diff --git a/.cirrus-DISABLED.yml b/.cirrus-DISABLED.yml deleted file mode 100644 index f20835cb6cac2a..00000000000000 --- a/.cirrus-DISABLED.yml +++ /dev/null @@ -1,29 +0,0 @@ -# gh-91960: Job disabled since Python is out of free credit (September 2023): -# https://discuss.python.org/t/freebsd-gets-a-new-cirrus-ci-github-action-job-and-a-new-buildbot/33122/26 - -freebsd_task: - freebsd_instance: - matrix: - - image: freebsd-13-2-release-amd64 - # Turn off TCP and UDP blackhole. It is not enabled by default in FreeBSD, - # but it is in the FreeBSD GCE images as used by Cirrus-CI. It causes even - # local local connections to fail with ETIMEDOUT instead of ECONNREFUSED. - # For more information see https://reviews.freebsd.org/D41751 and - # https://github.com/cirruslabs/cirrus-ci-docs/issues/483. - sysctl_script: - - sysctl net.inet.tcp.blackhole=0 - - sysctl net.inet.udp.blackhole=0 - configure_script: - - mkdir build - - cd build - - ../configure --with-pydebug - build_script: - - cd build - - make -j$(sysctl -n hw.ncpu) - pythoninfo_script: - - cd build - - make pythoninfo - test_script: - - cd build - # dtrace fails to build on FreeBSD - see gh-73263 - - make buildbottest TESTOPTS="-j0 -x test_dtrace --timeout=600" diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 590d7834b2b8be..98ab4008bed7cf 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,12 +1,12 @@ -FROM docker.io/library/fedora:37 +FROM docker.io/library/fedora:40 ENV CC=clang -ENV WASI_SDK_VERSION=20 +ENV WASI_SDK_VERSION=21 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=9.0.1 +ENV WASMTIME_VERSION=22.0.0 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.editorconfig b/.editorconfig index 0169eed951cd3f..a6187d64f3ce46 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,6 @@ root = true -[*.{py,c,cpp,h,rst,md,yml}] +[*.{py,c,cpp,h,js,rst,md,yml}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space @@ -11,5 +11,5 @@ indent_size = 4 [*.rst] indent_size = 3 -[*.yml] +[*.{js,yml}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 8c37dbbb631022..5b81d2cb3c90e9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -76,14 +76,19 @@ Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated +Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated +Misc/sbom.spdx.json generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated @@ -92,7 +97,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/abstract_interp_cases.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cd35cba5b5c56d..95e30ac3001c9c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,8 @@ # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland # asyncio **/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc @@ -21,6 +23,7 @@ configure* @erlend-aasland @corona10 **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 +**/*jit* @brandtbucher Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -28,30 +31,65 @@ Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval.c @markshannon +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel +Python/instruction_sequence.c @iritkatriel Python/ast_opt.c @isidentical +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon +Python/optimizer_analysis.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner +Python/symtable.c @JelleZijlstra @carljm +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher -Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv +Tools/c-analyzer/ @ericsnowcurrently + +# dbm +**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka + +# runtime state/lifecycle +**/*pylifecycle* @ericsnowcurrently +**/*pystate* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently +**/*bltinmodule* @ericsnowcurrently +**/*gil* @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_tstate.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently # Exceptions -Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel -Lib/test/test_traceback.py @iritkatriel Objects/exceptions.c @iritkatriel -Python/traceback.c @iritkatriel # Hashing -**/*hashlib* @tiran -**/*pyhash* @tiran -**/*sha* @tiran -**/*md5* @tiran -**/*blake* @tiran -/Modules/_blake2/** @tiran -/Modules/_sha3/** @tiran +**/*hashlib* @gpshead @tiran +**/*pyhash* @gpshead @tiran +**/sha* @gpshead @tiran +Modules/md5* @gpshead @tiran +**/*blake* @gpshead @tiran +Modules/_blake2/** @gpshead @tiran +Modules/_hacl/** @gpshead # logging **/*logging* @vsajip @@ -71,8 +109,21 @@ Python/traceback.c @iritkatriel # Import (including importlib). **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw /Python/import.c @kumaraditya303 -**/*importlib/resources/* @jaraco @warsaw @FFY00 -**/importlib/metadata/* @jaraco @warsaw +Python/dynload_*.c @ericsnowcurrently +**/*freeze* @ericsnowcurrently +**/*frozen* @ericsnowcurrently +**/*modsupport* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently +**/*moduleobject* @ericsnowcurrently +**/*multiphase* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently +**/*pythonrun* @ericsnowcurrently +**/*runpy* @ericsnowcurrently +**/*singlephase* @ericsnowcurrently +Lib/test/test_module/ @ericsnowcurrently +Doc/c-api/module.rst @ericsnowcurrently +**/*importlib/resources/* @jaraco @warsaw @FFY00 +**/*importlib/metadata/* @jaraco @warsaw # Dates and times **/*datetime* @pganssle @abalkin @@ -102,16 +153,22 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/tokenize.py @pablogsal @lysnikolaou /Lib/test/test_tokenize.py @pablogsal @lysnikolaou +# Code generator +/Tools/cases_generator/ @markshannon + # AST -Python/ast.c @isidentical -Parser/asdl.py @isidentical -Parser/asdl_c.py @isidentical -Lib/ast.py @isidentical +Python/ast.c @isidentical @JelleZijlstra +Parser/asdl.py @isidentical @JelleZijlstra +Parser/asdl_c.py @isidentical @JelleZijlstra +Lib/ast.py @isidentical @JelleZijlstra # Mock /Lib/unittest/mock.py @cjw296 /Lib/test/test_unittest/testmock/* @cjw296 +# multiprocessing +**/*multiprocessing* @gpshead + # SQLite 3 **/*sqlite* @berkerpeksag @erlend-aasland @@ -120,6 +177,10 @@ Lib/ast.py @isidentical /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# debugger +**/*pdb* @gaogaotiantian +**/*bdb* @gaogaotiantian + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -141,7 +202,6 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger @@ -149,9 +209,12 @@ Doc/c-api/stable.rst @encukou **/*dataclasses* @ericvsmith +**/*ensurepip* @pfmoore @pradyunsg + **/*idlelib* @terryjreedy +/Doc/library/idle.rst @terryjreedy -**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood +**/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo **/*shutil @giampaolo @@ -177,9 +240,37 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland @AlexWaygood -/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +/Tools/clinic/** @erlend-aasland +/Lib/test/test_clinic.py @erlend-aasland Doc/howto/clinic.rst @erlend-aasland +# Subinterpreters +**/*interpreteridobject.* @ericsnowcurrently +**/*crossinterp* @ericsnowcurrently +Lib/test/support/interpreters/ @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently + +# Android +**/*Android* @mhsmith +**/*android* @mhsmith + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + # WebAssembly /Tools/wasm/ @brettcannon + +# SBOM +/Misc/externals.spdx.json @sethmlarson +/Misc/sbom.spdx.json @sethmlarson +/Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 21a375361bf58e..395516f29b037c 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -39,6 +39,7 @@ body: - "3.10" - "3.11" - "3.12" + - "3.13" - "CPython main branch" validations: required: true diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 1ef9178b95e5f6..570b8779994a0f 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -19,7 +19,7 @@ jobs: permissions: issues: write steps: - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81e85e630bb001..e1d9e1632f136c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,21 +8,11 @@ on: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' + - '3.*' permissions: contents: read @@ -64,7 +54,7 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi # Check if we should run hypothesis tests @@ -99,7 +89,7 @@ jobs: - name: Get a list of the changed documentation-related files if: github.event_name == 'pull_request' id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 + uses: Ana06/get-changed-files@v2.3.0 with: filter: | Doc/** @@ -122,42 +112,50 @@ jobs: check_generated_files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - uses: actions/setup-python@v4 - with: - python-version: '3.x' + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Check Autoconf and aclocal versions run: | grep "Generated by GNU Autoconf 2.71" configure - grep "aclocal 1.16.4" aclocal.m4 + grep "aclocal 1.16.5" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - name: Configure CPython run: | # Build Python with the libpython dynamic library ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -180,170 +178,86 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals - build_win32: - name: 'Windows (x86)' - runs-on: windows-latest - timeout-minutes: 60 + build_windows: + name: >- + Windows + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci + if: fromJSON(needs.check_source.outputs.run_tests) + strategy: + matrix: + arch: + - Win32 + - x64 + - arm64 + free-threading: + - false + - true + uses: ./.github/workflows/reusable-windows.yml + with: + arch: ${{ matrix.arch }} + free-threading: ${{ matrix.free-threading }} - build_win_amd64: - name: 'Windows (x64)' - runs-on: windows-latest - timeout-minutes: 60 + build_macos: + name: 'macOS' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci + uses: ./.github/workflows/reusable-macos.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # Cirrus used for upstream, macos-14 for forks. + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14", "macos-13"]' - build_win_arm64: - name: 'Windows (arm64)' - runs-on: windows-latest - timeout-minutes: 60 + build_macos_free_threading: + name: 'macOS (free-threading)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 + uses: ./.github/workflows/reusable-macos.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + free-threading: true + # Cirrus and macos-14 are M1. + # Cirrus used for upstream, macos-14 for forks. + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]' - build_macos: - name: 'macOS' - runs-on: macos-latest - timeout-minutes: 60 + build_ubuntu: + name: 'Ubuntu' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - HOMEBREW_NO_ANALYTICS: 1 - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - - name: Configure CPython - run: | - GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ - GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure \ + uses: ./.github/workflows/reusable-ubuntu.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | + ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: make test + --with-openssl=$OPENSSL_DIR - build_ubuntu: - name: 'Ubuntu' - runs-on: ubuntu-20.04 - timeout-minutes: 60 + build_ubuntu_free_threading: + name: 'Ubuntu (free-threading)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 3.0.11 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | + uses: ./.github/workflows/reusable-ubuntu.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + --with-openssl=$OPENSSL_DIR \ + --disable-gil build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.11, 3.1.3] + openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -351,11 +265,13 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -367,7 +283,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -379,6 +295,8 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Configure CPython run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython @@ -388,14 +306,22 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -410,7 +336,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -422,6 +348,8 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV @@ -430,11 +358,13 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -464,12 +394,12 @@ jobs: ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v3 + uses: actions/cache@v4 with: - path: ./hypothesis + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - - hypothesis-database- + hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -490,30 +420,32 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db - path: .hypothesis/examples/ + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ build_asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 3.0.11 + OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -529,7 +461,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -541,6 +473,9 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -550,6 +485,28 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan: + name: 'Thread sanitizer' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --with-thread-sanitizer --with-pydebug + suppressions_path: Tools/tsan/supressions.txt + tsan_logs_artifact_name: tsan-logs-default + + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug + suppressions_path: Tools/tsan/suppressions_free_threading.txt + tsan_logs_artifact_name: tsan-logs-free-threading + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz @@ -578,14 +535,14 @@ jobs: output-sarif: true sanitizer: ${{ matrix.sanitizer }} - name: Upload crash - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() && steps.build.outcome == 'success' with: name: ${{ matrix.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif @@ -598,14 +555,17 @@ jobs: - check_source # Transitive dependency, needed to access `run_tests` value - check-docs - check_generated_files - - build_win32 - - build_win_amd64 - - build_win_arm64 - build_macos + - build_macos_free_threading - build_ubuntu + - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi + - build_windows - test_hypothesis - build_asan + - build_tsan + - build_tsan_free_threading - cifuzz runs-on: ubuntu-latest @@ -615,10 +575,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_macos, build_ubuntu_ssltests, - build_win32, - build_win_arm64, cifuzz, test_hypothesis, allowed-skips: >- @@ -633,13 +590,16 @@ jobs: needs.check_source.outputs.run_tests != 'true' && ' check_generated_files, - build_win32, - build_win_amd64, - build_win_arm64, build_macos, + build_macos_free_threading, build_ubuntu, + build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, + build_windows, build_asan, + build_tsan, + build_tsan_free_threading, ' || '' }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 29282dffa37ec0..65d32c734e7745 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -32,6 +32,8 @@ jobs: strategy: matrix: type: [x86, x64, arm64] + env: + IncludeFreethreaded: true steps: - uses: actions/checkout@v4 - name: Build CPython installer diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml new file mode 100644 index 00000000000000..5e3ac9e9e0fada --- /dev/null +++ b/.github/workflows/jit.yml @@ -0,0 +1,183 @@ +name: JIT +on: + pull_request: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' + push: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - x86_64-unknown-linux-gnu/clang + - aarch64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/clang + debug: + - true + - false + llvm: + - 18 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + compiler: msvc + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + compiler: msvc + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-latest + compiler: msvc + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + compiler: clang + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + compiler: clang + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-latest + compiler: gcc + - target: x86_64-unknown-linux-gnu/clang + architecture: x86_64 + runner: ubuntu-latest + compiler: clang + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-latest + compiler: gcc + - target: aarch64-unknown-linux-gnu/clang + architecture: aarch64 + runner: ubuntu-latest + compiler: clang + env: + CC: ${{ matrix.compiler }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Native Windows + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' + run: | + choco upgrade llvm -y + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No PGO or tests (yet): + - name: Emulated Windows + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + run: | + choco upgrade llvm -y + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + + - name: Native macOS + if: runner.os == 'macOS' + run: | + brew update + brew install llvm@${{ matrix.llvm }} + SDKROOT="$(xcrun --show-sdk-path)" \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux + if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Emulated Linux + if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --prefix="$(pwd)/../build" + make install --jobs 4 + make clean --jobs 4 + export HOST=${{ matrix.architecture }}-linux-gnu + sudo apt install --yes "gcc-$HOST" qemu-user + ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} + ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} + export QEMU_LD_PREFIX="/usr/$HOST" + CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ + CPP="$CC --preprocess" \ + HOSTRUNNER=qemu-${{ matrix.architecture }} \ + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + make all --jobs 4 + ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + jit-with-disabled-gil: + name: Free-Threaded (Debug) + needs: interpreter + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18 + export PATH="$(llvm-config-18 --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug --disable-gil + make all --jobs 4 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6c1c29a58cf4fc..ccde03f91983df 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 405511ca6820b3..1b2d998182e0f7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,8 +8,12 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" + - "Lib/test/libregrtest/**" + - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" - "Tools/clinic/**" + - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - "Tools/wasm/**" @@ -30,10 +34,15 @@ concurrency: jobs: mypy: strategy: + fail-fast: false matrix: target: [ + "Lib/_pyrepl", + "Lib/test/libregrtest", + "Tools/build", "Tools/cases_generator", "Tools/clinic", + "Tools/jit", "Tools/peg_generator", "Tools/wasm", ] @@ -42,7 +51,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 80514b4d2ca572..9f1a8a824e5f19 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index a220896f2cd7be..fb485bd4f82bd2 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,9 +1,11 @@ #!/bin/sh apt-get update +# autoconf-archive is needed by autoreconf (check_generated_files job) apt-get -yq install \ build-essential \ pkg-config \ + autoconf-archive \ ccache \ gdb \ lcov \ @@ -13,12 +15,14 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ lzma \ lzma-dev \ + strace \ tk-dev \ uuid-dev \ xvfb \ diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 7574bfc208ff76..066d8593a70cf6 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -23,7 +23,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.1.0 + - uses: actions/add-to-project@v1.0.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 080204bcfd3b94..ff5cbdf3eda749 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -11,6 +11,7 @@ permissions: jobs: label: name: DO-NOT-MERGE / unresolved review + if: github.repository_owner == 'python' runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 1c4fa4239c1e34..859f78d043ba92 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -41,7 +41,7 @@ jobs: git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3' cache: 'pip' @@ -62,7 +62,8 @@ jobs: python Doc/tools/check-warnings.py \ --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ - --fail-if-improved + --fail-if-improved \ + --fail-if-new-news-nit # This build doesn't use problem matchers or check annotations build_doc_oldest_supported_sphinx: @@ -72,9 +73,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' # known to work with Sphinx 4.2 + python-version: '3.12' # known to work with Sphinx 6.2.1 cache: 'pip' cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - name: 'Install build dependencies' @@ -89,7 +90,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml new file mode 100644 index 00000000000000..f825d1a7b3f69a --- /dev/null +++ b/.github/workflows/reusable-macos.yml @@ -0,0 +1,64 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + free-threading: + required: false + type: boolean + default: false + os-matrix: + required: false + type: string + +jobs: + build_macos: + name: build and test (${{ matrix.os }}) + timeout-minutes: 60 + env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux + strategy: + fail-fast: false + matrix: + os: ${{fromJson(inputs.os-matrix)}} + is-fork: + - ${{ github.repository_owner != 'python' }} + exclude: + - os: "ghcr.io/cirruslabs/macos-runner:sonoma" + is-fork: true + - os: "macos-14" + is-fork: false + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Homebrew dependencies + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + - name: Configure CPython + run: | + GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ + GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ + ./configure \ + --config-cache \ + --with-pydebug \ + ${{ inputs.free-threading && '--disable-gil' || '' }} \ + --prefix=/opt/python-dev \ + --with-openssl="$(brew --prefix openssl@3.0)" + - name: Build CPython + run: make -j8 + - name: Display build info + run: make pythoninfo + - name: Tests + run: make test diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..b6d5d8fa1c7157 --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,76 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + suppressions_path: + description: 'A repo relative path to the suppressions file' + required: true + type: string + tsan_logs_artifact_name: + description: 'Name of the TSAN logs artifact. Must be unique for each job.' + required: true + type: string + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + sudo update-alternatives --set clang /usr/bin/clang-18 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-18 + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ${{ inputs.options }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + - name: Display TSAN logs + if: always() + run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.tsan_logs_artifact_name }} + path: tsan_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml new file mode 100644 index 00000000000000..fa450ed3376321 --- /dev/null +++ b/.github/workflows/reusable-ubuntu.yml @@ -0,0 +1,78 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + +jobs: + build_ubuntu_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-22.04 + env: + FORCE_COLOR: 1 + OPENSSL_VER: 3.0.13 + PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux + steps: + - uses: actions/checkout@v4 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ${{ inputs.options }} + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make test diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..db6c04ec2ac1c5 --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,75 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-22.04 + env: + WASMTIME_VERSION: 22.0.0 + WASI_SDK_VERSION: 21 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasi + steps: + - uses: actions/checkout@v4 + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: bytecodealliance/actions/wasmtime/setup@v1 + with: + version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir ${{ env.WASI_SDK_PATH }} && \ + curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ + tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. + # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # (Make sure to keep the key in sync with the other config.cache step below.) + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + # Should be kept in sync with the other config.cache step above. + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + - name: "Test" + run: make --directory ${{ env.CROSS_BUILD_WASI }} test diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml new file mode 100644 index 00000000000000..94a3abe530cb9b --- /dev/null +++ b/.github/workflows/reusable-windows.yml @@ -0,0 +1,45 @@ +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + free-threading: + description: Whether to use no-GIL mode + required: false + type: boolean + default: false + +env: + IncludeUwp: >- + true + +jobs: + build: + name: >- + build${{ inputs.arch != 'arm64' && ' and test' || '' }} + (${{ inputs.arch }}) + runs-on: windows-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Register MSVC problem matcher + if: inputs.arch != 'Win32' + run: echo "::add-matcher::.github/problem-matchers/msvc.json" + - name: Build CPython + run: >- + .\PCbuild\build.bat + -e -d -v + -p ${{ inputs.arch }} + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Display build info + if: inputs.arch != 'arm64' + run: .\python.bat -m test.pythoninfo + - name: Tests + if: inputs.arch != 'arm64' + run: >- + .\PCbuild\rt.bat + -p ${{ inputs.arch }} + -d -q --fast-ci + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 94676f5ee5fffc..f97587e68cbbe4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Mark stale pull requests on: schedule: - - cron: "0 0 * * *" + - cron: "0 */6 * * *" permissions: pull-requests: write @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 4a545037bf6e2b..83b007f1c9c2ef 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.gitignore b/.gitignore index dddf28da016192..8872e9d5508ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ gmon.out .coverage .mypy_cache/ .pytest_cache/ +.ruff_cache/ .DS_Store *.exe @@ -68,6 +69,17 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +/iOSTestbed.* +iOS/Frameworks/ +iOS/Resources/Info.plist +iOS/testbed/build +iOS/testbed/Python.xcframework/ios-*/bin +iOS/testbed/Python.xcframework/ios-*/include +iOS/testbed/Python.xcframework/ios-*/lib +iOS/testbed/Python.xcframework/ios-*/Python.framework +iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace +iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +iOS/testbed/iOSTestbed.xcodeproj/xcshareddata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -124,11 +136,12 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build/ +/jit_stencils.h /platform /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config @@ -156,5 +169,5 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained +# main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4835b641b3106a..fde9d9149bf62b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.3.4 hooks: - id: ruff name: Run Ruff on Lib/test/ @@ -11,9 +11,19 @@ repos: args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.4.2 + hooks: + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ + language_version: python3.12 + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: + - id: check-case-conflict + - id: check-merge-conflict - id: check-toml exclude: ^Lib/test/test_tomllib/ - id: check-yaml @@ -24,12 +34,11 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.7.0 + rev: v0.9.1 hooks: - id: sphinx-lint - args: [--enable=default-role, -j1] - files: ^Doc/|^Misc/NEWS.d/next/ - types: [rst] + args: [--enable=default-role] + files: ^Doc/|^Misc/NEWS.d/ - repo: meta hooks: diff --git a/.readthedocs.yml b/.readthedocs.yml index 59830c79a404e0..d0d0c3b93ed207 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,6 +26,9 @@ build: exit 183; fi + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html diff --git a/Android/README.md b/Android/README.md new file mode 100644 index 00000000000000..f5f463ca116589 --- /dev/null +++ b/Android/README.md @@ -0,0 +1,97 @@ +# Python for Android + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. If you're looking to +use Python on Android, one of the following tools will provide a much more +approachable user experience: + +* [Briefcase](https://briefcase.readthedocs.io), from the BeeWare project +* [Buildozer](https://buildozer.readthedocs.io), from the Kivy project +* [Chaquopy](https://chaquo.com/chaquopy/) + + +## Prerequisites + +Export the `ANDROID_HOME` environment variable to point at your Android SDK. If +you don't already have the SDK, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +The `android.py` script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` +* `tar` +* `unzip` + + +## Building + +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. This involves doing a +cross-build where you use a "build" Python (for your development machine) to +help produce a "host" Python for Android. + +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. The only Android tool you need to install +is the command line tools package above: the build script will download the +rest. + +The easiest way to do a build is to use the `android.py` script. You can either +have it perform the entire build process from start to finish in one step, or +you can do it in discrete steps that mirror running `configure` and `make` for +each of the two builds of Python you end up producing. + +The discrete steps for building via `android.py` are: + +```sh +./android.py configure-build +./android.py make-build +./android.py configure-host HOST +./android.py make-host HOST +``` + +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. + +To do all steps in a single command, run: + +```sh +./android.py build HOST +``` + +In the end you should have a build Python in `cross-build/build`, and an Android +build in `cross-build/HOST`. + +You can use `--` as a separator for any of the `configure`-related commands – +including `build` itself – to pass arguments to the underlying `configure` +call. For example, if you want a pydebug build that also caches the results from +`configure`, you can do: + +```sh +./android.py build HOST -- -C --with-pydebug +``` + + +## Testing + +To run the Python test suite on Android: + +* Install Android Studio, if you don't already have it. +* Follow the instructions in the previous section to build all supported + architectures. +* Run `./android.py setup-testbed` to download the Gradle wrapper. +* Open the `testbed` directory in Android Studio. +* In the *Device Manager* dock, connect a device or start an emulator. + Then select it from the drop-down list in the toolbar. +* Click the "Run" button in the toolbar. +* The testbed app displays nothing on screen while running. To see its output, + open the [Logcat window](https://developer.android.com/studio/debug/logcat). + +To run specific tests, or pass any other arguments to the test suite, edit the +command line in testbed/app/src/main/python/main.py. diff --git a/Android/android-env.sh b/Android/android-env.sh new file mode 100644 index 00000000000000..545d559d93ab36 --- /dev/null +++ b/Android/android-env.sh @@ -0,0 +1,93 @@ +# This script must be sourced with the following variables already set: +: ${ANDROID_HOME:?} # Path to Android SDK +: ${HOST:?} # GNU target triplet + +# You may also override the following: +: ${api_level:=21} # Minimum Android API level the build will run on +: ${PREFIX:-} # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=26.2.11394342 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e $ndk ]; then + log "Installing NDK: this may take several minutes" + yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" +fi + +if [ $HOST = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet=$HOST +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ $HOST = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix=$(realpath $PREFIX) + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# Use the same variable name as conda-build +if [ $(uname) = "Darwin" ]; then + export CPU_COUNT=$(sysctl -n hw.ncpu) +else + export CPU_COUNT=$(nproc) +fi diff --git a/Android/android.py b/Android/android.py new file mode 100755 index 00000000000000..0a1393e61ddb0e --- /dev/null +++ b/Android/android.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import shutil +import subprocess +import sys +import sysconfig +from os.path import basename, relpath +from pathlib import Path +from tempfile import TemporaryDirectory + +SCRIPT_NAME = Path(__file__).name +CHECKOUT = Path(__file__).resolve().parent.parent +CROSS_BUILD_DIR = CHECKOUT / "cross-build" + + +def delete_if_exists(path): + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def subdir(name, *, clean=None): + path = CROSS_BUILD_DIR / name + if clean: + delete_if_exists(path) + if not path.exists(): + if clean is None: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}.") + else: + path.mkdir(parents=True) + return path + + +def run(command, *, host=None, **kwargs): + env = os.environ.copy() + if host: + env_script = CHECKOUT / "Android/android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={subdir(host)}/prefix; " + f". {env_script}; " + f"export", + check=True, shell=True, text=True, stdout=subprocess.PIPE + ).stdout + + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if env.get(key) != value: + print(line) + env[key] = value + + if env == os.environ: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + print(">", " ".join(map(str, command))) + try: + subprocess.run(command, check=True, env=env, **kwargs) + except subprocess.CalledProcessError as e: + sys.exit(e) + + +def build_python_path(): + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{build_dir}") + + return binary + + +def configure_build_python(context): + os.chdir(subdir("build", clean=context.clean)) + + command = [relpath(CHECKOUT / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context): + os.chdir(subdir("build")) + run(["make", "-j", str(os.cpu_count())]) + + +def unpack_deps(host): + deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1", + "sqlite-3.45.1-0", "xz-5.4.6-0"]: + filename = f"{name_ver}-{host}.tar.gz" + download(f"{deps_url}/{name_ver}/{filename}") + run(["tar", "-xf", filename]) + os.remove(filename) + + +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "-o", out_path, url]) + return out_path + + +def configure_host_python(context): + host_dir = subdir(context.host, clean=context.clean) + + prefix_dir = host_dir / "prefix" + if not prefix_dir.exists(): + prefix_dir.mkdir() + os.chdir(prefix_dir) + unpack_deps(context.host) + + build_dir = host_dir / "build" + build_dir.mkdir(exist_ok=True) + os.chdir(build_dir) + + command = [ + # Basic cross-compiling configuration + relpath(CHECKOUT / "configure"), + f"--host={context.host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--without-ensurepip", + + # Android always uses a shared libpython. + "--enable-shared", + "--without-static-libpython", + + # Dependent libraries. The others are found using pkg-config: see + # android-env.sh. + f"--with-openssl={prefix_dir}", + ] + + if context.args: + command.extend(context.args) + run(command, host=context.host) + + +def make_host_python(context): + host_dir = subdir(context.host) + os.chdir(host_dir / "build") + run(["make", "-j", str(os.cpu_count())], host=context.host) + run(["make", "install", f"prefix={host_dir}/prefix"], host=context.host) + + +def build_all(context): + steps = [configure_build_python, make_build_python, configure_host_python, + make_host_python] + for step in steps: + step(context) + + +def clean_all(context): + delete_if_exists(CROSS_BUILD_DIR) + + +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle release. +def setup_testbed(context): + ver_long = "8.7.0" + ver_short = ver_long.removesuffix(".0") + testbed_dir = CHECKOUT / "Android/testbed" + + for filename in ["gradlew", "gradlew.bat"]: + out_path = download( + f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", + testbed_dir) + os.chmod(out_path, 0o755) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + os.chdir(temp_dir) + bin_zip = download( + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip") + outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" + run(["unzip", bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar, + "gradle-wrapper.jar"]) + + +def main(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for Android") + make_host = subcommands.add_parser("make-host", + help="Run `make` for Android") + subcommands.add_parser( + "clean", help="Delete the cross-build directory") + subcommands.add_parser( + "setup-testbed", help="Download the testbed Gradle wrapper") + + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--clean", action="store_true", default=False, dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_host, make_host: + subcommand.add_argument( + "host", metavar="HOST", + choices=["aarch64-linux-android", "x86_64-linux-android"], + help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + + context = parser.parse_args() + dispatch = {"configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all, + "setup-testbed": setup_testbed} + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 00000000000000..b9a7d611c943cf --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,21 @@ +# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/deploymentTargetDropdown.xml +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 00000000000000..42afabfd2abebf --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 00000000000000..7690d3fd86b2fd --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,129 @@ +import com.android.build.api.variant.* + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val PYTHON_DIR = File(projectDir, "../../..").canonicalPath +val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" +val ABIS = mapOf( + "arm64-v8a" to "aarch64-linux-android", + "x86_64" to "x86_64-linux-android", +) + +val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines { + for (line in it) { + val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find Python version") +} + + +android { + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(ABIS.keys) + externalNativeBuild.cmake.arguments( + "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", + "-DPYTHON_VERSION=$PYTHON_VERSION") + } + + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + generateTask(variant, variant.sources.assets!!) { + into("python") { + for (triplet in ABIS.values) { + for (subDir in listOf("include", "lib")) { + into(subDir) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir") + include("python$PYTHON_VERSION/**") + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + } + } + into("lib/python$PYTHON_VERSION") { + // Uncomment this to pick up edits from the source directory + // without having to rerun `make install`. + // from("$PYTHON_DIR/Lib") + // duplicatesStrategy = DuplicatesStrategy.INCLUDE + + into("site-packages") { + from("$projectDir/src/main/python") + } + } + } + exclude("**/__pycache__") + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((abi, triplet) in ABIS.entries) { + into(abi) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000000..2be8a82d426099 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 00000000000000..1d5df9a73465b6 --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) +include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 00000000000000..73aba4164d000f --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,147 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +static StreamInfo STREAMS[] = { + {stdout, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {stderr, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python intialization ---------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 00000000000000..5a590d5d04e954 --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,61 @@ +package org.python.testbed + +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + runPython(pythonHome.toString(), "main") + findViewById(R.id.tvHello).text = "Python complete" + } + + private fun extractAssets() : File { + val pythonHome = File(filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) +} \ No newline at end of file diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 00000000000000..a1b6def34ede81 --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,17 @@ +import runpy +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the "Signal Catcher" thread. That +# thread's functionality is only relevant to the JVM ("forcing GC (no HPROF) and +# profile save"), so disabling it should not weaken the tests. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +# To run specific tests, or pass any other arguments to the test suite, edit +# this command line. +sys.argv[1:] = [ + "--use", "all,-cpu", + "--verbose3", +] +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000000..741d6580d60e05 Binary files /dev/null and b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000000000..21398609ec9c78 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000000..352d2f9e885a2a --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Python testbed + \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 00000000000000..53f4a67287fcc5 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.2.2" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} \ No newline at end of file diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 00000000000000..3c5031eb7d63f7 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000000..2dc3339a3ef213 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 00000000000000..5e08773e02450f --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Doc/Makefile b/Doc/Makefile index 78ee4271e25f79..c70768754834dd 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -28,7 +32,9 @@ help: @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" + @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" @@ -135,10 +141,24 @@ pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: gettext +gettext: BUILDER = gettext +gettext: SPHINXOPTS += '-d build/doctrees-gettext' +gettext: build + .PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: ensure-sphinx-autobuild +ensure-sphinx-autobuild: venv + $(call ensure_package,sphinx-autobuild) + +.PHONY: htmllive +htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 +htmllive: ensure-sphinx-autobuild html + .PHONY: clean clean: clean-venv -rm -rf build/* @@ -153,10 +173,16 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ - $(PYTHON) -m venv $(VENVDIR); \ - $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ - $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ - echo "The venv has been created in the $(VENVDIR) directory"; \ + echo "Creating venv in $(VENVDIR)"; \ + if uv --version > /dev/null; then \ + uv venv $(VENVDIR); \ + VIRTUAL_ENV=$(VENVDIR) uv pip install -r $(REQUIREMENTS); \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ + $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + echo "The venv has been created in the $(VENVDIR) directory"; \ + fi; \ fi .PHONY: dist @@ -214,9 +240,17 @@ dist: rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar +define ensure_package + if uv --version > /dev/null; then \ + $(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || VIRTUAL_ENV=$(VENVDIR) uv pip install $(1); \ + else \ + $(VENVDIR)/bin/python3 -m $(1) --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install $(1); \ + fi +endef + .PHONY: check check: venv - $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(call ensure_package,pre_commit) $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve diff --git a/Doc/README.rst b/Doc/README.rst index a3bb5fa5445c23..efcee0db428908 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -28,7 +28,7 @@ install the tools into there. Using make ---------- -To get started on UNIX, you can create a virtual environment and build +To get started on Unix, you can create a virtual environment and build documentation with the commands:: make venv @@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location, you can specify it using the ``VENVDIR`` variable. You can also skip creating the virtual environment altogether, in which case -the Makefile will look for instances of ``sphinx-build`` and ``blurb`` +the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb`` installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and ``BLURB`` variables). -On Windows, we try to emulate the Makefile as closely as possible with a +On Windows, we try to emulate the ``Makefile`` as closely as possible with a ``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable. +set the ``PYTHON`` environment variable. Available make targets are: @@ -62,15 +62,19 @@ Available make targets are: * "htmlview", which re-uses the "html" builder, but then opens the main page in your default web browser. +* "htmllive", which re-uses the "html" builder, rebuilds the docs, + starts a local server, and automatically reloads the page in your browser + when you make changes to reST files (Unix only). + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for + over the generated project (.hhp) file. The ``make.bat`` script does this for you on Windows. -* "latex", which builds LaTeX source files as input to "pdflatex" to produce +* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce PDF documents. * "text", which builds a plain text file for each source file. @@ -95,8 +99,6 @@ Available make targets are: * "check", which checks for frequent markup errors. -* "serve", which serves the build/html directory on port 8000. - * "dist", (Unix only) which creates distributable archives of HTML, text, PDF, and EPUB builds. diff --git a/Doc/bugs.rst b/Doc/bugs.rst index d98192b369603e..9aff2f0ff5187d 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -22,6 +22,10 @@ have a suggestion on how to fix it, include that as well. You can also open a discussion item on our `Documentation Discourse forum `_. +If you find a bug in the theme (HTML / CSS / JavaScript) of the +documentation, please submit a bug report on the `python-doc-theme bug +tracker `_. + If you're short on time, you can also email documentation bug reports to docs@python.org (behavioral bugs can be sent to python-list@python.org). 'docs@' is a mailing list run by volunteers; your request will be noticed, @@ -38,7 +42,7 @@ though it may take a while to be processed. `Helping with Documentation `_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations `_ + `Documentation Translations `_ A list of GitHub pages for documentation translation and their primary contacts. diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 62d87d898e682c..834aae9372fe3b 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -413,7 +413,7 @@ API Functions than a variable number of arguments. -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) +.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, ...) Parse the parameters of a function that takes both positional and keyword parameters into local variables. @@ -424,15 +424,24 @@ API Functions Returns true on success; on failure, it returns false and raises the appropriate exception. + .. note:: + + The *keywords* parameter declaration is :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + .. versionchanged:: 3.6 Added support for :ref:`positional-only parameters `. .. versionchanged:: 3.13 + The *keywords* parameter has now type :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. Added support for non-ASCII keyword parameter names. -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) + +.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, va_list vargs) Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a va_list rather than a variable number of arguments. @@ -505,6 +514,19 @@ API Functions PyArg_ParseTuple(args, "O|O:ref", &object, &callback) +.. c:macro:: PY_CXX_CONST + + The value to be inserted, if any, before :c:expr:`char * const *` + in the *keywords* parameter declaration of + :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords`. + Default empty for C and ``const`` for C++ + (:c:expr:`const char * const *`). + To override, define it to the desired value before including + :file:`Python.h`. + + .. versionadded:: 3.13 + --------------- Building values diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index b14fa6a0a982e2..b4dc4849d044e1 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -26,19 +26,19 @@ are available, however. .. c:var:: PyObject* Py_False The Python ``False`` object. This object has no methods and is - `immortal `_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_False` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_False` is :term:`immortal`. .. c:var:: PyObject* Py_True The Python ``True`` object. This object has no methods and is - `immortal `_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_True` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_True` is :term:`immortal`. .. c:macro:: Py_RETURN_FALSE diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index e572815ffd6259..9500fe465c7d94 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -29,7 +29,7 @@ without intermediate copying. Python provides such a facility at the C level in the form of the :ref:`buffer protocol `. This protocol has two sides: -.. index:: single: PyBufferProcs +.. index:: single: PyBufferProcs (C type) - on the producer side, a type can export a "buffer interface" which allows objects of that type to expose information about their underlying buffer. @@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 61a68f52773882..bca78a9c369385 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -155,6 +155,7 @@ called with a non-bytes parameter. Return the null-terminated contents of the object *obj* through the output variables *buffer* and *length*. + Returns ``0`` on success. If *length* is ``NULL``, the bytes object may not contain embedded null bytes; @@ -190,10 +191,10 @@ called with a non-bytes parameter. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an + Resize a bytes object. *newsize* will be the new length of the bytes object. + You can think of it as creating a new bytes object and destroying the old + one, only more efficiently. + Pass the address of an existing bytes object as an lvalue (it may be written into), and the new size desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index aed4ae44c76eea..7198d6bc056eb4 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -108,6 +108,8 @@ This is a pointer to a function with the following signature: Doing so will allow callables such as bound methods to make their onward calls (which include a prepended *self* argument) very efficiently. + .. versionadded:: 3.8 + To call an object that implements vectorcall, use a :ref:`call API ` function as with any other callable. :c:func:`PyObject_Vectorcall` will usually be most efficient. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index f8cd0344fdd1c0..61eb994c370946 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return the contents of the cell *cell*, which can be ``NULL``. + If *cell* is not a cell object, returns ``NULL`` with an exception set. .. c:function:: PyObject* PyCell_GET(PyObject *cell) @@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere. Set the contents of the cell object *cell* to *value*. This releases the reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. + must be non-``NULL``. + + On success, return ``0``. + If *cell* is not a cell object, set an exception and return ``-1``. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 5082b0cb6ad3f3..968c472219c643 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -22,16 +22,27 @@ bound into a function. .. c:var:: PyTypeObject PyCode_Type This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. + :ref:`code object `. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. This function always succeeds. + Return true if *co* is a :ref:`code object `. + This function always succeeds. -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in *co*. + Return the number of free variables in a code object. + +.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) + + Return the position of the first free variable in a code object. + + .. versionchanged:: 3.13 + + Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) @@ -48,7 +59,7 @@ bound into a function. .. versionchanged:: 3.11 Added ``qualname`` and ``exceptiontable`` parameters. - .. index:: single: PyCode_New + .. index:: single: PyCode_New (C function) .. versionchanged:: 3.12 @@ -61,7 +72,7 @@ bound into a function. Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs + .. index:: single: PyCode_NewWithPosOnlyArgs (C function) .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` @@ -220,7 +231,7 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex + .. index:: single: _PyEval_RequestCodeExtraIndex (C function) .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` @@ -238,7 +249,7 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra + .. index:: single: _PyCode_GetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_GetExtra`` @@ -253,7 +264,7 @@ may change without deprecation warnings. Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra + .. index:: single: _PyCode_SetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_SetExtra`` diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index e3fd001c599c80..5a0474869071d9 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -117,11 +117,29 @@ Complex Numbers as Python Objects Return the real part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns its result. Upon failure, this method returns ``-1.0``, so one + should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: double PyComplex_ImagAsDouble(PyObject *op) Return the imaginary part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns ``0.0`` on success. Upon failure, this method returns ``-1.0``, so + one should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index d970f5443b1df5..fe7b8f93f2c6cf 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -6,6 +6,8 @@ Context Variables Objects ------------------------- .. _contextvarsobjects_pointertype_change: +.. versionadded:: 3.7 + .. versionchanged:: 3.7.1 .. note:: @@ -24,8 +26,6 @@ Context Variables Objects See :issue:`34762` for more details. -.. versionadded:: 3.7 - This section details the public C API for the :mod:`contextvars` module. .. c:type:: PyContext diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c5350123dfdfdc..4aaf3905e81c8a 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows: The following functions provide locale-independent string to number conversions. +.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`unsigned + long` value according to the given ``base``, which must be between ``2`` and + ``36`` inclusive, or be the special value ``0``. + + Leading white space and case of characters are ignored. If ``base`` is zero + it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base. If + these are absent it defaults to ``10``. Base must be 0 or between 2 and 36 + (inclusive). If ``ptr`` is non-``NULL`` it will contain a pointer to the + end of the scan. + + If the converted value falls out of range of corresponding return type, + range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and + :c:macro:`!ULONG_MAX` is returned. If no conversion can be performed, ``0`` + is returned. + + See also the Unix man page :manpage:`strtoul(3)`. + + .. versionadded:: 3.2 + + +.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`long` value + according to the given ``base``, which must be between ``2`` and ``36`` + inclusive, or be the special value ``0``. + + Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead + and :c:macro:`LONG_MAX` on overflows. + + See also the Unix man page :manpage:`strtol(3)`. + + .. versionadded:: 3.2 + + .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) Convert a string ``s`` to a :c:expr:`double`, raising a Python diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8ee351918006e4..ce73fa0cc60ebb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -156,7 +156,7 @@ Dictionary Objects .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) - Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a + Similar to :c:func:`PyDict_GetItemRef`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. @@ -173,6 +173,54 @@ Dictionary Objects .. versionadded:: 3.4 + +.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) + + Inserts *default_value* into the dictionary *p* with a key of *key* if the + key is not already present in the dictionary. If *result* is not ``NULL``, + then *\*result* is set to a :term:`strong reference` to either + *default_value*, if the key was not present, or the existing value, if *key* + was already present in the dictionary. + Returns ``1`` if the key was present and *default_value* was not inserted, + or ``0`` if the key was not present and *default_value* was inserted. + On failure, returns ``-1``, sets an exception, and sets ``*result`` + to ``NULL``. + + For clarity: if you have a strong reference to *default_value* before + calling this function, then after it returns, you hold a strong reference + to both *default_value* and *\*result* (if it's not ``NULL``). + These may refer to the same object: in that case you hold two separate + references to it. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) + + Remove *key* from dictionary *p* and optionally return the removed value. + Do not raise :exc:`KeyError` if the key missing. + + - If the key is present, set *\*result* to a new reference to the removed + value if *result* is not ``NULL``, and return ``1``. + - If the key is missing, set *\*result* to ``NULL`` if *result* is not + ``NULL``, and return ``0``. + - On error, raise an exception and return ``-1``. + + Similar to :meth:`dict.pop`, but without the default value and + not raising :exc:`KeyError` if the key missing. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result) + + Similar to :c:func:`PyDict_Pop`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. @@ -242,6 +290,17 @@ Dictionary Objects Py_DECREF(o); } + The function is not thread-safe in the :term:`free-threaded ` + build without external synchronization. You can use + :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating + over it:: + + Py_BEGIN_CRITICAL_SECTION(self->dict); + while (PyDict_Next(self->dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 2139da051e0193..499bfb47cc4be5 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -88,9 +88,29 @@ Printing and clearing The function is called with a single argument *obj* that identifies the context in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + If *obj* is ``NULL``, only the traceback is printed. An exception must be set when calling this function. + .. versionchanged:: 3.4 + Print a traceback. Print only traceback if *obj* is ``NULL``. + + .. versionchanged:: 3.8 + Use :func:`sys.unraisablehook`. + + +.. c:function:: void PyErr_FormatUnraisable(const char *format, ...) + + Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent + parameters help format the warning message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. + ``PyErr_WriteUnraisable(obj)`` is roughly equivalent to + ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. + If *format* is ``NULL``, only the traceback is printed. + + .. versionadded:: 3.13 + + .. c:function:: void PyErr_DisplayException(PyObject *exc) Print the standard traceback display of ``exc`` to ``sys.stderr``, including @@ -98,6 +118,7 @@ Printing and clearing .. versionadded:: 3.12 + Raising exceptions ================== @@ -159,7 +180,7 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - .. index:: single: strerror() + .. index:: single: strerror (C function) This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a @@ -200,13 +221,14 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - This is a convenience function to raise :exc:`WindowsError`. If called with + This is a convenience function to raise :exc:`OSError`. If called with *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + then it constructs a :exc:`OSError` object with the :attr:`~OSError.winerror` + attribute set to the error code, the :attr:`~OSError.strerror` attribute + set to the corresponding error message (gotten from + :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_OSError, object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -375,7 +397,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :class:`!warnings.WarningMessage`. .. versionadded:: 3.6 @@ -419,7 +441,7 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) Return the exception currently being raised, clearing the error indicator at - the same time. + the same time. Return ``NULL`` if the error indicator is not set. This function is used by code that needs to catch exceptions, or code that needs to save and restore the error indicator temporarily. @@ -520,7 +542,8 @@ Querying the error indicator .. note:: - This function *does not* implicitly set the ``__traceback__`` + This function *does not* implicitly set the + :attr:`~BaseException.__traceback__` attribute on the exception value. If setting the traceback appropriately is desired, the following additional snippet is needed:: @@ -613,7 +636,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) This function interacts with Python's signal handling. @@ -644,7 +667,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) Simulate the effect of a :c:macro:`!SIGINT` signal arriving. @@ -710,7 +733,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`__module__` attribute of the new class is set to the first part (up + The :attr:`!__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* @@ -732,7 +755,8 @@ Exception Objects .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no + accessible from Python through the :attr:`~BaseException.__traceback__` + attribute. If there is no traceback associated, this returns ``NULL``. @@ -746,8 +770,8 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. + Python through the :attr:`~BaseException.__context__` attribute. + If there is no context associated, this returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) @@ -761,7 +785,8 @@ Exception Objects Return the cause (either an exception instance, or ``None``, set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. + reference, as accessible from Python through the + :attr:`~BaseException.__cause__` attribute. .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) @@ -770,7 +795,8 @@ Exception Objects it. There is no type check to make sure that *cause* is either an exception instance or ``None``. This steals a reference to *cause*. - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + The :attr:`~BaseException.__suppress_context__` attribute is implicitly set + to ``True`` by this function. .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) @@ -879,8 +905,8 @@ because the :ref:`call protocol ` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it sets a :exc:`MemoryError` and returns a nonzero value. The function then checks if the recursion limit is reached. If this is the @@ -943,59 +969,59 @@ All standard Python exceptions are available as global variables whose names are the variables: .. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError + single: PyExc_BaseException (C var) + single: PyExc_Exception (C var) + single: PyExc_ArithmeticError (C var) + single: PyExc_AssertionError (C var) + single: PyExc_AttributeError (C var) + single: PyExc_BlockingIOError (C var) + single: PyExc_BrokenPipeError (C var) + single: PyExc_BufferError (C var) + single: PyExc_ChildProcessError (C var) + single: PyExc_ConnectionAbortedError (C var) + single: PyExc_ConnectionError (C var) + single: PyExc_ConnectionRefusedError (C var) + single: PyExc_ConnectionResetError (C var) + single: PyExc_EOFError (C var) + single: PyExc_FileExistsError (C var) + single: PyExc_FileNotFoundError (C var) + single: PyExc_FloatingPointError (C var) + single: PyExc_GeneratorExit (C var) + single: PyExc_ImportError (C var) + single: PyExc_IndentationError (C var) + single: PyExc_IndexError (C var) + single: PyExc_InterruptedError (C var) + single: PyExc_IsADirectoryError (C var) + single: PyExc_KeyError (C var) + single: PyExc_KeyboardInterrupt (C var) + single: PyExc_LookupError (C var) + single: PyExc_MemoryError (C var) + single: PyExc_ModuleNotFoundError (C var) + single: PyExc_NameError (C var) + single: PyExc_NotADirectoryError (C var) + single: PyExc_NotImplementedError (C var) + single: PyExc_OSError (C var) + single: PyExc_OverflowError (C var) + single: PyExc_PermissionError (C var) + single: PyExc_ProcessLookupError (C var) + single: PyExc_RecursionError (C var) + single: PyExc_ReferenceError (C var) + single: PyExc_RuntimeError (C var) + single: PyExc_StopAsyncIteration (C var) + single: PyExc_StopIteration (C var) + single: PyExc_SyntaxError (C var) + single: PyExc_SystemError (C var) + single: PyExc_SystemExit (C var) + single: PyExc_TabError (C var) + single: PyExc_TimeoutError (C var) + single: PyExc_TypeError (C var) + single: PyExc_UnboundLocalError (C var) + single: PyExc_UnicodeDecodeError (C var) + single: PyExc_UnicodeEncodeError (C var) + single: PyExc_UnicodeError (C var) + single: PyExc_UnicodeTranslateError (C var) + single: PyExc_ValueError (C var) + single: PyExc_ZeroDivisionError (C var) +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | @@ -1126,18 +1152,18 @@ the variables: These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError + single: PyExc_EnvironmentError (C var) + single: PyExc_IOError (C var) + single: PyExc_WindowsError (C var) +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | +| :c:data:`!PyExc_EnvironmentError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | +| :c:data:`!PyExc_IOError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | [2]_ | +| :c:data:`!PyExc_WindowsError` | [2]_ | +-------------------------------------+----------+ .. versionchanged:: 3.3 @@ -1163,17 +1189,17 @@ names are ``PyExc_`` followed by the Python exception name. These have the type the variables: .. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning + single: PyExc_Warning (C var) + single: PyExc_BytesWarning (C var) + single: PyExc_DeprecationWarning (C var) + single: PyExc_FutureWarning (C var) + single: PyExc_ImportWarning (C var) + single: PyExc_PendingDeprecationWarning (C var) + single: PyExc_ResourceWarning (C var) + single: PyExc_RuntimeWarning (C var) + single: PyExc_SyntaxWarning (C var) + single: PyExc_UnicodeWarning (C var) + single: PyExc_UserWarning (C var) +------------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index b36c800e00444a..e9019a0d500f7e 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -65,8 +65,14 @@ the :mod:`io` APIs instead. Overrides the normal behavior of :func:`io.open_code` to pass its parameter through the provided handler. - The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path, - void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + The *handler* is a function of type: + + .. c:namespace:: NULL + .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *) + + Equivalent of :c:expr:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be + :c:type:`PyUnicodeObject`. The *userData* pointer is passed into the hook function. Since hook functions may be called from different runtimes, this pointer should not @@ -90,7 +96,7 @@ the :mod:`io` APIs instead. .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - .. index:: single: Py_PRINT_RAW + .. index:: single: Py_PRINT_RAW (C macro) Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1accee2767a485..638a740e0c24da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -50,7 +50,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) - Get the *frame*'s ``f_builtins`` attribute. + Get the *frame*'s :attr:`~frame.f_builtins` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -81,7 +81,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) - Get the *frame*'s ``f_globals`` attribute. + Get the *frame*'s :attr:`~frame.f_globals` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -90,7 +90,7 @@ See also :ref:`Reflection `. .. c:function:: int PyFrame_GetLasti(PyFrameObject *frame) - Get the *frame*'s ``f_lasti`` attribute. + Get the *frame*'s :attr:`~frame.f_lasti` attribute. Returns -1 if ``frame.f_lasti`` is ``None``. @@ -120,12 +120,20 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s ``f_locals`` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + As part of :pep:`667`, return a proxy object for optimized scopes. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5857dba82c11c6..e7fb5090c09933 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -34,18 +34,20 @@ There are a few functions specific to Python functions. Return a new function object associated with the code object *code*. *globals* must be a dictionary with the global variables accessible to the function. - The function's docstring and name are retrieved from the code object. *__module__* + The function's docstring and name are retrieved from the code object. + :attr:`~function.__module__` is retrieved from *globals*. The argument defaults, annotations and closure are - set to ``NULL``. *__qualname__* is set to the same value as the code object's - ``co_qualname`` field. + set to ``NULL``. :attr:`~function.__qualname__` is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; - if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's ``co_qualname`` field. + :attr:`~function.__qualname__` attribute. + *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the :attr:`!__qualname__` attribute is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. versionadded:: 3.3 @@ -62,11 +64,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - Return a :term:`borrowed reference` to the *__module__* attribute of the - function object *op*. It can be *NULL*. + Return a :term:`borrowed reference` to the :attr:`~function.__module__` + attribute of the :ref:`function object ` *op*. + It can be *NULL*. - This is normally a string containing the module name, but can be set to any - other object by Python code. + This is normally a :class:`string ` containing the module name, + but can be set to any other object by Python code. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 6b2494ee4f0ed4..621da3eb069949 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -83,10 +83,15 @@ rules: .. versionadded:: 3.12 -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) +.. c:macro:: PyObject_GC_Resize(TYPE, op, newsize) - Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the - resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. + Resize an object allocated by :c:macro:`PyObject_NewVar`. + Returns the resized object of type ``TYPE*`` (refers to any C type) + or ``NULL`` on failure. + + *op* must be of type :c:expr:`PyVarObject *` + and must not be tracked by the collector yet. + *newsize* must be of type :c:type:`Py_ssize_t`. .. c:function:: void PyObject_GC_Track(PyObject *op) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst new file mode 100644 index 00000000000000..7345a048a4128b --- /dev/null +++ b/Doc/c-api/hash.rst @@ -0,0 +1,101 @@ +.. highlight:: c + +PyHash API +---------- + +See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. + +.. c:type:: Py_hash_t + + Hash value type: signed integer. + + .. versionadded:: 3.2 + +.. c:type:: Py_uhash_t + + Hash value type: unsigned integer. + + .. versionadded:: 3.2 + +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_BITS + + The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_MULTIPLIER + + Prime multiplier used in string and various other hashes. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_INF + + The hash value returned for a positive infinity. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_IMAG + + The multiplier used for the imaginary part of a complex number. + + .. versionadded:: 3.13 + +.. c:type:: PyHash_FuncDef + + Hash function definition used by :c:func:`PyHash_GetFuncDef`. + + .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + + Hash function. + + .. c:member:: const char *name + + Hash function name (UTF-8 encoded string). + + .. c:member:: const int hash_bits + + Internal size of the hash value in bits. + + .. c:member:: const int seed_bits + + Size of seed input in bits. + + .. versionadded:: 3.4 + + +.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void) + + Get the hash function definition. + + .. seealso:: + :pep:`456` "Secure and interchangeable hash algorithm". + + .. versionadded:: 3.4 + + +.. c:function:: Py_hash_t Py_HashPointer(const void *ptr) + + Hash a pointer value: process the pointer value as an integer (cast it to + ``uintptr_t`` internally). The pointer is not dereferenced. + + The function cannot fail: it cannot return ``-1``. + + .. versionadded:: 3.13 + +.. c:function:: Py_hash_t PyObject_GenericHash(PyObject *obj) + + Generic hashing function that is meant to be put into a type + object's ``tp_hash`` slot. + Its result only depends on the object's identity. + + .. impl-detail:: + In CPython, it is equivalent to :c:func:`Py_HashPointer`. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 137780cc359cf9..1054b38cb92f7d 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -13,20 +13,8 @@ Importing Modules single: __all__ (package variable) single: modules (in module sys) - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - + This is a wrapper around :c:func:`PyImport_Import()` which takes a + :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) @@ -154,7 +142,7 @@ Importing Modules :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`__file__` attribute will be set to the code object's - :attr:`!co_filename`. If applicable, :attr:`__cached__` will also + :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also be set. This function will reload the module if it was already imported. See @@ -320,7 +308,7 @@ Importing Modules The module name, as an ASCII encoded string. - .. c: member:: PyObject* (*initfunc)(void) + .. c:member:: PyObject* (*initfunc)(void) Initialization function for a module built into the interpreter. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc..ba56b03c6ac8e7 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index d164d1a752e295..1fab3f577f2f89 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -29,6 +29,8 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetAllocator` * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` * :c:func:`PySys_ResetWarnOptions` * Informative functions: @@ -53,13 +55,18 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + .. note:: The following functions **should not be called** before :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - and :c:func:`Py_GetProgramName`. + :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. .. _global-conf-vars: @@ -326,13 +333,14 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: + single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) pair: module; builtins pair: module; __main__ pair: module; sys triple: module; search; path - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; see @@ -388,9 +396,16 @@ Initializing and finalizing the interpreter :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -425,6 +440,34 @@ Process-wide parameters ======================= +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated:: 3.11 + + .. c:function:: wchar_t* Py_GetProgramName() Return the program name set with :c:member:`PyConfig.program_name`, or the default. @@ -626,6 +669,106 @@ Process-wide parameters ``sys.version``. +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; + check w/ Guido. + + .. deprecated:: 3.11 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated:: 3.11 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated:: 3.11 + + .. c:function:: wchar_t* Py_GetPythonHome() Return the default "home", that is, the value set by @@ -661,7 +804,7 @@ operations could cause problems in a multi-threaded program: for example, when two threads simultaneously increment the reference count of the same object, the reference count could end up being incremented only once instead of twice. -.. index:: single: setswitchinterval() (in module sys) +.. index:: single: setswitchinterval (in module sys) Therefore, the rule exists that only the thread that has acquired the :term:`GIL` may operate on Python objects or call Python/C API functions. @@ -671,8 +814,7 @@ released around potentially blocking I/O operations like reading or writing a file, so that other Python threads can run in the meantime. .. index:: - single: PyThreadState - single: PyThreadState + single: PyThreadState (C type) The Python interpreter keeps some thread-specific bookkeeping information inside a data structure called :c:type:`PyThreadState`. There's also one @@ -698,8 +840,8 @@ This is so common that a pair of macros exists to simplify it:: Py_END_ALLOW_THREADS .. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the @@ -714,8 +856,8 @@ The block above expands to the following code:: PyEval_RestoreThread(_save); .. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) Here is how these functions work: the global interpreter lock is used to protect the pointer to the current thread state. When releasing the lock and saving the thread state, @@ -842,6 +984,33 @@ code, or when embedding the Python interpreter: This thread's interpreter state. +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + .. c:function:: PyThreadState* PyEval_SaveThread() Release the global interpreter lock (if it has been created) and reset the @@ -1399,8 +1568,8 @@ function. You can create and destroy them using the following functions: may be stored internally on the :c:type:`PyInterpreterState`. .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) Extension modules are shared between (sub-)interpreters as follows: @@ -1428,7 +1597,7 @@ function. You can create and destroy them using the following functions: As with multi-phase initialization, this means that only C-level static and global variables are shared between these modules. - .. index:: single: close() (in module os) + .. index:: single: close (in module os) .. c:function:: PyThreadState* Py_NewInterpreter(void) @@ -1451,7 +1620,7 @@ function. You can create and destroy them using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_FinalizeEx() + .. index:: single: Py_FinalizeEx (C function) Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the @@ -1485,7 +1654,7 @@ otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared because of the refcount. One simple but less-efficient approach around this is to use a global lock around all use of some state (or object). Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them "immortal". +can be made safe in spite of their refcounts by making them :term:`immortal`. In fact, this has been done for the builtin singletons, small integers, and a number of other builtin objects. @@ -1543,8 +1712,6 @@ pointer and a void pointer argument. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - .. index:: single: Py_AddPendingCall() - Schedule a function to be called from the main interpreter thread. On success, ``0`` is returned and *func* is queued for being called in the main thread. On failure, ``-1`` is returned without setting any exception. @@ -1578,14 +1745,14 @@ pointer and a void pointer argument. function is generally **not** suitable for calling Python code from arbitrary C threads. Instead, use the :ref:`PyGILState API`. + .. versionadded:: 3.1 + .. versionchanged:: 3.9 If this function is called in a subinterpreter, the function *func* is now scheduled to be called from the subinterpreter, rather than being called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. - .. versionadded:: 3.1 - .. _profiling: Profiling and Tracing @@ -1662,7 +1829,8 @@ Python-level trace functions in previous versions. The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. .. c:var:: int PyTrace_RETURN @@ -1694,7 +1862,7 @@ Python-level trace functions in previous versions. The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not profiling functions) when a new opcode is about to be executed. This event is not emitted by default: it must be explicitly requested by setting - :attr:`f_trace_opcodes` to *1* on the frame. + :attr:`~frame.f_trace_opcodes` to *1* on the frame. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) @@ -1748,6 +1916,58 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: @@ -1944,3 +2164,145 @@ be used in new code. .. c:function:: void PyThread_delete_key_value(int key) .. c:function:: void PyThread_ReInitTLS() +Synchronization Primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily release the :term:`GIL` if it is held. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. _python-critical-section-api: + +Python Critical Section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. +When :c:func:`PyEval_RestoreThread` is called, the most recent critical section +is resumed, and its locks reacquired. This means the critical section API +provides weaker guarantees than traditional locks -- they are useful because +their behavior is similar to the :term:`GIL`. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potentital deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-objects locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection_Begin2(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End2(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0240e25b6f1607..5195f6cccfe9df 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -253,11 +253,21 @@ PyPreConfig * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc memory allocator ` with :ref:`debug hooks `. + * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast + malloc replacement. + * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast + malloc replacement with :ref:`debug hooks `. + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are not supported if Python is :option:`configured using --without-pymalloc <--without-pymalloc>`. + ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-mimalloc + <--without-mimalloc>` or if the underlying atomic support isn't + available. + See :ref:`Memory Management `. Default: ``PYMEM_ALLOCATOR_NOT_SET``. @@ -716,7 +726,7 @@ PyConfig Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. - Need a special build of Python with the ``Py_TRACE_REFS`` macro defined: + Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined: see the :option:`configure --with-trace-refs option <--with-trace-refs>`. Default: ``0``. @@ -1048,7 +1058,7 @@ PyConfig Incremented by the :option:`-d` command line option. Set to the :envvar:`PYTHONDEBUG` environment variable value. - Need a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro must be defined). Default: ``0``. @@ -1100,6 +1110,7 @@ PyConfig Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and the :envvar:`PYTHONPYCACHEPREFIX` environment variable. + The command-line option takes precedence. If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. @@ -1143,13 +1154,27 @@ PyConfig Default: ``NULL``. + .. c:member:: wchar_t* run_presite + + ``package.module`` path to module that should be imported before + ``site.py`` is run. + + Set by the :option:`-X presite=package.module <-X>` command-line + option and the :envvar:`PYTHON_PRESITE` environment variable. + The command-line option takes precedence. + + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + must be defined). + + Default: ``NULL``. + .. c:member:: int show_ref_count - Show total reference count at exit (excluding immortal objects)? + Show total reference count at exit (excluding :term:`immortal` objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. - Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` + Needs a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` macro must be defined). Default: ``0``. @@ -1225,8 +1250,11 @@ PyConfig If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` for more information. - Set by :option:`-X perf <-X>` command line option and by the - :envvar:`PYTHONPERFSUPPORT` environment variable. + Set by :option:`-X perf <-X>` command-line option and by the + :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support + with stack pointers and :option:`-X perf_jit <-X>` command-line option + and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf + support with DWARF JIT information. Default: ``-1``. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4dbca92b18b5cd..8ef463e3f88ca8 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -148,7 +148,7 @@ complete listing. worse performances (due to increased code size for example). The compiler is usually smarter than the developer for the cost/benefit analysis. - If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. It must be specified before the function return type. Usage:: @@ -325,8 +325,8 @@ objects that reference each other here; for now, the solution is "don't do that.") .. index:: - single: Py_INCREF() - single: Py_DECREF() + single: Py_INCREF (C function) + single: Py_DECREF (C function) Reference counts are always manipulated explicitly. The normal way is to use the macro :c:func:`Py_INCREF` to take a new reference to an @@ -401,8 +401,8 @@ function, that function assumes that it now owns that reference, and you are not responsible for it any longer. .. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() + single: PyList_SetItem (C function) + single: PyTuple_SetItem (C function) Few functions steal references; the two notable exceptions are :c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference @@ -491,8 +491,8 @@ using :c:func:`PySequence_GetItem` (which happens to take exactly the same arguments), you do own a reference to the returned object. .. index:: - single: PyList_GetItem() - single: PySequence_GetItem() + single: PyList_GetItem (C function) + single: PySequence_GetItem (C function) Here is an example of how you could write a function that computes the sum of the items in a list of integers; once using :c:func:`PyList_GetItem`, and once @@ -587,7 +587,7 @@ caller, then to the caller's caller, and so on, until they reach the top-level interpreter, where they are reported to the user accompanied by a stack traceback. -.. index:: single: PyErr_Occurred() +.. index:: single: PyErr_Occurred (C function) For C programmers, however, error checking always has to be explicit. All functions in the Python/C API can raise exceptions, unless an explicit claim is @@ -601,8 +601,8 @@ ambiguous return value, and require explicit testing for errors with :c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. .. index:: - single: PyErr_SetString() - single: PyErr_Clear() + single: PyErr_SetString (C function) + single: PyErr_Clear (C function) Exception state is maintained in per-thread storage (this is equivalent to using global storage in an unthreaded application). A thread can be in one of @@ -624,7 +624,7 @@ an exception is being passed on between C functions until it reaches the Python bytecode interpreter's main loop, which takes care of transferring it to ``sys.exc_info()`` and friends. -.. index:: single: exc_info() (in module sys) +.. index:: single: exc_info (in module sys) Note that starting with Python 1.5, the preferred, thread-safe way to access the exception state from Python code is to call the function :func:`sys.exc_info`, @@ -709,9 +709,9 @@ Here is the corresponding C code, in all its glory:: .. index:: single: incr_item() .. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() + single: PyErr_ExceptionMatches (C function) + single: PyErr_Clear (C function) + single: Py_XDECREF (C function) This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and @@ -735,7 +735,7 @@ the finalization, of the Python interpreter. Most functionality of the interpreter can only be used after the interpreter has been initialized. .. index:: - single: Py_Initialize() + single: Py_Initialize (C function) pair: module; builtins pair: module; __main__ pair: module; sys @@ -770,10 +770,10 @@ environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. .. index:: - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() + single: Py_GetPath (C function) + single: Py_GetPrefix (C function) + single: Py_GetExecPrefix (C function) + single: Py_GetProgramFullPath (C function) The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling @@ -784,7 +784,7 @@ control has to provide its own implementation of :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and :c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). -.. index:: single: Py_IsInitialized() +.. index:: single: Py_IsInitialized (C function) Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to @@ -812,12 +812,14 @@ available that support tracing of reference counts, debugging the memory allocator, or low-level profiling of the main interpreter loop. Only the most frequently used builds will be described in the remainder of this section. -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +.. c:macro:: Py_DEBUG + +Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding +:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding :option:`--with-pydebug` to the :file:`./configure` command. It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled +not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled in the Unix build, compiler optimization is disabled. In addition to the reference count debugging described below, extra checks are @@ -832,4 +834,3 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. - diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c15cecd41b89d1..53eb54d3e1021a 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -56,13 +56,21 @@ List Objects Similar to :c:func:`PyList_Size`, but without error checking. -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) +.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), + supported. If *index* is out of bounds (:code:`<0 or >=len(list)`), return ``NULL`` and set an :exc:`IndexError` exception. + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Like :c:func:`PyList_GetItemRef`, but returns a + :term:`borrowed reference` instead of a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) @@ -128,6 +136,30 @@ List Objects list is not supported. +.. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) + + Extend *list* with the contents of *iterable*. This is the same as + ``PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable)`` + and analogous to ``list.extend(iterable)`` or ``list += iterable``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + +.. c:function:: int PyList_Clear(PyObject *list) + + Remove all items from *list*. This is the same as + ``PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL)`` and analogous to + ``list.clear()`` or ``del list[:]``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + .. c:function:: int PyList_Sort(PyObject *list) Sort the items of *list* in place. Return ``0`` on success, ``-1`` on diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 045604870d3c84..42162914c0aec8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -113,11 +113,37 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as a two's-complement signed number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is a sign bit. Passing + ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` will produce the same result as calling + :c:func:`PyLong_FromUnsignedNativeBytes`. Other flags are ignored. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as an unsigned number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is not a sign bit. Flags other than endian are ignored. + + .. versionadded:: 3.13 + + .. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: - single: LONG_MAX + single: LONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an @@ -210,7 +236,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) .. index:: - single: PY_SSIZE_T_MAX + single: PY_SSIZE_T_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must @@ -225,7 +251,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) .. index:: - single: ULONG_MAX + single: ULONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`unsigned long` representation of *pylong*. *pylong* @@ -241,7 +267,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) .. index:: - single: SIZE_MAX + single: SIZE_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be @@ -332,6 +358,162 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int flags) + + Copy the Python integer value *pylong* to a native *buffer* of size + *n_bytes*. The *flags* can be set to ``-1`` to behave similarly to a C cast, + or to values documented below to control the behavior. + + Returns ``-1`` with an exception raised on error. This may happen if + *pylong* cannot be interpreted as an integer, or if *pylong* was negative + and the ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` flag was set. + + Otherwise, returns the number of bytes required to store the value. + If this is equal to or less than *n_bytes*, the entire value was copied. + All *n_bytes* of the buffer are written: large buffers are padded with + zeroes. + + If the returned value is greater than than *n_bytes*, the value was + truncated: as many of the lowest bits of the value as could fit are written, + and the higher bits are ignored. This matches the typical behavior + of a C-style downcast. + + .. note:: + + Overflow is not considered an error. If the returned value + is larger than *n_bytes*, most significant bits were discarded. + + ``0`` will never be returned. + + Values are always copied as two's-complement. + + Usage example:: + + int32_t value; + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); + if (bytes < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + else if (bytes <= (Py_ssize_t)sizeof(value)) { + // Success! + } + else { + // Overflow occurred, but 'value' contains the truncated + // lowest bits of pylong. + } + + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. + + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of the value. + + To get at the entire Python value of an unknown size, the function can be + called twice: first to determine the buffer size, then to fill it:: + + // Ask how much space we need. + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); + if (expected < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + assert(expected != 0); // Impossible per the API definition. + uint8_t *bignum = malloc(expected); + if (!bignum) { + PyErr_SetString(PyExc_MemoryError, "bignum malloc failed."); + return NULL; + } + // Safely get the entire value. + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception has been set. + free(bignum); + return NULL; + } + else if (bytes > expected) { // This should not be possible. + PyErr_SetString(PyExc_RuntimeError, + "Unexpected bignum truncation after a size check."); + free(bignum); + return NULL; + } + // The expected success given the above pre-check. + // ... use bignum ... + free(bignum); + + *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults + that behave most like a C cast, or a combintation of the other flags in + the table below. + Note that ``-1`` cannot be combined with other flags. + + Currently, ``-1`` corresponds to + ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + + ============================================= ====== + Flag Value + ============================================= ====== + .. c:macro:: Py_ASNATIVEBYTES_DEFAULTS ``-1`` + .. c:macro:: Py_ASNATIVEBYTES_BIG_ENDIAN ``0`` + .. c:macro:: Py_ASNATIVEBYTES_LITTLE_ENDIAN ``1`` + .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` + .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` + .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` + ============================================= ====== + + Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian + flags. Passing ``2`` is reserved. + + By default, sufficient buffer will be requested to include a sign bit. + For example, when converting 128 with *n_bytes=1*, the function will return + 2 (or more) in order to store a zero sign bit. + + If ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` is specified, a zero sign bit + will be omitted from size calculations. This allows, for example, 128 to fit + in a single-byte buffer. If the destination buffer is later treated as + signed, a positive input value may become negative. + Note that the flag does not affect handling of negative values: for those, + space for a sign bit is always requested. + + Specifying ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` causes an exception to be set + if *pylong* is negative. Without this flag, negative values will be copied + provided there is enough space for at least one sign bit, regardless of + whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. + + .. note:: + + With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without + *REJECT_NEGATIVE*), multiple Python integers can map to a single value + without overflow. For example, both ``255`` and ``-1`` fit a single-byte + buffer and set all its bits. + This matches typical C cast behavior. + + .. versionadded:: 3.13 + + +.. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) + + Get the sign of the integer object *obj*. + + On success, set *\*sign* to the integer sign (0, -1 or +1 for zero, negative or + positive integer, respectively) and return 0. + + On failure, return -1 with an exception set. This function always succeeds + if *obj* is a :c:type:`PyLongObject` or its subtype. + + .. versionadded:: 3.14 + + .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) Return 1 if *op* is compact, 0 otherwise. @@ -340,7 +522,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. a “fast path†for small integers. For compact values use :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a :c:func:`PyLong_As* ` function or - :c:func:`calling ` :meth:`int.to_bytes`. + :c:func:`PyLong_AsNativeBytes`. The speedup is expected to be negligible for most users. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index e98c178ac27a37..9da09a21607f61 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -41,10 +41,10 @@ buffers is performed on demand by the Python memory manager through the Python/C API functions listed in this document. .. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() + single: malloc (C function) + single: calloc (C function) + single: realloc (C function) + single: free (C function) To avoid memory corruption, extension writers should never try to operate on Python objects with the functions exported by the C library: :c:func:`malloc`, @@ -267,14 +267,14 @@ The following type-oriented macros are provided for convenience. Note that .. c:macro:: PyMem_New(TYPE, n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have + memory. Returns a pointer cast to ``TYPE*``. The memory will not have been initialized in any way. .. c:macro:: PyMem_Resize(p, TYPE, n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return, + sizeof(TYPE))`` bytes. Returns a pointer cast to ``TYPE*``. On return, *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. @@ -391,6 +391,8 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. +* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc + allocator will be used if mimalloc support isn't available. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -626,7 +628,8 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit +platforms. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the @@ -671,6 +674,16 @@ Customize pymalloc Arena Allocator Set the arena allocator. +.. _mimalloc: + +The mimalloc allocator +====================== + +.. versionadded:: 3.13 + +Python supports the mimalloc allocator when the underlying platform support is available. +mimalloc "is a general purpose allocator with excellent performance characteristics. +Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." tracemalloc C API ================= diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 2aa43318e7a455..f6038032805259 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -20,6 +20,17 @@ any other object. read/write, otherwise it may be either read-only or read/write at the discretion of the exporter. + +.. c:macro:: PyBUF_READ + + Flag to request a readonly buffer. + + +.. c:macro:: PyBUF_WRITE + + Flag to request a writable buffer. + + .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) Create a memoryview object using *mem* as the underlying buffer. @@ -41,6 +52,8 @@ any other object. original memory. Otherwise, a copy is made and the memoryview points to a new bytes object. + *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. + .. c:function:: int PyMemoryView_Check(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 979b22261efa3b..3ea7aeb55d0d74 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -43,6 +43,8 @@ Module Objects to ``None``); the caller is responsible for providing a :attr:`__file__` attribute. + Return ``NULL`` with an exception set on error. + .. versionadded:: 3.3 .. versionchanged:: 3.4 @@ -265,6 +267,8 @@ of the following two module creation functions: API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_Create` @@ -411,6 +415,31 @@ The available slot types are: .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`free-threaded-cpython` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded:: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -436,6 +465,8 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,7 +517,7 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + Return ``-1`` if *value* is ``NULL``. It must be called with an exception raised in this case. Example usage:: @@ -576,15 +607,16 @@ state: .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. + used from the module's initialization function. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddIntMacro(module, macro) @@ -592,7 +624,7 @@ state: Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddStringMacro(module, macro) @@ -605,10 +637,23 @@ state: The type object is finalized by calling internally :c:func:`PyType_Ready`. The name of the type object is taken from the last component of :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.9 +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` with an exception set on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ @@ -644,14 +689,14 @@ since multiple such modules can be created from a single definition. The caller must hold the GIL. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. The caller must hold the GIL. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 00000000000000..b34035b5548f02 --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,164 @@ +.. highlight:: c + +.. _monitoring: + +Monitoring C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) + + Fire a ``STOP_ITERATION`` event. If ``value`` is an instance of :exc:`StopIteration`, it is used. Otherwise, + a new :exc:`StopIteration` instance is created with ``value`` as its argument. + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. :c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + ``PyMonitoring_EnterScope`` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by ``PyMonitoring_EnterScope`` itelf. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + ``PyMonitoring_EnterScope`` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + +.. :c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with ``PyMonitoring_EnterScope``. diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index dd8bfb56104251..f1941fc4bc4e85 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -16,10 +16,10 @@ same reason. .. c:var:: PyObject* Py_None The Python ``None`` object, denoting lack of value. This object has no methods - and is `immortal `_. + and is :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_None` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_None` is :term:`immortal`. .. c:macro:: Py_RETURN_NONE diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a4e3e74861a315..2103a64d8ffbb7 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -6,6 +6,56 @@ Object Protocol =============== +.. c:function:: PyObject* Py_GetConstant(unsigned int constant_id) + + Get a :term:`strong reference` to a constant. + + Set an exception and return ``NULL`` if *constant_id* is invalid. + + *constant_id* must be one of these constant identifiers: + + .. c:namespace:: NULL + + ======================================== ===== ========================= + Constant Identifier Value Returned object + ======================================== ===== ========================= + .. c:macro:: Py_CONSTANT_NONE ``0`` :py:data:`None` + .. c:macro:: Py_CONSTANT_FALSE ``1`` :py:data:`False` + .. c:macro:: Py_CONSTANT_TRUE ``2`` :py:data:`True` + .. c:macro:: Py_CONSTANT_ELLIPSIS ``3`` :py:data:`Ellipsis` + .. c:macro:: Py_CONSTANT_NOT_IMPLEMENTED ``4`` :py:data:`NotImplemented` + .. c:macro:: Py_CONSTANT_ZERO ``5`` ``0`` + .. c:macro:: Py_CONSTANT_ONE ``6`` ``1`` + .. c:macro:: Py_CONSTANT_EMPTY_STR ``7`` ``''`` + .. c:macro:: Py_CONSTANT_EMPTY_BYTES ``8`` ``b''`` + .. c:macro:: Py_CONSTANT_EMPTY_TUPLE ``9`` ``()`` + ======================================== ===== ========================= + + Numeric values are only given for projects which cannot use the constant + identifiers. + + + .. versionadded:: 3.13 + + .. impl-detail:: + + In CPython, all of these constants are :term:`immortal`. + + +.. c:function:: PyObject* Py_GetConstantBorrowed(unsigned int constant_id) + + Similar to :c:func:`Py_GetConstant`, but return a :term:`borrowed + reference`. + + This function is primarily intended for backwards compatibility: + using :c:func:`Py_GetConstant` is recommended for new code. + + The reference is borrowed from the interpreter, and is valid until the + interpreter finalization. + + .. versionadded:: 3.13 + + .. c:var:: PyObject* Py_NotImplemented The ``NotImplemented`` singleton, used to signal that an operation is @@ -16,7 +66,15 @@ Object Protocol Properly handle returning :c:data:`Py_NotImplemented` from within a C function (that is, create a new :term:`strong reference` - to NotImplemented and return it). + to :const:`NotImplemented` and return it). + + +.. c:macro:: Py_PRINT_RAW + + Flag to be used with multiple functions that print the object (like + :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). + If passed, these function would use the :func:`str` of the object + instead of the :func:`repr`. .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -47,9 +105,8 @@ Object Protocol .. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. + This function always succeeds. .. note:: @@ -222,12 +279,8 @@ Object Protocol .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. + like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if + the result is false, ``1`` otherwise. .. note:: If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 118af7a1a8cf90..bf50107347e0e7 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -17,26 +17,27 @@ of Python objects. Note that the returned value may not actually reflect how many references to the object are actually held. For example, some - objects are "immortal" and have a very high refcount that does not + objects are :term:`immortal` and have a very high refcount that does not reflect the actual number of references. Consequently, do not rely on the returned value to be accurate, other than a value of 0 or 1. Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. - .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. + .. versionchanged:: 3.11 + The parameter type is no longer :c:expr:`const PyObject*`. + .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) Set the object *o* reference counter to *refcnt*. - Note that this function has no effect on - `immortal `_ - objects. + On :ref:`Python build with Free Threading `, if + *refcnt* is larger than ``UINT32_MAX``, the object is made :term:`immortal`. + + This function has no effect on :term:`immortal` objects. .. versionadded:: 3.9 @@ -49,6 +50,8 @@ of Python objects. Indicate taking a new :term:`strong reference` to object *o*, indicating it is in use and should not be destroyed. + This function has no effect on :term:`immortal` objects. + This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. @@ -113,6 +116,8 @@ of Python objects. Release a :term:`strong reference` to object *o*, indicating the reference is no longer used. + This function has no effect on :term:`immortal` objects. + Once the last :term:`strong reference` is released (i.e. the object's reference count reaches 0), the object's type's deallocation diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 4b1c4770848a30..038e6977104560 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -7,18 +7,48 @@ Reflection .. c:function:: PyObject* PyEval_GetBuiltins(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameBuiltins` instead. + Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. .. c:function:: PyObject* PyEval_GetLocals(void) - Return a dictionary of the local variables in the current execution frame, + .. deprecated:: 3.13 + + Use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result + of :c:func:`PyEval_GetFrame` to access the :attr:`~frame.f_locals` attribute of the + currently executing frame. + + Return a mapping providing access to the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + Refer to :func:`locals` for details of the mapping returned at different scopes. + + As this function returns a :term:`borrowed reference`, the dictionary returned for + :term:`optimized scopes ` is cached on the frame object and will remain + alive as long as the frame object does. Unlike :c:func:`PyEval_GetFrameLocals` and + :func:`locals`, subsequent calls to this function in the same frame will update the + contents of the cached dictionary to reflect changes in the state of the local variables + rather than returning a new snapshot. + + .. versionchanged:: 3.13 + As part of :pep:`667`, :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals ` no longer make use of the shared cache + dictionary. Refer to the :ref:`What's New entry ` for + additional details. + .. c:function:: PyObject* PyEval_GetGlobals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameGlobals` instead. + Return a dictionary of the global variables in the current execution frame, or ``NULL`` if no frame is currently executing. @@ -31,6 +61,36 @@ Reflection See also :c:func:`PyThreadState_GetFrame`. +.. c:function:: PyObject* PyEval_GetFrameBuiltins(void) + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameLocals(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`locals` in Python code. + + To access :attr:`~frame.f_locals` on the current frame without making an independent + snapshot in :term:`optimized scopes `, call :c:func:`PyFrame_GetLocals` + on the result of :c:func:`PyEval_GetFrame`. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameGlobals(void) + + Return a dictionary of the global variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`globals` in Python code. + + .. versionadded:: 3.13 + + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) Return the name of *func* if it is a function, class or instance object, else the diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 1e8a09509032f5..cba823aa027bd6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. @@ -163,4 +163,6 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: int PySet_Clear(PyObject *set) - Empty an existing set of all elements. + Empty an existing set of all elements. Return ``0`` on + success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of + :class:`set` or its subtype. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 9e880c6b7f25ad..8adf6a961378a3 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -23,7 +23,9 @@ Slice Objects Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if + ``None`` will be used for the corresponding attribute. + + Return ``NULL`` with an exception set if the new object could not be allocated. @@ -52,7 +54,7 @@ Slice Objects of bounds indices are clipped in a manner consistent with the handling of normal slices. - Returns ``0`` on success and ``-1`` on error with exception set. + Return ``0`` on success and ``-1`` on error with an exception set. .. note:: This function is considered not safe for resizable sequences. @@ -95,7 +97,7 @@ Slice Objects ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.6.1 @@ -119,8 +121,7 @@ Ellipsis Object .. c:var:: PyObject *Py_Ellipsis The Python ``Ellipsis`` object. This object has no methods. Like - :c:data:`Py_None`, it is an `immortal `_. - singleton object. + :c:data:`Py_None`, it is an :term:`immortal` singleton object. .. versionchanged:: 3.12 :c:data:`Py_Ellipsis` is immortal. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 63a100a6f26f24..5b9e43874c7f2b 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -16,7 +16,7 @@ CPython's Application Binary Interface (ABI) is forward- and backwards-compatible across a minor release (if these are compiled the same way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, -but will need to be compiled separately for 3.9.x and 3.10.x. +but will need to be compiled separately for 3.9.x and 3.11.x. There are two tiers of C API with different stability expectations: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 25cb4ed40f63e7..f9461ab01f6049 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -187,26 +187,26 @@ Implementing functions and methods PyObject *kwargs); -.. c:type:: _PyCFunctionFast +.. c:type:: PyCFunctionFast Type of the functions used to implement Python callables in C with signature :c:macro:`METH_FASTCALL`. The function signature is:: - PyObject *_PyCFunctionFast(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs); + PyObject *PyCFunctionFast(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs); -.. c:type:: _PyCFunctionFastWithKeywords +.. c:type:: PyCFunctionFastWithKeywords Type of the functions used to implement Python callables in C with signature :ref:`METH_FASTCALL | METH_KEYWORDS `. The function signature is:: - PyObject *_PyCFunctionFastWithKeywords(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames); + PyObject *PyCFunctionFastWithKeywords(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); .. c:type:: PyCMethod @@ -290,7 +290,7 @@ There are these calling conventions: .. c:macro:: METH_FASTCALL Fast calling convention supporting only positional arguments. - The methods have the type :c:type:`_PyCFunctionFast`. + The methods have the type :c:type:`PyCFunctionFast`. The first parameter is *self*, the second parameter is a C array of :c:expr:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). @@ -306,7 +306,7 @@ There are these calling conventions: :c:expr:`METH_FASTCALL | METH_KEYWORDS` Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, - with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + with methods of type :c:type:`PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol `: there is an additional fourth :c:expr:`PyObject*` parameter @@ -399,6 +399,40 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. +.. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) + + Turn *ml* into a Python :term:`callable` object. + The caller must ensure that *ml* outlives the :term:`callable`. + Typically, *ml* is defined as a static variable. + + The *self* parameter will be passed as the *self* argument + to the C function in ``ml->ml_meth`` when invoked. + *self* can be ``NULL``. + + The :term:`callable` object's ``__module__`` attribute + can be set from the given *module* argument. + *module* should be a Python string, + which will be used as name of the module the function is defined in. + If unavailable, it can be set to :const:`None` or ``NULL``. + + .. seealso:: :attr:`function.__module__` + + The *cls* parameter will be passed as the *defining_class* + argument to the C function. + Must be set if :c:macro:`METH_METHOD` is set on ``ml->ml_flags``. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) + + Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. + + +.. c:function:: PyObject * PyCFunction_New(PyMethodDef *ml, PyObject *self) + + Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. + Accessing attributes of extension types --------------------------------------- @@ -419,15 +453,15 @@ Accessing attributes of extension types The string should be static, no copy is made of it. - .. c:member:: Py_ssize_t offset - - The offset in bytes that the member is located on the type’s object struct. - .. c:member:: int type The type of the member in the C struct. See :ref:`PyMemberDef-types` for the possible values. + .. c:member:: Py_ssize_t offset + + The offset in bytes that the member is located on the type’s object struct. + .. c:member:: int flags Zero or more of the :ref:`PyMemberDef-flags`, combined using bitwise OR. @@ -517,19 +551,19 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: from ``PyObject``. Can only be used as part of :c:member:`Py_tp_members ` - :c:type:`slot ` when creating a class using negative + :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - This flag is only used in :c:type:`PyTypeSlot`. + This flag is only used in :c:type:`PyType_Slot`. When setting :c:member:`~PyTypeObject.tp_members` during class creation, Python clears it and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED + single: READ_RESTRICTED (C macro) + single: WRITE_RESTRICTED (C macro) + single: RESTRICTED (C macro) .. versionchanged:: 3.10 @@ -540,7 +574,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing. .. index:: - single: READONLY + single: READONLY (C macro) .. versionchanged:: 3.12 @@ -592,7 +626,7 @@ Macro name C type Python type (*): Zero-terminated, UTF8-encoded C string. With :c:macro:`!Py_T_STRING` the C representation is a pointer; - with :c:macro:`!Py_T_STRING_INLINE` the string is stored directly + with :c:macro:`!Py_T_STRING_INPLACE` the string is stored directly in the structure. (**): String of length 1. Only ASCII is accepted. @@ -603,24 +637,24 @@ Macro name C type Python type Reading a ``NULL`` pointer raises :py:exc:`AttributeError`. .. index:: - single: T_BYTE - single: T_SHORT - single: T_INT - single: T_LONG - single: T_LONGLONG - single: T_UBYTE - single: T_USHORT - single: T_UINT - single: T_ULONG - single: T_ULONGULONG - single: T_PYSSIZET - single: T_FLOAT - single: T_DOUBLE - single: T_BOOL - single: T_CHAR - single: T_STRING - single: T_STRING_INPLACE - single: T_OBJECT_EX + single: T_BYTE (C macro) + single: T_SHORT (C macro) + single: T_INT (C macro) + single: T_LONG (C macro) + single: T_LONGLONG (C macro) + single: T_UBYTE (C macro) + single: T_USHORT (C macro) + single: T_UINT (C macro) + single: T_ULONG (C macro) + single: T_ULONGULONG (C macro) + single: T_PYSSIZET (C macro) + single: T_FLOAT (C macro) + single: T_DOUBLE (C macro) + single: T_BOOL (C macro) + single: T_CHAR (C macro) + single: T_STRING (C macro) + single: T_STRING_INPLACE (C macro) + single: T_OBJECT_EX (C macro) single: structmember.h .. versionadded:: 3.12 @@ -659,7 +693,8 @@ Defining Getters and Setters .. c:member:: setter set - Optional C function to set or delete the attribute, if omitted the attribute is readonly. + Optional C function to set or delete the attribute. + If ``NULL``, the attribute is read-only. .. c:member:: const char* doc @@ -667,20 +702,20 @@ Defining Getters and Setters .. c:member:: void* closure - Optional function pointer, providing additional data for getter and setter. + Optional user data pointer, providing additional data for getter and setter. - The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``):: +.. c:type:: PyObject *(*getter)(PyObject *, void *) - typedef PyObject *(*getter)(PyObject *, void *); + The ``get`` function takes one :c:expr:`PyObject*` parameter (the + instance) and a user data pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: +.. c:type:: int (*setter)(PyObject *, PyObject *, void *) - typedef int (*setter)(PyObject *, PyObject *, void *); + ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and + the value to be set) and a user data pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e3c54b075114ff..d6fca1a0b0a219 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -5,6 +5,7 @@ Operating System Utilities ========================== + .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a @@ -97,27 +98,30 @@ Operating System Utilities .. c:function:: int PyOS_CheckStack() + .. index:: single: USE_STACKCHECK (C macro) + Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently + check, but is only available when :c:macro:`!USE_STACKCHECK` is defined (currently on certain versions of Windows using the Microsoft Visual C++ compiler). - :c:macro:`USE_STACKCHECK` will be defined automatically; you should never + :c:macro:`!USE_STACKCHECK` will be defined automatically; you should never change the definition in your own code. +.. c:type:: void (*PyOS_sighandler_t)(int) + + .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void - (\*)(int)`. + directly! .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:expr:`void (\*)(int)`. + not call those functions directly! .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) @@ -342,10 +346,8 @@ accessible to C code. They all work with the current interpreter thread's silently abort the operation by raising an error subclassed from :class:`Exception` (other errors will not be silenced). - The hook function is of type :c:expr:`int (*)(const char *event, PyObject - *args, void *userData)`, where *args* is guaranteed to be a - :c:type:`PyTupleObject`. The hook function is always called with the GIL - held by the Python interpreter that raised the event. + The hook function is always called with the GIL held by the Python + interpreter that raised the event. See :pep:`578` for a detailed description of auditing. Functions in the runtime and standard library that raise events are listed in the @@ -354,12 +356,21 @@ accessible to C code. They all work with the current interpreter thread's .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - If the interpreter is initialized, this function raises a auditing event + If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an exception derived from :class:`Exception`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. + .. c:namespace:: NULL + .. c:type:: int (*Py_AuditHookFunction) (const char *event, PyObject *args, void *userData) + + The type of the hook function. + *event* is the C string event argument passed to :c:func:`PySys_Audit` or + :c:func:`PySys_AuditTuple`. + *args* is guaranteed to be a :c:type:`PyTupleObject`. + *userData* is the argument passed to PySys_AddAuditHook(). + .. versionadded:: 3.8 @@ -371,7 +382,7 @@ Process Control .. c:function:: void Py_FatalError(const char *message) - .. index:: single: abort() + .. index:: single: abort (C function) Print a fatal error message and kill the process. No cleanup is performed. This function should only be invoked when a condition is detected that would @@ -391,8 +402,8 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_FinalizeEx() - single: exit() + single: Py_FinalizeEx (C function) + single: exit (C function) Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` @@ -405,7 +416,7 @@ Process Control .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) single: cleanup functions Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst new file mode 100644 index 00000000000000..5cfdef71b3e191 --- /dev/null +++ b/Doc/c-api/time.rst @@ -0,0 +1,112 @@ +.. highlight:: c + +PyTime C API +============ + +.. versionadded:: 3.13 + +The clock C API provides access to system clocks. +It is similar to the Python :mod:`time` module. + +For C API related to the :mod:`datetime` module, see :ref:`datetimeobjects`. + + +Types +----- + +.. c:type:: PyTime_t + + A timestamp or duration in nanoseconds, represented as a signed 64-bit + integer. + + The reference point for timestamps depends on the clock used. For example, + :c:func:`PyTime_Time` returns timestamps relative to the UNIX epoch. + + The supported range is around [-292.3 years; +292.3 years]. + Using the Unix epoch (January 1st, 1970) as reference, the supported date + range is around [1677-09-21; 2262-04-11]. + The exact limits are exposed as constants: + +.. c:var:: PyTime_t PyTime_MIN + + Minimum value of :c:type:`PyTime_t`. + +.. c:var:: PyTime_t PyTime_MAX + + Maximum value of :c:type:`PyTime_t`. + + +Clock Functions +--------------- + +The following functions take a pointer to a :c:expr:`PyTime_t` that they +set to the value of a particular clock. +Details of each clock are given in the documentation of the corresponding +Python function. + +The functions return ``0`` on success, or ``-1`` (with an exception set) +on failure. + +On integer overflow, they set the :c:data:`PyExc_OverflowError` exception and +set ``*result`` to the value clamped to the ``[PyTime_MIN; PyTime_MAX]`` +range. +(On current systems, integer overflows are likely caused by misconfigured +system time.) + +As any other C API (unless otherwise specified), the functions must be called +with the :term:`GIL` held. + +.. c:function:: int PyTime_Monotonic(PyTime_t *result) + + Read the monotonic clock. + See :func:`time.monotonic` for important details on this clock. + +.. c:function:: int PyTime_PerfCounter(PyTime_t *result) + + Read the performance counter. + See :func:`time.perf_counter` for important details on this clock. + +.. c:function:: int PyTime_Time(PyTime_t *result) + + Read the “wall clock†time. + See :func:`time.time` for details important on this clock. + + +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to hold the GIL. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, acquire the GIL and call the +regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require holding the GIL. + + +Conversion functions +-------------------- + +.. c:function:: double PyTime_AsSecondsDouble(PyTime_t t) + + Convert a timestamp to a number of seconds as a C :c:expr:`double`. + + The function cannot fail, but note that :c:expr:`double` has limited + accuracy for large values. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b3710560ebe7ac..52668d16b74436 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -59,6 +59,12 @@ Tuple Objects Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + The returned reference is borrowed from the tuple *p* + (that is: it is only valid as long as you hold a reference to *p*). + To get a :term:`strong reference`, use + :c:func:`Py_NewRef(PyTuple_GetItem(...)) ` + or :c:func:`PySequence_GetItem`. + .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -99,6 +105,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5aaa8147dd3176..0cae5c09505ebe 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -185,6 +185,21 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) + + Return the type's fully qualified name. Equivalent to + ``f"{type.__module__}.{type.__qualname__}"``, or ``type.__qualname__`` if + ``type.__module__`` is not a string or is equal to ``"builtins"``. + + .. versionadded:: 3.13 + +.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) + + Return the type's module name. Equivalent to getting the ``type.__module__`` + attribute. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 10c05beda7c66f..0091e084308245 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -343,13 +343,13 @@ slot typedefs | | :c:type:`PyTypeObject` * | | | | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`destructor` | void * | void | +| :c:type:`destructor` | :c:type:`PyObject` * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`freefunc` | void * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`traverseproc` | .. line-block:: | int | | | | | -| | void * | | +| | :c:type:`PyObject` * | | | | :c:type:`visitproc` | | | | void * | | +-----------------------------+-----------------------------+----------------------+ @@ -426,7 +426,7 @@ slot typedefs | | :c:type:`PyObject` * | | | | :c:type:`Py_buffer` * | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`inquiry` | void * | int | +| :c:type:`inquiry` | :c:type:`PyObject` * | int | +-----------------------------+-----------------------------+----------------------+ | :c:type:`unaryfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | @@ -883,6 +883,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. + **Default:** + + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericHash`. + .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -1030,7 +1034,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). + DECREF'ed). Heap types should also :ref:`support garbage collection ` + as they can form a reference cycle with their own module object. **Inheritance:** @@ -1323,8 +1328,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) To indicate that a class has changed call :c:func:`PyType_Modified` .. warning:: - This flag is present in header files, but is an internal feature and should - not be used. It will be removed in a future version of CPython + This flag is present in header files, but is not be used. + It will be removed in a future version of CPython .. c:member:: const char* PyTypeObject.tp_doc @@ -1376,7 +1381,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_VISIT(Py_TYPE(self)); It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditionnal:: + line must be conditional:: #if PY_VERSION_HEX >= 0x03090000 Py_VISIT(Py_TYPE(self)); @@ -1587,7 +1592,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) weak references to the type object itself. It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and - :c:member:`~PyTypeObject.tp_weaklist`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. **Inheritance:** @@ -1599,7 +1604,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Default:** If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value, to indicate that it is unsafe to use this field. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5ab9f1cab23ef8..246cf47df62e78 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -75,19 +75,19 @@ Python: The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: -.. c:function:: int PyUnicode_Check(PyObject *o) +.. c:function:: int PyUnicode_Check(PyObject *obj) - Return true if the object *o* is a Unicode object or an instance of a Unicode + Return true if the object *obj* is a Unicode object or an instance of a Unicode subtype. This function always succeeds. -.. c:function:: int PyUnicode_CheckExact(PyObject *o) +.. c:function:: int PyUnicode_CheckExact(PyObject *obj) - Return true if the object *o* is a Unicode object, but not an instance of a + Return true if the object *obj* is a Unicode object, but not an instance of a subtype. This function always succeeds. -.. c:function:: int PyUnicode_READY(PyObject *o) +.. c:function:: int PyUnicode_READY(PyObject *unicode) Returns ``0``. This API is kept only for backward compatibility. @@ -97,17 +97,17 @@ access to internal read-only data of Unicode objects: This API does nothing since Python 3.12. -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) +.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *unicode) - Return the length of the Unicode string, in code points. *o* has to be a + Return the length of the Unicode string, in code points. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) +.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *unicode) + Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *unicode) + Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *unicode) Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 integer types for direct character access. No checks are performed if the @@ -129,18 +129,18 @@ access to internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` has been removed. -.. c:function:: int PyUnicode_KIND(PyObject *o) +.. c:function:: int PyUnicode_KIND(PyObject *unicode) Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to + bytes per character this Unicode object uses to store its data. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: void* PyUnicode_DATA(PyObject *o) +.. c:function:: void* PyUnicode_DATA(PyObject *unicode) - Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -168,25 +168,25 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) +.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) - Read a character from a Unicode object *o*, which must be in the "canonical" + Read a character from a Unicode object *unicode*, which must be in the "canonical" representation. This is less efficient than :c:func:`PyUnicode_READ` if you do multiple consecutive reads. .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *unicode) Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is + based on *unicode*, which must be in the "canonical" representation. This is always an approximation but more efficient than iterating over the string. .. versionadded:: 3.3 -.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) +.. c:function:: int PyUnicode_IsIdentifier(PyObject *unicode) Return ``1`` if the string is a valid identifier according to the language definition, section :ref:`identifiers`. Return ``0`` otherwise. @@ -358,9 +358,9 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *str, Py_ssize_t size) - Create a Unicode object from the char buffer *u*. The bytes will be + Create a Unicode object from the char buffer *str*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new object. The return value might be a shared object, i.e. modification of the data is @@ -369,16 +369,16 @@ APIs: This function raises :exc:`SystemError` when: * *size* < 0, - * *u* is ``NULL`` and *size* > 0 + * *str* is ``NULL`` and *size* > 0 .. versionchanged:: 3.12 - *u* == ``NULL`` with *size* > 0 is not allowed anymore. + *str* == ``NULL`` with *size* > 0 is not allowed anymore. -.. c:function:: PyObject *PyUnicode_FromString(const char *u) +.. c:function:: PyObject *PyUnicode_FromString(const char *str) Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. + *str*. .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) @@ -518,6 +518,26 @@ APIs: - :c:expr:`PyObject*` - The result of calling :c:func:`PyObject_Repr`. + * - ``T`` + - :c:expr:`PyObject*` + - Get the fully qualified name of an object type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#T`` + - :c:expr:`PyObject*` + - Similar to ``T`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + + * - ``N`` + - :c:expr:`PyTypeObject*` + - Get the fully qualified name of a type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#N`` + - :c:expr:`PyTypeObject*` + - Similar to ``N`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes or :c:type:`wchar_t` @@ -553,6 +573,9 @@ APIs: In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. + .. versionchanged:: 3.13 + Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -646,29 +669,29 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ +.. c:function:: PyObject* PyUnicode_Substring(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t end) - Return a substring of *str*, from character index *start* (included) to + Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ +.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *unicode, Py_UCS4 *buffer, \ Py_ssize_t buflen, int copy_null) - Copy the string *u* into a UCS4 buffer, including a null character, if + Copy the string *unicode* into a UCS4 buffer, including a null character, if *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. + *unicode*). *buffer* is returned on success. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) +.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *unicode) - Copy the string *u* into a new UCS4 buffer that is allocated using + Copy the string *unicode* into a new UCS4 buffer that is allocated using :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -683,7 +706,7 @@ The current locale encoding can be used to decode text from the operating system. .. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ + Py_ssize_t length, \ const char *errors) Decode a string from UTF-8 on Android and VxWorks, or from the current @@ -788,7 +811,7 @@ conversion function: Accepts a :term:`path-like object`. -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *str, Py_ssize_t size) Decode a string from the :term:`filesystem encoding and error handler`. @@ -804,7 +827,7 @@ conversion function: handler>` is now used. -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) +.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *str) Decode a null-terminated string from the :term:`filesystem encoding and error handler`. @@ -841,20 +864,25 @@ wchar_t Support :c:type:`wchar_t` support for platforms which support it: -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *wstr, Py_ssize_t size) - Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. + Create a Unicode object from the :c:type:`wchar_t` buffer *wstr* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. + using :c:func:`!wcslen`. Return ``NULL`` on failure. -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) +.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *wstr, Py_ssize_t size) - Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most + Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` + copied or ``-1`` in case of an error. + + When *wstr* is ``NULL``, instead return the *size* that would be required + to store all of *unicode* including a terminating null. + + Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is required by the application. Also, note that the :c:expr:`wchar_t*` string @@ -915,10 +943,10 @@ Generic Codecs These are the generic codec APIs: -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_Decode(const char *str, Py_ssize_t size, \ const char *encoding, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s*. + Create a Unicode object by decoding *size* bytes of the encoded string *str*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by @@ -941,13 +969,13 @@ UTF-8 Codecs These are the UTF-8 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If @@ -971,8 +999,8 @@ These are the UTF-8 codec APIs: returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. + On error, set an exception, set *size* to ``-1`` (if it's not NULL) and + return ``NULL``. This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not @@ -1004,7 +1032,7 @@ UTF-32 Codecs These are the UTF-32 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the @@ -1031,7 +1059,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If @@ -1054,7 +1082,7 @@ UTF-16 Codecs These are the UTF-16 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the @@ -1082,7 +1110,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If @@ -1105,13 +1133,13 @@ UTF-7 Codecs These are the UTF-7 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If @@ -1126,11 +1154,11 @@ Unicode-Escape Codecs These are the "Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return ``NULL`` if an exception was raised by the codec. + string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) @@ -1146,11 +1174,11 @@ Raw-Unicode-Escape Codecs These are the "Raw Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return ``NULL`` if an exception was raised by the codec. + encoded string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) @@ -1167,10 +1195,10 @@ These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode ordinals and only these are accepted by the codecs during encoding. -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) @@ -1187,10 +1215,10 @@ These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other codes generate errors. -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) @@ -1211,10 +1239,10 @@ decode characters. The mapping objects provided must support the These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *str, Py_ssize_t length, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* + Create a Unicode object by decoding *size* bytes of the encoded string *str* using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. @@ -1241,7 +1269,7 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, PyObject *table, const char *errors) Translate a string by applying a character mapping table to it and return the resulting Unicode object. Return ``NULL`` if an exception was raised by the @@ -1266,13 +1294,13 @@ use the Win32 MBCS converters to implement the conversions. Note that MBCS (or DBCS) is a class of encodings, not just one. The target encoding is defined by the user settings on the machine running the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *str, Py_ssize_t size, const char *errors) - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. + Create a Unicode object by decoding *size* bytes of the MBCS encoded string *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If @@ -1318,7 +1346,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Concat two strings giving a new Unicode string. -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +.. c:function:: PyObject* PyUnicode_Split(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given @@ -1326,10 +1354,10 @@ They all return ``NULL`` or ``-1`` if an exception occurs. set. Separators are not included in the resulting list. -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) +.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *unicode, int keepends) Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the line break + CRLF is considered to be one line break. If *keepends* is ``0``, the Line break characters are not included in the resulting strings. @@ -1339,28 +1367,28 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Unicode string. -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end + Return ``1`` if *substr* matches ``unicode[start:end]`` at the given tail end (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), ``0`` otherwise. Return ``-1`` if an error occurred. -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of *substr* in ``str[start:end]`` using the given + Return the first position of *substr* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` indicates that an error occurred and an exception has been set. -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ +.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *unicode, Py_UCS4 ch, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of the character *ch* in ``str[start:end]`` using + Return the first position of the character *ch* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` @@ -1369,20 +1397,20 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.3 .. versionchanged:: 3.7 - *start* and *end* are now adjusted to behave like ``str[start:end]``. + *start* and *end* are now adjusted to behave like ``unicode[start:end]``. -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end) Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. + ``unicode[start:end]``. Return ``-1`` if an error occurred. -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ +.. c:function:: PyObject* PyUnicode_Replace(PyObject *unicode, PyObject *substr, \ PyObject *replstr, Py_ssize_t maxcount) - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and + Replace at most *maxcount* occurrences of *substr* in *unicode* with *replstr* and return the resulting Unicode object. *maxcount* == ``-1`` means replace all occurrences. @@ -1418,9 +1446,9 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.13 -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) +.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *unicode, const char *string) - Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *unicode*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1428,7 +1456,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function does not raise exceptions. -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) +.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) Rich compare two Unicode strings and return one of the following: @@ -1446,31 +1474,153 @@ They all return ``NULL`` or ``-1`` if an exception occurs. ``format % args``. -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) +.. c:function:: int PyUnicode_Contains(PyObject *unicode, PyObject *substr) - Check whether *element* is contained in *container* and return true or false + Check whether *substr* is contained in *unicode* and return true or false accordingly. - *element* has to coerce to a one element Unicode string. ``-1`` is returned + *substr* has to coerce to a one element Unicode string. ``-1`` is returned if there was an error. -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) +.. c:function:: void PyUnicode_InternInPlace(PyObject **p_unicode) - Intern the argument *\*string* in place. The argument must be the address of a + Intern the argument :c:expr:`*p_unicode` in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to + existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - *\*string* alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). (Clarification: even though there is a lot of talk about references, think of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) +.. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string object that has been interned, or a new ("owned") reference to an earlier interned string object with the same value. + +PyUnicodeWriter +^^^^^^^^^^^^^^^ + +The :c:type:`PyUnicodeWriter` API can be used to create a Python :class:`str` +object. + +.. versionadded:: 3.14 + +.. c:type:: PyUnicodeWriter + + A Unicode writer instance. + + The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on + success, or :c:func:`PyUnicodeWriter_Discard` on error. + +.. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) + + Create a Unicode writer instance. + + Set an exception and return ``NULL`` on error. + +.. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) + + Return the final Python :class:`str` object and destroy the writer instance. + + Set an exception and return ``NULL`` on error. + +.. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) + + Discard the internal Unicode buffer and destroy the writer instance. + +.. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) + + Write the single Unicode character *ch* into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Decode the string *str* from UTF-8 in strict mode and write the output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +.. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) + + Writer the wide string *str* into *writer*. + + *size* is a number of wide characters. If *size* is equal to ``-1``, call + ``wcslen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) + + Writer the UCS4 string *str* into *writer*. + + *size* is a number of UCS4 characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Str` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) + + Write the substring ``str[start:end]`` into *writer*. + + *str* must be Python :class:`str` object. *start* must be greater than or + equal to 0, and less than or equal to *end*. *end* must be less than or + equal to *str* length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) + + Similar to :c:func:`PyUnicode_FromFormat`, but write the output directly into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed) + + Decode the string *str* from UTF-8 with *errors* error handler and write the + output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *errors* is an error handler name, such as ``"replace"``. If *errors* is + ``NULL``, use the strict error handler. + + If *consumed* is not ``NULL``, set *\*consumed* to the number of decoded + bytes on success. + If *consumed* is ``NULL``, treat trailing incomplete UTF-8 byte sequences + as an error. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_WriteUTF8`. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index ccbf14e1850f68..9d0abf440f791d 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -17,6 +17,8 @@ and parsing function arguments and constructing Python values from C values. marshal.rst arg.rst conversion.rst + hash.rst reflection.rst codec.rst + time.rst perfmaps.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 324518c035096b..67167444d0a685 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -322,7 +322,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_eval_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for isolated expressions; for use with :c:func:`Py_CompileString`. @@ -330,7 +330,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_file_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for sequences of statements as read from a file or other source; for use with :c:func:`Py_CompileString`. This is @@ -339,7 +339,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_single_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for a single statement; for use with :c:func:`Py_CompileString`. This is the symbol used for the interactive diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..8f233e16fb17cf 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -96,3 +96,19 @@ as much as it can. This iterates through the weak references for *object* and calls callbacks for those references which have one. It returns when all callbacks have been attempted. + + +.. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object) + + Clears the weakrefs for *object* without calling the callbacks. + + This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler + for types with finalizers (i.e., :meth:`~object.__del__`). The handler for + those objects first calls :c:func:`PyObject_ClearWeakRefs` to clear weakrefs + and call their callbacks, then the finalizer, and finally this function to + clear any weakrefs that may have been created by the finalizer. + + In most circumstances, it's more appropriate to use + :c:func:`PyObject_ClearWeakRefs` to clear weakrefs instead of this function. + + .. versionadded:: 3.13 diff --git a/Doc/conf.py b/Doc/conf.py index c92ea60ee07094..29b1b2db32718b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,10 +6,14 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import sys, os, time +import os +import sys +import time sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) +from pyspecific import SOURCE_URI + # General configuration # --------------------- @@ -22,9 +26,16 @@ 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', ] -# Skip if downstream redistributors haven't installed it +# Skip if downstream redistributors haven't installed them +try: + import notfound.extension +except ImportError: + pass +else: + extensions.append('notfound.extension') try: import sphinxext.opengraph except ImportError: @@ -49,13 +60,19 @@ # General substitutions. project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') +copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. import patchlevel version, release = patchlevel.get_version_info() +rst_epilog = f""" +.. |python_version_literal| replace:: ``Python {version}`` +.. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` +""" + # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: today = '' @@ -68,6 +85,10 @@ # Minimum version of sphinx required needs_sphinx = '4.2' +# Create table of contents entries for domain objects (e.g. functions, classes, +# attributes, etc.). Default is True. +toc_object_entries = False + # Ignore any .rst files in the includes/ directory; # they're embedded in pages but not rendered individually. # Ignore any .rst files in the venv/ directory. @@ -79,25 +100,38 @@ nitpick_ignore = [ # Standard C functions ('c:func', 'calloc'), + ('c:func', 'ctime'), ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), + ('c:func', 'flock'), ('c:func', 'fork'), ('c:func', 'free'), + ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), + ('c:func', 'grantpt'), + ('c:func', 'ioctl'), + ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), ('c:func', 'malloc'), + ('c:func', 'mktime'), + ('c:func', 'posix_openpt'), ('c:func', 'printf'), + ('c:func', 'ptsname'), + ('c:func', 'ptsname_r'), ('c:func', 'realloc'), ('c:func', 'snprintf'), ('c:func', 'sprintf'), ('c:func', 'stat'), + ('c:func', 'strftime'), ('c:func', 'system'), ('c:func', 'time'), + ('c:func', 'unlockpt'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), @@ -113,11 +147,14 @@ ('c:type', 'wchar_t'), ('c:type', '__int64'), ('c:type', 'unsigned __int64'), + ('c:type', 'double'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), ('c:struct', 'stat'), ('c:struct', 'statvfs'), + ('c:struct', 'timeval'), + ('c:struct', 'timespec'), # Standard C macros ('c:macro', 'LLONG_MAX'), ('c:macro', 'LLONG_MIN'), @@ -157,6 +194,13 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), + # Deprecated function that was never documented: + ('py:func', 'getargspec'), + ('py:func', 'inspect.getargspec'), + # Undocumented modules that users shouldn't have to worry about + # (implementation details of `os.path`): + ('py:mod', 'ntpath'), + ('py:mod', 'posixpath'), ] # Temporary undocumented names. @@ -228,18 +272,28 @@ ('c:data', 'PyExc_UnicodeWarning'), ('c:data', 'PyExc_UserWarning'), ('c:data', 'PyExc_Warning'), + # Undocumented public C macros + ('c:macro', 'Py_BUILD_ASSERT'), + ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), + # Attributes/methods/etc. that definitely should be documented better, + # but are deferred for now: + ('py:attr', '__annotations__'), + ('py:meth', '__missing__'), + ('py:attr', '__wrapped__'), + ('py:attr', 'decimal.Context.clamp'), + ('py:meth', 'index'), # list.index, tuple.index, etc. ] -# gh-106948: Copy standard C types declared in the "c:type" domain to the -# "c:identifier" domain, since "c:function" markup looks for types in the -# "c:identifier" domain. Use list() to not iterate on items which are being -# added +# gh-106948: Copy standard C types declared in the "c:type" domain and C +# structures declared in the "c:struct" domain to the "c:identifier" domain, +# since "c:function" markup looks for types in the "c:identifier" domain. Use +# list() to not iterate on items which are being added for role, name in list(nitpick_ignore): - if role == 'c:type': + if role in ('c:type', 'c:struct'): nitpick_ignore.append(('c:identifier', name)) del role, name @@ -248,8 +302,8 @@ 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], } -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' +# Avoid a warning with Sphinx >= 4.0 +root_doc = 'contents' # Allow translation of index directives gettext_additional_targets = [ @@ -269,6 +323,9 @@ 'root_include_title': False # We use the version switcher instead. } +if os.getenv("READTHEDOCS"): + html_theme_options["hosted_on"] = 'Read the Docs' + # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 if any('htmlhelp' in arg for arg in sys.argv): @@ -277,7 +334,7 @@ print("It may be removed in the future\n") # Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release +html_short_title = f'{release} Documentation' # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) @@ -285,12 +342,12 @@ html_context = { "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") + "pr_id": os.getenv("READTHEDOCS_VERSION"), + "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +# This 'Last updated on:' timestamp is inserted at the bottom of every page. +html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) # Path to find HTML templates. templates_path = ['tools/templates'] @@ -321,18 +378,17 @@ # Split the index html_split_index = True +# Split pot files one per reST file +gettext_compact = False # Options for LaTeX output # ------------------------ latex_engine = 'xelatex' -# Get LaTeX to handle Unicode correctly latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' + # For the LaTeX preamble. + 'preamble': r''' \authoraddress{ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} @@ -340,13 +396,12 @@ \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' +''', + # The paper size ('letter' or 'a4'). + 'papersize': 'a4', + # The font size ('10pt', '11pt' or '12pt'). + 'pointsize': '10pt', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -386,6 +441,10 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +# index pages are not valid xhtml +# https://github.com/sphinx-doc/sphinx/issues/12359 +epub_use_index = False + # Options for the coverage checker # -------------------------------- @@ -409,9 +468,9 @@ # Regexes to find C items in the source files. coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), + 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)', + 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)', + 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]', } # The coverage checker will ignore all C items whose names match these regexes @@ -468,6 +527,19 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] +# Options for sphinx.ext.extlinks +# ------------------------------- + +# This config is a dictionary of external sites, +# mapping unique short aliases to a base URL and a prefix. +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +extlinks = { + "cve": ("https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-%s", "CVE-%s"), + "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"), + "pypi": ("https://pypi.org/project/%s/", "%s"), + "source": (SOURCE_URI, "%s"), +} +extlinks_detect_hardcoded_links = True # Options for extensions # ---------------------- diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 9b71683155eebe..8629ed1fc38009 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ef9ac1617a284b..a7d06e076a1b55 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -402,6 +402,21 @@ PyContextVar_Reset:int::: PyContextVar_Reset:PyObject*:var:0: PyContextVar_Reset:PyObject*:token:-1: +PyCFunction_New:PyObject*::+1: +PyCFunction_New:PyMethodDef*:ml:: +PyCFunction_New:PyObject*:self:+1: + +PyCFunction_NewEx:PyObject*::+1: +PyCFunction_NewEx:PyMethodDef*:ml:: +PyCFunction_NewEx:PyObject*:self:+1: +PyCFunction_NewEx:PyObject*:module:+1: + +PyCMethod_New:PyObject*::+1: +PyCMethod_New:PyMethodDef*:ml:: +PyCMethod_New:PyObject*:self:+1: +PyCMethod_New:PyObject*:module:+1: +PyCMethod_New:PyObject*:cls:+1: + PyDate_Check:int::: PyDate_Check:PyObject*:ob:0: @@ -775,6 +790,12 @@ PyEval_GetGlobals:PyObject*::0: PyEval_GetFrame:PyObject*::0: +PyEval_GetFrameBuiltins:PyObject*::+1: + +PyEval_GetFrameLocals:PyObject*::+1: + +PyEval_GetFrameGlobals:PyObject*::+1: + PyEval_GetFuncDesc:const char*::: PyEval_GetFuncDesc:PyObject*:func:0: @@ -901,6 +922,32 @@ PyFloat_FromString:PyObject*:str:0: PyFloat_GetInfo:PyObject*::+1: PyFloat_GetInfo::void:: +PyFrame_GetBack:PyObject*::+1: +PyFrame_GetBack:PyFrameObject*:frame:0: + +PyFrame_GetBuiltins:PyObject*::+1: +PyFrame_GetBuiltins:PyFrameObject*:frame:0: + +PyFrame_GetCode:PyObject*::+1: +PyFrame_GetCode:PyFrameObject*:frame:0: + +PyFrame_GetGenerator:PyObject*::+1: +PyFrame_GetGenerator:PyFrameObject*:frame:0: + +PyFrame_GetGlobals:PyObject*::+1: +PyFrame_GetGlobals:PyFrameObject*:frame:0: + +PyFrame_GetLocals:PyObject*::+1: +PyFrame_GetLocals:PyFrameObject*:frame:0: + +PyFrame_GetVar:PyObject*::+1: +PyFrame_GetVar:PyFrameObject*:frame:0: +PyFrame_GetVar:PyObject*:name:0: + +PyFrame_GetVarString:PyObject*::+1: +PyFrame_GetVarString:PyFrameObject*:frame:0: +PyFrame_GetVarString:const char*:name:: + PyFrozenSet_Check:int::: PyFrozenSet_Check:PyObject*:p:0: @@ -1118,6 +1165,10 @@ PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: PyList_GetItem:Py_ssize_t:index:: +PyList_GetItemRef:PyObject*::+1: +PyList_GetItemRef:PyObject*:list:0: +PyList_GetItemRef:Py_ssize_t:index:: + PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: PyList_GetSlice:Py_ssize_t:low:: @@ -1589,6 +1640,13 @@ PyObject_Call:PyObject*:callable_object:0: PyObject_Call:PyObject*:args:0: PyObject_Call:PyObject*:kw:0: +PyObject_CallNoArgs:PyObject*::+1: +PyObject_CallNoArgs:PyObject*:callable_object:0: + +PyObject_CallOneArg:PyObject*::+1: +PyObject_CallOneArg:PyObject*:callable_object:0: +PyObject_CallOneArg:PyObject*:arg:0: + PyObject_CallFunction:PyObject*::+1: PyObject_CallFunction:PyObject*:callable_object:0: PyObject_CallFunction:const char*:format:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 6ec9c907254b04..1f7af436a4150b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -42,6 +42,8 @@ function,PyBytes_Repr,3.2,, function,PyBytes_Size,3.2,, var,PyBytes_Type,3.2,, type,PyCFunction,3.2,, +type,PyCFunctionFast,3.13,, +type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, function,PyCFunction_GetFlags,3.2,, function,PyCFunction_GetFunction,3.2,, @@ -186,10 +188,14 @@ function,PyEval_EvalFrame,3.2,, function,PyEval_EvalFrameEx,3.2,, function,PyEval_GetBuiltins,3.2,, function,PyEval_GetFrame,3.2,, +function,PyEval_GetFrameBuiltins,3.13,, +function,PyEval_GetFrameGlobals,3.13,, +function,PyEval_GetFrameLocals,3.13,, function,PyEval_GetFuncDesc,3.2,, function,PyEval_GetFuncName,3.2,, function,PyEval_GetGlobals,3.2,, function,PyEval_GetLocals,3.2,, +function,PyEval_InitThreads,3.2,, function,PyEval_ReleaseThread,3.2,, function,PyEval_RestoreThread,3.2,, function,PyEval_SaveThread,3.2,, @@ -335,6 +341,7 @@ var,PyListRevIter_Type,3.2,, function,PyList_Append,3.2,, function,PyList_AsTuple,3.2,, function,PyList_GetItem,3.2,, +function,PyList_GetItemRef,3.13,, function,PyList_GetSlice,3.2,, function,PyList_Insert,3.2,, function,PyList_New,3.2,, @@ -388,6 +395,10 @@ function,PyMapping_Values,3.2,, function,PyMem_Calloc,3.7,, function,PyMem_Free,3.2,, function,PyMem_Malloc,3.2,, +function,PyMem_RawCalloc,3.13,, +function,PyMem_RawFree,3.13,, +function,PyMem_RawMalloc,3.13,, +function,PyMem_RawRealloc,3.13,, function,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi var,PyMemberDescr_Type,3.2,, @@ -602,11 +613,15 @@ function,PyStructSequence_NewType,3.2,, function,PyStructSequence_SetItem,3.2,, var,PyStructSequence_UnnamedField,3.11,, var,PySuper_Type,3.2,, +function,PySys_Audit,3.13,, +function,PySys_AuditTuple,3.13,, function,PySys_FormatStderr,3.2,, function,PySys_FormatStdout,3.2,, function,PySys_GetObject,3.2,, function,PySys_GetXOptions,3.7,, function,PySys_ResetWarnOptions,3.2,, +function,PySys_SetArgv,3.2,, +function,PySys_SetArgvEx,3.2,, function,PySys_SetObject,3.2,, function,PySys_WriteStderr,3.2,, function,PySys_WriteStdout,3.2,, @@ -667,7 +682,10 @@ function,PyType_FromSpecWithBases,3.3,, function,PyType_GenericAlloc,3.2,, function,PyType_GenericNew,3.2,, function,PyType_GetFlags,3.2,, +function,PyType_GetFullyQualifiedName,3.13,, function,PyType_GetModule,3.10,, +function,PyType_GetModuleByDef,3.13,, +function,PyType_GetModuleName,3.13,, function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, function,PyType_GetQualName,3.11,, @@ -826,6 +844,8 @@ function,Py_GenericAlias,3.9,, var,Py_GenericAliasType,3.9,, function,Py_GetBuildInfo,3.2,, function,Py_GetCompiler,3.2,, +function,Py_GetConstant,3.13,, +function,Py_GetConstantBorrowed,3.13,, function,Py_GetCopyright,3.2,, function,Py_GetExecPrefix,3.2,, function,Py_GetPath,3.2,, @@ -853,7 +873,10 @@ function,Py_NewInterpreter,3.2,, function,Py_NewRef,3.10,, function,Py_ReprEnter,3.2,, function,Py_ReprLeave,3.2,, +function,Py_SetProgramName,3.2,, +function,Py_SetPythonHome,3.2,, function,Py_SetRecursionLimit,3.2,, +function,Py_TYPE,3.14,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, var,Py_UTF8Mode,3.8,, diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 1ee7f28b2ba220..b0493bed75b151 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -547,7 +547,7 @@ reference count of an object and are safe in the presence of ``NULL`` pointers (but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. -.. index:: single: PyObject_CallObject() +.. index:: single: PyObject_CallObject (C function) Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to @@ -638,7 +638,7 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple() +.. index:: single: PyArg_ParseTuple (C function) The :c:func:`PyArg_ParseTuple` function is declared as follows:: @@ -730,12 +730,12 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== -.. index:: single: PyArg_ParseTupleAndKeywords() +.. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); + const char *format, char * const *kwlist, ...); The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of @@ -868,7 +868,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 9f166eb8a4c3ff..fd05c82b41629a 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -89,8 +89,8 @@ If your type supports garbage collection, the destructor should call } .. index:: - single: PyErr_Fetch() - single: PyErr_Restore() + single: PyErr_Fetch (C function) + single: PyErr_Restore (C function) One important requirement of the deallocator function is that it leaves any pending exceptions alone. This is important since deallocators are frequently @@ -296,7 +296,7 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`__doc__` attribute. +class object, and get the doc string using its :attr:`!__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. @@ -545,7 +545,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ae02c443e5938b..c8beb64e39bc1a 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- @@ -451,7 +456,7 @@ on the key and a per-process seed; for example, ``'Python'`` could hash to to ``1142331976``. The hash code is then used to calculate a location in an internal array where the value will be stored. Assuming that you're storing keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. +constant time -- *O*\ (1), in Big-O notation -- to retrieve a key. Why must dictionary keys be immutable? diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..1cff2c4091df06 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -50,7 +50,7 @@ to learn Python's C API. If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG <https://www.swig.org>`_. `SIP -<https://riverbankcomputing.com/software/sip/intro>`__, `CXX +<https://github.com/Python-SIP/sip>`__, `CXX <https://cxx.sourceforge.net/>`_ `Boost <https://www.boost.org/libs/python/doc/index.html>`_, or `Weave <https://github.com/scipy/weave>`_ are also diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 8727332594bda6..eb859c5d5992da 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index <https://pypi.org>`_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -133,8 +135,6 @@ Python versions are numbered "A.B.C" or "A.B": changes. * *C* is the micro version number -- it is incremented for each bugfix release. -See :pep:`6` for more information about bugfix releases. - Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; @@ -157,7 +157,11 @@ unreleased versions, built directly from the CPython development repository. In practice, after a final minor release is made, the version is incremented to the next minor version, which becomes the "a0" version, e.g. "2.4a0". -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and +See the `Developer's Guide +<https://devguide.python.org/developer-workflow/development-cycle/>`__ +for more information about the development cycle, and +:pep:`387` to learn more about Python's backward compatibility policy. See also +the documentation for :data:`sys.version`, :data:`sys.hexversion`, and :data:`sys.version_info`. @@ -181,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 476a43d9c288f1..a2900952d7bef6 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -405,22 +405,37 @@ lists. When in doubt, use a mutex! Can't we get rid of the Global Interpreter Lock? ------------------------------------------------ -.. XXX link to dbeazley's talk about GIL? - The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive +With the approval of :pep:`703` work is now underway to remove the GIL from the +CPython implementation of Python. Initially it will be implemented as an +optional compiler flag when building the interpreter, and so separate +builds will be available with and without the GIL. Long-term, the hope is +to settle on a single build, once the performance implications of removing the +GIL are fully understood. Python 3.13 is likely to be the first release +containing this work, although it may not be completely functional in this +release. + +The current work to remove the GIL is based on a +`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_ +by Sam Gross. +Prior to that, +in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment +with fine-grained locking. Adam Olsen did a similar experiment in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread +project. Unfortunately, both of these earlier experiments exhibited a sharp +drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. +necessary to compensate for the removal of the GIL. The Python 3.9 fork +is the first attempt at removing the GIL with an acceptable performance +impact. -This doesn't mean that you can't make good use of Python on multi-CPU machines! +The presence of the GIL in current Python releases +doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The :class:`~concurrent.futures.ProcessPoolExecutor` class in the new @@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` already do this. -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? +An alternative approach to reducing the impact of the GIL is +to make the GIL a per-interpreter-state lock rather than truly global. +This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is +available in the C API. A Python interface to it is expected in Python 3.13. +The main limitation to it at the moment is likely to be 3rd party extension +modules, since these must be written with multiple interpreters in mind in +order to be usable, so many older extension modules will not be usable. Input and Output @@ -535,91 +541,12 @@ Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need use ``p.read(n)``. -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - <https://pypi.org/project/pexpect/>`_. - - How do I access the serial (RS232) port? ---------------------------------------- For Win32, OSX, Linux, BSD, Jython, IronPython: - https://pypi.org/project/pyserial/ + :pypi:`pyserial` For Unix, see a Usenet post by Mitch Chapman: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 0a88c5f6384f2b..61fbd1bb92ada6 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1741,11 +1741,31 @@ but effective way to define class private variables. Any identifier of the form is textually replaced with ``_classname__spam``, where ``classname`` is the current class name with any leading underscores stripped. -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. +The identifier can be used unchanged within the class, but to access it outside +the class, the mangled name must be used: +.. code-block:: python + + class A: + def __one(self): + return 1 + def two(self): + return 2 * self.__one() + + class B(A): + def three(self): + return 3 * self._A__one() + + four = 4 * A()._A__one() + +In particular, this does not guarantee privacy since an outside user can still +deliberately access the private attribute; many Python programmers never bother +to use private variable names at all. + +.. seealso:: + + The :ref:`private name mangling specifications <private-name-mangling>` + for details and special cases. My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4cd05b0cf019d..281dde30dc78ed 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -151,9 +152,9 @@ Glossary A :term:`file object` able to read and write :term:`bytes-like objects <bytes-like object>`. Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer <sys.stdin>`, + :data:`sys.stdout.buffer <sys.stdout>`, and instances of + :class:`io.BytesIO` and :class:`gzip.GzipFile`. See also :term:`text file` for a file object able to read and write :class:`str` objects. @@ -239,7 +240,7 @@ Glossary context manager An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`__enter__` and :meth:`__exit__` methods. + statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. See :pep:`343`. context variable @@ -304,8 +305,9 @@ Glossary :ref:`class definitions <class>` for more about decorators. descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special + Any object which defines the methods :meth:`~object.__get__`, + :meth:`~object.__set__`, or :meth:`~object.__delete__`. + When a class attribute is a descriptor, its special binding behavior is triggered upon attribute lookup. Normally, using *a.b* to get, set or delete an attribute looks up the object named *b* in the class dictionary for *a*, but if *b* is a descriptor, the respective @@ -319,7 +321,8 @@ Glossary dictionary An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. + keys can be any object with :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods. Called a hash in Perl. dictionary comprehension @@ -339,7 +342,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute + recognized by the compiler and put into the :attr:`!__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -383,7 +386,7 @@ Glossary file object An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending + :meth:`!read` or :meth:`!write`) to an underlying resource. Depending on the way it was created, a file object can mediate access to a real on-disk file or to another type of storage or communication device (for example standard input/output, in-memory buffers, sockets, pipes, @@ -422,11 +425,11 @@ Glossary An object that tries to find the :term:`loader` for a module that is being imported. - Since Python 3.3, there are two types of finder: :term:`meta path finders + There are two types of finder: :term:`meta path finders <meta path finder>` for use with :data:`sys.meta_path`, and :term:`path entry finders <path entry finder>` for use with :data:`sys.path_hooks`. - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + See :ref:`importsystem` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor @@ -435,6 +438,12 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments <argument>` which may be used in @@ -502,7 +511,7 @@ Glossary .. index:: single: generator expression generator expression - An expression that returns an iterator. It looks like a normal expression + An :term:`expression` that returns an :term:`iterator`. It looks like a normal expression followed by a :keyword:`!for` clause defining a loop variable, range, and an optional :keyword:`!if` clause. The combined expression generates values for an enclosing function:: @@ -545,12 +554,12 @@ Glossary tasks such as compression or hashing. Also, the GIL is always released when doing I/O. - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. - + As of Python 3.13, the GIL can be disabled using the :option:`--disable-gil` + build configuration. After building Python with this option, code must be + run with :option:`-X gil 0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 <PYTHON_GIL>` + environment variable. This feature enables improved performance for + multi-threaded applications and makes it easier to use multi-core CPUs + efficiently. For more details, see :pep:`703`. hash-based pyc A bytecode cache file that uses the hash rather than the last-modified @@ -559,8 +568,9 @@ Glossary hashable An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which + its lifetime (it needs a :meth:`~object.__hash__` method), and can be + compared to other objects (it needs an :meth:`~object.__eq__` method). + Hashable objects which compare equal must have the same hash value. Hashability makes an object usable as a dictionary key and a set member, @@ -579,6 +589,16 @@ Glossary :ref:`idle` is a basic editor and interpreter environment which ships with the standard distribution of Python. + immortal + If an object is immortal, its reference count is never modified, and + therefore it is never deallocated. + + Built-in strings and singletons are immortal objects. For example, + :const:`True` and :const:`None` singletons are immortal. + + See `PEP 683 – Immortal Objects, Using a Fixed Refcount + <https://peps.python.org/pep-0683/>`_ for more information. + immutable An object with a fixed value. Immutable objects include numbers, strings and tuples. Such an object cannot be altered. A new object has to @@ -607,7 +627,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -636,7 +657,8 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects <file object>`, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`__getitem__` method + with an :meth:`~iterator.__iter__` method or with a + :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. Iterables can be @@ -645,7 +667,7 @@ Glossary as an argument to the built-in function :func:`iter`, it returns an iterator for the object. This iterator is good for one pass over the set of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` + :func:`iter` or deal with iterator objects yourself. The :keyword:`for` statement does that automatically for you, creating a temporary unnamed variable to hold the iterator for the duration of the loop. See also :term:`iterator`, :term:`sequence`, and :term:`generator`. @@ -656,8 +678,8 @@ Glossary :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator + :meth:`!__next__` method just raise :exc:`StopIteration` again. Iterators + are required to have an :meth:`~iterator.__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most places where other iterables are accepted. One notable exception is code which attempts multiple iteration passes. A container object (such as a @@ -671,7 +693,10 @@ Glossary .. impl-detail:: CPython does not consistently apply the requirement that an iterator - define :meth:`__iter__`. + define :meth:`~iterator.__iter__`. + And also please note that the free-threading CPython does not guarantee + the thread-safety of iterator operations. + key function A key function or collation function is a callable that returns a value @@ -713,22 +738,10 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) <locale.setlocale>`. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. - - See also the :term:`filesystem encoding and error handler`. - list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to - elements is O(1). + elements is *O*\ (1). list comprehension A compact way to process all or part of the elements in a sequence and @@ -744,6 +757,18 @@ Glossary :term:`finder`. See :pep:`302` for details and :class:`importlib.abc.Loader` for an :term:`abstract base class`. + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + :func:`locale.setlocale(locale.LC_CTYPE, new_locale) <locale.setlocale>`. + + On Windows, it is the ANSI code page (ex: ``"cp1252"``). + + On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. + + :func:`locale.getencoding` can be used to get the locale encoding. + + See also the :term:`filesystem encoding and error handler`. + magic method .. index:: pair: magic; method @@ -786,8 +811,7 @@ Glossary method resolution order Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - <https://www.python.org/download/releases/2.3/mro/>`_ for details of the + for a member during lookup. See :ref:`python_2.3_mro` for details of the algorithm used by the Python interpreter since the 2.3 release. module @@ -827,10 +851,11 @@ Glossary Some named tuples are built-in types (such as the above examples). Alternatively, a named tuple can be created from a regular class definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. + fields. Such a class can be written by hand, or it can be created by + inheriting :class:`typing.NamedTuple`, or with the factory function + :func:`collections.namedtuple`. The latter techniques also add some + extra methods that may not be found in hand-written or built-in named + tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -865,13 +890,23 @@ Glossary Old name for the flavor of classes now used for all class objects. In earlier Python versions, only new-style classes could use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. + properties, :meth:`~object.__getattribute__`, class methods, and static + methods. object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -945,7 +980,7 @@ Glossary finders implement. path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path + A callable on the :data:`sys.path_hooks` list which returns a :term:`path entry finder` if it knows how to find modules on a specific :term:`path entry`. @@ -1056,7 +1091,7 @@ Glossary reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are - "immortal" and have reference counts that are never modified, and + :term:`immortal` and have reference counts that are never modified, and therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the @@ -1069,6 +1104,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though @@ -1078,21 +1117,23 @@ Glossary sequence An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. + indices via the :meth:`~object.__getitem__` special method and defines a + :meth:`~object.__len__` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`__getitem__` and :meth:`__len__`, but is considered a + supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary :term:`immutable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded + :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding + :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and + :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. + :func:`~abc.ABCMeta.register`. For more documentation on sequence + methods generally, see + :ref:`Common Sequence Operations <typesseq-common>`. set comprehension A compact way to process all or part of the elements in an iterable and @@ -1138,6 +1179,11 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. + static type checker + An external tool that reads Python code and analyzes it, looking for + issues such as incorrect types. See also :term:`type hints <type hint>` + and the :mod:`typing` module. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1214,8 +1260,8 @@ Glossary attribute, or a function parameter or return value. Type hints are optional and are not enforced by Python but - they are useful to static type analysis tools, and aid IDEs with code - completion and refactoring. + they are useful to :term:`static type checkers <static type checker>`. + They can also aid IDEs with code completion and refactoring. Type hints of global variables, class attributes, and functions, but not local variables, can be accessed using diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 1134686c947d66..be8c7e6c827f57 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -153,7 +153,8 @@ on an arbitrary object ``o``: unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. + :attr:`o.__globals__ <function.__globals__>` as the globals when calling + :func:`eval`. However, not all string values used as annotations can be successfully turned into Python values by :func:`eval`. diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 4828e2fa29bd24..f9ad81e38f8dc3 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -43,7 +43,7 @@ appearance---and the curses library will figure out what control codes need to be sent to the terminal to produce the right output. curses doesn't provide many user-interface concepts such as buttons, checkboxes, or dialogs; if you need such features, consider a user interface library such as -`Urwid <https://pypi.org/project/urwid/>`_. +:pypi:`Urwid`. The curses library was originally written for BSD Unix; the later System V versions of Unix from AT&T added many enhancements and new functions. BSD curses @@ -56,8 +56,7 @@ versions of curses carried by some proprietary Unixes may not support everything, though. The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -<https://pypi.org/project/UniCurses>`_ is available. +module. A ported version called :pypi:`UniCurses` is available. The Python curses module @@ -429,8 +428,7 @@ User Input The C curses library offers only very simple input mechanisms. Python's :mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid <https://pypi.org/project/urwid/>`_ have more extensive -collections of widgets.) +such as :pypi:`Urwid` have more extensive collections of widgets.) There are two methods for getting input from a window: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 1d9424cb735a46..b29488be39a0a3 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1,8 +1,8 @@ .. _descriptorhowto: -====================== -Descriptor HowTo Guide -====================== +================ +Descriptor Guide +================ :Author: Raymond Hettinger :Contact: <python at rcn dot com> @@ -521,11 +521,11 @@ everyday Python programs. Descriptor protocol ------------------- -``descr.__get__(self, obj, type=None) -> value`` +``descr.__get__(self, obj, type=None)`` -``descr.__set__(self, obj, value) -> None`` +``descr.__set__(self, obj, value)`` -``descr.__delete__(self, obj) -> None`` +``descr.__delete__(self, obj)`` That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up @@ -787,7 +787,7 @@ Invocation from super --------------------- The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. +object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``B`` immediately following ``A`` and then returns @@ -943,6 +943,10 @@ it can be updated: >>> Movie('Star Wars').director 'J.J. Abrams' +.. testcleanup:: + + conn.close() + Pure Python Equivalents ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1000,26 +1004,43 @@ here is a pure Python equivalent: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc - self._name = '' + self._name = None def __set_name__(self, owner, name): self._name = name + @property + def __name__(self): + return self._name if self._name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self._name = value + def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: - raise AttributeError(f"property '{self._name}' has no getter") + raise AttributeError( + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no getter' + ) return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError(f"property '{self._name}' has no setter") + raise AttributeError( + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no setter' + ) self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError(f"property '{self._name}' has no deleter") + raise AttributeError( + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no deleter' + ) self.fdel(obj) def getter(self, fget): @@ -1050,6 +1071,11 @@ here is a pure Python equivalent: def delx(self): del self.__x x = Property(getx, setx, delx, "I'm the 'x' property.") + no_getter = Property(None, setx, delx, "I'm the 'x' property.") + no_setter = Property(getx, None, delx, "I'm the 'x' property.") + no_deleter = Property(getx, setx, None, "I'm the 'x' property.") + no_doc = Property(getx, setx, delx, None) + # Now do it again but use the decorator style @@ -1088,6 +1114,32 @@ here is a pure Python equivalent: >>> hasattr(ccc, 'x') False + >>> cc = CC() + >>> cc.x = 33 + >>> try: + ... cc.no_getter + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_getter' of 'CC' object has no getter" + + >>> try: + ... cc.no_setter = 33 + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_setter' of 'CC' object has no setter" + + >>> try: + ... del cc.no_deleter + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_deleter' of 'CC' object has no deleter" + + >>> CC.no_doc.__doc__ is None + True + The :func:`property` builtin helps whenever a user interface has granted attribute access and then subsequent changes require the intervention of a method. @@ -1141,6 +1193,20 @@ roughly equivalent to: obj = self.__self__ return func(obj, *args, **kwargs) + def __getattribute__(self, name): + "Emulate method_getset() in Objects/classobject.c" + if name == '__doc__': + return self.__func__.__doc__ + return object.__getattribute__(self, name) + + def __getattr__(self, name): + "Emulate method_getattro() in Objects/classobject.c" + return getattr(self.__func__, name) + + def __get__(self, obj, objtype=None): + "Emulate method_descr_get() in Objects/classobject.c" + return self + To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods @@ -1163,8 +1229,20 @@ descriptor works in practice: .. testcode:: class D: - def f(self, x): - return x + def f(self): + return self + + class D2: + pass + +.. doctest:: + :hide: + + >>> d = D() + >>> d2 = D2() + >>> d2.f = d.f.__get__(d2, D2) + >>> d2.f() is d + True The function has a :term:`qualified name` attribute to support introspection: @@ -1199,7 +1277,7 @@ instance:: <function D.f at 0x00C45070> >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> + <__main__.D object at 0x00B18C90> If you have ever wondered where *self* comes from in regular methods or where *cls* comes from in class methods, this is it! @@ -1291,7 +1369,8 @@ Using the non-data descriptor protocol, a pure Python version of The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``. +function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, +:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. .. testcode:: :hide: @@ -1420,10 +1499,6 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, cls=None): if cls is None: cls = type(obj) - if hasattr(type(self.f), '__get__'): - # This code path was added in Python 3.9 - # and was deprecated in Python 3.11. - return self.f.__get__(cls, cls) return MethodType(self.f, cls) .. testcode:: @@ -1436,11 +1511,6 @@ Using the non-data descriptor protocol, a pure Python version of "Class method that returns a tuple" return (cls.__name__, x, y) - @ClassMethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - .. doctest:: :hide: @@ -1453,10 +1523,6 @@ Using the non-data descriptor protocol, a pure Python version of >>> t.cm(11, 22) ('T', 11, 22) - # Check the alternate path for chained descriptors - >>> T.__doc__ - "A doc for 'T'" - # Verify that T uses our emulation >>> type(vars(T)['cm']).__name__ 'ClassMethod' @@ -1481,29 +1547,12 @@ Using the non-data descriptor protocol, a pure Python version of ('T', 11, 22) -The code path for ``hasattr(type(self.f), '__get__')`` was added in -Python 3.9 and makes it possible for :func:`classmethod` to support -chained decorators. For example, a classmethod and property could be -chained together. In Python 3.11, this functionality was deprecated. - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look -like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``, -and ``__annotations__``. +like the wrapped function: :attr:`~function.__name__`, +:attr:`~function.__qualname__`, :attr:`~function.__doc__`, +and :attr:`~function.__annotations__`. Member objects and __slots__ diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index ebaa1cfe4c8b58..18e13fcf9f59bd 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1,3 +1,5 @@ +.. _enum-howto: + ========== Enum HOWTO ========== @@ -483,6 +485,7 @@ Dataclass support When inheriting from a :class:`~dataclasses.dataclass`, the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: + >>> from dataclasses import dataclass, field >>> @dataclass ... class CreatureDataMixin: ... size: str @@ -496,13 +499,30 @@ the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: >>> Creature.DOG <Creature.DOG: size='medium', legs=4> -Use the :func:`!dataclass` argument ``repr=False`` +Use the :func:`~dataclasses.dataclass` argument ``repr=False`` to use the standard :func:`repr`. .. versionchanged:: 3.12 Only the dataclass fields are shown in the value area, not the dataclass' name. +.. note:: + + Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + and its subclasses is not supported. It will not raise any errors, + but it will produce very strange results at runtime, such as members + being equal to each other:: + + >>> @dataclass # don't do this: it does not make any sense + ... class Color(Enum): + ... RED = 1 + ... BLUE = 2 + ... + >>> Color.RED is Color.BLUE + False + >>> Color.RED == Color.BLUE # problem is here: they should not be equal + True + Pickling -------- @@ -527,7 +547,8 @@ It is possible to modify how enum members are pickled/unpickled by defining :meth:`__reduce_ex__` in the enumeration class. The default method is by-value, but enums with complicated values may want to use by-name:: - >>> class MyEnum(Enum): + >>> import enum + >>> class MyEnum(enum.Enum): ... __reduce_ex__ = enum.pickle_by_enum_name .. note:: @@ -605,9 +626,9 @@ The complete signature is:: start=1, ) -:value: What the new enum class will record as its name. +* *value*: What the new enum class will record as its name. -:names: The enum members. This can be a whitespace- or comma-separated string +* *names*: The enum members. This can be a whitespace- or comma-separated string (values will start at 1 unless otherwise specified):: 'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE' @@ -624,13 +645,13 @@ The complete signature is:: {'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42} -:module: name of module where new enum class can be found. +* *module*: name of module where new enum class can be found. -:qualname: where in module new enum class can be found. +* *qualname*: where in module new enum class can be found. -:type: type to mix in to new enum class. +* *type*: type to mix in to new enum class. -:start: number to start counting at if only names are passed in. +* *start*: number to start counting at if only names are passed in. .. versionchanged:: 3.5 The *start* parameter was added. @@ -770,7 +791,7 @@ be combined with them (but may lose :class:`IntFlag` membership:: >>> Perm.X | 4 <Perm.R|X: 5> - >>> Perm.X | 8 + >>> Perm.X + 8 9 .. note:: @@ -866,7 +887,7 @@ Others While :class:`IntEnum` is part of the :mod:`enum` module, it would be very simple to implement independently:: - class IntEnum(int, Enum): + class IntEnum(int, ReprEnum): # or Enum instead of ReprEnum pass This demonstrates how similar derived enumerations can be defined; for example @@ -874,8 +895,8 @@ a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` +1. When subclassing :class:`Enum`, mix-in types must appear before the + :class:`Enum` class itself in the sequence of bases, as in the :class:`IntEnum` example above. 2. Mix-in types must be subclassable. For example, :class:`bool` and :class:`range` are not subclassable and will throw an error during Enum @@ -959,30 +980,34 @@ all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. See `MultiValueEnum`_ for an example. -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used by the `Functional API`_ and by - :class:`auto` to get an appropriate value for an enum member; may be - overridden + .. note:: -.. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can be provided. It will be checked against the actual order of the enumeration @@ -1127,6 +1152,14 @@ the following are true: >>> (Color.RED | Color.GREEN).name 'RED|GREEN' + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... + >>> (Perm.R & Perm.W).name is None # effectively Perm(0) + True + - multi-bit flags, aka aliases, can be returned from operations:: >>> Color.RED | Color.BLUE @@ -1435,7 +1468,7 @@ alias:: ... GRENE = 2 ... Traceback (most recent call last): - ... + ... ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' .. note:: @@ -1445,6 +1478,29 @@ alias:: disallowing aliases, the :func:`unique` decorator can be used instead. +MultiValueEnum +^^^^^^^^^^^^^^^^^ + +Supports having more than one value per member:: + + >>> class MultiValueEnum(Enum): + ... def __new__(cls, value, *values): + ... self = object.__new__(cls) + ... self._value_ = value + ... for v in values: + ... self._add_value_alias_(v) + ... return self + ... + >>> class DType(MultiValueEnum): + ... float32 = 'f', 8 + ... double64 = 'd', 9 + ... + >>> DType('f') + <DType.float32: 'f'> + >>> DType(9) + <DType.double64: 'd'> + + Planet ^^^^^^ diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst new file mode 100644 index 00000000000000..1ba91b09516f9c --- /dev/null +++ b/Doc/howto/free-threading-extensions.rst @@ -0,0 +1,272 @@ +.. highlight:: c + +.. _freethreading-extensions-howto: + +****************************************** +C API Extension Support for Free Threading +****************************************** + +Starting with the 3.13 release, CPython has experimental support for running +with the :term:`global interpreter lock` (GIL) disabled in a configuration +called :term:`free threading`. This document describes how to adapt C API +extensions to support free threading. + + +Identifying the Free-Threaded Build in C +======================================== + +The CPython C API exposes the ``Py_GIL_DISABLED`` macro: in the free-threaded +build it's defined to ``1``, and in the regular build it's not defined. +You can use it to enable code that only runs under the free-threaded build:: + + #ifdef Py_GIL_DISABLED + /* code that only runs in the free-threaded build */ + #endif + +Module Initialization +===================== + +Extension modules need to explicitly indicate that they support running with +the GIL disabled; otherwise importing the extension will raise a warning and +enable the GIL at runtime. + +There are two ways to indicate that an extension module supports running with +the GIL disabled depending on whether the extension uses multi-phase or +single-phase initialization. + +Multi-Phase Initialization +.......................... + +Extensions that use multi-phase initialization (i.e., +:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the +module definition. If your extension supports older versions of CPython, +you should guard the slot with a :c:data:`PY_VERSION_HEX` check. + +:: + + static struct PyModuleDef_Slot module_slots[] = { + ... + #if PY_VERSION_HEX >= 0x030D0000 + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + {0, NULL} + }; + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_slots = module_slots, + ... + }; + + +Single-Phase Initialization +........................... + +Extensions that use single-phase initialization (i.e., +:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +indicate that they support running with the GIL disabled. The function is +only defined in the free-threaded build, so you should guard the call with +``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. + +:: + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + ... + }; + + PyMODINIT_FUNC + PyInit_mymodule(void) + { + PyObject *m = PyModule_Create(&moduledef); + if (m == NULL) { + return NULL; + } + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); + #endif + return m; + } + + +General API Guidelines +====================== + +Most of the C API is thread-safe, but there are some exceptions. + +* **Struct Fields**: Accessing fields in Python C API objects or structs + directly is not thread-safe if the field may be concurrently modified. +* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM` and + :c:macro:`PyList_SET_ITEM` do not perform any error checking or locking. + These macros are not thread-safe if the container object may be modified + concurrently. +* **Borrowed References**: C API functions that return + :term:`borrowed references <borrowed reference>` may not be thread-safe if + the containing object is modified concurrently. See the section on + :ref:`borrowed references <borrowed-references>` for more information. + + +Container Thread Safety +....................... + +Containers like :c:struct:`PyListObject`, +:c:struct:`PyDictObject`, and :c:struct:`PySetObject` perform internal locking +in the free-threaded build. For example, the :c:func:`PyList_Append` will +lock the list before appending an item. + +.. _PyDict_Next: + +``PyDict_Next`` +''''''''''''''' + +A notable exception is :c:func:`PyDict_Next`, which does not lock the +dictionary. You should use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to protect +the dictionary while iterating over it if the dictionary may be concurrently +modified:: + + Py_BEGIN_CRITICAL_SECTION(dict); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + + +Borrowed References +=================== + +.. _borrowed-references: + +Some C API functions return :term:`borrowed references <borrowed reference>`. +These APIs are not thread-safe if the containing object is modified +concurrently. For example, it's not safe to use :c:func:`PyList_GetItem` +if the list may be modified concurrently. + +The following table lists some borrowed reference APIs and their replacements +that return :term:`strong references <strong reference>`. + ++-----------------------------------+-----------------------------------+ +| Borrowed reference API | Strong reference API | ++===================================+===================================+ +| :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemString` | :c:func:`PyDict_GetItemStringRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_SetDefault` | :c:func:`PyDict_SetDefaultRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | ++-----------------------------------+-----------------------------------+ + +Not all APIs that return borrowed references are problematic. For +example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. +Similarly, not all uses of the above APIs are problematic. For example, +:c:func:`PyDict_GetItem` is often used for parsing keyword argument +dictionaries in function calls; those keyword argument dictionaries are +effectively private (not accessible by other threads), so using borrowed +references in that context is safe. + +Some of these functions were added in Python 3.13. You can use the +`pythoncapi-compat <https://github.com/python/pythoncapi-compat>`_ package +to provide implementations of these functions for older Python versions. + + +Memory Allocation APIs +====================== + +Python's memory management C API provides functions in three different +:ref:`allocation domains <allocator-domains>`: "raw", "mem", and "object". +For thread-safety, the free-threaded build requires that only Python objects +are allocated using the object domain, and that all Python object are +allocated using that domain. This differes from the prior Python versions, +where this was only a best practice and not a hard requirement. + +.. note:: + + Search for uses of :c:func:`PyObject_Malloc` in your + extension and check that the allocated memory is used for Python objects. + Use :c:func:`PyMem_Malloc` to allocate buffers instead of + :c:func:`PyObject_Malloc`. + + +Thread State and GIL APIs +========================= + +Python provides a set of functions and macros to manage thread state and the +GIL, such as: + +* :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` +* :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` +* :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` + +These functions should still be used in the free-threaded build to manage +thread state even when the :term:`GIL` is disabled. For example, if you +create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +before calling into the Python API to ensure that the thread has a valid +Python thread state. + +You should continue to call :c:func:`PyEval_SaveThread` or +:c:macro:`Py_BEGIN_ALLOW_THREADS` around blocking operations, such as I/O or +lock acquisitions, to allow other threads to run the +:term:`cyclic garbage collector <garbage collection>`. + + +Protecting Internal Extension State +=================================== + +Your extension may have internal state that was previously protected by the +GIL. You may need to add locking to protect this state. The approach will +depend on your extension, but some common patterns include: + +* **Caches**: global caches are a common source of shared state. Consider + using a lock to protect the cache or disabling it in the free-threaded build + if the cache is not critical for performance. +* **Global State**: global state may need to be protected by a lock or moved + to thread local storage. C11 and C++11 provide the ``thread_local`` or + ``_Thread_local`` for + `thread-local storage <https://en.cppreference.com/w/c/language/storage_duration>`_. + + +Building Extensions for the Free-Threaded Build +=============================================== + +C API extensions need to be built specifically for the free-threaded build. +The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. + +* `pypa/manylinux <https://github.com/pypa/manylinux>`_ supports the + free-threaded build, with the ``t`` suffix, such as ``python3.13t``. +* `pypa/cibuildwheel <https://github.com/pypa/cibuildwheel>`_ supports the + free-threaded build if you set + `CIBW_FREE_THREADED_SUPPORT <https://cibuildwheel.pypa.io/en/stable/options/#free-threaded-support>`_. + +Limited C API and Stable ABI +............................ + +The free-threaded build does not currently support the +:ref:`Limited C API <limited-c-api>` or the stable ABI. If you use +`setuptools <https://setuptools.pypa.io/en/latest/setuptools.html>`_ to build +your extension and currently set ``py_limited_api=True`` you can use +``py_limited_api=not sysconfig.get_config_var("Py_GIL_DISABLED")`` to opt out +of the limited API when building with the free-threaded build. + +.. note:: + You will need to build separate wheels specifically for the free-threaded + build. If you currently use the stable ABI, you can continue to build a + single wheel for multiple non-free-threaded Python versions. + + +Windows +....... + +Due to a limitation of the official Windows installer, you will need to +manually define ``Py_GIL_DISABLED=1`` when building extensions from source. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index b0f9d22d74f0e3..1f0608fb0fc53f 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1,3 +1,5 @@ +.. _functional-howto: + ******************************** Functional Programming HOWTO ******************************** diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst new file mode 100644 index 00000000000000..53bbf7ddaa2ab9 --- /dev/null +++ b/Doc/howto/gdb_helpers.rst @@ -0,0 +1,449 @@ +.. _gdb: + +========================================================= +Debugging C API extensions and CPython Internals with GDB +========================================================= + +.. highlight:: none + +This document explains how the Python GDB extension, ``python-gdb.py``, can +be used with the GDB debugger to debug CPython extensions and the +CPython interpreter itself. + +When debugging low-level problems such as crashes or deadlocks, a low-level +debugger, such as GDB, is useful to diagnose and correct the issue. +By default, GDB (or any of its front-ends) doesn't support high-level +information specific to the CPython interpreter. + +The ``python-gdb.py`` extension adds CPython interpreter information to GDB. +The extension helps introspect the stack of currently executing Python functions. +Given a Python object represented by a :c:expr:`PyObject *` pointer, +the extension surfaces the type and value of the object. + +Developers who are working on CPython extensions or tinkering with parts +of CPython that are written in C can use this document to learn how to use the +``python-gdb.py`` extension with GDB. + +.. note:: + + This document assumes that you are familiar with the basics of GDB and the + CPython C API. It consolidates guidance from the + `devguide <https://devguide.python.org>`_ and the + `Python wiki <https://wiki.python.org/moin/DebuggingWithGdb>`_. + + +Prerequisites +============= + +You need to have: + +- GDB 7 or later. (For earlier versions of GDB, see ``Misc/gdbinit`` in the + sources of Python 3.11 or earlier.) +- GDB-compatible debugging information for Python and any extension you are + debugging. +- The ``python-gdb.py`` extension. + +The extension is built with Python, but might be distributed separately or +not at all. Below, we include tips for a few common systems as examples. +Note that even if the instructions match your system, they might be outdated. + + +Setup with Python built from source +----------------------------------- + +When you build CPython from source, debugging information should be available, +and the build should add a ``python-gdb.py`` file to the root directory of +your repository. + +To activate support, you must add the directory containing ``python-gdb.py`` +to GDB's "auto-load-safe-path". +If you haven't done this, recent versions of GDB will print out a warning +with instructions on how to do this. + +.. note:: + + If you do not see instructions for your version of GDB, put this in your + configuration file (``~/.gdbinit`` or ``~/.config/gdb/gdbinit``):: + + add-auto-load-safe-path /path/to/cpython + + You can also add multiple paths, separated by ``:``. + + +Setup for Python from a Linux distro +------------------------------------ + +Most Linux systems provide debug information for the system Python +in a package called ``python-debuginfo``, ``python-dbg`` or similar. +For example: + +- Fedora: + + .. code-block:: shell + + sudo dnf install gdb + sudo dnf debuginfo-install python3 + +- Ubuntu: + + .. code-block:: shell + + sudo apt install gdb python3-dbg + +On several recent Linux systems, GDB can download debugging symbols +automatically using *debuginfod*. +However, this will not install the ``python-gdb.py`` extension; +you generally do need to install the debug info package separately. + + +Using the Debug build and Development mode +========================================== + +For easier debugging, you might want to: + +- Use a :ref:`debug build <debug-build>` of Python. (When building from source, + use ``configure --with-pydebug``. On Linux distros, install and run a package + like ``python-debug`` or ``python-dbg``, if available.) +- Use the runtime :ref:`development mode <devmode>` (``-X dev``). + +Both enable extra assertions and disable some optimizations. +Sometimes this hides the bug you are trying to find, but in most cases they +make the process easier. + + +Using the ``python-gdb`` extension +================================== + +When the extension is loaded, it provides two main features: +pretty printers for Python values, and additional commands. + +Pretty-printers +--------------- + +This is what a GDB backtrace looks like (truncated) when this extension is +enabled:: + + #0 0x000000000041a6b1 in PyObject_Malloc (nbytes=Cannot access memory at address 0x7fffff7fefe8 + ) at Objects/obmalloc.c:748 + #1 0x000000000041b7c0 in _PyObject_DebugMallocApi (id=111 'o', nbytes=24) at Objects/obmalloc.c:1445 + #2 0x000000000041b717 in _PyObject_DebugMalloc (nbytes=24) at Objects/obmalloc.c:1412 + #3 0x000000000044060a in _PyUnicode_New (length=11) at Objects/unicodeobject.c:346 + #4 0x00000000004466aa in PyUnicodeUCS2_DecodeUTF8Stateful (s=0x5c2b8d "__lltrace__", size=11, errors=0x0, consumed= + 0x0) at Objects/unicodeobject.c:2531 + #5 0x0000000000446647 in PyUnicodeUCS2_DecodeUTF8 (s=0x5c2b8d "__lltrace__", size=11, errors=0x0) + at Objects/unicodeobject.c:2495 + #6 0x0000000000440d1b in PyUnicodeUCS2_FromStringAndSize (u=0x5c2b8d "__lltrace__", size=11) + at Objects/unicodeobject.c:551 + #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 + #8 0x0000000000584abd in PyDict_GetItemString (v= + {'Yuck': <type at remote 0xad4730>, '__builtins__': <module at remote 0x7ffff7fd5ee8>, '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': <Yuck(i=0) at remote 0xaacd80>, 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': <Yuck(i=0) at remote 0xaace60>, '__doc__': None}, key= + 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 + +Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed +as its ``repr()``, rather than an opaque ``PyObject *`` pointer. + +The extension works by supplying a custom printing routine for values of type +``PyObject *``. If you need to access lower-level details of an object, then +cast the value to a pointer of the appropriate type. For example:: + + (gdb) p globals + $1 = {'__builtins__': <module at remote 0x7ffff7fb1868>, '__name__': + '__main__', 'ctypes': <module at remote 0x7ffff7f14360>, '__doc__': None, + '__package__': None} + + (gdb) p *(PyDictObject*)globals + $2 = {ob_refcnt = 3, ob_type = 0x3dbdf85820, ma_fill = 5, ma_used = 5, + ma_mask = 7, ma_table = 0x63d0f8, ma_lookup = 0x3dbdc7ea70 + <lookdict_string>, ma_smalltable = {{me_hash = 7065186196740147912, + me_key = '__builtins__', me_value = <module at remote 0x7ffff7fb1868>}, + {me_hash = -368181376027291943, me_key = '__name__', + me_value ='__main__'}, {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = -9177857982131165996, me_key = 'ctypes', + me_value = <module at remote 0x7ffff7f14360>}, + {me_hash = -8518757509529533123, me_key = '__doc__', me_value = None}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, { + me_hash = 6614918939584953775, me_key = '__package__', me_value = None}}} + +Note that the pretty-printers do not actually call ``repr()``. +For basic types, they try to match its result closely. + +An area that can be confusing is that the custom printer for some types look a +lot like GDB's built-in printer for standard types. For example, the +pretty-printer for a Python ``int`` (:c:expr:`PyLongObject *`) +gives a representation that is not distinguishable from one of a +regular machine-level integer:: + + (gdb) p some_machine_integer + $3 = 42 + + (gdb) p some_python_integer + $4 = 42 + +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: + + (gdb) p *(PyLongObject*)some_python_integer + $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, + ob_digit = {42}} + +A similar confusion can arise with the ``str`` type, where the output looks a +lot like gdb's built-in printer for ``char *``:: + + (gdb) p ptr_to_python_str + $6 = '__builtins__' + +The pretty-printer for ``str`` instances defaults to using single-quotes (as +does Python's ``repr`` for strings) whereas the standard printer for ``char *`` +values uses double-quotes and contains a hexadecimal address:: + + (gdb) p ptr_to_char_star + $7 = 0x6d72c0 "hello world" + +Again, the implementation details can be revealed with a cast to +:c:expr:`PyUnicodeObject *`:: + + (gdb) p *(PyUnicodeObject*)$6 + $8 = {ob_base = {ob_refcnt = 33, ob_type = 0x3dad3a95a0}, length = 12, + str = 0x7ffff2128500, hash = 7065186196740147912, state = 1, defenc = 0x0} + +``py-list`` +----------- + + The extension adds a ``py-list`` command, which + lists the Python source code (if any) for the current frame in the selected + thread. The current line is marked with a ">":: + + (gdb) py-list + 901 if options.profile: + 902 options.profile = False + 903 profile_me() + 904 return + 905 + >906 u = UI() + 907 if not u.quit: + 908 try: + 909 gtk.main() + 910 except KeyboardInterrupt: + 911 # properly quit on a keyboard interrupt... + + Use ``py-list START`` to list at a different line number within the Python + source, and ``py-list START,END`` to list a specific range of lines within + the Python source. + +``py-up`` and ``py-down`` +------------------------- + + The ``py-up`` and ``py-down`` commands are analogous to GDB's regular ``up`` + and ``down`` commands, but try to move at the level of CPython frames, rather + than C frames. + + GDB is not always able to read the relevant frame information, depending on + the optimization level with which CPython was compiled. Internally, the + commands look for C frames that are executing the default frame evaluation + function (that is, the core bytecode interpreter loop within CPython) and + look up the value of the related ``PyFrameObject *``. + + They emit the frame number (at the C level) within the thread. + + For example:: + + (gdb) py-up + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-up + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/gnome_sudoku.py, line 22, in start_game(main=<module at remote 0xb771b7f4>) + main.start_game() + (gdb) py-up + Unable to find an older python frame + + so we're at the top of the Python stack. + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + The command skips C frames which are not executing Python code. + + Going back down:: + + (gdb) py-down + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-down + #34 (unable to read python frame information) + (gdb) py-down + #23 (unable to read python frame information) + (gdb) py-down + #19 (unable to read python frame information) + (gdb) py-down + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=<NewOrSavedGameSelector(new_game_model=<gtk.ListStore at remote 0x98fab44>, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': <float at remote 0x984b474>, 'gsd.hints': 0, 'timer.active_time': <float at remote 0x984b494>, 'timer.total_time': <float at remote 0x984b464>}], dialog=<gtk.Dialog at remote 0x98faaa4>, saved_game_model=<gtk.ListStore at remote 0x98fad24>, sudoku_maker=<SudokuMaker(terminated=False, played=[], batch_siz...(truncated) + swallower.run_dialog(self.dialog) + (gdb) py-down + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=<SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, main_page=0) at remote 0x98fa6e4>, d=<gtk.Dialog at remote 0x98faaa4>) + gtk.main() + (gdb) py-down + #8 (unable to read python frame information) + (gdb) py-down + Unable to find a newer python frame + + and we're at the bottom of the Python stack. + + Note that in Python 3.12 and newer, the same C stack frame can be used for + multiple Python stack frames. This means that ``py-up`` and ``py-down`` + may move multiple Python frames at once. For example:: + + (gdb) py-up + #6 Frame 0x7ffff7fb62b0, for file /tmp/rec.py, line 5, in recursive_function (n=0) + time.sleep(5) + #6 Frame 0x7ffff7fb6240, for file /tmp/rec.py, line 7, in recursive_function (n=1) + recursive_function(n-1) + #6 Frame 0x7ffff7fb61d0, for file /tmp/rec.py, line 7, in recursive_function (n=2) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6160, for file /tmp/rec.py, line 7, in recursive_function (n=3) + recursive_function(n-1) + #6 Frame 0x7ffff7fb60f0, for file /tmp/rec.py, line 7, in recursive_function (n=4) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6080, for file /tmp/rec.py, line 7, in recursive_function (n=5) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6020, for file /tmp/rec.py, line 9, in <module> () + recursive_function(5) + (gdb) py-up + Unable to find an older python frame + + +``py-bt`` +--------- + + The ``py-bt`` command attempts to display a Python-level backtrace of the + current thread. + + For example:: + + (gdb) py-bt + #8 (unable to read python frame information) + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=<SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, main_page=0) at remote 0x98fa6e4>, d=<gtk.Dialog at remote 0x98faaa4>) + gtk.main() + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=<NewOrSavedGameSelector(new_game_model=<gtk.ListStore at remote 0x98fab44>, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': <float at remote 0x984b474>, 'gsd.hints': 0, 'timer.active_time': <float at remote 0x984b494>, 'timer.total_time': <float at remote 0x984b464>}], dialog=<gtk.Dialog at remote 0x98faaa4>, saved_game_model=<gtk.ListStore at remote 0x98fad24>, sudoku_maker=<SudokuMaker(terminated=False, played=[], batch_siz...(truncated) + swallower.run_dialog(self.dialog) + #19 (unable to read python frame information) + #23 (unable to read python frame information) + #34 (unable to read python frame information) + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/gnome_sudoku/gnome_sudoku.py, line 22, in start_game (main=<module at remote 0xb771b7f4>) + main.start_game() + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + +``py-print`` +------------ + + The ``py-print`` command looks up a Python name and tries to print it. + It looks in locals within the current thread, then globals, then finally + builtins:: + + (gdb) py-print self + local 'self' = <SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, + main_page=0) at remote 0x98fa6e4> + (gdb) py-print __name__ + global '__name__' = 'gnome_sudoku.dialog_swallower' + (gdb) py-print len + builtin 'len' = <built-in function len> + (gdb) py-print scarlet_pimpernel + 'scarlet_pimpernel' not found + + If the current C frame corresponds to multiple Python frames, ``py-print`` + only considers the first one. + +``py-locals`` +------------- + + The ``py-locals`` command looks up all Python locals within the current + Python frame in the selected thread, and prints their representations:: + + (gdb) py-locals + self = <SwappableArea(running=<gtk.Dialog at remote 0x98faaa4>, + main_page=0) at remote 0x98fa6e4> + d = <gtk.Dialog at remote 0x98faaa4> + + If the current C frame corresponds to multiple Python frames, locals from + all of them will be shown:: + + (gdb) py-locals + Locals for recursive_function + n = 0 + Locals for recursive_function + n = 1 + Locals for recursive_function + n = 2 + Locals for recursive_function + n = 3 + Locals for recursive_function + n = 4 + Locals for recursive_function + n = 5 + Locals for <module> + + +Use with GDB commands +===================== + +The extension commands complement GDB's built-in commands. +For example, you can use a frame numbers shown by ``py-bt`` with the ``frame`` +command to go a specific frame within the selected thread, like this:: + + (gdb) py-bt + (output snipped) + #68 Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in <module> () + main() + (gdb) frame 68 + #68 0x00000000004cd1e6 in PyEval_EvalFrameEx (f=Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in <module> (), throwflag=0) at Python/ceval.c:2665 + 2665 x = call_function(&sp, oparg); + (gdb) py-list + 1543 # Run the tests in a context manager that temporary changes the CWD to a + 1544 # temporary and writable directory. If it's not possible to create or + 1545 # change the CWD, the original CWD will be used. The original CWD is + 1546 # available from test_support.SAVEDCWD. + 1547 with test_support.temp_cwd(TESTCWD, quiet=True): + >1548 main() + +The ``info threads`` command will give you a list of the threads within the +process, and you can use the ``thread`` command to select a different one:: + + (gdb) info threads + 105 Thread 0x7fffefa18710 (LWP 10260) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + 104 Thread 0x7fffdf5fe710 (LWP 10259) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + * 1 Thread 0x7ffff7fe2700 (LWP 10145) 0x00000038e46d73e3 in select () at ../sysdeps/unix/syscall-template.S:82 + +You can use ``thread apply all COMMAND`` or (``t a a COMMAND`` for short) to run +a command on all threads. With ``py-bt``, this lets you see what every +thread is doing at the Python level:: + + (gdb) t a a py-bt + + Thread 105 (Thread 0x7fffefa18710 (LWP 10260)): + #5 Frame 0x7fffd00019d0, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140737213728528), count=1, owner=140737213728528) + self.__block.acquire() + #8 Frame 0x7fffac001640, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, acquire=<instancemethod at remote 0xd80260>, _is_owned=<instancemethod at remote 0xd80160>, _release_save=<instancemethod at remote 0xd803e0>, release=<instancemethod at remote 0xd802e0>, _acquire_restore=<instancemethod at remote 0xd7ee60>, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=<thread.lock at remote 0x858a90>, saved_state=(1, 140737213728528)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffb8001a10, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffb8001c40, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140737213728528) + f() + + Thread 104 (Thread 0x7fffdf5fe710 (LWP 10259)): + #5 Frame 0x7fffe4001580, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140736940992272), count=1, owner=140736940992272) + self.__block.acquire() + #8 Frame 0x7fffc8002090, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=<thread.lock at remote 0x858770>, _RLock__count=1) at remote 0xd7ff40>, acquire=<instancemethod at remote 0xd80260>, _is_owned=<instancemethod at remote 0xd80160>, _release_save=<instancemethod at remote 0xd803e0>, release=<instancemethod at remote 0xd802e0>, _acquire_restore=<instancemethod at remote 0xd7ee60>, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=<thread.lock at remote 0x858860>, saved_state=(1, 140736940992272)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffac001c90, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffac0011c0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140736940992272) + f() + + Thread 1 (Thread 0x7ffff7fe2700 (LWP 10145)): + #5 Frame 0xcb5380, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 16, in _wait () + time.sleep(0.01) + #8 Frame 0x7fffd00024a0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 378, in _check_notify (self=<ConditionTests(_testMethodName='test_notify', _resultForDoCleanups=<TestResult(_original_stdout=<cStringIO.StringO at remote 0xc191e0>, skipped=[], _mirrorOutput=False, testsRun=39, buffer=False, _original_stderr=<file at remote 0x7ffff7fc6340>, _stdout_buffer=<cStringIO.StringO at remote 0xc9c7f8>, _stderr_buffer=<cStringIO.StringO at remote 0xc9c790>, _moduleSetUpFailed=False, expectedFailures=[], errors=[], _previousTestClass=<type at remote 0x928310>, unexpectedSuccesses=[], failures=[], shouldStop=False, failfast=False) at remote 0xc185a0>, _threads=(0,), _cleanups=[], _type_equality_funcs={<type at remote 0x7eba00>: <instancemethod at remote 0xd750e0>, <type at remote 0x7e7820>: <instancemethod at remote 0xd75160>, <type at remote 0x7e30e0>: <instancemethod at remote 0xd75060>, <type at remote 0x7e7d20>: <instancemethod at remote 0xd751e0>, <type at remote 0x7f19e0...(truncated) + _wait() diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index a835bb5f13bd1c..a882f1747084fe 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -2,21 +2,19 @@ Python HOWTOs *************** -Python HOWTOs are documents that cover a single, specific topic, -and attempt to cover it fairly completely. Modelled on the Linux -Documentation Project's HOWTO collection, this collection is an +Python HOWTOs are documents that cover a specific topic in-depth. +Modeled on the Linux Documentation Project's HOWTO collection, this collection is an effort to foster documentation that's more detailed than the Python Library Reference. -Currently, the HOWTOs are: - .. toctree:: :maxdepth: 1 + :hidden: - pyporting.rst cporting.rst curses.rst descriptor.rst + gdb_helpers.rst enum.rst functional.rst logging.rst @@ -33,4 +31,36 @@ Currently, the HOWTOs are: annotations.rst isolating-extensions.rst timerfd.rst + mro.rst + free-threading-extensions.rst + +General: + +* :ref:`annotations-howto` +* :ref:`argparse-tutorial` +* :ref:`descriptorhowto` +* :ref:`enum-howto` +* :ref:`functional-howto` +* :ref:`ipaddress-howto` +* :ref:`logging-howto` +* :ref:`logging-cookbook` +* :ref:`regex-howto` +* :ref:`sortinghowto` +* :ref:`unicode-howto` +* :ref:`urllib-howto` + +Advanced development: + +* :ref:`curses-howto` +* :ref:`freethreading-extensions-howto` +* :ref:`isolating-extensions-howto` +* :ref:`python_2.3_mro` +* :ref:`socket-howto` +* :ref:`timerfd-howto` +* :ref:`cporting-howto` + +Debugging and profiling: +* :ref:`gdb` +* :ref:`instrumentation` +* :ref:`perf_profiling` diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 8f3787f2d2f145..e35855deedbe5f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -337,14 +337,46 @@ That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - Define a traverse function using ``Py_tp_traverse``, which - visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`). + visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). -Please refer to the :ref:`the documentation <type-structs>` of +Please refer to the the documentation of :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` for additional considerations. -If your traverse function delegates to the ``tp_traverse`` of its base class -(or another type), ensure that ``Py_TYPE(self)`` is visited only once. +The API for defining heap types grew organically, leaving it +somewhat awkward to use in its current state. +The following sections will guide you through common issues. + + +``tp_traverse`` in Python 3.8 and lower +....................................... + +The requirement to visit the type from ``tp_traverse`` was added in Python 3.9. +If you support Python 3.8 and lower, the traverse function must *not* +visit the type, so it must be more complicated:: + + static int my_traverse(PyObject *self, visitproc visit, void *arg) + { + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } + return 0; + } + +Unfortunately, :c:data:`Py_Version` was only added in Python 3.11. +As a replacement, use: + +* :c:macro:`PY_VERSION_HEX`, if not using the stable ABI, or +* :py:data:`sys.version_info` (via :c:func:`PySys_GetObject` and + :c:func:`PyArg_ParseTuple`). + + +Delegating ``tp_traverse`` +.......................... + +If your traverse function delegates to the :c:member:`~PyTypeObject.tp_traverse` +of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited +only once. Note that only heap type are expected to visit the type in ``tp_traverse``. For example, if your traverse function includes:: @@ -356,11 +388,70 @@ For example, if your traverse function includes:: if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { // a heap type's tp_traverse already visited Py_TYPE(self) } else { - Py_VISIT(Py_TYPE(self)); + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } } -It is not necessary to handle the type's reference count in ``tp_new`` -and ``tp_clear``. +It is not necessary to handle the type's reference count in +:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_clear`. + + +Defining ``tp_dealloc`` +....................... + +If your type has a custom :c:member:`~PyTypeObject.tp_dealloc` function, +it needs to: + +- call :c:func:`PyObject_GC_UnTrack` before any fields are invalidated, and +- decrement the reference count of the type. + +To keep the type valid while ``tp_free`` is called, the type's refcount needs +to be decremented *after* the instance is deallocated. For example:: + + static void my_dealloc(PyObject *self) + { + PyObject_GC_UnTrack(self); + ... + PyTypeObject *type = Py_TYPE(self); + type->tp_free(self); + Py_DECREF(type); + } + +The default ``tp_dealloc`` function does this, so +if your type does *not* override +``tp_dealloc`` you don't need to add it. + + +Not overriding ``tp_free`` +.......................... + +The :c:member:`~PyTypeObject.tp_free` slot of a heap type must be set to +:c:func:`PyObject_GC_Del`. +This is the default; do not override it. + + +Avoiding ``PyObject_New`` +......................... + +GC-tracked objects need to be allocated using GC-aware functions. + +If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: + +- Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. + That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: + + TYPE *o = typeobj->tp_alloc(typeobj, 0); + + Replace ``o = PyObject_NewVar(TYPE, typeobj, size)`` with the same, + but use size instead of the 0. + +- If the above is not possible (e.g. inside a custom ``tp_alloc``), + call :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`:: + + TYPE *o = PyObject_GC_New(TYPE, typeobj); + + TYPE *o = PyObject_GC_NewVar(TYPE, typeobj, size); Module State Access from Classes @@ -391,7 +482,7 @@ The largest roadblock is getting *the class a method was defined in*, or that method's "defining class" for short. The defining class can have a reference to the module it is part of. -Do not confuse the defining class with :c:expr:`Py_TYPE(self)`. If the method +Do not confuse the defining class with ``Py_TYPE(self)``. If the method is called on a *subclass* of your type, ``Py_TYPE(self)`` will refer to that subclass, which may be defined in different module than yours. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 588f5a0a53ded0..3ed2dd6251afe9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -332,10 +332,10 @@ Suppose you configure logging with the following JSON: } } -This configuration does *almost* what we want, except that ``sys.stdout`` would -show messages of severity ``ERROR`` and above as well as ``INFO`` and -``WARNING`` messages. To prevent this, we can set up a filter which excludes -those messages and add it to the relevant handler. This can be configured by +This configuration does *almost* what we want, except that ``sys.stdout`` would show messages +of severity ``ERROR`` and only events of this severity and higher will be tracked +as well as ``INFO`` and ``WARNING`` messages. To prevent this, we can set up a filter which +excludes those messages and add it to the relevant handler. This can be configured by adding a ``filters`` section parallel to ``formatters`` and ``handlers``: .. code-block:: json @@ -1744,13 +1744,11 @@ to the above, as in the following example:: return self.fmt.format(*self.args) class StyleAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super().__init__(logger, extra or {}) - - def log(self, level, msg, /, *args, **kwargs): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, Message(msg, args), (), **kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) logger = StyleAdapter(logging.getLogger(__name__)) @@ -1762,7 +1760,7 @@ to the above, as in the following example:: main() The above script should log the message ``Hello, world!`` when run with -Python 3.2 or later. +Python 3.8 or later. .. currentmodule:: logging @@ -1848,8 +1846,11 @@ the use of a :class:`Filter` does not provide the desired result. .. _zeromq-handlers: -Subclassing QueueHandler - a ZeroMQ example -------------------------------------------- +Subclassing QueueHandler and QueueListener- a ZeroMQ example +------------------------------------------------------------ + +Subclass ``QueueHandler`` +^^^^^^^^^^^^^^^^^^^^^^^^^ You can use a :class:`QueueHandler` subclass to send messages to other kinds of queues, for example a ZeroMQ 'publish' socket. In the example below,the @@ -1887,8 +1888,8 @@ data needed by the handler to create the socket:: self.queue.close() -Subclassing QueueListener - a ZeroMQ example --------------------------------------------- +Subclass ``QueueListener`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: @@ -1905,25 +1906,194 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: msg = self.queue.recv_json() return logging.makeLogRecord(msg) +.. _pynng-handlers: -.. seealso:: +Subclassing QueueHandler and QueueListener- a ``pynng`` example +--------------------------------------------------------------- - Module :mod:`logging` - API reference for the logging module. +In a similar way to the above section, we can implement a listener and handler +using :pypi:`pynng`, which is a Python binding to +`NNG <https://nng.nanomsg.org/>`_, billed as a spiritual successor to ZeroMQ. +The following snippets illustrate -- you can test them in an environment which has +``pynng`` installed. Just for variety, we present the listener first. - Module :mod:`logging.config` - Configuration API for the logging module. - Module :mod:`logging.handlers` - Useful handlers included with the logging module. +Subclass ``QueueListener`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + # listener.py + import json + import logging + import logging.handlers - :ref:`A basic logging tutorial <logging-basic-tutorial>` + import pynng - :ref:`A more advanced logging tutorial <logging-advanced-tutorial>` + DEFAULT_ADDR = "tcp://localhost:13232" + interrupted = False + + class NNGSocketListener(logging.handlers.QueueListener): + + def __init__(self, uri, /, *handlers, **kwargs): + # Have a timeout for interruptability, and open a + # subscriber socket + socket = pynng.Sub0(listen=uri, recv_timeout=500) + # The b'' subscription matches all topics + topics = kwargs.pop('topics', None) or b'' + socket.subscribe(topics) + # We treat the socket as a queue + super().__init__(socket, *handlers, **kwargs) + + def dequeue(self, block): + data = None + # Keep looping while not interrupted and no data received over the + # socket + while not interrupted: + try: + data = self.queue.recv(block=block) + break + except pynng.Timeout: + pass + except pynng.Closed: # sometimes happens when you hit Ctrl-C + break + if data is None: + return None + # Get the logging event sent from a publisher + event = json.loads(data.decode('utf-8')) + return logging.makeLogRecord(event) + + def enqueue_sentinel(self): + # Not used in this implementation, as the socket isn't really a + # queue + pass + + logging.getLogger('pynng').propagate = False + listener = NNGSocketListener(DEFAULT_ADDR, logging.StreamHandler(), topics=b'') + listener.start() + print('Press Ctrl-C to stop.') + try: + while True: + pass + except KeyboardInterrupt: + interrupted = True + finally: + listener.stop() + + +Subclass ``QueueHandler`` +^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: logging +.. code-block:: python + + # sender.py + import json + import logging + import logging.handlers + import time + import random + + import pynng + + DEFAULT_ADDR = "tcp://localhost:13232" + + class NNGSocketHandler(logging.handlers.QueueHandler): + + def __init__(self, uri): + socket = pynng.Pub0(dial=uri, send_timeout=500) + super().__init__(socket) + + def enqueue(self, record): + # Send the record as UTF-8 encoded JSON + d = dict(record.__dict__) + data = json.dumps(d) + self.queue.send(data.encode('utf-8')) + + def close(self): + self.queue.close() + + logging.getLogger('pynng').propagate = False + handler = NNGSocketHandler(DEFAULT_ADDR) + # Make sure the process ID is in the output + logging.basicConfig(level=logging.DEBUG, + handlers=[logging.StreamHandler(), handler], + format='%(levelname)-8s %(name)10s %(process)6s %(message)s') + levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + logger_names = ('myapp', 'myapp.lib1', 'myapp.lib2') + msgno = 1 + while True: + # Just randomly select some loggers and levels and log away + level = random.choice(levels) + logger = logging.getLogger(random.choice(logger_names)) + logger.log(level, 'Message no. %5d' % msgno) + msgno += 1 + delay = random.random() * 2 + 0.5 + time.sleep(delay) + +You can run the above two snippets in separate command shells. If we run the +listener in one shell and run the sender in two separate shells, we should see +something like the following. In the first sender shell: + +.. code-block:: console + + $ python sender.py + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + CRITICAL myapp.lib2 613 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + DEBUG myapp 613 Message no. 6 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + (and so on) + +In the second sender shell: + +.. code-block:: console + + $ python sender.py + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +In the listener shell: + +.. code-block:: console + + $ python listener.py + Press Ctrl-C to stop. + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 613 Message no. 3 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + DEBUG myapp 613 Message no. 6 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +As you can see, the logging from the two sender processes is interleaved in the +listener's output. + + An example dictionary-based configuration ----------------------------------------- @@ -1933,30 +2103,28 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'formatters': { 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', }, 'simple': { - 'format': '%(levelname)s %(message)s' + 'format': '{levelname} {message}', + 'style': '{', }, }, 'filters': { 'special': { '()': 'project.logging.SpecialFilter', 'foo': 'bar', - } + }, }, 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', }, 'mail_admins': { 'level': 'ERROR', @@ -1966,9 +2134,8 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration }, 'loggers': { 'django': { - 'handlers':['null'], + 'handlers': ['console'], 'propagate': True, - 'level':'INFO', }, 'django.request': { 'handlers': ['mail_admins'], @@ -2783,7 +2950,7 @@ When run, this produces a file with exactly two lines: .. code-block:: none 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| + 28/01/2015 07:21:23|ERROR|ZeroDivisionError: division by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: division by zero'| While the above treatment is simplistic, it points the way to how exception information can be formatted to your liking. The :mod:`traceback` module may be @@ -3408,9 +3575,8 @@ A Qt GUI for logging A question that comes up from time to time is about how to log to a GUI application. The `Qt <https://www.qt.io/>`_ framework is a popular -cross-platform UI framework with Python bindings using `PySide2 -<https://pypi.org/project/PySide2/>`_ or `PyQt5 -<https://pypi.org/project/PyQt5/>`_ libraries. +cross-platform UI framework with Python bindings using :pypi:`PySide2` +or :pypi:`PyQt5` libraries. The following example shows how to log to a Qt GUI. This introduces a simple ``QtHandler`` class which takes a callable, which should be a slot in the main @@ -3423,9 +3589,10 @@ The worker thread is implemented using Qt's ``QThread`` class rather than the :mod:`threading` module, as there are circumstances where one has to use ``QThread``, which offers better integration with other ``Qt`` components. -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. +The code should work with recent releases of any of ``PySide6``, ``PyQt6``, +``PySide2`` or ``PyQt5``. You should be able to adapt the approach to earlier +versions of Qt. Please refer to the comments in the code snippet for more +detailed information. .. code-block:: python3 @@ -3435,16 +3602,25 @@ refer to the comments in the code snippet for more detailed information. import sys import time - # Deal with minor differences between PySide2 and PyQt5 + # Deal with minor differences between different Qt packages try: - from PySide2 import QtCore, QtGui, QtWidgets + from PySide6 import QtCore, QtGui, QtWidgets Signal = QtCore.Signal Slot = QtCore.Slot except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - + try: + from PyQt6 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + except ImportError: + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot logger = logging.getLogger(__name__) @@ -3516,8 +3692,14 @@ refer to the comments in the code snippet for more detailed information. while not QtCore.QThread.currentThread().isInterruptionRequested(): delay = 0.5 + random.random() * 2 time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + try: + if random.random() < 0.1: + raise ValueError('Exception raised: %d' % i) + else: + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + except ValueError as e: + logger.exception('Failed: %s', e, extra=extra) i += 1 # @@ -3544,7 +3726,10 @@ refer to the comments in the code snippet for more detailed information. self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) + if hasattr(f, 'Monospace'): + f.setStyleHint(f.Monospace) + else: + f.setStyleHint(f.StyleHint.Monospace) # for Qt6 te.setFont(f) te.setReadOnly(True) PB = QtWidgets.QPushButton @@ -3631,7 +3816,11 @@ refer to the comments in the code snippet for more detailed information. app = QtWidgets.QApplication(sys.argv) example = Window(app) example.show() - sys.exit(app.exec_()) + if hasattr(app, 'exec'): + rc = app.exec() + else: + rc = app.exec_() + sys.exit(rc) if __name__=='__main__': main() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 7330cf675baa36..cbfe93319ddaa4 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -1,3 +1,5 @@ +.. _logging-howto: + ============= Logging HOWTO ============= @@ -25,10 +27,12 @@ or *severity*. When to use logging ^^^^^^^^^^^^^^^^^^^ -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. +You can access logging functionality by creating a logger via ``logger = +getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +:meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and +:meth:`~Logger.critical` methods. To determine when to use logging, and to see +which logger methods to use when, see the table below. It states, for each of a +set of common tasks, the best tool to use for that task. +-------------------------------------+--------------------------------------+ | Task you want to perform | The best tool for the task | @@ -37,8 +41,8 @@ states, for each of a set of common tasks, the best tool to use for it. | usage of a command line script or | | | program | | +-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | +| Report events that occur during | A logger's :meth:`~Logger.info` (or | +| normal operation of a program (e.g. | :meth:`~Logger.debug` method for very| | for status monitoring or fault | detailed output for diagnostic | | investigation) | purposes) | +-------------------------------------+--------------------------------------+ @@ -47,22 +51,23 @@ states, for each of a set of common tasks, the best tool to use for it. | | the client application should be | | | modified to eliminate the warning | | | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | +| | A logger's :meth:`~Logger.warning` | +| | method if there is nothing the client| +| | application can do about the | +| | situation, but the event should still| +| | be noted | +-------------------------------------+--------------------------------------+ | Report an error regarding a | Raise an exception | | particular runtime event | | +-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | +| Report suppression of an error | A logger's :meth:`~Logger.error`, | +| without raising an exception (e.g. | :meth:`~Logger.exception` or | +| error handler in a long-running | :meth:`~Logger.critical` method as | | server process) | appropriate for the specific error | | | and application domain | +-------------------------------------+--------------------------------------+ -The logging functions are named after the level or severity of the events +The logger methods are named after the level or severity of the events they are used to track. The standard levels and their applicability are described below (in increasing order of severity): @@ -89,9 +94,8 @@ described below (in increasing order of severity): | | itself may be unable to continue running. | +--------------+---------------------------------------------+ -The default level is ``WARNING``, which means that only events of this level -and above will be tracked, unless the logging package is configured to do -otherwise. +The default level is ``WARNING``, which means that only events of this severity and higher +will be tracked, unless the logging package is configured to do otherwise. Events that are tracked can be handled in different ways. The simplest way of handling tracked events is to print them to the console. Another common way @@ -116,12 +120,18 @@ If you type these lines into a script and run it, you'll see: WARNING:root:Watch out! printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - +default level is ``WARNING``. The printed message includes the indication of the +level and the description of the event provided in the logging call, i.e. +'Watch out!'. The actual output can be formatted quite flexibly if you need +that; formatting options will also be explained later. + +Notice that in this example, we use functions directly on the ``logging`` +module, like ``logging.debug``, rather than creating a logger and calling +functions on it. These functions operation on the root logger, but can be useful +as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in +this example. In larger programs you'll usually want to control the logging +configuration explicitly however - so for that reason as well as others, it's +better to create loggers and call their methods. Logging to a file ^^^^^^^^^^^^^^^^^ @@ -131,11 +141,12 @@ look at that next. Be sure to try the following in a newly started Python interpreter, and don't just continue from the session described above:: import logging + logger = logging.getLogger(__name__) logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + logger.debug('This message should go to the log file') + logger.info('So should this') + logger.warning('And this, too') + logger.error('And non-ASCII stuff, too, like Øresund and Malmö') .. versionchanged:: 3.9 The *encoding* argument was added. In earlier Python versions, or if not @@ -149,10 +160,10 @@ messages: .. code-block:: none - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö + DEBUG:__main__:This message should go to the log file + INFO:__main__:So should this + WARNING:__main__:And this, too + ERROR:__main__:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -181,11 +192,9 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. +The call to :func:`basicConfig` should come *before* any calls to a logger's +methods such as :meth:`~Logger.debug`, :meth:`~Logger.info`, etc. Otherwise, +that logging event may not be handled in the desired manner. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, @@ -198,50 +207,6 @@ The output will be the same as before, but the log file is no longer appended to, so the messages from earlier runs are lost. -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - Logging variable data ^^^^^^^^^^^^^^^^^^^^^ @@ -417,8 +382,48 @@ Logging Flow The flow of log event information in loggers and handlers is illustrated in the following diagram. -.. image:: logging_flow.png - :class: invert-in-dark-mode +.. raw:: html + :file: logging_flow.svg + +.. raw:: html + + <script> + /* + * This snippet is needed to handle the case where a light or dark theme is + * chosen via the theme is selected in the page. We call the existing handler + * and then add a dark-theme class to the body when the dark theme is selected. + * The SVG styling (above) then does the rest. + * + * If the pydoc theme is updated to set the dark-theme class, this snippet + * won't be needed any more. + */ + (function() { + var oldActivateTheme = activateTheme; + + function updateBody(theme) { + let elem = document.body; + + elem.classList.remove('dark-theme'); + elem.classList.remove('light-theme'); + if (theme === 'dark') { + elem.classList.add('dark-theme'); + } + else if (theme === 'light') { + elem.classList.add('light-theme'); + } + } + + activateTheme = function(theme) { + oldActivateTheme(theme); + updateBody(theme); + }; + /* + * If the page is refreshed, make sure we update the body - the overriding + * of activateTheme won't have taken effect yet. + */ + updateBody(localStorage.getItem('currentTheme') || 'auto'); + })(); + </script> Loggers ^^^^^^^ @@ -521,7 +526,7 @@ custom handlers) are the following configuration methods: * The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger + are there two :meth:`~Handler.setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. @@ -775,29 +780,29 @@ What happens if no configuration is provided If no logging configuration is provided, it is possible to have a situation where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. +output the event. -For versions of Python prior to 3.2, the behaviour is as follows: +The event is output using a 'handler of last resort', stored in +:data:`lastResort`. This internal handler is not associated with any +logger, and acts like a :class:`~logging.StreamHandler` which writes the +event description message to the current value of ``sys.stderr`` (therefore +respecting any redirections which may be in effect). No formatting is +done on the message - just the bare event description message is printed. +The handler's level is set to ``WARNING``, so all events at this and +greater severities will be output. -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. +.. versionchanged:: 3.2 -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. + For versions of Python prior to 3.2, the behaviour is as follows: -In Python 3.2 and later, the behaviour is as follows: + * If :data:`raiseExceptions` is ``False`` (production mode), the event is + silently dropped. -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. + * If :data:`raiseExceptions` is ``True`` (development mode), a message + 'No handlers could be found for logger X.Y.Z' is printed once. -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. + To obtain the pre-3.2 behaviour, + :data:`lastResort` can be set to ``None``. .. _library-config: @@ -999,7 +1004,7 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format +:class:`BufferingFormatter` can be used. In addition to the format string (which is applied to each message in the batch), there is provision for header and trailer format strings. @@ -1035,7 +1040,8 @@ checks to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is +.. note:: + The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. @@ -1073,7 +1079,7 @@ You can write code like this:: expensive_func2()) so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. +``expensive_func1`` and ``expensive_func2`` are never made. .. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more expensive than you'd like (e.g. for deeply nested loggers where an explicit diff --git a/Doc/howto/logging_flow.png b/Doc/howto/logging_flow.png index d65e597f811db5..d60ed7c031585a 100644 Binary files a/Doc/howto/logging_flow.png and b/Doc/howto/logging_flow.png differ diff --git a/Doc/howto/logging_flow.svg b/Doc/howto/logging_flow.svg new file mode 100644 index 00000000000000..4974994ac6b400 --- /dev/null +++ b/Doc/howto/logging_flow.svg @@ -0,0 +1,327 @@ +<svg width="22cm" height="23cm" viewBox="1 1 439 446" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> + <!-- Invert color in dark mode --> + <style type="text/css"> + svg { + background-color: transparent !important; + } + line { + stroke: #000000; + fill: none; + stroke-opacity: 1; + } + polygon, rect { + fill: none; + stroke: #000000; + fill-opacity: 1; + stroke-opacity: 1; + } + polygon.filled { + fill: #000000; + } + polyline { + fill: none; + stroke-opacity: 1; + stroke: #000000; + } + text { + fill: #000000; + fill-opacity: 1; + stroke: none; + font-family: sans-serif; + font-style: normal; + font-weight: normal; + text-anchor: start; + } + @media (prefers-color-scheme: dark) { + polygon, rect, polyline, line { + stroke: #ffffff; + } + polygon.filled { + fill: #ffffff; + } + text { + fill: #ffffff; + } + image { + filter: invert(100%) hue-rotate(180deg) saturate(1.25); + } + } + /* These rules are for when the theme selector is used, perhaps in contrast to the browser theme. */ + body.dark-theme polygon, body.dark-theme rect, body.dark-theme polyline, body.dark-theme line { + stroke: #ffffff; + } + body.dark-theme polygon.filled { + fill: #ffffff; + } + body.dark-theme text { + fill: #ffffff; + } + body.light-theme polygon, body.light-theme rect, body.light-theme polyline, body.light-theme line { + stroke: #000000; + } + body.light-theme polygon.filled { + fill: #000000; + } + body.light-theme text { + fill: #000000; + } + </style> + + <defs /> + <g id="Background"> + <rect style="stroke-width: 1.8; stroke-dasharray: 3;" x="227.504" y="1.91011" width="211.607" height="207.375" rx="0" ry="0" /> + <rect style="stroke-width: 1.2;" x="230.966" y="5.62454" width="68.4216" height="21.9166" rx="0" ry="0" /> + <rect style="stroke-width: 1.2;" x="5.27912" y="5.70399" width="68.4216" height="21.9166" rx="0" ry="0" /> + <text font-size="6.77333" style="font-weight: 700;" x="17.0649" y="19.0071"> + <tspan x="17.0649" y="19.0071">Logger flow</tspan> + </text> + <g> + <rect x="81.5533" y="106.469" width="44.45" height="25.9333" rx="0" ry="0" /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="117.256"> + <tspan x="103.778" y="117.256">Create</tspan> + <tspan x="103.778" y="125.723" style="font-family: monospace">LogRecord</tspan> + </text> + </g> + <g> + <line x1="103.778" y1="82.8734" x2="103.778" y2="102.351" /> + <polygon fill-rule="evenodd" points="103.778,105.351 101.778,101.351 103.778,102.351 105.778,101.351 " /> + </g> + <g> + <line x1="103.774" y1="3.65583" x2="103.778" y2="30.3755" /> + <polygon fill-rule="evenodd" points="103.778,33.3755 101.778,29.3758 103.778,30.3755 105.778,29.3752 " /> + </g> + <text font-size="6.77333" style="text-anchor: middle;" x="144.365" y="10.4604"> + <tspan x="143.798" y="10.4604">Logging call in user</tspan> + <tspan x="143.798" y="18.927">code, e.g.</tspan> + </text> + <text font-size="6.77333" style="font-family: monospace;" x="110.837" y="27.4696"> + <tspan x="111.893" y="27.4696">logger.info(...)</tspan> + </text> + <g> + <line x1="155.62" y1="58.6834" x2="183.943" y2="58.6924" /> + <polygon fill-rule="evenodd" points="186.943,58.6933 182.942,60.6921 183.943,58.6924 182.943,56.6921 " /> + </g> + <g> + <rect x="188.061" y="49.9604" width="24.4" height="17.4667" rx="6" ry="6" /> + <text font-size="6.77333" style="text-anchor: middle;" x="200.261" y="60.7475"> + <tspan x="200.261" y="60.7475">Stop</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="103.778,162.482 169.307,193.042 103.778,223.603 38.2493,193.042 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="186.629"> + <tspan x="103.778" y="188.741">Does a filter attached</tspan> + <tspan x="103.778" y="197.208">to logger reject the</tspan> + <tspan x="103.778" y="205.675">record?</tspan> + </text> + </g> + <g> + <line x1="103.778" y1="132.402" x2="103.778" y2="158.364" /> + <polygon class="filled" fill-rule="evenodd" points="103.778,161.364 101.778,157.364 103.778,158.364 105.778,157.364 " /> + </g> + <g> + <rect x="75.2033" y="249.478" width="57.15" height="34.4" rx="0" ry="0" /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="260.265"> + <tspan x="103.778" y="260.265">Pass record to</tspan> + <tspan x="103.778" y="268.732">handlers of</tspan> + <tspan x="103.778" y="277.198">current logger</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="103.778,326.569 158.193,352.399 103.778,378.229 49.3637,352.399 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="350.22"> + <tspan x="103.778" y="351.276">Is propagate true for</tspan> + <tspan x="103.778" y="359.742">current logger?</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="103.778,399.9 150.573,422.994 103.778,446.087 56.984,422.994 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="420.814"> + <tspan x="103.778" y="422.926">Is there a parent</tspan> + <tspan x="103.778" y="431.393">logger?</tspan> + </text> + </g> + <g> + <rect x="2.43852" y="295.984" width="63.9" height="26.5909" rx="0" ry="0" /> + <text font-size="6.77333" style="text-anchor: middle;" x="34.3885" y="307.1"> + <tspan x="34.3885" y="307.1">Set current</tspan> + <tspan x="34.3885" y="315.566">logger to parent</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="278.422,240.202 330.96,266.686 278.422,293.169 225.885,266.686 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="278.422" y="264.506"> + <tspan x="278.422" y="266.618">At least one handler</tspan> + <tspan x="278.422" y="275.085">in hierarchy?</tspan> + </text> + </g> + <g> + <rect x="298.963" y="312.257" width="57.75" height="25.9333" rx="0" ry="0" /> + <text font-size="6.77333" x="327.838" y="323.044"> + <tspan x="301" y="324.100">Use</tspan> + <tspan x="315" y="324.100" style="font-family: monospace">lastResort</tspan> + <tspan x="316" y="332.567">handler</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="320.041,35.7307 373.377,60.8536 320.041,85.9765 266.704,60.8536 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="320.041" y="58.6741"> + <tspan x="320.041" y="58.6741">Handler enabled for</tspan> + <tspan x="320.041" y="67.1407">level of record?</tspan> + </text> + </g> + <g> + <polygon fill-rule="evenodd" points="320.041,105.448 386.002,135.748 320.041,166.047 254.08,135.748 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="320.041" y="129.335"> + <tspan x="320.041" y="132.508">Does a filter attached</tspan> + <tspan x="320.041" y="140.970">to handler reject the</tspan> + <tspan x="320.041" y="149.436">record?</tspan> + </text> + </g> + <g> + <rect x="409.532" y="52.4436" width="24.4" height="17.4667" rx="6" ry="6" /> + <text font-size="6.77333" style="text-anchor: middle;" x="421.732" y="63.2307"> + <tspan x="421.732" y="63.2307">Stop</tspan> + </text> + </g> + <g> + <rect x="271.091" y="185.519" width="97.9" height="17.4667" rx="6" ry="6" /> + <text font-size="6.77333" style="text-anchor: middle;" x="320.041" y="196.306"> + <tspan x="320.041" y="196.306">Emit (includes formatting)</tspan> + </text> + </g> + <text font-size="6.77333" style="font-weight: 700;" x="241.002" y="18.9277"> + <tspan x="241.002" y="18.9277">Handler flow</tspan> + </text> + <g> + <polygon fill-rule="evenodd" points="103.778,34.4935 155.62,58.6834 103.778,82.8734 51.9368,58.6834 " /> + <text font-size="6.77333" style="text-anchor: middle;" x="103.778" y="56.5039"> + <tspan x="103.778" y="57.560">Logger enabled for</tspan> + <tspan x="103.778" y="66.026">level of call?</tspan> + </text> + </g> + <g> + <line x1="103.778" y1="223.603" x2="103.778" y2="245.36" /> + <polygon class="filled" fill-rule="evenodd" points="103.778,248.36 101.778,244.36 103.778,245.36 105.778,244.36 " /> + </g> + <g> + <line x1="103.778" y1="283.878" x2="103.778" y2="322.451" /> + <polygon class="filled" fill-rule="evenodd" points="103.778,325.451 101.778,321.451 103.778,322.451 105.778,321.451 " /> + </g> + <g> + <line x1="103.778" y1="378.229" x2="103.778" y2="395.782" /> + <polygon class="filled" fill-rule="evenodd" points="103.778,398.782 101.778,394.782 103.778,395.782 105.778,394.782 " /> + </g> + <g> + <polyline points="56.984,422.994 34.3885,422.994 34.3885,326.693 " /> + <polygon class="filled" fill-rule="evenodd" points="34.3885,323.693 36.3885,327.693 34.3885,326.693 32.3885,327.693 " /> + </g> + <g> + <polyline points="34.3885,295.984 34.3885,266.678 71.0853,266.678 " /> + <polygon class="filled" fill-rule="evenodd" points="74.0853,266.678 70.0853,268.678 71.0853,266.678 70.0853,264.678 " /> + </g> + <g> + <polyline points="150.573,422.994 200.261,422.994 200.261,71.5451 " /> + <polygon class="filled" fill-rule="evenodd" points="200.261,68.5451 202.261,72.5451 200.261,71.5451 198.261,72.5451 " /> + </g> + <g> + <line style="stroke-dasharray: 5" x1="132.353" y1="266.678" x2="221.767" y2="266.685" /> + <polygon class="filled" fill-rule="evenodd" points="224.767,266.686 220.766,268.685 221.767,266.685 220.767,264.685 " /> + </g> + <g> + <polyline style="stroke-dasharray: 5" points="278.422,293.169 278.422,325.224 294.845,325.224 " /> + <polygon class="filled" fill-rule="evenodd" points="297.845,325.224 293.845,327.224 294.845,325.224 293.845,323.224 " /> + </g> + <g> + <polyline points="169.307,193.042 200.261,193.042 200.261,71.5451 " /> + <polygon class="filled" fill-rule="evenodd" points="200.261,68.5451 202.261,72.5451 200.261,71.5451 198.261,72.5451 " /> + </g> + <g> + <polyline points="158.193,352.399 200.261,352.399 200.261,71.5451 " /> + <polygon class="filled" fill-rule="evenodd" points="200.261,68.5451 202.261,72.5451 200.261,71.5451 198.261,72.5451 " /> + </g> + <g> + <line x1="319.981" y1="4.27261" x2="320.033" y2="31.6127" /> + <polygon class="filled" fill-rule="evenodd" points="320.039,34.6127 318.031,30.6165 320.033,31.6127 322.031,30.6089 " /> + </g> + <g> + <line x1="320.041" y1="85.9765" x2="320.041" y2="101.33" /> + <polygon class="filled" fill-rule="evenodd" points="320.041,104.33 318.041,100.33 320.041,101.33 322.041,100.33 " /> + </g> + <g> + <line x1="320.041" y1="166.047" x2="320.041" y2="181.401" /> + <polygon class="filled" fill-rule="evenodd" points="320.041,184.401 318.041,180.401 320.041,181.401 322.041,180.401 " /> + </g> + <g> + <polyline points="386.002,135.748 421.732,135.748 421.732,74.0283 " /> + <polygon class="filled" fill-rule="evenodd" points="421.732,71.0283 423.732,75.0283 421.732,74.0283 419.732,75.0283 " /> + </g> + <g> + <line x1="373.377" y1="60.8536" x2="405.415" y2="61.1401" /> + <polygon class="filled" fill-rule="evenodd" points="408.414,61.1669 404.397,63.1311 405.415,61.1401 404.433,59.1312 " /> + </g> + <text font-size="6.77333" x="164.96" y="55.5649"> + <tspan x="164.96" y="55.5649">No</tspan> + </text> + <text font-size="6.77333" x="106.571" y="97.8453"> + <tspan x="106.571" y="97.8453">Yes</tspan> + </text> + <text font-size="6.77333" x="173.856" y="188.452"> + <tspan x="173.856" y="188.452">Yes</tspan> + </text> + <text font-size="6.77333" x="107.446" y="239.221"> + <tspan x="107.446" y="239.221">No</tspan> + </text> + <text font-size="6.77333" x="174.731" y="349.418"> + <tspan x="174.731" y="349.418">No</tspan> + </text> + <text font-size="6.77333" x="106.571" y="390.507"> + <tspan x="106.571" y="390.507">Yes</tspan> + </text> + <text font-size="6.77333" x="39.4722" y="417.667"> + <tspan x="39.4722" y="417.667">Yes</tspan> + </text> + <text font-size="6.77333" x="174.731" y="417.667"> + <tspan x="174.731" y="417.667">No</tspan> + </text> + <text font-size="6.77333" x="281.451" y="313.406"> + <tspan x="281.451" y="313.406">No</tspan> + </text> + <text font-size="6.77333" x="333.909" y="263.96"> + <tspan x="333.909" y="263.96">Yes</tspan> + </text> + <text font-size="6.77333" x="333.307" y="105.598"> + <tspan x="333.307" y="105.598"></tspan> + </text> + <text font-size="6.77333" x="385.766" y="56.9098"> + <tspan x="385.766" y="56.9098">No</tspan> + </text> + <text font-size="6.77333" x="333.307" y="105.598"> + <tspan x="333.307" y="105.598"></tspan> + </text> + <text font-size="6.77333" x="333.307" y="105.598"> + <tspan x="333.307" y="105.598"></tspan> + </text> + <text font-size="6.77333" x="397.102" y="130.471"> + <tspan x="397.102" y="130.471">Yes</tspan> + </text> + <text font-size="6.77333" x="323.563" y="178.785"> + <tspan x="323.563" y="178.785">No</tspan> + </text> + <text font-size="6.77333" x="333.307" y="105.598"> + <tspan x="333.307" y="105.598"></tspan> + </text> + <text font-size="6.77333" x="323.75" y="99.0042"> + <tspan x="323.75" y="99.0042">Yes</tspan> + </text> + <text font-size="6.77323" style="text-anchor: middle;" x="355.762" y="18.2449"> + <tspan x="355.762" y="18.2449">Record passed</tspan> + <tspan x="355.762" y="26.7116">to handler</tspan> + </text> + <line style="stroke-dasharray: 5" x1="330.96" y1="266.686" x2="377.733" y2="267.908" /> + <g> + <polyline style="fill: none; stroke-opacity: 1; stroke-dasharray: 5" points="356.713,325.224 377.733,325.224 377.733,214.711 " /> + <polygon class="filled" fill-rule="evenodd" points="377.733,211.711 379.733,215.711 377.733,214.711 375.733,215.711 " /> + </g> + </g> +</svg> diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst new file mode 100644 index 00000000000000..f44b4f98e570bd --- /dev/null +++ b/Doc/howto/mro.rst @@ -0,0 +1,671 @@ +.. _python_2.3_mro: + +The Python 2.3 Method Resolution Order +====================================== + +.. note:: + + This is a historical document, provided as an appendix to the official + documentation. + The Method Resolution Order discussed here was *introduced* in Python 2.3, + but it is still used in later versions -- including Python 3. + +By `Michele Simionato <https://www.phyast.pitt.edu/~micheles/>`__. + +:Abstract: + + *This document is intended for Python programmers who want to + understand the C3 Method Resolution Order used in Python 2.3. + Although it is not intended for newbies, it is quite pedagogical with + many worked out examples. I am not aware of other publicly available + documents with the same scope, therefore it should be useful.* + +Disclaimer: + + *I donate this document to the Python Software Foundation, under the + Python 2.3 license. As usual in these circumstances, I warn the + reader that what follows* should *be correct, but I don't give any + warranty. Use it at your own risk and peril!* + +Acknowledgments: + + *All the people of the Python mailing list who sent me their support. + Paul Foley who pointed out various imprecisions and made me to add the + part on local precedence ordering. David Goodger for help with the + formatting in reStructuredText. David Mertz for help with the editing. + Finally, Guido van Rossum who enthusiastically added this document to + the official Python 2.3 home-page.* + +The beginning +------------- + + *Felix qui potuit rerum cognoscere causas* -- Virgilius + +Everything started with a post by Samuele Pedroni to the Python +development mailing list [#]_. In his post, Samuele showed that the +Python 2.2 method resolution order is not monotonic and he proposed to +replace it with the C3 method resolution order. Guido agreed with his +arguments and therefore now Python 2.3 uses C3. The C3 method itself +has nothing to do with Python, since it was invented by people working +on Dylan and it is described in a paper intended for lispers [#]_. The +present paper gives a (hopefully) readable discussion of the C3 +algorithm for Pythonistas who want to understand the reasons for the +change. + +First of all, let me point out that what I am going to say only applies +to the *new style classes* introduced in Python 2.2: *classic classes* +maintain their old method resolution order, depth first and then left to +right. Therefore, there is no breaking of old code for classic classes; +and even if in principle there could be breaking of code for Python 2.2 +new style classes, in practice the cases in which the C3 resolution +order differs from the Python 2.2 method resolution order are so rare +that no real breaking of code is expected. Therefore: + + *Don't be scared!* + +Moreover, unless you make strong use of multiple inheritance and you +have non-trivial hierarchies, you don't need to understand the C3 +algorithm, and you can easily skip this paper. On the other hand, if +you really want to know how multiple inheritance works, then this paper +is for you. The good news is that things are not as complicated as you +might expect. + +Let me begin with some basic definitions. + +1) Given a class C in a complicated multiple inheritance hierarchy, it + is a non-trivial task to specify the order in which methods are + overridden, i.e. to specify the order of the ancestors of C. + +2) The list of the ancestors of a class C, including the class itself, + ordered from the nearest ancestor to the furthest, is called the + class precedence list or the *linearization* of C. + +3) The *Method Resolution Order* (MRO) is the set of rules that + construct the linearization. In the Python literature, the idiom + "the MRO of C" is also used as a synonymous for the linearization of + the class C. + +4) For instance, in the case of single inheritance hierarchy, if C is a + subclass of C1, and C1 is a subclass of C2, then the linearization of + C is simply the list [C, C1 , C2]. However, with multiple + inheritance hierarchies, the construction of the linearization is + more cumbersome, since it is more difficult to construct a + linearization that respects *local precedence ordering* and + *monotonicity*. + +5) I will discuss the local precedence ordering later, but I can give + the definition of monotonicity here. A MRO is monotonic when the + following is true: *if C1 precedes C2 in the linearization of C, + then C1 precedes C2 in the linearization of any subclass of C*. + Otherwise, the innocuous operation of deriving a new class could + change the resolution order of methods, potentially introducing very + subtle bugs. Examples where this happens will be shown later. + +6) Not all classes admit a linearization. There are cases, in + complicated hierarchies, where it is not possible to derive a class + such that its linearization respects all the desired properties. + +Here I give an example of this situation. Consider the hierarchy + + >>> O = object + >>> class X(O): pass + >>> class Y(O): pass + >>> class A(X,Y): pass + >>> class B(Y,X): pass + +which can be represented with the following inheritance graph, where I +have denoted with O the ``object`` class, which is the beginning of any +hierarchy for new style classes: + + .. code-block:: text + + ----------- + | | + | O | + | / \ | + - X Y / + | / | / + | / |/ + A B + \ / + ? + +In this case, it is not possible to derive a new class C from A and B, +since X precedes Y in A, but Y precedes X in B, therefore the method +resolution order would be ambiguous in C. + +Python 2.3 raises an exception in this situation (TypeError: MRO +conflict among bases Y, X) forbidding the naive programmer from creating +ambiguous hierarchies. Python 2.2 instead does not raise an exception, +but chooses an *ad hoc* ordering (CABXYO in this case). + +The C3 Method Resolution Order +------------------------------ + +Let me introduce a few simple notations which will be useful for the +following discussion. I will use the shortcut notation:: + + C1 C2 ... CN + +to indicate the list of classes [C1, C2, ... , CN]. + +The *head* of the list is its first element:: + + head = C1 + +whereas the *tail* is the rest of the list:: + + tail = C2 ... CN. + +I shall also use the notation:: + + C + (C1 C2 ... CN) = C C1 C2 ... CN + +to denote the sum of the lists [C] + [C1, C2, ... ,CN]. + +Now I can explain how the MRO works in Python 2.3. + +Consider a class C in a multiple inheritance hierarchy, with C +inheriting from the base classes B1, B2, ... , BN. We want to +compute the linearization L[C] of the class C. The rule is the +following: + + *the linearization of C is the sum of C plus the merge of the + linearizations of the parents and the list of the parents.* + +In symbolic notation:: + + L[C(B1 ... BN)] = C + merge(L[B1] ... L[BN], B1 ... BN) + +In particular, if C is the ``object`` class, which has no parents, the +linearization is trivial:: + + L[object] = object. + +However, in general one has to compute the merge according to the following +prescription: + + *take the head of the first list, i.e L[B1][0]; if this head is not in + the tail of any of the other lists, then add it to the linearization + of C and remove it from the lists in the merge, otherwise look at the + head of the next list and take it, if it is a good head. Then repeat + the operation until all the class are removed or it is impossible to + find good heads. In this case, it is impossible to construct the + merge, Python 2.3 will refuse to create the class C and will raise an + exception.* + +This prescription ensures that the merge operation *preserves* the +ordering, if the ordering can be preserved. On the other hand, if the +order cannot be preserved (as in the example of serious order +disagreement discussed above) then the merge cannot be computed. + +The computation of the merge is trivial if C has only one parent +(single inheritance); in this case:: + + L[C(B)] = C + merge(L[B],B) = C + L[B] + +However, in the case of multiple inheritance things are more cumbersome +and I don't expect you can understand the rule without a couple of +examples ;-) + +Examples +-------- + +First example. Consider the following hierarchy: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(D,E): pass + >>> class A(B,C): pass + +In this case the inheritance graph can be drawn as: + + .. code-block:: text + + 6 + --- + Level 3 | O | (more general) + / --- \ + / | \ | + / | \ | + / | \ | + --- --- --- | + Level 2 3 | D | 4| E | | F | 5 | + --- --- --- | + \ \ _ / | | + \ / \ _ | | + \ / \ | | + --- --- | + Level 1 1 | B | | C | 2 | + --- --- | + \ / | + \ / \ / + --- + Level 0 0 | A | (more specialized) + --- + + +The linearizations of O,D,E and F are trivial:: + + L[O] = O + L[D] = D O + L[E] = E O + L[F] = F O + +The linearization of B can be computed as:: + + L[B] = B + merge(DO, EO, DE) + +We see that D is a good head, therefore we take it and we are reduced to +compute ``merge(O,EO,E)``. Now O is not a good head, since it is in the +tail of the sequence EO. In this case the rule says that we have to +skip to the next sequence. Then we see that E is a good head; we take +it and we are reduced to compute ``merge(O,O)`` which gives O. Therefore:: + + L[B] = B D E O + +Using the same procedure one finds:: + + L[C] = C + merge(DO,FO,DF) + = C + D + merge(O,FO,F) + = C + D + F + merge(O,O) + = C D F O + +Now we can compute:: + + L[A] = A + merge(BDEO,CDFO,BC) + = A + B + merge(DEO,CDFO,C) + = A + B + C + merge(DEO,DFO) + = A + B + C + D + merge(EO,FO) + = A + B + C + D + E + merge(O,FO) + = A + B + C + D + E + F + merge(O,O) + = A B C D E F O + +In this example, the linearization is ordered in a pretty nice way +according to the inheritance level, in the sense that lower levels (i.e. +more specialized classes) have higher precedence (see the inheritance +graph). However, this is not the general case. + +I leave as an exercise for the reader to compute the linearization for +my second example: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(E,D): pass + >>> class A(B,C): pass + +The only difference with the previous example is the change B(D,E) --> +B(E,D); however even such a little modification completely changes the +ordering of the hierarchy: + + .. code-block:: text + + 6 + --- + Level 3 | O | + / --- \ + / | \ + / | \ + / | \ + --- --- --- + Level 2 2 | E | 4 | D | | F | 5 + --- --- --- + \ / \ / + \ / \ / + \ / \ / + --- --- + Level 1 1 | B | | C | 3 + --- --- + \ / + \ / + --- + Level 0 0 | A | + --- + + +Notice that the class E, which is in the second level of the hierarchy, +precedes the class C, which is in the first level of the hierarchy, i.e. +E is more specialized than C, even if it is in a higher level. + +A lazy programmer can obtain the MRO directly from Python 2.2, since in +this case it coincides with the Python 2.3 linearization. It is enough +to invoke the .mro() method of class A: + + >>> A.mro() # doctest: +NORMALIZE_WHITESPACE + [<class 'A'>, <class 'B'>, <class 'E'>, + <class 'C'>, <class 'D'>, <class 'F'>, + <class 'object'>] + +Finally, let me consider the example discussed in the first section, +involving a serious order disagreement. In this case, it is +straightforward to compute the linearizations of O, X, Y, A and B: + + .. code-block:: text + + L[O] = 0 + L[X] = X O + L[Y] = Y O + L[A] = A X Y O + L[B] = B Y X O + +However, it is impossible to compute the linearization for a class C +that inherits from A and B:: + + L[C] = C + merge(AXYO, BYXO, AB) + = C + A + merge(XYO, BYXO, B) + = C + A + B + merge(XYO, YXO) + +At this point we cannot merge the lists XYO and YXO, since X is in the +tail of YXO whereas Y is in the tail of XYO: therefore there are no +good heads and the C3 algorithm stops. Python 2.3 raises an error and +refuses to create the class C. + +Bad Method Resolution Orders +---------------------------- + +A MRO is *bad* when it breaks such fundamental properties as local +precedence ordering and monotonicity. In this section, I will show +that both the MRO for classic classes and the MRO for new style classes +in Python 2.2 are bad. + +It is easier to start with the local precedence ordering. Consider the +following example: + + >>> F=type('Food',(),{'remember2buy':'spam'}) + >>> E=type('Eggs',(F,),{'remember2buy':'eggs'}) + >>> G=type('GoodFood',(F,E),{}) # under Python 2.3 this is an error! # doctest: +SKIP + +with inheritance diagram + + .. code-block:: text + + O + | + (buy spam) F + | \ + | E (buy eggs) + | / + G + + (buy eggs or spam ?) + + +We see that class G inherits from F and E, with F *before* E: therefore +we would expect the attribute *G.remember2buy* to be inherited by +*F.rembermer2buy* and not by *E.remember2buy*: nevertheless Python 2.2 +gives + + >>> G.remember2buy # doctest: +SKIP + 'eggs' + +This is a breaking of local precedence ordering since the order in the +local precedence list, i.e. the list of the parents of G, is not +preserved in the Python 2.2 linearization of G:: + + L[G,P22]= G E F object # F *follows* E + +One could argue that the reason why F follows E in the Python 2.2 +linearization is that F is less specialized than E, since F is the +superclass of E; nevertheless the breaking of local precedence ordering +is quite non-intuitive and error prone. This is particularly true since +it is a different from old style classes: + + >>> class F: remember2buy='spam' + >>> class E(F): remember2buy='eggs' + >>> class G(F,E): pass # doctest: +SKIP + >>> G.remember2buy # doctest: +SKIP + 'spam' + +In this case the MRO is GFEF and the local precedence ordering is +preserved. + +As a general rule, hierarchies such as the previous one should be +avoided, since it is unclear if F should override E or vice-versa. +Python 2.3 solves the ambiguity by raising an exception in the creation +of class G, effectively stopping the programmer from generating +ambiguous hierarchies. The reason for that is that the C3 algorithm +fails when the merge:: + + merge(FO,EFO,FE) + +cannot be computed, because F is in the tail of EFO and E is in the tail +of FE. + +The real solution is to design a non-ambiguous hierarchy, i.e. to derive +G from E and F (the more specific first) and not from F and E; in this +case the MRO is GEF without any doubt. + + .. code-block:: text + + O + | + F (spam) + / | + (eggs) E | + \ | + G + (eggs, no doubt) + + +Python 2.3 forces the programmer to write good hierarchies (or, at +least, less error-prone ones). + +On a related note, let me point out that the Python 2.3 algorithm is +smart enough to recognize obvious mistakes, as the duplication of +classes in the list of parents: + + >>> class A(object): pass + >>> class C(A,A): pass # error + Traceback (most recent call last): + File "<stdin>", line 1, in ? + TypeError: duplicate base class A + +Python 2.2 (both for classic classes and new style classes) in this +situation, would not raise any exception. + +Finally, I would like to point out two lessons we have learned from this +example: + +1. despite the name, the MRO determines the resolution order of + attributes, not only of methods; + +2. the default food for Pythonistas is spam ! (but you already knew + that ;-) + +Having discussed the issue of local precedence ordering, let me now +consider the issue of monotonicity. My goal is to show that neither the +MRO for classic classes nor that for Python 2.2 new style classes is +monotonic. + +To prove that the MRO for classic classes is non-monotonic is rather +trivial, it is enough to look at the diamond diagram: + + .. code-block:: text + + + C + / \ + / \ + A B + \ / + \ / + D + +One easily discerns the inconsistency:: + + L[B,P21] = B C # B precedes C : B's methods win + L[D,P21] = D A C B C # B follows C : C's methods win! + +On the other hand, there are no problems with the Python 2.2 and 2.3 +MROs, they give both:: + + L[D] = D A B C + +Guido points out in his essay [#]_ that the classic MRO is not so bad in +practice, since one can typically avoids diamonds for classic classes. +But all new style classes inherit from ``object``, therefore diamonds are +unavoidable and inconsistencies shows up in every multiple inheritance +graph. + +The MRO of Python 2.2 makes breaking monotonicity difficult, but not +impossible. The following example, originally provided by Samuele +Pedroni, shows that the MRO of Python 2.2 is non-monotonic: + + >>> class A(object): pass + >>> class B(object): pass + >>> class C(object): pass + >>> class D(object): pass + >>> class E(object): pass + >>> class K1(A,B,C): pass + >>> class K2(D,B,E): pass + >>> class K3(D,A): pass + >>> class Z(K1,K2,K3): pass + +Here are the linearizations according to the C3 MRO (the reader should +verify these linearizations as an exercise and draw the inheritance +diagram ;-) :: + + L[A] = A O + L[B] = B O + L[C] = C O + L[D] = D O + L[E] = E O + L[K1]= K1 A B C O + L[K2]= K2 D B E O + L[K3]= K3 D A O + L[Z] = Z K1 K2 K3 D A B C E O + +Python 2.2 gives exactly the same linearizations for A, B, C, D, E, K1, +K2 and K3, but a different linearization for Z:: + + L[Z,P22] = Z K1 K3 A K2 D B C E O + +It is clear that this linearization is *wrong*, since A comes before D +whereas in the linearization of K3 A comes *after* D. In other words, in +K3 methods derived by D override methods derived by A, but in Z, which +still is a subclass of K3, methods derived by A override methods derived +by D! This is a violation of monotonicity. Moreover, the Python 2.2 +linearization of Z is also inconsistent with local precedence ordering, +since the local precedence list of the class Z is [K1, K2, K3] (K2 +precedes K3), whereas in the linearization of Z K2 *follows* K3. These +problems explain why the 2.2 rule has been dismissed in favor of the C3 +rule. + +The end +------- + +This section is for the impatient reader, who skipped all the previous +sections and jumped immediately to the end. This section is for the +lazy programmer too, who didn't want to exercise her/his brain. +Finally, it is for the programmer with some hubris, otherwise s/he would +not be reading a paper on the C3 method resolution order in multiple +inheritance hierarchies ;-) These three virtues taken all together (and +*not* separately) deserve a prize: the prize is a short Python 2.2 +script that allows you to compute the 2.3 MRO without risk to your +brain. Simply change the last line to play with the various examples I +have discussed in this paper.:: + + #<mro.py> + + """C3 algorithm by Samuele Pedroni (with readability enhanced by me).""" + + class __metaclass__(type): + "All classes are metamagically modified to be nicely printed" + __repr__ = lambda cls: cls.__name__ + + class ex_2: + "Serious order disagreement" #From Guido + class O: pass + class X(O): pass + class Y(O): pass + class A(X,Y): pass + class B(Y,X): pass + try: + class Z(A,B): pass #creates Z(A,B) in Python 2.2 + except TypeError: + pass # Z(A,B) cannot be created in Python 2.3 + + class ex_5: + "My first example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(D,E): pass + class A(B,C): pass + + class ex_6: + "My second example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(E,D): pass + class A(B,C): pass + + class ex_9: + "Difference between Python 2.2 MRO and C3" #From Samuele + class O: pass + class A(O): pass + class B(O): pass + class C(O): pass + class D(O): pass + class E(O): pass + class K1(A,B,C): pass + class K2(D,B,E): pass + class K3(D,A): pass + class Z(K1,K2,K3): pass + + def merge(seqs): + print '\n\nCPL[%s]=%s' % (seqs[0][0],seqs), + res = []; i=0 + while 1: + nonemptyseqs=[seq for seq in seqs if seq] + if not nonemptyseqs: return res + i+=1; print '\n',i,'round: candidates...', + for seq in nonemptyseqs: # find merge candidates among seq heads + cand = seq[0]; print ' ',cand, + nothead=[s for s in nonemptyseqs if cand in s[1:]] + if nothead: cand=None #reject candidate + else: break + if not cand: raise "Inconsistent hierarchy" + res.append(cand) + for seq in nonemptyseqs: # remove cand + if seq[0] == cand: del seq[0] + + def mro(C): + "Compute the class precedence list (mro) according to C3" + return merge([[C]]+map(mro,C.__bases__)+[list(C.__bases__)]) + + def print_mro(C): + print '\nMRO[%s]=%s' % (C,mro(C)) + print '\nP22 MRO[%s]=%s' % (C,C.mro()) + + print_mro(ex_9.Z) + + #</mro.py> + +That's all folks, + + enjoy ! + + +Resources +--------- + +.. [#] The thread on python-dev started by Samuele Pedroni: + https://mail.python.org/pipermail/python-dev/2002-October/029035.html + +.. [#] The paper *A Monotonic Superclass Linearization for Dylan*: + https://doi.org/10.1145/236337.236343 + +.. [#] Guido van Rossum's essay, *Unifying types and classes in Python 2.2*: + https://web.archive.org/web/20140210194412/http://www.python.org/download/releases/2.2.2/descrintro diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index af7b67d2042950..06459d1b222964 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -97,7 +97,7 @@ Then we can use ``perf report`` to analyze the data: | | | | | |--2.97%--_PyObject_Malloc ... -As you can see, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` +As you can see, the Python functions are not shown in the output, only ``_PyEval_EvalFrameDefault`` (the function that evaluates the Python bytecode) shows up. Unfortunately that's not very useful because all Python functions use the same C function to evaluate bytecode so we cannot know which Python function corresponds to which bytecode-evaluating function. @@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: ...then:: - $ python ./example.py + $ perf record -F 9999 -g -o perf.data python ./example.py $ perf report -g -i perf.data @@ -205,3 +205,62 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without +frame pointers, you can still use the ``perf`` profiler, but the overhead will be +a bit higher because Python needs to generate unwinding information for every +Python function call on the fly. Additionally, ``perf`` will take more time to +process the data because it will need to use the DWARF debugging information to +unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable +:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option, +which will enable the JIT mode for the ``perf`` profiler. + +.. note:: + + Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8 + will work with the JIT mode. The fix was also backported to the v6.7.2 + version of the tool. + + Note that when checking the version of the ``perf`` tool (which can be done + by running ``perf version``) you must take into account that some distros + add some custom version numbers including a ``-`` character. This means + that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``. + +When using the perf JIT mode, you need an extra step before you can run ``perf +report``. You need to call the ``perf inject`` command to inject the JIT +information into the ``perf.data`` file.:: + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +or using the environment variable:: + + $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +``perf inject --jit`` command will read ``perf.data``, +automatically pick up the perf dump file that Python creates (in +``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the +JIT information together. It should also create a lot of ``jitted-XXXX-N.so`` +files in the current directory which are ELF images for all the JIT trampolines +that were created by Python. + +.. warning:: + Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + snapshots of the stack of the process being profiled and save the + information in the ``perf.data`` file. By default the size of the stack dump + is 8192 bytes but the user can change the size by passing the size after + comma like ``--call-graph dwarf,4096``. The size of the stack dump is + important because if the size is too small ``perf`` will not be able to + unwind the stack and the output will be incomplete. On the other hand, if + the size is too big, then ``perf`` won't be able to sample the process as + frequently as it would like as the overhead will be higher. + diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 6c30a0dd7d6bcc..9f73c811cfcbc0 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -1,3 +1,5 @@ +:orphan: + .. _pyporting-howto: ************************************* @@ -6,422 +8,30 @@ How to port Python 2 Code to Python 3 :author: Brett Cannon -.. topic:: Abstract - - Python 2 reached its official end-of-life at the start of 2020. This means - that no new bug reports, fixes, or changes will be made to Python 2 - it's - no longer supported. - - This guide is intended to provide you with a path to Python 3 for your - code, that includes compatibility with Python 2 as a first step. - - If you are looking to port an extension module instead of pure Python code, - please see :ref:`cporting-howto`. - - The archived python-porting_ mailing list may contain some useful guidance. - - -The Short Explanation -===================== - -To achieve Python 2/3 compatibility in a single code base, the basic steps -are: - -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``python -m pip install coverage``) -#. Learn the differences between Python 2 and 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``python -m pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``python -m pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 and 3 (tox_ can help test - against multiple versions of Python; ``python -m pip install tox``) -#. Consider using optional static type checking to make sure your type usage - works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both - Python 2 and Python 3; ``python -m pip install mypy``). - -.. note:: - - Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke - is the one installed for the Python currently in use, whether it be - a system-wide ``pip`` or one installed within a - :ref:`virtual environment <tut-venv>`. - -Details -======= - -Even if other factors - say, dependencies over which you have no control - -still require you to support Python 2, that does not prevent you taking the -step of including Python 3 support. - -Most changes required to support Python 3 lead to cleaner code using newer -practices even in Python 2 code. - - -Different versions of Python 2 ------------------------------- - -Ideally, your code should be compatible with Python 2.7, which was the -last supported version of Python 2. - -Some of the tools mentioned in this guide will not work with Python 2.6. - -If absolutely necessary, the six_ project can help you support Python 2.5 and -3 simultaneously. Do realize, though, that nearly all the projects listed in -this guide will not be available to you. - -If you are able to skip Python 2.5 and older, the required changes to your -code will be minimal. At worst you will have to use a function instead of a -method in some instances or have to import a function instead of using a -built-in one. - - -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- - -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. - - -Have good test coverage ------------------------ - -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you find it hard to get better than 90% -coverage). If you don't already have a tool to measure test coverage then -coverage.py_ is recommended. - - -Be aware of the differences between Python 2 and 3 --------------------------------------------------- - -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. - -Some resources for understanding the differences and their implications for you -code: - -* the :ref:`"What's New" <whatsnew-index>` doc for each release of Python 3 -* the `Porting to Python 3`_ book (which is free online) -* the handy `cheat sheet`_ from the Python-Future project. - - -Update your code ----------------- - -There are tools available that can port your code automatically. - -Futurize_ does its best to make Python 3 idioms and practices exist in Python -2, e.g. backporting the ``bytes`` type from Python 3 so that you have -semantic parity between the major versions of Python. This is the better -approach for most cases. - -Modernize_, on the other hand, is more conservative and targets a Python 2/3 -subset of Python, directly relying on six_ to help provide compatibility. - -A good approach is to run the tool over your test suite first and visually -inspect the diff to make sure the transformation is accurate. After you have -transformed your test suite and verified that all the tests still pass as -expected, then you can transform your application code knowing that any tests -which fail is a translation failure. - -Unfortunately the tools can't automate everything to make your code work under -Python 3, and you will also need to read the tools' documentation in case some -options you need are turned off by default. - -Key issues to be aware of and check for: - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all -division between ``int`` values result in a ``float``. This change has -actually been planned since Python 2.2 which was released in 2002. Since then -users have been encouraged to add ``from __future__ import division`` to any -and all files which use the ``/`` and ``//`` operators or to be running the -interpreter with the ``-Q`` flag. If you have not been doing this then you -will need to go through your code and do two things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). +Python 2 reached its official end-of-life at the start of 2020. This means +that no new bug reports, fixes, or changes will be made to Python 2 - it's +no longer supported: see :pep:`373` and +`status of Python versions <https://devguide.python.org/versions>`_. +If you are looking to port an extension module instead of pure Python code, +please see :ref:`cporting-howto`. -Text versus binary data -+++++++++++++++++++++++ +The archived python-porting_ mailing list may contain some useful guidance. -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. +Since Python 3.11 the original porting guide was discontinued. +You can find the old guide in the +`archive <https://docs.python.org/3.10/howto/pyporting.html>`_. -Python 3 made text and binary data distinct types that cannot simply be mixed -together. For any code that deals only with text or only binary data, this -separation doesn't pose an issue. But for code that has to deal with both, it -does mean you might have to now care about when you are using text compared -to binary data, which is why this cannot be entirely automated. -Decide which APIs take text and which take binary (it is **highly** recommended -you don't design APIs that can take both due to the difficulty of keeping the -code working; as stated earlier it is difficult to do well). In Python 2 this -means making sure the APIs that take text can work with ``unicode`` and those -that work with binary data work with the ``bytes`` type from Python 3 -(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` -type in Python 2). Usually the biggest issue is realizing which methods exist -on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` -in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in -Python 2 and ``bytes`` in Python 3). +Third-party guides +================== -The following table lists the **unique** methods of each data type across -Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary -data type in either Python 2 or 3, but it can't be used by the textual data -type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't -have the method). Do note that as of Python 3.5 the ``__mod__`` method was -added to the bytes type. +There are also multiple third-party guides that might be useful: -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== +- `Guide by Fedora <https://portingguide.readthedocs.io>`_ +- `PyCon 2020 tutorial <https://www.youtube.com/watch?v=JgIgEjASOlk>`_ +- `Guide by DigitalOcean <https://www.digitalocean.com/community/tutorials/how-to-port-python-2-code-to-python-3>`_ +- `Guide by ActiveState <https://www.activestate.com/blog/how-to-migrate-python-2-applications-to-python-3>`_ -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. -The next issue is making sure you know whether the string literals in your code -represent text or binary data. You should add a ``b`` prefix to any -literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (There is a :mod:`__future__` import to force all unspecified -literals to be Unicode, but usage has shown it isn't as effective as adding a -``b`` or ``u`` prefix to all literals explicitly) - -You also need to be careful about opening files. Possibly you have not always -bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for -binary reading). Under Python 3, binary files and text files are clearly -distinct and mutually incompatible; see the :mod:`io` module for details. -Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing binary data to be read and/or written) or textual access -(allowing text data to be read and/or written). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). Do not bother with the -outdated practice of using :func:`codecs.open` as that's only necessary for -keeping compatibility with Python 2.5. - -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. - -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. - -To summarize: - -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, textual literals with a ``u`` - prefix -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing into binary data - - -Use feature detection instead of version detection -++++++++++++++++++++++++++++++++++++++++++++++++++ - -Inevitably you will have code that has to choose what to do based on what -version of Python is running. The best way to do this is with feature detection -of whether the version of Python you're running under supports what you need. -If for some reason that doesn't work then you should make the version check be -against Python 2 and not Python 3. To help explain this, let's look at an -example. - -Let's pretend that you need access to a feature of :mod:`importlib` that -is available in Python's standard library since Python 3.3 and available for -Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the :mod:`importlib.abc` module by doing the following:: - - import sys - - if sys.version_info[0] == 3: - from importlib import abc - else: - from importlib2 import abc - -The problem with this code is what happens when Python 4 comes out? It would -be better to treat Python 2 as the exceptional case instead of Python 3 and -assume that future Python versions will be more compatible with Python 3 than -Python 2:: - - import sys - - if sys.version_info[0] > 2: - from importlib import abc - else: - from importlib2 import abc - -The best solution, though, is to do no version detection at all and instead rely -on feature detection. That avoids any potential issues of getting the version -detection wrong and helps keep you future-compatible:: - - try: - from importlib import abc - except ImportError: - from importlib2 import abc - - -Prevent compatibility regressions ---------------------------------- - -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. - -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. - -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. - - -Check which dependencies block your transition ----------------------------------------------- - -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com. - -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. - - -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- - -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell anyone using your code that you -support Python 2 **and** 3. Ideally you will also want to add classifiers for -each major/minor version of Python you now support. - - -Use continuous integration to stay compatible ---------------------------------------------- - -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 and 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. - -You may also want to use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. - - -Consider using optional static type checking --------------------------------------------- - -Another way to help port your code is to use a static type checker like -mypy_ or pytype_ on your code. These tools can be used to analyze your code as -if it's being run under Python 2, then you can run the tool a second time as if -your code is running under Python 3. By running a static type checker twice like -this you can discover if you're e.g. misusing binary data type in one version -of Python compared to another. If you add optional type hints to your code you -can also explicitly state whether your APIs use textual or binary data, helping -to make sure everything functions as expected in both versions of Python. - - -.. _caniusepython3: https://pypi.org/project/caniusepython3 -.. _cheat sheet: https://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.org/project/coverage -.. _Futurize: https://python-future.org/automatic_conversion.html -.. _importlib2: https://pypi.org/project/importlib2 -.. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: https://mypy-lang.org/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.org/project/pylint - -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html - -.. _pytype: https://github.com/google/pytype -.. _python-future: https://python-future.org/ .. _python-porting: https://mail.python.org/pipermail/python-porting/ -.. _six: https://pypi.org/project/six -.. _tox: https://pypi.org/project/tox -.. _trove classifier: https://pypi.org/classifiers - -.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index c19c48301f5848..5e2f9a9d1837fe 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -245,6 +245,9 @@ You can omit either *m* or *n*; in that case, a reasonable value is assumed for the missing value. Omitting *m* is interpreted as a lower limit of 0, while omitting *n* results in an upper bound of infinity. +The simplest case ``{m}`` matches the preceding item exactly *m* times. +For example, ``a/{2}b`` will only match ``'a//b'``. + Readers of a reductionist bent may notice that the three other quantifiers can all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 38dd09f0a721d2..b98f91e023bdfc 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -1,10 +1,9 @@ .. _sortinghowto: -Sorting HOW TO -************** +Sorting Techniques +****************** :Author: Andrew Dalke and Raymond Hettinger -:Release: 0.1 Python lists have a built-in :meth:`list.sort` method that modifies the list @@ -56,7 +55,7 @@ For example, here's a case-insensitive string comparison: .. doctest:: - >>> sorted("This is a test string from Andrew".split(), key=str.lower) + >>> sorted("This is a test string from Andrew".split(), key=str.casefold) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] The value of the *key* parameter should be a function (or other callable) that @@ -97,10 +96,14 @@ The same technique works for objects with named attributes. For example: >>> sorted(student_objects, key=lambda student: student.age) # sort by age [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] -Operator Module Functions -========================= +Objects with named attributes can be made by a regular class as shown +above, or they can be instances of :class:`~dataclasses.dataclass` or +a :term:`named tuple`. -The key-function patterns shown above are very common, so Python provides +Operator Module Functions and Partial Function Evaluation +========================================================= + +The :term:`key function` patterns shown above are very common, so Python provides convenience functions to make accessor functions easier and faster. The :mod:`operator` module has :func:`~operator.itemgetter`, :func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. @@ -128,6 +131,24 @@ sort by *grade* then by *age*: >>> sorted(student_objects, key=attrgetter('grade', 'age')) [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] +The :mod:`functools` module provides another helpful tool for making +key-functions. The :func:`~functools.partial` function can reduce the +`arity <https://en.wikipedia.org/wiki/Arity>`_ of a multi-argument +function making it suitable for use as a key-function. + +.. doctest:: + + >>> from functools import partial + >>> from unicodedata import normalize + + >>> names = 'Zoë Åbjørn Núñez Élana Zeke Abe Nubia Eloise'.split() + + >>> sorted(names, key=partial(normalize, 'NFD')) + ['Abe', 'Åbjørn', 'Eloise', 'Élana', 'Nubia', 'Núñez', 'Zeke', 'Zoë'] + + >>> sorted(names, key=partial(normalize, 'NFC')) + ['Abe', 'Eloise', 'Nubia', 'Núñez', 'Zeke', 'Zoë', 'Åbjørn', 'Élana'] + Ascending and Descending ======================== @@ -200,6 +221,8 @@ This idiom is called Decorate-Sort-Undecorate after its three steps: For example, to sort the student data by *grade* using the DSU approach: +.. doctest:: + >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] >>> decorated.sort() >>> [student for grade, i, student in decorated] # undecorate @@ -282,7 +305,11 @@ Odds and Ends [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] However, note that ``<`` can fall back to using :meth:`~object.__gt__` if - :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` + for details on the mechanics). To avoid surprises, :pep:`8` + recommends that all six comparison methods be implemented. + The :func:`~functools.total_ordering` decorator is provided to make that + task easier. * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades @@ -295,3 +322,24 @@ Odds and Ends >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} >>> sorted(students, key=newgrades.__getitem__) ['jane', 'dave', 'john'] + +Partial Sorts +============= + +Some applications require only some of the data to be ordered. The standard +library provides several tools that do less work than a full sort: + +* :func:`min` and :func:`max` return the smallest and largest values, + respectively. These functions make a single pass over the input data and + require almost no auxiliary memory. + +* :func:`heapq.nsmallest` and :func:`heapq.nlargest` return + the *n* smallest and largest values, respectively. These functions + make a single pass over the data keeping only *n* elements in memory + at a time. For values of *n* that are small relative to the number of + inputs, these functions make far fewer comparisons than a full sort. + +* :func:`heapq.heappush` and :func:`heapq.heappop` create and maintain a + partially sorted arrangement of data that keeps the smallest element + at position ``0``. These functions are suitable for implementing + priority queues which are commonly used for task scheduling. diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst index 98f0294f9d082d..b5fc06ae8c6810 100644 --- a/Doc/howto/timerfd.rst +++ b/Doc/howto/timerfd.rst @@ -108,7 +108,7 @@ descriptors to wait until the file descriptor is ready for reading: # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. # # If a timer file descriptor is signaled more than once since - # the last os.read() call, os.read() returns the nubmer of signaled + # the last os.read() call, os.read() returns the number of signaled # as host order of class bytes. print(f"Signaled events={events}") for fd, event in events: diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 570435d48866d3..33a2a7ea89ea07 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -392,16 +392,16 @@ info and geturl =============== The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two -useful methods :meth:`info` and :meth:`geturl` and is defined in the module -:mod:`urllib.response`.. +useful methods :meth:`!info` and :meth:`!geturl` and is defined in the module +:mod:`urllib.response`. -**geturl** - this returns the real URL of the page fetched. This is useful -because ``urlopen`` (or the opener object used) may have followed a -redirect. The URL of the page fetched may not be the same as the URL requested. +* **geturl** - this returns the real URL of the page fetched. This is useful + because ``urlopen`` (or the opener object used) may have followed a + redirect. The URL of the page fetched may not be the same as the URL requested. -**info** - this returns a dictionary-like object that describes the page -fetched, particularly the headers sent by the server. It is currently an -:class:`http.client.HTTPMessage` instance. +* **info** - this returns a dictionary-like object that describes the page + fetched, particularly the headers sent by the server. It is currently an + :class:`http.client.HTTPMessage` instance. Typical headers include 'Content-length', 'Content-type', and so on. See the `Quick Reference to HTTP Headers <https://jkorpela.fi/http.html>`_ @@ -507,7 +507,7 @@ than the URL you pass to .add_password() will also match. :: In the above example we only supplied our ``HTTPBasicAuthHandler`` to ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` + -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`!http_proxy` environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. @@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - <https://code.activestate.com/recipes/456195/>`_. + <https://code.activestate.com/recipes/456195-urrlib2-opener-for-ssl-proxy-connect-method/>`_. diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 2fc1570e654db6..aa2a5c7cda52aa 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -53,7 +53,7 @@ def main(): # Guess the content type based on the file's extension. Encoding # will be ignored, although we should check for simple things like # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) + ctype, encoding = mimetypes.guess_file_type(path) if ctype is None or encoding is not None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. diff --git a/Doc/includes/wasm-ios-notavail.rst b/Doc/includes/wasm-ios-notavail.rst new file mode 100644 index 00000000000000..c820665f5e403c --- /dev/null +++ b/Doc/includes/wasm-ios-notavail.rst @@ -0,0 +1,8 @@ +.. include for modules that don't work on WASM or iOS + +.. availability:: not WASI, not iOS. + + This module does not work or is not available on WebAssembly platforms, or + on iOS. See :ref:`wasm-availability` for more information on WASM + availability; see :ref:`iOS-availability` for more information on iOS + availability. diff --git a/Doc/includes/wasm-notavail.rst b/Doc/includes/wasm-notavail.rst index e680e1f9b43807..c1b79d2a4a0508 100644 --- a/Doc/includes/wasm-notavail.rst +++ b/Doc/includes/wasm-notavail.rst @@ -1,7 +1,6 @@ .. include for modules that don't work on WASM -.. availability:: not Emscripten, not WASI. +.. availability:: not WASI. - This module does not work or is not available on WebAssembly platforms - ``wasm32-emscripten`` and ``wasm32-wasi``. See + This module does not work or is not available on WebAssembly. See :ref:`wasm-availability` for more information. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index d261e4a4f338a5..1ebff4409b1e95 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -1,5 +1,5 @@ -:mod:`__future__` --- Future statement definitions -================================================== +:mod:`!__future__` --- Future statement definitions +=================================================== .. module:: __future__ :synopsis: Future statement definitions @@ -8,20 +8,68 @@ -------------- -:mod:`__future__` is a real module, and serves three purposes: +Imports of the form ``from __future__ import feature`` are called +:ref:`future statements <future>`. These are special-cased by the Python compiler +to allow the use of new Python features in modules containing the future statement +before the release in which the feature becomes standard. + +While these future statements are given additional special meaning by the +Python compiler, they are still executed like any other import statement and +the :mod:`__future__` exists and is handled by the import system the same way +any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to find the modules they're importing. -* To ensure that :ref:`future statements <future>` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and can be inspected programmatically via importing :mod:`__future__` and examining its contents. +* To ensure that :ref:`future statements <future>` run under releases prior to + Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + will fail, because there was no module of that name prior to 2.1). + +Module Contents +--------------- + +No feature description will ever be deleted from :mod:`__future__`. Since its +introduction in Python 2.1 the following features have found their way into the +language using this mechanism: + ++------------------+-------------+--------------+---------------------------------------------+ +| feature | optional in | mandatory in | effect | ++==================+=============+==============+=============================================+ +| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | +| | | | *Statically Nested Scopes* | ++------------------+-------------+--------------+---------------------------------------------+ +| generators | 2.2.0a1 | 2.3 | :pep:`255`: | +| | | | *Simple Generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| division | 2.2.0a2 | 3.0 | :pep:`238`: | +| | | | *Changing the Division Operator* | ++------------------+-------------+--------------+---------------------------------------------+ +| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | +| | | | *Imports: Multi-Line and Absolute/Relative* | ++------------------+-------------+--------------+---------------------------------------------+ +| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | +| | | | *The "with" Statement* | ++------------------+-------------+--------------+---------------------------------------------+ +| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | +| | | | *Make print a function* | ++------------------+-------------+--------------+---------------------------------------------+ +| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | +| | | | *Bytes literals in Python 3000* | ++------------------+-------------+--------------+---------------------------------------------+ +| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | +| | | | *StopIteration handling inside generators* | ++------------------+-------------+--------------+---------------------------------------------+ +| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations* | ++------------------+-------------+--------------+---------------------------------------------+ + +.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. + .. _future-classes: .. class:: _Feature @@ -65,43 +113,6 @@ dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag` attribute on :class:`_Feature` instances. -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | -+------------------+-------------+--------------+---------------------------------------------+ - -.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. - .. [1] ``from __future__ import annotations`` was previously scheduled to become mandatory in Python 3.10, but the Python Steering Council @@ -115,3 +126,6 @@ language using this mechanism: :ref:`future` How the compiler treats future imports. + + :pep:`236` - Back to the __future__ + The original proposal for the __future__ mechanism. diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index 24a32b30bba673..647ff9da04d10d 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -1,5 +1,5 @@ -:mod:`__main__` --- Top-level code environment -============================================== +:mod:`!__main__` --- Top-level code environment +=============================================== .. module:: __main__ :synopsis: The environment where top-level code is run. Covers command-line @@ -227,7 +227,7 @@ students:: import sys from .student import search_students - student_name = sys.argv[2] if len(sys.argv) >= 2 else '' + student_name = sys.argv[1] if len(sys.argv) >= 2 else '' print(f'Found student: {search_students(student_name)}') Note that ``from .student import search_students`` is an example of a relative @@ -251,9 +251,9 @@ attribute will include the package's path if imported:: >>> asyncio.__main__.__name__ 'asyncio.__main__' -This won't work for ``__main__.py`` files in the root directory of a .zip file -though. Hence, for consistency, minimal ``__main__.py`` like the :mod:`venv` -one mentioned below are preferred. +This won't work for ``__main__.py`` files in the root directory of a +``.zip`` file though. Hence, for consistency, a minimal ``__main__.py`` +without a ``__name__`` check is preferred. .. seealso:: diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index d7c61c3d7ef126..81f0cac947f602 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -1,5 +1,5 @@ -:mod:`_thread` --- Low-level threading API -========================================== +:mod:`!_thread` --- Low-level threading API +=========================================== .. module:: _thread :synopsis: Low-level threading API. @@ -120,10 +120,13 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + Added support for GNU/kFreeBSD. + .. function:: stack_size([size]) @@ -166,14 +169,14 @@ Lock objects have the following methods: time can acquire a lock --- that's their reason for existence). If the *blocking* argument is present, the action depends on its - value: if it is False, the lock is only acquired if it can be acquired - immediately without waiting, while if it is True, the lock is acquired + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired unconditionally as above. If the floating-point *timeout* argument is present and positive, it specifies the maximum wait time in seconds before returning. A negative *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is False. + a *timeout* if *blocking* is false. The return value is ``True`` if the lock is acquired successfully, ``False`` if not. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 274b2d69f0ab5c..168ef3ec00d81b 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -1,5 +1,5 @@ -:mod:`abc` --- Abstract Base Classes -==================================== +:mod:`!abc` --- Abstract Base Classes +===================================== .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. @@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition, the :mod:`collections.abc` submodule has some ABCs that can be used to test whether a class or instance provides a particular interface, for example, if it is -:term:`hashable` or if it is a mapping. +:term:`hashable` or if it is a :term:`mapping`. This module provides the metaclass :class:`ABCMeta` for defining ABCs and @@ -30,7 +30,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: .. class:: ABC A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC` + an abstract base class can be created by simply deriving from :class:`!ABC` avoiding sometimes confusing metaclass usage, for example:: from abc import ABC @@ -38,11 +38,11 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: class MyABC(ABC): pass - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding + Note that the type of :class:`!ABC` is still :class:`ABCMeta`, therefore + inheriting from :class:`!ABC` requires the usual precautions regarding metaclass usage, as multiple inheritance may lead to metaclass conflicts. One may also define an abstract base class by passing the metaclass - keyword and using :class:`ABCMeta` directly, for example:: + keyword and using :class:`!ABCMeta` directly, for example:: from abc import ABCMeta @@ -65,7 +65,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: implementations defined by the registering ABC be callable (not even via :func:`super`). [#]_ - Classes created with a metaclass of :class:`ABCMeta` have the following method: + Classes created with a metaclass of :class:`!ABCMeta` have the following method: .. method:: register(subclass) @@ -86,7 +86,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Returns the registered subclass, to allow usage as a class decorator. .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the + To detect calls to :meth:`!register`, you can use the :func:`get_cache_token` function. You can also override this method in an abstract base class: @@ -96,16 +96,16 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: (Must be defined as a class method.) Check whether *subclass* is considered a subclass of this ABC. This means - that you can customize the behavior of ``issubclass`` further without the + that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`__subclasscheck__` method of the ABC.) + :meth:`~class.__subclasscheck__` method of the ABC.) - This method should return ``True``, ``False`` or ``NotImplemented``. If + This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. If it returns ``False``, the *subclass* is not considered a subclass of this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual + :data:`!NotImplemented`, the subclass check is continued with the usual mechanism. .. XXX explain the "usual mechanism" @@ -142,7 +142,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The ABC ``MyIterable`` defines the standard iterable method, :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method + here can still be called from subclasses. The :meth:`!get_iterator` method is also part of the ``MyIterable`` abstract base class, but it does not have to be overridden in non-abstract derived classes. @@ -153,14 +153,14 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and - :meth:`__getitem__`). Note that this will not make ``get_iterator`` + the old-style iterable protocol, defined in terms of :meth:`~object.__len__` and + :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. -The :mod:`abc` module also provides the following decorator: +The :mod:`!abc` module also provides the following decorator: .. decorator:: abstractmethod @@ -168,19 +168,19 @@ The :mod:`abc` module also provides the following decorator: Using this decorator requires that the class's metaclass is :class:`ABCMeta` or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods + :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`abstractmethod` may be used + of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`abstractmethod` only affects subclasses derived using regular - inheritance; "virtual subclasses" registered with the ABC's :meth:`register` - method are not affected. + :func:`!abstractmethod` only affects subclasses derived using regular + inheritance; "virtual subclasses" registered with the ABC's + :meth:`~ABCMeta.register` method are not affected. - When :func:`abstractmethod` is applied in combination with other method + When :func:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -216,7 +216,7 @@ The :mod:`abc` module also provides the following decorator: In order to correctly interoperate with the abstract base class machinery, the descriptor must identify itself as abstract using - :attr:`__isabstractmethod__`. In general, this attribute should be ``True`` + :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For example, Python's built-in :class:`property` does the equivalent of:: @@ -236,7 +236,7 @@ The :mod:`abc` module also provides the following decorator: super-call in a framework that uses cooperative multiple-inheritance. -The :mod:`abc` module also supports the following legacy decorators: +The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod @@ -323,7 +323,7 @@ The :mod:`abc` module also supports the following legacy decorators: ... -The :mod:`abc` module also provides the following functions: +The :mod:`!abc` module also provides the following functions: .. function:: get_cache_token() diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index f7105d8af8e28b..0223c1054ea5d8 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -16,7 +16,6 @@ but they are available on most other systems as well. Here's an overview: io.rst time.rst argparse.rst - getopt.rst logging.rst logging.config.rst logging.handlers.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index fbffa71d200735..0367c83d9369d3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1,5 +1,5 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== +:mod:`!argparse` --- Parser for command-line options, arguments and sub-commands +================================================================================ .. module:: argparse :synopsis: Command-line option and argument parsing library. @@ -745,7 +745,7 @@ The add_argument() method .. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) + [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: @@ -777,6 +777,8 @@ The add_argument() method * dest_ - The name of the attribute to be added to the object returned by :meth:`parse_args`. + * deprecated_ - Whether or not use of the argument is deprecated. + The following sections describe how each of these are used. @@ -1439,6 +1441,34 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') + +.. _deprecated: + +deprecated +^^^^^^^^^^ + +During a project's lifetime, some arguments may need to be removed from the +command line. Before removing them, you should inform +your users that the arguments are deprecated and will be removed. +The ``deprecated`` keyword argument of +:meth:`~ArgumentParser.add_argument`, which defaults to ``False``, +specifies if the argument is deprecated and will be removed +in the future. +For arguments, if ``deprecated`` is ``True``, then a warning will be +printed to standard error when the argument is used:: + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='snake.py') + >>> parser.add_argument('--legs', default=0, type=int, deprecated=True) + >>> parser.parse_args([]) + Namespace(legs=0) + >>> parser.parse_args(['--legs', '4']) # doctest: +SKIP + snake.py: warning: option '--legs' is deprecated + Namespace(legs=4) + +.. versionchanged:: 3.13 + + Action classes ^^^^^^^^^^^^^^ @@ -1842,7 +1872,8 @@ Sub-commands {foo,bar} additional help - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, + Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional + *aliases* argument, which allows multiple strings to refer to the same subparser. This example, like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: @@ -1853,6 +1884,20 @@ Sub-commands >>> parser.parse_args(['co', 'bar']) Namespace(foo='bar') + :meth:`~_SubParsersAction.add_parser` supports also an additional + *deprecated* argument, which allows to deprecate the subparser. + + >>> import argparse + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> run = subparsers.add_parser('run') + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> parser.parse_args(['fly']) # doctest: +SKIP + chicken.py: warning: command 'fly' is deprecated + Namespace() + + .. versionadded:: 3.13 + One particularly effective way of handling sub-commands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For @@ -1936,8 +1981,8 @@ FileType objects >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='<stdin>' encoding='UTF-8'>) - .. versionadded:: 3.4 - The *encodings* and *errors* keyword arguments. + .. versionchanged:: 3.4 + Added the *encodings* and *errors* parameters. Argument groups diff --git a/Doc/library/array.rst b/Doc/library/array.rst index ad622627724217..d34a1888342e27 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -1,5 +1,5 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== +:mod:`!array` --- Efficient arrays of numeric values +==================================================== .. module:: array :synopsis: Space efficient arrays of uniformly typed numeric values. @@ -79,14 +79,16 @@ The module defines the following type: .. class:: array(typecode[, initializer]) A new array whose items are restricted by *typecode*, and initialized - from the optional *initializer* value, which must be a list, a - :term:`bytes-like object`, or iterable over elements of the - appropriate type. + from the optional *initializer* value, which must be a :class:`bytes` + or :class:`bytearray` object, a Unicode string, or iterable over elements + of the appropriate type. - If given a list or string, the initializer is passed to the new array's - :meth:`fromlist`, :meth:`frombytes`, or :meth:`fromunicode` method (see below) - to add initial items to the array. Otherwise, the iterable initializer is - passed to the :meth:`extend` method. + If given a :class:`bytes` or :class:`bytearray` object, the initializer + is passed to the new array's :meth:`frombytes` method; + if given a Unicode string, the initializer is passed to the + :meth:`fromunicode` method; + otherwise, the initializer's iterator is passed to the :meth:`extend` method + to add initial items to the array. Array objects support the ordinary sequence operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned @@ -152,10 +154,11 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(s) + .. method:: frombytes(buffer) - Appends items from the string, interpreting the string as an array of machine - values (as if it had been read from a file using the :meth:`fromfile` method). + Appends items from the :term:`bytes-like object`, interpreting + its content as an array of machine values (as if it had been read + from a file using the :meth:`fromfile` method). .. versionadded:: 3.2 :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. @@ -177,7 +180,7 @@ The module defines the following type: .. method:: fromunicode(s) - Extends this array with data from the given unicode string. + Extends this array with data from the given Unicode string. The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. @@ -212,6 +215,13 @@ The module defines the following type: Remove the first occurrence of *x* from the array. + .. method:: clear() + + Remove all elements from the array. + + .. versionadded:: 3.13 + + .. method:: reverse() Reverse the order of the items in the array. @@ -239,24 +249,27 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to - obtain a unicode string from an array of some other type. + obtain a Unicode string from an array of some other type. -When an array object is printed or converted to a string, it is represented as -``array(typecode, initializer)``. The *initializer* is omitted if the array is -empty, otherwise it is a string if the *typecode* is ``'u'`` or ``'w'``, -otherwise it is a list of numbers. -The string is guaranteed to be able to be converted back to an +The string representation of array objects has the form +``array(typecode, initializer)``. +The *initializer* is omitted if the array is empty, otherwise it is +a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a list of numbers. +The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. +Variables ``inf`` and ``nan`` must also be defined if it contains +corresponding floating point values. Examples:: array('l') array('w', 'hello \u2641') array('l', [1, 2, 3, 4, 5]) - array('d', [1.0, 2.0, 3.14]) + array('d', [1.0, 2.0, 3.14, -inf, nan]) .. seealso:: @@ -266,4 +279,3 @@ Examples:: `NumPy <https://numpy.org/>`_ The NumPy package defines another array type. - diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 4ebbe0e5471c88..f7e8afa7000392 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,5 +1,5 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== +:mod:`!ast` --- Abstract Syntax Trees +===================================== .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. @@ -45,7 +45,7 @@ Node classes This is the base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`above <abstract-grammar>`. They are defined in the :mod:`_ast` C + :ref:`above <abstract-grammar>`. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`ast`. There is one class defined for each left-hand side symbol in the abstract @@ -61,7 +61,7 @@ Node classes .. attribute:: _fields - Each concrete class has an attribute :attr:`_fields` which gives the names + Each concrete class has an attribute :attr:`!_fields` which gives the names of all child nodes. Each instance of a concrete class has one attribute for each child node, @@ -74,6 +74,18 @@ Node classes as Python lists. All possible attributes must be present and have valid values when compiling an AST with :func:`compile`. + .. attribute:: _field_types + + The :attr:`!_field_types` attribute on each concrete class is a dictionary + mapping field names (as also listed in :attr:`_fields`) to their types. + + .. doctest:: + + >>> ast.TypeVar._field_types + {'name': <class 'str'>, 'bound': ast.expr | None, 'default_value': ast.expr | None} + + .. versionadded:: 3.13 + .. attribute:: lineno col_offset end_lineno @@ -103,20 +115,16 @@ Node classes For example, to create and populate an :class:`ast.UnaryOp` node, you could use :: - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) + If a field that is optional in the grammar is omitted from the constructor, + it defaults to ``None``. If a list field is omitted, it defaults to the empty + list. If a field of type :class:`!ast.expr_context` is omitted, it defaults to + :class:`Load() <ast.Load>`. If any other field is omitted, a :exc:`DeprecationWarning` is raised + and the AST node will not have this field. In Python 3.15, this condition will + raise an error. + .. versionchanged:: 3.8 Class :class:`ast.Constant` is now used for all constants. @@ -128,18 +136,26 @@ Node classes .. deprecated:: 3.8 - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, + Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, + :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, but they will be removed in future Python releases. In the meantime, instantiating them will return an instance of a different class. .. deprecated:: 3.9 - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + Old classes :class:`!ast.Index` and :class:`!ast.ExtSlice` are still available, but they will be removed in future Python releases. In the meantime, instantiating them will return an instance of a different class. +.. deprecated-removed:: 3.13 3.15 + + Previous versions of Python allowed the creation of AST nodes that were missing + required fields. Similarly, AST node constructors allowed arbitrary keyword + arguments that were set as attributes of the AST node, even if they did not + match any of the fields of the AST node. This behavior is deprecated and will + be removed in Python 3.15. + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree @@ -170,8 +186,7 @@ Root nodes Assign( targets=[ Name(id='x', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) .. class:: Expression(body) @@ -299,8 +314,7 @@ Literals value=Call( func=Name(id='sin', ctx=Load()), args=[ - Name(id='a', ctx=Load())], - keywords=[]), + Name(id='a', ctx=Load())]), conversion=-1, format_spec=JoinedStr( values=[ @@ -395,8 +409,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) + value=Name(id='a', ctx=Load()))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -404,16 +417,14 @@ Variables Assign( targets=[ Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('del a'), indent=4)) Module( body=[ Delete( targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) + Name(id='a', ctx=Del())])]) .. class:: Starred(value, ctx) @@ -436,8 +447,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) + value=Name(id='it', ctx=Load()))]) .. _ast-expressions: @@ -460,8 +470,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) + operand=Name(id='a', ctx=Load())))]) .. class:: UnaryOp(op, operand) @@ -588,8 +597,7 @@ Expressions * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. + The ``args`` and ``keywords`` arguments are optional and default to empty lists. .. doctest:: @@ -726,7 +734,10 @@ Comprehensions .. doctest:: - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('[x for x in numbers]', mode='eval'), + ... indent=4, + ... )) Expression( body=ListComp( elt=Name(id='x', ctx=Load()), @@ -734,9 +745,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=DictComp( key=Name(id='x', ctx=Load()), @@ -748,9 +761,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=SetComp( elt=Name(id='x', ctx=Load()), @@ -758,7 +773,6 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) @@ -781,18 +795,15 @@ Comprehensions elt=Call( func=Name(id='ord', ctx=Load()), args=[ - Name(id='c', ctx=Load())], - keywords=[]), + Name(id='c', ctx=Load())]), generators=[ comprehension( target=Name(id='line', ctx=Store()), iter=Name(id='file', ctx=Load()), - ifs=[], is_async=0), comprehension( target=Name(id='c', ctx=Store()), iter=Name(id='line', ctx=Load()), - ifs=[], is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -831,7 +842,6 @@ Comprehensions comprehension( target=Name(id='i', ctx=Store()), iter=Name(id='soc', ctx=Load()), - ifs=[], is_async=1)])) @@ -861,8 +871,7 @@ Statements targets=[ Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking Module( @@ -874,8 +883,7 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) + value=Name(id='c', ctx=Load()))]) .. class:: AnnAssign(target, annotation, value, simple) @@ -883,9 +891,13 @@ Statements An assignment with a type annotation. ``target`` is a single node and can be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. + node. ``value`` is a single optional node. + + ``simple`` is always either 0 (indicating a "complex" target) or 1 + (indicating a "simple" target). A "simple" target consists solely of a + :class:`Name` node that does not appear between parentheses; all other + targets are considered complex. Only simple targets appear in + the :attr:`__annotations__` dictionary of modules and classes. .. doctest:: @@ -895,8 +907,7 @@ Statements AnnAssign( target=Name(id='c', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) + simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis Module( @@ -905,8 +916,7 @@ Statements target=Name(id='a', ctx=Store()), annotation=Name(id='int', ctx=Load()), value=Constant(value=1), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation Module( @@ -917,8 +927,7 @@ Statements attr='b', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation Module( @@ -929,8 +938,7 @@ Statements slice=Constant(value=1), ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) .. class:: AugAssign(target, op, value) @@ -951,8 +959,7 @@ Statements AugAssign( target=Name(id='x', ctx=Store()), op=Add(), - value=Constant(value=2))], - type_ignores=[]) + value=Constant(value=2))]) .. class:: Raise(exc, cause) @@ -968,8 +975,7 @@ Statements body=[ Raise( exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) + cause=Name(id='y', ctx=Load()))]) .. class:: Assert(test, msg) @@ -984,8 +990,7 @@ Statements body=[ Assert( test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) + msg=Name(id='y', ctx=Load()))]) .. class:: Delete(targets) @@ -1002,8 +1007,7 @@ Statements targets=[ Name(id='x', ctx=Del()), Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) + Name(id='z', ctx=Del())])]) .. class:: Pass() @@ -1015,8 +1019,7 @@ Statements >>> print(ast.dump(ast.parse('pass'), indent=4)) Module( body=[ - Pass()], - type_ignores=[]) + Pass()]) .. class:: TypeAlias(name, type_params, value) @@ -1033,9 +1036,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - type_params=[], - value=Name(id='int', ctx=Load()))], - type_ignores=[]) + value=Name(id='int', ctx=Load()))]) .. versionadded:: 3.12 @@ -1058,8 +1059,7 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])], - type_ignores=[]) + alias(name='z')])]) .. class:: ImportFrom(module, names, level) @@ -1080,8 +1080,7 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)], - type_ignores=[]) + level=0)]) .. class:: alias(name, asname) @@ -1099,8 +1098,7 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)], - type_ignores=[]) + level=2)]) Control flow ^^^^^^^^^^^^ @@ -1143,8 +1141,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. class:: For(target, iter, body, orelse, type_comment) @@ -1178,8 +1175,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: While(test, body, orelse) @@ -1204,8 +1200,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: Break @@ -1239,9 +1234,7 @@ Control flow body=[ Break()], orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) + Continue()])])]) .. class:: Try(body, handlers, orelse, finalbody) @@ -1286,8 +1279,7 @@ Control flow value=Constant(value=Ellipsis))], finalbody=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: TryStar(body, handlers, orelse, finalbody) @@ -1315,10 +1307,7 @@ Control flow type=Name(id='Exception', ctx=Load()), body=[ Expr( - value=Constant(value=Ellipsis))])], - orelse=[], - finalbody=[])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.11 @@ -1350,10 +1339,7 @@ Control flow ExceptHandler( type=Name(id='TypeError', ctx=Load()), body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) + Pass()])])]) .. class:: With(items, body, type_comment) @@ -1395,9 +1381,7 @@ Control flow func=Name(id='something', ctx=Load()), args=[ Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) + Name(id='d', ctx=Load())]))])]) Pattern matching @@ -1454,14 +1438,10 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[]), + cls=Name(id='tuple', ctx=Load())), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1489,8 +1469,7 @@ Pattern matching value=Constant(value='Relevant')), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1516,8 +1495,7 @@ Pattern matching pattern=MatchSingleton(value=None), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1549,8 +1527,7 @@ Pattern matching value=Constant(value=2))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1591,8 +1568,7 @@ Pattern matching MatchStar()]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1636,11 +1612,10 @@ Pattern matching Expr( value=Constant(value=Ellipsis))]), match_case( - pattern=MatchMapping(keys=[], patterns=[], rest='rest'), + pattern=MatchMapping(rest='rest'), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1682,16 +1657,13 @@ Pattern matching MatchValue( value=Constant(value=0)), MatchValue( - value=Constant(value=0))], - kwd_attrs=[], - kwd_patterns=[]), + value=Constant(value=0))]), body=[ Expr( value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( cls=Name(id='Point3D', ctx=Load()), - patterns=[], kwd_attrs=[ 'x', 'y', @@ -1705,8 +1677,7 @@ Pattern matching value=Constant(value=0))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1748,8 +1719,7 @@ Pattern matching pattern=MatchAs(), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1782,8 +1752,7 @@ Pattern matching MatchAs(name='y')]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1795,15 +1764,17 @@ Type parameters :ref:`Type parameters <type-params>` can exist on classes, functions, and type aliases. -.. class:: TypeVar(name, bound) +.. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. ``name`` is the name of the type variable. - ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. + A :class:`typing.TypeVar`. *name* is the name of the type variable. + *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. *default_value* + is the default value; if the :class:`!TypeVar` has no default, this + attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[T: int = bool] = list[T]"), indent=4)) Module( body=[ TypeAlias( @@ -1811,28 +1782,39 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()))], + bound=Name(id='int', ctx=Load()), + default_value=Name(id='bool', ctx=Load()))], value=Subscript( value=Name(id='list', ctx=Load()), slice=Name(id='T', ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: ParamSpec(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. + *default_value* is the default value; if the :class:`!ParamSpec` has no default, + this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - ParamSpec(name='P')], + ParamSpec( + name='P', + default_value=Tuple( + elts=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + ctx=Load()))], value=Subscript( value=Name(id='Callable', ctx=Load()), slice=Tuple( @@ -1840,24 +1822,30 @@ aliases. Name(id='P', ctx=Load()), Name(id='int', ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. + *default_value* is the default value; if the :class:`!TypeVarTuple` has no + default, this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[*Ts = ()] = tuple[*Ts]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple(name='Ts')], + TypeVarTuple( + name='Ts', + default_value=Tuple(ctx=Load()))], value=Subscript( value=Name(id='tuple', ctx=Load()), slice=Tuple( @@ -1866,11 +1854,13 @@ aliases. value=Name(id='Ts', ctx=Load()), ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *default_value* parameter. + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1907,15 +1897,10 @@ Function and class definitions Expr( value=Lambda( args=arguments( - posonlyargs=[], args=[ arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) + arg(arg='y')]), + body=Constant(value=Ellipsis)))]) .. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) @@ -1935,8 +1920,7 @@ Function and class definitions .. class:: arg(arg, annotation, type_comment) A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. + name; ``annotation`` is its annotation, such as a :class:`Name` node. .. attribute:: type_comment @@ -1955,7 +1939,6 @@ Function and class definitions FunctionDef( name='f', args=arguments( - posonlyargs=[], args=[ arg( arg='a', @@ -1978,9 +1961,7 @@ Function and class definitions decorator_list=[ Name(id='decorator1', ctx=Load()), Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'), - type_params=[])], - type_ignores=[]) + returns=Constant(value='return annotation'))]) .. class:: Return(value) @@ -1993,8 +1974,7 @@ Function and class definitions Module( body=[ Return( - value=Constant(value=4))], - type_ignores=[]) + value=Constant(value=4))]) .. class:: Yield(value) @@ -2010,16 +1990,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) .. class:: Global(names) @@ -2036,8 +2014,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) Module( @@ -2046,8 +2023,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) .. class:: ClassDef(name, bases, keywords, body, decorator_list, type_params) @@ -2087,9 +2063,7 @@ Function and class definitions Pass()], decorator_list=[ Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - type_params=[])], - type_ignores=[]) + Name(id='decorator2', ctx=Load())])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2121,22 +2095,12 @@ Async and await body=[ AsyncFunctionDef( name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), + args=arguments(), body=[ Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + func=Name(id='other_func', ctx=Load()))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2181,14 +2145,17 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 7)``; the highest is ``sys.version_info[0:2]``. + Setting ``feature_version`` to a tuple ``(major, minor)`` will result in + a "best-effort" attempt to parse using that Python version's grammar. + For example, setting ``feature_version=(3, 9)`` will attempt to disallow + parsing of :keyword:`match` statements. + Currently ``major`` must equal to ``3``. The lowest supported version is + ``(3, 7)`` (and this may increase in future Python versions); + the highest is ``sys.version_info[0:2]``. "Best-effort" attempt means there + is no guarantee that the parse (or success of the parse) is the same as + when run on the Python version corresponding to ``feature_version``. - If source contains a null character ('\0'), :exc:`ValueError` is raised. + If source contains a null character (``\0``), :exc:`ValueError` is raised. .. warning:: Note that successfully parsing source code into an AST object doesn't @@ -2210,7 +2177,7 @@ and classes for traversing abstract syntax trees: Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. .. versionchanged:: 3.13 - The minimum supported version for feature_version is now (3,7) + The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. @@ -2286,8 +2253,8 @@ and classes for traversing abstract syntax trees: .. function:: get_source_segment(source, node, *, padded=False) Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + If some location information (:attr:`~ast.AST.lineno`, :attr:`~ast.AST.end_lineno`, + :attr:`~ast.AST.col_offset`, or :attr:`~ast.AST.end_col_offset`) is missing, return ``None``. If *padded* is ``True``, the first line of a multi-line statement will be padded with spaces to match its original position. @@ -2298,7 +2265,7 @@ and classes for traversing abstract syntax trees: .. function:: fix_missing_locations(node) When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + :attr:`~ast.AST.lineno` and :attr:`~ast.AST.col_offset` attributes for every node that supports them. This is rather tedious to fill in for generated nodes, so this helper adds these attributes recursively where not already set, by setting them to the values of the parent node. It works recursively starting at *node*. @@ -2313,8 +2280,8 @@ and classes for traversing abstract syntax trees: .. function:: copy_location(new_node, old_node) - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + Copy source location (:attr:`~ast.AST.lineno`, :attr:`~ast.AST.col_offset`, :attr:`~ast.AST.end_lineno`, + and :attr:`~ast.AST.end_col_offset`) from *old_node* to *new_node* if possible, and return *new_node*. @@ -2360,14 +2327,18 @@ and classes for traversing abstract syntax trees: visited unless the visitor calls :meth:`generic_visit` or visits them itself. + .. method:: visit_Constant(node) + + Handles all constant nodes. + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. .. deprecated:: 3.8 - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated now and will not be called in future Python versions. Add the :meth:`visit_Constant` method to handle all constant nodes. @@ -2396,7 +2367,7 @@ and classes for traversing abstract syntax trees: ) Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` + either transform the child nodes yourself or call the :meth:`~ast.NodeVisitor.generic_visit` method for the node first. For nodes that were part of a collection of statements (that applies to all @@ -2405,7 +2376,7 @@ and classes for traversing abstract syntax trees: If :class:`NodeTransformer` introduces new nodes (that weren't part of original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with + :attr:`~ast.AST.lineno`), :func:`fix_missing_locations` should be called with the new sub-tree to recalculate the location information:: tree = ast.parse('foo', mode='eval') @@ -2416,7 +2387,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2433,9 +2404,42 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. + If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` + will be omitted from the output. + .. versionchanged:: 3.9 Added the *indent* option. + .. versionchanged:: 3.13 + Added the *show_empty* option. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4, show_empty=True)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[], + type_params=[])], + type_ignores=[]) + .. _ast-compiler-flags: @@ -2457,6 +2461,13 @@ effects on the compilation of a program: Generates and returns an abstract syntax tree instead of returning a compiled code object. +.. data:: PyCF_OPTIMIZED_AST + + The returned AST is optimized according to the *optimize* argument + in :func:`compile` or :func:`ast.parse`. + + .. versionadded:: 3.13 + .. data:: PyCF_TYPE_COMMENTS Enables support for :pep:`484` and :pep:`526` style type comments @@ -2465,6 +2476,20 @@ effects on the compilation of a program: .. versionadded:: 3.8 +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + + .. _ast-cli: Command-Line Usage @@ -2520,7 +2545,8 @@ to stdout. Otherwise, the content is read from stdin. code that generated them. This is helpful for tools that make source code transformations. - `leoAst.py <https://leoeditor.com/appendices.html#leoast-py>`_ unifies the + `leoAst.py <https://leo-editor.github.io/leo-editor/appendices.html#leoast-py>`_ + unifies the token-based and parse-tree-based views of python programs by inserting two-way links between tokens and ast nodes. @@ -2532,4 +2558,4 @@ to stdout. Otherwise, the content is read from stdin. `Parso <https://parso.readthedocs.io>`_ is a Python parser that supports error recovery and round-trip parsing for different Python versions (in multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. + in your Python file. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 361e7bb9c8f2fa..70bdd154d6c406 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -243,9 +243,9 @@ Scheduling callbacks See the :ref:`concurrency and multithreading <asyncio-multithreading>` section of the documentation. -.. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. + .. versionchanged:: 3.7 + The *context* keyword-only parameter was added. See :pep:`567` + for more details. .. _asyncio-pass-keywords: @@ -509,7 +509,7 @@ Opening network connections .. versionchanged:: 3.6 - The socket option :py:const:`~socket.TCP_NODELAY` is set by default + The socket option :ref:`socket.TCP_NODELAY <socket-unix-constants>` is set by default for all TCP connections. .. versionchanged:: 3.7 @@ -581,7 +581,7 @@ Opening network connections * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows - and some Unixes. If the :py:const:`~socket.SO_REUSEPORT` constant is not + and some Unixes. If the :ref:`socket.SO_REUSEPORT <socket-unix-constants>` constant is not defined then this capability is unsupported. * *allow_broadcast* tells the kernel to allow this endpoint to send @@ -605,9 +605,13 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port*, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8 + Added support for Windows. + .. versionchanged:: 3.8.1 The *reuse_address* parameter is no longer supported, as using - :py:const:`~sockets.SO_REUSEADDR` poses a significant security concern for + :ref:`socket.SO_REUSEADDR <socket-unix-constants>` + poses a significant security concern for UDP. Explicitly passing ``reuse_address=True`` will raise an exception. When multiple processes with differing UIDs assign sockets to an @@ -616,15 +620,13 @@ Opening network connections For supported platforms, *reuse_port* can be used as a replacement for similar functionality. With *reuse_port*, - :py:const:`~sockets.SO_REUSEPORT` is used instead, which specifically + :ref:`socket.SO_REUSEPORT <socket-unix-constants>` + is used instead, which specifically prevents processes with differing UIDs from assigning sockets to the same socket address. - .. versionchanged:: 3.8 - Added support for Windows. - .. versionchanged:: 3.11 - The *reuse_address* parameter, disabled since Python 3.9.0, 3.8.1, + The *reuse_address* parameter, disabled since Python 3.8.1, 3.7.6 and 3.6.10, has been entirely removed. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ @@ -661,12 +663,15 @@ Opening network connections Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ +.. _loop_create_server: + .. coroutinemethod:: loop.create_server(protocol_factory, \ host=None, port=None, *, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ reuse_address=None, reuse_port=None, \ + keep_alive=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ start_serving=True) @@ -731,6 +736,13 @@ Creating network servers set this flag when being created. This option is not supported on Windows. + * *keep_alive* set to ``True`` keeps connections active by enabling the + periodic transmission of messages. + + .. versionchanged:: 3.13 + + Added the *keep_alive* parameter. + * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). @@ -756,7 +768,7 @@ Creating network servers .. versionchanged:: 3.6 Added *ssl_handshake_timeout* and *start_serving* parameters. - The socket option :py:const:`~socket.TCP_NODELAY` is set by default + The socket option :ref:`socket.TCP_NODELAY <socket-unix-constants>` is set by default for all TCP connections. .. versionchanged:: 3.11 @@ -774,7 +786,7 @@ Creating network servers *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ - start_serving=True) + start_serving=True, cleanup_socket=True) Similar to :meth:`loop.create_server` but works with the :py:const:`~socket.AF_UNIX` socket family. @@ -784,6 +796,10 @@ Creating network servers :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are supported. + If *cleanup_socket* is true then the Unix socket will automatically + be removed from the filesystem when the server is closed, unless the + socket has been replaced after the server has been created. + See the documentation of the :meth:`loop.create_server` method for information about arguments to this method. @@ -798,6 +814,10 @@ Creating network servers Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + + Added the *cleanup_socket* parameter. + .. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ sock, *, ssl=None, ssl_handshake_timeout=None, \ @@ -1135,6 +1155,14 @@ DNS Asynchronous version of :meth:`socket.getnameinfo`. +.. note:: + Both *getaddrinfo* and *getnameinfo* internally utilize their synchronous + versions through the loop's default thread pool executor. + When this executor is saturated, these methods may experience delays, + which higher-level networking libraries may report as increased timeouts. + To mitigate this, consider using a custom executor for other user tasks, + or setting a default executor with a larger number of workers. + .. versionchanged:: 3.7 Both *getaddrinfo* and *getnameinfo* methods were always documented to return a coroutine, but prior to Python 3.7 they were, in fact, @@ -1191,6 +1219,8 @@ Working with pipes Unix signals ^^^^^^^^^^^^ +.. _loop_add_signal_handler: + .. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1232,6 +1262,9 @@ Executing code in thread or process pools The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. + The default executor can be set by :meth:`loop.set_default_executor`, + otherwise, a :class:`concurrent.futures.ThreadPoolExecutor` will be + lazy-initialized and used by :func:`run_in_executor` if needed. Example:: @@ -1391,6 +1424,14 @@ Enabling debug mode The new :ref:`Python Development Mode <devmode>` can now also be used to enable the debug mode. +.. attribute:: loop.slow_callback_duration + + This attribute can be used to set the + minimum execution duration in seconds that is considered "slow". + When debug mode is enabled, "slow" callbacks are logged. + + Default value is 100 milliseconds. + .. seealso:: The :ref:`debug mode of asyncio <asyncio-debug-mode>`. @@ -1411,6 +1452,8 @@ async/await code consider using the high-level :ref:`Subprocess Support on Windows <asyncio-windows-subprocess>` for details. +.. _loop_subprocess_exec: + .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1605,8 +1648,34 @@ Do not instantiate the :class:`Server` class directly. The sockets that represent existing incoming client connections are left open. - The server is closed asynchronously, use the :meth:`wait_closed` - coroutine to wait until the server is closed. + The server is closed asynchronously; use the :meth:`wait_closed` + coroutine to wait until the server is closed (and no more + connections are active). + + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 .. method:: get_loop() @@ -1664,7 +1733,8 @@ Do not instantiate the :class:`Server` class directly. .. coroutinemethod:: wait_closed() - Wait until the :meth:`close` method completes. + Wait until the :meth:`close` method completes and all active + connections have finished. .. attribute:: sockets @@ -1880,7 +1950,7 @@ Set signal handlers for SIGINT and SIGTERM (This ``signals`` example only works on Unix.) -Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` +Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: import asyncio diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 19ec726c1be060..a2a3114ad6e4c5 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -77,11 +77,6 @@ Subprocess Support on Windows On Windows, the default event loop :class:`ProactorEventLoop` supports subprocesses, whereas :class:`SelectorEventLoop` does not. -The :meth:`policy.set_child_watcher() -<AbstractEventLoopPolicy.set_child_watcher>` function is also -not supported, as :class:`ProactorEventLoop` has a different mechanism -to watch child processes. - macOS ===== diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 0d7821e608ec98..837ccc6606786e 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -79,25 +79,6 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. - .. method:: get_child_watcher() - - Get a child process watcher object. - - Return a watcher object implementing the - :class:`AbstractChildWatcher` interface. - - This function is Unix specific. - - .. deprecated:: 3.12 - - .. method:: set_child_watcher(watcher) - - Set the current child process watcher to *watcher*. - - This function is Unix specific. - - .. deprecated:: 3.12 - .. _asyncio-policy-builtin: @@ -139,172 +120,6 @@ asyncio ships with the following built-in policies: .. availability:: Windows. -.. _asyncio-watchers: - -Process Watchers -================ - -A process watcher allows customization of how an event loop monitors -child processes on Unix. Specifically, the event loop needs to know -when a child process has exited. - -In asyncio, child processes are created with -:func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` -functions. - -asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child -watchers should implement, and has four different implementations: -:class:`ThreadedChildWatcher` (configured to be used by default), -:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and -:class:`FastChildWatcher`. - -See also the :ref:`Subprocess and Threads <asyncio-subprocess-threads>` -section. - -The following two functions can be used to customize the child process watcher -implementation used by the asyncio event loop: - -.. function:: get_child_watcher() - - Return the current child watcher for the current policy. - - .. deprecated:: 3.12 - -.. function:: set_child_watcher(watcher) - - Set the current child watcher to *watcher* for the current - policy. *watcher* must implement methods defined in the - :class:`AbstractChildWatcher` base class. - - .. deprecated:: 3.12 - -.. note:: - Third-party event loops implementations might not support - custom child watchers. For such event loops, using - :func:`set_child_watcher` might be prohibited or have no effect. - -.. class:: AbstractChildWatcher - - .. method:: add_child_handler(pid, callback, *args) - - Register a new child handler. - - Arrange for ``callback(pid, returncode, *args)`` to be called - when a process with PID equal to *pid* terminates. Specifying - another callback for the same process replaces the previous - handler. - - The *callback* callable must be thread-safe. - - .. method:: remove_child_handler(pid) - - Removes the handler for process with PID equal to *pid*. - - The function returns ``True`` if the handler was successfully - removed, ``False`` if there was nothing to remove. - - .. method:: attach_loop(loop) - - Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then - it is first detached before attaching to the new loop. - - Note: loop may be ``None``. - - .. method:: is_active() - - Return ``True`` if the watcher is ready to use. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - - .. versionadded:: 3.8 - - .. method:: close() - - Close the watcher. - - This method has to be called to ensure that underlying - resources are cleaned-up. - - .. deprecated:: 3.12 - - -.. class:: ThreadedChildWatcher - - This implementation starts a new waiting thread for every subprocess spawn. - - It works reliably even when the asyncio event loop is run in a non-main OS thread. - - There is no noticeable overhead when handling a big number of children (*O(1)* each - time a child terminates), but starting a thread per process requires extra memory. - - This watcher is used by default. - - .. versionadded:: 3.8 - -.. class:: MultiLoopChildWatcher - - This implementation registers a :py:data:`SIGCHLD` signal handler on - instantiation. That can break third-party code that installs a custom handler for - :py:data:`SIGCHLD` signal. - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - There is no limitation for running subprocesses from different threads once the - watcher is installed. - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - :py:data:`SIGCHLD` is received). - - .. versionadded:: 3.8 - - .. deprecated:: 3.12 - -.. class:: SafeChildWatcher - - This implementation uses active event loop from the main thread to handle - :py:data:`SIGCHLD` signal. If the main thread has no running event loop another - thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* - complexity but requires a running event loop in the main thread to work. - - .. deprecated:: 3.12 - -.. class:: FastChildWatcher - - This implementation reaps every terminated processes by calling - ``os.waitpid(-1)`` directly, possibly breaking other code spawning - processes and waiting for their termination. - - There is no noticeable overhead when handling a big number of - children (*O(1)* each time a child terminates). - - This solution requires a running event loop in the main thread to work, as - :class:`SafeChildWatcher`. - - .. deprecated:: 3.12 - -.. class:: PidfdChildWatcher - - This implementation polls process file descriptors (pidfds) to await child - process termination. In some respects, :class:`PidfdChildWatcher` is a - "Goldilocks" child watcher implementation. It doesn't require signals or - threads, doesn't interfere with any processes launched outside the event - loop, and scales linearly with the number of subprocesses launched by the - event loop. The main disadvantage is that pidfds are specific to Linux, and - only work on recent (5.3+) kernels. - - .. versionadded:: 3.9 - - .. _asyncio-custom-policies: Custom Policies diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3f734f544afe21..7c08d65f26bc27 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -362,6 +362,11 @@ Datagram Transports This method does not block; it buffers the data and arranges for it to be sent out asynchronously. + .. versionchanged:: 3.13 + This method can be called with an empty bytes object to send a + zero-length datagram. The buffer size calculation used for flow + control is also updated to account for the datagram header. + .. method:: DatagramTransport.abort() Close the transport immediately, without waiting for pending @@ -417,8 +422,8 @@ Subprocess Transports Stop the subprocess. - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to + On POSIX systems, this method sends :py:const:`~signal.SIGTERM` to the subprocess. + On Windows, the Windows API function :c:func:`!TerminateProcess` is called to stop the subprocess. See also :meth:`subprocess.Popen.terminate`. diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d86fbc21351e2d..9b579cc1d5fdfe 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -62,6 +62,9 @@ Queue Remove and return an item from the queue. If queue is empty, wait until an item is available. + Raises :exc:`QueueShutDown` if the queue has been shut down and + is empty, or if the queue has been shut down immediately. + .. method:: get_nowait() Return an item if one is immediately available, else raise @@ -82,6 +85,8 @@ Queue Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: put_nowait(item) Put an item into the queue without blocking. @@ -92,6 +97,22 @@ Queue Return the number of items in the queue. + .. method:: shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + raise :exc:`QueueShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only + raise once the queue is empty. Set *immediate* to true to make + :meth:`~Queue.get` raise immediately instead. + + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` + will be unblocked. If *immediate* is true, a task will be marked + as done for each remaining item in the queue, which may unblock + callers of :meth:`~Queue.join`. + + .. versionadded:: 3.13 + .. method:: task_done() Indicate that a formerly enqueued task is complete. @@ -105,6 +126,9 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each + remaining item in the queue. + Raises :exc:`ValueError` if called more times than there were items placed in the queue. @@ -145,6 +169,14 @@ Exceptions on a queue that has reached its *maxsize*. +.. exception:: QueueShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is + called on a queue which has been shut down. + + .. versionadded:: 3.13 + + Examples ======== diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d8186b6ce75c79..3fdc79b3c6896c 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -77,8 +77,8 @@ and work with streams: .. versionchanged:: 3.7 Added the *ssl_handshake_timeout* parameter. - .. versionadded:: 3.8 - Added *happy_eyeballs_delay* and *interleave* parameters. + .. versionchanged:: 3.8 + Added the *happy_eyeballs_delay* and *interleave* parameters. .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -204,6 +204,10 @@ StreamReader directly; use :func:`open_connection` and :func:`start_server` instead. + .. method:: feed_eof() + + Acknowledge the EOF. + .. coroutinemethod:: read(n=-1) Read up to *n* bytes from the stream. @@ -256,8 +260,19 @@ StreamReader buffer is reset. The :attr:`IncompleteReadError.partial` attribute may contain a portion of the separator. + The *separator* may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of :exc:`LimitOverrunError`, + the shortest possible separator is considered to be the one that + matched. + .. versionadded:: 3.5.2 + .. versionchanged:: 3.13 + + The *separator* parameter may now be a :class:`tuple` of + separators. + .. method:: at_eof() Return ``True`` if the buffer is empty and :meth:`feed_eof` @@ -343,7 +358,7 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. coroutinemethod:: start_tls(sslcontext, \*, server_hostname=None, \ + .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \ ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Upgrade an existing stream-based connection to TLS. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index bf35b1cb798aee..b477a7fa2d37ef 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -240,7 +240,7 @@ their completion. .. note:: - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. + On Windows, :py:const:`~signal.SIGTERM` is an alias for :meth:`terminate`. ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes started with a *creationflags* parameter which includes ``CREATE_NEW_PROCESS_GROUP``. @@ -249,10 +249,10 @@ their completion. Stop the child process. - On POSIX systems this method sends :py:const:`signal.SIGTERM` to the + On POSIX systems this method sends :py:const:`~signal.SIGTERM` to the child process. - On Windows the Win32 API function :c:func:`TerminateProcess` is + On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child process. .. method:: kill() @@ -316,18 +316,6 @@ default. On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), :class:`SelectorEventLoop` has no subprocess support. -On UNIX *child watchers* are used for subprocess finish waiting, see -:ref:`asyncio-watchers` for more info. - - -.. versionchanged:: 3.8 - - UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from - different threads without any limitation. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - Note that alternative event loop implementations might have own limitations; please refer to their documentation. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 05bdf5488af143..3cf8e2737e85dc 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -216,8 +216,8 @@ Condition .. method:: notify(n=1) - Wake up at most *n* tasks (1 by default) waiting on this - condition. The method is no-op if no tasks are waiting. + Wake up *n* tasks (1 by default) waiting on this + condition. If fewer than *n* tasks are waiting they are all awakened. The lock must be acquired before this method is called and released shortly after. If called with an *unlocked* lock @@ -257,12 +257,18 @@ Condition Once awakened, the Condition re-acquires its lock and this method returns ``True``. + Note that a task *may* return from this call spuriously, + which is why the caller should always re-check the state + and be prepared to :meth:`wait` again. For this reason, you may + prefer to use :meth:`wait_for` instead. + .. coroutinemethod:: wait_for(predicate) Wait until a predicate becomes *true*. The predicate must be a callable which result will be - interpreted as a boolean value. The final value is the + interpreted as a boolean value. The method will repeatedly + :meth:`wait` until the predicate evaluates to *true*. The final value is the return value. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 88fdf41c7b2a3e..c5deac7e2748ae 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -334,6 +334,13 @@ and reliable way to wait for all tasks in the group to finish. Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. + If the task group is inactive (e.g. not yet entered, + already finished, or in the process of shutting down), + we will close the given ``coro``. + + .. versionchanged:: 3.13 + + Close the given coroutine if the task group is not active. Example:: @@ -385,6 +392,27 @@ is also included in the exception group. The same special case is made for :exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. +Task groups are careful not to mix up the internal cancellation used to +"wake up" their :meth:`~object.__aexit__` with cancellation requests +for the task in which they are running made by other parties. +In particular, when one task group is syntactically nested in another, +and both experience an exception in one of their child tasks simultaneously, +the inner task group will process its exceptions, and then the outer task group +will receive another cancellation and process its own exceptions. + +In the case where a task group is cancelled externally and also must +raise an :exc:`ExceptionGroup`, it will call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next +:keyword:`await`, so the cancellation is not lost. + +Task groups preserve the cancellation count +reported by :meth:`asyncio.Task.cancelling`. + +.. versionchanged:: 3.13 + + Improved handling of simultaneous internal and external cancellations + and correct preservation of cancellation counts. Sleeping ======== @@ -510,7 +538,7 @@ Running Tasks Concurrently # [2, 6, 24] .. note:: - If *return_exceptions* is False, cancelling gather() after it + If *return_exceptions* is false, cancelling gather() after it has been marked done won't cancel any submitted awaitables. For instance, gather can be marked done after propagating an exception to the caller, therefore, calling ``gather.cancel()`` @@ -592,7 +620,7 @@ Shielding From Cancellation is equivalent to:: - res = await shield(something()) + res = await something() *except* that if the coroutine containing it is cancelled, the Task running in ``something()`` is not cancelled. From the point @@ -828,23 +856,22 @@ Waiting Primitives *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. @@ -861,19 +888,50 @@ Waiting Primitives .. function:: as_completed(aws, *, timeout=None) - Run :ref:`awaitable objects <asyncio-awaitables>` in the *aws* - iterable concurrently. Return an iterator of coroutines. - Each coroutine returned can be awaited to get the earliest next - result from the iterable of the remaining awaitables. + Run :ref:`awaitable objects <asyncio-awaitables>` in the *aws* iterable + concurrently. The returned object can be iterated to obtain the results + of the awaitables as they finish. - Raises :exc:`TimeoutError` if the timeout occurs before - all Futures are done. + The object returned by ``as_completed()`` can be iterated as an + :term:`asynchronous iterator` or a plain :term:`iterator`. When asynchronous + iteration is used, the originally-supplied awaitables are yielded if they + are tasks or futures. This makes it easy to correlate previously-scheduled + tasks with their results. Example:: - Example:: + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - for coro in as_completed(aws): - earliest_result = await coro - # ... + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect + + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") + + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. + + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13:: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect + + A :exc:`TimeoutError` is raised if the timeout occurs before all awaitables + are done. This is raised by the ``async for`` loop during asynchronous + iteration or by the coroutines yielded during plain iteration. .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -885,6 +943,10 @@ Waiting Primitives .. versionchanged:: 3.12 Added support for generators yielding tasks. + .. versionchanged:: 3.13 + The result can now be used as either an :term:`asynchronous iterator` + or as a plain :term:`iterator` (previously it was only a plain iterator). + Running in Threads ================== @@ -1324,10 +1386,19 @@ Task Object with :meth:`uncancel`. :class:`TaskGroup` context managers use :func:`uncancel` in a similar fashion. - If end-user code is, for some reason, suppresing cancellation by + If end-user code is, for some reason, suppressing cancellation by catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. + When this method decrements the cancellation count to zero, + the method checks if a previous :meth:`cancel` call had arranged + for :exc:`CancelledError` to be thrown into the task. + If it hasn't been thrown yet, that arrangement will be + rescinded (by resetting the internal ``_must_cancel`` flag). + + .. versionchanged:: 3.13 + Changed to rescind pending cancellation requests upon reaching zero. + .. method:: cancelling() Return the number of pending cancellation requests to this Task, i.e., diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index c75ab47404c1e4..184f981c1021aa 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -1,5 +1,5 @@ -:mod:`asyncio` --- Asynchronous I/O -=================================== +:mod:`!asyncio` --- Asynchronous I/O +==================================== .. module:: asyncio :synopsis: Asynchronous I/O. @@ -46,9 +46,9 @@ Additionally, there are **low-level** APIs for *library and framework developers* to: * create and manage :ref:`event loops <asyncio-event-loop>`, which - provide asynchronous APIs for :meth:`networking <loop.create_server>`, - running :meth:`subprocesses <loop.subprocess_exec>`, - handling :meth:`OS signals <loop.add_signal_handler>`, etc; + provide asynchronous APIs for :ref:`networking <loop_create_server>`, + running :ref:`subprocesses <loop_subprocess_exec>`, + handling :ref:`OS signals <loop_add_signal_handler>`, etc; * implement efficient protocols using :ref:`transports <asyncio-transports-protocols>`; diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 3dbef69580d9b3..02d2f0807df8f6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -1,11 +1,11 @@ -:mod:`atexit` --- Exit handlers -=============================== +:mod:`!atexit` --- Exit handlers +================================ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro <skip@pobox.com> -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. moduleauthor:: Skip Montanaro <skip.montanaro@gmail.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index 8227a7955bef81..a2a90a00d0cfca 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -7,7 +7,7 @@ Audit events table This table contains all events raised by :func:`sys.audit` or :c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later (see :pep:`578`). +standard library. These calls were added in 3.8 or later (see :pep:`578`). See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for information on handling these events. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index d5b6af8c1928ef..834ab2536e6c14 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -1,5 +1,5 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== +:mod:`!base64` --- Base16, Base32, Base64, Base85 Data Encodings +================================================================ .. module:: base64 :synopsis: RFC 4648: Base16, Base32, Base64 Data Encodings; @@ -193,7 +193,7 @@ The modern interface provides: *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be - at most this many characters long. + at most this many characters long, excluding the trailing newline. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. @@ -244,6 +244,24 @@ The modern interface provides: .. versionadded:: 3.4 +.. function:: z85encode(s) + + Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) + and return the encoded :class:`bytes`. See `Z85 specification + <https://rfc.zeromq.org/spec/32/>`_ for more information. + + .. versionadded:: 3.13 + + +.. function:: z85decode(s) + + Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and + return the decoded :class:`bytes`. See `Z85 specification + <https://rfc.zeromq.org/spec/32/>`_ for more information. + + .. versionadded:: 3.13 + + The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index d201dc963b5995..85df7914a9a014 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -1,5 +1,5 @@ -:mod:`bdb` --- Debugger framework -================================= +:mod:`!bdb` --- Debugger framework +================================== .. module:: bdb :synopsis: Debugger framework. @@ -86,7 +86,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: temporary - True if a :class:`Breakpoint` at (file, line) is temporary. + ``True`` if a :class:`Breakpoint` at (file, line) is temporary. .. attribute:: cond @@ -99,7 +99,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: enabled - True if :class:`Breakpoint` is enabled. + ``True`` if :class:`Breakpoint` is enabled. .. attribute:: bpbynumber @@ -132,8 +132,8 @@ The :mod:`bdb` module also defines two classes: frame is considered to originate in a certain module is determined by the ``__name__`` in the frame globals. - .. versionadded:: 3.1 - The *skip* argument. + .. versionchanged:: 3.1 + Added the *skip* parameter. The following methods of :class:`Bdb` normally don't need to be overridden. @@ -148,8 +148,8 @@ The :mod:`bdb` module also defines two classes: .. method:: reset() - Set the :attr:`botframe`, :attr:`stopframe`, :attr:`returnframe` and - :attr:`quitting` attributes with values ready to start debugging. + Set the :attr:`!botframe`, :attr:`!stopframe`, :attr:`!returnframe` and + :attr:`quitting <Bdb.set_quit>` attributes with values ready to start debugging. .. method:: trace_dispatch(frame, event, arg) @@ -182,7 +182,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on the current line, invoke the :meth:`user_line` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_line`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -190,7 +190,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function call, invoke the :meth:`user_call` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_call`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -198,7 +198,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop on this function return, invoke the :meth:`user_return` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_return`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -206,7 +206,7 @@ The :mod:`bdb` module also defines two classes: If the debugger should stop at this exception, invokes the :meth:`user_exception` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set + Raise a :exc:`BdbQuit` exception if the :attr:`quitting <Bdb.set_quit>` flag is set (which can be set from :meth:`user_exception`). Return a reference to the :meth:`trace_dispatch` method for further tracing in that scope. @@ -215,22 +215,22 @@ The :mod:`bdb` module also defines two classes: .. method:: is_skipped_line(module_name) - Return True if *module_name* matches any skip pattern. + Return ``True`` if *module_name* matches any skip pattern. .. method:: stop_here(frame) - Return True if *frame* is below the starting frame in the stack. + Return ``True`` if *frame* is below the starting frame in the stack. .. method:: break_here(frame) - Return True if there is an effective breakpoint for this line. + Return ``True`` if there is an effective breakpoint for this line. Check whether a line or function breakpoint exists and is in effect. Delete temporary breakpoints based on information from :func:`effective`. .. method:: break_anywhere(frame) - Return True if any breakpoint exists for *frame*'s filename. + Return ``True`` if any breakpoint exists for *frame*'s filename. Derived classes should override these methods to gain control over debugger operation. @@ -240,6 +240,9 @@ The :mod:`bdb` module also defines two classes: Called from :meth:`dispatch_call` if a break might stop inside the called function. + *argument_list* is not used anymore and will always be ``None``. + The argument is kept for backwards compatibility. + .. method:: user_line(frame) Called from :meth:`dispatch_line` when either :meth:`stop_here` or @@ -286,6 +289,10 @@ The :mod:`bdb` module also defines two classes: Start debugging from *frame*. If *frame* is not specified, debugging starts from caller's frame. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + .. method:: set_continue() Stop only at breakpoints or when finished. If there are no breakpoints, @@ -293,8 +300,10 @@ The :mod:`bdb` module also defines two classes: .. method:: set_quit() - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in - the next call to one of the :meth:`dispatch_\*` methods. + .. index:: single: quitting (bdb.Bdb attribute) + + Set the :attr:`!quitting` attribute to ``True``. This raises :exc:`BdbQuit` in + the next call to one of the :meth:`!dispatch_\*` methods. Derived classes and clients can call the following methods to manipulate @@ -339,7 +348,7 @@ The :mod:`bdb` module also defines two classes: .. method:: get_break(filename, lineno) - Return True if there is a breakpoint for *lineno* in *filename*. + Return ``True`` if there is a breakpoint for *lineno* in *filename*. .. method:: get_breaks(filename, lineno) @@ -383,7 +392,7 @@ The :mod:`bdb` module also defines two classes: .. method:: run(cmd, globals=None, locals=None) Debug a statement executed via the :func:`exec` function. *globals* - defaults to :attr:`__main__.__dict__`, *locals* defaults to *globals*. + defaults to :attr:`!__main__.__dict__`, *locals* defaults to *globals*. .. method:: runeval(expr, globals=None, locals=None) @@ -403,7 +412,7 @@ Finally, the module defines the following functions: .. function:: checkfuncname(b, frame) - Return True if we should break here, depending on the way the + Return ``True`` if we should break here, depending on the way the :class:`Breakpoint` *b* was set. If it was set via line number, it checks if @@ -422,14 +431,14 @@ Finally, the module defines the following functions: :attr:`bplist <bdb.Breakpoint.bplist>` for the (:attr:`file <bdb.Breakpoint.file>`, :attr:`line <bdb.Breakpoint.line>`) (which must exist) that is :attr:`enabled <bdb.Breakpoint.enabled>`, for - which :func:`checkfuncname` is True, and that has neither a False + which :func:`checkfuncname` is true, and that has neither a false :attr:`condition <bdb.Breakpoint.cond>` nor positive :attr:`ignore <bdb.Breakpoint.ignore>` count. The *flag*, meaning that a - temporary breakpoint should be deleted, is False only when the + temporary breakpoint should be deleted, is ``False`` only when the :attr:`cond <bdb.Breakpoint.cond>` cannot be evaluated (in which case, :attr:`ignore <bdb.Breakpoint.ignore>` count is ignored). - If no such entry exists, then (None, None) is returned. + If no such entry exists, then ``(None, None)`` is returned. .. function:: set_trace() diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 39fabb59bb1984..1bab785684bbab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -1,5 +1,5 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== +:mod:`!binascii` --- Convert between binary and ASCII +===================================================== .. module:: binascii :synopsis: Tools for converting between binary and various ASCII-encoded binary diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 8022c596f0af97..78da563397b625 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -1,5 +1,5 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== +:mod:`!bisect` --- Array bisection algorithm +============================================ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. @@ -19,9 +19,9 @@ linear searches or frequent resorting. The module is called :mod:`bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an -insertion point. Accordingly, the functions never call an :meth:`__eq__` +insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` method to determine whether a value has been found. Instead, the -functions only call the :meth:`__lt__` method and will return an insertion +functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. .. _bisect functions: @@ -73,13 +73,13 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is applied to *x* for the search step but not for the insertion step. - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) + Keep in mind that the *O*\ (log *n*) search is dominated by the slow *O*\ (*n*) insertion step. .. versionchanged:: 3.10 @@ -93,13 +93,13 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is applied to *x* for the search step but not for the insertion step. - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) + Keep in mind that the *O*\ (log *n*) search is dominated by the slow *O*\ (*n*) insertion step. .. versionchanged:: 3.10 @@ -115,7 +115,7 @@ thoughts in mind: * Bisection is effective for searching ranges of values. For locating specific values, dictionaries are more performant. -* The *insort()* functions are ``O(n)`` because the logarithmic search step +* The *insort()* functions are *O*\ (*n*) because the logarithmic search step is dominated by the linear time insertion step. * The search functions are stateless and discard key function results after diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst index 7e4f8fe0531567..644344e7fef29a 100644 --- a/Doc/library/builtins.rst +++ b/Doc/library/builtins.rst @@ -1,5 +1,5 @@ -:mod:`builtins` --- Built-in objects -==================================== +:mod:`!builtins` --- Built-in objects +===================================== .. module:: builtins :synopsis: The module that provides the built-in namespace. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ec4aeaa04395ac..ebe2e43febaefa 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -1,5 +1,5 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= +:mod:`!bz2` --- Support for :program:`bzip2` compression +======================================================== .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. @@ -91,7 +91,7 @@ The :mod:`bz2` module contains: and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following method: + :class:`BZ2File` also provides the following methods and attributes: .. method:: peek([n]) @@ -106,19 +106,69 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. method:: fileno() + + Return the file descriptor for the underlying file. + + .. versionadded:: 3.3 + + .. method:: readable() + + Return whether the file was opened for reading. + + .. versionadded:: 3.3 + + .. method:: seekable() + + Return whether the file supports seeking. + + .. versionadded:: 3.3 + + .. method:: writable() + + Return whether the file was opened for writing. + + .. versionadded:: 3.3 + + .. method:: read1(size=-1) + + Read up to *size* uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns ``b''`` if the file is at EOF. + + .. versionadded:: 3.3 + + .. method:: readinto(b) + + Read bytes into *b*. + + Returns the number of bytes read (0 for EOF). + + .. versionadded:: 3.3 + + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The bzip2 file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. - .. versionchanged:: 3.3 - The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`, - :meth:`read1` and :meth:`readinto` methods were added. - .. versionchanged:: 3.3 Support was added for *filename* being a :term:`file object` instead of an actual filename. - .. versionchanged:: 3.3 The ``'a'`` (append) mode was added, along with support for reading multi-stream files. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 157a7537f97dc6..d5876054da3eee 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -1,5 +1,5 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== +:mod:`!calendar` --- General calendar-related functions +======================================================= .. module:: calendar :synopsis: Functions for working with calendars, including some emulation @@ -196,6 +196,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is output (defaulting to the system default encoding). + .. method:: formatmonthname(theyear, themonth, withyear=True) + + Return a month name as an HTML table row. If *withyear* is true the year + will be included in the row, otherwise just the month name will be + used. + + :class:`!HTMLCalendar` has the following attributes you can override to customize the CSS classes used by the calendar: @@ -289,7 +296,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. note:: - The constructor, :meth:`formatweekday` and :meth:`formatmonthname` methods + The constructor, :meth:`!formatweekday` and :meth:`!formatmonthname` methods of these two classes temporarily change the ``LC_TIME`` locale to the given *locale*. Because the current locale is a process-wide setting, they are not thread-safe. @@ -358,7 +365,7 @@ For simple text calendars this module provides the following functions. .. function:: month(theyear, themonth, w=0, l=0) - Returns a month's calendar in a multi-line string using the :meth:`formatmonth` + Returns a month's calendar in a multi-line string using the :meth:`~TextCalendar.formatmonth` of the :class:`TextCalendar` class. @@ -370,7 +377,7 @@ For simple text calendars this module provides the following functions. .. function:: calendar(year, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using - the :meth:`formatyear` of the :class:`TextCalendar` class. + the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. .. function:: timegm(tuple) @@ -505,7 +512,7 @@ to interactively print a calendar. python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] - [year] [month] + [-f FIRST_WEEKDAY] [year] [month] For example, to print a calendar for the year 2000: @@ -579,10 +586,17 @@ The following options are accepted: or as an HTML document. +.. option:: --first-weekday FIRST_WEEKDAY, -f FIRST_WEEKDAY + + The weekday to start each week. + Must be a number between 0 (Monday) and 6 (Sunday). + Defaults to 0. + + .. versionadded:: 3.13 + .. option:: year The year to print the calendar for. - Must be a number between 1 and 9999. Defaults to the current year. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index fdac51d9603ceb..381a8332f4b187 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -1,5 +1,5 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== +:mod:`!cmath` --- Mathematical functions for complex numbers +============================================================ .. module:: cmath :synopsis: Mathematical functions for complex numbers. @@ -43,10 +43,7 @@ Conversions to and from polar coordinates A Python complex number ``z`` is stored internally using *rectangular* or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j +part* ``z.real`` and its *imaginary part* ``z.imag``. *Polar coordinates* give an alternative way to represent a complex number. In polar coordinates, a complex number *z* is defined by the @@ -90,7 +87,7 @@ rectangular coordinates to polar coordinates and back. .. function:: rect(r, phi) Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. + Equivalent to ``complex(r * math.cos(phi), r * math.sin(phi))``. Power and logarithmic functions diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index fd5df96dfd0b3d..66544f82f6ff3f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -1,5 +1,5 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= +:mod:`!cmd` --- Support for line-oriented command interpreters +============================================================== .. module:: cmd :synopsis: Build line-oriented command interpreters. @@ -26,6 +26,13 @@ interface. key; it defaults to :kbd:`Tab`. If *completekey* is not :const:`None` and :mod:`readline` is available, command completion is done automatically. + The default, ``'tab'``, is treated specially, so that it refers to the + :kbd:`Tab` key on every :data:`readline.backend`. + Specifically, if :data:`readline.backend` is ``editline``, + ``Cmd`` will use ``'^I'`` instead of ``'tab'``. + Note that other values are not treated this way, and might only work + with a specific backend. + The optional arguments *stdin* and *stdout* specify the input and output file objects that the Cmd instance or subclass instance will use for input and output. If not specified, they will default to :data:`sys.stdin` and @@ -35,6 +42,9 @@ interface. :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be ignored. + .. versionchanged:: 3.13 + ``completekey='tab'`` is replaced by ``'^I'`` for ``editline``. + .. _cmd-objects: @@ -66,29 +76,32 @@ A :class:`Cmd` instance has the following methods: single: ! (exclamation); in a command interpreter An interpreter instance will recognize a command name ``foo`` if and only if it - has a method :meth:`do_foo`. As a special case, a line beginning with the + has a method :meth:`!do_foo`. As a special case, a line beginning with the character ``'?'`` is dispatched to the method :meth:`do_help`. As another special case, a line beginning with the character ``'!'`` is dispatched to the - method :meth:`do_shell` (if such a method is defined). + method :meth:`!do_shell` (if such a method is defined). This method will return when the :meth:`postcmd` method returns a true value. The *stop* argument to :meth:`postcmd` is the return value from the command's - corresponding :meth:`do_\*` method. + corresponding :meth:`!do_\*` method. If completion is enabled, completing commands will be done automatically, and - completing of commands args is done by calling :meth:`complete_foo` with + completing of commands args is done by calling :meth:`!complete_foo` with arguments *text*, *line*, *begidx*, and *endidx*. *text* is the string prefix we are attempting to match: all returned matches must begin with it. *line* is the current input line with leading whitespace removed, *begidx* and *endidx* are the beginning and ending indexes of the prefix text, which could be used to provide different completion depending upon which position the argument is in. - All subclasses of :class:`Cmd` inherit a predefined :meth:`do_help`. This + +.. method:: Cmd.do_help(arg) + + All subclasses of :class:`Cmd` inherit a predefined :meth:`!do_help`. This method, called with an argument ``'bar'``, invokes the corresponding method - :meth:`help_bar`, and if that is not present, prints the docstring of - :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all + :meth:`!help_bar`, and if that is not present, prints the docstring of + :meth:`!do_bar`, if available. With no argument, :meth:`!do_help` lists all available help topics (that is, all commands with corresponding - :meth:`help_\*` methods or commands that have docstrings), and also lists any + :meth:`!help_\*` methods or commands that have docstrings), and also lists any undocumented commands. @@ -98,7 +111,7 @@ A :class:`Cmd` instance has the following methods: This may be overridden, but should not normally need to be; see the :meth:`precmd` and :meth:`postcmd` methods for useful execution hooks. The return value is a flag indicating whether interpretation of commands by the - interpreter should stop. If there is a :meth:`do_\*` method for the command + interpreter should stop. If there is a :meth:`!do_\*` method for the command *str*, the return value of that method is returned, otherwise the return value from the :meth:`default` method is returned. @@ -118,7 +131,7 @@ A :class:`Cmd` instance has the following methods: .. method:: Cmd.completedefault(text, line, begidx, endidx) Method called to complete an input line when no command-specific - :meth:`complete_\*` method is available. By default, it returns an empty list. + :meth:`!complete_\*` method is available. By default, it returns an empty list. .. method:: Cmd.columnize(list, displaywidth=80) @@ -199,14 +212,14 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.misc_header The header to issue if the help output has a section for miscellaneous help - topics (that is, there are :meth:`help_\*` methods without corresponding - :meth:`do_\*` methods). + topics (that is, there are :meth:`!help_\*` methods without corresponding + :meth:`!do_\*` methods). .. attribute:: Cmd.undoc_header The header to issue if the help output has a section for undocumented commands - (that is, there are :meth:`do_\*` methods without corresponding :meth:`help_\*` + (that is, there are :meth:`!do_\*` methods without corresponding :meth:`!help_\*` methods). @@ -219,8 +232,8 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.use_rawinput A flag, defaulting to true. If true, :meth:`cmdloop` uses :func:`input` to - display a prompt and read the next command; if false, :meth:`sys.stdout.write` - and :meth:`sys.stdin.readline` are used. (This means that by importing + display a prompt and read the next command; if false, :data:`sys.stdout.write() <sys.stdout>` + and :data:`sys.stdin.readline() <sys.stdin>` are used. (This means that by importing :mod:`readline`, on systems that support it, the interpreter will automatically support :program:`Emacs`\ -like line editing and command-history keystrokes.) @@ -239,14 +252,14 @@ This section presents a simple example of how to build a shell around a few of the commands in the :mod:`turtle` module. Basic turtle commands such as :meth:`~turtle.forward` are added to a -:class:`Cmd` subclass with method named :meth:`do_forward`. The argument is +:class:`Cmd` subclass with method named :meth:`!do_forward`. The argument is converted to a number and dispatched to the turtle module. The docstring is used in the help utility provided by the shell. The example also includes a basic record and playback facility implemented with the :meth:`~Cmd.precmd` method which is responsible for converting the input to -lowercase and writing the commands to a file. The :meth:`do_playback` method -reads the file and adds the recorded commands to the :attr:`cmdqueue` for +lowercase and writing the commands to a file. The :meth:`!do_playback` method +reads the file and adds the recorded commands to the :attr:`~Cmd.cmdqueue` for immediate playback:: import cmd, sys diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index b2379befeffcba..5174515ffc23ed 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -36,6 +36,7 @@ The following modules have a command-line interface. * :mod:`pyclbr` * :mod:`pydoc` * :mod:`quopri` +* :ref:`random <random-cli>` * :mod:`runpy` * :ref:`site <site-commandline>` * :ref:`sqlite3 <sqlite3-cli>` diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 3d7f43c86a0557..8f7692df9fb22d 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -1,5 +1,5 @@ -:mod:`code` --- Interpreter base classes -======================================== +:mod:`!code` --- Interpreter base classes +========================================= .. module:: code :synopsis: Facilities to implement read-eval-print loops. @@ -18,25 +18,30 @@ build applications which provide an interactive interpreter prompt. This class deals with parsing and interpreter state (the user's namespace); it does not deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. + specifies a mapping to use as the namespace in which code will be executed; + it defaults to a newly created dictionary with key ``'__name__'`` set to + ``'__console__'`` and key ``'__doc__'`` set to ``None``. -.. class:: InteractiveConsole(locals=None, filename="<console>") +.. class:: InteractiveConsole(locals=None, filename="<console>", local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is true, + ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but + instead return to the calling code. + .. versionchanged:: 3.13 + Added *local_exit* parameter. -.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None) +.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False) Convenience function to run a read-eval-print loop. This creates a new instance of :class:`InteractiveConsole` and sets *readfunc* to be used as the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` + use as the default namespace for the interpreter loop. If *local_exit* is provided, + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`~InteractiveConsole.interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. @@ -44,6 +49,8 @@ build applications which provide an interactive interpreter prompt. .. versionchanged:: 3.6 Added *exitmsg* parameter. + .. versionchanged:: 3.13 + Added *local_exit* parameter. .. function:: compile_command(source, filename="<input>", symbol="single") diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2db4a67d1973d5..2cfd8a1eaee806 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1,5 +1,5 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= +:mod:`!codecs` --- Codec registry and base classes +================================================== .. module:: codecs :synopsis: Encode and decode data and streams. @@ -520,44 +520,46 @@ The base :class:`Codec` class defines these methods which also define the function interfaces of the stateless encoder and decoder: -.. method:: Codec.encode(input, errors='strict') +.. class:: Codec - Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular - character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). + .. method:: encode(input, errors='strict') - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + Encodes the object *input* and returns a tuple (output object, length consumed). + For instance, :term:`text encoding` converts + a string object to a bytes object using a particular + character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The encoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamWriter` for codecs which have to keep state in order to make + encoding efficient. + The encoder must be able to handle zero length input and return an empty object + of the output object type in this situation. -.. method:: Codec.decode(input, errors='strict') - Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular - character set encoding to a string object. + .. method:: decode(input, errors='strict') - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only - buffer interface -- for example, buffer objects and memory mapped files. + Decodes the object *input* and returns a tuple (output object, length + consumed). For instance, for a :term:`text encoding`, decoding converts + a bytes object encoded using a particular + character set encoding to a string object. - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + For text encodings and bytes-to-bytes codecs, + *input* must be a bytes object or one which provides the read-only + buffer interface -- for example, buffer objects and memory mapped files. - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The decoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamReader` for codecs which have to keep state in order to make + decoding efficient. + + The decoder must be able to handle zero length input and return an empty object + of the output object type in this situation. Incremental Encoding and Decoding @@ -705,7 +707,7 @@ Stream Encoding and Decoding The :class:`StreamWriter` and :class:`StreamReader` classes provide generic working interfaces which can be used to implement new encoding submodules very -easily. See :mod:`encodings.utf_8` for an example of how this is done. +easily. See :mod:`!encodings.utf_8` for an example of how this is done. .. _stream-writer-objects: @@ -895,9 +897,10 @@ The design is such that one can use the factory functions returned by the .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* - work on the backend — the data in *stream*. + *encode* and *decode* work on the frontend — the data visible to + code calling :meth:`~StreamReader.read` and :meth:`~StreamWriter.write`, + while *Reader* and *Writer* + work on the backend — the data in *stream*. You can use these objects to do transparent transcodings, e.g., from Latin-1 to UTF-8 and back. @@ -1129,7 +1132,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp875 | | Greek | +-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | +| cp932 | 932, ms932, mskanji, ms-kanji, | Japanese | +| | windows-31j | | +-----------------+--------------------------------+--------------------------------+ | cp949 | 949, ms949, uhc | Korean | +-----------------+--------------------------------+--------------------------------+ @@ -1417,8 +1421,8 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` | | quotedprintable, | quoted printable. | ``quotetabs=True`` / | | | quoted_printable | | :meth:`quopri.decode` | +----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode. | :meth:`uu.decode` | +| uu_codec | uu | Convert the operand using | | +| | | uuencode. | | +----------------------+------------------+------------------------------+------------------------------+ | zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | | | | gzip. | :meth:`zlib.decompress` | @@ -1474,7 +1478,7 @@ Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the -third-party `idna module <https://pypi.org/project/idna/>`_. +third-party :pypi:`idna` module. These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as @@ -1537,13 +1541,13 @@ This module implements the ANSI codepage (CP_ACP). .. availability:: Windows. -.. versionchanged:: 3.3 - Support any error handler. - .. versionchanged:: 3.2 Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used to encode, and ``'ignore'`` to decode. +.. versionchanged:: 3.3 + Support any error handler. + :mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature ------------------------------------------------------------- diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 55606e1c5f09ac..16f674adb4b22b 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -1,5 +1,5 @@ -:mod:`codeop` --- Compile Python code -===================================== +:mod:`!codeop` --- Compile Python code +====================================== .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 43a3286ba832cf..a63a55caa0f66d 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -1,5 +1,5 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== +:mod:`!collections.abc` --- Abstract Base Classes for Containers +================================================================ .. module:: collections.abc :synopsis: Abstract base classes for containers @@ -22,7 +22,7 @@ This module provides :term:`abstract base classes <abstract base class>` that can be used to test whether a class provides a particular interface; for -example, whether it is :term:`hashable` or whether it is a mapping. +example, whether it is :term:`hashable` or whether it is a :term:`mapping`. An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. @@ -73,7 +73,7 @@ of the API: >>> isinstance(D(), Sequence) True -In this example, class :class:`D` does not need to define +In this example, class :class:`!D` does not need to define ``__contains__``, ``__iter__``, and ``__reversed__`` because the :ref:`in-operator <comparisons>`, the :term:`iteration <iterable>` logic, and the :func:`reversed` function automatically fall back to @@ -87,7 +87,7 @@ the required methods (unless those methods have been set to class E: def __iter__(self): ... - def __next__(next): ... + def __next__(self): ... .. doctest:: @@ -136,14 +136,11 @@ ABC Inherits from Abstract Methods Mi :class:`Collection` ``__len__`` ``index``, and ``count`` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` + ``__setitem__``, ``append``, ``clear``, ``reverse``, ``extend``, + ``__delitem__``, ``pop``, ``remove``, and ``__iadd__`` ``__len__``, ``insert`` -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` @@ -183,16 +180,16 @@ ABC Inherits from Abstract Methods Mi .. rubric:: Footnotes -.. [1] These ABCs override :meth:`object.__subclasshook__` to support +.. [1] These ABCs override :meth:`~abc.ABCMeta.__subclasshook__` to support testing an interface by verifying the required methods are present and have not been set to :const:`None`. This only works for simple interfaces. More complex interfaces require registration or direct subclassing. .. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are - registered as :class:`Iterable` or that have an :meth:`__iter__` + registered as :class:`Iterable` or that have an :meth:`~container.__iter__` method, but it does not detect classes that iterate with the - :meth:`__getitem__` method. The only reliable way to determine + :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -202,27 +199,28 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Container - ABC for classes that provide the :meth:`__contains__` method. + ABC for classes that provide the :meth:`~object.__contains__` method. .. class:: Hashable - ABC for classes that provide the :meth:`__hash__` method. + ABC for classes that provide the :meth:`~object.__hash__` method. .. class:: Sized - ABC for classes that provide the :meth:`__len__` method. + ABC for classes that provide the :meth:`~object.__len__` method. .. class:: Callable - ABC for classes that provide the :meth:`__call__` method. + ABC for classes that provide the :meth:`~object.__call__` method. .. class:: Iterable - ABC for classes that provide the :meth:`__iter__` method. + ABC for classes that provide the :meth:`~container.__iter__` method. Checking ``isinstance(obj, Iterable)`` detects classes that are registered - as :class:`Iterable` or that have an :meth:`__iter__` method, but it does - not detect classes that iterate with the :meth:`__getitem__` method. + as :class:`Iterable` or that have an :meth:`~container.__iter__` method, + but it does + not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -240,30 +238,29 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Reversible - ABC for iterable classes that also provide the :meth:`__reversed__` + ABC for iterable classes that also provide the :meth:`~object.__reversed__` method. .. versionadded:: 3.6 .. class:: Generator - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, + ABC for :term:`generator` classes that implement the protocol defined in + :pep:`342` that extends :term:`iterators <iterator>` with the + :meth:`~generator.send`, :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. .. versionadded:: 3.5 .. class:: Sequence MutableSequence - ByteString ABCs for read-only and mutable :term:`sequences <sequence>`. Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`__getitem__` method. - Consequently, if :meth:`__getitem__` is implemented with constant + :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make + repeated calls to the underlying :meth:`~object.__getitem__` method. + Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will @@ -273,16 +270,10 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. - .. deprecated-removed:: 3.12 3.14 - The :class:`ByteString` ABC has been deprecated. - For use in typing, prefer a union, like ``bytes | bytearray``, or - :class:`collections.abc.Buffer`. - For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - .. class:: Set MutableSet - ABCs for read-only and mutable sets. + ABCs for read-only and mutable :ref:`sets <types-set>`. .. class:: Mapping MutableMapping @@ -299,16 +290,16 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Awaitable ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. + expressions. Custom implementations must provide the + :meth:`~object.__await__` method. :term:`Coroutine <coroutine>` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators <generator>` + decorated with :func:`@types.coroutine <types.coroutine>`) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -316,17 +307,17 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Coroutine - ABC for coroutine compatible classes. These implement the + ABC for :term:`coroutine` compatible classes. These implement the following methods, defined in :ref:`coroutine-objects`: :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. + :meth:`~object.__await__`. All :class:`Coroutine` instances are also + instances of :class:`Awaitable`. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators <generator>` + decorated with :func:`@types.coroutine <types.coroutine>`) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -334,7 +325,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncIterable - ABC for classes that provide ``__aiter__`` method. See also the + ABC for classes that provide an ``__aiter__`` method. See also the definition of :term:`asynchronous iterable`. .. versionadded:: 3.5 @@ -348,7 +339,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncGenerator - ABC for asynchronous generator classes that implement the protocol + ABC for :term:`asynchronous generator` classes that implement the protocol defined in :pep:`525` and :pep:`492`. .. versionadded:: 3.6 @@ -373,9 +364,9 @@ particular functionality, for example:: Several of the ABCs are also useful as mixins that make it easier to develop classes supporting container APIs. For example, to write a class supporting the full :class:`Set` API, it is only necessary to supply the three underlying -abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. -The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: +abstract methods: :meth:`~object.__contains__`, :meth:`~container.__iter__`, and +:meth:`~object.__len__`. The ABC supplies the remaining methods such as +:meth:`!__and__` and :meth:`~frozenset.isdisjoint`:: class ListBasedSet(collections.abc.Set): ''' Alternate set implementation favoring space over speed @@ -403,23 +394,24 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: (1) Since some set operations create new sets, the default mixin methods need - a way to create new instances from an iterable. The class constructor is + a way to create new instances from an :term:`iterable`. The class constructor is assumed to have a signature in the form ``ClassName(iterable)``. - That assumption is factored-out to an internal classmethod called - :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. + That assumption is factored-out to an internal :class:`classmethod` called + :meth:`!_from_iterable` which calls ``cls(iterable)`` to produce a new set. If the :class:`Set` mixin is being used in a class with a different - constructor signature, you will need to override :meth:`_from_iterable` + constructor signature, you will need to override :meth:`!_from_iterable` with a classmethod or regular method that can construct new instances from an iterable argument. (2) To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, + semantics are fixed), redefine :meth:`~object.__le__` and + :meth:`~object.__ge__`, then the other operations will automatically follow suit. (3) - The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value - for the set; however, :meth:`__hash__` is not defined because not all sets + The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value + for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 02df6e9f137926..ce89101d6b667c 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1,5 +1,5 @@ -:mod:`collections` --- Container datatypes -========================================== +:mod:`!collections` --- Container datatypes +=========================================== .. module:: collections :synopsis: Container datatypes @@ -99,7 +99,7 @@ The class can be used to simulate nested scopes and is useful in templating. :func:`super` function. A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. - Note, the iteration order of a :class:`ChainMap()` is determined by + Note, the iteration order of a :class:`ChainMap` is determined by scanning the mappings last to first:: >>> baseline = {'music': 'bach', 'art': 'rembrandt'} @@ -134,7 +134,7 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - <https://code.activestate.com/recipes/577434/>`_ has options to control + <https://code.activestate.com/recipes/577434-nested-contexts-a-chain-of-mapping-objects/>`_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. @@ -343,7 +343,7 @@ superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. All of those tests treat missing elements as having zero counts so that ``Counter(a=1) == Counter(a=1, b=0)`` returns true. -.. versionadded:: 3.10 +.. versionchanged:: 3.10 Rich comparison operations were added. .. versionchanged:: 3.10 @@ -458,10 +458,10 @@ or subtracting from an empty counter. Deques are a generalization of stacks and queues (the name is pronounced "deck" and is short for "double-ended queue"). Deques support thread-safe, memory efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. + same *O*\ (1) performance in either direction. Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for + fast fixed-length operations and incur *O*\ (*n*) memory movement costs for ``pop(0)`` and ``insert(0, v)`` operations which change both the size and position of the underlying data representation. @@ -585,7 +585,7 @@ or subtracting from an empty counter. In addition to the above, deques support iteration, pickling, ``len(d)``, ``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in +the first element. Indexed access is *O*\ (1) at both ends but slows to *O*\ (*n*) in the middle. For fast random access, use lists instead. Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, @@ -743,12 +743,12 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, If calling :attr:`default_factory` raises an exception this exception is propagated unchanged. - This method is called by the :meth:`__getitem__` method of the + This method is called by the :meth:`~object.__getitem__` method of the :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. + returns or raises is then returned or raised by :meth:`~object.__getitem__`. Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal + :meth:`~object.__getitem__`. This means that :meth:`get` will, like normal dictionaries, return ``None`` as a default rather than using :attr:`default_factory`. @@ -981,6 +981,10 @@ field names, the method and attribute names start with an underscore. Named tuples are also supported by generic function :func:`copy.replace`. + .. versionchanged:: 3.13 + Raise :exc:`TypeError` instead of :exc:`ValueError` for invalid + keyword arguments. + .. attribute:: somenamedtuple._fields Tuple of strings listing the field names. Useful for introspection diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index b672a05b39145d..125d62b174088a 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -1,5 +1,5 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== +:mod:`!colorsys` --- Conversions between color systems +====================================================== .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index df1eefab839cc1..d9c0cb67a92aa7 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -1,5 +1,5 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== +:mod:`!compileall` --- Byte-compile Python libraries +==================================================== .. module:: compileall :synopsis: Tools for byte-compiling all Python source files in a directory tree. @@ -226,7 +226,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. @@ -289,7 +289,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.9 Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. @@ -318,7 +318,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index deefb8606ead84..e3b24451188cc4 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -1,5 +1,5 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== +:mod:`!concurrent.futures` --- Launching parallel tasks +======================================================= .. module:: concurrent.futures :synopsis: Execute computations concurrently using threads or processes. @@ -39,14 +39,14 @@ Executor Objects future = executor.submit(pow, 323, 1235) print(future.result()) - .. method:: map(func, *iterables, timeout=None, chunksize=1) + .. method:: map(fn, *iterables, timeout=None, chunksize=1) - Similar to :func:`map(func, *iterables) <map>` except: + Similar to :func:`map(fn, *iterables) <map>` except: * the *iterables* are collected immediately rather than lazily; - * *func* is executed asynchronously and several calls to - *func* may be made concurrently. + * *fn* is executed asynchronously and several calls to + *fn* may be made concurrently. The returned iterator raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` is called and the result isn't available @@ -54,7 +54,7 @@ Executor Objects *timeout* can be an int or a float. If *timeout* is not specified or ``None``, there is no limit to the wait time. - If a *func* call raises an exception, then that exception will be + If a *fn* call raises an exception, then that exception will be raised when its value is retrieved from the iterator. When using :class:`ProcessPoolExecutor`, this method chops *iterables* @@ -171,8 +171,8 @@ And:: should be higher than the number of workers for :class:`ProcessPoolExecutor`. - .. versionadded:: 3.6 - The *thread_name_prefix* argument was added to allow users to + .. versionchanged:: 3.6 + Added the *thread_name_prefix* parameter to allow users to control the :class:`threading.Thread` names for worker threads created by the pool for easier debugging. @@ -275,7 +275,8 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a - :exc:`BrokenProcessPool` error is now raised. Previously, behaviour + :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. + Previously, behaviour was undefined but operations on the executor or its futures would often freeze or deadlock. @@ -493,23 +494,22 @@ Module Functions *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ + .. list-table:: + :header-rows: 1 + + * - Constant + - Description + + * - .. data:: FIRST_COMPLETED + - The function will return when any future finishes or is cancelled. + + * - .. data:: FIRST_EXCEPTION + - The function will return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to :const:`ALL_COMPLETED`. + + * - .. data:: ALL_COMPLETED + - The function will return when all futures finish or are cancelled. .. function:: as_completed(fs, timeout=None) @@ -570,7 +570,8 @@ Exception classes .. exception:: BrokenThreadPool Derived from :exc:`~concurrent.futures.BrokenExecutor`, this exception - class is raised when one of the workers of a :class:`ThreadPoolExecutor` + class is raised when one of the workers + of a :class:`~concurrent.futures.ThreadPoolExecutor` has failed initializing. .. versionadded:: 3.7 @@ -581,7 +582,8 @@ Exception classes Derived from :exc:`~concurrent.futures.BrokenExecutor` (formerly :exc:`RuntimeError`), this exception class is raised when one of the - workers of a :class:`ProcessPoolExecutor` has terminated in a non-clean + workers of a :class:`~concurrent.futures.ProcessPoolExecutor` + has terminated in a non-clean fashion (for example, if it was killed from the outside). .. versionadded:: 3.3 diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb282428c5fffc..6fae03cd5c96fd 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1,5 +1,5 @@ -:mod:`configparser` --- Configuration file parser -================================================= +:mod:`!configparser` --- Configuration file parser +================================================== .. module:: configparser :synopsis: Configuration file parser. @@ -147,23 +147,28 @@ case-insensitive and stored in lowercase [1]_. It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent -configuration while the previously existing keys are retained. +configuration while the previously existing keys are retained. The example +below reads in an ``override.ini`` file, which will override any conflicting +keys from the ``example.ini`` file. + +.. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 .. doctest:: - >>> another_config = configparser.ConfigParser() - >>> another_config.read('example.ini') - ['example.ini'] - >>> another_config['topsecret.server.example']['Port'] - '50022' - >>> another_config.read_string("[topsecret.server.example]\nPort=48484") - >>> another_config['topsecret.server.example']['Port'] - '48484' - >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) - >>> another_config['topsecret.server.example']['Port'] - '21212' - >>> another_config['topsecret.server.example']['ForwardX11'] - 'no' + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + This behaviour is equivalent to a :meth:`ConfigParser.read` call with several files passed to the *filenames* parameter. @@ -208,7 +213,7 @@ converters and customize the provided ones. [1]_ Fallback Values --------------- -As with a dictionary, you can use a section's :meth:`get` method to +As with a dictionary, you can use a section's :meth:`~ConfigParser.get` method to provide fallback values: .. doctest:: @@ -232,7 +237,7 @@ even if we specify a fallback: >>> topsecret.get('CompressionLevel', '3') '9' -One more thing to be aware of is that the parser-level :meth:`get` method +One more thing to be aware of is that the parser-level :meth:`~ConfigParser.get` method provides a custom, more complex interface, maintained for backwards compatibility. When using this method, a fallback value can be provided via the ``fallback`` keyword-only argument: @@ -271,9 +276,14 @@ out. Values can also span multiple lines, as long as they are indented deeper than the first line of the value. Depending on the parser's mode, blank lines may be treated as parts of multiline values or ignored. -By default, a valid section name can be any string that does not contain '\\n' or ']'. +By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. +The first section name may be omitted if the parser is configured to allow an +unnamed top level section with ``allow_unnamed_section=True``. In this case, +the keys/values may be retrieved by :const:`UNNAMED_SECTION` as in +``config[UNNAMED_SECTION]``. + Configuration files may include comments, prefixed by specific characters (``#`` and ``;`` by default [1]_). Comments may appear on their own on an otherwise empty line, possibly indented. [1]_ @@ -325,6 +335,27 @@ For example: # Did I mention we can indent comments, too? +.. _unnamed-sections: + +Unnamed Sections +---------------- + +The name of the first section (or unique) may be omitted and values +retrieved by the :const:`UNNAMED_SECTION` attribute. + +.. doctest:: + + >>> config = """ + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> unnamed = configparser.ConfigParser(allow_unnamed_section=True) + >>> unnamed.read_string(config) + >>> unnamed.get(configparser.UNNAMED_SECTION, 'option') + 'value' + Interpolation of values ----------------------- @@ -481,7 +512,7 @@ historical background and it's very likely that you will want to customize some of the features. The most common way to change the way a specific config parser works is to use -the :meth:`__init__` options: +the :meth:`!__init__` options: * *defaults*, default value: ``None`` @@ -491,7 +522,7 @@ the :meth:`__init__` options: the documented default. Hint: if you want to specify default values for a specific section, use - :meth:`read_dict` before you read the actual file. + :meth:`~ConfigParser.read_dict` before you read the actual file. * *dict_type*, default value: :class:`dict` @@ -635,8 +666,8 @@ the :meth:`__init__` options: * *strict*, default value: ``True`` When set to ``True``, the parser will not allow for any section or option - duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict + duplicates while reading from a single source (using :meth:`~ConfigParser.read_file`, + :meth:`~ConfigParser.read_string` or :meth:`~ConfigParser.read_dict`). It is recommended to use strict parsers in new applications. .. versionchanged:: 3.2 @@ -697,7 +728,7 @@ the :meth:`__init__` options: desirable, users may define them in a subclass or pass a dictionary where each key is a name of the converter and each value is a callable implementing said conversion. For instance, passing ``{'decimal': decimal.Decimal}`` would add - :meth:`getdecimal` on both the parser object and all section proxies. In + :meth:`!getdecimal` on both the parser object and all section proxies. In other words, it will be possible to write both ``parser_instance.getdecimal('section', 'key', fallback=0)`` and ``parser_instance['section'].getdecimal('key', 0)``. @@ -955,9 +986,34 @@ ConfigParser Objects When *converters* is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser + converter gets its own corresponding :meth:`!get*()` method on the parser object and section proxies. + It is possible to read several configurations into a single + :class:`ConfigParser`, where the most recently added configuration has the + highest priority. Any conflicting keys are taken from the more recent + configuration while the previously existing keys are retained. The example + below reads in an ``override.ini`` file, which will override any conflicting + keys from the ``example.ini`` file. + + .. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 + + .. doctest:: + + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -978,6 +1034,10 @@ ConfigParser Objects The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + Raise a :exc:`MultilineContinuationError` when *allow_no_value* is + ``True``, and a key without a value is continued with an indented line. + .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1045,14 +1105,14 @@ ConfigParser Objects config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')], encoding='cp1250') - .. versionadded:: 3.2 - The *encoding* parameter. Previously, all files were read using the - default encoding for :func:`open`. + .. versionchanged:: 3.2 + Added the *encoding* parameter. + Previously, all files were read using the default encoding for :func:`open`. - .. versionadded:: 3.6.1 + .. versionchanged:: 3.6.1 The *filenames* parameter accepts a :term:`path-like object`. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 The *filenames* parameter accepts a :class:`bytes` object. @@ -1062,11 +1122,11 @@ ConfigParser Objects yielding Unicode strings (for example files opened in text mode). Optional argument *source* specifies the name of the file being read. If - not given and *f* has a :attr:`name` attribute, that is used for + not given and *f* has a :attr:`!name` attribute, that is used for *source*; the default is ``'<???>'``. .. versionadded:: 3.2 - Replaces :meth:`readfp`. + Replaces :meth:`!readfp`. .. method:: read_string(string, source='<string>') @@ -1212,9 +1272,14 @@ ConfigParser Objects names is stripped before :meth:`optionxform` is called. +.. data:: UNNAMED_SECTION + + A special object representing a section name used to reference the unnamed section (see :ref:`unnamed-sections`). + + .. data:: MAX_INTERPOLATION_DEPTH - The maximum depth for recursive interpolation for :meth:`get` when the *raw* + The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* parameter is false. This is relevant only when the default *interpolation* is used. @@ -1287,13 +1352,13 @@ Exceptions .. exception:: DuplicateSectionError - Exception raised if :meth:`add_section` is called with the name of a section + Exception raised if :meth:`~ConfigParser.add_section` is called with the name of a section that is already present or in strict parsers when a section if found more than once in a single input file, string or dictionary. - .. versionadded:: 3.2 - Optional ``source`` and ``lineno`` attributes and arguments to - :meth:`__init__` were added. + .. versionchanged:: 3.2 + Added the optional *source* and *lineno* attributes and parameters to + :meth:`!__init__`. .. exception:: DuplicateOptionError @@ -1345,9 +1410,16 @@ Exceptions Exception raised when errors occur attempting to parse a file. -.. versionchanged:: 3.12 - The ``filename`` attribute and :meth:`__init__` constructor argument were - removed. They have been available using the name ``source`` since 3.2. + .. versionchanged:: 3.12 + The ``filename`` attribute and :meth:`!__init__` constructor argument were + removed. They have been available using the name ``source`` since 3.2. + +.. exception:: MultilineContinuationError + + Exception raised when a key without a corresponding value is continued with + an indented line. + + .. versionadded:: 3.13 .. rubric:: Footnotes diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 401dc9a320c5e0..6c1063cda6690e 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -33,29 +33,32 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. .. note:: - When a binary (or in-place) method returns ``NotImplemented`` the + When a binary (or in-place) method returns :data:`!NotImplemented` the interpreter will try the reflected operation on the other type (or some other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. + :data:`!NotImplemented`, the interpreter will raise an appropriate exception. + Incorrectly returning :data:`!NotImplemented` will result in a misleading + error message or the :data:`!NotImplemented` value being returned to Python code. See :ref:`implementing-the-arithmetic-operations` for examples. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`!NotImplemented` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`!NotImplemented` in a boolean context was deprecated. + + .. versionchanged:: 3.14 + Evaluating :data:`!NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. .. index:: single: ...; ellipsis literal diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 7cd081d1f54f43..27cf99446e5980 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -45,7 +45,7 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`with` statement context managers, without needing to - create a class or separate :meth:`__enter__` and :meth:`__exit__` methods. + create a class or separate :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. While many objects natively support use in with statements, sometimes a resource needs to be managed that isn't a context manager in its own right, @@ -106,8 +106,8 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`async with` statement asynchronous context managers, - without needing to create a class or separate :meth:`__aenter__` and - :meth:`__aexit__` methods. It must be applied to an :term:`asynchronous + without needing to create a class or separate :meth:`~object.__aenter__` and + :meth:`~object.__aexit__` methods. It must be applied to an :term:`asynchronous generator` function. A simple example:: @@ -182,6 +182,14 @@ Functions and classes provided: without needing to explicitly close ``page``. Even if an error occurs, ``page.close()`` will be called when the :keyword:`with` block is exited. + .. note:: + + Most types managing resources support the :term:`context manager` protocol, + which closes *thing* on leaving the :keyword:`with` statement. + As such, :func:`!closing` is most useful for third party types that don't + support context managers. + This example is purely for illustration purposes, + as :func:`~urllib.request.urlopen` would normally be used in a context manager. .. function:: aclosing(thing) @@ -304,15 +312,17 @@ Functions and classes provided: This context manager is :ref:`reentrant <reentrant-cms>`. - If the code within the :keyword:`!with` block raises an - :exc:`ExceptionGroup`, suppressed exceptions are removed from the - group. If any exceptions in the group are not suppressed, a group containing them is re-raised. + If the code within the :keyword:`!with` block raises a + :exc:`BaseExceptionGroup`, suppressed exceptions are removed from the + group. Any exceptions of the group which are not suppressed are re-raised in + a new group which is created using the original group's :meth:`~BaseExceptionGroup.derive` + method. .. versionadded:: 3.4 .. versionchanged:: 3.12 ``suppress`` now supports suppressing exceptions raised as - part of an :exc:`ExceptionGroup`. + part of an :exc:`BaseExceptionGroup`. .. function:: redirect_stdout(new_target) @@ -515,7 +525,7 @@ Functions and classes provided: # the with statement, even if attempts to open files later # in the list raise an exception - The :meth:`__enter__` method returns the :class:`ExitStack` instance, and + The :meth:`~object.__enter__` method returns the :class:`ExitStack` instance, and performs no additional operations. Each instance maintains a stack of registered callbacks that are called in @@ -543,9 +553,9 @@ Functions and classes provided: .. method:: enter_context(cm) - Enters a new context manager and adds its :meth:`__exit__` method to + Enters a new context manager and adds its :meth:`~object.__exit__` method to the callback stack. The return value is the result of the context - manager's own :meth:`__enter__` method. + manager's own :meth:`~object.__enter__` method. These context managers may suppress exceptions just as they normally would if used directly as part of a :keyword:`with` statement. @@ -556,18 +566,18 @@ Functions and classes provided: .. method:: push(exit) - Adds a context manager's :meth:`__exit__` method to the callback stack. + Adds a context manager's :meth:`~object.__exit__` method to the callback stack. As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`__enter__` implementation with a context manager's own - :meth:`__exit__` method. + part of an :meth:`~object.__enter__` implementation with a context manager's own + :meth:`~object.__exit__` method. If passed an object that is not a context manager, this method assumes it is a callback with the same signature as a context manager's - :meth:`__exit__` method and adds it directly to the callback stack. + :meth:`~object.__exit__` method and adds it directly to the callback stack. By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`__exit__` methods can. + same way context manager :meth:`~object.__exit__` methods can. The passed in object is returned from the function, allowing this method to be used as a function decorator. @@ -616,12 +626,12 @@ Functions and classes provided: asynchronous context managers, as well as having coroutines for cleanup logic. - The :meth:`close` method is not implemented, :meth:`aclose` must be used + The :meth:`~ExitStack.close` method is not implemented; :meth:`aclose` must be used instead. .. coroutinemethod:: enter_async_context(cm) - Similar to :meth:`enter_context` but expects an asynchronous context + Similar to :meth:`ExitStack.enter_context` but expects an asynchronous context manager. .. versionchanged:: 3.11 @@ -630,16 +640,16 @@ Functions and classes provided: .. method:: push_async_exit(exit) - Similar to :meth:`push` but expects either an asynchronous context manager + Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. .. method:: push_async_callback(callback, /, *args, **kwds) - Similar to :meth:`callback` but expects a coroutine function. + Similar to :meth:`ExitStack.callback` but expects a coroutine function. .. coroutinemethod:: aclose() - Similar to :meth:`close` but properly handles awaitables. + Similar to :meth:`ExitStack.close` but properly handles awaitables. Continuing the example for :func:`asynccontextmanager`:: @@ -714,7 +724,7 @@ Cleaning up in an ``__enter__`` implementation As noted in the documentation of :meth:`ExitStack.push`, this method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`__enter__` implementation fail. +steps in the :meth:`~object.__enter__` implementation fail. Here's an example of doing this for a context manager that accepts resource acquisition and release functions, along with an optional validation function, @@ -788,7 +798,7 @@ executing that callback:: if result: stack.pop_all() -This allows the intended cleanup up behaviour to be made explicit up front, +This allows the intended cleanup behaviour to be made explicit up front, rather than requiring a separate flag variable. If a particular application uses this pattern a lot, it can be simplified @@ -871,7 +881,7 @@ And also as a function decorator:: Note that there is one additional limitation when using context managers as function decorators: there's no way to access the return value of -:meth:`__enter__`. If that value is needed, then it is still necessary to use +:meth:`~object.__enter__`. If that value is needed, then it is still necessary to use an explicit ``with`` statement. .. seealso:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 0ac2f3d85749b7..8ae386b489fb4e 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -1,5 +1,5 @@ -:mod:`contextvars` --- Context Variables -======================================== +:mod:`!contextvars` --- Context Variables +========================================= .. module:: contextvars :synopsis: Context Variables @@ -131,7 +131,7 @@ Manual Context Management ctx: Context = copy_context() print(list(ctx.items())) - The function has an O(1) complexity, i.e. works equally fast for + The function has an *O*\ (1) complexity, i.e. works equally fast for contexts with a few context variables and for contexts that have a lot of them. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 74333b2e934814..95b41f988a035b 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -1,5 +1,5 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ +:mod:`!copy` --- Shallow and deep copy operations +================================================= .. module:: copy :synopsis: Shallow and deep copy operations. diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2a28c043f80723..6e3144824ebe91 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -1,5 +1,5 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== +:mod:`!copyreg` --- Register :mod:`!pickle` support functions +============================================================= .. module:: copyreg :synopsis: Register pickle support functions. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index aba398b8ee1e54..533cdf13974be6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -1,10 +1,10 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== +:mod:`!csv` --- CSV File Reading and Writing +============================================ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/csv.py` @@ -55,10 +55,11 @@ The :mod:`csv` module defines the following functions: .. function:: reader(csvfile, dialect='excel', **fmtparams) - Return a reader object which will iterate over lines in the given *csvfile*. - *csvfile* can be any object which supports the :term:`iterator` protocol and returns a - string each time its :meth:`!__next__` method is called --- :term:`file objects - <file object>` and list objects are both suitable. If *csvfile* is a file object, + Return a :ref:`reader object <reader-objects>` that will process + lines from the given *csvfile*. A csvfile must be an iterable of + strings, each in the reader's defined csv format. + A csvfile is most commonly a file-like object or list. + If *csvfile* is a file object, it should be opened with ``newline=''``. [1]_ An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of @@ -87,7 +88,7 @@ The :mod:`csv` module defines the following functions: Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with + :meth:`~io.TextIOBase.write` method. If *csvfile* is a file object, it should be opened with ``newline=''`` [1]_. An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of the @@ -155,8 +156,10 @@ The :mod:`csv` module defines the following classes: The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. + fieldnames and will be omitted from the results. If + *fieldnames* is provided, they will be used and the first row will be + included in the results. Regardless of how the fieldnames are determined, + the dictionary preserves their original ordering. If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults @@ -196,10 +199,10 @@ The :mod:`csv` module defines the following classes: Create an object which operates like a regular writer but maps dictionaries onto output rows. The *fieldnames* parameter is a :mod:`sequence <collections.abc>` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file + dictionary passed to the :meth:`~csvwriter.writerow` method are written to file *f*. The optional *restval* parameter specifies the value to be written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in + dictionary passed to the :meth:`~csvwriter.writerow` method contains a key not found in *fieldnames*, the optional *extrasaction* parameter indicates what action to take. If it is set to ``'raise'``, the default value, a :exc:`ValueError` @@ -243,7 +246,6 @@ The :mod:`csv` module defines the following classes: with open('students.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, dialect='unix') - ^^^^^^^^^^^^^^ .. class:: excel() @@ -309,6 +311,8 @@ An example for :class:`Sniffer` use:: # ... process CSV file contents here ... +.. _csv-constants: + The :mod:`csv` module defines the following constants: .. data:: QUOTE_ALL @@ -345,8 +349,10 @@ The :mod:`csv` module defines the following constants: ``None``. This is similar to :data:`QUOTE_ALL`, except that if a field value is ``None`` an empty (unquoted) string is written. - Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and - to otherwise behave as :data:`QUOTE_ALL`. + Instructs :class:`reader` objects to interpret an empty (unquoted) field + as ``None`` and to otherwise behave as :data:`QUOTE_ALL`. + + .. versionadded:: 3.12 .. data:: QUOTE_STRINGS @@ -357,6 +363,8 @@ The :mod:`csv` module defines the following constants: Instructs :class:`reader` objects to interpret an empty (unquoted) string as ``None`` and to otherwise behave as :data:`QUOTE_NONNUMERIC`. + .. versionadded:: 3.12 + The :mod:`csv` module defines the following exception: @@ -371,8 +379,8 @@ Dialects and Formatting Parameters To make it easier to specify the format of input and output records, specific formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or +subclass of the :class:`Dialect` class containing various attributes +describing the format of the CSV file. When creating :class:`reader` or :class:`writer` objects, the programmer can specify a string or a subclass of the :class:`Dialect` class as the dialect parameter. In addition to, or instead of, the *dialect* parameter, the programmer can also specify individual @@ -432,8 +440,8 @@ Dialects support the following attributes: .. attribute:: Dialect.quoting Controls when quotes should be generated by the writer and recognised by the - reader. It can take on any of the :const:`QUOTE_\*` constants (see section - :ref:`csv-contents`) and defaults to :const:`QUOTE_MINIMAL`. + reader. It can take on any of the :ref:`QUOTE_\* constants <csv-constants>` + and defaults to :const:`QUOTE_MINIMAL`. .. attribute:: Dialect.skipinitialspace @@ -447,6 +455,8 @@ Dialects support the following attributes: When ``True``, raise exception :exc:`Error` on bad CSV input. The default is ``False``. +.. _reader-objects: + Reader Objects -------------- @@ -487,9 +497,9 @@ DictReader objects have the following public attribute: Writer Objects -------------- -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by +:class:`writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ef3a9a0f5898af..e3d74d7dc0d91c 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1,5 +1,5 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= +:mod:`!ctypes` --- A foreign function library for Python +======================================================== .. module:: ctypes :synopsis: A foreign function library for Python. @@ -93,7 +93,6 @@ Accessing functions from loaded dlls Functions are accessed as attributes of dll objects:: - >>> from ctypes import * >>> libc.printf <_FuncPtr object at 0x...> >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS @@ -200,7 +199,7 @@ calls). Python objects that can directly be used as parameters in these function calls. ``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed as pointer to the memory block that contains their data (:c:expr:`char *` or -:c:expr:`wchar_t *`). Python integers are passed as the platforms default C +:c:expr:`wchar_t *`). Python integers are passed as the platform's default C :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn @@ -267,6 +266,20 @@ Fundamental data types (1) The constructor accepts any object with a truth value. +Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported, the following +complex types are available: + ++----------------------------------+---------------------------------+-----------------+ +| ctypes type | C type | Python type | ++==================================+=================================+=================+ +| :class:`c_float_complex` | :c:expr:`float complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_double_complex` | :c:expr:`double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ + + All these types can be created by calling them with an optional initializer of the correct type and value:: @@ -662,14 +675,22 @@ for debugging because they can provide useful information:: guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Structure/union layout, alignment and byte order +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`~Structure._pack_` class attribute in the subclass definition. -This must be set to a positive integer and specifies the maximum alignment for the fields. -This is what ``#pragma pack(n)`` also does in MSVC. +By default, Structure and Union fields are laid out in the same way the C +compiler does it. It is possible to override this behavior entirely by specifying a +:attr:`~Structure._layout_` class attribute in the subclass definition; see +the attribute documentation for details. + +It is possible to specify the maximum alignment for the fields by setting +the :attr:`~Structure._pack_` class attribute to a positive integer. +This matches what ``#pragma pack(n)`` does in MSVC. + +It is also possible to set a minimum alignment for how the subclass itself is packed in the +same way ``#pragma align(n)`` works in MSVC. +This can be achieved by specifying a ::attr:`~Structure._align_` class attribute +in the subclass definition. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -1113,10 +1134,6 @@ api:: >>> print(hex(version.value)) 0x30c00a0 -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - An extended example which also demonstrates the use of pointers accesses the :c:data:`PyImport_FrozenModules` pointer exported by Python. @@ -1335,8 +1352,9 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: +On macOS and Android, :func:`~ctypes.util.find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: >>> from ctypes.util import find_library >>> find_library("c") @@ -1441,7 +1459,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or +parameter, otherwise the underlying platform's :c:func:`!dlopen` or :c:func:`!LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1452,7 +1470,7 @@ configurable. The *use_errno* parameter, when set to true, enables a ctypes mechanism that allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` +:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` variable; if you call foreign functions created with ``use_errno=True`` then the :data:`errno` value before the function call is swapped with the ctypes private copy, the same happens immediately after the function call. @@ -2077,13 +2095,13 @@ Utility functions Does the same as the C ``sizeof`` operator. -.. function:: string_at(address, size=-1) +.. function:: string_at(ptr, size=-1) - This function returns the C string starting at memory address *address* as a bytes - object. If size is specified, it is used as size, otherwise the string is assumed + Return the byte string at *void \*ptr*. + If *size* is specified, it is used as size, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.string_at address,size ctypes.string_at + .. audit-event:: ctypes.string_at ptr,size ctypes.string_at .. function:: WinError(code=None, descr=None) @@ -2099,14 +2117,14 @@ Utility functions alias of :exc:`OSError`. -.. function:: wstring_at(address, size=-1) +.. function:: wstring_at(ptr, size=-1) - This function returns the wide character string starting at memory address - *address* as a string. If *size* is specified, it is used as the number of + Return the wide-character string at *void \*ptr*. + If *size* is specified, it is used as the number of characters of the string, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at + .. audit-event:: ctypes.wstring_at ptr,size ctypes.wstring_at .. _ctypes-data-types: @@ -2280,6 +2298,30 @@ These are the fundamental ctypes data types: optional float initializer. +.. class:: c_double_complex + + Represents the C :c:expr:`double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_float_complex + + Represents the C :c:expr:`float complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_longdouble_complex + + Represents the C :c:expr:`long double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + .. class:: c_int Represents the C :c:expr:`signed int` datatype. The constructor accepts an @@ -2534,6 +2576,37 @@ fields, or any other data types containing pointer type fields. Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _align_ + + An optional small integer that allows overriding the alignment of + the structure when being packed or unpacked to/from memory. + Setting this attribute to 0 is the same as not setting it at all. + + .. attribute:: _layout_ + + An optional string naming the struct/union layout. It can currently + be set to: + + - ``"ms"``: the layout used by the Microsoft compiler (MSVC). + On GCC and Clang, this layout can be selected with + ``__attribute__((ms_struct))``. + - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like†+ data model, as used on Linux and macOS. + With this layout, :attr:`~Structure._pack_` must be unset or zero. + + If not set explicitly, ``ctypes`` will use a default that + matches the platform conventions. This default may change in future + Python releases (for example, when a new platform gains official support, + or when a difference between similar platforms is found). + Currently the default will be: + + - On Windows: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - Otherwise: ``"gcc-sysv"`` + + :attr:`!_layout_` must already be defined when + :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 410b76e77c025b..cb895664ff1b11 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -1,5 +1,5 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== +:mod:`!curses.ascii` --- Utilities for ASCII characters +======================================================= .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index d770c03c8375f4..11fd841d381f69 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -1,5 +1,5 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== +:mod:`!curses.panel` --- A panel stack extension for curses +=========================================================== .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 9b8a98f05f7cbb..91ea6150fb15ba 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1,5 +1,5 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== +:mod:`!curses` --- Terminal handling for character-cell displays +================================================================ .. module:: curses :synopsis: An interface to the curses library, providing portable @@ -21,6 +21,8 @@ for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. +.. include:: ../includes/wasm-ios-notavail.rst + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -922,7 +924,7 @@ the following methods and attributes: .. method:: window.getbegyx() - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. .. method:: window.getbkgd() diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index bbbbcb00d8fef8..fcb5e8bad295a0 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -1,5 +1,5 @@ -:mod:`dataclasses` --- Data Classes -=================================== +:mod:`!dataclasses` --- Data Classes +==================================== .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. @@ -12,7 +12,7 @@ -------------- This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`~object.__init__` and +adding generated :term:`special methods <special method>` such as :meth:`~object.__init__` and :meth:`~object.__repr__` to user-defined classes. It was originally described in :pep:`557`. @@ -31,7 +31,7 @@ using :pep:`526` type annotations. For example, this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -will add, among other things, a :meth:`~object.__init__` that looks like:: +will add, among other things, a :meth:`!__init__` that looks like:: def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name @@ -39,7 +39,7 @@ will add, among other things, a :meth:`~object.__init__` that looks like:: self.quantity_on_hand = quantity_on_hand Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. +directly specified in the :class:`!InventoryItem` definition shown above. .. versionadded:: 3.7 @@ -49,26 +49,26 @@ Module contents .. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. + :term:`special methods <special method>` to classes, as described below. - The :func:`dataclass` decorator examines the class to find + The ``@dataclass`` decorator examines the class to find ``field``\s. A ``field`` is defined as a class variable that has a :term:`type annotation <variable annotation>`. With two - exceptions described below, nothing in :func:`dataclass` + exceptions described below, nothing in ``@dataclass`` examines the type specified in the variable annotation. The order of the fields in all of the generated methods is the order in which they appear in the class definition. - The :func:`dataclass` decorator will add various "dunder" methods to + The ``@dataclass`` decorator will add various "dunder" methods to the class, described below. If any of the added methods already exist in the class, the behavior depends on the parameter, as documented below. The decorator returns the same class that it is called on; no new class is created. - If :func:`dataclass` is used just as a simple decorator with no parameters, + If ``@dataclass`` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are + signature. That is, these three uses of ``@dataclass`` are equivalent:: @dataclass @@ -84,122 +84,125 @@ Module contents class C: ... - The parameters to :func:`dataclass` are: + The parameters to ``@dataclass`` are: - - ``init``: If true (the default), a :meth:`~object.__init__` method will be + - *init*: If true (the default), a :meth:`~object.__init__` method will be generated. - If the class already defines :meth:`~object.__init__`, this parameter is + If the class already defines :meth:`!__init__`, this parameter is ignored. - - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be + - *repr*: If true (the default), a :meth:`~object.__repr__` method will be generated. The generated repr string will have the class name and the name and repr of each field, in the order they are defined in the class. Fields that are marked as being excluded from the repr are not included. For example: ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - If the class already defines :meth:`~object.__repr__`, this parameter is + If the class already defines :meth:`!__repr__`, this parameter is ignored. - - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be + - *eq*: If true (the default), an :meth:`~object.__eq__` method will be generated. This method compares the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the identical type. - If the class already defines :meth:`~object.__eq__`, this parameter is + If the class already defines :meth:`!__eq__`, this parameter is ignored. - - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, + - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be generated. These compare the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a + identical type. If *order* is true and *eq* is false, a :exc:`ValueError` is raised. - If the class already defines any of :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then + If the class already defines any of :meth:`!__lt__`, + :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. + - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method + is generated according to how *eq* and *frozen* are set. - :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are + :meth:`!__hash__` is used by built-in :meth:`hash()`, and when objects are added to hashed collections such as dictionaries and sets. Having a - :meth:`~object.__hash__` implies that instances of the class are immutable. + :meth:`!__hash__` implies that instances of the class are immutable. Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`~object.__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + intent, the existence and behavior of :meth:`!__eq__`, and the values of + the *eq* and *frozen* flags in the ``@dataclass`` decorator. - By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` + By default, ``@dataclass`` will not implicitly add a :meth:`~object.__hash__` method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`~object.__hash__` method. Setting the class + existing explicitly defined :meth:`!__hash__` method. Setting the class attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`~object.__hash__` documentation. + described in the :meth:`!__hash__` documentation. - If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. + If :meth:`!__hash__` is not explicitly defined, or if it is set to ``None``, + then ``@dataclass`` *may* add an implicit :meth:`!__hash__` method. + Although not recommended, you can force ``@dataclass`` to create a + :meth:`!__hash__` method with ``unsafe_hash=True``. This might be the case + if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. - Here are the rules governing implicit creation of a :meth:`~object.__hash__` - method. Note that you cannot both have an explicit :meth:`~object.__hash__` + Here are the rules governing implicit creation of a :meth:`!__hash__` + method. Note that you cannot both have an explicit :meth:`!__hash__` method in your dataclass and set ``unsafe_hash=True``; this will result in a :exc:`TypeError`. - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` + If *eq* and *frozen* are both true, by default ``@dataclass`` will + generate a :meth:`!__hash__` method for you. If *eq* is true and + *frozen* is false, :meth:`!__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If *eq* is false, + :meth:`!__hash__` will be left untouched meaning the :meth:`!__hash__` method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - - ``frozen``: If true (the default is ``False``), assigning to fields will + - *frozen*: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. - - ``match_args``: If true (the default is ``True``), the - ``__match_args__`` tuple will be created from the list of + - *match_args*: If true (the default is ``True``), the + :attr:`~object.__match_args__` tuple will be created from the list of parameters to the generated :meth:`~object.__init__` method (even if - :meth:`~object.__init__` is not generated, see above). If false, or if - ``__match_args__`` is already defined in the class, then - ``__match_args__`` will not be generated. + :meth:`!__init__` is not generated, see above). If false, or if + :attr:`!__match_args__` is already defined in the class, then + :attr:`!__match_args__` will not be generated. .. versionadded:: 3.10 - - ``kw_only``: If true (the default value is ``False``), then all + - *kw_only*: If true (the default value is ``False``), then all fields will be marked as keyword-only. If a field is marked as keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`~object.__init__` is called. There is no + with a keyword when :meth:`!__init__` is called. There is no effect on any other aspect of dataclasses. See the :term:`parameter` glossary entry for details. Also see the :const:`KW_ONLY` section. .. versionadded:: 3.10 - - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute + - *slots*: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. - If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` - is raised. + If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` + is raised. Calling no-arg :func:`super` in dataclasses using ``slots=True`` will result in + the following exception being raised: + ``TypeError: super(type, obj): obj must be an instance or subtype of type``. + The two-arg :func:`super` is a valid workaround. See :gh:`90562` for full details. .. versionadded:: 3.10 .. versionchanged:: 3.11 - If a field name is already included in the ``__slots__`` - of a base class, it will not be included in the generated ``__slots__`` + If a field name is already included in the :attr:`!__slots__` + of a base class, it will not be included in the generated :attr:`!__slots__` to prevent :ref:`overriding them <datamodel-note-slots>`. - Therefore, do not use ``__slots__`` to retrieve the field names of a + Therefore, do not use :attr:`!__slots__` to retrieve the field names of a dataclass. Use :func:`fields` instead. To be able to determine inherited slots, - base class ``__slots__`` may be any iterable, but *not* an iterator. + base class :attr:`!__slots__` may be any iterable, but *not* an iterator. - - ``weakref_slot``: If true (the default is ``False``), add a slot + - *weakref_slot*: If true (the default is ``False``), add a slot named "__weakref__", which is required to make an instance weakref-able. It is an error to specify ``weakref_slot=True`` without also specifying ``slots=True``. @@ -214,7 +217,7 @@ Module contents a: int # 'a' has no default value b: int = 0 # assign a default value for 'b' - In this example, both ``a`` and ``b`` will be included in the added + In this example, both :attr:`!a` and :attr:`!b` will be included in the added :meth:`~object.__init__` method, which will be defined as:: def __init__(self, a: int, b: int = 0): @@ -229,7 +232,7 @@ Module contents required. There are, however, some dataclass features that require additional per-field information. To satisfy this need for additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: + with a call to the provided :func:`!field` function. For example:: @dataclass class C: @@ -243,27 +246,27 @@ Module contents used because ``None`` is a valid value for some parameters with a distinct meaning. No code should directly use the :const:`MISSING` value. - The parameters to :func:`field` are: + The parameters to :func:`!field` are: - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself + - *default*: If provided, this will be the default value for this + field. This is needed because the :func:`!field` call itself replaces the normal position of the default value. - - ``default_factory``: If provided, it must be a zero-argument + - *default_factory*: If provided, it must be a zero-argument callable that will be called when a default value is needed for this field. Among other purposes, this can be used to specify fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. + error to specify both *default* and *default_factory*. - - ``init``: If true (the default), this field is included as a + - *init*: If true (the default), this field is included as a parameter to the generated :meth:`~object.__init__` method. - - ``repr``: If true (the default), this field is included in the + - *repr*: If true (the default), this field is included in the string returned by the generated :meth:`~object.__repr__` method. - - ``hash``: This can be a bool or ``None``. If true, this field is + - *hash*: This can be a bool or ``None``. If true, this field is included in the generated :meth:`~object.__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be + default), use the value of *compare*: this would normally be the expected behavior. A field should be considered in the hash if it's used for comparisons. Setting this value to anything other than ``None`` is discouraged. @@ -274,11 +277,11 @@ Module contents fields that contribute to the type's hash value. Even if a field is excluded from the hash, it will still be used for comparisons. - - ``compare``: If true (the default), this field is included in the + - *compare*: If true (the default), this field is included in the generated equality and comparison methods (:meth:`~object.__eq__`, :meth:`~object.__gt__`, et al.). - - ``metadata``: This can be a mapping or None. None is treated as + - *metadata*: This can be a mapping or ``None``. ``None`` is treated as an empty dict. This value is wrapped in :func:`~types.MappingProxyType` to make it read-only, and exposed on the :class:`Field` object. It is not used at all by Data @@ -286,17 +289,17 @@ Module contents Multiple third-parties can each have their own key, to use as a namespace in the metadata. - - ``kw_only``: If true, this field will be marked as keyword-only. + - *kw_only*: If true, this field will be marked as keyword-only. This is used when the generated :meth:`~object.__init__` method's parameters are computed. .. versionadded:: 3.10 If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is + :func:`!field`, then the class attribute for this field will be + replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class + that after the :func:`@dataclass <dataclass>` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -308,21 +311,21 @@ Module contents z: int = field(repr=False, default=10) t: int = 20 - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. + The class attribute :attr:`!C.z` will be ``10``, the class attribute + :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and + :attr:`!C.y` will not be set. .. class:: Field - :class:`Field` objects describe each defined field. These objects + :class:`!Field` objects describe each defined field. These objects are created internally, and are returned by the :func:`fields` module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: + :class:`!Field` object directly. Its documented attributes are: - - ``name``: The name of the field. - - ``type``: The type of the field. - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, ``metadata``, and ``kw_only`` have the identical + - :attr:`!name`: The name of the field. + - :attr:`!type`: The type of the field. + - :attr:`!default`, :attr:`!default_factory`, :attr:`!init`, :attr:`!repr`, :attr:`!hash`, + :attr:`!compare`, :attr:`!metadata`, and :attr:`!kw_only` have the identical meaning and values as they do in the :func:`field` function. Other attributes may exist, but they are private and must not be @@ -337,13 +340,13 @@ Module contents .. function:: asdict(obj, *, dict_factory=dict) - Converts the dataclass ``obj`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted + Converts the dataclass *obj* to a dict (by using the + factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. - Example of using :func:`asdict` on nested dataclasses:: + Example of using :func:`!asdict` on nested dataclasses:: @dataclass class Point: @@ -362,15 +365,15 @@ Module contents To create a shallow copy, the following workaround may be used:: - dict((field.name, getattr(obj, field.name)) for field in fields(obj)) + {field.name: getattr(obj, field.name) for field in fields(obj)} - :func:`asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!asdict` raises :exc:`TypeError` if *obj* is not a dataclass instance. .. function:: astuple(obj, *, tuple_factory=tuple) - Converts the dataclass ``obj`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted + Converts the dataclass *obj* to a tuple (by using the + factory function *tuple_factory*). Each dataclass is converted to a tuple of its field values. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. @@ -384,28 +387,28 @@ Module contents tuple(getattr(obj, field.name) for field in dataclasses.fields(obj)) - :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. .. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an + Creates a new dataclass with name *cls_name*, fields as defined + in *fields*, base classes as given in *bases*, and initialized + with a namespace as given in *namespace*. *fields* is an iterable whose elements are each either ``name``, ``(name, type)``, or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, - ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have - the same meaning as they do in :func:`dataclass`. + :data:`typing.Any` is used for ``type``. The values of *init*, + *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, + *match_args*, *kw_only*, *slots*, and *weakref_slot* have + the same meaning as they do in :func:`@dataclass <dataclass>`. - If ``module`` is defined, the ``__module__`` attribute + If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. By default, it is set to the module name of the caller. This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to + mechanism for creating a new class with :attr:`!__annotations__` can + then apply the :func:`@dataclass <dataclass>` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -428,38 +431,38 @@ Module contents .. function:: replace(obj, /, **changes) - Creates a new object of the same type as ``obj``, replacing - fields with values from ``changes``. If ``obj`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. + Creates a new object of the same type as *obj*, replacing + fields with values from *changes*. If *obj* is not a Data + Class, raises :exc:`TypeError`. If keys in *changes* are not + field names of the given dataclass, raises :exc:`TypeError`. The newly returned object is created by calling the :meth:`~object.__init__` method of the dataclass. This ensures that :meth:`__post_init__`, if present, is also called. Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :meth:`__post_init__`. + specified on the call to :func:`!replace` so that they can be passed to + :meth:`!__init__` and :meth:`__post_init__`. - It is an error for ``changes`` to contain any fields that are + It is an error for *changes* to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised in this case. Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but + :func:`!replace`. They are not copied from the source object, but rather are initialized in :meth:`__post_init__`, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance + :func:`!replace` (or similarly named) method which handles instance copying. Dataclass instances are also supported by generic function :func:`copy.replace`. .. function:: is_dataclass(obj) - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. + Return ``True`` if its parameter is a dataclass (including subclasses of a + dataclass) or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -475,11 +478,11 @@ Module contents .. data:: KW_ONLY A sentinel value used as a type annotation. Any fields after a - pseudo-field with the type of :const:`KW_ONLY` are marked as + pseudo-field with the type of :const:`!KW_ONLY` are marked as keyword-only fields. Note that a pseudo-field of type - :const:`KW_ONLY` is otherwise completely ignored. This includes the + :const:`!KW_ONLY` is otherwise completely ignored. This includes the name of such a field. By convention, a name of ``_`` is used for a - :const:`KW_ONLY` field. Keyword-only fields signify + :const:`!KW_ONLY` field. Keyword-only fields signify :meth:`~object.__init__` parameters that must be specified as keywords when the class is instantiated. @@ -495,7 +498,7 @@ Module contents p = Point(0, y=1.5, z=2.0) In a single dataclass, it is an error to specify more than one - field whose type is :const:`KW_ONLY`. + field whose type is :const:`!KW_ONLY`. .. versionadded:: 3.10 @@ -513,11 +516,11 @@ Post-init processing .. function:: __post_init__() When defined on the class, it will be called by the generated - :meth:`~object.__init__`, normally as ``self.__post_init__()``. + :meth:`~object.__init__`, normally as :meth:`!self.__post_init__`. However, if any ``InitVar`` fields are defined, they will also be - passed to :meth:`__post_init__` in the order they were defined in the - class. If no :meth:`~object.__init__` method is generated, then - :meth:`__post_init__` will not automatically be called. + passed to :meth:`!__post_init__` in the order they were defined in the + class. If no :meth:`!__init__` method is generated, then + :meth:`!__post_init__` will not automatically be called. Among other uses, this allows for initializing field values that depend on one or more other fields. For example:: @@ -531,15 +534,15 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base -class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method +The :meth:`~object.__init__` method generated by :func:`@dataclass <dataclass>` does not call base +class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: - @dataclass class Rectangle: - height: float - width: float + def __init__(self, height, width): + self.height = height + self.width = width @dataclass class Square(Rectangle): @@ -548,29 +551,33 @@ that has to be called, it is common to call this method in a def __post_init__(self): super().__init__(self.side, self.side) -Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods +Note, however, that in general the dataclass-generated :meth:`!__init__` methods don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how +parameters to :meth:`!__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. +.. _dataclasses-class-variables: + Class variables --------------- -One of the few places where :func:`dataclass` actually inspects the type +One of the few places where :func:`@dataclass <dataclass>` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +:data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded from consideration as a field and is ignored by the dataclass mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the module-level :func:`fields` function. +.. _dataclasses-init-only-variables: + Init-only variables ------------------- -Another place where :func:`dataclass` inspects a type annotation is to +Another place where :func:`@dataclass <dataclass>` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type ``dataclasses.InitVar``. If a field is an ``InitVar``, it is considered a pseudo-field called an init-only @@ -595,14 +602,16 @@ value is not provided when creating the class:: c = C(10, database=my_database) -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. +In this case, :func:`fields` will return :class:`Field` objects for :attr:`!i` and +:attr:`!j`, but not for :attr:`!database`. + +.. _dataclasses-frozen: Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +by passing ``frozen=True`` to the :func:`@dataclass <dataclass>` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -611,10 +620,14 @@ There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and must use :meth:`!object.__setattr__`. +.. Make sure to not remove "object" from "object.__setattr__" in the above markup! + +.. _dataclasses-inheritance: + Inheritance ----------- -When the dataclass is being created by the :meth:`dataclass` decorator, +When the dataclass is being created by the :func:`@dataclass <dataclass>` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -634,15 +647,15 @@ example:: z: int = 10 x: int = 15 -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. +The final list of fields is, in order, :attr:`!x`, :attr:`!y`, :attr:`!z`. The final +type of :attr:`!x` is :class:`int`, as specified in class :class:`!C`. -The generated :meth:`~object.__init__` method for ``C`` will look like:: +The generated :meth:`~object.__init__` method for :class:`!C` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): -Re-ordering of keyword-only parameters in :meth:`~object.__init__` ------------------------------------------------------------------- +Re-ordering of keyword-only parameters in :meth:`!__init__` +----------------------------------------------------------- After the parameters needed for :meth:`~object.__init__` are computed, any keyword-only parameters are moved to come after all regular @@ -650,8 +663,8 @@ keyword-only parameters are moved to come after all regular keyword-only parameters are implemented in Python: they must come after non-keyword-only parameters. -In this example, ``Base.y``, ``Base.w``, and ``D.t`` are keyword-only -fields, and ``Base.x`` and ``D.z`` are regular fields:: +In this example, :attr:`!Base.y`, :attr:`!Base.w`, and :attr:`!D.t` are keyword-only +fields, and :attr:`!Base.x` and :attr:`!D.z` are regular fields:: @dataclass class Base: @@ -665,7 +678,7 @@ fields, and ``Base.x`` and ``D.z`` are regular fields:: z: int = 10 t: int = field(kw_only=True, default=0) -The generated :meth:`~object.__init__` method for ``D`` will look like:: +The generated :meth:`!__init__` method for :class:`!D` will look like:: def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): @@ -674,22 +687,22 @@ the list of fields: parameters derived from regular fields are followed by parameters derived from keyword-only fields. The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`~object.__init__` parameter list. +re-ordered :meth:`!__init__` parameter list. Default factory functions ------------------------- -If a :func:`field` specifies a ``default_factory``, it is called with +If a :func:`field` specifies a *default_factory*, it is called with zero arguments when a default value for the field is needed. For example, to create a new instance of a list, use:: mylist: list = field(default_factory=list) If a field is excluded from :meth:`~object.__init__` (using ``init=False``) -and the field also specifies ``default_factory``, then the default +and the field also specifies *default_factory*, then the default factory function will always be called from the generated -:meth:`~object.__init__` function. This happens because there is no other +:meth:`!__init__` function. This happens because there is no other way to give the field an initial value. Mutable default values @@ -710,8 +723,8 @@ Consider this example, not using dataclasses:: assert o1.x == [1, 2] assert o1.x is o2.x -Note that the two instances of class ``C`` share the same class -variable ``x``, as expected. +Note that the two instances of class :class:`!C` share the same class +variable :attr:`!x`, as expected. Using dataclasses, *if* this code was valid:: @@ -719,7 +732,7 @@ Using dataclasses, *if* this code was valid:: class D: x: list = [] # This code raises ValueError def add(self, element): - self.x += element + self.x.append(element) it would generate code similar to:: @@ -728,17 +741,17 @@ it would generate code similar to:: def __init__(self, x=x): self.x = x def add(self, element): - self.x += element + self.x.append(element) assert D().x is D().x -This has the same issue as the original example using class ``C``. -That is, two instances of class ``D`` that do not specify a value -for ``x`` when creating a class instance will share the same copy -of ``x``. Because dataclasses just use normal Python class +This has the same issue as the original example using class :class:`!C`. +That is, two instances of class :class:`!D` that do not specify a value +for :attr:`!x` when creating a class instance will share the same copy +of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`ValueError` if it +:func:`@dataclass <dataclass>` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -753,8 +766,8 @@ mutable types as default values for fields:: assert D().x is not D().x .. versionchanged:: 3.11 - Instead of looking for and disallowing objects of type ``list``, - ``dict``, or ``set``, unhashable objects are now not allowed as + Instead of looking for and disallowing objects of type :class:`list`, + :class:`dict`, or :class:`set`, unhashable objects are now not allowed as default values. Unhashability is used to approximate mutability. @@ -764,15 +777,17 @@ Descriptor-typed fields Fields that are assigned :ref:`descriptor objects <descriptors>` as their default value have the following special behaviors: -* The value for the field passed to the dataclass's ``__init__`` method is - passed to the descriptor's ``__set__`` method rather than overwriting the +* The value for the field passed to the dataclass's :meth:`~object.__init__` method is + passed to the descriptor's :meth:`~object.__set__` method rather than overwriting the descriptor object. + * Similarly, when getting or setting the field, the descriptor's - ``__get__`` or ``__set__`` method is called rather than returning or + :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, ``dataclasses`` - will call the descriptor's ``__get__`` method using its class access - form (i.e. ``descriptor.__get__(obj=None, type=cls)``. If the + +* To determine whether a field contains a default value, :func:`@dataclass <dataclass>` + will call the descriptor's :meth:`!__get__` method using its class access + form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the field's default. On the other hand, if the descriptor raises :exc:`AttributeError` in this situation, no default value will be diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 0b9d42f32e3bd6..b6d8e6e6df07fa 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1,5 +1,5 @@ -:mod:`datetime` --- Basic date and time types -============================================= +:mod:`!datetime` --- Basic date and time types +============================================== .. module:: datetime :synopsis: Basic date and time types. @@ -14,7 +14,7 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times. +The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. @@ -37,8 +37,9 @@ on efficient attribute extraction for output formatting and manipulation. Package `dateutil <https://dateutil.readthedocs.io/en/stable/>`_ Third-party library with expanded time zone and parsing support. - Package `DateType <https://pypi.org/project/datetype/>`_ - Third-party library that introduces distinct static types to e.g. allow static type checkers + Package :pypi:`DateType` + Third-party library that introduces distinct static types to e.g. allow + :term:`static type checkers <static type checker>` to differentiate between naive and aware datetimes. .. _datetime-naive-aware: @@ -69,7 +70,7 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can +supplied by the :mod:`!datetime` module. The :class:`timezone` class can represent simple timezones with fixed offsets from UTC, such as UTC itself or North American EST and EDT timezones. Supporting timezones at deeper levels of detail is up to the application. The rules for time adjustment across the @@ -79,18 +80,18 @@ standard suitable for every application aside from UTC. Constants --------- -The :mod:`datetime` module exports the following constants: +The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR The smallest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MINYEAR` is ``1``. + :const:`MINYEAR` is 1. .. data:: MAXYEAR The largest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MAXYEAR` is ``9999``. + :const:`MAXYEAR` is 9999. .. attribute:: UTC @@ -129,8 +130,8 @@ Available Types .. class:: timedelta :noindex: - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. + A duration expressing the difference between two :class:`.datetime` + or :class:`date` instances to microsecond resolution. .. class:: tzinfo @@ -202,11 +203,11 @@ objects. -------------------------- A :class:`timedelta` object represents a duration, the difference between two -dates or times. +:class:`.datetime` or :class:`date` instances. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - All arguments are optional and default to ``0``. Arguments may be integers + All arguments are optional and default to 0. Arguments may be integers or floats, and may be positive or negative. Only *days*, *seconds* and *microseconds* are stored internally. @@ -279,7 +280,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. -Note that, because of normalization, ``timedelta.max`` > ``-timedelta.min``. +Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. Instance attributes (read-only): @@ -301,26 +302,27 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ -| ``t1 = t2 + t3`` | Sum of *t2* and *t3*. Afterwards *t1*-*t2* == | -| | *t3* and *t1*-*t3* == *t2* are true. (1) | +| ``t1 = t2 + t3`` | Sum of ``t2`` and ``t3``. | +| | Afterwards ``t1 - t2 == t3`` and | +| | ``t1 - t3 == t2`` are true. (1) | +--------------------------------+-----------------------------------------------+ -| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | -| | == *t2* - *t3* and *t2* == *t1* + *t3* are | +| ``t1 = t2 - t3`` | Difference of ``t2`` and ``t3``. Afterwards | +| | ``t1 == t2 - t3`` and ``t2 == t1 + t3`` are | | | true. (1)(6) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | -| | Afterwards *t1* // i == *t2* is true, | +| | Afterwards ``t1 // i == t2`` is true, | | | provided ``i != 0``. | +--------------------------------+-----------------------------------------------+ -| | In general, *t1* \* i == *t1* \* (i-1) + *t1* | +| | In general, ``t1 * i == t1 * (i-1) + t1`` | | | is true. (1) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * f or t1 = f * t2`` | Delta multiplied by a float. The result is | | | rounded to the nearest multiple of | | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of overall duration *t2* by | -| | interval unit *t3*. Returns a :class:`float` | +| ``f = t2 / t3`` | Division (3) of overall duration ``t2`` by | +| | interval unit ``t3``. Returns a :class:`float`| | | object. | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| @@ -342,13 +344,12 @@ Supported operations: | ``+t1`` | Returns a :class:`timedelta` object with the | | | same value. (2) | +--------------------------------+-----------------------------------------------+ -| ``-t1`` | equivalent to | -| | :class:`timedelta`\ (-*t1.days*, | -| | -*t1.seconds*, -*t1.microseconds*), | -| | and to *t1*\* -1. (1)(4) | +| ``-t1`` | Equivalent to ``timedelta(-t1.days, | +| | -t1.seconds*, -t1.microseconds)``, | +| | and to ``t1 * -1``. (1)(4) | +--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | -| | and to -*t* when ``t.days < 0``. (2) | +| ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, | +| | and to ``-t`` when ``t.days < 0``. (2) | +--------------------------------+-----------------------------------------------+ | ``str(t)`` | Returns a string in the form | | | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | @@ -369,10 +370,10 @@ Notes: This is exact and cannot overflow. (3) - Division by 0 raises :exc:`ZeroDivisionError`. + Division by zero raises :exc:`ZeroDivisionError`. (4) - -*timedelta.max* is not representable as a :class:`timedelta` object. + ``-timedelta.max`` is not representable as a :class:`timedelta` object. (5) String representations of :class:`timedelta` objects are normalized @@ -399,30 +400,7 @@ objects (see below). the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. - -Comparisons of :class:`timedelta` objects are supported, with some caveats. - -The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter -the type of the compared object:: - - >>> from datetime import timedelta - >>> delta1 = timedelta(seconds=57) - >>> delta2 = timedelta(hours=25, seconds=2) - >>> delta2 != delta1 - True - >>> delta2 == 5 - False - -For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` -object is compared to an object of a different type, :exc:`TypeError` -is raised:: - - >>> delta2 > delta1 - True - >>> delta2 > 5 - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' +:class:`timedelta` objects support equality and order comparisons. In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. @@ -535,7 +513,15 @@ Other constructors, all class methods: .. classmethod:: date.fromisoformat(date_string) Return a :class:`date` corresponding to a *date_string* given in any valid - ISO 8601 format, except ordinal dates (e.g. ``YYYY-DDD``):: + ISO 8601 format, with the following exceptions: + + 1. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 2. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 3. Ordinal dates are not currently supported (``YYYY-OOO``). + + Examples:: >>> from datetime import date >>> date.fromisoformat('2019-12-04') @@ -597,16 +583,21 @@ Supported operations: +-------------------------------+----------------------------------------------+ | Operation | Result | +===============================+==============================================+ -| ``date2 = date1 + timedelta`` | *date2* will be ``timedelta.days`` days | -| | after *date1*. (1) | +| ``date2 = date1 + timedelta`` | ``date2`` will be ``timedelta.days`` days | +| | after ``date1``. (1) | +-------------------------------+----------------------------------------------+ -| ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | +| ``date2 = date1 - timedelta`` | Computes ``date2`` such that ``date2 + | | | timedelta == date1``. (2) | +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | +-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | +| | ``date1 == date2`` | Equality comparison. (4) | +| | ``date1 != date2`` | | ++-------------------------------+----------------------------------------------+ +| | ``date1 < date2`` | Order comparison. (5) | +| | ``date1 > date2`` | | +| | ``date1 <= date2`` | | +| | ``date1 >= date2`` | | +-------------------------------+----------------------------------------------+ Notes: @@ -622,19 +613,32 @@ Notes: ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) - This is exact, and cannot overflow. timedelta.seconds and - timedelta.microseconds are 0, and date2 + timedelta == date1 after. + This is exact, and cannot overflow. ``timedelta.seconds`` and + ``timedelta.microseconds`` are 0, and ``date2 + timedelta == date1`` after. (4) + :class:`date` objects are equal if they represent the same date. + + :class:`!date` objects that are not also :class:`.datetime` instances + are never equal to :class:`!datetime` objects, even if they represent + the same date. + +(5) + *date1* is considered less than *date2* when *date1* precedes *date2* in time. In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. + date2.toordinal()``. + + Order comparison between a :class:`!date` object that is not also a + :class:`.datetime` instance and a :class:`!datetime` object raises + :exc:`TypeError`. + +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer converts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. In Boolean contexts, all :class:`date` objects are considered to be true. @@ -667,7 +671,7 @@ Instance methods: time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January 1st. + is the day number within the current year starting with 1 for January 1st. .. method:: date.toordinal() @@ -858,8 +862,8 @@ Constructor: If an argument outside those ranges is given, :exc:`ValueError` is raised. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. Other constructors, all class methods: @@ -987,8 +991,8 @@ Other constructors, all class methods: .. classmethod:: datetime.fromordinal(ordinal) Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, - where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and + where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless + ``1 <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. @@ -1016,8 +1020,12 @@ Other constructors, all class methods: 1. Time zone offsets may have fractional seconds. 2. The ``T`` separator may be replaced by any single unicode character. - 3. Ordinal dates are not currently supported. - 4. Fractional hours and minutes are not supported. + 3. Fractional hours and minutes are not supported. + 4. Reduced precision dates are not currently supported (``YYYY-MM``, + ``YYYY``). + 5. Extended date representations are not currently supported + (``±YYYYYY-MM-DD``). + 6. Ordinal dates are not currently supported (``YYYY-OOO``). Examples:: @@ -1071,6 +1079,24 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. + .. versionchanged:: 3.13 + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import datetime + >>> date_string = "02/29" + >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' Class attributes: @@ -1141,8 +1167,8 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 @@ -1157,21 +1183,26 @@ Supported operations: +---------------------------------------+--------------------------------+ | ``timedelta = datetime1 - datetime2`` | \(3) | +---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | +| | ``datetime1 == datetime2`` | Equality comparison. (4) | +| | ``datetime1 != datetime2`` | | ++---------------------------------------+--------------------------------+ +| | ``datetime1 < datetime2`` | Order comparison. (5) | +| | ``datetime1 > datetime2`` | | +| | ``datetime1 <= datetime2`` | | +| | ``datetime1 >= datetime2`` | | +---------------------------------------+--------------------------------+ (1) - datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The + ``datetime2`` is a duration of ``timedelta`` removed from ``datetime1``, moving forward in + time if ``timedelta.days > 0``, or backward if ``timedelta.days < 0``. The result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and - datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if - datetime2.year would be smaller than :const:`MINYEAR` or larger than + ``datetime2 - datetime1 == timedelta`` after. :exc:`OverflowError` is raised if + ``datetime2.year`` would be smaller than :const:`MINYEAR` or larger than :const:`MAXYEAR`. Note that no time zone adjustments are done even if the input is an aware object. (2) - Computes the datetime2 such that datetime2 + timedelta == datetime1. As for + Computes the ``datetime2`` such that ``datetime2 + timedelta == datetime1``. As for addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and no time zone adjustments are done even if the input is aware. @@ -1186,39 +1217,50 @@ Supported operations: are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. (4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. + :class:`.datetime` objects are equal if they represent the same date + and time, taking into account the time zone. - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. + Naive and aware :class:`!datetime` objects are never equal. - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. + :class:`!datetime` instances in a repeated interval are never equal to + :class:`!datetime` instances in other time zone. - .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`.datetime` - instances don't raise :exc:`TypeError`. +(5) + *datetime1* is considered less than *datetime2* when *datetime1* precedes + *datetime2* in time, taking into account the time zone. - .. note:: + Order comparison between naive and aware :class:`.datetime` objects + raises :exc:`TypeError`. + + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. + If both comparands are aware and have different :attr:`~.datetime.tzinfo` + attributes, the comparison acts as comparands were first converted to UTC + datetimes except that the implementation never overflows. - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`.datetime` + instances don't raise :exc:`TypeError`. + +.. versionchanged:: 3.13 + Comparison between :class:`.datetime` object and an instance of + the :class:`date` subclass that is not a :class:`!datetime` subclass + no longer converts the latter to :class:`!date`, ignoring the time part + and the time zone. + The default behavior can be changed by overriding the special comparison + methods in subclasses. Instance methods: @@ -1254,11 +1296,11 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. - :class:`datetime` objects are also supported by generic function + :class:`.datetime` objects are also supported by generic function :func:`copy.replace`. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. .. method:: datetime.astimezone(tz=None) @@ -1345,24 +1387,24 @@ Instance methods: d.weekday(), yday, dst)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January - 1st. The :attr:`tm_isdst` flag of the result is set according to the + is the day number within the current year starting with 1 for January + 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns - ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is - set to ``0``. + ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`!tm_isdst` is set to 1; else :attr:`!tm_isdst` is + set to 0. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what + ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note - that an :exc:`OverflowError` may be raised if *d*.year was + normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note + that an :exc:`OverflowError` may be raised if ``d.year`` was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1501,8 +1543,8 @@ Instance methods: >>> dt.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' - .. versionadded:: 3.6 - Added the *timespec* argument. + .. versionchanged:: 3.6 + Added the *timespec* parameter. .. method:: datetime.__str__() @@ -1549,7 +1591,7 @@ Instance methods: Examples of Usage: :class:`.datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with :class:`~datetime.datetime` objects: +Examples of working with :class:`.datetime` objects: .. doctest:: @@ -1677,7 +1719,7 @@ Usage of ``KabulTz`` from above:: :class:`.time` Objects ---------------------- -A :class:`time` object represents a (local) time of day, independent of any particular +A :class:`.time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) @@ -1693,7 +1735,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to ``0`` except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes: @@ -1748,29 +1790,26 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 -:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, -where *a* is considered less -than *b* when *a* precedes *b* in time. If one comparand is naive and the other -is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality -comparisons, naive instances are never equal to aware instances. +:class:`.time` objects support equality and order comparisons, +where *a* is considered less than *b* when *a* precedes *b* in time. -If both comparands are aware, and have -the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +Naive and aware :class:`!time` objects are never equal. +Order comparison between naive and aware :class:`!time` objects raises +:exc:`TypeError`. + +If both comparands are aware, and have the same :attr:`~.time.tzinfo` +attribute, the :attr:`!tzinfo` and :attr:`!fold` attributes are ignored and the base times are compared. If both comparands are aware and -have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by -subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order -to stop mixed-type comparisons from falling back to the default comparison by -object address, when a :class:`.time` object is compared to an object of a -different type, :exc:`TypeError` is raised unless the comparison is ``==`` or -``!=``. The latter cases return :const:`False` or :const:`True`, respectively. +have different :attr:`!tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`~datetime.time` instances + Equality comparisons between aware and naive :class:`.time` instances don't raise :exc:`TypeError`. In Boolean contexts, a :class:`.time` object is always considered to be true. @@ -1796,7 +1835,9 @@ Other constructor: be truncated). 4. Fractional hours and minutes are not supported. - Examples:: + Examples: + + .. doctest:: >>> from datetime import time >>> time.fromisoformat('04:23:01') @@ -1807,7 +1848,7 @@ Other constructor: datetime.time(4, 23, 1) >>> time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000') + >>> time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) >>> time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1833,11 +1874,11 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. - :class:`time` objects are also supported by generic function + :class:`.time` objects are also supported by generic function :func:`copy.replace`. - .. versionadded:: 3.6 - Added the ``fold`` argument. + .. versionchanged:: 3.6 + Added the *fold* parameter. .. method:: time.isoformat(timespec='auto') @@ -1880,8 +1921,8 @@ Instance methods: >>> dt.isoformat(timespec='auto') '12:34:56' - .. versionadded:: 3.6 - Added the *timespec* argument. + .. versionchanged:: 3.6 + Added the *timespec* parameter. .. method:: time.__str__() @@ -1978,19 +2019,20 @@ Examples of working with a :class:`.time` object:: You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can represent timezones with fixed offset from UTC such as UTC itself or North American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, otherwise it can be + :meth:`~object.__init__` method that can be called with no arguments, + otherwise it can be pickled but possibly not unpickled again. This is a technical requirement that may be relaxed in the future. A concrete subclass of :class:`tzinfo` may need to implement the following methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + :mod:`!datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) @@ -2031,7 +2073,7 @@ Examples of working with a :class:`.time` object:: already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag + attribute's :meth:`dst` method to determine how the :attr:`~time.struct_time.tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -2041,13 +2083,13 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz`` For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. + :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: @@ -2076,9 +2118,9 @@ Examples of working with a :class:`.time` object:: .. method:: tzinfo.tzname(dt) Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, + a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, - "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all + ``"GMT"``, ``"UTC"``, ``"-500"``, ``"-5:00"``, ``"EDT"``, ``"US/Eastern"``, ``"America/New York"`` are all valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value @@ -2111,7 +2153,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) - This is called from the default :class:`datetime.astimezone()` + This is called from the default :meth:`datetime.astimezone` implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an @@ -2124,7 +2166,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and + for political reasons. The default implementations of :meth:`~.datetime.astimezone` and :meth:`fromutc` may not produce the result you want if the result is one of the hours straddling the moment the standard offset changes. @@ -2190,10 +2232,10 @@ hour that can't be spelled unambiguously in local wall time: the last hour of daylight time. In Eastern, that's times of the form 5:MM UTC on the day daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC +:meth:`~.datetime.astimezone` mimics the local clock's behavior by mapping two adjacent UTC hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. +have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) @@ -2208,10 +2250,10 @@ For example, at the Fall back transition of 2016, we get:: 07:00:00 UTC = 02:00:00 EST 0 Note that the :class:`.datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. +:attr:`~.datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid +value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, or any other fixed-offset :class:`tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). @@ -2219,7 +2261,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: :mod:`zoneinfo` - The :mod:`datetime` module has a basic :class:`timezone` class (for + The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). @@ -2237,7 +2279,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: :class:`timezone` Objects --------------------------- +------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from @@ -2312,8 +2354,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior +-------------------------------------------------------------------- :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2323,8 +2365,8 @@ Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a corresponding format string. -The table below provides a high-level comparison of :meth:`strftime` -versus :meth:`strptime`: +The table below provides a high-level comparison of :meth:`~.datetime.strftime` +versus :meth:`~.datetime.strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ | | ``strftime`` | ``strptime`` | @@ -2341,8 +2383,8 @@ versus :meth:`strptime`: .. _format-codes: -:meth:`strftime` and :meth:`strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: @@ -2481,13 +2523,13 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`strftime` +These may not be available on all platforms when used with the :meth:`~.datetime.strftime` method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform +calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your platform, consult the :manpage:`strftime(3)` documentation. There are also differences between platforms in handling of unsupported format specifiers. @@ -2503,9 +2545,9 @@ Technical Detail Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a -:meth:`timetuple` method. +:meth:`~date.timetuple` method. -For the :meth:`datetime.strptime` class method, the default value is +For the :meth:`.datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: any components not specified in the format string will be pulled from the default value. [#]_ @@ -2518,12 +2560,12 @@ information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not -be used, as :class:`time` objects have no such values. If they're used anyway, -``1900`` is substituted for the year, and ``1`` for the month and day. +be used, as :class:`!time` objects have no such values. If they're used anyway, +1900 is substituted for the year, and 1 for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. +values. If they're used anyway, 0 is substituted for them. For the same reason, handling of format strings containing Unicode code points that can't be represented in the charset of the current locale is also @@ -2540,27 +2582,27 @@ Notes: contain non-ASCII characters. (2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but + The :meth:`~.datetime.strptime` method can parse years in the full [1, 9999] range, but years < 1000 must be zero-filled to 4-digit width. .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to + In previous versions, :meth:`~.datetime.strftime` method was restricted to years >= 1900. .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to + In version 3.2, :meth:`~.datetime.strftime` method was restricted to years >= 1000. (3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects + When used with the :meth:`~.datetime.strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. (4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support + Unlike the :mod:`time` module, the :mod:`!datetime` module does not support leap seconds. (5) - When used with the :meth:`strptime` method, the ``%f`` directive + When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always @@ -2573,7 +2615,7 @@ Notes: For an aware object: ``%z`` - :meth:`utcoffset` is transformed into a string of the form + :meth:`~.datetime.utcoffset` is transformed into a string of the form ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset @@ -2581,14 +2623,14 @@ Notes: offset microseconds. The ``ffffff`` part is omitted when the offset is a whole number of seconds and both the ``ffffff`` and the ``SS`` part is omitted when the offset is a whole number of minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is + :meth:`~.datetime.utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. .. versionchanged:: 3.7 - When the ``%z`` directive is provided to the :meth:`strptime` method, + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. For example, ``'+01:00:00'`` will be parsed as an offset of one hour. @@ -2599,11 +2641,11 @@ Notes: hours, minutes and seconds. ``%Z`` - In :meth:`strftime`, ``%Z`` is replaced by an empty string if - :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the + In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if + :meth:`~.datetime.tzname` returns ``None``; otherwise ``%Z`` is replaced by the returned value, which must be a string. - :meth:`strptime` only accepts certain values for ``%Z``: + :meth:`~.datetime.strptime` only accepts certain values for ``%Z``: 1. any value in ``time.tzname`` for your machine's locale 2. the hard-coded values ``UTC`` and ``GMT`` @@ -2613,26 +2655,45 @@ Notes: invalid values. .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an + When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, an aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + When used with the :meth:`~.datetime.strptime` method, ``%U`` and ``%W`` are only used in calculations when the day of the week and the calendar year (``%Y``) are specified. (8) Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + :meth:`~.datetime.strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. (9) - When used with the :meth:`strptime` method, the leading zero is optional + When used with the :meth:`~.datetime.strptime` method, the leading zero is optional for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. +(10) + When parsing a month and day using :meth:`~.datetime.strptime`, always + include a year in the format. If the value you need to parse lacks a year, + append an explicit dummy leap year. Otherwise your code will raise an + exception when it encounters leap day because the default year used by the + parser is not a leap year. Users run into this bug every four years... + + .. doctest:: + + >>> month_day = "02/29" + >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + datetime.datetime(1984, 2, 29, 0, 0) + + .. deprecated-removed:: 3.13 3.15 + :meth:`~.datetime.strptime` calls using a format string containing + a day of month without a year now emit a + :exc:`DeprecationWarning`. In 3.15 or later we may change this into + an error or change the default year to a leap year. See :gh:`70647`. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity @@ -2647,4 +2708,4 @@ Notes: <https://web.archive.org/web/20220531051136/https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm>`_ for a good explanation. -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 766847b971b645..77148a558d1909 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -1,5 +1,5 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= +:mod:`!dbm` --- Interfaces to Unix "databases" +============================================== .. module:: dbm :synopsis: Interfaces to various Unix "database" formats. @@ -8,12 +8,18 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the +:mod:`dbm` is a generic interface to variants of the DBM database: + +* :mod:`dbm.sqlite3` +* :mod:`dbm.gnu` +* :mod:`dbm.ndbm` + +If none of these modules are installed, the slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface <https://www.jcea.es/programacion/pybsddb.htm>`_ to the Oracle Berkeley DB. +.. include:: ../includes/wasm-ios-notavail.rst .. exception:: error @@ -25,73 +31,84 @@ the Oracle Berkeley DB. .. function:: whichdb(filename) This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. + available --- :mod:`dbm.sqlite3`, :mod:`dbm.gnu`, :mod:`dbm.ndbm`, + or :mod:`dbm.dumb` --- should be used to open a given file. - Returns one of the following values: ``None`` if the file can't be opened - because it's unreadable or doesn't exist; the empty string (``''``) if the - file's format can't be guessed; or a string containing the required module - name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. + Return one of the following values: -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + * ``None`` if the file can't be opened because it's unreadable or doesn't exist + * the empty string (``''``) if the file's format can't be guessed + * a string containing the required module name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'`` -.. function:: open(file, flag='r', mode=0o666) + .. versionchanged:: 3.11 + *filename* accepts a :term:`path-like object`. - Open the database file *file* and return a corresponding object. +.. Substitutions for the open() flag param docs; + all submodules use the same text. - If the database file already exists, the :func:`whichdb` function is used to - determine its type and the appropriate module is used; if it does not exist, - the first module listed above that can be imported is used. +.. |flag_r| replace:: + Open existing database for reading only. - The optional *flag* argument can be: +.. |flag_w| replace:: + Open existing database for reading and writing. - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ +.. |flag_c| replace:: + Open database for reading and writing, creating it if it doesn't exist. - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). +.. |flag_n| replace:: + Always create a new, empty database, open for reading and writing. +.. |mode_param_doc| replace:: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. -The object returned by :func:`.open` supports the same basic functionality as -dictionaries; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`keys` method are -available, as well as :meth:`get` and :meth:`setdefault`. +.. function:: open(file, flag='r', mode=0o666) -.. versionchanged:: 3.2 - :meth:`get` and :meth:`setdefault` are now available in all database modules. + Open a database and return the corresponding database object. -.. versionchanged:: 3.8 - Deleting a key from a read-only database raises database module specific error - instead of :exc:`KeyError`. + :param file: + The database file to open. + + If the database file already exists, the :func:`whichdb` function is used to + determine its type and the appropriate module is used; if it does not exist, + the first submodule listed above that can be imported is used. + :type file: :term:`path-like object` -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for file. + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| -Key and values are always stored as bytes. This means that when + :param int mode: + |mode_param_doc| + + .. versionchanged:: 3.11 + *file* accepts a :term:`path-like object`. + +The object returned by :func:`~dbm.open` supports the same basic functionality as a +:class:`dict`; keys and their corresponding values can be stored, retrieved, and +deleted, and the :keyword:`in` operator and the :meth:`!keys` method are +available, as well as :meth:`!get` and :meth:`!setdefault` methods. + +Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before being stored. These objects also support being used in a :keyword:`with` statement, which will automatically close them when done. +.. versionchanged:: 3.2 + :meth:`!get` and :meth:`!setdefault` methods are now available for all + :mod:`dbm` backends. + .. versionchanged:: 3.4 Added native support for the context management protocol to the objects - returned by :func:`.open`. + returned by :func:`~dbm.open`. + +.. versionchanged:: 3.8 + Deleting a key from a read-only database raises a database module specific exception + instead of :exc:`KeyError`. The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -129,28 +146,66 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. +:mod:`dbm.sqlite3` --- SQLite backend for dbm +--------------------------------------------- + +.. module:: dbm.sqlite3 + :platform: All + :synopsis: SQLite backend for dbm + +.. versionadded:: 3.13 + +**Source code:** :source:`Lib/dbm/sqlite3.py` + +-------------- + +This module uses the standard library :mod:`sqlite3` module to provide an +SQLite backend for the :mod:`dbm` module. +The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +or any other SQLite browser, including the SQLite CLI. + +.. function:: open(filename, /, flag="r", mode=0o666) + + Open an SQLite database. + The returned object behaves like a :term:`mapping`, + implements a :meth:`!close` method, + and supports a "closing" context manager via the :keyword:`with` keyword. + + :param filename: + The path to the database to be opened. + :type filename: :term:`path-like object` -:mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------- + :param str flag: + + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + :param mode: + The Unix file access mode of the file (default: octal ``0o666``), + used only when the database has to be created. + + +:mod:`dbm.gnu` --- GNU database manager +--------------------------------------- .. module:: dbm.gnu :platform: Unix - :synopsis: GNU's reinterpretation of dbm. + :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -This module is quite similar to the :mod:`dbm` module, but uses the GNU library -``gdbm`` instead to provide some additional functionality. Please note that the -file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible. +The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +library, similar to the :mod:`dbm.ndbm` module, but with additional +functionality like crash tolerance. -The :mod:`dbm.gnu` module provides an interface to the GNU DBM library. -``dbm.gnu.gdbm`` objects behave like mappings (dictionaries), except that keys and -values are always converted to bytes before storing. Printing a ``gdbm`` -object doesn't print the -keys and values, and the :meth:`items` and :meth:`values` methods are not -supported. +.. note:: + + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. .. exception:: error @@ -158,62 +213,53 @@ supported. raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) - - Open a ``gdbm`` database and return a :class:`gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The following additional characters may be appended to the flag to control - how the database is opened: - - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ - - Not all flags are valid for all versions of ``gdbm``. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. - - In addition to the dictionary-like methods, ``gdbm`` objects have the - following methods: +.. function:: open(filename, flag="r", mode=0o666, /) + + Open a GDBM database and return a :class:`!gdbm` object. + + :param filename: + The database file to open. + :type filename: :term:`path-like object` + + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| + + The following additional characters may be appended + to control how the database is opened: + + * ``'f'``: Open the database in fast mode. + Writes to the database will not be synchronized. + * ``'s'``: Synchronized mode. + Changes to the database will be written immediately to the file. + * ``'u'``: Do not lock database. + + Not all flags are valid for all versions of GDBM. + See the :data:`open_flags` member for a list of supported flag characters. + + :param int mode: + |mode_param_doc| + + :raises error: + If an invalid *flag* argument is passed. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. + + .. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + :class:`!gdbm` objects behave similar to :term:`mappings <mapping>`, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the - :meth:`nextkey` method. The traversal is ordered by ``gdbm``'s internal + :meth:`nextkey` method. The traversal is ordered by GDBM's internal hash values, and won't be sorted by the key values. This method returns the starting key. @@ -231,7 +277,7 @@ supported. .. method:: gdbm.reorganize() If you have carried out a lot of deletions and would like to shrink the space - used by the ``gdbm`` file, this routine will reorganize the database. ``gdbm`` + used by the GDBM file, this routine will reorganize the database. :class:`!gdbm` objects will not shorten the length of a database file except by using this reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. @@ -243,34 +289,42 @@ supported. .. method:: gdbm.close() - Close the ``gdbm`` database. + Close the GDBM database. .. method:: gdbm.clear() - Remove all items from the ``gdbm`` database. + Remove all items from the GDBM database. .. versionadded:: 3.13 -:mod:`dbm.ndbm` --- Interface based on ndbm -------------------------------------------- +:mod:`dbm.ndbm` --- New Database Manager +---------------------------------------- .. module:: dbm.ndbm :platform: Unix - :synopsis: The standard "database" interface, based on ndbm. + :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the Unix "(n)dbm" library. -Dbm objects behave like mappings (dictionaries), except that keys and values are -always stored as bytes. Printing a ``dbm`` object doesn't print the keys and -values, and the :meth:`items` and :meth:`values` methods are not supported. +The :mod:`dbm.ndbm` module provides an interface to the +:abbr:`NDBM (New Database Manager)` library. +This module can be used with the "classic" NDBM interface or the +:abbr:`GDBM (GNU dbm)` compatibility interface. + +.. note:: -This module can be used with the "classic" ndbm interface or the GNU GDBM -compatibility interface. On Unix, the :program:`configure` script will attempt -to locate the appropriate header file to simplify building this module. + The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + and can not be used interchangeably. + +.. warning:: + + The NDBM library shipped as part of macOS has an undocumented limitation on the + size of values, which can result in corrupted database files + when storing values larger than this limit. Reading such corrupted files can + result in a hard crash (segmentation fault). .. exception:: error @@ -280,49 +334,41 @@ to locate the appropriate header file to simplify building this module. .. data:: library - Name of the ``ndbm`` implementation library used. + Name of the NDBM implementation library used. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="r", mode=0o666, /) - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the - name of the database file (without the :file:`.dir` or :file:`.pag` extensions). + Open an NDBM database and return an :class:`!ndbm` object. - The optional *flag* argument must be one of these values: + :param filename: + The basename of the database file + (without the :file:`.dir` or :file:`.pag` extensions). + :type filename: :term:`path-like object` - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + :param str flag: + * ``'r'`` (default): |flag_r| + * ``'w'``: |flag_w| + * ``'c'``: |flag_c| + * ``'n'``: |flag_n| - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). + :param int mode: + |mode_param_doc| - In addition to the dictionary-like methods, ``ndbm`` objects - provide the following method: + :class:`!ndbm` objects behave similar to :term:`mappings <mapping>`, + but :meth:`!items` and :meth:`!values` methods are not supported. + The following methods are also provided: .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. .. method:: ndbm.close() - Close the ``ndbm`` database. + Close the NDBM database. .. method:: ndbm.clear() - Remove all items from the ``ndbm`` database. + Remove all items from the NDBM database. .. versionadded:: 3.13 @@ -346,13 +392,12 @@ to locate the appropriate header file to simplify building this module. -------------- -The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which -is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no -external library is required. As with other persistent mappings, the keys and -values are always stored as bytes. - -The module defines the following: +The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +interface which is written entirely in Python. +Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +external library is required. +The :mod:`!dbm.dumb` module defines the following: .. exception:: error @@ -360,34 +405,29 @@ The module defines the following: raised for general mapping errors like specifying an incorrect key. -.. function:: open(filename[, flag[, mode]]) +.. function:: open(filename, flag="c", mode=0o666) - Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is - the basename of the database file (without any specific extensions). When a - dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions - are created. + Open a :mod:`!dbm.dumb` database. + The returned database object behaves similar to a :term:`mapping`, + in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` + methods. - The optional *flag* argument can be: + :param filename: + The basename of the database file (without extensions). + A new database creates the following files: - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ + - :file:`{filename}.dat` + - :file:`{filename}.dir` + :type database: :term:`path-like object` - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be modified - by the prevailing umask). + :param str flag: + * ``'r'``: |flag_r| + * ``'w'``: |flag_w| + * ``'c'`` (default): |flag_c| + * ``'n'``: |flag_n| + + :param int mode: + |mode_param_doc| .. warning:: It is possible to crash the Python interpreter when loading a database @@ -395,20 +435,18 @@ The module defines the following: Python's AST compiler. .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. + :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. .. versionchanged:: 3.8 - A database opened with flags ``'r'`` is now read-only. Opening with - flags ``'r'`` and ``'w'`` no longer creates a database if it does not - exist. + A database opened read-only if *flag* is ``'r'``. + A database is not created if it does not exist if *flag* is ``'r'`` or ``'w'``. .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. + *filename* accepts a :term:`path-like object`. In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: + :class:`collections.abc.MutableMapping` class, + the following methods are provided: .. method:: dumbdbm.sync() @@ -417,5 +455,4 @@ The module defines the following: .. method:: dumbdbm.close() - Close the ``dumbdbm`` database. - + Close the database. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index fb8bbf72adf268..db323802a6f68c 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1,5 +1,5 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== +:mod:`!decimal` --- Decimal fixed point and floating point arithmetic +===================================================================== .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. @@ -897,6 +897,48 @@ Decimal objects :const:`Rounded`. If given, applies *rounding*; otherwise, uses the rounding method in either the supplied *context* or the current context. + Decimal numbers can be rounded using the :func:`.round` function: + + .. describe:: round(number) + .. describe:: round(number, ndigits) + + If *ndigits* is not given or ``None``, + returns the nearest :class:`int` to *number*, + rounding ties to even, and ignoring the rounding mode of the + :class:`Decimal` context. Raises :exc:`OverflowError` if *number* is an + infinity or :exc:`ValueError` if it is a (quiet or signaling) NaN. + + If *ndigits* is an :class:`int`, the context's rounding mode is respected + and a :class:`Decimal` representing *number* rounded to the nearest + multiple of ``Decimal('1E-ndigits')`` is returned; in this case, + ``round(number, ndigits)`` is equivalent to + ``self.quantize(Decimal('1E-ndigits'))``. Returns ``Decimal('NaN')`` if + *number* is a quiet NaN. Raises :class:`InvalidOperation` if *number* + is an infinity, a signaling NaN, or if the length of the coefficient after + the quantize operation would be greater than the current context's + precision. In other words, for the non-corner cases: + + * if *ndigits* is positive, return *number* rounded to *ndigits* decimal + places; + * if *ndigits* is zero, return *number* rounded to the nearest integer; + * if *ndigits* is negative, return *number* rounded to the nearest + multiple of ``10**abs(ndigits)``. + + For example:: + + >>> from decimal import Decimal, getcontext, ROUND_DOWN + >>> getcontext().rounding = ROUND_DOWN + >>> round(Decimal('3.75')) # context rounding ignored + 4 + >>> round(Decimal('3.5')) # round-ties-to-even + 4 + >>> round(Decimal('3.75'), 0) # uses the context rounding + Decimal('3') + >>> round(Decimal('3.75'), 1) + Decimal('3.7') + >>> round(Decimal('3.75'), -1) + Decimal('0E+1') + .. _logical_operands_label: @@ -1517,7 +1559,7 @@ are also included in the pure Python version for compatibility. the C version uses a thread-local rather than a coroutine-local context and the value is ``False``. This is slightly faster in some nested context scenarios. -.. versionadded:: 3.9 backported to 3.7 and 3.8. + .. versionadded:: 3.8.3 Rounding modes diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index c553611401d018..ce948a6860f02c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -1,5 +1,5 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== +:mod:`!difflib` --- Helpers for computing deltas +================================================ .. module:: difflib :synopsis: Helpers for computing differences between objects. @@ -52,8 +52,8 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. the purpose of sequence matching. This heuristic can be turned off by setting the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. - .. versionadded:: 3.2 - The *autojunk* parameter. + .. versionchanged:: 3.2 + Added the *autojunk* parameter. .. class:: Differ @@ -171,9 +171,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. expressed in the ISO 8601 format. If not specified, the strings default to blanks. + >>> import sys + >>> from difflib import * >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', tofile='after.py')) + >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', + ... tofile='after.py')) *** before.py --- after.py *************** @@ -294,13 +297,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. - The context diff format normally has a header for filenames and modification + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] >>> sys.stdout.writelines(unified_diff(s1, s2, fromfile='before.py', tofile='after.py')) @@ -381,8 +383,8 @@ The :class:`SequenceMatcher` class has this constructor: The optional argument *autojunk* can be used to disable the automatic junk heuristic. - .. versionadded:: 3.2 - The *autojunk* parameter. + .. versionchanged:: 3.2 + Added the *autojunk* parameter. SequenceMatcher objects get three data attributes: *bjunk* is the set of elements of *b* for which *isjunk* is ``True``; *bpopular* is the set of @@ -629,7 +631,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - <https://code.activestate.com/recipes/576729/>`_ for a small application + <https://code.activestate.com/recipes/576729-simple-version-control/>`_ for a small application built with :class:`SequenceMatcher`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1f773b7b337fa3..e932b865a825a0 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1,5 +1,5 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== +:mod:`!dis` --- Disassembler for Python bytecode +================================================ .. module:: dis :synopsis: Disassembler for Python bytecode. @@ -42,6 +42,19 @@ interpreter. bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. + .. versionchanged:: 3.12 + The argument of a jump is the offset of the target instruction relative + to the instruction that appears immediately after the jump instruction's + :opcode:`CACHE` entries. + + As a consequence, the presence of the :opcode:`CACHE` instructions is + transparent for forward jumps but needs to be taken into account when + reasoning about backward jumps. + + .. versionchanged:: 3.13 + The output shows logical labels rather than instruction offsets + for jump targets and exception handlers. The ``-O`` command line + option and the ``show_offsets`` argument were added. Example: Given the function :func:`!myfunc`:: @@ -54,12 +67,12 @@ the following command can be used to display the disassembly of .. doctest:: >>> dis.dis(myfunc) - 2 0 RESUME 0 + 2 RESUME 0 <BLANKLINE> - 3 2 LOAD_GLOBAL 1 (len + NULL) - 12 LOAD_FAST 0 (alist) - 14 CALL 1 - 22 RETURN_VALUE + 3 LOAD_GLOBAL 1 (len + NULL) + LOAD_FAST 0 (alist) + CALL 1 + RETURN_VALUE (The "2" is a line number). @@ -72,7 +85,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [infile] + python -m dis [-h] [-C] [-O] [infile] The following options are accepted: @@ -86,8 +99,12 @@ The following options are accepted: Show inline caches. +.. cmdoption:: -O, --show-offsets + + Show offsets of instructions. + If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. +Otherwise, disassembly is performed on compiled source code received from stdin. Bytecode analysis ----------------- @@ -99,7 +116,7 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False) + show_caches=False, adaptive=False, show_offsets=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -124,6 +141,9 @@ code. If *adaptive* is ``True``, :meth:`.dis` will display specialized bytecode that may be different from the original bytecode. + If *show_offsets* is ``True``, :meth:`.dis` will include instruction + offsets in the output. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -246,7 +266,8 @@ operation is being performed, so the intermediate analysis object isn't useful: Added the *show_caches* and *adaptive* parameters. -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False, + show_offset=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -261,9 +282,12 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False, + show_offsets=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -288,6 +312,8 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. .. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) @@ -302,26 +328,33 @@ operation is being performed, so the intermediate analysis object isn't useful: source line information (if any) is taken directly from the disassembled code object. - The *show_caches* and *adaptive* parameters work as they do in :func:`dis`. + The *adaptive* parameter works as it does in :func:`dis`. .. versionadded:: 3.4 .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + The *show_caches* parameter is deprecated and has no effect. The iterator + generates the :class:`Instruction` instances with the *cache_info* + field populated (regardless of the value of *show_caches*) and it no longer + generates separate items for the cache entries. .. function:: findlinestarts(code) - This generator function uses the ``co_lines`` method - of the code object *code* to find the offsets which are starts of + This generator function uses the :meth:`~codeobject.co_lines` method + of the :ref:`code object <code-objects>` *code* to find the offsets which + are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. .. versionchanged:: 3.6 Line numbers can be decreasing. Before, they were always increasing. .. versionchanged:: 3.10 - The :pep:`626` ``co_lines`` method is used instead of the ``co_firstlineno`` - and ``co_lnotab`` attributes of the code object. + The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the + :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + attributes of the :ref:`code object <code-objects>`. .. versionchanged:: 3.13 Line numbers can be ``None`` for bytecode that does not map to source lines. @@ -455,6 +488,14 @@ details of bytecode instructions as :class:`Instruction` instances: :class:`dis.Positions` object holding the start and end locations that are covered by this instruction. + .. data::cache_info + + Information about the cache entries of this instruction, as + triplets of the form ``(name, size, data)``, where the ``name`` + and ``size`` describe the cache format and data is the contents + of the cache. ``cache_info`` is ``None`` if the instruction does not have + caches. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -466,8 +507,8 @@ details of bytecode instructions as :class:`Instruction` instances: Changed field ``starts_line``. Added fields ``start_offset``, ``cache_offset``, ``end_offset``, - ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, and - ``line_number``. + ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, + ``line_number`` and ``cache_info``. .. class:: Positions @@ -506,13 +547,21 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: END_FOR - Removes the top two values from the stack. - Equivalent to ``POP_TOP``; ``POP_TOP``. + Removes the top-of-stack item. + Equivalent to ``POP_TOP``. Used to clean up at the end of loops, hence the name. .. versionadded:: 3.12 +.. opcode:: END_SEND + + Implements ``del STACK[-2]``. + Used to clean up when a generator exits. + + .. versionadded:: 3.12 + + .. opcode:: COPY (i) Push the i-th item to the top of the stack without removing it from its original @@ -528,7 +577,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to Swap the top of the stack with the i-th element:: - STACK[-i], STACK[-1] = stack[-1], STACK[-i] + STACK[-i], STACK[-1] = STACK[-1], STACK[-i] .. versionadded:: 3.11 @@ -731,16 +780,6 @@ not have to be) the original ``STACK[-2]``. .. versionadded:: 3.12 -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from ``STACK[-1]``. - Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack:: - - STACK.extend((__aexit__, __aenter__()) - - .. versionadded:: 3.5 - - **Miscellaneous opcodes** @@ -807,6 +846,9 @@ iterations of the loop. .. versionchanged:: 3.12 oparg set to be the exception block depth, for efficient closing of generators. + .. versionchanged:: 3.13 + oparg is ``1`` if this instruction is part of a yield-from or await, and ``0`` + otherwise. .. opcode:: SETUP_ANNOTATIONS @@ -828,8 +870,8 @@ iterations of the loop. .. opcode:: RERAISE Re-raises the exception currently on top of the stack. If oparg is non-zero, - pops an additional value from the stack which is used to set ``f_lasti`` - of the current frame. + pops an additional value from the stack which is used to set + :attr:`~frame.f_lasti` of the current frame. .. versionadded:: 3.9 @@ -878,12 +920,13 @@ iterations of the loop. Exception representation on the stack now consist of one, not three, items. -.. opcode:: LOAD_ASSERTION_ERROR +.. opcode:: LOAD_COMMON_CONSTANT - Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` - statement. + Pushes a common constant onto the stack. The interpreter contains a hardcoded + list of constants supported by this instruction. Used by the :keyword:`assert` + statement to load :exc:`AssertionError`. - .. versionadded:: 3.9 + .. versionadded:: 3.14 .. opcode:: LOAD_BUILD_CLASS @@ -891,18 +934,6 @@ iterations of the loop. Pushes :func:`!builtins.__build_class__` onto the stack. It is later called to construct a class. - -.. opcode:: BEFORE_WITH - - This opcode performs several operations before a with block starts. First, - it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, - :meth:`~object.__enter__` is called. Finally, the result of calling the - ``__enter__()`` method is pushed onto the stack. - - .. versionadded:: 3.11 - - .. opcode:: GET_LEN Perform ``STACK.append(len(STACK[-1]))``. @@ -946,13 +977,13 @@ iterations of the loop. .. opcode:: STORE_NAME (namei) Implements ``name = STACK.pop()``. *namei* is the index of *name* in the attribute - :attr:`!co_names` of the :ref:`code object <code-objects>`. + :attr:`~codeobject.co_names` of the :ref:`code object <code-objects>`. The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. .. opcode:: DELETE_NAME (namei) - Implements ``del name``, where *namei* is the index into :attr:`!co_names` + Implements ``del name``, where *namei* is the index into :attr:`~codeobject.co_names` attribute of the :ref:`code object <code-objects>`. @@ -992,7 +1023,7 @@ iterations of the loop. value = STACK.pop() obj.name = value - where *namei* is the index of name in :attr:`!co_names` of the + where *namei* is the index of name in :attr:`~codeobject.co_names` of the :ref:`code object <code-objects>`. .. opcode:: DELETE_ATTR (namei) @@ -1002,7 +1033,7 @@ iterations of the loop. obj = STACK.pop() del obj.name - where *namei* is the index of name into :attr:`!co_names` of the + where *namei* is the index of name into :attr:`~codeobject.co_names` of the :ref:`code object <code-objects>`. @@ -1159,15 +1190,22 @@ iterations of the loop. .. opcode:: LOAD_SUPER_ATTR (namei) - This opcode implements :func:`super` (e.g. ``super().method()`` and - ``super().attr``). It works the same as :opcode:`LOAD_ATTR`, except that - ``namei`` is shifted left by 2 bits instead of 1, and instead of expecting a - single receiver on the stack, it expects three objects (from top of stack - down): ``self`` (the first argument to the current method), ``cls`` (the - class within which the current method was defined), and the global ``super``. + This opcode implements :func:`super`, both in its zero-argument and + two-argument forms (e.g. ``super().method()``, ``super().attr`` and + ``super(cls, self).method()``, ``super(cls, self).attr``). + + It pops three values from the stack (from top of stack down): + + * ``self``: the first argument to the current method + * ``cls``: the class within which the current method was defined + * the global ``super`` + + With respect to its argument, it works similarly to :opcode:`LOAD_ATTR`, + except that ``namei`` is shifted left by 2 bits instead of 1. The low bit of ``namei`` signals to attempt a method load, as with - :opcode:`LOAD_ATTR`. + :opcode:`LOAD_ATTR`, which results in pushing ``NULL`` and the loaded method. + When it is unset a single value is pushed to the stack. The second-low bit of ``namei``, if set, means that this was a two-argument call to :func:`super` (unset means zero-argument). @@ -1359,7 +1397,7 @@ iterations of the loop. Pushes a reference to the object the cell contains on the stack. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: LOAD_FROM_DICT_OR_DEREF (i) @@ -1381,7 +1419,7 @@ iterations of the loop. storage. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: DELETE_DEREF (i) @@ -1392,7 +1430,7 @@ iterations of the loop. .. versionadded:: 3.2 .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: COPY_FREE_VARS (n) @@ -1548,7 +1586,7 @@ iterations of the loop. value = STACK.pop() result = func(value) - STACK.push(result) + STACK.append(result) * ``oparg == 1``: call :func:`str` on *value* * ``oparg == 2``: call :func:`repr` on *value* @@ -1565,7 +1603,7 @@ iterations of the loop. value = STACK.pop() result = value.__format__("") - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). @@ -1578,7 +1616,7 @@ iterations of the loop. spec = STACK.pop() value = STACK.pop() result = value.__format__(spec) - STACK.push(result) + STACK.append(result) Used for implementing formatted literal strings (f-strings). @@ -1603,11 +1641,12 @@ iterations of the loop. success (``True``) or failure (``False``). -.. opcode:: RESUME (where) +.. opcode:: RESUME (context) A no-op. Performs internal tracing, debugging and optimization checks. - The ``where`` operand marks where the ``RESUME`` occurs: + The ``context`` operand consists of two parts. The lowest two bits + indicate where the ``RESUME`` occurs: * ``0`` The start of a function, which is neither a generator, coroutine nor an async generator @@ -1615,8 +1654,14 @@ iterations of the loop. * ``2`` After a ``yield from`` expression * ``3`` After an ``await`` expression + The next bit is ``1`` if the RESUME is at except-depth ``1``, and ``0`` + otherwise. + .. versionadded:: 3.11 + .. versionchanged:: 3.13 + The oparg value changed to include information about except-depth + .. opcode:: RETURN_GENERATOR @@ -1632,9 +1677,9 @@ iterations of the loop. Equivalent to ``STACK[-1] = STACK[-2].send(STACK[-1])``. Used in ``yield from`` and ``await`` statements. - If the call raises :exc:`StopIteration`, pop both items, push the - exception's ``value`` attribute, and increment the bytecode counter by - *delta*. + If the call raises :exc:`StopIteration`, pop the top value from the stack, + push the exception's ``value`` attribute, and increment the bytecode counter + by *delta*. .. versionadded:: 3.11 @@ -1664,7 +1709,7 @@ iterations of the loop. Calls an intrinsic function with one argument. Passes ``STACK[-1]`` as the argument and sets ``STACK[-1]`` to the result. Used to implement - functionality that is necessary but not performance critical. + functionality that is not performance critical. The operand determines which intrinsic function is called: @@ -1712,9 +1757,13 @@ iterations of the loop. .. opcode:: CALL_INTRINSIC_2 - Calls an intrinsic function with two arguments. Passes ``STACK[-2]``, ``STACK[-1]`` as the - arguments and sets ``STACK[-1]`` to the result. Used to implement functionality that is - necessary but not performance critical. + Calls an intrinsic function with two arguments. Used to implement functionality + that is not performance critical:: + + arg2 = STACK.pop() + arg1 = STACK.pop() + result = intrinsic2(arg1, arg2) + STACK.append(result) The operand determines which intrinsic function is called: @@ -1741,6 +1790,17 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: LOAD_SPECIAL + + Performs special method lookup on ``STACK[-1]``. + If ``type(STACK[-1]).__xxx__`` is a method, leave + ``type(STACK[-1]).__xxx__; STACK[-1]`` on the stack. + If ``type(STACK[-1]).__xxx__`` is not a method, leave + ``STACK[-1].__xxx__; NULL`` on the stack. + + .. versionadded:: 3.14 + + **Pseudo-instructions** These opcodes do not appear in Python bytecode. They are used by the compiler @@ -1810,8 +1870,9 @@ These collections are provided for automatic introspection of bytecode instructions: .. versionchanged:: 3.12 - The collections now contain pseudo instructions as well. These are - opcodes with values ``>= MIN_PSEUDO_OPCODE``. + The collections now contain pseudo instructions and instrumented + instructions as well. These are opcodes with values ``>= MIN_PSEUDO_OPCODE`` + and ``>= MIN_INSTRUMENTED_OPCODE``. .. data:: opname diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 54a8e79a0ef941..6b0282eed49566 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,5 @@ -:mod:`doctest` --- Test interactive Python examples -=================================================== +:mod:`!doctest` --- Test interactive Python examples +==================================================== .. module:: doctest :synopsis: Test pieces of code within docstrings. @@ -123,10 +123,10 @@ And so on, eventually ending with: OverflowError: n too large ok 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. + 1 test in __main__ + 6 tests in __main__.factorial + 7 tests in 2 items. + 7 passed. Test passed. $ @@ -134,7 +134,7 @@ That's all you need to know to start making productive use of :mod:`doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file -:file:`Lib/test/test_doctest.py`. +:file:`Lib/test/test_doctest/test_doctest.py`. .. _doctest-simple-testmod: @@ -143,13 +143,13 @@ Simple Usage: Checking Examples in Docstrings --------------------------------------------- The simplest way to start using doctest (but not necessarily the way you'll -continue to do it) is to end each module :mod:`M` with:: +continue to do it) is to end each module :mod:`!M` with:: if __name__ == "__main__": import doctest doctest.testmod() -:mod:`doctest` then examines docstrings in module :mod:`M`. +:mod:`!doctest` then examines docstrings in module :mod:`!M`. Running the module as a script causes the examples in the docstrings to get executed and verified:: @@ -277,13 +277,34 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. -In addition, if ``M.__test__`` exists and "is true", it must be a dict, and each +In addition, there are cases when you want tests to be part of a module but not part +of the help text, which requires that the tests not be included in the docstring. +Doctest looks for a module-level variable called ``__test__`` and uses it to locate other +tests. If ``M.__test__`` exists, it must be a dict, and each entry maps a (string) name to a function object, class object, or string. Function and class object docstrings found from ``M.__test__`` are searched, and strings are treated as if they were docstrings. In output, a key ``K`` in -``M.__test__`` appears with name :: +``M.__test__`` appears with name ``M.__test__.K``. - <name of M>.__test__.K +For example, place this block of code at the top of :file:`example.py`: + +.. code-block:: python + + __test__ = { + 'numbers': """ + >>> factorial(6) + 720 + + >>> [factorial(n) for n in range(6)] + [1, 1, 2, 6, 24, 120] + """ + } + +The value of ``example.__test__["numbers"]`` will be treated as a +docstring and all the tests inside it will be run. It is +important to note that the value can be mapped to a function, +class object, or module; if so, :mod:`!doctest` +searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. @@ -382,10 +403,10 @@ What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, each time :mod:`doctest` finds a docstring to test, it uses a -*shallow copy* of :mod:`M`'s globals, so that running tests doesn't change the -module's real globals, and so that one test in :mod:`M` can't leave behind +*shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the +module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can -freely use any names defined at top-level in :mod:`M`, and names defined earlier +freely use any names defined at top-level in :mod:`!M`, and names defined earlier in the docstring being run. Examples cannot see names defined in other docstrings. @@ -409,10 +430,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "<stdin>", line 1, in <module> - ValueError: 42 is not in list + ValueError: list.remove(x): x not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` -detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): +x not in list`` detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of @@ -779,18 +800,18 @@ guarantee about output. For example, when printing a set, Python doesn't guarantee that the element is printed in any particular order, so a test like :: >>> foo() - {"Hermione", "Harry"} + {"spam", "eggs"} is vulnerable! One workaround is to do :: - >>> foo() == {"Hermione", "Harry"} + >>> foo() == {"spam", "eggs"} True instead. Another is to do :: >>> d = sorted(foo()) >>> d - ['Harry', 'Hermione'] + ['eggs', 'spam'] There are others, but you get the idea. @@ -923,8 +944,8 @@ and :ref:`doctest-simple-testfile`. (or module :mod:`__main__` if *m* is not supplied or is ``None``), starting with ``m.__doc__``. - Also test examples reachable from dict ``m.__test__``, if it exists and is not - ``None``. ``m.__test__`` maps names (strings) to functions, classes and + Also test examples reachable from dict ``m.__test__``, if it exists. + ``m.__test__`` maps names (strings) to functions, classes and strings; function and class docstrings are searched for examples; strings are searched directly, as if they were docstrings. @@ -937,7 +958,8 @@ and :ref:`doctest-simple-testfile`. Optional argument *exclude_empty* defaults to false. If true, objects for which no doctests are found are excluded from consideration. The default is a backward - compatibility hack, so that code still using :meth:`doctest.master.summarize` in + compatibility hack, so that code still using + :meth:`doctest.master.summarize <DocTestRunner.summarize>` in conjunction with :func:`testmod` continues to get output for objects with no tests. The *exclude_empty* argument to the newer :class:`DocTestFinder` constructor defaults to true. @@ -976,7 +998,7 @@ As your collection of doctest'ed modules grows, you'll want a way to run all their doctests systematically. :mod:`doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include -a :func:`load_tests` function in your test module:: +a :ref:`load_tests <load_tests-protocol>` function in your test module:: import unittest import doctest @@ -999,7 +1021,8 @@ from text files and modules with doctests: and runs the interactive examples in each file. If an example in any file fails, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. + (sometimes approximate) line number. If all the examples in a file are + skipped, then the synthesized unit test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1065,7 +1088,8 @@ from text files and modules with doctests: and runs each doctest in the module. If any of the doctests fail, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) - line number. + line number. If all the examples in a docstring are skipped, then the + synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1090,19 +1114,24 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. +.. exception:: failureException + + When doctests which have been converted to unit tests by :func:`DocFileSuite` + or :func:`DocTestSuite` fail, this exception is raised showing the name of + the file containing the test and a (sometimes approximate) line number. Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out -of :class:`doctest.DocTestCase` instances, and :class:`DocTestCase` is a -subclass of :class:`unittest.TestCase`. :class:`DocTestCase` isn't documented +of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a +subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented here (it's an internal detail), but studying its code can answer questions about the exact details of :mod:`unittest` integration. Similarly, :func:`DocFileSuite` creates a :class:`unittest.TestSuite` out of -:class:`doctest.DocFileCase` instances, and :class:`DocFileCase` is a subclass -of :class:`DocTestCase`. +:class:`!doctest.DocFileCase` instances, and :class:`!DocFileCase` is a subclass +of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of -:class:`DocTestCase`. This is important for a subtle reason: when you run +:class:`!DocTestCase`. This is important for a subtle reason: when you run :mod:`doctest` functions yourself, you can control the :mod:`doctest` options in use directly, by passing option flags to :mod:`doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls @@ -1123,14 +1152,14 @@ reporting flags specific to :mod:`unittest` support, via this function: section :ref:`doctest-options`. Only "reporting flags" can be used. This is a module-global setting, and affects all future doctests run by module - :mod:`unittest`: the :meth:`runTest` method of :class:`DocTestCase` looks at - the option flags specified for the test case when the :class:`DocTestCase` + :mod:`unittest`: the :meth:`!runTest` method of :class:`!DocTestCase` looks at + the option flags specified for the test case when the :class:`!DocTestCase` instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`doctest`'s :mod:`unittest` reporting flags are + typical and expected case), :mod:`!doctest`'s :mod:`unittest` reporting flags are :ref:`bitwise ORed <bitwise>` into the option flags, and the option flags so augmented are passed to the :class:`DocTestRunner` instance created to run the doctest. If any reporting flags were specified when the - :class:`DocTestCase` instance was constructed, :mod:`doctest`'s + :class:`!DocTestCase` instance was constructed, :mod:`!doctest`'s :mod:`unittest` reporting flags are ignored. The value of the :mod:`unittest` reporting flags in effect before the function @@ -1300,7 +1329,8 @@ Example Objects A dictionary mapping from option flags to ``True`` or ``False``, which is used to override default options for this example. Any option flags not contained in this dictionary are left at their default value (as specified by the - :class:`DocTestRunner`'s :attr:`optionflags`). By default, no options are set. + :class:`DocTestRunner`'s :ref:`optionflags <doctest-options>`). + By default, no options are set. .. _doctest-doctestfinder: @@ -1474,7 +1504,7 @@ DocTestRunner objects :attr:`failures` and :attr:`skips` attributes. The :meth:`run` and :meth:`summarize` methods return a :class:`TestResults` instance. - :class:`DocTestParser` defines the following methods: + :class:`DocTestRunner` defines the following methods: .. method:: report_start(out, test, example) @@ -1539,7 +1569,7 @@ DocTestRunner objects The output of each example is checked using the :class:`DocTestRunner`'s output checker, and the results are formatted by the - :meth:`DocTestRunner.report_\*` methods. + :meth:`!DocTestRunner.report_\*` methods. .. method:: summarize(verbose=None) @@ -1714,12 +1744,12 @@ code under the debugger: module) of the object with the doctests of interest. The result is a string, containing the object's docstring converted to a Python script, as described for :func:`script_from_examples` above. For example, if module :file:`a.py` - contains a top-level function :func:`f`, then :: + contains a top-level function :func:`!f`, then :: import a, doctest print(doctest.testsource(a, "a.f")) - prints a script version of function :func:`f`'s docstring, with doctests + prints a script version of function :func:`!f`'s docstring, with doctests converted to code, and the rest placed in comments. @@ -1905,7 +1935,7 @@ such a test runner:: optionflags=flags) else: fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) + print(f"{fail} failures out of {total} tests") .. rubric:: Footnotes diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index aa0134412f3a60..6875af2be49d7a 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -1,5 +1,5 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- +:mod:`!email.charset`: Representing character sets +-------------------------------------------------- .. module:: email.charset :synopsis: Character Sets diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 5bef155a4af310..c4c322a82e1f44 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -367,7 +367,7 @@ Here are the methods of the :class:`Message` class: .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (defaults to ``None``). Here are some additional useful methods: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 5b49339650f0e9..34121f8c0a7727 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -1,5 +1,5 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- +:mod:`!email.contentmanager`: Managing MIME Content +--------------------------------------------------- .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 3bd377e33f6c15..9c8c8c9234ed7a 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -1,5 +1,5 @@ -:mod:`email.encoders`: Encoders -------------------------------- +:mod:`!email.encoders`: Encoders +-------------------------------- .. module:: email.encoders :synopsis: Encoders for email message payloads. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 194a98696f437d..33ab4265116178 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -1,5 +1,5 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- +:mod:`!email.errors`: Exception and Defect classes +-------------------------------------------------- .. module:: email.errors :synopsis: The exception classes used by the email package. @@ -58,6 +58,15 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). +.. exception:: MessageDefect() + + This is the base class for all defects found when parsing email messages. + It is derived from :exc:`ValueError`. + +.. exception:: HeaderDefect() + + This is the base class for all defects found when parsing email headers. + It is derived from :exc:`MessageDefect`. Here is the list of the defects that the :class:`~email.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index afa0038ea2d6c4..a3132d02687bc9 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -1,5 +1,5 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- +:mod:`!email.generator`: Generating MIME documents +-------------------------------------------------- .. module:: email.generator :synopsis: Generate flat text email messages from a message structure. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e093f138936b36..219fad0d2f6745 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -1,5 +1,5 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- +:mod:`!email.header`: Internationalized headers +----------------------------------------------- .. module:: email.header :synopsis: Representing non-ASCII headers @@ -77,7 +77,7 @@ Here is the :class:`Header` class description: The maximum line length can be specified explicitly via *maxlinelen*. For splitting the first line to a shorter value (to account for the field header which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value + field in *header_name*. The default *maxlinelen* is 78, and the default value for *header_name* is ``None``, meaning it is not taken into account for the first line of a long, split header. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 00a954e0307ea6..bcbd00c833e28e 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -1,5 +1,5 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- +:mod:`!email.headerregistry`: Custom Header Objects +--------------------------------------------------- .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index d53ab33b8904a7..090981d84b4de3 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -1,5 +1,5 @@ -:mod:`email.iterators`: Iterators ---------------------------------- +:mod:`!email.iterators`: Iterators +---------------------------------- .. module:: email.iterators :synopsis: Iterate over a message object tree. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 225f498781fa86..e9cce1af186526 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -1,5 +1,5 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- +:mod:`!email.message`: Representing an email message +---------------------------------------------------- .. module:: email.message :synopsis: The base class representing email messages. @@ -40,9 +40,9 @@ over the object tree. The :class:`EmailMessage` dictionary-like interface is indexed by the header names, which must be ASCII values. The values of the dictionary are strings with some extra methods. Headers are stored and returned in case-preserving -form, but field names are matched case-insensitively. Unlike a real dict, -there is an ordering to the keys, and there can be duplicate keys. Additional -methods are provided for working with headers that have duplicate keys. +form, but field names are matched case-insensitively. The keys are ordered, +but unlike a real dict, there can be duplicates. Additional methods are +provided for working with headers that have duplicate keys. The *payload* is either a string or bytes object, in the case of simple message objects, or a list of :class:`EmailMessage` objects, for MIME container @@ -247,7 +247,7 @@ message objects. .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (*failobj* defaults to ``None``). diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d7c0d203d191f8..b85673a4acd0d0 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -1,5 +1,5 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- +:mod:`!email.mime`: Creating email and MIME objects from scratch +---------------------------------------------------------------- .. module:: email.mime :synopsis: Build MIME messages. @@ -28,7 +28,7 @@ make things easier. Here are the classes: -.. currentmodule:: email.mime.base +.. module:: email.mime.base .. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params) @@ -58,7 +58,7 @@ Here are the classes: Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.nonmultipart +.. module:: email.mime.nonmultipart .. class:: MIMENonMultipart() @@ -72,7 +72,7 @@ Here are the classes: is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. -.. currentmodule:: email.mime.multipart +.. module:: email.mime.multipart .. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \ *, policy=compat32, **_params) @@ -104,7 +104,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.application +.. module:: email.mime.application .. class:: MIMEApplication(_data, _subtype='octet-stream', \ _encoder=email.encoders.encode_base64, \ @@ -135,7 +135,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.audio +.. module:: email.mime.audio .. class:: MIMEAudio(_audiodata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -169,7 +169,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.image +.. module:: email.mime.image .. class:: MIMEImage(_imagedata, _subtype=None, \ _encoder=email.encoders.encode_base64, \ @@ -205,7 +205,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.message +.. module:: email.mime.message .. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32) @@ -225,7 +225,7 @@ Here are the classes: .. versionchanged:: 3.6 Added *policy* keyword-only parameter. -.. currentmodule:: email.mime.text +.. module:: email.mime.text .. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index dda0466a6afa7d..439b5c8f34b65a 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -1,5 +1,5 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- +:mod:`!email.parser`: Parsing email messages +-------------------------------------------- .. module:: email.parser :synopsis: Parse flat text email messages to produce a message object structure. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index fd47dd0dc5df36..83feedf728351e 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -1,5 +1,5 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ +:mod:`!email.policy`: Policy Objects +------------------------------------ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages @@ -219,7 +219,6 @@ added matters. To illustrate:: Default: :const:`False`. .. versionadded:: 3.5 - The *mangle_from_* parameter. .. attribute:: message_factory diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 3a6039004fcaae..66c42e4a5008ee 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -1,5 +1,5 @@ -:mod:`email` --- An email and MIME handling package -=================================================== +:mod:`!email` --- An email and MIME handling package +==================================================== .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 345b64001c1ace..43e5b25df01f79 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -1,5 +1,5 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- +:mod:`!email.utils`: Miscellaneous utilities +-------------------------------------------- .. module:: email.utils :synopsis: Miscellaneous email package utilities. @@ -17,8 +17,7 @@ module: arguments, return current time. Otherwise *dt* argument should be a :class:`~datetime.datetime` instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. The - *isdst* parameter is ignored. + is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. .. versionadded:: 3.3 @@ -58,13 +57,18 @@ of the new API. begins with angle brackets, they are stripped off. -.. function:: parseaddr(address) +.. function:: parseaddr(address, *, strict=True) Parse address -- which should be the value of some address-containing field such as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and *email address* parts. Returns a tuple of that information, unless the parse fails, in which case a 2-tuple of ``('', '')`` is returned. + If *strict* is true, use a strict parser which rejects malformed inputs. + + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: formataddr(pair, charset='utf-8') @@ -82,12 +86,15 @@ of the new API. Added the *charset* option. -.. function:: getaddresses(fieldvalues) +.. function:: getaddresses(fieldvalues, *, strict=True) This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple - example that gets all the recipients of a message:: + :meth:`Message.get_all <email.message.Message.get_all>`. + + If *strict* is true, use a strict parser which rejects malformed inputs. + + Here's a simple example that gets all the recipients of a message:: from email.utils import getaddresses @@ -97,6 +104,9 @@ of the new API. resent_ccs = msg.get_all('resent-cc', []) all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: parsedate(date) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index de3b93f5e61073..518a2940edcf69 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -1,5 +1,5 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== +:mod:`!ensurepip` --- Bootstrapping the ``pip`` installer +========================================================= .. module:: ensurepip :synopsis: Bootstrapping the "pip" installer into an existing Python @@ -38,7 +38,7 @@ when creating a virtual environment) or after explicitly uninstalling :pep:`453`: Explicit bootstrapping of pip in Python installations The original rationale and specification for this module. -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-ios-notavail.rst Command line interface ---------------------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index d5f46ed58e8d24..8b3f397ea862f4 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1,5 +1,5 @@ -:mod:`enum` --- Support for enumerations -======================================== +:mod:`!enum` --- Support for enumerations +========================================= .. module:: enum :synopsis: Implementation of an enumeration class. @@ -165,12 +165,12 @@ Data Types to subclass *EnumType* -- see :ref:`Subclassing EnumType <enumtype-examples>` for details. - *EnumType* is responsible for setting the correct :meth:`!__repr__`, + ``EnumType`` is responsible for setting the correct :meth:`!__repr__`, :meth:`!__str__`, :meth:`!__format__`, and :meth:`!__reduce__` methods on the final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. - .. method:: EnumType.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -198,11 +198,12 @@ Data Types >>> some_var = Color.RED >>> some_var in Color True + >>> Color.RED.value in Color + True - .. note:: + .. versionchanged:: 3.12 - In Python 3.12 it will be possible to check for member values and not - just members; until then, a ``TypeError`` will be raised if a + Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. .. method:: EnumType.__dir__(cls) @@ -234,6 +235,10 @@ Data Types >>> len(Color) 3 + .. attribute:: EnumType.__members__ + + Returns a mapping of every enum name to its member, including aliases + .. method:: EnumType.__reversed__(cls) Returns each member in *cls* in reverse definition order:: @@ -241,9 +246,19 @@ Data Types >>> list(reversed(Color)) [<Color.BLUE: 3>, <Color.GREEN: 2>, <Color.RED: 1>] + .. method:: EnumType._add_alias_ + + Adds a new name as an alias to an existing member. Raises a + :exc:`NameError` if the name is already assigned to a different member. + + .. method:: EnumType._add_value_alias_ + + Adds a new value as an alias to an existing member. Raises a + :exc:`ValueError` if the value is already linked with a different member. + .. versionadded:: 3.11 - Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. .. class:: Enum @@ -264,6 +279,8 @@ Data Types >>> Color.RED.value 1 + Value of the member, can be set in :meth:`~Enum.__new__`. + .. note:: Enum member values Member values can be anything: :class:`int`, :class:`str`, etc. If @@ -271,6 +288,24 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + While mutable/unhashable values, such as :class:`dict`, :class:`list` or + a mutable :class:`~dataclasses.dataclass`, can be used, they will have a + quadratic performance impact during creation relative to the + total number of mutable/unhashable values in the enum. + + .. attribute:: Enum._name_ + + Name of the member. + + .. attribute:: Enum._value_ + + Value of the member, can be set in :meth:`~Enum.__new__`. + + .. attribute:: Enum._order_ + + No longer used, kept for backward compatibility. + (class attribute, removed during class creation). + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -322,6 +357,17 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. method:: Enum.__init__(self, *args, **kwds) + + By default, does nothing. If multiple values are given in the member + assignment, those values become separate arguments to ``__init__``; e.g. + + >>> from enum import Enum + >>> class Weekday(Enum): + ... MONDAY = 1, 'Mon' + + ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. @@ -349,6 +395,23 @@ Data Types >>> Build('deBUG') <Build.DEBUG: 'debug'> + .. method:: Enum.__new__(cls, *args, **kwds) + + By default, doesn't exist. If specified, either in the enum class + definition or in a mixin class (such as ``int``), all values given + in the member assignment will be passed; e.g. + + >>> from enum import Enum + >>> class MyIntEnum(int, Enum): + ... TWENTYSIX = '1a', 16 + + results in the call ``int('1a', 16)`` and a value of ``26`` for the member. + + .. note:: + + When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the @@ -405,7 +468,7 @@ Data Types .. class:: IntEnum - *IntEnum* is the same as *Enum*, but its members are also integers and can be + *IntEnum* is the same as :class:`Enum`, but its members are also integers and can be used anywhere that an integer can be used. If any integer operation is performed with an *IntEnum* member, the resulting value loses its enumeration status. @@ -436,7 +499,7 @@ Data Types .. class:: StrEnum - *StrEnum* is the same as *Enum*, but its members are also strings and can be used + ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used in most of the same places that a string can be used. The result of any string operation performed on or with a *StrEnum* member is not part of the enumeration. @@ -462,9 +525,9 @@ Data Types .. class:: Flag - *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), - ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members - of the enumeration. + ``Flag`` is the same as :class:`Enum`, but its members support the bitwise + operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); + the results of those operations are (aliases of) members of the enumeration. .. method:: __contains__(self, value) @@ -496,9 +559,7 @@ Data Types >>> list(purple) [<Color.RED: 1>, <Color.BLUE: 4>] - .. versionchanged:: 3.11 - - Aliases are no longer returned during iteration. + .. versionadded:: 3.11 .. method:: __len__(self): @@ -548,7 +609,7 @@ Data Types .. method:: __invert__(self): - Returns all the flags in *type(self)* that are not in self:: + Returns all the flags in *type(self)* that are not in *self*:: >>> ~white <Color: 0> @@ -568,14 +629,14 @@ Data Types of two, starting with ``1``. .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It - is now:: + is now: >>> Color(0) # doctest: +SKIP <Color: 0> .. class:: IntFlag - *IntFlag* is the same as *Flag*, but its members are also integers and can be + ``IntFlag`` is the same as :class:`Flag`, but its members are also integers and can be used anywhere that an integer can be used. >>> from enum import IntFlag, auto @@ -595,12 +656,12 @@ Data Types >>> Color.RED + 2 3 - If a *Flag* operation is performed with an *IntFlag* member and: + If a :class:`Flag` operation is performed with an *IntFlag* member and: * the result is a valid *IntFlag*: an *IntFlag* is returned - * the result is not a valid *IntFlag*: the result depends on the *FlagBoundary* setting + * the result is not a valid *IntFlag*: the result depends on the :class:`FlagBoundary` setting - The *repr()* of unnamed zero-valued flags has changed. It is now: + The :func:`repr()` of unnamed zero-valued flags has changed. It is now: >>> Color(0) <Color: 0> @@ -696,7 +757,7 @@ Data Types .. class:: FlagBoundary - *FlagBoundary* controls how out-of-range values are handled in *Flag* and its + ``FlagBoundary`` controls how out-of-range values are handled in :class:`Flag` and its subclasses. .. attribute:: STRICT @@ -719,7 +780,7 @@ Data Types .. attribute:: CONFORM - Out-of-range values have invalid values removed, leaving a valid *Flag* + Out-of-range values have invalid values removed, leaving a valid :class:`Flag` value:: >>> from enum import Flag, CONFORM, auto @@ -733,7 +794,7 @@ Data Types .. attribute:: EJECT - Out-of-range values lose their *Flag* membership and revert to :class:`int`. + Out-of-range values lose their :class:`Flag` membership and revert to :class:`int`. >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): @@ -746,7 +807,7 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. + Out-of-range values are kept, and the :class:`Flag` membership is kept. This is the default for :class:`IntFlag`:: >>> from enum import Flag, KEEP, auto @@ -768,37 +829,47 @@ Supported ``__dunder__`` names :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`!_value_` appropriately. Once -all the members are created it is no longer used. +:meth:`~Enum.__new__`, if specified, must create and return the enum members; +it is also a very good idea to set the member's :attr:`!_value_` appropriately. +Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used to get an appropriate value for an enum - member; may be overridden +- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an + existing member. +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :attr:`~Enum._order_` -- no longer used, kept for backward + compatibility (class attribute, removed during class creation) +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden .. note:: - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. + power-of-two. + +- While ``_sunder_`` names are generally reserved for the further development + of the :class:`Enum` class and can not be used, some are explicitly allowed: + + - ``_repr_*`` (e.g. ``_repr_html_``), as used in `IPython's rich display`_ .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_``, ``_repr_*`` +.. _`IPython's rich display`: https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display --------------- @@ -808,10 +879,10 @@ Utilities and Decorators .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will - call an *Enum*'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For *Enum* and *IntEnum* that appropriate value will be the last value plus - one; for *Flag* and *IntFlag* it will be the first power-of-two greater - than the highest value; for *StrEnum* it will be the lower-cased version of + call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus + one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. @@ -819,7 +890,7 @@ Utilities and Decorators * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is - used to create the ``SECOND`` enum member; + used to create the ``SECOND`` enum member; * ``THREE = [auto(), -3]`` will *not* work (``<auto instance>, -3`` is used to create the ``THREE`` enum member) diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 283e8b013265d9..4983b8961b1c3f 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -1,5 +1,5 @@ -:mod:`errno` --- Standard errno system symbols -============================================== +:mod:`!errno` --- Standard errno system symbols +=============================================== .. module:: errno :synopsis: Standard errno system symbols. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index cd85df8723a76b..7910b306f143d7 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -16,7 +16,7 @@ equivalent, even if they have the same name. .. index:: pair: statement; raise -The built-in exceptions listed below can be generated by the interpreter or +The built-in exceptions listed in this chapter can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" indicating the detailed cause of the error. This may be a string or a tuple of several items of information (e.g., an error code and a string explaining the @@ -38,36 +38,48 @@ information on defining exceptions is available in the Python Tutorial under Exception context ----------------- -When raising a new exception while another exception -is already being handled, the new exception's -:attr:`__context__` attribute is automatically set to the handled -exception. An exception may be handled when an :keyword:`except` or -:keyword:`finally` clause, or a :keyword:`with` statement, is used. +.. index:: pair: exception; chaining + __cause__ (exception attribute) + __context__ (exception attribute) + __suppress_context__ (exception attribute) -This implicit exception context can be -supplemented with an explicit cause by using :keyword:`!from` with -:keyword:`raise`:: +Three attributes on exception objects provide information about the context in +which the exception was raised: - raise new_exc from original_exc +.. attribute:: BaseException.__context__ + BaseException.__cause__ + BaseException.__suppress_context__ -The expression following :keyword:`from<raise>` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. + When raising a new exception while another exception + is already being handled, the new exception's + :attr:`!__context__` attribute is automatically set to the handled + exception. An exception may be handled when an :keyword:`except` or + :keyword:`finally` clause, or a :keyword:`with` statement, is used. -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. + This implicit exception context can be + supplemented with an explicit cause by using :keyword:`!from` with + :keyword:`raise`:: -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. + raise new_exc from original_exc + + The expression following :keyword:`from<raise>` must be an exception or ``None``. It + will be set as :attr:`!__cause__` on the raised exception. Setting + :attr:`!__cause__` also implicitly sets the :attr:`!__suppress_context__` + attribute to ``True``, so that using ``raise new_exc from None`` + effectively replaces the old exception with the new one for display + purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while + leaving the old exception available in :attr:`!__context__` for introspection + when debugging. + + The default traceback display code shows these chained exceptions in + addition to the traceback for the exception itself. An explicitly chained + exception in :attr:`!__cause__` is always shown when present. An implicitly + chained exception in :attr:`!__context__` is shown only if :attr:`!__cause__` + is :const:`None` and :attr:`!__suppress_context__` is false. + + In either case, the exception itself is always shown after any chained + exceptions so that the final line of the traceback always shows the last + exception that was raised. Inheriting from built-in exceptions @@ -126,6 +138,12 @@ The following exceptions are used mostly as base classes for other exceptions. tb = sys.exception().__traceback__ raise OtherException(...).with_traceback(tb) + .. attribute:: __traceback__ + + A writable field that holds the + :ref:`traceback object <traceback-objects>` associated with this + exception. See also: :ref:`raise`. + .. method:: add_note(note) Add the string ``note`` to the exception's notes which appear in the standard @@ -317,9 +335,9 @@ The following exceptions are the exceptions that are usually raised. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`NotImplemented` are not interchangeable, even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. + :data:`!NotImplemented` for details on when to use it. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) @@ -398,6 +416,24 @@ The following exceptions are the exceptions that are usually raised. handling in C, most floating point operations are not checked. +.. exception:: PythonFinalizationError + + This exception is derived from :exc:`RuntimeError`. It is raised when + an operation is blocked during interpreter shutdown also known as + :term:`Python finalization <interpreter shutdown>`. + + Examples of operations which can be blocked with a + :exc:`PythonFinalizationError` during the Python finalization: + + * Creating a new Python thread. + * :func:`os.fork`. + + See also the :func:`sys.is_finalizing` function. + + .. versionadded:: 3.13 + Previously, a plain :exc:`RuntimeError` was raised. + + .. exception:: RecursionError This exception is derived from :exc:`RuntimeError`. It is raised when the @@ -429,9 +465,11 @@ The following exceptions are the exceptions that are usually raised. :meth:`~iterator.__next__` method to signal that there are no further items produced by the iterator. - The exception object has a single attribute :attr:`value`, which is - given as an argument when constructing the exception, and defaults - to :const:`None`. + .. attribute:: StopIteration.value + + The exception object has a single attribute :attr:`!value`, which is + given as an argument when constructing the exception, and defaults + to :const:`None`. When a :term:`generator` or :term:`coroutine` function returns, a new :exc:`StopIteration` instance is @@ -927,8 +965,10 @@ their subgroups based on the types of the contained exceptions. true for the exceptions that should be in the subgroup. The nesting structure of the current exception is preserved in the result, - as are the values of its :attr:`message`, :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields. + as are the values of its :attr:`message`, + :attr:`~BaseException.__traceback__`, :attr:`~BaseException.__cause__`, + :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields. Empty nested groups are omitted from the result. The condition is checked for all exceptions in the nested exception group, @@ -949,15 +989,20 @@ their subgroups based on the types of the contained exceptions. Returns an exception group with the same :attr:`message`, but which wraps the exceptions in ``excs``. - This method is used by :meth:`subgroup` and :meth:`split`. A + This method is used by :meth:`subgroup` and :meth:`split`, which + are used in various contexts to break up an exception group. A subclass needs to override it in order to make :meth:`subgroup` and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. - :meth:`subgroup` and :meth:`split` copy the :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields from + :meth:`subgroup` and :meth:`split` copy the + :attr:`~BaseException.__traceback__`, + :attr:`~BaseException.__cause__`, :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields from the original exception group to the one returned by :meth:`derive`, so - these fields do not need to be updated by :meth:`derive`. :: + these fields do not need to be updated by :meth:`derive`. + + .. doctest:: >>> class MyGroup(ExceptionGroup): ... def derive(self, excs): @@ -983,9 +1028,9 @@ their subgroups based on the types of the contained exceptions. True - Note that :exc:`BaseExceptionGroup` defines :meth:`__new__`, so + Note that :exc:`BaseExceptionGroup` defines :meth:`~object.__new__`, so subclasses that need a different constructor signature need to - override that rather than :meth:`__init__`. For example, the following + override that rather than :meth:`~object.__init__`. For example, the following defines an exception group subclass which accepts an exit_code and and constructs the group's message from it. :: diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index f64dfeb5e081c7..4067d7912b88b2 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -1,5 +1,5 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= +:mod:`!faulthandler` --- Dump the Python traceback +================================================== .. module:: faulthandler :synopsis: Dump the Python traceback. @@ -10,14 +10,15 @@ This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also +install fault handlers for the :const:`~signal.SIGSEGV`, +:const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS`, and +:const:`~signal.SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment variable or by using the :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the +if the :c:func:`!sigaltstack` function is available. This allows it to dump the traceback even on a stack overflow. The fault handler is called on catastrophic cases and therefore can only use @@ -70,8 +71,9 @@ Fault handler state .. function:: enable(file=sys.stderr, all_threads=True) - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` + Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, + :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` + and :const:`~signal.SIGILL` signals to dump the Python traceback. If *all_threads* is ``True``, produce tracebacks for every running thread. Otherwise, dump only the current thread. @@ -106,8 +108,8 @@ Dumping the tracebacks after a timeout Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any + :c:func:`!_exit` with status=1 after dumping the tracebacks. (Note + :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a sub-second resolution. @@ -118,12 +120,12 @@ Dumping the tracebacks after a timeout This function is implemented using a watchdog thread. - .. versionchanged:: 3.7 - This function is now always available. - .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.7 + This function is now always available. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 969a79fa873395..7bd64e43dd5bfe 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,5 +1,5 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= +:mod:`!fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================== .. module:: fcntl :platform: Unix @@ -13,12 +13,12 @@ ---------------- -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. +This module performs file and I/O control on file descriptors. It is an +interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. +See the :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages +for full details. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not WASI. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by @@ -31,26 +31,26 @@ descriptor. raise an :exc:`OSError`. .. versionchanged:: 3.8 - The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and + The :mod:`!fcntl` module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file descriptors. .. versionchanged:: 3.9 - On macOS, the fcntl module exposes the ``F_GETPATH`` constant, which obtains - the path of a file from a file descriptor. - On Linux(>=3.15), the fcntl module exposes the ``F_OFD_GETLK``, ``F_OFD_SETLK`` - and ``F_OFD_SETLKW`` constants, which are used when working with open file - description locks. + On macOS, the :mod:`!fcntl` module exposes the ``F_GETPATH`` constant, + which obtains the path of a file from a file descriptor. + On Linux(>=3.15), the :mod:`!fcntl` module exposes the ``F_OFD_GETLK``, + ``F_OFD_SETLK`` and ``F_OFD_SETLKW`` constants, which are used when working + with open file description locks. .. versionchanged:: 3.10 - On Linux >= 2.6.11, the fcntl module exposes the ``F_GETPIPE_SZ`` and + On Linux >= 2.6.11, the :mod:`!fcntl` module exposes the ``F_GETPIPE_SZ`` and ``F_SETPIPE_SZ`` constants, which allow to check and modify a pipe's size respectively. .. versionchanged:: 3.11 - On FreeBSD, the fcntl module exposes the ``F_DUP2FD`` and ``F_DUP2FD_CLOEXEC`` - constants, which allow to duplicate a file descriptor, the latter setting - ``FD_CLOEXEC`` flag in addition. + On FreeBSD, the :mod:`!fcntl` module exposes the ``F_DUP2FD`` and + ``F_DUP2FD_CLOEXEC`` constants, which allow to duplicate a file descriptor, + the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and @@ -58,6 +58,27 @@ descriptor. another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". +.. versionchanged:: 3.13 + On Linux >= 2.6.32, the :mod:`!fcntl` module exposes the + ``F_GETOWN_EX``, ``F_SETOWN_EX``, ``F_OWNER_TID``, ``F_OWNER_PID``, ``F_OWNER_PGRP`` constants, which allow to direct I/O availability signals + to a specific thread, process, or process group. + On Linux >= 4.13, the :mod:`!fcntl` module exposes the + ``F_GET_RW_HINT``, ``F_SET_RW_HINT``, ``F_GET_FILE_RW_HINT``, + ``F_SET_FILE_RW_HINT``, and ``RWH_WRITE_LIFE_*`` constants, which allow + to inform the kernel about the relative expected lifetime of writes on + a given inode or via a particular open file description. + On Linux >= 5.1 and NetBSD, the :mod:`!fcntl` module exposes the + ``F_SEAL_FUTURE_WRITE`` constant for use with ``F_ADD_SEALS`` and + ``F_GET_SEALS`` operations. + On FreeBSD, the :mod:`!fcntl` module exposes the ``F_READAHEAD``, ``F_ISUNIONSTACK``, and ``F_KINFO`` constants. + On macOS and FreeBSD, the :mod:`!fcntl` module exposes the ``F_RDAHEAD`` + constant. + On NetBSD and AIX, the :mod:`!fcntl` module exposes the ``F_CLOSEM`` + constant. + On NetBSD, the :mod:`!fcntl` module exposes the ``F_MAXFD`` constant. + On macOS and NetBSD, the :mod:`!fcntl` module exposes the ``F_GETNOSIGPIPE`` + and ``F_SETNOSIGPIPE`` constant. + The module defines the following functions: @@ -80,7 +101,7 @@ The module defines the following functions: most likely to result in a segmentation violation or a more subtle data corruption. - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + If the :c:func:`fcntl` call fails, an :exc:`OSError` is raised. .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl @@ -118,7 +139,7 @@ The module defines the following functions: buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back into the supplied buffer. - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. + If the :c:func:`ioctl` call fails, an :exc:`OSError` exception is raised. An example:: @@ -143,7 +164,7 @@ The module defines the following functions: :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + If the :c:func:`flock` call fails, an :exc:`OSError` exception is raised. .. audit-event:: fcntl.flock fd,operation fcntl.flock @@ -155,17 +176,28 @@ The module defines the following functions: method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock + .. data:: LOCK_UN + + Release an existing lock. + + .. data:: LOCK_SH + + Acquire a shared lock. + + .. data:: LOCK_EX + + Acquire an exclusive lock. + + .. data:: LOCK_NB + + Bitwise OR with any of the other three ``LOCK_*`` constants to make + the request non-blocking. - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an + If :const:`!LOCK_NB` is used and the lock cannot be acquired, an :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the + attribute set to :const:`~errno.EACCES` or :const:`~errno.EAGAIN` (depending on the operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a + systems, :const:`!LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. *len* is the number of bytes to lock, *start* is the byte offset at diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index dfe4b7c59fd578..282d0e0d8db5cf 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -1,5 +1,5 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= +:mod:`!filecmp` --- File and Directory Comparisons +================================================== .. module:: filecmp :synopsis: Compare files efficiently. @@ -70,7 +70,7 @@ The :mod:`filecmp` module defines the following functions: The :class:`dircmp` class ------------------------- -.. class:: dircmp(a, b, ignore=None, hide=None) +.. class:: dircmp(a, b, ignore=None, hide=None, *, shallow=True) Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to @@ -78,7 +78,12 @@ The :class:`dircmp` class defaults to ``[os.curdir, os.pardir]``. The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. + as described for :func:`filecmp.cmp` by default using the *shallow* + parameter. + + .. versionchanged:: 3.13 + + Added the *shallow* parameter. The :class:`dircmp` class provides the following methods: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index f93e9a58791eeb..8f32b11e565365 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -1,5 +1,5 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== +:mod:`!fileinput` --- Iterate over lines from multiple input streams +==================================================================== .. module:: fileinput :synopsis: Loop over standard input or a list of files. @@ -47,7 +47,7 @@ Lines are returned with any newlines intact, which means that the last line in a file may not have one. You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The +*openhook* parameter to :func:`fileinput.input` or :func:`FileInput`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* are specified, they will be passed to the hook as additional keyword arguments. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index aed8991d44772f..fda44923f204fc 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -1,5 +1,5 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= +:mod:`!fnmatch` --- Unix filename pattern matching +================================================== .. module:: fnmatch :synopsis: Unix shell style filename pattern matching. @@ -50,10 +50,10 @@ Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used t cache the compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. -.. function:: fnmatch(filename, pattern) +.. function:: fnmatch(name, pat) - Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. Both parameters are case-normalized + Test whether the filename string *name* matches the pattern string *pat*, + returning ``True`` or ``False``. Both parameters are case-normalized using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a case-sensitive comparison, regardless of whether that's standard for the operating system. @@ -69,22 +69,24 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, print(file) -.. function:: fnmatchcase(filename, pattern) +.. function:: fnmatchcase(name, pat) - Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive and does not apply - :func:`os.path.normcase`. + Test whether the filename string *name* matches the pattern string *pat*, + returning ``True`` or ``False``; + the comparison is case-sensitive and does not apply :func:`os.path.normcase`. -.. function:: filter(names, pattern) +.. function:: filter(names, pat) - Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as - ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. + Construct a list from those elements of the :term:`iterable` *names* + that match pattern *pat*. + It is the same as ``[n for n in names if fnmatch(n, pat)]``, + but implemented more efficiently. -.. function:: translate(pattern) +.. function:: translate(pat) - Return the shell-style *pattern* converted to a regular expression for + Return the shell-style pattern *pat* converted to a regular expression for using with :func:`re.match`. Example: diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 509c63686f5a7f..552d6030b1ceda 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -1,5 +1,5 @@ -:mod:`fractions` --- Rational numbers -===================================== +:mod:`!fractions` --- Rational numbers +====================================== .. module:: fractions :synopsis: Rational numbers. @@ -106,6 +106,10 @@ another rational number, or from a string. presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` and ``"%""``. + .. versionchanged:: 3.13 + Formatting of :class:`Fraction` instances without a presentation type + now supports fill, alignment, sign handling, minimum width and grouping. + .. attribute:: numerator Numerator of the Fraction in lowest term. @@ -201,17 +205,36 @@ another rational number, or from a string. .. method:: __format__(format_spec, /) - Provides support for float-style formatting of :class:`Fraction` - instances via the :meth:`str.format` method, the :func:`format` built-in - function, or :ref:`Formatted string literals <f-strings>`. The - presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` - and ``"%"`` are supported. For these presentation types, formatting for a - :class:`Fraction` object ``x`` follows the rules outlined for - the :class:`float` type in the :ref:`formatspec` section. + Provides support for formatting of :class:`Fraction` instances via the + :meth:`str.format` method, the :func:`format` built-in function, or + :ref:`Formatted string literals <f-strings>`. + + If the ``format_spec`` format specification string does not end with one + of the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the general rules for fill, + alignment, sign handling, minimum width, and grouping as described in the + :ref:`format specification mini-language <formatspec>`. The "alternate + form" flag ``'#'`` is supported: if present, it forces the output string + to always include an explicit denominator, even when the value being + formatted is an exact integer. The zero-fill flag ``'0'`` is not + supported. + + If the ``format_spec`` format specification string ends with one of + the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the rules outlined for the + :class:`float` type in the :ref:`formatspec` section. Here are some examples:: >>> from fractions import Fraction + >>> format(Fraction(103993, 33102), '_') + '103_993/33_102' + >>> format(Fraction(1, 7), '.^+10') + '...+1/7...' + >>> format(Fraction(3, 1), '') + '3' + >>> format(Fraction(3, 1), '#') + '3/1' >>> format(Fraction(1, 7), '.40g') '0.1428571428571428571428571428571428571429' >>> format(Fraction('1234567.855'), '_.2f') diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index d1fe6414ea020c..bb15322067245e 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -1,5 +1,5 @@ -:mod:`ftplib` --- FTP protocol client -===================================== +:mod:`!ftplib` --- FTP protocol client +====================================== .. module:: ftplib :synopsis: FTP protocol client (requires sockets). @@ -45,19 +45,73 @@ Here's a sample session using the :mod:`ftplib` module:: '221 Goodbye.' -The module defines the following items: +.. _ftplib-reference: -.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None, *, encoding='utf-8') +Reference +--------- - Return a new instance of the :class:`FTP` class. When *host* is given, the - method call ``connect(host)`` is made. When *user* is given, additionally - the method call ``login(user, passwd, acct)`` is made (where *passwd* and - *acct* default to the empty string when not given). The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if is not specified, the global default timeout setting - will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket - to bind to as its source address before connecting. The *encoding* parameter - specifies the encoding for directories and filenames. +.. _ftp-objects: + +FTP objects +^^^^^^^^^^^ + +.. Use substitutions for some param docs so we don't need to repeat them + in multiple places. + +.. |param_doc_user| replace:: + The username to log in with (default: ``'anonymous'``). + +.. |param_doc_passwd| replace:: + The password to use when logging in. + If not given, and if *passwd* is the empty string or ``"-"``, + a password will be automatically generated. + +.. Ideally, we'd like to use the :rfc: directive, but Sphinx will not allow it. + +.. |param_doc_acct| replace:: + Account information to be used for the ``ACCT`` FTP command. + Few systems implement this. + See `RFC-959 <https://datatracker.ietf.org/doc/html/rfc959.html>`__ + for more details. + +.. |param_doc_source_address| replace:: + A 2-tuple ``(host, port)`` for the socket to bind to as its + source address before connecting. + +.. |param_doc_encoding| replace:: + The encoding for directories and filenames (default: ``'utf-8'``). + +.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, \ + source_address=None, *, encoding='utf-8') + + Return a new instance of the :class:`FTP` class. + + :param str host: + The hostname to connect to. + If given, :code:`connect(host)` is implicitly called by the constructor. + + :param str user: + |param_doc_user| + If given, :code:`login(host, passwd, acct)` is implicitly called + by the constructor. + + :param str passwd: + |param_doc_passwd| + + :param str acct: + |param_doc_acct| + + :param timeout: + A timeout in seconds for blocking operations like :meth:`connect` + (default: the global default timeout setting). + :type timeout: float | None + + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None + + :param str encoding: + |param_doc_encoding| The :class:`FTP` class supports the :keyword:`with` statement, e.g.: @@ -85,376 +139,460 @@ The module defines the following items: The *encoding* parameter was added, and the default was changed from Latin-1 to UTF-8 to follow :rfc:`2640`. -.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, - timeout=None, source_address=None, encoding='utf-8') + Several :class:`!FTP` methods are available in two flavors: + one for handling text files and another for binary files. + The methods are named for the command which is used followed by + ``lines`` for the text version or ``binary`` for the binary version. - A :class:`FTP` subclass which adds TLS support to FTP as described in - :rfc:`4217`. - Connect as usual to port 21 implicitly securing the FTP control connection - before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. + :class:`FTP` instances have the following methods: - .. versionadded:: 3.2 + .. method:: FTP.set_debuglevel(level) - .. versionchanged:: 3.3 - *source_address* parameter was added. + Set the instance's debugging level as an :class:`int`. + This controls the amount of debugging output printed. + The debug levels are: - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). + * ``0`` (default): No debug output. + * ``1``: Produce a moderate amount of debug output, + generally a single line per request. + * ``2`` or higher: Produce the maximum amount of debugging output, + logging each line sent and received on the control connection. - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - The *encoding* parameter was added, and the default was changed from - Latin-1 to UTF-8 to follow :rfc:`2640`. + .. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) - .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + Connect to the given host and port. + This function should be called only once for each instance; + it should not be called if a *host* argument was given + when the :class:`FTP` instance was created. + All other :class:`!FTP` methods can only be called + after a connection has successfully been made. - Here's a sample session using the :class:`FTP_TLS` class:: + :param str host: + The host to connect to. - >>> ftps = FTP_TLS('ftp.pureftpd.org') - >>> ftps.login() - '230 Anonymous user logged in' - >>> ftps.prot_p() - '200 Data protection level set to "private"' - >>> ftps.nlst() - ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] + :param int port: + The TCP port to connect to (default: ``21``, + as specified by the FTP protocol specification). + It is rarely needed to specify a different port number. + :param timeout: + A timeout in seconds for the connection attempt + (default: the global default timeout setting). + :type timeout: float | None -.. exception:: error_reply + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None - Exception raised when an unexpected reply is received from the server. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 + *source_address* parameter was added. -.. exception:: error_temp - Exception raised when an error code signifying a temporary error (response - codes in the range 400--499) is received. + .. method:: FTP.getwelcome() + Return the welcome message sent by the server in reply to the initial + connection. (This message sometimes contains disclaimers or help information + that may be relevant to the user.) -.. exception:: error_perm - Exception raised when an error code signifying a permanent error (response - codes in the range 500--599) is received. + .. method:: FTP.login(user='anonymous', passwd='', acct='') + Log on to the connected FTP server. + This function should be called only once for each instance, + after a connection has been established; + it should not be called if the *host* and *user* arguments were given + when the :class:`FTP` instance was created. + Most FTP commands are only allowed after the client has logged in. -.. exception:: error_proto + :param str user: + |param_doc_user| - Exception raised when a reply is received from the server that does not fit - the response specifications of the File Transfer Protocol, i.e. begin with a - digit in the range 1--5. + :param str passwd: + |param_doc_passwd| + :param str acct: + |param_doc_acct| -.. data:: all_errors - The set of all exceptions (as a tuple) that methods of :class:`FTP` - instances may raise as a result of problems with the FTP connection (as - opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. + .. method:: FTP.abort() + Abort a file transfer that is in progress. Using this does not always work, but + it's worth a try. -.. seealso:: - Module :mod:`netrc` - Parser for the :file:`.netrc` file format. The file :file:`.netrc` is - typically used by FTP clients to load user authentication information - before prompting the user. + .. method:: FTP.sendcmd(cmd) + Send a simple command string to the server and return the response string. -.. _ftp-objects: + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd -FTP Objects ------------ -Several methods are available in two flavors: one for handling text files and -another for binary files. These are named for the command which is used -followed by ``lines`` for the text version or ``binary`` for the binary version. + .. method:: FTP.voidcmd(cmd) -:class:`FTP` instances have the following methods: + Send a simple command string to the server and handle the response. Return + the response string if the response code corresponds to success (codes in + the range 200--299). Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd -.. method:: FTP.set_debuglevel(level) - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) + Retrieve a file in binary transfer mode. -.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) + :param str cmd: + An appropriate ``RETR`` command: :samp:`"RETR {filename}"`. - Connect to the given host and port. The default port number is ``21``, as - specified by the FTP protocol specification. It is rarely needed to specify a - different port number. This function should be called only once for each - instance; it should not be called at all if a host was given when the instance - was created. All other methods can only be used after a connection has been - made. - The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If no *timeout* is passed, the global default timeout - setting will be used. - *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as - its source address before connecting. + :param callback: + A single parameter callable that is called + for each block of data received, + with its single argument being the data as :class:`bytes`. + :type callback: :term:`callable` - .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + :param int blocksize: + The maximum chunk size to read on the low-level + :class:`~socket.socket` object created to do the actual transfer. + This also corresponds to the largest size of data + that will be passed to *callback*. + Defaults to ``8192``. - .. versionchanged:: 3.3 - *source_address* parameter was added. + :param int rest: + A ``REST`` command to be sent to the server. + See the documentation for the *rest* parameter of the :meth:`transfercmd` method. -.. method:: FTP.getwelcome() + .. method:: FTP.retrlines(cmd, callback=None) - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) + Retrieve a file or directory listing in the encoding specified by the + *encoding* parameter at initialization. + *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or + a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). + ``LIST`` retrieves a list of files and information about those files. + ``NLST`` retrieves a list of file names. + The *callback* function is called for each line with a string argument + containing the line with the trailing CRLF stripped. The default *callback* + prints the line to :data:`sys.stdout`. -.. method:: FTP.login(user='anonymous', passwd='', acct='') + .. method:: FTP.set_pasv(val) - Log in as the given *user*. The *passwd* and *acct* parameters are optional and - default to the empty string. If no *user* is specified, it defaults to - ``'anonymous'``. If *user* is ``'anonymous'``, the default *passwd* is - ``'anonymous@'``. This function should be called only once for each instance, - after a connection has been established; it should not be called at all if a - host and user were given when the instance was created. Most FTP commands are - only allowed after the client has logged in. The *acct* parameter supplies - "accounting information"; few systems implement this. + Enable "passive" mode if *val* is true, otherwise disable passive mode. + Passive mode is on by default. -.. method:: FTP.abort() + .. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) - Abort a file transfer that is in progress. Using this does not always work, but - it's worth a try. + Store a file in binary transfer mode. + :param str cmd: + An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. -.. method:: FTP.sendcmd(cmd) + :param fp: + A file object (opened in binary mode) which is read until EOF, + using its :meth:`~io.RawIOBase.read` method in blocks of size *blocksize* + to provide the data to be stored. + :type fp: :term:`file object` - Send a simple command string to the server and return the response string. + :param int blocksize: + The read block size. + Defaults to ``8192``. - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + :param callback: + A single parameter callable that is called + for each block of data sent, + with its single argument being the data as :class:`bytes`. + :type callback: :term:`callable` + :param int rest: + A ``REST`` command to be sent to the server. + See the documentation for the *rest* parameter of the :meth:`transfercmd` method. -.. method:: FTP.voidcmd(cmd) + .. versionchanged:: 3.2 + The *rest* parameter was added. - Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.storlines(cmd, fp, callback=None) + Store a file in line mode. *cmd* should be an appropriate + ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the + :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` + method to provide the data to be stored. *callback* is an optional single + parameter callable that is called on each line after it is sent. -.. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) - Retrieve a file in binary transfer mode. *cmd* should be an appropriate - ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single bytes argument giving the data - block. The optional *blocksize* argument specifies the maximum chunk size to - read on the low-level socket object created to do the actual transfer (which - will also be the largest size of the data blocks passed to *callback*). A - reasonable default is chosen. *rest* means the same thing as in the - :meth:`transfercmd` method. + .. method:: FTP.transfercmd(cmd, rest=None) + Initiate a transfer over the data connection. If the transfer is active, send an + ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and + accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` + command, connect to it, and start the transfer command. Either way, return the + socket for the connection. -.. method:: FTP.retrlines(cmd, callback=None) + If optional *rest* is given, a ``REST`` command is sent to the server, passing + *rest* as an argument. *rest* is usually a byte offset into the requested file, + telling the server to restart sending the file's bytes at the requested offset, + skipping over the initial bytes. Note however that the :meth:`transfercmd` + method converts *rest* to a string with the *encoding* parameter specified + at initialization, but no check is performed on the string's contents. If the + server does not recognize the ``REST`` command, an :exc:`error_reply` exception + will be raised. If this happens, simply call :meth:`transfercmd` without a + *rest* argument. - Retrieve a file or directory listing in the encoding specified by the - *encoding* parameter at initialization. - *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or - a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). - ``LIST`` retrieves a list of files and information about those files. - ``NLST`` retrieves a list of file names. - The *callback* function is called for each line with a string argument - containing the line with the trailing CRLF stripped. The default *callback* - prints the line to ``sys.stdout``. + .. method:: FTP.ntransfercmd(cmd, rest=None) -.. method:: FTP.set_pasv(val) + Like :meth:`transfercmd`, but returns a tuple of the data connection and the + expected size of the data. If the expected size could not be computed, ``None`` + will be returned as the expected size. *cmd* and *rest* means the same thing as + in :meth:`transfercmd`. - Enable "passive" mode if *val* is true, otherwise disable passive mode. - Passive mode is on by default. + .. method:: FTP.mlsd(path="", facts=[]) -.. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) + List a directory in a standardized format by using ``MLSD`` command + (:rfc:`3659`). If *path* is omitted the current directory is assumed. + *facts* is a list of strings representing the type of information desired + (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a + tuple of two elements for every file found in path. First element is the + file name, the second one is a dictionary containing facts about the file + name. Content of this dictionary might be limited by the *facts* argument + but server is not guaranteed to return all requested facts. - Store a file in binary transfer mode. *cmd* should be an appropriate - ``STOR`` command: ``"STOR filename"``. *fp* is a :term:`file object` - (opened in binary mode) which is read until EOF using its :meth:`~io.IOBase.read` - method in blocks of size *blocksize* to provide the data to be stored. - The *blocksize* argument defaults to 8192. *callback* is an optional single - parameter callable that is called on each block of data after it is sent. - *rest* means the same thing as in the :meth:`transfercmd` method. + .. versionadded:: 3.3 - .. versionchanged:: 3.2 - *rest* parameter added. + .. method:: FTP.nlst(argument[, ...]) + + Return a list of file names as returned by the ``NLST`` command. The + optional *argument* is a directory to list (default is the current server + directory). Multiple arguments can be used to pass non-standard options to + the ``NLST`` command. -.. method:: FTP.storlines(cmd, fp, callback=None) + .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - Store a file in line mode. *cmd* should be an appropriate - ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the - :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` - method to provide the data to be stored. *callback* is an optional single - parameter callable that is called on each line after it is sent. + .. method:: FTP.dir(argument[, ...]) -.. method:: FTP.transfercmd(cmd, rest=None) + Produce a directory listing as returned by the ``LIST`` command, printing it to + standard output. The optional *argument* is a directory to list (default is the + current server directory). Multiple arguments can be used to pass non-standard + options to the ``LIST`` command. If the last argument is a function, it is used + as a *callback* function as for :meth:`retrlines`; the default prints to + :data:`sys.stdout`. This method returns ``None``. - Initiate a transfer over the data connection. If the transfer is active, send an - ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and - accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` - command, connect to it, and start the transfer command. Either way, return the - socket for the connection. + .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - If optional *rest* is given, a ``REST`` command is sent to the server, passing - *rest* as an argument. *rest* is usually a byte offset into the requested file, - telling the server to restart sending the file's bytes at the requested offset, - skipping over the initial bytes. Note however that the :meth:`transfercmd` - method converts *rest* to a string with the *encoding* parameter specified - at initialization, but no check is performed on the string's contents. If the - server does not recognize the ``REST`` command, an :exc:`error_reply` exception - will be raised. If this happens, simply call :meth:`transfercmd` without a - *rest* argument. + .. method:: FTP.rename(fromname, toname) -.. method:: FTP.ntransfercmd(cmd, rest=None) + Rename file *fromname* on the server to *toname*. - Like :meth:`transfercmd`, but returns a tuple of the data connection and the - expected size of the data. If the expected size could not be computed, ``None`` - will be returned as the expected size. *cmd* and *rest* means the same thing as - in :meth:`transfercmd`. + .. method:: FTP.delete(filename) -.. method:: FTP.mlsd(path="", facts=[]) + Remove the file named *filename* from the server. If successful, returns the + text of the response, otherwise raises :exc:`error_perm` on permission errors or + :exc:`error_reply` on other errors. - List a directory in a standardized format by using ``MLSD`` command - (:rfc:`3659`). If *path* is omitted the current directory is assumed. - *facts* is a list of strings representing the type of information desired - (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a - tuple of two elements for every file found in path. First element is the - file name, the second one is a dictionary containing facts about the file - name. Content of this dictionary might be limited by the *facts* argument - but server is not guaranteed to return all requested facts. - .. versionadded:: 3.3 + .. method:: FTP.cwd(pathname) + Set the current directory on the server. -.. method:: FTP.nlst(argument[, ...]) - Return a list of file names as returned by the ``NLST`` command. The - optional *argument* is a directory to list (default is the current server - directory). Multiple arguments can be used to pass non-standard options to - the ``NLST`` command. + .. method:: FTP.mkd(pathname) - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + Create a new directory on the server. -.. method:: FTP.dir(argument[, ...]) + .. method:: FTP.pwd() - Produce a directory listing as returned by the ``LIST`` command, printing it to - standard output. The optional *argument* is a directory to list (default is the - current server directory). Multiple arguments can be used to pass non-standard - options to the ``LIST`` command. If the last argument is a function, it is used - as a *callback* function as for :meth:`retrlines`; the default prints to - ``sys.stdout``. This method returns ``None``. + Return the pathname of the current directory on the server. - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + .. method:: FTP.rmd(dirname) -.. method:: FTP.rename(fromname, toname) + Remove the directory named *dirname* on the server. - Rename file *fromname* on the server to *toname*. + .. method:: FTP.size(filename) -.. method:: FTP.delete(filename) + Request the size of the file named *filename* on the server. On success, the + size of the file is returned as an integer, otherwise ``None`` is returned. + Note that the ``SIZE`` command is not standardized, but is supported by many + common server implementations. - Remove the file named *filename* from the server. If successful, returns the - text of the response, otherwise raises :exc:`error_perm` on permission errors or - :exc:`error_reply` on other errors. + .. method:: FTP.quit() -.. method:: FTP.cwd(pathname) + Send a ``QUIT`` command to the server and close the connection. This is the + "polite" way to close a connection, but it may raise an exception if the server + responds with an error to the ``QUIT`` command. This implies a call to the + :meth:`close` method which renders the :class:`FTP` instance useless for + subsequent calls (see below). - Set the current directory on the server. + .. method:: FTP.close() -.. method:: FTP.mkd(pathname) + Close the connection unilaterally. This should not be applied to an already + closed connection such as after a successful call to :meth:`~FTP.quit`. + After this call the :class:`FTP` instance should not be used any more (after + a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the + connection by issuing another :meth:`login` method). - Create a new directory on the server. +FTP_TLS objects +^^^^^^^^^^^^^^^ -.. method:: FTP.pwd() +.. class:: FTP_TLS(host='', user='', passwd='', acct='', *, context=None, \ + timeout=None, source_address=None, encoding='utf-8') - Return the pathname of the current directory on the server. + An :class:`FTP` subclass which adds TLS support to FTP as described in + :rfc:`4217`. + Connect to port 21 implicitly securing the FTP control connection + before authenticating. + .. note:: + The user must explicitly secure the data connection + by calling the :meth:`prot_p` method. -.. method:: FTP.rmd(dirname) + :param str host: + The hostname to connect to. + If given, :code:`connect(host)` is implicitly called by the constructor. - Remove the directory named *dirname* on the server. + :param str user: + |param_doc_user| + If given, :code:`login(host, passwd, acct)` is implicitly called + by the constructor. + :param str passwd: + |param_doc_passwd| -.. method:: FTP.size(filename) + :param str acct: + |param_doc_acct| - Request the size of the file named *filename* on the server. On success, the - size of the file is returned as an integer, otherwise ``None`` is returned. - Note that the ``SIZE`` command is not standardized, but is supported by many - common server implementations. + :param context: + An SSL context object which allows bundling SSL configuration options, + certificates and private keys into a single, potentially long-lived, + structure. + Please read :ref:`ssl-security` for best practices. + :type context: :class:`ssl.SSLContext` + :param timeout: + A timeout in seconds for blocking operations like :meth:`~FTP.connect` + (default: the global default timeout setting). + :type timeout: float | None -.. method:: FTP.quit() + :param source_address: + |param_doc_source_address| + :type source_address: tuple | None - Send a ``QUIT`` command to the server and close the connection. This is the - "polite" way to close a connection, but it may raise an exception if the server - responds with an error to the ``QUIT`` command. This implies a call to the - :meth:`close` method which renders the :class:`FTP` instance useless for - subsequent calls (see below). + :param str encoding: + |param_doc_encoding| + .. versionadded:: 3.2 -.. method:: FTP.close() + .. versionchanged:: 3.3 + Added the *source_address* parameter. - Close the connection unilaterally. This should not be applied to an already - closed connection such as after a successful call to :meth:`~FTP.quit`. - After this call the :class:`FTP` instance should not be used any more (after - a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the - connection by issuing another :meth:`login` method). + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :const:`ssl.HAS_SNI`). + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + The *encoding* parameter was added, and the default was changed from + Latin-1 to UTF-8 to follow :rfc:`2640`. -FTP_TLS Objects ---------------- + .. versionchanged:: 3.12 + The deprecated *keyfile* and *certfile* parameters have been removed. -:class:`FTP_TLS` class inherits from :class:`FTP`, defining these additional objects: + Here's a sample session using the :class:`FTP_TLS` class:: -.. attribute:: FTP_TLS.ssl_version + >>> ftps = FTP_TLS('ftp.pureftpd.org') + >>> ftps.login() + '230 Anonymous user logged in' + >>> ftps.prot_p() + '200 Data protection level set to "private"' + >>> ftps.nlst() + ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). + :class:`!FTP_TLS` class inherits from :class:`FTP`, + defining these additional methods and attributes: -.. method:: FTP_TLS.auth() + .. attribute:: FTP_TLS.ssl_version - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. + The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). + .. method:: FTP_TLS.auth() + + Set up a secure control connection by using TLS or SSL, depending on what + is specified in the :attr:`ssl_version` attribute. -.. method:: FTP_TLS.ccc() + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :const:`ssl.HAS_SNI`). - Revert control channel back to plaintext. This can be useful to take - advantage of firewalls that know how to handle NAT with non-secure FTP - without opening fixed ports. + .. method:: FTP_TLS.ccc() - .. versionadded:: 3.3 + Revert control channel back to plaintext. This can be useful to take + advantage of firewalls that know how to handle NAT with non-secure FTP + without opening fixed ports. -.. method:: FTP_TLS.prot_p() + .. versionadded:: 3.3 - Set up secure data connection. + .. method:: FTP_TLS.prot_p() -.. method:: FTP_TLS.prot_c() + Set up secure data connection. + + .. method:: FTP_TLS.prot_c() + + Set up clear text data connection. + + +Module variables +^^^^^^^^^^^^^^^^ + +.. exception:: error_reply + + Exception raised when an unexpected reply is received from the server. - Set up clear text data connection. + +.. exception:: error_temp + + Exception raised when an error code signifying a temporary error (response + codes in the range 400--499) is received. + + +.. exception:: error_perm + + Exception raised when an error code signifying a permanent error (response + codes in the range 500--599) is received. + + +.. exception:: error_proto + + Exception raised when a reply is received from the server that does not fit + the response specifications of the File Transfer Protocol, i.e. begin with a + digit in the range 1--5. + + +.. data:: all_errors + + The set of all exceptions (as a tuple) that methods of :class:`FTP` + instances may raise as a result of problems with the FTP connection (as + opposed to programming errors made by the caller). This set includes the + four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. + + +.. seealso:: + + Module :mod:`netrc` + Parser for the :file:`.netrc` file format. The file :file:`.netrc` is + typically used by FTP clients to load user authentication information + before prompting the user. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 624ff954c663a0..26ab086f914d7d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -57,7 +57,8 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing :meth:`__abs__`. + integer, a floating point number, or an object implementing + :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -140,10 +141,11 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool(x=False) +.. class:: bool(object=False, /) - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure <truth>`. If *x* is false + Return a Boolean value, i.e. one of ``True`` or ``False``. The argument + is converted using the standard :ref:`truth testing procedure <truth>`. + If the argument is false or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and @@ -152,7 +154,7 @@ are always available. They are listed here in alphabetical order. .. index:: pair: Boolean; type .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. function:: breakpoint(*args, **kws) @@ -235,7 +237,7 @@ are always available. They are listed here in alphabetical order. :const:`False` if not. If this returns ``True``, it is still possible that a call fails, but if it is ``False``, calling *object* will never succeed. Note that classes are callable (calling a class returns a new instance); - instances are callable if their class has a :meth:`__call__` method. + instances are callable if their class has a :meth:`~object.__call__` method. .. versionadded:: 3.2 This function was first removed in Python 3.0 and then brought back @@ -285,7 +287,7 @@ are always available. They are listed here in alphabetical order. ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and have a new ``__wrapped__`` attribute. - .. versionchanged:: 3.11 + .. deprecated-removed:: 3.11 3.13 Class methods can no longer wrap other :term:`descriptors <descriptor>` such as :func:`property`. @@ -370,31 +372,75 @@ are always available. They are listed here in alphabetical order. support for top-level ``await``, ``async for``, and ``async with``. -.. class:: complex(real=0, imag=0) - complex(string) +.. class:: complex(number=0, /) + complex(string, /) + complex(real=0, imag=0) + + Convert a single string or number to a complex number, or create a + complex number from real and imaginary parts. + + Examples: + + .. doctest:: + + >>> complex('+1.23') + (1.23+0j) + >>> complex('-4.5j') + -4.5j + >>> complex('-1.23+4.5j') + (-1.23+4.5j) + >>> complex('\t( -1.23+4.5J )\n') + (-1.23+4.5j) + >>> complex('-Infinity+NaNj') + (-inf+nanj) + >>> complex(1.23) + (1.23+0j) + >>> complex(imag=-4.5) + -4.5j + >>> complex(-1.23, 4.5) + (-1.23+4.5j) + + If the argument is a string, it must contain either a real part (in the + same format as for :func:`float`) or an imaginary part (in the same + format but with a ``'j'`` or ``'J'`` suffix), or both real and imaginary + parts (the sign of the imaginary part is mandatory in this case). + The string can optionally be surrounded by whitespaces and the round + parentheses ``'('`` and ``')'``, which are ignored. + The string must not contain whitespace between ``'+'``, ``'-'``, the + ``'j'`` or ``'J'`` suffix, and the decimal number. + For example, ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises + :exc:`ValueError`. + More precisely, the input must conform to the :token:`~float:complexvalue` + production rule in the following grammar, after parentheses and leading and + trailing whitespace characters are removed: - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. + .. productionlist:: float + complexvalue: `floatvalue` | + : `floatvalue` ("j" | "J") | + : `floatvalue` `sign` `absfloatvalue` ("j" | "J") + If the argument is a number, the constructor serves as a numeric + conversion like :class:`int` and :class:`float`. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back - to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + ``x.__complex__()``. + If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. - See also :meth:`complex.from_number` which only accepts a single numeric argument. + If two arguments are provided or keyword arguments are used, each argument + may be any numeric type (including complex). + If both arguments are real numbers, return a complex number with the real + component *real* and the imaginary component *imag*. + If both arguments are complex numbers, return a complex number with the real + component ``real.real-imag.imag`` and the imaginary component + ``real.imag+imag.real``. + If one of arguments is a real number, only its real component is used in + the above expressions. - .. note:: + See also :meth:`complex.from_number` which only accepts a single numeric argument. - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. + If all arguments are omitted, returns ``0j``. The complex type is described in :ref:`typesnumeric`. @@ -405,6 +451,10 @@ are always available. They are listed here in alphabetical order. Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and :meth:`~object.__float__` are not defined. + .. deprecated:: 3.14 + Passing a complex number as the *real* or *imag* argument is now + deprecated; it should only be passed as a single positional argument. + .. function:: delattr(object, name) @@ -434,15 +484,18 @@ are always available. They are listed here in alphabetical order. Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. - If the object has a method named :meth:`__dir__`, this method will be called and + If the object has a method named :meth:`~object.__dir__`, + this method will be called and must return the list of attributes. This allows objects that implement a custom - :func:`__getattr__` or :func:`__getattribute__` function to customize the way + :func:`~object.__getattr__` or :func:`~object.__getattribute__` function + to customize the way :func:`dir` reports their attributes. - If the object does not provide :meth:`__dir__`, the function tries its best to - gather information from the object's :attr:`~object.__dict__` attribute, if defined, and + If the object does not provide :meth:`~object.__dir__`, + the function tries its best to gather information from the object's + :attr:`~object.__dict__` attribute, if defined, and from its type object. The resulting list is not necessarily complete and may - be inaccurate when the object has a custom :func:`__getattr__`. + be inaccurate when the object has a custom :func:`~object.__getattr__`. The default :func:`dir` mechanism behaves differently with different types of objects, as it attempts to produce the most relevant, rather than complete, @@ -522,29 +575,40 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(source, /, globals=None, locals=None) + + :param source: + A Python expression. + :type source: :class:`str` | :ref:`code object <code-objects>` + + :param globals: + The global namespace (default: ``None``). + :type globals: :class:`dict` | ``None`` - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + :param locals: + The local namespace (default: ``None``). + :type locals: :term:`mapping` | ``None`` + + :returns: The result of the evaluated expression. + :raises: Syntax errors are reported as exceptions. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is + mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *expression* is parsed. That way you can control what builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is + :func:`eval`. If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the expression is executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the + :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes <nested scope>` (non-locals) in the enclosing - environment. + environment if they are already referenced in the scope that is calling + :func:`eval` (e.g. via a :keyword:`nonlocal` statement). - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: + Example: >>> x = 1 >>> eval('x+1') @@ -571,11 +635,20 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event <auditing>` ``exec`` with the code object as the argument. Code compilation events may also be raised. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. index:: pair: built-in function; exec -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) +.. function:: exec(source, /, globals=None, locals=None, *, closure=None) - This function supports dynamic execution of Python code. *object* must be + This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, @@ -592,9 +665,15 @@ are always available. They are listed here in alphabetical order. will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember - that at the module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. + that at the module level, globals and locals are the same dictionary. + + .. note:: + + When ``exec`` gets two separate objects as *globals* and *locals*, the + code will be executed as if it were embedded in a class definition. This + means functions and classes defined in the executed code will not be able + to access variables assigned at the top level (as the "top level" + variables are treated as class variables in a class definition). If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -615,19 +694,27 @@ are always available. They are listed here in alphabetical order. .. note:: The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around + global and local namespace, respectively, which may be useful to pass around for use as the second and third argument to :func:`exec`. .. note:: - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. + The default *locals* act as described for function :func:`locals` below. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. .. versionchanged:: 3.11 Added the *closure* parameter. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. function:: filter(function, iterable) @@ -646,36 +733,53 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* is false. -.. class:: float(x=0.0) +.. class:: float(number=0.0, /) + float(string, /) .. index:: single: NaN single: Infinity - Return a floating point number constructed from a number or string *x*. + Return a floating point number constructed from a number or a string. + + Examples: + + .. doctest:: + + >>> float('+1.23') + 1.23 + >>> float(' -12345\n') + -12345.0 + >>> float('1e-003') + 0.001 + >>> float('+1E6') + 1000000.0 + >>> float('-Infinity') + -inf If the argument is a string, it should contain a decimal number, optionally preceded by a sign, and optionally embedded in whitespace. The optional sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN - (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the ``floatvalue`` production rule in the following - grammar, after leading and trailing whitespace characters are removed: + (not-a-number), or positive or negative infinity. + More precisely, the input must conform to the :token:`~float:floatvalue` + production rule in the following grammar, after leading and trailing + whitespace characters are removed: .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" + digit: <a Unicode decimal digit, i.e. characters in Unicode general category Nd> digitpart: `digit` (["_"] `digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] - exponent: ("e" | "E") ["+" | "-"] `digitpart` - floatnumber: number [`exponent`] - floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + exponent: ("e" | "E") [`sign`] `digitpart` + floatnumber: `number` [`exponent`] + absfloatvalue: `floatnumber` | `infinity` | `nan` + floatvalue: [`sign`] `absfloatvalue` - Here ``digit`` is a Unicode decimal digit (character in the Unicode general - category ``Nd``). Case is not significant, so, for example, "inf", "Inf", - "INFINITY", and "iNfINity" are all acceptable spellings for positive - infinity. + Case is not significant, so, for example, "inf", "Inf", "INFINITY", and + "iNfINity" are all acceptable spellings for positive infinity. Otherwise, if the argument is an integer or a floating point number, a floating point number with the same value (within Python's floating point @@ -690,26 +794,13 @@ are always available. They are listed here in alphabetical order. If no argument is given, ``0.0`` is returned. - Examples:: - - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf - The float type is described in :ref:`typesnumeric`. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. @@ -731,8 +822,8 @@ are always available. They are listed here in alphabetical order. A call to ``format(value, format_spec)`` is translated to ``type(value).__format__(value, format_spec)`` which bypasses the instance - dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches + dictionary when searching for the value's :meth:`~object.__format__` method. + A :exc:`TypeError` exception is raised if the method search reaches :mod:`object` and the *format_spec* is non-empty, or if either the *format_spec* or the return value are not strings. @@ -796,9 +887,9 @@ are always available. They are listed here in alphabetical order. .. note:: - For objects with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`~object.__hash__` methods, + note that :func:`hash` truncates the return value based on the bit width of the host machine. - See :meth:`__hash__ <object.__hash__>` for details. .. function:: help() help(request) @@ -893,17 +984,35 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int(x=0) - int(x, base=10) +.. class:: int(number=0, /) + int(string, /, base=10) - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, - it returns ``x.__trunc__()``. + Return an integer object constructed from a number or a string, or return + ``0`` if no arguments are given. + + Examples: + + .. doctest:: + + >>> int(123.45) + 123 + >>> int('123') + 123 + >>> int(' -12_345\n') + -12345 + >>> int('FACE', 16) + 64206 + >>> int('0xface', 0) + 64206 + >>> int('01110011', base=2) + 115 + + If the argument defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If the argument defines + :meth:`~object.__index__`, it returns ``x.__index__()``. For floating point numbers, this truncates towards zero. - If *x* is not a number or if *base* is given, then *x* must be a string, + If the argument is not a number or if *base* is given, then it must be a string, :class:`bytes`, or :class:`bytearray` instance representing an integer in radix *base*. Optionally, the string can be preceded by ``+`` or ``-`` (with no space in between), have leading zeros, be surrounded by whitespace, @@ -933,22 +1042,22 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The first parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. - .. versionchanged:: 3.11 - The delegation to :meth:`~object.__trunc__` is deprecated. - .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to help avoid denial of service attacks. A :exc:`ValueError` is raised when - the limit is exceeded while converting a string *x* to an :class:`int` or + the limit is exceeded while converting a string to an :class:`int` or when converting an :class:`int` into a string would exceed the limit. See the :ref:`integer string conversion length limitation <int_max_str_digits>` documentation. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. + .. function:: isinstance(object, classinfo) Return ``True`` if the *object* argument is an instance of the *classinfo* @@ -986,8 +1095,9 @@ are always available. They are listed here in alphabetical order. Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a second argument, *object* must be a collection object which supports the - :term:`iterable` protocol (the :meth:`__iter__` method), or it must support - the sequence protocol (the :meth:`__getitem__` method with integer arguments + :term:`iterable` protocol (the :meth:`~object.__iter__` method), + or it must support + the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, then *object* must be a callable object. The iterator created in this case @@ -1031,14 +1141,56 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. + + .. versionchanged:: 3.12 + The behaviour of ``locals()`` in a comprehension has been updated as + described in :pep:`709`. + + .. versionchanged:: 3.13 + As part of :pep:`667`, the semantics of mutating the mapping objects + returned from this function are now defined. The behavior in + :term:`optimized scopes <optimized scope>` is now as described above. + Aside from being defined, the behaviour in other scopes remains + unchanged from previous versions. - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. .. function:: map(function, iterable, *iterables) @@ -1073,8 +1225,8 @@ are always available. They are listed here in alphabetical order. such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and ``heapq.nlargest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. + .. versionchanged:: 3.4 + Added the *default* keyword-only parameter. .. versionchanged:: 3.8 The *key* can be ``None``. @@ -1111,8 +1263,8 @@ are always available. They are listed here in alphabetical order. such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. + .. versionchanged:: 3.4 + Added the *default* keyword-only parameter. .. versionchanged:: 3.8 The *key* can be ``None``. @@ -1225,7 +1377,7 @@ are always available. They are listed here in alphabetical order. *buffering* is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line - buffering (only usable in text mode), and an integer > 1 to indicate the size + buffering (only usable when writing in text mode), and an integer > 1 to indicate the size in bytes of a fixed-size chunk buffer. Note that specifying a buffer size this way applies for binary buffered I/O, but ``TextIOWrapper`` (i.e., files opened with ``mode='r+'``) would have another buffering. To disable buffering in @@ -1413,7 +1565,9 @@ are always available. They are listed here in alphabetical order. returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of type :class:`int` or :class:`float` and a non-integral exponent, a complex result is delivered. For example, ``pow(-9, 0.5)`` returns a value close - to ``3j``. + to ``3j``. Whereas, for a negative base of type :class:`int` or :class:`float` + with an integral exponent, a float result is delivered. For example, + ``pow(-9, 2.0)`` returns ``81.0``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and @@ -1504,38 +1658,44 @@ are always available. They are listed here in alphabetical order. """Get the current voltage.""" return self._voltage - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" + The ``@property`` decorator turns the :meth:`!voltage` method into a "getter" for a read-only attribute with the same name, and it sets the docstring for *voltage* to "Get the current voltage." - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: + .. decorator:: property.getter + .. decorator:: property.setter + .. decorator:: property.deleter - class C: - def __init__(self): - self._x = None + A property object has ``getter``, ``setter``, + and ``deleter`` methods usable as decorators that create a + copy of the property with the corresponding accessor function set to the + decorated function. This is best explained with an example: - @property - def x(self): - """I'm the 'x' property.""" - return self._x + .. testcode:: - @x.setter - def x(self, value): - self._x = value + class C: + def __init__(self): + self._x = None - @x.deleter - def x(self): - del self._x + @property + def x(self): + """I'm the 'x' property.""" + return self._x - This code is exactly equivalent to the first example. Be sure to give the - additional functions the same name as the original property (``x`` in this - case.) + @x.setter + def x(self, value): + self._x = value - The returned property object also has the attributes ``fget``, ``fset``, and - ``fdel`` corresponding to the constructor arguments. + @x.deleter + def x(self): + del self._x + + This code is exactly equivalent to the first example. Be sure to give the + additional functions the same name as the original property (``x`` in this + case.) + + The returned property object also has the attributes ``fget``, ``fset``, and + ``fdel`` corresponding to the constructor arguments. .. versionchanged:: 3.5 The docstrings of property objects are now writeable. @@ -1558,17 +1718,28 @@ are always available. They are listed here in alphabetical order. representation is a string enclosed in angle brackets that contains the name of the type of the object together with additional information often including the name and address of the object. A class can control what this - function returns for its instances by defining a :meth:`__repr__` method. + function returns for its instances + by defining a :meth:`~object.__repr__` method. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. + This class has a custom representation that can be evaluated:: + + class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def __repr__(self): + return f"Person('{self.name}', {self.age})" + .. function:: reversed(seq) Return a reverse :term:`iterator`. *seq* must be an object which has - a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`__getitem__` method with integer - arguments starting at ``0``). + a :meth:`~object.__reversed__` method or supports the sequence protocol (the + :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method + with integer arguments starting at ``0``). .. function:: round(number, ndigits=None) @@ -1639,13 +1810,21 @@ are always available. They are listed here in alphabetical order. Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop`, and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + ``None``. + + .. attribute:: slice.start + .. attribute:: slice.stop + .. attribute:: slice.step + + Slice objects have read-only data attributes :attr:`!start`, + :attr:`!stop`, and :attr:`!step` which merely return the argument + values (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. + Slice objects are also generated when extended indexing syntax is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. + :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1698,8 +1877,9 @@ are always available. They are listed here in alphabetical order. :ref:`function` for details. A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). + an instance (such as ``C().f()``). + Moreover, the static method :term:`descriptor` is also callable, so it can + be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1758,6 +1938,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.12 Summation of floats switched to an algorithm that gives higher accuracy and better commutativity on most builds. + .. versionchanged:: 3.14 + Added specialization for summation of complexes, + using same algorithm as for summation of floats. + .. class:: super() super(type, object_or_type=None) @@ -1784,6 +1968,13 @@ are always available. They are listed here in alphabetical order. the second argument is a type, ``issubclass(type2, type)`` must be true (this is useful for classmethods). + When called directly within an ordinary method of a class, both arguments may + be omitted ("zero-argument :func:`!super`"). In this case, *type* will be the + enclosing class, and *obj* will be the first argument of the immediately + enclosing function (typically ``self``). (This means that zero-argument + :func:`!super` will not work as expected within nested functions, including + generator expressions, which implicitly create nested functions.) + There are two typical use cases for *super*. In a class hierarchy with single inheritance, *super* can be used to refer to parent classes without naming them explicitly, thus making the code more maintainable. This use @@ -1812,7 +2003,8 @@ are always available. They are listed here in alphabetical order. Note that :func:`super` is implemented as part of the binding process for explicit dotted attribute lookups such as ``super().__getitem__(name)``. - It does so by implementing its own :meth:`__getattribute__` method for searching + It does so by implementing its own :meth:`~object.__getattribute__` method + for searching classes in a predictable order that supports cooperative multiple inheritance. Accordingly, :func:`super` is undefined for implicit lookups using statements or operators such as ``super()[name]``. @@ -1890,14 +2082,18 @@ are always available. They are listed here in alphabetical order. :attr:`~object.__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. + Without an argument, :func:`vars` acts like :func:`locals`. A :exc:`TypeError` exception is raised if an object is specified but it doesn't have a :attr:`~object.__dict__` attribute (for example, if its class defines the :attr:`~object.__slots__` attribute). + .. versionchanged:: 3.13 + + The result of calling this function without an argument has been + updated as described for the :func:`locals` builtin. + + .. function:: zip(*iterables, strict=False) Iterate over several iterables in parallel, producing tuples with an item diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 69ec1eb3ecd89d..9d5c72802a21f2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -1,5 +1,5 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== +:mod:`!functools` --- Higher-order functions and operations on callable objects +=============================================================================== .. module:: functools :synopsis: Higher-order functions and operations on callable objects. @@ -194,7 +194,7 @@ The :mod:`functools` module defines the following functions: In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. - The wrapped function is instrumented with a :func:`cache_parameters` + The wrapped function is instrumented with a :func:`!cache_parameters` function that returns a new :class:`dict` showing the values for *maxsize* and *typed*. This is for information purposes only. Mutating the values has no effect. @@ -275,8 +275,8 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.8 Added the *user_function* option. - .. versionadded:: 3.9 - Added the function :func:`cache_parameters` + .. versionchanged:: 3.9 + Added the function :func:`!cache_parameters` .. decorator:: total_ordering @@ -325,7 +325,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for + Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. .. function:: partial(func, /, *args, **keywords) @@ -646,8 +646,9 @@ The :mod:`functools` module defines the following functions: attributes of the wrapper function are updated with the corresponding attributes from the original function. The default values for these arguments are the module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which + function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__``, + ``__type_params__``, and ``__doc__``, the documentation string) + and ``WRAPPER_UPDATES`` (which updates the wrapper function's ``__dict__``, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes @@ -667,13 +668,9 @@ The :mod:`functools` module defines the following functions: on the wrapper function). :exc:`AttributeError` is still raised if the wrapper function itself is missing any attributes named in *updated*. - .. versionadded:: 3.2 - Automatic addition of the ``__wrapped__`` attribute. - - .. versionadded:: 3.2 - Copying of the ``__annotations__`` attribute by default. - .. versionchanged:: 3.2 + The ``__wrapped__`` attribute is now automatically added. + The ``__annotations__`` attribute is now copied by default. Missing attributes no longer trigger an :exc:`AttributeError`. .. versionchanged:: 3.4 @@ -681,6 +678,9 @@ The :mod:`functools` module defines the following functions: function, even if that function defined a ``__wrapped__`` attribute. (see :issue:`17482`) + .. versionchanged:: 3.12 + The ``__type_params__`` attribute is now copied by default. + .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -742,7 +742,7 @@ have three read-only attributes: called. :class:`partial` objects are like :class:`function` objects in that they are -callable, weak referencable, and can have attributes. There are some important +callable, weak referenceable, and can have attributes. There are some important differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 331c071cda7692..790dfdfd00b196 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -1,5 +1,5 @@ -:mod:`gc` --- Garbage Collector interface -========================================= +:mod:`!gc` --- Garbage Collector interface +========================================== .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. @@ -42,8 +42,8 @@ The :mod:`gc` module provides the following functions: With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. + :exc:`ValueError` is raised if the generation number is invalid. The sum of + collected objects and uncollectable objects is returned. The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) @@ -69,7 +69,7 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by + returned. If *generation* is not ``None``, return only the objects tracked by the collector that are in that generation. .. versionchanged:: 3.8 @@ -96,7 +96,7 @@ The :mod:`gc` module provides the following functions: .. versionadded:: 3.4 -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) +.. function:: set_threshold(threshold0, [threshold1, [threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index ada68b240143e8..d43d3250732306 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -1,5 +1,5 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= +:mod:`!getopt` --- C-style parser for command line options +========================================================== .. module:: getopt :synopsis: Portable parser for command line options; support both short and diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index d5bbe67fb30a62..9d67250033df81 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -1,5 +1,5 @@ -:mod:`getpass` --- Portable password input -========================================== +:mod:`!getpass` --- Portable password input +=========================================== .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. @@ -43,10 +43,13 @@ The :mod:`getpass` module provides two functions: Return the "login name" of the user. This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and + :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and returns the value of the first one which is set to a non-empty string. If none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. + systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` + is raised. In general, this function should be preferred over :func:`os.getlogin()`. + + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index dc6cf5533fccbe..d0de83907eb297 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -1,5 +1,5 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= +:mod:`!gettext` --- Multilingual internationalization services +============================================================== .. module:: gettext :synopsis: Multilingual internationalization services. @@ -257,7 +257,7 @@ are the methods of :class:`!NullTranslations`: .. method:: info() - Return the "protected" :attr:`_info` variable, a dictionary containing + Return a dictionary containing the metadata found in the message catalog file. @@ -296,9 +296,9 @@ are the methods of :class:`!NullTranslations`: The :class:`GNUTranslations` class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :mod:`gettext` module provides one additional class derived from +The :mod:`!gettext` module provides one additional class derived from :class:`NullTranslations`: :class:`GNUTranslations`. This class overrides -:meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files +:meth:`!_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files in both big-endian and little-endian format. :class:`GNUTranslations` parses optional metadata out of the translation @@ -306,7 +306,7 @@ catalog. It is convention with GNU :program:`gettext` to include metadata as the translation for the empty string. This metadata is in :rfc:`822`\ -style ``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the key ``Content-Type`` is found, then the ``charset`` property is used to -initialize the "protected" :attr:`_charset` instance variable, defaulting to +initialize the "protected" :attr:`!_charset` instance variable, defaulting to ``None`` if not found. If the charset encoding is specified, then all message ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. @@ -315,7 +315,7 @@ Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the -"protected" :attr:`_info` instance variable. +"protected" :attr:`!_info` instance variable. If the :file:`.mo` file's magic number is invalid, the major version number is unexpected, or if other problems occur while reading the file, instantiating a @@ -636,9 +636,9 @@ implementations, and valuable experience to the creation of this module: .. rubric:: Footnotes -.. [#] The default locale directory is system dependent; for example, on RedHat Linux +.. [#] The default locale directory is system dependent; for example, on Red Hat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. - The :mod:`gettext` module does not try to support these system dependent + The :mod:`!gettext` module does not try to support these system dependent defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see :data:`sys.base_prefix`). For this reason, it is always best to call :func:`bindtextdomain` with an explicit absolute path at the start of your diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 0e4cfe7ebed797..684466d354aef8 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -1,5 +1,5 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== +:mod:`!glob` --- Unix style pathname pattern expansion +====================================================== .. module:: glob :synopsis: Unix shell style pathname pattern expansion. @@ -34,9 +34,7 @@ unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. +The :mod:`glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -77,6 +75,10 @@ For example, ``'[?]'`` matches the character ``'?'``. Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -96,6 +98,10 @@ For example, ``'[?]'`` matches the character ``'?'``. .. audit-event:: glob.glob pathname,recursive glob.iglob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -117,7 +123,48 @@ For example, ``'[?]'`` matches the character ``'?'``. .. versionadded:: 3.4 -For example, consider a directory containing the following files: +.. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) + + Convert the given path specification to a regular expression for use with + :func:`re.match`. The path specification can contain shell-style wildcards. + + For example: + + >>> import glob, re + >>> + >>> regex = glob.translate('**/*.txt', recursive=True, include_hidden=True) + >>> regex + '(?s:(?:.+/)?[^/]*\\.txt)\\Z' + >>> reobj = re.compile(regex) + >>> reobj.match('foo/bar/baz.txt') + <re.Match object; span=(0, 15), match='foo/bar/baz.txt'> + + Path separators and segments are meaningful to this function, unlike + :func:`fnmatch.translate`. By default wildcards do not match path + separators, and ``*`` pattern segments match precisely one path segment. + + If *recursive* is true, the pattern segment "``**``" will match any number + of path segments. + + If *include_hidden* is true, wildcards can match path segments that start + with a dot (``.``). + + A sequence of path separators may be supplied to the *seps* argument. If + not given, :data:`os.sep` and :data:`~os.altsep` (if available) are used. + + .. seealso:: + + :meth:`pathlib.PurePath.full_match` and :meth:`pathlib.Path.glob` + methods, which call this function to implement pattern matching and + globbing. + + .. versionadded:: 3.13 + + +Examples +-------- + +Consider a directory containing the following files: :file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub` which contains only the file :file:`3.txt`. :func:`glob` will produce the following results. Notice how any leading components of the path are @@ -146,6 +193,7 @@ default. For example, consider a directory containing :file:`card.gif` and ['.card.gif'] .. seealso:: + The :mod:`fnmatch` module offers shell-style filename (not path) expansion. - Module :mod:`fnmatch` - Shell-style filename (not path) expansion +.. seealso:: + The :mod:`pathlib` module offers high-level path objects. diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 5414d6370b78ce..a0b16576fad219 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -1,5 +1,5 @@ -:mod:`graphlib` --- Functionality to operate with graph-like structures -========================================================================= +:mod:`!graphlib` --- Functionality to operate with graph-like structures +======================================================================== .. module:: graphlib :synopsis: Functionality to operate with graph-like structures diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 8f88f82e1c21ad..30caea328baa79 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -1,5 +1,5 @@ -:mod:`grp` --- The group database -================================= +:mod:`!grp` --- The group database +================================== .. module:: grp :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix group database. It is available on all Unix versions. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not WASI, not iOS. Group database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``group`` structure (Attribute field below, see diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index f931d0e399c9f2..6b6e158f6eba2c 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -1,5 +1,5 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= +:mod:`!gzip` --- Support for :program:`gzip` files +================================================== .. module:: gzip :synopsis: Interfaces for gzip compression and decompression using file objects. @@ -61,7 +61,7 @@ The module defines the following items: .. exception:: BadGzipFile - An exception raised for invalid gzip files. It inherits :exc:`OSError`. + An exception raised for invalid gzip files. It inherits from :exc:`OSError`. :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip files. @@ -100,12 +100,14 @@ The module defines the following items: compression, and ``9`` is slowest and produces the most compression. ``0`` is no compression. The default is ``9``. - The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. + The optional *mtime* argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 + to generate a compressed stream that does not depend on creation time. - Calling a :class:`GzipFile` object's :meth:`close` method does not close + See below for the :attr:`mtime` attribute that is set when decompressing. + + Calling a :class:`GzipFile` object's :meth:`!close` method does not close *fileobj*, since you might wish to append more material after the compressed data. This also allows you to pass an :class:`io.BytesIO` object opened for writing as *fileobj*, and retrieve the resulting memory buffer using the @@ -131,17 +133,19 @@ The module defines the following items: .. versionadded:: 3.2 - .. attribute:: mtime + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionchanged:: 3.13 + In previous versions it was an integer ``1`` or ``2``. - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. + .. attribute:: mtime - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. + When decompressing, this attribute is set to the last timestamp in the most + recently read header. It is an integer, holding the number of seconds + since the Unix epoch (00:00:00 UTC, January 1, 1970). + The initial value before reading any headers is ``None``. .. attribute:: name @@ -171,22 +175,20 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.12 - Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` - attribute instead. - .. deprecated:: 3.9 Opening :class:`GzipFile` for writing without specifying the *mode* argument is deprecated. + .. versionchanged:: 3.12 + Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` + attribute instead. + .. function:: compress(data, compresslevel=9, *, mtime=None) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. + the :class:`GzipFile` constructor above. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -194,7 +196,13 @@ The module defines the following items: .. versionchanged:: 3.11 Speed is improved by compressing all data at once instead of in a streamed fashion. Calls with *mtime* set to ``0`` are delegated to - :func:`zlib.compress` for better speed. + :func:`zlib.compress` for better speed. In this situation the + output may contain a gzip header "OS" byte value other than 255 + "unknown" as supplied by the underlying zlib implementation. + + .. versionchanged:: 3.13 + The gzip header OS byte is guaranteed to be set to 255 when this function + is used as was the case in 3.10 and earlier. .. function:: decompress(data) @@ -287,4 +295,3 @@ Command line options .. option:: -h, --help Show the help message. - diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 761dd84edee299..5d24b77e13bfce 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -1,5 +1,5 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== +:mod:`!hashlib` --- Secure hashes and message digests +===================================================== .. module:: hashlib :synopsis: Secure hash and message digest algorithms. @@ -77,8 +77,6 @@ accessible by name via :func:`new`. See :data:`algorithms_available`. SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` were added. - -.. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. .. _hashlib-usedforsecurity: @@ -121,7 +119,7 @@ More condensed: Constructors ------------ -.. function:: new(name[, data], \*, usedforsecurity=True) +.. function:: new(name[, data], *, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the @@ -330,7 +328,7 @@ include a `salt <https://en.wikipedia.org/wiki/Salt_%28cryptography%29>`_. your application, read *Appendix A.2.2* of NIST-SP-800-132_. The answers on the `stackexchange pbkdf2 iterations question`_ explain in detail. - *dklen* is the length of the derived key. If *dklen* is ``None`` then the + *dklen* is the length of the derived key in bytes. If *dklen* is ``None`` then the digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. >>> from hashlib import pbkdf2_hmac @@ -359,7 +357,7 @@ include a `salt <https://en.wikipedia.org/wiki/Salt_%28cryptography%29>`_. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). - *dklen* is the length of the derived key. + *dklen* is the length of the derived key in bytes. .. versionadded:: 3.6 diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 8b00f7b27959b6..d3c4b920ba500a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -1,5 +1,5 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== +:mod:`!heapq` --- Heap queue algorithm +====================================== .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). @@ -17,7 +17,9 @@ This module provides an implementation of the heap queue algorithm, also known as the priority queue algorithm. Heaps are binary trees for which every parent node has a value less than or -equal to any of its children. This implementation uses arrays for which +equal to any of its children. We refer to this condition as the heap invariant. + +This implementation uses arrays for which ``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]`` for all *k*, counting elements from zero. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that its @@ -270,7 +272,7 @@ winner. The simplest algorithmic way to remove it and find the "next" winner is to move some loser (let's say cell 30 in the diagram above) into the 0 position, and then percolate this new 0 down the tree, exchanging values, until the invariant is re-established. This is clearly logarithmic on the total number of -items in the tree. By iterating over all items, you get an O(n log n) sort. +items in the tree. By iterating over all items, you get an *O*\ (*n* log *n*) sort. A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the @@ -319,4 +321,3 @@ applications, and I think it is good to keep a 'heap' module around. :-) backwards, and this was also used to avoid the rewinding time. Believe me, real good tape sorts were quite spectacular to watch! From all times, sorting has always been a Great Art! :-) - diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index b2ca0455d3745c..d6692033b2d4c3 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -1,5 +1,5 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== +:mod:`!hmac` --- Keyed-Hashing for Message Authentication +========================================================= .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation @@ -14,7 +14,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. -.. function:: new(key, msg=None, digestmod='') +.. function:: new(key, msg=None, digestmod) Return a new hmac object. *key* is a bytes or bytearray object giving the secret key. If *msg* is present, the method call ``update(msg)`` is made. @@ -27,10 +27,9 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. Parameter *msg* can be of any type supported by :mod:`hashlib`. Parameter *digestmod* can be the name of a hash algorithm. - .. deprecated-removed:: 3.4 3.8 - MD5 as implicit default digest for *digestmod* is deprecated. - The digestmod parameter is now required. Pass it as a keyword - argument to avoid awkwardness when you do not have an initial msg. + .. versionchanged:: 3.8 + The *digestmod* argument is now required. Pass it as a keyword + argument to avoid awkwardness when you do not have an initial *msg*. .. function:: digest(key, msg, digest) @@ -114,11 +113,9 @@ A hash object has the following attributes: .. versionadded:: 3.4 -.. deprecated:: 3.9 - - The undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, and - ``HMAC.outer`` are internal implementation details and will be removed in - Python 3.10. +.. versionchanged:: 3.10 + Removed the undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, + and ``HMAC.outer``. This module also provides the following helper function: diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 10529561a92cd0..add18e4c87d220 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -1,5 +1,5 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= +:mod:`!html.entities` --- Definitions of HTML general entities +============================================================== .. module:: html.entities :synopsis: Definitions of HTML general entities. diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index d35090111e0822..6d433b5a04fc4a 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -1,5 +1,5 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== +:mod:`!html.parser` --- Simple HTML and XHTML parser +==================================================== .. module:: html.parser :synopsis: A simple parser that can handle HTML and XHTML. diff --git a/Doc/library/html.rst b/Doc/library/html.rst index c2b01e14ea7555..9aa39ba9a42b0f 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -1,5 +1,5 @@ -:mod:`html` --- HyperText Markup Language support -================================================= +:mod:`!html` --- HyperText Markup Language support +================================================== .. module:: html :synopsis: Helpers for manipulating HTML. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index c46314fc5e253b..2835c8d0eb711e 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -1,5 +1,5 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== +:mod:`!http.client` --- HTTP protocol client +============================================ .. module:: http.client :synopsis: HTTP and HTTPS protocol client (requires sockets). @@ -92,7 +92,7 @@ The module provides the following classes: .. versionchanged:: 3.4.3 This class now performs all the necessary certificate and hostname checks by default. To revert to the previous, unverified, behavior - :func:`ssl._create_unverified_context` can be passed to the *context* + :func:`!ssl._create_unverified_context` can be passed to the *context* parameter. .. versionchanged:: 3.8 @@ -103,7 +103,7 @@ The module provides the following classes: .. versionchanged:: 3.10 This class now sends an ALPN extension with protocol indicator ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. .. versionchanged:: 3.12 The deprecated *key_file*, *cert_file* and *check_hostname* parameters @@ -124,7 +124,7 @@ This module provides the following function: .. function:: parse_headers(fp) Parse the headers from a file pointer *fp* representing a HTTP - request/response. The file has to be a :class:`BufferedIOBase` reader + request/response. The file has to be a :class:`~io.BufferedIOBase` reader (i.e. not text) and must provide a valid :rfc:`2822` style header. This function returns an instance of :class:`http.client.HTTPMessage` @@ -311,7 +311,7 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. - .. versionadded:: 3.2 + .. versionchanged:: 3.2 *body* can now be an iterable. .. versionchanged:: 3.6 @@ -416,7 +416,7 @@ HTTPConnection Objects .. versionadded:: 3.7 -As an alternative to using the :meth:`request` method described above, you can +As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can also send your request step by step, by using the four functions below. @@ -461,9 +461,8 @@ also send your request step by step, by using the four functions below. This is to avoid premature termination of the read of the request by the target server due to malformed encoding. - .. versionadded:: 3.6 - Chunked encoding support. The *encode_chunked* parameter was - added. + .. versionchanged:: 3.6 + Added chunked encoding support and the *encode_chunked* parameter. .. method:: HTTPConnection.send(data) @@ -648,6 +647,8 @@ method attribute. Here is an example session that uses the ``PUT`` method:: HTTPMessage Objects ------------------- +.. class:: HTTPMessage(email.message.Message) + An :class:`http.client.HTTPMessage` instance holds the headers from an HTTP response. It is implemented using the :class:`email.message.Message` class. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 12a6d768437ea5..31ac8bafb6ab4b 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -1,5 +1,5 @@ -:mod:`http.cookiejar` --- Cookie handling for HTTP clients -========================================================== +:mod:`!http.cookiejar` --- Cookie handling for HTTP clients +=========================================================== .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. @@ -649,6 +649,11 @@ internal consistency, so you should know what you're doing if you do that. :const:`None`. +.. attribute:: Cookie.domain + + Cookie domain (a string). + + .. attribute:: Cookie.path Cookie path (a string, eg. ``'/acme/rocket_launchers'``). diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index a2c1eb00d8b33d..4ce2e3c4f4cb42 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -1,5 +1,5 @@ -:mod:`http.cookies` --- HTTP state management -============================================= +:mod:`!http.cookies` --- HTTP state management +============================================== .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). @@ -18,16 +18,17 @@ cookie value. The module formerly strictly applied the parsing rules described in the :rfc:`2109` and :rfc:`2068` specifications. It has since been discovered that -MSIE 3.0x doesn't follow the character rules outlined in those specs and also -many current day browsers and servers have relaxed parsing rules when comes to -Cookie handling. As a result, the parsing rules used are a bit less strict. +MSIE 3.0x didn't follow the character rules outlined in those specs; many +current-day browsers and servers have also relaxed parsing rules when it comes +to cookie handling. As a result, this module now uses parsing rules that are a +bit less strict than they once were. The character set, :data:`string.ascii_letters`, :data:`string.digits` and ``!#$%&'*+-.^_`|~:`` denote the set of valid characters allowed by this module -in Cookie name (as :attr:`~Morsel.key`). +in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed ':' as a valid Cookie name character. + Allowed ':' as a valid cookie name character. .. note:: @@ -54,9 +55,10 @@ in Cookie name (as :attr:`~Morsel.key`). .. class:: SimpleCookie([input]) - This class derives from :class:`BaseCookie` and overrides :meth:`value_decode` - and :meth:`value_encode`. SimpleCookie supports strings as cookie values. - When setting the value, SimpleCookie calls the builtin :func:`str()` to convert + This class derives from :class:`BaseCookie` and overrides :meth:`~BaseCookie.value_decode` + and :meth:`~BaseCookie.value_encode`. :class:`!SimpleCookie` supports + strings as cookie values. When setting the value, :class:`!SimpleCookie` + calls the builtin :func:`str` to convert the value to a string. Values received from HTTP are kept as strings. .. seealso:: @@ -129,17 +131,17 @@ Morsel Objects Abstract a key/value pair, which has some :rfc:`2109` attributes. Morsels are dictionary-like objects, whose set of keys is constant --- the valid - :rfc:`2109` attributes, which are - - * ``expires`` - * ``path`` - * ``comment`` - * ``domain`` - * ``max-age`` - * ``secure`` - * ``version`` - * ``httponly`` - * ``samesite`` + :rfc:`2109` attributes, which are: + + .. attribute:: expires + path + comment + domain + max-age + secure + version + httponly + samesite The attribute :attr:`httponly` specifies that the cookie is only transferred in HTTP requests, and is not accessible through JavaScript. This is intended @@ -152,7 +154,7 @@ Morsel Objects The keys are case-insensitive and their default value is ``''``. .. versionchanged:: 3.5 - :meth:`~Morsel.__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` + :meth:`!__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` into account. .. versionchanged:: 3.7 diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 5e1912716e5319..ce3fb9f8120502 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -1,5 +1,5 @@ -:mod:`http` --- HTTP modules -============================ +:mod:`!http` --- HTTP modules +============================= .. module:: http :synopsis: HTTP status codes and messages @@ -59,63 +59,63 @@ available in :class:`http.HTTPStatus` are: ======= =================================== ================================================================== Code Enum Name Details ======= =================================== ================================================================== -``100`` ``CONTINUE`` HTTP/1.1 :rfc:`7231`, Section 6.2.1 -``101`` ``SWITCHING_PROTOCOLS`` HTTP/1.1 :rfc:`7231`, Section 6.2.2 +``100`` ``CONTINUE`` HTTP Semantics :rfc:`9110`, Section 15.2.1 +``101`` ``SWITCHING_PROTOCOLS`` HTTP Semantics :rfc:`9110`, Section 15.2.2 ``102`` ``PROCESSING`` WebDAV :rfc:`2518`, Section 10.1 ``103`` ``EARLY_HINTS`` An HTTP Status Code for Indicating Hints :rfc:`8297` -``200`` ``OK`` HTTP/1.1 :rfc:`7231`, Section 6.3.1 -``201`` ``CREATED`` HTTP/1.1 :rfc:`7231`, Section 6.3.2 -``202`` ``ACCEPTED`` HTTP/1.1 :rfc:`7231`, Section 6.3.3 -``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP/1.1 :rfc:`7231`, Section 6.3.4 -``204`` ``NO_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.5 -``205`` ``RESET_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.6 -``206`` ``PARTIAL_CONTENT`` HTTP/1.1 :rfc:`7233`, Section 4.1 +``200`` ``OK`` HTTP Semantics :rfc:`9110`, Section 15.3.1 +``201`` ``CREATED`` HTTP Semantics :rfc:`9110`, Section 15.3.2 +``202`` ``ACCEPTED`` HTTP Semantics :rfc:`9110`, Section 15.3.3 +``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP Semantics :rfc:`9110`, Section 15.3.4 +``204`` ``NO_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.5 +``205`` ``RESET_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.6 +``206`` ``PARTIAL_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.7 ``207`` ``MULTI_STATUS`` WebDAV :rfc:`4918`, Section 11.1 ``208`` ``ALREADY_REPORTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.1 (Experimental) ``226`` ``IM_USED`` Delta Encoding in HTTP :rfc:`3229`, Section 10.4.1 -``300`` ``MULTIPLE_CHOICES`` HTTP/1.1 :rfc:`7231`, Section 6.4.1 -``301`` ``MOVED_PERMANENTLY`` HTTP/1.1 :rfc:`7231`, Section 6.4.2 -``302`` ``FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.4.3 -``303`` ``SEE_OTHER`` HTTP/1.1 :rfc:`7231`, Section 6.4.4 -``304`` ``NOT_MODIFIED`` HTTP/1.1 :rfc:`7232`, Section 4.1 -``305`` ``USE_PROXY`` HTTP/1.1 :rfc:`7231`, Section 6.4.5 -``307`` ``TEMPORARY_REDIRECT`` HTTP/1.1 :rfc:`7231`, Section 6.4.7 -``308`` ``PERMANENT_REDIRECT`` Permanent Redirect :rfc:`7238`, Section 3 (Experimental) -``400`` ``BAD_REQUEST`` HTTP/1.1 :rfc:`7231`, Section 6.5.1 -``401`` ``UNAUTHORIZED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.1 -``402`` ``PAYMENT_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.2 -``403`` ``FORBIDDEN`` HTTP/1.1 :rfc:`7231`, Section 6.5.3 -``404`` ``NOT_FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.5.4 -``405`` ``METHOD_NOT_ALLOWED`` HTTP/1.1 :rfc:`7231`, Section 6.5.5 -``406`` ``NOT_ACCEPTABLE`` HTTP/1.1 :rfc:`7231`, Section 6.5.6 -``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.2 -``408`` ``REQUEST_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.5.7 -``409`` ``CONFLICT`` HTTP/1.1 :rfc:`7231`, Section 6.5.8 -``410`` ``GONE`` HTTP/1.1 :rfc:`7231`, Section 6.5.9 -``411`` ``LENGTH_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.10 -``412`` ``PRECONDITION_FAILED`` HTTP/1.1 :rfc:`7232`, Section 4.2 -``413`` ``REQUEST_ENTITY_TOO_LARGE`` HTTP/1.1 :rfc:`7231`, Section 6.5.11 -``414`` ``REQUEST_URI_TOO_LONG`` HTTP/1.1 :rfc:`7231`, Section 6.5.12 -``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP/1.1 :rfc:`7231`, Section 6.5.13 -``416`` ``REQUESTED_RANGE_NOT_SATISFIABLE`` HTTP/1.1 Range Requests :rfc:`7233`, Section 4.4 -``417`` ``EXPECTATION_FAILED`` HTTP/1.1 :rfc:`7231`, Section 6.5.14 +``300`` ``MULTIPLE_CHOICES`` HTTP Semantics :rfc:`9110`, Section 15.4.1 +``301`` ``MOVED_PERMANENTLY`` HTTP Semantics :rfc:`9110`, Section 15.4.2 +``302`` ``FOUND`` HTTP Semantics :rfc:`9110`, Section 15.4.3 +``303`` ``SEE_OTHER`` HTTP Semantics :rfc:`9110`, Section 15.4.4 +``304`` ``NOT_MODIFIED`` HTTP Semantics :rfc:`9110`, Section 15.4.5 +``305`` ``USE_PROXY`` HTTP Semantics :rfc:`9110`, Section 15.4.6 +``307`` ``TEMPORARY_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.8 +``308`` ``PERMANENT_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.9 +``400`` ``BAD_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.1 +``401`` ``UNAUTHORIZED`` HTTP Semantics :rfc:`9110`, Section 15.5.2 +``402`` ``PAYMENT_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.3 +``403`` ``FORBIDDEN`` HTTP Semantics :rfc:`9110`, Section 15.5.4 +``404`` ``NOT_FOUND`` HTTP Semantics :rfc:`9110`, Section 15.5.5 +``405`` ``METHOD_NOT_ALLOWED`` HTTP Semantics :rfc:`9110`, Section 15.5.6 +``406`` ``NOT_ACCEPTABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.7 +``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.8 +``408`` ``REQUEST_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.5.9 +``409`` ``CONFLICT`` HTTP Semantics :rfc:`9110`, Section 15.5.10 +``410`` ``GONE`` HTTP Semantics :rfc:`9110`, Section 15.5.11 +``411`` ``LENGTH_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.12 +``412`` ``PRECONDITION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.13 +``413`` ``CONTENT_TOO_LARGE`` HTTP Semantics :rfc:`9110`, Section 15.5.14 +``414`` ``URI_TOO_LONG`` HTTP Semantics :rfc:`9110`, Section 15.5.15 +``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP Semantics :rfc:`9110`, Section 15.5.16 +``416`` ``RANGE_NOT_SATISFIABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.17 +``417`` ``EXPECTATION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.18 ``418`` ``IM_A_TEAPOT`` HTCPCP/1.0 :rfc:`2324`, Section 2.3.2 -``421`` ``MISDIRECTED_REQUEST`` HTTP/2 :rfc:`7540`, Section 9.1.2 -``422`` ``UNPROCESSABLE_ENTITY`` WebDAV :rfc:`4918`, Section 11.2 +``421`` ``MISDIRECTED_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.20 +``422`` ``UNPROCESSABLE_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.5.21 ``423`` ``LOCKED`` WebDAV :rfc:`4918`, Section 11.3 ``424`` ``FAILED_DEPENDENCY`` WebDAV :rfc:`4918`, Section 11.4 ``425`` ``TOO_EARLY`` Using Early Data in HTTP :rfc:`8470` -``426`` ``UPGRADE_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.15 +``426`` ``UPGRADE_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.22 ``428`` ``PRECONDITION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585` ``429`` ``TOO_MANY_REQUESTS`` Additional HTTP Status Codes :rfc:`6585` ``431`` ``REQUEST_HEADER_FIELDS_TOO_LARGE`` Additional HTTP Status Codes :rfc:`6585` ``451`` ``UNAVAILABLE_FOR_LEGAL_REASONS`` An HTTP Status Code to Report Legal Obstacles :rfc:`7725` -``500`` ``INTERNAL_SERVER_ERROR`` HTTP/1.1 :rfc:`7231`, Section 6.6.1 -``501`` ``NOT_IMPLEMENTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.2 -``502`` ``BAD_GATEWAY`` HTTP/1.1 :rfc:`7231`, Section 6.6.3 -``503`` ``SERVICE_UNAVAILABLE`` HTTP/1.1 :rfc:`7231`, Section 6.6.4 -``504`` ``GATEWAY_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.6.5 -``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.6 +``500`` ``INTERNAL_SERVER_ERROR`` HTTP Semantics :rfc:`9110`, Section 15.6.1 +``501`` ``NOT_IMPLEMENTED`` HTTP Semantics :rfc:`9110`, Section 15.6.2 +``502`` ``BAD_GATEWAY`` HTTP Semantics :rfc:`9110`, Section 15.6.3 +``503`` ``SERVICE_UNAVAILABLE`` HTTP Semantics :rfc:`9110`, Section 15.6.4 +``504`` ``GATEWAY_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.6.5 +``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP Semantics :rfc:`9110`, Section 15.6.6 ``506`` ``VARIANT_ALSO_NEGOTIATES`` Transparent Content Negotiation in HTTP :rfc:`2295`, Section 8.1 (Experimental) ``507`` ``INSUFFICIENT_STORAGE`` WebDAV :rfc:`4918`, Section 11.5 ``508`` ``LOOP_DETECTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.2 (Experimental) @@ -137,6 +137,10 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionadded:: 3.9 Added ``103 EARLY_HINTS``, ``418 IM_A_TEAPOT`` and ``425 TOO_EARLY`` status codes. +.. versionchanged:: 3.13 + Implemented RFC9110 naming for status constants. Old constant names are preserved for + backwards compatibility. + HTTP status category -------------------- @@ -144,15 +148,15 @@ HTTP status category The enum values have several properties to indicate the HTTP status category: -==================== ======================== =============================== +==================== ======================== ====================================== Property Indicates that Details -==================== ======================== =============================== -``is_informational`` ``100 <= status <= 199`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_success`` ``200 <= status <= 299`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_redirection`` ``300 <= status <= 399`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_client_error`` ``400 <= status <= 499`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_server_error`` ``500 <= status <= 599`` HTTP/1.1 :rfc:`7231`, Section 6 -==================== ======================== =============================== +==================== ======================== ====================================== +``is_informational`` ``100 <= status <= 199`` HTTP Semantics :rfc:`9110`, Section 15 +``is_success`` ``200 <= status <= 299`` HTTP Semantics :rfc:`9110`, Section 15 +``is_redirection`` ``300 <= status <= 399`` HTTP Semantics :rfc:`9110`, Section 15 +``is_client_error`` ``400 <= status <= 499`` HTTP Semantics :rfc:`9110`, Section 15 +``is_server_error`` ``500 <= status <= 599`` HTTP Semantics :rfc:`9110`, Section 15 +==================== ======================== ====================================== Usage:: @@ -203,13 +207,13 @@ available in :class:`http.HTTPMethod` are: =========== =================================== ================================================================== Method Enum Name Details =========== =================================== ================================================================== -``GET`` ``GET`` HTTP/1.1 :rfc:`7231`, Section 4.3.1 -``HEAD`` ``HEAD`` HTTP/1.1 :rfc:`7231`, Section 4.3.2 -``POST`` ``POST`` HTTP/1.1 :rfc:`7231`, Section 4.3.3 -``PUT`` ``PUT`` HTTP/1.1 :rfc:`7231`, Section 4.3.4 -``DELETE`` ``DELETE`` HTTP/1.1 :rfc:`7231`, Section 4.3.5 -``CONNECT`` ``CONNECT`` HTTP/1.1 :rfc:`7231`, Section 4.3.6 -``OPTIONS`` ``OPTIONS`` HTTP/1.1 :rfc:`7231`, Section 4.3.7 -``TRACE`` ``TRACE`` HTTP/1.1 :rfc:`7231`, Section 4.3.8 +``GET`` ``GET`` HTTP Semantics :rfc:`9110`, Section 9.3.1 +``HEAD`` ``HEAD`` HTTP Semantics :rfc:`9110`, Section 9.3.2 +``POST`` ``POST`` HTTP Semantics :rfc:`9110`, Section 9.3.3 +``PUT`` ``PUT`` HTTP Semantics :rfc:`9110`, Section 9.3.4 +``DELETE`` ``DELETE`` HTTP Semantics :rfc:`9110`, Section 9.3.5 +``CONNECT`` ``CONNECT`` HTTP Semantics :rfc:`9110`, Section 9.3.6 +``OPTIONS`` ``OPTIONS`` HTTP Semantics :rfc:`9110`, Section 9.3.7 +``TRACE`` ``TRACE`` HTTP Semantics :rfc:`9110`, Section 9.3.8 ``PATCH`` ``PATCH`` HTTP/1.1 :rfc:`5789` =========== =================================== ================================================================== diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 6f79b222790094..3c80fa747d5f1f 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -1,5 +1,5 @@ -:mod:`http.server` --- HTTP servers -=================================== +:mod:`!http.server` --- HTTP servers +==================================== .. module:: http.server :synopsis: HTTP server and request handlers. @@ -65,10 +65,10 @@ provides three different variants: The handler will parse the request and the headers, then call a method specific to the request type. The method name is constructed from the - request. For example, for the request method ``SPAM``, the :meth:`do_SPAM` + request. For example, for the request method ``SPAM``, the :meth:`!do_SPAM` method will be called with no arguments. All of the relevant information is stored in instance variables of the handler. Subclasses should not need to - override or extend the :meth:`__init__` method. + override or extend the :meth:`!__init__` method. :class:`BaseHTTPRequestHandler` has the following instance variables: @@ -187,13 +187,13 @@ provides three different variants: Calls :meth:`handle_one_request` once (or, if persistent connections are enabled, multiple times) to handle incoming HTTP requests. You should - never need to override it; instead, implement appropriate :meth:`do_\*` + never need to override it; instead, implement appropriate :meth:`!do_\*` methods. .. method:: handle_one_request() This method will parse and dispatch the request to the appropriate - :meth:`do_\*` method. You should never need to override it. + :meth:`!do_\*` method. You should never need to override it. .. method:: handle_expect_100() @@ -328,8 +328,8 @@ provides three different variants: or the current directory if *directory* is not provided, directly mapping the directory structure to HTTP requests. - .. versionadded:: 3.7 - The *directory* parameter. + .. versionchanged:: 3.7 + Added the *directory* parameter. .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. @@ -438,11 +438,11 @@ to bind to localhost only:: python -m http.server --bind 127.0.0.1 -.. versionadded:: 3.4 - ``--bind`` argument was introduced. +.. versionchanged:: 3.4 + Added the ``--bind`` option. -.. versionadded:: 3.8 - ``--bind`` argument enhanced to support IPv6 +.. versionchanged:: 3.8 + Support IPv6 in the ``--bind`` option. By default, the server uses the current directory. The option ``-d/--directory`` specifies a directory to which it should serve the files. For example, @@ -450,8 +450,8 @@ the following command uses a specific directory:: python -m http.server --directory /tmp/ -.. versionadded:: 3.7 - ``--directory`` argument was introduced. +.. versionchanged:: 3.7 + Added the ``--directory`` option. By default, the server is conformant to HTTP/1.0. The option ``-p/--protocol`` specifies the HTTP version to which the server is conformant. For example, the @@ -459,8 +459,8 @@ following command runs an HTTP/1.1 conformant server:: python -m http.server --protocol HTTP/1.1 -.. versionadded:: 3.11 - ``--protocol`` argument was introduced. +.. versionchanged:: 3.11 + Added the ``--protocol`` option. .. class:: CGIHTTPRequestHandler(request, client_address, server) @@ -520,6 +520,12 @@ the ``--cgi`` option:: :mod:`http.server` command line ``--cgi`` support is being removed because :class:`CGIHTTPRequestHandler` is being removed. +.. warning:: + + :class:`CGIHTTPRequestHandler` and the ``--cgi`` command line option + are not intended for use by untrusted clients and may be vulnerable + to exploitation. Always use within a secure environment. + .. _http.server-security: Security Considerations @@ -537,5 +543,5 @@ default :class:`BaseHTTPRequestHandler` ``.log_message`` implementation. This could allow remote clients connecting to your server to send nefarious control codes to your terminal. -.. versionadded:: 3.12 +.. versionchanged:: 3.12 Control characters are scrubbed in stderr logs. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index e710d0bacf3fee..59b181aab3e484 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -18,8 +18,6 @@ IDLE is Python's Integrated Development and Learning Environment. IDLE has the following features: -* coded in 100% pure Python, using the :mod:`tkinter` GUI toolkit - * cross-platform: works mostly the same on Windows, Unix, and macOS * Python shell window (interactive interpreter) with colorizing @@ -422,41 +420,34 @@ and that other files do not. Run Python code with the Run menu. Key bindings ^^^^^^^^^^^^ -In this section, 'C' refers to the :kbd:`Control` key on Windows and Unix and -the :kbd:`Command` key on macOS. - -* :kbd:`Backspace` deletes to the left; :kbd:`Del` deletes to the right - -* :kbd:`C-Backspace` delete word left; :kbd:`C-Del` delete word to the right - -* Arrow keys and :kbd:`Page Up`/:kbd:`Page Down` to move around - -* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves by words +The IDLE insertion cursor is a thin vertical bar between character +positions. When characters are entered, the insertion cursor and +everything to its right moves right one character and +the new character is entered in the new space. -* :kbd:`Home`/:kbd:`End` go to begin/end of line +Several non-character keys move the cursor and possibly +delete characters. Deletion does not puts text on the clipboard, +but IDLE has an undo list. Wherever this doc discusses keys, +'C' refers to the :kbd:`Control` key on Windows and +Unix and the :kbd:`Command` key on macOS. (And all such discussions +assume that the keys have not been re-bound to something else.) -* :kbd:`C-Home`/:kbd:`C-End` go to begin/end of file +* Arrow keys move the cursor one character or line. -* Some useful Emacs bindings are inherited from Tcl/Tk: +* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves left or right one word. - * :kbd:`C-a` beginning of line +* :kbd:`Home` and :kbd:`End` go to the beginning or end of the line. - * :kbd:`C-e` end of line +* :kbd:`Page Up` and :kbd:`Page Down` go up or down one screen. - * :kbd:`C-k` kill line (but doesn't put it in clipboard) +* :kbd:`C-Home` and :kbd:`C-End` go to beginning or end of the file. - * :kbd:`C-l` center window around the insertion point +* :kbd:`Backspace` and :kbd:`Del` (or :kbd:`C-d`) delete the previous + or next character. - * :kbd:`C-b` go backward one character without deleting (usually you can - also use the cursor key for this) +* :kbd:`C-Backspace` and :kbd:`C-Del` delete one word left or right. - * :kbd:`C-f` go forward one character without deleting (usually you can - also use the cursor key for this) - - * :kbd:`C-p` go up one line (usually you can also use the cursor key for - this) - - * :kbd:`C-d` delete next character +* :kbd:`C-k` deletes ('kills') everything to the right. Standard keybindings (like :kbd:`C-c` to copy and :kbd:`C-v` to paste) may work. Keybindings are selected in the Configure IDLE dialog. @@ -611,23 +602,18 @@ when one requests a restart on the Shell menu, or when one runs code in an editor window. The editing features described in previous subsections work when entering -code interactively. IDLE's Shell window also responds to the following keys. - -* :kbd:`C-c` interrupts executing command - -* :kbd:`C-d` sends end-of-file; closes window if typed at a ``>>>`` prompt - -* :kbd:`Alt-/` (Expand word) is also useful to reduce typing +code interactively. IDLE's Shell window also responds to the following: - Command history +* :kbd:`C-c` attempts to interrupt statement execution (but may fail). - * :kbd:`Alt-p` retrieves previous command matching what you have typed. On - macOS use :kbd:`C-p`. +* :kbd:`C-d` closes Shell if typed at a ``>>>`` prompt. - * :kbd:`Alt-n` retrieves next. On macOS use :kbd:`C-n`. +* :kbd:`Alt-p` and :kbd:`Alt-n` (:kbd:`C-p` and :kbd:`C-n` on macOS) + retrieve to the current prompt the previous or next previously + entered statement that matches anything already typed. - * :kbd:`Return` while the cursor is on any previous command - retrieves that command +* :kbd:`Return` while the cursor is on any previous statement + appends the latter to anything already typed at the prompt. Text colors ^^^^^^^^^^^ diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 1f774e64b0eae3..a2dad58b00b9fa 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -1,5 +1,5 @@ -:mod:`imaplib` --- IMAP4 protocol client -======================================== +:mod:`!imaplib` --- IMAP4 protocol client +========================================= .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). @@ -39,7 +39,7 @@ base class: initialized. If *host* is not specified, ``''`` (the local host) is used. If *port* is omitted, the standard IMAP4 port (143) is used. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout is used. + If timeout is not given or is ``None``, the global default socket timeout is used. The :class:`IMAP4` class supports the :keyword:`with` statement. When used like this, the IMAP4 ``LOGOUT`` command is issued automatically when the @@ -97,7 +97,7 @@ There's also a subclass for secure connections: best practices. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If timeout is not given or is None, the global default + connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. .. versionchanged:: 3.3 @@ -360,7 +360,7 @@ An :class:`IMAP4` instance has the following methods: Opens socket to *port* at *host*. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout + If timeout is not given or is ``None``, the global default socket timeout is used. Also note that if the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to reject creating a non-blocking socket. This method is implicitly called by the :class:`IMAP4` constructor. @@ -531,7 +531,7 @@ An :class:`IMAP4` instance has the following methods: allowed creation of such tags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also create such tags. Although it is an RFC violation and IMAP clients and - servers are supposed to be strict, imaplib nonetheless continues to allow + servers are supposed to be strict, imaplib still continues to allow such tags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. @@ -622,7 +622,7 @@ retrieves and prints all messages:: import getpass, imaplib - M = imaplib.IMAP4() + M = imaplib.IMAP4(host='example.org') M.login(getpass.getuser(), getpass.getpass()) M.select() typ, data = M.search(None, 'ALL') diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index d2cc769e2c8400..9c0879f5ca850f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -26,7 +26,7 @@ this package can eliminate the need to use the older and less efficient ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as -`pip <https://pypi.org/project/pip/>`_. +:pypi:`pip`. Specifically, it works with distributions with discoverable ``dist-info`` or ``egg-info`` directories, and metadata defined by the `Core metadata specifications <https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata>`_. @@ -41,7 +41,7 @@ and metadata defined by the `Core metadata specifications <https://packaging.pyt and one top-level *import package* may map to multiple *distribution packages* if it is a namespace package. - You can use :ref:`package_distributions() <package-distributions>` + You can use :ref:`packages_distributions() <package-distributions>` to get a mapping between them. By default, distribution metadata can live on the file system @@ -171,16 +171,18 @@ group. Read `the setuptools docs <https://setuptools.pypa.io/en/latest/userguide/entry_point.html>`_ for more information on entry points, their definition, and usage. -*Compatibility Note* - -The "selectable" entry points were introduced in ``importlib_metadata`` -3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted -no parameters and always returned a dictionary of entry points, keyed -by group. With ``importlib_metadata`` 5.0 and Python 3.12, -``entry_points`` always returns an ``EntryPoints`` object. See -`backports.entry_points_selectable <https://pypi.org/project/backports.entry-points-selectable>`_ -for compatibility options. +.. versionchanged:: 3.12 + The "selectable" entry points were introduced in ``importlib_metadata`` + 3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted + no parameters and always returned a dictionary of entry points, keyed + by group. With ``importlib_metadata`` 5.0 and Python 3.12, + ``entry_points`` always returns an ``EntryPoints`` object. See + :pypi:`backports.entry_points_selectable` + for compatibility options. +.. versionchanged:: 3.13 + ``EntryPoint`` objects no longer present a tuple-like interface + (:meth:`~object.__getitem__`). .. _metadata: @@ -217,7 +219,6 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. -.. versionadded:: 3.10 The ``json`` attribute was added. @@ -342,9 +343,17 @@ instance:: >>> dist.metadata['License'] # doctest: +SKIP 'MIT' +For editable packages, an ``origin`` property may present :pep:`610` +metadata:: + + >>> dist.origin.url + 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + The full set of available metadata is not described here. See the `Core metadata specifications <https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata>`_ for additional details. +.. versionadded:: 3.13 + The ``.origin`` property was added. Distribution Discovery ====================== @@ -397,6 +406,84 @@ metadata in locations other than the file system, subclass a custom finder, return instances of this derived ``Distribution`` in the ``find_distributions()`` method. +Example +------- + +Consider for example a custom finder that loads Python +modules from a database:: + + class DatabaseImporter(importlib.abc.MetaPathFinder): + def __init__(self, db): + self.db = db + + def find_spec(self, fullname, target=None) -> ModuleSpec: + return self.db.spec_from_name(fullname) + + sys.meta_path.append(DatabaseImporter(connect_db(...))) + +That importer now presumably provides importable modules from a +database, but it provides no metadata or entry points. For this +custom importer to provide metadata, it would also need to implement +``DistributionFinder``:: + + from importlib.metadata import DistributionFinder + + class DatabaseImporter(DistributionFinder): + ... + + def find_distributions(self, context=DistributionFinder.Context()): + query = dict(name=context.name) if context.name else {} + for dist_record in self.db.query_distributions(query): + yield DatabaseDistribution(dist_record) + +In this way, ``query_distributions`` would return records for +each distribution served by the database matching the query. For +example, if ``requests-1.0`` is in the database, ``find_distributions`` +would yield a ``DatabaseDistribution`` for ``Context(name='requests')`` +or ``Context(name=None)``. + +For the sake of simplicity, this example ignores ``context.path``\. The +``path`` attribute defaults to ``sys.path`` and is the set of import paths to +be considered in the search. A ``DatabaseImporter`` could potentially function +without any concern for a search path. Assuming the importer does no +partitioning, the "path" would be irrelevant. In order to illustrate the +purpose of ``path``, the example would need to illustrate a more complex +``DatabaseImporter`` whose behavior varied depending on +``sys.path``/``PYTHONPATH``. In that case, the ``find_distributions`` should +honor the ``context.path`` and only yield ``Distribution``\ s pertinent to that +path. + +``DatabaseDistribution``, then, would look something like:: + + class DatabaseDistribution(importlib.metadata.Distributon): + def __init__(self, record): + self.record = record + + def read_text(self, filename): + """ + Read a file like "METADATA" for the current distribution. + """ + if filename == "METADATA": + return f"""Name: {self.record.name} + Version: {self.record.version} + """ + if filename == "entry_points.txt": + return "\n".join( + f"""[{ep.group}]\n{ep.name}={ep.value}""" + for ep in self.record.entry_points) + + def locate_file(self, path): + raise RuntimeError("This distribution has no file system") + +This basic implementation should provide metadata and entry points for +packages served by the ``DatabaseImporter``, assuming that the +``record`` supplies suitable ``.name``, ``.version``, and +``.entry_points`` attributes. + +The ``DatabaseDistribution`` may also provide other metadata files, like +``RECORD`` (required for ``Distribution.files``) or override the +implementation of ``Distribution.files``. See the source for more inspiration. + .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index c508b6ba965cc0..5ea8044e1ec6ca 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources.abc` -- Abstract base classes for resources ---------------------------------------------------------------------- +:mod:`!importlib.resources.abc` -- Abstract base classes for resources +---------------------------------------------------------------------- .. module:: importlib.resources.abc :synopsis: Abstract base classes for resources @@ -103,19 +103,41 @@ .. abstractmethod:: is_dir() - Return True if self is a directory. + Return ``True`` if self is a directory. .. abstractmethod:: is_file() - Return True if self is a file. + Return ``True`` if self is a file. - .. abstractmethod:: joinpath(child) + .. abstractmethod:: joinpath(*pathsegments) - Return Traversable child in self. + Traverse directories according to *pathsegments* and return + the result as :class:`!Traversable`. + + Each *pathsegments* argument may contain multiple names separated by + forward slashes (``/``, ``posixpath.sep`` ). + For example, the following are equivalent:: + + files.joinpath('subdir', 'subsuddir', 'file.txt') + files.joinpath('subdir/subsuddir/file.txt') + + Note that some :class:`!Traversable` implementations + might not be updated to the latest version of the protocol. + For compatibility with such implementations, provide a single argument + without path separators to each call to ``joinpath``. For example:: + + files.joinpath('subdir').joinpath('subsubdir').joinpath('file.txt') + + .. versionchanged:: 3.11 + + ``joinpath`` accepts multiple *pathsegments*, and these segments + may contain forward slashes as path separators. + Previously, only a single *child* argument was accepted. .. abstractmethod:: __truediv__(child) Return Traversable child in self. + Equivalent to ``joinpath(child)``. .. abstractmethod:: open(mode='r', *args, **kwargs) diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 3de97e80311a17..e002198899c8b8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Package resource reading, opening and access --------------------------------------------------------------------------- +:mod:`!importlib.resources` -- Package resource reading, opening and access +--------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -50,7 +50,7 @@ for example, a package and its resources can be imported from a zip file using ``get_resource_reader(fullname)`` method as specified by :class:`importlib.resources.abc.ResourceReader`. -.. data:: Anchor +.. class:: Anchor Represents an anchor for resources, either a :class:`module object <types.ModuleType>` or a module name as a string. Defined as @@ -63,7 +63,7 @@ for example, a package and its resources can be imported from a zip file using (think files). A Traversable may contain other containers (think subdirectories). - *anchor* is an optional :data:`Anchor`. If the anchor is a + *anchor* is an optional :class:`Anchor`. If the anchor is a package, resources are resolved from that package. If a module, resources are resolved adjacent to that module (in the same package or the package root). If the anchor is omitted, the caller's module @@ -72,10 +72,10 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - "package" parameter was renamed to "anchor". "anchor" can now + *package* parameter was renamed to *anchor*. *anchor* can now be a non-package module and if omitted will default to the caller's - module. "package" is still accepted for compatibility but will raise - a DeprecationWarning. Consider passing the anchor positionally or + module. *package* is still accepted for compatibility but will raise + a :exc:`DeprecationWarning`. Consider passing the anchor positionally or using ``importlib_resources >= 5.10`` for a compatible interface on older Pythons. @@ -96,4 +96,182 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - Added support for ``traversable`` representing a directory. + Added support for *traversable* representing a directory. + + +.. _importlib_resources_functional: + +Functional API +^^^^^^^^^^^^^^ + +A set of simplified, backwards-compatible helpers is available. +These allow common operations in a single function call. + +For all the following functions: + +- *anchor* is an :class:`~importlib.resources.Anchor`, + as in :func:`~importlib.resources.files`. + Unlike in ``files``, it may not be omitted. + +- *path_names* are components of a resource's path name, relative to + the anchor. + For example, to get the text of resource named ``info.txt``, use:: + + importlib.resources.read_text(my_module, "info.txt") + + Like :meth:`Traversable.joinpath <importlib.resources.abc.Traversable>`, + The individual components should use forward slashes (``/``) + as path separators. + For example, the following are equivalent:: + + importlib.resources.read_binary(my_module, "pics/painting.png") + importlib.resources.read_binary(my_module, "pics", "painting.png") + + For backward compatibility reasons, functions that read text require + an explicit *encoding* argument if multiple *path_names* are given. + For example, to get the text of ``info/chapter1.txt``, use:: + + importlib.resources.read_text(my_module, "info", "chapter1.txt", + encoding='utf-8') + +.. function:: open_binary(anchor, *path_names) + + Open the named resource for binary reading. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + + This function returns a :class:`~typing.BinaryIO` object, + that is, a binary stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('rb') + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: open_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Open the named resource for text reading. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function returns a :class:`~typing.TextIO` object, + that is, a text stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('r', encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: read_binary(anchor, *path_names) + + Read and return the contents of the named resource as :class:`bytes`. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_bytes() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: read_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Read and return the contents of the named resource as :class:`str`. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_text(encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: path(anchor, *path_names) + + Provides the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary files created, e.g. + when the resource needs to be extracted from a zip file. + + For example, the :meth:`~pathlib.Path.stat` method requires + an actual file system path; it can be used like this:: + + with importlib.resources.path(anchor, "resource.txt") as fspath: + result = fspath.stat() + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + as_file(files(anchor).joinpath(*path_names)) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: is_resource(anchor, *path_names) + + Return ``True`` if the named resource exists, otherwise ``False``. + This function does not consider directories to be resources. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).is_file() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: contents(anchor, *path_names) + + Return an iterable over the named items within the package or path. + The iterable returns names of resources (e.g. files) and non-resources + (e.g. directories) as :class:`str`. + The iterable does not recurse into subdirectories. + + See :ref:`the introduction <importlib_resources_functional>` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + for resource in files(anchor).joinpath(*path_names).iterdir(): + yield resource.name + + .. deprecated:: 3.11 + Prefer ``iterdir()`` as above, which offers more control over the + results and richer functionality. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index fc954724bb72fe..2ec15dd171c18a 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -265,7 +265,7 @@ ABC hierarchy:: when invalidating the caches of all finders on :data:`sys.meta_path`. .. versionchanged:: 3.4 - Returns ``None`` when called instead of ``NotImplemented``. + Returns ``None`` when called instead of :data:`NotImplemented`. .. class:: PathEntryFinder @@ -1145,7 +1145,7 @@ find and load modules. .. versionadded:: 3.4 -.. class:: NamespaceLoader(name, path, path_finder): +.. class:: NamespaceLoader(name, path, path_finder) A concrete implementation of :class:`importlib.abc.InspectLoader` for namespace packages. This is an alias for a private class and is only made @@ -1241,6 +1241,69 @@ find and load modules. and how the module's :attr:`__file__` is populated. +.. class:: AppleFrameworkLoader(name, path) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be dynamic libraries, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + To accommodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traversal. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: name + + Name of the module the loader supports. + + .. attribute:: path + + Path to the ``.fwork`` file for the extension module. + + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3efd3be5909254..7838eeed2843c4 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,5 +1,10 @@ -:mod:`inspect` --- Inspect live objects -======================================= +:mod:`!inspect` --- Inspect live objects +======================================== + +.. testsetup:: * + + import inspect + from inspect import * .. module:: inspect :synopsis: Extract information and source code from live objects. @@ -37,210 +42,233 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. this function name is too big to fit in the ascii-art table below .. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` -+-----------+-------------------+---------------------------+ -| Type | Attribute | Description | -+===========+===================+===========================+ -| class | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-------------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this method was defined | -+-----------+-------------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-------------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-------------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-------------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| | __builtins__ | builtins namespace | -+-----------+-------------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-------------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-------------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-------------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-------------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-------------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-------------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including keyword only | -| | | arguments, \* or \*\* | -| | | args) | -+-----------+-------------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-------------------+---------------------------+ -| | co_cellvars | tuple of names of cell | -| | | variables (referenced by | -| | | containing scopes) | -+-----------+-------------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-------------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-------------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | <inspect-module-co-flags>`| -+-----------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-------------------+---------------------------+ -| | co_freevars | tuple of names of free | -| | | variables (referenced via | -| | | a function's closure) | -+-----------+-------------------+---------------------------+ -| | co_posonlyargcount| number of positional only | -| | | arguments | -+-----------+-------------------+---------------------------+ -| | co_kwonlyargcount | number of keyword only | -| | | arguments (not including | -| | | \*\* arg) | -+-----------+-------------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-------------------+---------------------------+ -| | co_qualname | fully qualified name with | -| | | which this code object | -| | | was defined | -+-----------+-------------------+---------------------------+ -| | co_names | tuple of names other | -| | | than arguments and | -| | | function locals | -+-----------+-------------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-------------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-------------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-------------------+---------------------------+ -| generator | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | gi_frame | frame | -+-----------+-------------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-------------------+---------------------------+ -| | gi_code | code | -+-----------+-------------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-------------------+---------------------------+ -| | cr_frame | frame | -+-----------+-------------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-------------------+---------------------------+ -| | cr_code | code | -+-----------+-------------------+---------------------------+ -| | cr_origin | where coroutine was | -| | | created, or ``None``. See | -| | | |coroutine-origin-link| | -+-----------+-------------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ ++-----------------+-------------------+---------------------------+ +| Type | Attribute | Description | ++=================+===================+===========================+ +| class | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | <type-params>` of | +| | | a generic class | ++-----------------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this method was defined | ++-----------------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| | __builtins__ | builtins namespace | ++-----------------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | <type-params>` of | +| | | a generic function | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | <inspect-module-co-flags>`| ++-----------------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------------+-------------------+---------------------------+ +| | co_posonlyargcount| number of positional only | +| | | arguments | ++-----------------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------------+-------------------+---------------------------+ +| | co_qualname | fully qualified name with | +| | | which this code object | +| | | was defined | ++-----------------+-------------------+---------------------------+ +| | co_names | tuple of names other | +| | | than arguments and | +| | | function locals | ++-----------------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | gi_code | code | ++-----------------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| async generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | ag_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | ag_frame | frame | ++-----------------+-------------------+---------------------------+ +| | ag_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | ag_code | code | ++-----------------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------------+-------------------+---------------------------+ +| | cr_code | code | ++-----------------+-------------------+---------------------------+ +| | cr_origin | where coroutine was | +| | | created, or ``None``. See | +| | | |coroutine-origin-link| | ++-----------------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -268,7 +296,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): :func:`getmembers` will only return class attributes defined in the metaclass when the argument is a class and those attributes have been - listed in the metaclass' custom :meth:`__dir__`. + listed in the metaclass' custom :meth:`~object.__dir__`. .. function:: getmembers_static(object[, predicate]) @@ -335,6 +363,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a Python generator function. .. function:: isgenerator(object) @@ -358,6 +389,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Sync functions marked with :func:`markcoroutinefunction` now return ``True``. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. + .. function:: markcoroutinefunction(func) @@ -387,7 +422,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object can be used in :keyword:`await` expression. Can also be used to distinguish generator-based coroutines from regular - generators:: + generators: + + .. testcode:: + + import types def gen(): yield @@ -404,13 +443,15 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isasyncgenfunction(object) Return ``True`` if the object is an :term:`asynchronous generator` function, - for example:: + for example: + + .. doctest:: - >>> async def agen(): - ... yield 1 - ... - >>> inspect.isasyncgenfunction(agen) - True + >>> async def agen(): + ... yield 1 + ... + >>> inspect.isasyncgenfunction(agen) + True .. versionadded:: 3.6 @@ -418,6 +459,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a :term:`asynchronous generator` function. + .. versionchanged:: 3.13 + Functions wrapped in :func:`functools.partialmethod` now return ``True`` + if the wrapped function is a :term:`coroutine function`. .. function:: isasyncgen(object) @@ -473,15 +517,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): are true. This, for example, is true of ``int.__add__``. An object passing this test - has a :meth:`~object.__get__` method but not a :meth:`~object.__set__` - method, but beyond that the set of attributes varies. A - :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`__doc__` often is. + has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` + method or a :meth:`~object.__delete__` method. Beyond that, the set of + attributes varies. A :attr:`~definition.__name__` attribute is usually + sensible, and :attr:`!__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the other tests promise more -- you can, e.g., count on having the - :attr:`__func__` attribute (etc) when an object passes :func:`ismethod`. + :attr:`~method.__func__` attribute (etc) when an object passes + :func:`ismethod`. + + .. versionchanged:: 3.13 + This function no longer incorrectly reports objects with :meth:`~object.__get__` + and :meth:`~object.__delete__`, but not :meth:`~object.__set__`, as being method + descriptors (such objects are data descriptors, not method descriptors). .. function:: isdatadescriptor(object) @@ -492,7 +542,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Examples are properties (defined in Python), getsets, and members. The latter two are defined in C and there are more specific tests available for those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`__doc__` attributes + descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed. @@ -614,13 +664,16 @@ Introspecting callables with the Signature object .. versionadded:: 3.3 -The Signature object represents the call signature of a callable object and its -return annotation. To retrieve a Signature object, use the :func:`signature` +The :class:`Signature` object represents the call signature of a callable object +and its return annotation. To retrieve a :class:`!Signature` object, +use the :func:`!signature` function. .. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) - Return a :class:`Signature` object for the given ``callable``:: + Return a :class:`Signature` object for the given *callable*: + + .. doctest:: >>> from inspect import signature >>> def foo(a, *, b:int, **kwargs): @@ -629,10 +682,10 @@ function. >>> sig = signature(foo) >>> str(sig) - '(a, *, b:int, **kwargs)' + '(a, *, b: int, **kwargs)' >>> str(sig.parameters['b']) - 'b:int' + 'b: int' >>> sig.parameters['b'].annotation <class 'int'> @@ -643,29 +696,30 @@ function. For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`inspect.get_annotations()`. The - ``global``, ``locals``, and ``eval_str`` parameters are passed - into :func:`inspect.get_annotations()` when resolving the - annotations; see the documentation for :func:`inspect.get_annotations()` + :func:`get_annotations`. The + *globals*, *locals*, and *eval_str* parameters are passed + into :func:`get_annotations` when resolving the + annotations; see the documentation for :func:`get_annotations` for instructions on how to use these parameters. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, - if the annotations are stringized, and ``eval_str`` is not false, - the ``eval()`` call(s) to un-stringize the annotations could - potentially raise any kind of exception. + if the annotations are stringized, and *eval_str* is not false, + the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + could potentially raise any kind of exception. A slash(/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters <faq-positional-only-arguments>`. - .. versionadded:: 3.5 - ``follow_wrapped`` parameter. Pass ``False`` to get a signature of - ``callable`` specifically (``callable.__wrapped__`` will not be used to + .. versionchanged:: 3.5 + The *follow_wrapped* parameter was added. + Pass ``False`` to get a signature of + *callable* specifically (``callable.__wrapped__`` will not be used to unwrap decorated callables.) - .. versionadded:: 3.10 - ``globals``, ``locals``, and ``eval_str`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. note:: @@ -673,10 +727,18 @@ function. Python. For example, in CPython, some built-in functions defined in C provide no metadata about their arguments. + .. impl-detail:: + + If the passed object has a :attr:`!__signature__` attribute, + we may use it to create the signature. + The exact semantics are an implementation detail and are subject to + unannounced changes. Consult the source code for current semantics. + .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) - A Signature object represents the call signature of a function and its return + A :class:`!Signature` object represents the call signature of a function + and its return annotation. For each parameter accepted by the function it stores a :class:`Parameter` object in its :attr:`parameters` collection. @@ -686,14 +748,14 @@ function. positional-only first, then positional-or-keyword, and that parameters with defaults follow parameters without defaults. - The optional *return_annotation* argument, can be an arbitrary Python object, - is the "return" annotation of the callable. + The optional *return_annotation* argument can be an arbitrary Python object. + It represents the "return" annotation of the callable. - Signature objects are *immutable*. Use :meth:`Signature.replace` or + :class:`!Signature` objects are *immutable*. Use :meth:`Signature.replace` or :func:`copy.replace` to make a modified copy. .. versionchanged:: 3.5 - Signature objects are picklable and :term:`hashable`. + :class:`!Signature` objects are now picklable and :term:`hashable`. .. attribute:: Signature.empty @@ -730,13 +792,15 @@ function. .. method:: Signature.replace(*[, parameters][, return_annotation]) - Create a new Signature instance based on the instance :meth:`replace` was invoked - on. It is possible to pass different ``parameters`` and/or - ``return_annotation`` to override the corresponding properties of the base - signature. To remove return_annotation from the copied Signature, pass in + Create a new :class:`Signature` instance based on the instance + :meth:`replace` was invoked on. + It is possible to pass different *parameters* and/or + *return_annotation* to override the corresponding properties of the base + signature. To remove ``return_annotation`` from the copied + :class:`!Signature`, pass in :attr:`Signature.empty`. - :: + .. doctest:: >>> def test(a, b): ... pass @@ -746,36 +810,50 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" - Signature objects are also supported by generic function + :class:`Signature` objects are also supported by the generic function :func:`copy.replace`. - .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globalns=None, localns=None) + .. method:: format(*, max_width=None) + + Create a string representation of the :class:`Signature` object. + + If *max_width* is passed, the method will attempt to fit + the signature into lines of at most *max_width* characters. + If the signature is longer than *max_width*, + all parameters will be on separate lines. + + .. versionadded:: 3.13 + + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable - ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj`` - without unwrapping its ``__wrapped__`` chain. ``globalns`` and - ``localns`` will be used as the namespaces when resolving annotations. + *obj*. - This method simplifies subclassing of :class:`Signature`:: + This method simplifies subclassing of :class:`Signature`: - class MySignature(Signature): - pass - sig = MySignature.from_callable(min) - assert isinstance(sig, MySignature) + .. testcode:: + + class MySignature(Signature): + pass + sig = MySignature.from_callable(sum) + assert isinstance(sig, MySignature) + + Its behavior is otherwise identical to that of :func:`signature`. .. versionadded:: 3.5 - .. versionadded:: 3.10 - ``globalns`` and ``localns`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) - Parameter objects are *immutable*. Instead of modifying a Parameter object, + :class:`!Parameter` objects are *immutable*. + Instead of modifying a :class:`!Parameter` object, you can use :meth:`Parameter.replace` or :func:`copy.replace` to create a modified copy. .. versionchanged:: 3.5 - Parameter objects are picklable and :term:`hashable`. + Parameter objects are now picklable and :term:`hashable`. .. attribute:: Parameter.empty @@ -794,7 +872,7 @@ function. expressions. .. versionchanged:: 3.6 - These parameter names are exposed by this module as names like + These parameter names are now exposed by this module as names like ``implicit0``. .. attribute:: Parameter.default @@ -844,7 +922,9 @@ function. | | definition. | +------------------------+----------------------------------------------+ - Example: print all keyword-only arguments without default values:: + Example: print all keyword-only arguments without default values: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -858,11 +938,13 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of Parameter.kind. + Describes a enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 - Example: print all descriptions of arguments:: + Example: print all descriptions of arguments: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -877,12 +959,12 @@ function. .. method:: Parameter.replace(*[, name][, kind][, default][, annotation]) - Create a new Parameter instance based on the instance replaced was invoked - on. To override a :class:`Parameter` attribute, pass the corresponding + Create a new :class:`Parameter` instance based on the instance replaced was invoked + on. To override a :class:`!Parameter` attribute, pass the corresponding argument. To remove a default value or/and an annotation from a - Parameter, pass :attr:`Parameter.empty`. + :class:`!Parameter`, pass :attr:`Parameter.empty`. - :: + .. doctest:: >>> from inspect import Parameter >>> param = Parameter('foo', Parameter.KEYWORD_ONLY, default=42) @@ -893,12 +975,13 @@ function. 'foo=42' >>> str(param.replace(default=Parameter.empty, annotation='spam')) - "foo:'spam'" + "foo: 'spam'" - Parameter objects are also supported by generic function :func:`copy.replace`. + :class:`Parameter` objects are also supported by the generic function + :func:`copy.replace`. .. versionchanged:: 3.4 - In Python 3.3 Parameter objects were allowed to have ``name`` set + In Python 3.3 :class:`Parameter` objects were allowed to have ``name`` set to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. This is no longer permitted. @@ -951,18 +1034,20 @@ function. For variable-keyword arguments (``**kwargs``) the default is an empty dict. - :: + .. doctest:: - >>> def foo(a, b='ham', *args): pass - >>> ba = inspect.signature(foo).bind('spam') - >>> ba.apply_defaults() - >>> ba.arguments - {'a': 'spam', 'b': 'ham', 'args': ()} + >>> def foo(a, b='ham', *args): pass + >>> ba = inspect.signature(foo).bind('spam') + >>> ba.apply_defaults() + >>> ba.arguments + {'a': 'spam', 'b': 'ham', 'args': ()} .. versionadded:: 3.5 The :attr:`args` and :attr:`kwargs` properties can be used to invoke - functions:: + functions: + + .. testcode:: def test(a, *, b): ... @@ -1081,20 +1166,22 @@ Classes and functions ``**`` arguments, if any) to their values from *args* and *kwds*. In case of invoking *func* incorrectly, i.e. whenever ``func(*args, **kwds)`` would raise an exception because of incompatible signature, an exception of the same type - and the same or similar message is raised. For example:: - - >>> from inspect import getcallargs - >>> def f(a, b=1, *pos, **named): - ... pass - ... - >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} - True - >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} - True - >>> getcallargs(f) - Traceback (most recent call last): - ... - TypeError: f() missing 1 required positional argument: 'a' + and the same or similar message is raised. For example: + + .. doctest:: + + >>> from inspect import getcallargs + >>> def f(a, b=1, *pos, **named): + ... pass + ... + >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} + True + >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} + True + >>> getcallargs(f) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' .. versionadded:: 3.2 @@ -1180,9 +1267,10 @@ Classes and functions * If ``obj`` is a class, ``globals`` defaults to ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to ``obj.__globals__``, + * If ``obj`` is a callable, ``globals`` defaults to + :attr:`obj.__globals__ <function.__globals__>`, although if ``obj`` is a wrapped function (using - ``functools.update_wrapper()``) it is first unwrapped. + :func:`functools.update_wrapper`) it is first unwrapped. Calling ``get_annotations`` is best practice for accessing the annotations dict of any object. See :ref:`annotations-howto` for @@ -1396,7 +1484,8 @@ Fetching attributes statically Both :func:`getattr` and :func:`hasattr` can trigger code execution when fetching or checking for the existence of attributes. Descriptors, like -properties, will be invoked and :meth:`__getattr__` and :meth:`__getattribute__` +properties, will be invoked and :meth:`~object.__getattr__` and +:meth:`~object.__getattribute__` may be called. For cases where you want passive introspection, like documentation tools, this @@ -1406,7 +1495,8 @@ but avoids executing code when it fetches attributes. .. function:: getattr_static(obj, attr, default=None) Retrieve attributes without triggering dynamic lookup via the - descriptor protocol, :meth:`__getattr__` or :meth:`__getattribute__`. + descriptor protocol, :meth:`~object.__getattr__` + or :meth:`~object.__getattribute__`. Note: this function may not be able to retrieve all attributes that getattr can fetch (like dynamically created attributes) @@ -1549,8 +1639,8 @@ updated as expected: Code Objects Bit Flags ---------------------- -Python code objects have a ``co_flags`` attribute, which is a bitmap of -the following flags: +Python code objects have a :attr:`~codeobject.co_flags` attribute, +which is a bitmap of the following flags: .. data:: CO_OPTIMIZED @@ -1558,8 +1648,8 @@ the following flags: .. data:: CO_NEWLOCALS - If set, a new dict will be created for the frame's ``f_locals`` when - the code object is executed. + If set, a new dict will be created for the frame's :attr:`~frame.f_locals` + when the code object is executed. .. data:: CO_VARARGS diff --git a/Doc/library/intro.rst b/Doc/library/intro.rst index 5a4c9b8b16ab3b..ffc8939d21157d 100644 --- a/Doc/library/intro.rst +++ b/Doc/library/intro.rst @@ -58,7 +58,7 @@ Notes on availability operating system. * If not separately noted, all functions that claim "Availability: Unix" are - supported on macOS, which builds on a Unix core. + supported on macOS and iOS, both of which build on a Unix core. * If an availability note contains both a minimum Kernel version and a minimum libc version, then both conditions must hold. For example a feature with note @@ -119,3 +119,44 @@ DOM APIs as well as limited networking capabilities with JavaScript's .. _wasmtime: https://wasmtime.dev/ .. _Pyodide: https://pyodide.org/ .. _PyScript: https://pyscript.net/ + +.. _iOS-availability: + +iOS +--- + +iOS is, in most respects, a POSIX operating system. File I/O, socket handling, +and threading all behave as they would on any POSIX operating system. However, +there are several major differences between iOS and other POSIX systems. + +* iOS can only use Python in "embedded" mode. There is no Python REPL, and no + ability to execute binaries that are part of the normal Python developer + experience, such as :program:`pip`. To add Python code to your iOS app, you must use + the :ref:`Python embedding API <embedding>` to add a Python interpreter to an + iOS app created with Xcode. See the :ref:`iOS usage guide <using-ios>` for + more details. + +* An iOS app cannot use any form of subprocessing, background processing, or + inter-process communication. If an iOS app attempts to create a subprocess, + the process creating the subprocess will either lock up, or crash. An iOS app + has no visibility of other applications that are running, nor any ability to + communicate with other running applications, outside of the iOS-specific APIs + that exist for this purpose. + +* iOS apps have limited access to modify system resources (such as the system + clock). These resources will often be *readable*, but attempts to modify + those resources will usually fail. + +* iOS apps have a limited concept of console input and output. ``stdout`` and + ``stderr`` *exist*, and content written to ``stdout`` and ``stderr`` will be + visible in logs when running in Xcode, but this content *won't* be recorded + in the system log. If a user who has installed your app provides their app + logs as a diagnostic aid, they will not include any detail written to + ``stdout`` or ``stderr``. + + iOS apps have no concept of ``stdin`` at all. While iOS apps can have a + keyboard, this is a software feature, not something that is attached to + ``stdin``. + + As a result, Python library that involve console manipulation (such as + :mod:`curses` and :mod:`readline`) are not available on iOS. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 6736aa9ee2b0ef..748c49968f505c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1,5 +1,5 @@ -:mod:`io` --- Core tools for working with streams -================================================= +:mod:`!io` --- Core tools for working with streams +================================================== .. module:: io :synopsis: Core tools for working with streams. @@ -466,7 +466,7 @@ I/O Base Classes .. class:: RawIOBase - Base class for raw binary streams. It inherits :class:`IOBase`. + Base class for raw binary streams. It inherits from :class:`IOBase`. Raw binary streams typically provide low-level access to an underlying OS device or API, and do not try to encapsulate it in high-level primitives @@ -519,7 +519,7 @@ I/O Base Classes .. class:: BufferedIOBase Base class for binary streams that support some kind of buffering. - It inherits :class:`IOBase`. + It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, :meth:`readinto` and :meth:`write` will try (respectively) to read as much @@ -633,7 +633,7 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits :class:`RawIOBase`. + inherits from :class:`RawIOBase`. The *name* can be one of two things: @@ -696,7 +696,7 @@ than raw I/O does. .. class:: BytesIO(initial_bytes=b'') - A binary stream using an in-memory bytes buffer. It inherits + A binary stream using an in-memory bytes buffer. It inherits from :class:`BufferedIOBase`. The buffer is discarded when the :meth:`~IOBase.close` method is called. @@ -745,7 +745,7 @@ than raw I/O does. .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a readable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When reading data from this object, a larger amount of data may be @@ -783,7 +783,7 @@ than raw I/O does. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non - seekable :class:`RawIOBase` raw binary stream. It inherits + seekable :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedIOBase`. When writing to this object, data is normally placed into an internal @@ -818,7 +818,7 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits :class:`BufferedReader` + :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` and :class:`BufferedWriter`. The constructor creates a reader and writer for a seekable raw stream, given @@ -834,7 +834,7 @@ than raw I/O does. A buffered binary stream providing higher-level access to two non seekable :class:`RawIOBase` raw binary streams---one readable, the other writeable. - It inherits :class:`BufferedIOBase`. + It inherits from :class:`BufferedIOBase`. *reader* and *writer* are :class:`RawIOBase` objects that are readable and writeable respectively. If the *buffer_size* is omitted it defaults to @@ -857,7 +857,7 @@ Text I/O .. class:: TextIOBase Base class for text streams. This class provides a character and line based - interface to stream I/O. It inherits :class:`IOBase`. + interface to stream I/O. It inherits from :class:`IOBase`. :class:`TextIOBase` provides or overrides these data attributes and methods in addition to those from :class:`IOBase`: @@ -946,7 +946,7 @@ Text I/O line_buffering=False, write_through=False) A buffered text stream providing higher-level access to a - :class:`BufferedIOBase` buffered binary stream. It inherits + :class:`BufferedIOBase` buffered binary stream. It inherits from :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or @@ -1073,7 +1073,7 @@ Text I/O .. class:: StringIO(initial_value='', newline='\n') - A text stream using an in-memory text buffer. It inherits + A text stream using an in-memory text buffer. It inherits from :class:`TextIOBase`. The text buffer is discarded when the :meth:`~IOBase.close` method is @@ -1124,7 +1124,7 @@ Text I/O .. class:: IncrementalNewlineDecoder A helper codec that decodes newlines for :term:`universal newlines` mode. - It inherits :class:`codecs.IncrementalDecoder`. + It inherits from :class:`codecs.IncrementalDecoder`. Performance diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9c2dff55703273..f58c0ea75a4753 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -1,5 +1,5 @@ -:mod:`ipaddress` --- IPv4/IPv6 manipulation library -=================================================== +:mod:`!ipaddress` --- IPv4/IPv6 manipulation library +==================================================== .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. @@ -121,22 +121,12 @@ write code that handles both IP versions correctly. Address objects are Leading zeros are tolerated, even in ambiguous cases that look like octal notation. - .. versionchanged:: 3.10 + .. versionchanged:: 3.9.5 Leading zeros are no longer tolerated and are treated as an error. IPv4 address strings are now parsed as strict as glibc :func:`~socket.inet_pton`. - .. versionchanged:: 3.9.5 - - The above change was also included in Python 3.9 starting with - version 3.9.5. - - .. versionchanged:: 3.8.12 - - The above change was also included in Python 3.8 starting with - version 3.8.12. - .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. @@ -188,18 +178,53 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives. + + * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and + ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). + * ``64:ff9b:1::/48`` is considered private. + * ``2002::/16`` is considered private. + * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, + ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. + The exceptions are not considered private. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives, see :attr:`is_private` for details. + .. attribute:: is_unspecified ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) @@ -219,6 +244,13 @@ write code that handles both IP versions correctly. Address objects are ``True`` if the address is reserved for link-local usage. See :RFC:`3927`. + .. attribute:: ipv6_mapped + + :class:`IPv4Address` object representing the IPv4-mapped IPv6 address. See :RFC:`4291`. + + .. versionadded:: 3.13 + + .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml @@ -302,14 +334,14 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_multicast .. attribute:: is_private .. attribute:: is_global + + .. versionadded:: 3.4 + .. attribute:: is_unspecified .. attribute:: is_reserved .. attribute:: is_loopback .. attribute:: is_link_local - .. versionadded:: 3.4 - is_global - .. attribute:: is_site_local ``True`` if the address is reserved for site-local usage. Note that @@ -472,7 +504,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 32-bits integer, a 4-bytes packed - integer, or an existing IPv4Address object; and the netmask is either + integer, or an existing :class:`IPv4Address` object; and the netmask is either an integer representing the prefix length (e.g. ``24``) or a string representing the prefix mask (e.g. ``255.255.255.0``). @@ -693,7 +725,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 128-bits integer, a 16-bytes packed - integer, or an existing IPv6Address object; and the netmask is an + integer, or an existing :class:`IPv6Address` object; and the netmask is an integer representing the prefix length. An :exc:`AddressValueError` is raised if *address* is not a valid IPv6 @@ -749,7 +781,7 @@ dictionaries. .. attribute:: is_site_local - These attribute is true for the network as a whole if it is true + This attribute is true for the network as a whole if it is true for both the network address and the broadcast address. @@ -958,7 +990,7 @@ The module also provides the following module level functions: .. function:: collapse_addresses(addresses) Return an iterator of the collapsed :class:`IPv4Network` or - :class:`IPv6Network` objects. *addresses* is an iterator of + :class:`IPv6Network` objects. *addresses* is an :term:`iterable` of :class:`IPv4Network` or :class:`IPv6Network` objects. A :exc:`TypeError` is raised if *addresses* contains mixed version objects. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5846d784c88ccc..1fdd00a69da1f3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1,5 +1,5 @@ -:mod:`itertools` --- Functions creating iterators for efficient looping -======================================================================= +:mod:`!itertools` --- Functions creating iterators for efficient looping +======================================================================== .. module:: itertools :synopsis: Functions creating iterators for efficient looping. @@ -41,9 +41,9 @@ operator can be mapped across two vectors to form an efficient dot-product: ================== ================= ================================================= ========================================= Iterator Arguments Results Example ================== ================= ================================================= ========================================= -:func:`count` start, [step] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` ================== ================= ================================================= ========================================= **Iterators terminating on the shortest input sequence:** @@ -51,20 +51,20 @@ Iterator Arguments Results ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` +:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` +:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` +:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','ABC'], len) → (1, A B) (3, ABC)`` +:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` +:func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` +:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= **Combinatoric iterators:** @@ -84,72 +84,70 @@ Examples Results ``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` ``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` ``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` +``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` ============================================== ============================================================= .. _itertools-functions: -Itertool functions +Itertool Functions ------------------ The following module functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. -.. function:: accumulate(iterable[, func, *, initial=None]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). +.. function:: accumulate(iterable[, function, *, initial=None]) - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) + Make an iterator that returns accumulated sums or accumulated + results from other binary functions. - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. + The *function* defaults to addition. The *function* should accept + two arguments, an accumulated total and a value from the *iterable*. + + If an *initial* value is provided, the accumulation will start with + that value and the output will have one more element than the input + iterable. Roughly equivalent to:: - def accumulate(iterable, func=operator.add, *, initial=None): + def accumulate(iterable, function=operator.add, *, initial=None): 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 - it = iter(iterable) + # accumulate([1,2,3,4,5]) → 1 3 6 10 15 + # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 + # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 + + iterator = iter(iterable) total = initial if initial is None: try: - total = next(it) + total = next(iterator) except StopIteration: return + yield total - for element in it: - total = func(total, element) + for element in iterator: + total = function(total, element) yield total - There are a number of uses for the *func* argument. It can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: + The *function* argument can be set to :func:`min` for a running + minimum, :func:`max` for a running maximum, or :func:`operator.mul` + for a running product. `Amortization tables + <https://www.ramseysolutions.com/real-estate/amortization-schedule>`_ + can be built by accumulating interest and applying payments: .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] + >>> list(accumulate(data, operator.mul)) # running product + [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] # Amortize a 5% loan of 1000 with 10 annual payments of 90 - >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt - >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + >>> update = lambda balance, payment: round(balance * 1.05) - payment + >>> list(accumulate(repeat(90, 10), update, initial=1_000)) [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the @@ -158,17 +156,20 @@ loops that truncate the stream. .. versionadded:: 3.2 .. versionchanged:: 3.3 - Added the optional *func* parameter. + Added the optional *function* parameter. .. versionchanged:: 3.8 Added the optional *initial* parameter. -.. function:: batched(iterable, n) +.. function:: batched(iterable, n, *, strict=False) Batch data from the *iterable* into tuples of length *n*. The last batch may be shorter than *n*. + If *strict* is true, will raise a :exc:`ValueError` if the final + batch is shorter than *n*. + Loops over the input iterable and accumulates data into tuples up to size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input @@ -181,25 +182,23 @@ loops that truncate the stream. >>> unflattened [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] - >>> for batch in batched('ABCDEFG', 3): - ... print(batch) - ... - ('A', 'B', 'C') - ('D', 'E', 'F') - ('G',) - Roughly equivalent to:: - def batched(iterable, n): - # batched('ABCDEFG', 3) --> ABC DEF G + def batched(iterable, n, *, strict=False): + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): + if strict and len(batch) != n: + raise ValueError('batched(): incomplete batch') yield batch .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *strict* option. + .. function:: chain(*iterables) @@ -209,10 +208,9 @@ loops that truncate the stream. Roughly equivalent to:: def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain('ABC', 'DEF') → A B C D E F + for iterable in iterables: + yield from iterable .. classmethod:: chain.from_iterable(iterable) @@ -221,34 +219,40 @@ loops that truncate the stream. single iterable argument that is evaluated lazily. Roughly equivalent to:: def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain.from_iterable(['ABC', 'DEF']) → A B C D E F + for iterable in iterables: + yield from iterable .. function:: combinations(iterable, r) Return *r* length subsequences of elements from the input *iterable*. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` keeping only entries that + are subsequences of the *iterable*. The length of the output is given + by :func:`math.comb` which computes ``n! / r! / (n - r)!`` when ``0 ≤ r + ≤ n`` or zero when ``r > n``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. + value. If the input elements are unique, there will be no repeated + values within each combination. Roughly equivalent to:: def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 + # combinations('ABCD', 2) → AB AC AD BC BD CD + # combinations(range(4), 3) → 012 013 023 123 + pool = tuple(iterable) n = len(pool) if r > n: return indices = list(range(r)) + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -261,42 +265,36 @@ loops that truncate the stream. indices[j] = indices[j-1] + 1 yield tuple(pool[i] for i in indices) - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: combinations_with_replacement(iterable, r) Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` that keeps only entries + that are subsequences (with possible repeated elements) of the + *iterable*. The number of subsequence returned is ``(n + r - 1)! / r! / + (n - 1)!`` when ``n > 0``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations + value. If the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC + # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC + pool = tuple(iterable) n = len(pool) if not n and r: return indices = [0] * r + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -307,45 +305,33 @@ loops that truncate the stream. indices[i:] = [indices[i] + 1] * (r - i) yield tuple(pool[i] for i in indices) - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - .. versionadded:: 3.1 .. function:: compress(data, selectors) - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. - Roughly equivalent to:: + Make an iterator that returns elements from *data* where the + corresponding element in *selectors* is true. Stops when either the + *data* or *selectors* iterables have been exhausted. Roughly + equivalent to:: def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F - return (d for d, s in zip(data, selectors) if s) + # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F + return (datum for datum, selector in zip(data, selectors) if selector) .. versionadded:: 3.1 .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: + Make an iterator that returns evenly spaced values beginning with + *start*. Can be used with :func:`map` to generate consecutive data + points or with :func:`zip` to add sequence numbers. Roughly + equivalent to:: def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) --> 2.5 3.0 3.5 ... + # count(10) → 10 11 12 13 14 ... + # count(2.5, 0.5) → 2.5 3.0 3.5 ... n = start while True: yield n @@ -358,51 +344,58 @@ loops that truncate the stream. .. versionchanged:: 3.1 Added *step* argument and allowed non-integer arguments. + .. function:: cycle(iterable) - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: + Make an iterator returning elements from the *iterable* and saving a + copy of each. When the iterable is exhausted, return elements from + the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... + # cycle('ABCD') → A B C D A B C D A B C D ... saved = [] for element in iterable: yield element saved.append(element) while saved: for element in saved: - yield element + yield element - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). + This itertool may require significant auxiliary storage (depending on + the length of the iterable). .. function:: dropwhile(predicate, iterable) - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: + Make an iterator that drops elements from the *iterable* while the + *predicate* is true and afterwards returns every element. Roughly + equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 - iterable = iter(iterable) - for x in iterable: + # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 + + iterator = iter(iterable) + for x in iterator: if not predicate(x): yield x break - for x in iterable: + + for x in iterator: yield x + Note this does not produce *any* output until the predicate first + becomes false, so this itertool may have a lengthy start-up time. + + .. function:: filterfalse(predicate, iterable) - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: + Make an iterator that filters elements from the *iterable* returning + only those for which the *predicate* returns a false value. If + *predicate* is ``None``, returns the items that are false. Roughly + equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 + # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 if predicate is None: predicate = bool for x in iterable: @@ -438,82 +431,77 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: - class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D - - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: return - self.currkey = self.keyfunc(self.currvalue) + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass .. function:: islice(iterable, stop) islice(iterable, start, stop[, step]) - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. + Make an iterator that returns selected elements from the iterable. + Works like sequence slicing but does not support negative values for + *start*, *stop*, or *step*. + + If *start* is zero or ``None``, iteration starts at zero. Otherwise, + elements from the iterable are skipped until *start* is reached. - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. + If *stop* is ``None``, iteration continues until the iterator is + exhausted, if at all. Otherwise, it stops at the specified position. - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). + If *step* is ``None``, the step defaults to one. Elements are returned + consecutively unless *step* is set higher than one which results in + items being skipped. Roughly equivalent to:: def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step .. function:: pairwise(iterable) @@ -527,43 +515,54 @@ loops that truncate the stream. Roughly equivalent to:: def pairwise(iterable): - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) + # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) + a = next(iterator, None) + for b in iterator: + yield a, b + a = b .. versionadded:: 3.10 .. function:: permutations(iterable, r=None) - Return successive *r* length permutations of elements in the *iterable*. + Return successive *r* length `permutations of elements + <https://www.britannica.com/science/permutation>`_ from the *iterable*. If *r* is not specified or is ``None``, then *r* defaults to the length of the *iterable* and all possible full-length permutations are generated. + The output is a subsequence of :func:`product` where entries with + repeated elements have been filtered out. The length of the output is + given by :func:`math.perm` which computes ``n! / (n - r)!`` when + ``0 ≤ r ≤ n`` or zero when ``r > n``. + The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 + # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC + # permutations(range(3)) → 012 021 102 120 201 210 + pool = tuple(iterable) n = len(pool) r = n if r is None else r if r > n: return + indices = list(range(n)) cycles = list(range(n, n-r, -1)) yield tuple(pool[i] for i in indices[:r]) + while n: for i in reversed(range(r)): cycles[i] -= 1 @@ -578,20 +577,6 @@ loops that truncate the stream. else: return - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: product(*iterables, repeat=1) @@ -612,13 +597,16 @@ loops that truncate the stream. This function is roughly equivalent to the following code, except that the actual implementation does not build up intermediate results in memory:: - def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat + def product(*iterables, repeat=1): + # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 + + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -626,6 +614,7 @@ loops that truncate the stream. keeping pools of values in memory to generate the products. Accordingly, it is only useful with finite inputs. + .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely @@ -634,7 +623,7 @@ loops that truncate the stream. Roughly equivalent to:: def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 + # repeat(10, 3) → 10 10 10 if times is None: while True: yield object @@ -650,59 +639,65 @@ loops that truncate the stream. >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + .. function:: starmap(function, iterable) - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). + Make an iterator that computes the *function* using arguments obtained + from the *iterable*. Used instead of :func:`map` when argument + parameters have already been "pre-zipped" into tuples. The difference between :func:`map` and :func:`starmap` parallels the distinction between ``function(a,b)`` and ``function(*c)``. Roughly equivalent to:: def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 + # starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000 for args in iterable: yield function(*args) .. function:: takewhile(predicate, iterable) - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: + Make an iterator that returns elements from the *iterable* as long as + the *predicate* is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 + # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 for x in iterable: - if predicate(x): - yield x - else: + if not predicate(x): break + yield x + + Note, the element that first fails the predicate condition is + consumed from the input iterator and there is no way to access it. + This could be an issue if an application wants to further consume the + input iterator after *takewhile* has been run to exhaustion. To work + around this problem, consider using `more-iterools before_and_after() + <https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.before_and_after>`_ + instead. .. function:: tee(iterable, n=2) Return *n* independent iterators from a single iterable. - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue):: + Roughly equivalent to:: def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): + iterator = iter(iterable) + shared_link = [None, None] + return tuple(_tee(iterator, shared_link) for _ in range(n)) + + def _tee(iterator, link): + try: while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) + if link[1] is None: + link[0] = next(iterator) + link[1] = [None, None] + value, link = link + yield value + except StopIteration: + return Once a :func:`tee` has been created, the original *iterable* should not be used anywhere else; otherwise, the *iterable* could get advanced without @@ -720,21 +715,29 @@ loops that truncate the stream. .. function:: zip_longest(*iterables, fillvalue=None) - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: + Make an iterator that aggregates elements from each of the + *iterables*. + + If the iterables are of uneven length, missing values are filled-in + with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. + + Iteration continues until the longest iterable is exhausted. + + Roughly equivalent to:: + + def zip_longest(*iterables, fillvalue=None): + # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- - def zip_longest(*args, fillvalue=None): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- - iterators = [iter(it) for it in args] + iterators = list(map(iter, iterables)) num_active = len(iterators) if not num_active: return + while True: values = [] - for i, it in enumerate(iterators): + for i, iterator in enumerate(iterators): try: - value = next(it) + value = next(iterator) except StopIteration: num_active -= 1 if not num_active: @@ -746,8 +749,7 @@ loops that truncate the stream. If one of the iterables is potentially infinite, then the :func:`zip_longest` function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. + (for example :func:`islice` or :func:`takewhile`). .. _itertools-recipes: @@ -762,261 +764,201 @@ The primary purpose of the itertools recipes is educational. The recipes show various ways of thinking about individual tools — for example, that ``chain.from_iterable`` is related to the concept of flattening. The recipes also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns +``starmap()`` and ``repeat()`` can work together. The recipes also show patterns for using itertools with the :mod:`operator` and :mod:`collections` modules as well as with the built-in itertools such as ``map()``, ``filter()``, ``reversed()``, and ``enumerate()``. A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()`` and ``iter_index()`` recipes -are being tested to see whether they prove their worth. +recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from -the `more-itertools project <https://pypi.org/project/more-itertools/>`_ found +the :pypi:`more-itertools` project found on the Python Package Index:: python -m pip install more-itertools Many of the recipes offer the same high performance as the underlying toolset. -Superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. +Superior memory performance is kept by processing elements one at a time rather +than bringing the whole iterable into memory all at once. Code volume is kept +small by linking the tools together in a `functional style +<https://www.cs.kent.ac.uk/people/staff/dat/miranda/whyfp90.pdf>`_. High speed +is retained by preferring "vectorized" building blocks over the use of for-loops +and :term:`generators <generator>` which incur interpreter overhead. .. testcode:: import collections + import contextlib import functools import math import operator import random def take(n, iterable): - "Return first n items of the iterable as a list" + "Return first n items of the iterable as a list." return list(islice(iterable, n)) - def prepend(value, iterator): - "Prepend a single value in front of an iterator" - # prepend(1, [2, 3, 4]) --> 1 2 3 4 - return chain([value], iterator) + def prepend(value, iterable): + "Prepend a single value in front of an iterable." + # prepend(1, [2, 3, 4]) → 1 2 3 4 + return chain([value], iterable) def tabulate(function, start=0): "Return function(0), function(1), ..." return map(function, count(start)) def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ + "Repeat calls to func with specified arguments." if times is None: return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) def flatten(list_of_lists): - "Flatten one level of nesting" + "Flatten one level of nesting." return chain.from_iterable(list_of_lists) def ncycles(iterable, n): - "Returns the sequence elements n times" + "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) def tail(n, iterable): - "Return an iterator over the last n items" - # tail(3, 'ABCDEFG') --> E F G + "Return an iterator over the last n items." + # tail(3, 'ABCDEFG') → E F G return iter(collections.deque(iterable, maxlen=n)) def consume(iterator, n=None): "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - # feed the entire iterator into a zero-length deque collections.deque(iterator, maxlen=0) else: - # advance to the empty slice starting at position n next(islice(iterator, n, n), None) def nth(iterable, n, default=None): - "Returns the nth item or a default value" + "Returns the nth item or a default value." return next(islice(iterable, n, None), default) - def quantify(iterable, pred=bool): + def quantify(iterable, predicate=bool): "Given a predicate that returns True or False, count the True results." - return sum(map(pred, iterable)) + return sum(map(predicate, iterable)) - def all_equal(iterable): - "Returns True if all the elements are equal to each other" - g = groupby(iterable) - return next(g, True) and not next(g, False) + def first_true(iterable, default=False, predicate=None): + "Returns the first true value or the *default* if there is no true value." + # first_true([a,b,c], x) → a or b or c or x + # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + return next(filter(predicate, iterable), default) - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. + def all_equal(iterable, key=None): + "Returns True if all the elements are equal to each other." + # all_equal('4٤௪౪໔', key=int) → True + return len(take(2, groupby(iterable, key))) <= 1 - If no true value is found, returns *default* - - If *pred* is not None, returns the first item - for which pred(item) is true. - - """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) + def unique_justseen(iterable, key=None): + "Yield unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') → A B C D A B + # unique_justseen('ABBcCAD', str.casefold) → A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - def iter_index(iterable, value, start=0, stop=None): - "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Slow path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i + def unique_everseen(iterable, key=None): + "Yield unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') → A B C D + # unique_everseen('ABBcCAD', str.casefold) → A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element else: - # Fast path for sequences - stop = len(iterable) if stop is None else stop - i = start - 1 - try: - while True: - yield (i := seq_index(value, i+1, stop)) - except ValueError: - pass - - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element - def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': - return zip(*args, strict=True) - if incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') + def unique(iterable, key=None, reverse=False): + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + return unique_justseen(sorted(iterable, key=key, reverse=reverse), key=key) def sliding_window(iterable, n): - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: + "Collect data into overlapping fixed-length chunks or blocks." + # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + iterator = iter(iterable) + window = collections.deque(islice(iterator, n - 1), maxlen=n) + for x in iterator: window.append(x) yield tuple(window) + def grouper(iterable, n, *, incomplete='fill', fillvalue=None): + "Collect data into non-overlapping fixed-length chunks or blocks." + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF + iterators = [iter(iterable)] * n + match incomplete: + case 'fill': + return zip_longest(*iterators, fillvalue=fillvalue) + case 'strict': + return zip(*iterators, strict=True) + case 'ignore': + return zip(*iterators) + case _: + raise ValueError('Expected fill, strict, or ignore') + def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) - - def partition(pred, iterable): + "Visit input iterables in a cycle until each is exhausted." + # roundrobin('ABC', 'D', 'EF') → A D E B F C + # Algorithm credited to George Sakkis + iterators = map(iter, iterables) + for num_active in range(len(iterables), 0, -1): + iterators = cycle(islice(iterators, num_active)) + yield from map(next, iterators) + + def partition(predicate, iterable): """Partition entries into false entries and true entries. - If *pred* is slow, consider wrapping it with functools.lru_cache(). + If *predicate* is slow, consider wrapping it with functools.lru_cache(). """ - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 + # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) - return filterfalse(pred, t1), filter(pred, t2) + return filterfalse(predicate, t1), filter(predicate, t2) def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D + "Return all contiguous non-empty subslices of a sequence." + # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) - def before_and_after(predicate, it): - """ Variant of takewhile() that allows complete - access to the remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. - """ - it = iter(it) - transition = [] - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() - - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) + def iter_index(iterable, value, start=0, stop=None): + "Return indices where a value occurs in a sequence or iterable." + # iter_index('AABCADEAF', 'A') → 0 1 4 7 + seq_index = getattr(iterable, 'index', None) + if seq_index is None: + iterator = islice(iterable, start, stop) + for i, element in enumerate(iterator, start): + if element is value or element == value: + yield i else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() + stop = len(iterable) if stop is None else stop + i = start + with contextlib.suppress(ValueError): + while True: + yield (i := seq_index(value, i, stop)) + i += 1 - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_except(func, exception, first=None): + "Convert a call-until-exception interface to an iterator interface." + # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator + with contextlib.suppress(exception): + if first is not None: + yield first() + while True: + yield func() The following recipes have a more mathematical flavor: @@ -1024,42 +966,46 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + "powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sum_of_squares(it): + def sum_of_squares(iterable): "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 - return math.sumprod(*tee(it)) + # sum_of_squares([10, 20, 30]) → 1400 + return math.sumprod(*tee(iterable)) - def transpose(it): - "Swap the rows and columns of the input." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) - return zip(*it, strict=True) + def reshape(matrix, cols): + "Reshape a 2-D matrix to have a given number of columns." + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + return batched(chain.from_iterable(matrix), cols, strict=True) + + def transpose(matrix): + "Swap the rows and columns of a 2-D matrix." + # transpose([(1, 2, 3), (11, 22, 33)]) → (1, 11) (2, 22) (3, 33) + return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) --> (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) n = len(m2[0]) return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) def convolve(signal, kernel): """Discrete linear convolution of two iterables. + Equivalent to polynomial multiplication. - The kernel is fully consumed before the calculations begin. - The signal is consumed lazily and can be infinite. - - Convolutions are mathematically commutative. - If the signal and kernel are swapped, - the output will be the same. + Convolutions are mathematically commutative; however, the inputs are + evaluated differently. The signal is consumed lazily and can be + infinite. The kernel is fully consumed before the calculations begin. Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate - # convolve(data, [1, -2, 1]) --> 2nd derivative estimate + # convolve([1, -1, -20], [1, -3]) → 1 -4 -17 60 + # convolve(data, [0.25, 0.25, 0.25, 0.25]) → Moving average (blur) + # convolve(data, [1/2, 0, -1/2]) → 1st derivative estimate + # convolve(data, [1, -2, 1]) → 2nd derivative estimate kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) @@ -1071,7 +1017,7 @@ The following recipes have a more mathematical flavor: (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] + # polynomial_from_roots([5, -4, 3]) → [1, -4, -17, 60] factors = zip(repeat(1), map(operator.neg, roots)) return list(functools.reduce(convolve, factors, [1])) @@ -1080,8 +1026,8 @@ The following recipes have a more mathematical flavor: Computes with better numeric stability than Horner's method. """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 + # Evaluate x³ -4x² -17x + 60 at x = 5 + # polynomial_eval([1, -4, -17, 60], x=5) → 0 n = len(coefficients) if not n: return type(x)(0) @@ -1094,30 +1040,26 @@ The following recipes have a more mathematical flavor: f(x) = x³ -4x² -17x + 60 f'(x) = 3x² -8x -17 """ - # polynomial_derivative([1, -4, -17, 60]) -> [3, -8, -17] + # polynomial_derivative([1, -4, -17, 60]) → [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) return list(map(operator.mul, coefficients, powers)) def sieve(n): "Primes less than n." - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 - start = 3 data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p*p) + for p in iter_index(data, 1, start=3, stop=math.isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - start = p*p - yield from iter_index(data, 1, start) + yield from iter_index(data, 1, start=3) def factor(n): "Prime factors of n." - # factor(99) --> 3 3 11 - # factor(1_000_000_000_000_007) --> 47 59 360620266859 - # factor(1_000_000_000_000_403) --> 1000000000000403 + # factor(99) → 3 3 11 + # factor(1_000_000_000_000_007) → 47 59 360620266859 + # factor(1_000_000_000_000_403) → 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while not n % prime: yield prime @@ -1127,23 +1069,13 @@ The following recipes have a more mathematical flavor: if n > 1: yield n - def nth_combination(iterable, r, index): - "Equivalent to list(combinations(iterable, r))[index]" - pool = tuple(iterable) - n = len(pool) - c = math.comb(n, r) - if index < 0: - index += c - if index < 0 or index >= c: - raise IndexError - result = [] - while r: - c, n, r = c*r//n, n-1, r-1 - while index >= c: - index -= c - c, n = c*(n-r)//n, n-1 - result.append(pool[-1-n]) - return tuple(result) + def totient(n): + "Count of natural numbers up to n that are coprime to n." + # https://mathworld.wolfram.com/TotientFunction.html + # totient(12) → 4 because len([1, 5, 7, 11]) == 4 + for prime in set(factor(n)): + n -= n // prime + return n .. doctest:: @@ -1206,6 +1138,12 @@ The following recipes have a more mathematical flavor: >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> # Verify that the input is consumed lazily + >>> it = iter('abcdef') + >>> take(3, it) + ['a', 'b', 'c'] + >>> list(it) + ['d', 'e', 'f'] >>> list(prepend(1, [2, 3, 4])) [1, 2, 3, 4] @@ -1218,23 +1156,45 @@ The following recipes have a more mathematical flavor: >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] + >>> # Verify the input is consumed greedily + >>> input_iterator = iter('ABCDEFG') + >>> output_iterator = tail(3, input_iterator) + >>> list(input_iterator) + [] >>> it = iter(range(10)) >>> consume(it, 3) + >>> # Verify the input is consumed lazily >>> next(it) 3 + >>> # Verify the input is consumed completely >>> consume(it) >>> next(it, 'Done') 'Done' >>> nth('abcde', 3) 'd' - >>> nth('abcde', 9) is None True + >>> # Verify that the input is consumed lazily + >>> it = iter('abcde') + >>> nth(it, 2) + 'c' + >>> list(it) + ['d', 'e'] >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] + [True, True, True, False, False] + >>> # Verify that the input is consumed lazily and that only + >>> # one element of a second equivalence class is used to disprove + >>> # the assertion that all elements are equal. + >>> it = iter('aaabbbccc') + >>> all_equal(it) + False + >>> ''.join(it) + 'bbccc' >>> quantify(range(99), lambda x: x%2==0) 50 @@ -1242,7 +1202,7 @@ The following recipes have a more mathematical flavor: >>> quantify([True, False, False, True, True]) 3 - >>> quantify(range(12), pred=lambda x: x%2==1) + >>> quantify(range(12), predicate=lambda x: x%2==1) 6 >>> a = [[1, 2, 3], [4, 5, 6]] @@ -1257,18 +1217,53 @@ The following recipes have a more mathematical flavor: >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] + >>> # Verify greedy consumption of input iterator + >>> input_iterator = iter('abc') + >>> output_iterator = ncycles(input_iterator, 3) + >>> list(input_iterator) + [] >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) + [(0, 1, 2), (3, 4, 5)] + >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 1)) + [(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,)] + >>> list(reshape(M, 2)) + [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11)] + >>> list(reshape(M, 3)) + [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, 10, 11)] + >>> list(reshape(M, 4)) + [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 5)) + Traceback (most recent call last): + ... + ValueError: batched(): incomplete batch + >>> list(reshape(M, 6)) + [(0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10, 11)] + >>> list(reshape(M, 12)) + [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] + >>> # Verify that the inputs are consumed lazily + >>> input1 = iter([1, 2, 3]) + >>> input2 = iter([11, 22, 33]) + >>> output_iterator = transpose([input1, input2]) + >>> next(output_iterator) + (1, 11) + >>> list(zip(input1, input2)) + [(2, 22), (3, 33)] >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) [(49, 80), (41, 60)] >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] + >>> list(convolve([1, -1, -20], [1, -3])) == [1, -4, -17, 60] + True >>> data = [20, 40, 24, 32, 20, 28, 16] >>> list(convolve(data, [0.25, 0.25, 0.25, 0.25])) [5.0, 15.0, 21.0, 29.0, 29.0, 26.0, 24.0, 16.0, 11.0, 4.0] @@ -1276,13 +1271,25 @@ The following recipes have a more mathematical flavor: [20, 20, -16, 8, -12, 8, -12, -16] >>> list(convolve(data, [1, -2, 1])) [20, 0, -36, 24, -20, 20, -20, -4, 16] + >>> # Verify signal is consumed lazily and the kernel greedily + >>> signal_iterator = iter([10, 20, 30, 40, 50]) + >>> kernel_iterator = iter([1, 2, 3]) + >>> output_iterator = convolve(signal_iterator, kernel_iterator) + >>> list(kernel_iterator) + [] + >>> next(output_iterator) + 10 + >>> next(output_iterator) + 40 + >>> list(signal_iterator) + [30, 40, 50] >>> from fractions import Fraction >>> from decimal import Decimal - >>> polynomial_eval([1, -4, -17, 60], x=2) - 18 - >>> x = 2; x**3 - 4*x**2 -17*x + 60 - 18 + >>> polynomial_eval([1, -4, -17, 60], x=5) + 0 + >>> x = 5; x**3 - 4*x**2 -17*x + 60 + 0 >>> polynomial_eval([1, -4, -17, 60], x=2.5) 8.125 >>> x = 2.5; x**3 - 4*x**2 -17*x + 60 @@ -1363,6 +1370,33 @@ The following recipes have a more mathematical flavor: >>> # Test list input. Lists do not support None for the stop argument >>> list(iter_index(list('AABCADEAF'), 'A')) [0, 1, 4, 7] + >>> # Verify that input is consumed lazily + >>> input_iterator = iter('AABCADEAF') + >>> output_iterator = iter_index(input_iterator, 'A') + >>> next(output_iterator) + 0 + >>> next(output_iterator) + 1 + >>> next(output_iterator) + 4 + >>> ''.join(input_iterator) + 'DEAF' + + >>> # Verify that the target value can be a sequence. + >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] + >>> target = [30, 40] + >>> list(iter_index(seq, target)) + [1, 4] + + >>> # Verify faithfulness to type specific index() method behaviors. + >>> # For example, bytes and str perform continuous-subsequence searches + >>> # that do not match the general behavior specified + >>> # in collections.abc.Sequence.index(). + >>> seq = 'abracadabra' + >>> target = 'ab' + >>> list(iter_index(seq, target)) + [0, 7] + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] @@ -1428,6 +1462,25 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 + 0 + >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, + ... 18, 8, 12, 10, 22, 8, 20, 12, 18, 12, 28, 8, 30, 16, 20, 16, 24, 12, 36, 18, + ... 24, 16, 40, 12, 42, 20, 24, 22, 46, 16, 42, 20, 32, 24, 52, 18, 40, 24, 36, + ... 28, 58, 16, 60, 30, 36, 32, 48, 20, 66, 32, 44] # https://oeis.org/A000010 + ... + >>> list(map(totient, range(1, 70))) == first_totients + True + >>> reference_totient = lambda n: sum(math.gcd(t, n) == 1 for t in range(1, n+1)) + >>> all(totient(n) == reference_totient(n) for n in range(1000)) + True + >>> totient(128_884_753_939) == 128_884_753_938 # large prime + True + >>> totient(999953 * 999983) == 999952 * 999982 # large semiprime + True + >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] @@ -1486,6 +1539,16 @@ The following recipes have a more mathematical flavor: >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] + >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] + >>> collections.Counter(roundrobin(*ranges)) == collections.Counter(chain(*ranges)) + True + >>> # Verify that the inputs are consumed lazily + >>> input_iterators = list(map(iter, ['abcd', 'ef', '', 'ghijk', 'l', 'mnopqr'])) + >>> output_iterator = roundrobin(*input_iterators) + >>> ''.join(islice(output_iterator, 10)) + 'aeglmbfhnc' + >>> ''.join(chain(*input_iterators)) + 'dijkopqr' >>> def is_odd(x): ... return x % 2 == 1 @@ -1495,13 +1558,17 @@ The following recipes have a more mathematical flavor: [0, 2, 4, 6, 8] >>> list(odds) [1, 3, 5, 7, 9] - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) - 'dEfGhI' + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter(range(10)) + >>> evens, odds = partition(is_odd, input_iterator) + >>> next(odds) + 1 + >>> next(odds) + 3 + >>> next(evens) + 0 + >>> list(input_iterator) + [4, 5, 6, 7, 8, 9] >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] @@ -1517,17 +1584,38 @@ The following recipes have a more mathematical flavor: >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) + >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) + >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_everseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) + >>> list(unique_justseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) + >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_justseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' + + >>> list(unique([[1, 2], [3, 4], [1, 2]])) + [[1, 2], [3, 4]] + >>> list(unique('ABBcCAD', str.casefold)) + ['A', 'B', 'c', 'D'] + >>> list(unique('ABBcCAD', str.casefold, reverse=True)) + ['D', 'c', 'B', 'A'] >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) @@ -1548,20 +1636,12 @@ The following recipes have a more mathematical flavor: >>> first_true('ABC0DEF1', '9', str.isdigit) '0' - - >>> population = 'ABCDEFGH' - >>> for r in range(len(population) + 1): - ... seq = list(combinations(population, r)) - ... for i in range(len(seq)): - ... assert nth_combination(population, r, i) == seq[i] - ... for i in range(-len(seq), 0): - ... assert nth_combination(population, r, i) == seq[i] - - >>> iterable = 'abcde' - >>> r = 3 - >>> combos = list(combinations(iterable, r)) - >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) - True + >>> # Verify that inputs are consumed lazily + >>> it = iter('ABC0DEF1') + >>> first_true(it, predicate=str.isdigit) + '0' + >>> ''.join(it) + 'DEF1' .. testcode:: @@ -1585,10 +1665,54 @@ The following recipes have a more mathematical flavor: def triplewise(iterable): "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG + # triplewise('ABCDEFG') → ABC BCD CDE DEF EFG for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c + def nth_combination(iterable, r, index): + "Equivalent to list(combinations(iterable, r))[index]" + pool = tuple(iterable) + n = len(pool) + c = math.comb(n, r) + if index < 0: + index += c + if index < 0 or index >= c: + raise IndexError + result = [] + while r: + c, n, r = c*r//n, n-1, r-1 + while index >= c: + index -= c + c, n = c*(n-r)//n, n-1 + result.append(pool[-1-n]) + return tuple(result) + + def before_and_after(predicate, it): + """ Variant of takewhile() that allows complete + access to the remainder of the iterator. + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) # takewhile() would lose the 'd' + 'dEfGhI' + + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. + """ + it = iter(it) + transition = [] + + def true_iterator(): + for elem in it: + if predicate(elem): + yield elem + else: + transition.append(elem) + return + + return true_iterator(), chain(transition, it) .. doctest:: :hide: @@ -1604,3 +1728,24 @@ The following recipes have a more mathematical flavor: >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + + >>> population = 'ABCDEFGH' + >>> for r in range(len(population) + 1): + ... seq = list(combinations(population, r)) + ... for i in range(len(seq)): + ... assert nth_combination(population, r, i) == seq[i] + ... for i in range(-len(seq), 0): + ... assert nth_combination(population, r, i) == seq[i] + + >>> iterable = 'abcde' + >>> r = 3 + >>> combos = list(combinations(iterable, r)) + >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) + True + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) + 'dEfGhI' diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3..42cb1f850fe9c5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -1,5 +1,5 @@ -:mod:`json` --- JSON encoder and decoder -======================================== +:mod:`!json` --- JSON encoder and decoder +========================================= .. module:: json :synopsis: Encode and decode the JSON format. @@ -106,7 +106,7 @@ Extending :class:`JSONEncoder`:: ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) + ... return super().default(obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) '[2.0, 1.0]' @@ -504,7 +504,7 @@ Encoders and Decoders else: return list(iterable) # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) + return super().default(o) .. method:: encode(o) diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png index f4504895699974..4c26f26292faa5 100644 Binary files a/Doc/library/kde_example.png and b/Doc/library/kde_example.png differ diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index c3b4699cb05af6..ac57140f888024 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -1,5 +1,5 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== +:mod:`!keyword` --- Testing for Python keywords +=============================================== .. module:: keyword :synopsis: Test whether a string is a keyword in Python. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index dd9f4ee45ba82e..88c6079a05b7fa 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -1,5 +1,5 @@ -:mod:`linecache` --- Random access to text lines -================================================ +:mod:`!linecache` --- Random access to text lines +================================================= .. module:: linecache :synopsis: Provides random access to individual lines from text files. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index afd5677deac3f8..0a8cbd4f95f473 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -1,5 +1,5 @@ -:mod:`locale` --- Internationalization services -=============================================== +:mod:`!locale` --- Internationalization services +================================================ .. module:: locale :synopsis: Internationalization services. @@ -18,7 +18,7 @@ know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`_locale` module, +The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. The :mod:`locale` module defines the following exception and functions: @@ -192,7 +192,13 @@ The :mod:`locale` module defines the following exception and functions: Get a format string for :func:`time.strftime` to represent time in the am/pm format. - .. data:: DAY_1 ... DAY_7 + .. data:: DAY_1 + DAY_2 + DAY_3 + DAY_4 + DAY_5 + DAY_6 + DAY_7 Get the name of the n-th day of the week. @@ -202,15 +208,43 @@ The :mod:`locale` module defines the following exception and functions: international convention (ISO 8601) that Monday is the first day of the week. - .. data:: ABDAY_1 ... ABDAY_7 + .. data:: ABDAY_1 + ABDAY_2 + ABDAY_3 + ABDAY_4 + ABDAY_5 + ABDAY_6 + ABDAY_7 Get the abbreviated name of the n-th day of the week. - .. data:: MON_1 ... MON_12 + .. data:: MON_1 + MON_2 + MON_3 + MON_4 + MON_5 + MON_6 + MON_7 + MON_8 + MON_9 + MON_10 + MON_11 + MON_12 Get the name of the n-th month. - .. data:: ABMON_1 ... ABMON_12 + .. data:: ABMON_1 + ABMON_2 + ABMON_3 + ABMON_4 + ABMON_5 + ABMON_6 + ABMON_7 + ABMON_8 + ABMON_9 + ABMON_10 + ABMON_11 + ABMON_12 Get the abbreviated name of the n-th month. @@ -229,14 +263,14 @@ The :mod:`locale` module defines the following exception and functions: .. data:: NOEXPR - Get a regular expression that can be used with the regex(3) function to + Get a regular expression that can be used with the ``regex(3)`` function to recognize a negative response to a yes/no question. .. note:: The regular expressions for :const:`YESEXPR` and :const:`NOEXPR` use syntax suitable for the - :c:func:`regex` function from the C library, which might + ``regex`` function from the C library, which might differ from the syntax used in :mod:`re`. .. data:: CRNCYSTR @@ -303,13 +337,13 @@ The :mod:`locale` module defines the following exception and functions: *language code* and *encoding* may be ``None`` if their values cannot be determined. - .. deprecated-removed:: 3.11 3.13 + .. deprecated-removed:: 3.11 3.15 .. function:: getlocale(category=LC_CTYPE) Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`LC_\*` values + *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. @@ -454,11 +488,16 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_CTYPE - .. index:: pair: module; string + Locale category for the character type functions. Most importantly, this + category defines the text encoding, i.e. how bytes are interpreted as + Unicode codepoints. See :pep:`538` and :pep:`540` for how this variable + might be automatically coerced to ``C.UTF-8`` to avoid issues created by + invalid settings in containers or incompatible settings passed over remote + SSH connections. - Locale category for the character type functions. Depending on the settings of - this category, the functions of module :mod:`string` dealing with case change - their behaviour. + Python doesn't internally use locale-dependent character transformation functions + from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`!Py_TOLOWER`. .. data:: LC_COLLATE @@ -576,9 +615,9 @@ the locale is ``C``). When Python code uses the :mod:`locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want -this to happen, it should remove the :mod:`_locale` extension module (which does +this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, -and make sure that the :mod:`_locale` module is not accessible as a shared +and make sure that the :mod:`!_locale` module is not accessible as a shared library. @@ -592,17 +631,18 @@ Access to message catalogs .. function:: dcgettext(domain, msg, category) .. function:: textdomain(domain) .. function:: bindtextdomain(domain, dir) +.. function:: bind_textdomain_codeset(domain, codeset) The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`!gettext`, -:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, -and :func:`!bind_textdomain_codeset`. These are similar to the same functions in +provide this interface. It consists of the functions :func:`gettext`, +:func:`dgettext`, :func:`dcgettext`, :func:`textdomain`, :func:`bindtextdomain`, +and :func:`bind_textdomain_codeset`. These are similar to the same functions in the :mod:`gettext` module, but use the C library's binary format for message catalogs, and the C library's search algorithms for locating message catalogs. Python applications should normally find no need to invoke these functions, and should use :mod:`gettext` instead. A known exception to this rule are applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be +C functions ``gettext`` or ``dcgettext``. For these applications, it may be necessary to bind the text domain, so that the libraries can properly locate their message catalogs. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 85a53e6aa7a78b..dfbf0b1cf2f9ff 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -1,5 +1,5 @@ -:mod:`logging.config` --- Logging configuration -=============================================== +:mod:`!logging.config` --- Logging configuration +================================================ .. module:: logging.config :synopsis: Configuration of the logging module. @@ -93,8 +93,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in :param fname: A filename, or a file-like object, or an instance derived from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is + :class:`!RawConfigParser`-derived instance is passed, it is used as + is. Otherwise, a :class:`~configparser.ConfigParser` is instantiated, and the configuration read by it from the object passed in ``fname``. If that has a :meth:`readline` method, it is assumed to be a file-like object and read using @@ -103,7 +103,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in :meth:`~configparser.ConfigParser.read`. - :param defaults: Defaults to be passed to the ConfigParser can be specified + :param defaults: Defaults to be passed to the :class:`!ConfigParser` can be specified in this argument. :param disable_existing_loggers: If specified as ``False``, loggers which @@ -127,8 +127,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionadded:: 3.10 - The *encoding* parameter is added. + .. versionchanged:: 3.10 + Added the *encoding* parameter. .. versionchanged:: 3.12 An exception will be thrown if the provided file diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 2a825db54aed5c..5a081f9e7add99 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -1,5 +1,5 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ +:mod:`!logging.handlers` --- Logging handlers +============================================= .. module:: logging.handlers :synopsis: Handlers for the logging module. @@ -66,7 +66,7 @@ and :meth:`flush` methods). :param stream: The stream that the handler should use. - :return: the old stream, if the stream was changed, or *None* if it wasn't. + :return: the old stream, if the stream was changed, or ``None`` if it wasn't. .. versionadded:: 3.7 @@ -656,9 +656,7 @@ supports sending logging messages to a remote or local Unix syslog. to the other end. This method is called during handler initialization, but it's not regarded as an error if the other end isn't listening at this point - the method will be called again when emitting an event, if - but it's not regarded as an error if the other end isn't listening yet - --- the method will be called again when emitting an event, - if there is no socket at that point. + there is no socket at that point. .. versionadded:: 3.11 @@ -873,8 +871,8 @@ supports sending logging messages to an email address via SMTP. A timeout can be specified for communication with the SMTP server using the *timeout* argument. - .. versionadded:: 3.3 - The *timeout* argument was added. + .. versionchanged:: 3.3 + Added the *timeout* parameter. .. method:: emit(record) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index acdeb88a546261..4ba520c139ebce 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1,5 +1,5 @@ -:mod:`logging` --- Logging facility for Python -============================================== +:mod:`!logging` --- Logging facility for Python +=============================================== .. module:: logging :synopsis: Flexible event logging system for applications. @@ -30,20 +30,61 @@ is that all Python modules can participate in logging, so your application log can include your own messages integrated with messages from third-party modules. -The simplest example: +Here's a simple example of idiomatic usage: :: + + # myapp.py + import logging + import mylib + logger = logging.getLogger(__name__) + + def main(): + logging.basicConfig(filename='myapp.log', level=logging.INFO) + logger.info('Started') + mylib.do_something() + logger.info('Finished') + + if __name__ == '__main__': + main() + +:: + + # mylib.py + import logging + logger = logging.getLogger(__name__) + + def do_something(): + logger.info('Doing something') + +If you run *myapp.py*, you should see this in *myapp.log*: .. code-block:: none - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! + INFO:__main__:Started + INFO:mylib:Doing something + INFO:__main__:Finished + +The key feature of this idiomatic usage is that the majority of code is simply +creating a module level logger with ``getLogger(__name__)``, and using that +logger to do any needed logging. This is concise, while allowing downstream +code fine-grained control if needed. Logged messages to the module-level logger +get forwarded to handlers of loggers in higher-level modules, all the way up to +the highest-level logger known as the root logger; this approach is known as +hierarchical logging. + +For logging to be useful, it needs to be configured: setting the levels and +destinations for each logger, potentially changing how specific modules log, +often based on command-line arguments or application configuration. In most +cases, like the one above, only the root logger needs to be so configured, since +all the lower level loggers at module level eventually forward their messages to +its handlers. :func:`~logging.basicConfig` provides a quick way to configure +the root logger that handles many use cases. The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the tutorials (**see the links above and on the right**). -The basic classes defined by the module, together with their functions, are -listed below. +The basic classes defined by the module, together with their attributes and +methods, are listed in the sections below. * Loggers expose the interface that application code directly uses. * Handlers send the log records (created by loggers) to the appropriate @@ -68,15 +109,36 @@ The ``name`` is potentially a period-separated hierarchical value, like Loggers that are further down in the hierarchical list are children of loggers higher up in the list. For example, given a logger with a name of ``foo``, loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. +descendants of ``foo``. In addition, all loggers are descendants of the root +logger. The logger name hierarchy is analogous to the Python package hierarchy, +and identical to it if you organise your loggers on a per-module basis using +the recommended construction ``logging.getLogger(__name__)``. That's because +in a module, ``__name__`` is the module's name in the Python package namespace. .. class:: Logger + .. attribute:: Logger.name + + This is the logger's name, and is the value that was passed to :func:`getLogger` + to obtain the logger. + + .. note:: This attribute should be treated as read-only. + + .. attribute:: Logger.level + + The threshold of this logger, as set by the :meth:`setLevel` method. + + .. note:: Do not set this attribute directly - always use :meth:`setLevel`, + which has checks for the level passed to it. + + .. attribute:: Logger.parent + + The parent logger of this logger. It may change based on later instantiation + of loggers which are higher up in the namespace hierarchy. + + .. note:: This value should be treated as read-only. + .. attribute:: Logger.propagate If this attribute evaluates to true, events logged to this logger will be @@ -108,6 +170,21 @@ is the module's name in the Python package namespace. scenario is to attach handlers only to the root logger, and to let propagation take care of the rest. + .. attribute:: Logger.handlers + + The list of handlers directly attached to this logger instance. + + .. note:: This attribute should be treated as read-only; it is normally changed via + the :meth:`addHandler` and :meth:`removeHandler` methods, which use locks to ensure + thread-safe operation. + + .. attribute:: Logger.disabled + + This attribute disables handling of any events. It is set to ``False`` in the + initializer, and only changed by logging configuration code. + + .. note:: This attribute should be treated as read-only. + .. method:: Logger.setLevel(level) Sets the threshold for this logger to *level*. Logging messages which are less @@ -531,12 +608,12 @@ subclasses. However, the :meth:`!__init__` method in subclasses needs to call This method should be called from handlers when an exception is encountered during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + :data:`raiseExceptions` is ``False``, exceptions get silently ignored. This is what is mostly wanted for a logging system - most users will not care about errors in the logging system, they are more interested in application errors. You could, however, replace this with a custom handler if you wish. The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + occurred. (The default value of :data:`raiseExceptions` is ``True``, as that is more useful during development). @@ -615,14 +692,14 @@ Formatter Objects ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` :type defaults: dict[str, Any] - .. versionadded:: 3.2 - The *style* parameter. + .. versionchanged:: 3.2 + Added the *style* parameter. - .. versionadded:: 3.8 - The *validate* parameter. + .. versionchanged:: 3.8 + Added the *validate* parameter. - .. versionadded:: 3.10 - The *defaults* parameter. + .. versionchanged:: 3.10 + Added the *defaults* parameter. .. method:: format(record) @@ -926,7 +1003,7 @@ the options available to you. | | | portion of the time). | +----------------+-------------------------+-----------------------------------------------+ | created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | +| | | (as returned by :func:`time.time_ns` / 1e9). | +----------------+-------------------------+-----------------------------------------------+ | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | @@ -1080,10 +1157,12 @@ functions. .. function:: getLogger(name=None) - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. + Return a logger with the specified name or, if name is ``None``, return the + root logger of the hierarchy. If specified, the name is typically a + dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. Choice + of these names is entirely up to the developer who is using logging, though + it is recommended that ``__name__`` be used unless you have a specific + reason for not doing that, as mentioned in :ref:`logger`. All calls to this function with a given name return the same logger instance. This means that logger instances never need to be passed between different parts @@ -1115,89 +1194,31 @@ functions. .. function:: debug(msg, *args, **kwargs) - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) + This is a convenience function that calls :meth:`Logger.debug`, on the root + logger. The handling of the arguments is in every way identical + to what is described in that method. - would print something like: + The only difference is that if the root logger has no handlers, then + :func:`basicConfig` is called, prior to calling ``debug`` on the root logger. - .. code-block:: none + For very short scripts or quick demonstrations of ``logging`` facilities, + ``debug`` and the other module-level functions may be convenient. However, + most programs will want to carefully and explicitly control the logging + configuration, and should therefore prefer creating a module-level logger and + calling :meth:`Logger.debug` (or other level-specific methods) on it, as + described at the beginning of this documentation. - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. .. function:: info(msg, *args, **kwargs) - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`INFO` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: warning(msg, *args, **kwargs) - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`WARNING` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. note:: There is an obsolete function ``warn`` which is functionally identical to ``warning``. As ``warn`` is deprecated, please do not use @@ -1210,26 +1231,26 @@ functions. .. function:: error(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: critical(msg, *args, **kwargs) - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`CRITICAL` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: exception(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. Exception info is added to the logging message. This function should only be called from an exception handler. .. function:: log(level, msg, *args, **kwargs) - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. + Logs a message with level *level* on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: disable(level=CRITICAL) @@ -1494,6 +1515,18 @@ Module-Level Attributes .. versionadded:: 3.2 +.. attribute:: raiseExceptions + + Used to see if exceptions during handling should be propagated. + + Default: ``True``. + + If :data:`raiseExceptions` is ``False``, + exceptions get silently ignored. This is what is mostly wanted + for a logging system - most users will not care about errors in + the logging system, they are more interested in application errors. + + Integration with the warnings module ------------------------------------ diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 0d69c3bc01d1e2..69f7cb8d48d7ae 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -1,5 +1,5 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== +:mod:`!lzma` --- Compression using the LZMA algorithm +===================================================== .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. @@ -104,7 +104,7 @@ Reading and writing compressed files and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - The following method is also provided: + The following method and attributes are also provided: .. method:: peek(size=-1) @@ -117,6 +117,20 @@ Reading and writing compressed files file object (e.g. if the :class:`LZMAFile` was constructed by passing a file object for *filename*). + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The lzma file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + + .. versionchanged:: 3.4 Added support for the ``"x"`` and ``"xb"`` modes. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 91df07d914cae2..40ea71cd342b47 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1,5 +1,5 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== +:mod:`!mailbox` --- Manipulate mailboxes in various formats +=========================================================== .. module:: mailbox :synopsis: Manipulate mailboxes in various formats @@ -13,8 +13,8 @@ This module defines two classes, :class:`Mailbox` and :class:`Message`, for accessing and manipulating on-disk mailboxes and the messages they contain. -:class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's +:class:`!Mailbox` offers a dictionary-like mapping from keys to messages. +:class:`!Message` extends the :mod:`email.message` module's :class:`~email.message.Message` class with format-specific state and behavior. Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. @@ -27,37 +27,38 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-objects: -:class:`Mailbox` objects ------------------------- +:class:`!Mailbox` objects +------------------------- .. class:: Mailbox A mailbox, which may be inspected and modified. - The :class:`Mailbox` class defines an interface and is not intended to be + The :class:`!Mailbox` class defines an interface and is not intended to be instantiated. Instead, format-specific subclasses should inherit from - :class:`Mailbox` and your code should instantiate a particular subclass. + :class:`!Mailbox` and your code should instantiate a particular subclass. - The :class:`Mailbox` interface is dictionary-like, with small keys - corresponding to messages. Keys are issued by the :class:`Mailbox` instance - with which they will be used and are only meaningful to that :class:`Mailbox` + The :class:`!Mailbox` interface is dictionary-like, with small keys + corresponding to messages. Keys are issued by the :class:`!Mailbox` instance + with which they will be used and are only meaningful to that :class:`!Mailbox` instance. A key continues to identify a message even if the corresponding message is modified, such as by replacing it with another message. - Messages may be added to a :class:`Mailbox` instance using the set-like + Messages may be added to a :class:`!Mailbox` instance using the set-like method :meth:`add` and removed using a ``del`` statement or the set-like methods :meth:`remove` and :meth:`discard`. - :class:`Mailbox` interface semantics differ from dictionary semantics in some + :class:`!Mailbox` interface semantics differ from dictionary semantics in some noteworthy ways. Each time a message is requested, a new representation (typically a :class:`Message` instance) is generated based upon the current state of the mailbox. Similarly, when a message is added to a - :class:`Mailbox` instance, the provided message representation's contents are + :class:`!Mailbox` instance, the provided message representation's contents are copied. In neither case is a reference to the message representation kept by - the :class:`Mailbox` instance. + the :class:`!Mailbox` instance. - The default :class:`Mailbox` iterator iterates over message representations, - not keys as the default dictionary iterator does. Moreover, modification of a + The default :class:`!Mailbox` :term:`iterator` iterates over message + representations, not keys as the default :class:`dictionary <dict>` + iterator does. Moreover, modification of a mailbox during iteration is safe and well-defined. Messages added to the mailbox after an iterator is created will not be seen by the iterator. Messages removed from the mailbox before the iterator yields them @@ -69,14 +70,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Be very cautious when modifying mailboxes that might be simultaneously changed by some other process. The safest mailbox format to use for such - tasks is Maildir; try to avoid using single-file formats such as mbox for + tasks is :class:`Maildir`; try to avoid using single-file formats such as + :class:`mbox` for concurrent writing. If you're modifying a mailbox, you *must* lock it by calling the :meth:`lock` and :meth:`unlock` methods *before* reading any messages in the file or making any changes by adding or deleting a message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. - :class:`Mailbox` instances have the following methods: + :class:`!Mailbox` instances have the following methods: .. method:: add(message) @@ -127,21 +129,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: iterkeys() - keys() - Return an iterator over all keys if called as :meth:`iterkeys` or return a - list of keys if called as :meth:`keys`. + Return an :term:`iterator` over all keys + + + .. method:: keys() + + The same as :meth:`iterkeys`, except that a :class:`list` is returned + rather than an :term:`iterator` .. method:: itervalues() __iter__() - values() - Return an iterator over representations of all messages if called as - :meth:`itervalues` or :meth:`__iter__` or return a list of such - representations if called as :meth:`values`. The messages are represented + Return an :term:`iterator` over representations of all messages. + The messages are represented as instances of the appropriate format-specific :class:`Message` subclass - unless a custom message factory was specified when the :class:`Mailbox` + unless a custom message factory was specified when the :class:`!Mailbox` instance was initialized. .. note:: @@ -150,15 +154,25 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. iterate over keys. + .. method:: values() + + The same as :meth:`itervalues`, except that a :class:`list` is returned + rather than an :term:`iterator` + + .. method:: iteritems() - items() - Return an iterator over (*key*, *message*) pairs, where *key* is a key and - *message* is a message representation, if called as :meth:`iteritems` or - return a list of such pairs if called as :meth:`items`. The messages are + Return an :term:`iterator` over (*key*, *message*) pairs, where *key* is + a key and *message* is a message representation. The messages are represented as instances of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. + + + .. method:: items() + + The same as :meth:`iteritems`, except that a :class:`list` of pairs is + returned rather than an :term:`iterator` of pairs. .. method:: get(key, default=None) @@ -167,9 +181,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Return a representation of the message corresponding to *key*. If no such message exists, *default* is returned if the method was called as :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`__getitem__`. The message is represented as an instance + called as :meth:`!__getitem__`. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a - custom message factory was specified when the :class:`Mailbox` instance + custom message factory was specified when the :class:`!Mailbox` instance was initialized. @@ -198,21 +212,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_file(key) - Return a file-like representation of the message corresponding to *key*, + Return a :term:`file-like <file-like object>` representation of the + message corresponding to *key*, or raise a :exc:`KeyError` exception if no such message exists. The file-like object behaves as if open in binary mode. This file should be closed once it is no longer needed. .. versionchanged:: 3.2 - The file object really is a binary file; previously it was incorrectly - returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to - automatically close it. + The file object really is a :term:`binary file`; previously it was + incorrectly returned in text mode. Also, the :term:`file-like object` + now supports the :term:`context manager` protocol: you can use a + :keyword:`with` statement to automatically close it. .. note:: - Unlike other representations of messages, file-like representations are - not necessarily independent of the :class:`Mailbox` instance that + Unlike other representations of messages, + :term:`file-like <file-like object>` representations are not + necessarily independent of the :class:`!Mailbox` instance that created them or of the underlying mailbox. More specific documentation is provided by each subclass. @@ -238,7 +254,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. the message. If no such message exists, return *default*. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: popitem() @@ -248,7 +264,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. If the mailbox is empty, raise a :exc:`KeyError` exception. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: update(arg) @@ -259,7 +275,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. *message* as if by using :meth:`__setitem__`. As with :meth:`__setitem__`, each *key* must already correspond to a message in the mailbox or else a :exc:`KeyError` exception will be raised, so in general it is incorrect - for *arg* to be a :class:`Mailbox` instance. + for *arg* to be a :class:`!Mailbox` instance. .. note:: @@ -269,7 +285,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: flush() Write any pending changes to the filesystem. For some :class:`Mailbox` - subclasses, changes are always written immediately and :meth:`flush` does + subclasses, changes are always written immediately and :meth:`!flush` does nothing, but you should still make a habit of calling this method. @@ -290,13 +306,13 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() Flush the mailbox, unlock it if necessary, and close any open files. For - some :class:`Mailbox` subclasses, this method does nothing. + some :class:`!Mailbox` subclasses, this method does nothing. .. _mailbox-maildir: -:class:`Maildir` -^^^^^^^^^^^^^^^^ +:class:`!Maildir` objects +^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Maildir(dirname, factory=None, create=True) @@ -330,11 +346,11 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Folders of the style introduced by the Courier mail transfer agent are also supported. Any subdirectory of the main mailbox is considered a folder if ``'.'`` is the first character in its name. Folder names are represented by - :class:`Maildir` without the leading ``'.'``. Each folder is itself a Maildir + :class:`!Maildir` without the leading ``'.'``. Each folder is itself a Maildir mailbox but should not contain other folders. Instead, a logical nesting is indicated using ``'.'`` to delimit levels, e.g., "Archived.2005.07". - .. note:: + .. attribute:: Maildir.colon The Maildir specification requires the use of a colon (``':'``) in certain message file names. However, some operating systems do not permit this @@ -346,9 +362,12 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. import mailbox mailbox.Maildir.colon = '!' - The :attr:`colon` attribute may also be set on a per-instance basis. + The :attr:`!colon` attribute may also be set on a per-instance basis. - :class:`Maildir` instances have all of the methods of :class:`Mailbox` in + .. versionchanged:: 3.13 + :class:`Maildir` now ignores files with a leading dot. + + :class:`!Maildir` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -359,14 +378,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return a :class:`Maildir` instance representing the folder whose name is + Return a :class:`!Maildir` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return a :class:`Maildir` + Create a folder whose name is *folder* and return a :class:`!Maildir` instance representing it. @@ -383,7 +402,109 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. last 36 hours. The Maildir specification says that mail-reading programs should do this occasionally. - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special + + .. method:: get_flags(key) + + Return as a string the flags that are set on the message + corresponding to *key*. + This is the same as ``get_message(key).get_flags()`` but much + faster, because it does not open the message file. + Use this method when iterating over the keys to determine which + messages are interesting to get. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.get_flags` method instead, because + changes made by the message's :meth:`~MaildirMessage.set_flags`, + :meth:`~MaildirMessage.add_flag` and :meth:`~MaildirMessage.remove_flag` + methods are not reflected here until the mailbox's + :meth:`__setitem__` method is called. + + .. versionadded:: 3.13 + + + .. method:: set_flags(key, flags) + + On the message corresponding to *key*, set the flags specified + by *flags* and unset all others. + Calling ``some_mailbox.set_flags(key, flags)`` is similar to :: + + one_message = some_mailbox.get_message(key) + one_message.set_flags(flags) + some_mailbox[key] = one_message + + but faster, because it does not open the message file. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.set_flags` method instead, because + changes made with this mailbox method will not be visible to the + message object's method, :meth:`~MaildirMessage.get_flags`. + + .. versionadded:: 3.13 + + + .. method:: add_flag(key, flag) + + On the message corresponding to *key*, set the flags specified + by *flag* without changing other flags. To add more than one + flag at a time, *flag* may be a string of more than one character. + + Considerations for using this method versus the message object's + :meth:`~MaildirMessage.add_flag` method are similar to + those for :meth:`set_flags`; see the discussion there. + + .. versionadded:: 3.13 + + + .. method:: remove_flag(key, flag) + + On the message corresponding to *key*, unset the flags specified + by *flag* without changing other flags. To remove more than one + flag at a time, *flag* may be a string of more than one character. + + Considerations for using this method versus the message object's + :meth:`~MaildirMessage.remove_flag` method are similar to + those for :meth:`set_flags`; see the discussion there. + + .. versionadded:: 3.13 + + + .. method:: get_info(key) + + Return a string containing the info for the message + corresponding to *key*. + This is the same as ``get_message(key).get_info()`` but much + faster, because it does not open the message file. + Use this method when iterating over the keys to determine which + messages are interesting to get. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.get_info` method instead, because + changes made by the message's :meth:`~MaildirMessage.set_info` method + are not reflected here until the mailbox's :meth:`__setitem__` method + is called. + + .. versionadded:: 3.13 + + + .. method:: set_info(key, info) + + Set the info of the message corresponding to *key* to *info*. + Calling ``some_mailbox.set_info(key, flags)`` is similar to :: + + one_message = some_mailbox.get_message(key) + one_message.set_info(info) + some_mailbox[key] = one_message + + but faster, because it does not open the message file. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.set_info` method instead, because + changes made with this mailbox method will not be visible to the + message object's method, :meth:`~MaildirMessage.get_info`. + + .. versionadded:: 3.13 + + Some :class:`Mailbox` methods implemented by :class:`!Maildir` deserve special remarks: @@ -414,7 +535,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() - :class:`Maildir` instances do not keep any open files and the underlying + :class:`!Maildir` instances do not keep any open files and the underlying mailboxes do not support locking, so this method does nothing. @@ -437,8 +558,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mbox: -:class:`mbox` -^^^^^^^^^^^^^ +:class:`!mbox` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: mbox(path, factory=None, create=True) @@ -455,22 +576,22 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. each message indicated by a line whose first five characters are "From ". Several variations of the mbox format exist to address perceived shortcomings in - the original. In the interest of compatibility, :class:`mbox` implements the + the original. In the interest of compatibility, :class:`!mbox` implements the original format, which is sometimes referred to as :dfn:`mboxo`. This means that the :mailheader:`Content-Length` header, if present, is ignored and that any occurrences of "From " at the beginning of a line in a message body are transformed to ">From " when storing the message, although occurrences of ">From " are not transformed to "From " when reading the message. - Some :class:`Mailbox` methods implemented by :class:`mbox` deserve special + Some :class:`Mailbox` methods implemented by :class:`!mbox` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`mbox` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!mbox` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -494,8 +615,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mh: -:class:`MH` -^^^^^^^^^^^ +:class:`!MH` objects +^^^^^^^^^^^^^^^^^^^^ .. class:: MH(path, factory=None, create=True) @@ -515,14 +636,18 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. messages without moving them to sub-folders. Sequences are defined in a file called :file:`.mh_sequences` in each folder. - The :class:`MH` class manipulates MH mailboxes, but it does not attempt to + The :class:`!MH` class manipulates MH mailboxes, but it does not attempt to emulate all of :program:`mh`'s behaviors. In particular, it does not modify and is not affected by the :file:`context` or :file:`.mh_profile` files that are used by :program:`mh` to store its state and configuration. - :class:`MH` instances have all of the methods of :class:`Mailbox` in addition + :class:`!MH` instances have all of the methods of :class:`Mailbox` in addition to the following: + .. versionchanged:: 3.13 + + Supported folders that don't contain a :file:`.mh_sequences` file. + .. method:: list_folders() @@ -531,14 +656,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return an :class:`MH` instance representing the folder whose name is + Return an :class:`!MH` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return an :class:`MH` instance + Create a folder whose name is *folder* and return an :class:`!MH` instance representing it. @@ -572,7 +697,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Already-issued keys are invalidated by this operation and should not be subsequently used. - Some :class:`Mailbox` methods implemented by :class:`MH` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MH` deserve special remarks: @@ -608,7 +733,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() - :class:`MH` instances do not keep any open files, so this method is + :class:`!MH` instances do not keep any open files, so this method is equivalent to :meth:`unlock`. @@ -624,8 +749,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-babyl: -:class:`Babyl` -^^^^^^^^^^^^^^ +:class:`!Babyl` objects +^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Babyl(path, factory=None, create=True) @@ -652,7 +777,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message, and a list of all user-defined labels found in the mailbox is kept in the Babyl options section. - :class:`Babyl` instances have all of the methods of :class:`Mailbox` in + :class:`!Babyl` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -667,7 +792,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. options section, but the Babyl section is updated whenever the mailbox is modified. - Some :class:`Mailbox` methods implemented by :class:`Babyl` deserve special + Some :class:`Mailbox` methods implemented by :class:`!Babyl` deserve special remarks: @@ -700,8 +825,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mmdf: -:class:`MMDF` -^^^^^^^^^^^^^ +:class:`!MMDF` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDF(path, factory=None, create=True) @@ -722,15 +847,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. ">From " when storing messages because the extra message separator lines prevent mistaking such occurrences for the starts of subsequent messages. - Some :class:`Mailbox` methods implemented by :class:`MMDF` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MMDF` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`MMDF` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!MMDF` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -752,20 +877,20 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-message-objects: -:class:`Message` objects ------------------------- +:class:`!Message` objects +------------------------- .. class:: Message(message=None) A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add + :class:`~email.message.Message`. Subclasses of :class:`!mailbox.Message` add mailbox-format-specific state and behavior. If *message* is omitted, the new instance is created in a default, empty state. If *message* is an :class:`email.message.Message` instance, its contents are copied; furthermore, any format-specific information is converted insofar as - possible if *message* is a :class:`Message` instance. If *message* is a string, + possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, or a file, it should contain an :rfc:`2822`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files @@ -780,18 +905,18 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. such as whether a message has been read by the user or marked as important is retained, because it applies to the message itself. - There is no requirement that :class:`Message` instances be used to represent + There is no requirement that :class:`!Message` instances be used to represent messages retrieved using :class:`Mailbox` instances. In some situations, the - time and memory required to generate :class:`Message` representations might - not be acceptable. For such situations, :class:`Mailbox` instances also + time and memory required to generate :class:`!Message` representations might + not be acceptable. For such situations, :class:`!Mailbox` instances also offer string and file-like representations, and a custom message factory may - be specified when a :class:`Mailbox` instance is initialized. + be specified when a :class:`!Mailbox` instance is initialized. .. _mailbox-maildirmessage: -:class:`MaildirMessage` -^^^^^^^^^^^^^^^^^^^^^^^ +:class:`!MaildirMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MaildirMessage(message=None) @@ -826,7 +951,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. | T | Trashed | Marked for subsequent deletion | +------+---------+--------------------------------+ - :class:`MaildirMessage` instances offer the following methods: + :class:`!MaildirMessage` instances offer the following methods: .. method:: get_subdir() @@ -838,7 +963,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. note:: A message is typically moved from :file:`new` to :file:`cur` after its - mailbox has been accessed, whether or not the message is has been + mailbox has been accessed, whether or not the message has been read. A message ``msg`` has been read if ``"S" in msg.get_flags()`` is ``True``. @@ -903,7 +1028,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Set "info" to *info*, which should be a string. -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -923,7 +1048,7 @@ take place: | T flag | D flag | +--------------------+----------------------------------------------+ -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------------------+--------------------------+ @@ -938,7 +1063,7 @@ When a :class:`MaildirMessage` instance is created based upon an | R flag | "replied" sequence | +-------------------------------+--------------------------+ -When a :class:`MaildirMessage` instance is created based upon a +When a :class:`!MaildirMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------------------+-------------------------------+ @@ -958,8 +1083,8 @@ When a :class:`MaildirMessage` instance is created based upon a .. _mailbox-mboxmessage: -:class:`mboxMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!mboxMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: mboxMessage(message=None) @@ -995,7 +1120,7 @@ When a :class:`MaildirMessage` instance is created based upon a "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`mboxMessage` instances offer the following methods: + :class:`!mboxMessage` instances offer the following methods: .. method:: get_from() @@ -1011,8 +1136,8 @@ When a :class:`MaildirMessage` instance is created based upon a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() @@ -1043,7 +1168,7 @@ When a :class:`MaildirMessage` instance is created based upon a remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1062,7 +1187,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`mboxMessage` instance is created based upon an +When an :class:`!mboxMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1077,7 +1202,7 @@ When an :class:`mboxMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1092,7 +1217,8 @@ When an :class:`mboxMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When a :class:`Message` instance is created based upon an :class:`MMDFMessage` +When a :class:`!mboxMessage` instance is created based upon an +:class:`MMDFMessage` instance, the "From " line is copied and all flags directly correspond: +-----------------+----------------------------+ @@ -1112,8 +1238,8 @@ instance, the "From " line is copied and all flags directly correspond: .. _mailbox-mhmessage: -:class:`MHMessage` -^^^^^^^^^^^^^^^^^^ +:class:`!MHMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MHMessage(message=None) @@ -1137,7 +1263,7 @@ instance, the "From " line is copied and all flags directly correspond: | flagged | Marked as important | +----------+------------------------------------------+ - :class:`MHMessage` instances offer the following methods: + :class:`!MHMessage` instances offer the following methods: .. method:: get_sequences() @@ -1159,7 +1285,7 @@ instance, the "From " line is copied and all flags directly correspond: Remove *sequence* from the list of sequences that include this message. -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +--------------------+-------------------------------+ @@ -1172,7 +1298,7 @@ When an :class:`MHMessage` instance is created based upon a | "flagged" sequence | F flag | +--------------------+-------------------------------+ -When an :class:`MHMessage` instance is created based upon an +When an :class:`!MHMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1188,7 +1314,7 @@ take place: | "flagged" sequence | F flag | +--------------------+----------------------------------------------+ -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +--------------------+-----------------------------+ @@ -1202,8 +1328,8 @@ When an :class:`MHMessage` instance is created based upon a .. _mailbox-babylmessage: -:class:`BabylMessage` -^^^^^^^^^^^^^^^^^^^^^ +:class:`!BabylMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: BabylMessage(message=None) @@ -1232,11 +1358,11 @@ When an :class:`MHMessage` instance is created based upon a | resent | Resent | +-----------+------------------------------------------+ - By default, Rmail displays only visible headers. The :class:`BabylMessage` + By default, Rmail displays only visible headers. The :class:`!BabylMessage` class, though, uses the original headers because they are more complete. Visible headers may be accessed explicitly if desired. - :class:`BabylMessage` instances offer the following methods: + :class:`!BabylMessage` instances offer the following methods: .. method:: get_labels() @@ -1275,7 +1401,7 @@ When an :class:`MHMessage` instance is created based upon a .. method:: update_visible() - When a :class:`BabylMessage` instance's original headers are modified, the + When a :class:`!BabylMessage` instance's original headers are modified, the visible headers are not automatically modified to correspond. This method updates the visible headers as follows: each visible header with a corresponding original header is set to the value of the original header, @@ -1285,7 +1411,7 @@ When an :class:`MHMessage` instance is created based upon a present in the original headers but not the visible headers are added to the visible headers. -When a :class:`BabylMessage` instance is created based upon a +When a :class:`!BabylMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +-------------------+-------------------------------+ @@ -1300,7 +1426,7 @@ When a :class:`BabylMessage` instance is created based upon a | "forwarded" label | P flag | +-------------------+-------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1316,7 +1442,7 @@ take place: | "answered" label | A flag | +------------------+----------------------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +------------------+--------------------------+ @@ -1330,8 +1456,8 @@ When a :class:`BabylMessage` instance is created based upon an .. _mailbox-mmdfmessage: -:class:`MMDFMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!MMDFMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDFMessage(message=None) @@ -1365,7 +1491,7 @@ When a :class:`BabylMessage` instance is created based upon an "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`MMDFMessage` instances offer the following methods, which are + :class:`!MMDFMessage` instances offer the following methods, which are identical to those offered by :class:`mboxMessage`: @@ -1382,8 +1508,8 @@ When a :class:`BabylMessage` instance is created based upon an leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). + tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use + :func:`time.gmtime`). .. method:: get_flags() @@ -1414,7 +1540,7 @@ When a :class:`BabylMessage` instance is created based upon an remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1433,7 +1559,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1448,7 +1574,7 @@ When an :class:`MMDFMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1463,7 +1589,7 @@ When an :class:`MMDFMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`mboxMessage` instance, the "From " line is copied and all flags directly correspond: @@ -1485,7 +1611,7 @@ correspond: Exceptions ---------- -The following exception classes are defined in the :mod:`mailbox` module: +The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 0556f19699dc15..f9ba4d554b0c22 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -1,5 +1,5 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= +:mod:`!marshal` --- Internal Python object serialization +======================================================== .. module:: marshal :synopsis: Convert Python objects to streams of bytes and back (with different @@ -10,7 +10,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are +transport the file to a Mac, and read it back there). Details of the format are undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ @@ -23,7 +23,11 @@ transfer of Python objects through RPC calls, see the modules :mod:`pickle` and :mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format -in backward incompatible ways should the need arise. If you're serializing and +in backward incompatible ways should the need arise. +The format of code objects is not compatible between Python versions, +even if the version of the format is the same. +De-serializing a code object in the incorrect Python version has undefined behavior. +If you're serializing and de-serializing Python objects, use the :mod:`pickle` module instead -- the performance is comparable, version independence is guaranteed, and pickle supports a substantially wider range of objects than marshal. @@ -40,7 +44,8 @@ Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by this module. The following types are supported: booleans, integers, floating point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects, where it should be understood that +frozensets, dictionaries, and code objects (if *allow_code* is true), +where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as the values contained therein are themselves supported. The singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be @@ -54,7 +59,7 @@ bytes-like objects. The module defines these functions: -.. function:: dump(value, file[, version]) +.. function:: dump(value, file, version=version, /, *, allow_code=True) Write the value on the open file. The value must be a supported type. The file must be a writeable :term:`binary file`. @@ -62,19 +67,24 @@ The module defines these functions: If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written to the file. The object will not be properly read back by :func:`load`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. The *version* argument indicates the data format that ``dump`` should use (see below). .. audit-event:: marshal.dumps value,version marshal.dump + .. versionchanged:: 3.13 + Added the *allow_code* parameter. -.. function:: load(file) + +.. function:: load(file, /, *, allow_code=True) Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal - format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be a readable :term:`binary file`. + format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. + The file must be a readable :term:`binary file`. .. audit-event:: marshal.load "" marshal.load @@ -88,24 +98,32 @@ The module defines these functions: This call used to raise a ``code.__new__`` audit event for each code object. Now it raises a single ``marshal.load`` event for the entire load operation. + .. versionchanged:: 3.13 + Added the *allow_code* parameter. + -.. function:: dumps(value[, version]) +.. function:: dumps(value, version=version, /, *, allow_code=True) Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. The *version* argument indicates the data format that ``dumps`` should use (see below). .. audit-event:: marshal.dumps value,version marshal.dump + .. versionchanged:: 3.13 + Added the *allow_code* parameter. -.. function:: loads(bytes) + +.. function:: loads(bytes, /, *, allow_code=True) Convert the :term:`bytes-like object` to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the - input are ignored. + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. + :ref:`Code objects <code-objects>` are only supported if *allow_code* is true. + Extra bytes in the input are ignored. .. audit-event:: marshal.loads bytes marshal.load @@ -114,6 +132,9 @@ The module defines these functions: This call used to raise a ``code.__new__`` audit event for each code object. Now it raises a single ``marshal.loads`` event for the entire load operation. + .. versionchanged:: 3.13 + Added the *allow_code* parameter. + In addition, the following constants are defined: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9caf7230eed0aa..316144992d6832 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -1,5 +1,5 @@ -:mod:`math` --- Mathematical functions -====================================== +:mod:`!math` --- Mathematical functions +======================================= .. module:: math :synopsis: Mathematical functions (sin() etc.). @@ -82,6 +82,22 @@ Number-theoretic and representation functions should return an :class:`~numbers.Integral` value. +.. function:: fma(x, y, z) + + Fused multiply-add operation. Return ``(x * y) + z``, computed as though with + infinite precision and range followed by a single round to the ``float`` + format. This operation often provides better accuracy than the direct + expression ``(x * y) + z``. + + This function follows the specification of the fusedMultiplyAdd operation + described in the IEEE 754 standard. The standard leaves one case + implementation-defined, namely the result of ``fma(0, inf, nan)`` + and ``fma(inf, 0, nan)``. In these cases, ``math.fma`` returns a NaN, + and does not raise any exception. + + .. versionadded:: 3.13 + + .. function:: fmod(x, y) Return ``fmod(x, y)``, as defined by the platform C library. Note that the @@ -118,7 +134,7 @@ Number-theoretic and representation functions For further discussion and two alternative approaches, see the `ASPN cookbook recipes for accurate floating point summation - <https://code.activestate.com/recipes/393090/>`_\. + <https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/>`_\. .. function:: gcd(*integers) @@ -239,11 +255,11 @@ Number-theoretic and representation functions See also :func:`math.ulp`. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 Added the *steps* argument. - .. versionadded:: 3.9 - .. function:: perm(n, k=None) Return the number of ways to choose *k* items from *n* items @@ -252,7 +268,7 @@ Number-theoretic and representation functions Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates to zero when ``k > n``. - If *k* is not specified or is None, then *k* defaults to *n* + If *k* is not specified or is ``None``, then *k* defaults to *n* and the function returns ``n!``. Raises :exc:`TypeError` if either of the arguments are not integers. @@ -592,7 +608,7 @@ Special functions The :func:`erf` function can be used to compute traditional statistical functions such as the `cumulative standard normal distribution - <https://en.wikipedia.org/wiki/Normal_distribution#Cumulative_distribution_functions>`_:: + <https://en.wikipedia.org/wiki/Cumulative_distribution_function>`_:: def phi(x): 'Cumulative distribution function for the standard normal distribution' @@ -680,11 +696,11 @@ Constants >>> math.isnan(float('nan')) True + .. versionadded:: 3.5 + .. versionchanged:: 3.11 It is now always available. - .. versionadded:: 3.5 - .. impl-detail:: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f610032acbe417..91e8c30f8607b3 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -1,5 +1,5 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ +:mod:`!mimetypes` --- Map filenames to MIME types +================================================= .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. @@ -52,7 +52,22 @@ the information :func:`init` sets up. are also recognized. .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. + Added support for *url* being a :term:`path-like object`. + + .. deprecated:: 3.13 + Passing a file path instead of URL is :term:`soft deprecated`. + Use :func:`guess_file_type` for this. + + +.. function:: guess_file_type(path, *, strict=True) + + .. index:: pair: MIME; headers + + Guess the type of a file based on its path, given by *path*. + Similar to the :func:`guess_type` function, but accepts a path instead of URL. + Path can be a string, a bytes object or a :term:`path-like object`. + + .. versionadded:: 3.13 .. function:: guess_all_extensions(type, strict=True) @@ -61,7 +76,7 @@ the information :func:`init` sets up. return value is a list of strings giving all possible filename extensions, including the leading dot (``'.'``). The extensions are not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. + type *type* by :func:`guess_type` and :func:`guess_file_type`. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -72,8 +87,8 @@ the information :func:`init` sets up. return value is a string giving a filename extension, including the leading dot (``'.'``). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. + :func:`guess_type` and :func:`guess_file_type`. + If no extension can be guessed for *type*, ``None`` is returned. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -238,6 +253,14 @@ than one MIME-type database; it provides an interface similar to the one of the the object. + .. method:: MimeTypes.guess_file_type(path, *, strict=True) + + Similar to the :func:`guess_file_type` function, using the tables stored + as part of the object. + + .. versionadded:: 3.13 + + .. method:: MimeTypes.guess_all_extensions(type, strict=True) Similar to the :func:`guess_all_extensions` function, using the tables stored diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 69afadff1f5f42..4e20c07331a220 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -1,5 +1,5 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== +:mod:`!mmap` --- Memory-mapped file support +=========================================== .. module:: mmap :synopsis: Interface to memory-mapped files for Unix and Windows. @@ -19,7 +19,7 @@ the current file position, and :meth:`seek` through the file to different positi A memory-mapped file is created by the :class:`~mmap.mmap` constructor, which is different on Unix and on Windows. In either case you must provide a file descriptor for a file opened for update. If you wish to map an existing Python -file object, use its :meth:`fileno` method to obtain the correct value for the +file object, use its :meth:`~io.IOBase.fileno` method to obtain the correct value for the *fileno* parameter. Otherwise, you can open the file using the :func:`os.open` function, which returns a file descriptor directly (the file still needs to be closed when done). @@ -48,7 +48,7 @@ update the underlying file. To map anonymous memory, -1 should be passed as the fileno along with the length. -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset]) +.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT, offset=0) **(Windows version)** Maps *length* bytes from the file specified by the file handle *fileno*, and creates a mmap object. If *length* is larger @@ -62,8 +62,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length the same file. If you specify the name of an existing tag, that tag is opened, otherwise a new tag of this name is created. If this parameter is omitted or ``None``, the mapping is created without a name. Avoiding the - use of the tag parameter will assist in keeping your code portable between - Unix and Windows. + use of the *tagname* parameter will assist in keeping your code portable + between Unix and Windows. *offset* may be specified as a non-negative integer offset. mmap references will be relative to the offset from the beginning of the file. *offset* @@ -71,7 +71,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap -.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) +.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, \ + access=ACCESS_DEFAULT, offset=0, *, trackfd=True) :noindex: **(Unix version)** Maps *length* bytes from the file specified by the file @@ -102,10 +103,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length defaults to 0. *offset* must be a multiple of :const:`ALLOCATIONGRANULARITY` which is equal to :const:`PAGESIZE` on Unix systems. + If *trackfd* is ``False``, the file descriptor specified by *fileno* will + not be duplicated, and the resulting :class:`!mmap` object will not + be associated with the map's underlying file. + This means that the :meth:`~mmap.mmap.size` and :meth:`~mmap.mmap.resize` + methods will fail. + This mode is useful to limit the number of open file descriptors. + To ensure validity of the created memory mapping the file specified by the descriptor *fileno* is internally automatically synchronized with the physical backing store on macOS. + .. versionchanged:: 3.13 + The *trackfd* parameter was added. + This example shows a simple way of using :class:`~mmap.mmap`:: import mmap @@ -254,9 +265,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. method:: resize(newsize) - Resizes the map and the underlying file, if any. If the mmap was created - with :const:`ACCESS_READ` or :const:`ACCESS_COPY`, resizing the map will - raise a :exc:`TypeError` exception. + Resizes the map and the underlying file, if any. + + Resizing a map created with *access* of :const:`ACCESS_READ` or + :const:`ACCESS_COPY`, will raise a :exc:`TypeError` exception. + Resizing a map created with with *trackfd* set to ``False``, + will raise a :exc:`ValueError` exception. **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other maps against the same named file. Resizing an anonymous map (ie against the @@ -285,6 +299,14 @@ To map anonymous memory, -1 should be passed as the fileno along with the length values are ``os.SEEK_CUR`` or ``1`` (seek relative to the current position) and ``os.SEEK_END`` or ``2`` (seek relative to the file's end). + .. versionchanged:: 3.13 + Return the new absolute position instead of ``None``. + + .. method:: seekable() + + Return whether the file supports seeking, and the return value is always ``True``. + + .. versionadded:: 3.13 .. method:: size() @@ -364,14 +386,25 @@ MAP_* Constants .. data:: MAP_SHARED MAP_PRIVATE - MAP_DENYWRITE - MAP_EXECUTABLE + MAP_32BIT + MAP_ALIGNED_SUPER MAP_ANON MAP_ANONYMOUS + MAP_CONCEAL + MAP_DENYWRITE + MAP_EXECUTABLE + MAP_HASSEMAPHORE + MAP_JIT + MAP_NOCACHE + MAP_NOEXTEND + MAP_NORESERVE MAP_POPULATE + MAP_RESILIENT_CODESIGN + MAP_RESILIENT_MEDIA MAP_STACK - MAP_ALIGNED_SUPER - MAP_CONCEAL + MAP_TPRO + MAP_TRANSLATED_ALLOW_EXECUTE + MAP_UNIX03 These are the various flags that can be passed to :meth:`mmap.mmap`. :data:`MAP_ALIGNED_SUPER` is only available at FreeBSD and :data:`MAP_CONCEAL` is only available at OpenBSD. Note @@ -384,5 +417,12 @@ MAP_* Constants Added :data:`MAP_STACK` constant. .. versionadded:: 3.12 - Added :data:`MAP_ALIGNED_SUPER` constant. - Added :data:`MAP_CONCEAL` constant. + Added :data:`MAP_ALIGNED_SUPER` and :data:`MAP_CONCEAL` constants. + + .. versionadded:: 3.13 + Added :data:`MAP_32BIT`, :data:`MAP_HASSEMAPHORE`, :data:`MAP_JIT`, + :data:`MAP_NOCACHE`, :data:`MAP_NOEXTEND`, :data:`MAP_NORESERVE`, + :data:`MAP_RESILIENT_CODESIGN`, :data:`MAP_RESILIENT_MEDIA`, + :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and + :data:`MAP_UNIX03` constants. + diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 526f0ff868c2b7..823d853f1ed8eb 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -1,5 +1,5 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== +:mod:`!modulefinder` --- Find modules used by a script +====================================================== .. module:: modulefinder :synopsis: Find modules used by a script. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 0b059e746c61af..327cc3602b1a77 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -1,5 +1,5 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== +:mod:`!msvcrt` --- Useful routines from the MS VC++ runtime +=========================================================== .. module:: msvcrt :platform: Windows @@ -75,10 +75,14 @@ File Operations .. function:: open_osfhandle(handle, flags) Create a C runtime file descriptor from the file handle *handle*. The *flags* - parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter + parameter should be a bitwise OR of :const:`os.O_APPEND`, + :const:`os.O_RDONLY`, :const:`os.O_TEXT` and :const:`os.O_NOINHERIT`. + The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + The file descriptor is inheritable by default. Pass :const:`os.O_NOINHERIT` + flag to make it non inheritable. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle @@ -207,7 +211,7 @@ Other Functions After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, you can specify the file handle to receive the message text. *type* must be - one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + one of the :const:`!CRT_\*` constants listed below. *file* should be the file handle your want specified. Only available in :ref:`debug build of Python <debug-build>`. @@ -248,3 +252,18 @@ Other Functions .. data:: CRTDBG_REPORT_MODE Returns current *mode* for the specified *type*. + + +.. data:: CRT_ASSEMBLY_VERSION + + The CRT Assembly version, from the :file:`crtassem.h` header file. + + +.. data:: VC_ASSEMBLY_PUBLICKEYTOKEN + + The VC Assembly public key token, from the :file:`crtassem.h` header file. + + +.. data:: LIBRARIES_ASSEMBLY_NAME_PREFIX + + The Libraries Assembly name prefix, from the :file:`crtassem.h` header file. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 231038e6678dbc..bc615b8e297046 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== +:mod:`!multiprocessing` --- Process-based parallelism +===================================================== .. module:: multiprocessing :synopsis: Process-based parallelism. @@ -8,7 +8,7 @@ -------------- -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-ios-notavail.rst Introduction ------------ @@ -56,7 +56,7 @@ will print to standard output :: The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` @@ -102,7 +102,7 @@ necessary, see :ref:`multiprocessing-programming`. .. _multiprocessing-start-methods: Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are @@ -150,18 +150,18 @@ to start a process. These *start methods* are over Unix pipes such as Linux. -.. versionchanged:: 3.8 - - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess as macOS system libraries may start threads. See :issue:`33725`. - .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for some POSIX platforms. Child processes no longer inherit all of the parents inheritable handles on Windows. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess as macOS system libraries may start threads. See :issue:`33725`. + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or @@ -231,7 +231,7 @@ library user. Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` supports two types of communication channel between processes: @@ -254,6 +254,7 @@ processes: p.join() Queues are thread and process safe. + Any object put into a :mod:`~multiprocessing` queue will be serialized. **Pipes** @@ -281,9 +282,11 @@ processes: of corruption from processes using different ends of the pipe at the same time. + The :meth:`~Connection.send` method serializes the the object and + :meth:`~Connection.recv` re-creates the object. Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :mod:`multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure @@ -309,7 +312,7 @@ mixed up. Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As mentioned above, when doing concurrent programming it is usually best to avoid using shared state as far as possible. This is particularly true when @@ -399,7 +402,7 @@ However, if you really do need to use some shared data then Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The :class:`~multiprocessing.pool.Pool` class represents a pool of worker processes. It has methods which allows tasks to be offloaded to the worker @@ -490,7 +493,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the :class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ *, daemon=None) @@ -519,7 +522,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the to the process. .. versionchanged:: 3.3 - Added the *daemon* argument. + Added the *daemon* parameter. .. method:: run() @@ -649,8 +652,8 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: terminate() - Terminate the process. On POSIX this is done using the ``SIGTERM`` signal; - on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and + Terminate the process. On POSIX this is done using the :py:const:`~signal.SIGTERM` signal; + on Windows :c:func:`!TerminateProcess` is used. Note that exit handlers and finally clauses, etc., will not be executed. Note that descendant processes of the process will *not* be terminated -- @@ -724,7 +727,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Raised by methods with a timeout when the timeout expires. Pipes and Queues -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ When using multiple processes, one generally uses message passing for communication between processes and avoids having to use any synchronization @@ -745,6 +748,11 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. +One difference from other Python queue implementations, is that :mod:`multiprocessing` +queues serializes all objects that are put into them using :mod:`pickle`. +The object return by the get method is a re-created object that does not share memory +with the original object. + Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. @@ -811,6 +819,8 @@ For an example of the usage of queues for interprocess communication see used for receiving messages and ``conn2`` can only be used for sending messages. + The :meth:`~multiprocessing.Connection.send` method serializes the the object using + :mod:`pickle` and the :meth:`~multiprocessing.Connection.recv` re-creates the object. .. class:: Queue([maxsize]) @@ -837,6 +847,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. Because of multithreading/multiprocessing semantics, this is not reliable. + May raise an :exc:`OSError` on closed queues. (not guaranteed) + .. method:: full() Return ``True`` if the queue is full, ``False`` otherwise. Because of @@ -940,6 +952,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. + Always raises an :exc:`OSError` if the SimpleQueue is closed. + .. method:: get() Remove and return an item from the queue. @@ -981,7 +995,7 @@ For an example of the usage of queues for interprocess communication see Miscellaneous -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. function:: active_children() @@ -1084,13 +1098,13 @@ Miscellaneous The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. -.. versionchanged:: 3.8 + .. versionadded:: 3.4 - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. + .. versionchanged:: 3.8 - .. versionadded:: 3.4 + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. function:: set_executable(executable) @@ -1150,7 +1164,7 @@ Miscellaneous Connection Objects -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing.connection @@ -1245,8 +1259,7 @@ Connection objects are usually created using Connection objects themselves can now be transferred between processes using :meth:`Connection.send` and :meth:`Connection.recv`. - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see + Connection objects also now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -1292,7 +1305,7 @@ For example: Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: multiprocessing @@ -1481,7 +1494,7 @@ object -- see :ref:`multiprocessing-managers`. Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to create shared objects using shared memory which can be inherited by child processes. @@ -1543,7 +1556,7 @@ inherited by child processes. The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. @@ -1709,7 +1722,7 @@ The results printed are :: .. _multiprocessing-managers: Managers -~~~~~~~~ +^^^^^^^^ Managers provide a way to create data which can be shared between different processes, including sharing over a network between processes running on @@ -1954,7 +1967,7 @@ their parent process exits. The manager classes are defined in the Customized managers ->>>>>>>>>>>>>>>>>>> +""""""""""""""""""" To create one's own manager, one creates a subclass of :class:`BaseManager` and uses the :meth:`~BaseManager.register` classmethod to register new types or @@ -1981,7 +1994,7 @@ callables with the manager class. For example:: Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> +"""""""""""""""""""""" It is possible to run a manager server on one machine and have clients use it from other machines (assuming that the firewalls involved allow it). @@ -2044,7 +2057,7 @@ client to access it remotely:: .. _multiprocessing-proxy_objects: Proxy Objects -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ A proxy is an object which *refers* to a shared object which lives (presumably) in a different process. The shared object is said to be the *referent* of the @@ -2196,7 +2209,7 @@ demonstrates a level of control over the synchronization. Cleanup ->>>>>>> +""""""" A proxy object uses a weakref callback so that when it gets garbage collected it deregisters itself from the manager which owns its referent. @@ -2206,7 +2219,7 @@ any proxies referring to it. Process Pools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ .. module:: multiprocessing.pool :synopsis: Create pools of processes. @@ -2250,11 +2263,11 @@ with the :class:`Pool` class. as CPython does not assure that the finalizer of the pool will be called (see :meth:`object.__del__` for more information). - .. versionadded:: 3.2 - *maxtasksperchild* + .. versionchanged:: 3.2 + Added the *maxtasksperchild* parameter. - .. versionadded:: 3.4 - *context* + .. versionchanged:: 3.4 + Added the *context* parameter. .. versionchanged:: 3.13 *processes* uses :func:`os.process_cpu_count` by default, instead of @@ -2380,7 +2393,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2442,7 +2455,7 @@ The following example demonstrates the use of a pool:: .. _multiprocessing-listeners-clients: Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.connection :synopsis: API for dealing with sockets. @@ -2484,9 +2497,9 @@ multiple connections at the same time. generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2519,9 +2532,9 @@ multiple connections at the same time. to the :meth:`~socket.socket.listen` method of the socket once it has been bound. - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2549,7 +2562,7 @@ multiple connections at the same time. The address from which the last accepted connection came. If this is unavailable then it is ``None``. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -2582,7 +2595,7 @@ multiple connections at the same time. **Windows**: An item in *object_list* must either be an integer handle which is waitable (according to the definition used by the documentation of the Win32 function ``WaitForMultipleObjects()``) - or it can be an object with a :meth:`fileno` method which returns a + or it can be an object with a :meth:`~io.IOBase.fileno` method which returns a socket handle or pipe handle. (Note that pipe handles and socket handles are **not** waitable handles.) @@ -2665,7 +2678,7 @@ wait for messages from multiple processes at once:: .. _multiprocessing-address-formats: Address Formats ->>>>>>>>>>>>>>> +""""""""""""""" * An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where *hostname* is a string and *port* is an integer. @@ -2685,7 +2698,7 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. .. _multiprocessing-auth-keys: Authentication keys -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ When one uses :meth:`Connection.recv <Connection.recv>`, the data received is automatically @@ -2711,7 +2724,7 @@ Suitable authentication keys can also be generated by using :func:`os.urandom`. Logging -~~~~~~~ +^^^^^^^ Some support for logging is available. Note, however, that the :mod:`logging` package does not use process shared locks so it is possible (depending on the @@ -2759,7 +2772,7 @@ For a full table of logging levels, see the :mod:`logging` module. The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. @@ -2818,7 +2831,7 @@ There are certain guidelines and idioms which should be adhered to when using All start methods -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The following applies to all start methods. @@ -2977,9 +2990,9 @@ Beware of replacing :data:`sys.stdin` with a "file like object" For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a few extra restriction which don't apply to the *fork* +There are a few extra restrictions which don't apply to the *fork* start method. More picklability diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index f453e6403d932d..e8f04a6ac7b95d 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes -========================================================================================= +:mod:`!multiprocessing.shared_memory` --- Shared memory for direct access across processes +========================================================================================== .. module:: multiprocessing.shared_memory :synopsis: Provides shared memory for direct access across processes. @@ -20,10 +20,10 @@ and management of shared memory to be accessed by one or more processes on a multicore or symmetric multiprocessor (SMP) machine. To assist with the life-cycle management of shared memory especially across distinct processes, a :class:`~multiprocessing.managers.BaseManager` subclass, -:class:`SharedMemoryManager`, is also provided in the -``multiprocessing.managers`` module. +:class:`~multiprocessing.managers.SharedMemoryManager`, is also provided in the +:mod:`multiprocessing.managers` module. -In this module, shared memory refers to "System V style" shared memory blocks +In this module, shared memory refers to "POSIX style" shared memory blocks (though is not necessarily implemented explicitly as such) and does not refer to "distributed shared memory". This style of shared memory permits distinct processes to potentially read and write to a common (or shared) region of @@ -36,9 +36,10 @@ or other communications requiring the serialization/deserialization and copying of data. -.. class:: SharedMemory(name=None, create=False, size=0) +.. class:: SharedMemory(name=None, create=False, size=0, *, track=True) - Creates a new shared memory block or attaches to an existing shared + Create an instance of the :class:`!SharedMemory` class for either + creating a new shared memory block or attaching to an existing shared memory block. Each shared memory block is assigned a unique name. In this way, one process can create a shared memory block with a particular name and a different process can attach to that same shared @@ -47,43 +48,69 @@ copying of data. As a resource for sharing data across processes, shared memory blocks may outlive the original process that created them. When one process no longer needs access to a shared memory block that might still be - needed by other processes, the :meth:`close()` method should be called. + needed by other processes, the :meth:`close` method should be called. When a shared memory block is no longer needed by any process, the - :meth:`unlink()` method should be called to ensure proper cleanup. - - *name* is the unique name for the requested shared memory, specified as - a string. When creating a new shared memory block, if ``None`` (the - default) is supplied for the name, a novel name will be generated. - - *create* controls whether a new shared memory block is created (``True``) - or an existing shared memory block is attached (``False``). - - *size* specifies the requested number of bytes when creating a new shared - memory block. Because some platforms choose to allocate chunks of memory - based upon that platform's memory page size, the exact size of the shared - memory block may be larger or equal to the size requested. When attaching - to an existing shared memory block, the ``size`` parameter is ignored. + :meth:`unlink` method should be called to ensure proper cleanup. + + :param name: + The unique name for the requested shared memory, specified as a string. + When creating a new shared memory block, if ``None`` (the default) + is supplied for the name, a novel name will be generated. + :type name: str | None + + :param bool create: + Control whether a new shared memory block is created (``True``) + or an existing shared memory block is attached (``False``). + + :param int size: + The requested number of bytes when creating a new shared memory block. + Because some platforms choose to allocate chunks of memory + based upon that platform's memory page size, the exact size of the shared + memory block may be larger or equal to the size requested. + When attaching to an existing shared memory block, + the *size* parameter is ignored. + + :param bool track: + When ``True``, register the shared memory block with a resource + tracker process on platforms where the OS does not do this automatically. + The resource tracker ensures proper cleanup of the shared memory even + if all other processes with access to the memory exit without doing so. + Python processes created from a common ancestor using :mod:`multiprocessing` + facilities share a single resource tracker process, and the lifetime of + shared memory segments is handled automatically among these processes. + Python processes created in any other way will receive their own + resource tracker when accessing shared memory with *track* enabled. + This will cause the shared memory to be deleted by the resource tracker + of the first process that terminates. + To avoid this issue, users of :mod:`subprocess` or standalone Python + processes should set *track* to ``False`` when there is already another + process in place that does the bookkeeping. + *track* is ignored on Windows, which has its own tracking and + automatically deletes shared memory when all handles to it have been closed. + + .. versionchanged:: 3.13 + Added the *track* parameter. .. method:: close() - Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances should call - ``close()`` once the instance is no longer needed. Note that calling - ``close()`` does not cause the shared memory block itself to be - destroyed. + Close the file descriptor/handle to the shared memory from this + instance. :meth:`close` should be called once access to the shared + memory block from this instance is no longer needed. Depending + on operating system, the underlying memory may or may not be freed + even if all handles to it have been closed. To ensure proper cleanup, + use the :meth:`unlink` method. .. method:: unlink() - Requests that the underlying shared memory block be destroyed. In - order to ensure proper cleanup of resources, ``unlink()`` should be - called once (and only once) across all processes which have need - for the shared memory block. After requesting its destruction, a - shared memory block may or may not be immediately destroyed and - this behavior may differ across platforms. Attempts to access data - inside the shared memory block after ``unlink()`` has been called may - result in memory access errors. Note: the last process relinquishing - its hold on a shared memory block may call ``unlink()`` and - :meth:`close()` in either order. + Delete the underlying shared memory block. This should be called only + once per shared memory block regardless of the number of handles to it, + even in other processes. + :meth:`unlink` and :meth:`close` can be called in any order, but + trying to access data inside a shared memory block after :meth:`unlink` + may result in memory access errors, depending on platform. + + This method has no effect on Windows, where the only way to delete a + shared memory block is to close all handles. .. attribute:: buf @@ -126,7 +153,7 @@ instances:: The following example demonstrates a practical use of the :class:`SharedMemory` class with `NumPy arrays <https://numpy.org/>`_, accessing the -same ``numpy.ndarray`` from two distinct Python shells: +same :class:`!numpy.ndarray` from two distinct Python shells: .. doctest:: :options: +SKIP @@ -178,43 +205,43 @@ same ``numpy.ndarray`` from two distinct Python shells: .. class:: SharedMemoryManager([address[, authkey]]) :module: multiprocessing.managers - A subclass of :class:`~multiprocessing.managers.BaseManager` which can be + A subclass of :class:`multiprocessing.managers.BaseManager` which can be used for the management of shared memory blocks across processes. A call to :meth:`~multiprocessing.managers.BaseManager.start` on a - :class:`SharedMemoryManager` instance causes a new process to be started. + :class:`!SharedMemoryManager` instance causes a new process to be started. This new process's sole purpose is to manage the life cycle of all shared memory blocks created through it. To trigger the release of all shared memory blocks managed by that process, call - :meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance. - This triggers a :meth:`SharedMemory.unlink()` call on all of the - :class:`SharedMemory` objects managed by that process and then - stops the process itself. By creating ``SharedMemory`` instances - through a ``SharedMemoryManager``, we avoid the need to manually track + :meth:`~multiprocessing.managers.BaseManager.shutdown` on the instance. + This triggers a :meth:`~multiprocessing.shared_memory.SharedMemory.unlink` call + on all of the :class:`SharedMemory` objects managed by that process and then + stops the process itself. By creating :class:`!SharedMemory` instances + through a :class:`!SharedMemoryManager`, we avoid the need to manually track and trigger the freeing of shared memory resources. This class provides methods for creating and returning :class:`SharedMemory` instances and for creating a list-like object (:class:`ShareableList`) backed by shared memory. - Refer to :class:`multiprocessing.managers.BaseManager` for a description + Refer to :class:`~multiprocessing.managers.BaseManager` for a description of the inherited *address* and *authkey* optional input arguments and how - they may be used to connect to an existing ``SharedMemoryManager`` service + they may be used to connect to an existing :class:`!SharedMemoryManager` service from other processes. .. method:: SharedMemory(size) Create and return a new :class:`SharedMemory` object with the - specified ``size`` in bytes. + specified *size* in bytes. .. method:: ShareableList(sequence) Create and return a new :class:`ShareableList` object, initialized - by the values from the input ``sequence``. + by the values from the input *sequence*. The following example demonstrates the basic mechanisms of a -:class:`SharedMemoryManager`: +:class:`~multiprocessing.managers.SharedMemoryManager`: .. doctest:: :options: +SKIP @@ -232,9 +259,9 @@ The following example demonstrates the basic mechanisms of a >>> smm.shutdown() # Calls unlink() on sl, raw_shm, and another_sl The following example depicts a potentially more convenient pattern for using -:class:`SharedMemoryManager` objects via the :keyword:`with` statement to -ensure that all shared memory blocks are released after they are no longer -needed: +:class:`~multiprocessing.managers.SharedMemoryManager` objects via the +:keyword:`with` statement to ensure that all shared memory blocks are released +after they are no longer needed: .. doctest:: :options: +SKIP @@ -250,38 +277,46 @@ needed: ... p2.join() # Wait for all work to complete in both processes ... total_result = sum(sl) # Consolidate the partial results now in sl -When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the -shared memory blocks created using that manager are all released when the -:keyword:`with` statement's code block finishes execution. +When using a :class:`~multiprocessing.managers.SharedMemoryManager` +in a :keyword:`with` statement, the shared memory blocks created using that +manager are all released when the :keyword:`!with` statement's code block +finishes execution. + + +.. class:: ShareableList(sequence=None, *, name=None) + Provide a mutable list-like object where all values stored within are + stored in a shared memory block. + This constrains storable values to the following built-in data types: -.. class:: ShareableList(sequence=None, \*, name=None) + * :class:`int` (signed 64-bit) + * :class:`float` + * :class:`bool` + * :class:`str` (less than 10M bytes each when encoded as UTF-8) + * :class:`bytes` (less than 10M bytes each) + * ``None`` - Provides a mutable list-like object where all values stored within are - stored in a shared memory block. This constrains storable values to - only the ``int`` (signed 64-bit), ``float``, ``bool``, ``str`` (less - than 10M bytes each when encoded as utf-8), ``bytes`` (less than 10M - bytes each), and ``None`` built-in data types. It also notably - differs from the built-in ``list`` type in that these lists can not - change their overall length (i.e. no append, insert, etc.) and do not - support the dynamic creation of new :class:`ShareableList` instances + It also notably differs from the built-in :class:`list` type + in that these lists can not change their overall length + (i.e. no :meth:`!append`, :meth:`!insert`, etc.) and do not + support the dynamic creation of new :class:`!ShareableList` instances via slicing. - *sequence* is used in populating a new ``ShareableList`` full of values. + *sequence* is used in populating a new :class:`!ShareableList` full of values. Set to ``None`` to instead attach to an already existing - ``ShareableList`` by its unique shared memory name. + :class:`!ShareableList` by its unique shared memory name. *name* is the unique name for the requested shared memory, as described in the definition for :class:`SharedMemory`. When attaching to an - existing ``ShareableList``, specify its shared memory block's unique - name while leaving ``sequence`` set to ``None``. + existing :class:`!ShareableList`, specify its shared memory block's unique + name while leaving *sequence* set to ``None``. .. note:: A known issue exists for :class:`bytes` and :class:`str` values. If they end with ``\x00`` nul bytes or characters, those may be *silently stripped* when fetching them by index from the - :class:`ShareableList`. This ``.rstrip(b'\x00')`` behavior is + :class:`!ShareableList`. This ``.rstrip(b'\x00')`` behavior is considered a bug and may go away in the future. See :gh:`106939`. For applications where rstripping of trailing nulls is a problem, @@ -307,12 +342,12 @@ shared memory blocks created using that manager are all released when the .. method:: count(value) - Returns the number of occurrences of ``value``. + Return the number of occurrences of *value*. .. method:: index(value) - Returns first index position of ``value``. Raises :exc:`ValueError` if - ``value`` is not present. + Return first index position of *value*. + Raise :exc:`ValueError` if *value* is not present. .. attribute:: format @@ -372,8 +407,8 @@ behind it: >>> c.shm.close() >>> c.shm.unlink() -The following examples demonstrates that ``ShareableList`` -(and underlying ``SharedMemory``) objects +The following examples demonstrates that :class:`ShareableList` +(and underlying :class:`SharedMemory`) objects can be pickled and unpickled if needed. Note, that it will still be the same shared object. This happens, because the deserialized object has diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index c36e5cfecfc6a8..f6260383b2b057 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -1,6 +1,5 @@ - -:mod:`netrc` --- netrc file processing -====================================== +:mod:`!netrc` --- netrc file processing +======================================= .. module:: netrc :synopsis: Loading of .netrc files. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 2a05b56db051f9..d0ae79c7a3df76 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -1,5 +1,5 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ +:mod:`!numbers` --- Numeric abstract base classes +================================================= .. module:: numbers :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). @@ -8,7 +8,7 @@ -------------- -The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric +The :mod:`!numbers` module (:pep:`3141`) defines a hierarchy of numeric :term:`abstract base classes <abstract base class>` which progressively define more operations. None of the types defined in this module are intended to be instantiated. @@ -45,7 +45,7 @@ The numeric tower .. class:: Real - To :class:`Complex`, :class:`Real` adds the operations that work on real + To :class:`Complex`, :class:`!Real` adds the operations that work on real numbers. In short, those are: a conversion to :class:`float`, :func:`math.trunc`, @@ -84,10 +84,10 @@ The numeric tower ``~``. -Notes for type implementors +Notes for type implementers --------------------------- -Implementors should be careful to make equal numbers equal and hash +Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different extensions of the real numbers. For example, :class:`fractions.Fraction` implements :func:`hash` as follows:: @@ -126,7 +126,8 @@ We want to implement the arithmetic operations so that mixed-mode operations either call an implementation whose author knew about the types of both arguments, or convert both to the nearest built in type and do the operation there. For subtypes of :class:`Integral`, this -means that :meth:`__add__` and :meth:`__radd__` should be defined as:: +means that :meth:`~object.__add__` and :meth:`~object.__radd__` should be +defined as:: class MyIntegral(Integral): @@ -160,15 +161,15 @@ refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as of :class:`Complex` (``a : A <: Complex``), and ``b : B <: Complex``. I'll consider ``a + b``: -1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is +1. If ``A`` defines an :meth:`~object.__add__` which accepts ``b``, all is well. 2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the - boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at + return a value from :meth:`~object.__add__`, we'd miss the possibility + that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the + boilerplate should return :data:`NotImplemented` from + :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) -3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts +3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts ``a``, all is well. 4. If it falls back to the boilerplate, there are no more possible methods to try, so this is where the default implementation @@ -180,7 +181,7 @@ Complex``. I'll consider ``a + b``: If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, then the appropriate shared operation is the one involving the built -in :class:`complex`, and both :meth:`__radd__` s land there, so ``a+b +in :class:`complex`, and both :meth:`~object.__radd__` s land there, so ``a+b == b+a``. Because most of the operations on any given type will be very similar, diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 57c67bcf3aa12e..a9a6026af406fe 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -1,5 +1,5 @@ -:mod:`operator` --- Standard operators as functions -=================================================== +:mod:`!operator` --- Standard operators as functions +==================================================== .. module:: operator :synopsis: Functions corresponding to the standard operators. @@ -306,7 +306,7 @@ expect a function argument. itemgetter(*items) Return a callable object that fetches *item* from its operand using the - operand's :meth:`__getitem__` method. If multiple items are specified, + operand's :meth:`~object.__getitem__` method. If multiple items are specified, returns a tuple of lookup values. For example: * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. @@ -326,7 +326,7 @@ expect a function argument. return tuple(obj[item] for item in items) return g - The items can be any type accepted by the operand's :meth:`__getitem__` + The items can be any type accepted by the operand's :meth:`~object.__getitem__` method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and strings accept an index or a slice: diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 015e83ed2ce5f7..3e96259f94d47b 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1,5 +1,5 @@ -:mod:`optparse` --- Parser for command line options -=================================================== +:mod:`!optparse` --- Parser for command line options +==================================================== .. module:: optparse :synopsis: Command-line option parsing library. @@ -1739,7 +1739,7 @@ seen, but blow up if it comes after ``-b`` in the command-line. :: Callback example 3: check option order (generalized) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you want to re-use this callback for several similar options (set a flag, but +If you want to reuse this callback for several similar options (set a flag, but blow up if ``-b`` has already been seen), it needs a bit of work: the error message and the flag that it sets must be generalized. :: diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 95933f56d50542..52487b4737ae2f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -1,10 +1,10 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ +:mod:`!os.path` --- Common pathname manipulations +================================================= .. module:: os.path :synopsis: Operations on pathnames. -**Source code:** :source:`Lib/posixpath.py` (for POSIX) and +**Source code:** :source:`Lib/genericpath.py`, :source:`Lib/posixpath.py` (for POSIX) and :source:`Lib/ntpath.py` (for Windows). .. index:: single: path; operations @@ -79,19 +79,20 @@ the :mod:`glob` module.) .. function:: commonpath(paths) - Return the longest common sub-path of each pathname in the sequence + Return the longest common sub-path of each pathname in the iterable *paths*. Raise :exc:`ValueError` if *paths* contain both absolute and relative pathnames, the *paths* are on the different drives or if *paths* is empty. Unlike :func:`commonprefix`, this returns a valid path. - .. availability:: Unix, Windows. - .. versionadded:: 3.5 .. versionchanged:: 3.6 Accepts a sequence of :term:`path-like objects <path-like object>`. + .. versionchanged:: 3.13 + Any iterable can now be passed, rather than just sequences. + .. function:: commonprefix(list) @@ -144,7 +145,7 @@ the :mod:`glob` module.) .. function:: lexists(path) - Return ``True`` if *path* refers to an existing path. Returns ``True`` for + Return ``True`` if *path* refers to an existing path, including broken symbolic links. Equivalent to :func:`exists` on platforms lacking :func:`os.lstat`. @@ -239,12 +240,16 @@ the :mod:`glob` module.) .. function:: isabs(path) Return ``True`` if *path* is an absolute pathname. On Unix, that means it - begins with a slash, on Windows that it begins with a (back)slash after chopping - off a potential drive letter. + begins with a slash, on Windows that it begins with two (back)slashes, or a + drive letter, colon, and (back)slash together. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + On Windows, returns ``False`` if the given path starts with exactly one + (back)slash. + .. function:: isfile(path) @@ -297,8 +302,8 @@ the :mod:`glob` module.) always mount points, and for any other path ``GetVolumePathName`` is called to see if it is different from the input path. - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. + .. versionchanged:: 3.4 + Added support for detecting non-root mount points on Windows. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -317,9 +322,32 @@ the :mod:`glob` module.) Dev Drives. See `the Windows documentation <https://learn.microsoft.com/windows/dev-drive/>`_ for information on enabling and creating Dev Drives. + .. versionadded:: 3.12 + + .. versionchanged:: 3.13 + The function is now available on all platforms, and will always return ``False`` on those that have no support for Dev Drives + + +.. function:: isreserved(path) + + Return ``True`` if *path* is a reserved pathname on the current system. + + On Windows, reserved filenames include those that end with a space or dot; + those that contain colons (i.e. file streams such as "name:stream"), + wildcard characters (i.e. ``'*?"<>'``), pipe, or ASCII control characters; + as well as DOS device names such as "NUL", "CON", "CONIN$", "CONOUT$", + "AUX", "PRN", "COM1", and "LPT1". + + .. note:: + + This function approximates rules for reserved paths on most Windows + systems. These rules change over time in various Windows releases. + This function may be updated in future Python releases as changes to + the rules become broadly available. + .. availability:: Windows. - .. versionadded:: 3.12 + .. versionadded:: 3.13 .. function:: join(path, *paths) @@ -361,7 +389,7 @@ the :mod:`glob` module.) that contains symbolic links. On Windows, it converts forward slashes to backward slashes. To normalize case, use :func:`normcase`. - .. note:: + .. note:: On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 Pathname Resolution <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13>`_, if a pathname begins with exactly two slashes, the first component @@ -381,9 +409,8 @@ the :mod:`glob` module.) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors + are ignored, and so the result might be missing or otherwise inaccessible. .. note:: This function emulates the operating system's procedure for making a path @@ -413,8 +440,6 @@ the :mod:`glob` module.) *start* defaults to :data:`os.curdir`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -425,8 +450,6 @@ the :mod:`glob` module.) This is determined by the device number and i-node number and raises an exception if an :func:`os.stat` call on either pathname fails. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -441,8 +464,6 @@ the :mod:`glob` module.) Return ``True`` if the file descriptors *fp1* and *fp2* refer to the same file. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -457,8 +478,6 @@ the :mod:`glob` module.) :func:`os.lstat`, or :func:`os.stat`. This function implements the underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.4 Added Windows support. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fe573f188ab066..2878d425310d75 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1,5 +1,5 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= +:mod:`!os` --- Miscellaneous operating system interfaces +======================================================== .. module:: os :synopsis: Miscellaneous operating system interfaces. @@ -34,12 +34,12 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, large - parts of the :mod:`os` module are not available or behave differently. API - related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`), signals - (e.g. :func:`~os.kill`, :func:`~os.wait`), and resources - (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` - and :func:`~os.getpid` are emulated or stubs. +* On WebAssembly platforms, and on iOS, large parts of the :mod:`os` module are + not available or behave differently. API related to processes (e.g. + :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) + are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are + emulated or stubs. WebAssembly platforms also lack support for signals (e.g. + :func:`~os.kill`, :func:`~os.wait`). .. note:: @@ -178,7 +178,7 @@ process and user. Return the filename corresponding to the controlling terminal of the process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: environ @@ -193,6 +193,10 @@ process and user. to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. + The :meth:`!os.environ.refresh()` method updates :data:`os.environ` with + changes to the environment made by :func:`os.putenv`, by + :func:`os.unsetenv`, or made outside Python in the same process. + This mapping may be used to modify the environment as well as query the environment. :func:`putenv` will be called automatically when the mapping is modified. @@ -225,6 +229,9 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.14 + Added the :meth:`!os.environ.refresh()` method. + .. data:: environb @@ -355,7 +362,7 @@ process and user. Return the effective group id of the current process. This corresponds to the "set id" bit on the file being executed in the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: geteuid() @@ -364,7 +371,7 @@ process and user. Return the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getgid() @@ -375,8 +382,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getgrouplist(user, group, /) @@ -386,7 +393,7 @@ process and user. field from the password record for *user*, because that group ID will otherwise be potentially omitted. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -395,7 +402,7 @@ process and user. Return list of supplemental group ids associated with the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -423,7 +430,7 @@ process and user. falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the current real user id. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. function:: getpgid(pid) @@ -431,7 +438,7 @@ process and user. Return the process group id of the process with process id *pid*. If *pid* is 0, the process group id of the current process is returned. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpgrp() @@ -439,7 +446,7 @@ process and user. Return the id of the current process group. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpid() @@ -448,8 +455,8 @@ process and user. Return the current process id. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getppid() @@ -459,7 +466,7 @@ process and user. the id returned is the one of the init process (1), on Windows it is still the same id, which may be already reused by another process. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionchanged:: 3.2 Added support for Windows. @@ -477,7 +484,7 @@ process and user. (respectively) the calling process, the process group of the calling process, or the real user ID of the calling process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -488,7 +495,7 @@ process and user. Parameters for the :func:`getpriority` and :func:`setpriority` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -509,7 +516,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -519,7 +526,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -532,8 +539,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: initgroups(username, gid, /) @@ -542,7 +549,7 @@ process and user. the groups of which the specified username is a member, plus the specified group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -561,6 +568,8 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. + See also the :data:`os.environ.refresh() <os.environ>` method. + .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may @@ -576,21 +585,21 @@ process and user. Set the current process's effective group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: seteuid(euid, /) Set the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setgid(gid, /) Set the current process' group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setgroups(groups, /) @@ -599,7 +608,7 @@ process and user. *groups*. *groups* must be a sequence, and each element must be an integer identifying a group. This operation is typically available only to the superuser. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: On macOS, the length of *groups* may not exceed the system-defined maximum number of effective group ids, typically 16. @@ -649,7 +658,7 @@ process and user. Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpgid(pid, pgrp, /) @@ -658,7 +667,7 @@ process and user. process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpriority(which, who, priority) @@ -675,7 +684,7 @@ process and user. *priority* is a value in the range -20 to 19. The default priority is 0; lower priorities cause more favorable scheduling. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -684,14 +693,14 @@ process and user. Set the current process's real and effective group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setresgid(rgid, egid, sgid, /) Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -700,7 +709,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -709,21 +718,21 @@ process and user. Set the current process's real and effective user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getsid(pid, /) Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setsid() Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setuid(uid, /) @@ -732,7 +741,7 @@ process and user. Set the current process's user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. placed in this section since it relates to errno.... a little weak @@ -755,8 +764,8 @@ process and user. Set the current numeric umask and return the previous umask. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: uname() @@ -784,6 +793,11 @@ process and user. :func:`socket.gethostname` or even ``socket.gethostbyaddr(socket.gethostname())``. + On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname()` + can be used to get the user-facing operating system name and version on iOS and + Android. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -804,6 +818,8 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. + See also the :data:`os.environ.refresh() <os.environ>` method. + .. audit-event:: os.unsetenv key os.unsetenv .. versionchanged:: 3.9 @@ -918,10 +934,10 @@ as internal buffering of data. Copy *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - If *offset_src* is None, then *src* is read from the current position; + If *offset_src* is ``None``, then *src* is read from the current position; respectively for *offset_dst*. - In Linux kernel older than 5.3, the files pointed by *src* and *dst* + In Linux kernel older than 5.3, the files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1001,10 +1017,13 @@ as internal buffering of data. .. audit-event:: os.chmod path,mode,dir_fd os.fchmod - .. availability:: Unix. + .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. + + .. versionchanged:: 3.13 + Added support on Windows. .. function:: fchown(fd, uid, gid) @@ -1018,8 +1037,8 @@ as internal buffering of data. .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. function:: fdatasync(fd) @@ -1109,8 +1128,8 @@ as internal buffering of data. .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1119,6 +1138,20 @@ as internal buffering of data. .. versionchanged:: 3.12 Added support for pipes on Windows. + +.. function:: grantpt(fd, /) + + Grant access to the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`grantpt`. + + .. availability:: Unix, not WASI. + + .. versionadded:: 3.13 + + .. function:: isatty(fd, /) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -1158,7 +1191,7 @@ as internal buffering of data. Make the calling process a session leader; make the tty the controlling tty, the stdin, the stdout, and the stderr of the calling process; close fd. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1251,8 +1284,8 @@ as internal buffering of data. :meth:`~file.read` and :meth:`~file.write` methods (and many more). To wrap a file descriptor in a file object, use :func:`fdopen`. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1342,7 +1375,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo descriptors are :ref:`non-inheritable <fd_inheritance>`. For a (slightly) more portable approach, use the :mod:`pty` module. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.4 The new file descriptors are now non-inheritable. @@ -1368,7 +1401,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1378,7 +1411,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix, not Emscripten. + .. availability:: Unix. .. versionadded:: 3.3 @@ -1426,6 +1459,23 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.3 +.. function:: posix_openpt(oflag, /) + + Open and return a file descriptor for a master pseudo-terminal device. + + Calls the C standard library function :c:func:`posix_openpt`. The *oflag* + argument is used to set file status flags and file access modes as + specified in the manual page of :c:func:`posix_openpt` of your system. + + The returned file descriptor is :ref:`non-inheritable <fd_inheritance>`. + If the value :data:`O_CLOEXEC` is available on the system, it is added to + *oflag*. + + .. availability:: Unix, not WASI. + + .. versionadded:: 3.13 + + .. function:: preadv(fd, buffers, offset, flags=0, /) Read from a file descriptor *fd* at a position of *offset* into mutable @@ -1483,6 +1533,21 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. versionadded:: 3.7 +.. function:: ptsname(fd, /) + + Return the name of the slave pseudo-terminal device associated with the + master pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the reentrant C standard library function :c:func:`ptsname_r` if + it is available; otherwise, the C standard library function + :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. + + .. availability:: Unix, not WASI. + + .. versionadded:: 3.13 + + .. function:: pwrite(fd, str, offset, /) Write the bytestring in *str* to file descriptor *fd* at position of @@ -1605,7 +1670,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Cross-platform applications should not use *headers*, *trailers* and *flags* arguments. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -1625,7 +1690,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Parameters to the :func:`sendfile` function, if the implementation supports them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1634,7 +1699,7 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Parameter to the :func:`sendfile` function, if the implementation supports it. The data won't be cached in the virtual memory and will be freed afterwards. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1648,8 +1713,8 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1659,14 +1724,31 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo Added support for pipes on Windows. -.. function:: splice(src, dst, count, offset_src=None, offset_dst=None) +.. function:: splice(src, dst, count, offset_src=None, offset_dst=None, flags=0) Transfer *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + + The splicing behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + * If :const:`SPLICE_F_MOVE` is specified, + the kernel is asked to move pages instead of copying, + but pages may still be copied if the kernel cannot move the pages from the pipe. + + * If :const:`SPLICE_F_NONBLOCK` is specified, + the kernel is asked to not block on I/O. + This makes the splice pipe operations nonblocking, + but splice may nevertheless block because the spliced file descriptors may block. + + * If :const:`SPLICE_F_MORE` is specified, + it hints to the kernel that more data will be coming in a subsequent splice. + At least one of the file descriptors must refer to a pipe. If *offset_src* - is None, then *src* is read from the current position; respectively for + is ``None``, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in + pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1681,6 +1763,8 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo make sense to block because there are no writers connected to the write end of the pipe. + .. seealso:: The :manpage:`splice(2)` man page. + .. availability:: Linux >= 2.6.17 with glibc >= 2.5 .. versionadded:: 3.10 @@ -1735,6 +1819,19 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo .. availability:: Unix. +.. function:: unlockpt(fd, /) + + Unlock the slave pseudo-terminal device associated with the master + pseudo-terminal device to which the file descriptor *fd* refers. + The file descriptor *fd* is not closed upon failure. + + Calls the C standard library function :c:func:`unlockpt`. + + .. availability:: Unix, not WASI. + + .. versionadded:: 3.13 + + .. function:: write(fd, str, /) Write the bytestring in *str* to file descriptor *fd*. @@ -1831,8 +1928,7 @@ Using the :mod:`subprocess` module, all file descriptors except standard streams are closed, and inheritable handles are only inherited if the *close_fds* parameter is ``False``. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, the file -descriptor cannot be modified. +On WebAssembly platforms, the file descriptor cannot be modified. .. function:: get_inheritable(fd, /) @@ -1988,7 +2084,7 @@ features: .. audit-event:: os.chdir path os.chdir - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -2018,10 +2114,10 @@ features: .. audit-event:: os.chflags path,flags os.chflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. - .. versionadded:: 3.3 - The *follow_symlinks* argument. + .. versionchanged:: 3.3 + Added the *follow_symlinks* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2062,19 +2158,24 @@ features: Although Windows supports :func:`chmod`, you can only set the file's read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + The default value of *follow_symlinks* is ``False`` on Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. audit-event:: os.chmod path,mode,dir_fd os.chmod - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support for a file descriptor and the *follow_symlinks* argument + on Windows. + .. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) @@ -2092,10 +2193,10 @@ features: .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -2107,7 +2208,7 @@ features: Change the root directory of the current process to *path*. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2147,7 +2248,7 @@ features: .. audit-event:: os.chflags path,flags os.lchflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2160,13 +2261,19 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + ``lchmod()`` is not part of POSIX, but Unix implementations may have it if + changing the mode of symbolic links is supported. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod - .. availability:: Unix. + .. availability:: Unix, Windows, not Linux, FreeBSD >= 1.3, NetBSD >= 1.3, not OpenBSD .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support on Windows. + .. function:: lchown(path, uid, gid) Change the owner and group id of *path* to the numeric *uid* and *gid*. This @@ -2191,13 +2298,13 @@ features: .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link - .. availability:: Unix, Windows, not Emscripten. + .. availability:: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. - .. versionadded:: 3.3 - Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* arguments. + .. versionchanged:: 3.3 + Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* parameters. .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. @@ -2233,7 +2340,7 @@ features: .. versionchanged:: 3.2 The *path* parameter became optional. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.6 @@ -2353,6 +2460,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors <dir_fd>`. @@ -2361,12 +2472,15 @@ features: .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) @@ -2395,8 +2509,8 @@ features: .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs - .. versionadded:: 3.2 - The *exist_ok* parameter. + .. versionchanged:: 3.2 + Added the *exist_ok* parameter. .. versionchanged:: 3.4.1 @@ -2427,10 +2541,10 @@ features: FIFO for reading, and the client opens it for writing. Note that :func:`mkfifo` doesn't open the FIFO --- it just creates the rendezvous point. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2449,10 +2563,10 @@ features: This function can also support :ref:`paths relative to directory descriptors <dir_fd>`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2532,8 +2646,8 @@ features: .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object` on Unix. @@ -2541,7 +2655,6 @@ features: .. versionchanged:: 3.8 Accepts a :term:`path-like object` and a bytes object on Windows. - .. versionchanged:: 3.8 Added support for directory junctions, and changed to return the substitution path (which typically includes ``\\?\`` prefix) rather than the optional "print name" field that was previously returned. @@ -2563,8 +2676,8 @@ features: .. audit-event:: os.remove path,dir_fd os.remove - .. versionadded:: 3.3 - The *dir_fd* argument. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2614,8 +2727,8 @@ features: .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename - .. versionadded:: 3.3 - The *src_dir_fd* and *dst_dir_fd* arguments. + .. versionchanged:: 3.3 + Added the *src_dir_fd* and *dst_dir_fd* parameters. .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. @@ -2670,8 +2783,8 @@ features: .. audit-event:: os.rmdir path,dir_fd os.rmdir - .. versionadded:: 3.3 - The *dir_fd* parameter. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2745,7 +2858,7 @@ features: .. versionadded:: 3.5 - .. versionadded:: 3.6 + .. versionchanged:: 3.6 Added support for the :term:`context manager` protocol and the :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted @@ -2959,9 +3072,9 @@ features: :func:`fstat` and :func:`lstat` functions. - .. versionadded:: 3.3 - Added the *dir_fd* and *follow_symlinks* arguments, specifying a file - descriptor instead of a path. + .. versionchanged:: 3.3 + Added the *dir_fd* and *follow_symlinks* parameters, + specifying a file descriptor instead of a path. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3042,16 +3155,22 @@ features: Time of most recent access expressed in nanoseconds as an integer. + .. versionadded:: 3.3 + .. attribute:: st_mtime_ns Time of most recent content modification expressed in nanoseconds as an integer. + .. versionadded:: 3.3 + .. attribute:: st_ctime_ns Time of most recent metadata change expressed in nanoseconds as an integer. + .. versionadded:: 3.3 + .. versionchanged:: 3.12 ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` for the file creation time. In the future, ``st_ctime`` will contain @@ -3152,6 +3271,8 @@ features: See the :const:`!FILE_ATTRIBUTE_* <stat.FILE_ATTRIBUTE_ARCHIVE>` constants in the :mod:`stat` module. + .. versionadded:: 3.5 + .. attribute:: st_reparse_tag When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` @@ -3172,21 +3293,14 @@ features: some implementations. For compatibility with older Python versions, accessing :class:`stat_result` as a tuple always returns integers. - .. versionadded:: 3.3 - Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and - :attr:`st_ctime_ns` members. - - .. versionadded:: 3.5 - Added the :attr:`st_file_attributes` member on Windows. - .. versionchanged:: 3.5 Windows now returns the file index as :attr:`st_ino` when available. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 Added the :attr:`st_fstype` member to Solaris/derivatives. - .. versionadded:: 3.8 + .. versionchanged:: 3.8 Added the :attr:`st_reparse_tag` member on Windows. .. versionchanged:: 3.8 @@ -3200,16 +3314,13 @@ features: platforms, but for now still contains creation time. Use :attr:`st_birthtime` for the creation time. - .. versionchanged:: 3.12 On Windows, :attr:`st_ino` may now be up to 128 bits, depending on the file system. Previously it would not be above 64 bits, and larger file identifiers would be arbitrarily packed. - .. versionchanged:: 3.12 On Windows, :attr:`st_rdev` no longer returns a value. Previously it would contain the same as :attr:`st_dev`, which was incorrect. - .. versionadded:: 3.12 Added the :attr:`st_birthtime` member on Windows. @@ -3243,7 +3354,7 @@ features: .. versionchanged:: 3.2 The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.4 @@ -3255,8 +3366,8 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionadded:: 3.7 - Added :attr:`f_fsid`. + .. versionchanged:: 3.7 + Added the :attr:`f_fsid` attribute. .. data:: supports_dir_fd @@ -3374,14 +3485,14 @@ features: .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - Added the *dir_fd* argument, and now allow *target_is_directory* + .. versionchanged:: 3.3 + Added the *dir_fd* parameter, and now allow *target_is_directory* on non-Windows platforms. .. versionchanged:: 3.6 @@ -3429,8 +3540,8 @@ features: .. audit-event:: os.remove path,dir_fd os.unlink - .. versionadded:: 3.3 - The *dir_fd* parameter. + .. versionchanged:: 3.3 + Added the *dir_fd* parameter. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3468,7 +3579,7 @@ features: .. audit-event:: os.utime path,times,ns,dir_fd os.utime - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. @@ -4170,7 +4281,7 @@ to be ignored. The "l" and "v" variants of the :func:`exec\* <execl>` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the :func:`execl\*` + individual parameters simply become additional parameters to the :func:`!execl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process should start with @@ -4201,9 +4312,9 @@ to be ignored. .. audit-event:: os.exec path,args,env os.execl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor for :func:`execve`. @@ -4244,49 +4355,49 @@ written in Python, such as a mail server's external command delivery program. Exit code that means the command was used incorrectly, such as when the wrong number of arguments are given. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_DATAERR Exit code that means the input data was incorrect. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOINPUT Exit code that means an input file did not exist or was not readable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOUSER Exit code that means a specified user did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOHOST Exit code that means a specified host did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_UNAVAILABLE Exit code that means that a required service is unavailable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_SOFTWARE Exit code that means an internal software error was detected. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSERR @@ -4294,7 +4405,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means an operating system error was detected, such as the inability to fork or create a pipe. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSFILE @@ -4302,21 +4413,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means some system file did not exist, could not be opened, or had some other kind of error. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CANTCREAT Exit code that means a user specified output file could not be created. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_IOERR Exit code that means that an error occurred while doing I/O on some file. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_TEMPFAIL @@ -4325,7 +4436,7 @@ written in Python, such as a mail server's external command delivery program. that may not really be an error, such as a network connection that couldn't be made during a retryable operation. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_PROTOCOL @@ -4333,7 +4444,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that a protocol exchange was illegal, invalid, or not understood. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOPERM @@ -4341,21 +4452,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that there were insufficient permissions to perform the operation (but not intended for file system problems). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CONFIG Exit code that means that some kind of configuration error occurred. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOTFOUND Exit code that means something like "an entry was not found". - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: fork() @@ -4373,6 +4484,11 @@ written in Python, such as a mail server's external command delivery program. If you use TLS sockets in an application calling ``fork()``, see the warning in the :mod:`ssl` documentation. + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -4399,7 +4515,7 @@ written in Python, such as a mail server's external command delivery program. for technical details of why we're surfacing this longstanding platform compatibility problem to developers. - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not iOS. .. function:: forkpty() @@ -4412,16 +4528,21 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.forkpty "" os.forkpty - .. versionchanged:: 3.12 - If Python is able to detect that your process has multiple - threads, this now raises a :exc:`DeprecationWarning`. See the - longer explanation on :func:`os.fork`. + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. .. versionchanged:: 3.8 Calling ``forkpty()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). - .. availability:: Unix, not Emscripten, not WASI. + .. versionchanged:: 3.12 + If Python is able to detect that your process has multiple + threads, this now raises a :exc:`DeprecationWarning`. See the + longer explanation on :func:`os.fork`. + + .. availability:: Unix, not WASI, not iOS. .. function:: kill(pid, sig, /) @@ -4445,10 +4566,10 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.kill pid,sig os.kill - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. - .. versionadded:: 3.2 - Windows support. + .. versionchanged:: 3.2 + Added Windows support. .. function:: killpg(pgid, sig, /) @@ -4461,14 +4582,14 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.killpg pgid,sig os.killpg - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: nice(increment, /) Add *increment* to the process's "niceness". Return the new niceness. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: pidfd_open(pid, flags=0) @@ -4498,7 +4619,7 @@ written in Python, such as a mail server's external command delivery program. Lock program segments into memory. The value of *op* (defined in ``<sys/lock.h>``) determines which segments are locked. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -4530,7 +4651,7 @@ written in Python, such as a mail server's external command delivery program. documentation for more powerful ways to manage and communicate with subprocesses. - .. availability:: not Emscripten, not WASI. + .. availability:: not WASI, not iOS. .. note:: The :ref:`Python UTF-8 Mode <utf8-mode>` affects encodings used @@ -4540,6 +4661,10 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4550,7 +4675,8 @@ written in Python, such as a mail server's external command delivery program. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. The positional-only arguments *path*, *args*, and *env* are similar to - :func:`execve`. + :func:`execve`. *env* is allowed to be ``None``, in which case current + process' environment is used. The *path* parameter is the path to the executable file. The *path* should contain a directory. Use :func:`posix_spawnp` to pass an executable file @@ -4580,10 +4706,17 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. + .. data:: POSIX_SPAWN_CLOSEFROM + + (``os.POSIX_SPAWN_CLOSEFROM``, *fd*) + + Performs ``os.closerange(fd, INF)``. + These tuples correspond to the C library :c:func:`!posix_spawn_file_actions_addopen`, - :c:func:`!posix_spawn_file_actions_addclose`, and - :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + :c:func:`!posix_spawn_file_actions_addclose`, + :c:func:`!posix_spawn_file_actions_adddup2`, and + :c:func:`!posix_spawn_file_actions_addclosefrom_np` API calls used to prepare for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value @@ -4625,7 +4758,12 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.8 - .. availability:: Unix, not Emscripten, not WASI. + .. versionchanged:: 3.13 + *env* parameter accepts ``None``. + ``os.POSIX_SPAWN_CLOSEFROM`` is available on platforms where + :c:func:`!posix_spawn_file_actions_addclosefrom_np` exists. + + .. availability:: Unix, not WASI, not iOS. .. function:: posix_spawnp(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4641,7 +4779,7 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.8 - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not iOS. See :func:`posix_spawn` documentation. @@ -4674,7 +4812,7 @@ written in Python, such as a mail server's external command delivery program. There is no way to unregister a function. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. versionadded:: 3.7 @@ -4708,7 +4846,7 @@ written in Python, such as a mail server's external command delivery program. command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the individual parameters simply become additional parameters to the - :func:`spawnl\*` functions. The "v" variants are good when the number of + :func:`!spawnl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process must start with the name of the command being run. @@ -4743,7 +4881,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.spawn mode,path,args,env os.spawnl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and @@ -4753,12 +4891,16 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.14 + These functions are :term:`soft deprecated` and should no longer be used + to write new code. The :mod:`subprocess` module is recommended instead. + .. data:: P_NOWAIT P_NOWAITO Possible values for the *mode* parameter to the :func:`spawn\* <spawnl>` family of - functions. If either of these values is given, the :func:`spawn\*` functions + functions. If either of these values is given, the :func:`spawn\* <spawnl>` functions will return as soon as the new process has been created, with the process id as the return value. @@ -4768,7 +4910,7 @@ written in Python, such as a mail server's external command delivery program. .. data:: P_WAIT Possible value for the *mode* parameter to the :func:`spawn\* <spawnl>` family of - functions. If this is given as *mode*, the :func:`spawn\*` functions will not + functions. If this is given as *mode*, the :func:`spawn\* <spawnl>` functions will not return until the new process has run to completion and will return the exit code of the process the run is successful, or ``-signal`` if a signal kills the process. @@ -4857,7 +4999,7 @@ written in Python, such as a mail server's external command delivery program. shell documentation. The :mod:`subprocess` module provides more powerful facilities for spawning - new processes and retrieving their results; using that module is preferable + new processes and retrieving their results; using that module is recommended to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. @@ -4867,7 +5009,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.system command os.system - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. .. function:: times() @@ -4911,7 +5053,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. seealso:: @@ -4945,10 +5087,13 @@ written in Python, such as a mail server's external command delivery program. Otherwise, if there are no matching children that could be waited for, :exc:`ChildProcessError` is raised. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. versionadded:: 3.3 + .. versionchanged:: 3.13 + This function is now available on macOS as well. + .. function:: waitpid(pid, options, /) @@ -4983,7 +5128,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -5003,7 +5148,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: wait4(pid, options) @@ -5017,7 +5162,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. data:: P_PID @@ -5034,7 +5179,7 @@ written in Python, such as a mail server's external command delivery program. * :data:`!P_PIDFD` - wait for the child identified by the file descriptor *id* (a process file descriptor created with :func:`pidfd_open`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. @@ -5049,7 +5194,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` causes child processes to be reported if they have been continued from a job control stop since they were last reported. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. data:: WEXITED @@ -5060,7 +5205,7 @@ written in Python, such as a mail server's external command delivery program. The other ``wait*`` functions always report children that have terminated, so this option is not available for them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. versionadded:: 3.3 @@ -5072,7 +5217,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. versionadded:: 3.3 @@ -5085,7 +5230,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. data:: WNOHANG @@ -5094,7 +5239,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` to return right away if no child process status is available immediately. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. data:: WNOWAIT @@ -5104,7 +5249,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. data:: CLD_EXITED @@ -5117,7 +5262,7 @@ written in Python, such as a mail server's external command delivery program. These are the possible values for :attr:`!si_code` in the result returned by :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. versionadded:: 3.3 @@ -5152,7 +5297,7 @@ written in Python, such as a mail server's external command delivery program. :func:`WIFEXITED`, :func:`WEXITSTATUS`, :func:`WIFSIGNALED`, :func:`WTERMSIG`, :func:`WIFSTOPPED`, :func:`WSTOPSIG` functions. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. .. versionadded:: 3.9 @@ -5168,7 +5313,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WIFCONTINUED(status) @@ -5179,7 +5324,7 @@ used to determine the disposition of a process. See :data:`WCONTINUED` option. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WIFSTOPPED(status) @@ -5191,14 +5336,14 @@ used to determine the disposition of a process. done using :data:`WUNTRACED` option or when the process is being traced (see :manpage:`ptrace(2)`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WIFSIGNALED(status) Return ``True`` if the process was terminated by a signal, otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WIFEXITED(status) @@ -5207,7 +5352,7 @@ used to determine the disposition of a process. by calling ``exit()`` or ``_exit()``, or by returning from ``main()``; otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WEXITSTATUS(status) @@ -5216,7 +5361,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFEXITED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WSTOPSIG(status) @@ -5225,7 +5370,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSTOPPED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: WTERMSIG(status) @@ -5234,7 +5379,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. Interface to the scheduler @@ -5606,20 +5751,20 @@ Random numbers easy-to-use interface to the random number generator provided by your platform, please see :class:`random.SystemRandom`. - .. versionchanged:: 3.6.0 - On Linux, ``getrandom()`` is now used in blocking mode to increase the - security. - - .. versionchanged:: 3.5.2 - On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool - is not initialized yet), fall back on reading ``/dev/urandom``. - .. versionchanged:: 3.5 On Linux 3.17 and newer, the ``getrandom()`` syscall is now used when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` function is now used. These functions avoid the usage of an internal file descriptor. + .. versionchanged:: 3.5.2 + On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool + is not initialized yet), fall back on reading ``/dev/urandom``. + + .. versionchanged:: 3.6 + On Linux, ``getrandom()`` is now used in blocking mode to increase the + security. + .. versionchanged:: 3.11 On Windows, ``BCryptGenRandom()`` is used instead of ``CryptGenRandom()`` which is deprecated. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 8ee89a003a339a..f139abd2454d69 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1,13 +1,12 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== +:mod:`!pathlib` --- Object-oriented filesystem paths +==================================================== .. module:: pathlib :synopsis: Object-oriented filesystem paths .. versionadded:: 3.4 -**Source code:** :source:`Lib/pathlib.py` +**Source code:** :source:`Lib/pathlib/` .. index:: single: path; operations @@ -22,6 +21,12 @@ inherit from pure paths but also provide I/O operations. .. image:: pathlib-inheritance.png :align: center :class: invert-in-dark-mode + :alt: Inheritance diagram showing the classes available in pathlib. The + most basic class is PurePath, which has three direct subclasses: + PurePosixPath, PureWindowsPath, and Path. Further to these four + classes, there are two classes that use multiple inheritance: + PosixPath subclasses PurePosixPath and Path, and WindowsPath + subclasses PureWindowsPath and Path. If you've never used this module before or just aren't sure which class is right for your task, :class:`Path` is most likely what you need. It instantiates @@ -303,10 +308,10 @@ Methods and properties Pure paths provide the following methods and properties: -.. attribute:: PurePath.pathmod +.. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - operations: either ``posixpath`` or ``ntpath``. + parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. .. versionadded:: 3.13 @@ -450,6 +455,10 @@ Pure paths provide the following methods and properties: This is commonly called the file extension. + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.suffixes A list of the path's suffixes, often called file extensions:: @@ -461,6 +470,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffixes [] + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.stem @@ -485,19 +498,6 @@ Pure paths provide the following methods and properties: 'c:/windows' -.. method:: PurePath.as_uri() - - Represent the path as a ``file`` URI. :exc:`ValueError` is raised if - the path isn't absolute. - - >>> p = PurePosixPath('/etc/passwd') - >>> p.as_uri() - 'file:///etc/passwd' - >>> p = PureWindowsPath('c:/Windows') - >>> p.as_uri() - 'file:///c:/Windows' - - .. method:: PurePath.is_absolute() Return whether the path is absolute or not. A path is considered absolute @@ -528,6 +528,13 @@ Pure paths provide the following methods and properties: >>> p.is_relative_to('/usr') False + This method is string-based; it neither accesses the filesystem nor treats + "``..``" segments specially. The following code is equivalent: + + >>> u = PurePath('/usr') + >>> u == p or u in p.parents + False + .. versionadded:: 3.9 .. deprecated-removed:: 3.12 3.14 @@ -541,14 +548,13 @@ Pure paths provide the following methods and properties: reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, ``False`` is always returned. - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. + .. versionchanged:: 3.13 + Windows path names that contain a colon, or end with a dot or a space, + are considered reserved. UNC paths may be reserved. + .. deprecated-removed:: 3.13 3.15 + This method is deprecated; use :func:`os.path.isreserved` to detect + reserved paths on Windows. .. method:: PurePath.joinpath(*pathsegments) @@ -565,51 +571,57 @@ Pure paths provide the following methods and properties: PureWindowsPath('c:/Program Files') -.. method:: PurePath.match(pattern, *, case_sensitive=None) +.. method:: PurePath.full_match(pattern, *, case_sensitive=None) Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. - - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: + if matching is successful, ``False`` otherwise. For example:: - >>> PurePath('a/b.py').match('*.py') - True - >>> PurePath('/a/b/c.py').match('b/*.py') + >>> PurePath('a/b.py').full_match('a/*.py') True - >>> PurePath('/a/b/c.py').match('a/*.py') + >>> PurePath('a/b.py').full_match('*.py') False - - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: - - >>> PurePath('/a.py').match('/*.py') + >>> PurePath('/a/b/c.py').full_match('/a/**') True - >>> PurePath('a/b.py').match('/*.py') - False - - The *pattern* may be another path object; this speeds up matching the same - pattern against multiple files:: - - >>> pattern = PurePath('*.py') - >>> PurePath('a/b.py').match(pattern) + >>> PurePath('/a/b/c.py').full_match('**/*.py') True + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + As with other methods, case-sensitivity follows platform defaults:: - >>> PurePosixPath('b.py').match('*.PY') + >>> PurePosixPath('b.py').full_match('*.PY') False - >>> PureWindowsPath('b.py').match('*.PY') + >>> PureWindowsPath('b.py').full_match('*.PY') True Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. versionadded:: 3.13 + + +.. method:: PurePath.match(pattern, *, case_sensitive=None) + + Match this path against the provided non-recursive glob-style pattern. + Return ``True`` if matching is successful, ``False`` otherwise. + + This method is similar to :meth:`~PurePath.full_match`, but empty patterns + aren't allowed (:exc:`ValueError` is raised), the recursive wildcard + "``**``" isn't supported (it acts like non-recursive "``*``"), and if a + relative pattern is provided, then matching is done from the right:: + + >>> PurePath('a/b.py').match('*.py') + True + >>> PurePath('/a/b/c.py').match('b/*.py') + True + >>> PurePath('/a/b/c.py').match('a/*.py') + False + .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. + The *pattern* parameter accepts a :term:`path-like object`. - .. versionchanged:: 3.13 - Support for the recursive wildcard "``**``" was added. In previous - versions, it acted like the non-recursive wildcard "``*``". + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. .. method:: PurePath.relative_to(other, walk_up=False) @@ -629,8 +641,8 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. - When *walk_up* is False (the default), the path must start with *other*. - When the argument is True, ``..`` entries may be added to form the + When *walk_up* is false (the default), the path must start with *other*. + When the argument is true, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -715,6 +727,11 @@ Pure paths provide the following methods and properties: >>> p.with_suffix('') PureWindowsPath('README') + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. In previous + versions, :exc:`ValueError` is raised if a single dot is supplied. + .. method:: PurePath.with_segments(*pathsegments) @@ -809,33 +826,74 @@ bugs or failures in your application):: % (cls.__name__,)) UnsupportedOperation: cannot instantiate 'WindowsPath' on your system +Some concrete path methods can raise an :exc:`OSError` if a system call fails +(for example because the path doesn't exist). -Methods -^^^^^^^ -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). +Parsing and generating URIs +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionchanged:: 3.8 +Concrete path objects can be created from, and represented as, 'file' URIs +conforming to :rfc:`8089`. - :meth:`~Path.exists()`, :meth:`~Path.is_dir()`, :meth:`~Path.is_file()`, - :meth:`~Path.is_mount()`, :meth:`~Path.is_symlink()`, - :meth:`~Path.is_block_device()`, :meth:`~Path.is_char_device()`, - :meth:`~Path.is_fifo()`, :meth:`~Path.is_socket()` now return ``False`` - instead of raising an exception for paths that contain characters - unrepresentable at the OS level. +.. note:: + File URIs are not portable across machines with different + :ref:`filesystem encodings <filesystem-encoding>`. -.. classmethod:: Path.cwd() +.. classmethod:: Path.from_uri(uri) - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: + Return a new path object from parsing a 'file' URI. For example:: - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') + >>> p = Path.from_uri('file:///etc/hosts') + PosixPath('/etc/hosts') + + On Windows, DOS device and UNC paths may be parsed from URIs:: + + >>> p = Path.from_uri('file:///c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file://server/share') + WindowsPath('//server/share') + + Several variant forms are supported:: + + >>> p = Path.from_uri('file:////server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file://///server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file:c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file:/c|/windows') + WindowsPath('c:/windows') + + :exc:`ValueError` is raised if the URI does not start with ``file:``, or + the parsed path isn't absolute. + + .. versionadded:: 3.13 +.. method:: Path.as_uri() + + Represent the path as a 'file' URI. :exc:`ValueError` is raised if + the path isn't absolute. + + .. code-block:: pycon + + >>> p = PosixPath('/etc/passwd') + >>> p.as_uri() + 'file:///etc/passwd' + >>> p = WindowsPath('c:/Windows') + >>> p.as_uri() + 'file:///c:/Windows' + + For historical reasons, this method is also available from + :class:`PurePath` objects. However, its use of :func:`os.fsencode` makes + it strictly impure. + + +Expanding and resolving paths +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + .. classmethod:: Path.home() Return a new path object representing the user's home directory (as @@ -850,40 +908,110 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. classmethod:: Path.from_uri(uri) +.. method:: Path.expanduser() + + Return a new path with expanded ``~`` and ``~user`` constructs, + as returned by :meth:`os.path.expanduser`. If a home directory can't be + resolved, :exc:`RuntimeError` is raised. - Return a new path object from parsing a 'file' URI conforming to - :rfc:`8089`. For example:: + :: - >>> p = Path.from_uri('file:///etc/hosts') - PosixPath('/etc/hosts') + >>> p = PosixPath('~/films/Monty Python') + >>> p.expanduser() + PosixPath('/home/eric/films/Monty Python') - On Windows, DOS device and UNC paths may be parsed from URIs:: + .. versionadded:: 3.5 - >>> p = Path.from_uri('file:///c:/windows') - WindowsPath('c:/windows') - >>> p = Path.from_uri('file://server/share') - WindowsPath('//server/share') - Several variant forms are supported:: +.. classmethod:: Path.cwd() - >>> p = Path.from_uri('file:////server/share') - WindowsPath('//server/share') - >>> p = Path.from_uri('file://///server/share') - WindowsPath('//server/share') - >>> p = Path.from_uri('file:c:/windows') - WindowsPath('c:/windows') - >>> p = Path.from_uri('file:/c|/windows') - WindowsPath('c:/windows') + Return a new path object representing the current directory (as returned + by :func:`os.getcwd`):: - :exc:`ValueError` is raised if the URI does not start with ``file:``, or - the parsed path isn't absolute. + >>> Path.cwd() + PosixPath('/home/antoine/pathlib') - :func:`os.fsdecode` is used to decode percent-escaped byte sequences, and - so file URIs are not portable across machines with different - :ref:`filesystem encodings <filesystem-encoding>`. - .. versionadded:: 3.13 +.. method:: Path.absolute() + + Make the path absolute, without normalization or resolving symlinks. + Returns a new path object:: + + >>> p = Path('tests') + >>> p + PosixPath('tests') + >>> p.absolute() + PosixPath('/home/antoine/pathlib/tests') + + +.. method:: Path.resolve(strict=False) + + Make the path absolute, resolving any symlinks. A new path object is + returned:: + + >>> p = Path() + >>> p + PosixPath('.') + >>> p.resolve() + PosixPath('/home/antoine/pathlib') + + "``..``" components are also eliminated (this is the only method to do so):: + + >>> p = Path('docs/../setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. + + .. versionchanged:: 3.6 + The *strict* parameter was added (pre-3.6 behavior is strict). + + .. versionchanged:: 3.13 + Symlink loops are treated like other errors: :exc:`OSError` is raised in + strict mode, and no exception is raised in non-strict mode. In previous + versions, :exc:`RuntimeError` is raised no matter the value of *strict*. + + +.. method:: Path.readlink() + + Return the path to which the symbolic link points (as returned by + :func:`os.readlink`):: + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.readlink() + PosixPath('setup.py') + + .. versionadded:: 3.9 + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + +Querying file type and status +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionchanged:: 3.8 + + :meth:`~Path.exists()`, :meth:`~Path.is_dir()`, :meth:`~Path.is_file()`, + :meth:`~Path.is_mount()`, :meth:`~Path.is_symlink()`, + :meth:`~Path.is_block_device()`, :meth:`~Path.is_char_device()`, + :meth:`~Path.is_fifo()`, :meth:`~Path.is_socket()` now return ``False`` + instead of raising an exception for paths that contain characters + unrepresentable at the OS level. + +.. versionchanged:: 3.14 + + The methods given above now return ``False`` instead of raising any + :exc:`OSError` exception from the operating system. In previous versions, + some kinds of :exc:`OSError` exception are raised, and others suppressed. + The new behaviour is consistent with :func:`os.path.exists`, + :func:`os.path.isdir`, etc. Use :meth:`~Path.stat` to retrieve the file + status without suppressing exceptions. .. method:: Path.stat(*, follow_symlinks=True) @@ -905,29 +1033,18 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - :: +.. method:: Path.lstat() - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. .. method:: Path.exists(*, follow_symlinks=True) Return ``True`` if the path points to an existing file or directory. + ``False`` will be returned if the path is invalid, inaccessible or missing. + Use :meth:`Path.stat` to distinguish between these cases. This method normally follows symlinks; to check if a symlink exists, add the argument ``follow_symlinks=False``. @@ -946,186 +1063,207 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 The *follow_symlinks* parameter was added. -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`. If a home directory can't be - resolved, :exc:`RuntimeError` is raised. - - :: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - .. versionadded:: 3.5 +.. method:: Path.is_file(*, follow_symlinks=True) + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. -.. method:: Path.glob(pattern, *, case_sensitive=None, follow_symlinks=None) + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. - Glob the given relative *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" - which means "this directory and all subdirectories, recursively". In other - words, it enables recursive globbing:: +.. method:: Path.is_dir(*, follow_symlinks=True) - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. + This method normally follows symlinks; to exclude symlinks to directories, + add the argument ``follow_symlinks=False``. - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. +.. method:: Path.is_symlink() - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. +.. method:: Path.is_junction() - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + Return ``True`` if the path points to a junction, and ``False`` for any other + type of file. Currently only Windows supports junctions. - .. versionchanged:: 3.13 - Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a - future Python release, patterns with this ending will match both files - and directories. Add a trailing slash to match only directories. + .. versionadded:: 3.12 -.. method:: Path.group() - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. +.. method:: Path.is_mount() - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not - available. In previous versions, :exc:`NotImplementedError` was raised. + Return ``True`` if the path is a :dfn:`mount point`: a point in a + file system where a different file system has been mounted. On POSIX, the + function checks whether *path*'s parent, :file:`path/..`, is on a different + device than *path*, or whether :file:`path/..` and *path* point to the same + i-node on the same device --- this should detect mount points for all Unix + and POSIX variants. On Windows, a mount point is considered to be a drive + letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a + mounted filesystem directory. + .. versionadded:: 3.7 -.. method:: Path.is_dir(*, follow_symlinks=True) + .. versionchanged:: 3.12 + Windows support was added. - Return ``True`` if the path points to a directory, ``False`` if it points - to another kind of file. +.. method:: Path.is_socket() - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. - This method normally follows symlinks; to exclude symlinks to directories, - add the argument ``follow_symlinks=False``. - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. +.. method:: Path.is_fifo() + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. -.. method:: Path.is_file(*, follow_symlinks=True) - Return ``True`` if the path points to a regular file, ``False`` if it - points to another kind of file. +.. method:: Path.is_block_device() - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. - This method normally follows symlinks; to exclude symlinks, add the - argument ``follow_symlinks=False``. - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. +.. method:: Path.is_char_device() + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. -.. method:: Path.is_junction() - Return ``True`` if the path points to a junction, and ``False`` for any other - type of file. Currently only Windows supports junctions. +.. method:: Path.samefile(other_path) - .. versionadded:: 3.12 + Return whether this path points to the same file as *other_path*, which + can be either a Path object, or a string. The semantics are similar + to :func:`os.path.samefile` and :func:`os.path.samestat`. + An :exc:`OSError` can be raised if either file cannot be accessed for some + reason. -.. method:: Path.is_mount() + :: - Return ``True`` if the path is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. On Windows, a mount point is considered to be a drive - letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a - mounted filesystem directory. + >>> p = Path('spam') + >>> q = Path('eggs') + >>> p.samefile(q) + False + >>> p.samefile('spam') + True - .. versionadded:: 3.7 + .. versionadded:: 3.5 - .. versionchanged:: 3.12 - Windows support was added. +Reading and writing files +^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Path.is_symlink() - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. +.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. + Open the file pointed to by the path, like the built-in :func:`open` + function does:: + >>> p = Path('setup.py') + >>> with p.open() as f: + ... f.readline() + ... + '#!/usr/bin/env python3\n' -.. method:: Path.is_socket() - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. +.. method:: Path.read_text(encoding=None, errors=None, newline=None) - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return the decoded contents of the pointed-to file as a string:: + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' -.. method:: Path.is_fifo() + The file is opened and then closed. The optional parameters have the same + meaning as in :func:`open`. - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. + .. versionadded:: 3.5 - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + .. versionchanged:: 3.13 + The *newline* parameter was added. -.. method:: Path.is_block_device() +.. method:: Path.read_bytes() - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. + Return the binary contents of the pointed-to file as a bytes object:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + .. versionadded:: 3.5 -.. method:: Path.is_char_device() +.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) + + Open the file pointed to in text mode, write *data* to it, and close the + file:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + An existing file of the same name is overwritten. The optional parameters + have the same meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.10 + The *newline* parameter was added. - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. +.. method:: Path.write_bytes(data) + + Open the file pointed to in bytes mode, write *data* to it, and close the + file:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + An existing file of the same name is overwritten. + + .. versionadded:: 3.5 +Reading directories +^^^^^^^^^^^^^^^^^^^ + .. method:: Path.iterdir() When the path points to a directory, yield path objects of the directory @@ -1144,8 +1282,77 @@ call fails (for example because the path doesn't exist). The children are yielded in arbitrary order, and the special entries ``'.'`` and ``'..'`` are not included. If a file is removed from or added - to the directory after creating the iterator, whether a path object for - that file be included is unspecified. + to the directory after creating the iterator, it is unspecified whether + a path object for that file is included. + + If the path is not a directory or otherwise inaccessible, :exc:`OSError` is + raised. + + +.. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* in the directory represented by this path, + yielding all matching files (of any kind):: + + >>> sorted(Path('.').glob('*.py')) + [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] + >>> sorted(Path('.').glob('*/*.py')) + [PosixPath('docs/conf.py')] + >>> sorted(Path('.').glob('**/*.py')) + [PosixPath('build/lib/pathlib.py'), + PosixPath('docs/conf.py'), + PosixPath('pathlib.py'), + PosixPath('setup.py'), + PosixPath('test_pathlib.py')] + + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + + By default, or when the *recurse_symlinks* keyword-only argument is set to + ``False``, this method follows symlinks except when expanding "``**``" + wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + + .. versionchanged:: 3.13 + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. In previous versions, such exceptions are suppressed in many + cases, but not all. + + +.. method:: Path.rglob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* recursively. This is like calling + :func:`Path.glob` with "``**/``" added in front of the *pattern*. + + .. seealso:: + :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. + + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + .. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False) @@ -1242,22 +1449,27 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.12 -.. method:: Path.lchmod(mode) - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. +Creating files and directories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. method:: Path.touch(mode=0o666, exist_ok=True) -.. method:: Path.lstat() + Create a file at this given path. If *mode* is given, it is combined + with the process's ``umask`` value to determine the file mode and access + flags. If the file already exists, the function succeeds when *exist_ok* + is true (and its modification time is updated to the current time), + otherwise :exc:`FileExistsError` is raised. - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. + .. seealso:: + The :meth:`~Path.open`, :meth:`~Path.write_text` and + :meth:`~Path.write_bytes` methods are often used to create files. .. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode + combined with the process's ``umask`` value to determine the file mode and access flags. If the path already exists, :exc:`FileExistsError` is raised. @@ -1271,87 +1483,112 @@ call fails (for example because the path doesn't exist). If *exist_ok* is false (the default), :exc:`FileExistsError` is raised if the target directory already exists. - If *exist_ok* is true, :exc:`FileExistsError` exceptions will be - ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the - last path component is not an existing non-directory file. + If *exist_ok* is true, :exc:`FileExistsError` will not be raised unless the given + path already exists in the file system and is not a directory (same + behavior as the POSIX ``mkdir -p`` command). .. versionchanged:: 3.5 The *exist_ok* parameter was added. -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) +.. method:: Path.symlink_to(target, target_is_directory=False) - Open the file pointed to by the path, like the built-in :func:`open` - function does:: + Make this path a symbolic link pointing to *target*. - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' + On Windows, a symlink represents either a file or a directory, and does not + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is true or a file symlink (the + default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. + :: -.. method:: Path.owner() + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + >>> p.stat().st_size + 956 + >>> p.lstat().st_size + 8 - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.symlink`'s. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.read_bytes() +.. method:: Path.hardlink_to(target) - Return the binary contents of the pointed-to file as a bytes object:: + Make this path a hard link to the same file as *target*. - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.link`'s. - .. versionadded:: 3.5 + .. versionadded:: 3.10 + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.link` is not + available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.read_text(encoding=None, errors=None) - Return the decoded contents of the pointed-to file as a string:: +Copying, renaming and deleting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' +.. method:: Path.copy(target, *, follow_symlinks=True, preserve_metadata=False) - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. + Copy the contents of this file to the *target* file. If *target* specifies + a file that already exists, it will be replaced. - .. versionadded:: 3.5 + If *follow_symlinks* is false, and this file is a symbolic link, *target* + will be created as a symbolic link. If *follow_symlinks* is true and this + file is a symbolic link, *target* will be a copy of the symlink target. + If *preserve_metadata* is false (the default), only the file data is + guaranteed to be copied. Set *preserve_metadata* to true to ensure that the + file mode (permissions), flags, last access and modification times, and + extended attributes are copied where supported. This argument has no effect + on Windows, where metadata is always preserved when copying. -.. method:: Path.readlink() + .. versionadded:: 3.14 - Return the path to which the symbolic link points (as returned by - :func:`os.readlink`):: - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.readlink() - PosixPath('setup.py') +.. method:: Path.copytree(target, *, follow_symlinks=True, dirs_exist_ok=False, \ + ignore=None, on_error=None) - .. versionadded:: 3.9 + Recursively copy this directory tree to the given destination. - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. + If a symlink is encountered in the source tree, and *follow_symlinks* is + true (the default), the symlink's target is copied. Otherwise, the symlink + is recreated in the destination tree. + + If the destination is an existing directory and *dirs_exist_ok* is false + (the default), a :exc:`FileExistsError` is raised. Otherwise, the copying + operation will continue if it encounters existing directories, and files + within the destination tree will be overwritten by corresponding files from + the source tree. + + If *ignore* is given, it should be a callable accepting one argument: a + file or directory path within the source tree. The callable may return true + to suppress copying of the path. + + If *on_error* is given, it should be a callable accepting one argument: an + instance of :exc:`OSError`. The callable may re-raise the exception or do + nothing, in which case the copying operation continues. If *on_error* isn't + given, exceptions are propagated to the caller. + + .. versionadded:: 3.14 .. method:: Path.rename(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. On Unix, if *target* exists + and is a file, it will be replaced silently if the user has permission. On Windows, if *target* exists, :exc:`FileExistsError` will be raised. *target* can be either a string or another path object:: @@ -1365,249 +1602,236 @@ call fails (for example because the path doesn't exist). 'some text' The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. It is implemented in terms of :func:`os.rename` and gives the same guarantees. .. versionchanged:: 3.8 - Added return value, return the new Path instance. + Added return value, return the new :class:`!Path` instance. .. method:: Path.replace(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. If *target* points to an + existing file or empty directory, it will be unconditionally replaced. The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - -.. method:: Path.absolute() - - Make the path absolute, without normalization or resolving symlinks. - Returns a new path object:: - - >>> p = Path('tests') - >>> p - PosixPath('tests') - >>> p.absolute() - PosixPath('/home/antoine/pathlib/tests') - - -.. method:: Path.resolve(strict=False) - - Make the path absolute, resolving any symlinks. A new path object is - returned:: - - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') - - "``..``" components are also eliminated (this is the only method to do so):: - - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') + Added return value, return the new :class:`!Path` instance. - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. - - .. versionchanged:: 3.6 - The *strict* parameter was added (pre-3.6 behavior is strict). - - .. versionchanged:: 3.13 - Symlink loops are treated like other errors: :exc:`OSError` is raised in - strict mode, and no exception is raised in non-strict mode. In previous - versions, :exc:`RuntimeError` is raised no matter the value of *strict*. - -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* recursively. This is like calling - :func:`Path.glob` with "``**/``" added in front of the *pattern*, where - *patterns* are the same as for :mod:`fnmatch`:: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. +.. method:: Path.unlink(missing_ok=False) - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. .. method:: Path.rmdir() Remove this directory. The directory must be empty. -.. method:: Path.samefile(other_path) +Permissions and ownership +^^^^^^^^^^^^^^^^^^^^^^^^^ - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. +.. method:: Path.owner(*, follow_symlinks=True) - :: + Return the name of the user owning the file. :exc:`KeyError` is raised + if the file's user identifier (UID) isn't found in the system database. - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link to *target*. Under Windows, - *target_is_directory* must be true (default ``False``) if the link's target - is a directory. Under POSIX, *target_is_directory*'s value is ignored. - - :: - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. + This method normally follows symlinks; to get the owner of the symlink, add + the argument ``follow_symlinks=False``. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. + Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. -.. method:: Path.hardlink_to(target) - Make this path a hard link to the same file as *target*. +.. method:: Path.group(*, follow_symlinks=True) - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.link`'s. + Return the name of the group owning the file. :exc:`KeyError` is raised + if the file's group identifier (GID) isn't found in the system database. - .. versionadded:: 3.10 + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.link` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. + Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. -.. method:: Path.unlink(missing_ok=False) - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. +.. method:: Path.chmod(mode, *, follow_symlinks=True) - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. + Change the file mode and permissions, like :func:`os.chmod`. - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. + :: + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 -.. method:: Path.write_bytes(data) + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' +.. method:: Path.lchmod(mode) - An existing file of the same name is overwritten. + Like :meth:`Path.chmod` but, if the path points to a symbolic link, the + symbolic link's mode is changed rather than its target's. - .. versionadded:: 3.5 +.. _pathlib-pattern-language: -.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) +Pattern language +---------------- - Open the file pointed to in text mode, write *data* to it, and close the - file:: +The following wildcards are supported in patterns for +:meth:`~PurePath.full_match`, :meth:`~Path.glob` and :meth:`~Path.rglob`: - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' +``**`` (entire segment) + Matches any number of file or directory segments, including zero. +``*`` (entire segment) + Matches one file or directory segment. +``*`` (part of a segment) + Matches any number of non-separator characters, including zero. +``?`` + Matches one non-separator character. +``[seq]`` + Matches one character in *seq*. +``[!seq]`` + Matches one character not in *seq*. - An existing file of the same name is overwritten. The optional parameters - have the same meaning as in :func:`open`. +For a literal match, wrap the meta-characters in brackets. +For example, ``"[?]"`` matches the character ``"?"``. - .. versionadded:: 3.5 +The "``**``" wildcard enables recursive globbing. A few examples: - .. versionchanged:: 3.10 - The *newline* parameter was added. +========================= =========================================== +Pattern Meaning +========================= =========================================== +"``**/*``" Any path with at least one segment. +"``**/*.py``" Any path with a final segment ending "``.py``". +"``assets/**``" Any path starting with "``assets/``". +"``assets/**/*``" Any path starting with "``assets/``", excluding "``assets/``" itself. +========================= =========================================== -Correspondence to tools in the :mod:`os` module ------------------------------------------------ +.. note:: + Globbing with the "``**``" wildcard visits every directory in the tree. + Large directory trees may take a long time to search. + +.. versionchanged:: 3.13 + Globbing with a pattern that ends with "``**``" returns both files and + directories. In previous versions, only directories were returned. + +In :meth:`Path.glob` and :meth:`~Path.rglob`, a trailing slash may be added to +the pattern to match only directories. + +.. versionchanged:: 3.11 + Globbing with a pattern that ends with a pathname components separator + (:data:`~os.sep` or :data:`~os.altsep`) returns only directories. + + +Comparison to the :mod:`glob` module +------------------------------------ + +The patterns accepted and results generated by :meth:`Path.glob` and +:meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: + +1. Files beginning with a dot are not special in pathlib. This is + like passing ``include_hidden=True`` to :func:`glob.glob`. +2. "``**``" pattern components are always recursive in pathlib. This is like + passing ``recursive=True`` to :func:`glob.glob`. +3. "``**``" pattern components do not follow symlinks by default in pathlib. + This behaviour has no equivalent in :func:`glob.glob`, but you can pass + ``recurse_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. +4. Like all :class:`PurePath` and :class:`Path` objects, the values returned + from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing + slashes. +5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + include the *path* as a prefix, unlike the results of + ``glob.glob(root_dir=path)``. +6. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + may include *path* itself, for example when globbing "``**``", whereas the + results of ``glob.glob(root_dir=path)`` never include an empty string that + would correspond to *path*. + + +Comparison to the :mod:`os` and :mod:`os.path` modules +------------------------------------------------------ + +pathlib implements path operations using :class:`PurePath` and :class:`Path` +objects, and so it's said to be *object-oriented*. On the other hand, the +:mod:`os` and :mod:`os.path` modules supply functions that work with low-level +``str`` and ``bytes`` objects, which is a more *procedural* approach. Some +users consider the object-oriented style to be more readable. + +Many functions in :mod:`os` and :mod:`os.path` support ``bytes`` paths and +:ref:`paths relative to directory descriptors <dir_fd>`. These features aren't +available in pathlib. + +Python's ``str`` and ``bytes`` types, and portions of the :mod:`os` and +:mod:`os.path` modules, are written in C and are very speedy. pathlib is +written in pure Python and is often slower, but rarely slow enough to matter. + +pathlib's path normalization is slightly more opinionated and consistent than +:mod:`os.path`. For example, whereas :func:`os.path.abspath` eliminates +"``..``" segments from a path, which may change its meaning if symlinks are +involved, :meth:`Path.absolute` preserves these segments for greater safety. + +pathlib's path normalization may render it unsuitable for some applications: + +1. pathlib normalizes ``Path("my_folder/")`` to ``Path("my_folder")``, which + changes a path's meaning when supplied to various operating system APIs and + command-line utilities. Specifically, the absence of a trailing separator + may allow the path to be resolved as either a file or directory, rather + than a directory only. +2. pathlib normalizes ``Path("./my_program")`` to ``Path("my_program")``, + which changes a path's meaning when used as an executable search path, such + as in a shell or when spawning a child process. Specifically, the absence + of a separator in the path may force it to be looked up in :envvar:`PATH` + rather than the current directory. + +As a consequence of these differences, pathlib is not a drop-in replacement +for :mod:`os.path`. + + +Corresponding tools +^^^^^^^^^^^^^^^^^^^ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. -.. note:: - - Not all pairs of functions/methods below are equivalent. Some of them, - despite having some overlapping use-cases, have different semantics. They - include :func:`os.path.abspath` and :meth:`Path.absolute`, - :func:`os.path.relpath` and :meth:`PurePath.relative_to`. - ==================================== ============================== :mod:`os` and :mod:`os.path` :mod:`pathlib` ==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` [#]_ +:func:`os.path.abspath` :meth:`Path.absolute` :func:`os.path.realpath` :meth:`Path.resolve` :func:`os.chmod` :meth:`Path.chmod` :func:`os.mkdir` :meth:`Path.mkdir` @@ -1628,7 +1852,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :func:`os.link` :meth:`Path.hardlink_to` :func:`os.symlink` :meth:`Path.symlink_to` :func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` [#]_ +:func:`os.path.relpath` :meth:`PurePath.relative_to` :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, :meth:`Path.group` @@ -1640,8 +1864,3 @@ Below is a table mapping various :mod:`os` functions to their corresponding :func:`os.path.splitext` :attr:`PurePath.stem` and :attr:`PurePath.suffix` ==================================== ============================== - -.. rubric:: Footnotes - -.. [#] :func:`os.path.abspath` normalizes the resulting path, which may change its meaning in the presence of symlinks, while :meth:`Path.absolute` does not. -.. [#] :meth:`PurePath.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 002eeef4c09b5d..b1e9392ecfd927 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -48,7 +48,7 @@ at the location you want to break into the debugger, and then run the program. You can then step through the code following this statement, and continue running without the debugger using the :pdbcmd:`continue` command. -.. versionadded:: 3.7 +.. versionchanged:: 3.7 The built-in :func:`breakpoint()`, when called with defaults, can be used instead of ``import pdb; pdb.set_trace()``. @@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command. The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - > ...(3)double() - -> return x * 2 + > ...(2)double() + -> breakpoint() (Pdb) p x 3 (Pdb) continue @@ -86,12 +86,12 @@ after normal exit of the program), pdb will restart the program. Automatic restarting preserves pdb's state (such as breakpoints) and in most cases is more useful than quitting the debugger upon program's exit. -.. versionadded:: 3.2 - ``-c`` option is introduced to execute commands as if given - in a :file:`.pdbrc` file, see :ref:`debugger-commands`. +.. versionchanged:: 3.2 + Added the ``-c`` option to execute commands as if given + in a :file:`.pdbrc` file; see :ref:`debugger-commands`. -.. versionadded:: 3.7 - ``-m`` option is introduced to execute modules similar to the way +.. versionchanged:: 3.7 + Added the ``-m`` option to execute modules similar to the way ``python -m`` does. As with a script, the debugger will pause execution just before the first line of the module. @@ -123,6 +123,11 @@ The typical usage to inspect a crashed program is:: 0 (Pdb) +.. versionchanged:: 3.13 + The implementation of :pep:`667` means that name assignments made via ``pdb`` + will immediately affect the active scope, even when running inside an + :term:`optimized scope`. + The module defines the following functions; each enters the debugger in a slightly different way: @@ -164,6 +169,9 @@ slightly different way: .. versionchanged:: 3.7 The keyword-only argument *header*. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. .. function:: post_mortem(traceback=None) @@ -209,12 +217,12 @@ access further features, you have to do this yourself: .. audit-event:: pdb.Pdb "" pdb.Pdb - .. versionadded:: 3.1 - The *skip* argument. + .. versionchanged:: 3.1 + Added the *skip* parameter. - .. versionadded:: 3.2 - The *nosigint* argument. Previously, a SIGINT handler was never set by - Pdb. + .. versionchanged:: 3.2 + Added the *nosigint* parameter. + Previously, a SIGINT handler was never set by Pdb. .. versionchanged:: 3.6 The *readrc* argument. @@ -282,25 +290,28 @@ There are three preset *convenience variables*: .. versionadded:: 3.12 + Added the *convenience variable* feature. + .. index:: pair: .pdbrc; file triple: debugger; configuration; file If a file :file:`.pdbrc` exists in the user's home directory or in the current directory, it is read with ``'utf-8'`` encoding and executed as if it had been -typed at the debugger prompt. This is particularly useful for aliases. If both +typed at the debugger prompt, with the exception that empty lines and lines +starting with ``#`` are ignored. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. -.. versionchanged:: 3.11 - :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read - with the system locale encoding. - .. versionchanged:: 3.2 :file:`.pdbrc` can now contain commands that continue debugging, such as :pdbcmd:`continue` or :pdbcmd:`next`. Previously, these commands had no effect. +.. versionchanged:: 3.11 + :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read + with the system locale encoding. + .. pdbcommand:: h(elp) [command] @@ -310,11 +321,17 @@ can be overridden by the local file. argument must be an identifier, ``help exec`` must be entered to get help on the ``!`` command. -.. pdbcommand:: w(here) +.. pdbcommand:: w(here) [count] - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) + Print a stack trace, with the most recent frame at the bottom. if *count* + is 0, print the current frame entry. If *count* is negative, print the least + recent - *count* frames. If *count* is positive, print the most recent + *count* frames. An arrow (``>``) indicates the current frame, which determines the context of most commands. + .. versionchanged:: 3.14 + *count* argument is added. + .. pdbcommand:: d(own) [count] Move the current frame *count* (default one) levels down in the stack trace @@ -327,12 +344,16 @@ can be overridden by the local file. .. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. + With a *lineno* argument, set a break at line *lineno* in the current file. + The line number may be prefixed with a *filename* and a colon, + to specify a breakpoint in another file (possibly one that hasn't been loaded + yet). The file is searched on :data:`sys.path`. Acceptable forms of *filename* + are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and + ``package.module``. + + With a *function* argument, set a break at the first executable statement within + that function. *function* can be any expression that evaluates to a function + in the current namespace. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -341,6 +362,9 @@ can be overridden by the local file. of times that breakpoint has been hit, the current ignore count, and the associated condition if any. + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + .. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] Temporary breakpoint, which is removed automatically when it is first hit. @@ -467,8 +491,8 @@ can be overridden by the local file. raised or propagated is indicated by ``>>``, if it differs from the current line. - .. versionadded:: 3.2 - The ``>>`` marker. + .. versionchanged:: 3.2 + Added the ``>>`` marker. .. pdbcommand:: ll | longlist @@ -568,19 +592,35 @@ can be overridden by the local file. .. pdbcommand:: interact - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. + Start an interactive interpreter (using the :mod:`code` module) in a new + global namespace initialised from the local and global namespaces for the + current scope. Use ``exit()`` or ``quit()`` to exit the interpreter and + return to the debugger. + + .. note:: + + As ``interact`` creates a new dedicated namespace for code execution, + assignments to variables will not affect the original namespaces. + However, modifications to any referenced mutable objects will be reflected + in the original namespaces as usual. .. versionadded:: 3.2 + .. versionchanged:: 3.13 + ``exit()`` and ``quit()`` can be used to exit the :pdbcmd:`interact` + command. + + .. versionchanged:: 3.13 + :pdbcmd:`interact` directs its output to the debugger's + output channel rather than :data:`sys.stderr`. + .. _debugger-aliases: .. pdbcommand:: alias [name [command]] Create an alias called *name* that executes *command*. The *command* must *not* be enclosed in quotes. Replaceable parameters can be indicated by - ``%1``, ``%2``, and so on, while ``%*`` is replaced by all the parameters. + ``%1``, ``%2``, ... and ``%9``, while ``%*`` is replaced by all the parameters. If *command* is omitted, the current alias for *name* is shown. If no arguments are given, all aliases are listed. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 93387fb0b45038..57fbe5b6ece6b6 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -1,5 +1,5 @@ -:mod:`pickle` --- Python object serialization -============================================= +:mod:`!pickle` --- Python object serialization +============================================== .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. @@ -272,13 +272,13 @@ The :mod:`pickle` module defines three exceptions: .. exception:: PickleError - Common base class for the other pickling exceptions. It inherits + Common base class for the other pickling exceptions. It inherits from :exc:`Exception`. .. exception:: PicklingError Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. + It inherits from :exc:`PickleError`. Refer to :ref:`pickle-picklable` to learn what kinds of objects can be pickled. @@ -286,7 +286,7 @@ The :mod:`pickle` module defines three exceptions: .. exception:: UnpicklingError Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. + corruption or a security violation. It inherits from :exc:`PickleError`. Note that other exceptions may also be raised during unpickling, including (but not necessarily limited to) AttributeError, EOFError, ImportError, and @@ -314,16 +314,16 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. - If *buffer_callback* is None (the default), buffer views are + If *buffer_callback* is ``None`` (the default), buffer views are serialized into *file* as part of the pickle stream. - If *buffer_callback* is not None, then it can be called any number + If *buffer_callback* is not ``None``, then it can be called any number of times with a buffer view. If the callback returns a false value - (such as None), the given buffer is :ref:`out-of-band <pickle-oob>`; + (such as ``None``), the given buffer is :ref:`out-of-band <pickle-oob>`; otherwise the buffer is serialized in-band, i.e. inside the pickle stream. - It is an error if *buffer_callback* is not None and *protocol* is - None or smaller than 5. + It is an error if *buffer_callback* is not ``None`` and *protocol* is + ``None`` or smaller than 5. .. versionchanged:: 3.8 The *buffer_callback* argument was added. @@ -345,6 +345,10 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, See :ref:`pickle-persistent` for details and examples of uses. + .. versionchanged:: 3.13 + Add the default implementation of this method in the C implementation + of :class:`!Pickler`. + .. attribute:: dispatch_table A pickler object's dispatch table is a registry of *reduction @@ -352,7 +356,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, :func:`copyreg.pickle`. It is a mapping whose keys are classes and whose values are reduction functions. A reduction function takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` + conform to the same interface as a :meth:`~object.__reduce__` method. By default, a pickler object will not have a @@ -372,8 +376,8 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It - should conform to the same interface as a :meth:`__reduce__` method, and - can optionally return ``NotImplemented`` to fallback on + should conform to the same interface as a :meth:`~object.__reduce__` method, and + can optionally return :data:`NotImplemented` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. For a detailed example, see :ref:`reducer_override`. @@ -416,12 +420,12 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, instances of :class:`~datetime.datetime`, :class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2. - If *buffers* is None (the default), then all data necessary for + If *buffers* is ``None`` (the default), then all data necessary for deserialization must be contained in the pickle stream. This means - that the *buffer_callback* argument was None when a :class:`Pickler` + that the *buffer_callback* argument was ``None`` when a :class:`Pickler` was instantiated (or when :func:`dump` or :func:`dumps` was called). - If *buffers* is not None, it should be an iterable of buffer-enabled + If *buffers* is not ``None``, it should be an iterable of buffer-enabled objects that is consumed each time the pickle stream references an :ref:`out-of-band <pickle-oob>` buffer view. Such buffers have been given in order to the *buffer_callback* of a Pickler object. @@ -446,6 +450,10 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, See :ref:`pickle-persistent` for details and examples of uses. + .. versionchanged:: 3.13 + Add the default implementation of this method in the C implementation + of :class:`!Unpickler`. + .. method:: find_class(module, name) Import *module* if necessary and return the object called *name* from it, @@ -495,7 +503,7 @@ What can be pickled and unpickled? The following types can be pickled: * built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and - ``NotImplemented``); + :data:`NotImplemented`); * integers, floating-point numbers, complex numbers; @@ -508,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`__getstate__` +* instances of such classes whose the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -544,7 +552,7 @@ purpose, so you can fix bugs in a class or add methods to the class and still load objects that were created with an earlier version of the class. If you plan to have long-lived objects that will see many versions of a class, it may be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. +conversions can be made by the class's :meth:`~object.__setstate__` method. .. _pickle-inst: @@ -559,7 +567,7 @@ customize, and control how class instances are pickled and unpickled. In most cases, no additional code is needed to make instances picklable. By default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method +introspection. When a class instance is unpickled, its :meth:`~object.__init__` method is usually *not* invoked. The default behaviour first creates an uninitialized instance and then restores the saved attributes. The following code shows an implementation of this behaviour:: @@ -645,35 +653,35 @@ methods: .. note:: - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. + If :meth:`__reduce__` returns a state with value ``None`` at pickling, + the :meth:`__setstate__` method will not be called upon unpickling. Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. +the methods :meth:`~object.__getstate__` and :meth:`~object.__setstate__`. .. note:: - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the + At unpickling time, some methods like :meth:`~object.__getattr__`, + :meth:`~object.__getattribute__`, or :meth:`~object.__setattr__` may be called upon the instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__new__` to establish such an - invariant, as :meth:`__init__` is not called when unpickling an + true, the type should implement :meth:`~object.__new__` to establish such an + invariant, as :meth:`~object.__init__` is not called when unpickling an instance. .. index:: pair: copy; protocol As we shall see, pickle does not use directly the methods described above. In fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified +:meth:`~object.__reduce__` special method. The copy protocol provides a unified interface for retrieving the data necessary for pickling and copying objects. [#]_ -Although powerful, implementing :meth:`__reduce__` directly in your classes is +Although powerful, implementing :meth:`~object.__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling +interface (i.e., :meth:`~object.__getnewargs_ex__`, :meth:`~object.__getstate__` and +:meth:`~object.__setstate__`) whenever possible. We will show, however, cases where +using :meth:`!__reduce__` is the only option or leads to more efficient pickling or both. .. method:: object.__reduce__() @@ -708,8 +716,9 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is + classes as long as they have + :ref:`append and extend methods <typesseq-common>` with + the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -785,8 +794,8 @@ any other code which depends on pickling, then one can create a pickler with a private dispatch table. The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a +available as :data:`!copyreg.dispatch_table`. Therefore, one may +choose to use a modified copy of :data:`!copyreg.dispatch_table` as a private dispatch table. For example :: @@ -825,12 +834,12 @@ Handling Stateful Objects single: __setstate__() (copy protocol) Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and +The :class:`!TextReader` class below opens a text file, and returns the line number and line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object +:class:`!TextReader` instance is pickled, all attributes *except* the file object member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: +reading resumes from the last location. The :meth:`!__setstate__` and +:meth:`!__getstate__` methods are used to implement this behavior. :: class TextReader: """Print and number lines in a text file.""" @@ -895,8 +904,8 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. +arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return +:data:`NotImplemented` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then @@ -963,7 +972,7 @@ provided by pickle protocol 5 and higher. Provider API ^^^^^^^^^^^^ -The large data objects to be pickled must implement a :meth:`__reduce_ex__` +The large data objects to be pickled must implement a :meth:`~object.__reduce_ex__` method specialized for protocol 5 and higher, which returns a :class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) for any large data. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 41930f8cbe8412..e072605974f6c2 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -1,5 +1,5 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== +:mod:`!pickletools` --- Tools for pickle developers +=================================================== .. module:: pickletools :synopsis: Contains extensive comments about the pickle protocols and @@ -94,8 +94,8 @@ Programmatic Interface a short description. The value of *annotate* is used as a hint for the column where annotation should start. - .. versionadded:: 3.2 - The *annotate* argument. + .. versionchanged:: 3.2 + Added the *annotate* parameter. .. function:: genops(pickle) diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 891a867d1ceb68..5d4ff34ba029a0 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -1,5 +1,5 @@ -:mod:`pkgutil` --- Package extension utility -============================================ +:mod:`!pkgutil` --- Package extension utility +============================================= .. module:: pkgutil :synopsis: Utilities for the import system. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index ec2a7ebd5d6e0b..f082393ef9363c 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -1,5 +1,5 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== +:mod:`!platform` --- Access to underlying platform's identifying data +====================================================================== .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. @@ -148,6 +148,9 @@ Cross Platform Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, ``'Windows'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this returns the user-facing OS name (i.e, ``'iOS``, + ``'iPadOS'`` or ``'Android'``). To obtain the kernel name (``'Darwin'`` or + ``'Linux'``), use :func:`os.uname()`. .. function:: system_alias(system, release, version) @@ -161,6 +164,8 @@ Cross Platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS version. To obtain the + Darwin or Linux kernel version, use :func:`os.uname()`. .. function:: uname() @@ -196,6 +201,10 @@ Java Platform ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to the defaults given as parameters (which all default to ``''``). + .. deprecated-removed:: 3.13 3.15 + It was largely untested, had a confusing API, + and was only useful for Jython support. + Windows Platform ---------------- @@ -210,8 +219,8 @@ Windows Platform default to an empty string). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines - and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers - to the OS version being free of debugging code. It could also state *'Checked'* + and ``'Multiprocessor Free'`` on multi processor machines. The ``'Free'`` refers + to the OS version being free of debugging code. It could also state ``'Checked'`` which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. @@ -234,7 +243,6 @@ Windows Platform macOS Platform -------------- - .. function:: mac_ver(release='', versioninfo=('','',''), machine='') Get macOS version information and return it as tuple ``(release, versioninfo, @@ -244,6 +252,24 @@ macOS Platform Entries which cannot be determined are set to ``''``. All tuple entries are strings. +iOS Platform +------------ + +.. function:: ios_ver(system='', release='', model='', is_simulator=False) + + Get iOS version information and return it as a + :func:`~collections.namedtuple` with the following attributes: + + * ``system`` is the OS name; either ``'iOS'`` or ``'iPadOS'``. + * ``release`` is the iOS version number as a string (e.g., ``'17.2'``). + * ``model`` is the device model identifier; this will be a string like + ``'iPhone13,2'`` for a physical device, or ``'iPhone'`` on a simulator. + * ``is_simulator`` is a boolean describing if the app is running on a + simulator or a physical device. + + Entries which cannot be determined are set to the defaults given as + parameters. + Unix Platforms -------------- @@ -297,3 +323,39 @@ Linux Platforms return ids .. versionadded:: 3.10 + + +Android Platform +---------------- + +.. function:: android_ver(release="", api_level=0, manufacturer="", \ + model="", device="", is_emulator=False) + + Get Android device information. Returns a :func:`~collections.namedtuple` + with the following attributes. Values which cannot be determined are set to + the defaults given as parameters. + + * ``release`` - Android version, as a string (e.g. ``"14"``). + + * ``api_level`` - API level of the running device, as an integer (e.g. ``34`` + for Android 14). To get the API level which Python was built against, see + :func:`sys.getandroidapilevel`. + + * ``manufacturer`` - `Manufacturer name + <https://developer.android.com/reference/android/os/Build#MANUFACTURER>`__. + + * ``model`` - `Model name + <https://developer.android.com/reference/android/os/Build#MODEL>`__ – + typically the marketing name or model number. + + * ``device`` - `Device name + <https://developer.android.com/reference/android/os/Build#DEVICE>`__ – + typically the model number or a codename. + + * ``is_emulator`` - ``True`` if the device is an emulator; ``False`` if it's + a physical device. + + Google maintains a `list of known model and device names + <https://storage.googleapis.com/play_public/supported_devices.html>`__. + + .. versionadded:: 3.13 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 732ef3536863cc..78b3c2697bd696 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -1,5 +1,5 @@ -:mod:`plistlib` --- Generate and parse Apple ``.plist`` files -============================================================= +:mod:`!plistlib` --- Generate and parse Apple ``.plist`` files +============================================================== .. module:: plistlib :synopsis: Generate and parse Apple plist files. @@ -27,7 +27,7 @@ top level object is a dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. -To work with plist data in bytes objects, use :func:`dumps` +To work with plist data in bytes or string objects, use :func:`dumps` and :func:`loads`. Values can be strings, integers, floats, booleans, tuples, lists, dictionaries @@ -52,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, *, fmt=None, dict_type=dict) +.. function:: load(fp, *, fmt=None, dict_type=dict, aware_datetime=False) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -69,6 +69,10 @@ This module defines the following functions: The *dict_type* is the type used for dictionaries that are read from the plist file. + When *aware_datetime* is true, fields with type ``datetime.datetime`` will + be created as :ref:`aware object <datetime-naive-aware>`, with + :attr:`!tzinfo` as :attr:`datetime.UTC`. + XML data for the :data:`FMT_XML` format is parsed using the Expat parser from :mod:`xml.parsers.expat` -- see its documentation for possible exceptions on ill-formed XML. Unknown elements will simply be ignored @@ -79,16 +83,21 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: loads(data, *, fmt=None, dict_type=dict) +.. function:: loads(data, *, fmt=None, dict_type=dict, aware_datetime=False) - Load a plist from a bytes object. See :func:`load` for an explanation of - the keyword arguments. + Load a plist from a bytes or string object. See :func:`load` for an + explanation of the keyword arguments. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + *data* can be a string when *fmt* equals :data:`FMT_XML`. -.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Write *value* to a plist file. *Fp* should be a writable, binary file object. @@ -107,6 +116,10 @@ This module defines the following functions: When *skipkeys* is false (the default) the function raises :exc:`TypeError` when a key of a dictionary is not a string, otherwise such keys are skipped. + When *aware_datetime* is true and any field with type ``datetime.datetime`` + is set as a :ref:`aware object <datetime-naive-aware>`, it will convert to + UTC timezone before writing it. + A :exc:`TypeError` will be raised if the object is of an unsupported type or a container that contains objects of unsupported types. @@ -115,8 +128,11 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Return *value* as a plist-formatted bytes object. See the documentation for :func:`dump` for an explanation of the keyword diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 943eb21f6eec02..23f20b00e6dc6d 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -1,5 +1,5 @@ -:mod:`poplib` --- POP3 protocol client -====================================== +:mod:`!poplib` --- POP3 protocol client +======================================= .. module:: poplib :synopsis: POP3 protocol client (requires sockets). diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 0413f9d02a8d57..14ab3e91e8a8e4 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -1,5 +1,5 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== +:mod:`!posix` --- The most common POSIX system calls +==================================================== .. module:: posix :platform: Unix @@ -11,6 +11,8 @@ This module provides access to operating system functionality that is standardized by the C Standard and the POSIX standard (a thinly disguised Unix interface). +.. availability:: Unix. + .. index:: pair: module; os **Do not import this module directly.** Instead, import the module :mod:`os`, diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index e883acd67d6c72..1b3498e51f766d 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -1,5 +1,5 @@ -:mod:`pprint` --- Data pretty printer -===================================== +:mod:`!pprint` --- Data pretty printer +====================================== .. module:: pprint :synopsis: Data pretty printer. @@ -19,9 +19,8 @@ such as files, sockets or classes are included, as well as many other objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. +breaks them onto multiple lines if they don't fit within the allowed width, +adjustable by the *width* parameter defaulting to 80 characters. Dictionaries are sorted by key before the display is computed. @@ -31,86 +30,82 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.10 Added support for pretty-printing :class:`dataclasses.dataclass`. -The :mod:`pprint` module defines one class: +.. _pprint-functions: + +Functions +--------- + +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=False, underscore_numbers=False) + + Prints the formatted representation of *object*, followed by a newline. + This function may be used in the interactive interpreter + instead of the :func:`print` function for inspecting values. + Tip: you can reassign ``print = pprint.pp`` for use within a scope. + + :param object: + The object to be printed. + + :param stream: + A file-like object to which the output will be written + by calling its :meth:`!write` method. + If ``None`` (the default), :data:`sys.stdout` is used. + :type stream: :term:`file-like object` | None + + :param int indent: + The amount of indentation added for each nesting level. + + :param int width: + The desired maximum number of characters per line in the output. + If a structure cannot be formatted within the width constraint, + a best effort will be made. + + :param depth: + The number of nesting levels which may be printed. + If the data structure being printed is too deep, + the next contained level is replaced by ``...``. + If ``None`` (the default), there is no constraint + on the depth of the objects being formatted. + :type depth: int | None + + :param bool compact: + Control the way long :term:`sequences <sequence>` are formatted. + If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + + :param bool sort_dicts: + If ``True``, dictionaries will be formatted with + their keys sorted, otherwise + they will be displayed in insertion order (the default). + + :param bool underscore_numbers: + If ``True``, + integers will be formatted with the ``_`` character for a thousands separator, + otherwise underscores are not displayed (the default). -.. First the implementation class: - - -.. index:: single: ...; placeholder - -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default ``sys.stdout``) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. - - Other values configure the manner in which nesting of complex data - structures is displayed. - - *indent* (default 1) specifies the amount of indentation added for - each nesting level. - - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pp(stuff) + [<Recursion on list with id=...>, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). + .. versionadded:: 3.8 - .. versionchanged:: 3.4 - Added the *compact* parameter. - .. versionchanged:: 3.8 - Added the *sort_dicts* parameter. +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) - .. versionchanged:: 3.10 - Added the *underscore_numbers* parameter. + Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, + which would automatically sort the dictionaries' keys, + you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. - .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) .. function:: pformat(object, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) @@ -118,44 +113,8 @@ The :mod:`pprint` module defines one class: Return the formatted representation of *object* as a string. *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - + and their meanings are as described in the documentation above. -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [<Recursion on list with id=...>, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] .. function:: isreadable(object) @@ -176,8 +135,6 @@ The :mod:`pprint` module defines one class: :exc:`RecursionError` if it fails to detect a recursive object. -One more support function is also defined: - .. function:: saferepr(object) Return a string representation of *object*, protected against recursion in @@ -190,12 +147,58 @@ One more support function is also defined: >>> pprint.saferepr(stuff) "[<Recursion on list with id=...>, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - .. _prettyprinter-objects: PrettyPrinter Objects --------------------- +.. index:: single: ...; placeholder + +.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Construct a :class:`PrettyPrinter` instance. + + Arguments have the same meaning as for :func:`~pprint.pp`. + Note that they are in a different order, and that *sort_dicts* defaults to ``True``. + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff[:]) + >>> pp = pprint.PrettyPrinter(indent=4) + >>> pp.pprint(stuff) + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] + >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', + ... ('parrot', ('fresh fruit',)))))))) + >>> pp = pprint.PrettyPrinter(depth=6) + >>> pp.pprint(tup) + ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) + + + .. versionchanged:: 3.4 + Added the *compact* parameter. + + .. versionchanged:: 3.8 + Added the *sort_dicts* parameter. + + .. versionchanged:: 3.10 + Added the *underscore_numbers* parameter. + + .. versionchanged:: 3.11 + No longer attempts to write to :data:`!sys.stdout` if it is ``None``. + + :class:`PrettyPrinter` instances have the following methods: @@ -258,7 +261,7 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pp` function and its parameters, let's fetch information about a project from `PyPI <https://pypi.org>`_:: >>> import json @@ -267,9 +270,9 @@ let's fetch information about a project from `PyPI <https://pypi.org>`_:: >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`pprint` shows the whole object:: +In its basic form, :func:`~pprint.pp` shows the whole object:: - >>> pprint.pprint(project_info) + >>> pprint.pp(project_info) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -326,7 +329,7 @@ In its basic form, :func:`pprint` shows the whole object:: The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: - >>> pprint.pprint(project_info, depth=1) + >>> pprint.pp(project_info, depth=1) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -372,7 +375,7 @@ contents):: Additionally, maximum character *width* can be suggested. If a long object cannot be split, the specified width will be exceeded:: - >>> pprint.pprint(project_info, depth=1, width=60) + >>> pprint.pp(project_info, depth=1, width=60) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 4c60a1e0d781b0..d7940b3040bbdb 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -82,8 +82,8 @@ the following:: The first line indicates that 214 calls were monitored. Of those calls, 207 were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time``, indicates that the text string in the -far right column was used to sort the output. The column headings include: +next line: ``Ordered by: cumulative time`` indicates the output is sorted +by the ``cumtime`` values. The column headings include: ncalls for the number of calls. @@ -234,7 +234,7 @@ functions: .. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine + globals and locals mappings for the *command* string. This routine executes:: exec(command, globals, locals) @@ -299,6 +299,13 @@ functions: Create a :class:`~pstats.Stats` object based on the current profile and print the results to stdout. + The *sort* parameter specifies the sorting order of the displayed + statistics. It accepts a single key or a tuple of keys to enable + multi-level sorting, as in :func:`Stats.sort_stats <pstats.Stats.sort_stats>`. + + .. versionadded:: 3.13 + :meth:`~Profile.print_stats` now accepts a tuple of keys. + .. method:: dump_stats(filename) Write the results of the current profile to *filename*. @@ -692,7 +699,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, As the :class:`cProfile.Profile` class cannot be calibrated, custom timer functions should be used with care and should be as fast as possible. For the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. + in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index ad4981c97119fa..1a44bb13a841de 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -1,5 +1,5 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== +:mod:`!pty` --- Pseudo-terminal utilities +========================================= .. module:: pty :platform: Unix @@ -16,6 +16,8 @@ The :mod:`pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. +.. availability:: Unix. + Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). @@ -31,6 +33,9 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + .. warning:: On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. function:: openpty() diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index 300419301b9ff5..e1ff32912132f7 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -1,5 +1,5 @@ -:mod:`pwd` --- The password database -==================================== +:mod:`!pwd` --- The password database +===================================== .. module:: pwd :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix user account and password database. It is available on all Unix versions. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not WASI, not iOS. Password database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``passwd`` structure (Attribute field below, diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 38c416f9ad0305..75aa739d1003b8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -1,5 +1,5 @@ -:mod:`py_compile` --- Compile Python source files -================================================= +:mod:`!py_compile` --- Compile Python source files +================================================== .. module:: py_compile :synopsis: Generate byte-code files from Python source files. @@ -96,7 +96,7 @@ byte-code cache files in the directory containing the source code. .. class:: PycInvalidationMode - A enumeration of possible methods the interpreter can use to determine + An enumeration of possible methods the interpreter can use to determine whether a bytecode file is up to date with a source file. The ``.pyc`` file indicates the desired invalidation mode in its header. See :ref:`pyc-invalidation` for more information on how Python invalidates diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 1c40ba4838ca75..5efb11d89dd143 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -1,5 +1,5 @@ -:mod:`pyclbr` --- Python module browser support -=============================================== +:mod:`!pyclbr` --- Python module browser support +================================================ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. @@ -58,106 +58,115 @@ of these classes. Function Objects ---------------- -Class :class:`Function` instances describe functions defined by def -statements. They have the following attributes: +.. class:: Function -.. attribute:: Function.file + Class :class:`!Function` instances describe functions defined by def + statements. They have the following attributes: - Name of the file in which the function is defined. + .. attribute:: file -.. attribute:: Function.module + Name of the file in which the function is defined. - The name of the module defining the function described. + .. attribute:: module -.. attribute:: Function.name + The name of the module defining the function described. - The name of the function. + .. attribute:: name -.. attribute:: Function.lineno + The name of the function. - The line number in the file where the definition starts. + .. attribute:: lineno -.. attribute:: Function.parent + The line number in the file where the definition starts. - For top-level functions, None. For nested functions, the parent. - .. versionadded:: 3.7 + .. attribute:: parent + For top-level functions, ``None``. For nested functions, the parent. -.. attribute:: Function.children + .. versionadded:: 3.7 - A dictionary mapping names to descriptors for nested functions and - classes. - .. versionadded:: 3.7 + .. attribute:: children + A :class:`dictionary <dict>` mapping names to descriptors for nested functions and + classes. -.. attribute:: Function.is_async + .. versionadded:: 3.7 - ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. - .. versionadded:: 3.10 + .. attribute:: is_async + + ``True`` for functions that are defined with the + :keyword:`async <async def>` prefix, ``False`` otherwise. + + .. versionadded:: 3.10 .. _pyclbr-class-objects: Class Objects ------------- -Class :class:`Class` instances describe classes defined by class -statements. They have the same attributes as Functions and two more. + +.. class:: Class + + Class :class:`!Class` instances describe classes defined by class + statements. They have the same attributes as :class:`Functions <Function>` + and two more. -.. attribute:: Class.file + .. attribute:: file - Name of the file in which the class is defined. + Name of the file in which the class is defined. -.. attribute:: Class.module + .. attribute:: module - The name of the module defining the class described. + The name of the module defining the class described. -.. attribute:: Class.name + .. attribute:: name - The name of the class. + The name of the class. -.. attribute:: Class.lineno + .. attribute:: lineno - The line number in the file where the definition starts. + The line number in the file where the definition starts. -.. attribute:: Class.parent + .. attribute:: parent - For top-level classes, None. For nested classes, the parent. + For top-level classes, ``None``. For nested classes, the parent. - .. versionadded:: 3.7 + .. versionadded:: 3.7 -.. attribute:: Class.children + .. attribute:: children - A dictionary mapping names to descriptors for nested functions and - classes. + A dictionary mapping names to descriptors for nested functions and + classes. - .. versionadded:: 3.7 + .. versionadded:: 3.7 -.. attribute:: Class.super + .. attribute:: super - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule_ex` - are listed as a string with the class name instead of as - :class:`Class` objects. + A list of :class:`!Class` objects which describe the immediate base + classes of the class being described. Classes which are named as + superclasses but which are not discoverable by :func:`readmodule_ex` + are listed as a string with the class name instead of as + :class:`!Class` objects. -.. attribute:: Class.methods + .. attribute:: methods - A dictionary mapping method names to line numbers. This can be - derived from the newer children dictionary, but remains for - back-compatibility. + A :class:`dictionary <dict>` mapping method names to line numbers. + This can be derived from the newer :attr:`children` dictionary, + but remains for + back-compatibility. diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 03e0915bf6d135..f7ca1e045699eb 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -1,5 +1,5 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== +:mod:`!pydoc` --- Documentation generator and online help system +================================================================ .. module:: pydoc :synopsis: Documentation generator and online help system. @@ -16,19 +16,19 @@ -------------- -The :mod:`pydoc` module automatically generates documentation from Python +The :mod:`!pydoc` module automatically generates documentation from Python modules. The documentation can be presented as pages of text on the console, served to a web browser, or saved to HTML files. For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, +derived from the docstring (i.e. the :attr:`!__doc__` attribute) of the object, and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just +:mod:`!pydoc` tries to obtain a description from the block of comment lines just above the definition of the class, function or method in the source file, or at the top of the module (see :func:`inspect.getcomments`). The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation +interactive interpreter, which uses :mod:`!pydoc` to generate its documentation as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: @@ -46,7 +46,7 @@ produced for that file. .. note:: - In order to find objects and their documentation, :mod:`pydoc` imports the + In order to find objects and their documentation, :mod:`!pydoc` imports the module(s) to be documented. Therefore, any code on module level will be executed on that occasion. Use an ``if __name__ == '__main__':`` guard to only execute code when a file is invoked as a script and not just imported. @@ -90,7 +90,7 @@ Python interpreter and typed ``import spam``. Module docs for core modules are assumed to reside in ``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable +be overridden by setting the :envvar:`!PYTHONDOCS` environment variable to a different URL or to a local directory containing the Library Reference Manual pages. @@ -101,7 +101,7 @@ Reference Manual pages. The ``-g`` command line option was removed. .. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than + :mod:`!pydoc` now uses :func:`inspect.signature` rather than :func:`inspect.getfullargspec` to extract signature information from callables. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 935e872480efda..c0e9999f4b1270 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -1,5 +1,5 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= +:mod:`!xml.parsers.expat` --- Fast XML parsing using Expat +========================================================== .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. @@ -196,6 +196,42 @@ XMLParser Objects :exc:`ExpatError` to be raised with the :attr:`code` attribute set to ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. +.. method:: xmlparser.SetReparseDeferralEnabled(enabled) + + .. warning:: + + Calling ``SetReparseDeferralEnabled(False)`` has security implications, + as detailed below; please make sure to understand these consequences + prior to using the ``SetReparseDeferralEnabled`` method. + + Expat 2.6.0 introduced a security mechanism called "reparse deferral" + where instead of causing denial of service through quadratic runtime + from reparsing large tokens, reparsing of unfinished tokens is now delayed + by default until a sufficient amount of input is reached. + Due to this delay, registered handlers may — depending of the sizing of + input chunks pushed to Expat — no longer be called right after pushing new + input to the parser. Where immediate feedback and taking over responsibility + of protecting against denial of service from large tokens are both wanted, + calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral + for the current Expat parser instance, temporarily or altogether. + Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse + deferral. + + Note that :meth:`SetReparseDeferralEnabled` has been backported to some + prior releases of CPython as a security fix. Check for availability of + :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code + running across a variety of Python versions. + + .. versionadded:: 3.13 + +.. method:: xmlparser.GetReparseDeferralEnabled() + + Returns whether reparse deferral is currently enabled for the given + Expat parser instance. + + .. versionadded:: 3.13 + + :class:`xmlparser` objects have the following attributes: @@ -214,7 +250,8 @@ XMLParser Objects :meth:`CharacterDataHandler` callback whenever possible. This can improve performance substantially since Expat normally breaks character data into chunks at every line ending. This attribute is false by default, and may be changed at - any time. + any time. Note that when it is false, data that does not contain newlines + may be chunked too. .. attribute:: xmlparser.buffer_used @@ -372,7 +409,10 @@ otherwise stated. marked content, and ignorable whitespace. Applications which must distinguish these cases can use the :attr:`StartCdataSectionHandler`, :attr:`EndCdataSectionHandler`, and :attr:`ElementDeclHandler` callbacks to - collect the required information. + collect the required information. Note that the character data may be + chunked even if it is short and so you may receive more than one call to + :meth:`CharacterDataHandler`. Set the :attr:`buffer_text` instance attribute + to ``True`` to avoid that. .. method:: xmlparser.UnparsedEntityDeclHandler(entityName, base, systemId, publicId, notationName) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index b2b787c5a8260c..fbbebcf4ed8f92 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -1,5 +1,5 @@ -:mod:`queue` --- A synchronized queue class -=========================================== +:mod:`!queue` --- A synchronized queue class +============================================ .. module:: queue :synopsis: A synchronized queue class. @@ -93,6 +93,14 @@ The :mod:`queue` module defines the following classes and exceptions: on a :class:`Queue` object which is full. +.. exception:: ShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is called on + a :class:`Queue` object which has been shut down. + + .. versionadded:: 3.13 + + .. _queueobjects: Queue Objects @@ -135,6 +143,8 @@ provide the public methods described below. immediately available, else raise the :exc:`Full` exception (*timeout* is ignored in that case). + Raises :exc:`ShutDown` if the queue has been shut down. + .. method:: Queue.put_nowait(item) @@ -155,6 +165,9 @@ provide the public methods described below. an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + Raises :exc:`ShutDown` if the queue has been shut down and is empty, or if + the queue has been shut down immediately. + .. method:: Queue.get_nowait() @@ -174,6 +187,9 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item + in the queue. + Raises a :exc:`ValueError` if called more times than there were items placed in the queue. @@ -214,6 +230,29 @@ Example of how to wait for enqueued tasks to be completed:: print('All work completed') +Terminating queues +^^^^^^^^^^^^^^^^^^ + +:class:`Queue` objects can be made to prevent further interaction by shutting +them down. + +.. method:: Queue.shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise + :exc:`ShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only raise once the + queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise + immediately instead. + + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be + unblocked. If *immediate* is true, a task will be marked as done for each + remaining item in the queue, which may unblock callers of + :meth:`~Queue.join`. + + .. versionadded:: 3.13 + + SimpleQueue Objects ------------------- diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index 86717c00c3c136..977cb08d836afe 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -1,5 +1,5 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== +:mod:`!quopri` --- Encode and decode MIME quoted-printable data +=============================================================== .. module:: quopri :synopsis: Encode and decode files using the MIME quoted-printable encoding. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 76ae97a8be7e63..755d1c8908c966 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -1,5 +1,5 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ +:mod:`!random` --- Generate pseudo-random numbers +================================================= .. module:: random :synopsis: Generate pseudo-random numbers with various common distributions. @@ -34,10 +34,8 @@ instance of the :class:`random.Random` class. You can instantiate your own instances of :class:`Random` to get generators that don't share state. Class :class:`Random` can also be subclassed if you want to use a different -basic generator of your own devising: in that case, override the :meth:`~Random.random`, -:meth:`~Random.seed`, :meth:`~Random.getstate`, and :meth:`~Random.setstate` methods. -Optionally, a new generator can supply a :meth:`~Random.getrandbits` method --- this -allows :meth:`randrange` to produce selections over an arbitrarily large range. +basic generator of your own devising: see the documentation on that class for +more details. The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers @@ -57,10 +55,16 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - <https://code.activestate.com/recipes/576707/>`_ for a compatible alternative + <https://code.activestate.com/recipes/576707-long-period-random-number-generator/>`_ for a compatible alternative random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions --------------------- @@ -88,7 +92,7 @@ Bookkeeping functions .. versionchanged:: 3.11 The *seed* must be one of the following types: - *NoneType*, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. .. function:: getstate() @@ -220,8 +224,8 @@ Functions for sequences generated. For example, a sequence of length 2080 is the largest that can fit within the period of the Mersenne Twister random number generator. - .. deprecated-removed:: 3.9 3.11 - The optional parameter *random*. + .. versionchanged:: 3.11 + Removed the optional parameter *random*. .. function:: sample(population, k, *, counts=None) @@ -303,7 +307,8 @@ be found in any statistics text. ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. + depending on floating-point rounding in the expression + ``a + (b-a) * random()``. .. function:: triangular(low, high, mode) @@ -407,11 +412,42 @@ Alternative Generator Class that implements the default pseudo-random number generator used by the :mod:`random` module. - .. deprecated-removed:: 3.9 3.11 + .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: - :class:`NoneType`, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. + Subclasses of :class:`!Random` should override the following methods if they + wish to make use of a different basic generator: + + .. method:: Random.seed(a=None, version=2) + + Override this method in subclasses to customise the :meth:`~random.seed` + behaviour of :class:`!Random` instances. + + .. method:: Random.getstate() + + Override this method in subclasses to customise the :meth:`~random.getstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.setstate(state) + + Override this method in subclasses to customise the :meth:`~random.setstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.random() + + Override this method in subclasses to customise the :meth:`~random.random` + behaviour of :class:`!Random` instances. + + Optionally, a custom generator subclass can also supply the following method: + + .. method:: Random.getrandbits(k) + + Override this method in subclasses to customise the + :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + + .. class:: SystemRandom([seed]) Class that uses the :func:`os.urandom` function for generating random numbers @@ -445,30 +481,30 @@ Examples Basic examples:: - >>> random() # Random float: 0.0 <= x < 1.0 + >>> random() # Random float: 0.0 <= x < 1.0 0.37444887175646646 - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 + >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 3.1800146073117523 - >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds + >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds 5.148957571865031 - >>> randrange(10) # Integer from 0 to 9 inclusive + >>> randrange(10) # Integer from 0 to 9 inclusive 7 - >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive + >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive 26 - >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence + >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence 'draw' >>> deck = 'ace two three four'.split() - >>> shuffle(deck) # Shuffle a list + >>> shuffle(deck) # Shuffle a list >>> deck ['four', 'two', 'ace', 'three'] - >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement + >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement [40, 10, 50, 30] Simulations:: @@ -572,14 +608,14 @@ Simulation of arrival times and service deliveries for a multiserver queue:: including simulation, sampling, shuffling, and cross-validation. `Economics Simulation - <https://nbviewer.jupyter.org/url/norvig.com/ipython/Economics.ipynb>`_ + <https://nbviewer.org/url/norvig.com/ipython/Economics.ipynb>`_ a simulation of a marketplace by `Peter Norvig <https://norvig.com/bio.html>`_ that shows effective use of many of the tools and distributions provided by this module (gauss, uniform, sample, betavariate, choice, triangular, and randrange). `A Concrete Introduction to Probability (using Python) - <https://nbviewer.jupyter.org/url/norvig.com/ipython/Probability.ipynb>`_ + <https://nbviewer.org/url/norvig.com/ipython/Probability.ipynb>`_ a tutorial by `Peter Norvig <https://norvig.com/bio.html>`_ covering the basics of probability theory, how to write simulations, and how to perform data analysis using Python. @@ -670,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.) <https://allendowney.com/research/rand/downey07randfloat.pdf>`_ a paper by Allen B. Downey describing ways to generate more fine-grained floats than normally generated by :func:`.random`. + +.. _random-cli: + +Command-line usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`!random` module can be executed from the command line. + +.. code-block:: sh + + python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...] + +The following options are accepted: + +.. program:: random + +.. option:: -h, --help + + Show the help message and exit. + +.. option:: -c CHOICE [CHOICE ...] + --choice CHOICE [CHOICE ...] + + Print a random choice, using :meth:`choice`. + +.. option:: -i <N> + --integer <N> + + Print a random integer between 1 and N inclusive, using :meth:`randint`. + +.. option:: -f <N> + --float <N> + + Print a random floating point number between 1 and N inclusive, + using :meth:`uniform`. + +If no options are given, the output depends on the input: + +* String or multiple: same as :option:`--choice`. +* Integer: same as :option:`--integer`. +* Float: same as :option:`--float`. + +.. _random-cli-example: + +Command-line example +-------------------- + +Here are some examples of the :mod:`!random` command-line interface: + +.. code-block:: console + + $ # Choose one at random + $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + Lobster Thermidor aux crevettes with a Mornay sauce + + $ # Random integer + $ python -m random 6 + 6 + + $ # Random floating-point number + $ python -m random 1.8 + 1.7080016272295635 + + $ # With explicit arguments + $ python -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + egg + + $ python -m random --integer 6 + 3 + + $ python -m random --float 1.8 + 1.5666339105010318 + + $ python -m random --integer 6 + 5 + + $ python -m random --float 6 + 3.1942323316565915 diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 251ec8ca0021a6..cc979fe66f7fe9 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1,5 +1,5 @@ -:mod:`re` --- Regular expression operations -=========================================== +:mod:`!re` --- Regular expression operations +============================================ .. module:: re :synopsis: Regular expression operations. @@ -17,7 +17,7 @@ those found in Perl. Both patterns and strings to be searched can be Unicode strings (:class:`str`) as well as 8-bit strings (:class:`bytes`). However, Unicode strings and 8-bit strings cannot be mixed: -that is, you cannot match a Unicode string with a byte pattern or +that is, you cannot match a Unicode string with a bytes pattern or vice-versa; similarly, when asking for a substitution, the replacement string must be of the same type as both the pattern and the search string. @@ -48,7 +48,7 @@ fine-tuning parameters. .. seealso:: - The third-party `regex <https://pypi.org/project/regex/>`_ module, + The third-party :pypi:`regex` module, which has an API compatible with the standard library :mod:`re` module, but offers additional functionality and a more thorough Unicode support. @@ -101,7 +101,7 @@ The special characters are: ``.`` (Dot.) In the default mode, this matches any character except a newline. If the :const:`DOTALL` flag has been specified, this matches any character - including a newline. + including a newline. ``(?s:.)`` matches any character regardless of flags. .. index:: single: ^ (caret); in regular expressions @@ -257,8 +257,7 @@ The special characters are: .. index:: single: \ (backslash); in regular expressions * Character classes such as ``\w`` or ``\S`` (defined below) are also accepted - inside a set, although the characters they match depends on whether - :const:`ASCII` or :const:`LOCALE` mode is in force. + inside a set, although the characters they match depend on the flags_ used. .. index:: single: ^ (caret); in regular expressions @@ -326,18 +325,24 @@ The special characters are: currently supported extensions. ``(?aiLmsux)`` - (One or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``.) The group matches the empty string; the - letters set the corresponding flags: :const:`re.A` (ASCII-only matching), - :const:`re.I` (ignore case), :const:`re.L` (locale dependent), - :const:`re.M` (multi-line), :const:`re.S` (dot matches all), - :const:`re.U` (Unicode matching), and :const:`re.X` (verbose), - for the entire regular expression. + (One or more letters from the set + ``'a'``, ``'i'``, ``'L'``, ``'m'``, ``'s'``, ``'u'``, ``'x'``.) + The group matches the empty string; + the letters set the corresponding flags for the entire regular expression: + + * :const:`re.A` (ASCII-only matching) + * :const:`re.I` (ignore case) + * :const:`re.L` (locale dependent) + * :const:`re.M` (multi-line) + * :const:`re.S` (dot matches all) + * :const:`re.U` (Unicode matching) + * :const:`re.X` (verbose) + (The flags are described in :ref:`contents-of-module-re`.) This is useful if you wish to include the flags as part of the regular expression, instead of passing a *flag* argument to the - :func:`re.compile` function. Flags should be used first in the - expression string. + :func:`re.compile` function. + Flags should be used first in the expression string. .. versionchanged:: 3.11 This construction can only be used at the start of the expression. @@ -351,14 +356,20 @@ The special characters are: pattern. ``(?aiLmsux-imsx:...)`` - (Zero or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``, optionally followed by ``'-'`` followed by + (Zero or more letters from the set + ``'a'``, ``'i'``, ``'L'``, ``'m'``, ``'s'``, ``'u'``, ``'x'``, + optionally followed by ``'-'`` followed by one or more letters from the ``'i'``, ``'m'``, ``'s'``, ``'x'``.) - The letters set or remove the corresponding flags: - :const:`re.A` (ASCII-only matching), :const:`re.I` (ignore case), - :const:`re.L` (locale dependent), :const:`re.M` (multi-line), - :const:`re.S` (dot matches all), :const:`re.U` (Unicode matching), - and :const:`re.X` (verbose), for the part of the expression. + The letters set or remove the corresponding flags for the part of the expression: + + * :const:`re.A` (ASCII-only matching) + * :const:`re.I` (ignore case) + * :const:`re.L` (locale dependent) + * :const:`re.M` (multi-line) + * :const:`re.S` (dot matches all) + * :const:`re.U` (Unicode matching) + * :const:`re.X` (verbose) + (The flags are described in :ref:`contents-of-module-re`.) The letters ``'a'``, ``'L'`` and ``'u'`` are mutually exclusive when used @@ -366,7 +377,7 @@ The special characters are: when one of them appears in an inline group, it overrides the matching mode in the enclosing group. In Unicode patterns ``(?a:...)`` switches to ASCII-only matching, and ``(?u:...)`` switches to Unicode matching - (default). In byte pattern ``(?L:...)`` switches to locale depending + (default). In bytes patterns ``(?L:...)`` switches to locale dependent matching, and ``(?a:...)`` switches to ASCII-only matching (default). This override is only in effect for the narrow inline group, and the original matching mode is restored outside of the group. @@ -529,47 +540,61 @@ character ``'$'``. ``\b`` Matches the empty string, but only at the beginning or end of a word. - A word is defined as a sequence of word characters. Note that formally, - ``\b`` is defined as the boundary between a ``\w`` and a ``\W`` character - (or vice versa), or between ``\w`` and the beginning/end of the string. - This means that ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, - ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - - By default Unicode alphanumerics are the ones used in Unicode patterns, but - this can be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - Inside a character range, ``\b`` represents the backspace character, for - compatibility with Python's string literals. + A word is defined as a sequence of word characters. + Note that formally, ``\b`` is defined as the boundary + between a ``\w`` and a ``\W`` character (or vice versa), + or between ``\w`` and the beginning or end of the string. + This means that ``r'\bat\b'`` matches ``'at'``, ``'at.'``, ``'(at)'``, + and ``'as at ay'`` but not ``'attempt'`` or ``'atlas'``. + + The default word characters in Unicode (str) patterns + are Unicode alphanumerics and the underscore, + but this can be changed by using the :py:const:`~re.ASCII` flag. + Word boundaries are determined by the current locale + if the :py:const:`~re.LOCALE` flag is used. + + .. note:: + + Inside a character range, ``\b`` represents the backspace character, + for compatibility with Python's string literals. .. index:: single: \B; in regular expressions ``\B`` - Matches the empty string, but only when it is *not* at the beginning or end - of a word. This means that ``r'py\B'`` matches ``'python'``, ``'py3'``, - ``'py2'``, but not ``'py'``, ``'py.'``, or ``'py!'``. - ``\B`` is just the opposite of ``\b``, so word characters in Unicode - patterns are Unicode alphanumerics or the underscore, although this can - be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. + Matches the empty string, + but only when it is *not* at the beginning or end of a word. + This means that ``r'at\B'`` matches ``'athens'``, ``'atom'``, + ``'attorney'``, but not ``'at'``, ``'at.'``, or ``'at!'``. + ``\B`` is the opposite of ``\b``, + so word characters in Unicode (str) patterns + are Unicode alphanumerics or the underscore, + although this can be changed by using the :py:const:`~re.ASCII` flag. + Word boundaries are determined by the current locale + if the :py:const:`~re.LOCALE` flag is used. .. index:: single: \d; in regular expressions ``\d`` For Unicode (str) patterns: - Matches any Unicode decimal digit (that is, any character in - Unicode character category [Nd]). This includes ``[0-9]``, and - also many other digit characters. If the :const:`ASCII` flag is - used only ``[0-9]`` is matched. + Matches any Unicode decimal digit + (that is, any character in Unicode character category `[Nd]`__). + This includes ``[0-9]``, and also many other digit characters. + + Matches ``[0-9]`` if the :py:const:`~re.ASCII` flag is used. + + __ https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G134153 For 8-bit (bytes) patterns: - Matches any decimal digit; this is equivalent to ``[0-9]``. + Matches any decimal digit in the ASCII character set; + this is equivalent to ``[0-9]``. .. index:: single: \D; in regular expressions ``\D`` - Matches any character which is not a decimal digit. This is - the opposite of ``\d``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^0-9]``. + Matches any character which is not a decimal digit. + This is the opposite of ``\d``. + + Matches ``[^0-9]`` if the :py:const:`~re.ASCII` flag is used. .. index:: single: \s; in regular expressions @@ -578,8 +603,9 @@ character ``'$'``. Matches Unicode whitespace characters (which includes ``[ \t\n\r\f\v]``, and also many other characters, for example the non-breaking spaces mandated by typography rules in many - languages). If the :const:`ASCII` flag is used, only - ``[ \t\n\r\f\v]`` is matched. + languages). + + Matches ``[ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. For 8-bit (bytes) patterns: Matches characters considered whitespace in the ASCII character set; @@ -589,30 +615,39 @@ character ``'$'``. ``\S`` Matches any character which is not a whitespace character. This is - the opposite of ``\s``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^ \t\n\r\f\v]``. + the opposite of ``\s``. + + Matches ``[^ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. .. index:: single: \w; in regular expressions ``\w`` For Unicode (str) patterns: - Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) + Matches Unicode word characters; + this includes all Unicode alphanumeric characters + (as defined by :py:meth:`str.isalnum`), as well as the underscore (``_``). - If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. + + Matches ``[a-zA-Z0-9_]`` if the :py:const:`~re.ASCII` flag is used. For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; - this is equivalent to ``[a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. + this is equivalent to ``[a-zA-Z0-9_]``. + If the :py:const:`~re.LOCALE` flag is used, + matches characters considered alphanumeric in the current locale and the underscore. .. index:: single: \W; in regular expressions ``\W`` - Matches any character which is not a word character. This is - the opposite of ``\w``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters which are neither alphanumeric in the current locale + Matches any character which is not a word character. + This is the opposite of ``\w``. + By default, matches non-underscore (``_``) characters + for which :py:meth:`str.isalnum` returns ``False``. + + Matches ``[^a-zA-Z0-9_]`` if the :py:const:`~re.ASCII` flag is used. + + If the :py:const:`~re.LOCALE` flag is used, + matches characters which are neither alphanumeric in the current locale nor the underscore. .. index:: single: \Z; in regular expressions @@ -644,9 +679,11 @@ string literals are also accepted by the regular expression parser:: (Note that ``\b`` is used to represent word boundaries, and means "backspace" only inside character classes.) -``'\u'``, ``'\U'``, and ``'\N'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are errors. Unknown escapes of ASCII -letters are reserved for future use and treated as errors. +``'\u'``, ``'\U'``, and ``'\N'`` escape sequences are +only recognized in Unicode (str) patterns. +In bytes patterns they are errors. +Unknown escapes of ASCII letters are reserved +for future use and treated as errors. Octal escapes are included in a limited form. If the first digit is a 0, or if there are three octal digits, it is considered an octal escape. Otherwise, it is @@ -694,30 +731,37 @@ Flags Make ``\w``, ``\W``, ``\b``, ``\B``, ``\d``, ``\D``, ``\s`` and ``\S`` perform ASCII-only matching instead of full Unicode matching. This is only - meaningful for Unicode patterns, and is ignored for byte patterns. + meaningful for Unicode (str) patterns, and is ignored for bytes patterns. + Corresponds to the inline flag ``(?a)``. - Note that for backward compatibility, the :const:`re.U` flag still - exists (as well as its synonym :const:`re.UNICODE` and its embedded - counterpart ``(?u)``), but these are redundant in Python 3 since - matches are Unicode by default for strings (and Unicode matching - isn't allowed for bytes). + .. note:: + + The :py:const:`~re.U` flag still exists for backward compatibility, + but is redundant in Python 3 since + matches are Unicode by default for ``str`` patterns, + and Unicode matching isn't allowed for bytes patterns. + :py:const:`~re.UNICODE` and the inline flag ``(?u)`` are similarly redundant. .. data:: DEBUG Display debug information about compiled expression. + No corresponding inline flag. .. data:: I IGNORECASE - Perform case-insensitive matching; expressions like ``[A-Z]`` will also - match lowercase letters. Full Unicode matching (such as ``Ü`` matching - ``ü``) also works unless the :const:`re.ASCII` flag is used to disable - non-ASCII matches. The current locale does not change the effect of this - flag unless the :const:`re.LOCALE` flag is also used. + Perform case-insensitive matching; + expressions like ``[A-Z]`` will also match lowercase letters. + Full Unicode matching (such as ``Ü`` matching ``ü``) + also works unless the :py:const:`~re.ASCII` flag + is used to disable non-ASCII matches. + The current locale does not change the effect of this flag + unless the :py:const:`~re.LOCALE` flag is also used. + Corresponds to the inline flag ``(?i)``. Note that when the Unicode patterns ``[a-z]`` or ``[A-Z]`` are used in @@ -725,29 +769,35 @@ Flags letters and 4 additional non-ASCII letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131, Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and 'K' (U+212A, Kelvin sign). - If the :const:`ASCII` flag is used, only letters 'a' to 'z' + If the :py:const:`~re.ASCII` flag is used, only letters 'a' to 'z' and 'A' to 'Z' are matched. .. data:: L LOCALE Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching - dependent on the current locale. This flag can be used only with bytes - patterns. The use of this flag is discouraged as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. + dependent on the current locale. + This flag can be used only with bytes patterns. + Corresponds to the inline flag ``(?L)``. + .. warning:: + + This flag is discouraged; consider Unicode matching instead. + The locale mechanism is very unreliable + as it only handles one "culture" at a time + and only works with 8-bit locales. + Unicode matching is enabled by default for Unicode (str) patterns + and it is able to handle different locales and languages. + .. versionchanged:: 3.6 - :const:`re.LOCALE` can be used only with bytes patterns and is - not compatible with :const:`re.ASCII`. + :py:const:`~re.LOCALE` can be used only with bytes patterns + and is not compatible with :py:const:`~re.ASCII`. .. versionchanged:: 3.7 - Compiled regular expression objects with the :const:`re.LOCALE` flag no - longer depend on the locale at compile time. Only the locale at - matching time affects the result of matching. + Compiled regular expression objects with the :py:const:`~re.LOCALE` flag + no longer depend on the locale at compile time. + Only the locale at matching time affects the result of matching. .. data:: M @@ -759,6 +809,7 @@ Flags end of each line (immediately preceding each newline). By default, ``'^'`` matches only at the beginning of the string, and ``'$'`` only at the end of the string and immediately before the newline (if any) at the end of the string. + Corresponds to the inline flag ``(?m)``. .. data:: NOFLAG @@ -778,19 +829,19 @@ Flags Make the ``'.'`` special character match any character at all, including a newline; without this flag, ``'.'`` will match anything *except* a newline. + Corresponds to the inline flag ``(?s)``. .. data:: U UNICODE - In Python 2, this flag made :ref:`special sequences <re-special-sequences>` - include Unicode characters in matches. Since Python 3, Unicode characters - are matched by default. - - See :const:`A` for restricting matching on ASCII characters instead. + In Python 3, Unicode characters are matched by default + for ``str`` patterns. + This flag is therefore redundant with **no effect** + and is only kept for backward compatibility. - This flag is only kept for backward compatibility. + See :py:const:`~re.ASCII` to restrict matching to ASCII characters instead. .. data:: X VERBOSE @@ -829,8 +880,8 @@ Functions below. The expression's behaviour can be modified by specifying a *flags* value. - Values can be any of the following variables, combined using bitwise OR (the - ``|`` operator). + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). The sequence :: @@ -860,6 +911,10 @@ Functions ``None`` if no position in the string matches the pattern; note that this is different from finding a zero-length match at some point in the string. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: match(pattern, string, flags=0) @@ -874,6 +929,10 @@ Functions If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: fullmatch(pattern, string, flags=0) @@ -881,6 +940,10 @@ Functions corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionadded:: 3.4 @@ -914,6 +977,8 @@ Functions Empty matches for the pattern split the string only when not adjacent to a previous empty match. + .. code:: pycon + >>> re.split(r'\b', 'Words, words, words.') ['', 'Words', ', ', 'words', ', ', 'words', '.'] >>> re.split(r'\W*', '...words...') @@ -921,6 +986,10 @@ Functions >>> re.split(r'(\W*)', '...words...') ['', '...', '', '', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', '', '', ''] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -951,6 +1020,10 @@ Functions >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') [('width', '20'), ('height', '10')] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -962,6 +1035,10 @@ Functions is scanned left-to-right, and matches are returned in the order found. Empty matches are included in the result. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1017,6 +1094,10 @@ Functions character ``'0'``. The backreference ``\g<0>`` substitutes in the entire substring matched by the RE. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -1049,6 +1130,10 @@ Functions Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: escape(pattern) @@ -1093,12 +1178,12 @@ Functions Exceptions ^^^^^^^^^^ -.. exception:: error(msg, pattern=None, pos=None) +.. exception:: PatternError(msg, pattern=None, pos=None) Exception raised when a string passed to one of the functions here is not a valid regular expression (for example, it might contain unmatched parentheses) or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has + error if a string contains no match for a pattern. The ``PatternError`` instance has the following additional attributes: .. attribute:: msg @@ -1124,6 +1209,10 @@ Exceptions .. versionchanged:: 3.5 Added additional attributes. + .. versionchanged:: 3.13 + ``PatternError`` was originally named ``error``; the latter is kept as an alias for + backward compatibility. + .. _re-objects: Regular Expression Objects @@ -1233,7 +1322,7 @@ Regular Expression Objects The regex matching flags. This is a combination of the flags given to :func:`.compile`, any ``(?...)`` inline flags in the pattern, and implicit - flags such as :data:`UNICODE` if the pattern is a Unicode string. + flags such as :py:const:`~re.UNICODE` if the pattern is a Unicode string. .. attribute:: Pattern.groups @@ -1287,7 +1376,8 @@ when there is no match, you can test whether there was a match with a simple Escapes such as ``\n`` are converted to the appropriate characters, and numeric backreferences (``\1``, ``\2``) and named backreferences (``\g<1>``, ``\g<name>``) are replaced by the contents of the - corresponding group. + corresponding group. The backreference ``\g<0>`` will be + replaced by the entire match. .. versionchanged:: 3.5 Unmatched groups are replaced with an empty string. @@ -1540,7 +1630,7 @@ To find out what card the pair consists of, one could use the Simulating scanf() ^^^^^^^^^^^^^^^^^^ -.. index:: single: scanf() +.. index:: single: scanf (C function) Python does not currently have an equivalent to :c:func:`!scanf`. Regular expressions are generally more powerful, though also more verbose, than diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 8fb0eca8df74d8..5658b93c81dc99 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -1,11 +1,11 @@ -:mod:`readline` --- GNU readline interface -========================================== +:mod:`!readline` --- GNU readline interface +=========================================== .. module:: readline :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> -------------- @@ -24,19 +24,20 @@ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. +.. include:: ../includes/wasm-ios-notavail.rst + .. note:: The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. + the ``editline`` (``libedit``) library instead of GNU readline. On macOS the :mod:`readline` module detects which library is being used at run time. - The configuration file for ``libedit`` is different from that + The configuration file for ``editline`` is different from that of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. + you can use :data:`backend` to determine which library is being used. - If you use *editline*/``libedit`` readline emulation on macOS, the + If you use ``editline``/``libedit`` readline emulation on macOS, the initialization file located in your home directory is named ``.editrc``. For example, the following content in ``~/.editrc`` will turn ON *vi* keybindings and TAB completion:: @@ -44,6 +45,12 @@ Readline library in general. python:bind -v python:bind ^I rl_complete +.. data:: backend + + The name of the underlying Readline library being used, either + ``"readline"`` or ``"editline"``. + + .. versionadded:: 3.13 Init file --------- @@ -213,6 +220,8 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. _readline-completion: + Completion ---------- diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 5ebb0a7780c37b..28c7855dfeeef3 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -1,5 +1,5 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== +:mod:`!reprlib` --- Alternate :func:`repr` implementation +========================================================= .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. @@ -10,7 +10,7 @@ -------------- -The :mod:`reprlib` module provides a means for producing object representations +The :mod:`!reprlib` module provides a means for producing object representations with limits on the size of the resulting strings. This is used in the Python debugger and may be useful in other contexts as well. @@ -58,29 +58,31 @@ This module provides a class, an instance, and a function: limits on most sizes. In addition to size-limiting tools, the module also provides a decorator for -detecting recursive calls to :meth:`__repr__` and substituting a placeholder -string instead. +detecting recursive calls to :meth:`~object.__repr__` and substituting a +placeholder string instead. .. index:: single: ...; placeholder .. decorator:: recursive_repr(fillvalue="...") - Decorator for :meth:`__repr__` methods to detect recursive calls within the + Decorator for :meth:`~object.__repr__` methods to detect recursive calls within the same thread. If a recursive call is made, the *fillvalue* is returned, - otherwise, the usual :meth:`__repr__` call is made. For example: - - >>> from reprlib import recursive_repr - >>> class MyList(list): - ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' - ... - >>> m = MyList('abc') - >>> m.append(m) - >>> m.append('x') - >>> print(m) - <'a'|'b'|'c'|...|'x'> + otherwise, the usual :meth:`!__repr__` call is made. For example: + + .. doctest:: + + >>> from reprlib import recursive_repr + >>> class MyList(list): + ... @recursive_repr() + ... def __repr__(self): + ... return '<' + '|'.join(map(repr, self)) + '>' + ... + >>> m = MyList('abc') + >>> m.append(m) + >>> m.append('x') + >>> print(m) + <'a'|'b'|'c'|...|'x'> .. versionadded:: 3.2 @@ -148,10 +150,10 @@ which format specific object types. with no line breaks or indentation, like the standard :func:`repr`. For example: - .. code-block:: pycon + .. doctest:: indent >>> example = [ - 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] + ... 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] >>> import reprlib >>> aRepr = reprlib.Repr() >>> print(aRepr.repr(example)) @@ -160,7 +162,7 @@ which format specific object types. If :attr:`~Repr.indent` is set to a string, each recursion level is placed on its own line, indented by that string: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = '-->' >>> print(aRepr.repr(example)) @@ -181,7 +183,7 @@ which format specific object types. Setting :attr:`~Repr.indent` to a positive integer value behaves as if it was set to a string with that number of spaces: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = 4 >>> print(aRepr.repr(example)) @@ -234,7 +236,9 @@ Subclassing Repr Objects The use of dynamic dispatching by :meth:`Repr.repr1` allows subclasses of :class:`Repr` to add support for additional built-in object types or to modify the handling of types already supported. This example shows how special support -for file objects could be added:: +for file objects could be added: + +.. testcode:: import reprlib import sys @@ -248,3 +252,7 @@ for file objects could be added:: aRepr = MyRepr() print(aRepr.repr(sys.stdin)) # prints '<stdin>' + +.. testoutput:: + + <stdin> diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index a5324c82c63484..dd80b1e6670d92 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -1,5 +1,5 @@ -:mod:`resource` --- Resource usage information -============================================== +:mod:`!resource` --- Resource usage information +=============================================== .. module:: resource :platform: Unix @@ -13,7 +13,7 @@ This module provides basic mechanisms for measuring and controlling system resources utilized by a program. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not WASI. Symbolic constants are used to specify particular system resources and to request usage information about either the current process or its children. @@ -177,6 +177,8 @@ platform. The largest area of mapped memory which the process may occupy. + .. availability:: FreeBSD >= 11. + .. data:: RLIMIT_AS @@ -277,7 +279,7 @@ These functions are used to retrieve resource usage information: This function returns an object that describes the resources consumed by either the current process or its children, as specified by the *who* parameter. The - *who* parameter should be specified using one of the :const:`RUSAGE_\*` + *who* parameter should be specified using one of the :const:`!RUSAGE_\*` constants described below. A simple example:: @@ -353,7 +355,7 @@ These functions are used to retrieve resource usage information: Returns the number of bytes in a system page. (This need not be the same as the hardware page size.) -The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` +The following :const:`!RUSAGE_\*` symbols are passed to the :func:`getrusage` function to specify which processes information should be provided for. diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 40b09ce897880e..91779feb525013 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -1,5 +1,5 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== +:mod:`!rlcompleter` --- Completion function for GNU readline +============================================================ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. @@ -10,12 +10,14 @@ -------------- -The :mod:`rlcompleter` module defines a completion function suitable for the -:mod:`readline` module by completing valid Python identifiers and keywords. +The :mod:`!rlcompleter` module defines a completion function suitable to be +passed to :func:`~readline.set_completer` in the :mod:`readline` module. When this module is imported on a Unix platform with the :mod:`readline` module available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. +and its :meth:`~Completer.complete` method is set as the +:ref:`readline completer <readline-completion>`. The method provides +completion of valid Python :ref:`identifiers and keywords <identifiers>`. Example:: @@ -28,7 +30,7 @@ Example:: readline.__name__ readline.parse_and_bind( >>> readline. -The :mod:`rlcompleter` module is designed for use with Python's +The :mod:`!rlcompleter` module is designed for use with Python's :ref:`interactive mode <tut-interactive>`. Unless Python is run with the :option:`-S` option, the module is automatically imported and configured (see :ref:`rlcompleter-config`). @@ -39,23 +41,25 @@ this module can still be used for custom purposes. .. _completer-objects: -Completer Objects ------------------ +.. class:: Completer -Completer objects have the following method: + Completer objects have the following method: + .. method:: Completer.complete(text, state) -.. method:: Completer.complete(text, state) + Return the next possible completion for *text*. - Return the *state*\ th completion for *text*. + When called by the :mod:`readline` module, this method is called + successively with ``state == 0, 1, 2, ...`` until the method returns + ``None``. - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. + If called for *text* that doesn't include a period character (``'.'``), it will + complete from names currently defined in :mod:`__main__`, :mod:`builtins` and + keywords (as defined by the :mod:`keyword` module). + If called for a dotted name, it will try to evaluate anything without obvious + side-effects (functions will not be evaluated, but it can generate calls to + :meth:`~object.__getattr__`) up to the last part, and find matches for the + rest via the :func:`dir` function. Any exception raised during the + evaluation of the expression is caught, silenced and :const:`None` is + returned. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 406b080b7be30f..b07ec6e93f80ab 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -1,5 +1,5 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== +:mod:`!runpy` --- Locating and executing Python modules +======================================================= .. module:: runpy :synopsis: Locate and run Python modules without importing them first. @@ -32,7 +32,7 @@ The :mod:`runpy` module provides two functions: .. index:: pair: module; __main__ - Execute the code of the specified module and return the resulting module + Execute the code of the specified module and return the resulting module's globals dictionary. The module's code is first located using the standard import mechanism (refer to :pep:`302` for details) and then executed in a fresh module namespace. @@ -44,16 +44,16 @@ The :mod:`runpy` module provides two functions: returned. The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are + the module's globals dictionary before the code is executed. + *init_globals* will not be modified. If any of the special global variables + below are defined in *init_globals*, those definitions are overridden by :func:`run_module`. The special global variables ``__name__``, ``__spec__``, ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a + dictionary before the module code is executed. (Note that this is a minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + interpreter implementation detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a @@ -61,7 +61,7 @@ The :mod:`runpy` module provides two functions: ``__spec__`` will be set appropriately for the *actually* imported module (that is, ``__spec__.name`` will always be *mod_name* or - ``mod_name + '.__main__``, never *run_name*). + ``mod_name + '.__main__'``, never *run_name*). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are :ref:`set as normal <import-mod-attrs>` based on the module spec. @@ -104,11 +104,11 @@ The :mod:`runpy` module provides two functions: pair: module; __main__ Execute the code at the named filesystem location and return the resulting - module globals dictionary. As with a script name supplied to the CPython - command line, the supplied path may refer to a Python source file, a + module's globals dictionary. As with a script name supplied to the CPython + command line, *file_path* may refer to a Python source file, a compiled bytecode file or a valid :data:`sys.path` entry containing a :mod:`__main__` module - (e.g. a zipfile containing a top-level ``__main__.py`` file). + (e.g. a zipfile containing a top-level :file:`__main__.py` file). For a simple script, the specified code is simply executed in a fresh module namespace. For a valid :data:`sys.path` entry (typically a zipfile or @@ -119,26 +119,26 @@ The :mod:`runpy` module provides two functions: there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are + the module's globals dictionary before the code is executed. + *init_globals* will not be modified. If any of the special global variables + below are defined in *init_globals*, those definitions are overridden by :func:`run_path`. The special global variables ``__name__``, ``__spec__``, ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a + dictionary before the module code is executed. (Note that this is a minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + interpreter implementation detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``'<run_path>'`` otherwise. - If the supplied path directly references a script file (whether as source - or as precompiled byte code), then ``__file__`` will be set to the - supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and + If *file_path* directly references a script file (whether as source + or as precompiled byte code), then ``__file__`` will be set to + *file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid :data:`sys.path` entry, then + If *file_path* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be @@ -146,7 +146,7 @@ The :mod:`runpy` module provides two functions: A number of alterations are also made to the :mod:`sys` module. Firstly, :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of ``path_name`` and ``sys.modules[__name__]`` is updated + with the value of *file_path* and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function returns. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 01bac5afd0b9b3..517dbe8c321898 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -1,5 +1,5 @@ -:mod:`sched` --- Event scheduler -================================ +:mod:`!sched` --- Event scheduler +================================= .. module:: sched :synopsis: General purpose event scheduler. @@ -36,7 +36,7 @@ scheduler: Example:: >>> import sched, time - >>> s = sched.scheduler(time.monotonic, time.sleep) + >>> s = sched.scheduler(time.time, time.sleep) >>> def print_time(a='default'): ... print("From print_time", time.time(), a) ... diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 4405dfc0535973..1401a925103517 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -1,5 +1,5 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== +:mod:`!secrets` --- Generate secure random numbers for managing secrets +======================================================================= .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. @@ -42,13 +42,13 @@ randomness that your operating system provides. sources provided by the operating system. See :class:`random.SystemRandom` for additional details. -.. function:: choice(sequence) +.. function:: choice(seq) Return a randomly chosen element from a non-empty sequence. -.. function:: randbelow(n) +.. function:: randbelow(exclusive_upper_bound) - Return a random int in the range [0, *n*). + Return a random int in the range [0, *exclusive_upper_bound*). .. function:: randbits(k) @@ -155,7 +155,7 @@ Generate an eight-character alphanumeric password: .. note:: Applications should not - `store passwords in a recoverable format <https://cwe.mitre.org/data/definitions/257.html>`_, + :cwe:`store passwords in a recoverable format <257>`, whether plain text or encrypted. They should be salted and hashed using a cryptographically strong one-way (irreversible) hash function. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index c2941e628d9d78..06ebaf0201e0e7 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -1,5 +1,5 @@ -:mod:`select` --- Waiting for I/O completion -============================================ +:mod:`!select` --- Waiting for I/O completion +============================================= .. module:: select :synopsis: Wait for I/O completion on multiple streams. @@ -185,8 +185,8 @@ The module defines the following: ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is -O(highest file descriptor) and :c:func:`!poll` is O(number of file -descriptors), ``/dev/poll`` is O(active file descriptors). +*O*\ (*highest file descriptor*) and :c:func:`!poll` is *O*\ (*number of file +descriptors*), ``/dev/poll`` is *O*\ (*active file descriptors*). ``/dev/poll`` behaviour is very close to the standard :c:func:`!poll` object. @@ -381,8 +381,8 @@ scalability for network servers that service many, many clients at the same time. :c:func:`!poll` scales better because the system call only requires listing the file descriptors of interest, while :c:func:`!select` builds a bitmap, turns on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`!select` is O(highest file descriptor), while -:c:func:`!poll` is O(number of file descriptors). +linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), while +:c:func:`!poll` is *O*\ (*number of file descriptors*). .. method:: poll.register(fd[, eventmask]) diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index dd50bac37e49b8..de8c3ef0ea2275 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -1,5 +1,5 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ +:mod:`!selectors` --- High-level I/O multiplexing +================================================= .. module:: selectors :synopsis: High-level I/O multiplexing. @@ -21,7 +21,7 @@ It defines a :class:`BaseSelector` abstract base class, along with several concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), that can be used to wait for I/O readiness notification on multiple file objects. In the following, "file object" refers to any object with a -:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. +:meth:`~io.IOBase.fileno` method, or a raw file descriptor. See :term:`file object`. :class:`DefaultSelector` is an alias to the most efficient implementation available on the current platform: this should be the default choice for most diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 219219af6fd87f..6e74a59b82b8ec 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -1,5 +1,5 @@ -:mod:`shelve` --- Python object persistence -=========================================== +:mod:`!shelve` --- Python object persistence +============================================ .. module:: shelve :synopsis: Python object persistence. @@ -86,7 +86,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe <https://code.activestate.com/recipes/576642/>`_ + `Persistent dictionary recipe <https://code.activestate.com/recipes/576642-persistent-dict-with-multiple-standard-file-format/>`_ with widely supported storage formats and having the speed of native dictionaries. @@ -113,6 +113,9 @@ Restrictions differs across Unix versions and requires knowledge about the database implementation used. +* On macOS :mod:`dbm.ndbm` can silently corrupt the database file on updates, + which can cause hard crashes when trying to read from the database. + .. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') @@ -149,13 +152,14 @@ Restrictions .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - A subclass of :class:`Shelf` which exposes :meth:`first`, :meth:`!next`, - :meth:`previous`, :meth:`last` and :meth:`set_location` which are available - in the third-party :mod:`bsddb` module from `pybsddb + A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, + :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. + These are available + in the third-party :mod:`!bsddb` module from `pybsddb <https://www.jcea.es/programacion/pybsddb.htm>`_ but not in other database modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of - :func:`bsddb.hashopen`, :func:`bsddb.btopen` or :func:`bsddb.rnopen`. The + :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, and *keyencoding* parameters have the same interpretation as for the :class:`Shelf` class. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index f94833ad5331a9..a96f0864dc1260 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -1,5 +1,5 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== +:mod:`!shlex` --- Simple lexical analysis +========================================= .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. @@ -412,17 +412,17 @@ otherwise. To illustrate, you can see the difference in the following snippet: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> import shlex - >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> s = shlex.shlex(text, posix=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] - >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', - '(', 'def', 'ghi', ')'] + >>> import shlex + >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -431,10 +431,10 @@ Instead of passing ``True`` as the value for the punctuation_chars parameter, you can pass a string with specific characters, which will be used to determine which characters constitute punctuation. For example:: - >>> import shlex - >>> s = shlex.shlex("a && b || c", punctuation_chars="|") - >>> list(s) - ['a', '&', '&', 'b', '||', 'c'] + >>> import shlex + >>> s = shlex.shlex("a && b || c", punctuation_chars="|") + >>> list(s) + ['a', '&', '&', 'b', '||', 'c'] .. note:: When ``punctuation_chars`` is specified, the :attr:`~shlex.wordchars` attribute is augmented with the characters ``~-./*?=``. That is because these diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d1949d698f5614..fd32479195eca8 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -1,5 +1,5 @@ -:mod:`shutil` --- High-level file operations -============================================ +:mod:`!shutil` --- High-level file operations +============================================= .. module:: shutil :synopsis: High-level file operations, including copying. @@ -39,7 +39,7 @@ Directory and files operations .. function:: copyfileobj(fsrc, fdst[, length]) - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. + Copy the contents of the :term:`file-like object <file object>` *fsrc* to the file-like object *fdst*. The integer *length*, if given, is the buffer size. In particular, a negative *length* value means to copy the data without looping over the source data in chunks; by default the data is read in chunks to avoid uncontrolled memory @@ -52,7 +52,7 @@ Directory and files operations Copy the contents (no metadata) of the file named *src* to a file named *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path-like objects or path names given as strings. + *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* @@ -94,7 +94,7 @@ Directory and files operations .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path names + group are unaffected. *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, :func:`copymode` will attempt to modify the mode of *dst* itself (rather @@ -113,7 +113,7 @@ Directory and files operations Copy the permission bits, last access time, last modification time, and flags from *src* to *dst*. On Linux, :func:`copystat` also copies the "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path + group are unaffected. *src* and *dst* are :term:`path-like objects <path-like object>` or path names given as strings. If *follow_symlinks* is false, and *src* and *dst* both @@ -242,7 +242,7 @@ Directory and files operations be copied as far as the platform allows; if false or omitted, the contents and metadata of the linked files are copied to the new tree. - When *symlinks* is false, if the file pointed by the symlink doesn't + When *symlinks* is false, if the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you @@ -274,23 +274,23 @@ Directory and files operations .. audit-event:: shutil.copytree src,dst shutil.copytree - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - .. versionchanged:: 3.2 Added the *copy_function* argument to be able to provide a custom copy function. Added the *ignore_dangling_symlinks* argument to silence dangling symlinks errors when *symlinks* is false. + .. versionchanged:: 3.3 + Copy metadata when *symlinks* is false. + Now returns *dst*. + .. versionchanged:: 3.8 Platform-specific fast-copy syscalls may be used internally in order to copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. - .. versionadded:: 3.8 - The *dirs_exist_ok* parameter. + .. versionchanged:: 3.8 + Added the *dirs_exist_ok* parameter. .. function:: rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None) @@ -338,11 +338,17 @@ Directory and files operations before removing the junction. .. versionchanged:: 3.11 - The *dir_fd* parameter. + Added the *dir_fd* parameter. .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. + .. versionchanged:: 3.13 + :func:`!rmtree` now ignores :exc:`FileNotFoundError` exceptions for all + but the top-level path. + Exceptions other than :exc:`OSError` and subclasses of :exc:`!OSError` + are now always propagated to the caller. + .. attribute:: rmtree.avoids_symlink_attacks Indicates whether the current platform and implementation provides a @@ -354,21 +360,24 @@ Directory and files operations .. function:: move(src, dst, copy_function=copy2) - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. + Recursively move a file or directory (*src*) to another location and return + the destination. - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. + If *dst* is an existing directory or a symlink to a directory, then *src* + is moved inside that directory. The destination path in that directory must + not already exist. + + If *dst* already exists but is not a directory, it may be overwritten + depending on :func:`os.rename` semantics. If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. + used. Otherwise, *src* is copied to the destination using *copy_function* + and then removed. In case of symlinks, a new symlink pointing to the target + of *src* will be created as the destination and *src* will be removed. - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dst* if - :func:`os.rename` cannot be used. If the source is a directory, + If *copy_function* is given, it must be a callable that takes two arguments, + *src* and the destination, and will be used to copy *src* to the destination + if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the *copy_function*. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also @@ -412,7 +421,8 @@ Directory and files operations .. availability:: Unix, Windows. -.. function:: chown(path, user=None, group=None) +.. function:: chown(path, user=None, group=None, *, dir_fd=None, \ + follow_symlinks=True) Change owner *user* and/or *group* of the given *path*. @@ -427,6 +437,9 @@ Directory and files operations .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Added *dir_fd* and *follow_symlinks* parameters. + .. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) @@ -434,10 +447,11 @@ Directory and files operations called. If no *cmd* would be called, return ``None``. *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. + determining if the file exists and is executable. - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the lookup directory list. When no + *path* is specified, the results of :func:`os.environ` are used, returning + either the "PATH" value or a fallback of :data:`os.defpath`. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -580,7 +594,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of + any format-specific extension. + + *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is available), or "xztar" (if the :mod:`lzma` module is available). diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 7ee5ece8859825..48c6841c648ca4 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -1,5 +1,5 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== +:mod:`!signal` --- Set handlers for asynchronous events +======================================================= .. module:: signal :synopsis: Set handlers for asynchronous events. @@ -26,9 +26,9 @@ explicitly reset (Python emulates the BSD style interface regardless of the underlying implementation), with the exception of the handler for :const:`SIGCHLD`, which follows the underlying implementation. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, signals -are emulated and therefore behave differently. Several functions and signals -are not available on these platforms. +On WebAssembly platforms, signals are emulated and therefore behave +differently. Several functions and signals are not available on these +platforms. Execution of Python signal handlers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -157,6 +157,8 @@ The variables defined in the :mod:`signal` module are: Alias to :data:`SIGCHLD`. + .. availability:: not macOS. + .. data:: SIGCONT Continue the process if it is currently stopped diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 2dc9fb09d727e2..1c420419568a90 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -1,5 +1,5 @@ -:mod:`site` --- Site-specific configuration hook -================================================ +:mod:`!site` --- Site-specific configuration hook +================================================= .. module:: site :synopsis: Module responsible for site-specific configuration. @@ -74,6 +74,10 @@ with ``import`` (followed by space or tab) are executed. Limiting a code chunk to a single line is a deliberate measure to discourage putting anything more complex here. +.. versionchanged:: 3.13 + The :file:`.pth` files are now decoded by UTF-8 at first and then by the + :term:`locale encoding` if it fails. + .. index:: single: package triple: path; configuration; file diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index aaec2aa1ef1dbe..7cd530a5fd6438 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -1,5 +1,5 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= +:mod:`!smtplib` --- SMTP protocol client +======================================== .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). @@ -556,34 +556,33 @@ This example prompts the user for addresses needed in the message envelope ('To' and 'From' addresses), and the message to be delivered. Note that the headers to be included with the message must be included in the message as entered; this example doesn't do any processing of the :rfc:`822` headers. In particular, the -'To' and 'From' addresses must be included in the message headers explicitly. :: +'To' and 'From' addresses must be included in the message headers explicitly:: import smtplib - def prompt(prompt): - return input(prompt).strip() + def prompt(title): + return input(title).strip() - fromaddr = prompt("From: ") - toaddrs = prompt("To: ").split() + from_addr = prompt("From: ") + to_addrs = prompt("To: ").split() print("Enter message, end with ^D (Unix) or ^Z (Windows):") # Add the From: and To: headers at the start! - msg = ("From: %s\r\nTo: %s\r\n\r\n" - % (fromaddr, ", ".join(toaddrs))) + lines = [f"From: {from_addr}", f"To: {', '.join(to_addrs)}", ""] while True: try: line = input() except EOFError: break - if not line: - break - msg = msg + line + else: + lines.append(line) + msg = "\r\n".join(lines) print("Message length is", len(msg)) - server = smtplib.SMTP('localhost') + server = smtplib.SMTP("localhost") server.set_debuglevel(1) - server.sendmail(fromaddr, toaddrs, msg) + server.sendmail(from_addr, to_addrs, msg) server.quit() .. note:: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c3cf48316b3248..782fb9b27ae1ba 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1,5 +1,5 @@ -:mod:`socket` --- Low-level networking interface -================================================ +:mod:`!socket` --- Low-level networking interface +================================================= .. module:: socket :synopsis: Low-level networking interface. @@ -23,7 +23,7 @@ all modern Unix systems, Windows, MacOS, and probably additional platforms. The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement +:func:`~socket.socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -185,7 +185,7 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.7 - :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple + The addresses are represented by the tuple ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - *ifname* - String specifying the device name. @@ -193,7 +193,6 @@ created. Socket addresses are represented as follows: May be :data:`ETH_P_ALL` to capture all protocols, one of the :ref:`ETHERTYPE_* constants <socket-ethernet-types>` or any other Ethernet protocol number. - Value must be in network-byte-order. - *pkttype* - Optional integer specifying the packet type: - ``PACKET_HOST`` (the default) - Packet addressed to the local host. @@ -312,7 +311,7 @@ Exceptions The accompanying value is a pair ``(error, string)`` representing an error returned by a library call. *string* represents the description of *error*, as returned by the :c:func:`gai_strerror` C function. The - numeric *error* value will match one of the :const:`EAI_\*` constants + numeric *error* value will match one of the :const:`!EAI_\*` constants defined in this module. .. versionchanged:: 3.3 @@ -348,10 +347,15 @@ Constants AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`~socket.socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. More constants may be available depending on the system. +.. data:: AF_UNSPEC + + :const:`AF_UNSPEC` means that + :func:`getaddrinfo` should return socket addresses for any + address family (either IPv4, IPv6, or any other) that can be used. .. data:: SOCK_STREAM SOCK_DGRAM @@ -360,7 +364,7 @@ Constants SOCK_SEQPACKET These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. + :func:`~socket.socket`. More constants may be available depending on the system. (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally useful.) @@ -380,6 +384,8 @@ Constants .. versionadded:: 3.2 +.. _socket-unix-constants: + .. data:: SO_* SOMAXCONN MSG_* @@ -397,7 +403,7 @@ Constants Many constants of these forms, documented in the Unix documentation on sockets and/or the IP protocol, are also defined in the socket module. They are - generally used in arguments to the :meth:`setsockopt` and :meth:`getsockopt` + generally used in arguments to the :meth:`~socket.setsockopt` and :meth:`~socket.getsockopt` methods of socket objects. In most cases, only those symbols that are defined in the Unix header files are defined; for a few symbols, default values are provided. @@ -439,6 +445,15 @@ Constants Added ``IP_PKTINFO``, ``IP_UNBLOCK_SOURCE``, ``IP_BLOCK_SOURCE``, ``IP_ADD_SOURCE_MEMBERSHIP``, ``IP_DROP_SOURCE_MEMBERSHIP``. + .. versionchanged:: 3.13 + Added ``SO_BINDTOIFINDEX``. On Linux this constant can be used in the + same way that ``SO_BINDTODEVICE`` is used, but with the index of a + network interface instead of its name. + + .. versionchanged:: 3.14 + Added missing ``IP_RECVERR``, ``IP_RECVTTL``, and ``IP_RECVORIGDSTADDR`` + on Linux. + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -763,7 +778,7 @@ The following functions all create :ref:`socket objects <socket-objects>`. Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable <fd_inheritance>`. @@ -858,8 +873,8 @@ The following functions all create :ref:`socket objects <socket-objects>`. .. function:: fromfd(fd, family, type, proto=0) Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function + :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address + family, socket type and protocol number are as for the :func:`~socket.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on @@ -924,7 +939,7 @@ The :mod:`socket` module also offers various network-related services: ``(family, type, proto, canonname, sockaddr)`` In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be + meant to be passed to the :func:`~socket.socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -1040,7 +1055,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: getprotobyname(protocolname) Translate an internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` + suitable for passing as the (optional) third argument to the :func:`~socket.socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -1202,7 +1217,7 @@ The :mod:`socket` module also offers various network-related services: buffer. Raises :exc:`OverflowError` if *length* is outside the permissible range of values. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. Most Unix platforms. @@ -1225,7 +1240,7 @@ The :mod:`socket` module also offers various network-related services: amount of ancillary data that can be received, since additional data may be able to fit into the padding area. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. most Unix platforms. @@ -1265,7 +1280,7 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1292,7 +1307,7 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1309,7 +1324,7 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1324,9 +1339,9 @@ The :mod:`socket` module also offers various network-related services: Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. + Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1338,9 +1353,9 @@ The :mod:`socket` module also offers various network-related services: Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. Return ``(msg, list(fds), flags, addr)``. - Consult :meth:`recvmsg` for the documentation of these parameters. + Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1511,7 +1526,7 @@ to sockets. .. method:: socket.getsockopt(level, optname[, buflen]) Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:const:`SO_\*` etc.) + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. <socket-unix-constants>`) are defined in this module. If *buflen* is absent, an integer option is assumed and its integer value is returned by the function. If *buflen* is present, it specifies the maximum length of the buffer used to receive the option in, and @@ -1578,7 +1593,8 @@ to sockets. Return a :term:`file object` associated with the socket. The exact returned type depends on the arguments given to :meth:`makefile`. These arguments are interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. The socket must be in blocking mode; it can have a timeout, but the file object's internal buffer may end up in an inconsistent state if a timeout @@ -1599,8 +1615,9 @@ to sockets. Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. .. note:: @@ -1931,8 +1948,8 @@ to sockets. .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer, + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. <socket-unix-constants>`). The value can be an integer, ``None`` or a :term:`bytes-like object` representing a buffer. In the later case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to @@ -2057,10 +2074,10 @@ Example Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, +using it. Note that a server must perform the sequence :func:`~socket.socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`~socket.socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 5fd213fa613c8d..69f06e6cf4d923 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -1,5 +1,5 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= +:mod:`!socketserver` --- A framework for network servers +======================================================== .. module:: socketserver :synopsis: A framework for network servers. @@ -126,6 +126,12 @@ server is the address family. waits until all non-daemon threads complete, except if :attr:`block_on_close` attribute is ``False``. + .. attribute:: max_children + + Specify how many child processes will exist to handle requests at a time + for :class:`ForkingMixIn`. If the limit is reached, + new requests will wait until one child process has finished. + .. attribute:: daemon_threads For :class:`ThreadingMixIn` use daemonic threads by setting @@ -494,7 +500,7 @@ This is the server side:: def handle(self): # self.request is the TCP socket connected to the client self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) + print("Received from {}:".format(self.client_address[0])) print(self.data) # just send back the same data, but upper-cased self.request.sendall(self.data.upper()) @@ -525,8 +531,9 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. +single ``recv()`` call in the first handler will just return what has been +received so far from the client's ``sendall()`` call (typically all of it, but +this is not guaranteed by the TCP protocol). This is the client side:: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fc87aa94d3363b..5dc22a7e431ae4 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1,5 +1,5 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ +:mod:`!sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================= .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. @@ -16,6 +16,8 @@ src = sqlite3.connect(":memory:", isolation_level=None) dst = sqlite3.connect("tutorial.db", isolation_level=None) src.backup(dst) + src.close() + dst.close() del src, dst .. _sqlite3-intro: @@ -220,6 +222,7 @@ creating a new cursor, then querying the database: >>> title, year = res.fetchone() >>> print(f'The highest scoring Monty Python movie is {title!r}, released in {year}') The highest scoring Monty Python movie is 'Monty Python and the Holy Grail', released in 1975 + >>> new_con.close() You've now created an SQLite database using the :mod:`!sqlite3` module, inserted data and retrieved values from it in multiple ways. @@ -343,17 +346,17 @@ Module functions .. audit-event:: sqlite3.connect database sqlite3.connect .. audit-event:: sqlite3.connect/handle connection_handle sqlite3.connect - .. versionadded:: 3.4 - The *uri* parameter. + .. versionchanged:: 3.4 + Added the *uri* parameter. .. versionchanged:: 3.7 *database* can now also be a :term:`path-like object`, not only a string. - .. versionadded:: 3.10 - The ``sqlite3.connect/handle`` auditing event. + .. versionchanged:: 3.10 + Added the ``sqlite3.connect/handle`` auditing event. - .. versionadded:: 3.12 - The *autocommit* parameter. + .. versionchanged:: 3.12 + Added the *autocommit* parameter. .. versionchanged:: 3.13 Positional use of the parameters *timeout*, *detect_types*, @@ -394,29 +397,11 @@ Module functions will get tracebacks from callbacks on :data:`sys.stderr`. Use ``False`` to disable the feature again. - Register an :func:`unraisable hook handler <sys.unraisablehook>` for an - improved debug experience: - - .. testsetup:: sqlite3.trace - - import sqlite3 + .. note:: - .. doctest:: sqlite3.trace - - >>> sqlite3.enable_callback_tracebacks(True) - >>> con = sqlite3.connect(":memory:") - >>> def evil_trace(stmt): - ... 5/0 - ... - >>> con.set_trace_callback(evil_trace) - >>> def debug(unraisable): - ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") - ... print(f"Error message: {unraisable.err_msg}") - >>> import sys - >>> sys.unraisablehook = debug - >>> cur = con.execute("SELECT 1") - ZeroDivisionError('division by zero') in callback evil_trace - Error message: None + Errors in user-defined function callbacks are logged as unraisable exceptions. + Use an :func:`unraisable hook handler <sys.unraisablehook>` for + introspection of the failed callback. .. function:: register_adapter(type, adapter, /) @@ -560,26 +545,6 @@ Module constants .. versionchanged:: 3.11 Set *threadsafety* dynamically instead of hard-coding it to ``1``. -.. data:: version - - Version number of this module as a :class:`string <str>`. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - -.. data:: version_info - - Version number of this module as a :class:`tuple` of :class:`integers <int>`. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - .. _sqlite3-dbconfig-constants: .. data:: SQLITE_DBCONFIG_DEFENSIVE @@ -612,6 +577,8 @@ Module constants https://www.sqlite.org/c3ref/c_dbconfig_defensive.html SQLite docs: Database Connection Configuration Options +.. deprecated-removed:: 3.12 3.14 + The :data:`!version` and :data:`!version_info` constants. .. _sqlite3-connection-objects: @@ -747,8 +714,8 @@ Connection objects `deterministic <https://sqlite.org/deterministic.html>`_, which allows SQLite to perform additional optimizations. - .. versionadded:: 3.8 - The *deterministic* parameter. + .. versionchanged:: 3.8 + Added the *deterministic* parameter. Example: @@ -762,6 +729,7 @@ Connection objects >>> for row in con.execute("SELECT md5(?)", (b"foo",)): ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + >>> con.close() .. versionchanged:: 3.13 @@ -908,6 +876,7 @@ Connection objects FROM test ORDER BY x """) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -1068,13 +1037,10 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.enable_load_extension`` auditing event. - .. testsetup:: sqlite3.loadext - - import sqlite3 - con = sqlite3.connect(":memory:") + .. We cannot doctest the load extension API, since there is no convenient + way to skip it. - .. testcode:: sqlite3.loadext - :skipif: True # not testable at the moment + .. code-block:: con.enable_load_extension(True) @@ -1098,14 +1064,6 @@ Connection objects for row in con.execute("SELECT rowid, name, ingredients FROM recipe WHERE name MATCH 'pie'"): print(row) - con.close() - - .. testoutput:: sqlite3.loadext - :hide: - - (2, 'broccoli pie', 'broccoli cheese onions flour') - (3, 'pumpkin pie', 'pumpkin sugar flour butter') - .. method:: load_extension(path, /, *, entrypoint=None) Load an SQLite extension from a shared library. @@ -1132,17 +1090,24 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.load_extension`` auditing event. - .. versionadded:: 3.12 - The *entrypoint* parameter. + .. versionchanged:: 3.12 + Added the *entrypoint* parameter. - .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ + .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension - .. method:: iterdump + .. method:: iterdump(*, filter=None) Return an :term:`iterator` to dump the database as SQL source code. Useful when saving an in-memory database for later restoration. Similar to the ``.dump`` command in the :program:`sqlite3` shell. + :param filter: + + An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``. + If ``None`` (the default), all database objects will be included. + + :type filter: str | None + Example: .. testcode:: @@ -1154,6 +1119,12 @@ Connection objects f.write('%s\n' % line) con.close() + .. seealso:: + + :ref:`sqlite3-howto-encoding` + + .. versionchanged:: 3.13 + Added the *filter* parameter. .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) @@ -1217,9 +1188,15 @@ Connection objects src = sqlite3.connect('example.db') dst = sqlite3.connect(':memory:') src.backup(dst) + dst.close() + src.close() .. versionadded:: 3.7 + .. seealso:: + + :ref:`sqlite3-howto-encoding` + .. method:: getlimit(category, /) Get a connection runtime limit. @@ -1279,6 +1256,10 @@ Connection objects >>> con.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) 1 + .. testcleanup:: sqlite3.limits + + con.close() + .. versionadded:: 3.11 .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html @@ -1441,39 +1422,8 @@ Connection objects and returns a text representation of it. The callable is invoked for SQLite values with the ``TEXT`` data type. By default, this attribute is set to :class:`str`. - If you want to return ``bytes`` instead, set *text_factory* to ``bytes``. - Example: - - .. testcode:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - - AUSTRIA = "Österreich" - - # by default, rows are returned as str - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert row[0] == AUSTRIA - - # but we can make sqlite3 always return bytestrings ... - con.text_factory = bytes - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert type(row[0]) is bytes - # the bytestrings will be encoded in UTF-8, unless you stored garbage in the - # database ... - assert row[0] == AUSTRIA.encode("utf-8") - - # we can also implement a custom text_factory ... - # here we implement one that appends "foo" to all strings - con.text_factory = lambda x: x.decode("utf-8") + "foo" - cur.execute("SELECT ?", ("bar",)) - row = cur.fetchone() - assert row[0] == "barfoo" - - con.close() + See :ref:`sqlite3-howto-encoding` for more details. .. attribute:: total_changes @@ -1525,7 +1475,7 @@ Cursor objects .. method:: execute(sql, parameters=(), /) - Execute SQL a single SQL statement, + Execute a single SQL statement, optionally binding Python values using :ref:`placeholders <sqlite3-placeholders>`. @@ -1591,6 +1541,10 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. testcleanup:: sqlite3.cursor + + con.close() + .. note:: Any resulting rows are discarded, @@ -1632,7 +1586,6 @@ Cursor objects COMMIT; """) - .. method:: fetchone() If :attr:`~Cursor.row_factory` is ``None``, @@ -1697,6 +1650,7 @@ Cursor objects >>> cur = con.cursor() >>> cur.connection == con True + >>> con.close() .. attribute:: description @@ -1786,10 +1740,10 @@ Row objects Blob objects ^^^^^^^^^^^^ -.. versionadded:: 3.11 - .. class:: Blob + .. versionadded:: 3.11 + A :class:`Blob` instance is a :term:`file-like object` that can read and write data in an SQLite :abbr:`BLOB (Binary Large OBject)`. Call :func:`len(blob) <len>` to get the size (number of bytes) of the blob. @@ -1817,6 +1771,7 @@ Blob objects greeting = blob.read() print(greeting) # outputs "b'Hello, world!'" + con.close() .. testoutput:: :hide: @@ -2129,6 +2084,7 @@ Here's an example of both styles: params = (1972,) cur.execute("SELECT * FROM lang WHERE first_appeared = ?", params) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -2187,6 +2143,7 @@ The object passed to *protocol* will be of type :class:`PrepareProtocol`. cur.execute("SELECT ?", (Point(4.0, -3.2),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2217,6 +2174,7 @@ This function can then be registered using :func:`register_adapter`. cur.execute("SELECT ?", (Point(1.0, 2.5),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2301,6 +2259,8 @@ The following example illustrates the implicit and explicit approaches: cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) cur.execute('SELECT p AS "p [point]" FROM test') print("with column names:", cur.fetchone()[0]) + cur.close() + con.close() .. testoutput:: :hide: @@ -2507,6 +2467,8 @@ Some useful URI tricks include: res = con2.execute("SELECT data FROM shared") assert res.fetchone() == (28,) + con1.close() + con2.close() More information about this feature, including a list of parameters, can be found in the `SQLite URI documentation`_. @@ -2553,6 +2515,7 @@ Queries now return :class:`!Row` objects: 'Earth' >>> row["RADIUS"] # Column names are case-insensitive. 6378 + >>> con.close() .. note:: @@ -2579,6 +2542,7 @@ Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): ... print(row) {'a': 1, 'b': 2} + >>> con.close() The following row factory returns a :term:`named tuple`: @@ -2605,12 +2569,54 @@ The following row factory returns a :term:`named tuple`: 1 >>> row.b # Attribute access. 2 + >>> con.close() With some adjustments, the above recipe can be adapted to use a :class:`~dataclasses.dataclass`, or any other custom class, instead of a :class:`~collections.namedtuple`. +.. _sqlite3-howto-encoding: + +How to handle non-UTF-8 text encodings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, :mod:`!sqlite3` uses :class:`str` to adapt SQLite values +with the ``TEXT`` data type. +This works well for UTF-8 encoded text, but it might fail for other encodings +and invalid UTF-8. +You can use a custom :attr:`~Connection.text_factory` to handle such cases. + +Because of SQLite's `flexible typing`_, it is not uncommon to encounter table +columns with the ``TEXT`` data type containing non-UTF-8 encodings, +or even arbitrary data. +To demonstrate, let's assume we have a database with ISO-8859-2 (Latin-2) +encoded text, for example a table of Czech-English dictionary entries. +Assuming we now have a :class:`Connection` instance :py:data:`!con` +connected to this database, +we can decode the Latin-2 encoded text using this :attr:`~Connection.text_factory`: + +.. testcode:: + + con.text_factory = lambda data: str(data, encoding="latin2") + +For invalid UTF-8 or arbitrary data in stored in ``TEXT`` table columns, +you can use the following technique, borrowed from the :ref:`unicode-howto`: + +.. testcode:: + + con.text_factory = lambda data: str(data, errors="surrogateescape") + +.. note:: + + The :mod:`!sqlite3` module API does not support strings + containing surrogates. + +.. seealso:: + + :ref:`unicode-howto` + + .. _sqlite3-explanation: Explanation @@ -2721,3 +2727,11 @@ regardless of the value of :attr:`~Connection.isolation_level`. .. _SQLite transaction behaviour: https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions + +.. testcleanup:: + + import os + os.remove("backup.db") + os.remove("dump.sql") + os.remove("example.db") + os.remove("tutorial.db") diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 21b38ae62fe02f..dc72f67c6361e2 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1,5 +1,5 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= +:mod:`!ssl` --- TLS/SSL wrapper for socket objects +================================================== .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects @@ -25,7 +25,7 @@ probably additional platforms, as long as OpenSSL is installed on that platform. Some behavior may be platform dependent, since calls are made to the operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.3 with OpenSSL version + cause variations in behavior. For example, TLSv1.3 comes with OpenSSL version 1.1.1. .. warning:: @@ -151,6 +151,12 @@ purposes. variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` enables key logging. + The default settings for this context include + :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. + These make the underlying OpenSSL implementation behave more like + a conforming implementation of :rfc:`5280`, in exchange for a small + amount of incompatibility with older X.509 certificates. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +178,15 @@ purposes. ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) ctx.options &= ~ssl.OP_NO_SSLv3 + .. note:: + This context enables :data:`VERIFY_X509_STRICT` by default, which + may reject pre-:rfc:`5280` or malformed certificates that the + underlying OpenSSL implementation otherwise would accept. While disabling + this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + .. versionadded:: 3.4 .. versionchanged:: 3.4.4 @@ -194,6 +209,11 @@ purposes. :data:`PROTOCOL_TLS_SERVER` protocol instead of generic :data:`PROTOCOL_TLS`. + .. versionchanged:: 3.13 + + The context now uses :data:`VERIFY_X509_PARTIAL_CHAIN` and + :data:`VERIFY_X509_STRICT` in its default verify flags. + Exceptions ^^^^^^^^^^ @@ -737,11 +757,11 @@ Constants When Python has been compiled against an older version of OpenSSL, the flag defaults to *0*. - .. versionadded:: 3.7 + .. versionadded:: 3.6.3 .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15, - 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2. + The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15 and + 3.6.3 for backwards compatibility with OpenSSL 1.0.2. .. data:: OP_NO_RENEGOTIATION @@ -761,7 +781,7 @@ Constants .. data:: OP_SINGLE_DH_USE - Prevents re-use of the same DH key for distinct SSL sessions. This + Prevents reuse of the same DH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -769,7 +789,7 @@ Constants .. data:: OP_SINGLE_ECDH_USE - Prevents re-use of the same ECDH key for distinct SSL sessions. This + Prevents reuse of the same ECDH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -908,6 +928,12 @@ Constants .. versionadded:: 3.7 +.. data:: HAS_PSK + + Whether the OpenSSL library has built-in support for TLS-PSK. + + .. versionadded:: 3.13 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -1446,6 +1472,19 @@ to speed up repeated connections from the same clients. :data:`PROTOCOL_TLS`, :data:`PROTOCOL_TLS_CLIENT`, and :data:`PROTOCOL_TLS_SERVER` use TLS 1.2 as minimum TLS version. + .. note:: + + :class:`SSLContext` only supports limited mutation once it has been used + by a connection. Adding new certificates to the internal trust store is + allowed, but changing ciphers, verification settings, or mTLS + certificates may result in surprising behavior. + + .. note:: + + :class:`SSLContext` is designed to be shared and used by multiple + connections. + Thus, it is thread-safe as long as it is not reconfigured after being + used by a connection. :class:`SSLContext` objects have the following methods and attributes: @@ -1784,6 +1823,9 @@ to speed up repeated connections from the same clients. *session*, see :attr:`~SSLSocket.session`. + To wrap an :class:`SSLSocket` in another :class:`SSLSocket`, use + :meth:`SSLContext.wrap_bio`. + .. versionchanged:: 3.5 Always allow a server_hostname to be passed, even if OpenSSL does not have SNI. @@ -1791,7 +1833,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.6 *session* argument was added. - .. versionchanged:: 3.7 + .. versionchanged:: 3.7 The method returns an instance of :attr:`SSLContext.sslsocket_class` instead of hard-coded :class:`SSLSocket`. @@ -1969,7 +2011,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.10 - The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + The flag had no effect with OpenSSL before version 1.1.1l. Python 3.8.9, 3.9.3, and 3.10 include workarounds for previous versions. .. attribute:: SSLContext.security_level @@ -2006,6 +2048,100 @@ to speed up repeated connections from the same clients. >>> ssl.create_default_context().verify_mode # doctest: +SKIP <VerifyMode.CERT_REQUIRED: 2> +.. method:: SSLContext.set_psk_client_callback(callback) + + Enables TLS-PSK (pre-shared key) authentication on a client-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(hint: str | None) -> tuple[str | None, bytes]``. + The ``hint`` parameter is an optional identity hint sent by the server. + The return value is a tuple in the form (client-identity, psk). + Client-identity is an optional string which may be used by the server to + select a corresponding PSK for the client. The string must be less than or + equal to ``256`` octets when UTF-8 encoded. PSK is a + :term:`bytes-like object` representing the pre-shared key. Return a zero + length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + .. note:: + When using TLS 1.3: + + - the ``hint`` parameter is always :const:`None`. + - client-identity must be a non-empty string. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_client_callback(lambda hint: (None, psk)) + + # A table using the hint from the server: + psk_table = { 'ServerId_1': bytes.fromhex('c0ffee'), + 'ServerId_2': bytes.fromhex('facade') + } + def callback(hint): + return 'ClientId_1', psk_table.get(hint, b'') + context.set_psk_client_callback(callback) + + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + + .. versionadded:: 3.13 + +.. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) + + Enables TLS-PSK (pre-shared key) authentication on a server-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(identity: str | None) -> bytes``. + The ``identity`` parameter is an optional identity sent by the client which can + be used to select a corresponding PSK. + The return value is a :term:`bytes-like object` representing the pre-shared key. + Return a zero length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + The parameter ``identity_hint`` is an optional identity hint string sent to + the client. The string must be less than or equal to ``256`` octets when + UTF-8 encoded. + + .. note:: + When using TLS 1.3 the ``identity_hint`` parameter is not sent to the client. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_server_callback(lambda identity: psk) + + # A table using the identity of the client: + psk_table = { 'ClientId_1': bytes.fromhex('c0ffee'), + 'ClientId_2': bytes.fromhex('facade') + } + def callback(identity): + return psk_table.get(identity, b'') + context.set_psk_server_callback(callback, 'ServerId_1') + + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + + .. versionadded:: 3.13 + .. index:: single: certificates .. index:: single: X509 certificate @@ -2474,12 +2610,8 @@ provided. :exc:`SSLWantReadError` if it needs more data than the incoming BIO has available. - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - .. versionchanged:: 3.7 - :class:`SSLObject` instances must to created with + :class:`SSLObject` instances must be created with :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 77538514598a50..8434b2e8c75cf4 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -1,5 +1,5 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== +:mod:`!stat` --- Interpreting :func:`~os.stat` results +====================================================== .. module:: stat :synopsis: Utilities for interpreting the results of os.stat(), @@ -350,6 +350,12 @@ The following flags can also be used in the *mode* argument of :func:`os.chmod`: The following flags can be used in the *flags* argument of :func:`os.chflags`: +.. data:: UF_SETTABLE + + All user settable flags. + + .. versionadded:: 3.13 + .. data:: UF_NODUMP Do not dump the file. @@ -374,10 +380,44 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is stored compressed (macOS 10.6+). +.. data:: UF_TRACKED + + Used for handling document IDs (macOS) + + .. versionadded:: 3.13 + +.. data:: UF_DATAVAULT + + The file needs an entitlement for reading or writing (macOS 10.13+) + + .. versionadded:: 3.13 + .. data:: UF_HIDDEN The file should not be displayed in a GUI (macOS 10.5+). +.. data:: SF_SETTABLE + + All super-user changeable flags + + .. versionadded:: 3.13 + +.. data:: SF_SUPPORTED + + All super-user supported flags + + .. availability:: macOS + + .. versionadded:: 3.13 + +.. data:: SF_SYNTHETIC + + All super-user read-only synthetic flags + + .. availability:: macOS + + .. versionadded:: 3.13 + .. data:: SF_ARCHIVED The file may be archived. @@ -390,6 +430,12 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file may only be appended to. +.. data:: SF_RESTRICTED + + The file needs an entitlement to write to (macOS 10.13+) + + .. versionadded:: 3.13 + .. data:: SF_NOUNLINK The file may not be renamed or deleted. @@ -398,6 +444,18 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is a snapshot file. +.. data:: SF_FIRMLINK + + The file is a firmlink (macOS 10.15+) + + .. versionadded:: 3.13 + +.. data:: SF_DATALESS + + The file is a dataless object (macOS 10.15+) + + .. versionadded:: 3.13 + See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. On Windows, the following file attribute constants are available for use when diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 5c8ad3a7dd7380..8453135d2e164d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1,5 +1,5 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= +:mod:`!statistics` --- Mathematical statistics functions +======================================================== .. module:: statistics :synopsis: Mathematical statistics functions @@ -76,10 +76,12 @@ or sample. :func:`fmean` Fast, floating point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. +:func:`kde` Estimate the probability density distribution of the data. +:func:`kde_random` Random sampling from the PDF generated by kde(). :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`median_grouped` Median (50th percentile) of grouped data. :func:`mode` Single mode (most common value) of discrete or nominal data. :func:`multimode` List of modes (most common values) of discrete or nominal data. :func:`quantiles` Divide data into intervals with equal probability. @@ -218,7 +220,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data, weights=None) Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then + real-valued numbers. If *weights* is omitted or ``None``, then equal weighting is assumed. The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the @@ -259,6 +261,81 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionchanged:: 3.10 Added support for *weights*. + +.. function:: kde(data, h, kernel='normal', *, cumulative=False) + + `Kernel Density Estimation (KDE) + <https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf>`_: + Create a continuous probability density function or cumulative + distribution function from discrete samples. + + The basic idea is to smooth the data using `a kernel function + <https://en.wikipedia.org/wiki/Kernel_(statistics)>`_. + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter *h* + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The *kernel* determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point include + *normal* (*gauss*), *logistic*, and *sigmoid*. + + Kernels that only give weight to sample points within the bandwidth + include *rectangular* (*uniform*), *triangular*, *parabolic* + (*epanechnikov*), *quartic* (*biweight*), *triweight*, and *cosine*. + + If *cumulative* is true, will return a cumulative distribution function. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + `Wikipedia has an example + <https://en.wikipedia.org/wiki/Kernel_density_estimation#Example>`_ + where we can use :func:`kde` to generate and plot a probability + density function estimated from a small sample: + + .. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + + The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + + .. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + + .. versionadded:: 3.13 + + +.. function:: kde_random(data, h, kernel='normal', *, seed=None) + + Return a function that makes a random selection from the estimated + probability density function produced by ``kde(data, h, kernel)``. + + Providing a *seed* allows reproducible selections. In the future, the + values may change slightly as more accurate kernel inverse CDF estimates + are implemented. The seed may be an integer, float, str, or bytes. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + Continuing the example for :func:`kde`, we can use + :func:`kde_random` to generate new random selections from an + estimated probability density function: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -329,55 +406,56 @@ However, for reading convenience, most of the examples show sorted sequences. be an actual data point rather than interpolated. -.. function:: median_grouped(data, interval=1) +.. function:: median_grouped(data, interval=1.0) - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. + Estimates the median for numeric data that has been `grouped or binned + <https://en.wikipedia.org/wiki/Data_binning>`_ around the midpoints + of consecutive, fixed-width intervals. - .. doctest:: + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *interval* is the width of each bin. - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: .. doctest:: - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: + >>> from collections import Counter + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) + ... + + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular :func:`median` function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use + :func:`median_grouped`: .. doctest:: - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - <https://help.gnome.org/users/gnumeric/stable/gnumeric.html#gnumeric-function-SSMEDIAN>`_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - <https://mail.gnome.org/archives/gnumeric-list/2011-April/msg00018.html>`_. + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. .. function:: mode(data) @@ -448,9 +526,9 @@ However, for reading convenience, most of the examples show sorted sequences. variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), + If the optional second argument *mu* is given, it should be the *population* + mean of the *data*. It can also be used to compute the second moment around + a point that is not the mean. If it is missing or ``None`` (the default), the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To @@ -520,8 +598,8 @@ However, for reading convenience, most of the examples show sorted sequences. the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is + If the optional second argument *xbar* is given, it should be the *sample* + mean of *data*. If it is missing or ``None`` (the default), the mean is automatically calculated. Use this function when your data is a sample from a population. To calculate @@ -537,8 +615,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> variance(data) 1.3720238095238095 - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: + If you have already calculated the sample mean of your data, you can pass it + as the optional second argument *xbar* to avoid recalculation: .. doctest:: @@ -948,8 +1026,8 @@ of applications in statistics. .. versionadded:: 3.8 -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Examples and Recipes +-------------------- Classic probability problems @@ -984,7 +1062,7 @@ Find the `quartiles <https://en.wikipedia.org/wiki/Quartile>`_ and `deciles Monte Carlo inputs for simulations ********************************** -To estimate the distribution for a model than isn't easy to solve +To estimate the distribution for a model that isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation <https://en.wikipedia.org/wiki/Monte_Carlo_method>`_: @@ -1026,19 +1104,16 @@ probability that the Python room will stay within its capacity limits? >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) 0.8402 - >>> # Solution using the cumulative binomial distribution + >>> # Exact solution using the cumulative binomial distribution >>> from math import comb, fsum >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) 0.8402 >>> # Approximation using a simulation - >>> from random import seed, choices + >>> from random import seed, binomialvariate >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - ... - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 + >>> mean(binomialvariate(n, p) <= k for i in range(10_000)) + 0.8406 Naive bayesian classifier @@ -1098,48 +1173,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Kernel density estimation -************************* - -It is possible to estimate a continuous probability density function -from a fixed number of discrete samples. - -The basic idea is to smooth the data using `a kernel function such as a -normal distribution, triangular distribution, or uniform distribution -<https://en.wikipedia.org/wiki/Kernel_(statistics)#Kernel_functions_in_common_use>`_. -The degree of smoothing is controlled by a single -parameter, ``h``, representing the variance of the kernel function. - -.. testcode:: - - import math - - def kde_normal(sample, h): - "Create a continuous probability density function from a sample." - # Smooth the sample with a normal distribution of variance h. - kernel_h = NormalDist(0.0, math.sqrt(h)).pdf - n = len(sample) - def pdf(x): - return sum(kernel_h(x - x_i) for x_i in sample) / n - return pdf - -`Wikipedia has an example -<https://en.wikipedia.org/wiki/Kernel_density_estimation#Example>`_ -where we can use the ``kde_normal()`` recipe to generate and plot -a probability density function estimated from a small sample: - -.. doctest:: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde_normal(sample, h=2.25) - >>> xarr = [i/100 for i in range(-750, 1100)] - >>> yarr = [f_hat(x) for x in xarr] - -The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: - -.. image:: kde_example.png - :alt: Scatter plot of the estimated probability density function. - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 779d5ba49236ad..73bf6847a4ca65 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -44,7 +44,8 @@ Any object can be tested for truth value, for use in an :keyword:`if` or .. index:: single: true By default, an object is considered true unless its class defines either a -:meth:`~object.__bool__` method that returns ``False`` or a :meth:`__len__` method that +:meth:`~object.__bool__` method that returns ``False`` or a +:meth:`~object.__len__` method that returns zero, when called with the object. [1]_ Here are most of the built-in objects considered false: @@ -197,7 +198,7 @@ exception. Two more operations with the same syntactic priority, :keyword:`in` and :keyword:`not in`, are supported by types that are :term:`iterable` or -implement the :meth:`__contains__` method. +implement the :meth:`~object.__contains__` method. .. _typesnumeric: @@ -753,7 +754,7 @@ that's defined for any rational number, and hence applies to all instances of :class:`int` and :class:`fractions.Fraction`, and all finite instances of :class:`float` and :class:`decimal.Decimal`. Essentially, this function is given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is -made available to Python as the :attr:`modulus` attribute of +made available to Python as the :attr:`~sys.hash_info.modulus` attribute of :data:`sys.hash_info`. .. impl-detail:: @@ -942,9 +943,9 @@ Generator Types --------------- Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`__iter__` method is implemented as a +protocol. If a container object's :meth:`~iterator.__iter__` method is implemented as a generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` +generator object) supplying the :meth:`!__iter__` and :meth:`~generator.__next__` methods. More information about generators can be found in :ref:`the documentation for the yield expression <yieldexpr>`. @@ -1255,7 +1256,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). Notes: (1) - *t* must have the same length as the slice it is replacing. + If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) The optional argument *i* defaults to ``-1``, so that by default the last @@ -1526,13 +1527,12 @@ objects that compare equal might have different :attr:`~range.start`, sequence of values they define (instead of comparing based on object identity). -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` + Added the :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` attributes. .. seealso:: - * The `linspace recipe <https://code.activestate.com/recipes/579000/>`_ + * The `linspace recipe <https://code.activestate.com/recipes/579000-equally-spaced-numbers-linspace/>`_ shows how to implement a lazy version of range suitable for floating point applications. @@ -1563,7 +1563,7 @@ between them will be implicitly converted to a single string literal. That is, ``("spam " "eggs") == "spam eggs"``. See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that +including supported :ref:`escape sequences <escape-sequences>`, and the ``r`` ("raw") prefix that disables most escape sequence processing. Strings may also be created from other objects using the :class:`str` @@ -1804,7 +1804,7 @@ expression support in the :mod:`re` module). cases. -.. method:: str.format_map(mapping) +.. method:: str.format_map(mapping, /) Similar to ``str.format(**mapping)``, except that ``mapping`` is used directly and not copied to a :class:`dict`. This is useful @@ -2131,8 +2131,9 @@ expression support in the :mod:`re` module). If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns ``['1', '', '2']``). The *sep* argument may consist of multiple characters - (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). - Splitting an empty string with a specified separator returns ``['']``. + as a single delimiter (to split with multiple delimiters, use + :func:`re.split`). Splitting an empty string with a specified separator + returns ``['']``. For example:: @@ -2142,6 +2143,8 @@ expression support in the :mod:`re` module). ['1', '2,3'] >>> '1,2,,3,'.split(',') ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive whitespace are regarded as a single separator, @@ -2303,7 +2306,7 @@ expression support in the :mod:`re` module). Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements - indexing via :meth:`__getitem__`, typically a :term:`mapping` or + indexing via :meth:`~object.__getitem__`, typically a :term:`mapping` or :term:`sequence`. When indexed by a Unicode ordinal (an integer), the table object can do any of the following: return a Unicode ordinal or a string, to map the character to one or more other characters; return @@ -2375,7 +2378,13 @@ String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a string), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. +For example: + +.. doctest:: + + >>> print('%s has %d quote types.' % ('Python', 2)) + Python has 2 quote types. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -3179,10 +3188,9 @@ produce new objects. If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any + multibyte sequence as a single delimiter. Splitting an empty sequence with + a specified separator returns ``[b'']`` or ``[bytearray(b'')]`` depending + on the type of object being split. The *sep* argument may be any :term:`bytes-like object`. For example:: @@ -3193,6 +3201,8 @@ produce new objects. [b'1', b'2,3'] >>> b'1,2,,3,'.split(b',') [b'1', b'2', b'', b'3', b''] + >>> b'1<>2<>3<4'.split(b'<>') + [b'1', b'2', b'3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive ASCII whitespace are regarded as a single @@ -3708,7 +3718,7 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'b'`` | Bytes (any object that follows the | \(5) | | | :ref:`buffer protocol <bufferobjects>` or has | | -| | :meth:`__bytes__`). | | +| | :meth:`~object.__bytes__`). | | +------------+-----------------------------------------------------+-------+ | ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | | | be used for Python2/3 code bases. | | @@ -4446,7 +4456,8 @@ The constructors for both classes work the same: :meth:`symmetric_difference_update` methods will accept any iterable as an argument. - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and + Note, the *elem* argument to the :meth:`~object.__contains__`, + :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. @@ -4594,7 +4605,7 @@ can be used interchangeably to index the same dictionary entry. Return a shallow copy of the dictionary. - .. classmethod:: fromkeys(iterable[, value]) + .. classmethod:: fromkeys(iterable, value=None, /) Create a new dictionary with keys from *iterable* and values set to *value*. @@ -4604,7 +4615,7 @@ can be used interchangeably to index the same dictionary entry. such as an empty list. To get distinct values, use a :ref:`dict comprehension <dict>` instead. - .. method:: get(key[, default]) + .. method:: get(key, default=None) Return the value for *key* if *key* is in the dictionary, else *default*. If *default* is not given, it defaults to ``None``, so that this method @@ -4646,7 +4657,7 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.8 - .. method:: setdefault(key[, default]) + .. method:: setdefault(key, default=None) If *key* is in the dictionary, return its value. If not, insert *key* with a value of *default* and return *default*. *default* defaults to @@ -4791,14 +4802,17 @@ support membership tests: .. versionadded:: 3.10 -Keys views are set-like since their entries are unique and :term:`hashable`. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like +Keys views are set-like since their entries are unique and :term:`hashable`. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. +If all values in an items view are hashable as well, +then the items view can interoperate with other sets. +(Values views are not treated as set-like since the entries are generally not unique.) For set-like views, all of the operations defined for the abstract base class :class:`collections.abc.Set` are available (for example, ``==``, ``<``, or ``^``). While using set operators, -set-like views accept any iterable as the other operand, unlike sets which only -accept sets as the input. +set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. An example of dictionary view usage:: @@ -4893,7 +4907,7 @@ before the statement body is executed and exited when the statement ends: The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`__exit__` + context management code to easily detect whether or not an :meth:`~object.__exit__` method has actually failed. Python defines several context managers to support easy thread synchronisation, @@ -5087,7 +5101,6 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` -* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5269,9 +5282,11 @@ instantiated from the type:: TypeError: cannot create 'types.UnionType' instances .. note:: - The :meth:`__or__` method for type objects was added to support the syntax - ``X | Y``. If a metaclass implements :meth:`__or__`, the Union may - override it:: + The :meth:`!__or__` method for type objects was added to support the syntax + ``X | Y``. If a metaclass implements :meth:`!__or__`, the Union may + override it: + + .. doctest:: >>> class M(type): ... def __or__(self, other): @@ -5283,7 +5298,7 @@ instantiated from the type:: >>> C | int 'Hello' >>> int | C - int | __main__.C + int | C .. seealso:: @@ -5357,25 +5372,30 @@ Methods .. index:: pair: object; method Methods are functions that are called using the attribute notation. There are -two flavors: built-in methods (such as :meth:`append` on lists) and class -instance methods. Built-in methods are described with the types that support -them. +two flavors: :ref:`built-in methods <builtin-methods>` (such as :meth:`append` +on lists) and :ref:`class instance method <instance-methods>`. +Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an instance, you get a special object: a :dfn:`bound method` (also called -:dfn:`instance method`) object. When called, it will add the ``self`` argument +:ref:`instance method <instance-methods>`) object. When called, it will add +the ``self`` argument to the argument list. Bound methods have two special read-only attributes: -``m.__self__`` is the object on which the method operates, and ``m.__func__`` is +:attr:`m.__self__ <method.__self__>` is the object on which the method +operates, and :attr:`m.__func__ <method.__func__>` is the function implementing the method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)``. -Like function objects, bound method objects support getting arbitrary +Like :ref:`function objects <user-defined-funcs>`, bound method objects support +getting arbitrary attributes. However, since method attributes are actually stored on the -underlying function object (``meth.__func__``), setting method attributes on +underlying function object (:attr:`method.__func__`), setting method attributes on bound methods is disallowed. Attempting to set an attribute on a method results in an :exc:`AttributeError` being raised. In order to set a method -attribute, you need to explicitly set it on the underlying function object:: +attribute, you need to explicitly set it on the underlying function object: + +.. doctest:: >>> class C: ... def method(self): @@ -5390,7 +5410,7 @@ attribute, you need to explicitly set it on the underlying function object:: >>> c.method.whoami 'my name is method' -See :ref:`types` for more information. +See :ref:`instance-methods` for more information. .. index:: object; code, code object @@ -5408,10 +5428,10 @@ Code objects are used by the implementation to represent "pseudo-compiled" executable Python code such as a function body. They differ from function objects because they don't contain a reference to their global execution environment. Code objects are returned by the built-in :func:`compile` function -and can be extracted from function objects through their :attr:`__code__` -attribute. See also the :mod:`code` module. +and can be extracted from function objects through their +:attr:`~function.__code__` attribute. See also the :mod:`code` module. -Accessing ``__code__`` raises an :ref:`auditing event <auditing>` +Accessing :attr:`~function.__code__` raises an :ref:`auditing event <auditing>` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: @@ -5474,10 +5494,10 @@ The NotImplemented Object This object is returned from comparisons and binary operations when they are asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. +information. There is exactly one :data:`NotImplemented` object. +:code:`type(NotImplemented)()` produces the singleton instance. -It is written as ``NotImplemented``. +It is written as :code:`NotImplemented`. .. _typesinternal: @@ -5485,8 +5505,9 @@ It is written as ``NotImplemented``. Internal Objects ---------------- -See :ref:`types` for this information. It describes stack frame objects, -traceback objects, and slice objects. +See :ref:`types` for this information. It describes +:ref:`stack frame objects <frame-objects>`, +:ref:`traceback objects <traceback-objects>`, and slice objects. .. _specialattrs: @@ -5560,6 +5581,13 @@ types, where they are relevant. Some of these are not reported by the [<class 'bool'>, <enum 'IntEnum'>, <flag 'IntFlag'>, <class 're._constants._NamedIntConstant'>] +.. attribute:: class.__static_attributes__ + + A tuple containing names of attributes of this class which are accessed + through ``self.X`` from any function in its body. + + .. versionadded:: 3.13 + .. _int_max_str_digits: Integer string conversion length limitation @@ -5577,8 +5605,7 @@ a string to a binary integer or a binary integer to a string in linear time, have sub-quadratic complexity. Converting a large value such as ``int('1' * 500_000)`` can take over a second on a fast CPU. -Limiting conversion size offers a practical way to avoid `CVE-2020-10735 -<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. +Limiting conversion size offers a practical way to avoid :cve:`2020-10735`. The limit is applied to the number of digit characters in the input or output string when a non-linear conversion algorithm would be involved. Underscores diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 262b785bbcbfc1..c3c0d732cf18d4 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -1,5 +1,5 @@ -:mod:`string` --- Common string operations -========================================== +:mod:`!string` --- Common string operations +=========================================== .. module:: string :synopsis: Common string operations. @@ -208,13 +208,13 @@ The grammar for a replacement field is as follows: .. productionlist:: format-string replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" - field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `digit`+] - attribute_name: `identifier` - element_index: `digit`+ | `index_string` + field_name: `arg_name` ("." `attribute_name` | "[" `element_index` "]")* + arg_name: [`~python-grammar:identifier` | `~python-grammar:digit`+] + attribute_name: `~python-grammar:identifier` + element_index: `~python-grammar:digit`+ | `index_string` index_string: <any source character except "]"> + conversion: "r" | "s" | "a" - format_spec: <described in the next section> + format_spec: `format-spec:format_spec` In less formal terms, the replacement field can start with a *field_name* that specifies the object whose value is to be formatted and inserted @@ -316,9 +316,9 @@ The general form of a *standard format specifier* is: fill: <any character> align: "<" | ">" | "=" | "^" sign: "+" | "-" | " " - width: `digit`+ + width: `~python-grammar:digit`+ grouping_option: "_" | "," - precision: `digit`+ + precision: `~python-grammar:digit`+ type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" If a valid *align* value is specified, it can be preceded by a *fill* diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index c6d78a356d97bc..37d5adf0fa9541 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -1,5 +1,5 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= +:mod:`!stringprep` --- Internet String Preparation +================================================== .. module:: stringprep :synopsis: String preparation, as per RFC 3453 diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index e2e6fc542e3e67..a2c293443e23d3 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,5 +1,5 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= +:mod:`!struct` --- Interpret bytes as packed binary data +======================================================== .. testsetup:: * @@ -160,6 +160,21 @@ following table: If the first character is not one of these, ``'@'`` is assumed. +.. note:: + + The number 1023 (``0x3ff`` in hexadecimal) has the following byte representations: + + * ``03 ff`` in big-endian (``>``) + * ``ff 03`` in little-endian (``<``) + + Python example: + + >>> import struct + >>> struct.pack('>h', 1023) + b'\x03\xff' + >>> struct.pack('<h', 1023) + b'\xff\x03' + Native byte order is big-endian or little-endian, depending on the host system. For example, Intel x86, AMD64 (x86-64), and Apple M1 are little-endian; IBM z and many legacy architectures are big-endian. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 7f22a5d1852a89..f520d989e0c70d 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1,5 +1,5 @@ -:mod:`subprocess` --- Subprocess management -=========================================== +:mod:`!subprocess` --- Subprocess management +============================================ .. module:: subprocess :synopsis: Subprocess management. @@ -25,7 +25,7 @@ modules and functions can be found in the following sections. :pep:`324` -- PEP proposing the subprocess module -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-ios-notavail.rst Using the :mod:`subprocess` Module ---------------------------------- @@ -52,10 +52,12 @@ underlying :class:`Popen` interface can be used directly. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with - ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. If you wish to capture - and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` - instead of *capture_output*. + *stdout* and *stderr* both set to :data:`~subprocess.PIPE`. + The *stdout* and *stderr* arguments may not be supplied at the same time as *capture_output*. + If you wish to capture and combine both streams into one, + set *stdout* to :data:`~subprocess.PIPE` + and *stderr* to :data:`~subprocess.STDOUT`, + instead of using *capture_output*. A *timeout* may be specified in seconds, it is internally passed on to :meth:`Popen.communicate`. If the timeout expires, the child process will be @@ -69,7 +71,8 @@ underlying :class:`Popen` interface can be used directly. subprocess's stdin. If used it must be a byte sequence, or a string if *encoding* or *errors* is specified or *text* is true. When used, the internal :class:`Popen` object is automatically created with - ``stdin=PIPE``, and the *stdin* argument may not be used as well. + *stdin* set to :data:`~subprocess.PIPE`, + and the *stdin* argument may not be used as well. If *check* is true, and the process exits with a non-zero exit code, a :exc:`CalledProcessError` exception will be raised. Attributes of that @@ -308,10 +311,10 @@ default values. The arguments that are most commonly needed are: If text mode is not used, *stdin*, *stdout* and *stderr* will be opened as binary streams. No encoding or line ending conversion is performed. - .. versionadded:: 3.6 - Added *encoding* and *errors* parameters. + .. versionchanged:: 3.6 + Added the *encoding* and *errors* parameters. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 Added the *text* parameter as an alias for *universal_newlines*. .. note:: @@ -664,7 +667,8 @@ functions. If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be one or more of the following flags: + + If given, *creationflags*, can be one or more of the following flags: * :data:`CREATE_NEW_CONSOLE` * :data:`CREATE_NEW_PROCESS_GROUP` @@ -684,8 +688,8 @@ functions. is only changed on platforms that support this (only Linux at this time of writing). Other platforms will ignore this parameter. - .. versionadded:: 3.10 - The ``pipesize`` parameter was added. + .. versionchanged:: 3.10 + Added the *pipesize* parameter. Popen objects are supported as context managers via the :keyword:`with` statement: on exit, standard file descriptors are closed, and the process is waited for. @@ -750,8 +754,8 @@ Exceptions defined in this module all inherit from :exc:`SubprocessError`. Security Considerations ----------------------- -Unlike some other popen functions, this implementation will never -implicitly call a system shell. This means that all characters, +Unlike some other popen functions, this library will not +implicitly choose to call a system shell. This means that all characters, including shell metacharacters, can safely be passed to child processes. If the shell is invoked explicitly, via ``shell=True``, it is the application's responsibility to ensure that all whitespace and metacharacters are @@ -760,6 +764,14 @@ quoted appropriately to avoid vulnerabilities. On :ref:`some platforms <shlex-quote-warning>`, it is possible to use :func:`shlex.quote` for this escaping. +On Windows, batch files (:file:`*.bat` or :file:`*.cmd`) may be launched by the +operating system in a system shell regardless of the arguments passed to this +library. This could result in arguments being parsed according to shell rules, +but without any escaping added by Python. If you are intentionally launching a +batch file with arguments from untrusted sources, consider passing +``shell=True`` to allow Python to escape special characters. See :gh:`114539` +for additional discussion. + Popen Objects ------------- @@ -791,9 +803,10 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. + When the ``timeout`` parameter is not ``None``, then (on POSIX) the + function is implemented using a busy loop (non-blocking call and short + sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. @@ -855,8 +868,8 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.terminate() - Stop the child. On POSIX OSs the method sends SIGTERM to the - child. On Windows the Win32 API function :c:func:`TerminateProcess` is called + Stop the child. On POSIX OSs the method sends :py:const:`~signal.SIGTERM` to the + child. On Windows the Win32 API function :c:func:`!TerminateProcess` is called to stop the child. @@ -1053,6 +1066,22 @@ The :mod:`subprocess` module exposes the following constants. Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains additional information. +.. data:: STARTF_FORCEONFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the + *Working in Background* mouse cursor will be displayed while a + process is launching. This is the default behavior for GUI + processes. + + .. versionadded:: 3.13 + +.. data:: STARTF_FORCEOFFFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the mouse + cursor will not be changed when launching a process. + + .. versionadded:: 3.13 + .. data:: CREATE_NEW_CONSOLE The new process has a new console, instead of inheriting its parent's @@ -1414,36 +1443,8 @@ Environment example:: -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) +Replacing :func:`os.popen` +^^^^^^^^^^^^^^^^^^^^^^^^^^ Return code handling translates as follows:: @@ -1460,44 +1461,6 @@ Return code handling translates as follows:: print("There were some errors") -Replacing functions from the :mod:`popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* The *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - Legacy Shell Invocation Functions --------------------------------- @@ -1537,8 +1500,8 @@ handling consistency are valid for these functions. as it did in Python 3.3.3 and earlier. exitcode has the same value as :attr:`~Popen.returncode`. - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. + .. versionchanged:: 3.11 + Added the *encoding* and *errors* parameters. .. function:: getoutput(cmd, *, encoding=None, errors=None) @@ -1555,8 +1518,8 @@ handling consistency are valid for these functions. .. versionchanged:: 3.3.4 Windows support added - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. + .. versionchanged:: 3.11 + Added the *encoding* and *errors* parameters. Notes diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 85eae5f3822575..aa5f8d95925ada 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -1,5 +1,5 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== +:mod:`!symtable` --- Access to the compiler's symbol tables +=========================================================== .. module:: symtable :synopsis: Interface to the compiler's internal symbol tables. @@ -31,21 +31,74 @@ Generating Symbol Tables Examining Symbol Tables ----------------------- +.. class:: SymbolTableType + + An enumeration indicating the type of a :class:`SymbolTable` object. + + .. attribute:: MODULE + :value: "module" + + Used for the symbol table of a module. + + .. attribute:: FUNCTION + :value: "function" + + Used for the symbol table of a function. + + .. attribute:: CLASS + :value: "class" + + Used for the symbol table of a class. + + The following members refer to different flavors of + :ref:`annotation scopes <annotation-scopes>`. + + .. attribute:: ANNOTATION + :value: "annotation" + + Used for annotations if ``from __future__ import annotations`` is active. + + .. attribute:: TYPE_ALIAS + :value: "type alias" + + Used for the symbol table of :keyword:`type` constructions. + + .. attribute:: TYPE_PARAMETERS + :value: "type parameters" + + Used for the symbol table of :ref:`generic functions <generic-functions>` + or :ref:`generic classes <generic-classes>`. + + .. attribute:: TYPE_VARIABLE + :value: "type variable" + + Used for the symbol table of the bound, the constraint tuple or the + default value of a single type variable in the formal sense, i.e., + a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two do + not support a bound or a constraint tuple). + + .. versionadded:: 3.13 + .. class:: SymbolTable A namespace table for a block. The constructor is not public. .. method:: get_type() - Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, ``'function'``, ``'annotation'``, ``'TypeVar bound'``, - ``'type alias'``, and ``'type parameter'``. The latter four refer to - different flavors of :ref:`annotation scopes <annotation-scopes>`. + Return the type of the symbol table. Possible values are members + of the :class:`SymbolTableType` enumeration. .. versionchanged:: 3.12 Added ``'annotation'``, ``'TypeVar bound'``, ``'type alias'``, and ``'type parameter'`` as possible return values. + .. versionchanged:: 3.13 + Return values are members of the :class:`SymbolTableType` enumeration. + + The exact values of the returned string may change in the future, + and thus, it is recommended to use :class:`SymbolTableType` members + instead of hard-coded strings. + .. method:: get_id() Return the table's identifier. @@ -97,7 +150,7 @@ Examining Symbol Tables .. class:: Function - A namespace for a function or method. This class inherits + A namespace for a function or method. This class inherits from :class:`SymbolTable`. .. method:: get_parameters() @@ -123,12 +176,43 @@ Examining Symbol Tables .. class:: Class - A namespace of a class. This class inherits :class:`SymbolTable`. + A namespace of a class. This class inherits from :class:`SymbolTable`. .. method:: get_methods() - Return a tuple containing the names of methods declared in the class. - + Return a tuple containing the names of method-like functions declared + in the class. + + Here, the term 'method' designates *any* function defined in the class + body via :keyword:`def` or :keyword:`async def`. + + Functions defined in a deeper scope (e.g., in an inner class) are not + picked up by :meth:`get_methods`. + + For example: + + >>> import symtable + >>> st = symtable.symtable(''' + ... def outer(): pass + ... + ... class A: + ... def f(): + ... def w(): pass + ... + ... def g(self): pass + ... + ... @classmethod + ... async def h(cls): pass + ... + ... global outer + ... def outer(self): pass + ... ''', 'test', 'exec') + >>> class_A = st.get_children()[2] + >>> class_A.get_methods() + ('f', 'g', 'h') + + Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still + considered as a method-like function. .. class:: Symbol @@ -151,6 +235,12 @@ Examining Symbol Tables Return ``True`` if the symbol is a parameter. + .. method:: is_type_parameter() + + Return ``True`` if the symbol is a type parameter. + + .. versionadded:: 3.14 + .. method:: is_global() Return ``True`` if the symbol is global. @@ -178,10 +268,42 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_free_class() + + Return *True* if a class-scoped symbol is free from + the perspective of a method. + + Consider the following example:: + + def f(): + x = 1 # function-scoped + class C: + x = 2 # class-scoped + def method(self): + return x + + In this example, the class-scoped symbol ``x`` is considered to + be free from the perspective of ``C.method``, thereby allowing + the latter to return *1* at runtime and not *2*. + + .. versionadded:: 3.14 + .. method:: is_assigned() Return ``True`` if the symbol is assigned to in its block. + .. method:: is_comp_iter() + + Return ``True`` if the symbol is a comprehension iteration variable. + + .. versionadded:: 3.14 + + .. method:: is_comp_cell() + + Return ``True`` if the symbol is a cell in an inlined comprehension. + + .. versionadded:: 3.14 + .. method:: is_namespace() Return ``True`` if name binding introduces new namespace. @@ -207,3 +329,21 @@ Examining Symbol Tables Return the namespace bound to this name. If more than one or no namespace is bound to this name, a :exc:`ValueError` is raised. + + +.. _symtable-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`symtable` module can be executed as a script from the command line. + +.. code-block:: sh + + python -m symtable [infile...] + +Symbol tables are generated for the specified Python source files and +dumped to stdout. +If no input file is specified, the content is read from stdin. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 5dcbdaf8e5d0e4..0fa06da522049f 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -1,14 +1,16 @@ -:mod:`sys.monitoring` --- Execution event monitoring -==================================================== +:mod:`!sys.monitoring` --- Execution event monitoring +===================================================== -.. module:: sys.monitoring - :synopsis: Access and control event monitoring +.. module:: sys.monitoring + :synopsis: Access and control event monitoring + +.. versionadded:: 3.12 ----------------- .. note:: - ``sys.monitoring`` is a namespace within the ``sys`` module, + :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, not an independent module, so there is no need to ``import sys.monitoring``, simply ``import sys`` and then use ``sys.monitoring``. @@ -18,45 +20,52 @@ This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`!sys.monitoring` namespace provides means to +monitor execution. The :mod:`sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: -* Tool identifiers -* Events -* Callbacks +* `Tool identifiers`_ +* `Events`_ +* :ref:`Callbacks <callbacks>` Tool identifiers ---------------- -A tool identifier is an integer and associated name. +A tool identifier is an integer and the associated name. Tool identifiers are used to discourage tools from interfering with each other and to allow multiple tools to operate at the same time. Currently tools are completely independent and cannot be used to monitor each other. This restriction may be lifted in the future. Before registering or activating events, a tool should choose an identifier. -Identifiers are integers in the range 0 to 5. +Identifiers are integers in the range 0 to 5 inclusive. Registering and using tools ''''''''''''''''''''''''''' -.. function:: use_tool_id(id: int, name: str) -> None +.. function:: use_tool_id(tool_id: int, name: str, /) -> None + + Must be called before *tool_id* can be used. + *tool_id* must be in the range 0 to 5 inclusive. + Raises a :exc:`ValueError` if *tool_id* is in use. - Must be called before ``id`` can be used. - ``id`` must be in the range 0 to 5 inclusive. - Raises a ``ValueError`` if ``id`` is in use. +.. function:: free_tool_id(tool_id: int, /) -> None -.. function:: free_tool_id(id: int) -> None + Should be called once a tool no longer requires *tool_id*. + +.. note:: - Should be called once a tool no longer requires ``id``. + :func:`free_tool_id` will not disable global or local events associated + with *tool_id*, nor will it unregister any callback functions. This + function is only intended to be used to notify the VM that the + particular *tool_id* is no longer in use. -.. function:: get_tool(id: int) -> str | None +.. function:: get_tool(tool_id: int, /) -> str | None - Returns the name of the tool if ``id`` is in use, + Returns the name of the tool if *tool_id* is in use, otherwise it returns ``None``. - ``id`` must be in the range 0 to 5 inclusive. + *tool_id* must be in the range 0 to 5 inclusive. All IDs are treated the same by the VM with regard to events, but the following IDs are pre-defined to make co-operation of tools easier:: @@ -66,57 +75,95 @@ following IDs are pre-defined to make co-operation of tools easier:: sys.monitoring.PROFILER_ID = 2 sys.monitoring.OPTIMIZER_ID = 5 -There is no obligation to set an ID, nor is there anything preventing a tool -from using an ID even it is already in use. -However, tools are encouraged to use a unique ID and respect other tools. Events ------ The following events are supported: -BRANCH - A conditional branch is taken (or not). -CALL - A call in Python code (event occurs before the call). -C_RAISE - Exception raised from any callable, except Python functions (event occurs after the exit). -C_RETURN - Return from any callable, except Python functions (event occurs after the return). -EXCEPTION_HANDLED - An exception is handled. -INSTRUCTION - A VM instruction is about to be executed. -JUMP - An unconditional jump in the control flow graph is made. -LINE - An instruction is about to be executed that has a different line number from the preceding instruction. -PY_RESUME - Resumption of a Python function (for generator and coroutine functions), except for throw() calls. -PY_RETURN - Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). -PY_START - Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) -PY_THROW - A Python function is resumed by a throw() call. -PY_UNWIND - Exit from a Python function during exception unwinding. -PY_YIELD - Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). -RAISE - An exception is raised, except those that cause a ``STOP_ITERATION`` event. -RERAISE - An exception is re-raised, for example at the end of a ``finally`` block. -STOP_ITERATION - An artificial ``StopIteration`` is raised; see `the STOP_ITERATION event`_. +.. monitoring-event:: BRANCH + + A conditional branch is taken (or not). + +.. monitoring-event:: CALL + + A call in Python code (event occurs before the call). + +.. monitoring-event:: C_RAISE + + An exception raised from any callable, except for Python functions (event occurs after the exit). + +.. monitoring-event:: C_RETURN + + Return from any callable, except for Python functions (event occurs after the return). + +.. monitoring-event:: EXCEPTION_HANDLED + + An exception is handled. + +.. monitoring-event:: INSTRUCTION + + A VM instruction is about to be executed. + +.. monitoring-event:: JUMP + + An unconditional jump in the control flow graph is made. + +.. monitoring-event:: LINE + + An instruction is about to be executed that has a different line number from the preceding instruction. + +.. monitoring-event:: PY_RESUME + + Resumption of a Python function (for generator and coroutine functions), except for ``throw()`` calls. + +.. monitoring-event:: PY_RETURN + + Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). + +.. monitoring-event:: PY_START + + Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) + +.. monitoring-event:: PY_THROW + + A Python function is resumed by a ``throw()`` call. + +.. monitoring-event:: PY_UNWIND + + Exit from a Python function during exception unwinding. + +.. monitoring-event:: PY_YIELD + + Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). + +.. monitoring-event:: RAISE + + An exception is raised, except those that cause a :monitoring-event:`STOP_ITERATION` event. + +.. monitoring-event:: RERAISE + + An exception is re-raised, for example at the end of a :keyword:`finally` block. + +.. monitoring-event:: STOP_ITERATION + + An artificial :exc:`StopIteration` is raised; see `the STOP_ITERATION event`_. + More events may be added in the future. These events are attributes of the :mod:`!sys.monitoring.events` namespace. Each event is represented as a power-of-2 integer constant. To define a set of events, simply bitwise or the individual events together. -For example, to specify both ``PY_RETURN`` and ``PY_START`` events, use the -expression ``PY_RETURN | PY_START``. +For example, to specify both :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_START` +events, use the expression ``PY_RETURN | PY_START``. + +.. monitoring-event:: NO_EVENTS + + An alias for ``0`` so users can do explicit comparisons like:: + + if get_events(DEBUGGER_ID) == NO_EVENTS: + ... Events are divided into three groups: @@ -127,16 +174,16 @@ Local events are associated with normal execution of the program and happen at clearly defined locations. All local events can be disabled. The local events are: -* PY_START -* PY_RESUME -* PY_RETURN -* PY_YIELD -* CALL -* LINE -* INSTRUCTION -* JUMP -* BRANCH -* STOP_ITERATION +* :monitoring-event:`PY_START` +* :monitoring-event:`PY_RESUME` +* :monitoring-event:`PY_RETURN` +* :monitoring-event:`PY_YIELD` +* :monitoring-event:`CALL` +* :monitoring-event:`LINE` +* :monitoring-event:`INSTRUCTION` +* :monitoring-event:`JUMP` +* :monitoring-event:`BRANCH` +* :monitoring-event:`STOP_ITERATION` Ancillary events '''''''''''''''' @@ -144,12 +191,13 @@ Ancillary events Ancillary events can be monitored like other events, but are controlled by another event: -* C_RAISE -* C_RETURN +* :monitoring-event:`C_RAISE` +* :monitoring-event:`C_RETURN` -The ``C_RETURN`` and ``C_RAISE`` events are controlled by the ``CALL`` -event. ``C_RETURN`` and ``C_RAISE`` events will only be seen if the -corresponding ``CALL`` event is being monitored. +The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events +are controlled by the :monitoring-event:`CALL` event. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the +corresponding :monitoring-event:`CALL` event is being monitored. Other events '''''''''''' @@ -159,30 +207,31 @@ program and cannot be individually disabled. The other events that can be monitored are: -* PY_THROW -* PY_UNWIND -* RAISE -* EXCEPTION_HANDLED +* :monitoring-event:`PY_THROW` +* :monitoring-event:`PY_UNWIND` +* :monitoring-event:`RAISE` +* :monitoring-event:`EXCEPTION_HANDLED` The STOP_ITERATION event '''''''''''''''''''''''' :pep:`PEP 380 <380#use-of-stopiteration-to-return-values>` -specifies that a ``StopIteration`` exception is raised when returning a value +specifies that a :exc:`StopIteration` exception is raised when returning a value from a generator or coroutine. However, this is a very inefficient way to return a value, so some Python implementations, notably CPython 3.12+, do not raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators -and coroutines, the ``STOP_ITERATION`` event is provided. -``STOP_ITERATION`` can be locally disabled, unlike ``RAISE``. +and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. Turning events on and off ------------------------- -In order to monitor an event, it must be turned on and a callback registered. +In order to monitor an event, it must be turned on and a corresponding callback +must be registered. Events can be turned on or off by setting the events either globally or for a particular code object. @@ -192,30 +241,33 @@ Setting events globally Events can be controlled globally by modifying the set of events being monitored. -.. function:: get_events(tool_id: int) -> int +.. function:: get_events(tool_id: int, /) -> int Returns the ``int`` representing all the active events. -.. function:: set_events(tool_id: int, event_set: int) +.. function:: set_events(tool_id: int, event_set: int, /) -> None - Activates all events which are set in ``event_set``. - Raises a ``ValueError`` if ``tool_id`` is not in use. + Activates all events which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. No events are active by default. Per code object events '''''''''''''''''''''' -Events can also be controlled on a per code object basis. +Events can also be controlled on a per code object basis. The functions +defined below which accept a :class:`types.CodeType` should be prepared +to accept a look-alike object from functions which are not defined +in Python (see :ref:`monitoring`). -.. function:: get_local_events(tool_id: int, code: CodeType) -> int +.. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for ``code`` + Returns all the local events for *code* -.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int) +.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for ``code`` which are set in ``event_set``. - Raises a ``ValueError`` if ``tool_id`` is not in use. + Activates all the local events for *code* which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. Local events add to global events, but do not mask them. In other words, all global events will trigger for a code object, @@ -225,8 +277,13 @@ regardless of the local events. Disabling events '''''''''''''''' +.. data:: DISABLE + + A special value that can be returned from a callback function to disable + events for the current code location. + Local events can be disabled for a specific code location by returning -``sys.monitoring.DISABLE`` from a callback function. This does not change +:data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. Disabling events for specific locations is very important for high @@ -234,19 +291,26 @@ performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +.. function:: restart_events() -> None + + Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` + for all tools. + + +.. _callbacks: Registering callback functions ------------------------------ To register a callable for events call -.. function:: register_callback(tool_id: int, event: int, func: Callable | None) -> Callable | None +.. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None - Registers the callable ``func`` for the ``event`` with the given ``tool_id`` + Registers the callable *func* for the *event* with the given *tool_id* - If another callback was registered for the given ``tool_id`` and ``event``, + If another callback was registered for the given *tool_id* and *event*, it is unregistered and returned. - Otherwise ``register_callback`` returns ``None``. + Otherwise :func:`register_callback` returns ``None``. Functions can be unregistered by calling @@ -254,47 +318,51 @@ Functions can be unregistered by calling Callback functions can be registered and unregistered at any time. -Registering or unregistering a callback function will generate a ``sys.audit`` event. +Registering or unregistering a callback function will generate a :func:`sys.audit` event. Callback function arguments ''''''''''''''''''''''''''' +.. data:: MISSING + + A special value that is passed to a callback function to indicate + that there are no arguments to the call. + When an active event occurs, the registered callback function is called. Different events will provide the callback function with different arguments, as follows: -* ``PY_START`` and ``PY_RESUME``:: +* :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: func(code: CodeType, instruction_offset: int) -> DISABLE | Any -* ``PY_RETURN`` and ``PY_YIELD``: +* :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - ``func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any -* ``CALL``, ``C_RAISE`` and ``C_RETURN``: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: - ``func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any - If there are no arguments, ``arg0`` is set to ``MISSING``. + If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. -* ``RAISE``, ``RERAISE``, ``EXCEPTION_HANDLED``, ``PY_UNWIND``, ``PY_THROW`` and ``STOP_ITERATION``: +* :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, + :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - ``func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any -* ``LINE``: +* :monitoring-event:`LINE`:: - ``func(code: CodeType, line_number: int) -> DISABLE | Any`` + func(code: CodeType, line_number: int) -> DISABLE | Any -* ``BRANCH`` and ``JUMP``: +* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: - ``func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any - Note that the ``destination_offset`` is where the code will next execute. + Note that the *destination_offset* is where the code will next execute. For an untaken branch this will be the offset of the instruction following the branch. -* ``INSTRUCTION``: - - ``func(code: CodeType, instruction_offset: int) -> DISABLE | Any`` - +* :monitoring-event:`INSTRUCTION`:: + func(code: CodeType, instruction_offset: int) -> DISABLE | Any diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f9f556306f5827..ed809d04167ffd 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1,5 +1,5 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= +:mod:`!sys` --- System-specific parameters and functions +======================================================== .. module:: sys :synopsis: Access system-specific parameters and functions. @@ -16,12 +16,12 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionadded:: 3.2 + .. versionchanged:: 3.8 Default flags became an empty string (``m`` flag for pymalloc has been removed). - .. versionadded:: 3.2 - .. availability:: Unix. @@ -175,7 +175,11 @@ always available. Call ``func(*args)``, while tracing is enabled. The tracing state is saved, and restored afterwards. This is intended to be called from a debugger from - a checkpoint, to recursively debug some other code. + a checkpoint, to recursively debug or profile some other code. + + Tracing is suspended while calling a tracing function set by + :func:`settrace` or :func:`setprofile` to avoid infinite recursion. + :func:`!call_tracing` enables explicit recursion of the tracing function. .. data:: copyright @@ -191,6 +195,17 @@ always available. This function should be used for internal and specialized purposes only. + .. deprecated:: 3.13 + Use the more general :func:`_clear_internal_caches` function instead. + + +.. function:: _clear_internal_caches() + + Clear all internal performance-related caches. Use this function *only* to + release unnecessary references and memory blocks when hunting for leaks. + + .. versionadded:: 3.13 + .. function:: _current_frames() @@ -720,7 +735,7 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more + :func:`_clear_internal_caches()` and :func:`gc.collect()` to get more predictable results. If a Python build or implementation cannot reasonably compute this @@ -738,7 +753,9 @@ always available. .. function:: getandroidapilevel() - Return the build time API version of Android as an integer. + Return the build-time API level of Android as an integer. This represents the + minimum version of Android this build of Python can run on. For runtime + version information, see :func:`platform.android_ver`. .. availability:: Android. @@ -827,7 +844,7 @@ always available. Note that the returned value may not actually reflect how many references to the object are actually held. For example, some - objects are "immortal" and have a very high refcount that does not + objects are :term:`immortal` and have a very high refcount that does not reflect the actual number of references. Consequently, do not rely on the returned value to be accurate, other than a value of 0 or 1. @@ -860,7 +877,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe <https://code.activestate.com/recipes/577504/>`_ + See `recursive sizeof recipe <https://code.activestate.com/recipes/577504-compute-memory-footprint-of-an-object-and-its-cont/>`_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. @@ -1178,8 +1195,16 @@ always available. names used in Python programs are automatically interned, and the dictionaries used to hold module, class or instance attributes have interned keys. - Interned strings are not immortal; you must keep a reference to the return - value of :func:`intern` around to benefit from it. + Interned strings are not :term:`immortal`; you must keep a reference to the + return value of :func:`intern` around to benefit from it. + + +.. function:: _is_gil_enabled() + + Return :const:`True` if the :term:`GIL` is enabled and :const:`False` if + it is disabled. + + .. versionadded:: 3.13 .. function:: is_finalizing() @@ -1187,6 +1212,8 @@ always available. Return :const:`True` if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. Return :const:`False` otherwise. + See also the :exc:`PythonFinalizationError` exception. + .. versionadded:: 3.5 .. data:: last_exc @@ -1201,6 +1228,18 @@ always available. .. versionadded:: 3.12 +.. function:: _is_interned(string) + + Return :const:`True` if the given string is "interned", :const:`False` + otherwise. + + .. versionadded:: 3.13 + + .. impl-detail:: + + It is not guaranteed to exist in all implementations of Python. + + .. data:: last_type last_value last_traceback @@ -1252,10 +1291,13 @@ always available. .. versionchanged:: 3.4 :term:`Module specs <module spec>` were introduced in Python 3.4, by - :pep:`451`. Earlier versions of Python looked for a method called - :meth:`!find_module`. - This is still called as a fallback if a :data:`meta_path` entry doesn't - have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. + :pep:`451`. + + .. versionchanged:: 3.12 + + Removed the fallback that looked for a :meth:`!find_module` method + if a :data:`meta_path` entry didn't have a + :meth:`~importlib.abc.MetaPathFinder.find_spec` method. .. data:: modules @@ -1274,7 +1316,10 @@ always available. The list of the original command line arguments passed to the Python executable. - See also :data:`sys.argv`. + The elements of :data:`sys.orig_argv` are the arguments to the Python interpreter, + while the elements of :data:`sys.argv` are the arguments to the user's program. + Arguments consumed by the interpreter itself will be present in :data:`sys.orig_argv` + and missing from :data:`sys.argv`. .. versionadded:: 3.10 @@ -1332,47 +1377,42 @@ always available. .. data:: platform - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: + A string containing a platform identifier. Known values are: ================ =========================== System ``platform`` value ================ =========================== AIX ``'aix'`` + Android ``'android'`` Emscripten ``'emscripten'`` + iOS ``'ios'`` Linux ``'linux'`` - WASI ``'wasi'`` + macOS ``'darwin'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` + WASI ``'wasi'`` ================ =========================== + On Unix systems not listed in the table, the value is the lowercased OS name + as returned by ``uname -s``, with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time + when Python was built*. Unless you want to test for a specific system + version, it is therefore recommended to use the following idiom:: + + if sys.platform.startswith('freebsd'): + # FreeBSD-specific code here... + .. versionchanged:: 3.3 On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. .. versionchanged:: 3.8 On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. + + .. versionchanged:: 3.13 + On Android, :data:`sys.platform` now returns ``'android'`` rather than + ``'linux'``. .. seealso:: @@ -1473,13 +1513,16 @@ always available. its return value is not used, so it can simply return ``None``. Error in the profile function will cause itself unset. + .. note:: + The same tracing mechanism is used for :func:`!setprofile` as :func:`settrace`. + To trace calls with :func:`!setprofile` inside a tracing function + (e.g. in a debugger breakpoint), see :func:`call_tracing`. + Profile functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. - .. audit-event:: sys.setprofile "" sys.setprofile - The events have the following meaning: ``'call'`` @@ -1501,6 +1544,9 @@ always available. ``'c_exception'`` A C function has raised an exception. *arg* is the C function object. + .. audit-event:: sys.setprofile "" sys.setprofile + + .. function:: setrecursionlimit(limit) Set the maximum depth of the Python interpreter stack to *limit*. This limit @@ -1554,13 +1600,16 @@ always available. function to be used for the new scope, or ``None`` if the scope shouldn't be traced. - The local trace function should return a reference to itself (or to another - function for further tracing in that scope), or ``None`` to turn off tracing - in that scope. + The local trace function should return a reference to itself, or to another + function which would then be used as the local trace function for the scope. If there is any error occurred in the trace function, it will be unset, just like ``settrace(None)`` is called. + .. note:: + Tracing is disabled while calling the trace function (e.g. a function set by + :func:`!settrace`). For recursive tracing see :func:`call_tracing`. + The events have the following meaning: ``'call'`` @@ -1575,7 +1624,8 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame <frame-objects>`. + :attr:`~frame.f_trace_lines` to :const:`False` on that + :ref:`frame <frame-objects>`. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1593,7 +1643,7 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + requested by setting :attr:`~frame.f_trace_opcodes` to :const:`True` on the :ref:`frame <frame-objects>`. Note that as an exception is propagated down the chain of callers, an @@ -1623,10 +1673,10 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`!f_trace_lines` and - :attr:`!f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`~frame.f_trace_lines` and + :attr:`~frame.f_trace_opcodes` attributes added to frames -.. function:: set_asyncgen_hooks(firstiter, finalizer) +.. function:: set_asyncgen_hooks([firstiter] [, finalizer]) Accepts two optional keyword arguments which are callables that accept an :term:`asynchronous generator iterator` as an argument. The *firstiter* @@ -1658,7 +1708,7 @@ always available. contain a tuple of (filename, line number, function name) tuples describing the traceback where the coroutine object was created, with the most recent call first. When disabled, ``cr_origin`` will - be None. + be ``None``. To enable, pass a *depth* value greater than zero; this sets the number of frames whose information will be captured. To disable, @@ -1718,9 +1768,17 @@ always available. .. availability:: Windows. + .. note:: + Changing the filesystem encoding after Python startup is risky because + the old fsencoding or paths encoded by the old fsencoding may be cached + somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. versionadded:: 3.6 See :pep:`529` for more details. + .. deprecated-removed:: 3.13 3.16 + Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. data:: stdin stdout stderr @@ -1781,7 +1839,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:!buffer` attribute. + do not support the :attr:`!buffer` attribute. .. data:: __stdin__ diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 905abc3a7c9f9b..9556da808f8c63 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -1,5 +1,5 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= +:mod:`!sysconfig` --- Provide access to Python's configuration information +========================================================================== .. module:: sysconfig :synopsis: Python's configuration information @@ -9,7 +9,7 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/sysconfig.py` +**Source code:** :source:`Lib/sysconfig` .. index:: single: configuration information diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index f29ef03267b1ba..548898a37bc6ea 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -1,5 +1,5 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== +:mod:`!syslog` --- Unix syslog library routines +=============================================== .. module:: syslog :platform: Unix @@ -11,11 +11,11 @@ This module provides an interface to the Unix ``syslog`` library routines. Refer to the Unix manual pages for a detailed description of the ``syslog`` facility. +.. availability:: Unix, not WASI, not iOS. + This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the -:mod:`logging.handlers` module as :class:`SysLogHandler`. - -.. include:: ../includes/wasm-notavail.rst +:mod:`logging.handlers` module as :class:`~logging.handlers.SysLogHandler`. The module defines the following functions: @@ -107,22 +107,62 @@ The module defines the following functions: The module defines the following constants: -Priority levels (high to low): - :const:`LOG_EMERG`, :const:`LOG_ALERT`, :const:`LOG_CRIT`, :const:`LOG_ERR`, - :const:`LOG_WARNING`, :const:`LOG_NOTICE`, :const:`LOG_INFO`, - :const:`LOG_DEBUG`. - -Facilities: - :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, - :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ``<syslog.h>``, - :const:`LOG_AUTHPRIV`. - -Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ``<syslog.h>``, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. + +.. data:: LOG_EMERG + LOG_ALERT + LOG_CRIT + LOG_ERR + LOG_WARNING + LOG_NOTICE + LOG_INFO + LOG_DEBUG + + Priority levels (high to low). + + +.. data:: LOG_AUTH + LOG_AUTHPRIV + LOG_CRON + LOG_DAEMON + LOG_FTP + LOG_INSTALL + LOG_KERN + LOG_LAUNCHD + LOG_LPR + LOG_MAIL + LOG_NETINFO + LOG_NEWS + LOG_RAS + LOG_REMOTEAUTH + LOG_SYSLOG + LOG_USER + LOG_UUCP + LOG_LOCAL0 + LOG_LOCAL1 + LOG_LOCAL2 + LOG_LOCAL3 + LOG_LOCAL4 + LOG_LOCAL5 + LOG_LOCAL6 + LOG_LOCAL7 + + Facilities, depending on availability in ``<syslog.h>`` for :const:`LOG_AUTHPRIV`, + :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL` and :const:`LOG_RAS`. + + .. versionchanged:: 3.13 + Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD`. + +.. data:: LOG_PID + LOG_CONS + LOG_NDELAY + LOG_ODELAY + LOG_NOWAIT + LOG_PERROR + + Log options, depending on availability in ``<syslog.h>`` for + :const:`LOG_ODELAY`, :const:`LOG_NOWAIT` and :const:`LOG_PERROR`. Examples diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index dfe688a2f93e0c..4f61b3dd761400 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -1,5 +1,5 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== +:mod:`!tabnanny` --- Detection of ambiguous indentation +======================================================= .. module:: tabnanny :synopsis: Tool for detecting white space related problems in Python diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 3e5723a66780ca..5b624f3533136f 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1,5 +1,5 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== +:mod:`!tarfile` --- Read and write tar archive files +==================================================== .. module:: tarfile :synopsis: Read and write tar-format archive files. @@ -116,10 +116,12 @@ Some facts and figures: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will be done on the file. If given, *fileobj* may be any object that has a - :meth:`read` or :meth:`write` method (depending on the *mode*). *bufsize* - specifies the blocksize and defaults to ``20 * 512`` bytes. Use this variant - in combination with e.g. ``sys.stdin``, a socket :term:`file object` or a tape - device. However, such a :class:`TarFile` object is limited in that it does + :meth:`~io.RawIOBase.read` or :meth:`~io.RawIOBase.write` method + (depending on the *mode*) that works with bytes. + *bufsize* specifies the blocksize and defaults to ``20 * 512`` bytes. + Use this variant in combination with e.g. ``sys.stdin.buffer``, a socket + :term:`file object` or a tape device. + However, such a :class:`TarFile` object is limited in that it does not allow random access, see :ref:`tar-examples`. The currently possible modes: @@ -255,6 +257,51 @@ The following constants are available at the module level: The default character encoding: ``'utf-8'`` on Windows, the value returned by :func:`sys.getfilesystemencoding` otherwise. +.. data:: REGTYPE + AREGTYPE + + A regular file :attr:`~TarInfo.type`. + +.. data:: LNKTYPE + + A link (inside tarfile) :attr:`~TarInfo.type`. + +.. data:: SYMTYPE + + A symbolic link :attr:`~TarInfo.type`. + +.. data:: CHRTYPE + + A character special device :attr:`~TarInfo.type`. + +.. data:: BLKTYPE + + A block special device :attr:`~TarInfo.type`. + +.. data:: DIRTYPE + + A directory :attr:`~TarInfo.type`. + +.. data:: FIFOTYPE + + A FIFO special device :attr:`~TarInfo.type`. + +.. data:: CONTTYPE + + A contiguous file :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_LONGNAME + + A GNU tar longname :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_LONGLINK + + A GNU tar longlink :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_SPARSE + + A GNU tar sparse file :attr:`~TarInfo.type`. + Each of the following constants defines a tar archive format that the :mod:`tarfile` module is able to create. See section :ref:`tar-formats` for @@ -325,7 +372,7 @@ be finalized; only the internally used file object will be closed. See the *name* is the pathname of the archive. *name* may be a :term:`path-like object`. It can be omitted if *fileobj* is given. - In this case, the file object's :attr:`name` attribute is used if it exists. + In this case, the file object's :attr:`!name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append data to an existing file, ``'w'`` to create a new file overwriting an existing @@ -359,7 +406,7 @@ be finalized; only the internally used file object will be closed. See the messages). The messages are written to ``sys.stderr``. *errorlevel* controls how extraction errors are handled, - see :attr:`the corresponding attribute <~TarFile.errorlevel>`. + see :attr:`the corresponding attribute <TarFile.errorlevel>`. The *encoding* and *errors* arguments define the character encoding to be used for reading or writing the archive and how conversion errors are going @@ -518,6 +565,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. + .. versionchanged:: 3.13 + The returned :class:`io.BufferedReader` object has the :attr:`!mode` + attribute which is always equal to ``'rb'``. + .. attribute:: TarFile.errorlevel :type: int @@ -590,11 +641,15 @@ be finalized; only the internally used file object will be closed. See the .. method:: TarFile.addfile(tarinfo, fileobj=None) - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can + Add the :class:`TarInfo` object *tarinfo* to the archive. If *tarinfo* represents + a non zero-size regular file, the *fileobj* argument should be a :term:`binary file`, + and ``tarinfo.size`` bytes are read from it and added to the archive. You can create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. + .. versionchanged:: 3.13 + + *fileobj* must be given for non-zero-sized regular files. + .. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) @@ -626,6 +681,7 @@ be finalized; only the internally used file object will be closed. See the .. attribute:: TarFile.pax_headers + :type: dict A dictionary containing key-value pairs of pax global headers. @@ -645,8 +701,8 @@ It does *not* contain the file's data itself. :meth:`~TarFile.getmember`, :meth:`~TarFile.getmembers` and :meth:`~TarFile.gettarinfo`. -Modifying the objects returned by :meth:`~!TarFile.getmember` or -:meth:`~!TarFile.getmembers` will affect all subsequent +Modifying the objects returned by :meth:`~TarFile.getmember` or +:meth:`~TarFile.getmembers` will affect all subsequent operations on the archive. For cases where this is unwanted, you can use :mod:`copy.copy() <copy>` or call the :meth:`~TarInfo.replace` method to create a modified copy in one step. @@ -790,13 +846,48 @@ A ``TarInfo`` object has the following public data attributes: :meth:`~TarFile.extractall`, causing extraction to skip applying this attribute. +.. attribute:: TarInfo.chksum + :type: int + + Header checksum. + + +.. attribute:: TarInfo.devmajor + :type: int + + Device major number. + + +.. attribute:: TarInfo.devminor + :type: int + + Device minor number. + + +.. attribute:: TarInfo.offset + :type: int + + The tar header starts here. + + +.. attribute:: TarInfo.offset_data + :type: int + + The file's data starts here. + + +.. attribute:: TarInfo.sparse + + Sparse member information. + + .. attribute:: TarInfo.pax_headers :type: dict A dictionary containing key-value pairs of an associated pax extended header. -.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., - uid=..., gid=..., uname=..., gname=..., +.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., \ + uid=..., gid=..., uname=..., gname=..., \ deep=True) .. versionadded:: 3.12 @@ -816,7 +907,7 @@ A :class:`TarInfo` object also provides some convenient query methods: .. method:: TarInfo.isfile() - Return :const:`True` if the :class:`Tarinfo` object is a regular file. + Return :const:`True` if the :class:`TarInfo` object is a regular file. .. method:: TarInfo.isreg() @@ -952,7 +1043,7 @@ reused in custom filters: path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. - Clear high mode bits (setuid, setgid, sticky) and group/other write bits - (:const:`~stat.S_IWGRP`|:const:`~stat.S_IWOTH`). + (:const:`~stat.S_IWGRP` | :const:`~stat.S_IWOTH`). Return the modified ``TarInfo`` member. @@ -977,9 +1068,9 @@ reused in custom filters: - For regular files, including hard links: - Set the owner read and write permissions - (:const:`~stat.S_IRUSR`|:const:`~stat.S_IWUSR`). + (:const:`~stat.S_IRUSR` | :const:`~stat.S_IWUSR`). - Remove the group & other executable permission - (:const:`~stat.S_IXGRP`|:const:`~stat.S_IXOTH`) + (:const:`~stat.S_IXGRP` | :const:`~stat.S_IXOTH`) if the owner doesn’t have it (:const:`~stat.S_IXUSR`). - For other files (directories), set ``mode`` to ``None``, so diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 42314648fec2bd..f0a81a093b435b 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -1,5 +1,5 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ +:mod:`!tempfile` --- Generate temporary files and directories +============================================================= .. module:: tempfile :synopsis: Generate temporary files and directories. @@ -18,7 +18,7 @@ This module creates temporary files and directories. It works on all supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, :class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and +:term:`context managers <context manager>`. :func:`mkstemp` and :func:`mkdtemp` are lower-level functions which require manual cleanup. All the user-callable functions and constructors take additional arguments which @@ -41,7 +41,7 @@ The module defines the following user-callable items: this; your code should not rely on a temporary file created using this function having or not having a visible name in the file system. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the file object the temporary file will be removed from the filesystem. @@ -87,9 +87,9 @@ The module defines the following user-callable items: determine whether and how the named file should be automatically deleted. The returned object is always a :term:`file-like object` whose :attr:`!file` - attribute is the underlying true file object. This :term:`file-like object` + attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. The - name of the temporary file can be retrieved from the :attr:`name` attribute + name of the temporary file can be retrieved from the :attr:`!name` attribute of the returned file-like object. On Unix, unlike with the :func:`TemporaryFile`, the directory entry does not get unlinked immediately after the file creation. @@ -147,22 +147,24 @@ The module defines the following user-callable items: This class operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the + until the file's :func:`~io.IOBase.fileno` method is called, at which point the contents are written to disk and operation proceeds as with :func:`TemporaryFile`. - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. + .. method:: SpooledTemporaryFile.rollover - The returned object is a file-like object whose :attr:`_file` attribute + The resulting file has one additional method, :meth:`!rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`!_file` attribute is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This + object, depending on whether :meth:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. + the truncate method now accepts a *size* argument. .. versionchanged:: 3.8 Added *errors* parameter. @@ -176,24 +178,28 @@ The module defines the following user-callable items: .. class:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False, *, delete=True) This class securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the temporary directory object, the newly created temporary directory and all its contents are removed from the filesystem. - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. + .. attribute:: TemporaryDirectory.name + + The directory name can be retrieved from the :attr:`!name` attribute of the + returned object. When the returned object is used as a :term:`context manager`, the + :attr:`!name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled - exceptions during explicit or implicit cleanup (such as a - :exc:`PermissionError` removing open files on Windows) will be ignored, - and the remaining removable items deleted on a "best-effort" basis. - Otherwise, errors will be raised in whatever context cleanup occurs - (the :func:`cleanup` call, exiting the context manager, when the object - is garbage-collected or during interpreter shutdown). + .. method:: TemporaryDirectory.cleanup + + The directory can be explicitly cleaned up by calling the + :meth:`!cleanup` method. If *ignore_cleanup_errors* is true, any unhandled + exceptions during explicit or implicit cleanup (such as a + :exc:`PermissionError` removing open files on Windows) will be ignored, + and the remaining removable items deleted on a "best-effort" basis. + Otherwise, errors will be raised in whatever context cleanup occurs + (the :meth:`!cleanup` call, exiting the context manager, when the object + is garbage-collected or during interpreter shutdown). The *delete* parameter can be used to disable cleanup of the directory tree upon exiting the context. While it may seem unusual for a context manager @@ -404,13 +410,13 @@ Here are some examples of typical usage of the :mod:`tempfile` module:: # create a temporary file using a context manager # close the file, use the name to open the file again - >>> with tempfile.TemporaryFile(delete_on_close=False) as fp: - ... fp.write(b'Hello world!') - ... fp.close() - # the file is closed, but not removed - # open the file again by using its name - ... with open(fp.name) as f - ... f.read() + >>> with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + ... fp.write(b'Hello world!') + ... fp.close() + ... # the file is closed, but not removed + ... # open the file again by using its name + ... with open(fp.name, mode='rb') as f: + ... f.read() b'Hello world!' >>> # file is now removed diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index fb1ff567d49e5c..0c6f3059fe71d1 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -1,5 +1,5 @@ -:mod:`termios` --- POSIX style tty control -========================================== +:mod:`!termios` --- POSIX style tty control +=========================================== .. module:: termios :platform: Unix @@ -16,6 +16,8 @@ complete description of these calls, see :manpage:`termios(3)` Unix manual page. It is only available for those Unix versions that support POSIX *termios* style tty I/O control configured during installation. +.. availability:: Unix. + All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by ``sys.stdin.fileno()``, or a :term:`file object`, such as ``sys.stdin`` itself. @@ -43,10 +45,20 @@ The module defines the following functions: Set the tty attributes for file descriptor *fd* from the *attributes*, which is a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: :const:`TCSANOW` to change - immediately, :const:`TCSADRAIN` to change after transmitting all queued output, - or :const:`TCSAFLUSH` to change after transmitting all queued output and - discarding all queued input. + determines when the attributes are changed: + + .. data:: TCSANOW + + Change attributes immediately. + + .. data:: TCSADRAIN + + Change attributes after transmitting all queued output. + + .. data:: TCSAFLUSH + + Change attributes after transmitting all queued output and + discarding all queued input. .. function:: tcsendbreak(fd, duration) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index de60151bb32ce1..2a61f0aaef2b4c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1,5 +1,5 @@ -:mod:`test` --- Regression tests package for Python -=================================================== +:mod:`!test` --- Regression tests package for Python +==================================================== .. module:: test :synopsis: Regression tests package containing the testing suite for Python. @@ -143,7 +143,7 @@ guidelines to be followed: arg = (1, 2, 3) When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above + :class:`unittest.TestCase` are run as tests. The :class:`!TestFuncAcceptsSequencesMixin` class in the example above does not have any data and so can't be run by itself, thus it does not inherit from :class:`unittest.TestCase`. @@ -159,6 +159,9 @@ guidelines to be followed: Running tests using the command-line interface ---------------------------------------------- +.. module:: test.regrtest + :synopsis: Drives the regression test suite. + The :mod:`test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under the hood, it uses :mod:`test.regrtest`; the call :program:`python -m @@ -321,9 +324,9 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python is built with the :c:macro:`Py_DEBUG` macro defined: if - Python is :ref:`built in debug mode <debug-build>` - (:option:`./configure --with-pydebug <--with-pydebug>`). + ``True`` if Python was built with the :c:macro:`Py_DEBUG` macro + defined, that is, if + Python was :ref:`built in debug mode <debug-build>`. .. versionadded:: 3.12 @@ -498,44 +501,6 @@ The :mod:`test.support` module defines the following functions: rather than looking directly in the path directories. -.. function:: match_test(test) - - Determine whether *test* matches the patterns set in :func:`set_match_tests`. - - -.. function:: set_match_tests(accept_patterns=None, ignore_patterns=None) - - Define match patterns on test filenames and test method names for filtering tests. - - -.. function:: run_unittest(*classes) - - Execute :class:`unittest.TestCase` subclasses passed to the function. The - function scans the classes for methods starting with the prefix ``test_`` - and executes the tests individually. - - It is also legal to pass strings as parameters; these should be keys in - ``sys.modules``. Each associated module will be scanned by - ``unittest.TestLoader.loadTestsFromModule()``. This is usually seen in the - following :func:`test_main` function:: - - def test_main(): - support.run_unittest(__name__) - - This will run all tests defined in the named module. - - -.. function:: run_doctest(module, verbosity=None, optionflags=0) - - Run :func:`doctest.testmod` on the given *module*. Return - ``(failure_count, test_count)``. - - If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity - set to :data:`verbose`. Otherwise, it is run with verbosity set to - ``None``. *optionflags* is passed as ``optionflags`` to - :func:`doctest.testmod`. - - .. function:: get_pagesize() Get size of a page in bytes. @@ -766,6 +731,12 @@ The :mod:`test.support` module defines the following functions: macOS version is less than the minimum, the test is skipped. +.. decorator:: requires_gil_enabled + + Decorator for skipping tests on the free-threaded build. If the + :term:`GIL` is disabled, the test is skipped. + + .. decorator:: requires_IEEE_754 Decorator for skipping tests on non-IEEE 754 platforms. @@ -1043,7 +1014,7 @@ The :mod:`test.support` module defines the following classes: :const:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file creation. - On both platforms, the old value is restored by :meth:`__exit__`. + On both platforms, the old value is restored by :meth:`~object.__exit__`. .. class:: SaveSignals() @@ -1443,7 +1414,8 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. class:: FakePath(path) - Simple :term:`path-like object`. It implements the :meth:`__fspath__` + Simple :term:`path-like object`. It implements the + :meth:`~os.PathLike.__fspath__` method which just returns the *path* argument. If *path* is an exception, it will be raised in :meth:`!__fspath__`. diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 7445410f91808c..a58b460fef409c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -1,5 +1,5 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= +:mod:`!textwrap` --- Text wrapping and filling +============================================== .. module:: textwrap :synopsis: Text wrapping and filling @@ -154,7 +154,7 @@ hyphenated words; only then will long words be broken if necessary, unless wrapper = TextWrapper() wrapper.initial_indent = "* " - You can re-use the same :class:`TextWrapper` object many times, and you can + You can reuse the same :class:`TextWrapper` object many times, and you can change any of its options through direct assignment to instance attributes between uses. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 23d8cd158abd5d..7b259e22dc7124 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -1,5 +1,5 @@ -:mod:`threading` --- Thread-based parallelism -============================================= +:mod:`!threading` --- Thread-based parallelism +============================================== .. module:: threading :synopsis: Thread-based parallelism. @@ -127,10 +127,13 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + Added support for GNU/kFreeBSD. + .. function:: enumerate() @@ -360,12 +363,12 @@ since it is impossible to detect the termination of alien threads. base class constructor (``Thread.__init__()``) before doing anything else to the thread. + .. versionchanged:: 3.3 + Added the *daemon* parameter. + .. versionchanged:: 3.10 Use the *target* name if *name* argument is omitted. - .. versionchanged:: 3.3 - Added the *daemon* argument. - .. method:: start() Start the thread's activity. @@ -531,9 +534,10 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - Note that ``Lock`` is actually a factory function which returns an instance - of the most efficient version of the concrete Lock class that is supported - by the platform. + .. versionchanged:: 3.13 + ``Lock`` is now a class. In earlier Pythons, ``Lock`` was a factory + function which returned an instance of the underlying private lock + type. .. method:: acquire(blocking=True, timeout=-1) @@ -594,14 +598,25 @@ and "recursion level" in addition to the locked/unlocked state used by primitive locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. +Threads call a lock's :meth:`~RLock.acquire` method to lock it, +and its :meth:`~Lock.release` method to unlock it. -Reentrant locks also support the :ref:`context management protocol <with-locks>`. +.. note:: + + Reentrant locks support the :ref:`context management protocol <with-locks>`, + so it is recommended to use :keyword:`with` instead of manually calling + :meth:`~RLock.acquire` and :meth:`~RLock.release` + to handle acquiring and releasing the lock for a block of code. + +RLock's :meth:`~RLock.acquire`/:meth:`~RLock.release` call pairs may be nested, +unlike Lock's :meth:`~Lock.acquire`/:meth:`~Lock.release`. Only the final +:meth:`~RLock.release` (the :meth:`~Lock.release` of the outermost pair) resets +the lock to an unlocked state and allows another thread blocked in +:meth:`~RLock.acquire` to proceed. + +:meth:`~RLock.acquire`/:meth:`~RLock.release` must be used in pairs: each acquire +must have a release in the thread that has acquired the lock. Failing to +call release as many times the lock has been acquired can lead to deadlock. .. class:: RLock() @@ -620,25 +635,41 @@ Reentrant locks also support the :ref:`context management protocol <with-locks>` Acquire a lock, blocking or non-blocking. - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. + .. seealso:: - When invoked with the *blocking* argument set to ``True``, do the same thing as when - called without arguments, and return ``True``. + :ref:`Using RLock as a context manager <with-locks>` + Recommended over manual :meth:`!acquire` and :meth:`release` calls + whenever practical. - When invoked with the *blocking* argument set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do the - same thing as when called without arguments, and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return ``True`` if the lock has - been acquired, ``False`` if the timeout has elapsed. + When invoked with the *blocking* argument set to ``True`` (the default): + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, block until we are able to acquire + lock, or *timeout*, if set to a positive float value. + + * If the same thread owns the lock, acquire the lock again, and + return immediately. This is the difference between :class:`Lock` and + :class:`!RLock`; :class:`Lock` handles this case the same as the previous, + blocking until the lock can be acquired. + + When invoked with the *blocking* argument set to ``False``: + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, return immediately. + + * If the same thread owns the lock, acquire the lock again and return + immediately. + + In all cases, if the thread was able to acquire the lock, return ``True``. + If the thread was unable to acquire the lock (i.e. if not blocking or + the timeout was reached) return ``False``. + + If called multiple times, failing to call :meth:`~RLock.release` as many times + may lead to deadlock. Consider using :class:`!RLock` as a context manager rather than + calling acquire/release directly. .. versionchanged:: 3.2 The *timeout* parameter is new. @@ -654,7 +685,7 @@ Reentrant locks also support the :ref:`context management protocol <with-locks>` Only call this method when the calling thread owns the lock. A :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. + not acquired. There is no return value. @@ -983,18 +1014,15 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. method:: wait(timeout=None) - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. + Block as long as the internal flag is false and the timeout, if given, + has not expired. The return value represents the + reason that this blocking method returned; ``True`` if returning because + the internal flag is set to true, or ``False`` if a timeout is given and + the the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns ``True`` if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. + floating point number specifying a timeout for the operation in seconds, + or fractions thereof. .. versionchanged:: 3.1 Previously, the method always returned ``None``. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 577600881676b3..4d7661715aa0af 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -1,5 +1,5 @@ -:mod:`time` --- Time access and conversions -=========================================== +:mod:`!time` --- Time access and conversions +============================================ .. module:: time :synopsis: Time access and conversions. @@ -287,6 +287,15 @@ Functions The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. + Clock: + + * On Windows, call ``QueryPerformanceCounter()`` and + ``QueryPerformanceFrequency()``. + * On macOS, call ``mach_absolute_time()`` and ``mach_timebase_info()``. + * On HP-UX, call ``gethrtime()``. + * Call ``clock_gettime(CLOCK_HIGHRES)`` if available. + * Otherwise, call ``clock_gettime(CLOCK_MONOTONIC)``. + Use :func:`monotonic_ns` to avoid the precision loss caused by the :class:`float` type. @@ -316,6 +325,11 @@ Functions point of the returned value is undefined, so that only the difference between the results of two calls is valid. + .. impl-detail:: + + On CPython, use the same clock than :func:`time.monotonic()` and is a + monotonic clock, i.e. a clock that cannot go backwards. + Use :func:`perf_counter_ns` to avoid the precision loss caused by the :class:`float` type. @@ -324,6 +338,10 @@ Functions .. versionchanged:: 3.10 On Windows, the function is now system-wide. + .. versionchanged:: 3.13 + Use the same clock than :func:`time.monotonic()`. + + .. function:: perf_counter_ns() -> int Similar to :func:`perf_counter`, but return time as nanoseconds. @@ -383,15 +401,14 @@ Functions .. audit-event:: time.sleep secs - .. versionchanged:: 3.11 - On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now - used if available. On Windows, a waitable timer is now used. - .. versionchanged:: 3.5 The function now sleeps at least *secs* even if the sleep is interrupted by a signal, except if the signal handler raises an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.11 + On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now + used if available. On Windows, a waitable timer is now used. .. versionchanged:: 3.13 Raises an auditing event. @@ -433,6 +450,10 @@ Functions | ``%d`` | Day of the month as a decimal number [01,31]. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%f`` | Microseconds as a decimal number | \(1) | + | | [000000,999999]. | | + | | | | + +-----------+------------------------------------------------+-------+ | ``%H`` | Hour (24-hour clock) as a decimal number | | | | [00,23]. | | +-----------+------------------------------------------------+-------+ @@ -448,13 +469,13 @@ Functions | ``%M`` | Minute as a decimal number [00,59]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | + | ``%p`` | Locale's equivalent of either AM or PM. | \(2) | | | | | +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | + | ``%S`` | Second as a decimal number [00,61]. | \(3) | | | | | +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | + | ``%U`` | Week number of the year (Sunday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Sunday are considered to be in week 0. | | @@ -465,7 +486,7 @@ Functions | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | + | ``%W`` | Week number of the year (Monday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Monday are considered to be in week 0. | | @@ -500,17 +521,23 @@ Functions Notes: (1) + The ``%f`` format directive only applies to :func:`strptime`, + not to :func:`strftime`. However, see also :meth:`datetime.datetime.strptime` and + :meth:`datetime.datetime.strftime` where the ``%f`` format directive + :ref:`applies to microseconds <format-codes>`. + + (2) When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. .. _leap-second: - (2) + (3) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. - (3) + (4) When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. @@ -590,7 +617,7 @@ Functions - range [1, 12] * - 2 - - .. attribute:: tm_day + - .. attribute:: tm_mday - range [1, 31] * - 3 @@ -657,6 +684,12 @@ Functions :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. + Clock: + + * On Windows, call ``GetSystemTimeAsFileTime()``. + * Call ``clock_gettime(CLOCK_REALTIME)`` if available. + * Otherwise, call ``gettimeofday()``. + Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. @@ -840,6 +873,15 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.3 +.. data:: CLOCK_MONOTONIC_RAW_APPROX + + Similar to :data:`CLOCK_MONOTONIC_RAW`, but reads a value cached by + the system at context switch and hence has less accuracy. + + .. availability:: macOS >= 10.12. + + .. versionadded:: 3.13 + .. data:: CLOCK_PROCESS_CPUTIME_ID @@ -899,6 +941,15 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.8 +.. data:: CLOCK_UPTIME_RAW_APPROX + + Like :data:`CLOCK_UPTIME_RAW`, but the value is cached by the system + at context switches and therefore has less accuracy. + + .. availability:: macOS >= 10.12. + + .. versionadded:: 3.13 + The following constant is the only parameter that can be sent to :func:`clock_settime`. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 616f8365b80f6c..548a3ee0540506 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -1,5 +1,5 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== +:mod:`!timeit` --- Measure execution time of small code snippets +================================================================ .. module:: timeit :synopsis: Measure the execution time of small code snippets. diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index 6e8479c1dea1e2..df2b324fd5d3a7 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.colorchooser` --- Color choosing dialog -===================================================== +:mod:`!tkinter.colorchooser` --- Color choosing dialog +====================================================== .. module:: tkinter.colorchooser :platform: Tk diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 02de0fd331958d..62298d96c26459 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.dnd` --- Drag and drop support -============================================ +:mod:`!tkinter.dnd` --- Drag and drop support +============================================= .. module:: tkinter.dnd :platform: Tk @@ -25,8 +25,8 @@ Selection of a target object occurs as follows: #. Top-down search of area under mouse for target widget * Target widget should have a callable *dnd_accept* attribute - * If *dnd_accept* is not present or returns None, search moves to parent widget - * If no target widget is found, then the target object is None + * If *dnd_accept* is not present or returns ``None``, search moves to parent widget + * If no target widget is found, then the target object is ``None`` 2. Call to *<old_target>.dnd_leave(source, event)* #. Call to *<new_target>.dnd_enter(source, event)* diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index c7c2b7b566cf8f..ed01bd5f483943 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.font` --- Tkinter font wrapper -============================================ +:mod:`!tkinter.font` --- Tkinter font wrapper +============================================= .. module:: tkinter.font :platform: Tk diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 56c1d6c132afd2..0dc9632ca73304 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.messagebox` --- Tkinter message prompts -===================================================== +:mod:`!tkinter.messagebox` --- Tkinter message prompts +====================================================== .. module:: tkinter.messagebox :platform: Tk @@ -11,7 +11,8 @@ The :mod:`tkinter.messagebox` module provides a template base class as well as a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (True, False, OK, None, Yes, No) based on +boxes are modal and will return a subset of (``True``, ``False``, ``None``, +:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on the user's selection. Common message box styles and layouts include but are not limited to: @@ -19,21 +20,175 @@ limited to: .. class:: Message(master=None, **options) - Create a default information message box. + Create a message window with an application-specified message, an icon + and a set of buttons. + Each of the buttons in the message window is identified by a unique symbolic name (see the *type* options). + + The following options are supported: + + *command* + Specifies the function to invoke when the user closes the dialog. + The name of the button clicked by the user to close the dialog is + passed as argument. + This is only available on macOS. + + *default* + Gives the :ref:`symbolic name <messagebox-buttons>` of the default button + for this message window (:data:`OK`, :data:`CANCEL`, and so on). + If this option is not specified, the first button in the dialog will + be made the default. + + *detail* + Specifies an auxiliary message to the main message given by the + *message* option. + The message detail will be presented beneath the main message and, + where supported by the OS, in a less emphasized font than the main + message. + + *icon* + Specifies an :ref:`icon <messagebox-icons>` to display. + If this option is not specified, then the :data:`INFO` icon will be + displayed. + + *message* + Specifies the message to display in this message box. + The default value is an empty string. + + *parent* + Makes the specified window the logical parent of the message box. + The message box is displayed on top of its parent window. + + *title* + Specifies a string to display as the title of the message box. + This option is ignored on macOS, where platform guidelines forbid + the use of a title on this kind of dialog. + + *type* + Arranges for a :ref:`predefined set of buttons <messagebox-types>` + to be displayed. + + .. method:: show(**options) + + Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. + Keyword arguments can override options specified in the constructor. + **Information message box** -.. method:: showinfo(title=None, message=None, **options) +.. function:: showinfo(title=None, message=None, **options) + + Creates and displays an information message box with the specified title + and message. **Warning message boxes** -.. method:: showwarning(title=None, message=None, **options) - showerror(title=None, message=None, **options) +.. function:: showwarning(title=None, message=None, **options) + + Creates and displays a warning message box with the specified title + and message. + +.. function:: showerror(title=None, message=None, **options) + + Creates and displays an error message box with the specified title + and message. **Question message boxes** -.. method:: askquestion(title=None, message=None, **options) - askokcancel(title=None, message=None, **options) - askretrycancel(title=None, message=None, **options) - askyesno(title=None, message=None, **options) - askyesnocancel(title=None, message=None, **options) +.. function:: askquestion(title=None, message=None, *, type=YESNO, **options) + + Ask a question. By default shows buttons :data:`YES` and :data:`NO`. + Returns the symbolic name of the selected button. + +.. function:: askokcancel(title=None, message=None, **options) + + Ask if operation should proceed. Shows buttons :data:`OK` and :data:`CANCEL`. + Returns ``True`` if the answer is ok and ``False`` otherwise. + +.. function:: askretrycancel(title=None, message=None, **options) + + Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. + Return ``True`` if the answer is yes and ``False`` otherwise. + +.. function:: askyesno(title=None, message=None, **options) + + Ask a question. Shows buttons :data:`YES` and :data:`NO`. + Returns ``True`` if the answer is yes and ``False`` otherwise. + +.. function:: askyesnocancel(title=None, message=None, **options) + + Ask a question. Shows buttons :data:`YES`, :data:`NO` and :data:`CANCEL`. + Return ``True`` if the answer is yes, ``None`` if cancelled, and ``False`` + otherwise. + + +.. _messagebox-buttons: + +Symbolic names of buttons: + +.. data:: ABORT + :value: 'abort' +.. data:: RETRY + :value: 'retry' +.. data:: IGNORE + :value: 'ignore' +.. data:: OK + :value: 'ok' +.. data:: CANCEL + :value: 'cancel' +.. data:: YES + :value: 'yes' +.. data:: NO + :value: 'no' + +.. _messagebox-types: + +Predefined sets of buttons: + +.. data:: ABORTRETRYIGNORE + :value: 'abortretryignore' + + Displays three buttons whose symbolic names are :data:`ABORT`, + :data:`RETRY` and :data:`IGNORE`. + +.. data:: OK + :value: 'ok' + :noindex: + + Displays one button whose symbolic name is :data:`OK`. + +.. data:: OKCANCEL + :value: 'okcancel' + + Displays two buttons whose symbolic names are :data:`OK` and + :data:`CANCEL`. + +.. data:: RETRYCANCEL + :value: 'retrycancel' + + Displays two buttons whose symbolic names are :data:`RETRY` and + :data:`CANCEL`. + +.. data:: YESNO + :value: 'yesno' + + Displays two buttons whose symbolic names are :data:`YES` and + :data:`NO`. + +.. data:: YESNOCANCEL + :value: 'yesnocancel' + + Displays three buttons whose symbolic names are :data:`YES`, + :data:`NO` and :data:`CANCEL`. + +.. _messagebox-icons: + +Icon images: + +.. data:: ERROR + :value: 'error' +.. data:: INFO + :value: 'info' +.. data:: QUESTION + :value: 'question' +.. data:: WARNING + :value: 'warning' diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 5fab26db67633c..f40790c1175800 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -1,5 +1,5 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= +:mod:`!tkinter` --- Python interface to Tcl/Tk +============================================== .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers <https://tkdocs.com/book.html>`_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming <https://www.packtpub.com/product/python-gui-programming-with-tkinter/9781788835886>`_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter <https://www.packtpub.com/product/python-gui-programming-with-tkinter/9781788835886>`_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python <https://learning-python.com/about-pp4e.html>`_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) @@ -232,6 +232,9 @@ The modules that provide Tk support include: Additional modules: +.. module:: _tkinter + :synopsis: A binary module that contains the low-level interface to Tcl/Tk. + :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. It is automatically imported by the main :mod:`tkinter` module, @@ -976,6 +979,15 @@ of :class:`tkinter.Image`: Either type of image is created through either the ``file`` or the ``data`` option (other options are available as well). +.. versionchanged:: 3.13 + Added the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + The image object can then be used wherever an ``image`` option is supported by some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index d20365baa38690..763e24929d74b5 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== +:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +===================================================== .. module:: tkinter.scrolledtext :platform: Tk diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index dc31a1a4c1850a..628e9f945ac365 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== +:mod:`!tkinter.ttk` --- Tk themed widgets +========================================= .. module:: tkinter.ttk :synopsis: Tk themed widget set @@ -986,19 +986,19 @@ ttk.Treeview The valid options/values are: - id + *id* Returns the column name. This is a read-only option. - anchor: One of the standard Tk anchor values. + *anchor*: One of the standard Tk anchor values. Specifies how the text in this column should be aligned with respect to the cell. - minwidth: width + *minwidth*: width The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - stretch: ``True``/``False`` + *stretch*: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. - width: width + *width*: width The width of the column in pixels. To configure the tree column, call this with column = "#0" @@ -1041,14 +1041,14 @@ ttk.Treeview The valid options/values are: - text: text + *text*: text The text to display in the column heading. - image: imageName + *image*: imageName Specifies an image to display to the right of the column heading. - anchor: anchor + *anchor*: anchor Specifies how the heading text should be aligned. One of the standard Tk anchor values. - command: callback + *command*: callback A callback to be invoked when the heading label is pressed. To configure the tree column heading, call this with column = "#0". @@ -1118,7 +1118,7 @@ ttk.Treeview as the item identifier; *iid* must not already exist in the tree. Otherwise, a new unique identifier is generated. - See `Item Options`_ for the list of available points. + See `Item Options`_ for the list of available options. .. method:: item(item, option=None, **kw) @@ -1391,8 +1391,8 @@ option. If you don't know the class name of a widget, use the method .. method:: element_create(elementname, etype, *args, **kw) Create a new element in the current theme, of the given *etype* which is - expected to be either "image", "from" or "vsapi". The latter is only - available in Tk 8.6a for Windows XP and Vista and is not described here. + expected to be either "image", "from" or "vsapi". + The latter is only available in Tk 8.6 on Windows. If "image" is used, *args* should contain the default image name followed by statespec/value pairs (this is the imagespec), and *kw* may have the @@ -1418,6 +1418,16 @@ option. If you don't know the class name of a widget, use the method Specifies a minimum width for the element. If less than zero, the base image's width is used as a default. + Example:: + + img1 = tkinter.PhotoImage(master=root, file='button.png') + img1 = tkinter.PhotoImage(master=root, file='button-pressed.png') + img1 = tkinter.PhotoImage(master=root, file='button-active.png') + style = ttk.Style(root) + style.element_create('Button.button', 'image', + img1, ('pressed', img2), ('active', img3), + border=(2, 4), sticky='we') + If "from" is used as the value of *etype*, :meth:`element_create` will clone an existing element. *args* is expected to contain a themename, from which @@ -1425,6 +1435,68 @@ option. If you don't know the class name of a widget, use the method If this element to clone from is not specified, an empty element will be used. *kw* is discarded. + Example:: + + style = ttk.Style(root) + style.element_create('plain.background', 'from', 'default') + + If "vsapi" is used as the value of *etype*, :meth:`element_create` + will create a new element in the current theme whose visual appearance + is drawn using the Microsoft Visual Styles API which is responsible + for the themed styles on Windows XP and Vista. + *args* is expected to contain the Visual Styles class and part as + given in the Microsoft documentation followed by an optional sequence + of tuples of ttk states and the corresponding Visual Styles API state + value. + *kw* may have the following options: + + padding=padding + Specify the element's interior padding. + *padding* is a list of up to four integers specifying the left, + top, right and bottom padding quantities respectively. + If fewer than four elements are specified, bottom defaults to top, + right defaults to left, and top defaults to left. + In other words, a list of three numbers specify the left, vertical, + and right padding; a list of two numbers specify the horizontal + and the vertical padding; a single number specifies the same + padding all the way around the widget. + This option may not be mixed with any other options. + + margins=padding + Specifies the elements exterior padding. + *padding* is a list of up to four integers specifying the left, top, + right and bottom padding quantities respectively. + This option may not be mixed with any other options. + + width=width + Specifies the width for the element. + If this option is set then the Visual Styles API will not be queried + for the recommended size or the part. + If this option is set then *height* should also be set. + The *width* and *height* options cannot be mixed with the *padding* + or *margins* options. + + height=height + Specifies the height of the element. + See the comments for *width*. + + Example:: + + style = ttk.Style(root) + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(style='Explorer.Pin') + pin.pack(expand=True, fill='both') + + .. versionchanged:: 3.13 + Added support of the "vsapi" element factory. .. method:: element_names() @@ -1501,23 +1573,24 @@ Layouts A layout can be just ``None``, if it takes no options, or a dict of options specifying how to arrange the element. The layout mechanism uses a simplified version of the pack geometry manager: given an -initial cavity, each element is allocated a parcel. Valid -options/values are: +initial cavity, each element is allocated a parcel. + +The valid options/values are: -side: whichside +*side*: whichside Specifies which side of the cavity to place the element; one of top, right, bottom or left. If omitted, the element occupies the entire cavity. -sticky: nswe +*sticky*: nswe Specifies where the element is placed inside its allocated parcel. -unit: 0 or 1 +*unit*: 0 or 1 If set to 1, causes the element and all of its descendants to be treated as a single element for the purposes of :meth:`Widget.identify` et al. It's used for things like scrollbar thumbs with grips. -children: [sublayout... ] +*children*: [sublayout... ] Specifies a list of elements to place inside the element. Each element is a tuple (or other sequence type) where the first item is the layout name, and the other is a `Layout`_. diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e6dc37d7ad852c..919ff590b72916 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -1,5 +1,5 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= +:mod:`!token` --- Constants used with Python parse trees +======================================================== .. module:: token :synopsis: Constants representing terminal nodes of the parse tree. diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 92bdb052267a68..f719319a302a23 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -1,5 +1,5 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== +:mod:`!tokenize` --- Tokenizer for Python source +================================================ .. module:: tokenize :synopsis: Lexical scanner for Python source code. diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 918576eb37eaee..b523ad93b35f9d 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -1,5 +1,5 @@ -:mod:`tomllib` --- Parse TOML files -=================================== +:mod:`!tomllib` --- Parse TOML files +==================================== .. module:: tomllib :synopsis: Parse TOML files. @@ -19,14 +19,14 @@ support writing TOML. .. seealso:: - The `Tomli-W package <https://pypi.org/project/tomli-w/>`__ + The :pypi:`Tomli-W package <tomli-w>` is a TOML writer that can be used in conjunction with this module, providing a write API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. .. seealso:: - The `TOML Kit package <https://pypi.org/project/tomlkit/>`__ + The :pypi:`TOML Kit package <tomlkit>` is a style-preserving TOML library with both read and write capability. It is a recommended replacement for this module for editing already existing TOML files. @@ -95,7 +95,7 @@ Conversion Table +------------------+--------------------------------------------------------------------------------------+ | TOML | Python | +==================+======================================================================================+ -| table | dict | +| TOML document | dict | +------------------+--------------------------------------------------------------------------------------+ | string | str | +------------------+--------------------------------------------------------------------------------------+ @@ -115,3 +115,9 @@ Conversion Table +------------------+--------------------------------------------------------------------------------------+ | array | list | +------------------+--------------------------------------------------------------------------------------+ +| table | dict | ++------------------+--------------------------------------------------------------------------------------+ +| inline table | dict | ++------------------+--------------------------------------------------------------------------------------+ +| array of tables | list of dicts | ++------------------+--------------------------------------------------------------------------------------+ diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index e9b59a6d186ba2..cae94ea08e17e5 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -1,5 +1,5 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== +:mod:`!trace` --- Trace or track Python statement execution +=========================================================== .. module:: trace :synopsis: Trace or track Python statement execution. @@ -187,7 +187,8 @@ Programmatic Interface Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None,\ + *, ignore_missing_files=False) Write coverage results. Set *show_missing* to show lines that had no hits. Set *summary* to include in the output the coverage summary per @@ -195,6 +196,13 @@ Programmatic Interface result files will be output. If ``None``, the results for each source file are placed in its directory. + If *ignore_missing_files* is ``True``, coverage counts for files that no + longer exist are silently ignored. Otherwise, a missing file will + raise a :exc:`FileNotFoundError`. + + .. versionchanged:: 3.13 + Added *ignore_missing_files* parameter. + A simple example demonstrating the use of the programmatic interface:: import sys diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 67ee73d4b2e1e5..bfd2c3efc4b1f6 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -1,5 +1,5 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== +:mod:`!traceback` --- Print or retrieve a stack traceback +========================================================= .. module:: traceback :synopsis: Print or retrieve a stack traceback. @@ -16,8 +16,10 @@ interpreter. .. index:: pair: object; traceback -The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, -which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. +The module uses :ref:`traceback objects <traceback-objects>` --- these are +objects of type :class:`types.TracebackType`, +which are assigned to the :attr:`~BaseException.__traceback__` field of +:class:`BaseException` instances. .. seealso:: @@ -31,11 +33,13 @@ The module defines the following functions: .. function:: print_tb(tb, limit=None, file=None) - Print up to *limit* stack trace entries from traceback object *tb* (starting + Print up to *limit* stack trace entries from + :ref:`traceback object <traceback-objects>` *tb* (starting from the caller's frame) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to + :data:`sys.stderr`; otherwise it should be an open + :term:`file <file object>` or :term:`file-like object` to receive the output. .. versionchanged:: 3.5 @@ -45,7 +49,8 @@ The module defines the following functions: .. function:: print_exception(exc, /[, value, tb], limit=None, \ file=None, chain=True) - Print exception information and stack trace entries from traceback object + Print exception information and stack trace entries from + :ref:`traceback object <traceback-objects>` *tb* to *file*. This differs from :func:`print_tb` in the following ways: @@ -66,7 +71,8 @@ The module defines the following functions: The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the - :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be + :attr:`~BaseException.__cause__` or :attr:`~BaseException.__context__` + attributes of the exception) will be printed as well, like the interpreter itself does when printing an unhandled exception. @@ -96,7 +102,8 @@ The module defines the following functions: Print up to *limit* stack trace entries (starting from the invocation point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame + The optional *f* argument can be used to specify an alternate + :ref:`stack frame <frame-objects>` to start. The optional *file* argument has the same meaning as for :func:`print_tb`. @@ -107,20 +114,20 @@ The module defines the following functions: .. function:: extract_tb(tb, limit=None) Return a :class:`StackSummary` object representing a list of "pre-processed" - stack trace entries extracted from the traceback object *tb*. It is useful + stack trace entries extracted from the + :ref:`traceback object <traceback-objects>` *tb*. It is useful for alternate formatting of stack traces. The optional *limit* argument has the same meaning as for :func:`print_tb`. A "pre-processed" stack trace entry is a :class:`FrameSummary` object containing attributes :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the - information that is usually printed for a stack trace. The - :attr:`~FrameSummary.line` is a string with leading and trailing - whitespace stripped; if the source is not available it is ``None``. + information that is usually printed for a stack trace. .. function:: extract_stack(f=None, limit=None) - Extract the raw traceback from the current stack frame. The return value has + Extract the raw traceback from the current + :ref:`stack frame <frame-objects>`. The return value has the same format as for :func:`extract_tb`. The optional *f* and *limit* arguments have the same meaning as for :func:`print_stack`. @@ -135,10 +142,10 @@ The module defines the following functions: text line is not ``None``. -.. function:: format_exception_only(exc, /[, value]) +.. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as - given by ``sys.last_value``. The return value is a list of strings, each + given by :data:`sys.last_value`. The return value is a list of strings, each ending in a newline. The list contains the exception's message, which is normally a single string; however, for :exc:`SyntaxError` exceptions, it contains several lines that (when printed) display detailed information @@ -149,12 +156,20 @@ The module defines the following functions: can be passed as the first argument. If *value* is provided, the first argument is ignored in order to provide backwards compatibility. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. + .. versionchanged:: 3.10 The *etype* parameter has been renamed to *exc* and is now positional-only. .. versionchanged:: 3.11 - The returned list now includes any notes attached to the exception. + The returned list now includes any + :attr:`notes <BaseException.__notes__>` attached to the exception. + + .. versionchanged:: 3.13 + *show_group* parameter was added. .. function:: format_exception(exc, /[, value, tb], limit=None, chain=True) @@ -190,14 +205,17 @@ The module defines the following functions: .. function:: clear_frames(tb) - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. + Clears the local variables of all the stack frames in a + :ref:`traceback <traceback-objects>` *tb* + by calling the :meth:`~frame.clear` method of each + :ref:`frame object <frame-objects>`. .. versionadded:: 3.4 .. function:: walk_stack(f) - Walk a stack following ``f.f_back`` from the given frame, yielding the frame + Walk a stack following :attr:`f.f_back <frame.f_back>` from the given frame, + yielding the frame and line number for each frame. If *f* is ``None``, the current stack is used. This helper is used with :meth:`StackSummary.extract`. @@ -205,19 +223,20 @@ The module defines the following functions: .. function:: walk_tb(tb) - Walk a traceback following ``tb_next`` yielding the frame and line number + Walk a traceback following :attr:`~traceback.tb_next` yielding the frame and + line number for each frame. This helper is used with :meth:`StackSummary.extract`. .. versionadded:: 3.5 The module also defines the following classes: -:class:`TracebackException` Objects ------------------------------------ +:class:`!TracebackException` Objects +------------------------------------ .. versionadded:: 3.5 -:class:`TracebackException` objects are created from actual exceptions to +:class:`!TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) @@ -225,10 +244,11 @@ capture data for later printing in a lightweight fashion. Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. - If *compact* is true, only data that is required by :class:`TracebackException`'s - ``format`` method is saved in the class attributes. In particular, the - ``__context__`` field is calculated only if ``__cause__`` is ``None`` and - ``__suppress_context__`` is false. + If *compact* is true, only data that is required by + :class:`!TracebackException`'s :meth:`format` method + is saved in the class attributes. In particular, the + :attr:`__context__` field is calculated only if :attr:`__cause__` is + ``None`` and :attr:`__suppress_context__` is false. Note that when locals are captured, they are also shown in the traceback. @@ -246,27 +266,31 @@ capture data for later printing in a lightweight fashion. .. attribute:: __cause__ - A :class:`TracebackException` of the original ``__cause__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__cause__`. .. attribute:: __context__ - A :class:`TracebackException` of the original ``__context__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__context__`. .. attribute:: exceptions If ``self`` represents an :exc:`ExceptionGroup`, this field holds a list of - :class:`TracebackException` instances representing the nested exceptions. + :class:`!TracebackException` instances representing the nested exceptions. Otherwise it is ``None``. .. versionadded:: 3.11 .. attribute:: __suppress_context__ - The ``__suppress_context__`` value from the original exception. + The :attr:`~BaseException.__suppress_context__` value from the original + exception. .. attribute:: __notes__ - The ``__notes__`` value from the original exception, or ``None`` + The :attr:`~BaseException.__notes__` value from the original exception, + or ``None`` if the exception does not have any notes. If it is not ``None`` is it formatted in the traceback after the exception string. @@ -280,6 +304,14 @@ capture data for later printing in a lightweight fashion. The class of the original traceback. + .. deprecated:: 3.13 + + .. attribute:: exc_type_str + + String display of the class of the original exception. + + .. versionadded:: 3.13 + .. attribute:: filename For syntax errors - the file name where the error occurred. @@ -332,8 +364,8 @@ capture data for later printing in a lightweight fashion. Format the exception. - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. + If *chain* is not ``True``, :attr:`__cause__` and :attr:`__context__` + will not be formatted. The return value is a generator of strings, each ending in a newline and some containing internal newlines. :func:`~traceback.print_exception` @@ -356,33 +388,34 @@ capture data for later printing in a lightweight fashion. well, recursively, with indentation relative to their nesting depth. .. versionchanged:: 3.11 - The exception's notes are now included in the output. + The exception's :attr:`notes <BaseException.__notes__>` are now + included in the output. .. versionchanged:: 3.13 Added the *show_group* parameter. -:class:`StackSummary` Objects ------------------------------ +:class:`!StackSummary` Objects +------------------------------ .. versionadded:: 3.5 -:class:`StackSummary` objects represent a call stack ready for formatting. +:class:`!StackSummary` objects represent a call stack ready for formatting. .. class:: StackSummary .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - Construct a :class:`StackSummary` object from a frame generator (such as + Construct a :class:`!StackSummary` object from a frame generator (such as is returned by :func:`~traceback.walk_stack` or :func:`~traceback.walk_tb`). If *limit* is supplied, only this many frames are taken from *frame_gen*. If *lookup_lines* is ``False``, the returned :class:`FrameSummary` objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it + creating the :class:`!StackSummary` cheaper (which may be valuable if it may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object + local variables in each :class:`!FrameSummary` are captured as object representations. .. versionchanged:: 3.12 @@ -391,14 +424,16 @@ capture data for later printing in a lightweight fashion. .. classmethod:: from_list(a_list) - Construct a :class:`StackSummary` object from a supplied list of + Construct a :class:`!StackSummary` object from a supplied list of :class:`FrameSummary` objects or old-style list of tuples. Each tuple - should be a 4-tuple with filename, lineno, name, line as the elements. + should be a 4-tuple with *filename*, *lineno*, *name*, *line* as the + elements. .. method:: format() Returns a list of strings ready for printing. Each string in the - resulting list corresponds to a single frame from the stack. + resulting list corresponds to a single :ref:`frame <frame-objects>` from + the stack. Each string ends in a newline; the strings may contain internal newlines as well, for those items with source text lines. @@ -411,7 +446,8 @@ capture data for later printing in a lightweight fashion. .. method:: format_frame_summary(frame_summary) - Returns a string for printing one of the frames involved in the stack. + Returns a string for printing one of the :ref:`frames <frame-objects>` + involved in the stack. This method is called for each :class:`FrameSummary` object to be printed by :meth:`StackSummary.format`. If it returns ``None``, the frame is omitted from the output. @@ -419,25 +455,50 @@ capture data for later printing in a lightweight fashion. .. versionadded:: 3.11 -:class:`FrameSummary` Objects ------------------------------ +:class:`!FrameSummary` Objects +------------------------------ .. versionadded:: 3.5 -A :class:`FrameSummary` object represents a single frame in a traceback. +A :class:`!FrameSummary` object represents a single :ref:`frame <frame-objects>` +in a :ref:`traceback <traceback-objects>`. .. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames + Represents a single :ref:`frame <frame-objects>` in the + :ref:`traceback <traceback-objects>` or stack that is being formatted + or printed. It may optionally have a stringified version of the frame's locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). + looked up until the :class:`!FrameSummary` has the :attr:`~FrameSummary.line` + attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the + mapping, and if supplied the variable representations are stored in the summary for later display. + :class:`!FrameSummary` instances have the following attributes: + + .. attribute:: FrameSummary.filename + + The filename of the source code for this frame. Equivalent to accessing + :attr:`f.f_code.co_filename <codeobject.co_filename>` on a + :ref:`frame object <frame-objects>` *f*. + + .. attribute:: FrameSummary.lineno + + The line number of the source code for this frame. + + .. attribute:: FrameSummary.name + + Equivalent to accessing :attr:`f.f_code.co_name <codeobject.co_name>` on + a :ref:`frame object <frame-objects>` *f*. + + .. attribute:: FrameSummary.line + + A string representing the source code for this frame, with leading and + trailing whitespace stripped. + If the source is not available, it is ``None``. + .. _traceback-example: Traceback Examples @@ -508,27 +569,32 @@ The output for the example would look similar to this: *** print_tb: File "<doctest...>", line 10, in <module> lumberjack() + ~~~~~~~~~~^^ *** print_exception: Traceback (most recent call last): File "<doctest...>", line 10, in <module> lumberjack() + ~~~~~~~~~~^^ File "<doctest...>", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** print_exc: Traceback (most recent call last): File "<doctest...>", line 10, in <module> lumberjack() + ~~~~~~~~~~^^ File "<doctest...>", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** format_exc, first and last line: Traceback (most recent call last): IndexError: tuple index out of range *** format_exception: ['Traceback (most recent call last):\n', - ' File "<doctest default[0]>", line 10, in <module>\n lumberjack()\n', - ' File "<doctest default[0]>", line 4, in lumberjack\n bright_side_of_life()\n', + ' File "<doctest default[0]>", line 10, in <module>\n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "<doctest default[0]>", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "<doctest default[0]>", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n', 'IndexError: tuple index out of range\n'] *** extract_tb: @@ -536,8 +602,8 @@ The output for the example would look similar to this: <FrameSummary file <doctest...>, line 4 in lumberjack>, <FrameSummary file <doctest...>, line 7 in bright_side_of_life>] *** format_tb: - [' File "<doctest default[0]>", line 10, in <module>\n lumberjack()\n', - ' File "<doctest default[0]>", line 4, in lumberjack\n bright_side_of_life()\n', + [' File "<doctest default[0]>", line 10, in <module>\n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "<doctest default[0]>", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "<doctest default[0]>", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n'] *** tb_lineno: 10 diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 68432aeaecbcc1..2370d927292eb0 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -1,5 +1,5 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== +:mod:`!tracemalloc` --- Trace memory allocations +================================================ .. module:: tracemalloc :synopsis: Trace memory allocations. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index a4777772e1fc6c..37778bf20bdcc7 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -1,5 +1,5 @@ -:mod:`tty` --- Terminal control functions -========================================= +:mod:`!tty` --- Terminal control functions +========================================== .. module:: tty :platform: Unix @@ -15,6 +15,8 @@ The :mod:`tty` module defines functions for putting the tty into cbreak and raw modes. +.. availability:: Unix. + Because it requires the :mod:`termios` module, it will work only on Unix. The :mod:`tty` module defines the following functions: @@ -33,8 +35,15 @@ The :mod:`tty` module defines the following functions: Convert the tty attribute list *mode*, which is a list like the one returned by :func:`termios.tcgetattr`, to that of a tty in cbreak mode. + This clears the ``ECHO`` and ``ICANON`` local mode flags in *mode* as well + as setting the minimum input to 1 byte with no delay. + .. versionadded:: 3.12 + .. versionchanged:: 3.12.2 + The ``ICRNL`` flag is no longer cleared. This matches Linux and macOS + ``stty cbreak`` behavior and what :func:`setcbreak` historically did. + .. function:: setraw(fd, when=termios.TCSAFLUSH) @@ -44,7 +53,7 @@ The :mod:`tty` module defines the following functions: is saved before setting *fd* to raw mode; this value is returned. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. function:: setcbreak(fd, when=termios.TCSAFLUSH) @@ -54,8 +63,16 @@ The :mod:`tty` module defines the following functions: :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` is saved before setting *fd* to cbreak mode; this value is returned. + This clears the ``ECHO`` and ``ICANON`` local mode flags as well as setting + the minimum input to 1 byte with no delay. + .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. + + .. versionchanged:: 3.12.2 + The ``ICRNL`` flag is no longer cleared. This restores the behavior + of Python 3.11 and earlier as well as matching what Linux, macOS, & BSDs + describe in their ``stty(1)`` man pages regarding cbreak mode. .. seealso:: diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 88b1f09eb3c8b7..afda3685d606bb 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -120,7 +120,7 @@ off-screen):: home() The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: +know them, get the turtle's x-y coordinates with:: pos() @@ -606,7 +606,7 @@ Turtle motion >>> turtle.pos() (20.00,30.00) - .. versionadded: 3.12 + .. versionadded:: 3.12 .. function:: setx(x) diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 54c3907dec98cc..116868c24be864 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -1,5 +1,5 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== +:mod:`!types` --- Dynamic type creation and names for built-in types +==================================================================== .. module:: types :synopsis: Names for built-in types. @@ -188,7 +188,7 @@ Standard names are defined for the following types: .. index:: pair: built-in function; compile - The type for code objects such as returned by :func:`compile`. + The type of :ref:`code objects <code-objects>` such as returned by :func:`compile`. .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType @@ -196,14 +196,6 @@ Standard names are defined for the following types: required by the initializer. The audit event only occurs for direct instantiation of code objects, and is not raised for normal compilation. - .. method:: CodeType.replace(**kwargs) - - Return a copy of the code object with new values for the specified fields. - - Code objects are also supported by generic function :func:`copy.replace`. - - .. versionadded:: 3.8 - .. data:: CellType The type for cell objects: such objects are used as containers for @@ -378,17 +370,15 @@ Standard names are defined for the following types: .. data:: FrameType - The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a - traceback object. - - See :ref:`the language reference <frame-objects>` for details of the - available attributes and operations. + The type of :ref:`frame objects <frame-objects>` such as found in + :attr:`tb.tb_frame <traceback.tb_frame>` if ``tb`` is a traceback object. .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such - as ``FrameType.f_locals`` or ``array.array.typecode``. This type is used as + as :attr:`FrameType.f_locals <frame.f_locals>` or ``array.array.typecode``. + This type is used as descriptor for object attributes; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. @@ -400,6 +390,10 @@ Standard names are defined for the following types: data members which use standard conversion functions; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. + In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for + each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute + on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + .. impl-detail:: In other implementations of Python, this type may be identical to @@ -487,14 +481,25 @@ Additional Utility Classes and Functions A simple :class:`object` subclass that provides attribute access to its namespace, as well as a meaningful repr. - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. + Unlike :class:`object`, with :class:`!SimpleNamespace` you can add and remove + attributes. + + :py:class:`SimpleNamespace` objects may be initialized + in the same way as :class:`dict`: either with keyword arguments, + with a single positional argument, or with both. + When initialized with keyword arguments, + those are directly added to the underlying namespace. + Alternatively, when initialized with a positional argument, + the underlying namespace will be updated with key-value pairs + from that argument (either a mapping object or + an :term:`iterable` object producing key-value pairs). + All such keys must be strings. The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, /, **kwargs): + def __init__(self, mapping_or_iterable=(), /, **kwargs): + self.__dict__.update(mapping_or_iterable) self.__dict__.update(kwargs) def __repr__(self): @@ -518,6 +523,9 @@ Additional Utility Classes and Functions Attribute order in the repr changed from alphabetical to insertion (like ``dict``). + .. versionchanged:: 3.13 + Added support for an optional positional argument. + .. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) Route attribute access on a class to __getattr__. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 8f691932201225..7d1d317b9f8f8a 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -18,32 +18,31 @@ .. note:: The Python runtime does not enforce function and variable type annotations. - They can be used by third party tools such as type checkers, IDEs, linters, - etc. + They can be used by third party tools such as :term:`type checkers <static type checker>`, + IDEs, linters, etc. -------------- -This module provides runtime support for type hints. For the original -specification of the typing system, see :pep:`484`. For a simplified -introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. +Consider the function below:: -The function below takes and returns a string and is annotated as follows:: + def surface_area_of_cube(edge_length: float) -> str: + return f"The surface area of the cube is {6 * edge_length ** 2}." - def greeting(name: str) -> str: - return 'Hello ' + name +The function ``surface_area_of_cube`` takes an argument expected to +be an instance of :class:`float`, as indicated by the :term:`type hint` +``edge_length: float``. The function is expected to return an instance +of :class:`str`, as indicated by the ``-> str`` hint. -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. +While type hints can be simple classes like :class:`float` or :class:`str`, +they can also be more complex. The :mod:`typing` module provides a vocabulary of +more advanced type hints. New features are frequently added to the ``typing`` module. -The `typing_extensions <https://pypi.org/project/typing-extensions/>`_ package +The :pypi:`typing_extensions` package provides backports of these new features to older versions of Python. -For a summary of deprecated features and a deprecation timeline, please see -`Deprecation Timeline of Major Features`_. - .. seealso:: `"Typing cheat sheet" <https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html>`_ @@ -61,67 +60,11 @@ For a summary of deprecated features and a deprecation timeline, please see .. _relevant-peps: -Relevant PEPs -============= - -Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a -number of PEPs have modified and enhanced Python's framework for type -annotations: - -.. raw:: html - - <details> - <summary><a style="cursor:pointer;">The full list of PEPs</a></summary> - -* :pep:`526`: Syntax for Variable Annotations - *Introducing* syntax for annotating variables outside of function - definitions, and :data:`ClassVar` -* :pep:`544`: Protocols: Structural subtyping (static duck typing) - *Introducing* :class:`Protocol` and the - :func:`@runtime_checkable<runtime_checkable>` decorator -* :pep:`585`: Type Hinting Generics In Standard Collections - *Introducing* :class:`types.GenericAlias` and the ability to use standard - library classes as :ref:`generic types<types-genericalias>` -* :pep:`586`: Literal Types - *Introducing* :data:`Literal` -* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys - *Introducing* :class:`TypedDict` -* :pep:`591`: Adding a final qualifier to typing - *Introducing* :data:`Final` and the :func:`@final<final>` decorator -* :pep:`593`: Flexible function and variable annotations - *Introducing* :data:`Annotated` -* :pep:`604`: Allow writing union types as ``X | Y`` - *Introducing* :data:`types.UnionType` and the ability to use - the binary-or operator ``|`` to signify a - :ref:`union of types<types-union>` -* :pep:`612`: Parameter Specification Variables - *Introducing* :class:`ParamSpec` and :data:`Concatenate` -* :pep:`613`: Explicit Type Aliases - *Introducing* :data:`TypeAlias` -* :pep:`646`: Variadic Generics - *Introducing* :data:`TypeVarTuple` -* :pep:`647`: User-Defined Type Guards - *Introducing* :data:`TypeGuard` -* :pep:`655`: Marking individual TypedDict items as required or potentially missing - *Introducing* :data:`Required` and :data:`NotRequired` -* :pep:`673`: Self type - *Introducing* :data:`Self` -* :pep:`675`: Arbitrary Literal String Type - *Introducing* :data:`LiteralString` -* :pep:`681`: Data Class Transforms - *Introducing* the :func:`@dataclass_transform<dataclass_transform>` decorator -* :pep:`692`: Using ``TypedDict`` for more precise ``**kwargs`` typing - *Introducing* a new way of typing ``**kwargs`` with :data:`Unpack` and - :data:`TypedDict` -* :pep:`695`: Type Parameter Syntax - *Introducing* builtin syntax for creating generic functions, classes, and type aliases. -* :pep:`698`: Adding an override decorator to typing - *Introducing* the :func:`@override<override>` decorator - -.. raw:: html - - </details> - <br> +Specification for the Python Type System +======================================== + +The canonical, up-to-date specification of the Python type system can be +found at `"Specification for the Python type system" <https://typing.readthedocs.io/en/latest/spec/index.html>`_. .. _type-aliases: @@ -155,8 +98,9 @@ Type aliases are useful for simplifying complex type signatures. For example:: # The static type checker will treat the previous type signature as # being exactly equivalent to this one. def broadcast_message( - message: str, - servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: + message: str, + servers: Sequence[tuple[tuple[str, int], dict[str, str]]] + ) -> None: ... The :keyword:`type` statement is new in Python 3.12. For backwards @@ -304,7 +248,7 @@ a callable with any arbitrary parameter list would be acceptable: x = concat # Also OK ``Callable`` cannot express complex signatures such as functions that take a -variadic number of arguments, :func:`overloaded functions <overload>`, or +variadic number of arguments, :ref:`overloaded functions <overload>`, or functions that have keyword-only parameters. However, these signatures can be expressed by defining a :class:`Protocol` class with a :meth:`~object.__call__` method: @@ -526,7 +470,7 @@ A user-defined class can be defined as a generic class. self.logger.info('%s: %s', self.name, message) This syntax indicates that the class ``LoggedVar`` is parameterised around a -single :class:`type variable <TypeVar>` ``T`` . This also makes ``T`` valid as +single :ref:`type variable <typevar>` ``T`` . This also makes ``T`` valid as a type within the class body. Generic classes implicitly inherit from :class:`Generic`. For compatibility @@ -910,14 +854,25 @@ using ``[]``. .. versionadded:: 3.11 .. data:: Never + NoReturn - The `bottom type <https://en.wikipedia.org/wiki/Bottom_type>`_, + :data:`!Never` and :data:`!NoReturn` represent the + `bottom type <https://en.wikipedia.org/wiki/Bottom_type>`_, a type that has no members. - This can be used to define a function that should never be - called, or a function that never returns:: + They can be used to indicate that a function never returns, + such as :func:`sys.exit`:: + + from typing import Never # or NoReturn + + def stop() -> Never: + raise RuntimeError('no way') + + Or to define a function that should never be + called, as there are no valid arguments, such as + :func:`assert_never`:: - from typing import Never + from typing import Never # or NoReturn def never_call_me(arg: Never) -> None: pass @@ -930,32 +885,18 @@ using ``[]``. case str(): print("It's a str") case _: - never_call_me(arg) # OK, arg is of type Never - - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. + never_call_me(arg) # OK, arg is of type Never (or NoReturn) -.. data:: NoReturn + :data:`!Never` and :data:`!NoReturn` have the same meaning in the type system + and static type checkers treat both equivalently. - Special type indicating that a function never returns. - - For example:: - - from typing import NoReturn + .. versionadded:: 3.6.2 - def stop() -> NoReturn: - raise RuntimeError('no way') + Added :data:`NoReturn`. - ``NoReturn`` can also be used as a - `bottom type <https://en.wikipedia.org/wiki/Bottom_type>`_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. + .. versionadded:: 3.11 - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + Added :data:`Never`. .. data:: Self @@ -1145,16 +1086,13 @@ These can be used as types in annotations. They all support subscription using from collections.abc import Callable from threading import Lock - from typing import Concatenate, ParamSpec, TypeVar - - P = ParamSpec('P') - R = TypeVar('R') + from typing import Concatenate # Use this lock to ensure that only one thread is executing a function # at any time. my_lock = Lock() - def with_lock(f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: + def with_lock[**P, R](f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: '''A type-safe decorator which provides a lock.''' def inner(*args: P.args, **kwargs: P.kwargs) -> R: # Provide the lock as the first argument. @@ -1237,6 +1175,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.5.3 + .. versionchanged:: 3.13 + + :data:`ClassVar` can now be nested in :data:`Final` and vice versa. + .. data:: Final Special typing construct to indicate final names to type checkers. @@ -1260,6 +1202,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.8 + .. versionchanged:: 3.13 + + :data:`Final` can now be nested in :data:`ClassVar` and vice versa. + .. data:: Required Special typing construct to mark a :class:`TypedDict` key as required. @@ -1278,6 +1224,26 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.11 +.. data:: ReadOnly + + A special typing construct to mark an item of a :class:`TypedDict` as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + + See :class:`TypedDict` and :pep:`705` for more details. + + .. versionadded:: 3.13 + .. data:: Annotated Special typing form to add context-specific metadata to an annotation. @@ -1419,22 +1385,23 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.9 -.. data:: TypeGuard +.. data:: TypeIs - Special typing construct for marking user-defined type guard functions. + Special typing construct for marking user-defined type predicate functions. - ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and take at + least one positional argument. - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard":: + conditional expression here is sometimes referred to as a "type predicate":: def is_str(val: str | float): - # "isinstance" type guard + # "isinstance" type predicate if isinstance(val, str): # Type of ``val`` is narrowed to ``str`` ... @@ -1443,8 +1410,73 @@ These can be used as types in annotations. They all support subscription using ... Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeIs[...]`` or + :data:`TypeGuard` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + If a ``TypeIs`` function is a class or instance method, then the type in + ``TypeIs`` maps to the type of the second parameter (after ``cls`` or + ``self``). + + In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, + means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance + of ``TypeB``, and if it returns ``False``, it is not an instance of ``TypeB``. + + ``TypeIs`` also works with type variables. For more information, see + :pep:`742` (Narrowing types with ``TypeIs``). + + .. versionadded:: 3.13 + + +.. data:: TypeGuard + + Special typing construct for marking user-defined type predicate functions. + + Type predicate functions are user-defined functions that return whether their + argument is an instance of a particular type. + ``TypeGuard`` works similarly to :data:`TypeIs`, but has subtly different + effects on type checking behavior (see below). Using ``-> TypeGuard`` tells the static type checker that for a given function: @@ -1453,6 +1485,8 @@ These can be used as types in annotations. They all support subscription using 2. If the return value is ``True``, the type of its argument is the type inside ``TypeGuard``. + ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + For example:: def is_str_list(val: list[object]) -> TypeGuard[list[str]]: @@ -1467,23 +1501,19 @@ These can be used as types in annotations. They all support subscription using # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") - If ``is_str_list`` is a class or instance method, then the type in - ``TypeGuard`` maps to the type of the second parameter after ``cls`` or - ``self``. - - In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, - means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from - ``TypeA`` to ``TypeB``. - - .. note:: - - ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a - wider form. The main reason is to allow for things like - narrowing ``list[object]`` to ``list[str]`` even though the latter - is not a subtype of the former, since ``list`` is invariant. - The responsibility of writing type-safe type guards is left to the user. + ``TypeIs`` and ``TypeGuard`` differ in the following ways: - ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + * ``TypeIs`` requires the narrowed type to be a subtype of the input type, while + ``TypeGuard`` does not. The main reason is to allow for things like + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. + * When a ``TypeGuard`` function returns ``True``, type checkers narrow the type of the + variable to exactly the ``TypeGuard`` type. When a ``TypeIs`` function returns ``True``, + type checkers can infer a more precise type combining the previously known type of the + variable with the ``TypeIs`` type. (Technically, this is known as an intersection type.) + * When a ``TypeGuard`` function returns ``False``, type checkers cannot narrow the type of + the variable at all. When a ``TypeIs`` function returns ``False``, type checkers can narrow + the type of the variable to exclude the ``TypeIs`` type. .. versionadded:: 3.10 @@ -1493,7 +1523,7 @@ These can be used as types in annotations. They all support subscription using Typing operator to conceptually mark an object as having been unpacked. For example, using the unpack operator ``*`` on a - :class:`type variable tuple <TypeVarTuple>` is equivalent to using ``Unpack`` + :ref:`type variable tuple <typevartuple>` is equivalent to using ``Unpack`` to mark the type variable tuple as having been unpacked:: Ts = TypeVarTuple('Ts') @@ -1584,7 +1614,9 @@ without the dedicated syntax, as documented below. ... # Etc. -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) +.. _typevar: + +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable. @@ -1722,15 +1754,37 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. attribute:: __default__ + + The default value of the type variable, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated <lazy-evaluation>` default value. + + .. versionadded:: 3.13 + .. versionchanged:: 3.12 Type variables can now be declared using the :ref:`type parameter <type-params>` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 - Type variable tuple. A specialized form of :class:`type variable <TypeVar>` + Support for default values was added. + +.. _typevartuple: + +.. class:: TypeVarTuple(name, *, default=typing.NoDefault) + + Type variable tuple. A specialized form of :ref:`type variable <typevar>` that enables *variadic* generics. Type variable tuples can be declared in :ref:`type parameter lists <type-params>` @@ -1819,8 +1873,8 @@ without the dedicated syntax, as documented below. of ``*args``:: def call_soon[*Ts]( - callback: Callable[[*Ts], None], - *args: *Ts + callback: Callable[[*Ts], None], + *args: *Ts ) -> None: ... callback(*args) @@ -1838,6 +1892,22 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __default__ + + The default value of the type variable tuple, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable tuple has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated <lazy-evaluation>` default value. + + .. versionadded:: 3.13 + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1845,10 +1915,14 @@ without the dedicated syntax, as documented below. Type variable tuples can now be declared using the :ref:`type parameter <type-params>` syntax introduced by :pep:`695`. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + .. versionchanged:: 3.13 + + Support for default values was added. + +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of - :class:`type variables <TypeVar>`. + :ref:`type variables <typevar>`. In :ref:`type parameter lists <type-params>`, parameter specifications can be declared with two asterisks (``**``):: @@ -1914,6 +1988,22 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __default__ + + The default value of the parameter specification, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the parameter specification has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated <lazy-evaluation>` default value. + + .. versionadded:: 3.13 + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1927,6 +2017,10 @@ without the dedicated syntax, as documented below. Parameter specifications can now be declared using the :ref:`type parameter <type-params>` syntax introduced by :pep:`695`. + .. versionchanged:: 3.13 + + Support for default values was added. + .. note:: Only parameter specification variables defined in global scope can be pickled. @@ -1938,7 +2032,7 @@ without the dedicated syntax, as documented below. * :ref:`annotating-callables` .. data:: ParamSpecArgs -.. data:: ParamSpecKwargs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, @@ -1950,7 +2044,7 @@ without the dedicated syntax, as documented below. .. doctest:: - >>> from typing import ParamSpec + >>> from typing import ParamSpec, get_origin >>> P = ParamSpec("P") >>> get_origin(P.args) is P True @@ -2406,7 +2500,7 @@ types. This attribute reflects *only* the value of the ``total`` argument to the current ``TypedDict`` class, not whether the class is semantically - total. For example, a ``TypedDict`` with ``__total__`` set to True may + total. For example, a ``TypedDict`` with ``__total__`` set to ``True`` may have keys marked with :data:`NotRequired`, or it may inherit from another ``TypedDict`` with ``total=False``. Therefore, it is generally better to use :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. @@ -2454,6 +2548,22 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. + Support for :data:`ReadOnly` is reflected in the following attributes: + + .. attribute:: __readonly_keys__ + + A :class:`frozenset` containing the names of all read-only keys. Keys + are read-only if they carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + + .. attribute:: __mutable_keys__ + + A :class:`frozenset` containing the names of all mutable keys. Keys + are mutable if they do not carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 @@ -2468,6 +2578,9 @@ types. .. versionchanged:: 3.13 Removed support for the keyword-argument method of creating ``TypedDict``\ s. + .. versionchanged:: 3.13 + Support for the :data:`ReadOnly` qualifier was added. + .. deprecated-removed:: 3.13 3.15 When using the functional syntax to create a TypedDict class, failing to pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is @@ -2603,10 +2716,10 @@ Functions and decorators .. function:: reveal_type(obj, /) - Reveal the inferred static type of an expression. + Ask a static type checker to reveal the inferred type of an expression. When a static type checker encounters a call to this function, - it emits a diagnostic with the type of the argument. For example:: + it emits a diagnostic with the inferred type of the argument. For example:: x: int = 1 reveal_type(x) # Revealed type is "builtins.int" @@ -2614,22 +2727,21 @@ Functions and decorators This can be useful when you want to debug how your type checker handles a particular piece of code. - The function returns its argument unchanged, which allows using - it within an expression:: + At runtime, this function prints the runtime type of its argument to + :data:`sys.stderr` and returns the argument unchanged (allowing the call to + be used within an expression):: - x = reveal_type(1) # Revealed type is "builtins.int" + x = reveal_type(1) # prints "Runtime type is int" + print(x) # prints "1" + + Note that the runtime type may be different from (more or less specific + than) the type statically inferred by a type checker. Most type checkers support ``reveal_type()`` anywhere, even if the name is not imported from ``typing``. Importing the name from - ``typing`` allows your code to run without runtime errors and + ``typing``, however, allows your code to run without runtime errors and communicates intent more clearly. - At runtime, this function prints the runtime type of its argument to stderr - and returns it unchanged:: - - x = reveal_type(1) # prints "Runtime type is int" - print(x) # prints "1" - .. versionadded:: 3.11 .. decorator:: dataclass_transform(*, eq_default=True, order_default=False, \ @@ -2772,6 +2884,8 @@ Functions and decorators .. versionadded:: 3.11 +.. _overload: + .. decorator:: overload Decorator for creating overloaded functions and methods. @@ -2968,33 +3082,37 @@ Introspection helpers Return a dictionary containing type hints for a function, method, module or class object. - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: - - .. testcode:: - - class Student(NamedTuple): - name: Annotated[str, 'some marker'] - - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + This is often the same as ``obj.__annotations__``, but this function makes + the following changes to the annotations dictionary: + + * Forward references encoded as string literals or :class:`ForwardRef` + objects are handled by evaluating them in *globalns*, *localns*, and + (where applicable) *obj*'s :ref:`type parameter <type-params>` namespace. + If *globalns* or *localns* is not given, appropriate namespace + dictionaries are inferred from *obj*. + * ``None`` is replaced with :class:`types.NoneType`. + * If :func:`@no_type_check <no_type_check>` has been applied to *obj*, an + empty dictionary is returned. + * If *obj* is a class ``C``, the function returns a dictionary that merges + annotations from ``C``'s base classes with those on ``C`` directly. This + is done by traversing ``C.__mro__`` and iteratively combining + ``__annotations__`` dictionaries. Annotations on classes appearing + earlier in the :term:`method resolution order` always take precedence over + annotations on classes appearing later in the method resolution order. + * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + with ``T``, unless *include_extras* is set to ``True`` (see + :class:`Annotated` for more information). + + See also :func:`inspect.get_annotations`, a lower-level function that + returns annotations more directly. .. note:: - :func:`get_type_hints` does not work with imported - :ref:`type aliases <type-aliases>` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. + If any forward references in the annotations of *obj* are not resolvable + or are not valid Python code, this function will raise an exception + such as :exc:`NameError`. For example, this can happen with imported + :ref:`type aliases <type-aliases>` that include forward references, + or with names imported under :data:`if TYPE_CHECKING <TYPE_CHECKING>`. .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. @@ -3053,14 +3171,14 @@ Introspection helpers Return the set of members defined in a :class:`Protocol`. - :: + .. doctest:: >>> from typing import Protocol, get_protocol_members >>> class P(Protocol): ... def a(self) -> str: ... ... b: int - >>> get_protocol_members(P) - frozenset({'a', 'b'}) + >>> get_protocol_members(P) == frozenset({'a', 'b'}) + True Raise :exc:`TypeError` for arguments that are not Protocols. @@ -3117,6 +3235,22 @@ Introspection helpers .. versionadded:: 3.7.4 +.. data:: NoDefault + + A sentinel object used to indicate that a type parameter has no default + value. For example: + + .. doctest:: + + >>> T = TypeVar("T") + >>> T.__default__ is typing.NoDefault + True + >>> S = TypeVar("S", default=None) + >>> S.__default__ is None + True + + .. versionadded:: 3.13 + Constant -------- @@ -3290,7 +3424,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.ChainMap`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3301,7 +3434,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.Counter`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3312,7 +3444,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.deque`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3375,19 +3506,11 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: ByteString(Sequence[int]) - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. - - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. - .. versionadded:: 3.6.0 + .. versionadded:: 3.6 .. deprecated:: 3.9 :class:`collections.abc.Collection` now supports subscripting (``[]``). @@ -3521,8 +3644,14 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` is no ``ReturnType`` type parameter. As with :class:`Generator`, the ``SendType`` behaves contravariantly. - If your generator will only yield values, set the ``SendType`` to - ``None``:: + The ``SendType`` defaults to :const:`!None`:: + + async def infinite_stream(start: int) -> AsyncGenerator[int]: + while True: + yield start + start = await increment(start) + + It is also possible to set this type explicitly:: async def infinite_stream(start: int) -> AsyncGenerator[int, None]: while True: @@ -3544,6 +3673,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + The ``SendType`` parameter now has a default. + .. class:: AsyncIterable(Generic[T_co]) Deprecated alias to :class:`collections.abc.AsyncIterable`. @@ -3627,8 +3759,14 @@ Aliases to other ABCs in :mod:`collections.abc` of :class:`Generator` behaves contravariantly, not covariantly or invariantly. - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`:: + + def infinite_stream(start: int) -> Generator[int]: + while True: + yield start + start += 1 + + It is also possible to set these types explicitly:: def infinite_stream(start: int) -> Generator[int, None, None]: while True: @@ -3647,6 +3785,9 @@ Aliases to other ABCs in :mod:`collections.abc` :class:`collections.abc.Generator` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Default values for the send and return types were added. + .. class:: Hashable Deprecated alias to :class:`collections.abc.Hashable`. @@ -3674,23 +3815,34 @@ Aliases to other ABCs in :mod:`collections.abc` Aliases to :mod:`contextlib` ABCs """"""""""""""""""""""""""""""""" -.. class:: ContextManager(Generic[T_co]) +.. class:: ContextManager(Generic[T_co, ExitT_co]) Deprecated alias to :class:`contextlib.AbstractContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__enter__` method. The optional second type parameter, ``ExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__exit__` method. + .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 .. deprecated:: 3.9 :class:`contextlib.AbstractContextManager` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: AsyncContextManager(Generic[T_co]) + .. versionchanged:: 3.13 + Added the optional second type parameter, ``ExitT_co``. + +.. class:: AsyncContextManager(Generic[T_co, AExitT_co]) Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.5.4 + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__aenter__` method. The optional second type parameter, ``AExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__aexit__` method. + .. versionadded:: 3.6.2 .. deprecated:: 3.9 @@ -3698,6 +3850,9 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Added the optional second type parameter, ``AExitT_co``. + Deprecation Timeline of Major Features ====================================== @@ -3716,10 +3871,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` - * - :class:`typing.ByteString` - - 3.9 - - 3.14 - - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 7db47d48022a0e..37dc37513fa15d 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -1,5 +1,5 @@ -:mod:`unicodedata` --- Unicode Database -======================================= +:mod:`!unicodedata` --- Unicode Database +======================================== .. module:: unicodedata :synopsis: Access the Unicode Database. diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f2bdde80bdbd64..00cc9bfc0a5f2b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1,5 +1,5 @@ -:mod:`unittest.mock` --- getting started -======================================== +:mod:`!unittest.mock` --- getting started +========================================= .. moduleauthor:: Michael Foord <michael@python.org> .. currentmodule:: unittest.mock diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ea6412abf72d1a..01206e05f4ba01 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1,6 +1,5 @@ - -:mod:`unittest.mock` --- mock object library -============================================ +:mod:`!unittest.mock` --- mock object library +============================================= .. module:: unittest.mock :synopsis: Mock object library. @@ -35,7 +34,7 @@ is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI <https://pypi.org/project/mock>`_. +available as :pypi:`mock` on PyPI. Quick Guide @@ -415,13 +414,13 @@ the *new_callable* argument to :func:`patch`. This can be useful where you want to make a series of assertions that reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have + :attr:`return_value`, :attr:`side_effect` or any child attributes you have set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding + :attr:`return_value` or :attr:`side_effect`, then pass the corresponding parameter as ``True``. Child mocks and the return value mock (if any) are reset as well. - .. note:: *return_value*, and :attr:`side_effect` are keyword-only + .. note:: *return_value*, and *side_effect* are keyword-only arguments. @@ -824,8 +823,9 @@ apply to method calls on the mock object. .. class:: PropertyMock(*args, **kwargs) - A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods + A mock intended to be used as a :class:`property`, or other + :term:`descriptor`, on a class. :class:`PropertyMock` provides + :meth:`~object.__get__` and :meth:`~object.__set__` methods so you can specify a return value when it is fetched. Fetching a :class:`PropertyMock` instance from an object calls the mock, with @@ -1624,7 +1624,8 @@ patch.dict .. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is a + copy of the dictionary as it was before the test. *in_dict* can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -1707,8 +1708,9 @@ Keywords can be used in the :func:`patch.dict` call to set values in the diction :func:`patch.dict` can be used with dictionary like objects that aren't actually dictionaries. At the very minimum they must support item getting, setting, deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. +magic methods :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__` and either :meth:`~container.__iter__` or +:meth:`~object.__contains__`. >>> class Container: ... def __init__(self): @@ -2007,8 +2009,8 @@ Mocking Magic Methods ~~~~~~~~~~~~~~~~~~~~~ :class:`Mock` supports mocking the Python protocol methods, also known as -"magic methods". This allows mock objects to replace containers or other -objects that implement Python protocols. +:term:`"magic methods" <magic method>`. This allows mock objects to replace +containers or other objects that implement Python protocols. Because magic methods are looked up differently from normal methods [#]_, this support has been specially implemented. This means that only specific magic @@ -2106,8 +2108,8 @@ There are two ``MagicMock`` variants: :class:`MagicMock` and :class:`NonCallable .. class:: MagicMock(*args, **kw) ``MagicMock`` is a subclass of :class:`Mock` with default implementations - of most of the magic methods. You can use ``MagicMock`` without having to - configure the magic methods yourself. + of most of the :term:`magic methods <magic method>`. You can use + ``MagicMock`` without having to configure the magic methods yourself. The constructor parameters have the same meaning as for :class:`Mock`. @@ -2141,10 +2143,10 @@ to change the default. Methods and their defaults: -* ``__lt__``: ``NotImplemented`` -* ``__gt__``: ``NotImplemented`` -* ``__le__``: ``NotImplemented`` -* ``__ge__``: ``NotImplemented`` +* ``__lt__``: :data:`NotImplemented` +* ``__gt__``: :data:`!NotImplemented` +* ``__le__``: :data:`!NotImplemented` +* ``__ge__``: :data:`!NotImplemented` * ``__int__``: ``1`` * ``__contains__``: ``False`` * ``__len__``: ``0`` @@ -2171,7 +2173,7 @@ For example: >>> object() in mock False -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. +The two equality methods, :meth:`!__eq__` and :meth:`!__ne__`, are special. They do the default equality comparison on identity, using the :attr:`~Mock.side_effect` attribute, unless you change their return value to return something else:: @@ -2424,6 +2426,14 @@ passed in. >>> m.mock_calls == [call(1), call(1, 2), ANY] True +:data:`ANY` is not limited to comparisons with call objects and so +can also be used in test assertions:: + + class TestStringMethods(unittest.TestCase): + + def test_split(self): + s = 'hello world' + self.assertEqual(s.split(), ['hello', ANY]) FILTER_DIR @@ -2521,8 +2531,8 @@ mock_open *read_data* is now reset on each call to the *mock*. .. versionchanged:: 3.8 - Added :meth:`__iter__` to implementation so that iteration (such as in for - loops) correctly consumes *read_data*. + Added :meth:`~container.__iter__` to implementation so that iteration + (such as in for loops) correctly consumes *read_data*. Using :func:`open` as a context manager is a great way to ensure your file handles are closed properly and is becoming common:: @@ -2531,8 +2541,8 @@ are closed properly and is becoming common:: f.write('something') The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`__enter__` and -:meth:`__exit__` called). +*returned object* that is used as a context manager (and has :meth:`~object.__enter__` and +:meth:`~object.__exit__` called). Mocking context managers with a :class:`MagicMock` is common enough and fiddly enough that a helper function is useful. :: @@ -2575,40 +2585,16 @@ called incorrectly. Before I explain how auto-speccing works, here's why it is needed. -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. +:class:`Mock` is a very powerful and flexible object, but it suffers from a flaw which +is general to mocking. If you refactor some of your code, rename members and so on, any +tests for code that is still using the *old api* but uses mocks instead of the real +objects will still pass. This means your tests can all pass even though your code is +broken. - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. +.. versionchanged:: 3.5 -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) # Intentional typo! - -Your tests can pass silently and incorrectly because of the typo. - -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. + Before 3.5, tests with a typo in the word assert would silently pass when they should + raise an error. You can still achieve this behavior by passing ``unsafe=True`` to Mock. Note that this is another reason why you need integration tests as well as unit tests. Testing everything in isolation is all fine and dandy, but if you @@ -2704,7 +2690,7 @@ able to use autospec. On the other hand it is much better to design your objects so that introspection is safe [#]_. A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. +created in the :meth:`~object.__init__` method and not to exist on the class at all. *autospec* can't know about any dynamically created attributes and restricts the api to visible attributes. :: @@ -2745,8 +2731,9 @@ this particular scenario: AttributeError: Mock object has no attribute 'a' Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via +default values for instance members initialised in :meth:`~object.__init__`. +Note that if +you are only setting default attributes in :meth:`!__init__` then providing them via class attributes (shared between instances of course) is faster too. e.g. .. code-block:: python @@ -2820,3 +2807,123 @@ Sealing mocks >>> mock.not_submock.attribute2 # This won't raise. .. versionadded:: 3.7 + + +Order of precedence of :attr:`side_effect`, :attr:`return_value` and *wraps* +---------------------------------------------------------------------------- + +The order of their precedence is: + +1. :attr:`~Mock.side_effect` +2. :attr:`~Mock.return_value` +3. *wraps* + +If all three are set, mock will return the value from :attr:`~Mock.side_effect`, +ignoring :attr:`~Mock.return_value` and the wrapped object altogether. If any +two are set, the one with the higher precedence will return the value. +Regardless of the order of which was set first, the order of precedence +remains unchanged. + + >>> from unittest.mock import Mock + >>> class Order: + ... @staticmethod + ... def get_value(): + ... return "third" + ... + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first"] + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'first' + +As ``None`` is the default value of :attr:`~Mock.side_effect`, if you reassign +its value back to ``None``, the order of precedence will be checked between +:attr:`~Mock.return_value` and the wrapped object, ignoring +:attr:`~Mock.side_effect`. + + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + +If the value being returned by :attr:`~Mock.side_effect` is :data:`DEFAULT`, +it is ignored and the order of precedence moves to the successor to obtain the +value to return. + + >>> from unittest.mock import DEFAULT + >>> order_mock.get_value.side_effect = [DEFAULT] + >>> order_mock.get_value() + 'second' + +When :class:`Mock` wraps an object, the default value of +:attr:`~Mock.return_value` will be :data:`DEFAULT`. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.return_value + sentinel.DEFAULT + >>> order_mock.get_value.return_value + sentinel.DEFAULT + +The order of precedence will ignore this value and it will move to the last +successor which is the wrapped object. + +As the real call is being made to the wrapped object, creating an instance of +this mock will return the real instance of the class. The positional arguments, +if any, required by the wrapped object must be passed. + + >>> order_mock_instance = order_mock() + >>> isinstance(order_mock_instance, Order) + True + >>> order_mock_instance.get_value() + 'third' + + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'second' + +But if you assign ``None`` to it, this will not be ignored as it is an +explicit assignment. So, the order of precedence will not move to the wrapped +object. + + >>> order_mock.get_value.return_value = None + >>> order_mock.get_value() is None + True + +Even if you set all three at once when initializing the mock, the order of +precedence remains the same: + + >>> order_mock = Mock(spec=Order, wraps=Order, + ... **{"get_value.side_effect": ["first"], + ... "get_value.return_value": "second"} + ... ) + ... + >>> order_mock.get_value() + 'first' + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + +If :attr:`~Mock.side_effect` is exhausted, the order of precedence will not +cause a value to be obtained from the successors. Instead, ``StopIteration`` +exception is raised. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first side effect value", + ... "another side effect value"] + >>> order_mock.get_value.return_value = "second" + + >>> order_mock.get_value() + 'first side effect value' + >>> order_mock.get_value() + 'another side effect value' + + >>> order_mock.get_value() + Traceback (most recent call last): + ... + StopIteration diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 21abc583f853a7..dc76374d5181eb 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1,5 +1,5 @@ -:mod:`unittest` --- Unit testing framework -========================================== +:mod:`!unittest` --- Unit testing framework +=========================================== .. module:: unittest :synopsis: Unit testing framework for Python. @@ -346,8 +346,8 @@ the `load_tests protocol`_. ``python -m unittest discover -s root/namespace -t root``). .. versionchanged:: 3.11 - Python 3.11 dropped the :term:`namespace packages <namespace package>` - support. It has been broken since Python 3.7. Start directory and + :mod:`unittest` dropped the :term:`namespace packages <namespace package>` + support in Python 3.11. It has been broken since Python 3.7. Start directory and subdirectories containing tests must be regular package that have ``__init__.py`` file. @@ -390,8 +390,8 @@ testing code:: widget = Widget('The widget') self.assertEqual(widget.size(), (50, 50)) -Note that in order to test something, we use one of the :meth:`assert\*` -methods provided by the :class:`TestCase` base class. If the test fails, an +Note that in order to test something, we use one of the :ref:`assert\* methods <assert-methods>` +provided by the :class:`TestCase` base class. If the test fails, an exception will be raised with an explanatory message, and :mod:`unittest` will identify the test case as a :dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. @@ -1571,6 +1571,14 @@ Test cases .. versionadded:: 3.8 + .. attribute:: loop_factory + + The *loop_factory* passed to :class:`asyncio.Runner`. Override + in subclasses with :class:`asyncio.EventLoop` to avoid using the + asyncio policy system. + + .. versionadded:: 3.13 + .. coroutinemethod:: asyncSetUp() Method called to prepare the test fixture. This is called after :meth:`setUp`. @@ -1725,7 +1733,7 @@ Grouping tests .. method:: __iter__() Tests grouped by a :class:`TestSuite` are always accessed by iteration. - Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note + Subclasses can lazily provide tests by overriding :meth:`!__iter__`. Note that this method may be called several times on a single suite (for example when counting tests or comparing for equality) so the tests returned by repeated iterations before :meth:`TestSuite.run` must be the @@ -1736,7 +1744,7 @@ Grouping tests .. versionchanged:: 3.2 In earlier versions the :class:`TestSuite` accessed tests directly rather - than through iteration, so overriding :meth:`__iter__` wasn't sufficient + than through iteration, so overriding :meth:`!__iter__` wasn't sufficient for providing tests. .. versionchanged:: 3.4 @@ -1872,8 +1880,8 @@ Loading and running tests Python identifiers) will be loaded. All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. + the start directory is not the top level directory then *top_level_dir* + must be specified separately. If importing a module fails, for example due to a syntax error, then this will be recorded as a single error and discovery will continue. If @@ -1893,9 +1901,11 @@ Loading and running tests package. The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. + packages can continue discovery themselves. + + *top_level_dir* is stored internally, and used as a default to any + nested calls to ``discover()``. That is, if a package's ``load_tests`` + calls ``loader.discover()``, it does not need to pass this argument. *start_dir* can be a dotted module name as well as a directory. @@ -1922,6 +1932,9 @@ Loading and running tests *start_dir* can not be a :term:`namespace packages <namespace package>`. It has been broken since Python 3.7 and Python 3.11 officially remove it. + .. versionchanged:: 3.13 + *top_level_dir* is only stored for the duration of *discover* call. + The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -1932,14 +1945,14 @@ Loading and running tests String giving the prefix of method names which will be interpreted as test methods. The default value is ``'test'``. - This affects :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` + This affects :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: sortTestMethodsUsing Function to be used to compare method names when sorting them in - :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` methods. + :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: suiteClass @@ -1948,7 +1961,7 @@ Loading and running tests methods on the resulting object are needed. The default value is the :class:`TestSuite` class. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. attribute:: testNamePatterns @@ -1961,7 +1974,7 @@ Loading and running tests so unlike patterns passed to the ``-k`` option, simple substring patterns will have to be converted using ``*`` wildcards. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. versionadded:: 3.7 @@ -1995,7 +2008,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :ref:`assert\* methods <assert-methods>`. .. attribute:: skipped @@ -2188,8 +2201,8 @@ Loading and running tests .. versionadded:: 3.2 - .. versionadded:: 3.12 - Added *durations* keyword argument. + .. versionchanged:: 3.12 + Added the *durations* keyword parameter. .. data:: defaultTestLoader @@ -2282,7 +2295,7 @@ Loading and running tests The *testRunner* argument can either be a test runner class or an already created instance of it. By default ``main`` calls :func:`sys.exit` with an exit code indicating success (0) or failure (1) of the tests run. - An exit code of 5 indicates that no tests were run. + An exit code of 5 indicates that no tests were run or skipped. The *testLoader* argument has to be a :class:`TestLoader` instance, and defaults to :data:`defaultTestLoader`. @@ -2303,8 +2316,8 @@ Loading and running tests (see :ref:`Warning control <using-on-warnings>`), otherwise it will be set to ``'default'``. - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. + Calling ``main`` returns an object with the ``result`` attribute that contains + the result of the tests run as a :class:`unittest.TestResult`. .. versionchanged:: 3.1 The *exit* parameter was added. @@ -2318,6 +2331,8 @@ Loading and running tests test names. +.. _load_tests-protocol: + load_tests Protocol ################### diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index facb11f42a40c5..1686ddd09caa48 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -1,5 +1,5 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== +:mod:`!urllib.error` --- Exception classes raised by urllib.request +=================================================================== .. module:: urllib.error :synopsis: Exception classes raised by urllib.request. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 53e5f0395715d7..fb5353e1895bf9 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -1,5 +1,5 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== +:mod:`!urllib.parse` --- Parse URLs into components +=================================================== .. module:: urllib.parse :synopsis: Parse URLs into or assemble them from components. @@ -22,15 +22,28 @@ to an absolute URL given a "base URL." The module has been designed to match the internet RFC on Relative Uniform Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, +``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``itms-services``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, ``wais``, ``ws``, ``wss``. +.. impl-detail:: + + The inclusion of the ``itms-services`` URL scheme can prevent an app from + passing Apple's App Store review process for the macOS and iOS App Stores. + Handling for the ``itms-services`` scheme is always removed on iOS; on + macOS, it *may* be removed if CPython has been built with the + :option:`--with-app-store-compliance` option. + The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. +This module's functions use the deprecated term ``netloc`` (or ``net_loc``), +which was introduced in :rfc:`1808`. However, this term has been obsoleted by +:rfc:`3986`, which introduced the term ``authority`` as its replacement. +The use of ``netloc`` is continued for backward compatibility. + URL Parsing ----------- @@ -168,7 +181,7 @@ or on combining URL components into a URL string. Added IPv6 URL parsing capabilities. .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is + The fragment is now parsed for all URL schemes (unless *allow_fragments* is false), in accordance with :rfc:`3986`. Previously, an allowlist of schemes that support fragments existed. @@ -729,8 +742,8 @@ task isn't already covered by the URL parsing functions above. .. versionchanged:: 3.2 *query* supports bytes and string objects. - .. versionadded:: 3.5 - *quote_via* parameter. + .. versionchanged:: 3.5 + Added the *quote_via* parameter. .. seealso:: diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index bf3af1bef0714c..754405e0fbe5b2 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1,5 +1,5 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= +:mod:`!urllib.request` --- Extensible library for opening URLs +============================================================== .. module:: urllib.request :synopsis: Extensible library for opening URLs. @@ -21,6 +21,14 @@ authentication, redirections, cookies and more. The `Requests package <https://requests.readthedocs.io/en/master/>`_ is recommended for a higher-level HTTP client interface. +.. warning:: + + On macOS it is unsafe to use this module in programs using + :func:`os.fork` because the :func:`getproxies` implementation for + macOS uses a higher-level system API. Set the environment variable + ``no_proxy`` to ``*`` to avoid this problem + (e.g. ``os.environ["no_proxy"] = "*"``). + .. include:: ../includes/wasm-notavail.rst The :mod:`urllib.request` module defines the following functions: @@ -70,7 +78,7 @@ The :mod:`urllib.request` module defines the following functions: :class:`UnknownHandler` to ensure this never happens). In addition, if proxy settings are detected (for example, when a ``*_proxy`` - environment variable like :envvar:`http_proxy` is set), + environment variable like :envvar:`!http_proxy` is set), :class:`ProxyHandler` is default installed and makes sure the requests are handled through the proxy. @@ -89,11 +97,9 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.2 *cafile* and *capath* were added. - .. versionchanged:: 3.2 HTTPS virtual hosts are now supported if possible (that is, if :const:`ssl.HAS_SNI` is true). - .. versionadded:: 3.2 *data* can be an iterable object. .. versionchanged:: 3.3 @@ -105,11 +111,11 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.10 HTTPS connection now send an ALPN extension with protocol indicator ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. - .. versionchanged:: 3.13 - Remove *cafile*, *capath* and *cadefault* parameters: use the *context* - parameter instead. + .. versionchanged:: 3.13 + Remove *cafile*, *capath* and *cadefault* parameters: use the *context* + parameter instead. .. function:: install_opener(opener) @@ -212,7 +218,7 @@ The following classes are provided: An appropriate ``Content-Type`` header should be included if the *data* argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will + is not ``None``, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. The next two arguments are only of interest for correct handling @@ -610,25 +616,25 @@ OpenerDirector Objects the actual HTTP code, for example :meth:`http_error_404` would handle HTTP 404 errors. - * :meth:`<protocol>_open` --- signal that the handler knows how to open *protocol* + * :meth:`!<protocol>_open` --- signal that the handler knows how to open *protocol* URLs. See |protocol_open|_ for more information. - * :meth:`http_error_\<type\>` --- signal that the handler knows how to handle HTTP + * :meth:`!http_error_\<type\>` --- signal that the handler knows how to handle HTTP errors with HTTP error code *type*. See |http_error_nnn|_ for more information. - * :meth:`<protocol>_error` --- signal that the handler knows how to handle errors + * :meth:`!<protocol>_error` --- signal that the handler knows how to handle errors from (non-\ ``http``) *protocol*. - * :meth:`<protocol>_request` --- signal that the handler knows how to pre-process + * :meth:`!<protocol>_request` --- signal that the handler knows how to pre-process *protocol* requests. See |protocol_request|_ for more information. - * :meth:`<protocol>_response` --- signal that the handler knows how to + * :meth:`!<protocol>_response` --- signal that the handler knows how to post-process *protocol* responses. See |protocol_response|_ for more information. @@ -655,7 +661,7 @@ OpenerDirector Objects Handle an error of the given protocol. This will call the registered error handlers for the given protocol with the given arguments (which are protocol specific). The HTTP protocol is a special case which uses the HTTP response - code to determine the specific error handler; refer to the :meth:`http_error_\<type\>` + code to determine the specific error handler; refer to the :meth:`!http_error_\<type\>` methods of the handler classes. Return values and exceptions raised are the same as those of :func:`urlopen`. @@ -665,25 +671,25 @@ OpenerDirector objects open URLs in three stages: The order in which these methods are called within each stage is determined by sorting the handler instances. -#. Every handler with a method named like :meth:`<protocol>_request` has that +#. Every handler with a method named like :meth:`!<protocol>_request` has that method called to pre-process the request. -#. Handlers with a method named like :meth:`<protocol>_open` are called to handle +#. Handlers with a method named like :meth:`!<protocol>_open` are called to handle the request. This stage ends when a handler either returns a non-\ :const:`None` value (ie. a response), or raises an exception (usually :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. In fact, the above algorithm is first tried for methods named - :meth:`default_open`. If all such methods return :const:`None`, the algorithm - is repeated for methods named like :meth:`<protocol>_open`. If all such methods + :meth:`~BaseHandler.default_open`. If all such methods return :const:`None`, the algorithm + is repeated for methods named like :meth:`!<protocol>_open`. If all such methods return :const:`None`, the algorithm is repeated for methods named - :meth:`unknown_open`. + :meth:`~BaseHandler.unknown_open`. Note that the implementation of these methods may involve calls of the parent :class:`OpenerDirector` instance's :meth:`~OpenerDirector.open` and :meth:`~OpenerDirector.error` methods. -#. Every handler with a method named like :meth:`<protocol>_response` has that +#. Every handler with a method named like :meth:`!<protocol>_response` has that method called to post-process the response. @@ -712,8 +718,8 @@ The following attribute and methods should only be used by classes derived from .. note:: The convention has been adopted that subclasses defining - :meth:`<protocol>_request` or :meth:`<protocol>_response` methods are named - :class:`\*Processor`; all others are named :class:`\*Handler`. + :meth:`!<protocol>_request` or :meth:`!<protocol>_response` methods are named + :class:`!\*Processor`; all others are named :class:`!\*Handler`. .. attribute:: BaseHandler.parent @@ -732,7 +738,7 @@ The following attribute and methods should only be used by classes derived from the return value of the :meth:`~OpenerDirector.open` method of :class:`OpenerDirector`, or ``None``. It should raise :exc:`~urllib.error.URLError`, unless a truly exceptional thing happens (for example, :exc:`MemoryError` should not be mapped to - :exc:`URLError`). + :exc:`~urllib.error.URLError`). This method will be called before any protocol-specific open method. @@ -745,7 +751,7 @@ The following attribute and methods should only be used by classes derived from define it if they want to handle URLs with the given protocol. This method, if defined, will be called by the parent :class:`OpenerDirector`. - Return values should be the same as for :meth:`default_open`. + Return values should be the same as for :meth:`~BaseHandler.default_open`. .. method:: BaseHandler.unknown_open(req) @@ -785,7 +791,7 @@ The following attribute and methods should only be used by classes derived from Subclasses should override this method to handle specific HTTP errors. Arguments, return values and exceptions raised should be the same as for - :meth:`http_error_default`. + :meth:`~BaseHandler.http_error_default`. .. _protocol_request: @@ -825,7 +831,7 @@ HTTPRedirectHandler Objects is the case, :exc:`~urllib.error.HTTPError` is raised. See :rfc:`2616` for details of the precise meanings of the various redirection codes. - An :class:`HTTPError` exception raised as a security consideration if the + An :exc:`~urllib.error.HTTPError` exception raised as a security consideration if the HTTPRedirectHandler is presented with a redirected URL which is not an HTTP, HTTPS or FTP URL. @@ -833,9 +839,9 @@ HTTPRedirectHandler Objects .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) Return a :class:`Request` or ``None`` in response to a redirect. This is called - by the default implementations of the :meth:`http_error_30\*` methods when a + by the default implementations of the :meth:`!http_error_30\*` methods when a redirection is received from the server. If a redirection should take place, - return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the + return a new :class:`Request` to allow :meth:`!http_error_30\*` to perform the redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if no other handler should try to handle this URL, or return ``None`` if you can't but another handler might. @@ -902,7 +908,7 @@ ProxyHandler Objects .. method:: ProxyHandler.<protocol>_open(request) :noindex: - The :class:`ProxyHandler` will have a method :meth:`<protocol>_open` for every + The :class:`ProxyHandler` will have a method :meth:`!<protocol>_open` for every *protocol* which has a proxy in the *proxies* dictionary given in the constructor. The method will modify requests to go through the proxy, by calling ``request.set_proxy()``, and call the next handler in the chain to @@ -1158,7 +1164,7 @@ HTTPErrorProcessor Objects For 200 error codes, the response object is returned immediately. For non-200 error codes, this simply passes the job on to the - :meth:`http_error_\<type\>` handler methods, via :meth:`OpenerDirector.error`. + :meth:`!http_error_\<type\>` handler methods, via :meth:`OpenerDirector.error`. Eventually, :class:`HTTPDefaultErrorHandler` will raise an :exc:`~urllib.error.HTTPError` if no other handler handles the error. @@ -1265,7 +1271,7 @@ Use of Basic HTTP Authentication:: :func:`build_opener` provides many handlers by default, including a :class:`ProxyHandler`. By default, :class:`ProxyHandler` uses the environment variables named ``<scheme>_proxy``, where ``<scheme>`` is the URL scheme -involved. For example, the :envvar:`http_proxy` environment variable is read to +involved. For example, the :envvar:`!http_proxy` environment variable is read to obtain the HTTP proxy's URL. This example replaces the default :class:`ProxyHandler` with one that uses @@ -1360,7 +1366,7 @@ some point in the future. points to a local file, the object will not be copied unless filename is supplied. Return a tuple ``(filename, headers)`` where *filename* is the local file name under which the object can be found, and *headers* is whatever - the :meth:`info` method of the object returned by :func:`urlopen` returned (for + the :meth:`!info` method of the object returned by :func:`urlopen` returned (for a remote object). Exceptions are the same as for :func:`urlopen`. The second argument, if present, specifies the file location to copy to (if @@ -1385,7 +1391,7 @@ some point in the future. :mimetype:`application/x-www-form-urlencoded` format; see the :func:`urllib.parse.urlencode` function. - :func:`urlretrieve` will raise :exc:`ContentTooShortError` when it detects that + :func:`urlretrieve` will raise :exc:`~urllib.error.ContentTooShortError` when it detects that the amount of data available was less than the expected amount (which is the size reported by a *Content-Length* header). This can occur, for example, when the download is interrupted. @@ -1394,8 +1400,8 @@ some point in the future. urlretrieve reads more data, but if less data is available, it raises the exception. - You can still retrieve the downloaded data in this case, it is stored in the - :attr:`content` attribute of the exception instance. + You can still retrieve the downloaded data in this case, it is stored in the + :attr:`!content` attribute of the exception instance. If no *Content-Length* header was supplied, urlretrieve can not check the size of the data it has downloaded, and just returns it. In this case you just have @@ -1489,7 +1495,7 @@ some point in the future. authentication is performed. For the 30x response codes, recursion is bounded by the value of the *maxtries* attribute, which defaults to 10. - For all other response codes, the method :meth:`http_error_default` is called + For all other response codes, the method :meth:`~BaseHandler.http_error_default` is called which you can override in subclasses to handle the error appropriately. .. note:: diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f063e463753e0b..016fcdc75da67a 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -1,11 +1,11 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== +:mod:`!urllib.robotparser` --- Parser for robots.txt +===================================================== .. module:: urllib.robotparser :synopsis: Load a robots.txt file and answer questions about fetchability of other URLs. -.. sectionauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com> **Source code:** :source:`Lib/urllib/robotparser.py` diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 624e164625556a..7d9f39ef070fc3 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -1,5 +1,5 @@ -:mod:`urllib` --- URL handling modules -====================================== +:mod:`!urllib` --- URL handling modules +======================================= .. module:: urllib diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index e2d231da38fd9a..0f2d7820cb25c8 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -1,5 +1,5 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== +:mod:`!uuid` --- UUID objects according to :rfc:`4122` +====================================================== .. module:: uuid :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index da8942c554dea1..fff1075c2473eb 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -1,5 +1,5 @@ -:mod:`venv` --- Creation of virtual environments -================================================ +:mod:`!venv` --- Creation of virtual environments +================================================= .. module:: venv :synopsis: Creation of virtual environments. @@ -27,7 +27,7 @@ optionally be isolated from the packages in the base environment, so only those explicitly installed in the virtual environment are available. When used from within a virtual environment, common installation tools such as -`pip`_ will install Python packages into a virtual environment +:pypi:`pip` will install Python packages into a virtual environment without needing to be told to do so explicitly. A virtual environment is (amongst other things): @@ -54,9 +54,9 @@ See :pep:`405` for more background on Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - <https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment>`__ + <https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments>`__ -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-ios-notavail.rst Creating virtual environments ----------------------------- @@ -201,13 +201,13 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.4 Added the ``with_pip`` parameter - .. versionadded:: 3.6 + .. versionchanged:: 3.6 Added the ``prompt`` parameter - .. versionadded:: 3.9 + .. versionchanged:: 3.9 Added the ``upgrade_deps`` parameter - .. versionadded:: 3.13 + .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter Creators of third-party virtual environment tools will be free to use the @@ -286,15 +286,15 @@ creation according to their needs, the :class:`EnvBuilder` class. the virtual environment. - .. versionchanged:: 3.12 - The attribute ``lib_path`` was added to the context, and the context - object was documented. - .. versionchanged:: 3.11 The *venv* :ref:`sysconfig installation scheme <installation_paths>` is used to construct the paths of the created directories. + .. versionchanged:: 3.12 + The attribute ``lib_path`` was added to the context, and the context + object was documented. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. @@ -614,7 +614,3 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online <https://gist.github.com/vsajip/4673395>`_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 884de08eab1b16..68b9ff5ce2f78c 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -1,5 +1,5 @@ -:mod:`warnings` --- Warning control -=================================== +:mod:`!warnings` --- Warning control +==================================== .. module:: warnings :synopsis: Issue warning messages and control their disposition. @@ -145,6 +145,8 @@ the disposition of the match. Each entry is a tuple of the form (*action*, +---------------+----------------------------------------------+ | ``"always"`` | always print matching warnings | +---------------+----------------------------------------------+ + | ``"all"`` | alias to "always" | + +---------------+----------------------------------------------+ | ``"module"`` | print the first occurrence of matching | | | warnings for each module where the warning | | | is issued (regardless of line number) | @@ -396,7 +398,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, \*, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class <warning-categories>`; it @@ -522,6 +524,56 @@ Available Functions and calls to :func:`simplefilter`. +.. decorator:: deprecated(msg, *, category=DeprecationWarning, stacklevel=1) + + Decorator to indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, + deprecation warnings may be emitted at runtime when the object is used. + :term:`static type checkers <static type checker>` + will also generate a diagnostic on usage of the deprecated object. + + Usage:: + + from warnings import deprecated + from typing import overload + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the :func:`@overload <typing.overload>` decorator + for the attribute to exist on the overload as returned by + :func:`typing.get_overloads`. + + .. versionadded:: 3.13 + See :pep:`702`. + + Available Context Managers -------------------------- diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 55b029bc742b24..89664693cc87b4 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -1,5 +1,5 @@ -:mod:`wave` --- Read and write WAV files -======================================== +:mod:`!wave` --- Read and write WAV files +========================================= .. module:: wave :synopsis: Provide an interface to the WAV sound format. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 4667b81e38ada2..334f21f01c14c0 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -1,5 +1,5 @@ -:mod:`webbrowser` --- Convenient web-browser controller -======================================================= +:mod:`!webbrowser` --- Convenient web-browser controller +======================================================== .. module:: webbrowser :synopsis: Easy-to-use controller for web browsers. @@ -33,11 +33,21 @@ allow the remote browser to maintain its own windows on the display. If remote browsers are not available on Unix, the controlling process will launch a new browser and wait. +On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments +controlling autoraise, browser preference, and new tab/window creation will be +ignored. Web pages will *always* be opened in the user's preferred browser, in +a new tab, with the browser being brought to the foreground. The use of the +:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`ctypes` isn't available, calls to :func:`.open` will fail. + The script :program:`webbrowser` can be used as a command-line interface for the module. It accepts a URL as the argument. It accepts the following optional -parameters: ``-n`` opens the URL in a new browser window, if possible; -``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: +parameters: + +* ``-n``/``--new-window`` opens the URL in a new browser window, if possible. +* ``-t``/``--new-tab`` opens the URL in a new browser page ("tab"). + +The options are, naturally, mutually exclusive. Usage example:: python -m webbrowser -t "https://www.python.org" @@ -147,6 +157,8 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'chromium-browser'`` | ``Chromium('chromium-browser')`` | | +------------------------+-----------------------------------------+-------+ +| ``'iosbrowser'`` | ``IOSBrowser`` | \(4) | ++------------------------+-----------------------------------------+-------+ Notes: @@ -161,7 +173,10 @@ Notes: Only on Windows platforms. (3) - Only on macOS platform. + Only on macOS. + +(4) + Only on iOS. .. versionadded:: 3.2 A new :class:`!MacOSXOSAScript` class has been added @@ -176,6 +191,9 @@ Notes: Removed browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, and Firefox versions 35 and below. +.. versionchanged:: 3.13 + Support for iOS has been added. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 06bd4d87eb03c6..b3a824fb69a49f 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -1,5 +1,5 @@ -:mod:`winreg` --- Windows registry access -========================================= +:mod:`!winreg` --- Windows registry access +========================================== .. module:: winreg :platform: Windows diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 370c5216652ba7..f7ca9dc57bbe28 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -1,5 +1,5 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= +:mod:`!winsound` --- Sound-playing interface for Windows +======================================================== .. module:: winsound :platform: Windows diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 39a4c1ba142338..e46730f1716761 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -1,5 +1,5 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== +:mod:`!wsgiref` --- WSGI Utilities and Reference Implementation +=============================================================== .. module:: wsgiref :synopsis: WSGI Utilities and Reference Implementation. @@ -180,7 +180,7 @@ also provides these miscellaneous utilities: print(chunk) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. :mod:`wsgiref.headers` -- WSGI response header tools @@ -201,8 +201,9 @@ manipulation of WSGI response headers using a mapping-like interface. an empty list. :class:`Headers` objects support typical mapping operations including - :meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, - :meth:`__delitem__` and :meth:`__contains__`. For each of + :meth:`~object.__getitem__`, :meth:`~dict.get`, :meth:`~object.__setitem__`, + :meth:`~dict.setdefault`, + :meth:`~object.__delitem__` and :meth:`~object.__contains__`. For each of these methods, the key is the header name (treated case-insensitively), and the value is the first value associated with that header name. Setting a header deletes any existing values for that header, then adds a new value at the end of @@ -520,8 +521,10 @@ input, output, and error streams. want to subclass this instead of :class:`BaseCGIHandler`. This class is a subclass of :class:`BaseHandler`. It overrides the - :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, - :meth:`_write`, and :meth:`_flush` methods to support explicitly setting the + :meth:`!__init__`, :meth:`~BaseHandler.get_stdin`, + :meth:`~BaseHandler.get_stderr`, :meth:`~BaseHandler.add_cgi_vars`, + :meth:`~BaseHandler._write`, and :meth:`~BaseHandler._flush` methods to + support explicitly setting the environment and streams via the constructor. The supplied environment and streams are stored in the :attr:`stdin`, :attr:`stdout`, :attr:`stderr`, and :attr:`environ` attributes. @@ -862,7 +865,7 @@ directory and port number (default: 8000) on the command line:: fn = os.path.join(path, environ["PATH_INFO"][1:]) if "." not in fn.split(os.path.sep)[-1]: fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] + mime_type = mimetypes.guess_file_type(fn)[0] # Return 200 OK if file exists, otherwise 404 Not Found if os.path.exists(fn): diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 72a7a98c2ac4f2..00a18751207e7a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== +:mod:`!xml.dom.minidom` --- Minimal DOM implementation +====================================================== .. module:: xml.dom.minidom :synopsis: Minimal Document Object Model (DOM) implementation. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index d1df465a598e53..fd96765cbe3c96 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= +:mod:`!xml.dom.pulldom` --- Support for building partial DOM trees +================================================================== .. module:: xml.dom.pulldom :synopsis: Support for building partial DOM trees from SAX events. @@ -115,7 +115,7 @@ DOMEventStream Objects .. class:: DOMEventStream(stream, parser, bufsize) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. .. method:: getEvent() diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index b387240a3716cc..f33b19bc2724d0 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ +:mod:`!xml.dom` --- The Document Object Model API +================================================= .. module:: xml.dom :synopsis: Document Object Model API for Python. @@ -734,7 +734,7 @@ NamedNodeMap Objects attribute node. Get its value with the :attr:`value` attribute. There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`getAttribute\*` family +You can use them or you can use the standardized :meth:`!getAttribute\*` family of methods on the :class:`Element` objects. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 54c93008503c02..4c1e7bd7e6734a 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1,5 +1,5 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== +:mod:`!xml.etree.ElementTree` --- The ElementTree XML API +========================================================= .. module:: xml.etree.ElementTree :synopsis: Implementation of the ElementTree API. @@ -49,7 +49,7 @@ and its sub-elements are done on the :class:`Element` level. Parsing XML ^^^^^^^^^^^ -We'll be using the following XML document as the sample data for this section: +We'll be using the fictive :file:`country_data.xml` XML document as the sample data for this section: .. code-block:: xml @@ -154,6 +154,7 @@ elements, call :meth:`XMLPullParser.read_events`. Here is an example:: ... print(elem.tag, 'text=', elem.text) ... end + mytag text= sometext more text The obvious use case is applications that operate in a non-blocking fashion where the XML data is being received from a socket or read incrementally from @@ -165,6 +166,11 @@ data but would still like to have incremental parsing capabilities, take a look at :func:`iterparse`. It can be useful when you're reading a large XML document and don't want to hold it wholly in memory. +Where *immediate* feedback through events is wanted, calling method +:meth:`XMLPullParser.flush` can help reduce delay; +please make sure to study the related security notes. + + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -502,7 +508,7 @@ Functions `C14N 2.0 <https://www.w3.org/TR/xml-c14n2/>`_ transformation function. Canonicalization is a way to normalise XML output in a way that allows - byte-by-byte comparisons and digital signatures. It reduced the freedom + byte-by-byte comparisons and digital signatures. It reduces the freedom that XML serializers have and instead generates a more constrained XML representation. The main restrictions regard the placement of namespace declarations, the ordering of attributes, and ignorable whitespace. @@ -621,7 +627,11 @@ Functions *parser* is an optional parser instance. If not given, the standard :class:`XMLParser` parser is used. *parser* must be a subclass of :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a - target. Returns an :term:`iterator` providing ``(event, elem)`` pairs. + target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; + it has a ``root`` attribute that references the root element of the + resulting XML tree once *source* is fully read. + The iterator has the :meth:`!close` method that closes the internal + file object if *source* is a filename. Note that while :func:`iterparse` builds the tree incrementally, it issues blocking reads on *source* (or the file it names). As such, it's unsuitable @@ -644,6 +654,9 @@ Functions .. versionchanged:: 3.8 The ``comment`` and ``pi`` events were added. + .. versionchanged:: 3.13 + Added the :meth:`!close` method. + .. function:: parse(source, parser=None) @@ -661,7 +674,7 @@ Functions given. Returns an element instance, representing a processing instruction. Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An + in the input instead of creating PI objects for them. An :class:`ElementTree` will only contain processing instruction nodes if they have been inserted into to the tree using one of the :class:`Element` methods. @@ -702,11 +715,11 @@ Functions meaning as in :meth:`ElementTree.write`. Returns an (optionally) encoded string containing the XML data. - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + Added the *xml_declaration* and *default_namespace* parameters. .. versionchanged:: 3.8 The :func:`tostring` function now preserves the attribute order @@ -729,11 +742,11 @@ Functions .. versionadded:: 3.2 - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + Added the *xml_declaration* and *default_namespace* parameters. .. versionchanged:: 3.8 The :func:`tostringlist` function now preserves the attribute order @@ -827,36 +840,31 @@ Functions .. module:: xml.etree.ElementInclude -.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) - :module: +.. function:: default_loader(href, parse, encoding=None) - Default loader. This default loader reads an included resource from disk. *href* is a URL. - *parse* is for parse mode either "xml" or "text". *encoding* - is an optional text encoding. If not given, encoding is ``utf-8``. Returns the - expanded resource. If the parse mode is ``"xml"``, this is an ElementTree - instance. If the parse mode is "text", this is a Unicode string. If the - loader fails, it can return None or raise an exception. + Default loader. This default loader reads an included resource from disk. + *href* is a URL. *parse* is for parse mode either "xml" or "text". + *encoding* is an optional text encoding. If not given, encoding is ``utf-8``. + Returns the expanded resource. + If the parse mode is ``"xml"``, this is an :class:`~xml.etree.ElementTree.Element` instance. + If the parse mode is ``"text"``, this is a string. + If the loader fails, it can return ``None`` or raise an exception. -.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ - max_depth=6) - :module: +.. function:: include(elem, loader=None, base_url=None, max_depth=6) - This function expands XInclude directives. *elem* is the root element. *loader* is - an optional resource loader. If omitted, it defaults to :func:`default_loader`. + This function expands XInclude directives in-place in tree pointed by *elem*. + *elem* is either the root :class:`~xml.etree.ElementTree.Element` or an + :class:`~xml.etree.ElementTree.ElementTree` instance to find such element. + *loader* is an optional resource loader. If omitted, it defaults to :func:`default_loader`. If given, it should be a callable that implements the same interface as :func:`default_loader`. *base_url* is base URL of the original file, to resolve relative include file references. *max_depth* is the maximum number of recursive - inclusions. Limited to reduce the risk of malicious content explosion. Pass a - negative value to disable the limitation. + inclusions. Limited to reduce the risk of malicious content explosion. + Pass ``None`` to disable the limitation. - Returns the expanded resource. If the parse mode is - ``"xml"``, this is an ElementTree instance. If the parse mode is "text", - this is a Unicode string. If the loader fails, it can return None or - raise an exception. - - .. versionadded:: 3.9 - The *base_url* and *max_depth* parameters. + .. versionchanged:: 3.9 + Added the *base_url* and *max_depth* parameters. .. _elementtree-element-objects: @@ -1050,9 +1058,10 @@ Element Objects :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__len__`. - Caution: Elements with no subelements will test as ``False``. Testing the - truth value of an Element is deprecated and will raise an exception in - Python 3.14. Use specific ``len(elem)`` or ``elem is None`` test instead.:: + Caution: Elements with no subelements will test as ``False``. In a future + release of Python, all elements will test as ``True`` regardless of whether + subelements exist. Instead, prefer explicit ``len(elem)`` or + ``elem is not None`` tests.:: element = root.find('foo') @@ -1186,8 +1195,8 @@ ElementTree Objects :term:`file object`; make sure you do not try to write a string to a binary stream and vice versa. - .. versionadded:: 3.4 - The *short_empty_elements* parameter. + .. versionchanged:: 3.4 + Added the *short_empty_elements* parameter. .. versionchanged:: 3.8 The :meth:`write` method now preserves the attribute order specified @@ -1299,8 +1308,8 @@ TreeBuilder Objects .. method:: pi(target, text) - Creates a comment with the given *target* name and *text*. If - ``insert_pis`` is true, this will also add it to the tree. + Creates a process instruction with the given *target* name and *text*. + If ``insert_pis`` is true, this will also add it to the tree. .. versionadded:: 3.8 @@ -1379,6 +1388,24 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. + + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.13 + + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data is processed by method ``data(data)``. For further supported callback @@ -1440,6 +1467,22 @@ XMLPullParser Objects Feed the given bytes data to the parser. + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.13 + .. method:: close() Signal the parser that the data stream is terminated. Unlike diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 1e49b6568dfc28..d495995398959d 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -68,12 +68,13 @@ quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable* external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) `DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe decompression bomb Safe Safe Safe Safe **Vulnerable** +large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) ========================= ================== ================== ================== ================== ================== 1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to potential reliance on system-provided libraries. Check - :const:`pyexpat.EXPAT_VERSION`. + :const:`!pyexpat.EXPAT_VERSION`. 2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. 3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns @@ -81,6 +82,11 @@ decompression bomb Safe Safe Safe 4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. 5. Since Python 3.7.1, external general entities are no longer processed by default. +6. Expat 2.6.0 and newer is not vulnerable to denial of service + through quadratic runtime caused by parsing large tokens. + Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :const:`!pyexpat.EXPAT_VERSION`. billion laughs / exponential entity expansion @@ -114,7 +120,13 @@ decompression bomb files. For an attacker it can reduce the amount of transmitted data by three magnitudes or more. -The documentation for `defusedxml`_ on PyPI has further information about +large tokens + Expat needs to re-parse unfinished tokens; without the protection + introduced in Expat 2.6.0, this can lead to quadratic runtime that can + be used to cause denial of service in the application parsing XML. + The issue is known as :cve:`2023-52425`. + +The documentation for :pypi:`defusedxml` on PyPI has further information about all known attack vectors with examples and references. .. _defusedxml-package: @@ -122,14 +134,13 @@ all known attack vectors with examples and references. The :mod:`!defusedxml` Package ------------------------------ -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib +:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib XML parsers that prevent any potentially malicious operation. Use of this package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -.. _defusedxml: https://pypi.org/project/defusedxml/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index e2f28e3244cb09..c2c9d6424b5072 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== +:mod:`!xml.sax.handler` --- Base classes for SAX handlers +========================================================= .. module:: xml.sax.handler :synopsis: Base classes for SAX event handlers. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 113e9e93fb04ff..b0bc84062e0719 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== +:mod:`!xml.sax.xmlreader` --- Interface for XML parsers +======================================================= .. module:: xml.sax.xmlreader :synopsis: Interface which SAX-compliant XML parsers must implement. diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 6d351dfb4d7072..c60e9e505f7544 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== +:mod:`!xml.sax` --- Support for SAX2 parsers +============================================ .. module:: xml.sax :synopsis: Package containing SAX2 base classes and convenience functions. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index e57e76dcac7820..5ee11d58c3dd26 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= +:mod:`!xml.sax.saxutils` --- SAX Utilities +========================================== .. module:: xml.sax.saxutils :synopsis: Convenience functions and classes for use with SAX. @@ -71,8 +71,8 @@ or as base classes. content: if ``False`` (the default) they are emitted as a pair of start/end tags, if set to ``True`` they are emitted as a single self-closed tag. - .. versionadded:: 3.2 - The *short_empty_elements* parameter. + .. versionchanged:: 3.2 + Added the *short_empty_elements* parameter. .. class:: XMLFilterBase(base) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 146c4fd768233b..614fb19d1f56b6 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== +:mod:`!xmlrpc.client` --- XML-RPC client access +=============================================== .. module:: xmlrpc.client :synopsis: XML-RPC client access. @@ -269,8 +269,9 @@ DateTime Objects Write the XML-RPC encoding of this :class:`DateTime` item to the *out* stream object. - It also supports certain of Python's built-in operators through rich comparison - and :meth:`__repr__` methods. + It also supports certain of Python's built-in operators through + :meth:`rich comparison <object.__lt__>` and :meth:`~object.__repr__` + methods. A working example follows. The server code:: @@ -334,8 +335,8 @@ Binary Objects which was the de facto standard base64 specification when the XML-RPC spec was written. - It also supports certain of Python's built-in operators through :meth:`__eq__` - and :meth:`__ne__` methods. + It also supports certain of Python's built-in operators through + :meth:`~object.__eq__` and :meth:`~object.__ne__` methods. Example usage of the binary objects. We're going to transfer an image over XMLRPC:: diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 016369d2b89d2c..06169c7eca8b0c 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== +:mod:`!xmlrpc.server` --- Basic XML-RPC servers +=============================================== .. module:: xmlrpc.server :synopsis: Basic XML-RPC server implementations. @@ -84,12 +84,12 @@ alone XML-RPC servers. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. @@ -298,12 +298,12 @@ requests sent to Python CGI scripts. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 104afca23a20b4..cf561b454e934f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -1,5 +1,5 @@ -:mod:`zipapp` --- Manage executable Python zip archives -======================================================= +:mod:`!zipapp` --- Manage executable Python zip archives +======================================================== .. module:: zipapp :synopsis: Manage executable Python zip archives @@ -171,8 +171,8 @@ The module defines two convenience functions: passed to the ``zipfile.ZipFile`` class, and must supply the methods needed by that class. - .. versionadded:: 3.7 - Added the *filter* and *compressed* arguments. + .. versionchanged:: 3.7 + Added the *filter* and *compressed* parameters. .. function:: get_interpreter(archive) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index a77e49a7643826..5583c6b24be5c6 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -1,5 +1,5 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= +:mod:`!zipfile` --- Work with ZIP archives +========================================== .. module:: zipfile :synopsis: Read and write ZIP-format archive files. @@ -79,6 +79,11 @@ The module defines the following items: of the last modification to the file; the fields are described in section :ref:`zipinfo-objects`. + .. versionchanged:: 3.13 + A public :attr:`!compress_level` attribute has been added to expose the + formerly protected :attr:`!_compresslevel`. The older protected name + continues to work as a property for backwards compatibility. + .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, @@ -213,7 +218,7 @@ ZipFile Objects That flag takes precedence over *metadata_encoding*, which is a Python-specific extension. - .. versionadded:: 3.2 + .. versionchanged:: 3.2 Added the ability to use :class:`ZipFile` as a context manager. .. versionchanged:: 3.3 @@ -236,8 +241,8 @@ ZipFile Objects .. versionchanged:: 3.7 Add the *compresslevel* parameter. - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument + .. versionchanged:: 3.8 + The *strict_timestamps* keyword-only parameter. .. versionchanged:: 3.11 Added support for specifying member name encoding for reading @@ -296,6 +301,10 @@ ZipFile Objects attempting to read or write other files in the ZIP file will raise a :exc:`ValueError`. + In both cases the file-like object has also attributes :attr:`!name`, + which is equivalent to the name of a file within the archive, and + :attr:`!mode`, which is ``'rb'`` or ``'wb'`` depending on the input mode. + When writing a file, if the file size is not known in advance but may exceed 2 GiB, pass ``force_zip64=True`` to ensure that the header format is capable of supporting large files. If the file size is known in advance, @@ -320,6 +329,12 @@ ZipFile Objects Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.13 + Added attributes :attr:`!name` and :attr:`!mode` for the writeable + file-like object. + The value of the :attr:`!mode` attribute for the readable file-like + object was changed from ``'r'`` to ``'rb'``. + .. method:: ZipFile.extract(member, path=None, pwd=None) @@ -570,6 +585,15 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Return ``True`` if the current context references a file. +.. method:: Path.is_symlink() + + Return ``True`` if the current context references a symbolic link. + + .. versionadded:: 3.12 + + .. versionchanged:: 3.13 + Previously, ``is_symlink`` would unconditionally return ``False``. + .. method:: Path.exists() Return ``True`` if the current context references a file or @@ -627,7 +651,7 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Prior to 3.10, ``joinpath`` was undocumented and accepted exactly one parameter. -The `zipp <https://pypi.org/project/zipp>`_ project provides backports +The :pypi:`zipp` project provides backports of the latest path object functionality to older Pythons. Use ``zipp.Path`` in place of ``zipfile.Path`` for early access to changes. @@ -643,8 +667,8 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ optimize=-1) - .. versionadded:: 3.2 - The *optimize* parameter. + .. versionchanged:: 3.2 + Added the *optimize* parameter. .. versionchanged:: 3.4 ZIP64 extensions are enabled by default. @@ -699,8 +723,8 @@ The :class:`PyZipFile` constructor takes the same parameters as the test/bogus/__init__.pyc # Subpackage directory test/bogus/myfile.pyc # Submodule test.bogus.myfile - .. versionadded:: 3.4 - The *filterfunc* parameter. + .. versionchanged:: 3.4 + Added the *filterfunc* parameter. .. versionchanged:: 3.6.2 The *pathname* parameter accepts a :term:`path-like object`. @@ -744,8 +768,8 @@ file: .. versionchanged:: 3.6.2 The *filename* parameter accepts a :term:`path-like object`. - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument + .. versionchanged:: 3.8 + Added the *strict_timestamps* keyword-only parameter. Instances have the following methods and attributes: diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 47c81f0e63603d..9353a45bdcecba 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -1,5 +1,5 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== +:mod:`!zipimport` --- Import modules from Zip archives +====================================================== .. module:: zipimport :synopsis: Support for importing Python modules from ZIP archives. @@ -30,6 +30,9 @@ Any files may be present in the ZIP archive, but importers are only invoked for corresponding :file:`.pyc` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. +.. versionchanged:: 3.13 + ZIP64 is supported + .. versionchanged:: 3.8 Previously, ZIP archives with an archive comment were not supported. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ac179722dee2be..965b82a3daffb9 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -1,5 +1,5 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== +:mod:`!zlib` --- Compression compatible with :program:`gzip` +============================================================ .. module:: zlib :synopsis: Low-level interface to compression and decompression routines diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index f8624da6e51dbb..a57f3b8b3e858c 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -1,5 +1,5 @@ -:mod:`zoneinfo` --- IANA time zone support -========================================== +:mod:`!zoneinfo` --- IANA time zone support +=========================================== .. module:: zoneinfo :synopsis: IANA time zone support @@ -17,7 +17,7 @@ The :mod:`zoneinfo` module provides a concrete time zone implementation to support the IANA time zone database as originally specified in :pep:`615`. By default, :mod:`zoneinfo` uses the system's time zone data if available; if no system time zone data is available, the library will fall back to using the -first-party `tzdata`_ package available on PyPI. +first-party :pypi:`tzdata` package available on PyPI. .. seealso:: @@ -25,7 +25,7 @@ first-party `tzdata`_ package available on PyPI. Provides the :class:`~datetime.time` and :class:`~datetime.datetime` types with which the :class:`ZoneInfo` class is designed to be used. - Package `tzdata`_ + Package :pypi:`tzdata` First-party package maintained by the CPython core developers to supply time zone data via PyPI. @@ -93,7 +93,7 @@ Data sources The ``zoneinfo`` module does not directly provide time zone data, and instead pulls time zone information from the system time zone database or the -first-party PyPI package `tzdata`_, if available. Some systems, including +first-party PyPI package :pypi:`tzdata`, if available. Some systems, including notably Windows systems, do not have an IANA database available, and so for projects targeting cross-platform compatibility that require time zone data, it is recommended to declare a dependency on tzdata. If neither system data nor @@ -413,5 +413,3 @@ Exceptions and warnings be filtered out, such as a relative path. .. Links and references: - -.. _tzdata: https://pypi.org/project/tzdata/ diff --git a/Doc/license.rst b/Doc/license.rst index 82039fd9aaf9d0..674ac5f56e6f97 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -100,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2023 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2024 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -1040,3 +1040,92 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +.. _mimalloc-license: + +mimalloc +-------- + +MIT License:: + + Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + +asyncio +---------- + +Parts of the :mod:`asyncio` module are incorporated from +`uvloop 0.16 <https://github.com/MagicStack/uvloop/tree/v0.16.0>`_, +which is distributed under the MIT license:: + + Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Global Unbounded Sequences (GUS) +-------------------------------- + +The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded +Sequences" safe memory reclamation scheme in +`subr_smr.c <https://github.com/freebsd/freebsd-src/blob/main/sys/kern/subr_smr.c>`_. +The file is distributed under the 2-Clause BSD License:: + + Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice unmodified, this list of conditions, and the following + disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f79db828604b24..8181b9759517f6 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -245,13 +245,12 @@ handler is started. This search inspects the :keyword:`!except` clauses in turn until one is found that matches the exception. An expression-less :keyword:`!except` clause, if present, must be last; it matches any exception. -For an :keyword:`!except` clause with an expression, -that expression is evaluated, and the clause matches the exception -if the resulting object is "compatible" with the exception. An object is -compatible with an exception if the object is the class or a -:term:`non-virtual base class <abstract base class>` of the exception object, -or a tuple containing an item that is the class or a non-virtual base class -of the exception object. + +For an :keyword:`!except` clause with an expression, the +expression must evaluate to an exception type or a tuple of exception types. +The raised exception matches an :keyword:`!except` clause whose expression evaluates +to the class or a :term:`non-virtual base class <abstract base class>` of the exception object, +or to a tuple that contains such a class. If no :keyword:`!except` clause matches the exception, the search for an exception handler @@ -378,8 +377,10 @@ exception group with an empty message string. :: ... ExceptionGroup('', (BlockingIOError())) -An :keyword:`!except*` clause must have a matching type, -and this type cannot be a subclass of :exc:`BaseExceptionGroup`. +An :keyword:`!except*` clause must have a matching expression; it cannot be ``except*:``. +Furthermore, this expression cannot contain exception group types, because that would +have ambiguous semantics. + It is not possible to mix :keyword:`except` and :keyword:`!except*` in the same :keyword:`try`. :keyword:`break`, :keyword:`continue` and :keyword:`return` @@ -489,37 +490,37 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo #. The context expression (the expression given in the :token:`~python-grammar:with_item`) is evaluated to obtain a context manager. -#. The context manager's :meth:`__enter__` is loaded for later use. +#. The context manager's :meth:`~object.__enter__` is loaded for later use. -#. The context manager's :meth:`__exit__` is loaded for later use. +#. The context manager's :meth:`~object.__exit__` is loaded for later use. -#. The context manager's :meth:`__enter__` method is invoked. +#. The context manager's :meth:`~object.__enter__` method is invoked. #. If a target was included in the :keyword:`with` statement, the return value - from :meth:`__enter__` is assigned to it. + from :meth:`~object.__enter__` is assigned to it. .. note:: - The :keyword:`with` statement guarantees that if the :meth:`__enter__` - method returns without an error, then :meth:`__exit__` will always be + The :keyword:`with` statement guarantees that if the :meth:`~object.__enter__` + method returns without an error, then :meth:`~object.__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would be. See step 7 below. #. The suite is executed. -#. The context manager's :meth:`__exit__` method is invoked. If an exception +#. The context manager's :meth:`~object.__exit__` method is invoked. If an exception caused the suite to be exited, its type, value, and traceback are passed as - arguments to :meth:`__exit__`. Otherwise, three :const:`None` arguments are + arguments to :meth:`~object.__exit__`. Otherwise, three :const:`None` arguments are supplied. If the suite was exited due to an exception, and the return value from the - :meth:`__exit__` method was false, the exception is reraised. If the return + :meth:`~object.__exit__` method was false, the exception is reraised. If the return value was true, the exception is suppressed, and execution continues with the statement following the :keyword:`with` statement. If the suite was exited for any reason other than an exception, the return - value from :meth:`__exit__` is ignored, and execution proceeds at the normal + value from :meth:`~object.__exit__` is ignored, and execution proceeds at the normal location for the kind of exit that was taken. The following code:: @@ -1058,7 +1059,7 @@ subject value: .. note:: Key-value pairs are matched using the two-argument form of the mapping subject's ``get()`` method. Matched key-value pairs must already be present in the mapping, and not created on-the-fly via :meth:`__missing__` or - :meth:`__getitem__`. + :meth:`~object.__getitem__`. In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following happens: @@ -1261,7 +1262,8 @@ except that the original function is not temporarily bound to the name ``func``. A list of :ref:`type parameters <type-params>` may be given in square brackets between the function's name and the opening parenthesis for its parameter list. This indicates to static type checkers that the function is generic. At runtime, -the type parameters can be retrieved from the function's ``__type_params__`` +the type parameters can be retrieved from the function's +:attr:`~function.__type_params__` attribute. See :ref:`generic-functions` for more. .. versionchanged:: 3.12 @@ -1361,12 +1363,15 @@ access the local variables of the function containing the def. See section :pep:`526` - Syntax for Variable Annotations Ability to type hint variable declarations, including class - variables and instance variables + variables and instance variables. :pep:`563` - Postponed Evaluation of Annotations Support for forward references within annotations by preserving annotations in a string form at runtime instead of eager evaluation. + :pep:`318` - Decorators for Functions and Methods + Function and method decorators were introduced. + Class decorators were introduced in :pep:`3129`. .. _class: @@ -1616,15 +1621,18 @@ Type parameter lists .. versionadded:: 3.12 +.. versionchanged:: 3.13 + Support for default values was added (see :pep:`696`). + .. index:: single: type parameters .. productionlist:: python-grammar type_params: "[" `type_param` ("," `type_param`)* "]" type_param: `typevar` | `typevartuple` | `paramspec` - typevar: `identifier` (":" `expression`)? - typevartuple: "*" `identifier` - paramspec: "**" `identifier` + typevar: `identifier` (":" `expression`)? ("=" `expression`)? + typevartuple: "*" `identifier` ("=" `expression`)? + paramspec: "**" `identifier` ("=" `expression`)? :ref:`Functions <def>` (including :ref:`coroutines <async def>`), :ref:`classes <class>` and :ref:`type aliases <type>` may @@ -1690,19 +1698,31 @@ evaluated in a separate :ref:`annotation scope <annotation-scopes>`. :data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds or constraints. +All three flavors of type parameters can also have a *default value*, which is used +when the type parameter is not explicitly provided. This is added by appending +a single equals sign (``=``) followed by an expression. Like the bounds and +constraints of type variables, the default value is not evaluated when the +object is created, but only when the type parameter's ``__default__`` attribute +is accessed. To this end, the default value is evaluated in a separate +:ref:`annotation scope <annotation-scopes>`. If no default value is specified +for a type parameter, the ``__default__`` attribute is set to the special +sentinel object :data:`typing.NoDefault`. + The following example indicates the full set of allowed type parameter declarations:: def overly_generic[ SimpleTypeVar, + TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), - *SimpleTypeVarTuple, - **SimpleParamSpec, + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), ]( a: SimpleTypeVar, - b: TypeVarWithBound, - c: Callable[SimpleParamSpec, TypeVarWithConstraints], - *d: SimpleTypeVarTuple, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, ): ... .. _generic-functions: @@ -1868,8 +1888,8 @@ like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. are mappings. .. [#] A string literal appearing as the first statement in the function body is - transformed into the function's ``__doc__`` attribute and therefore the - function's :term:`docstring`. + transformed into the function's :attr:`~function.__doc__` attribute and + therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is transformed into the namespace's ``__doc__`` item and therefore the class's diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index cc81f73f824352..34d8bbee4c6806 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -34,7 +34,7 @@ represented by objects.) Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the +memory. The :keyword:`is` operator compares the identity of two objects; the :func:`id` function returns an integer representing its identity. .. impl-detail:: @@ -81,16 +81,16 @@ are still reachable. Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep +an exception with a :keyword:`try`...\ :keyword:`except` statement may keep objects alive. Some objects contain references to "external" resources such as open files or windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, -usually a :meth:`close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. +usually a :meth:`!close` method. Programs are strongly recommended to explicitly +close such objects. The :keyword:`try`...\ :keyword:`finally` statement +and the :keyword:`with` statement provide convenient ways to do this. .. index:: single: container @@ -159,7 +159,7 @@ NotImplemented .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``NotImplemented``. Numeric methods +object is accessed through the built-in name :data:`NotImplemented`. Numeric methods and rich comparison methods should return this value if they do not implement the operation for the operands provided. (The interpreter will then try the reflected operation, or some other fallback, depending on the operator.) It @@ -170,9 +170,12 @@ See for more details. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`NotImplemented` in a boolean context was deprecated. + +.. versionchanged:: 3.14 + Evaluating :data:`NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. Ellipsis @@ -299,14 +302,17 @@ Sequences These represent finite ordered sets indexed by non-negative numbers. The built-in function :func:`len` returns the number of items of a sequence. When the length of a sequence is *n*, the index set contains the numbers 0, 1, -..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. +..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. Some sequences, +including built-in sequences, interpret negative subscripts by adding the +sequence length. For example, ``a[-2]`` equals ``a[n-2]``, the second to last +item of sequence a with length ``n``. .. index:: single: slicing Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. This implies that the index set is renumbered so -that it starts at 0. +sequence of the same type. The comment above about negative indexes also applies +to negative slice positions. Some sequences also support "extended slicing" with a third "step" parameter: ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* @@ -519,6 +525,8 @@ These are the types to which the function call operation (see section :ref:`calls`) can be applied: +.. _user-defined-funcs: + User-defined functions ^^^^^^^^^^^^^^^^^^^^^^ @@ -532,9 +540,34 @@ section :ref:`function`). It should be called with an argument list containing the same number of items as the function's formal parameter list. -Special attributes: +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: __closure__ (function attribute) + single: __globals__ (function attribute) + pair: global; namespace + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__globals__ + - A reference to the :class:`dictionary <dict>` that holds the function's + :ref:`global variables <naming>` -- the global namespace of the module + in which the function was defined. -.. tabularcolumns:: |l|L|l| + * - .. attribute:: function.__closure__ + - ``None`` or a :class:`tuple` of cells that contain bindings for the + function's free variables. + + A cell object has the attribute ``cell_contents``. + This can be used to get the value of the cell, as well as set the value. + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. index:: single: __doc__ (function attribute) @@ -542,97 +575,84 @@ Special attributes: single: __module__ (function attribute) single: __dict__ (function attribute) single: __defaults__ (function attribute) - single: __closure__ (function attribute) single: __code__ (function attribute) - single: __globals__ (function attribute) single: __annotations__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) - pair: global; namespace -+-------------------------+-------------------------------+-----------+ -| Attribute | Meaning | | -+=========================+===============================+===========+ -| :attr:`__doc__` | The function's documentation | Writable | -| | string, or ``None`` if | | -| | unavailable; not inherited by | | -| | subclasses. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's name. | Writable | -| __name__` | | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's | Writable | -| __qualname__` | :term:`qualified name`. | | -| | | | -| | .. versionadded:: 3.3 | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__module__` | The name of the module the | Writable | -| | function was defined in, or | | -| | ``None`` if unavailable. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__defaults__` | A tuple containing default | Writable | -| | argument values for those | | -| | arguments that have defaults, | | -| | or ``None`` if no arguments | | -| | have a default value. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__code__` | The code object representing | Writable | -| | the compiled function body. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__globals__` | A reference to the dictionary | Read-only | -| | that holds the function's | | -| | global variables --- the | | -| | global namespace of the | | -| | module in which the function | | -| | was defined. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~object.__dict__`| The namespace supporting | Writable | -| | arbitrary function | | -| | attributes. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | -| | that contain bindings for the | | -| | function's free variables. | | -| | See below for information on | | -| | the ``cell_contents`` | | -| | attribute. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__annotations__` | A dict containing annotations | Writable | -| | of parameters. The keys of | | -| | the dict are the parameter | | -| | names, and ``'return'`` for | | -| | the return annotation, if | | -| | provided. For more | | -| | information on working with | | -| | this attribute, see | | -| | :ref:`annotations-howto`. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__kwdefaults__` | A dict containing defaults | Writable | -| | for keyword-only parameters. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__type_params__` | A tuple containing the | Writable | -| | :ref:`type parameters | | -| | <type-params>` of a | | -| | :ref:`generic function | | -| | <generic-functions>`. | | -+-------------------------+-------------------------------+-----------+ - -Most of the attributes labelled "Writable" check the type of the assigned value. +Most of these attributes check the type of the assigned value: + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__doc__ + - The function's documentation string, or ``None`` if unavailable. + Not inherited by subclasses. + + * - .. attribute:: function.__name__ + - The function's name. + See also: :attr:`__name__ attributes <definition.__name__>`. + + * - .. attribute:: function.__qualname__ + - The function's :term:`qualified name`. + See also: :attr:`__qualname__ attributes <definition.__qualname__>`. + + .. versionadded:: 3.3 + + * - .. attribute:: function.__module__ + - The name of the module the function was defined in, + or ``None`` if unavailable. + + * - .. attribute:: function.__defaults__ + - A :class:`tuple` containing default :term:`parameter` values + for those parameters that have defaults, + or ``None`` if no parameters have a default value. + + * - .. attribute:: function.__code__ + - The :ref:`code object <code-objects>` representing + the compiled function body. + + * - .. attribute:: function.__dict__ + - The namespace supporting arbitrary function attributes. + See also: :attr:`__dict__ attributes <object.__dict__>`. + + * - .. attribute:: function.__annotations__ + - A :class:`dictionary <dict>` containing annotations of + :term:`parameters <parameter>`. + The keys of the dictionary are the parameter names, + and ``'return'`` for the return annotation, if provided. + See also: :ref:`annotations-howto`. + + * - .. attribute:: function.__kwdefaults__ + - A :class:`dictionary <dict>` containing defaults for keyword-only + :term:`parameters <parameter>`. + + * - .. attribute:: function.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters <type-params>` of + a :ref:`generic function <generic-functions>`. + + .. versionadded:: 3.12 Function objects also support getting and setting arbitrary attributes, which can be used, for example, to attach metadata to functions. Regular attribute -dot-notation is used to get and set such attributes. *Note that the current -implementation only supports function attributes on user-defined functions. -Function attributes on built-in functions may be supported in the future.* +dot-notation is used to get and set such attributes. + +.. impl-detail:: -A cell object has the attribute ``cell_contents``. This can be used to get -the value of the cell, as well as set the value. + CPython's current implementation only supports function attributes + on user-defined functions. Function attributes on + :ref:`built-in functions <builtin-functions>` may be supported in the + future. Additional information about a function's definition can be retrieved from its -code object; see the description of internal types below. The -:data:`cell <types.CellType>` type can be accessed in the :mod:`types` -module. +:ref:`code object <code-objects>` +(accessible via the :attr:`~function.__code__` attribute). + +.. _instance-methods: Instance methods ^^^^^^^^^^^^^^^^ @@ -652,43 +672,66 @@ callable object (normally a user-defined function). single: __name__ (method attribute) single: __module__ (method attribute) -Special read-only attributes: :attr:`__self__` is the class instance object, -:attr:`__func__` is the function object; :attr:`__doc__` is the method's -documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the -method name (same as ``__func__.__name__``); :attr:`__module__` is the -name of the module the method was defined in, or ``None`` if unavailable. +Special read-only attributes: + +.. list-table:: + + * - .. attribute:: method.__self__ + - Refers to the class instance object to which the method is + :ref:`bound <method-binding>` + + * - .. attribute:: method.__func__ + - Refers to the original :ref:`function object <user-defined-funcs>` + + * - .. attribute:: method.__doc__ + - The method's documentation + (same as :attr:`method.__func__.__doc__ <function.__doc__>`). + A :class:`string <str>` if the original function had a docstring, else + ``None``. + + * - .. attribute:: method.__name__ + - The name of the method + (same as :attr:`method.__func__.__name__ <function.__name__>`) + + * - .. attribute:: method.__module__ + - The name of the module the method was defined in, or ``None`` if + unavailable. Methods also support accessing (but not setting) the arbitrary function -attributes on the underlying function object. +attributes on the underlying :ref:`function object <user-defined-funcs>`. User-defined method objects may be created when getting an attribute of a class (perhaps via an instance of that class), if that attribute is a -user-defined function object or a class method object. +user-defined :ref:`function object <user-defined-funcs>` or a +:class:`classmethod` object. + +.. _method-binding: When an instance method object is created by retrieving a user-defined -function object from a class via one of its instances, its -:attr:`__self__` attribute is the instance, and the method object is said -to be bound. The new method's :attr:`__func__` attribute is the original -function object. - -When an instance method object is created by retrieving a class method -object from a class or instance, its :attr:`__self__` attribute is the -class itself, and its :attr:`__func__` attribute is the function object +:ref:`function object <user-defined-funcs>` from a class via one of its +instances, its :attr:`~method.__self__` attribute is the instance, and the +method object is said to be *bound*. The new method's :attr:`~method.__func__` +attribute is the original function object. + +When an instance method object is created by retrieving a :class:`classmethod` +object from a class or instance, its :attr:`~method.__self__` attribute is the +class itself, and its :attr:`~method.__func__` attribute is the function object underlying the class method. When an instance method object is called, the underlying function -(:attr:`__func__`) is called, inserting the class instance -(:attr:`__self__`) in front of the argument list. For instance, when -:class:`C` is a class which contains a definition for a function -:meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is +(:attr:`~method.__func__`) is called, inserting the class instance +(:attr:`~method.__self__`) in front of the argument list. For instance, when +:class:`!C` is a class which contains a definition for a function +:meth:`!f`, and ``x`` is an instance of :class:`!C`, calling ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``. -When an instance method object is derived from a class method object, the -"class instance" stored in :attr:`__self__` will actually be the class +When an instance method object is derived from a :class:`classmethod` object, the +"class instance" stored in :attr:`~method.__self__` will actually be the class itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is the underlying function. -Note that the transformation from function object to instance method +Note that the transformation from :ref:`function object <user-defined-funcs>` +to instance method object happens each time the attribute is retrieved from the instance. In some cases, a fruitful optimization is to assign the attribute to a local variable and call that local variable. Also notice that this @@ -754,6 +797,8 @@ is raised and the asynchronous iterator will have reached the end of the set of values to be yielded. +.. _builtin-functions: + Built-in functions ^^^^^^^^^^^^^^^^^^ @@ -766,12 +811,18 @@ A built-in function object is a wrapper around a C function. Examples of built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a standard built-in module). The number and type of the arguments are determined by the C function. Special read-only attributes: -:attr:`__doc__` is the function's documentation string, or ``None`` if -unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is -set to ``None`` (but see the next item); :attr:`__module__` is the name of -the module the function was defined in or ``None`` if unavailable. + +* :attr:`!__doc__` is the function's documentation string, or ``None`` if + unavailable. See :attr:`function.__doc__`. +* :attr:`!__name__` is the function's name. See :attr:`function.__name__`. +* :attr:`!__self__` is set to ``None`` (but see the next item). +* :attr:`!__module__` is the name of + the module the function was defined in or ``None`` if unavailable. + See :attr:`function.__module__`. +.. _builtin-methods: + Built-in methods ^^^^^^^^^^^^^^^^ @@ -783,8 +834,9 @@ Built-in methods This is really a different disguise of a built-in function, this time containing an object passed to the C function as an implicit extra argument. An example of a built-in method is ``alist.append()``, assuming *alist* is a list object. In -this case, the special read-only attribute :attr:`__self__` is set to the object -denoted by *alist*. +this case, the special read-only attribute :attr:`!__self__` is set to the object +denoted by *alist*. (The attribute has the same semantics as it does with +:attr:`other instance methods <method.__self__>`.) Classes @@ -793,7 +845,7 @@ Classes Classes are callable. These objects normally act as factories for new instances of themselves, but variations are possible for class types that override :meth:`~object.__new__`. The arguments of the call are passed to -:meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to +:meth:`!__new__` and, in the typical case, to :meth:`~object.__init__` to initialize the new instance. @@ -816,7 +868,8 @@ the :ref:`import system <importsystem>` as invoked either by the :keyword:`import` statement, or by calling functions such as :func:`importlib.import_module` and built-in :func:`__import__`. A module object has a namespace implemented by a -dictionary object (this is the dictionary referenced by the ``__globals__`` +:class:`dictionary <dict>` object (this is the dictionary referenced by the +:attr:`~function.__globals__` attribute of functions defined in the module). Attribute references are translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object does not contain the code object used @@ -882,11 +935,8 @@ name is not found there, the attribute search continues in the base classes. This search of the base classes uses the C3 method resolution order which behaves correctly even in the presence of 'diamond' inheritance structures where there are multiple inheritance paths leading back to a common ancestor. -Additional details on the C3 MRO used by Python can be found in the -documentation accompanying the 2.3 release at -https://www.python.org/download/releases/2.3/mro/. - -.. XXX: Could we add that MRO doc as an appendix to the language ref? +Additional details on the C3 MRO used by Python can be found at +:ref:`python_2.3_mro`. .. index:: pair: object; class @@ -897,10 +947,11 @@ https://www.python.org/download/releases/2.3/mro/. pair: object; dictionary pair: class; attribute -When a class attribute reference (for class :class:`C`, say) would yield a +When a class attribute reference (for class :class:`!C`, say) would yield a class method object, it is transformed into an instance method object whose -:attr:`__self__` attribute is :class:`C`. When it would yield a static -method object, it is transformed into the object wrapped by the static method +:attr:`~method.__self__` attribute is :class:`!C`. +When it would yield a :class:`staticmethod` object, +it is transformed into the object wrapped by the static method object. See section :ref:`descriptors` for another way in which attributes retrieved from a class may differ from those actually contained in its :attr:`~object.__dict__`. @@ -922,6 +973,8 @@ A class object can be called (see above) to yield a class instance (see below). single: __doc__ (class attribute) single: __annotations__ (class attribute) single: __type_params__ (class attribute) + single: __static_attributes__ (class attribute) + single: __firstlineno__ (class attribute) Special attributes: @@ -952,6 +1005,13 @@ Special attributes: A tuple containing the :ref:`type parameters <type-params>` of a :ref:`generic class <generic-classes>`. + :attr:`~class.__static_attributes__` + A tuple containing names of attributes of this class which are accessed + through ``self.X`` from any function in its body. + + :attr:`__firstlineno__` + The line number of the first line of the class definition, including decorators. + Class instances --------------- @@ -968,7 +1028,7 @@ in which attribute references are searched. When an attribute is not found there, and the instance's class has an attribute by that name, the search continues with the class attributes. If a class attribute is found that is a user-defined function object, it is transformed into an instance method -object whose :attr:`__self__` attribute is the instance. Static method and +object whose :attr:`~method.__self__` attribute is the instance. Static method and class method objects are also transformed; see above under "Classes". See section :ref:`descriptors` for another way in which attributes of a class retrieved via its instances may differ from the objects actually stored in @@ -1075,59 +1135,115 @@ indirectly) to mutable objects. single: co_freevars (code object attribute) single: co_qualname (code object attribute) -Special read-only attributes: :attr:`co_name` gives the function name; -:attr:`co_qualname` gives the fully qualified function name; -:attr:`co_argcount` is the total number of positional arguments -(including positional-only arguments and arguments with default values); -:attr:`co_posonlyargcount` is the number of positional-only arguments -(including arguments with default values); :attr:`co_kwonlyargcount` is -the number of keyword-only arguments (including arguments with default -values); :attr:`co_nlocals` is the number of local variables used by the -function (including arguments); :attr:`co_varnames` is a tuple containing -the names of the local variables (starting with the argument names); -:attr:`co_cellvars` is a tuple containing the names of local variables -that are referenced by nested functions; :attr:`co_freevars` is a tuple -containing the names of free variables; :attr:`co_code` is a string -representing the sequence of bytecode instructions; :attr:`co_consts` is -a tuple containing the literals used by the bytecode; :attr:`co_names` is -a tuple containing the names used by the bytecode; :attr:`co_filename` is -the filename from which the code was compiled; :attr:`co_firstlineno` is -the first line number of the function; :attr:`co_lnotab` is a string -encoding the mapping from bytecode offsets to line numbers (for details -see the source code of the interpreter, is deprecated since 3.12 -and may be removed in 3.14); :attr:`co_stacksize` is the -required stack size; :attr:`co_flags` is an integer encoding a number -of flags for the interpreter. +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: codeobject.co_name + - The function name + + * - .. attribute:: codeobject.co_qualname + - The fully qualified function name + + .. versionadded:: 3.11 + + * - .. attribute:: codeobject.co_argcount + - The total number of positional :term:`parameters <parameter>` + (including positional-only parameters and parameters with default values) + that the function has + + * - .. attribute:: codeobject.co_posonlyargcount + - The number of positional-only :term:`parameters <parameter>` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_kwonlyargcount + - The number of keyword-only :term:`parameters <parameter>` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_nlocals + - The number of :ref:`local variables <naming>` used by the function + (including parameters) + + * - .. attribute:: codeobject.co_varnames + - A :class:`tuple` containing the names of the local variables in the + function (starting with the parameter names) + + * - .. attribute:: codeobject.co_cellvars + - A :class:`tuple` containing the names of :ref:`local variables <naming>` + that are referenced by nested functions inside the function + + * - .. attribute:: codeobject.co_freevars + - A :class:`tuple` containing the names of free variables in the function + + * - .. attribute:: codeobject.co_code + - A string representing the sequence of :term:`bytecode` instructions in + the function + + * - .. attribute:: codeobject.co_consts + - A :class:`tuple` containing the literals used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_names + - A :class:`tuple` containing the names used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_filename + - The name of the file from which the code was compiled + + * - .. attribute:: codeobject.co_firstlineno + - The line number of the first line of the function + + * - .. attribute:: codeobject.co_lnotab + - A string encoding the mapping from :term:`bytecode` offsets to line + numbers. For details, see the source code of the interpreter. + + .. deprecated:: 3.12 + This attribute of code objects is deprecated, and may be removed in + Python 3.14. + + * - .. attribute:: codeobject.co_stacksize + - The required stack size of the code object + + * - .. attribute:: codeobject.co_flags + - An :class:`integer <int>` encoding a number of flags for the + interpreter. .. index:: pair: object; generator -The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if +The following flag bits are defined for :attr:`~codeobject.co_flags`: +bit ``0x04`` is set if the function uses the ``*arguments`` syntax to accept an arbitrary number of positional arguments; bit ``0x08`` is set if the function uses the ``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set -if the function is a generator. +if the function is a generator. See :ref:`inspect-module-co-flags` for details +on the semantics of each flags that might be present. Future feature declarations (``from __future__ import division``) also use bits -in :attr:`co_flags` to indicate whether a code object was compiled with a +in :attr:`~codeobject.co_flags` to indicate whether a code object was compiled with a particular feature enabled: bit ``0x2000`` is set if the function was compiled with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier versions of Python. -Other bits in :attr:`co_flags` are reserved for internal use. +Other bits in :attr:`~codeobject.co_flags` are reserved for internal use. .. index:: single: documentation string -If a code object represents a function, the first item in :attr:`co_consts` is +If a code object represents a function, the first item in +:attr:`~codeobject.co_consts` is the documentation string of the function, or ``None`` if undefined. +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ + .. method:: codeobject.co_positions() - Returns an iterable over the source code positions of each bytecode + Returns an iterable over the source code positions of each :term:`bytecode` instruction in the code object. - The iterator returns tuples containing the ``(start_line, end_line, + The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. + position of the source code that compiled to the *i-th* code unit. Column information is 0-indexed utf-8 byte offsets on the given source line. @@ -1153,6 +1269,49 @@ the documentation string of the function, or ``None`` if undefined. :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` environment variable can be used. +.. method:: codeobject.co_lines() + + Returns an iterator that yields information about successive ranges of + :term:`bytecode`\s. Each item yielded is a ``(start, end, lineno)`` + :class:`tuple`: + + * ``start`` (an :class:`int`) represents the offset (inclusive) of the start + of the :term:`bytecode` range + * ``end`` (an :class:`int`) represents the offset (exclusive) of the end of + the :term:`bytecode` range + * ``lineno`` is an :class:`int` representing the line number of the + :term:`bytecode` range, or ``None`` if the bytecodes in the given range + have no line number + + The items yielded will have the following properties: + + * The first range yielded will have a ``start`` of 0. + * The ``(start, end)`` ranges will be non-decreasing and consecutive. That + is, for any pair of :class:`tuple`\s, the ``start`` of the second will be + equal to the ``end`` of the first. + * No range will be backwards: ``end >= start`` for all triples. + * The last :class:`tuple` yielded will have ``end`` equal to the size of the + :term:`bytecode`. + + Zero-width ranges, where ``start == end``, are allowed. Zero-width ranges + are used for lines that are present in the source code, but have been + eliminated by the :term:`bytecode` compiler. + + .. versionadded:: 3.10 + + .. seealso:: + + :pep:`626` - Precise line numbers for debugging and other tools. + The PEP that introduced the :meth:`!co_lines` method. + +.. method:: codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + Code objects are also supported by the generic function :func:`copy.replace`. + + .. versionadded:: 3.8 + .. _frame-objects: @@ -1161,8 +1320,9 @@ Frame objects .. index:: pair: object; frame -Frame objects represent execution frames. They may occur in traceback objects -(see below), and are also passed to registered trace functions. +Frame objects represent execution frames. They may occur in +:ref:`traceback objects <traceback-objects>`, +and are also passed to registered trace functions. .. index:: single: f_back (frame attribute) @@ -1172,16 +1332,41 @@ Frame objects represent execution frames. They may occur in traceback objects single: f_lasti (frame attribute) single: f_builtins (frame attribute) -Special read-only attributes: :attr:`f_back` is to the previous stack frame -(towards the caller), or ``None`` if this is the bottom stack frame; -:attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` -is the dictionary used to look up local variables; :attr:`f_globals` is used for -global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; -:attr:`f_lasti` gives the precise instruction (this is an index into the -bytecode string of the code object). +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Accessing ``f_code`` raises an :ref:`auditing event <auditing>` -``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. +.. list-table:: + + * - .. attribute:: frame.f_back + - Points to the previous stack frame (towards the caller), + or ``None`` if this is the bottom stack frame + + * - .. attribute:: frame.f_code + - The :ref:`code object <code-objects>` being executed in this frame. + Accessing this attribute raises an :ref:`auditing event <auditing>` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + + * - .. attribute:: frame.f_locals + - The mapping used by the frame to look up + :ref:`local variables <naming>`. + If the frame refers to an :term:`optimized scope`, + this may return a write-through proxy object. + + .. versionchanged:: 3.13 + Return a proxy for optimized scopes. + + * - .. attribute:: frame.f_globals + - The dictionary used by the frame to look up + :ref:`global variables <naming>` + + * - .. attribute:: frame.f_builtins + - The dictionary used by the frame to look up + :ref:`built-in (intrinsic) names <naming>` + + * - .. attribute:: frame.f_lasti + - The "precise instruction" of the frame object + (this is an index into the :term:`bytecode` string of the + :ref:`code object <code-objects>`) .. index:: single: f_trace (frame attribute) @@ -1189,35 +1374,54 @@ Accessing ``f_code`` raises an :ref:`auditing event <auditing>` single: f_trace_opcodes (frame attribute) single: f_lineno (frame attribute) -Special writable attributes: :attr:`f_trace`, if not ``None``, is a function -called for various events during code execution (this is used by the debugger). -Normally an event is triggered for each new source line - this can be -disabled by setting :attr:`f_trace_lines` to :const:`False`. +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Implementations *may* allow per-opcode events to be requested by setting -:attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to -undefined interpreter behaviour if exceptions raised by the trace -function escape to the function being traced. +.. list-table:: -:attr:`f_lineno` is the current line number of the frame --- writing to this -from within a trace function jumps to the given line (only for the bottom-most -frame). A debugger can implement a Jump command (aka Set Next Statement) -by writing to f_lineno. + * - .. attribute:: frame.f_trace + - If not ``None``, this is a function called for various events during + code execution (this is used by debuggers). Normally an event is + triggered for each new source line (see :attr:`~frame.f_trace_lines`). + + * - .. attribute:: frame.f_trace_lines + - Set this attribute to :const:`False` to disable triggering a tracing + event for each source line. + + * - .. attribute:: frame.f_trace_opcodes + - Set this attribute to :const:`True` to allow per-opcode events to be + requested. Note that this may lead to + undefined interpreter behaviour if exceptions raised by the trace + function escape to the function being traced. + + * - .. attribute:: frame.f_lineno + - The current line number of the frame -- writing to this + from within a trace function jumps to the given line (only for the bottom-most + frame). A debugger can implement a Jump command (aka Set Next Statement) + by writing to this attribute. + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ Frame objects support one method: .. method:: frame.clear() - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator + This method clears all references to :ref:`local variables <naming>` held by the + frame. Also, if the frame belonged to a :term:`generator`, the generator is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). + objects (for example when catching an :ref:`exception <bltin-exceptions>` + and storing its :ref:`traceback <traceback-objects>` for later use). - :exc:`RuntimeError` is raised if the frame is currently executing. + :exc:`RuntimeError` is raised if the frame is currently executing + or suspended. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + Attempting to clear a suspended frame raises :exc:`RuntimeError` + (as has always been the case for executing frames). + .. _traceback-objects: @@ -1235,26 +1439,31 @@ Traceback objects single: sys.exception single: sys.last_traceback -Traceback objects represent a stack trace of an exception. A traceback object +Traceback objects represent the stack trace of an :ref:`exception <tut-errors>`. +A traceback object is implicitly created when an exception occurs, and may also be explicitly created by calling :class:`types.TracebackType`. +.. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code. + For implicitly created tracebacks, when the search for an exception handler unwinds the execution stack, at each unwound level a traceback object is inserted in front of the current traceback. When an exception handler is entered, the stack trace is made available to the program. (See section :ref:`try`.) It is accessible as the third item of the -tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute +tuple returned by :func:`sys.exc_info`, and as the +:attr:`~BaseException.__traceback__` attribute of the caught exception. When the program contains no suitable handler, the stack trace is written (nicely formatted) to the standard error stream; if the interpreter is interactive, it is also made available to the user -as ``sys.last_traceback``. +as :data:`sys.last_traceback`. For explicitly created tracebacks, it is up to the creator of the traceback -to determine how the ``tb_next`` attributes should be linked to form a -full stack trace. +to determine how the :attr:`~traceback.tb_next` attributes should be linked to +form a full stack trace. .. index:: single: tb_frame (traceback attribute) @@ -1263,27 +1472,40 @@ full stack trace. pair: statement; try Special read-only attributes: -:attr:`tb_frame` points to the execution frame of the current level; -:attr:`tb_lineno` gives the line number where the exception occurred; -:attr:`tb_lasti` indicates the precise instruction. + +.. list-table:: + + * - .. attribute:: traceback.tb_frame + - Points to the execution :ref:`frame <frame-objects>` of the current + level. + + Accessing this attribute raises an + :ref:`auditing event <auditing>` ``object.__getattr__`` with arguments + ``obj`` and ``"tb_frame"``. + + * - .. attribute:: traceback.tb_lineno + - Gives the line number where the exception occurred + + * - .. attribute:: traceback.tb_lasti + - Indicates the "precise instruction". + The line number and last instruction in the traceback may differ from the -line number of its frame object if the exception occurred in a +line number of its :ref:`frame object <frame-objects>` if the exception +occurred in a :keyword:`try` statement with no matching except clause or with a -finally clause. - -Accessing ``tb_frame`` raises an :ref:`auditing event <auditing>` -``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. +:keyword:`finally` clause. .. index:: single: tb_next (traceback attribute) -Special writable attribute: :attr:`tb_next` is the next level in the stack -trace (towards the frame where the exception occurred), or ``None`` if -there is no next level. +.. attribute:: traceback.tb_next -.. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. + The special writable attribute :attr:`!tb_next` is the next level in the + stack trace (towards the frame where the exception occurred), or ``None`` if + there is no next level. + + .. versionchanged:: 3.7 + This attribute is now writable Slice objects @@ -1334,7 +1556,7 @@ Class method objects A class method object, like a static method object, is a wrapper around another object that alters the way in which that object is retrieved from classes and class instances. The behaviour of class method objects upon such retrieval is -described above, under "User-defined methods". Class method objects are created +described above, under :ref:`"instance methods" <instance-methods>`. Class method objects are created by the built-in :func:`classmethod` constructor. @@ -1445,6 +1667,8 @@ Basic customization It is not guaranteed that :meth:`__del__` methods are called for objects that still exist when the interpreter exits. + :class:`weakref.finalize` provides a straightforward way to register + a cleanup function to be called when an object is garbage collected. .. note:: @@ -1582,7 +1806,7 @@ Basic customization ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls ``x.__ge__(y)``. - A rich comparison method may return the singleton ``NotImplemented`` if it does + A rich comparison method may return the singleton :data:`NotImplemented` if it does not implement the operation for a given pair of arguments. By convention, ``False`` and ``True`` are returned for a successful comparison. However, these methods can return any value, so if the comparison operator is used in a Boolean @@ -1590,10 +1814,10 @@ Basic customization :func:`bool` on the value to determine if the result is true or false. By default, ``object`` implements :meth:`__eq__` by using ``is``, returning - ``NotImplemented`` in the case of a false comparison: + :data:`NotImplemented` in the case of a false comparison: ``True if x is y else NotImplemented``. For :meth:`__ne__`, by default it delegates to :meth:`__eq__` and inverts the result unless it is - ``NotImplemented``. There are no other implied relationships among the + :data:`!NotImplemented`. There are no other implied relationships among the comparison operators or default implementations; for example, the truth of ``(x<y or x==y)`` does not imply ``x<=y``. To automatically generate ordering operations from a single root operation, see :func:`functools.total_ordering`. @@ -1607,12 +1831,15 @@ Basic customization rather, :meth:`__lt__` and :meth:`__gt__` are each other's reflection, :meth:`__le__` and :meth:`__ge__` are each other's reflection, and :meth:`__eq__` and :meth:`__ne__` are their own reflection. - If the operands are of different types, and right operand's type is + If the operands are of different types, and the right operand's type is a direct or indirect subclass of the left operand's type, the reflected method of the right operand has priority, otherwise the left operand's method has priority. Virtual subclassing is not considered. + When no appropriate method returns any value other than :data:`NotImplemented`, the + ``==`` and ``!=`` operators will fall back to ``is`` and ``is not``, respectively. + .. method:: object.__hash__(self) .. index:: @@ -1681,7 +1908,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a - dict insertion, O(n\ :sup:`2`) complexity. See + dict insertion, *O*\ (*n*\ :sup:`2`) complexity. See http://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. @@ -1751,7 +1978,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. note:: This method may still be bypassed when looking up special methods as the - result of implicit invocation via language syntax or built-in functions. + result of implicit invocation via language syntax or + :ref:`built-in functions <builtin-functions>`. See :ref:`special-lookup`. .. audit-event:: object.__getattr__ obj,name object.__getattribute__ @@ -1792,8 +2020,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__dir__(self) - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. + Called when :func:`dir` is called on the object. An iterable must be + returned. :func:`dir` converts the returned iterable to a list and sorts it. Customizing module attribute access @@ -1813,7 +2041,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a sequence of +The ``__dir__`` function should accept no arguments, and return an iterable of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. @@ -1896,13 +2124,17 @@ class' :attr:`~object.__dict__`. Called to delete the attribute on an instance *instance* of the owner class. +Instances of descriptors may also have the :attr:`!__objclass__` attribute +present: + +.. attribute:: object.__objclass__ -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). + The attribute :attr:`!__objclass__` is interpreted by the :mod:`inspect` module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class attributes). + For callables, it may indicate that an instance of the given type (or a + subclass) is expected or required as the first positional argument (for example, + CPython sets this attribute for unbound methods that are implemented in C). .. _descriptor-invocation: @@ -1983,13 +2215,14 @@ For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`~object.__get__`, :meth:`~object.__set__` and :meth:`~object.__delete__`. If it does not -define :meth:`__get__`, then accessing the attribute will return the descriptor +define :meth:`!__get__`, then accessing the attribute will return the descriptor object itself unless there is a value in the object's instance dictionary. If -the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data +the descriptor defines :meth:`!__set__` and/or :meth:`!__delete__`, it is a data descriptor; if it defines neither, it is a non-data descriptor. Normally, data -descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data -descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__get__` and :meth:`__set__` (and/or :meth:`__delete__`) defined always override a redefinition in an +descriptors define both :meth:`!__get__` and :meth:`!__set__`, while non-data +descriptors have just the :meth:`!__get__` method. Data descriptors with +:meth:`!__get__` and :meth:`!__set__` (and/or :meth:`!__delete__`) defined +always override a redefinition in an instance dictionary. In contrast, non-data descriptors can be overridden by instances. @@ -2107,7 +2340,7 @@ class defining the method. this method is implicitly converted to a class method. Keyword arguments which are given to a new class are passed to - the parent's class ``__init_subclass__``. For compatibility with + the parent class's ``__init_subclass__``. For compatibility with other classes using ``__init_subclass__``, one should take out the needed keyword arguments and pass the others over to the base class, as in:: @@ -2566,16 +2799,17 @@ either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a range of items. It is also recommended that mappings provide the methods -:meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard :class:`dictionary <dict>` +:meth:`!keys`, :meth:`!values`, :meth:`!items`, :meth:`!get`, :meth:`!clear`, +:meth:`!setdefault`, :meth:`!pop`, :meth:`!popitem`, :meth:`!copy`, and +:meth:`!update` behaving similar to those for Python's standard :class:`dictionary <dict>` objects. The :mod:`collections.abc` module provides a :class:`~collections.abc.MutableMapping` :term:`abstract base class` to help create those methods from a base set of -:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`. -Mutable sequences should provide methods :meth:`append`, :meth:`count`, -:meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard :class:`list` +:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__`, and :meth:`!keys`. +Mutable sequences should provide methods :meth:`!append`, :meth:`!count`, +:meth:`!index`, :meth:`!extend`, :meth:`!insert`, :meth:`!pop`, :meth:`!remove`, +:meth:`!reverse` and :meth:`!sort`, like Python standard :class:`list` objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods @@ -2588,7 +2822,7 @@ operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and sequences implement the :meth:`~object.__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should iterate +through the container; for mappings, :meth:`!__iter__` should iterate through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) @@ -2616,7 +2850,7 @@ through the object's keys; for sequences, it should iterate through the values. Called to implement :func:`operator.length_hint`. Should return an estimated length for the object (which may be greater or less than the actual length). The length must be an integer ``>=`` 0. The return value may also be - :const:`NotImplemented`, which is treated the same as if the + :data:`NotImplemented`, which is treated the same as if the ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. @@ -2641,10 +2875,10 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__getitem__(self, key) Called to implement evaluation of ``self[key]``. For :term:`sequence` types, - the accepted keys should be integers and slice objects. Note that the - special interpretation of negative indexes (if the class wishes to emulate a - :term:`sequence` type) is up to the :meth:`__getitem__` method. If *key* is - of an inappropriate type, :exc:`TypeError` may be raised; if of a value + the accepted keys should be integers. Optionally, they may support + :class:`slice` objects as well. Negative index support is also optional. + If *key* is + of an inappropriate type, :exc:`TypeError` may be raised; if *key* is a value outside the set of indexes for the sequence (after any special interpretation of negative values), :exc:`IndexError` should be raised. For :term:`mapping` types, if *key* is missing (not in the container), @@ -2768,7 +3002,7 @@ left undefined. function is to be supported. If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. + arguments, it should return :data:`NotImplemented`. .. method:: object.__radd__(self, other) @@ -2798,7 +3032,7 @@ left undefined. types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is an instance of a class that has an :meth:`__rsub__` method, ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - *NotImplemented*. + :data:`NotImplemented`. .. index:: pair: built-in function; pow @@ -2832,10 +3066,12 @@ left undefined. (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the + but does not have to be, *self*). If a specific method is not defined, or if + that method returns :data:`NotImplemented`, the augmented assignment falls back to the normal methods. For instance, if *x* is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and + equivalent to ``x = x.__iadd__(y)`` . If :meth:`__iadd__` does not exist, or if ``x.__iadd__(y)`` + returns :data:`!NotImplemented`, ``x.__add__(y)`` and ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In certain situations, augmented assignment can result in unexpected errors (see :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact @@ -2893,11 +3129,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - The built-in function :func:`int` falls back to :meth:`__trunc__` if neither - :meth:`__int__` nor :meth:`__index__` is defined. - - .. versionchanged:: 3.11 - The delegation of :func:`int` to :meth:`__trunc__` is deprecated. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. .. _context-managers: @@ -2939,7 +3172,7 @@ For more information on context managers, see :ref:`typecontextmanager`. (i.e., prevent it from being propagated), it should return a true value. Otherwise, the exception will be processed normally upon exit from this method. - Note that :meth:`__exit__` methods should not reraise the passed-in exception; + Note that :meth:`~object.__exit__` methods should not reraise the passed-in exception; this is the caller's responsibility. @@ -3167,7 +3400,7 @@ generators, coroutines do not directly support iteration. to the :meth:`~generator.send` method of the iterator that caused the coroutine to suspend. The result (return value, :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. + iterating over the :meth:`!__await__` return value, described above. .. method:: coroutine.throw(value) coroutine.throw(type[, value[, traceback]]) @@ -3257,12 +3490,12 @@ Asynchronous context managers can be used in an :keyword:`async with` statement. .. method:: object.__aenter__(self) - Semantically similar to :meth:`__enter__`, the only + Semantically similar to :meth:`~object.__enter__`, the only difference being that it must return an *awaitable*. .. method:: object.__aexit__(self, exc_type, exc_value, traceback) - Semantically similar to :meth:`__exit__`, the only + Semantically similar to :meth:`~object.__exit__`, the only difference being that it must return an *awaitable*. An example of an asynchronous context manager class:: @@ -3291,7 +3524,7 @@ An example of an asynchronous context manager class:: the behavior that ``None`` is not callable. .. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to + the method returns :data:`NotImplemented`. Do not set the method to ``None`` if you want to force fallback to the right operand's reflected method—that will instead have the opposite effect of explicitly *blocking* such fallback. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cea3a56ba51644..f24e1537af39ed 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -139,8 +139,9 @@ namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The global namespace is searched first. If the names are not found there, the -builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the listed names. +builtins namespace is searched next. If the names are also not found in the +builtins namespace, new variables are created in the global namespace. +The global statement must precede all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains @@ -204,7 +205,7 @@ Annotation scopes are used in the following contexts: * Type parameter lists for :ref:`generic classes <generic-classes>`. A generic class's base classes and keyword arguments are executed within the annotation scope, but its decorators are not. -* The bounds and constraints for type variables +* The bounds, constraints, and default values for type parameters (:ref:`lazily evaluated <lazy-evaluation>`). * The value of type aliases (:ref:`lazily evaluated <lazy-evaluation>`). @@ -231,13 +232,17 @@ Annotation scopes differ from function scopes in the following ways: .. versionadded:: 3.12 Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. +.. versionchanged:: 3.13 + Annotation scopes are also used for type parameter defaults, as + introduced by :pep:`696`. + .. _lazy-evaluation: Lazy evaluation --------------- The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds and constraints of type +*lazily evaluated*. The same applies to the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax <type-params>`. This means that they are not evaluated when the type alias or type variable is created. Instead, they are only evaluated when doing so is necessary to resolve diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 5e9d12c0087a94..cfada6e824c4bf 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -14,7 +14,7 @@ be used to describe syntax, not lexical analysis. When (one alternative of) a syntax rule has the form .. productionlist:: python-grammar - name: `othername` + name: othername and no semantics are given, the semantics of this form of ``name`` are the same as for ``othername``. @@ -83,18 +83,47 @@ exception. pair: name; mangling pair: private; names -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. +Private name mangling +^^^^^^^^^^^^^^^^^^^^^ +When an identifier that textually occurs in a class definition begins with two +or more underscore characters and does not end in two or more underscores, it +is considered a :dfn:`private name` of that class. + +.. seealso:: + + The :ref:`class specifications <class>`. + +More precisely, private names are transformed to a longer form before code is +generated for them. If the transformed name is longer than 255 characters, +implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which the +identifier is used but only the following private identifiers are mangled: + +- Any name used as the name of a variable that is assigned or read or any + name of an attribute being accessed. + + The ``__name__`` attribute of nested functions, classes, and type aliases + is however not mangled. + +- The name of imported modules, e.g., ``__spam`` in ``import __spam``. + If the module is part of a package (i.e., its name contains a dot), + the name is *not* mangled, e.g., the ``__foo`` in ``import __foo.bar`` + is not mangled. + +- The name of an imported member, e.g., ``__f`` in ``from spam import __f``. + +The transformation rule is defined as follows: + +- The class name, with leading underscores removed and a single leading + underscore inserted, is inserted in front of the identifier, e.g., the + identifier ``__spam`` occurring in a class named ``Foo``, ``_Foo`` or + ``__Foo`` is transformed to ``_Foo__spam``. + +- If the class name consists only of underscores, the transformation is the + identity, e.g., the identifier ``__spam`` occurring in a class named ``_`` + or ``__`` is left as is. .. _atom-literals: @@ -218,10 +247,12 @@ A comprehension in an :keyword:`!async def` function may consist of either a :keyword:`!for` or :keyword:`!async for` clause following the leading expression, may contain additional :keyword:`!for` or :keyword:`!async for` clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`!async for` clauses or -:keyword:`!await` expressions or other asynchronous comprehensions it is called -an :dfn:`asynchronous comprehension`. An asynchronous comprehension may -suspend the execution of the coroutine function in which it appears. + +If a comprehension contains :keyword:`!async for` clauses, or if it contains +:keyword:`!await` expressions or other asynchronous comprehensions anywhere except +the iterable expression in the leftmost :keyword:`!for` clause, it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the +execution of the coroutine function in which it appears. See also :pep:`530`. .. versionadded:: 3.6 @@ -422,7 +453,8 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" - yield_expression: "yield" [`expression_list` | "from" `expression`] + yield_from: "yield" "from" `expression` + yield_expression: "yield" `expression_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -823,12 +855,18 @@ An attribute reference is a primary followed by a period and a name: The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier. This production can be customized by -overriding the :meth:`__getattr__` method. If this attribute is not available, -the exception :exc:`AttributeError` is raised. Otherwise, the type and value of -the object produced is determined by the object. Multiple evaluations of the -same attribute reference may yield different objects. +attribute whose name is the identifier. The type and value produced is +determined by the object. Multiple evaluations of the same attribute +reference may yield different objects. + +This production can be customized by overriding the +:meth:`~object.__getattribute__` method or the :meth:`~object.__getattr__` +method. The :meth:`!__getattribute__` method is called first and either +returns a value or raises :exc:`AttributeError` if the attribute is not +available. +If an :exc:`AttributeError` is raised and the object has a :meth:`!__getattr__` +method, that method is called as a fallback. .. _subscriptions: @@ -889,7 +927,7 @@ to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The resulting value must be a nonnegative integer less than the number of items in the sequence, and the subscription selects the item whose index is that value (counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`__getitem__` method, subclasses overriding +occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding this method will need to explicitly add that support. .. index:: @@ -944,7 +982,7 @@ slice list contains no proper slice). single: step (slice object attribute) The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`__getitem__` method as +same :meth:`~object.__getitem__` method as normal subscription) with a key that is constructed from the slice list, as follows. If the slice list contains at least one comma, the key is a tuple containing the conversion of the slice items; otherwise, the conversion of the @@ -994,7 +1032,7 @@ but does not affect the semantics. The primary must evaluate to a callable object (user-defined functions, built-in functions, methods of built-in objects, class objects, methods of class -instances, and all objects having a :meth:`__call__` method are callable). All +instances, and all objects having a :meth:`~object.__call__` method are callable). All argument expressions are evaluated before the call is attempted. Please refer to section :ref:`function` for the syntax of formal :term:`parameter` lists. @@ -1152,7 +1190,7 @@ a class instance: pair: instance; call single: __call__() (object method) - The class must define a :meth:`__call__` method; the effect is then the same as + The class must define a :meth:`~object.__call__` method; the effect is then the same as if that method was called. @@ -1204,7 +1242,8 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) -This operation can be customized using the special :meth:`__pow__` method. +This operation can be customized using the special :meth:`~object.__pow__` and +:meth:`~object.__rpow__` methods. .. _unary: @@ -1227,7 +1266,7 @@ All unary arithmetic and bitwise operations have the same priority: single: - (minus); unary operator The unary ``-`` (minus) operator yields the negation of its numeric argument; the -operation can be overridden with the :meth:`__neg__` special method. +operation can be overridden with the :meth:`~object.__neg__` special method. .. index:: single: plus @@ -1235,7 +1274,7 @@ operation can be overridden with the :meth:`__neg__` special method. single: + (plus); unary operator The unary ``+`` (plus) operator yields its numeric argument unchanged; the -operation can be overridden with the :meth:`__pos__` special method. +operation can be overridden with the :meth:`~object.__pos__` special method. .. index:: single: inversion @@ -1244,7 +1283,7 @@ operation can be overridden with the :meth:`__pos__` special method. The unary ``~`` (invert) operator yields the bitwise inversion of its integer argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only applies to integral numbers or to custom objects that override the -:meth:`__invert__` special method. +:meth:`~object.__invert__` special method. @@ -1282,8 +1321,8 @@ the other must be a sequence. In the former case, the numbers are converted to a common type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. -This operation can be customized using the special :meth:`__mul__` and -:meth:`__rmul__` methods. +This operation can be customized using the special :meth:`~object.__mul__` and +:meth:`~object.__rmul__` methods. .. index:: single: matrix multiplication @@ -1292,6 +1331,9 @@ This operation can be customized using the special :meth:`__mul__` and The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. +This operation can be customized using the special :meth:`~object.__matmul__` and +:meth:`~object.__rmatmul__` methods. + .. versionadded:: 3.5 .. index:: @@ -1307,8 +1349,10 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`__truediv__` and -:meth:`__floordiv__` methods. +The division operation can be customized using the special :meth:`~object.__truediv__` +and :meth:`~object.__rtruediv__` methods. +The floor division operation can be customized using the special +:meth:`~object.__floordiv__` and :meth:`~object.__rfloordiv__` methods. .. index:: single: modulo @@ -1333,7 +1377,8 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. -The *modulo* operation can be customized using the special :meth:`__mod__` method. +The *modulo* operation can be customized using the special :meth:`~object.__mod__` +and :meth:`~object.__rmod__` methods. The floor division operator, the modulo operator, and the :func:`divmod` function are not defined for complex numbers. Instead, convert to a floating @@ -1349,8 +1394,8 @@ must either both be numbers or both be sequences of the same type. In the former case, the numbers are converted to a common type and then added together. In the latter case, the sequences are concatenated. -This operation can be customized using the special :meth:`__add__` and -:meth:`__radd__` methods. +This operation can be customized using the special :meth:`~object.__add__` and +:meth:`~object.__radd__` methods. .. index:: single: subtraction @@ -1360,7 +1405,8 @@ This operation can be customized using the special :meth:`__add__` and The ``-`` (subtraction) operator yields the difference of its arguments. The numeric arguments are first converted to a common type. -This operation can be customized using the special :meth:`__sub__` method. +This operation can be customized using the special :meth:`~object.__sub__` and +:meth:`~object.__rsub__` methods. .. _shifting: @@ -1381,8 +1427,10 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. -This operation can be customized using the special :meth:`__lshift__` and -:meth:`__rshift__` methods. +The left shift operation can be customized using the special :meth:`~object.__lshift__` +and :meth:`~object.__rlshift__` methods. +The right shift operation can be customized using the special :meth:`~object.__rshift__` +and :meth:`~object.__rrshift__` methods. .. index:: pair: exception; ValueError @@ -1409,8 +1457,8 @@ Each of the three bitwise operations has a different priority level: pair: operator; & (ampersand) The ``&`` operator yields the bitwise AND of its arguments, which must be -integers or one of them must be a custom object overriding :meth:`__and__` or -:meth:`__rand__` special methods. +integers or one of them must be a custom object overriding :meth:`~object.__and__` or +:meth:`~object.__rand__` special methods. .. index:: pair: bitwise; xor @@ -1418,8 +1466,8 @@ integers or one of them must be a custom object overriding :meth:`__and__` or pair: operator; ^ (caret) The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__xor__` or -:meth:`__rxor__` special methods. +must be integers or one of them must be a custom object overriding :meth:`~object.__xor__` or +:meth:`~object.__rxor__` special methods. .. index:: pair: bitwise; or @@ -1427,8 +1475,8 @@ must be integers or one of them must be a custom object overriding :meth:`__xor_ pair: operator; | (vertical bar) The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__or__` or -:meth:`__ror__` special methods. +must be integers or one of them must be a custom object overriding :meth:`~object.__or__` or +:meth:`~object.__ror__` special methods. .. _comparisons: @@ -1495,7 +1543,7 @@ comparison implementation. Because all types are (direct or indirect) subtypes of :class:`object`, they inherit the default comparison behavior from :class:`object`. Types can customize their comparison behavior by implementing -:dfn:`rich comparison methods` like :meth:`__lt__`, described in +:dfn:`rich comparison methods` like :meth:`~object.__lt__`, described in :ref:`customization`. The default behavior for equality comparison (``==`` and ``!=``) is based on @@ -1532,7 +1580,7 @@ built-in types. ``x == x`` are all false, while ``x != x`` is true. This behavior is compliant with IEEE 754. -* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that +* ``None`` and :data:`NotImplemented` are singletons. :PEP:`8` advises that comparisons for singletons should always be done with ``is`` or ``is not``, never the equality operators. @@ -1659,18 +1707,18 @@ substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. -For user-defined classes which define the :meth:`__contains__` method, ``x in +For user-defined classes which define the :meth:`~object.__contains__` method, ``x in y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. -For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +For user-defined classes which do not define :meth:`~object.__contains__` but do define +:meth:`~object.__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the expression ``x is z or x == z`` is true, is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative +:meth:`~object.__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). @@ -1781,10 +1829,11 @@ Or, when processing a file stream in chunks: while chunk := file.read(9000): process(chunk) -Assignment expressions must be surrounded by parentheses when used -as sub-expressions in slicing, conditional, lambda, -keyword-argument, and comprehension-if expressions -and in ``assert`` and ``with`` statements. +Assignment expressions must be surrounded by parentheses when +used as expression statements and when used as sub-expressions in +slicing, conditional, lambda, +keyword-argument, and comprehension-if expressions and +in ``assert``, ``with``, and ``assignment`` statements. In all other places where they can be used, parentheses are not required, including in ``if`` and ``while`` statements. @@ -1882,8 +1931,9 @@ the unpacking. .. index:: pair: trailing; comma -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a +A trailing comma is required only to create a one-item tuple, +such as ``1,``; it is optional in all other cases. +A single expression without a trailing comma doesn't create a tuple, but rather yields the value of that expression. (To create an empty tuple, use an empty pair of parentheses: ``()``.) diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index bc1db7b039cd5a..b9cca4444c9141 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -1,3 +1,5 @@ +.. _full-grammar-specification: + Full Grammar specification ========================== diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 1a2677e7562b9c..f8c9724114da9e 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -327,14 +327,15 @@ modules, and one that knows how to import modules from an :term:`import path` finders replaced :meth:`!find_module`, which is now deprecated. While it will continue to work without change, the import machinery will try it only if the finder does not implement - ``find_spec()``. + :meth:`~importlib.abc.MetaPathFinder.find_spec`. .. versionchanged:: 3.10 Use of :meth:`!find_module` by the import system now raises :exc:`ImportWarning`. .. versionchanged:: 3.12 - ``find_module()`` has been removed. Use :meth:`find_spec` instead. + :meth:`!find_module` has been removed. + Use :meth:`~importlib.abc.MetaPathFinder.find_spec` instead. Loading @@ -559,7 +560,7 @@ listed below. functionality, for example getting data associated with a loader. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of this attribute. + instead of this attribute. .. versionchanged:: 3.12 The value of ``__loader__`` is expected to be the same as @@ -580,7 +581,7 @@ listed below. relative imports for main modules, as defined in :pep:`366`. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of this attribute. + instead of this attribute. .. versionchanged:: 3.6 The value of ``__package__`` is expected to be the same as @@ -650,7 +651,7 @@ listed below. from a file, that atypical scenario may be appropriate. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of ``__cached__``. + instead of ``__cached__``. .. _package-path-rules: @@ -812,7 +813,7 @@ attributes on package objects are also used. These provide additional ways that the import machinery can be customized. :data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` +modules and packages. It is initialized from the :envvar:`PYTHONPATH` environment variable and various other installation- and implementation-specific defaults. Entries in :data:`sys.path` can name directories on the file system, zip files, and potentially other "locations" diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 81f0a5c5d43883..cf186705e6e987 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -90,7 +90,8 @@ Notation .. index:: BNF, grammar, syntax, notation -The descriptions of lexical analysis and syntax use a modified BNF grammar +The descriptions of lexical analysis and syntax use a modified +`Backus–Naur form (BNF) <https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form>`_ grammar notation. This uses the following style of definition: .. productionlist:: notation diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 3e07d16068a627..41ea89fd234122 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -96,10 +96,9 @@ which is recognized also by GNU Emacs, and :: which is recognized by Bram Moolenaar's VIM. -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). +If no encoding declaration is found, the default encoding is UTF-8. If the +implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order +mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The @@ -514,7 +513,6 @@ is not supported. The ``'rb'`` prefix of raw bytes literals has been added as a synonym of ``'br'``. -.. versionadded:: 3.3 Support for the unicode legacy literal (``u'value'``) was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. @@ -708,10 +706,12 @@ and formatted string literals may be concatenated with plain string literals. single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals + .. _f-strings: +.. _formatted-string-literals: -Formatted string literals -------------------------- +f-strings +--------- .. versionadded:: 3.6 @@ -732,7 +732,7 @@ for the contents of the string is: : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* + format_spec: (`literal_char` | `replacement_field`)* literal_char: <any code point except "{", "}" or NULL> The parts of the string outside curly braces are treated literally, diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a9e65be1eda340..618664b23f0680 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -214,7 +214,7 @@ Assignment of an object to a single target is recursively defined as follows. object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the :meth:`__setitem__` method is called with + For user-defined objects, the :meth:`~object.__setitem__` method is called with appropriate arguments. .. index:: pair: slicing; assignment @@ -293,7 +293,7 @@ statements, cannot be an unpacking) and the expression list, performs the binary operation specific to the type of assignment on the two operands, and assigns the result to the original target. The target is only evaluated once. -An augmented assignment expression like ``x += 1`` can be rewritten as ``x = x + +An augmented assignment statement like ``x += 1`` can be rewritten as ``x = x + 1`` to achieve a similar, but not exactly equal effect. In the augmented version, ``x`` is only evaluated once. Also, when possible, the actual operation is performed *in-place*, meaning that rather than creating a new object and @@ -333,7 +333,9 @@ statement, of a variable or attribute annotation and an optional assignment stat The difference from normal :ref:`assignment` is that only a single target is allowed. -For simple names as assignment targets, if in class or module scope, +The assignment target is considered "simple" if it consists of a single +name that is not enclosed in parentheses. +For simple assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module attribute :attr:`__annotations__` that is a dictionary mapping from variable names (mangled if private) to @@ -341,7 +343,8 @@ evaluated annotations. This attribute is writable and is automatically created at the start of class or module body execution, if annotations are found statically. -For expressions as assignment targets, the annotations are evaluated if +If the assignment target is not simple (an attribute, subscript node, or +parenthesized name), the annotation is evaluated if in class or module scope, but not stored. If a name is annotated in a function scope, then this name is local for @@ -351,7 +354,7 @@ If the right hand side is present, an annotated assignment performs the actual assignment before evaluating annotations (where applicable). If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last -:meth:`__setitem__` or :meth:`__setattr__` call. +:meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. .. seealso:: @@ -578,7 +581,7 @@ The :dfn:`type` of the exception is the exception instance's class, the .. index:: pair: object; traceback A traceback object is normally created automatically when an exception is raised -and attached to it as the :attr:`__traceback__` attribute, which is writable. +and attached to it as the :attr:`~BaseException.__traceback__` attribute. You can create an exception and set your own traceback in one step using the :meth:`~BaseException.with_traceback` exception method (which returns the same exception instance, with its traceback set to its argument), like so:: @@ -592,11 +595,13 @@ same exception instance, with its traceback set to its argument), like so:: The ``from`` clause is used for exception chaining: if given, the second *expression* must be another exception class or instance. If the second expression is an exception instance, it will be attached to the raised -exception as the :attr:`__cause__` attribute (which is writable). If the +exception as the :attr:`~BaseException.__cause__` attribute (which is writable). If the expression is an exception class, the class will be instantiated and the resulting exception instance will be attached to the raised exception as the -:attr:`__cause__` attribute. If the raised exception is not handled, both -exceptions will be printed:: +:attr:`!__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -605,19 +610,24 @@ exceptions will be printed:: ... Traceback (most recent call last): File "<stdin>", line 2, in <module> + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: Traceback (most recent call last): File "<stdin>", line 4, in <module> + raise RuntimeError("Something bad happened") from exc RuntimeError: Something bad happened A similar mechanism works implicitly if a new exception is raised when an exception is already being handled. An exception may be handled when an :keyword:`except` or :keyword:`finally` clause, or a :keyword:`with` statement, is used. The previous exception is then -attached as the new exception's :attr:`__context__` attribute:: +attached as the new exception's :attr:`~BaseException.__context__` attribute: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -626,16 +636,21 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "<stdin>", line 2, in <module> + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<stdin>", line 4, in <module> + raise RuntimeError("Something bad happened") RuntimeError: Something bad happened Exception chaining can be explicitly suppressed by specifying :const:`None` in -the ``from`` clause:: +the ``from`` clause: + +.. doctest:: >>> try: ... print(1 / 0) @@ -652,9 +667,8 @@ and information about handling exceptions is in section :ref:`try`. .. versionchanged:: 3.3 :const:`None` is now permitted as ``Y`` in ``raise X from Y``. -.. versionadded:: 3.3 - The ``__suppress_context__`` attribute to suppress automatic display of the - exception context. + Added the :attr:`~BaseException.__suppress_context__` attribute to suppress + automatic display of the exception context. .. versionchanged:: 3.11 If the traceback of the active exception is modified in an :keyword:`except` @@ -920,7 +934,7 @@ That is not a future statement; it's an ordinary import statement with no special semantics or syntax restrictions. Code compiled by calls to the built-in functions :func:`exec` and :func:`compile` -that occur in a module :mod:`M` containing a future statement will, by default, +that occur in a module :mod:`!M` containing a future statement will, by default, use the new syntax or semantics associated with the future statement. This can be controlled by optional arguments to :func:`compile` --- see the documentation of that function for details. @@ -995,25 +1009,29 @@ The :keyword:`!nonlocal` statement .. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The :keyword:`nonlocal` statement +causes the listed identifiers to refer to names previously bound in +nonlocal scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a :exc:`SyntaxError` is raised. -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). - -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. +The nonlocal statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. .. seealso:: :pep:`3104` - Access to Names in Outer Scopes The specification for the :keyword:`nonlocal` statement. +**Programmer's note:** :keyword:`nonlocal` is a directive to the parser +and applies only to code parsed along with it. See the note for the +:keyword:`global` statement. + + .. _type: The :keyword:`!type` statement diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 5de739fc10b085..068fe0cb426ecd 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -7,30 +7,29 @@ blurb python-docs-theme>=2022.1 # Generated from: -# pip install "Sphinx~=4.2.0" +# pip install "Sphinx~=6.2.1" # pip freeze # -# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``. +# Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``. -alabaster==0.7.13 -Babel==2.13.0 -certifi==2023.7.22 -charset-normalizer==3.3.0 -colorama==0.4.6 -docutils==0.17.1 -idna==3.4 +alabaster==0.7.16 +Babel==2.15.0 +certifi==2024.7.4 +charset-normalizer==3.3.2 +docutils==0.19 +idna==3.7 imagesize==1.4.1 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -packaging==23.2 -Pygments==2.16.1 -requests==2.31.0 +Jinja2==3.1.4 +MarkupSafe==2.1.5 +packaging==24.1 +Pygments==2.18.0 +requests==2.32.3 snowballstemmer==2.2.0 -Sphinx==4.2.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 +Sphinx==6.2.1 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.6 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +urllib3==2.2.2 diff --git a/Doc/requirements.txt b/Doc/requirements.txt index d4f23ea8c400fe..b47a9d8a8635ab 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,12 +6,12 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -# PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. -sphinx==6.2.1 +sphinx~=7.3.0 blurb sphinxext-opengraph==0.7.5 +sphinx-notfound-page==1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f62ed55282b8d2..66914f79f3d4ec 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -3,125 +3,60 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/descriptor.rst -Doc/c-api/exceptions.rst -Doc/c-api/file.rst Doc/c-api/float.rst -Doc/c-api/gcsupport.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/memory.rst -Doc/c-api/memoryview.rst Doc/c-api/module.rst -Doc/c-api/object.rst -Doc/c-api/set.rst Doc/c-api/stable.rst -Doc/c-api/structures.rst -Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/extending/extending.rst -Doc/extending/newtypes.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/isolating-extensions.rst -Doc/howto/logging.rst -Doc/howto/urllib2.rst -Doc/library/abc.rst Doc/library/ast.rst -Doc/library/asyncio-dev.rst -Doc/library/asyncio-eventloop.rst Doc/library/asyncio-extending.rst -Doc/library/asyncio-policy.rst -Doc/library/asyncio-stream.rst Doc/library/asyncio-subprocess.rst -Doc/library/asyncio-task.rst -Doc/library/bdb.rst -Doc/library/bisect.rst -Doc/library/bz2.rst -Doc/library/calendar.rst -Doc/library/cmd.rst -Doc/library/codecs.rst -Doc/library/collections.abc.rst Doc/library/collections.rst -Doc/library/concurrent.futures.rst -Doc/library/configparser.rst -Doc/library/contextlib.rst -Doc/library/csv.rst -Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst -Doc/library/doctest.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.errors.rst -Doc/library/email.headerregistry.rst -Doc/library/email.mime.rst Doc/library/email.parser.rst Doc/library/email.policy.rst -Doc/library/enum.rst Doc/library/exceptions.rst -Doc/library/faulthandler.rst -Doc/library/fcntl.rst -Doc/library/ftplib.rst -Doc/library/functions.rst Doc/library/functools.rst -Doc/library/getpass.rst -Doc/library/gettext.rst -Doc/library/gzip.rst -Doc/library/http.client.rst Doc/library/http.cookiejar.rst -Doc/library/http.cookies.rst Doc/library/http.server.rst -Doc/library/importlib.resources.rst Doc/library/importlib.rst -Doc/library/inspect.rst -Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst Doc/library/lzma.rst -Doc/library/mailbox.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst -Doc/library/multiprocessing.shared_memory.rst -Doc/library/numbers.rst Doc/library/optparse.rst -Doc/library/os.path.rst Doc/library/os.rst -Doc/library/pickle.rst Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/profile.rst -Doc/library/pyclbr.rst -Doc/library/pydoc.rst Doc/library/pyexpat.rst -Doc/library/random.rst Doc/library/readline.rst -Doc/library/reprlib.rst Doc/library/resource.rst -Doc/library/rlcompleter.rst Doc/library/select.rst -Doc/library/selectors.rst -Doc/library/shelve.rst Doc/library/signal.rst Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst Doc/library/stdtypes.rst -Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/syslog.rst -Doc/library/tarfile.rst -Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst -Doc/library/traceback.rst -Doc/library/tty.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst @@ -130,7 +65,6 @@ Doc/library/wsgiref.rst Doc/library/xml.dom.minidom.rst Doc/library/xml.dom.pulldom.rst Doc/library/xml.dom.rst -Doc/library/xml.rst Doc/library/xml.sax.handler.rst Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst @@ -139,23 +73,13 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/reference/expressions.rst -Doc/reference/import.rst -Doc/reference/simple_stmts.rst Doc/tutorial/datastructures.rst -Doc/tutorial/introduction.rst Doc/using/windows.rst -Doc/whatsnew/2.0.rst -Doc/whatsnew/2.1.rst -Doc/whatsnew/2.2.rst -Doc/whatsnew/2.3.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst Doc/whatsnew/3.0.rst -Doc/whatsnew/3.1.rst -Doc/whatsnew/3.2.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index 809a8d63087e12..67623b83d3a67d 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -13,6 +13,9 @@ from pathlib import Path from typing import TextIO +# Fail if NEWS nit found before this line number +NEWS_NIT_THRESHOLD = 1700 + # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. EXCLUDE_FILES = { @@ -245,6 +248,32 @@ def fail_if_improved( return 0 +def fail_if_new_news_nit(warnings: list[str], threshold: int) -> int: + """ + Ensure no warnings are found in the NEWS file before a given line number. + """ + news_nits = ( + warning + for warning in warnings + if "/build/NEWS:" in warning + ) + + # Nits found before the threshold line + new_news_nits = [ + nit + for nit in news_nits + if int(nit.split(":")[1]) <= threshold + ] + + if new_news_nits: + print("\nError: new NEWS nits:\n") + for warning in new_news_nits: + print(warning) + return -1 + + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -264,6 +293,14 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Fail if new files with no nits are found", ) + parser.add_argument( + "--fail-if-new-news-nit", + metavar="threshold", + type=int, + nargs="?", + const=NEWS_NIT_THRESHOLD, + help="Fail if new NEWS nit found before threshold line number", + ) args = parser.parse_args(argv) if args.annotate_diff is not None and len(args.annotate_diff) > 2: @@ -304,6 +341,9 @@ def main(argv: list[str] | None = None) -> int: if args.fail_if_improved: exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + if args.fail_if_new_news_nit: + exit_code += fail_if_new_news_nit(warnings, args.fail_if_new_news_nit) + return exit_code diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 3551bfa4c0f133..7916b178f1c0f1 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ c_annotations.py ~~~~~~~~~~~~~~~~ @@ -20,7 +19,6 @@ """ from os import path -import docutils from docutils import nodes from docutils.parsers.rst import directives from docutils.parsers.rst import Directive @@ -34,24 +32,13 @@ REST_ROLE_MAP = { 'function': 'func', - 'var': 'data', - 'type': 'type', 'macro': 'macro', - 'type': 'type', 'member': 'member', + 'type': 'type', + 'var': 'data', } -# Monkeypatch nodes.Node.findall for forwards compatability -# This patch can be dropped when the minimum Sphinx version is 4.4.0 -# or the minimum Docutils version is 0.18.1. -if docutils.__version_info__ < (0, 18, 1): - def findall(self, *args, **kwargs): - return iter(self.traverse(*args, **kwargs)) - - nodes.Node.findall = findall - - class RCEntry: def __init__(self, name): self.name = name @@ -63,7 +50,7 @@ def __init__(self, name): class Annotations: def __init__(self, refcount_filename, stable_abi_file): self.refcount_data = {} - with open(refcount_filename, 'r') as fp: + with open(refcount_filename, encoding='utf8') as fp: for line in fp: line = line.strip() if line[:1] in ("", "#"): @@ -71,7 +58,7 @@ def __init__(self, refcount_filename, stable_abi_file): continue parts = line.split(":", 4) if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) + raise ValueError(f"Wrong field count in {line!r}") function, type, arg, refcount, comment = parts # Get the entry, creating it if needed: try: @@ -91,9 +78,8 @@ def __init__(self, refcount_filename, stable_abi_file): entry.result_refs = refcount self.stable_abi_data = {} - with open(stable_abi_file, 'r') as fp: + with open(stable_abi_file, encoding='utf8') as fp: for record in csv.DictReader(fp): - role = record['role'] name = record['name'] self.stable_abi_data[name] = record @@ -126,7 +112,8 @@ def add_annotations(self, app, doctree): f"Object type mismatch in limited API annotation " f"for {name}: {record['role']!r} != {objtype!r}") stable_added = record['added'] - message = ' Part of the ' + message = sphinx_gettext('Part of the') + message = message.center(len(message) + 2) emph_node = nodes.emphasis(message, message, classes=['stableabi']) ref_node = addnodes.pending_xref( @@ -134,40 +121,40 @@ def add_annotations(self, app, doctree): reftype='ref', refexplicit="False") struct_abi_kind = record['struct_abi_kind'] if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text('Limited API') + ref_node += nodes.Text(sphinx_gettext('Limited API')) else: - ref_node += nodes.Text('Stable ABI') + ref_node += nodes.Text(sphinx_gettext('Stable ABI')) emph_node += ref_node if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' (as an opaque struct)') + emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' (including all members)') + emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) if record['ifdef_note']: emph_node += nodes.Text(' ' + record['ifdef_note']) if stable_added == '3.2': # Stable ABI was introduced in 3.2. pass else: - emph_node += nodes.Text(f' since version {stable_added}') + emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) emph_node += nodes.Text('.') if struct_abi_kind == 'members': emph_node += nodes.Text( - ' (Only some members are part of the stable ABI.)') + ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) node.insert(0, emph_node) # Unstable API annotation. if name.startswith('PyUnstable'): warn_node = nodes.admonition( classes=['unstable-c-api', 'warning']) - message = 'This is ' + message = sphinx_gettext('This is') + ' ' emph_node = nodes.emphasis(message, message) ref_node = addnodes.pending_xref( 'Unstable API', refdomain="std", reftarget='unstable-c-api', reftype='ref', refexplicit="False") - ref_node += nodes.Text('Unstable API') + ref_node += nodes.Text(sphinx_gettext('Unstable API')) emph_node += ref_node - emph_node += nodes.Text('. It may change without warning in minor releases.') + emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) warn_node += emph_node node.insert(0, warn_node) @@ -179,13 +166,17 @@ def add_annotations(self, app, doctree): continue elif not entry.result_type.endswith("Object*"): continue + classes = ['refcount'] if entry.result_refs is None: rc = sphinx_gettext('Return value: Always NULL.') + classes.append('return_null') elif entry.result_refs: rc = sphinx_gettext('Return value: New reference.') + classes.append('return_new_ref') else: rc = sphinx_gettext('Return value: Borrowed reference.') - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) + classes.append('return_borrowed_ref') + node.insert(0, nodes.emphasis(rc, rc, classes=classes)) def init_annotations(app): @@ -227,6 +218,7 @@ def setup(app): 'stableabi': directives.flag, } old_handle_signature = CObject.handle_signature + def new_handle_signature(self, sig, signode): signode.parent['stableabi'] = 'stableabi' in self.options return old_handle_signature(self, sig, signode) diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 59a6862ea3d3f4..7c93b1e4990603 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -20,13 +20,13 @@ def process_glossary_nodes(app, doctree, fromdocname): - if app.builder.format != 'html': + if app.builder.format != 'html' or app.builder.embedded: return terms = {} - for node in doctree.traverse(glossary): - for glossary_item in node.traverse(definition_list_item): + for node in doctree.findall(glossary): + for glossary_item in node.findall(definition_list_item): term = glossary_item[0].astext().lower() definition = glossary_item[1] diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 4ba5f5a1fbdc1b..8b592d4b4adcea 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -26,19 +26,13 @@ from sphinx.locale import _ as sphinx_gettext from sphinx.util import logging from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator - -try: - # Sphinx 6+ - from sphinx.util.display import status_iterator -except ImportError: - # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 - from sphinx.util import status_iterator +from sphinx.util.display import status_iterator ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' +# Used in conf.py and updated here by python/release-tools/run_release.py SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists @@ -48,6 +42,12 @@ Body.enum.converters['lowerroman'] = \ Body.enum.converters['upperroman'] = lambda x: None +# monkey-patch the productionlist directive to allow hyphens in group names +# https://github.com/sphinx-doc/sphinx/issues/11854 +from sphinx.domains import std + +std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`') + # Support for marking up and linking to bugs.python.org issues @@ -80,16 +80,6 @@ def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): return [refnode], [] -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - # Support for marking up implementation details class ImplementationDetail(Directive): @@ -127,8 +117,8 @@ class Availability(SphinxDirective): # known platform, libc, and threading implementations known_platforms = frozenset({ "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", "Unix", "VxWorks", - "WASI", "Windows", "macOS", + "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", + "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS", # libc "BSD libc", "glibc", "musl", # POSIX platforms with pthreads @@ -159,7 +149,7 @@ def parse_platforms(self): Example:: - .. availability:: Windows, Linux >= 4.2, not Emscripten, not WASI + .. availability:: Windows, Linux >= 4.2, not WASI Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not parsed into separate tokens. @@ -189,7 +179,6 @@ def parse_platforms(self): return platforms - # Support for documenting audit event def audit_events_purge(app, env, docname): @@ -607,8 +596,15 @@ def parse_pdb_command(env, sig, signode): return fullname +def parse_monitoring_event(env, sig, signode): + """Transform a monitoring event signature into RST nodes.""" + signode += addnodes.desc_addname('sys.monitoring.events.', 'sys.monitoring.events.') + signode += addnodes.desc_name(sig, sig) + return sig + + def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): break else: return @@ -667,7 +663,7 @@ def process_audit_events(app, doctree, fromdocname): body += row - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): node.replace_self(table) @@ -698,7 +694,6 @@ def patch_pairindextypes(app, _env) -> None: def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) - app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) app.add_directive('audit-event', AuditEvent) @@ -707,6 +702,7 @@ def setup(app): app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) + app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event) app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js index c881a9bd4c84a7..0a77c0d71ae937 100644 --- a/Doc/tools/static/changelog_search.js +++ b/Doc/tools/static/changelog_search.js @@ -1,53 +1,59 @@ -$(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '<p>Filter entries by content:', - '<input type="text" value="" id="searchbox" style="width: 50%">', - '<input type="submit" id="searchbox-submit" value="Filter"></p>' - ].join('\n')); +document.addEventListener("DOMContentLoaded", function () { + // add the search form and bind the events + document + .querySelector("h1") + .insertAdjacentHTML( + "afterend", + [ + "<p>Filter entries by content:", + '<input type="text" value="" id="searchbox" style="width: 50%">', + '<input type="submit" id="searchbox-submit" value="Filter"></p>', + ].join("\n"), + ); - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); + function doFilter() { + let query; + try { + query = new RegExp(document.querySelector("#searchbox").value, "i"); + } catch (e) { + return; // not a valid regex (yet) + } + // find headers for the versions (What's new in Python X.Y.Z?) + const h2s = document.querySelectorAll("#changelog h2"); + for (const h2 of h2s) { + let sections_found = 0; + // find headers for the sections (Core, Library, etc.) + const h3s = h2.parentNode.querySelectorAll("h3"); + for (const h3 of h3s) { + let entries_found = 0; + // find all the entries + const lis = h3.parentNode.querySelectorAll("li"); + for (let li of lis) { + // check if the query matches the entry + if (query.test(li.textContent)) { + li.style.display = "block"; + entries_found++; + } else { + li.style.display = "none"; + } } - catch (e) { - return; // not a valid regex (yet) + // if there are entries, show the section, otherwise hide it + if (entries_found > 0) { + h3.parentNode.style.display = "block"; + sections_found++; + } else { + h3.parentNode.style.display = "none"; } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); + } + if (sections_found > 0) { + h2.parentNode.style.display = "block"; + } else { + h2.parentNode.style.display = "none"; + } } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); + } + document.querySelector("#searchbox").addEventListener("keyup", doFilter); + document + .querySelector("#searchbox-submit") + .addEventListener("click", doFilter); }); diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js new file mode 100644 index 00000000000000..f5dc7045a0dbc4 --- /dev/null +++ b/Doc/tools/static/rtd_switcher.js @@ -0,0 +1,55 @@ + function onSwitch(event) { + const option = event.target.selectedIndex; + const item = event.target.options[option]; + window.location.href = item.dataset.url; + } + + document.addEventListener("readthedocs-addons-data-ready", function(event) { + const config = event.detail.data() + const versionSelect = ` + <select id="version_select"> + ${ config.versions.active.map( + (version) => ` + <option + value="${ version.slug }" + ${ config.versions.current.slug === version.slug ? 'selected="selected"' : '' } + data-url="${ version.urls.documentation }"> + ${ version.slug } + </option>` + ).join("\n") } + </select> + `; + + // Prepend the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + + const languageSelect = ` + <select id="language_select"> + ${ languages.map( + (translation) => ` + <option + value="${ translation.slug }" + ${ config.projects.current.slug === translation.slug ? 'selected="selected"' : '' } + data-url="${ translation.urls.documentation }"> + ${ translation.language.name } + </option>` + ).join("\n") } + </select> + `; + + // Query all the placeholders because there are different ones for Desktop/Mobile + const versionPlaceholders = document.querySelectorAll(".version_switcher_placeholder"); + for (placeholder of versionPlaceholders) { + placeholder.innerHTML = versionSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + + const languagePlaceholders = document.querySelectorAll(".language_switcher_placeholder"); + for (placeholder of languagePlaceholders) { + placeholder.innerHTML = languageSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + }); diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 7920e0619f9337..b5353d6fb77ab4 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -49,12 +49,12 @@ <h2>Unpacking</h2> <p>Unix users should download the .tar.bz2 archives; these are bzipped tar archives and can be handled in the usual way using tar and the bzip2 -program. The <a href="http://www.info-zip.org">InfoZIP</a> unzip program can be +program. The <a href="https://infozip.sourceforge.net">Info-ZIP</a> unzip program can be used to handle the ZIP archives if desired. The .tar.bz2 archives provide the best compression and fastest download times.</p> <p>Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the InfoZIP zip program.</p> +platform. These are created on Unix using the Info-ZIP zip program.</p> <h2>Problems</h2> diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index bab4aaeb4604b8..49c2a71a5e40cf 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -9,6 +9,16 @@ In extensions/c_annotations.py: +{% trans %}Part of the{% endtrans %} +{% trans %}Limited API{% endtrans %} +{% trans %}Stable ABI{% endtrans %} +{% trans %}(as an opaque struct){% endtrans %} +{% trans %}(including all members){% endtrans %} +{% trans %}since version %s{% endtrans %} +{% trans %}(Only some members are part of the stable ABI.){% endtrans %} +{% trans %}This is{% endtrans %} +{% trans %}Unstable API{% endtrans %} +{% trans %}. It may change without warning in minor releases.{% endtrans %} {% trans %}Return value: Always NULL.{% endtrans %} {% trans %}Return value: New reference.{% endtrans %} {% trans %}Return value: Borrowed reference.{% endtrans %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1e3ab7cfe02fee..6f854e86ab8ef1 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -7,62 +7,62 @@ <h1>{{ docstitle|e }}</h1> <p> {% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %} </p> - <p><strong>{% trans %}Parts of the documentation:{% endtrans %}</strong></p> + <p><strong>{% trans %}Documentation sections:{% endtrans %}</strong></p> <table class="contentstable" align="center"><tr> <td width="50%"> <p class="biglink"><a class="biglink" href="{{ pathto("whatsnew/" + version) }}">{% trans %}What's new in Python {{ version }}?{% endtrans %}</a><br/> - <span class="linkdescr"> {% trans whatsnew_index=pathto("whatsnew/index") %}or <a href="{{ whatsnew_index }}">all "What's new" documents</a> since 2.0{% endtrans %}</span></p> + <span class="linkdescr"> {% trans whatsnew_index=pathto("whatsnew/index") %}Or <a href="{{ whatsnew_index }}">all "What's new" documents since Python 2.0</a>{% endtrans %}</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("tutorial/index") }}">{% trans %}Tutorial{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}start here{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("library/index") }}">{% trans %}Library Reference{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}keep this under your pillow{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">{% trans %}Language Reference{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}describes syntax and language elements{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("using/index") }}">{% trans %}Python Setup and Usage{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}how to use Python on different platforms{% endtrans %}</span></p> + <span class="linkdescr">{% trans %}Start here: a tour of Python's syntax and features{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("library/index") }}">{% trans %}Library reference{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}Standard library and builtins{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">{% trans %}Language reference{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}Syntax and language elements{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("using/index") }}">{% trans %}Python setup and usage{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}How to install, configure, and use Python{% endtrans %}</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("howto/index") }}">{% trans %}Python HOWTOs{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}in-depth documents on specific topics{% endtrans %}</span></p> + <span class="linkdescr">{% trans %}In-depth topic manuals{% endtrans %}</span></p> </td><td width="50%"> - <p class="biglink"><a class="biglink" href="{{ pathto("installing/index") }}">{% trans %}Installing Python Modules{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}installing from the Python Package Index & other sources{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("distributing/index") }}">{% trans %}Distributing Python Modules{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}publishing modules for installation by others{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("extending/index") }}">{% trans %}Extending and Embedding{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}tutorial for C/C++ programmers{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">{% trans %}Python/C API{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}reference for C/C++ programmers{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("installing/index") }}">{% trans %}Installing Python modules{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}Third-party modules and PyPI.org{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("distributing/index") }}">{% trans %}Distributing Python modules{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}Publishing modules for use by other people{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("extending/index") }}">{% trans %}Extending and embedding{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}For C/C++ programmers{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">{% trans %}Python's C API{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}C API reference{% endtrans %}</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("faq/index") }}">{% trans %}FAQs{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}frequently asked questions (with answers!){% endtrans %}</span></p> + <span class="linkdescr">{% trans %}Frequently asked questions (with answers!){% endtrans %}</span></p> </td></tr> </table> - <p><strong>{% trans %}Indices and tables:{% endtrans %}</strong></p> + <p><strong>{% trans %}Indices, glossary, and search:{% endtrans %}</strong></p> <table class="contentstable" align="center"><tr> <td width="50%"> - <p class="biglink"><a class="biglink" href="{{ pathto("py-modindex") }}">{% trans %}Global Module Index{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}quick access to all modules{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">{% trans %}General Index{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}all functions, classes, terms{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("py-modindex") }}">{% trans %}Global module index{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}All modules and libraries{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">{% trans %}General index{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}All functions, classes, and terms{% endtrans %}</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("glossary") }}">{% trans %}Glossary{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}the most important terms explained{% endtrans %}</span></p> + <span class="linkdescr">{% trans %}Terms explained{% endtrans %}</span></p> </td><td width="50%"> <p class="biglink"><a class="biglink" href="{{ pathto("search") }}">{% trans %}Search page{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}search this documentation{% endtrans %}</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("contents") }}">{% trans %}Complete Table of Contents{% endtrans %}</a><br/> - <span class="linkdescr">{% trans %}lists all sections and subsections{% endtrans %}</span></p> + <span class="linkdescr">{% trans %}Search this documentation{% endtrans %}</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("contents") }}">{% trans %}Complete table of contents{% endtrans %}</a><br/> + <span class="linkdescr">{% trans %}Lists all sections and subsections{% endtrans %}</span></p> </td></tr> </table> - <p><strong>{% trans %}Meta information:{% endtrans %}</strong></p> + <p><strong>{% trans %}Project information:{% endtrans %}</strong></p> <table class="contentstable" align="center"><tr> <td width="50%"> - <p class="biglink"><a class="biglink" href="{{ pathto("bugs") }}">{% trans %}Reporting bugs{% endtrans %}</a></p> + <p class="biglink"><a class="biglink" href="{{ pathto("bugs") }}">{% trans %}Reporting issues{% endtrans %}</a></p> <p class="biglink"><a class="biglink" href="https://devguide.python.org/docquality/#helping-with-documentation">{% trans %}Contributing to Docs{% endtrans %}</a></p> - <p class="biglink"><a class="biglink" href="{{ pathto("about") }}">{% trans %}About the documentation{% endtrans %}</a></p> + <p class="biglink"><a class="biglink" href="{{ pathto("download") }}">{% trans %}Download the documentation{% endtrans %}</a></p> </td><td width="50%"> - <p class="biglink"><a class="biglink" href="{{ pathto("license") }}">{% trans %}History and License of Python{% endtrans %}</a></p> + <p class="biglink"><a class="biglink" href="{{ pathto("license") }}">{% trans %}History and license of Python{% endtrans %}</a></p> <p class="biglink"><a class="biglink" href="{{ pathto("copyright") }}">{% trans %}Copyright{% endtrans %}</a></p> - <p class="biglink"><a class="biglink" href="{{ pathto("download") }}">{% trans %}Download the documentation{% endtrans %}</a></p> + <p class="biglink"><a class="biglink" href="{{ pathto("about") }}">{% trans %}About the documentation{% endtrans %}</a></p> </td></tr> </table> {% endblock %} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9498b2ccc5af92..b09fd21a8ddcc9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,6 +26,9 @@ {% endblock %} {% block extrahead %} + {% if builder == "html" and enable_analytics %} + <script defer data-domain="docs.python.org" src="https://plausible.io/js/script.js"></script> + {% endif %} <link rel="canonical" href="https://docs.python.org/3/{{pagename}}.html" /> {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} @@ -41,4 +44,9 @@ {{ "}" }} </style> {{ super() }} + +{%- if not embedded %} + <script type="text/javascript" src="{{ pathto('_static/rtd_switcher.js', 1) }}"></script> + <meta name="readthedocs-addons-api-version" content="1"> +{%- endif %} {% endblock %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 588591fcdb726f..b8faf756698097 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and +:mod:`readline` support, a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history without output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling @@ -20,7 +42,7 @@ In interactive mode, it then returns to the primary prompt; when input came from a file, it exits with a nonzero exit status after printing the stack trace. (Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement are not errors in this context.) Some errors are unconditionally fatal and -cause an exit with a nonzero exit; this applies to internal inconsistencies and +cause an exit with a nonzero exit status; this applies to internal inconsistencies and some cases of running out of memory. All error messages are written to the standard error stream; normal output from executed commands is written to standard output. @@ -40,7 +62,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #!/usr/bin/env python3.5 + #!/usr/bin/env python3 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the @@ -107,7 +129,7 @@ of your user site-packages directory. Start Python and run this code:: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' + '/home/user/.local/lib/python3.x/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 7b92e1a51b6e67..675faa8c52477d 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -338,11 +338,7 @@ code will print the value ``16``, without leaving a trace:: del x.counter The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) +function that "belongs to" an object. .. index:: pair: object; method @@ -386,12 +382,11 @@ general, calling a method with a list of *n* arguments is equivalent to calling the corresponding function with an argument list that is created by inserting the method's instance object before the first argument. -If you still don't understand how methods work, a look at the implementation can -perhaps clarify matters. When a non-data attribute of an instance is -referenced, the instance's class is searched. If the name denotes a valid class -attribute that is a function object, a method object is created by packing -(pointers to) the instance object and the function object just found together in -an abstract object: this is the method object. When the method object is called +In general, methods work as follows. When a non-data attribute +of an instance is referenced, the instance's class is searched. +If the name denotes a valid class attribute that is a function object, +references to both the instance object and the function object +are packed into a method object. When the method object is called with an argument list, a new argument list is constructed from the instance object and the argument list, and the function object is called with this new argument list. @@ -666,7 +661,7 @@ class, that calls each parent only once, and that is monotonic (meaning that a class can be subclassed without affecting the precedence order of its parents). Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. +:ref:`python_2.3_mro`. .. _tut-private: @@ -693,6 +688,11 @@ current class name with leading underscore(s) stripped. This mangling is done without regard to the syntactic position of the identifier, as long as it occurs within the definition of a class. +.. seealso:: + + The :ref:`private name mangling specifications <private-name-mangling>` + for details and special cases. + Name mangling is helpful for letting subclasses override methods without breaking intraclass method calls. For example:: @@ -769,8 +769,10 @@ data from a string buffer instead, and pass it as an argument. or arithmetic operators, and assigning such a "pseudo-file" to sys.stdin will not cause the interpreter to read further input from it.) -Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`!m`, and ``m.__func__`` is the function object +:ref:`Instance method objects <instance-methods>` have attributes, too: +:attr:`m.__self__ <method.__self__>` is the instance +object with the method :meth:`!m`, and :attr:`m.__func__ <method.__func__>` is +the :ref:`function object <user-defined-funcs>` corresponding to the method. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index aa9caa101da40a..77444f9cb8358d 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -559,10 +559,10 @@ defined to allow. For example:: def ask_ok(prompt, retries=4, reminder='Please try again!'): while True: - ok = input(prompt) - if ok in ('y', 'ye', 'yes'): + reply = input(prompt) + if reply in {'y', 'ye', 'yes'}: return True - if ok in ('n', 'no', 'nop', 'nope'): + if reply in {'n', 'no', 'nop', 'nope'}: return False retries = retries - 1 if retries < 0: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 87614d082a1d4e..a1492298bdb867 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -48,10 +48,9 @@ objects: :noindex: Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) + is specified, ``a.pop()`` removes and returns the last item in the list. + It raises an :exc:`IndexError` if the list is empty or the index is + outside the list range. .. method:: list.clear() @@ -127,7 +126,7 @@ Python. Another thing you might notice is that not all data can be sorted or compared. For instance, ``[None, 'hello', 10]`` doesn't sort because -integers can't be compared to strings and *None* can't be compared to +integers can't be compared to strings and ``None`` can't be compared to other types. Also, there are some types that don't have a defined ordering relation. For example, ``3+4j < 5+7j`` isn't a valid comparison. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 1ec59767e9ce12..981b14f5a4212b 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -20,12 +20,12 @@ complaint you get while you are still learning Python:: >>> while True print('Hello world') File "<stdin>", line 1 while True print('Hello world') - ^ + ^^^^^ SyntaxError: invalid syntax -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the +The parser repeats the offending line and displays little 'arrow's pointing +at the token in the line where the error was detected. The error may be +caused by the absence of a token *before* the indicated token. In the example, the error is detected at the function :func:`print`, since a colon (``':'``) is missing before it. File name and line number are printed so you know where to look in case the input came from a script. @@ -108,8 +108,7 @@ The :keyword:`try` statement works as follows. * If an exception occurs which does not match the exception named in the *except clause*, it is passed on to outer :keyword:`try` statements; if no handler is - found, it is an *unhandled exception* and execution stops with a message as - shown above. + found, it is an *unhandled exception* and execution stops with an error message. A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. @@ -120,9 +119,9 @@ may name multiple exceptions as a parenthesized tuple, for example:: ... except (RuntimeError, TypeError, NameError): ... pass -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -*except clause* listing a derived class is not compatible with a base class). +A class in an :keyword:`except` clause matches exceptions which are instances of the +class itself or one of its derived classes (but not the other way around --- an +*except clause* listing a derived class does not match instances of its base classes). For example, the following code will print B, C, D in that order:: class B(Exception): diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 30f3dfb6b238b4..0795e2fef98830 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -150,7 +150,7 @@ section. See `Examples of Floating Point Problems <https://jvns.ca/blog/2023/01/13/examples-of-floating-point-problems/>`_ for a pleasant summary of how binary floating-point works and the kinds of problems commonly encountered in practice. Also see -`The Perils of Floating Point <https://www.lahey.com/float.htm>`_ +`The Perils of Floating Point <http://www.indowsway.com/floatingpoint.htm>`_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fe9ca9ccb9c7e0..b93a0e8cec2d38 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -37,16 +37,23 @@ printing space-separated values. There are several ways to format output. * The :meth:`str.format` method of strings requires more manual effort. You'll still use ``{`` and ``}`` to mark where a variable will be substituted and can provide detailed formatting directives, - but you'll also need to provide the information to be formatted. + but you'll also need to provide the information to be formatted. In the following code + block there are two examples of how to format variables: + :: >>> yes_votes = 42_572_654 - >>> no_votes = 43_132_495 - >>> percentage = yes_votes / (yes_votes + no_votes) + >>> total_votes = 85_705_149 + >>> percentage = yes_votes / total_votes >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) ' 42572654 YES votes 49.67%' + Notice how the ``yes_votes`` are padded with spaces and a negative sign only for negative numbers. + The example also prints ``percentage`` multiplied by 100, with 2 decimal + places and followed by a percent sign (see :ref:`formatspec` for details). + + * Finally, you can do all the string handling yourself by using string slicing and concatenation operations to create any layout you can imagine. The string type has some methods that perform useful operations for padding @@ -197,7 +204,12 @@ notation. :: Jack: 4098; Sjoerd: 4127; Dcab: 8637678 This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. +:func:`vars`, which returns a dictionary containing all local variables:: + + >>> table = {k: str(v) for k, v in vars().items()} + >>> message = " ".join([f'{k}: ' + '{' + k +'};' for k in table.keys()]) + >>> print(message.format(**table)) + __name__: __main__; __doc__: None; __package__: None; __loader__: ... As an example, the following lines produce a tidily aligned set of columns giving integers and their squares and cubes:: @@ -267,9 +279,11 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The % operator (modulo) can also be used for string formatting. Given ``'string' -% values``, instances of ``%`` in ``string`` are replaced with zero or more -elements of ``values``. This operation is commonly known as string +The % operator (modulo) can also be used for string formatting. +Given ``format % values`` (where *format* is a string), +``%`` conversion specifications in *format* are replaced with +zero or more elements of *values*. +This operation is commonly known as string interpolation. For example:: >>> import math diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst index 0d3896a4832b59..4e054c4e6c2c32 100644 --- a/Doc/tutorial/interactive.rst +++ b/Doc/tutorial/interactive.rst @@ -51,4 +51,4 @@ bpython_. .. _GNU Readline: https://tiswww.case.edu/php/chet/readline/rltop.html .. _IPython: https://ipython.org/ -.. _bpython: https://www.bpython-interpreter.org/ +.. _bpython: https://bpython-interpreter.org/ diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 42ebf2b3d294a8..02e7de77322e99 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` +The Python interpreter is usually installed as |usr_local_bin_python_x_dot_y_literal| on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.13 + python3.14 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -<windows-store>`, the :file:`python3.13` command will be available. If you have +<windows-store>`, the |python_x_dot_y_literal| command will be available. If you have the :ref:`py.exe launcher <launcher>` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.13 - Python 3.13 (default, April 4 2023, 09:25:04) + $ python3.14 + Python 3.14 (default, April 4 2024, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 172611dd0cfb41..0f16dae8b1418f 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -405,13 +405,6 @@ indexed and sliced:: >>> squares[-3:] # slicing returns a new list [9, 16, 25] -All slice operations return a new list containing the requested elements. This -means that the following slice returns a -:ref:`shallow copy <shallow_vs_deep_copy>` of the list:: - - >>> squares[:] - [1, 4, 9, 16, 25] - Lists also support operations like concatenation:: >>> squares + [36, 49, 64, 81, 100] @@ -428,13 +421,37 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`~list.append` *method* (we will see more about methods later):: +the :meth:`!list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 >>> cubes [1, 8, 27, 64, 125, 216, 343] +Simple assignment in Python never copies data. When you assign a list +to a variable, the variable refers to the *existing list*. +Any changes you make to the list through one variable will be seen +through all other variables that refer to it.:: + + >>> rgb = ["Red", "Green", "Blue"] + >>> rgba = rgb + >>> id(rgb) == id(rgba) # they reference the same object + True + >>> rgba.append("Alph") + >>> rgb + ["Red", "Green", "Blue", "Alph"] + +All slice operations return a new list containing the requested elements. This +means that the following slice returns a +:ref:`shallow copy <shallow_vs_deep_copy>` of the list:: + + >>> correct_rgba = rgba[:] + >>> correct_rgba[-1] = "Alpha" + >>> correct_rgba + ["Red", "Green", "Blue", "Alpha"] + >>> rgba + ["Red", "Green", "Blue", "Alph"] + Assignment to slices is also possible, and this can even change the size of the list or clear it entirely:: diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index bf9e8e0b7b8066..0316239e776a95 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -437,7 +437,8 @@ When importing the package, Python searches through the directories on ``sys.path`` looking for the package subdirectory. The :file:`__init__.py` files are required to make Python treat directories -containing the file as packages. This prevents directories with a common name, +containing the file as packages (unless using a :term:`namespace package`, a +relatively advanced feature). This prevents directories with a common name, such as ``string``, from unintentionally hiding valid modules that occur later on the module search path. In the simplest case, :file:`__init__.py` can just be an empty file, but it can also execute initialization code for the package or diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 6bae279c5e9cde..86c94429339dfc 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python312' + 'C:\\Python314' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -153,7 +153,7 @@ The :mod:`random` module provides tools for making random selections:: 'apple' >>> random.sample(range(100), 10) # sampling without replacement [30, 83, 16, 4, 8, 81, 41, 50, 18, 33] - >>> random.random() # random float + >>> random.random() # random float from the interval [0.0, 1.0) 0.17970987693706186 >>> random.randrange(6) # random integer chosen from range(6) 4 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 33f311db3a24d2..719f772e687008 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "<stdin>", line 1, in <module> d['primary'] # entry was automatically removed - File "C:/python312/lib/weakref.py", line 46, in __getitem__ + File "C:/python314/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' @@ -293,7 +293,7 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`~array.array()` object that is like +The :mod:`array` module provides an :class:`~array.array` object that is like a list that stores only homogeneous data and stores it more compactly. The following example shows an array of numbers stored as two byte unsigned binary numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular @@ -306,7 +306,7 @@ lists of Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`~collections.deque()` object +The :mod:`collections` module provides a :class:`~collections.deque` object that is like a list with faster appends and pops from the left side but slower lookups in the middle. These objects are well suited for implementing queues and breadth first tree searches:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index a6dead2eac11f6..91e4ce18acef1d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -36,10 +36,10 @@ Creating Virtual Environments ============================= The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. +:mod:`venv`. :mod:`venv` will install the Python version from which +the command was run (as reported by the :option:`--version` option). +For instance, executing the command with ``python3.12`` will install +version 3.12. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b133d2c5893797..a575760c963327 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -242,12 +243,13 @@ Miscellaneous options .. option:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the - option is given twice (:option:`!-bb`). + Issue a warning when converting :class:`bytes` or :class:`bytearray` to + :class:`str` without specifying encoding or comparing :class:`!bytes` or + :class:`!bytearray` with :class:`!str` or :class:`!bytes` with :class:`int`. + Issue an error when the option is given twice (:option:`!-bb`). .. versionchanged:: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. + Affects also comparisons of :class:`bytes` with :class:`int`. .. option:: -B @@ -288,9 +290,15 @@ Miscellaneous options .. option:: -i - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. The + Enter interactive mode after execution. + + Using the :option:`-i` option will enter interactive mode in any of the following circumstances\: + + * When a script is passed as first argument + * When the :option:`-c` option is used + * When the :option:`-m` option is used + + Interactive mode will start even when :data:`sys.stdin` does not appear to be a terminal. The :envvar:`PYTHONSTARTUP` file is not read. This can be useful to inspect global variables or a stack trace when a script @@ -369,23 +377,25 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst - case performance of a dict construction, O(n\ :sup:`2`) complexity. See + case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See http://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. + .. versionadded:: 3.2.3 + .. versionchanged:: 3.7 The option is no longer ignored. - .. versionadded:: 3.2.3 - .. option:: -s Don't add the :data:`user site-packages directory <site.USER_SITE>` to :data:`sys.path`. + See also :envvar:`PYTHONNOUSERSITE`. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -437,6 +447,7 @@ Miscellaneous options -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -497,55 +508,101 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`. See also :envvar:`PYTHONFAULTHANDLER`. + + .. versionadded:: 3.3 + * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the interactive interpreter. This only works on :ref:`debug builds <debug-build>`. + + .. versionadded:: 3.4 + * ``-X tracemalloc`` to start tracing Python memory allocations using the :mod:`tracemalloc` module. By default, only the most recent frame is stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start tracing with a traceback limit of *NFRAME* frames. See :func:`tracemalloc.start` and :envvar:`PYTHONTRACEMALLOC` for more information. + + .. versionadded:: 3.4 + * ``-X int_max_str_digits`` configures the :ref:`integer string conversion length limitation <int_max_str_digits>`. See also :envvar:`PYTHONINTMAXSTRDIGITS`. + + .. versionadded:: 3.11 + * ``-X importtime`` to show how long each import takes. It shows module name, cumulative time (including nested imports) and self time (excluding nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. + + .. versionadded:: 3.7 + * ``-X dev``: enable :ref:`Python Development Mode <devmode>`, introducing additional runtime checks that are too expensive to be enabled by - default. + default. See also :envvar:`PYTHONDEVMODE`. + + .. versionadded:: 3.7 + * ``-X utf8`` enables the :ref:`Python UTF-8 Mode <utf8-mode>`. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode <utf8-mode>` (even when it would otherwise activate automatically). See also :envvar:`PYTHONUTF8`. + + .. versionadded:: 3.7 + * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. + + .. versionadded:: 3.8 + * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when the locale-specific default encoding is used for opening files. See also :envvar:`PYTHONWARNDEFAULTENCODING`. + + .. versionadded:: 3.10 + * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller code objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. + + .. versionadded:: 3.11 + * ``-X frozen_modules`` determines whether or not frozen modules are - ignored by the import machinery. A value of "on" means they get - imported and "off" means they are ignored. The default is "on" + ignored by the import machinery. A value of ``on`` means they get + imported and ``off`` means they are ignored. The default is ``on`` if this is an installed Python (the normal case). If it's under - development (running from the source tree) then the default is "off". - Note that the "importlib_bootstrap" and "importlib_bootstrap_external" - frozen modules are always used, even if this flag is set to "off". + development (running from the source tree) then the default is ``off``. + Note that the :mod:`!importlib_bootstrap` and + :mod:`!importlib_bootstrap_external` frozen modules are always used, even + if this flag is set to ``off``. See also :envvar:`PYTHON_FROZEN_MODULES`. + + .. versionadded:: 3.11 + * ``-X perf`` enables support for the Linux ``perf`` profiler. When this option is provided, the ``perf`` profiler will be able to report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + + .. versionadded:: 3.12 + + * ``-X perf_jit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF information. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHON_PERF_JIT_SUPPORT` + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. @@ -553,55 +610,66 @@ Miscellaneous options container system. See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, nothing is overridden. - It also allows passing arbitrary values and retrieving them through the - :data:`sys._xoptions` dictionary. + .. versionadded:: 3.13 - .. versionchanged:: 3.2 - The :option:`-X` option was added. + * :samp:`-X presite={package.module}` specifies a module that should be + imported before the :mod:`site` module is executed and before the + :mod:`__main__` module exists. Therefore, the imported module isn't + :mod:`__main__`. This can be used to execute code early during Python + initialization. Python needs to be :ref:`built in debug mode <debug-build>` + for this option to exist. See also :envvar:`PYTHON_PRESITE`. - .. versionadded:: 3.3 - The ``-X faulthandler`` option. + .. versionadded:: 3.13 - .. versionadded:: 3.4 - The ``-X showrefcount`` and ``-X tracemalloc`` options. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL` and + :ref:`free-threaded-cpython`. - .. versionadded:: 3.6 - The ``-X showalloccount`` option. + .. versionadded:: 3.13 - .. versionadded:: 3.7 - The ``-X importtime``, ``-X dev`` and ``-X utf8`` options. + It also allows passing arbitrary values and retrieving them through the + :data:`sys._xoptions` dictionary. - .. versionadded:: 3.8 - The ``-X pycache_prefix`` option. The ``-X dev`` option now logs - ``close()`` exceptions in :class:`io.IOBase` destructor. + .. versionadded:: 3.2 .. versionchanged:: 3.9 - Using ``-X dev`` option, check *encoding* and *errors* arguments on - string encoding and decoding operations. + Removed the ``-X showalloccount`` option. - The ``-X showalloccount`` option has been removed. + .. versionchanged:: 3.10 + Removed the ``-X oldparser`` option. - .. versionadded:: 3.10 - The ``-X warn_default_encoding`` option. +.. _using-on-controlling-color: - .. deprecated-removed:: 3.9 3.10 - The ``-X oldparser`` option. +Controlling color +~~~~~~~~~~~~~~~~~ - .. versionadded:: 3.11 - The ``-X no_debug_ranges`` option. +The Python interpreter is configured by default to use colors to highlight +output in certain situations such as when displaying tracebacks. This +behavior can be controlled by setting different environment variables. - .. versionadded:: 3.11 - The ``-X frozen_modules`` option. +Setting the environment variable ``TERM`` to ``dumb`` will disable color. - .. versionadded:: 3.11 - The ``-X int_max_str_digits`` option. +If the |FORCE_COLOR|_ environment variable is set, then color will be +enabled regardless of the value of TERM. This is useful on CI systems which +aren’t terminals but can still display ANSI escape sequences. - .. versionadded:: 3.12 - The ``-X perf`` option. +If the |NO_COLOR|_ environment variable is set, Python will disable all color +in the output. This takes precedence over ``FORCE_COLOR``. - .. versionadded:: 3.13 - The ``-X cpu_count`` option. +All these environment variables are used also by other tools to control color +output. To control the color output only in the Python interpreter, the +:envvar:`PYTHON_COLORS` environment variable can be used. This variable takes +precedence over ``NO_COLOR``, which in turn takes precedence over +``FORCE_COLOR``. + +.. Apparently this how you hack together a formatted link: +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -848,6 +916,7 @@ conflict. PYTHONWARNINGS=default # Warn once per call location PYTHONWARNINGS=error # Convert to exceptions PYTHONWARNINGS=always # Warn every time + PYTHONWARNINGS=all # Same as PYTHONWARNINGS=always PYTHONWARNINGS=module # Warn once per calling module PYTHONWARNINGS=once # Warn once per Python process PYTHONWARNINGS=ignore # Never warn @@ -912,6 +981,9 @@ conflict. * ``pymalloc``: use the :ref:`pymalloc allocator <pymalloc>` for :c:macro:`PYMEM_DOMAIN_MEM` and :c:macro:`PYMEM_DOMAIN_OBJ` domains and use the :c:func:`malloc` function for the :c:macro:`PYMEM_DOMAIN_RAW` domain. + * ``mimalloc``: use the :ref:`mimalloc allocator <mimalloc>` for + :c:macro:`PYMEM_DOMAIN_MEM` and :c:macro:`PYMEM_DOMAIN_OBJ` domains and use + the :c:func:`malloc` function for the :c:macro:`PYMEM_DOMAIN_RAW` domain. Install :ref:`debug hooks <pymem-debug-hooks>`: @@ -919,12 +991,13 @@ conflict. allocators <default-memory-allocators>`. * ``malloc_debug``: same as ``malloc`` but also install debug hooks. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. + * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + + .. versionadded:: 3.6 .. versionchanged:: 3.7 Added the ``"default"`` allocator. - .. versionadded:: 3.6 - .. envvar:: PYTHONMALLOCSTATS @@ -1073,6 +1146,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHON_PERF_JIT_SUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perf_jit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return @@ -1082,6 +1170,56 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_FROZEN_MODULES + + If this variable is set to ``on`` or ``off``, it determines whether or not + frozen modules are ignored by the import machinery. A value of ``on`` means + they get imported and ``off`` means they are ignored. The default is ``on`` + for non-debug builds (the normal case) and ``off`` for debug builds. + Note that the :mod:`!importlib_bootstrap` and + :mod:`!importlib_bootstrap_external` frozen modules are always used, even + if this flag is set to ``off``. + + See also the :option:`-X frozen_modules <-X>` command-line option. + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_COLORS + + If this variable is set to ``1``, the interpreter will colorize various kinds + of output. Setting it to ``0`` deactivates this behavior. + See also :ref:`using-on-controlling-color`. + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to ``1``, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_HISTORY + + This environment variable can be used to set the location of a + ``.python_history`` file (by default, it is ``.python_history`` in the + user's home directory). + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off. + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable, and :ref:`free-threaded-cpython`. + + Needs Python configured with the :option:`--disable-gil` build option. + + .. versionadded:: 3.13 Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -1091,13 +1229,33 @@ Debug-mode variables If set, Python will dump objects and reference counts still alive after shutting down the interpreter. - Need Python configured with the :option:`--with-trace-refs` build option. + Needs Python configured with the :option:`--with-trace-refs` build option. -.. envvar:: PYTHONDUMPREFSFILE=FILENAME +.. envvar:: PYTHONDUMPREFSFILE If set, Python will dump objects and reference counts still alive - after shutting down the interpreter into a file called *FILENAME*. + after shutting down the interpreter into a file under the path given + as the value to this environment variable. - Need Python configured with the :option:`--with-trace-refs` build option. + Needs Python configured with the :option:`--with-trace-refs` build option. .. versionadded:: 3.11 + +.. envvar:: PYTHON_PRESITE + + If this variable is set to a module, that module will be imported + early in the interpreter lifecycle, before the :mod:`site` module is + executed, and before the :mod:`__main__` module is created. + Therefore, the imported module is not treated as :mod:`__main__`. + + This can be used to execute code early during Python initialization. + + To import a submodule, use ``package.module`` as the value, like in + an import statement. + + See also the :option:`-X presite <-X>` command-line option, + which takes precedence over this variable. + + Needs Python configured with the :option:`--with-pydebug` build option. + + .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 5f9e695d10ad44..2c73c224e4e8a1 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -2,6 +2,8 @@ Configure Python **************** +.. highlight:: sh + Build Requirements ================== @@ -30,31 +32,31 @@ Features and minimum versions required to build CPython: * Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the :file:`configure` script. -.. versionchanged:: 3.13: - Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required. -.. versionchanged:: 3.11 - C11 compiler, IEEE 754 and NaN support are now required. - On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is now required. + Tcl/Tk version 8.4 is now required. -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. +.. versionchanged:: 3.6 + Selected C99 features are now required, like ``<stdint.h>`` and ``static + inline`` functions. .. versionchanged:: 3.7 Thread support and OpenSSL 1.0.2 are now required. -.. versionchanged:: 3.6 - Selected C99 features are now required, like ``<stdint.h>`` and ``static - inline`` functions. +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required. + Require SQLite 3.7.15. -.. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. + Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. +.. versionchanged:: 3.13 + Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -74,14 +76,21 @@ files. Commands to regenerate all generated files:: The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used to regenerate them. Search for ``regen-*`` make targets. -The ``make regen-configure`` command runs `tiran/cpython_autoconf -<https://github.com/tiran/cpython_autoconf>`_ container for reproducible build; -see container ``entry.sh`` script. The container is optional, the following -command can be run locally, the generated files depend on autoconf and aclocal -versions:: +configure script +---------------- + +The ``make regen-configure`` command regenerates the ``aclocal.m4`` file and +the ``configure`` script using the ``Tools/build/regen-configure.sh`` shell +script which uses an Ubuntu container to get the same tools versions and have a +reproducible output. + +The container is optional, the following command can be run locally:: autoreconf -ivf -Werror +The generated files can change depending on the exact ``autoconf-archive``, +``aclocal`` and ``pkg-config`` versions. + .. _configure-options: @@ -268,7 +277,7 @@ General Options * to/from free lists; * dictionary materialized/dematerialized; * type cache; - * optimization attemps; + * optimization attempts; * optimization traces created/executed; * uops executed. @@ -280,12 +289,17 @@ General Options .. versionadded:: 3.11 +.. _free-threading-build: + .. option:: --disable-gil Enables **experimental** support for running Python without the - :term:`global interpreter lock` (GIL). + :term:`global interpreter lock` (GIL): free threading build. + + Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to + :data:`sys.abiflags`. - See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + See :ref:`free-threaded-cpython` for more detail. .. versionadded:: 3.13 @@ -375,6 +389,17 @@ Options for third-party dependencies C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, overriding ``pkg-config``. +.. option:: LIBMPDEC_CFLAGS +.. option:: LIBMPDEC_LIBS + + C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, + overriding ``pkg-config``. + + .. note:: + + These environment variables have no effect unless + :option:`--with-system-libmpdec` is specified. + .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -402,7 +427,7 @@ Options for third-party dependencies .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS - C compiler and Linker flags for PANEL, overriding ``pkg-config``. + C compiler and linker flags for PANEL, overriding ``pkg-config``. C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by :mod:`curses.panel` module, overriding ``pkg-config``. @@ -504,6 +529,15 @@ also be used to improve performance. GCC is used: add ``-fno-semantic-interposition`` to the compiler and linker flags. + .. note:: + + During the build, you may encounter compiler warnings about + profile data not being available for some source files. + These warnings are harmless, as only a subset of the code is exercised + during profile data acquisition. + To disable these warnings on Clang, manually suppress them by adding + ``-Wno-profile-instr-unprofiled`` to :envvar:`CFLAGS`. + .. versionadded:: 3.6 .. versionchanged:: 3.10 @@ -579,6 +613,13 @@ also be used to improve performance. Enable computed gotos in evaluation loop (enabled by default on supported compilers). +.. option:: --without-mimalloc + + Disable the fast :ref:`mimalloc <mimalloc>` allocator + (enabled by default). + + See also :envvar:`PYTHONMALLOC` environment variable. + .. option:: --without-pymalloc Disable the specialized Python memory allocator :ref:`pymalloc <pymalloc>` @@ -675,12 +716,12 @@ Debug options :ref:`Statically allocated objects <static-types>` are not traced. + .. versionadded:: 3.8 + .. versionchanged:: 3.13 This build is now ABI compatible with release build and :ref:`debug build <debug-build>`. - .. versionadded:: 3.8 - .. option:: --with-assertions Build with C assertions enabled (default is no): ``assert(...);`` and @@ -726,6 +767,13 @@ Debug options .. versionadded:: 3.6 +.. option:: --with-thread-sanitizer + + Enable ThreadSanitizer data race detector, ``tsan`` + (default is no). + + .. versionadded:: 3.13 + Linker options -------------- @@ -756,11 +804,20 @@ Libraries options .. option:: --with-system-libmpdec - Build the ``_decimal`` extension module using an installed ``mpdec`` - library, see the :mod:`decimal` module (default is no). + Build the ``_decimal`` extension module using an installed ``mpdecimal`` + library, see the :mod:`decimal` module (default is yes). .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Default to using the installed ``mpdecimal`` library. + + .. deprecated-removed:: 3.13 3.15 + A copy of the ``mpdecimal`` library sources will no longer be distributed + with Python 3.15. + + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. + .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -853,7 +910,7 @@ Security Options macOS Options ------------- -See ``Mac/README.rst``. +See :source:`Mac/README.rst`. .. option:: --enable-universalsdk .. option:: --enable-universalsdk=SDKDIR @@ -888,6 +945,31 @@ See ``Mac/README.rst``. Specify the name for the python framework on macOS only valid when :option:`--enable-framework` is set (default: ``Python``). +.. option:: --with-app-store-compliance +.. option:: --with-app-store-compliance=PATCH-FILE + + The Python standard library contains strings that are known to trigger + automated inspection tool errors when submitted for distribution by + the macOS and iOS App Stores. If enabled, this option will apply the list of + patches that are known to correct app store compliance. A custom patch + file can also be specified. This option is disabled by default. + + .. versionadded:: 3.13 + +iOS Options +----------- + +See :source:`iOS/README.rst`. + +.. option:: --enable-framework=INSTALLDIR + + Create a Python.framework. Unlike macOS, the *INSTALLDIR* argument + specifying the installation path is mandatory. + +.. option:: --with-framework-name=FRAMEWORK + + Specify the name for the framework (default: ``Python``). + Cross Compiling Options ----------------------- @@ -915,7 +997,9 @@ the version of the cross compiled host Python. An environment variable that points to a file with configure overrides. - Example *config.site* file:: + Example *config.site* file: + + .. code-block:: ini # config.site-aarch64 ac_cv_buggy_getaddrinfo=no @@ -961,39 +1045,108 @@ Main build steps Main Makefile targets --------------------- -* ``make``: Build Python with the standard library. -* ``make platform:``: build the ``python`` program, but don't build the - standard library extension modules. -* ``make profile-opt``: build Python using Profile Guided Optimization (PGO). - You can use the configure :option:`--enable-optimizations` option to make - this the default target of the ``make`` command (``make all`` or just - ``make``). - -* ``make test``: Build Python and run the Python test suite with ``--fast-ci`` - option. Variables: - - * ``TESTOPTS``: additional regrtest command line options. - * ``TESTPYTHONOPTS``: additional Python command line options. - * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). - -* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` - option and default timeout of 20 minutes, instead of ``--fast-ci`` option - and a default timeout of 10 minutes. - -* ``make install``: Build and install Python. -* ``make regen-all``: Regenerate (almost) all generated files; - ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately - for the remaining generated files. -* ``make clean``: Remove built files. -* ``make distclean``: Same than ``make clean``, but remove also files created - by the configure script. +make +^^^^ + +For the most part, when rebuilding after editing some code or +refreshing your checkout from upstream, all you need to do is execute +``make``, which (per Make's semantics) builds the default target, the +first one defined in the Makefile. By tradition (including in the +CPython project) this is usually the ``all`` target. The +``configure`` script expands an ``autoconf`` variable, +``@DEF_MAKE_ALL_RULE@`` to describe precisely which targets ``make +all`` will build. The three choices are: + +* ``profile-opt`` (configured with ``--enable-optimizations``) +* ``build_wasm`` (configured with ``--with-emscripten-target``) +* ``build_all`` (configured without explicitly using either of the others) + +Depending on the most recent source file changes, Make will rebuild +any targets (object files and executables) deemed out-of-date, +including running ``configure`` again if necessary. Source/target +dependencies are many and maintained manually however, so Make +sometimes doesn't have all the information necessary to correctly +detect all targets which need to be rebuilt. Depending on which +targets aren't rebuilt, you might experience a number of problems. If +you have build or test problems which you can't otherwise explain, +``make clean && make`` should work around most dependency problems, at +the expense of longer build times. + + +make platform +^^^^^^^^^^^^^ + +Build the ``python`` program, but don't build the standard library +extension modules. This generates a file named ``platform`` which +contains a single line describing the details of the build platform, +e.g., ``macosx-14.3-arm64-3.12`` or ``linux-x86_64-3.13``. + + +make profile-opt +^^^^^^^^^^^^^^^^ + +Build Python using profile-guided optimization (PGO). You can use the +configure :option:`--enable-optimizations` option to make this the +default target of the ``make`` command (``make all`` or just +``make``). + + + +make clean +^^^^^^^^^^ + +Remove built files. + + +make distclean +^^^^^^^^^^^^^^ + +In addition to the the work done by ``make clean``, remove files +created by the configure script. ``configure`` will have to be run +before building again. [#]_ + + +make install +^^^^^^^^^^^^ + +Build the ``all`` target and install Python. + + +make test +^^^^^^^^^ + +Build the ``all`` target and run the Python test suite with the +``--fast-ci`` option. Variables: + +* ``TESTOPTS``: additional regrtest command-line options. +* ``TESTPYTHONOPTS``: additional Python command-line options. +* ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). + + +make buildbottest +^^^^^^^^^^^^^^^^^ + +This is similar to ``make test``, but uses the ``--slow-ci`` +option and default timeout of 20 minutes, instead of ``--fast-ci`` option. + + +make regen-all +^^^^^^^^^^^^^^ + +Regenerate (almost) all generated files. These include (but are not +limited to) bytecode cases, and parser generator file. +``make regen-stdlib-module-names`` and ``autoconf`` must be run +separately for the remaining `generated files <#generated-files>`_. + C extensions ------------ Some C extensions are built as built-in modules, like the ``sys`` module. They are built with the ``Py_BUILD_CORE_BUILTIN`` macro defined. -Built-in modules have no ``__file__`` attribute:: +Built-in modules have no ``__file__`` attribute: + +.. code-block:: pycon >>> import sys >>> sys @@ -1005,7 +1158,9 @@ Built-in modules have no ``__file__`` attribute:: Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. -Example on Linux x86-64:: +Example on Linux x86-64: + +.. code-block:: pycon >>> import _asyncio >>> _asyncio @@ -1277,3 +1432,14 @@ Linker flags Linker flags used for building the interpreter object files. .. versionadded:: 3.8 + + +.. rubric:: Footnotes + +.. [#] ``git clean -fdx`` is an even more extreme way to "clean" your + checkout. It removes all files not known to Git. + When bug hunting using ``git bisect``, this is + `recommended between probes <https://github.com/python/cpython/issues/114505#issuecomment-1907021718>`_ + to guarantee a completely clean build. **Use with care**, as it + will delete all files not checked into Git, including your + new, uncommitted work. diff --git a/Doc/using/index.rst b/Doc/using/index.rst index e1a3111f36a44f..f55a12f1ab8a0d 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -18,4 +18,5 @@ interpreter and things that make working with Python easier. configure.rst windows.rst mac.rst + ios.rst editors.rst diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst new file mode 100644 index 00000000000000..774856e8aec2ac --- /dev/null +++ b/Doc/using/ios.rst @@ -0,0 +1,332 @@ +.. _using-ios: + +=================== +Using Python on iOS +=================== + +:Authors: + Russell Keith-Magee (2024-03) + +Python on iOS is unlike Python on desktop platforms. On a desktop platform, +Python is generally installed as a system resource that can be used by any user +of that computer. Users then interact with Python by running a :program:`python` +executable and entering commands at an interactive prompt, or by running a +Python script. + +On iOS, there is no concept of installing as a system resource. The only unit +of software distribution is an "app". There is also no console where you could +run a :program:`python` executable, or interact with a Python REPL. + +As a result, the only way you can use Python on iOS is in embedded mode - that +is, by writing a native iOS application, and embedding a Python interpreter +using ``libPython``, and invoking Python code using the :ref:`Python embedding +API <embedding>`. The full Python interpreter, the standard library, and all +your Python code is then packaged as a standalone bundle that can be +distributed via the iOS App Store. + +If you're looking to experiment for the first time with writing an iOS app in +Python, projects such as `BeeWare <https://beeware.org>`__ and `Kivy +<https://kivy.org>`__ will provide a much more approachable user experience. +These projects manage the complexities associated with getting an iOS project +running, so you only need to deal with the Python code itself. + +Python at runtime on iOS +======================== + +Platform identification +----------------------- + +When executing on iOS, ``sys.platform`` will report as ``ios``. This value will +be returned on an iPhone or iPad, regardless of whether the app is running on +the simulator or a physical device. + +Information about the specific runtime environment, including the iOS version, +device model, and whether the device is a simulator, can be obtained using +:func:`platform.ios_ver()`. :func:`platform.system()` will report ``iOS`` or +``iPadOS``, depending on the device. + +:func:`os.uname()` reports kernel-level details; it will report a name of +``Darwin``. + +Standard library availability +----------------------------- + +The Python standard library has some notable omissions and restrictions on +iOS. See the :ref:`API availability guide for iOS <iOS-availability>` for +details. + +Binary extension modules +------------------------ + +One notable difference about iOS as a platform is that App Store distribution +imposes hard requirements on the packaging of an application. One of these +requirements governs how binary extension modules are distributed. + +The iOS App Store requires that *all* binary modules in an iOS app must be +dynamic libraries, contained in a framework with appropriate metadata, stored +in the ``Frameworks`` folder of the packaged app. There can be only a single +binary per framework, and there can be no executable binary material outside +the ``Frameworks`` folder. + +This conflicts with the usual Python approach for distributing binaries, which +allows a binary extension module to be loaded from any location on +``sys.path``. To ensure compliance with App Store policies, an iOS project must +post-process any Python packages, converting ``.so`` binary modules into +individual standalone frameworks with appropriate metadata and signing. For +details on how to perform this post-processing, see the guide for :ref:`adding +Python to your project <adding-ios>`. + +To help Python discover binaries in their new location, the original ``.so`` +file on ``sys.path`` is replaced with a ``.fwork`` file. This file is a text +file containing the location of the framework binary, relative to the app +bundle. To allow the framework to resolve back to the original location, the +framework must contain a ``.origin`` file that contains the location of the +``.fwork`` file, relative to the app bundle. + +For example, consider the case of an import ``from foo.bar import _whiz``, +where ``_whiz`` is implemented with the binary module +``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location +registered on ``sys.path``, relative to the application bundle. This module +*must* be distributed as ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` +(creating the framework name from the full import path of the module), with an +``Info.plist`` file in the ``.framework`` directory identifying the binary as a +framework. The ``foo.bar._whiz`` module would be represented in the original +location with a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing +the path ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also +contain ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing +the path to the ``.fwork`` file. + +When running on iOS, the Python interpreter will install an +:class:`~importlib.machinery.AppleFrameworkLoader` that is able to read and +import ``.fwork`` files. Once imported, the ``__file__`` attribute of the +binary module will report as the location of the ``.fwork`` file. However, the +:class:`~importlib.machinery.ModuleSpec` for the loaded module will report the +``origin`` as the location of the binary in the framework folder. + +Compiler stub binaries +---------------------- + +Xcode doesn't expose explicit compilers for iOS; instead, it uses an ``xcrun`` +script that resolves to a full compiler path (e.g., ``xcrun --sdk iphoneos +clang`` to get the ``clang`` for an iPhone device). However, using this script +poses two problems: + +* The output of ``xcrun`` includes paths that are machine specific, resulting + in a sysconfig module that cannot be shared between users; and + +* It results in ``CC``/``CPP``/``LD``/``AR`` definitions that include spaces. + There is a lot of C ecosystem tooling that assumes that you can split a + command line at the first space to get the path to the compiler executable; + this isn't the case when using ``xcrun``. + +To avoid these problems, Python provided stubs for these tools. These stubs are +shell script wrappers around the underingly ``xcrun`` tools, distributed in a +``bin`` folder distributed alongside the compiled iOS framework. These scripts +are relocatable, and will always resolve to the appropriate local system paths. +By including these scripts in the bin folder that accompanies a framework, the +contents of the ``sysconfig`` module becomes useful for end-users to compile +their own modules. When compiling third-party Python modules for iOS, you +should ensure these stub binaries are on your path. + +Installing Python on iOS +======================== + +Tools for building iOS apps +--------------------------- + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting from the Platforms tab of the Xcode Settings panel. + +.. _adding-ios: + +Adding Python to an iOS project +------------------------------- + +Python can be added to any iOS project, using either Swift or Objective C. The +following examples will use Objective C; if you are using Swift, you may find a +library like `PythonKit <https://github.com/pvieito/PythonKit>`__ to be +helpful. + +To add Python to an iOS Xcode project: + +1. Build or obtain a Python ``XCFramework``. See the instructions in + :source:`iOS/README.rst` (in the CPython source distribution) for details on + how to build a Python ``XCFramework``. At a minimum, you will need a build + that supports ``arm64-apple-ios``, plus one of either + ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. + +2. Drag the ``XCframework`` into your iOS project. In the following + instructions, we'll assume you've dropped the ``XCframework`` into the root + of your project; however, you can use any other location that you want by + adjusting paths as needed. + +3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, + and ensure it is associated with the app target. + +4. Add your application code as a folder in your Xcode project. In the + following instructions, we'll assume that your user code is in a folder + named ``app`` in the root of your project; you can use any other location by + adjusting paths as needed. Ensure that this folder is associated with your + app target. + +5. Select the app target by selecting the root node of your Xcode project, then + the target name in the sidebar that appears. + +6. In the "General" settings, under "Frameworks, Libraries and Embedded + Content", add ``Python.xcframework``, with "Embed & Sign" selected. + +7. In the "Build Settings" tab, modify the following: + + - Build Options + + * User Script Sandboxing: No + * Enable Testability: Yes + + - Search Paths + + * Framework Search Paths: ``$(PROJECT_DIR)`` + * Header Search Paths: ``"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers"`` + + - Apple Clang - Warnings - All languages + + * Quoted Include In Framework Header: No + +8. Add a build step that copies the Python standard library into your app. In + the "Build Phases" tab, add a new "Run Script" build step *before* the + "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name + the step "Install Target Specific Python Standard Library", disable the + "Based on dependency analysis" checkbox, and set the script content to: + + .. code-block:: bash + + set -e + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + echo "Installing Python modules for iOS Device" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + fi + + Note that the name of the simulator "slice" in the XCframework may be + different, depending the CPU architectures your ``XCFramework`` supports. + +9. Add a second build step that processes the binary extension modules in the + standard library into "Framework" format. Add a "Run Script" build step + *directly after* the one you added in step 8, named "Prepare Python Binary + Modules". It should also have "Based on dependency analysis" unchecked, with + the following script content: + + .. code-block:: bash + + set -e + + install_dylib () { + INSTALL_BASE=$1 + FULL_EXT=$2 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + } + + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do + install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" + done + + # Clean up dylib template + rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" + + echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; + +10. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: + + * :c:member:`UTF-8 mode <PyPreConfig.utf8_mode>` is *enabled*; + * :c:member:`Buffered stdio <PyConfig.buffered_stdio>` is *disabled*; + * :c:member:`Writing bytecode <PyConfig.write_bytecode>` is *disabled*; + * :c:member:`Signal handlers <PyConfig.install_signal_handlers>` are *enabled*; + * ``PYTHONHOME`` for the interpreter is configured to point at the + ``python`` subfolder of your app's bundle; and + * The ``PYTHONPATH`` for the interpreter includes: + + - the ``python/lib/python3.X`` subfolder of your app's bundle, + - the ``python/lib/python3.X/lib-dynload`` subfolder of your app's bundle, and + - the ``app`` subfolder of your app's bundle + + Your app's bundle location can be determined using ``[[NSBundle mainBundle] + resourcePath]``. + +Steps 8, 9 and 10 of these instructions assume that you have a single folder of +pure Python application code, named ``app``. If you have third-party binary +modules in your app, some additional steps will be required: + +* You need to ensure that any folders containing third-party binaries are + either associated with the app target, or copied in as part of step 8. Step 8 + should also purge any binaries that are not appropriate for the platform a + specific build is targeting (i.e., delete any device binaries if you're + building an app targeting the simulator). + +* Any folders that contain third-party binaries must be processed into + framework form by step 9. The invocation of ``install_dylib`` that processes + the ``lib-dynload`` folder can be copied and adapted for this purpose. + +* If you're using a separate folder for third-party packages, ensure that folder + is included as part of the ``PYTHONPATH`` configuration in step 10. + +App Store Compliance +==================== + +The only mechanism for distributing apps to third-party iOS devices is to +submit the app to the iOS App Store; apps submitted for distribution must pass +Apple's app review process. This process includes a set of automated validation +rules that inspect the submitted application bundle for problematic code. + +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged; so, it is necessary to modify the Python +standard library for an app to pass App Store review. + +The Python source tree contains +:source:`a patch file <Mac/Resources/app-store-compliance.patch>` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when building for iOS. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index eb1413af2cbc3d..44fb00de3733c5 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -10,41 +10,46 @@ Using Python on a Mac Python on a Mac running macOS is in principle very similar to Python on any other Unix platform, but there are a number of additional features such as -the IDE and the Package Manager that are worth pointing out. +the integrated development environment (IDE) and the Package Manager that are +worth pointing out. + .. _getting-osx: +.. _getting-and-installing-macpython: -Getting and Installing MacPython -================================ +Getting and Installing Python +============================= macOS used to come with Python 2.7 pre-installed between versions 10.8 and `12.3 <https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python>`_. -You are invited to install the most recent version of Python 3 from the Python -website (https://www.python.org). A current "universal binary" build of Python, -which runs natively on the Mac's new Intel and legacy PPC CPU's, is available -there. +You are invited to install the most recent version of Python 3 from the `Python +website <https://www.python.org/downloads/macos/>`__. +A current "universal2 binary" build of Python, which runs natively on the Mac's +new Apple Silicon and legacy Intel processors, is available there. What you get after installing is a number of things: -* A :file:`Python 3.12` folder in your :file:`Applications` folder. In here +* A |python_version_literal| folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official - Python distributions; and PythonLauncher, which handles double-clicking Python + Python distributions; and :program:`Python Launcher`, which handles double-clicking Python scripts from the Finder. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell - path. To uninstall MacPython, you can simply remove these three things. A - symlink to the Python executable is placed in /usr/local/bin/. - -The Apple-provided build of Python is installed in -:file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, -respectively. You should never modify or delete these, as they are -Apple-controlled and are used by Apple- or third-party software. Remember that -if you choose to install a newer Python version from python.org, you will have -two different but functional Python installations on your computer, so it will -be important that your paths and usages are consistent with what you want to do. - -IDLE includes a help menu that allows you to access Python documentation. If you + path. To uninstall Python, you can remove these three things. A + symlink to the Python executable is placed in :file:`/usr/local/bin/`. + +.. note:: + + On macOS 10.8-12.3, the Apple-provided build of Python is installed in + :file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, + respectively. You should never modify or delete these, as they are + Apple-controlled and are used by Apple- or third-party software. Remember that + if you choose to install a newer Python version from python.org, you will have + two different but functional Python installations on your computer, so it will + be important that your paths and usages are consistent with what you want to do. + +IDLE includes a Help menu that allows you to access Python documentation. If you are completely new to Python you should start reading the tutorial introduction in that document. @@ -56,29 +61,29 @@ How to run a Python script -------------------------- Your best way to get started with Python on macOS is through the IDLE -integrated development environment, see section :ref:`ide` and use the Help menu +integrated development environment; see section :ref:`ide` and use the Help menu when the IDE is running. If you want to run Python scripts from the Terminal window command line or from the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` and -:program:`emacs` among them. If you want a more Mac-like editor, -:program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see -http://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` -(http://aquamacs.org/). +number of standard Unix command line editors, :program:`vim` +:program:`nano` among them. If you want a more Mac-like editor, +:program:`BBEdit` from Bare Bones Software (see +https://www.barebones.com/products/bbedit/index.html) are good choices, as is +:program:`TextMate` (see https://macromates.com). Other editors include +:program:`MacVim` (https://macvim.org) and :program:`Aquamacs` +(https://aquamacs.org). To run your script from the Terminal window you must make sure that :file:`/usr/local/bin` is in your shell search path. To run your script from the Finder you have two options: -* Drag it to :program:`PythonLauncher` +* Drag it to :program:`Python Launcher`. -* Select :program:`PythonLauncher` as the default application to open your - script (or any .py script) through the finder Info window and double-click it. - :program:`PythonLauncher` has various preferences to control how your script is +* Select :program:`Python Launcher` as the default application to open your + script (or any ``.py`` script) through the finder Info window and double-click it. + :program:`Python Launcher` has various preferences to control how your script is launched. Option-dragging allows you to change these for one invocation, or use its Preferences menu to change things globally. @@ -103,10 +108,11 @@ Python on macOS honors all standard Unix environment variables such as :envvar:`PYTHONPATH`, but setting these variables for programs started from the Finder is non-standard as the Finder does not read your :file:`.profile` or :file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's Technical Document QA1067 for -details. +:file:`~/.MacOSX/environment.plist`. See Apple's +`Technical Q&A QA1067 <https://developer.apple.com/library/archive/qa/qa1067/_index.html>`__ +for details. -For more information on installation Python packages in MacPython, see section +For more information on installation Python packages, see section :ref:`mac-package-manager`. @@ -115,9 +121,9 @@ For more information on installation Python packages in MacPython, see section The IDE ======= -MacPython ships with the standard IDLE development environment. A good +Python ships with the standard IDLE development environment. A good introduction to using IDLE can be found at -http://www.hashcollision.org/hkn/python/idle_intro/index.html. +https://www.hashcollision.org/hkn/python/idle_intro/index.html. .. _mac-package-manager: @@ -130,45 +136,88 @@ This section has moved to the `Python Packaging User Guide`_. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ -GUI Programming on the Mac -========================== +.. _gui-programming-on-the-mac: + +GUI Programming +=============== There are several options for building GUI applications on the Mac with Python. *PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is the foundation of most modern Mac development. Information on PyObjC is -available from https://pypi.org/project/pyobjc/. +available from :pypi:`pyobjc`. The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with macOS by Apple, and the latest version can be downloaded and installed from https://www.activestate.com; it can also be built from source. -*wxPython* is another popular cross-platform GUI toolkit that runs natively on -macOS. Packages and documentation are available from https://www.wxpython.org. +A number of alternative macOS GUI toolkits are available: + +* `PySide <https://www.qt.io/qt-for-python>`__: Official Python bindings to the + `Qt GUI toolkit <https://qt.io>`__. + +* `PyQt <https://riverbankcomputing.com/software/pyqt/intro>`__: Alternative + Python bindings to Qt. + +* `Kivy <https://kivy.org>`__: A cross-platform GUI toolkit that supports + desktop and mobile platforms. -*PyQt* is another popular cross-platform GUI toolkit that runs natively on -macOS. More information can be found at -https://riverbankcomputing.com/software/pyqt/intro. +* `Toga <https://toga.readthedocs.io>`__: Part of the `BeeWare Project + <https://beeware.org>`__; supports desktop, mobile, web and console apps. +* `wxPython <https://www.wxpython.org>`__: A cross-platform toolkit that + supports desktop operating systems. -Distributing Python Applications on the Mac -=========================================== +.. _distributing-python-applications-on-the-mac: -The standard tool for deploying standalone Python applications on the Mac is -:program:`py2app`. More information on installing and using py2app can be found -at https://pypi.org/project/py2app/. +Distributing Python Applications +================================ + +A range of tools exist for converting your Python code into a standalone +distributable application: + +* :pypi:`py2app`: Supports creating macOS ``.app`` + bundles from a Python project. + +* `Briefcase <https://briefcase.readthedocs.io>`__: Part of the `BeeWare Project + <https://beeware.org>`__; a cross-platform packaging tool that supports + creation of ``.app`` bundles on macOS, as well as managing signing and + notarization. + +* `PyInstaller <https://pyinstaller.org/>`__: A cross-platform packaging tool that creates + a single file or folder as a distributable artifact. +App Store Compliance +-------------------- + +Apps submitted for distribution through the macOS App Store must pass Apple's +app review process. This process includes a set of automated validation rules +that inspect the submitted application bundle for problematic code. + +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged. Therefore, it is necessary to modify the +Python standard library for an app to pass App Store review. + +The Python source tree contains +:source:`a patch file <Mac/Resources/app-store-compliance.patch>` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when CPython is configured with the +:option:`--with-app-store-compliance` option. + +This patch is not normally required to use CPython on a Mac; nor is it required +if you are distributing an app *outside* the macOS App Store. It is *only* +required if you are using the macOS App Store as a distribution channel. Other Resources =============== -The MacPython mailing list is an excellent support resource for Python users and -developers on the Mac: +The Pythonmac-SIG mailing list is an excellent support resource for Python users +and developers on the Mac: https://www.python.org/community/sigs/current/pythonmac-sig/ Another useful resource is the MacPython wiki: https://wiki.python.org/moin/MacPython - diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 1cf438b198a9af..354eb1541ceac2 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -14,14 +14,14 @@ used at environment creation time). It also creates an (initially empty) ``Lib\site-packages``). If an existing directory is specified, it will be re-used. +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + .. deprecated:: 3.6 ``pyvenv`` was the recommended tool for creating virtual environments for Python 3.3 and 3.4, and is :ref:`deprecated in Python 3.6 <whatsnew36-venv>`. -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - .. highlight:: none On Windows, invoke the ``venv`` command as follows:: diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png new file mode 100644 index 00000000000000..0aa01c1df6e051 Binary files /dev/null and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 598bf3ca9bcc04..ef98d32e8674ec 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -14,8 +14,8 @@ know about when using Python on Microsoft Windows. Unlike most Unix systems and services, Windows does not include a system supported installation of Python. To make Python available, the CPython team -has compiled Windows installers (MSI packages) with every `release -<https://www.python.org/download/releases/>`_ for many years. These installers +has compiled Windows installers with every `release +<https://www.python.org/downloads/>`_ for many years. These installers are primarily intended to add a per-user installation of Python, with the core interpreter and library being used by a single user. The installer is also able to install for all users of a single machine, and a separate ZIP file is @@ -307,6 +307,46 @@ settings and replace any that have been removed or modified. "Uninstall" will remove Python entirely, with the exception of the :ref:`launcher`, which has its own entry in Programs and Features. +.. _install-freethreaded-windows: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +To install pre-built binaries with free-threading enabled (see :pep:`703`), you +should select "Customize installation". The second page of options includes the +"Download free-threaded binaries" checkbox. + +.. image:: win_install_freethreaded.png + +Selecting this option will download and install additional binaries to the same +location as the main Python install. The main executable is called +``python3.13t.exe``, and other binaries either receive a ``t`` suffix or a full +ABI suffix. Python source files and bundled third-party dependencies are shared +with the main install. + +The free-threaded version is registered as a regular Python install with the +tag ``3.13t`` (with a ``-32`` or ``-arm64`` suffix as normal for those +platforms). This allows tools to discover it, and for the :ref:`launcher` to +support ``py.exe -3.13t``. Note that the launcher will interpret ``py.exe -3`` +(or a ``python3`` shebang) as "the latest 3.x install", which will prefer the +free-threaded binaries over the regular ones, while ``py.exe -3.13`` will not. +If you use the short style of option, you may prefer to not install the +free-threaded binaries at this time. + +To specify the install option at the command line, use +``Include_freethreaded=1``. See :ref:`install-layout-option` for instructions on +pre-emptively downloading the additional binaries for offline install. The +options to include debug symbols and binaries also apply to the free-threaded +builds. + +Free-threaded binaries are also available :ref:`on nuget.org <windows-nuget>`. .. _windows-store: @@ -450,9 +490,29 @@ automatically use the headers and import libraries in your build. The package information pages on nuget.org are `www.nuget.org/packages/python <https://www.nuget.org/packages/python>`_ -for the 64-bit version and `www.nuget.org/packages/pythonx86 -<https://www.nuget.org/packages/pythonx86>`_ for the 32-bit version. +for the 64-bit version, `www.nuget.org/packages/pythonx86 +<https://www.nuget.org/packages/pythonx86>`_ for the 32-bit version, and +`www.nuget.org/packages/pythonarm64 +<https://www.nuget.org/packages/pythonarm64>`_ for the ARM64 version + +Free-threaded packages +---------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. +Packages containing free-threaded binaries are named +`python-freethreaded <https://www.nuget.org/packages/python-freethreaded>`_ +for the 64-bit version, `pythonx86-freethreaded +<https://www.nuget.org/packages/pythonx86-freethreaded>`_ for the 32-bit +version, and `pythonarm64-freethreaded +<https://www.nuget.org/packages/pythonarm64-freethreaded>`_ for the ARM64 +version. These packages contain both the ``python3.13t.exe`` and +``python.exe`` entry points, both of which run free threaded. .. _windows-embeddable: @@ -1225,7 +1285,7 @@ The Windows-specific standard modules are documented in PyWin32 ------- -The `PyWin32 <https://pypi.org/project/pywin32>`_ module by Mark Hammond +The :pypi:`PyWin32` module by Mark Hammond is a collection of modules for advanced Windows-specific support. This includes utilities for: diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index c2b0ae8c76302a..1a949ec4035807 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -130,7 +130,7 @@ Guidelines": Read the rest of :pep:`1` for the details of the PEP editorial process, style, and format. PEPs are kept in the Python CVS tree on SourceForge, though they're not part of the Python 2.0 distribution, and are also available in HTML form from -https://peps.python.org/. As of September 2000, there are 25 PEPS, ranging +https://peps.python.org/. As of September 2000, there are 25 PEPs, ranging from :pep:`201`, "Lockstep Iteration", to PEP 225, "Elementwise/Objectwise Operators". @@ -217,13 +217,13 @@ often use the ``codecs.lookup(encoding)`` function, which returns a was consumed. * *stream_reader* is a class that supports decoding input from a stream. - *stream_reader(file_obj)* returns an object that supports the :meth:`read`, - :meth:`readline`, and :meth:`readlines` methods. These methods will all + *stream_reader(file_obj)* returns an object that supports the :meth:`!read`, + :meth:`!readline`, and :meth:`!readlines` methods. These methods will all translate from the given encoding and return Unicode strings. * *stream_writer*, similarly, is a class that supports encoding output to a stream. *stream_writer(file_obj)* returns an object that supports the - :meth:`write` and :meth:`writelines` methods. These methods expect Unicode + :meth:`!write` and :meth:`!writelines` methods. These methods expect Unicode strings, translating them to the given encoding on output. For example, the following code writes a Unicode string into a file, encoding @@ -356,8 +356,8 @@ variable ``a`` by 2, equivalent to the slightly lengthier ``a = a + 2``. The full list of supported assignment operators is ``+=``, ``-=``, ``*=``, ``/=``, ``%=``, ``**=``, ``&=``, ``|=``, ``^=``, ``>>=``, and ``<<=``. Python classes can override the augmented assignment operators by defining methods -named :meth:`__iadd__`, :meth:`__isub__`, etc. For example, the following -:class:`Number` class stores a number and supports using += to create a new +named :meth:`!__iadd__`, :meth:`!__isub__`, etc. For example, the following +:class:`!Number` class stores a number and supports using += to create a new instance with an incremented value. .. The empty groups below prevent conversion to guillemets. @@ -374,7 +374,7 @@ instance with an incremented value. n += 3 print n.value -The :meth:`__iadd__` special method is called with the value of the increment, +The :meth:`!__iadd__` special method is called with the value of the increment, and should return a new instance with an appropriately modified value; this return value is bound as the new value of the variable on the left-hand side. @@ -390,10 +390,10 @@ String Methods ============== Until now string-manipulation functionality was in the :mod:`string` module, -which was usually a front-end for the :mod:`strop` module written in C. The -addition of Unicode posed a difficulty for the :mod:`strop` module, because the +which was usually a front-end for the :mod:`!strop` module written in C. The +addition of Unicode posed a difficulty for the :mod:`!strop` module, because the functions would all need to be rewritten in order to accept either 8-bit or -Unicode strings. For functions such as :func:`string.replace`, which takes 3 +Unicode strings. For functions such as :func:`!string.replace`, which takes 3 string arguments, that means eight possible permutations, and correspondingly complicated code. @@ -416,13 +416,13 @@ The old :mod:`string` module is still around for backwards compatibility, but it mostly acts as a front-end to the new string methods. Two methods which have no parallel in pre-2.0 versions, although they did exist -in JPython for quite some time, are :meth:`startswith` and :meth:`endswith`. +in JPython for quite some time, are :meth:`!startswith` and :meth:`!endswith`. ``s.startswith(t)`` is equivalent to ``s[:len(t)] == t``, while ``s.endswith(t)`` is equivalent to ``s[-len(t):] == t``. -One other method which deserves special mention is :meth:`join`. The -:meth:`join` method of a string receives one parameter, a sequence of strings, -and is equivalent to the :func:`string.join` function from the old :mod:`string` +One other method which deserves special mention is :meth:`!join`. The +:meth:`!join` method of a string receives one parameter, a sequence of strings, +and is equivalent to the :func:`!string.join` function from the old :mod:`string` module, with the arguments reversed. In other words, ``s.join(seq)`` is equivalent to the old ``string.join(seq, s)``. @@ -503,9 +503,9 @@ Minor Language Changes A new syntax makes it more convenient to call a given function with a tuple of arguments and/or a dictionary of keyword arguments. In Python 1.5 and earlier, -you'd use the :func:`apply` built-in function: ``apply(f, args, kw)`` calls the -function :func:`f` with the argument tuple *args* and the keyword arguments in -the dictionary *kw*. :func:`apply` is the same in 2.0, but thanks to a patch +you'd use the :func:`!apply` built-in function: ``apply(f, args, kw)`` calls the +function :func:`!f` with the argument tuple *args* and the keyword arguments in +the dictionary *kw*. :func:`!apply` is the same in 2.0, but thanks to a patch from Greg Ewing, ``f(*args, **kw)`` is a shorter and clearer way to achieve the same effect. This syntax is symmetrical with the syntax for defining functions:: @@ -518,7 +518,7 @@ functions:: The ``print`` statement can now have its output directed to a file-like object by following the ``print`` with ``>> file``, similar to the redirection operator in Unix shells. Previously you'd either have to use the -:meth:`write` method of the file-like object, which lacks the convenience and +:meth:`!write` method of the file-like object, which lacks the convenience and simplicity of ``print``, or you could assign a new value to ``sys.stdout`` and then restore the old value. For sending output to standard error, it's much easier to write this:: @@ -540,7 +540,7 @@ Previously there was no way to implement a class that overrode Python's built-in true if *obj* is present in the sequence *seq*; Python computes this by simply trying every index of the sequence until either *obj* is found or an :exc:`IndexError` is encountered. Moshe Zadka contributed a patch which adds a -:meth:`__contains__` magic method for providing a custom implementation for +:meth:`!__contains__` magic method for providing a custom implementation for :keyword:`!in`. Additionally, new built-in objects written in C can define what :keyword:`!in` means for them via a new slot in the sequence protocol. @@ -562,7 +562,7 @@ the python-dev mailing list for the discussion leading up to this implementation, and some useful relevant links. Note that comparisons can now also raise exceptions. In earlier versions of Python, a comparison operation such as ``cmp(a,b)`` would always produce an answer, even if a user-defined -:meth:`__cmp__` method encountered an error, since the resulting exception would +:meth:`!__cmp__` method encountered an error, since the resulting exception would simply be silently swallowed. .. Starting URL: @@ -607,7 +607,7 @@ seq1, seq2)`` is that :func:`map` pads the sequences with ``None`` if the sequences aren't all of the same length, while :func:`zip` truncates the returned list to the length of the shortest argument sequence. -The :func:`int` and :func:`long` functions now accept an optional "base" +The :func:`int` and :func:`!long` functions now accept an optional "base" parameter when the first argument is a string. ``int('123', 10)`` returns 123, while ``int('123', 16)`` returns 291. ``int(123, 16)`` raises a :exc:`TypeError` exception with the message "can't convert non-string with @@ -620,8 +620,8 @@ would be ``(2, 0, 1, 'beta', 1)``. *level* is a string such as ``"alpha"``, ``"beta"``, or ``"final"`` for a final release. Dictionaries have an odd new method, ``setdefault(key, default)``, which -behaves similarly to the existing :meth:`get` method. However, if the key is -missing, :meth:`setdefault` both returns the value of *default* as :meth:`get` +behaves similarly to the existing :meth:`!get` method. However, if the key is +missing, :meth:`!setdefault` both returns the value of *default* as :meth:`!get` would do, and also inserts it into the dictionary as the value for *key*. Thus, the following lines of code:: @@ -656,7 +656,7 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`append` and :meth:`insert`. In earlier versions of Python, if ``L`` is +:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an @@ -671,9 +671,9 @@ errors. If you absolutely must use 2.0 but can't fix your code, you can edit ``NO_STRICT_LIST_APPEND`` to preserve the old behaviour; this isn't recommended. Some of the functions in the :mod:`socket` module are still forgiving in this -way. For example, :func:`socket.connect( ('hostname', 25) )` is the correct -form, passing a tuple representing an IP address, but :func:`socket.connect( -'hostname', 25 )` also works. :func:`socket.connect_ex` and :func:`socket.bind` +way. For example, ``socket.connect( ('hostname', 25) )`` is the correct +form, passing a tuple representing an IP address, but ``socket.connect('hostname', 25)`` +also works. :meth:`socket.connect_ex <socket.socket.connect_ex>` and :meth:`socket.bind <socket.socket.bind>` are similarly easy-going. 2.0alpha1 tightened these functions up, but because the documentation actually used the erroneous multiple argument form, many people wrote code which would break with the stricter checking. GvR backed out @@ -693,7 +693,7 @@ advantage of this fact will break in 2.0. Some work has been done to make integers and long integers a bit more interchangeable. In 1.5.2, large-file support was added for Solaris, to allow -reading files larger than 2 GiB; this made the :meth:`tell` method of file +reading files larger than 2 GiB; this made the :meth:`!tell` method of file objects return a long integer instead of a regular integer. Some code would subtract two file offsets and attempt to use the result to multiply a sequence or slice a string, but this raised a :exc:`TypeError`. In 2.0, long integers @@ -701,7 +701,7 @@ can be used to multiply or slice a sequence, and it'll behave as you'd intuitively expect it to; ``3L * 'abc'`` produces 'abcabcabc', and ``(0,1,2,3)[2L:4L]`` produces (2,3). Long integers can also be used in various contexts where previously only integers were accepted, such as in the -:meth:`seek` method of file objects, and in the formats supported by the ``%`` +:meth:`!seek` method of file objects, and in the formats supported by the ``%`` operator (``%d``, ``%i``, ``%x``, etc.). For example, ``"%d" % 2L**64`` will produce the string ``18446744073709551616``. @@ -715,7 +715,7 @@ digit. Taking the :func:`repr` of a float now uses a different formatting precision than :func:`str`. :func:`repr` uses ``%.17g`` format string for C's -:func:`sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that +:func:`!sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that :func:`repr` may occasionally show more decimal places than :func:`str`, for certain numbers. For example, the number 8.1 can't be represented exactly in binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is @@ -723,7 +723,7 @@ binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is The ``-X`` command-line option, which turned all standard exceptions into strings instead of classes, has been removed; the standard exceptions will now -always be classes. The :mod:`exceptions` module containing the standard +always be classes. The :mod:`!exceptions` module containing the standard exceptions was translated from Python to a built-in C module, written by Barry Warsaw and Fredrik Lundh. @@ -879,11 +879,11 @@ joins the basic set of Python documentation. XML Modules =========== -Python 1.5.2 included a simple XML parser in the form of the :mod:`xmllib` +Python 1.5.2 included a simple XML parser in the form of the :mod:`!xmllib` module, contributed by Sjoerd Mullender. Since 1.5.2's release, two different interfaces for processing XML have become common: SAX2 (version 2 of the Simple API for XML) provides an event-driven interface with some similarities to -:mod:`xmllib`, and the DOM (Document Object Model) provides a tree-based +:mod:`!xmllib`, and the DOM (Document Object Model) provides a tree-based interface, transforming an XML document into a tree of nodes that can be traversed and modified. Python 2.0 includes a SAX2 interface and a stripped-down DOM interface as part of the :mod:`xml` package. Here we will give a brief @@ -898,9 +898,9 @@ SAX2 Support SAX defines an event-driven interface for parsing XML. To use SAX, you must write a SAX handler class. Handler classes inherit from various classes provided by SAX, and override various methods that will then be called by the -XML parser. For example, the :meth:`startElement` and :meth:`endElement` +XML parser. For example, the :meth:`~xml.sax.handler.ContentHandler.startElement` and :meth:`~xml.sax.handler.ContentHandler.endElement` methods are called for every starting and end tag encountered by the parser, the -:meth:`characters` method is called for every chunk of character data, and so +:meth:`~xml.sax.handler.ContentHandler.characters` method is called for every chunk of character data, and so forth. The advantage of the event-driven approach is that the whole document doesn't @@ -940,8 +940,8 @@ DOM Support ----------- The Document Object Model is a tree-based representation for an XML document. A -top-level :class:`Document` instance is the root of the tree, and has a single -child which is the top-level :class:`Element` instance. This :class:`Element` +top-level :class:`!Document` instance is the root of the tree, and has a single +child which is the top-level :class:`!Element` instance. This :class:`!Element` has children nodes representing character data and any sub-elements, which may have further children of their own, and so forth. Using the DOM you can traverse the resulting tree any way you like, access element and attribute @@ -955,18 +955,18 @@ simply writing ``<tag1>``...\ ``</tag1>`` to a file. The DOM implementation included with Python lives in the :mod:`xml.dom.minidom` module. It's a lightweight implementation of the Level 1 DOM with support for -XML namespaces. The :func:`parse` and :func:`parseString` convenience +XML namespaces. The :func:`!parse` and :func:`!parseString` convenience functions are provided for generating a DOM tree:: from xml.dom import minidom doc = minidom.parse('hamlet.xml') -``doc`` is a :class:`Document` instance. :class:`Document`, like all the other -DOM classes such as :class:`Element` and :class:`Text`, is a subclass of the -:class:`Node` base class. All the nodes in a DOM tree therefore support certain -common methods, such as :meth:`toxml` which returns a string containing the XML +``doc`` is a :class:`!Document` instance. :class:`!Document`, like all the other +DOM classes such as :class:`!Element` and :class:`Text`, is a subclass of the +:class:`!Node` base class. All the nodes in a DOM tree therefore support certain +common methods, such as :meth:`!toxml` which returns a string containing the XML representation of the node and its children. Each class also has special -methods of its own; for example, :class:`Element` and :class:`Document` +methods of its own; for example, :class:`!Element` and :class:`!Document` instances have a method to find all child elements with a given tag name. Continuing from the previous 2-line example:: @@ -995,7 +995,7 @@ its children can be easily modified by deleting, adding, or removing nodes:: root.insertBefore( root.childNodes[0], root.childNodes[20] ) Again, I will refer you to the Python documentation for a complete listing of -the different :class:`Node` classes and their various methods. +the different :class:`!Node` classes and their various methods. Relationship to PyXML @@ -1020,7 +1020,7 @@ features in PyXML include: * The xmlproc validating parser, written by Lars Marius Garshol. -* The :mod:`sgmlop` parser accelerator module, written by Fredrik Lundh. +* The :mod:`!sgmlop` parser accelerator module, written by Fredrik Lundh. .. ====================================================================== @@ -1030,8 +1030,8 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, -:mod:`ConfigParser`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, -:mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, +:mod:`ConfigParser <configparser>`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, +:mod:`!xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. Brian Gallew contributed OpenSSL support for the :mod:`socket` module. OpenSSL @@ -1039,16 +1039,17 @@ is an implementation of the Secure Socket Layer, which encrypts the data being sent over a socket. When compiling Python, you can edit :file:`Modules/Setup` to include SSL support, which adds an additional function to the :mod:`socket` module: ``socket.ssl(socket, keyfile, certfile)``, which takes a socket -object and returns an SSL socket. The :mod:`httplib` and :mod:`urllib` modules +object and returns an SSL socket. The :mod:`httplib <http>` and :mod:`urllib` modules were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. -The :mod:`httplib` module has been rewritten by Greg Stein to support HTTP/1.1. -Backward compatibility with the 1.5 version of :mod:`httplib` is provided, +The :mod:`httplib <http>` module has been rewritten by Greg Stein to support HTTP/1.1. + +Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. -The :mod:`Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and +The :mod:`!Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and support for the older 7.x versions has been dropped. The Tkinter module now supports displaying Unicode strings in Tk widgets. Also, Fredrik Lundh contributed an optimization which makes operations like ``create_line`` and @@ -1083,11 +1084,11 @@ module. calling :func:`atexit.register` with the function to be called on exit. (Contributed by Skip Montanaro.) -* :mod:`codecs`, :mod:`encodings`, :mod:`unicodedata`: Added as part of the new +* :mod:`codecs`, :mod:`!encodings`, :mod:`unicodedata`: Added as part of the new Unicode support. -* :mod:`filecmp`: Supersedes the old :mod:`cmp`, :mod:`cmpcache` and - :mod:`dircmp` modules, which have now become deprecated. (Contributed by Gordon +* :mod:`filecmp`: Supersedes the old :mod:`!cmp`, :mod:`!cmpcache` and + :mod:`!dircmp` modules, which have now become deprecated. (Contributed by Gordon MacMillan and Moshe Zadka.) * :mod:`gettext`: This module provides internationalization (I18N) and @@ -1095,8 +1096,8 @@ module. GNU gettext message catalog library. (Integrated by Barry Warsaw, from separate contributions by Martin von Löwis, Peter Funk, and James Henstridge.) -* :mod:`linuxaudiodev`: Support for the :file:`/dev/audio` device on Linux, a - twin to the existing :mod:`sunaudiodev` module. (Contributed by Peter Bosch, +* :mod:`!linuxaudiodev`: Support for the :file:`/dev/audio` device on Linux, a + twin to the existing :mod:`!sunaudiodev` module. (Contributed by Peter Bosch, with fixes by Jeremy Hylton.) * :mod:`mmap`: An interface to memory-mapped files on both Windows and Unix. A @@ -1105,10 +1106,10 @@ module. be passed to functions that expect ordinary strings, such as the :mod:`re` module. (Contributed by Sam Rushing, with some extensions by A.M. Kuchling.) -* :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul +* :mod:`!pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) -* :mod:`robotparser`: Parse a :file:`robots.txt` file, which is used for writing +* :mod:`robotparser <urllib.robotparser>`: Parse a :file:`robots.txt` file, which is used for writing web spiders that politely avoid certain areas of a web site. The parser accepts the contents of a :file:`robots.txt` file, builds a set of rules from it, and can then answer questions about the fetchability of a given URL. (Contributed @@ -1117,7 +1118,7 @@ module. * :mod:`tabnanny`: A module/script to check Python source code for ambiguous indentation. (Contributed by Tim Peters.) -* :mod:`UserString`: A base class useful for deriving objects that behave like +* :mod:`!UserString`: A base class useful for deriving objects that behave like strings. * :mod:`webbrowser`: A module that provides a platform independent way to launch @@ -1129,18 +1130,18 @@ module. :file:`Tools/idle/BrowserControl.py`, and adapted for the standard library by Fred.) -* :mod:`_winreg`: An interface to the Windows registry. :mod:`_winreg` is an +* :mod:`_winreg <winreg>`: An interface to the Windows registry. :mod:`!_winreg` is an adaptation of functions that have been part of PythonWin since 1995, but has now been added to the core distribution, and enhanced to support Unicode. - :mod:`_winreg` was written by Bill Tutt and Mark Hammond. + :mod:`!_winreg` was written by Bill Tutt and Mark Hammond. * :mod:`zipfile`: A module for reading and writing ZIP-format archives. These are archives produced by :program:`PKZIP` on DOS/Windows or :program:`zip` on Unix, not to be confused with :program:`gzip`\ -format files (which are supported by the :mod:`gzip` module) (Contributed by James C. Ahlstrom.) -* :mod:`imputil`: A module that provides a simpler way for writing customized - import hooks, in comparison to the existing :mod:`ihooks` module. (Implemented +* :mod:`!imputil`: A module that provides a simpler way for writing customized + import hooks, in comparison to the existing :mod:`!ihooks` module. (Implemented by Greg Stein, with much discussion on python-dev along the way.) .. ====================================================================== @@ -1184,13 +1185,13 @@ Deleted and Deprecated Modules ============================== A few modules have been dropped because they're obsolete, or because there are -now better ways to do the same thing. The :mod:`stdwin` module is gone; it was +now better ways to do the same thing. The :mod:`!stdwin` module is gone; it was for a platform-independent windowing toolkit that's no longer developed. A number of modules have been moved to the :file:`lib-old` subdirectory: -:mod:`cmp`, :mod:`cmpcache`, :mod:`dircmp`, :mod:`dump`, :mod:`find`, -:mod:`grep`, :mod:`packmail`, :mod:`poly`, :mod:`util`, :mod:`whatsound`, -:mod:`zmod`. If you have code which relies on a module that's been moved to +:mod:`!cmp`, :mod:`!cmpcache`, :mod:`!dircmp`, :mod:`!dump`, :mod:`!find`, +:mod:`!grep`, :mod:`!packmail`, :mod:`!poly`, :mod:`!util`, :mod:`!whatsound`, +:mod:`!zmod`. If you have code which relies on a module that's been moved to :file:`lib-old`, you can simply add that directory to ``sys.path`` to get them back, but you're encouraged to update any code that uses these modules. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index f0e1ded75a9d27..b4002f06e92adc 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -48,7 +48,7 @@ nested recursive function definition doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -104,7 +104,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -161,7 +161,7 @@ PEP 207: Rich Comparisons In earlier versions, Python's support for implementing comparisons on user-defined classes and extension types was quite simple. Classes could implement a -:meth:`__cmp__` method that was given two instances of a class, and could only +:meth:`!__cmp__` method that was given two instances of a class, and could only return 0 if they were equal or +1 or -1 if they weren't; the method couldn't raise an exception or return anything other than a Boolean value. Users of Numeric Python often found this model too weak and restrictive, because in the @@ -175,21 +175,21 @@ In Python 2.1, rich comparisons were added in order to support this need. Python classes can now individually overload each of the ``<``, ``<=``, ``>``, ``>=``, ``==``, and ``!=`` operations. The new magic method names are: -+-----------+----------------+ -| Operation | Method name | -+===========+================+ -| ``<`` | :meth:`__lt__` | -+-----------+----------------+ -| ``<=`` | :meth:`__le__` | -+-----------+----------------+ -| ``>`` | :meth:`__gt__` | -+-----------+----------------+ -| ``>=`` | :meth:`__ge__` | -+-----------+----------------+ -| ``==`` | :meth:`__eq__` | -+-----------+----------------+ -| ``!=`` | :meth:`__ne__` | -+-----------+----------------+ ++-----------+------------------------+ +| Operation | Method name | ++===========+========================+ +| ``<`` | :meth:`~object.__lt__` | ++-----------+------------------------+ +| ``<=`` | :meth:`~object.__le__` | ++-----------+------------------------+ +| ``>`` | :meth:`~object.__gt__` | ++-----------+------------------------+ +| ``>=`` | :meth:`~object.__ge__` | ++-----------+------------------------+ +| ``==`` | :meth:`~object.__eq__` | ++-----------+------------------------+ +| ``!=`` | :meth:`~object.__ne__` | ++-----------+------------------------+ (The magic methods are named after the corresponding Fortran operators ``.LT.``. ``.LE.``, &c. Numeric programmers are almost certainly quite familiar with @@ -208,7 +208,7 @@ The built-in ``cmp(A,B)`` function can use the rich comparison machinery, and now accepts an optional argument specifying which comparison operation to use; this is given as one of the strings ``"<"``, ``"<="``, ``">"``, ``">="``, ``"=="``, or ``"!="``. If called without the optional third argument, -:func:`cmp` will only return -1, 0, or +1 as in previous versions of Python; +:func:`!cmp` will only return -1, 0, or +1 as in previous versions of Python; otherwise it will call the appropriate method and can return any Python object. There are also corresponding changes of interest to C programmers; there's a new @@ -245,7 +245,7 @@ out warnings that you don't want to be displayed. Third-party modules can also use this framework to deprecate old features that they no longer wish to support. -For example, in Python 2.1 the :mod:`regex` module is deprecated, so importing +For example, in Python 2.1 the :mod:`!regex` module is deprecated, so importing it causes a warning to be printed:: >>> import regex @@ -262,7 +262,7 @@ can be used to specify a particular warning category. Filters can be added to disable certain warnings; a regular expression pattern can be applied to the message or to the module name in order to suppress a -warning. For example, you may have a program that uses the :mod:`regex` module +warning. For example, you may have a program that uses the :mod:`!regex` module and not want to spare the time to convert it to use the :mod:`re` module right now. The warning can be suppressed by calling :: @@ -274,7 +274,7 @@ now. The warning can be suppressed by calling :: This adds a filter that will apply only to warnings of the class :class:`DeprecationWarning` triggered in the :mod:`__main__` module, and applies -a regular expression to only match the message about the :mod:`regex` module +a regular expression to only match the message about the :mod:`!regex` module being deprecated, and will cause such warnings to be ignored. Warnings can also be printed only once, printed every time the offending code is executed, or turned into exceptions that will cause the program to stop (unless the @@ -368,7 +368,7 @@ dictionary:: This version works for simple things such as integers, but it has a side effect; the ``_cache`` dictionary holds a reference to the return values, so they'll never be deallocated until the Python process exits and cleans up. This isn't -very noticeable for integers, but if :func:`f` returns an object, or a data +very noticeable for integers, but if :func:`!f` returns an object, or a data structure that takes up a lot of memory, this can be a problem. Weak references provide a way to implement a cache that won't keep objects alive @@ -379,7 +379,7 @@ created by calling ``wr = weakref.ref(obj)``. The object being referred to is returned by calling the weak reference as if it were a function: ``wr()``. It will return the referenced object, or ``None`` if the object no longer exists. -This makes it possible to write a :func:`memoize` function whose cache doesn't +This makes it possible to write a :func:`!memoize` function whose cache doesn't keep objects alive, by storing weak references in the cache. :: _cache = {} @@ -402,7 +402,7 @@ weak references --- an object referenced only by proxy objects is deallocated -- but instead of requiring an explicit call to retrieve the object, the proxy transparently forwards all operations to the object as long as the object still exists. If the object is deallocated, attempting to use a proxy will cause a -:exc:`weakref.ReferenceError` exception to be raised. :: +:exc:`!weakref.ReferenceError` exception to be raised. :: proxy = weakref.proxy(obj) proxy.attr # Equivalent to obj.attr @@ -424,7 +424,8 @@ PEP 232: Function Attributes In Python 2.1, functions can now have arbitrary information attached to them. People were often using docstrings to hold information about functions and -methods, because the ``__doc__`` attribute was the only way of attaching any +methods, because the :attr:`~function.__doc__` attribute was the only way of +attaching any information to a function. For example, in the Zope web application server, functions are marked as safe for public access by having a docstring, and in John Aycock's SPARK parsing framework, docstrings hold parts of the BNF grammar @@ -445,7 +446,7 @@ The dictionary containing attributes can be accessed as the function's :attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be -tricky and set it to a :class:`UserDict` instance, or any other random object +tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. @@ -583,11 +584,11 @@ available from the Distutils SIG at https://www.python.org/community/sigs/curren New and Improved Modules ======================== -* Ka-Ping Yee contributed two new modules: :mod:`inspect.py`, a module for - getting information about live Python code, and :mod:`pydoc.py`, a module for +* Ka-Ping Yee contributed two new modules: :mod:`!inspect.py`, a module for + getting information about live Python code, and :mod:`!pydoc.py`, a module for interactively converting docstrings to HTML or text. As a bonus, :file:`Tools/scripts/pydoc`, which is now automatically installed, uses - :mod:`pydoc.py` to display documentation given a Python module, package, or + :mod:`!pydoc.py` to display documentation given a Python module, package, or class name. For example, ``pydoc xml.dom`` displays the following:: Python Library Documentation: package xml.dom in xml @@ -616,7 +617,7 @@ New and Improved Modules Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. -* The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which +* The :mod:`difflib` module contains a class, :class:`~difflib.SequenceMatcher`, which compares two sequences and computes the changes required to transform one sequence into the other. For example, this module can be used to write a tool similar to the Unix :program:`diff` program, and in fact the sample program @@ -632,7 +633,7 @@ New and Improved Modules 2.1 includes an updated version of the :mod:`xml` package. Some of the noteworthy changes include support for Expat 1.2 and later versions, the ability for Expat parsers to handle files in any encoding supported by Python, and - various bugfixes for SAX, DOM, and the :mod:`minidom` module. + various bugfixes for SAX, DOM, and the :mod:`!minidom` module. * Ping also contributed another hook for handling uncaught exceptions. :func:`sys.excepthook` can be set to a callable object. When an exception isn't @@ -642,8 +643,8 @@ New and Improved Modules printing an extended traceback that not only lists the stack frames, but also lists the function arguments and the local variables for each frame. -* Various functions in the :mod:`time` module, such as :func:`asctime` and - :func:`localtime`, require a floating point argument containing the time in +* Various functions in the :mod:`time` module, such as :func:`~time.asctime` and + :func:`~time.localtime`, require a floating point argument containing the time in seconds since the epoch. The most common use of these functions is to work with the current time, so the floating point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file @@ -723,10 +724,10 @@ of the more notable changes are: a discussion in comp.lang.python. A new module and method for file objects was also added, contributed by Jeff - Epler. The new method, :meth:`xreadlines`, is similar to the existing - :func:`xrange` built-in. :func:`xreadlines` returns an opaque sequence object + Epler. The new method, :meth:`!xreadlines`, is similar to the existing + :func:`!xrange` built-in. :func:`!xreadlines` returns an opaque sequence object that only supports being iterated over, reading a line on every iteration but - not reading the entire file into memory as the existing :meth:`readlines` method + not reading the entire file into memory as the existing :meth:`!readlines` method does. You'd use it like this:: for line in sys.stdin.xreadlines(): @@ -736,7 +737,7 @@ of the more notable changes are: For a fuller discussion of the line I/O changes, see the python-dev summary for January 1--15, 2001 at https://mail.python.org/pipermail/python-dev/2001-January/. -* A new method, :meth:`popitem`, was added to dictionaries to enable +* A new method, :meth:`~dict.popitem`, was added to dictionaries to enable destructively iterating through the contents of a dictionary; this can be faster for large dictionaries because there's no need to construct a list containing all the keys or values. ``D.popitem()`` removes a random ``(key, value)`` pair diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index d9ead57413cbbf..d4dbe0570fbda5 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -53,9 +53,9 @@ A long time ago I wrote a web page listing flaws in Python's design. One of the most significant flaws was that it's impossible to subclass Python types implemented in C. In particular, it's not possible to subclass built-in types, so you can't just subclass, say, lists in order to add a single useful method to -them. The :mod:`UserList` module provides a class that supports all of the +them. The :mod:`!UserList` module provides a class that supports all of the methods of lists and that can be subclassed further, but there's lots of C code -that expects a regular Python list and won't accept a :class:`UserList` +that expects a regular Python list and won't accept a :class:`~collections.UserList` instance. Python 2.2 fixes this, and in the process adds some exciting new capabilities. @@ -69,7 +69,7 @@ A brief summary: * It's also possible to automatically call methods on accessing or setting an instance attribute by using a new mechanism called :dfn:`properties`. Many uses - of :meth:`__getattr__` can be rewritten to use properties instead, making the + of :meth:`~object.__getattr__` can be rewritten to use properties instead, making the resulting code simpler and faster. As a small side benefit, attributes can now have docstrings, too. @@ -120,7 +120,7 @@ added so if no built-in type is suitable, you can just subclass This means that :keyword:`class` statements that don't have any base classes are always classic classes in Python 2.2. (Actually you can also change this by -setting a module-level variable named :attr:`__metaclass__` --- see :pep:`253` +setting a module-level variable named :attr:`!__metaclass__` --- see :pep:`253` for the details --- but it's easier to just subclass :class:`object`.) The type objects for the built-in types are available as built-ins, named using @@ -134,8 +134,8 @@ type objects that behave as factories when called. :: 123 To make the set of types complete, new type objects such as :func:`dict` and -:func:`file` have been added. Here's a more interesting example, adding a -:meth:`lock` method to file objects:: +:func:`!file` have been added. Here's a more interesting example, adding a +:meth:`!lock` method to file objects:: class LockableFile(file): def lock (self, operation, length=0, start=0, whence=0): @@ -143,10 +143,10 @@ To make the set of types complete, new type objects such as :func:`dict` and return fcntl.lockf(self.fileno(), operation, length, start, whence) -The now-obsolete :mod:`posixfile` module contained a class that emulated all of -a file object's methods and also added a :meth:`lock` method, but this class +The now-obsolete :mod:`!posixfile` module contained a class that emulated all of +a file object's methods and also added a :meth:`!lock` method, but this class couldn't be passed to internal functions that expected a built-in file, -something which is possible with our new :class:`LockableFile`. +something which is possible with our new :class:`!LockableFile`. Descriptors @@ -154,11 +154,11 @@ Descriptors In previous versions of Python, there was no consistent way to discover what attributes and methods were supported by an object. There were some informal -conventions, such as defining :attr:`__members__` and :attr:`__methods__` +conventions, such as defining :attr:`!__members__` and :attr:`!__methods__` attributes that were lists of names, but often the author of an extension type or a class wouldn't bother to define them. You could fall back on inspecting the :attr:`~object.__dict__` of an object, but when class inheritance or an arbitrary -:meth:`__getattr__` hook were in use this could still be inaccurate. +:meth:`!__getattr__` hook were in use this could still be inaccurate. The one big idea underlying the new class model is that an API for describing the attributes of an object using :dfn:`descriptors` has been formalized. @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`__doc__` is the attribute's docstring. +* :attr:`!__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,7 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`descriptor.__get__` returns a temporary object that's +For methods, :meth:`!descriptor.__get__` returns a temporary object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -204,7 +204,7 @@ methods are defined like this:: ... g = classmethod(g) -The :func:`staticmethod` function takes the function :func:`f`, and returns it +The :func:`staticmethod` function takes the function :func:`!f`, and returns it wrapped up in a descriptor so it can be stored in the class object. You might expect there to be special syntax for creating such methods (``def static f``, ``defstatic f()``, or something like that) but no such syntax has been defined @@ -232,10 +232,10 @@ like this:: f = eiffelmethod(f, pre_f, post_f) -Note that a person using the new :func:`eiffelmethod` doesn't have to understand +Note that a person using the new :func:`!eiffelmethod` doesn't have to understand anything about descriptors. This is why I think the new features don't increase the basic complexity of the language. There will be a few wizards who need to -know about it in order to write :func:`eiffelmethod` or the ZODB or whatever, +know about it in order to write :func:`!eiffelmethod` or the ZODB or whatever, but most users will just write code on top of the resulting libraries and ignore the implementation details. @@ -263,10 +263,10 @@ from :pep:`253` by Guido van Rossum):: The lookup rule for classic classes is simple but not very smart; the base classes are searched depth-first, going from left to right. A reference to -:meth:`D.save` will search the classes :class:`D`, :class:`B`, and then -:class:`A`, where :meth:`save` would be found and returned. :meth:`C.save` -would never be found at all. This is bad, because if :class:`C`'s :meth:`save` -method is saving some internal state specific to :class:`C`, not calling it will +:meth:`!D.save` will search the classes :class:`!D`, :class:`!B`, and then +:class:`!A`, where :meth:`!save` would be found and returned. :meth:`!C.save` +would never be found at all. This is bad, because if :class:`!C`'s :meth:`!save` +method is saving some internal state specific to :class:`!C`, not calling it will result in that state never getting saved. New-style classes follow a different algorithm that's a bit more complicated to @@ -276,22 +276,22 @@ produces more useful results for really complicated inheritance graphs.) #. List all the base classes, following the classic lookup rule and include a class multiple times if it's visited repeatedly. In the above example, the list - of visited classes is [:class:`D`, :class:`B`, :class:`A`, :class:`C`, - :class:`A`]. + of visited classes is [:class:`!D`, :class:`!B`, :class:`!A`, :class:`!C`, + :class:`!A`]. #. Scan the list for duplicated classes. If any are found, remove all but one occurrence, leaving the *last* one in the list. In the above example, the list - becomes [:class:`D`, :class:`B`, :class:`C`, :class:`A`] after dropping + becomes [:class:`!D`, :class:`!B`, :class:`!C`, :class:`!A`] after dropping duplicates. -Following this rule, referring to :meth:`D.save` will return :meth:`C.save`, +Following this rule, referring to :meth:`!D.save` will return :meth:`!C.save`, which is the behaviour we're after. This lookup rule is the same as the one followed by Common Lisp. A new built-in function, :func:`super`, provides a way to get at a class's superclasses without having to reimplement Python's algorithm. The most commonly used form will be ``super(class, obj)``, which returns a bound superclass object (not the actual class object). This form will be used in methods to call a method in the superclass; for example, -:class:`D`'s :meth:`save` method would look like this:: +:class:`!D`'s :meth:`!save` method would look like this:: class D (B,C): def save (self): @@ -309,7 +309,7 @@ Attribute Access ---------------- A fair number of sophisticated Python classes define hooks for attribute access -using :meth:`__getattr__`; most commonly this is done for convenience, to make +using :meth:`~object.__getattr__`; most commonly this is done for convenience, to make code more readable by automatically mapping an attribute access such as ``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds some new ways of controlling attribute access. @@ -321,22 +321,22 @@ instance's dictionary. New-style classes also support a new method, ``__getattribute__(attr_name)``. The difference between the two methods is -that :meth:`__getattribute__` is *always* called whenever any attribute is -accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't +that :meth:`~object.__getattribute__` is *always* called whenever any attribute is +accessed, while the old :meth:`~object.__getattr__` is only called if ``foo`` isn't found in the instance's dictionary. However, Python 2.2's support for :dfn:`properties` will often be a simpler way -to trap attribute references. Writing a :meth:`__getattr__` method is +to trap attribute references. Writing a :meth:`!__getattr__` method is complicated because to avoid recursion you can't use regular attribute accesses inside them, and instead have to mess around with the contents of -:attr:`~object.__dict__`. :meth:`__getattr__` methods also end up being called by Python -when it checks for other methods such as :meth:`__repr__` or :meth:`__coerce__`, +:attr:`~object.__dict__`. :meth:`~object.__getattr__` methods also end up being called by Python +when it checks for other methods such as :meth:`~object.__repr__` or :meth:`!__coerce__`, and so have to be written with this in mind. Finally, calling a function on every attribute access results in a sizable performance loss. :class:`property` is a new built-in type that packages up three functions that get, set, or delete an attribute, and a docstring. For example, if you want to -define a :attr:`size` attribute that's computed, but also settable, you could +define a :attr:`!size` attribute that's computed, but also settable, you could write:: class C(object): @@ -355,9 +355,9 @@ write:: "Storage size of this instance") That is certainly clearer and easier to write than a pair of -:meth:`__getattr__`/:meth:`__setattr__` methods that check for the :attr:`size` +:meth:`!__getattr__`/:meth:`!__setattr__` methods that check for the :attr:`!size` attribute and handle it specially while retrieving all other attributes from the -instance's :attr:`~object.__dict__`. Accesses to :attr:`size` are also the only ones +instance's :attr:`~object.__dict__`. Accesses to :attr:`!size` are also the only ones which have to perform the work of calling a function, so references to other attributes run at their usual speed. @@ -424,22 +424,22 @@ Another significant addition to 2.2 is an iteration interface at both the C and Python levels. Objects can define how they can be looped over by callers. In Python versions up to 2.1, the usual way to make ``for item in obj`` work is -to define a :meth:`__getitem__` method that looks something like this:: +to define a :meth:`~object.__getitem__` method that looks something like this:: def __getitem__(self, index): return <next item> -:meth:`__getitem__` is more properly used to define an indexing operation on an +:meth:`~object.__getitem__` is more properly used to define an indexing operation on an object so that you can write ``obj[5]`` to retrieve the sixth element. It's a bit misleading when you're using this only to support :keyword:`for` loops. Consider some file-like object that wants to be looped over; the *index* parameter is essentially meaningless, as the class probably assumes that a -series of :meth:`__getitem__` calls will be made with *index* incrementing by -one each time. In other words, the presence of the :meth:`__getitem__` method +series of :meth:`~object.__getitem__` calls will be made with *index* incrementing by +one each time. In other words, the presence of the :meth:`~object.__getitem__` method doesn't mean that using ``file[5]`` to randomly access the sixth element will work, though it really should. -In Python 2.2, iteration can be implemented separately, and :meth:`__getitem__` +In Python 2.2, iteration can be implemented separately, and :meth:`~object.__getitem__` methods can be limited to classes that really do support random access. The basic idea of iterators is simple. A new built-in function, ``iter(obj)`` or ``iter(C, sentinel)``, is used to get an iterator. ``iter(obj)`` returns @@ -447,7 +447,7 @@ an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an iterator that will invoke the callable object *C* until it returns *sentinel* to signal that the iterator is done. -Python classes can define an :meth:`__iter__` method, which should create and +Python classes can define an :meth:`!__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so +sequences that don't implement :meth:`!__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: @@ -510,8 +510,8 @@ Iterator support has been added to some of Python's basic types. Calling Oct 10 That's just the default behaviour. If you want to iterate over keys, values, or -key/value pairs, you can explicitly call the :meth:`iterkeys`, -:meth:`itervalues`, or :meth:`iteritems` methods to get an appropriate iterator. +key/value pairs, you can explicitly call the :meth:`!iterkeys`, +:meth:`!itervalues`, or :meth:`!iteritems` methods to get an appropriate iterator. In a minor related change, the :keyword:`in` operator now works on dictionaries, so ``key in dict`` is now equivalent to ``dict.has_key(key)``. @@ -580,7 +580,7 @@ allowed inside the :keyword:`!try` block of a :keyword:`try`...\ :keyword:`finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -641,7 +641,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -671,7 +671,7 @@ PEP 237: Unifying Long Integers and Integers In recent versions, the distinction between regular integers, which are 32-bit values on most machines, and long integers, which can be of arbitrary size, was becoming an annoyance. For example, on platforms that support files larger than -``2**32`` bytes, the :meth:`tell` method of file objects has to return a long +``2**32`` bytes, the :meth:`!tell` method of file objects has to return a long integer. However, there were various bits of Python that expected plain integers and would raise an error if a long integer was provided instead. For example, in Python 1.5, only regular integers could be used as a slice index, and @@ -752,7 +752,7 @@ Here are the changes 2.2 introduces: 0.5. Without the ``__future__`` statement, ``/`` still means classic division. The default meaning of ``/`` will not change until Python 3.0. -* Classes can define methods called :meth:`__truediv__` and :meth:`__floordiv__` +* Classes can define methods called :meth:`~object.__truediv__` and :meth:`~object.__floordiv__` to overload the two division operators. At the C level, there are also slots in the :c:type:`PyNumberMethods` structure so extension types can define the two operators. @@ -785,17 +785,17 @@ support.) When built to use UCS-4 (a "wide Python"), the interpreter can natively handle Unicode characters from U+000000 to U+110000, so the range of legal values for -the :func:`unichr` function is expanded accordingly. Using an interpreter +the :func:`!unichr` function is expanded accordingly. Using an interpreter compiled to use UCS-2 (a "narrow Python"), values greater than 65535 will still -cause :func:`unichr` to raise a :exc:`ValueError` exception. This is all +cause :func:`!unichr` to raise a :exc:`ValueError` exception. This is all described in :pep:`261`, "Support for 'wide' Unicode characters"; consult it for further details. Another change is simpler to explain. Since their introduction, Unicode strings -have supported an :meth:`encode` method to convert the string to a selected +have supported an :meth:`!encode` method to convert the string to a selected encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])`` method has been added to 8-bit strings (though not to Unicode strings) in 2.2. -:meth:`decode` assumes that the string is in the specified encoding and decodes +:meth:`!decode` assumes that the string is in the specified encoding and decodes it, returning whatever is returned by the codec. Using this new feature, codecs have been added for tasks not directly related to @@ -819,10 +819,10 @@ encoding, and compression with the :mod:`zlib` module:: >>> "sheesh".encode('rot-13') 'furrfu' -To convert a class instance to Unicode, a :meth:`__unicode__` method can be -defined by a class, analogous to :meth:`__str__`. +To convert a class instance to Unicode, a :meth:`!__unicode__` method can be +defined by a class, analogous to :meth:`!__str__`. -:meth:`encode`, :meth:`decode`, and :meth:`__unicode__` were implemented by +:meth:`!encode`, :meth:`!decode`, and :meth:`!__unicode__` were implemented by Marc-André Lemburg. The changes to support using UCS-4 internally were implemented by Fredrik Lundh and Martin von Löwis. @@ -859,7 +859,7 @@ doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -915,7 +915,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -933,7 +933,7 @@ anyway). New and Improved Modules ======================== -* The :mod:`xmlrpclib` module was contributed to the standard library by Fredrik +* The :mod:`xmlrpclib <xmlrpc.client>` module was contributed to the standard library by Fredrik Lundh, providing support for writing XML-RPC clients. XML-RPC is a simple remote procedure call protocol built on top of HTTP and XML. For example, the following snippet retrieves a list of RSS channels from the O'Reilly Network, @@ -956,7 +956,7 @@ New and Improved Modules # 'description': 'A utility which converts HTML to XSL FO.', # 'title': 'html2fo 0.3 (Default)'}, ... ] - The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward + The :mod:`SimpleXMLRPCServer <xmlrpc.server>` module makes it easy to create straightforward XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC. * The new :mod:`hmac` module implements the HMAC algorithm described by @@ -964,9 +964,9 @@ New and Improved Modules * Several functions that originally returned lengthy tuples now return pseudo-sequences that still behave like tuples but also have mnemonic attributes such - as memberst_mtime or :attr:`tm_year`. The enhanced functions include - :func:`stat`, :func:`fstat`, :func:`statvfs`, and :func:`fstatvfs` in the - :mod:`os` module, and :func:`localtime`, :func:`gmtime`, and :func:`strptime` in + as :attr:`!memberst_mtime` or :attr:`~time.struct_time.tm_year`. The enhanced functions include + :func:`~os.stat`, :func:`~os.fstat`, :func:`~os.statvfs`, and :func:`~os.fstatvfs` in the + :mod:`os` module, and :func:`~time.localtime`, :func:`~time.gmtime`, and :func:`~time.strptime` in the :mod:`time` module. For example, to obtain a file's size using the old tuples, you'd end up writing @@ -999,7 +999,7 @@ New and Improved Modules underlying the :mod:`re` module. For example, the :func:`re.sub` and :func:`re.split` functions have been rewritten in C. Another contributed patch speeds up certain Unicode character ranges by a factor of two, and a new - :meth:`finditer` method that returns an iterator over all the non-overlapping + :meth:`~re.finditer` method that returns an iterator over all the non-overlapping matches in a given string. (SRE is maintained by Fredrik Lundh. The BIGCHARSET patch was contributed by Martin von Löwis.) @@ -1012,33 +1012,33 @@ New and Improved Modules new extensions: the NAMESPACE extension defined in :rfc:`2342`, SORT, GETACL and SETACL. (Contributed by Anthony Baxter and Michel Pelletier.) -* The :mod:`rfc822` module's parsing of email addresses is now compliant with +* The :mod:`!rfc822` module's parsing of email addresses is now compliant with :rfc:`2822`, an update to :rfc:`822`. (The module's name is *not* going to be changed to ``rfc2822``.) A new package, :mod:`email`, has also been added for parsing and generating e-mail messages. (Contributed by Barry Warsaw, and arising out of his work on Mailman.) -* The :mod:`difflib` module now contains a new :class:`Differ` class for +* The :mod:`difflib` module now contains a new :class:`!Differ` class for producing human-readable lists of changes (a "delta") between two sequences of - lines of text. There are also two generator functions, :func:`ndiff` and - :func:`restore`, which respectively return a delta from two sequences, or one of + lines of text. There are also two generator functions, :func:`!ndiff` and + :func:`!restore`, which respectively return a delta from two sequences, or one of the original sequences from a delta. (Grunt work contributed by David Goodger, from ndiff.py code by Tim Peters who then did the generatorization.) -* New constants :const:`ascii_letters`, :const:`ascii_lowercase`, and - :const:`ascii_uppercase` were added to the :mod:`string` module. There were - several modules in the standard library that used :const:`string.letters` to +* New constants :const:`!ascii_letters`, :const:`!ascii_lowercase`, and + :const:`!ascii_uppercase` were added to the :mod:`string` module. There were + several modules in the standard library that used :const:`!string.letters` to mean the ranges A-Za-z, but that assumption is incorrect when locales are in - use, because :const:`string.letters` varies depending on the set of legal + use, because :const:`!string.letters` varies depending on the set of legal characters defined by the current locale. The buggy modules have all been fixed - to use :const:`ascii_letters` instead. (Reported by an unknown person; fixed by + to use :const:`!ascii_letters` instead. (Reported by an unknown person; fixed by Fred L. Drake, Jr.) * The :mod:`mimetypes` module now makes it easier to use alternative MIME-type - databases by the addition of a :class:`MimeTypes` class, which takes a list of + databases by the addition of a :class:`~mimetypes.MimeTypes` class, which takes a list of filenames to be parsed. (Contributed by Fred L. Drake, Jr.) -* A :class:`Timer` class was added to the :mod:`threading` module that allows +* A :class:`~threading.Timer` class was added to the :mod:`threading` module that allows scheduling an activity to happen at some future time. (Contributed by Itamar Shtull-Trauring.) @@ -1062,7 +1062,7 @@ code, none of the changes described here will affect you very much. simply been changed to use the new C-level interface. (Contributed by Fred L. Drake, Jr.) -* Another low-level API, primarily of interest to implementors of Python +* Another low-level API, primarily of interest to implementers of Python debuggers and development tools, was added. :c:func:`PyInterpreterState_Head` and :c:func:`PyInterpreterState_Next` let a caller walk through all the existing interpreter objects; :c:func:`PyInterpreterState_ThreadHead` and @@ -1114,7 +1114,7 @@ code, none of the changes described here will affect you very much. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new :c:func:`snprintf` and :c:func:`vsnprintf` C lib APIs. In contrast to the standard - :c:func:`sprintf` and :c:func:`vsprintf` functions, the Python versions check the + :c:func:`sprintf` and :c:func:`!vsprintf` functions, the Python versions check the bounds of the buffer used to protect against buffer overruns. (Contributed by M.-A. Lemburg.) @@ -1212,12 +1212,12 @@ Some of the more notable changes are: * The :file:`Tools/scripts/ftpmirror.py` script now parses a :file:`.netrc` file, if you have one. (Contributed by Mike Romberg.) -* Some features of the object returned by the :func:`xrange` function are now +* Some features of the object returned by the :func:`!xrange` function are now deprecated, and trigger warnings when they're accessed; they'll disappear in - Python 2.3. :class:`xrange` objects tried to pretend they were full sequence + Python 2.3. :class:`!xrange` objects tried to pretend they were full sequence types by supporting slicing, sequence multiplication, and the :keyword:`in` operator, but these features were rarely used and therefore buggy. The - :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` + :meth:`!tolist` method and the :attr:`!start`, :attr:`!stop`, and :attr:`!step` attributes are also being deprecated. At the C level, the fourth argument to the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index ec1ca417ee139d..8adf36e316c6fb 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -40,10 +40,10 @@ new feature. PEP 218: A Standard Set Datatype ================================ -The new :mod:`sets` module contains an implementation of a set datatype. The +The new :mod:`!sets` module contains an implementation of a set datatype. The :class:`Set` class is for mutable sets, sets that can have members added and -removed. The :class:`ImmutableSet` class is for sets that can't be modified, -and instances of :class:`ImmutableSet` can therefore be used as dictionary keys. +removed. The :class:`!ImmutableSet` class is for sets that can't be modified, +and instances of :class:`!ImmutableSet` can therefore be used as dictionary keys. Sets are built on top of dictionaries, so the elements within a set must be hashable. @@ -63,10 +63,10 @@ Here's a simple example:: Set([1, 2, 5]) >>> -The union and intersection of sets can be computed with the :meth:`union` and -:meth:`intersection` methods; an alternative notation uses the bitwise operators +The union and intersection of sets can be computed with the :meth:`~frozenset.union` and +:meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`union_update` and :meth:`intersection_update`. :: +:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -97,7 +97,7 @@ in-place version with the ungainly name :meth:`symmetric_difference_update`. :: Set([1, 2, 5, 6]) >>> -There are also :meth:`issubset` and :meth:`issuperset` methods for checking +There are also :meth:`!issubset` and :meth:`!issuperset` methods for checking whether one set is a subset or superset of another:: >>> S1 = sets.Set([1,2,3]) @@ -166,7 +166,7 @@ statement isn't allowed inside the :keyword:`try` block of a :keyword:`!try`...\ :keyword:`!finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -227,7 +227,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -345,7 +345,7 @@ Python now allows using arbitrary Unicode strings (within the limitations of the file system) for all functions that expect file names, most notably the :func:`open` built-in function. If a Unicode string is passed to :func:`os.listdir`, Python now returns a list of Unicode strings. A new -function, :func:`os.getcwdu`, returns the current directory as a Unicode string. +function, :func:`!os.getcwdu`, returns the current directory as a Unicode string. Byte strings still work as file names, and on Windows Python will transparently convert them to Unicode using the ``mbcs`` encoding. @@ -386,10 +386,10 @@ one followed by the platform on which Python is running. Opening a file with the mode ``'U'`` or ``'rU'`` will open a file for reading in :term:`universal newlines` mode. All three line ending conventions will be translated to a ``'\n'`` in the strings returned by the various file methods such as -:meth:`read` and :meth:`readline`. +:meth:`!read` and :meth:`!readline`. Universal newline support is also used when importing modules and when executing -a file with the :func:`execfile` function. This means that Python modules can +a file with the :func:`!execfile` function. This means that Python modules can be shared between all three operating systems without needing to convert the line-endings. @@ -450,16 +450,16 @@ records to standard error or to a file or socket, send them to the system log, or even e-mail them to a particular address; of course, it's also possible to write your own handler classes. -The :class:`Logger` class is the primary class. Most application code will deal -with one or more :class:`Logger` objects, each one used by a particular -subsystem of the application. Each :class:`Logger` is identified by a name, and +The :class:`~logging.Logger` class is the primary class. Most application code will deal +with one or more :class:`~logging.Logger` objects, each one used by a particular +subsystem of the application. Each :class:`~logging.Logger` is identified by a name, and names are organized into a hierarchy using ``.`` as the component separator. -For example, you might have :class:`Logger` instances named ``server``, +For example, you might have :class:`~logging.Logger` instances named ``server``, ``server.auth`` and ``server.network``. The latter two instances are below ``server`` in the hierarchy. This means that if you turn up the verbosity for ``server`` or direct ``server`` messages to a different handler, the changes will also apply to records logged to ``server.auth`` and ``server.network``. -There's also a root :class:`Logger` that's the parent of all other loggers. +There's also a root :class:`~logging.Logger` that's the parent of all other loggers. For simple uses, the :mod:`logging` package contains some convenience functions that always use the root log:: @@ -480,14 +480,14 @@ This produces the following output:: In the default configuration, informational and debugging messages are suppressed and the output is sent to standard error. You can enable the display -of informational and debugging messages by calling the :meth:`setLevel` method +of informational and debugging messages by calling the :meth:`~logging.Logger.setLevel` method on the root logger. -Notice the :func:`warning` call's use of string formatting operators; all of the +Notice the :func:`~logging.warning` call's use of string formatting operators; all of the functions for logging messages take the arguments ``(msg, arg1, arg2, ...)`` and log the string resulting from ``msg % (arg1, arg2, ...)``. -There's also an :func:`exception` function that records the most recent +There's also an :func:`~logging.exception` function that records the most recent traceback. Any of the other functions will also record the traceback if you specify a true value for the keyword argument *exc_info*. :: @@ -517,16 +517,16 @@ it if it doesn't exist yet. ``getLogger(None)`` returns the root logger. :: ... Log records are usually propagated up the hierarchy, so a message logged to -``server.auth`` is also seen by ``server`` and ``root``, but a :class:`Logger` -can prevent this by setting its :attr:`propagate` attribute to :const:`False`. +``server.auth`` is also seen by ``server`` and ``root``, but a :class:`~logging.Logger` +can prevent this by setting its :attr:`~logging.Logger.propagate` attribute to :const:`False`. There are more classes provided by the :mod:`logging` package that can be -customized. When a :class:`Logger` instance is told to log a message, it -creates a :class:`LogRecord` instance that is sent to any number of different -:class:`Handler` instances. Loggers and handlers can also have an attached list -of filters, and each filter can cause the :class:`LogRecord` to be ignored or +customized. When a :class:`~logging.Logger` instance is told to log a message, it +creates a :class:`~logging.LogRecord` instance that is sent to any number of different +:class:`~logging.Handler` instances. Loggers and handlers can also have an attached list +of filters, and each filter can cause the :class:`~logging.LogRecord` to be ignored or can modify the record before passing it along. When they're finally output, -:class:`LogRecord` instances are converted to text by a :class:`Formatter` +:class:`~logging.LogRecord` instances are converted to text by a :class:`~logging.Formatter` class. All of these classes can be replaced by your own specially written classes. @@ -550,7 +550,7 @@ PEP 285: A Boolean Type ======================= A Boolean type was added to Python 2.3. Two new constants were added to the -:mod:`__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and +:mod:`!__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and :const:`False` constants were added to the built-ins in Python 2.2.1, but the 2.2.1 versions are simply set to integer values of 1 and 0 and aren't a different type.) @@ -662,7 +662,7 @@ a central catalog server. The resulting catalog is available from https://pypi.org. To make the catalog a bit more useful, a new optional *classifiers* keyword -argument has been added to the Distutils :func:`setup` function. A list of +argument has been added to the Distutils :func:`!setup` function. A list of `Trove <http://catb.org/~esr/trove/>`_-style strings can be supplied to help classify the software. @@ -703,14 +703,14 @@ PEP 302: New Import Hooks ========================= While it's been possible to write custom import hooks ever since the -:mod:`ihooks` module was introduced in Python 1.3, no one has ever been really +:mod:`!ihooks` module was introduced in Python 1.3, no one has ever been really happy with it because writing new import hooks is difficult and messy. There -have been various proposed alternatives such as the :mod:`imputil` and :mod:`iu` +have been various proposed alternatives such as the :mod:`!imputil` and :mod:`!iu` modules, but none of them has ever gained much acceptance, and none of them were easily usable from C code. :pep:`302` borrows ideas from its predecessors, especially from Gordon -McMillan's :mod:`iu` module. Three new items are added to the :mod:`sys` +McMillan's :mod:`!iu` module. Three new items are added to the :mod:`sys` module: * ``sys.path_hooks`` is a list of callable objects; most often they'll be @@ -790,7 +790,7 @@ package is much simpler:: for line in reader: print line -The :func:`reader` function takes a number of different options. The field +The :func:`~csv.reader` function takes a number of different options. The field separator isn't limited to the comma and can be changed to any character, and so can the quoting and line-ending characters. @@ -814,7 +814,7 @@ of tuples or lists, quoting strings that contain the delimiter. PEP 307: Pickle Enhancements ============================ -The :mod:`pickle` and :mod:`cPickle` modules received some attention during the +The :mod:`pickle` and :mod:`!cPickle` modules received some attention during the 2.3 development cycle. In 2.2, new-style classes could be pickled without difficulty, but they weren't pickled very compactly; :pep:`307` quotes a trivial example where a new-style class results in a pickled string three times longer @@ -829,13 +829,13 @@ fanciest protocol available. Unpickling is no longer considered a safe operation. 2.2's :mod:`pickle` provided hooks for trying to prevent unsafe classes from being unpickled -(specifically, a :attr:`__safe_for_unpickling__` attribute), but none of this +(specifically, a :attr:`!__safe_for_unpickling__` attribute), but none of this code was ever audited and therefore it's all been ripped out in 2.3. You should not unpickle untrusted data in any version of Python. To reduce the pickling overhead for new-style classes, a new interface for customizing pickling was added using three special methods: -:meth:`__getstate__`, :meth:`__setstate__`, and :meth:`__getnewargs__`. Consult +:meth:`~object.__getstate__`, :meth:`~object.__setstate__`, and :meth:`~object.__getnewargs__`. Consult :pep:`307` for the full semantics of these methods. As a way to compress pickles yet further, it's now possible to use integer codes @@ -925,7 +925,7 @@ Deletion is more straightforward:: >>> a [1, 3] -One can also now pass slice objects to the :meth:`__getitem__` methods of the +One can also now pass slice objects to the :meth:`~object.__getitem__` methods of the built-in sequences:: >>> range(10).__getitem__(slice(0, 5, 2)) @@ -939,7 +939,7 @@ Or use slice objects directly in subscripts:: To simplify implementing sequences that support extended slicing, slice objects now have a method ``indices(length)`` which, given the length of a sequence, returns a ``(start, stop, step)`` tuple that can be passed directly to -:func:`range`. :meth:`indices` handles omitted and out-of-bounds indices in a +:func:`range`. :meth:`!indices` handles omitted and out-of-bounds indices in a manner consistent with regular slices (and this innocuous phrase hides a welter of confusing details!). The method is intended to be used like this:: @@ -1042,7 +1042,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. execute any assertions. * Most type objects are now callable, so you can use them to create new objects - such as functions, classes, and modules. (This means that the :mod:`new` module + such as functions, classes, and modules. (This means that the :mod:`!new` module can be deprecated in a future Python version, because you can now use the type objects available in the :mod:`types` module.) For example, you can create a new module object with the following code: @@ -1069,11 +1069,11 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * Using ``None`` as a variable name will now result in a :exc:`SyntaxWarning` warning. In a future version of Python, ``None`` may finally become a keyword. -* The :meth:`xreadlines` method of file objects, introduced in Python 2.1, is no +* The :meth:`!xreadlines` method of file objects, introduced in Python 2.1, is no longer necessary because files now behave as their own iterator. - :meth:`xreadlines` was originally introduced as a faster way to loop over all + :meth:`!xreadlines` was originally introduced as a faster way to loop over all the lines in a file, but now you can simply write ``for line in file_obj``. - File objects also have a new read-only :attr:`encoding` attribute that gives the + File objects also have a new read-only :attr:`!encoding` attribute that gives the encoding used by the file; Unicode strings written to the file will be automatically converted to bytes using the given encoding. @@ -1084,7 +1084,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. C3 algorithm as described in the paper `"A Monotonic Superclass Linearization for Dylan" <https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.19.3910>`_. To understand the motivation for this change, read Michele Simionato's article - `"Python 2.3 Method Resolution Order" <http://www.phyast.pitt.edu/~micheles/mro.html>`_, or + :ref:`python_2.3_mro`, or read the thread on python-dev starting with the message at https://mail.python.org/pipermail/python-dev/2002-October/029035.html. Samuele Pedroni first pointed out the problem and also implemented the fix by coding the @@ -1096,12 +1096,12 @@ Here are all of the changes that Python 2.3 makes to the core Python language. switching overhead. Some multithreaded applications may suffer slower response time, but that's easily fixed by setting the limit back to a lower number using ``sys.setcheckinterval(N)``. The limit can be retrieved with the new - :func:`sys.getcheckinterval` function. + :func:`!sys.getcheckinterval` function. * One minor but far-reaching change is that the names of extension types defined by the modules included with Python now contain the module and a ``'.'`` in front of the type name. For example, in Python 2.2, if you created a socket and - printed its :attr:`__class__`, you'd get this output:: + printed its :attr:`!__class__`, you'd get this output:: >>> s = socket.socket() >>> s.__class__ @@ -1138,9 +1138,9 @@ String Changes True Note that this doesn't tell you where the substring starts; if you need that - information, use the :meth:`find` string method. + information, use the :meth:`~str.find` string method. -* The :meth:`strip`, :meth:`lstrip`, and :meth:`rstrip` string methods now have +* The :meth:`~str.strip`, :meth:`~str.lstrip`, and :meth:`~str.rstrip` string methods now have an optional argument for specifying the characters to strip. The default is still to remove all whitespace characters:: @@ -1156,13 +1156,13 @@ String Changes (Suggested by Simon Brunning and implemented by Walter Dörwald.) -* The :meth:`startswith` and :meth:`endswith` string methods now accept negative +* The :meth:`~str.startswith` and :meth:`~str.endswith` string methods now accept negative numbers for the *start* and *end* parameters. -* Another new string method is :meth:`zfill`, originally a function in the - :mod:`string` module. :meth:`zfill` pads a numeric string with zeros on the +* Another new string method is :meth:`~str.zfill`, originally a function in the + :mod:`string` module. :meth:`~str.zfill` pads a numeric string with zeros on the left until it's the specified width. Note that the ``%`` operator is still more - flexible and powerful than :meth:`zfill`. :: + flexible and powerful than :meth:`~str.zfill`. :: >>> '45'.zfill(4) '0045' @@ -1173,10 +1173,10 @@ String Changes (Contributed by Walter Dörwald.) -* A new type object, :class:`basestring`, has been added. Both 8-bit strings and +* A new type object, :class:`!basestring`, has been added. Both 8-bit strings and Unicode strings inherit from this type, so ``isinstance(obj, basestring)`` will return :const:`True` for either kind of string. It's a completely abstract - type, so you can't create :class:`basestring` instances. + type, so you can't create :class:`!basestring` instances. * Interned strings are no longer immortal and will now be garbage-collected in the usual way when the only reference to them is from the internal dictionary of @@ -1191,19 +1191,19 @@ Optimizations * The creation of new-style class instances has been made much faster; they're now faster than classic classes! -* The :meth:`sort` method of list objects has been extensively rewritten by Tim +* The :meth:`~list.sort` method of list objects has been extensively rewritten by Tim Peters, and the implementation is significantly faster. * Multiplication of large long integers is now much faster thanks to an implementation of Karatsuba multiplication, an algorithm that scales better than - the O(n\*n) required for the grade-school multiplication algorithm. (Original + the *O*\ (*n*\ :sup:`2`) required for the grade-school multiplication algorithm. (Original patch by Christopher A. Craig, and significantly reworked by Tim Peters.) * The ``SET_LINENO`` opcode is now gone. This may provide a small speed increase, depending on your compiler's idiosyncrasies. See section :ref:`23section-other` for a longer explanation. (Removed by Michael Hudson.) -* :func:`xrange` objects now have their own iterator, making ``for i in +* :func:`!xrange` objects now have their own iterator, making ``for i in xrange(n)`` slightly faster than ``for i in range(n)``. (Patch by Raymond Hettinger.) @@ -1230,21 +1230,21 @@ complete list of changes, or look through the CVS logs for all the details. operator to add another array's contents, and the ``*=`` assignment operator to repeat an array. (Contributed by Jason Orendorff.) -* The :mod:`bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB +* The :mod:`!bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB <https://pybsddb.sourceforge.net>`_ package, providing a more complete interface to the transactional features of the BerkeleyDB library. - The old version of the module has been renamed to :mod:`bsddb185` and is no + The old version of the module has been renamed to :mod:`!bsddb185` and is no longer built automatically; you'll have to edit :file:`Modules/Setup` to enable - it. Note that the new :mod:`bsddb` package is intended to be compatible with + it. Note that the new :mod:`!bsddb` package is intended to be compatible with the old module, so be sure to file bugs if you discover any incompatibilities. When upgrading to Python 2.3, if the new interpreter is compiled with a new version of the underlying BerkeleyDB library, you will almost certainly have to convert your database files to the new version. You can do this fairly easily with the new scripts :file:`db2pickle.py` and :file:`pickle2db.py` which you will find in the distribution's :file:`Tools/scripts` directory. If you've - already been using the PyBSDDB package and importing it as :mod:`bsddb3`, you - will have to change your ``import`` statements to import it as :mod:`bsddb`. + already been using the PyBSDDB package and importing it as :mod:`!bsddb3`, you + will have to change your ``import`` statements to import it as :mod:`!bsddb`. * The new :mod:`bz2` module is an interface to the bz2 data compression library. bz2-compressed data is usually smaller than corresponding @@ -1253,11 +1253,11 @@ complete list of changes, or look through the CVS logs for all the details. * A set of standard date/time types has been added in the new :mod:`datetime` module. See the following section for more details. -* The Distutils :class:`Extension` class now supports an extra constructor +* The Distutils :class:`!Extension` class now supports an extra constructor argument named *depends* for listing additional source files that an extension depends on. This lets Distutils recompile the module if any of the dependency files are modified. For example, if :file:`sampmodule.c` includes the header - file :file:`sample.h`, you would create the :class:`Extension` object like + file :file:`sample.h`, you would create the :class:`!Extension` object like this:: ext = Extension("samp", @@ -1268,21 +1268,21 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Jeremy Hylton.) * Other minor changes to Distutils: it now checks for the :envvar:`CC`, - :envvar:`CFLAGS`, :envvar:`CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` + :envvar:`CFLAGS`, :envvar:`!CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` environment variables, using them to override the settings in Python's configuration (contributed by Robert Weber). * Previously the :mod:`doctest` module would only search the docstrings of public methods and functions for test cases, but it now also examines private - ones as well. The :func:`DocTestSuite` function creates a + ones as well. The :func:`~doctest.DocTestSuite` function creates a :class:`unittest.TestSuite` object from a set of :mod:`doctest` tests. * The new ``gc.get_referents(object)`` function returns a list of all the objects referenced by *object*. -* The :mod:`getopt` module gained a new function, :func:`gnu_getopt`, that - supports the same arguments as the existing :func:`getopt` function but uses - GNU-style scanning mode. The existing :func:`getopt` stops processing options as +* The :mod:`getopt` module gained a new function, :func:`~getopt.gnu_getopt`, that + supports the same arguments as the existing :func:`~getopt.getopt` function but uses + GNU-style scanning mode. The existing :func:`~getopt.getopt` stops processing options as soon as a non-option argument is encountered, but in GNU-style mode processing continues, meaning that options and arguments can be mixed. For example:: @@ -1308,10 +1308,10 @@ complete list of changes, or look through the CVS logs for all the details. partially sorted order such that, for every index *k*, ``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]``. This makes it quick to remove the smallest item, and inserting a new item while maintaining the heap property is - O(lg n). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more + *O*\ (log *n*). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more information about the priority queue data structure.) - The :mod:`heapq` module provides :func:`heappush` and :func:`heappop` functions + The :mod:`heapq` module provides :func:`~heapq.heappush` and :func:`~heapq.heappop` functions for adding and removing items while maintaining the heap property on top of some other mutable Python sequence type. Here's an example that uses a Python list:: @@ -1343,7 +1343,7 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`itertools` contains a number of useful functions for use with iterators, inspired by various functions provided by the ML and Haskell languages. For example, ``itertools.ifilter(predicate, iterator)`` returns all - elements in the iterator for which the function :func:`predicate` returns + elements in the iterator for which the function :func:`!predicate` returns :const:`True`, and ``itertools.repeat(obj, N)`` returns ``obj`` *N* times. There are a number of other functions in the module; see the package's reference documentation for details. @@ -1356,21 +1356,21 @@ complete list of changes, or look through the CVS logs for all the details. was added to :func:`math.log` to make it easier to compute logarithms for bases other than ``e`` and ``10``. (Contributed by Raymond Hettinger.) -* Several new POSIX functions (:func:`getpgid`, :func:`killpg`, :func:`lchown`, - :func:`loadavg`, :func:`major`, :func:`makedev`, :func:`minor`, and - :func:`mknod`) were added to the :mod:`posix` module that underlies the +* Several new POSIX functions (:func:`!getpgid`, :func:`!killpg`, :func:`!lchown`, + :func:`!loadavg`, :func:`!major`, :func:`!makedev`, :func:`!minor`, and + :func:`!mknod`) were added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by Gustavo Niemeyer, Geert Jansen, and Denis S. Otkidach.) -* In the :mod:`os` module, the :func:`\*stat` family of functions can now report +* In the :mod:`os` module, the :func:`!\*stat` family of functions can now report fractions of a second in a timestamp. Such time stamps are represented as floats, similar to the value returned by :func:`time.time`. During testing, it was found that some applications will break if time stamps are floats. For compatibility, when using the tuple interface of the - :class:`stat_result` time stamps will be represented as integers. When using + :class:`~os.stat_result` time stamps will be represented as integers. When using named fields (a feature first introduced in Python 2.2), time stamps are still - represented as integers, unless :func:`os.stat_float_times` is invoked to enable + represented as integers, unless :func:`!os.stat_float_times` is invoked to enable float return values:: >>> os.stat("/tmp").st_mtime @@ -1391,7 +1391,7 @@ complete list of changes, or look through the CVS logs for all the details. automatically generate a usage message. See the following section for more details. -* The old and never-documented :mod:`linuxaudiodev` module has been deprecated, +* The old and never-documented :mod:`!linuxaudiodev` module has been deprecated, and a new version named :mod:`!ossaudiodev` has been added. The module was renamed because the OSS sound drivers can be used on platforms other than Linux, and the interface has also been tidied and brought up to date in various ways. @@ -1402,14 +1402,14 @@ complete list of changes, or look through the CVS logs for all the details. functions for getting the architecture, CPU type, the Windows OS version, and even the Linux distribution version. (Contributed by Marc-André Lemburg.) -* The parser objects provided by the :mod:`pyexpat` module can now optionally +* The parser objects provided by the :mod:`pyexpat <xml.parsers.expat>` module can now optionally buffer character data, resulting in fewer calls to your character data handler and therefore faster performance. Setting the parser object's - :attr:`buffer_text` attribute to :const:`True` will enable buffering. + :attr:`~xml.parsers.expat.xmlparser.buffer_text` attribute to :const:`True` will enable buffering. * The ``sample(population, k)`` function was added to the :mod:`random` - module. *population* is a sequence or :class:`xrange` object containing the - elements of a population, and :func:`sample` chooses *k* elements from the + module. *population* is a sequence or :class:`!xrange` object containing the + elements of a population, and :func:`~random.sample` chooses *k* elements from the population without replacing chosen elements. *k* can be any value up to ``len(population)``. For example:: @@ -1436,20 +1436,20 @@ complete list of changes, or look through the CVS logs for all the details. (All changes contributed by Raymond Hettinger.) * The :mod:`readline` module also gained a number of new functions: - :func:`get_history_item`, :func:`get_current_history_length`, and - :func:`redisplay`. + :func:`~readline.get_history_item`, :func:`~readline.get_current_history_length`, and + :func:`~readline.redisplay`. -* The :mod:`rexec` and :mod:`Bastion` modules have been declared dead, and +* The :mod:`!rexec` and :mod:`!Bastion` modules have been declared dead, and attempts to import them will fail with a :exc:`RuntimeError`. New-style classes provide new ways to break out of the restricted execution environment provided - by :mod:`rexec`, and no one has interest in fixing them or time to do so. If - you have applications using :mod:`rexec`, rewrite them to use something else. + by :mod:`!rexec`, and no one has interest in fixing them or time to do so. If + you have applications using :mod:`!rexec`, rewrite them to use something else. (Sticking with Python 2.2 or 2.1 will not make your applications any safer - because there are known bugs in the :mod:`rexec` module in those versions. To - repeat: if you're using :mod:`rexec`, stop using it immediately.) + because there are known bugs in the :mod:`!rexec` module in those versions. To + repeat: if you're using :mod:`!rexec`, stop using it immediately.) -* The :mod:`rotor` module has been deprecated because the algorithm it uses for +* The :mod:`!rotor` module has been deprecated because the algorithm it uses for encryption is not believed to be secure. If you need encryption, use one of the several AES Python modules that are available separately. @@ -1474,9 +1474,9 @@ complete list of changes, or look through the CVS logs for all the details. * On Windows, the :mod:`socket` module now ships with Secure Sockets Layer (SSL) support. -* The value of the C :c:macro:`PYTHON_API_VERSION` macro is now exposed at the +* The value of the C :c:macro:`!PYTHON_API_VERSION` macro is now exposed at the Python level as ``sys.api_version``. The current exception can be cleared by - calling the new :func:`sys.exc_clear` function. + calling the new :func:`!sys.exc_clear` function. * The new :mod:`tarfile` module allows reading from and writing to :program:`tar`\ -format archive files. (Contributed by Lars Gustäbel.) @@ -1486,7 +1486,7 @@ complete list of changes, or look through the CVS logs for all the details. string and returns a list containing the text split into lines of no more than the chosen width. The ``fill(text, width)`` function returns a single string, reformatted to fit into lines no longer than the chosen width. (As you - can guess, :func:`fill` is built on top of :func:`wrap`. For example:: + can guess, :func:`~textwrap.fill` is built on top of :func:`~textwrap.wrap`. For example:: >>> import textwrap >>> paragraph = "Not a whit, we defy augury: ... more text ..." @@ -1503,15 +1503,15 @@ complete list of changes, or look through the CVS logs for all the details. it will come: the readiness is all. >>> - The module also contains a :class:`TextWrapper` class that actually implements - the text wrapping strategy. Both the :class:`TextWrapper` class and the - :func:`wrap` and :func:`fill` functions support a number of additional keyword + The module also contains a :class:`~textwrap.TextWrapper` class that actually implements + the text wrapping strategy. Both the :class:`~textwrap.TextWrapper` class and the + :func:`~textwrap.wrap` and :func:`~textwrap.fill` functions support a number of additional keyword arguments for fine-tuning the formatting; consult the module's documentation for details. (Contributed by Greg Ward.) -* The :mod:`thread` and :mod:`threading` modules now have companion modules, - :mod:`dummy_thread` and :mod:`dummy_threading`, that provide a do-nothing - implementation of the :mod:`thread` module's interface for platforms where +* The :mod:`!thread` and :mod:`threading` modules now have companion modules, + :mod:`!dummy_thread` and :mod:`!dummy_threading`, that provide a do-nothing + implementation of the :mod:`!thread` module's interface for platforms where threads are not supported. The intention is to simplify thread-aware modules (ones that *don't* rely on threads to run) by putting the following code at the top:: @@ -1521,26 +1521,26 @@ complete list of changes, or look through the CVS logs for all the details. except ImportError: import dummy_threading as _threading - In this example, :mod:`_threading` is used as the module name to make it clear + In this example, :mod:`!_threading` is used as the module name to make it clear that the module being used is not necessarily the actual :mod:`threading` - module. Code can call functions and use classes in :mod:`_threading` whether or + module. Code can call functions and use classes in :mod:`!_threading` whether or not threads are supported, avoiding an :keyword:`if` statement and making the code slightly clearer. This module will not magically make multithreaded code run without threads; code that waits for another thread to return or to do something will simply hang forever. -* The :mod:`time` module's :func:`strptime` function has long been an annoyance - because it uses the platform C library's :func:`strptime` implementation, and +* The :mod:`time` module's :func:`~time.strptime` function has long been an annoyance + because it uses the platform C library's :func:`~time.strptime` implementation, and different platforms sometimes have odd bugs. Brett Cannon contributed a portable implementation that's written in pure Python and should behave identically on all platforms. * The new :mod:`timeit` module helps measure how long snippets of Python code take to execute. The :file:`timeit.py` file can be run directly from the - command line, or the module's :class:`Timer` class can be imported and used + command line, or the module's :class:`~timeit.Timer` class can be imported and used directly. Here's a short example that figures out whether it's faster to convert an 8-bit string to Unicode by appending an empty Unicode string to it or - by using the :func:`unicode` function:: + by using the :func:`!unicode` function:: import timeit @@ -1558,46 +1558,46 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`!Tix` module has received various bug fixes and updates for the current version of the Tix package. -* The :mod:`Tkinter` module now works with a thread-enabled version of Tcl. +* The :mod:`!Tkinter` module now works with a thread-enabled version of Tcl. Tcl's threading model requires that widgets only be accessed from the thread in which they're created; accesses from another thread can cause Tcl to panic. For - certain Tcl interfaces, :mod:`Tkinter` will now automatically avoid this when a + certain Tcl interfaces, :mod:`!Tkinter` will now automatically avoid this when a widget is accessed from a different thread by marshalling a command, passing it to the correct thread, and waiting for the results. Other interfaces can't be - handled automatically but :mod:`Tkinter` will now raise an exception on such an + handled automatically but :mod:`!Tkinter` will now raise an exception on such an access so that you can at least find out about the problem. See https://mail.python.org/pipermail/python-dev/2002-December/031107.html for a more detailed explanation of this change. (Implemented by Martin von Löwis.) -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. This behavior can be controlled through - the :meth:`wantobjects` method of :class:`tkapp` objects. + the :meth:`!wantobjects` method of :class:`!tkapp` objects. - When using :mod:`_tkinter` through the :mod:`Tkinter` module (as most Tkinter + When using :mod:`!_tkinter` through the :mod:`!Tkinter` module (as most Tkinter applications will), this feature is always activated. It should not cause compatibility problems, since Tkinter would always convert string results to Python types where possible. If any incompatibilities are found, the old behavior can be restored by setting - the :attr:`wantobjects` variable in the :mod:`Tkinter` module to false before - creating the first :class:`tkapp` object. :: + the :attr:`!wantobjects` variable in the :mod:`!Tkinter` module to false before + creating the first :class:`!tkapp` object. :: import Tkinter Tkinter.wantobjects = 0 Any breakage caused by this change should be reported as a bug. -* The :mod:`UserDict` module has a new :class:`DictMixin` class which defines +* The :mod:`!UserDict` module has a new :class:`!DictMixin` class which defines all dictionary methods for classes that already have a minimum mapping interface. This greatly simplifies writing classes that need to be substitutable for dictionaries, such as the classes in the :mod:`shelve` module. Adding the mix-in as a superclass provides the full dictionary interface - whenever the class defines :meth:`__getitem__`, :meth:`__setitem__`, - :meth:`__delitem__`, and :meth:`keys`. For example:: + whenever the class defines :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, + :meth:`~object.__delitem__`, and :meth:`!keys`. For example:: >>> import UserDict >>> class SeqDict(UserDict.DictMixin): @@ -1640,15 +1640,15 @@ complete list of changes, or look through the CVS logs for all the details. * The DOM implementation in :mod:`xml.dom.minidom` can now generate XML output in a particular encoding by providing an optional encoding argument to the - :meth:`toxml` and :meth:`toprettyxml` methods of DOM nodes. + :meth:`~xml.dom.minidom.Node.toxml` and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of DOM nodes. -* The :mod:`xmlrpclib` module now supports an XML-RPC extension for handling nil +* The :mod:`!xmlrpclib` module now supports an XML-RPC extension for handling nil data values such as Python's ``None``. Nil values are always supported on unmarshalling an XML-RPC response. To generate requests containing ``None``, you must supply a true value for the *allow_none* parameter when creating a - :class:`Marshaller` instance. + :class:`!Marshaller` instance. -* The new :mod:`DocXMLRPCServer` module allows writing self-documenting XML-RPC +* The new :mod:`!DocXMLRPCServer` module allows writing self-documenting XML-RPC servers. Run it in demo mode (as a program) to see it in action. Pointing the web browser to the RPC server produces pydoc-style documentation; pointing xmlrpclib to the server allows invoking the actual methods. (Contributed by @@ -1663,8 +1663,8 @@ complete list of changes, or look through the CVS logs for all the details. The :mod:`socket` module has also been extended to transparently convert Unicode hostnames to the ACE version before passing them to the C library. - Modules that deal with hostnames such as :mod:`httplib` and :mod:`ftplib`) - also support Unicode host names; :mod:`httplib` also sends HTTP ``Host`` + Modules that deal with hostnames such as :mod:`!httplib` and :mod:`ftplib`) + also support Unicode host names; :mod:`!httplib` also sends HTTP ``Host`` headers using the ACE version of the domain name. :mod:`urllib` supports Unicode URLs with non-ASCII host names as long as the ``path`` part of the URL is ASCII only. @@ -1682,17 +1682,17 @@ Date and time types suitable for expressing timestamps were added as the :mod:`datetime` module. The types don't support different calendars or many fancy features, and just stick to the basics of representing time. -The three primary types are: :class:`date`, representing a day, month, and year; +The three primary types are: :class:`~datetime.date`, representing a day, month, and year; :class:`~datetime.time`, consisting of hour, minute, and second; and :class:`~datetime.datetime`, -which contains all the attributes of both :class:`date` and :class:`~datetime.time`. -There's also a :class:`timedelta` class representing differences between two +which contains all the attributes of both :class:`~datetime.date` and :class:`~datetime.time`. +There's also a :class:`~datetime.timedelta` class representing differences between two points in time, and time zone logic is implemented by classes inheriting from -the abstract :class:`tzinfo` class. +the abstract :class:`~datetime.tzinfo` class. -You can create instances of :class:`date` and :class:`~datetime.time` by either supplying +You can create instances of :class:`~datetime.date` and :class:`~datetime.time` by either supplying keyword arguments to the appropriate constructor, e.g. ``datetime.date(year=1972, month=10, day=15)``, or by using one of a number of -class methods. For example, the :meth:`date.today` class method returns the +class methods. For example, the :meth:`~datetime.date.today` class method returns the current local date. Once created, instances of the date/time classes are all immutable. There are a @@ -1707,8 +1707,8 @@ number of methods for producing formatted strings from objects:: >>> now.strftime('%Y %d %b') '2002 30 Dec' -The :meth:`replace` method allows modifying one or more fields of a -:class:`date` or :class:`~datetime.datetime` instance, returning a new instance:: +The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a +:class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: >>> d = datetime.datetime.now() >>> d @@ -1718,10 +1718,10 @@ The :meth:`replace` method allows modifying one or more fields of a >>> Instances can be compared, hashed, and converted to strings (the result is the -same as that of :meth:`isoformat`). :class:`date` and :class:`~datetime.datetime` -instances can be subtracted from each other, and added to :class:`timedelta` +same as that of :meth:`~datetime.datetime.isoformat`). :class:`~datetime.date` and :class:`~datetime.datetime` +instances can be subtracted from each other, and added to :class:`~datetime.timedelta` instances. The largest missing feature is that there's no standard library -support for parsing strings and getting back a :class:`date` or +support for parsing strings and getting back a :class:`~datetime.date` or :class:`~datetime.datetime`. For more information, refer to the module's reference documentation. @@ -1739,7 +1739,7 @@ command-line parsing that follows the Unix conventions, automatically creates the output for :option:`!--help`, and can perform different actions for different options. -You start by creating an instance of :class:`OptionParser` and telling it what +You start by creating an instance of :class:`~optparse.OptionParser` and telling it what your program's options are. :: import sys @@ -1753,7 +1753,7 @@ your program's options are. :: action='store', type='int', dest='length', help='set maximum length of output') -Parsing a command line is then done by calling the :meth:`parse_args` method. :: +Parsing a command line is then done by calling the :meth:`~optparse.OptionParser.parse_args` method. :: options, args = op.parse_args(sys.argv[1:]) print options @@ -1925,7 +1925,7 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`__module__` and + aware of a change in the rules relating to the :attr:`!__module__` and :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired @@ -1940,7 +1940,7 @@ Port-Specific Changes Support for a port to IBM's OS/2 using the EMX runtime environment was merged into the main Python source tree. EMX is a POSIX emulation layer over the OS/2 system APIs. The Python port for EMX tries to support all the POSIX-like -capability exposed by the EMX runtime, and mostly succeeds; :func:`fork` and +capability exposed by the EMX runtime, and mostly succeeds; :func:`!fork` and :func:`fcntl` are restricted by the limitations of the underlying emulation layer. The standard OS/2 port, which uses IBM's Visual Age compiler, also gained support for case-sensitive import semantics as part of the integration of @@ -1998,13 +1998,13 @@ Some of the more notable changes are: It would be difficult to detect any resulting difference from Python code, apart from a slight speed up when Python is run without :option:`-O`. - C extensions that access the :attr:`f_lineno` field of frame objects should + C extensions that access the :attr:`~frame.f_lineno` field of frame objects should instead call ``PyCode_Addr2Line(f->f_code, f->f_lasti)``. This will have the added effect of making the code work as desired under "python -O" in earlier versions of Python. A nifty new feature is that trace functions can now assign to the - :attr:`f_lineno` attribute of frame objects, changing the line that will be + :attr:`~frame.f_lineno` attribute of frame objects, changing the line that will be executed next. A ``jump`` command has been added to the :mod:`pdb` debugger taking advantage of this new feature. (Implemented by Richie Hindle.) @@ -2031,9 +2031,9 @@ code: the file's encoding (UTF-8, Latin-1, or whatever) by adding a comment to the top of the file. See section :ref:`section-encodings` for more information. -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. * Large octal and hex literals such as ``0xffffffff`` now trigger a @@ -2049,10 +2049,10 @@ code: * You can no longer disable assertions by assigning to ``__debug__``. -* The Distutils :func:`setup` function has gained various new keyword arguments +* The Distutils :func:`!setup` function has gained various new keyword arguments such as *depends*. Old versions of the Distutils will abort if passed unknown keywords. A solution is to check for the presence of the new - :func:`get_distutil_options` function in your :file:`setup.py` and only uses the + :func:`!get_distutil_options` function in your :file:`setup.py` and only uses the new keywords with a version of the Distutils that supports them:: from distutils import core diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index cab321c3e54d18..7e235d4370edaa 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -324,7 +324,8 @@ function, as previously described. In other words, ``@A @B @C(args)`` becomes:: Getting this right can be slightly brain-bending, but it's not too difficult. -A small related change makes the :attr:`func_name` attribute of functions +A small related change makes the :attr:`func_name <function.__name__>` +attribute of functions writable. This attribute is used to display function names in tracebacks, so decorators should change the name of any new function that's constructed and returned. @@ -386,13 +387,13 @@ The standard library provides a number of ways to execute a subprocess, offering different features and different levels of complexity. ``os.system(command)`` is easy to use, but slow (it runs a shell process which executes the command) and dangerous (you have to be careful about escaping -the shell's metacharacters). The :mod:`popen2` module offers classes that can +the shell's metacharacters). The :mod:`!popen2` module offers classes that can capture standard output and standard error from the subprocess, but the naming is confusing. The :mod:`subprocess` module cleans this up, providing a unified interface that offers all the features you might need. -Instead of :mod:`popen2`'s collection of classes, :mod:`subprocess` contains a -single class called :class:`Popen` whose constructor supports a number of +Instead of :mod:`!popen2`'s collection of classes, :mod:`subprocess` contains a +single class called :class:`subprocess.Popen` whose constructor supports a number of different keyword arguments. :: class Popen(args, bufsize=0, executable=None, @@ -994,7 +995,7 @@ fixes. Here's a partial list of the most notable changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the CVS logs for all the details. -* The :mod:`asyncore` module's :func:`loop` function now has a *count* parameter +* The :mod:`!asyncore` module's :func:`!loop` function now has a *count* parameter that lets you perform a limited number of passes through the polling loop. The default is still to loop forever. @@ -1051,9 +1052,9 @@ complete list of changes, or look through the CVS logs for all the details. advantage of :class:`collections.deque` for improved performance. (Contributed by Raymond Hettinger.) -* The :mod:`ConfigParser` classes have been enhanced slightly. The :meth:`read` +* The :mod:`ConfigParser <configparser>` classes have been enhanced slightly. The :meth:`~configparser.ConfigParser.read` method now returns a list of the files that were successfully parsed, and the - :meth:`set` method raises :exc:`TypeError` if passed a *value* argument that + :meth:`~configparser.ConfigParser.set` method raises :exc:`TypeError` if passed a *value* argument that isn't a string. (Contributed by John Belmonte and David Goodger.) * The :mod:`curses` module now supports the ncurses extension @@ -1080,7 +1081,7 @@ complete list of changes, or look through the CVS logs for all the details. :func:`nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) -* The :mod:`httplib` module now contains constants for HTTP status codes defined +* The :mod:`httplib <http>` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by @@ -1163,7 +1164,7 @@ complete list of changes, or look through the CVS logs for all the details. * A number of functions were added to the :mod:`locale` module, such as :func:`bind_textdomain_codeset` to specify a particular encoding and a family of - :func:`l\*gettext` functions that return messages in the chosen encoding. + :func:`!l\*gettext` functions that return messages in the chosen encoding. (Contributed by Gustavo Niemeyer.) * Some keyword arguments were added to the :mod:`logging` package's @@ -1217,10 +1218,10 @@ complete list of changes, or look through the CVS logs for all the details. now include the string ``'%default'``, which will be replaced by the option's default value. (Contributed by Greg Ward.) -* The long-term plan is to deprecate the :mod:`rfc822` module in some future +* The long-term plan is to deprecate the :mod:`!rfc822` module in some future Python release in favor of the :mod:`email` package. To this end, the - :func:`email.Utils.formatdate` function has been changed to make it usable as a - replacement for :func:`rfc822.formatdate`. You may want to write new e-mail + :func:`email.Utils.formatdate <email.utils.formatdate>` function has been changed to make it usable as a + replacement for :func:`!rfc822.formatdate`. You may want to write new e-mail processing code with this in mind. (Change implemented by Anthony Baxter.) * A new ``urandom(n)`` function was added to the :mod:`os` module, returning @@ -1307,7 +1308,7 @@ complete list of changes, or look through the CVS logs for all the details. sockets, and regular expression pattern objects. (Contributed by Raymond Hettinger.) -* The :mod:`xmlrpclib` module now supports a multi-call extension for +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports a multi-call extension for transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) @@ -1322,8 +1323,8 @@ complete list of changes, or look through the CVS logs for all the details. cookielib --------- -The :mod:`cookielib` library supports client-side handling for HTTP cookies, -mirroring the :mod:`Cookie` module's server-side cookie support. Cookies are +The :mod:`cookielib <http.cookiejar>` library supports client-side handling for HTTP cookies, +mirroring the :mod:`Cookie <http.cookies>` module's server-side cookie support. Cookies are stored in cookie jars; the library transparently stores cookies offered by the web server in the cookie jar, and fetches the cookie from the jar when connecting to the server. As in web browsers, policy objects control whether @@ -1334,7 +1335,7 @@ are provided: one that stores cookies in the Netscape format so applications can use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. -:mod:`urllib2` has been changed to interact with :mod:`cookielib`: +:mod:`urllib2 <urllib.request>` has been changed to interact with :mod:`cookielib <http.cookiejar>`: :class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. @@ -1528,7 +1529,7 @@ code: will now always be unequal, and relative comparisons (``<``, ``>``) will raise a :exc:`TypeError`. -* :func:`dircache.listdir` now passes exceptions to the caller instead of +* :func:`!dircache.listdir` now passes exceptions to the caller instead of returning empty lists. * :func:`LexicalHandler.startDTD` used to receive the public and system IDs in diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 2df6d603207a10..3430ac8668e280 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -575,15 +575,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -609,7 +609,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -652,10 +652,10 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``'as VAR'`` clause is present, the value is simply discarded. @@ -669,7 +669,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -703,7 +703,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback (self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -715,7 +715,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -748,10 +748,10 @@ are useful for writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Our database example from the previous section could be written using this @@ -1167,10 +1167,10 @@ marked in the following list. * It's now illegal to mix iterating over a file with ``for line in file`` and calling the file object's :meth:`read`/:meth:`readline`/:meth:`readlines` - methods. Iteration uses an internal buffer and the :meth:`read\*` methods + methods. Iteration uses an internal buffer and the :meth:`!read\*` methods don't use that buffer. Instead they would return the data following the buffer, causing the data to appear out of order. Mixing iteration and these - methods will now trigger a :exc:`ValueError` from the :meth:`read\*` method. + methods will now trigger a :exc:`ValueError` from the :meth:`!read\*` method. (Implemented by Thomas Wouters.) .. Patch 1397960 @@ -1347,7 +1347,7 @@ complete list of changes, or look through the SVN logs for all the details. :func:`input` function to allow opening files in binary or :term:`universal newlines` mode. Another new parameter, *openhook*, lets you use a function other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`fileno` returns + the set of files, the :class:`FileInput` object's new :meth:`~fileinput.fileno` returns the file descriptor for the currently opened file. (Contributed by Georg Brandl.) @@ -1478,8 +1478,8 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 790710 -* The :mod:`pickle` and :mod:`cPickle` modules no longer accept a return value - of ``None`` from the :meth:`__reduce__` method; the method must return a tuple +* The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a return value + of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The ability to return ``None`` was deprecated in Python 2.4, so this completes the removal of the feature. @@ -1519,7 +1519,7 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now have a +* The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. @@ -1650,9 +1650,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #754022 -* The :mod:`xmlrpclib` module now supports returning :class:`~datetime.datetime` objects - for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`loads` - function or the :class:`Unmarshaller` class to enable this feature. (Contributed +* The :mod:`xmlrpclib <xmlrpc.client>` module now supports returning :class:`~datetime.datetime` objects + for the XML-RPC date type. Supply ``use_datetime=True`` to the :func:`~xmlrpc.client.loads` + function or the :class:`!Unmarshaller` class to enable this feature. (Contributed by Skip Montanaro.) .. Patch 1120353 @@ -1680,7 +1680,7 @@ The ctypes package The :mod:`ctypes` package, written by Thomas Heller, has been added to the standard library. :mod:`ctypes` lets you call arbitrary functions in shared -libraries or DLLs. Long-time users may remember the :mod:`dl` module, which +libraries or DLLs. Long-time users may remember the :mod:`!dl` module, which provides functions for loading shared libraries and calling functions in them. The :mod:`ctypes` package is much fancier. @@ -1724,7 +1724,7 @@ attribute of the function object to change this:: :mod:`ctypes` also provides a wrapper for Python's C API as the ``ctypes.pythonapi`` object. This object does *not* release the global interpreter lock before calling a function, because the lock must be held when -calling into the interpreter's code. There's a :class:`py_object()` type +calling into the interpreter's code. There's a :class:`~ctypes.py_object` type constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: import ctypes @@ -1734,7 +1734,7 @@ constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: ctypes.py_object("abc"), ctypes.py_object(1)) # d is now {'abc', 1}. -Don't forget to use :class:`py_object()`; if it's omitted you end up with a +Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end up with a segmentation fault. :mod:`ctypes` has been around for a while, but people still write and @@ -1877,12 +1877,12 @@ The hashlib package ------------------- A new :mod:`hashlib` module, written by Gregory P. Smith, has been added to -replace the :mod:`md5` and :mod:`sha` modules. :mod:`hashlib` adds support for +replace the :mod:`!md5` and :mod:`!sha` modules. :mod:`hashlib` adds support for additional secure hashes (SHA-224, SHA-256, SHA-384, and SHA-512). When available, the module uses OpenSSL for fast platform optimized implementations of algorithms. -The old :mod:`md5` and :mod:`sha` modules still exist as wrappers around hashlib +The old :mod:`!md5` and :mod:`!sha` modules still exist as wrappers around hashlib to preserve backwards compatibility. The new module's interface is very close to that of the old modules, but not identical. The most significant difference is that the constructor functions for creating new hashing objects are named @@ -2253,12 +2253,12 @@ code: appeared. In Python 2.5, the argument must be exactly one %char specifier with no surrounding text. -* Library: The :mod:`pickle` and :mod:`cPickle` modules no longer accept a - return value of ``None`` from the :meth:`__reduce__` method; the method must +* Library: The :mod:`pickle` and :mod:`!cPickle` modules no longer accept a + return value of ``None`` from the :meth:`~object.__reduce__` method; the method must return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer` and :mod:`DocXMLRPCServer` classes now +* Library: The :mod:`SimpleXMLRPCServer <xmlrpc.server>` and :mod:`DocXMLRPCServer <xmlrpc.server>` classes now have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 198d5152194e93..fc2de7124859a8 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -4,8 +4,6 @@ What's New in Python 2.6 **************************** -.. XXX add trademark info for Apple, Microsoft, SourceForge. - :Author: A.M. Kuchling (amk at amk.ca) .. $Id$ @@ -128,7 +126,7 @@ and to C extension code as :c:data:`!Py_Py3kWarningFlag`. The 3\ *xxx* series of PEPs, which contains proposals for Python 3.0. :pep:`3000` describes the development process for Python 3.0. Start with :pep:`3100` that describes the general goals for Python - 3.0, and then explore the higher-numbered PEPS that propose + 3.0, and then explore the higher-numbered PEPs that propose specific features. @@ -269,15 +267,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods). -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -296,7 +294,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -339,16 +337,16 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply discarded. * The code in *BLOCK* is executed. -* If *BLOCK* raises an exception, the context manager's :meth:`__exit__` method +* If *BLOCK* raises an exception, the context manager's :meth:`~object.__exit__` method is called with three arguments, the exception details (``type, value, traceback``, the same values returned by :func:`sys.exc_info`, which can also be ``None`` if no exception occurred). The method's return value controls whether an exception @@ -357,7 +355,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -391,7 +389,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback(self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -403,7 +401,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -436,10 +434,10 @@ are useful when writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Using this decorator, our database example from the previous section @@ -1051,8 +1049,6 @@ the :mod:`io` module: sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. - .. XXX should 2.6 register them in io.py? - * :class:`BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. @@ -1086,7 +1082,7 @@ the :mod:`io` module: (In Python 2.6, :class:`io.StringIO` is implemented in pure Python, so it's pretty slow. You should therefore stick with the - existing :mod:`StringIO` module or :mod:`cStringIO` for now. At some + existing :mod:`!StringIO` module or :mod:`!cStringIO` for now. At some point Python 3.0's :mod:`io` module will be rewritten into C for speed, and perhaps the C implementation will be backported to the 2.x releases.) @@ -1133,8 +1129,6 @@ while an external caller could be modifying the contents, so there's a corresponding ``PyBuffer_Release(Py_buffer *view)`` to indicate that the external caller is done. -.. XXX PyObject_GetBuffer not documented in c-api - The *flags* argument to :c:func:`PyObject_GetBuffer` specifies constraints upon the memory returned. Some examples are: @@ -1677,8 +1671,9 @@ Some smaller changes made to the core Python language are: (:issue:`1591665`) * Instance method objects have new attributes for the object and function - comprising the method; the new synonym for :attr:`im_self` is - :attr:`__self__`, and :attr:`im_func` is also available as :attr:`__func__`. + comprising the method; the new synonym for :attr:`!im_self` is + :attr:`~method.__self__`, and :attr:`!im_func` is also available as + :attr:`~method.__func__`. The old names are still supported in Python 2.6, but are gone in 3.0. * An obscure change: when you use the :func:`locals` function inside a @@ -1737,7 +1732,7 @@ Optimizations (Contributed by Antoine Pitrou.) Memory usage is reduced by using pymalloc for the Unicode string's data. -* The ``with`` statement now stores the :meth:`__exit__` method on the stack, +* The ``with`` statement now stores the :meth:`~object.__exit__` method on the stack, producing a small speedup. (Implemented by Jeffrey Yasskin.) * To reduce memory usage, the garbage collector will now clear internal @@ -1788,7 +1783,7 @@ changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the Subversion logs for all the details. -* The :mod:`asyncore` and :mod:`asynchat` modules are +* The :mod:`!asyncore` and :mod:`!asynchat` modules are being actively maintained again, and a number of patches and bugfixes were applied. (Maintained by Josiah Carlson; see :issue:`1736190` for one patch.) @@ -1812,7 +1807,7 @@ changes, or look through the Subversion logs for all the details. Nubis; :issue:`1817`.) The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`!cgi` module to the :mod:`urlparse` module. + relocated from the :mod:`!cgi` module to the :mod:`urlparse <urllib.parse>` module. The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1900,8 +1895,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) -* The :mod:`Cookie` module's :class:`Morsel` objects now support an - :attr:`httponly` attribute. In some browsers. cookies with this attribute +* The :mod:`Cookie <http.cookies>` module's :class:`~http.cookies.Morsel` objects now support an + :attr:`~http.cookies.Morsel.httponly` attribute. In some browsers. cookies with this attribute set cannot be accessed or manipulated by JavaScript code. (Contributed by Arvin Schnell; :issue:`1638033`.) @@ -1992,8 +1987,8 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter, specifying a timeout measured in - seconds, was added to the :class:`httplib.HTTPConnection` and - :class:`HTTPSConnection` class constructors. (Added by Facundo + seconds, was added to the :class:`httplib.HTTPConnection <http.client.HTTPConnection>` and + :class:`HTTPSConnection <http.client.HTTPSConnection>` class constructors. (Added by Facundo Batista.) * Most of the :mod:`inspect` module's functions, such as @@ -2376,10 +2371,10 @@ changes, or look through the Subversion logs for all the details. ``socket(socket.AF_INET, ...)`` may be all that's required to make your code work with IPv6. -* The base classes in the :mod:`SocketServer` module now support - calling a :meth:`handle_timeout` method after a span of inactivity - specified by the server's :attr:`timeout` attribute. (Contributed - by Michael Pomraning.) The :meth:`serve_forever` method +* The base classes in the :mod:`SocketServer <socketserver>` module now support + calling a :meth:`~socketserver.BaseServer.handle_timeout` method after a span of inactivity + specified by the server's :attr:`~socketserver.BaseServer.timeout` attribute. (Contributed + by Michael Pomraning.) The :meth:`~socketserver.BaseServer.serve_forever` method now takes an optional poll interval measured in seconds, controlling how often the server will check for a shutdown request. (Contributed by Pedro Werneck and Jeffrey Yasskin; @@ -2393,11 +2388,11 @@ changes, or look through the Subversion logs for all the details. using the format character ``'?'``. (Contributed by David Remahl.) -* The :class:`Popen` objects provided by the :mod:`subprocess` module - now have :meth:`terminate`, :meth:`kill`, and :meth:`send_signal` methods. - On Windows, :meth:`send_signal` only supports the :const:`SIGTERM` +* The :class:`~subprocess.Popen` objects provided by the :mod:`subprocess` module + now have :meth:`~subprocess.Popen.terminate`, :meth:`~subprocess.Popen.kill`, and :meth:`~subprocess.Popen.send_signal` methods. + On Windows, :meth:`!send_signal` only supports the :py:const:`~signal.SIGTERM` signal, and all these methods are aliases for the Win32 API function - :c:func:`TerminateProcess`. + :c:func:`!TerminateProcess`. (Contributed by Christian Heimes.) * A new variable in the :mod:`sys` module, :attr:`float_info`, is an @@ -2483,9 +2478,9 @@ changes, or look through the Subversion logs for all the details. ``with tempfile.NamedTemporaryFile() as tmp: ...``. (Contributed by Alexander Belopolsky; :issue:`2021`.) -* The :mod:`test.test_support` module gained a number +* The :mod:`test.test_support <test.support>` module gained a number of context managers useful for writing tests. - :func:`EnvironmentVarGuard` is a + :func:`~test.support.os_helper.EnvironmentVarGuard` is a context manager that temporarily changes environment variables and automatically restores them to their old values. @@ -2582,9 +2577,9 @@ changes, or look through the Subversion logs for all the details. (:issue:`1513695`) * An optional ``timeout`` parameter was added to the - :func:`urllib.urlopen` function and the + :func:`urllib.urlopen <urllib.request.urlopen>` function and the :class:`urllib.ftpwrapper` class constructor, as well as the - :func:`urllib2.urlopen` function. The parameter specifies a timeout + :func:`urllib2.urlopen <urllib.request.urlopen>` function. The parameter specifies a timeout measured in seconds. For example:: >>> u = urllib2.urlopen("http://slow.example.com", @@ -2609,7 +2604,7 @@ changes, or look through the Subversion logs for all the details. intended for testing purposes that lets you temporarily modify the warning filters and then restore their original values (:issue:`3781`). -* The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` +* The XML-RPC :class:`SimpleXMLRPCServer <xmlrpc.server>` and :class:`DocXMLRPCServer <xmlrpc.server>` classes can now be prevented from immediately opening and binding to their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's @@ -2626,11 +2621,11 @@ changes, or look through the Subversion logs for all the details. information. (Contributed by Alan McIntyre as part of his project for Google's Summer of Code 2007.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) The code can also handle dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) and 64-bit integers represented by using ``<i8>`` in XML-RPC responses @@ -2915,8 +2910,8 @@ Deprecations and Removals * Changes to the :class:`Exception` interface as dictated by :pep:`352` continue to be made. For 2.6, - the :attr:`message` attribute is being deprecated in favor of the - :attr:`args` attribute. + the :attr:`!message` attribute is being deprecated in favor of the + :attr:`~BaseException.args` attribute. * (3.0-warning mode) Python 3.0 will feature a reorganized standard library that will drop many outdated modules and rename others. @@ -2924,51 +2919,51 @@ Deprecations and Removals when they are imported. The list of deprecated modules is: - :mod:`audiodev`, - :mod:`bgenlocations`, - :mod:`buildtools`, - :mod:`bundlebuilder`, - :mod:`Canvas`, - :mod:`compiler`, - :mod:`dircache`, - :mod:`dl`, - :mod:`fpformat`, - :mod:`gensuitemodule`, - :mod:`ihooks`, - :mod:`imageop`, - :mod:`imgfile`, - :mod:`linuxaudiodev`, - :mod:`mhlib`, - :mod:`mimetools`, - :mod:`multifile`, - :mod:`new`, - :mod:`pure`, - :mod:`statvfs`, - :mod:`sunaudiodev`, - :mod:`test.testall`, and - :mod:`toaiff`. - -* The :mod:`gopherlib` module has been removed. - -* The :mod:`MimeWriter` module and :mod:`mimify` module + :mod:`!audiodev`, + :mod:`!bgenlocations`, + :mod:`!buildtools`, + :mod:`!bundlebuilder`, + :mod:`!Canvas`, + :mod:`!compiler`, + :mod:`!dircache`, + :mod:`!dl`, + :mod:`!fpformat`, + :mod:`!gensuitemodule`, + :mod:`!ihooks`, + :mod:`!imageop`, + :mod:`!imgfile`, + :mod:`!linuxaudiodev`, + :mod:`!mhlib`, + :mod:`!mimetools`, + :mod:`!multifile`, + :mod:`!new`, + :mod:`!pure`, + :mod:`!statvfs`, + :mod:`!sunaudiodev`, + :mod:`!test.testall`, and + :mod:`!toaiff`. + +* The :mod:`!gopherlib` module has been removed. + +* The :mod:`!MimeWriter` module and :mod:`!mimify` module have been deprecated; use the :mod:`email` package instead. -* The :mod:`md5` module has been deprecated; use the :mod:`hashlib` module +* The :mod:`!md5` module has been deprecated; use the :mod:`hashlib` module instead. -* The :mod:`posixfile` module has been deprecated; :func:`fcntl.lockf` +* The :mod:`!posixfile` module has been deprecated; :func:`fcntl.lockf` provides better locking. -* The :mod:`popen2` module has been deprecated; use the :mod:`subprocess` +* The :mod:`!popen2` module has been deprecated; use the :mod:`subprocess` module. -* The :mod:`rgbimg` module has been removed. +* The :mod:`!rgbimg` module has been removed. -* The :mod:`sets` module has been deprecated; it's better to +* The :mod:`!sets` module has been deprecated; it's better to use the built-in :class:`set` and :class:`frozenset` types. -* The :mod:`sha` module has been deprecated; use the :mod:`hashlib` module +* The :mod:`!sha` module has been deprecated; use the :mod:`hashlib` module instead. @@ -2997,6 +2992,32 @@ Changes to Python's build process and to the C API include: architectures (x86, PowerPC), 64-bit (x86-64 and PPC-64), or both. (Contributed by Ronald Oussoren.) +* A new function added in Python 2.6.6, :c:func:`!PySys_SetArgvEx`, sets + the value of ``sys.argv`` and can optionally update ``sys.path`` to + include the directory containing the script named by ``sys.argv[0]`` + depending on the value of an *updatepath* parameter. + + This function was added to close a security hole for applications + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would + always update ``sys.path``, and sometimes it would add the current + directory. This meant that, if you ran an application embedding + Python in a directory controlled by someone else, attackers could + put a Trojan-horse module in the directory (say, a file named + :file:`os.py`) that your application would then import and run. + + If you maintain a C/C++ application that embeds Python, check + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` + with *updatepath* set to false. Note that using this function will + break compatibility with Python versions 2.6.5 and earlier; if you + have to continue working with earlier versions, you can leave + the call to :c:func:`!PySys_SetArgv` alone and call + ``PyRun_SimpleString("sys.path.pop(0)\n")`` afterwards to discard + the first ``sys.path`` component. + + Security issue reported as :cve:`2008-5983`; + discussed in :gh:`50003`, and fixed by Antoine Pitrou. + * The BerkeleyDB module now has a C API object, available as ``bsddb.db.api``. This object can be used by other C extensions that wish to use the :mod:`bsddb` module for their own purposes. @@ -3109,8 +3130,8 @@ Port-Specific Changes: Windows * The :mod:`msvcrt` module now supports both the normal and wide char variants of the console I/O - API. The :func:`getwch` function reads a keypress and returns a Unicode - value, as does the :func:`getwche` function. The :func:`putwch` function + API. The :func:`~msvcrt.getwch` function reads a keypress and returns a Unicode + value, as does the :func:`~msvcrt.getwche` function. The :func:`~msvcrt.putwch` function takes a Unicode character and writes it to the console. (Contributed by Christian Heimes.) @@ -3119,24 +3140,24 @@ Port-Specific Changes: Windows directory path. (Contributed by Josiah Carlson; :issue:`957650`.) * The :mod:`socket` module's socket objects now have an - :meth:`ioctl` method that provides a limited interface to the + :meth:`~socket.socket.ioctl` method that provides a limited interface to the :c:func:`WSAIoctl` system interface. -* The :mod:`_winreg` module now has a function, - :func:`ExpandEnvironmentStrings`, +* The :mod:`_winreg <winreg>` module now has a function, + :func:`~winreg.ExpandEnvironmentStrings`, that expands environment variable references such as ``%NAME%`` in an input string. The handle objects provided by this module now support the context protocol, so they can be used in :keyword:`with` statements. (Contributed by Christian Heimes.) - :mod:`_winreg` also has better support for x64 systems, - exposing the :func:`DisableReflectionKey`, :func:`EnableReflectionKey`, - and :func:`QueryReflectionKey` functions, which enable and disable + :mod:`_winreg <winreg>` also has better support for x64 systems, + exposing the :func:`~winreg.DisableReflectionKey`, :func:`~winreg.EnableReflectionKey`, + and :func:`~winreg.QueryReflectionKey` functions, which enable and disable registry reflection for 32-bit processes running on 64-bit systems. (:issue:`1753245`) -* The :mod:`!msilib` module's :class:`Record` object - gained :meth:`GetInteger` and :meth:`GetString` methods that +* The :mod:`!msilib` module's :class:`!Record` object + gained :meth:`!GetInteger` and :meth:`!GetString` methods that return field values as an integer or a string. (Contributed by Floris Bruynooghe; :issue:`2125`.) @@ -3150,49 +3171,49 @@ Port-Specific Changes: Mac OS X :option:`!--with-framework-name=` option to the :program:`configure` script. -* The :mod:`macfs` module has been removed. This in turn required the - :func:`macostools.touched` function to be removed because it depended on the - :mod:`macfs` module. (:issue:`1490190`) +* The :mod:`!macfs` module has been removed. This in turn required the + :func:`!macostools.touched` function to be removed because it depended on the + :mod:`!macfs` module. (:issue:`1490190`) * Many other Mac OS modules have been deprecated and will be removed in Python 3.0: - :mod:`_builtinSuites`, - :mod:`aepack`, - :mod:`aetools`, - :mod:`aetypes`, - :mod:`applesingle`, - :mod:`appletrawmain`, - :mod:`appletrunner`, - :mod:`argvemulator`, - :mod:`Audio_mac`, - :mod:`autoGIL`, - :mod:`Carbon`, - :mod:`cfmfile`, - :mod:`CodeWarrior`, - :mod:`ColorPicker`, - :mod:`EasyDialogs`, - :mod:`Explorer`, - :mod:`Finder`, - :mod:`FrameWork`, - :mod:`findertools`, - :mod:`ic`, - :mod:`icglue`, - :mod:`icopen`, - :mod:`macerrors`, - :mod:`MacOS`, - :mod:`macfs`, - :mod:`macostools`, - :mod:`macresource`, - :mod:`MiniAEFrame`, - :mod:`Nav`, - :mod:`Netscape`, - :mod:`OSATerminology`, - :mod:`pimp`, - :mod:`PixMapWrapper`, - :mod:`StdSuites`, - :mod:`SystemEvents`, - :mod:`Terminal`, and - :mod:`terminalcommand`. + :mod:`!_builtinSuites`, + :mod:`!aepack`, + :mod:`!aetools`, + :mod:`!aetypes`, + :mod:`!applesingle`, + :mod:`!appletrawmain`, + :mod:`!appletrunner`, + :mod:`!argvemulator`, + :mod:`!Audio_mac`, + :mod:`!autoGIL`, + :mod:`!Carbon`, + :mod:`!cfmfile`, + :mod:`!CodeWarrior`, + :mod:`!ColorPicker`, + :mod:`!EasyDialogs`, + :mod:`!Explorer`, + :mod:`!Finder`, + :mod:`!FrameWork`, + :mod:`!findertools`, + :mod:`!ic`, + :mod:`!icglue`, + :mod:`!icopen`, + :mod:`!macerrors`, + :mod:`!MacOS`, + :mod:`!macfs`, + :mod:`!macostools`, + :mod:`!macresource`, + :mod:`!MiniAEFrame`, + :mod:`!Nav`, + :mod:`!Netscape`, + :mod:`!OSATerminology`, + :mod:`!pimp`, + :mod:`!PixMapWrapper`, + :mod:`!StdSuites`, + :mod:`!SystemEvents`, + :mod:`!Terminal`, and + :mod:`!terminalcommand`. .. ====================================================================== @@ -3201,29 +3222,29 @@ Port-Specific Changes: IRIX A number of old IRIX-specific modules were deprecated and will be removed in Python 3.0: -:mod:`al` and :mod:`AL`, -:mod:`cd`, -:mod:`cddb`, -:mod:`cdplayer`, -:mod:`CL` and :mod:`cl`, -:mod:`DEVICE`, -:mod:`ERRNO`, -:mod:`FILE`, -:mod:`FL` and :mod:`fl`, -:mod:`flp`, -:mod:`fm`, -:mod:`GET`, -:mod:`GLWS`, -:mod:`GL` and :mod:`gl`, -:mod:`IN`, -:mod:`IOCTL`, -:mod:`jpeg`, -:mod:`panelparser`, -:mod:`readcd`, -:mod:`SV` and :mod:`sv`, -:mod:`torgb`, -:mod:`videoreader`, and -:mod:`WAIT`. +:mod:`!al` and :mod:`!AL`, +:mod:`!cd`, +:mod:`!cddb`, +:mod:`!cdplayer`, +:mod:`!CL` and :mod:`!cl`, +:mod:`!DEVICE`, +:mod:`!ERRNO`, +:mod:`!FILE`, +:mod:`!FL` and :mod:`!fl`, +:mod:`!flp`, +:mod:`!fm`, +:mod:`!GET`, +:mod:`!GLWS`, +:mod:`!GL` and :mod:`!gl`, +:mod:`!IN`, +:mod:`!IOCTL`, +:mod:`!jpeg`, +:mod:`!panelparser`, +:mod:`!readcd`, +:mod:`!SV` and :mod:`!sv`, +:mod:`!torgb`, +:mod:`!videoreader`, and +:mod:`!WAIT`. .. ====================================================================== @@ -3279,11 +3300,11 @@ that may require changes to your code: :exc:`StandardError` but now it is, through :exc:`IOError`. (Implemented by Gregory P. Smith; :issue:`1706815`.) -* The :mod:`xmlrpclib` module no longer automatically converts +* The :mod:`xmlrpclib <xmlrpc.client>` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the - :class:`xmlrpclib.DateTime` type; the conversion semantics were + :class:`xmlrpclib.DateTime <xmlrpc.client.DateTime>` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` instances. (:issue:`1330538`) * (3.0-warning mode) The :class:`Exception` class now warns @@ -3299,6 +3320,15 @@ that may require changes to your code: scoping rules, also cause warnings because such comparisons are forbidden entirely in 3.0. +For applications that embed Python: + +* The :c:func:`!PySys_SetArgvEx` function was added in Python 2.6.6, + letting applications close a security hole when the existing + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with + *updatepath* set to false. + .. ====================================================================== diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index f45d45f5b5bf72..c45f0887b41f4f 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -196,7 +196,7 @@ A partial list of 3.1 features that were backported to 2.7: Other new Python3-mode warnings include: -* :func:`operator.isCallable` and :func:`operator.sequenceIncludes`, +* :func:`!operator.isCallable` and :func:`!operator.sequenceIncludes`, which are not supported in 3.x, now trigger warnings. * The :option:`!-3` switch now automatically enables the :option:`!-Qwarn` switch that causes warnings @@ -282,12 +282,12 @@ How does the :class:`~collections.OrderedDict` work? It maintains a doubly linked list of keys, appending new keys to the list as they're inserted. A secondary dictionary maps keys to their corresponding list node, so deletion doesn't have to traverse the entire linked list and therefore -remains O(1). +remains *O*\ (1). The standard library now supports use of ordered dictionaries in several modules. -* The :mod:`ConfigParser` module uses them by default, meaning that +* The :mod:`ConfigParser <configparser>` module uses them by default, meaning that configuration files can now be read, modified, and then written back in their original order. @@ -455,11 +455,11 @@ a varying number of handlers. All this flexibility can require a lot of configuration. You can write Python statements to create objects and set their properties, but a complex set-up requires verbose but boring code. -:mod:`logging` also supports a :func:`~logging.fileConfig` +:mod:`logging` also supports a :func:`~logging.config.fileConfig` function that parses a file, but the file format doesn't support configuring filters, and it's messier to generate programmatically. -Python 2.7 adds a :func:`~logging.dictConfig` function that +Python 2.7 adds a :func:`~logging.config.dictConfig` function that uses a dictionary to configure logging. There are many ways to produce a dictionary from different sources: construct one with code; parse a file containing JSON; or use a YAML parsing library if one is @@ -533,7 +533,7 @@ implemented by Vinay Sajip, are: ``getLogger('app.network.listen')``. * The :class:`~logging.LoggerAdapter` class gained an - :meth:`~logging.LoggerAdapter.isEnabledFor` method that takes a + :meth:`~logging.Logger.isEnabledFor` method that takes a *level* and returns whether the underlying logger would process a message of that level of importance. @@ -554,8 +554,8 @@ called a :dfn:`view` instead of a fully materialized list. It's not possible to change the return values of :meth:`~dict.keys`, :meth:`~dict.values`, and :meth:`~dict.items` in Python 2.7 because too much code would break. Instead the 3.x versions were added -under the new names :meth:`~dict.viewkeys`, :meth:`~dict.viewvalues`, -and :meth:`~dict.viewitems`. +under the new names :meth:`!viewkeys`, :meth:`!viewvalues`, +and :meth:`!viewitems`. :: @@ -720,7 +720,7 @@ Some smaller changes made to the core Python language are: with B() as b: ... suite of statements ... - The :func:`contextlib.nested` function provides a very similar + The :func:`!contextlib.nested` function provides a very similar function, so it's no longer necessary and has been deprecated. (Proposed in https://codereview.appspot.com/53094; implemented by @@ -785,7 +785,7 @@ Some smaller changes made to the core Python language are: implemented by Mark Dickinson; :issue:`1811`.) * Implicit coercion for complex numbers has been removed; the interpreter - will no longer ever attempt to call a :meth:`__coerce__` method on complex + will no longer ever attempt to call a :meth:`!__coerce__` method on complex objects. (Removed by Meador Inge and Mark Dickinson; :issue:`5211`.) * The :meth:`str.format` method now supports automatic numbering of the replacement @@ -817,7 +817,7 @@ Some smaller changes made to the core Python language are: A low-level change: the :meth:`object.__format__` method now triggers a :exc:`PendingDeprecationWarning` if it's passed a format string, - because the :meth:`__format__` method for :class:`object` converts + because the :meth:`!__format__` method for :class:`object` converts the object to a string representation and formats that. Previously the method silently applied the format string to the string representation, but that could hide mistakes in Python code. If @@ -825,7 +825,7 @@ Some smaller changes made to the core Python language are: precision, presumably you're expecting the formatting to be applied in some object-specific way. (Fixed by Eric Smith; :issue:`7994`.) -* The :func:`int` and :func:`long` types gained a ``bit_length`` +* The :func:`int` and :func:`!long` types gained a ``bit_length`` method that returns the number of bits necessary to represent its argument in binary:: @@ -848,8 +848,8 @@ Some smaller changes made to the core Python language are: statements that were only working by accident. (Fixed by Meador Inge; :issue:`7902`.) -* It's now possible for a subclass of the built-in :class:`unicode` type - to override the :meth:`__unicode__` method. (Implemented by +* It's now possible for a subclass of the built-in :class:`!unicode` type + to override the :meth:`!__unicode__` method. (Implemented by Victor Stinner; :issue:`1583863`.) * The :class:`bytearray` type's :meth:`~bytearray.translate` method now accepts @@ -858,9 +858,11 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using ``@classmethod`` and ``@staticmethod`` to wrap +* When using :class:`@classmethod <classmethod>` and + :class:`@staticmethod <staticmethod>` to wrap methods as class or static methods, the wrapper object now - exposes the wrapped function as their :attr:`__func__` attribute. + exposes the wrapped function as their :attr:`~method.__func__` + attribute. (Contributed by Amaury Forgeot d'Arc, after a suggestion by George Sakkis; :issue:`5982`.) @@ -874,7 +876,7 @@ Some smaller changes made to the core Python language are: Forgeot d'Arc in :issue:`1616979`; CP858 contributed by Tim Hatch in :issue:`8016`.) -* The :class:`file` object will now set the :attr:`filename` attribute +* The :class:`!file` object will now set the :attr:`!filename` attribute on the :exc:`IOError` exception when trying to open a directory on POSIX platforms (noted by Jan Kaliszewski; :issue:`4764`), and now explicitly checks for and forbids writing to read-only file objects @@ -913,7 +915,7 @@ used with the :option:`-W` switch, separated by commas. (Contributed by Brian Curtin; :issue:`7301`.) For example, the following setting will print warnings every time -they occur, but turn warnings from the :mod:`Cookie` module into an +they occur, but turn warnings from the :mod:`Cookie <http.cookies>` module into an error. (The exact syntax for setting an environment variable varies across operating systems and shells.) @@ -930,8 +932,8 @@ Optimizations Several performance enhancements have been added: * A new opcode was added to perform the initial setup for - :keyword:`with` statements, looking up the :meth:`__enter__` and - :meth:`__exit__` methods. (Contributed by Benjamin Peterson.) + :keyword:`with` statements, looking up the :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. (Contributed by Benjamin Peterson.) * The garbage collector now performs better for one common usage pattern: when many objects are being allocated without deallocating @@ -964,7 +966,7 @@ Several performance enhancements have been added: Apart from the performance improvements this change should be invisible to end users, with one exception: for testing and - debugging purposes there's a new structseq :data:`sys.long_info` that + debugging purposes there's a new structseq :data:`!sys.long_info` that provides information about the internal format, giving the number of bits per digit and the size in bytes of the C type used to store each digit:: @@ -1003,19 +1005,19 @@ Several performance enhancements have been added: conversion function that supports arbitrary bases. (Patch by Gawain Bolton; :issue:`6713`.) -* The :meth:`split`, :meth:`replace`, :meth:`rindex`, - :meth:`rpartition`, and :meth:`rsplit` methods of string-like types +* The :meth:`!split`, :meth:`!replace`, :meth:`!rindex`, + :meth:`!rpartition`, and :meth:`!rsplit` methods of string-like types (strings, Unicode strings, and :class:`bytearray` objects) now use a fast reverse-search algorithm instead of a character-by-character scan. This is sometimes faster by a factor of 10. (Added by Florent Xicluna; :issue:`7462` and :issue:`7622`.) -* The :mod:`pickle` and :mod:`cPickle` modules now automatically +* The :mod:`pickle` and :mod:`!cPickle` modules now automatically intern the strings used for attribute names, reducing memory usage of the objects resulting from unpickling. (Contributed by Jake McGuire; :issue:`5084`.) -* The :mod:`cPickle` module now special-cases dictionaries, +* The :mod:`!cPickle` module now special-cases dictionaries, nearly halving the time required to pickle them. (Contributed by Collin Winter; :issue:`5670`.) @@ -1042,7 +1044,7 @@ changes, or look through the Subversion logs for all the details. used with :class:`memoryview` instances and other similar buffer objects. (Backported from 3.x by Florent Xicluna; :issue:`7703`.) -* Updated module: the :mod:`bsddb` module has been updated from 4.7.2devel9 +* Updated module: the :mod:`!bsddb` module has been updated from 4.7.2devel9 to version 4.8.4 of `the pybsddb package <https://www.jcea.es/programacion/pybsddb.htm>`__. The new version features better Python 3.x compatibility, various bug fixes, @@ -1127,12 +1129,12 @@ changes, or look through the Subversion logs for all the details. (Added by Raymond Hettinger; :issue:`1818`.) - Finally, the :class:`~collections.Mapping` abstract base class now - returns :const:`NotImplemented` if a mapping is compared to + Finally, the :class:`~collections.abc.Mapping` abstract base class now + returns :data:`NotImplemented` if a mapping is compared to another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) -* Constructors for the parsing classes in the :mod:`ConfigParser` module now +* Constructors for the parsing classes in the :mod:`ConfigParser <configparser>` module now take an *allow_no_value* parameter, defaulting to false; if true, options without values will be allowed. For example:: @@ -1156,12 +1158,12 @@ changes, or look through the Subversion logs for all the details. (Contributed by Mats Kindahl; :issue:`7005`.) -* Deprecated function: :func:`contextlib.nested`, which allows +* Deprecated function: :func:`!contextlib.nested`, which allows handling more than one context manager with a single :keyword:`with` statement, has been deprecated, because the :keyword:`!with` statement now supports multiple context managers. -* The :mod:`cookielib` module now ignores cookies that have an invalid +* The :mod:`cookielib <http.cookiejar>` module now ignores cookies that have an invalid version field, one that doesn't contain an integer value. (Fixed by John J. Lee; :issue:`3924`.) @@ -1182,7 +1184,7 @@ changes, or look through the Subversion logs for all the details. * New method: the :class:`~decimal.Decimal` class gained a :meth:`~decimal.Decimal.from_float` class method that performs an exact - conversion of a floating-point number to a :class:`~decimal.Decimal`. + conversion of a floating-point number to a :class:`!Decimal`. This exact conversion strives for the closest decimal approximation to the floating-point representation's value; the resulting decimal value will therefore still include the inaccuracy, @@ -1196,9 +1198,9 @@ changes, or look through the Subversion logs for all the details. of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine - :class:`Decimal` and floating-point in other operations such as addition, + :class:`!Decimal` and floating-point in other operations such as addition, since you should be explicitly choosing how to convert between float and - :class:`~decimal.Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) + :class:`!Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) The constructor for :class:`~decimal.Decimal` now accepts floating-point numbers (added by Raymond Hettinger; :issue:`8257`) @@ -1216,7 +1218,7 @@ changes, or look through the Subversion logs for all the details. more sensible for numeric types. (Changed by Mark Dickinson; :issue:`6857`.) Comparisons involving a signaling NaN value (or ``sNAN``) now signal - :const:`InvalidOperation` instead of silently returning a true or + :const:`~decimal.InvalidOperation` instead of silently returning a true or false value depending on the comparison operator. Quiet NaN values (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) @@ -1233,13 +1235,13 @@ changes, or look through the Subversion logs for all the details. created some new files that should be included. (Fixed by Tarek Ziadé; :issue:`8688`.) -* The :mod:`doctest` module's :const:`IGNORE_EXCEPTION_DETAIL` flag +* The :mod:`doctest` module's :const:`~doctest.IGNORE_EXCEPTION_DETAIL` flag will now ignore the name of the module containing the exception being tested. (Patch by Lennart Regebro; :issue:`7490`.) * The :mod:`email` module's :class:`~email.message.Message` class will now accept a Unicode-valued payload, automatically converting the - payload to the encoding specified by :attr:`output_charset`. + payload to the encoding specified by :attr:`!output_charset`. (Added by R. David Murray; :issue:`1368247`.) * The :class:`~fractions.Fraction` class now accepts a single float or @@ -1266,10 +1268,10 @@ changes, or look through the Subversion logs for all the details. :issue:`6845`.) * New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` - module takes a class that defines an :meth:`__eq__` method and one of - :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, + module takes a class that defines an :meth:`~object.__eq__` method and one of + :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the - :meth:`__cmp__` method is being deprecated in Python 3.x, + :meth:`!__cmp__` method is being deprecated in Python 3.x, this decorator makes it easier to define ordered classes. (Added by Raymond Hettinger; :issue:`5479`.) @@ -1298,24 +1300,24 @@ changes, or look through the Subversion logs for all the details. :mod:`gzip` module will now consume these trailing bytes. (Fixed by Tadek Pietraszek and Brian Curtin; :issue:`2846`.) -* New attribute: the :mod:`hashlib` module now has an :attr:`~hashlib.hashlib.algorithms` +* New attribute: the :mod:`hashlib` module now has an :attr:`!algorithms` attribute containing a tuple naming the supported algorithms. In Python 2.7, ``hashlib.algorithms`` contains ``('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')``. (Contributed by Carl Chenet; :issue:`7418`.) -* The default :class:`~httplib.HTTPResponse` class used by the :mod:`httplib` module now +* The default :class:`~http.client.HTTPResponse` class used by the :mod:`httplib <http>` module now supports buffering, resulting in much faster reading of HTTP responses. (Contributed by Kristján Valur Jónsson; :issue:`4879`.) - The :class:`~httplib.HTTPConnection` and :class:`~httplib.HTTPSConnection` classes + The :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` classes now support a *source_address* parameter, a ``(host, port)`` 2-tuple giving the source address that will be used for the connection. (Contributed by Eldon Ziegler; :issue:`3972`.) -* The :mod:`ihooks` module now supports relative imports. Note that - :mod:`ihooks` is an older module for customizing imports, - superseded by the :mod:`imputil` module added in Python 2.0. +* The :mod:`!ihooks` module now supports relative imports. Note that + :mod:`!ihooks` is an older module for customizing imports, + superseded by the :mod:`!imputil` module added in Python 2.0. (Relative import support added by Neil Schemenauer.) .. revision 75423 @@ -1346,10 +1348,10 @@ changes, or look through the Subversion logs for all the details. * Updated module: The :mod:`io` library has been upgraded to the version shipped with Python 3.1. For 3.1, the I/O library was entirely rewritten in C and is 2 to 20 times faster depending on the task being performed. The - original Python version was renamed to the :mod:`_pyio` module. + original Python version was renamed to the :mod:`!_pyio` module. One minor resulting change: the :class:`io.TextIOBase` class now - has an :attr:`errors` attribute giving the error setting + has an :attr:`~io.TextIOBase.errors` attribute giving the error setting used for encoding and decoding errors (one of ``'strict'``, ``'replace'``, ``'ignore'``). @@ -1415,16 +1417,16 @@ changes, or look through the Subversion logs for all the details. :func:`~math.lgamma` for the natural log of the Gamma function. (Contributed by Mark Dickinson and nirinA raseliarison; :issue:`3366`.) -* The :mod:`multiprocessing` module's :class:`Manager*` classes +* The :mod:`multiprocessing` module's :class:`!Manager*` classes can now be passed a callable that will be called whenever a subprocess is started, along with a set of arguments that will be passed to the callable. (Contributed by lekma; :issue:`5585`.) - The :class:`~multiprocessing.Pool` class, which controls a pool of worker processes, + The :class:`~multiprocessing.pool.Pool` class, which controls a pool of worker processes, now has an optional *maxtasksperchild* parameter. Worker processes will perform the specified number of tasks and then exit, causing the - :class:`~multiprocessing.Pool` to start a new worker. This is useful if tasks may leak + :class:`!Pool` to start a new worker. This is useful if tasks may leak memory or other resources, or if some tasks will cause the worker to become very large. (Contributed by Charles Cazabon; :issue:`6963`.) @@ -1496,7 +1498,7 @@ changes, or look through the Subversion logs for all the details. global site-packages directories, :func:`~site.getusersitepackages` returns the path of the user's site-packages directory, and - :func:`~site.getuserbase` returns the value of the :envvar:`USER_BASE` + :func:`~site.getuserbase` returns the value of the :data:`~site.USER_BASE` environment variable, giving the path to a directory that can be used to store data. (Contributed by Tarek Ziadé; :issue:`6693`.) @@ -1516,16 +1518,16 @@ changes, or look through the Subversion logs for all the details. the :class:`bytearray` and :class:`memoryview` objects. (Implemented by Antoine Pitrou; :issue:`8104`.) -* The :mod:`SocketServer` module's :class:`~SocketServer.TCPServer` class now +* The :mod:`SocketServer <socketserver>` module's :class:`~socketserver.TCPServer` class now supports socket timeouts and disabling the Nagle algorithm. - The :attr:`~SocketServer.TCPServer.disable_nagle_algorithm` class attribute + The :attr:`!disable_nagle_algorithm` class attribute defaults to ``False``; if overridden to be true, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold + The :attr:`~socketserver.BaseServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` - will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. + no request is received within that time, :meth:`~socketserver.BaseServer.handle_timeout` + will be called and :meth:`~socketserver.BaseServer.handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1538,11 +1540,11 @@ changes, or look through the Subversion logs for all the details. * The :mod:`ssl` module's :class:`~ssl.SSLSocket` objects now support the buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set - OpenSSL's :c:macro:`SSL_MODE_AUTO_RETRY`, which will prevent an error + OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error code being returned from :meth:`recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). - The :func:`ssl.wrap_socket` constructor function now takes a + The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a *ciphers* argument that's a string listing the encryption algorithms to be allowed; the format of the string is described `in the OpenSSL documentation @@ -1566,8 +1568,8 @@ changes, or look through the Subversion logs for all the details. code (one of ``bBhHiIlLqQ``); it now always raises a :exc:`struct.error` exception. (Changed by Mark Dickinson; :issue:`1523`.) The :func:`~struct.pack` function will also - attempt to use :meth:`__index__` to convert and pack non-integers - before trying the :meth:`__int__` method or reporting an error. + attempt to use :meth:`~object.__index__` to convert and pack non-integers + before trying the :meth:`~object.__int__` method or reporting an error. (Changed by Mark Dickinson; :issue:`8300`.) * New function: the :mod:`subprocess` module's @@ -1588,7 +1590,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Gregory P. Smith.) The :mod:`subprocess` module will now retry its internal system calls - on receiving an :const:`EINTR` signal. (Reported by several people; final + on receiving an :const:`~errno.EINTR` signal. (Reported by several people; final patch by Gregory P. Smith in :issue:`1068268`.) * New function: :func:`~symtable.Symbol.is_declared_global` in the :mod:`symtable` module @@ -1600,16 +1602,16 @@ changes, or look through the Subversion logs for all the details. identifier instead of the previous default value of ``'python'``. (Changed by Sean Reifschneider; :issue:`8451`.) -* The ``sys.version_info`` value is now a named tuple, with attributes - named :attr:`major`, :attr:`minor`, :attr:`micro`, - :attr:`releaselevel`, and :attr:`serial`. (Contributed by Ross +* The :attr:`sys.version_info` value is now a named tuple, with attributes + named :attr:`!major`, :attr:`!minor`, :attr:`!micro`, + :attr:`!releaselevel`, and :attr:`!serial`. (Contributed by Ross Light; :issue:`4285`.) :func:`sys.getwindowsversion` also returns a named tuple, - with attributes named :attr:`major`, :attr:`minor`, :attr:`build`, - :attr:`platform`, :attr:`service_pack`, :attr:`service_pack_major`, - :attr:`service_pack_minor`, :attr:`suite_mask`, and - :attr:`product_type`. (Contributed by Brian Curtin; :issue:`7766`.) + with attributes named :attr:`!major`, :attr:`!minor`, :attr:`!build`, + :attr:`!platform`, :attr:`!service_pack`, :attr:`!service_pack_major`, + :attr:`!service_pack_minor`, :attr:`!suite_mask`, and + :attr:`!product_type`. (Contributed by Brian Curtin; :issue:`7766`.) * The :mod:`tarfile` module's default error handling has changed, to no longer suppress fatal errors. The default error level was previously 0, @@ -1646,7 +1648,7 @@ changes, or look through the Subversion logs for all the details. and has been updated to version 5.2.0 (updated by Florent Xicluna; :issue:`8024`). -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -1673,7 +1675,7 @@ changes, or look through the Subversion logs for all the details. (Python 2.7 actually produces slightly different output, since it returns a named tuple instead of a standard tuple.) - The :mod:`urlparse` module also supports IPv6 literal addresses as defined by + The :mod:`urlparse <urllib.parse>` module also supports IPv6 literal addresses as defined by :rfc:`2732` (contributed by Senthil Kumaran; :issue:`2987`). .. doctest:: @@ -1689,18 +1691,18 @@ changes, or look through the Subversion logs for all the details. (Originally implemented in Python 3.x by Raymond Hettinger, and backported to 2.7 by Michael Foord.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library, no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ``<?xml-stylesheet href="#style1"?>``) or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The XML-RPC client and server, provided by the :mod:`xmlrpclib` and - :mod:`SimpleXMLRPCServer` modules, have improved performance by +* The XML-RPC client and server, provided by the :mod:`xmlrpclib <xmlrpc.client>` and + :mod:`SimpleXMLRPCServer <xmlrpc.server>` modules, have improved performance by supporting HTTP/1.1 keep-alive and by optionally using gzip encoding to compress the XML being exchanged. The gzip compression is - controlled by the :attr:`encode_threshold` attribute of - :class:`SimpleXMLRPCRequestHandler`, which contains a size in bytes; + controlled by the :attr:`!encode_threshold` attribute of + :class:`~xmlrpc.server.SimpleXMLRPCRequestHandler`, which contains a size in bytes; responses larger than this will be compressed. (Contributed by Kristján Valur Jónsson; :issue:`6267`.) @@ -1711,7 +1713,8 @@ changes, or look through the Subversion logs for all the details. :mod:`zipfile` now also supports archiving empty directories and extracts them correctly. (Fixed by Kuba Wieczorek; :issue:`4710`.) Reading files out of an archive is faster, and interleaving - :meth:`~zipfile.ZipFile.read` and :meth:`~zipfile.ZipFile.readline` now works correctly. + :meth:`read() <io.BufferedIOBase.read>` and + :meth:`readline() <io.IOBase.readline>` now works correctly. (Contributed by Nir Aides; :issue:`7610`.) The :func:`~zipfile.is_zipfile` function now @@ -1735,7 +1738,7 @@ New module: importlib Python 3.1 includes the :mod:`importlib` package, a re-implementation of the logic underlying Python's :keyword:`import` statement. -:mod:`importlib` is useful for implementors of Python interpreters and +:mod:`importlib` is useful for implementers of Python interpreters and to users who wish to write new importers that can participate in the import process. Python 2.7 doesn't contain the complete :mod:`importlib` package, but instead has a tiny subset that contains @@ -1805,14 +1808,14 @@ closely resemble the native platform's widgets. This widget set was originally called Tile, but was renamed to Ttk (for "themed Tk") on being added to Tcl/Tck release 8.5. -To learn more, read the :mod:`ttk` module documentation. You may also +To learn more, read the :mod:`~tkinter.ttk` module documentation. You may also wish to read the Tcl/Tk manual page describing the Ttk theme engine, available at -https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.htm. Some +https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.html. Some screenshots of the Python/Ttk code in use are at https://code.google.com/archive/p/python-ttk/wikis/Screenshots.wiki. -The :mod:`ttk` module was written by Guilherme Polo and added in +The :mod:`tkinter.ttk` module was written by Guilherme Polo and added in :issue:`2983`. An alternate version called ``Tile.py``, written by Martin Franklin and maintained by Kevin Walzer, was proposed for inclusion in :issue:`2618`, but the authors argued that Guilherme @@ -1828,8 +1831,7 @@ The :mod:`unittest` module was greatly enhanced; many new features were added. Most of these features were implemented by Michael Foord, unless otherwise noted. The enhanced version of the module is downloadable separately for use with Python versions 2.4 to 2.6, -packaged as the :mod:`unittest2` package, from -https://pypi.org/project/unittest2. +packaged as the :mod:`!unittest2` package, from :pypi:`unittest2`. When used from the command line, the module can automatically discover tests. It's not as fancy as `py.test <https://pytest.org>`__ or @@ -1936,19 +1938,20 @@ GvR worked on merging them into Python's version of :mod:`unittest`. differences in the two strings. This comparison is now used by default when Unicode strings are compared with :meth:`~unittest.TestCase.assertEqual`. -* :meth:`~unittest.TestCase.assertRegexpMatches` and - :meth:`~unittest.TestCase.assertNotRegexpMatches` checks whether the +* :meth:`assertRegexpMatches() <unittest.TestCase.assertRegex>` and + :meth:`assertNotRegexpMatches() <unittest.TestCase.assertNotRegex>` checks whether the first argument is a string matching or not matching the regular expression provided as the second argument (:issue:`8038`). -* :meth:`~unittest.TestCase.assertRaisesRegexp` checks whether a particular exception +* :meth:`assertRaisesRegexp() <unittest.TestCase.assertRaisesRegex>` checks + whether a particular exception is raised, and then also checks that the string representation of the exception matches the provided regular expression. * :meth:`~unittest.TestCase.assertIn` and :meth:`~unittest.TestCase.assertNotIn` tests whether *first* is or is not in *second*. -* :meth:`~unittest.TestCase.assertItemsEqual` tests whether two provided sequences +* :meth:`assertItemsEqual() <unittest.TestCase.assertCountEqual>` tests whether two provided sequences contain the same elements. * :meth:`~unittest.TestCase.assertSetEqual` compares whether two sets are equal, and @@ -1964,7 +1967,7 @@ GvR worked on merging them into Python's version of :mod:`unittest`. * :meth:`~unittest.TestCase.assertDictEqual` compares two dictionaries and reports the differences; it's now used by default when you compare two dictionaries - using :meth:`~unittest.TestCase.assertEqual`. :meth:`~unittest.TestCase.assertDictContainsSubset` checks whether + using :meth:`~unittest.TestCase.assertEqual`. :meth:`!assertDictContainsSubset` checks whether all of the key/value pairs in *first* are found in *second*. * :meth:`~unittest.TestCase.assertAlmostEqual` and :meth:`~unittest.TestCase.assertNotAlmostEqual` test @@ -2021,8 +2024,8 @@ version 1.3. Some of the new features are: p = ET.XMLParser(encoding='utf-8') t = ET.XML("""<root/>""", parser=p) - Errors in parsing XML now raise a :exc:`ParseError` exception, whose - instances have a :attr:`position` attribute + Errors in parsing XML now raise a :exc:`~xml.etree.ElementTree.ParseError` exception, whose + instances have a :attr:`!position` attribute containing a (*line*, *column*) tuple giving the location of the problem. * ElementTree's code for converting trees to a string has been @@ -2032,7 +2035,8 @@ version 1.3. Some of the new features are: "xml" (the default), "html", or "text". HTML mode will output empty elements as ``<empty></empty>`` instead of ``<empty/>``, and text mode will skip over elements and only output the text chunks. If - you set the :attr:`tag` attribute of an element to ``None`` but + you set the :attr:`~xml.etree.ElementTree.Element.tag` attribute of an + element to ``None`` but leave its children in place, the element will be omitted when the tree is written out, so you don't need to do more extensive rearrangement to remove a single element. @@ -2062,14 +2066,14 @@ version 1.3. Some of the new features are: # Outputs <root><item>1</item>...</root> print ET.tostring(new) -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.iter` yields the children of the element as a generator. It's also possible to write ``for child in elem:`` to loop over an element's children. The existing method - :meth:`getiterator` is now deprecated, as is :meth:`getchildren` + :meth:`!getiterator` is now deprecated, as is :meth:`!getchildren` which constructs and returns a list of children. -* New :class:`Element` method: +* New :class:`~xml.etree.ElementTree.Element` method: :meth:`~xml.etree.ElementTree.Element.itertext` yields all chunks of text that are descendants of the element. For example:: @@ -2128,7 +2132,7 @@ Changes to Python's build process and to the C API include: only the filename, function name, and first line number are required. This is useful for extension modules that are attempting to construct a more useful traceback stack. Previously such - extensions needed to call :c:func:`PyCode_New`, which had many + extensions needed to call :c:func:`!PyCode_New`, which had many more arguments. (Added by Jeffrey Yasskin.) * New function: :c:func:`PyErr_NewExceptionWithDoc` creates a new @@ -2173,8 +2177,7 @@ Changes to Python's build process and to the C API include: whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. - Security issue reported as `CVE-2008-5983 - <https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5983>`_; + Security issue reported as :cve:`2008-5983`; discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: @@ -2225,7 +2228,7 @@ Changes to Python's build process and to the C API include: (Fixed by Thomas Wouters; :issue:`1590864`.) * The :c:func:`Py_Finalize` function now calls the internal - :func:`threading._shutdown` function; this prevents some exceptions from + :func:`!threading._shutdown` function; this prevents some exceptions from being raised when an interpreter shuts down. (Patch by Adam Olsen; :issue:`1722344`.) @@ -2240,7 +2243,7 @@ Changes to Python's build process and to the C API include: Heller; :issue:`3102`.) * New configure option: the :option:`!--with-system-expat` switch allows - building the :mod:`pyexpat` module to use the system Expat library. + building the :mod:`pyexpat <xml.parsers.expat>` module to use the system Expat library. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`7609`.) * New configure option: the @@ -2327,34 +2330,34 @@ Port-Specific Changes: Windows * The :mod:`msvcrt` module now contains some constants from the :file:`crtassem.h` header file: - :data:`CRT_ASSEMBLY_VERSION`, - :data:`VC_ASSEMBLY_PUBLICKEYTOKEN`, - and :data:`LIBRARIES_ASSEMBLY_NAME_PREFIX`. + :data:`~msvcrt.CRT_ASSEMBLY_VERSION`, + :data:`~msvcrt.VC_ASSEMBLY_PUBLICKEYTOKEN`, + and :data:`~msvcrt.LIBRARIES_ASSEMBLY_NAME_PREFIX`. (Contributed by David Cournapeau; :issue:`4365`.) -* The :mod:`_winreg` module for accessing the registry now implements - the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` +* The :mod:`_winreg <winreg>` module for accessing the registry now implements + the :func:`~winreg.CreateKeyEx` and :func:`~winreg.DeleteKeyEx` functions, extended versions of previously supported functions that - take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, - :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` + take several extra arguments. The :func:`~winreg.DisableReflectionKey`, + :func:`~winreg.EnableReflectionKey`, and :func:`~winreg.QueryReflectionKey` were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) -* The new :c:func:`_beginthreadex` API is used to start threads, and +* The new :c:func:`!_beginthreadex` API is used to start threads, and the native thread-local storage functions are now used. (Contributed by Kristján Valur Jónsson; :issue:`3582`.) * The :func:`os.kill` function now works on Windows. The signal value - can be the constants :const:`CTRL_C_EVENT`, - :const:`CTRL_BREAK_EVENT`, or any integer. The first two constants + can be the constants :const:`~signal.CTRL_C_EVENT`, + :const:`~signal.CTRL_BREAK_EVENT`, or any integer. The first two constants will send :kbd:`Control-C` and :kbd:`Control-Break` keystroke events to - subprocesses; any other value will use the :c:func:`TerminateProcess` + subprocesses; any other value will use the :c:func:`!TerminateProcess` API. (Contributed by Miki Tebeka; :issue:`1220212`.) * The :func:`os.listdir` function now correctly fails for an empty path. (Fixed by Hirokazu Yamamoto; :issue:`5913`.) -* The :mod:`mimelib` module will now read the MIME database from +* The :mod:`mimetypes` module will now read the MIME database from the Windows registry when initializing. (Patch by Gabriel Genellina; :issue:`4969`.) @@ -2383,8 +2386,8 @@ Port-Specific Changes: Mac OS X Port-Specific Changes: FreeBSD ----------------------------------- -* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with - :func:`~socket.getsockopt`/:func:`~socket.setsockopt` to select an +* FreeBSD 7.1's :const:`!SO_SETFIB` constant, used with the :func:`~socket.socket` methods + :func:`~socket.socket.getsockopt`/:func:`~socket.socket.setsockopt` to select an alternate routing table, is now available in the :mod:`socket` module. (Added by Kyle VanderBeek; :issue:`8235`.) @@ -2404,7 +2407,7 @@ Other Changes and Fixes :issue:`5464`.) * When importing a module from a :file:`.pyc` or :file:`.pyo` file - with an existing :file:`.py` counterpart, the :attr:`co_filename` + with an existing :file:`.py` counterpart, the :attr:`~codeobject.co_filename` attributes of the resulting code objects are overwritten when the original filename is obsolete. This can happen if the file has been renamed, moved, or is accessed through different paths. (Patch by @@ -2439,7 +2442,7 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :func:`range` function processes its arguments more - consistently; it will now call :meth:`__int__` on non-float, + consistently; it will now call :meth:`~object.__int__` on non-float, non-integer arguments that are supplied to it. (Fixed by Alexander Belopolsky; :issue:`1533`.) @@ -2449,13 +2452,13 @@ that may require changes to your code: (Changed by Eric Smith; :issue:`5920`.) * Because of an optimization for the :keyword:`with` statement, the special - methods :meth:`__enter__` and :meth:`__exit__` must belong to the object's + methods :meth:`~object.__enter__` and :meth:`~object.__exit__` must belong to the object's type, and cannot be directly attached to the object's instance. This affects new-style classes (derived from :class:`object`) and C extension types. (:issue:`6101`.) * Due to a bug in Python 2.6, the *exc_value* parameter to - :meth:`__exit__` methods was often the string representation of the + :meth:`~object.__exit__` methods was often the string representation of the exception, not an instance. This was fixed in 2.7, so *exc_value* will be an instance as expected. (Fixed by Florent Xicluna; :issue:`7853`.) @@ -2484,13 +2487,13 @@ In the standard library: (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) -* The ElementTree library, :mod:`xml.etree`, no longer escapes +* The :mod:`xml.etree.ElementTree` library no longer escapes ampersands and angle brackets when outputting an XML processing instruction (which looks like ``<?xml-stylesheet href="#style1"?>``) or comment (which looks like ``<!-- comment -->``). (Patch by Neil Muller; :issue:`2746`.) -* The :meth:`~StringIO.StringIO.readline` method of :class:`~StringIO.StringIO` objects now does +* The :meth:`!readline` method of :class:`~io.StringIO` objects now does nothing when a negative length is requested, as other file-like objects do. (:issue:`7348`). @@ -2506,7 +2509,7 @@ In the standard library: which raises an exception if there's an error. (Changed by Lars Gustäbel; :issue:`7357`.) -* The :mod:`urlparse` module's :func:`~urlparse.urlsplit` now handles +* The :mod:`urlparse <urllib.parse>` module's :func:`~urllib.parse.urlsplit` now handles unknown URL schemes in a fashion compliant with :rfc:`3986`: if the URL is of the form ``"<something>://..."``, the text before the ``://`` is treated as the scheme, even if it's a made-up scheme that @@ -2575,11 +2578,11 @@ Two new environment variables for debug mode -------------------------------------------- In debug mode, the ``[xxx refs]`` statistic is not written by default, the -:envvar:`PYTHONSHOWREFCOUNT` environment variable now must also be set. +:envvar:`!PYTHONSHOWREFCOUNT` environment variable now must also be set. (Contributed by Victor Stinner; :issue:`31733`.) When Python is compiled with ``COUNT_ALLOC`` defined, allocation counts are no -longer dumped by default anymore: the :envvar:`PYTHONSHOWALLOCCOUNT` environment +longer dumped by default anymore: the :envvar:`!PYTHONSHOWALLOCCOUNT` environment variable must now also be set. Moreover, allocation counts are now dumped into stderr, rather than stdout. (Contributed by Victor Stinner; :issue:`31692`.) @@ -2621,7 +2624,7 @@ with the first of those changes appearing in the Python 2.7.7 release. 2 applications. (Contributed by Alex Gaynor; :issue:`21304`.) * OpenSSL 1.0.1h was upgraded for the official Windows installers published on - python.org. (contributed by Zachary Ware in :issue:`21671` for CVE-2014-0224) + python.org. (Contributed by Zachary Ware in :issue:`21671` for :cve:`2014-0224`.) :pep:`466` related features added in Python 2.7.9: @@ -2709,8 +2712,9 @@ and :ref:`setuptools-index`. PEP 476: Enabling certificate verification by default for stdlib http clients ----------------------------------------------------------------------------- -:pep:`476` updated :mod:`httplib` and modules which use it, such as -:mod:`urllib2` and :mod:`xmlrpclib`, to now verify that the server +:pep:`476` updated :mod:`httplib <http>` and modules which use it, such as +:mod:`urllib2 <urllib.request>` and :mod:`xmlrpclib <xmlrpc.client>`, to now +verify that the server presents a certificate which is signed by a Certificate Authority in the platform trust store and whose hostname matches the hostname being requested by default, significantly improving security for many applications. This @@ -2751,7 +2755,7 @@ entire Python process back to the default permissive behaviour of Python 2.7.8 and earlier. For cases where the connection establishment code can't be modified, but the -overall application can be, the new :func:`ssl._https_verify_certificates` +overall application can be, the new :func:`!ssl._https_verify_certificates` function can be used to adjust the default behaviour at runtime. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index b0c2529e780213..888e6279754fc2 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -337,7 +337,7 @@ changed. (However, the standard library remains ASCII-only with the exception of contributor names in comments.) -* The :mod:`StringIO` and :mod:`cStringIO` modules are gone. Instead, +* The :mod:`!StringIO` and :mod:`!cStringIO` modules are gone. Instead, import the :mod:`io` module and use :class:`io.StringIO` or :class:`io.BytesIO` for text and data respectively. @@ -555,15 +555,15 @@ very extensive changes to the standard library. :pep:`3108` is the reference for the major changes to the library. Here's a capsule review: -* Many old modules were removed. Some, like :mod:`gopherlib` (no - longer used) and :mod:`md5` (replaced by :mod:`hashlib`), were +* Many old modules were removed. Some, like :mod:`!gopherlib` (no + longer used) and :mod:`!md5` (replaced by :mod:`hashlib`), were already deprecated by :pep:`4`. Others were removed as a result of the removal of support for various platforms such as Irix, BeOS and Mac OS 9 (see :pep:`11`). Some modules were also selected for removal in Python 3.0 due to lack of use or because a better replacement exists. See :pep:`3108` for an exhaustive list. -* The :mod:`bsddb3` package was removed because its presence in the +* The :mod:`!bsddb3` package was removed because its presence in the core standard library has proved over time to be a particular burden for the core developers due to testing instability and Berkeley DB's release schedule. However, the package is alive and well, @@ -588,45 +588,45 @@ review: * A common pattern in Python 2.x is to have one version of a module implemented in pure Python, with an optional accelerated version implemented as a C extension; for example, :mod:`pickle` and - :mod:`cPickle`. This places the burden of importing the accelerated + :mod:`!cPickle`. This places the burden of importing the accelerated version and falling back on the pure Python version on each user of these modules. In Python 3.0, the accelerated versions are considered implementation details of the pure Python versions. Users should always import the standard version, which attempts to import the accelerated version and falls back to the pure Python - version. The :mod:`pickle` / :mod:`cPickle` pair received this + version. The :mod:`pickle` / :mod:`!cPickle` pair received this treatment. The :mod:`profile` module is on the list for 3.1. The - :mod:`StringIO` module has been turned into a class in the :mod:`io` + :mod:`!StringIO` module has been turned into a class in the :mod:`io` module. * Some related modules have been grouped into packages, and usually the submodule names have been simplified. The resulting new packages are: - * :mod:`dbm` (:mod:`anydbm`, :mod:`dbhash`, :mod:`dbm`, - :mod:`dumbdbm`, :mod:`gdbm`, :mod:`whichdb`). + * :mod:`dbm` (:mod:`!anydbm`, :mod:`!dbhash`, :mod:`!dbm`, + :mod:`!dumbdbm`, :mod:`!gdbm`, :mod:`!whichdb`). - * :mod:`html` (:mod:`HTMLParser`, :mod:`htmlentitydefs`). + * :mod:`html` (:mod:`!HTMLParser`, :mod:`!htmlentitydefs`). - * :mod:`http` (:mod:`httplib`, :mod:`BaseHTTPServer`, - :mod:`CGIHTTPServer`, :mod:`SimpleHTTPServer`, :mod:`Cookie`, - :mod:`cookielib`). + * :mod:`http` (:mod:`!httplib`, :mod:`!BaseHTTPServer`, + :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, + :mod:`!cookielib`). * :mod:`tkinter` (all :mod:`Tkinter`-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. - * :mod:`urllib` (:mod:`urllib`, :mod:`urllib2`, :mod:`urlparse`, - :mod:`robotparse`). + * :mod:`urllib` (:mod:`!urllib`, :mod:`!urllib2`, :mod:`!urlparse`, + :mod:`!robotparse`). - * :mod:`xmlrpc` (:mod:`xmlrpclib`, :mod:`DocXMLRPCServer`, - :mod:`SimpleXMLRPCServer`). + * :mod:`xmlrpc` (:mod:`!xmlrpclib`, :mod:`!DocXMLRPCServer`, + :mod:`!SimpleXMLRPCServer`). Some other changes to standard library modules, not covered by :pep:`3108`: -* Killed :mod:`sets`. Use the built-in :func:`set` class. +* Killed :mod:`!sets`. Use the built-in :func:`set` class. * Cleanup of the :mod:`sys` module: removed :func:`sys.exitfunc`, :func:`sys.exc_clear`, :data:`sys.exc_type`, :data:`sys.exc_value`, @@ -642,13 +642,13 @@ Some other changes to standard library modules, not covered by * Cleanup of the :mod:`operator` module: removed :func:`sequenceIncludes` and :func:`isCallable`. -* Cleanup of the :mod:`thread` module: :func:`acquire_lock` and - :func:`release_lock` are gone; use :func:`acquire` and - :func:`release` instead. +* Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and + :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and + :meth:`~threading.Lock.release` instead. * Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. -* The :mod:`new` module is gone. +* The :mod:`!new` module is gone. * The functions :func:`os.tmpnam`, :func:`os.tempnam` and :func:`os.tmpfile` have been removed in favor of the :mod:`tempfile` @@ -711,7 +711,7 @@ new powerful features added: {Exception}({args})` instead of :samp:`raise {Exception}, {args}`. Additionally, you can no longer explicitly specify a traceback; instead, if you *have* to do this, you can assign directly to the - :attr:`__traceback__` attribute (see below). + :attr:`~BaseException.__traceback__` attribute (see below). * :pep:`3110`: Catching exceptions. You must now use :samp:`except {SomeException} as {variable}` instead @@ -725,7 +725,7 @@ new powerful features added: handler block. This usually happens due to a bug in the handler block; we call this a *secondary* exception. In this case, the original exception (that was being handled) is saved as the - :attr:`__context__` attribute of the secondary exception. + :attr:`~BaseException.__context__` attribute of the secondary exception. Explicit chaining is invoked with this syntax:: raise SecondaryException() from primary_exception @@ -733,14 +733,15 @@ new powerful features added: (where *primary_exception* is any expression that produces an exception object, probably an exception that was previously caught). In this case, the primary exception is stored on the - :attr:`__cause__` attribute of the secondary exception. The + :attr:`~BaseException.__cause__` attribute of the secondary exception. The traceback printed when an unhandled exception occurs walks the chain - of :attr:`__cause__` and :attr:`__context__` attributes and prints a + of :attr:`!__cause__` and :attr:`~BaseException.__context__` attributes and + prints a separate traceback for each component of the chain, with the primary exception at the top. (Java users may recognize this behavior.) * :pep:`3134`: Exception objects now store their traceback as the - :attr:`__traceback__` attribute. This means that an exception + :attr:`~BaseException.__traceback__` attribute. This means that an exception object now contains all the information pertaining to an exception, and there are fewer reasons to use :func:`sys.exc_info` (though the latter is not removed). @@ -779,14 +780,15 @@ Operators And Special Methods * Removed support for :attr:`__members__` and :attr:`__methods__`. -* The function attributes named :attr:`func_X` have been renamed to - use the :data:`__X__` form, freeing up these names in the function +* The function attributes named :attr:`!func_X` have been renamed to + use the :attr:`!__X__` form, freeing up these names in the function attribute namespace for user-defined attributes. To wit, - :attr:`func_closure`, :attr:`func_code`, :attr:`func_defaults`, - :attr:`func_dict`, :attr:`func_doc`, :attr:`func_globals`, - :attr:`func_name` were renamed to :attr:`__closure__`, - :attr:`__code__`, :attr:`__defaults__`, :attr:`~object.__dict__`, - :attr:`__doc__`, :attr:`__globals__`, :attr:`~definition.__name__`, + :attr:`!func_closure`, :attr:`!func_code`, :attr:`!func_defaults`, + :attr:`!func_dict`, :attr:`!func_doc`, :attr:`!func_globals`, + :attr:`!func_name` were renamed to :attr:`~function.__closure__`, + :attr:`~function.__code__`, :attr:`~function.__defaults__`, + :attr:`~function.__dict__`, :attr:`~function.__doc__`, + :attr:`~function.__globals__`, :attr:`~function.__name__`, respectively. * :meth:`!__nonzero__` is now :meth:`~object.__bool__`. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index e237179f4b1829..69b273e58385d2 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -80,6 +80,28 @@ Support was also added for third-party tools like `PyYAML <https://pyyaml.org/>` PEP written by Armin Ronacher and Raymond Hettinger. Implementation written by Raymond Hettinger. +Since an ordered dictionary remembers its insertion order, it can be used +in conjunction with sorting to make a sorted dictionary:: + + >>> # regular unsorted dictionary + >>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2} + + >>> # dictionary sorted by key + >>> OrderedDict(sorted(d.items(), key=lambda t: t[0])) + OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)]) + + >>> # dictionary sorted by value + >>> OrderedDict(sorted(d.items(), key=lambda t: t[1])) + OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)]) + + >>> # dictionary sorted by length of the key string + >>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0]))) + OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)]) + +The new sorted dictionaries maintain their sort order when entries +are deleted. But when new keys are added, the keys are appended +to the end and the sort is not maintained. + PEP 378: Format Specifier for Thousands Separator ================================================= @@ -152,7 +174,7 @@ Some smaller changes made to the core Python language are: (Contributed by Eric Smith; :issue:`5237`.) -* The :func:`string.maketrans` function is deprecated and is replaced by new +* The :func:`!string.maketrans` function is deprecated and is replaced by new static methods, :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, :class:`bytes`, and @@ -169,7 +191,7 @@ Some smaller changes made to the core Python language are: ... if '<critical>' in line: ... outfile.write(line) - With the new syntax, the :func:`contextlib.nested` function is no longer + With the new syntax, the :func:`!contextlib.nested` function is no longer needed and is now deprecated. (Contributed by Georg Brandl and Mattias Brändström; @@ -359,16 +381,20 @@ New, Improved, and Deprecated Modules x / 0 In addition, several new assertion methods were added including - :func:`assertSetEqual`, :func:`assertDictEqual`, - :func:`assertDictContainsSubset`, :func:`assertListEqual`, - :func:`assertTupleEqual`, :func:`assertSequenceEqual`, - :func:`assertRaisesRegexp`, :func:`assertIsNone`, - and :func:`assertIsNotNone`. + :meth:`~unittest.TestCase.assertSetEqual`, + :meth:`~unittest.TestCase.assertDictEqual`, + :meth:`!assertDictContainsSubset`, + :meth:`~unittest.TestCase.assertListEqual`, + :meth:`~unittest.TestCase.assertTupleEqual`, + :meth:`~unittest.TestCase.assertSequenceEqual`, + :meth:`assertRaisesRegexp() <unittest.TestCase.assertRaisesRegex>`, + :meth:`~unittest.TestCase.assertIsNone`, + and :meth:`~unittest.TestCase.assertIsNotNone`. (Contributed by Benjamin Peterson and Antoine Pitrou.) -* The :mod:`io` module has three new constants for the :meth:`seek` - method :data:`SEEK_SET`, :data:`SEEK_CUR`, and :data:`SEEK_END`. +* The :mod:`io` module has three new constants for the :meth:`~io.IOBase.seek` + method: :data:`~os.SEEK_SET`, :data:`~os.SEEK_CUR`, and :data:`~os.SEEK_END`. * The :data:`sys.version_info` tuple is now a named tuple:: diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index df821d68eb8d9f..b939ccd17903f2 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -352,7 +352,7 @@ was expecting an indentation, including the location of the statement: AttributeErrors ~~~~~~~~~~~~~~~ -When printing :exc:`AttributeError`, :c:func:`PyErr_Display` will offer +When printing :exc:`AttributeError`, :c:func:`!PyErr_Display` will offer suggestions of similar attribute names in the object that the exception was raised from: @@ -366,14 +366,14 @@ raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. NameErrors ~~~~~~~~~~ -When printing :exc:`NameError` raised by the interpreter, :c:func:`PyErr_Display` +When printing :exc:`NameError` raised by the interpreter, :c:func:`!PyErr_Display` will offer suggestions of similar variable names in the function that the exception was raised from: @@ -388,7 +388,7 @@ was raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error, which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. @@ -399,10 +399,14 @@ PEP 626: Precise line numbers for debugging and other tools PEP 626 brings more precise and reliable line numbers for debugging, profiling and coverage tools. Tracing events, with the correct line number, are generated for all lines of code executed and only for lines of code that are executed. -The ``f_lineno`` attribute of frame objects will always contain the expected line number. +The :attr:`~frame.f_lineno` attribute of frame objects will always contain the +expected line number. -The ``co_lnotab`` attribute of code objects is deprecated and will be removed in 3.12. -Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead. +The :attr:`~codeobject.co_lnotab` attribute of +:ref:`code objects <code-objects>` is deprecated and +will be removed in 3.12. +Code that needs to convert from offset to line number should use the new +:meth:`~codeobject.co_lines` method instead. PEP 634: Structural Pattern Matching ------------------------------------ @@ -686,7 +690,7 @@ are in :pep:`635`, and a longer tutorial is in :pep:`636`. Optional ``EncodingWarning`` and ``encoding="locale"`` option ------------------------------------------------------------- -The default encoding of :class:`TextIOWrapper` and :func:`open` is +The default encoding of :class:`~io.TextIOWrapper` and :func:`open` is platform and locale dependent. Since UTF-8 is used on most Unix platforms, omitting ``encoding`` option when opening UTF-8 files (e.g. JSON, YAML, TOML, Markdown) is a very common bug. For example:: @@ -781,7 +785,7 @@ especially when forward references or invalid types were involved. Compare:: StrCache = 'Cache[str]' # a type alias LOG_PREFIX = 'LOG[DEBUG]' # a module constant -Now the :mod:`typing` module has a special value :data:`TypeAlias` +Now the :mod:`typing` module has a special value :data:`~typing.TypeAlias` which lets you declare type aliases more explicitly:: StrCache: TypeAlias = 'Cache[str]' # a type alias @@ -794,10 +798,10 @@ See :pep:`613` for more details. PEP 647: User-Defined Type Guards --------------------------------- -:data:`TypeGuard` has been added to the :mod:`typing` module to annotate +:data:`~typing.TypeGuard` has been added to the :mod:`typing` module to annotate type guard functions and improve information provided to static type checkers -during type narrowing. For more information, please see :data:`TypeGuard`\ 's -documentation, and :pep:`647`. +during type narrowing. For more information, please see +:data:`~typing.TypeGuard`\ 's documentation, and :pep:`647`. (Contributed by Ken Jin and Guido van Rossum in :issue:`43766`. PEP written by Eric Traut.) @@ -824,7 +828,7 @@ Other Language Changes :meth:`~object.__index__` method). (Contributed by Serhiy Storchaka in :issue:`37999`.) -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will +* If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. (Contributed by Alex Shkop in :issue:`38302`.) @@ -968,8 +972,8 @@ and objects representing asynchronously released resources. Add asynchronous context manager support to :func:`contextlib.nullcontext`. (Contributed by Tom Gringauz in :issue:`41543`.) -Add :class:`AsyncContextDecorator`, for supporting usage of async context managers -as decorators. +Add :class:`~contextlib.AsyncContextDecorator`, for supporting usage of async +context managers as decorators. curses ------ @@ -1085,8 +1089,8 @@ encodings enum ---- -:class:`Enum` :func:`__repr__` now returns ``enum_name.member_name`` and -:func:`__str__` now returns ``member_name``. Stdlib enums available as +:class:`~enum.Enum` :func:`~object.__repr__` now returns ``enum_name.member_name`` and +:func:`~object.__str__` now returns ``member_name``. Stdlib enums available as module constants have a :func:`repr` of ``module_name.member_name``. (Contributed by Ethan Furman in :issue:`40066`.) @@ -1100,7 +1104,7 @@ Add *encoding* and *errors* parameters in :func:`fileinput.input` and :class:`fileinput.FileInput`. (Contributed by Inada Naoki in :issue:`43712`.) -:func:`fileinput.hook_compressed` now returns :class:`TextIOWrapper` object +:func:`fileinput.hook_compressed` now returns :class:`~io.TextIOWrapper` object when *mode* is "r" and file is compressed, like uncompressed files. (Contributed by Inada Naoki in :issue:`5758`.) @@ -1198,12 +1202,12 @@ Feature parity with ``importlib_metadata`` 4.6 :ref:`importlib.metadata entry points <entry-points>` now provide a nicer experience for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. See the Compatibility +:ref:`importlib.metadata.EntryPoints <entry-points>` class. See the Compatibility Note in the docs for more info on the deprecation and usage. -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. +Added :ref:`importlib.metadata.packages_distributions() <package-distributions>` +for resolving top-level Python modules and packages to their +:ref:`importlib.metadata.Distribution <distributions>`. inspect ------- @@ -1220,7 +1224,7 @@ best practice for accessing the annotations dict defined on any Python object; for more information on best practices for working with annotations, please see :ref:`annotations-howto`. Relatedly, :func:`inspect.signature`, -:func:`inspect.Signature.from_callable`, and :func:`inspect.Signature.from_function` +:func:`inspect.Signature.from_callable`, and :func:`!inspect.Signature.from_function` now call :func:`inspect.get_annotations` to retrieve annotations. This means :func:`inspect.signature` and :func:`inspect.Signature.from_callable` can also now un-stringize stringized annotations. @@ -1276,7 +1280,7 @@ Add negative indexing support to :attr:`PurePath.parents (Contributed by Yaroslav Pankovych in :issue:`21041`.) Add :meth:`Path.hardlink_to <pathlib.Path.hardlink_to>` method that -supersedes :meth:`~pathlib.Path.link_to`. The new method has the same argument +supersedes :meth:`!link_to`. The new method has the same argument order as :meth:`~pathlib.Path.symlink_to`. (Contributed by Barney Gale in :issue:`39950`.) @@ -1312,8 +1316,8 @@ pyclbr ------ Add an ``end_lineno`` attribute to the ``Function`` and ``Class`` -objects in the tree returned by :func:`pyclbr.readline` and -:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +objects in the tree returned by :func:`pyclbr.readmodule` and +:func:`pyclbr.readmodule_ex`. It matches the existing (start) ``lineno``. (Contributed by Aviral Srivastava in :issue:`38307`.) shelve @@ -1480,9 +1484,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`runtime_checkable`. Previously, these checks +are decorated with :func:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`runtime_checkable` decorator +subclasses with the :func:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1513,6 +1517,13 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + xml --- @@ -1584,8 +1595,8 @@ Optimizations :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. (Contributed by Donghee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) -* :class:`BZ2File` performance is improved by removing internal ``RLock``. - This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous +* :class:`~bz2.BZ2File` performance is improved by removing internal ``RLock``. + This makes :class:`!BZ2File` thread unsafe in the face of multiple simultaneous readers or writers, just like its equivalent classes in :mod:`gzip` and :mod:`lzma` have always been. (Contributed by Inada Naoki in :issue:`43785`.) @@ -1609,7 +1620,7 @@ Deprecated cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), + (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system @@ -1636,7 +1647,7 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` instead. (Contributed by Brett Cannon in :issue:`26131`.) -* :meth:`zimport.zipimporter.load_module` has been deprecated in +* :meth:`!zimport.zipimporter.load_module` has been deprecated in preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1738,7 +1749,7 @@ Deprecated (Contributed by Jelle Zijlstra in :gh:`87889`.) -* :meth:`pathlib.Path.link_to` is deprecated and slated for removal in +* :meth:`!pathlib.Path.link_to` is deprecated and slated for removal in Python 3.12. Use :meth:`pathlib.Path.hardlink_to` instead. (Contributed by Barney Gale in :issue:`39950`.) @@ -1748,28 +1759,28 @@ Deprecated * The following :mod:`ssl` features have been deprecated since Python 3.6, Python 3.7, or OpenSSL 1.1.0 and will be removed in 3.11: - * :data:`~ssl.OP_NO_SSLv2`, :data:`~ssl.OP_NO_SSLv3`, :data:`~ssl.OP_NO_TLSv1`, - :data:`~ssl.OP_NO_TLSv1_1`, :data:`~ssl.OP_NO_TLSv1_2`, and - :data:`~ssl.OP_NO_TLSv1_3` are replaced by - :attr:`sslSSLContext.minimum_version` and - :attr:`sslSSLContext.maximum_version`. + * :data:`!OP_NO_SSLv2`, :data:`!OP_NO_SSLv3`, :data:`!OP_NO_TLSv1`, + :data:`!OP_NO_TLSv1_1`, :data:`!OP_NO_TLSv1_2`, and + :data:`!OP_NO_TLSv1_3` are replaced by + :attr:`~ssl.SSLContext.minimum_version` and + :attr:`~ssl.SSLContext.maximum_version`. - * :data:`~ssl.PROTOCOL_SSLv2`, :data:`~ssl.PROTOCOL_SSLv3`, - :data:`~ssl.PROTOCOL_SSLv23`, :data:`~ssl.PROTOCOL_TLSv1`, - :data:`~ssl.PROTOCOL_TLSv1_1`, :data:`~ssl.PROTOCOL_TLSv1_2`, and - :const:`~ssl.PROTOCOL_TLS` are deprecated in favor of + * :data:`!PROTOCOL_SSLv2`, :data:`!PROTOCOL_SSLv3`, + :data:`!PROTOCOL_SSLv23`, :data:`!PROTOCOL_TLSv1`, + :data:`!PROTOCOL_TLSv1_1`, :data:`!PROTOCOL_TLSv1_2`, and + :const:`!PROTOCOL_TLS` are deprecated in favor of :const:`~ssl.PROTOCOL_TLS_CLIENT` and :const:`~ssl.PROTOCOL_TLS_SERVER` - * :func:`~ssl.wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` + * :func:`!wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` - * :func:`~ssl.match_hostname` + * :func:`!match_hostname` - * :func:`~ssl.RAND_pseudo_bytes`, :func:`~ssl.RAND_egd` + * :func:`!RAND_pseudo_bytes`, :func:`!RAND_egd` * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. -* The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +* The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a :ref:`debug build of Python <debug-build>`. (Contributed by Victor Stinner in :issue:`44584`.) @@ -1959,11 +1970,11 @@ Changes in the C API source_buf = PyBytes_AsString(source_bytes_object); code = Py_CompileString(source_buf, filename, Py_file_input); - * For ``FrameObject`` objects, the ``f_lasti`` member now represents a wordcode + * For ``FrameObject`` objects, the :attr:`~frame.f_lasti` member now represents a wordcode offset instead of a simple offset into the bytecode string. This means that this number needs to be multiplied by 2 to be used with APIs that expect a byte offset instead (like :c:func:`PyCode_Addr2Line` for example). Notice as well that the - ``f_lasti`` member of ``FrameObject`` objects is not considered stable: please + :attr:`!f_lasti` member of ``FrameObject`` objects is not considered stable: please use :c:func:`PyFrame_GetLineNumber` instead. CPython bytecode changes @@ -2311,3 +2322,42 @@ Removed * The ``PyThreadState.use_tracing`` member has been removed to optimize Python. (Contributed by Mark Shannon in :issue:`43760`.) + + +Notable security feature in 3.10.7 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable security feature in 3.10.8 +================================== + +The deprecated :mod:`!mailcap` module now refuses to inject unsafe text +(filenames, MIME types, parameters) into shell commands. Instead of using such +text, it will warn and act as if a match was not found (or for test commands, +as if the test failed). +(Contributed by Petr Viktorin in :gh:`98966`.) + +Notable changes in 3.10.12 +========================== + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 257025da91a7ed..b601bd453f5be5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -544,8 +544,7 @@ Other CPython Implementation Changes (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the - algorithmic complexity. This is a mitigation for `CVE-2020-10735 - <https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735>`_. + algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation <int_max_str_digits>` documentation. The default limit @@ -672,7 +671,7 @@ enum * Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for :func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce - the same result as :meth:`Enum.__str__()`: for enums inheriting from + the same result as :meth:`Enum.__str__() <enum.Enum.__str__>`: for enums inheriting from :class:`~enum.ReprEnum` it will be the member's value; for all other enums it will be the enum and member name (e.g. ``Color.RED``). @@ -769,6 +768,21 @@ functools (Contributed by Yurii Karabas in :issue:`46014`.) +.. _whatsnew311-gzip: + +gzip +---- + +* The :func:`gzip.compress` function is now faster when used with the + **mtime=0** argument as it delegates the compression entirely to a single + :func:`zlib.compress` operation. There is one side effect of this change: The + gzip file header contains an "OS" byte in its header. That was traditionally + always set to a value of 255 representing "unknown" by the :mod:`gzip` + module. Now, when using :func:`~gzip.compress` with **mtime=0**, it may be + set to a different value by the underlying zlib C library Python was linked + against. + (See :gh:`112346` for details on the side effect.) + .. _whatsnew311-hashlib: hashlib @@ -1604,7 +1618,7 @@ raw, adaptive bytecode containing quickened data. New opcodes ----------- -* :opcode:`ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, +* :opcode:`!ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, used in generators and co-routines. * :opcode:`COPY_FREE_VARS`, @@ -1615,7 +1629,7 @@ New opcodes * :opcode:`MAKE_CELL`, to create :ref:`cell-objects`. -* :opcode:`CHECK_EG_MATCH` and :opcode:`PREP_RERAISE_STAR`, +* :opcode:`CHECK_EG_MATCH` and :opcode:`!PREP_RERAISE_STAR`, to handle the :ref:`new exception groups and except* <whatsnew311-pep654>` added in :pep:`654`. @@ -1630,38 +1644,38 @@ New opcodes Replaced opcodes ---------------- -+------------------------------------+-----------------------------------+-----------------------------------------+ -| Replaced Opcode(s) | New Opcode(s) | Notes | -+====================================+===================================+=========================================+ -| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | -| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | -| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`KW_NAMES` | from handling of keyword arguments; | -| | :opcode:`!CALL_METHOD` | | :opcode:`PRECALL` | allows better specialization of calls | -| | | :opcode:`PUSH_NULL` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | -| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | -| | :opcode:`!ROT_TWO` | | | -| | :opcode:`!ROT_THREE` | | | -| | :opcode:`!ROT_FOUR` | | | -| | :opcode:`!ROT_N` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | -| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`POP_JUMP_BACKWARD_IF_* | ``TRUE``, ``FALSE``, | -| | :opcode:`!POP_JUMP_IF_TRUE` | <POP_JUMP_BACKWARD_IF_TRUE>` | ``NONE`` and ``NOT_NONE`` variants | -| | | :opcode:`POP_JUMP_FORWARD_IF_* | for each direction | -| | <POP_JUMP_FORWARD_IF_TRUE>` | | -+------------------------------------+-----------------------------------+-----------------------------------------+ -| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | -| | :opcode:`!SETUP_ASYNC_WITH` | | | -+------------------------------------+-----------------------------------+-----------------------------------------+ ++------------------------------------+------------------------------------+-----------------------------------------+ +| Replaced Opcode(s) | New Opcode(s) | Notes | ++====================================+====================================+=========================================+ +| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | +| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | +| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`!KW_NAMES` | from handling of keyword arguments; | +| | :opcode:`!CALL_METHOD` | | :opcode:`!PRECALL` | allows better specialization of calls | +| | | :opcode:`PUSH_NULL` | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | +| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | +| | :opcode:`!ROT_TWO` | | | +| | :opcode:`!ROT_THREE` | | | +| | :opcode:`!ROT_FOUR` | | | +| | :opcode:`!ROT_N` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | +| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`!POP_JUMP_BACKWARD_IF_*`| ``TRUE``, ``FALSE``, | +| | :opcode:`!POP_JUMP_IF_TRUE` | | :opcode:`!POP_JUMP_FORWARD_IF_*` | ``NONE`` and ``NOT_NONE`` variants | +| | | for each direction | +| | | | ++------------------------------------+------------------------------------+-----------------------------------------+ +| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_ASYNC_WITH` | | | ++------------------------------------+------------------------------------+-----------------------------------------+ .. [#bytecode-jump] All jump opcodes are now relative, including the - existing :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP`. + existing :opcode:`!JUMP_IF_TRUE_OR_POP` and :opcode:`!JUMP_IF_FALSE_OR_POP`. The argument is now an offset from the current instruction rather than an absolute location. @@ -1747,7 +1761,7 @@ Modules (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in :gh:`68966`.) -* The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +* The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) @@ -1773,7 +1787,7 @@ Standard Library * the :class:`!configparser.SafeConfigParser` class * the :attr:`!configparser.ParsingError.filename` property - * the :meth:`configparser.RawConfigParser.readfp` method + * the :meth:`!configparser.RawConfigParser.readfp` method (Contributed by Hugo van Kemenade in :issue:`45173`.) @@ -1789,21 +1803,21 @@ Standard Library and will be removed in a future Python version, due to not supporting resources located within package subdirectories: - * :func:`importlib.resources.contents` - * :func:`importlib.resources.is_resource` - * :func:`importlib.resources.open_binary` - * :func:`importlib.resources.open_text` - * :func:`importlib.resources.read_binary` - * :func:`importlib.resources.read_text` - * :func:`importlib.resources.path` + * :func:`!importlib.resources.contents` + * :func:`!importlib.resources.is_resource` + * :func:`!importlib.resources.open_binary` + * :func:`!importlib.resources.open_text` + * :func:`!importlib.resources.read_binary` + * :func:`!importlib.resources.read_text` + * :func:`!importlib.resources.path` * The :func:`locale.getdefaultlocale` function is deprecated and will be - removed in Python 3.13. Use :func:`locale.setlocale`, + removed in Python 3.15. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) <locale.getpreferredencoding>` and :func:`locale.getlocale` functions instead. (Contributed by Victor Stinner in :gh:`90817`.) -* The :func:`locale.resetlocale` function is deprecated and will be +* The :func:`!locale.resetlocale` function is deprecated and will be removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. (Contributed by Victor Stinner in :gh:`90817`.) @@ -1860,7 +1874,7 @@ Standard Library (Contributed by Erlend E. Aasland in :issue:`5846`.) -* :meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed +* :meth:`!unittest.TestProgram.usageExit` is marked deprecated, to be removed in 3.13. (Contributed by Carlos Damázio in :gh:`67048`.) @@ -1877,8 +1891,8 @@ and will be removed in Python 3.12. C APIs pending removal are :ref:`listed separately <whatsnew311-c-api-pending-removal>`. -* The :mod:`asynchat` module -* The :mod:`asyncore` module +* The :mod:`!asynchat` module +* The :mod:`!asyncore` module * The :ref:`entire distutils package <distutils-deprecated>` * The :mod:`!imp` module * The :class:`typing.io <typing.IO>` namespace @@ -1902,10 +1916,10 @@ C APIs pending removal are * :func:`!importlib.util.set_package_wrapper` * :class:`!pkgutil.ImpImporter` * :class:`!pkgutil.ImpLoader` -* :meth:`pathlib.Path.link_to` +* :meth:`!pathlib.Path.link_to` * :func:`!sqlite3.enable_shared_cache` * :func:`!sqlite3.OptimizedUnicode` -* :envvar:`PYTHONTHREADDEBUG` environment variable +* :envvar:`!PYTHONTHREADDEBUG` environment variable * The following deprecated aliases in :mod:`unittest`: ============================ =============================== =============== @@ -1967,7 +1981,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. (Contributed by Victor Stinner in :issue:`45085`.) -* Removed the :mod:`distutils` ``bdist_msi`` command deprecated in Python 3.9. +* Removed the :mod:`!distutils` ``bdist_msi`` command deprecated in Python 3.9. Use ``bdist_wheel`` (wheel packages) instead. (Contributed by Hugo van Kemenade in :issue:`45124`.) @@ -2007,7 +2021,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. because it was not used and added by mistake in previous versions. (Contributed by Nikita Sobolev in :issue:`46483`.) -* Removed the :class:`!MailmanProxy` class in the :mod:`smtpd` module, +* Removed the :class:`!MailmanProxy` class in the :mod:`!smtpd` module, as it is unusable without the external :mod:`!mailman` package. (Contributed by Donghee Na in :issue:`35800`.) @@ -2028,7 +2042,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`. (and corresponding :c:macro:`!EXPERIMENTAL_ISOLATED_SUBINTERPRETERS` macro) have been removed. -* `Pynche <https://pypi.org/project/pynche/>`_ +* :pypi:`Pynche` --- The Pythonically Natural Color and Hue Editor --- has been moved out of ``Tools/scripts`` and is `being developed independently <https://gitlab.com/warsaw/pynche/-/tree/main>`_ from the Python source tree. @@ -2295,7 +2309,7 @@ Porting to Python 3.11 as its second parameter, instead of ``PyFrameObject*``. See :pep:`523` for more details of how to use this function pointer type. -* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take +* :c:func:`!PyCode_New` and :c:func:`!PyCode_NewWithPosOnlyArgs` now take an additional ``exception_table`` argument. Using these functions should be avoided, if at all possible. To get a custom code object: create a code object using the compiler, @@ -2402,7 +2416,7 @@ Porting to Python 3.11 been included directly, consider including ``Python.h`` instead. (Contributed by Victor Stinner in :issue:`35134`.) -* The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +* The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. (Contributed by Victor Stinner in :issue:`46007`.) @@ -2458,13 +2472,14 @@ Porting to Python 3.11 * ``f_valuestack``: removed. The Python frame object is now created lazily. A side effect is that the - ``f_back`` member must not be accessed directly, since its value is now also + :attr:`~frame.f_back` member must not be accessed directly, + since its value is now also computed lazily. The :c:func:`PyFrame_GetBack` function must be called instead. - Debuggers that accessed the ``f_locals`` directly *must* call + Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call :c:func:`PyFrame_GetLocals` instead. They no longer need to call - :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`, + :c:func:`!PyFrame_FastToLocalsWithError` or :c:func:`!PyFrame_LocalsToFast`, in fact they should not call those functions. The necessary updating of the frame is now managed by the virtual machine. @@ -2603,8 +2618,8 @@ and will be removed in Python 3.12. * :c:func:`!PyUnicode_GET_DATA_SIZE` * :c:func:`!PyUnicode_GET_SIZE` * :c:func:`!PyUnicode_GetSize` -* :c:func:`PyUnicode_IS_COMPACT` -* :c:func:`PyUnicode_IS_READY` +* :c:func:`!PyUnicode_IS_COMPACT` +* :c:func:`!PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` * :c:func:`!PyUnicode_WSTR_LENGTH` * :c:func:`!_PyUnicode_AsUnicode` @@ -2659,7 +2674,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) * Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never - worked since the :c:type:`PyWeakReference` structure is opaque in the + worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) @@ -2700,4 +2715,30 @@ Removed (Contributed by Inada Naoki in :issue:`44029`.) +Notable changes in 3.11.4 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) + + +Notable changes in 3.11.5 +========================= + +OpenSSL +------- + +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + + .. _libb2: https://www.blake2.net/ diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dc0cc82475a2e4..cfc87cb9089c66 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -343,7 +343,9 @@ cores. This is currently only available through the C-API, though a Python API is :pep:`anticipated for 3.13 <554>`. Use the new :c:func:`Py_NewInterpreterFromConfig` function to -create an interpreter with its own GIL:: +create an interpreter with its own GIL: + +.. code-block:: c PyInterpreterConfig config = { .check_multi_interp_extensions = 1, @@ -650,14 +652,14 @@ asyncio making some use-cases 2x to 5x faster. (Contributed by Jacob Bower & Itamar Oren in :gh:`102853`, :gh:`104140`, and :gh:`104138`) -* On Linux, :mod:`asyncio` uses :class:`asyncio.PidfdChildWatcher` by default +* On Linux, :mod:`asyncio` uses :class:`!asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of - :class:`asyncio.ThreadedChildWatcher`. + :class:`!asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) * The event loop now uses the best available child watcher for each platform - (:class:`asyncio.PidfdChildWatcher` if supported and - :class:`asyncio.ThreadedChildWatcher` otherwise), so manually + (:class:`!asyncio.PidfdChildWatcher` if supported and + :class:`!asyncio.ThreadedChildWatcher` otherwise), so manually configuring a child watcher is not recommended. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -688,7 +690,7 @@ csv * Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to provide finer grained control of ``None`` and empty strings by - :class:`csv.writer` objects. + :class:`~csv.reader` and :class:`~csv.writer` objects. dis --- @@ -716,12 +718,15 @@ importlib.resources * :func:`importlib.resources.as_file` now supports resource directories. (Contributed by Jason R. Coombs in :gh:`97930`.) +* Rename first parameter of :func:`importlib.resources.files` to *anchor*. + (Contributed by Jason R. Coombs in :gh:`100598`.) + inspect ------- * Add :func:`inspect.markcoroutinefunction` to mark sync functions that return a :term:`coroutine` for use with :func:`inspect.iscoroutinefunction`. - (Contributed Carlton Gibson in :gh:`99247`.) + (Contributed by Carlton Gibson in :gh:`99247`.) * Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` for determining the current state of asynchronous generators. @@ -729,13 +734,12 @@ inspect * The performance of :func:`inspect.getattr_static` has been considerably improved. Most calls to the function should be at least 2x faster than they - were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex - Waygood in :gh:`103193`.) + were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) itertools --------- -* Add :class:`itertools.batched()` for collecting into even-sized +* Add :func:`itertools.batched` for collecting into even-sized tuples where the last batch may be shorter than the rest. (Contributed by Raymond Hettinger in :gh:`98363`.) @@ -746,8 +750,8 @@ math (Contributed by Raymond Hettinger in :gh:`100485`.) * Extend :func:`math.nextafter` to include a *steps* argument - for moving up or down multiple steps at a time. - (By Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) + for moving up or down multiple steps at a time. (Contributed by + Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) os -- @@ -835,8 +839,7 @@ shutil * :func:`shutil.rmtree` now accepts a new argument *onexc* which is an error handler like *onerror* but which expects an exception instance - rather than a *(typ, val, tb)* triplet. *onerror* is deprecated and - will be removed in Python 3.14. + rather than a *(typ, val, tb)* triplet. *onerror* is deprecated. (Contributed by Irit Katriel in :gh:`102828`.) * :func:`shutil.which` now consults the *PATHEXT* environment variable to @@ -923,8 +926,6 @@ tempfile * :func:`tempfile.mkdtemp` now always returns an absolute path, even if the argument provided to the *dir* parameter is a relative path. -.. _whatsnew-typing-py312: - threading --------- @@ -959,6 +960,8 @@ types :ref:`user-defined-generics` when subclassed. (Contributed by James Hilton-Balfe and Alex Waygood in :gh:`101827`.) +.. _whatsnew-typing-py312: + typing ------ @@ -1002,8 +1005,8 @@ typing :func:`runtime-checkable protocols <typing.runtime_checkable>` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with fourteen - or more members may be slower than in Python 3.11. (Contributed by Alex + faster or more. However, ``isinstance()`` checks against protocols with many + members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) * All :data:`typing.TypedDict` and :data:`typing.NamedTuple` classes now have the @@ -1159,15 +1162,15 @@ Deprecated * :mod:`asyncio`: - * The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, - :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` - and :class:`asyncio.SafeChildWatcher` are deprecated and + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) - * :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, - :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and - :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -1180,7 +1183,7 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) @@ -1248,7 +1251,7 @@ Deprecated :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing + We added the warning to raise awareness as issues encountered by code doing this are becoming more frequent. See the :func:`os.fork` documentation for more details along with `this discussion on fork being incompatible with threads <https://discuss.python.org/t/33555>`_ for *why* we're now surfacing this @@ -1258,8 +1261,8 @@ Deprecated :mod:`concurrent.futures` the fix is to use a different :mod:`multiprocessing` start method such as ``"spawn"`` or ``"forkserver"``. -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated and will be removed - in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated; + use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) * :mod:`sqlite3`: @@ -1286,10 +1289,11 @@ Deprecated * :mod:`typing`: - * :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` - and :class:`collections.abc.Sized`. (:gh:`94309`.) + * :class:`typing.Hashable` and :class:`typing.Sized`, aliases for + :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are + deprecated. (:gh:`94309`.) - * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1320,7 +1324,8 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing ``co_lnotab`` on code objects was deprecated in Python 3.10 via :pep:`626`, +* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in + Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12, therefore it will be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) @@ -1360,7 +1365,6 @@ Other modules: APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* :func:`locale.getdefaultlocale` (:gh:`90817`) * ``locale.resetlocale()`` (:gh:`90817`) * :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) * :func:`!unittest.findTestCases` (:gh:`50096`) @@ -1369,6 +1373,18 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy + <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy>`_ (:gh:`106531`) Pending Removal in Python 3.14 ------------------------------ @@ -1424,11 +1440,20 @@ and will be removed in Python 3.14. * :mod:`typing`: :class:`!typing.ByteString` -* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. - * The ``__package__`` and ``__cached__`` attributes on module objects. -* The ``co_lnotab`` attribute of code objects. +* The :attr:`~codeobject.co_lnotab` attribute of code objects. + +Pending Removal in Python 3.15 +------------------------------ + +The following APIs have been deprecated +and will be removed in Python 3.15. + +APIs: + +* :func:`locale.getdefaultlocale` (:gh:`90817`) + Pending Removal in Future Versions ---------------------------------- @@ -1440,6 +1465,11 @@ although there is currently no date scheduled for their removal. * :class:`typing.Text` (:gh:`92332`) +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return True. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + * Currently Python accepts numeric literals immediately followed by keywords, for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and ambiguous expressions like ``[0x1for x in y]`` (which can be @@ -1626,14 +1656,15 @@ locale use :func:`locale.format_string` instead. (Contributed by Victor Stinner in :gh:`94226`.) -* ``smtpd``: The module has been removed according to the schedule in :pep:`594`, +smtpd +----- + +* The ``smtpd`` module has been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.4.7 and 3.5.4. - Use aiosmtpd_ PyPI module or any other + Use the :pypi:`aiosmtpd` PyPI module or any other :mod:`asyncio`-based server instead. (Contributed by Oleg Iarygin in :gh:`93243`.) -.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ - sqlite3 ------- @@ -1670,9 +1701,8 @@ ssl instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a - SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 - <https://cwe.mitre.org/data/definitions/295.html>`_: Improper Certificate - Validation. + SNI TLS extension nor validates the server hostname. Code is subject to :cwe:`295` + (Improper Certificate Validation). (Contributed by Victor Stinner in :gh:`94199`.) unittest @@ -1712,7 +1742,7 @@ unittest * Undocumented :meth:`TestLoader.loadTestsFromModule <unittest.TestLoader.loadTestsFromModule>` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). + (deprecated and ignored since Python 3.5). * An alias of the :class:`~unittest.TextTestResult` class: ``_TextTestResult`` (deprecated in Python 3.2). @@ -1759,6 +1789,14 @@ Others (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) +* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. + (Contributed by Nikita Sobolev in :gh:`99482`.) + +* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag + from :mod:`ctypes` module. + (Contributed by Nikita Sobolev in :gh:`99285`.) + + .. _whatsnew312-porting-to-python312: Porting to Python 3.12 @@ -1793,7 +1831,7 @@ Changes in the Python API * Remove the ``asyncore``-based ``smtpd`` module deprecated in Python 3.4.7 and 3.5.4. A recommended replacement is the - :mod:`asyncio`-based aiosmtpd_ PyPI module. + :mod:`asyncio`-based :pypi:`aiosmtpd` PyPI module. * :func:`shlex.split`: Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. The feature was deprecated @@ -1870,6 +1908,15 @@ Changes in the Python API * Mixing tabs and spaces as indentation in the same file is not supported anymore and will raise a :exc:`TabError`. +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + should provide ``_is_main_interpreter()``. + (See :gh:`112826`.) + Build Changes ============= @@ -1908,6 +1955,8 @@ Build Changes :file:`!configure`. (Contributed by Christian Heimes in :gh:`89886`.) +* Windows builds and macOS installers from python.org now use OpenSSL 3.0. + C API Changes ============= @@ -2410,10 +2459,3 @@ Removed * Remove the ``PyUnicode_InternImmortal()`` function macro. (Contributed by Victor Stinner in :gh:`85858`.) - -* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. - (Contributed by Nikita Sobolev in :gh:`99482`.) - -* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag - from :mod:`ctypes` module. - (Contributed by Nikita Sobolev in :gh:`99285`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a804b1e3acd147..5cadcb7a9e5d09 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: TBD +:Editor: Thomas Wouters .. Rules for maintenance: @@ -49,6 +49,10 @@ This article explains the new features in Python 3.13, compared to 3.12. For full details, see the :ref:`changelog <changelog>`. +.. seealso:: + + :pep:`719` -- Python 3.13 Release Schedule + .. note:: Prerelease users should be aware that this document is currently in draft @@ -56,21 +60,364 @@ For full details, see the :ref:`changelog <changelog>`. so it's worth checking back even after reading earlier versions. -Summary -- Release highlights +Summary -- Release Highlights ============================= .. This section singles out the most important changes in Python 3.13. Brevity is key. +Python 3.13 beta is the pre-release of the next version of the Python +programming language, with a mix of changes to the language, the +implementation and the standard library. The biggest changes to the +implementation include a new interactive interpreter, and experimental +support for dropping the Global Interpreter Lock (:pep:`703`) and a +Just-In-Time compiler (:pep:`744`). The library changes contain removal of +deprecated APIs and modules, as well as the usual improvements in +user-friendliness and correctness. .. PEP-sized items next. +Interpreter improvements: + +* A greatly improved :ref:`interactive interpreter + <whatsnew313-better-interactive-interpreter>` and + :ref:`improved error messages <whatsnew313-improved-error-messages>`. + +* Color support in the new :ref:`interactive interpreter + <whatsnew313-better-interactive-interpreter>`, + as well as in :ref:`tracebacks <whatsnew313-improved-error-messages>` + and :ref:`doctest <whatsnew313-doctest>` output. This can be disabled through the + :envvar:`PYTHON_COLORS` and |NO_COLOR|_ environment variables. + +* :pep:`744`: A basic :ref:`JIT compiler <whatsnew313-jit-compiler>` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to be improving this + over the next few releases. + +* :pep:`667`: The :func:`locals` builtin now has + :ref:`defined semantics <whatsnew313-locals-semantics>` when mutating the + returned mapping. Python debuggers and similar tools may now more reliably + update local variables in optimized scopes even during concurrent code + execution. + +New typing features: + +* :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, + and :data:`typing.TypeVarTuple`) now support defaults. + +* :pep:`702`: Support for marking deprecations in the type system using the + new :func:`warnings.deprecated` decorator. + +* :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive + type narrowing behavior. + +* :pep:`705`: :data:`typing.ReadOnly` was added, to mark an item of a + :class:`typing.TypedDict` as read-only for type checkers. +Free-threading: + +* :pep:`703`: CPython 3.13 has experimental support for running with the + :term:`global interpreter lock` disabled when built with ``--disable-gil``. + See :ref:`Free-threaded CPython <free-threaded-cpython>` for more details. + +Platform support: + +* :pep:`730`: Apple's iOS is now an officially supported platform. Official + Android support (:pep:`738`) is in the works as well. + +Removed modules: + +* :ref:`PEP 594 <whatsnew313-pep594>`: The remaining 19 "dead batteries" + have been removed from the standard library: + :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, + :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, + :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. + +* Also removed were the :mod:`!tkinter.tix` and :mod:`!lib2to3` modules, and the + ``2to3`` program. + +Release schedule changes: + +* :pep:`602` ("Annual Release Cycle for Python") has been updated: + + * Python 3.9 - 3.12 have one and a half years of full support, + followed by three and a half years of security fixes. + * Python 3.13 and later have two years of full support, + followed by three years of security fixes. New Features ============ +.. _whatsnew313-better-interactive-interpreter: + +A Better Interactive Interpreter +-------------------------------- + +On Unix-like systems like Linux or macOS as well as Windows, Python now +uses a new :term:`interactive` shell. When the user starts the +:term:`REPL` from an interactive terminal the interactive shell now +supports the following new features: + +* Colorized prompts. +* Multiline editing with history preservation. +* Interactive help browsing using :kbd:`F1` with a separate command + history. +* History browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts. +* "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt). +* The ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit` without the need to use call parentheses after the command + name. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + +The new shell requires :mod:`curses` on Unix-like systems. + +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Salgado, Åukasz Langa, and +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. +Windows support contributed by Dino Viehland and Anthony Shaw.) + +.. _whatsnew313-improved-error-messages: + +Improved Error Messages +----------------------- + +* The interpreter now colorizes error messages when displaying tracebacks by default. + This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Pablo Galindo Salgado in :gh:`112730`.) + +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + +* A common mistake is to write a script with the same name as a + standard library module. When this results in errors, we now + display a more helpful error message: + + .. code-block:: shell-session + + $ python random.py + Traceback (most recent call last): + File "/home/random.py", line 1, in <module> + import random; print(random.randint(5)) + ^^^^^^^^^^^^^ + File "/home/random.py", line 1, in <module> + import random; print(random.randint(5)) + ^^^^^^^^^^^^^^ + AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/random.py' since it has the same name as the standard library module named 'random' and the import system gives it precedence) + + Similarly, if a script has the same name as a third-party + module it attempts to import, and this results in errors, + we also display a more helpful error message: + + .. code-block:: shell-session + + $ python numpy.py + Traceback (most recent call last): + File "/home/numpy.py", line 1, in <module> + import numpy as np; np.array([1,2,3]) + ^^^^^^^^^^^^^^^^^^ + File "/home/numpy.py", line 1, in <module> + import numpy as np; np.array([1,2,3]) + ^^^^^^^^ + AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/numpy.py' if it has the same name as a third-party module you intended to import) + + (Contributed by Shantanu Jain in :gh:`95754`.) + +* When an incorrect keyword argument is passed to a function, the error message + now potentially suggests the correct keyword argument. + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) + + >>> "better error messages!".split(max_split=1) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + "better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + +* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, + with a tuple of names of attributes of this class which are accessed + through ``self.X`` from any function in its body. (Contributed by Irit Katriel + in :gh:`115775`.) + +.. _whatsnew313-locals-semantics: + +Defined mutation semantics for ``locals()`` +------------------------------------------- + +Historically, the expected result of mutating the return value of :func:`locals` +has been left to individual Python implementations to define. + +Through :pep:`667`, Python 3.13 standardises the historical behaviour of CPython +for most code execution scopes, but changes +:term:`optimized scopes <optimized scope>` (functions, generators, coroutines, +comprehensions, and generator expressions) to explicitly return independent +snapshots of the currently assigned local variables, including locally +referenced nonlocal variables captured in closures. + +This change to the semantics of :func:`locals` in optimized scopes also affects the default +behaviour of code execution functions that implicitly target ``locals()`` if no explicit +namespace is provided (such as :func:`exec` and :func:`eval`). In previous versions, whether +or not changes could be accessed by calling ``locals()`` after calling the code execution +function was implementation dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based on other code +(including debuggers and code execution tracing tools) potentially resetting the shared +snapshot in that scope. Now, the code will always run against an independent snapshot of the +local variables in optimized scopes, and hence the changes will never be visible in +subsequent calls to ``locals()``. To access the changes made in these cases, an explicit +namespace reference must now be passed to the relevant function. Alternatively, it may make +sense to update affected code to use a higher level code execution API that returns the +resulting code execution namespace (e.g. :func:`runpy.run_path` when executing Python +files from disk). + +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals <frame.f_locals>` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. + +See :pep:`667` for more details, including related C API changes and deprecations. Porting +notes are also provided below for the affected :ref:`Python APIs <pep667-porting-notes-py>` +and :ref:`C APIs <pep667-porting-notes-c>`. + +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) + +Incremental Garbage Collection +------------------------------ + +* The cycle garbage collector is now incremental. + This means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + +Support For Mobile Platforms +---------------------------- + +* iOS is now a :pep:`11` supported platform. ``arm64-apple-ios`` + (iPhone and iPad devices released after 2013) and + ``arm64-apple-ios-simulator`` (Xcode iOS simulator running on Apple Silicon + hardware) are now tier 3 platforms. + + ``x86_64-apple-ios-simulator`` (Xcode iOS simulator running on older x86_64 + hardware) is not a tier 3 supported platform, but will be supported on a + best-effort basis. + + See :pep:`730`: for more details. + + (PEP written and implementation contributed by Russell Keith-Magee in + :gh:`114099`.) + +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which may speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 <whatsnew311-pep659>` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. + +* When the JIT is enabled, the optimized + Tier 2 IR is translated to machine code, which is then executed. + +* The machine code translation process uses a technique called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +The ``--enable-experimental-jit`` flag has the following optional values: +* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. + +* ``yes`` (default if the flag is present without optional value) + -- Enable the JIT. To disable the JIT at runtime, + pass the environment variable ``PYTHON_JIT=0``. + +* ``yes-off`` -- Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable + ``PYTHON_JIT=1``. + +* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with + ``PYTHON_JIT=0``. + +(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) + +See :pep:`744` for more details. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + +.. _free-threaded-cpython: + +Free-threaded CPython +===================== + +CPython will run with the :term:`global interpreter lock` (GIL) disabled when +configured using the ``--disable-gil`` option at build time. This is an +experimental feature and therefore isn't used by default. Users need to +either compile their own interpreter, or install one of the experimental +builds that are marked as *free-threaded*. See :pep:`703` "Making the Global +Interpreter Lock Optional in CPython" for more detail. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multicore hardware. + +Work is still ongoing: expect some bugs and a substantial single-threaded +performance hit. + +The free-threaded build still supports optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command line option :option:`-X gil`. + +To check if the current interpreter is configured with ``--disable-gil``, +use ``sysconfig.get_config_var("Py_GIL_DISABLED")``. To check if the :term:`GIL` +is actually disabled in the running process, the :func:`!sys._is_gil_enabled` +function can be used. + +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. + +pip 24.1b1 or newer is required to install packages with C extensions in the +free-threaded build. Other Language Changes ====================== @@ -106,22 +453,111 @@ Other Language Changes the file is not accessible. (Contributed by Moonsik Park in :gh:`82367`.) +* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block + is rejected when the global is used in the :keyword:`else` block. + (Contributed by Irit Katriel in :gh:`111123`.) + +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + +* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It + determines whether or not frozen modules are ignored by the import machinery, + equivalent of the :option:`-X frozen_modules <-X>` command-line option. + (Contributed by Yilei Yang in :gh:`111374`.) + +* Add :ref:`support for the perf profiler <perf_profiling>` working without + frame pointers through the new environment variable + :envvar:`PYTHON_PERF_JIT_SUPPORT` and command-line option :option:`-X perf_jit + <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) + +* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change + the location of a ``.python_history`` file. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in + :gh:`73965`.) + +* Add :exc:`PythonFinalizationError` exception. This exception derived from + :exc:`RuntimeError` is raised when an operation is blocked during + the :term:`Python finalization <interpreter shutdown>`. + + The following functions now raise PythonFinalizationError, instead of + :exc:`RuntimeError`: + + * :func:`_thread.start_new_thread`. + * :class:`subprocess.Popen`. + * :func:`os.fork`. + * :func:`os.forkpty`. + + (Contributed by Victor Stinner in :gh:`114570`.) + +* Added :attr:`!name` and :attr:`!mode` attributes for compressed + and archived file-like objects in modules :mod:`bz2`, :mod:`lzma`, + :mod:`tarfile` and :mod:`zipfile`. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) + +* The :func:`ssl.create_default_context` API now includes + :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` + in its default flags. + + .. note:: + + :data:`ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` or malformed + certificates that the underlying OpenSSL implementation otherwise would + accept. While disabling this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + + (Contributed by William Woodruff in :gh:`112389`.) + +* The :class:`configparser.ConfigParser` now accepts unnamed sections before named + ones if configured to do so. + (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + +* :ref:`annotation scope <annotation-scopes>` within class scopes can now + contain lambdas and comprehensions. Comprehensions that are located within + class scopes are not inlined into their parent scope. (Contributed by + Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) + +* Classes have a new :attr:`!__firstlineno__` attribute, + populated by the compiler, with the line number of the first line + of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) + +* ``from __future__ import ...`` statements are now just normal + relative imports if dots are present before the module name. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) + + New Modules =========== -* None yet. +* None. Improved Modules ================ -ast ---- +argparse +-------- -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. - (Contributed by Irit Katriel in :gh:`108113`). +* Add parameter *deprecated* in methods + :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` + which allows to deprecate command-line options, positional arguments and + subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`.) array ----- @@ -130,6 +566,108 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +* Add ``clear()`` method in order to implement ``MutableSequence``. + (Contributed by Mike Zimin in :gh:`114894`.) + +ast +--- + +* The constructors of node types in the :mod:`ast` module are now stricter + in the arguments they accept, and have more intuitive behaviour when + arguments are omitted. + + If an optional field on an AST node is not included as an argument when + constructing an instance, the field will now be set to ``None``. Similarly, + if a list field is omitted, that field will now be set to an empty list, + and if a :class:`!ast.expr_context` field is omitted, it defaults to + :class:`Load() <ast.Load>`. + (Previously, in all cases, the attribute would be missing on the newly + constructed AST node instance.) + + If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. + This will cause an exception in Python 3.15. Similarly, passing a keyword + argument that does not map to a field on the AST node is now deprecated, + and will raise an exception in Python 3.15. + + These changes do not apply to user-defined subclasses of :class:`ast.AST`, + unless the class opts in to the new behavior by setting the attribute + :attr:`ast.AST._field_types`. + + (Contributed by Jelle Zijlstra in :gh:`105858`, :gh:`117486`, and :gh:`118851`.) + +* :func:`ast.parse` now accepts an optional argument *optimize* + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized AST. + (Contributed by Irit Katriel in :gh:`108113`.) + +asyncio +------- + +* :meth:`asyncio.loop.create_unix_server` will now automatically remove + the Unix socket when the server is closed. + (Contributed by Pierre Ossman in :gh:`111246`.) + +* :meth:`asyncio.DatagramTransport.sendto` will now send zero-length + datagrams if called with an empty bytes object. The transport flow + control also now accounts for the datagram header when calculating + the buffer size. + (Contributed by Jamie Phan in :gh:`115199`.) + +* Add :meth:`asyncio.Server.close_clients` and + :meth:`asyncio.Server.abort_clients` methods which allow to more + forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + +* :func:`asyncio.as_completed` now returns an object that is both an + :term:`asynchronous iterator` and a plain :term:`iterator` of awaitables. + The awaitables yielded by asynchronous iteration include original task or + future objects that were passed in, making it easier to associate results + with the tasks being completed. + (Contributed by Justin Arthur in :gh:`77714`.) + +* When :func:`asyncio.TaskGroup.create_task` is called on an inactive + :class:`asyncio.TaskGroup`, the given coroutine will be closed (which + prevents a :exc:`RuntimeWarning` about the given coroutine being + never awaited). + (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) + +* Improved behavior of :class:`asyncio.TaskGroup` when an external cancellation + collides with an internal cancellation. For example, when two task groups + are nested and both experience an exception in a child task simultaneously, + it was possible that the outer task group would hang, because its internal + cancellation was swallowed by the inner task group. + + In the case where a task group is cancelled externally and also must + raise an :exc:`ExceptionGroup`, it will now call the parent task's + :meth:`~asyncio.Task.cancel` method. This ensures that a + :exc:`asyncio.CancelledError` will be raised at the next + :keyword:`await`, so the cancellation is not lost. + + An added benefit of these changes is that task groups now preserve the + cancellation count (:meth:`asyncio.Task.cancelling`). + + In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now + reset the undocumented ``_must_cancel`` flag when the cancellation count + reaches zero. + + (Inspired by an issue reported by Arthur Tacca in :gh:`116720`.) + +* Add :meth:`asyncio.Queue.shutdown` (along with + :exc:`asyncio.QueueShutDown`) for queue termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104228`.) + +* Accept a tuple of separators in :meth:`asyncio.StreamReader.readuntil`, + stopping when one of them is encountered. + (Contributed by Bruce Merry in :gh:`81322`.) + +base64 +------ + +* Add :func:`base64.z85encode` and :func:`base64.z85decode` functions which allow encoding + and decoding Z85 data. + See `Z85 specification <https://rfc.zeromq.org/spec/32/>`_ for more information. + (Contributed by Matan Perelman in :gh:`75299`.) + copy ---- @@ -142,6 +680,18 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* The layout of :ref:`bit fields <ctypes-bit-fields-in-structures-unions>` in + :class:`~ctypes.Structure` and :class:`~ctypes.Union` was improved to better + match platform defaults (GCC/Clang or MSC). In particular, fields no longer + overlap. + (Contributed by Matthias Görgens in :gh:`97702`.) +* A :attr:`ctypes.Structure._layout_` class attribute can be set + to help match a non-default ABI. + (Contributed by Petr Viktorin in :gh:`97702`.) + dbm --- @@ -149,22 +699,182 @@ dbm from the database. (Contributed by Donghee Na in :gh:`107122`.) +* Add new :mod:`dbm.sqlite3` backend, and make it the default :mod:`!dbm` backend. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) + +dis +--- + +* Change the output of :mod:`dis` module functions to show logical + labels for jump targets and exception handlers, rather than offsets. + The offsets can be added with the new ``-O`` command line option or + the ``show_offsets`` parameter. + (Contributed by Irit Katriel in :gh:`112137`.) + +* :meth:`~dis.get_instructions` no longer represents cache entries as + separate instructions. Instead, it returns them as part of the + :class:`~dis.Instruction`, in the new *cache_info* field. The + *show_caches* argument to :meth:`~dis.get_instructions` is + deprecated and no longer has any effect. + (Contributed by Irit Katriel in :gh:`112962`.) + +.. _whatsnew313-doctest: + doctest ------- +* Color is added to the output by default. + This can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`117225`.) + * The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped tests. Add :attr:`doctest.DocTestRunner.skips` and :attr:`doctest.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) +email +----- + +* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return + ``('', '')`` 2-tuples in more situations where invalid email addresses are + encountered instead of potentially inaccurate values. Add optional *strict* + parameter to these two functions: use ``strict=False`` to get the old + behavior, accept malformed inputs. + ``getattr(email.utils, 'supports_strict_parsing', False)`` can be used to + check if the *strict* parameter is available. + (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve + the :cve:`2023-27043` fix.) + +fractions +--------- + +* Formatting for objects of type :class:`fractions.Fraction` now supports + the standard format specification mini-language rules for fill, alignment, + sign handling, minimum width and grouping. (Contributed by Mark Dickinson + in :gh:`111320`.) + +gc +-- + +* The cyclic garbage collector is now incremental, which changes the meanings + of the results of :meth:`gc.get_threshold` and :meth:`gc.set_threshold` as + well as :meth:`gc.get_count` and :meth:`gc.get_stats`. + + * :meth:`gc.get_threshold` returns a three-item tuple for backwards compatibility. + The first value is the threshold for young collections, as before; the second + value determines the rate at which the old collection is scanned (the + default is 10, and higher values mean that the old collection is scanned more slowly). + The third value is meaningless and is always zero. + * :meth:`gc.set_threshold` ignores any items after the second. + * :meth:`gc.get_count` and :meth:`gc.get_stats` + return the same format of results as before. + The only difference is that instead of the results referring to + the young, aging and old generations, the results refer to the + young generation and the aging and collecting spaces of the old generation. + + In summary, code that attempted to manipulate the behavior of the cycle GC may + not work exactly as intended, but it is very unlikely to be harmful. + All other code will work just fine. + +glob +---- + +* Add :func:`glob.translate` function that converts a path specification with + shell-style wildcards to a regular expression. + (Contributed by Barney Gale in :gh:`72904`.) + +importlib +--------- + +* Previously deprecated :mod:`importlib.resources` functions are un-deprecated: + + * :func:`~importlib.resources.is_resource()` + * :func:`~importlib.resources.open_binary()` + * :func:`~importlib.resources.open_text()` + * :func:`~importlib.resources.path()` + * :func:`~importlib.resources.read_binary()` + * :func:`~importlib.resources.read_text()` + + All now allow for a directory (or tree) of resources, using multiple positional + arguments. + + For text-reading functions, the *encoding* and *errors* must now be given as + keyword arguments. + + The :func:`~importlib.resources.contents()` remains deprecated in favor of + the full-featured :class:`~importlib.resources.abc.Traversable` API. + However, there is now no plan to remove it. + + (Contributed by Petr Viktorin in :gh:`106532`.) + io -- -The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with -:data:`sys.unraisablehook`. Previously, errors were ignored silently by default, -and only logged in :ref:`Python Development Mode <devmode>` or on :ref:`Python -built on debug mode <debug-build>`. -(Contributed by Victor Stinner in :gh:`62948`.) +* The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with + :data:`sys.unraisablehook`. Previously, errors were ignored silently by default, + and only logged in :ref:`Python Development Mode <devmode>` or on :ref:`Python + built on debug mode <debug-build>`. + (Contributed by Victor Stinner in :gh:`62948`.) + +ipaddress +--------- + +* Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. + (Contributed by Charles Machalow in :gh:`109466`.) +* Fix ``is_global`` and ``is_private`` behavior in + :class:`~ipaddress.IPv4Address`, + :class:`~ipaddress.IPv6Address`, + :class:`~ipaddress.IPv4Network` and + :class:`~ipaddress.IPv6Network`. + +itertools +--------- + +* Added a ``strict`` option to :func:`itertools.batched`. + This raises a :exc:`ValueError` if the final batch is shorter + than the specified batch size. + (Contributed by Raymond Hettinger in :gh:`113202`.) + +marshal +------- + +* Add the *allow_code* parameter in module functions. + Passing ``allow_code=False`` prevents serialization and de-serialization of + code objects which are incompatible between Python versions. + (Contributed by Serhiy Storchaka in :gh:`113626`.) + +math +---- + +* A new function :func:`~math.fma` for fused multiply-add operations has been + added. This function computes ``x * y + z`` with only a single round, and so + avoids any intermediate loss of precision. It wraps the ``fma()`` function + provided by C99, and follows the specification of the IEEE 754 + "fusedMultiplyAdd" operation for special cases. + (Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) + +mimetypes +--------- + +* Add the :func:`~mimetypes.guess_file_type` function which works with file path. + Passing file path instead of URL in :func:`~mimetypes.guess_type` is :term:`soft deprecated`. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + +mmap +---- + +* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method + that can be used when a seekable file-like object is required. + The :meth:`~mmap.mmap.seek` method now returns the new absolute position. + (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) +* :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, + the file descriptor specified by *fileno* will not be duplicated. + (Contributed by Zackery Spytz and Petr Viktorin in :gh:`78502`.) +* :class:`mmap.mmap` is now protected from crashing on Windows when the mapped memory + is inaccessible due to file system errors or access violations. + (Contributed by Jannis Weigend in :gh:`118209`.) opcode ------ @@ -197,7 +907,46 @@ os the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). - (Contributed by Donghee Na in :gh:`109595`) + (Contributed by Donghee Na in :gh:`109595`.) + +* Add support of :func:`os.lchmod` and the *follow_symlinks* argument + in :func:`os.chmod` on Windows. + Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is + ``False`` on Windows. + (Contributed by Serhiy Storchaka in :gh:`59616`.) + +* Add support of :func:`os.fchmod` and a file descriptor + in :func:`os.chmod` on Windows. + (Contributed by Serhiy Storchaka in :gh:`113191`.) + +* :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned + process use the current process environment. + (Contributed by Jakub Kulik in :gh:`113119`.) + +* :func:`os.posix_spawn` gains an :attr:`os.POSIX_SPAWN_CLOSEFROM` attribute for + use in ``file_actions=`` on platforms that support + :c:func:`!posix_spawn_file_actions_addclosefrom_np`. + (Contributed by Jakub Kulik in :gh:`113117`.) + +* :func:`os.mkdir` and :func:`os.makedirs` on Windows now support passing a + *mode* value of ``0o700`` to apply access control to the new directory. This + implicitly affects :func:`tempfile.mkdtemp` and is a mitigation for + :cve:`2024-4030`. Other values for *mode* continue to be ignored. + (Contributed by Steve Dower in :gh:`118486`.) + +os.path +------- + +* Add :func:`os.path.isreserved` to check if a path is reserved on the current + system. This function is only available on Windows. + (Contributed by Barney Gale in :gh:`88569`.) +* On Windows, :func:`os.path.isabs` no longer considers paths starting with + exactly one (back)slash to be absolute. + (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) + +* Add support of *dir_fd* and *follow_symlinks* keyword arguments in + :func:`shutil.chown`. + (Contributed by Berker Peksag and Tahia K in :gh:`62308`) pathlib ------- @@ -207,16 +956,31 @@ pathlib (Contributed by Barney Gale in :gh:`89812`.) * Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file:/``). + object from a 'file' URI (``file://``). (Contributed by Barney Gale in :gh:`107465`.) -* Add support for recursive wildcards in :meth:`pathlib.PurePath.match`. +* Add :meth:`pathlib.PurePath.full_match` for matching paths with + shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) -* Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, - :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, and - :meth:`~pathlib.Path.is_dir`. - (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +* Add :attr:`pathlib.PurePath.parser` class attribute that stores the + implementation of :mod:`os.path` used for low-level path parsing and + joining: either ``posixpath`` or ``ntpath``. + +* Add *recurse_symlinks* keyword-only argument to :meth:`pathlib.Path.glob` + and :meth:`~pathlib.Path.rglob`. + (Contributed by Barney Gale in :gh:`77609`.) + +* Add *follow_symlinks* keyword-only argument to :meth:`~pathlib.Path.is_file`, + :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, + :meth:`~pathlib.Path.group`. + (Contributed by Barney Gale in :gh:`105793`, and Kamil Turek in + :gh:`107962`.) + +* Return files and directories from :meth:`pathlib.Path.glob` and + :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In + earlier versions, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`.) pdb --- @@ -225,10 +989,49 @@ pdb the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias Bussonnier in :gh:`106676`.) -* Expressions/Statements whose prefix is a pdb command are now correctly +* Expressions/statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) +* ``sys.path[0]`` will no longer be replaced by the directory of the script + being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` + command line option or :envvar:`PYTHONSAFEPATH` environment variable). + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + +* :mod:`zipapp` is supported as a debugging target. + (Contributed by Tian Gao in :gh:`118501`.) + +* ``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately + rather than on the next line of code to be executed. This change prevents the + debugger from breaking outside of the context when ``breakpoint()`` is positioned + at the end of the context. + (Contributed by Tian Gao in :gh:`118579`.) + +queue +----- + +* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue + termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + +random +------ + +* Add a :ref:`command-line interface <random-cli>`. + (Contributed by Hugo van Kemenade in :gh:`118131`.) + +re +-- +* Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. + :exc:`!re.error` is kept for backward compatibility. + +site +---- + +* :file:`.pth` files are now decoded by UTF-8 first, and then by the + :term:`locale encoding` if the UTF-8 decoding fails. + (Contributed by Inada Naoki in :gh:`117802`.) + sqlite3 ------- @@ -236,6 +1039,66 @@ sqlite3 object is not :meth:`closed <sqlite3.Connection.close>` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` + for filtering database objects to dump. + (Contributed by Mariusz Felisiak in :gh:`91602`.) + +statistics +---------- + +* Add :func:`statistics.kde` for kernel density estimation. + This makes it possible to estimate a continuous probability density function + from a fixed number of discrete samples. Also added :func:`statistics.kde_random` + for sampling from the estimated probability density function. + (Contributed by Raymond Hettinger in :gh:`115863`.) + +.. _whatsnew313-subprocess: + +subprocess +---------- + +* The :mod:`subprocess` module now uses the :func:`os.posix_spawn` function in + more situations. Notably in the default case of ``close_fds=True`` on more + recent versions of platforms including Linux, FreeBSD, and Solaris where the + C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. + On Linux this should perform similar to our existing Linux :c:func:`!vfork` + based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can + be set to ``False`` if you need to force :mod:`subprocess` not to ever use + :func:`os.posix_spawn`. Please report your reason and platform details in + the CPython issue tracker if you set this so that we can improve our API + selection logic for everyone. + (Contributed by Jakub Kulik in :gh:`113117`.) + +sys +--- + +* Add the :func:`sys._is_interned` function to test if the string was interned. + This function is not guaranteed to exist in all implementations of Python. + (Contributed by Serhiy Storchaka in :gh:`78573`.) + +tempfile +-------- + +* On Windows, the default mode ``0o700`` used by :func:`tempfile.mkdtemp` now + limits access to the new directory due to changes to :func:`os.mkdir`. This + is a mitigation for :cve:`2024-4030`. + (Contributed by Steve Dower in :gh:`118486`.) + +time +---- + +* On Windows, :func:`time.monotonic()` now uses the + ``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, + instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. + (Contributed by Victor Stinner in :gh:`88494`.) + +* On Windows, :func:`time.time()` now uses the + ``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better + than 1 μs, instead of the ``GetSystemTimeAsFileTime()`` clock which has a + resolution of 15.6 ms. + (Contributed by Victor Stinner in :gh:`63207`.) + + tkinter ------- @@ -245,6 +1108,44 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts + the attribute name without the minus prefix to get window attributes, + e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and + values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. + Add new optional keyword-only parameter *return_python_dict*: calling + ``w.wm_attributes(return_python_dict=True)`` returns the attributes as + a dict instead of a tuple. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + +* Add new optional keyword-only parameter *return_ints* in + the :meth:`!Text.count` method. + Passing ``return_ints=True`` makes it always returning the single count + as an integer instead of a 1-tuple or ``None``. + (Contributed by Serhiy Storchaka in :gh:`97928`.) + +* Add support of the "vsapi" element type in + the :meth:`~tkinter.ttk.Style.element_create` method of + :class:`tkinter.ttk.Style`. + (Contributed by Serhiy Storchaka in :gh:`68166`.) + +* Add the :meth:`!after_info` method for Tkinter widgets. + (Contributed by Cheryl Sabella in :gh:`77020`.) + +* Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add the :class:`!PhotoImage` methods :meth:`!read` to read + an image from a file and :meth:`!data` to get the image data. + Add *background* and *grayscale* parameters to :class:`!PhotoImage` method + :meth:`!write`. + (Contributed by Serhiy Storchaka in :gh:`118271`.) + traceback --------- @@ -252,6 +1153,20 @@ traceback to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. (Contributed by Irit Katriel in :gh:`105292`.) +* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which + holds a string display of the *exc_type*. Deprecate the field *exc_type* + which holds the type object itself. Add parameter *save_exc_type* (default + ``True``) to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + +types +----- + +* :class:`~types.SimpleNamespace` constructor now allows specifying initial + values of attributes as a positional argument which must be a mapping or + an iterable of key-value pairs. + (Contributed by Serhiy Storchaka in :gh:`108191`.) + typing ------ @@ -260,6 +1175,20 @@ typing check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in :gh:`104873`.) +* Add :data:`typing.ReadOnly`, a special typing construct to mark + an item of a :class:`typing.TypedDict` as read-only for type checkers. + See :pep:`705` for more details. + +* Add :data:`typing.NoDefault`, a sentinel object used to represent the defaults + of some parameters in the :mod:`typing` module. (Contributed by Jelle Zijlstra in + :gh:`116126`.) + +unicodedata +----------- + +* The Unicode database has been updated to version 15.1.0. (Contributed by + James Gerity in :gh:`109559`.) + venv ---- @@ -270,41 +1199,400 @@ venv (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in :gh:`108125`.) +warnings +-------- + +* The new :func:`warnings.deprecated` decorator provides a way to communicate + deprecations to :term:`static type checkers <static type checker>` and + to warn on usage of deprecated classes and functions. A runtime deprecation + warning may also be emitted when a decorated function or class is used at runtime. + See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) + +xml.etree.ElementTree +--------------------- + +* Add the :meth:`!close` method for the iterator returned by + :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. + (Contributed by Serhiy Storchaka in :gh:`69893`.) + +zipimport +--------- + +* Gains support for ZIP64 format files. Everybody loves huge code right? + (Contributed by Tim Hatch in :gh:`94146`.) + +.. Add improved modules above alphabetically, not here at the end. + Optimizations ============= * :func:`textwrap.indent` is now ~30% faster than before for large input. (Contributed by Inada Naoki in :gh:`107369`.) +* The :mod:`subprocess` module uses :func:`os.posix_spawn` in more situations + including the default where ``close_fds=True`` on many modern platforms. This + should provide a noteworthy performance increase launching processes on + FreeBSD and Solaris. See the :ref:`subprocess <whatsnew313-subprocess>` + section above for details. + (Contributed by Jakub Kulik in :gh:`113117`.) + +* Several standard library modules have had their import times significantly + improved. For example, the import time of the :mod:`typing` module has been + reduced by around a third by removing dependencies on :mod:`re` and + :mod:`contextlib`. Other modules to enjoy import-time speedups include + :mod:`importlib.metadata`, :mod:`threading`, :mod:`enum`, :mod:`functools` + and :mod:`email.utils`. + (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas and + others in :gh:`109653`.) + +Removed Modules And APIs +======================== + +.. _whatsnew313-pep594: + +PEP 594: dead batteries (and other module removals) +--------------------------------------------------- + +* :pep:`594` removed 19 modules from the standard library, + deprecated in Python 3.11: + + * :mod:`!aifc`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!audioop`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!chunk`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!cgi` and :mod:`!cgitb`. + + * ``cgi.FieldStorage`` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or :pypi:`multipart` + PyPI project for ``POST`` and ``PUT``. + + * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` + directly on the desired query string, except for ``multipart/form-data`` + input, which can be handled as described for ``cgi.parse_multipart()``. + + * ``cgi.parse_header()`` can be replaced with the functionality in the + :mod:`email` package, which implements the same MIME RFCs. For example, + with :class:`email.message.EmailMessage`:: + + from email.message import EmailMessage + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params + + * ``cgi.parse_multipart()`` can be replaced with the functionality in the + :mod:`email` package (e.g. :class:`email.message.EmailMessage` and + :class:`email.message.Message`) which implements the same MIME RFCs, or + with the :pypi:`multipart` PyPI project. + + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!crypt` module and its private :mod:`!_crypt` extension. + The :mod:`hashlib` module is a potential replacement for certain use cases. + Otherwise, the following PyPI projects can be used: + + * :pypi:`bcrypt`: + Modern password hashing for your software and your servers. + * :pypi:`passlib`: + Comprehensive password hashing framework supporting over 30 schemes. + * :pypi:`argon2-cffi`: + The secure Argon2 password hashing algorithm. + * :pypi:`legacycrypt`: + :mod:`ctypes` wrapper to the POSIX crypt library call and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library + call and associated functionality. + + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!imghdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + The ``puremagic.what()`` function can be used to replace + the ``imghdr.what()`` function for all file formats that + were supported by ``imghdr``. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!mailcap`. + The :mod:`mimetypes` module provides an alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!msilib`. + (Contributed by Zachary Ware in :gh:`104773`.) + + * :mod:`!nis`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!nntplib`: + the :pypi:`nntplib` PyPI project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!ossaudiodev`: use the + `pygame project <https://www.pygame.org/>`_ for audio playback. + (Contributed by Victor Stinner in :gh:`104780`.) + + * :mod:`!pipes`: use the :mod:`subprocess` module instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sndhdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!spwd`: + the :pypi:`python-pam` project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sunau`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3` or + :pypi:`Exscript` instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!uu`: the :mod:`base64` module is a modern alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!xdrlib`. + (Contributed by Victor Stinner in :gh:`104773`.) + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The + third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) + +configparser +------------ + +* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) + +importlib +--------- + +* Remove deprecated :meth:`~object.__getitem__` access for + :class:`!importlib.metadata.EntryPoint` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) + +locale +------ + +* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: + use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) + +logging +------- + +* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and + ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated + since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` + method, :meth:`!logging.LoggerAdapter.warning` method and + :func:`logging.warning` function. + (Contributed by Victor Stinner in :gh:`105376`.) + +pathlib +------- + +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + +re +-- + +* Remove undocumented, never working, and deprecated ``re.template`` function + and ``re.TEMPLATE`` flag (and ``re.T`` alias). + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) + + +turtle +------ + +* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, + deprecated in docs since Python 3.1 + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) + +typing +------ + +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + +* Remove support for the keyword-argument method of creating + :class:`typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) + +unittest +-------- + +* Remove the following :mod:`unittest` functions, deprecated in Python 3.11: + + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` methods instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Hugo van Kemenade in :gh:`104835`.) + +* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) + +urllib +------ + +* Remove *cafile*, *capath* and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6: pass + the *context* parameter instead. Use + :meth:`ssl.SSLContext.load_cert_chain` to load specific certificates, or + let :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + (Contributed by Victor Stinner in :gh:`105382`.) + +webbrowser +---------- + +* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) + +* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. + Use :attr:`webbrowser.MacOSXOSAScript.name <webbrowser.controller.name>` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) + +New Deprecations +================ -Deprecated -========== +* Removed chained :class:`classmethod` descriptors (introduced in + :gh:`63272`). This can no longer be used to wrap other descriptors + such as :class:`property`. The core design of this feature was flawed + and caused a number of downstream problems. To "pass-through" a + :class:`classmethod`, consider using the :attr:`!__wrapped__` + attribute that was added in Python 3.10. (Contributed by Raymond + Hettinger in :gh:`89519`.) * :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, emits :exc:`DeprecationWarning` since 3.13 and will be removed in Python 3.16. Use the ``'w'`` format code instead. - (contributed by Hugo van Kemenade in :gh:`80480`) + (Contributed by Hugo van Kemenade in :gh:`80480`.) * :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` and :func:`!ctypes.ARRAY` functions. Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) +* :mod:`decimal`: Deprecate non-standard format specifier "N" for + :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check + membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + +* :ref:`frame-objects`: + Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` + (as has always been the case for an executing frame). + (Contributed by Irit Katriel in :gh:`79932`.) + * :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. Previously, the :mod:`optparse` module was already deprecated, its removal was not scheduled, and no warnings was emitted: so there is no change in practice. (Contributed by Victor Stinner in :gh:`106535`.) +* :mod:`gettext`: Emit deprecation warning for non-integer numbers in + :mod:`gettext` functions and methods that consider plural forms even if the + translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + +* :mod:`glob`: The undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions are deprecated. Use :func:`glob.glob` and pass a directory to its + *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`117337`.) + * :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI - http servers have been out of favor for a very long time. This code was + :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI + HTTP servers have been out of favor for a very long time. This code was outdated, unmaintained, and rarely used. It has a high potential for both security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. +* :mod:`mimetypes`: Passing file path instead of URL in :func:`~mimetypes.guess_type` is + :term:`soft deprecated`. Use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + +* :mod:`re`: Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. In future Python versions these parameters will be + :ref:`keyword-only <keyword-only_parameter>`. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`platform`: + :func:`~platform.java_ver` is deprecated and will be removed in 3.15. + It was largely untested, had a confusing API, + and was only useful for Jython support. + (Contributed by Nikita Sobolev in :gh:`116349`.) + +* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) + +* :mod:`sqlite3`: Passing more than one positional argument to + :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is + deprecated. The remaining parameters will become keyword-only in Python 3.15. + + Deprecate passing name, number of arguments, and the callable as keyword + arguments for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) + +* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. + Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. + (Contributed by Inada Naoki in :gh:`73427`.) + +* :mod:`tarfile`: + The undocumented and unused ``tarfile`` attribute of :class:`tarfile.TarFile` + is deprecated and scheduled for removal in Python 3.16. + +* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` + is deprecated. Use *exc_type_str* instead. + * :mod:`typing`: * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote @@ -334,44 +1622,19 @@ Deprecated the new :ref:`type parameter syntax <type-params>` instead. (Contributed by Michael The in :gh:`107116`.) +* :ref:`user-defined-funcs`: + Assignment to a function's :attr:`~function.__code__` attribute where the new code + object's type does not match the function's type, is deprecated. The + different types are: plain function, generator, async generator and + coroutine. + (Contributed by Irit Katriel in :gh:`81137`.) + * :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing more than one positional argument to :func:`sqlite3.connect` and the - :class:`sqlite3.Connection` constructor is deprecated. The remaining - parameters will become keyword-only in Python 3.15. - - Deprecate passing name, number of arguments, and the callable as keyword - arguments, for the following :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` - - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: - - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` - - The affected parameters will become positional-only in Python 3.15. - - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) - -* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership - in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) - -* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions - and methods that consider plural forms even if the translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) - +.. Add deprecations above alphabetically, not here at the end. Pending Removal in Python 3.14 ------------------------------ @@ -394,7 +1657,7 @@ Pending Removal in Python 3.14 Use :class:`ast.Constant` instead. (Contributed by Serhiy Storchaka in :gh:`90953`.) -* :mod:`collections.abc`: Deprecated :class:`~collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. @@ -433,11 +1696,11 @@ Pending Removal in Python 3.14 :func:`~multiprocessing.set_start_method` APIs to explicitly specify when your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`, +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -447,12 +1710,9 @@ Pending Removal in Python 3.14 * ``master_open()``: use :func:`pty.openpty`. * ``slave_open()``: use :func:`pty.openpty`. -* :func:`shutil.rmtree` *onerror* parameter is deprecated in 3.12, - and will be removed in 3.14: use the *onexc* parameter instead. - * :mod:`sqlite3`: - * :data:`~sqlite3.version` and :data:`~sqlite3.version_info`. + * :data:`!version` and :data:`!version_info`. * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` if :ref:`named placeholders <sqlite3-placeholders>` are used and @@ -461,23 +1721,21 @@ Pending Removal in Python 3.14 * date and datetime adapter, date and timestamp converter: see the :mod:`sqlite3` documentation for suggested replacement recipes. -* :class:`types.CodeType`: Accessing ``co_lnotab`` was deprecated in :pep:`626` +* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) -* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9, +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. -* :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a +* :mod:`urllib`: + :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. (Contributed by Gregory P. Smith in :gh:`88168`.) -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an - exception in Python 3.14. - Pending Removal in Python 3.15 ------------------------------ @@ -486,24 +1744,49 @@ Pending Removal in Python 3.15 rarely used. No direct replacement exists. *Anything* is better than CGI to interface a web server with a request handler. +* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 + and originally planned for removal in Python 3.13 (:gh:`90817`), + but removal has been postponed to Python 3.15. + Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and + :func:`locale.getlocale()` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`platform`: + :func:`~platform.java_ver` is deprecated and will be removed in 3.15. + It was largely untested, had a confusing API, + and was only useful for Jython support. + (Contributed by Nikita Sobolev in :gh:`116349`.) + +* :mod:`threading`: + Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + * :class:`typing.NamedTuple`: - * The undocumented keyword argument syntax for creating NamedTuple classes + * The undocumented keyword argument syntax for creating :class:`!NamedTuple` classes (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in 3.15. Use the class-based syntax or the functional syntax instead. - * When using the functional syntax to create a NamedTuple class, failing to - pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the 'fields' parameter + * When using the functional syntax to create a :class:`!NamedTuple` class, failing to + pass a value to the *fields* parameter (``NT = NamedTuple("NT")``) is + deprecated. Passing ``None`` to the *fields* parameter (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a NamedTuple class with 0 fields, use + disallowed in Python 3.15. To create a :class:`!NamedTuple` class with 0 fields, use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. * :class:`typing.TypedDict`: When using the functional syntax to create a - TypedDict class, failing to pass a value to the 'fields' parameter (``TD = - TypedDict("TD")``) is deprecated. Passing ``None`` to the 'fields' parameter + :class:`!TypedDict` class, failing to pass a value to the *fields* parameter (``TD = + TypedDict("TD")``) is deprecated. Passing ``None`` to the *fields* parameter (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed - in Python 3.15. To create a TypedDict class with 0 fields, use ``class + in Python 3.15. To create a :class:`!TypedDict` class with 0 fields, use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. * :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` @@ -511,12 +1794,6 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) -* Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - Pending Removal in Python 3.16 ------------------------------ @@ -563,6 +1840,9 @@ although there is currently no date scheduled for their removal. :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + * :mod:`datetime`: * :meth:`~datetime.datetime.utcnow`: @@ -583,363 +1863,100 @@ although there is currently no date scheduled for their removal. * ``EntryPoints`` tuple interface. * Implicit ``None`` on return values. -* :mod:`importlib.resources`: First parameter to files is renamed to 'anchor'. -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use ``files()`` instead. Refer to `importlib-resources: Migrating from Legacy - <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy>`_ - for migration advice. - -* :func:`locale.getdefaultlocale`: use :func:`locale.setlocale()`, - :func:`locale.getencoding()` and :func:`locale.getlocale()` instead - (:gh:`90817`) - * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. - -* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is - deprecated, use an exception instance. - -* :mod:`re`: More strict rules are now applied for numerical group references - and group names in regular expressions. Only sequence of ASCII digits is now - accepted as a numerical reference. The group name in bytes patterns and - replacement strings can now only contain ASCII letters and digits and - underscore. - (Contributed by Serhiy Storchaka in :gh:`91760`.) - -* :mod:`ssl` options and protocols: - - * :class:`ssl.SSLContext` without protocol argument is deprecated. - * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN - instead. - * ``ssl.OP_NO_SSL*`` options - * ``ssl.OP_NO_TLS*`` options - * ``ssl.PROTOCOL_SSLv3`` - * ``ssl.PROTOCOL_TLS`` - * ``ssl.PROTOCOL_TLSv1`` - * ``ssl.PROTOCOL_TLSv1_1`` - * ``ssl.PROTOCOL_TLSv1_2`` - * ``ssl.TLSVersion.SSLv3`` - * ``ssl.TLSVersion.TLSv1`` - * ``ssl.TLSVersion.TLSv1_1`` - -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* ``types.CodeType.co_lnotab``: use the ``co_lines`` attribute instead. - -* :class:`typing.Text` (:gh:`92332`). - -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - -* :mod:`threading` methods: - - * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. - * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. - * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: - use :attr:`threading.Thread.daemon` attribute. - * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: - use :attr:`threading.Thread.name` attribute. - * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. - * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not None from a test case. - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. - -* :func:`!urllib.parse.to_bytes`. - -* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead - - * ``splitattr()`` - * ``splithost()`` - * ``splitnport()`` - * ``splitpasswd()`` - * ``splitport()`` - * ``splitquery()`` - * ``splittag()`` - * ``splittype()`` - * ``splituser()`` - * ``splitvalue()`` - -* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial - writes. - -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. - - -Removed -======= - -* :pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11: - use the projects `telnetlib3 <https://pypi.org/project/telnetlib3/>`_ or - `Exscript <https://pypi.org/project/Exscript/>`_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, - deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) - -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) - -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) - -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. - -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) - -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) - -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: - - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` - - Use :class:`~unittest.TestLoader` methods instead: - - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` - - (Contributed by Hugo van Kemenade in :gh:`104835`.) - -* :pep:`594`: Remove the :mod:`!cgi` and :mod:`!cgitb` modules, - deprecated in Python 3.11. - - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, and the - :mod:`email.message` module or `multipart - <https://pypi.org/project/multipart/>`__ PyPI project for ``POST`` and - ``PUT``. - - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. - - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the `multipart <https://pypi.org/project/multipart/>`__ PyPI project. - - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: - - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params - - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!sndhdr` module, deprecated in Python 3.11: use - the projects `filetype <https://pypi.org/project/filetype/>`_, `puremagic - <https://pypi.org/project/puremagic/>`_, or `python-magic - <https://pypi.org/project/python-magic/>`_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!pipes` module, deprecated in Python 3.11: - use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!ossaudiodev` module, deprecated in Python 3.11: - use the `pygame project <https://www.pygame.org/>`_ for audio playback. - (Contributed by Victor Stinner in :gh:`104780`.) - -* :pep:`594`: Remove the :mod:`!sunau` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!mailcap` module, deprecated in Python 3.11. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!spwd` module, deprecated in Python 3.11: - the `python-pam project <https://pypi.org/project/python-pam/>`_ can be used - instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!nntplib` module, deprecated in Python 3.11: - the `PyPI nntplib project <https://pypi.org/project/nntplib/>`_ can be used - instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!nis` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!xdrlib` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!msilib` module, deprecated in Python 3.11. - (Contributed by Zachary Ware in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!crypt` module and its private :mod:`!_crypt` - extension, deprecated in Python 3.11. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: - - * `bcrypt <https://pypi.org/project/bcrypt/>`_: - Modern password hashing for your software and your servers. - * `passlib <https://pypi.org/project/passlib/>`_: - Comprehensive password hashing framework supporting over 30 schemes. - * `argon2-cffi <https://pypi.org/project/argon2-cffi/>`_: - The secure Argon2 password hashing algorithm. - * `legacycrypt <https://pypi.org/project/legacycrypt/>`_: - Wrapper to the POSIX crypt library call and associated functionality. - - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!uu` module, deprecated in Python 3.11: - the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!aifc` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!audioop` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* :pep:`594`: Remove the :mod:`!chunk` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) - -* Remove support for the keyword-argument method of creating - :class:`typing.TypedDict` types, deprecated in Python 3.11. - (Contributed by Tomas Roun in :gh:`104786`.) - -* :pep:`594`: Remove the :mod:`!imghdr` module, deprecated in Python 3.11: - use the projects - `filetype <https://pypi.org/project/filetype/>`_, - `puremagic <https://pypi.org/project/puremagic/>`_, - or `python-magic <https://pypi.org/project/python-magic/>`_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) - -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) - -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) - -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were - added in Python 3.8 and the old macros were deprecated in Python 3.11. - (Contributed by Irit Katriel in :gh:`105111`.) - -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) - -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) - -* Remove *cafile*, *capath* and *cadefault* parameters of the - :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the - *context* parameter instead. Please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - (Contributed by Victor Stinner in :gh:`105382`.) +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. -* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. - Use :attr:`webbrowser.MacOSXOSAScript.name <webbrowser.controller.name>` - attribute instead. - (Contributed by Nikita Sobolev in :gh:`105546`.) +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. -Porting to Python 3.13 -====================== +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. -This section lists previously described changes and other bugfixes -that may require changes to your code. +* :mod:`ssl` options and protocols: -* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` - were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` - and ``Py_TRASHCAN_END``. + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` - A tp_dealloc function that has the old macros, such as:: +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_SAFE_BEGIN(p); - ... - Py_TRASHCAN_SAFE_END - } +* :mod:`threading` methods: - should migrate to the new macros as follows:: + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_BEGIN(p, mytype_dealloc) - ... - Py_TRASHCAN_END - } +* :class:`typing.Text` (:gh:`92332`). - Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead -Build Changes -============= + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` -* Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate - the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. - (Contributed by Erlend Aasland in :gh:`105875`.) +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with Python release build and - :ref:`debug build <debug-build>`. - (Contributed by Victor Stinner in :gh:`108634`.) +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + it will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. -* The ``_stat`` C extension is now built with the :ref:`limited C API - <limited-c-api>`. - (Contributed by Victor Stinner in :gh:`85283`.) + +CPython Bytecode Changes +======================== + +* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a + yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was + changed to add a bit indicating whether the except-depth is 1, which + is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) C API Changes @@ -954,6 +1971,16 @@ New Features APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. (Contributed by Inada Naoki in :gh:`104922`.) +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` + in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + It makes these functions compatible with arguments of type + :c:expr:`const char * const *`, :c:expr:`const char **` or + :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C + without an explicit type cast. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) + * Add :c:func:`PyImport_AddModuleRef`: similar to :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead of a :term:`borrowed reference`. @@ -995,7 +2022,6 @@ New Features * If Python is built in :ref:`debug mode <debug-build>` or :option:`with assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. - If the assertion fails, make sure that the size is set before. (Contributed by Victor Stinner in :gh:`106168`.) * Add :c:func:`PyModule_Add` function: similar to @@ -1003,18 +2029,28 @@ New Features always steals a reference to the value. (Contributed by Serhiy Storchaka in :gh:`86493`.) -* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` +* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` functions: similar to :c:func:`PyDict_GetItemWithError` but returning a :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, these functions return -1 on error and so checking ``PyErr_Occurred()`` is not needed. (Contributed by Victor Stinner in :gh:`106004`.) -* Added :c:func:`PyDict_ContainsString` function: same as +* Added :c:func:`PyDict_SetDefaultRef`, which is similar to + :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on + insertion, and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) + +* Add :c:func:`PyDict_ContainsString` function: same as :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. (Contributed by Victor Stinner in :gh:`108314`.) +* Added :c:func:`PyList_GetItemRef` function: similar to + :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of + a :term:`borrowed reference`. + * Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is :term:`shutting down <interpreter shutdown>`. (Contributed by Victor Stinner in :gh:`108014`.) @@ -1058,8 +2094,211 @@ New Features parameter names. (Contributed by Serhiy Storchaka in :gh:`110815`.) +* Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, + :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API + (version 3.13). + (Contributed by Victor Stinner in :gh:`85283`.) + +* Add :c:func:`PySys_Audit` and :c:func:`PySys_AuditTuple` functions to the + limited C API. + (Contributed by Victor Stinner in :gh:`85283`.) + +* Add :c:func:`PyErr_FormatUnraisable` function: similar to + :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. + (Contributed by Serhiy Storchaka in :gh:`108082`.) + +* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to + Python ``list.extend()`` and ``list.clear()`` methods. + (Contributed by Victor Stinner in :gh:`111138`.) + +* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a + key from a dictionary and optionally return the removed value. This is + similar to :meth:`dict.pop`, but without the default value and not raising + :exc:`KeyError` if the key is missing. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) + +* Add :c:func:`Py_HashPointer` function to hash a pointer. + (Contributed by Victor Stinner in :gh:`111545`.) + +* Add :c:func:`PyObject_GenericHash` function that implements the default + hashing function of a Python object. + (Contributed by Serhiy Storchaka in :gh:`113024`.) + +* Add PyTime C API: + + * :c:type:`PyTime_t` type. + * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. + * Add functions: + + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. + + (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) + +* Add :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and + :c:func:`PyLong_FromUnsignedNativeBytes` functions to simplify converting + between native integer types and Python :class:`int` objects. + (Contributed by Steve Dower in :gh:`111140`.) + +* Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully + qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, + or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal + to ``"builtins"``. + (Contributed by Victor Stinner in :gh:`111696`.) + +* Add :c:func:`PyType_GetModuleName` function to get the type's module name. + Equivalent to getting the ``type.__module__`` attribute. + (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) + +* Add support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats to + :c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object + type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for + more information. + (Contributed by Victor Stinner in :gh:`111696`.) + +* Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions + to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a + :term:`strong reference` to the constant zero. + (Contributed by Victor Stinner in :gh:`115754`.) + +* Add :c:func:`PyType_GetModuleByDef` to the limited C API + (Contributed by Victor Stinner in :gh:`116936`.) + +* Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and + :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + destruction the same way the :mod:`tracemalloc` module does. (Contributed + by Pablo Galindo in :gh:`93502`.) + +* Add :c:func:`PyEval_GetFrameBuiltins`, :c:func:`PyEval_GetFrameGlobals`, and + :c:func:`PyEval_GetFrameLocals` to the C API. These replacements for + :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and + :c:func:`PyEval_GetLocals` return :term:`strong references <strong reference>` + rather than borrowed references. (Added as part of :pep:`667`.) + +* Add :c:type:`PyMutex` API, a lightweight mutex that occupies a single byte. + The :c:func:`PyMutex_Lock` function will release the GIL (if currently held) + if the operation needs to block. + (Contributed by Sam Gross in :gh:`108724`.) + +Build Changes +============= + +* The :file:`configure` option :option:`--with-system-libmpdec` now defaults + to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python + 3.15. + +* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886`.) + +* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) + +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) is now ABI compatible with the Python release build and + :ref:`debug build <debug-build>`. + (Contributed by Victor Stinner in :gh:`108634`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API <limited-c-api>`. + (Contributed by Victor Stinner in :gh:`85283`.) + +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* Python now bundles the `mimalloc library <https://github.com/microsoft/mimalloc>`__. + It is licensed under the MIT license; see :ref:`mimalloc license <mimalloc-license>`. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) + +* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI + flags. For example, the free-threaded build generates ``python-3.13t.pc`` + and the debug build generates ``python-3.13d.pc``. + + Porting to Python 3.13 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + must provide ``_is_main_interpreter()``, just like the module's + other "private" attributes. + (See :gh:`112826`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot. + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was + changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* :class:`functools.partial` now emits a :exc:`FutureWarning` when it is + used as a method. + Its behavior will be changed in future Python versions. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`121027`.) + +.. _pep667-porting-notes-py: + +* Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behaviour now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to ``locals()``. Code execution functions that + implicitly target ``locals()`` (such as ``exec`` and ``eval``) must be + passed an explicit namespace to access their results in an optimized scope. + (Changed as part of :pep:`667`.) + +* Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + +* Accessing :attr:`FrameType.f_locals <frame.f_locals>` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) + +Changes in the C API +-------------------- * ``Python.h`` no longer includes the ``<ieeefp.h>`` standard header. It was included for the ``finite()`` function which is now provided by the @@ -1067,16 +2306,6 @@ Porting to Python 3.13 also the ``HAVE_IEEEFP_H`` macro. (Contributed by Victor Stinner in :gh:`108765`.) -* ``Python.h`` no longer includes the ``<unistd.h>`` standard header file. If - needed, it should now be included explicitly. For example, it provides the - functions: ``read()``, ``write()``, ``close()``, ``isatty()``, ``lseek()``, - ``getpid()``, ``getcwd()``, ``sysconf()`` and ``getpagesize()``. - As a consequence, ``_POSIX_SEMAPHORES`` and ``_POSIX_THREADS`` macros are no - longer defined by ``Python.h``. The ``HAVE_UNISTD_H`` and ``HAVE_PTHREAD_H`` - macros defined by ``Python.h`` can be used to decide if ``<unistd.h>`` and - ``<pthread.h>`` header files can be included. - (Contributed by Victor Stinner in :gh:`108765`.) - * ``Python.h`` no longer includes these standard header files: ``<time.h>``, ``<sys/select.h>`` and ``<sys/time.h>``. If needed, they should now be included explicitly. For example, ``<time.h>`` provides the ``clock()`` and @@ -1085,11 +2314,11 @@ Porting to Python 3.13 and ``setitimer()`` functions. (Contributed by Victor Stinner in :gh:`108765`.) -* ``Python.h`` no longer includes the ``<ctype.h>`` standard header file. If - needed, it should now be included explicitly. For example, it provides - ``isalpha()`` and ``tolower()`` functions which are locale dependent. Python - provides locale independent functions, like :c:func:`!Py_ISALPHA` and - :c:func:`!Py_TOLOWER`. +* On Windows, ``Python.h`` no longer includes the ``<stddef.h>`` standard + header file. If needed, it should now be included explicitly. For example, it + provides ``offsetof()`` function, and ``size_t`` and ``ptrdiff_t`` types. + Including ``<stddef.h>`` explicitly was already needed by all other + platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. (Contributed by Victor Stinner in :gh:`108765`.) * If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, @@ -1097,51 +2326,79 @@ Porting to Python 3.13 are now undefined by ``<Python.h>``. (Contributed by Victor Stinner in :gh:`85283`.) -Deprecated ----------- - -* Passing optional arguments *maxsplit*, *count* and *flags* in module-level - functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional - arguments is now deprecated. - In future Python versions these parameters will be - :ref:`keyword-only <keyword-only_parameter>`. - (Contributed by Serhiy Storchaka in :gh:`56166`.) - -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) +* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` + were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. -* Deprecate old Python initialization functions: + A ``tp_dealloc`` function that has the old macros, such as:: - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_SAFE_BEGIN(p); + ... + Py_TRASHCAN_SAFE_END + } - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) + should migrate to the new macros as follows:: -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_BEGIN(p, mytype_dealloc) + ... + Py_TRASHCAN_END + } -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - <https://github.com/python/pythoncapi-compat/>`__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) + Note that ``Py_TRASHCAN_BEGIN`` has a second argument which + should be the deallocation function it is in. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) -Removed -------- +* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) + +* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed + to :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) + +.. _pep667-porting-notes-c: + +* The effects of mutating the dictionary returned from :c:func:`PyEval_GetLocals` in an + :term:`optimized scope` have changed. New dict entries added this way will now *only* be + visible to subsequent :c:func:`PyEval_GetLocals` calls in that frame, as + :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals <frame.f_locals>` no longer access the same underlying cached + dictionary. Changes made to entries for actual variable names and names added via the + write-through proxy interfaces will be overwritten on subsequent calls to + :c:func:`PyEval_GetLocals` in that frame. The recommended code update depends on how the + function was being used, so refer to the deprecation notice on the function for details. + (Changed as part of :pep:`667`.) + +* Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` now returns a + write-through proxy rather than a snapshot that gets updated at ill-specified times. + If a snapshot is desired, it must be created explicitly (e.g. with :c:func:`PyDict_Copy`) + or by calling the new :c:func:`PyEval_GetFrameLocals` API. (Changed as part of :pep:`667`.) + +* :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` + no longer have any effect. Calling these functions has been redundant since + Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. + (Changed as part of :pep:`667`.) + +* :c:func:`!PyFrame_LocalsToFast` no longer has any effect. Calling this function + is redundant now that :c:func:`PyFrame_GetLocals` returns a write-through proxy + for :term:`optimized scopes <optimized scope>`. (Changed as part of :pep:`667`.) + +Removed C APIs +-------------- * Remove many APIs (functions, macros, variables) with names prefixed by ``_Py`` or ``_PY`` (considered as private API). If your project is affected @@ -1150,14 +2407,14 @@ Removed add ``cc @vstinner`` to the issue to notify Victor Stinner. (Contributed by Victor Stinner in :gh:`106320`.) -* Remove functions deprecated in Python 3.9. +* Remove functions deprecated in Python 3.9: * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be *NULL*, keyword arguments must be a - :class:`dict` or *NULL*, whereas removed functions checked arguments type - and accepted *NULL* positional and keyword arguments. + :class:`tuple` and must not be ``NULL``, keyword arguments must be a + :class:`dict` or ``NULL``, whereas removed functions checked arguments type + and accepted ``NULL`` positional and keyword arguments. To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using :c:func:`PyTuple_New(0) <PyTuple_New>`. @@ -1210,12 +2467,8 @@ Removed * ``PySys_AddWarnOption()``: use :c:member:`PyConfig.warnoptions` instead. * ``PySys_AddXOption()``: use :c:member:`PyConfig.xoptions` instead. * ``PySys_HasWarnOptions()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. * ``PySys_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. - * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on Windows). @@ -1225,9 +2478,9 @@ Removed Configuration <init-config>` instead (:pep:`587`), added to Python 3.8. (Contributed by Victor Stinner in :gh:`105145`.) -* Remove ``PyEval_InitThreads()`` and ``PyEval_ThreadsInitialized()`` - functions, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` - always creates the GIL: calling ``PyEval_InitThreads()`` did nothing and +* Remove ``PyEval_ThreadsInitialized()`` + function, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` + always creates the GIL: calling ``PyEval_InitThreads()`` does nothing and ``PyEval_ThreadsInitialized()`` always returned non-zero. (Contributed by Victor Stinner in :gh:`105182`.) @@ -1241,20 +2494,6 @@ Removed (Contributed by Victor Stinner in :gh:`105182`.) -* Remove the old aliases to functions calling functions which were kept for - backward compatibility with Python 3.8 provisional API: - - * ``_PyObject_CallMethodNoArgs()``: use ``PyObject_CallMethodNoArgs()`` - * ``_PyObject_CallMethodOneArg()``: use ``PyObject_CallMethodOneArg()`` - * ``_PyObject_CallOneArg()``: use ``PyObject_CallOneArg()`` - * ``_PyObject_FastCallDict()``: use ``PyObject_VectorcallDict()`` - * ``_PyObject_Vectorcall()``: use ``PyObject_Vectorcall()`` - * ``_PyObject_VectorcallMethod()``: use ``PyObject_VectorcallMethod()`` - * ``_PyVectorcall_Function()``: use ``PyVectorcall_Function()`` - - Just remove the underscore prefix to update your code. - (Contributed by Victor Stinner in :gh:`106084`.) - * Remove private ``_PyObject_FastCall()`` function: use ``PyObject_Vectorcall()`` which is available since Python 3.8 (:pep:`590`). @@ -1275,11 +2514,63 @@ Removed be used instead. (Contributed by Serhiy Storchaka in :gh:`86493`.) +Deprecated C APIs +----------------- + +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly + the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and + ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + :envvar:`PYTHONHOME` environment variable instead. + + Functions scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just + an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + Scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105396`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed + reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it + returns a :term:`strong reference`. The `pythoncapi-compat project + <https://github.com/python/pythoncapi-compat/>`__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + +* Deprecate the :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and + :c:func:`PyEval_GetLocals` functions, which return a :term:`borrowed reference`. + Refer to the deprecation notices on each function for their recommended replacements. + (Soft deprecated as part of :pep:`667`.) + Pending Removal in Python 3.14 ------------------------------ * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. + +* Functions to configure the Python initialization, deprecated in Python 3.11: + + * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. + * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. + * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. + * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + * Global configuration variables: * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` @@ -1311,6 +2602,7 @@ Pending Removal in Python 3.14 Pending Removal in Python 3.15 ------------------------------ +* The bundled copy of ``libmpdecimal``. * :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. * :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. * :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. @@ -1361,3 +2653,13 @@ removed, although there is currently no date scheduled for their removal. * Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. (Contributed by Victor Stinner in :gh:`110014`.) + + +Regression Test Changes +======================= + +* Python built with :file:`configure` :option:`--with-pydebug` now + supports a :option:`-X presite=package.module <-X>` command-line + option. If used, it specifies a module that should be imported early + in the lifecycle of the interpreter, before ``site.py`` is executed. + (Contributed by Åukasz Langa in :gh:`110769`.) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst new file mode 100644 index 00000000000000..da9b45cd8e58b3 --- /dev/null +++ b/Doc/whatsnew/3.14.rst @@ -0,0 +1,381 @@ + +**************************** + What's New In Python 3.14 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.14, compared to 3.13. + +For full details, see the :ref:`changelog <changelog>`. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.14 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.14. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + +ast +--- + +* Added :func:`ast.compare` for comparing two ASTs. + (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) + +* Add support for :func:`copy.replace` for AST nodes. + + (Contributed by Bénédikt Tran in :gh:`121141`.) + +os +-- + +* Added the :data:`os.environ.refresh() <os.environ>` method to update + :data:`os.environ` with changes to the environment made by :func:`os.putenv`, + by :func:`os.unsetenv`, or made outside Python in the same process. + (Contributed by Victor Stinner in :gh:`120057`.) + +pathlib +------- + +* Add :meth:`pathlib.Path.copy`, which copies the content of one file to + another, like :func:`shutil.copyfile`. + (Contributed by Barney Gale in :gh:`73991`.) +* Add :meth:`pathlib.Path.copytree`, which copies one directory tree to + another. + (Contributed by Barney Gale in :gh:`73991`.) + +pdb +--- + +* Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace()`) now + reuse the most recent :class:`~pdb.Pdb` instance that calls + :meth:`~pdb.Pdb.set_trace()`, instead of creating a new one each time. + As a result, all the instance specific data like :pdbcmd:`display` and + :pdbcmd:`commands` are preserved across hard-coded breakpoints. + (Contributed by Tian Gao in :gh:`121450`.) + +symtable +-------- + +* Expose the following :class:`symtable.Symbol` methods: + + * :meth:`~symtable.Symbol.is_free_class` + * :meth:`~symtable.Symbol.is_comp_iter` + * :meth:`~symtable.Symbol.is_comp_cell` + + (Contributed by Bénédikt Tran in :gh:`120029`.) + + +Optimizations +============= + +asyncio +------- + +* :mod:`asyncio` now uses double linked list implementation for native tasks + which speeds up execution by 10% on standard pyperformance benchmarks and + reduces memory usage. + (Contributed by Kumar Aditya in :gh:`107803`.) + + + + +Deprecated +========== + +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :term:`Soft deprecate <soft deprecated>` :func:`os.popen` and + :func:`os.spawn* <os.spawnl>` functions. They should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + (Contributed by Victor Stinner in :gh:`120743`.) + + +Removed +======= + +argparse +-------- + +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction`. + They were deprecated since 3.12. + +ast +--- + +* Remove the following classes. They were all deprecated since Python 3.8, + and have emitted deprecation warnings since Python 3.12: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. As a consequence of these removals, + user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, + ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom + :class:`ast.NodeVisitor` subclasses will no longer be called when the + ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + method instead. + + Also, remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!ast.Constant.n` + * :attr:`!ast.Constant.s` + + Use :attr:`!ast.Constant.value` instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + +asyncio +------- + +* Remove the following classes and functions. They were all deprecated and + emitted deprecation warnings since Python 3.12: + + * :class:`!asyncio.AbstractChildWatcher` + * :class:`!asyncio.SafeChildWatcher` + * :class:`!asyncio.MultiLoopChildWatcher` + * :class:`!asyncio.FastChildWatcher` + * :class:`!asyncio.ThreadedChildWatcher` + * :class:`!asyncio.PidfdChildWatcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` + * :func:`!asyncio.get_child_watcher` + * :func:`!asyncio.set_child_watcher` + + (Contributed by Kumar Aditya in :gh:`120804`.) + + +collections.abc +--------------- + +* Remove :class:`!collections.abc.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +email +----- + +* Remove the *isdst* parameter from :func:`email.utils.localtime`. + (Contributed by Hugo van Kemenade in :gh:`118798`.) + +importlib +--------- + +* Remove deprecated :mod:`importlib.abc` classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +itertools +--------- + +* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +pathlib +------- + +* Remove support for passing additional keyword arguments to + :class:`pathlib.Path`. In previous versions, any such arguments are ignored. +* Remove support for passing additional positional arguments to + :meth:`pathlib.PurePath.relative_to` and + :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such + arguments are joined onto *other*. + +pty +___ + +* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. + They had previously raised a :exc:`DeprecationWarning` since Python 3.12. + Use :func:`pty.openpty` instead. + (Contributed by Nikita Sobolev in :gh:`118824`.) + +sqlite3 +------- + +* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. + (Contributed by Hugo van Kemenade in :gh:`118924`.) + +* Disallow using a sequence of parameters with named placeholders. + This had previously raised a :exc:`DeprecationWarning` since Python 3.12; + it will now raise a :exc:`sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) + +typing +------ + +* Remove :class:`!typing.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +urllib +------ + +* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.11. + (Contributed by Nikita Sobolev in :gh:`118827`.) + +Others +------ + +* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed + by Jelle Zijlstra in :gh:`118767`.) + +* The :func:`int` built-in no longer delegates to + :meth:`~object.__trunc__`. Classes that want to support conversion to + integer must implement either :meth:`~object.__int__` or + :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) + + +Porting to Python 3.14 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +Build Changes +============= + + +C API Changes +============= + +New Features +------------ + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object: + + * :c:func:`PyUnicodeWriter_Create`. + * :c:func:`PyUnicodeWriter_Discard`. + * :c:func:`PyUnicodeWriter_Finish`. + * :c:func:`PyUnicodeWriter_WriteChar`. + * :c:func:`PyUnicodeWriter_WriteUTF8`. + * :c:func:`PyUnicodeWriter_WriteUCS4`. + * :c:func:`PyUnicodeWriter_WriteWideChar`. + * :c:func:`PyUnicodeWriter_WriteStr`. + * :c:func:`PyUnicodeWriter_WriteRepr`. + * :c:func:`PyUnicodeWriter_WriteSubstring`. + * :c:func:`PyUnicodeWriter_Format`. + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + + (Contributed by Victor Stinner in :gh:`119182`.) + +Porting to Python 3.14 +---------------------- + +* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` is now implemented as + an opaque function call to hide implementation details. + (Contributed by Victor Stinner in :gh:`120600`.) + + +Deprecated +---------- + +* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` + and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, + use instead :c:macro:`!isnan`, :c:macro:`!isinf` and + :c:macro:`!isfinite` available from :file:`math.h` + since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +Removed +------- + +* Creating :c:data:`immutable types <Py_TPFLAGS_IMMUTABLETYPE>` with mutable + bases was deprecated since 3.12 and now raises a :exc:`TypeError`. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index df32b76b6d7b03..a6b38207b70c93 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -183,7 +183,7 @@ PEP 391: Dictionary Based Configuration for Logging The :mod:`logging` module provided two kinds of configuration, one style with function calls for each option or another style driven by an external file saved -in a :mod:`ConfigParser` format. Those options did not provide the flexibility +in a :mod:`configparser` format. Those options did not provide the flexibility to create configurations from JSON or YAML files, nor did they support incremental configuration, which is needed for specifying logger options from a command line. @@ -344,8 +344,8 @@ aspects that are visible to the programmer: * The :mod:`importlib.abc` module has been updated with new :term:`abstract base classes <abstract base class>` for loading bytecode files. The obsolete - ABCs, :class:`~importlib.abc.PyLoader` and - :class:`~importlib.abc.PyPycLoader`, have been deprecated (instructions on how + ABCs, :class:`!PyLoader` and + :class:`!PyPycLoader`, have been deprecated (instructions on how to stay Python 3.1 compatible are included with the documentation). .. seealso:: @@ -401,7 +401,7 @@ The *native strings* are always of type :class:`str` but are restricted to code points between *U+0000* through *U+00FF* which are translatable to bytes using *Latin-1* encoding. These strings are used for the keys and values in the environment dictionary and for response headers and statuses in the -:func:`start_response` function. They must follow :rfc:`2616` with respect to +:func:`!start_response` function. They must follow :rfc:`2616` with respect to encoding. That is, they must either be *ISO-8859-1* characters or use :rfc:`2047` MIME encoding. @@ -415,8 +415,8 @@ points: encoded in utf-8 was using ``h.encode('utf-8')`` now needs to convert from bytes to native strings using ``h.encode('utf-8').decode('latin-1')``. -* Values yielded by an application or sent using the :meth:`write` method - must be byte strings. The :func:`start_response` function and environ +* Values yielded by an application or sent using the :meth:`!write` method + must be byte strings. The :func:`!start_response` function and environ must use native strings. The two cannot be mixed. For server implementers writing CGI-to-WSGI pathways or other CGI-style @@ -499,7 +499,7 @@ Some smaller changes made to the core Python language are: * The :func:`hasattr` function works by calling :func:`getattr` and detecting whether an exception is raised. This technique allows it to detect methods - created dynamically by :meth:`__getattr__` or :meth:`__getattribute__` which + created dynamically by :meth:`~object.__getattr__` or :meth:`~object.__getattribute__` which would otherwise be absent from the class dictionary. Formerly, *hasattr* would catch any exception, possibly masking genuine errors. Now, *hasattr* has been tightened to only catch :exc:`AttributeError` and let other @@ -620,7 +620,7 @@ Some smaller changes made to the core Python language are: * :class:`range` objects now support *index* and *count* methods. This is part of an effort to make more objects fully implement the - :class:`collections.Sequence` :term:`abstract base class`. As a result, the + :class:`collections.Sequence <collections.abc.Sequence>` :term:`abstract base class`. As a result, the language will have a more uniform API. In addition, :class:`range` objects now support slicing and negative indices, even with values larger than :data:`sys.maxsize`. This makes *range* more interoperable with lists:: @@ -720,7 +720,7 @@ format. elementtree ----------- -The :mod:`xml.etree.ElementTree` package and its :mod:`xml.etree.cElementTree` +The :mod:`xml.etree.ElementTree` package and its :mod:`!xml.etree.cElementTree` counterpart have been updated to version 1.3. Several new and useful functions and methods have been added: @@ -743,8 +743,8 @@ Several new and useful functions and methods have been added: Two methods have been deprecated: -* :meth:`xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. -* :meth:`xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. +* :meth:`!xml.etree.ElementTree.getchildren` use ``list(elem)`` instead. +* :meth:`!xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. For details of the update, see `Introducing ElementTree <https://web.archive.org/web/20200703234532/http://effbot.org/zone/elementtree-13-intro.htm>`_ @@ -785,14 +785,15 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - <https://code.activestate.com/recipes/498245/>`_\, `recipe 577479 - <https://code.activestate.com/recipes/577479/>`_\, :issue:`10586`, and + <https://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators/>`_\, `recipe 577479 + <https://code.activestate.com/recipes/577479-simple-caching-decorator/>`_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to - be introspected. It also copies :attr:`__annotations__` if defined. And now - it also gracefully skips over missing attributes such as :attr:`__doc__` which + be introspected. It also copies :attr:`~function.__annotations__` if + defined. And now it also gracefully skips over missing attributes such as + :attr:`~function.__doc__` which might not be defined for the wrapped callable. In the above example, the cache can be removed by recovering the original @@ -1007,13 +1008,13 @@ datetime and time after 1900. The new supported year range is from 1000 to 9999 inclusive. * Whenever a two-digit year is used in a time tuple, the interpretation has been - governed by :data:`time.accept2dyear`. The default is ``True`` which means that + governed by :data:`!time.accept2dyear`. The default is ``True`` which means that for a two-digit year, the century is guessed according to the POSIX rules governing the ``%y`` strptime format. Starting with Py3.2, use of the century guessing heuristic will emit a :exc:`DeprecationWarning`. Instead, it is recommended that - :data:`time.accept2dyear` be set to ``False`` so that large date ranges + :data:`!time.accept2dyear` be set to ``False`` so that large date ranges can be used without guesswork:: >>> import time, warnings @@ -1031,7 +1032,7 @@ datetime and time 'Fri Jan 1 12:34:56 11' Several functions now have significantly expanded date ranges. When - :data:`time.accept2dyear` is false, the :func:`time.asctime` function will + :data:`!time.accept2dyear` is false, the :func:`time.asctime` function will accept any year that fits in a C int, while the :func:`time.mktime` and :func:`time.strftime` functions will accept the full range supported by the corresponding operating system functions. @@ -1147,15 +1148,15 @@ for slice notation are well-suited to in-place editing:: reprlib ------- -When writing a :meth:`__repr__` method for a custom container, it is easy to +When writing a :meth:`~object.__repr__` method for a custom container, it is easy to forget to handle the case where a member refers back to the container itself. Python's builtin objects such as :class:`list` and :class:`set` handle self-reference by displaying "..." in the recursive part of the representation string. -To help write such :meth:`__repr__` methods, the :mod:`reprlib` module has a new +To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to -:meth:`__repr__` and substituting a placeholder string instead:: +:meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): ... @recursive_repr() @@ -1307,7 +1308,7 @@ used for the imaginary part of a number: >>> sys.hash_info # doctest: +SKIP sys.hash_info(width=64, modulus=2305843009213693951, inf=314159, nan=0, imag=1000003) -An early decision to limit the inter-operability of various numeric types has +An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary @@ -1335,7 +1336,7 @@ Decimal('1.100000000000000088817841970012523233890533447265625') Fraction(2476979795053773, 2251799813685248) Another useful change for the :mod:`decimal` module is that the -:attr:`Context.clamp` attribute is now public. This is useful in creating +:attr:`Context.clamp <decimal.Context.clamp>` attribute is now public. This is useful in creating contexts that correspond to the decimal interchange formats specified in IEEE 754 (see :issue:`8540`). @@ -1427,7 +1428,7 @@ before compressing and decompressing: Aides and Brian Curtin in :issue:`9962`, :issue:`1675951`, :issue:`7471` and :issue:`2846`.) -Also, the :class:`zipfile.ZipExtFile` class was reworked internally to represent +Also, the :class:`zipfile.ZipExtFile <zipfile.ZipFile.open>` class was reworked internally to represent files stored inside an archive. The new implementation is significantly faster and can be wrapped in an :class:`io.BufferedReader` object for more speedups. It also solves an issue where interleaved calls to *read* and *readline* gave the @@ -1595,7 +1596,7 @@ sqlite3 The :mod:`sqlite3` module was updated to pysqlite version 2.6.0. It has two new capabilities. -* The :attr:`sqlite3.Connection.in_transit` attribute is true if there is an +* The :attr:`!sqlite3.Connection.in_transit` attribute is true if there is an active transaction for uncommitted changes. * The :meth:`sqlite3.Connection.enable_load_extension` and @@ -1642,11 +1643,11 @@ for secure (encrypted, authenticated) internet connections: other options. It includes a :meth:`~ssl.SSLContext.wrap_socket` for creating an SSL socket from an SSL context. -* A new function, :func:`ssl.match_hostname`, supports server identity +* A new function, :func:`!ssl.match_hostname`, supports server identity verification for higher-level protocols by implementing the rules of HTTPS (from :rfc:`2818`) which are also suitable for other protocols. -* The :func:`ssl.wrap_socket` constructor function now takes a *ciphers* +* The :func:`ssl.wrap_socket() <ssl.SSLContext.wrap_socket>` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation <https://www.openssl.org/docs/man1.0.2/man1/ciphers.html#CIPHER-LIST-FORMAT>`__. @@ -1758,7 +1759,7 @@ names. (Contributed by Michael Foord.) * Experimentation at the interactive prompt is now easier because the - :class:`unittest.case.TestCase` class can now be instantiated without + :class:`unittest.TestCase` class can now be instantiated without arguments: >>> from unittest import TestCase @@ -1796,7 +1797,7 @@ names. * In addition, the method names in the module have undergone a number of clean-ups. For example, :meth:`~unittest.TestCase.assertRegex` is the new name for - :meth:`~unittest.TestCase.assertRegexpMatches` which was misnamed because the + :meth:`!assertRegexpMatches` which was misnamed because the test uses :func:`re.search`, not :func:`re.match`. Other methods using regular expressions are now named using short form "Regex" in preference to "Regexp" -- this matches the names used in other unittest implementations, @@ -1811,11 +1812,11 @@ names. =============================== ============================== Old Name Preferred Name =============================== ============================== - :meth:`assert_` :meth:`.assertTrue` - :meth:`assertEquals` :meth:`.assertEqual` - :meth:`assertNotEquals` :meth:`.assertNotEqual` - :meth:`assertAlmostEquals` :meth:`.assertAlmostEqual` - :meth:`assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` + :meth:`!assert_` :meth:`.assertTrue` + :meth:`!assertEquals` :meth:`.assertEqual` + :meth:`!assertNotEquals` :meth:`.assertNotEqual` + :meth:`!assertAlmostEquals` :meth:`.assertAlmostEqual` + :meth:`!assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` =============================== ============================== Likewise, the ``TestCase.fail*`` methods deprecated in Python 3.1 are expected @@ -1823,7 +1824,7 @@ names. (Contributed by Ezio Melotti; :issue:`9424`.) -* The :meth:`~unittest.TestCase.assertDictContainsSubset` method was deprecated +* The :meth:`!assertDictContainsSubset` method was deprecated because it was misimplemented with the arguments in the wrong order. This created hard-to-debug optical illusions where tests like ``TestCase().assertDictContainsSubset({'a':1, 'b':2}, {'a':1})`` would fail. @@ -1857,12 +1858,12 @@ structure. asyncore -------- -:class:`asyncore.dispatcher` now provides a -:meth:`~asyncore.dispatcher.handle_accepted()` method +:class:`!asyncore.dispatcher` now provides a +:meth:`!handle_accepted()` method returning a ``(sock, addr)`` pair which is called when a connection has actually been established with a new remote endpoint. This is supposed to be used as a -replacement for old :meth:`~asyncore.dispatcher.handle_accept()` and avoids -the user to call :meth:`~asyncore.dispatcher.accept()` directly. +replacement for old :meth:`!handle_accept()` and avoids +the user to call :meth:`!accept()` directly. (Contributed by Giampaolo Rodolà; :issue:`6706`.) @@ -1996,7 +1997,7 @@ under-the-hood. dbm --- -All database modules now support the :meth:`get` and :meth:`setdefault` methods. +All database modules now support the :meth:`!get` and :meth:`!setdefault` methods. (Suggested by Ray Allen in :issue:`9523`.) @@ -2117,7 +2118,7 @@ The :mod:`pdb` debugger module gained a number of usability improvements: :file:`.pdbrc` script file. * A :file:`.pdbrc` script file can contain ``continue`` and ``next`` commands that continue debugging. -* The :class:`Pdb` class constructor now accepts a *nosigint* argument. +* The :class:`~pdb.Pdb` class constructor now accepts a *nosigint* argument. * New commands: ``l(list)``, ``ll(long list)`` and ``source`` for listing source code. * New commands: ``display`` and ``undisplay`` for showing or hiding @@ -2133,7 +2134,7 @@ configparser The :mod:`configparser` module was modified to improve usability and predictability of the default parser and its supported INI syntax. The old -:class:`ConfigParser` class was removed in favor of :class:`SafeConfigParser` +:class:`!ConfigParser` class was removed in favor of :class:`!SafeConfigParser` which has in turn been renamed to :class:`~configparser.ConfigParser`. Support for inline comments is now turned off by default and section or option duplicates are not allowed in a single configuration source. @@ -2393,11 +2394,11 @@ A number of small performance enhancements have been added: (Contributed by Antoine Pitrou; :issue:`3001`.) -* The fast-search algorithm in stringlib is now used by the :meth:`split`, - :meth:`rsplit`, :meth:`splitlines` and :meth:`replace` methods on +* The fast-search algorithm in stringlib is now used by the :meth:`~str.split`, + :meth:`~str.rsplit`, :meth:`~str.splitlines` and :meth:`~str.replace` methods on :class:`bytes`, :class:`bytearray` and :class:`str` objects. Likewise, the - algorithm is also used by :meth:`rfind`, :meth:`rindex`, :meth:`rsplit` and - :meth:`rpartition`. + algorithm is also used by :meth:`~str.rfind`, :meth:`~str.rindex`, :meth:`~str.rsplit` and + :meth:`~str.rpartition`. (Patch by Florent Xicluna in :issue:`7622` and :issue:`7462`.) @@ -2409,11 +2410,11 @@ A number of small performance enhancements have been added: There were several other minor optimizations. Set differencing now runs faster when one operand is much larger than the other (patch by Andress Bennetts in -:issue:`8685`). The :meth:`array.repeat` method has a faster implementation -(:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` +:issue:`8685`). The :meth:`!array.repeat` method has a faster implementation +(:issue:`1569291` by Alexander Belopolsky). The :class:`~http.server.BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by -Christos Georgiou). And :class:`ConfigParser` loads multi-line arguments a bit +Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit faster (:issue:`7113` by Åukasz Langa). @@ -2561,11 +2562,11 @@ Changes to Python's build process and to the C API include: (Suggested by Raymond Hettinger and implemented by Benjamin Peterson; :issue:`9778`.) -* A new macro :c:macro:`Py_VA_COPY` copies the state of the variable argument +* A new macro :c:macro:`!Py_VA_COPY` copies the state of the variable argument list. It is equivalent to C99 *va_copy* but available on all Python platforms (:issue:`2443`). -* A new C API function :c:func:`PySys_SetArgvEx` allows an embedded interpreter +* A new C API function :c:func:`!PySys_SetArgvEx` allows an embedded interpreter to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). @@ -2613,8 +2614,8 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :mod:`configparser` module has a number of clean-ups. The major change is - to replace the old :class:`ConfigParser` class with long-standing preferred - alternative :class:`SafeConfigParser`. In addition there are a number of + to replace the old :class:`!ConfigParser` class with long-standing preferred + alternative :class:`!SafeConfigParser`. In addition there are a number of smaller incompatibilities: * The interpolation syntax is now validated on @@ -2649,8 +2650,9 @@ require changes to your code: * :class:`bytearray` objects can no longer be used as filenames; instead, they should be converted to :class:`bytes`. -* The :meth:`array.tostring` and :meth:`array.fromstring` have been renamed to - :meth:`array.tobytes` and :meth:`array.frombytes` for clarity. The old names +* The :meth:`!array.tostring` and :meth:`!array.fromstring` have been renamed to + :meth:`array.tobytes() <array.array.tobytes>` and + :meth:`array.frombytes() <array.array.frombytes>` for clarity. The old names have been deprecated. (See :issue:`8990`.) * ``PyArg_Parse*()`` functions: @@ -2663,7 +2665,7 @@ require changes to your code: instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. -* The :func:`sys.setfilesystemencoding` function was removed because +* The :func:`!sys.setfilesystemencoding` function was removed because it had a flawed design. * The :func:`random.seed` function and method now salt string seeds with an @@ -2671,7 +2673,7 @@ require changes to your code: reproduce Python 3.1 sequences, set the *version* argument to *1*, ``random.seed(s, version=1)``. -* The previously deprecated :func:`string.maketrans` function has been removed +* The previously deprecated :func:`!string.maketrans` function has been removed in favor of the static methods :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, @@ -2681,7 +2683,7 @@ require changes to your code: (Contributed by Georg Brandl; :issue:`5675`.) -* The previously deprecated :func:`contextlib.nested` function has been removed +* The previously deprecated :func:`!contextlib.nested` function has been removed in favor of a plain :keyword:`with` statement which can accept multiple context managers. The latter technique is faster (because it is built-in), and it does a better job finalizing multiple context managers when one of them @@ -2736,8 +2738,8 @@ require changes to your code: thread-state aware APIs (such as :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`) should be used instead. -* Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and - a new function, :func:`asyncore.handle_accepted`, was added to replace it. +* Due to security risks, :func:`!asyncore.handle_accept` has been deprecated, and + a new function, :func:`!asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 5674bc7f359b72..29b4034e32821c 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -174,7 +174,7 @@ Features b or c are now hashable. (Contributed by Antoine Pitrou in :issue:`13411`.) * Arbitrary slicing of any 1-D arrays type is supported. For example, it - is now possible to reverse a memoryview in O(1) by using a negative step. + is now possible to reverse a memoryview in *O*\ (1) by using a negative step. API changes ----------- @@ -1052,7 +1052,7 @@ their ``__init__`` method (for example, file objects) or in their crypt ----- -Addition of salt and modular crypt format (hashing method) and the :func:`~!crypt.mksalt` +Addition of salt and modular crypt format (hashing method) and the :func:`!mksalt` function to the :mod:`!crypt` module. (:issue:`10924`) @@ -1845,7 +1845,7 @@ signal smtpd ----- -The :mod:`smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` +The :mod:`!smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` (size extension). Per the standard, these extensions are enabled if and only if the client initiates the session with an ``EHLO`` command. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 2ddab76814369e..938dd273e7e102 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -605,15 +605,15 @@ Using ``ABC`` as a base class has essentially the same effect as specifying aifc ---- -The :meth:`~!aifc.aifc.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) :func:`!aifc.open` now supports the context management protocol: when used in a -:keyword:`with` block, the :meth:`~!aifc.aifc.close` method of the returned +:keyword:`with` block, the :meth:`!close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) -The :meth:`~!aifc.aifc.writeframesraw` and :meth:`~!aifc.aifc.writeframes` +The :meth:`!writeframesraw` and :meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -632,7 +632,7 @@ audioop :mod:`!audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka in :issue:`12866`.) -New :func:`~!audioop.byteswap` function converts big-endian samples to +New :func:`!byteswap` function converts big-endian samples to little-endian and vice versa. (Contributed by Serhiy Storchaka in :issue:`19641`.) @@ -872,7 +872,7 @@ multiple implementations of an operation that allows it to work with PEP written and implemented by Åukasz Langa. :func:`~functools.total_ordering` now supports a return value of -:const:`NotImplemented` from the underlying comparison function. (Contributed +:data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) A pure-python version of the :func:`~functools.partial` function is now in the @@ -1369,9 +1369,9 @@ error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in smtpd ----- -The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +The :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *map* keyword argument which, if specified, is passed in to -:class:`asynchat.async_chat` as its *map* argument. This allows an application +:class:`!asynchat.async_chat` as its *map* argument. This allows an application to avoid affecting the global socket map. (Contributed by Vinay Sajip in :issue:`11959`.) @@ -1495,7 +1495,7 @@ The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional stat ---- -The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C +The :mod:`stat` module is now backed by a C implementation in :mod:`!_stat`. A C implementation is required as most of the values aren't standardized and are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) @@ -1528,7 +1528,7 @@ work on Windows. This change was actually inadvertently made in 3.3.4. sunau ----- -The :meth:`~!sunau.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) :meth:`!sunau.open` now supports the context management protocol: when used in a @@ -1540,8 +1540,8 @@ in :issue:`18878`.) support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) -The :meth:`~!sunau.AU_write.writeframesraw` and -:meth:`~!sunau.AU_write.writeframes` methods now accept any :term:`bytes-like +The :meth:`!writeframesraw` and +:meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -1936,7 +1936,7 @@ Other Improvements * The :ref:`python <using-on-cmdline>` command has a new :ref:`option <using-on-misc-options>`, ``-I``, which causes it to run in "isolated mode", which means that :data:`sys.path` contains neither the script's directory nor - the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment + the user's ``site-packages`` directory, and all :envvar:`!PYTHON*` environment variables are ignored (it implies both ``-s`` and ``-E``). Other restrictions may also be applied in the future, with the goal being to isolate the execution of a script from the user's environment. This is @@ -2370,7 +2370,7 @@ Changes in the Python API :issue:`18011`.) Note: this change was also inadvertently applied in Python 3.3.3. -* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when +* The :attr:`!file` attribute is now automatically closed when the creating :class:`!cgi.FieldStorage` instance is garbage collected. If you were pulling the file object out separately from the :class:`!cgi.FieldStorage` instance and not keeping the instance alive, then you should either store the @@ -2405,15 +2405,15 @@ Changes in the Python API storage). (:issue:`17094`.) * Parameter names in ``__annotations__`` dicts are now mangled properly, - similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in - :issue:`20625`.) + similarly to :attr:`~function.__kwdefaults__`. + (Contributed by Yury Selivanov in :issue:`20625`.) * :attr:`hashlib.hash.name` now always returns the identifier in lower case. Previously some builtin hashes had uppercase names, but now that it is a formal public interface the naming has been made consistent (:issue:`18532`). * Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + are run, test harnesses that reuse a :class:`~unittest.TestSuite` to re-run a set of tests may fail. Test suites should not be re-used in this fashion since it means state is retained between test runs, breaking the test isolation that :mod:`unittest` is designed to provide. However, if the lack diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index ae6affcab664c6..75654f3e78eb16 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -878,7 +878,7 @@ size of decompressed data. (Contributed by Nikolaus Rath in :issue:`15955`.) cgi --- -The :class:`~cgi.FieldStorage` class now supports the :term:`context manager` +The :class:`!FieldStorage` class now supports the :term:`context manager` protocol. (Contributed by Berker Peksag in :issue:`20289`.) @@ -951,7 +951,7 @@ New :class:`~collections.abc.Awaitable`, :class:`~collections.abc.Coroutine`, (Contributed by Yury Selivanov in :issue:`24184`.) For earlier Python versions, a backport of the new ABCs is available in an -external `PyPI package <https://pypi.org/project/backports_abc>`_. +external :pypi:`PyPI package <backports_abc>`. compileall @@ -1252,7 +1252,7 @@ Oberkirch in :issue:`21800`.) imghdr ------ -The :func:`~!imghdr.what` function now recognizes the +The :func:`!what` function now recognizes the `OpenEXR <https://www.openexr.com>`_ format (contributed by Martin Vignali and Claudiu Popa in :issue:`20295`), and the `WebP <https://en.wikipedia.org/wiki/WebP>`_ format @@ -1663,34 +1663,34 @@ during debugging, instead of integer "magic numbers". smtpd ----- -Both the :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +Both the :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *decode_data* keyword argument to determine if the ``DATA`` portion of the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead provided to the -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` method as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set to ``False``, the ``process_message`` method must be prepared to accept keyword arguments. (Contributed by Maciej Szulik in :issue:`19662`.) -The :class:`~smtpd.SMTPServer` class now advertises the ``8BITMIME`` extension +The :class:`!SMTPServer` class now advertises the ``8BITMIME`` extension (:rfc:`6152`) if *decode_data* has been set ``True``. If the client specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) -The :class:`~smtpd.SMTPServer` class now also supports the ``SMTPUTF8`` +The :class:`!SMTPServer` class now also supports the ``SMTPUTF8`` extension (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`SMTPServer.process_message() <smtpd.SMTPServer.process_message>` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. It is the responsibility of the ``process_message`` method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan Oberkirch in :issue:`21725`.) It is now possible to provide, directly or via name resolution, IPv6 -addresses in the :class:`~smtpd.SMTPServer` constructor, and have it +addresses in the :class:`!SMTPServer` constructor, and have it successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.) @@ -1714,7 +1714,7 @@ support :rfc:`6531` (SMTPUTF8). sndhdr ------ -The :func:`~sndhdr.what` and :func:`~sndhdr.whathdr` functions now return +The :func:`!what` and :func:`!whathdr` functions now return a :func:`~collections.namedtuple`. (Contributed by Claudiu Popa in :issue:`18615`.) @@ -1935,8 +1935,8 @@ specifying the namespace in which the code will be running. tkinter ------- -The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment -on Windows has been replaced by a private function in the :mod:`_tkinter` +The :mod:`!tkinter._fix` module used for setting up the Tcl/Tk environment +on Windows has been replaced by a private function in the :mod:`!_tkinter` module which makes no permanent changes to environment variables. (Contributed by Zachary Ware in :issue:`20035`.) @@ -1947,7 +1947,8 @@ traceback --------- New :func:`~traceback.walk_stack` and :func:`~traceback.walk_tb` -functions to conveniently traverse frame and traceback objects. +functions to conveniently traverse frame and +:ref:`traceback objects <traceback-objects>`. (Contributed by Robert Collins in :issue:`17911`.) New lightweight classes: :class:`~traceback.TracebackException`, @@ -2295,9 +2296,9 @@ slated for removal in Python 3.6. The :func:`asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. -The :mod:`smtpd` module has in the past always decoded the DATA portion of +The :mod:`!smtpd` module has in the past always decoded the DATA portion of email messages using the ``utf-8`` codec. This can now be controlled by the -new *decode_data* keyword to :class:`~smtpd.SMTPServer`. The default value is +new *decode_data* keyword to :class:`!SMTPServer`. The default value is ``True``, but this default is deprecated. Specify the *decode_data* keyword with an appropriate value to avoid the deprecation warning. @@ -2417,7 +2418,7 @@ Changes in the Python API (Contributed by Victor Stinner in :issue:`21205`.) * The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`, - :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been + :meth:`!HTMLParser.error`, and the :exc:`!HTMLParserError` exception have been removed. (Contributed by Ezio Melotti in :issue:`15114`.) The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c15d8be651fd17..68ab43462b743a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1472,6 +1472,10 @@ Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` were added. (Contributed by Christian Heimes in :issue:`28085`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) statistics ---------- @@ -1961,14 +1965,14 @@ Deprecated Python modules, functions and methods asynchat ~~~~~~~~ -The :mod:`asynchat` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asynchat` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) asyncore ~~~~~~~~ -The :mod:`asyncore` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asyncore` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) @@ -2063,6 +2067,15 @@ connected to and thus what Python interpreter will be used by the virtual environment. (Contributed by Brett Cannon in :issue:`25154`.) +xml +--- + +* As mitigation against DTD and external entity retrieval, the + :mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process + external entities by default. + (Contributed by Christian Heimes in :gh:`61441`.) + + Deprecated functions and types of the C API ------------------------------------------- @@ -2160,14 +2173,15 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the ``co_lnotab`` attribute of code objects changed to support +* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects + changed to support a negative line number delta. By default, Python does not emit bytecode with - a negative line number delta. Functions using ``frame.f_lineno``, + a negative line number delta. Functions using :attr:`frame.f_lineno`, ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. - Functions directly decoding ``co_lnotab`` should be updated to use a signed + Functions directly decoding :attr:`!co_lnotab` should be updated to use a signed 8-bit integer type for the line number delta, but this is only required to support applications using a negative line number delta. See - ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode + ``Objects/lnotab_notes.txt`` for the :attr:`!co_lnotab` format and how to decode it, and see the :pep:`511` for the rationale. * The functions in the :mod:`compileall` module now return booleans instead @@ -2188,7 +2202,7 @@ Changes in the Python API :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, - :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`smtpd`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`!smtpd`, :mod:`subprocess`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. @@ -2218,11 +2232,11 @@ Changes in the Python API an error (e.g. ``EBADF``) was reported by the underlying system call. (Contributed by Martin Panter in :issue:`26685`.) -* The *decode_data* argument for the :class:`smtpd.SMTPChannel` and - :class:`smtpd.SMTPServer` constructors is now ``False`` by default. +* The *decode_data* argument for the :class:`!smtpd.SMTPChannel` and + :class:`!smtpd.SMTPServer` constructors is now ``False`` by default. This means that the argument passed to - :meth:`~smtpd.SMTPServer.process_message` is now a bytes object by - default, and ``process_message()`` will be passed keyword arguments. + :meth:`!process_message` is now a bytes object by + default, and :meth:`!process_message` will be passed keyword arguments. Code that has already been updated in accordance with the deprecation warning generated by 3.5 will not be affected. @@ -2322,10 +2336,10 @@ Changes in the Python API * With the introduction of :exc:`ModuleNotFoundError`, import system consumers may start expecting import system replacements to raise that more specific exception when appropriate, rather than the less-specific :exc:`ImportError`. - To provide future compatibility with such consumers, implementors of + To provide future compatibility with such consumers, implementers of alternative import systems that completely replace :func:`__import__` will need to update their implementations to raise the new subclass when a module - can't be found at all. Implementors of compliant plugins to the default + can't be found at all. Implementers of compliant plugins to the default import system shouldn't need to make any changes, as the default import system will raise the new subclass when appropriate. @@ -2429,9 +2443,13 @@ The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. (Contributed by Victor Stinner in :issue:`31900`.) + Notable changes in Python 3.6.7 =============================== +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. See also :gh:`61441`. + In 3.6.7 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token when provided with input that does not have a trailing new line. This behavior now matches what the C tokenizer does internally. @@ -2459,3 +2477,19 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.6.14 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a7d5c3db6ddcb2..ae750cb9bba696 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -525,8 +525,8 @@ Other Language Changes * In order to better support dynamic creation of stack traces, :class:`types.TracebackType` can now be instantiated from Python code, and - the ``tb_next`` attribute on :ref:`tracebacks <traceback-objects>` is now - writable. + the :attr:`~traceback.tb_next` attribute on + :ref:`tracebacks <traceback-objects>` is now writable. (Contributed by Nathaniel J. Smith in :issue:`30579`.) * When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded @@ -669,8 +669,8 @@ include: * The new :func:`asyncio.current_task` function returns the currently running :class:`~asyncio.Task` instance, and the new :func:`asyncio.all_tasks` function returns a set of all existing ``Task`` instances in a given loop. - The :meth:`Task.current_task() <asyncio.Task.current_task>` and - :meth:`Task.all_tasks() <asyncio.Task.all_tasks>` methods have been deprecated. + The :meth:`!Task.current_task` and + :meth:`!Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) * The new *provisional* :class:`~asyncio.BufferedProtocol` class allows @@ -851,7 +851,7 @@ crypt The :mod:`!crypt` module now supports the Blowfish hashing method. (Contributed by Serhiy Storchaka in :issue:`31664`.) -The :func:`~!crypt.mksalt` function now allows specifying the number of rounds +The :func:`!mksalt` function now allows specifying the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) @@ -1380,6 +1380,10 @@ Supported protocols are indicated by several new flags, such as :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) +Added :attr:`ssl.SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :gh:`78851`.) string ------ @@ -1599,6 +1603,15 @@ at the interactive prompt. See :ref:`whatsnew37-pep565` for details. (Contributed by Nick Coghlan in :issue:`31975`.) +xml +--- + +As mitigation against DTD and external entity retrieval, the +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. +(Contributed by Christian Heimes in :gh:`61441`.) + + xml.etree --------- @@ -1891,7 +1904,7 @@ Other CPython Implementation Changes * Trace hooks may now opt out of receiving the ``line`` and opt into receiving the ``opcode`` events from the interpreter by setting the corresponding new - ``f_trace_lines`` and ``f_trace_opcodes`` attributes on the + :attr:`~frame.f_trace_lines` and :attr:`~frame.f_trace_opcodes` attributes on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) * Fixed some consistency problems with namespace package module attributes. @@ -1956,7 +1969,7 @@ asynchronous context manager must be used in order to acquire and release the synchronization resource. (Contributed by Andrew Svetlov in :issue:`32253`.) -The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) @@ -2004,15 +2017,15 @@ importlib --------- Methods -:meth:`MetaPathFinder.find_module() <!importlib.abc.MetaPathFinder.find_module>` +:meth:`!MetaPathFinder.find_module()` (replaced by :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) and -:meth:`PathEntryFinder.find_loader() <!importlib.abc.PathEntryFinder.find_loader>` +:meth:`!PathEntryFinder.find_loader()` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. -(Contributed by Matthias Bussonnier in :issue:`29576`) +(Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. @@ -2035,7 +2048,7 @@ The :mod:`macpath` is now deprecated and will be removed in Python 3.8. threading --------- -:mod:`dummy_threading` and :mod:`_dummy_thread` have been deprecated. It is +:mod:`!dummy_threading` and :mod:`!_dummy_thread` have been deprecated. It is no longer possible to build Python with threading disabled. Use :mod:`threading` instead. (Contributed by Antoine Pitrou in :issue:`31370`.) @@ -2144,9 +2157,9 @@ The following features and APIs have been removed from Python 3.7: * Removed support of the *exclude* argument in :meth:`tarfile.TarFile.add`. It was deprecated in Python 2.7 and 3.2. Use the *filter* argument instead. -* The ``splitunc()`` function in the :mod:`ntpath` module was deprecated in - Python 3.1, and has now been removed. Use the :func:`~os.path.splitdrive` - function instead. +* The :func:`!ntpath.splitunc` function was deprecated in + Python 3.1, and has now been removed. Use :func:`~os.path.splitdrive` + instead. * :func:`collections.namedtuple` no longer supports the *verbose* parameter or ``_source`` attribute which showed the generated source code for the @@ -2171,7 +2184,7 @@ The following features and APIs have been removed from Python 3.7: ``socket.socketpair`` on Python 3.5 and newer. * :mod:`asyncio` no longer exports the :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and + :mod:`!_overlapped` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. @@ -2304,9 +2317,9 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) -* :func:`~cgi.parse_multipart` now accepts the *encoding* and *errors* +* :func:`!cgi.parse_multipart` now accepts the *encoding* and *errors* arguments and returns the same results as - :class:`~FieldStorage`: for non-file fields, the value associated to a key + :class:`!FieldStorage`: for non-file fields, the value associated to a key is a list of strings, not bytes. (Contributed by Pierre Quentel in :issue:`29979`.) @@ -2571,3 +2584,33 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.7.11 +================================ + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates RFC 3986, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser :func:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.7.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index a2a67264deff66..1356f24547b424 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -123,7 +123,7 @@ There is a new function parameter syntax ``/`` to indicate that some function parameters must be specified positionally and cannot be used as keyword arguments. This is the same notation shown by ``help()`` for C functions annotated with Larry Hastings' -`Argument Clinic <devguide.python.org/development-tools/clinic/>`__ tool. +`Argument Clinic <https://devguide.python.org/development-tools/clinic/>`__ tool. In the following example, parameters *a* and *b* are positional-only, while *c* or *d* can be positional or keyword, and *e* or *f* are @@ -754,8 +754,8 @@ datetime -------- Added new alternate constructors :meth:`datetime.date.fromisocalendar` and -:meth:`datetime.datetime.fromisocalendar`, which construct :class:`date` and -:class:`datetime` objects respectively from ISO year, week number, and weekday; +:meth:`datetime.datetime.fromisocalendar`, which construct :class:`~datetime.date` and +:class:`~datetime.datetime` objects respectively from ISO year, week number, and weekday; these are the inverse of each class's ``isocalendar`` method. (Contributed by Paul Ganssle in :issue:`36004`.) @@ -1086,7 +1086,7 @@ pathlib contain characters unrepresentable at the OS level. (Contributed by Serhiy Storchaka in :issue:`33721`.) -Added :meth:`pathlib.Path.link_to()` which creates a hard link pointing +Added :meth:`!pathlib.Path.link_to()` which creates a hard link pointing to a path. (Contributed by Joannah Nanjekye in :issue:`26978`) Note that ``link_to`` was deprecated in 3.10 and removed in 3.12 in @@ -1623,8 +1623,8 @@ Build and C API Changes allocation or deallocation may need to be adjusted. (Contributed by Eddie Elizondo in :issue:`35810`.) -* The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create - code objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +* The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create + code objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) @@ -1653,7 +1653,7 @@ Deprecated deprecated and will be prohibited in Python 3.9. (Contributed by Elvis Pranskevichus in :issue:`34075`.) -* The :meth:`__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, +* The :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput` have been deprecated. @@ -2243,6 +2243,21 @@ details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) +Notable changes in Python 3.8.2 +=============================== + +Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`. +The argument types are now str and List[str] again. +(Contributed by Manuel Barkhau and Giampaolo Rodola in :gh:`83571`.) + +Notable changes in Python 3.8.3 +=============================== + +The constant values of future flags in the :mod:`__future__` module +are updated in order to prevent collision with compiler flags. Previously +``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. +(Contributed by Batuhan Taskaya in :gh:`83743`) + Notable changes in Python 3.8.8 =============================== @@ -2256,9 +2271,55 @@ functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) +Notable changes in Python 3.8.9 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.8.10 +================================ + +macOS 11.0 (Big Sur) and Apple Silicon Mac support +-------------------------------------------------- + +As of 3.8.10, Python now supports building and running on macOS 11 +(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture). +A new universal build variant, ``universal2``, is now available to natively +support both ``ARM64`` and ``Intel 64`` in one set of executables. +Note that support for "weaklinking", building binaries targeted for newer +versions of macOS that will also run correctly on older versions by +testing at runtime for missing features, is not included in this backport +from Python 3.9; to support a range of macOS versions, continue to target +for and build on the oldest version in the range. + +(Originally contributed by Ronald Oussoren and Lawrence D'Anna in :gh:`85272`, +with fixes by FX Coudert and Eli Rykoff, and backported to 3.8 by Maxime Bélanger +and Ned Deily) + +Notable changes in Python 3.8.10 +================================ + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :issue:`43882`) + + Notable changes in Python 3.8.12 ================================ +Changes in the Python API +------------------------- + Starting with Python 3.8.12 the :mod:`ipaddress` module no longer accepts any leading zeros in IPv4 address strings. Leading zeros are ambiguous and interpreted as octal notation by some libraries. For example the legacy @@ -2268,3 +2329,32 @@ any leading zeros. (Originally contributed by Christian Heimes in :issue:`36384`, and backported to 3.8 by Achraf Merzouki.) + +Notable security feature in 3.8.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.8.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index cb2482ee48d7fa..1bce90d248d631 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -81,13 +81,13 @@ Interpreter improvements: * a number of Python builtins (range, tuple, set, frozenset, list, dict) are now sped up using :pep:`590` vectorcall; * garbage collection does not block on resurrected objects; -* a number of Python modules (:mod:`_abc`, :mod:`!audioop`, :mod:`_bz2`, - :mod:`_codecs`, :mod:`_contextvars`, :mod:`!_crypt`, :mod:`_functools`, - :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, - :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined +* a number of Python modules (:mod:`!_abc`, :mod:`!audioop`, :mod:`!_bz2`, + :mod:`!_codecs`, :mod:`!_contextvars`, :mod:`!_crypt`, :mod:`!_functools`, + :mod:`!_json`, :mod:`!_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, + :mod:`time`, :mod:`!_weakref`) now use multiphase initialization as defined by PEP 489; * a number of standard library modules (:mod:`!audioop`, :mod:`ast`, :mod:`grp`, - :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, + :mod:`!_hashlib`, :mod:`pwd`, :mod:`!_posixsubprocess`, :mod:`random`, :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using the stable ABI defined by PEP 384. @@ -203,7 +203,7 @@ The :mod:`ast` module uses the new parser and produces the same AST as the old parser. In Python 3.10, the old parser will be deleted and so will all -functionality that depends on it (primarily the :mod:`parser` module, +functionality that depends on it (primarily the :mod:`!parser` module, which has long been deprecated). In Python 3.9 *only*, you can switch back to the LL(1) parser using a command line switch (``-X oldparser``) or an environment variable (``PYTHONOLDPARSER=1``). @@ -300,12 +300,9 @@ Example:: As a fall-back source of data for platforms that don't ship the IANA database, -the |tzdata|_ module was released as a first-party package -- distributed via +the :pypi:`tzdata` module was released as a first-party package -- distributed via PyPI and maintained by the CPython core team. -.. |tzdata| replace:: ``tzdata`` -.. _tzdata: https://pypi.org/project/tzdata/ - .. seealso:: :pep:`615` -- Support for the IANA Time Zone Database in the Standard Library @@ -355,7 +352,7 @@ that schedules a shutdown for the default executor that waits on the :func:`asyncio.run` has been updated to use the new :term:`coroutine`. (Contributed by Kyle Stanley in :issue:`34037`.) -Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher +Added :class:`!asyncio.PidfdChildWatcher`, a Linux-specific child watcher implementation that polls process file descriptors. (:issue:`38692`) Added a new :term:`coroutine` :func:`asyncio.to_thread`. It is mainly used for @@ -369,7 +366,7 @@ wait until the cancellation is complete also in the case when *timeout* is <= 0, like it does with positive timeouts. (Contributed by Elvis Pranskevichus in :issue:`32751`.) -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. (Contributed by Ido Michael in :issue:`37404`.) @@ -585,14 +582,14 @@ queue. nntplib ------- -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a :class:`ValueError` +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) os -- -Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`si_code`. +Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`!si_code`. (Contributed by Donghee Na in :issue:`38493`.) Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and @@ -864,7 +861,7 @@ Deprecated Python versions it will raise a :exc:`TypeError` for all floats. (Contributed by Serhiy Storchaka in :issue:`37315`.) -* The :mod:`parser` and :mod:`symbol` modules are deprecated and will be +* The :mod:`!parser` and :mod:`!symbol` modules are deprecated and will be removed in future versions of Python. For the majority of use cases, users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. @@ -892,20 +889,20 @@ Deprecated it for writing and silencing a warning. (Contributed by Serhiy Storchaka in :issue:`28286`.) -* Deprecated the ``split()`` method of :class:`_tkinter.TkappType` in +* Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and - predicable behavior. + predictable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) * The explicit passing of coroutine objects to :func:`asyncio.wait` has been deprecated and will be removed in version 3.11. (Contributed by Yury Selivanov and Kyle Stanley in :issue:`34790`.) -* binhex4 and hexbin4 standards are now deprecated. The :mod:`binhex` module +* binhex4 and hexbin4 standards are now deprecated. The :mod:`!binhex` module and the following :mod:`binascii` functions are now deprecated: - * :func:`~binascii.b2a_hqx`, :func:`~binascii.a2b_hqx` - * :func:`~binascii.rlecode_hqx`, :func:`~binascii.rledecode_hqx` + * :func:`!b2a_hqx`, :func:`!a2b_hqx` + * :func:`!rlecode_hqx`, :func:`!rledecode_hqx` (Contributed by Victor Stinner in :issue:`39353`.) @@ -931,7 +928,7 @@ Deprecated * Passing ``None`` as the first argument to the :func:`shlex.split` function has been deprecated. (Contributed by Zackery Spytz in :issue:`33262`.) -* :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without +* :func:`!smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) * The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. @@ -953,7 +950,7 @@ Deprecated Removed ======= -* The erroneous version at :data:`unittest.mock.__version__` has been removed. +* The erroneous version at :data:`!unittest.mock.__version__` has been removed. * :class:`!nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. These methods are deprecated since Python 3.3. Generally, these extensions @@ -990,7 +987,7 @@ Removed removed. They were deprecated since Python 3.7. (Contributed by Victor Stinner in :issue:`37320`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`!isAlive()` method of :class:`threading.Thread` has been removed. It was deprecated since Python 3.8. Use :meth:`~threading.Thread.is_alive()` instead. (Contributed by Donghee Na in :issue:`37804`.) @@ -1038,7 +1035,7 @@ Removed ``asyncio.Condition`` and ``asyncio.Semaphore``. (Contributed by Andrew Svetlov in :issue:`34793`.) -* The :func:`sys.getcounts` function, the ``-X showalloccount`` command line +* The :func:`!sys.getcounts` function, the ``-X showalloccount`` command line option and the ``show_alloc_count`` field of the C structure :c:type:`PyConfig` have been removed. They required a special Python build by defining ``COUNT_ALLOCS`` macro. @@ -1049,11 +1046,11 @@ Removed the ``__annotations__`` attribute instead. (Contributed by Serhiy Storchaka in :issue:`40182`.) -* The :meth:`symtable.SymbolTable.has_exec` method has been removed. It was +* The :meth:`!symtable.SymbolTable.has_exec` method has been removed. It was deprecated since 2006, and only returning ``False`` when it's called. (Contributed by Batuhan Taskaya in :issue:`40208`) -* The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +* The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` have been removed. They were deprecated since Python 3.7 and you can use :func:`asyncio.current_task` and :func:`asyncio.all_tasks` instead. (Contributed by Rémi Lapeyre in :issue:`40967`) @@ -1126,7 +1123,7 @@ Changes in the Python API ``logging.getLogger(__name__)`` in some top-level module called ``'root.py'``. (Contributed by Vinay Sajip in :issue:`37742`.) -* Division handling of :class:`~pathlib.PurePath` now returns ``NotImplemented`` +* Division handling of :class:`~pathlib.PurePath` now returns :data:`NotImplemented` instead of raising a :exc:`TypeError` when passed something other than an instance of ``str`` or :class:`~pathlib.PurePath`. This allows creating compatible classes that don't inherit from those mentioned types. @@ -1233,7 +1230,7 @@ Build Changes * The ``COUNT_ALLOCS`` special build macro has been removed. (Contributed by Victor Stinner in :issue:`39489`.) -* On non-Windows platforms, the :c:func:`setenv` and :c:func:`unsetenv` +* On non-Windows platforms, the :c:func:`!setenv` and :c:func:`!unsetenv` functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) @@ -1322,7 +1319,7 @@ New Features the garbage collector respectively. (Contributed by Pablo Galindo Salgado in :issue:`40241`.) -* Added :c:func:`_PyObject_FunctionStr` to get a user-friendly string +* Added :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. (Patch by Jeroen Demeyer in :issue:`37645`.) @@ -1364,7 +1361,7 @@ Porting to Python 3.9 and refers to a constant string. (Contributed by Serhiy Storchaka in :issue:`38650`.) -* The :c:type:`PyGC_Head` structure is now opaque. It is only defined in the +* The :c:type:`!PyGC_Head` structure is now opaque. It is only defined in the internal C API (``pycore_gc.h``). (Contributed by Victor Stinner in :issue:`40241`.) @@ -1387,12 +1384,12 @@ Porting to Python 3.9 * :c:func:`PyObject_IS_GC` macro was converted to a function. - * The :c:func:`PyObject_NEW` macro becomes an alias to the - :c:macro:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + * The :c:func:`!PyObject_NEW` macro becomes an alias to the + :c:macro:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:macro:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. - * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: + * :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` member. @@ -1562,3 +1559,54 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.9.3 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :gh:`87451`) + +Notable changes in Python 3.9.5 +=============================== + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :gh:`88048`) + +Notable security feature in 3.9.14 +================================== + +Converting between :class:`int` and :class:`str` in bases other than 2 +(binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) +now raises a :exc:`ValueError` if the number of digits in string form is +above a limit to avoid potential denial of service attacks due to the +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. +This limit can be configured or disabled by environment variable, command +line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion +length limitation <int_max_str_digits>` documentation. The default limit +is 4300 digits in string form. + +Notable changes in 3.9.17 +========================= + +tarfile +------- + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.12, use without the *filter* argument will show a + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index b9c19602653219..6ff722a1894585 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.14.rst 3.13.rst 3.12.rst 3.11.rst @@ -34,8 +35,8 @@ anyone wishing to stay up-to-date after a new release. 2.1.rst 2.0.rst -The "Changelog" is an HTML version of the `file built -<https://pypi.org/project/blurb>`_ from the contents of the +The "Changelog" is an HTML version of the :pypi:`file built<blurb>` +from the contents of the :source:`Misc/NEWS.d` directory tree, which contains *all* nontrivial changes to Python for the current version. diff --git a/Grammar/python.gram b/Grammar/python.gram index 8b479a716ac44d..b14e5dd096cdf4 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -78,6 +78,9 @@ _PyPegen_parse(Parser *p) # Fail if e can be parsed, without consuming any input. # ~ # Commit to the current alternative, even if it fails to parse. +# &&e +# Eager parse e. The parser will not backtrack and will immediately +# fail with SyntaxError if e cannot be parsed. # # STARTING RULES @@ -265,11 +268,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } - | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'async' 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -391,8 +394,8 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent - | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NULL, EXTRA)) } + | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { @@ -637,27 +640,24 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { +type_params[asdl_type_param_seq*]: + | invalid_type_params + | '[' t=type_param_seq ']' { CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } type_param[type_param_ty] (memo): - | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } - | '*' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with TypeVarTuple" - : "cannot use bound with TypeVarTuple") - } - | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } - | '**' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with ParamSpec" - : "cannot use bound with ParamSpec") - } - | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } + | invalid_type_param + | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } + | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } +type_param_default[expr_ty]: '=' e=expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } +type_param_starred_default[expr_ty]: '=' e=star_expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } # EXPRESSIONS # ----------- @@ -777,6 +777,7 @@ bitwise_and[expr_ty]: shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } + | invalid_arithmetic | sum # Arithmetic operators @@ -793,6 +794,7 @@ term[expr_ty]: | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } + | invalid_factor | factor factor[expr_ty] (memo): @@ -912,7 +914,7 @@ fstring_middle[expr_ty]: | fstring_replacement_field | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } fstring_replacement_field[expr_ty]: - | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { + | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) } | invalid_replacement_field fstring_conversion[ResultTokenWithMetadata*]: @@ -967,6 +969,7 @@ for_if_clause[comprehension_ty]: CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } + | invalid_for_if_clause | invalid_for_target listcomp[expr_ty]: @@ -1006,8 +1009,9 @@ kwargs[asdl_seq*]: | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: - | invalid_starred_expression + | invalid_starred_expression_unpacking | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } + | invalid_starred_expression kwarg_or_starred[KeywordOrStarred*]: | invalid_kwarg @@ -1128,8 +1132,8 @@ func_type_comment[Token*]: # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") } + | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { @@ -1161,6 +1165,18 @@ invalid_legacy_expression: _PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL} +invalid_type_param: + | '*' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with TypeVarTuple" + : "cannot use bound with TypeVarTuple") + } + | '**' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with ParamSpec" + : "cannot use bound with ParamSpec") + } + invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME @@ -1195,7 +1211,7 @@ invalid_assignment: | (star_targets '=')* a=star_expressions '=' { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } - | a=star_expressions augassign (yield_expr | star_expressions) { + | a=star_expressions augassign annotated_rhs { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "'%s' is an illegal expression for augmented assignment", @@ -1281,6 +1297,10 @@ invalid_with_item: | expression 'as' a=expression &(',' | ')' | ':') { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } +invalid_for_if_clause: + | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { + RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + invalid_for_target: | 'async'? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } @@ -1293,10 +1313,14 @@ invalid_group: invalid_import: | a='import' ','.dotted_name+ 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } + | 'import' token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } + | token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } @@ -1369,11 +1393,12 @@ invalid_for_stmt: | ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) } invalid_def_raw: - | ['async'] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { + | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } + | ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block invalid_class_def_raw: - | 'class' NAME ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | a='class' NAME ['(' [arguments] ')'] ':' NEWLINE !INDENT { + | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } invalid_double_starred_kvpairs: @@ -1385,26 +1410,40 @@ invalid_kvpair: RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } -invalid_starred_expression: +invalid_starred_expression_unpacking: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression: + | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } + invalid_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") } | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") } - | '{' !(yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} - | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') { + | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} + | '{' annotated_rhs !('=' | '!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') { + | '{' annotated_rhs '=' !('!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '='? invalid_conversion_character - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') { + | '{' annotated_rhs '='? invalid_conversion_character + | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' { + | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' { + | '{' annotated_rhs '='? ['!' NAME] !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") } invalid_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") } + +invalid_arithmetic: + | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } +invalid_factor: + | ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } + +invalid_type_params: + | '[' token=']' { + RAISE_SYNTAX_ERROR_STARTING_FROM( + token, + "Type parameter list cannot be empty")} diff --git a/Include/Python.h b/Include/Python.h index 7312cc87d5cc33..8fffa22df9da48 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -22,23 +22,38 @@ #include <math.h> // HUGE_VAL #include <stdarg.h> // va_list #include <wchar.h> // wchar_t -#ifdef HAVE_STDDEF_H -# include <stddef.h> // size_t -#endif #ifdef HAVE_SYS_TYPES_H # include <sys/types.h> // ssize_t #endif -// errno.h, stdio.h, stdlib.h and string.h headers are no longer used by -// Python, but kept for backward compatibility (avoid compiler warnings). -// They are no longer included by limited C API version 3.11 and newer. +// <errno.h>, <stdio.h>, <stdlib.h> and <string.h> headers are no longer used +// by Python, but kept for the backward compatibility of existing third party C +// extensions. They are not included by limited C API version 3.11 and newer. +// +// The <ctype.h> and <unistd.h> headers are not included by limited C API +// version 3.13 and newer. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # include <errno.h> // errno # include <stdio.h> // FILE* # include <stdlib.h> // getenv() # include <string.h> // memcpy() #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 +# include <ctype.h> // tolower() +# ifndef MS_WINDOWS +# include <unistd.h> // close() +# endif +#endif +// gh-111506: The free-threaded build is not compatible with the limited API +// or the stable ABI. +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +# error "The limited API is not currently supported in the free-threaded build" +#endif + +#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) +# include <intrin.h> // __readgsqword() +#endif // Include Python header files #include "pyport.h" @@ -49,7 +64,9 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" +#include "lock.h" #include "object.h" +#include "refcount.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" @@ -57,6 +74,7 @@ #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" +#include "pyerrors.h" #include "longobject.h" #include "cpython/longintrepr.h" #include "boolobject.h" @@ -72,6 +90,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" +#include "monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -91,8 +110,8 @@ #include "weakrefobject.h" #include "structseq.h" #include "cpython/picklebufobject.h" +#include "cpython/pytime.h" #include "codecs.h" -#include "pyerrors.h" #include "pythread.h" #include "cpython/context.h" #include "modsupport.h" @@ -106,12 +125,12 @@ #include "import.h" #include "abstract.h" #include "bltinmodule.h" +#include "critical_section.h" #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" #include "fileutils.h" #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" -#include "cpython/optimizer.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index bd12a54963c13f..f0e49c1afb8164 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -852,15 +852,15 @@ PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key); PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key); -/* On success, return a list or tuple of the keys in mapping object 'o'. +/* On success, return a list of the keys in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); -/* On success, return a list or tuple of the values in mapping object 'o'. +/* On success, return a list of the values in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o); -/* On success, return a list or tuple of the items in mapping object 'o', +/* On success, return a list of the items in mapping object 'o', where each item is a tuple containing a key-value pair. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); diff --git a/Include/boolobject.h b/Include/boolobject.h index 976fa35201d035..3037e61bbf6d0c 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -7,7 +7,7 @@ extern "C" { #endif -PyAPI_DATA(PyTypeObject) PyBool_Type; +// PyBool_Type is declared by object.h #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type) @@ -18,8 +18,13 @@ PyAPI_DATA(PyLongObject) _Py_FalseStruct; PyAPI_DATA(PyLongObject) _Py_TrueStruct; /* Use these macros */ -#define Py_False _PyObject_CAST(&_Py_FalseStruct) -#define Py_True _PyObject_CAST(&_Py_TrueStruct) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_False Py_GetConstantBorrowed(Py_CONSTANT_FALSE) +# define Py_True Py_GetConstantBorrowed(Py_CONSTANT_TRUE) +#else +# define Py_False _PyObject_CAST(&_Py_FalseStruct) +# define Py_True _PyObject_CAST(&_Py_TrueStruct) +#endif // Test if an object is the True singleton, the same as "x is True" in Python. PyAPI_FUNC(int) Py_IsTrue(PyObject *x); diff --git a/Include/ceval.h b/Include/ceval.h index 9885bdb7febc21..32ab38972e548f 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); @@ -38,7 +42,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); level exceeds "current recursion limit + 50". By construction, this protection can only be triggered when the "overflowed" flag is set. It means the cleanup code has itself gone into an infinite loop, or the - RecursionError has been mistakingly ignored. When this protection is + RecursionError has been mistakenly ignored. When this protection is triggered, the interpreter aborts with a Fatal Error. In addition, the "overflowed" flag is automatically reset when the @@ -107,6 +111,8 @@ PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(PyFrameObject *f, int exc); PyAPI_FUNC(PyThreadState *) PyEval_SaveThread(void); PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *); +Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void); + PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); @@ -127,6 +133,13 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 +/* Special methods used by LOAD_SPECIAL */ +#define SPECIAL___ENTER__ 0 +#define SPECIAL___EXIT__ 1 +#define SPECIAL___AENTER__ 2 +#define SPECIAL___AEXIT__ 3 +#define SPECIAL_MAX 3 + #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H # include "cpython/ceval.h" diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 1f495f19df280b..4e7b7a46703a6d 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -2,6 +2,28 @@ # error "this header file must not be included directly" #endif +/* === Object Protocol ================================================== */ + +/* Like PyObject_CallMethod(), but expect a _Py_Identifier* + as the method name. */ +PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( + PyObject *obj, + _Py_Identifier *name, + const char *format, ...); + +/* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) + format to a Python dictionary ("kwargs" dict). + + The type of kwnames keys is not checked. The final function getting + arguments is responsible to check if all keys are strings, for example using + PyArg_ParseTupleAndKeywords() or PyArg_ValidateKeywordArguments(). + + Duplicate keys are merged using the last value. If duplicate keys must raise + an exception, the caller is responsible to implement an explicit keys on + kwnames. */ +PyAPI_FUNC(PyObject*) _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); + + /* === Vectorcall protocol (PEP 590) ============================= */ // PyVectorcall_NARGS() is exported as a function for the stable ABI. @@ -16,6 +38,16 @@ _PyVectorcall_NARGS(size_t n) PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); +// Backwards compatibility aliases (PEP 590) for API that was provisional +// in Python 3.8 +#define _PyObject_Vectorcall PyObject_Vectorcall +#define _PyObject_VectorcallMethod PyObject_VectorcallMethod +#define _PyObject_FastCallDict PyObject_VectorcallDict +#define _PyVectorcall_Function PyVectorcall_Function +#define _PyObject_CallOneArg PyObject_CallOneArg +#define _PyObject_CallMethodNoArgs PyObject_CallMethodNoArgs +#define _PyObject_CallMethodOneArg PyObject_CallMethodOneArg + /* Same as PyObject_Vectorcall except that keyword arguments are passed as dict, which may be NULL if there are no keyword arguments. */ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( diff --git a/Include/cpython/code.h b/Include/cpython/code.h index cf715c55a2b3b8..58d93fcfc1066b 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -24,47 +24,6 @@ typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_GlobalMonitors; -/* Each instruction in a code object is a fixed-width value, - * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG - * opcode allows for larger values but the current limit is 3 uses - * of EXTENDED_ARG (see Python/compile.c), for a maximum - * 32-bit value. This aligns with the note in Python/compile.c - * (compiler_addop_i_line) indicating that the max oparg value is - * 2**32 - 1, rather than INT_MAX. - */ - -typedef union { - uint16_t cache; - struct { - uint8_t code; - uint8_t arg; - } op; -} _Py_CODEUNIT; - - -/* These macros only remain defined for compatibility. */ -#define _Py_OPCODE(word) ((word).op.code) -#define _Py_OPARG(word) ((word).op.arg) - -static inline _Py_CODEUNIT -_py_make_codeunit(uint8_t opcode, uint8_t oparg) -{ - // No designated initialisers because of C++ compat - _Py_CODEUNIT word; - word.op.code = opcode; - word.op.arg = oparg; - return word; -} - -static inline void -_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) -{ - word->op.code = opcode; -} - -#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) -#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) - typedef struct { PyObject *_co_code; @@ -73,7 +32,7 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; -/* Ancilliary data structure used for instrumentation. +/* Ancillary data structure used for instrumentation. Line instrumentation creates an array of these. One entry per code unit.*/ typedef struct { @@ -208,12 +167,14 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_FUTURE_GENERATOR_STOP 0x800000 #define CO_FUTURE_ANNOTATIONS 0x1000000 +#define CO_NO_MONITORING_EVENTS 0x2000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ #define PY_PARSER_REQUIRES_FUTURE_KEYWORD -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ +#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */ PyAPI_DATA(PyTypeObject) PyCode_Type; @@ -224,13 +185,14 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -static inline int PyCode_GetFirstFree(PyCodeObject *op) { +static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) { assert(PyCode_Check(op)); return op->co_nlocalsplus - op->co_nfreevars; } -#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) +Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { + return PyUnstable_Code_GetFirstFree(op); +} /* Unstable public interface */ PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 265f5397b45116..cfdb7080d45f2b 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -32,28 +32,8 @@ typedef struct { #define _PyCompilerFlags_INIT \ (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -/* source location information */ -typedef struct { - int lineno; - int end_lineno; - int col_offset; - int end_col_offset; -} _PyCompilerSrcLocation; - -#define SRC_LOCATION_FROM_AST(n) \ - (_PyCompilerSrcLocation){ \ - .lineno = (n)->lineno, \ - .end_lineno = (n)->end_lineno, \ - .col_offset = (n)->col_offset, \ - .end_col_offset = (n)->end_col_offset } - /* Future feature support */ -typedef struct { - int ff_features; /* flags set by future statements */ - _PyCompilerSrcLocation ff_location; /* location of last future statement */ -} PyFutureFeatures; - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" @@ -68,15 +48,3 @@ typedef struct { #define PY_INVALID_STACK_EFFECT INT_MAX PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); - -PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); - -PyAPI_FUNC(PyObject*) PyUnstable_GetUnaryIntrinsicName(int index); -PyAPI_FUNC(PyObject*) PyUnstable_GetBinaryIntrinsicName(int index); diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index b524ec42c24371..fbdc6a91fe895c 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,6 +7,16 @@ typedef struct { double imag; } Py_complex; +// Operations on complex numbers. +PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); +PyAPI_FUNC(double) _Py_c_abs(Py_complex); + + /* Complex object interface */ /* diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h new file mode 100644 index 00000000000000..35db3fb6a59ce6 --- /dev/null +++ b/Include/cpython/critical_section.h @@ -0,0 +1,134 @@ +#ifndef Py_CPYTHON_CRITICAL_SECTION_H +# error "this header file must not be included directly" +#endif + +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// Straightforward per-object locking could introduce deadlocks that were not +// present when running with the GIL. Threads may hold locks for multiple +// objects simultaneously because Python operations can nest. If threads were +// to acquire the same locks in different orders, they would deadlock. +// +// One way to avoid deadlocks is to allow threads to hold only the lock (or +// locks) for a single operation at a time (typically a single lock, but some +// operations involve two locks). When a thread begins a nested operation it +// could suspend the locks for any outer operation: before beginning the nested +// operation, the locks for the outer operation are released and when the +// nested operation completes, the locks for the outer operation are +// reacquired. +// +// To improve performance, this API uses a variation of the above scheme. +// Instead of immediately suspending locks any time a nested operation begins, +// locks are only suspended if the thread would block. This reduces the number +// of lock acquisitions and releases for nested operations, while still +// avoiding deadlocks. +// +// Additionally, the locks for any active operation are suspended around +// other potentially blocking operations, such as I/O. This is because the +// interaction between locks and blocking operations can lead to deadlocks in +// the same way as the interaction between multiple locks. +// +// Each thread's critical sections and their corresponding locks are tracked in +// a stack in `PyThreadState.critical_section`. When a thread calls +// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when +// waiting to acquire a lock, the thread suspends all of its active critical +// sections, temporarily releasing the associated locks. When the thread calls +// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) +// critical section by reacquiring the associated lock or locks. See +// `_PyCriticalSection_Resume()`. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +typedef struct PyCriticalSection PyCriticalSection; +typedef struct PyCriticalSection2 PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +#ifndef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { +# define Py_END_CRITICAL_SECTION() \ + } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { +# define Py_END_CRITICAL_SECTION2() \ + } +#else /* !Py_GIL_DISABLED */ + +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + PyMutex *_cs_mutex; +}; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + PyMutex *_cs_mutex2; +}; + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + +#endif diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index b05ca3ef453816..e2861c963266ea 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -17,6 +17,9 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ #ifdef Py_BUILD_CORE + /* Bits 0-7 are for dict watchers. + * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) + * The remaining bits (12-63) are the actual version tag. */ uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; @@ -32,20 +35,43 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +// PyDict_GetItemStringRef() can be used instead +Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); + /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; assert(PyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); +#else return mp->ma_used; +#endif } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); +PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); + +PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); +PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); +PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); /* Dictionary watchers */ diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..dbbfbb5105ba7a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index de2013323d2c72..598cd330bc9ca9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -8,7 +8,7 @@ extern "C" { #endif -#define COMMON_FIELDS(PREFIX) \ +#define _Py_COMMON_FIELDS(PREFIX) \ PyObject *PREFIX ## globals; \ PyObject *PREFIX ## builtins; \ PyObject *PREFIX ## name; \ @@ -19,7 +19,7 @@ extern "C" { PyObject *PREFIX ## closure; /* NULL or a tuple of cell objects */ typedef struct { - COMMON_FIELDS(fc_) + _Py_COMMON_FIELDS(fc_) } PyFrameConstructor; /* Function objects and code objects should not be confused with each other: @@ -35,12 +35,13 @@ typedef struct { typedef struct { PyObject_HEAD - COMMON_FIELDS(func_) + _Py_COMMON_FIELDS(func_) PyObject *func_doc; /* The __doc__ attribute, can be anything */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. @@ -60,6 +61,8 @@ typedef struct { */ } PyFunctionObject; +#undef _Py_COMMON_FIELDS + PyAPI_DATA(PyTypeObject) PyFunction_Type; #define PyFunction_Check(op) Py_IS_TYPE((op), &PyFunction_Type) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 49e46c277d75ae..f75884e597e2c2 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -9,29 +9,7 @@ extern "C" { /* --- Generators --------------------------------------------------------- */ -/* _PyGenObject_HEAD defines the initial segment of generator - and coroutine objects. */ -#define _PyGenObject_HEAD(prefix) \ - PyObject_HEAD \ - /* List of weak reference. */ \ - PyObject *prefix##_weakreflist; \ - /* Name of the generator. */ \ - PyObject *prefix##_name; \ - /* Qualified name of the generator. */ \ - PyObject *prefix##_qualname; \ - _PyErr_StackItem prefix##_exc_state; \ - PyObject *prefix##_origin_or_finalizer; \ - char prefix##_hooks_inited; \ - char prefix##_closed; \ - char prefix##_running_async; \ - /* The frame */ \ - int8_t prefix##_frame_state; \ - PyObject *prefix##_iframe[1]; \ - -typedef struct { - /* The gi_ prefix is intended to remind of generator-iterator. */ - _PyGenObject_HEAD(gi) -} PyGenObject; +typedef struct _PyGenObject PyGenObject; PyAPI_DATA(PyTypeObject) PyGen_Type; @@ -46,9 +24,7 @@ PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); /* --- PyCoroObject ------------------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(cr) -} PyCoroObject; +typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; @@ -59,9 +35,7 @@ PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, /* --- Asynchronous Generators -------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(ag) -} PyAsyncGenObject; +typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; @@ -73,7 +47,6 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, #define PyAsyncGenASend_CheckExact(op) Py_IS_TYPE((op), &_PyAsyncGenASend_Type) - #undef _PyGenObject_HEAD #ifdef __cplusplus diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 808c1056498b49..5da5ef9e5431b1 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -181,6 +181,9 @@ typedef struct PyConfig { int int_max_str_digits; int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; @@ -225,6 +228,12 @@ typedef struct PyConfig { // If non-zero, turns on statistics gathering. int _pystats; #endif + +#ifdef Py_DEBUG + // If not empty, import a non-__main__ module before site.py is executed. + // PYTHON_PRESITE=package.module or -X presite=package.module + wchar_t *run_presite; +#endif } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h deleted file mode 100644 index 4ab9ad5d315f80..00000000000000 --- a/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *); diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index 661610548733fd..49f5e8d6d1a0d6 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -29,7 +29,11 @@ typedef struct { static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { PyListObject *list = _PyList_CAST(op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size)); +#else return Py_SIZE(list); +#endif } #define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op)) @@ -39,8 +43,11 @@ static inline void PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyListObject *list = _PyList_CAST(op); assert(0 <= index); - assert(index < Py_SIZE(list)); + assert(index < list->allocated); list->ob_item[index] = value; } #define PyList_SET_ITEM(op, index, value) \ PyList_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable); +PyAPI_FUNC(int) PyList_Clear(PyObject *self); diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h new file mode 100644 index 00000000000000..8ee03e82f74dfd --- /dev/null +++ b/Include/cpython/lock.h @@ -0,0 +1,63 @@ +#ifndef Py_CPYTHON_LOCK_H +# error "this header file must not be included directly" +#endif + +#define _Py_UNLOCKED 0 +#define _Py_LOCKED 1 + +// A mutex that occupies one byte. The lock can be zero initialized to +// represent the unlocked state. +// +// Typical initialization: +// PyMutex m = (PyMutex){0}; +// +// Or initialize as global variables: +// static PyMutex m; +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); +// +// The contents of the PyMutex are not part of the public API, but are +// described to aid in understanding the implementation and debugging. Only +// the two least significant bits are used. The remaining bits are always zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +typedef struct PyMutex { + uint8_t _bits; // (private) +} PyMutex; + +// exported function for locking the mutex +PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); + +// exported function for unlocking the mutex +PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +_PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { + PyMutex_Lock(m); + } +} +#define PyMutex_Lock _PyMutex_Lock + +// Unlocks the mutex. +static inline void +_PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_UNLOCKED)) { + PyMutex_Unlock(m); + } +} +#define PyMutex_Unlock _PyMutex_Unlock diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index fb82f83dc50e42..d841c043f37fc4 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { @@ -89,6 +100,16 @@ struct _longobject { _PyLongValue long_value; }; +PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); + +// Return a copy of src. +PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); + +PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( + int negative, + Py_ssize_t digit_count, + digit *digits); + /* Inline some internals for speed. These should be in pycore_long.h * if user code didn't need them inlined. */ @@ -98,19 +119,20 @@ struct _longobject { static inline int -_PyLong_IsCompact(const PyLongObject* op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); +_PyLong_IsCompact(PyLongObject* op) { + assert(PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } #define PyUnstable_Long_IsCompact _PyLong_IsCompact static inline Py_ssize_t -_PyLong_CompactValue(const PyLongObject *op) +_PyLong_CompactValue(PyLongObject *op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + Py_ssize_t sign; + assert(PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 57834173490c99..e7e0c3d9764f20 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -4,6 +4,117 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); +#define Py_ASNATIVEBYTES_DEFAULTS -1 +#define Py_ASNATIVEBYTES_BIG_ENDIAN 0 +#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1 +#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 +#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 +#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 + +/* PyLong_AsNativeBytes: Copy the integer value to a native variable. + buffer points to the first byte of the variable. + n_bytes is the number of bytes available in the buffer. Pass 0 to request + the required size for the value. + flags is a bitfield of the following flags: + * 1 - little endian + * 2 - native endian + * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) + * 8 - raise an exception for negative inputs + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. + Big endian mode will write the most significant byte into the address + directly referenced by buffer; little endian will write the least significant + byte into that address. + + If an exception is raised, returns a negative value. + Otherwise, returns the number of bytes that are required to store the value. + To check that the full value is represented, ensure that the return value is + equal or less than n_bytes. + All n_bytes are guaranteed to be written (unless an exception occurs), and + so ignoring a positive return value is the equivalent of a downcast in C. + In cases where the full value could not be represented, the returned value + may be larger than necessary - this function is not an accurate way to + calculate the bit length of an integer object. + */ +PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, + Py_ssize_t n_bytes, int flags); + +/* PyLong_FromNativeBytes: Create an int value from a native integer + n_bytes is the number of bytes to read from the buffer. Passing 0 will + always produce the zero int. + PyLong_FromUnsignedNativeBytes always produces a non-negative int. + flags is the same as for PyLong_AsNativeBytes, but only supports selecting + the endianness or forcing an unsigned buffer. + + Returns the int object, or NULL with an exception set. */ +PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, + int flags); +PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, + size_t n_bytes, int flags); + PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); +/* PyLong_GetSign. Get the sign of an integer object: + 0, -1 or +1 for zero, negative or positive integer, respectively. + + - On success, set '*sign' to the integer sign, and return 0. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_GetSign(PyObject *v, int *sign); + +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + (size_t)-1 is returned and OverflowError set if the true result doesn't + fit in a size_t. +*/ +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + +/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in + base 256, and return a Python int with the same numeric value. + If n is 0, the integer is 0. Else: + If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; + else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the + LSB. + If is_signed is 0/false, view the bytes as a non-negative integer. + If is_signed is 1/true, view the bytes as a 2's-complement integer, + non-negative if bit 0x80 of the MSB is clear, negative if set. + Error returns: + + Return NULL with the appropriate exception set if there's not + enough memory to create the Python int. +*/ +PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long + v to a base-256 integer, stored in array bytes. Normally return 0, + return -1 on error. + If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at + bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and + the LSB at bytes[n-1]. + If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes + are filled and there's nothing special about bit 0x80 of the MSB. + If is_signed is 1/true, bytes is filled with the 2's-complement + representation of v's value. Bit 0x80 of the MSB is the sign bit. + Error returns (-1): + + is_signed is 0 and v < 0. TypeError is set in this case, and bytes + isn't altered. + + n isn't big enough to hold the full mathematical value of v. For + example, if is_signed is 0 and there are more digits in the v than + fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of + being large enough to hold a sign bit. OverflowError is set in this + case, but bytes holds the least-significant n bytes of the true value. +*/ +PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, + unsigned char* bytes, size_t n, + int little_endian, int is_signed, int with_exceptions); + +/* For use by the gcd function in mathmodule.c */ +PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index cfc2c2cdb5a7f4..d3b88f58c82ca3 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -2,72 +2,25 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyArg_UnpackStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *name, - Py_ssize_t min, - Py_ssize_t max, - ...); - -PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); -PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); -#define _PyArg_NoKeywords(funcname, kwargs) \ - ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) -#define _PyArg_NoPositional(funcname, args) \ - ((args) == NULL || _PyArg_NoPositional((funcname), (args))) - -#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) - -PyAPI_FUNC(void) _PyArg_BadArgument(const char *, const char *, const char *, PyObject *); -PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, - Py_ssize_t, Py_ssize_t); -#define _PyArg_CheckPositional(funcname, nargs, min, max) \ - ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ - || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; typedef struct _PyArg_Parser { - int initialized; const char *format; const char * const *keywords; const char *fname; const char *custom_msg; - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ struct _PyArg_Parser *next; } _PyArg_Parser; PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); -PyAPI_FUNC(int) _PyArg_ParseStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *format, - ...); -PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames, - struct _PyArg_Parser *, - ...); -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - PyObject **buf); - -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf); - -#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ - (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ - (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ - _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ - (minpos), (maxpos), (minkw), (buf))) diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h new file mode 100644 index 00000000000000..797ba51246b1c6 --- /dev/null +++ b/Include/cpython/monitoring.h @@ -0,0 +1,250 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ede394d9673d7e..90cd7b54b34161 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -4,6 +4,7 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); +PyAPI_FUNC(void) _Py_ResurrectReference(PyObject *op); #ifdef Py_REF_DEBUG /* These are useful as debugging aids when chasing down refleaks. */ @@ -14,6 +15,48 @@ PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GetRefTotal(PyInterpreterState *); #endif +/********************* String Literals ****************************************/ +/* This structure helps managing static strings. The basic usage goes like this: + Instead of doing + + r = PyObject_CallMethod(o, "foo", "args", ...); + + do + + _Py_IDENTIFIER(foo); + ... + r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); + + PyId_foo is a static variable, either on block level or file level. On first + usage, the string "foo" is interned, and the structures are linked. On interpreter + shutdown, all strings are released. + + Alternatively, _Py_static_string allows choosing the variable name. + _PyUnicode_FromId returns a borrowed reference to the interned string. + _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. +*/ +typedef struct _Py_Identifier { + const char* string; + // Index in PyInterpreterState.unicode.ids.array. It is process-wide + // unique and must be initialized to -1. + Py_ssize_t index; + // Hidden PyMutex struct for non free-threaded build. + struct { + uint8_t v; + } mutex; +} _Py_Identifier; + +#ifndef Py_BUILD_CORE +// For now we are keeping _Py_IDENTIFIER for continued use +// in non-builtin extensions (and naughty PyPI modules). + +#define _Py_static_string_init(value) { .string = (value), .index = -1 } +#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) +#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) + +#endif /* !Py_BUILD_CORE */ + + typedef struct { /* Number implementations must check *both* arguments for proper type and implement the necessary conversions @@ -186,6 +229,7 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + uint16_t tp_versions_used; }; /* This struct is used by the specializer @@ -231,17 +275,21 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); + PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); + /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ PyAPI_FUNC(PyObject *) @@ -267,7 +315,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); * triggered as a side-effect of `dst` getting torn down no longer believes * `dst` points to a valid object. * - * Temporary variables are used to only evalutate macro arguments once and so + * Temporary variables are used to only evaluate macro arguments once and so * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for * implementation details about type punning. @@ -403,8 +451,8 @@ without deallocating anything (and so unbounded call-stack depth is avoided). When the call stack finishes unwinding again, code generated by the END macro notices this, and calls another routine to deallocate all the objects that may have been added to the list of deferred deallocations. In effect, a -chain of N deallocations is broken into (N-1)/(_PyTrash_UNWIND_LEVEL-1) pieces, -with the call stack never exceeding a depth of _PyTrash_UNWIND_LEVEL. +chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces, +with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM. Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base class, we need to ensure that the trashcan is only triggered on the tp_dealloc @@ -416,35 +464,39 @@ passed as second argument to Py_TRASHCAN_BEGIN(). /* Python 3.9 private API, invoked by the macros below. */ PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate); + +PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); +PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); + + /* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */ -PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); -#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \ - do { \ - PyThreadState *_tstate = NULL; \ - /* If "cond" is false, then _tstate remains NULL and the deallocator \ - * is run normally without involving the trashcan */ \ - if (cond) { \ - _tstate = PyThreadState_GetUnchecked(); \ - if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ - break; \ - } \ - } - /* The body of the deallocator is here. */ -#define Py_TRASHCAN_END \ - if (_tstate) { \ - _PyTrash_end(_tstate); \ - } \ - } while (0); +/* To avoid raising recursion errors during dealloc trigger trashcan before we reach + * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until + * we have headroom above the trigger limit */ +#define Py_TRASHCAN_HEADROOM 50 #define Py_TRASHCAN_BEGIN(op, dealloc) \ - Py_TRASHCAN_BEGIN_CONDITION((op), \ - _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc))) +do { \ + PyThreadState *tstate = PyThreadState_Get(); \ + if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \ + _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \ + break; \ + } \ + tstate->c_recursion_remaining--; + /* The body of the deallocator is here. */ +#define Py_TRASHCAN_END \ + tstate->c_recursion_remaining++; \ + if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \ + _PyTrash_thread_destroy_chain(tstate); \ + } \ +} while (0); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 @@ -461,3 +513,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 58a30aeea6ac64..e0c2ce286f13ce 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -85,3 +85,20 @@ PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t); + + +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h deleted file mode 100644 index 47536108a9665e..00000000000000 --- a/Include/cpython/optimizer.h +++ /dev/null @@ -1,58 +0,0 @@ - -#ifndef Py_LIMITED_API -#ifndef Py_OPTIMIZER_H -#define Py_OPTIMIZER_H -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - uint8_t opcode; - uint8_t oparg; -} _PyVMData; - -typedef struct _PyExecutorObject { - PyObject_VAR_HEAD - /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ - struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); - _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - /* Data needed by the executor goes here, but is opaque to the VM */ -} _PyExecutorObject; - -typedef struct _PyOptimizerObject _PyOptimizerObject; - -/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); - -typedef struct _PyOptimizerObject { - PyObject_HEAD - optimize_func optimize; - uint16_t resume_threshold; - uint16_t backedge_threshold; - /* Data needed by the optimizer goes here, but is opaque to the VM */ -} _PyOptimizerObject; - -PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); - -PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); - -PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); - -PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); - -int -_PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer); - -extern _PyOptimizerObject _PyOptimizer_Default; - -/* For testing */ -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); - -#define OPTIMIZER_BITS_IN_COUNTER 4 - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPTIMIZER_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/pthread_stubs.h b/Include/cpython/pthread_stubs.h index 5246968ea05476..e542eaa5bff0cf 100644 --- a/Include/cpython/pthread_stubs.h +++ b/Include/cpython/pthread_stubs.h @@ -83,6 +83,7 @@ PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread, void *(*start_routine)(void *), void *restrict arg); PyAPI_FUNC(int) pthread_detach(pthread_t thread); +PyAPI_FUNC(int) pthread_join(pthread_t thread, void** value_ptr); PyAPI_FUNC(pthread_t) pthread_self(void); PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__)); PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr); diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 7a783058c173aa..4ecef4f56edf42 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -360,6 +360,8 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); static inline void * _Py_atomic_load_ptr_relaxed(const void *obj); +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj); // --- _Py_atomic_store ------------------------------------------------------ // Atomically performs `*obj = value` (sequential consistency) @@ -452,6 +454,10 @@ _Py_atomic_store_ptr_relaxed(void *obj, void *value); static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -459,10 +465,42 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); static inline void * _Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + // Stores `*obj = value` (release operation) static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); + +static inline void +_Py_atomic_store_int_release(int *obj, int value); + +static inline int +_Py_atomic_load_int_acquire(const int *obj); + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + + + // --- _Py_atomic_fence ------------------------------------------------------ @@ -472,6 +510,9 @@ _Py_atomic_store_ptr_release(void *obj, void *value); // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence static inline void _Py_atomic_fence_seq_cst(void); +// Acquire fence +static inline void _Py_atomic_fence_acquire(void); + // Release fence static inline void _Py_atomic_fence_release(void); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index f1a38c7b52871a..ef09954d53ac1d 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -297,7 +297,7 @@ _Py_atomic_load_ssize(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); } // --- _Py_atomic_load_relaxed ----------------------------------------------- @@ -356,7 +356,11 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr_relaxed(const void *obj) -{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); } + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } // --- _Py_atomic_store ------------------------------------------------------ @@ -476,17 +480,61 @@ static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * _Py_atomic_load_ptr_acquire(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } // --- _Py_atomic_fence ------------------------------------------------------ @@ -494,6 +542,10 @@ static inline void _Py_atomic_fence_seq_cst(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); } + static inline void +_Py_atomic_fence_acquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + static inline void _Py_atomic_fence_release(void) { __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 287ed43b5714cd..84da21bdcbff4f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -712,6 +712,12 @@ _Py_atomic_load_ptr_relaxed(const void *obj) return *(void * volatile *)obj; } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -886,6 +892,14 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) *(volatile Py_ssize_t *)obj = value; } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * @@ -900,6 +914,18 @@ _Py_atomic_load_ptr_acquire(const void *obj) #endif } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -912,6 +938,119 @@ _Py_atomic_store_ptr_release(void *obj, void *value) #endif } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_int_release" +#endif +} + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_int_acquire" +#endif +} + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32_acquire" +#endif +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} // --- _Py_atomic_fence ------------------------------------------------------ @@ -927,6 +1066,18 @@ _Py_atomic_fence_seq_cst(void) #else # error "no implementation of _Py_atomic_fence_seq_cst" #endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif } static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index bf74a90887c634..7c71e94c68f8e6 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -619,6 +619,14 @@ _Py_atomic_load_ptr_relaxed(const void *obj) memory_order_relaxed); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -835,6 +843,15 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) memory_order_relaxed); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -846,6 +863,14 @@ _Py_atomic_load_ptr_acquire(const void *obj) memory_order_acquire); } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -854,6 +879,78 @@ _Py_atomic_store_ptr_release(void *obj, void *value) memory_order_release); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_release); +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_acquire); +} + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); +} + // --- _Py_atomic_fence ------------------------------------------------------ @@ -864,6 +961,13 @@ _Py_atomic_fence_seq_cst(void) atomic_thread_fence(memory_order_seq_cst); } + static inline void +_Py_atomic_fence_acquire(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_acquire); +} + static inline void _Py_atomic_fence_release(void) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index da96eec4b35aab..b36b4681f5dddb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,6 +88,10 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif +/* Context manipulation (PEP 3134) */ + +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + /* In exceptions.c */ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( @@ -96,7 +100,7 @@ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( /* In signalmodule.c */ -int PySignal_SetWakeupFd(int fd); +PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); /* Support for adding program text to SyntaxErrors */ @@ -120,4 +124,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( const char *func, const char *message); +PyAPI_FUNC(void) PyErr_FormatUnraisable(const char *, ...); + +PyAPI_DATA(PyObject *) PyExc_PythonFinalizationError; + #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f69..51529763923ec3 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); @@ -26,7 +28,7 @@ struct _PyInterpreterFrame; * Does not raise an exception. */ PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); -/* Returns a byte ofsset into the last executed instruction. +/* Returns a byte offset into the last executed instruction. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h new file mode 100644 index 00000000000000..825c034a8d8474 --- /dev/null +++ b/Include/cpython/pyhash.h @@ -0,0 +1,47 @@ +#ifndef Py_CPYTHON_HASH_H +# error "this header file must not be included directly" +#endif + +/* Prime multiplier used in string and various other hashes. */ +#define PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +# define PyHASH_BITS 61 +#else +# define PyHASH_BITS 31 +#endif + +#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_INF 314159 +#define PyHASH_IMAG PyHASH_MULTIPLIER + +/* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_MULTIPLIER PyHASH_MULTIPLIER +#define _PyHASH_BITS PyHASH_BITS +#define _PyHASH_MODULUS PyHASH_MODULUS +#define _PyHASH_INF PyHASH_INF +#define _PyHASH_IMAG PyHASH_IMAG + +/* Helpers for hash functions */ +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); + +// Kept for backward compatibility +#define _Py_HashPointer Py_HashPointer + + +/* hash function definition */ +typedef struct { + Py_hash_t (*const hash)(const void *, Py_ssize_t); + const char *name; + const int hash_bits; + const int seed_bits; +} PyHash_FuncDef; + +PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); + +PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); +PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index d425a233f71000..e46dfe59ec4630 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -63,6 +63,15 @@ typedef struct { .gil = PyInterpreterConfig_OWN_GIL, \ } +// gh-117649: The free-threaded build does not currently support single-phase +// init extensions in subinterpreters. For now, we ensure that +// `check_multi_interp_extensions` is always `1`, even in the legacy config. +#ifdef Py_GIL_DISABLED +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 1 +#else +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 0 +#endif + #define _PyInterpreterConfig_LEGACY_INIT \ { \ .use_main_obmalloc = 1, \ @@ -70,7 +79,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ - .check_multi_interp_extensions = 0, \ + .check_multi_interp_extensions = _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS, \ .gil = PyInterpreterConfig_SHARED_GIL, \ } diff --git a/Include/cpython/pymem.h b/Include/cpython/pymem.h index b75f1c4d2425dd..76b3221f7b9f39 100644 --- a/Include/cpython/pymem.h +++ b/Include/cpython/pymem.h @@ -2,12 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); -PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyMem_RawFree(void *ptr); - - typedef enum { /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ PYMEM_DOMAIN_RAW, @@ -29,6 +23,10 @@ typedef enum { PYMEM_ALLOCATOR_PYMALLOC = 5, PYMEM_ALLOCATOR_PYMALLOC_DEBUG = 6, #endif +#ifdef WITH_MIMALLOC + PYMEM_ALLOCATOR_MIMALLOC = 7, + PYMEM_ALLOCATOR_MIMALLOC_DEBUG = 8, +#endif } PyMemAllocatorName; diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 40102f8855090e..bb2af78a376d75 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -56,11 +56,6 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; -struct _py_trashcan { - int delete_nesting; - PyObject *delete_later; -}; - struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -68,6 +63,11 @@ struct _ts { PyThreadState *next; PyInterpreterState *interp; + /* The global instrumentation version in high bits, plus flags indicating + when to break out of the interpreter loop in lower bits. See details in + pycore_ceval.h. */ + uintptr_t eval_breaker; + struct { /* Has been initialized to a safe state. @@ -83,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -90,19 +92,20 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; #ifdef Py_BUILD_CORE # define _PyThreadState_WHENCE_NOTSET -1 # define _PyThreadState_WHENCE_UNKNOWN 0 -# define _PyThreadState_WHENCE_INTERP 1 -# define _PyThreadState_WHENCE_THREADING 2 -# define _PyThreadState_WHENCE_GILSTATE 3 -# define _PyThreadState_WHENCE_EXEC 4 +# define _PyThreadState_WHENCE_INIT 1 +# define _PyThreadState_WHENCE_FINI 2 +# define _PyThreadState_WHENCE_THREADING 3 +# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_EXEC 5 #endif int _whence; - /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC). + /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). See Include/internal/pycore_pystate.h for more details. */ int state; @@ -147,33 +150,14 @@ struct _ts { */ unsigned long native_thread_id; - struct _py_trashcan trash; - - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. + PyObject *delete_later; + + /* Tagged pointer to top-most critical section, or zero if there is no + * active critical section. Critical sections are only used in + * `--disable-gil` builds (i.e., when Py_GIL_DISABLED is defined to 1). In the + * default build, this field is always zero. */ - void (*on_delete)(void *); - void *on_delete_data; + uintptr_t critical_section; int coroutine_origin_tracking_depth; @@ -205,16 +189,33 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; + PyObject *previous_executor; + + uint64_t dict_global_version; }; -#ifdef __wasi__ - // WASI has limited call stack. Python's recursion limit depends on code - // layout, optimization, and WASI runtime. Wasmtime can handle about 700 - // recursions, sometimes less. 500 is a more conservative limit. +#ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. # define Py_C_RECURSION_LIMIT 500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) && defined(_M_ARM64) +# define Py_C_RECURSION_LIMIT 1000 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 3000 +#elif defined(__ANDROID__) + // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21 + // crashed in test_compiler_recursion_limit. +# define Py_C_RECURSION_LIMIT 3000 +#elif defined(_Py_ADDRESS_SANITIZER) +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 5000 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 1500 +# define Py_C_RECURSION_LIMIT 10000 #endif @@ -224,6 +225,9 @@ struct _ts { * if it is NULL. */ PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); +// Alias kept for backward compatibility +#define _PyThreadState_UncheckedGet PyThreadState_GetUnchecked + // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); @@ -240,6 +244,11 @@ PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); +/* The implementation of sys._current_frames() Returns a dict mapping + thread id to that thread's current frame. +*/ +PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); @@ -258,80 +267,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interp; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 4988caa803723d..38480a4f6cd78f 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -19,6 +19,8 @@ // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. +#include "pycore_uop_ids.h" + #ifndef Py_CPYTHON_PYSTATS_H # error "this header file must not be included directly" #endif @@ -75,12 +77,11 @@ typedef struct _object_stats { uint64_t frees; uint64_t to_freelist; uint64_t from_freelist; - uint64_t new_values; + uint64_t inline_values; uint64_t dict_materialized_on_request; uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; @@ -98,6 +99,8 @@ typedef struct _gc_stats { typedef struct _uop_stats { uint64_t execution_count; + uint64_t miss; + uint64_t pair_count[MAX_UOP_ID + 1]; } UOpStats; #define _Py_UOP_HIST_SIZE 32 @@ -113,18 +116,43 @@ typedef struct _optimization_stats { uint64_t trace_too_short; uint64_t inner_loop; uint64_t recursive_call; - UOpStats opcode[512]; + uint64_t low_confidence; + uint64_t executors_invalidated; + UOpStats opcode[MAX_UOP_ID+1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimizer_attempts; + uint64_t optimizer_successes; + uint64_t optimizer_failure_reason_no_memory; + uint64_t remove_globals_builtins_changed; + uint64_t remove_globals_incorrect_keys; + uint64_t error_in_opcode[MAX_UOP_ID+1]; } OptimizationStats; +typedef struct _rare_event_stats { + /* Setting an object's class, obj.__class__ = ... */ + uint64_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint64_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint64_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint64_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint64_t func_modification; + /* Modifying a dict that is being watched */ + uint64_t watched_dict_modification; + uint64_t watched_globals_modification; +} RareEventStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; + RareEventStats rare_event_stats; GCStats *gc_stats; } PyStats; diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h new file mode 100644 index 00000000000000..5c68110aeedb86 --- /dev/null +++ b/Include/cpython/pytime.h @@ -0,0 +1,27 @@ +// PyTime_t C API: see Doc/c-api/time.rst for the documentation. + +#ifndef Py_LIMITED_API +#ifndef Py_PYTIME_H +#define Py_PYTIME_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef int64_t PyTime_t; +#define PyTime_MIN INT64_MIN +#define PyTime_MAX INT64_MAX + +PyAPI_FUNC(double) PyTime_AsSecondsDouble(PyTime_t t); +PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); +PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); + +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); + +#ifdef __cplusplus +} +#endif +#endif /* Py_PYTIME_H */ +#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 1778c778a05324..89565cb29212fc 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -62,6 +62,10 @@ typedef struct { (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PySet_CAST(so)->used)); +#else return _PySet_CAST(so)->used; +#endif } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index 36c4f89432067b..a3ac07f538a94f 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -4,12 +4,19 @@ typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *format, - ...); PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); -PyAPI_FUNC(int) PySys_AuditTuple( - const char *event, - PyObject *args); +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 859ab7178e920a..91799137101280 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -9,6 +9,7 @@ Py_DEPRECATED(3.13) typedef wchar_t PY_UNICODE_TYPE; Py_DEPRECATED(3.13) typedef wchar_t Py_UNICODE; + /* --- Internal Unicode Operations ---------------------------------------- */ // Static inline functions to work with surrogates @@ -43,6 +44,7 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { return (0xDC00 + (ch & 0x3FF)); } + /* --- Unicode Type ------------------------------------------------------- */ /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject @@ -375,6 +377,7 @@ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) #define PyUnicode_MAX_CHAR_VALUE(op) \ PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) + /* === Public API ========================================================= */ /* With PEP 393, this is the recommended way to allocate a new unicode object. @@ -440,6 +443,170 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); + +/* --- Public PyUnicodeWriter API ----------------------------------------- */ + +typedef struct PyUnicodeWriter PyUnicodeWriter; + +PyAPI_FUNC(PyUnicodeWriter*) PyUnicodeWriter_Create(Py_ssize_t length); +PyAPI_FUNC(void) PyUnicodeWriter_Discard(PyUnicodeWriter *writer); +PyAPI_FUNC(PyObject*) PyUnicodeWriter_Finish(PyUnicodeWriter *writer); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteChar( + PyUnicodeWriter *writer, + Py_UCS4 ch); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( + PyUnicodeWriter *writer, + const wchar_t *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( + PyUnicodeWriter *writer, + Py_UCS4 *str, + Py_ssize_t size); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteRepr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteSubstring( + PyUnicodeWriter *writer, + PyObject *str, + Py_ssize_t start, + Py_ssize_t end); +PyAPI_FUNC(int) PyUnicodeWriter_Format( + PyUnicodeWriter *writer, + const char *format, + ...); +PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful( + PyUnicodeWriter *writer, + const char *string, /* UTF-8 encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed */ + + +/* --- Private _PyUnicodeWriter API --------------------------------------- */ + +typedef struct { + PyObject *buffer; + void *data; + int kind; + Py_UCS4 maxchar; + Py_ssize_t size; + Py_ssize_t pos; + + /* minimum number of allocated characters (default: 0) */ + Py_ssize_t min_length; + + /* minimum character (default: 127, ASCII) */ + Py_UCS4 min_char; + + /* If non-zero, overallocate the buffer (default: 0). */ + unsigned char overallocate; + + /* If readonly is 1, buffer is a shared string (cannot be modified) + and size is set to 0. */ + unsigned char readonly; +} _PyUnicodeWriter; + +// Initialize a Unicode writer. +// +// By default, the minimum buffer size is 0 character and overallocation is +// disabled. Set min_length, min_char and overallocate attributes to control +// the allocation of the buffer. +PyAPI_FUNC(void) +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); + +/* Prepare the buffer to write 'length' characters + with the specified maximum character. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ + (((MAXCHAR) <= (WRITER)->maxchar \ + && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ + ? 0 \ + : (((LENGTH) == 0) \ + ? 0 \ + : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) + +/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro + instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar); + +/* Prepare the buffer to have at least the kind KIND. + For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will + support characters in range U+000-U+FFFF. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ + ((KIND) <= (WRITER)->kind \ + ? 0 \ + : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) + +/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() + macro instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + int kind); + +/* Append a Unicode character. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, + Py_UCS4 ch + ); + +/* Append a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, + PyObject *str /* Unicode string */ + ); + +/* Append a substring of a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, + PyObject *str, /* Unicode string */ + Py_ssize_t start, + Py_ssize_t end + ); + +/* Append an ASCII-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, + const char *str, /* ASCII-encoded byte string */ + Py_ssize_t len /* number of bytes, or -1 if unknown */ + ); + +/* Append a latin1-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, + const char *str, /* latin1-encoded byte string */ + Py_ssize_t len /* length in bytes */ + ); + +/* Get the value of the writer as a Unicode string. Clear the + buffer of the writer. Raise an exception and return NULL + on error. */ +PyAPI_FUNC(PyObject *) +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); + + /* --- Manage the default encoding ---------------------------------------- */ /* Returns a pointer to the default encoding (UTF-8) of the @@ -448,12 +615,19 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation in the unicodeobject. + _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to + support the previous internal function with the same behaviour. + Use of this API is DEPRECATED since no size information can be extracted from the returned data. */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); +// Alias kept for backward compatibility +#define _PyUnicode_AsString PyUnicode_AsUTF8 + + /* === Characters Type APIs =============================================== */ /* These should not be used directly. Use the Py_UNICODE_IS* and @@ -567,3 +741,10 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { || Py_UNICODE_ISDIGIT(ch) || Py_UNICODE_ISNUMERIC(ch)); } + + +/* === Misc functions ===================================================== */ + +// Return an interned Unicode object for an Identifier; may fail if there is no +// memory. +PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 1559e2def61260..28acf7265a0856 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -30,8 +30,18 @@ struct _PyWeakReference { PyWeakReference *wr_prev; PyWeakReference *wr_next; vectorcallfunc vectorcall; + +#ifdef Py_GIL_DISABLED + /* Pointer to the lock used when clearing in free-threaded builds. + * Normally this can be derived from wr_object, but in some cases we need + * to lock after wr_object has been set to Py_None. + */ + PyMutex *weakrefs_lock; +#endif }; +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { PyWeakReference *ref; diff --git a/Include/critical_section.h b/Include/critical_section.h new file mode 100644 index 00000000000000..3b37615a8b17e2 --- /dev/null +++ b/Include/critical_section.h @@ -0,0 +1,16 @@ +#ifndef Py_CRITICAL_SECTION_H +#define Py_CRITICAL_SECTION_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_CRITICAL_SECTION_H +# include "cpython/critical_section.h" +# undef Py_CPYTHON_CRITICAL_SECTION_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CRITICAL_SECTION_H */ diff --git a/Include/errcode.h b/Include/errcode.h index 8d44e9ae559193..dac5cf068c99d6 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -19,24 +19,25 @@ extern "C" { #endif -#define E_OK 10 /* No error */ -#define E_EOF 11 /* End Of File */ -#define E_INTR 12 /* Interrupted */ -#define E_TOKEN 13 /* Bad token */ -#define E_SYNTAX 14 /* Syntax error */ -#define E_NOMEM 15 /* Ran out of memory */ -#define E_DONE 16 /* Parsing complete */ -#define E_ERROR 17 /* Execution error */ -#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ -#define E_OVERFLOW 19 /* Node had too many children */ -#define E_TOODEEP 20 /* Too many indentation levels */ -#define E_DEDENT 21 /* No matching outer block for dedent */ -#define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ -#define E_LINECONT 25 /* Unexpected characters after a line continuation */ -#define E_BADSINGLE 27 /* Ill-formed single statement input */ -#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */ +#define E_OK 10 /* No error */ +#define E_EOF 11 /* End Of File */ +#define E_INTR 12 /* Interrupted */ +#define E_TOKEN 13 /* Bad token */ +#define E_SYNTAX 14 /* Syntax error */ +#define E_NOMEM 15 /* Ran out of memory */ +#define E_DONE 16 /* Parsing complete */ +#define E_ERROR 17 /* Execution error */ +#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ +#define E_OVERFLOW 19 /* Node had too many children */ +#define E_TOODEEP 20 /* Too many indentation levels */ +#define E_DEDENT 21 /* No matching outer block for dedent */ +#define E_DECODE 22 /* Error in decoding into Unicode */ +#define E_EOFS 23 /* EOF in triple-quoted string */ +#define E_EOLS 24 /* EOL in single-quoted string */ +#define E_LINECONT 25 /* Unexpected characters after a line continuation */ +#define E_BADSINGLE 27 /* Ill-formed single statement input */ +#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */ +#define E_COLUMNOVERFLOW 29 /* Column offset overflow */ #ifdef __cplusplus } diff --git a/Include/exports.h b/Include/exports.h index ce601216f17156..0c646d5beb6ad6 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -41,11 +41,8 @@ * we may still need to support gcc >= 4, as some Ubuntu LTS and Centos versions * have 4 < gcc < 5. */ - #ifndef __has_attribute - #define __has_attribute(x) 0 // Compatibility with non-clang compilers. - #endif #if (defined(__GNUC__) && (__GNUC__ >= 4)) ||\ - (defined(__clang__) && __has_attribute(visibility)) + (defined(__clang__) && _Py__has_attribute(visibility)) #define Py_IMPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_LOCAL_SYMBOL __attribute__ ((visibility ("hidden"))) diff --git a/Include/internal/mimalloc/mimalloc.h b/Include/internal/mimalloc/mimalloc.h new file mode 100644 index 00000000000000..821129e7690b1b --- /dev/null +++ b/Include/internal/mimalloc/mimalloc.h @@ -0,0 +1,565 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_H +#define MIMALLOC_H + +#define MI_MALLOC_VERSION 212 // major + 2 digits minor + +// ------------------------------------------------------ +// Compiler specific attributes +// ------------------------------------------------------ + +#ifdef __cplusplus + #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 + #define mi_attr_noexcept noexcept + #else + #define mi_attr_noexcept throw() + #endif +#else + #define mi_attr_noexcept +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201703) + #define mi_decl_nodiscard [[nodiscard]] +#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl + #define mi_decl_nodiscard __attribute__((warn_unused_result)) +#elif defined(_HAS_NODISCARD) + #define mi_decl_nodiscard _NODISCARD +#elif (_MSC_VER >= 1700) + #define mi_decl_nodiscard _Check_return_ +#else + #define mi_decl_nodiscard +#endif + +#if defined(_MSC_VER) || defined(__MINGW32__) + #if !defined(MI_SHARED_LIB) + #define mi_decl_export + #elif defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __declspec(dllexport) + #else + #define mi_decl_export __declspec(dllimport) + #endif + #if defined(__MINGW32__) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #else + #if (_MSC_VER >= 1900) && !defined(__EDG__) + #define mi_decl_restrict __declspec(allocator) __declspec(restrict) + #else + #define mi_decl_restrict __declspec(restrict) + #endif + #define mi_attr_malloc + #endif + #define mi_cdecl __cdecl + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) + #elif defined(__INTEL_COMPILER) + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) + #else + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) __attribute__((alloc_align(p))) + #endif +#else + #define mi_cdecl + #define mi_decl_export + #define mi_decl_restrict + #define mi_attr_malloc + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#endif + +// ------------------------------------------------------ +// Includes +// ------------------------------------------------------ + +#include <stddef.h> // size_t +#include <stdbool.h> // bool +#include <stdint.h> // INTPTR_MAX + +#ifdef __cplusplus +extern "C" { +#endif + +// ------------------------------------------------------ +// Standard malloc interface +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_export void mi_free(void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; + +// ------------------------------------------------------ +// Extended functionality +// ------------------------------------------------------ +#define MI_SMALL_WSIZE_MAX (128) +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; + + +// ------------------------------------------------------ +// Internals +// ------------------------------------------------------ + +typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); +mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); +mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_error_fun)(int err, void* arg); +mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); + +mi_decl_export void mi_collect(bool force) mi_attr_noexcept; +mi_decl_export int mi_version(void) mi_attr_noexcept; +mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; +mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; +mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL +mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; + +mi_decl_export void mi_process_init(void) mi_attr_noexcept; +mi_decl_export void mi_thread_init(void) mi_attr_noexcept; +mi_decl_export void mi_thread_done(void) mi_attr_noexcept; +mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; + +mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, + size_t* current_rss, size_t* peak_rss, + size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; + +// ------------------------------------------------------------------------------------- +// Aligned allocation +// Note that `alignment` always follows `size` for consistency with unaligned +// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. +// ------------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); + + +// ------------------------------------------------------------------------------------- +// Heaps: first-class, but can only allocate from the same thread that created it. +// ------------------------------------------------------------------------------------- + +struct mi_heap_s; +typedef struct mi_heap_s mi_heap_t; + +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); +mi_decl_export void mi_heap_delete(mi_heap_t* heap); +mi_decl_export void mi_heap_destroy(mi_heap_t* heap); +mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); +mi_decl_export mi_heap_t* mi_heap_get_default(void); +mi_decl_export mi_heap_t* mi_heap_get_backing(void); +mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); + + +// -------------------------------------------------------------------------------- +// Zero initialized re-allocation. +// Only valid on memory that was originally allocated with zero initialization too. +// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc. +// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992> +// -------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); + + +// ------------------------------------------------------ +// Analysis +// ------------------------------------------------------ + +mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); +mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); +mi_decl_export bool mi_check_owned(const void* p); + +// An area of heap space contains blocks of a single size. +typedef struct mi_heap_area_s { + void* blocks; // start of the area containing heap blocks + size_t reserved; // bytes reserved for this area (virtual) + size_t committed; // current available bytes for this area + size_t used; // number of allocated blocks + size_t block_size; // size in bytes of each block + size_t full_block_size; // size in bytes of a full block including padding and metadata. +} mi_heap_area_t; + +typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); + +mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); + +// Experimental +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; + +mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; +mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; + +mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; + +mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; + +// Experimental: heaps associated with specific memory arena's +typedef int mi_arena_id_t; +mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); +mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; + +#if MI_MALLOC_VERSION >= 182 +// Create a heap that only allocates in the specified arena +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); +#endif + +// deprecated +mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; + + +// ------------------------------------------------------ +// Convenience +// ------------------------------------------------------ + +#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) +#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) +#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) +#define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) +#define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) + +#define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) +#define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) +#define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) +#define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) +#define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp))) +#define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp))) + + +// ------------------------------------------------------ +// Options +// ------------------------------------------------------ + +typedef enum mi_option_e { + // stable options + mi_option_show_errors, // print error messages + mi_option_show_stats, // print statistics on termination + mi_option_verbose, // print verbose messages + // the following options are experimental (see src/options.h) + mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) + mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) + mi_option_purge_decommits, // should a memory purge decommit (or only reset) (=1) + mi_option_allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit + mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup + mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node + mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup + mi_option_deprecated_segment_cache, + mi_option_deprecated_page_reset, + mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination + mi_option_deprecated_segment_reset, + mi_option_eager_commit_delay, + mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. + mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. + mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) + mi_option_os_tag, // tag used for OS logging (macOS only for now) + mi_option_max_errors, // issue at most N error messages + mi_option_max_warnings, // issue at most N warning messages + mi_option_max_segment_reclaim, + mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe. + mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit) + mi_option_arena_purge_mult, + mi_option_purge_extend_delay, + _mi_option_last, + // legacy option names + mi_option_large_os_pages = mi_option_allow_large_os_pages, + mi_option_eager_region_commit = mi_option_arena_eager_commit, + mi_option_reset_decommits = mi_option_purge_decommits, + mi_option_reset_delay = mi_option_purge_delay, + mi_option_abandoned_page_reset = mi_option_abandoned_page_purge +} mi_option_t; + + +mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); +mi_decl_export void mi_option_enable(mi_option_t option); +mi_decl_export void mi_option_disable(mi_option_t option); +mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); +mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); + +mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); +mi_decl_export void mi_option_set(mi_option_t option, long value); +mi_decl_export void mi_option_set_default(mi_option_t option, long value); + + +// ------------------------------------------------------------------------------------------------------- +// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. +// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) +// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. +// ------------------------------------------------------------------------------------------------------- + +mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; + +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); + +mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; +mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; + +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; + +// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. +// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); + +#ifdef __cplusplus +} +#endif + +// --------------------------------------------------------------------------------------------- +// Implement the C++ std::allocator interface for use in STL containers. +// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally) +// --------------------------------------------------------------------------------------------- +#ifdef __cplusplus + +#include <cstddef> // std::size_t +#include <cstdint> // PTRDIFF_MAX +#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 +#include <type_traits> // std::true_type +#include <utility> // std::forward +#endif + +template<class T> struct _mi_stl_allocator_common { + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type& reference; + typedef value_type const& const_reference; + typedef value_type* pointer; + typedef value_type const* const_pointer; + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); } + template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); } + #else + void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } + void destroy(pointer p) { p->~value_type(); } + #endif + + size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +}; + +template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> { + using typename _mi_stl_allocator_common<T>::size_type; + using typename _mi_stl_allocator_common<T>::value_type; + using typename _mi_stl_allocator_common<T>::pointer; + template <class U> struct rebind { typedef mi_stl_allocator<U> other; }; + + mi_stl_allocator() mi_attr_noexcept = default; + mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; + template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { } + mi_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::true_type; + #endif +}; + +template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; } +template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; } + + +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 +#define MI_HAS_HEAP_STL_ALLOCATOR 1 + +#include <memory> // std::shared_ptr + +// Common base class for STL allocators in a specific heap +template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> { + using typename _mi_stl_allocator_common<T>::size_type; + using typename _mi_stl_allocator_common<T>::value_type; + using typename _mi_stl_allocator_common<T>::pointer; + + _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp) { } /* will not delete nor destroy the passed in heap */ + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::false_type; + #endif + + void collect(bool force) { mi_heap_collect(this->heap.get(), force); } + template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); } + +protected: + std::shared_ptr<mi_heap_t> heap; + template<class U, bool D> friend struct _mi_heap_stl_allocator_common; + + _mi_heap_stl_allocator_common() { + mi_heap_t* hp = mi_heap_new(); + this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ + } + _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { } + +private: + static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } + static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } +}; + +// STL allocator allocation in a specific heap +template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> { + using typename _mi_heap_stl_allocator_common<T, false>::size_type; + mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called + mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap + template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { } + + mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; }; +}; + +template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } +template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } + + +// STL allocator allocation in a specific heap, where `free` does nothing and +// the heap is destroyed in one go on destruction -- use with care! +template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> { + using typename _mi_heap_stl_allocator_common<T, true>::size_type; + mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap + template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { } + + mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ } + template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; }; +}; + +template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } +template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } + +#endif // C++11 + +#endif // __cplusplus + +#endif diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h new file mode 100644 index 00000000000000..cdd9c372beafd5 --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -0,0 +1,389 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_ATOMIC_H +#define MIMALLOC_ATOMIC_H + +// -------------------------------------------------------------------------------------------- +// Atomics +// We need to be portable between C, C++, and MSVC. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. +// This is why we try to use only `uintptr_t` and `<type>*` as atomic types. +// To gain better insight in the range of used atomics, we use explicitly named memory order operations +// instead of passing the memory order as a parameter. +// ----------------------------------------------------------------------------------------------- + +#if defined(__cplusplus) +// Use C++ atomics +#include <atomic> +#define _Atomic(tp) std::atomic<tp> +#define mi_atomic(name) std::atomic_##name +#define mi_memory_order(name) std::memory_order_##name +#if (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#elif defined(_MSC_VER) +// Use MSVC C wrapper for C11 atomics +#define _Atomic(tp) tp +#define MI_ATOMIC_VAR_INIT(x) x +#define mi_atomic(name) mi_atomic_##name +#define mi_memory_order(name) mi_memory_order_##name +#else +// Use C11 atomics +#include <stdatomic.h> +#define mi_atomic(name) atomic_##name +#define mi_memory_order(name) memory_order_##name +#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#endif + +// Various defines for all used memory orders in mimalloc +#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) +#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) + +#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) + +#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) +#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) +#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) +#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) + +static inline void mi_atomic_yield(void); +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); + + +#if defined(__cplusplus) || !defined(_MSC_VER) + +// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) +// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well +#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p) +#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p) + +// In C++ we need to add casts to help resolve templates if NULL is passed +#if defined(__cplusplus) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) +#else +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) +#endif + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { + return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); +} +static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { + int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); + while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; +} + +// Used by timers +#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) + +#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d) +#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i) + + +#elif defined(_MSC_VER) + +// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <intrin.h> +#ifdef _WIN64 +typedef LONG64 msc_intptr_t; +#define MI_64(f) f##64 +#else +typedef LONG msc_intptr_t; +#define MI_64(f) f +#endif + +typedef enum mi_memory_order_e { + mi_memory_order_relaxed, + mi_memory_order_consume, + mi_memory_order_acquire, + mi_memory_order_release, + mi_memory_order_acq_rel, + mi_memory_order_seq_cst +} mi_memory_order; + +static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); +} +static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)); +} +static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + (void)(mo1); (void)(mo2); + uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected)); + if (read == *expected) { + return true; + } + else { + *expected = read; + return false; + } +} +static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2); +} +static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); +} +static inline void mi_atomic_thread_fence(mi_memory_order mo) { + (void)(mo); + _Atomic(uintptr_t) x = 0; + mi_atomic_exchange_explicit(&x, 1, mo); +} +static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { + (void)(mo); +#if defined(_M_IX86) || defined(_M_X64) + return *p; +#else + uintptr_t x = *p; + if (mo > mi_memory_order_relaxed) { + while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; + } + return x; +#endif +} +static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); +#if defined(_M_IX86) || defined(_M_X64) + *p = x; +#else + mi_atomic_exchange_explicit(p, x, mo); +#endif +} +static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) { + (void)(mo); +#if defined(_M_X64) + return *p; +#else + int64_t old = *p; + int64_t x = old; + while ((old = InterlockedCompareExchange64(p, x, old)) != x) { + x = old; + } + return x; +#endif +} +static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) { + (void)(mo); +#if defined(x_M_IX86) || defined(_M_X64) + *p = x; +#else + InterlockedExchange64(p, x); +#endif +} + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { +#ifdef _WIN64 + return (int64_t)mi_atomic_addi((int64_t*)p, add); +#else + int64_t current; + int64_t sum; + do { + current = *p; + sum = current + add; + } while (_InterlockedCompareExchange64(p, sum, current) != current); + return current; +#endif +} +static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { + int64_t current; + do { + current = *p; + } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); +} + +static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) { + mi_atomic_addi64_relaxed(p, i); +} + +static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) { + int64_t read = _InterlockedCompareExchange64(p, des, *exp); + if (read == *exp) { + return true; + } + else { + *exp = read; + return false; + } +} + +// The pointer macros cast to `uintptr_t`. +#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) + +#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed)) + + +#endif + + +// Atomically add a signed value; returns the previous value. +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { + return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); +} + +// Atomically subtract a signed value; returns the previous value. +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { + return (intptr_t)mi_atomic_addi(p, -sub); +} + +typedef _Atomic(uintptr_t) mi_atomic_once_t; + +// Returns true only on the first invocation +static inline bool mi_atomic_once( mi_atomic_once_t* once ) { + if (mi_atomic_load_relaxed(once) != 0) return false; // quick test + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1 +} + +typedef _Atomic(uintptr_t) mi_atomic_guard_t; + +// Allows only one thread to execute at a time +#define mi_atomic_guard(guard) \ + uintptr_t _mi_guard_expected = 0; \ + for(bool _mi_guard_once = true; \ + _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ + (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) + + + +// Yield +#if defined(__cplusplus) +#include <thread> +static inline void mi_atomic_yield(void) { + std::this_thread::yield(); +} +#elif defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +static inline void mi_atomic_yield(void) { + YieldProcessor(); +} +#elif defined(__SSE2__) +#include <emmintrin.h> +static inline void mi_atomic_yield(void) { + _mm_pause(); +} +#elif (defined(__GNUC__) || defined(__clang__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \ + defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) || defined(__POWERPC__) +#if defined(__x86_64__) || defined(__i386__) +static inline void mi_atomic_yield(void) { + __asm__ volatile ("pause" ::: "memory"); +} +#elif defined(__aarch64__) +static inline void mi_atomic_yield(void) { + __asm__ volatile("wfe"); +} +#elif (defined(__arm__) && __ARM_ARCH__ >= 7) +static inline void mi_atomic_yield(void) { + __asm__ volatile("yield" ::: "memory"); +} +#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__) +#ifdef __APPLE__ +static inline void mi_atomic_yield(void) { + __asm__ volatile ("or r27,r27,r27" ::: "memory"); +} +#else +static inline void mi_atomic_yield(void) { + __asm__ __volatile__ ("or 27,27,27" ::: "memory"); +} +#endif +#elif defined(__armel__) || defined(__ARMEL__) +static inline void mi_atomic_yield(void) { + __asm__ volatile ("nop" ::: "memory"); +} +#endif +#elif defined(__sun) +// Fallback for other archs +#include <synch.h> +static inline void mi_atomic_yield(void) { + smt_pause(); +} +#elif defined(__wasi__) +#include <sched.h> +static inline void mi_atomic_yield(void) { + sched_yield(); +} +#else +#include <unistd.h> +static inline void mi_atomic_yield(void) { + sleep(0); +} +#endif + + +#endif // __MIMALLOC_ATOMIC_H diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h new file mode 100644 index 00000000000000..d97f51b8eefbe5 --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -0,0 +1,969 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_INTERNAL_H +#define MIMALLOC_INTERNAL_H + + +// -------------------------------------------------------------------------- +// This file contains the internal API's of mimalloc and various utility +// functions and macros. +// -------------------------------------------------------------------------- + +#include "types.h" +#include "track.h" + +#if (MI_DEBUG>0) +#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) +#else +#define mi_trace_message(...) +#endif + +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#else +#define mi_decl_externc +#endif + +// pthreads +#if !defined(_WIN32) && !defined(__wasi__) +#define MI_USE_PTHREADS +#include <pthread.h> +#endif + +// "options.c" +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); +void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); +void _mi_warning_message(const char* fmt, ...); +void _mi_verbose_message(const char* fmt, ...); +void _mi_trace_message(const char* fmt, ...); +void _mi_options_init(void); +void _mi_error_message(int err, const char* fmt, ...); + +// random.c +void _mi_random_init(mi_random_ctx_t* ctx); +void _mi_random_init_weak(mi_random_ctx_t* ctx); +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); +uintptr_t _mi_random_next(mi_random_ctx_t* ctx); +uintptr_t _mi_heap_random_next(mi_heap_t* heap); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); +static inline uintptr_t _mi_random_shuffle(uintptr_t x); + +// init.c +extern mi_decl_cache_align mi_stats_t _mi_stats_main; +extern mi_decl_cache_align const mi_page_t _mi_page_empty; +bool _mi_is_main_thread(void); +size_t _mi_current_thread_count(void); +bool _mi_preloading(void); // true while the C runtime is not initialized yet +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap +void _mi_thread_done(mi_heap_t* heap); +void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); + +// os.c +void _mi_os_init(void); // called from process init +void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats); +void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats); +void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats); + +size_t _mi_os_page_size(void); +size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); +bool _mi_os_has_virtual_reserve(void); + +bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); +bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); +bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats); + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats); +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats); + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_use_large_page(size_t size, size_t alignment); +size_t _mi_os_large_page_size(void); + +void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); + +// arena.c +mi_arena_id_t _mi_arena_id_none(void); +void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats); +void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); +bool _mi_arena_contains(const void* p); +void _mi_arena_collect(bool force_purge, mi_stats_t* stats); +void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); + +// "segment-map.c" +void _mi_segment_map_allocated_at(const mi_segment_t* segment); +void _mi_segment_map_freed_at(const mi_segment_t* segment); + +// "segment.c" +extern mi_abandoned_pool_t _mi_abandoned_default; // global abandoned pool +mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); +void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); +void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); +bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); +void _mi_segment_thread_collect(mi_segments_tld_t* tld); +bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + + +#if MI_HUGE_PAGE_ABANDON +void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +#else +void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +#endif + +uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page +void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); +void _mi_abandoned_await_readers(mi_abandoned_pool_t *pool); +void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); + +// "page.c" +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; + +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks +void _mi_page_unfull(mi_page_t* page); +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_heap_delayed_free_all(mi_heap_t* heap); +bool _mi_heap_delayed_free_partial(mi_heap_t* heap); +void _mi_heap_collect_retired(mi_heap_t* heap, bool force); + +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); +void _mi_deferred_free(mi_heap_t* heap, bool force); + +void _mi_page_free_collect(mi_page_t* page,bool force); +void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments + +size_t _mi_bin_size(uint8_t bin); // for stats +uint8_t _mi_bin(size_t size); // for stats + +// "heap.c" +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool no_reclaim, uint8_t tag); +void _mi_heap_destroy_pages(mi_heap_t* heap); +void _mi_heap_collect_abandon(mi_heap_t* heap); +void _mi_heap_set_default_direct(mi_heap_t* heap); +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); +void _mi_heap_unsafe_destroy_all(void); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_block_visit_fun* visitor, void* arg); + +// "stats.c" +void _mi_stats_done(mi_stats_t* stats); +mi_msecs_t _mi_clock_now(void); +mi_msecs_t _mi_clock_end(mi_msecs_t start); +mi_msecs_t _mi_clock_start(void); + +// "alloc.c" +void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; +void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` +void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); +bool _mi_free_delayed_block(mi_block_t* block); +void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); + +// option.c, c primitives +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +void _mi_strlcpy(char* dest, const char* src, size_t dest_size); +void _mi_strlcat(char* dest, const char* src, size_t dest_size); +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); + + +#if MI_DEBUG>1 +bool _mi_page_is_valid(mi_page_t* page); +#endif + + +// ------------------------------------------------------ +// Branches +// ------------------------------------------------------ + +#if defined(__GNUC__) || defined(__clang__) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] +#else +#define mi_unlikely(x) (x) +#define mi_likely(x) (x) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + + +/* ----------------------------------------------------------- + Error codes passed to `_mi_fatal_error` + All are recoverable but EFAULT is a serious error and aborts by default in secure mode. + For portability define undefined error codes using common Unix codes: + <https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html> +----------------------------------------------------------- */ +#include <errno.h> +#ifndef EAGAIN // double free +#define EAGAIN (11) +#endif +#ifndef ENOMEM // out of memory +#define ENOMEM (12) +#endif +#ifndef EFAULT // corrupted free-list or meta-data +#define EFAULT (14) +#endif +#ifndef EINVAL // trying to free an invalid pointer +#define EINVAL (22) +#endif +#ifndef EOVERFLOW // count*size overflow +#define EOVERFLOW (75) +#endif + + +/* ----------------------------------------------------------- + Inlined definitions +----------------------------------------------------------- */ +#define MI_UNUSED(x) (void)(x) +#if (MI_DEBUG>0) +#define MI_UNUSED_RELEASE(x) +#else +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) +#endif + +#define MI_INIT4(x) x(),x(),x(),x() +#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) +#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) +#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) +#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) +#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) +#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) + + +#include <string.h> +// initialize a local variable to zero; use memset as compilers optimize constant sized memset's +#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) + +// Is `x` a power of two? (0 is considered a power of two) +static inline bool _mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} + +// Is a pointer aligned? +static inline bool _mi_is_aligned(void* p, size_t alignment) { + mi_assert_internal(alignment != 0); + return (((uintptr_t)p % alignment) == 0); +} + +// Align upwards +static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + +// Align downwards +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz / alignment) * alignment); + } +} + +// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. +static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { + mi_assert_internal(divider != 0); + return (divider == 0 ? size : ((size + divider - 1) / divider)); +} + +// Is memory zero initialized? +static inline bool mi_mem_is_zero(const void* p, size_t size) { + for (size_t i = 0; i < size; i++) { + if (((uint8_t*)p)[i] != 0) return false; + } + return true; +} + + +// Align a byte size to a size in _machine words_, +// i.e. byte size == `wsize*sizeof(void*)`. +static inline size_t _mi_wsize_from_size(size_t size) { + mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); + return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); +} + +// Overflow detecting multiply +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) +#include <limits.h> // UINT_MAX, ULONG_MAX +#if defined(_CLOCK_T) // for Illumos +#undef _CLOCK_T +#endif +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #if (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); + #elif (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, (unsigned int *)total); + #else + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); + #endif +} +#else /* __builtin_umul_overflow is unavailable */ +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) + *total = count * size; + // note: gcc/clang optimize this to directly check the overflow flag + return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); +} +#endif + +// Safe multiply `count*size` into `total`; return `true` on overflow. +static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { + if (count==1) { // quick check for the case where count is one (common for C++ allocators) + *total = size; + return false; + } + else if mi_unlikely(mi_mul_overflow(count, size, total)) { + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); + #endif + *total = SIZE_MAX; + return true; + } + else return false; +} + + +/*---------------------------------------------------------------------------------------- + Heap functions +------------------------------------------------------------------------------------------- */ + +extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap + +static inline bool mi_heap_is_backing(const mi_heap_t* heap) { + return (heap->tld->heap_backing == heap); +} + +static inline bool mi_heap_is_initialized(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + return (heap != &_mi_heap_empty); +} + +static inline uintptr_t _mi_ptr_cookie(const void* p) { + extern mi_heap_t _mi_heap_main; + mi_assert_internal(_mi_heap_main.cookie != 0); + return ((uintptr_t)p ^ _mi_heap_main.cookie); +} + +/* ----------------------------------------------------------- + Pages +----------------------------------------------------------- */ + +static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { + mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = _mi_wsize_from_size(size); + mi_assert_internal(idx < MI_PAGES_DIRECT); + return heap->pages_free_direct[idx]; +} + +// Segment that contains the pointer +// Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE), +// and we need align "down" to the segment info which is `MI_SEGMENT_SIZE` bytes before it; +// therefore we align one byte before `p`. +static inline mi_segment_t* _mi_ptr_segment(const void* p) { + mi_assert_internal(p != NULL); + return (mi_segment_t*)(((uintptr_t)p - 1) & ~MI_SEGMENT_MASK); +} + +static inline mi_page_t* mi_slice_to_page(mi_slice_t* s) { + mi_assert_internal(s->slice_offset== 0 && s->slice_count > 0); + return (mi_page_t*)(s); +} + +static inline mi_slice_t* mi_page_to_slice(mi_page_t* p) { + mi_assert_internal(p->slice_offset== 0 && p->slice_count > 0); + return (mi_slice_t*)(p); +} + +// Segment belonging to a page +static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { + mi_segment_t* segment = _mi_ptr_segment(page); + mi_assert_internal(segment == NULL || ((mi_slice_t*)page >= segment->slices && (mi_slice_t*)page < segment->slices + segment->slice_entries)); + return segment; +} + +static inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) { + mi_slice_t* start = (mi_slice_t*)((uint8_t*)slice - slice->slice_offset); + mi_assert_internal(start >= _mi_ptr_segment(slice)->slices); + mi_assert_internal(start->slice_offset == 0); + mi_assert_internal(start + start->slice_count > slice); + return start; +} + +// Get the page containing the pointer (performance critical as it is called in mi_free) +static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { + mi_assert_internal(p > (void*)segment); + ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; + mi_assert_internal(diff > 0 && diff <= (ptrdiff_t)MI_SEGMENT_SIZE); + size_t idx = (size_t)diff >> MI_SEGMENT_SLICE_SHIFT; + mi_assert_internal(idx <= segment->slice_entries); + mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx]; + mi_slice_t* slice = mi_slice_first(slice0); // adjust to the block that holds the page data + mi_assert_internal(slice->slice_offset == 0); + mi_assert_internal(slice >= segment->slices && slice < segment->slices + segment->slice_entries); + return mi_slice_to_page(slice); +} + +// Quick page start for initialized pages +static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { + return _mi_segment_page_start(segment, page, page_size); +} + +// Get the page containing the pointer +static inline mi_page_t* _mi_ptr_page(void* p) { + return _mi_segment_page_of(_mi_ptr_segment(p), p); +} + +// Get the block size of a page (special case for huge objects) +static inline size_t mi_page_block_size(const mi_page_t* page) { + const size_t bsize = page->xblock_size; + mi_assert_internal(bsize > 0); + if mi_likely(bsize < MI_HUGE_BLOCK_SIZE) { + return bsize; + } + else { + size_t psize; + _mi_segment_page_start(_mi_page_segment(page), page, &psize); + return psize; + } +} + +static inline bool mi_page_is_huge(const mi_page_t* page) { + return (_mi_page_segment(page)->kind == MI_SEGMENT_HUGE); +} + +// Get the usable block size of a page without fixed padding. +// This may still include internal padding due to alignment and rounding up size classes. +static inline size_t mi_page_usable_block_size(const mi_page_t* page) { + return mi_page_block_size(page) - MI_PADDING_SIZE; +} + +// size of a segment +static inline size_t mi_segment_size(mi_segment_t* segment) { + return segment->segment_slices * MI_SEGMENT_SLICE_SIZE; +} + +static inline uint8_t* mi_segment_end(mi_segment_t* segment) { + return (uint8_t*)segment + mi_segment_size(segment); +} + +// Thread free access +static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { + return (mi_block_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & ~3); +} + +static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) { + return (mi_delayed_t)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & 3); +} + +// Heap access +static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { + return (mi_heap_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xheap)); +} + +static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { + mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING); + mi_atomic_store_release(&page->xheap,(uintptr_t)heap); +} + +// Thread free flag helpers +static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { + return (mi_block_t*)(tf & ~0x03); +} +static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) { + return (mi_delayed_t)(tf & 0x03); +} +static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) { + return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed); +} +static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { + return mi_tf_make(mi_tf_block(tf),delayed); +} +static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) { + return mi_tf_make(block, mi_tf_delayed(tf)); +} + +// are all blocks in a page freed? +// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. +static inline bool mi_page_all_free(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->used == 0); +} + +// are there any available blocks? +static inline bool mi_page_has_any_available(const mi_page_t* page) { + mi_assert_internal(page != NULL && page->reserved > 0); + return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); +} + +// are there immediately available blocks, i.e. blocks available on the free list. +static inline bool mi_page_immediate_available(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->free != NULL); +} + +// is more than 7/8th of a page in use? +static inline bool mi_page_mostly_used(const mi_page_t* page) { + if (page==NULL) return true; + uint16_t frac = page->reserved / 8U; + return (page->reserved - page->used <= frac); +} + +static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { + return &((mi_heap_t*)heap)->pages[_mi_bin(size)]; +} + + + +//----------------------------------------------------------- +// Page flags +//----------------------------------------------------------- +static inline bool mi_page_is_in_full(const mi_page_t* page) { + return page->flags.x.in_full; +} + +static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { + page->flags.x.in_full = in_full; +} + +static inline bool mi_page_has_aligned(const mi_page_t* page) { + return page->flags.x.has_aligned; +} + +static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { + page->flags.x.has_aligned = has_aligned; +} + + +/* ------------------------------------------------------------------- +Encoding/Decoding the free list next pointers + +This is to protect against buffer overflow exploits where the +free list is mutated. Many hardened allocators xor the next pointer `p` +with a secret key `k1`, as `p^k1`. This prevents overwriting with known +values but might be still too weak: if the attacker can guess +the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). +Moreover, if multiple blocks can be read as well, the attacker can +xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot +about the pointers (and subsequently `k1`). + +Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<<k1)+k1`. +Since these operations are not associative, the above approaches do not +work so well any more even if the `p` can be guesstimated. For example, +for the read case we can subtract two entries to discard the `+k1` term, +but that leads to `((p1^k2)<<<k1) - ((p2^k2)<<<k1)` at best. +We include the left-rotation since xor and addition are otherwise linear +in the lowest bit. Finally, both keys are unique per page which reduces +the re-use of keys by a large factor. + +We also pass a separate `null` value to be used as `NULL` or otherwise +`(k2<<<k1)+k1` would appear (too) often as a sentinel value. +------------------------------------------------------------------- */ + +static inline bool mi_is_in_same_segment(const void* p, const void* q) { + return (_mi_ptr_segment(p) == _mi_ptr_segment(q)); +} + +static inline bool mi_is_in_same_page(const void* p, const void* q) { + mi_segment_t* segment = _mi_ptr_segment(p); + if (_mi_ptr_segment(q) != segment) return false; + // assume q may be invalid // return (_mi_segment_page_of(segment, p) == _mi_segment_page_of(segment, q)); + mi_page_t* page = _mi_segment_page_of(segment, p); + size_t psize; + uint8_t* start = _mi_segment_page_start(segment, page, &psize); + return (start <= (uint8_t*)q && (uint8_t*)q < start + psize); +} + +static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) { + shift %= MI_INTPTR_BITS; + return (shift==0 ? x : ((x << shift) | (x >> (MI_INTPTR_BITS - shift)))); +} +static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) { + shift %= MI_INTPTR_BITS; + return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift)))); +} + +static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { + void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); + return (p==null ? NULL : p); +} + +static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { + uintptr_t x = (uintptr_t)(p==NULL ? null : p); + return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; +} + +static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { + mi_track_mem_defined(block,sizeof(mi_block_t)); + mi_block_t* next; + #ifdef MI_ENCODE_FREELIST + next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + next = (mi_block_t*)block->next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); + return next; +} + +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { + mi_track_mem_undefined(block,sizeof(mi_block_t)); + #ifdef MI_ENCODE_FREELIST + block->next = mi_ptr_encode(null, next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + block->next = (mi_encoded_t)next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); +} + +static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { + #ifdef MI_ENCODE_FREELIST + mi_block_t* next = mi_block_nextx(page,block,page->keys); + // check for free list corruption: is `next` at least in the same page? + // TODO: check if `next` is `page->block_size` aligned? + if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + next = NULL; + } + return next; + #else + MI_UNUSED(page); + return mi_block_nextx(page,block,NULL); + #endif +} + +static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { + #ifdef MI_ENCODE_FREELIST + mi_block_set_nextx(page,block,next, page->keys); + #else + MI_UNUSED(page); + mi_block_set_nextx(page,block,next,NULL); + #endif +} + + +// ------------------------------------------------------------------- +// commit mask +// ------------------------------------------------------------------- + +static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + cm->mask[i] = 0; + } +} + +static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + cm->mask[i] = ~((size_t)0); + } +} + +static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if (cm->mask[i] != 0) return false; + } + return true; +} + +static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if (cm->mask[i] != ~((size_t)0)) return false; + } + return true; +} + +// defined in `segment.c`: +size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); +size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx); + +#define mi_commit_mask_foreach(cm,idx,count) \ + idx = 0; \ + while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) { + +#define mi_commit_mask_foreach_end() \ + idx += count; \ + } + + + +/* ----------------------------------------------------------- + memory id's +----------------------------------------------------------- */ + +static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) { + mi_memid_t memid; + _mi_memzero_var(memid); + memid.memkind = memkind; + return memid; +} + +static inline mi_memid_t _mi_memid_none(void) { + return _mi_memid_create(MI_MEM_NONE); +} + +static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) { + mi_memid_t memid = _mi_memid_create(MI_MEM_OS); + memid.initially_committed = committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_large; + return memid; +} + + +// ------------------------------------------------------------------- +// Fast "random" shuffle +// ------------------------------------------------------------------- + +static inline uintptr_t _mi_random_shuffle(uintptr_t x) { + if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros +#if (MI_INTPTR_SIZE==8) + // by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c> + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9UL; + x ^= x >> 27; + x *= 0x94d049bb133111ebUL; + x ^= x >> 31; +#elif (MI_INTPTR_SIZE==4) + // by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/> + x ^= x >> 16; + x *= 0x7feb352dUL; + x ^= x >> 15; + x *= 0x846ca68bUL; + x ^= x >> 16; +#endif + return x; +} + +// ------------------------------------------------------------------- +// Optimize numa node access for the common case (= one node) +// ------------------------------------------------------------------- + +int _mi_os_numa_node_get(mi_os_tld_t* tld); +size_t _mi_os_numa_node_count_get(void); + +extern _Atomic(size_t) _mi_numa_node_count; +static inline int _mi_os_numa_node(mi_os_tld_t* tld) { + if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; } + else return _mi_os_numa_node_get(tld); +} +static inline size_t _mi_os_numa_node_count(void) { + const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); + if mi_likely(count > 0) { return count; } + else return _mi_os_numa_node_count_get(); +} + + + +// ----------------------------------------------------------------------- +// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) +// ----------------------------------------------------------------------- + +#if defined(__GNUC__) + +#include <limits.h> // LONG_MAX +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_clzl(x); +#else + return __builtin_clzll(x); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_ctzl(x); +#else + return __builtin_ctzll(x); +#endif +} + +#elif defined(_MSC_VER) + +#include <limits.h> // LONG_MAX +#include <intrin.h> // BitScanReverse64 +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanReverse(&idx, x); +#else + _BitScanReverse64(&idx, x); +#endif + return ((MI_INTPTR_BITS - 1) - idx); +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanForward(&idx, x); +#else + _BitScanForward64(&idx, x); +#endif + return idx; +} + +#else +static inline size_t mi_ctz32(uint32_t x) { + // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf> + static const unsigned char debruijn[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + if (x==0) return 32; + return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27]; +} +static inline size_t mi_clz32(uint32_t x) { + // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf> + static const uint8_t debruijn[32] = { + 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, + 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0 + }; + if (x==0) return 32; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27]; +} + +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_clz32((uint32_t)x); +#else + size_t count = mi_clz32((uint32_t)(x >> 32)); + if (count < 32) return count; + return (32 + mi_clz32((uint32_t)x)); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_ctz32((uint32_t)x); +#else + size_t count = mi_ctz32((uint32_t)x); + if (count < 32) return count; + return (32 + mi_ctz32((uint32_t)(x>>32))); +#endif +} + +#endif + +// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero) +static inline size_t mi_bsr(uintptr_t x) { + return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x)); +} + + +// --------------------------------------------------------------------------------- +// Provide our own `_mi_memcpy` for potential performance optimizations. +// +// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if +// we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support +// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. +// --------------------------------------------------------------------------------- + +#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#include <intrin.h> +extern bool _mi_cpu_has_fsrm; +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + if (_mi_cpu_has_fsrm) { + __movsb((unsigned char*)dst, (const unsigned char*)src, n); + } + else { + memcpy(dst, src, n); + } +} +static inline void _mi_memzero(void* dst, size_t n) { + if (_mi_cpu_has_fsrm) { + __stosb((unsigned char*)dst, 0, n); + } + else { + memset(dst, 0, n); + } +} +#else +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + memcpy(dst, src, n); +} +static inline void _mi_memzero(void* dst, size_t n) { + memset(dst, 0, n); +} +#endif + +// ------------------------------------------------------------------------------- +// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned +// This is used for example in `mi_realloc`. +// ------------------------------------------------------------------------------- + +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +// On GCC/CLang we provide a hint that the pointers are word aligned. +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); + _mi_memcpy(adst, asrc, n); +} + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + _mi_memzero(adst, n); +} +#else +// Default fallback on `_mi_memcpy` +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + _mi_memcpy(dst, src, n); +} + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + _mi_memzero(dst, n); +} +#endif + + +#endif diff --git a/Include/internal/mimalloc/mimalloc/prim.h b/Include/internal/mimalloc/mimalloc/prim.h new file mode 100644 index 00000000000000..8a60d528458e6c --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/prim.h @@ -0,0 +1,329 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_H +#define MIMALLOC_PRIM_H + + +// -------------------------------------------------------------------------- +// This file specifies the primitive portability API. +// Each OS/host needs to implement these primitives, see `src/prim` +// for implementations on Window, macOS, WASI, and Linux/Unix. +// +// note: on all primitive functions, we always have result parameters != NUL, and: +// addr != NULL and page aligned +// size > 0 and page aligned +// return value is an error code an int where 0 is success. +// -------------------------------------------------------------------------- + +// OS memory configuration +typedef struct mi_os_mem_config_s { + size_t page_size; // 4KiB + size_t large_page_size; // 2MiB + size_t alloc_granularity; // smallest allocation size (on Windows 64KiB) + bool has_overcommit; // can we reserve more memory than can be actually committed? + bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) + bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) +} mi_os_mem_config_t; + +// Initialize +void _mi_prim_mem_init( mi_os_mem_config_t* config ); + +// Free OS memory +int _mi_prim_free(void* addr, size_t size ); + +// Allocate OS memory. Return NULL on error. +// The `try_alignment` is just a hint and the returned pointer does not have to be aligned. +// If `commit` is false, the virtual memory range only needs to be reserved (with no access) +// which will later be committed explicitly using `_mi_prim_commit`. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// pre: !commit => !allow_large +// try_alignment >= _mi_os_page_size() and a power of 2 +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); + +// Commit memory. Returns error code or 0 on success. +// For example, on Linux this would make the memory PROT_READ|PROT_WRITE. +// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows) +int _mi_prim_commit(void* addr, size_t size, bool* is_zero); + +// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true +// if the memory would need to be re-committed. For example, on Windows this is always true, +// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit. +// pre: needs_recommit != NULL +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); + +// Reset memory. The range keeps being accessible but the content might be reset. +// Returns error code or 0 on success. +int _mi_prim_reset(void* addr, size_t size); + +// Protect memory. Returns error code or 0 on success. +int _mi_prim_protect(void* addr, size_t size, bool protect); + +// Allocate huge (1GiB) pages possibly associated with a NUMA node. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// pre: size > 0 and a multiple of 1GiB. +// numa_node is either negative (don't care), or a numa node number. +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); + +// Return the current NUMA node +size_t _mi_prim_numa_node(void); + +// Return the number of logical NUMA nodes +size_t _mi_prim_numa_node_count(void); + +// Clock ticks +mi_msecs_t _mi_prim_clock_now(void); + +// Return process information (only for statistics) +typedef struct mi_process_info_s { + mi_msecs_t elapsed; + mi_msecs_t utime; + mi_msecs_t stime; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; +} mi_process_info_t; + +void _mi_prim_process_info(mi_process_info_t* pinfo); + +// Default stderr output. (only for warnings etc. with verbose enabled) +// msg != NULL && _mi_strlen(msg) > 0 +void _mi_prim_out_stderr( const char* msg ); + +// Get an environment variable. (only for options) +// name != NULL, result != NULL, result_size >= 64 +bool _mi_prim_getenv(const char* name, char* result, size_t result_size); + + +// Fill a buffer with strong randomness; return `false` on error or if +// there is no strong randomization available. +bool _mi_prim_random_buf(void* buf, size_t buf_len); + +// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination. +void _mi_prim_thread_init_auto_done(void); + +// Called on process exit and may take action to clean up resources associated with the thread auto done. +void _mi_prim_thread_done_auto_done(void); + +// Called when the default heap for a thread changes +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); + + +//------------------------------------------------------------------- +// Thread id: `_mi_prim_thread_id()` +// +// Getting the thread id should be performant as it is called in the +// fast path of `_mi_free` and we specialize for various platforms as +// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. +// We only require _mi_prim_thread_id() to return a unique id +// for each thread (unequal to zero). +//------------------------------------------------------------------- + +// defined in `init.c`; do not use these directly +extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from +extern bool _mi_process_is_initialized; // has mi_process_init been called? + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; + +#ifdef MI_PRIM_THREAD_ID + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return MI_PRIM_THREAD_ID(); +} + +#elif defined(_WIN32) + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + // Windows: works on Intel and ARM in both 32- and 64-bit + return (uintptr_t)NtCurrentTeb(); +} + +// We use assembly for a fast thread id on the main platforms. The TLS layout depends on +// both the OS and libc implementation so we use specific tests for each main platform. +// If you test on another platform and it works please send a PR :-) +// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. +#elif defined(__GNUC__) && ( \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + ) + +static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { + void* res; + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + res = tcb[slot]; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + res = tcb[slot]; + #endif + return res; +} + +// setting a tls slot is only used on macOS for now +static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + tcb[slot] = value; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + tcb[slot] = value; + #endif +} + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + #if defined(__BIONIC__) + // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id + // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 + return (uintptr_t)mi_prim_tls_slot(1); + #else + // in all our other targets, slot 0 is the thread id + // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h + // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 + return (uintptr_t)mi_prim_tls_slot(0); + #endif +} + +#else + +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return (uintptr_t)&_mi_heap_default; +} + +#endif + + + +/* ---------------------------------------------------------------------------------------- +The thread local default heap: `_mi_prim_get_default_heap()` +This is inlined here as it is on the fast path for allocation functions. + +On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a +__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures +that the storage will always be available (allocated on the thread stacks). + +On some platforms though we cannot use that when overriding `malloc` since the underlying +TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. +We try to circumvent this in an efficient way: +- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the + loader itself calls `malloc` even before the modules are initialized. +- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). +- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) +------------------------------------------------------------------------------------------- */ + +static inline mi_heap_t* mi_prim_get_default_heap(void); + +#if defined(MI_MALLOC_OVERRIDE) +#if defined(__APPLE__) // macOS + #define MI_TLS_SLOT 89 // seems unused? + // #define MI_TLS_RECURSE_GUARD 1 + // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) + // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h> +#elif defined(__OpenBSD__) + // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) + // see <https://github.com/openbsd/src/blob/master/lib/libc/include/thread_private.h#L371> + #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) + // #elif defined(__DragonFly__) + // #warning "mimalloc is not working correctly on DragonFly yet." + // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) <https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/lib/libthread_xu/thread/thr_private.h#L458> +#elif defined(__ANDROID__) + // See issue #381 + #define MI_TLS_PTHREAD +#endif +#endif + + +#if defined(MI_TLS_SLOT) + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); + if mi_unlikely(heap == NULL) { + #ifdef __GNUC__ + __asm(""); // prevent conditional load of the address of _mi_heap_empty + #endif + heap = (mi_heap_t*)&_mi_heap_empty; + } + return heap; +} + +#elif defined(MI_TLS_PTHREAD_SLOT_OFS) + +static inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) { + pthread_t self = pthread_self(); + #if defined(__DragonFly__) + if (self==NULL) return NULL; + #endif + return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); +} + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot(); + if mi_unlikely(pheap == NULL) return _mi_heap_main_get(); + mi_heap_t* heap = *pheap; + if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty; + return heap; +} + +#elif defined(MI_TLS_PTHREAD) + +extern pthread_key_t _mi_heap_default_key; +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); + return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); +} + +#else // default using a thread local variable; used on most platforms. + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + #if defined(MI_TLS_RECURSE_GUARD) + if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); + #endif + return _mi_heap_default; +} + +#endif // mi_prim_get_default_heap() + + + +#endif // MIMALLOC_PRIM_H diff --git a/Include/internal/mimalloc/mimalloc/track.h b/Include/internal/mimalloc/mimalloc/track.h new file mode 100644 index 00000000000000..fa1a048d846a9c --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/track.h @@ -0,0 +1,147 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_TRACK_H +#define MIMALLOC_TRACK_H + +/* ------------------------------------------------------------------------------------------------------ +Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. +These can be defined for tracking allocation: + + #define mi_track_malloc_size(p,reqsize,size,zero) + #define mi_track_free_size(p,_size) + +The macros are set up such that the size passed to `mi_track_free_size` +always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). +The `reqsize` is what the user requested, and `size >= reqsize`. +The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, +or otherwise it is the usable block size which may be larger than the original request. +Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). +The `zero` parameter is `true` if the allocated block is zero initialized. + +Optional: + + #define mi_track_align(p,alignedp,offset,size) + #define mi_track_resize(p,oldsize,newsize) + #define mi_track_init() + +The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. +The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). +The `mi_track_resize` is currently unused but could be called on reallocations within a block. +`mi_track_init` is called at program start. + +The following macros are for tools like asan and valgrind to track whether memory is +defined, undefined, or not accessible at all: + + #define mi_track_mem_defined(p,size) + #define mi_track_mem_undefined(p,size) + #define mi_track_mem_noaccess(p,size) + +-------------------------------------------------------------------------------------------------------*/ + +#if MI_TRACK_VALGRIND +// valgrind tool + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy +#define MI_TRACK_TOOL "valgrind" + +#include <valgrind/valgrind.h> +#include <valgrind/memcheck.h> + +#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) +#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) +#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) +#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) +#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) +#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) + +#elif MI_TRACK_ASAN +// address sanitizer + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "asan" + +#include <sanitizer/asan_interface.h> + +#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) + +#elif MI_TRACK_ETW +// windows event tracing + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 +#define MI_TRACK_TOOL "ETW" + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include "../src/prim/windows/etw.h" + +#define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); +#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) +#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) + +#else +// no tracking + +#define MI_TRACK_ENABLED 0 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "none" + +#define mi_track_malloc_size(p,reqsize,size,zero) +#define mi_track_free_size(p,_size) + +#endif + +// ------------------- +// Utility definitions + +#ifndef mi_track_resize +#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) +#endif + +#ifndef mi_track_align +#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) +#endif + +#ifndef mi_track_init +#define mi_track_init() +#endif + +#ifndef mi_track_mem_defined +#define mi_track_mem_defined(p,size) +#endif + +#ifndef mi_track_mem_undefined +#define mi_track_mem_undefined(p,size) +#endif + +#ifndef mi_track_mem_noaccess +#define mi_track_mem_noaccess(p,size) +#endif + + +#if MI_PADDING +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)==(reqsize)); \ + mi_track_malloc_size(p,reqsize,reqsize,zero); \ + } +#else +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ + } +#endif + +#endif diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h new file mode 100644 index 00000000000000..354839ba955b36 --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -0,0 +1,721 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_TYPES_H +#define MIMALLOC_TYPES_H + +// -------------------------------------------------------------------------- +// This file contains the main type definitions for mimalloc: +// mi_heap_t : all data for a thread-local heap, contains +// lists of all managed heap pages. +// mi_segment_t : a larger chunk of memory (32GiB) from where pages +// are allocated. +// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from +// where objects are allocated. +// -------------------------------------------------------------------------- + + +#include <stddef.h> // ptrdiff_t +#include <stdint.h> // uintptr_t, uint16_t, etc +#include "atomic.h" // _Atomic + +#ifdef _MSC_VER +#pragma warning(disable:4214) // bitfield is not int +#endif + +// Minimal alignment necessary. On most platforms 16 bytes are needed +// due to SSE registers for example. This must be at least `sizeof(void*)` +#ifndef MI_MAX_ALIGN_SIZE +#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) +#endif + +#define MI_CACHE_LINE 64 +#if defined(_MSC_VER) +#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning +#define mi_decl_noinline __declspec(noinline) +#define mi_decl_thread __declspec(thread) +#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc +#define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_thread __thread +#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) +#else +#define mi_decl_noinline +#define mi_decl_thread __thread // hope for the best :-) +#define mi_decl_cache_align +#endif + +// ------------------------------------------------------ +// Variants +// ------------------------------------------------------ + +// Define NDEBUG in the release version to disable assertions. +// #define NDEBUG + +// Define MI_TRACK_<tool> to enable tracking support +// #define MI_TRACK_VALGRIND 1 +// #define MI_TRACK_ASAN 1 +// #define MI_TRACK_ETW 1 + +// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STAT 1 + +// Define MI_SECURE to enable security mitigations +// #define MI_SECURE 1 // guard page around metadata +// #define MI_SECURE 2 // guard page around each mimalloc page +// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) +// #define MI_SECURE 4 // checks for double free. (may be more expensive) + +#if !defined(MI_SECURE) +#define MI_SECURE 0 +#endif + +// Define MI_DEBUG for debug mode +// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. +// #define MI_DEBUG 2 // + internal assertion checks +// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON) +#if !defined(MI_DEBUG) +#if !defined(NDEBUG) || defined(_DEBUG) +#define MI_DEBUG 2 +#else +#define MI_DEBUG 0 +#endif +#endif + +// Reserve extra padding at the end of each block to be more resilient against heap block overflows. +// The padding can detect buffer overflow on free. +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#define MI_PADDING 1 +#endif + +// Check padding bytes; allows byte-precise buffer overflow detection +#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_PADDING_CHECK 1 +#endif + + +// Encoded free lists allow detection of corrupted free lists +// and can detect buffer overflows, modify after free, and double `free`s. +#if (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_ENCODE_FREELIST 1 +#endif + + +// We used to abandon huge pages but to eagerly deallocate if freed from another thread, +// but that makes it not possible to visit them during a heap walk or include them in a +// `mi_heap_destroy`. We therefore instead reset/decommit the huge blocks if freed from +// another thread so most memory is available until it gets properly freed by the owning thread. +// #define MI_HUGE_PAGE_ABANDON 1 + + +// ------------------------------------------------------ +// Platform specific values +// ------------------------------------------------------ + +// ------------------------------------------------------ +// Size of a pointer. +// We assume that `sizeof(void*)==sizeof(intptr_t)` +// and it holds for all platforms we know of. +// +// However, the C standard only requires that: +// p == (void*)((intptr_t)p)) +// but we also need: +// i == (intptr_t)((void*)i) +// or otherwise one might define an intptr_t type that is larger than a pointer... +// ------------------------------------------------------ + +#if INTPTR_MAX > INT64_MAX +# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#elif INTPTR_MAX == INT64_MAX +# define MI_INTPTR_SHIFT (3) +#elif INTPTR_MAX == INT32_MAX +# define MI_INTPTR_SHIFT (2) +#else +#error platform pointers must be 32, 64, or 128 bits +#endif + +#if SIZE_MAX == UINT64_MAX +# define MI_SIZE_SHIFT (3) +typedef int64_t mi_ssize_t; +#elif SIZE_MAX == UINT32_MAX +# define MI_SIZE_SHIFT (2) +typedef int32_t mi_ssize_t; +#else +#error platform objects must be 32 or 64 bits +#endif + +#if (SIZE_MAX/2) > LONG_MAX +# define MI_ZU(x) x##ULL +# define MI_ZI(x) x##LL +#else +# define MI_ZU(x) x##UL +# define MI_ZI(x) x##L +#endif + +#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT) +#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8) + +#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT) +#define MI_SIZE_BITS (MI_SIZE_SIZE*8) + +#define MI_KiB (MI_ZU(1024)) +#define MI_MiB (MI_KiB*MI_KiB) +#define MI_GiB (MI_MiB*MI_KiB) + + +// ------------------------------------------------------ +// Main internal data-structures +// ------------------------------------------------------ + +// Main tuning parameters for segment and page sizes +// Sizes for 64-bit (usually divide by two for 32-bit) +#define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB (32KiB on 32-bit) + +#if MI_INTPTR_SIZE > 4 +#define MI_SEGMENT_SHIFT ( 9 + MI_SEGMENT_SLICE_SHIFT) // 32MiB +#else +#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit +#endif + +#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB + + +// Derived constants +#define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT) +#define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE +#define MI_SEGMENT_MASK ((uintptr_t)(MI_SEGMENT_ALIGN - 1)) +#define MI_SEGMENT_SLICE_SIZE (MI_ZU(1)<< MI_SEGMENT_SLICE_SHIFT) +#define MI_SLICES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_SEGMENT_SLICE_SIZE) // 1024 + +#define MI_SMALL_PAGE_SIZE (MI_ZU(1)<<MI_SMALL_PAGE_SHIFT) +#define MI_MEDIUM_PAGE_SIZE (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT) + +#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit +#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit +#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) +#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit +#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) + +// Maximum number of size classes. (spaced exponentially in 12.5% increments) +#define MI_BIN_HUGE (73U) + +#if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360) +#error "mimalloc internal: define more bins" +#endif + +// Maximum slice offset (15) +#define MI_MAX_SLICE_OFFSET ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1) + +// Used as a special value to encode block sizes in 32 bits. +#define MI_HUGE_BLOCK_SIZE ((uint32_t)(2*MI_GiB)) + +// blocks up to this size are always allocated aligned +#define MI_MAX_ALIGN_GUARANTEE (8*MI_MAX_ALIGN_SIZE) + +// Alignments over MI_ALIGNMENT_MAX are allocated in dedicated huge page segments +#define MI_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1) + + +// ------------------------------------------------------ +// Mimalloc pages contain allocated blocks +// ------------------------------------------------------ + +// The free lists use encoded next fields +// (Only actually encodes when MI_ENCODED_FREELIST is defined.) +typedef uintptr_t mi_encoded_t; + +// thread id's +typedef size_t mi_threadid_t; + +// free lists contain blocks +typedef struct mi_block_s { + mi_encoded_t next; +} mi_block_t; + + +// The delayed flags are used for efficient multi-threaded free-ing +typedef enum mi_delayed_e { + MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list + MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap + MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list + MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim +} mi_delayed_t; + + +// The `in_full` and `has_aligned` page flags are put in a union to efficiently +// test if both are false (`full_aligned == 0`) in the `mi_free` routine. +#if !MI_TSAN +typedef union mi_page_flags_s { + uint8_t full_aligned; + struct { + uint8_t in_full : 1; + uint8_t has_aligned : 1; + } x; +} mi_page_flags_t; +#else +// under thread sanitizer, use a byte for each flag to suppress warning, issue #130 +typedef union mi_page_flags_s { + uint16_t full_aligned; + struct { + uint8_t in_full; + uint8_t has_aligned; + } x; +} mi_page_flags_t; +#endif + +// Thread free list. +// We use the bottom 2 bits of the pointer for mi_delayed_t flags +typedef uintptr_t mi_thread_free_t; + +// A page contains blocks of one specific size (`block_size`). +// Each page has three list of free blocks: +// `free` for blocks that can be allocated, +// `local_free` for freed blocks that are not yet available to `mi_malloc` +// `thread_free` for freed blocks by other threads +// The `local_free` and `thread_free` lists are migrated to the `free` list +// when it is exhausted. The separate `local_free` list is necessary to +// implement a monotonic heartbeat. The `thread_free` list is needed for +// avoiding atomic operations in the common case. +// +// +// `used - |thread_free|` == actual blocks that are in use (alive) +// `used - |thread_free| + |free| + |local_free| == capacity` +// +// We don't count `freed` (as |free|) but use `used` to reduce +// the number of memory accesses in the `mi_page_all_free` function(s). +// +// Notes: +// - Access is optimized for `mi_free` and `mi_page_alloc` (in `alloc.c`) +// - Using `uint16_t` does not seem to slow things down +// - The size is 8 words on 64-bit which helps the page index calculations +// (and 10 words on 32-bit, and encoded free lists add 2 words. Sizes 10 +// and 12 are still good for address calculation) +// - To limit the structure size, the `xblock_size` is 32-bits only; for +// blocks > MI_HUGE_BLOCK_SIZE the size is determined from the segment page size +// - `thread_free` uses the bottom bits as a delayed-free flags to optimize +// concurrent frees where only the first concurrent free adds to the owning +// heap `thread_delayed_free` list (see `alloc.c:mi_free_block_mt`). +// The invariant is that no-delayed-free is only set if there is +// at least one block that will be added, or as already been added, to +// the owning heap `thread_delayed_free` list. This guarantees that pages +// will be freed correctly even if only other threads free blocks. +typedef struct mi_page_s { + // "owned" by the segment + uint32_t slice_count; // slices in this page (0 if not a page) + uint32_t slice_offset; // distance from the actual page data slice (0 if a page) + uint8_t is_committed : 1; // `true` if the page virtual memory is committed + uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized + uint8_t use_qsbr : 1; // delay page freeing using qsbr + uint8_t tag : 4; // tag from the owning heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory + + // layout like this to optimize access in `mi_malloc` and `mi_free` + uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` + uint16_t reserved; // number of blocks reserved in memory + mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) + uint8_t free_is_zero : 1; // `true` if the blocks in the free list are zero initialized + uint8_t retire_expire : 7; // expiration count for retired blocks + + mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) + uint32_t xblock_size; // size available in each block (always `>0`) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary + #endif + + _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads + _Atomic(uintptr_t) xheap; + + struct mi_page_s* next; // next page owned by this thread with the same `block_size` + struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` + +#ifdef Py_GIL_DISABLED + struct llist_node qsbr_node; + uint64_t qsbr_goal; +#endif + + // 64-bit 9 words, 32-bit 12 words, (+2 for secure) + #if MI_INTPTR_SIZE==8 && !defined(Py_GIL_DISABLED) + uintptr_t padding[1]; + #endif +} mi_page_t; + + + +// ------------------------------------------------------ +// Mimalloc segments contain mimalloc pages +// ------------------------------------------------------ + +typedef enum mi_page_kind_e { + MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment + MI_PAGE_MEDIUM, // medium blocks go into medium pages inside a segment + MI_PAGE_LARGE, // larger blocks go into a page of just one block + MI_PAGE_HUGE, // huge blocks (> 16 MiB) are put into a single page in a single segment. +} mi_page_kind_t; + +typedef enum mi_segment_kind_e { + MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside. + MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside. +} mi_segment_kind_t; + +// ------------------------------------------------------ +// A segment holds a commit mask where a bit is set if +// the corresponding MI_COMMIT_SIZE area is committed. +// The MI_COMMIT_SIZE must be a multiple of the slice +// size. If it is equal we have the most fine grained +// decommit (but setting it higher can be more efficient). +// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will +// be committed in one go which can be set higher than +// MI_COMMIT_SIZE for efficiency (while the decommit mask +// is still tracked in fine-grained MI_COMMIT_SIZE chunks) +// ------------------------------------------------------ + +#define MI_MINIMAL_COMMIT_SIZE (1*MI_SEGMENT_SLICE_SIZE) +#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB +#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) +#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS +#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) + +#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS)) +#error "the segment size must be exactly divisible by the (commit size * size_t bits)" +#endif + +typedef struct mi_commit_mask_s { + size_t mask[MI_COMMIT_MASK_FIELD_COUNT]; +} mi_commit_mask_t; + +typedef mi_page_t mi_slice_t; +typedef int64_t mi_msecs_t; + + +// Memory can reside in arena's, direct OS allocated, or statically allocated. The memid keeps track of this. +typedef enum mi_memkind_e { + MI_MEM_NONE, // not allocated + MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) + MI_MEM_STATIC, // allocated in a static area and should not be freed (for arena meta data for example) + MI_MEM_OS, // allocated from the OS + MI_MEM_OS_HUGE, // allocated as huge os pages + MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) + MI_MEM_ARENA // allocated from an arena (the usual case) +} mi_memkind_t; + +static inline bool mi_memkind_is_os(mi_memkind_t memkind) { + return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP); +} + +typedef struct mi_memid_os_info { + void* base; // actual base address of the block (used for offset aligned allocations) + size_t alignment; // alignment at allocation +} mi_memid_os_info_t; + +typedef struct mi_memid_arena_info { + size_t block_index; // index in the arena + mi_arena_id_t id; // arena id (>= 1) + bool is_exclusive; // the arena can only be used for specific arena allocations +} mi_memid_arena_info_t; + +typedef struct mi_memid_s { + union { + mi_memid_os_info_t os; // only used for MI_MEM_OS + mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA + } mem; + bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large OS pages) + bool initially_committed;// `true` if the memory was originally allocated as committed + bool initially_zero; // `true` if the memory was originally zero initialized + mi_memkind_t memkind; +} mi_memid_t; + + +// Segments are large allocated memory blocks (8mb on 64 bit) from +// the OS. Inside segments we allocated fixed size _pages_ that +// contain blocks. +typedef struct mi_segment_s { + // constant fields + mi_memid_t memid; // memory id for arena allocation + bool allow_decommit; + bool allow_purge; + size_t segment_size; + + // segment fields + mi_msecs_t purge_expire; + mi_commit_mask_t purge_mask; + mi_commit_mask_t commit_mask; + + _Atomic(struct mi_segment_s*) abandoned_next; + + // from here is zero initialized + struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) + + size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t used; // count of pages in use + uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` + + size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT` + size_t segment_info_slices; // initial slices we are using segment info and possible guard pages. + + // layout like this to optimize access in `mi_free` + mi_segment_kind_t kind; + size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT` + _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment + + mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one more for huge blocks with large alignment +} mi_segment_t; + +typedef uintptr_t mi_tagged_segment_t; + +// Segments unowned by any thread are put in a shared pool +typedef struct mi_abandoned_pool_s { + // This is a list of visited abandoned pages that were full at the time. + // this list migrates to `abandoned` when that becomes NULL. The use of + // this list reduces contention and the rate at which segments are visited. + mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = NULL + + // The abandoned page list (tagged as it supports pop) + mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL + + // Maintain these for debug purposes (these counts may be a bit off) + mi_decl_cache_align _Atomic(size_t) abandoned_count; + mi_decl_cache_align _Atomic(size_t) abandoned_visited_count; + + // We also maintain a count of current readers of the abandoned list + // in order to prevent resetting/decommitting segment memory if it might + // still be read. + mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0 +} mi_abandoned_pool_t; + + +// ------------------------------------------------------ +// Heaps +// Provide first-class heaps to allocate from. +// A heap just owns a set of pages for allocation and +// can only be allocate/reallocate from the thread that created it. +// Freeing blocks can be done from any thread though. +// Per thread, the segments are shared among its heaps. +// Per thread, there is always a default heap that is +// used for allocation; it is initialized to statically +// point to an empty heap to avoid initialization checks +// in the fast path. +// ------------------------------------------------------ + +// Thread local data +typedef struct mi_tld_s mi_tld_t; + +// Pages of a certain block size are held in a queue. +typedef struct mi_page_queue_s { + mi_page_t* first; + mi_page_t* last; + size_t block_size; +} mi_page_queue_t; + +#define MI_BIN_FULL (MI_BIN_HUGE+1) + +// Random context +typedef struct mi_random_cxt_s { + uint32_t input[16]; + uint32_t output[16]; + int output_available; + bool weak; +} mi_random_ctx_t; + + +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows +#if (MI_PADDING) +typedef struct mi_padding_s { + uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) + uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) +} mi_padding_t; +#define MI_PADDING_SIZE (sizeof(mi_padding_t)) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#else +#define MI_PADDING_SIZE 0 +#define MI_PADDING_WSIZE 0 +#endif + +#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) + + +// A heap owns a set of pages. +struct mi_heap_s { + mi_tld_t* tld; + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") + _Atomic(mi_block_t*) thread_delayed_free; + mi_threadid_t thread_id; // thread this heap belongs too + mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0) + uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) + uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list + mi_random_ctx_t random; // random number context used for secure allocation + size_t page_count; // total number of pages in the `pages` queues. + size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) + size_t page_retired_max; // largest retired index into the `pages` array. + mi_heap_t* next; // list of heaps per thread + bool no_reclaim; // `true` if this heap should not reclaim abandoned pages + uint8_t tag; // custom identifier for this heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory + bool page_use_qsbr; // should freeing pages be delayed using QSBR +}; + + + +// ------------------------------------------------------ +// Debug +// ------------------------------------------------------ + +#if !defined(MI_DEBUG_UNINIT) +#define MI_DEBUG_UNINIT (0xD0) +#endif +#if !defined(MI_DEBUG_FREED) +#define MI_DEBUG_FREED (0xDF) +#endif +#if !defined(MI_DEBUG_PADDING) +#define MI_DEBUG_PADDING (0xDE) +#endif + +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + +// ------------------------------------------------------ +// Statistics +// ------------------------------------------------------ + +#ifndef MI_STAT +#if (MI_DEBUG>0) +#define MI_STAT 2 +#else +#define MI_STAT 0 +#endif +#endif + +typedef struct mi_stat_count_s { + int64_t allocated; + int64_t freed; + int64_t peak; + int64_t current; +} mi_stat_count_t; + +typedef struct mi_stat_counter_s { + int64_t total; + int64_t count; +} mi_stat_counter_t; + +typedef struct mi_stats_s { + mi_stat_count_t segments; + mi_stat_count_t pages; + mi_stat_count_t reserved; + mi_stat_count_t committed; + mi_stat_count_t reset; + mi_stat_count_t purged; + mi_stat_count_t page_committed; + mi_stat_count_t segments_abandoned; + mi_stat_count_t pages_abandoned; + mi_stat_count_t threads; + mi_stat_count_t normal; + mi_stat_count_t huge; + mi_stat_count_t large; + mi_stat_count_t malloc; + mi_stat_count_t segments_cache; + mi_stat_counter_t pages_extended; + mi_stat_counter_t mmap_calls; + mi_stat_counter_t commit_calls; + mi_stat_counter_t reset_calls; + mi_stat_counter_t purge_calls; + mi_stat_counter_t page_no_retire; + mi_stat_counter_t searches; + mi_stat_counter_t normal_count; + mi_stat_counter_t huge_count; + mi_stat_counter_t large_count; +#if MI_STAT>1 + mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; +#endif +} mi_stats_t; + + +void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); +void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); +void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); + +#if (MI_STAT) +#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) +#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) +#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) +#else +#define mi_stat_increase(stat,amount) (void)0 +#define mi_stat_decrease(stat,amount) (void)0 +#define mi_stat_counter_increase(stat,amount) (void)0 +#endif + +#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) + +// ------------------------------------------------------ +// Thread Local data +// ------------------------------------------------------ + +// A "span" is is an available range of slices. The span queues keep +// track of slice spans of at most the given `slice_count` (but more than the previous size class). +typedef struct mi_span_queue_s { + mi_slice_t* first; + mi_slice_t* last; + size_t slice_count; +} mi_span_queue_t; + +#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT) + +// OS thread local data +typedef struct mi_os_tld_s { + size_t region_idx; // start point for next allocation + mi_stats_t* stats; // points to tld stats +} mi_os_tld_t; + + +// Segments thread local data +typedef struct mi_segments_tld_s { + mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments + size_t count; // current number of segments; + size_t peak_count; // peak number of segments + size_t current_size; // current size of all segments + size_t peak_size; // peak size of all segments + mi_stats_t* stats; // points to tld stats + mi_os_tld_t* os; // points to os stats + mi_abandoned_pool_t* abandoned; // pool of abandoned segments +} mi_segments_tld_t; + +// Thread local data +struct mi_tld_s { + unsigned long long heartbeat; // monotonic heartbeat count + bool recurse; // true if deferred was called; used to prevent infinite recursion. + mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) + mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) + mi_segments_tld_t segments; // segment tld + mi_os_tld_t os; // os tld + mi_stats_t stats; // statistics +}; + +#endif diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index f222d485e0b54b..f5bf1205a82be9 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -657,14 +657,17 @@ struct _type_param { struct { identifier name; expr_ty bound; + expr_ty default_value; } TypeVar; struct { identifier name; + expr_ty default_value; } ParamSpec; struct { identifier name; + expr_ty default_value; } TypeVarTuple; } v; @@ -892,14 +895,15 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, expr_ty + default_value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 0c0d53f3e5d7e9..09ae95465495c0 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -2,6 +2,9 @@ #ifndef Py_INTERNAL_AST_STATE_H #define Py_INTERNAL_AST_STATE_H + +#include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -11,9 +14,8 @@ extern "C" { #endif struct ast_state { - int initialized; - int recursion_depth; - int recursion_limit; + _PyOnceFlag once; + int finalized; PyObject *AST_type; PyObject *Add_singleton; PyObject *Add_type; @@ -182,6 +184,7 @@ struct ast_state { PyObject *conversion; PyObject *ctx; PyObject *decorator_list; + PyObject *default_value; PyObject *defaults; PyObject *elt; PyObject *elts; diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 3966df70e2616f..507a5c03cbc792 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_ATEXIT_H #define Py_INTERNAL_ATEXIT_H + +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -15,7 +18,7 @@ extern "C" { typedef void (*atexit_callbackfunc)(void); struct _atexit_runtime_state { - PyThread_type_lock mutex; + PyMutex mutex; #define NEXITFUNCS 32 atexit_callbackfunc callbacks[NEXITFUNCS]; int ncallbacks; @@ -51,7 +54,7 @@ struct atexit_state { int callback_len; }; -// Export for '_xxinterpchannels' shared extension +// Export for '_interpchannels' shared extension PyAPI_FUNC(int) _Py_AtExit( PyInterpreterState *interp, atexit_datacallbackfunc func, diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h deleted file mode 100644 index 22ce971a64f3df..00000000000000 --- a/Include/internal/pycore_atomic.h +++ /dev/null @@ -1,557 +0,0 @@ -#ifndef Py_INTERNAL_ATOMIC_H -#define Py_INTERNAL_ATOMIC_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "pyconfig.h" // HAVE_STD_ATOMIC -#include "dynamic_annotations.h" // _Py_ANNOTATE_MEMORY_ORDER - -#ifdef HAVE_STD_ATOMIC -# include <stdatomic.h> // atomic_store_explicit() -#endif - - -#if defined(_MSC_VER) -# include <intrin.h> // _InterlockedExchange64() -# if defined(_M_IX86) || defined(_M_X64) -# include <immintrin.h> // _InterlockedExchange_HLEAcquire() -# endif -#endif - -/* This is modeled after the atomics interface from C1x, according to - * the draft at - * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. - * Operations and types are named the same except with a _Py_ prefix - * and have the same semantics. - * - * Beware, the implementations here are deep magic. - */ - -#if defined(HAVE_STD_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = memory_order_relaxed, - _Py_memory_order_acquire = memory_order_acquire, - _Py_memory_order_release = memory_order_release, - _Py_memory_order_acq_rel = memory_order_acq_rel, - _Py_memory_order_seq_cst = memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - atomic_uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - atomic_int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - atomic_store_explicit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - atomic_load_explicit(&((ATOMIC_VAL)->_value), ORDER) - -// Use builtin atomic operations in GCC >= 4.7 and clang -#elif defined(HAVE_BUILTIN_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = __ATOMIC_RELAXED, - _Py_memory_order_acquire = __ATOMIC_ACQUIRE, - _Py_memory_order_release = __ATOMIC_RELEASE, - _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, - _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - __atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - __atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_RELEASE), \ - __atomic_store_n(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER)) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_ACQUIRE \ - || (ORDER) == __ATOMIC_CONSUME), \ - __atomic_load_n(&((ATOMIC_VAL)->_value), ORDER)) - -/* Only support GCC (for expression statements) and x86 (for simple - * atomic semantics) and MSVC x86/x64/ARM */ -#elif defined(__GNUC__) && (defined(__i386__) || defined(__amd64)) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; - - -static __inline__ void -_Py_atomic_signal_fence(_Py_memory_order order) -{ - if (order != _Py_memory_order_relaxed) - __asm__ volatile("":::"memory"); -} - -static __inline__ void -_Py_atomic_thread_fence(_Py_memory_order order) -{ - if (order != _Py_memory_order_relaxed) - __asm__ volatile("mfence":::"memory"); -} - -/* Tell the race checker about this operation's effects. */ -static __inline__ void -_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order) -{ - (void)address; /* shut up -Wunused-parameter */ - switch(order) { - case _Py_memory_order_release: - case _Py_memory_order_acq_rel: - case _Py_memory_order_seq_cst: - _Py_ANNOTATE_HAPPENS_BEFORE(address); - break; - case _Py_memory_order_relaxed: - case _Py_memory_order_acquire: - break; - } - switch(order) { - case _Py_memory_order_acquire: - case _Py_memory_order_acq_rel: - case _Py_memory_order_seq_cst: - _Py_ANNOTATE_HAPPENS_AFTER(address); - break; - case _Py_memory_order_relaxed: - case _Py_memory_order_release: - break; - } -} - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - __extension__ ({ \ - __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ - __typeof__(atomic_val->_value) new_val = NEW_VAL;\ - volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \ - _Py_memory_order order = ORDER; \ - _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ - \ - /* Perform the operation. */ \ - _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ - switch(order) { \ - case _Py_memory_order_release: \ - _Py_atomic_signal_fence(_Py_memory_order_release); \ - /* fallthrough */ \ - case _Py_memory_order_relaxed: \ - *volatile_data = new_val; \ - break; \ - \ - case _Py_memory_order_acquire: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - __asm__ volatile("xchg %0, %1" \ - : "+r"(new_val) \ - : "m"(atomic_val->_value) \ - : "memory"); \ - break; \ - } \ - _Py_ANNOTATE_IGNORE_WRITES_END(); \ - }) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - __extension__ ({ \ - __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ - __typeof__(atomic_val->_value) result; \ - volatile __typeof__(result) *volatile_data = &atomic_val->_value; \ - _Py_memory_order order = ORDER; \ - _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ - \ - /* Perform the operation. */ \ - _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ - switch(order) { \ - case _Py_memory_order_release: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - /* Loads on x86 are not releases by default, so need a */ \ - /* thread fence. */ \ - _Py_atomic_thread_fence(_Py_memory_order_release); \ - break; \ - default: \ - /* No fence */ \ - break; \ - } \ - result = *volatile_data; \ - switch(order) { \ - case _Py_memory_order_acquire: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - /* Loads on x86 are automatically acquire operations so */ \ - /* can get by with just a compiler fence. */ \ - _Py_atomic_signal_fence(_Py_memory_order_acquire); \ - break; \ - default: \ - /* No fence */ \ - break; \ - } \ - _Py_ANNOTATE_IGNORE_READS_END(); \ - result; \ - }) - -#elif defined(_MSC_VER) -/* _Interlocked* functions provide a full memory barrier and are therefore - enough for acq_rel and seq_cst. If the HLE variants aren't available - in hardware they will fall back to a full memory barrier as well. - - This might affect performance but likely only in some very specific and - hard to measure scenario. -*/ -#if defined(_M_IX86) || defined(_M_X64) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - volatile uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - volatile int _value; -} _Py_atomic_int; - - -#if defined(_M_X64) -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange64_HLEAcquire((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange64_HLERelease((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - default: \ - _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - } -#else -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); -#endif - -#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange_HLEAcquire((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange_HLERelease((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - default: \ - _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - } - -#if defined(_M_X64) -/* This has to be an intptr_t for now. - gil_created() uses -1 as a sentinel value, if this returns - a uintptr_t it will do an unsigned compare and crash -*/ -inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { - __int64 old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_HLEAcquire((volatile __int64*)value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_HLERelease((volatile __int64*)value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange64((volatile __int64*)value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) - -#else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) -#endif - -inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { - long old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange_HLEAcquire((volatile long*)value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange_HLERelease((volatile long*)value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange((volatile long*)value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit((ATOMIC_VAL), NEW_VAL, ORDER) } else { \ - _Py_atomic_store_32bit((ATOMIC_VAL), NEW_VAL, ORDER) } - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ( \ - sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit((ATOMIC_VAL), ORDER) : \ - _Py_atomic_load_32bit((ATOMIC_VAL), ORDER) \ - ) -#elif defined(_M_ARM) || defined(_M_ARM64) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - volatile uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - volatile int _value; -} _Py_atomic_int; - - -#if defined(_M_ARM64) -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange64_acq((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange64_rel((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - default: \ - _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - } -#else -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); -#endif - -#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange_acq((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange_rel((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - default: \ - _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - } - -#if defined(_M_ARM64) -/* This has to be an intptr_t for now. - gil_created() uses -1 as a sentinel value, if this returns - a uintptr_t it will do an unsigned compare and crash -*/ -inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { - uintptr_t old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_acq(value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_rel(value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange64(value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) - -#else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) -#endif - -inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { - int old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange_acq(value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange_rel(value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange(value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } else { \ - _Py_atomic_store_32bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ( \ - sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit((ATOMIC_VAL), (ORDER)) : \ - _Py_atomic_load_32bit((ATOMIC_VAL), (ORDER)) \ - ) -#endif -#else /* !gcc x86 !_msc_ver */ -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; -/* Fall back to other compilers and processors by assuming that simple - volatile accesses are atomic. This is false, so people should port - this. */ -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0) -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0) -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - ((ATOMIC_VAL)->_value = NEW_VAL) -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ((ATOMIC_VAL)->_value) -#endif - -/* Standardized shortcuts. */ -#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_seq_cst) -#define _Py_atomic_load(ATOMIC_VAL) \ - _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_seq_cst) - -/* Python-local extensions */ - -#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed) -#define _Py_atomic_load_relaxed(ATOMIC_VAL) \ - _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed) - -#ifdef __cplusplus -} -#endif -#endif /* Py_INTERNAL_ATOMIC_H */ diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h new file mode 100644 index 00000000000000..3db3aa3eb77879 --- /dev/null +++ b/Include/internal/pycore_backoff.h @@ -0,0 +1,145 @@ + +#ifndef Py_INTERNAL_BACKOFF_H +#define Py_INTERNAL_BACKOFF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> + + +typedef struct { + union { + struct { + uint16_t backoff : 4; + uint16_t value : 12; + }; + uint16_t as_counter; // For printf("%#x", ...) + }; +} _Py_BackoffCounter; + + +/* 16-bit countdown counters using exponential backoff. + + These are used by the adaptive specializer to count down until + it is time to specialize an instruction. If specialization fails + the counter is reset using exponential backoff. + + Another use is for the Tier 2 optimizer to decide when to create + a new Tier 2 trace (executor). Again, exponential backoff is used. + + The 16-bit counter is structured as a 12-bit unsigned 'value' + and a 4-bit 'backoff' field. When resetting the counter, the + backoff field is incremented (until it reaches a limit) and the + value is set to a bit mask representing the value 2**backoff - 1. + The maximum backoff is 12 (the number of value bits). + + There is an exceptional value which must not be updated, 0xFFFF. +*/ + +#define UNREACHABLE_BACKOFF 0xFFFF + +static inline bool +is_unreachable_backoff_counter(_Py_BackoffCounter counter) +{ + return counter.as_counter == UNREACHABLE_BACKOFF; +} + +static inline _Py_BackoffCounter +make_backoff_counter(uint16_t value, uint16_t backoff) +{ + assert(backoff <= 15); + assert(value <= 0xFFF); + _Py_BackoffCounter result; + result.value = value; + result.backoff = backoff; + return result; +} + +static inline _Py_BackoffCounter +forge_backoff_counter(uint16_t counter) +{ + _Py_BackoffCounter result; + result.as_counter = counter; + return result; +} + +static inline _Py_BackoffCounter +restart_backoff_counter(_Py_BackoffCounter counter) +{ + assert(!is_unreachable_backoff_counter(counter)); + if (counter.backoff < 12) { + return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1); + } + else { + return make_backoff_counter((1 << 12) - 1, 12); + } +} + +static inline _Py_BackoffCounter +pause_backoff_counter(_Py_BackoffCounter counter) +{ + return make_backoff_counter(counter.value | 1, counter.backoff); +} + +static inline _Py_BackoffCounter +advance_backoff_counter(_Py_BackoffCounter counter) +{ + if (!is_unreachable_backoff_counter(counter)) { + return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff); + } + else { + return counter; + } +} + +static inline bool +backoff_counter_triggers(_Py_BackoffCounter counter) +{ + return counter.value == 0; +} + +/* Initial JUMP_BACKWARD counter. + * This determines when we create a trace for a loop. +* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ +#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_BACKOFF 4 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(void) +{ + return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, + JUMP_BACKWARD_INITIAL_BACKOFF); +} + +/* Initial exit temperature. + * Must be larger than ADAPTIVE_COOLDOWN_VALUE, + * otherwise when a side exit warms up we may construct + * a new trace before the Tier 1 code has properly re-specialized. + * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ +#define SIDE_EXIT_INITIAL_VALUE 64 +#define SIDE_EXIT_INITIAL_BACKOFF 6 + +static inline _Py_BackoffCounter +initial_temperature_backoff_counter(void) +{ + return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, + SIDE_EXIT_INITIAL_BACKOFF); +} + +/* Unreachable backoff counter. */ +static inline _Py_BackoffCounter +initial_unreachable_backoff_counter(void) +{ + return forge_backoff_counter(UNREACHABLE_BACKOFF); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BACKOFF_H */ diff --git a/Include/internal/pycore_brc.h b/Include/internal/pycore_brc.h new file mode 100644 index 00000000000000..3453d83b57ca97 --- /dev/null +++ b/Include/internal/pycore_brc.h @@ -0,0 +1,74 @@ +#ifndef Py_INTERNAL_BRC_H +#define Py_INTERNAL_BRC_H + +#include <stdint.h> +#include "pycore_llist.h" // struct llist_node +#include "pycore_lock.h" // PyMutex +#include "pycore_object_stack.h" // _PyObjectStack + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// Prime number to avoid correlations with memory addresses. +#define _Py_BRC_NUM_BUCKETS 257 + +// Hash table bucket +struct _brc_bucket { + // Mutex protects both the bucket and thread state queues in this bucket. + PyMutex mutex; + + // Linked list of _PyThreadStateImpl objects hashed to this bucket. + struct llist_node root; +}; + +// Per-interpreter biased reference counting state +struct _brc_state { + // Hash table of thread states by thread-id. Thread states within a bucket + // are chained using a doubly-linked list. + struct _brc_bucket table[_Py_BRC_NUM_BUCKETS]; +}; + +// Per-thread biased reference counting state +struct _brc_thread_state { + // Linked-list of thread states per hash bucket + struct llist_node bucket_node; + + // Thread-id as determined by _PyThread_Id() + uintptr_t tid; + + // Objects with refcounts to be merged (protected by bucket mutex) + _PyObjectStack objects_to_merge; + + // Local stack of objects to be merged (not accessed by other threads) + _PyObjectStack local_objects_to_merge; +}; + +// Initialize/finalize the per-thread biased reference counting state +void _Py_brc_init_thread(PyThreadState *tstate); +void _Py_brc_remove_thread(PyThreadState *tstate); + +// Initialize per-interpreter state +void _Py_brc_init_state(PyInterpreterState *interp); + +void _Py_brc_after_fork(PyInterpreterState *interp); + +// Enqueues an object to be merged by it's owning thread (tid). This +// steals a reference to the object. +void _Py_brc_queue_object(PyObject *ob); + +// Merge the refcounts of queued objects for the current thread. +void _Py_brc_merge_refcounts(PyThreadState *tstate); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BRC_H */ diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 11e8ab20e91367..059dc2599bbd77 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -26,14 +26,23 @@ extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); -extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg); -extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); +extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 8846155b38defb..49f5c3322de267 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_pystate.h" // _PyThreadState_GET() /* Suggested size (number of positional arguments) for arrays of PyObject* @@ -31,18 +30,6 @@ PyAPI_FUNC(PyObject*) _Py_CheckFunctionResult( PyObject *result, const char *where); -/* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) - format to a Python dictionary ("kwargs" dict). - - The type of kwnames keys is not checked. The final function getting - arguments is responsible to check if all keys are strings, for example using - PyArg_ParseTupleAndKeywords() or PyArg_ValidateKeywordArguments(). - - Duplicate keys are merged using the last value. If duplicate keys must raise - an exception, the caller is responsible to implement an explicit keys on - kwnames. */ -extern PyObject* _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); - extern PyObject* _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, @@ -75,13 +62,6 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -/* Like PyObject_CallMethod(), but expect a _Py_Identifier* - as the method name. */ -extern PyObject* _PyObject_CallMethodId( - PyObject *obj, - _Py_Identifier *name, - const char *format, ...); - extern PyObject* _PyObject_CallMethodIdObjArgs( PyObject *obj, _Py_Identifier *name, diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h new file mode 100644 index 00000000000000..27f67d57b2fb79 --- /dev/null +++ b/Include/internal/pycore_cell.h @@ -0,0 +1,48 @@ +#ifndef Py_INTERNAL_CELL_H +#define Py_INTERNAL_CELL_H + +#include "pycore_critical_section.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Sets the cell contents to `value` and return previous contents. Steals a +// reference to `value`. +static inline PyObject * +PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value; + Py_BEGIN_CRITICAL_SECTION(cell); + old_value = cell->ob_ref; + cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); + return old_value; +} + +static inline void +PyCell_SetTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value = PyCell_SwapTakeRef(cell, value); + Py_XDECREF(old_value); +} + +// Gets the cell contents. Returns a new reference. +static inline PyObject * +PyCell_GetRef(PyCellObject *cell) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(cell); + res = Py_XNewRef(cell->ob_ref); + Py_END_CRITICAL_SECTION(); + return res; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CELL_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 312d67ee0b37aa..b472d5d446b246 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "dynamic_annotations.h" // _Py_ANNOTATE_RWLOCK_CREATE + #include "pycore_interp.h" // PyInterpreterState.eval_frame #include "pycore_pystate.h" // _PyThreadState_GET() @@ -20,6 +22,8 @@ PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyO extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); + // Helper to look up a builtin object // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); @@ -37,32 +41,24 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); #endif extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); -extern void _PyEval_FiniState(struct _ceval_state *ceval); -extern void _PyEval_SignalReceived(PyInterpreterState *interp); +extern void _PyEval_InitState(PyInterpreterState *); +extern void _PyEval_SignalReceived(void); // bitwise flags: #define _Py_PENDING_MAINTHREADONLY 1 #define _Py_PENDING_RAWFREE 2 +typedef int _Py_add_pending_call_result; +#define _Py_ADD_PENDING_SUCCESS 0 +#define _Py_ADD_PENDING_FULL -1 + // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(int) _PyEval_AddPendingCall( +PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall( PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags); -typedef int (*_Py_simple_func)(void *); -extern int _Py_CallInInterpreter( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); -extern int _Py_CallInInterpreterAndRawFree( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); - -extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -107,10 +103,12 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); +extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* @@ -130,11 +128,56 @@ _PyEval_Vector(PyThreadState *tstate, PyObject *kwnames); extern int _PyEval_ThreadsInitialized(void); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); +extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); + +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, + int final_release); + +#ifdef Py_GIL_DISABLED +// Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or +// enabled, respectively. +// +// The enabled state of the GIL will not change while one or more threads are +// attached. +static inline int +_PyEval_IsGILEnabled(PyThreadState *tstate) +{ + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + return _Py_atomic_load_int_relaxed(&gil->enabled) != 0; +} + +// Enable or disable the GIL used by the interpreter that owns tstate, which +// must be the current thread. This may affect other interpreters, if the GIL +// is shared. All three functions will be no-ops (and return 0) if the +// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT. +// +// Every call to _PyEval_EnableGILTransient() must be paired with exactly one +// call to either _PyEval_EnableGILPermanent() or +// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL() +// must only be called while the GIL is enabled from a call to +// _PyEval_EnableGILTransient(). +// +// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the +// GIL was already enabled, whether transiently or permanently. The caller will +// hold the GIL upon return. +// +// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL +// (which must already be enabled), or 0 if it was already permanently +// enabled. Once _PyEval_EnableGILPermanent() has been called once, all +// subsequent calls to any of the three functions will be no-ops. +// +// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was +// kept enabled because of another request, whether transient or permanent. +// +// All three functions must be called by an attached thread (this implies that +// if the GIL is enabled, the current thread must hold it). +extern int _PyEval_EnableGILTransient(PyThreadState *tstate); +extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); +extern int _PyEval_DisableGIL(PyThreadState *state); +#endif extern void _PyEval_DeactivateOpCache(void); @@ -145,12 +188,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 + return (tstate->c_recursion_remaining-- < 0 || (tstate->c_recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->c_recursion_remaining-- <= 0; + return tstate->c_recursion_remaining-- < 0; } #endif @@ -168,6 +211,11 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); } +static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) { + assert(tstate->c_recursion_remaining > 0); + tstate->c_recursion_remaining--; +} + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); @@ -184,60 +232,79 @@ static inline void _Py_LeaveRecursiveCall(void) { extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); -extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ -extern int _Py_HandlePending(PyThreadState *tstate); +PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); extern PyObject * _PyEval_GetFrameLocals(void); -extern const binaryfunc _PyEval_BinaryOps[]; -int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); -void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); -void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); -void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); -PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); -PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); -void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); - - -#define _PY_GIL_DROP_REQUEST_BIT 0 -#define _PY_SIGNALS_PENDING_BIT 1 -#define _PY_CALLS_TO_DO_BIT 2 -#define _PY_ASYNC_EXCEPTION_BIT 3 -#define _PY_GC_SCHEDULED_BIT 4 +typedef PyObject *(*conversion_func)(PyObject *); + +PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[]; +PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[]; + +typedef struct _special_method { + PyObject *name; + const char *error; +} _Py_SpecialMethod; + +PyAPI_DATA(const _Py_SpecialMethod) _Py_SpecialMethods[]; + +PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); +PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); +PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); +PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); +PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); + +PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); + + +/* Bits that can be set in PyThreadState.eval_breaker */ +#define _PY_GIL_DROP_REQUEST_BIT (1U << 0) +#define _PY_SIGNALS_PENDING_BIT (1U << 1) +#define _PY_CALLS_TO_DO_BIT (1U << 2) +#define _PY_ASYNC_EXCEPTION_BIT (1U << 3) +#define _PY_GC_SCHEDULED_BIT (1U << 4) +#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) +#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void -_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +_Py_set_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) { - assert(set == 0 || set == 1); - uintptr_t to_set = set << bit; - uintptr_t mask = ((uintptr_t)1) << bit; - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - if ((old & mask) == to_set) { - return; - } - uintptr_t new; - do { - new = (old & ~mask) | to_set; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + _Py_atomic_or_uintptr(&tstate->eval_breaker, bit); +} + +static inline void +_Py_unset_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) +{ + _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit); } -static inline bool -_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +static inline int +_Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) { - return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); + uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + return (b & bit) != 0; } +// Free-threaded builds use these functions to set or unset a bit on all +// threads in the given interpreter. +void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1717ec4f41c36b..009a1ea41eb985 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -8,33 +8,64 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_gil.h" // struct _gil_runtime_state typedef int (*_Py_pending_call_func)(void *); +struct _pending_call { + _Py_pending_call_func func; + void *arg; + int flags; +}; + +#define PENDINGCALLSARRAYSIZE 300 + +#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE +/* For interpreter-level pending calls, we want to avoid spending too + much time on pending calls in any one thread, so we apply a limit. */ +#if MAXPENDINGCALLS > 100 +# define MAXPENDINGCALLSLOOP 100 +#else +# define MAXPENDINGCALLSLOOP MAXPENDINGCALLS +#endif + +/* We keep the number small to preserve as much compatibility + as possible with earlier versions. */ +#define MAXPENDINGCALLS_MAIN 32 +/* For the main thread, we want to make sure all pending calls are + run at once, for the sake of prompt signal handling. This is + unlikely to cause any problems since there should be very few + pending calls for the main thread. */ +#define MAXPENDINGCALLSLOOP_MAIN 0 + struct _pending_calls { - int busy; - PyThread_type_lock lock; + PyThreadState *handling_thread; + PyMutex mutex; /* Request for running pending calls. */ - int32_t calls_to_do; -#define NPENDINGCALLS 32 - struct _pending_call { - _Py_pending_call_func func; - void *arg; - int flags; - } calls[NPENDINGCALLS]; + int32_t npending; + /* The maximum allowed number of pending calls. + If the queue fills up to this point then _PyEval_AddPendingCall() + will return _Py_ADD_PENDING_FULL. */ + int32_t max; + /* We don't want a flood of pending calls to interrupt any one thread + for too long, so we keep a limit on the number handled per pass. + A value of 0 means there is no limit (other than the maximum + size of the list of pending calls). */ + int32_t maxloop; + struct _pending_call calls[PENDINGCALLSARRAYSIZE]; int first; - int last; + int next; }; + typedef enum { PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed } perf_status_t; - #ifdef PY_HAVE_PERF_TRAMPOLINE struct code_arena_st; @@ -44,30 +75,41 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif + struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; FILE *map_file; + Py_ssize_t persist_after_fork; #else int _not_used; #endif } perf; /* Pending calls to be made only on the main thread. */ + // The signal machinery falls back on this + // so it must be especially stable and efficient. + // For example, we use a preallocated array + // for the list of pending calls. struct _pending_calls pending_mainthread; + PyMutex sys_trace_profile_mutex; }; + #ifdef PY_HAVE_PERF_TRAMPOLINE # define _PyEval_RUNTIME_PERF_INIT \ { \ .status = PERF_STATUS_NO_INIT, \ .extra_code_index = -1, \ + .persist_after_fork = 0, \ } #else # define _PyEval_RUNTIME_PERF_INIT {0} @@ -75,13 +117,10 @@ struct _ceval_runtime_state { struct _ceval_state { - /* This single variable consolidates all requests to break out of - * the fast path in the eval loop. - * It is by far the hottest field in this struct and - * should be placed at the beginning. */ - uintptr_t eval_breaker; - /* Avoid false sharing */ - int64_t padding[7]; + /* This variable holds the global instrumentation version. When a thread is + running, this value is overlaid onto PyThreadState.eval_breaker so that + changes in the instrumentation version will trigger the eval breaker. */ + uintptr_t instrumentation_version; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index d31d8363d771ca..cd77fe408fc5d9 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,73 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_stackref.h" // _PyStackRef +#include "pycore_lock.h" // PyMutex +#include "pycore_backoff.h" // _Py_BackoffCounter + + +/* Each instruction in a code object is a fixed-width value, + * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG + * opcode allows for larger values but the current limit is 3 uses + * of EXTENDED_ARG (see Python/compile.c), for a maximum + * 32-bit value. This aligns with the note in Python/compile.c + * (compiler_addop_i_line) indicating that the max oparg value is + * 2**32 - 1, rather than INT_MAX. + */ + +typedef union { + uint16_t cache; + struct { + uint8_t code; + uint8_t arg; + } op; + _Py_BackoffCounter counter; // First cache entry of specializable op +} _Py_CODEUNIT; + +#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + + +/* These macros only remain defined for compatibility. */ +#define _Py_OPCODE(word) ((word).op.code) +#define _Py_OPARG(word) ((word).op.arg) + +static inline _Py_CODEUNIT +_py_make_codeunit(uint8_t opcode, uint8_t oparg) +{ + // No designated initialisers because of C++ compat + _Py_CODEUNIT word; + word.op.code = opcode; + word.op.arg = oparg; + return word; +} + +static inline void +_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) +{ + word->op.code = opcode; +} + +#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) +#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) + + +// We hide some of the newer PyCodeObject fields behind macros. +// This helps with backporting certain changes to 3.12. +#define _PyCode_HAS_EXECUTORS(CODE) \ + (CODE->co_executors != NULL) +#define _PyCode_HAS_INSTRUMENTATION(CODE) \ + (CODE->_co_instrumentation_version > 0) + +struct _py_code_state { + PyMutex mutex; + // Interned constants from code objects. Used by the free-threaded build. + struct _Py_hashtable_t *constants; +}; + +extern PyStatus _PyCode_Init(PyInterpreterState *interp); +extern void _PyCode_Fini(PyInterpreterState *interp); + #define CODE_MAX_WATCHERS 8 /* PEP 659 @@ -22,7 +89,7 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t module_keys_version; uint16_t builtin_keys_version; uint16_t index; @@ -31,46 +98,49 @@ typedef struct { #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; uint16_t index; } _PyAttrCache; typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t type_version[2]; - uint16_t keys_version[2]; + union { + uint16_t keys_version[2]; + uint16_t dict_offset; + }; uint16_t descr[4]; } _PyLoadMethodCache; @@ -81,37 +151,43 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t func_version[2]; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySendCache; #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; } _PyToBoolCache; #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyContainsOpCache; + +#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) + // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; @@ -236,41 +312,44 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); +#ifdef Py_GIL_DISABLED +// gh-115999 tracks progress on addressing this. +#define ENABLE_SPECIALIZATION 0 +#else #define ENABLE_SPECIALIZATION 1 +#endif /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, +extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, +extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container, _Py_CODEUNIT *instr); -extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, +extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, +extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals); -extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, +extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals); +extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, +extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr); - -/* Finalizer function for static codeobjects used in deepfreeze.py */ -extern void _PyStaticCode_Fini(PyCodeObject *co); -/* Function to intern strings of codeobjects and quicken the bytecode */ -extern int _PyStaticCode_Init(PyCodeObject *co); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); #ifdef Py_STATS +#include "pycore_bitutils.h" // _Py_bit_length + #define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) #define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) #define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) @@ -283,8 +362,16 @@ extern int _PyStaticCode_Init(PyCodeObject *co); do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define UOP_EXE_INC(opname) do { if (_Py_stats) _Py_stats->optimization_stats.opcode[opname].execution_count++; } while (0) +#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + if (lastuop && _Py_stats) { \ + _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) #define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) #define OPT_HIST(length, name) \ do { \ if (_Py_stats) { \ @@ -293,6 +380,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); _Py_stats->optimization_stats.name[bucket]++; \ } \ } while (0) +#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -308,9 +396,12 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) #define GC_STAT_ADD(gen, name, n) ((void)0) #define OPT_STAT_INC(name) ((void)0) -#define UOP_EXE_INC(opname) ((void)0) +#define UOP_STAT_INC(opname, name) ((void)0) +#define UOP_PAIR_INC(uopcode, lastuop) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) +#define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. @@ -392,45 +483,43 @@ write_varint(uint8_t *ptr, unsigned int val) val >>= 6; written++; } - *ptr = val; + *ptr = (uint8_t)val; return written; } static inline int write_signed_varint(uint8_t *ptr, int val) { + unsigned int uval; if (val < 0) { - val = ((-val)<<1) | 1; + // (unsigned int)(-val) has an undefined behavior for INT_MIN + uval = ((0 - (unsigned int)val) << 1) | 1; } else { - val = val << 1; + uval = (unsigned int)val << 1; } - return write_varint(ptr, val); + return write_varint(ptr, uval); } static inline int write_location_entry_start(uint8_t *ptr, int code, int length) { assert((code & 15) == code); - *ptr = 128 | (code << 3) | (length - 1); + *ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1); return 1; } /** Counters * The first 16-bit value in each inline cache is a counter. - * When counting misses, the counter is treated as a simple unsigned value. * * When counting executions until the next specialization attempt, * exponential backoff is used to reduce the number of specialization failures. - * The high 12 bits store the counter, the low 4 bits store the backoff exponent. - * On a specialization failure, the backoff exponent is incremented and the - * counter set to (2**backoff - 1). - * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023. + * See pycore_backoff.h for more details. + * On a specialization failure, the backoff counter is restarted. */ -/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ -#define ADAPTIVE_BACKOFF_BITS 4 +#include "pycore_backoff.h" // A value of 1 means that we attempt to specialize the *second* time each // instruction is executed. Executing twice is a much better indicator of @@ -448,36 +537,30 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_COOLDOWN_VALUE 52 #define ADAPTIVE_COOLDOWN_BACKOFF 0 -#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) +// Can't assert this in pycore_backoff.h because of header order dependencies +static_assert(SIDE_EXIT_INITIAL_VALUE > ADAPTIVE_COOLDOWN_VALUE, + "Cold exit value should be larger than adaptive cooldown value"); - -static inline uint16_t -adaptive_counter_bits(int value, int backoff) { - return (value << ADAPTIVE_BACKOFF_BITS) | - (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); +static inline _Py_BackoffCounter +adaptive_counter_bits(uint16_t value, uint16_t backoff) { + return make_backoff_counter(value, backoff); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_warmup(void) { return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, ADAPTIVE_WARMUP_BACKOFF); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_cooldown(void) { return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, ADAPTIVE_COOLDOWN_BACKOFF); } -static inline uint16_t -adaptive_counter_backoff(uint16_t counter) { - unsigned int backoff = counter & ((1<<ADAPTIVE_BACKOFF_BITS)-1); - backoff++; - if (backoff > MAX_BACKOFF_VALUE) { - backoff = MAX_BACKOFF_VALUE; - } - unsigned int value = (1 << backoff) - 1; - return adaptive_counter_bits(value, backoff); +static inline _Py_BackoffCounter +adaptive_counter_backoff(_Py_BackoffCounter counter) { + return restart_backoff_counter(counter); } diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2a7151d50ade7..5e2d5c5ce9d868 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -8,6 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + +/* Initialize codecs-related state for the given interpreter, including + registering the first codec search function. Must be called before any other + PyCodec-related functions, and while only one thread is active. */ +extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp); + +/* Finalize codecs-related state for the given interpreter. No PyCodec-related + functions other than PyCodec_Unregister() may be called after this. */ +extern void _PyCodec_Fini(PyInterpreterState *interp); + extern PyObject* _PyCodec_Lookup(const char *encoding); /* Text codec specific encoding and decoding API. @@ -48,6 +59,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder( PyObject *codec_info, const char *errors); +// Per-interpreter state used by codecs.c. +struct codecs_state { + // A list of callable objects used to search for codecs. + PyObject *search_path; + + // A dict mapping codec names to codecs returned from a callable in + // search_path. + PyObject *search_cache; + + // A dict mapping error handling strategies to functions to implement them. + PyObject *error_registry; + +#ifdef Py_GIL_DISABLED + // Used to safely delete a specific item from search_path. + PyMutex search_path_mutex; +#endif + + // Whether or not the rest of the state is initialized. + int initialized; +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a5a7146f5ee917..325243e6a64e1f 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_instruction_sequence.h" + struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h @@ -27,7 +30,7 @@ extern int _PyCompile_AstOptimize( int optimize, struct _arena *arena); -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; +struct _Py_SourceLocation; extern int _PyAST_Optimize( struct _mod *, @@ -35,37 +38,6 @@ extern int _PyAST_Optimize( int optimize, int ff_features); -typedef struct { - int h_label; - int h_startdepth; - int h_preserve_lasti; -} _PyCompile_ExceptHandlerInfo; - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - _PyCompile_ExceptHandlerInfo i_except_handler_info; - - /* Used by the assembler */ - int i_target; - int i_offset; -} _PyCompile_Instruction; - -typedef struct { - _PyCompile_Instruction *s_instrs; - int s_allocated; - int s_used; - - int *s_labelmap; /* label id --> instr offset */ - int s_labelmap_size; - int s_next_free_label; /* next free label id */ -} _PyCompile_InstructionSequence; - -int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); -int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, - int opcode, int oparg, - _PyCompilerSrcLocation loc); typedef struct { PyObject *u_name; @@ -102,6 +74,11 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); + +// Export for '_opcode' extension module +PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); +PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); + /* Access compiler internals for unit testing */ // Export for '_testinternalcapi' shared extension @@ -115,12 +92,6 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( - PyObject *instructions, - PyObject *consts, - int nlocals); - // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h index a6fee9d23f3a9f..54713536eedc46 100644 --- a/Include/internal/pycore_complexobject.h +++ b/Include/internal/pycore_complexobject.h @@ -10,16 +10,6 @@ extern "C" { #include "pycore_unicodeobject.h" // _PyUnicodeWriter -// Operations on complex numbers. -// Export functions for 'cmath' shared extension. -PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); -PyAPI_FUNC(double) _Py_c_abs(Py_complex); - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ extern int _PyComplex_FormatAdvancedWriter( diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 34c21aaad43197..ee9533484e8048 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -35,14 +35,14 @@ #include <windows.h> // CRITICAL_SECTION /* options */ -/* non-emulated condition variables are provided for those that want - * to target Windows Vista. Modify this macro to enable them. +/* emulated condition variables are provided for those that want + * to target Windows XP or earlier. Modify this macro to enable them. */ #ifndef _PY_EMULATED_WIN_CV -#define _PY_EMULATED_WIN_CV 1 /* use emulated condition variables */ +#define _PY_EMULATED_WIN_CV 0 /* use non-emulated condition variables */ #endif -/* fall back to emulation if not targeting Vista */ +/* fall back to emulation if targeting earlier than Vista */ #if !defined NTDDI_VISTA || NTDDI_VERSION < NTDDI_VISTA #undef _PY_EMULATED_WIN_CV #define _PY_EMULATED_WIN_CV 1 @@ -77,7 +77,7 @@ typedef struct _PyCOND_T #else /* !_PY_EMULATED_WIN_CV */ -/* Use native Win7 primitives if build target is Win7 or higher */ +/* Use native Windows primitives if build target is Vista or higher */ /* SRWLOCK is faster and better than CriticalSection */ typedef SRWLOCK PyMUTEX_T; diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ec884e9e0f55a9..ae5c47f195eb7f 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,6 +5,7 @@ # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject @@ -13,7 +14,6 @@ extern PyTypeObject _PyContextTokenMissing_Type; /* runtime lifecycle */ PyStatus _PyContext_Init(PyInterpreterState *); -void _PyContext_Fini(PyInterpreterState *); /* other API */ @@ -22,23 +22,6 @@ typedef struct { PyObject_HEAD } _PyContextTokenMissing; -#ifndef WITH_FREELISTS -// without freelists -# define PyContext_MAXFREELIST 0 -#endif - -#ifndef PyContext_MAXFREELIST -# define PyContext_MAXFREELIST 255 -#endif - -struct _Py_context_state { -#if PyContext_MAXFREELIST > 0 - // List of free PyContext objects - PyContext *freelist; - int numfree; -#endif -}; - struct _pycontextobject { PyObject_HEAD PyContext *ctx_prev; diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h new file mode 100644 index 00000000000000..78cd0d54972660 --- /dev/null +++ b/Include/internal/pycore_critical_section.h @@ -0,0 +1,233 @@ +#ifndef Py_INTERNAL_CRITICAL_SECTION_H +#define Py_INTERNAL_CRITICAL_SECTION_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_lock.h" // PyMutex +#include "pycore_pystate.h" // _PyThreadState_GET() +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +// Tagged pointers to critical sections use the two least significant bits to +// mark if the pointed-to critical section is inactive and whether it is a +// PyCriticalSection2 object. +#define _Py_CRITICAL_SECTION_INACTIVE 0x1 +#define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2 +#define _Py_CRITICAL_SECTION_MASK 0x3 + +#ifdef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ + { \ + PyCriticalSection _py_cs; \ + _PyCriticalSection_BeginMutex(&_py_cs, mutex) + +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + +// Specialized version of critical section locking to safely use +// PySequence_Fast APIs without the GIL. For performance, the argument *to* +// PySequence_Fast() is provided to the macro, not the *result* of +// PySequence_Fast(), which would require an extra test to determine if the +// lock must be acquired. +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) \ + { \ + PyObject *_orig_seq = _PyObject_CAST(original); \ + const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ + PyCriticalSection _cs; \ + if (_should_lock_cs) { \ + _PyCriticalSection_Begin(&_cs, _orig_seq); \ + } + +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ + if (_should_lock_cs) { \ + PyCriticalSection_End(&_cs); \ + } \ + } + +// Asserts that the mutex is locked. The mutex must be held by the +// top-most critical section otherwise there's the possibility +// that the mutex would be swalled out in some code paths. +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ + _PyCriticalSection_AssertHeld(mutex) + +// Asserts that the mutex for the given object is locked. The mutex must +// be held by the top-most critical section otherwise there's the +// possibility that the mutex would be swalled out in some code paths. +#ifdef Py_DEBUG + +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ + if (Py_REFCNT(op) != 1) { \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ + } + +#else /* Py_DEBUG */ + +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) + +#endif /* Py_DEBUG */ + +#else /* !Py_GIL_DISABLED */ +// The critical section APIs are no-ops with the GIL. +# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } +# define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) +#endif /* !Py_GIL_DISABLED */ + +// Resumes the top-most critical section. +PyAPI_FUNC(void) +_PyCriticalSection_Resume(PyThreadState *tstate); + +// (private) slow path for locking the mutex +PyAPI_FUNC(void) +_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); + +PyAPI_FUNC(void) +_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, + int is_m1_locked); + +PyAPI_FUNC(void) +_PyCriticalSection_SuspendAll(PyThreadState *tstate); + +#ifdef Py_GIL_DISABLED + +static inline int +_PyCriticalSection_IsActive(uintptr_t tag) +{ + return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; +} + +static inline void +_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +{ + if (PyMutex_LockFast(&m->_bits)) { + PyThreadState *tstate = _PyThreadState_GET(); + c->_cs_mutex = m; + c->_cs_prev = tstate->critical_section; + tstate->critical_section = (uintptr_t)c; + } + else { + _PyCriticalSection_BeginSlow(c, m); + } +} + +static inline void +_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +{ + _PyCriticalSection_BeginMutex(c, &op->ob_mutex); +} +#define PyCriticalSection_Begin _PyCriticalSection_Begin + +// Removes the top-most critical section from the thread's stack of critical +// sections. If the new top-most critical section is inactive, then it is +// resumed. +static inline void +_PyCriticalSection_Pop(PyCriticalSection *c) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = c->_cs_prev; + tstate->critical_section = prev; + + if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) { + _PyCriticalSection_Resume(tstate); + } +} + +static inline void +_PyCriticalSection_End(PyCriticalSection *c) +{ + PyMutex_Unlock(c->_cs_mutex); + _PyCriticalSection_Pop(c); +} +#define PyCriticalSection_End _PyCriticalSection_End + +static inline void +_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ + if (m1 == m2) { + // If the two mutex arguments are the same, treat this as a critical + // section with a single mutex. + c->_cs_mutex2 = NULL; + _PyCriticalSection_BeginMutex(&c->_cs_base, m1); + return; + } + + if ((uintptr_t)m2 < (uintptr_t)m1) { + // Sort the mutexes so that the lower address is locked first. + // The exact order does not matter, but we need to acquire the mutexes + // in a consistent order to avoid lock ordering deadlocks. + PyMutex *tmp = m1; + m1 = m2; + m2 = tmp; + } + + if (PyMutex_LockFast(&m1->_bits)) { + if (PyMutex_LockFast(&m2->_bits)) { + PyThreadState *tstate = _PyThreadState_GET(); + c->_cs_base._cs_mutex = m1; + c->_cs_mutex2 = m2; + c->_cs_base._cs_prev = tstate->critical_section; + + uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; + tstate->critical_section = p; + } + else { + _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + } + } + else { + _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + } +} + +static inline void +_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +{ + _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); +} +#define PyCriticalSection2_Begin _PyCriticalSection2_Begin + +static inline void +_PyCriticalSection2_End(PyCriticalSection2 *c) +{ + if (c->_cs_mutex2) { + PyMutex_Unlock(c->_cs_mutex2); + } + PyMutex_Unlock(c->_cs_base._cs_mutex); + _PyCriticalSection_Pop(&c->_cs_base); +} +#define PyCriticalSection2_End _PyCriticalSection2_End + +static inline void +_PyCriticalSection_AssertHeld(PyMutex *mutex) +{ +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = tstate->critical_section; + if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex)); + } + else { + PyCriticalSection *cs = (PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->_cs_mutex == mutex); + } + +#endif +} + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CRITICAL_SECTION_H */ diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h new file mode 100644 index 00000000000000..2dd165eae74850 --- /dev/null +++ b/Include/internal/pycore_crossinterp.h @@ -0,0 +1,340 @@ +#ifndef Py_INTERNAL_CROSSINTERP_H +#define Py_INTERNAL_CROSSINTERP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_lock.h" // PyMutex +#include "pycore_pyerrors.h" + +/**************/ +/* exceptions */ +/**************/ + +PyAPI_DATA(PyObject *) PyExc_InterpreterError; +PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError; + + +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +typedef int (*_Py_simple_func)(void *); +extern int _Py_CallInInterpreter( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); +extern int _Py_CallInInterpreterAndRawFree( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + +PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); +PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); + +#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data) +#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj) +#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid) +// Users should not need getters for "new_object" or "free". + + +/* defining cross-interpreter data */ + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + +// Normally the Init* functions are sufficient. The only time +// additional initialization might be needed is to set the "free" func, +// though that should be infrequent. +#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \ + do { \ + (DATA)->free = (FUNC); \ + } while (0) +// Additionally, some shareable types are essentially light wrappers +// around other shareable types. The crossinterpdatafunc of the wrapper +// can often be implemented by calling the wrapped object's +// crossinterpdatafunc and then changing the "new_object" function. +// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that, +// but might be better to have a function like +// _PyCrossInterpreterData_AdaptToWrapper() instead. +#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \ + do { \ + (DATA)->new_object = (FUNC); \ + } while (0) + + +/* using cross-interpreter data */ + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + + +/* cross-interpreter data registry */ + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// crossinterpdatafunc. It would be simpler and more efficient. + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + int global; /* builtin types or heap types */ + int initialized; + PyMutex mutex; + struct _xidregitem *head; +}; + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + + +/*****************************/ +/* runtime state & lifecycle */ +/*****************************/ + +struct _xi_runtime_state { + // builtin types + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry registry; +}; + +struct _xi_state { + // heap types + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry registry; + + // heap types + PyObject *PyExc_NotShareableError; +}; + +extern PyStatus _PyXI_Init(PyInterpreterState *interp); +extern void _PyXI_Fini(PyInterpreterState *interp); + +extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); +extern void _PyXI_FiniTypes(PyInterpreterState *interp); + +#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi) + + +/***************************/ +/* short-term data sharing */ +/***************************/ + +// Ultimately we'd like to preserve enough information about the +// exception and traceback that we could re-constitute (or at least +// simulate, a la traceback.TracebackException), and even chain, a copy +// of the exception in the calling interpreter. + +typedef struct _excinfo { + struct _excinfo_type { + PyTypeObject *builtin; + const char *name; + const char *qualname; + const char *module; + } type; + const char *msg; + const char *errdisplay; +} _PyXI_excinfo; + +PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); +PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); +PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); + + +typedef enum error_code { + _PyXI_ERR_NO_ERROR = 0, + _PyXI_ERR_UNCAUGHT_EXCEPTION = -1, + _PyXI_ERR_OTHER = -2, + _PyXI_ERR_NO_MEMORY = -3, + _PyXI_ERR_ALREADY_RUNNING = -4, + _PyXI_ERR_MAIN_NS_FAILURE = -5, + _PyXI_ERR_APPLY_NS_FAILURE = -6, + _PyXI_ERR_NOT_SHAREABLE = -7, +} _PyXI_errcode; + + +typedef struct _sharedexception { + // The originating interpreter. + PyInterpreterState *interp; + // The kind of error to propagate. + _PyXI_errcode code; + // The exception information to propagate, if applicable. + // This is populated only for some error codes, + // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. + _PyXI_excinfo uncaught; +} _PyXI_error; + +PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); + + +typedef struct xi_session _PyXI_session; +typedef struct _sharedns _PyXI_namespace; + +PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns); +PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names); +PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict( + _PyXI_namespace *ns, + PyObject *nsobj, + _PyXI_session *session); +PyAPI_FUNC(int) _PyXI_ApplyNamespace( + _PyXI_namespace *ns, + PyObject *nsobj, + PyObject *dflt); + + +// A cross-interpreter session involves entering an interpreter +// (_PyXI_Enter()), doing some work with it, and finally exiting +// that interpreter (_PyXI_Exit()). +// +// At the boundaries of the session, both entering and exiting, +// data may be exchanged between the previous interpreter and the +// target one in a thread-safe way that does not violate the +// isolation between interpreters. This includes setting objects +// in the target's __main__ module on the way in, and capturing +// uncaught exceptions on the way out. +struct xi_session { + // Once a session has been entered, this is the tstate that was + // current before the session. If it is different from cur_tstate + // then we must have switched interpreters. Either way, this will + // be the current tstate once we exit the session. + PyThreadState *prev_tstate; + // Once a session has been entered, this is the current tstate. + // It must be current when the session exits. + PyThreadState *init_tstate; + // This is true if init_tstate needs cleanup during exit. + int own_init_tstate; + + // This is true if, while entering the session, init_thread took + // "ownership" of the interpreter's __main__ module. This means + // it is the only thread that is allowed to run code there. + // (Caveat: for now, users may still run exec() against the + // __main__ module's dict, though that isn't advisable.) + int running; + // This is a cached reference to the __dict__ of the entered + // interpreter's __main__ module. It is looked up when at the + // beginning of the session as a convenience. + PyObject *main_ns; + + // This is set if the interpreter is entered and raised an exception + // that needs to be handled in some special way during exit. + _PyXI_errcode *error_override; + // This is set if exit captured an exception to propagate. + _PyXI_error *error; + + // -- pre-allocated memory -- + _PyXI_error _error; + _PyXI_errcode _error_override; +}; + +PyAPI_FUNC(int) _PyXI_Enter( + _PyXI_session *session, + PyInterpreterState *interp, + PyObject *nsupdates); +PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); + +PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); +PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); + + +/*************/ +/* other API */ +/*************/ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter( + PyInterpreterConfig *config, + long *maybe_whence, + PyThreadState **p_tstate, + PyThreadState **p_save_tstate); +PyAPI_FUNC(void) _PyXI_EndInterpreter( + PyInterpreterState *interp, + PyThreadState *tstate, + PyThreadState **p_save_tstate); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CROSSINTERP_H */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 47b5948f66343a..56cc49432cc61e 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -1,4 +1,3 @@ - #ifndef Py_INTERNAL_DICT_H #define Py_INTERNAL_DICT_H #ifdef __cplusplus @@ -9,8 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyDictOrValues +#include "pycore_freelist.h" // _PyFreeListState +#include "pycore_object.h" // PyManagedDictPointer +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); @@ -19,9 +19,6 @@ extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value)); // "KnownHash" variants -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, PyObject *item, Py_hash_t hash); @@ -44,14 +41,9 @@ extern int _PyDict_HasOnlyStringKeys(PyObject *mp); extern void _PyDict_MaybeUntrack(PyObject *mp); -extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); - // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -// Export for '_socket' shared extension (Windows remove_unusable_flags()) -PyAPI_FUNC(PyObject*) _PyDict_Pop(PyObject *, PyObject *, PyObject *); - #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, @@ -59,7 +51,7 @@ PyAPI_FUNC(PyObject*) _PyDict_Pop(PyObject *, PyObject *, PyObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); @@ -74,12 +66,6 @@ typedef struct { extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); - -/* runtime lifecycle */ - -extern void _PyDict_Fini(PyInterpreterState *interp); - - /* other API */ typedef struct { @@ -110,16 +96,24 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); * -1 when no entry found, -3 when compare raises error. */ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); -extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); /* Consumes references to key and value */ -extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); - -extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObject *); +PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); +extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); +extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); + +extern int _PyDict_Pop_KnownHash( + PyDictObject *dict, + PyObject *key, + Py_hash_t hash, + PyObject **result); #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ @@ -145,6 +139,11 @@ struct _dictkeysobject { /* Kind of keys */ uint8_t dk_kind; +#ifdef Py_GIL_DISABLED + /* Lock used to protect shared keys */ + PyMutex dk_mutex; +#endif + /* Version number -- Reset to 0 by any modification to keys */ uint32_t dk_version; @@ -154,6 +153,7 @@ struct _dictkeysobject { /* Number of used entries in dk_entries. */ Py_ssize_t dk_nentries; + /* Actual hash table of dk_size entries. It holds indices in dk_entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). @@ -170,7 +170,7 @@ struct _dictkeysobject { char dk_indices[]; /* char is required to avoid strict aliasing. */ /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows: - see the DK_ENTRIES() macro */ + see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; /* This must be no more than 250, for the prefix size to fit in one byte. */ @@ -184,6 +184,10 @@ struct _dictkeysobject { * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order. */ struct _dictvalues { + uint8_t capacity; + uint8_t size; + uint8_t embedded; + uint8_t valid; PyObject *values[1]; }; @@ -199,6 +203,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { size_t index = (size_t)1 << dk->dk_log2_index_bytes; return (&indices[index]); } + static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { assert(dk->dk_kind == DICT_KEYS_GENERAL); return (PyDictKeyEntry*)_DK_ENTRIES(dk); @@ -210,13 +215,37 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL) -#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) -#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) +#define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) +#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) + +#ifdef Py_GIL_DISABLED + +#define THREAD_LOCAL_DICT_VERSION_COUNT 256 +#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT + +static inline uint64_t +dict_next_version(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyThreadState_GET(); + uint64_t cur_progress = (tstate->dict_global_version & + (THREAD_LOCAL_DICT_VERSION_BATCH - 1)); + if (cur_progress == 0) { + uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version, + THREAD_LOCAL_DICT_VERSION_BATCH); + tstate->dict_global_version = next; + } + return tstate->dict_global_version += DICT_VERSION_INCREMENT; +} + +#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP) +#else #define DICT_NEXT_VERSION(INTERP) \ ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) +#endif -void +PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, PyDictObject *mp, @@ -231,33 +260,70 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; + int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { + RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(interp); + return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } -extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); -extern PyObject *_PyDict_FromItems( +extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); + +PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); +static inline uint8_t * +get_insertion_order_array(PyDictValues *values) +{ + return (uint8_t *)&values->values[values->capacity]; +} + static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { assert(ix < SHARED_KEYS_MAX_SIZE); - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; - assert(size+2 < ((uint8_t *)values)[-1]); - size++; - size_ptr[-size] = (uint8_t)ix; - *size_ptr = size; + int size = values->size; + uint8_t *array = get_insertion_order_array(values); + assert(size < values->capacity); + assert(((uint8_t)ix) == ix); + array[size] = (uint8_t)ix; + values->size = size+1; } +static inline size_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + return dk_nentries + dk_usable; +} + +static inline size_t +_PyInlineValuesSize(PyTypeObject *tp) +{ + PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys; + assert(keys != NULL); + size_t size = shared_keys_usable_size(keys); + size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *)); + assert(prefix_size < 256); + return prefix_size + (size + 1) * sizeof(PyObject *); +} + +int +_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); + +PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index ece0f10ca25170..1a44755c7a01a3 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -8,17 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#ifndef WITH_FREELISTS -// without freelists -# define PyDict_MAXFREELIST 0 -#endif - -#ifndef PyDict_MAXFREELIST -# define PyDict_MAXFREELIST 80 -#endif - #define DICT_MAX_WATCHERS 8 +#define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { /*Global counter used to set ma_version_tag field of dictionary. @@ -26,15 +17,6 @@ struct _Py_dict_state { * time that a dictionary is modified. */ uint64_t global_version; uint32_t next_keys_version; - -#if PyDict_MAXFREELIST > 0 - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int numfree; - int keys_numfree; -#endif - PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index ac62a4d300720a..e4222c5267d6be 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -11,8 +11,6 @@ extern "C" { #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR -#if _PY_SHORT_FLOAT_REPR == 1 - typedef uint32_t ULong; struct @@ -22,19 +20,22 @@ Bigint { ULong x[1]; }; -#ifdef Py_USING_MEMORY_DEBUGGER +#if defined(Py_USING_MEMORY_DEBUGGER) || _PY_SHORT_FLOAT_REPR == 0 struct _dtoa_state { int _not_used; }; -#define _dtoa_interp_state_INIT(INTERP) \ +#define _dtoa_state_INIT(INTERP) \ {0} -#else // !Py_USING_MEMORY_DEBUGGER +#else // !Py_USING_MEMORY_DEBUGGER && _PY_SHORT_FLOAT_REPR != 0 /* The size of the Bigint freelist */ #define Bigint_Kmax 7 +/* The size of the cached powers of 5 array */ +#define Bigint_Pow5size 8 + #ifndef PRIVATE_MEM #define PRIVATE_MEM 2304 #endif @@ -42,9 +43,10 @@ struct _dtoa_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_state { - /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // p5s is an array of powers of 5 of the form: + // 5**(2**(i+2)) for 0 <= i < Bigint_Pow5size + struct Bigint *p5s[Bigint_Pow5size]; // XXX This should be freed during runtime fini. - struct Bigint *p5s; struct Bigint *freelist[Bigint_Kmax+1]; double preallocated[Bigint_PREALLOC_SIZE]; double *preallocated_next; @@ -57,15 +59,15 @@ struct _dtoa_state { #endif // !Py_USING_MEMORY_DEBUGGER -/* These functions are used by modules compiled as C extension like math: - they must be exported. */ - extern double _Py_dg_strtod(const char *str, char **ptr); extern char* _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); extern void _Py_dg_freedtoa(char *s); -#endif // _PY_SHORT_FLOAT_REPR == 1 + +extern PyStatus _PyDtoa_Init(PyInterpreterState *interp); +extern void _PyDtoa_Fini(PyInterpreterState *interp); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2f89da2c6ecd91..13f86b01bbfe8f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -290,6 +290,8 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +extern void _Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, Py_ssize_t *rootsize); + // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. // Usage: @@ -320,6 +322,13 @@ PyAPI_FUNC(char*) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject extern int _PyFile_Flush(PyObject *); +#ifndef MS_WINDOWS +extern int _Py_GetTicksPerSecond(long *ticks_per_second); +#endif + +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 4e5474841bc25d..f984df695696c3 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,14 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); -extern void _PyFloat_Fini(PyInterpreterState *); extern void _PyFloat_FiniType(PyInterpreterState *); @@ -33,26 +32,9 @@ struct _Py_float_runtime_state { }; -#ifndef WITH_FREELISTS -// without freelists -# define PyFloat_MAXFREELIST 0 -#endif - -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif -struct _Py_float_state { -#if PyFloat_MAXFREELIST > 0 - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *free_list; -#endif -}; -void _PyFloat_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); extern void _PyFloat_DebugMallocStats(FILE* out); diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 58fed46886ea45..5043260d2fd99f 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -8,17 +8,14 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int id; -} _PyCfgJumpTargetLabel; +#include "pycore_instruction_sequence.h" +#include "pycore_opcode_utils.h" struct _PyCfgBuilder; -int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc); struct _PyCfgBuilder* _PyCfgBuilder_New(void); void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); @@ -27,16 +24,23 @@ int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq); + _PyInstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, - PyObject *consts, int maxdepth, _PyCompile_InstructionSequence *instrs, + PyObject *consts, int maxdepth, _PyInstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( + PyObject *instructions, + PyObject *consts, + int nlocals); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index bd448272e058ab..506c20ca1950bd 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -11,8 +11,9 @@ extern "C" { #include <stdbool.h> #include <stddef.h> // offsetof() #include "pycore_code.h" // STATS +#include "pycore_stackref.h" // _PyStackRef -/* See Objects/frame_layout.md for an explanation of the frame stack +/* See InternalDocs/frames.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame * structs. */ @@ -25,7 +26,7 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -36,13 +37,17 @@ extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ typedef enum _framestate { - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, + FRAME_CREATED = -3, + FRAME_SUSPENDED = -2, + FRAME_SUSPENDED_YIELD_FROM = -1, FRAME_EXECUTING = 0, FRAME_COMPLETED = 1, FRAME_CLEARED = 4 } PyFrameState; +#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) +#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) + enum _frameowner { FRAME_OWNED_BY_THREAD = 0, FRAME_OWNED_BY_GENERATOR = 1, @@ -51,56 +56,46 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - PyObject *f_executable; /* Strong reference */ + PyObject *f_executable; /* Strong reference (code object or None) */ struct _PyInterpreterFrame *previous; PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ - // NOTE: This is not necessarily the last instruction started in the given - // frame. Rather, it is the code unit *prior to* the *next* instruction. For - // example, it may be an inline CACHE entry, an instruction we just jumped - // over, or (in the case of a newly-created frame) a totally invalid value: - _Py_CODEUNIT *prev_instr; + _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ - /* The return_offset determines where a `RETURN` should go in the caller, - * relative to `prev_instr`. - * It is only meaningful to the callee, - * so it needs to be set in any CALL (to a Python function) - * or SEND (to a coroutine or generator). - * If there is no callee, then it is meaningless. */ - uint16_t return_offset; + uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ - PyObject *localsplus[1]; + _PyStackRef localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { assert(PyCode_Check(f->f_executable)); return (PyCodeObject *)f->f_executable; } -static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus; +static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { + return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); + assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1])); return f->localsplus[f->stacktop-1]; } -static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); f->stacktop--; return f->localsplus[f->stacktop]; } -static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { +static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) { f->localsplus[f->stacktop] = value; f->stacktop++; } @@ -116,7 +111,17 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) return code->co_framesize - FRAME_SPECIALS_SIZE; } -void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); +static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) +{ + assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + *dest = *src; + for (int i = 1; i < src->stacktop; i++) { + dest->localsplus[i] = src->localsplus[i]; + } + // Don't leave a dangling pointer to the old frame when creating generators + // and coroutines: + dest->previous = NULL; +} /* Consumes reference to func and locals. Does not initialize frame->previous, which happens @@ -134,19 +139,19 @@ _PyFrame_Initialize( frame->f_locals = locals; frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) - 1; + frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { - frame->localsplus[i] = NULL; + frame->localsplus[i] = PyStackRef_NULL; } } /* Gets the pointer to the locals array * that precedes this frame. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) { return frame->localsplus; @@ -156,16 +161,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) Having stacktop <= 0 ensures that invalid values are not visible to the cycle GC. We choose -1 rather than 0 to assist debugging. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus + frame->stacktop; + _PyStackRef *sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; } static inline void -_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { frame->stacktop = (int)(stack_pointer - frame->localsplus); } @@ -185,7 +190,7 @@ _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) return true; } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; + frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -210,7 +215,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily * creating it if necessary. - * Returns a borrowed referennce */ + * Returns a borrowed reference */ static inline PyFrameObject * _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { @@ -223,6 +228,9 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame); + /* Clears all references in the frame. * If take is non-zero, then the _PyInterpreterFrame frame * may be transferred to the frame object it references @@ -238,14 +246,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) @@ -262,7 +267,7 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) extern _PyInterpreterFrame * _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); -void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); /* Pushes a frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() @@ -297,19 +302,16 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code); + frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; return frame; } -static inline -PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) -{ - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); - return (PyGenObject *)(((char *)frame) - offset_in_gen); -} +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, _PyStackRef const* args, + size_t argcount, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h new file mode 100644 index 00000000000000..e684e084b8bef8 --- /dev/null +++ b/Include/internal/pycore_freelist.h @@ -0,0 +1,153 @@ +#ifndef Py_INTERNAL_FREELIST_H +#define Py_INTERNAL_FREELIST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// PyTuple_MAXSAVESIZE - largest tuple to save on free list +// PyTuple_MAXFREELIST - maximum number of tuples of each size to save + +#ifdef WITH_FREELISTS +// with freelists +# define PyTuple_MAXSAVESIZE 20 +# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE +# define PyTuple_MAXFREELIST 2000 +# define PyList_MAXFREELIST 80 +# define PyDict_MAXFREELIST 80 +# define PyFloat_MAXFREELIST 100 +# define PyContext_MAXFREELIST 255 +# define _PyAsyncGen_MAXFREELIST 80 +# define _PyObjectStackChunk_MAXFREELIST 4 +#else +# define PyTuple_NFREELISTS 0 +# define PyTuple_MAXFREELIST 0 +# define PyList_MAXFREELIST 0 +# define PyDict_MAXFREELIST 0 +# define PyFloat_MAXFREELIST 0 +# define PyContext_MAXFREELIST 0 +# define _PyAsyncGen_MAXFREELIST 0 +# define _PyObjectStackChunk_MAXFREELIST 0 +#endif + +struct _Py_list_freelist { +#ifdef WITH_FREELISTS + PyListObject *items[PyList_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_tuple_freelist { +#if WITH_FREELISTS + /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. + The empty tuple is handled separately. + + Each tuple stored in the array is the head of the linked list + (and the next available tuple) for that size. The actual tuple + object is used as the linked list node, with its first item + (ob_item[0]) pointing to the next node (i.e. the previous head). + Each linked list is initially NULL. */ + PyTupleObject *items[PyTuple_NFREELISTS]; + int numfree[PyTuple_NFREELISTS]; +#else + char _unused; // Empty structs are not allowed. +#endif +}; + +struct _Py_float_freelist { +#ifdef WITH_FREELISTS + /* Special free list + free_list is a singly-linked list of available PyFloatObjects, + linked via abuse of their ob_type members. */ + int numfree; + PyFloatObject *items; +#endif +}; + +struct _Py_dict_freelist { +#ifdef WITH_FREELISTS + /* Dictionary reuse scheme to save calls to malloc and free */ + PyDictObject *items[PyDict_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_dictkeys_freelist { +#ifdef WITH_FREELISTS + /* Dictionary keys reuse scheme to save calls to malloc and free */ + PyDictKeysObject *items[PyDict_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_slice_freelist { +#ifdef WITH_FREELISTS + /* Using a cache is very effective since typically only a single slice is + created and then deleted again. */ + PySliceObject *slice_cache; +#endif +}; + +struct _Py_context_freelist { +#ifdef WITH_FREELISTS + // List of free PyContext objects + PyContext *items; + int numfree; +#endif +}; + +struct _Py_async_gen_freelist { +#ifdef WITH_FREELISTS + /* Freelists boost performance 6-10%; they also reduce memory + fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend + are short-living objects that are instantiated for every + __anext__() call. */ + struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_async_gen_asend_freelist { +#ifdef WITH_FREELISTS + struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; + int numfree; +#endif +}; + +struct _PyObjectStackChunk; + +struct _Py_object_stack_freelist { + struct _PyObjectStackChunk *items; + Py_ssize_t numfree; +}; + +struct _Py_object_freelists { + struct _Py_float_freelist floats; + struct _Py_tuple_freelist tuples; + struct _Py_list_freelist lists; + struct _Py_dict_freelist dicts; + struct _Py_dictkeys_freelist dictkeys; + struct _Py_slice_freelist slices; + struct _Py_context_freelist contexts; + struct _Py_async_gen_freelist async_gens; + struct _Py_async_gen_asend_freelist async_gen_asends; + struct _Py_object_stack_freelist object_stacks; +}; + +extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_H */ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 3f3da8a44b77e4..6d44e933e8a8cb 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_lock.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -17,20 +19,32 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 #define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + +struct _func_version_cache_item { + PyFunctionObject *func; + PyObject *code; +}; + struct _py_func_state { +#ifdef Py_GIL_DISABLED + // Protects next_version + PyMutex mutex; +#endif + uint32_t next_version; - // Borrowed references to function objects whose + // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE // once was equal to the index in the table. - // They are cleared when the function is deallocated. - PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; + // They are cleared when the function or code object is deallocated. + struct _func_version_cache_item func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); +PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +void _PyFunction_ClearCodeByVersion(uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2d33aa76d78229..28e34d3809634c 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState + /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. @@ -35,10 +37,57 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) + * + * Setting the bits requires a relaxed store. The per-object lock must also be + * held, except when the object is only visible to a single thread (e.g. during + * object initialization or destruction). + * + * Reading the bits requires using a relaxed load, but does not require holding + * the per-object lock. + */ +#ifdef Py_GIL_DISABLED +# define _PyGC_BITS_TRACKED (1) // Tracked by the GC +# define _PyGC_BITS_FINALIZED (2) // tp_finalize was called +# define _PyGC_BITS_UNREACHABLE (4) +# define _PyGC_BITS_FROZEN (8) +# define _PyGC_BITS_SHARED (16) +# define _PyGC_BITS_SHARED_INLINE (32) +# define _PyGC_BITS_DEFERRED (64) // Use deferred reference counting +#endif + +#ifdef Py_GIL_DISABLED + +static inline void +_PyObject_SET_GC_BITS(PyObject *op, uint8_t new_bits) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits | new_bits); +} + +static inline int +_PyObject_HAS_GC_BITS(PyObject *op, uint8_t bits) +{ + return (_Py_atomic_load_uint8_relaxed(&op->ob_gc_bits) & bits) != 0; +} + +static inline void +_PyObject_CLEAR_GC_BITS(PyObject *op, uint8_t bits_to_clear) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits & ~bits_to_clear); +} + +#endif + /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_TRACKED); +#else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); +#endif } #define _PyObject_GC_IS_TRACKED(op) _PyObject_GC_IS_TRACKED(_Py_CAST(PyObject*, op)) @@ -54,24 +103,97 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { return 1; } +#ifdef Py_GIL_DISABLED + +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ +static inline int _PyObject_GC_IS_SHARED(PyObject *op) { + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED); +} +#define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED(PyObject *op) { + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED); +} +#define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) + +/* True if the memory of the object is shared between multiple + * threads and needs special purpose when freeing due to + * the possibility of in-flight lock-free reads occurring. + * Objects with this bit that are GC objects will automatically + * delay-freed by PyObject_GC_Del. */ +static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); +} +#define _PyObject_GC_IS_SHARED_INLINE(op) \ + _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); +} +#define _PyObject_GC_SET_SHARED_INLINE(op) \ + _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +#endif /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ -#define _PyGC_PREV_MASK_FINALIZED (1) +#define _PyGC_PREV_MASK_FINALIZED 1 /* Bit 1 is set when the object is in generation which is GCed currently. */ -#define _PyGC_PREV_MASK_COLLECTING (2) -/* The (N-2) most significant bits contain the real address. */ -#define _PyGC_PREV_SHIFT (2) +#define _PyGC_PREV_MASK_COLLECTING 2 + +/* Bit 0 in _gc_next is the old space bit. + * It is set as follows: + * Young: gcstate->visited_space + * old[0]: 0 + * old[1]: 1 + * permanent: 0 + * + * During a collection all objects handled should have the bit set to + * gcstate->visited_space, as objects are moved from the young gen + * and the increment into old[gcstate->visited_space]. + * When object are moved from the pending space, old[gcstate->visited_space^1] + * into the increment, the old space bit is flipped. +*/ +#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 + +#define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) +/* set for debugging information */ +#define _PyGC_DEBUG_STATS (1<<0) /* print collection statistics */ +#define _PyGC_DEBUG_COLLECTABLE (1<<1) /* print collectable objects */ +#define _PyGC_DEBUG_UNCOLLECTABLE (1<<2) /* print uncollectable objects */ +#define _PyGC_DEBUG_SAVEALL (1<<5) /* save all garbage in gc.garbage */ +#define _PyGC_DEBUG_LEAK _PyGC_DEBUG_COLLECTABLE | \ + _PyGC_DEBUG_UNCOLLECTABLE | \ + _PyGC_DEBUG_SAVEALL + +typedef enum { + // GC was triggered by heap allocation + _Py_GC_REASON_HEAP, + + // GC was called during shutdown + _Py_GC_REASON_SHUTDOWN, + + // GC was called by gc.collect() or PyGC_Collect() + _Py_GC_REASON_MANUAL +} _PyGC_Reason; + // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next; + uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - gc->_gc_next = (uintptr_t)next; + uintptr_t unext = (uintptr_t)next; + assert((unext & ~_PyGC_PREV_MASK) == 0); + gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -79,26 +201,36 @@ static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); return (PyGC_Head*)prev; } + static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { uintptr_t uprev = (uintptr_t)prev; assert((uprev & ~_PyGC_PREV_MASK) == 0); gc->_gc_prev = ((gc->_gc_prev & ~_PyGC_PREV_MASK) | uprev); } -static inline int _PyGCHead_FINALIZED(PyGC_Head *gc) { - return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); -} -static inline void _PyGCHead_SET_FINALIZED(PyGC_Head *gc) { - gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; -} - static inline int _PyGC_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_FINALIZED); +#else PyGC_Head *gc = _Py_AS_GC(op); - return _PyGCHead_FINALIZED(gc); + return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); +#endif } static inline void _PyGC_SET_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + _PyObject_SET_GC_BITS(op, _PyGC_BITS_FINALIZED); +#else PyGC_Head *gc = _Py_AS_GC(op); - _PyGCHead_SET_FINALIZED(gc); + gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; +#endif +} +static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { +#ifdef Py_GIL_DISABLED + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_FINALIZED); +#else + PyGC_Head *gc = _Py_AS_GC(op); + gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; +#endif } @@ -155,6 +287,13 @@ struct gc_generation { generations */ }; +struct gc_collection_stats { + /* number of collected objects */ + Py_ssize_t collected; + /* total number of uncollectable objects (put into gc.garbage) */ + Py_ssize_t uncollectable; +}; + /* Running stats per generation */ struct gc_generation_stats { /* total number of collections */ @@ -176,8 +315,8 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ - struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; + struct gc_generation young; + struct gc_generation old[2]; /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; @@ -187,6 +326,13 @@ struct _gc_runtime_state { PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + + Py_ssize_t heap_size; + Py_ssize_t work_to_do; + /* Which of the old spaces is the visited space */ + int visited_space; + +#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -198,24 +344,50 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; + + /* gh-117783: Deferred reference counting is not fully implemented yet, so + as a temporary measure we treat objects using deferred reference + counting as immortal. The value may be zero, one, or a negative number: + 0: immortalize deferred RC objects once the first thread is created + 1: immortalize all deferred RC objects immediately + <0: suppressed; don't immortalize objects */ + int immortalize; +#endif +}; + +#ifdef Py_GIL_DISABLED +struct _gc_thread_state { + /* Thread-local allocation count. */ + Py_ssize_t alloc_count; }; +#endif extern void _PyGC_InitState(struct _gc_runtime_state *); -extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); +extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); +extern void _PyGC_CollectNoFail(PyThreadState *tstate); +/* Freeze objects tracked by the GC and ignore them in future collections. */ +extern void _PyGC_Freeze(PyInterpreterState *interp); +/* Unfreezes objects placing them in the oldest generation */ +extern void _PyGC_Unfreeze(PyInterpreterState *interp); +/* Number of frozen objects */ +extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp); + +extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation); +extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists -extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); -extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); -extern void _PyList_ClearFreeList(PyInterpreterState *interp); -extern void _PyDict_ClearFreeList(PyInterpreterState *interp); -extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); -extern void _PyContext_ClearFreeList(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +#ifdef Py_GIL_DISABLED +// gh-117783: Immortalize objects that use deferred reference counting +extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index cf58a2750a31f9..f6d7e6d367177b 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -8,7 +8,51 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyObject *_PyGen_yf(PyGenObject *); +#include "pycore_frame.h" + +/* _PyGenObject_HEAD defines the initial segment of generator + and coroutine objects. */ +#define _PyGenObject_HEAD(prefix) \ + PyObject_HEAD \ + /* List of weak reference. */ \ + PyObject *prefix##_weakreflist; \ + /* Name of the generator. */ \ + PyObject *prefix##_name; \ + /* Qualified name of the generator. */ \ + PyObject *prefix##_qualname; \ + _PyErr_StackItem prefix##_exc_state; \ + PyObject *prefix##_origin_or_finalizer; \ + char prefix##_hooks_inited; \ + char prefix##_closed; \ + char prefix##_running_async; \ + /* The frame */ \ + int8_t prefix##_frame_state; \ + struct _PyInterpreterFrame prefix##_iframe; \ + +struct _PyGenObject { + /* The gi_ prefix is intended to remind of generator-iterator. */ + _PyGenObject_HEAD(gi) +}; + +struct _PyCoroObject { + _PyGenObject_HEAD(cr) +}; + +struct _PyAsyncGenObject { + _PyGenObject_HEAD(ag) +}; + +#undef _PyGenObject_HEAD + +static inline +PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); + return (PyGenObject *)(((char *)frame) - offset_in_gen); +} + +PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension @@ -17,44 +61,13 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); +PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; -/* runtime lifecycle */ - -extern void _PyAsyncGen_Fini(PyInterpreterState *); - - -/* other API */ - -#ifndef WITH_FREELISTS -// without freelists -# define _PyAsyncGen_MAXFREELIST 0 -#endif - -#ifndef _PyAsyncGen_MAXFREELIST -# define _PyAsyncGen_MAXFREELIST 80 -#endif - -struct _Py_async_gen_state { -#if _PyAsyncGen_MAXFREELIST > 0 - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; - - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; -#endif -}; - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index daf1e73e7827e8..a2de5077371eba 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" // _Py_atomic_int #include "pycore_condvar.h" // PyCOND_T #ifndef Py_HAVE_CONDVAR @@ -21,6 +20,23 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* If this GIL is disabled, enabled == 0. + + If this GIL is enabled transiently (most likely to initialize a module + of unknown safety), enabled indicates the number of active transient + requests. + + If this GIL is enabled permanently, enabled == INT_MAX. + + It must not be modified directly; use _PyEval_EnableGILTransiently(), + _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL() + + It is always read and written atomically, but a thread can assume its + value will be stable as long as that thread is attached or knows that no + other threads are attached (e.g., during a stop-the-world.). */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us @@ -28,7 +44,7 @@ struct _gil_runtime_state { PyThreadState* last_holder; /* Whether the GIL is already taken (-1 if uninitialized). This is atomic because it can be read without any lock taken in ceval.c. */ - _Py_atomic_int locked; + int locked; /* Number of GIL switches since the beginning. */ unsigned long switch_number; /* This condition variable allows one or several threads to wait diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8fb22f70505808..c0840f9eb7eca2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -551,21 +551,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(defaults)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); @@ -577,7 +573,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); @@ -590,6 +585,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__all__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); @@ -624,6 +620,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__firstlineno__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__format__)); @@ -724,12 +721,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slotnames__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slots__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__spec__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__static_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__str__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__sub__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasscheck__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasshook__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__truediv__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__trunc__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__type_params__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_is_unpacked_typevartuple__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_prepare_subst__)); @@ -742,7 +739,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); @@ -752,6 +749,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); @@ -762,23 +760,25 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_layout_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_xoptions)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(a)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(abs_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); @@ -787,8 +787,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -798,7 +802,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(b)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); @@ -816,16 +819,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byteorder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes_per_sep)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -856,6 +860,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_stacksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); @@ -869,15 +874,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(desired_access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); @@ -893,17 +899,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(displayhook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dklen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(doc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(done)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_lineno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); @@ -912,13 +918,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -934,15 +941,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fileno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filepath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fillvalue)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(final)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(find_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fix_imports)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(frequency)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -950,7 +958,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -964,15 +971,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(has_location)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); @@ -981,9 +990,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inherit_handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inheritable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_bytes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_state)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inner_size)); @@ -1015,6 +1027,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1038,11 +1052,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(logoption)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); @@ -1052,15 +1068,19 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_sequence_fields)); @@ -1071,6 +1091,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); @@ -1105,7 +1126,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(p)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password)); @@ -1133,7 +1153,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(r)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read1)); @@ -1157,10 +1176,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(s)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); @@ -1186,9 +1206,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sleep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sock)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source_traceback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); @@ -1228,6 +1248,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); @@ -1237,6 +1258,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); @@ -1247,6 +1269,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wait_all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warn_on_full_buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); @@ -1258,7 +1282,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 39ffda8419a77d..51735a8a726e11 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -37,21 +37,17 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_setcomp, "<setcomp>") STRUCT_FOR_STR(anon_string, "<string>") STRUCT_FOR_STR(anon_unknown, "<unknown>") - STRUCT_FOR_STR(close_br, "}") STRUCT_FOR_STR(dbl_close_br, "}}") STRUCT_FOR_STR(dbl_open_br, "{{") STRUCT_FOR_STR(dbl_percent, "%%") STRUCT_FOR_STR(defaults, ".defaults") - STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".<locals>") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") - STRUCT_FOR_STR(newline, "\n") - STRUCT_FOR_STR(open_br, "{") - STRUCT_FOR_STR(percent, "%") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; @@ -66,7 +62,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) - STRUCT_FOR_ID(_) STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) @@ -79,6 +74,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__all__) STRUCT_FOR_ID(__and__) STRUCT_FOR_ID(__anext__) + STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) STRUCT_FOR_ID(__asyncio_running_event_loop__) @@ -113,6 +109,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) STRUCT_FOR_ID(__file__) + STRUCT_FOR_ID(__firstlineno__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) STRUCT_FOR_ID(__format__) @@ -213,12 +210,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(__slotnames__) STRUCT_FOR_ID(__slots__) STRUCT_FOR_ID(__spec__) + STRUCT_FOR_ID(__static_attributes__) STRUCT_FOR_ID(__str__) STRUCT_FOR_ID(__sub__) STRUCT_FOR_ID(__subclasscheck__) STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) - STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__type_params__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_prepare_subst__) @@ -231,7 +228,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) - STRUCT_FOR_ID(_annotation) + STRUCT_FOR_ID(_align_) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) @@ -241,6 +238,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_check_retval_) STRUCT_FOR_ID(_dealloc_warn) STRUCT_FOR_ID(_feature_version) + STRUCT_FOR_ID(_field_types) STRUCT_FOR_ID(_fields_) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) @@ -251,23 +249,25 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_layout_) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) + STRUCT_FOR_ID(_only_immortal) STRUCT_FOR_ID(_pack_) STRUCT_FOR_ID(_restype_) STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) + STRUCT_FOR_ID(_strptime) STRUCT_FOR_ID(_strptime_datetime) STRUCT_FOR_ID(_swappedbytes_) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) STRUCT_FOR_ID(_xoptions) - STRUCT_FOR_ID(a) STRUCT_FOR_ID(abs_tol) STRUCT_FOR_ID(access) STRUCT_FOR_ID(aclose) @@ -276,8 +276,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) + STRUCT_FOR_ID(arg) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -287,7 +291,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(autocommit) - STRUCT_FOR_ID(b) STRUCT_FOR_ID(backtick) STRUCT_FOR_ID(base) STRUCT_FOR_ID(before) @@ -305,16 +308,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(byteorder) STRUCT_FOR_ID(bytes) STRUCT_FOR_ID(bytes_per_sep) - STRUCT_FOR_ID(c) STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(category) @@ -345,6 +349,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_stacksize) STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) + STRUCT_FOR_ID(col_offset) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) @@ -358,15 +363,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(count) STRUCT_FOR_ID(covariant) STRUCT_FOR_ID(cwd) - STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(day) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) STRUCT_FOR_ID(defaultaction) STRUCT_FOR_ID(delete) STRUCT_FOR_ID(depth) + STRUCT_FOR_ID(desired_access) STRUCT_FOR_ID(detect_types) STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) @@ -382,17 +388,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(displayhook) STRUCT_FOR_ID(dklen) STRUCT_FOR_ID(doc) + STRUCT_FOR_ID(done) STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) - STRUCT_FOR_ID(duration) - STRUCT_FOR_ID(e) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) + STRUCT_FOR_ID(end_col_offset) STRUCT_FOR_ID(end_lineno) STRUCT_FOR_ID(end_offset) STRUCT_FOR_ID(endpos) @@ -401,13 +407,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) - STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -423,15 +430,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(fileno) STRUCT_FOR_ID(filepath) STRUCT_FOR_ID(fillvalue) + STRUCT_FOR_ID(filter) STRUCT_FOR_ID(filters) STRUCT_FOR_ID(final) STRUCT_FOR_ID(find_class) STRUCT_FOR_ID(fix_imports) STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) + STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) - STRUCT_FOR_ID(frequency) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -439,7 +447,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) - STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -453,15 +460,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) - STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) + STRUCT_FOR_ID(handle_seq) + STRUCT_FOR_ID(has_location) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) - STRUCT_FOR_ID(id) + STRUCT_FOR_ID(hour) STRUCT_FOR_ID(ident) + STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(importlib) @@ -470,9 +479,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) + STRUCT_FOR_ID(inherit_handle) STRUCT_FOR_ID(inheritable) STRUCT_FOR_ID(initial) STRUCT_FOR_ID(initial_bytes) + STRUCT_FOR_ID(initial_owner) + STRUCT_FOR_ID(initial_state) STRUCT_FOR_ID(initial_value) STRUCT_FOR_ID(initval) STRUCT_FOR_ID(inner_size) @@ -504,6 +516,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw) STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) + STRUCT_FOR_ID(kwdefaults) + STRUCT_FOR_ID(label) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -527,11 +541,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(locals) STRUCT_FOR_ID(logoption) STRUCT_FOR_ID(loop) + STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) + STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) @@ -541,15 +557,19 @@ struct _Py_global_strings { STRUCT_FOR_ID(metaclass) STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) + STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(milliseconds) + STRUCT_FOR_ID(minute) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) + STRUCT_FOR_ID(mutex) STRUCT_FOR_ID(mycmp) - STRUCT_FOR_ID(n) STRUCT_FOR_ID(n_arg) STRUCT_FOR_ID(n_fields) STRUCT_FOR_ID(n_sequence_fields) @@ -560,6 +580,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) @@ -594,7 +615,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) - STRUCT_FOR_ID(p) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parent) STRUCT_FOR_ID(password) @@ -622,7 +642,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(ps2) STRUCT_FOR_ID(query) STRUCT_FOR_ID(quotetabs) - STRUCT_FOR_ID(r) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) STRUCT_FOR_ID(read1) @@ -646,10 +665,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) - STRUCT_FOR_ID(s) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) + STRUCT_FOR_ID(second) + STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) @@ -675,9 +695,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(sleep) STRUCT_FOR_ID(sock) STRUCT_FOR_ID(sort) - STRUCT_FOR_ID(sound) STRUCT_FOR_ID(source) STRUCT_FOR_ID(source_traceback) + STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) STRUCT_FOR_ID(stacklevel) @@ -717,6 +737,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) + STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) @@ -726,6 +747,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(type) STRUCT_FOR_ID(type_params) STRUCT_FOR_ID(tz) + STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) STRUCT_FOR_ID(unlink) @@ -736,6 +758,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) STRUCT_FOR_ID(volume) + STRUCT_FOR_ID(wait_all) + STRUCT_FOR_ID(warn_on_full_buffer) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) @@ -747,7 +771,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) - STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) } identifiers; @@ -770,6 +793,10 @@ struct _Py_global_strings { (_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base)) #define _Py_STR(NAME) \ (_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base)) +#define _Py_LATIN1_CHR(CH) \ + ((CH) < 128 \ + ? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \ + : (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128]) /* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function. diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h deleted file mode 100644 index 0e015a40c831f4..00000000000000 --- a/Include/internal/pycore_identifier.h +++ /dev/null @@ -1,54 +0,0 @@ -/* String Literals: _Py_Identifier API */ - -#ifndef Py_INTERNAL_IDENTIFIER_H -#define Py_INTERNAL_IDENTIFIER_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -/* This structure helps managing static strings. The basic usage goes like this: - Instead of doing - - r = PyObject_CallMethod(o, "foo", "args", ...); - - do - - _Py_IDENTIFIER(foo); - ... - r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); - - PyId_foo is a static variable, either on block level or file level. On first - usage, the string "foo" is interned, and the structures are linked. On interpreter - shutdown, all strings are released. - - Alternatively, _Py_static_string allows choosing the variable name. - _PyUnicode_FromId returns a borrowed reference to the interned string. - _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. -*/ -typedef struct _Py_Identifier { - const char* string; - // Index in PyInterpreterState.unicode.ids.array. It is process-wide - // unique and must be initialized to -1. - Py_ssize_t index; -} _Py_Identifier; - -// For now we are keeping _Py_IDENTIFIER for continued use -// in non-builtin extensions (and naughty PyPI modules). - -#define _Py_static_string_init(value) { .string = (value), .index = -1 } -#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) -#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) - -extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -extern PyObject* _PyObject_GetAttrId(PyObject *, _Py_Identifier *); -extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_IDENTIFIER_H diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 117e46bb86285d..290ba95e1a0ad7 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -9,8 +9,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_time.h" // _PyTime_t extern int _PyImport_IsInitialized(PyInterpreterState *); @@ -20,15 +20,15 @@ PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); -extern int _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReleaseLock(PyInterpreterState *interp); +// This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( + PyThreadState *tstate, PyObject *mod, const char *name, /* UTF-8 encoded string */ PyObject *modules ); -extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); // Export for many shared extensions, like '_json' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); @@ -47,12 +47,12 @@ struct _import_runtime_state { Py_ssize_t last_module_index; struct { /* A lock to guard the cache. */ - PyThread_type_lock mutex; + PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added - and only if they support multiple initialization (m_size >- 0) + and only if they support multiple initialization (m_size >= 0) or are imported in the main interpreter. - This is initialized lazily in _PyImport_FixupExtensionObject(). + This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ _Py_hashtable_t *hashtable; } extensions; @@ -94,15 +94,11 @@ struct _import_state { #endif PyObject *import_func; /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; + _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; - _PyTime_t accumulated; + PyTime_t accumulated; int header; } find_and_load; }; @@ -123,11 +119,6 @@ struct _import_state { #define IMPORTS_INIT \ { \ DLOPENFLAGS_INIT \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ .find_and_load = { \ .header = 1, \ }, \ @@ -180,11 +171,6 @@ extern void _PyImport_FiniCore(PyInterpreterState *interp); extern void _PyImport_FiniExternal(PyInterpreterState *interp); -#ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); -#endif - - extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { @@ -206,6 +192,19 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); +#ifdef Py_GIL_DISABLED +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), resolve the transient request depending on the +// state of the module argument: +// - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, +// call _PyEval_DisableGIL(). +// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already +// enabled permanently, issue a warning referencing the module's name. +// +// This function may raise an exception. +extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index dee64241c763f3..e5f222b371a113 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,9 +14,87 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); +typedef enum ext_module_kind { + _Py_ext_module_kind_UNKNOWN = 0, + _Py_ext_module_kind_SINGLEPHASE = 1, + _Py_ext_module_kind_MULTIPHASE = 2, + _Py_ext_module_kind_INVALID = 3, +} _Py_ext_module_kind; + +typedef enum ext_module_origin { + _Py_ext_module_origin_CORE = 1, + _Py_ext_module_origin_BUILTIN = 2, + _Py_ext_module_origin_DYNAMIC = 3, +} _Py_ext_module_origin; + +/* Input for loading an extension module. */ +struct _Py_ext_module_loader_info { + PyObject *filename; +#ifndef MS_WINDOWS + PyObject *filename_encoded; +#endif + PyObject *name; + PyObject *name_encoded; + /* path is always a borrowed ref of name or filename, + * depending on if it's builtin or not. */ + PyObject *path; + _Py_ext_module_origin origin; + const char *hook_prefix; + const char *newcontext; +}; +extern void _Py_ext_module_loader_info_clear( + struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_init( + struct _Py_ext_module_loader_info *info, + PyObject *name, + PyObject *filename, + _Py_ext_module_origin origin); +extern int _Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +extern int _Py_ext_module_loader_info_init_for_builtin( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +extern int _Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *info, + PyObject *spec); + +/* The result from running an extension module's init function. */ +struct _Py_ext_module_loader_result { + PyModuleDef *def; + PyObject *module; + _Py_ext_module_kind kind; + struct _Py_ext_module_loader_result_error *err; + struct _Py_ext_module_loader_result_error { + enum _Py_ext_module_loader_result_error_kind { + _Py_ext_module_loader_result_EXCEPTION = 0, + _Py_ext_module_loader_result_ERR_MISSING = 1, + _Py_ext_module_loader_result_ERR_UNREPORTED_EXC = 2, + _Py_ext_module_loader_result_ERR_UNINITIALIZED = 3, + _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE = 4, + _Py_ext_module_loader_result_ERR_NOT_MODULE = 5, + _Py_ext_module_loader_result_ERR_MISSING_DEF = 6, + } kind; + PyObject *exc; + } _err; +}; +extern void _Py_ext_module_loader_result_clear( + struct _Py_ext_module_loader_result *res); +extern void _Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name); + +/* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +extern PyModInitFunction _PyImport_GetModInitFunc( + struct _Py_ext_module_loader_info *info, + FILE *fp); +extern int _PyImport_RunModInitFunc( + PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res); + /* Max length of module suffix searched for -- accommodates "module.slb" */ #define MAXSUFFIXSIZE 12 @@ -31,7 +109,7 @@ typedef FARPROC dl_funcptr; # define PYD_DEBUG_SUFFIX "" #endif -#ifdef Py_NOGIL +#ifdef Py_GIL_DISABLED # define PYD_THREADING_TAG "t" #else # define PYD_THREADING_TAG "" diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c86988234f6a05..6bf1b53bffd3ba 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be re-enabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h new file mode 100644 index 00000000000000..d6a79616db71fa --- /dev/null +++ b/Include/internal/pycore_instruction_sequence.h @@ -0,0 +1,73 @@ +#ifndef Py_INTERNAL_INSTRUCTION_SEQUENCE_H +#define Py_INTERNAL_INSTRUCTION_SEQUENCE_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_symtable.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct { + int h_label; + int h_startdepth; + int h_preserve_lasti; +} _PyExceptHandlerInfo; + +typedef struct { + int i_opcode; + int i_oparg; + _Py_SourceLocation i_loc; + _PyExceptHandlerInfo i_except_handler_info; + + /* Temporary fields, used by the assembler and in instr_sequence_to_cfg */ + int i_target; + int i_offset; +} _PyInstruction; + +typedef struct instruction_sequence { + PyObject_HEAD + _PyInstruction *s_instrs; + int s_allocated; + int s_used; + + int s_next_free_label; /* next free label id */ + + /* Map of a label id to instruction offset (index into s_instrs). + * If s_labelmap is NULL, then each label id is the offset itself. + */ + int *s_labelmap; + int s_labelmap_size; + + /* PyList of instruction sequences of nested functions */ + PyObject *s_nested; +} _PyInstructionSequence; + +typedef struct { + int id; +} _PyJumpTargetLabel; + +PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void); + +int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); +int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, + int opcode, int oparg, + _Py_SourceLocation loc); +_PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); +int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); +int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, + int opcode, int oparg, _Py_SourceLocation loc); +int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); +void PyInstructionSequence_Fini(_PyInstructionSequence *seq); + +extern PyTypeObject _PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUCTION_SEQUENCE_H */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 97dcfb9f8672f7..4e5b374968ea98 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -13,38 +13,6 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Local events. - * These require bytecode instrumentation */ - -#define PY_MONITORING_EVENT_PY_START 0 -#define PY_MONITORING_EVENT_PY_RESUME 1 -#define PY_MONITORING_EVENT_PY_RETURN 2 -#define PY_MONITORING_EVENT_PY_YIELD 3 -#define PY_MONITORING_EVENT_CALL 4 -#define PY_MONITORING_EVENT_LINE 5 -#define PY_MONITORING_EVENT_INSTRUCTION 6 -#define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 - -#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) - -/* Other events, mainly exceptions */ - -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 - - -/* Ancilliary events */ - -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 - - typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ @@ -55,7 +23,7 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_MONITORING_PROFILER_ID 2 #define PY_MONITORING_OPTIMIZER_ID 5 -/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +/* Internal IDs used to support sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 @@ -63,6 +31,8 @@ typedef uint32_t _PyMonitoringEventSet; PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events); extern int _Py_call_instrumentation(PyThreadState *tstate, int event, diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 60d333ad7baa2e..4a83862ac13e26 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -14,20 +14,26 @@ extern "C" { #include "pycore_atexit.h" // struct atexit_state #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache +#include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state +#include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state +#include "pycore_optimizer.h" // _PyOptimizerObject #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_qsbr.h" // struct _qsbr_state +#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -38,31 +44,45 @@ struct _Py_long_state { int max_str_digits; }; +// Support for stop-the-world events. This exists in both the PyRuntime struct +// for global pauses and in each PyInterpreterState for per-interpreter pauses. +struct _stoptheworld_state { + PyMutex mutex; // Serializes stop-the-world attempts. -/* cross-interpreter data registry */ + // NOTE: The below fields are protected by HEAD_LOCK(runtime), not by the + // above mutex. + bool requested; // Set when a pause is requested. + bool world_stopped; // Set when the world is stopped. + bool is_global; // Set when contained in PyRuntime struct. -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - /* This can be a dangling pointer, but only if weakref is set. */ - PyTypeObject *cls; - /* This is NULL for builtin types. */ - PyObject *weakref; - size_t refcount; - crossinterpdatafunc getdata; -}; + PyEvent stop_event; // Set when thread_countdown reaches zero. + Py_ssize_t thread_countdown; // Number of threads that must pause. -struct _xidregistry { - PyThread_type_lock mutex; - struct _xidregitem *head; + PyThreadState *requester; // Thread that requested the pause (may be NULL). }; +#ifdef Py_GIL_DISABLED +// This should be prime but otherwise the choice is arbitrary. A larger value +// increases concurrency at the expense of memory. +# define NUM_WEAKREF_LIST_LOCKS 127 +#endif + +/* cross-interpreter data registry */ + +/* Tracks some rare events per-interpreter, used by the optimizer to turn on/off + specific optimizations. */ +typedef struct _rare_events { + /* Setting an object's class, obj.__class__ = ... */ + uint8_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint8_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint8_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint8_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint8_t func_modification; +} _rare_events; /* interpreter state */ @@ -73,7 +93,7 @@ struct _xidregistry { */ struct _is { - /* This struct countains the eval_breaker, + /* This struct contains the eval_breaker, * which is by far the hottest field in this struct * and should be placed at the beginning. */ struct _ceval_state ceval; @@ -85,11 +105,23 @@ struct _is { int requires_idref; PyThread_type_lock id_mutex; +#define _PyInterpreterState_WHENCE_NOTSET -1 +#define _PyInterpreterState_WHENCE_UNKNOWN 0 +#define _PyInterpreterState_WHENCE_RUNTIME 1 +#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2 +#define _PyInterpreterState_WHENCE_CAPI 3 +#define _PyInterpreterState_WHENCE_XI 4 +#define _PyInterpreterState_WHENCE_STDLIB 5 +#define _PyInterpreterState_WHENCE_MAX 5 + long _whence; + /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right after allocation. */ int _initialized; + /* Has been fully initialized via pylifecycle.c. */ + int _ready; int finalizing; uintptr_t last_restart_version; @@ -100,7 +132,7 @@ struct _is { /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; /* Used in Modules/_threadmodule.c. */ - long count; + Py_ssize_t count; /* Support for runtime thread stack size tuning. A value of 0 means using the platform's default stack size or the size specified by the THREAD_STACK_SIZE macro. */ @@ -152,10 +184,7 @@ struct _is { possible to facilitate out-of-process observability tools. */ - PyObject *codec_search_path; - PyObject *codec_search_cache; - PyObject *codec_error_registry; - int codecs_initialized; + struct codecs_state codecs; PyConfig config; unsigned long feature_flags; @@ -174,8 +203,8 @@ struct _is { Py_ssize_t co_extra_user_count; freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry xidregistry; + /* cross-interpreter data and utils */ + struct _xi_state xi; #ifdef HAVE_FORK PyObject *before_forkers; @@ -185,8 +214,26 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; + struct _stoptheworld_state stoptheworld; + struct _qsbr_shared qsbr; - struct _obmalloc_state obmalloc; +#if defined(Py_GIL_DISABLED) + struct _mimalloc_interp_state mimalloc; + struct _brc_state brc; // biased reference counting state + PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; +#endif + + // Per-interpreter state for the obmalloc allocator. For the main + // interpreter and for all interpreters that don't have their + // own obmalloc state, this points to the static structure in + // obmalloc.c obmalloc_state_main. For other interpreters, it is + // heap allocated by _PyMem_init_obmalloc() and freed when the + // interpreter structure is freed. In the case of a heap allocated + // obmalloc state, it is not safe to hold on to or use memory after + // the interpreter is freed. The obmalloc state corresponding to + // that allocated memory is gone. See free_obmalloc_arenas() for + // more comments. + struct _obmalloc_state *obmalloc; PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; @@ -196,31 +243,25 @@ struct _is { struct _py_object_state object_state; struct _Py_unicode_state unicode; - struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; + struct _py_code_state code_state; - struct _Py_tuple_state tuple; - struct _Py_list_state list; struct _Py_dict_state dict_state; - struct _Py_async_gen_state async_gen; - struct _Py_context_state context; struct _Py_exc_state exc_state; + struct _Py_mem_interp_free_queue mem_free_queue; struct ast_state ast; struct types_state types; struct callable_cache callable_cache; _PyOptimizerObject *optimizer; - uint16_t optimizer_resume_threshold; - uint16_t optimizer_backedge_threshold; - uint32_t next_func_version; + _PyExecutorObject *executor_list_head; + + _rare_events rare_events; + PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; - bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ @@ -231,8 +272,8 @@ struct _is { struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; - /* the initial PyInterpreterState.threads.head */ - PyThreadState _initial_thread; + /* the initial PyInterpreterState.threads.head */ + _PyThreadStateImpl _initial_thread; Py_ssize_t _interactive_src_count; }; @@ -267,12 +308,21 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } -// Export for the _xxinterpchannels module. + +// Exports for the _testinternalcapi module. +PyAPI_FUNC(int64_t) _PyInterpreterState_ObjectToID(PyObject *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); +PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); + +PyAPI_FUNC(int) _PyInterpreterState_IsReady(PyInterpreterState *interp); -extern int _PyInterpreterState_IDInitref(PyInterpreterState *); -extern int _PyInterpreterState_IDIncref(PyInterpreterState *); -extern void _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp); +extern void _PyInterpreterState_SetWhence( + PyInterpreterState *interp, + long whence); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); @@ -349,6 +399,22 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( PyInterpreterState **pinterp); +#define RARE_EVENT_INTERP_INC(interp, name) \ + do { \ + /* saturating add */ \ + int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + if (val < UINT8_MAX) { \ + FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ + } \ + RARE_EVENT_STAT_INC(name); \ + } while (0); \ + +#define RARE_EVENT_INC(name) \ + do { \ + PyInterpreterState *interp = PyInterpreterState_Get(); \ + RARE_EVENT_INTERP_INC(interp, name); \ + } while (0); \ + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 3a8dd95cff8e5d..39c2a30f6e979d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -28,8 +28,9 @@ #define INTRINSIC_TYPEVAR_WITH_BOUND 2 #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 +#define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 -#define MAX_INTRINSIC_2 4 +#define MAX_INTRINSIC_2 5 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); @@ -44,7 +45,7 @@ typedef struct { const char *name; } intrinsic_func2_info; -extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; -extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; +PyAPI_DATA(const intrinsic_func1_info) _PyIntrinsics_UnaryFunctions[]; +PyAPI_DATA(const intrinsic_func2_info) _PyIntrinsics_BinaryFunctions[]; #endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h new file mode 100644 index 00000000000000..4d6cc35a7a3de7 --- /dev/null +++ b/Include/internal/pycore_jit.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_JIT_H +#define Py_INTERNAL_JIT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef _Py_JIT + +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); + +int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); +void _PyJIT_Free(_PyExecutorObject *executor); + +#endif // _Py_JIT + +#ifdef __cplusplus +} +#endif + +#endif // !Py_INTERNAL_JIT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 056be2c80c8ce6..73695d10e0c372 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,51 +8,32 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState -extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); - -/* runtime lifecycle */ - -extern void _PyList_Fini(PyInterpreterState *); - - -/* other API */ - -#ifndef WITH_FREELISTS -// without freelists -# define PyList_MAXFREELIST 0 -#endif - -/* Empty list reuse scheme to save calls to malloc and free */ -#ifndef PyList_MAXFREELIST -# define PyList_MAXFREELIST 80 -#endif - -struct _Py_list_state { -#if PyList_MAXFREELIST > 0 - PyListObject *free_list[PyList_MAXFREELIST]; - int numfree; -#endif -}; - #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) -extern int +PyAPI_FUNC(int) _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); +// In free-threaded build: self should be locked by the caller, if it should be thread-safe. static inline int _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) { assert(self != NULL && newitem != NULL); assert(PyList_Check(self)); - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { - Py_SET_SIZE(self, len + 1); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&self->ob_item[len], newitem); +#else PyList_SET_ITEM(self, len, newitem); +#endif + Py_SET_SIZE(self, len + 1); return 0; } return _PyList_AppendTakeRefListResize(self, newitem); @@ -77,7 +58,7 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } _PyListIterObject; -extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); #ifdef __cplusplus } diff --git a/Include/internal/pycore_llist.h b/Include/internal/pycore_llist.h index 5fd261da05fa5d..f629902fda9ff1 100644 --- a/Include/internal/pycore_llist.h +++ b/Include/internal/pycore_llist.h @@ -37,8 +37,7 @@ struct llist_node { }; // Get the struct containing a node. -#define llist_data(node, type, member) \ - (type*)((char*)node - offsetof(type, member)) +#define llist_data(node, type, member) (_Py_CONTAINER_OF(node, type, member)) // Iterate over a list. #define llist_for_each(node, head) \ diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index c4bb76a40e7b12..e6da083b807ce5 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,68 +13,29 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - - -// A mutex that occupies one byte. The lock can be zero initialized. -// -// Only the two least significant bits are used. The remaining bits should be -// zero: -// 0b00: unlocked -// 0b01: locked -// 0b10: unlocked and has parked threads -// 0b11: locked and has parked threads -// -// Typical initialization: -// PyMutex m = (PyMutex){0}; -// -// Typical usage: -// PyMutex_Lock(&m); -// ... -// PyMutex_Unlock(&m); -typedef struct _PyMutex { - uint8_t v; -} PyMutex; - -#define _Py_UNLOCKED 0 -#define _Py_LOCKED 1 +//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h #define _Py_HAS_PARKED 2 +#define _Py_ONCE_INITIALIZED 4 -// (private) slow path for locking the mutex -PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); - -// (private) slow path for unlocking the mutex -PyAPI_FUNC(void) _PyMutex_UnlockSlow(PyMutex *m); - -// Locks the mutex. -// -// If the mutex is currently locked, the calling thread will be parked until -// the mutex is unlocked. If the current thread holds the GIL, then the GIL -// will be released while the thread is parked. -static inline void -PyMutex_Lock(PyMutex *m) +static inline int +PyMutex_LockFast(uint8_t *lock_bits) { uint8_t expected = _Py_UNLOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { - _PyMutex_LockSlow(m); - } -} - -// Unlocks the mutex. -static inline void -PyMutex_Unlock(PyMutex *m) -{ - uint8_t expected = _Py_LOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_UNLOCKED)) { - _PyMutex_UnlockSlow(m); - } + return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } // Checks if the mutex is currently locked. static inline int PyMutex_IsLocked(PyMutex *m) { - return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; +} + +// Re-initializes the mutex after a fork to the unlocked state. +static inline void +_PyMutex_at_fork_reinit(PyMutex *m) +{ + memset(m, 0, sizeof(*m)); } typedef enum _PyLockFlags { @@ -91,7 +52,17 @@ typedef enum _PyLockFlags { // Lock a mutex with an optional timeout and additional options. See // _PyLockFlags for details. extern PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); + +// Lock a mutex with additional options. See _PyLockFlags for details. +static inline void +PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { + _PyMutex_LockTimed(m, -1, flags); + } +} // Unlock a mutex, returns 0 if the mutex is not locked (used for improved // error messages). @@ -103,6 +74,10 @@ typedef struct { uint8_t v; } PyEvent; +// Check if the event is set without blocking. Returns 1 if the event is set or +// 0 otherwise. +PyAPI_FUNC(int) _PyEvent_IsSet(PyEvent *evt); + // Set the event and notify any waiting threads. // Export for '_testinternalcapi' shared extension PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); @@ -113,9 +88,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns); - +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot @@ -152,6 +128,113 @@ _PyRawMutex_Unlock(_PyRawMutex *m) _PyRawMutex_UnlockSlow(m); } +// Type signature for one-time initialization functions. The function should +// return 0 on success and -1 on failure. +typedef int _Py_once_fn_t(void *arg); + +// (private) slow path for one time initialization +PyAPI_FUNC(int) +_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg); + +// Calls `fn` once using `flag`. The `arg` is passed to the call to `fn`. +// +// Returns 0 on success and -1 on failure. +// +// If `fn` returns 0 (success), then subsequent calls immediately return 0. +// If `fn` returns -1 (failure), then subsequent calls will retry the call. +static inline int +_PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) +{ + if (_Py_atomic_load_uint8(&flag->v) == _Py_ONCE_INITIALIZED) { + return 0; + } + return _PyOnceFlag_CallOnceSlow(flag, fn, arg); +} + +// A recursive mutex. The mutex should zero-initialized. +typedef struct { + PyMutex mutex; + unsigned long long thread; // i.e., PyThread_get_thread_ident_ex() + size_t level; +} _PyRecursiveMutex; + +PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); + + +// A readers-writer (RW) lock. The lock supports multiple concurrent readers or +// a single writer. The lock is write-preferring: if a writer is waiting while +// the lock is read-locked then, new readers will be blocked. This avoids +// starvation of writers. +// +// In C++, the equivalent synchronization primitive is std::shared_mutex +// with shared ("read") and exclusive ("write") locking. +// +// The two least significant bits are used to indicate if the lock is +// write-locked and if there are parked threads (either readers or writers) +// waiting to acquire the lock. The remaining bits are used to indicate the +// number of readers holding the lock. +// +// 0b000..00000: unlocked +// 0bnnn..nnn00: nnn..nnn readers holding the lock +// 0bnnn..nnn10: nnn..nnn readers holding the lock and a writer is waiting +// 0b00000..010: unlocked with awoken writer about to acquire lock +// 0b00000..001: write-locked +// 0b00000..011: write-locked and readers or other writers are waiting +// +// Note that reader_count must be zero if the lock is held by a writer, and +// vice versa. The lock can only be held by readers or a writer, but not both. +// +// The design is optimized for simplicity of the implementation. The lock is +// not fair: if fairness is desired, use an additional PyMutex to serialize +// writers. The lock is also not reentrant. +typedef struct { + uintptr_t bits; +} _PyRWMutex; + +// Read lock (i.e., shared lock) +PyAPI_FUNC(void) _PyRWMutex_RLock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); + +// Write lock (i.e., exclusive lock) +PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); + +// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock +// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd +// sequence means we're locked. Readers will read the sequence before attempting to read the +// underlying data and then read the sequence number again after reading the data. If the +// sequence has not changed the data is valid. +// +// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock. +// The writer can also detect that the undelering data has not changed and abandon the write +// and restore the previous sequence. +typedef struct { + uint32_t sequence; +} _PySeqLock; + +// Lock the sequence lock for the writer +PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); + +// Unlock the sequence lock and move to the next sequence number. +PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); + +// Abandon the current update indicating that no mutations have occurred +// and restore the previous sequence value. +PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); + +// Begin a read operation and return the current sequence number. +PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); + +// End the read operation and confirm that the sequence number has not changed. +// Returns 1 if the read was successful or 0 if the read should be retried. +PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); + +// Check if the lock was held during a fork and clear the lock. Returns 1 +// if the lock was held and any associated data should be cleared. +PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3c253ed7ff556b..2bf6ff459ddc96 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,35 +47,6 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif -extern PyLongObject* _PyLong_New(Py_ssize_t); - -// Return a copy of src. -extern PyObject* _PyLong_Copy(PyLongObject *src); - -// Export for '_decimal' shared extension -PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( - int negative, - Py_ssize_t digit_count, - digit *digits); - -// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. -// v must not be NULL, and must be a normalized long. -// There are no error cases. -// -// Export for '_pickle' shared extension. -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - -// _PyLong_NumBits. Return the number of bits needed to represent the -// absolute value of a long. For example, this returns 1 for 1 and -1, 2 -// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. -// v must not be NULL, and must be a normalized long. -// (size_t)-1 is returned and OverflowError set if the true result doesn't -// fit in a size_t. -// -// Export for 'math' shared extension. -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - - /* runtime lifecycle */ extern PyStatus _PyLong_InitTypes(PyInterpreterState *); @@ -128,66 +99,20 @@ extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); // Export for '_datetime' shared extension. PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); -// _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in -// base 256, and return a Python int with the same numeric value. -// If n is 0, the integer is 0. Else: -// If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; -// else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the -// LSB. -// If is_signed is 0/false, view the bytes as a non-negative integer. -// If is_signed is 1/true, view the bytes as a 2's-complement integer, -// non-negative if bit 0x80 of the MSB is clear, negative if set. -// Error returns: -// + Return NULL with the appropriate exception set if there's not -// enough memory to create the Python int. -// -// Export for '_multibytecodec' shared extension. -PyAPI_DATA(PyObject*) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -// _PyLong_AsByteArray: Convert the least-significant 8*n bits of long -// v to a base-256 integer, stored in array bytes. Normally return 0, -// return -1 on error. -// If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at -// bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and -// the LSB at bytes[n-1]. -// If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes -// are filled and there's nothing special about bit 0x80 of the MSB. -// If is_signed is 1/true, bytes is filled with the 2's-complement -// representation of v's value. Bit 0x80 of the MSB is the sign bit. -// Error returns (-1): -// + is_signed is 0 and v < 0. TypeError is set in this case, and bytes -// isn't altered. -// + n isn't big enough to hold the full mathematical value of v. For -// example, if is_signed is 0 and there are more digits in the v than -// fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of -// being large enough to hold a sign bit. OverflowError is set in this -// case, but bytes holds the least-significant n bytes of the true value. -// -// Export for '_struct' shared extension. -PyAPI_DATA(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); - // _PyLong_Format: Convert the long to a string object with given base, // appending a base prefix of 0[box] if base is 2, 8 or 16. // Export for '_tkinter' shared extension. PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); -// For use by the math.gcd() function. -// Export for 'math' shared extension. -PyAPI_DATA(PyObject*) _PyLong_GCD(PyObject *, PyObject *); - // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); -extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -315,7 +240,7 @@ static inline int _PyLong_CompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(_PyLong_IsCompact(op)); + assert(_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -323,7 +248,7 @@ static inline int _PyLong_NonCompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(!_PyLong_IsCompact(op)); + assert(!_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -363,7 +288,7 @@ _PyLong_FlipSign(PyLongObject *op) { #define _PyLong_DIGIT_INIT(val) \ { \ - .ob_base = _PyObject_HEAD_INIT(&PyLong_Type) \ + .ob_base = _PyObject_HEAD_INIT(&PyLong_Type), \ .long_value = { \ .lv_tag = TAG_FROM_SIGN_AND_SIZE( \ (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \ diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h new file mode 100644 index 00000000000000..d10b01d5b49b19 --- /dev/null +++ b/Include/internal/pycore_mimalloc.h @@ -0,0 +1,60 @@ +#ifndef Py_INTERNAL_MIMALLOC_H +#define Py_INTERNAL_MIMALLOC_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#if defined(MIMALLOC_H) || defined(MIMALLOC_TYPES_H) +# error "pycore_mimalloc.h must be included before mimalloc.h" +#endif + +typedef enum { + _Py_MIMALLOC_HEAP_MEM = 0, // PyMem_Malloc() and friends + _Py_MIMALLOC_HEAP_OBJECT = 1, // non-GC objects + _Py_MIMALLOC_HEAP_GC = 2, // GC objects without pre-header + _Py_MIMALLOC_HEAP_GC_PRE = 3, // GC objects with pre-header + _Py_MIMALLOC_HEAP_COUNT +} _Py_mimalloc_heap_id; + +#include "pycore_pymem.h" + +#ifdef WITH_MIMALLOC +# ifdef Py_GIL_DISABLED +# define MI_PRIM_THREAD_ID _Py_ThreadId +# endif +# define MI_DEBUG_UNINIT PYMEM_CLEANBYTE +# define MI_DEBUG_FREED PYMEM_DEADBYTE +# define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +#ifdef Py_DEBUG +# define MI_DEBUG 2 +#else +# define MI_DEBUG 0 +#endif + +#ifdef _Py_THREAD_SANITIZER +# define MI_TSAN 1 +#endif + +#include "mimalloc/mimalloc.h" +#include "mimalloc/mimalloc/types.h" +#include "mimalloc/mimalloc/internal.h" +#endif + +#ifdef Py_GIL_DISABLED +struct _mimalloc_interp_state { + // When exiting, threads place any segments with live blocks in this + // shared pool for other threads to claim and reuse. + mi_abandoned_pool_t abandoned_pool; +}; + +struct _mimalloc_thread_state { + mi_heap_t *current_object_heap; + mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; + mi_tld_t tld; + int initialized; + struct llist_node page_list; +}; +#endif + +#endif // Py_INTERNAL_MIMALLOC_H diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index e577c6ba856b77..11fde814875938 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODSUPPORT_H #define Py_INTERNAL_MODSUPPORT_H + +#include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -13,6 +16,24 @@ extern int _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); #define _PyArg_NoKwnames(funcname, kwnames) \ ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) +// Export for '_bz2' shared extension +PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); +#define _PyArg_NoPositional(funcname, args) \ + ((args) == NULL || _PyArg_NoPositional((funcname), (args))) + +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); +#define _PyArg_NoKeywords(funcname, kwargs) \ + ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) + +// Export for 'zlib' shared extension +PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, + Py_ssize_t, Py_ssize_t); +#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) +#define _PyArg_CheckPositional(funcname, nargs, min, max) \ + ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ + || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) + extern PyObject ** _Py_VaBuildStack( PyObject **small_stack, Py_ssize_t small_stack_len, @@ -22,6 +43,63 @@ extern PyObject ** _Py_VaBuildStack( extern PyObject* _PyModule_CreateInitialized(PyModuleDef*, int apiver); +// Export for '_curses' shared extension +PyAPI_FUNC(int) _PyArg_ParseStack( + PyObject *const *args, + Py_ssize_t nargs, + const char *format, + ...); + +extern int _PyArg_UnpackStack( + PyObject *const *args, + Py_ssize_t nargs, + const char *name, + Py_ssize_t min, + Py_ssize_t max, + ...); + +// Export for '_heapq' shared extension +PyAPI_FUNC(void) _PyArg_BadArgument( + const char *fname, + const char *displayname, + const char *expected, + PyObject *arg); + +// --- _PyArg_Parser API --------------------------------------------------- + +// Export for '_dbm' shared extension +PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames, + struct _PyArg_Parser *, + ...); + +// Export for 'math' shared extension +PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwargs, + PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, + int maxpos, + int minkw, + PyObject **buf); +#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ + (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ + (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ + _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ + (minpos), (maxpos), (minkw), (buf))) + +// Export for '_testclinic' shared extension +PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, int maxpos, int minkw, + int vararg, PyObject **buf); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 5644bbe5e0552b..049677b292e235 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -22,6 +22,9 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; +#ifdef Py_GIL_DISABLED + void *md_gil; +#endif } PyModuleObject; static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2d50f42c9c614d..d385b20b0e964d 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() #include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() /* Check if an object is consistent. For example, ensure that the reference @@ -54,18 +55,26 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); Furthermore, we can't use designated initializers in Extensions since these are not supported pre-C++20. Thus, keeping an internal copy here is the most backwards compatible solution */ +#if defined(Py_GIL_DISABLED) +#define _PyObject_HEAD_INIT(type) \ + { \ + .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ + .ob_type = (type) \ + } +#else #define _PyObject_HEAD_INIT(type) \ { \ .ob_refcnt = _Py_IMMORTAL_REFCNT, \ .ob_type = (type) \ - }, + } +#endif #define _PyVarObject_HEAD_INIT(type, size) \ { \ - .ob_base = _PyObject_HEAD_INIT(type) \ + .ob_base = _PyObject_HEAD_INIT(type), \ .ob_size = size \ - }, + } -extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -78,9 +87,9 @@ extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( built against the pre-3.12 stable ABI. */ PyAPI_DATA(Py_ssize_t) _Py_RefTotal; -extern void _Py_AddRefTotal(PyInterpreterState *, Py_ssize_t); -extern void _Py_IncRefTotal(PyInterpreterState *); -extern void _Py_DecRefTotal(PyInterpreterState *); +extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); +extern void _Py_IncRefTotal(PyThreadState *); +extern void _Py_DecRefTotal(PyThreadState *); # define _Py_DEC_REFTOTAL(interp) \ interp->object_state.reftotal-- @@ -93,26 +102,54 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) return; } #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), n); + _Py_AddRefTotal(_PyThreadState_GET(), n); #endif +#if !defined(Py_GIL_DISABLED) op->ob_refcnt += n; +#else + if (_Py_IsOwnedByCurrentThread(op)) { + uint32_t local = op->ob_ref_local; + Py_ssize_t refcnt = (Py_ssize_t)local + n; +# if PY_SSIZE_T_MAX > UINT32_MAX + if (refcnt > (Py_ssize_t)UINT32_MAX) { + // Make the object immortal if the 32-bit local reference count + // would overflow. + refcnt = _Py_IMMORTAL_REFCNT_LOCAL; + } +# endif + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); + } +#endif } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) -static inline void _Py_SetImmortal(PyObject *op) +PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op); +PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); + +// Makes an immortal object mortal again with the specified refcnt. Should only +// be used during runtime finalization. +static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) { if (op) { - op->ob_refcnt = _Py_IMMORTAL_REFCNT; + assert(_Py_IsImmortal(op)); +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); +#else + op->ob_refcnt = refcnt; +#endif } } -#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) /* _Py_ClearImmortal() should only be used during runtime finalization. */ static inline void _Py_ClearImmortal(PyObject *op) { if (op) { - assert(op->ob_refcnt == _Py_IMMORTAL_REFCNT); - op->ob_refcnt = 1; + _Py_SetMortal(op, 1); Py_DECREF(op); } } @@ -122,6 +159,7 @@ static inline void _Py_ClearImmortal(PyObject *op) op = NULL; \ } while (0) +#if !defined(Py_GIL_DISABLED) static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { @@ -130,7 +168,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -151,7 +189,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); + _Py_DEC_REFTOTAL(PyInterpreterState_Get()); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -161,6 +199,37 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) #endif } +#else +// TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build +static inline void +_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) +{ + Py_DECREF(op); +} + +static inline void +_Py_DECREF_NO_DEALLOC(PyObject *op) +{ + Py_DECREF(op); +} + +static inline int +_Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) +{ + return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; +} + +static inline int +_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) +{ + return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; +} + +// Merge the local and shared reference count fields and add `extra` to the +// refcount when merging. +Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); +#endif // !defined(Py_GIL_DISABLED) + #ifdef Py_REF_DEBUG # undef _Py_DEC_REFTOTAL #endif @@ -173,12 +242,12 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); +extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((type->tp_flags & feature) != 0); + return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); } extern void _PyType_InitCache(PyInterpreterState *interp); @@ -197,9 +266,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { - Py_INCREF(typeobj); - } + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + Py_INCREF(typeobj); _Py_NewReference(op); } @@ -236,7 +304,9 @@ static inline void _PyObject_GC_TRACK( _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), "object already tracked by the garbage collector", filename, lineno, __func__); - +#ifdef Py_GIL_DISABLED + _PyObject_SET_GC_BITS(op, _PyGC_BITS_TRACKED); +#else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, @@ -244,12 +314,14 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = interp->gc.generation0; + PyGC_Head *generation0 = &interp->gc.young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, generation0); + /* Young objects will be moved into the visited space during GC, so set the bit here */ + gc->_gc_next = ((uintptr_t)generation0) | interp->gc.visited_space; generation0->_gc_prev = (uintptr_t)gc; +#endif } /* Tell the GC to stop tracking this object. @@ -273,6 +345,9 @@ static inline void _PyObject_GC_UNTRACK( "object not tracked by the garbage collector", filename, lineno, __func__); +#ifdef Py_GIL_DISABLED + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_TRACKED); +#else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); PyGC_Head *next = _PyGCHead_NEXT(gc); @@ -280,6 +355,7 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; +#endif } // Macros to accept any type for the parameter, and to automatically pass @@ -297,6 +373,179 @@ static inline void _PyObject_GC_UNTRACK( _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op)) #endif +#ifdef Py_GIL_DISABLED + +/* Tries to increment an object's reference count + * + * This is a specialized version of _Py_TryIncref that only succeeds if the + * object is immortal or local to this thread. It does not handle the case + * where the reference count modification requires an atomic operation. This + * allows call sites to specialize for the immortal/local case. + */ +static inline int +_Py_TryIncrefFast(PyObject *op) { + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + local += 1; + if (local == 0) { + // immortal + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_INCREF_STAT_INC(); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + return 1; + } + return 0; +} + +static inline int +_Py_TryIncRefShared(PyObject *op) +{ + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + for (;;) { + // If the shared refcount is zero and the object is either merged + // or may not have weak references, then we cannot incref it. + if (shared == 0 || shared == _Py_REF_MERGED) { + return 0; + } + + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + shared + (1 << _Py_REF_SHARED_SHIFT))) { +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + _Py_INCREF_STAT_INC(); + return 1; + } + } +} + +/* Tries to incref the object op and ensures that *src still points to it. */ +static inline int +_Py_TryIncrefCompare(PyObject **src, PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return 1; + } + if (!_Py_TryIncRefShared(op)) { + return 0; + } + if (op != _Py_atomic_load_ptr(src)) { + Py_DECREF(op); + return 0; + } + return 1; +} + +/* Loads and increfs an object from ptr, which may contain a NULL value. + Safe with concurrent (atomic) updates to ptr. + NOTE: The writer must set maybe-weakref on the stored object! */ +static inline PyObject * +_Py_XGetRef(PyObject **ptr) +{ + for (;;) { + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncrefCompare(ptr, value)) { + return value; + } + } +} + +/* Attempts to loads and increfs an object from ptr. Returns NULL + on failure, which may be due to a NULL value or a concurrent update. */ +static inline PyObject * +_Py_TryXGetRef(PyObject **ptr) +{ + PyObject *value = _Py_atomic_load_ptr(ptr); + if (value == NULL) { + return value; + } + if (_Py_TryIncrefCompare(ptr, value)) { + return value; + } + return NULL; +} + +/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF + on objects owned by a different thread. */ +static inline PyObject * +_Py_NewRefWithLock(PyObject *op) +{ + if (_Py_TryIncrefFast(op)) { + return op; + } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + _Py_INCREF_STAT_INC(); + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT); + if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) { + new_shared |= _Py_REF_MAYBE_WEAKREF; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, + &shared, + new_shared)) { + return op; + } + } +} + +static inline PyObject * +_Py_XNewRefWithLock(PyObject *obj) +{ + if (obj == NULL) { + return NULL; + } + return _Py_NewRefWithLock(obj); +} + +static inline void +_PyObject_SetMaybeWeakref(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +} + +#endif + +/* Tries to incref op and returns 1 if successful or 0 otherwise. */ +static inline int +_Py_TryIncref(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); +#else + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; +#endif +} + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); @@ -325,7 +574,7 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState( + managed_static_type_state *state = _PyStaticType_GetState( interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } @@ -355,7 +604,6 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) return (PyWeakReference **)((char *)op + offset); } - // Fast inlined version of PyObject_IS_GC() static inline int _PyObject_IS_GC(PyObject *obj) @@ -365,14 +613,32 @@ _PyObject_IS_GC(PyObject *obj) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } +// Fast inlined version of PyObject_Hash() +static inline Py_hash_t +_PyObject_HashFast(PyObject *op) +{ + if (PyUnicode_CheckExact(op)) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( + _PyASCIIObject_CAST(op)->hash); + if (hash != -1) { + return hash; + } + } + return PyObject_Hash(op); +} + // Fast inlined version of PyType_IS_GC() #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { - return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *); + return ( +#ifndef Py_GIL_DISABLED + _PyType_IS_GC(tp) * sizeof(PyGC_Head) + +#endif + _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + ); } void _PyObject_GC_Link(PyObject *op); @@ -394,65 +660,59 @@ extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); +extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); + +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, + PyObject *name, PyObject *value); +extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, + PyObject **attr); -extern int _PyObject_InitializeDict(PyObject *obj); -int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); -extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name, PyObject *value); -PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name); +#ifdef Py_GIL_DISABLED +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) +#else +# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) +# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) +#endif typedef union { - PyObject *dict; - /* Use a char* to generate a warning if directly assigning a PyDictValues */ - char *values; -} PyDictOrValues; + PyDictObject *dict; +} PyManagedDictPointer; -static inline PyDictOrValues * -_PyObject_DictOrValuesPointer(PyObject *obj) +static inline PyManagedDictPointer * +_PyObject_ManagedDictPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return ((PyDictOrValues *)obj)-3; + return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } -static inline int -_PyDictOrValues_IsValues(PyDictOrValues dorv) +static inline PyDictObject * +_PyObject_GetManagedDict(PyObject *obj) { - return ((uintptr_t)dorv.values) & 1; + PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); + return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); } static inline PyDictValues * -_PyDictOrValues_GetValues(PyDictOrValues dorv) -{ - assert(_PyDictOrValues_IsValues(dorv)); - return (PyDictValues *)(dorv.values + 1); -} - -static inline PyObject * -_PyDictOrValues_GetDict(PyDictOrValues dorv) -{ - assert(!_PyDictOrValues_IsValues(dorv)); - return dorv.dict; -} - -static inline void -_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) +_PyObject_InlineValues(PyObject *obj) { - ptr->values = ((char *)values) - 1; + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject)); + return (PyDictValues *)((char *)obj + sizeof(PyObject)); } -#define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) - extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); -extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null); extern int _PyObject_IsAbstract(PyObject *); -extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); extern PyObject* _PyObject_NextNotImplemented(PyObject *); // Pickle support. @@ -473,13 +733,7 @@ PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); * Third party code unintentionally rely on problematic fpcasts. The call * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#define _PyCFunction_TrampolineCall(meth, self, args) \ - _PyCFunctionWithKeywords_TrampolineCall( \ - (*(PyCFunctionWithKeywords)(void(*)(void))(meth)), (self), (args), NULL) -extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( - PyCFunctionWithKeywords meth, PyObject *, PyObject *, PyObject *); -#else +#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) #define _PyCFunction_TrampolineCall(meth, self, args) \ (meth)((self), (args)) #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ @@ -494,6 +748,8 @@ PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; // Export for the stable ABI. PyAPI_DATA(int) _Py_SwappedOp[]; +extern void _Py_GetConstant_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object_alloc.h b/Include/internal/pycore_object_alloc.h new file mode 100644 index 00000000000000..8cc7a444bc93e7 --- /dev/null +++ b/Include/internal/pycore_object_alloc.h @@ -0,0 +1,71 @@ +#ifndef Py_INTERNAL_OBJECT_ALLOC_H +#define Py_INTERNAL_OBJECT_ALLOC_H + +#include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED +static inline mi_heap_t * +_PyObject_GetAllocationHeap(_PyThreadStateImpl *tstate, PyTypeObject *tp) +{ + struct _mimalloc_thread_state *m = &tstate->mimalloc; + if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) { + return &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE]; + } + else if (_PyType_IS_GC(tp)) { + return &m->heaps[_Py_MIMALLOC_HEAP_GC]; + } + else { + return &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; + } +} +#endif + +// Sets the heap used for PyObject_Malloc(), PyObject_Realloc(), etc. calls in +// Py_GIL_DISABLED builds. We use different heaps depending on if the object +// supports GC and if it has a pre-header. We smuggle the choice of heap +// through the _mimalloc_thread_state. In the default build, this simply +// calls PyObject_Malloc(). +static inline void * +_PyObject_MallocWithType(PyTypeObject *tp, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _mimalloc_thread_state *m = &tstate->mimalloc; + m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp); +#endif + void *mem = PyObject_Malloc(size); +#ifdef Py_GIL_DISABLED + m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif + return mem; +} + +static inline void * +_PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _mimalloc_thread_state *m = &tstate->mimalloc; + m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp); +#endif + void *mem = PyObject_Realloc(ptr, size); +#ifdef Py_GIL_DISABLED + m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif + return mem; +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_ALLOC_H diff --git a/Include/internal/pycore_object_deferred.h b/Include/internal/pycore_object_deferred.h new file mode 100644 index 00000000000000..c070d768b7d771 --- /dev/null +++ b/Include/internal/pycore_object_deferred.h @@ -0,0 +1,32 @@ +#ifndef Py_INTERNAL_OBJECT_DEFERRED_H +#define Py_INTERNAL_OBJECT_DEFERRED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "pycore_gc.h" + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Mark an object as supporting deferred reference counting. This is a no-op +// in the default (with GIL) build. Objects that use deferred reference +// counting should be tracked by the GC so that they are eventually collected. +extern void _PyObject_SetDeferredRefcount(PyObject *op); + +static inline int +_PyObject_HasDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); +#else + return 0; +#endif +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_DEFERRED_H diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h new file mode 100644 index 00000000000000..fc130b1e9920b4 --- /dev/null +++ b/Include/internal/pycore_object_stack.h @@ -0,0 +1,87 @@ +#ifndef Py_INTERNAL_OBJECT_STACK_H +#define Py_INTERNAL_OBJECT_STACK_H + +#include "pycore_freelist.h" // _PyFreeListState + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// _PyObjectStack is a stack of Python objects implemented as a linked list of +// fixed size buffers. + +// Chosen so that _PyObjectStackChunk is a power-of-two size. +#define _Py_OBJECT_STACK_CHUNK_SIZE 254 + +typedef struct _PyObjectStackChunk { + struct _PyObjectStackChunk *prev; + Py_ssize_t n; + PyObject *objs[_Py_OBJECT_STACK_CHUNK_SIZE]; +} _PyObjectStackChunk; + +typedef struct _PyObjectStack { + _PyObjectStackChunk *head; +} _PyObjectStack; + + +extern _PyObjectStackChunk * +_PyObjectStackChunk_New(void); + +extern void +_PyObjectStackChunk_Free(_PyObjectStackChunk *); + +// Push an item onto the stack. Return -1 on allocation failure, 0 on success. +static inline int +_PyObjectStack_Push(_PyObjectStack *stack, PyObject *obj) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL || buf->n == _Py_OBJECT_STACK_CHUNK_SIZE) { + buf = _PyObjectStackChunk_New(); + if (buf == NULL) { + return -1; + } + buf->prev = stack->head; + buf->n = 0; + stack->head = buf; + } + + assert(buf->n >= 0 && buf->n < _Py_OBJECT_STACK_CHUNK_SIZE); + buf->objs[buf->n] = obj; + buf->n++; + return 0; +} + +// Pop the top item from the stack. Return NULL if the stack is empty. +static inline PyObject * +_PyObjectStack_Pop(_PyObjectStack *stack) +{ + _PyObjectStackChunk *buf = stack->head; + if (buf == NULL) { + return NULL; + } + assert(buf->n > 0 && buf->n <= _Py_OBJECT_STACK_CHUNK_SIZE); + buf->n--; + PyObject *obj = buf->objs[buf->n]; + if (buf->n == 0) { + stack->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } + return obj; +} + +// Merge src into dst, leaving src empty +extern void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); + +// Remove all items from the stack +extern void +_PyObjectStack_Clear(_PyObjectStack *stack); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_STACK_H diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 9eac27b1a9a4e3..cd7c9335b3e611 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyObject_freelists #include "pycore_hashtable.h" // _Py_hashtable_t struct _py_object_runtime_state { @@ -18,6 +19,9 @@ struct _py_object_runtime_state { }; struct _py_object_state { +#if !defined(Py_GIL_DISABLED) + struct _Py_object_freelists freelists; +#endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; #endif diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index b0dbf53d4e3d15..9140d8f08f0af1 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -665,7 +665,9 @@ struct _obmalloc_global_state { struct _obmalloc_state { struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; +#if WITH_PYMALLOC_RADIX_TREE struct _obmalloc_usage usage; +#endif }; @@ -684,6 +686,8 @@ extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void); _Py_GetGlobalAllocatedBlocks() extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *); extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); +extern int _PyMem_init_obmalloc(PyInterpreterState *interp); +extern bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp); #ifdef WITH_PYMALLOC diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index 8ee72ff2d4126f..e6811b7aeca73c 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -59,13 +59,6 @@ extern "C" { .dump_debug_stats = -1, \ } -#define _obmalloc_state_INIT(obmalloc) \ - { \ - .pools = { \ - .used = _obmalloc_pools_INIT(obmalloc.pools), \ - }, \ - } - #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 926c0041c34c28..7b495238d7a9f3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1,22 +1,25 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/opcode_metadata_generator.py // from: // Python/bytecodes.c // Do not edit! +#ifndef Py_CORE_OPCODE_METADATA_H +#define Py_CORE_OPCODE_METADATA_H +#ifdef __cplusplus +extern "C" { +#endif + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif #include <stdbool.h> // bool +#include "opcode_ids.h" #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ ((OP) == STORE_FAST_MAYBE_NULL) || \ - ((OP) == LOAD_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_ATTR) || \ - ((OP) == LOAD_METHOD) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ ((OP) == SETUP_FINALLY) || \ @@ -25,697 +28,417 @@ ((OP) == POP_BLOCK) || \ 0) -#define _EXIT_TRACE 300 -#define _SET_IP 301 -#define _GUARD_BOTH_INT 302 -#define _BINARY_OP_MULTIPLY_INT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_SUBTRACT_INT 305 -#define _GUARD_BOTH_FLOAT 306 -#define _BINARY_OP_MULTIPLY_FLOAT 307 -#define _BINARY_OP_ADD_FLOAT 308 -#define _BINARY_OP_SUBTRACT_FLOAT 309 -#define _GUARD_BOTH_UNICODE 310 -#define _BINARY_OP_ADD_UNICODE 311 -#define _BINARY_OP_INPLACE_ADD_UNICODE 312 -#define _POP_FRAME 313 -#define _GUARD_GLOBALS_VERSION 314 -#define _GUARD_BUILTINS_VERSION 315 -#define _LOAD_GLOBAL_MODULE 316 -#define _LOAD_GLOBAL_BUILTINS 317 -#define _GUARD_TYPE_VERSION 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _LOAD_ATTR_INSTANCE_VALUE 320 -#define _CHECK_ATTR_MODULE 321 -#define _LOAD_ATTR_MODULE 322 -#define _CHECK_ATTR_WITH_HINT 323 -#define _LOAD_ATTR_WITH_HINT 324 -#define _LOAD_ATTR_SLOT 325 -#define _CHECK_ATTR_CLASS 326 -#define _LOAD_ATTR_CLASS 327 -#define _GUARD_DORV_VALUES 328 -#define _STORE_ATTR_INSTANCE_VALUE 329 -#define _STORE_ATTR_SLOT 330 -#define _IS_NONE 331 -#define _ITER_CHECK_LIST 332 -#define _ITER_JUMP_LIST 333 -#define _IS_ITER_EXHAUSTED_LIST 334 -#define _ITER_NEXT_LIST 335 -#define _ITER_CHECK_TUPLE 336 -#define _ITER_JUMP_TUPLE 337 -#define _IS_ITER_EXHAUSTED_TUPLE 338 -#define _ITER_NEXT_TUPLE 339 -#define _ITER_CHECK_RANGE 340 -#define _ITER_JUMP_RANGE 341 -#define _IS_ITER_EXHAUSTED_RANGE 342 -#define _ITER_NEXT_RANGE 343 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344 -#define _GUARD_KEYS_VERSION 345 -#define _LOAD_ATTR_METHOD_WITH_VALUES 346 -#define _LOAD_ATTR_METHOD_NO_DICT 347 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 348 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 349 -#define _CHECK_ATTR_METHOD_LAZY_DICT 350 -#define _LOAD_ATTR_METHOD_LAZY_DICT 351 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 352 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 353 -#define _CHECK_PEP_523 354 -#define _CHECK_FUNCTION_EXACT_ARGS 355 -#define _CHECK_STACK_SPACE 356 -#define _INIT_CALL_PY_EXACT_ARGS 357 -#define _PUSH_FRAME 358 -#define _POP_JUMP_IF_FALSE 359 -#define _POP_JUMP_IF_TRUE 360 -#define _JUMP_TO_TOP 361 -#define _SAVE_CURRENT_IP 362 -#define _INSERT 363 - -extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); +#include "pycore_uop_ids.h" +extern int _PyOpcode_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { +int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { - case NOP: - return 0; - case RESUME: - return 0; - case RESUME_CHECK: - return 0; - case INSTRUMENTED_RESUME: - return 0; - case LOAD_CLOSURE: - return 0; - case LOAD_FAST_CHECK: - return 0; - case LOAD_FAST: - return 0; - case LOAD_FAST_AND_CLEAR: - return 0; - case LOAD_FAST_LOAD_FAST: - return 0; - case LOAD_CONST: - return 0; - case STORE_FAST: - return 1; - case STORE_FAST_MAYBE_NULL: - return 1; - case STORE_FAST_LOAD_FAST: - return 1; - case STORE_FAST_STORE_FAST: - return 2; - case POP_TOP: - return 1; - case PUSH_NULL: - return 0; - case END_FOR: - return 2; - case INSTRUMENTED_END_FOR: - return 2; - case END_SEND: + case BINARY_OP: return 2; - case INSTRUMENTED_END_SEND: + case BINARY_OP_ADD_FLOAT: return 2; - case UNARY_NEGATIVE: - return 1; - case UNARY_NOT: - return 1; - case TO_BOOL: - return 1; - case TO_BOOL_BOOL: - return 1; - case TO_BOOL_INT: - return 1; - case TO_BOOL_LIST: - return 1; - case TO_BOOL_NONE: - return 1; - case TO_BOOL_STR: - return 1; - case TO_BOOL_ALWAYS_TRUE: - return 1; - case UNARY_INVERT: - return 1; - case _GUARD_BOTH_INT: + case BINARY_OP_ADD_INT: return 2; - case _BINARY_OP_MULTIPLY_INT: + case BINARY_OP_ADD_UNICODE: return 2; - case _BINARY_OP_ADD_INT: + case BINARY_OP_INPLACE_ADD_UNICODE: return 2; - case _BINARY_OP_SUBTRACT_INT: + case BINARY_OP_MULTIPLY_FLOAT: return 2; case BINARY_OP_MULTIPLY_INT: return 2; - case BINARY_OP_ADD_INT: + case BINARY_OP_SUBTRACT_FLOAT: return 2; case BINARY_OP_SUBTRACT_INT: return 2; - case _GUARD_BOTH_FLOAT: + case BINARY_SLICE: + return 3; + case BINARY_SUBSCR: return 2; - case _BINARY_OP_MULTIPLY_FLOAT: + case BINARY_SUBSCR_DICT: return 2; - case _BINARY_OP_ADD_FLOAT: + case BINARY_SUBSCR_GETITEM: return 2; - case _BINARY_OP_SUBTRACT_FLOAT: + case BINARY_SUBSCR_LIST_INT: return 2; - case BINARY_OP_MULTIPLY_FLOAT: + case BINARY_SUBSCR_STR_INT: return 2; - case BINARY_OP_ADD_FLOAT: + case BINARY_SUBSCR_TUPLE_INT: return 2; - case BINARY_OP_SUBTRACT_FLOAT: + case BUILD_CONST_KEY_MAP: + return 1 + oparg; + case BUILD_LIST: + return oparg; + case BUILD_MAP: + return oparg*2; + case BUILD_SET: + return oparg; + case BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case BUILD_STRING: + return oparg; + case BUILD_TUPLE: + return oparg; + case CACHE: + return 0; + case CALL: + return 2 + oparg; + case CALL_ALLOC_AND_ENTER_INIT: + return 2 + oparg; + case CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; + case CALL_BUILTIN_CLASS: + return 2 + oparg; + case CALL_BUILTIN_FAST: + return 2 + oparg; + case CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case CALL_BUILTIN_O: + return 2 + oparg; + case CALL_FUNCTION_EX: + return 3 + (oparg & 1); + case CALL_INTRINSIC_1: + return 1; + case CALL_INTRINSIC_2: return 2; - case _GUARD_BOTH_UNICODE: + case CALL_ISINSTANCE: + return 2 + oparg; + case CALL_KW: + return 3 + oparg; + case CALL_LEN: + return 2 + oparg; + case CALL_LIST_APPEND: + return 3; + case CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case CALL_NON_PY_GENERAL: + return 2 + oparg; + case CALL_PY_EXACT_ARGS: + return 2 + oparg; + case CALL_PY_GENERAL: + return 2 + oparg; + case CALL_STR_1: + return 3; + case CALL_TUPLE_1: + return 3; + case CALL_TYPE_1: + return 3; + case CHECK_EG_MATCH: return 2; - case _BINARY_OP_ADD_UNICODE: + case CHECK_EXC_MATCH: return 2; - case BINARY_OP_ADD_UNICODE: + case CLEANUP_THROW: + return 3; + case COMPARE_OP: return 2; - case _BINARY_OP_INPLACE_ADD_UNICODE: + case COMPARE_OP_FLOAT: return 2; - case BINARY_OP_INPLACE_ADD_UNICODE: + case COMPARE_OP_INT: return 2; - case BINARY_SUBSCR: + case COMPARE_OP_STR: return 2; - case BINARY_SLICE: - return 3; - case STORE_SLICE: - return 4; - case BINARY_SUBSCR_LIST_INT: + case CONTAINS_OP: return 2; - case BINARY_SUBSCR_STR_INT: + case CONTAINS_OP_DICT: return 2; - case BINARY_SUBSCR_TUPLE_INT: + case CONTAINS_OP_SET: return 2; - case BINARY_SUBSCR_DICT: + case CONVERT_VALUE: + return 1; + case COPY: + return 1 + (oparg-1); + case COPY_FREE_VARS: + return 0; + case DELETE_ATTR: + return 1; + case DELETE_DEREF: + return 0; + case DELETE_FAST: + return 0; + case DELETE_GLOBAL: + return 0; + case DELETE_NAME: + return 0; + case DELETE_SUBSCR: return 2; - case BINARY_SUBSCR_GETITEM: + case DICT_MERGE: + return 5 + (oparg - 1); + case DICT_UPDATE: + return 2 + (oparg - 1); + case END_ASYNC_FOR: return 2; - case LIST_APPEND: - return (oparg-1) + 2; - case SET_ADD: - return (oparg-1) + 2; - case STORE_SUBSCR: - return 3; - case STORE_SUBSCR_LIST_INT: - return 3; - case STORE_SUBSCR_DICT: - return 3; - case DELETE_SUBSCR: + case END_FOR: + return 1; + case END_SEND: return 2; - case CALL_INTRINSIC_1: + case ENTER_EXECUTOR: + return 0; + case EXIT_INIT_CHECK: return 1; - case CALL_INTRINSIC_2: + case EXTENDED_ARG: + return 0; + case FORMAT_SIMPLE: + return 1; + case FORMAT_WITH_SPEC: return 2; - case RAISE_VARARGS: - return oparg; - case INTERPRETER_EXIT: + case FOR_ITER: return 1; - case _POP_FRAME: + case FOR_ITER_GEN: return 1; - case RETURN_VALUE: + case FOR_ITER_LIST: return 1; - case INSTRUMENTED_RETURN_VALUE: + case FOR_ITER_RANGE: + return 1; + case FOR_ITER_TUPLE: return 1; - case RETURN_CONST: - return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case GET_AITER: return 1; case GET_ANEXT: return 1; case GET_AWAITABLE: return 1; - case SEND: - return 2; - case SEND_GEN: - return 2; - case INSTRUMENTED_YIELD_VALUE: + case GET_ITER: return 1; - case YIELD_VALUE: + case GET_LEN: return 1; - case POP_EXCEPT: + case GET_YIELD_FROM_ITER: return 1; - case RERAISE: - return oparg + 1; - case END_ASYNC_FOR: + case IMPORT_FROM: + return 1; + case IMPORT_NAME: return 2; - case CLEANUP_THROW: - return 3; - case LOAD_ASSERTION_ERROR: + case INSTRUMENTED_CALL: return 0; - case LOAD_BUILD_CLASS: + case INSTRUMENTED_CALL_FUNCTION_EX: return 0; - case STORE_NAME: - return 1; - case DELETE_NAME: + case INSTRUMENTED_CALL_KW: return 0; - case UNPACK_SEQUENCE: - return 1; - case UNPACK_SEQUENCE_TWO_TUPLE: - return 1; - case UNPACK_SEQUENCE_TUPLE: - return 1; - case UNPACK_SEQUENCE_LIST: - return 1; - case UNPACK_EX: - return 1; - case STORE_ATTR: + case INSTRUMENTED_END_FOR: return 2; - case DELETE_ATTR: - return 1; - case STORE_GLOBAL: - return 1; - case DELETE_GLOBAL: + case INSTRUMENTED_END_SEND: + return 2; + case INSTRUMENTED_FOR_ITER: return 0; - case LOAD_LOCALS: + case INSTRUMENTED_INSTRUCTION: return 0; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; - case LOAD_NAME: + case INSTRUMENTED_JUMP_BACKWARD: return 0; - case LOAD_GLOBAL: + case INSTRUMENTED_JUMP_FORWARD: return 0; - case _GUARD_GLOBALS_VERSION: + case INSTRUMENTED_LOAD_SUPER_ATTR: + return 3; + case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; - case _GUARD_BUILTINS_VERSION: + case INSTRUMENTED_POP_JUMP_IF_NONE: return 0; - case _LOAD_GLOBAL_MODULE: + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: return 0; - case _LOAD_GLOBAL_BUILTINS: + case INSTRUMENTED_POP_JUMP_IF_TRUE: return 0; - case LOAD_GLOBAL_MODULE: + case INSTRUMENTED_RESUME: return 0; - case LOAD_GLOBAL_BUILTIN: + case INSTRUMENTED_RETURN_CONST: return 0; - case DELETE_FAST: + case INSTRUMENTED_RETURN_VALUE: + return 1; + case INSTRUMENTED_YIELD_VALUE: + return 1; + case INTERPRETER_EXIT: + return 1; + case IS_OP: + return 2; + case JUMP: return 0; - case MAKE_CELL: + case JUMP_BACKWARD: return 0; - case DELETE_DEREF: + case JUMP_BACKWARD_NO_INTERRUPT: return 0; - case LOAD_FROM_DICT_OR_DEREF: - return 1; - case LOAD_DEREF: + case JUMP_FORWARD: return 0; - case STORE_DEREF: - return 1; - case COPY_FREE_VARS: + case JUMP_NO_INTERRUPT: return 0; - case BUILD_STRING: - return oparg; - case BUILD_TUPLE: - return oparg; - case BUILD_LIST: - return oparg; + case LIST_APPEND: + return 2 + (oparg-1); case LIST_EXTEND: - return (oparg-1) + 2; - case SET_UPDATE: - return (oparg-1) + 2; - case BUILD_SET: - return oparg; - case BUILD_MAP: - return oparg*2; - case SETUP_ANNOTATIONS: - return 0; - case BUILD_CONST_KEY_MAP: - return oparg + 1; - case DICT_UPDATE: - return (oparg - 1) + 2; - case DICT_MERGE: - return (oparg - 1) + 5; - case MAP_ADD: - return (oparg - 1) + 3; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return 3; - case LOAD_SUPER_ATTR: - return 3; - case LOAD_SUPER_METHOD: - return 3; - case LOAD_ZERO_SUPER_METHOD: - return 3; - case LOAD_ZERO_SUPER_ATTR: - return 3; - case LOAD_SUPER_ATTR_ATTR: - return 3; - case LOAD_SUPER_ATTR_METHOD: - return 3; + return 2 + (oparg-1); case LOAD_ATTR: return 1; - case LOAD_METHOD: - return 1; - case _GUARD_TYPE_VERSION: - return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case LOAD_ATTR_CLASS: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1; - case _CHECK_ATTR_MODULE: - return 1; - case _LOAD_ATTR_MODULE: - return 1; - case LOAD_ATTR_MODULE: - return 1; - case _CHECK_ATTR_WITH_HINT: - return 1; - case _LOAD_ATTR_WITH_HINT: - return 1; - case LOAD_ATTR_WITH_HINT: + case LOAD_ATTR_METHOD_LAZY_DICT: return 1; - case _LOAD_ATTR_SLOT: + case LOAD_ATTR_METHOD_NO_DICT: return 1; - case LOAD_ATTR_SLOT: + case LOAD_ATTR_METHOD_WITH_VALUES: return 1; - case _CHECK_ATTR_CLASS: + case LOAD_ATTR_MODULE: return 1; - case _LOAD_ATTR_CLASS: + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case LOAD_ATTR_CLASS: + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; case LOAD_ATTR_PROPERTY: return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1; - case _GUARD_DORV_VALUES: + case LOAD_ATTR_SLOT: return 1; - case _STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_WITH_HINT: - return 2; - case _STORE_ATTR_SLOT: - return 2; - case STORE_ATTR_SLOT: - return 2; - case COMPARE_OP: - return 2; - case COMPARE_OP_FLOAT: - return 2; - case COMPARE_OP_INT: - return 2; - case COMPARE_OP_STR: - return 2; - case IS_OP: - return 2; - case CONTAINS_OP: - return 2; - case CHECK_EG_MATCH: - return 2; - case CHECK_EXC_MATCH: - return 2; - case IMPORT_NAME: - return 2; - case IMPORT_FROM: + case LOAD_ATTR_WITH_HINT: return 1; - case JUMP_FORWARD: + case LOAD_BUILD_CLASS: return 0; - case JUMP_BACKWARD: + case LOAD_CLOSURE: return 0; - case JUMP: + case LOAD_COMMON_CONSTANT: return 0; - case JUMP_NO_INTERRUPT: + case LOAD_CONST: return 0; - case ENTER_EXECUTOR: + case LOAD_DEREF: return 0; - case POP_JUMP_IF_FALSE: - return 1; - case POP_JUMP_IF_TRUE: - return 1; - case _IS_NONE: + case LOAD_FAST: + return 0; + case LOAD_FAST_AND_CLEAR: + return 0; + case LOAD_FAST_CHECK: + return 0; + case LOAD_FAST_LOAD_FAST: + return 0; + case LOAD_FROM_DICT_OR_DEREF: return 1; - case POP_JUMP_IF_NONE: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; - case POP_JUMP_IF_NOT_NONE: + case LOAD_GLOBAL: + return 0; + case LOAD_GLOBAL_BUILTIN: + return 0; + case LOAD_GLOBAL_MODULE: + return 0; + case LOAD_LOCALS: + return 0; + case LOAD_NAME: + return 0; + case LOAD_SPECIAL: return 1; - case JUMP_BACKWARD_NO_INTERRUPT: + case LOAD_SUPER_ATTR: + return 3; + case LOAD_SUPER_ATTR_ATTR: + return 3; + case LOAD_SUPER_ATTR_METHOD: + return 3; + case MAKE_CELL: return 0; - case GET_LEN: + case MAKE_FUNCTION: return 1; + case MAP_ADD: + return 3 + (oparg - 1); case MATCH_CLASS: return 3; - case MATCH_MAPPING: - return 1; - case MATCH_SEQUENCE: - return 1; case MATCH_KEYS: return 2; - case GET_ITER: - return 1; - case GET_YIELD_FROM_ITER: - return 1; - case FOR_ITER: - return 1; - case INSTRUMENTED_FOR_ITER: - return 0; - case _ITER_CHECK_LIST: - return 1; - case _ITER_JUMP_LIST: - return 1; - case _IS_ITER_EXHAUSTED_LIST: - return 1; - case _ITER_NEXT_LIST: - return 1; - case FOR_ITER_LIST: - return 1; - case _ITER_CHECK_TUPLE: - return 1; - case _ITER_JUMP_TUPLE: - return 1; - case _IS_ITER_EXHAUSTED_TUPLE: - return 1; - case _ITER_NEXT_TUPLE: - return 1; - case FOR_ITER_TUPLE: - return 1; - case _ITER_CHECK_RANGE: - return 1; - case _ITER_JUMP_RANGE: - return 1; - case _IS_ITER_EXHAUSTED_RANGE: - return 1; - case _ITER_NEXT_RANGE: - return 1; - case FOR_ITER_RANGE: - return 1; - case FOR_ITER_GEN: - return 1; - case BEFORE_ASYNC_WITH: + case MATCH_MAPPING: return 1; - case BEFORE_WITH: + case MATCH_SEQUENCE: return 1; - case WITH_EXCEPT_START: - return 4; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: + case NOP: return 0; case POP_BLOCK: return 0; - case PUSH_EXC_INFO: - return 1; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 1; - case _GUARD_KEYS_VERSION: - return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: - return 1; - case LOAD_ATTR_METHOD_WITH_VALUES: - return 1; - case _LOAD_ATTR_METHOD_NO_DICT: - return 1; - case LOAD_ATTR_METHOD_NO_DICT: - return 1; - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case POP_EXCEPT: return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case POP_JUMP_IF_FALSE: return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case POP_JUMP_IF_NONE: return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case POP_JUMP_IF_NOT_NONE: return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: + case POP_JUMP_IF_TRUE: return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: + case POP_TOP: return 1; - case LOAD_ATTR_METHOD_LAZY_DICT: + case PUSH_EXC_INFO: return 1; - case INSTRUMENTED_CALL: + case PUSH_NULL: return 0; - case CALL: - return oparg + 2; - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _CHECK_PEP_523: - return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; - case _CHECK_STACK_SPACE: - return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: - return oparg + 2; - case _PUSH_FRAME: - return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case CALL_PY_EXACT_ARGS: - return oparg + 2; - case CALL_PY_WITH_DEFAULTS: - return oparg + 2; - case CALL_TYPE_1: - return oparg + 2; - case CALL_STR_1: - return oparg + 2; - case CALL_TUPLE_1: - return oparg + 2; - case CALL_ALLOC_AND_ENTER_INIT: - return oparg + 2; - case EXIT_INIT_CHECK: - return 1; - case CALL_BUILTIN_CLASS: - return oparg + 2; - case CALL_BUILTIN_O: - return oparg + 2; - case CALL_BUILTIN_FAST: - return oparg + 2; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_LEN: - return oparg + 2; - case CALL_ISINSTANCE: - return oparg + 2; - case CALL_LIST_APPEND: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_O: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_NOARGS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST: - return oparg + 2; - case INSTRUMENTED_CALL_KW: + case RAISE_VARARGS: + return oparg; + case RERAISE: + return 1 + oparg; + case RESERVED: return 0; - case CALL_KW: - return oparg + 3; - case INSTRUMENTED_CALL_FUNCTION_EX: + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case RETURN_CONST: return 0; - case CALL_FUNCTION_EX: - return ((oparg & 1) ? 1 : 0) + 3; - case MAKE_FUNCTION: - return 1; - case SET_FUNCTION_ATTRIBUTE: - return 2; case RETURN_GENERATOR: return 0; - case BUILD_SLICE: - return ((oparg == 3) ? 1 : 0) + 2; - case CONVERT_VALUE: - return 1; - case FORMAT_SIMPLE: + case RETURN_VALUE: return 1; - case FORMAT_WITH_SPEC: + case SEND: return 2; - case COPY: - return (oparg-1) + 1; - case BINARY_OP: + case SEND_GEN: return 2; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: - return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: - return 0; - case EXTENDED_ARG: - return 0; - case CACHE: - return 0; - case RESERVED: - return 0; - case _POP_JUMP_IF_FALSE: - return 1; - case _POP_JUMP_IF_TRUE: - return 1; - case _JUMP_TO_TOP: - return 0; - case _SET_IP: - return 0; - case _SAVE_CURRENT_IP: - return 0; - case _EXIT_TRACE: - return 0; - case _INSERT: - return oparg + 1; - default: - return -1; - } -} -#endif // NEED_OPCODE_METADATA - -extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); -#ifdef NEED_OPCODE_METADATA -int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { - switch(opcode) { - case NOP: + case SETUP_ANNOTATIONS: return 0; - case RESUME: + case SETUP_CLEANUP: return 0; - case RESUME_CHECK: + case SETUP_FINALLY: return 0; - case INSTRUMENTED_RESUME: + case SETUP_WITH: return 0; - case LOAD_CLOSURE: - return 1; - case LOAD_FAST_CHECK: - return 1; - case LOAD_FAST: - return 1; - case LOAD_FAST_AND_CLEAR: - return 1; - case LOAD_FAST_LOAD_FAST: + case SET_ADD: + return 2 + (oparg-1); + case SET_FUNCTION_ATTRIBUTE: return 2; - case LOAD_CONST: + case SET_UPDATE: + return 2 + (oparg-1); + case STORE_ATTR: + return 2; + case STORE_ATTR_INSTANCE_VALUE: + return 2; + case STORE_ATTR_SLOT: + return 2; + case STORE_ATTR_WITH_HINT: + return 2; + case STORE_DEREF: return 1; case STORE_FAST: - return 0; - case STORE_FAST_MAYBE_NULL: - return 0; - case STORE_FAST_LOAD_FAST: - return 1; - case STORE_FAST_STORE_FAST: - return 0; - case POP_TOP: - return 0; - case PUSH_NULL: return 1; - case END_FOR: - return 0; - case INSTRUMENTED_END_FOR: - return 0; - case END_SEND: + case STORE_FAST_LOAD_FAST: return 1; - case INSTRUMENTED_END_SEND: + case STORE_FAST_MAYBE_NULL: return 1; - case UNARY_NEGATIVE: + case STORE_FAST_STORE_FAST: + return 2; + case STORE_GLOBAL: return 1; - case UNARY_NOT: + case STORE_NAME: return 1; + case STORE_SLICE: + return 4; + case STORE_SUBSCR: + return 3; + case STORE_SUBSCR_DICT: + return 3; + case STORE_SUBSCR_LIST_INT: + return 3; + case SWAP: + return 2 + (oparg-2); case TO_BOOL: return 1; + case TO_BOOL_ALWAYS_TRUE: + return 1; case TO_BOOL_BOOL: return 1; case TO_BOOL_INT: @@ -726,264 +449,141 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case TO_BOOL_STR: return 1; - case TO_BOOL_ALWAYS_TRUE: - return 1; case UNARY_INVERT: return 1; - case _GUARD_BOTH_INT: - return 2; - case _BINARY_OP_MULTIPLY_INT: - return 1; - case _BINARY_OP_ADD_INT: + case UNARY_NEGATIVE: return 1; - case _BINARY_OP_SUBTRACT_INT: + case UNARY_NOT: return 1; - case BINARY_OP_MULTIPLY_INT: + case UNPACK_EX: return 1; - case BINARY_OP_ADD_INT: + case UNPACK_SEQUENCE: return 1; - case BINARY_OP_SUBTRACT_INT: + case UNPACK_SEQUENCE_LIST: return 1; - case _GUARD_BOTH_FLOAT: - return 2; - case _BINARY_OP_MULTIPLY_FLOAT: + case UNPACK_SEQUENCE_TUPLE: return 1; - case _BINARY_OP_ADD_FLOAT: + case UNPACK_SEQUENCE_TWO_TUPLE: return 1; - case _BINARY_OP_SUBTRACT_FLOAT: + case WITH_EXCEPT_START: + return 5; + case YIELD_VALUE: return 1; - case BINARY_OP_MULTIPLY_FLOAT: + default: + return -1; + } +} + +#endif + +extern int _PyOpcode_num_pushed(int opcode, int oparg); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_pushed(int opcode, int oparg) { + switch(opcode) { + case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: return 1; - case BINARY_OP_SUBTRACT_FLOAT: - return 1; - case _GUARD_BOTH_UNICODE: - return 2; - case _BINARY_OP_ADD_UNICODE: + case BINARY_OP_ADD_INT: return 1; case BINARY_OP_ADD_UNICODE: return 1; - case _BINARY_OP_INPLACE_ADD_UNICODE: - return 0; case BINARY_OP_INPLACE_ADD_UNICODE: return 0; - case BINARY_SUBSCR: + case BINARY_OP_MULTIPLY_FLOAT: return 1; - case BINARY_SLICE: + case BINARY_OP_MULTIPLY_INT: return 1; - case STORE_SLICE: - return 0; - case BINARY_SUBSCR_LIST_INT: + case BINARY_OP_SUBTRACT_FLOAT: return 1; - case BINARY_SUBSCR_STR_INT: + case BINARY_OP_SUBTRACT_INT: return 1; - case BINARY_SUBSCR_TUPLE_INT: + case BINARY_SLICE: + return 1; + case BINARY_SUBSCR: return 1; case BINARY_SUBSCR_DICT: return 1; case BINARY_SUBSCR_GETITEM: return 1; - case LIST_APPEND: - return (oparg-1) + 1; - case SET_ADD: - return (oparg-1) + 1; - case STORE_SUBSCR: - return 0; - case STORE_SUBSCR_LIST_INT: - return 0; - case STORE_SUBSCR_DICT: - return 0; - case DELETE_SUBSCR: - return 0; - case CALL_INTRINSIC_1: - return 1; - case CALL_INTRINSIC_2: + case BINARY_SUBSCR_LIST_INT: return 1; - case RAISE_VARARGS: - return 0; - case INTERPRETER_EXIT: - return 0; - case _POP_FRAME: - return 0; - case RETURN_VALUE: - return 0; - case INSTRUMENTED_RETURN_VALUE: - return 0; - case RETURN_CONST: - return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; - case GET_AITER: + case BINARY_SUBSCR_STR_INT: return 1; - case GET_ANEXT: - return 2; - case GET_AWAITABLE: + case BINARY_SUBSCR_TUPLE_INT: return 1; - case SEND: - return 2; - case SEND_GEN: - return 2; - case INSTRUMENTED_YIELD_VALUE: + case BUILD_CONST_KEY_MAP: return 1; - case YIELD_VALUE: + case BUILD_LIST: return 1; - case POP_EXCEPT: - return 0; - case RERAISE: - return oparg; - case END_ASYNC_FOR: - return 0; - case CLEANUP_THROW: - return 2; - case LOAD_ASSERTION_ERROR: + case BUILD_MAP: return 1; - case LOAD_BUILD_CLASS: + case BUILD_SET: return 1; - case STORE_NAME: - return 0; - case DELETE_NAME: - return 0; - case UNPACK_SEQUENCE: - return oparg; - case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; - case UNPACK_SEQUENCE_TUPLE: - return oparg; - case UNPACK_SEQUENCE_LIST: - return oparg; - case UNPACK_EX: - return (oparg & 0xFF) + (oparg >> 8) + 1; - case STORE_ATTR: - return 0; - case DELETE_ATTR: - return 0; - case STORE_GLOBAL: - return 0; - case DELETE_GLOBAL: - return 0; - case LOAD_LOCALS: + case BUILD_SLICE: return 1; - case LOAD_FROM_DICT_OR_GLOBALS: + case BUILD_STRING: return 1; - case LOAD_NAME: + case BUILD_TUPLE: return 1; - case LOAD_GLOBAL: - return ((oparg & 1) ? 1 : 0) + 1; - case _GUARD_GLOBALS_VERSION: - return 0; - case _GUARD_BUILTINS_VERSION: - return 0; - case _LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case _LOAD_GLOBAL_BUILTINS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_GLOBAL_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_GLOBAL_BUILTIN: - return (oparg & 1 ? 1 : 0) + 1; - case DELETE_FAST: - return 0; - case MAKE_CELL: - return 0; - case DELETE_DEREF: + case CACHE: return 0; - case LOAD_FROM_DICT_OR_DEREF: + case CALL: return 1; - case LOAD_DEREF: + case CALL_ALLOC_AND_ENTER_INIT: return 1; - case STORE_DEREF: + case CALL_BOUND_METHOD_EXACT_ARGS: return 0; - case COPY_FREE_VARS: + case CALL_BOUND_METHOD_GENERAL: return 0; - case BUILD_STRING: + case CALL_BUILTIN_CLASS: return 1; - case BUILD_TUPLE: + case CALL_BUILTIN_FAST: return 1; - case BUILD_LIST: + case CALL_BUILTIN_FAST_WITH_KEYWORDS: return 1; - case LIST_EXTEND: - return (oparg-1) + 1; - case SET_UPDATE: - return (oparg-1) + 1; - case BUILD_SET: + case CALL_BUILTIN_O: return 1; - case BUILD_MAP: + case CALL_FUNCTION_EX: return 1; - case SETUP_ANNOTATIONS: - return 0; - case BUILD_CONST_KEY_MAP: + case CALL_INTRINSIC_1: return 1; - case DICT_UPDATE: - return (oparg - 1) + 1; - case DICT_MERGE: - return (oparg - 1) + 4; - case MAP_ADD: - return (oparg - 1) + 1; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_ATTR_ATTR: + case CALL_INTRINSIC_2: return 1; - case LOAD_SUPER_ATTR_METHOD: - return 2; - case LOAD_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; - case _GUARD_TYPE_VERSION: + case CALL_ISINSTANCE: return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case CALL_KW: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_INSTANCE_VALUE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_MODULE: + case CALL_LEN: return 1; - case _LOAD_ATTR_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_WITH_HINT: + case CALL_LIST_APPEND: return 1; - case _LOAD_ATTR_WITH_HINT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_WITH_HINT: - return (oparg & 1 ? 1 : 0) + 1; - case _LOAD_ATTR_SLOT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_SLOT: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_CLASS: + case CALL_METHOD_DESCRIPTOR_FAST: return 1; - case _LOAD_ATTR_CLASS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_CLASS: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ATTR_PROPERTY: + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + case CALL_METHOD_DESCRIPTOR_NOARGS: return 1; - case _GUARD_DORV_VALUES: + case CALL_METHOD_DESCRIPTOR_O: return 1; - case _STORE_ATTR_INSTANCE_VALUE: - return 0; - case STORE_ATTR_INSTANCE_VALUE: - return 0; - case STORE_ATTR_WITH_HINT: - return 0; - case _STORE_ATTR_SLOT: + case CALL_NON_PY_GENERAL: + return 1; + case CALL_PY_EXACT_ARGS: return 0; - case STORE_ATTR_SLOT: + case CALL_PY_GENERAL: return 0; + case CALL_STR_1: + return 1; + case CALL_TUPLE_1: + return 1; + case CALL_TYPE_1: + return 1; + case CHECK_EG_MATCH: + return 2; + case CHECK_EXC_MATCH: + return 2; + case CLEANUP_THROW: + return 2; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -992,272 +592,351 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case COMPARE_OP_STR: return 1; - case IS_OP: - return 1; case CONTAINS_OP: return 1; - case CHECK_EG_MATCH: - return 2; - case CHECK_EXC_MATCH: - return 2; - case IMPORT_NAME: + case CONTAINS_OP_DICT: return 1; - case IMPORT_FROM: - return 2; - case JUMP_FORWARD: + case CONTAINS_OP_SET: + return 1; + case CONVERT_VALUE: + return 1; + case COPY: + return 2 + (oparg-1); + case COPY_FREE_VARS: return 0; - case JUMP_BACKWARD: + case DELETE_ATTR: return 0; - case JUMP: + case DELETE_DEREF: return 0; - case JUMP_NO_INTERRUPT: + case DELETE_FAST: return 0; - case ENTER_EXECUTOR: + case DELETE_GLOBAL: return 0; - case POP_JUMP_IF_FALSE: + case DELETE_NAME: return 0; - case POP_JUMP_IF_TRUE: + case DELETE_SUBSCR: + return 0; + case DICT_MERGE: + return 4 + (oparg - 1); + case DICT_UPDATE: + return 1 + (oparg - 1); + case END_ASYNC_FOR: + return 0; + case END_FOR: return 0; - case _IS_NONE: + case END_SEND: return 1; - case POP_JUMP_IF_NONE: + case ENTER_EXECUTOR: return 0; - case POP_JUMP_IF_NOT_NONE: + case EXIT_INIT_CHECK: return 0; - case JUMP_BACKWARD_NO_INTERRUPT: + case EXTENDED_ARG: return 0; - case GET_LEN: - return 2; - case MATCH_CLASS: - return 1; - case MATCH_MAPPING: - return 2; - case MATCH_SEQUENCE: - return 2; - case MATCH_KEYS: - return 3; - case GET_ITER: + case FORMAT_SIMPLE: return 1; - case GET_YIELD_FROM_ITER: + case FORMAT_WITH_SPEC: return 1; case FOR_ITER: return 2; - case INSTRUMENTED_FOR_ITER: - return 0; - case _ITER_CHECK_LIST: - return 1; - case _ITER_JUMP_LIST: + case FOR_ITER_GEN: return 1; - case _IS_ITER_EXHAUSTED_LIST: - return 2; - case _ITER_NEXT_LIST: - return 2; case FOR_ITER_LIST: return 2; - case _ITER_CHECK_TUPLE: - return 1; - case _ITER_JUMP_TUPLE: - return 1; - case _IS_ITER_EXHAUSTED_TUPLE: - return 2; - case _ITER_NEXT_TUPLE: + case FOR_ITER_RANGE: return 2; case FOR_ITER_TUPLE: return 2; - case _ITER_CHECK_RANGE: - return 1; - case _ITER_JUMP_RANGE: + case GET_AITER: return 1; - case _IS_ITER_EXHAUSTED_RANGE: - return 2; - case _ITER_NEXT_RANGE: - return 2; - case FOR_ITER_RANGE: - return 2; - case FOR_ITER_GEN: - return 2; - case BEFORE_ASYNC_WITH: - return 2; - case BEFORE_WITH: - return 2; - case WITH_EXCEPT_START: - return 5; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: - return 0; - case POP_BLOCK: - return 0; - case PUSH_EXC_INFO: + case GET_ANEXT: return 2; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + case GET_AWAITABLE: return 1; - case _GUARD_KEYS_VERSION: + case GET_ITER: return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case _LOAD_ATTR_METHOD_NO_DICT: - return 2; - case LOAD_ATTR_METHOD_NO_DICT: + case GET_LEN: return 2; - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case GET_YIELD_FROM_ITER: return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case IMPORT_FROM: + return 2; + case IMPORT_NAME: return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case INSTRUMENTED_CALL: + return 0; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; + case INSTRUMENTED_CALL_KW: + return 0; + case INSTRUMENTED_END_FOR: return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case INSTRUMENTED_END_SEND: return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: + case INSTRUMENTED_FOR_ITER: + return 0; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_LOAD_SUPER_ATTR: + return 1 + (oparg & 1); + case INSTRUMENTED_POP_JUMP_IF_FALSE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_RESUME: + return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; + case INSTRUMENTED_RETURN_VALUE: + return 0; + case INSTRUMENTED_YIELD_VALUE: return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case INSTRUMENTED_CALL: + case INTERPRETER_EXIT: return 0; - case CALL: + case IS_OP: return 1; - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _CHECK_PEP_523: + case JUMP: + return 0; + case JUMP_BACKWARD: + return 0; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case JUMP_FORWARD: return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; - case _CHECK_STACK_SPACE: - return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: + case JUMP_NO_INTERRUPT: + return 0; + case LIST_APPEND: + return 1 + (oparg-1); + case LIST_EXTEND: + return 1 + (oparg-1); + case LOAD_ATTR: + return 1 + (oparg & 1); + case LOAD_ATTR_CLASS: + return 1 + (oparg & 1); + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; - case _PUSH_FRAME: + case LOAD_ATTR_INSTANCE_VALUE: + return 1 + (oparg & 1); + case LOAD_ATTR_METHOD_LAZY_DICT: + return 2; + case LOAD_ATTR_METHOD_NO_DICT: + return 2; + case LOAD_ATTR_METHOD_WITH_VALUES: + return 2; + case LOAD_ATTR_MODULE: + return 1 + (oparg & 1); + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case CALL_PY_EXACT_ARGS: + case LOAD_ATTR_PROPERTY: return 1; - case CALL_PY_WITH_DEFAULTS: + case LOAD_ATTR_SLOT: + return 1 + (oparg & 1); + case LOAD_ATTR_WITH_HINT: + return 1 + (oparg & 1); + case LOAD_BUILD_CLASS: return 1; - case CALL_TYPE_1: + case LOAD_CLOSURE: return 1; - case CALL_STR_1: + case LOAD_COMMON_CONSTANT: return 1; - case CALL_TUPLE_1: + case LOAD_CONST: return 1; - case CALL_ALLOC_AND_ENTER_INIT: + case LOAD_DEREF: return 1; - case EXIT_INIT_CHECK: - return 0; - case CALL_BUILTIN_CLASS: + case LOAD_FAST: return 1; - case CALL_BUILTIN_O: + case LOAD_FAST_AND_CLEAR: return 1; - case CALL_BUILTIN_FAST: + case LOAD_FAST_CHECK: return 1; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: + case LOAD_FAST_LOAD_FAST: + return 2; + case LOAD_FROM_DICT_OR_DEREF: return 1; - case CALL_LEN: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; - case CALL_ISINSTANCE: + case LOAD_GLOBAL: + return 1 + (oparg & 1); + case LOAD_GLOBAL_BUILTIN: + return 1 + (oparg & 1); + case LOAD_GLOBAL_MODULE: + return 1 + (oparg & 1); + case LOAD_LOCALS: return 1; - case CALL_LIST_APPEND: + case LOAD_NAME: return 1; - case CALL_METHOD_DESCRIPTOR_O: + case LOAD_SPECIAL: + return 2; + case LOAD_SUPER_ATTR: + return 1 + (oparg & 1); + case LOAD_SUPER_ATTR_ATTR: return 1; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + case LOAD_SUPER_ATTR_METHOD: + return 2; + case MAKE_CELL: + return 0; + case MAKE_FUNCTION: return 1; - case CALL_METHOD_DESCRIPTOR_NOARGS: + case MAP_ADD: + return 1 + (oparg - 1); + case MATCH_CLASS: return 1; - case CALL_METHOD_DESCRIPTOR_FAST: + case MATCH_KEYS: + return 3; + case MATCH_MAPPING: + return 2; + case MATCH_SEQUENCE: + return 2; + case NOP: + return 0; + case POP_BLOCK: + return 0; + case POP_EXCEPT: + return 0; + case POP_JUMP_IF_FALSE: + return 0; + case POP_JUMP_IF_NONE: + return 0; + case POP_JUMP_IF_NOT_NONE: + return 0; + case POP_JUMP_IF_TRUE: + return 0; + case POP_TOP: + return 0; + case PUSH_EXC_INFO: + return 2; + case PUSH_NULL: return 1; - case INSTRUMENTED_CALL_KW: + case RAISE_VARARGS: + return 0; + case RERAISE: + return oparg; + case RESERVED: return 0; - case CALL_KW: - return 1; - case INSTRUMENTED_CALL_FUNCTION_EX: + case RESUME: return 0; - case CALL_FUNCTION_EX: - return 1; - case MAKE_FUNCTION: - return 1; - case SET_FUNCTION_ATTRIBUTE: + case RESUME_CHECK: + return 0; + case RETURN_CONST: return 1; case RETURN_GENERATOR: - return 0; - case BUILD_SLICE: return 1; - case CONVERT_VALUE: + case RETURN_VALUE: return 1; - case FORMAT_SIMPLE: + case SEND: + return 2; + case SEND_GEN: + return 2; + case SETUP_ANNOTATIONS: + return 0; + case SETUP_CLEANUP: + return 2; + case SETUP_FINALLY: return 1; - case FORMAT_WITH_SPEC: + case SETUP_WITH: return 1; - case COPY: - return (oparg-1) + 2; - case BINARY_OP: + case SET_ADD: + return 1 + (oparg-1); + case SET_FUNCTION_ATTRIBUTE: return 1; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: + case SET_UPDATE: + return 1 + (oparg-1); + case STORE_ATTR: return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: + case STORE_ATTR_INSTANCE_VALUE: return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: + case STORE_ATTR_SLOT: return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: + case STORE_ATTR_WITH_HINT: return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + case STORE_DEREF: return 0; - case EXTENDED_ARG: + case STORE_FAST: return 0; - case CACHE: + case STORE_FAST_LOAD_FAST: + return 1; + case STORE_FAST_MAYBE_NULL: return 0; - case RESERVED: + case STORE_FAST_STORE_FAST: return 0; - case _POP_JUMP_IF_FALSE: + case STORE_GLOBAL: return 0; - case _POP_JUMP_IF_TRUE: + case STORE_NAME: return 0; - case _JUMP_TO_TOP: + case STORE_SLICE: return 0; - case _SET_IP: + case STORE_SUBSCR: return 0; - case _SAVE_CURRENT_IP: + case STORE_SUBSCR_DICT: return 0; - case _EXIT_TRACE: + case STORE_SUBSCR_LIST_INT: return 0; - case _INSERT: - return oparg + 1; + case SWAP: + return 2 + (oparg-2); + case TO_BOOL: + return 1; + case TO_BOOL_ALWAYS_TRUE: + return 1; + case TO_BOOL_BOOL: + return 1; + case TO_BOOL_INT: + return 1; + case TO_BOOL_LIST: + return 1; + case TO_BOOL_NONE: + return 1; + case TO_BOOL_STR: + return 1; + case UNARY_INVERT: + return 1; + case UNARY_NEGATIVE: + return 1; + case UNARY_NOT: + return 1; + case UNPACK_EX: + return 1 + (oparg >> 8) + (oparg & 0xFF); + case UNPACK_SEQUENCE: + return oparg; + case UNPACK_SEQUENCE_LIST: + return oparg; + case UNPACK_SEQUENCE_TUPLE: + return oparg; + case UNPACK_SEQUENCE_TWO_TUPLE: + return 2; + case WITH_EXCEPT_START: + return 6; + case YIELD_VALUE: + return 1; default: return -1; } } -#endif // NEED_OPCODE_METADATA + +#endif enum InstructionFormat { - INSTR_FMT_IB, - INSTR_FMT_IBC, - INSTR_FMT_IBC0, - INSTR_FMT_IBC00, - INSTR_FMT_IBC000, - INSTR_FMT_IBC00000000, - INSTR_FMT_IX, - INSTR_FMT_IXC, - INSTR_FMT_IXC0, - INSTR_FMT_IXC00, - INSTR_FMT_IXC000, + INSTR_FMT_IB = 1, + INSTR_FMT_IBC = 2, + INSTR_FMT_IBC00 = 3, + INSTR_FMT_IBC000 = 4, + INSTR_FMT_IBC00000000 = 5, + INSTR_FMT_IX = 6, + INSTR_FMT_IXC = 7, + INSTR_FMT_IXC00 = 8, + INSTR_FMT_IXC000 = 9, }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ + (((OP) >= 0) && ((OP) < 264) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -1269,6 +948,12 @@ enum InstructionFormat { #define HAS_EVAL_BREAK_FLAG (64) #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) +#define HAS_ESCAPES_FLAG (512) +#define HAS_EXIT_FLAG (1024) +#define HAS_PURE_FLAG (2048) +#define HAS_PASSTHROUGH_FLAG (4096) +#define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_ERROR_NO_POP_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1278,17 +963,12 @@ enum InstructionFormat { #define OPCODE_HAS_EVAL_BREAK(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EVAL_BREAK_FLAG)) #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) - -struct opcode_metadata { - bool valid_entry; - enum InstructionFormat instr_format; - int flags; -}; - -struct opcode_macro_expansion { - int nuops; - struct { int16_t uop; int8_t size; int8_t offset; } uops[12]; -}; +#define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) +#define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) +#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) +#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1296,577 +976,433 @@ struct opcode_macro_expansion { #define OPARG_CACHE_4 4 #define OPARG_TOP 5 #define OPARG_BOTTOM 6 -#define OPARG_SET_IP 7 - -#define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) -#define SAME_OPCODE_METADATA(OP1, OP2) \ - (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2)) +#define OPARG_SAVE_RETURN_OFFSET 7 +#define OPARG_REPLACED 9 -#define OPCODE_METADATA_SIZE 512 -#define OPCODE_UOP_NAME_SIZE 512 -#define OPCODE_MACRO_EXPANSION_SIZE 256 +struct opcode_metadata { + uint8_t valid_entry; + int8_t instr_format; + int16_t flags; +}; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[264]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { - [NOP] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, - [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, +const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { + [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, - [_POP_FRAME] = { true, INSTR_FMT_IX, 0 }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG }, - [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [POP_EXCEPT] = { true, INSTR_FMT_IX, 0 }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, - [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, - [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, - [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [LOAD_METHOD] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, 0 }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, 0 }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, - [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, - [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [JUMP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, - [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, - [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_ITER_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, 0 }, - [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_ITER_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, 0 }, - [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_ITER_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, 0 }, - [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [SETUP_FINALLY] = { true, INSTR_FMT_IX, 0 }, - [SETUP_CLEANUP] = { true, INSTR_FMT_IX, 0 }, - [SETUP_WITH] = { true, INSTR_FMT_IX, 0 }, - [POP_BLOCK] = { true, INSTR_FMT_IX, 0 }, - [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, 0 }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, + [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, + [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, - [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, - [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SAVE_CURRENT_IP] = { true, INSTR_FMT_IX, 0 }, - [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, - [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, + [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; -#endif // NEED_OPCODE_METADATA +#endif + +#define MAX_UOP_PER_EXPANSION 9 +struct opcode_macro_expansion { + int nuops; + struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; +}; +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; -extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; #ifdef NEED_OPCODE_METADATA -const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { - [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, - [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, - [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, - [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, - [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { LOAD_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { STORE_FAST, 0, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { STORE_FAST, 6, 0 } } }, - [POP_TOP] = { .nuops = 1, .uops = { { POP_TOP, 0, 0 } } }, - [PUSH_NULL] = { .nuops = 1, .uops = { { PUSH_NULL, 0, 0 } } }, - [END_FOR] = { .nuops = 2, .uops = { { POP_TOP, 0, 0 }, { POP_TOP, 0, 0 } } }, - [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, - [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, - [TO_BOOL] = { .nuops = 1, .uops = { { TO_BOOL, 0, 0 } } }, - [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, - [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, - [TO_BOOL_NONE] = { .nuops = 1, .uops = { { TO_BOOL_NONE, 0, 0 } } }, - [TO_BOOL_STR] = { .nuops = 1, .uops = { { TO_BOOL_STR, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { UNARY_INVERT, 0, 0 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, +const struct opcode_macro_expansion +_PyOpcode_macro_expansion[256] = { + [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, - [BINARY_SUBSCR] = { .nuops = 1, .uops = { { BINARY_SUBSCR, 0, 0 } } }, - [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, - [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, - [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, - [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, - [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, - [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, - [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 0, 0 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, - [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 2, .uops = { { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 3, .uops = { { LOAD_CONST, 0, 0 }, { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, - [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, - [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, - [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, - [POP_EXCEPT] = { .nuops = 1, .uops = { { POP_EXCEPT, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { LOAD_ASSERTION_ERROR, 0, 0 } } }, - [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, - [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, - [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, - [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE, 0, 0 } } }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, - [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, - [STORE_ATTR] = { .nuops = 1, .uops = { { STORE_ATTR, 0, 0 } } }, - [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, - [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, - [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, - [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, - [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, - [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, - [STORE_DEREF] = { .nuops = 1, .uops = { { STORE_DEREF, 0, 0 } } }, - [COPY_FREE_VARS] = { .nuops = 1, .uops = { { COPY_FREE_VARS, 0, 0 } } }, - [BUILD_STRING] = { .nuops = 1, .uops = { { BUILD_STRING, 0, 0 } } }, - [BUILD_TUPLE] = { .nuops = 1, .uops = { { BUILD_TUPLE, 0, 0 } } }, - [BUILD_LIST] = { .nuops = 1, .uops = { { BUILD_LIST, 0, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { LIST_EXTEND, 0, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { SET_UPDATE, 0, 0 } } }, - [BUILD_SET] = { .nuops = 1, .uops = { { BUILD_SET, 0, 0 } } }, - [BUILD_MAP] = { .nuops = 1, .uops = { { BUILD_MAP, 0, 0 } } }, - [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { SETUP_ANNOTATIONS, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { BUILD_CONST_KEY_MAP, 0, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, - [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, - [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, - [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, - [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, - [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { IS_OP, 0, 0 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, - [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, - [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, - [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, - [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, - [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { MATCH_SEQUENCE, 0, 0 } } }, - [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, - [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, - [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, - [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_BUILTIN_O, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_LEN] = { .nuops = 1, .uops = { { CALL_LEN, 0, 0 } } }, - [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_ISINSTANCE, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { MAKE_FUNCTION, 0, 0 } } }, - [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, - [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, - [CONVERT_VALUE] = { .nuops = 1, .uops = { { CONVERT_VALUE, 0, 0 } } }, - [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, - [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, - [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, - [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } }, - [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, -}; -#endif // NEED_OPCODE_METADATA - -extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; -#ifdef NEED_OPCODE_METADATA -const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { - [_EXIT_TRACE] = "_EXIT_TRACE", - [_SET_IP] = "_SET_IP", - [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", - [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", - [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", - [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", - [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", - [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", - [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", - [_POP_FRAME] = "_POP_FRAME", - [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", - [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", - [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", - [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", - [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", - [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", - [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", - [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", - [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", - [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", - [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", - [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", - [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", - [_IS_NONE] = "_IS_NONE", - [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", - [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", - [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST", - [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", - [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", - [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", - [_IS_ITER_EXHAUSTED_TUPLE] = "_IS_ITER_EXHAUSTED_TUPLE", - [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", - [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", - [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", - [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", - [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", - [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", - [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", - [_CHECK_PEP_523] = "_CHECK_PEP_523", - [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", - [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", - [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", - [_PUSH_FRAME] = "_PUSH_FRAME", - [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", - [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", - [_JUMP_TO_TOP] = "_JUMP_TO_TOP", - [_SAVE_CURRENT_IP] = "_SAVE_CURRENT_IP", - [_INSERT] = "_INSERT", -}; -#endif // NEED_OPCODE_METADATA - -extern const char *const _PyOpcode_OpName[268]; -#ifdef NEED_OPCODE_METADATA -const char *const _PyOpcode_OpName[268] = { - [CACHE] = "CACHE", - [RESERVED] = "RESERVED", - [RESUME] = "RESUME", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [BINARY_SLICE] = "BINARY_SLICE", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CLEANUP_THROW] = "CLEANUP_THROW", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", - [FORMAT_SIMPLE] = "FORMAT_SIMPLE", - [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [GET_ITER] = "GET_ITER", - [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_LOCALS] = "LOAD_LOCALS", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [MATCH_KEYS] = "MATCH_KEYS", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [NOP] = "NOP", - [POP_EXCEPT] = "POP_EXCEPT", - [POP_TOP] = "POP_TOP", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [PUSH_NULL] = "PUSH_NULL", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [RETURN_VALUE] = "RETURN_VALUE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [STORE_SLICE] = "STORE_SLICE", - [STORE_SUBSCR] = "STORE_SUBSCR", - [TO_BOOL] = "TO_BOOL", - [UNARY_INVERT] = "UNARY_INVERT", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, 0, 0 } } }, + [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, + [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_DICT, 0, 0 } } }, + [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_LIST_INT, 0, 0 } } }, + [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_STR_INT, 0, 0 } } }, + [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, + [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, + [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, + [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, + [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, + [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, + [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, + [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, + [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, + [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, + [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _COMPARE_OP_STR, 0, 0 } } }, + [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 1, .uops = { { _CONTAINS_OP_DICT, 0, 0 } } }, + [CONTAINS_OP_SET] = { .nuops = 1, .uops = { { _CONTAINS_OP_SET, 0, 0 } } }, + [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, + [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, + [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, 0, 0 } } }, + [DELETE_ATTR] = { .nuops = 1, .uops = { { _DELETE_ATTR, 0, 0 } } }, + [DELETE_DEREF] = { .nuops = 1, .uops = { { _DELETE_DEREF, 0, 0 } } }, + [DELETE_FAST] = { .nuops = 1, .uops = { { _DELETE_FAST, 0, 0 } } }, + [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, 0, 0 } } }, + [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, 0, 0 } } }, + [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, + [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, + [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, + [END_FOR] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, + [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, + [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, + [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, + [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, 0, 0 } } }, + [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 9, 0 } } }, + [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _FOR_ITER_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 9, 1 }, { _ITER_NEXT_LIST, 0, 0 } } }, + [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 9, 1 }, { _ITER_NEXT_RANGE, 0, 0 } } }, + [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 9, 1 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, + [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, 0, 0 } } }, + [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, 0, 0 } } }, + [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, 0, 0 } } }, + [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, + [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, + [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, + [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, + [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, + [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, + [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } }, + [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, + [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, + [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, + [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } }, + [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, + [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, + [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, + [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, + [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, 0, 0 } } }, + [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, 0, 0 } } }, + [MATCH_CLASS] = { .nuops = 1, .uops = { { _MATCH_CLASS, 0, 0 } } }, + [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, 0, 0 } } }, + [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, 0, 0 } } }, + [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, 0, 0 } } }, + [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, + [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 9, 1 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 9, 1 } } }, + [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, + [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, 0, 0 } } }, + [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, 0, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _RETURN_VALUE, 0, 0 } } }, + [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, 0, 0 } } }, + [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, + [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, + [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, + [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, + [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, + [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _STORE_FAST, 6, 0 } } }, + [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, 0, 0 } } }, + [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, 0, 0 } } }, + [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, 0, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_DICT, 0, 0 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, + [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, + [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } }, + [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, + [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, + [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, + [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, 0, 0 } } }, + [TO_BOOL_STR] = { .nuops = 1, .uops = { { _TO_BOOL_STR, 0, 0 } } }, + [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, 0, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE, 0, 0 } } }, + [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, 0, 0 } } }, + [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_LIST, 0, 0 } } }, + [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, + [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, +}; +#endif // NEED_OPCODE_METADATA + +extern const char *_PyOpcode_OpName[264]; +#ifdef NEED_OPCODE_METADATA +const char *_PyOpcode_OpName[264] = { [BINARY_OP] = "BINARY_OP", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", @@ -1874,13 +1410,42 @@ const char *const _PyOpcode_OpName[268] = { [BUILD_SLICE] = "BUILD_SLICE", [BUILD_STRING] = "BUILD_STRING", [BUILD_TUPLE] = "BUILD_TUPLE", + [CACHE] = "CACHE", [CALL] = "CALL", + [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_O] = "CALL_BUILTIN_O", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_LEN] = "CALL_LEN", + [CALL_LIST_APPEND] = "CALL_LIST_APPEND", + [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", + [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_GENERAL] = "CALL_PY_GENERAL", + [CALL_STR_1] = "CALL_STR_1", + [CALL_TUPLE_1] = "CALL_TUPLE_1", + [CALL_TYPE_1] = "CALL_TYPE_1", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", [COMPARE_OP] = "COMPARE_OP", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [CONTAINS_OP] = "CONTAINS_OP", + [CONTAINS_OP_DICT] = "CONTAINS_OP_DICT", + [CONTAINS_OP_SET] = "CONTAINS_OP_SET", [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", [COPY_FREE_VARS] = "COPY_FREE_VARS", @@ -1889,21 +1454,74 @@ const char *const _PyOpcode_OpName[268] = { [DELETE_FAST] = "DELETE_FAST", [DELETE_GLOBAL] = "DELETE_GLOBAL", [DELETE_NAME] = "DELETE_NAME", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [EXTENDED_ARG] = "EXTENDED_ARG", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", [FOR_ITER] = "FOR_ITER", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [IS_OP] = "IS_OP", + [JUMP] = "JUMP", [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", @@ -1913,139 +1531,88 @@ const char *const _PyOpcode_OpName[268] = { [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_LOCALS] = "LOAD_LOCALS", [LOAD_NAME] = "LOAD_NAME", + [LOAD_SPECIAL] = "LOAD_SPECIAL", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [MAKE_CELL] = "MAKE_CELL", + [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", [MATCH_CLASS] = "MATCH_CLASS", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_BLOCK] = "POP_BLOCK", + [POP_EXCEPT] = "POP_EXCEPT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", [RERAISE] = "RERAISE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [RESUME_CHECK] = "RESUME_CHECK", [RETURN_CONST] = "RETURN_CONST", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_GEN] = "SEND_GEN", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", [SET_ADD] = "SET_ADD", [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", [SET_UPDATE] = "SET_UPDATE", [STORE_ATTR] = "STORE_ATTR", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_DEREF] = "STORE_DEREF", [STORE_FAST] = "STORE_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", [STORE_GLOBAL] = "STORE_GLOBAL", [STORE_NAME] = "STORE_NAME", - [SWAP] = "SWAP", - [UNPACK_EX] = "UNPACK_EX", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [YIELD_VALUE] = "YIELD_VALUE", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_BUILTIN_O] = "CALL_BUILTIN_O", - [CALL_ISINSTANCE] = "CALL_ISINSTANCE", - [CALL_LEN] = "CALL_LEN", - [CALL_LIST_APPEND] = "CALL_LIST_APPEND", - [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", - [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_STR_1] = "CALL_STR_1", - [CALL_TUPLE_1] = "CALL_TUPLE_1", - [CALL_TYPE_1] = "CALL_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [RESUME_CHECK] = "RESUME_CHECK", - [SEND_GEN] = "SEND_GEN", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [SWAP] = "SWAP", + [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", [TO_BOOL_INT] = "TO_BOOL_INT", [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_CLOSURE] = "LOAD_CLOSURE", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [POP_BLOCK] = "POP_BLOCK", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_WITH] = "SETUP_WITH", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [YIELD_VALUE] = "YIELD_VALUE", }; -#endif // NEED_OPCODE_METADATA +#endif extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, @@ -2056,22 +1623,20 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_SUPER_ATTR] = 1, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, - [POP_JUMP_IF_FALSE] = 1, + [CONTAINS_OP] = 1, [POP_JUMP_IF_TRUE] = 1, + [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, [BINARY_OP] = 1, - [JUMP_BACKWARD] = 1, }; -#endif // NEED_OPCODE_METADATA +#endif extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2099,6 +1664,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_GENERAL] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -2114,8 +1680,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_PY_GENERAL] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -2127,6 +1694,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [COMPARE_OP_INT] = COMPARE_OP, [COMPARE_OP_STR] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, + [CONTAINS_OP_DICT] = CONTAINS_OP, + [CONTAINS_OP_SET] = CONTAINS_OP, [CONVERT_VALUE] = CONVERT_VALUE, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, @@ -2185,7 +1754,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [JUMP_FORWARD] = JUMP_FORWARD, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, @@ -2200,6 +1768,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_SLOT] = LOAD_ATTR, [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, @@ -2213,6 +1782,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_LOCALS] = LOAD_LOCALS, [LOAD_NAME] = LOAD_NAME, + [LOAD_SPECIAL] = LOAD_SPECIAL, [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, @@ -2279,9 +1849,11 @@ const uint8_t _PyOpcode_Deopt[256] = { [WITH_EXCEPT_START] = WITH_EXCEPT_START, [YIELD_VALUE] = YIELD_VALUE, }; + #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 118: \ case 119: \ case 120: \ case 121: \ @@ -2312,10 +1884,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ case 223: \ case 224: \ case 225: \ @@ -2331,4 +1899,36 @@ const uint8_t _PyOpcode_Deopt[256] = { case 235: \ case 255: \ ; +struct pseudo_targets { + uint8_t targets[3]; +}; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[8]; +#ifdef NEED_OPCODE_METADATA +const struct pseudo_targets _PyOpcode_PseudoTargets[8] = { + [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, + [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, + [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, + [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, + [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, + [SETUP_WITH-256] = { { NOP, 0, 0 } }, + [POP_BLOCK-256] = { { NOP, 0, 0 } }, +}; +#endif // NEED_OPCODE_METADATA +static inline bool +is_pseudo_target(int pseudo, int target) { + if (pseudo < 256 || pseudo >= 264) { + return false; + } + for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { + if (_PyOpcode_PseudoTargets[pseudo-256].targets[i] == target) return true; + } + return false; +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_OPCODE_METADATA_H */ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index c4acb00a4b291e..e76f4840a66891 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -57,7 +57,21 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 +/* Values used as the oparg for LOAD_COMMON_CONSTANT */ +#define CONSTANT_ASSERTIONERROR 0 +#define CONSTANT_NOTIMPLEMENTEDERROR 1 +#define NUM_COMMON_CONSTANTS 2 + +/* Values used in the oparg for RESUME */ +#define RESUME_AT_FUNC_START 0 +#define RESUME_AFTER_YIELD 1 +#define RESUME_AFTER_YIELD_FROM 2 +#define RESUME_AFTER_AWAIT 3 + +#define RESUME_OPARG_LOCATION_MASK 0x3 +#define RESUME_OPARG_DEPTH1_MASK 0x4 #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f9f16c48cbc21c..bcbb8b73706359 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,11 +8,256 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_uops.h" // _PyUOpInstruction +#include "pycore_uop_ids.h" +#include <stdbool.h> -int _Py_uop_analyze_and_optimize(PyCodeObject *code, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries); +typedef struct _PyExecutorLinkListNode { + struct _PyExecutorObject *next; + struct _PyExecutorObject *previous; +} _PyExecutorLinkListNode; + + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +typedef struct { + uint8_t opcode; + uint8_t oparg; + uint8_t valid; + uint8_t linked; + int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + _PyBloomFilter bloom; + _PyExecutorLinkListNode links; + PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). +} _PyVMData; + +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct { + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand; // A cache entry +} _PyUOpInstruction; + +typedef struct { + uint32_t target; + _Py_BackoffCounter temperature; + const struct _PyExecutorObject *executor; +} _PyExitData; + +typedef struct _PyExecutorObject { + PyObject_VAR_HEAD + const _PyUOpInstruction *trace; + _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + uint32_t exit_count; + uint32_t code_size; + size_t jit_size; + void *jit_code; + void *jit_side_entry; + _PyExitData exits[1]; +} _PyExecutorObject; + +typedef struct _PyOptimizerObject _PyOptimizerObject; + +/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ +typedef int (*_Py_optimize_func)( + _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, + int curr_stackentries); + +struct _PyOptimizerObject { + PyObject_HEAD + _Py_optimize_func optimize; + /* Data needed by the optimizer goes here, but is opaque to the VM */ +}; + +/** Test support **/ +typedef struct { + _PyOptimizerObject base; + int64_t count; +} _PyCounterOptimizerObject; + +_PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); + + +// Export for '_opcode' shared extension (JIT compiler). +PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); + +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +void _Py_ExecutorDetach(_PyExecutorObject *); +void _Py_BloomFilter_Init(_PyBloomFilter *); +void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); +PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); + +// For testing +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); +PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); + +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 + +#ifdef _Py_TIER2 +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +#else +# define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) +# define _Py_Executors_InvalidateAll(A, B) ((void)0) +#endif + + +// This is the length of the trace we project initially. +#define UOP_MAX_TRACE_LENGTH 800 + +#define TRACE_STACK_SIZE 5 + +int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries, + _PyBloomFilter *dependencies); + +extern PyTypeObject _PyCounterExecutor_Type; +extern PyTypeObject _PyCounterOptimizer_Type; +extern PyTypeObject _PyDefaultOptimizer_Type; +extern PyTypeObject _PyUOpExecutor_Type; +extern PyTypeObject _PyUOpOptimizer_Type; + +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + +struct _Py_UopsSymbol { + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyTypeObject *typ; // Borrowed reference + PyObject *const_val; // Owned reference (!) + unsigned int type_version; // currently stores type version +}; + +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_JUMP 1 + +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; +} + +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} + +// Holds locals, stack, locals, stack ... co_consts (in that order) +#define MAX_ABSTRACT_INTERP_SIZE 4096 + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; + +struct _Py_UOpsAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; +}; + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + +struct _Py_UOpsContext { + char done; + char out_of_space; + bool contradiction; + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsContext _Py_UOpsContext; + +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( + _Py_UOpsContext *ctx, PyTypeObject *typ); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); +extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); + + +extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len); +extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); + +PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); + +PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr); #ifdef __cplusplus } diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h index f444da730055e8..8c9260e2636fbc 100644 --- a/Include/internal/pycore_parking_lot.h +++ b/Include/internal/pycore_parking_lot.h @@ -18,8 +18,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - enum { // The thread was unparked by another thread. @@ -61,7 +59,7 @@ enum { // } PyAPI_FUNC(int) _PyParkingLot_Park(const void *address, const void *expected, - size_t address_size, _PyTime_t timeout_ns, + size_t address_size, PyTime_t timeout_ns, void *park_arg, int detach); // Callback for _PyParkingLot_Unpark: diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h new file mode 100644 index 00000000000000..a1bb383bcd22e9 --- /dev/null +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -0,0 +1,94 @@ +// This header file provides wrappers around the atomic operations found in +// `pyatomic.h` that are only atomic in free-threaded builds. +// +// These are intended to be used in places where atomics are required in +// free-threaded builds, but not in the default build, and we don't want to +// introduce the potential performance overhead of an atomic operation in the +// default build. +// +// All usages of these macros should be replaced with unconditionally atomic or +// non-atomic versions, and this file should be removed, once the dust settles +// on free threading. +#ifndef Py_ATOMIC_FT_WRAPPERS_H +#define Py_ATOMIC_FT_WRAPPERS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +#error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED +#define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \ + _Py_atomic_load_ssize_acquire(&value) +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ + _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) \ + _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ + _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ + _Py_atomic_load_uintptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ + _Py_atomic_load_ptr_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT8(value) \ + _Py_atomic_load_uint8(&value) +#define FT_ATOMIC_STORE_UINT8(value, new_value) \ + _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ + _Py_atomic_load_ulong_relaxed(&value) +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ + _Py_atomic_store_ptr_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ + _Py_atomic_store_ptr_release(&value, new_value) +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ + _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ + _Py_atomic_store_uint8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) + +#else +#define FT_ATOMIC_LOAD_PTR(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SSIZE(value) value +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT8(value) value +#define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value + +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_ATOMIC_FT_WRAPPERS_H */ diff --git a/Include/internal/pycore_pybuffer.h b/Include/internal/pycore_pybuffer.h index 3cbc290b2ea3ee..9439d2bd770587 100644 --- a/Include/internal/pycore_pybuffer.h +++ b/Include/internal/pycore_pybuffer.h @@ -9,7 +9,7 @@ extern "C" { #endif -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreter( PyInterpreterState *interp, Py_buffer *view); PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreterAndRawFree( diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 184eb35e52b47b..15071638203457 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -95,7 +95,7 @@ extern void _PyErr_Fetch( extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); -extern int _PyErr_ExceptionMatches( +PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); @@ -114,18 +114,18 @@ extern void _PyErr_SetObject( extern void _PyErr_ChainStackItem(void); -extern void _PyErr_Clear(PyThreadState *tstate); +PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern void _PyErr_SetString( +PyAPI_FUNC(void) _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); -extern PyObject* _PyErr_Format( +PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, const char *format, @@ -167,13 +167,10 @@ void _PyErr_FormatNote(const char *format, ...); Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -// Export for '_zoneinfo' shared extension -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); - -// Export for '_lsprof' shared extension -PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( - const char *err_msg, - PyObject *obj); +// implementation detail for the codeop module. +// Exported for test.test_peg_generator.test_c_parser +PyAPI_DATA(PyTypeObject) _PyExc_IncompleteInputError; +#define PyExc_IncompleteInputError ((PyObject *)(&_PyExc_IncompleteInputError)) #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 78bf0c7d07eb10..0ce08900e96f0b 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -5,35 +5,24 @@ # error "this header requires Py_BUILD_CORE define" #endif -/* Helpers for hash functions */ -extern Py_hash_t _Py_HashDouble(PyObject *, double); +// Similar to Py_HashPointer(), but don't replace -1 with -2. +static inline Py_hash_t +_Py_HashPointerRaw(const void *ptr) +{ + uintptr_t x = (uintptr_t)ptr; + Py_BUILD_ASSERT(sizeof(x) == sizeof(ptr)); -// Export for '_decimal' shared extension -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); + // Bottom 3 or 4 bits are likely to be 0; rotate x by 4 to the right + // to avoid excessive hash collisions for dicts and sets. + x = (x >> 4) | (x << (8 * sizeof(uintptr_t) - 4)); -// Similar to _Py_HashPointer(), but don't replace -1 with -2 -extern Py_hash_t _Py_HashPointerRaw(const void*); + Py_BUILD_ASSERT(sizeof(x) == sizeof(Py_hash_t)); + return (Py_hash_t)x; +} // Export for '_datetime' shared extension PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); -/* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ - -/* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ - -#if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 -#else -# define _PyHASH_BITS 31 -#endif - -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER - /* Hash secret * * memory layout on 64 bit systems diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index ec003a1dad2595..f426ae0e103b9c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -40,10 +40,8 @@ extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); -extern PyStatus _PyTime_Init(void); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); -extern int _Py_Deepfreeze_Init(void); /* Various internal finalizers */ @@ -59,12 +57,11 @@ extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); -extern void _Py_Deepfreeze_Fini(void); extern void _PyArg_Fini(void); extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); extern PyStatus _PyGILState_Init(PyInterpreterState *interp); -extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); +extern void _PyGILState_SetTstate(PyThreadState *tstate); extern void _PyGILState_Fini(PyInterpreterState *interp); extern void _PyGC_DumpShutdownStats(PyInterpreterState *interp); @@ -114,6 +111,25 @@ extern int _Py_LegacyLocaleDetected(int warn); // Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); +// Export for special main.c string compiling with source tracebacks +int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + + +/* interpreter config */ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 6b5113714dbeb2..dd6b0762370c92 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,5 +1,9 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H + +#include "pycore_llist.h" // struct llist_node +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -30,7 +34,7 @@ typedef struct { } debug_alloc_api_t; struct _pymem_allocators { - PyThread_type_lock mutex; + PyMutex mutex; struct { PyMemAllocatorEx raw; PyMemAllocatorEx mem; @@ -41,9 +45,15 @@ struct _pymem_allocators { debug_alloc_api_t mem; debug_alloc_api_t obj; } debug; + int is_debug_enabled; PyObjectArenaAllocator obj_arena; }; +struct _Py_mem_interp_free_queue { + int has_work; // true if the queue is not empty + PyMutex mutex; // protects the queue + struct llist_node head; // queue of _mem_work_chunk items +}; /* Set the memory allocator of the specified domain to the default. Save the old allocator into *old_alloc if it's non-NULL. @@ -61,7 +71,7 @@ extern int _PyMem_SetDefaultAllocator( - PYMEM_FORBIDDENBYTE: untouchable bytes at each end of a block Byte patterns 0xCB, 0xDB and 0xFB have been replaced with 0xCD, 0xDD and - 0xFD to use the same values than Windows CRT debug malloc() and free(). + 0xFD to use the same values as Windows CRT debug malloc() and free(). If modified, _PyMem_IsPtrFreed() should be updated as well. */ #define PYMEM_CLEANBYTE 0xCD #define PYMEM_DEADBYTE 0xDD @@ -103,6 +113,24 @@ extern int _PyMem_GetAllocatorName( PYMEM_ALLOCATOR_NOT_SET does nothing. */ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); +/* Is the debug allocator enabled? */ +extern int _PyMem_DebugEnabled(void); + +// Enqueue a pointer to be freed possibly after some delay. +extern void _PyMem_FreeDelayed(void *ptr); + +// Enqueue an object to be freed possibly after some delay +extern void _PyObject_FreeDelayed(void *ptr); + +// Periodically process delayed free requests. +extern void _PyMem_ProcessDelayed(PyThreadState *tstate); + +// Abandon all thread-local delayed free requests and push them to the +// interpreter's queue. +extern void _PyMem_AbandonDelayed(PyThreadState *tstate); + +// On interpreter shutdown, frees all delayed free requests. +extern void _PyMem_FiniDelayed(PyInterpreterState *interp); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 119fa16fb911ae..c593edc86d9952 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -18,17 +18,30 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t); extern void _PyMem_RawFree(void *, void *); #define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} -#ifdef WITH_PYMALLOC +#ifdef Py_GIL_DISABLED +// Py_GIL_DISABLED requires mimalloc +extern void* _PyObject_MiMalloc(void *, size_t); +extern void* _PyObject_MiCalloc(void *, size_t, size_t); +extern void _PyObject_MiFree(void *, void *); +extern void* _PyObject_MiRealloc(void *, void *, size_t); +# define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree} +extern void* _PyMem_MiMalloc(void *, size_t); +extern void* _PyMem_MiCalloc(void *, size_t, size_t); +extern void _PyMem_MiFree(void *, void *); +extern void* _PyMem_MiRealloc(void *, void *, size_t); +# define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +#elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); extern void _PyObject_Free(void *, void *); extern void* _PyObject_Realloc(void *, void *, size_t); # define PYOBJ_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} +# define PYMEM_ALLOC PYOBJ_ALLOC #else # define PYOBJ_ALLOC PYRAW_ALLOC +# define PYMEM_ALLOC PYOBJ_ALLOC #endif // WITH_PYMALLOC -#define PYMEM_ALLOC PYOBJ_ALLOC extern void* _PyMem_DebugRawMalloc(void *, size_t); extern void* _PyMem_DebugRawCalloc(void *, size_t, size_t); @@ -57,6 +70,7 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); PYDBGMEM_ALLOC(runtime), \ PYDBGOBJ_ALLOC(runtime), \ } +# define _pymem_is_debug_enabled_INIT 1 #else # define _pymem_allocators_standard_INIT(runtime) \ { \ @@ -64,6 +78,7 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); PYMEM_ALLOC, \ PYOBJ_ALLOC, \ } +# define _pymem_is_debug_enabled_INIT 0 #endif #define _pymem_allocators_debug_INIT \ @@ -77,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); { NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree } +#define _Py_mem_free_queue_INIT(queue) \ + { \ + .head = LLIST_INIT(queue.head), \ + } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7135b1e966feb5..b0e72523f58ed8 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime +#include "pycore_tstate.h" // _PyThreadStateImpl // Values for PyThreadState.state. A thread must be in the "attached" state @@ -19,23 +21,27 @@ extern "C" { // interpreter at the same time. Only the "bound" thread may perform the // transitions between "attached" and "detached" on its own PyThreadState. // -// The "gc" state is used to implement stop-the-world pauses, such as for -// cyclic garbage collection. It is only used in `--disable-gil` builds. It is -// similar to the "detached" state, but only the thread performing a -// stop-the-world pause may transition threads between the "detached" and "gc" -// states. A thread trying to "attach" from the "gc" state will block until -// it is transitioned back to "detached" when the stop-the-world pause is -// complete. +// The "suspended" state is used to implement stop-the-world pauses, such as +// for cyclic garbage collection. It is only used in `--disable-gil` builds. +// The "suspended" state is similar to the "detached" state in that in both +// states the thread is not allowed to call most Python APIs. However, unlike +// the "detached" state, a thread may not transition itself out from the +// "suspended" state. Only the thread performing a stop-the-world pause may +// transition a thread from the "suspended" state back to the "detached" state. // // State transition diagram: // // (bound thread) (stop-the-world thread) -// [attached] <-> [detached] <-> [gc] +// [attached] <-> [detached] <-> [suspended] +// | ^ +// +---------------------------->---------------------------+ +// (bound thread) // -// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`. +// The (bound thread) and (stop-the-world thread) labels indicate which thread +// is allowed to perform the transition. #define _Py_THREAD_DETACHED 0 #define _Py_THREAD_ATTACHED 1 -#define _Py_THREAD_GC 2 +#define _Py_THREAD_SUSPENDED 2 /* Check if the current thread is the main thread. @@ -71,12 +77,18 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &_PyRuntime._main_interpreter); } -// Export for _xxsubinterpreters module. +// Export for _interpreters module. +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); + +// Export for _interpreters module. PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); +extern int _PyThreadState_IsRunningMain(PyThreadState *); +extern void _PyInterpreterState_ReinitRunningMain(PyThreadState *); + static inline const PyConfig * _Py_GetMainConfig(void) @@ -138,13 +150,36 @@ _PyThreadState_GET(void) // // High-level code should generally call PyEval_RestoreThread() instead, which // calls this function. -void _PyThreadState_Attach(PyThreadState *tstate); +extern void _PyThreadState_Attach(PyThreadState *tstate); // Detaches the current thread from the interpreter. // // High-level code should generally call PyEval_SaveThread() instead, which // calls this function. -void _PyThreadState_Detach(PyThreadState *tstate); +extern void _PyThreadState_Detach(PyThreadState *tstate); + +// Detaches the current thread to the "suspended" state if a stop-the-world +// pause is in progress. +// +// If there is no stop-the-world pause in progress, then the thread switches +// to the "detached" state. +extern void _PyThreadState_Suspend(PyThreadState *tstate); + +// Perform a stop-the-world pause for all threads in the all interpreters. +// +// Threads in the "attached" state are paused and transitioned to the "GC" +// state. Threads in the "detached" state switch to the "GC" state, preventing +// them from reattaching until the stop-the-world pause is complete. +// +// NOTE: This is a no-op outside of Py_GIL_DISABLED builds. +extern void _PyEval_StopTheWorldAll(_PyRuntimeState *runtime); +extern void _PyEval_StartTheWorldAll(_PyRuntimeState *runtime); + +// Perform a stop-the-world pause for threads in the specified interpreter. +// +// NOTE: This is a no-op outside of Py_GIL_DISABLED builds. +extern void _PyEval_StopTheWorld(PyInterpreterState *interp); +extern void _PyEval_StartTheWorld(PyInterpreterState *interp); static inline void @@ -182,20 +217,21 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { // PyThreadState functions -extern PyThreadState * _PyThreadState_New( +// Export for _testinternalcapi +PyAPI_FUNC(PyThreadState *) _PyThreadState_New( PyInterpreterState *interp, int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); -extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +PyAPI_FUNC(PyThreadState *) _PyThreadState_NewBound( + PyInterpreterState *interp, + int whence); +extern PyThreadState * _PyThreadState_RemoveExcept(PyThreadState *tstate); +extern void _PyThreadState_DeleteList(PyThreadState *list); +extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); -/* The implementation of sys._current_frames() Returns a dict mapping - thread id to that thread's current frame. -*/ -extern PyObject* _PyThread_CurrentFrames(void); - /* The implementation of sys._current_exceptions() Returns a dict mapping thread id to that thread's current exception. */ @@ -225,9 +261,9 @@ PyAPI_FUNC(int) _PyState_AddModule( extern int _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ - PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK) + PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ - PyThread_release_lock((runtime)->interpreters.mutex) + PyMutex_Unlock(&(runtime)->interpreters.mutex) // Get the configuration of the current interpreter. // The caller must hold the GIL. @@ -243,6 +279,20 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); +static inline struct _Py_object_freelists* _Py_object_freelists_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif + +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelists; +#else + return &tstate->interp->object_state.freelists; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index f679c1bdb75499..f3f5942444e851 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX +#include "pycore_llist.h" // struct llist_node + // Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available #if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ && !defined(_POSIX_SEMAPHORES)) @@ -43,7 +46,8 @@ extern "C" { #if defined(HAVE_PTHREAD_STUBS) -#include <stdbool.h> // bool +#include "cpython/pthread_stubs.h" // PTHREAD_KEYS_MAX +#include <stdbool.h> // bool // pthread_key struct py_stub_tls_entry { @@ -73,17 +77,82 @@ struct _pythread_runtime_state { struct py_stub_tls_entry tls_entries[PTHREAD_KEYS_MAX]; } stubs; #endif + + // Linked list of ThreadHandles + struct llist_node handles; }; +#define _pythread_RUNTIME_INIT(pythread) \ + { \ + .handles = LLIST_INIT(pythread.handles), \ + } #ifdef HAVE_FORK /* Private function to reinitialize a lock at fork in the child process. Reset the lock to the unlocked state. Return 0 on success, return -1 on error. */ extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); #endif /* HAVE_FORK */ +// unset: -1 seconds, in nanoseconds +#define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000)) + +// Exported for the _interpchannels module. +PyAPI_FUNC(int) PyThread_ParseTimeoutArg( + PyObject *arg, + int blocking, + PY_TIMEOUT_T *timeout); + +/* Helper to acquire an interruptible lock with a timeout. If the lock acquire + * is interrupted, signal handlers are run, and if they raise an exception, + * PY_LOCK_INTR is returned. Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE + * are returned, depending on whether the lock can be acquired within the + * timeout. + */ +// Exported for the _interpchannels module. +PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries( + PyThread_type_lock, + PY_TIMEOUT_T microseconds); + +typedef unsigned long long PyThread_ident_t; +typedef Py_uintptr_t PyThread_handle_t; + +#define PY_FORMAT_THREAD_IDENT_T "llu" +#define Py_PARSE_THREAD_IDENT_T "K" + +PyAPI_FUNC(PyThread_ident_t) PyThread_get_thread_ident_ex(void); + +/* Thread joining APIs. + * + * These APIs have a strict contract: + * - Either PyThread_join_thread or PyThread_detach_thread must be called + * exactly once with the given handle. + * - Calling neither PyThread_join_thread nor PyThread_detach_thread results + * in a resource leak until the end of the process. + * - Any other usage, such as calling both PyThread_join_thread and + * PyThread_detach_thread, or calling them more than once (including + * simultaneously), results in undefined behavior. + */ +PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), + void *arg, + PyThread_ident_t* ident, + PyThread_handle_t* handle); +/* + * Join a thread started with `PyThread_start_joinable_thread`. + * This function cannot be interrupted. It returns 0 on success, + * a non-zero value on failure. + */ +PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); +/* + * Detach a thread started with `PyThread_start_joinable_thread`, such + * that its resources are released as soon as it exits. + * This function cannot be interrupted. It returns 0 on success, + * a non-zero value on failure. + */ +PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h new file mode 100644 index 00000000000000..20e643e172b38d --- /dev/null +++ b/Include/internal/pycore_qsbr.h @@ -0,0 +1,154 @@ +// The QSBR APIs (quiescent state-based reclamation) provide a mechanism for +// the free-threaded build to safely reclaim memory when there may be +// concurrent accesses. +// +// Many operations in the free-threaded build are protected by locks. However, +// in some cases, we want to allow reads to happen concurrently with updates. +// In this case, we need to delay freeing ("reclaiming") any memory that may be +// concurrently accessed by a reader. The QSBR APIs provide a way to do this. +#ifndef Py_INTERNAL_QSBR_H +#define Py_INTERNAL_QSBR_H + +#include <stdbool.h> +#include <stdint.h> +#include "pycore_lock.h" // PyMutex + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. This avoids collisions +// between the offline state and any valid sequence number even if the +// sequences numbers wrap around. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + +struct _qsbr_shared; +struct _PyThreadStateImpl; // forward declare to avoid circular dependency + +// Per-thread state +struct _qsbr_thread_state { + // Last observed write sequence (or 0 if detached) + uint64_t seq; + + // Shared (per-interpreter) QSBR state + struct _qsbr_shared *shared; + + // Thread state (or NULL) + PyThreadState *tstate; + + // Used to defer advancing write sequence a fixed number of times + int deferrals; + + // Is this thread state allocated? + bool allocated; + struct _qsbr_thread_state *freelist_next; +}; + +// Padding to avoid false sharing +struct _qsbr_pad { + struct _qsbr_thread_state qsbr; + char __padding[64 - sizeof(struct _qsbr_thread_state)]; +}; + +// Per-interpreter state +struct _qsbr_shared { + // Write sequence: always odd, incremented by two + uint64_t wr_seq; + + // Minimum observed read sequence of all QSBR thread states + uint64_t rd_seq; + + // Array of QSBR thread states. + struct _qsbr_pad *array; + Py_ssize_t size; + + // Freelist of unused _qsbr_thread_states (protected by mutex) + PyMutex mutex; + struct _qsbr_thread_state *freelist; +}; + +static inline uint64_t +_Py_qsbr_shared_current(struct _qsbr_shared *shared) +{ + return _Py_atomic_load_uint64_acquire(&shared->wr_seq); +} + +// Reports a quiescent state: the caller no longer holds any pointer to shared +// data not protected by locks or reference counts. +static inline void +_Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) +{ + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64_release(&qsbr->seq, seq); +} + +// Have the read sequences advanced to the given goal? Like `_Py_qsbr_poll()`, +// but does not perform a scan of threads. +static inline bool +_Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + return QSBR_LEQ(goal, rd_seq); +} + +// Advance the write sequence and return the new goal. This should be called +// after data is removed. The returned goal is used with `_Py_qsbr_poll()` to +// determine when it is safe to reclaim (free) the memory. +extern uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared); + +// Batches requests to advance the write sequence. This advances the write +// sequence every N calls, which reduces overhead but increases time to +// reclamation. Returns the new goal. +extern uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); + +// Have the read sequences advanced to the given goal? If this returns true, +// it safe to reclaim any memory tagged with the goal (or earlier goal). +extern bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal); + +// Called when thread attaches to interpreter +extern void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr); + +// Called when thread detaches from interpreter +extern void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr); + +// Reserves (allocates) a QSBR state and returns its index. +extern Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp); + +// Associates a PyThreadState with the QSBR state at the given index +extern void +_Py_qsbr_register(struct _PyThreadStateImpl *tstate, + PyInterpreterState *interp, Py_ssize_t index); + +// Disassociates a PyThreadState from the QSBR state and frees the QSBR state. +extern void +_Py_qsbr_unregister(PyThreadState *tstate); + +extern void +_Py_qsbr_fini(PyInterpreterState *interp); + +extern void +_Py_qsbr_after_fork(struct _PyThreadStateImpl *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_QSBR_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f2383b43fabaf1..d4ffd977940a02 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_ceval_state.h" // struct _ceval_runtime_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state @@ -20,13 +21,11 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_signal.h" // struct _signals_runtime_state -#include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state struct _getargs_runtime_state { - PyThread_type_lock mutex; struct _PyArg_Parser *static_parsers; }; @@ -56,76 +55,102 @@ typedef struct _Py_DebugOffsets { uint64_t version; // Runtime state offset; struct _runtime_state { - off_t finalizing; - off_t interpreters_head; + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; } runtime_state; // Interpreter state offset; struct _interpreter_state { - off_t next; - off_t threads_head; - off_t gc; - off_t imports_modules; - off_t sysdict; - off_t builtins; - off_t ceval_gil; - off_t gil_runtime_state_locked; - off_t gil_runtime_state_holder; + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; } interpreter_state; // Thread state offset; struct _thread_state{ - off_t prev; - off_t next; - off_t interp; - off_t current_frame; - off_t thread_id; - off_t native_thread_id; + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; } thread_state; // InterpreterFrame offset; struct _interpreter_frame { - off_t previous; - off_t executable; - off_t prev_instr; - off_t localsplus; - off_t owner; + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; } interpreter_frame; - // CFrame offset; - struct _cframe { - off_t current_frame; - off_t previous; - } cframe; - // Code object offset; struct _code_object { - off_t filename; - off_t name; - off_t linetable; - off_t firstlineno; - off_t argcount; - off_t localsplusnames; - off_t localspluskinds; - off_t co_code_adaptive; + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; } code_object; // PyObject offset; struct _pyobject { - off_t ob_type; + uint64_t size; + uint64_t ob_type; } pyobject; // PyTypeObject object offset; struct _type_object { - off_t tp_name; + uint64_t size; + uint64_t tp_name; } type_object; // PyTuple object offset; struct _tuple_object { - off_t ob_item; + uint64_t size; + uint64_t ob_item; } tuple_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + size_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; } _Py_DebugOffsets; +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. @@ -174,7 +199,7 @@ typedef struct pyruntimestate { unsigned long _finalizing_id; struct pyinterpreters { - PyThread_type_lock mutex; + PyMutex mutex; /* The linked list of interpreters, newest first. */ PyInterpreterState *head; /* The runtime's initial interpreter, which has a special role @@ -192,20 +217,22 @@ typedef struct pyruntimestate { int64_t next_id; } interpreters; + /* Platform-specific identifier and PyThreadState, respectively, for the + main thread in the main interpreter. */ unsigned long main_thread; + PyThreadState *main_tstate; /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability tools. */ - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry xidregistry; + /* cross-interpreter data and utils */ + struct _xi_runtime_state xi; struct _pymem_allocators allocators; struct _obmalloc_global_state obmalloc; struct pyhash_runtime_state pyhash_state; - struct _time_runtime_state time; struct _pythread_runtime_state threads; struct _signals_runtime_state signals; @@ -228,6 +255,14 @@ typedef struct pyruntimestate { struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state ref_tracer; + + // The rwmutex is used to prevent overlapping global and per-interpreter + // stop-the-world events. Global stop-the-world events lock the mutex + // exclusively (as a "writer"), while per-interpreter stop-the-world events + // lock it non-exclusively (as "readers"). + _PyRWMutex stoptheworld_mutex; + struct _stoptheworld_state stoptheworld; PyPreConfig preconfig; @@ -236,7 +271,7 @@ typedef struct pyruntimestate { Py_OpenCodeHookFunction open_code_hook; void *open_code_userdata; struct { - PyThread_type_lock mutex; + PyMutex mutex; _Py_AuditHookEntry *head; } audit_hooks; @@ -263,7 +298,7 @@ typedef struct pyruntimestate { a pointer type. */ - /* PyInterpreterState.interpreters.main */ + /* _PyRuntimeState.interpreters.main */ PyInterpreterState _main_interpreter; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 574a3c1a9db66c..da2b8d5570de62 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -16,6 +16,8 @@ extern "C" { #include "pycore_parser.h" // _parser_runtime_state_INIT #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_pythread.h" // _pythread_RUNTIME_INIT +#include "pycore_qsbr.h" // QSBR_INITIAL #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -33,10 +35,13 @@ extern PyTypeObject _PyExc_MemoryError; .cookie = "xdebugpy", \ .version = PY_VERSION_HEX, \ .runtime_state = { \ + .size = sizeof(_PyRuntimeState), \ .finalizing = offsetof(_PyRuntimeState, _finalizing), \ .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ }, \ .interpreter_state = { \ + .size = sizeof(PyInterpreterState), \ + .id = offsetof(PyInterpreterState, id), \ .next = offsetof(PyInterpreterState, next), \ .threads_head = offsetof(PyInterpreterState, threads.head), \ .gc = offsetof(PyInterpreterState, gc), \ @@ -48,23 +53,29 @@ extern PyTypeObject _PyExc_MemoryError; .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ }, \ .thread_state = { \ + .size = sizeof(PyThreadState), \ .prev = offsetof(PyThreadState, prev), \ .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ + .status = offsetof(PyThreadState, _status), \ }, \ .interpreter_frame = { \ + .size = sizeof(_PyInterpreterFrame), \ .previous = offsetof(_PyInterpreterFrame, previous), \ .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ .code_object = { \ + .size = sizeof(PyCodeObject), \ .filename = offsetof(PyCodeObject, co_filename), \ .name = offsetof(PyCodeObject, co_name), \ + .qualname = offsetof(PyCodeObject, co_qualname), \ .linetable = offsetof(PyCodeObject, co_linetable), \ .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ .argcount = offsetof(PyCodeObject, co_argcount), \ @@ -73,33 +84,57 @@ extern PyTypeObject _PyExc_MemoryError; .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ }, \ .pyobject = { \ + .size = sizeof(PyObject), \ .ob_type = offsetof(PyObject, ob_type), \ }, \ .type_object = { \ + .size = sizeof(PyTypeObject), \ .tp_name = offsetof(PyTypeObject, tp_name), \ }, \ .tuple_object = { \ + .size = sizeof(PyTupleObject), \ .ob_item = offsetof(PyTupleObject, ob_item), \ }, \ + .unicode_object = { \ + .size = sizeof(PyUnicodeObject), \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ + .gc = { \ + .size = sizeof(struct _gc_runtime_state), \ + .collecting = offsetof(struct _gc_runtime_state, collecting), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ .obj_arena = _pymem_allocators_obj_arena_INIT, \ + .is_debug_enabled = _pymem_is_debug_enabled_INIT, \ }, \ .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ + .threads = _pythread_RUNTIME_INIT(runtime.threads), \ .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ .next_id = -1, \ }, \ + .xi = { \ + .registry = { \ + .global = 1, \ + }, \ + }, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ .ceval = { \ + .pending_mainthread = { \ + .max = MAXPENDINGCALLS_MAIN, \ + .maxloop = MAXPENDINGCALLSLOOP_MAIN, \ + }, \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ .gilstate = { \ @@ -110,6 +145,13 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .ref_tracer = { \ + .tracer_func = NULL, \ + .tracer_data = NULL, \ + }, \ + .stoptheworld = { \ + .is_global = 1, \ + }, \ .float_state = { \ .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ @@ -129,13 +171,13 @@ extern PyTypeObject _PyExc_MemoryError; .latin1 = _Py_str_latin1_INIT, \ }, \ .tuple_empty = { \ - .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0), \ }, \ .hamt_bitmap_node_empty = { \ - .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0), \ }, \ .context_token_missing = { \ - .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \ + .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type), \ }, \ }, \ }, \ @@ -145,23 +187,31 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyInterpreterState_INIT(INTERP) \ { \ .id_refcount = -1, \ + ._whence = _PyInterpreterState_WHENCE_NOTSET, \ .imports = IMPORTS_INIT, \ - .obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ + .pending = { \ + .max = MAXPENDINGCALLS, \ + .maxloop = MAXPENDINGCALLSLOOP, \ + }, \ }, \ .gc = { \ .enabled = 1, \ - .generations = { \ - /* .head is set in _PyGC_InitState(). */ \ - { .threshold = 700, }, \ - { .threshold = 10, }, \ + .young = { .threshold = 2000, }, \ + .old = { \ { .threshold = 10, }, \ + { .threshold = 0, }, \ }, \ + .work_to_do = -5000, \ + }, \ + .qsbr = { \ + .wr_seq = QSBR_INITIAL, \ + .rd_seq = QSBR_INITIAL, \ }, \ - .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ + .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ .next_version = 1, \ }, \ @@ -172,15 +222,21 @@ extern PyTypeObject _PyExc_MemoryError; .singletons = { \ ._not_used = 1, \ .hamt_empty = { \ - .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type) \ + .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ .last_resort_memory_error = { \ - _PyObject_HEAD_INIT(&_PyExc_MemoryError) \ + _PyObject_HEAD_INIT(&_PyExc_MemoryError), \ + .args = (PyObject*)&_Py_SINGLETON(tuple_empty) \ }, \ }, \ }, \ - ._initial_thread = _PyThreadState_INIT, \ + ._initial_thread = _PyThreadStateImpl_INIT, \ + } + +#define _PyThreadStateImpl_INIT \ + { \ + .base = _PyThreadState_INIT, \ } #define _PyThreadState_INIT \ @@ -190,22 +246,12 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } -#ifdef Py_TRACE_REFS -# define _py_object_state_INIT(INTERP) \ - { \ - .refchain = NULL, \ - } -#else -# define _py_object_state_INIT(INTERP) \ - { 0 } -#endif - // global objects #define _PyBytes_SIMPLE_INIT(CH, LEN) \ { \ - _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \ + _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)), \ .ob_shash = -1, \ .ob_sval = { (CH) }, \ } @@ -216,7 +262,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \ { \ - .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type) \ + .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type), \ .length = sizeof(LITERAL) - 1, \ .hash = -1, \ .state = { \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 43a8243bf41b7e..c5be67c6d80b9d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -546,21 +546,17 @@ extern "C" { INIT_STR(anon_setcomp, "<setcomp>"), \ INIT_STR(anon_string, "<string>"), \ INIT_STR(anon_unknown, "<unknown>"), \ - INIT_STR(close_br, "}"), \ INIT_STR(dbl_close_br, "}}"), \ INIT_STR(dbl_open_br, "{{"), \ INIT_STR(dbl_percent, "%%"), \ INIT_STR(defaults, ".defaults"), \ - INIT_STR(dot, "."), \ INIT_STR(dot_locals, ".<locals>"), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ - INIT_STR(newline, "\n"), \ - INIT_STR(open_br, "{"), \ - INIT_STR(percent, "%"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } @@ -575,7 +571,6 @@ extern "C" { INIT_ID(TextIOWrapper), \ INIT_ID(True), \ INIT_ID(WarningMessage), \ - INIT_ID(_), \ INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ @@ -588,6 +583,7 @@ extern "C" { INIT_ID(__all__), \ INIT_ID(__and__), \ INIT_ID(__anext__), \ + INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ INIT_ID(__asyncio_running_event_loop__), \ @@ -622,6 +618,7 @@ extern "C" { INIT_ID(__eq__), \ INIT_ID(__exit__), \ INIT_ID(__file__), \ + INIT_ID(__firstlineno__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ INIT_ID(__format__), \ @@ -722,12 +719,12 @@ extern "C" { INIT_ID(__slotnames__), \ INIT_ID(__slots__), \ INIT_ID(__spec__), \ + INIT_ID(__static_attributes__), \ INIT_ID(__str__), \ INIT_ID(__sub__), \ INIT_ID(__subclasscheck__), \ INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ - INIT_ID(__trunc__), \ INIT_ID(__type_params__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_prepare_subst__), \ @@ -740,7 +737,7 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ - INIT_ID(_annotation), \ + INIT_ID(_align_), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ @@ -750,6 +747,7 @@ extern "C" { INIT_ID(_check_retval_), \ INIT_ID(_dealloc_warn), \ INIT_ID(_feature_version), \ + INIT_ID(_field_types), \ INIT_ID(_fields_), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ @@ -760,23 +758,25 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_layout_), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ + INIT_ID(_only_immortal), \ INIT_ID(_pack_), \ INIT_ID(_restype_), \ INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ + INIT_ID(_strptime), \ INIT_ID(_strptime_datetime), \ INIT_ID(_swappedbytes_), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ INIT_ID(_xoptions), \ - INIT_ID(a), \ INIT_ID(abs_tol), \ INIT_ID(access), \ INIT_ID(aclose), \ @@ -785,8 +785,12 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ + INIT_ID(allow_code), \ INIT_ID(append), \ + INIT_ID(arg), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -796,7 +800,6 @@ extern "C" { INIT_ID(attribute), \ INIT_ID(authorizer_callback), \ INIT_ID(autocommit), \ - INIT_ID(b), \ INIT_ID(backtick), \ INIT_ID(base), \ INIT_ID(before), \ @@ -814,16 +817,17 @@ extern "C" { INIT_ID(byteorder), \ INIT_ID(bytes), \ INIT_ID(bytes_per_sep), \ - INIT_ID(c), \ INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callback), \ INIT_ID(cancel), \ INIT_ID(capath), \ INIT_ID(category), \ @@ -854,6 +858,7 @@ extern "C" { INIT_ID(co_stacksize), \ INIT_ID(co_varnames), \ INIT_ID(code), \ + INIT_ID(col_offset), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ @@ -867,15 +872,16 @@ extern "C" { INIT_ID(count), \ INIT_ID(covariant), \ INIT_ID(cwd), \ - INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(day), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ INIT_ID(defaultaction), \ INIT_ID(delete), \ INIT_ID(depth), \ + INIT_ID(desired_access), \ INIT_ID(detect_types), \ INIT_ID(deterministic), \ INIT_ID(device), \ @@ -891,17 +897,17 @@ extern "C" { INIT_ID(displayhook), \ INIT_ID(dklen), \ INIT_ID(doc), \ + INIT_ID(done), \ INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ - INIT_ID(duration), \ - INIT_ID(e), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ + INIT_ID(end_col_offset), \ INIT_ID(end_lineno), \ INIT_ID(end_offset), \ INIT_ID(endpos), \ @@ -910,13 +916,14 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc_type), \ + INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ - INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -932,15 +939,16 @@ extern "C" { INIT_ID(fileno), \ INIT_ID(filepath), \ INIT_ID(fillvalue), \ + INIT_ID(filter), \ INIT_ID(filters), \ INIT_ID(final), \ INIT_ID(find_class), \ INIT_ID(fix_imports), \ INIT_ID(flags), \ INIT_ID(flush), \ + INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ - INIT_ID(frequency), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -948,7 +956,6 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ - INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -962,15 +969,17 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ - INIT_ID(h), \ INIT_ID(handle), \ + INIT_ID(handle_seq), \ + INIT_ID(has_location), \ INIT_ID(hash_name), \ INIT_ID(header), \ INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ - INIT_ID(id), \ + INIT_ID(hour), \ INIT_ID(ident), \ + INIT_ID(identity_hint), \ INIT_ID(ignore), \ INIT_ID(imag), \ INIT_ID(importlib), \ @@ -979,9 +988,12 @@ extern "C" { INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ + INIT_ID(inherit_handle), \ INIT_ID(inheritable), \ INIT_ID(initial), \ INIT_ID(initial_bytes), \ + INIT_ID(initial_owner), \ + INIT_ID(initial_state), \ INIT_ID(initial_value), \ INIT_ID(initval), \ INIT_ID(inner_size), \ @@ -1013,6 +1025,8 @@ extern "C" { INIT_ID(kw), \ INIT_ID(kw1), \ INIT_ID(kw2), \ + INIT_ID(kwdefaults), \ + INIT_ID(label), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1036,11 +1050,13 @@ extern "C" { INIT_ID(locals), \ INIT_ID(logoption), \ INIT_ID(loop), \ + INIT_ID(manual_reset), \ INIT_ID(mapping), \ INIT_ID(match), \ INIT_ID(max_length), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ + INIT_ID(maxlen), \ INIT_ID(maxmem), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ @@ -1050,15 +1066,19 @@ extern "C" { INIT_ID(metaclass), \ INIT_ID(metadata), \ INIT_ID(method), \ + INIT_ID(microsecond), \ + INIT_ID(milliseconds), \ + INIT_ID(minute), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ + INIT_ID(mutex), \ INIT_ID(mycmp), \ - INIT_ID(n), \ INIT_ID(n_arg), \ INIT_ID(n_fields), \ INIT_ID(n_sequence_fields), \ @@ -1069,6 +1089,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(nested), \ INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ @@ -1103,7 +1124,6 @@ extern "C" { INIT_ID(outgoing), \ INIT_ID(overlapped), \ INIT_ID(owner), \ - INIT_ID(p), \ INIT_ID(pages), \ INIT_ID(parent), \ INIT_ID(password), \ @@ -1131,7 +1151,6 @@ extern "C" { INIT_ID(ps2), \ INIT_ID(query), \ INIT_ID(quotetabs), \ - INIT_ID(r), \ INIT_ID(raw), \ INIT_ID(read), \ INIT_ID(read1), \ @@ -1155,10 +1174,11 @@ extern "C" { INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ - INIT_ID(s), \ INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ + INIT_ID(second), \ + INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ @@ -1184,9 +1204,9 @@ extern "C" { INIT_ID(sleep), \ INIT_ID(sock), \ INIT_ID(sort), \ - INIT_ID(sound), \ INIT_ID(source), \ INIT_ID(source_traceback), \ + INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ INIT_ID(stacklevel), \ @@ -1226,6 +1246,7 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ + INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ @@ -1235,6 +1256,7 @@ extern "C" { INIT_ID(type), \ INIT_ID(type_params), \ INIT_ID(tz), \ + INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ INIT_ID(unlink), \ @@ -1245,6 +1267,8 @@ extern "C" { INIT_ID(values), \ INIT_ID(version), \ INIT_ID(volume), \ + INIT_ID(wait_all), \ + INIT_ID(warn_on_full_buffer), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ @@ -1256,7 +1280,6 @@ extern "C" { INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ - INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ } diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 4c37df7b39a48a..ffcc6d80344d6e 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -8,7 +8,6 @@ #endif #include "pycore_pythread.h" // _POSIX_SEMAPHORES -#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -48,7 +47,7 @@ typedef struct _PySemaphore { // If `detach` is true, then the thread will detach/release the GIL while // sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 34a00e6d45fe69..0494c07fe1869d 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -8,19 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// Export for '_pickle' shared extension +// Export for '_abc' shared extension PyAPI_FUNC(int) _PySet_NextEntry( PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); +// Export for '_pickle' shared extension +PyAPI_FUNC(int) _PySet_NextEntryRef( + PyObject *set, + Py_ssize_t *pos, + PyObject **key, + Py_hash_t *hash); + // Export for '_pickle' shared extension PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); // Export for the gdb plugin's (python-gdb.py) benefit PyAPI_DATA(PyObject *) _PySet_Dummy; +PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key); + +// Clears the set without acquiring locks. Used by _PyCode_Fini. +extern void _PySet_ClearInternal(PySetObject *so); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 98665c3859d574..ba8b1f1cb27dee 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,9 +11,7 @@ extern "C" { /* runtime lifecycle */ -extern void _PySlice_Fini(PyInterpreterState *); - -extern PyObject * +PyAPI_FUNC(PyObject *) _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); #ifdef __cplusplus diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 00000000000000..8d3d559814bfd9 --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,244 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_object_deferred.h" + +#include <stddef.h> + +/* + This file introduces a new API for handling references on the stack, called + _PyStackRef. This API is inspired by HPy. + + There are 3 main operations, that convert _PyStackRef to PyObject* and + vice versa: + + 1. Borrow (discouraged) + 2. Steal + 3. New + + Borrow means that the reference is converted without any change in ownership. + This is discouraged because it makes verification much harder. It also makes + unboxed integers harder in the future. + + Steal means that ownership is transferred to something else. The total + number of references to the object stays the same. + + New creates a new reference from the old reference. The old reference + is still valid. + + With these 3 API, a strict stack discipline must be maintained. All + _PyStackRef must be operated on by the new reference operations: + + 1. DUP + 2. CLOSE + + DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old + reference. The old reference remains unchanged. + + CLOSE is roughly equivalent to Py_DECREF. It destroys a reference. + + Note that it is unsafe to borrow a _PyStackRef and then do normal + CPython refcounting operations on it! +*/ + +typedef union _PyStackRef { + uintptr_t bits; +} _PyStackRef; + + +#define Py_TAG_DEFERRED (1) + +#define Py_TAG_PTR (0) +#define Py_TAG_BITS (1) + +#ifdef Py_GIL_DISABLED + static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED}; +#else + static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; +#endif + +#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) + + +#ifdef Py_GIL_DISABLED +# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) }) +#endif + +#ifdef Py_GIL_DISABLED +# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) +#endif + +#ifdef Py_GIL_DISABLED +# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) +#endif + +// Note: the following are all macros because MSVC (Windows) has trouble inlining them. + +#define PyStackRef_Is(a, b) ((a).bits == (b).bits) + +#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) + + +#ifdef Py_GIL_DISABLED +// Gets a PyObject * from a _PyStackRef +static inline PyObject * +PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) +{ + PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); + return cleared; +} +#else +# define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits) +#endif + +// Converts a PyStackRef back to a PyObject *, stealing the +// PyStackRef. +#ifdef Py_GIL_DISABLED +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef stackref) +{ + if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) { + return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); + } + return PyStackRef_AsPyObjectBorrow(stackref); +} +#else +# define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref) +#endif + +// Converts a PyStackRef back to a PyObject *, converting the +// stackref to a new reference. +#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + +// Converts a PyObject * to a PyStackRef, stealing the reference +#ifdef Py_GIL_DISABLED +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR; + return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); +} +# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) +#else +# define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) +#endif + + +// Converts a PyObject * to a PyStackRef, with a new reference +#ifdef Py_GIL_DISABLED +static inline _PyStackRef +PyStackRef_FromPyObjectNew(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + // TODO (gh-117139): Add deferred objects later. + if (_Py_IsImmortal(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; + } +} +# define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) +#else +# define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }) +#endif + +#ifdef Py_GIL_DISABLED +// Same as PyStackRef_FromPyObjectNew but only for immortal objects. +static inline _PyStackRef +PyStackRef_FromPyObjectImmortal(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + assert(_Py_IsImmortal(obj)); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; +} +# define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) +#else +# define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) }) +#endif + + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (!PyStackRef_IsNull(_tmp_old_op)) { \ + *_tmp_op_ptr = PyStackRef_NULL; \ + PyStackRef_CLOSE(_tmp_old_op); \ + } \ + } while (0) + +#ifdef Py_GIL_DISABLED +static inline void +PyStackRef_CLOSE(_PyStackRef stackref) +{ + if (PyStackRef_IsDeferred(stackref)) { + // No assert for being immortal or deferred here. + // The GC unsets deferred objects right before clearing. + return; + } + Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)); +} +#else +# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)); +#endif + +#define PyStackRef_XCLOSE(stackref) \ + do { \ + _PyStackRef _tmp = (stackref); \ + if (!PyStackRef_IsNull(_tmp)) { \ + PyStackRef_CLOSE(_tmp); \ + } \ + } while (0); + + +#ifdef Py_GIL_DISABLED +static inline _PyStackRef +PyStackRef_DUP(_PyStackRef stackref) +{ + if (PyStackRef_IsDeferred(stackref)) { + assert(PyStackRef_IsNull(stackref) || + _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref))); + return stackref; + } + Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); + return stackref; +} +#else +# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))); +#endif + +static inline void +_PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_AsPyObjectBorrow(src[i]); + } +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 1d782ca2c96e05..d9ed16a3d2321f 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,14 +12,27 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, - // Used for generics and type aliases. These work mostly like functions - // (see PEP 695 for details). The three different blocks function identically; - // they are different enum entries only so that error messages can be more - // precise. - TypeVarBoundBlock, TypeAliasBlock, TypeParamBlock + + // The following blocks are used for generics and type aliases. These work + // mostly like functions (see PEP 695 for details). The three different + // blocks function identically; they are different enum entries only so + // that error messages can be more precise. + + // The block to enter when processing a "type" (PEP 695) construction, + // e.g., "type MyGeneric[T] = list[T]". + TypeAliasBlock, + // The block to enter when processing a "generic" (PEP 695) object, + // e.g., "def foo[T](): pass" or "class A[T]: pass". + TypeParametersBlock, + // The block to enter when processing the bound, the constraint tuple + // or the default value of a single "type variable" in the formal sense, + // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two + // do not support a bound or a constraint tuple). + TypeVariableBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -29,6 +42,29 @@ typedef enum _comprehension_type { SetComprehension = 3, GeneratorExpression = 4 } _Py_comprehension_ty; +/* source location information */ +typedef struct { + int lineno; + int end_lineno; + int col_offset; + int end_col_offset; +} _Py_SourceLocation; + +#define SRC_LOCATION_FROM_AST(n) \ + (_Py_SourceLocation){ \ + .lineno = (n)->lineno, \ + .end_lineno = (n)->end_lineno, \ + .col_offset = (n)->col_offset, \ + .end_col_offset = (n)->end_col_offset } + +static const _Py_SourceLocation NO_LOCATION = {-1, -1, -1, -1}; + +/* __future__ information */ +typedef struct { + int ff_features; /* flags set by future statements */ + _Py_SourceLocation ff_location; /* location of last future statement */ +} _PyFutureFeatures; + struct _symtable_entry; struct symtable { @@ -44,7 +80,7 @@ struct symtable { consistency with the corresponding compiler structure */ PyObject *st_private; /* name of current class or NULL */ - PyFutureFeatures *st_future; /* module's future features that affect + _PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -58,13 +94,24 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ + PyObject *ste_mangled_names; /* set of names for which mangling should be applied */ + _Py_block_ty ste_type; + // Optional string set by symtable.c and used when reporting errors. + // The content of that string is a description of the current "context". + // + // For instance, if we are processing the default value of the type + // variable "T" in "def foo[T = int](): pass", `ste_scope_info` is + // set to "a TypeVar default". + const char *ste_scope_info; + int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ unsigned ste_child_free : 1; /* true if a child block has free vars, including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -80,12 +127,8 @@ typedef struct _symtable_entry { unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ - int ste_lineno; /* first line of block */ - int ste_col_offset; /* offset of first line of block */ - int ste_end_lineno; /* end line of block */ - int ste_end_col_offset; /* end offset of first line of block */ - int ste_opt_lineno; /* lineno of last exec or import * */ - int ste_opt_col_offset; /* offset of last exec or import * */ + _Py_SourceLocation ste_loc; /* source location of block */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -100,32 +143,33 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - PyFutureFeatures *future); + _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); +extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define DEF_NONLOCAL 2<<2 /* nonlocal stmt */ -#define USE 2<<3 /* name is used */ -#define DEF_FREE 2<<4 /* name used but not defined in nested block */ -#define DEF_FREE_CLASS 2<<5 /* free variable from class's method */ -#define DEF_IMPORT 2<<6 /* assignment occurred via import */ -#define DEF_ANNOT 2<<7 /* this name is annotated */ -#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ -#define DEF_TYPE_PARAM 2<<9 /* this name is a type parameter */ -#define DEF_COMP_CELL 2<<10 /* this name is a cell in an inlined comprehension */ +#define DEF_GLOBAL 1 /* global stmt */ +#define DEF_LOCAL 2 /* assignment in code block */ +#define DEF_PARAM (2<<1) /* formal parameter */ +#define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ +#define USE (2<<3) /* name is used */ +#define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ +#define DEF_IMPORT (2<<6) /* assignment occurred via import */ +#define DEF_ANNOT (2<<7) /* this name is annotated */ +#define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ +#define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ +#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol - table. GLOBAL is returned from PyST_GetScope() for either of them. + table. GLOBAL is returned from _PyST_GetScope() for either of them. It is stored in ste_symbols at bits 13-16. */ #define SCOPE_OFFSET 12 @@ -137,9 +181,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define FREE 4 #define CELL 5 -#define GENERATOR 1 -#define GENERATOR_EXPRESSION 2 - // Used by symtablemodule.c extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, @@ -150,7 +191,7 @@ extern struct symtable* _Py_SymtableStringObjectFlags( int _PyFuture_FromAST( struct _mod * mod, PyObject *filename, - PyFutureFeatures* futures); + _PyFutureFeatures* futures); #ifdef __cplusplus } diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 46713f91d190ff..15806552e0a384 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -1,34 +1,39 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. +// Internal PyTime_t C API: see Doc/c-api/time.rst for the documentation. // -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. +// The PyTime_t type is an integer to support directly common arithmetic +// operations such as t1 + t2. // -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. +// Time formats: // -// Formats: +// * Seconds. +// * Seconds as a floating point number (C double). +// * Milliseconds (10^-3 seconds). +// * Microseconds (10^-6 seconds). +// * 100 nanoseconds (10^-7 seconds), used on Windows. +// * Nanoseconds (10^-9 seconds). +// * timeval structure, 1 microsecond (10^-6 seconds). +// * timespec structure, 1 nanosecond (10^-9 seconds). // -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// Note that PyTime_t is now specified as int64_t, in nanoseconds. +// (If we need to change this, we'll need new public API with new names.) +// Previously, PyTime_t was configurable (in theory); some comments and code +// might still allude to that. // // Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). +// resolution larger than 1 nanosecond is rounded correctly with the requested +// rounding mode. Available rounding modes: // -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. +// * Round towards minus infinity (-inf). For example, used to read a clock. +// * Round towards infinity (+inf). For example, used for timeout to wait "at +// least" N seconds. +// * Round to nearest with ties going to nearest even integer. For example, used +// to round from a Python float. +// * Round away from zero. For example, used for timeout. +// +// Some functions clamp the result in the range [PyTime_MIN; PyTime_MAX]. The +// caller doesn't have to handle errors and so doesn't need to hold the GIL to +// handle exceptions. For example, _PyTime_Add(t1, t2) computes t1+t2 and +// clamps the result on overflow. // // Clocks: // @@ -36,10 +41,11 @@ // * Monotonic clock // * Performance counter // -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). +// Internally, operations like (t * k / q) with integers are implemented in a +// way to reduce the risk of integer overflow. Such operation is used to convert a +// clock value expressed in ticks with a frequency to PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency() on Windows. + #ifndef Py_INTERNAL_TIME_H #define Py_INTERNAL_TIME_H @@ -52,28 +58,10 @@ extern "C" { #endif -struct _time_runtime_state { -#ifdef HAVE_TIMES - int ticks_per_second_initialized; - long ticks_per_second; -#else - int _not_used; -#endif -}; - - #ifdef __clang__ struct timeval; #endif -// _PyTime_t: Python timestamp with subsecond precision. It can be used to -// store a duration, and so indirectly a date (related to another date, like -// UNIX epoch). -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX #define _SIZEOF_PYTIME_T 8 typedef enum { @@ -141,76 +129,66 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( // Create a timestamp from a number of seconds. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); +PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. -// Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +extern int _PyTime_FromSecondsDouble( + double seconds, + _PyTime_round_t round, + PyTime_t *result); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -// Create a timestamp from a number of nanoseconds. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + ((PyTime_t)(seconds) * (1000 * 1000 * 1000)) // Create a timestamp from a number of microseconds. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from nanoseconds (Python int). +// Create a timestamp from a Python int object (number of nanoseconds). // Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, PyObject *obj); // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromSecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); // Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); -// Convert a timestamp to a number of seconds as a C double. -// Export for '_socket' shared extension. -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - // Convert timestamp to a number of milliseconds (10^-3 seconds). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMilliseconds(PyTime_t t, _PyTime_round_t round); // Convert timestamp to a number of microseconds (10^-6 seconds). // Export for '_queue' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMicroseconds(PyTime_t t, _PyTime_round_t round); -// Convert timestamp to a number of nanoseconds (10^-9 seconds). -extern _PyTime_t _PyTime_AsNanoseconds(_PyTime_t t); - #ifdef MS_WINDOWS // Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, +extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int -// object. +// Convert a timestamp (number of nanoseconds) as a Python int object. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); +PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +extern int _PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv); #endif // Convert a timestamp to a timeval structure (microsecond resolution). @@ -218,14 +196,14 @@ extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); // Raise an exception and return -1 if the conversion overflowed, // return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, +PyAPI_FUNC(int) _PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round); // Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for 'select' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -237,7 +215,7 @@ PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, // return 0 on success. // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, + PyTime_t t, time_t *secs, int *us, _PyTime_round_t round); @@ -245,30 +223,23 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) // Create a timestamp from a timespec structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); +extern int _PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts); // Convert a timestamp to a timespec structure (nanosecond resolution). // tv_nsec is always positive. // Raise an exception and return -1 on error, return 0 on success. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(int) _PyTime_AsTimespec(PyTime_t t, struct timespec *ts); // Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts); #endif -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); - -// Compute ticks * mul / div. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -// The caller must ensure that ((div - 1) * mul) cannot overflow. -extern _PyTime_t _PyTime_MulDiv(_PyTime_t ticks, - _PyTime_t mul, - _PyTime_t div); +// Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTime_Add(PyTime_t t1, PyTime_t t2); // Structure used by time.get_clock_info() typedef struct { @@ -278,36 +249,13 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Get the current time from the system clock. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetSystemClockWithInfo() to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. -extern int _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, +extern int _PyTime_TimeWithInfo( + PyTime_t *t, _Py_clock_info_t *info); -// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. -// The clock is not affected by system clock updates. The reference point of -// the returned value is undefined, so that only the difference between the -// results of consecutive calls is valid. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetMonotonicClockWithInfo() to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of // the returned value is undefined, so that only the difference between the @@ -317,8 +265,8 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); // // Return 0 on success, raise an exception and return -1 on error. // Export for '_testsinglephase' shared extension. -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, +PyAPI_FUNC(int) _PyTime_MonotonicWithInfo( + PyTime_t *t, _Py_clock_info_t *info); @@ -332,16 +280,6 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Get the performance counter: clock with the highest available resolution to -// measure a short duration. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetPerfCounterWithInfo() to check for failure. -// Export for '_lsprof' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -349,20 +287,48 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Fill info (if set) with information of the function used to get the time. // // Return 0 on success, raise an exception and return -1 on error. -extern int _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, +extern int _PyTime_PerfCounterWithInfo( + PyTime_t *t, _Py_clock_info_t *info); +// --- _PyDeadline ----------------------------------------------------------- + // Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +// Pseudo code: return PyTime_MonotonicRaw() + timeout // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); +PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +// Pseudo code: return deadline - PyTime_MonotonicRaw() // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); + + +// --- _PyTimeFraction ------------------------------------------------------- + +typedef struct { + PyTime_t numer; + PyTime_t denom; +} _PyTimeFraction; + +// Set a fraction. +// Return 0 on success. +// Return -1 if the fraction is invalid. +extern int _PyTimeFraction_Set( + _PyTimeFraction *frac, + PyTime_t numer, + PyTime_t denom); + +// Compute ticks * frac.numer / frac.denom. +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTimeFraction_Mul( + PyTime_t ticks, + const _PyTimeFraction *frac); + +// Compute a clock resolution: frac.numer / frac.denom / 1e9. +extern double _PyTimeFraction_Resolution( + const _PyTimeFraction *frac); #ifdef __cplusplus diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h new file mode 100644 index 00000000000000..befca950920bac --- /dev/null +++ b/Include/internal/pycore_tstate.h @@ -0,0 +1,44 @@ +#ifndef Py_INTERNAL_TSTATE_H +#define Py_INTERNAL_TSTATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_freelist.h" // struct _Py_freelist_state +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr + + +// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The +// PyThreadState fields are exposed as part of the C API, although most fields +// are intended to be private. The _PyThreadStateImpl fields not exposed. +typedef struct _PyThreadStateImpl { + // semi-public fields are in PyThreadState. + PyThreadState base; + + struct _qsbr_thread_state *qsbr; // only used by free-threaded build + struct llist_node mem_free_queue; // delayed free queue + +#ifdef Py_GIL_DISABLED + struct _gc_thread_state gc; + struct _mimalloc_thread_state mimalloc; + struct _Py_object_freelists freelists; + struct _brc_thread_state brc; +#endif + +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + Py_ssize_t reftotal; // this thread's total refcount operations +#endif + +} _PyThreadStateImpl; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TSTATE_H */ diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 4fa7a12206bcb2..dfbbd6fd0c7de5 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -14,58 +14,15 @@ extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); -extern void _PyTuple_Fini(PyInterpreterState *); /* other API */ -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save - -#if defined(PyTuple_MAXSAVESIZE) && PyTuple_MAXSAVESIZE <= 0 - // A build indicated that tuple freelists should not be used. -# define PyTuple_NFREELISTS 0 -# undef PyTuple_MAXSAVESIZE -# undef PyTuple_MAXFREELIST - -#elif !defined(WITH_FREELISTS) -# define PyTuple_NFREELISTS 0 -# undef PyTuple_MAXSAVESIZE -# undef PyTuple_MAXFREELIST - -#else - // We are using a freelist for tuples. -# ifndef PyTuple_MAXSAVESIZE -# define PyTuple_MAXSAVESIZE 20 -# endif -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# ifndef PyTuple_MAXFREELIST -# define PyTuple_MAXFREELIST 2000 -# endif -#endif - -struct _Py_tuple_state { -#if PyTuple_NFREELISTS > 0 - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *free_list[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; -#else - char _unused; // Empty structs are not allowed. -#endif -}; - #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); -extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); - +PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefSteal(const union _PyStackRef *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 27c6c8731cb3f9..df6bfef715dd34 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_lock.h" // PyMutex /* state */ @@ -16,11 +17,26 @@ extern "C" { #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) +/* For now we hard-code this to a value for which we are confident + all the static builtin types will fit (for all builds). */ +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 +#define _Py_MAX_MANAGED_STATIC_TYPES \ + (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; + + struct { + struct { + PyTypeObject *type; + PyTypeObject def; + int64_t interp_count; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; }; @@ -28,6 +44,9 @@ struct _types_runtime_state { // see _PyType_Lookup(). struct type_cache_entry { unsigned int version; // initialized from type->tp_version_tag +#ifdef Py_GIL_DISABLED + _PySeqLock sequence; +#endif PyObject *name; // reference to exactly a str or None PyObject *value; // borrowed reference or NULL }; @@ -38,12 +57,9 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -/* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_STATIC_BUILTIN_TYPES 200 - typedef struct { PyTypeObject *type; + int isbuiltin; int readying; int ready; // XXX tp_dict can probably be statically allocated, @@ -55,7 +71,9 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; -} static_builtin_state; +} managed_static_type_state; + +#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct types_state { /* Used to set PyTypeObject.tp_version_tag. @@ -64,8 +82,60 @@ struct types_state { unsigned int next_version_tag; struct type_cache type_cache; - size_t num_builtins_initialized; - static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + + /* Every static builtin type is initialized for each interpreter + during its own initialization, including for the main interpreter + during global runtime initialization. This is done by calling + _PyStaticType_InitBuiltin(). + + The first time a static builtin type is initialized, all the + normal PyType_Ready() stuff happens. The only difference from + normal is that there are three PyTypeObject fields holding + objects which are stored here (on PyInterpreterState) rather + than in the corresponding PyTypeObject fields. Those are: + tp_dict (cls.__dict__), tp_subclasses (cls.__subclasses__), + and tp_weaklist. + + When a subinterpreter is initialized, each static builtin type + is still initialized, but only the interpreter-specific portion, + namely those three objects. + + Those objects are stored in the PyInterpreterState.types.builtins + array, at the index corresponding to each specific static builtin + type. That index (a size_t value) is stored in the tp_subclasses + field. For static builtin types, we re-purposed the now-unused + tp_subclasses to avoid adding another field to PyTypeObject. + In all other cases tp_subclasses holds a dict like before. + (The field was previously defined as PyObject*, but is now void* + to reflect its dual use.) + + The index for each static builtin type isn't statically assigned. + Instead it is calculated the first time a type is initialized + (by the main interpreter). The index matches the order in which + the type was initialized relative to the others. The actual + value comes from the current value of num_builtins_initialized, + as each type is initialized for the main interpreter. + + num_builtins_initialized is incremented once for each static + builtin type. Once initialization is over for a subinterpreter, + the value will be the same as for all other interpreters. */ + struct { + size_t num_initialized; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]; + } builtins; + /* We apply a similar strategy for managed extension modules. */ + struct { + size_t num_initialized; + size_t next_index; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; + } for_extensions; + PyMutex mutex; + + // Borrowed references to type objects whose + // tp_version_tag % TYPE_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the type object is deallocated. + PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE]; }; @@ -73,8 +143,9 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); - +extern void _PyTypes_AfterFork(void); /* other API */ @@ -88,12 +159,31 @@ typedef struct wrapperbase pytype_slotdef; static inline PyObject ** -_PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) +_PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) { assert(state != NULL); return &state->tp_weaklist; } +extern int _PyStaticType_InitBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_ClearWeakRefs( + PyInterpreterState *interp, + PyTypeObject *type); +extern managed_static_type_state * _PyStaticType_GetState( + PyInterpreterState *interp, + PyTypeObject *type); + +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyStaticType_InitForExtension( + PyInterpreterState *interp, + PyTypeObject *self); + + /* Like PyType_GetModuleState, but skips verification * that type is a heap type with an associated module */ static inline void * @@ -109,11 +199,6 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); - // Export for 'math' shared extension, used via _PyType_IsReady() static inline // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); @@ -122,6 +207,7 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // PyType_Ready() must be called if _PyType_IsReady() is false. // See also the Py_TPFLAGS_READY flag. @@ -133,16 +219,36 @@ _PyType_IsReady(PyTypeObject *type) extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); -extern PyObject* _Py_type_getattro(PyTypeObject *type, PyObject *name); +extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name); + +extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; -extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, +PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); + +// Perform the following operation, in a thread-safe way when required by the +// build mode. +// +// self->tp_flags = (self->tp_flags & ~mask) | flags; +extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, + unsigned long flags); + +// Like _PyType_SetFlags(), but apply the operation to self and any of its +// subclasses without Py_TPFLAGS_IMMUTABLETYPE set. +extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, + unsigned long flags); + +extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); +PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); +PyTypeObject *_PyType_LookupByVersion(unsigned int version); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index c9fa97d6820757..a368edebd622a1 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -13,10 +13,13 @@ extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern PyTypeObject _PyTypeAlias_Type; +extern PyTypeObject _PyNoDefault_Type; +extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus } diff --git a/Include/internal/pycore_ucnhash.h b/Include/internal/pycore_ucnhash.h index 187dd68e7347ff..1561dfbb3150d3 100644 --- a/Include/internal/pycore_ucnhash.h +++ b/Include/internal/pycore_ucnhash.h @@ -28,6 +28,8 @@ typedef struct { } _PyUnicode_Name_CAPI; +extern _PyUnicode_Name_CAPI* _PyUnicode_GetNameCAPI(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 360a9e1819f8e8..20497ee93016d0 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -8,9 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_fileutils.h" // _Py_error_handler -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_global_objects.h" // _Py_SINGLETON /* --- Characters Type APIs ----------------------------------------------- */ @@ -30,8 +31,9 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); -extern void _PyUnicode_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); +extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void); // Get a copy of a Unicode string. // Export for '_datetime' shared extension. @@ -73,122 +75,6 @@ extern Py_UCS4 _PyUnicode_FindMaxChar ( /* --- _PyUnicodeWriter API ----------------------------------------------- */ -typedef struct { - PyObject *buffer; - void *data; - int kind; - Py_UCS4 maxchar; - Py_ssize_t size; - Py_ssize_t pos; - - /* minimum number of allocated characters (default: 0) */ - Py_ssize_t min_length; - - /* minimum character (default: 127, ASCII) */ - Py_UCS4 min_char; - - /* If non-zero, overallocate the buffer (default: 0). */ - unsigned char overallocate; - - /* If readonly is 1, buffer is a shared string (cannot be modified) - and size is set to 0. */ - unsigned char readonly; -} _PyUnicodeWriter ; - -// Initialize a Unicode writer. -// -// By default, the minimum buffer size is 0 character and overallocation is -// disabled. Set min_length, min_char and overallocate attributes to control -// the allocation of the buffer. -// -// Export the _PyUnicodeWriter API for '_multibytecodec' shared extension. -PyAPI_FUNC(void) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); - -/* Prepare the buffer to write 'length' characters - with the specified maximum character. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ - (((MAXCHAR) <= (WRITER)->maxchar \ - && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ - ? 0 \ - : (((LENGTH) == 0) \ - ? 0 \ - : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) - -/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro - instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar); - -/* Prepare the buffer to have at least the kind KIND. - For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will - support characters in range U+000-U+FFFF. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ - ((KIND) <= (WRITER)->kind \ - ? 0 \ - : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) - -/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() - macro instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - int kind); - -/* Append a Unicode character. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, - Py_UCS4 ch - ); - -/* Append a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, - PyObject *str /* Unicode string */ - ); - -/* Append a substring of a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, - PyObject *str, /* Unicode string */ - Py_ssize_t start, - Py_ssize_t end - ); - -/* Append an ASCII-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *str, /* ASCII-encoded byte string */ - Py_ssize_t len /* number of bytes, or -1 if unknown */ - ); - -/* Append a latin1-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, /* latin1-encoded byte string */ - Py_ssize_t len /* length in bytes */ - ); - -/* Get the value of the writer as a Unicode string. Clear the - buffer of the writer. Raise an exception and return NULL - on error. */ -PyAPI_FUNC(PyObject *) -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); - - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ extern int _PyUnicode_FormatAdvancedWriter( @@ -304,7 +190,7 @@ extern PyObject* _PyUnicode_EncodeCharmap( /* --- Decimal Encoder ---------------------------------------------------- */ -// Coverts a Unicode object holding a decimal value to an ASCII string +// Converts a Unicode object holding a decimal value to an ASCII string // for using in int, float and complex parsers. // Transforms code points that have decimal digit property to the // corresponding ASCII digit code points. Transforms spaces to ASCII. @@ -317,7 +203,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ -extern PyObject* _PyUnicode_JoinArray( +PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( PyObject *separator, PyObject *const *items, Py_ssize_t seqlen @@ -366,11 +252,6 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -// Return an interned Unicode object for an Identifier; may fail if there is no -// memory. -// Export for '_testembed' program. -PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); - /* Fast equality check when the inputs are known to be exact unicode types and where the hash values are equal (i.e. a very probable match) */ extern int _PyUnicode_EQ(PyObject *, PyObject *); @@ -395,10 +276,22 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *); extern PyTypeObject _PyUnicodeASCIIIter_Type; +/* --- Interning ---------------------------------------------------------- */ + +// All these are "ref-neutral", like the public PyUnicode_InternInPlace. + +// Explicit interning routines: +PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **); +PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **); +// Left here to help backporting: +PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); +// Only for singletons in the _PyRuntime struct: +extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **); + /* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { - PyThread_type_lock lock; + PyMutex mutex; // next_index value must be preserved when Py_Initialize()/Py_Finalize() // is called multiple times: see _PyUnicode_FromId() implementation. Py_ssize_t next_index; @@ -431,9 +324,12 @@ struct _Py_unicode_state { struct _Py_unicode_ids ids; }; -extern void _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); +// Like PyUnicode_AsUTF8(), but check for embedded null characters. +// Export for '_sqlite3' shared extension. +PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 729d54bbb951e7..0e0ad6518771e9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -13,2084 +13,2973 @@ static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; string = &_Py_ID(CANCELLED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(FINISHED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(False); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(PENDING); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(Py_Repr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(TextIOWrapper); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(True); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(WarningMessage); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_WindowsConsoleIO); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__IOBase_closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abc_tpflags__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abstractmethods__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__add__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aenter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aexit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aiter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__all__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__and__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__anext__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__annotate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__annotations__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__asyncio_running_event_loop__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__await__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bases__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bool__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__build_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__builtins__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bytes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__call__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__cantrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class_getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdictcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__complex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__contains__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__copy__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ctypes_from_outparam__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__del__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delete__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dictoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dir__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__divmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__doc__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__enter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__eq__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__exit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__file__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__firstlineno__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__float__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__floordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__format__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__fspath__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ge__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__get__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattribute__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getinitargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__gt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__hash__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iadd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ifloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ilshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__import__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__index__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init_subclass__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__instancecheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__int__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__invert__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ior__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ipow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__irshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isabstractmethod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__itruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ixor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__len__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__length_hint__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lltrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__loader__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__main__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__match_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__matmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__missing__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__module__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mro_entries__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ne__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__neg__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__new__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__next__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__notes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__or__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__orig_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__origin__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__package__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__parameters__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__path__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pos__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__prepare__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__qualname__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__radd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rdivmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__release_buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__repr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reversed__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rfloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rlshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ror__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__round__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rpow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rrshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rsub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rtruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rxor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set_name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sizeof__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slotnames__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slots__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__spec__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__static_attributes__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__str__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasscheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasshook__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__truediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__trunc__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__type_params__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_is_unpacked_typevartuple__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_prepare_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_unpacked_tuple_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__warningregistry__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weaklistoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weakref__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__xor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abc_impl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abstract_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_active); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_align_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_anonymous_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_argtypes_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_as_parameter_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_asyncio_future_blocking); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_bootstrap); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_check_retval_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_dealloc_warn); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_feature_version); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_field_types); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fields_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_finalizing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_find_and_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_flags_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_get_sourcefile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_handle_fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_initializing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_io); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_is_text_encoding); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_layout_); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_limbo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_lock_unlock_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_needs_com_addref_); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_only_immortal); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_pack_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_restype_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_showwarnmsg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_shutdown); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_slotnames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_strptime_datetime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_swappedbytes_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_type_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_uninitialized_submodules); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_warn_unawaited_coroutine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_xoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(a); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(abs_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aclose); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add_done_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aggregate_class); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(alias); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(allow_code); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(append); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argdefs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(args); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(arguments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argv); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(as_integer_ratio); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(asend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ast); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(athrow); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(attribute); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(authorizer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(autocommit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(b); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(backtick); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(base); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(before); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(big); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(binary_form); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(block); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bound); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bufsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(builtins); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(byteorder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes_per_sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(c); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_return); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cached_datetime_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cafile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_exception_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_soon); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(callback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cancel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(category); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cb_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(certfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(check_same_thread); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(clear); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(close); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closefd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closure); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_argcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_cellvars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_exceptiontable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_firstlineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_freevars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_kwonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_linetable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_names); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_posonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_qualname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_stacksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_varnames); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(code); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(col_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(comment_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(compile_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(context); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(contravariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cookie); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copyreg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(coro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(count); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(covariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cwd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(d); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(data); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(database); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(day); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decoder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(default); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(defaultaction); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(delete); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(depth); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(desired_access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(detect_types); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(deterministic); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(device); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digestmod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(discard); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dispatch_table); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(displayhook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dklen); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(doc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(done); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dont_inherit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(duration); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(e); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eager_start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(effective_ids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(element_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encoding); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(end_col_offset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(endpos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(entrypoint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(env); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(errors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(event); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eventmask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_type); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(excepthook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(existing_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extra_tokens); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(f); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(facility); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(family); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fanout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fdel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fget); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file_actions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fileno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filepath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fillvalue); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(filter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filters); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(final); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(find_class); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fix_imports); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flush); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fold); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(format); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(frequency); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromtimestamp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromutc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(future); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(g); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(genexpr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_debug); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_event_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getattr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(gid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groupindex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groups); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(h); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(handle); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(handle_seq); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(has_location); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hash_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(header); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(headers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hi); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(id); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(hour); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ident); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(identity_hint); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ignore); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(in_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(incoming); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(indexgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inf); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(infer_variance); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(inherit_handle); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inheritable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_bytes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(initial_owner); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(initial_state); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inner_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(input); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_comments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_pis); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(instructions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intersection); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(interval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(is_running); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isatty); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isinstance); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isoformat); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isolation_level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(istext); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(item); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(items); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterations); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(join); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(jump); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keepends); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(key); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keyfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kind); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw2); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(kwdefaults); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(label); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lambda); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_exc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_node); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(latin1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(leaf_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(len); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line_buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(listcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(little); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locale); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(logoption); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(loop); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(manual_reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mapping); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(max_length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxevents); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(maxlen); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxmem); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxvalue); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memLevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memlimit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(message); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metaclass); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(method); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(microsecond); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(milliseconds); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(minute); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module_globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(modules); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(month); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(msg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(mycmp); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mutex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(n); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mycmp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_sequence_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_unnamed_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name_from); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespace_separator); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespaces); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(narg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(nested); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newlines); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_depth); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ns); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nstype); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(null); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(number); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(obj); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(object); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(on_type_read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(onceregistry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(oparg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opcode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opener); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(operation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(optimize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(options); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(order); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(origin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(outgoing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(overlapped); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(owner); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(p); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(password); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(path); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pattern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(peek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_id); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(person); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pi_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(policy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(posix); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(proto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(protocol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(r); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(raw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readall); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readonly); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(real); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(registry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(rel_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(release); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reload); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(repl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(replace); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(resetids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reverse); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reversed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(s); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(scheduler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(second); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(security_attributes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seekable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(selectors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(self); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(send); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sequence); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_hostname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_side); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(session); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setpgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigdef); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigmask); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(shape); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(signed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sizehint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(skip_file_prefixes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sock); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sort); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(sound); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(spam); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stderr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(step); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(steps); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strategy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strftime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(string); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sub_key); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(symmetric_difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tabsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target_is_directory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_frame); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lasti); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tell); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(template); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(term); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(text); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(threading); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(throw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(times); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(top); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trace_callback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trailers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(translate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(true); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(truncate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(twice); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(txt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type_params); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tz); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(tzinfo); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tzname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unraisablehook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(usedforsecurity); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(values); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(version); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(volume); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(wait_all); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(warn_on_full_buffer); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnings); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(wbits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(week); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(weekday); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(who); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(withdata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write_through); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(x); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(year); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(zdict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(empty); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_percent); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dot_locals); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(defaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(format); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(generic_base); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(kwdefaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(type_params); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_null); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_dictcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_genexpr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_lambda); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_listcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_module); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_setcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_string); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_unknown); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(json_decoder); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(list_err); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(utf_8); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_open_br); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_close_br); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); } /* End auto-generated code */ #ifdef __cplusplus diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cf..6ece7134cdeca0 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -8,9 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyTypeObject _PyUnion_Type; +// For extensions created by test_peg_generator +PyAPI_DATA(PyTypeObject) _PyUnion_Type; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject *, PyObject *); + #define _PyUnion_Check(op) Py_IS_TYPE((op), &_PyUnion_Type) -extern PyObject *_Py_union_type_or(PyObject *, PyObject *); #define _PyGenericAlias_Check(op) PyObject_TypeCheck((op), &Py_GenericAliasType) extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h new file mode 100644 index 00000000000000..bd1d27b03b3d00 --- /dev/null +++ b/Include/internal/pycore_uop_ids.h @@ -0,0 +1,292 @@ +// This file is generated by Tools/cases_generator/uop_id_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_CORE_UOP_IDS_H +#define Py_CORE_UOP_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + +#define _EXIT_TRACE 300 +#define _SET_IP 301 +#define _BINARY_OP 302 +#define _BINARY_OP_ADD_FLOAT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_ADD_UNICODE 305 +#define _BINARY_OP_MULTIPLY_FLOAT 306 +#define _BINARY_OP_MULTIPLY_INT 307 +#define _BINARY_OP_SUBTRACT_FLOAT 308 +#define _BINARY_OP_SUBTRACT_INT 309 +#define _BINARY_SLICE BINARY_SLICE +#define _BINARY_SUBSCR 310 +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM +#define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT +#define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT +#define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _BUILD_LIST BUILD_LIST +#define _BUILD_MAP BUILD_MAP +#define _BUILD_SET BUILD_SET +#define _BUILD_SLICE BUILD_SLICE +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _CALL 311 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _CALL_BUILTIN_CLASS 312 +#define _CALL_BUILTIN_FAST 313 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 314 +#define _CALL_BUILTIN_O 315 +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX +#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 +#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW CALL_KW +#define _CALL_LEN CALL_LEN +#define _CALL_METHOD_DESCRIPTOR_FAST 316 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 318 +#define _CALL_METHOD_DESCRIPTOR_O 319 +#define _CALL_NON_PY_GENERAL 320 +#define _CALL_STR_1 321 +#define _CALL_TUPLE_1 322 +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CHECK_ATTR_CLASS 323 +#define _CHECK_ATTR_METHOD_LAZY_DICT 324 +#define _CHECK_ATTR_MODULE 325 +#define _CHECK_ATTR_WITH_HINT 326 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 327 +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _CHECK_FUNCTION 328 +#define _CHECK_FUNCTION_EXACT_ARGS 329 +#define _CHECK_FUNCTION_VERSION 330 +#define _CHECK_IS_NOT_PY_CALLABLE 331 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 332 +#define _CHECK_METHOD_VERSION 333 +#define _CHECK_PEP_523 334 +#define _CHECK_PERIODIC 335 +#define _CHECK_STACK_SPACE 336 +#define _CHECK_STACK_SPACE_OPERAND 337 +#define _CHECK_VALIDITY 338 +#define _CHECK_VALIDITY_AND_SET_IP 339 +#define _COMPARE_OP 340 +#define _COMPARE_OP_FLOAT 341 +#define _COMPARE_OP_INT 342 +#define _COMPARE_OP_STR 343 +#define _CONTAINS_OP 344 +#define _CONTAINS_OP_DICT CONTAINS_OP_DICT +#define _CONTAINS_OP_SET CONTAINS_OP_SET +#define _CONVERT_VALUE CONVERT_VALUE +#define _COPY COPY +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _DELETE_ATTR DELETE_ATTR +#define _DELETE_DEREF DELETE_DEREF +#define _DELETE_FAST DELETE_FAST +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _DELETE_NAME DELETE_NAME +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _DEOPT 345 +#define _DICT_MERGE DICT_MERGE +#define _DICT_UPDATE DICT_UPDATE +#define _DYNAMIC_EXIT 346 +#define _END_SEND END_SEND +#define _ERROR_POP_N 347 +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _EXPAND_METHOD 348 +#define _FATAL_ERROR 349 +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _FOR_ITER 350 +#define _FOR_ITER_GEN_FRAME 351 +#define _FOR_ITER_TIER_TWO 352 +#define _GET_AITER GET_AITER +#define _GET_ANEXT GET_ANEXT +#define _GET_AWAITABLE GET_AWAITABLE +#define _GET_ITER GET_ITER +#define _GET_LEN GET_LEN +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _GUARD_BOTH_FLOAT 353 +#define _GUARD_BOTH_INT 354 +#define _GUARD_BOTH_UNICODE 355 +#define _GUARD_BUILTINS_VERSION 356 +#define _GUARD_DORV_NO_DICT 357 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 358 +#define _GUARD_GLOBALS_VERSION 359 +#define _GUARD_IS_FALSE_POP 360 +#define _GUARD_IS_NONE_POP 361 +#define _GUARD_IS_NOT_NONE_POP 362 +#define _GUARD_IS_TRUE_POP 363 +#define _GUARD_KEYS_VERSION 364 +#define _GUARD_NOS_FLOAT 365 +#define _GUARD_NOS_INT 366 +#define _GUARD_NOT_EXHAUSTED_LIST 367 +#define _GUARD_NOT_EXHAUSTED_RANGE 368 +#define _GUARD_NOT_EXHAUSTED_TUPLE 369 +#define _GUARD_TOS_FLOAT 370 +#define _GUARD_TOS_INT 371 +#define _GUARD_TYPE_VERSION 372 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 373 +#define _INIT_CALL_PY_EXACT_ARGS 374 +#define _INIT_CALL_PY_EXACT_ARGS_0 375 +#define _INIT_CALL_PY_EXACT_ARGS_1 376 +#define _INIT_CALL_PY_EXACT_ARGS_2 377 +#define _INIT_CALL_PY_EXACT_ARGS_3 378 +#define _INIT_CALL_PY_EXACT_ARGS_4 379 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE +#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE +#define _INTERNAL_INCREMENT_OPT_COUNTER 380 +#define _IS_NONE 381 +#define _IS_OP IS_OP +#define _ITER_CHECK_LIST 382 +#define _ITER_CHECK_RANGE 383 +#define _ITER_CHECK_TUPLE 384 +#define _ITER_JUMP_LIST 385 +#define _ITER_JUMP_RANGE 386 +#define _ITER_JUMP_TUPLE 387 +#define _ITER_NEXT_LIST 388 +#define _ITER_NEXT_RANGE 389 +#define _ITER_NEXT_TUPLE 390 +#define _JUMP_TO_TOP 391 +#define _LIST_APPEND LIST_APPEND +#define _LIST_EXTEND LIST_EXTEND +#define _LOAD_ATTR 392 +#define _LOAD_ATTR_CLASS 393 +#define _LOAD_ATTR_CLASS_0 394 +#define _LOAD_ATTR_CLASS_1 395 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _LOAD_ATTR_INSTANCE_VALUE 396 +#define _LOAD_ATTR_INSTANCE_VALUE_0 397 +#define _LOAD_ATTR_INSTANCE_VALUE_1 398 +#define _LOAD_ATTR_METHOD_LAZY_DICT 399 +#define _LOAD_ATTR_METHOD_NO_DICT 400 +#define _LOAD_ATTR_METHOD_WITH_VALUES 401 +#define _LOAD_ATTR_MODULE 402 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 403 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 404 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_SLOT 405 +#define _LOAD_ATTR_SLOT_0 406 +#define _LOAD_ATTR_SLOT_1 407 +#define _LOAD_ATTR_WITH_HINT 408 +#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT +#define _LOAD_CONST LOAD_CONST +#define _LOAD_CONST_INLINE 409 +#define _LOAD_CONST_INLINE_BORROW 410 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 411 +#define _LOAD_CONST_INLINE_WITH_NULL 412 +#define _LOAD_DEREF LOAD_DEREF +#define _LOAD_FAST 413 +#define _LOAD_FAST_0 414 +#define _LOAD_FAST_1 415 +#define _LOAD_FAST_2 416 +#define _LOAD_FAST_3 417 +#define _LOAD_FAST_4 418 +#define _LOAD_FAST_5 419 +#define _LOAD_FAST_6 420 +#define _LOAD_FAST_7 421 +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF +#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_GLOBAL 422 +#define _LOAD_GLOBAL_BUILTINS 423 +#define _LOAD_GLOBAL_MODULE 424 +#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_NAME LOAD_NAME +#define _LOAD_SPECIAL LOAD_SPECIAL +#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR +#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _MAKE_CELL MAKE_CELL +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAP_ADD MAP_ADD +#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_KEYS MATCH_KEYS +#define _MATCH_MAPPING MATCH_MAPPING +#define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _NOP NOP +#define _POP_EXCEPT POP_EXCEPT +#define _POP_JUMP_IF_FALSE 425 +#define _POP_JUMP_IF_TRUE 426 +#define _POP_TOP POP_TOP +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 427 +#define _PUSH_EXC_INFO PUSH_EXC_INFO +#define _PUSH_FRAME 428 +#define _PUSH_NULL PUSH_NULL +#define _PY_FRAME_GENERAL 429 +#define _REPLACE_WITH_TRUE 430 +#define _RESUME_CHECK RESUME_CHECK +#define _RETURN_GENERATOR RETURN_GENERATOR +#define _RETURN_VALUE RETURN_VALUE +#define _SAVE_RETURN_OFFSET 431 +#define _SEND 432 +#define _SEND_GEN SEND_GEN +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _SET_ADD SET_ADD +#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE +#define _SET_UPDATE SET_UPDATE +#define _START_EXECUTOR 433 +#define _STORE_ATTR 434 +#define _STORE_ATTR_INSTANCE_VALUE 435 +#define _STORE_ATTR_SLOT 436 +#define _STORE_ATTR_WITH_HINT 437 +#define _STORE_DEREF STORE_DEREF +#define _STORE_FAST 438 +#define _STORE_FAST_0 439 +#define _STORE_FAST_1 440 +#define _STORE_FAST_2 441 +#define _STORE_FAST_3 442 +#define _STORE_FAST_4 443 +#define _STORE_FAST_5 444 +#define _STORE_FAST_6 445 +#define _STORE_FAST_7 446 +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_GLOBAL STORE_GLOBAL +#define _STORE_NAME STORE_NAME +#define _STORE_SLICE STORE_SLICE +#define _STORE_SUBSCR 447 +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT +#define _SWAP SWAP +#define _TIER2_RESUME_CHECK 448 +#define _TO_BOOL 449 +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _UNARY_INVERT UNARY_INVERT +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _UNPACK_EX UNPACK_EX +#define _UNPACK_SEQUENCE 450 +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _YIELD_VALUE YIELD_VALUE +#define MAX_UOP_ID 450 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_UOP_IDS_H */ diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h new file mode 100644 index 00000000000000..2a2d6e923b7617 --- /dev/null +++ b/Include/internal/pycore_uop_metadata.h @@ -0,0 +1,1010 @@ +// This file is generated by Tools/cases_generator/uop_metadata_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_CORE_UOP_METADATA_H +#define Py_CORE_UOP_METADATA_H +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include "pycore_uop_ids.h" +extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; + +extern int _PyUop_num_popped(int opcode, int oparg); + +#ifdef NEED_OPCODE_METADATA +const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { + [_NOP] = HAS_PURE_FLAG, + [_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_POP_TOP] = HAS_PURE_FLAG, + [_PUSH_NULL] = HAS_PURE_FLAG, + [_END_SEND] = HAS_PURE_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NOT] = HAS_PURE_FLAG, + [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, + [_REPLACE_WITH_TRUE] = 0, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, + [_GUARD_NOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_INT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = 0, + [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_POP_EXCEPT] = HAS_ESCAPES_FLAG, + [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, + [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, + [_COPY_FREE_VARS] = HAS_ARG_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, + [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, + [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_CLASS_0] = 0, + [_LOAD_ATTR_CLASS_1] = 0, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG, + [_STORE_ATTR_INSTANCE_VALUE] = 0, + [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_ATTR_SLOT] = 0, + [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG, + [_IS_OP] = HAS_ARG_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IS_NONE] = 0, + [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_MAPPING] = 0, + [_MATCH_SEQUENCE] = 0, + [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, + [_ITER_NEXT_LIST] = 0, + [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, + [_ITER_NEXT_TUPLE] = 0, + [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_EXC_INFO] = 0, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, + [_CHECK_PEP_523] = HAS_DEOPT_FLAG, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_PUSH_FRAME] = 0, + [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, + [_JUMP_TO_TOP] = 0, + [_SET_IP] = 0, + [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, + [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, + [_EXIT_TRACE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, + [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, + [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, + [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, + [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, + [_FATAL_ERROR] = 0, + [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_DEOPT] = 0, + [_ERROR_POP_N] = HAS_ARG_FLAG, + [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, +}; + +const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = 8, + [_STORE_FAST] = 8, + [_INIT_CALL_PY_EXACT_ARGS] = 5, +}; + +const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { + [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_BINARY_SUBSCR_DICT] = "_BINARY_SUBSCR_DICT", + [_BINARY_SUBSCR_LIST_INT] = "_BINARY_SUBSCR_LIST_INT", + [_BINARY_SUBSCR_STR_INT] = "_BINARY_SUBSCR_STR_INT", + [_BINARY_SUBSCR_TUPLE_INT] = "_BINARY_SUBSCR_TUPLE_INT", + [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", + [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_LEN] = "_CALL_LEN", + [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_FUNCTION] = "_CHECK_FUNCTION", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", + [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_COPY] = "_COPY", + [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DEOPT] = "_DEOPT", + [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_END_SEND] = "_END_SEND", + [_ERROR_POP_N] = "_ERROR_POP_N", + [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_FATAL_ERROR] = "_FATAL_ERROR", + [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_GET_AITER] = "_GET_AITER", + [_GET_ANEXT] = "_GET_ANEXT", + [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_ITER] = "_GET_ITER", + [_GET_LEN] = "_GET_LEN", + [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", + [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", + [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", + [_IS_NONE] = "_IS_NONE", + [_IS_OP] = "_IS_OP", + [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", + [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_EXTEND] = "_LIST_EXTEND", + [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", + [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", + [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", + [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", + [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", + [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", + [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", + [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", + [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", + [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAP_ADD] = "_MAP_ADD", + [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_NOP] = "_NOP", + [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_TOP] = "_POP_TOP", + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_NULL] = "_PUSH_NULL", + [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_RESUME_CHECK] = "_RESUME_CHECK", + [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_VALUE] = "_RETURN_VALUE", + [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SET_ADD] = "_SET_ADD", + [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_IP] = "_SET_IP", + [_SET_UPDATE] = "_SET_UPDATE", + [_START_EXECUTOR] = "_START_EXECUTOR", + [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_DEREF] = "_STORE_DEREF", + [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_7] = "_STORE_FAST_7", + [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", + [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_NAME] = "_STORE_NAME", + [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_SWAP] = "_SWAP", + [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NOT] = "_UNARY_NOT", + [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", +}; +int _PyUop_num_popped(int opcode, int oparg) +{ + switch(opcode) { + case _NOP: + return 0; + case _RESUME_CHECK: + return 0; + case _LOAD_FAST_CHECK: + return 0; + case _LOAD_FAST_0: + return 0; + case _LOAD_FAST_1: + return 0; + case _LOAD_FAST_2: + return 0; + case _LOAD_FAST_3: + return 0; + case _LOAD_FAST_4: + return 0; + case _LOAD_FAST_5: + return 0; + case _LOAD_FAST_6: + return 0; + case _LOAD_FAST_7: + return 0; + case _LOAD_FAST: + return 0; + case _LOAD_FAST_AND_CLEAR: + return 0; + case _LOAD_FAST_LOAD_FAST: + return 0; + case _LOAD_CONST: + return 0; + case _STORE_FAST_0: + return 1; + case _STORE_FAST_1: + return 1; + case _STORE_FAST_2: + return 1; + case _STORE_FAST_3: + return 1; + case _STORE_FAST_4: + return 1; + case _STORE_FAST_5: + return 1; + case _STORE_FAST_6: + return 1; + case _STORE_FAST_7: + return 1; + case _STORE_FAST: + return 1; + case _STORE_FAST_LOAD_FAST: + return 1; + case _STORE_FAST_STORE_FAST: + return 2; + case _POP_TOP: + return 1; + case _PUSH_NULL: + return 0; + case _END_SEND: + return 2; + case _UNARY_NEGATIVE: + return 1; + case _UNARY_NOT: + return 1; + case _TO_BOOL: + return 1; + case _TO_BOOL_BOOL: + return 1; + case _TO_BOOL_INT: + return 1; + case _TO_BOOL_LIST: + return 1; + case _TO_BOOL_NONE: + return 1; + case _TO_BOOL_STR: + return 1; + case _REPLACE_WITH_TRUE: + return 1; + case _UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _GUARD_NOS_INT: + return 2; + case _GUARD_TOS_INT: + return 1; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _GUARD_NOS_FLOAT: + return 2; + case _GUARD_TOS_FLOAT: + return 1; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_SUBSCR: + return 2; + case _BINARY_SLICE: + return 3; + case _STORE_SLICE: + return 4; + case _BINARY_SUBSCR_LIST_INT: + return 2; + case _BINARY_SUBSCR_STR_INT: + return 2; + case _BINARY_SUBSCR_TUPLE_INT: + return 2; + case _BINARY_SUBSCR_DICT: + return 2; + case _LIST_APPEND: + return 2 + (oparg-1); + case _SET_ADD: + return 2 + (oparg-1); + case _STORE_SUBSCR: + return 3; + case _STORE_SUBSCR_LIST_INT: + return 3; + case _STORE_SUBSCR_DICT: + return 3; + case _DELETE_SUBSCR: + return 2; + case _CALL_INTRINSIC_1: + return 1; + case _CALL_INTRINSIC_2: + return 2; + case _RETURN_VALUE: + return 1; + case _GET_AITER: + return 1; + case _GET_ANEXT: + return 1; + case _GET_AWAITABLE: + return 1; + case _YIELD_VALUE: + return 1; + case _POP_EXCEPT: + return 1; + case _LOAD_COMMON_CONSTANT: + return 0; + case _LOAD_BUILD_CLASS: + return 0; + case _STORE_NAME: + return 1; + case _DELETE_NAME: + return 0; + case _UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_TUPLE: + return 1; + case _UNPACK_SEQUENCE_LIST: + return 1; + case _UNPACK_EX: + return 1; + case _STORE_ATTR: + return 2; + case _DELETE_ATTR: + return 1; + case _STORE_GLOBAL: + return 1; + case _DELETE_GLOBAL: + return 0; + case _LOAD_LOCALS: + return 0; + case _LOAD_GLOBAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _DELETE_FAST: + return 0; + case _MAKE_CELL: + return 0; + case _DELETE_DEREF: + return 0; + case _LOAD_FROM_DICT_OR_DEREF: + return 1; + case _LOAD_DEREF: + return 0; + case _STORE_DEREF: + return 1; + case _COPY_FREE_VARS: + return 0; + case _BUILD_STRING: + return oparg; + case _BUILD_TUPLE: + return oparg; + case _BUILD_LIST: + return oparg; + case _LIST_EXTEND: + return 2 + (oparg-1); + case _SET_UPDATE: + return 2 + (oparg-1); + case _BUILD_MAP: + return oparg*2; + case _SETUP_ANNOTATIONS: + return 0; + case _BUILD_CONST_KEY_MAP: + return 1 + oparg; + case _DICT_UPDATE: + return 2 + (oparg - 1); + case _DICT_MERGE: + return 5 + (oparg - 1); + case _MAP_ADD: + return 3 + (oparg - 1); + case _LOAD_SUPER_ATTR_ATTR: + return 3; + case _LOAD_SUPER_ATTR_METHOD: + return 3; + case _LOAD_ATTR: + return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_0: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_1: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_SLOT_0: + return 1; + case _LOAD_ATTR_SLOT_1: + return 1; + case _LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS_0: + return 1; + case _LOAD_ATTR_CLASS_1: + return 1; + case _LOAD_ATTR_CLASS: + return 1; + case _GUARD_DORV_NO_DICT: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case _STORE_ATTR_WITH_HINT: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case _COMPARE_OP: + return 2; + case _COMPARE_OP_FLOAT: + return 2; + case _COMPARE_OP_INT: + return 2; + case _COMPARE_OP_STR: + return 2; + case _IS_OP: + return 2; + case _CONTAINS_OP: + return 2; + case _CONTAINS_OP_SET: + return 2; + case _CONTAINS_OP_DICT: + return 2; + case _CHECK_EG_MATCH: + return 2; + case _CHECK_EXC_MATCH: + return 2; + case _IS_NONE: + return 1; + case _GET_LEN: + return 1; + case _MATCH_CLASS: + return 3; + case _MATCH_MAPPING: + return 1; + case _MATCH_SEQUENCE: + return 1; + case _MATCH_KEYS: + return 2; + case _GET_ITER: + return 1; + case _GET_YIELD_FROM_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 1; + case _ITER_CHECK_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; + case _ITER_CHECK_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; + case _FOR_ITER_GEN_FRAME: + return 1; + case _LOAD_SPECIAL: + return 1; + case _WITH_EXCEPT_START: + return 5; + case _PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case _CHECK_PERIODIC: + return 0; + case _PY_FRAME_GENERAL: + return 2 + oparg; + case _CHECK_FUNCTION_VERSION: + return 2 + oparg; + case _CHECK_METHOD_VERSION: + return 2 + oparg; + case _EXPAND_METHOD: + return 2 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE: + return 2 + oparg; + case _CALL_NON_PY_GENERAL: + return 2 + oparg; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return 2 + oparg; + case _CHECK_STACK_SPACE: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_0: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_1: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_2: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_3: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_4: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case _PUSH_FRAME: + return 1; + case _CALL_TYPE_1: + return 3; + case _CALL_STR_1: + return 3; + case _CALL_TUPLE_1: + return 3; + case _EXIT_INIT_CHECK: + return 1; + case _CALL_BUILTIN_CLASS: + return 2 + oparg; + case _CALL_BUILTIN_O: + return 2 + oparg; + case _CALL_BUILTIN_FAST: + return 2 + oparg; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_LEN: + return 2 + oparg; + case _CALL_ISINSTANCE: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case _MAKE_FUNCTION: + return 1; + case _SET_FUNCTION_ATTRIBUTE: + return 2; + case _RETURN_GENERATOR: + return 0; + case _BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case _CONVERT_VALUE: + return 1; + case _FORMAT_SIMPLE: + return 1; + case _FORMAT_WITH_SPEC: + return 2; + case _COPY: + return 1 + (oparg-1); + case _BINARY_OP: + return 2; + case _SWAP: + return 2 + (oparg-2); + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _CHECK_STACK_SPACE_OPERAND: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _CHECK_VALIDITY: + return 0; + case _LOAD_CONST_INLINE: + return 0; + case _LOAD_CONST_INLINE_BORROW: + return 0; + case _POP_TOP_LOAD_CONST_INLINE_BORROW: + return 1; + case _LOAD_CONST_INLINE_WITH_NULL: + return 0; + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: + return 0; + case _CHECK_FUNCTION: + return 0; + case _INTERNAL_INCREMENT_OPT_COUNTER: + return 1; + case _DYNAMIC_EXIT: + return 0; + case _START_EXECUTOR: + return 0; + case _FATAL_ERROR: + return 0; + case _CHECK_VALIDITY_AND_SET_IP: + return 0; + case _DEOPT: + return 0; + case _ERROR_POP_N: + return oparg; + case _TIER2_RESUME_CHECK: + return 0; + default: + return -1; + } +} + +#endif // NEED_OPCODE_METADATA + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_UOP_METADATA_H */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h deleted file mode 100644 index d8a7d978f1304e..00000000000000 --- a/Include/internal/pycore_uops.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef Py_INTERNAL_UOPS_H -#define Py_INTERNAL_UOPS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "pycore_frame.h" // _PyInterpreterFrame - -#define _Py_UOP_MAX_TRACE_LENGTH 128 - -typedef struct { - uint32_t opcode; - uint32_t oparg; - uint64_t operand; // A cache entry -} _PyUOpInstruction; - -typedef struct { - _PyExecutorObject base; - _PyUOpInstruction trace[1]; -} _PyUOpExecutorObject; - -_PyInterpreterFrame *_PyUopExecute( - _PyExecutorObject *executor, - _PyInterpreterFrame *frame, - PyObject **stack_pointer); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_UOPS_H */ diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index 9785d7cc467de2..f9f6559312f4ef 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -14,6 +14,7 @@ struct _warnings_runtime_state { PyObject *filters; /* List */ PyObject *once_registry; /* Dict */ PyObject *default_action; /* String */ + PyMutex mutex; long filters_version; }; diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index 51b2bb6b11ede9..94aadb2c1547dd 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -8,50 +8,112 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { - assert(PyWeakref_Check(ref_obj)); - PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_lock.h" +#include "pycore_object.h" // _Py_REF_IS_MERGED() +#include "pycore_pyatomic_ft_wrappers.h" - if (obj == Py_None) { - // clear_weakref() was called - return NULL; - } +#ifdef Py_GIL_DISABLED + +#define WEAKREF_LIST_LOCK(obj) \ + _PyInterpreterState_GET() \ + ->weakref_locks[((uintptr_t)obj) % NUM_WEAKREF_LIST_LOCKS] + +// Lock using the referenced object +#define LOCK_WEAKREFS(obj) \ + PyMutex_LockFlags(&WEAKREF_LIST_LOCK(obj), _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&WEAKREF_LIST_LOCK(obj)) + +// Lock using a weakref +#define LOCK_WEAKREFS_FOR_WR(wr) \ + PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) +#else + +#define LOCK_WEAKREFS(obj) +#define UNLOCK_WEAKREFS(obj) + +#define LOCK_WEAKREFS_FOR_WR(wr) +#define UNLOCK_WEAKREFS_FOR_WR(wr) + +#endif + +static inline int _is_dead(PyObject *obj) +{ // Explanation for the Py_REFCNT() check: when a weakref's target is part // of a long chain of deallocations which triggers the trashcan mechanism, // clearing the weakrefs can be delayed long after the target's refcount // has dropped to zero. In the meantime, code accessing the weakref will // be able to "see" the target object even though it is supposed to be // unreachable. See issue gh-60806. - Py_ssize_t refcnt = Py_REFCNT(obj); - if (refcnt == 0) { +#if defined(Py_GIL_DISABLED) + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + return shared == _Py_REF_SHARED(0, _Py_REF_MERGED); +#else + return (Py_REFCNT(obj) == 0); +#endif +} + +static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) +{ + assert(PyWeakref_Check(ref_obj)); + PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); + + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); + if (obj == Py_None) { + // clear_weakref() was called return NULL; } - assert(refcnt > 0); - return Py_NewRef(obj); + LOCK_WEAKREFS(obj); +#ifdef Py_GIL_DISABLED + if (ref->wr_object == Py_None) { + // clear_weakref() was called + UNLOCK_WEAKREFS(obj); + return NULL; + } +#endif + if (_Py_TryIncref(obj)) { + UNLOCK_WEAKREFS(obj); + return obj; + } + UNLOCK_WEAKREFS(obj); + return NULL; } -static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { +static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) +{ assert(PyWeakref_Check(ref_obj)); + int ret = 0; PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called - return 1; + ret = 1; } - - // See _PyWeakref_GET_REF() for the rationale of this test - return (Py_REFCNT(obj) == 0); + else { + LOCK_WEAKREFS(obj); + // See _PyWeakref_GET_REF() for the rationale of this test +#ifdef Py_GIL_DISABLED + ret = (ref->wr_object == Py_None) || _is_dead(obj); +#else + ret = _is_dead(obj); +#endif + UNLOCK_WEAKREFS(obj); + } + return ret; } -extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); +extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); -extern void _PyWeakref_ClearRef(PyWeakReference *self); +// Clear all the weak references to obj but leave their callbacks uncalled and +// intact. +extern void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj); + +PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_WEAKREF_H */ - diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index 8432632f339e92..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/listobject.h b/Include/listobject.h index 6b7041ba0b05d5..e1e059b0ba7466 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,6 +29,9 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); +#endif PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/lock.h b/Include/lock.h new file mode 100644 index 00000000000000..782b9dbc70d056 --- /dev/null +++ b/Include/lock.h @@ -0,0 +1,16 @@ +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_LOCK_H +# include "cpython/lock.h" +# undef Py_CPYTHON_LOCK_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_LOCK_H */ diff --git a/Include/longobject.h b/Include/longobject.h index 7393254cd24a9b..19104cd9d1bef9 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -7,7 +7,7 @@ extern "C" { /* Long (arbitrary precision) integer object interface */ -PyAPI_DATA(PyTypeObject) PyLong_Type; +// PyLong_Type is declared by object.h #define PyLong_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) @@ -40,7 +40,24 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); #if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT #define _Py_PARSE_PID "i" #define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define PyLong_AsPid PyLong_AsInt +# elif SIZEOF_INT == SIZEOF_LONG +# define PyLong_AsPid PyLong_AsLong +# else +static inline int +PyLong_AsPid(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow || result > INT_MAX || result < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} +# endif #elif SIZEOF_PID_T == SIZEOF_LONG #define _Py_PARSE_PID "l" #define PyLong_FromPid PyLong_FromLong diff --git a/Include/methodobject.h b/Include/methodobject.h index 2381e8482b82a8..39272815b127f4 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -17,15 +17,22 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Type; #define PyCFunction_Check(op) PyObject_TypeCheck((op), &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); +typedef PyObject *(*PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *, - PyObject *const *, Py_ssize_t, - PyObject *); +typedef PyObject *(*PyCFunctionFastWithKeywords) (PyObject *, + PyObject *const *, Py_ssize_t, + PyObject *); typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +// For backwards compatibility. `METH_FASTCALL` was added to the stable API in +// 3.10 alongside `_PyCFunctionFastWithKeywords` and `_PyCFunctionFast`. +// Note that the underscore-prefixed names were documented in public docs; +// people may be using them. +typedef PyCFunctionFast _PyCFunctionFast; +typedef PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords; + // Cast an function to the PyCFunction type to use it with PyMethodDef. // // This macro can be used to prevent compiler warnings if the first parameter diff --git a/Include/modsupport.h b/Include/modsupport.h index 6efe9dfaa9089e..af995f567b004c 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -9,10 +9,10 @@ extern "C" { PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, ...); + const char *, PY_CXX_CONST char * const *, ...); PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list); PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, va_list); + const char *, PY_CXX_CONST char * const *, va_list); PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *); PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...); diff --git a/Include/moduleobject.h b/Include/moduleobject.h index ea08145381cee6..2a17c891dda811 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -53,7 +53,7 @@ typedef struct PyModuleDef_Base { /* A copy of the module's __dict__ after the first time it was loaded. This is only set/used for legacy modules that do not support multiple initializations. - It is set by _PyImport_FixupExtensionObject(). */ + It is set by fix_up_extension() in import.c. */ PyObject* m_copy; } PyModuleDef_Base; @@ -73,18 +73,36 @@ struct PyModuleDef_Slot { #define Py_mod_create 1 #define Py_mod_exec 2 -#define Py_mod_multiple_interpreters 3 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 +# define Py_mod_multiple_interpreters 3 +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_mod_gil 4 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 3 +#define _Py_mod_LAST_SLOT 4 #endif +#endif /* New in 3.5 */ + /* for Py_mod_multiple_interpreters: */ -#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) -#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) -#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 +# define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) +# define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) +# define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) +#endif -#endif /* New in 3.5 */ +/* for Py_mod_gil: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_MOD_GIL_USED ((void *)0) +# define Py_MOD_GIL_NOT_USED ((void *)1) +#endif + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +#endif struct PyModuleDef { PyModuleDef_Base m_base; diff --git a/Include/monitoring.h b/Include/monitoring.h new file mode 100644 index 00000000000000..985f7f230e44e3 --- /dev/null +++ b/Include/monitoring.h @@ -0,0 +1,18 @@ +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifdef __cplusplus +extern "C" { +#endif + +// There is currently no limited API for monitoring + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MONITORING_H +# include "cpython/monitoring.h" +# undef Py_CPYTHON_MONITORING_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index 9058558e3cd4d9..abfdb6ce24df21 100644 --- a/Include/object.h +++ b/Include/object.h @@ -59,68 +59,32 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -/* -Immortalization: - -The following indicates the immortalization strategy depending on the amount -of available bits in the reference count field. All strategies are backwards -compatible but the specific reference count value or immortalization check -might change depending on the specializations for the underlying system. - -Proper deallocation of immortal instances requires distinguishing between -statically allocated immortal instances vs those promoted by the runtime to be -immortal. The latter should be the only instances that require -cleanup during runtime finalization. -*/ - -#if SIZEOF_VOID_P > 4 -/* -In 64+ bit systems, an object will be marked as immortal by setting all of the -lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF - -Using the lower 32 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases will use saturated arithmetic, taking advantage of -having all the lower 32 bits set, which will avoid the reference count to go -beyond the refcount limit. Immortality checks for reference count decreases will -be done by checking the bit sign flag in the lower 32 bits. -*/ -#define _Py_IMMORTAL_REFCNT UINT_MAX +// Kept for backward compatibility. It was needed by Py_TRACE_REFS build. +#define _PyObject_EXTRA_INIT +/* Make all uses of PyObject_HEAD_INIT immortal. + * + * Statically allocated objects might be shared between + * interpreters, so must be marked as immortal. + */ +#if defined(Py_GIL_DISABLED) +#define PyObject_HEAD_INIT(type) \ + { \ + 0, \ + 0, \ + { 0 }, \ + 0, \ + _Py_IMMORTAL_REFCNT_LOCAL, \ + 0, \ + (type), \ + }, #else -/* -In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF - -Using the lower 30 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases and decreases will first go through an immortality -check by comparing the reference count field to the immortality reference count. -*/ -#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) -#endif - -// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the -// C-API expectation that the refcnt will be set to 1. -#ifdef Py_BUILD_CORE #define PyObject_HEAD_INIT(type) \ { \ { _Py_IMMORTAL_REFCNT }, \ (type) \ }, -#else -#define PyObject_HEAD_INIT(type) \ - { \ - { 1 }, \ - (type) \ - }, -#endif /* Py_BUILD_CORE */ +#endif #define PyVarObject_HEAD_INIT(type, size) \ { \ @@ -142,6 +106,7 @@ check by comparing the reference count field to the immortality reference count. * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ +#ifndef Py_GIL_DISABLED struct _object { #if (defined(__GNUC__) || defined(__clang__)) \ && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) @@ -166,6 +131,25 @@ struct _object { PyTypeObject *ob_type; }; +#else +// Objects that are not owned by any thread use a thread id (tid) of zero. +// This includes both immortal objects and objects whose reference count +// fields have been merged. +#define _Py_UNOWNED_TID 0 + +struct _object { + // ob_tid stores the thread id (or zero). It is also used by the GC and the + // trashcan mechanism as a linked list pointer and by the GC to store the + // computed "gc_refs" refcount. + uintptr_t ob_tid; + uint16_t _padding; + PyMutex ob_mutex; // per-object lock + uint8_t ob_gc_bits; // gc-related state + uint32_t ob_ref_local; // local reference count + Py_ssize_t ob_ref_shared; // shared (atomic) reference count + PyTypeObject *ob_type; +}; +#endif /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) @@ -183,21 +167,95 @@ typedef struct { PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) +#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); -static inline Py_ssize_t Py_REFCNT(PyObject *ob) { - return ob->ob_refcnt; +static inline uintptr_t +_Py_ThreadId(void) +{ + uintptr_t tid; +#if defined(_MSC_VER) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(_MSC_VER) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(_MSC_VER) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__i386__) + __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS +#elif defined(__MACH__) && defined(__x86_64__) + __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS +#elif defined(__x86_64__) + __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS +#elif defined(__arm__) + __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); +#elif defined(__aarch64__) && defined(__APPLE__) + __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); +#elif defined(__aarch64__) + __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif +#else + // Fallback to a portable implementation if we do not have a faster + // platform-specific implementation. + tid = _Py_GetThreadLocal_Addr(); +#endif + return tid; +} + +static inline Py_ALWAYS_INLINE int +_Py_IsOwnedByCurrentThread(PyObject *ob) +{ +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); +#else + return ob->ob_tid == _Py_ThreadId(); +#endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) #endif +// Py_TYPE() implementation for the stable ABI +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); -// bpo-39573: The Py_SET_TYPE() function must be used to set an object type. -static inline PyTypeObject* Py_TYPE(PyObject *ob) { - return ob->ob_type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_TYPE() as a function call + // on limited C API version 3.14 and newer. +#else + static inline PyTypeObject* _Py_TYPE(PyObject *ob) + { + return ob->ob_type; + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) + #else + # define Py_TYPE(ob) _Py_TYPE(ob) + #endif #endif PyAPI_DATA(PyTypeObject) PyLong_Type; @@ -205,25 +263,14 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { - assert(ob->ob_type != &PyLong_Type); - assert(ob->ob_type != &PyBool_Type); - PyVarObject *var_ob = _PyVarObject_CAST(ob); - return var_ob->ob_size; + assert(Py_TYPE(ob) != &PyLong_Type); + assert(Py_TYPE(ob) != &PyBool_Type); + return _PyVarObject_CAST(ob)->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif -static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) -{ -#if SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; -#else - return op->ob_refcnt == _Py_IMMORTAL_REFCNT; -#endif -} -#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) - static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } @@ -232,21 +279,6 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif -static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { - // This immortal check is for code that is unaware of immortal objects. - // The runtime tracks these objects and we should avoid as much - // as possible having extensions inadvertently change the refcnt - // of an immortalized object. - if (_Py_IsImmortal(ob)) { - return; - } - ob->ob_refcnt = refcnt; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) -#endif - - static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -255,9 +287,13 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { #endif static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { - assert(ob->ob_base.ob_type != &PyLong_Type); - assert(ob->ob_base.ob_type != &PyBool_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); +#else ob->ob_size = size; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) @@ -346,6 +382,10 @@ PyAPI_FUNC(void *) PyType_GetModuleState(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *type); +PyAPI_FUNC(PyObject *) PyType_GetModuleName(PyTypeObject *type); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); @@ -450,13 +490,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* The values array is placed inline directly after the rest of + * the object. Implies Py_TPFLAGS_HAVE_GC. + */ +#define Py_TPFLAGS_INLINE_VALUES (1 << 2) + /* Placement of weakref pointers are managed by the VM, not by the type. * The VM will automatically set tp_weaklistoffset. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. + * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) @@ -509,7 +554,7 @@ given type object has a specified feature. /* Objects behave like an unbound method */ #define Py_TPFLAGS_METHOD_DESCRIPTOR (1UL << 17) -/* Object has up-to-date type attribute cache */ +/* Unused. Legacy flag */ #define Py_TPFLAGS_VALID_VERSION_TAG (1UL << 19) /* Type is abstract and cannot be instantiated */ @@ -554,264 +599,20 @@ given type object has a specified feature. #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) -/* -The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement -reference counts. Py_DECREF calls the object's deallocator function when -the refcount falls to 0; for -objects that don't contain references to other objects or heap memory -this can be the standard function free(). Both macros can be used -wherever a void expression is allowed. The argument must not be a -NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. -The macro _Py_NewReference(op) initialize reference counts to 1, and -in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional -bookkeeping appropriate to the special build. - -We assume that the reference count field can never overflow; this can -be proven when the size of the field is the same as the pointer size, so -we ignore the possibility. Provided a C int is at least 32 bits (which -is implicitly assumed in many parts of this code), that's enough for -about 2**31 references to an object. - -XXX The following became out of date in Python 2.2, but I'm not sure -XXX what the full truth is now. Certainly, heap-allocated type objects -XXX can and should be deallocated. -Type objects should never be deallocated; the type pointer in an object -is not considered to be a reference to the type object, to save -complications in the deallocation function. (This is actually a -decision that's up to the implementer of each new type so if you want, -you can count such references to the type object.) -*/ - -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) -PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, - PyObject *op); -PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); -PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); -#endif // Py_REF_DEBUG && !Py_LIMITED_API - -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); - -/* -These are provided as conveniences to Python runtime embedders, so that -they can have object code that is not dependent on Python compilation flags. -*/ -PyAPI_FUNC(void) Py_IncRef(PyObject *); -PyAPI_FUNC(void) Py_DecRef(PyObject *); - -// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. -// Private functions used by Py_INCREF() and Py_DECREF(). -PyAPI_FUNC(void) _Py_IncRef(PyObject *); -PyAPI_FUNC(void) _Py_DecRef(PyObject *); - -static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) -{ -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) - // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. - // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_IncRef(op); -# else - Py_IncRef(op); -# endif -#else - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. -#if SIZEOF_VOID_P > 4 - // Portable saturated add, branching on the carry flag and set low bits - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; - PY_UINT32_T new_refcnt = cur_refcnt + 1; - if (new_refcnt == 0) { - return; - } - op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; -#else - // Explicitly check immortality against the immortal value - if (_Py_IsImmortal(op)) { - return; - } - op->ob_refcnt++; -#endif - _Py_INCREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_INCREF_IncRefTotal(); -#endif -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) -#endif - -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) -// Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. -// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. -static inline void Py_DECREF(PyObject *op) { -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_DecRef(op); -# else - Py_DecRef(op); -# endif -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - if (op->ob_refcnt <= 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#else -static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) -{ - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) -#endif - +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 -/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear - * and tp_dealloc implementations. - * - * Note that "the obvious" code can be deadly: - * - * Py_XDECREF(op); - * op = NULL; - * - * Typically, `op` is something like self->containee, and `self` is done - * using its `containee` member. In the code sequence above, suppose - * `containee` is non-NULL with a refcount of 1. Its refcount falls to - * 0 on the first line, which can trigger an arbitrary amount of code, - * possibly including finalizers (like __del__ methods or weakref callbacks) - * coded in Python, which in turn can release the GIL and allow other threads - * to run, etc. Such code may even invoke methods of `self` again, or cause - * cyclic gc to trigger, but-- oops! --self->containee still points to the - * object being torn down, and it may be in an insane state while being torn - * down. This has in fact been a rich historic source of miserable (rare & - * hard-to-diagnose) segfaulting (and other) bugs. - * - * The safe way is: - * - * Py_CLEAR(op); - * - * That arranges to set `op` to NULL _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. - * - * There are cases where it's safe to use the naive code, but they're brittle. - * For example, if `op` points to a Python integer, you know that destroying - * one of those can't cause problems -- but in part that relies on that - * Python integers aren't currently weakly referencable. Best practice is - * to use Py_CLEAR() even if you can't think of a reason for why you need to. - * - * gh-98724: Use a temporary variable to only evaluate the macro argument once, - * to avoid the duplication of side effects if the argument has side effects. - * - * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, - * the code can be miscompiled with strict aliasing because of type punning. - * With strict aliasing, a compiler considers that two pointers of different - * types cannot read or write the same memory which enables optimization - * opportunities. - * - * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, - * and so avoid type punning. Otherwise, use memcpy() which causes type erasure - * and so prevents the compiler to reuse an old cached 'op' value after - * Py_CLEAR(). - */ -#ifdef _Py_TYPEOF -#define Py_CLEAR(op) \ - do { \ - _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ - _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - *_tmp_op_ptr = _Py_NULL; \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#else -#define Py_CLEAR(op) \ - do { \ - PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ - PyObject *_tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - PyObject *_null_ptr = _Py_NULL; \ - memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#endif - - -/* Function to use in case the object pointer can be NULL: */ -static inline void Py_XINCREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_INCREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) -#endif - -static inline void Py_XDECREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_DECREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) -#endif - -// Create a new strong reference to an object: -// increment the reference count of the object and return the object. -PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); - -// Similar to Py_NewRef(), but the object can be NULL. -PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); - -static inline PyObject* _Py_NewRef(PyObject *obj) -{ - Py_INCREF(obj); - return obj; -} - -static inline PyObject* _Py_XNewRef(PyObject *obj) -{ - Py_XINCREF(obj); - return obj; -} - -// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overridden with macros by static inline functions for best -// performances. -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) -# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) -#else -# define Py_NewRef(obj) _Py_NewRef(obj) -# define Py_XNewRef(obj) _Py_XNewRef(obj) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject*) Py_GetConstant(unsigned int constant_id); +PyAPI_FUNC(PyObject*) Py_GetConstantBorrowed(unsigned int constant_id); #endif @@ -820,7 +621,12 @@ _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ -#define Py_None (&_Py_NoneStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_None Py_GetConstantBorrowed(Py_CONSTANT_NONE) +#else +# define Py_None (&_Py_NoneStruct) +#endif // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); @@ -834,7 +640,12 @@ Py_NotImplemented is a singleton used to signal that an operation is not implemented for a given type combination. */ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ -#define Py_NotImplemented (&_Py_NotImplementedStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_NotImplemented Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED) +#else +# define Py_NotImplemented (&_Py_NotImplementedStruct) +#endif /* Macro for returning Py_NotImplemented from a function */ #define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented @@ -943,7 +754,11 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else - flags = type->tp_flags; +# ifdef Py_GIL_DISABLED + flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); +# else + flags = type->tp_flags; +# endif #endif return ((flags & feature) != 0); } @@ -967,6 +782,10 @@ static inline int PyType_CheckExact(PyObject *op) { # define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op)) #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/objimpl.h b/Include/objimpl.h index 967e2776767756..56472a72e42d34 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -35,7 +35,7 @@ Functions and macros for modules that implement new object types. fields, this also fills in the ob_size field. - PyObject_Free(op) releases the memory allocated for an object. It does not - run a destructor -- it only frees the memory. PyObject_Free is identical. + run a destructor -- it only frees the memory. - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) don't allocate memory. Instead of a 'type' parameter, they take a pointer to a @@ -153,25 +153,6 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); - -#if !defined(Py_LIMITED_API) -/* Visit all live GC-capable objects, similar to gc.get_objects(None). The - * supplied callback is called on every such object with the void* arg set - * to the supplied arg. Returning 0 from the callback ends iteration, returning - * 1 allows iteration to continue. Returning any other value may result in - * undefined behaviour. - * - * If new objects are (de)allocated by the callback it is undefined if they - * will be visited. - - * Garbage collection is disabled during operation. Explicitly running a - * collection in the callback may lead to undefined behaviour e.g. visiting the - * same objects multiple times or not at all. - */ -typedef int (*gcvisitobjects_t)(PyObject*, void*); -PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); -#endif - /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index ba25bd459c1bcd..2ae6e5c1ba51ec 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/opcode_id_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -11,125 +11,123 @@ extern "C" { /* Instruction opcodes for compiled code */ #define CACHE 0 -#define BEFORE_ASYNC_WITH 1 -#define BEFORE_WITH 2 +#define BINARY_SLICE 1 +#define BINARY_SUBSCR 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define BINARY_SLICE 4 -#define BINARY_SUBSCR 5 -#define CHECK_EG_MATCH 6 -#define CHECK_EXC_MATCH 7 -#define CLEANUP_THROW 8 -#define DELETE_SUBSCR 9 -#define END_ASYNC_FOR 10 -#define END_FOR 11 -#define END_SEND 12 -#define EXIT_INIT_CHECK 13 -#define FORMAT_SIMPLE 14 -#define FORMAT_WITH_SPEC 15 -#define GET_AITER 16 +#define CHECK_EG_MATCH 4 +#define CHECK_EXC_MATCH 5 +#define CLEANUP_THROW 6 +#define DELETE_SUBSCR 7 +#define END_ASYNC_FOR 8 +#define END_FOR 9 +#define END_SEND 10 +#define EXIT_INIT_CHECK 11 +#define FORMAT_SIMPLE 12 +#define FORMAT_WITH_SPEC 13 +#define GET_AITER 14 +#define GET_ANEXT 15 +#define GET_ITER 16 #define RESERVED 17 -#define GET_ANEXT 18 -#define GET_ITER 19 -#define GET_LEN 20 -#define GET_YIELD_FROM_ITER 21 -#define INTERPRETER_EXIT 22 -#define LOAD_ASSERTION_ERROR 23 -#define LOAD_BUILD_CLASS 24 -#define LOAD_LOCALS 25 -#define MAKE_FUNCTION 26 -#define MATCH_KEYS 27 -#define MATCH_MAPPING 28 -#define MATCH_SEQUENCE 29 -#define NOP 30 -#define POP_EXCEPT 31 -#define POP_TOP 32 -#define PUSH_EXC_INFO 33 -#define PUSH_NULL 34 -#define RETURN_GENERATOR 35 -#define RETURN_VALUE 36 -#define SETUP_ANNOTATIONS 37 -#define STORE_SLICE 38 -#define STORE_SUBSCR 39 -#define TO_BOOL 40 -#define UNARY_INVERT 41 -#define UNARY_NEGATIVE 42 -#define UNARY_NOT 43 -#define WITH_EXCEPT_START 44 -#define HAVE_ARGUMENT 45 -#define BINARY_OP 45 -#define BUILD_CONST_KEY_MAP 46 -#define BUILD_LIST 47 -#define BUILD_MAP 48 -#define BUILD_SET 49 -#define BUILD_SLICE 50 -#define BUILD_STRING 51 -#define BUILD_TUPLE 52 -#define CALL 53 -#define CALL_FUNCTION_EX 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define ENTER_EXECUTOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 -#define LOAD_CONST 83 -#define LOAD_DEREF 84 -#define LOAD_FAST 85 -#define LOAD_FAST_AND_CLEAR 86 -#define LOAD_FAST_CHECK 87 -#define LOAD_FAST_LOAD_FAST 88 -#define LOAD_FROM_DICT_OR_DEREF 89 -#define LOAD_FROM_DICT_OR_GLOBALS 90 -#define LOAD_GLOBAL 91 -#define LOAD_NAME 92 -#define LOAD_SUPER_ATTR 93 -#define MAKE_CELL 94 -#define MAP_ADD 95 -#define MATCH_CLASS 96 -#define POP_JUMP_IF_FALSE 97 -#define POP_JUMP_IF_NONE 98 -#define POP_JUMP_IF_NOT_NONE 99 -#define POP_JUMP_IF_TRUE 100 -#define RAISE_VARARGS 101 -#define RERAISE 102 -#define RETURN_CONST 103 -#define SEND 104 -#define SET_ADD 105 -#define SET_FUNCTION_ATTRIBUTE 106 -#define SET_UPDATE 107 -#define STORE_ATTR 108 -#define STORE_DEREF 109 -#define STORE_FAST 110 -#define STORE_FAST_LOAD_FAST 111 -#define STORE_FAST_STORE_FAST 112 -#define STORE_GLOBAL 113 -#define STORE_NAME 114 -#define SWAP 115 -#define UNPACK_EX 116 -#define UNPACK_SEQUENCE 117 -#define YIELD_VALUE 118 +#define GET_LEN 18 +#define GET_YIELD_FROM_ITER 19 +#define INTERPRETER_EXIT 20 +#define LOAD_BUILD_CLASS 21 +#define LOAD_LOCALS 22 +#define MAKE_FUNCTION 23 +#define MATCH_KEYS 24 +#define MATCH_MAPPING 25 +#define MATCH_SEQUENCE 26 +#define NOP 27 +#define POP_EXCEPT 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_CONST_KEY_MAP 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_FUNCTION_EX 51 +#define CALL_INTRINSIC_1 52 +#define CALL_INTRINSIC_2 53 +#define CALL_KW 54 +#define COMPARE_OP 55 +#define CONTAINS_OP 56 +#define CONVERT_VALUE 57 +#define COPY 58 +#define COPY_FREE_VARS 59 +#define DELETE_ATTR 60 +#define DELETE_DEREF 61 +#define DELETE_FAST 62 +#define DELETE_GLOBAL 63 +#define DELETE_NAME 64 +#define DICT_MERGE 65 +#define DICT_UPDATE 66 +#define ENTER_EXECUTOR 67 +#define EXTENDED_ARG 68 +#define FOR_ITER 69 +#define GET_AWAITABLE 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SPECIAL 91 +#define LOAD_SUPER_ATTR 92 +#define MAKE_CELL 93 +#define MAP_ADD 94 +#define MATCH_CLASS 95 +#define POP_JUMP_IF_FALSE 96 +#define POP_JUMP_IF_NONE 97 +#define POP_JUMP_IF_NOT_NONE 98 +#define POP_JUMP_IF_TRUE 99 +#define RAISE_VARARGS 100 +#define RERAISE 101 +#define RETURN_CONST 102 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -145,62 +143,65 @@ extern "C" { #define BINARY_SUBSCR_TUPLE_INT 161 #define CALL_ALLOC_AND_ENTER_INIT 162 #define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BUILTIN_CLASS 164 -#define CALL_BUILTIN_FAST 165 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 -#define CALL_BUILTIN_O 167 -#define CALL_ISINSTANCE 168 -#define CALL_LEN 169 -#define CALL_LIST_APPEND 170 -#define CALL_METHOD_DESCRIPTOR_FAST 171 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 -#define CALL_METHOD_DESCRIPTOR_NOARGS 173 -#define CALL_METHOD_DESCRIPTOR_O 174 -#define CALL_PY_EXACT_ARGS 175 -#define CALL_PY_WITH_DEFAULTS 176 -#define CALL_STR_1 177 -#define CALL_TUPLE_1 178 -#define CALL_TYPE_1 179 -#define COMPARE_OP_FLOAT 180 -#define COMPARE_OP_INT 181 -#define COMPARE_OP_STR 182 -#define FOR_ITER_GEN 183 -#define FOR_ITER_LIST 184 -#define FOR_ITER_RANGE 185 -#define FOR_ITER_TUPLE 186 -#define LOAD_ATTR_CLASS 187 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 188 -#define LOAD_ATTR_INSTANCE_VALUE 189 -#define LOAD_ATTR_METHOD_LAZY_DICT 190 -#define LOAD_ATTR_METHOD_NO_DICT 191 -#define LOAD_ATTR_METHOD_WITH_VALUES 192 -#define LOAD_ATTR_MODULE 193 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 194 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 195 -#define LOAD_ATTR_PROPERTY 196 -#define LOAD_ATTR_SLOT 197 -#define LOAD_ATTR_WITH_HINT 198 -#define LOAD_GLOBAL_BUILTIN 199 -#define LOAD_GLOBAL_MODULE 200 -#define LOAD_SUPER_ATTR_ATTR 201 -#define LOAD_SUPER_ATTR_METHOD 202 -#define RESUME_CHECK 203 -#define SEND_GEN 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 -#define MIN_INSTRUMENTED_OPCODE 236 +#define CALL_BOUND_METHOD_GENERAL 164 +#define CALL_BUILTIN_CLASS 165 +#define CALL_BUILTIN_FAST 166 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 +#define CALL_BUILTIN_O 168 +#define CALL_ISINSTANCE 169 +#define CALL_LEN 170 +#define CALL_LIST_APPEND 171 +#define CALL_METHOD_DESCRIPTOR_FAST 172 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 +#define CALL_METHOD_DESCRIPTOR_NOARGS 174 +#define CALL_METHOD_DESCRIPTOR_O 175 +#define CALL_NON_PY_GENERAL 176 +#define CALL_PY_EXACT_ARGS 177 +#define CALL_PY_GENERAL 178 +#define CALL_STR_1 179 +#define CALL_TUPLE_1 180 +#define CALL_TYPE_1 181 +#define COMPARE_OP_FLOAT 182 +#define COMPARE_OP_INT 183 +#define COMPARE_OP_STR 184 +#define CONTAINS_OP_DICT 185 +#define CONTAINS_OP_SET 186 +#define FOR_ITER_GEN 187 +#define FOR_ITER_LIST 188 +#define FOR_ITER_RANGE 189 +#define FOR_ITER_TUPLE 190 +#define LOAD_ATTR_CLASS 191 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 +#define LOAD_ATTR_INSTANCE_VALUE 193 +#define LOAD_ATTR_METHOD_LAZY_DICT 194 +#define LOAD_ATTR_METHOD_NO_DICT 195 +#define LOAD_ATTR_METHOD_WITH_VALUES 196 +#define LOAD_ATTR_MODULE 197 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 +#define LOAD_ATTR_PROPERTY 200 +#define LOAD_ATTR_SLOT 201 +#define LOAD_ATTR_WITH_HINT 202 +#define LOAD_GLOBAL_BUILTIN 203 +#define LOAD_GLOBAL_MODULE 204 +#define LOAD_SUPER_ATTR_ATTR 205 +#define LOAD_SUPER_ATTR_METHOD 206 +#define RESUME_CHECK 207 +#define SEND_GEN 208 +#define STORE_ATTR_INSTANCE_VALUE 209 +#define STORE_ATTR_SLOT 210 +#define STORE_ATTR_WITH_HINT 211 +#define STORE_SUBSCR_DICT 212 +#define STORE_SUBSCR_LIST_INT 213 +#define TO_BOOL_ALWAYS_TRUE 214 +#define TO_BOOL_BOOL 215 +#define TO_BOOL_INT 216 +#define TO_BOOL_LIST 217 +#define TO_BOOL_NONE 218 +#define TO_BOOL_STR 219 +#define UNPACK_SEQUENCE_LIST 220 +#define UNPACK_SEQUENCE_TUPLE 221 +#define UNPACK_SEQUENCE_TWO_TUPLE 222 #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 @@ -223,15 +224,14 @@ extern "C" { #define JUMP 256 #define JUMP_NO_INTERRUPT 257 #define LOAD_CLOSURE 258 -#define LOAD_METHOD 259 -#define LOAD_SUPER_METHOD 260 -#define LOAD_ZERO_SUPER_ATTR 261 -#define LOAD_ZERO_SUPER_METHOD 262 -#define POP_BLOCK 263 -#define SETUP_CLEANUP 264 -#define SETUP_FINALLY 265 -#define SETUP_WITH 266 -#define STORE_FAST_MAYBE_NULL 267 +#define POP_BLOCK 259 +#define SETUP_CLEANUP 260 +#define SETUP_FINALLY 261 +#define SETUP_WITH 262 +#define STORE_FAST_MAYBE_NULL 263 + +#define HAVE_ARGUMENT 41 +#define MIN_INSTRUMENTED_OPCODE 236 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c2ffa3d5a75c06..d63af11dbd220a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 13 +#define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.13.0a1+" +#define PY_VERSION "3.14.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..79b1b01fcfa594 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -23,23 +23,40 @@ # endif #endif -#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) -/* The following definition is necessary for ncurses 5.7; without it, - some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ +#if defined(WINDOW_HAS_FLAGS) && defined(__APPLE__) +/* gh-109617, gh-115383: we can rely on the default value for NCURSES_OPAQUE on + most platforms, but not on macOS. This is because, starting with Xcode 15, + Apple-provided ncurses.h comes from ncurses 6 (which defaults to opaque + structs) but can still be linked to older versions of ncurses dynamic + libraries which don't provide functions such as is_pad() to deal with opaque + structs. Setting NCURSES_OPAQUE to 0 is harmless in all ncurses releases to + this date (provided that a thread-safe implementation is not required), but + this might change in the future. This fix might become irrelevant once + support for macOS 13 or earlier is dropped. */ #define NCURSES_OPAQUE 0 #endif -#ifdef HAVE_NCURSES_H -#include <ncurses.h> -#else -#include <curses.h> +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> #endif -#ifdef HAVE_NCURSES_H +#ifdef NCURSES_VERSION /* configure was checking <curses.h>, but we will use <ncurses.h>, which has some or all these features. */ -#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) +#if !defined(WINDOW_HAS_FLAGS) && \ + (NCURSES_VERSION_PATCH+0 < 20070303 || !(NCURSES_OPAQUE+0)) +/* the WINDOW flags field was always accessible in ncurses prior to 20070303; + after that, it depends on the value of NCURSES_OPAQUE. */ #define WINDOW_HAS_FLAGS 1 #endif #if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 07020b5dc964cb..9824d099c3df7d 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -48,8 +48,10 @@ struct PyExpat_CAPI enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); - /* might be none for expat < 2.1.0 */ + /* might be NULL for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.6.0 */ + XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); /* always add new stuff to the end! */ }; diff --git a/Include/pyhash.h b/Include/pyhash.h index 6e969f86fa2625..3e23e2758808d7 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -4,19 +4,6 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -/* hash function definition */ -typedef struct { - Py_hash_t (*const hash)(const void *, Py_ssize_t); - const char *name; - const int hash_bits; - const int seed_bits; -} PyHash_FuncDef; - -PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); -#endif - - /* Cutoff for small string DJBX33A optimization in range [1, cutoff). * * About 50% of the strings in a typical Python application are smaller than @@ -60,6 +47,12 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); # endif /* uint64_t && uint32_t && aligned */ #endif /* Py_HASH_ALGORITHM */ +#ifndef Py_LIMITED_API +# define Py_CPYTHON_HASH_H +# include "cpython/pyhash.h" +# undef Py_CPYTHON_HASH_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index c1e2bc5e323358..de1bcb1d2cb632 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -34,8 +34,12 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 806e41955efd7f..615abe103ca038 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -7,7 +7,9 @@ #define PY_MACCONFIG_H #ifdef __APPLE__ +#undef ALIGNOF_MAX_ALIGN_T #undef SIZEOF_LONG +#undef SIZEOF_LONG_DOUBLE #undef SIZEOF_PTHREAD_T #undef SIZEOF_SIZE_T #undef SIZEOF_TIME_T @@ -20,6 +22,7 @@ #undef DOUBLE_IS_BIG_ENDIAN_IEEE754 #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #undef HAVE_GCC_ASM_FOR_X87 +#undef HAVE_GCC_ASM_FOR_X64 #undef VA_LIST_IS_ARRAY #if defined(__LP64__) && defined(__x86_64__) @@ -74,8 +77,14 @@ # define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #endif -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) # define HAVE_GCC_ASM_FOR_X87 +# define ALIGNOF_MAX_ALIGN_T 16 +# define HAVE_GCC_ASM_FOR_X64 1 +# define SIZEOF_LONG_DOUBLE 16 +#else +# define ALIGNOF_MAX_ALIGN_T 8 +# define SIZEOF_LONG_DOUBLE 8 #endif #endif // __APPLE__ diff --git a/Include/pymacro.h b/Include/pymacro.h index 9d264fe6eea1d4..a7945ef84a46fc 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -46,24 +46,42 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ + && !defined(_MSC_VER)) +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array @@ -160,6 +178,9 @@ Py_FatalError("Unreachable C code path reached") #endif +#define _Py_CONTAINER_OF(ptr, type, member) \ + (type*)((char*)ptr - offsetof(type, member)) + // Prevent using an expression as a l-value. // For example, "int x; _Py_RVALUE(x) = 1;" fails with a compiler error. #define _Py_RVALUE(EXPR) ((void)0, (EXPR)) diff --git a/Include/pymath.h b/Include/pymath.h index 4c1e3d9984894b..d8f763f808d662 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -29,14 +29,17 @@ // Py_IS_NAN(X) // Return 1 if float or double arg is a NaN, else 0. +// Soft deprecated since Python 3.14, use isnan() instead. #define Py_IS_NAN(X) isnan(X) // Py_IS_INFINITY(X) // Return 1 if float or double arg is an infinity, else 0. +// Soft deprecated since Python 3.14, use isinf() instead. #define Py_IS_INFINITY(X) isinf(X) // Py_IS_FINITE(X) // Return 1 if float or double arg is neither infinite nor NAN, else 0. +// Soft deprecated since Python 3.14, use isfinite() instead. #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. diff --git a/Include/pymem.h b/Include/pymem.h index 68e33f90b7b913..a80da99e1dd7fc 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -87,6 +87,17 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #define PyMem_DEL(p) PyMem_Free((p)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +// Memory allocator which doesn't require the GIL to be held. +// Usually, it's just a thin wrapper to functions of the standard C library: +// malloc(), calloc(), realloc() and free(). The difference is that +// tracemalloc can track these memory allocations. +PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); +PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); +PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); +PyAPI_FUNC(void) PyMem_RawFree(void *ptr); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYMEM_H # include "cpython/pymem.h" diff --git a/Include/pyport.h b/Include/pyport.h index d30fcd7f6cb7da..2b6bd4c21110e5 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -9,6 +9,24 @@ #endif +// Preprocessor check for a builtin preprocessor function. Always return 0 +// if __has_builtin() macro is not defined. +// +// __has_builtin() is available on clang and GCC 10. +#ifdef __has_builtin +# define _Py__has_builtin(x) __has_builtin(x) +#else +# define _Py__has_builtin(x) 0 +#endif + +// Preprocessor check for a compiler __attribute__. Always return 0 +// if __has_attribute() macro is not defined. +#ifdef __has_attribute +# define _Py__has_attribute(x) __has_attribute(x) +#else +# define _Py__has_attribute(x) 0 +#endif + // Macro to use C++ static_cast<> in the Python C API. #ifdef __cplusplus # define _Py_STATIC_CAST(type, expr) static_cast<type>(expr) @@ -180,6 +198,7 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif +// Soft deprecated since Python 3.14, use memcpy() instead. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif @@ -470,6 +489,14 @@ extern "C" { # define WITH_THREAD #endif +/* Some WebAssembly platforms do not provide a working pthread implementation. + * Thread support is stubbed and any attempt to create a new thread fails. + */ +#if (!defined(HAVE_PTHREAD_STUBS) && \ + (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) +# define Py_CAN_START_THREADS 1 +#endif + #ifdef WITH_THREAD # ifdef Py_BUILD_CORE # ifdef HAVE_THREAD_LOCAL @@ -523,16 +550,6 @@ extern "C" { #endif -// Preprocessor check for a builtin preprocessor function. Always return 0 -// if __has_builtin() macro is not defined. -// -// __has_builtin() is available on clang and GCC 10. -#ifdef __has_builtin -# define _Py__has_builtin(x) __has_builtin(x) -#else -# define _Py__has_builtin(x) 0 -#endif - // _Py_TYPEOF(expr) gets the type of an expression. // // Example: _Py_TYPEOF(x) x_copy = (x); @@ -555,10 +572,18 @@ extern "C" { # define _Py_ADDRESS_SANITIZER # endif # endif +# if __has_feature(thread_sanitizer) +# if !defined(_Py_THREAD_SANITIZER) +# define _Py_THREAD_SANITIZER +# endif +# endif #elif defined(__GNUC__) # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER # endif +# if defined(__SANITIZE_THREAD__) +# define _Py_THREAD_SANITIZER +# endif #endif @@ -578,8 +603,33 @@ extern "C" { # define ALIGNOF_MAX_ALIGN_T _Alignof(long double) #endif +#ifndef PY_CXX_CONST +# ifdef __cplusplus +# define PY_CXX_CONST const +# else +# define PY_CXX_CONST +# endif +#endif + #if defined(__sgi) && !defined(_SGI_MP_SOURCE) # define _SGI_MP_SOURCE #endif +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. +// +// Usage example: +// +// switch (value) { +// case 1: _Py_FALLTHROUGH; +// case 2: code; break; +// } +// +// __attribute__((fallthrough)) was introduced in GCC 7. +#if _Py__has_attribute(fallthrough) +# define _Py_FALLTHROUGH __attribute__((fallthrough)) +#else +# define _Py_FALLTHROUGH do { } while (0) +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pythread.h b/Include/pythread.h index 0784f6b2e5391f..a3216c51d66165 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -21,7 +21,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ - || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) \ + || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \ + || defined(__OpenBSD__) || defined(__NetBSD__) \ || defined(__DragonFly__) || defined(_AIX)) #define PY_HAVE_THREAD_NATIVE_ID PyAPI_FUNC(unsigned long) PyThread_get_thread_native_id(void); diff --git a/Include/refcount.h b/Include/refcount.h new file mode 100644 index 00000000000000..a0bd2087fb1b57 --- /dev/null +++ b/Include/refcount.h @@ -0,0 +1,500 @@ +#ifndef Py_REFCOUNT_H +#define Py_REFCOUNT_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, an object will be marked as immortal by setting all of the +lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. +*/ +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) + +#else +/* +In 32 bit systems, an object will be marked as immortal by setting all of the +lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF + +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the immortality reference count. +*/ +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) +#endif + +// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is +// always 32-bits. +#ifdef Py_GIL_DISABLED +#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX +#endif + + +#ifdef Py_GIL_DISABLED + // The shared reference count uses the two least-significant bits to store + // flags. The remaining bits are used to store the reference count. +# define _Py_REF_SHARED_SHIFT 2 +# define _Py_REF_SHARED_FLAG_MASK 0x3 + + // The shared flags are initialized to zero. +# define _Py_REF_SHARED_INIT 0x0 +# define _Py_REF_MAYBE_WEAKREF 0x1 +# define _Py_REF_QUEUED 0x2 +# define _Py_REF_MERGED 0x3 + + // Create a shared field from a refcnt and desired flags +# define _Py_REF_SHARED(refcnt, flags) \ + (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) +#endif // Py_GIL_DISABLED + + +static inline Py_ssize_t Py_REFCNT(PyObject *ob) { +#if !defined(Py_GIL_DISABLED) + return ob->ob_refcnt; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) +#endif + + +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) + return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == + _Py_IMMORTAL_REFCNT_LOCAL); +#elif SIZEOF_VOID_P > 4 + return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); +#else + return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) + + +// Py_SET_REFCNT() implementation for stable ABI +PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); + +static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 + // Stable ABI implements Py_SET_REFCNT() as a function call + // on limited C API version 3.13 and newer. + _Py_SetRefcnt(ob, refcnt); +#else + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } + +#ifndef Py_GIL_DISABLED + ob->ob_refcnt = refcnt; +#else + if (_Py_IsOwnedByCurrentThread(ob)) { + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } + } + else { + // Set local refcount to zero and shared refcount to desired refcount. + // Mark the object as merged. + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = 0; + ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API+0 < 0x030d0000 +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) +#endif + + +/* +The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement +reference counts. Py_DECREF calls the object's deallocator function when +the refcount falls to 0; for +objects that don't contain references to other objects or heap memory +this can be the standard function free(). Both macros can be used +wherever a void expression is allowed. The argument must not be a +NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. +The macro _Py_NewReference(op) initialize reference counts to 1, and +in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional +bookkeeping appropriate to the special build. + +We assume that the reference count field can never overflow; this can +be proven when the size of the field is the same as the pointer size, so +we ignore the possibility. Provided a C int is at least 32 bits (which +is implicitly assumed in many parts of this code), that's enough for +about 2**31 references to an object. + +XXX The following became out of date in Python 2.2, but I'm not sure +XXX what the full truth is now. Certainly, heap-allocated type objects +XXX can and should be deallocated. +Type objects should never be deallocated; the type pointer in an object +is not considered to be a reference to the type object, to save +complications in the deallocation function. (This is actually a +decision that's up to the implementer of each new type so if you want, +you can count such references to the type object.) +*/ + +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) +PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, + PyObject *op); +PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); +PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); +#endif // Py_REF_DEBUG && !Py_LIMITED_API + +PyAPI_FUNC(void) _Py_Dealloc(PyObject *); + + +/* +These are provided as conveniences to Python runtime embedders, so that +they can have object code that is not dependent on Python compilation flags. +*/ +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); + +// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. +// Private functions used by Py_INCREF() and Py_DECREF(). +PyAPI_FUNC(void) _Py_IncRef(PyObject *); +PyAPI_FUNC(void) _Py_DecRef(PyObject *); + +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) +{ +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_IncRef(op); +# else + Py_IncRef(op); +# endif +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. +#if defined(Py_GIL_DISABLED) + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + uint32_t new_local = local + 1; + if (new_local == 0) { + // local is equal to _Py_IMMORTAL_REFCNT: do nothing + return; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); + } +#elif SIZEOF_VOID_P > 4 + // Portable saturated add, branching on the carry flag and set low bits + PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T new_refcnt = cur_refcnt + 1; + if (new_refcnt == 0) { + // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, + // do nothing + return; + } + op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; +#else + // Explicitly check immortality against the immortal value + if (_Py_IsImmortal(op)) { + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); +#ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +#endif +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) +#endif + + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +// Implements Py_DECREF on objects not owned by the current thread. +PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); +PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); + +// Called from Py_DECREF by the owning thread when the local refcount reaches +// zero. The call will deallocate the object if the shared refcount is also +// zero. Otherwise, the thread gives up ownership and merges the reference +// count fields. +PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); +#endif + +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. +static inline void Py_DECREF(PyObject *op) { +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_DecRef(op); +# else + Py_DecRef(op); +# endif +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (_Py_IsOwnedByCurrentThread(op)) { + if (local == 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefSharedDebug(op, filename, lineno); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) +static inline void Py_DECREF(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return; + } + _Py_DECREF_STAT_INC(); + if (_Py_IsOwnedByCurrentThread(op)) { + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefShared(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#else +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) +{ + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) +#endif + + +/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear + * and tp_dealloc implementations. + * + * Note that "the obvious" code can be deadly: + * + * Py_XDECREF(op); + * op = NULL; + * + * Typically, `op` is something like self->containee, and `self` is done + * using its `containee` member. In the code sequence above, suppose + * `containee` is non-NULL with a refcount of 1. Its refcount falls to + * 0 on the first line, which can trigger an arbitrary amount of code, + * possibly including finalizers (like __del__ methods or weakref callbacks) + * coded in Python, which in turn can release the GIL and allow other threads + * to run, etc. Such code may even invoke methods of `self` again, or cause + * cyclic gc to trigger, but-- oops! --self->containee still points to the + * object being torn down, and it may be in an insane state while being torn + * down. This has in fact been a rich historic source of miserable (rare & + * hard-to-diagnose) segfaulting (and other) bugs. + * + * The safe way is: + * + * Py_CLEAR(op); + * + * That arranges to set `op` to NULL _before_ decref'ing, so that any code + * triggered as a side-effect of `op` getting torn down no longer believes + * `op` points to a valid object. + * + * There are cases where it's safe to use the naive code, but they're brittle. + * For example, if `op` points to a Python integer, you know that destroying + * one of those can't cause problems -- but in part that relies on that + * Python integers aren't currently weakly referencable. Best practice is + * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). + */ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + + +/* Function to use in case the object pointer can be NULL: */ +static inline void Py_XINCREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_INCREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#endif + +static inline void Py_XDECREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_DECREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#endif + +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef(), but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overridden with macros by static inline functions for best +// performances. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#else +# define Py_NewRef(obj) _Py_NewRef(obj) +# define Py_XNewRef(obj) _Py_XNewRef(obj) +#endif + + +#ifdef __cplusplus +} +#endif +#endif // !Py_REFCOUNT_H diff --git a/Include/sliceobject.h b/Include/sliceobject.h index c13863f27c2e63..35e2ea254ca80a 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -8,7 +8,11 @@ extern "C" { PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ -#define Py_Ellipsis (&_Py_EllipsisObject) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_Ellipsis Py_GetConstantBorrowed(Py_CONSTANT_ELLIPSIS) +#else +# define Py_Ellipsis (&_Py_EllipsisObject) +#endif /* Slice object interface */ diff --git a/Include/structmember.h b/Include/structmember.h index f6e8fd829892f4..5f29fbcfed99e3 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -11,7 +11,7 @@ extern "C" { * New definitions are in descrobject.h. * * However, there's nothing wrong with old code continuing to use it, - * and there's not much mainenance overhead in maintaining a few aliases. + * and there's not much maintenance overhead in maintaining a few aliases. * So, don't be too eager to convert old code. * * It uses names not prefixed with Py_. diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 225e0602d191e1..5a0af2e1578eb7 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -1,6 +1,3 @@ - -/* System module interface */ - #ifndef Py_SYSMODULE_H #define Py_SYSMODULE_H #ifdef __cplusplus @@ -10,6 +7,9 @@ extern "C" { PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); + PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) @@ -21,17 +21,15 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#if !defined(Py_LIMITED_API) -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; -} PerfMapState; - -PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); - -PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); -PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); #endif #ifndef Py_LIMITED_API diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 727ba6934bbacb..a6e71eb178b124 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -28,7 +28,10 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); +#endif #ifndef Py_LIMITED_API diff --git a/InternalDocs/README.md b/InternalDocs/README.md new file mode 100644 index 00000000000000..95181a420f1dfb --- /dev/null +++ b/InternalDocs/README.md @@ -0,0 +1,23 @@ + +# CPython Internals Documentation + +The documentation in this folder is intended for CPython maintainers. +It describes implementation details of CPython, which should not be +assumed to be part of the Python language specification. These details +can change between any two CPython versions and should not be assumed +to hold for other implementations of the Python language. + +The core dev team attempts to keep this documentation up to date. If +it is not, please report that through the +[issue tracker](https://github.com/python/cpython/issues). + + +[Compiler Design](compiler.md) + +[Frames](frames.md) + +[Adaptive Instruction Families](adaptive.md) + +[The Source Code Locations Table](locations.md) + +[Exception Handling](exception_handling.md) diff --git a/Python/adaptive.md b/InternalDocs/adaptive.md similarity index 93% rename from Python/adaptive.md rename to InternalDocs/adaptive.md index d978c089b237e0..09245730b271fa 100644 --- a/Python/adaptive.md +++ b/InternalDocs/adaptive.md @@ -2,8 +2,9 @@ ## Families of instructions -The core part of PEP 659 (specializing adaptive interpreter) is the families -of instructions that perform the adaptive specialization. +The core part of [PEP 659](https://peps.python.org/pep-0659/) +(specializing adaptive interpreter) is the families of +instructions that perform the adaptive specialization. A family of instructions has the following fundamental properties: @@ -30,8 +31,9 @@ although these are not fundamental and may change: ## Example family -The `LOAD_GLOBAL` instruction (in Python/bytecodes.c) already has an adaptive -family that serves as a relatively simple example. +The `LOAD_GLOBAL` instruction (in +[Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. The `LOAD_GLOBAL` instruction performs adaptive specialization, calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md new file mode 100644 index 00000000000000..b3dc0a48069969 --- /dev/null +++ b/InternalDocs/compiler.md @@ -0,0 +1,651 @@ + +Compiler design +=============== + +Abstract +-------- + +In CPython, the compilation from source code to bytecode involves several steps: + +1. Tokenize the source code + [Parser/lexer/](https://github.com/python/cpython/blob/main/Parser/lexer/) + and [Parser/tokenizer/](https://github.com/python/cpython/blob/main/Parser/tokenizer/). +2. Parse the stream of tokens into an Abstract Syntax Tree + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c). +3. Transform AST into an instruction sequence + [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). +4. Construct a Control Flow Graph and apply optimizations to it + [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). +5. Emit bytecode based on the Control Flow Graph + [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + +This document outlines how these steps of the process work. + +This document only describes parsing in enough depth to explain what is needed +for understanding compilation. This document provides a detailed, though not +exhaustive, view of the how the entire system works. You will most likely need +to read some source code to have an exact understanding of all details. + + +Parsing +======= + +As of Python 3.9, Python's parser is a PEG parser of a somewhat +unusual design. It is unusual in the sense that the parser's input is a stream +of tokens rather than a stream of characters which is more common with PEG +parsers. + +The grammar file for Python can be found in +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). +The definitions for literal tokens (such as ``:``, numbers, etc.) can be found in +[Grammar/Tokens](https://github.com/python/cpython/blob/main/Grammar/Tokens). +Various C files, including +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +are generated from these. + +See Also: + +* [Guide to the parser](https://devguide.python.org/internals/parser/index.html) + for a detailed description of the parser. + +* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) + for a detailed description of the grammar. + + +Abstract syntax trees (AST) +=========================== + + +The abstract syntax tree (AST) is a high-level representation of the +program structure without the necessity of containing the source code; +it can be thought of as an abstract representation of the source code. The +specification of the AST nodes is specified using the Zephyr Abstract +Syntax Definition Language (ASDL) [^1], [^2]. + +The definition of the AST nodes for Python is found in the file +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl). + +Each AST node (representing statements, expressions, and several +specialized types, like list comprehensions and exception handlers) is +defined by the ASDL. Most definitions in the AST correspond to a +particular source construct, such as an 'if' statement or an attribute +lookup. The definition is independent of its realization in any +particular programming language. + +The following fragment of the Python ASDL construct demonstrates the +approach and syntax: + +``` + module Python + { + stmt = FunctionDef(identifier name, arguments args, stmt* body, + expr* decorators) + | Return(expr? value) | Yield(expr? value) + attributes (int lineno) + } +``` + +The preceding example describes two different kinds of statements and an +expression: function definitions, return statements, and yield expressions. +All three kinds are considered of type ``stmt`` as shown by ``|`` separating +the various kinds. They all take arguments of various kinds and amounts. + +Modifiers on the argument type specify the number of values needed; ``?`` +means it is optional, ``*`` means 0 or more, while no modifier means only one +value for the argument and it is required. ``FunctionDef``, for instance, +takes an ``identifier`` for the *name*, ``arguments`` for *args*, zero or more +``stmt`` arguments for *body*, and zero or more ``expr`` arguments for +*decorators*. + +Do notice that something like 'arguments', which is a node type, is +represented as a single AST node and not as a sequence of nodes as with +stmt as one might expect. + +All three kinds also have an 'attributes' argument; this is shown by the +fact that 'attributes' lacks a '|' before it. + +The statement definitions above generate the following C structure type: + + +``` + typedef struct _stmt *stmt_ty; + + struct _stmt { + enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + } FunctionDef; + + struct { + expr_ty value; + } Return; + + struct { + expr_ty value; + } Yield; + } v; + int lineno; + } +``` + +Also generated are a series of constructor functions that allocate (in +this case) a ``stmt_ty`` struct with the appropriate initialization. The +``kind`` field specifies which component of the union is initialized. The +``FunctionDef()`` constructor function sets 'kind' to ``FunctionDef_kind`` and +initializes the *name*, *args*, *body*, and *attributes* fields. + +See also +[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) + by Thomas Kluyver. + +Memory management +================= + +Before discussing the actual implementation of the compiler, a discussion of +how memory is handled is in order. To make memory management simple, an **arena** +is used that pools memory in a single location for easy +allocation and removal. This enables the removal of explicit memory +deallocation. Because memory allocation for all needed memory in the compiler +registers that memory with the arena, a single call to free the arena is all +that is needed to completely free all memory used by the compiler. + +In general, unless you are working on the critical core of the compiler, memory +management can be completely ignored. But if you are working at either the +very beginning of the compiler or the end, you need to care about how the arena +works. All code relating to the arena is in either +[Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) +or [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + +``PyArena_New()`` will create a new arena. The returned ``PyArena`` structure +will store pointers to all memory given to it. This does the bookkeeping of +what memory needs to be freed when the compiler is finished with the memory it +used. That freeing is done with ``PyArena_Free()``. This only needs to be +called in strategic areas where the compiler exits. + +As stated above, in general you should not have to worry about memory +management when working on the compiler. The technical details of memory +management have been designed to be hidden from you for most cases. + +The only exception comes about when managing a PyObject. Since the rest +of Python uses reference counting, there is extra support added +to the arena to cleanup each PyObject that was allocated. These cases +are very rare. However, if you've allocated a PyObject, you must tell +the arena about it by calling ``PyArena_AddPyObject()``. + + +Source code to AST +================== + +The AST is generated from source code using the function +``_PyParser_ASTFromString()`` or ``_PyParser_ASTFromFile()`` +[Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + +After some checks, a helper function in +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +begins applying production rules on the source code it receives; converting source +code to tokens and matching these tokens recursively to their corresponding rule. The +production rule's corresponding rule function is called on every match. These rule +functions follow the format `xx_rule`. Where *xx* is the grammar rule +that the function handles and is automatically derived from +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram) by +[Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py). + +Each rule function in turn creates an AST node as it goes along. It does this +by allocating all the new nodes it needs, calling the proper AST node creation +functions for any required supporting functions and connecting them as needed. +This continues until all nonterminal symbols are replaced with terminals. If an +error occurs, the rule functions backtrack and try another rule function. If +there are no more rules, an error is set and the parsing ends. + +The AST node creation helper functions have the name `_PyAST_{xx}` +where *xx* is the AST node that the function creates. These are defined by the +ASDL grammar and contained in +[Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) +(which is generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +from +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl)). +This all leads to a sequence of AST nodes stored in ``asdl_seq`` structs. + +To demonstrate everything explained so far, here's the +rule function responsible for a simple named import statement such as +``import sys``. Note that error-checking and debugging code has been +omitted. Removed parts are represented by ``...``. +Furthermore, some comments have been added for explanation. These comments +may not be present in the actual code. + + +``` + // This is the production rule (from python.gram) the rule function + // corresponds to: + // import_name: 'import' dotted_as_names + static stmt_ty + import_name_rule(Parser *p) + { + ... + stmt_ty _res = NULL; + { // 'import' dotted_as_names + ... + Token * _keyword; + asdl_alias_seq* a; + // The tokenizing steps. + if ( + (_keyword = _PyPegen_expect_token(p, 513)) // token='import' + && + (a = dotted_as_names_rule(p)) // dotted_as_names + ) + { + ... + // Generate an AST for the import statement. + _res = _PyAST_Import ( a , ...); + ... + goto done; + } + ... + } + _res = NULL; + done: + ... + return _res; + } +``` + + +To improve backtracking performance, some rules (chosen by applying a +``(memo)`` flag in the grammar file) are memoized. Each rule function checks if +a memoized version exists and returns that if so, else it continues in the +manner stated in the previous paragraphs. + +There are macros for creating and using ``asdl_xx_seq *`` types, where *xx* is +a type of the ASDL sequence. Three main types are defined +manually -- ``generic``, ``identifier`` and ``int``. These types are found in +[Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c) +and its corresponding header file +[Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h). +Functions and macros for creating ``asdl_xx_seq *`` types are as follows: + +``_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_generic_seq`` of the specified length +``_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_identifier_seq`` of the specified length +``_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_int_seq`` of the specified length + +In addition to the three types mentioned above, some ASDL sequence types are +automatically generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +and found in +[Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). +Macros for using both manually defined and automatically generated ASDL +sequence types are as follows: + +``asdl_seq_GET(asdl_xx_seq *, int)`` + Get item held at a specific position in an ``asdl_xx_seq`` +``asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_xx_seq`` to the specified value + +Untyped counterparts exist for some of the typed macros. These are useful +when a function needs to manipulate a generic ASDL sequence: + +``asdl_seq_GET_UNTYPED(asdl_seq *, int)`` + Get item held at a specific position in an ``asdl_seq`` +``asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_seq`` to the specified value +``asdl_seq_LEN(asdl_seq *)`` + Return the length of an ``asdl_seq`` or ``asdl_xx_seq`` + +Note that typed macros and functions are recommended over their untyped +counterparts. Typed macros carry out checks in debug mode and aid +debugging errors caused by incorrectly casting from ``void *``. + +If you are working with statements, you must also worry about keeping +track of what line number generated the statement. Currently the line +number is passed as the last parameter to each ``stmt_ty`` function. + +See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/). + + +Control flow graphs +=================== + +A **control flow graph** (often referenced by its acronym, **CFG**) is a +directed graph that models the flow of a program. A node of a CFG is +not an individual bytecode instruction, but instead represents a +sequence of bytecode instructions that always execute sequentially. +Each node is called a *basic block* and must always execute from +start to finish, with a single entry point at the beginning and a +single exit point at the end. If some bytecode instruction *a* needs +to jump to some other bytecode instruction *b*, then *a* must occur at +the end of its basic block, and *b* must occur at the start of its +basic block. + +As an example, consider the following code snippet: + +.. code-block:: Python + + if x < 10: + f1() + f2() + else: + g() + end() + +The ``x < 10`` guard is represented by its own basic block that +compares ``x`` with ``10`` and then ends in a conditional jump based on +the result of the comparison. This conditional jump allows the block +to point to both the body of the ``if`` and the body of the ``else``. The +``if`` basic block contains the ``f1()`` and ``f2()`` calls and points to +the ``end()`` basic block. The ``else`` basic block contains the ``g()`` +call and similarly points to the ``end()`` block. + +Note that more complex code in the guard, the ``if`` body, or the ``else`` +body may be represented by multiple basic blocks. For instance, +short-circuiting boolean logic in a guard like ``if x or y:`` +will produce one basic block that tests the truth value of ``x`` +and then points both (1) to the start of the ``if`` body and (2) to +a different basic block that tests the truth value of y. + +CFGs are useful as an intermediate representation of the code because +they are a convenient data structure for optimizations. + +AST to CFG to bytecode +====================== + +The conversion of an ``AST`` to bytecode is initiated by a call to the function +``_PyAST_Compile()`` in +[Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). + +The first step is to construct the symbol table. This is implemented by +``_PySymtable_Build()`` in +[Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). +This function begins by entering the starting code block for the AST (passed-in) +and then calling the proper `symtable_visit_{xx}` function (with *xx* being the +AST node type). Next, the AST tree is walked with the various code blocks that +delineate the reach of a local variable as blocks are entered and exited using +``symtable_enter_block()`` and ``symtable_exit_block()``, respectively. + +Once the symbol table is created, the ``AST`` is transformed by ``compiler_codegen()`` +in [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c) +into a sequence of pseudo instructions. These are similar to bytecode, but +in some cases they are more abstract, and are resolved later into actual +bytecode. The construction of this instruction sequence is handled by several +functions that break the task down by various AST node types. The functions are +all named `compiler_visit_{xx}` where *xx* is the name of the node type (such +as ``stmt``, ``expr``, etc.). Each function receives a ``struct compiler *`` +and `{xx}_ty` where *xx* is the AST node type. Typically these functions +consist of a large 'switch' statement, branching based on the kind of +node type passed to it. Simple things are handled inline in the +'switch' statement with more complex transformations farmed out to other +functions named `compiler_{xx}` with *xx* being a descriptive name of what is +being handled. + +When transforming an arbitrary AST node, use the ``VISIT()`` macro. +The appropriate `compiler_visit_{xx}` function is called, based on the value +passed in for <node type> (so `VISIT({c}, expr, {node})` calls +`compiler_visit_expr({c}, {node})`). The ``VISIT_SEQ()`` macro is very similar, +but is called on AST node sequences (those values that were created as +arguments to a node that used the '*' modifier). + +Emission of bytecode is handled by the following macros: + +* ``ADDOP(struct compiler *, location, int)`` + add a specified opcode +* ``ADDOP_IN_SCOPE(struct compiler *, location, int)`` + like ``ADDOP``, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* ``ADDOP_I(struct compiler *, location, int, Py_ssize_t)`` + add an opcode that takes an integer argument +* ``ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`` + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (``names`` or ``varnames``) +* ``ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but steals a reference to PyObject +* ``ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but name mangling is also handled; used for + attribute loading or importing based on name +* ``ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`` + add the ``LOAD_CONST`` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* ``ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`` + just like ``ADDOP_LOAD_CONST_NEW``, but steals a reference to PyObject +* ``ADDOP_JUMP(struct compiler *, location, int, basicblock *)`` + create a jump to a basic block + +The ``location`` argument is a struct with the source location to be +associated with this instruction. It is typically extracted from an +``AST`` node with the ``LOC`` macro. The ``NO_LOCATION`` can be used +for *synthetic* instructions, which we do not associate with a line +number at this stage. For example, the implicit ``return None`` +which is added at the end of a function is not associated with any +line in the source code. + +There are several helper functions that will emit pseudo-instructions +and are named `compiler_{xx}()` where *xx* is what the function helps +with (``list``, ``boolop``, etc.). A rather useful one is ``compiler_nameop()``. +This function looks up the scope of a variable and, based on the +expression context, emits the proper opcode to load, store, or delete +the variable. + +Once the instruction sequence is created, it is transformed into a CFG +by ``_PyCfg_FromInstructionSequence()``. Then ``_PyCfg_OptimizeCodeUnit()`` +applies various peephole optimizations, and +``_PyCfg_OptimizedCfgToInstructionSequence()`` converts the optimized ``CFG`` +back into an instruction sequence. These conversions and optimizations are +implemented in +[Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). + +Finally, the sequence of pseudo-instructions is converted into actual +bytecode. This includes transforming pseudo instructions into actual instructions, +converting jump targets from logical labels to relative offsets, and +construction of the +[exception table](exception_handling.md) and +[locations table](https://github.com/python/cpython/blob/main/Objects/locations.md). +The bytecode and tables are then wrapped into a ``PyCodeObject`` along with additional +metadata, including the ``consts`` and ``names`` arrays, information about function +reference to the source code (filename, etc). All of this is implemented by +``_PyAssemble_MakeCodeObject()`` in +[Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + + +Code objects +============ + +The result of ``PyAST_CompileObject()`` is a ``PyCodeObject`` which is defined in +[Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h). +And with that you now have executable Python bytecode! + +The code objects (byte code) are executed in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c). +This file will also need a new case statement for the new opcode in the big switch +statement in ``_PyEval_EvalFrameDefault()``. + + +Important files +=============== + +* [Parser/](https://github.com/python/cpython/blob/main/Parser/) + + * [Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl): + ASDL syntax file. + + * [Parser/asdl.py](https://github.com/python/cpython/blob/main/Parser/asdl.py): + Parser for ASDL definition files. + Reads in an ASDL description and parses it into an AST that describes it. + + * [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py): + Generate C code from an ASDL description. Generates + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + and + [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). + + * [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c): + The new PEG parser introduced in Python 3.9. + Generated by + [Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py) + from the grammar [Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). + Creates the AST from source code. Rule functions for their corresponding production + rules are found here. + + * [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c): + Contains high-level functions which are + used by the interpreter to create an AST from source code. + + * [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c): + Contains helper functions which are used by functions in + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) + to construct the AST. Also contains helper functions which help raise better error messages + when parsing source code. + + * [Parser/pegen.h](https://github.com/python/cpython/blob/main/Parser/pegen.h): + Header file for the corresponding + [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c). + Also contains definitions of the ``Parser`` and ``Token`` structs. + +* [Python/](https://github.com/python/cpython/blob/main/Python) + + * [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c): + Creates C structs corresponding to the ASDL types. Also contains code for + marshalling AST nodes (core ASDL types have marshalling code in + [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c)). + "File automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + This file must be committed separately after every grammar change + is committed since the ``__version__`` value is set to the latest + grammar change revision number. + + * [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c): + Contains code to handle the ASDL sequence type. + Also has code to handle marshalling the core ASDL types, such as number + and identifier. Used by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + for marshalling AST nodes. + + * [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c): + Used for validating the AST. + + * [Python/ast_opt.c](https://github.com/python/cpython/blob/main/Python/ast_opt.c): + Optimizes the AST. + + * [Python/ast_unparse.c](https://github.com/python/cpython/blob/main/Python/ast_unparse.c): + Converts the AST expression node back into a string (for string annotations). + + * [Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c): + Executes byte code (aka, eval loop). + + * [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c): + Generates a symbol table from AST. + + * [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c): + Implementation of the arena memory manager. + + * [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c): + Emits pseudo bytecode based on the AST. + + * [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c): + Implements peephole optimizations. + + * [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c): + Constructs a code object from a sequence of pseudo instructions. + + * [Python/instruction_sequence.c.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c.c): + A data structure representing a sequence of bytecode-like pseudo-instructions. + +* [Include/](https://github.com/python/cpython/blob/main/Include/) + + * [Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h) + : Header file for + [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c); + contains definition of ``PyCodeObject``. + + * [Include/opcode.h](https://github.com/python/cpython/blob/main/Include/opcode.h) + : One of the files that must be modified if + [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) is. + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Contains the actual definitions of the C structs as generated by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + "Automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + + * [Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h) + : Header for the corresponding + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c). + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Declares ``_PyAST_Validate()`` external (from + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c)). + + * [Include/internal/pycore_symtable.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_symtable.h) + : Header for + [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). + ``struct symtable`` and ``PySTEntryObject`` are defined here. + + * [Include/internal/pycore_parser.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_parser.h) + : Header for the corresponding + [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + + * [Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) + : Header file for the corresponding + [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + + * [Include/opcode_ids.h](https://github.com/python/cpython/blob/main/Include/opcode_ids.h) + : List of opcodes. Generated from + [Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c) + by + [Tools/cases_generator/opcode_id_generator.py](https://github.com/python/cpython/blob/main/Tools/cases_generator/opcode_id_generator.py). + +* [Objects/](https://github.com/python/cpython/blob/main/Objects/) + + * [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c) + : Contains PyCodeObject-related code. + + * [Objects/frameobject.c](https://github.com/python/cpython/blob/main/Objects/frameobject.c) + : Contains the ``frame_setlineno()`` function which should determine whether it is allowed + to make a jump between two points in a bytecode. + +* [Lib/](https://github.com/python/cpython/blob/main/Lib/) + + * [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) + : opcode utilities exposed to Python. + + * [Lib/importlib/_bootstrap_external.py](https://github.com/python/cpython/blob/main/Lib/importlib/_bootstrap_external.py) + : Home of the magic number (named ``MAGIC_NUMBER``) for bytecode versioning. + + +Objects +======= + +* [Locations](locations.md): Describes the location table +* [Frames](frames.md): Describes frames and the frame stack +* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Describes object layout for 3.11 and later +* [Exception Handling](exception_handling.md): Describes the exception table + + +Specializing Adaptive Interpreter +================================= + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. These documents provide more information: + +* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). +* [Adding or extending a family of adaptive instructions](adaptive.md) + + +References +========== + +[^1]: Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris + S. Serra. `The Zephyr Abstract Syntax Description Language.`_ + In Proceedings of the Conference on Domain-Specific Languages, + pp. 213--227, 1997. + +[^2]: The Zephyr Abstract Syntax Description Language.: + https://www.cs.princeton.edu/research/techreps/TR-554-97 diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md new file mode 100644 index 00000000000000..ec09e0769929fa --- /dev/null +++ b/InternalDocs/exception_handling.md @@ -0,0 +1,184 @@ +Description of exception handling +--------------------------------- + +Python uses a technique known as "zero-cost" exception handling, which +minimizes the cost of supporting exceptions. In the common case (where +no exception is raised) the cost is reduced to zero (or close to zero). +The cost of raising an exception is increased, but not by much. + +The following code: + +``` +try: + g(0) +except: + res = "fail" + +``` + +compiles into intermediate code like the following: + +``` + RESUME 0 + + 1 SETUP_FINALLY 8 (to L1) + + 2 LOAD_NAME 0 (g) + PUSH_NULL + LOAD_CONST 0 (0) + CALL 1 + POP_TOP + POP_BLOCK + + -- L1: PUSH_EXC_INFO + + 3 POP_TOP + + 4 LOAD_CONST 1 ('fail') + STORE_NAME 1 (res) +``` + +`SETUP_FINALLY` and `POP_BLOCK` are pseudo-instructions. This means +that they can appear in intermediate code but they are not bytecode +instructions. `SETUP_FINALLY` specifies that henceforth, exceptions +are handled by the code at label L1. The `POP_BLOCK` instruction +reverses the effect of the last `SETUP` instruction, so that the +active exception handler reverts to what it was before. + +`SETUP_FINALLY` and `POP_BLOCK` have no effect when no exceptions +are raised. The idea of zero-cost exception handling is to replace +these pseudo-instructions by metadata which is stored alongside the +bytecode, and which is inspected only when an exception occurs. +This metadata is the exception table, and it is stored in the code +object's `co_exceptiontable` field. + +When the pseudo-instructions are translated into bytecode, +`SETUP_FINALLY` and `POP_BLOCK` are removed, and the exception +table is constructed, mapping each instruction to the exception +handler that covers it, if any. Instructions which are not +covered by any exception handler within the same code object's +bytecode, do not appear in the exception table at all. + +For the code object in our example above, the table has a single +entry specifying that all instructions that were between the +`SETUP_FINALLY` and the `POP_BLOCK` are covered by the exception +handler located at label `L1`. + +Handling Exceptions +------------------- + +At runtime, when an exception occurs, the interpreter calls +``get_exception_handler()`` in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c) +to look up the offset of the current instruction in the exception +table. If it finds a handler, control flow transfers to it. Otherwise, the +exception bubbles up to the caller, and the caller's frame is +checked for a handler covering the `CALL` instruction. This +repeats until a handler is found or the topmost frame is reached. +If no handler is found, the program terminates. During unwinding, +the traceback is constructed as each frame is added to it by +``PyTraceBack_Here()``, which is in +[Python/traceback.c](https://github.com/python/cpython/blob/main/Python/traceback.c). + +Along with the location of an exception handler, each entry of the +exception table also contains the stack depth of the `try` instruction +and a boolean `lasti` value, which indicates whether the instruction +offset of the raising instruction should be pushed to the stack. + +Handling an exception, once an exception table entry is found, consists +of the following steps: + + 1. pop values from the stack until it matches the stack depth for the handler. + 2. if `lasti` is true, then push the offset that the exception was raised at. + 3. push the exception to the stack. + 4. jump to the target offset and resume execution. + + +Reraising Exceptions and `lasti` +-------------------------------- + +The purpose of pushing `lasti` to the stack is for cases where an exception +needs to be re-raised, and be associated with the original instruction that +raised it. This happens, for example, at the end of a `finally` block, when +any in-flight exception needs to be propagated on. As the frame's instruction +pointer now points into the finally block, a `RERAISE` instruction +(with `oparg > 0`) sets it to the `lasti` value from the stack. + +Format of the exception table +----------------------------- + +Conceptually, the exception table consists of a sequence of 5-tuples: +``` + 1. `start-offset` (inclusive) + 2. `end-offset` (exclusive) + 3. `target` + 4. `stack-depth` + 5. `push-lasti` (boolean) +``` + +All offsets and lengths are in code units, not bytes. + +We want the format to be compact, but quickly searchable. +For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. +For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. + +It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: + `start, size, target, depth, push-lasti`. + +Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +It also happens that depth is generally quite small. + +So, we need to encode: +``` + `start` (up to 30 bits) + `size` (up to 30 bits) + `target` (up to 30 bits) + `depth` (up to ~8 bits) + `lasti` (1 bit) +``` + +We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. +Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. +Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. +The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. + +In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. + +For example, the exception entry: +``` + `start`: 20 + `end`: 28 + `target`: 100 + `depth`: 3 + `lasti`: False +``` + +is encoded by first converting to the more compact four value form: +``` + `start`: 20 + `size`: 8 + `target`: 100 + `depth<<1+lasti`: 6 +``` + +which is then encoded as: +``` + 148 (MSB + 20 for start) + 8 (size) + 65 (Extend bit + 1) + 36 (Remainder of target, 100 == (1<<6)+36) + 6 +``` + +for a total of five bytes. + +The code to construct the exception table is in ``assemble_exception_table()`` +in [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + +The interpreter's function to lookup the table by instruction offset is +``get_exception_handler()`` in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c). +The Python function ``_parse_exception_table()`` in +[Lib/dis.py](https://github.com/python/cpython/blob/main/Lib/dis.py) +returns the exception table content as a list of namedtuple instances. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md new file mode 100644 index 00000000000000..34682adb1b422e --- /dev/null +++ b/InternalDocs/frames.md @@ -0,0 +1,138 @@ +# Frames + +Each call to a Python function has an activation record, commonly known as a +"frame". It contains information about the function being executed, consisting +of three conceptual sections: + +* Local variables (including arguments, cells and free variables) +* Evaluation stack +* Specials: The per-frame object references needed by the VM, including + globals dict, code object, instruction pointer, stack depth, the + previous frame, etc. + +The definition of the ``_PyInterpreterFrame`` struct is in +[Include/internal/pycore_frame.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_frame.h). + +# Allocation + +Python semantics allows frames to outlive the activation, so they need to +be allocated outside the C call stack. To reduce overhead and improve locality +of reference, most frames are allocated contiguously in a per-thread stack +(see ``_PyThreadState_PushFrame`` in +[Python/pystate.c](https://github.com/python/cpython/blob/main/Python/pystate.c)). + +Frames of generators and coroutines are embedded in the generator and coroutine +objects, so are not allocated in the per-thread stack. See ``PyGenObject`` in +[Include/internal/pycore_genobject.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_genobject.h). + +## Layout + +Each activation record is laid out as: +* Specials +* Locals +* Stack + +This seems to provide the best performance without excessive complexity. +The specials have a fixed size, so the offset of the locals is know. The +interpreter needs to hold two pointers, a frame pointer and a stack pointer. + +#### Alternative layout + +An alternative layout that was used for part of 3.11 alpha was: + +* Locals +* Specials +* Stack + +This has the advantage that no copying is required when making a call, +as the arguments on the stack are (usually) already in the correct +location for the parameters. However, it requires the VM to maintain +an extra pointer for the locals, which can hurt performance. + +### Generators and Coroutines + +Generators and coroutines contain a `_PyInterpreterFrame` +The specials sections contains the following pointers: + +* Globals dict +* Builtins dict +* Locals dict (not the "fast" locals, but the locals for eval and class creation) +* Code object +* Heap allocated `PyFrameObject` for this activation record, if any. +* The function. + +The pointer to the function is not strictly required, but it is cheaper to +store a strong reference to the function and borrowed references to the globals +and builtins, than strong references to both globals and builtins. + +### Frame objects + +When creating a backtrace or when calling `sys._getframe()` the frame becomes +visible to Python code. When this happens a new `PyFrameObject` is created +and a strong reference to it placed in the `frame_obj` field of the specials +section. The `frame_obj` field is initially `NULL`. + +The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. +If it does then `_PyInterpreterFrame` is copied into the `PyFrameObject`, +except the evaluation stack which must be empty at this point. +The previous frame link is updated to reflect the new location of the frame. + +This mechanism provides the appearance of persistent, heap-allocated +frames for each activation, but with low runtime overhead. + +### Generators and Coroutines + +Generators (objects of type ``PyGen_Type``, ``PyCoro_Type`` or +``PyAsyncGen_Type``) have a `_PyInterpreterFrame` embedded in them, so +that they can be created with a single memory allocation. +When such an embedded frame is iterated or awaited, it can be linked with +frames on the per-thread stack via the linkage fields. + +If a frame object associated with a generator outlives the generator, then +the embedded `_PyInterpreterFrame` is copied into the frame object (see +``take_ownership()`` in +[Python/frame.c](https://github.com/python/cpython/blob/main/Python/frame.c)). + +### Field names + +Many of the fields in `_PyInterpreterFrame` were copied from the 3.10 `PyFrameObject`. +Thus, some of the field names may be a bit misleading. + +For example the `f_globals` field has a `f_` prefix implying it belongs to the +`PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct. +We may rationalize this naming scheme for a later version. + + +### Shim frames + +On entry to `_PyEval_EvalFrameDefault()` a shim `_PyInterpreterFrame` is pushed. +This frame is stored on the C stack, and popped when `_PyEval_EvalFrameDefault()` +returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and +`RETURN_GENERATOR` do not need to check whether the current frame is the entry frame. +The shim frame points to a special code object containing the `INTERPRETER_EXIT` +instruction which cleans up the shim frame and returns. + + +### The Instruction Pointer + +`_PyInterpreterFrame` has two fields which are used to maintain the instruction +pointer: `instr_ptr` and `return_offset`. + +When a frame is executing, `instr_ptr` points to the instruction currently being +executed. In a suspended frame, it points to the instruction that would execute +if the frame were to resume. After `frame.f_lineno` is set, `instr_ptr` points to +the next instruction to be executed. During a call to a python function, +`instr_ptr` points to the call instruction, because this is what we would expect +to see in an exception traceback. + +The `return_offset` field determines where a `RETURN` should go in the caller, +relative to `instr_ptr`. It is only meaningful to the callee, so it needs to +be set in any instruction that implements a call (to a Python function), +including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no +callee, then return_offset is meaningless. It is necessary to have a separate +field for the return offset because (1) if we apply this offset to `instr_ptr` +while executing the `RETURN`, this is too early and would lose us information +about the previous instruction which we could need for introspecting and +debugging. (2) `SEND` needs to pass two offsets to the generator: one for +`RETURN` and one for `YIELD`. It uses the `oparg` for one, and the +`return_offset` for the other. diff --git a/Objects/locations.md b/InternalDocs/locations.md similarity index 88% rename from Objects/locations.md rename to InternalDocs/locations.md index 18a338a95978a9..91a7824e2a8e4d 100644 --- a/Objects/locations.md +++ b/InternalDocs/locations.md @@ -1,10 +1,10 @@ # Locations table -For versions up to 3.10 see ./lnotab_notes.txt +The `co_linetable` bytes object of code objects contains a compact +representation of the source code positions of instructions, which are +returned by the `co_positions()` iterator. -In version 3.11 the `co_linetable` bytes object of code objects contains a compact representation of the positions returned by the `co_positions()` iterator. - -The `co_linetable` consists of a sequence of location entries. +`co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with most significant bit unset. Each entry contains the following information: diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md new file mode 100644 index 00000000000000..930ea110d857ac --- /dev/null +++ b/InternalDocs/string_interning.md @@ -0,0 +1,122 @@ +# String interning + +*Interned* strings are conceptually part of an interpreter-global +*set* of interned strings, meaning that: +- no two interned strings have the same content (across an interpreter); +- two interned strings can be safely compared using pointer equality + (Python `is`). + +This is used to optimize dict and attribute lookups, among other things. + +Python uses three different mechanisms to intern strings: + +- Singleton strings marked in C source with `_Py_STR` and `_Py_ID` macros. + These are statically allocated, and collected using `make regen-global-objects` + (`Tools/build/generate_global_objects.py`), which generates code + for declaration, initialization and finalization. + + The difference between the two kinds is not important. (A `_Py_ID` string is + a valid C name, with which we can refer to it; a `_Py_STR` may e.g. contain + non-identifier characters, so it needs a separate C-compatible name.) + + The empty string is in this category (as `_Py_STR(empty)`). + + These singletons are interned in a runtime-global lookup table, + `_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`), + at runtime initialization. + +- The 256 possible one-character latin-1 strings are singletons, + which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in runtime-global + arrays, `_PyRuntime.static_objects.strings.ascii` and + `_PyRuntime.static_objects.strings.latin1`. + + These are NOT interned at startup in the normal build. + In the free-threaded build, they are; this avoids modifying the + global lookup table after threads are started. + + Interning a one-char latin-1 string will always intern the corresponding + singleton. + +- All other strings are allocated dynamically, and have their + `_PyUnicode_STATE(s).statically_allocated` flag set to zero. + When interned, such strings are added to an interpreter-wide dict, + `PyInterpreterState.cached_objects.interned_strings`. + + The key and value of each entry in this dict reference the same object. + +The three sets of singletons (`_Py_STR`, `_Py_ID`, `_Py_LATIN1_CHR`) +are disjoint. +If you have such a singleton, it (and no other copy) will be interned. + + +## Immortality and reference counting + +Invariant: Every immortal string is interned, *except* the one-char latin-1 +singletons (which might but might not be interned). + +In practice, this means that you must not use `_Py_SetImmortal` on +a string. (If you know it's already immortal, don't immortalize it; +if you know it's not interned you might be immortalizing a redundant copy; +if it's interned and mortal it needs extra processing in +`_PyUnicode_InternImmortal`.) + +The converse is not true: interned strings can be mortal. +For mortal interned strings: +- the 2 references from the interned dict (key & value) are excluded from + their refcount +- the deallocator (`unicode_dealloc`) removes the string from the interned dict +- at shutdown, when the interned dict is cleared, the references are added back + +As with any type, you should only immortalize strings that will live until +interpreter shutdown. +We currently also immortalize strings contained in code objects and similar, +specifically in the compiler and in `marshal`. +These are “close enough†to immortal: even in use cases like hot reloading +or `eval`-ing user input, the number of distinct identifiers and string +constants expected to stay low. + + +## Internal API + +We have the following *internal* API for interning: + +- `_PyUnicode_InternMortal`: just intern the string +- `_PyUnicode_InternImmortal`: intern, and immortalize the result +- `_PyUnicode_InternStatic`: intern a static singleton (`_Py_STR`, `_Py_ID` + or one-byte). Not for general use. + +All take an interpreter state, and a pointer to a `PyObject*` which they +modify in place. + +The functions take ownership of (“stealâ€) the reference to their argument, +and update the argument with a *new* reference. +This means: +- They're “reference neutralâ€. +- They must not be called with a borrowed reference. + + +## State + +The intern state (retrieved by `PyUnicode_CHECK_INTERNED(s)`; +stored in `_PyUnicode_STATE(s).interned`) can be: + +- `SSTATE_NOT_INTERNED` (defined as 0, which is useful in a boolean context) +- `SSTATE_INTERNED_MORTAL` (1) +- `SSTATE_INTERNED_IMMORTAL` (2) +- `SSTATE_INTERNED_IMMORTAL_STATIC` (3) + +The valid transitions between these states are: + +- For dynamically allocated strings: + + - 0 -> 1 (`_PyUnicode_InternMortal`) + - 1 -> 2 or 0 -> 2 (`_PyUnicode_InternImmortal`) + + Using `_PyUnicode_InternStatic` on these is an error; the other cases + don't change the state. + +- One-char latin-1 singletons can be interned (0 -> 3) using any interning + function; after that the functions don't change the state. + +- Other statically allocated strings are interned (0 -> 3) at runtime init; + after that all interning functions don't change the state. diff --git a/LICENSE b/LICENSE index f26bcf4d2de6eb..14603b95c2e23b 100644 --- a/LICENSE +++ b/LICENSE @@ -83,10 +83,8 @@ grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. +i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..590e85ea8c2db1 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,94 @@ +import io +import sys + + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +# longer than this will be be truncated by logcat. This limit has already been +# reduced at least once in the history of Android (from 4076 to 4068 between API +# level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then TextIOWrapper may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + sys.stdout = TextLogStream( + android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + sys.stderr = TextLogStream( + android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, android_log_write, prio, tag, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) + self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + + def __repr__(self): + return f"<TextLogStream {self.buffer.tag!r}>" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string before sending it to the superclass. Note that + # "".splitlines() == [], so nothing will be logged for an empty string. + for line in s.splitlines(keepends=True): + while line: + super().write(line[:MAX_CHARS_PER_WRITE]) + line = line[MAX_CHARS_PER_WRITE:] + + return len(s) + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, android_log_write, prio, tag): + self.android_log_write = android_log_write + self.prio = prio + self.tag = tag + + def __repr__(self): + return f"<BinaryLogStream {self.tag!r}>" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.android_log_write(self.prio, self.tag, + b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 601107d2d86771..75252b3a87f9c4 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", + "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -85,6 +85,10 @@ def _f(): pass dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) +def _get_framelocalsproxy(): + return type(sys._getframe().f_locals) +framelocalsproxy = _get_framelocalsproxy() +del _get_framelocalsproxy generator = type((lambda: (yield))()) ## coroutine ## async def _coro(): pass @@ -836,6 +840,7 @@ def __eq__(self, other): __reversed__ = None Mapping.register(mappingproxy) +Mapping.register(framelocalsproxy) class MappingView(Sized): @@ -1068,40 +1073,10 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) -class _DeprecateByteStringMeta(ABCMeta): - def __new__(cls, name, bases, namespace, **kwargs): - if name != "ByteString": - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__new__(cls, name, bases, namespace, **kwargs) - - def __instancecheck__(cls, instance): - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__instancecheck__(instance) - -class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): - """This unifies bytes and bytearray. - - XXX Should add all their methods. - """ - - __slots__ = () - -ByteString.register(bytes) -ByteString.register(bytearray) - class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1170,4 +1145,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 00000000000000..845fb57a90abb8 --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,64 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False) -> ANSIColors: + if colorize or can_colorize(): + return ANSIColors() + else: + return NoColors + + +def can_colorize() -> bool: + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if "NO_COLOR" in os.environ: + return False + if not COLORIZE: + return False + if not sys.flags.ignore_environment: + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(sys.stderr, "fileno"): + return False + + try: + return os.isatty(sys.stderr.fileno()) + except io.UnsupportedOperation: + return sys.stderr.isatty() diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 00000000000000..20467a7c2bcaeb --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 5dd06ae487dfcf..2d0a5ba46b3e10 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -1,8 +1,7 @@ -# This file is generated by Tools/cases_generator/generate_cases.py +# This file is generated by Tools/cases_generator/py_metadata_generator.py # from: # Python/bytecodes.c # Do not edit! - _specializations = { "RESUME": [ "RESUME_CHECK", @@ -23,6 +22,7 @@ "BINARY_OP_ADD_FLOAT", "BINARY_OP_SUBTRACT_FLOAT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", ], "BINARY_SUBSCR": [ "BINARY_SUBSCR_DICT", @@ -75,6 +75,10 @@ "COMPARE_OP_INT", "COMPARE_OP_STR", ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_TUPLE", @@ -84,7 +88,6 @@ "CALL": [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", "CALL_TYPE_1", "CALL_STR_1", "CALL_TUPLE_1", @@ -100,17 +103,17 @@ "CALL_METHOD_DESCRIPTOR_NOARGS", "CALL_METHOD_DESCRIPTOR_FAST", "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", ], } -# An irregular case: -_specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") - _specialized_opmap = { - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'BINARY_OP_ADD_FLOAT': 150, 'BINARY_OP_ADD_INT': 151, 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'BINARY_OP_MULTIPLY_FLOAT': 153, 'BINARY_OP_MULTIPLY_INT': 154, 'BINARY_OP_SUBTRACT_FLOAT': 155, @@ -122,183 +125,187 @@ 'BINARY_SUBSCR_TUPLE_INT': 161, 'CALL_ALLOC_AND_ENTER_INIT': 162, 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BUILTIN_CLASS': 164, - 'CALL_BUILTIN_FAST': 165, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, - 'CALL_BUILTIN_O': 167, - 'CALL_ISINSTANCE': 168, - 'CALL_LEN': 169, - 'CALL_LIST_APPEND': 170, - 'CALL_METHOD_DESCRIPTOR_FAST': 171, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, - 'CALL_METHOD_DESCRIPTOR_O': 174, - 'CALL_PY_EXACT_ARGS': 175, - 'CALL_PY_WITH_DEFAULTS': 176, - 'CALL_STR_1': 177, - 'CALL_TUPLE_1': 178, - 'CALL_TYPE_1': 179, - 'COMPARE_OP_FLOAT': 180, - 'COMPARE_OP_INT': 181, - 'COMPARE_OP_STR': 182, - 'FOR_ITER_GEN': 183, - 'FOR_ITER_LIST': 184, - 'FOR_ITER_RANGE': 185, - 'FOR_ITER_TUPLE': 186, - 'LOAD_ATTR_CLASS': 187, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 188, - 'LOAD_ATTR_INSTANCE_VALUE': 189, - 'LOAD_ATTR_METHOD_LAZY_DICT': 190, - 'LOAD_ATTR_METHOD_NO_DICT': 191, - 'LOAD_ATTR_METHOD_WITH_VALUES': 192, - 'LOAD_ATTR_MODULE': 193, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 194, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 195, - 'LOAD_ATTR_PROPERTY': 196, - 'LOAD_ATTR_SLOT': 197, - 'LOAD_ATTR_WITH_HINT': 198, - 'LOAD_GLOBAL_BUILTIN': 199, - 'LOAD_GLOBAL_MODULE': 200, - 'LOAD_SUPER_ATTR_ATTR': 201, - 'LOAD_SUPER_ATTR_METHOD': 202, - 'RESUME_CHECK': 203, - 'SEND_GEN': 204, - 'STORE_ATTR_INSTANCE_VALUE': 205, - 'STORE_ATTR_SLOT': 206, - 'STORE_ATTR_WITH_HINT': 207, - 'STORE_SUBSCR_DICT': 208, - 'STORE_SUBSCR_LIST_INT': 209, - 'TO_BOOL_ALWAYS_TRUE': 210, - 'TO_BOOL_BOOL': 211, - 'TO_BOOL_INT': 212, - 'TO_BOOL_LIST': 213, - 'TO_BOOL_NONE': 214, - 'TO_BOOL_STR': 215, - 'UNPACK_SEQUENCE_LIST': 216, - 'UNPACK_SEQUENCE_TUPLE': 217, - 'UNPACK_SEQUENCE_TWO_TUPLE': 218, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_LEN': 170, + 'CALL_LIST_APPEND': 171, + 'CALL_METHOD_DESCRIPTOR_FAST': 172, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, + 'CALL_METHOD_DESCRIPTOR_O': 175, + 'CALL_NON_PY_GENERAL': 176, + 'CALL_PY_EXACT_ARGS': 177, + 'CALL_PY_GENERAL': 178, + 'CALL_STR_1': 179, + 'CALL_TUPLE_1': 180, + 'CALL_TYPE_1': 181, + 'COMPARE_OP_FLOAT': 182, + 'COMPARE_OP_INT': 183, + 'COMPARE_OP_STR': 184, + 'CONTAINS_OP_DICT': 185, + 'CONTAINS_OP_SET': 186, + 'FOR_ITER_GEN': 187, + 'FOR_ITER_LIST': 188, + 'FOR_ITER_RANGE': 189, + 'FOR_ITER_TUPLE': 190, + 'LOAD_ATTR_CLASS': 191, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, + 'LOAD_ATTR_INSTANCE_VALUE': 193, + 'LOAD_ATTR_METHOD_LAZY_DICT': 194, + 'LOAD_ATTR_METHOD_NO_DICT': 195, + 'LOAD_ATTR_METHOD_WITH_VALUES': 196, + 'LOAD_ATTR_MODULE': 197, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, + 'LOAD_ATTR_PROPERTY': 200, + 'LOAD_ATTR_SLOT': 201, + 'LOAD_ATTR_WITH_HINT': 202, + 'LOAD_GLOBAL_BUILTIN': 203, + 'LOAD_GLOBAL_MODULE': 204, + 'LOAD_SUPER_ATTR_ATTR': 205, + 'LOAD_SUPER_ATTR_METHOD': 206, + 'RESUME_CHECK': 207, + 'SEND_GEN': 208, + 'STORE_ATTR_INSTANCE_VALUE': 209, + 'STORE_ATTR_SLOT': 210, + 'STORE_ATTR_WITH_HINT': 211, + 'STORE_SUBSCR_DICT': 212, + 'STORE_SUBSCR_LIST_INT': 213, + 'TO_BOOL_ALWAYS_TRUE': 214, + 'TO_BOOL_BOOL': 215, + 'TO_BOOL_INT': 216, + 'TO_BOOL_LIST': 217, + 'TO_BOOL_NONE': 218, + 'TO_BOOL_STR': 219, + 'UNPACK_SEQUENCE_LIST': 220, + 'UNPACK_SEQUENCE_TUPLE': 221, + 'UNPACK_SEQUENCE_TWO_TUPLE': 222, } opmap = { 'CACHE': 0, - 'BEFORE_ASYNC_WITH': 1, - 'BEFORE_WITH': 2, - 'BINARY_SLICE': 4, - 'BINARY_SUBSCR': 5, - 'CHECK_EG_MATCH': 6, - 'CHECK_EXC_MATCH': 7, - 'CLEANUP_THROW': 8, - 'DELETE_SUBSCR': 9, - 'END_ASYNC_FOR': 10, - 'END_FOR': 11, - 'END_SEND': 12, - 'EXIT_INIT_CHECK': 13, - 'FORMAT_SIMPLE': 14, - 'FORMAT_WITH_SPEC': 15, - 'GET_AITER': 16, 'RESERVED': 17, - 'GET_ANEXT': 18, - 'GET_ITER': 19, - 'GET_LEN': 20, - 'GET_YIELD_FROM_ITER': 21, - 'INTERPRETER_EXIT': 22, - 'LOAD_ASSERTION_ERROR': 23, - 'LOAD_BUILD_CLASS': 24, - 'LOAD_LOCALS': 25, - 'MAKE_FUNCTION': 26, - 'MATCH_KEYS': 27, - 'MATCH_MAPPING': 28, - 'MATCH_SEQUENCE': 29, - 'NOP': 30, - 'POP_EXCEPT': 31, - 'POP_TOP': 32, - 'PUSH_EXC_INFO': 33, - 'PUSH_NULL': 34, - 'RETURN_GENERATOR': 35, - 'RETURN_VALUE': 36, - 'SETUP_ANNOTATIONS': 37, - 'STORE_SLICE': 38, - 'STORE_SUBSCR': 39, - 'TO_BOOL': 40, - 'UNARY_INVERT': 41, - 'UNARY_NEGATIVE': 42, - 'UNARY_NOT': 43, - 'WITH_EXCEPT_START': 44, - 'BINARY_OP': 45, - 'BUILD_CONST_KEY_MAP': 46, - 'BUILD_LIST': 47, - 'BUILD_MAP': 48, - 'BUILD_SET': 49, - 'BUILD_SLICE': 50, - 'BUILD_STRING': 51, - 'BUILD_TUPLE': 52, - 'CALL': 53, - 'CALL_FUNCTION_EX': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'ENTER_EXECUTOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_CONST': 83, - 'LOAD_DEREF': 84, - 'LOAD_FAST': 85, - 'LOAD_FAST_AND_CLEAR': 86, - 'LOAD_FAST_CHECK': 87, - 'LOAD_FAST_LOAD_FAST': 88, - 'LOAD_FROM_DICT_OR_DEREF': 89, - 'LOAD_FROM_DICT_OR_GLOBALS': 90, - 'LOAD_GLOBAL': 91, - 'LOAD_NAME': 92, - 'LOAD_SUPER_ATTR': 93, - 'MAKE_CELL': 94, - 'MAP_ADD': 95, - 'MATCH_CLASS': 96, - 'POP_JUMP_IF_FALSE': 97, - 'POP_JUMP_IF_NONE': 98, - 'POP_JUMP_IF_NOT_NONE': 99, - 'POP_JUMP_IF_TRUE': 100, - 'RAISE_VARARGS': 101, - 'RERAISE': 102, - 'RETURN_CONST': 103, - 'SEND': 104, - 'SET_ADD': 105, - 'SET_FUNCTION_ATTRIBUTE': 106, - 'SET_UPDATE': 107, - 'STORE_ATTR': 108, - 'STORE_DEREF': 109, - 'STORE_FAST': 110, - 'STORE_FAST_LOAD_FAST': 111, - 'STORE_FAST_STORE_FAST': 112, - 'STORE_GLOBAL': 113, - 'STORE_NAME': 114, - 'SWAP': 115, - 'UNPACK_EX': 116, - 'UNPACK_SEQUENCE': 117, - 'YIELD_VALUE': 118, 'RESUME': 149, + 'INSTRUMENTED_LINE': 254, + 'BINARY_SLICE': 1, + 'BINARY_SUBSCR': 2, + 'CHECK_EG_MATCH': 4, + 'CHECK_EXC_MATCH': 5, + 'CLEANUP_THROW': 6, + 'DELETE_SUBSCR': 7, + 'END_ASYNC_FOR': 8, + 'END_FOR': 9, + 'END_SEND': 10, + 'EXIT_INIT_CHECK': 11, + 'FORMAT_SIMPLE': 12, + 'FORMAT_WITH_SPEC': 13, + 'GET_AITER': 14, + 'GET_ANEXT': 15, + 'GET_ITER': 16, + 'GET_LEN': 18, + 'GET_YIELD_FROM_ITER': 19, + 'INTERPRETER_EXIT': 20, + 'LOAD_BUILD_CLASS': 21, + 'LOAD_LOCALS': 22, + 'MAKE_FUNCTION': 23, + 'MATCH_KEYS': 24, + 'MATCH_MAPPING': 25, + 'MATCH_SEQUENCE': 26, + 'NOP': 27, + 'POP_EXCEPT': 28, + 'POP_TOP': 29, + 'PUSH_EXC_INFO': 30, + 'PUSH_NULL': 31, + 'RETURN_GENERATOR': 32, + 'RETURN_VALUE': 33, + 'SETUP_ANNOTATIONS': 34, + 'STORE_SLICE': 35, + 'STORE_SUBSCR': 36, + 'TO_BOOL': 37, + 'UNARY_INVERT': 38, + 'UNARY_NEGATIVE': 39, + 'UNARY_NOT': 40, + 'WITH_EXCEPT_START': 41, + 'BINARY_OP': 42, + 'BUILD_CONST_KEY_MAP': 43, + 'BUILD_LIST': 44, + 'BUILD_MAP': 45, + 'BUILD_SET': 46, + 'BUILD_SLICE': 47, + 'BUILD_STRING': 48, + 'BUILD_TUPLE': 49, + 'CALL': 50, + 'CALL_FUNCTION_EX': 51, + 'CALL_INTRINSIC_1': 52, + 'CALL_INTRINSIC_2': 53, + 'CALL_KW': 54, + 'COMPARE_OP': 55, + 'CONTAINS_OP': 56, + 'CONVERT_VALUE': 57, + 'COPY': 58, + 'COPY_FREE_VARS': 59, + 'DELETE_ATTR': 60, + 'DELETE_DEREF': 61, + 'DELETE_FAST': 62, + 'DELETE_GLOBAL': 63, + 'DELETE_NAME': 64, + 'DICT_MERGE': 65, + 'DICT_UPDATE': 66, + 'ENTER_EXECUTOR': 67, + 'EXTENDED_ARG': 68, + 'FOR_ITER': 69, + 'GET_AWAITABLE': 70, + 'IMPORT_FROM': 71, + 'IMPORT_NAME': 72, + 'IS_OP': 73, + 'JUMP_BACKWARD': 74, + 'JUMP_BACKWARD_NO_INTERRUPT': 75, + 'JUMP_FORWARD': 76, + 'LIST_APPEND': 77, + 'LIST_EXTEND': 78, + 'LOAD_ATTR': 79, + 'LOAD_COMMON_CONSTANT': 80, + 'LOAD_CONST': 81, + 'LOAD_DEREF': 82, + 'LOAD_FAST': 83, + 'LOAD_FAST_AND_CLEAR': 84, + 'LOAD_FAST_CHECK': 85, + 'LOAD_FAST_LOAD_FAST': 86, + 'LOAD_FROM_DICT_OR_DEREF': 87, + 'LOAD_FROM_DICT_OR_GLOBALS': 88, + 'LOAD_GLOBAL': 89, + 'LOAD_NAME': 90, + 'LOAD_SPECIAL': 91, + 'LOAD_SUPER_ATTR': 92, + 'MAKE_CELL': 93, + 'MAP_ADD': 94, + 'MATCH_CLASS': 95, + 'POP_JUMP_IF_FALSE': 96, + 'POP_JUMP_IF_NONE': 97, + 'POP_JUMP_IF_NOT_NONE': 98, + 'POP_JUMP_IF_TRUE': 99, + 'RAISE_VARARGS': 100, + 'RERAISE': 101, + 'RETURN_CONST': 102, + 'SEND': 103, + 'SET_ADD': 104, + 'SET_FUNCTION_ATTRIBUTE': 105, + 'SET_UPDATE': 106, + 'STORE_ATTR': 107, + 'STORE_DEREF': 108, + 'STORE_FAST': 109, + 'STORE_FAST_LOAD_FAST': 110, + 'STORE_FAST_STORE_FAST': 111, + 'STORE_GLOBAL': 112, + 'STORE_NAME': 113, + 'SWAP': 114, + 'UNPACK_EX': 115, + 'UNPACK_SEQUENCE': 116, + 'YIELD_VALUE': 117, 'INSTRUMENTED_RESUME': 236, 'INSTRUMENTED_END_FOR': 237, 'INSTRUMENTED_END_SEND': 238, @@ -317,19 +324,15 @@ 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, - 'INSTRUMENTED_LINE': 254, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, - 'LOAD_METHOD': 259, - 'LOAD_SUPER_METHOD': 260, - 'LOAD_ZERO_SUPER_ATTR': 261, - 'LOAD_ZERO_SUPER_METHOD': 262, - 'POP_BLOCK': 263, - 'SETUP_CLEANUP': 264, - 'SETUP_FINALLY': 265, - 'SETUP_WITH': 266, - 'STORE_FAST_MAYBE_NULL': 267, + 'POP_BLOCK': 259, + 'SETUP_CLEANUP': 260, + 'SETUP_FINALLY': 261, + 'SETUP_WITH': 262, + 'STORE_FAST_MAYBE_NULL': 263, } + +HAVE_ARGUMENT = 41 MIN_INSTRUMENTED_OPCODE = 236 -HAVE_ARGUMENT = 45 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index aa66c8b9f4189f..0cb064fcd791be 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine): # MACOSX_DEPLOYMENT_TARGET. macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + if macver and '.' not in macver: + # Ensure that the version includes at least a major + # and minor version, even if MACOSX_DEPLOYMENT_TARGET + # is set to a single-label version like "14". + macver += '.0' macrelease = _get_system_version() or macver macver = macver or macrelease diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bca2acf1fc88cf..27cacb8e01ff3b 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -204,6 +204,17 @@ def _format_offset(off, sep=':'): s += '.%06d' % ss.microseconds return s +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -261,6 +272,14 @@ def _wrap_strftime(object, format, timetuple): # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) + elif ch in 'YG' and object.year < 1000 and _need_normalize_century(): + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = object.year + push('{:04}'.format(year)) else: push('%') push(ch) @@ -556,10 +575,6 @@ def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -970,6 +985,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @@ -1113,32 +1130,33 @@ def replace(self, year=None, month=None, day=None): # Comparisons of date objects with other. def __eq__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) + assert not isinstance(other, datetime) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) @@ -1809,7 +1827,7 @@ def fromtimestamp(cls, timestamp, tz=None): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" import warnings - warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " + warnings.warn("datetime.datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.fromtimestamp(t, datetime.UTC).", @@ -1827,8 +1845,8 @@ def now(cls, tz=None): def utcnow(cls): "Construct a UTC datetime from time.time()." import warnings - warnings.warn("datetime.utcnow() is deprecated and scheduled for " - "removal in a future version. Instead, Use timezone-aware " + warnings.warn("datetime.datetime.utcnow() is deprecated and scheduled for " + "removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " "datetime.datetime.now(datetime.UTC).", DeprecationWarning, @@ -2137,42 +2155,32 @@ def dst(self): def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) @@ -2347,6 +2355,9 @@ def __new__(cls, offset, name=_Omitted): "timedelta(hours=24).") return cls._create(offset, name) + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 2692f2fcba45bf..613123ec7b4329 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -13,104 +13,7 @@ # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" +"""Python decimal arithmetic module""" __all__ = [ # Two major classes @@ -2228,10 +2131,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2281,13 +2190,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 32698abac78d25..75b5ad1b1a47d2 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,7 +16,7 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : @@ -1197,7 +1197,8 @@ def _readinto(self, buf, read1): return written def tell(self): - return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos + # GH-95782: Keep return value non-negative + return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0) def seek(self, pos, whence=0): if whence not in valid_seek_flags: @@ -1495,6 +1496,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') @@ -1571,6 +1577,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): self._blksize = getattr(fdfstat, 'st_blksize', 0) if self._blksize <= 1: self._blksize = DEFAULT_BUFFER_SIZE + self._estimated_size = fdfstat.st_size if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1648,14 +1655,18 @@ def readall(self): """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._estimated_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + bufsize = self._estimated_size + 1 + + if self._estimated_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._estimated_size >= pos: + bufsize = self._estimated_size - pos + 1 + except OSError: + pass result = bytearray() while True: @@ -1731,6 +1742,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._estimated_size = size return size def close(self): @@ -2198,8 +2210,9 @@ def write(self, s): self.buffer.write(b) if self._line_buffering and (haslf or "\r" in s): self.flush() - self._set_decoded_chars('') - self._snapshot = None + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None if self._decoder: self._decoder.reset() return length @@ -2513,8 +2526,9 @@ def read(self, size=None): # Read everything. result = (self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)) - self._set_decoded_chars('') - self._snapshot = None + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None return result else: # Keep reading chunks until we have size characters to return. diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 936346e187ff69..a8bf5cd3e638a4 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -14,7 +14,127 @@ import re import decimal - +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra + + d = {} + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: + if show: + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. + if show: + print("square", end="") + result = d[lo] * d[lo] # same object + if hi != lo: + if show: + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check def int_to_decimal(n): """Asymptotically fast conversion of an 'int' to Decimal.""" @@ -29,61 +149,76 @@ def int_to_decimal(n): # "clever" recursive way. If we want a string representation, we # apply str to _that_. - D = decimal.Decimal - D2 = D(2) - - BITLIM = 128 - - mem = {} - - def w2pow(w): - """Return D(2)**w and store the result. Also possibly save some - intermediate results. In context, these are likely to be reused - across various levels of the conversion to Decimal.""" - if (result := mem.get(w)) is None: - if w <= BITLIM: - result = D2**w - elif w - 1 in mem: - result = (t := mem[w - 1]) + t - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w2pow(w2) * w2pow(w - w2) - mem[w] = result - return result + from decimal import Decimal as D + BITLIM = 200 + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. def inner(n, w): if w <= BITLIM: return D(n) w2 = w >> 1 hi = n >> w2 - lo = n - (hi << w2) - return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2) - - with decimal.localcontext() as ctx: - ctx.prec = decimal.MAX_PREC - ctx.Emax = decimal.MAX_EMAX - ctx.Emin = decimal.MIN_EMIN - ctx.traps[decimal.Inexact] = 1 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) if n < 0: negate = True n = -n else: negate = False - result = inner(n, n.bit_length()) + result = inner(n, nbits) if negate: result = -result return result - def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" - return str(int_to_decimal(n)) - + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s def _str_to_int_inner(s): """Asymptotically fast conversion of a 'str' to an 'int'.""" @@ -100,38 +235,157 @@ def _str_to_int_inner(s): DIGLIM = 2048 - mem = {} - - def w5pow(w): - """Return 5**w and store the result. - Also possibly save some intermediate results. In context, these - are likely to be reused across various levels of the conversion - to 'int'. - """ - if (result := mem.get(w)) is None: - if w <= DIGLIM: - result = 5**w - elif w - 1 in mem: - result = mem[w - 1] * 5 - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w5pow(w2) * w5pow(w - w2) - mem[w] = result - return result - def inner(a, b): if b - a <= DIGLIM: return int(s[a:b]) mid = (a + b + 1) >> 1 - return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid)) + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + w5pow = compute_powers(len(s), 5, DIGLIM) return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platfrom can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -140,8 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) - + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -274,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -283,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operationa, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limir, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limir, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000000..1693cbd0b98b74 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com> +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000000..efb6d343cc9a7c --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,3 @@ +if __name__ == "__main__": + from .main import interactive_console as __pyrepl_interactive_console + __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..849617bf7585e4 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib() -> str: + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000000..c3fce91013b001 --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,483 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .historical_reader import HistoricalReader + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + self.reader.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class ctrl_c(Command): + def do(self) -> None: + self.reader.console.finish() + self.reader.finish() + raise KeyboardInterrupt + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if r.bol() == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if new_y > r.max_row(): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + text = self.event * r.get_arg() + r.insert(text) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + with self.reader.suspend(): + pager = get_pager() + pager(history, gethistoryfile()) + + +class paste_mode(Command): + + def do(self) -> None: + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + self.reader.in_bracketed_paste = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.in_bracketed_paste = False + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000000..05770aaf5060cc --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,290 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amout + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + r.cmpltn_menu_visible = True + r.cmpltn_message_visible = False + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif not r.cmpltn_menu_visible: + r.cmpltn_message_visible = True + if stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + if r.cmpltn_menu_visible: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_visible: bool = field(init=False) + cmpltn_message_visible: bool = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_visible: + ly = self.lxy[1] + screen[ly:ly] = self.cmpltn_menu + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_visible = False + self.cmpltn_message_visible = False + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000000..a8d3f520340dcf --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,201 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] + +from abc import ABC, abstractmethod +import ast +import code +from dataclasses import dataclass, field +import os.path +import sys + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + from typing import Callable + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self, timeout: float | None) -> bool: + """Wait for an event. The return value is True if an event is + available, False if the timeout has been reached. If timeout is + None, wait forever. The timeout is in milliseconds.""" + ... + + @property + def input_hook(self) -> Callable[[], int] | None: + """Returns the current input hook.""" + ... + + @abstractmethod + def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "<console>", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None): + super().showsyntaxerror(colorize=self.can_colorize) + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + def runsource(self, source, filename="<input>", symbol="single"): + try: + tree = ast.parse(source) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol, dont_inherit=True) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000000..3a624d9f6835d1 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000000..5b85cb0f52521f --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,74 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000000..7f4d0672d02094 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,346 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\<backspace>", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\<page down>", "last-history"), + (r"\<page up>", "first-history"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i].rstrip() + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + self.last_refresh_cache.invalidated = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(): + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000000..21c24eb5cde3e3 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self) -> bool: + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000000..d11df4b5164696 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,213 @@ +# Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com> +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. + +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. + +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. + +The extension to readline is that the sequence \\<KEY> denotes the +sequence of characters produced by hitting KEY. + +Examples: +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\<UP>' - the up arrow key +'\\<up>' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\<tab>', '\\<TAB>', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") + + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py new file mode 100644 index 00000000000000..041a4009f42ed8 --- /dev/null +++ b/Lib/_pyrepl/main.py @@ -0,0 +1,55 @@ +import os +import sys + +CAN_USE_PYREPL: bool +if sys.platform != "win32": + CAN_USE_PYREPL = True +else: + CAN_USE_PYREPL = sys.getwindowsversion().build >= 10586 # Windows 10 TH2 + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + global CAN_USE_PYREPL + if not CAN_USE_PYREPL: + return sys._baserepl() + + if mainmodule: + namespace = mainmodule.__dict__ + else: + import __main__ + namespace = __main__.__dict__ + namespace.pop("__pyrepl_interactive_console", None) + + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, namespace) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + run_interactive = None + try: + import errno + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) + from .simple_interact import run_multiline_interactive_console + run_interactive = run_multiline_interactive_console + except Exception as e: + from .trace import trace + msg = f"warning: can't use pyrepl: {e}" + trace(msg) + print(msg, file=sys.stderr) + CAN_USE_PYREPL = False + if run_interactive is None: + return sys._baserepl() + run_interactive(namespace) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000000..395f5945ab740b --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,24 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000000..66dcd99111adfc --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000000..b2da038906939b --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,787 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .trace import trace + + +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed representation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\<return>", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + (r"\C-z", "suspend"), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + # (r'\M-\n', 'insert-nl'), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\<up>", "up"), + (r"\<down>", "down"), + (r"\<left>", "left"), + (r"\C-\<left>", "backward-word"), + (r"\<right>", "right"), + (r"\C-\<right>", "forward-word"), + (r"\<delete>", "delete"), + (r"\<backspace>", "backspace"), + (r"\M-\<backspace>", "backward-kill-word"), + (r"\<end>", "end-of-line"), # was 'end' + (r"\<home>", "beginning-of-line"), # was 'home' + (r"\<f1>", "help"), + (r"\<f2>", "show-history"), + (r"\<f3>", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into 'buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to 'syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + in_bracketed_paste: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + self.invalidated = False + + def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # <https://bugs.python.org/issue25660>. + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus + + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] + + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] + + pos = self.pos + pos -= offset + + lines = "".join(self.buffer[offset:]).split("\n") + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): + ll = len(line) + if 0 <= pos <= ll: + self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + i = 0 + while l: + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + if len(l) > index_to_wrap_before: + offset += index_to_wrap_before + post = "\\" + after = [1] + else: + offset += index_to_wrap_before + 1 # Takes the newline + post = "" + after = [] + last_refresh_line_end_offsets.append(offset) + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + i += 1 + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) + return screen + + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning 'default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + 'lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = f"(arg: {self.arg}) " + elif self.paste_mode and not self.in_bracketed_paste: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif self.ps4 and lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if self.can_colorize: + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + + # this call sets up self.cxy, so call it first. + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + input_hook = self.console.input_hook + if input_hook: + input_hook() + # We use the same timeout as in readline.c: 100ms + while not self.console.wait(100): + input_hook() + event = self.console.get_event(block=False) + else: + event = self.console.get_event(block) + if not event: # can only happen if we're not blocking + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + else: + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000000..28f592d80b1b03 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,587 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +import os +from site import gethistoryfile # type: ignore[attr-defined] +import sys +from rlcompleter import Completer as RLCompleter + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +MoreLinesCallable = Callable[[str], bool] + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = None + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\<backspace>", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n": # ignore whitespaces + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + + if self.reader.in_bracketed_paste: + r.insert("\n") + return + + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _should_auto_indent(r.buffer, r.pos): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = None + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = Console(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + reader.ps1 = str(prompt) + return reader.readline(startup_hook=self.startup_hook) + + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" + with warnings.catch_warnings(action="ignore"): + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + # Ignore readline history file header + if line.startswith("_HiStOrY_V2_"): + continue + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = "".join(buffer).replace("\r", "") + line + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> bytes: + buf_str = self.get_reader().get_unicode() + return buf_str.encode(ENCODING) + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if str(buf[i]) in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(namespace: dict[str, Any]) -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # set up namespace in rlcompleter + _wrapper.config.readline_completer = RLCompleter(namespace).complete + + # this is not really what readline.c does. Better than nothing I guess + import builtins + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000000..bc16c1f6a23159 --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,161 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _sitebuiltins +import linecache +import builtins +import sys +import code +from types import ModuleType + +from .console import InteractiveColoredConsole +from .readline import _get_reader, multiline_input + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + return str(e) or repr(e) or "unknown error" + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": "help", + "clear": _clear_screen, +} + + +def run_multiline_interactive_console( + namespace: dict[str, Any], + future_flags: int = 0, + console: code.InteractiveConsole | None = None, +) -> None: + from .readline import _setup + _setup(namespace) + + if console is None: + console = InteractiveColoredConsole( + namespace, filename="<stdin>" + ) + if future_flags: + console.compile.compiler.flags |= future_flags + + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + command() + return True + + if isinstance(command, str): + # Internal readline commands require a prepared reader like + # inside multiline_input. + reader.prepare() + reader.refresh() + reader.do_cmd((command, [statement])) + reader.restore() + return True + + return False + + def more_lines(unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "<stdin>", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"<python-input-{input_n}>" + linecache._register_code(input_name, statement, "<stdin>") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + console.write("KeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000000..a8eb2433cd3cce --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000000..f9d48b828c720b --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000000..18b2bba91c8c9b --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,785 @@ +# Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com> +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import termios +import time +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +TYPE_CHECKING = False + +# types +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + super().__init__(f_in, f_out, term, encoding) + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 + curses.setupterm(term or None, self.output_fd) + self.term = term + + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def __read(self, n: int) -> bytes: + if not self.input_buffer or self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.__posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.__posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.__posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen.copy() + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.__posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + if not block and not self.wait(timeout=0): + return None + while self.event_queue.empty(): + while True: + try: + self.push_char(self.__read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() + + def wait(self, timeout: float | None = None) -> bool: + """ + Wait for events on the console. + """ + return bool(self.pollob.poll(timeout)) + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + height, width = struct.unpack( + "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + )[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.__posxy = 0, 0 + self.screen = [] + + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x_coord > self.__posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.__posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x: int) -> None: + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x: int) -> None: + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x: int) -> None: + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000000..70cfade26e23b1 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com> +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000000..20dbb1f7e17229 --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,24 @@ +import re +import unicodedata +import functools + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +@functools.cache +def str_width(c: str) -> int: + if ord(c) < 128: + return 1 + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + if len(s) == 1: + return str_width(s) + length = sum(str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + return length - sum(len(i) for i in sequence) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 00000000000000..9e97b1524e29a0 --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,602 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +import os +import sys +import time +import msvcrt + +from collections import deque +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x79: "f17", # VK_F17 + 0x80: "f18", # VK_F18 + 0x81: "f19", # VK_F19 + 0x82: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.__posxy = self.__posxy[0], self.__posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.__posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.__posxy = 0, y + 1 + else: + self.__posxy = wlen(newline), y + + if "\x1b" in newline or y != self.__posxy[1]: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current __posxy""" + dx = x - self.__posxy[0] + dy = y - self.__posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.__posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self) -> INPUT_RECORD | None: + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + if read.value == 0: + return None + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input() + if rec is None: + if block: + continue + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key = rec.Event.KeyEvent.uChar.UnicodeChar + + if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": + # Make enter make unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + # Turn backspace directly into the command + return Event( + evt="key", + data="backspace", + raw=rec.Event.KeyEvent.uChar.UnicodeChar, + ) + elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) + if code: + return Event( + evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar + ) + if block: + continue + + return None + + return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.__posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + while self._read_input() is not None: + pass + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self, timeout: float | None) -> bool: + """Wait for an event.""" + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout: + return False + time.sleep(0.01) + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 77ccdc9e1d789b..e42af75af74bf5 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,6 +10,7 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar @@ -250,12 +251,30 @@ def pattern(self, format): format = regex_chars.sub(r"\\\1", format) whitespace_replacement = re_compile(r'\s+') format = whitespace_replacement.sub(r'\\s+', format) + year_in_format = False + day_of_month_in_format = False while '%' in format: directive_index = format.index('%')+1 + format_char = format[directive_index] processed_format = "%s%s%s" % (processed_format, format[:directive_index-1], - self[format[directive_index]]) + self[format_char]) format = format[directive_index+1:] + match format_char: + case 'Y' | 'y' | 'G': + year_in_format = True + case 'd': + day_of_month_in_format = True + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) return "%s%s" % (processed_format, format) def compile(self, format): @@ -342,8 +361,6 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): tz = -1 gmtoff = None gmtoff_fraction = 0 - # Default to -1 to signify that values not known; not critical to have, - # though iso_week = week_of_year = None week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate @@ -470,17 +487,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # Deal with the cases where ambiguities arise # don't assume default values for ISO week/year - if year is None and iso_year is not None: - if iso_week is None or weekday is None: - raise ValueError("ISO year directive '%G' must be used with " - "the ISO week directive '%V' and a weekday " - "directive ('%A', '%a', '%w', or '%u').") + if iso_year is not None: if julian is not None: raise ValueError("Day of the year directive '%j' is not " "compatible with ISO year directive '%G'. " "Use '%Y' instead.") - elif week_of_year is None and iso_week is not None: - if weekday is None: + elif iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + elif iso_week is not None: + if year is None or weekday is None: raise ValueError("ISO week directive '%V' must be used with " "the ISO year directive '%G' and a weekday " "directive ('%A', '%a', '%w', or '%u').") @@ -490,11 +507,12 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): "instead.") leap_year_fix = False - if year is None and month == 2 and day == 29: - year = 1904 # 1904 is first leap year of 20th century - leap_year_fix = True - elif year is None: - year = 1900 + if year is None: + if month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + else: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. diff --git a/Lib/argparse.py b/Lib/argparse.py index a32884db80d1ea..83189115bf85f7 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -223,7 +223,8 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) else: heading = '' @@ -327,17 +328,8 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(usage) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = self._get_actions_usage_parts(optionals, groups) + pos_parts = self._get_actions_usage_parts(positionals, groups) # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -390,6 +382,9 @@ def get_lines(parts, indent, prefix=None): return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): + return ' '.join(self._get_actions_usage_parts(actions, groups)) + + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -397,56 +392,26 @@ def _format_actions_usage(self, actions, groups): if not group._group_actions: raise ValueError(f'empty group {group}') + if all(action.help is SUPPRESS for action in group._group_actions): + continue + try: start = actions.index(group._group_actions[0]) except ValueError: continue else: - group_action_count = len(group._group_actions) - end = start + group_action_count + end = start + len(group._group_actions) if actions[start:end] == group._group_actions: - - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 - - exposed_actions_count = group_action_count - suppressed_actions_count - - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + group_actions.update(group._group_actions) + inserts[start, end] = group # collect all actions format strings parts = [] - for i, action in enumerate(actions): + for action in actions: # suppressed arguments are marked with None - # remove | separators for suppressed arguments if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) + part = None # produce all arg strings elif not action.option_strings: @@ -458,9 +423,6 @@ def _format_actions_usage(self, actions, groups): if part[0] == '[' and part[-1] == ']': part = part[1:-1] - # add the action string to the list - parts.append(part) - # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] @@ -481,26 +443,23 @@ def _format_actions_usage(self, actions, groups): if not action.required and action not in group_actions: part = '[%s]' % part - # add the action string to the list - parts.append(part) + # add the action string to the list + parts.append(part) - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() + # group mutually exclusive actions + for start, end in sorted(inserts, reverse=True): + group = inserts[start, end] + group_parts = [item for item in parts[start:end] if item is not None] + if group.required: + open, close = "()" if len(group_parts) > 1 else ("", "") + else: + open, close = "[]" + parts[start] = open + " | ".join(group_parts) + close + for i in range(start + 1, end): + parts[i] = None - # return the text - return text + # return the usage parts + return [item for item in parts if item is not None] def _format_text(self, text): if '%(prog)' in text: @@ -564,22 +523,18 @@ def _format_action_invocation(self, action): return metavar else: - parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - parts.extend(action.option_strings) + return ', '.join(action.option_strings) # if the Optional takes a value, format is: - # -s ARGS, --long ARGS + # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) + return ', '.join(action.option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -702,14 +657,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): """ def _get_help_string(self, action): - """ - Add the default value to the option help message. - - ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't - already present. This code will do that, detecting cornercases to - prevent duplicates or cases where it wouldn't make sense to the end - user. - """ help = action.help if help is None: help = '' @@ -718,7 +665,7 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + help += _(' (default: %(default)s)') return help @@ -847,7 +794,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): self.option_strings = option_strings self.dest = dest self.nargs = nargs @@ -858,6 +806,7 @@ def __init__(self, self.required = required self.help = help self.metavar = metavar + self.deprecated = deprecated def _get_kwargs(self): names = [ @@ -871,6 +820,7 @@ def _get_kwargs(self): 'required', 'help', 'metavar', + 'deprecated', ] return [(name, getattr(self, name)) for name in names] @@ -881,19 +831,14 @@ def __call__(self, parser, namespace, values, option_string=None): raise NotImplementedError(_('.__call__() not defined')) -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() - class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default): + deprecated=False): _option_strings = [] for option_string in option_strings: @@ -903,35 +848,14 @@ def __init__(self, option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar) + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): @@ -954,7 +878,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for store actions must be != 0; if you ' 'have nothing to store, actions such as store ' @@ -971,7 +896,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) @@ -986,7 +912,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -994,7 +921,8 @@ def __init__(self, const=const, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) @@ -1007,14 +935,16 @@ def __init__(self, dest, default=False, required=False, - help=None): + help=None, + deprecated=False): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, - default=default, + deprecated=deprecated, required=required, - help=help) + help=help, + default=default) class _StoreFalseAction(_StoreConstAction): @@ -1024,14 +954,16 @@ def __init__(self, dest, default=True, required=False, - help=None): + help=None, + deprecated=False): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) class _AppendAction(Action): @@ -1046,7 +978,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for append actions must be != 0; if arg ' 'strings are not supplying the value to append, ' @@ -1063,7 +996,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1081,7 +1015,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1090,7 +1025,8 @@ def __init__(self, default=default, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1106,14 +1042,16 @@ def __init__(self, dest, default=None, required=False, - help=None): + help=None, + deprecated=False): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) @@ -1128,13 +1066,15 @@ def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, - help=None): + help=None, + deprecated=False): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() @@ -1148,7 +1088,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit"): + help=None, + deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1192,6 +1135,7 @@ def __init__(self, self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] + self._deprecated = set() super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1202,7 +1146,7 @@ def __init__(self, help=help, metavar=metavar) - def add_parser(self, name, **kwargs): + def add_parser(self, name, *, deprecated=False, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) @@ -1230,6 +1174,10 @@ def add_parser(self, name, **kwargs): for alias in aliases: self._name_parser_map[alias] = parser + if deprecated: + self._deprecated.add(name) + self._deprecated.update(aliases) + return parser def _get_subactions(self): @@ -1245,13 +1193,17 @@ def __call__(self, parser, namespace, values, option_string=None): # select the parser try: - parser = self._name_parser_map[parser_name] + subparser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) + if parser_name in self._deprecated: + parser._warning(_("command '%(parser_name)s' is deprecated") % + {'parser_name': parser_name}) + # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them @@ -1259,7 +1211,7 @@ def __call__(self, parser, namespace, values, option_string=None): # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) @@ -1839,7 +1791,7 @@ def _get_kwargs(self): # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ArgumentError(None, _('cannot have multiple subparser arguments')) # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1891,8 +1843,11 @@ def _get_positional_actions(self): def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): @@ -1979,6 +1934,7 @@ def _parse_known_args(self, arg_strings, namespace): # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() + warned = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) @@ -2005,7 +1961,7 @@ def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + action, option_string, sep, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2032,18 +1988,27 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != '' ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: @@ -2074,6 +2039,10 @@ def consume_optional(start_index): # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: + if action.deprecated and option_string not in warned: + self._warning(_("option '%(option)s' is deprecated") % + {'option': option_string}) + warned.add(option_string) take_action(action, args, option_string) return stop @@ -2093,6 +2062,10 @@ def consume_positionals(start_index): for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count + if args and action.deprecated and action.dest not in warned: + self._warning(_("argument '%(argument_name)s' is deprecated") % + {'argument_name': action.dest}) + warned.add(action.dest) take_action(action, args) # slice off the Positionals that we just parsed and return the @@ -2111,10 +2084,11 @@ def consume_positionals(start_index): while start_index <= max_option_string_index: # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) + next_option_string_index = start_index + while next_option_string_index <= max_option_string_index: + if next_option_string_index in option_string_indices: + break + next_option_string_index += 1 if start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) @@ -2162,7 +2136,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2178,7 +2152,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2205,7 +2179,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2263,18 +2237,17 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None + return action, arg_string, None, None # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, sep, explicit_arg # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations @@ -2283,10 +2256,10 @@ def _parse_optional(self, arg_string): # if multiple actions match, the option string was ambiguous if len(option_tuples) > 1: options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) + for action, option_string, sep, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) + raise ArgumentError(None, msg % args) # if exactly one action matched, this segmentation is good, # so return the parsed action @@ -2307,7 +2280,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None + return None, arg_string, None, None def _get_option_tuples(self, option_string): result = [] @@ -2317,15 +2290,13 @@ def _get_option_tuples(self, option_string): chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if self.allow_abbrev: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # single character options can be concatenated with their arguments @@ -2333,23 +2304,22 @@ def _get_option_tuples(self, option_string): # separate elif option_string[0] in chars and option_string[1] not in chars: option_prefix = option_string - explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg + tup = action, option_string, '', short_explicit_arg result.append(tup) elif option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, None, None result.append(tup) # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2406,8 +2376,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): @@ -2489,7 +2462,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: + if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: try: arg_strings.remove('--') except ValueError: @@ -2654,3 +2627,7 @@ def error(self, message): self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + + def _warning(self, message): + args = {'prog': self.prog, 'message': message} + self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) diff --git a/Lib/ast.py b/Lib/ast.py index f7888d18859ae4..a954d4a97d3c22 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,7 +25,6 @@ :license: Python License. """ import sys -import re from _ast import * from contextlib import contextmanager, nullcontext from enum import IntEnum, auto, _simple_enum @@ -114,7 +113,11 @@ def _convert(node): return _convert(node_or_string) -def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): +def dump( + node, annotate_fields=True, include_attributes=False, + *, + indent=None, show_empty=False, +): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), @@ -125,6 +128,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): include_attributes can be set to true. If indent is a non-negative integer or string, then the tree will be pretty-printed with that indent level. None (the default) selects the single line representation. + If show_empty is False, then empty lists and fields that are None + will be omitted from the output for better readability. """ def _format(node, level=0): if indent is not None: @@ -137,6 +142,7 @@ def _format(node, level=0): if isinstance(node, AST): cls = type(node) args = [] + args_buffer = [] allsimple = True keywords = annotate_fields for name in node._fields: @@ -148,6 +154,18 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue + if ( + not show_empty + and (value is None or value == []) + # Special cases: + # `Constant(value=None)` and `MatchSingleton(value=None)` + and not isinstance(node, (Constant, MatchSingleton)) + ): + args_buffer.append(repr(value)) + continue + elif not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: @@ -306,12 +324,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -382,6 +406,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -418,27 +524,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -498,151 +583,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -728,12 +668,11 @@ class _Unparser(NodeVisitor): output source code for the abstract syntax; original formatting is disregarded.""" - def __init__(self, *, _avoid_backslashes=False): + def __init__(self): self._source = [] self._precedences = {} self._type_ignores = {} self._indent = 0 - self._avoid_backslashes = _avoid_backslashes self._in_try_star = False def interleave(self, inter, f, seq): @@ -1106,12 +1045,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") @@ -1270,14 +1218,18 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node, scape_newlines=False): + def _write_fstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value, scape_newlines=scape_newlines) + self._write_fstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") - if scape_newlines: + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): @@ -1301,10 +1253,7 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner( - node.format_spec, - scape_newlines=True - ) + self._write_fstring_inner(node.format_spec, is_format_spec=True) def visit_Name(self, node): self.write(node.id) @@ -1324,8 +1273,6 @@ def _write_constant(self, value): .replace("inf", _INFSTR) .replace("nan", f"({_INFSTR}-{_INFSTR})") ) - elif self._avoid_backslashes and isinstance(value, str): - self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) @@ -1787,33 +1734,17 @@ def visit_MatchOr(self, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - def main(): import argparse parser = argparse.ArgumentParser(prog='python -m ast') - parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?', - default='-', + parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), @@ -1827,9 +1758,14 @@ def main(): help='indentation of nodes (number of spaces)') args = parser.parse_args() - with args.infile as infile: - source = infile.read() - tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments) + if args.infile == '-': + name = '<stdin>' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + tree = parse(source, name, args.mode, type_comments=args.no_type_comments) print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) if __name__ == '__main__': diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 18bb87a5bc4ffd..95147171c2b79d 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,41 +1,49 @@ import ast import asyncio -import code import concurrent.futures import inspect +import os +import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="<stdin>") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -56,10 +64,12 @@ def callback(): try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -68,18 +78,57 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize(): + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + try: + import errno + if os.getenv("PYTHON_BASIC_REPL"): + raise RuntimeError("user environment requested basic REPL") + if not os.isatty(sys.stdin.fileno()): + return_code = errno.ENOTTY + raise OSError(return_code, "tty required", "stdin") + + # This import will fail on operating systems with no termios. + from _pyrepl.simple_interact import ( + check, + run_multiline_interactive_console, + ) + if err := check(): + raise RuntimeError(err) + except Exception as e: + console.interact(banner="", exitmsg="") + else: + try: + run_multiline_interactive_console(console=console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 finally: warnings.filterwarnings( 'ignore', @@ -90,6 +139,9 @@ def run(self): if __name__ == '__main__': + CAN_USE_PYREPL = True + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -102,14 +154,30 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA except ImportError: - pass + readline = None + + interactive_hook = getattr(sys, "__interactivehook__", None) - repl_thread = REPLThread() + if interactive_hook is not None: + interactive_hook() + + if interactive_hook is site.register_readline: + # Fix the completer function to use the interactive console locals + try: + import rlcompleter + except: + pass + else: + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) + + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -117,9 +185,12 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 0476de631a6a52..f0e690b61a73dd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -16,6 +16,7 @@ import collections import collections.abc import concurrent.futures +import errno import functools import heapq import itertools @@ -44,6 +45,7 @@ from . import sslproto from . import staggered from . import tasks +from . import timeouts from . import transports from . import trsock from .log import logger @@ -277,7 +279,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -290,14 +294,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -305,7 +308,7 @@ def _wakeup(self): self._waiters = None for waiter in waiters: if not waiter.done(): - waiter.set_result(waiter) + waiter.set_result(None) def _start_serving(self): if self._serving: @@ -346,9 +349,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' @@ -377,7 +388,27 @@ async def serve_forever(self): self._serving_forever_fut = None async def wait_closed(self): - if self._waiters is None or self._active_count == 0: + """Wait until server is closed and all connections are dropped. + + - If the server is not closed, wait. + - If it is closed, but there are still active connections, wait. + + Anyone waiting here will be unblocked once both conditions + (server is closed and all connections have been dropped) + have become true, in either order. + + Historical note: In 3.11 and before, this was broken, returning + immediately if the server was already closed, even if there + were still active connections. An attempted fix in 3.12.0 was + still broken, returning immediately if the server was still + open and there were no active connections. Hopefully in 3.12.1 + we have it right. + """ + # Waiters are unblocked by self._wakeup(), which is called + # from two places: self.close() and self._detach(), but only + # when both conditions have become true. To signal that this + # has happened, self._wakeup() sets self._waiters to None. + if self._waiters is None: return waiter = self._loop.create_future() self._waiters.append(waiter) @@ -577,23 +608,24 @@ async def shutdown_default_executor(self, timeout=None): thread = threading.Thread(target=self._do_shutdown, args=(future,)) thread.start() try: - await future - finally: - thread.join(timeout) - - if thread.is_alive(): + async with timeouts.timeout(timeout): + await future + except TimeoutError: warnings.warn("The executor did not finishing joining " - f"its threads within {timeout} seconds.", - RuntimeWarning, stacklevel=2) + f"its threads within {timeout} seconds.", + RuntimeWarning, stacklevel=2) self._default_executor.shutdown(wait=False) + else: + thread.join() def _do_shutdown(self, future): try: self._default_executor.shutdown(wait=True) if not self.is_closed(): - self.call_soon_threadsafe(future.set_result, None) + self.call_soon_threadsafe(futures._set_result_unless_cancelled, + future, None) except Exception as ex: - if not self.is_closed(): + if not self.is_closed() and not future.cancelled(): self.call_soon_threadsafe(future.set_exception, ex) def _check_running(self): @@ -1319,9 +1351,9 @@ async def create_datagram_endpoint(self, protocol_factory, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if sock.type != socket.SOCK_DGRAM: + if sock.type == socket.SOCK_STREAM: raise ValueError( - f'A UDP Socket was expected, got {sock!r}') + f'A datagram socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or reuse_port or allow_broadcast): @@ -1476,6 +1508,7 @@ async def create_server( ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -1549,6 +1582,9 @@ async def create_server( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) if reuse_port: _set_reuseport(sock) + if keep_alive: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. @@ -1561,9 +1597,22 @@ async def create_server( try: sock.bind(sa) except OSError as err: - raise OSError(err.errno, 'error while attempting ' - 'to bind on address %r: %s' - % (sa, err.strerror.lower())) from None + msg = ('error while attempting ' + 'to bind on address %r: %s' + % (sa, err.strerror.lower())) + if err.errno == errno.EADDRNOTAVAIL: + # Assume the family is not enabled (bpo-30945) + sockets.pop() + sock.close() + if self._debug: + logger.warning(msg) + continue + raise OSError(err.errno, msg) from None + + if not sockets: + raise OSError('could not bind on any address out of %r' + % ([info[4] for info in infos],)) + completed = True finally: if not completed: diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 4c9b0dd5653c0c..9c2ba679ce2bf1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -1,6 +1,9 @@ import collections import subprocess import warnings +import os +import signal +import sys from . import protocols from . import transports @@ -115,7 +118,8 @@ def close(self): try: self._proc.kill() - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # the process may have already exited or may be running setuid pass # Don't clear the _proc reference yet: _post_init() may still run @@ -141,17 +145,31 @@ def _check_proc(self): if self._proc is None: raise ProcessLookupError() - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) + if sys.platform == 'win32': + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + else: + def send_signal(self, signal): + self._check_proc() + try: + os.kill(self._proc.pid, signal) + except ProcessLookupError: + pass - def terminate(self): - self._check_proc() - self._proc.terminate() + def terminate(self): + self.send_signal(signal.SIGTERM) - def kill(self): - self._check_proc() - self._proc.kill() + def kill(self): + self.send_signal(signal.SIGKILL) async def _connect_pipes(self, waiter): try: diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index f0ce0433a7a8a6..b60c1e4236af1f 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import enum # After the connection is lost, log warnings after this many write()s. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0ccf85105e6673..b63fe6aa79604b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -1,12 +1,15 @@ """Event loop and event loop policy.""" +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + __all__ = ( 'AbstractEventLoopPolicy', 'AbstractEventLoop', 'AbstractServer', 'Handle', 'TimerHandle', 'get_event_loop_policy', 'set_event_loop_policy', 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', '_set_running_loop', 'get_running_loop', '_get_running_loop', ) @@ -50,7 +53,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -86,7 +90,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, @@ -169,6 +174,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError @@ -316,6 +329,7 @@ async def create_server( *, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -354,6 +368,9 @@ async def create_server( they all set this flag when being created. This option is not supported on Windows. + keep_alive set to True keeps connections active by enabling the + periodic transmission of messages. + ssl_handshake_timeout is the time in seconds that an SSL server will wait for completion of the SSL handshake before aborting the connection. Default is 60s. @@ -634,17 +651,6 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. @@ -819,17 +825,6 @@ def new_event_loop(): return get_event_loop_policy().new_event_loop() -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) - - # Alias pure-Python implementations for testing purposes. _py__get_running_loop = _get_running_loop _py__set_running_loop = _set_running_loop diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9553..93737b7708a67b 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 97fc4e3fcb60ee..5f6fa2348726cf 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -43,7 +43,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,7 +60,7 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between # `await Future()` or`yield from Future()` (correct) vs. # `yield Future()` (incorrect). @@ -138,9 +137,6 @@ def _make_cancelled_error(self): exc = exceptions.CancelledError() else: exc = exceptions.CancelledError(self._cancel_message) - exc.__context__ = self._cancelled_exc - # Remove the reference since we don't need this anymore. - self._cancelled_exc = None return exc def cancel(self, msg=None): @@ -272,9 +268,13 @@ def set_exception(self, exception): raise exceptions.InvalidStateError(f'{self._state}: {self!r}') if isinstance(exception, type): exception = exception() - if type(exception) is StopIteration: - raise TypeError("StopIteration interacts badly with generators " - "and cannot be raised into a Future") + if isinstance(exception, StopIteration): + new_exc = RuntimeError("StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future") + new_exc.__cause__ = exception + new_exc.__context__ = exception + exception = new_exc self._exception = exception self._exception_tb = exception.__traceback__ self._state = _FINISHED @@ -318,11 +318,9 @@ def _set_result_unless_cancelled(fut, result): def _convert_future_exc(exc): exc_class = type(exc) if exc_class is concurrent.futures.CancelledError: - return exceptions.CancelledError(*exc.args) - elif exc_class is concurrent.futures.TimeoutError: - return exceptions.TimeoutError(*exc.args) + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) elif exc_class is concurrent.futures.InvalidStateError: - return exceptions.InvalidStateError(*exc.args) + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) else: return exc diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index ce5d8d5bfb2e81..aaee8ff0702923 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'await'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. Locks also support the asynchronous context management protocol. 'async with lock' statement should be used. @@ -95,6 +93,8 @@ async def acquire(self): This method blocks until the lock is unlocked, then sets it to locked and returns True. """ + # Implement fair scheduling, where thread always waits + # its turn. Jumping the queue if all are cancelled is an optimization. if (not self._locked and (self._waiters is None or all(w.cancelled() for w in self._waiters))): self._locked = True @@ -105,19 +105,22 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: + # Currently the only exception designed be able to occur here. + + # Ensure the lock invariant: If lock is not claimed (or about + # to be claimed by us) and there is a Task in waiters, + # ensure that the Task at the head will run. if not self._locked: self._wake_up_first() raise + # assert self._locked is False self._locked = True return True @@ -125,7 +128,7 @@ def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -139,7 +142,7 @@ def release(self): raise RuntimeError('Lock is not acquired.') def _wake_up_first(self): - """Wake up the first waiter if it isn't done.""" + """Ensure that the first waiter will wake up.""" if not self._waiters: return try: @@ -147,9 +150,7 @@ def _wake_up_first(self): except StopIteration: return - # .done() necessarily means that a waiter will wake up later on and - # either take the lock, or, if it was cancelled and lock wasn't - # taken already, will hit this again and wake up a new waiter. + # .done() means that the waiter is already set to wake up. if not fut.done(): fut.set_result(True) @@ -179,8 +180,8 @@ def is_set(self): return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -191,7 +192,7 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False @@ -200,7 +201,7 @@ async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: @@ -219,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ @@ -247,45 +248,64 @@ def __repr__(self): async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._get_loop().create_future() - self._waiters.append(fut) try: - await fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must reacquire lock even if wait is cancelled - cancelled = False - while True: + self._waiters.append(fut) try: - await self.acquire() - break - except exceptions.CancelledError: - cancelled = True + await fut + return True + finally: + self._waiters.remove(fut) - if cancelled: - raise exceptions.CancelledError + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() @@ -295,20 +315,22 @@ async def wait_for(self, predicate): return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -357,6 +379,7 @@ def __repr__(self): def locked(self): """Returns True if semaphore cannot be acquired immediately.""" + # Due to state, or FIFO rules (must allow others to run first). return self._value == 0 or ( any(not w.cancelled() for w in (self._waiters or ()))) @@ -365,11 +388,12 @@ async def acquire(self): If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ if not self.locked(): + # Maintain FIFO, wait for others to start even if _value > 0. self._value -= 1 return True @@ -378,29 +402,34 @@ async def acquire(self): fut = self._get_loop().create_future() self._waiters.append(fut) - # Finally block should be called before the CancelledError - # handling as we don't want CancelledError to call - # _wake_up_first() and attempt to wake up itself. try: try: await fut finally: self._waiters.remove(fut) except exceptions.CancelledError: - if not fut.cancelled(): + # Currently the only exception designed be able to occur here. + if fut.done() and not fut.cancelled(): + # Our Future was successfully set to True via _wake_up_next(), + # but we are not about to successfully acquire(). Therefore we + # must undo the bookkeeping already done and attempt to wake + # up someone else. self._value += 1 - self._wake_up_next() raise - if self._value > 0: - self._wake_up_next() + finally: + # New waiters may have arrived but had to wait due to FIFO. + # Wake up as many as are allowed. + while self._value > 0: + if not self._wake_up_next(): + break # There was no-one to wake up. return True def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() @@ -408,13 +437,15 @@ def release(self): def _wake_up_next(self): """Wake up the first waiter that isn't done.""" if not self._waiters: - return + return False for fut in self._waiters: if not fut.done(): self._value -= 1 fut.set_result(True) - return + # `fut` is now `done()` and not `cancelled()`. + return True + return False class BoundedSemaphore(Semaphore): diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 1e2a730cf368a9..7eb55bd63ddb73 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True @@ -487,9 +487,6 @@ def sendto(self, data, addr=None): raise TypeError('data argument must be bytes-like object (%r)', type(data)) - if not data: - return - if self._address is not None and addr not in (None, self._address): raise ValueError( f'Invalid address: must be None or {self._address}') @@ -502,7 +499,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes if self._write_fut is None: # No current write operations are active, kick one off @@ -724,6 +721,8 @@ async def sock_sendto(self, sock, data, address): return await self._proactor.sendto(sock, data, 0, address) async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") return await self._proactor.connect(sock, address) async def sock_accept(self, sock): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba..2f3865114a84f9 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,11 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +53,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -81,6 +94,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -112,8 +127,12 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raises QueueShutDown if the queue has been shut down. """ while self.full(): + if self._is_shutdown: + raise QueueShutDown putter = self._get_loop().create_future() self._putters.append(putter) try: @@ -125,7 +144,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take @@ -138,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -150,8 +173,13 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): + if self._is_shutdown and self.empty(): + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -163,7 +191,7 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take @@ -176,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -194,6 +227,9 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises ValueError if called more times than there were items placed in the queue. """ @@ -214,6 +250,34 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index d521b4e2e255a9..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -235,6 +235,10 @@ async def _accept_connection2( await waiter except BaseException: transport.close() + # gh-109534: When an exception is raised by the SSLProtocol object the + # exception set in this future can keep the protocol object alive and + # cause a reference cycle. + waiter = None raise # It's now up to the protocol to handle the connection. @@ -261,15 +265,11 @@ def _ensure_fd_no_transport(self, fd): except (AttributeError, TypeError, ValueError): # This code matches selectors._fileobj_to_fd function. raise ValueError(f"Invalid file object: {fd!r}") from None - try: - transport = self._transports[fileno] - except KeyError: - pass - else: - if not transport.is_closing(): - raise RuntimeError( - f'File descriptor {fd!r} is used by transport ' - f'{transport!r}') + transport = self._transports.get(fileno) + if transport and not transport.is_closing(): + raise RuntimeError( + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): @@ -1241,8 +1243,6 @@ def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError(f'data argument must be a bytes-like object, ' f'not {type(data).__name__!r}') - if not data: - return if self._address: if addr not in (None, self._address): @@ -1278,7 +1278,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes self._maybe_pause_protocol() def _sendto_ready(self): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 3eb65a8a08b5a0..fa99d4533aa0a6 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import collections import enum import warnings @@ -243,13 +247,12 @@ def abort(self): The protocol's connection_lost() method will (eventually) be called with None as its argument. """ - self._closed = True - if self._ssl_protocol is not None: - self._ssl_protocol._abort() + self._force_close(None) def _force_close(self, exc): self._closed = True - self._ssl_protocol._abort(exc) + if self._ssl_protocol is not None: + self._ssl_protocol._abort(exc) def _test__append_write_backlog(self, data): # for test only @@ -576,6 +579,7 @@ def _on_handshake_complete(self, handshake_exc): peercert = sslobj.getpeercert() except Exception as exc: + handshake_exc = None self._set_state(SSLProtocolState.UNWRAPPED) if isinstance(exc, ssl.CertificateError): msg = 'SSL handshake failed on verifying the certificate' @@ -614,7 +618,7 @@ def _start_shutdown(self): if self._app_transport is not None: self._app_transport._closed = True if self._state == SSLProtocolState.DO_HANDSHAKE: - self._abort() + self._abort(None) else: self._set_state(SSLProtocolState.FLUSHING) self._shutdown_timeout_handle = self._loop.call_later( @@ -661,10 +665,10 @@ def _on_shutdown_complete(self, shutdown_exc): else: self._loop.call_soon(self._transport.close) - def _abort(self): + def _abort(self, exc): self._set_state(SSLProtocolState.UNWRAPPED) if self._transport is not None: - self._transport.abort() + self._transport._force_close(exc) # Outgoing flow diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 451a53a16f3831..c3a7441a7b091d 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -3,7 +3,6 @@ __all__ = 'staggered_race', import contextlib -import typing from . import events from . import exceptions as exceptions_mod @@ -11,16 +10,7 @@ from . import tasks -async def staggered_race( - coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]], - delay: typing.Optional[float], - *, - loop: events.AbstractEventLoop = None, -) -> typing.Tuple[ - typing.Any, - typing.Optional[int], - typing.List[typing.Optional[Exception]] -]: +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is @@ -79,8 +69,7 @@ async def staggered_race( exceptions = [] running_tasks = [] - async def run_one_coro( - previous_failed: typing.Optional[locks.Event]) -> None: + async def run_one_coro(previous_failed) -> None: # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bc84e53b8443cf..64aac4cc50d15a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,13 +236,25 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): + def callback(task): + if task.cancelled(): + transport.close() + return + exc = task.exception() + if exc is not None: + self._loop.call_exception_handler({ + 'message': 'Unhandled exception in client_connected_cb', + 'exception': exc, + 'transport': transport, + }) + transport.close() + self._task = self._loop.create_task(res) + self._task.add_done_callback(callback) + self._strong_reader = None def connection_lost(self, exc): @@ -390,13 +399,15 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) def __del__(self, warnings=warnings): if not self._transport.is_closing(): - self.close() - warnings.warn(f"unclosed {self!r}", ResourceWarning) - + if self._loop.is_closed(): + warnings.warn("loop is closed", ResourceWarning) + else: + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) class StreamReader: @@ -579,20 +590,34 @@ async def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, tuple): + # Makes sure shortest matches wins + separator = sorted(separator, key=len) + else: + separator = [separator] + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -605,26 +630,35 @@ async def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', @@ -633,7 +667,7 @@ async def readuntil(self, separator=b'\n'): # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() @@ -642,12 +676,12 @@ async def readuntil(self, separator=b'\n'): # _wait_for_data() will resume reading if stream was paused. await self._wait_for_data('readuntil') - if isep > self._limit: + if match_start > self._limit: raise exceptions.LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 24238c4f5f998d..f2ee9648c43876 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -54,16 +54,14 @@ def __repr__(self): async def __aenter__(self): if self._entered: raise RuntimeError( - f"TaskGroup {self!r} has been already entered") - self._entered = True - + f"TaskGroup {self!r} has already been entered") if self._loop is None: self._loop = events.get_running_loop() - self._parent_task = tasks.current_task(self._loop) if self._parent_task is None: raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') + self._entered = True return self @@ -75,14 +73,10 @@ async def __aexit__(self, et, exc, tb): self._base_error is None): self._base_error = exc - propagate_cancellation_error = \ - exc if et is exceptions.CancelledError else None - if self._parent_cancel_requested: - # If this flag is set we *must* call uncancel(). - if self._parent_task.uncancel() == 0: - # If there are no pending cancellations left, - # don't propagate CancelledError. - propagate_cancellation_error = None + if et is not None and issubclass(et, exceptions.CancelledError): + propagate_cancellation_error = exc + else: + propagate_cancellation_error = None if et is not None: if not self._aborting: @@ -130,15 +124,28 @@ async def __aexit__(self, et, exc, tb): if self._base_error is not None: raise self._base_error + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None + # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error and not self._errors: + if propagate_cancellation_error is not None and not self._errors: raise propagate_cancellation_error - if et is not None and et is not exceptions.CancelledError: + if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) if self._errors: + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() # Exceptions are heavy objects that can have object # cycles (bad for GC); let's not keep a reference to # a bunch of them. @@ -154,16 +161,19 @@ def create_task(self, coro, *, name=None, context=None): Similar to `asyncio.create_task`. """ if not self._entered: + coro.close() raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is finished") if self._aborting: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: - task = self._loop.create_task(coro) + task = self._loop.create_task(coro, name=name) else: - task = self._loop.create_task(coro, context=context) - task.set_name(name) + task = self._loop.create_task(coro, name=name, context=context) + # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 72f4cc07173f0a..2f8f4f08a1e111 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,6 +25,7 @@ from . import events from . import exceptions from . import futures +from . import queues from . import timeouts # Helper to generate new task names @@ -254,6 +255,8 @@ def uncancel(self): """ if self._num_cancels_requested > 0: self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False return self._num_cancels_requested def __eager_start(self): @@ -278,7 +281,7 @@ def __eager_start(self): def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() @@ -376,7 +379,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -404,11 +407,10 @@ def create_task(coro, *, name=None, context=None): loop = events.get_running_loop() if context is None: # Use legacy API if context is not needed - task = loop.create_task(coro) + task = loop.create_task(coro, name=name) else: - task = loop.create_task(coro, context=context) + task = loop.create_task(coro, name=name, context=context) - task.set_name(name) return task @@ -424,8 +426,6 @@ async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): The fs iterable must not be empty. - Coroutines will be wrapped in Tasks. - Returns two sets of Future: (done, pending). Usage: @@ -467,7 +467,7 @@ async def wait_for(fut, timeout): If the wait is cancelled, the task is also cancelled. - If the task supresses the cancellation and returns a value instead, + If the task suppresses the cancellation and returns a value instead, that value is returned. This function is a coroutine. @@ -567,62 +567,125 @@ async def _cancel_and_wait(fut): fut.remove_done_callback(cb) -# This is *not* a @coroutine! It is just an iterator (yielding Futures). +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. + + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. + """ + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None + + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} + for f in todo: + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f + + def as_completed(fs, *, timeout=None): - """Return an iterator whose values are coroutines. + """Create an iterator of awaitables or their results in completion order. - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. - This differs from PEP 3148; the proper way to use this is: + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: - for f in as_completed(fs): - result = await f # The 'await' may raise. - # Use result. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - If a timeout is specified, the 'await' will raise - TimeoutError when the timeout occurs before all Futures are done. + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect - Note: The futures 'f' are not necessarily members of fs. - """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") - from .queues import Queue # Import here to avoid circular import problem. - done = Queue() + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - loop = events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - timeout_handle = None + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect - async def _wait_for_one(): - f = await done.get() - if f is None: - # Dummy value from _on_timeout(). - raise exceptions.TimeoutError - return f.result() # May raise f.exception(). + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + return _AsCompletedIterator(fs, timeout) @types.coroutine @@ -1034,14 +1097,14 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1053,3 +1116,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 029c468739bf2d..e6f5100691d362 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -49,8 +49,9 @@ def when(self) -> Optional[float]: def reschedule(self, when: Optional[float]) -> None: """Reschedule the timeout.""" - assert self._state is not _State.CREATED if self._state is not _State.ENTERED: + if self._state is _State.CREATED: + raise RuntimeError("Timeout has not been entered") raise RuntimeError( f"Cannot change state of {self._state.value} Timeout", ) @@ -82,11 +83,14 @@ def __repr__(self) -> str: return f"<Timeout [{self._state.value}]{info_str}>" async def __aenter__(self) -> "Timeout": + if self._state is not _State.CREATED: + raise RuntimeError("Timeout has already been entered") + task = tasks.current_task() + if task is None: + raise RuntimeError("Timeout should be used inside a task") self._state = _State.ENTERED - self._task = tasks.current_task() + self._task = task self._cancelling = self._task.cancelling() - if self._task is None: - raise RuntimeError("Timeout should be used inside a task") self.reschedule(self._when) return self @@ -105,10 +109,16 @@ async def __aexit__( if self._state is _State.EXPIRING: self._state = _State.EXPIRED - if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError: + if self._task.uncancel() <= self._cancelling and exc_type is not None: # Since there are no new cancel requests, we're # handling this. - raise TimeoutError from exc_val + if issubclass(exc_type, exceptions.CancelledError): + raise TimeoutError from exc_val + elif exc_val is not None: + self._insert_timeout_error(exc_val) + if isinstance(exc_val, ExceptionGroup): + for exc in exc_val.exceptions: + self._insert_timeout_error(exc) elif self._state is _State.ENTERED: self._state = _State.EXITED @@ -121,6 +131,16 @@ def _on_timeout(self) -> None: # drop the reference early self._timeout_handler = None + @staticmethod + def _insert_timeout_error(exc_val: BaseException) -> None: + while exc_val.__context__ is not None: + if isinstance(exc_val.__context__, exceptions.CancelledError): + te = TimeoutError() + te.__context__ = te.__cause__ = exc_val.__context__ + exc_val.__context__ = te + break + exc_val = exc_val.__context__ + def timeout(delay: Optional[float]) -> Timeout: """Timeout async context manager. diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 30fd41d49af71f..34c7ad44ffd8ab 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -181,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 809f29eecba198..2796e397c0e84d 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,9 +28,6 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', 'DefaultEventLoopPolicy', 'EventLoop', ) @@ -64,6 +61,11 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} + self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -196,33 +198,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -284,7 +275,7 @@ async def create_unix_server( sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - start_serving=True): + start_serving=True, cleanup_socket=True): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') @@ -340,6 +331,15 @@ async def create_unix_server( raise ValueError( f'A UNIX Domain Stream Socket was expected, got {sock!r}') + if cleanup_socket: + path = sock.getsockname() + # Check for abstract socket. `str` and `bytes` paths are supported. + if path[0] not in (0, '\x00'): + try: + self._unix_server_sockets[sock] = os.stat(path).st_ino + except FileNotFoundError: + pass + sock.setblocking(False) server = base_events.Server(self, [sock], protocol_factory, ssl, backlog, ssl_handshake_timeout, @@ -460,6 +460,27 @@ def cb(fut): self.remove_writer(fd) fut.add_done_callback(cb) + def _stop_serving(self, sock): + # Is this a unix socket that needs cleanup? + if sock in self._unix_server_sockets: + path = sock.getsockname() + else: + path = None + + super()._stop_serving(sock) + + if path is not None: + prev_ino = self._unix_server_sockets[sock] + del self._unix_server_sockets[sock] + try: + if os.stat(path).st_ino == prev_ino: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as err: + logger.error('Unable to clean up listening UNIX socket ' + '%r: %r', path, err) + class _UnixReadPipeTransport(transports.ReadTransport): @@ -834,93 +855,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -932,21 +867,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -971,386 +891,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1367,25 +908,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - self._join_threads() - - def _join_threads(self): - """Internal: Join all non-daemon threads""" - threads = [thread for thread in list(self._threads.values()) - if thread.is_alive() and not thread.daemon] - for thread in threads: - thread.join() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1397,21 +919,12 @@ def __del__(self, _warn=warnings.warn): def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() thread = threading.Thread(target=self._do_waitpid, - name=f"waitpid-{next(self._pid_counter)}", + name=f"asyncio-waitpid-{next(self._pid_counter)}", args=(loop, pid, callback, args), daemon=True) self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1451,61 +964,9 @@ def can_use_pidfd(): class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index b62ea75fee3858..bf99bc271c7acd 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -8,6 +8,7 @@ import _overlapped import _winapi import errno +from functools import partial import math import msvcrt import socket @@ -324,13 +325,13 @@ def _run_forever_cleanup(self): if self._self_reading_future is not None: ov = self._self_reading_future._ov self._self_reading_future.cancel() - # self_reading_future was just cancelled so if it hasn't been - # finished yet, it never will be (it's possible that it has - # already finished and its callback is waiting in the queue, - # where it could still happen if the event loop is restarted). - # Unregister it otherwise IocpProactor.close will wait for it - # forever - if ov is not None: + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: self._proactor._unregister(ov) self._self_reading_future = None @@ -467,6 +468,18 @@ def finish_socket_func(trans, key, ov): else: raise + @classmethod + def _finish_recvfrom(cls, trans, key, ov, *, empty_result): + try: + return cls.finish_socket_func(trans, key, ov) + except OSError as exc: + # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same + # socket is used to send to an address that is not listening. + if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE: + return empty_result, None + else: + raise + def recv(self, conn, nbytes, flags=0): self._register_with_iocp(conn) ov = _overlapped.Overlapped(NULL) @@ -501,7 +514,8 @@ def recvfrom(self, conn, nbytes, flags=0): except BrokenPipeError: return self._result((b'', None)) - return self._register(ov, conn, self.finish_socket_func) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=b'')) def recvfrom_into(self, conn, buf, flags=0): self._register_with_iocp(conn) @@ -511,17 +525,8 @@ def recvfrom_into(self, conn, buf, flags=0): except BrokenPipeError: return self._result((0, None)) - def finish_recv(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, - _overlapped.ERROR_OPERATION_ABORTED): - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_recv) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=0)) def sendto(self, conn, buf, flags=0, addr=None): self._register_with_iocp(conn) diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index e233647ee76639..61be4fb856e92c --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module @@ -18,7 +16,7 @@ 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', + 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -164,7 +162,6 @@ def urlsafe_b64decode(s): _b32rev = {} def _b32encode(alphabet, s): - global _b32tab2 # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32tab2: @@ -200,7 +197,6 @@ def _b32encode(alphabet, s): return bytes(encoded) def _b32decode(alphabet, s, casefold=False, map01=None): - global _b32rev # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32rev: @@ -334,7 +330,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. @@ -499,6 +495,33 @@ def b85decode(b): result = result[:-padding] return result +_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') +# Translating b85 valid but z85 invalid chars to b'\x00' is required +# to prevent them from being decoded as b85 valid chars. +_z85_b85_decode_diff = b';_`|~' +_z85_decode_translation = bytes.maketrans( + _z85alphabet + _z85_b85_decode_diff, + _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) +) +_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + +def z85encode(s): + """Encode bytes-like object b in z85 format and return a bytes object.""" + return b85encode(s).translate(_z85_encode_translation) + +def z85decode(s): + """Decode the z85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_z85_decode_translation) + try: + return b85decode(s) + except ValueError as e: + raise ValueError(e.args[0].replace('base85', 'z85')) from None + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. diff --git a/Lib/bdb.py b/Lib/bdb.py index 0f3eec653baaad..d7543017940d4f 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -32,7 +32,10 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None self._load_breaks() @@ -84,6 +87,9 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ + + self.enterframe = frame + if self.quitting: return # None if event == 'line': @@ -100,6 +106,8 @@ def trace_dispatch(self, frame, event, arg): return self.trace_dispatch if event == 'c_return': return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch @@ -157,6 +165,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -186,6 +199,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -272,7 +296,21 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -285,6 +323,16 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -299,16 +347,12 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) + def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) @@ -325,14 +369,19 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + sys.settrace(None) if frame is None: frame = sys._getframe().f_back self.reset() + self.enterframe = frame while frame: frame.f_trace = self.trace_dispatch self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True frame = frame.f_back - self.set_step() + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -349,6 +398,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8d808..2420cd019069b4 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -17,7 +17,7 @@ from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py old mode 100755 new mode 100644 index 135a12c3965c00..e7c868b8d55543 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Python interface for the 'lsprof' profiler. Compatible with the 'profile' module. """ @@ -41,7 +39,9 @@ class Profile(_lsprof.Profiler): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): import marshal diff --git a/Lib/calendar.py b/Lib/calendar.py index 03469d8ac96bcd..069dd5174112ae 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -159,8 +159,8 @@ def weekday(year, month, day): def monthrange(year, month): - """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for - year, month.""" + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" if not 1 <= month <= 12: raise IllegalMonthError(month) day1 = weekday(year, month, 1) @@ -734,10 +734,15 @@ def main(args=None): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -761,6 +766,7 @@ def main(args=None): cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() @@ -775,6 +781,7 @@ def main(args=None): cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing diff --git a/Lib/cmd.py b/Lib/cmd.py index 88ee7d3ddc4694..c333e099bd8c9a 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,26 +23,26 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ -import string, sys +import inspect, string, sys __all__ = ["Cmd"] @@ -108,7 +108,15 @@ def cmdloop(self, intro=None): import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") + if readline.backend == "editline": + if self.completekey == 'tab': + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = f"bind {self.completekey} rl_complete" + else: + command_string = f"{self.completekey}: complete" + readline.parse_and_bind(command_string) except ImportError: pass try: @@ -210,9 +218,8 @@ def onecmd(self, line): if cmd == '': return self.default(line) else: - try: - func = getattr(self, 'do_' + cmd) - except AttributeError: + func = getattr(self, 'do_' + cmd, None) + if func is None: return self.default(line) return func(arg) @@ -298,6 +305,7 @@ def do_help(self, arg): except AttributeError: try: doc=getattr(self, 'do_' + arg).__doc__ + doc = inspect.cleandoc(doc) if doc: self.stdout.write("%s\n"%str(doc)) return diff --git a/Lib/code.py b/Lib/code.py index 2bd5fa3e795a61..a55fced0704b1d 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -24,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -93,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -105,6 +106,7 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) type, value, tb = sys.exc_info() sys.last_exc = value sys.last_type = type @@ -122,14 +124,14 @@ def showsyntaxerror(self, filename=None): value = SyntaxError(msg, (filename, lineno, offset, line)) sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) + lines = traceback.format_exception_only(type, value, colorize=colorize) self.write(''.join(lines)) else: # If someone has set sys.excepthook, we let that take precedence # over self.write sys.excepthook(type, value, tb) - def showtraceback(self): + def showtraceback(self, **kwargs): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -137,11 +139,12 @@ def showtraceback(self): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() sys.last_traceback = last_tb sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) + lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next, colorize=colorize) if sys.excepthook is sys.__excepthook__: self.write(''.join(lines)) else: @@ -169,7 +172,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="<console>"): + def __init__(self, locals=None, filename="<console>", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -181,6 +184,7 @@ def __init__(self, locals=None, filename="<console>"): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -219,29 +223,66 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) - - def push(self, line): + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) + + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -257,7 +298,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -276,8 +319,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -290,14 +347,15 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) diff --git a/Lib/codecs.py b/Lib/codecs.py index 9b35b6127dd01c..a887e5d4c94a38 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -1129,4 +1129,4 @@ def make_encoding_map(decoding_map): # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index a574aa4b70f1a8..a0276b52d484e3 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -65,15 +65,20 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None + except _IncompleteInputError as e: + return None except SyntaxError as e: - if "incomplete input" in str(e): - return None + pass # fallthrough - return compiler(source, filename, symbol) + return compiler(source, filename, symbol, incomplete_input=False) -def _compile(source, filename, symbol): - return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT) +def _compile(source, filename, symbol, incomplete_input=True): + flags = 0 + if incomplete_input: + flags |= PyCF_ALLOW_INCOMPLETE_INPUT + flags |= PyCF_DONT_IMPLY_DEDENT + return compile(source, filename, symbol, flags) def compile_command(source, filename="<input>", symbol="single"): r"""Compile a command and determine whether it is incomplete. @@ -104,8 +109,12 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol): - codeob = compile(source, filename, symbol, self.flags, True) + def __call__(self, source, filename, symbol, **kwargs): + flags = self.flags + if kwargs.get('incomplete_input', True) is False: + flags &= ~PyCF_DONT_IMPLY_DEDENT + flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT + codeob = compile(source, filename, symbol, flags, True) for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index a461550ea40da7..b47e728484c8ac 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -46,7 +46,8 @@ _collections_abc.MutableSequence.register(deque) try: - from _collections import _deque_iterator + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 except ImportError: pass @@ -457,7 +458,7 @@ def _make(cls, iterable): def _replace(self, /, **kwds): result = self._make(_map(kwds.pop, field_names, self)) if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') return result _replace.__doc__ = (f'Return a new {typename} object replacing specified ' @@ -639,7 +640,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) @@ -680,7 +682,7 @@ def update(self, iterable=None, /, **kwds): ''' # The regular dict.update() operation makes no sense here because the - # replace behavior results in the some of original untouched counts + # replace behavior results in some of the original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs @@ -1015,7 +1017,7 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible @@ -1027,7 +1029,10 @@ def __iter__(self): return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) @@ -1037,9 +1042,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 86ca8b8a8414b3..bff76291634604 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias +from _collections_abc import __all__ # noqa: F401 +from _collections_abc import _CallableGenericAlias # noqa: F401 diff --git a/Lib/compileall.py b/Lib/compileall.py index d394156cedc4e7..47e2446356e7d7 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -116,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, prependdir=prependdir, limit_sl_dest=limit_sl_dest, hardlink_dupes=hardlink_dupes), - files) + files, + chunksize=4) success = min(results, default=True) else: for file in files: @@ -172,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 292e886d5a88ac..72de617a5b6f61 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,6 +23,7 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 6742a07753c921..707fcdfde79acd 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -23,14 +23,6 @@ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED' FINISHED = 'FINISHED' -_FUTURE_STATES = [ - PENDING, - RUNNING, - CANCELLED, - CANCELLED_AND_NOTIFIED, - FINISHED -] - _STATE_TO_DESCRIPTION_MAP = { PENDING: "pending", RUNNING: "running", diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ffaffdb8b3d0aa..7092b4757b5429 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -190,16 +190,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -306,8 +296,9 @@ def __init__(self, executor): # if there is no pending work item. def weakref_cb(_, thread_wakeup=self.thread_wakeup, - shutdown_lock=self.shutdown_lock): - mp.util.debug('Executor collected: triggering callback for' + shutdown_lock=self.shutdown_lock, + mp_util_debug=mp.util.debug): + mp_util_debug('Executor collected: triggering callback for' ' QueueManager wakeup') with shutdown_lock: thread_wakeup.wakeup() @@ -598,7 +589,7 @@ def _check_system_limits(): raise NotImplementedError(_system_limited) _system_limits_checked = True try: - import multiprocessing.synchronize + import multiprocessing.synchronize # noqa: F401 except ImportError: _system_limited = ( "This Python build lacks multiprocessing.synchronize, usually due " @@ -847,7 +838,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results) diff --git a/Lib/configparser.py b/Lib/configparser.py index 71362d23ec3757..4344a9e8baca44 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=<unset>, converters=<unset>): - + interpolation=<unset>, converters=<unset>, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,23 +143,28 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys +import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", + "MultilineContinuationError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -297,15 +306,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -322,6 +349,28 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + +class _UnnamedSection: + + def __repr__(self): + return "<UNNAMED_SECTION>" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -490,6 +539,52 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] + + def __init__(self): + self.elements_added = set() + self.errors = list() + + +class _Line(str): + + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) + + def __init__(self, val, prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + + class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" @@ -536,7 +631,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -555,8 +651,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = types.SimpleNamespace( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -575,6 +673,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -848,13 +947,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified 'fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -940,110 +1045,117 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + st.sectname = UNNAMED_SECTION + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo, fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, mo, fpname): + st.sectname = mo.group('header') + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1063,12 +1175,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 6994690ebf7eb2..5b646fabca0225 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -461,7 +461,7 @@ def __exit__(self, exctype, excinst, exctb): return if issubclass(exctype, self._exceptions): return True - if issubclass(exctype, ExceptionGroup): + if issubclass(exctype, BaseExceptionGroup): match, rest = excinst.split(self._exceptions) if rest is None: return True diff --git a/Lib/copy.py b/Lib/copy.py index a69bc4e78c20b3..a79976d3a658f0 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -4,8 +4,9 @@ import copy - x = copy.copy(y) # make a shallow copy of y - x = copy.deepcopy(y) # make a deep copy of y + x = copy.copy(y) # make a shallow copy of y + x = copy.deepcopy(y) # make a deep copy of y + x = copy.replace(y, a=1, b=2) # new object with fields replaced, as defined by `__replace__` For module specific errors, copy.Error is raised. @@ -56,7 +57,7 @@ class Error(Exception): pass error = Error # backward compatibility -__all__ = ["Error", "copy", "deepcopy"] +__all__ = ["Error", "copy", "deepcopy", "replace"] def copy(x): """Shallow copy operation on arbitrary Python objects. @@ -121,6 +122,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -129,14 +135,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -167,26 +171,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Lib/csv.py b/Lib/csv.py index 77f30c8d2b1f61..75e35b23236795 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,28 +1,90 @@ -""" -csv.py - read/write/investigate CSV files +r""" +CSV parsing and writing. + +This module provides classes that assist in the reading and writing +of Comma Separated Value (CSV) files, and implements the interface +described by PEP 305. Although many CSV files are simple to parse, +the format is not formally defined by a stable specification and +is subtle enough that parsing lines of a CSV file with something +like line.split(",") is bound to fail. The module supports three +basic APIs: reading, writing, and registration of dialects. + + +DIALECT REGISTRATION: + +Readers and writers support a dialect argument, which is a convenient +handle on a group of settings. When the dialect argument is a string, +it identifies one of the dialects previously registered with the module. +If it is a class or instance, the attributes of the argument are used as +the settings for the reader or writer: + + class excel: + delimiter = ',' + quotechar = '"' + escapechar = None + doublequote = True + skipinitialspace = False + lineterminator = '\r\n' + quoting = QUOTE_MINIMAL + +SETTINGS: + + * quotechar - specifies a one-character string to use as the + quoting character. It defaults to '"'. + * delimiter - specifies a one-character string to use as the + field separator. It defaults to ','. + * skipinitialspace - specifies how to interpret spaces which + immediately follow a delimiter. It defaults to False, which + means that spaces immediately following a delimiter is part + of the following field. + * lineterminator - specifies the character sequence which should + terminate rows. + * quoting - controls when quotes should be generated by the writer. + It can take on any of the following module constants: + + csv.QUOTE_MINIMAL means only when required, for example, when a + field contains either the quotechar or the delimiter + csv.QUOTE_ALL means that quotes are always placed around fields. + csv.QUOTE_NONNUMERIC means that quotes are always placed around + fields which do not parse as integers or floating point + numbers. + csv.QUOTE_STRINGS means that quotes are always placed around + fields which are strings. Note that the Python value None + is not a string. + csv.QUOTE_NOTNULL means that quotes are only placed around fields + that are not the Python value None. + csv.QUOTE_NONE means that quotes are never placed around fields. + * escapechar - specifies a one-character string used to escape + the delimiter when quoting is set to QUOTE_NONE. + * doublequote - controls the handling of quotes inside fields. When + True, two consecutive quotes are interpreted as one during read, + and when writing, each quote character embedded in the data is + written as two quotes """ import re import types -from _csv import Error, __version__, writer, reader, register_dialect, \ +from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ - QUOTE_STRINGS, QUOTE_NOTNULL, \ - __doc__ + QUOTE_STRINGS, QUOTE_NOTNULL from _csv import Dialect as _Dialect from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "QUOTE_STRINGS", "QUOTE_NOTNULL", - "Error", "Dialect", "__doc__", "excel", "excel_tab", + "Error", "Dialect", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unregister_dialect", "DictReader", "DictWriter", "unix_dialect"] +__version__ = "1.0" + + class Dialect: """Describe a CSV dialect. @@ -51,8 +113,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 141142a57dcb3e..721522caeeac92 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -205,6 +205,16 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "C" + class c_float_complex(_SimpleCData): + _type_ = "E" + class c_longdouble_complex(_SimpleCData): + _type_ = "F" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -348,6 +358,17 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: @@ -468,6 +489,8 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) +elif _sys.platform == "android": + pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) else: @@ -521,9 +544,9 @@ def cast(obj, typ): _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): - """string_at(addr[, size]) -> string + """string_at(ptr[, size]) -> string - Return the string at addr.""" + Return the byte string at void *ptr.""" return _string_at(ptr, size) try: @@ -533,9 +556,9 @@ def string_at(ptr, size=-1): else: _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string + """wstring_at(ptr[, size]) -> string - Return the string at addr.""" + Return the wide-character string at void *ptr.""" return _wstring_at(ptr, size) diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index 3febb3118b8230..6382dd22b8acc8 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -15,8 +15,8 @@ def _other_endian(typ): # if typ is array if isinstance(typ, _array_type): return _other_endian(typ._type_) * typ._length_ - # if typ is structure - if issubclass(typ, Structure): + # if typ is structure or union + if issubclass(typ, (Structure, Union)): return typ raise TypeError("This type does not support other endian: %s" % typ) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 0c2510e1619c8e..117bf06cb01013 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -89,6 +89,15 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -96,8 +105,11 @@ def find_library(name): def _is_elf(filename): "Return True if the given file is an ELF file" elf_header = b'\x7fELF' - with open(filename, 'br') as thefile: - return thefile.read(4) == elf_header + try: + with open(filename, 'br') as thefile: + return thefile.read(4) == elf_header + except FileNotFoundError: + return False def _findLib_gcc(name): # Run GCC's linker with the -t (aka --trace) option and examine the diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 69270bfcd2b205..6165fe6c9875c0 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -53,7 +53,7 @@ def start_color(): try: has_key except NameError: - from .has_key import has_key + from .has_key import has_key # noqa: F401 # Wrapper for the entire curses-based application. Runs a function which # should be the rest of your curses-based application. If the application diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 2fba32b5ffbc1e..74011b7e28b9f3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,11 +4,9 @@ import types import inspect import keyword -import functools import itertools import abc -import _thread -from types import FunctionType, GenericAlias +from reprlib import recursive_repr __all__ = ['dataclass', @@ -245,25 +243,6 @@ def __repr__(self): property, }) -# This function's logic is copied from "recursive_repr" function in -# reprlib module to avoid dependency. -def _recursive_repr(user_function): - # Decorator to make a repr function return "..." for a recursive - # call. - repr_running = set() - - @functools.wraps(user_function) - def wrapper(self): - key = id(self), _thread.get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - return wrapper class InitVar: __slots__ = ('type', ) @@ -322,7 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, self.kw_only = kw_only self._field_type = None - @_recursive_repr + @recursive_repr() def __repr__(self): return ('Field(' f'name={self.name!r},' @@ -353,7 +332,7 @@ def __set_name__(self, owner, name): # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(types.GenericAlias) class _DataclassParams: @@ -446,32 +425,95 @@ def _tuple_str(obj_name, fields): return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' -def _create_fn(name, args, body, *, globals=None, locals=None, - return_type=MISSING): - # Note that we may mutate locals. Callers beware! - # The only callers are internal to this module, so no - # worries about external callers. - if locals is None: - locals = {} - return_annotation = '' - if return_type is not MISSING: - locals['__dataclass_return_type__'] = return_type - return_annotation = '->__dataclass_return_type__' - args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) - - # Compute the text of the entire function. - txt = f' def {name}({args}){return_annotation}:\n{body}' - - # Free variables in exec are resolved in the global namespace. - # The global namespace we have is user-provided, so we can't modify it for - # our purposes. So we put the things we need into locals and introduce a - # scope to allow the function we're creating to close over them. - local_vars = ', '.join(locals.keys()) - txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} - exec(txt, globals, ns) - return ns['__create_fn__'](**locals) +class _FuncBuilder: + def __init__(self, globals): + self.names = [] + self.src = [] + self.globals = globals + self.locals = {} + self.overwrite_errors = {} + self.unconditional_adds = {} + + def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, + overwrite_error=False, unconditional_add=False, decorator=None): + if locals is not None: + self.locals.update(locals) + + # Keep track if this method is allowed to be overwritten if it already + # exists in the class. The error is method-specific, so keep it with + # the name. We'll use this when we generate all of the functions in + # the add_fns_to_class call. overwrite_error is either True, in which + # case we'll raise an error, or it's a string, in which case we'll + # raise an error and append this string. + if overwrite_error: + self.overwrite_errors[name] = overwrite_error + + # Should this function always overwrite anything that's already in the + # class? The default is to not overwrite a function that already + # exists. + if unconditional_add: + self.unconditional_adds[name] = True + + self.names.append(name) + + if return_type is not MISSING: + self.locals[f'__dataclass_{name}_return_type__'] = return_type + return_annotation = f'->__dataclass_{name}_return_type__' + else: + return_annotation = '' + args = ','.join(args) + body = '\n'.join(body) + + # Compute the text of the entire function, add it to the text we're generating. + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + + def add_fns_to_class(self, cls): + # The source to all of the functions we're generating. + fns_src = '\n'.join(self.src) + + # The locals they use. + local_vars = ','.join(self.locals.keys()) + + # The names of all of the functions, used for the return value of the + # outer function. Need to handle the 0-tuple specially. + if len(self.names) == 0: + return_names = '()' + else: + return_names =f'({",".join(self.names)},)' + + # txt is the entire function we're going to execute, including the + # bodies of the functions we're defining. Here's a greatly simplified + # version: + # def __create_fn__(): + # def __init__(self, x, y): + # self.x = x + # self.y = y + # @recursive_repr + # def __repr__(self): + # return f"cls(x={self.x!r},y={self.y!r})" + # return __init__,__repr__ + + txt = f"def __create_fn__({local_vars}):\n{fns_src}\n return {return_names}" + ns = {} + exec(txt, self.globals, ns) + fns = ns['__create_fn__'](**self.locals) + + # Now that we've generated the functions, assign them into cls. + for name, fn in zip(self.names, fns): + fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + + # See if it's an error to overwrite this particular function. + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + + raise TypeError(error_msg) def _field_assign(frozen, name, value, self_name): @@ -482,8 +524,8 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' - return f'{self_name}.{name}={value}' + return f' __dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' + return f' {self_name}.{name}={value}' def _field_init(f, frozen, globals, self_name, slots): @@ -566,7 +608,7 @@ def _init_param(f): def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, - self_name, globals, slots): + self_name, func_builder, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -585,11 +627,11 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} - locals.update({ - '__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - }) + locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, + **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, + } + } body_lines = [] for f in fields: @@ -603,11 +645,11 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') + body_lines.append(f' {self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: - body_lines = ['pass'] + body_lines = [' pass'] _init_params = [_init_param(f) for f in std_fields] if kw_only_fields: @@ -616,68 +658,34 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # (instead of just concatenting the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] - return _create_fn('__init__', - [self_name] + _init_params, - body_lines, - locals=locals, - globals=globals, - return_type=None) - - -def _repr_fn(fields, globals): - fn = _create_fn('__repr__', - ('self',), - ['return f"{self.__class__.__qualname__}(' + - ', '.join([f"{f.name}={{self.{f.name}!r}}" - for f in fields]) + - ')"'], - globals=globals) - return _recursive_repr(fn) - - -def _frozen_get_del_attr(cls, fields, globals): + func_builder.add_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + return_type=None) + + +def _frozen_get_del_attr(cls, fields, func_builder): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} condition = 'type(self) is cls' if fields: condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' - return (_create_fn('__setattr__', - ('self', 'name', 'value'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', - f'super(cls, self).__setattr__(name, value)'), - locals=locals, - globals=globals), - _create_fn('__delattr__', - ('self', 'name'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot delete field {name!r}")', - f'super(cls, self).__delattr__(name)'), - locals=locals, - globals=globals), - ) - - -def _cmp_fn(name, op, self_tuple, other_tuple, globals): - # Create a comparison function. If the fields in the object are - # named 'x' and 'y', then self_tuple is the string - # '(self.x,self.y)' and other_tuple is the string - # '(other.x,other.y)'. - - return _create_fn(name, - ('self', 'other'), - [ 'if other.__class__ is self.__class__:', - f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented'], - globals=globals) - -def _hash_fn(fields, globals): - self_tuple = _tuple_str('self', fields) - return _create_fn('__hash__', - ('self',), - [f'return hash({self_tuple})'], - globals=globals) + func_builder.add_fn('__setattr__', + ('self', 'name', 'value'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f' super(cls, self).__setattr__(name, value)'), + locals=locals, + overwrite_error=True) + func_builder.add_fn('__delattr__', + ('self', 'name'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f' super(cls, self).__delattr__(name)'), + locals=locals, + overwrite_error=True) def _is_classvar(a_type, typing): @@ -854,19 +862,11 @@ def _get_field(cls, a_name, a_type, default_kw_only): return f -def _set_qualname(cls, value): - # Ensure that the functions returned from _create_fn uses the proper - # __qualname__ (the class they belong to). - if isinstance(value, FunctionType): - value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" - return value - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True - _set_qualname(cls, value) setattr(cls, name, value) return False @@ -876,14 +876,22 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields, globals): - return None +def _hash_set_none(cls, fields, func_builder): + # It's sort of a hack that I'm setting this here, instead of at + # func_builder.add_fns_to_class time, but since this is an exceptional case + # (it's not setting an attribute to a function, but to a scalar value), + # just do it directly here. I might come to regret this. + cls.__hash__ = None -def _hash_add(cls, fields, globals): +def _hash_add(cls, fields, func_builder): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _set_qualname(cls, _hash_fn(flds, globals)) + self_tuple = _tuple_str('self', flds) + func_builder.add_fn('__hash__', + ('self',), + [f' return hash({self_tuple})'], + unconditional_add=True) -def _hash_exception(cls, fields, globals): +def _hash_exception(cls, fields, func_builder): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -1061,24 +1069,26 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, (std_init_fields, kw_only_init_fields) = _fields_in_init_order(all_init_fields) + func_builder = _FuncBuilder(globals) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - _set_new_attribute(cls, '__init__', - _init_fn(all_init_fields, - std_init_fields, - kw_only_init_fields, - frozen, - has_post_init, - # The name to use for the "self" - # param in __init__. Use "self" - # if possible. - '__dataclass_self__' if 'self' in fields - else 'self', - globals, - slots, - )) + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + func_builder, + slots, + ) + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is @@ -1087,7 +1097,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) + func_builder.add_fn('__repr__', + ('self',), + [' return f"{self.__class__.__qualname__}(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in flds]) + ')"'], + locals={'__dataclasses_recursive_repr': recursive_repr}, + decorator="@__dataclasses_recursive_repr()") if eq: # Create __eq__ method. There's no need for a __ne__ method, @@ -1095,14 +1111,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cmp_fields = (field for field in field_list if field.compare) terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] field_comparisons = ' and '.join(terms) or 'True' - body = [f'if other.__class__ is self.__class__:', - f' return {field_comparisons}', - f'return NotImplemented'] - func = _create_fn('__eq__', - ('self', 'other'), - body, - globals=globals) - _set_new_attribute(cls, '__eq__', func) + func_builder.add_fn('__eq__', + ('self', 'other'), + [ ' if self is other:', + ' return True', + ' if other.__class__ is self.__class__:', + f' return {field_comparisons}', + ' return NotImplemented']) if order: # Create and set the ordering methods. @@ -1114,18 +1129,19 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, ('__gt__', '>'), ('__ge__', '>='), ]: - if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple, - globals=globals)): - raise TypeError(f'Cannot overwrite attribute {name} ' - f'in class {cls.__name__}. Consider using ' - 'functools.total_ordering') + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list, globals): - if _set_new_attribute(cls, fn.__name__, fn): - raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' - f'in class {cls.__name__}') + _frozen_get_del_attr(cls, field_list, func_builder) # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), @@ -1133,9 +1149,12 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, bool(frozen), has_explicit_hash] if hash_action: - # No need to call _set_new_attribute here, since by the time - # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list, globals) + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): # Create a class doc-string. @@ -1148,7 +1167,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cls.__doc__ = (cls.__name__ + text_sig) if match_args: - # I could probably compute this once + # I could probably compute this once. _set_new_attribute(cls, '__match_args__', tuple(f.name for f in std_init_fields)) @@ -1179,8 +1198,17 @@ def _dataclass_setstate(self, state): def _get_slots(cls): match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation case None: - return + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictrefoffset__', -1) != 0: + slots.append('__dict__') + yield from slots case str(slot): yield slot # Slots may be any iterable, but we cannot handle an iterator @@ -1332,58 +1360,69 @@ class C: def _asdict_inner(obj, dict_factory): - if type(obj) in _ATOMIC_TYPES: + obj_type = type(obj) + if obj_type in _ATOMIC_TYPES: return obj - elif _is_dataclass_instance(obj): - # fast path for the common case + elif hasattr(obj_type, _FIELDS): + # dataclass instance: fast path for the common case if dict_factory is dict: return { f.name: _asdict_inner(getattr(obj, f.name), dict) for f in fields(obj) } else: - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) - elif isinstance(obj, tuple) and hasattr(obj, '_fields'): - # obj is a namedtuple. Recurse into it, but the returned - # object is another namedtuple of the same type. This is - # similar to how other list- or tuple-derived classes are - # treated (see below), but we just need to create them - # differently because a namedtuple's __init__ needs to be - # called differently (see bpo-34363). - - # I'm not using namedtuple's _asdict() - # method, because: - # - it does not recurse in to the namedtuple fields and - # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The main - # use case here is json.dumps, and it handles converting - # namedtuples to lists. Admittedly we're losing some - # information here when we produce a json list instead of a - # dict. Note that if we returned dicts here instead of - # namedtuples, we could no longer call asdict() on a data - # structure where a namedtuple was used as a dict key. - - return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) - elif isinstance(obj, (list, tuple)): - # Assume we can create an object of this type by passing in a - # generator (which is not true for namedtuples, handled - # above). - return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict): - if hasattr(type(obj), 'default_factory'): + return dict_factory([ + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in fields(obj) + ]) + # handle the builtin types first for speed; subclasses handled below + elif obj_type is list: + return [_asdict_inner(v, dict_factory) for v in obj] + elif obj_type is dict: + return { + _asdict_inner(k, dict_factory): _asdict_inner(v, dict_factory) + for k, v in obj.items() + } + elif obj_type is tuple: + return tuple([_asdict_inner(v, dict_factory) for v in obj]) + elif issubclass(obj_type, tuple): + if hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) + else: + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + elif issubclass(obj_type, dict): + if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from # dict as it requires the default_factory as its first arg. - result = type(obj)(getattr(obj, 'default_factory')) + result = obj_type(obj.default_factory) for k, v in obj.items(): result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) return result - return type(obj)((_asdict_inner(k, dict_factory), - _asdict_inner(v, dict_factory)) - for k, v in obj.items()) + return obj_type((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + elif issubclass(obj_type, list): + # Assume we can create an object of this type by passing in a + # generator + return obj_type(_asdict_inner(v, dict_factory) for v in obj) else: return copy.deepcopy(obj) @@ -1416,11 +1455,10 @@ def _astuple_inner(obj, tuple_factory): if type(obj) in _ATOMIC_TYPES: return obj elif _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _astuple_inner(getattr(obj, f.name), tuple_factory) - result.append(value) - return tuple_factory(result) + return tuple_factory([ + _astuple_inner(getattr(obj, f.name), tuple_factory) + for f in fields(obj) + ]) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is @@ -1558,14 +1596,14 @@ class C: return _replace(obj, **changes) -def _replace(obj, /, **changes): +def _replace(self, /, **changes): # We're going to mutate 'changes', but that's okay because it's a - # new dict, even if called with 'replace(obj, **my_changes)'. + # new dict, even if called with 'replace(self, **my_changes)'. # It's an error to have init=False fields in 'changes'. - # If a field is not in 'changes', read its value from the provided obj. + # If a field is not in 'changes', read its value from the provided 'self'. - for f in getattr(obj, _FIELDS).values(): + for f in getattr(self, _FIELDS).values(): # Only consider normal fields or InitVars. if f._field_type is _FIELD_CLASSVAR: continue @@ -1582,11 +1620,11 @@ def _replace(obj, /, **changes): if f._field_type is _FIELD_INITVAR and f.default is MISSING: raise TypeError(f"InitVar {f.name!r} " f'must be specified with replace()') - changes[f.name] = getattr(obj, f.name) + changes[f.name] = getattr(self, f.name) # Create the new object, which calls __init__() and # __post_init__() (if defined), using all of the init fields we've # added and/or left in 'changes'. If there are values supplied in # changes that aren't fields, this will correctly raise a # TypeError. - return obj.__class__(**changes) + return self.__class__(**changes) diff --git a/Lib/datetime.py b/Lib/datetime.py index a33d2d724cb33d..b4f7bd045c7b68 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,9 +1,9 @@ try: from _datetime import * - from _datetime import __doc__ + from _datetime import __doc__ # noqa: F401 except ImportError: from _pydatetime import * - from _pydatetime import __doc__ + from _pydatetime import __doc__ # noqa: F401 __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index 8055d3769f9dd0..4fdbc54e74cfb6 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -164,6 +164,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 754624ccc8f500..def120ffc3778b 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -98,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -134,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 00000000000000..7e0ae2a29e3a64 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,140 @@ +import os +import sqlite3 +from pathlib import Path +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE IF NOT EXISTS Dict ( + key BLOB UNIQUE NOT NULL, + value BLOB NOT NULL + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" +ITER_KEYS = "SELECT key FROM Dict" + + +class error(OSError): + pass + + +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + +def _normalize_uri(path): + path = Path(path) + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri + + +class _Database(MutableMapping): + + def __init__(self, path, /, *, flag, mode): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + + path = os.fsdecode(path) + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) + case "n": + flag = "rwc" + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") + + # We use the URI format when opening the database. + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" + + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) + + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") + + if flag == "rwc": + self._execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) + try: + return closing(self._cx.execute(*args, **kwargs)) + except sqlite3.Error as exc: + raise error(str(exc)) + + def __len__(self): + with self._execute(GET_SIZE) as cu: + row = cu.fetchone() + return row[0] + + def __getitem__(self, key): + with self._execute(LOOKUP_KEY, (key,)) as cu: + row = cu.fetchone() + if not row: + raise KeyError(key) + return row[0] + + def __setitem__(self, key, value): + self._execute(STORE_KV, (key, value)) + + def __delitem__(self, key): + with self._execute(DELETE_KEY, (key,)) as cu: + if not cu.rowcount: + raise KeyError(key) + + def __iter__(self): + try: + with self._execute(ITER_KEYS) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + if self._cx: + self._cx.close() + self._cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def open(filename, /, flag="r", mode=0o666): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. + """ + return _Database(filename, flag=flag, mode=mode) diff --git a/Lib/decimal.py b/Lib/decimal.py index 7746ea2601024c..13a0dcb77f1267 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,11 +1,108 @@ +"""Decimal fixed point and floating point arithmetic. + +This is an implementation of decimal floating point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" try: from _decimal import * - from _decimal import __doc__ - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + from _pydecimal import __version__ # noqa: F401 + from _pydecimal import __libmpdec_version__ # noqa: F401 diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969ebff..7f595b6c72e641 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1266,6 +1264,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) diff --git a/Lib/dis.py b/Lib/dis.py index cad62b95990c30..bb922b786f5307 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,12 +11,16 @@ _cache_format, _inline_cache_entries, _nb_ops, + _common_constants, _intrinsic_1_descs, _intrinsic_2_descs, + _special_method_names, _specializations, _specialized_opmap, ) +from _opcode import get_executor + __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all @@ -30,6 +34,7 @@ SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] LOAD_CONST = opmap['LOAD_CONST'] RETURN_CONST = opmap['RETURN_CONST'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] @@ -41,6 +46,8 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] +LOAD_SPECIAL = opmap['LOAD_SPECIAL'] LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] @@ -72,7 +79,8 @@ def _try_compile(source, name): pass return compile(source, name, 'exec') -def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): +def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, + show_offsets=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -82,7 +90,8 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): in a special attribute. """ if x is None: - distb(file=file, show_caches=show_caches, adaptive=adaptive) + distb(file=file, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -103,21 +112,28 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) elif isinstance(x, (bytes, bytearray)): # Raw bytecode - _disassemble_bytes(x, file=file, show_caches=show_caches) + labels_map = _make_labels_map(x) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + offset_width=len(str(max(len(x) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(labels_map=labels_map) + _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -128,7 +144,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False): except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to @@ -195,7 +211,27 @@ def _deoptop(op): return _all_opmap[deoptmap[name]] if name in deoptmap else op def _get_code_array(co, adaptive): - return co._co_code_adaptive if adaptive else co.co_code + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = get_executor(co, i) + except (ValueError, RuntimeError): + ex = None + + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code def code_info(x): """Formatted details of methods, functions, or code.""" @@ -263,10 +299,11 @@ def show_code(co, *, file=None): 'start_offset', 'starts_line', 'line_number', - 'is_jump_target', - 'positions' + 'label', + 'positions', + 'cache_info', ], - defaults=[None] + defaults=[None, None, None] ) _Instruction.opname.__doc__ = "Human readable name for operation" @@ -281,12 +318,16 @@ def show_code(co, *, file=None): ) _Instruction.starts_line.__doc__ = "True if this opcode starts a source line, otherwise False" _Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" -_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" +_Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" _Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" +_Instruction.cache_info.__doc__ = "list of (name, size, data), one for each cache entry of the instruction" -_ExceptionTableEntry = collections.namedtuple("_ExceptionTableEntry", +_ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", "start end target depth lasti") +class _ExceptionTableEntry(_ExceptionTableEntryBase): + pass + _OPNAME_WIDTH = 20 _OPARG_WIDTH = 5 @@ -325,9 +366,11 @@ class Instruction(_Instruction): otherwise equal to Instruction.offset starts_line - True if this opcode starts a source line, otherwise False line_number - source line number associated with this opcode (if any), otherwise None - is_jump_target - True if other code jumps to here, otherwise False + label - A label if this instruction is a jump target, otherwise None positions - Optional dis.Positions object holding the span of source code covered by this instruction + cache_info - information about the format and content of the instruction's cache + entries (if any) """ @property @@ -369,53 +412,210 @@ def jump_target(self): """ return _get_jump_target(self.opcode, self.arg, self.offset) - def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4): - """Format instruction details for inclusion in disassembly output. + @property + def is_jump_target(self): + """True if other code jumps to here, otherwise False""" + return self.label is not None + + def __str__(self): + output = io.StringIO() + formatter = Formatter(file=output) + formatter.print_instruction(self, False) + return output.getvalue() + +class Formatter: + + def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, + line_offset=0, show_caches=False): + """Create a Formatter + + *file* where to write the output *lineno_width* sets the width of the line number field (0 omits it) - *mark_as_current* inserts a '-->' marker arrow as part of the line *offset_width* sets the width of the instruction offset field + *label_width* sets the width of the label field + *show_caches* is a boolean indicating whether to display cache lines + """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + self.show_caches = show_caches + + def print_instruction(self, instr, mark_as_current=False): + self.print_instruction_line(instr, mark_as_current) + if self.show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + self.print_instruction_line( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + False) + + def print_instruction_line(self, instr, mark_as_current): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + fields = [] # Column: Source code line number if lineno_width: - if self.starts_line: - lineno_fmt = "%%%dd" if self.line_number is not None else "%%%ds" + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" lineno_fmt = lineno_fmt % lineno_width - fields.append(lineno_fmt % self.line_number) + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) else: fields.append(' ' * lineno_width) + # Column: Label + if instr.label is not None: + lbl = f"L{instr.label}:" + fields.append(f"{lbl:>{label_width}}") + else: + fields.append(' ' * label_width) + # Column: Instruction offset from start of code sequence + if offset_width > 0: + fields.append(f"{repr(instr.offset):>{offset_width}} ") # Column: Current instruction indicator if mark_as_current: fields.append('-->') else: fields.append(' ') - # Column: Jump target marker - if self.is_jump_target: - fields.append('>>') - else: - fields.append(' ') - # Column: Instruction offset from start of code sequence - fields.append(repr(self.offset).rjust(offset_width)) # Column: Opcode name - fields.append(self.opname.ljust(_OPNAME_WIDTH)) + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument - if self.arg is not None: - arg = repr(self.arg) + if instr.arg is not None: + arg = repr(instr.arg) # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into # the space reserved for oparg. This results in fewer misaligned opargs # in the disassembly output. - opname_excess = max(0, len(self.opname) - _OPNAME_WIDTH) - fields.append(repr(self.arg).rjust(_OPARG_WIDTH - opname_excess)) + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) # Column: Opcode argument details - if self.argrepr: - fields.append('(' + self.argrepr + ')') - return ' '.join(fields).rstrip() - - def __str__(self): - return self._disassemble() + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_map=None): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map or {} + + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + return argval + return None + + def get_label_for_offset(self, offset): + return self.labels_map.get(offset, None) + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + argrepr = f"to L{lbl}" + elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1<<i)) + elif deop == BINARY_OP: + _, argrepr = _nb_ops[arg] + elif deop == CALL_INTRINSIC_1: + argrepr = _intrinsic_1_descs[arg] + elif deop == CALL_INTRINSIC_2: + argrepr = _intrinsic_2_descs[arg] + elif deop == LOAD_COMMON_CONSTANT: + obj = _common_constants[arg] + if isinstance(obj, type): + argrepr = obj.__name__ + else: + argrepr = repr(obj) + elif deop == LOAD_SPECIAL: + argrepr = _special_method_names[arg] + return argval, argrepr -def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False): +def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): """Iterator for the opcodes in methods, functions or code Generates a series of Instruction named tuples giving the details of @@ -432,13 +632,18 @@ def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False): line_offset = first_line - co.co_firstlineno else: line_offset = 0 + + original_code = co.co_code + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=_make_labels_map(original_code)) return _get_instructions_bytes(_get_code_array(co, adaptive), - co._varname_from_oparg, - co.co_names, co.co_consts, - linestarts, line_offset, + linestarts=linestarts, + line_offset=line_offset, co_positions=co.co_positions(), - show_caches=show_caches, - original_code=co.co_code) + original_code=original_code, + arg_resolver=arg_resolver) def _get_const_value(op, arg, co_consts): """Helper to get the value of the const in a hasconst op. @@ -506,19 +711,15 @@ def _parse_exception_table(code): return entries def _is_backward_jump(op): - return 'JUMP_BACKWARD' in opname[op] + return opname[op] in ('JUMP_BACKWARD', + 'JUMP_BACKWARD_NO_INTERRUPT') -def _get_instructions_bytes(code, varname_from_oparg=None, - names=None, co_consts=None, - linestarts=None, line_offset=0, - exception_entries=(), co_positions=None, - show_caches=False, original_code=None): +def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, + original_code=None, arg_resolver=None): """Iterate over the instructions in a bytecode string. Generates a sequence of Instruction namedtuples giving the details of each - opcode. Additional information about the code's runtime environment - (e.g. variable names, co_consts) can be specified using optional - arguments. + opcode. """ # Use the basic, unadaptive code for finding labels and actually walking the @@ -526,11 +727,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, # mess that logic up pretty badly: original_code = original_code or code co_positions = co_positions or iter(()) - get_name = None if names is None else names.__getitem__ - labels = set(findlabels(original_code)) - for start, end, target, _, _ in exception_entries: - for i in range(start, end): - labels.add(target) + starts_line = False local_line_number = None line_number = None @@ -543,109 +740,56 @@ def _get_instructions_bytes(code, varname_from_oparg=None, line_number = local_line_number + line_offset else: line_number = None - is_jump_target = offset in labels - argval = None - argrepr = '' positions = Positions(*next(co_positions, ())) deop = _deoptop(op) - caches = _get_cache_size(_all_opname[deop]) op = code[offset] - if arg is not None: - # Set argval to the dereferenced value of the argument when - # available, and argrepr to the string representation of argval. - # _disassemble_bytes needs the string repr of the - # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. - argval = arg - if deop in hasconst: - argval, argrepr = _get_const_info(deop, arg, co_consts) - elif deop in hasname: - if deop == LOAD_GLOBAL: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL" - elif deop == LOAD_ATTR: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - elif deop == LOAD_SUPER_ATTR: - argval, argrepr = _get_name_info(arg//4, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - else: - argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = "to " + repr(argval) - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - argval += 2 * caches - argrepr = "to " + repr(argval) - elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): - arg1 = arg >> 4 - arg2 = arg & 15 - val1, argrepr1 = _get_name_info(arg1, varname_from_oparg) - val2, argrepr2 = _get_name_info(arg2, varname_from_oparg) - argrepr = argrepr1 + ", " + argrepr2 - argval = val1, val2 - elif deop in haslocal or deop in hasfree: - argval, argrepr = _get_name_info(arg, varname_from_oparg) - elif deop in hascompare: - argval = cmp_op[arg >> 5] - argrepr = argval - if arg & 16: - argrepr = f"bool({argrepr})" - elif deop == CONVERT_VALUE: - argval = (None, str, repr, ascii)[arg] - argrepr = ('', 'str', 'repr', 'ascii')[arg] - elif deop == SET_FUNCTION_ATTRIBUTE: - argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) - if arg & (1<<i)) - elif deop == BINARY_OP: - _, argrepr = _nb_ops[arg] - elif deop == CALL_INTRINSIC_1: - argrepr = _intrinsic_1_descs[arg] - elif deop == CALL_INTRINSIC_2: - argrepr = _intrinsic_2_descs[arg] - yield Instruction(_all_opname[op], op, - arg, argval, argrepr, + + if arg_resolver: + argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset) + else: + argval, argrepr = arg, repr(arg) + + caches = _get_cache_size(_all_opname[deop]) + # Advance the co_positions iterator: + for _ in range(caches): + next(co_positions, ()) + + if caches: + cache_info = [] + for name, size in _cache_format[opname[deop]].items(): + data = code[offset + 2: offset + 2 + 2 * size] + cache_info.append((name, size, data)) + else: + cache_info = None + + label = arg_resolver.get_label_for_offset(offset) if arg_resolver else None + yield Instruction(_all_opname[op], op, arg, argval, argrepr, offset, start_offset, starts_line, line_number, - is_jump_target, positions) - if not caches: - continue - if not show_caches: - # We still need to advance the co_positions iterator: - for _ in range(caches): - next(co_positions, ()) - continue - for name, size in _cache_format[opname[deop]].items(): - for i in range(size): - offset += 2 - # Only show the fancy argrepr for a CACHE instruction when it's - # the first entry for a particular cache value: - if i == 0: - data = code[offset: offset + 2 * size] - argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" - else: - argrepr = "" - yield Instruction( - "CACHE", CACHE, 0, None, argrepr, offset, offset, False, None, False, - Positions(*next(co_positions, ())) - ) + label, positions, cache_info) + -def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False): +def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, + show_offsets=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) - _disassemble_bytes(_get_code_array(co, adaptive), - lasti, co._varname_from_oparg, - co.co_names, co.co_consts, linestarts, file=file, - exception_entries=exception_entries, - co_positions=co.co_positions(), show_caches=show_caches, - original_code=co.co_code) - -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive) + labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + lineno_width=_get_lineno_width(linestarts), + offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts, + exception_entries=exception_entries, co_positions=co.co_positions(), + original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) + +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -654,62 +798,66 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print(file=file) print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( - x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive + x, file=file, depth=depth, show_caches=show_caches, + adaptive=adaptive, show_offsets=show_offsets ) -def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, - names=None, co_consts=None, linestarts=None, - *, file=None, line_offset=0, exception_entries=(), - co_positions=None, show_caches=False, original_code=None): - # Omit the line number column entirely if we have no line number info - if bool(linestarts): - linestarts_ints = [line for line in linestarts.values() if line is not None] - show_lineno = len(linestarts_ints) > 0 - else: - show_lineno = False - - if show_lineno: - maxlineno = max(linestarts_ints) + line_offset - if maxlineno >= 1000: - lineno_width = len(str(maxlineno)) - else: - lineno_width = 3 - if lineno_width < len(str(None)) and None in linestarts.values(): - lineno_width = len(str(None)) - else: - lineno_width = 0 - maxoffset = len(code) - 2 - if maxoffset >= 10000: - offset_width = len(str(maxoffset)) - else: - offset_width = 4 - for instr in _get_instructions_bytes(code, varname_from_oparg, names, - co_consts, linestarts, - line_offset=line_offset, - exception_entries=exception_entries, - co_positions=co_positions, - show_caches=show_caches, - original_code=original_code): - new_source_line = (show_lineno and - instr.starts_line and - instr.offset > 0) - if new_source_line: - print(file=file) - if show_caches: - is_current_instr = instr.offset == lasti - else: - # Each CACHE takes 2 bytes - is_current_instr = instr.offset <= lasti \ - <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) - print(instr._disassemble(lineno_width, is_current_instr, offset_width), - file=file) - if exception_entries: - print("ExceptionTable:", file=file) - for entry in exception_entries: - lasti = " lasti" if entry.lasti else "" - end = entry.end-2 - print(f" {entry.start} to {end} -> {entry.target} [{entry.depth}]{lasti}", file=file) +def _make_labels_map(original_code, exception_entries=()): + jump_targets = set(findlabels(original_code)) + labels = set(jump_targets) + for start, end, target, _, _ in exception_entries: + labels.add(start) + labels.add(end) + labels.add(target) + labels = sorted(labels) + labels_map = {offset: i+1 for (i, offset) in enumerate(sorted(labels))} + for e in exception_entries: + e.start_label = labels_map[e.start] + e.end_label = labels_map[e.end] + e.target_label = labels_map[e.target] + return labels_map + +_NO_LINENO = ' --' + +def _get_lineno_width(linestarts): + if linestarts is None: + return 0 + maxlineno = max(filter(None, linestarts.values()), default=-1) + if maxlineno == -1: + # Omit the line number column entirely if we have no line number info + return 0 + lineno_width = max(3, len(str(maxlineno))) + if lineno_width < len(_NO_LINENO) and None in linestarts.values(): + lineno_width = len(_NO_LINENO) + return lineno_width + + +def _disassemble_bytes(code, lasti=-1, linestarts=None, + *, line_offset=0, exception_entries=(), + co_positions=None, original_code=None, + arg_resolver=None, formatter=None): + + assert formatter is not None + assert arg_resolver is not None + + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + original_code=original_code, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, lasti=-1): + for instr in instrs: + # Each CACHE takes 2 bytes + is_current_instr = instr.offset <= lasti \ + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + formatter.print_instruction(instr, is_current_instr) + + formatter.print_exception_table(exception_entries) def _disassemble_str(source, **kwargs): """Compile the source string, then disassemble the code object.""" @@ -830,7 +978,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -844,18 +992,22 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.exception_entries = _parse_exception_table(co) self.show_caches = show_caches self.adaptive = adaptive + self.show_offsets = show_offsets def __iter__(self): co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) return _get_instructions_bytes(_get_code_array(co, self.adaptive), - co._varname_from_oparg, - co.co_names, co.co_consts, - self._linestarts, + linestarts=self._linestarts, line_offset=self._line_offset, - exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches, - original_code=co.co_code) + original_code=original_code, + arg_resolver=arg_resolver) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, @@ -882,17 +1034,32 @@ def dis(self): else: offset = -1 with io.StringIO() as output: - _disassemble_bytes(_get_code_array(co, self.adaptive), - varname_from_oparg=co._varname_from_oparg, - names=co.co_names, co_consts=co.co_consts, + code = _get_code_array(co, self.adaptive) + offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 + + + labels_map = _make_labels_map(co.co_code, self.exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=output, + lineno_width=_get_lineno_width(self._linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=self._line_offset, + show_caches=self.show_caches) + + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(code, linestarts=self._linestarts, line_offset=self._line_offset, - file=output, lasti=offset, exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches, - original_code=co.co_code) + original_code=co.co_code, + arg_resolver=arg_resolver, + formatter=formatter) return output.getvalue() @@ -902,12 +1069,19 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('-C', '--show-caches', action='store_true', help='show inline caches') - parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') + parser.add_argument('-O', '--show-offsets', action='store_true', + help='show instruction offsets') + parser.add_argument('infile', nargs='?', default='-') args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") - dis(code, show_caches=args.show_caches) + if args.infile == '-': + name = '<stdin>' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + code = compile(source, name, "exec") + dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) if __name__ == "__main__": main() diff --git a/Lib/doctest.py b/Lib/doctest.py index 46b4dd6769b063..ea7d275c91db04 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,6 +104,8 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -591,9 +593,11 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented - return ((self.name, self.filename, self.lineno, id(self)) + self_lno = self.lineno if self.lineno is not None else -1 + other_lno = other.lineno if other.lineno is not None else -1 + return ((self.name, self.filename, self_lno, id(self)) < - (other.name, other.filename, other.lineno, id(other))) + (other.name, other.filename, other_lno, id(other))) ###################################################################### ## 3. DocTestParser @@ -1134,9 +1138,18 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ + if isinstance(obj, property): + obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): @@ -1168,6 +1181,9 @@ class DocTestRunner: The `run` method is used to process a single DocTest case. It returns a TestResults instance. + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) >>> tests.sort(key = lambda test: test.name) @@ -1187,9 +1203,9 @@ class DocTestRunner: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) @@ -1211,13 +1227,15 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1296,7 +1314,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1393,7 +1414,24 @@ def __run(self, test, compileflags, out): # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exception[:2]) + if issubclass(exception[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exception_line_prefixes = ( + f"{exception[0].__qualname__}:", + f"{exception[0].__module__}.{exception[0].__qualname__}:", + ) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(exception_line_prefixes) + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: got += _exception_traceback(exception) @@ -1519,7 +1557,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1528,6 +1570,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1547,49 +1593,70 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] + + notests, passed, failed = [], [], [] total_tries = total_failures = total_skips = 0 - for item in self._stats.items(): - name, (failures, tries, skips) = item + + for name, (failures, tries, skips) in self._stats.items(): assert failures <= tries total_tries += tries total_failures += failures total_skips += skips + if tries == 0: notests.append(name) elif failures == 0: passed.append((name, tries)) else: - failed.append(item) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(f"{len(notests)} items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() for name in notests: print(f" {name}") + if passed: - print(f"{len(passed)} items passed all tests:") - passed.sort() - for name, count in passed: - print(f" {count:3d} tests in {name}") + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(f"{len(failed)} items had failures:") - failed.sort() - for name, (failures, tries, skips) in failed: + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(f"{total_tries} tests in {len(self._stats)} items.") - print(f"{total_tries - total_failures} passed and {total_failures} failed.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + if total_failures: - msg = f"***Test Failed*** {total_failures} failures" + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" if total_skips: - msg = f"{msg} and {total_skips} skipped tests" + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" print(f"{msg}.") elif verbose: - print("Test passed.") + print(f"{bold_green}Test passed.{reset}") + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// @@ -1606,9 +1673,18 @@ def merge(self, other): d[name] = (failures, tries, skips) +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -2204,13 +2280,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) @@ -2241,12 +2317,13 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: + if results.failed: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): @@ -2657,7 +2734,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 0d6bd812475eea..ab3c3031ef590c 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -566,12 +566,14 @@ def display_name(self): if res[0].token_type == 'cfws': res.pop(0) else: - if res[0][0].token_type == 'cfws': + if (isinstance(res[0], TokenList) and + res[0][0].token_type == 'cfws'): res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: - if res[-1][-1].token_type == 'cfws': + if (isinstance(res[-1], TokenList) and + res[-1][-1].token_type == 'cfws'): res[-1] = TokenList(res[-1][:-1]) return res.value @@ -586,9 +588,13 @@ def value(self): quote = True if len(self) != 0 and quote: pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': + if (self[0].token_type == 'cfws' or + isinstance(self[0], TokenList) and + self[0][0].token_type == 'cfws'): pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': + if (self[-1].token_type == 'cfws' or + isinstance(self[-1], TokenList) and + self[-1][-1].token_type == 'cfws'): post = ' ' return pre+quote_string(self.display_name)+post else: @@ -949,6 +955,8 @@ class _InvalidEwError(errors.HeaderParseError): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -1206,7 +1214,7 @@ def get_bare_quoted_string(value): value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ - if value[0] != '"': + if not value or value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() @@ -1447,7 +1455,7 @@ def get_local_part(value): """ local_part = LocalPart() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1513,13 +1521,18 @@ def get_obs_local_part(value): raise token, value = get_cfws(value) obs_local_part.append(token) + if not obs_local_part: + raise errors.HeaderParseError( + "expected obs-local-part but found '{}'".format(value)) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) @@ -1601,7 +1614,7 @@ def get_domain(value): """ domain = Domain() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1677,6 +1690,8 @@ def get_obs_route(value): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) + if not value: + break if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) @@ -1695,7 +1710,7 @@ def get_angle_addr(value): """ angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': @@ -1705,7 +1720,7 @@ def get_angle_addr(value): value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. - if value[0] == '>': + if value and value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) @@ -1757,6 +1772,9 @@ def get_name_addr(value): name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: @@ -1771,7 +1789,10 @@ def get_name_addr(value): raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: - token[0][:0] = [leader] + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) @@ -2022,7 +2043,7 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value @@ -2764,10 +2785,15 @@ def _refold_parse_tree(parse_tree, *, policy): # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' - lines = [''] - last_ew = None + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line + last_charset = None wrap_as_ew_blocked = 0 - want_encoding = False + want_encoding = False # This is set to True if we need to encode this part end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: @@ -2791,10 +2817,12 @@ def _refold_parse_tree(parse_tree, *, policy): # 'charset' property on the policy. charset = 'utf-8' want_encoding = True + if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) continue + if want_encoding and not wrap_as_ew_blocked: if not part.as_ew_allowed: want_encoding = False @@ -2817,24 +2845,55 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a Terminal, do each piece individually. parts = list(part) + parts - else: + want_encoding = False + continue + elif part.as_ew_allowed: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. + if (last_ew is not None and + charset != last_charset and + (last_charset == 'unknown-8bit' or + last_charset == 'utf-8' and charset != 'us-ascii')): + last_ew = None last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, - part.ew_combine_allowed, charset) - want_encoding = False - continue + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' + last_charset = charset + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through + if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr continue + # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) last_ew = None continue if not hasattr(part, 'encode'): @@ -2858,9 +2917,10 @@ def _refold_parse_tree(parse_tree, *, policy): else: # We can't fold it onto the next line either... lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep -def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. @@ -2875,7 +2935,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] - if to_encode[0] in WSP: + elif to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. leading_wsp = to_encode[0] @@ -2883,6 +2943,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): if (len(lines[-1]) == maxlen): lines.append(_steal_trailing_WSP_if_exists(lines)) lines[-1] += leading_wsp + trailing_wsp = '' if to_encode[-1] in WSP: # Likewise for the trailing space. @@ -2902,11 +2963,20 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): while to_encode: remaining_space = maxlen - len(lines[-1]) - text_space = remaining_space - chrome_len + text_space = remaining_space - chrome_len - len(leading_whitespace) if text_space <= 0: lines.append(' ') continue + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index febe411355d6be..36625e35ffb6a7 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -194,6 +194,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] @@ -221,7 +224,7 @@ class AddrlistClass: def __init__(self, field): """Initialize a new instance. - `field' is an unparsed address header field, containing + 'field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' @@ -230,7 +233,7 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it + # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') @@ -420,14 +423,14 @@ def getdomain(self): def getdelimited(self, beginchar, endchars, allowcomments=True): """Parse a header fragment delimited by special characters. - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then + 'beginchar' is the start character for the fragment. + If self is not looking at an instance of 'beginchar' then getdelimited returns the empty string. - `endchars' is a sequence of allowable end-delimiting characters. + 'endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed + If 'allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: @@ -471,7 +474,7 @@ def getatom(self, atomends=None): Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which + getphraselist() since phrase endings must not include the '.' (which is legal in phrases).""" atomlist = [''] if atomends is None: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48..1c76ed63b61ae8 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -150,9 +150,9 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): wrapping is done. Default is 78. mangle_from_ -- a flag that, when True escapes From_ lines in the - body of the message by putting a `>' in front of + body of the message by putting a '>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 4cdf22666e3016..d440de95255bf1 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -15,7 +15,7 @@ with Base64 encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:, From:, Cc:, etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 043801107b60e5..cfd5a0c456e497 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -175,7 +175,7 @@ class Charset: module expose the following information about a character set: input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 + are converted to their 'official' email names (e.g. latin_1 is converted to iso-8859-1). Defaults to 7-bit us-ascii. header_encoding: If the character set must be encoded before it can be @@ -245,7 +245,7 @@ def __eq__(self, other): def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. - This is either the string `quoted-printable' or `base64' depending on + This is either the string 'quoted-printable' or 'base64' depending on the encoding used, or it is a function in which case you should call the function with a single argument, the Message object being encoded. The function should then set the Content-Transfer-Encoding diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 7ccbe10eb76856..9d058ceada24f8 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -41,7 +41,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, Optional mangle_from_ is a flag that, when True (the default if policy is not set), escapes From_ lines in the body of the message by putting - a `>' in front of them. + a '>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs @@ -74,7 +74,7 @@ def flatten(self, msg, unixfrom=False, linesep=None): unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this + has no From_ delimiter, a 'standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. Note that for subobjects, no From_ line is printed. @@ -243,7 +243,7 @@ def _handle_text(self, msg): # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] - msg.set_payload(payload, charset) + msg.set_payload(msg._payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) @@ -456,7 +456,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, argument is allowed. Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. + type 'text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in diff --git a/Lib/email/header.py b/Lib/email/header.py index 984851a7d9a679..66a1d46db50c45 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -192,7 +192,7 @@ def __init__(self, s=None, charset=None, The maximum line length can be specified explicitly via maxlinelen. For splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of + header which isn't included in s, e.g. 'Subject') pass in the name of the field in header_name. The default maxlinelen is 78 as recommended by RFC 2822. @@ -276,7 +276,7 @@ def append(self, s, charset=None, errors='strict'): output codec of the charset. If the string cannot be encoded to the output codec, a UnicodeError will be raised. - Optional `errors' is passed as the errors argument to the decode + Optional 'errors' is passed as the errors argument to the decode call if s is a byte string. """ if charset is None: @@ -326,7 +326,7 @@ def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): Optional splitchars is a string containing characters which should be given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of RFC 2822's `higher level + wrapping. This is in very rough support of RFC 2822's 'higher level syntactic breaks': split points preceded by a splitchar are preferred during line splitting, with the characters preferred in the order in which they appear in the string. Space and tab may be included in the diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index 3410935e38f476..2f436aefc2300b 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -43,8 +43,8 @@ def body_line_iterator(msg, decode=False): def typed_subpart_iterator(msg, maintype='text', subtype=None): """Iterate over the subparts with a given MIME type. - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if + Use 'maintype' as the main MIME type to match against; this defaults to + "text". Optional 'subtype' is the MIME subtype to match against; if omitted, only the main type is matched. """ for subpart in msg.walk(): diff --git a/Lib/email/message.py b/Lib/email/message.py index 411118c74dabb4..08192c50a8ff5c 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -21,7 +21,7 @@ SEMISPACE = '; ' -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') @@ -141,7 +141,7 @@ class Message: multipart or a message/rfc822), then the payload is a list of Message objects, otherwise it is a string. - Message objects implement part of the `mapping' interface, which assumes + Message objects implement part of the 'mapping' interface, which assumes there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of @@ -289,25 +289,26 @@ def get_payload(self, i=None, decode=False): # cte might be a Header, so for now stringify it. cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. - if isinstance(payload, str): - if utils._has_surrogates(payload): - bpayload = payload.encode('ascii', 'surrogateescape') - if not decode: + if not decode: + if isinstance(payload, str) and utils._has_surrogates(payload): + try: + bpayload = payload.encode('ascii', 'surrogateescape') try: - payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') + payload = bpayload.decode(self.get_content_charset('ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') - elif decode: - try: - bpayload = payload.encode('ascii') - except UnicodeError: - # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). - # If it does happen, turn the string into bytes in a way - # guaranteed not to fail. - bpayload = payload.encode('raw-unicode-escape') - if not decode: + except UnicodeEncodeError: + pass return payload + if isinstance(payload, str): + try: + bpayload = payload.encode('ascii', 'surrogateescape') + except UnicodeEncodeError: + # This won't happen for RFC compliant messages (messages + # containing only ASCII code points in the unicode input). + # If it does happen, turn the string into bytes in a way + # guaranteed not to fail. + bpayload = payload.encode('raw-unicode-escape') if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': @@ -339,7 +340,7 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset) + payload = payload.encode(charset.output_charset, 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: @@ -596,7 +597,7 @@ def get_content_type(self): """Return the message's content type. The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the + 'maintype/subtype'. If there was no Content-Type header in the message, the default type as given by get_default_type() will be returned. Since according to RFC 2045, messages always have a default type this will always return a value. @@ -619,7 +620,7 @@ def get_content_type(self): def get_content_maintype(self): """Return the message's main content type. - This is the `maintype' part of the string returned by + This is the 'maintype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() @@ -628,14 +629,14 @@ def get_content_maintype(self): def get_content_subtype(self): """Returns the message's sub-content type. - This is the `subtype' part of the string returned by + This is the 'subtype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[1] def get_default_type(self): - """Return the `default' content type. + """Return the 'default' content type. Most messages have a default content type of text/plain, except for messages that are subparts of multipart/digest containers. Such @@ -644,7 +645,7 @@ def get_default_type(self): return self._default_type def set_default_type(self, ctype): - """Set the `default' content type. + """Set the 'default' content type. ctype should be either "text/plain" or "message/rfc822", although this is not enforced. The default content type is not stored in the @@ -677,8 +678,8 @@ def get_params(self, failobj=None, header='content-type', unquote=True): """Return the message's Content-Type parameters, as a list. The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in + split on the '=' sign. The left hand side of the '=' is the key, + while the right hand side is the value. If there is no '=' sign in the parameter the value is the empty string. The value is as described in the get_param() method. @@ -838,9 +839,9 @@ def get_filename(self, failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. + 'filename' parameter, and it is unquoted. If that header is missing + the 'filename' parameter, this method falls back to looking for the + 'name' parameter. """ missing = object() filename = self.get_param('filename', missing, 'content-disposition') @@ -853,7 +854,7 @@ def get_filename(self, failobj=None): def get_boundary(self, failobj=None): """Return the boundary associated with the payload if present. - The boundary is extracted from the Content-Type header's `boundary' + The boundary is extracted from the Content-Type header's 'boundary' parameter, and it is unquoted. """ missing = object() diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 94d81c771a474e..47fc218e1ae032 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -21,7 +21,7 @@ def __init__(self, _subtype='mixed', boundary=None, _subparts=None, Content-Type and MIME-Version headers. _subtype is the subtype of the multipart content type, defaulting to - `mixed'. + 'mixed'. boundary is the multipart boundary string. By default it is calculated as needed. diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 06d99b17f2f9c4..475aa2b1a66680 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -22,7 +22,7 @@ def __init__(self, _class=None, *, policy=compat32): textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -82,7 +82,7 @@ def __init__(self, *args, **kw): textual representation of the message. The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 611deb50bb5290..46b7de5bb6d8ae 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -21,7 +21,7 @@ 'HTTP', ] -linesep_splitter = re.compile(r'\n|\r') +linesep_splitter = re.compile(r'\n|\r\n?') @_extend_docstrings class EmailPolicy(Policy): @@ -205,13 +205,21 @@ def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) maxlen = self.max_line_length if self.max_line_length else sys.maxsize - lines = value.splitlines() + # We can't use splitlines here because it splits on more than \r and \n. + lines = linesep_splitter.split(value) refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): + + if not refold: + if not self.utf8: + refold = not value.isascii() + elif refold_binary: + refold = _has_surrogates(value) + if refold: return self.header_factory(name, ''.join(lines)).fold(policy=self) + return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index 27fcbb5a26e3ae..500bbc5151769d 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -5,7 +5,7 @@ """Quoted-printable content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to +to encode US ASCII-like 8-bit data called 'quoted-printable'. It is used to safely encode text that is in a character set similar to the 7-bit US ASCII character set, but that includes some 8-bit characters that are normally not allowed in email bodies or headers. @@ -17,7 +17,7 @@ with quoted-printable encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:/From:/Cc: etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character @@ -127,7 +127,7 @@ def quote(c): def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with quoted-printable (like) encoding. - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but + Defined in RFC 2045, this 'Q' encoding is similar to quoted-printable, but used specifically for email header fields to allow charsets with mostly 7 bit characters (and some 8 bit) to remain more or less readable in non-RFC 2045 aware mail clients. @@ -290,7 +290,7 @@ def _unquote_match(match): # Header decoding is done a bit differently def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. + """Decode a string encoded with RFC 2045 MIME header 'Q' encoding. This function does not parse a full MIME header value encoded with quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a49a8fa986ce0c..6d897ca8eeee91 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -43,11 +43,12 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') + def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" + """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. + # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings. try: s.encode() return False @@ -103,12 +104,127 @@ def formataddr(pair, charset='utf-8'): return address +def _iter_escaped_chars(addr): + pos = 0 + escape = False + for pos, ch in enumerate(addr): + if escape: + yield (pos, '\\' + ch) + escape = False + elif ch == '\\': + escape = True + else: + yield (pos, ch) + if escape: + yield (pos, '\\') + + +def _strip_quoted_realnames(addr): + """Strip real names between quotes.""" + if '"' not in addr: + # Fast path + return addr + + start = 0 + open_pos = None + result = [] + for pos, ch in _iter_escaped_chars(addr): + if ch == '"': + if open_pos is None: + open_pos = pos + else: + if start != open_pos: + result.append(addr[start:open_pos]) + start = pos + 1 + open_pos = None + + if start < len(addr): + result.append(addr[start:]) + + return ''.join(result) -def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(str(v) for v in fieldvalues) - a = _AddressList(all) - return a.addresslist + +supports_strict_parsing = True + +def getaddresses(fieldvalues, *, strict=True): + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. + + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If strict is true, use a strict parser which rejects malformed inputs. + """ + + # If strict is true, if the resulting list of parsed addresses is greater + # than the number of fieldvalues in the input list, a parsing error has + # occurred and consequently a list containing a single empty 2-tuple [('', + # '')] is returned in its place. This is done to avoid invalid output. + # + # Malformed input: getaddresses(['alice@example.com <bob@example.com>']) + # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] + # Safe output: [('', '')] + + if not strict: + all = COMMASPACE.join(str(v) for v in fieldvalues) + a = _AddressList(all) + return a.addresslist + + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + addr = COMMASPACE.join(fieldvalues) + a = _AddressList(addr) + result = _post_parse_validation(a.addresslist) + + # Treat output as invalid if the number of addresses is not equal to the + # expected number of addresses. + n = 0 + for v in fieldvalues: + # When a comma is used in the Real Name part it is not a deliminator. + # So strip those out before counting the commas. + v = _strip_quoted_realnames(v) + # Expected number of addresses: 1 + number of commas + n += 1 + v.count(',') + if len(result) != n: + return [('', '')] + + return result + + +def _check_parenthesis(addr): + # Ignore parenthesis in quoted real names. + addr = _strip_quoted_realnames(addr) + + opens = 0 + for pos, ch in _iter_escaped_chars(addr): + if ch == '(': + opens += 1 + elif ch == ')': + opens -= 1 + if opens < 0: + return False + return (opens == 0) + + +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + if not _check_parenthesis(v): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values def _format_timetuple_and_zone(timetuple, zone): @@ -207,16 +323,33 @@ def parsedate_to_datetime(data): tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) -def parseaddr(addr): +def parseaddr(addr, *, strict=True): """ Parse addr into its constituent realname and email address parts. Return a tuple of realname and email address, unless the parse fails, in which case return a 2-tuple of ('', ''). + + If strict is True, use a strict parser which rejects malformed inputs. """ - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' + if not strict: + addrs = _AddressList(addr).addresslist + if not addrs: + return ('', '') + return addrs[0] + + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] @@ -333,23 +466,15 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=None): +def localtime(dt=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - The isdst parameter is ignored. """ - if isdst is not None: - import warnings - warnings._deprecated( - "The 'isdst' parameter to 'localtime'", - message='{name} is deprecated and slated for removal in Python {remove}', - remove=(3, 14), - ) if dt is None: dt = datetime.datetime.now() return dt.astimezone() diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5cf704..6a5ca046b5eb6c 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7fb0b8..60a8d5eb227f82 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -202,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError("Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c5726447077b1..4622fc8c9206f3 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355c710..082771e1c86677 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242be8c7..d3b9980026666f 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d14129a62..1924bedbb74c68 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 1fb1d505cfd0c5..23fe7a82eb029d 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.2.1" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "24.1.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None + +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None -def _find_packages(path): - packages = {} + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -105,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False, with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl deleted file mode 100644 index ba28ef02e265f0..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl new file mode 100644 index 00000000000000..e27568eb8b39c9 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index f5448a1788e4d2..c36fc75a24a239 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -4,7 +4,7 @@ __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', @@ -61,8 +61,8 @@ def _is_sunder(name): return ( len(name) > 2 and name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' + name[1] != '_' and + name[-2] != '_' ) def _is_internal_class(cls_name, obj): @@ -81,7 +81,6 @@ def _is_private(cls_name, name): if ( len(name) > pat_len and name.startswith(pattern) - and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -156,7 +155,6 @@ def _dedent(text): Like textwrap.dedent. Rewritten because we cannot import textwrap. """ lines = text.split('\n') - blanks = 0 for i, ch in enumerate(lines[0]): if ch != ' ': break @@ -164,6 +162,11 @@ def _dedent(text): lines[j] = l[i:] return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('<not given>') +_not_given = _not_given() + class _auto_null: def __repr__(self): return '_auto_null' @@ -281,9 +284,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -311,49 +315,13 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be @@ -362,9 +330,10 @@ def __set_name__(self, enum_class, member_name): except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. @@ -373,7 +342,7 @@ class _EnumDict(dict): """ def __init__(self): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False @@ -395,7 +364,11 @@ def __setitem__(self, key, value): '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', - ): + '_add_alias_', '_add_value_alias_', + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -446,10 +419,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -464,12 +438,21 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singlely + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -480,6 +463,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -491,7 +476,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() + enum_dict = EnumDict() enum_dict._cls_name = cls # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) @@ -554,6 +539,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} classdict['_unhashable_values_'] = [] + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -569,13 +555,20 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_inverted_'] = None try: exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) + except Exception as e: + # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError + # recreate the exception to discard + exc = type(e)(str(e)) + exc.__cause__ = e.__cause__ + exc.__context__ = e.__context__ + tb = e.__traceback__ if exc is not None: - raise exc + raise exc.with_traceback(tb) # # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) @@ -695,7 +688,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -724,18 +717,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -752,7 +745,10 @@ def __contains__(cls, value): """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + try: + return value in cls._value2member_map_ + except TypeError: + return value in cls._unhashable_values_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1048,7 +1044,55 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1114,12 +1158,14 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, values in cls._unhashable_values_map_.items(): + if value in values: + return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1157,6 +1203,33 @@ def __new__(cls, value): def __init__(self, *args, **kwds): pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) + @staticmethod def _generate_next_value_(name, start, count, last_values): """ @@ -1191,14 +1264,13 @@ def __str__(self): def __dir__(self): """ - Returns all members and all public methods + Returns public methods and other interesting attributes. """ - if self.__class__._member_type_ is object: - interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) - else: + interesting = set() + if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): - if name[0] != '_': + if name[0] != '_' and name not in self._member_map_: interesting.add(name) for cls in self.__class__.mro(): for name, obj in cls.__dict__.items(): @@ -1211,7 +1283,7 @@ def __dir__(self): else: # in case it was added by `dir(self)` interesting.discard(name) - else: + elif name not in self._member_map_: interesting.add(name) names = sorted( set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) @@ -1499,37 +1571,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) @@ -1596,7 +1681,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1639,7 +1724,7 @@ def _simple_enum(etype=Enum, *, boundary=None, use_args=None): Class decorator that converts a normal class into an :class:`Enum`. No safety checks are done, and some advanced behavior (such as :func:`__init_subclass__`) is not available. Enum creation can be faster - using :func:`simple_enum`. + using :func:`_simple_enum`. >>> from enum import Enum, _simple_enum >>> @_simple_enum(Enum) @@ -1670,7 +1755,8 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1717,34 +1803,40 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ @@ -1767,37 +1859,51 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ @@ -1931,7 +2037,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 30bd900fa805aa..020ea694ca63e9 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -121,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -132,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -184,7 +188,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -196,7 +200,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 1b25f28f3d3432..3dba3d2fbfa967 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -53,7 +53,7 @@ sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. -Optional in-place filtering: if the keyword argument inplace=1 is +Optional in-place filtering: if the keyword argument inplace=True is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index d5e296f7748c1c..73acb1fe8d4106 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -78,6 +78,11 @@ def translate(pat): """ STAR = object() + parts = _translate(pat, STAR, '.') + return _join_translated_parts(parts, STAR) + + +def _translate(pat, STAR, QUESTION_MARK): res = [] add = res.append i, n = 0, len(pat) @@ -89,7 +94,7 @@ def translate(pat): if (not res) or res[-1] is not STAR: add(STAR) elif c == '?': - add('.') + add(QUESTION_MARK) elif c == '[': j = i if j < n and pat[j] == '!': @@ -146,9 +151,11 @@ def translate(pat): else: add(re.escape(c)) assert i == n + return res + +def _join_translated_parts(inp, STAR): # Deal with STARs. - inp = res res = [] add = res.append i, n = 0, len(inp) diff --git a/Lib/fractions.py b/Lib/fractions.py index c95db0730e5b6d..565503911bbe97 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -55,17 +55,17 @@ def _hash_algorithm(numerator, denominator): return -2 if result == -1 else result _RATIONAL_FORMAT = re.compile(r""" - \A\s* # optional whitespace at the start, - (?P<sign>[-+]?) # an optional sign, then - (?=\d|\.\d) # lookahead for digit or .digit - (?P<num>\d*|\d+(_\d+)*) # numerator (possibly empty) - (?: # followed by - (?:\s*/\s*(?P<denom>\d+(_\d+)*))? # an optional denominator - | # or - (?:\.(?P<decimal>d*|\d+(_\d+)*))? # an optional fractional part - (?:E(?P<exp>[-+]?\d+(_\d+)*))? # and optional exponent + \A\s* # optional whitespace at the start, + (?P<sign>[-+]?) # an optional sign, then + (?=\d|\.\d) # lookahead for digit or .digit + (?P<num>\d*|\d+(_\d+)*) # numerator (possibly empty) + (?: # followed by + (?:\s*/\s*(?P<denom>\d+(_\d+)*))? # an optional denominator + | # or + (?:\.(?P<decimal>\d*|\d+(_\d+)*))? # an optional fractional part + (?:E(?P<exp>[-+]?\d+(_\d+)*))? # and optional exponent ) - \s*\Z # and optional whitespace to finish + \s*\Z # and optional whitespace to finish """, re.VERBOSE | re.IGNORECASE) @@ -139,6 +139,23 @@ def _round_to_figures(n, d, figures): return sign, significand, exponent +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P<fill>.)? + (?P<align>[<>=^]) + )? + (?P<sign>[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P<alt>\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P<minimumwidth>0|[1-9][0-9]*)? + (?P<thousands_sep>[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + # Pattern for matching float-style format specifications; # supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. _FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" @@ -414,27 +431,42 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def __format__(self, format_spec, /): - """Format this fraction according to the given format specification.""" - - # Backwards compatiblility with existing formatting. - if not format_spec: - return str(self) + def _format_general(self, match): + """Helper method for __format__. + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ # Validate and parse the format specifier. - match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec) - if match is None: - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}" - ) - elif match["align"] is not None and match["zeropad"] is not None: - # Avoid the temptation to guess. - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}; " - "can't use explicit alignment when zero-padding" - ) + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" fill = match["fill"] or " " align = match["align"] or ">" pos_sign = "" if match["sign"] == "-" else match["sign"] @@ -530,7 +562,25 @@ def __format__(self, format_spec, /): else: # align == "=" return sign + padding + body - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -617,8 +667,8 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) + elif handle_complex and isinstance(b, complex): + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -630,8 +680,8 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + elif handle_complex and isinstance(a, numbers.Complex): + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -781,7 +831,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -789,16 +839,16 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -806,6 +856,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator @@ -825,8 +877,10 @@ def __pow__(a, b): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented def __rpow__(b, a): """a ** b""" diff --git a/Lib/ftplib.py b/Lib/ftplib.py index a56e0c3085701b..50771e8c17c250 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. @@ -900,11 +900,17 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): def test(): '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + Usage: ftplib [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... - -d dir - -l list - -p password + Options: + -d increase debugging level + -r[file] set alternate ~/.netrc file + + Commands: + -l[dir] list directory + -d[dir] change the current directory + -p toggle passive and active mode + file retrieve the file and write it to stdout ''' if len(sys.argv) < 2: @@ -930,15 +936,14 @@ def test(): netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") + print("Could not open account file -- using anonymous login.", + file=sys.stderr) else: try: userid, acct, passwd = netrcobj.authenticators(host) - except KeyError: + except (KeyError, TypeError): # no account for host - sys.stderr.write( - "No account -- using anonymous login.") + print("No account -- using anonymous login.", file=sys.stderr) ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': @@ -951,7 +956,9 @@ def test(): ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) + sys.stdout.buffer.write, 1024) + sys.stdout.buffer.flush() + sys.stdout.flush() ftp.quit() diff --git a/Lib/functools.py b/Lib/functools.py index 55990e742bf23f..a10493f0e25360 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -18,6 +18,7 @@ from collections import namedtuple # import types, weakref # Deferred to single_dispatch() from reprlib import recursive_repr +from types import MethodType from _thread import RLock # Avoid importing types, so we can speedup import time @@ -285,7 +286,7 @@ def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - if hasattr(func, "func"): + if isinstance(func, partial): args = func.args + args keywords = {**func.keywords, **keywords} func = func.func @@ -303,13 +304,18 @@ def __call__(self, /, *args, **keywords): @recursive_repr() def __repr__(self): - qualname = type(self).__qualname__ + cls = type(self) + qualname = cls.__qualname__ + module = cls.__module__ args = [repr(self.func)] args.extend(repr(x) for x in self.args) args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + return f"{module}.{qualname}({', '.join(args)})" + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return MethodType(self, obj) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -373,22 +379,20 @@ def __init__(self, func, /, *args, **keywords): self.keywords = keywords def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(cls_or_self, *self.args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self + _method.__partialmethod__ = self return _method def __get__(self, obj, cls=None): @@ -424,6 +428,17 @@ def _unwrap_partial(func): func = func.func return func +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + ################################################################################ ### LRU Cache function decorator ################################################################################ @@ -662,7 +677,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from https://www.python.org/download/releases/2.3/mro/. + Adapted from https://docs.python.org/3/howto/mro.html. """ result = [] @@ -907,7 +922,6 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') @@ -957,7 +971,11 @@ def __get__(self, obj, cls=None): return _method dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') def _method(*args, **kwargs): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 1bd5b3897c3af9..ba7b0a13c7f81d 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -7,8 +7,8 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat'] # Does a path exist? @@ -22,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -57,6 +66,21 @@ def islink(path): return stat.S_ISLNK(st.st_mode) +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d774e..e5fd04fe12a7ee 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -105,7 +105,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. diff --git a/Lib/getpass.py b/Lib/getpass.py index 8b42c0a536b4c4..bd0097ced94c5e 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -156,7 +156,11 @@ def getuser(): First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +168,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/glob.py b/Lib/glob.py index a7256422d520fb..574e5ad51b601d 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -132,7 +142,8 @@ def glob1(dirname, pattern): def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False): assert _isrecursive(pattern) - yield pattern[:0] + if not dirname or _isdir(dirname, dir_fd): + yield pattern[:0] yield from _rlistdir(dirname, dir_fd, dironly, include_hidden=include_hidden) @@ -248,4 +259,293 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Abstract methods + + @staticmethod + def lexists(path): + """Implements os.path.lexists(). + """ + raise NotImplementedError + + @staticmethod + def scandir(path): + """Implements os.scandir(). + """ + raise NotImplementedError + + @staticmethod + def add_slash(path): + """Returns a path with a trailing slash added. + """ + raise NotImplementedError + + @staticmethod + def concat_path(path, text): + """Implements path concatenation. + """ + raise NotImplementedError + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + raise NotImplementedError + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.parse_entry(entry) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.parse_entry(entry) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + elif self.lexists(path): + yield path + + +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/gzip.py b/Lib/gzip.py index 177f9080dc5af8..ba753ce3050dd8 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -15,7 +15,8 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 @@ -178,9 +179,10 @@ def __init__(self, filename=None, mode=None, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. - The mtime argument is an optional numeric timestamp to be written - to the last modification time field in the stream when compressing. - If omitted or None, the current time is used. + The optional mtime argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If mtime is omitted or None, the current time is used. Use mtime = 0 + to generate a compressed stream that does not depend on creation time. """ @@ -349,7 +351,7 @@ def closed(self): def close(self): fileobj = self.fileobj - if fileobj is None: + if fileobj is None or self._buffer.closed: return try: if self.mode == WRITE: @@ -578,27 +580,6 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255) - - def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None): """Compress data in one shot and return the compressed string. @@ -606,15 +587,14 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None): mtime can be used to set the modification time. The modification time is set to the current time by default. """ - if mtime == 0: - # Use zlib as it creates the header with 0 mtime by default. - # This is faster and with less overhead. - return zlib.compress(data, level=compresslevel, wbits=31) - header = _create_simple_gzip_header(compresslevel, mtime) - trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff)) - # Wbits=-15 creates a raw deflate block. - return (header + zlib.compress(data, level=compresslevel, wbits=-15) + - trailer) + # Wbits=31 automatically includes a gzip header and trailer. + gzip_data = zlib.compress(data, level=compresslevel, wbits=31) + if mtime is None: + mtime = time.time() + # Reuse gzip header created by zlib, replace mtime and OS byte for + # consistency. + header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255) + return header + gzip_data[10:] def decompress(data): diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 1b16441cb60ba7..da0577023cf47d 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -187,7 +187,7 @@ def __hash_new(name, data=b'', **kwargs): try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt + from _hashlib import scrypt # noqa: F401 except ImportError: pass diff --git a/Lib/heapq.py b/Lib/heapq.py index 2fd9d1ff4bf827..9649da251f2a83 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -42,7 +42,7 @@ property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory -representation for a tournament. The numbers below are `k', not a[k]: +representation for a tournament. The numbers below are 'k', not a[k]: 0 @@ -55,7 +55,7 @@ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications @@ -78,7 +78,7 @@ not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled -time. When an event schedule other events for execution, they are +time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). @@ -91,14 +91,14 @@ Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted -sequences, which size is usually related to the amount of CPU memory), +sequences, whose size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way -to that. If, using all the memory available to hold a tournament, you -replace and percolate items that happen to fit the current run, you'll -produce runs which are twice the size of the memory for random input, -and much better for input fuzzily ordered. +to achieve that. If, using all the memory available to hold a +tournament, you replace and percolate items that happen to fit the +current run, you'll produce runs which are twice the size of the +memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over @@ -110,7 +110,7 @@ effective! In a word, heaps are useful memory structures to know. I use them in -a few applications, and I think it is good to keep a `heap' module +a few applications, and I think it is good to keep a 'heap' module around. :-) -------------------- diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db954ca..8b4eb2fe741e60 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -53,7 +53,7 @@ def __init__(self, key, msg=None, digestmod=''): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") if _hashopenssl and isinstance(digestmod, (str, _functype)): try: diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index e093a1fec4dffc..d64741ec0dd29a 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -9,7 +9,7 @@ class HTTPStatus: Status codes from the following RFCs are all observed: - * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 9110: HTTP Semantics, obsoletes 7231, which obsoleted 2616 * RFC 6585: Additional HTTP Status Codes * RFC 3229: Delta encoding in HTTP * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 @@ -26,7 +26,6 @@ class HTTPStatus: def __new__(cls, value, phrase, description=''): obj = int.__new__(cls, value) obj._value_ = value - obj.phrase = phrase obj.description = description return obj @@ -115,22 +114,25 @@ def is_server_error(self): 'Client must specify Content-Length') PRECONDITION_FAILED = (412, 'Precondition Failed', 'Precondition in headers is false') - REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large', - 'Entity is too large') - REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long', + CONTENT_TOO_LARGE = (413, 'Content Too Large', + 'Content is too large') + REQUEST_ENTITY_TOO_LARGE = CONTENT_TOO_LARGE + URI_TOO_LONG = (414, 'URI Too Long', 'URI is too long') + REQUEST_URI_TOO_LONG = URI_TOO_LONG UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', 'Entity body in unsupported format') - REQUESTED_RANGE_NOT_SATISFIABLE = (416, - 'Requested Range Not Satisfiable', + RANGE_NOT_SATISFIABLE = (416, 'Range Not Satisfiable', 'Cannot satisfy request range') + REQUESTED_RANGE_NOT_SATISFIABLE = RANGE_NOT_SATISFIABLE EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') IM_A_TEAPOT = (418, 'I\'m a Teapot', 'Server refuses to brew coffee because it is a teapot.') MISDIRECTED_REQUEST = (421, 'Misdirected Request', 'Server is not able to produce a response') - UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity' + UNPROCESSABLE_CONTENT = 422, 'Unprocessable Content' + UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT LOCKED = 423, 'Locked' FAILED_DEPENDENCY = 424, 'Failed Dependency' TOO_EARLY = 425, 'Too Early' @@ -177,7 +179,7 @@ class HTTPMethod: Methods from the following RFCs are all observed: - * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFF 9110: HTTP Semantics, obsoletes 7231, which obsoleted 2616 * RFC 5789: PATCH Method for HTTP """ def __new__(cls, value, description): diff --git a/Lib/http/client.py b/Lib/http/client.py index b35b1d6368aae7..fab90a0ba4eb83 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -172,6 +172,13 @@ def _encode(data, name='data'): "if you want to send it encoded in UTF-8." % (name.title(), data[err.start:err.end], name)) from None +def _strip_ipv6_iface(enc_name: bytes) -> bytes: + """Remove interface scope from IPv6 address.""" + enc_name, percent, _ = enc_name.partition(b"%") + if percent: + assert enc_name.startswith(b'['), enc_name + enc_name += b']' + return enc_name class HTTPMessage(email.message.Message): # XXX The only usage of this method is in @@ -658,6 +665,8 @@ def read1(self, n=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def peek(self, n=-1): @@ -682,6 +691,8 @@ def readline(self, limit=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def _read1_chunked(self, n): @@ -925,17 +936,23 @@ def _get_hostport(self, host, port): host = host[:i] else: port = self.default_port - if host and host[0] == '[' and host[-1] == ']': - host = host[1:-1] + if host and host[0] == '[' and host[-1] == ']': + host = host[1:-1] return (host, port) def set_debuglevel(self, level): self.debuglevel = level + def _wrap_ipv6(self, ip): + if b':' in ip and ip[0] != b'['[0]: + return b"[" + ip + b"]" + return ip + def _tunnel(self): connect = b"CONNECT %s:%d %s\r\n" % ( - self._tunnel_host.encode("idna"), self._tunnel_port, + self._wrap_ipv6(self._tunnel_host.encode("idna")), + self._tunnel_port, self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): @@ -1008,7 +1025,7 @@ def close(self): response.close() def send(self, data): - """Send `data' to the server. + """Send 'data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. """ @@ -1120,10 +1137,10 @@ def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): """Send a request to the server. - `method' specifies an HTTP request method, e.g. 'GET'. - `url' specifies the object being requested, e.g. '/index.html'. - `skip_host' if True does not add automatically a 'Host:' header - `skip_accept_encoding' if True does not add automatically an + 'method' specifies an HTTP request method, e.g. 'GET'. + 'url' specifies the object being requested, e.g. '/index.html'. + 'skip_host' if True does not add automatically a 'Host:' header + 'skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ @@ -1194,7 +1211,7 @@ def putrequest(self, method, url, skip_host=False, netloc_enc = netloc.encode("ascii") except UnicodeEncodeError: netloc_enc = netloc.encode("idna") - self.putheader('Host', netloc_enc) + self.putheader('Host', _strip_ipv6_iface(netloc_enc)) else: if self._tunnel_host: host = self._tunnel_host @@ -1210,9 +1227,9 @@ def putrequest(self, method, url, skip_host=False, # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - - if host.find(':') >= 0: - host_enc = b'[' + host_enc + b']' + host_enc = self._wrap_ipv6(host_enc) + if ":" in host: + host_enc = _strip_ipv6_iface(host_enc) if port == self.default_port: self.putheader('Host', host_enc) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index bd89370e16831e..fb0fd2e97999af 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1986,7 +1986,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. curl and lynx use this file format, too. + 'cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if diff --git a/Lib/http/server.py b/Lib/http/server.py index ee7a9b6aa55b88..2d010649e56b51 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -129,7 +129,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = True def server_bind(self): """Override server_bind to store the server name.""" @@ -897,7 +898,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt index 731fabd185fbbf..a601b25b5f838f 100644 --- a/Lib/idlelib/HISTORY.txt +++ b/Lib/idlelib/HISTORY.txt @@ -277,7 +277,7 @@ Command to format a paragraph. Debug menu: JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer -automaticall pops up when you get a traceback. +automatically pops up when you get a traceback. Windows menu: diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/News3.txt similarity index 98% rename from Lib/idlelib/NEWS.txt rename to Lib/idlelib/News3.txt index f258797c6e0fb3..b1b652dc562c8e 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/News3.txt @@ -1,9 +1,38 @@ +What's New in IDLE 3.13.0 +(since 3.12.0) +Released on 2024-10-xx +========================= + + +gh-78955: Use user-selected color theme for Help => IDLE Doc. + +gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. + +gh-72284: Improve the lists of features, editor key bindings, +and shell key bingings in the IDLE doc. + +gh-113903: Fix rare failure of test.test_idle, in test_configdialog. + +gh-113729: Fix the "Help -> IDLE Doc" menu bug in 3.11.7 and 3.12.1. + +gh-57795: Enter selected text into the Find box when opening +a Replace dialog. Patch by Roger Serwy and Zackery Spytz. + +gh-113269: Fix test_editor hang on macOS Catalina. +Patch by Terry Reedy. + +gh-112939: Fix processing unsaved files when quitting IDLE on macOS. +Patch by Ronald Oussoren and Christopher Chavez. + +gh-79871: Add docstrings to debugger.py. Fix two bugs in +test_debugger and expand coverage by 47%. Patch by Anthony Shaw. + + What's New in IDLE 3.12.0 (since 3.11.0) Released on 2023-10-02 ========================= - gh-104719: Remove IDLE's modification of tokenize.tabsize and test other uses of tokenize data and methods. @@ -539,14 +568,14 @@ bpo-33679: Enable theme-specific color configuration for Code Context. color setting, default or custom, on the extensions tab, that applied to all themes.) For built-in themes, the foreground is the same as normal text and the background is a contrasting gray. Context colors for -custom themes are set on the Hightlights tab along with other colors. +custom themes are set on the Highlights tab along with other colors. When one starts IDLE from a console and loads a custom theme without definitions for 'context', one will see a warning message on the console. bpo-33642: Display up to maxlines non-blank lines for Code Context. If there is no current context, show a single blank line. (Previously, -the Code Contex had numlines lines, usually with some blank.) The use +the Code Context had numlines lines, usually with some blank.) The use of a new option, 'maxlines' (default 15), avoids possible interference with user settings of the old option, 'numlines' (default 3). @@ -700,7 +729,7 @@ not affect their keyset-specific customization after 3.6.3. and vice versa. Initial patch by Charles Wohlganger, revised by Terry Jan Reedy. -bpo-31051: Rearrange condigdialog General tab. +bpo-31051: Rearrange configdialog General tab. Sort non-Help options into Window (Shell+Editor) and Editor (only). Leave room for the addition of new options. Patch by Terry Jan Reedy. diff --git a/Lib/idlelib/TODO.txt b/Lib/idlelib/TODO.txt index e2f1ac0f274001..41b86b0c6d5bbd 100644 --- a/Lib/idlelib/TODO.txt +++ b/Lib/idlelib/TODO.txt @@ -179,7 +179,7 @@ it -- i.e. you can only edit the current command, and the cursor can't escape from the command area. (Albert Brandl) - Set X11 class to "idle/Idle", set icon and title to something -beginning with "idle" -- for window manangers. (Randall Hopper) +beginning with "idle" -- for window managers. (Randall Hopper) - Config files editable through a preferences dialog. (me) DONE diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 4fe64dced60aca..8b9060e57072ea 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -250,9 +250,11 @@ def closure(): class Nested_in_closure: pass ModuleBrowser(parent, file, _htest=True) + if __name__ == "__main__": if len(sys.argv) == 1: # If pass file on command line, unittest fails. from unittest import main main('idlelib.idle_test.test_browser', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_module_browser) diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py index 278546064adde2..9386376058c791 100644 --- a/Lib/idlelib/calltip_w.py +++ b/Lib/idlelib/calltip_w.py @@ -193,6 +193,7 @@ def calltip_hide(event): text.focus_set() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_calltip_w', verbosity=2, exit=False) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 2b09d79470b47c..6a5acac9be8888 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -158,8 +158,9 @@ def __init__(self, _utest=False): self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg + + # See https://bugs.python.org/issue4630#msg356516 for following. # self.blink_off_time = <first editor text>['insertofftime'] - # See https:/bugs.python.org/issue4630, msg356516. if not _utest: self.CreateConfigHandlers() @@ -597,7 +598,9 @@ def GetCoreKeys(self, keySetName=None): problem getting any core binding there will be an 'ultimate last resort fallback' to the CUA-ish bindings defined here. """ + # TODO: = dict(sorted([(v-event, keys), ...]))? keyBindings={ + # vitual-event: list of key events. '<<copy>>': ['<Control-c>', '<Control-C>'], '<<cut>>': ['<Control-x>', '<Control-X>'], '<<paste>>': ['<Control-v>', '<Control-V>'], @@ -880,7 +883,7 @@ def _dump(): # htest # (not really, but ignore in coverage) line, crc = 0, 0 def sprint(obj): - global line, crc + nonlocal line, crc txt = str(obj) line += 1 crc = crc32(txt.encode(encoding='utf-8'), crc) @@ -889,7 +892,7 @@ def sprint(obj): def dumpCfg(cfg): print('\n', cfg, '\n') # Cfg has variable '0xnnnnnnnn' address. - for key in sorted(cfg.keys()): + for key in sorted(cfg): sections = cfg[key].sections() sprint(key) sprint(sections) @@ -904,8 +907,11 @@ def dumpCfg(cfg): dumpCfg(idleConf.userCfg) print('\nlines = ', line, ', crc = ', crc, sep='') + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_config', verbosity=2, exit=False) - # Run revised _dump() as htest? + _dump() + # Run revised _dump() (700+ lines) as htest? More sorting. + # Perhaps as window with tabs for textviews, making it config viewer. diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b6..e5f67e8d4069ee 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index cda7966d558a51..eedf97bf74fe6a 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -211,14 +211,8 @@ def help(self): contents=help_common+help_pages.get(page, '')) def deactivate_current_config(self): - """Remove current key bindings. - Iterate over window instances defined in parent and remove - the keybindings. - """ - # Before a config is saved, some cleanup of current - # config must be done - remove the previous keybindings. - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + """Remove current key bindings in current windows.""" + for instance in self.parent.instance_dict: instance.RemoveKeybindings() def activate_config_changes(self): @@ -227,8 +221,7 @@ def activate_config_changes(self): Dynamically update the current parent window instances with some of the configuration changes. """ - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + for instance in self.parent.instance_dict: instance.ResetColorizer() instance.ResetFont() instance.set_notabs_indentwidth() @@ -583,22 +576,23 @@ def create_page_highlight(self): (*)theme_message: Label """ self.theme_elements = { - 'Normal Code or Text': ('normal', '00'), - 'Code Context': ('context', '01'), - 'Python Keywords': ('keyword', '02'), - 'Python Definitions': ('definition', '03'), - 'Python Builtins': ('builtin', '04'), - 'Python Comments': ('comment', '05'), - 'Python Strings': ('string', '06'), - 'Selected Text': ('hilite', '07'), - 'Found Text': ('hit', '08'), - 'Cursor': ('cursor', '09'), - 'Editor Breakpoint': ('break', '10'), - 'Shell Prompt': ('console', '11'), - 'Error Text': ('error', '12'), - 'Shell User Output': ('stdout', '13'), - 'Shell User Exception': ('stderr', '14'), - 'Line Number': ('linenumber', '16'), + # Display-name: internal-config-tag-name. + 'Normal Code or Text': 'normal', + 'Code Context': 'context', + 'Python Keywords': 'keyword', + 'Python Definitions': 'definition', + 'Python Builtins': 'builtin', + 'Python Comments': 'comment', + 'Python Strings': 'string', + 'Selected Text': 'hilite', + 'Found Text': 'hit', + 'Cursor': 'cursor', + 'Editor Breakpoint': 'break', + 'Shell Prompt': 'console', + 'Error Text': 'error', + 'Shell User Output': 'stdout', + 'Shell User Exception': 'stderr', + 'Line Number': 'linenumber', } self.builtin_name = tracers.add( StringVar(self), self.var_changed_builtin_name) @@ -658,7 +652,7 @@ def tem(event, elem=element): # event.widget.winfo_top_level().highlight_target.set(elem) self.highlight_target.set(elem) text.tag_bind( - self.theme_elements[element][0], '<ButtonPress-1>', tem) + self.theme_elements[element], '<ButtonPress-1>', tem) text['state'] = 'disabled' self.style.configure('frame_color_set.TFrame', borderwidth=1, relief='solid') @@ -765,8 +759,7 @@ def load_theme_cfg(self): self.builtinlist.SetMenu(item_list, item_list[0]) self.set_theme_type() # Load theme element option menu. - theme_names = list(self.theme_elements.keys()) - theme_names.sort(key=lambda x: self.theme_elements[x][1]) + theme_names = list(self.theme_elements) self.targetlist.SetMenu(theme_names, theme_names[0]) self.paint_theme_sample() self.set_highlight_target() @@ -893,7 +886,7 @@ def on_new_color_set(self): new_color = self.color.get() self.style.configure('frame_color_set.TFrame', background=new_color) plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - sample_element = self.theme_elements[self.highlight_target.get()][0] + sample_element = self.theme_elements[self.highlight_target.get()] self.highlight_sample.tag_config(sample_element, **{plane: new_color}) theme = self.custom_name.get() theme_element = sample_element + '-' + plane @@ -1007,7 +1000,7 @@ def set_color_sample(self): frame_color_set """ # Set the color sample area. - tag = self.theme_elements[self.highlight_target.get()][0] + tag = self.theme_elements[self.highlight_target.get()] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' color = self.highlight_sample.tag_cget(tag, plane) self.style.configure('frame_color_set.TFrame', background=color) @@ -1037,7 +1030,7 @@ def paint_theme_sample(self): else: # User theme theme = self.custom_name.get() for element_title in self.theme_elements: - element = self.theme_elements[element_title][0] + element = self.theme_elements[element_title] colors = idleConf.GetHighlight(theme, element) if element == 'cursor': # Cursor sample needs special painting. colors['background'] = idleConf.GetHighlight( @@ -1477,12 +1470,13 @@ def load_keys_list(self, keyset_name): reselect = True list_index = self.bindingslist.index(ANCHOR) keyset = idleConf.GetKeySet(keyset_name) - bind_names = list(keyset.keys()) + # 'set' is dict mapping virtual event to list of key events. + bind_names = list(keyset) bind_names.sort() self.bindingslist.delete(0, END) for bind_name in bind_names: key = ' '.join(keyset[bind_name]) - bind_name = bind_name[2:-2] # Trim off the angle brackets. + bind_name = bind_name[2:-2] # Trim double angle brackets. if keyset_name in changes['keys']: # Handle any unsaved changes to this key set. if bind_name in changes['keys'][keyset_name]: diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index 452c62b42655b3..d90dbcd11f9f61 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -1,3 +1,20 @@ +"""Debug user code with a GUI interface to a subclass of bdb.Bdb. + +The Idb idb and Debugger gui instances each need a reference to each +other or to an rpc proxy for each other. + +If IDLE is started with '-n', so that user code and idb both run in the +IDLE process, Debugger is called without an idb. Debugger.__init__ +calls Idb with its incomplete self. Idb.__init__ stores gui and gui +then stores idb. + +If IDLE is started normally, so that user code executes in a separate +process, debugger_r.start_remote_debugger is called, executing in the +IDLE process. It calls 'start the debugger' in the remote process, +which calls Idb with a gui proxy. Then Debugger is called in the IDLE +for more. +""" + import bdb import os @@ -10,66 +27,95 @@ class Idb(bdb.Bdb): + "Supply user_line and user_exception functions for Bdb." def __init__(self, gui): - self.gui = gui # An instance of Debugger or proxy of remote. - bdb.Bdb.__init__(self) + self.gui = gui # An instance of Debugger or proxy thereof. + super().__init__() def user_line(self, frame): - if self.in_rpc_code(frame): + """Handle a user stopping or breaking at a line. + + Convert frame to a string and send it to gui. + """ + if _in_rpc_code(frame): self.set_step() return - message = self.__frame2message(frame) + message = _frame2message(frame) try: self.gui.interaction(message, frame) except TclError: # When closing debugger window with [x] in 3.x pass - def user_exception(self, frame, info): - if self.in_rpc_code(frame): + def user_exception(self, frame, exc_info): + """Handle an the occurrence of an exception.""" + if _in_rpc_code(frame): self.set_step() return - message = self.__frame2message(frame) - self.gui.interaction(message, frame, info) - - def in_rpc_code(self, frame): - if frame.f_code.co_filename.count('rpc.py'): - return True - else: - prev_frame = frame.f_back - prev_name = prev_frame.f_code.co_filename - if 'idlelib' in prev_name and 'debugger' in prev_name: - # catch both idlelib/debugger.py and idlelib/debugger_r.py - # on both Posix and Windows - return False - return self.in_rpc_code(prev_frame) - - def __frame2message(self, frame): - code = frame.f_code - filename = code.co_filename - lineno = frame.f_lineno - basename = os.path.basename(filename) - message = f"{basename}:{lineno}" - if code.co_name != "?": - message = f"{message}: {code.co_name}()" - return message + message = _frame2message(frame) + self.gui.interaction(message, frame, exc_info) + +def _in_rpc_code(frame): + "Determine if debugger is within RPC code." + if frame.f_code.co_filename.count('rpc.py'): + return True # Skip this frame. + else: + prev_frame = frame.f_back + if prev_frame is None: + return False + prev_name = prev_frame.f_code.co_filename + if 'idlelib' in prev_name and 'debugger' in prev_name: + # catch both idlelib/debugger.py and idlelib/debugger_r.py + # on both Posix and Windows + return False + return _in_rpc_code(prev_frame) + +def _frame2message(frame): + """Return a message string for frame.""" + code = frame.f_code + filename = code.co_filename + lineno = frame.f_lineno + basename = os.path.basename(filename) + message = f"{basename}:{lineno}" + if code.co_name != "?": + message = f"{message}: {code.co_name}()" + return message class Debugger: - - vstack = vsource = vlocals = vglobals = None + """The debugger interface. + + This class handles the drawing of the debugger window and + the interactions with the underlying debugger session. + """ + vstack = None + vsource = None + vlocals = None + vglobals = None + stackviewer = None + localsviewer = None + globalsviewer = None def __init__(self, pyshell, idb=None): + """Instantiate and draw a debugger window. + + :param pyshell: An instance of the PyShell Window + :type pyshell: :class:`idlelib.pyshell.PyShell` + + :param idb: An instance of the IDLE debugger (optional) + :type idb: :class:`idlelib.debugger.Idb` + """ if idb is None: idb = Idb(self) self.pyshell = pyshell self.idb = idb # If passed, a proxy of remote instance. self.frame = None self.make_gui() - self.interacting = 0 + self.interacting = False self.nesting_level = 0 def run(self, *args): + """Run the debugger.""" # Deal with the scenario where we've already got a program running # in the debugger and we want to start another. If that is the case, # our second 'run' was invoked from an event dispatched not from @@ -104,12 +150,13 @@ def run(self, *args): self.root.after(100, lambda: self.run(*args)) return try: - self.interacting = 1 + self.interacting = True return self.idb.run(*args) finally: - self.interacting = 0 + self.interacting = False def close(self, event=None): + """Close the debugger and window.""" try: self.quit() except Exception: @@ -127,6 +174,7 @@ def close(self, event=None): self.top.destroy() def make_gui(self): + """Draw the debugger gui on the screen.""" pyshell = self.pyshell self.flist = pyshell.flist self.root = root = pyshell.root @@ -135,11 +183,11 @@ def make_gui(self): self.top.wm_iconname("Debug") top.wm_protocol("WM_DELETE_WINDOW", self.close) self.top.bind("<Escape>", self.close) - # + self.bframe = bframe = Frame(top) self.bframe.pack(anchor="w") self.buttons = bl = [] - # + self.bcont = b = Button(bframe, text="Go", command=self.cont) bl.append(b) self.bstep = b = Button(bframe, text="Step", command=self.step) @@ -150,14 +198,14 @@ def make_gui(self): bl.append(b) self.bret = b = Button(bframe, text="Quit", command=self.quit) bl.append(b) - # + for b in bl: b.configure(state="disabled") b.pack(side="left") - # + self.cframe = cframe = Frame(bframe) self.cframe.pack(side="left") - # + if not self.vstack: self.__class__.vstack = BooleanVar(top) self.vstack.set(1) @@ -180,20 +228,20 @@ def make_gui(self): self.bglobals = Checkbutton(cframe, text="Globals", command=self.show_globals, variable=self.vglobals) self.bglobals.grid(row=1, column=1) - # + self.status = Label(top, anchor="w") self.status.pack(anchor="w") self.error = Label(top, anchor="w") self.error.pack(anchor="w", fill="x") self.errorbg = self.error.cget("background") - # + self.fstack = Frame(top, height=1) self.fstack.pack(expand=1, fill="both") self.flocals = Frame(top) self.flocals.pack(expand=1, fill="both") self.fglobals = Frame(top, height=1) self.fglobals.pack(expand=1, fill="both") - # + if self.vstack.get(): self.show_stack() if self.vlocals.get(): @@ -204,7 +252,7 @@ def make_gui(self): def interaction(self, message, frame, info=None): self.frame = frame self.status.configure(text=message) - # + if info: type, value, tb = info try: @@ -223,28 +271,28 @@ def interaction(self, message, frame, info=None): tb = None bg = self.errorbg self.error.configure(text=m1, background=bg) - # + sv = self.stackviewer if sv: stack, i = self.idb.get_stack(self.frame, tb) sv.load_stack(stack, i) - # + self.show_variables(1) - # + if self.vsource.get(): self.sync_source_line() - # + for b in self.buttons: b.configure(state="normal") - # + self.top.wakeup() # Nested main loop: Tkinter's main loop is not reentrant, so use # Tcl's vwait facility, which reenters the event loop until an - # event handler sets the variable we're waiting on + # event handler sets the variable we're waiting on. self.nesting_level += 1 self.root.tk.call('vwait', '::idledebugwait') self.nesting_level -= 1 - # + for b in self.buttons: b.configure(state="disabled") self.status.configure(text="") @@ -288,8 +336,6 @@ def quit(self): def abort_loop(self): self.root.tk.call('set', '::idledebugwait', '1') - stackviewer = None - def show_stack(self): if not self.stackviewer and self.vstack.get(): self.stackviewer = sv = StackViewer(self.fstack, self.flist, self) @@ -311,9 +357,6 @@ def show_frame(self, stackitem): self.frame = stackitem[0] # lineno is stackitem[1] self.show_variables() - localsviewer = None - globalsviewer = None - def show_locals(self): lv = self.localsviewer if self.vlocals.get(): @@ -354,26 +397,32 @@ def show_variables(self, force=0): if gv: gv.load_dict(gdict, force, self.pyshell.interp.rpcclt) - def set_breakpoint_here(self, filename, lineno): + def set_breakpoint(self, filename, lineno): + """Set a filename-lineno breakpoint in the debugger. + + Called from self.load_breakpoints and EW.setbreakpoint + """ self.idb.set_break(filename, lineno) - def clear_breakpoint_here(self, filename, lineno): + def clear_breakpoint(self, filename, lineno): self.idb.clear_break(filename, lineno) def clear_file_breaks(self, filename): self.idb.clear_all_file_breaks(filename) def load_breakpoints(self): - "Load PyShellEditorWindow breakpoints into subprocess debugger" + """Load PyShellEditorWindow breakpoints into subprocess debugger.""" for editwin in self.pyshell.flist.inversedict: filename = editwin.io.filename try: for lineno in editwin.breakpoints: - self.set_breakpoint_here(filename, lineno) + self.set_breakpoint(filename, lineno) except AttributeError: continue + class StackViewer(ScrolledList): + "Code stack viewer for debugger GUI." def __init__(self, master, flist, gui): if macosx.isAquaTk(): @@ -414,12 +463,12 @@ def load_stack(self, stack, index=None): self.select(index) def popup_event(self, event): - "override base method" + "Override base method." if self.stack: return ScrolledList.popup_event(self, event) def fill_menu(self): - "override base method" + "Override base method." menu = self.menu menu.add_command(label="Go to source line", command=self.goto_source_line) @@ -427,12 +476,12 @@ def fill_menu(self): command=self.show_stack_frame) def on_select(self, index): - "override base method" + "Override base method." if 0 <= index < len(self.stack): self.gui.show_frame(self.stack[index]) def on_double(self, index): - "override base method" + "Override base method." self.show_source(index) def goto_source_line(self): @@ -457,12 +506,13 @@ def show_source(self, index): class NamespaceViewer: + "Global/local namespace viewer for debugger GUI." - def __init__(self, master, title, dict=None): + def __init__(self, master, title, odict=None): # XXX odict never passed. width = 0 height = 40 - if dict: - height = 20*len(dict) # XXX 20 == observed height of Entry widget + if odict: + height = 20*len(odict) # XXX 20 == observed height of Entry widget self.master = master self.title = title import reprlib @@ -483,24 +533,24 @@ def __init__(self, master, title, dict=None): canvas["yscrollcommand"] = vbar.set self.subframe = subframe = Frame(canvas) self.sfid = canvas.create_window(0, 0, window=subframe, anchor="nw") - self.load_dict(dict) + self.load_dict(odict) - dict = -1 + prev_odict = -1 # Needed for initial comparison below. - def load_dict(self, dict, force=0, rpc_client=None): - if dict is self.dict and not force: + def load_dict(self, odict, force=0, rpc_client=None): + if odict is self.prev_odict and not force: return subframe = self.subframe frame = self.frame for c in list(subframe.children.values()): c.destroy() - self.dict = None - if not dict: + self.prev_odict = None + if not odict: l = Label(subframe, text="None") l.grid(row=0, column=0) else: #names = sorted(dict) - ### + # # Because of (temporary) limitations on the dict_keys type (not yet # public or pickleable), have the subprocess to send a list of # keys, not a dict_keys object. sorted() will take a dict_keys @@ -509,13 +559,13 @@ def load_dict(self, dict, force=0, rpc_client=None): # There is also an obscure bug in sorted(dict) where the # interpreter gets into a loop requesting non-existing dict[0], # dict[1], dict[2], etc from the debugger_r.DictProxy. - ### - keys_list = dict.keys() + # TODO recheck above; see debugger_r 159ff, debugobj 60. + keys_list = odict.keys() names = sorted(keys_list) - ### + row = 0 for name in names: - value = dict[name] + value = odict[name] svalue = self.repr.repr(value) # repr(value) # Strip extra quotes caused by calling repr on the (already) # repr'd value sent across the RPC interface: @@ -527,7 +577,7 @@ def load_dict(self, dict, force=0, rpc_client=None): l.insert(0, svalue) l.grid(row=row, column=1, sticky="nw") row = row+1 - self.dict = dict + self.prev_odict = odict # XXX Could we use a <Configure> callback for the following? subframe.update_idletasks() # Alas! width = subframe.winfo_reqwidth() @@ -544,6 +594,7 @@ def load_dict(self, dict, force=0, rpc_client=None): def close(self): self.frame.destroy() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index 26204438858d8a..ad3355d9f82765 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -125,16 +125,16 @@ def frame_attr(self, fid, name): def frame_globals(self, fid): frame = frametable[fid] - dict = frame.f_globals - did = id(dict) - dicttable[did] = dict + gdict = frame.f_globals + did = id(gdict) + dicttable[did] = gdict return did def frame_locals(self, fid): frame = frametable[fid] - dict = frame.f_locals - did = id(dict) - dicttable[did] = dict + ldict = frame.f_locals + did = id(ldict) + dicttable[did] = ldict return did def frame_code(self, fid): @@ -158,20 +158,17 @@ def code_filename(self, cid): def dict_keys(self, did): raise NotImplementedError("dict_keys not public or pickleable") -## dict = dicttable[did] -## return dict.keys() +## return dicttable[did].keys() - ### Needed until dict_keys is type is finished and pickealable. + ### Needed until dict_keys type is finished and pickleable. + # xxx finished. pickleable? ### Will probably need to extend rpc.py:SocketIO._proxify at that time. def dict_keys_list(self, did): - dict = dicttable[did] - return list(dict.keys()) + return list(dicttable[did].keys()) def dict_item(self, did, key): - dict = dicttable[did] - value = dict[key] - value = reprlib.repr(value) ### can't pickle module 'builtins' - return value + value = dicttable[did][key] + return reprlib.repr(value) # Can't pickle module 'builtins'. #----------end class IdbAdapter---------- diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 71d01c7070df54..fb448ece2fa25e 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -1,3 +1,5 @@ +"""Define tree items for debug stackviewer, which is only user. +""" # XXX TO DO: # - popup menu # - support partial or total redisplay @@ -17,9 +19,9 @@ myrepr.maxother = 100 class ObjectTreeItem(TreeItem): - def __init__(self, labeltext, object, setfunction=None): + def __init__(self, labeltext, object_, setfunction=None): self.labeltext = labeltext - self.object = object + self.object = object_ self.setfunction = setfunction def GetLabelText(self): return self.labeltext @@ -51,8 +53,8 @@ def GetSubList(self): item = make_objecttreeitem( str(key) + " =", value, - lambda value, key=key, object=self.object: - setattr(object, key, value)) + lambda value, key=key, object_=self.object: + setattr(object_, key, value)) sublist.append(item) return sublist @@ -85,15 +87,16 @@ def GetSubList(self): value = self.object[key] except KeyError: continue - def setfunction(value, key=key, object=self.object): - object[key] = value + def setfunction(value, key=key, object_=self.object): + object_[key] = value item = make_objecttreeitem(f"{key!r}:", value, setfunction) sublist.append(item) return sublist class DictTreeItem(SequenceTreeItem): def keys(self): - keys = list(self.object.keys()) + # TODO return sorted(self.object) + keys = list(self.object) try: keys.sort() except: @@ -110,16 +113,16 @@ def keys(self): type: ClassTreeItem, } -def make_objecttreeitem(labeltext, object, setfunction=None): - t = type(object) +def make_objecttreeitem(labeltext, object_, setfunction=None): + t = type(object_) if t in dispatch: c = dispatch[t] else: c = ObjectTreeItem - return c(labeltext, object, setfunction) + return c(labeltext, object_, setfunction) -def _object_browser(parent): # htest # +def _debug_object_browser(parent): # htest # import sys from tkinter import Toplevel top = Toplevel(parent) @@ -134,9 +137,10 @@ def _object_browser(parent): # htest # node = TreeNode(sc.canvas, None, item) node.update() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_debugobj', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_object_browser) + run(_debug_object_browser) diff --git a/Lib/idlelib/delegator.py b/Lib/idlelib/delegator.py index 55c95da8532f47..93ae8bbd43ff44 100644 --- a/Lib/idlelib/delegator.py +++ b/Lib/idlelib/delegator.py @@ -28,6 +28,7 @@ def setdelegate(self, delegate): self.resetcache() self.delegate = delegate + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_delegator', verbosity=2) diff --git a/Lib/idlelib/dynoption.py b/Lib/idlelib/dynoption.py index d5dfc3eda13f60..b8937f7106ca75 100644 --- a/Lib/idlelib/dynoption.py +++ b/Lib/idlelib/dynoption.py @@ -29,6 +29,7 @@ def SetMenu(self,valueList,value=None): if value: self.variable.set(value) + def _dyn_option_menu(parent): # htest # from tkinter import Toplevel # + StringVar, Button @@ -49,6 +50,7 @@ def update(): button = Button(top, text="Change option set", command=update) button.pack() + if __name__ == '__main__': # Only module without unittests because of intention to replace. from idlelib.idle_test.htest import run diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 69b27d0683a104..7bfa0932500d81 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,8 +166,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<3>",self.right_menu_event) text.bind('<MouseWheel>', wheel_event) - text.bind('<Button-4>', wheel_event) - text.bind('<Button-5>', wheel_event) + if text._windowingsystem == 'x11': + text.bind('<Button-4>', wheel_event) + text.bind('<Button-5>', wheel_event) text.bind('<Configure>', self.handle_winconfig) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) @@ -1043,7 +1044,9 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() - if short and long: + if short and long and not macosx.isCocoaTk(): + # Don't use both values on macOS because + # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: title = short @@ -1058,6 +1061,13 @@ def saved_change_hook(self): self.top.wm_title(title) self.top.wm_iconname(icon) + if macosx.isCocoaTk(): + # Add a proxy icon to the window title + self.top.wm_attributes("-titlepath", long) + + # Maintain the modification status for the window + self.top.wm_attributes("-modified", not self.get_saved()) + def get_saved(self): return self.undo.get_saved() @@ -1748,6 +1758,7 @@ def _editor_window(parent): # htest # # Does not stop error, neither does following # edit.text.bind("<<close-window>>", edit.close_event) + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_editor', verbosity=2, exit=False) diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index f87781d2570fe0..e27e5d32a0ff63 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -124,6 +124,7 @@ def _test(): # TODO check and convert to htest if flist.inversedict: root.mainloop() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_filelist', verbosity=2) diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 12513594b76f8f..42048ff2395fe1 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -190,7 +190,7 @@ def grep_it(self, prog, path): def _grep_dialog(parent): # htest # - from tkinter import Toplevel, Text, SEL, END + from tkinter import Toplevel, Text, SEL from tkinter.ttk import Frame, Button from idlelib.pyshell import PyShellFileList @@ -204,15 +204,17 @@ def _grep_dialog(parent): # htest # frame.pack() text = Text(frame, height=5) text.pack() + text.insert('1.0', 'import grep') def show_grep_dialog(): - text.tag_add(SEL, "1.0", END) + text.tag_add(SEL, "1.0", '1.end') grep(text, flist=flist) - text.tag_remove(SEL, "1.0", END) + text.tag_remove(SEL, "1.0", '1.end') button = Button(frame, text="Show GrepDialog", command=show_grep_dialog) button.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_grep', verbosity=2, exit=False) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 722406b81a8ae6..827d230b54e159 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -1,33 +1,31 @@ - <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> - <title>IDLE — Python 3.12.0a0 documentation + IDLE — Python 3.13.0a2 documentation - + + - - - + - + @@ -41,35 +39,48 @@ } } - + + + +
-

Next topic

@@ -1117,7 +1123,7 @@

Navigation

next |
  • - previous |
  • python logo
  • @@ -1130,7 +1136,7 @@

    Navigation

  • - 3.12.0a0 Documentation » + 3.13.0a2 Documentation »
  • @@ -1141,19 +1147,26 @@

    Navigation

    | +
  • + |
  • diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index cc027b9cef4f5b..d8613b2eadd6aa 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -33,6 +33,7 @@ from tkinter import font as tkfont from idlelib.config import idleConf +from idlelib.colorizer import color_config ## About IDLE ## @@ -102,7 +103,7 @@ def handle_starttag(self, tag, attrs): if self.level > 0: self.nested_dl = True elif tag == 'li': - s = '\n* ' if self.simplelist else '\n\n* ' + s = '\n* ' elif tag == 'dt': s = '\n\n' if not self.nested_dl else '\n' # Avoid extra line. self.nested_dl = False @@ -177,14 +178,16 @@ def __init__(self, parent, filename): normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) + color_config(self) self['font'] = (normalfont, 12) self.tag_configure('em', font=(normalfont, 12, 'italic')) self.tag_configure('h1', font=(normalfont, 20, 'bold')) self.tag_configure('h2', font=(normalfont, 18, 'bold')) self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') + self.tag_configure('pre', font=(fixedfont, 12)) + preback = self['selectbackground'] self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') + background=preback) self.tag_configure('l1', lmargin1=25, lmargin2=25) self.tag_configure('l2', lmargin1=50, lmargin2=50) self.tag_configure('l3', lmargin1=75, lmargin2=75) @@ -241,12 +244,13 @@ def __init__(self, parent, filename, title): Toplevel.__init__(self, parent) self.wm_title(title) self.protocol("WM_DELETE_WINDOW", self.destroy) - HelpFrame(self, filename).grid(column=0, row=0, sticky='nsew') + self.frame = HelpFrame(self, filename) + self.frame.grid(column=0, row=0, sticky='nsew') self.grid_columnconfigure(0, weight=1) self.grid_rowconfigure(0, weight=1) -def copy_strip(): +def copy_strip(): # pragma: no cover """Copy idle.html to idlelib/help.html, stripping trailing whitespace. Files with trailing whitespace cannot be pushed to the git cpython @@ -278,13 +282,15 @@ def copy_strip(): out.write(line.rstrip() + b'\n') print(f'{src} copied to {dst}') + def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." filename = join(abspath(dirname(__file__)), 'help.html') - if not isfile(filename): + if not isfile(filename): # pragma: no cover # Try copy_strip, present message. return - HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) + return HelpWindow(parent, filename, 'IDLE Doc (%s)' % python_version()) + if __name__ == '__main__': from unittest import main diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index cfa4ca781f087d..aa1c352897f9e7 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -129,11 +129,11 @@ def create_widgets(self): idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) - self.readme = Button(idle_buttons, text='README', width=8, + self.readme = Button(idle_buttons, text='Readme', width=8, highlightbackground=self.bg, command=self.show_readme) self.readme.pack(side=LEFT, padx=10, pady=10) - self.idle_news = Button(idle_buttons, text='NEWS', width=8, + self.idle_news = Button(idle_buttons, text='News', width=8, highlightbackground=self.bg, command=self.show_idle_news) self.idle_news.pack(side=LEFT, padx=10, pady=10) @@ -167,7 +167,7 @@ def show_readme(self): def show_idle_news(self): "Handle News button event." - self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') + self.display_file_text('About - News', 'News3.txt', 'utf-8') def display_printer_text(self, title, printer): """Create textview for built-in constants. diff --git a/Lib/idlelib/idle_test/example_stub.pyi b/Lib/idlelib/idle_test/example_stub.pyi index a9811a78d10a6e..17b58010a9d8de 100644 --- a/Lib/idlelib/idle_test/example_stub.pyi +++ b/Lib/idlelib/idle_test/example_stub.pyi @@ -1,2 +1,4 @@ +" Example to test recognition of .pyi file as Python source code. + class Example: def method(self, argument1: str, argument2: list[int]) -> None: ... diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094ee..a7293774eecaeb 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -1,38 +1,36 @@ -'''Run human tests of Idle's window, dialog, and popup widgets. - -run(*tests) -Create a master Tk window. Within that, run each callable in tests -after finding the matching test spec in this file. If tests is empty, -run an htest for each spec dict in this file after finding the matching -callable in the module named in the spec. Close the window to skip or -end the test. - -In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attribute is also X (the usual situation). -The first parameter of X must be 'parent'. When called, the parent -argument will be the root window. X must create a child Toplevel -window (or subclass thereof). The Toplevel may be a test widget or -dialog, in which case the callable is the corresponding class. Or the -Toplevel may contain the widget to be tested or set up a context in -which a test widget is invoked. In this latter case, the callable is a -wrapper function that sets up the Toplevel and other objects. Wrapper -function names, such as _editor_window', should start with '_'. +"""Run human tests of Idle's window, dialog, and popup widgets. + +run(*tests) Create a master Tk() htest window. Within that, run each +callable in tests after finding the matching test spec in this file. If +tests is empty, run an htest for each spec dict in this file after +finding the matching callable in the module named in the spec. Close +the master window to end testing. + +In a tested module, let X be a global name bound to a callable (class or +function) whose .__name__ attribute is also X (the usual situation). The +first parameter of X must be 'parent' or 'master'. When called, the +first argument will be the root window. X must create a child +Toplevel(parent/master) (or subclass thereof). The Toplevel may be a +test widget or dialog, in which case the callable is the corresponding +class. Or the Toplevel may contain the widget to be tested or set up a +context in which a test widget is invoked. In this latter case, the +callable is a wrapper function that sets up the Toplevel and other +objects. Wrapper function names, such as _editor_window', should start +with '_' and be lowercase. End the module with if __name__ == '__main__': - + from idlelib.idle_test.htest import run - run(X) + run(callable) # There could be multiple comma-separated callables. -To have wrapper functions and test invocation code ignored by coveragepy -reports, put '# htest #' on the def statement header line. - -def _wrapper(parent): # htest # - -Also make sure that the 'if __name__' line matches the above. Then have -make sure that .coveragerc includes the following. +To have wrapper functions ignored by coverage reports, tag the def +header like so: "def _wrapper(parent): # htest #". Use the same tag +for htest lines in widget code. Make sure that the 'if __name__' line +matches the above. Then have make sure that .coveragerc includes the +following: [report] exclude_lines = @@ -46,7 +44,7 @@ def _wrapper(parent): # htest # following template, with X.__name__ prepended to '_spec'. When all tests are run, the prefix is use to get X. -_spec = { +callable_spec = { 'file': '', 'kwds': {'title': ''}, 'msg': "" @@ -54,16 +52,16 @@ def _wrapper(parent): # htest # file (no .py): run() imports file.py. kwds: augmented with {'parent':root} and passed to X as **kwds. -title: an example kwd; some widgets need this, delete if not. +title: an example kwd; some widgets need this, delete line if not. msg: master window hints about testing the widget. -Modules and classes not being tested at the moment: -pyshell.PyShellEditorWindow -debugger.Debugger -autocomplete_w.AutoCompleteWindow -outwin.OutputWindow (indirectly being tested with grep test) -''' +TODO test these modules and classes: + autocomplete_w.AutoCompleteWindow + debugger.Debugger + outwin.OutputWindow (indirectly being tested with grep test) + pyshell.PyShellEditorWindow +""" import idlelib.pyshell # Set Windows DPI awareness before Tk(). from importlib import import_module @@ -91,15 +89,6 @@ def _wrapper(parent): # htest # "Force-open-calltip does not work here.\n" } -_module_browser_spec = { - 'file': 'browser', - 'kwds': {}, - 'msg': "Inspect names of module, class(with superclass if " - "applicable), methods and functions.\nToggle nested items.\n" - "Double clicking on items prints a traceback for an exception " - "that is ignored." - } - _color_delegator_spec = { 'file': 'colorizer', 'kwds': {}, @@ -109,16 +98,6 @@ def _wrapper(parent): # htest # "The default color scheme is in idlelib/config-highlight.def" } -CustomRun_spec = { - 'file': 'query', - 'kwds': {'title': 'Customize query.py Run', - '_htest': True}, - 'msg': "Enter with or [Run]. Print valid entry to Shell\n" - "Arguments are parsed into a list\n" - "Mode is currently restart True or False\n" - "Close dialog with valid entry, , [Cancel], [X]" - } - ConfigDialog_spec = { 'file': 'configdialog', 'kwds': {'title': 'ConfigDialogTest', @@ -135,6 +114,24 @@ def _wrapper(parent): # htest # "changes made have persisted." } +CustomRun_spec = { + 'file': 'query', + 'kwds': {'title': 'Customize query.py Run', + '_htest': True}, + 'msg': "Enter with or [OK]. Print valid entry to Shell\n" + "Arguments are parsed into a list\n" + "Mode is currently restart True or False\n" + "Close dialog with valid entry, , [Cancel], [X]" + } + +_debug_object_browser_spec = { + 'file': 'debugobj', + 'kwds': {}, + 'msg': "Double click on items up to the lowest level.\n" + "Attributes of the objects and related information " + "will be displayed side-by-side at each level." + } + # TODO Improve message _dyn_option_menu_spec = { 'file': 'dynoption', @@ -152,7 +149,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again', @@ -173,8 +170,8 @@ def _wrapper(parent): # htest # 'msg': "Click the 'Show GrepDialog' button.\n" "Test the various 'Find-in-files' functions.\n" "The results should be displayed in a new '*Output*' window.\n" - "'Right-click'->'Go to file/line' anywhere in the search results " - "should open that file \nin a new EditorWindow." + "'Right-click'->'Go to file/line' in the search results\n " + "should open that file in a new EditorWindow." } HelpSource_spec = { @@ -190,7 +187,7 @@ def _wrapper(parent): # htest # "Any url ('www...', 'http...') is accepted.\n" "Test Browse with and without path, as cannot unittest.\n" "[Ok] or prints valid entry to shell\n" - "[Cancel] or prints None to shell" + ", [Cancel], or [X] prints None to shell" } _io_binding_spec = { @@ -206,56 +203,36 @@ def _wrapper(parent): # htest # "Check that changes were saved by opening the file elsewhere." } -_linenumbers_drag_scrolling_spec = { - 'file': 'sidebar', +_multi_call_spec = { + 'file': 'multicall', 'kwds': {}, - 'msg': textwrap.dedent("""\ - 1. Click on the line numbers and drag down below the edge of the - window, moving the mouse a bit and then leaving it there for a while. - The text and line numbers should gradually scroll down, with the - selection updated continuously. - - 2. With the lines still selected, click on a line number above the - selected lines. Only the line whose number was clicked should be - selected. - - 3. Repeat step #1, dragging to above the window. The text and line - numbers should gradually scroll up, with the selection updated - continuously. - - 4. Repeat step #2, clicking a line number below the selection."""), + 'msg': "The following should trigger a print to console or IDLE Shell.\n" + "Entering and leaving the text area, key entry, ,\n" + ", , , \n" + ", and focusing elsewhere." } -_multi_call_spec = { - 'file': 'multicall', +_module_browser_spec = { + 'file': 'browser', 'kwds': {}, - 'msg': "The following actions should trigger a print to console or IDLE" - " Shell.\nEntering and leaving the text area, key entry, " - ",\n, , " - ", \n, and " - "focusing out of the window\nare sequences to be tested." + 'msg': textwrap.dedent(""" + "Inspect names of module, class(with superclass if applicable), + "methods and functions. Toggle nested items. Double clicking + "on items prints a traceback for an exception that is ignored.""") } _multistatus_bar_spec = { 'file': 'statusbar', 'kwds': {}, 'msg': "Ensure presence of multi-status bar below text area.\n" - "Click 'Update Status' to change the multi-status text" + "Click 'Update Status' to change the status text" } -_object_browser_spec = { - 'file': 'debugobj', - 'kwds': {}, - 'msg': "Double click on items up to the lowest level.\n" - "Attributes of the objects and related information " - "will be displayed side-by-side at each level." - } - -_path_browser_spec = { +PathBrowser_spec = { 'file': 'pathbrowser', - 'kwds': {}, + 'kwds': {'_htest': True}, 'msg': "Test for correct display of all paths in sys.path.\n" - "Toggle nested items up to the lowest level.\n" + "Toggle nested items out to the lowest level.\n" "Double clicking on an item prints a traceback\n" "for an exception that is ignored." } @@ -291,6 +268,15 @@ def _wrapper(parent): # htest # "Click [Close] or [X] to close the 'Replace Dialog'." } +_scrolled_list_spec = { + 'file': 'scrolledlist', + 'kwds': {}, + 'msg': "You should see a scrollable list of items\n" + "Selecting (clicking) or double clicking an item " + "prints the name to the console or Idle shell.\n" + "Right clicking an item will display a popup." + } + _search_dialog_spec = { 'file': 'search', 'kwds': {}, @@ -306,15 +292,6 @@ def _wrapper(parent): # htest # "Its only action is to close." } -_scrolled_list_spec = { - 'file': 'scrolledlist', - 'kwds': {}, - 'msg': "You should see a scrollable list of items\n" - "Selecting (clicking) or double clicking an item " - "prints the name to the console or Idle shell.\n" - "Right clicking an item will display a popup." - } - show_idlehelp_spec = { 'file': 'help', 'kwds': {}, @@ -322,12 +299,31 @@ def _wrapper(parent): # htest # "Text is selectable. Window is scrollable." } -_stack_viewer_spec = { +_sidebar_number_scrolling_spec = { + 'file': 'sidebar', + 'kwds': {}, + 'msg': textwrap.dedent("""\ + 1. Click on the line numbers and drag down below the edge of the + window, moving the mouse a bit and then leaving it there for a + while. The text and line numbers should gradually scroll down, + with the selection updated continuously. + + 2. With the lines still selected, click on a line number above + or below the selected lines. Only the line whose number was + clicked should be selected. + + 3. Repeat step #1, dragging to above the window. The text and + line numbers should gradually scroll up, with the selection + updated continuously. + + 4. Repeat step #2, clicking a line number below the selection."""), + } + +_stackbrowser_spec = { 'file': 'stackviewer', 'kwds': {}, 'msg': "A stacktrace for a NameError exception.\n" - "Expand 'idlelib ...' and ''.\n" - "Check that exc_value, exc_tb, and exc_type are correct.\n" + "Should have NameError and 1 traceback line." } _tooltip_spec = { @@ -370,11 +366,12 @@ def _wrapper(parent): # htest # } def run(*tests): + "Run callables in tests." root = tk.Tk() root.title('IDLE htest') root.resizable(0, 0) - # a scrollable Label like constant width text widget. + # A scrollable Label-like constant width text widget. frameLabel = tk.Frame(root, padx=10) frameLabel.pack() text = tk.Text(frameLabel, wrap='word') @@ -384,45 +381,44 @@ def run(*tests): scrollbar.pack(side='right', fill='y', expand=False) text.pack(side='left', fill='both', expand=True) - test_list = [] # List of tuples of the form (spec, callable widget) + test_list = [] # Make list of (spec, callable) tuples. if tests: for test in tests: test_spec = globals()[test.__name__ + '_spec'] test_spec['name'] = test.__name__ test_list.append((test_spec, test)) else: - for k, d in globals().items(): - if k.endswith('_spec'): - test_name = k[:-5] - test_spec = d + for key, dic in globals().items(): + if key.endswith('_spec'): + test_name = key[:-5] + test_spec = dic test_spec['name'] = test_name mod = import_module('idlelib.' + test_spec['file']) test = getattr(mod, test_name) test_list.append((test_spec, test)) + test_list.reverse() # So can pop in proper order in next_test. test_name = tk.StringVar(root) callable_object = None test_kwds = None def next_test(): - nonlocal test_name, callable_object, test_kwds if len(test_list) == 1: next_button.pack_forget() test_spec, callable_object = test_list.pop() test_kwds = test_spec['kwds'] - test_kwds['parent'] = root test_name.set('Test ' + test_spec['name']) - text.configure(state='normal') # enable text editing - text.delete('1.0','end') - text.insert("1.0",test_spec['msg']) - text.configure(state='disabled') # preserve read-only property + text['state'] = 'normal' # Enable text replacement. + text.delete('1.0', 'end') + text.insert("1.0", test_spec['msg']) + text['state'] = 'disabled' # Restore read-only property. def run_test(_=None): - widget = callable_object(**test_kwds) + widget = callable_object(root, **test_kwds) try: - print(widget.result) + print(widget.result) # Only true for query classes(?). except AttributeError: pass @@ -441,5 +437,6 @@ def close(_=None): next_test() root.mainloop() + if __name__ == '__main__': run() diff --git a/Lib/idlelib/idle_test/test_calltip.py b/Lib/idlelib/idle_test/test_calltip.py index 1ccb63b9dbd65f..28c196a42672fc 100644 --- a/Lib/idlelib/idle_test/test_calltip.py +++ b/Lib/idlelib/idle_test/test_calltip.py @@ -7,6 +7,7 @@ import types import re from idlelib.idle_test.mock_tk import Text +from test.support import MISSING_C_DOCSTRINGS # Test Class TC is used in multiple get_argspec test methods @@ -50,6 +51,8 @@ class Get_argspecTest(unittest.TestCase): # but a red buildbot is better than a user crash (as has happened). # For a simple mismatch, change the expected output to the actual. + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_builtins(self): def tiptest(obj, out): @@ -76,6 +79,7 @@ class SB: __call__ = None tiptest(list.append, '(self, object, /)' + append_doc) tiptest(List.append, '(self, object, /)' + append_doc) tiptest([].append, '(object, /)' + append_doc) + # The use of 'object' above matches the signature text. tiptest(types.MethodType, '(function, instance, /)\n' @@ -143,6 +147,8 @@ def f(): pass f.__doc__ = 'a'*300 self.assertEqual(get_spec(f), f"()\n{'a'*(calltip._MAX_COLS-3) + '...'}") + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_multiline_docstring(self): # Test fewer lines than max. self.assertEqual(get_spec(range), @@ -157,6 +163,7 @@ def test_multiline_docstring(self): bytes(int) -> bytes object of size given by the parameter initialized with null bytes bytes() -> empty bytes object''') + def test_multiline_docstring_2(self): # Test more than max lines def f(): pass f.__doc__ = 'a\n' * 15 diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 08ed76fe288294..6d75cf7aa67dcc 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -85,8 +85,8 @@ def test_load_nothing(self): self.assertEqual(parser.sections(), []) def test_load_file(self): - # Borrow test/cfgparser.1 from test_configparser. - config_path = findfile('cfgparser.1') + # Borrow test/configdata/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1', subdir='configdata') parser = config.IdleConfParser(config_path) parser.Load() @@ -274,8 +274,8 @@ def test_create_config_handlers(self): conf.CreateConfigHandlers() # Check keys are equal - self.assertCountEqual(conf.defaultCfg.keys(), conf.config_types) - self.assertCountEqual(conf.userCfg.keys(), conf.config_types) + self.assertCountEqual(conf.defaultCfg, conf.config_types) + self.assertCountEqual(conf.userCfg, conf.config_types) # Check conf parser are correct type for default_parser in conf.defaultCfg.values(): @@ -294,8 +294,8 @@ def test_create_config_handlers(self): def test_load_cfg_files(self): conf = self.new_config(_utest=True) - # Borrow test/cfgparser.1 from test_configparser. - config_path = findfile('cfgparser.1') + # Borrow test/configdata/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1', subdir='configdata') conf.defaultCfg['foo'] = config.IdleConfParser(config_path) conf.userCfg['foo'] = config.IdleUserConfParser(config_path) diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index e5d5b4013fca57..5099d093382445 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -420,20 +420,14 @@ def test_highlight_target_text_mouse(self): # Set highlight_target through clicking highlight_sample. eq = self.assertEqual d = self.page - - elem = {} - count = 0 hs = d.highlight_sample hs.focus_force() - hs.see(1.0) - hs.update_idletasks() - def tag_to_element(elem): - for element, tag in d.theme_elements.items(): - elem[tag[0]] = element - - def click_it(start): - x, y, dx, dy = hs.bbox(start) + def click_char(index): + "Simulate click on character at *index*." + hs.see(index) + hs.update_idletasks() + x, y, dx, dy = hs.bbox(index) x += dx // 2 y += dy // 2 hs.event_generate('', x=0, y=0) @@ -441,17 +435,20 @@ def click_it(start): hs.event_generate('', x=x, y=y) hs.event_generate('', x=x, y=y) - # Flip theme_elements to make the tag the key. - tag_to_element(elem) + # Reverse theme_elements to make the tag the key. + elem = {tag: element for element, tag in d.theme_elements.items()} # If highlight_sample has a tag that isn't in theme_elements, there # will be a KeyError in the test run. + count = 0 for tag in hs.tag_names(): - for start_index in hs.tag_ranges(tag)[0::2]: - count += 1 - click_it(start_index) + try: + click_char(hs.tag_nextrange(tag, "1.0")[0]) eq(d.highlight_target.get(), elem[tag]) + count += 1 eq(d.set_highlight_target.called, count) + except IndexError: + pass # Skip unused theme_elements tag, like 'sel'. def test_highlight_sample_double_click(self): # Test double click on highlight_sample. diff --git a/Lib/idlelib/idle_test/test_debugger.py b/Lib/idlelib/idle_test/test_debugger.py index 35efb3411c73b5..d1c9638dd5d711 100644 --- a/Lib/idlelib/idle_test/test_debugger.py +++ b/Lib/idlelib/idle_test/test_debugger.py @@ -1,16 +1,286 @@ -"Test debugger, coverage 19%" +"""Test debugger, coverage 66% + +Try to make tests pass with draft bdbx, which may replace bdb in 3.13+. +""" from idlelib import debugger -import unittest -from test.support import requires -requires('gui') +from collections import namedtuple +from textwrap import dedent from tkinter import Tk +from test.support import requires +import unittest +from unittest import mock +from unittest.mock import Mock, patch + +"""A test python script for the debug tests.""" +TEST_CODE = dedent(""" + i = 1 + i += 2 + if i == 3: + print(i) + """) + + +class MockFrame: + "Minimal mock frame." + + def __init__(self, code, lineno): + self.f_code = code + self.f_lineno = lineno + + +class IdbTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.gui = Mock() + cls.idb = debugger.Idb(cls.gui) + + # Create test and code objects to simulate a debug session. + code_obj = compile(TEST_CODE, 'idlelib/file.py', mode='exec') + frame1 = MockFrame(code_obj, 1) + frame1.f_back = None + frame2 = MockFrame(code_obj, 2) + frame2.f_back = frame1 + cls.frame = frame2 + cls.msg = 'file.py:2: ()' + + def test_init(self): + self.assertIs(self.idb.gui, self.gui) + # Won't test super call since two Bdbs are very different. + + def test_user_line(self): + # Test that .user_line() creates a string message for a frame. + self.gui.interaction = Mock() + self.idb.user_line(self.frame) + self.gui.interaction.assert_called_once_with(self.msg, self.frame) + + def test_user_exception(self): + # Test that .user_exception() creates a string message for a frame. + exc_info = (type(ValueError), ValueError(), None) + self.gui.interaction = Mock() + self.idb.user_exception(self.frame, exc_info) + self.gui.interaction.assert_called_once_with( + self.msg, self.frame, exc_info) + + +class FunctionTest(unittest.TestCase): + # Test module functions together. + + def test_functions(self): + rpc_obj = compile(TEST_CODE,'rpc.py', mode='exec') + rpc_frame = MockFrame(rpc_obj, 2) + rpc_frame.f_back = rpc_frame + self.assertTrue(debugger._in_rpc_code(rpc_frame)) + self.assertEqual(debugger._frame2message(rpc_frame), + 'rpc.py:2: ()') + + code_obj = compile(TEST_CODE, 'idlelib/debugger.py', mode='exec') + code_frame = MockFrame(code_obj, 1) + code_frame.f_back = None + self.assertFalse(debugger._in_rpc_code(code_frame)) + self.assertEqual(debugger._frame2message(code_frame), + 'debugger.py:1: ()') + + code_frame.f_back = code_frame + self.assertFalse(debugger._in_rpc_code(code_frame)) + code_frame.f_back = rpc_frame + self.assertTrue(debugger._in_rpc_code(code_frame)) + + +class DebuggerTest(unittest.TestCase): + "Tests for Debugger that do not need a real root." + + @classmethod + def setUpClass(cls): + cls.pyshell = Mock() + cls.pyshell.root = Mock() + cls.idb = Mock() + with patch.object(debugger.Debugger, 'make_gui'): + cls.debugger = debugger.Debugger(cls.pyshell, cls.idb) + cls.debugger.root = Mock() + + def test_cont(self): + self.debugger.cont() + self.idb.set_continue.assert_called_once() + + def test_step(self): + self.debugger.step() + self.idb.set_step.assert_called_once() + + def test_quit(self): + self.debugger.quit() + self.idb.set_quit.assert_called_once() + + def test_next(self): + with patch.object(self.debugger, 'frame') as frame: + self.debugger.next() + self.idb.set_next.assert_called_once_with(frame) + + def test_ret(self): + with patch.object(self.debugger, 'frame') as frame: + self.debugger.ret() + self.idb.set_return.assert_called_once_with(frame) + + def test_clear_breakpoint(self): + self.debugger.clear_breakpoint('test.py', 4) + self.idb.clear_break.assert_called_once_with('test.py', 4) + + def test_clear_file_breaks(self): + self.debugger.clear_file_breaks('test.py') + self.idb.clear_all_file_breaks.assert_called_once_with('test.py') + + def test_set_load_breakpoints(self): + # Test the .load_breakpoints() method calls idb. + FileIO = namedtuple('FileIO', 'filename') + + class MockEditWindow(object): + def __init__(self, fn, breakpoints): + self.io = FileIO(fn) + self.breakpoints = breakpoints + + self.pyshell.flist = Mock() + self.pyshell.flist.inversedict = ( + MockEditWindow('test1.py', [4, 4]), + MockEditWindow('test2.py', [13, 44, 45]), + ) + self.debugger.set_breakpoint('test0.py', 1) + self.idb.set_break.assert_called_once_with('test0.py', 1) + self.debugger.load_breakpoints() # Call set_breakpoint 5 times. + self.idb.set_break.assert_has_calls( + [mock.call('test0.py', 1), + mock.call('test1.py', 4), + mock.call('test1.py', 4), + mock.call('test2.py', 13), + mock.call('test2.py', 44), + mock.call('test2.py', 45)]) + + def test_sync_source_line(self): + # Test that .sync_source_line() will set the flist.gotofileline with fixed frame. + test_code = compile(TEST_CODE, 'test_sync.py', 'exec') + test_frame = MockFrame(test_code, 1) + self.debugger.frame = test_frame + + self.debugger.flist = Mock() + with patch('idlelib.debugger.os.path.exists', return_value=True): + self.debugger.sync_source_line() + self.debugger.flist.gotofileline.assert_called_once_with('test_sync.py', 1) + + +class DebuggerGuiTest(unittest.TestCase): + """Tests for debugger.Debugger that need tk root. + + close needs debugger.top set in make_gui. + """ + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = root = Tk() + root.withdraw() + cls.pyshell = Mock() + cls.pyshell.root = root + cls.idb = Mock() +# stack tests fail with debugger here. +## cls.debugger = debugger.Debugger(cls.pyshell, cls.idb) +## cls.debugger.root = root +## # real root needed for real make_gui +## # run, interacting, abort_loop + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.debugger = debugger.Debugger(self.pyshell, self.idb) + self.debugger.root = self.root + # real root needed for real make_gui + # run, interacting, abort_loop + + def test_run_debugger(self): + self.debugger.run(1, 'two') + self.idb.run.assert_called_once_with(1, 'two') + self.assertEqual(self.debugger.interacting, 0) + + def test_close(self): + # Test closing the window in an idle state. + self.debugger.close() + self.pyshell.close_debugger.assert_called_once() + + def test_show_stack(self): + self.debugger.show_stack() + self.assertEqual(self.debugger.stackviewer.gui, self.debugger) + + def test_show_stack_with_frame(self): + test_frame = MockFrame(None, None) + self.debugger.frame = test_frame + + # Reset the stackviewer to force it to be recreated. + self.debugger.stackviewer = None + self.idb.get_stack.return_value = ([], 0) + self.debugger.show_stack() + + # Check that the newly created stackviewer has the test gui as a field. + self.assertEqual(self.debugger.stackviewer.gui, self.debugger) + self.idb.get_stack.assert_called_once_with(test_frame, None) + + +class StackViewerTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.code = compile(TEST_CODE, 'test_stackviewer.py', 'exec') + self.stack = [ + (MockFrame(self.code, 1), 1), + (MockFrame(self.code, 2), 2) + ] + # Create a stackviewer and load the test stack. + self.sv = debugger.StackViewer(self.root, None, None) + self.sv.load_stack(self.stack) + + def test_init(self): + # Test creation of StackViewer. + gui = None + flist = None + master_window = self.root + sv = debugger.StackViewer(master_window, flist, gui) + self.assertTrue(hasattr(sv, 'stack')) + + def test_load_stack(self): + # Test the .load_stack() method against a fixed test stack. + # Check the test stack is assigned and the list contains the repr of them. + self.assertEqual(self.sv.stack, self.stack) + self.assertTrue('?.(), line 1:' in self.sv.get(0)) + self.assertEqual(self.sv.get(1), '?.(), line 2: ') + + def test_show_source(self): + # Test the .show_source() method against a fixed test stack. + # Patch out the file list to monitor it + self.sv.flist = Mock() + # Patch out isfile to pretend file exists. + with patch('idlelib.debugger.os.path.isfile', return_value=True) as isfile: + self.sv.show_source(1) + isfile.assert_called_once_with('test_stackviewer.py') + self.sv.flist.open.assert_called_once_with('test_stackviewer.py') + class NameSpaceTest(unittest.TestCase): @classmethod def setUpClass(cls): + requires('gui') cls.root = Tk() cls.root.withdraw() @@ -23,7 +293,5 @@ def test_init(self): debugger.NamespaceViewer(self.root, 'Test') -# Other classes are Idb, Debugger, and StackViewer. - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_debugobj.py b/Lib/idlelib/idle_test/test_debugobj.py index 131ce22b8bb69b..90ace4e1bc4f9e 100644 --- a/Lib/idlelib/idle_test/test_debugobj.py +++ b/Lib/idlelib/idle_test/test_debugobj.py @@ -37,7 +37,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.SequenceTreeItem('label', 'abc') - self.assertEqual(list(ti.keys()), [0, 1, 2]) + self.assertEqual(list(ti.keys()), [0, 1, 2]) # keys() is a range. class DictTreeItemTest(unittest.TestCase): @@ -50,7 +50,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.DictTreeItem('label', {1:1, 0:0, 2:2}) - self.assertEqual(ti.keys(), [0, 1, 2]) + self.assertEqual(ti.keys(), [0, 1, 2]) # keys() is a sorted list. if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 9296a6d235fbbe..0dfe2f3c58befa 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -95,7 +95,7 @@ def test_tabwidth_8(self): def insert(text, string): text.delete('1.0', 'end') text.insert('end', string) - text.update() # Force update for colorizer to finish. + text.update_idletasks() # Force update for colorizer to finish. class IndentAndNewlineTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_help.py b/Lib/idlelib/idle_test/test_help.py index b542659981894d..c528d4e77f8ca7 100644 --- a/Lib/idlelib/idle_test/test_help.py +++ b/Lib/idlelib/idle_test/test_help.py @@ -1,4 +1,4 @@ -"Test help, coverage 87%." +"Test help, coverage 94%." from idlelib import help import unittest @@ -8,25 +8,27 @@ from tkinter import Tk -class HelpFrameTest(unittest.TestCase): +class IdleDocTest(unittest.TestCase): @classmethod def setUpClass(cls): "By itself, this tests that file parsed without exception." cls.root = root = Tk() root.withdraw() - helpfile = join(dirname(dirname(abspath(__file__))), 'help.html') - cls.frame = help.HelpFrame(root, helpfile) + cls.window = help.show_idlehelp(root) @classmethod def tearDownClass(cls): - del cls.frame + del cls.window cls.root.update_idletasks() cls.root.destroy() del cls.root - def test_line1(self): - text = self.frame.text + def test_1window(self): + self.assertIn('IDLE Doc', self.window.wm_title()) + + def test_4text(self): + text = self.window.frame.text self.assertEqual(text.get('1.0', '1.end'), ' IDLE ') diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 8b79487b15d4cd..7e16abdb7c9f96 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -71,7 +71,7 @@ def test_file_buttons(self): """Test buttons that display files.""" dialog = self.dialog button_sources = [(self.dialog.readme, 'README.txt', 'readme'), - (self.dialog.idle_news, 'NEWS.txt', 'news'), + (self.dialog.idle_news, 'News3.txt', 'news'), (self.dialog.idle_credits, 'CREDITS.txt', 'credits')] for button, filename, name in button_sources: diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index a38e43dcb9d1c4..83ecbffa2a197e 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -8,6 +8,7 @@ from unittest import mock import idlelib from idlelib.idle_test.mock_idle import Func +from test.support import force_not_colorized idlelib.testing = True # Use {} for executing test user code. @@ -46,6 +47,7 @@ def __eq__(self, other): "Did you mean: 'real'?\n"), ) + @force_not_colorized def test_get_message(self): for code, exc, msg in self.data: with self.subTest(code=code): @@ -57,6 +59,7 @@ def test_get_message(self): expect = f'{exc.__name__}: {msg}' self.assertEqual(actual, expect) + @force_not_colorized @mock.patch.object(run, 'cleanup_traceback', new_callable=lambda: (lambda t, e: None)) def test_get_multiple_message(self, mock): diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index fb52b3a0179553..605e7a892570d7 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -690,16 +690,22 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll up using the event. - # The meaning of delta is platform-dependent. - delta = -1 if sys.platform == 'darwin' else 120 - sidebar.canvas.event_generate('', x=0, y=0, delta=delta) + # Delta for , whose meaning is platform-dependent. + delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 + + # Scroll up. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('', x=0, y=0) + else: + sidebar.canvas.event_generate('', x=0, y=0, delta=delta) yield - if sys.platform != 'darwin': # .update_idletasks() does not work. - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Scroll back down using the event. - sidebar.canvas.event_generate('', x=0, y=0) + # Scroll back down. + if sidebar.canvas._windowingsystem == 'x11': + sidebar.canvas.event_generate('', x=0, y=0) + else: + sidebar.canvas.event_generate('', x=0, y=0, delta=-delta) yield self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index af8159c2b33f51..464126e2df0668 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -7,7 +7,7 @@ from tkinter import filedialog from tkinter import messagebox -from tkinter.simpledialog import askstring +from tkinter.simpledialog import askstring # loadfile encoding. from idlelib.config import idleConf from idlelib.util import py_extensions @@ -180,24 +180,25 @@ def loadfile(self, filename): return True def maybesave(self): + """Return 'yes', 'no', 'cancel' as appropriate. + + Tkinter messagebox.askyesnocancel converts these tk responses + to True, False, None. Convert back, as now expected elsewhere. + """ if self.get_saved(): return "yes" - message = "Do you want to save %s before closing?" % ( - self.filename or "this untitled document") + message = ("Do you want to save " + f"{self.filename or 'this untitled document'}" + " before closing?") confirm = messagebox.askyesnocancel( title="Save On Close", message=message, default=messagebox.YES, parent=self.text) if confirm: - reply = "yes" self.save(None) - if not self.get_saved(): - reply = "cancel" - elif confirm is None: - reply = "cancel" - else: - reply = "no" + reply = "yes" if self.get_saved() else "cancel" + else: reply = "cancel" if confirm is None else "no" self.text.focus_set() return reply @@ -393,13 +394,15 @@ def updaterecentfileslist(self,filename): if self.editwin.flist: self.editwin.update_recent_files_list(filename) + def _io_binding(parent): # htest # from tkinter import Toplevel, Text - root = Toplevel(parent) - root.title("Test IOBinding") + top = Toplevel(parent) + top.title("Test IOBinding") x, y = map(int, parent.geometry().split('+')[1:]) - root.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) + class MyEditWin: def __init__(self, text): self.text = text @@ -423,12 +426,13 @@ def saveas(self, event): def savecopy(self, event): self.text.event_generate("<>") - text = Text(root) + text = Text(top) text.pack() text.focus_set() editwin = MyEditWin(text) IOBinding(editwin) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_iomenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 2ea02ec04d661a..332952f4572cbd 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -221,7 +221,7 @@ def help_dialog(event=None): # The binding above doesn't reliably work on all versions of Tk # on macOS. Adding command definition below does seem to do the # right thing for now. - root.createcommand('exit', flist.close_all_callback) + root.createcommand('::tk::mac::Quit', flist.close_all_callback) if isCarbonTk(): # for Carbon AquaTk, replace the default Tk apple menu diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py index 0200f445cc9340..41f81813113062 100644 --- a/Lib/idlelib/multicall.py +++ b/Lib/idlelib/multicall.py @@ -421,6 +421,8 @@ def _multi_call(parent): # htest # top.geometry("+%d+%d" % (x, y + 175)) text = MultiCallCreator(tkinter.Text)(top) text.pack() + text.focus_set() + def bindseq(seq, n=[0]): def handler(event): print(seq) @@ -440,6 +442,7 @@ def handler(event): bindseq("") bindseq("") + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_mainmenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 610031e26f1dff..5ed3f35a7af655 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -182,6 +182,7 @@ def setup(self): text.tag_raise('sel') self.write = self.owin.write + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index 6de242d0000bed..48a77875ba5801 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -99,13 +99,9 @@ def listmodules(self, allnames): return sorted -def _path_browser(parent): # htest # - PathBrowser(parent, _htest=True) - parent.mainloop() - if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_path_browser) + run(PathBrowser) diff --git a/Lib/idlelib/percolator.py b/Lib/idlelib/percolator.py index 1fe34d29f54eb2..aa73427c4915c8 100644 --- a/Lib/idlelib/percolator.py +++ b/Lib/idlelib/percolator.py @@ -86,11 +86,11 @@ def delete(self, *args): print(self.name, ": delete", args) self.delegate.delete(*args) - box = tk.Toplevel(parent) - box.title("Test Percolator") + top = tk.Toplevel(parent) + top.title("Test Percolator") x, y = map(int, parent.geometry().split('+')[1:]) - box.geometry("+%d+%d" % (x, y + 175)) - text = tk.Text(box) + top.geometry("+%d+%d" % (x, y + 175)) + text = tk.Text(top) p = Percolator(text) pin = p.insertfilter pout = p.removefilter @@ -103,13 +103,15 @@ def toggle2(): (pin if var2.get() else pout)(t2) text.pack() + text.focus_set() var1 = tk.IntVar(parent) - cb1 = tk.Checkbutton(box, text="Tracer1", command=toggle1, variable=var1) + cb1 = tk.Checkbutton(top, text="Tracer1", command=toggle1, variable=var1) cb1.pack() var2 = tk.IntVar(parent) - cb2 = tk.Checkbutton(box, text="Tracer2", command=toggle2, variable=var2) + cb2 = tk.Checkbutton(top, text="Tracer2", command=toggle2, variable=var2) cb2.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_percolator', verbosity=2, exit=False) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 6028700356b171..d8b2652d5d7979 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -11,15 +11,9 @@ "Your Python may not be configured for Tk. **", file=sys.__stderr__) raise SystemExit(1) -# Valid arguments for the ...Awareness call below are defined in the following. -# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx if sys.platform == 'win32': - try: - import ctypes - PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. - ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) - except (ImportError, AttributeError, OSError): - pass + from idlelib.util import fix_win_hidpi + fix_win_hidpi() from tkinter import messagebox @@ -133,8 +127,8 @@ class PyShellEditorWindow(EditorWindow): def __init__(self, *args): self.breakpoints = [] EditorWindow.__init__(self, *args) - self.text.bind("<>", self.set_breakpoint_here) - self.text.bind("<>", self.clear_breakpoint_here) + self.text.bind("<>", self.set_breakpoint_event) + self.text.bind("<>", self.clear_breakpoint_event) self.text.bind("<>", self.flist.open_shell) #TODO: don't read/write this from/to .idlerc when testing @@ -155,8 +149,8 @@ def filename_changed_hook(old_hook=self.io.filename_change_hook, ("Copy", "<>", "rmenu_check_copy"), ("Paste", "<>", "rmenu_check_paste"), (None, None, None), - ("Set Breakpoint", "<>", None), - ("Clear Breakpoint", "<>", None) + ("Set Breakpoint", "<>", None), + ("Clear Breakpoint", "<>", None) ] def color_breakpoint_text(self, color=True): @@ -181,11 +175,11 @@ def set_breakpoint(self, lineno): self.breakpoints.append(lineno) try: # update the subprocess debugger debug = self.flist.pyshell.interp.debugger - debug.set_breakpoint_here(filename, lineno) + debug.set_breakpoint(filename, lineno) except: # but debugger may not be active right now.... pass - def set_breakpoint_here(self, event=None): + def set_breakpoint_event(self, event=None): text = self.text filename = self.io.filename if not filename: @@ -194,7 +188,7 @@ def set_breakpoint_here(self, event=None): lineno = int(float(text.index("insert"))) self.set_breakpoint(lineno) - def clear_breakpoint_here(self, event=None): + def clear_breakpoint_event(self, event=None): text = self.text filename = self.io.filename if not filename: @@ -209,7 +203,7 @@ def clear_breakpoint_here(self, event=None): "insert lineend +1char") try: debug = self.flist.pyshell.interp.debugger - debug.clear_breakpoint_here(filename, lineno) + debug.clear_breakpoint(filename, lineno) except: pass @@ -747,10 +741,11 @@ def showtraceback(self): self.tkconsole.open_stack_viewer() def checklinecache(self): - c = linecache.cache - for key in list(c.keys()): + "Remove keys other than ''." + cache = linecache.cache + for key in list(cache): # Iterate list because mutate cache. if key[:1] + key[-1:] != "<>": - del c[key] + del cache[key] def runcommand(self, code): "Run the code without invoking the debugger" @@ -1693,6 +1688,7 @@ def main(): root.destroy() capture_warnings(False) + if __name__ == "__main__": main() diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index df02f2123ab02f..57230e2aaca66d 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -368,7 +368,7 @@ def create_extra(self): sticky='we') def cli_args_ok(self): - "Validity check and parsing for command line arguments." + "Return command line arg list or None if error." cli_string = self.entry.get().strip() try: cli_args = shlex.split(cli_string, posix=True) diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 4928340e98df68..8e2ba68d3815bf 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -106,6 +106,7 @@ def dispatch(self, operation, *args): to *args to accomplish that. For an example, see colorizer.py. ''' + operation = str(operation) # can be a Tcl_Obj m = self._operations.get(operation) try: if m: @@ -164,6 +165,7 @@ def my_insert(*args): original_insert(*args) original_insert = redir.register("insert", my_insert) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_redirector', verbosity=2, exit=False) diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index ca83173877ad1d..3716d841568d30 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -25,7 +25,8 @@ def replace(text, insert_tags=None): if not hasattr(engine, "_replacedialog"): engine._replacedialog = ReplaceDialog(root, engine) dialog = engine._replacedialog - dialog.open(text, insert_tags=insert_tags) + searchphrase = text.get("sel.first", "sel.last") + dialog.open(text, searchphrase, insert_tags=insert_tags) class ReplaceDialog(SearchDialogBase): @@ -51,27 +52,17 @@ def __init__(self, root, engine): self.replvar = StringVar(root) self.insert_tags = None - def open(self, text, insert_tags=None): + def open(self, text, searchphrase=None, *, insert_tags=None): """Make dialog visible on top of others and ready to use. - Also, highlight the currently selected text and set the - search to include the current selection (self.ok). + Also, set the search to include the current selection + (self.ok). Args: text: Text widget being searched. + searchphrase: String phrase to search. """ - SearchDialogBase.open(self, text) - try: - first = text.index("sel.first") - except TclError: - first = None - try: - last = text.index("sel.last") - except TclError: - last = None - first = first or text.index("insert") - last = last or first - self.show_hit(first, last) + SearchDialogBase.open(self, text, searchphrase) self.ok = True self.insert_tags = insert_tags @@ -120,7 +111,7 @@ def _replace_expand(self, m, repl): if self.engine.isre(): try: new = m.expand(repl) - except re.error: + except re.PatternError: self.engine.report_error(repl, 'Invalid Replace Expression') new = None else: @@ -299,6 +290,7 @@ def show_replace(): button = Button(frame, text="Replace", command=show_replace) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_replace', verbosity=2, exit=False) diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py index b08b80c9004551..3f0b2230dd185d 100644 --- a/Lib/idlelib/rpc.py +++ b/Lib/idlelib/rpc.py @@ -158,8 +158,8 @@ def debug(self, *args): s = s + " " + str(a) print(s, file=sys.__stderr__) - def register(self, oid, object): - self.objtable[oid] = object + def register(self, oid, object_): + self.objtable[oid] = object_ def unregister(self, oid): try: diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 53e80a9b42801f..8974b52674fb8c 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -91,13 +91,20 @@ def capture_warnings(capture): _warnings_showwarning = None capture_warnings(True) -tcl = tkinter.Tcl() -def handle_tk_events(tcl=tcl): - """Process any tk events that are ready to be dispatched if tkinter - has been imported, a tcl interpreter has been created and tk has been - loaded.""" - tcl.eval("update") +if idlelib.testing: + # gh-121008: When testing IDLE, don't create a Tk object to avoid side + # effects such as installing a PyOS_InputHook hook. + def handle_tk_events(): + pass +else: + tcl = tkinter.Tcl() + + def handle_tk_events(tcl=tcl): + """Process any tk events that are ready to be dispatched if tkinter + has been imported, a tcl interpreter has been created and tk has been + loaded.""" + tcl.eval("update") # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global diff --git a/Lib/idlelib/scrolledlist.py b/Lib/idlelib/scrolledlist.py index 71fd18ab19ec8a..4fb418db326255 100644 --- a/Lib/idlelib/scrolledlist.py +++ b/Lib/idlelib/scrolledlist.py @@ -132,6 +132,7 @@ def _scrolled_list(parent): # htest # top = Toplevel(parent) x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" % (x+200, y + 175)) + class MyScrolledList(ScrolledList): def fill_menu(self): self.menu.add_command(label="right click") def on_select(self, index): print("select", self.get(index)) @@ -141,9 +142,10 @@ def on_double(self, index): print("double", self.get(index)) for i in range(30): scrolled_list.append("Item %02d" % i) + if __name__ == '__main__': from unittest import main - main('idlelib.idle_test.test_scrolledlist', verbosity=2,) + main('idlelib.idle_test.test_scrolledlist', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_scrolled_list) diff --git a/Lib/idlelib/search.py b/Lib/idlelib/search.py index b35f3b59c3d2e8..935a4832257fa4 100644 --- a/Lib/idlelib/search.py +++ b/Lib/idlelib/search.py @@ -156,6 +156,7 @@ def show_find(): button = Button(frame, text="Search (selection ignored)", command=show_find) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_search', verbosity=2, exit=False) diff --git a/Lib/idlelib/searchengine.py b/Lib/idlelib/searchengine.py index 0684142f43644a..ceb38cfaef900b 100644 --- a/Lib/idlelib/searchengine.py +++ b/Lib/idlelib/searchengine.py @@ -84,7 +84,7 @@ def getprog(self): flags = flags | re.IGNORECASE try: prog = re.compile(pat, flags) - except re.error as e: + except re.PatternError as e: self.report_error(pat, e.msg, e.pos) return None return prog diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index fb1084dbf3f18b..aa19a24e3edef2 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -25,10 +25,9 @@ def get_end_linenumber(text): def get_displaylines(text, index): """Display height, in lines, of a logical line in a Tk text widget.""" - res = text.count(f"{index} linestart", - f"{index} lineend", - "displaylines") - return res[0] if res else 0 + return text.count(f"{index} linestart", + f"{index} lineend", + "displaylines", return_ints=True) def get_widget_padding(widget): """Get the total padding of a Tk widget, including its border.""" @@ -514,16 +513,16 @@ def update_colors(self): self.change_callback() -def _linenumbers_drag_scrolling(parent): # htest # +def _sidebar_number_scrolling(parent): # htest # from idlelib.idle_test.test_sidebar import Dummy_editwin - toplevel = tk.Toplevel(parent) - text_frame = tk.Frame(toplevel) + top = tk.Toplevel(parent) + text_frame = tk.Frame(top) text_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) text_frame.rowconfigure(1, weight=1) text_frame.columnconfigure(1, weight=1) - font = idleConf.GetFont(toplevel, 'main', 'EditorWindow') + font = idleConf.GetFont(top, 'main', 'EditorWindow') text = tk.Text(text_frame, width=80, height=24, wrap=tk.NONE, font=font) text.grid(row=1, column=1, sticky=tk.NSEW) @@ -541,4 +540,4 @@ def _linenumbers_drag_scrolling(parent): # htest # main('idlelib.idle_test.test_sidebar', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_linenumbers_drag_scrolling) + run(_sidebar_number_scrolling) diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 7b00c4cdb7d033..95042d4debdc03 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -1,3 +1,5 @@ +# Rename to stackbrowser or possibly consolidate with browser. + import linecache import os @@ -99,19 +101,19 @@ def IsExpandable(self): def GetSubList(self): sublist = [] - for key in self.object.keys(): + for key in self.object.keys(): # self.object not necessarily dict. try: value = self.object[key] except KeyError: continue - def setfunction(value, key=key, object=self.object): - object[key] = value + def setfunction(value, key=key, object_=self.object): + object_[key] = value item = make_objecttreeitem(key + " =", value, setfunction) sublist.append(item) return sublist -def _stack_viewer(parent): # htest # +def _stackbrowser(parent): # htest # from idlelib.pyshell import PyShellFileList top = tk.Toplevel(parent) top.title("Test StackViewer") @@ -129,4 +131,4 @@ def _stack_viewer(parent): # htest # main('idlelib.idle_test.test_stackviewer', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_stack_viewer) + run(_stackbrowser) diff --git a/Lib/idlelib/statusbar.py b/Lib/idlelib/statusbar.py index 755fafb0ac6438..8445d4cc8dfdb9 100644 --- a/Lib/idlelib/statusbar.py +++ b/Lib/idlelib/statusbar.py @@ -26,6 +26,7 @@ def _multistatus_bar(parent): # htest # x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" %(x, y + 175)) top.title("Test multistatus bar") + frame = Frame(top) text = Text(frame, height=5, width=40) text.pack() @@ -42,6 +43,7 @@ def change(): button.pack(side='bottom') frame.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_statusbar', verbosity=2, exit=False) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 5f30f0f6092bfa..0726d7e23660f6 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -285,8 +285,9 @@ def drawtext(self): self.label.bind("<1>", self.select_or_edit) self.label.bind("", self.flip) self.label.bind("", lambda e: wheel_event(e, self.canvas)) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) + if self.label._windowingsystem == 'x11': + self.label.bind("", lambda e: wheel_event(e, self.canvas)) + self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.text_id = id def select_or_edit(self, event=None): @@ -460,8 +461,9 @@ def __init__(self, master, **opts): self.canvas.bind("", self.unit_up) self.canvas.bind("", self.unit_down) self.canvas.bind("", wheel_event) - self.canvas.bind("", wheel_event) - self.canvas.bind("", wheel_event) + if self.canvas._windowingsystem == 'x11': + self.canvas.bind("", wheel_event) + self.canvas.bind("", wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("", self.zoom_height) self.canvas.focus_set() @@ -492,6 +494,7 @@ def _tree_widget(parent): # htest # node = TreeNode(sc.canvas, None, item) node.expand() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_tree', verbosity=2, exit=False) diff --git a/Lib/idlelib/undo.py b/Lib/idlelib/undo.py index 5f10c0f05c1acb..f52446d5fcdcf8 100644 --- a/Lib/idlelib/undo.py +++ b/Lib/idlelib/undo.py @@ -339,25 +339,26 @@ def bump_depth(self, incr=1): def _undo_delegator(parent): # htest # from tkinter import Toplevel, Text, Button from idlelib.percolator import Percolator - undowin = Toplevel(parent) - undowin.title("Test UndoDelegator") + top = Toplevel(parent) + top.title("Test UndoDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - undowin.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) - text = Text(undowin, height=10) + text = Text(top, height=10) text.pack() text.focus_set() p = Percolator(text) d = UndoDelegator() p.insertfilter(d) - undo = Button(undowin, text="Undo", command=lambda:d.undo_event(None)) + undo = Button(top, text="Undo", command=lambda:d.undo_event(None)) undo.pack(side='left') - redo = Button(undowin, text="Redo", command=lambda:d.redo_event(None)) + redo = Button(top, text="Redo", command=lambda:d.redo_event(None)) redo.pack(side='left') - dump = Button(undowin, text="Dump", command=lambda:d.dump_event(None)) + dump = Button(top, text="Dump", command=lambda:d.dump_event(None)) dump.pack(side='left') + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_undo', verbosity=2, exit=False) diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index ede670a4db5536..e05604ab4853f6 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -12,9 +12,25 @@ * std streams (pyshell, run), * warning stuff (pyshell, run). """ +import sys + +# .pyw is for Windows; .pyi is for typing stub files. +# The extension order is needed for iomenu open/save dialogs. +py_extensions = ('.py', '.pyw', '.pyi') + + +# Fix for HiDPI screens on Windows. CALL BEFORE ANY TK OPERATIONS! +# URL for arguments for the ...Awareness call below. +# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx +if sys.platform == 'win32': # pragma: no cover + def fix_win_hidpi(): # Called in pyshell and turtledemo. + try: + import ctypes + PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. + ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) + except (ImportError, AttributeError, OSError): + pass -# .pyw is for Windows; .pyi is for stub files. -py_extensions = ('.py', '.pyw', '.pyi') # Order needed for open/save dialogs. if __name__ == '__main__': from unittest import main diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 577b4b9b03a88d..e576c29e67dc0a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -239,7 +239,7 @@ def _connect(self): if __debug__: self._cmd_log_len = 10 self._cmd_log_idx = 0 - self._cmd_log = {} # Last `_cmd_log_len' interactions + self._cmd_log = {} # Last '_cmd_log_len' interactions if self.debug >= 1: self._mesg('imaplib version %s' % __version__) self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) @@ -396,7 +396,7 @@ def append(self, mailbox, flags, date_time, message): (typ, [data]) = .append(mailbox, flags, date_time, message) - All args except `message' can be None. + All args except 'message' can be None. """ name = 'APPEND' if not mailbox: @@ -927,7 +927,7 @@ def xatom(self, name, *args): (typ, [data]) = .xatom(name, arg, ...) - Returns response appropriate to extension command `name'. + Returns response appropriate to extension command 'name'. """ name = name.upper() #if not name in self.capabilities: # Let the server decide! @@ -1167,7 +1167,7 @@ def _get_tagged_response(self, tag, expect_bye=False): # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and - # I'll update the code in `_get_response()'. + # I'll update the code in '_get_response()'. try: self._get_response() @@ -1259,7 +1259,7 @@ def _dump_ur(self, untagged_resp_dict): self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) def _log(self, line): - # Keep log of last `_cmd_log_len' interactions for debugging. + # Keep log of last '_cmd_log_len' interactions for debugging. self._cmd_log[self._cmd_log_idx] = (line, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ec2e56f6ea9ca1..de5651f0a7fc36 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to @@ -824,10 +824,16 @@ def _module_repr_from_spec(spec): """Return the repr to use for the module.""" name = '?' if spec.name is None else spec.name if spec.origin is None: - if spec.loader is None: + loader = spec.loader + if loader is None: return f'' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'' else: - return f'' + return f'' else: if spec.has_location: return f'' @@ -1128,7 +1134,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 0019897c943e14..bf14d57b2503ea 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -81,6 +81,11 @@ def _pack_uint32(x): return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') +def _unpack_uint64(data): + """Convert 8 bytes in little-endian to an integer.""" + assert len(data) == 8 + return int.from_bytes(data, 'little') + def _unpack_uint32(data): """Convert 4 bytes in little-endian to an integer.""" assert len(data) == 4 @@ -413,6 +418,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) # Python 3.11a7 3494 (New location info table) +# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) # Python 3.12a1 3500 (Remove PRECALL opcode) # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) # Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) @@ -459,8 +465,18 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3561 (Add cache entry to branch instructions) # Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) # Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) - -# Python 3.14 will start with 3600 +# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) +# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) +# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) +# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) +# Python 3.13a1 3568 (Change semantics of END_FOR) +# Python 3.13a5 3569 (Specialize CONTAINS_OP) +# Python 3.13a6 3570 (Add __firstlineno__ class attribute) +# Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) +# Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) +# Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) + +# Python 3.15 will start with 3650 # Please don't copy-paste the same pre-release tag for new entries above!!! # You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -475,7 +491,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3563).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3602).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c @@ -1451,7 +1467,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. @@ -1463,6 +1479,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" @@ -1704,6 +1723,46 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + module.__file__ = path + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1736,10 +1795,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + else: + extension_loaders = [] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c135..eea6b38af6fa13 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -180,7 +164,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b275..6e294d59bfdcb9 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,9 +12,18 @@ from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 5c09666b6a40d9..8ce62dd864fc27 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,9 +1,13 @@ +from __future__ import annotations + import os import re import abc -import csv import sys +import json import email +import types +import inspect import pathlib import zipfile import operator @@ -13,9 +17,8 @@ import itertools import posixpath import collections -import inspect -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -25,8 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import List, Mapping, Optional, cast - +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -47,11 +49,11 @@ class PackageNotFoundError(ModuleNotFoundError): """The package was not found.""" - def __str__(self): + def __str__(self) -> str: return f"No package metadata was found for {self.name}" @property - def name(self): + def name(self) -> str: # type: ignore[override] (name,) = self.args return name @@ -117,38 +119,11 @@ def read(text, filter_=None): yield Pair(name, value) @staticmethod - def valid(line): + def valid(line: str): return line and not line.startswith('#') -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=2, - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): +class EntryPoint: """An entry point as defined by Python packaging conventions. See `the packaging docs on entry points @@ -190,34 +165,37 @@ class EntryPoint(DeprecatedTuple): value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None - def __init__(self, name, value, group): + def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @property - def module(self): + def module(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('module') @property - def attr(self): + def attr(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('attr') @property - def extras(self): + def extras(self) -> List[str]: match = self.pattern.match(self.value) + assert match is not None return re.findall(r'\w+', match.group('extras') or '') def _for(self, dist): @@ -265,7 +243,7 @@ def __repr__(self): f'group={self.group!r})' ) - def __hash__(self): + def __hash__(self) -> int: return hash(self._key()) @@ -276,7 +254,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name): # -> EntryPoint: + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] """ Get the EntryPoint in self matching name. """ @@ -285,7 +263,14 @@ def __getitem__(self, name): # -> EntryPoint: except StopIteration: raise KeyError(name) - def select(self, **params): + def __repr__(self): + """ + Repr with classname and tuple constructor to + signal that we deviate from regular tuple behavior. + """ + return '%s(%r)' % (self.__class__.__name__, tuple(self)) + + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -293,14 +278,14 @@ def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self): + def names(self) -> Set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self): + def groups(self) -> Set[str]: """ Return the set of all groups of all entry points. """ @@ -321,28 +306,31 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() + hash: Optional[FileHash] + size: int + dist: Distribution - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() + def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] + return self.locate().read_text(encoding=encoding) - def locate(self): + def read_binary(self) -> bytes: + return self.locate().read_bytes() + + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) class FileHash: - def __init__(self, spec): + def __init__(self, spec: str) -> None: self.mode, _, self.value = spec.partition('=') - def __repr__(self): + def __repr__(self) -> str: return f'' class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -362,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str): + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -393,21 +404,23 @@ def from_name(cls, name: str): if not name: raise ValueError("A distribution name is required.") try: - return next(cls.discover(name=name)) + return next(iter(cls.discover(name=name))) except StopIteration: raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs): + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -416,8 +429,8 @@ def discover(cls, **kwargs): ) @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -426,7 +439,7 @@ def at(path): @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -437,8 +450,15 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') @@ -451,7 +471,7 @@ def metadata(self) -> _meta.PackageMetadata: return _adapters.Message(email.message_from_string(text)) @property - def name(self): + def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] @@ -461,16 +481,22 @@ def _normalized_name(self): return Prepared.normalize(self.name) @property - def version(self): + def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" return self.metadata['Version'] @property - def entry_points(self): + def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self): + def files(self) -> Optional[List[PackagePath]]: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -479,6 +505,10 @@ def files(self): (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -490,6 +520,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none @@ -506,7 +540,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -533,7 +567,7 @@ def _read_files_egginfo_installed(self): paths = ( (subdir / name) .resolve() - .relative_to(self.locate_file('').resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) .as_posix() for name in text.splitlines() ) @@ -555,7 +589,7 @@ def _read_files_egginfo_sources(self): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self): + def requires(self) -> Optional[List[str]]: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -606,10 +640,23 @@ def url_req_space(req): space = url_req_space(section.value) yield section.value + space + quoted_marker(section.name) + @property + def origin(self): + return self._load_json('direct_url.json') + + def _load_json(self, filename): + return pass_none(json.loads)( + self.read_text(filename), + object_hook=lambda data: types.SimpleNamespace(**data), + ) + class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -622,6 +669,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -634,7 +692,7 @@ def __init__(self, **kwargs): vars(self).update(kwargs) @property - def path(self): + def path(self) -> List[str]: """ The sequence of directory path that a distribution finder should search. @@ -645,7 +703,7 @@ def path(self): return vars(self).get('path', sys.path) @abc.abstractmethod - def find_distributions(self, context=Context()): + def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ Find distributions. @@ -657,11 +715,18 @@ def find_distributions(self, context=Context()): class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -703,7 +768,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -724,7 +801,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -740,13 +820,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -774,7 +869,9 @@ def __bool__(self): class MetadataPathFinder(DistributionFinder): @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): + def find_distributions( + cls, context=DistributionFinder.Context() + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -794,19 +891,20 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) - def invalidate_caches(cls): + @classmethod + def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() class PathDistribution(Distribution): - def __init__(self, path: SimplePath): + def __init__(self, path: SimplePath) -> None: """Construct a distribution. :param path: SimplePath indicating the metadata directory. """ self._path = path - def read_text(self, filename): + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -816,9 +914,11 @@ def read_text(self, filename): ): return self._path.joinpath(filename).read_text(encoding='utf-8') + return None + read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property @@ -851,7 +951,7 @@ def _name_from_stem(stem): return name -def distribution(distribution_name): +def distribution(distribution_name: str) -> Distribution: """Get the ``Distribution`` instance for the named package. :param distribution_name: The name of the distribution package as a string. @@ -860,7 +960,7 @@ def distribution(distribution_name): return Distribution.from_name(distribution_name) -def distributions(**kwargs): +def distributions(**kwargs) -> Iterable[Distribution]: """Get all ``Distribution`` instances in the current environment. :return: An iterable of ``Distribution`` instances. @@ -868,7 +968,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> _meta.PackageMetadata: +def metadata(distribution_name: str) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -877,7 +977,7 @@ def metadata(distribution_name) -> _meta.PackageMetadata: return Distribution.from_name(distribution_name).metadata -def version(distribution_name): +def version(distribution_name: str) -> str: """Get the version string for the named package. :param distribution_name: The name of the distribution package to query. @@ -911,7 +1011,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name): +def files(distribution_name: str) -> Optional[List[PackagePath]]: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -920,11 +1020,11 @@ def files(distribution_name): return distribution(distribution_name).files -def requires(distribution_name): +def requires(distribution_name: str) -> Optional[List[str]]: """ Return a list of requirements for the named package. - :return: An iterator of requirements, suitable for + :return: An iterable of requirements, suitable for packaging.requirement.Requirement. """ return distribution(distribution_name).requires @@ -951,13 +1051,42 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() +def _topmost(name: PackagePath) -> Optional[str]: + """ + Return the top-most parent as long as there is a parent. + """ + top, *rest = name.parts + return top if rest else None + + +def _get_toplevel_name(name: PackagePath) -> str: + """ + Infer a possibly importable module name from a name presumed on + sys.path. + + >>> _get_toplevel_name(PackagePath('foo.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pyc')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo/__init__.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pth')) + 'foo.pth' + >>> _get_toplevel_name(PackagePath('foo.dist-info')) + 'foo.dist-info' + """ + return _topmost(name) or ( + # python/typeshed#10328 + inspect.getmodulename(name) # type: ignore + or str(name) + ) + + def _top_level_inferred(dist): - opt_names = { - f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) - for f in always_iterable(dist.files) - } + opt_names = set(map(_get_toplevel_name, always_iterable(dist.files))) - @pass_none def importable_name(name): return '.' not in name diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 6aed69a30857e4..591168808953ba 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -53,7 +53,7 @@ def __iter__(self): def __getitem__(self, item): """ Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. + missing key is supplied. Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index c9a7ef906a8a8c..1927d0f624d82f 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/importlib/metadata/diagnose.py b/Lib/importlib/metadata/diagnose.py new file mode 100644 index 00000000000000..e405471ac4d943 --- /dev/null +++ b/Lib/importlib/metadata/diagnose.py @@ -0,0 +1,21 @@ +import sys + +from . import Distribution + + +def inspect(path): + print("Inspecting", path) + dists = list(Distribution.discover(path=[path])) + if not dists: + return + print("Found", len(dists), "packages:", end=' ') + print(', '.join(dist.name for dist in dists)) + + +def run(): + for path in sys.path: + inspect(path) + + +if __name__ == '__main__': + run() diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index e6b60c18caa052..ec4441c9116118 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -4,6 +4,17 @@ as_file, files, Package, + Anchor, +) + +from ._functional import ( + contents, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, ) from .abc import ResourceReader @@ -11,7 +22,15 @@ __all__ = [ 'Package', + 'Anchor', 'ResourceReader', 'as_file', 'files', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', ] diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index a3902535342612..ca5b06743b46a6 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -27,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() @@ -109,6 +109,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000000..9e3ea1547d486a --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,85 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return ( + resource.name + for resource + in _get_resource(anchor, path_names).iterdir() + ) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbecb0..b86cdeff57c4c2 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,7 +1,10 @@ import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile from . import abc @@ -62,7 +65,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +133,36 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*map(self._resolve, namespace_path)) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + """ + (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return dir + + @classmethod + def _candidate_paths(cls, path_str): + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +174,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84fab..96f117fec62c10 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f4d6e82331516f..8403ef9b44ad1a 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -135,7 +135,7 @@ class _incompatible_extension_module_restrictions: may not be imported in a subinterpreter. That implies modules that do not implement multi-phase init or that explicitly of out. - Likewise for modules import in a subinterpeter with its own GIL + Likewise for modules import in a subinterpreter with its own GIL when the extension does not support a per-interpreter GIL. This implies the module does not have a Py_mod_multiple_interpreters slot set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. @@ -145,7 +145,7 @@ class _incompatible_extension_module_restrictions: You can get the same effect as this function by implementing the basic interface of multi-phase init (PEP 489) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). """ def __init__(self, *, disable_check): @@ -171,36 +171,57 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + __class__ = loader_state['__class__'] + + # Reentrant calls from the same thread must be allowed to proceed without + # triggering the load again. + # exec_module() and self-referential imports are the primary ways this can + # happen, but in any case we must return something to avoid deadlock. + if loader_state['is_loading']: + return __class__.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = __class__.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) + return getattr(self, attr) def __delattr__(self, attr): @@ -235,6 +256,9 @@ def create_module(self, spec): def exec_module(self, module): """Make the module load lazily.""" + # Threading is only needed for lazy loading, and importlib.util can + # be pulled in at interpreter startup, so defer until needed. + import threading module.__spec__.loader = self.loader module.__loader__ = self.loader # Don't need to worry about deep-copying as trying to set an attribute @@ -244,5 +268,13 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index aaa22bef896602..0e7b40eb39bce8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -160,6 +160,7 @@ from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict +from weakref import ref as make_weakref # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication @@ -219,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None + ann = obj.__annotations__ obj_globals = None module_name = getattr(obj, '__module__', None) @@ -279,7 +274,13 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): if globals is None: globals = obj_globals if locals is None: - locals = obj_locals + locals = obj_locals or {} + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + locals = {param.__name__: param for param in type_params} | locals return_value = {key: value if not isinstance(value, str) else eval(value, globals, locals) @@ -306,9 +307,10 @@ def ismethoddescriptor(object): But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute but not a __set__ - attribute, but beyond that the set of attributes varies. __name__ is - usually sensible, and __doc__ often is. + An object passing this test has a __get__ attribute, but not a + __set__ attribute or a __delete__ attribute. Beyond that, the set + of attributes varies; __name__ is usually sensible, and __doc__ + often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because @@ -318,7 +320,9 @@ def ismethoddescriptor(object): # mutual exclusion return False tp = type(object) - return hasattr(tp, "__get__") and not hasattr(tp, "__set__") + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) def isdatadescriptor(object): """Return true if the object is a data descriptor. @@ -383,8 +387,10 @@ def isfunction(object): def _has_code_flag(f, flag): """Return true if ``f`` is a function (or a method or functools.partial - wrapper wrapping a function) whose code object has the given ``flag`` + wrapper wrapping a function or a functools.partialmethod wrapping a + function) whose code object has the given ``flag`` set in its flags.""" + f = functools._unwrap_partialmethod(f) while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) @@ -400,13 +406,13 @@ def isgeneratorfunction(obj): return _has_code_flag(obj, CO_GENERATOR) # A marker for markcoroutinefunction and iscoroutinefunction. -_is_coroutine_marker = object() +_is_coroutine_mark = object() def _has_coroutine_mark(f): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark def markcoroutinefunction(func): """ @@ -414,7 +420,7 @@ def markcoroutinefunction(func): """ if hasattr(func, '__func__'): func = func.__func__ - func._is_coroutine_marker = _is_coroutine_marker + func._is_coroutine_marker = _is_coroutine_mark return func def iscoroutinefunction(obj): @@ -760,18 +766,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -832,9 +834,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -957,6 +958,10 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -987,6 +992,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1032,79 +1038,6 @@ class ClassFoundException(Exception): pass -class _ClassFinder(ast.NodeVisitor): - - def __init__(self, cls, tree, lines, qualname): - self.stack = [] - self.cls = cls - self.tree = tree - self.lines = lines - self.qualname = qualname - self.lineno_found = [] - - def visit_FunctionDef(self, node): - self.stack.append(node.name) - self.stack.append('') - self.generic_visit(node) - self.stack.pop() - self.stack.pop() - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_ClassDef(self, node): - self.stack.append(node.name) - if self.qualname == '.'.join(self.stack): - # Return the decorator for the class if present - if node.decorator_list: - line_number = node.decorator_list[0].lineno - else: - line_number = node.lineno - - # decrement by one since lines starts with indexing by zero - self.lineno_found.append((line_number - 1, node.end_lineno)) - self.generic_visit(node) - self.stack.pop() - - def get_lineno(self): - self.visit(self.tree) - lineno_found_number = len(self.lineno_found) - if lineno_found_number == 0: - raise OSError('could not find class definition') - elif lineno_found_number == 1: - return self.lineno_found[0][0] - else: - # We have multiple candidates for the class definition. - # Now we have to guess. - - # First, let's see if there are any method definitions - for member in self.cls.__dict__.values(): - if (isinstance(member, types.FunctionType) and - member.__module__ == self.cls.__module__): - for lineno, end_lineno in self.lineno_found: - if lineno <= member.__code__.co_firstlineno <= end_lineno: - return lineno - - class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) - for lineno, end_lineno in self.lineno_found] - - # Maybe the class has a docstring and it's unique? - if self.cls.__doc__: - ret = None - for candidate, lineno in class_strings: - if self.cls.__doc__.strip() in candidate: - if ret is None: - ret = lineno - else: - break - else: - if ret is not None: - return ret - - # We are out of ideas, just return the last one found, which is - # slightly better than previous ones - return self.lineno_found[-1][0] - - def findsource(object): """Return the entire source file and starting line number for an object. @@ -1122,7 +1055,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) @@ -1137,11 +1070,11 @@ def findsource(object): return lines, 0 if isclass(object): - qualname = object.__qualname__ - source = ''.join(lines) - tree = ast.parse(source) - class_finder = _ClassFinder(object, tree, lines, qualname) - return lines, class_finder.get_lineno() + try: + firstlineno = object.__firstlineno__ + except AttributeError: + raise OSError('source code not available') + return lines, object.__firstlineno__ - 1 if ismethod(object): object = object.__func__ @@ -1155,15 +1088,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - try: - line = lines[lnum] - except IndexError: - raise OSError('lineno is out of bounds') - if pat.match(line): - break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') @@ -1837,9 +1763,16 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel + @functools.lru_cache() -def _shadowed_dict_from_mro_tuple(mro): - for entry in mro: +def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): + for weakref_entry in weakref_mro: + # Normally we'd have to check whether the result of weakref_entry() + # is None here, in case the object the weakref is pointing to has died. + # In this specific case, however, we know that the only caller of this + # function is `_shadowed_dict()`, and that therefore this weakref is + # guaranteed to point to an object that is still alive. + entry = weakref_entry() dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1849,8 +1782,19 @@ def _shadowed_dict_from_mro_tuple(mro): return class_dict return _sentinel + def _shadowed_dict(klass): - return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + # gh-118013: the inner function here is decorated with lru_cache for + # performance reasons, *but* make sure not to pass strong references + # to the items in the mro. Doing so can lead to unexpected memory + # consumption in cases where classes are dynamically created and + # destroyed, and the dynamically created classes happen to be the only + # objects that hold strong references to other objects that take up a + # significant amount of memory. + return _shadowed_dict_from_weakref_mro_tuple( + *[make_weakref(entry) for entry in _static_getmro(klass)] + ) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the @@ -2042,15 +1986,17 @@ def _signature_get_user_defined_method(cls, method_name): named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None or isinstance(meth, _NonUserDefinedCallables): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -2163,8 +2109,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -2267,7 +2215,12 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: @@ -2495,6 +2448,15 @@ def _signature_from_function(cls, func, skip_bound_arg=True, __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, @@ -2561,7 +2523,7 @@ def _signature_from_callable(obj, *, return sig try: - partialmethod = obj._partialmethod + partialmethod = obj.__partialmethod__ except AttributeError: pass else: @@ -2588,6 +2550,10 @@ def _signature_from_callable(obj, *, new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: @@ -2599,11 +2565,6 @@ def _signature_from_callable(obj, *, return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): - wrapped_sig = _get_signature_of(obj.func) - return _signature_get_partial(wrapped_sig, obj) - - sig = None if isinstance(obj, type): # obj is a class or a metaclass @@ -2611,88 +2572,72 @@ def _signature_from_callable(obj, *, # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: - sig = _get_signature_of(call) - else: - factory_method = None - new = _signature_get_user_defined_method(obj, '__new__') - init = _signature_get_user_defined_method(obj, '__init__') - - # Go through the MRO and see if any class has user-defined - # pure Python __new__ or __init__ method - for base in obj.__mro__: - # Now we check if the 'obj' class has an own '__new__' method - if new is not None and '__new__' in base.__dict__: - factory_method = new - break - # or an own '__init__' method - elif init is not None and '__init__' in base.__dict__: - factory_method = init - break + return _get_signature_of(call) - if factory_method is not None: - sig = _get_signature_of(factory_method) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'base' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, base, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + new = _signature_get_user_defined_method(obj, '__new__') + init = _signature_get_user_defined_method(obj, '__init__') + + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. + try: + text_sig = base.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if (obj.__init__ is object.__init__ and + obj.__new__ is object.__new__): + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) - elif not isinstance(obj, _NonUserDefinedCallables): + else: # An object with __call__ - # We also check that the 'obj' is not an instance of - # types.WrapperDescriptorType or types.MethodWrapperType to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') + call = getattr_static(type(obj), '__call__', None) if call is not None: try: - sig = _get_signature_of(call) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) @@ -3164,6 +3109,8 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -3184,10 +3131,10 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -3269,20 +3216,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( @@ -3316,6 +3265,16 @@ def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): + return self.format() + + def format(self, *, max_width=None): + """Create a string representation of the Signature object. + + If *max_width* integer is passed, + signature will try to fit into the *max_width*. + If signature is longer than *max_width*, + all parameters will be on separate lines. + """ result = [] render_pos_only_separator = False render_kw_only_separator = True @@ -3353,6 +3312,8 @@ def __str__(self): result.append('/') rendered = '({})'.format(', '.join(result)) + if max_width is not None and len(rendered) > max_width: + rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: anno = formatannotation(self.return_annotation) diff --git a/Lib/io.py b/Lib/io.py index 50ce97436ac1d1..f0e2fa15d5abcf 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -45,7 +45,8 @@ "FileIO", "BytesIO", "StringIO", "BufferedIOBase", "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", - "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"] + "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] import _io diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index f5aba434fd4253..9cef275f7ae2fc 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -1086,7 +1086,11 @@ def is_private(self): """ return any(self.network_address in priv_network and self.broadcast_address in priv_network - for priv_network in self._constants._private_networks) + for priv_network in self._constants._private_networks) and all( + self.network_address not in network and + self.broadcast_address not in network + for network in self._constants._private_networks_exceptions + ) @property def is_global(self): @@ -1333,18 +1337,41 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -1389,6 +1416,16 @@ def is_link_local(self): """ return self in self._constants._linklocal_network + @property + def ipv6_mapped(self): + """Return the IPv4-mapped IPv6 address. + + Returns: + The IPv4-mapped IPv6 address per RFC 4291. + + """ + return IPv6Address(f'::ffff:{self}') + class IPv4Interface(IPv4Address): @@ -1548,13 +1585,15 @@ class _IPv4Constants: _public_network = IPv4Network('100.64.0.0/10') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.0/24'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), @@ -1565,6 +1604,11 @@ class _IPv4Constants: IPv4Network('255.255.255.255/32'), ] + _private_networks_exceptions = [ + IPv4Network('192.0.0.9/32'), + IPv4Network('192.0.0.10/32'), + ] + _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') @@ -1970,6 +2014,9 @@ def __eq__(self, other): return False return self._scope_id == getattr(other, '_scope_id', None) + def __reduce__(self): + return (self.__class__, (str(self),)) + @property def scope_id(self): """Identifier of a particular zone of the address's scope. @@ -2036,27 +2083,42 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: return ipv4_mapped.is_private - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return not self.is_private @@ -2080,6 +2142,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2196,7 +2261,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2297,19 +2362,31 @@ class _IPv6Constants: _multicast_network = IPv6Network('ff00::/8') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), + IPv6Network('64:ff9b:1::/48'), IPv6Network('100::/64'), IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), + # IANA says N/A, let's consider it not globally reachable to be safe + IPv6Network('2002::/16'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] + _private_networks_exceptions = [ + IPv6Network('2001:1::1/128'), + IPv6Network('2001:1::2/128'), + IPv6Network('2001:3::/32'), + IPv6Network('2001:4:112::/48'), + IPv6Network('2001:20::/28'), + IPv6Network('2001:30::/28'), + ] + _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index c5d9ae2d0d5d04..d69a45d6793069 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -200,10 +200,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': + if nextchar == '}': + raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx) raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end - 1) if object_pairs_hook is not None: @@ -240,13 +243,17 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() + nextchar = s[end:end + 1] except IndexError: pass + if nextchar == ']': + raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) return values, end diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a8..323332f064edf8 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -174,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' @@ -244,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -272,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 0490b8c0be11df..fdfc3372bcca02 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -13,7 +13,6 @@ import argparse import json import sys -from pathlib import Path def main(): @@ -22,11 +21,9 @@ def main(): 'to validate and pretty-print JSON objects.') parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument('infile', nargs='?', - type=argparse.FileType(encoding="utf-8"), help='a JSON file to be validated or pretty-printed', - default=sys.stdin) + default='-') parser.add_argument('outfile', nargs='?', - type=Path, help='write the output of infile to outfile', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, @@ -59,23 +56,30 @@ def main(): dump_args['indent'] = None dump_args['separators'] = ',', ':' - with options.infile as infile: + try: + if options.infile == '-': + infile = sys.stdin + else: + infile = open(options.infile, encoding='utf-8') try: if options.json_lines: objs = (json.loads(line) for line in infile) else: objs = (json.load(infile),) + finally: + if infile is not sys.stdin: + infile.close() - if options.outfile is None: - out = sys.stdout - else: - out = options.outfile.open('w', encoding='utf-8') - with out as outfile: - for obj in objs: - json.dump(obj, outfile, **dump_args) - outfile.write('\n') - except ValueError as e: - raise SystemExit(e) + if options.outfile is None: + outfile = sys.stdout + else: + outfile = open(options.outfile, 'w', encoding='utf-8') + with outfile: + for obj in objs: + json.dump(obj, outfile, **dump_args) + outfile.write('\n') + except ValueError as e: + raise SystemExit(e) if __name__ == '__main__': diff --git a/Lib/linecache.py b/Lib/linecache.py index c1c988d9df436c..3462f1c451ba29 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,11 +5,6 @@ that name. """ -import functools -import sys -import os -import tokenize - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -68,6 +63,11 @@ def checkcache(filename=None): size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ + try: + # This import can fail if the interpreter is shutting down + import os + except ImportError: + return try: stat = os.stat(fullname) except OSError: @@ -82,6 +82,13 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slows down the startup sequence. + import os + import sys + import tokenize + if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) @@ -137,7 +144,9 @@ def updatecache(filename, module_globals=None): lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname @@ -166,21 +175,21 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) cache[filename] = (get_lines,) return True return False + def _register_code(code, string, name): cache[code] = ( len(string), diff --git a/Lib/locale.py b/Lib/locale.py index 55c819ca80a160..d8c09f1123d318 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -541,12 +541,14 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ import warnings - warnings.warn( - "Use setlocale(), getencoding() and getlocale() instead", - DeprecationWarning, stacklevel=2 - ) + warnings._deprecated( + "locale.getdefaultlocale", + "{name!r} is deprecated and slated for removal in Python {remove}. " + "Use setlocale(), getencoding() and getlocale() instead.", + remove=(3, 15)) return _getdefaultlocale(envvars) + def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): try: # check if it's supported by the _locale module @@ -1458,7 +1460,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index eb7e020d1edfc0..3f4144226b40ec 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -56,7 +56,7 @@ # #_startTime is used as the base when calculating the relative time of events # -_startTime = time.time() +_startTime = time.time_ns() # #raiseExceptions is used to see if exceptions during handling should be @@ -300,7 +300,7 @@ def __init__(self, name, level, pathname, lineno, """ Initialize a logging record with interesting information. """ - ct = time.time() + ct = time.time_ns() self.name = name self.msg = msg # @@ -339,9 +339,17 @@ def __init__(self, name, level, pathname, lineno, self.stack_info = sinfo self.lineno = lineno self.funcName = func - self.created = ct - self.msecs = int((ct - int(ct)) * 1000) + 0.0 # see gh-89047 - self.relativeCreated = (self.created - _startTime) * 1000 + self.created = ct / 1e9 # ns to float seconds + # Get the number of whole milliseconds (0-999) in the fractional part of seconds. + # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms + # Convert to float by adding 0.0 for historical reasons. See gh-89047 + self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0 + if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000: + # ns -> sec conversion can round up, e.g: + # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec + self.msecs = 0.0 + + self.relativeCreated = (ct - _startTime) / 1e6 if logThreads: self.thread = threading.get_ident() self.threadName = threading.current_thread().name @@ -572,7 +580,7 @@ class Formatter(object): %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() + %(created)f Time when the LogRecord was created (time.time_ns() / 1e9 return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time @@ -1493,7 +1501,7 @@ def debug(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) + logger.debug("Houston, we have a %s", "thorny problem", exc_info=True) """ if self.isEnabledFor(DEBUG): self._log(DEBUG, msg, args, **kwargs) @@ -1505,7 +1513,7 @@ def info(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.info("Houston, we have a %s", "notable problem", exc_info=1) + logger.info("Houston, we have a %s", "notable problem", exc_info=True) """ if self.isEnabledFor(INFO): self._log(INFO, msg, args, **kwargs) @@ -1517,7 +1525,7 @@ def warning(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) + logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True) """ if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) @@ -1529,7 +1537,7 @@ def error(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.error("Houston, we have a %s", "major problem", exc_info=1) + logger.error("Houston, we have a %s", "major problem", exc_info=True) """ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) @@ -1547,7 +1555,7 @@ def critical(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.critical("Houston, we have a %s", "major disaster", exc_info=1) + logger.critical("Houston, we have a %s", "major disaster", exc_info=True) """ if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) @@ -1565,7 +1573,7 @@ def log(self, level, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.log(level, "We have a %s", "mysterious problem", exc_info=1) + logger.log(level, "We have a %s", "mysterious problem", exc_info=True) """ if not isinstance(level, int): if raiseExceptions: @@ -1949,18 +1957,11 @@ def hasHandlers(self): """ return self.logger.hasHandlers() - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + def _log(self, level, msg, args, **kwargs): """ Low-level log implementation, proxied to allow nested logger adapters. """ - return self.logger._log( - level, - msg, - args, - exc_info=exc_info, - extra=extra, - stack_info=stack_info, - ) + return self.logger._log(level, msg, args, **kwargs) @property def manager(self): @@ -2020,7 +2021,7 @@ def basicConfig(**kwargs): that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. handlers If specified, this should be an iterable of already created - handlers, which will be added to the root handler. Any handler + handlers, which will be added to the root logger. Any handler in the list which does not have a formatter assigned will be assigned the formatter created in this function. force If this keyword is specified as true, any existing handlers diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 951bba73913cb3..95e129ae988c24 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -482,10 +482,10 @@ def configure_custom(self, config): c = config.pop('()') if not callable(c): c = self.resolve(c) - props = config.pop('.', None) # Check for valid identifiers - kwargs = {k: config[k] for k in config if valid_ident(k)} + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} result = c(**kwargs) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -667,10 +667,9 @@ def configure_formatter(self, config): except TypeError as te: if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # logging.Formatter and its subclasses expect the `fmt` + # parameter instead of `format`. Retry passing configuration + # with `fmt`. config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) @@ -726,16 +725,16 @@ def add_filters(self, filterer, filters): def _configure_queue_handler(self, klass, **kwargs): if 'queue' in kwargs: - q = kwargs['queue'] + q = kwargs.pop('queue') else: q = queue.Queue() # unbounded - rhl = kwargs.get('respect_handler_level', False) - if 'listener' in kwargs: - lklass = kwargs['listener'] - else: - lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl) - handler = klass(q) + + rhl = kwargs.pop('respect_handler_level', False) + lklass = kwargs.pop('listener', logging.handlers.QueueListener) + handlers = kwargs.pop('handlers', []) + + listener = lklass(q, *handlers, respect_handler_level=rhl) + handler = klass(q, **kwargs) handler.listener = listener return handler @@ -762,37 +761,63 @@ def configure_handler(self, config): klass = cname else: klass = self.resolve(cname) - if issubclass(klass, logging.handlers.MemoryHandler) and\ - 'target' in config: - # Special case for handler which refers to another handler - try: - tn = config['target'] - th = self.config['handlers'][tn] - if not isinstance(th, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('target not configured yet') - config['target'] = th - except Exception as e: - raise ValueError('Unable to set target handler %r' % tn) from e + if issubclass(klass, logging.handlers.MemoryHandler): + if 'flushLevel' in config: + config['flushLevel'] = logging._checkLevel(config['flushLevel']) + if 'target' in config: + # Special case for handler which refers to another handler + try: + tn = config['target'] + th = self.config['handlers'][tn] + if not isinstance(th, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('target not configured yet') + config['target'] = th + except Exception as e: + raise ValueError('Unable to set target handler %r' % tn) from e elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers - if 'handlers' not in config: - raise ValueError('No handlers specified for a QueueHandler') + # if 'handlers' not in config: + # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: qspec = config['queue'] - if not isinstance(qspec, queue.Queue): - if isinstance(qspec, str): - q = self.resolve(qspec) - if not callable(q): - raise TypeError('Invalid queue specifier %r' % qspec) - q = q() - elif isinstance(qspec, dict): - if '()' not in qspec: - raise TypeError('Invalid queue specifier %r' % qspec) - q = self.configure_custom(dict(qspec)) - else: + + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = q() + elif isinstance(qspec, dict): + if '()' not in qspec: raise TypeError('Invalid queue specifier %r' % qspec) - config['queue'] = q + config['queue'] = self.configure_custom(dict(qspec)) + else: + from multiprocessing.queues import Queue as MPQueue + + if not isinstance(qspec, (queue.Queue, MPQueue)): + # Safely check if 'qspec' is an instance of Manager.Queue + # / Manager.JoinableQueue + + from multiprocessing import Manager as MM + from multiprocessing.managers import BaseProxy + + # if it's not an instance of BaseProxy, it also can't be + # an instance of Manager.Queue / Manager.JoinableQueue + if isinstance(qspec, BaseProxy): + # Sometimes manager or queue creation might fail + # (e.g. see issue gh-120868). In that case, any + # exception during the creation of these queues will + # propagate up to the caller and be wrapped in a + # `ValueError`, whose cause will indicate the details of + # the failure. + mm = MM() + proxy_queue = mm.Queue() + proxy_joinable_queue = mm.JoinableQueue() + if not isinstance(qspec, (type(proxy_queue), type(proxy_joinable_queue))): + raise TypeError('Invalid queue specifier %r' % qspec) + else: + raise TypeError('Invalid queue specifier %r' % qspec) + if 'listener' in config: lspec = config['listener'] if isinstance(lspec, type): @@ -813,18 +838,19 @@ def configure_handler(self, config): if not callable(listener): raise TypeError('Invalid listener specifier %r' % lspec) config['listener'] = listener - hlist = [] - try: - for hn in config['handlers']: - h = self.config['handlers'][hn] - if not isinstance(h, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('Required handler %r ' - 'is not configured yet' % hn) - hlist.append(h) - except Exception as e: - raise ValueError('Unable to set required handler %r' % hn) from e - config['handlers'] = hlist + if 'handlers' in config: + hlist = [] + try: + for hn in config['handlers']: + h = self.config['handlers'][hn] + if not isinstance(h, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('Required handler %r ' + 'is not configured yet' % hn) + hlist.append(h) + except Exception as e: + raise ValueError('Unable to set required handler %r' % hn) from e + config['handlers'] = hlist elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) @@ -835,8 +861,7 @@ def configure_handler(self, config): factory = functools.partial(self._configure_queue_handler, klass) else: factory = klass - props = config.pop('.', None) - kwargs = {k: config[k] for k in config if valid_ident(k)} + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} try: result = factory(**kwargs) except TypeError as te: @@ -854,6 +879,7 @@ def configure_handler(self, config): result.setLevel(logging._checkLevel(level)) if filters: self.add_filters(result, filters) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -977,7 +1003,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): A simple TCP socket-based logging config receiver. """ - allow_reuse_address = 1 + allow_reuse_address = True + allow_reuse_port = True def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index e75da9b7b1de64..0fa40f56e998d5 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -187,15 +193,15 @@ def shouldRollover(self, record): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ - # See bpo-45401: Never rollover anything other than regular files - if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): - return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) #due to non-posix-compliant Windows feature if self.stream.tell() + len(msg) >= self.maxBytes: + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False return True return False @@ -232,19 +238,19 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$" + extMatch = r"(? (plen + 1) and not fileName[plen+1].isdigit()): - continue - - if fileName[:plen] == prefix: - suffix = fileName[plen:] - # See bpo-45628: The date/time suffix could be anywhere in the - # filename - parts = suffix.split('.') - for part in parts: - if self.extMatch.match(part): + if self.namer is None: + prefix = baseName + '.' + plen = len(prefix) + for fileName in fileNames: + if fileName[:plen] == prefix: + suffix = fileName[plen:] + if self.extMatch.fullmatch(suffix): + result.append(os.path.join(dirName, fileName)) + else: + for fileName in fileNames: + # Our files could be just about anything after custom naming, + # but they should contain the datetime suffix. + # Try to find the datetime suffix in the file name and verify + # that the file name can be generated by this handler. + m = self.extMatch.search(fileName) + while m: + dfn = self.namer(self.baseFilename + "." + m[0]) + if os.path.basename(dfn) == fileName: result.append(os.path.join(dirName, fileName)) break + m = self.extMatch.search(fileName, m.start() + 1) + if len(result) < self.backupCount: result = [] else: @@ -410,17 +422,14 @@ def doRollover(self): then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ - if self.stream: - self.stream.close() - self.stream = None # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) - dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) + dstNow = time.localtime(currentTime)[-1] dstThen = timeTuple[-1] if dstNow != dstThen: if dstNow: @@ -431,26 +440,19 @@ def doRollover(self): dfn = self.rotation_filename(self.baseFilename + "." + time.strftime(self.suffix, timeTuple)) if os.path.exists(dfn): - os.remove(dfn) + # Already rolled over. + return + + if self.stream: + self.stream.close() + self.stream = None self.rotate(self.baseFilename, dfn) if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() - newRolloverAt = self.computeRollover(currentTime) - while newRolloverAt <= currentTime: - newRolloverAt = newRolloverAt + self.interval - #If DST changes and midnight or weekly rollover, adjust for this. - if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: - dstAtRollover = time.localtime(newRolloverAt)[-1] - if dstNow != dstAtRollover: - if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour - addend = -3600 - else: # DST bows out before next rollover, so we need to add an hour - addend = 3600 - newRolloverAt += addend - self.rolloverAt = newRolloverAt + self.rolloverAt = self.computeRollover(currentTime) class WatchedFileHandler(logging.FileHandler): """ @@ -466,8 +468,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -483,9 +484,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -495,6 +498,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -502,18 +508,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ @@ -830,10 +841,8 @@ class SysLogHandler(logging.Handler): "local7": LOG_LOCAL7, } - #The map below appears to be trivially lowercasing the key. However, - #there's more to it than meets the eye - in some locales, lowercasing - #gives unexpected results. See SF #1524081: in the Turkish locale, - #"INFO".lower() != "info" + # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept + # for backwards compatibility. priority_map = { "DEBUG" : "debug", "INFO" : "info", @@ -1588,6 +1597,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/lzma.py b/Lib/lzma.py index 800f52198fbb79..946066aa0fba56 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -25,11 +25,11 @@ import io import os from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 import _compression -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -92,7 +92,7 @@ def __init__(self, filename=None, mode="r", *, """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if mode in ("r", "rb"): if check != -1: @@ -137,7 +137,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -153,12 +153,20 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 59834a2b3b5243..b00d9e8634c785 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() @@ -540,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -698,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: @@ -778,10 +834,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1089,10 +1146,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1146,7 +1217,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1169,7 +1244,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 37228de4828de5..bacee8ba164b48 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -40,7 +40,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -116,11 +116,17 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) - scheme, url = urllib.parse._splittype(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -140,13 +146,25 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext_lower]) + base, ext = splitext(base + self.suffix_map[ext_lower]) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None ext = ext.lower() @@ -167,9 +185,9 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() @@ -186,11 +204,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -296,7 +314,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -304,17 +322,27 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -327,10 +355,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -528,6 +556,7 @@ def _default_mime_types(): '.tiff' : 'image/tiff', '.tif' : 'image/tiff', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -545,6 +574,8 @@ def _default_mime_types(): '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', @@ -554,9 +585,11 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', @@ -587,7 +620,6 @@ def _default_mime_types(): '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', - '.webp': 'image/webp', '.xul' : 'text/xul', } diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb9b4..ac478ee7f51722 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index dbbf106f680964..b7e1e132172d02 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -19,7 +19,6 @@ import tempfile import itertools -import _multiprocessing from . import util @@ -28,6 +27,7 @@ _ForkingPickler = reduction.ForkingPickler try: + import _multiprocessing import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: @@ -476,8 +476,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c @@ -1011,8 +1012,20 @@ def _exhaustive_wait(handles, timeout): # returning the first signalled might create starvation issues.) L = list(handles) ready = [] + # Windows limits WaitForMultipleObjects at 64 handles, and we use a + # few for synchronisation, so we switch to batched waits at 60. + if len(L) > 60: + try: + res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) + except TimeoutError: + return [] + ready.extend(L[i] for i in res) + if res: + L = [h for i, h in enumerate(L) if i > res[0] & i not in res] + timeout = 0 while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + short_L = L[:60] if len(L) > 60 else L + res = _winapi.WaitForMultipleObjects(short_L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index de8a264829dff3..ddcc7e7900999e 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -167,7 +167,7 @@ def allow_connection_pickling(self): ''' # This is undocumented. In previous versions of multiprocessing # its only effect was to make socket objects inheritable on Windows. - from . import connection + from . import connection # noqa: F401 def set_executable(self, executable): '''Sets the path to a python.exe or pythonw.exe binary used to run diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae2f4e..53b8c492675878 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -271,6 +272,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +281,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 273c22a7654f05..0f5f9f64c2de9e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -156,7 +156,7 @@ def __init__(self, registry, address, authkey, serializer): Listener, Client = listener_client[serializer] # do authentication later - self.listener = Listener(address=address, backlog=16) + self.listener = Listener(address=address, backlog=128) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} @@ -1152,10 +1152,10 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort', '__imul__' + '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__', + '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', + 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', + 'remove', 'reverse', 'sort', )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1165,16 +1165,24 @@ def __imul__(self, value): self._callmethod('__imul__', (value,)) return self + __class_getitem__ = classmethod(types.GenericAlias) + -DictProxy = MakeProxyType('DictProxy', ( - '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__setitem__', 'clear', 'copy', 'get', 'items', +_BaseDictProxy = MakeProxyType('DictProxy', ( + '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', + '__len__', '__or__', '__reversed__', '__ror__', + '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) -DictProxy._method_to_typeid_ = { +_BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } +class DictProxy(_BaseDictProxy): + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..a57ef6bdad5ccc 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index af044305709e56..62fb0ddbf91a5d 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -74,7 +75,8 @@ def __init__(self, process_obj): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) @@ -101,18 +103,20 @@ def duplicate_for_child(self, handle): return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): - if self.returncode is None: - if timeout is None: - msecs = _winapi.INFINITE - else: - msecs = max(0, int(timeout * 1000 + 0.5)) - - res = _winapi.WaitForSingleObject(int(self._handle), msecs) - if res == _winapi.WAIT_OBJECT_0: - code = _winapi.GetExitCodeProcess(self._handle) - if code == TERMINATE: - code = -signal.SIGTERM - self.returncode = code + if self.returncode is not None: + return self.returncode + + if timeout is None: + msecs = _winapi.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _winapi.WaitForSingleObject(int(self._handle), msecs) + if res == _winapi.WAIT_OBJECT_0: + code = _winapi.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code return self.returncode @@ -120,18 +124,22 @@ def poll(self): return self.wait(timeout=0) def terminate(self): - if self.returncode is None: - try: - _winapi.TerminateProcess(int(self._handle), TERMINATE) - except PermissionError: - # ERROR_ACCESS_DENIED (winerror 5) is received when the - # process already died. - code = _winapi.GetExitCodeProcess(int(self._handle)) - if code == _winapi.STILL_ACTIVE: - raise - self.returncode = code - else: - self.returncode = -signal.SIGTERM + if self.returncode is not None: + return + + try: + _winapi.TerminateProcess(int(self._handle), TERMINATE) + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: + raise + + # gh-113009: Don't set self.returncode. Even if GetExitCodeProcess() + # returns an exit code different than STILL_ACTIVE, the process can + # still be running. Only set self.returncode once WaitForSingleObject() + # returns WAIT_OBJECT_0 in wait(). kill = terminate diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd325138..b45f7df476f7d8 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 852ae87b276861..925f043900004e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index 66076509a1202e..b8afb0fbed3a3c 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -123,7 +123,7 @@ def _start(self): from .connection import Listener assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey) + self._listener = Listener(authkey=process.current_process().authkey, backlog=128) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 8e41f461cc934e..20ddd9c50e3d88 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -29,8 +29,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -61,6 +65,7 @@ def __init__(self): self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -84,9 +89,16 @@ def _stop(self): os.close(self._fd) self._fd = None - os.waitpid(self._pid, 0) + _, status = os.waitpid(self._pid, 0) + self._pid = None + try: + self._exitcode = os.waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None + def getfd(self): self.ensure_running() return self._fd @@ -119,6 +131,7 @@ def ensure_running(self): pass self._fd = None self._pid = None + self._exitcode = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') @@ -221,6 +234,8 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: @@ -242,6 +257,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -251,10 +267,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn( - f'resource_tracker: There appear to be {len(rtype_cache)} ' - f'leaked {rtype} objects to clean up at shutdown: {rtype_cache}' - ) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -265,6 +288,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 9a1e5aa17b87a2..99a8ce3320ad4e 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -71,8 +71,9 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -82,6 +83,7 @@ def __init__(self, name=None, create=False, size=0): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -116,8 +118,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - resource_tracker.register(self._name, "shared_memory") + if self._track: + resource_tracker.register(self._name, "shared_memory") else: @@ -236,12 +238,20 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" + if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) - resource_tracker.unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" @@ -529,6 +539,6 @@ def index(self, value): if value == entry: return position else: - raise ValueError(f"{value!r} not in this container") + raise ValueError("ShareableList.index(x): x not in list") __class_getitem__ = classmethod(types.GenericAlias) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 28c77df1c32ea8..4f471fbde71ace 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -14,7 +14,7 @@ import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does -from subprocess import _args_from_interpreter_flags +from subprocess import _args_from_interpreter_flags # noqa: F401 from . import process @@ -43,19 +43,19 @@ def sub_debug(msg, *args): if _logger: - _logger.log(SUBDEBUG, msg, *args) + _logger.log(SUBDEBUG, msg, *args, stacklevel=2) def debug(msg, *args): if _logger: - _logger.log(DEBUG, msg, *args) + _logger.log(DEBUG, msg, *args, stacklevel=2) def info(msg, *args): if _logger: - _logger.log(INFO, msg, *args) + _logger.log(INFO, msg, *args, stacklevel=2) def sub_warning(msg, *args): if _logger: - _logger.log(SUBWARNING, msg, *args) + _logger.log(SUBWARNING, msg, *args, stacklevel=2) def get_logger(): ''' @@ -102,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 3061a4a5ef4c56..1b1873f08b608b 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,17 +19,17 @@ import os import sys -import stat import genericpath from genericpath import * __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "curdir","pardir","sep","pathsep","defpath","altsep", + "ismount","isreserved","expanduser","expandvars","normpath", + "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] + "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", + "isdevdrive"] def _get_bothseps(path): if isinstance(path, bytes): @@ -77,12 +77,6 @@ def normcase(s): return s.replace('/', '\\').lower() -# Return whether a path is absolute. -# Trivial in Posix, harder on Windows. -# For Windows it is absolute if it starts with a slash or backslash (current -# volume), or if a pathname after the volume-letter-and-colon or UNC-resource -# starts with a slash or backslash. - def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) @@ -90,16 +84,15 @@ def isabs(s): sep = b'\\' altsep = b'/' colon_sep = b':\\' + double_sep = b'\\\\' else: sep = '\\' altsep = '/' colon_sep = ':\\' + double_sep = '\\\\' s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. - # LEGACY BUG: isabs("/x") should be false since the path has no drive. - if s.startswith(sep) or s.startswith(colon_sep, 1): - return True - return False + return s.startswith(colon_sep, 1) or s.startswith(double_sep) # Join two (or more) paths. @@ -108,16 +101,14 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_root, result_path = splitroot(path) - for p in map(os.fspath, paths): + for p in paths: p_drive, p_root, p_path = splitroot(p) if p_root: # Second path is absolute @@ -141,7 +132,7 @@ def join(path, *paths): result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and not result_root and - result_drive and result_drive[-1:] not in colon + seps): + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): @@ -175,56 +166,52 @@ def splitdrive(p): return drive, root + tail -def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: +try: + from nt import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' - normp = p.replace(altsep, sep) - if normp[:1] == sep: - if normp[1:2] == sep: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, empty, empty - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, empty, empty - return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: - # Relative path with root, e.g. \Windows - return empty, p[:1], p[1:] - elif normp[1:2] == colon: - if normp[2:3] == sep: - # Absolute drive-letter path, e.g. X:\Windows - return p[:2], p[2:3], p[3:] + sep = '\\' + altsep = '/' + colon = ':' + unc_prefix = '\\\\?\\UNC\\' + empty = '' + normp = p.replace(altsep, sep) + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] else: - # Relative path with drive, e.g. X:Windows - return p[:2], empty, p[2:] - else: - # Relative path, e.g. Windows - return empty, empty, p + # Relative path, e.g. Windows + return empty, empty, p # Split a path in head (everything up to the last '/') and tail (the @@ -276,33 +263,6 @@ def dirname(p): return split(p)[0] -# Is a path a junction? - -if hasattr(os.stat_result, 'st_reparse_tag'): - def isjunction(path): - """Test whether a path is a junction""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT) -else: - def isjunction(path): - """Test whether a path is a junction""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - st = os.lstat(path) - except (OSError, ValueError): - return False - return True - # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) @@ -337,6 +297,40 @@ def ismount(path): return False +_reserved_chars = frozenset( + {chr(i) for i in range(32)} | + {'"', '*', ':', '<', '>', '?', '|', '/', '\\'} +) + +_reserved_names = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +def isreserved(path): + """Return true if the pathname is reserved by the system.""" + # Refer to "Naming Files, Paths, and Namespaces": + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep) + return any(_isreservedname(name) for name in reversed(path.split(sep))) + +def _isreservedname(name): + """Return true if the filename is reserved by the system.""" + # Trailing dots and spaces are reserved. + if name[-1:] in ('.', ' '): + return name not in ('.', '..') + # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. + # ASCII control characters (0-31) are reserved. + # Colon is reserved for file streams (e.g. "name:stream[:type]"). + if _reserved_chars.intersection(name): + return True + # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules + # are complex and vary across Windows versions. On the side of + # caution, return True for names that may not be reserved. + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names + + # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, @@ -352,24 +346,23 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -520,7 +513,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -559,14 +552,6 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case @@ -709,7 +694,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -717,7 +703,8 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) if not had_prefix and not isabs(path): @@ -770,6 +757,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -781,22 +771,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -807,29 +795,28 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' @@ -843,9 +830,6 @@ def commonpath(paths): drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, r, p in drivesplits] - if len({r for d, r, p in drivesplits}) != 1: - raise ValueError("Can't mix absolute and relative paths") - # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. @@ -853,6 +837,12 @@ def commonpath(paths): raise ValueError("Paths don't have the same drive") drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] @@ -873,13 +863,15 @@ def commonpath(paths): try: - # The isdir(), isfile(), islink() and exists() implementations in - # genericpath use os.stat(). This is overkill on Windows. Use simpler + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler # builtin functions if they are available. from nt import _path_isdir as isdir from nt import _path_isfile as isfile from nt import _path_islink as islink + from nt import _path_isjunction as isjunction from nt import _path_exists as exists + from nt import _path_lexists as lexists except ImportError: # Use genericpath.* as imported above pass @@ -887,15 +879,12 @@ def commonpath(paths): try: from nt import _path_isdevdrive -except ImportError: - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - # Never a Dev Drive - return False -else: def isdevdrive(path): """Determines whether the specified path is on a Windows Dev Drive.""" try: return _path_isdevdrive(abspath(path)) except OSError: return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/opcode.py b/Lib/opcode.py index 88f4df7c0e8c38..2698609cd5636d 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -12,8 +12,8 @@ import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, - HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] @@ -36,6 +36,8 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_special_method_names = _opcode.get_special_method_names() +_common_constants = [AssertionError, NotImplementedError] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] @@ -56,6 +58,9 @@ "COMPARE_OP": { "counter": 1, }, + "CONTAINS_OP": { + "counter": 1, + }, "BINARY_SUBSCR": { "counter": 1, }, diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189a499..6d2a762bc95b6d 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -239,7 +239,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +257,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +276,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +288,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +315,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): @@ -415,7 +415,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/os.py b/Lib/os.py index a17946750ea7e7..aaa758d955fe4c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -131,6 +139,7 @@ def _add(str, fn): _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") @@ -171,6 +180,7 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") @@ -279,6 +289,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -359,58 +373,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs @@ -469,24 +470,59 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -512,31 +548,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") @@ -742,7 +765,18 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): + if _exists("_create_environ"): + def refresh(self): + data = _create_environ() + if name == 'nt': + data = {self.encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + self._data.clear() + self._data.update(data) + +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -772,8 +806,8 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping def getenv(key, default=None): diff --git a/Lib/pathlib.py b/Lib/pathlib.py deleted file mode 100644 index 5c1c71ecec2805..00000000000000 --- a/Lib/pathlib.py +++ /dev/null @@ -1,1716 +0,0 @@ -"""Object-oriented filesystem paths. - -This module provides classes to represent abstract paths and concrete -paths with operations that have semantics appropriate for different -operating systems. -""" - -import contextlib -import fnmatch -import functools -import io -import ntpath -import os -import posixpath -import re -import sys -import warnings -from _collections_abc import Sequence -from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO - -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - -# -# Internals -# - -# Maximum number of symlinks to follow in _PathBase.resolve() -_MAX_SYMLINKS = 40 - -# Reference for Windows paths can be found at -# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . -_WIN_RESERVED_NAMES = frozenset( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | - {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} -) - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - - -@functools.cache -def _is_case_sensitive(pathmod): - return pathmod.normcase('Aa') == 'Aa' - -# -# Globbing helpers -# - - -# fnmatch.translate() returns a regular expression that includes a prefix and -# a suffix, which enable matching newlines and ensure the end of the string is -# matched, respectively. These features are undesirable for our implementation -# of PurePatch.match(), which represents path separators as newlines and joins -# pattern segments together. As a workaround, we define a slice object that -# can remove the prefix and suffix from any translate() result. See the -# _compile_pattern_lines() function for more details. -_FNMATCH_PREFIX, _FNMATCH_SUFFIX = fnmatch.translate('_').split('_') -_FNMATCH_SLICE = slice(len(_FNMATCH_PREFIX), -len(_FNMATCH_SUFFIX)) -_SWAP_SEP_AND_NEWLINE = { - '/': str.maketrans({'/': '\n', '\n': '/'}), - '\\': str.maketrans({'\\': '\n', '\n': '\\'}), -} - - -@functools.lru_cache(maxsize=256) -def _compile_pattern(pat, case_sensitive): - """Compile given glob pattern to a re.Pattern object (observing case - sensitivity), or None if the pattern should match everything.""" - if pat == '*': - return None - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - return re.compile(fnmatch.translate(pat), flags).match - - -@functools.lru_cache() -def _compile_pattern_lines(pattern_lines, case_sensitive): - """Compile the given pattern lines to an `re.Pattern` object. - - The *pattern_lines* argument is a glob-style pattern (e.g. '**/*.py') with - its path separators and newlines swapped (e.g. '**\n*.py`). By using - newlines to separate path components, and not setting `re.DOTALL`, we - ensure that the `*` wildcard cannot match path separators. - - The returned `re.Pattern` object may have its `match()` method called to - match a complete pattern, or `search()` to match from the right. The - argument supplied to these methods must also have its path separators and - newlines swapped. - """ - - # Match the start of the path, or just after a path separator - parts = ['^'] - for part in pattern_lines.splitlines(keepends=True): - if part == '*\n': - part = r'.+\n' - elif part == '*': - part = r'.+' - elif part == '**\n': - # '**/' component: we use '[\s\S]' rather than '.' so that path - # separators (i.e. newlines) are matched. The trailing '^' ensures - # we terminate after a path separator (i.e. on a new line). - part = r'[\s\S]*^' - elif part == '**': - # '**' component. - part = r'[\s\S]*' - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - else: - # Any other component: pass to fnmatch.translate(). We slice off - # the common prefix and suffix added by translate() to ensure that - # re.DOTALL is not set, and the end of the string not matched, - # respectively. With DOTALL not set, '*' wildcards will not match - # path separators, because the '.' characters in the pattern will - # not match newlines. - part = fnmatch.translate(part)[_FNMATCH_SLICE] - parts.append(part) - # Match the end of the path, always. - parts.append(r'\Z') - flags = re.MULTILINE - if not case_sensitive: - flags |= re.IGNORECASE - return re.compile(''.join(parts), flags=flags) - - -def _select_children(parent_paths, dir_only, follow_symlinks, match): - """Yield direct children of given paths, filtering by name and type.""" - if follow_symlinks is None: - follow_symlinks = True - for parent_path in parent_paths: - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with parent_path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if dir_only: - try: - if not entry.is_dir(follow_symlinks=follow_symlinks): - continue - except OSError: - continue - name = entry.name - if match is None or match(name): - yield parent_path._make_child_relpath(name) - - -def _select_recursive(parent_paths, dir_only, follow_symlinks): - """Yield given paths and all their subdirectories, recursively.""" - if follow_symlinks is None: - follow_symlinks = False - for parent_path in parent_paths: - paths = [parent_path] - while paths: - path = paths.pop() - yield path - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - paths.append(path._make_child_relpath(entry.name)) - continue - except OSError: - pass - if not dir_only: - yield path._make_child_relpath(entry.name) - - -def _select_unique(paths): - """Yields the given paths, filtering out duplicates.""" - yielded = set() - try: - for path in paths: - path_str = str(path) - if path_str not in yielded: - yield path - yielded.add(path_str) - finally: - yielded.clear() - - -# -# Public API -# - -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is called on - a path object. - """ - pass - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -class PurePath: - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_lines_cached` slot stores the string path with path separators - # and newlines swapped. This is used to implement `match()`. - '_lines_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `_PathBase.resolve()`. This prevents duplicate - # work from occurring when `resolve()` calls `stat()` or `readlink()`. - '_resolving', - ) - pathmod = os.path - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - self._raw_paths = paths - self._resolving = False - - def with_segments(self, *pathsegments): - """Construct a new path object from any number of path-like objects. - Subclasses may override this method to customize how new path objects - are created from methods like `iterdir()`. - """ - return type(self)(*pathsegments) - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.pathmod.sep - altsep = cls.pathmod.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.pathmod.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - def _load_parts(self): - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.pathmod.join(*paths) - drv, root, tail = self._parse_path(path) - self._drv = drv - self._root = root - self._tail_cached = tail - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.pathmod.sep.join(tail) - elif tail and cls.pathmod.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.pathmod.sep.join(tail) - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - def __fspath__(self): - return str(self) - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - return str(self).replace(self.pathmod.sep, '/') - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _is_case_sensitive(self.pathmod): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) - return self._parts_normcase_cached - - @property - def _lines(self): - # Path with separators and newlines swapped, for pattern matching. - try: - return self._lines_cached - except AttributeError: - path_str = str(self) - if path_str == '.': - self._lines_cached = '' - else: - trans = _SWAP_SEP_AND_NEWLINE[self.pathmod.sep] - self._lines_cached = path_str.translate(trans) - return self._lines_cached - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._load_parts() - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._load_parts() - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._load_parts() - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - anchor = self.drive + self.root - return anchor - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - @property - def suffix(self): - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' - - @property - def suffixes(self): - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] - - @property - def stem(self): - """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name - - def with_name(self, name): - """Return a new path with the file name changed.""" - if not self.name: - raise ValueError("%r has an empty name" % (self,)) - m = self.pathmod - if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': - raise ValueError("Invalid name %r" % (name)) - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def with_stem(self, stem): - """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) - - def with_suffix(self, suffix): - """Return a new path with the file suffix changed. If the path - has no suffix, add given suffix. If the given suffix is an empty - string, remove the suffix from the path. - """ - m = self.pathmod - if m.sep in suffix or m.altsep and m.altsep in suffix: - raise ValueError("Invalid suffix %r" % (suffix,)) - if suffix and not suffix.startswith('.') or suffix == '.': - raise ValueError("Invalid suffix %r" % (suffix)) - name = self.name - if not name: - raise ValueError("%r has an empty name" % (self,)) - old_suffix = self.suffix - if not old_suffix: - name = name + suffix - else: - name = name[:-len(old_suffix)] + suffix - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, - remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - for step, path in enumerate([other] + list(other.parents)): - if self.is_relative_to(path): - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self.with_segments(*parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", - msg, remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - return other == self or other in self.parents - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.joinpath(key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - path = self._from_parsed_parts(drv, root, tail[:-1]) - path._resolving = self._resolving - return path - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.pathmod is ntpath: - # ntpath.isabs() is defective - see GH-44626. - return bool(self.drive and self.root) - elif self.pathmod is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - else: - return self.pathmod.isabs(str(self)) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - if self.pathmod is posixpath or not self._tail: - return False - - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if self.drive.startswith('\\\\'): - # UNC paths are never reserved. - return False - name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in _WIN_RESERVED_NAMES - - def match(self, path_pattern, *, case_sensitive=None): - """ - Return True if this path matches the given pattern. - """ - if not isinstance(path_pattern, PurePath): - path_pattern = self.with_segments(path_pattern) - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - pattern = _compile_pattern_lines(path_pattern._lines, case_sensitive) - if path_pattern.drive or path_pattern.root: - return pattern.match(self._lines) is not None - elif path_pattern._tail: - return pattern.search(self._lines) is not None - else: - raise ValueError("empty pattern") - - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = ntpath - __slots__ = () - - -# Filesystem-accessing classes - - -class _PathBase(PurePath): - """Base class for concrete path objects. - - This class provides dummy implementations for many methods that derived - classes can override selectively; the default implementations raise - UnsupportedOperation. The most basic methods, such as stat() and open(), - directly raise UnsupportedOperation; these basic methods are called by - other methods such as is_dir() and read_text(). - - The Path class derives this class to implement local filesystem paths. - Users may derive their own classes to implement virtual filesystem paths, - such as paths in archive files or on remote storage systems. - """ - __slots__ = () - __bytes__ = None - __fspath__ = None # virtual paths have no local file system representation - - @classmethod - def _unsupported(cls, method_name): - msg = f"{cls.__name__}.{method_name}() is unsupported" - if issubclass(cls, Path): - msg += " on this system" - raise UnsupportedOperation(msg) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - self._unsupported("stat") - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self, *, follow_symlinks=True): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self, *, follow_symlinks=True): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - # Junctions are a Windows-only feature, not present in POSIX nor the - # majority of virtual filesystems. There is no cross-platform idiom - # to check for junctions (using stat().st_mode). - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return (st.st_ino == other_st.st_ino and - st.st_dev == other_st.st_dev) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - self._unsupported("open") - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - encoding = io.text_encoding(encoding) - with self.open(mode='r', encoding=encoding, errors=errors) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - encoding = io.text_encoding(encoding) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - self._unsupported("iterdir") - - def _scandir(self): - # Emulate os.scandir(), which returns an object that can be used as a - # context manager. This method is called by walk() and glob(). - return contextlib.nullcontext(self.iterdir()) - - def _make_child_relpath(self, name): - sep = self.pathmod.sep - lines_name = name.replace('\n', sep) - lines_str = self._lines - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{sep}{name}' - lines_str = f'{lines_str}\n{lines_name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - lines_str = f'{lines_str}{lines_name}' - else: - path_str = name - lines_str = lines_name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - path._lines_cached = lines_str - return path - - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - return self._glob(pattern, case_sensitive, follow_symlinks) - - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - return self._glob(f'**/{pattern}', case_sensitive, follow_symlinks) - - def _glob(self, pattern, case_sensitive, follow_symlinks): - path_pattern = self.with_segments(pattern) - if path_pattern.drive or path_pattern.root: - raise NotImplementedError("Non-relative patterns are unsupported") - elif not path_pattern._tail: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - - pattern_parts = list(path_pattern._tail) - if pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - pattern_parts.append('') - if pattern_parts[-1] == '**': - # GH-70303: '**' only matches directories. Add trailing slash. - warnings.warn( - "Pattern ending '**' will match files and directories in a " - "future Python release. Add a trailing slash to match only " - "directories and remove this warning.", - FutureWarning, 3) - pattern_parts.append('') - - if case_sensitive is None: - # TODO: evaluate case-sensitivity of each directory in _select_children(). - case_sensitive = _is_case_sensitive(self.pathmod) - - # If symlinks are handled consistently, and the pattern does not - # contain '..' components, then we can use a 'walk-and-match' strategy - # when expanding '**' wildcards. When a '**' wildcard is encountered, - # all following pattern parts are immediately consumed and used to - # build a `re.Pattern` object. This pattern is used to filter the - # recursive walk. As a result, pattern parts following a '**' wildcard - # do not perform any filesystem access, which can be much faster! - filter_paths = follow_symlinks is not None and '..' not in pattern_parts - deduplicate_paths = False - paths = iter([self] if self.is_dir() else []) - part_idx = 0 - while part_idx < len(pattern_parts): - part = pattern_parts[part_idx] - part_idx += 1 - if part == '': - # Trailing slash. - pass - elif part == '..': - paths = (path._make_child_relpath('..') for path in paths) - elif part == '**': - # Consume adjacent '**' components. - while part_idx < len(pattern_parts) and pattern_parts[part_idx] == '**': - part_idx += 1 - - if filter_paths and part_idx < len(pattern_parts) and pattern_parts[part_idx] != '': - dir_only = pattern_parts[-1] == '' - paths = _select_recursive(paths, dir_only, follow_symlinks) - - # Filter out paths that don't match pattern. - prefix_len = len(self._make_child_relpath('_')._lines) - 1 - match = _compile_pattern_lines(path_pattern._lines, case_sensitive).match - paths = (path for path in paths if match(path._lines[prefix_len:])) - return paths - - dir_only = part_idx < len(pattern_parts) - paths = _select_recursive(paths, dir_only, follow_symlinks) - if deduplicate_paths: - # De-duplicate if we've already seen a '**' component. - paths = _select_unique(paths) - deduplicate_paths = True - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - else: - dir_only = part_idx < len(pattern_parts) - match = _compile_pattern(part, case_sensitive) - paths = _select_children(paths, dir_only, follow_symlinks, match) - return paths - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_obj = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_obj as scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - self._unsupported("absolute") - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - self._unsupported("expanduser") - - @classmethod - def home(cls): - """Return a new path pointing to expanduser('~'). - """ - return cls("~").expanduser() - - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - self._unsupported("readlink") - readlink._supported = False - - def _split_stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - return self._from_parsed_parts(self.drive, self.root, []), self._tail[::-1] - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - if self._resolving: - return self - try: - path = self.absolute() - except UnsupportedOperation: - path = self - - # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported - # and (in non-strict mode) we can improve performance by not calling `stat()`. - querying = strict or getattr(self.readlink, '_supported', True) - link_count = 0 - stat_cache = {} - target_cache = {} - path, parts = path._split_stack() - while parts: - part = parts.pop() - if part == '..': - if not path._tail: - if path.root: - # Delete '..' segment immediately following root - continue - elif path._tail[-1] != '..': - # Delete '..' segment and its predecessor - path = path.parent - continue - # Join the current part onto the path. - path_parent = path - path = path._make_child_relpath(part) - if querying and part != '..': - path._resolving = True - try: - st = stat_cache.get(path) - if st is None: - st = stat_cache[path] = path.stat(follow_symlinks=False) - if S_ISLNK(st.st_mode): - # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are - # encountered during resolution. - link_count += 1 - if link_count >= _MAX_SYMLINKS: - raise OSError(ELOOP, "Too many symbolic links in path", str(path)) - target = target_cache.get(path) - if target is None: - target = target_cache[path] = path.readlink() - target, target_parts = target._split_stack() - # If the symlink target is absolute (like '/etc/hosts'), set the current - # path to its uppermost parent (like '/'). If not, the symlink target is - # relative to the symlink parent, which we recorded earlier. - path = target if target.root else path_parent - # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to - # the stack of unresolved path parts. - parts.extend(target_parts) - elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", str(path)) - except OSError: - if strict: - raise - else: - querying = False - path._resolving = False - return path - - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - self._unsupported("symlink_to") - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - self._unsupported("hardlink_to") - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - self._unsupported("touch") - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - self._unsupported("mkdir") - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - self._unsupported("rename") - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - self._unsupported("replace") - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - self._unsupported("chmod") - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - self._unsupported("unlink") - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - self._unsupported("rmdir") - - def owner(self): - """ - Return the login name of the file owner. - """ - self._unsupported("owner") - - def group(self): - """ - Return the group name of the file gid. - """ - self._unsupported("group") - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - cls._unsupported("from_uri") - - def as_uri(self): - """Return the path as a URI.""" - self._unsupported("as_uri") - - -class Path(_PathBase): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - __bytes__ = PurePath.__bytes__ - __fspath__ = PurePath.__fspath__ - as_uri = PurePath.as_uri - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - return (self._make_child_relpath(name) for name in os.listdir(self)) - - def _scandir(self): - return os.scandir(self) - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - if not self.root and not self._tail: - result = self.with_segments(cwd) - result._str = cwd - return result - return self.with_segments(cwd, self) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self): - """ - Return the login name of the file owner. - """ - return pwd.getpwuid(self.stat().st_uid).pw_name - - if grp: - def group(self): - """ - Return the group name of the file gid. - """ - return grp.getgrgid(self.stat().st_gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000000..2298a249529460 --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,12 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +from ._os import * +from ._local import * + +__all__ = (_os.__all__ + + _local.__all__) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py new file mode 100644 index 00000000000000..05f55badd77c58 --- /dev/null +++ b/Lib/pathlib/_abc.py @@ -0,0 +1,937 @@ +""" +Abstract base classes for rich path objects. + +This module is published as a PyPI package called "pathlib-abc". + +This module is also a *PRIVATE* part of the Python standard library, where +it's developed alongside pathlib. If it finds success and maturity as a PyPI +package, it could become a public part of the standard library. + +Two base classes are defined here -- PurePathBase and PathBase -- that +resemble pathlib's PurePath and Path respectively. +""" + +import functools +import operator +import posixpath +from glob import _GlobberBase, _no_recurse_symlinks +from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from ._os import UnsupportedOperation, copyfileobj + + +@functools.cache +def _is_case_sensitive(parser): + return parser.normcase('Aa') == 'Aa' + + + +class ParserBase: + """Base class for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide PurePathBase functionality. Each PurePathBase + subclass references its path parser via a 'parser' class attribute. + + Every method in this base class raises an UnsupportedOperation exception. + """ + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" + + @property + def sep(self): + """The character used to separate path components.""" + raise UnsupportedOperation(self._unsupported_msg('sep')) + + def join(self, path, *paths): + """Join path segments.""" + raise UnsupportedOperation(self._unsupported_msg('join()')) + + def split(self, path): + """Split the path into a pair (head, tail), where *head* is everything + before the final path separator, and *tail* is everything after. + Either part may be empty. + """ + raise UnsupportedOperation(self._unsupported_msg('split()')) + + def splitdrive(self, path): + """Split the path into a 2-item tuple (drive, tail), where *drive* is + a device name or mount point, and *tail* is everything after the + drive. Either part may be empty.""" + raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + + def splitext(self, path): + """Split the path into a pair (root, ext), where *ext* is empty or + begins with a begins with a period and contains at most one period, + and *root* is everything before the extension.""" + raise UnsupportedOperation(self._unsupported_msg('splitext()')) + + def normcase(self, path): + """Normalize the case of the path.""" + raise UnsupportedOperation(self._unsupported_msg('normcase()')) + + def isabs(self, path): + """Returns whether the path is absolute, i.e. unaffected by the + current directory or drive.""" + raise UnsupportedOperation(self._unsupported_msg('isabs()')) + + +class PathGlobber(_GlobberBase): + """ + Class providing shell-style globbing for path objects. + """ + + lexists = operator.methodcaller('exists', follow_symlinks=False) + add_slash = operator.methodcaller('joinpath', '') + + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + import contextlib + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path.""" + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir().""" + return entry + + +class PurePathBase: + """Base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, + __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path + joining methods accept only strings, not os.PathLike objects more broadly. + """ + + __slots__ = ( + # The `_raw_path` slot store a joined string path. This is set in the + # `__init__()` method. + '_raw_path', + + # The '_resolving' slot stores a boolean indicating whether the path + # is being processed by `PathBase.resolve()`. This prevents duplicate + # work from occurring when `resolve()` calls `stat()` or `readlink()`. + '_resolving', + ) + parser = ParserBase() + _globber = PathGlobber + + def __init__(self, path, *paths): + self._raw_path = self.parser.join(path, *paths) if paths else path + if not isinstance(self._raw_path, str): + raise TypeError( + f"path should be a str, not {type(self._raw_path).__name__!r}") + self._resolving = False + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + return self._raw_path + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + return self.parser.splitdrive(self.anchor)[0] + + @property + def root(self): + """The root of the path, if any.""" + return self.parser.splitdrive(self.anchor)[1] + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self._stack[0] + + @property + def name(self): + """The final path component, if any.""" + return self.parser.split(self._raw_path)[1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + return self.parser.splitext(self.name)[1] + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + return self.parser.splitext(self.name)[0] + + def with_name(self, name): + """Return a new path with the file name changed.""" + split = self.parser.split + if split(name)[0]: + raise ValueError(f"Invalid name {name!r}") + return self.with_segments(split(self._raw_path)[0], name) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not isinstance(other, PurePathBase): + other = self.with_segments(other) + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if not part or part == '.': + pass + elif not walk_up: + raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") + elif part == '..': + raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") + else: + parts0.append('..') + return self.with_segments('', *reversed(parts0)) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not isinstance(other, PurePathBase): + other = self.with_segments(other) + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + return False + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if part and part != '.': + return False + return True + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + anchor, parts = self._stack + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self._raw_path, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self._raw_path, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self._raw_path) + except TypeError: + return NotImplemented + + @property + def _stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = self.parser.split + path = self._raw_path + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + @property + def parent(self): + """The logical parent of the path.""" + path = self._raw_path + parent = self.parser.split(path)[0] + if path != parent: + parent = self.with_segments(parent) + parent._resolving = self._resolving + return parent + return self + + @property + def parents(self): + """A sequence of this path's logical parents.""" + split = self.parser.split + path = self._raw_path + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + return self.parser.isabs(self._raw_path) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + return str(self) + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not isinstance(path_pattern, PurePathBase): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + sep = path_pattern.parser.sep + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = self._globber(sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern._pattern_str) + return match(self._pattern_str) is not None + + + +class PathBase(PurePathBase): + """Base class for concrete path objects. + + This class provides dummy implementations for many methods that derived + classes can override selectively; the default implementations raise + UnsupportedOperation. The most basic methods, such as stat() and open(), + directly raise UnsupportedOperation; these basic methods are called by + other methods such as is_dir() and read_text(). + + The Path class derives this class to implement local filesystem paths. + Users may derive their own classes to implement virtual filesystem paths, + such as paths in archive files or on remote storage systems. + """ + __slots__ = () + + # Maximum number of symlinks to follow in resolve() + _max_symlinks = 40 + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + raise UnsupportedOperation(self._unsupported_msg('stat()')) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return self.stat(follow_symlinks=False) + + + # Convenience functions for querying the stat results + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + try: + self.stat(follow_symlinks=follow_symlinks) + except (OSError, ValueError): + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + try: + parent_dev = self.parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = self.parent.stat().st_ino + return ino == parent_ino + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + try: + return S_ISLNK(self.lstat().st_mode) + except (OSError, ValueError): + return False + + def is_junction(self): + """ + Whether this path is a junction. + """ + # Junctions are a Windows-only feature, not present in POSIX nor the + # majority of virtual filesystems. There is no cross-platform idiom + # to check for junctions (using stat().st_mode). + return False + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def _samefile_safe(self, other_path): + """ + Like samefile(), but returns False rather than raising OSError. + """ + try: + return self.samefile(other_path) + except (OSError, ValueError): + return False + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + raise UnsupportedOperation(self._unsupported_msg('open()')) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + + def _glob_selector(self, parts, case_sensitive, recurse_symlinks): + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) + return globber.selector(parts) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + anchor, parts = pattern._stack + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + select = self._glob_selector(parts, case_sensitive, recurse_symlinks) + return select(self) + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + try: + if child.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + raise UnsupportedOperation(self._unsupported_msg('absolute()')) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls('').absolute() + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + raise UnsupportedOperation(self._unsupported_msg('expanduser()')) + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + return cls("~").expanduser() + + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + raise UnsupportedOperation(self._unsupported_msg('readlink()')) + readlink._supported = False + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + if self._resolving: + return self + + def getcwd(): + return str(self.with_segments().absolute()) + + if strict or getattr(self.readlink, '_supported', True): + def lstat(path_str): + path = self.with_segments(path_str) + path._resolving = True + return path.lstat() + + def readlink(path_str): + path = self.with_segments(path_str) + path._resolving = True + return str(path.readlink()) + else: + # If the user has *not* overridden the `readlink()` method, then + # symlinks are unsupported and (in non-strict mode) we can improve + # performance by not calling `path.lstat()`. + def skip(path_str): + # This exception will be internally consumed by `_realpath()`. + raise OSError("Operation skipped.") + + lstat = readlink = skip + + return self.with_segments(posixpath._realpath( + str(self), strict, self.parser.sep, + getcwd=getcwd, lstat=lstat, readlink=readlink, + maxlinks=self._max_symlinks)) + + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + raise UnsupportedOperation(self._unsupported_msg('touch()')) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + + # Metadata keys supported by this path type. + _readable_metadata = _writable_metadata = frozenset() + + def _read_metadata(self, keys=None, *, follow_symlinks=True): + """ + Returns path metadata as a dict with string keys. + """ + raise UnsupportedOperation(self._unsupported_msg('_read_metadata()')) + + def _write_metadata(self, metadata, *, follow_symlinks=True): + """ + Sets path metadata from the given dict with string keys. + """ + raise UnsupportedOperation(self._unsupported_msg('_write_metadata()')) + + def _copy_metadata(self, target, *, follow_symlinks=True): + """ + Copies metadata (permissions, timestamps, etc) from this path to target. + """ + # Metadata types supported by both source and target. + keys = self._readable_metadata & target._writable_metadata + if keys: + metadata = self._read_metadata(keys, follow_symlinks=follow_symlinks) + target._write_metadata(metadata, follow_symlinks=follow_symlinks) + + def copy(self, target, *, follow_symlinks=True, preserve_metadata=False): + """ + Copy the contents of this file to the given target. If this file is a + symlink and follow_symlinks is false, a symlink will be created at the + target. + """ + if not isinstance(target, PathBase): + target = self.with_segments(target) + if self._samefile_safe(target): + raise OSError(f"{self!r} and {target!r} are the same file") + if not follow_symlinks and self.is_symlink(): + target.symlink_to(self.readlink()) + if preserve_metadata: + self._copy_metadata(target, follow_symlinks=False) + return + with self.open('rb') as source_f: + try: + with target.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not target.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {target}') from e + else: + raise + if preserve_metadata: + self._copy_metadata(target) + + def copytree(self, target, *, follow_symlinks=True, dirs_exist_ok=False, + ignore=None, on_error=None): + """ + Recursively copy this directory tree to the given destination. + """ + if not isinstance(target, PathBase): + target = self.with_segments(target) + if on_error is None: + def on_error(err): + raise err + stack = [(self, target)] + while stack: + source_dir, target_dir = stack.pop() + try: + sources = source_dir.iterdir() + target_dir.mkdir(exist_ok=dirs_exist_ok) + for source in sources: + if ignore and ignore(source): + continue + try: + if source.is_dir(follow_symlinks=follow_symlinks): + stack.append((source, target_dir.joinpath(source.name))) + else: + source.copy(target_dir.joinpath(source.name), + follow_symlinks=follow_symlinks) + except OSError as err: + on_error(err) + except OSError as err: + on_error(err) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + raise UnsupportedOperation(self._unsupported_msg('rename()')) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + raise UnsupportedOperation(self._unsupported_msg('replace()')) + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + raise UnsupportedOperation(self._unsupported_msg('chmod()')) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + raise UnsupportedOperation(self._unsupported_msg('unlink()')) + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + raise UnsupportedOperation(self._unsupported_msg('rmdir()')) + + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + raise UnsupportedOperation(self._unsupported_msg('owner()')) + + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + raise UnsupportedOperation(self._unsupported_msg('group()')) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) + + def as_uri(self): + """Return the path as a URI.""" + raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000000..eae8a30c876f19 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,937 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +from glob import _StringGlobber +from itertools import chain +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from ._os import (UnsupportedOperation, copyfile, file_metadata_keys, + read_file_metadata, write_file_metadata) +from ._abc import PurePathBase, PathBase + + +__all__ = [ + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath(PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + _globber = _StringGlobber + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is ntpath and self.parser is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.parser.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + return PathBase.is_dir(self, follow_symlinks=follow_symlinks) + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + return PathBase.is_file(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.write_text(self, data, encoding, errors, newline) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if pattern.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + parts = pattern._tail.copy() + if not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + raw = pattern._raw_path + if raw[-1] in (self.parser.sep, self.parser.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + _readable_metadata = _writable_metadata = file_metadata_keys + _read_metadata = read_file_metadata + _write_metadata = write_file_metadata + + if copyfile: + def copy(self, target, *, follow_symlinks=True, preserve_metadata=False): + """ + Copy the contents of this file to the given target. If this file is a + symlink and follow_symlinks is false, a symlink will be created at the + target. + """ + try: + target = os.fspath(target) + except TypeError: + if not isinstance(target, PathBase): + raise + else: + try: + copyfile(os.fspath(self), target, follow_symlinks) + return + except UnsupportedOperation: + pass # Fall through to generic code. + PathBase.copy(self, target, follow_symlinks=follow_symlinks, + preserve_metadata=preserve_metadata) + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000000..164ee8e9034427 --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,277 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +import os +import stat +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +__all__ = ["UnsupportedOperation"] + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + +def get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def clonefd(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + clonefd = None + + +if posix and hasattr(posix, '_fcopyfile'): + def copyfd(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +elif hasattr(os, 'copy_file_range'): + def copyfd(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +elif hasattr(os, 'sendfile'): + def copyfd(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + copyfd = None + + +if _winapi and hasattr(_winapi, 'CopyFile2') and hasattr(os.stat_result, 'st_file_attributes'): + def _is_dirlink(path): + try: + st = os.lstat(path) + except (OSError, ValueError): + return False + return (st.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY and + st.st_reparse_tag == stat.IO_REPARSE_TAG_SYMLINK) + + def copyfile(source, target, follow_symlinks): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + if follow_symlinks: + _winapi.CopyFile2(source, target, 0) + else: + # Use COPY_FILE_COPY_SYMLINK to copy a file symlink. + flags = _winapi.COPY_FILE_COPY_SYMLINK + try: + _winapi.CopyFile2(source, target, flags) + return + except OSError as err: + # Check for ERROR_ACCESS_DENIED + if err.winerror == 5 and _is_dirlink(source): + pass + else: + raise + + # Add COPY_FILE_DIRECTORY to copy a directory symlink. + flags |= _winapi.COPY_FILE_DIRECTORY + try: + _winapi.CopyFile2(source, target, flags) + except OSError as err: + # Check for ERROR_INVALID_PARAMETER + if err.winerror == 87: + raise UnsupportedOperation(err) from None + else: + raise +else: + copyfile = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if clonefd: + try: + clonefd(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if copyfd: + copyfd(source_fd, target_fd) + return + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) + + +# Kinds of metadata supported by the operating system. +file_metadata_keys = {'mode', 'times_ns'} +if hasattr(os.stat_result, 'st_flags'): + file_metadata_keys.add('flags') +if hasattr(os, 'listxattr'): + file_metadata_keys.add('xattrs') +file_metadata_keys = frozenset(file_metadata_keys) + + +def read_file_metadata(path, keys=None, *, follow_symlinks=True): + """ + Returns local path metadata as a dict with string keys. + """ + if keys is None: + keys = file_metadata_keys + assert keys.issubset(file_metadata_keys) + result = {} + for key in keys: + if key == 'xattrs': + try: + result['xattrs'] = [ + (attr, os.getxattr(path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + continue + st = os.stat(path, follow_symlinks=follow_symlinks) + if key == 'mode': + result['mode'] = stat.S_IMODE(st.st_mode) + elif key == 'times_ns': + result['times_ns'] = st.st_atime_ns, st.st_mtime_ns + elif key == 'flags': + result['flags'] = st.st_flags + return result + + +def write_file_metadata(path, metadata, *, follow_symlinks=True): + """ + Sets local path metadata from the given dict with string keys. + """ + assert frozenset(metadata.keys()).issubset(file_metadata_keys) + + def _nop(*args, ns=None, follow_symlinks=None): + pass + + if follow_symlinks: + # use the real function if it exists + def lookup(name): + return getattr(os, name, _nop) + else: + # use the real function only if it exists + # *and* it supports follow_symlinks + def lookup(name): + fn = getattr(os, name, _nop) + if fn in os.supports_follow_symlinks: + return fn + return _nop + + times_ns = metadata.get('times_ns') + if times_ns is not None: + lookup("utime")(path, ns=times_ns, follow_symlinks=follow_symlinks) + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + xattrs = metadata.get('xattrs') + if xattrs is not None: + for attr, value in xattrs: + try: + os.setxattr(path, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + mode = metadata.get('mode') + if mode is not None: + try: + lookup("chmod")(path, mode, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + flags = metadata.get('flags') + if flags is not None: + try: + lookup("chflags")(path, flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise diff --git a/Lib/pdb.py b/Lib/pdb.py old mode 100755 new mode 100644 index 930cb91b8696a3..7ff973149b167b --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """ The Python Debugger Pdb ======================= @@ -76,17 +74,21 @@ import dis import code import glob +import token +import types import codeop import pprint import signal import inspect +import textwrap import tokenize -import functools import traceback import linecache +import _colorize from contextlib import contextmanager -from typing import Union +from rlcompleter import Completer +from types import CodeType class Restart(Exception): @@ -96,17 +98,50 @@ class Restart(Exception): __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] + +def find_first_executable_line(code): + """ Try to find the first executable line of the code object. + + Equivalently, find the line number of the instruction that's + after RESUME + + Return code.co_firstlineno if no executable line is found. + """ + prev = None + for instr in dis.get_instructions(code): + if prev is not None and prev.opname == 'RESUME': + if instr.positions.lineno is not None: + return instr.positions.lineno + return code.co_firstlineno + prev = instr + return code.co_firstlineno + def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: - return None + lines = linecache.getlines(filename) + if not lines: + return None + fp = io.StringIO(''.join(lines)) + funcdef = "" + funcstart = None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): - return funcname, filename, lineno + funcstart, funcdef = lineno, line + elif funcdef: + funcdef += line + + if funcdef: + try: + funccode = compile(funcdef, filename, 'exec').co_consts[0] + except SyntaxError: + continue + lineno_offset = find_first_executable_line(funccode) + return funcname, filename, funcstart + lineno_offset - 1 return None def lasti2lineno(code, lasti): @@ -124,68 +159,113 @@ def __repr__(self): return self -class _ScriptTarget(str): - def __new__(cls, val): - # Mutate self to be the "real path". - res = super().__new__(cls, os.path.realpath(val)) +class _ExecutableTarget: + filename: str + code: CodeType | str + namespace: dict - # Store the original path for error reporting. - res.orig = val - return res +class _ScriptTarget(_ExecutableTarget): + def __init__(self, target): + self._target = os.path.realpath(target) - def check(self): - if not os.path.exists(self): - print('Error:', self.orig, 'does not exist') + if not os.path.exists(self._target): + print(f'Error: {target} does not exist') + sys.exit(1) + if os.path.isdir(self._target): + print(f'Error: {target} is a directory') sys.exit(1) - # Replace pdb's dir with script's dir in front of module search path. - sys.path[0] = os.path.dirname(self) + # If safe_path(-P) is not set, sys.path[0] is the directory + # of pdb, and we should replace it with the directory of the script + if not sys.flags.safe_path: + sys.path[0] = os.path.dirname(self._target) + + def __repr__(self): + return self._target @property def filename(self): - return self + return self._target + + @property + def code(self): + # Open the file each time because the file may be modified + with io.open_code(self._target) as fp: + return f"exec(compile({fp.read()!r}, {self._target!r}, 'exec'))" @property def namespace(self): return dict( __name__='__main__', - __file__=self, + __file__=self._target, __builtins__=__builtins__, + __spec__=None, ) - @property - def code(self): - with io.open_code(self) as fp: - return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))" +class _ModuleTarget(_ExecutableTarget): + def __init__(self, target): + self._target = target -class _ModuleTarget(str): - def check(self): + import runpy try: - self._details + _, self._spec, self._code = runpy._get_module_details(self._target) + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) except Exception: traceback.print_exc() sys.exit(1) - @functools.cached_property - def _details(self): - import runpy - return runpy._get_module_details(self) + def __repr__(self): + return self._target @property def filename(self): - return self.code.co_filename + return self._code.co_filename @property def code(self): - name, spec, code = self._details - return code + return self._code @property - def _spec(self): - name, spec, code = self._details - return spec + def namespace(self): + return dict( + __name__='__main__', + __file__=os.path.normcase(os.path.abspath(self.filename)), + __package__=self._spec.parent, + __loader__=self._spec.loader, + __spec__=self._spec, + __builtins__=__builtins__, + ) + + +class _ZipTarget(_ExecutableTarget): + def __init__(self, target): + import runpy + + self._target = os.path.realpath(target) + sys.path.insert(0, self._target) + try: + _, self._spec, self._code = runpy._get_main_module_details() + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + def __repr__(self): + return self._target + + @property + def filename(self): + return self._code.co_filename + + @property + def code(self): + return self._code @property def namespace(self): @@ -199,6 +279,15 @@ def namespace(self): ) +class _PdbInteractiveConsole(code.InteractiveConsole): + def __init__(self, ns, message): + self._message = message + super().__init__(locals=ns, local_exit=True) + + def write(self, data): + self._message(data, end='') + + # Interaction prompt line will separate file and call info from code # text using value of line_prefix string. A newline and arrow may # be to your liking. You can set it once pdb is imported using the @@ -215,6 +304,10 @@ class Pdb(bdb.Bdb, cmd.Cmd): # but in case there are recursions, we stop at 999. MAX_CHAINED_EXCEPTION_DEPTH = 999 + _file_mtime_table = {} + + _last_pdb_instance = None + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -232,7 +325,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, try: import readline # remove some common file name delimiters - readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?') + readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') except ImportError: pass self.allow_kbdint = False @@ -268,6 +361,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 + def set_trace(self, frame=None): + Pdb._last_pdb_instance = self + if frame is None: + frame = sys._getframe().f_back + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -299,9 +398,12 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # cache it here to ensure that modifications are not overwritten. + # The f_locals dictionary used to be updated from the actual frame + # locals whenever the .f_locals accessor was called, so it was + # cached here to ensure that modifications were not overwritten. While + # the caching is no longer required now that f_locals is a direct proxy + # on optimized frames, it's also harmless, so the code structure has + # been left unchanged. self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) @@ -312,26 +414,12 @@ def setup(self, f, tb): self._chained_exceptions[self._chained_exception_index], ) - return self.execRcLines() - - # Can be executed earlier than 'setup' if desired - def execRcLines(self): - if not self.rcLines: - return - # local copy because of recursion - rcLines = self.rcLines - rcLines.reverse() - # execute every line only once - self.rcLines = [] - while rcLines: - line = rcLines.pop().strip() - if line and line[0] != '#': - if self.onecmd(line): - # if onecmd returns True, the command wants to exit - # from the interaction, save leftover rc lines - # to execute before next interaction - self.rcLines += reversed(rcLines) - return True + if self.rcLines: + self.cmdqueue = [ + line for line in self.rcLines + if line.strip() and not line.strip().startswith("#") + ] + self.rcLines = [] # Override Bdb methods @@ -354,6 +442,8 @@ def user_line(self, frame): if self.bp_commands(frame): self.interaction(frame, None) + user_opcode = user_line + def bp_commands(self, frame): """Call every command that was set for the current active breakpoint (if there is one). @@ -419,9 +509,23 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _validate_file_mtime(self): + """Check if the source file of the current frame has been modified since + the last time we saw it. If so, give a warning.""" + try: + filename = self.curframe.f_code.co_filename + mtime = os.path.getmtime(filename) + except Exception: + return + if (filename in self._file_mtime_table and + mtime != self._file_mtime_table[filename]): + self.message(f"*** WARNING: file '{filename}' was edited, " + "running stale code until the program is rerun") + self._file_mtime_table[filename] = mtime + # Called before loop, handles display expressions # Set up convenience variable containers - def preloop(self): + def _show_display(self): displaying = self.displaying.get(self.curframe) if displaying: for expr, oldvalue in displaying.items(): @@ -506,13 +610,17 @@ def interaction(self, frame, tb_or_exc): if isinstance(tb_or_exc, BaseException): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): - if self.setup(frame, tb): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") - self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) + self.setup(frame, tb) + # We should print the stack entry if and only if the user input + # is expected, and we should print it right before the user input. + # We achieve this by appending _pdbcmd_print_frame_status to the + # command queue. If cmdqueue is not exausted, the user input is + # not expected and we will not print the stack entry. + self.cmdqueue.append('_pdbcmd_print_frame_status') self._cmdloop() + # If _pdbcmd_print_frame_status is not used, pop it out + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() self.forget() def displayhook(self, obj): @@ -524,29 +632,108 @@ def displayhook(self, obj): self.message(repr(obj)) @contextmanager - def _disable_tab_completion(self): - if self.use_rawinput and self.completekey == 'tab': - try: - import readline - except ImportError: - yield - return - try: - readline.parse_and_bind('tab: self-insert') - yield - finally: - readline.parse_and_bind('tab: complete') - else: + def _disable_command_completion(self): + completenames = self.completenames + try: + self.completenames = self.completedefault yield + finally: + self.completenames = completenames + return + + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + + returns True if the source is executed, False otherwise + """ + + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + if not any(isinstance(const, CodeType) for const in code.co_consts): + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + except SyntaxError: + pass + else: + source = "__pdb_eval__['result'] = " + source + + # Add write-back to update the locals + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " __pdb_eval__['write_back'] = locals()") + + # Build a closure source code with freevars from locals like: + # def __pdb_outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + ns = {} + try: + exec(source_with_closure, {}, ns) + except Exception: + return False + code = ns["__pdb_outer"]() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) + + try: + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy["__pdb_eval__"] + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode - with self._disable_tab_completion(): + with self._disable_command_completion(): buffer = line continue_prompt = "... " while (code := codeop.compile_command(buffer, '', 'single')) is None: @@ -576,7 +763,8 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin @@ -584,6 +772,39 @@ def default(self, line): except: self._error_exc() + def _replace_convenience_variables(self, line): + """Replace the convenience variables in 'line' with their values. + e.g. $foo is replaced by __pdb_convenience_variables["foo"]. + Note: such pattern in string literals will be skipped""" + + if "$" not in line: + return line + + dollar_start = dollar_end = -1 + replace_variables = [] + try: + for t in tokenize.generate_tokens(io.StringIO(line).readline): + token_type, token_string, start, end, _ = t + if token_type == token.OP and token_string == '$': + dollar_start, dollar_end = start, end + elif start == dollar_end and token_type == token.NAME: + # line is a one-line command so we only care about column + replace_variables.append((dollar_start[1], end[1], token_string)) + except tokenize.TokenError: + return line + + if not replace_variables: + return line + + last_end = 0 + line_pieces = [] + for start, end, name in replace_variables: + line_pieces.append(line[last_end:start] + f'__pdb_convenience_variables["{name}"]') + last_end = end + line_pieces.append(line[last_end:]) + + return ''.join(line_pieces) + def precmd(self, line): """Handle alias expansion and ';;' separator.""" if not line.strip(): @@ -591,11 +812,20 @@ def precmd(self, line): args = line.split() while args[0] in self.aliases: line = self.aliases[args[0]] - ii = 1 - for tmpArg in args[1:]: - line = line.replace("%" + str(ii), - tmpArg) - ii += 1 + for idx in range(1, 10): + if f'%{idx}' in line: + if idx >= len(args): + self.error(f"Not enough arguments for alias '{args[0]}'") + # This is a no-op + return "!" + line = line.replace(f'%{idx}', args[idx]) + elif '%*' not in line: + if idx < len(args): + self.error(f"Too many arguments for alias '{args[0]}'") + # This is a no-op + return "!" + break + line = line.replace("%*", ' '.join(args[1:])) args = line.split() # split into ';;' separated commands @@ -605,11 +835,12 @@ def precmd(self, line): if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() - self.cmdqueue.append(next) + self.cmdqueue.insert(0, next) line = line[:marker].rstrip() # Replace all the convenience variables - line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) + line = self._replace_convenience_variables(line) + return line def onecmd(self, line): @@ -620,6 +851,11 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: + self._validate_file_mtime() + if line.startswith('_pdbcmd'): + command, arg, line = self.parseline(line) + if hasattr(self, command): + return getattr(self, command)(arg) return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) @@ -628,13 +864,15 @@ def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: - return + return False if cmd == 'silent': self.commands_silent[self.commands_bnum] = True - return # continue to handle other cmd def in the cmd list + return False # continue to handle other cmd def in the cmd list elif cmd == 'end': - self.cmdqueue = [] - return 1 # end of cmd list + return True # end of cmd list + elif cmd == 'EOF': + print('') + return True # end of cmd list cmdlist = self.commands[self.commands_bnum] if arg: cmdlist.append(cmd+' '+arg) @@ -648,14 +886,13 @@ def handle_command_def(self, line): # one of the resuming commands if func.__name__ in self.commands_resuming: self.commands_doprompt[self.commands_bnum] = False - self.cmdqueue = [] - return 1 - return + return True + return False # interface abstraction functions - def message(self, msg): - print(msg, file=self.stdout) + def message(self, msg, end='\n'): + print(msg, end=end, file=self.stdout) def error(self, msg): print('***', msg, file=self.stdout) @@ -670,6 +907,21 @@ def set_convenience_variable(self, frame, name, value): # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. + def completenames(self, text, line, begidx, endidx): + # Overwrite completenames() of cmd so for the command completion, + # if no current command matches, check for expressions as well + commands = super().completenames(text, line, begidx, endidx) + for alias in self.aliases: + if alias.startswith(text): + commands.append(alias) + if commands: + return commands + else: + expressions = self._complete_expression(text, line, begidx, endidx) + if expressions: + return expressions + return self.completedefault(text, line, begidx, endidx) + def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. if line.strip().endswith((':', ',')): @@ -704,6 +956,10 @@ def _complete_expression(self, text, line, begidx, endidx): # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. ns = {**self.curframe.f_globals, **self.curframe_locals} + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] if '.' in text: # Walk an attribute chain up to the last part, similar to what # rlcompleter does. This will bail if any of the parts are not @@ -721,6 +977,27 @@ def _complete_expression(self, text, line, begidx, endidx): # Complete a simple name. return [n for n in ns.keys() if n.startswith(text)] + def completedefault(self, text, line, begidx, endidx): + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] + + # Use rlcompleter to do the completion + state = 0 + matches = [] + completer = Completer(self.curframe.f_globals | self.curframe_locals) + while (match := completer.complete(text, state)) is not None: + matches.append(match) + state += 1 + return matches + + # Pdb meta commands, only intended to be used internally by pdb + + def _pdbcmd_print_frame_status(self, arg): + self.print_stack_trace(0) + self._show_display() + # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop @@ -881,7 +1158,7 @@ def do_break(self, arg, temporary = 0): #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name - lineno = code.co_firstlineno + lineno = find_first_executable_line(code) filename = code.co_filename except: # last thing to try @@ -959,7 +1236,7 @@ def lineinfo(self, identifier): if f: fname = f item = parts[1] - answer = find_function(item, fname) + answer = find_function(item, self.canonic(fname)) return answer or failed def checkline(self, filename, lineno): @@ -1151,16 +1428,24 @@ def do_clear(self, arg): complete_cl = _complete_location def do_where(self, arg): - """w(here) + """w(here) [count] - Print a stack trace, with the most recent frame at the bottom. + Print a stack trace. If count is not specified, print the full stack. + If count is 0, print the current frame entry. If count is positive, + print count entries from the most recent frame. If count is negative, + print -count entries from the least recent frame. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ - if arg: - self._print_invalid_arg(arg) - return - self.print_stack_trace() + if not arg: + count = None + else: + try: + count = int(arg) + except ValueError: + self.error('Invalid count (%s)' % arg) + return + self.print_stack_trace(count) do_w = do_where do_bt = do_where @@ -1735,7 +2020,9 @@ def do_interact(self, arg): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe_locals} - code.interact("*interactive*", local=ns) + console = _PdbInteractiveConsole(ns, message=self.message) + console.interact(banner="*pdb interact start*", + exitmsg="*exit from pdb interact command*") def do_alias(self, arg): """alias [name [command]] @@ -1774,7 +2061,18 @@ def do_alias(self, arg): else: self.error(f"Unknown alias '{args[0]}'") else: - self.aliases[args[0]] = ' '.join(args[1:]) + # Do a validation check to make sure no replaceable parameters + # are skipped if %* is not used. + alias = ' '.join(args[1:]) + if '%*' not in alias: + consecutive = True + for idx in range(1, 10): + if f'%{idx}' not in alias: + consecutive = False + if f'%{idx}' in alias and not consecutive: + self.error("Replaceable parameters must be consecutive") + return + self.aliases[args[0]] = alias def do_unalias(self, arg): """unalias name @@ -1802,10 +2100,22 @@ def complete_unalias(self, text, line, begidx, endidx): # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). - - def print_stack_trace(self): + # * if count is None, prints the full stack + # * if count = 0, prints the current frame entry + # * if count < 0, prints -count least recent frame entries + # * if count > 0, prints count most recent frame entries + + def print_stack_trace(self, count=None): + if count is None: + stack_to_print = self.stack + elif count == 0: + stack_to_print = [self.stack[self.curindex]] + elif count < 0: + stack_to_print = self.stack[:-count] + else: + stack_to_print = self.stack[-count:] try: - for frame_lineno in self.stack: + for frame_lineno in stack_to_print: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass @@ -1877,17 +2187,23 @@ def lookupmodule(self, filename): lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. + + filename could be in format of: + * an absolute path like '/path/to/file.py' + * a relative path like 'file.py' or 'dir/file.py' + * a module name like 'module' or 'package.module' + + files and modules will be searched in sys.path. """ - if os.path.isabs(filename) and os.path.exists(filename): - return filename - f = os.path.join(sys.path[0], filename) - if os.path.exists(f) and self.canonic(f) == self.mainpyfile: - return f - root, ext = os.path.splitext(filename) - if ext == '': - filename = filename + '.py' + if not filename.endswith('.py'): + # A module is passed in so convert it to equivalent file + filename = filename.replace('.', os.sep) + '.py' + if os.path.isabs(filename): - return filename + if os.path.exists(filename): + return filename + return None + for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) @@ -1896,7 +2212,7 @@ def lookupmodule(self, filename): return fullname return None - def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): + def _run(self, target: _ExecutableTarget): # When bdb sets tracing, a number of call and line events happen # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). Take special measures to @@ -1914,6 +2230,10 @@ def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): __main__.__dict__.clear() __main__.__dict__.update(target.namespace) + # Clear the mtime table for program reruns, assume all the files + # are up to date. + self._file_mtime_table.clear() + self.run(target.code) def _format_exc(self, exc: BaseException): @@ -2038,7 +2358,10 @@ def set_trace(*, header=None): an assertion fails). If given, *header* is printed to the console just before debugging begins. """ - pdb = Pdb() + if Pdb._last_pdb_instance is not None: + pdb = Pdb._last_pdb_instance + else: + pdb = Pdb() if header is not None: pdb.message(header) pdb.set_trace(sys._getframe().f_back) @@ -2113,15 +2436,19 @@ def main(): import argparse parser = argparse.ArgumentParser(prog="pdb", + usage="%(prog)s [-h] [-c command] (-m module | pyfile) [args ...]", description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False) - parser.add_argument('-c', '--command', action='append', default=[], metavar='command') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-m', metavar='module') - group.add_argument('pyfile', nargs='?') - parser.add_argument('args', nargs="*") + # We need to maunally get the script from args, because the first positional + # arguments could be either the script we need to debug, or the argument + # to the -m module + parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', + help='pdb commands to execute as if given in a .pdbrc file') + parser.add_argument('-m', metavar='module', dest='module') + parser.add_argument('args', nargs='*', + help="when -m is not specified, the first arg is the script to debug") if len(sys.argv) == 1: # If no arguments were given (python -m pdb), print the whole help message. @@ -2131,14 +2458,17 @@ def main(): opts = parser.parse_args() - if opts.m: - file = opts.m + if opts.module: + file = opts.module target = _ModuleTarget(file) else: - file = opts.pyfile - target = _ScriptTarget(file) - - target.check() + if not opts.args: + parser.error("no module or script to run") + file = opts.args.pop(0) + if file.endswith('.pyz'): + target = _ZipTarget(file) + else: + target = _ScriptTarget(file) sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list @@ -2147,13 +2477,10 @@ def main(): # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. pdb = Pdb() - pdb.rcLines.extend(opts.command) + pdb.rcLines.extend(opts.commands) while True: try: pdb._run(target) - if pdb._user_requested_quit: - break - print("The program finished and will be restarted") except Restart: print("Restarting", target, "with arguments:") print("\t" + " ".join(sys.argv[1:])) @@ -2161,16 +2488,19 @@ def main(): # In most cases SystemExit does not warrant a post-mortem session. print("The program exited via sys.exit(). Exit status:", end=' ') print(e) - except SyntaxError: - traceback.print_exc() - sys.exit(1) except BaseException as e: - traceback.print_exc() + traceback.print_exception(e, colorize=_colorize.can_colorize()) print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - pdb.interaction(None, e) - print("Post mortem debugger finished. The " + target + - " will be restarted") + try: + pdb.interaction(None, e) + except Restart: + print("Restarting", target, "with arguments:") + print("\t" + " ".join(sys.argv[1:])) + continue + if pdb._user_requested_quit: + break + print("The program finished and will be restarted") # When invoked as main program, invoke the debugger on a script diff --git a/Lib/pickle.py b/Lib/pickle.py index 4f5ad5b71e8899..d719ceb7a0b8e8 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -782,14 +782,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack(" 1: + name = '' if arg == '-' else arg + preamble = args.preamble.format(name=name) + output.write(preamble + '\n') + if arg == '-': + dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) + else: + with open(arg, 'rb') as f: + dis(f, output, memo, args.indentlevel, annotate) + finally: + if output is not sys.stdout: + output.close() diff --git a/Lib/platform.py b/Lib/platform.py old mode 100755 new mode 100644 index 7bb222088d5061..d6322c9d99d2f3 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ This module tries to retrieve as much platform-identifying data as possible. It makes this information available via function APIs. @@ -10,7 +8,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -118,6 +117,10 @@ import sys import functools import itertools +try: + import _wmi +except ImportError: + _wmi = None ### Globals & Constants @@ -312,24 +315,26 @@ def _syscmd_ver(system='', release='', version='', version = _norm_version(version) return system, release, version -try: - import _wmi -except ImportError: - def _wmi_query(*keys): + +def _wmi_query(table, *keys): + global _wmi + if not _wmi: raise OSError("not supported") -else: - def _wmi_query(table, *keys): - table = { - "OS": "Win32_OperatingSystem", - "CPU": "Win32_Processor", - }[table] + table = { + "OS": "Win32_OperatingSystem", + "CPU": "Win32_Processor", + }[table] + try: data = _wmi.exec_query("SELECT {} FROM {}".format( ",".join(keys), table, )).split("\0") - split_data = (i.partition("=") for i in data) - dict_data = {i[0]: i[2] for i in split_data} - return (dict_data[k] for k in keys) + except OSError: + _wmi = None + raise OSError("not supported") + split_data = (i.partition("=") for i in data) + dict_data = {i[0]: i[2] for i in split_data} + return (dict_data[k] for k in keys) _WIN32_CLIENT_RELEASES = [ @@ -364,10 +369,7 @@ def win32_is_iot(): def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -426,10 +428,7 @@ def _win32_ver(version, csd, ptype): csd = 'SP' + csd[13:] try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -496,8 +495,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -519,9 +542,11 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: - import java.lang + import java.lang # noqa: F401 except ImportError: return release, vendor, vminfo, osinfo @@ -540,6 +565,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -611,7 +677,7 @@ def _platform(*args): if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -652,7 +718,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -746,6 +812,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: @@ -814,6 +882,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -968,6 +1044,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -1066,17 +1151,16 @@ def _sys_version(sys_version=None): if result is not None: return result - sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1103,7 +1187,14 @@ def _sys_version(sys_version=None): else: # CPython - match = sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1247,11 +1338,14 @@ def platform(aliased=False, terse=False): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 3292c30d5fb29b..67e832db217319 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -140,7 +140,7 @@ def _decode_base64(s): _dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -149,10 +149,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -171,11 +175,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -277,7 +282,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -321,13 +327,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -360,7 +367,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -461,8 +469,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -556,8 +565,11 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) @@ -588,7 +600,8 @@ def _read_object(self, ref): obj_refs = self._read_refs(s) result = [] self._objects[ref] = result - result.extend(self._read_object(x) for x in obj_refs) + for x in obj_refs: + result.append(self._read_object(x)) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -629,10 +642,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -778,7 +792,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -862,7 +881,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -880,32 +899,41 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ + if isinstance(value, str): + if fmt == FMT_BINARY: + raise TypeError("value must be bytes-like object when fmt is " + "FMT_BINARY") + value = value.encode() fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/posixpath.py b/Lib/posixpath.py index e4f155e41a3221..fccca4e066b76f 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,6 +22,7 @@ altsep = None devnull = '/dev/null' +import errno import os import sys import stat @@ -35,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction"] + "commonpath", "isjunction","isdevdrive"] def _get_sep(path): @@ -77,12 +78,11 @@ def join(a, *p): sep = _get_sep(a) path = a try: - if not p: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): - if b.startswith(sep): + for b in p: + b = os.fspath(b) + if b.startswith(sep) or not path: path = b - elif not path or path.endswith(sep): + elif path.endswith(sep): path += b else: path += sep + b @@ -135,33 +135,30 @@ def splitdrive(p): return p[:0], p -def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: +try: + from posix import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if p[:1] != sep: - # Relative path, e.g.: 'foo' - return empty, empty, p - elif p[1:2] != sep or p[2:3] == sep: - # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. - return empty, sep, p[1:] - else: - # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see - # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - return empty, p[:2], p[2:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] # Return the tail (basename) part of a path, same as split(path)[1]. @@ -187,26 +184,6 @@ def dirname(p): return head -# Is a path a junction? - -def isjunction(path): - """Test whether a path is a junction - Junctions are not a part of posix semantics""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except (OSError, ValueError): - return False - return True - - # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) @@ -227,21 +204,17 @@ def ismount(path): parent = join(path, b'..') else: parent = join(path, '..') - parent = realpath(parent) try: s2 = os.lstat(parent) - except (OSError, ValueError): - return False + except OSError: + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except OSError: + return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -290,7 +263,7 @@ def expanduser(path): return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = os.fsdecode(name) try: pwent = pwd.getpwnam(name) except KeyError: @@ -303,11 +276,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. @@ -371,7 +341,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -379,21 +349,19 @@ def normpath(path): path = os.fspath(path) if isinstance(path, bytes): sep = b'/' - empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' - empty = '' dot = '.' dotdot = '..' - if path == empty: + if not path: return dot _, initial_slashes, path = splitroot(path) comps = path.split(sep) new_comps = [] for comp in comps: - if comp in (empty, dot): + if not comp or comp == dot: continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): @@ -404,24 +372,16 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -432,72 +392,104 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' - - if isabs(rest): - rest = rest[1:] - path = sep + getcwd = os.getcwd + return _realpath(filename, strict, sep, curdir, pardir, getcwd) + +def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, + getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() + + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 while rest: - name, _, rest = rest.partition(sep) + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - path, name = split(path) - if name == pardir: - path = join(path, pardir, pardir) - else: - path = pardir + path = path[:path.rindex(sep)] or sep continue - newpath = join(path, name) + if path == sep: + newpath = path + name + else: + newpath = path + sep + name try: - st = os.lstat(newpath) + st = lstat(newpath) + if not stat.S_ISLNK(st.st_mode): + path = newpath + continue + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + target = readlink(newpath) except OSError: if strict: raise - is_link = False - else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: path = newpath continue # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False - seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + rest.extend(target.split(sep)[::-1]) - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') @@ -505,10 +497,10 @@ def _joinrealpath(path, rest, strict, seen): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' @@ -524,15 +516,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise @@ -546,10 +540,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' @@ -561,7 +556,7 @@ def commonpath(paths): split_paths = [path.split(sep) for path in paths] try: - isabs, = set(p[:1] == sep for p in paths) + isabs, = {p.startswith(sep) for p in paths} except ValueError: raise ValueError("Can't mix absolute and relative paths") from None diff --git a/Lib/pprint.py b/Lib/pprint.py index 34ed12637e2288..9314701db340c7 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -128,6 +128,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, sort_dicts If true, dict keys are sorted. + underscore_numbers + If true, digit groups are separated with underscores. + """ indent = int(indent) width = int(width) diff --git a/Lib/profile.py b/Lib/profile.py old mode 100755 new mode 100644 index 4b82523b03d64b..a5afb12c9d121a --- a/Lib/profile.py +++ b/Lib/profile.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 # # Class for profiling python code. rev 1.0 6/2/94 # @@ -387,8 +386,9 @@ def simulate_cmd_complete(self): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort). \ - print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): with open(file, 'wb') as f: diff --git a/Lib/pstats.py b/Lib/pstats.py index 51bcca84188740..a174a545456e1a 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -329,7 +329,7 @@ def eval_print_amount(self, sel, list, msg): if isinstance(sel, str): try: rex = re.compile(sel) - except re.error: + except re.PatternError: msg += " \n" % sel return new_list, msg new_list = [] @@ -611,7 +611,7 @@ def f8(x): if __name__ == '__main__': import cmd try: - import readline + import readline # noqa: F401 except ImportError: pass diff --git a/Lib/pty.py b/Lib/pty.py index 1d97994abef3c8..eb3d5f1ff657bb 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -32,27 +32,18 @@ def openpty(): except (AttributeError, OSError): pass master_fd, slave_name = _open_terminal() - slave_fd = slave_open(slave_name) - return master_fd, slave_fd - -def master_open(): - """master_open() -> (master_fd, slave_name) - Open a pty master and return the fd, and the filename of the slave end. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + slave_fd = os.open(slave_name, os.O_RDWR) try: - master_fd, slave_fd = os.openpty() - except (AttributeError, OSError): + from fcntl import ioctl, I_PUSH + except ImportError: + return master_fd, slave_fd + try: + ioctl(result, I_PUSH, "ptem") + ioctl(result, I_PUSH, "ldterm") + except OSError: pass - else: - slave_name = os.ttyname(slave_fd) - os.close(slave_fd) - return master_fd, slave_name - - return _open_terminal() + return master_fd, slave_fd def _open_terminal(): """Open pty master and return (master_fd, tty_name).""" @@ -66,26 +57,6 @@ def _open_terminal(): return (fd, '/dev/tty' + x + y) raise OSError('out of pty devices') -def slave_open(tty_name): - """slave_open(tty_name) -> slave_fd - Open the pty slave and acquire the controlling terminal, returning - opened filedescriptor. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 - - result = os.open(tty_name, os.O_RDWR) - try: - from fcntl import ioctl, I_PUSH - except ImportError: - return result - try: - ioctl(result, I_PUSH, "ptem") - ioctl(result, I_PUSH, "ldterm") - except OSError: - pass - return result def fork(): """fork() -> (pid, master_fd) diff --git a/Lib/pydoc.py b/Lib/pydoc.py old mode 100755 new mode 100644 index c9a55799b39f0c..768c3dcb11ec59 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -76,6 +75,21 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -127,9 +141,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): @@ -201,7 +214,10 @@ def _getargspec(object): try: signature = inspect.signature(object) if signature: - return str(signature) + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width) except (ValueError, TypeError): argspec = getattr(object, '__text_signature__', None) if argspec: @@ -222,6 +238,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -298,7 +327,8 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 @@ -316,13 +346,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' if isinstance(value, property) and value.fset is None: kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -342,6 +374,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): @@ -532,7 +566,7 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): if docloc.startswith(("http://", "https://")): docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) else: @@ -676,6 +710,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '%s' % (object.__name__, object.__name__) @@ -817,9 +870,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -920,7 +973,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -1038,24 +1091,44 @@ def formatvalue(self, object): return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1066,10 +1139,13 @@ def docroutine(self, object, name=None, mod=None, if name == realname: title = '%s' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '%s' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '%s = %s' % ( @@ -1082,7 +1158,8 @@ def docroutine(self, object, name=None, mod=None, # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1097,7 +1174,7 @@ def docroutine(self, object, name=None, mod=None, doc = doc and '
    %s
    ' % doc return '
    %s
    %s
    \n' % (decl, doc) - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce html documentation for a data descriptor.""" results = [] push = results.append @@ -1208,7 +1285,7 @@ def formattree(self, tree, modname, parent=None, prefix=''): entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1237,9 +1314,9 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1387,7 +1464,7 @@ def spill(msg, attrs, predicate): push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1462,23 +1539,43 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname if (inspect.iscoroutinefunction(object) or inspect.isasyncgenfunction(object)): @@ -1489,8 +1586,11 @@ def docroutine(self, object, name=None, mod=None, cl=None): if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None @@ -1501,7 +1601,8 @@ def docroutine(self, object, name=None, mod=None, cl=None): # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note @@ -1512,7 +1613,7 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def docdata(self, object, name=None, mod=None, cl=None): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce text documentation for a data descriptor.""" results = [] push = results.append @@ -1528,7 +1629,8 @@ def docdata(self, object, name=None, mod=None, cl=None): docproperty = docdata - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: @@ -1549,140 +1651,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace') - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1782,7 +1755,15 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, """Display text documentation, given an object or a path to an object.""" if output is None: try: - pager(render_doc(thing, title, forceload)) + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') except ImportError as exc: if is_cli: raise @@ -2036,7 +2017,7 @@ def interact(self): if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -2063,7 +2044,7 @@ def help(self, request, is_cli=False): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:', is_cli=is_cli) + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) @@ -2073,20 +2054,22 @@ def help(self, request, is_cli=False): self.output.write('\n') def intro(self): - self.output.write(''' -Welcome to Python {0}'s help utility! - -If this is your first time using Python, you should definitely check out -the tutorial on the internet at https://docs.python.org/{0}/tutorial/. + self.output.write('''\ +Welcome to Python {0}'s help utility! If this is your first time using +Python, you should definitely check out the tutorial at +https://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing -Python programs and using Python modules. To quit this help utility and -return to the interpreter, just type "quit". +Python programs and using Python modules. To get a list of available +modules, keywords, symbols, or topics, enter "modules", "keywords", +"symbols", or "topics". + +Each module also comes with a one-line summary of what it does; to list +the modules whose name or summary contain a given string such as "spam", +enter "modules spam". -To get a list of available modules, keywords, symbols, or topics, type -"modules", "keywords", "symbols", or "topics". Each module also comes -with a one-line summary of what it does; to list the modules whose name -or summary contain a given string such as "spam", type "modules spam". +To quit this help utility and return to the interpreter, +enter "q", "quit" or "exit". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): @@ -2154,7 +2137,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2406,6 +2393,7 @@ def __init__(self, urlhandler, host, port): threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2438,9 +2426,9 @@ def stop(self): thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 87b0a2d669e5a3..4643df80e44aaf 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Fri Oct 13 10:51:21 2023 +# Autogenerated by Sphinx on Wed May 8 11:11:17 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -419,7 +419,7 @@ 'async': 'Coroutines\n' '**********\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -722,9 +722,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A ' - 'sequence must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' Called when "dir()" is called on the object. An ' + 'iterable must be\n' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -751,8 +751,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible ' + 'return an\n' + 'iterable of strings that represents the names accessible ' 'on module. If\n' 'present, this function overrides the standard "dir()" ' 'search on a\n' @@ -792,7 +792,7 @@ 'Changed in version 3.5: "__class__" module attribute is ' 'now writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -864,19 +864,25 @@ '*instance* of the\n' ' owner class.\n' '\n' - 'The attribute "__objclass__" is interpreted by the ' - '"inspect" module as\n' - 'specifying the class where this object was defined ' + 'Instances of descriptors may also have the ' + '"__objclass__" attribute\n' + 'present:\n' + '\n' + 'object.__objclass__\n' + '\n' + ' The attribute "__objclass__" is interpreted by the ' + '"inspect" module\n' + ' as specifying the class where this object was defined ' '(setting this\n' - 'appropriately can assist in runtime introspection of ' + ' appropriately can assist in runtime introspection of ' 'dynamic class\n' - 'attributes). For callables, it may indicate that an ' + ' attributes). For callables, it may indicate that an ' 'instance of the\n' - 'given type (or a subclass) is expected or required as ' + ' given type (or a subclass) is expected or required as ' 'the first\n' - 'positional argument (for example, CPython sets this ' + ' positional argument (for example, CPython sets this ' 'attribute for\n' - 'unbound methods that are implemented in C).\n' + ' unbound methods that are implemented in C).\n' '\n' '\n' 'Invoking Descriptors\n' @@ -1111,16 +1117,23 @@ 'attribute references, which most objects do. This ' 'object is then\n' 'asked to produce the attribute whose name is the ' - 'identifier. This\n' - 'production can be customized by overriding the ' - '"__getattr__()" method.\n' - 'If this attribute is not available, the exception ' - '"AttributeError" is\n' - 'raised. Otherwise, the type and value of the object ' - 'produced is\n' - 'determined by the object. Multiple evaluations of ' - 'the same attribute\n' - 'reference may yield different objects.\n', + 'identifier. The type\n' + 'and value produced is determined by the object. ' + 'Multiple evaluations\n' + 'of the same attribute reference may yield different ' + 'objects.\n' + '\n' + 'This production can be customized by overriding the\n' + '"__getattribute__()" method or the "__getattr__()" ' + 'method. The\n' + '"__getattribute__()" method is called first and ' + 'either returns a value\n' + 'or raises "AttributeError" if the attribute is not ' + 'available.\n' + '\n' + 'If an "AttributeError" is raised and the object has ' + 'a "__getattr__()"\n' + 'method, that method is called as a fallback.\n', 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' @@ -1193,7 +1206,7 @@ '\n' ' await_expr ::= "await" primary\n' '\n' - 'New in version 3.5.\n', + 'Added in version 3.5.\n', 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' @@ -1226,7 +1239,7 @@ 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' 'The "/" (division) and "//" (floor division) operators yield the\n' 'quotient of their arguments. The numeric arguments are first\n' @@ -2752,7 +2765,7 @@ 'The "match" statement\n' '=====================\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'The match statement is used for pattern matching. Syntax:\n' '\n' @@ -3045,8 +3058,7 @@ 'standard\n' 'Python grammar. Triple-quoted strings are supported. Raw ' 'strings and\n' - 'byte strings are supported. Formatted string literals are not\n' - 'supported.\n' + 'byte strings are supported. f-strings are not supported.\n' '\n' 'The forms "signed_number \'+\' NUMBER" and "signed_number \'-\' ' 'NUMBER"\n' @@ -3700,7 +3712,7 @@ ' **PEP 526** - Syntax for Variable Annotations\n' ' Ability to type hint variable declarations, including ' 'class\n' - ' variables and instance variables\n' + ' variables and instance variables.\n' '\n' ' **PEP 563** - Postponed Evaluation of Annotations\n' ' Support for forward references within annotations by ' @@ -3708,6 +3720,11 @@ ' annotations in a string form at runtime instead of eager\n' ' evaluation.\n' '\n' + ' **PEP 318** - Decorators for Functions and Methods\n' + ' Function and method decorators were introduced. Class ' + 'decorators\n' + ' were introduced in **PEP 3129**.\n' + '\n' '\n' 'Class definitions\n' '=================\n' @@ -3832,7 +3849,7 @@ 'Coroutines\n' '==========\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -3959,13 +3976,18 @@ 'Type parameter lists\n' '====================\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' + '\n' + 'Changed in version 3.13: Support for default values was added ' + '(see\n' + '**PEP 696**).\n' '\n' ' type_params ::= "[" type_param ("," type_param)* "]"\n' ' type_param ::= typevar | typevartuple | paramspec\n' - ' typevar ::= identifier (":" expression)?\n' - ' typevartuple ::= "*" identifier\n' - ' paramspec ::= "**" identifier\n' + ' typevar ::= identifier (":" expression)? ("=" ' + 'expression)?\n' + ' typevartuple ::= "*" identifier ("=" expression)?\n' + ' paramspec ::= "**" identifier ("=" expression)?\n' '\n' 'Functions (including coroutines), classes and type aliases may ' 'contain\n' @@ -4064,21 +4086,41 @@ 'bounds or\n' 'constraints.\n' '\n' + 'All three flavors of type parameters can also have a *default ' + 'value*,\n' + 'which is used when the type parameter is not explicitly ' + 'provided. This\n' + 'is added by appending a single equals sign ("=") followed by an\n' + 'expression. Like the bounds and constraints of type variables, ' + 'the\n' + 'default value is not evaluated when the object is created, but ' + 'only\n' + 'when the type parameter’s "__default__" attribute is accessed. ' + 'To this\n' + 'end, the default value is evaluated in a separate annotation ' + 'scope. If\n' + 'no default value is specified for a type parameter, the ' + '"__default__"\n' + 'attribute is set to the special sentinel object ' + '"typing.NoDefault".\n' + '\n' 'The following example indicates the full set of allowed type ' 'parameter\n' 'declarations:\n' '\n' ' def overly_generic[\n' ' SimpleTypeVar,\n' + ' TypeVarWithDefault = int,\n' ' TypeVarWithBound: int,\n' ' TypeVarWithConstraints: (str, bytes),\n' - ' *SimpleTypeVarTuple,\n' - ' **SimpleParamSpec,\n' + ' *SimpleTypeVarTuple = (int, float),\n' + ' **SimpleParamSpec = (str, bytearray),\n' ' ](\n' ' a: SimpleTypeVar,\n' - ' b: TypeVarWithBound,\n' - ' c: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' - ' *d: SimpleTypeVarTuple,\n' + ' b: TypeVarWithDefault,\n' + ' c: TypeVarWithBound,\n' + ' d: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' + ' *e: SimpleTypeVarTuple,\n' ' ): ...\n' '\n' '\n' @@ -4707,7 +4749,7 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s ' + ' operands are of different types, and the right operand’s ' 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' @@ -4717,6 +4759,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -4824,8 +4871,8 @@ 'denial-of-service caused\n' ' by carefully chosen inputs that exploit the worst ' 'case\n' - ' performance of a dict insertion, O(n^2) complexity. ' - 'See\n' + ' performance of a dict insertion, *O*(*n*^2) ' + 'complexity. See\n' ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' @@ -4904,7 +4951,7 @@ 'and continue running without the debugger using the "continue"\n' 'command.\n' '\n' - 'New in version 3.7: The built-in "breakpoint()", when called ' + 'Changed in version 3.7: The built-in "breakpoint()", when called ' 'with\n' 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' '\n' @@ -4918,8 +4965,8 @@ 'you are\n' 'in debug mode:\n' '\n' - ' > ...(3)double()\n' - ' -> return x * 2\n' + ' > ...(2)double()\n' + ' -> breakpoint()\n' ' (Pdb) p x\n' ' 3\n' ' (Pdb) continue\n' @@ -4948,11 +4995,11 @@ 'the\n' 'debugger upon program’s exit.\n' '\n' - 'New in version 3.2: "-c" option is introduced to execute ' + 'Changed in version 3.2: Added the "-c" option to execute ' 'commands as\n' - 'if given in a ".pdbrc" file, see Debugger Commands.\n' + 'if given in a ".pdbrc" file; see Debugger Commands.\n' '\n' - 'New in version 3.7: "-m" option is introduced to execute ' + 'Changed in version 3.7: Added the "-m" option to execute ' 'modules\n' 'similar to the way "python -m" does. As with a script, the ' 'debugger\n' @@ -5041,6 +5088,11 @@ '\n' ' Changed in version 3.7: The keyword-only argument *header*.\n' '\n' + ' Changed in version 3.13: "set_trace()" will enter the ' + 'debugger\n' + ' immediately, rather than on the next line of code to be ' + 'executed.\n' + '\n' 'pdb.post_mortem(traceback=None)\n' '\n' ' Enter post-mortem debugging of the given *traceback* object. ' @@ -5098,11 +5150,11 @@ '\n' ' Raises an auditing event "pdb.Pdb" with no arguments.\n' '\n' - ' New in version 3.1: The *skip* argument.\n' + ' Changed in version 3.1: Added the *skip* parameter.\n' '\n' - ' New in version 3.2: The *nosigint* argument. Previously, a ' - 'SIGINT\n' - ' handler was never set by Pdb.\n' + ' Changed in version 3.2: Added the *nosigint* parameter. ' + 'Previously,\n' + ' a SIGINT handler was never set by Pdb.\n' '\n' ' Changed in version 3.6: The *readrc* argument.\n' '\n' @@ -5193,28 +5245,29 @@ '* "$_exception": the exception if the frame is raising an ' 'exception\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' 'current directory, it is read with "\'utf-8\'" encoding and ' 'executed as\n' - 'if it had been typed at the debugger prompt. This is ' - 'particularly\n' - 'useful for aliases. If both files exist, the one in the home\n' - 'directory is read first and aliases defined there can be ' - 'overridden by\n' - 'the local file.\n' - '\n' - 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' - 'encoding.\n' - 'Previously, it was read with the system locale encoding.\n' + 'if it had been typed at the debugger prompt, with the exception ' + 'that\n' + 'empty lines and lines starting with "#" are ignored. This is\n' + 'particularly useful for aliases. If both files exist, the one ' + 'in the\n' + 'home directory is read first and aliases defined there can be\n' + 'overridden by the local file.\n' '\n' 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' 'continue debugging, such as "continue" or "next". Previously, ' 'these\n' 'commands had no effect.\n' '\n' + 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' + 'encoding.\n' + 'Previously, it was read with the system locale encoding.\n' + '\n' 'h(elp) [command]\n' '\n' ' Without argument, print the list of available commands. With ' @@ -5248,19 +5301,22 @@ '\n' 'b(reak) [([filename:]lineno | function) [, condition]]\n' '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' + ' With a *lineno* argument, set a break at line *lineno* in ' + 'the\n' + ' current file. The line number may be prefixed with a ' + '*filename* and\n' + ' a colon, to specify a breakpoint in another file (possibly ' + 'one that\n' + ' hasn’t been loaded yet). The file is searched on ' + '"sys.path".\n' + ' Accepatable forms of *filename* are "/abspath/to/file.py",\n' + ' "relpath/file.py", "module" and "package.module".\n' + '\n' ' With a *function* argument, set a break at the first ' 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' + ' statement within that function. *function* can be any ' + 'expression\n' + ' that evaluates to a function in the current namespace.\n' '\n' ' If a second argument is present, it is an expression which ' 'must\n' @@ -5272,6 +5328,9 @@ 'current\n' ' ignore count, and the associated condition if any.\n' '\n' + ' Each breakpoint is assigned a number to which all the other\n' + ' breakpoint commands refer.\n' + '\n' 'tbreak [([filename:]lineno | function) [, condition]]\n' '\n' ' Temporary breakpoint, which is removed automatically when it ' @@ -5449,14 +5508,14 @@ 'differs\n' ' from the current line.\n' '\n' - ' New in version 3.2: The ">>" marker.\n' + ' Changed in version 3.2: Added the ">>" marker.\n' '\n' 'll | longlist\n' '\n' ' List all source code for the current function or frame.\n' ' Interesting lines are marked as for "list".\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'a(rgs)\n' '\n' @@ -5489,7 +5548,7 @@ '\n' ' Try to get source code of *expression* and display it.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'display [expression]\n' '\n' @@ -5548,7 +5607,7 @@ ' display lst[:]: [1] [old: []]\n' ' (Pdb)\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'undisplay [expression]\n' '\n' @@ -5557,7 +5616,7 @@ ' *expression*, clear all display expressions for the current ' 'frame.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'interact\n' '\n' @@ -5565,16 +5624,37 @@ 'whose\n' ' global namespace contains all the (global and local) names ' 'found in\n' - ' the current scope.\n' + ' the current scope. Use "exit()" or "quit()" to exit the ' + 'interpreter\n' + ' and return to the debugger.\n' + '\n' + ' Note:\n' + '\n' + ' Because interact creates a new global namespace with the ' + 'current\n' + ' global and local namespace for execution, assignment to ' + 'variables\n' + ' will not affect the original namespaces. However, ' + 'modification to\n' + ' the mutable objects will be reflected in the original ' + 'namespaces.\n' + '\n' + ' Added in version 3.2.\n' '\n' - ' New in version 3.2.\n' + ' Changed in version 3.13: "exit()" and "quit()" can be used to ' + 'exit\n' + ' the "interact" command.\n' + '\n' + ' Changed in version 3.13: "interact" directs its output to ' + 'the\n' + ' debugger’s output channel rather than "sys.stderr".\n' '\n' 'alias [name [command]]\n' '\n' ' Create an alias called *name* that executes *command*. The\n' ' *command* must *not* be enclosed in quotes. Replaceable ' 'parameters\n' - ' can be indicated by "%1", "%2", and so on, while "%*" is ' + ' can be indicated by "%1", "%2", … and "%9", while "%*" is ' 'replaced\n' ' by all the parameters. If *command* is omitted, the current ' 'alias\n' @@ -5704,7 +5784,7 @@ ' > example.py(10)middle()\n' ' -> return inner(0)\n' '\n' - ' New in version 3.13.\n' + ' Added in version 3.13.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -5769,7 +5849,8 @@ 'dict\n' 'items and earlier dictionary unpackings.\n' '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' + 'Added in version 3.5: Unpacking into dictionary displays, ' + 'originally\n' 'proposed by **PEP 448**.\n' '\n' 'A dict comprehension, in contrast to list and set comprehensions,\n' @@ -6077,9 +6158,12 @@ 'of the module "builtins". The global namespace is searched ' 'first. If\n' 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed ' - 'names.\n' + 'searched\n' + 'next. If the names are also not found in the builtins ' + 'namespace, new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -6167,8 +6251,9 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type ' + 'parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -6211,9 +6296,13 @@ 'object were\n' ' defined in the enclosing scope.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in ' - 'Python 3.12\n' - 'as part of **PEP 695**.\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' + '\n' + 'Changed in version 3.13: Annotation scopes are also used for ' + 'type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' '\n' '\n' 'Lazy evaluation\n' @@ -6221,15 +6310,15 @@ '\n' 'The values of type aliases created through the "type" statement ' 'are\n' - '*lazily evaluated*. The same applies to the bounds and ' - 'constraints of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type ' - 'variable is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + '*lazily evaluated*. The same applies to the bounds, ' + 'constraints, and\n' + 'default values of type variables created through the type ' + 'parameter\n' + 'syntax. This means that they are not evaluated when the type ' + 'alias or\n' + 'type variable is created. Instead, they are only evaluated when ' + 'doing\n' + 'so is necessary to resolve an attribute access.\n' '\n' 'Example:\n' '\n' @@ -6273,7 +6362,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -6428,19 +6517,18 @@ 'the\n' 'unpacking.\n' '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' + 'Added in version 3.5: Iterable unpacking in expression lists,\n' + 'originally proposed by **PEP 448**.\n' '\n' - 'The trailing comma is required only to create a single tuple ' - '(a.k.a. a\n' - '*singleton*); it is optional in all other cases. A single ' - 'expression\n' - 'without a trailing comma doesn’t create a tuple, but rather ' - 'yields the\n' - 'value of that expression. (To create an empty tuple, use an ' - 'empty pair\n' - 'of parentheses: "()".)\n', + 'A trailing comma is required only to create a one-item tuple, ' + 'such as\n' + '"1,"; it is optional in all other cases. A single expression ' + 'without a\n' + 'trailing comma doesn’t create a tuple, but rather yields the ' + 'value of\n' + 'that expression. (To create an empty tuple, use an empty pair ' + 'of\n' + 'parentheses: "()".)\n', 'floating': 'Floating point literals\n' '***********************\n' '\n' @@ -6554,7 +6642,7 @@ ' index_string ::= ' '+\n' ' conversion ::= "r" | "s" | "a"\n' - ' format_spec ::= \n' + ' format_spec ::= format-spec:format_spec\n' '\n' 'In less formal terms, the replacement field can start with ' 'a\n' @@ -6689,12 +6777,11 @@ 'contained\n' 'within a format string to define how individual values are ' 'presented\n' - '(see Format String Syntax and Formatted string literals). ' - 'They can\n' - 'also be passed directly to the built-in "format()" ' - 'function. Each\n' - 'formattable type may define how the format specification is ' - 'to be\n' + '(see Format String Syntax and f-strings). They can also be ' + 'passed\n' + 'directly to the built-in "format()" function. Each ' + 'formattable type\n' + 'may define how the format specification is to be ' 'interpreted.\n' '\n' 'Most built-in types implement the following options for ' @@ -7509,13 +7596,18 @@ ' **PEP 526** - Syntax for Variable Annotations\n' ' Ability to type hint variable declarations, including ' 'class\n' - ' variables and instance variables\n' + ' variables and instance variables.\n' '\n' ' **PEP 563** - Postponed Evaluation of Annotations\n' ' Support for forward references within annotations by ' 'preserving\n' ' annotations in a string form at runtime instead of eager\n' - ' evaluation.\n', + ' evaluation.\n' + '\n' + ' **PEP 318** - Decorators for Functions and Methods\n' + ' Function and method decorators were introduced. Class ' + 'decorators\n' + ' were introduced in **PEP 3129**.\n', 'global': 'The "global" statement\n' '**********************\n' '\n' @@ -7721,7 +7813,7 @@ 'Soft Keywords\n' '=============\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'Some identifiers are only reserved under specific contexts. ' 'These are\n' @@ -8349,9 +8441,12 @@ 'namespace\n' 'of the module "builtins". The global namespace is searched ' 'first. If\n' - 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed names.\n' + 'the names are not found there, the builtins namespace is searched\n' + 'next. If the names are also not found in the builtins namespace, ' + 'new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -8434,8 +8529,8 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -8475,9 +8570,12 @@ 'were\n' ' defined in the enclosing scope.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in Python ' - '3.12\n' - 'as part of **PEP 695**.\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' + '\n' + 'Changed in version 3.13: Annotation scopes are also used for type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' '\n' '\n' 'Lazy evaluation\n' @@ -8485,15 +8583,15 @@ '\n' 'The values of type aliases created through the "type" statement ' 'are\n' - '*lazily evaluated*. The same applies to the bounds and constraints ' - 'of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type variable ' - 'is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + '*lazily evaluated*. The same applies to the bounds, constraints, ' + 'and\n' + 'default values of type variables created through the type ' + 'parameter\n' + 'syntax. This means that they are not evaluated when the type alias ' + 'or\n' + 'type variable is created. Instead, they are only evaluated when ' + 'doing\n' + 'so is necessary to resolve an attribute access.\n' '\n' 'Example:\n' '\n' @@ -8536,7 +8634,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -8593,32 +8691,36 @@ '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' + 'When the definition of a function or class is nested (enclosed) ' + 'within\n' + 'the definitions of other functions, its nonlocal scopes are the ' + 'local\n' + 'scopes of the enclosing functions. The "nonlocal" statement ' + 'causes the\n' + 'listed identifiers to refer to names previously bound in ' + 'nonlocal\n' + 'scopes. It allows encapsulated code to rebind such nonlocal\n' + 'identifiers. If a name is bound in more than one nonlocal ' + 'scope, the\n' + 'nearest binding is used. If a name is not bound in any nonlocal ' + 'scope,\n' + 'or if there is no nonlocal scope, a "SyntaxError" is raised.\n' + '\n' + 'The nonlocal statement applies to the entire scope of a function ' + 'or\n' + 'class body. A "SyntaxError" is raised if a variable is used or\n' + 'assigned to prior to its nonlocal declaration in the scope.\n' '\n' 'See also:\n' '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', + ' The specification for the "nonlocal" statement.\n' + '\n' + '**Programmer’s note:** "nonlocal" is a directive to the parser ' + 'and\n' + 'applies only to code parsed along with it. See the note for ' + 'the\n' + '"global" statement.\n', 'numbers': 'Numeric literals\n' '****************\n' '\n' @@ -8714,7 +8816,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -8757,14 +8859,18 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" ' + 'are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in ' @@ -8845,7 +8951,7 @@ 'Every object has an identity, a type and a value. An object’s\n' '*identity* never changes once it has been created; you may think ' 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' + 'as the object’s address in memory. The "is" operator compares ' 'the\n' 'identity of two objects; the "id()" function returns an integer\n' 'representing its identity.\n' @@ -8910,7 +9016,7 @@ 'Note that the use of the implementation’s tracing or debugging\n' 'facilities may keep objects alive that would normally be ' 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' + 'Also note that catching an exception with a "try"…"except" ' 'statement\n' 'may keep objects alive.\n' '\n' @@ -8925,8 +9031,9 @@ 'release the external resource, usually a "close()" method. ' 'Programs\n' 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' + '"try"…"finally" statement and the "with" statement provide ' + 'convenient\n' + 'ways to do this.\n' '\n' 'Some objects contain references to other objects; these are ' 'called\n' @@ -9212,15 +9319,13 @@ '\n' 'A traceback object is normally created automatically when an ' 'exception\n' - 'is raised and attached to it as the "__traceback__" attribute, ' - 'which\n' - 'is writable. You can create an exception and set your own traceback ' - 'in\n' - 'one step using the "with_traceback()" exception method (which ' - 'returns\n' - 'the same exception instance, with its traceback set to its ' - 'argument),\n' - 'like so:\n' + 'is raised and attached to it as the "__traceback__" attribute. You ' + 'can\n' + 'create an exception and set your own traceback in one step using ' + 'the\n' + '"with_traceback()" exception method (which returns the same ' + 'exception\n' + 'instance, with its traceback set to its argument), like so:\n' '\n' ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' '\n' @@ -9246,6 +9351,8 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' + ' print(1 / 0)\n' + ' ~~^~~\n' ' ZeroDivisionError: division by zero\n' '\n' ' The above exception was the direct cause of the following ' @@ -9253,6 +9360,7 @@ '\n' ' Traceback (most recent call last):\n' ' File "", line 4, in \n' + ' raise RuntimeError("Something bad happened") from exc\n' ' RuntimeError: Something bad happened\n' '\n' 'A similar mechanism works implicitly if a new exception is raised ' @@ -9271,6 +9379,8 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' + ' print(1 / 0)\n' + ' ~~^~~\n' ' ZeroDivisionError: division by zero\n' '\n' ' During handling of the above exception, another exception ' @@ -9278,6 +9388,7 @@ '\n' ' Traceback (most recent call last):\n' ' File "", line 4, in \n' + ' raise RuntimeError("Something bad happened")\n' ' RuntimeError: Something bad happened\n' '\n' 'Exception chaining can be explicitly suppressed by specifying ' @@ -9299,10 +9410,7 @@ 'The try statement.\n' '\n' 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' + 'from Y".Added the "__suppress_context__" attribute to suppress\n' 'automatic display of the exception context.\n' '\n' 'Changed in version 3.11: If the traceback of the active exception ' @@ -9445,7 +9553,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -9466,23 +9574,20 @@ '\n' ' Called to implement evaluation of "self[key]". For ' '*sequence*\n' - ' types, the accepted keys should be integers and slice ' - 'objects.\n' - ' Note that the special interpretation of negative ' - 'indexes (if the\n' - ' class wishes to emulate a *sequence* type) is up to ' - 'the\n' - ' "__getitem__()" method. If *key* is of an inappropriate ' - 'type,\n' - ' "TypeError" may be raised; if of a value outside the ' - 'set of indexes\n' - ' for the sequence (after any special interpretation of ' - 'negative\n' - ' values), "IndexError" should be raised. For *mapping* ' - 'types, if\n' - ' *key* is missing (not in the container), "KeyError" ' - 'should be\n' - ' raised.\n' + ' types, the accepted keys should be integers. ' + 'Optionally, they may\n' + ' support "slice" objects as well. Negative index ' + 'support is also\n' + ' optional. If *key* is of an inappropriate type, ' + '"TypeError" may be\n' + ' raised; if *key* is a value outside the set of indexes ' + 'for the\n' + ' sequence (after any special interpretation of negative ' + 'values),\n' + ' "IndexError" should be raised. For *mapping* types, if ' + '*key* is\n' + ' missing (not in the container), "KeyError" should be ' + 'raised.\n' '\n' ' Note:\n' '\n' @@ -9699,7 +9804,7 @@ 'descriptor, or\n' ' generator instance.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' 'definition.__type_params__\n' '\n' @@ -9707,7 +9812,7 @@ 'type\n' ' aliases.\n' '\n' - ' New in version 3.12.\n' + ' Added in version 3.12.\n' '\n' 'class.__mro__\n' '\n' @@ -9733,7 +9838,15 @@ '\n' ' >>> int.__subclasses__()\n' " [, , , " - "]\n", + "]\n" + '\n' + 'class.__static_attributes__\n' + '\n' + ' A tuple containing names of attributes of this class ' + 'which are\n' + ' accessed through "self.X" from any function in its body.\n' + '\n' + ' Added in version 3.13.\n', 'specialnames': 'Special method names\n' '********************\n' '\n' @@ -10090,8 +10203,8 @@ 'reflection,\n' ' and "__eq__()" and "__ne__()" are their own reflection. ' 'If the\n' - ' operands are of different types, and right operand’s type ' - 'is a\n' + ' operands are of different types, and the right operand’s ' + 'type is a\n' ' direct or indirect subclass of the left operand’s type, ' 'the\n' ' reflected method of the right operand has priority, ' @@ -10100,6 +10213,11 @@ 'is not\n' ' considered.\n' '\n' + ' When no appropriate method returns any value other than\n' + ' "NotImplemented", the "==" and "!=" operators will fall ' + 'back to\n' + ' "is" and "is not", respectively.\n' + '\n' 'object.__hash__(self)\n' '\n' ' Called by built-in function "hash()" and for operations ' @@ -10204,8 +10322,8 @@ ' intended to provide protection against a ' 'denial-of-service caused\n' ' by carefully chosen inputs that exploit the worst case\n' - ' performance of a dict insertion, O(n^2) complexity. ' - 'See\n' + ' performance of a dict insertion, *O*(*n*^2) ' + 'complexity. See\n' ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' @@ -10341,9 +10459,9 @@ '\n' 'object.__dir__(self)\n' '\n' - ' Called when "dir()" is called on the object. A sequence ' + ' Called when "dir()" is called on the object. An iterable ' 'must be\n' - ' returned. "dir()" converts the returned sequence to a ' + ' returned. "dir()" converts the returned iterable to a ' 'list and\n' ' sorts it.\n' '\n' @@ -10370,8 +10488,8 @@ 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a\n' - 'sequence of strings that represents the names accessible on ' + 'return an\n' + 'iterable of strings that represents the names accessible on ' 'module. If\n' 'present, this function overrides the standard "dir()" search ' 'on a\n' @@ -10411,7 +10529,7 @@ 'Changed in version 3.5: "__class__" module attribute is now ' 'writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -10483,19 +10601,25 @@ 'of the\n' ' owner class.\n' '\n' - 'The attribute "__objclass__" is interpreted by the "inspect" ' - 'module as\n' - 'specifying the class where this object was defined (setting ' - 'this\n' - 'appropriately can assist in runtime introspection of dynamic ' - 'class\n' - 'attributes). For callables, it may indicate that an instance ' - 'of the\n' - 'given type (or a subclass) is expected or required as the ' + 'Instances of descriptors may also have the "__objclass__" ' + 'attribute\n' + 'present:\n' + '\n' + 'object.__objclass__\n' + '\n' + ' The attribute "__objclass__" is interpreted by the ' + '"inspect" module\n' + ' as specifying the class where this object was defined ' + '(setting this\n' + ' appropriately can assist in runtime introspection of ' + 'dynamic class\n' + ' attributes). For callables, it may indicate that an ' + 'instance of the\n' + ' given type (or a subclass) is expected or required as the ' 'first\n' - 'positional argument (for example, CPython sets this ' + ' positional argument (for example, CPython sets this ' 'attribute for\n' - 'unbound methods that are implemented in C).\n' + ' unbound methods that are implemented in C).\n' '\n' '\n' 'Invoking Descriptors\n' @@ -10742,7 +10866,7 @@ '\n' ' Keyword arguments which are given to a new class are ' 'passed to the\n' - ' parent’s class "__init_subclass__". For compatibility ' + ' parent class’s "__init_subclass__". For compatibility ' 'with other\n' ' classes using "__init_subclass__", one should take out ' 'the needed\n' @@ -10774,7 +10898,7 @@ 'explicit\n' ' hint) can be accessed as "type(cls)".\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' 'When a class is created, "type.__new__()" scans the class ' 'variables\n' @@ -10806,7 +10930,7 @@ '\n' ' See Creating the class object for more details.\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' '\n' 'Metaclasses\n' @@ -11408,7 +11532,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -11429,22 +11553,20 @@ '\n' ' Called to implement evaluation of "self[key]". For ' '*sequence*\n' - ' types, the accepted keys should be integers and slice ' - 'objects.\n' - ' Note that the special interpretation of negative indexes ' - '(if the\n' - ' class wishes to emulate a *sequence* type) is up to the\n' - ' "__getitem__()" method. If *key* is of an inappropriate ' - 'type,\n' - ' "TypeError" may be raised; if of a value outside the set ' - 'of indexes\n' - ' for the sequence (after any special interpretation of ' - 'negative\n' - ' values), "IndexError" should be raised. For *mapping* ' - 'types, if\n' - ' *key* is missing (not in the container), "KeyError" ' - 'should be\n' - ' raised.\n' + ' types, the accepted keys should be integers. Optionally, ' + 'they may\n' + ' support "slice" objects as well. Negative index support ' + 'is also\n' + ' optional. If *key* is of an inappropriate type, ' + '"TypeError" may be\n' + ' raised; if *key* is a value outside the set of indexes ' + 'for the\n' + ' sequence (after any special interpretation of negative ' + 'values),\n' + ' "IndexError" should be raised. For *mapping* types, if ' + '*key* is\n' + ' missing (not in the container), "KeyError" should be ' + 'raised.\n' '\n' ' Note:\n' '\n' @@ -11635,7 +11757,7 @@ '"__rsub__()"\n' ' method, "type(y).__rsub__(y, x)" is called if ' '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' + ' y)" returns "NotImplemented".\n' '\n' ' Note that ternary "pow()" will not try calling ' '"__rpow__()" (the\n' @@ -11678,14 +11800,17 @@ 'the result\n' ' (which could be, but does not have to be, *self*). If a ' 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' + ' method is not defined, or if that method returns ' + '"NotImplemented",\n' + ' the augmented assignment falls back to the normal ' + 'methods. For\n' + ' instance, if *x* is an instance of a class with an ' + '"__iadd__()"\n' + ' method, "x += y" is equivalent to "x = x.__iadd__(y)" . ' + 'If\n' + ' "__iadd__()" does not exist, or if "x.__iadd__(y)" ' + 'returns\n' + ' "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are\n' ' considered, as with the evaluation of "x + y". In ' 'certain\n' ' situations, augmented assignment can result in unexpected ' @@ -11856,7 +11981,7 @@ 'will\n' 'raise a "TypeError".\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'See also:\n' '\n' @@ -11905,7 +12030,7 @@ 'to\n' ' implement this method.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' 'See also:\n' '\n' @@ -12069,7 +12194,7 @@ '‘Default\n' ' Case Folding’ of the Unicode Standard.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' 'str.center(width[, fillchar])\n' '\n' @@ -12253,7 +12378,7 @@ "{country}'.format_map(Default(name='Guido'))\n" " 'Guido was born in country'\n" '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'str.index(sub[, start[, end]])\n' '\n' @@ -12297,7 +12422,7 @@ 'have code\n' ' points in the range U+0000-U+007F.\n' '\n' - ' New in version 3.7.\n' + ' Added in version 3.7.\n' '\n' 'str.isdecimal()\n' '\n' @@ -12537,7 +12662,7 @@ " >>> 'BaseTestCase'.removeprefix('Test')\n" " 'BaseTestCase'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.removesuffix(suffix, /)\n' '\n' @@ -12552,7 +12677,7 @@ " >>> 'TmpDirMixin'.removesuffix('Tests')\n" " 'TmpDirMixin'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.replace(old, new, count=-1)\n' '\n' @@ -13031,11 +13156,10 @@ 'than\n' 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' - 'added as a synonym of "\'br\'".\n' - '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' + 'Added in version 3.3: The "\'rb\'" prefix of raw bytes literals ' + 'has been\n' + 'added as a synonym of "\'br\'".Support for the unicode legacy ' + 'literal\n' '("u\'value\'") was reintroduced to simplify the maintenance of ' 'dual\n' 'Python 2.x and 3.x codebases. See **PEP 414** for more ' @@ -13043,12 +13167,11 @@ '\n' 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' '*formatted\n' - 'string literal*; see Formatted string literals. The "\'f\'" may ' - 'be\n' - 'combined with "\'r\'", but not with "\'b\'" or "\'u\'", therefore ' - 'raw\n' - 'formatted strings are possible, but formatted bytes literals are ' - 'not.\n' + 'string literal*; see f-strings. The "\'f\'" may be combined with ' + '"\'r\'",\n' + 'but not with "\'b\'" or "\'u\'", therefore raw formatted strings ' + 'are\n' + 'possible, but formatted bytes literals are not.\n' '\n' 'In triple-quoted literals, unescaped newlines and quotes are ' 'allowed\n' @@ -13745,14 +13868,18 @@ 'contains\n' 'the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected ' 'by\n' - '"a[i]".\n' + '"a[i]". Some sequences, including built-in sequences, interpret\n' + 'negative subscripts by adding the sequence length. For example,\n' + '"a[-2]" equals "a[n-2]", the second to last item of sequence a ' + 'with\n' + 'length "n".\n' '\n' 'Sequences also support slicing: "a[i:j]" selects all items with ' 'index\n' '*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a\n' - 'slice is a sequence of the same type. This implies that the index ' - 'set\n' - 'is renumbered so that it starts at 0.\n' + 'slice is a sequence of the same type. The comment above about ' + 'negative\n' + 'indexes also applies to negative slice positions.\n' '\n' 'Some sequences also support “extended slicing†with a third “stepâ€\n' 'parameter: "a[i:j:k]" selects all items of *a* with index *x* where ' @@ -13952,130 +14079,117 @@ 'function’s\n' 'formal parameter list.\n' '\n' - 'Special attributes:\n' '\n' - '+---------------------------+---------------------------------+-------------+\n' - '| Attribute | Meaning ' - '| |\n' - '|===========================|=================================|=============|\n' - '| "__doc__" | The function’s documentation | ' - 'Writable |\n' - '| | string, or "None" if ' - '| |\n' - '| | unavailable; not inherited by ' - '| |\n' - '| | subclasses. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__name__" | The function’s name. | ' - 'Writable |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__qualname__" | The function’s *qualified | ' - 'Writable |\n' - '| | name*. New in version 3.3. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__module__" | The name of the module the | ' - 'Writable |\n' - '| | function was defined in, or ' - '| |\n' - '| | "None" if unavailable. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__defaults__" | A tuple containing default | ' - 'Writable |\n' - '| | argument values for those ' - '| |\n' - '| | arguments that have defaults, ' - '| |\n' - '| | or "None" if no arguments have ' - '| |\n' - '| | a default value. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__code__" | The code object representing | ' - 'Writable |\n' - '| | the compiled function body. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__globals__" | A reference to the dictionary | ' - 'Read-only |\n' - '| | that holds the function’s ' - '| |\n' - '| | global variables — the global ' - '| |\n' - '| | namespace of the module in ' - '| |\n' - '| | which the function was defined. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__dict__" | The namespace supporting | ' - 'Writable |\n' - '| | arbitrary function attributes. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__closure__" | "None" or a tuple of cells that | ' - 'Read-only |\n' - '| | contain bindings for the ' - '| |\n' - '| | function’s free variables. See ' - '| |\n' - '| | below for information on the ' - '| |\n' - '| | "cell_contents" attribute. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__annotations__" | A dict containing annotations | ' - 'Writable |\n' - '| | of parameters. The keys of the ' - '| |\n' - '| | dict are the parameter names, ' - '| |\n' - '| | and "\'return\'" for the return ' - '| |\n' - '| | annotation, if provided. For ' - '| |\n' - '| | more information on working ' - '| |\n' - '| | with this attribute, see ' - '| |\n' - '| | Annotations Best Practices. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__kwdefaults__" | A dict containing defaults for | ' - 'Writable |\n' - '| | keyword-only parameters. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '| "__type_params__" | A tuple containing the type | ' - 'Writable |\n' - '| | parameters of a generic ' - '| |\n' - '| | function. ' - '| |\n' - '+---------------------------+---------------------------------+-------------+\n' - '\n' - 'Most of the attributes labelled “Writable†check the type of the\n' - 'assigned value.\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| Attribute | ' + 'Meaning |\n' + '|====================================================|====================================================|\n' + '| function.__globals__ | A reference ' + 'to the "dictionary" that holds the |\n' + '| | function’s ' + 'global variables – the global namespace |\n' + '| | of the ' + 'module in which the function was defined. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__closure__ | "None" or a ' + '"tuple" of cells that contain bindings |\n' + '| | for the ' + 'function’s free variables. A cell object |\n' + '| | has the ' + 'attribute "cell_contents". This can be |\n' + '| | used to get ' + 'the value of the cell, as well as set |\n' + '| | the ' + 'value. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Special writable attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + 'Most of these attributes check the type of the assigned value:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| Attribute | ' + 'Meaning |\n' + '|====================================================|====================================================|\n' + '| function.__doc__ | The ' + 'function’s documentation string, or "None" if |\n' + '| | unavailable. ' + 'Not inherited by subclasses. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__name__ | The ' + 'function’s name. See also: "__name__ |\n' + '| | ' + 'attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__qualname__ | The ' + 'function’s *qualified name*. See also: |\n' + '| | ' + '"__qualname__ attributes". Added in version 3.3. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__module__ | The name of ' + 'the module the function was defined |\n' + '| | in, or ' + '"None" if unavailable. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__defaults__ | A "tuple" ' + 'containing default *parameter* values |\n' + '| | for those ' + 'parameters that have defaults, or "None" |\n' + '| | if no ' + 'parameters have a default value. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__code__ | The code ' + 'object representing the compiled function |\n' + '| | ' + 'body. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__dict__ | The ' + 'namespace supporting arbitrary function |\n' + '| | attributes. ' + 'See also: "__dict__ attributes". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__annotations__ | A ' + '"dictionary" containing annotations of |\n' + '| | ' + '*parameters*. The keys of the dictionary are the |\n' + '| | parameter ' + 'names, and "\'return\'" for the return |\n' + '| | annotation, ' + 'if provided. See also: Annotations |\n' + '| | Best ' + 'Practices. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__kwdefaults__ | A ' + '"dictionary" containing defaults for keyword- |\n' + '| | only ' + '*parameters*. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| function.__type_params__ | A "tuple" ' + 'containing the type parameters of a |\n' + '| | generic ' + 'function. Added in version 3.12. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Function objects also support getting and setting arbitrary\n' 'attributes, which can be used, for example, to attach metadata to\n' 'functions. Regular attribute dot-notation is used to get and set ' 'such\n' - 'attributes. *Note that the current implementation only supports\n' - 'function attributes on user-defined functions. Function attributes ' - 'on\n' - 'built-in functions may be supported in the future.*\n' + 'attributes.\n' '\n' - 'A cell object has the attribute "cell_contents". This can be used ' - 'to\n' - 'get the value of the cell, as well as set the value.\n' + '**CPython implementation detail:** CPython’s current ' + 'implementation\n' + 'only supports function attributes on user-defined functions. ' + 'Function\n' + 'attributes on built-in functions may be supported in the future.\n' '\n' 'Additional information about a function’s definition can be ' 'retrieved\n' - 'from its code object; see the description of internal types below. ' - 'The\n' - '"cell" type can be accessed in the "types" module.\n' + 'from its code object (accessible via the "__code__" attribute).\n' '\n' '\n' 'Instance methods\n' @@ -14085,14 +14199,34 @@ 'any\n' 'callable object (normally a user-defined function).\n' '\n' - 'Special read-only attributes: "__self__" is the class instance ' - 'object,\n' - '"__func__" is the function object; "__doc__" is the method’s\n' - 'documentation (same as "__func__.__doc__"); "__name__" is the ' - 'method\n' - 'name (same as "__func__.__name__"); "__module__" is the name of ' - 'the\n' - 'module the method was defined in, or "None" if unavailable.\n' + 'Special read-only attributes:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__self__ | Refers to ' + 'the class instance object to which the |\n' + '| | method is ' + 'bound |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__func__ | Refers to ' + 'the original function object |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__doc__ | The method’s ' + 'documentation (same as |\n' + '| | ' + '"method.__func__.__doc__"). A "string" if the |\n' + '| | original ' + 'function had a docstring, else "None". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__name__ | The name of ' + 'the method (same as |\n' + '| | ' + '"method.__func__.__name__") |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| method.__module__ | The name of ' + 'the module the method was defined in, |\n' + '| | or "None" if ' + 'unavailable. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Methods also support accessing (but not setting) the arbitrary\n' 'function attributes on the underlying function object.\n' @@ -14101,24 +14235,20 @@ 'attribute\n' 'of a class (perhaps via an instance of that class), if that ' 'attribute\n' - 'is a user-defined function object or a class method object.\n' + 'is a user-defined function object or a "classmethod" object.\n' '\n' 'When an instance method object is created by retrieving a ' 'user-defined\n' 'function object from a class via one of its instances, its ' '"__self__"\n' - 'attribute is the instance, and the method object is said to be ' - 'bound.\n' - 'The new method’s "__func__" attribute is the original function ' - 'object.\n' + 'attribute is the instance, and the method object is said to be\n' + '*bound*. The new method’s "__func__" attribute is the original\n' + 'function object.\n' '\n' - 'When an instance method object is created by retrieving a class ' - 'method\n' - 'object from a class or instance, its "__self__" attribute is the ' - 'class\n' - 'itself, and its "__func__" attribute is the function object ' - 'underlying\n' - 'the class method.\n' + 'When an instance method object is created by retrieving a\n' + '"classmethod" object from a class or instance, its "__self__"\n' + 'attribute is the class itself, and its "__func__" attribute is the\n' + 'function object underlying the class method.\n' '\n' 'When an instance method object is called, the underlying function\n' '("__func__") is called, inserting the class instance ("__self__") ' @@ -14129,7 +14259,7 @@ 'of\n' '"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)".\n' '\n' - 'When an instance method object is derived from a class method ' + 'When an instance method object is derived from a "classmethod" ' 'object,\n' 'the “class instance†stored in "__self__" will actually be the ' 'class\n' @@ -14217,13 +14347,18 @@ 'of built-in functions are "len()" and "math.sin()" ("math" is a\n' 'standard built-in module). The number and type of the arguments ' 'are\n' - 'determined by the C function. Special read-only attributes: ' - '"__doc__"\n' - 'is the function’s documentation string, or "None" if unavailable;\n' - '"__name__" is the function’s name; "__self__" is set to "None" ' - '(but\n' - 'see the next item); "__module__" is the name of the module the\n' - 'function was defined in or "None" if unavailable.\n' + 'determined by the C function. Special read-only attributes:\n' + '\n' + '* "__doc__" is the function’s documentation string, or "None" if\n' + ' unavailable. See "function.__doc__".\n' + '\n' + '* "__name__" is the function’s name. See "function.__name__".\n' + '\n' + '* "__self__" is set to "None" (but see the next item).\n' + '\n' + '* "__module__" is the name of the module the function was defined ' + 'in\n' + ' or "None" if unavailable. See "function.__module__".\n' '\n' '\n' 'Built-in methods\n' @@ -14235,7 +14370,9 @@ 'argument. An example of a built-in method is "alist.append()",\n' 'assuming *alist* is a list object. In this case, the special ' 'read-only\n' - 'attribute "__self__" is set to the object denoted by *alist*.\n' + 'attribute "__self__" is set to the object denoted by *alist*. (The\n' + 'attribute has the same semantics as it does with "other instance\n' + 'methods".)\n' '\n' '\n' 'Classes\n' @@ -14267,16 +14404,15 @@ 'statement, or by calling functions such as ' '"importlib.import_module()"\n' 'and built-in "__import__()". A module object has a namespace\n' - 'implemented by a dictionary object (this is the dictionary ' - 'referenced\n' - 'by the "__globals__" attribute of functions defined in the ' - 'module).\n' - 'Attribute references are translated to lookups in this dictionary,\n' - 'e.g., "m.x" is equivalent to "m.__dict__["x"]". A module object ' - 'does\n' - 'not contain the code object used to initialize the module (since ' - 'it\n' - 'isn’t needed once the initialization is done).\n' + 'implemented by a "dictionary" object (this is the dictionary\n' + 'referenced by the "__globals__" attribute of functions defined in ' + 'the\n' + 'module). Attribute references are translated to lookups in this\n' + 'dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A ' + 'module\n' + 'object does not contain the code object used to initialize the ' + 'module\n' + '(since it isn’t needed once the initialization is done).\n' '\n' 'Attribute assignment updates the module’s namespace dictionary, ' 'e.g.,\n' @@ -14343,21 +14479,19 @@ 'to\n' 'a common ancestor. Additional details on the C3 MRO used by Python ' 'can\n' - 'be found in the documentation accompanying the 2.3 release at\n' - 'https://www.python.org/download/releases/2.3/mro/.\n' + 'be found at The Python 2.3 Method Resolution Order.\n' '\n' 'When a class attribute reference (for class "C", say) would yield ' 'a\n' 'class method object, it is transformed into an instance method ' 'object\n' - 'whose "__self__" attribute is "C". When it would yield a static\n' - 'method object, it is transformed into the object wrapped by the ' - 'static\n' - 'method object. See section Implementing Descriptors for another way ' - 'in\n' - 'which attributes retrieved from a class may differ from those ' - 'actually\n' - 'contained in its "__dict__".\n' + 'whose "__self__" attribute is "C". When it would yield a\n' + '"staticmethod" object, it is transformed into the object wrapped ' + 'by\n' + 'the static method object. See section Implementing Descriptors for\n' + 'another way in which attributes retrieved from a class may differ ' + 'from\n' + 'those actually contained in its "__dict__".\n' '\n' 'Class attribute assignments update the class’s dictionary, never ' 'the\n' @@ -14393,6 +14527,15 @@ ' "__type_params__"\n' ' A tuple containing the type parameters of a generic class.\n' '\n' + ' "__static_attributes__"\n' + ' A tuple containing names of attributes of this class which ' + 'are\n' + ' accessed through "self.X" from any function in its body.\n' + '\n' + ' "__firstlineno__"\n' + ' The line number of the first line of the class definition,\n' + ' including decorators.\n' + '\n' '\n' 'Class instances\n' '===============\n' @@ -14480,43 +14623,108 @@ 'code objects are immutable and contain no references (directly or\n' 'indirectly) to mutable objects.\n' '\n' - 'Special read-only attributes: "co_name" gives the function name;\n' - '"co_qualname" gives the fully qualified function name; ' - '"co_argcount"\n' - 'is the total number of positional arguments (including ' - 'positional-only\n' - 'arguments and arguments with default values); "co_posonlyargcount" ' - 'is\n' - 'the number of positional-only arguments (including arguments with\n' - 'default values); "co_kwonlyargcount" is the number of keyword-only\n' - 'arguments (including arguments with default values); "co_nlocals" ' - 'is\n' - 'the number of local variables used by the function (including\n' - 'arguments); "co_varnames" is a tuple containing the names of the ' - 'local\n' - 'variables (starting with the argument names); "co_cellvars" is a ' - 'tuple\n' - 'containing the names of local variables that are referenced by ' - 'nested\n' - 'functions; "co_freevars" is a tuple containing the names of free\n' - 'variables; "co_code" is a string representing the sequence of ' - 'bytecode\n' - 'instructions; "co_consts" is a tuple containing the literals used ' - 'by\n' - 'the bytecode; "co_names" is a tuple containing the names used by ' - 'the\n' - 'bytecode; "co_filename" is the filename from which the code was\n' - 'compiled; "co_firstlineno" is the first line number of the ' - 'function;\n' - '"co_lnotab" is a string encoding the mapping from bytecode offsets ' - 'to\n' - 'line numbers (for details see the source code of the interpreter, ' - 'is\n' - 'deprecated since 3.12 and may be removed in 3.14); "co_stacksize" ' - 'is\n' - 'the required stack size; "co_flags" is an integer encoding a number ' - 'of\n' - 'flags for the interpreter.\n' + '\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_name | The function ' + 'name |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_qualname | The fully ' + 'qualified function name Added in |\n' + '| | version ' + '3.11. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_argcount | The total ' + 'number of positional *parameters* |\n' + '| | (including ' + 'positional-only parameters and |\n' + '| | parameters ' + 'with default values) that the function |\n' + '| | ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_posonlyargcount | The number ' + 'of positional-only *parameters* |\n' + '| | (including ' + 'arguments with default values) that the |\n' + '| | function ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_kwonlyargcount | The number ' + 'of keyword-only *parameters* (including |\n' + '| | arguments ' + 'with default values) that the function |\n' + '| | ' + 'has |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_nlocals | The number ' + 'of local variables used by the function |\n' + '| | (including ' + 'parameters) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_varnames | A "tuple" ' + 'containing the names of the local |\n' + '| | variables in ' + 'the function (starting with the |\n' + '| | parameter ' + 'names) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_cellvars | A "tuple" ' + 'containing the names of local variables |\n' + '| | that are ' + 'referenced by nested functions inside the |\n' + '| | ' + 'function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_freevars | A "tuple" ' + 'containing the names of free variables |\n' + '| | in the ' + 'function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_code | A string ' + 'representing the sequence of *bytecode* |\n' + '| | instructions ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_consts | A "tuple" ' + 'containing the literals used by the |\n' + '| | *bytecode* ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_names | A "tuple" ' + 'containing the names used by the |\n' + '| | *bytecode* ' + 'in the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_filename | The name of ' + 'the file from which the code was |\n' + '| | ' + 'compiled |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_firstlineno | The line ' + 'number of the first line of the function |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_lnotab | A string ' + 'encoding the mapping from *bytecode* |\n' + '| | offsets to ' + 'line numbers. For details, see the |\n' + '| | source code ' + 'of the interpreter. Deprecated since |\n' + '| | version ' + '3.12: This attribute of code objects is |\n' + '| | deprecated, ' + 'and may be removed in Python 3.14. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_stacksize | The required ' + 'stack size of the code object |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| codeobject.co_flags | An "integer" ' + 'encoding a number of flags for the |\n' + '| | ' + 'interpreter. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'The following flag bits are defined for "co_flags": bit "0x04" is ' 'set\n' @@ -14525,7 +14733,9 @@ 'number of positional arguments; bit "0x08" is set if the function ' 'uses\n' 'the "**keywords" syntax to accept arbitrary keyword arguments; bit\n' - '"0x20" is set if the function is a generator.\n' + '"0x20" is set if the function is a generator. See Code Objects Bit\n' + 'Flags for details on the semantics of each flags that might be\n' + 'present.\n' '\n' 'Future feature declarations ("from __future__ import division") ' 'also\n' @@ -14544,16 +14754,19 @@ 'is the documentation string of the function, or "None" if ' 'undefined.\n' '\n' + '\n' + 'Methods on code objects\n' + '~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' 'codeobject.co_positions()\n' '\n' - ' Returns an iterable over the source code positions of each ' - 'bytecode\n' - ' instruction in the code object.\n' + ' Returns an iterable over the source code positions of each\n' + ' *bytecode* instruction in the code object.\n' '\n' - ' The iterator returns tuples containing the "(start_line, ' - 'end_line,\n' - ' start_column, end_column)". The *i-th* tuple corresponds to the\n' - ' position of the source code that compiled to the *i-th*\n' + ' The iterator returns "tuple"s containing the "(start_line,\n' + ' end_line, start_column, end_column)". The *i-th* tuple ' + 'corresponds\n' + ' to the position of the source code that compiled to the *i-th*\n' ' instruction. Column information is 0-indexed utf-8 byte offsets ' 'on\n' ' the given source line.\n' @@ -14575,7 +14788,7 @@ ' When this occurs, some or all of the tuple elements can be ' '"None".\n' '\n' - ' New in version 3.11.\n' + ' Added in version 3.11.\n' '\n' ' Note:\n' '\n' @@ -14591,51 +14804,170 @@ 'the\n' ' "PYTHONNODEBUGRANGES" environment variable can be used.\n' '\n' + 'codeobject.co_lines()\n' '\n' - 'Frame objects\n' - '-------------\n' + ' Returns an iterator that yields information about successive ' + 'ranges\n' + ' of *bytecode*s. Each item yielded is a "(start, end, lineno)"\n' + ' "tuple":\n' '\n' - 'Frame objects represent execution frames. They may occur in ' - 'traceback\n' - 'objects (see below), and are also passed to registered trace\n' - 'functions.\n' + ' * "start" (an "int") represents the offset (inclusive) of the ' + 'start\n' + ' of the *bytecode* range\n' '\n' - 'Special read-only attributes: "f_back" is to the previous stack ' - 'frame\n' - '(towards the caller), or "None" if this is the bottom stack frame;\n' - '"f_code" is the code object being executed in this frame; ' - '"f_locals"\n' - 'is the dictionary used to look up local variables; "f_globals" is ' - 'used\n' - 'for global variables; "f_builtins" is used for built-in ' - '(intrinsic)\n' - 'names; "f_lasti" gives the precise instruction (this is an index ' - 'into\n' - 'the bytecode string of the code object).\n' + ' * "end" (an "int") represents the offset (exclusive) of the end ' + 'of\n' + ' the *bytecode* range\n' '\n' - 'Accessing "f_code" raises an auditing event "object.__getattr__" ' - 'with\n' - 'arguments "obj" and ""f_code"".\n' + ' * "lineno" is an "int" representing the line number of the\n' + ' *bytecode* range, or "None" if the bytecodes in the given ' + 'range\n' + ' have no line number\n' '\n' - 'Special writable attributes: "f_trace", if not "None", is a ' - 'function\n' - 'called for various events during code execution (this is used by ' + ' The items yielded will have the following properties:\n' + '\n' + ' * The first range yielded will have a "start" of 0.\n' + '\n' + ' * The "(start, end)" ranges will be non-decreasing and ' + 'consecutive.\n' + ' That is, for any pair of "tuple"s, the "start" of the second ' + 'will\n' + ' be equal to the "end" of the first.\n' + '\n' + ' * No range will be backwards: "end >= start" for all triples.\n' + '\n' + ' * The last "tuple" yielded will have "end" equal to the size of ' 'the\n' - 'debugger). Normally an event is triggered for each new source line ' - '-\n' - 'this can be disabled by setting "f_trace_lines" to "False".\n' - '\n' - 'Implementations *may* allow per-opcode events to be requested by\n' - 'setting "f_trace_opcodes" to "True". Note that this may lead to\n' - 'undefined interpreter behaviour if exceptions raised by the trace\n' - 'function escape to the function being traced.\n' - '\n' - '"f_lineno" is the current line number of the frame — writing to ' - 'this\n' - 'from within a trace function jumps to the given line (only for the\n' - 'bottom-most frame). A debugger can implement a Jump command (aka ' - 'Set\n' - 'Next Statement) by writing to f_lineno.\n' + ' *bytecode*.\n' + '\n' + ' Zero-width ranges, where "start == end", are allowed. ' + 'Zero-width\n' + ' ranges are used for lines that are present in the source code, ' + 'but\n' + ' have been eliminated by the *bytecode* compiler.\n' + '\n' + ' Added in version 3.10.\n' + '\n' + ' See also:\n' + '\n' + ' **PEP 626** - Precise line numbers for debugging and other ' + 'tools.\n' + ' The PEP that introduced the "co_lines()" method.\n' + '\n' + 'codeobject.replace(**kwargs)\n' + '\n' + ' Return a copy of the code object with new values for the ' + 'specified\n' + ' fields.\n' + '\n' + ' Code objects are also supported by the generic function\n' + ' "copy.replace()".\n' + '\n' + ' Added in version 3.8.\n' + '\n' + '\n' + 'Frame objects\n' + '-------------\n' + '\n' + 'Frame objects represent execution frames. They may occur in ' + 'traceback\n' + 'objects, and are also passed to registered trace functions.\n' + '\n' + '\n' + 'Special read-only attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_back | Points to ' + 'the previous stack frame (towards the |\n' + '| | caller), or ' + '"None" if this is the bottom stack |\n' + '| | ' + 'frame |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_code | The code ' + 'object being executed in this frame. |\n' + '| | Accessing ' + 'this attribute raises an auditing event |\n' + '| | ' + '"object.__getattr__" with arguments "obj" and |\n' + '| | ' + '""f_code"". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_locals | The ' + 'dictionary used by the frame to look up local |\n' + '| | variables. ' + 'If the frame refers to a function or |\n' + '| | ' + 'comprehension, this may return a write- through |\n' + '| | proxy ' + 'object. Changed in version 3.13: Return a |\n' + '| | proxy for ' + 'functions and comprehensions. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_globals | The ' + 'dictionary used by the frame to look up global |\n' + '| | ' + 'variables |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_builtins | The ' + 'dictionary used by the frame to look up built- |\n' + '| | in ' + '(intrinsic) names |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_lasti | The “precise ' + 'instruction†of the frame object |\n' + '| | (this is an ' + 'index into the *bytecode* string of |\n' + '| | the code ' + 'object) |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Special writable attributes\n' + '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace | If not ' + '"None", this is a function called for |\n' + '| | various ' + 'events during code execution (this is used |\n' + '| | by ' + 'debuggers). Normally an event is triggered for |\n' + '| | each new ' + 'source line (see "f_trace_lines"). |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace_lines | Set this ' + 'attribute to "False" to disable |\n' + '| | triggering a ' + 'tracing event for each source line. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_trace_opcodes | Set this ' + 'attribute to "True" to allow per-opcode |\n' + '| | events to be ' + 'requested. Note that this may lead to |\n' + '| | undefined ' + 'interpreter behaviour if exceptions |\n' + '| | raised by ' + 'the trace function escape to the |\n' + '| | function ' + 'being traced. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| frame.f_lineno | The current ' + 'line number of the frame – writing to |\n' + '| | this from ' + 'within a trace function jumps to the |\n' + '| | given line ' + '(only for the bottom-most frame). A |\n' + '| | debugger can ' + 'implement a Jump command (aka Set |\n' + '| | Next ' + 'Statement) by writing to this attribute. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + '\n' + 'Frame object methods\n' + '~~~~~~~~~~~~~~~~~~~~\n' '\n' 'Frame objects support one method:\n' '\n' @@ -14643,26 +14975,35 @@ '\n' ' This method clears all references to local variables held by ' 'the\n' - ' frame. Also, if the frame belonged to a generator, the ' + ' frame. Also, if the frame belonged to a *generator*, the ' 'generator\n' ' is finalized. This helps break reference cycles involving ' 'frame\n' ' objects (for example when catching an exception and storing its\n' ' traceback for later use).\n' '\n' - ' "RuntimeError" is raised if the frame is currently executing.\n' + ' "RuntimeError" is raised if the frame is currently executing or\n' + ' suspended.\n' + '\n' + ' Added in version 3.4.\n' '\n' - ' New in version 3.4.\n' + ' Changed in version 3.13: Attempting to clear a suspended frame\n' + ' raises "RuntimeError" (as has always been the case for ' + 'executing\n' + ' frames).\n' '\n' '\n' 'Traceback objects\n' '-----------------\n' '\n' - 'Traceback objects represent a stack trace of an exception. A\n' + 'Traceback objects represent the stack trace of an exception. A\n' 'traceback object is implicitly created when an exception occurs, ' 'and\n' 'may also be explicitly created by calling "types.TracebackType".\n' '\n' + 'Changed in version 3.7: Traceback objects can now be explicitly\n' + 'instantiated from Python code.\n' + '\n' 'For implicitly created tracebacks, when the search for an ' 'exception\n' 'handler unwinds the execution stack, at each unwound level a ' @@ -14685,30 +15026,40 @@ 'linked\n' 'to form a full stack trace.\n' '\n' - 'Special read-only attributes: "tb_frame" points to the execution ' - 'frame\n' - 'of the current level; "tb_lineno" gives the line number where the\n' - 'exception occurred; "tb_lasti" indicates the precise instruction. ' - 'The\n' - 'line number and last instruction in the traceback may differ from ' + 'Special read-only attributes:\n' + '\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_frame | Points to ' + 'the execution frame of the current |\n' + '| | level. ' + 'Accessing this attribute raises an |\n' + '| | auditing ' + 'event "object.__getattr__" with arguments |\n' + '| | "obj" and ' + '""tb_frame"". |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_lineno | Gives the ' + 'line number where the exception occurred |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '| traceback.tb_lasti | Indicates ' + 'the “precise instructionâ€. |\n' + '+----------------------------------------------------+----------------------------------------------------+\n' + '\n' + 'The line number and last instruction in the traceback may differ ' + 'from\n' + 'the line number of its frame object if the exception occurred in a\n' + '"try" statement with no matching except clause or with a "finally"\n' + 'clause.\n' + '\n' + 'traceback.tb_next\n' + '\n' + ' The special writable attribute "tb_next" is the next level in ' 'the\n' - 'line number of its frame object if the exception occurred in a ' - '"try"\n' - 'statement with no matching except clause or with a finally clause.\n' - '\n' - 'Accessing "tb_frame" raises an auditing event "object.__getattr__"\n' - 'with arguments "obj" and ""tb_frame"".\n' - '\n' - 'Special writable attribute: "tb_next" is the next level in the ' - 'stack\n' - 'trace (towards the frame where the exception occurred), or "None" ' - 'if\n' - 'there is no next level.\n' + ' stack trace (towards the frame where the exception occurred), ' + 'or\n' + ' "None" if there is no next level.\n' '\n' - 'Changed in version 3.7: Traceback objects can now be explicitly\n' - 'instantiated from Python code, and the "tb_next" attribute of ' - 'existing\n' - 'instances can be updated.\n' + ' Changed in version 3.7: This attribute is now writable\n' '\n' '\n' 'Slice objects\n' @@ -14765,8 +15116,8 @@ 'around another object that alters the way in which that object is\n' 'retrieved from classes and class instances. The behaviour of class\n' 'method objects upon such retrieval is described above, under ' - '“User-\n' - 'defined methodsâ€. Class method objects are created by the built-in\n' + '“instance\n' + 'methodsâ€. Class method objects are created by the built-in\n' '"classmethod()" constructor.\n', 'typesfunctions': 'Functions\n' '*********\n' @@ -15027,7 +15378,7 @@ 'dictionary. This\n' ' is a shortcut for "reversed(d.keys())".\n' '\n' - ' New in version 3.8.\n' + ' Added in version 3.8.\n' '\n' ' setdefault(key[, default])\n' '\n' @@ -15078,7 +15429,7 @@ ' *other* take priority when *d* and *other* share ' 'keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' d |= other\n' '\n' @@ -15090,7 +15441,7 @@ 'and *other*\n' ' share keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' Dictionaries compare equal if and only if they have the ' 'same "(key,\n' @@ -15212,25 +15563,27 @@ 'original\n' ' dictionary to which the view refers.\n' '\n' - ' New in version 3.10.\n' + ' Added in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' '*hashable*.\n' - 'If all values are hashable, so that "(key, value)" pairs are ' - 'unique\n' - 'and hashable, then the items view is also set-like. (Values ' - 'views are\n' - 'not treated as set-like since the entries are generally not ' - 'unique.)\n' - 'For set-like views, all of the operations defined for the ' - 'abstract\n' - 'base class "collections.abc.Set" are available (for example, ' - '"==",\n' - '"<", or "^"). While using set operators, set-like views ' - 'accept any\n' - 'iterable as the other operand, unlike sets which only accept ' - 'sets as\n' - 'the input.\n' + 'Items views also have set-like operations since the (key, ' + 'value) pairs\n' + 'are unique and the keys are hashable. If all values in an ' + 'items view\n' + 'are hashable as well, then the items view can interoperate ' + 'with other\n' + 'sets. (Values views are not treated as set-like since the ' + 'entries are\n' + 'generally not unique.) For set-like views, all of the ' + 'operations\n' + 'defined for the abstract base class "collections.abc.Set" ' + 'are\n' + 'available (for example, "==", "<", or "^"). While using ' + 'set\n' + 'operators, set-like views accept any iterable as the other ' + 'operand,\n' + 'unlike sets which only accept sets as the input.\n' '\n' 'An example of dictionary view usage:\n' '\n' @@ -15283,7 +15636,7 @@ 'notation.\n' 'There are two flavors: built-in methods (such as "append()" ' 'on lists)\n' - 'and class instance methods. Built-in methods are described ' + 'and class instance method. Built-in methods are described ' 'with the\n' 'types that support them.\n' '\n' @@ -15291,8 +15644,8 @@ 'namespace)\n' 'through an instance, you get a special object: a *bound ' 'method* (also\n' - 'called *instance method*) object. When called, it will add ' - 'the "self"\n' + 'called instance method) object. When called, it will add the ' + '"self"\n' 'argument to the argument list. Bound methods have two ' 'special read-\n' 'only attributes: "m.__self__" is the object on which the ' @@ -15307,7 +15660,7 @@ 'arbitrary\n' 'attributes. However, since method attributes are actually ' 'stored on\n' - 'the underlying function object ("meth.__func__"), setting ' + 'the underlying function object ("method.__func__"), setting ' 'method\n' 'attributes on bound methods is disallowed. Attempting to ' 'set an\n' @@ -15332,7 +15685,7 @@ ' >>> c.method.whoami\n' " 'my name is method'\n" '\n' - 'See The standard type hierarchy for more information.\n', + 'See Instance methods for more information.\n', 'typesmodules': 'Modules\n' '*******\n' '\n' @@ -15750,7 +16103,7 @@ 'mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" methods.\n' '\n' '6. The value *n* is an integer, or an object implementing\n' ' "__index__()". Zero and negative values of *n* clear the ' @@ -15862,7 +16215,7 @@ '\n' ' For sorting examples and a brief sorting tutorial, see ' 'Sorting\n' - ' HOW TO.\n' + ' Techniques.\n' '\n' ' **CPython implementation detail:** While a list is being ' 'sorted,\n' @@ -16077,9 +16430,8 @@ 'objects\n' 'based on the sequence of values they define (instead of ' 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' + 'based on object identity).Added the "start", "stop" and "step"\n' + 'attributes.\n' '\n' 'See also:\n' '\n' @@ -16219,7 +16571,8 @@ 'concrete mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" ' + 'methods.\n' '\n' '6. The value *n* is an integer, or an object ' 'implementing\n' diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460f9e..25beb46e30d6bd 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: @@ -25,6 +33,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +66,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -65,6 +80,9 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -129,8 +147,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +160,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +171,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +187,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +211,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +233,29 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise ShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index f36cf7b3951cda..129fd2f5c7c28a --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). diff --git a/Lib/random.py b/Lib/random.py index 3f7bfd272d65a5..bcc11c7cd3c208 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -50,7 +50,6 @@ # Adrian Baddeley. Adapted by Raymond Hettinger for use with # the Mersenne Twister and os.urandom() core generators. -from warnings import warn as _warn from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin from math import tau as TWOPI, floor as _floor, isfinite as _isfinite @@ -63,13 +62,6 @@ import os as _os import _random -try: - # hashlib is pretty heavy to load, try lean internal module first - from _sha2 import sha512 as _sha512 -except ImportError: - # fallback to official implementation - from hashlib import sha512 as _sha512 - __all__ = [ "Random", "SystemRandom", @@ -105,6 +97,7 @@ BPF = 53 # Number of bits in a float RECIP_BPF = 2 ** -BPF _ONE = 1 +_sha512 = None class Random(_random.Random): @@ -159,6 +152,16 @@ def seed(self, a=None, version=2): a = -2 if x == -1 else x elif version == 2 and isinstance(a, (str, bytes, bytearray)): + global _sha512 + if _sha512 is None: + try: + # hashlib is pretty heavy to load, try lean internal + # module first + from _sha2 import sha512 as _sha512 + except ImportError: + # fallback to official implementation + from hashlib import sha512 as _sha512 + if isinstance(a, str): a = a.encode() a = int.from_bytes(a + _sha512(a).digest()) @@ -257,9 +260,10 @@ def _randbelow_without_getrandbits(self, n, maxsize=1<= maxsize: - _warn("Underlying random() generator does not supply \n" - "enough bits to choose from a population range this large.\n" - "To remove the range limitation, add a getrandbits() method.") + from warnings import warn + warn("Underlying random() generator does not supply \n" + "enough bits to choose from a population range this large.\n" + "To remove the range limitation, add a getrandbits() method.") return _floor(random() * n) rem = maxsize % n limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 @@ -992,5 +996,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating point number between 1 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(1, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(1, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index 428d1b0d5fbd87..7e8abbf6ffe155 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -117,7 +117,8 @@ U UNICODE For compatibility only. Ignored for string patterns (it is the default), and forbidden for bytes patterns. -This module also defines an exception 'error'. +This module also defines exception 'PatternError', aliased to 'error' for +backward compatibility. """ @@ -133,7 +134,7 @@ "findall", "finditer", "compile", "purge", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", - "UNICODE", "NOFLAG", "RegexFlag", + "UNICODE", "NOFLAG", "RegexFlag", "PatternError" ] __version__ = "2.2.1" @@ -155,7 +156,7 @@ class RegexFlag: _numeric_repr_ = hex # sre exception -error = _compiler.error +PatternError = error = _compiler.PatternError # -------------------------------------------------------------------- # public interface diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index f87712d6d6f9f8..29109f8812ee7b 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -28,6 +28,8 @@ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), } +_CHARSET_ALL = [(NEGATE, None)] + def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=_parser.TYPE_FLAGS): if add_flags & TYPE_FLAGS: @@ -84,17 +86,22 @@ def _compile(code, pattern, flags): code[skip] = _len(code) - skip elif op is IN: charset, hascased = _optimize_charset(av, iscased, tolower, fixes) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - emit(IN_LOC_IGNORE) - elif not hascased: - emit(IN) - elif not fixes: # ascii - emit(IN_IGNORE) + if not charset: + emit(FAILURE) + elif charset == _CHARSET_ALL: + emit(ANY_ALL) else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - _compile_charset(charset, flags, code) - code[skip] = _len(code) - skip + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + emit(IN_LOC_IGNORE) + elif not hascased: + emit(IN) + elif not fixes: # ascii + emit(IN_IGNORE) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + _compile_charset(charset, flags, code) + code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(ANY_ALL) @@ -150,7 +157,7 @@ def _compile(code, pattern, flags): if lo > MAXCODE: raise error("looks too much behind") if lo != hi: - raise error("look-behind requires fixed-width pattern") + raise PatternError("look-behind requires fixed-width pattern") emit(lo) # look behind _compile(code, av[1], flags) emit(SUCCESS) @@ -209,7 +216,7 @@ def _compile(code, pattern, flags): else: code[skipyes] = _len(code) - skipyes + 1 else: - raise error("internal: unsupported operand type %r" % (op,)) + raise PatternError(f"internal: unsupported operand type {op!r}") def _compile_charset(charset, flags, code): # compile charset subprogram @@ -235,7 +242,7 @@ def _compile_charset(charset, flags, code): else: emit(av) else: - raise error("internal: unsupported set operator %r" % (op,)) + raise PatternError(f"internal: unsupported set operator {op!r}") emit(FAILURE) def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): @@ -277,6 +284,10 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[i] = 1 elif op is NEGATE: out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False else: tail.append((op, av)) except IndexError: @@ -519,13 +530,18 @@ def _compile_info(code, pattern, flags): # look for a literal prefix prefix = [] prefix_skip = 0 - charset = [] # not used + charset = None # not used if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): # look for literal prefix prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) # if no prefix, look for charset prefix if not prefix: charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -560,8 +576,6 @@ def _compile_info(code, pattern, flags): # generate overlap table code.extend(_generate_overlap_table(prefix)) elif charset: - charset, hascased = _optimize_charset(charset) - assert not hascased _compile_charset(charset, flags, code) code[skip] = len(code) - skip diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index d8e483ac4f23b4..d6f32302d37b2d 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -15,12 +15,12 @@ MAGIC = 20230612 -from _sre import MAXREPEAT, MAXGROUPS +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 # SRE standard exception (access as sre.error) # should this really be here? -class error(Exception): +class PatternError(Exception): """Exception raised for invalid regular expressions. Attributes: @@ -53,6 +53,9 @@ def __init__(self, msg, pattern=None, pos=None): super().__init__(msg) +# Backward compatibility after renaming in 3.13 +error = PatternError + class _NamedIntConstant(int): def __new__(cls, value, name): self = super(_NamedIntConstant, cls).__new__(cls, value) @@ -203,6 +206,8 @@ def _makecodes(*names): CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } +CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2])) + # flags SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 206d6fb511cdf6..23eb0020f42e8a 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -35,6 +35,7 @@ import keyword import re import __main__ +import warnings __all__ = ["Completer"] @@ -88,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: diff --git a/Lib/runpy.py b/Lib/runpy.py index 42f896c9cd5094..ef54d3282eee06 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,17 +247,17 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(run_name, fname): +def _get_code_from_file(fname): # Check for a compiled file first from pkgutil import read_code - decoded_path = os.path.abspath(os.fsdecode(fname)) - with io.open_code(decoded_path) as f: + code_path = os.path.abspath(fname) + with io.open_code(code_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(decoded_path) as f: + with io.open_code(code_path) as f: code = compile(f.read(), fname, 'exec') - return code, fname + return code def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location. @@ -279,12 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) + path_name = os.fsdecode(path_name) if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, fname = _get_code_from_file(run_name, path_name) + code = _get_code_from_file(path_name) return _run_module_code(code, init_globals, run_name, - pkg_name=pkg_name, script_name=fname) + pkg_name=pkg_name, script_name=path_name) else: # Finder is defined for path, so add it to # the start of sys.path diff --git a/Lib/shutil.py b/Lib/shutil.py index 0fed0117a63234..0235f6bae32f14 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -47,7 +47,8 @@ COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -306,7 +307,12 @@ def copymode(src, dst, *, follow_symlinks=True): else: return else: - stat_func, chmod_func = _stat, os.chmod + stat_func = _stat + if os.name == 'nt' and os.path.islink(dst): + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -550,7 +556,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -590,115 +596,155 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - def _rmtree_islink(path): - try: - st = os.lstat(path) - return (stat.S_ISLNK(st.st_mode) or - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False + def _rmtree_islink(st): + return (stat.S_ISLNK(st.st_mode) or + (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT + and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) else: - def _rmtree_islink(path): - return os.path.islink(path) + def _rmtree_islink(st): + return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onexc): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) + st = os.lstat(path) except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") + except OSError as err: + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, fullname, err) + os.rmdir(fullname) + except FileNotFoundError: continue - _rmtree_unsafe(fullname, onexc) - else: + except OSError as err: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) try: os.rmdir(path) + except FileNotFoundError: + pass except OSError as err: onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] try: - with os.scandir(topfd) as scandir_it: - entries = list(scandir_it) - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) - dirfd_closed = False + os.close(fd) except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - dirfd_closed = True - os.rmdir(entry.name, dir_fd=topfd) - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - os.close(dirfd) + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name + try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. + with os.scandir(topfd) as scandir_it: + entries = list(scandir_it) + for entry in entries: + fullname = os.path.join(path, entry.name) + try: + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue + except FileNotFoundError: + continue + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. @@ -721,11 +767,6 @@ def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): If both onerror and onexc are set, onerror is ignored and onexc is used. """ - if onerror is not None: - import warnings - warnings.warn("onerror argument is deprecated, use onexc instead", - DeprecationWarning, stacklevel=2) - sys.audit("shutil.rmtree", path, dir_fd) if ignore_errors: def onexc(*args): @@ -747,53 +788,7 @@ def onexc(*args): exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except Exception as err: - onexc(os.lstat, path, err) - return - try: - fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) - fd_closed = False - except Exception as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - fd_closed = True - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - finally: - if not fd_closed: - os.close(fd) - else: - if dir_fd is not None: - raise NotImplementedError("dir_fd unavailable on this platform") - try: - if _rmtree_islink(path): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - # can't continue even if onexc hook returns - return - return _rmtree_unsafe(path, onexc) + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform @@ -822,12 +817,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are @@ -846,7 +841,7 @@ def move(src, dst, copy_function=copy2): sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): - if _samefile(src, dst): + if _samefile(src, dst) and not os.path.islink(src): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) @@ -1402,11 +1397,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1432,7 +1434,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. diff --git a/Lib/signal.py b/Lib/signal.py index 50b215b29d2fad..c8cd3d4f597ca5 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -22,9 +22,11 @@ def _int_to_enum(value, enum_klass): - """Convert a numeric value to an IntEnum member. - If it's not a known member, return the numeric value itself. + """Convert a possible numeric value to an IntEnum member. + If it's not a known member, return the value itself. """ + if not isinstance(value, int): + return value try: return enum_klass(value) except ValueError: diff --git a/Lib/site.py b/Lib/site.py index 672fa7b000ad02..460269433f021c 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -74,6 +74,7 @@ import builtins import _sitebuiltins import io +import stat # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -168,37 +169,56 @@ def addpackage(sitedir, name, known_paths): else: reset = False fullname = os.path.join(sitedir, name) + try: + st = os.lstat(fullname) + except OSError: + return + if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or + (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)): + _trace(f"Skipping hidden .pth file: {fullname!r}") + return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -221,7 +241,8 @@ def addsitedir(sitedir, known_paths=None): names = os.listdir(sitedir) except OSError: return - names = [name for name in names if name.endswith(".pth")] + names = [name for name in names + if name.endswith(".pth") and not name.startswith(".")] for name in sorted(names): addpackage(sitedir, name, known_paths) if reset: @@ -260,14 +281,18 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'Python' + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -275,7 +300,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -287,15 +312,21 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/python/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -361,6 +392,12 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -368,7 +405,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - "python%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -425,64 +462,94 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit - try: - import readline - import rlcompleter - except ImportError: - return - - # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: - readline.parse_and_bind('bind ^I rl_complete') - else: - readline.parse_and_bind('tab: complete') + import atexit + try: + import readline + import rlcompleter # noqa: F401 + import _pyrepl.readline + import _pyrepl.unix_console + except ImportError: + return + + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + if readline.backend == 'editline': + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass + + if readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() try: - readline.read_init_file() - except OSError: - # An OSError here could have many causes, but the most likely one - # is that there's no .inputrc file (or .editrc file in the case of - # Mac OS X + libedit) in the expected location. In that case, we - # want to ignore the exception. + if os.getenv("PYTHON_BASIC_REPL"): + readline.read_history_file(history) + else: + _pyrepl.readline.read_history_file(history) + except (OSError,* _pyrepl.unix_console._error): pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + def write_history(): try: - readline.read_history_file(history) - except OSError: - pass + from _pyrepl.main import CAN_USE_PYREPL + except ImportError: + CAN_USE_PYREPL = False - def write_history(): - try: + try: + if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass + else: + _pyrepl.readline.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 + pass - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -543,7 +610,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -563,7 +630,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100755 new mode 100644 index b3cc68a789a7d8..84d6d858e7dec1 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP @@ -105,7 +103,7 @@ class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException - exceptions, this sets `sender' to the string that the SMTP refused. + exceptions, this sets 'sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): @@ -315,7 +313,7 @@ def _get_socket(self, host, port, timeout): def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. - If the hostname ends with a colon (`:') followed by a number, and + If the hostname ends with a colon (':') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. @@ -346,7 +344,7 @@ def connect(self, host='localhost', port=0, source_address=None): return (code, msg) def send(self, s): - """Send `s' to the server.""" + """Send 's' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if self.sock: diff --git a/Lib/socket.py b/Lib/socket.py index 321fcda51505c1..524ce1361b9091 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -306,7 +306,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: @@ -382,7 +383,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if timeout and not selector_select(timeout): raise TimeoutError('timed out') if count: - blocksize = count - total_sent + blocksize = min(count - total_sent, blocksize) if blocksize <= 0: break try: @@ -702,16 +703,15 @@ def readinto(self, b): self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - if e.errno in _blocking_errnos: - return None - raise + try: + return self._sock.recv_into(b) + except timeout: + self._timeout_occurred = True + raise + except error as e: + if e.errno in _blocking_errnos: + return None + raise def write(self, b): """Write the given bytes or bytearray object *b* to the socket diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b92ff..34a9c047dd607c 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index b93b84384a0925..d9423c25e34135 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -117,7 +117,7 @@ def main(*args): # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg="") diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c922..0315760516edf8 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index ead3360ce67608..57e6a3b4f1e6eb 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -15,7 +15,7 @@ def _quote_value(value): return "'{0}'".format(value.replace("'", "''")) -def _iterdump(connection): +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -26,17 +26,30 @@ def _iterdump(connection): writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. + # Disable foreign key constraints, if there is any foreign key violation. + violations = cu.execute("PRAGMA foreign_key_check").fetchall() + if violations: + yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': @@ -78,13 +91,14 @@ def _iterdump(connection): yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) diff --git a/Lib/ssl.py b/Lib/ssl.py index 62e55857141dfc..cc685c2cc405ab 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -110,13 +110,13 @@ try: from _ssl import RAND_egd except ImportError: - # LibreSSL does not provide RAND_egd + # RAND_egd is not supported on some platforms pass from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION @@ -704,6 +704,16 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, else: raise ValueError(purpose) + # `VERIFY_X509_PARTIAL_CHAIN` makes OpenSSL's chain building behave more + # like RFC 3280 and 5280, which specify that chain building stops with the + # first trust anchor, even if that anchor is not self-signed. + # + # `VERIFY_X509_STRICT` makes OpenSSL more conservative about the + # certificates it accepts, including "disabling workarounds for + # some broken certificates." + context.verify_flags |= (_ssl.VERIFY_X509_PARTIAL_CHAIN | + _ssl.VERIFY_X509_STRICT) + if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) elif context.verify_mode != CERT_NONE: @@ -994,71 +1004,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - sock_timeout = sock.gettimeout() sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - blocking = self.getblocking() - self.setblocking(False) + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + + # See if we are connected try: - # We are not connected so this is not supposed to block, but - # testing revealed otherwise on macOS and Windows so we do - # the non-blocking dance regardless. Our raise when any data - # is found means consuming the data is harmless. - notconn_pre_handshake_data = self.recv(1) + self.getpeername() except OSError as e: - # EINVAL occurs for recv(1) on non-connected on unix sockets. - if e.errno not in (errno.ENOTCONN, errno.EINVAL): + if e.errno != errno.ENOTCONN: raise - notconn_pre_handshake_data = b'' - self.setblocking(blocking) - if notconn_pre_handshake_data: - # This prevents pending data sent to the socket before it was - # closed from escaping to the caller who could otherwise - # presume it came through a successful TLS connection. - reason = "Closed before TLS handshake with data in recv buffer." - notconn_pre_handshake_data_error = SSLError(e.errno, reason) - # Add the SSLError attributes that _ssl.c always adds. - notconn_pre_handshake_data_error.reason = reason - notconn_pre_handshake_data_error.library = None + connected = False + blocking = self.getblocking() + self.setblocking(False) try: - self.close() - except OSError: - pass - try: - raise notconn_pre_handshake_data_error - finally: - # Explicitly break the reference cycle. - notconn_pre_handshake_data_error = None - else: - connected = True + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True - self.settimeout(sock_timeout) # Must come after setblocking() calls. - self._connected = connected - if connected: - # create the SSL object - try: + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1069,9 +1075,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property @@ -1270,10 +1279,14 @@ def recv(self, buflen=1024, flags=0): def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() - if buffer and (nbytes is None): - nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 + if nbytes is None: + if buffer is not None: + with memoryview(buffer) as view: + nbytes = view.nbytes + if not nbytes: + nbytes = 1024 + else: + nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( diff --git a/Lib/stat.py b/Lib/stat.py index 52cadbf04f6c88..1b4ed1ebc940ef 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -110,19 +110,25 @@ def S_ISWHT(mode): S_IXOTH = 0o0001 # execute by others # Names for file flags - +UF_SETTABLE = 0x0000ffff # owner settable flags UF_NODUMP = 0x00000001 # do not dump file UF_IMMUTABLE = 0x00000002 # file may not be changed UF_APPEND = 0x00000004 # file may only be appended to UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted -UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed -UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +UF_COMPRESSED = 0x00000020 # macOS: file is compressed +UF_TRACKED = 0x00000040 # macOS: used for handling document IDs +UF_DATAVAULT = 0x00000080 # macOS: entitlement needed for I/O +UF_HIDDEN = 0x00008000 # macOS: file should not be displayed +SF_SETTABLE = 0xffff0000 # superuser settable flags SF_ARCHIVED = 0x00010000 # file may be archived SF_IMMUTABLE = 0x00020000 # file may not be changed SF_APPEND = 0x00040000 # file may only be appended to +SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted SF_SNAPSHOT = 0x00200000 # file is a snapshot file +SF_FIRMLINK = 0x00800000 # macOS: file is a firmlink +SF_DATALESS = 0x40000000 # macOS: file is a dataless object _filemode_table = ( diff --git a/Lib/statistics.py b/Lib/statistics.py index 4da06889c6db46..c64c6fae4ab010 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -112,6 +112,8 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -137,449 +139,168 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) +_random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False + + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - Ties are averaged so that equal values receive the same rank: + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60]) + 48.0 - The operation is idempotent: + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: + If ``data`` is empty, or any element is less than zero, + ``harmonic_mean`` will raise ``StatisticsError``. - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] + """ + if iter(data) is data: + data = list(data) - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: + errmsg = 'harmonic mean does not support negative values' - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] + n = len(data) + if n < 1: + raise StatisticsError('harmonic_mean requires at least one data point') + elif n == 1 and weights is None: + x = data[0] + if isinstance(x, (numbers.Real, Decimal)): + if x < 0: + raise StatisticsError(errmsg) + return x + else: + raise TypeError('unsupported type') - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty, - if it contains a zero, or if it contains a negative value. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - try: - return exp(fmean(map(log, data))) - except ValueError: - raise StatisticsError('geometric mean requires a non-empty dataset ' - 'containing positive numbers') from None - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - If ``data`` is empty, or any element is less than zero, - ``harmonic_mean`` will raise ``StatisticsError``. - """ - if iter(data) is data: - data = list(data) - errmsg = 'harmonic mean does not support negative values' - n = len(data) - if n < 1: - raise StatisticsError('harmonic_mean requires at least one data point') - elif n == 1 and weights is None: - x = data[0] - if isinstance(x, (numbers.Real, Decimal)): - if x < 0: - raise StatisticsError(errmsg) - return x - else: - raise TypeError('unsupported type') if weights is None: weights = repeat(1, n) sum_weights = n @@ -589,16 +310,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -635,6 +359,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -778,6 +505,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -786,132 +514,48 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. +## Measures of spread ###################################################### -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". +def variance(data, xbar=None): + """Return the sample variance of data. -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. + Examples: -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Returns a list of (n - 1) cut points separating the intervals. + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. + + Decimals and Fractions are supported: + + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') + + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html - - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. - - Decimals and Fractions are supported: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) + # http://mathworld.wolfram.com/SampleVariance.html - """ T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') @@ -953,6 +597,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -995,46 +641,7 @@ def pstdev(data, mu=None): return _float_sqrt_of_frac(mss.numerator, mss.denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1053,6 +660,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1086,7 +694,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1094,6 +705,7 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) @@ -1103,9 +715,11 @@ def correlation(x, y, /, *, method='linear'): ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1153,28 +767,445 @@ def linear_regression(x, y, /, *, proportional=False): LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] # List because used three times below y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) + + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf + + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### def _normal_dist_inv_cdf(p, mu, sigma): # There is no closed-form solution to the inverse CDF for the normal @@ -1183,6 +1214,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. q = p - 0.5 + if fabs(q) <= 0.425: r = 0.180625 - q * q # Hash sum: 55.88319_28806_14901_4439 @@ -1204,6 +1236,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.0) x = num / den return mu + (x * sigma) + r = p if q <= 0.0 else 1.0 - p r = sqrt(-log(r)) if r <= 5.0: @@ -1244,9 +1277,11 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.36929_88092_27358_05310e-1) * r + 5.99832_20655_58879_37690e-1) * r + 1.0) + x = num / den if q < 0.0: x = -x + return mu + (x * sigma) @@ -1478,3 +1513,341 @@ def __getstate__(self): def __setstate__(self, state): self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636498..ff98e8c4cb3f1d 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 6df5dd551ea67e..bc08878db313df 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -74,15 +74,16 @@ else: _mswindows = True -# wasm32-emscripten and wasm32-wasi do not support processes -_can_fork_exec = sys.platform not in {"emscripten", "wasi"} +# some platforms do not support subprocesses +_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"} if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +94,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", @@ -103,18 +105,22 @@ if _can_fork_exec: from _posixsubprocess import fork_exec as _fork_exec # used in methods that are called by __del__ - _waitpid = os.waitpid - _waitstatus_to_exitcode = os.waitstatus_to_exitcode - _WIFSTOPPED = os.WIFSTOPPED - _WSTOPSIG = os.WSTOPSIG - _WNOHANG = os.WNOHANG + class _del_safe: + waitpid = os.waitpid + waitstatus_to_exitcode = os.waitstatus_to_exitcode + WIFSTOPPED = os.WIFSTOPPED + WSTOPSIG = os.WSTOPSIG + WNOHANG = os.WNOHANG + ECHILD = errno.ECHILD else: - _fork_exec = None - _waitpid = None - _waitstatus_to_exitcode = None - _WIFSTOPPED = None - _WSTOPSIG = None - _WNOHANG = None + class _del_safe: + waitpid = None + waitstatus_to_exitcode = None + WIFSTOPPED = None + WSTOPSIG = None + WNOHANG = None + ECHILD = errno.ECHILD + import select import selectors @@ -346,7 +352,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -744,6 +750,7 @@ def _use_posix_spawn(): # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() _USE_VFORK = True +_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') class Popen: @@ -834,6 +841,9 @@ def __init__(self, args, bufsize=-1, executable=None, if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if stdout is STDOUT: + raise ValueError("STDOUT can only be used for stderr") + if pipesize is None: pipesize = -1 # Restore default if not isinstance(pipesize, int): @@ -1581,6 +1591,8 @@ def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE + elif timeout <= 0: + timeout_millis = 0 else: timeout_millis = int(timeout * 1000) if self.returncode is None: @@ -1747,14 +1759,11 @@ def _get_handles(self, stdin, stdout, stderr): errread, errwrite) - def _posix_spawn(self, args, executable, env, restore_signals, + def _posix_spawn(self, args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program using os.posix_spawn().""" - if env is None: - env = os.environ - kwargs = {} if restore_signals: # See _Py_RestoreSignals() in Python/pylifecycle.c @@ -1776,6 +1785,10 @@ def _posix_spawn(self, args, executable, env, restore_signals, ): if fd != -1: file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) + + if close_fds: + file_actions.append((os.POSIX_SPAWN_CLOSEFROM, 3)) + if file_actions: kwargs['file_actions'] = file_actions @@ -1823,7 +1836,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None - and not close_fds + and (not close_fds or _HAVE_POSIX_SPAWN_CLOSEFROM) and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) @@ -1835,7 +1848,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, and gids is None and uid is None and umask < 0): - self._posix_spawn(args, executable, env, restore_signals, + self._posix_spawn(args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -1938,33 +1951,34 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) - child_exec_never_called = (err_msg == "noexec") - if child_exec_never_called: + if err_msg == "noexec:chdir": err_msg = "" # The error must be from chdir(cwd). err_filename = cwd + elif err_msg == "noexec": + err_msg = "" + err_filename = None else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) - raise child_exception_type(errno_num, err_msg, err_filename) + if err_filename is not None: + raise child_exception_type(errno_num, err_msg, err_filename) + else: + raise child_exception_type(errno_num, err_msg) raise child_exception_type(err_msg) - def _handle_exitstatus(self, sts, - _waitstatus_to_exitcode=_waitstatus_to_exitcode, - _WIFSTOPPED=_WIFSTOPPED, - _WSTOPSIG=_WSTOPSIG): + def _handle_exitstatus(self, sts, _del_safe=_del_safe): """All callers to this function MUST hold self._waitpid_lock.""" # This method is called (indirectly) by __del__, so it cannot # refer to anything outside of its local scope. - if _WIFSTOPPED(sts): - self.returncode = -_WSTOPSIG(sts) + if _del_safe.WIFSTOPPED(sts): + self.returncode = -_del_safe.WSTOPSIG(sts) else: - self.returncode = _waitstatus_to_exitcode(sts) + self.returncode = _del_safe.waitstatus_to_exitcode(sts) - def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, - _WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD): + def _internal_poll(self, _deadstate=None, _del_safe=_del_safe): """Check if child process has terminated. Returns returncode attribute. @@ -1980,13 +1994,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, try: if self.returncode is not None: return self.returncode # Another thread waited. - pid, sts = _waitpid(self.pid, _WNOHANG) + pid, sts = _del_safe.waitpid(self.pid, _del_safe.WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) except OSError as e: if _deadstate is not None: self.returncode = _deadstate - elif e.errno == _ECHILD: + elif e.errno == _del_safe.ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we diff --git a/Lib/symtable.py b/Lib/symtable.py index 4b0bc6f497a553..221aaf88d41670 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,13 +1,21 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref +from enum import StrEnum -__all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] +__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] def symtable(code, filename, compile_type): """ Return the toplevel *SymbolTable* for the source code. @@ -39,6 +47,16 @@ def __call__(self, table, filename): _newSymbolTable = SymbolTableFactory() +class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + + class SymbolTable: def __init__(self, raw_table, filename): @@ -62,23 +80,23 @@ def __repr__(self): def get_type(self): """Return the type of the symbol table. - The values returned are 'class', 'module', 'function', - 'annotation', 'TypeVar bound', 'type alias', and 'type parameter'. + The value returned is one of the values in + the ``SymbolTableType`` enumeration. """ if self._table.type == _symtable.TYPE_MODULE: - return "module" + return SymbolTableType.MODULE if self._table.type == _symtable.TYPE_FUNCTION: - return "function" + return SymbolTableType.FUNCTION if self._table.type == _symtable.TYPE_CLASS: - return "class" + return SymbolTableType.CLASS if self._table.type == _symtable.TYPE_ANNOTATION: - return "annotation" - if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: - return "TypeVar bound" + return SymbolTableType.ANNOTATION if self._table.type == _symtable.TYPE_TYPE_ALIAS: - return "type alias" - if self._table.type == _symtable.TYPE_TYPE_PARAM: - return "type parameter" + return SymbolTableType.TYPE_ALIAS + if self._table.type == _symtable.TYPE_TYPE_PARAMETERS: + return SymbolTableType.TYPE_PARAMETERS + if self._table.type == _symtable.TYPE_TYPE_VARIABLE: + return SymbolTableType.TYPE_VARIABLE assert False, f"unexpected type: {self._table.type}" def get_id(self): @@ -154,6 +172,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -179,7 +201,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -188,7 +210,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -203,7 +225,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -217,8 +239,25 @@ def get_methods(self): """ if self.__methods is None: d = {} + + def is_local_symbol(ident): + flags = self._table.symbols.get(ident, 0) + return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL + for st in self._table.children: - d[st.name] = 1 + # pick the function-like symbols that are local identifiers + if is_local_symbol(st.name): + match st.type: + case _symtable.TYPE_FUNCTION: + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + d[st.name] = 1 + break self.__methods = tuple(d) return self.__methods @@ -228,12 +267,21 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope def __repr__(self): - return "".format(self.__name) + flags_str = '|'.join(self._flags_str()) + return f'' + + def _scope_str(self): + return _scopes_value_to_name.get(self.__scope) or str(self.__scope) + + def _flags_str(self): + for flagname, flagvalue in _flags: + if self.__flags & flagvalue == flagvalue: + yield flagname def get_name(self): """Return a name of a symbol. @@ -244,13 +292,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ @@ -283,6 +336,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -293,6 +351,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. @@ -323,11 +391,43 @@ def get_namespace(self): else: return self.__namespaces[0] + +_flags = [('USE', USE)] +_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_')) +_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL') +_scopes_value_to_name = {globals()[n]: n for n in _scopes_names} + + +def main(args): + import sys + def print_symbols(table, level=0): + indent = ' ' * level + nested = "nested " if table.is_nested() else "" + if table.get_type() == 'module': + what = f'from file {table._filename!r}' + else: + what = f'{table.get_name()!r}' + print(f'{indent}symbol table for {nested}{table.get_type()} {what}:') + for ident in table.get_identifiers(): + symbol = table.lookup(ident) + flags = ', '.join(symbol._flags_str()).lower() + print(f' {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}') + print() + + for table2 in table.get_children(): + print_symbols(table2, level + 1) + + for filename in args or ['-']: + if filename == '-': + src = sys.stdin.read() + filename = '' + else: + with open(filename, 'rb') as f: + src = f.read() + mod = symtable(src, filename, 'exec') + print_symbols(mod) + + if __name__ == "__main__": - import os, sys - with open(sys.argv[0]) as f: - src = f.read() - mod = symtable(src, os.path.split(sys.argv[0])[1], "exec") - for ident in mod.get_identifiers(): - info = mod.lookup(ident) - print(info, info.is_local(), info.is_namespace()) + import sys + main(sys.argv[1:]) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 68d30c0f9e618f..83e057c177f8c0 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -21,29 +21,30 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -57,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -75,14 +77,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -104,6 +106,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'Python' # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -112,8 +116,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -121,7 +125,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -135,29 +139,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -288,6 +292,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -348,7 +353,7 @@ def _init_non_posix(vars): vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - # Add EXT_SUFFIX, SOABI, and Py_NOGIL + # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED vars.update(_sysconfig.config_vars()) vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) @@ -404,7 +409,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: @@ -459,6 +464,8 @@ def _init_config_vars(): _CONFIG_VARS['platbase'] = _EXEC_PREFIX _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() try: _CONFIG_VARS['abiflags'] = sys.abiflags except AttributeError: @@ -480,6 +487,9 @@ def _init_config_vars(): # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': @@ -594,10 +604,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -619,10 +641,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "12.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -631,6 +658,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index e2ac6837f157d5..c0097351b269f2 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation @@ -107,14 +105,14 @@ def check(file): errprint("%r: Token Error: %s" % (file, msg)) return - except SyntaxError as msg: - errprint("%r: Token Error: %s" % (file, msg)) - return - except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index ec32f9ba49b03f..d5d8a469779f50 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- @@ -330,10 +329,11 @@ def write(self, s): class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. + needs to have a read() or write() method that works with bytes, + and the method is accessed blockwise. + Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin.buffer, + sys.stdout.buffer, a socket, a tape device etc. _Stream is intended to be used only internally. """ @@ -635,6 +635,10 @@ def __init__(self, fileobj, offset, size, name, blockinfo=None): def flush(self): pass + @property + def mode(self): + return 'rb' + def readable(self): return True @@ -871,7 +875,7 @@ class TarInfo(object): pax_headers = ('A dictionary containing key-value pairs of an ' 'associated pax extended header.'), sparse = 'Sparse member information.', - tarfile = None, + _tarfile = None, _sparse_structs = None, _link_target = None, ) @@ -900,6 +904,24 @@ def __init__(self, name=""): self.sparse = None # sparse member information self.pax_headers = {} # pax header information + @property + def tarfile(self): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + return self._tarfile + + @tarfile.setter + def tarfile(self, tarfile): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + self._tarfile = tarfile + @property def path(self): 'In pax headers, "name" is called "path".' @@ -1194,7 +1216,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1640,13 +1662,13 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None, copybufsize=None, stream=False): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1975,7 +1997,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -2003,9 +2025,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -2029,7 +2051,7 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed + tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: @@ -2101,11 +2123,15 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ + # Convert tarinfo type to stat type. + type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, + FIFOTYPE: stat.S_IFIFO, CHRTYPE: stat.S_IFCHR, + DIRTYPE: stat.S_IFDIR, BLKTYPE: stat.S_IFBLK} self._check() if members is None: @@ -2115,7 +2141,8 @@ def list(self, verbose=True, *, members=None): if tarinfo.mode is None: _safe_print("??????????") else: - _safe_print(stat.filemode(tarinfo.mode)) + modetype = type2mode.get(tarinfo.type, 0) + _safe_print(stat.filemode(modetype | tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): @@ -2139,11 +2166,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2190,13 +2217,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) @@ -2223,7 +2253,7 @@ def _get_filter_function(self, filter): 'Python 3.14 will, by default, filter extracted tar ' + 'archives and reject files or modify their metadata. ' + 'Use the filter argument to control this behavior.', - DeprecationWarning) + DeprecationWarning, stacklevel=3) return fully_trusted_filter if isinstance(filter, str): raise TypeError( @@ -2242,12 +2272,12 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called on each member just + The 'filter' function will be called on each member just before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. @@ -2287,13 +2317,13 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called before extraction. + The 'filter' function will be called before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. """ @@ -2358,10 +2388,10 @@ def _handle_fatal_error(self, e): self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2405,7 +2435,7 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) @@ -2450,7 +2480,8 @@ def makedir(self, tarinfo, targetpath): # later in _extract_member(). os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. @@ -2558,7 +2589,7 @@ def chown(self, tarinfo, targetpath, numeric_owner): else: os.chown(targetpath, u, g) except (OSError, OverflowError) as e: - # OverflowError can be raised if an ID doesn't fit in `id_t` + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 2b4f4313247128..b5a15f7b72c872 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -269,6 +269,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") +def _dont_follow_symlinks(func, path, *args): + # Pass follow_symlinks=False, unless not supported on this platform. + if func in _os.supports_follow_symlinks: + func(path, *args, follow_symlinks=False) + elif not _os.path.islink(path): + func(path, *args) + +def _resetperms(path): + try: + chflags = _os.chflags + except AttributeError: + pass + else: + _dont_follow_symlinks(chflags, path, 0) + _dont_follow_symlinks(_os.chmod, path, 0o700) + # User visible interfaces. @@ -872,26 +888,37 @@ def __init__(self, suffix=None, prefix=None, dir=None, ignore_errors=self._ignore_cleanup_errors, delete=self._delete) @classmethod - def _rmtree(cls, name, ignore_errors=False): + def _rmtree(cls, name, ignore_errors=False, repeated=False): def onexc(func, path, exc): if isinstance(exc, PermissionError): - def resetperms(path): - try: - _os.chflags(path, 0) - except AttributeError: - pass - _os.chmod(path, 0o700) + if repeated and path == name: + if ignore_errors: + return + raise try: if path != name: - resetperms(_os.path.dirname(path)) - resetperms(path) + _resetperms(_os.path.dirname(path)) + _resetperms(path) try: _os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError): + except IsADirectoryError: cls._rmtree(path, ignore_errors=ignore_errors) + except PermissionError: + # The PermissionError handler was originally added for + # FreeBSD in directories, but it seems that it is raised + # on Windows too. + # bpo-43153: Calling _rmtree again may + # raise NotADirectoryError and mask the PermissionError. + # So we must re-raise the current PermissionError if + # path is not a directory. + if not _os.path.isdir(path) or _os.path.isjunction(path): + if ignore_errors: + return + raise + cls._rmtree(path, ignore_errors=ignore_errors, + repeated=(path == name)) except FileNotFoundError: pass elif isinstance(exc, FileNotFoundError): diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index 424c81f6ecd759..1c9bac507209b1 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,7 +1,4 @@ fix = true -select = [ - "F811", # Redefinition of unused variable (useful for finding test methods with the same name) -] extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -11,22 +8,18 @@ extend-exclude = [ "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", # TODO Fix: F811 Redefinition of unused name - "test__opcode.py", "test_buffer.py", - "test_ctypes/test_arrays.py", - "test_ctypes/test_functions.py", "test_dataclasses/__init__.py", "test_descr.py", "test_enum.py", "test_functools.py", - "test_genericclass.py", "test_grammar.py", "test_import/__init__.py", - "test_keywordonlyarg.py", "test_pkg.py", - "test_subclassinit.py", "test_yield_from.py", - "time_hashlib.py", - # Pending https://github.com/python/cpython/pull/109139 - "test_monitoring.py", +] + +[lint] +select = [ + "F811", # Redefinition of unused variable (useful for finding test methods with the same name) ] diff --git a/Lib/test/_test_atexit.py b/Lib/test/_test_atexit.py index 55d28083349175..f618c1fcbca52b 100644 --- a/Lib/test/_test_atexit.py +++ b/Lib/test/_test_atexit.py @@ -19,7 +19,9 @@ def assert_raises_unraisable(self, exc_type, func, *args): atexit.register(func, *args) atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, exc_type) self.assertEqual(type(cm.unraisable.exc_value), exc_type) @@ -125,7 +127,9 @@ def func(): try: with support.catch_unraisable_exception() as cm: atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, ZeroDivisionError) self.assertEqual(type(cm.unraisable.exc_value), ZeroDivisionError) finally: diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 15586f15dfab30..493932d6c6d441 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -18,6 +18,7 @@ import socket import subprocess import sys +import textwrap import time import unittest @@ -492,29 +493,31 @@ def test_devpoll(self): self.check_elapsed_time(dt) -class FNTLEINTRTest(EINTRBaseTest): +class FCNTLEINTRTest(EINTRBaseTest): def _lock(self, lock_func, lock_name): self.addCleanup(os_helper.unlink, os_helper.TESTFN) - code = '\n'.join(( - "import fcntl, time", - "with open('%s', 'wb') as f:" % os_helper.TESTFN, - " fcntl.%s(f, fcntl.LOCK_EX)" % lock_name, - " time.sleep(%s)" % self.sleep_time)) - start_time = time.monotonic() - proc = self.subprocess(code) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) with kill_on_error(proc): with open(os_helper.TESTFN, 'wb') as f: # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready start_time = time.monotonic() - for _ in support.sleeping_retry(support.LONG_TIMEOUT, error=False): - try: - lock_func(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - lock_func(f, fcntl.LOCK_UN) - except BlockingIOError: - break - else: - dt = time.monotonic() - start_time - raise Exception("failed to sync child in %.1f sec" % dt) + os.write(wr2, b"go") # the child locked the file just a moment ago for 'sleep_time' seconds # that means that the lock below will block for 'sleep_time' minus some diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index a2ddd133cf47c8..23423d5b7a583d 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -6,14 +6,51 @@ # (before the site module is run). import _testinternalcapi -import os import sys import unittest +from test import support from test.support import MS_WINDOWS MAX_HASH_SEED = 4294967295 + +BOOL_OPTIONS = [ + 'isolated', + 'use_environment', + 'dev_mode', + 'install_signal_handlers', + 'use_hash_seed', + 'faulthandler', + 'import_time', + 'code_debug_ranges', + 'show_ref_count', + 'dump_refs', + 'malloc_stats', + 'parse_argv', + 'site_import', + 'warn_default_encoding', + 'inspect', + 'interactive', + 'parser_debug', + 'write_bytecode', + 'quiet', + 'user_site_directory', + 'configure_c_stdio', + 'buffered_stdio', + 'use_frozen_modules', + 'safe_path', + 'pathconfig_warnings', + 'module_search_paths_set', + 'skip_source_first_line', + '_install_importlib', + '_init_main', + '_is_python_build', +] +if MS_WINDOWS: + BOOL_OPTIONS.append('legacy_windows_stdio') + + class SetConfigTests(unittest.TestCase): def setUp(self): self.old_config = _testinternalcapi.get_config() @@ -52,42 +89,15 @@ def test_set_invalid(self): ] # int (unsigned) - options = [ + int_options = [ '_config_init', - 'isolated', - 'use_environment', - 'dev_mode', - 'install_signal_handlers', - 'use_hash_seed', - 'faulthandler', - 'tracemalloc', - 'import_time', - 'code_debug_ranges', - 'show_ref_count', - 'dump_refs', - 'malloc_stats', - 'parse_argv', - 'site_import', 'bytes_warning', - 'inspect', - 'interactive', 'optimization_level', - 'parser_debug', - 'write_bytecode', + 'tracemalloc', 'verbose', - 'quiet', - 'user_site_directory', - 'configure_c_stdio', - 'buffered_stdio', - 'pathconfig_warnings', - 'module_search_paths_set', - 'skip_source_first_line', - '_install_importlib', - '_init_main', ] - if MS_WINDOWS: - options.append('legacy_windows_stdio') - for key in options: + int_options.extend(BOOL_OPTIONS) + for key in int_options: value_tests.append((key, invalid_uint)) type_tests.append((key, "abc")) type_tests.append((key, 2.0)) @@ -148,6 +158,7 @@ def test_set_invalid(self): _testinternalcapi.set_config(config) def test_flags(self): + bool_options = set(BOOL_OPTIONS) for sys_attr, key, value in ( ("debug", "parser_debug", 1), ("inspect", "inspect", 2), @@ -160,7 +171,10 @@ def test_flags(self): ): with self.subTest(sys=sys_attr, key=key, value=value): self.set_config(**{key: value, 'parse_argv': 0}) - self.assertEqual(getattr(sys.flags, sys_attr), value) + if key in bool_options: + self.assertEqual(getattr(sys.flags, sys_attr), int(bool(value))) + else: + self.assertEqual(getattr(sys.flags, sys_attr), value) self.set_config(write_bytecode=0) self.assertEqual(sys.flags.dont_write_bytecode, True) @@ -197,6 +211,19 @@ def test_flags(self): self.set_config(use_hash_seed=1, hash_seed=123) self.assertEqual(sys.flags.hash_randomization, 1) + if support.Py_GIL_DISABLED: + self.set_config(enable_gil=-1) + self.assertEqual(sys.flags.gil, None) + self.set_config(enable_gil=0) + self.assertEqual(sys.flags.gil, 0) + self.set_config(enable_gil=1) + self.assertEqual(sys.flags.gil, 1) + else: + # Builds without Py_GIL_DISABLED don't have + # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for + # consistency. + self.assertEqual(sys.flags.gil, 1) + def test_options(self): self.check(warnoptions=[]) self.check(warnoptions=["default", "ignore"]) diff --git a/Lib/test/_test_monitoring_shutdown.py b/Lib/test/_test_monitoring_shutdown.py new file mode 100644 index 00000000000000..3d0fbec029dc09 --- /dev/null +++ b/Lib/test/_test_monitoring_shutdown.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# gh-115832: An object destructor running during the final GC of interpreter +# shutdown triggered an infinite loop in the instrumentation code. + +import sys + +class CallableCycle: + def __init__(self): + self._cycle = self + + def __del__(self): + pass + + def __call__(self, code, instruction_offset): + pass + +def tracefunc(frame, event, arg): + pass + +def main(): + tool_id = sys.monitoring.PROFILER_ID + event_id = sys.monitoring.events.PY_START + + sys.monitoring.use_tool_id(tool_id, "test profiler") + sys.monitoring.set_events(tool_id, event_id) + sys.monitoring.register_callback(tool_id, event_id, CallableCycle()) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 0b333ca3b7e9dc..4b3a0645cfc84a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -22,7 +22,6 @@ import subprocess import struct import operator -import pathlib import pickle import weakref import warnings @@ -324,8 +323,9 @@ def test_set_executable(self): self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str - sys.executable.encode(), # bytes - pathlib.Path(sys.executable) # os.PathLike + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes ] for path in paths: self.set_executable(path) @@ -1332,6 +1332,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -2438,8 +2455,11 @@ def test_namespace(self): # # -def sqr(x, wait=0.0): - time.sleep(wait) +def sqr(x, wait=0.0, event=None): + if event is None: + time.sleep(wait) + else: + event.wait(wait) return x*x def mul(x, y): @@ -2578,10 +2598,18 @@ def test_async(self): self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1) def test_async_timeout(self): - res = self.pool.apply_async(sqr, (6, TIMEOUT2 + support.SHORT_TIMEOUT)) - get = TimingWrapper(res.get) - self.assertRaises(multiprocessing.TimeoutError, get, timeout=TIMEOUT2) - self.assertTimingAlmostEqual(get.elapsed, TIMEOUT2) + p = self.Pool(3) + try: + event = threading.Event() if self.TYPE == 'threads' else None + res = p.apply_async(sqr, (6, TIMEOUT2 + support.SHORT_TIMEOUT, event)) + get = TimingWrapper(res.get) + self.assertRaises(multiprocessing.TimeoutError, get, timeout=TIMEOUT2) + self.assertTimingAlmostEqual(get.elapsed, TIMEOUT2) + finally: + if event is not None: + event.set() + p.terminate() + p.join() def test_imap(self): it = self.pool.imap(sqr, list(range(10))) @@ -2683,10 +2711,20 @@ def test_make_pool(self): def test_terminate(self): # Simulate slow tasks which take "forever" to complete - args = [support.LONG_TIMEOUT for i in range(10_000)] - result = self.pool.map_async(time.sleep, args, chunksize=1) - self.pool.terminate() - self.pool.join() + sleep_time = support.LONG_TIMEOUT + + if self.TYPE == 'threads': + # Thread pool workers can't be forced to quit, so if the first + # task starts early enough, we will end up waiting for it. + # Sleep for a shorter time, so the test doesn't block. + sleep_time = 1 + + p = self.Pool(3) + args = [sleep_time for i in range(10_000)] + result = p.map_async(time.sleep, args, chunksize=1) + time.sleep(0.2) # give some tasks a chance to start + p.terminate() + p.join() def test_empty_iterable(self): # See Issue 12157 @@ -2791,8 +2829,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. @@ -3483,6 +3521,30 @@ def test_context(self): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b'' + + with self.connection.Listener(authkey=key) as listener: + thread = threading.Thread(target=run, args=(listener.address, key)) + thread.start() + try: + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + finally: + thread.join() + + if self.TYPE == 'processes': + with self.assertRaises(OSError): + listener.accept() + @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") def test_abstract_socket(self): @@ -3950,6 +4012,21 @@ def _new_shm_name(self, prefix): # test_multiprocessing_spawn, etc) in parallel. return prefix + str(os.getpid()) + def test_shared_memory_name_with_embedded_null(self): + name_tsmb = self._new_shm_name('test01_null') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + with self.assertRaises(ValueError): + shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512) + if shared_memory._USE_POSIX: + orig_name = sms._name + try: + sms._name = orig_name + '\0a' + with self.assertRaises(ValueError): + sms.unlink() + finally: + sms._name = orig_name + def test_shared_memory_basics(self): name_tsmb = self._new_shm_name('test01_tsmb') sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) @@ -4084,7 +4161,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - def test_invalid_shared_memory_cration(self): + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True, size=-1) @@ -4440,6 +4517,59 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_untracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=False, it must not cause the memory to be deleted + # when terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=False) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + # The resource tracker shares pipes with the subprocess, and so + # err existing means that the tracker process has terminated now. + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) + self.assertEqual(rc, 0) + mem2 = shared_memory.SharedMemory(create=False, name=mem.name) + mem2.close() + finally: + try: + mem.unlink() + except OSError: + pass + mem.close() + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_tracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=True, it must cause the memory to be deleted when + # terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=True) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertEqual(rc, 0) + self.assertIn( + b"resource_tracker: There appear to be 1 leaked " + b"shared_memory objects to clean up at shutdown", err) + finally: + try: + mem.unlink() + except OSError: + pass + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() + # # Test to verify that `Finalize` works. # @@ -4549,7 +4679,7 @@ def make_finalizers(): old_interval = sys.getswitchinterval() old_threshold = gc.get_threshold() try: - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) gc.set_threshold(5, 5, 5) threads = [threading.Thread(target=run_finalizers), threading.Thread(target=make_finalizers)] @@ -4656,6 +4786,29 @@ def test_level(self): root_logger.setLevel(root_level) logger.setLevel(level=LOG_LEVEL) + def test_filename(self): + logger = multiprocessing.get_logger() + original_level = logger.level + try: + logger.setLevel(util.DEBUG) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logging_format = '[%(levelname)s] [%(filename)s] %(message)s' + handler.setFormatter(logging.Formatter(logging_format)) + logger.addHandler(handler) + logger.info('1') + util.info('2') + logger.debug('3') + filename = os.path.basename(__file__) + log_record = stream.getvalue() + self.assertIn(f'[INFO] [{filename}] 1', log_record) + self.assertIn(f'[INFO] [{filename}] 2', log_record) + self.assertIn(f'[DEBUG] [{filename}] 3', log_record) + finally: + logger.setLevel(original_level) + logger.removeHandler(handler) + handler.close() + # class _TestLoggingProcessName(BaseTestCase): # @@ -5512,8 +5665,9 @@ def create_and_register_resource(rtype): ''' for rtype in resource_tracker._CLEANUP_FUNCS: with self.subTest(rtype=rtype): - if rtype == "noop": + if rtype in ("noop", "dummy"): # Artefact resource type used by the resource_tracker + # or tests continue r, w = os.pipe() p = subprocess.Popen([sys.executable, @@ -5633,6 +5787,38 @@ def test_too_long_name_resource(self): with self.assertRaises(ValueError): resource_tracker.register(too_long_name_resource, rtype) + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) class TestSimpleQueue(unittest.TestCase): @@ -5646,6 +5832,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -5895,12 +6090,30 @@ def _test_list(cls, obj): case.assertEqual(obj[0], 5) case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) + obj += [7] + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7]) + obj *= 2 + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7, 5, 7]) + double_obj = obj * 2 + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + double_obj = 2 * obj + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + copied_obj = obj.copy() + case.assertIsInstance(copied_obj, list) + case.assertListEqual(list(copied_obj), [5, 7, 5, 7]) + obj.extend(double_obj + copied_obj) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 1) - case.assertEqual(obj.pop(0), 5) + case.assertEqual(len(obj), 16) + case.assertEqual(obj.pop(0), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_list(self): o = self.manager.list() @@ -5919,7 +6132,29 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.keys()), ['foo']) case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) - case.assertTupleEqual(obj.popitem(), ('foo', 5)) + obj |= {'bar': 6} + case.assertIsInstance(obj, multiprocessing.managers.DictProxy) + case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6}) + x = reversed(obj) + case.assertIsInstance(x, type(iter([]))) + case.assertListEqual(list(x), ['bar', 'foo']) + x = {'bar': 7, 'baz': 7} | obj + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7}) + x = obj | {'bar': 7, 'baz': 7} + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) + x = obj.fromkeys(['bar'], 6) + case.assertIsInstance(x, dict) + case.assertDictEqual(x, {'bar': 6}) + x = obj.popitem() + case.assertIsInstance(x, tuple) + case.assertTupleEqual(x, ('bar', 6)) + obj.setdefault('bar', 0) + obj.update({'bar': 7}) + case.assertEqual(obj.pop('bar'), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_dict(self): o = self.manager.dict() @@ -5992,6 +6227,29 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded @@ -6016,6 +6274,24 @@ def test_spawn_sys_executable_none_allows_import(self): self.assertEqual(rc, 0) self.assertFalse(err, msg=err.decode('utf-8')) + def test_large_pool(self): + # + # gh-89240: Check that large pools are always okay + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing + def f(x): return x*x + if __name__ == '__main__': + with multiprocessing.Pool(200) as p: + print(sum(p.map(f, range(1000)))) + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + self.assertEqual("332833500", out.decode('utf-8').strip()) + self.assertFalse(err, msg=err.decode('utf-8')) + # # Mixins diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 00000000000000..24745941b08923 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,177 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import swap_attr +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/ziptestdata/README.md b/Lib/test/archivetestdata/README.md similarity index 62% rename from Lib/test/ziptestdata/README.md rename to Lib/test/archivetestdata/README.md index 6b9147db76e178..7b555fa32765bf 100644 --- a/Lib/test/ziptestdata/README.md +++ b/Lib/test/archivetestdata/README.md @@ -1,21 +1,23 @@ -# Test data for `test_zipfile` +# Test data for `test_zipfile`, `test_tarfile` (and even some others) -The test executables in this directory are created manually from header.sh and -the `testdata_module_inside_zip.py` file. You must have infozip's zip utility +## `test_zipfile` + +The test executables in this directory are created manually from `header.sh` and +the `testdata_module_inside_zip.py` file. You must have Info-ZIP's zip utility installed (`apt install zip` on Debian). -## Purpose +### Purpose of `exe_with_zip` and `exe_with_z64` These are used to test executable files with an appended zipfile, in a scenario where the executable is _not_ a Python interpreter itself so our automatic zipimport machinery (that'd look for `__main__.py`) is not being used. -## Updating the test executables +### Updating the test executables If you update header.sh or the testdata_module_inside_zip.py file, rerun the commands below. These are expected to be rarely changed, if ever. -### Standard old format (2.0) zip file +#### Standard old format (2.0) zip file ``` zip -0 zip2.zip testdata_module_inside_zip.py @@ -23,13 +25,12 @@ cat header.sh zip2.zip >exe_with_zip rm zip2.zip ``` -### Modern format (4.5) zip64 file +#### Modern format (4.5) zip64 file -Redirecting from stdin forces infozip's zip tool to create a zip64. +Redirecting from stdin forces Info-ZIP's zip tool to create a zip64. ``` zip -0 zip64.zip cat header.sh zip64.zip >exe_with_z64 rm zip64.zip ``` - diff --git a/Lib/test/ziptestdata/exe_with_z64 b/Lib/test/archivetestdata/exe_with_z64 similarity index 100% rename from Lib/test/ziptestdata/exe_with_z64 rename to Lib/test/archivetestdata/exe_with_z64 diff --git a/Lib/test/ziptestdata/exe_with_zip b/Lib/test/archivetestdata/exe_with_zip similarity index 100% rename from Lib/test/ziptestdata/exe_with_zip rename to Lib/test/archivetestdata/exe_with_zip diff --git a/Lib/test/ziptestdata/header.sh b/Lib/test/archivetestdata/header.sh similarity index 100% rename from Lib/test/ziptestdata/header.sh rename to Lib/test/archivetestdata/header.sh diff --git a/Lib/test/recursion.tar b/Lib/test/archivetestdata/recursion.tar similarity index 100% rename from Lib/test/recursion.tar rename to Lib/test/archivetestdata/recursion.tar diff --git a/Lib/test/ziptestdata/testdata_module_inside_zip.py b/Lib/test/archivetestdata/testdata_module_inside_zip.py similarity index 100% rename from Lib/test/ziptestdata/testdata_module_inside_zip.py rename to Lib/test/archivetestdata/testdata_module_inside_zip.py diff --git a/Lib/test/testtar.tar b/Lib/test/archivetestdata/testtar.tar similarity index 100% rename from Lib/test/testtar.tar rename to Lib/test/archivetestdata/testtar.tar diff --git a/Lib/test/testtar.tar.xz b/Lib/test/archivetestdata/testtar.tar.xz similarity index 100% rename from Lib/test/testtar.tar.xz rename to Lib/test/archivetestdata/testtar.tar.xz diff --git a/Lib/test/zip_cp437_header.zip b/Lib/test/archivetestdata/zip_cp437_header.zip similarity index 100% rename from Lib/test/zip_cp437_header.zip rename to Lib/test/archivetestdata/zip_cp437_header.zip diff --git a/Lib/test/zipdir.zip b/Lib/test/archivetestdata/zipdir.zip similarity index 100% rename from Lib/test/zipdir.zip rename to Lib/test/archivetestdata/zipdir.zip diff --git a/Lib/test/archivetestdata/zipdir_backslash.zip b/Lib/test/archivetestdata/zipdir_backslash.zip new file mode 100644 index 00000000000000..979126ef5e37eb Binary files /dev/null and b/Lib/test/archivetestdata/zipdir_backslash.zip differ diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index f0cedde308d53b..b9021467817f27 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -289,7 +289,7 @@ def hook(event, args): def test_unraisablehook(): - from _testinternalcapi import write_unraisable_exc + from _testcapi import err_formatunraisable def unraisablehook(hookargs): pass @@ -302,7 +302,8 @@ def hook(event, args): sys.addaudithook(hook) sys.unraisablehook = unraisablehook - write_unraisable_exc(RuntimeError("nonfatal-error"), "for audit hook test", None) + err_formatunraisable(RuntimeError("nonfatal-error"), + "Exception ignored for audit hook test") def test_winreg(): @@ -454,6 +455,9 @@ def __call__(self): i = _thread.start_new_thread(test_func(), ()) lock.acquire() + handle = _thread.start_joinable_thread(test_func()) + handle.join() + def test_threading_abort(): # Ensures that aborting PyThreadState_New raises the correct exception @@ -483,7 +487,13 @@ def hook(event, args): print(event, args[0]) sys.addaudithook(hook) - _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + try: + _wmi.exec_query("SELECT * FROM Win32_OperatingSystem") + except WindowsError as e: + # gh-112278: WMI may be slow response when first called, but we still + # get the audit event, so just ignore the timeout + if e.winerror != 258: + raise def test_syslog(): import syslog @@ -546,6 +556,17 @@ def hook(event, args): sys.monitoring.register_callback(1, 1, None) +def test_winapi_createnamedpipe(pipe_name): + import _winapi + + def hook(event, args): + if event == "_winapi.CreateNamedPipe": + print(event, args) + + sys.addaudithook(hook) + _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/bisect_cmd.py b/Lib/test/bisect_cmd.py index 5cb804bd469dc3..aee2e8ac120852 100755 --- a/Lib/test/bisect_cmd.py +++ b/Lib/test/bisect_cmd.py @@ -51,6 +51,7 @@ def python_cmd(): cmd = [sys.executable] cmd.extend(subprocess._args_from_interpreter_flags()) cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.extend(('-X', 'faulthandler')) return cmd @@ -77,9 +78,13 @@ def run_tests(args, tests, huntrleaks=None): write_tests(tmp, tests) cmd = python_cmd() - cmd.extend(['-m', 'test', '--matchfile', tmp]) + cmd.extend(['-u', '-m', 'test', '--matchfile', tmp]) cmd.extend(args.test_args) print("+ %s" % format_shell_args(cmd)) + + sys.stdout.flush() + sys.stderr.flush() + proc = subprocess.run(cmd) return proc.returncode finally: @@ -137,8 +142,8 @@ def main(): ntest = max(ntest // 2, 1) subtests = random.sample(tests, ntest) - print("[+] Iteration %s: run %s tests/%s" - % (iteration, len(subtests), len(tests))) + print(f"[+] Iteration {iteration}/{args.max_iter}: " + f"run {len(subtests)} tests/{len(tests)}") print() exitcode = run_tests(args, subtests) @@ -170,10 +175,10 @@ def main(): if len(tests) <= args.max_tests: print("Bisection completed in %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) - sys.exit(1) else: print("Bisection failed after %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) if __name__ == "__main__": diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem index e400e178a1f599..02f2c2e6346ef8 100644 --- a/Lib/test/certdata/allsans.pem +++ b/Lib/test/certdata/allsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAQKCAYAe -BaCCgdJk+xk1USg9cuo5ykBqzTSYlQLXdDlN2oO7sGehJhgvVEGX+QdM3ze+oM2B -wNd3tQDB2iKo11oCunDh4/m2xhq6wA+iPK8POoWRSUf+VJb6xlsTmurENV1s8IHz -GrPqM87OePFGqg/fEuQVuAotObzppVMfNdxHm0er4W6zRMw2rWqDnAOCQ5zDQ1/p -ryp5rYpA49M+R9NoAMlByHRbR7s+6Qnk3NuIMDmUcpF2xeQ/KIMUiHnLEU/gKDpi -bsk+VtyjlibR4zhh9/cJrLTApAIA+4eC176EJvKXCh5UIjd92JC7741HTNQXJpvG -9PXbzhyUCmncr04U+46snGHdwD+lG4LS7oBGACTLMtpcMrlgAm6XCg4T8gRVE/9n -FvCkqPHBR+vnhOxm+0x0yUY/DstJby6IPYPsfGK/s2n//j/vJrAZE1Pxlm9EPU13 -MRLcHstwjAc/NXRPnUN1DfcQvPLx6Tt6rqw3Wm1KO75kM+HZ56BX9/Bi1TgkiI0C -gcEA5JTlXssJ3W8Cz6w1ZtGsThHQBDbvHF2D5AdqO7y6/eqzCQgBQl9BTfXOzsvP -I1gf2CLEFBtGK09UjAuJQg90/NlKur7i7xt7HpAzEfGsDAL4P5BW5JnMNrzpJjjL -0uUDsPJlA75Wi29N2SFiaIslY0sZ6nckInat5GRe4O1AMSHoJ5suY9yTZTU3XB4O -A+XyddutI1GsFZgl8/8LyyNMcyNjxG3T5sr7IKf5/nIv6oMDjC2zLVZa8QS/MEnL -Kaa7AoHBANhEsxfcjw2MaPkrsqAsOP0dDf7g2rdz6wKT5BzZu9e+/E76NmvVDpns -e+kCjql9Os3/wonOMINvn1bTCQGTgk8+dw1fMyqg+zQCvH4ImcE6LSqhzblVHsIB -zZ7rW86trri1U9+olNHG4nwkus0i4LV8eeORns+j8DgXr6/eOvjX3ZW5TyU7/Qgm -SiSdBapzJbom3xJrbo9KQsrN5PVCOwuwrgY0o+2BeKyKhnt4uGv0bR+ii06EOJUA -WvjD7gLI9wKBwGVRXk3jH29IOm3EvjLh80bzfEmx89CV3tUfOEZcRGIyOsNhCfXa -dP7SWqWtDxZyhELwPgtPf43I7wfYQTHH2ioNQqN94ubrPmpwrkJg5cq5MkIyf2F6 -jlsg5xMrD6VeH4G6H25GWuQZJN9+fbkrHBpj+ovD3X9tLWzT1H5Miyx8BAQyM6DN -74Nn0C8Dn2C49vyor5i9JdK4ivIY9ahH8CYE5L73k3p0NFXoPtY61ORUyCjFROtu -oIa+fOQxgVzn6wKBwQC3DD7BnY7/Gq7m51ODOqrpoaPs7Qhyagyp298hhDD3hNEt -T56sWmLHaV/fcqipUDNrlGRmGzz4ooutA2YGDYIn7Gj7ym4WULcN6Jr92e25nLIJ -+XWUvjUQZFJThkXogxz1fZSGI7wCamHcTYJGipTDR54rPV+7w7hY4cN0CZbEdIE6 -buRMUZ/zO+VZZAYdpORz0N7SSlgDtAkgenCmHe64EEzbN8bgCcvHzl/RNfZyeSm7 -supSBJuXkfttvvg/JzUCgcEAlx0Pep9qCLvpk0WqzijBVHc3zK4wYxjhN2MBkF42 -SLWfogKpiPfIqxX6YF94roIA0VlW6Pj50v+sbPwq8nwsgFNhml80A4ODKr3O3Y3M -fXDBJW5W5ZRb/vhIKRjXyCSckSRfj7N8HUYjCLkxQansNWimrldmSet0H2mYJN0Y -JpBXdqpa76zoHzWpKFwD0fSVzvnMelPHSDCNOdIEHmR8e1x2F1/ufR/9/dBzPULY -HMj0OhQHoi8kJyMIj3+bQkbC +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCczEVv5D2UDtn6 +DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj +jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt +07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP +Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 +330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr +GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA +WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK +4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAQKCAYAB +m29nxPNjod5Wm4xydWQYbZj/J0qkcyru/i1qpqyDbGa1sRNcg5A/A/8BPuPcWxhR +/hvwVeD5XX2/i2cnQuv6D3DQP1cSNCxQPanwzknP2k7IVqUmG0RDErPWuoDIhCnR +ljp0NPQsnj0fLhEkcbgG0xwx7KceUDigGsiTbatIvvBHGhQzrmTpqlVVdtMWvGRt +HQEJYuMuIw6IwALHyy3CITv5wh/Bec5OhNoFF8iUZceR4ZkGWf8bYWIa25xlzH6K +4rhOOh1G2ObHHTjhZq4mGXTHY1MEkAxXKWlR3DJc0Lh5E1UETSI6WBHWRb08iwQ5 +AkLOPyMpt08xHFWbJqywvlxenpri+gjY3xbXqGNhyDYWHZqlQmJVnzxoUOuuHi2R +dO86IckUc4Thjbm7a6AilL9t8juNZvyeQUVgtVi25uPkm/cK6r5vo8y4UOUU41EN +NOathlF69gh93il4t6zJW9jPV2WENv1H/vhKUWKW6cabX3vy4rANwy3q4V//GDEC +gcEAuniGCHaEdSjV2sUHyt/yrCLbU6+eLTfNk6AQyXJk6Wrvj6g3gx90ewEq5i/C +ukmSKDslr9pupq8Z/UNfYHZfJfpwEsYvIZ8DdFSd62/h66DhIoEn1v3Lwt+aexgX +yGJHF0BG9JA2CU5Z5NGjlnQYqQBobO9uZMq62l15Ig1MAMHGL0ZYVvOqGZD7XvtC +4UnclK5kjp51Vd5rydEQxyi5qkyLl9Q6T0FQXOphGIOd8ifYeUGe7YC72cFPevdx +wDXZAoHBANdDVvCMrjmzdrS6td39/2nHTeerFPbsOe2LIQYzqjeEe6GWqd2NL9NZ +bk3/cAuVgbWtdvSQQhhmSqOC7JZic4hbZb3lK6v/sr4F/Zu0CfAu80swWFMeS7vq +eQeYzN4w4dKpJArvU3ll7N9AlZhdlYkbPf0WdeOIjZawdAOxNtNe0O+j+5MsXR59 +qkULatumhcKUnqxFCiVHzy21CVJtRzrtu6oGoSdFbmG82eSJ1rPXiuuDnCyzjyMW +iClYRM4NOQKBwERnO/vUxihYT4LOLlqcpl/A9aYQUT0TMGWMHTxYq2343WJceeiu +3ELXHc6NDKjbnjMF54BH57lbmHQQh+dR5PuAkCZC7z0tIM5G0Bty0nRmcs/+gwfZ +2Cpnbjrjjq3iZ2O/H4hNcpUdWdqXkKP7eKReUvBLMLrmp369NVdpe0z3yGTFMFjN +T8PLLHsePt14A+PCyX6L4E0cp3vEJpx4cwtmwvpyTuWN9xXuoKmmdoVDWqS4jr1f +MQnjYO2h4ed5mQKBwGVttWli4DUP+r7tuwP+ynptDqg6VIaEiEcFZ2okre+63QYm +l6NtAzvyx6a41XKf355bPdG+p2YXzNN+vTue6BE3/5iagxloQjCHYhgbnRMvDDRB +c1y2ybihoqWRufZ30fARAoqkehCZliMbq2E/t1YDIBJAowuzLAP04LVcqxitdIV2 +HvQZ00aqr7AY0SDuNdiZbqp9XWpzi4td4iaUlxuNKP/UX9rBPGGROpoU2LWkujB+ +svfdI3TFCSNyE/mDAQKBwQCP++WZKxExrSFRk3W+TcHKHZb2pusfoPWE7WH6EnDW +dkTZpa3PZaf0xgeglmNBv4Paxw2eMPsIhyNv62XY/6GbY6VJWRyx/s+NsazeP4ji +xUOufnwTePjYw6x0pcl6BknZrHn8LCJU741h0yTum8cDdNfRKdc0AMy0gVXk4ZTG +2cAtbEcWb3J+a5kYf6mp5yx3BNwtewkGZhc2VuQ9mQNbMmOOS/pHQQTRWcxsQwyt +GPAhMKawjrL1KFmu7vIqDSw= -----END PRIVATE KEY----- Certificate: Data: @@ -51,38 +51,38 @@ Certificate: Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c1:1a:f8:fe:53:2f:d5:53:24:d1:9a:62:00:d5: - 41:04:35:38:61:d3:ea:56:38:37:2f:bd:49:64:78: - 8a:3e:5d:32:77:16:11:78:fd:15:63:22:60:c7:8e: - 8c:e5:db:c5:df:d9:3c:ca:9e:f4:09:6b:52:82:a6: - a6:bb:7e:bf:1c:2f:d4:0c:c2:4e:29:1f:e4:b8:ba: - 5c:4e:bb:4d:81:97:76:1a:7f:1d:a8:25:55:0c:2f: - 7e:ef:72:22:60:fb:39:33:3f:1d:64:de:d5:c5:8d: - 79:2b:2c:68:d9:c0:ea:2e:7c:10:b7:02:63:ee:ab: - 6c:47:14:1c:c7:ae:fa:79:fc:32:11:1f:6b:25:40: - 53:3d:7e:95:59:cc:de:fb:81:8c:b3:c5:b6:b4:c0: - 27:c2:3e:90:9f:78:91:51:c9:82:96:f1:ce:cc:5e: - bc:27:33:cd:98:b4:4e:d1:96:77:f6:db:b1:24:09: - d3:0d:69:27:99:2d:42:31:79:5a:5c:9d:27:2a:66: - 5d:12:21:b4:77:60:48:95:d0:b3:c5:93:1d:30:24: - 59:bc:a9:41:05:53:f8:7e:d2:36:a6:83:2f:ce:37: - ed:75:9e:a9:8c:96:9d:c1:8c:d8:bf:25:35:6c:6b: - b3:7b:03:77:6f:74:70:bb:55:59:6b:5a:75:20:53: - a3:28:4a:78:b2:2f:a8:a3:a6:e7:32:1e:d6:73:2b: - 69:89:cb:4b:07:47:c3:da:74:72:a8:c3:43:b8:db: - 7f:f9:37:c1:8a:4d:23:af:68:63:17:4e:30:1e:38: - 6b:3e:f7:f3:f5:65:8a:37:22:38:d0:3f:3f:cd:57: - 74:25:84:af:33:46:ac:45:dd:c5:b4:7a:41:c7:91: - 3f:bf:8d:98:c2:bd:22:a6:ea:67:5b:31:0b:a7:28: - 4d:56:f9:da:24:01:cf:35:e6:96:f8:f0:cc:df:d5: - e5:8a:77:fe:d4:c9:47:fb:09:7b:ac:b3:20:1a:27: - 77:25:a5:a2:b5:b1:b6:e7:f6:6d + 00:9c:cc:45:6f:e4:3d:94:0e:d9:fa:0c:c9:99:fe: + e0:96:08:bc:8b:f8:ae:71:4d:9a:7f:e6:3d:dc:c8: + 84:a8:69:3b:94:d4:ab:b7:90:3f:7b:78:b8:29:1e: + 3e:05:82:4e:59:26:2d:b5:ce:c4:2a:7a:90:a3:8c: + 08:41:93:ab:63:e9:16:14:72:73:1b:6f:c3:2e:c5: + 49:10:30:40:d6:fb:92:df:4b:2a:7e:1c:26:9e:b6: + 51:e8:b1:67:61:10:3a:97:e2:4a:49:e5:a8:c1:c0: + d4:6d:d3:b9:b1:40:4c:6d:0b:89:1c:9e:8d:2e:b7: + da:c6:e5:7c:cb:37:6f:57:a4:2a:51:1e:51:45:ae: + 7b:4e:81:00:04:a2:2c:fd:fd:f7:3c:a9:a4:e0:6f: + 7e:d1:80:66:0f:42:cc:61:ec:f7:11:0d:17:b8:64: + 6c:75:97:4e:88:ae:bf:18:fb:65:b9:d3:35:82:7e: + 4c:02:4b:70:da:2c:33:8a:22:34:8e:7c:f1:cd:aa: + 67:dc:7e:c4:e9:ce:b8:fc:df:7d:1c:74:f9:25:ac: + c2:2e:88:1b:fe:88:69:25:0a:39:0f:64:98:84:87: + 1b:d8:da:4e:84:44:ce:07:03:1c:6e:3c:28:af:ac: + 7e:4c:09:e3:0d:62:f0:55:51:c2:eb:19:13:a6:ec: + 94:57:46:59:48:fd:7b:9b:c3:0a:00:68:06:fe:cb: + e7:59:b4:26:7d:02:7c:c7:21:bf:a7:c8:a8:89:c5: + 60:23:ce:34:7e:40:f2:fa:8c:7a:d5:58:15:40:59: + 37:3c:dd:83:f9:16:fc:54:5c:c7:d8:de:c5:6f:f6: + d3:52:09:5d:dc:a5:83:96:e7:e8:97:71:0c:cb:b9: + cc:d7:9c:e2:d1:62:4e:e9:f7:09:1b:d3:f8:64:f4: + ab:8a:e0:13:00:f7:47:81:86:f1:4c:08:58:07:27: + 5f:f1:26:0d:a7:16:60:47:3e:8f:f1:6d:cc:0f:1b: + f9:9a:e5:3f:cc:70:0f:26:02:51 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: - DNS:allsans, othername:, othername:, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 + DNS:allsans, othername: 1.2.3.4::some other identifier, othername: 1.3.6.1.5.2.2::, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 X509v3 Key Usage: critical Digital Signature, Key Encipherment X509v3 Extended Key Usage: @@ -90,59 +90,56 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - D4:F1:D8:23:E0:A7:E9:CA:12:45:A0:0D:03:C2:25:A6:E8:65:BC:EE + 31:5E:C0:5E:2F:47:FF:8B:92:F9:EE:3D:B1:87:D0:53:75:3B:B1:48 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 70:77:d8:82:b0:f4:ab:de:84:ce:88:32:63:5e:23:0f:b6:58: - a2:b1:65:ff:12:22:0b:88:a6:fa:06:40:9a:e7:63:a7:5d:ae: - 94:c5:68:3c:4b:e9:95:34:01:75:24:df:9d:6e:9b:e4:ff:3f: - 61:97:29:7b:ab:34:2c:14:d3:01:d2:eb:fb:84:40:db:12:54: - 7e:7a:44:bc:08:eb:9f:e2:15:0b:11:4f:25:d2:56:51:95:ad: - 6d:ad:07:aa:6a:61:f9:39:d5:82:8c:45:31:9f:2a:ff:18:98: - 49:0c:bb:17:ad:d5:24:d3:d1:c7:c4:10:3e:c4:79:26:58:f4: - c5:de:82:16:c4:c3:c4:a7:a3:62:22:41:90:36:0f:bc:4c:fd: - 6a:18:22:f2:87:e9:07:db:b4:3d:65:00:e4:70:f9:d6:e5:a8: - a1:b9:c9:9d:e7:5d:78:aa:98:d5:f8:f4:fd:5c:d9:4c:d0:6d: - bf:87:71:d3:5b:ec:f4:bf:46:f9:c8:f8:10:c5:72:af:c3:15: - b9:c4:06:67:0b:3f:f6:f4:64:c5:27:74:c1:6b:00:37:da:ea: - 18:36:77:36:a7:3e:80:2e:5d:54:0f:01:df:ce:9e:97:dd:c9: - f2:8b:59:82:c5:65:31:c8:73:20:fd:24:23:25:d8:00:df:90: - 93:26:76:08:0a:06:a9:0e:d3:d3:4c:6f:ef:a7:fb:de:eb:2a: - 40:b9:e4:b1:44:0c:37:ca:c6:9e:44:4a:b4:7c:2c:40:52:35: - bb:b3:71:28:3d:35:fd:be:c9:4f:54:b3:99:c5:5f:84:38:fb: - 2b:fb:ea:dd:88:e8:9d:c1:9b:67:87:3d:79:7b:3d:7e:61:1f: - 70:3c:b7:c8:4c:17:a5:0c:a3:28:c7:ab:48:11:14:f7:98:7a: - da:4e:fb:91:76:89:0a:a6:c6:72:e0:96:d9:f1:80:ea:68:90: - 37:5c:c6:69:c7:d7:bc:c7:d1:ae:5b:a9:12:59:c6:e4:6c:61: - a9:8b:ba:51:b3:13 + Signature Value: + 72:42:a6:fc:ee:3c:21:47:05:33:e8:8c:6b:27:07:4a:ed:e2: + 81:47:96:79:43:ff:0f:ef:5a:06:aa:4c:01:70:5b:21:c4:b7: + 5d:17:29:c8:10:02:c3:08:7b:8c:86:56:9e:e9:7c:6e:a8:b6: + 26:13:9e:1e:1f:93:66:85:67:63:9e:08:fb:55:39:56:82:f5: + be:0c:38:1e:eb:c4:54:b2:a7:7b:18:55:bb:00:87:43:50:50: + bb:e1:29:10:cf:3d:c9:07:c7:d2:5d:b6:45:68:1f:d6:de:00: + 96:3e:29:73:f6:22:70:21:a2:ba:68:28:94:ec:37:bc:a7:00: + 70:58:4e:d1:48:ae:ef:8d:11:a4:6e:10:2f:92:83:07:e2:76: + ac:bf:4f:bb:d6:9f:47:9e:a4:02:03:16:f8:a8:0a:3d:67:17: + 31:44:0e:68:d0:d3:24:d5:e7:bf:67:30:8f:88:97:92:0a:1e: + d7:74:df:7e:7b:4c:c6:d9:c3:84:92:2b:a0:89:11:08:4c:dd: + 32:49:df:36:23:d4:63:56:e4:f1:68:5a:6f:a0:c3:3c:e2:36: + ee:f3:46:60:78:4d:76:a5:5a:4a:61:c6:f8:ae:18:68:c2:8d: + 0e:2f:76:50:bb:be:b9:56:f1:04:5c:ac:ad:d7:d6:a4:1e:45: + 45:52:f4:10:a2:0f:9b:e3:d9:73:17:b6:52:42:a6:5b:c9:e9: + 8d:60:74:68:d0:1f:7a:ce:01:8e:9e:55:cb:cf:64:c1:cc:9a: + 72:aa:b4:5f:b5:55:13:41:10:51:a0:2c:a5:5b:43:12:ca:cc: + b7:c4:ac:f2:6f:72:fd:0d:50:6a:d6:81:c1:91:93:21:fe:de: + 9a:be:e5:3c:2a:98:95:a1:42:f8:f2:5c:75:c6:f1:fd:11:b1: + 22:26:33:5b:43:63:21:06:61:d2:cd:04:f3:30:c6:a8:3f:17: + d3:05:a3:87:45:2e:52:1e:51:88:e3:59:4c:78:51:b0:7b:b4: + 58:d9:27:22:6e:8c -----BEGIN CERTIFICATE----- MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs -bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAaOCAt4w +bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCczEVv5D2UDtn6 +DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj +jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt +07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP +Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 +330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr +GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA +WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK +4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAaOCAt4w ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt @@ -150,21 +147,21 @@ cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW -MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTU -8dgj4KfpyhJFoA0DwiWm6GW87jB9BgNVHSMEdjB0gBSziqCiunHxqCR51KRbJTYV -HknIzaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQx +XsBeL0f/i5L57j2xh9BTdTuxSDB9BgNVHSMEdjB0gBTz7JSO8o4wxI5owr+OahnA +wZ92ZaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y -ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAcHfYgrD0q96EzogyY14j -D7ZYorFl/xIiC4im+gZAmudjp12ulMVoPEvplTQBdSTfnW6b5P8/YZcpe6s0LBTT -AdLr+4RA2xJUfnpEvAjrn+IVCxFPJdJWUZWtba0Hqmph+TnVgoxFMZ8q/xiYSQy7 -F63VJNPRx8QQPsR5Jlj0xd6CFsTDxKejYiJBkDYPvEz9ahgi8ofpB9u0PWUA5HD5 -1uWoobnJneddeKqY1fj0/VzZTNBtv4dx01vs9L9G+cj4EMVyr8MVucQGZws/9vRk -xSd0wWsAN9rqGDZ3Nqc+gC5dVA8B386el93J8otZgsVlMchzIP0kIyXYAN+QkyZ2 -CAoGqQ7T00xv76f73usqQLnksUQMN8rGnkRKtHwsQFI1u7NxKD01/b7JT1SzmcVf -hDj7K/vq3YjoncGbZ4c9eXs9fmEfcDy3yEwXpQyjKMerSBEU95h62k77kXaJCqbG -cuCW2fGA6miQN1zGacfXvMfRrlupElnG5GxhqYu6UbMT +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAckKm/O48IUcFM+iMaycH +Su3igUeWeUP/D+9aBqpMAXBbIcS3XRcpyBACwwh7jIZWnul8bqi2JhOeHh+TZoVn +Y54I+1U5VoL1vgw4HuvEVLKnexhVuwCHQ1BQu+EpEM89yQfH0l22RWgf1t4Alj4p +c/YicCGiumgolOw3vKcAcFhO0Uiu740RpG4QL5KDB+J2rL9Pu9afR56kAgMW+KgK +PWcXMUQOaNDTJNXnv2cwj4iXkgoe13TffntMxtnDhJIroIkRCEzdMknfNiPUY1bk +8Whab6DDPOI27vNGYHhNdqVaSmHG+K4YaMKNDi92ULu+uVbxBFysrdfWpB5FRVL0 +EKIPm+PZcxe2UkKmW8npjWB0aNAfes4Bjp5Vy89kwcyacqq0X7VVE0EQUaAspVtD +EsrMt8Ss8m9y/Q1QataBwZGTIf7emr7lPCqYlaFC+PJcdcbx/RGxIiYzW0NjIQZh +0s0E8zDGqD8X0wWjh0UuUh5RiONZTHhRsHu0WNknIm6M -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 index 941d7919f8033e..aa9dbbe294f829 100644 --- a/Lib/test/certdata/capath/b1930218.0 +++ b/Lib/test/certdata/capath/b1930218.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 index 941d7919f8033e..aa9dbbe294f829 100644 --- a/Lib/test/certdata/capath/ceff1710.0 +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem index cbcac7818ddc67..07a42422af1fd3 100644 --- a/Lib/test/certdata/idnsans.pem +++ b/Lib/test/certdata/idnsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAQKCAYEA -uaYWWwHW6pzxOrnabcVLYX0WunW9LVShbIw97AElI2n/LuhkXh6xkK48BsqP0vaK -oDHJ5VYxgQdmoP03Zs8sX4BSWe7twg1u8wJxkA+cUXI1BAn0opHjpwJlalEEfe2v -s8PwjMrF59nsCq56W42PrDlms5UmuQ5WLsw6Co++hZmfxW7LPu+GIS6qBZfluNT5 -kBpZlDDCtkyteUD4SVI3wvmOSi+Wzv4e7P2wC9kByjENIcfhC5QQURRD4sA1hWCp -2SThYWqJOCEc2SvGgoqgTRaJuQ2aVG9qrntXt0N4V+WdJWXBK0jedkB2flLve1fR -KmDYuc9k/c1svmS3Y+iZohBha9H8jpuJmXYBxxg1iNg9m7qkfg8F8wxCYLQKB+U6 -tjRS7by+jSE08On7mpDDhJORnlh+rfEuWPPwAKQpLpdp76KDTvR++GvfOMUiOrFM -e9s5aXp+vcgkSSqYvigE+sFpCjQWwkGBkMdT16Pf9CzhQaM08YuLnzfLEYgLFw6R -AoHBAN5NQINBmlq/cptGSru66kfecqHfI7xHnnGWKAkto/B1x7Crrgs4Tk5b4vaA -JmAqatt5P1e7zco7uAXXebY5VURuH/30TlkuaB+oGFp0OMw6165n8RVPT2ZaDViK -ssJ9LT8fJ+23TWCCT2Z1zUlM/NnHAMjKOVsJK3/KEkVvlc7ROC7uVooc78AsQehg -zpL3GBYEeBukT8aNUMqUlesCsIs/dQHW7DzQL2xGkQagm5/PDsxaCsT7ynA8eL3X -TW+IXwKBwQDZTV3TaG6wqtL8y2DR0lN5jY/eYayX4e18iZ+XEZVTntPdVVyJIE4d -0A5ZfcILb9WE8R21iptROYSjcH/05j+3fQMJ1WAK0sNfGTUNNT3jYU8YzLvos+wW -G8E+mNMpFPWNvLV5Qrl4VvoifGh8AMvplUEz8uAzGJbXbRxUPcmjth2ph8zULEDn -/+o4OcT3gh1bp+HCqch0OuiJRn9qNUpsJG5GMm5FtjBjZM97ucZ1/0DaWl3JUxUN -/pueo3J9vCUCgcBg2Fjdlcvv8u2z1aijJmgATVm1SWfhE3ZkV50zem2sSTNotTJK -cwoyOveimeueA3ywBp9g0lFx5Bhkex3sFAggmrVXRoKHeZ8lA28woOdJmezybxfp -R7b4iQy9YRdFgZEfqawUdMHB5KNAqNt5LpANNBQUZX0dOt53eooBM/6Yri8CyxRq -cPbFysIfwWTdQ8Z7eRD2Qdv7TP9AcgDp9C8DSu7nkUEzsSKn0gpGT9vcgDEbN7Lv -ZB4qTT3wvoZeq5MCgcBIG18eDtJkN1sp3Yb0OTnP5QSvg3PVNngq0jQt2fzWMacW -FARP0HN7exW35n4kc2jD44q7OhJOAqsb3PHo3xqXlZkTg0WKceO4w9GR32/46spn -bVCRaFrX/z/BuM6hHD5bWRpS8aw/3YTFOsklFNKVYRyw01BIREmRlLhIz/QAKidv -oQt8AG9NTON44tqUUw3Q40WL5fEJeJ6/JrCTGrnmZrRdANEMuucVpFchNEVB1IC9 -tCzY6IPdD/atzojoZi0CgcB2x9oWLjJ0XJIp2pMAb8nCMVjkKrznKFjZbDm8EQBs -ou7pM2zkO3VRcWT1BXQocinJsjQqjQiTawP6IN2FQgT0d89V+pwd+jdvpdildQhP -1/6SErVRZV//oopKTsC6TIBL/EmW1TkP3ulQIZs8YklFgybeHdDyNFi+VgPXkVGe -IHp0nEzrui9q0YJsjHfFHBeGyzDSfbiBYiF7Auk66gYZbXufebP/LZNG/FIamPP3 -rwYIeeV1IVwk9tPBw6fGwrs= +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQCp6zt40WB3K7yj +BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 +nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd +1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc +LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO +jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV +WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj +nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx +MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAQKCAYAz +9ZdHkDsf5fN2pAznXfOOOOz8+2hMjmwkn42GAp1gdWr+Z5GFiyaC8oTTSp6N1AnZ +iqCk8jcrHYMFi1JIOG8TzFjWBcGsinxsmp4vGDmvq2Ddcw5IiD2+rHJsdKZAOBP9 +snpD9cTE3zQYAu0XbE617krrxRqoSBO/1SExRjoIgzPCgFGyarBQl/DGjC/3Tku2 +y6oL4qxFqdTMD9QTzUuycUJlz5xu2+gaaaQ3hcMUe2xnZq28Qz3FKpf2ivZmZqWf +4+AIe0lRosmFoLAFjIyyuGCkWZ2t9KDIZV0OOS4+DvVOC/Um9r4VojeikripCGKY +2FzkkuQP3jz6pJ1UxCDg7YXZdR2IbcS18F1OYmLViU8oLDR6T01s0Npmp39dDazf +A4U+WyV3o1ydiSpwAiN8pJadmr5BHrCSmawV8ztW5yholufrO+FR5ze2+QG99txm +6l7lUI8Nz01lYG6D10MjaQ9INk2WSjBPVNbfsTl73/hR76/319ctfOINRMHilJ0C +gcEAvFgTdc5Qf9E7xEZE5UtpSTPvZ24DxQ7hmGW3pTs6+Nw4ccknnW0lLkWvY4Sf +gXl4TyTTUhe6PAN3UHzbjN2vdsTmAVUlnkNH40ymF8FgRGFNDuvuCZvf5ZWNddSl +6Vu/e5TFPDAqS8rGnl23DgXhlT+3Y0/mrftWoigdOxgximLAsmmqp/tANGi9jqE1 +3L0BN+xKqMMKSXkMtuzJHuini8MaWQgQcW/4czh4ArdesMzuVrstOD8947XHreY9 +pesVAoHBAOb0y/AFEdR+mhk/cfqjTzsNU2sS9yHlzMVgecF8BA26xcfAwg4d47VS ++LK8fV6KC4hHk4uQWjQzCG2PYXBbFT52xeJ3EC8DwWxJP09b4HV/1mWxXl5htjnr +dfyTmXKvEe5ZBpKGWc8i7s7jBi7R5EpgIfc586iNRyjYAk60dyG0iP13SurRvXBg +ID25VR4wABl3HQ3Hhv61dqC9FPrdHZQJdysfUqNrAFniWsSR2eyG5i4S1uHa3G+i +MzBTOuBRlwKBwBNXUBhG6YlWqTaMqMKLLfKwfKM4bvargost1uAG5xVrN/inWYQX +EzxfN5WWpvKa0Ln/5BuICD3ldTk0uS8MDNq7eYslfUl1S0qSMnQ6DXK4MzuXCsi9 +0w42f2JcRfVi0JUWP/LgV1eVKTRWF1g/Tl0PP/vY1q2DI/BfAjFxWJUHcxZfN4Es +kflP0Dd3YpqaZieiAkC2VrYY0i9uvXCJH7uAe5Is+9NKVk8uu1Q8FGM/iDIr4obm +J6rcnfbDsAz7yQKBwGtIbW9qO3UU9ioiQaTmtYg90XEclzXk1HEfNo+9NvjVuMfo +b3w1QDBbgXEtg6MlxuOgNBaRkIVM625ROzcA6GZir9tZ6Wede/z8LW+Ew0hxgLsu +YCLBiu9uxBj2y0HttwubySTJSfChToNGC/o1v7EY5M492kSCk/qSFMhQpkI+5Z+w +CVn44eHQlUl2zOY/79vka9eZxsiMrLVP/+3kRrgciYG7hByrOLeIIRfMlIl9xHDE +iZLSorEsjFC3aNMIswKBwFELC2fvlziW9rECQcGXnvc1DPmZcxm1ATFZ93FpKleF +TjLIWSdst0PmO8CSIuEZ2ZXPoK9CMJyQG+kt3k7IgZ1xKXg9y6ThwbznurXp1jaW +NjEnYtFMBK9Ur3oaAsrG2XwZ2PMvnI/Yp8tciGvjJlzSM8gHJ9BL8Yf+3gIJi/0D +KtaF9ha9J/SDDZdEiLIQ4LvSqYmlUgsCgiUvY3SVwCh8xDfBWD1hKw9vUiZu5cnJ +81hAHFgeD4f+C8fLols/sA== -----END PRIVATE KEY----- Certificate: Data: @@ -51,34 +51,34 @@ Certificate: Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:bc:b2:aa:65:4e:e1:ee:2e:36:d6:4c:be:52:2f: - 60:f5:7f:0f:54:e9:64:33:2d:50:05:e6:0b:32:85: - de:65:af:61:69:7d:8c:23:ae:86:f9:31:ab:fd:d5: - a3:8b:6c:94:53:be:83:72:5e:c2:76:e8:53:d2:d1: - 1c:9c:44:65:db:a2:67:08:10:9c:57:1b:2f:5a:23: - ed:0c:2d:80:6d:d4:6e:66:82:a0:87:f8:77:15:0b: - 03:f3:08:d9:61:56:74:f0:be:98:00:ef:2f:33:b3: - 7f:ba:7b:36:40:3d:69:05:d3:25:8e:31:82:ed:4d: - ca:bd:03:91:96:79:ab:ed:e5:53:20:9b:52:99:17: - 78:0e:e2:4e:7c:a0:fc:a4:dc:07:bd:0f:42:c0:69: - 8b:17:e8:31:62:05:8c:78:d5:e5:11:e5:46:d2:6f: - 92:18:5d:a0:dd:f1:de:a3:a9:6f:e3:9d:88:60:73: - a0:b1:92:fd:60:4c:91:67:f3:b5:79:96:b1:b4:bd: - 83:12:45:4b:56:0d:0f:58:26:1f:c4:28:0c:63:1b: - 0e:c0:79:8e:36:f6:9e:93:13:85:28:26:10:e6:a0: - 56:11:d8:d2:ef:6b:08:4d:22:99:71:a2:5f:ef:d1: - fb:34:bd:e1:50:8c:8f:d4:b1:30:fc:da:d4:5f:9d: - 82:f8:21:7f:2c:ce:12:ec:13:9f:f9:22:af:1a:88: - b1:e3:55:b2:0c:c2:60:d8:01:ad:0f:eb:70:29:da: - 47:f5:6e:24:a7:f6:6a:43:2f:c9:50:6b:34:a8:ca: - bf:31:cc:8a:b6:41:2e:47:32:f1:9b:78:c0:26:4b: - 48:a1:d7:46:71:f3:8b:95:9a:45:a5:6a:f8:2f:b5: - 27:e5:c3:c2:bf:65:74:fd:73:bd:2b:66:9f:d3:74: - 11:98:f7:97:0e:16:c6:e0:e5:4f:f6:d0:cf:cb:96: - 98:ac:f6:d7:01:09:aa:15:69:84:85:ba:96:ad:ac: - ff:a9:f3:2d:7d:a8:fd:a7:79:bb + 00:a9:eb:3b:78:d1:60:77:2b:bc:a3:04:6b:a0:9d: + 1b:aa:23:70:29:7e:d4:e1:65:60:c8:a6:f5:b4:71: + 59:8d:d2:7a:90:c6:c3:ba:08:29:d2:e1:d4:bb:49: + 88:46:16:22:2a:5a:33:55:73:65:86:d2:0b:fb:9c: + 2e:43:16:0b:aa:6e:77:4d:66:4f:7f:4a:35:fc:10: + ec:4a:cf:80:0d:8d:de:b5:5a:74:e4:0c:ee:53:6e: + 28:b2:77:e4:ee:10:1c:61:99:b1:05:eb:d6:c5:f5: + db:5d:d5:28:28:f2:39:8c:ee:3f:9a:6b:a0:a5:62: + 1f:f8:f5:bc:7e:6e:52:43:17:be:84:19:2a:e6:07: + 5a:be:a1:b3:1e:65:44:61:3d:73:41:7a:88:aa:29: + 1f:37:c7:91:dc:2d:32:58:32:87:2f:6f:a9:bc:b8: + 00:70:b9:29:1a:d7:71:9b:dd:01:ae:d3:aa:79:95: + d2:ba:4c:32:b7:f0:7b:28:3b:2e:8c:14:ee:45:d4: + fa:e4:6a:41:bf:78:11:ce:8d:c0:4e:18:f7:cd:99: + 3c:16:3a:ab:f0:95:c4:16:ca:4a:f8:42:ca:56:4b: + ec:93:cd:d9:20:67:e9:b1:6a:bf:f2:42:ea:bb:c2: + e6:12:35:a1:1f:18:7e:72:a1:8b:55:58:88:0a:c3: + 06:51:0c:fa:a9:49:0b:95:7b:e2:7e:31:62:cf:43: + a4:53:33:ea:d7:20:75:73:1e:d3:e4:d5:ff:03:ae: + 03:7e:85:27:a9:27:2f:ec:54:10:44:40:2c:e3:9f: + 20:7e:b1:c9:fb:e8:44:d6:72:1d:e2:c9:5a:0c:8f: + 7a:15:38:6b:34:6f:85:d3:0c:c9:94:c3:f0:e6:12: + de:53:0d:a8:7b:4e:a1:37:1b:e7:e9:04:86:e0:31: + 28:f1:33:35:10:66:5d:bb:8b:39:8c:f6:74:02:25: + 10:28:b0:da:0d:c1:7a:a5:33:be:78:2b:d5:9c:63: + a8:13:f2:49:1c:8f:50:9f:4e:63 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,80 +90,77 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 5C:BE:18:7F:7B:3F:CE:99:66:80:79:53:4B:DD:33:1B:42:A5:7E:00 + 5B:93:42:58:B0:B4:18:CC:41:4C:15:EB:42:33:66:77:4C:71:2F:42 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 5d:7a:f8:81:e0:a7:c1:3f:39:eb:d3:52:2c:e1:cb:4d:29:b3: - 77:18:17:18:9e:12:fc:11:cc:3c:49:cb:6b:f4:4d:6c:b8:d2: - f4:e9:37:f8:6b:ed:f5:d7:f1:eb:5a:41:04:c7:f3:8c:da:e1: - 05:8e:ae:58:71:d9:01:8a:32:46:b2:dd:95:46:e1:ce:82:04: - fa:0b:1c:29:75:07:85:ce:cd:59:d4:cc:f3:56:b3:72:4d:cb: - 90:0f:ce:02:21:ce:5d:17:84:96:7f:6a:00:57:42:b7:24:5b: - 07:25:1e:77:a8:9d:da:41:09:8e:29:79:b4:b0:a1:45:c8:70: - ae:2c:86:24:ae:3d:9a:74:a7:04:78:d6:1f:1b:17:c5:c1:6d: - b1:1a:fd:f4:50:2e:61:16:84:89:d0:42:3f:b6:bf:bd:52:bd: - c8:3e:8e:87:b4:f0:bd:ad:c7:51:65:2f:77:e8:69:79:0e:03: - 63:89:e7:70:ad:c8:d1:2f:1a:a5:06:d2:90:db:7c:07:35:9a: - 0b:0e:85:87:d1:70:17:a7:88:0f:c6:b5:9c:88:00:fa:f9:b2: - 0a:19:5a:4b:8d:91:12:51:5e:0e:c1:d8:9e:02:78:d0:2d:24: - 09:fe:d4:97:3c:cb:a0:1f:9a:ab:f7:0f:e2:fa:64:23:4e:53: - 0a:15:3e:f5:04:01:86:29:8b:8e:24:40:2f:b1:90:87:5c:3b: - 7b:a7:4c:06:af:c3:90:7f:e9:c6:56:42:61:15:2c:83:f1:7c: - 4f:89:17:f3:a0:11:34:3f:8d:af:75:34:60:1e:e0:f2:f3:02: - e7:aa:b3:f7:9f:1c:f8:69:f4:fe:da:57:6e:1b:95:53:70:cd: - ed:b6:bb:2a:84:eb:ab:c3:a9:b4:d5:15:a0:b2:cc:81:2d:f1: - 56:c1:54:9b:5f:14:4c:5f:ad:5f:f5:06:ee:22:60:45:e4:50: - 35:64:ac:ac:ca:4a:bf:86:78:f8:53:2d:17:d8:e8:84:c8:07: - a4:c2:29:76:c7:1f + Signature Value: + 5f:d8:9b:dc:22:55:80:47:e1:9b:04:3e:46:53:9b:e5:a7:4a: + 8f:eb:53:01:39:d5:04:f6:cf:dc:48:84:8a:a9:c3:a5:35:22: + 2f:ab:74:77:ec:a6:fd:b1:e6:e6:74:82:38:54:0b:27:36:e6: + ec:3d:fe:92:1a:b2:7a:35:0d:a3:e5:7c:ff:e5:5b:1a:28:4b: + 29:1f:99:1b:3e:11:e9:e2:e0:d7:da:06:4f:e3:7b:8c:ad:30: + f4:39:24:e8:ad:2a:0e:71:74:ab:ed:62:e9:9f:85:7e:6a:b0: + bb:53:b4:d7:6b:b8:da:54:15:5c:9a:41:cf:61:f1:ab:67:d6: + 27:5c:0c:a3:d7:41:e7:27:3e:58:89:d6:1f:3f:2a:52:cc:13: + 0b:4b:e6:d6:ba:a0:c7:fd:e3:17:a4:b8:da:cc:cb:88:70:21: + 3b:70:df:09:40:6c:e7:02:81:08:80:b0:36:77:fb:44:c5:cf: + bf:19:54:7c:d1:4e:1f:a2:44:9e:d8:56:0e:bf:4b:0b:e0:84: + 6f:bc:f6:c6:7f:35:7a:17:ca:83:b3:82:c6:4e:d3:f3:d8:30: + 05:fd:6d:3c:8a:ab:63:55:6f:c5:18:ba:66:fe:e2:35:04:2b: + ae:76:34:f0:56:18:e8:54:db:83:b2:1b:93:0a:25:81:81:f0: + 25:ca:0a:95:be:8e:2f:05:3f:6c:e7:de:d1:7c:b8:a3:71:7c: + 6f:8a:05:c3:69:eb:6f:e6:76:8c:11:e1:59:0b:12:53:07:42: + 84:e8:89:ee:ab:7d:28:81:48:e8:79:d5:cf:a2:05:a4:fd:72: + 2c:7d:b4:1c:08:90:4e:0d:10:05:d1:9a:c0:69:4c:0a:14:39: + 17:fb:4d:5b:f6:42:bb:46:27:23:0f:5e:57:5b:b8:ae:9b:a3: + 0e:23:59:41:63:41:a4:f1:69:df:b3:a3:5c:10:d5:63:30:74: + a8:3c:0c:8e:1c:6b:10:e1:13:27:02:26:9b:fd:88:93:7e:91: + 9c:f9:c2:07:27:a4 -----BEGIN CERTIFICATE----- MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk -bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAaOCAo4w +bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCp6zt40WB3K7yj +BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 +nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd +1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc +LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO +jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV +WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj +nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx +MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAaOCAo4w ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB /wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ -BAIwADAdBgNVHQ4EFgQUXL4Yf3s/zplmgHlTS90zG0KlfgAwfQYDVR0jBHYwdIAU -s4qgorpx8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK +BAIwADAdBgNVHQ4EFgQUW5NCWLC0GMxBTBXrQjNmd0xxL0IwfQYDVR0jBHYwdIAU +8+yUjvKOMMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB -AF16+IHgp8E/OevTUizhy00ps3cYFxieEvwRzDxJy2v0TWy40vTpN/hr7fXX8eta -QQTH84za4QWOrlhx2QGKMkay3ZVG4c6CBPoLHCl1B4XOzVnUzPNWs3JNy5APzgIh -zl0XhJZ/agBXQrckWwclHneondpBCY4pebSwoUXIcK4shiSuPZp0pwR41h8bF8XB -bbEa/fRQLmEWhInQQj+2v71Svcg+joe08L2tx1FlL3foaXkOA2OJ53CtyNEvGqUG -0pDbfAc1mgsOhYfRcBeniA/GtZyIAPr5sgoZWkuNkRJRXg7B2J4CeNAtJAn+1Jc8 -y6Afmqv3D+L6ZCNOUwoVPvUEAYYpi44kQC+xkIdcO3unTAavw5B/6cZWQmEVLIPx -fE+JF/OgETQ/ja91NGAe4PLzAueqs/efHPhp9P7aV24blVNwze22uyqE66vDqbTV -FaCyzIEt8VbBVJtfFExfrV/1Bu4iYEXkUDVkrKzKSr+GePhTLRfY6ITIB6TCKXbH -Hw== +AF/Ym9wiVYBH4ZsEPkZTm+WnSo/rUwE51QT2z9xIhIqpw6U1Ii+rdHfspv2x5uZ0 +gjhUCyc25uw9/pIasno1DaPlfP/lWxooSykfmRs+Eeni4NfaBk/je4ytMPQ5JOit +Kg5xdKvtYumfhX5qsLtTtNdruNpUFVyaQc9h8atn1idcDKPXQecnPliJ1h8/KlLM +EwtL5ta6oMf94xekuNrMy4hwITtw3wlAbOcCgQiAsDZ3+0TFz78ZVHzRTh+iRJ7Y +Vg6/SwvghG+89sZ/NXoXyoOzgsZO0/PYMAX9bTyKq2NVb8UYumb+4jUEK652NPBW +GOhU24OyG5MKJYGB8CXKCpW+ji8FP2zn3tF8uKNxfG+KBcNp62/mdowR4VkLElMH +QoToie6rfSiBSOh51c+iBaT9cix9tBwIkE4NEAXRmsBpTAoUORf7TVv2QrtGJyMP +XldbuK6bow4jWUFjQaTxad+zo1wQ1WMwdKg8DI4caxDhEycCJpv9iJN+kZz5wgcn +pA== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem index c330c36d8f9fde..187021b8eeb9fa 100644 --- a/Lib/test/certdata/keycert.passwd.pem +++ b/Lib/test/certdata/keycert.passwd.pem @@ -1,69 +1,69 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH -ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M -mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2 -hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ -+wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV -jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC -rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX -I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K -PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe -tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c -4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ -t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2 -d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r -ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4 -CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc -e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN -ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC -wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5 -Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK -wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H -3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88 -GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu -pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz -g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8 -darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I -qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd -gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594 -nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q -G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI -tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl -ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4 -5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt -uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8 -hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI -ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm -9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI -IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx -PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p -j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5 +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIc17oH9riZswCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDwi0Mkj59S0hplpnDSNHwPBIIH +EFGdZuO4Cwzg0bspLhE1UpBN5cBq1rKbf4PyVtCczIqJt3KjO3H5I4KdQd9zihkN +A1qzMiqVZOnQZw1eWFXMdyWuCgvNe1S/PRLWY3iZfnuZ9gZXQvyMEHy4JU7pe2Ib +GNm9mzadzJtGv0YZ05Kkza20zRlOxC/cgaNUV6TPeTSwW9CR2bylxw0lTFKBph+o +uFGcAzhqQuw9vsURYJf1f1iE7bQsnWU2kKmb9cx6kaUXiGJpkUMUraBL/rShoHa0 +eet6saiFnK3XGMCIK0mhS9s92CIQV5H9oQQPo/7s6MOoUHjC/gFoWBXoIDOcN9aR +ngybosCLtofY2m14WcHXvu4NJnfnKStx73K3dy3ZLr2iyjnsqGD1OhqGEWOVG/ho +QiZEhZ+9sOnqWI2OuMhMoQJNvrLj7AY4QbdkahdjNvLjDAQSuMI2uSUDFDNfkQdy +hqF/iiEM28PmSHCapgCpzR4+VfEfXBoyBCqs973asa9qhrorfnBVxXnvsqmKNLGH +dymtEPei9scpoftE5T9TPqQj46446bXk23Xpg8QIFa8InQC2Y+yZqqlqvzCAbN6S +Qcq1DcTSAMnbmBXVu9hPmJYIYOlBMHL8JGbsGrkVOhLiiIou4w3G+DyAvIwPj6j9 +BHLqa7HgUnUEC+zL4azVHOSMqmDsOiF3w9fkBWNSkOyNoZpe+gBjbxq7sp+GjAJv +1CemRC3LSoNzLcjRG2IEGs1jlEHSSfijvwlE4lEy3JVc+QK8BOkKXXDVhY1SQHcS +pniEnj95RFVmAujdFDBoUgySyxK/y6Ju/tHPpSTG9VMNbJTKTdBWAVWWHVJzBFhR +0Ug62VrBK7fmfUdH1b37aIxqsPND2De6WLm0RX+7r3XPDJ7hm+baKCchI5CvnG19 +ky8InhMbU4qV+9LceMETmNKKDkhKl4Zx/Y3nab7DG9s/RZfrTdCHojc9Va/t0Ykp +qlVrvdj/893CdI78SW3VjWBJGWfKMyT16hBMY3TPz6ulbFXk6Pul/KcLLWslghS+ +GKZjyBe96UwfH4C7WjuIB+zo+De3Wr8xOCdJR5zwEutBMM+L/Wul8B6wIEGS71kB +TN/CAoeIgHLQFbcw4YE80dllTnSEsqF+ahVTTcCt3iLUaOgeTUxteMbXY9+nekSX +x8aUcvkMhbU9omdEowFr5/HIMKXo4UXat4fIGgh2pG8v8fA46hZXkhWUh/PhbnQw +StXzn4fA13erqVI679kHMmOIQebv4oqdcwkImrH5fEsACNjQbkYZF5fD4z+1GHkA +e2eGqejVT+OV14I8qfx9oqs2f8aqijH8fYLU0TymE7p53DYZy4WvDwk22I4rMzoQ +sGkOZwfKUYpdBI2t6tEf1ROBjoNG0E2Onq+5iooibN08rKXKAQMWsK+2vNHNHwBW +49vRheQNnRqSuLY+b7QAjA0KuRWo9YptCbnXyF/Aw64jMfAGjggDLoaZfALGZk3n +P+ZoL9xc7rYRpIca44BeYI6AhHFcWWIOX7Sm69FvmyHlfsgTAXVgY1lQPuGy68Au +PHSkgUyydDtkrfb2W2gJuqD/+h+9X2z+o/+nETYPCZm3sH5xvTY/DTcTx9kTpXxx +YQBaFTt12eVX7wZVr5K3u9M371rg+SeXC2SzL4T6APHD52cxbA1jgM0JFh3KJTuk +fADxIzM1NdzYQ45J6i2w+/Fh4VPnXZ0oiUSwE094XTBlvhI6zHgar2Q0Qx1P51vB +odd9XzyDLULuIzei0DYjTIg0KhE+wAGq1I5qtiMhmy5TdCKKNA9WGb1Pq38zpyjU +wGmztzSzCEjfLyhChaUObVRRxEfD5ioxKer/fczOhKQe8FXmGy5u/04tVmmEyNOO +JkkDtZy+UbKuJ257QnY72wPjgtHNy+S4Iv7zHUbNJNhxk+xBlRcmRNWCEM20LBSO +Tj4S9gyan+gH2+WFxy8FaENUhM+vHFEeJcjQIBFBeWICmSmdkh/r0YK1UVJ9NLfR +l0HiKm3lKg+kNCexTAPLMt2rGZ4PAKVnhVaxtuHMYYDpl2GYmyH73B9BfcPdA/Zx +GUBmd9hwcLz9SuUg+fjHcogZRRRlcZlKhw3zUCsqHSCQXZCQm7mBlG/5C/7cM7wQ +IRtsNospLStOg51gv21ClQ+uWx30XEcwmnIfVoLl1vMaguuf1u5u3dWBD/UgmqiP +1Ym8jv0BF/AS+u/CtUpwe7ZWxFT0vbyi10xxIF7O07fwFa+5dME3ycZwcyiE95K1 +ftcHlGOIhuVBMSNZXC4I9LM+7IWy+hanUcK+v5RvwBDSJV3fnAOdfrka1L/HyEEb +x/FYKEiU/TAjXDw2NtZ2itpADTSG5KbdJSwPr01Ak7aE+QYe7TIKJhBDZXGQlqq8 +1wv77zyv7V5Xq2cxSEKgSqzB9fhYZCASe8+HWlV2T+Sd -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== -----END CERTIFICATE----- - diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem index 0d398633739a51..a30d15ca4d61a6 100644 --- a/Lib/test/certdata/keycert.pem +++ b/Lib/test/certdata/keycert.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 +qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M +V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz +hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td +ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID +eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb +SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV +uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ +HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO +M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP +MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ +tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP +qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq +D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph +7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq +U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG +ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC +gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ +u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG +XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s +5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo +dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH +Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 +UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr +r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 +XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 +1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx +tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz +og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX +v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z +SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 +uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 ++yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE +5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV +AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o +VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F +G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R +kNdcwhtZZ3dpChUdf+w/LIc= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem index e59d45439d4b6d..c7c4aa74583c8a 100644 --- a/Lib/test/certdata/keycert2.pem +++ b/Lib/test/certdata/keycert2.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCf8FWxi4oVlDVx -e8NDFgb+IYAGr/hZWuY1Zq7d7g57yPoxJrgt+bN89+U7qTduqyB2Hy8G0TqeACOr -IdpPZ8P7V5E5YiASwfJ72nbVo7qR9DAKA5FE8PU0bJFmFLjDDihc970zc4ilRDfR -WylUpj68nefOY4CzFzeiqVOLX2wezs7Z0hflkSXGBmC0j1FbQU2I3YJg3CKCabhT -tU6OyKItzjJ2vVaOoQ+B0Kv8leaRQ6ANZBAFQF2LepSy5F2+oSD+QHjPr+012V5D -mrsdIc9We8YyonS1u/3HI7lLohf3W+qFroQWjn0DJI56ScV1uEr/B0+hn2jBRTM5 -d1F9BeVWm1u8BOJu50CvOeuxiVLsxJpa4T41DJznJk5V+hE4hKvDKmlrwulsRp8o -jUEyUi8dzWOBRfAijIWv3qAPjGA/J33n6+PllCczC2BsVZhVmLqSMCwp1g2JTCM/ -KC7T4vOl/EGkm76fcmLeA1Ef8oUdRg+3T77VP+HqZ2JP06J8O8MCAwEAAQKCAYAw -YvJZ82BEJQGCIrIxMpHNAm+MFmKpDdIFp9oRdDrXgjcG9bLU3e1KSmkEgq4tggIh -GlAM3PHB6ULhPC2ixj7JZHWgCaqwYhKtG6vF+HGyRFDgRrIFTGyyfoICgxReloLp -lV2dGj/l19yXLuAzJtRmFdOSYhIGnGiNgnKvAKBiNajoxyHJpv7piPZqyc0QMZJ2 -bKVMDm02TSuhz4FDuzktaGtl9uQf5GQfnvTZRrRpkC70vigGnrFuSBiCgopF6NLq -6AXl8YS3Jcu2oGWrZDfS/GlG1QmvGGsmr9wndJSGG43jcpcRZt0g1nJNu4Fioq3e -7y6Gap9TEsciuQOv/6RD457XkNARmTQxFpEwmSgOPQn2pFcDspo71Ej7azzL/Z+3 -jvnVo3wxgxBcrpyh+vhBtJARp4pT4anW4PcD6IcPSOWbnI8Ldoj1XN5QkJcBcykK -6LmsAUqsmEQDNsmnGZWyYSCns4P2vUJi0hwQz8UiQwgAta3xnq4v5On7l3cq35kC -gcEA0+joOFbZBeGlCb27tDW4VCW0cQuczzuNEoBUKnsNSqy0nx1O7hgHm/f/NQDD -cpxiD15bRQ0KM9QbQC4dGaVoLsM07hUGk97dCxQPs2zot4CodCKGohs7E154tEDP -zVg3YS5mubUmqdqtn8ZCKeeZye/Tv2ageyF300sEgj2Cd7EZ8S4sB0PxZ2tqT3jy -cBL5cDruLEWuHIQjN7WwSjxnXocpb1OU7dJ+v4zFPCkSCOoa0DTTw4jFhPEOBdqV -T619AoHBAME3QyW4QVtU2Ct9u0B1XThhqSEyOpUrcH9nOoefggwP4WF3phVx16BG -aDKUIGQ62klRa5fi2eooxcjQRLv1sWO0UzssnO6ABMnGkUiRdrowo6xukNak0RTp -0gvNoJ0SZxGF0yWSCw1Rq3qP2Koj7XDumFChAzLMyUsnoOl29SA7GfXcZp1pZTiq -kOfFMWt0CIHu/EK03YWcd4vfQEq6lus39RCSXuL++Jva3yiEl5s069RFZvP1bNrD -emkfetDSPwKBwQClk+8fVnzs44sZOW9ZOEB3P57mVbSJGHb6Zdtd9hhEqP3Y9gWe -dJg9fmGjAJ23CAp3B7s5ER9PsAQ6+c0zJNNq9ox9G2CwWgtNhLdf81FDUPxPAktA -jxZx4/dcoOe+A5gCD0elA67aOUxA86DvLVA1QXeqrn3muBfwuUUknvs6mt8yXGl6 -o9QUgxHmVxLYD3tn/iPr4+ZP0c/Sz9yXpOsAKYxuuFg+G6N9+HiEsXKuFH4vAZgV -yODNJ61VVZ4lS+ECgcAqFqOl39E81+qO7sCPdgFsermg5ZQlUmUbG52AVZq6jesG -lE21disGWs/v1JyJuNg8CGRrnZriiycqa1PNreOKWImY5kr5GSHx4jNbn3RBcr70 -nNEoMJbq+1QqBgzqqkuRYZlxIbMOn6++7v6/cTwT0aWUSr6rnjhrCqLeuG8FKlqp -V+1ydLb79QvDsQzm30vLIggJb+ShakgQS/1xSdv+OR5FEd1hjTESokbiSJ/Ny2Vj -xAp9MgUYUmSj6ZuTSXkCgcAggshdRQLom/EK2pYwffIpKfBiyLbi+KIjKxkiPEsb -jrrQbvh9ZN6iAG3StVAYB5c6vewfeIlcDT0YJDyy1hGRLRG7vf9ubPf+n7Xp1y0W -oo9L9qfCHu0jmWwtinkFYjpTDkXlxXCG2v3TllNsNX/5afYo8sb9oxXHLTpBlwZB -fw6IgNZblWQevdgmUMTP9W2W7AZUxEz4gOM6lQkOwC3U59Dx2yO6rD3An6G1tlZF -2MClyf8o5d5ePObH8rkxrpY= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCyAUXjczgUEn7m +mOwDMi/++wDRxqJAJ2f7F9ADxTuOm+EtdpfYr4mBn8Uz9e3I+ZheG5y3QZ1ddBYA +9YTfcUL0on8UXLOOBVZCetxsQXoSAuDMPV0IXeEgtZZDXe7STqKSQeYk7Cz+VtHe +lZ8j7oOOcx5sJgpbaD+OGJnPoAdB8l8nQfxqAG45sW4P6gfLKoJLviKctDe5pvgi +JC8tvytg/IhESKeefLZ4ix2dNjj2GNUaL+khU6UEuM1kJHcPVjPoYc+y8fop/qhQ +0ithBhO2OvJ+YmOFdCE67SyCwU3p8zJpN+XkwbHttgmNg4OSs7H6V7E52/CsTNTA +SthBHXtxqaM+vjbGARrz2Fpc/n+LwRt7MGIR0gVtntTgnP0HoeHskhAIeDtaPrZ6 +zHdl3aDwgAecVebTEBT5YPboz+X1lWdOrRD2JW3bqXSRIN3E4qz5IMuNx3VvhpSR +eFZzR6QIbxQqzO/Vp93Ivy8hPZ6WMgfSYWs7CGtu4NP79PJfdMsCAwEAAQKCAYAc +e3yp2NlbyNvaXRTCrCim5ZXrexuiJUwLjvNfbxNJDeM5iZThfLEFd0GwP0U1l86M +HGH2pr6d4gHVVHPW5wIeL9Qit3SZoHv9djhH8DAuqpw6wgTdXlw0BipNjD23FBMK +URYYyVuntM+vDITi1Hrjc8Ml7e5RUvx8aa5O3R3cLQKRvwq7EWeRvrTMQhfOJ/ai +VQGnzmRuRevFVsHf0YuI4M+TEYcUooL2BdiOu8rggfezUYA9r2sjtshSok0UvKeb +79pNzWmg9EWVeFk+A0HQpyLq+3EVyB5UZ3CZRkT0XhEm1B7mpKrtcGMjaumNAam7 +jkhidGdhT/PV9BB1TttcqwTf+JH9P9sSpY9ZTA1LkkeWe9Rwqpxjjssqxxl6Xnds ++wUfuovVvHuBinsO+5PLE5UQByn21WcIBNnPCAMvALy/68T7z8+ATJ+I2CqBXuM2 +z5663hNrvdCu93PpK4j/k/1l3NTrthaorbnPhkmNYHQkBicpAfFQywrv6enD+30C +gcEA7Vlw76og4oxI7SSD6gTfo85OqTLp2CUZhNNxzYiUOOssRnGHBKsGZ8p0OhLN +vk9/SgbeHL5jsmnyd8ZuYWmsOQHRWgg1zO3S25iuq+VAo4cL/7IynoJ0RP5xP0Pw +QD+xJLZQp0XuLUtXnlc6dM5Hg7tOTthOP9UxA1i57lzpYfkRnKmSeWi+4IDJymOt +WoWnEK7Yr7qSg6aScLWRyIvAPVmKF9LToSFaTq0eOD0GIwAQxqNbIwN3U0UJ5Ruc +KRBVAoHBAL/+DNGqnEzhhWS6zqZp2eH90YR+b3R4yOK4PROm2AVA3h1GhIAiWX68 +PvKYZK9dZ9EdAswlFf9PVQYIXUraR3az0UiIywnNMri+kO1ZxwofGvljrOfRRLg0 +B46wuHi6dVgTWzjTl503G9+FpAYNHv184xsr1tne0pf2TKEnN7oyQciCV8qtr8vV +HrL46uaD0w1fcXIXbO3F/7ErLsvsgLzKfxR5BeQo6Fq0GmzD+lCmzVNirtfl2CZj +2ukROXUQnwKBwQDR1CqFlm/wGHk4PPnp31ke5XqhFoOpNFM1HAEV5VK0ZyQDOsZU +mCXXiCHsXUdKodk0RpIB80cMKaHTxbc7o0JAO50q7OszOmUZAggZq1jTuMYgzRb3 +DvlfLVpMxfEVu7kNbagr2STRIjRZpV/md569lM+L4Kp8wCrOfJgTZExm8txhFYCK +mNF2hCThKfHNfy7NDuY9pMF2ZcI8pig1lWbkVc5BdX7miifeOinnKfvM4XfzQ+OE +NsI8+WHgC+KoYukCgcBwrOpdCmHchOZCbZfl9m1Wwh16QrGqi1BqLnI53EsfGijA +yaftgzs+s7/FpEZC3PCWuw3vPTyhr69YcQQ/b8dNFM8YYJ+4SuMfpUds5Kl5eTPd +dO/+xMQtzus4hOJeiB9h50o8GYH7VGJZVhcjLgQoBGlMgvf+uVSitnvWgCumbORK +hqR7YF+xoov3wToquubcDE2KBdF54h/jnFJEf7I2GilmnHgmpRNoWBbCCmoXdy09 +aMbwEgY+0Y+iBOfRmkUCgcEAoHJLw7VnZQGQQL4l3lnoGU9o06RPkNbpda9G/Ptz +v+K7DXmHiLFVDszvZaPVreohuc0tKdrT0cZpZ21h0GQD4B6JX66R/y6CCAr0QpdA +pFZO9sc5ky6RJ4xZCoCsNJzORNUb36cagEzBWExb7Jz2v6gNa044K5bs3CVv5h15 +rJtTxZNn/gcnIk+gt//67WUnKLS4PR5PVCCqYhSbhFwx/OvVTJmflIBUinAclf2Q +M4mhHOfwBicqYzzEYbOE9Vk9 -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIUF15VKdwjiTzzKgs6PnNpEekV9QQwDQYJKoZIhvcNAQEL +MIIEjDCCAvSgAwIBAgIUQ2S3jJ5nve5k5956sgsrWY3vw9MwDQYJKoZIhvcNAQEL BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv -c3RuYW1lMB4XDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFowYjELMAkGA1UE +c3RuYW1lMB4XDTIzMTEyNTA0MjEzN1oXDTQzMDEyNDA0MjEzN1owYjELMAkGA1UE BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAn/BVsYuKFZQ1cXvDQxYG/iGABq/4 -WVrmNWau3e4Oe8j6MSa4LfmzfPflO6k3bqsgdh8vBtE6ngAjqyHaT2fD+1eROWIg -EsHye9p21aO6kfQwCgORRPD1NGyRZhS4ww4oXPe9M3OIpUQ30VspVKY+vJ3nzmOA -sxc3oqlTi19sHs7O2dIX5ZElxgZgtI9RW0FNiN2CYNwigmm4U7VOjsiiLc4ydr1W -jqEPgdCr/JXmkUOgDWQQBUBdi3qUsuRdvqEg/kB4z6/tNdleQ5q7HSHPVnvGMqJ0 -tbv9xyO5S6IX91vqha6EFo59AySOeknFdbhK/wdPoZ9owUUzOXdRfQXlVptbvATi -budArznrsYlS7MSaWuE+NQyc5yZOVfoROISrwyppa8LpbEafKI1BMlIvHc1jgUXw -IoyFr96gD4xgPyd95+vj5ZQnMwtgbFWYVZi6kjAsKdYNiUwjPygu0+LzpfxBpJu+ -n3Ji3gNRH/KFHUYPt0++1T/h6mdiT9OifDvDAgMBAAGjGzAZMBcGA1UdEQQQMA6C -DGZha2Vob3N0bmFtZTANBgkqhkiG9w0BAQsFAAOCAYEARzdkuqa0Hexi/saMkdi3 -bubpQkc7X0RYKWnjy/PgcmbvQXLiWRMZOH9rMWvd5v+ZfkgAtsbOQuP8ycioNIFY -Il5SEmxHEN81z5UNSPLOib6ky13gzrnXRAxnnO7cICG7AaMu1dHv57fqjevcx/n/ -nxPNKwKL+TDpMw7ATVZw7Py7JciKyFAfwtkvt17j/ldvaQvuwmWHzyFVrQniQcQq -QEa4jy/Y/pXHAgCKq1qbe0ush17j1ChyH7l4SkF2xJKcYYQF5ipw8zg6WeOL2NFE -G1KDJN0SsMmM3PMN1e0lLQP3G+UaatervrKXu51QleKL32Xlby+pp1w9KKs39/Tb -RT8EMe9A6cecod6TL0ZUQHow6ykNYBkfSKDLTKWnL9ifZ0C/DvgmS7DpJg3oAa1e -GhIglMrgqJflTHAI/PvEsCKM1O0Un2dVGWsUCzPfhj1cKmagyb0Zd+2Tk9xGSRs9 -2ceXMxRCjOJwEHUCFuTYeqowabdlpi0nyPbSn7JIwCpT +BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsgFF43M4FBJ+5pjsAzIv/vsA0cai +QCdn+xfQA8U7jpvhLXaX2K+JgZ/FM/XtyPmYXhuct0GdXXQWAPWE33FC9KJ/FFyz +jgVWQnrcbEF6EgLgzD1dCF3hILWWQ13u0k6ikkHmJOws/lbR3pWfI+6DjnMebCYK +W2g/jhiZz6AHQfJfJ0H8agBuObFuD+oHyyqCS74inLQ3uab4IiQvLb8rYPyIREin +nny2eIsdnTY49hjVGi/pIVOlBLjNZCR3D1Yz6GHPsvH6Kf6oUNIrYQYTtjryfmJj +hXQhOu0sgsFN6fMyaTfl5MGx7bYJjYODkrOx+lexOdvwrEzUwErYQR17camjPr42 +xgEa89haXP5/i8EbezBiEdIFbZ7U4Jz9B6Hh7JIQCHg7Wj62esx3Zd2g8IAHnFXm +0xAU+WD26M/l9ZVnTq0Q9iVt26l0kSDdxOKs+SDLjcd1b4aUkXhWc0ekCG8UKszv +1afdyL8vIT2eljIH0mFrOwhrbuDT+/TyX3TLAgMBAAGjOjA4MBcGA1UdEQQQMA6C +DGZha2Vob3N0bmFtZTAdBgNVHQ4EFgQU5wVOIuQD/Jxmam/97g91+igosWQwDQYJ +KoZIhvcNAQELBQADggGBAFv5gW5x4ET5NXEw6vILlOtwxwplEbU/x6eUVR/AXtEz +jtq9zIk2svX/JIzSLRQnjJmb/nCDCeNcFMkkgIiB64I3yMJT9n50fO4EhSGEaITZ +vYAw0/U6QXw+B1VS1ijNA44X2zvC+aw1q9W+0SKtvnu7l16TQ654ey0Qh9hOF1HS +AZQ46593T9gaZMeexz4CShoBZ80oFOJezfNhyT3FK6tzXNbkVoJDhlLvr/ep81GG +mABUGtKQYYMhuSSp0TDvf7jnXxtQcZI5lQOxZp0fnWUcK4gMVJqFVicwY8NiOhAG +6TlvXYP4COLAvGmqBB+xUhekIS0jVzaMyek+hKK0sT/OE+W/fR5V9YT5QlHFJCf5 +IUIfDCpBZrBpsOTwsUm8eL0krLiBjYf0HgH5oFBc7aF4w1kuUJjlsJ68bzO9mLEF +HXDaOWbe00+7BMMDnyuEyLN8KaAGiN8x0NQRX+nTAjCdPs6E0NftcXtznWBID6tA +j5m7qjsoGurj6TlDsBJb1A== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem index f6887ba7a84e1a..20d9bd14e9678f 100644 --- a/Lib/test/certdata/keycert3.pem +++ b/Lib/test/certdata/keycert3.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDFtLOteQlQojN7 -ztkux7m0hmGKkP1hh0hbKqTcD87jkLAqAwZWenjZMjCbbZ3vP+AObCIkYIKzPXY7 -Yi+H5M3O2mXIDxoHGjL/GWtoEyDNXvm9UC+MRuSOq2MaLHHQG0Rx2TxcYrMVUM7b -93rpN1LGRrCv1gISXM4EvEJooAR7Aadj0pG/o0fqDAdFjH6QZbhn1iZle+eGbjcf -dgH/H0F8dn1PPGoViHXicbsQ4kB6002Pf+aXP4b2QKAbflyNHEKHPHEOOTXrFjMd -c+bqKW24epEsMZI59qx9hU/4Rvp3/v+vEwTL7Nm7ilptzZn2cvGCW39LC0nNYLOz -kO3H8xwA75h6uykdB+WO/v2CKIK9M/ZO+9QNrmaokfKDamCk39b8hlCwNL6LsVpv -d3XTS5Wn4YWn92EqiltUJJoPo7pc7VTdWCg4zVFn4Q8Zh4NFNn/qTB8lEMgrsNTV -5cyZ7zhoBiUMSO45bmo2NsnE7ce/JUhlqe5uh0PT1MIBgTV+oDMCAwEAAQKCAYEA -udsy4gwblqK0tVnxz0lQqYV+os3EdO/BNHr1Oi7eNg2pngTz603812mYSjUVOHma -vtQmkH3twGQyBoc52Y1dcGzdK+IOfMjDUg7qao840ffL3I1J9ZwbdodlhZBsec94 -W3J1jP/4DDzICf8vm5g3h0+i/9m2Xt7BibAU2dg7/grC+lNUUoxDqaEfIOF/hW0q -muq1c8e0EisAROIh5FzUqhWVnWxU6eM7tuFlkuyu4whLLHB3LI466Lo+CTqT9M+v -jJYlvS5+AZW3qMBp6WOI8C+VIiBL178mo+Igkyyy5AYXcWeNkjp6ygRWvtWXIhCv -CI29mf+BP/54jAY0rQRXJ2UcSHXmM6PTDkE/L2OKeiY1Ou8gLOwun3yBVdbkXJMb -PWmUW4N8qSIJQ+vE2TDqmkqAT6m+ilzOXl1O+LLTvGyMnOiiSLXK9mC4ND3tqaQu -hvKivnI1doErcWUaIf1DHiJmLrGxrTCUKjCEoefqVq2/dDdtCfx7CqUvjl3DYKMB -AoHBAP+Vdi6D07gZFepEGCaJ+YH6cxEyO73CNnea/F1whVAzOv91kHS32jC9PAI3 -/wYlX+DLcN9mVF/q62V4SLZYfOxTPW4vWO0A45URe9s9Z795fdAcQ5jt3QFOVSnk -3XSaCkIOwckuwabGJi4+foiUEOnLLzQi1/g7x12dwejxVNhqhz5KFkOQPv8fQRed -sb5LVLYDeprsB2Vsx0fHwg4z9FvTIxLBeI7+sJD30lNpYZrCl/T9x4e1SV2Rwn2W -bghxgQKBwQDGBx07biZK9RB5g4qPl+G6vz0M+/KBfpwQbMYxSyct7u6gfGD9mWBO -qocIIr39Unac3kUL237Cn3HbgiGCRe7Mwd7XqnSSGWM5oWSlVQxEKTXYUlTbd9O9 -DKuyQGOl/AMEwD4ZbEOfQNmnd1U4nh1AV052FQY8Ry/atGFT9fApA/5X/bbenOwQ -YGDsokLzPf2BIDncpE+VNevUMoMI7EnySgjjfpL+cRld0qpLqBMo2h5VddeJ/5YM -1YcNfMQiw7MCgcEAwXqXuKa7A8aZvHpH/gS9CRRbP01TxFbdfLWrDeE8SnY9111c -Ob9kQTk/0D4rpK9uYXIgxD1m6iWghXQFN2TNTOnGuz7EhsYBgrt1k4Zsn5qND5oV -4hNPFsoB1nEW5EooMdGSCYaHuoSOKrvMdgAAvbu+xC0MaTJ3vfrK7Fik7h/WueTD -7emohuFWGVabU38bZZ5EljrPboxmX4Rs9uuFtG2lQ3GKnlVXvKaeZd6EsO9WsXPc -NHOcUmUhYokaSvIBAoHAGCxGJTsM8Zl4qVylTWH87A7sJOmccLJD2r1sdBf4cGL6 -PhzwugQ+/VtToGqdRo8Ka5u2Ufw5PQi5nVIFRSHERLpluW3VTQBMXHyXDJeVJ7zg -Fcf3E9NMxYcGbnvtrhVVSP8ulWvh1U7VQtwOSxsB9xixOzjVygXmkYvzVYxwBJG4 -OoV+DS6aomUhb8Fe6tJmX5zPc1+bV1t9ril8VVqCrFDdROfuiaDEt+8/Wnzp2dLG -YShBZ1cLugVWtw7D4nqBAoHAF29k64iAxY5Y4OOibVkqjUCPyqG2oxiXqgO7CxZp -FGUat5UtV2mIBlSENs1o5AZ1nPlgWtPtg0xVCaG2t/Rq7ugvUfAnAhUK6zX8FS+T -gCXE+7iKuuIJiCo13/iAwF/CLfuXvj4CZ71ta0wX9w99f1FcPEk0x+ytiyuWJK8K -tyubL34JwNrnkh/8e3LcV3L88Sk9ZmxeTz31f3cA3Fy2ZJOAUMD9dKXeKtY7azzt -MkhXedRsdLSKqMh0VGeGHoLS +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCgKihxC+2g7d7M +JfIUBfFWiuMwxg0WhdxPyGUzMAjexbEOHo0ojntxPdH9KYRwiKRKb9jnmzXp2CKT +hqBYJIetq/3LYZp4bvDJ/hVCL9e4jqu1l+wd9DkqhKZ69b6C1/d12JAKvC5TIT+/ +b7EglYU8KMNx2WO5KxIJeVpX68jn49YtUzg0hT0QiXj4eugbDk1L1f99xgvkHaVW +VQxi6MFNWHJq/xXUb8E/hd/Q3oadN1BXMWl9P46D4R+YGKQQdZFkwEJsbqijFvWW +bOoaz7TFxf8n0q616803aXLfaWikfEXLnznEvKo7vyEivtT/y14Nm+SiR3nS6E0y +Dt8gmeHdaHcrmQT+yQ6yNOYDCcfeYM+rBuvOUHPudjMy0k8K/0IPjDo0KActKPH0 +UVbyDBMKDdGQ2+LhRFLsGXHlD9b05PxhqTULe3LeK6KZ+iuGbWtwVLaL5S42WiCA +YXNShE1Ko0Q7wugAippXCf+aWP3Wx9ZTrsfiDBbIfnY5mlfdG90CAwEAAQKCAYAA +ogoE4FoxD5+YyPGa+KcKg4QAVlgI5cCIJC+aMy9lyfw4JRDDv0RnnynsSTS3ySJ1 +FNoTmD5vTSZd1ONfVc2fdxWKrzkQDsgu1C07VLsShKXTEuWg/K0ZKOsLg1scY0Qc +GB4BnNrGA1SgKg3WJiEfqr2S/pvxSGVK2krsHAdwOytGhJStSHWEUjbDLKEsMjNG +AHOBCL5VSXS00aM55NeWuanCGH36l/J4kMvgpHB9wJE1twFGuHCUvtgEHtzPH9fQ +plmI0QDREm6UE6Qh01lxmwx3Xc5ASBURmxs+bxpk94BPRpj8/eF2HPiJalrkJksj +Xk3QQ7k23v6XnmHKV3QqpjUgJTdbuMoTrVMu14cIH6FtXfwVhtthPnCI8rk5Lh8N +cqLC7HT+NE1JyygzuMToOHMmSJTQ8L6BTIaRCZjvGTPYaZfFgeMHvvhAJtP5zAcc +xQzyCyNBU8RdPGT8tJTyDUIRs20poqe7dKrPEIocKJX7tvNSI2QxkQ96Adxo1gEC +gcEAvI8m6QCDGgDWI8yTH9EvZQwq+tF8I+WMC+jbPuDwKg5ZKC7VjRO//9JzPY+c +TxmLnQu64OkECHbu7pswDBbtnPMbodF9inYEY5RkfufEjEMJGEdkoBJWnNx78EkV +bcffWik0wXwdt6jd1CAnjmS9qaPz0T1NV8m5rQQn5JUYXlC9eB2kOojZYLbZBl3g +xUSRbIqHC7h8HuyAU26EPiprHsIxrOpbxABFOdvo2optr50U7X10Eqb4mRQ4z22W +ojJdAoHBANlzJjjEgGVB9W50pJqkTw8wXiTUG8AmhqrVvqEttLPcWpK6QwRkRC+i +5N1iUObf/kOlun2gNfHF6gM68Ja9wb2eGvE5sApq9hPpyYF0LS3g8BbJ9GOs6NU9 +BfM1CkPrDCdc4kzlUpDibvc6Fc9raCqvrZRlKEukqQS8dumVdb74IaPsP6q8sZMz +jibOk0eUrbx2c5vEnd0W8zMeNCuCwO1oXbfenPp/GLX9ZRlolWS/3cQoZYOSQc9J +lFQYkxL3gQKBwQCy3Pwk9AZoqTh4dvtsqArUSImQqRygFIQXXAh1ifxneHrcYijS +jVSIwEHuuIamhe3oyBK6fG8F9IPLtUwLe8hkJDwm8Misiiy5pS77LrFD9+btr/Nk +4GBmpcOveDQqkflt1j6j9y9dY4MhUGsVaLx86fhDmGoAh2tpEtMgwsl91gsUoNGD +cQL6+he+MVkg510nX/Sgipy63M8R1Xj+W1CHueBTTXBE6ZjBPLiSbdOETXZnnaR4 +eQjCdOs64JKOQ0UCgcBZ4kFAYel48aTT/Z801QphCus/afX2nXY5E5Vy5oO1fTZr +RFcDb7bHwhu8bzFl3d0qdUz7NMhXoimzIB/nD5UQHlSgtenQxJnnbVIAEtfCCSL1 +KJG+yfCMhGb7O0d8/6HMe5aHlptkjFS2GOp/DLTIQEoN9yqK6gt7i7PTphY/1C2D +ptpCZzE32a2+2NEEW67dIlFzZ/ihNSVeUfPasHezKtricECPQw4h3BZ4RETMmoq+ +1LvxgPl3B8EqaeYRhwECgcEAjjp/0hu/ukQhiNeR5a9p1ECBFP8qFh6Cpo0Az/DT +1kX0qU8tnT3cYYhwbVGwLxn2HVRdLrbjMj/t88W/LM2IaQ162m7TvvBMxNmr058y +sW/LADp5YWWsY70EJ8AfaTmdQriqKsNiLLpNdgcm1bkwHJ1CNlvEpDs1OOI3cCGi +BEuUmeKxpRhwCaZeaR5tREmbD70My+BMDTDLfrXoKqzl4JrRua4jFTpHeZaFdkkh +gDq3K6+KpVREQFEhyOtIB2kk -----END PRIVATE KEY----- Certificate: Data: @@ -51,34 +51,34 @@ Certificate: Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c5:b4:b3:ad:79:09:50:a2:33:7b:ce:d9:2e:c7: - b9:b4:86:61:8a:90:fd:61:87:48:5b:2a:a4:dc:0f: - ce:e3:90:b0:2a:03:06:56:7a:78:d9:32:30:9b:6d: - 9d:ef:3f:e0:0e:6c:22:24:60:82:b3:3d:76:3b:62: - 2f:87:e4:cd:ce:da:65:c8:0f:1a:07:1a:32:ff:19: - 6b:68:13:20:cd:5e:f9:bd:50:2f:8c:46:e4:8e:ab: - 63:1a:2c:71:d0:1b:44:71:d9:3c:5c:62:b3:15:50: - ce:db:f7:7a:e9:37:52:c6:46:b0:af:d6:02:12:5c: - ce:04:bc:42:68:a0:04:7b:01:a7:63:d2:91:bf:a3: - 47:ea:0c:07:45:8c:7e:90:65:b8:67:d6:26:65:7b: - e7:86:6e:37:1f:76:01:ff:1f:41:7c:76:7d:4f:3c: - 6a:15:88:75:e2:71:bb:10:e2:40:7a:d3:4d:8f:7f: - e6:97:3f:86:f6:40:a0:1b:7e:5c:8d:1c:42:87:3c: - 71:0e:39:35:eb:16:33:1d:73:e6:ea:29:6d:b8:7a: - 91:2c:31:92:39:f6:ac:7d:85:4f:f8:46:fa:77:fe: - ff:af:13:04:cb:ec:d9:bb:8a:5a:6d:cd:99:f6:72: - f1:82:5b:7f:4b:0b:49:cd:60:b3:b3:90:ed:c7:f3: - 1c:00:ef:98:7a:bb:29:1d:07:e5:8e:fe:fd:82:28: - 82:bd:33:f6:4e:fb:d4:0d:ae:66:a8:91:f2:83:6a: - 60:a4:df:d6:fc:86:50:b0:34:be:8b:b1:5a:6f:77: - 75:d3:4b:95:a7:e1:85:a7:f7:61:2a:8a:5b:54:24: - 9a:0f:a3:ba:5c:ed:54:dd:58:28:38:cd:51:67:e1: - 0f:19:87:83:45:36:7f:ea:4c:1f:25:10:c8:2b:b0: - d4:d5:e5:cc:99:ef:38:68:06:25:0c:48:ee:39:6e: - 6a:36:36:c9:c4:ed:c7:bf:25:48:65:a9:ee:6e:87: - 43:d3:d4:c2:01:81:35:7e:a0:33 + 00:a0:2a:28:71:0b:ed:a0:ed:de:cc:25:f2:14:05: + f1:56:8a:e3:30:c6:0d:16:85:dc:4f:c8:65:33:30: + 08:de:c5:b1:0e:1e:8d:28:8e:7b:71:3d:d1:fd:29: + 84:70:88:a4:4a:6f:d8:e7:9b:35:e9:d8:22:93:86: + a0:58:24:87:ad:ab:fd:cb:61:9a:78:6e:f0:c9:fe: + 15:42:2f:d7:b8:8e:ab:b5:97:ec:1d:f4:39:2a:84: + a6:7a:f5:be:82:d7:f7:75:d8:90:0a:bc:2e:53:21: + 3f:bf:6f:b1:20:95:85:3c:28:c3:71:d9:63:b9:2b: + 12:09:79:5a:57:eb:c8:e7:e3:d6:2d:53:38:34:85: + 3d:10:89:78:f8:7a:e8:1b:0e:4d:4b:d5:ff:7d:c6: + 0b:e4:1d:a5:56:55:0c:62:e8:c1:4d:58:72:6a:ff: + 15:d4:6f:c1:3f:85:df:d0:de:86:9d:37:50:57:31: + 69:7d:3f:8e:83:e1:1f:98:18:a4:10:75:91:64:c0: + 42:6c:6e:a8:a3:16:f5:96:6c:ea:1a:cf:b4:c5:c5: + ff:27:d2:ae:b5:eb:cd:37:69:72:df:69:68:a4:7c: + 45:cb:9f:39:c4:bc:aa:3b:bf:21:22:be:d4:ff:cb: + 5e:0d:9b:e4:a2:47:79:d2:e8:4d:32:0e:df:20:99: + e1:dd:68:77:2b:99:04:fe:c9:0e:b2:34:e6:03:09: + c7:de:60:cf:ab:06:eb:ce:50:73:ee:76:33:32:d2: + 4f:0a:ff:42:0f:8c:3a:34:28:07:2d:28:f1:f4:51: + 56:f2:0c:13:0a:0d:d1:90:db:e2:e1:44:52:ec:19: + 71:e5:0f:d6:f4:e4:fc:61:a9:35:0b:7b:72:de:2b: + a2:99:fa:2b:86:6d:6b:70:54:b6:8b:e5:2e:36:5a: + 20:80:61:73:52:84:4d:4a:a3:44:3b:c2:e8:00:8a: + 9a:57:09:ff:9a:58:fd:d6:c7:d6:53:ae:c7:e2:0c: + 16:c8:7e:76:39:9a:57:dd:1b:dd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 85:75:10:25:D0:2C:80:50:24:1A:5B:57:70:DE:B5:CB:71:A9:3B:7B + 3F:B1:E9:4F:A0:BE:30:66:3E:0A:18:C8:0F:47:1A:4F:34:6A:0F:42 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 95:f3:56:bb:d5:8c:70:bd:d1:de:da:63:b0:29:d7:db:60:27: - d6:59:fd:61:1b:30:c6:d0:5d:73:7d:34:e1:68:e3:28:a6:89: - e6:60:bd:89:d3:0e:f4:72:ad:72:76:f8:86:21:fd:75:3c:f8: - 6d:be:9c:04:e1:82:03:69:6c:ae:d0:55:ba:5e:f2:ca:f5:0f: - 8e:d6:d9:8d:c8:56:46:f4:f8:ac:74:2a:19:7b:8e:47:70:1f: - fb:fb:bd:69:02:a1:a5:4a:6e:21:1c:04:14:15:55:bf:bf:24: - 43:c8:17:03:be:3e:2c:ea:db:c8:af:1d:fd:52:df:d6:15:49: - 9e:c2:44:69:ef:f1:45:43:83:b2:1e:cf:14:1c:13:3f:fe:9c: - 71:cb:e7:1b:18:56:36:a7:af:44:f1:0b:a1:79:44:46:f9:43: - 46:29:d8:b0:ca:49:4d:65:60:d3:f6:8e:74:bc:62:9e:1e:8d: - 4b:29:9a:b4:0d:f0:a2:77:5b:34:e4:11:2f:a7:25:c5:e5:07: - 76:12:ae:be:75:73:15:e4:0a:7d:53:38:56:3f:79:6d:6e:ca: - ed:80:ab:56:ed:7e:8b:1c:e7:e3:d4:62:30:22:70:e7:29:b2: - 03:3c:fe:fa:3d:f0:36:c0:4d:11:a2:99:d3:29:31:27:b8:c5: - b8:15:a3:3c:4f:9b:73:5e:2b:b2:fb:cb:fd:75:47:b8:17:bd: - 21:d8:e6:c1:b9:ff:73:81:d8:25:08:6d:08:5e:1c:a5:83:50: - de:67:e6:da:d0:8e:5a:d3:f2:2a:b1:3f:b8:80:21:07:6a:71: - 15:6d:05:eb:51:b3:59:8d:d4:15:46:7e:02:a8:13:01:16:99: - bd:03:cc:70:71:2a:23:16:78:af:d1:d5:01:9d:04:b4:63:93: - 9a:04:3a:92:2e:e6:7e:73:93:a5:fe:50:9b:bd:0e:ea:54:86: - 6f:7c:e5:14:77:fe:c2:28:5a:4a:0e:d7:2d:8c:e9:ed:61:29: - b2:53:ff:6c:04:bc + Signature Value: + ca:34:ba:c5:d0:cf:27:31:32:d6:0d:27:30:b8:db:17:df:b7: + 39:a7:bb:b1:3b:86:c4:31:fd:fb:ab:db:63:1a:cc:90:ab:b9: + 4e:ab:34:49:0c:5e:8c:3e:70:a3:a7:6b:2f:a7:9a:25:7b:01: + 5a:18:96:48:76:f8:36:78:74:fa:bc:7d:68:7f:e5:ca:a6:9d: + 7b:dc:72:bd:a3:25:51:17:68:e8:e9:d7:02:86:2c:7d:16:7c: + b5:dc:44:b2:0a:e3:f7:c9:33:a3:51:36:83:bc:d4:70:cd:84: + 91:9f:06:ba:2d:d2:05:0a:65:c3:d9:55:09:a8:b8:09:69:bb: + 93:86:c2:b7:c2:90:74:7c:bf:f0:5d:bc:0e:63:13:8c:eb:fa: + 0f:f1:fa:e5:12:70:4d:0c:eb:8c:2e:a2:42:42:00:04:0f:fc: + f9:1f:41:9c:63:78:f0:66:93:b2:8f:2e:e8:93:1c:50:cb:2d: + 7f:b6:ba:57:6f:52:62:d7:39:0b:09:82:ab:a6:53:4d:cc:05: + 3e:19:f0:d4:c0:ce:a9:ad:10:ce:b9:71:e4:8f:f2:5a:3c:65: + ba:dc:cb:e0:04:90:2b:a5:15:a6:7d:da:dc:a3:b5:b7:bc:a0: + de:30:4e:64:cb:17:0d:3a:a0:52:d2:67:3b:a2:3a:00:d5:39: + aa:61:75:52:9f:fe:9b:c0:e8:a0:69:af:a8:b3:a3:1d:0a:40: + 52:04:e7:3d:c0:00:96:5f:2b:33:06:0c:30:f6:d3:18:72:ee: + 38:d0:64:d3:00:86:37:ec:4f:e9:38:49:e6:01:ff:a2:9a:7c: + dc:6a:d3:cb:a8:ba:58:fb:c3:86:78:47:f1:06:a6:45:e7:53: + de:99:1d:81:e6:bc:63:74:46:7c:70:23:57:29:60:70:9a:cc: + 6f:00:8e:c2:bf:6a:73:7d:6e:b0:62:e6:dc:13:1a:b9:fe:0f: + c2:d1:06:a1:79:62:7f:b6:30:a9:03:d0:47:57:25:db:48:10: + d1:cf:fb:7d:ac:3d -----BEGIN CERTIFICATE----- MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMW0s615CVCi -M3vO2S7HubSGYYqQ/WGHSFsqpNwPzuOQsCoDBlZ6eNkyMJttne8/4A5sIiRggrM9 -djtiL4fkzc7aZcgPGgcaMv8Za2gTIM1e+b1QL4xG5I6rYxoscdAbRHHZPFxisxVQ -ztv3euk3UsZGsK/WAhJczgS8QmigBHsBp2PSkb+jR+oMB0WMfpBluGfWJmV754Zu -Nx92Af8fQXx2fU88ahWIdeJxuxDiQHrTTY9/5pc/hvZAoBt+XI0cQoc8cQ45NesW -Mx1z5uopbbh6kSwxkjn2rH2FT/hG+nf+/68TBMvs2buKWm3NmfZy8YJbf0sLSc1g -s7OQ7cfzHADvmHq7KR0H5Y7+/YIogr0z9k771A2uZqiR8oNqYKTf1vyGULA0voux -Wm93ddNLlafhhaf3YSqKW1Qkmg+julztVN1YKDjNUWfhDxmHg0U2f+pMHyUQyCuw -1NXlzJnvOGgGJQxI7jluajY2ycTtx78lSGWp7m6HQ9PUwgGBNX6gMwIDAQABo4IB +Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt +3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY +IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh +P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd +pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW +9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo +TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o +8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa +IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUhXUQJdAsgFAkGltXcN61y3GpO3swfQYDVR0jBHYwdIAUs4qgorpx8agkedSk -WyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m +FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ +jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAJXzVrvVjHC90d7a -Y7Ap19tgJ9ZZ/WEbMMbQXXN9NOFo4yimieZgvYnTDvRyrXJ2+IYh/XU8+G2+nATh -ggNpbK7QVbpe8sr1D47W2Y3IVkb0+Kx0Khl7jkdwH/v7vWkCoaVKbiEcBBQVVb+/ -JEPIFwO+Pizq28ivHf1S39YVSZ7CRGnv8UVDg7IezxQcEz/+nHHL5xsYVjanr0Tx -C6F5REb5Q0Yp2LDKSU1lYNP2jnS8Yp4ejUspmrQN8KJ3WzTkES+nJcXlB3YSrr51 -cxXkCn1TOFY/eW1uyu2Aq1btfosc5+PUYjAicOcpsgM8/vo98DbATRGimdMpMSe4 -xbgVozxPm3NeK7L7y/11R7gXvSHY5sG5/3OB2CUIbQheHKWDUN5n5trQjlrT8iqx -P7iAIQdqcRVtBetRs1mN1BVGfgKoEwEWmb0DzHBxKiMWeK/R1QGdBLRjk5oEOpIu -5n5zk6X+UJu9DupUhm985RR3/sIoWkoO1y2M6e1hKbJT/2wEvA== +Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN +JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 ++DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN +hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu +okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA +zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV +OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 +SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ +anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem index 1003d67fd075ec..ff4dceac7907c5 100644 --- a/Lib/test/certdata/keycert4.pem +++ b/Lib/test/certdata/keycert4.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC34y3S6iXdmdvd -M/2aFBe6CvRvZwhh1huGl7IQRtdoakPqMLlEdNHJtNeF5M27xLei+p4wt7N1Jyi0 -2keHQb1m9TqH5AruOkE2ti+15zEoKoU9aWydTiH+epKTT0yjg2NcKQjRUaWcbhzB -H4EMKuCIlzIIz8/EIKkOqhCDwq6+Fv3Ays+z7Bz+yR80ixivKu/l7SjxQ7z7R/kC -I7OViRcIO5QBQPj7VLvCTz4VA6u/LdXngK2HNuau6WXm5yNNQbqrB11AEJcYZf/c -VrneV4F+ZjLloAKgSn9GB8eWOyilTQ18TcKd+H2icipRaP/+QR/KPx5GK/SXU3my -qm62QOGI7t/5ktVdjGhs6tHZxw1SRiipiLYWbtVRrSxa4wYlgpgoUwvrvvtC5kAN -nTw1VGWsxcs+6a7+PocYnJiq7k4b5OAUb3Ryvl9DLAMy8NqpRWo4cHD/XQ3FCYwF -HlOSgx/dL5Se0i3dW1KzbP6OvaNg6nl/1EXPUsJ1ATS8nzvzhccCAwEAAQKCAYEA -nD3GvaJ9MeB802JNZBEWZ9jO/6jHknldQeq6POI0PF+t/NoRUH0BkyS4yucxdw0a -CrxulG5BaJUxHRkqFV5iE4zhgnzcXLXamyYJO8GIHtyiASAGTVIJyDNVPxztvTDx -x2iGOXPqBxP4Eo82EqSLywLMXHhVzAsEGZWeGpXb61+Vk62+9Nz1dfZlMTvOaWdO -Fkp/sx8e/1KT3KGBANlOXIxioP4Xj1Tbg6nY0fogf3vud5j52B1pu8xL7PkPIaFq -DEGz3XvWhBF/+Cs5iDeYz8eQpfQig7HdHVn2D8dZmzQgpLw1yGbPAnqrgopWfm7R -MqiyFe82p2t+vfSoG5jz28XxPtzBJV3ljxKxlbnclqu/CAYSjzaYohDzyhjdZOZI -r9DOfWOqu01Ha3EEsApn95fusHHGTH2FOy0u61FSTrfLfqsLw9WRJPWleirKikhf -SZzi223QrmzZMtuCF7VgTx3ghDhBmFD8uzVVQ1SwPZ8CgftRkFcn1llXIAfJ3iHB -AoHBAOg3DOIdtUVgpjMKhpAyuH54fYvGl7afIMNbKRle0kCiP45wtGJ43RPMqiR8 -1rxZB3+iapICI/lnhk3O7vVRkR64yiqQBcl/hXZ1BhyD6iDXWYmm5mcnymcoqfwc -p9TfzEPyGPb3SM2YlI0cSPRqM/jDvGvnDeKIpzEKvUlwJ59WoN2HOHTIXf+XbN5n -unpuTt6YKJvc48DrXsPnUzkCmUfbOmgHfeb9/qBs/8kY4YJMsZEjqf88o7mCJCIy -BtDxTwKBwQDKuOwE8e0GIA01ZHd6RfR+ZCvmp2oauxal4EJsBx+ZZnhEWGaSm1fE -Bf/ih074ghcSKoSrdYpD1xGZ6fGVWMx3jcL11yLDOUiiPDJsm8hUBZ0IW1qXyfCP -l7xy1bUkWwPXdmFuGp1exrcjooKrFNuTdYiK4nQZSKuCfXQRADrmEJmM+gYwhqI7 -4XsYo848B9A4hbY6RLEox4uvo/RmafY0iR0PMhVEc+ydNLKB/4LpahZqBQ4kTpMv -o4+rEvYt1gkCgcB08gx177ozx1nMCLf99N0/LBUmCIytNvR8DfPjyAIg9NUHOjFO -CkpkR0VEfO50Cm4hVD1RbOyLFRzpIJbtSvfHvg5qYv/XG3auUn8Sa0jE408/aKNO -PhbL3wnEYvYO2ep4KXtzHNQ4XmgprJ39IWMtG/5PZRx0ApgYtazgSDBcKXd4OTow -bhwQtUTpuNmMAPONXJnO7O5yYNbn2B7sbiedrYV7kJJSe4X5awtiTjp7sX4XdxuM -5BAcQ7NI2WLfZTcCgcBp/X9hIoATmMRvKwUQx+yJ/KO7Z8KhETpJJdR0mNDbqmit -Cy8t7cxYb+6WqLoQUivv0o0k/EJ7L8JDH76woAnfZB4P3RiOy69/K0wN3vFBhOHS -kbju7aU53lKoE7YuuOtsRrewEng/KlRsbDY3bqNTGLt4KegbpBQQGLmLffxNd1Zh -EAQWcP33ou9yNYrJdihWtQpOssWRlash/O32ceZJF3s7C6t068tFclz2fPocQdxQ -OC5pqy9nU/P0tOhDlMkCgcEAosaBJLIeAYlOU0+2uSx5g5mIqOOTyrDEmqqad6T/ -wkB7vW2QaoDvLL22Yrzdn9vQ0V0rqzhVtan7sq5pn/BQJAueZYN8rFxS3uuW+UQk -Nsc4GLJzU8Az/2DvqEIrnE7zRc5E1FOI9gKLrBlpJB2o0hVcBznDe05Gax6Kjqbm -jHqzyU73SpxpEy3OesClCeCQIMr47HaL9aSqaEX4U9bMpgHi0HgTTHqvJ5pch0hY -dYl+WAE9LAyF1DF29BirEXVw +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDGKA1zZDjeNPh2 +J9WHVXXMUf8h5N4/bHCM3CbIaZ1dShkCgfmFWmOtruEihgbfRYaSWZAwCmVAQGjm +gvUfgOIgsFfM8yO+zDByPhza7XvWPZfEe7mNRFe5ZlYntbeM/vuWCM4VzwDq/mqF +TFxNRmwInqE7hx0WnfCoQWe9N41hJyl1K0OjADb+SjlpJ0/UJ63hsB+dowGjaaBv +J8HduQcRqNg8s6FcyJJ8Mjss1uRMFK2j9QrmgbA61XuIPCxzc3J57mW8FN2KsR8D +2HOhe9nsTGlxp+O5Cudf/RBWB443xcoyduwRXOFTdEAU45MS4tKGP2hzezuxMFQn +LKARXVW4/gFxZk7kU8TweZUS6LAYPfYJnlfteb6z37LAbtoDvzKUKBEDf/nmoa7C +uKxSPC5HIKhLbjU/6kuPglSVEfJPJWu2bZJDAkFL85Ot3gPs10EX2lMUy0Jt3tf+ +TaQjEvFZhpKN8KAdYj3eVgOfzIBbQyjotHJjFe9Jkq4q7RoI+ncCAwEAAQKCAYAH +tRsdRh1Z7JmHOasy+tPDsvhVuWLHMaYlScvAYhJh/W65YSKd56+zFKINlX3fYcp5 +Fz67Yy+uWahXVE2QgFou3KX0u+9ucRiLFXfYheWL3xSMXJgRee0LI/T7tRe7uAHu +CnoURqKCulIqzLOO1efx1eKasXmVuhEtmjhVpcmDGv8SChSKTIjzgOjqT7QGE9Xq +eSRhq7mulpq9zWq+/369yG+0SvPs60vTxNovDIaBn/RHSW5FjeDss5QnmYMh/ukN +dggoKllQlkTzHSxHmKrIJuryZC+bsqvEPUFXN0NMUYcZRvt1lwdjzq/A+w4gDDZG +7QqAzYMYQZMw9PJeHqu4mxfUX5hJWuAwG5I2eV3kBRheoFw7MxP0tw40fPlFU+Zh +pRXbKwhMAlIHi0D8NyMn3hkVPyToWVVY3vHRknBB/52RqRq3MjqEFaAZfp0nFkiF +ytv3Dd5aeBb1vraOIREyhxIxE/qY8CtZC+6JI8CpufLmFXB412WPwl0OrVpWYfEC +gcEA486zOI46xRDgDw0jqTpOFHzh+3VZ8UoPoiqCjKzJGnrh2EeLvTsXX/GZOj0m +5zl6RHEGFjm5vKCh2C72Vj/m+AFVy7V9iJRzTYzP8So/3paaqo7ZaROTa6uStxdD +VPnY1uIgVQz9w5coN4dmr+RLBpFvvWaHp1wuC08YIWxcC9HSTQpbi1EP5eo08fOk +8reNkDEHxihDGHr1xW0z0qJqK1IVyLP7wDkmapudMZjkjqJSGJwwefV4qyGMTV2b +suW1AoHBAN6t9n6LBH553MF5iUrNJYxXh/SCom4Zft9aD6W4bZV/xL4XPpKBB4HX +aWdeI0iYZU9U+CZ88tBoQCt+JMrJ9cz03ENOvA/MBMREwbZ2hKmQgnoDZsV0vNry +6UsxeQmeNpGQFUz9foVJQVRdQCceN2YEABdehV1HZoSBbuGZkzqGJXrWwaf/ZhpB +dPYGUGOsczoD2/QLuWy2M7f7v0Ews6Heww3zipWzvdxKE0IpyVs30ZwVi8CRQiWU +bEcleXP6+wKBwAi3xEwJxV39Q1XQHuk+/fXywYMp/oMpXmfKUKypgBivUy0/r61S +MZbOXBrKdE6s+GzeFmmLU/xP+WGYinzKfUBIbMwa6e7sH218UgjcoQ0Xnlugk9ld +kmqwajDvhvgdh5rRlIMsuBlgE33shJV+mxBpSGlrHw3cjTaJlFbTGsKpCO9B0jcG +pyEZUWVg+ZMASz6VYcLHj6nEKtufTjhlVsLJpWPE34F/rmSuB9n6C+UZeSLP91rz +dea2pfPf/TFfcQKBwF4DSj9Qx/vxzS7t9fXbuM+QoPitMpCTOQppRpPr0nA8uj6b +J7LIwPejj3+xsenTVWpx8DanqAgvC3CRWE05iQoYEupj0mhE9Xo7oSE81nOUbFHB +H+GbkKRLzA0P/Q7/egBouWWA3Kq/K9LHb+9UBYWPiM5U/K9OFs04rCyZHxylSCud +gbNA08Wf/xZjwgri4t8KhBF75bQtFJbHtY57Vkuv9d/tA4SCl1Tq/UiAxd86KMfi +HNeXPDsLd89t1eIOgwKBwQDJkqwZXkhwkhoNuHRdzPO/1f5FyKpQxFs+x+OBulzG +zuwVKIawsLlUR4TBtF7PChOSZSH50VZaBI5kVti79kEtfNjfAzg4kleHrY8jQ/eq +HludZ3nmiPqqlbH4MH8NWczPEjee6z4ODROsAe31pz3S8YQK7KVoEuSf0+usJ894 +FtzS5wl6POAXTo2QeSNg9zTbb6JjVYcq6KCTnflDm4YEvFKI+ARqAXQHxm05wEOe +DbKC6hxxQbDaNOvXEAda8wU= -----END PRIVATE KEY----- Certificate: Data: @@ -51,34 +51,34 @@ Certificate: Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b7:e3:2d:d2:ea:25:dd:99:db:dd:33:fd:9a:14: - 17:ba:0a:f4:6f:67:08:61:d6:1b:86:97:b2:10:46: - d7:68:6a:43:ea:30:b9:44:74:d1:c9:b4:d7:85:e4: - cd:bb:c4:b7:a2:fa:9e:30:b7:b3:75:27:28:b4:da: - 47:87:41:bd:66:f5:3a:87:e4:0a:ee:3a:41:36:b6: - 2f:b5:e7:31:28:2a:85:3d:69:6c:9d:4e:21:fe:7a: - 92:93:4f:4c:a3:83:63:5c:29:08:d1:51:a5:9c:6e: - 1c:c1:1f:81:0c:2a:e0:88:97:32:08:cf:cf:c4:20: - a9:0e:aa:10:83:c2:ae:be:16:fd:c0:ca:cf:b3:ec: - 1c:fe:c9:1f:34:8b:18:af:2a:ef:e5:ed:28:f1:43: - bc:fb:47:f9:02:23:b3:95:89:17:08:3b:94:01:40: - f8:fb:54:bb:c2:4f:3e:15:03:ab:bf:2d:d5:e7:80: - ad:87:36:e6:ae:e9:65:e6:e7:23:4d:41:ba:ab:07: - 5d:40:10:97:18:65:ff:dc:56:b9:de:57:81:7e:66: - 32:e5:a0:02:a0:4a:7f:46:07:c7:96:3b:28:a5:4d: - 0d:7c:4d:c2:9d:f8:7d:a2:72:2a:51:68:ff:fe:41: - 1f:ca:3f:1e:46:2b:f4:97:53:79:b2:aa:6e:b6:40: - e1:88:ee:df:f9:92:d5:5d:8c:68:6c:ea:d1:d9:c7: - 0d:52:46:28:a9:88:b6:16:6e:d5:51:ad:2c:5a:e3: - 06:25:82:98:28:53:0b:eb:be:fb:42:e6:40:0d:9d: - 3c:35:54:65:ac:c5:cb:3e:e9:ae:fe:3e:87:18:9c: - 98:aa:ee:4e:1b:e4:e0:14:6f:74:72:be:5f:43:2c: - 03:32:f0:da:a9:45:6a:38:70:70:ff:5d:0d:c5:09: - 8c:05:1e:53:92:83:1f:dd:2f:94:9e:d2:2d:dd:5b: - 52:b3:6c:fe:8e:bd:a3:60:ea:79:7f:d4:45:cf:52: - c2:75:01:34:bc:9f:3b:f3:85:c7 + 00:c6:28:0d:73:64:38:de:34:f8:76:27:d5:87:55: + 75:cc:51:ff:21:e4:de:3f:6c:70:8c:dc:26:c8:69: + 9d:5d:4a:19:02:81:f9:85:5a:63:ad:ae:e1:22:86: + 06:df:45:86:92:59:90:30:0a:65:40:40:68:e6:82: + f5:1f:80:e2:20:b0:57:cc:f3:23:be:cc:30:72:3e: + 1c:da:ed:7b:d6:3d:97:c4:7b:b9:8d:44:57:b9:66: + 56:27:b5:b7:8c:fe:fb:96:08:ce:15:cf:00:ea:fe: + 6a:85:4c:5c:4d:46:6c:08:9e:a1:3b:87:1d:16:9d: + f0:a8:41:67:bd:37:8d:61:27:29:75:2b:43:a3:00: + 36:fe:4a:39:69:27:4f:d4:27:ad:e1:b0:1f:9d:a3: + 01:a3:69:a0:6f:27:c1:dd:b9:07:11:a8:d8:3c:b3: + a1:5c:c8:92:7c:32:3b:2c:d6:e4:4c:14:ad:a3:f5: + 0a:e6:81:b0:3a:d5:7b:88:3c:2c:73:73:72:79:ee: + 65:bc:14:dd:8a:b1:1f:03:d8:73:a1:7b:d9:ec:4c: + 69:71:a7:e3:b9:0a:e7:5f:fd:10:56:07:8e:37:c5: + ca:32:76:ec:11:5c:e1:53:74:40:14:e3:93:12:e2: + d2:86:3f:68:73:7b:3b:b1:30:54:27:2c:a0:11:5d: + 55:b8:fe:01:71:66:4e:e4:53:c4:f0:79:95:12:e8: + b0:18:3d:f6:09:9e:57:ed:79:be:b3:df:b2:c0:6e: + da:03:bf:32:94:28:11:03:7f:f9:e6:a1:ae:c2:b8: + ac:52:3c:2e:47:20:a8:4b:6e:35:3f:ea:4b:8f:82: + 54:95:11:f2:4f:25:6b:b6:6d:92:43:02:41:4b:f3: + 93:ad:de:03:ec:d7:41:17:da:53:14:cb:42:6d:de: + d7:fe:4d:a4:23:12:f1:59:86:92:8d:f0:a0:1d:62: + 3d:de:56:03:9f:cc:80:5b:43:28:e8:b4:72:63:15: + ef:49:92:ae:2a:ed:1a:08:fa:77 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - C8:BD:A8:B4:C0:F2:32:10:73:47:9C:48:81:32:F8:BA:BB:26:84:97 + 1C:70:14:B0:20:DD:08:76:A4:3B:56:59:FA:5F:34:F8:36:66:E8:56 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 76:87:76:4d:e4:0f:88:bf:2c:f3:58:67:c0:97:6c:cd:59:18: - 82:83:4c:04:19:a5:6d:aa:fa:64:3d:49:32:3e:e1:56:95:b2: - 13:f7:cf:d3:11:b0:72:b7:5b:e7:d7:85:69:51:3c:b6:54:80: - 45:2f:28:10:21:20:b9:ba:e9:27:5a:b7:3f:82:b7:69:f5:46: - f5:bf:a2:8b:17:7f:f2:14:d1:46:97:b5:8b:47:fb:9f:e8:5c: - 05:0e:9d:11:bd:7c:9a:03:84:0b:ca:29:66:4a:ca:0d:6f:09: - 1e:7a:27:c1:7f:03:96:70:8d:18:a5:2f:a4:98:a5:19:aa:8c: - 5d:1e:8c:3e:bb:6d:3b:c0:33:c0:15:e1:bd:09:3d:9f:e8:dc: - 12:d4:cb:44:1d:06:f5:e8:d6:4e:a1:2d:5c:9f:5d:1f:5b:2a: - c3:4d:40:8d:da:d1:78:80:d0:c6:31:72:10:48:8a:e9:10:7a: - 13:30:11:b2:9e:67:0e:ed:a1:aa:ec:73:2d:f0:b8:8a:22:75: - 0f:30:69:5c:50:7e:91:ce:da:91:c7:70:8c:65:ff:f6:58:fb: - 00:bd:45:cc:e2:e4:e3:e5:16:36:7d:f3:a2:4a:9c:45:ff:d9: - a5:16:e0:2f:b5:5b:6c:e6:8a:13:15:48:73:bd:7c:80:33:c3: - d4:3b:3a:1d:85:0e:a4:f7:f7:fb:48:0c:e9:a0:4b:5e:8a:5c: - 67:f8:25:02:6f:cd:72:c1:aa:5a:93:64:7c:14:20:43:e0:13: - 7f:0d:e1:0d:61:5e:2e:2c:cd:7a:2e:2a:ae:b6:75:6a:5f:a0: - 1a:9b:b6:67:2d:b0:a5:1c:54:bc:8c:70:7e:15:2b:c0:50:e3: - 03:bb:a4:a5:fc:45:01:c9:3f:a7:b8:18:dc:3e:08:07:a1:9b: - f5:bd:95:bd:49:e8:10:7c:91:7d:2d:c4:c2:98:b6:b7:51:69: - d7:0a:68:40:b5:0f:85:a0:a9:67:77:c6:68:cb:0e:58:34:b3: - 58:e7:c8:7c:09:67 + Signature Value: + 75:14:e5:68:45:8d:ed:6c:f1:27:1e:0e:f3:35:ae:0e:60:c1: + 65:36:62:b8:07:78:e1:b9:8d:7a:50:70:af:06:c9:d4:ee:50: + ef:d2:76:b2:a2:b6:cb:dc:a6:18:b5:3d:d2:f7:eb:0e:ec:b7: + 95:cd:2e:b1:36:6f:a8:9f:b8:4d:ff:ce:8a:c4:8e:62:37:32: + 80:3e:05:4a:4d:39:87:69:09:00:e8:40:64:d2:9d:f9:1f:9f: + ab:67:1f:f9:c6:84:ba:7e:17:6c:8b:8d:08:ee:fb:8a:d7:cd: + 06:25:72:9f:4e:1a:c2:71:e1:1b:cf:a2:d7:1c:05:12:95:d6: + 49:4b:e9:95:95:89:cf:68:18:46:a3:ea:0d:9d:8e:ca:1c:28: + 55:49:6b:c0:4b:58:f5:42:b9:0a:ec:0e:6e:21:a4:ff:60:c0: + 1b:6e:40:72:d0:a5:c5:b5:db:4e:87:67:3a:31:70:cb:32:84: + 70:a9:e2:ff:e0:f2:db:cd:03:b4:85:45:d3:07:cc:0f:c7:49: + d8:c2:17:eb:73:f7:4a:c0:d9:8c:59:ef:c0:0a:ce:13:0b:84: + c9:aa:0d:11:14:b4:e5:74:aa:ec:18:de:5f:26:18:98:4a:76: + f0:7f:cd:e6:c4:b5:58:03:03:f5:10:01:5d:8f:63:88:ba:65: + d7:b4:7f:5a:1a:51:0e:ed:e5:68:fa:18:03:72:15:a1:ec:27: + 1f:ea:ac:24:46:18:6e:f1:97:db:4a:f4:d6:a1:91:a0:8c:b0: + 2f:be:87:3b:44:b0:8d:2a:89:85:5f:f2:d9:e3:2e:66:b2:88: + 98:04:2c:96:32:38:99:19:a9:83:fd:94:0c:dd:63:d4:1b:60: + 9d:43:98:35:ac:b4:23:38:de:7f:85:52:57:a0:37:df:a5:cf: + be:54:2c:3c:50:27:2b:d4:54:a9:9d:a3:d4:a5:b3:c0:ea:3d: + 0e:e2:70:6b:fb:cb:a5:56:05:ec:64:72:f0:1a:db:64:01:cb: + 5d:27:c4:a1:c4:63 -----BEGIN CERTIFICATE----- MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBALfjLdLq -Jd2Z290z/ZoUF7oK9G9nCGHWG4aXshBG12hqQ+owuUR00cm014XkzbvEt6L6njC3 -s3UnKLTaR4dBvWb1OofkCu46QTa2L7XnMSgqhT1pbJ1OIf56kpNPTKODY1wpCNFR -pZxuHMEfgQwq4IiXMgjPz8QgqQ6qEIPCrr4W/cDKz7PsHP7JHzSLGK8q7+XtKPFD -vPtH+QIjs5WJFwg7lAFA+PtUu8JPPhUDq78t1eeArYc25q7pZebnI01BuqsHXUAQ -lxhl/9xWud5XgX5mMuWgAqBKf0YHx5Y7KKVNDXxNwp34faJyKlFo//5BH8o/HkYr -9JdTebKqbrZA4Yju3/mS1V2MaGzq0dnHDVJGKKmIthZu1VGtLFrjBiWCmChTC+u+ -+0LmQA2dPDVUZazFyz7prv4+hxicmKruThvk4BRvdHK+X0MsAzLw2qlFajhwcP9d -DcUJjAUeU5KDH90vlJ7SLd1bUrNs/o69o2DqeX/URc9SwnUBNLyfO/OFxwIDAQAB +a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMYoDXNk +ON40+HYn1YdVdcxR/yHk3j9scIzcJshpnV1KGQKB+YVaY62u4SKGBt9FhpJZkDAK +ZUBAaOaC9R+A4iCwV8zzI77MMHI+HNrte9Y9l8R7uY1EV7lmVie1t4z++5YIzhXP +AOr+aoVMXE1GbAieoTuHHRad8KhBZ703jWEnKXUrQ6MANv5KOWknT9QnreGwH52j +AaNpoG8nwd25BxGo2DyzoVzIknwyOyzW5EwUraP1CuaBsDrVe4g8LHNzcnnuZbwU +3YqxHwPYc6F72exMaXGn47kK51/9EFYHjjfFyjJ27BFc4VN0QBTjkxLi0oY/aHN7 +O7EwVCcsoBFdVbj+AXFmTuRTxPB5lRLosBg99gmeV+15vrPfssBu2gO/MpQoEQN/ ++eahrsK4rFI8LkcgqEtuNT/qS4+CVJUR8k8la7ZtkkMCQUvzk63eA+zXQRfaUxTL +Qm3e1/5NpCMS8VmGko3woB1iPd5WA5/MgFtDKOi0cmMV70mSrirtGgj6dwIDAQAB o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd -BgNVHQ4EFgQUyL2otMDyMhBzR5xIgTL4ursmhJcwfQYDVR0jBHYwdIAUs4qgorpx -8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo +BgNVHQ4EFgQUHHAUsCDdCHakO1ZZ+l80+DZm6FYwfQYDVR0jBHYwdIAU8+yUjvKO +MMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l -dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHaHdk3k -D4i/LPNYZ8CXbM1ZGIKDTAQZpW2q+mQ9STI+4VaVshP3z9MRsHK3W+fXhWlRPLZU -gEUvKBAhILm66Sdatz+Ct2n1RvW/oosXf/IU0UaXtYtH+5/oXAUOnRG9fJoDhAvK -KWZKyg1vCR56J8F/A5ZwjRilL6SYpRmqjF0ejD67bTvAM8AV4b0JPZ/o3BLUy0Qd -BvXo1k6hLVyfXR9bKsNNQI3a0XiA0MYxchBIiukQehMwEbKeZw7toarscy3wuIoi -dQ8waVxQfpHO2pHHcIxl//ZY+wC9Rczi5OPlFjZ986JKnEX/2aUW4C+1W2zmihMV -SHO9fIAzw9Q7Oh2FDqT39/tIDOmgS16KXGf4JQJvzXLBqlqTZHwUIEPgE38N4Q1h -Xi4szXouKq62dWpfoBqbtmctsKUcVLyMcH4VK8BQ4wO7pKX8RQHJP6e4GNw+CAeh -m/W9lb1J6BB8kX0txMKYtrdRadcKaEC1D4WgqWd3xmjLDlg0s1jnyHwJZw== +dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHUU5WhF +je1s8SceDvM1rg5gwWU2YrgHeOG5jXpQcK8GydTuUO/SdrKitsvcphi1PdL36w7s +t5XNLrE2b6ifuE3/zorEjmI3MoA+BUpNOYdpCQDoQGTSnfkfn6tnH/nGhLp+F2yL +jQju+4rXzQYlcp9OGsJx4RvPotccBRKV1klL6ZWVic9oGEaj6g2djsocKFVJa8BL +WPVCuQrsDm4hpP9gwBtuQHLQpcW1206HZzoxcMsyhHCp4v/g8tvNA7SFRdMHzA/H +SdjCF+tz90rA2YxZ78AKzhMLhMmqDREUtOV0quwY3l8mGJhKdvB/zebEtVgDA/UQ +AV2PY4i6Zde0f1oaUQ7t5Wj6GANyFaHsJx/qrCRGGG7xl9tK9NahkaCMsC++hztE +sI0qiYVf8tnjLmayiJgELJYyOJkZqYP9lAzdY9QbYJ1DmDWstCM43n+FUlegN9+l +z75ULDxQJyvUVKmdo9Sls8DqPQ7icGv7y6VWBexkcvAa22QBy10nxKHEYw== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem index 81daa4ccb94217..bd109921898044 100644 --- a/Lib/test/certdata/keycertecc.pem +++ b/Lib/test/certdata/keycertecc.pem @@ -1,8 +1,8 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBcNwE+cm17mmr7Yg6d -0DNCnheGFOjkYH4tYzTyCkcZGShkmF/tKhIqb3imKz0Kx9+hZANiAATyp8ws6CuN -OI2/3MC4jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoG -JJYHhZNQXEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQAQ= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDRUbCeT3hMph4Y/ahL +1sy9Qfy4DYotuAP06UetzG6syv+EoQ02kX3xvazqwiJDrEyhZANiAAQef97STEPn +4Nk6C153VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn +4yLQbd/6YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qo= -----END PRIVATE KEY----- Certificate: Data: @@ -19,13 +19,13 @@ Certificate: Public Key Algorithm: id-ecPublicKey Public-Key: (384 bit) pub: - 04:f2:a7:cc:2c:e8:2b:8d:38:8d:bf:dc:c0:b8:8d: - 95:52:92:6a:03:ce:6f:d7:fd:9a:e4:12:6e:13:54: - 72:8f:49:9e:a4:cd:94:69:9a:65:25:2d:2f:65:ed: - 24:19:48:b6:32:0c:01:41:17:3a:06:24:96:07:85: - 93:50:5c:43:bb:1c:5b:33:9f:a2:a7:bc:b8:e1:c1: - 2e:9d:77:39:5a:1c:fc:e2:12:49:eb:4a:bd:0e:2a: - 12:66:74:91:eb:40:04 + 04:1e:7f:de:d2:4c:43:e7:e0:d9:3a:0b:5e:77:54: + 4c:76:e0:c3:64:25:47:26:2d:ee:fb:d6:ee:a5:af: + 74:3c:12:6d:c9:fd:83:da:03:58:bd:62:ce:ca:64: + 0d:d6:be:82:81:3e:85:9a:6a:29:27:e3:22:d0:6d: + df:fa:60:4e:80:02:33:2e:69:38:65:47:5f:dc:a8: + 7e:50:9e:64:b7:0d:71:d4:8e:6c:67:2e:35:b2:fd: + d6:65:d0:22:46:d6:aa ASN1 OID: secp384r1 NIST CURVE: P-384 X509v3 extensions: @@ -38,69 +38,66 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 79:11:98:86:15:4F:48:F4:31:0B:D2:CC:C8:26:3A:09:07:5D:96:40 + 45:ED:32:14:6D:51:A2:3B:B0:80:55:E0:A6:9B:74:4C:A5:56:88:B1 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 6e:42:e8:a2:2d:28:14:e3:25:5c:c1:7e:54:e9:3a:ff:30:db: - 94:ba:b2:f6:5f:ae:9a:c1:90:b3:4f:ce:65:1d:84:64:c0:71: - 2c:44:8e:7e:00:79:f5:8c:4a:1d:34:13:44:de:99:2e:db:53: - ee:ec:74:97:4d:59:1a:09:82:4f:98:75:91:a7:a0:b9:da:5e: - 68:f5:32:85:be:36:3d:83:d4:ee:f9:87:67:31:85:41:53:9a: - e7:05:96:13:1c:88:2e:7f:33:b1:ee:bd:f9:50:52:24:ed:3d: - 92:95:6e:30:c3:af:74:a9:ee:15:bb:da:7c:14:50:8e:e3:99: - ea:ba:b4:37:8a:50:61:26:de:01:93:b8:a2:6b:d9:c7:38:5e: - b2:f8:96:3d:a8:9f:7d:0c:71:d4:7e:cc:a0:57:af:7e:ce:3f: - a7:a7:27:68:c1:28:d7:4f:44:c1:b4:93:c3:c7:35:2b:50:c3: - 8e:2c:d0:46:c1:3f:e1:67:d3:f0:81:ae:f3:5c:3e:4f:d5:a8: - 07:8f:e0:eb:ef:d8:dc:47:e0:3d:58:eb:de:0e:7f:b2:58:cb: - 5c:f1:2f:65:7e:0f:0d:cc:ca:ba:83:53:63:bc:dd:18:0c:ee: - ed:ec:96:88:d0:38:c5:d7:ab:e7:55:79:7b:6d:ba:c0:a0:e9: - 5c:ca:7c:fb:f8:70:c7:fb:f5:b2:b5:74:cb:f7:c0:0d:20:9f: - 1d:b7:4c:bf:8a:8d:cd:e3:bc:4e:30:78:02:12:a0:9b:d5:8f: - 49:3c:95:91:76:6e:7c:54:dc:61:7a:2e:20:ed:35:25:e0:c5: - 17:50:02:83:00:74:8f:f0:1c:97:96:08:fc:2e:63:a4:f7:97: - 87:43:2a:32:04:2d:4c:f9:1a:07:bf:68:91:fc:50:21:a1:3c: - 8d:8f:fb:83:57:83:1f:b6:55:5c:55:2f:58:64:ad:f3:27:ba: - d0:e3:cd:58:01:a3:c9:ba:1d:95:dc:30:d5:af:b9:20:ad:d9: - 48:ba:8d:9a:66:ee + Signature Value: + 07:e4:91:0b:d3:ed:4b:52:7f:50:68:c7:8d:80:48:9f:b7:4a: + 13:66:bf:9d:4c:2d:18:19:68:a0:da:3b:12:85:05:16:fa:8d: + 9c:58:c6:81:b3:96:ba:11:62:65:d3:76:f1:1c:ab:95:e4:d8: + 2a:e0:1f:7b:c5:20:2e:7c:8f:de:87:7a:2b:52:54:ca:d1:41: + b0:5e:20:72:df:44:00:4a:69:1a:ef:10:63:52:13:ed:49:02: + ee:dc:9d:f3:c8:ba:c4:01:81:5a:a9:1c:15:12:b6:21:de:44: + a5:fd:7e:f9:22:d1:3e:ee:22:dd:31:55:32:4e:41:68:27:c5: + 95:1b:7e:6b:18:74:f9:22:d6:b7:b9:31:72:51:a0:5a:2c:ff: + 62:76:e9:a0:55:8d:78:33:52:4a:58:b2:f4:4b:0c:43:82:2f: + a9:84:68:05:dd:11:47:70:24:fe:5c:92:fd:17:21:63:bb:fa: + 93:fa:54:54:05:72:48:ed:81:48:ab:95:fc:6d:a8:62:96:f9: + 3b:e2:71:18:05:3e:76:bb:df:95:17:7b:81:4b:1f:7f:e1:67: + 76:c4:07:cb:65:a7:f2:cf:e6:b4:fb:75:7c:ee:df:a1:f5:34: + 20:2b:48:fd:2e:49:ff:f3:a6:3b:00:49:6c:88:79:ed:9c:16: + 2a:04:72:e2:93:e4:7e:3f:2a:dd:30:47:9a:99:84:2a:b9:c4: + 40:31:a6:68:f3:20:d1:75:f1:1e:c8:18:64:5b:f8:4c:ce:9a: + 3c:57:2c:e3:63:64:29:0a:c2:b6:8e:20:01:55:9f:fe:10:ba: + 12:42:38:0a:9b:53:01:a5:b4:08:76:ec:e8:a6:fc:69:2c:f7: + 7f:5e:0f:44:07:55:e1:7c:2e:58:e5:d6:fc:6f:c2:4d:83:65: + bd:f3:32:e3:14:48:22:8d:80:18:ea:44:f8:24:79:ff:ff:c6: + 04:c2:e9:90:34:40:d6:59:3f:59:1e:4a:9a:58:60:ce:ab:f9: + 76:0e:ef:f7:05:17 -----BEGIN CERTIFICATE----- MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATyp8ws6CuNOI2/3MC4 -jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoGJJYHhZNQ -XEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQASjggHEMIIBwDAYBgNV +Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQef97STEPn4Nk6C153 +VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn4yLQbd/6 +YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qqjggHEMIIBwDAYBgNV HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUeRGY -hhVPSPQxC9LMyCY6CQddlkAwfQYDVR0jBHYwdIAUs4qgorpx8agkedSkWyU2FR5J -yM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQURe0y +FG1RojuwgFXgppt0TKVWiLEwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/jmoZwMGf +dmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAG5C6KItKBTjJVzBflTpOv8w -25S6svZfrprBkLNPzmUdhGTAcSxEjn4AefWMSh00E0TemS7bU+7sdJdNWRoJgk+Y -dZGnoLnaXmj1MoW+Nj2D1O75h2cxhUFTmucFlhMciC5/M7HuvflQUiTtPZKVbjDD -r3Sp7hW72nwUUI7jmeq6tDeKUGEm3gGTuKJr2cc4XrL4lj2on30McdR+zKBXr37O -P6enJ2jBKNdPRMG0k8PHNStQw44s0EbBP+Fn0/CBrvNcPk/VqAeP4Ovv2NxH4D1Y -694Of7JYy1zxL2V+Dw3MyrqDU2O83RgM7u3slojQOMXXq+dVeXttusCg6VzKfPv4 -cMf79bK1dMv3wA0gnx23TL+Kjc3jvE4weAISoJvVj0k8lZF2bnxU3GF6LiDtNSXg -xRdQAoMAdI/wHJeWCPwuY6T3l4dDKjIELUz5Gge/aJH8UCGhPI2P+4NXgx+2VVxV -L1hkrfMnutDjzVgBo8m6HZXcMNWvuSCt2Ui6jZpm7g== +b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAAfkkQvT7UtSf1Box42ASJ+3 +ShNmv51MLRgZaKDaOxKFBRb6jZxYxoGzlroRYmXTdvEcq5Xk2CrgH3vFIC58j96H +eitSVMrRQbBeIHLfRABKaRrvEGNSE+1JAu7cnfPIusQBgVqpHBUStiHeRKX9fvki +0T7uIt0xVTJOQWgnxZUbfmsYdPki1re5MXJRoFos/2J26aBVjXgzUkpYsvRLDEOC +L6mEaAXdEUdwJP5ckv0XIWO7+pP6VFQFckjtgUirlfxtqGKW+TvicRgFPna735UX +e4FLH3/hZ3bEB8tlp/LP5rT7dXzu36H1NCArSP0uSf/zpjsASWyIee2cFioEcuKT +5H4/Kt0wR5qZhCq5xEAxpmjzINF18R7IGGRb+EzOmjxXLONjZCkKwraOIAFVn/4Q +uhJCOAqbUwGltAh27Oim/Gks939eD0QHVeF8Lljl1vxvwk2DZb3zMuMUSCKNgBjq +RPgkef//xgTC6ZA0QNZZP1keSppYYM6r+XYO7/cFFw== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.ca.pem b/Lib/test/certdata/leaf-missing-aki.ca.pem new file mode 100644 index 00000000000000..36b202ae02ee54 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.ca.pem @@ -0,0 +1,13 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUGjIb/aYm9u9fBh2o4GAYRJwk5XIwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAaMRgwFgYDVQQDDA94NTA5LWxpbWJvLXJvb3QwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAARUzBhjMOkO911U65Fvs4YmL1YPNj63P9Fa+g9U +KrUqiIy8WjaDXdIe8g8Zj0TalpbU1gYCs3atteMxgIp6qxwHo1cwVTAPBgNVHRMB +Af8EBTADAQH/MAsGA1UdDwQEAwICBDAWBgNVHREEDzANggtleGFtcGxlLmNvbTAd +BgNVHQ4EFgQUcv1fyqgezMGzmo+lhmUkdUuAbIowCgYIKoZIzj0EAwIDSQAwRgIh +AIOErPSRlWpnyMub9UgtPF/lSzdvnD4Q8KjLQppHx6oPAiEA373p4L/HvUbs0xg8 +6/pLyn0RT02toKKJcMV3ChohLtM= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.keycert.pem b/Lib/test/certdata/leaf-missing-aki.keycert.pem new file mode 100644 index 00000000000000..0fd2ab39bf2c70 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.keycert.pem @@ -0,0 +1,18 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIF5Re+/FP3rg+7c1odKEQPXhb9V65kXnlZIWHDG9gKrLoAoGCCqGSM49 +AwEHoUQDQgAE1WAQMdC7ims7T9lpK9uzaCuKqHb/oNMbGjh1f10pOHv3Z+oAvsqF +Sv3hGzreu69YLy01afA6sUCf1AA/95dKkg== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIBjjCCATWgAwIBAgIUVlBgclml+OXlrWzZfcgYCiNm96UwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABNVgEDHQu4prO0/ZaSvbs2griqh2/6DTGxo4dX9dKTh7 +92fqAL7KhUr94Rs63ruvWC8tNWnwOrFAn9QAP/eXSpKjWzBZMB0GA1UdDgQWBBS3 +yYRQQwo3syjGVQ8Yw7/XRZHbpzALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwFgYDVR0RBA8wDYILZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDRwAwRAIg +BVq7lw4Y5MPEyisPhowMWd4KnERupdM5qeImDO+dD7ICIE/ksd6Wz1b8rMAfllNV +yiYst9lfwTd2SkFgdDNUDFud +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py index 94a35a64ab1abe..6626b93976a585 100644 --- a/Lib/test/certdata/make_ssl_certs.py +++ b/Lib/test/certdata/make_ssl_certs.py @@ -109,7 +109,8 @@ subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer - basicConstraints = CA:true + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, keyCertSign, cRLSign """ diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem index ec10cdcabb9e35..c6ff8ea31bfa2e 100644 --- a/Lib/test/certdata/nosan.pem +++ b/Lib/test/certdata/nosan.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCv3sUoOE4F7Pye -AT2Q6XpXrGUOu1fYgdnItLLLhvn7ACuHMj7TA5UKXxsepJn5m2Ji9LvAbksr1IWd -LZAvNgjwsUR+E4HbY108BhVt9sk3HFkvE0OOFbAa14ICtYPe18P/4Hv6Zfu/GJDU -rwXHNCUu0p6i/mospZ5O3sx5MgVaShknGAEC3Kp7zOgusMmE8XSbkNQa3ARMkW4o -kTqWKAeAHDjVFVyyhzZQmo+gaLzhWfJVSZhlJsuiLoZGGrVTq85EiXsE4l8rPaI+ -mKkVzWP13IZW+Fx1tiIktumdHWb1OQWrvm8AiT9b8PcFCUUrvhQFcLDSCZjKlQ0t -RWrSSKrrVsSldOreqRLtpjGzFJpGnTcvslL7rP5pg5DjBsYmVcDjrmRuJuhGq52X -/6HEC97GouVK8tT1LVMv1wufVPn+i9TzwxOuRWeUvVqLAJgWQ9N3yKdymH+VrpZk -/oB9ScyDakGezZBW5CeOQbNJ8WoX58jNxefGjtqKxmyztu43r3ECAwEAAQKCAYBQ -fVoqYCqFV8L95X9x1QljGsldhqxbsIIl811o/KtoDtndFEfgd2E8z+4vhhHaRR0w -QOW02kWZF7jXCMVWdhp9XgQE15S0/bLsB7TDERFiIZ1HiD+AxbhFcKBV8REbahCQ -CQN0xDwFZ47RaBDy7JCf71EfM+UP7fSYECvww83jVspQNBIyZx+3bT5OMCbqqz88 -+3m3mT52dJDADEeN9WAJZ+Ey1IYKRwu6tCJLvePEF1BrbDVNBgZogXZ+mzalxpjr -4RpGPMMa+VWc8HmDVd+LtpwKJcQD00GvUP4fNywn+5jvNWl54FdQiTLPrieTWxas -XUQ2crxP7Aqr2/vsU5Ruru5uF7H+ssMHp9YQDhpJ2+SVhQ9P+/loXCuKGt+BrB2Z -MlitO3f+vfRtzATmJ8G0qFrOqZK1A/qsiyIze240C1hAl3oy2xpZqTDGp4gRWwoi -OIN0HmH9UbP7bbNQY1x/zstTbza4/7rGb1+DZKeZIMu7QjBCU0rtsJpGtUvcQGEC -gcEA42GMYSL/HljZMF1LsDhTX/cmP8FDNgONhWYxT+w0Csnj1usLNBaT63dYnEiW -QKydRR4casAR1Kdy4Yfcy2lCy1kCfwqkQYk8fxSsOSHRjUfwC1SnfdYlwKFMxw4a -oZF0R4oVCBYrfP+8kqrj+5gs/gXblsw72XkYtbCdIriKKdmUzTx7MegzSqh2PVRi -rJzuwCZQ/O0NfhwdOHxLQDo0dgD+vv9e+KOSoJ9FDv8HH1tnolpRMdkSA8AJR/Nk -DXt1AoHBAMYBfTKQZ2jqLKybe4tP+YKjvjVp8vJx0iNUXFN/P6hBaSBOgq85uxXL -X3s7N/pkOCjyE95B8QusIkbnbfdyEP89O4bTbUHPXyAkHyRkR7Vny49HYuaR/aXQ -mXC0J2z5bXVpCQ514l/R/Io3wBph+hbG3To7pp9pMOV4qzvibUZaTZFwH+q+xDwf -SKSFy3fcomgH4/K5/QuKVj0jOUQsYjQQWb8GukS2KZK3zYJIAG1bBcsCVpSuBdW0 -eCZgqjnwjQKBwCUyUwWc9QEg5b68tGIKhNEhHDe3xOf0ItWcxxpc+JJ/Pm9tGfMW -cnJFntBKK5I+6qdg6qMn8oLINcnhMORxvsSHNhpUQlSaP7RGTHo4JxCmoQUpfxDd -1GUzvdyeWQrvQYdmdlRRVCHpsA6KOCtzVIDlsmtz06Ka5cjrMHl6mNeJyYbdiwW6 -B5ICBv23bUDxlzkFy5/ko51qufkAlErYeraHKSVTn1SrZZQzGdf/LkoZ6NUtUzUF -XqYQZzRHA6oU9QKBwDslzLljC5D6ivfQxln6POV6dmJMUOd9erFVDPNgSqq/R2EA -MueXDjzXcKFGMlWYxHHuxmKZPiEnfWHC1kWZjFxCdVq0I6oKATd/stHTJtyYseUO -BQwtRiDXLE7PcguKgtkU1EC+lC3dc1vyhW8cH3HYW9N+aCqsaI/TuQr9e3kNlqhA -XzhnXgU7rx5+XSZkARukZ8JlLqLY4yQGNqAXxgoZbEW1A8VsyQRr5XbqfT4td5CK -FUT6qwGIlG+aZp9CLQKBwQCQkwdW9A/Q4Ffq8+XTL1hJ24m/q11OLAPODUypOhWw -OCbX2fkv59pSBe6niZDBls1NpHB9mzalBrJCfU+yKC667gKcKULOnWULIoOQvmcg -Ka3hkkW28gTnCjfDIYm3IdsLjc67zJplOixaKgxhO8NtJZGtg0oLIrofG8EYRInv -OmtGw+XE+s4TVs6WgXnEg9zWQ5ZYtqQVn6PT5jsz+Nrvipi61HWHVBd7g+78ojps -3suWxl0FvgzTW5HD16WRXeI= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC99xEYPTwFN/ji +i0lm11ckEGhcxciSsIgTgior54CLgQy7JXllTYmAWFTTg2zNBvDMexGI0h+xtZ4q +1Renghgt33N3Y6CT3v/L7JkE1abQbFveKW/ydlxH0+jLlsENSWjySwC80+f9L3bX +TcD8T4Fu9Uty2Rg1a/Eyekng5RmfkmLNgxfnX5R5nWhh0Aia7h3Ax2zCALfxqZIm +fxwavEgHsW/yZi+T+eoJwe0i7a6LaUoLqsPV9ZhagziNDaappPHH42NW39WlRhx1 +UjtiRm2Jihnzxcfs+90zLXSp5pxo/cE9Ia4d8ieq3Rxd/XgjlF6FXXFJjwfL36Dw +ehy8m3PKKAuO+fyMgPPPMQb7oaRy/MBG0NayRreTwyKILS2zafIW/iKpgICbxrWJ +r/H1b3S6PBKYUE2uQs0/ZPnRjjh0VeNnue7JcRoNbe27I2d56KUBsVEPdokjU59v +NYi6Se+ViZXtUbM1u/I0kvDMprAiobwtJFYgcE86N1lFJjHSwDMCAwEAAQKCAYBb +lvnJBA0iPwBiyeFUElNTcg2/XST9hNu2/DU1AeM6X7gxqznCnAXFudD8Qgt9NvF2 +xYeIvjbFydk+sYs8Gj9qLqhPUdukMAqI2cRVTmWla/lHPhdZgbOwdf1x23es3k4Z +NAxg/pKFwhK8cCKyA+tWAjKkZwODDk42ljt0kUEvbLbye1hVGAJQOJKRRmo/uLrj +rcNELnCBtc5ffT2hrlHUU7qz1ozt/brXhYa+JnbXhKZMxcKyMD2KtmXXrFNEy99o +jXbrpDCos82bzQfPDo8IpCbVbEd2J00aFmrNjQWhZuXX5dXflrujW4J0nzeHrZ78 +rNAz2/YuZ543BTB3XbogeFuLC5RqBgAMmw2WJ96Oa/UG8nZNvEw54N5r6dhfXj6A +VlJFLVwlfBQdAdaM3P4uZ6WECrH3EerQa27qyUdRrcDaGPLt7wG9FmMivnW1KQsy +5ow/gM0CsxFj2xNoGw1S5jtclbgSy8HNJaBsNk4XMQ+ORABZdG1MTTE+GMSjD/EC +gcEA+6JYiZEo+QrvItIZYB6Go4suu/F8df1pEjJlxwp2GmObitRhtV6r9g9IySFv +5SL7ZxARr4aQxvM7fNp57p9ssmkBtY0ofMjJAxhvs4T37bAcGK/2xCegNSmbqh24 +FAWpRDMgE5PjtuWC5jTvSOYFeUxwI/cu0HxWdxJl2dPUSL1nI2jP+ok3pZobEQk9 +E2+MlHpKmU+s/lAkuQiP+AW9a4M+ZJNWxocJjmtwj4OjJXPm7GddA/5x622DxFe6 +4K2vAoHBAMFC0An25bwGoFrCV/96s45K4qZcZcJ660+aK3xXaq6/8KfiusJnWds2 +nc0B6jYjKs8A7yTAGXke6fmyVsoLosZiXsbpW2m16g8jL79Tc85O9oDNmDIGk1uT +tRLZc2BvmHmy/dNrdbT/EHC3FKNWQVqWc2sHhPeB6F3hIEXDSUO/GB0njMZNXrPJ +547RlhN0xCLb3vTzzGHwNmwfI81YjV/XI4vpJjq1YceN8Xyd1r5ZOFfU8woIACO3 +I4dvBQ1avQKBwQCLDs9wzohfAFzg2Exvos7y6AKemDgYmD8NcE5wbWaQ9MTLNsz8 +RuIu64lkpRbKAMf/z5CGeI3fdCFGwRGq/e06tu7b3rMmKmtzS3jHM08zyiPsvKlZ +AzD00BaXLy8/2VUOPFaYmxy3QSRShaRKm9sgik5agcocKuo5iTBB7V8eB5VMqyps +IJJg8MXOZ1WaPQXqM56wFKjcLXvtyT6OaNWh6Xh8ajQFKDDuxI8CsFNjaiaONBzi +DSX1XaL4ySab7T8CgcEAsI+7xP6+EDP1mDVpc8zD8kHUI6zSgwUNqiHtjKHIo3JU +CO2JNkZ5v158eGlBcshaOdheozKlkxR9KlSWGezbf2crs4pKq585AS9iVeeGK3vU +lQRAAaQkSEv/6AKl9/q8UKMIZnkMhploibGZt0f8WSiOtb+e6QjUI8CjXVj2vF// +RdN2N01EMflKBh7Qf2H0NuytGxkJJojxD4K7kMVQE7lXjmEpPgWsGUZC01jYcfrN +EOFKUWXRys9sNDVnZjX5AoHAFRyOC1BlmVEtcOsgzum4+JEDWvRnO1hP1tm68eTZ +ijB/XppDtVESpq3+1+gx2YOmzlUNEhKlcn6eHPWEJwdVxJ87Gdh03rIV/ZQUKe46 +3+j6l/5coN4WfCBloy4b+Tcj+ZTL4sKaLm33RoD2UEWS5mmItfZuoEFQB958h3JD +1Ka1tgsLnuYGjcrg+ALvbM5nQlefzPqPJh0C8UV3Ny/4Gd02YgHw7Yoe4m6OUqQv +hctFUL/gjIGg9PVqTWzVVKaI -----END PRIVATE KEY----- Certificate: Data: @@ -51,80 +51,81 @@ Certificate: Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:af:de:c5:28:38:4e:05:ec:fc:9e:01:3d:90:e9: - 7a:57:ac:65:0e:bb:57:d8:81:d9:c8:b4:b2:cb:86: - f9:fb:00:2b:87:32:3e:d3:03:95:0a:5f:1b:1e:a4: - 99:f9:9b:62:62:f4:bb:c0:6e:4b:2b:d4:85:9d:2d: - 90:2f:36:08:f0:b1:44:7e:13:81:db:63:5d:3c:06: - 15:6d:f6:c9:37:1c:59:2f:13:43:8e:15:b0:1a:d7: - 82:02:b5:83:de:d7:c3:ff:e0:7b:fa:65:fb:bf:18: - 90:d4:af:05:c7:34:25:2e:d2:9e:a2:fe:6a:2c:a5: - 9e:4e:de:cc:79:32:05:5a:4a:19:27:18:01:02:dc: - aa:7b:cc:e8:2e:b0:c9:84:f1:74:9b:90:d4:1a:dc: - 04:4c:91:6e:28:91:3a:96:28:07:80:1c:38:d5:15: - 5c:b2:87:36:50:9a:8f:a0:68:bc:e1:59:f2:55:49: - 98:65:26:cb:a2:2e:86:46:1a:b5:53:ab:ce:44:89: - 7b:04:e2:5f:2b:3d:a2:3e:98:a9:15:cd:63:f5:dc: - 86:56:f8:5c:75:b6:22:24:b6:e9:9d:1d:66:f5:39: - 05:ab:be:6f:00:89:3f:5b:f0:f7:05:09:45:2b:be: - 14:05:70:b0:d2:09:98:ca:95:0d:2d:45:6a:d2:48: - aa:eb:56:c4:a5:74:ea:de:a9:12:ed:a6:31:b3:14: - 9a:46:9d:37:2f:b2:52:fb:ac:fe:69:83:90:e3:06: - c6:26:55:c0:e3:ae:64:6e:26:e8:46:ab:9d:97:ff: - a1:c4:0b:de:c6:a2:e5:4a:f2:d4:f5:2d:53:2f:d7: - 0b:9f:54:f9:fe:8b:d4:f3:c3:13:ae:45:67:94:bd: - 5a:8b:00:98:16:43:d3:77:c8:a7:72:98:7f:95:ae: - 96:64:fe:80:7d:49:cc:83:6a:41:9e:cd:90:56:e4: - 27:8e:41:b3:49:f1:6a:17:e7:c8:cd:c5:e7:c6:8e: - da:8a:c6:6c:b3:b6:ee:37:af:71 + 00:bd:f7:11:18:3d:3c:05:37:f8:e2:8b:49:66:d7: + 57:24:10:68:5c:c5:c8:92:b0:88:13:82:2a:2b:e7: + 80:8b:81:0c:bb:25:79:65:4d:89:80:58:54:d3:83: + 6c:cd:06:f0:cc:7b:11:88:d2:1f:b1:b5:9e:2a:d5: + 17:a7:82:18:2d:df:73:77:63:a0:93:de:ff:cb:ec: + 99:04:d5:a6:d0:6c:5b:de:29:6f:f2:76:5c:47:d3: + e8:cb:96:c1:0d:49:68:f2:4b:00:bc:d3:e7:fd:2f: + 76:d7:4d:c0:fc:4f:81:6e:f5:4b:72:d9:18:35:6b: + f1:32:7a:49:e0:e5:19:9f:92:62:cd:83:17:e7:5f: + 94:79:9d:68:61:d0:08:9a:ee:1d:c0:c7:6c:c2:00: + b7:f1:a9:92:26:7f:1c:1a:bc:48:07:b1:6f:f2:66: + 2f:93:f9:ea:09:c1:ed:22:ed:ae:8b:69:4a:0b:aa: + c3:d5:f5:98:5a:83:38:8d:0d:a6:a9:a4:f1:c7:e3: + 63:56:df:d5:a5:46:1c:75:52:3b:62:46:6d:89:8a: + 19:f3:c5:c7:ec:fb:dd:33:2d:74:a9:e6:9c:68:fd: + c1:3d:21:ae:1d:f2:27:aa:dd:1c:5d:fd:78:23:94: + 5e:85:5d:71:49:8f:07:cb:df:a0:f0:7a:1c:bc:9b: + 73:ca:28:0b:8e:f9:fc:8c:80:f3:cf:31:06:fb:a1: + a4:72:fc:c0:46:d0:d6:b2:46:b7:93:c3:22:88:2d: + 2d:b3:69:f2:16:fe:22:a9:80:80:9b:c6:b5:89:af: + f1:f5:6f:74:ba:3c:12:98:50:4d:ae:42:cd:3f:64: + f9:d1:8e:38:74:55:e3:67:b9:ee:c9:71:1a:0d:6d: + ed:bb:23:67:79:e8:a5:01:b1:51:0f:76:89:23:53: + 9f:6f:35:88:ba:49:ef:95:89:95:ed:51:b3:35:bb: + f2:34:92:f0:cc:a6:b0:22:a1:bc:2d:24:56:20:70: + 4f:3a:37:59:45:26:31:d2:c0:33 Exponent: 65537 (0x10001) Signature Algorithm: sha256WithRSAEncryption - 91:42:c2:15:57:42:47:77:e7:0f:c5:55:26:b1:5b:c3:5e:ba: - 81:db:e1:a4:9f:b8:42:5a:21:c9:8c:18:ae:0f:90:ab:9a:24: - e7:d2:78:fc:bd:97:29:b1:5c:46:1f:5b:b8:d2:a7:87:f1:50: - 53:5b:d3:be:57:74:bd:e5:75:db:50:81:f7:37:95:0b:69:ef: - 39:8c:5c:82:d5:64:62:d5:8b:e9:e0:31:e1:73:d2:5a:2c:de: - 43:5a:06:e5:d3:4d:d0:35:e0:9f:c2:73:31:bc:35:69:d4:fb: - 7d:f0:1a:33:f7:f6:25:72:9c:a6:84:05:08:f6:b5:e8:04:10: - f1:1f:f2:95:ad:a1:f8:d8:80:a5:eb:75:43:99:33:90:0c:79: - fc:c0:87:08:95:20:aa:c2:81:0b:22:6f:56:f4:8f:2a:23:f8: - 40:47:1c:03:a5:b1:04:0a:04:4a:df:d0:88:a8:bc:31:f2:42: - 9b:d8:11:14:9e:e3:68:ea:07:2c:15:de:d2:36:5a:15:38:ed: - d2:af:0e:b4:b6:1d:a0:57:94:ea:c3:c7:4c:14:57:81:00:57: - 94:d3:b0:27:69:d7:48:02:6c:e5:97:f7:be:22:7c:38:24:af: - b2:b0:7b:08:75:1e:ca:2e:c7:41:ef:8b:74:cf:c9:c3:6f:39: - b9:52:41:18:c6:70:24:54:51:04:fe:5f:88:70:35:e5:1c:8e: - d6:67:69:44:44:33:9b:8c:fe:a5:b9:95:48:66:84:f3:1a:04: - ab:a3:57:c1:b6:b4:2f:28:12:45:2b:cb:42:d3:f4:a5:ce:7b: - 6c:1f:e4:c8:a9:e7:d4:6d:c8:27:2d:69:26:c5:e8:73:10:54: - 1f:c3:bf:fd:aa:f5:95:6f:f6:ca:d5:06:8f:1b:79:93:e3:86: - ba:8d:fe:a8:10:8f:95:3e:14:09:bf:ca:88:59:e2:93:b6:ec: - 03:a9:7e:dd:1f:5f:13:d3:29:b3:a6:f3:6a:df:30:53:44:c8: - cd:e5:82:57:bc:9c + Signature Value: + 7e:dd:64:64:92:6c:b9:41:ce:f3:e3:f8:e6:9f:c8:5b:32:39: + 8c:03:5b:5e:7e:b3:23:ca:6c:d1:99:2f:53:af:9d:3c:84:cd: + c6:ce:0a:ee:94:de:ff:a7:06:81:7e:e2:38:a5:05:39:58:22: + dc:13:83:53:e7:f8:16:cb:93:dc:cf:4b:e6:1b:9f:9e:71:ef: + ee:ba:ea:b6:68:5c:32:22:7e:54:4f:46:a6:0b:11:8f:ef:05: + 6e:d3:0b:d0:a8:be:95:23:a2:e4:e7:a8:a2:a4:7d:98:52:86: + a4:15:fb:74:7a:9a:89:23:43:20:26:3a:56:9e:a3:6e:54:02: + 76:4e:25:9c:a1:8c:03:99:e5:eb:a6:61:b4:9c:2a:b1:ed:eb: + 94:f9:14:aa:a4:c3:f0:f7:7a:03:a3:b1:f8:c0:83:79:ab:8a: + 93:7f:0a:95:08:50:ff:55:19:ac:28:a2:c8:9f:a6:77:72:a3: + da:37:a9:ff:f3:57:70:c8:65:d9:55:14:84:b4:b3:78:86:82: + da:84:2c:48:19:51:ec:9d:20:b1:4d:18:fb:82:9f:7b:a7:80: + 22:69:25:83:4d:bf:ac:31:64:f5:39:11:f1:ed:53:fb:67:ab: + 91:86:c5:4d:87:e8:6b:fe:9a:84:fe:6a:92:6b:62:c1:ae:d2: + f0:cb:06:6e:f3:50:f4:8d:6d:fa:7d:6a:1c:64:c3:98:91:da: + c9:8c:a9:79:e5:48:4c:a2:de:42:28:e8:0e:9f:52:6a:a4:e0: + c7:ac:11:9c:ba:5d:d6:84:93:56:28:f1:6d:83:aa:62:b2:b7: + 56:c6:64:d9:96:4e:97:ab:4e:8f:ba:f6:ab:b9:17:52:98:32: + 7f:b5:12:fa:39:d7:34:2a:f3:ed:40:90:6f:66:7b:b6:c1:9d: + b9:53:d0:e3:e9:69:8c:cf:7a:fd:08:0a:62:47:d4:ce:72:f7: + 6f:80:b4:1d:18:7a:ba:a2:a9:45:49:ef:9c:0b:99:89:03:ab: + 5f:7a:9d:c5:77:b7 -----BEGIN CERTIFICATE----- MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC -AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAK/exSg4TgXs/J4BPZDpeles -ZQ67V9iB2ci0ssuG+fsAK4cyPtMDlQpfGx6kmfmbYmL0u8BuSyvUhZ0tkC82CPCx -RH4TgdtjXTwGFW32yTccWS8TQ44VsBrXggK1g97Xw//ge/pl+78YkNSvBcc0JS7S -nqL+aiylnk7ezHkyBVpKGScYAQLcqnvM6C6wyYTxdJuQ1BrcBEyRbiiROpYoB4Ac -ONUVXLKHNlCaj6BovOFZ8lVJmGUmy6IuhkYatVOrzkSJewTiXys9oj6YqRXNY/Xc -hlb4XHW2IiS26Z0dZvU5Bau+bwCJP1vw9wUJRSu+FAVwsNIJmMqVDS1FatJIqutW -xKV06t6pEu2mMbMUmkadNy+yUvus/mmDkOMGxiZVwOOuZG4m6EarnZf/ocQL3sai -5Ury1PUtUy/XC59U+f6L1PPDE65FZ5S9WosAmBZD03fIp3KYf5WulmT+gH1JzINq -QZ7NkFbkJ45Bs0nxahfnyM3F58aO2orGbLO27jevcQIDAQABMA0GCSqGSIb3DQEB -CwUAA4IBgQCRQsIVV0JHd+cPxVUmsVvDXrqB2+Gkn7hCWiHJjBiuD5CrmiTn0nj8 -vZcpsVxGH1u40qeH8VBTW9O+V3S95XXbUIH3N5ULae85jFyC1WRi1Yvp4DHhc9Ja -LN5DWgbl003QNeCfwnMxvDVp1Pt98Boz9/YlcpymhAUI9rXoBBDxH/KVraH42ICl -63VDmTOQDHn8wIcIlSCqwoELIm9W9I8qI/hARxwDpbEECgRK39CIqLwx8kKb2BEU -nuNo6gcsFd7SNloVOO3Srw60th2gV5Tqw8dMFFeBAFeU07AnaddIAmzll/e+Inw4 -JK+ysHsIdR7KLsdB74t0z8nDbzm5UkEYxnAkVFEE/l+IcDXlHI7WZ2lERDObjP6l -uZVIZoTzGgSro1fBtrQvKBJFK8tC0/SlzntsH+TIqefUbcgnLWkmxehzEFQfw7/9 -qvWVb/bK1QaPG3mT44a6jf6oEI+VPhQJv8qIWeKTtuwDqX7dH18T0ymzpvNq3zBT -RMjN5YJXvJw= +AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAL33ERg9PAU3+OKLSWbXVyQQ +aFzFyJKwiBOCKivngIuBDLsleWVNiYBYVNODbM0G8Mx7EYjSH7G1nirVF6eCGC3f +c3djoJPe/8vsmQTVptBsW94pb/J2XEfT6MuWwQ1JaPJLALzT5/0vdtdNwPxPgW71 +S3LZGDVr8TJ6SeDlGZ+SYs2DF+dflHmdaGHQCJruHcDHbMIAt/GpkiZ/HBq8SAex +b/JmL5P56gnB7SLtrotpSguqw9X1mFqDOI0Npqmk8cfjY1bf1aVGHHVSO2JGbYmK +GfPFx+z73TMtdKnmnGj9wT0hrh3yJ6rdHF39eCOUXoVdcUmPB8vfoPB6HLybc8oo +C475/IyA888xBvuhpHL8wEbQ1rJGt5PDIogtLbNp8hb+IqmAgJvGtYmv8fVvdLo8 +EphQTa5CzT9k+dGOOHRV42e57slxGg1t7bsjZ3nopQGxUQ92iSNTn281iLpJ75WJ +le1RszW78jSS8MymsCKhvC0kViBwTzo3WUUmMdLAMwIDAQABMA0GCSqGSIb3DQEB +CwUAA4IBgQB+3WRkkmy5Qc7z4/jmn8hbMjmMA1tefrMjymzRmS9Tr508hM3Gzgru +lN7/pwaBfuI4pQU5WCLcE4NT5/gWy5Pcz0vmG5+ece/uuuq2aFwyIn5UT0amCxGP +7wVu0wvQqL6VI6Lk56iipH2YUoakFft0epqJI0MgJjpWnqNuVAJ2TiWcoYwDmeXr +pmG0nCqx7euU+RSqpMPw93oDo7H4wIN5q4qTfwqVCFD/VRmsKKLIn6Z3cqPaN6n/ +81dwyGXZVRSEtLN4hoLahCxIGVHsnSCxTRj7gp97p4AiaSWDTb+sMWT1ORHx7VP7 +Z6uRhsVNh+hr/pqE/mqSa2LBrtLwywZu81D0jW36fWocZMOYkdrJjKl55UhMot5C +KOgOn1JqpODHrBGcul3WhJNWKPFtg6pisrdWxmTZlk6Xq06PuvaruRdSmDJ/tRL6 +Odc0KvPtQJBvZnu2wZ25U9Dj6WmMz3r9CApiR9TOcvdvgLQdGHq6oqlFSe+cC5mJ +A6tfep3Fd7c= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem index 360cd57426a5a8..0a48bf7d23539c 100644 --- a/Lib/test/certdata/pycacert.pem +++ b/Lib/test/certdata/pycacert.pem @@ -11,89 +11,92 @@ Certificate: Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b1:84:d3:4f:5c:04:80:91:4f:82:49:ba:30:0b: - f7:e8:cb:f9:14:ef:3d:9f:0b:3f:0a:62:fc:1b:20: - a5:20:d1:60:5f:87:5a:1f:16:d1:ed:97:70:a6:da: - 1b:03:2c:7e:a0:5b:3c:4e:2f:16:7e:0e:89:29:89: - e1:10:0d:38:da:6a:77:5f:37:13:b3:28:8f:7b:5c: - 76:ad:9e:e8:d3:f5:9e:f5:83:aa:10:07:8d:e6:51: - 98:f0:7c:0d:52:f2:0c:21:1e:d8:b9:99:26:a9:25: - 03:27:bb:5c:ab:2e:33:27:a2:d6:23:a8:83:87:44: - 29:9f:97:b5:24:6f:d7:b9:0a:fd:28:ee:bb:fb:41: - 58:ea:1d:99:dd:44:86:ab:98:be:1c:dc:cb:a9:89: - 1d:36:5c:a9:e8:47:b5:f4:52:48:aa:b5:a4:67:ef: - 3e:d7:e2:d3:33:de:98:29:d8:7a:b0:59:5c:e7:b1: - 0e:cc:fd:9f:eb:f6:d5:3a:0e:0b:cf:fe:0b:3d:a2: - bf:45:18:ce:94:e7:a9:55:60:88:d4:d8:84:50:79: - 05:2e:41:03:74:ae:67:26:f6:5b:12:08:98:ce:0a: - 97:ed:01:0f:89:4f:17:5c:fa:3e:1d:35:24:47:92: - 32:bf:f7:a4:18:2b:3c:d0:48:99:e1:a2:cd:a3:cc: - 50:53:20:b5:c6:e3:66:85:7b:57:10:ec:33:4f:c1: - 77:e7:1b:7e:81:c6:c4:f3:45:20:c0:91:dd:13:76: - 7b:03:af:f6:76:8e:a2:83:63:57:dd:63:bc:bb:5a: - 1c:17:52:8a:d6:06:48:cc:0f:c7:d3:4f:e8:da:22: - 6c:86:f9:4e:5c:a6:29:07:3b:d8:56:4c:59:b3:20: - 49:07:7b:94:84:cf:2b:c3:1c:1a:4e:87:64:92:ba: - 42:e1:e6:ad:7d:1d:f6:54:90:6f:2b:e9:b3:cc:4b: - 2b:33:26:23:fd:65:c0:3c:f0:79:ad:c9:c1:81:ef: - 37:04:e0:27:3e:b0:ee:15:be:51 + 00:d0:a0:9b:b1:b9:3b:79:ee:31:2f:b8:51:1c:01: + 75:ed:09:59:4c:ac:c8:bf:6a:0a:f8:7a:08:f0:25: + e3:25:7b:6a:f8:c4:d8:aa:a0:60:07:25:b0:cf:73: + 71:05:52:68:bf:06:93:ae:10:61:96:bc:b3:69:81: + 1a:7d:19:fc:20:86:8f:9a:68:1b:ed:cd:e2:6c:61: + 67:c7:4e:55:ea:43:06:21:1b:e9:b9:be:84:5f:c2: + da:eb:89:88:e0:42:a6:45:49:2a:d0:b9:6b:8c:93: + c9:44:3b:ca:fc:3c:94:7f:dd:70:c5:ad:d8:4b:3b: + dc:1e:f8:55:4b:b5:27:86:f8:df:b0:83:cf:f7:16: + 37:bf:13:17:34:d4:c9:55:ed:6b:16:c9:7f:ad:20: + 4e:f0:1e:d9:1b:bf:8d:ba:cd:ca:96:ef:1e:69:cb: + 51:66:61:48:0d:e8:79:a3:59:61:d4:10:8d:e0:0d: + 3b:0b:85:b6:d5:85:12:dd:a5:07:c5:4b:fa:23:fa: + 54:39:f7:97:0b:c9:44:47:1e:56:77:3c:3c:13:01: + 0b:6d:77:d6:14:ba:44:f4:53:35:56:d9:3d:b9:3e: + 35:5f:db:33:9a:4f:5a:b9:38:44:c9:32:49:fe:66: + f6:1f:1a:97:aa:ca:4c:4a:f6:b8:5e:40:7f:fb:0b: + 1d:f8:5b:a1:dc:f8:c0:2e:d0:6d:49:f5:d2:46:d4: + 90:57:fe:92:81:34:ae:2d:38:bb:a8:17:0c:e1:e5: + 3f:e2:f7:26:05:54:50:f5:64:b3:1c:6e:44:ff:6f: + a9:b4:03:96:e9:0e:c2:88:d8:72:52:90:99:c6:41: + 0f:46:90:59:b8:3e:6f:d2:e2:9e:1d:36:82:95:d3: + 58:8a:12:f3:e2:d8:0d:20:51:23:f0:90:2d:9a:3e: + 7d:26:86:b2:a7:d7:f9:28:60:03:e3:77:c7:88:04: + c9:fe:89:77:70:10:4f:c3:a0:8a:3b:f4:ab:42:7b: + e3:14:92:d8:ae:16:d6:a1:de:7d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD - - X509v3 Basic Constraints: + F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + X509v3 Basic Constraints: critical CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption - 6b:32:2f:e7:05:18:ea:5c:c9:95:f4:e0:c2:0c:41:5f:1a:0a: - 95:c9:c7:7d:05:ee:8a:56:29:35:50:40:b7:fe:9f:7b:5b:1c: - c3:69:2f:a0:cb:d2:b8:91:2f:50:19:62:f7:27:18:6d:95:7b: - 53:16:15:a2:5a:dc:14:e3:fb:b1:32:a9:69:db:a6:33:47:3c: - bb:1f:d2:dc:70:f9:6a:2e:0c:d8:8c:6d:e5:5d:1d:43:3c:4e: - 91:de:a0:c8:da:a0:4b:0e:9d:5e:b6:0f:4a:49:f0:7b:b6:53: - 9e:fd:35:14:5b:e3:4d:b4:18:a6:36:61:e8:8f:33:9b:d4:05: - f9:54:66:df:e0:cb:18:a3:4e:dc:17:a8:a0:b3:c1:a8:f4:d6: - 9d:ca:7f:68:53:1a:d7:95:da:e8:d3:9e:48:00:71:95:99:11: - 07:cf:96:c0:7d:ce:7d:30:e8:4f:e1:83:16:33:a1:ff:59:9b: - 3e:4c:e7:3a:38:01:9f:0f:67:4c:fd:2d:8b:4a:d4:01:46:37: - 33:e8:13:6b:15:a9:1d:68:76:45:a2:82:33:69:26:30:60:05: - c8:8f:bd:b4:75:ab:be:7a:8b:48:68:70:40:b4:1b:51:c5:e6: - 7a:ad:6b:4f:db:17:c0:60:67:2e:63:61:9b:2c:48:99:b8:76: - 45:a0:9e:cc:ef:33:1e:50:4e:ab:72:c3:65:c8:b2:79:b3:35: - 83:21:78:d3:8b:6c:3a:18:e8:65:32:39:b8:c0:9d:71:2f:35: - 36:8a:c0:17:62:d8:8b:3e:e1:22:18:2b:4c:63:a6:0e:9d:0a: - fa:ab:5b:35:fb:88:91:77:4c:8d:8c:9d:a9:cf:fc:ab:c2:e6: - 5a:05:7b:7e:04:6e:39:cf:93:ce:67:3b:7a:cb:af:b6:36:e1: - fb:71:64:45:d4:a6:f0:ce:ef:75:04:99:69:9a:e5:88:0a:10: - 02:74:89:ec:75:84:44:80:48:df:c1:f7:e9:37:ce:ce:92:92: - 5c:89:22:08:73:1f + Signature Value: + 8b:00:54:72:b3:8d:eb:f3:af:34:9f:d6:60:ea:de:84:3f:8c: + 04:8f:19:a6:be:02:67:c4:63:c5:74:e3:47:37:59:83:94:06: + f1:45:19:e8:07:2f:d6:4e:4b:4f:a8:3d:c7:07:07:27:92:f4: + 7e:73:4f:8b:32:19:94:46:7a:25:c4:d9:c4:27:b0:11:63:3a: + 60:8b:85:e1:73:4f:34:3b:6b:a4:34:8c:49:8e:cd:cf:4f:b2: + 65:27:41:19:b0:fc:80:31:78:f2:73:6a:9b:7d:71:34:50:fc: + 78:a8:da:05:b4:9c:5b:3a:99:7a:6b:5d:ef:3b:d3:e9:3b:33: + 01:12:65:cf:5e:07:d8:19:af:d5:53:ea:f0:10:ac:c4:b6:26: + 3c:34:2e:74:ee:64:dd:1d:36:75:89:44:00:b0:0d:fd:2f:b3: + 01:cc:1a:8b:02:cd:6c:e8:80:82:ca:bf:82:d7:00:9d:d8:36: + 15:d2:07:37:fc:6c:73:1d:da:a8:1c:e8:20:8e:32:7a:fe:6d: + 27:16:e4:58:6c:eb:3e:f0:fe:24:52:29:71:b8:96:7b:53:4b: + 45:20:55:40:5e:86:1b:ec:c9:46:91:92:ee:ac:93:65:91:2e: + 94:b6:b6:ac:e8:a3:34:89:a4:1a:12:0d:4d:44:a5:52:ed:8b: + 91:ee:2f:a6:af:a4:95:25:f9:ce:c7:5b:a7:00:d3:93:ca:b4: + 3c:5d:4d:f7:b1:3c:cc:6d:b4:45:be:82:ed:18:90:c8:86:d1: + 75:51:50:04:4c:e8:4f:d2:d6:50:aa:75:e7:5e:ff:a1:7b:27: + 19:1c:6b:49:2c:6c:4d:6b:63:cc:3b:73:00:f3:99:26:d0:82: + 87:d3:a9:36:9c:b4:3d:b9:48:68:a8:92:f0:27:8e:0c:cd:15: + 76:42:84:80:c9:3f:a3:7e:b4:dd:d7:f8:ac:76:ba:60:97:3c: + 5a:1f:4e:de:71:ee:09:39:10:dd:31:d5:68:57:5d:1a:e5:42: + ba:0e:68:e6:45:d3 -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem index 819bdef1ff9bfc..a6bf7356f4f684 100644 --- a/Lib/test/certdata/pycakey.pem +++ b/Lib/test/certdata/pycakey.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCxhNNPXASAkU+C -SbowC/foy/kU7z2fCz8KYvwbIKUg0WBfh1ofFtHtl3Cm2hsDLH6gWzxOLxZ+Dokp -ieEQDTjaandfNxOzKI97XHatnujT9Z71g6oQB43mUZjwfA1S8gwhHti5mSapJQMn -u1yrLjMnotYjqIOHRCmfl7Ukb9e5Cv0o7rv7QVjqHZndRIarmL4c3MupiR02XKno -R7X0UkiqtaRn7z7X4tMz3pgp2HqwWVznsQ7M/Z/r9tU6DgvP/gs9or9FGM6U56lV -YIjU2IRQeQUuQQN0rmcm9lsSCJjOCpftAQ+JTxdc+j4dNSRHkjK/96QYKzzQSJnh -os2jzFBTILXG42aFe1cQ7DNPwXfnG36BxsTzRSDAkd0TdnsDr/Z2jqKDY1fdY7y7 -WhwXUorWBkjMD8fTT+jaImyG+U5cpikHO9hWTFmzIEkHe5SEzyvDHBpOh2SSukLh -5q19HfZUkG8r6bPMSyszJiP9ZcA88HmtycGB7zcE4Cc+sO4VvlECAwEAAQKCAYB7 -gUnzALYxLOgAYYMkQm9si9zz768TpCNr+ooj5YZ9Wq6OSAEveBT+FErQCxaYErDW -qCNA0gn4Eezj9YWcQVa4vzHmEM+n6iRJU39ONC0Qqua5Ma10EY1sHIEnb2dlufku -YeOu3RrEu3eCgRxsDGySuvv5OxinV4kN++KPQzD3EOopPE+U81YFLCsMgsyfPlmm -gwc/IKIuXDHp5Vp2bXkZK98CYLV8RddjUw7SrkZNwx6cI9eET0CgTs7y4SrevoOy -jCdnA0j1HvL8AbLQuYoXo9fdGYDeq55hyYlxSMYLaEToZG3DJ0UAldrT+r7x52D8 -2QMnJUo2XHzVYPlXPJIAkFJisZZ36TkBvywCgXZMMLibPo9U6V0nfkybTtXKoory -nmgBv+XSGSNrVWMiygpDPqpX1G6bBgqUX3CiTlxtSkYYz1M4Vgj2cux5XEPTnVCq -CLVzvNIXZt1RyzXPxGWpPidCjOaiWBRT4u1Dol9fs3PmVvDaRxcKo9nspiUHCfEC -gcEA4GgxZ+IJwpAMHkdYId0oxjKgTqIg+Ua+EwfUoQT10ERl/k/V4cDwJRHT8lML -rKhTNQJMEE040jq+6mPJDl1KqMb/v05Q7fF22ToGw1HkZwK52O6CeEiJW4/J6bR1 -pZGN0irsa6GvzV65Y6gZVFEUl0JPRf8wPvQHXsWAw8/2LuXkXjV0ieIMq4pbWJf4 -kaid7dYLHnobiP9RVk7BGr7ifmCshoPjWp4TRMwYf6iIZrqMxUSX0QY8Xsqx6bch -LLx/AoHBAMqCvvwUKTrF4gKh5jyl6T6DTZ/Dujaz7BuAJdsSSHvuTa/Y1EfsQHZN -jABn89ZqHYDiyyCuVFO3dqhLtsPjhyFMSXj+98JYcL3FGKnqQqRTwtzzx2P2lV5X -U0WhrNRb3iLu79Tr8pE/2EPnvTr+J5b0DHEeRyM72LWs43zrDYHorH0/Aa5Qd37F -gDLCTBEl8jO5irRuAIq/KV9ZFnn8JDjNGVpXgHPW3354ON1YaMLnPASk7FQizSOQ -QZAsyxtdLwKBwGUosvTYYXvygXP4x1LkpmfKFJe94E1exXpAsmovmTvcSXn9tTXC -Sr77LWb0ZrPbYT7pHS7QEMg8MSnp941hIrG4mzs666KHkgLUdI4B0YtaIDsZMXlV -gY3j4KpYbhxH4/2U2eSfC2fxxnKVKW3n6vdQrfmo0q/eQ6BGOgiLK7fybCLHyBQL -8Zg2k3z5bNUEhMTdE0AW3WjBZ4IXmFcdK26616r/szJ7RcZilrydVXexqpmWlTVl -sTst9kucAPlwswKBwQCwf7my/GNezR8Jik+fZj7edBQQfcdra+8JnOvhfpLcKLte -2s1RjjA0q6usou1bYAsszP2bEzV97XWmgq7dFg4tUE7s/NO1d91zGDhBx2Gj1TkN -2A5dKonOuq9iDeITB6qYqcUvvyEfxRRZQr2jj+WzZCr/4BLCO6PJ29A9jKOuKLtF -QcfWRF2RiNMN6lffzkHFIR4p2YHxa2DEsGGtmbt8Ig3Jtl/HFmydzmxJRoev71dY -+ODdB6PhLhZmcRPoWpMCgcEAhGArwL68GwwRMqAX79gMv8tVT0CJnDyGk5mD/ZIB -Nzt0yQFO7rTEa1l1vAtOiVJ9IpAak2lgbEwodOfGnQst7lujNYDFzTRPTFt/lID1 -u6JBxmqawOSlqa00bt4l2YsTZV+BfSznBP6XO1PK4iR3o5G3NkoKJjZWm3e3asHk -6eTeMLcsIJ+Fp7gG0ve2EdQwhVSVMFEu4Q4C2FcJeU++L4kYpY7sTnAjUtiLvtHn -yp3jllEn3CBD8Uhs4B+sL/6p +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDQoJuxuTt57jEv +uFEcAXXtCVlMrMi/agr4egjwJeMle2r4xNiqoGAHJbDPc3EFUmi/BpOuEGGWvLNp +gRp9Gfwgho+aaBvtzeJsYWfHTlXqQwYhG+m5voRfwtrriYjgQqZFSSrQuWuMk8lE +O8r8PJR/3XDFrdhLO9we+FVLtSeG+N+wg8/3Fje/Exc01MlV7WsWyX+tIE7wHtkb +v426zcqW7x5py1FmYUgN6HmjWWHUEI3gDTsLhbbVhRLdpQfFS/oj+lQ595cLyURH +HlZ3PDwTAQttd9YUukT0UzVW2T25PjVf2zOaT1q5OETJMkn+ZvYfGpeqykxK9rhe +QH/7Cx34W6Hc+MAu0G1J9dJG1JBX/pKBNK4tOLuoFwzh5T/i9yYFVFD1ZLMcbkT/ +b6m0A5bpDsKI2HJSkJnGQQ9GkFm4Pm/S4p4dNoKV01iKEvPi2A0gUSPwkC2aPn0m +hrKn1/koYAPjd8eIBMn+iXdwEE/DoIo79KtCe+MUktiuFtah3n0CAwEAAQKCAYAD +iUK0/k2ZRqXJHXKBKy8rWjYMHCj3lvMM/M3g+tYWS7i88w00cIJ1geM006FDSf8i +LxjatvFd2OCg9ay+w8LSbvrJJGGbeXAQjo1v7ePRPttAPWphQ8RCS+8NAKhJcNJu +UzapZ13WJKfL2HLw1+VbziORXjMlLKRnAVDkzHMZO70C5MEQ0EIX+C6zrmBOl2HH +du6LPy8crSaDQg8YxFCI7WWnvRKp+Gp8aIfYnR+7ifT1qr5o9sEUw8GAReyooJ3a +yJ9uBUbcelO8fNjEABf9xjx+jOmOVsQfig2KuBEi0qXlQSpilZfUdYJhtNke9ADu +Hui6MBn04D4RIzeKXV+OLjiLwqkJyNlPuxJ2EGpIHNMcx3gpjXIApAwc47BQwLKJ +VhMWMXS0EWhCLtEzf5UrbMNX+Io3J7noEUu6jxmJV1BKhrnlYeoo4JryN0DUpkSb +rOAOJLOkpfj7+gvqmWI4MT6SQXSr6BK+3m4J5bVSq4pej9uG5NR3Utghi5hF7DEC +gcEA3cYNPYPFSTj9YAR3GUZvwUPVL3ZEFcwjrIeg87JhuZOH/hSQ33SgeEoAtaqL +cLbimj7YzUYx3FOUCp/7yK/bAF1dhAbFab1yZE46Qv2Vi4e+/KEBBftqxyJl5KyV +vc/HE1dXZGZIO1X5Z5MX8nO3rz/YayiozYVmMibrbHxgTEDC4BrbWtPJQNkflWEb +FXNjkm0s2+J3kFANpL94NUKMGtArxQV3hWydGN8wS3Fn7LDnHDoM5mOt/naeKRES +fwwpAoHBAPDTKsKs2LEe4YFzO1EClycDelppjxh5pHSnzTWSq40aKx533SG4aLyI +DmghzoA3OmY0xpAy1WpT9FeiDNbYpiFCH3qBkArQR2QCu+WGUQ9tDoeN0C2Dje4e +Yix49BjcGSWzSNvh+tU9PzRc/9eVBMAQuaCm3yNEL+Z7hFTzkrCWK23+jP/OzIIC +XhnKdOveIYVAjlVgv8CoWIy3xhwXyqPAcstcPmlv9sDAYn37Ot7rGIS7e0WyQxvg +gxnOxFzKNQKBwQDOPOn/NNV5HKh0bHKdbKVs4zoT4zW515etUIvbVR4QSCSFonZ/ +d6PreVZjmvAFp+3fZ2aSrx6bOJZJszGhFfjhw/G9X9aiWO1SXnVL6yrxERIJOWkM +ORy5h0GegOjYFauaTvUUhxHRLEi9i0sPy5EcRpFqReuFBPNe3Fa/EoMzJl6TriYj +tyRHTCNU9XMMZbxJZYH8EgUCjY/Cj9SoIvTL0p+Bn23hBHqrsJLm9dWhhXnHBC0O +68/Y/lJi+l9rCtECgcEAt6PfTJovl0j8HxF23vyBtK9TQtSR2NERlh9LPZn9lViq +Hs66YndT7sg1bDSzWlRDBSMjc1xAH5erkJOzBLYqYNwiUvGvnH9coSfwjkMRVxkL +ZlS+taZGuZiTtmP5h2d3CaegXIQDGU5d/xkXwxYQjEF0u8vkBel+OVxg+cLPTjcF +IRhl/r98dXtGtJYM+LvnhcxHfVWMg2YcOBn/SPbfgGVFZEuQECjf2fYaZQUJzGkr +xjOM+gXIZN6cOjbQyA0tAoHADgR5/bMbcf6Jk0w56c/khFZz/pusne5cjXw5a6qq +fClAqnqjGBpkRxs7HoCR3aje0Pd0pCS93a6Wiqneo4x4HDrpo+pWR2KGAAF4MeO3 +3K94hncmiLAiZo8iqULLKCqJW2EGB2b7QzGpY7jCPiI1g80KuYPesf4ZohSfrr1w +DoqGoNrcIVdVmUgX47lLqIiWarbbDRY0Am9j58dovmNINYr5wCYGbeh2RuUmHr4u +E2bb0CdekSHf05HPiF9QpK1z -----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl index 621675eb5c183d..431a831c4f196a 100644 --- a/Lib/test/certdata/revocation.crl +++ b/Lib/test/certdata/revocation.crl @@ -1,14 +1,14 @@ -----BEGIN X509 CRL----- MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFqgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQCd2GrHb4zr2R8eK7YMHwlkgICxbWP1 -4nuEi55yzUcmMcCZJ6ZQV3yYqTlAULGQ9qWAUdhsyH+yu3hRKFKHQv0DAdKKxgow -66YasAQQ99DskXOPxmRoIA7qtIWZbLtBwHQJWh+uUFlTdUXitGIX5Xie74xu5YIr -moa3QeuZyG5+gigSTUyst5T/J/cHfBzlAJLc2k3Ty4EPYXKHCVnrZWJbRmxq199l -A7S+eBb9qWXSYXCn6v+EZ76pUS3u/66kZ86PO3h9294BzdhxbCJ27dQXNHw6owe2 -Iyiv0aWx+TNSGSf4yCqaYTH6RtEoviI3h/inVFHNGgjlMzdaGw/0I3bkB0rt2WSR -Vck37HnXvQvVEkgO/39C0WKZus6m4gmOgZcbJbXaR8uIR5Hmw3SEyGEPEIBu6tXV -BLJOSOSu2vVUH5GUIrpvK9FTySKYa+MGryoPasuqZNfwpaXK+ON2G6QsmcXPWZY0 -Dry6t0w2geW6UYVGmb831i8ZP3JVVVwcwi0= +YS1zZXJ2ZXIXDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlqgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQDMZ4XLQlzUrqBbszEq9I/nXK3jN8/p +VZ2aScU2le0ySJqIthe0yXEYuoFu+I4ZULyNkCA79baStIl8/Lt48DOHfBVv8SVx +ZqF7/fdUZBCLJV1kuhuSSknbtNmja5NI4/lcRRXrodRWDMcOmqlKbAC6RMQz/gMG +vpewGPX1oj5AQnqqd9spKtHbeqeDiyyWYr9ZZFO/433lP7GdsoriTPggYJJMWJvs +819buE0iGwWf+rTLB51VyGluhcz2pqimej6Ra2cdnYh5IztZlDFR99HywzWhVz/A +2fwUA91GR7zATerweXVKNd59mcgF4PZWiXmQMwcE0qQOMqMmAqYPLim1mretZsAs +t1X+nDM0Ak3sKumIjteQF7I6VpSsG4NCtq23G8KpNHnBZVOt0U065lQEvx0ZmB94 +1z7SzjfSZMVXYxBjSXljwuoc1keGpNT5xCmHyrOIxaHsmizzwNESW4dGVLu7/JfK +w40uGbwH09w4Cfbwuo7w6sRWDWPnlW2mkoc= -----END X509 CRL----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem index de596717bd855f..427948252b786e 100644 --- a/Lib/test/certdata/ssl_cert.pem +++ b/Lib/test/certdata/ssl_cert.pem @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== -----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem index 46de61ab85b136..6ab7d57d003a35 100644 --- a/Lib/test/certdata/ssl_key.passwd.pem +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -1,42 +1,42 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH -ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn -Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC -aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7 -icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc -lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox -TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw -QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO -dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3 -3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq -a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo -YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD -II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT -maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf -fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2 -c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm -vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj -8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ -byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m -SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef -l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs -JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7 -TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M -SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX -LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK -JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8 -rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA -HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS -HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6 -UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu -KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H -3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk -Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0 -Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2 -kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv -qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d -bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ -8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L -oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6 +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIsc9l0YPybNICAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDxb9ekR9MERvIff73hFLc6BIIH +ENhkFePApZj7ZqpjBltINRnaZhu8sEfG1/y3ejDBOa5Sq3C/UPykPfJh0IXsraAB +STZO22UQEDpJzDnf1aLCo2cJpdz4Mr+Uj8OUdPiX83OlhC36gMrkgSYUdhSFQEas +MLiBnXU6Z5Mv1Lxe7TJrnMyA4A8JYXXu5XVTErJrC0YT6iCPQh7eAoEtml9a/tJM +OPg6kn58zmzVDp8LAau4Th1yhdD/cUQM09wg2i5JHLeC9akD+CkNlujVoAirLMTh +xoMXTy2dkv/lIwI9QVx6WE/VKIngBAPIi3Q+YCIm0PaTgWj5U10C8j4t7kW2AEZK +z82+vDOpLRGLo/ItNCO9F/a9e4PK4xxwFCOfR80tQNhs5gjKnbDz5IQv2p+pUfUX +u+AIO0rBb3M9Yya1MC2pc5VLAeQ3UF6YPrNyNjoDsQOytY3YtRVyxiKW72QzeUcX +Vpc3U6u8ZyHhkxK6bMv3dkPHGW1MOBd9/U5z+9lhHOfCGFStIQ9M8N48ZCWEGyty +oZT3UApxgqiBAi1h14ZyagA2mjsMNtTmmkSa3v26WUfrwnjm7LD1/0Vm+ptBOFH2 +CkP/aAvr8Ie+ehWobXGpqwB6rlOAwdpPrePtEZiZtdt58anmCquRgE5GIYtVz30f +flRABM8waJ196RDGkNAmDA3p/sqHy4vbsIOMl8faZ3QxvGVZlPbUEwPhiTIetA5Q +95fT/uIcuBLfpbaN23j/Av3LiJAeABSmGZ+dA+NXC5UMvuX8COyBU0YF2V6ofpIu +gP3UC7Tn4yV3Pbes81LEDCskaN6qVRil47l0G+dNcEHVkrGKcSaRCN+joBSCbuin +Rol34ir9azh8DqHRKdVlLlzTmDQcOwmi0Vx0ASgBXx4UI3IfK45gLJVoz6dkUz+3 +GIPrnh5cw2DvIgIApwmuCQUXPbWZwUW0zuyzhtny9W6S72GUE/P5oUCV+kGYBsup +FNiAyR9+n/xUuzB5HqIosj4rX+M4il4Ovt+KaCO6/COi+YjAO/9EnSttu8OTxsXl +wvgblsT7Y1d+iUfmIVNGtbc5NX46ktrbGiqgPX7oR7YDy5/+FQlnPS1YL0ThUiAC +2RbItu6b0uUyfu2jfWaGqy+SiRZ81rLwKPU3vJSEPfooVcJTG49EE006ZC4TvRzu +fNkId+P+BxvhEpUM4+VKzfzViEPuzR1u/DuwLAavS7nr5qb+zaUq+Fte5vDQmjjC +fflT8hS0BGpYEGndeZT4k+mZunHgs3NVUQ4/HW0nflf1j6qAn4+yIB79dH9d/ubt +RyBG29K+rN0TI/kH9BQZfsAcbnmhpT/ud0mJfeHZ0Lknn6mdJ/k4LXN0T1IlLKz3 +cSleOWY3zjKaOsbuju1o5IiVIr+AF/w+M4nzzDX6DDVpBPAt9iUnDGqjh6mJ3QWQ +CyCJDLNP0X8rZ8va2KOPorIBhmfDwJKEtIoXkb2hqWURTE0chC444QqiMsMXsX6+ +mOmiWGkdBFnEpGITISFTGERCjEfqOgTMweCANpquiLymJXgDURL603N2WexSgwnu +Gy1Ws1cA+1cT65ZLqjSqayZ6WdQvsKBBAnGW5LbwBhoCkX0vahs5nZiw0KnskP60 +wNMnyxaS1SuDJ65n+vuLUl7WeysRyz10RWliYZFiUE7jIXfWeYGonAo4eyCEeV/f +HInxxpswsg/na8BGBPMsx2SfBIiIvSIT4VNxHrL3sIfDrnb2HH/ut/oSLBgSKzY5 +DdkPz309kMM5dqnHANAgRrtVhqzLQE3kNGZ9mO/X1FAyXx8eB7NSeB6ysD8CAHvm +lkyfsGTzVsnuWWpeHqplds0wx5+XouVtFRI5J3RGa39mbpM1hMyIbS0O24CBKW6K +7n2UunbABwepL1hSa4e01OPdz4Zx/oayOevTtlfVqh68cEEc6ePdzf7z69pjot7B +eqlNaqa1POOmkuygL+fiP1BAR3rGEoQKXqb+6JjzLM9CnhCQHHPR2UdqukkEYwsa +bh9CU8AlfAJ19KFDria4JZXtl8LLMLLqWIO8fmQx7VqkEkEkl8jecO8YMaZTzFEb +bW7QtIZ1qHWH0UIHH3Qlav72NJTKvGIbtp1JNrLdsHcYNcojLZkEeA83UPaiTB2R +udltVUd016cktRVzLOKrust8kzPq3iSjpoIXFyFqIYHvWxGHgc7qD5gVBlazqSsV +qudDv+0PCBjLWLjS6HkFI8BfyXd3ME2wvSmTzSSgSh4nVJNNrZ/RVTtQ5MLVcdh0 +sJ3qsq2Pokf61XXjsTiQorX+cgI9zF6zETXHvnLf9FL+G/VSlcLUsQ0wC584qwQt +OSASYTbM79xgmjRmolZOptcYXGktfi2C4iq6V6zpFJuNMVgzZ+SbaQw9bvzUo2jG +VMwrTuQQ+fsAyn66WZvtkSGAdp58+3PNq31ZjafJXBzN -----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem index 1ea4578d81ecc4..ee927210511dfc 100644 --- a/Lib/test/certdata/ssl_key.pem +++ b/Lib/test/certdata/ssl_key.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 +qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M +V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz +hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td +ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID +eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb +SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV +uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ +HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO +M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP +MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ +tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP +qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq +D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph +7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq +U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG +ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC +gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ +u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG +XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s +5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo +dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH +Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 +UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr +r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 +XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 +1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx +tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz +og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX +v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z +SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 +uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 ++yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE +5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV +AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o +VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F +G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R +kNdcwhtZZ3dpChUdf+w/LIc= -----END PRIVATE KEY----- diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 2ef7a6e8eb2795..76214e6cda93b0 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -550,10 +550,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -563,10 +577,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -576,10 +604,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -589,10 +631,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -602,10 +658,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -615,10 +685,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -628,10 +712,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -641,10 +739,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -654,10 +766,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -667,10 +793,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -680,10 +820,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -693,10 +847,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -706,10 +874,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -719,10 +901,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -740,7 +936,7 @@ static PyObject * test_char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n) -/*[clinic end generated code: output=98589f02422fe6b1 input=e42330417a44feac]*/ +/*[clinic end generated code: output=ff11e203248582df input=e42330417a44feac]*/ /*[clinic input] @@ -1028,7 +1224,10 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("test_int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "test_int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1048,7 +1247,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=5aed87a7589eefb2 input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=fbcfb7554688663d input=d20541fc1ca0553e]*/ /*[clinic input] @@ -1980,7 +2179,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2014,7 +2213,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static PyObject * test_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=73d46a9ae3320f96 input=0d3484844749c05b]*/ +/*[clinic end generated code: output=13ba007e1c842a37 input=0d3484844749c05b]*/ /*[clinic input] @@ -2050,7 +2249,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2084,7 +2283,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=c9f02a41f425897d input=384adc78bfa0bff7]*/ +/*[clinic end generated code: output=789799a6d2d6eb4d input=384adc78bfa0bff7]*/ /*[clinic input] @@ -2121,7 +2320,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2168,7 +2367,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static PyObject * test_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b35d4e66f7283e46 input=eda7964f784f4607]*/ +/*[clinic end generated code: output=42430dd8ea5afde6 input=eda7964f784f4607]*/ /*[clinic input] @@ -2207,7 +2406,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2265,7 +2464,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ede7e6e65106bf2b input=209387a4815e5082]*/ +/*[clinic end generated code: output=f312c35c380d2bf9 input=209387a4815e5082]*/ /*[clinic input] @@ -2303,7 +2502,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2350,7 +2549,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=36d4df939a4c3eef input=18393cc64fa000f4]*/ +/*[clinic end generated code: output=3937da2a8233ebe0 input=18393cc64fa000f4]*/ /*[clinic input] @@ -2386,7 +2585,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2420,7 +2619,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=4835f4b6cf386c28 input=1767b0ebdf06060e]*/ +/*[clinic end generated code: output=6b4f6dd5f4db3877 input=1767b0ebdf06060e]*/ /*[clinic input] @@ -2457,7 +2656,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2491,7 +2690,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P static PyObject * test_posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *c) -/*[clinic end generated code: output=2570ea156a8d3cb5 input=9042f2818f664839]*/ +/*[clinic end generated code: output=8bef2a8198e70b26 input=9042f2818f664839]*/ /*[clinic input] @@ -2530,7 +2729,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2567,7 +2766,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=aaa0e6b5ce02900d input=29546ebdca492fea]*/ +/*[clinic end generated code: output=a44b8ae8300955e1 input=29546ebdca492fea]*/ /*[clinic input] @@ -2606,7 +2805,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2655,7 +2854,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na static PyObject * test_posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=1d9f2d8420d0a85f input=cdf5a9625e554e9b]*/ +/*[clinic end generated code: output=cae6647c9e8e0238 input=cdf5a9625e554e9b]*/ /*[clinic input] @@ -2693,7 +2892,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2740,7 +2939,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * test_posonly_keywords_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a83caa0505b296cf input=1581299d21d16f14]*/ +/*[clinic end generated code: output=6526fd08aafa2149 input=1581299d21d16f14]*/ /*[clinic input] @@ -2779,7 +2978,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2833,7 +3032,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ static PyObject * test_posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=0b24fba3dc04d26b input=408798ec3d42949f]*/ +/*[clinic end generated code: output=b8d01e98443738c2 input=408798ec3d42949f]*/ /*[clinic input] @@ -2873,7 +3072,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2922,7 +3121,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=592b217bca2f7bcc input=8d8e5643bbbc2309]*/ +/*[clinic end generated code: output=81d71c288f13d4dc input=8d8e5643bbbc2309]*/ /*[clinic input] @@ -2961,7 +3160,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3008,7 +3207,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_posonly_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b8b00420826bc11f input=f7e5eed94f75fff0]*/ +/*[clinic end generated code: output=a717d2a1a3310289 input=f7e5eed94f75fff0]*/ /*[clinic input] @@ -3048,7 +3247,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3102,7 +3301,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=3b9ee879ebee285a input=1e557dc979d120fd]*/ +/*[clinic end generated code: output=0f50b4b8d45cf2de input=1e557dc979d120fd]*/ /*[clinic input] @@ -3144,7 +3343,7 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3196,7 +3395,7 @@ static PyObject * test_posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d380f84f81cc0e45 input=c3884a4f956fdc89]*/ +/*[clinic end generated code: output=8dac8d2a4e6105fa input=c3884a4f956fdc89]*/ /*[clinic input] @@ -3236,7 +3435,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3285,7 +3484,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss static PyObject * test_posonly_keywords_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ee629e962cb06992 input=68d01d7c0f6dafb0]*/ +/*[clinic end generated code: output=5a96d521e6414f5d input=68d01d7c0f6dafb0]*/ /*[clinic input] @@ -3328,7 +3527,7 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3389,7 +3588,7 @@ static PyObject * test_posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=a2721babb42ecfd1 input=d0883d45876f186c]*/ +/*[clinic end generated code: output=d5a474dcd5dc3e9f input=d0883d45876f186c]*/ /*[clinic input] @@ -3432,7 +3631,7 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3498,7 +3697,7 @@ static PyObject * test_posonly_keywords_opt2_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=0626203eedb6e7e8 input=c95e2e1ec93035ad]*/ +/*[clinic end generated code: output=ac239c5ee8a74408 input=c95e2e1ec93035ad]*/ /*[clinic input] @@ -3543,7 +3742,7 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3617,7 +3816,7 @@ test_posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f) -/*[clinic end generated code: output=07d8acc04558a5a0 input=9914857713c5bbf8]*/ +/*[clinic end generated code: output=638bbd0005639342 input=9914857713c5bbf8]*/ /*[clinic input] test_keyword_only_parameter @@ -4013,7 +4212,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4048,7 +4247,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=880365c61ae205d7 input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=1411e464f358a7ba input=81d33815ad1bae6e]*/ /*[clinic input] test_vararg_with_default @@ -4085,7 +4284,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4131,7 +4330,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=291e9a5a09831128 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=f09d4b917063ca41 input=6e110b54acd9b22d]*/ /*[clinic input] test_vararg_with_only_defaults @@ -4168,7 +4367,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4219,7 +4418,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=dd21b28f0db26a4b input=fa56a709a035666e]*/ +/*[clinic end generated code: output=cc6590b8805d5433 input=fa56a709a035666e]*/ /*[clinic input] test_paramname_module @@ -4486,7 +4685,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4521,7 +4720,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) -/*[clinic end generated code: output=d89b99e83d442be0 input=af158077bd237ef9]*/ +/*[clinic end generated code: output=a3a968137b0f320a input=af158077bd237ef9]*/ /*[clinic input] @@ -4625,7 +4824,7 @@ Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); static PyObject * Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); return NULL; } @@ -4634,7 +4833,7 @@ Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_s static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ +/*[clinic end generated code: output=4d68b4652c144af3 input=e7e2e4e344e96a11]*/ /*[clinic input] @@ -4834,7 +5033,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4868,7 +5067,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/ +/*[clinic end generated code: output=0e1239b9bc247bc1 input=a8f9222a6ab35c59]*/ /*[clinic input] @@ -4951,6 +5150,69 @@ static PyObject * Test_meth_coexist_impl(TestObj *self) /*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ +/*[clinic input] +@getter +Test.property +[clinic start generated code]*/ + +#if defined(Test_property_HAS_DOCSTR) +# define Test_property_DOCSTR Test_property__doc__ +#else +# define Test_property_DOCSTR NULL +#endif +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, Test_property_DOCSTR}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, NULL, Test_property_DOCSTR}, +#endif + +static PyObject * +Test_property_get_impl(TestObj *self); + +static PyObject * +Test_property_get(TestObj *self, void *Py_UNUSED(context)) +{ + return Test_property_get_impl(self); +} + +static PyObject * +Test_property_get_impl(TestObj *self) +/*[clinic end generated code: output=27b519719d992e03 input=2d92b3449fbc7d2b]*/ + +/*[clinic input] +@setter +Test.property +[clinic start generated code]*/ + +#if defined(TEST_PROPERTY_HAS_DOCSTR) +# define Test_property_DOCSTR Test_property__doc__ +#else +# define Test_property_DOCSTR NULL +#endif +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, Test_property_DOCSTR}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", NULL, (setter)Test_property_set, NULL}, +#endif + +static int +Test_property_set_impl(TestObj *self, PyObject *value); + +static int +Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = Test_property_set_impl(self, value); + + return return_value; +} + +static int +Test_property_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ /*[clinic input] output push @@ -5039,7 +5301,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(i), }, + .ob_item = { _Py_LATIN1_CHR('i'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5074,7 +5336,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * mangled_c_keyword_identifier_impl(PyObject *module, int int_value) -/*[clinic end generated code: output=f24b37e0368e0eb8 input=060876448ab567a2]*/ +/*[clinic end generated code: output=01a8088b57632916 input=060876448ab567a2]*/ /*[clinic input] @@ -5268,52 +5530,6 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, /*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ -/*[clinic input] -Test.__init__ -> long -Test overriding the __init__ return converter -[clinic start generated code]*/ - -PyDoc_STRVAR(Test___init____doc__, -"Test()\n" -"--\n" -"\n" -"Test overriding the __init__ return converter"); - -static long -Test___init___impl(TestObj *self); - -static int -Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - PyTypeObject *base_tp = TestType; - long _return_value; - - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoPositional("Test", args)) { - goto exit; - } - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoKeywords("Test", kwargs)) { - goto exit; - } - _return_value = Test___init___impl((TestObj *)self); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong(_return_value); - -exit: - return return_value; -} - -static long -Test___init___impl(TestObj *self) -/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ - - /*[clinic input] fn_with_default_binop_expr arg: object(c_default='CONST_A + CONST_B') = a+b @@ -5432,7 +5648,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5466,4 +5682,249 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) -/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ +/*[clinic end generated code: output=3fff7b702f0065cb input=0cbe6a4d24bc2274]*/ + + +/*[clinic input] +@critical_section +test_critical_section +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section__doc__, +"test_critical_section($module, /)\n" +"--\n" +"\n"); + +#define TEST_CRITICAL_SECTION_METHODDEF \ + {"test_critical_section", (PyCFunction)test_critical_section, METH_NOARGS, test_critical_section__doc__}, + +static PyObject * +test_critical_section_impl(PyObject *module); + +static PyObject * +test_critical_section(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = test_critical_section_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +static PyObject * +test_critical_section_impl(PyObject *module) +/*[clinic end generated code: output=9d5a87bb28aa3f0c input=8c58956d6ff00f80]*/ + + +/*[clinic input] +@critical_section +test_critical_section_meth_o + a: object(subclass_of="&PyUnicode_Type") + / +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section_meth_o__doc__, +"test_critical_section_meth_o($module, a, /)\n" +"--\n" +"\n"); + +#define TEST_CRITICAL_SECTION_METH_O_METHODDEF \ + {"test_critical_section_meth_o", (PyCFunction)test_critical_section_meth_o, METH_O, test_critical_section_meth_o__doc__}, + +static PyObject * +test_critical_section_meth_o_impl(PyObject *module, PyObject *a); + +static PyObject * +test_critical_section_meth_o(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("test_critical_section_meth_o", "argument", "str", arg); + goto exit; + } + a = arg; + Py_BEGIN_CRITICAL_SECTION(module); + return_value = test_critical_section_meth_o_impl(module, a); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +static PyObject * +test_critical_section_meth_o_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=7a9d7420802d1202 input=376533f51eceb6c3]*/ + +/*[clinic input] +@critical_section a +test_critical_section_object + a: object(subclass_of="&PyUnicode_Type") + / +test_critical_section_object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section_object__doc__, +"test_critical_section_object($module, a, /)\n" +"--\n" +"\n" +"test_critical_section_object"); + +#define TEST_CRITICAL_SECTION_OBJECT_METHODDEF \ + {"test_critical_section_object", (PyCFunction)test_critical_section_object, METH_O, test_critical_section_object__doc__}, + +static PyObject * +test_critical_section_object_impl(PyObject *module, PyObject *a); + +static PyObject * +test_critical_section_object(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("test_critical_section_object", "argument", "str", arg); + goto exit; + } + a = arg; + Py_BEGIN_CRITICAL_SECTION(a); + return_value = test_critical_section_object_impl(module, a); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +static PyObject * +test_critical_section_object_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=ec06df92232b0fb5 input=6f67f91b523c875f]*/ + +PyDoc_STRVAR(test_critical_section_object__doc__, +"test_critical_section_object($module, a, /)\n" +"--\n" +"\n" +"test_critical_section_object"); + +#define TEST_CRITICAL_SECTION_OBJECT_METHODDEF \ + {"test_critical_section_object", (PyCFunction)test_critical_section_object, METH_O, test_critical_section_object__doc__}, + +static PyObject * +test_critical_section_object_impl(PyObject *module, PyObject *a); + +static PyObject * +test_critical_section_object(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *a; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("test_critical_section_object", "argument", "str", arg); + goto exit; + } + a = arg; + Py_BEGIN_CRITICAL_SECTION(a); + return_value = test_critical_section_object_impl(module, a); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +/*[clinic input] +@critical_section a b +test_critical_section_object2 + a: object(subclass_of="&PyUnicode_Type") + b: object(subclass_of="&PyUnicode_Type") + / +test_critical_section_object2 +[clinic start generated code]*/ + +PyDoc_STRVAR(test_critical_section_object2__doc__, +"test_critical_section_object2($module, a, b, /)\n" +"--\n" +"\n" +"test_critical_section_object2"); + +#define TEST_CRITICAL_SECTION_OBJECT2_METHODDEF \ + {"test_critical_section_object2", _PyCFunction_CAST(test_critical_section_object2), METH_FASTCALL, test_critical_section_object2__doc__}, + +static PyObject * +test_critical_section_object2_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_critical_section_object2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + + if (!_PyArg_CheckPositional("test_critical_section_object2", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("test_critical_section_object2", "argument 1", "str", args[0]); + goto exit; + } + a = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("test_critical_section_object2", "argument 2", "str", args[1]); + goto exit; + } + b = args[1]; + Py_BEGIN_CRITICAL_SECTION2(a, b); + return_value = test_critical_section_object2_impl(module, a, b); + Py_END_CRITICAL_SECTION2(); + +exit: + return return_value; +} + +static PyObject * +test_critical_section_object2_impl(PyObject *module, PyObject *a, + PyObject *b) +/*[clinic end generated code: output=d73a1657c18df17a input=638824e41419a466]*/ + +PyDoc_STRVAR(test_critical_section_object2__doc__, +"test_critical_section_object2($module, a, b, /)\n" +"--\n" +"\n" +"test_critical_section_object2"); + +#define TEST_CRITICAL_SECTION_OBJECT2_METHODDEF \ + {"test_critical_section_object2", _PyCFunction_CAST(test_critical_section_object2), METH_FASTCALL, test_critical_section_object2__doc__}, + +static PyObject * +test_critical_section_object2_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_critical_section_object2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + + if (!_PyArg_CheckPositional("test_critical_section_object2", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("test_critical_section_object2", "argument 1", "str", args[0]); + goto exit; + } + a = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("test_critical_section_object2", "argument 2", "str", args[1]); + goto exit; + } + b = args[1]; + Py_BEGIN_CRITICAL_SECTION2(a, b); + return_value = test_critical_section_object2_impl(module, a, b); + Py_END_CRITICAL_SECTION2(); + +exit: + return return_value; +} diff --git a/Lib/test/cfgparser.1 b/Lib/test/configdata/cfgparser.1 similarity index 100% rename from Lib/test/cfgparser.1 rename to Lib/test/configdata/cfgparser.1 diff --git a/Lib/test/cfgparser.2 b/Lib/test/configdata/cfgparser.2 similarity index 100% rename from Lib/test/cfgparser.2 rename to Lib/test/configdata/cfgparser.2 diff --git a/Lib/test/cfgparser.3 b/Lib/test/configdata/cfgparser.3 similarity index 100% rename from Lib/test/cfgparser.3 rename to Lib/test/configdata/cfgparser.3 diff --git a/Lib/test/cov.py b/Lib/test/cov.py new file mode 100644 index 00000000000000..e4699c7afe174a --- /dev/null +++ b/Lib/test/cov.py @@ -0,0 +1,48 @@ +"""A minimal hook for gathering line coverage of the standard library. + +Designed to be used with -Xpresite= which means: +* it installs itself on import +* it's not imported as `__main__` so can't use the ifmain idiom +* it can't import anything besides `sys` to avoid tainting gathered coverage +* filenames are not normalized + +To get gathered coverage back, look for 'test.cov' in `sys.modules` +instead of importing directly. That way you can determine if the module +was already in use. + +If you need to disable the hook, call the `disable()` function. +""" + +import sys + +mon = sys.monitoring + +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] + +coverage: set[Location] = set() + + +# `types` and `typing` aren't imported to avoid invalid coverage +def add_line( + code: "types.CodeType", + lineno: int, +) -> "typing.Literal[sys.monitoring.DISABLE]": + coverage.add((code.co_filename, lineno)) + return mon.DISABLE + + +def enable(): + mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage") + mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line) + mon.set_events(mon.COVERAGE_ID, mon.events.LINE) + + +def disable(): + mon.set_events(mon.COVERAGE_ID, 0) + mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, None) + mon.free_tool_id(mon.COVERAGE_ID) + + +enable() diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 0259a0688cbc67..7111946b93b280 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -8,13 +8,10 @@ Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py Put as much info into a docstring or comments to help determine the cause of the -failure, as well as a bugs.python.org issue number if it exists. Particularly -note if the cause is system or environment dependent and what the variables are. +failure, as well as an issue number or link if it exists. +Particularly note if the cause is system or environment dependent and +what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. - -Also see Lib/test_crashers.py which exercises the crashers in this directory. -In particular, make sure to add any new infinite loop crashers to the black -list so it doesn't try to run them. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 8bda17358db87f..b756413d7bf375 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -13,6 +13,7 @@ import re import struct import sys +import textwrap import unittest import warnings @@ -22,6 +23,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -37,6 +39,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None # Needed by test_datetime import _strptime @@ -301,6 +307,10 @@ def test_inheritance(self): self.assertIsInstance(timezone.utc, tzinfo) self.assertIsInstance(self.EST, tzinfo) + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class MyTimezone(timezone): pass + def test_utcoffset(self): dummy = self.DT for h in [0, 1.5, 12]: @@ -1331,6 +1341,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time @@ -1682,18 +1697,26 @@ def test_bool(self): self.assertTrue(self.theclass.max) def test_strftime_y2k(self): - for y in (1, 49, 70, 99, 100, 999, 1000, 1970): - d = self.theclass(y, 1, 1) - # Issue 13305: For years < 1000, the value is not always - # padded to 4 digits across platforms. The C standard - # assumes year >= 1900, so it does not specify the number - # of digits. - if d.strftime("%Y") != '%04d' % y: - # Year 42 returns '42', not padded - self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - if support.has_strftime_extensions: - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # Test that years less than 1000 are 0-padded; note that the beginning + # of an ISO 8601 year may fall in an ISO week of the year before, and + # therefore needs an offset of -1 when formatting with '%G'. + dataset = ( + (1, 0), + (49, -1), + (70, 0), + (99, 0), + (100, -1), + (999, 0), + (1000, 0), + (1970, 0), + ) + for year, offset in dataset: + for specifier in 'YG': + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += offset + self.assertEqual(d.strftime(f"%{specifier}"), f"{year:04d}") def test_replace(self): cls = self.theclass @@ -1719,11 +1742,24 @@ def test_replace(self): def test_subclass_replace(self): class DateSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result dt = DateSubclass(2012, 1, 1) - self.assertIs(type(dt.replace(year=2013)), DateSubclass) - self.assertIs(type(copy.replace(dt, year=2013)), DateSubclass) + + test_cases = [ + ('self.replace', dt.replace(year=2013)), + ('copy.replace', copy.replace(dt, year=2013)), + ] + + for name, res in test_cases: + with self.subTest(name): + self.assertIs(type(res), DateSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.month, 1) + self.assertEqual(res.extra, 7) def test_subclass_date(self): @@ -1910,6 +1946,10 @@ def test_fromisoformat_fails(self): '2009-02-29', # Invalid leap day '2019-W53-1', # No week 53 in 2019 '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -2776,6 +2816,20 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) @@ -3021,6 +3075,26 @@ def __new__(cls, *args, **kwargs): self.assertIsInstance(dt, DateTimeSubclass) self.assertEqual(dt.extra, 7) + def test_subclass_replace_fold(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2012, 1, 1) + dt2 = DateTimeSubclass(2012, 1, 1, fold=1) + + test_cases = [ + ('self.replace', dt.replace(year=2013), 0), + ('self.replace', dt2.replace(year=2013), 1), + ('copy.replace', copy.replace(dt, year=2013), 0), + ('copy.replace', copy.replace(dt2, year=2013), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), DateTimeSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.fold, fold) + def test_fromisoformat_datetime(self): # Test that isoformat() is reversible base_dates = [ @@ -3701,11 +3775,28 @@ def test_replace(self): def test_subclass_replace(self): class TimeSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result ctime = TimeSubclass(12, 30) - self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) - self.assertIs(type(copy.replace(ctime, hour=10)), TimeSubclass) + ctime2 = TimeSubclass(12, 30, fold=1) + + test_cases = [ + ('self.replace', ctime.replace(hour=10), 0), + ('self.replace', ctime2.replace(hour=10), 1), + ('copy.replace', copy.replace(ctime, hour=10), 0), + ('copy.replace', copy.replace(ctime2, hour=10), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), TimeSubclass) + self.assertEqual(res.hour, 10) + self.assertEqual(res.minute, 30) + self.assertEqual(res.extra, 7) + self.assertEqual(res.fold, fold) def test_subclass_time(self): @@ -4339,6 +4430,8 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute ] for bad_str in bad_strs: @@ -5431,42 +5524,50 @@ def fromutc(self, dt): class Oddballs(unittest.TestCase): - def test_bug_1028306(self): + def test_date_datetime_comparison(self): + # bpo-1028306, bpo-5516 (gh-49766) # Trying to compare a date to a datetime should act like a mixed- # type comparison, despite that datetime is a subclass of date. as_date = date.today() as_datetime = datetime.combine(as_date, time()) - self.assertTrue(as_date != as_datetime) - self.assertTrue(as_datetime != as_date) - self.assertFalse(as_date == as_datetime) - self.assertFalse(as_datetime == as_date) - self.assertRaises(TypeError, lambda: as_date < as_datetime) - self.assertRaises(TypeError, lambda: as_datetime < as_date) - self.assertRaises(TypeError, lambda: as_date <= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime <= as_date) - self.assertRaises(TypeError, lambda: as_date > as_datetime) - self.assertRaises(TypeError, lambda: as_datetime > as_date) - self.assertRaises(TypeError, lambda: as_date >= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime >= as_date) - - # Nevertheless, comparison should work with the base-class (date) - # projection if use of a date method is forced. - self.assertEqual(as_date.__eq__(as_datetime), True) - different_day = (as_date.day + 1) % 20 + 1 - as_different = as_datetime.replace(day= different_day) - self.assertEqual(as_date.__eq__(as_different), False) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. - date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) - self.assertEqual(as_date, date_sc) - self.assertEqual(date_sc, as_date) - # Ditto for datetimes. - datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, - as_date.day, 0, 0, 0) - self.assertEqual(as_datetime, datetime_sc) - self.assertEqual(datetime_sc, as_datetime) + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): @@ -6692,6 +6793,108 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + _datetime.date.max > _datetime.date.min + _datetime.time.max > _datetime.time.min + _datetime.datetime.max > _datetime.datetime.min + _datetime.timedelta.max > _datetime.timedelta.min + isinstance(_datetime.timezone.min, _datetime.tzinfo) + isinstance(_datetime.timezone.utc, _datetime.tzinfo) + isinstance(_datetime.timezone.max, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1eca123be0fecb..f2649aa2d41fef 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -40,6 +40,7 @@ BaseException ├── ReferenceError ├── RuntimeError │ ├── NotImplementedError + │ ├── PythonFinalizationError │ └── RecursionError ├── StopAsyncIteration ├── StopIteration diff --git a/Lib/test/imp_dummy.py b/Lib/test/imp_dummy.py deleted file mode 100644 index 2a4deb49547cf4..00000000000000 --- a/Lib/test/imp_dummy.py +++ /dev/null @@ -1,3 +0,0 @@ -# Fodder for test of issue24748 in test_imp - -dummy_name = True diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 5a01f1da7277e2..2ff4715e82a41b 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -2,8 +2,8 @@ import os.path import shlex import sys -from test.support import os_helper -from .utils import ALL_RESOURCES, RESOURCE_NAMES +from test.support import os_helper, Py_DEBUG +from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter USAGE = """\ @@ -161,10 +161,10 @@ def __init__(self, **kwargs) -> None: self.forever = False self.header = False self.failfast = False - self.match_tests = None - self.ignore_tests = None + self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False + self.tsan = False self.worker_json = None self.start = None self.timeout = None @@ -173,6 +173,8 @@ def __init__(self, **kwargs) -> None: self.fail_rerun = False self.tempdir = None self._add_python_opts = True + self.xmlpath = None + self.single_process = False super().__init__(**kwargs) @@ -183,6 +185,20 @@ def error(self, message): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -192,6 +208,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -251,17 +268,19 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') group.add_argument('-i', '--ignore', metavar='PAT', - dest='ignore_tests', action='append', + dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', - dest='ignore_filename', + dest='match_tests', + action=FromFileFilterAction, const=False, help='similar to --matchfile but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', @@ -289,6 +308,12 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -317,6 +342,8 @@ def _create_parser(): help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') @@ -331,6 +358,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") @@ -413,25 +442,39 @@ def _parse_args(args, **kwargs): else: ns._add_python_opts = False + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: # Similar to: -u "all" --timeout=1200 - if not ns.use: - ns.use = [['all']] + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all']) if ns.timeout is None: ns.timeout = 1200 # 20 minutes elif ns.fast_ci: # Similar to: -u "all,-cpu" --timeout=600 - if not ns.use: - ns.use = [['all', '-cpu']] + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all', '-cpu']) if ns.timeout is None: ns.timeout = 600 # 10 minutes if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") - if ns.use_mp is not None and ns.trace: - parser.error("-T and -j don't go together!") + if ns.trace: + if ns.use_mp is not None: + if not Py_DEBUG: + parser.error("need --with-pydebug to use -T and -j together") + else: + print( + "Warning: collecting coverage without -j is imprecise. Configure" + " --with-pydebug and run -m test -T -j for best results.", + file=sys.stderr + ) if ns.python is not None: if ns.use_mp is None: parser.error("-p requires -j!") @@ -475,23 +518,19 @@ def _parse_args(args, **kwargs): ns.randomize = True if ns.verbose: ns.header = True - if ns.huntrleaks and ns.verbose3: + + # When -jN option is used, a worker process does not use --verbose3 + # and so -R 3:3 -jN --verbose3 just works as expected: there is no false + # alarm about memory leak. + if ns.huntrleaks and ns.verbose3 and ns.use_mp is None: + # run_single_test() replaces sys.stdout with io.StringIO if verbose3 + # is true. In this case, huntrleaks sees an write into StringIO as + # a memory leak, whereas it is not (gh-71290). ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " - "--huntrleaks: see http://bugs.python.org/issue27103", + "--huntrleaks without -jN option", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) - if ns.ignore_filename: - if ns.ignore_tests is None: - ns.ignore_tests = [] - with open(ns.ignore_filename) as fp: - for line in fp: - ns.ignore_tests.append(line.strip()) + if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py new file mode 100644 index 00000000000000..41372e427ffd03 --- /dev/null +++ b/Lib/test/libregrtest/filter.py @@ -0,0 +1,77 @@ +import itertools +import operator +import re + + +# By default, don't filter tests +_test_matchers = () +_test_patterns = () + + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result + + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def get_match_tests(): + global _test_patterns + return _test_patterns + + +def set_match_tests(patterns): + global _test_matchers, _test_patterns + + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns + + +def _compile_match_function(patterns): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect_cmd utility only uses such full test identifiers. + return set(patterns).__contains__ + else: + import fnmatch + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id, regex_match=regex_match): + if regex_match(test_id): + # The regex matches the whole identifier, for example + # 'test.test_os.FileTests.test_access'. + return True + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + return match_test_regex diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index caf2872cfc9a18..4ac95e23a56b8f 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -4,8 +4,9 @@ from test import support +from .filter import match_test, set_match_tests from .utils import ( - StrPath, TestName, TestTuple, TestList, FilterTuple, + StrPath, TestName, TestTuple, TestList, TestFilter, abs_module_name, count, printlist) @@ -18,9 +19,11 @@ SPLITTESTDIRS: set[TestName] = { "test_asyncio", "test_concurrent_futures", + "test_doctests", "test_future_stmt", "test_gdb", "test_inspect", + "test_pydoc", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", @@ -79,15 +82,14 @@ def _list_cases(suite): if isinstance(test, unittest.TestSuite): _list_cases(test) elif isinstance(test, unittest.TestCase): - if support.match_test(test): + if match_test(test): print(test.id()) def list_cases(tests: TestTuple, *, - match_tests: FilterTuple | None = None, - ignore_tests: FilterTuple | None = None, + match_tests: TestFilter | None = None, test_dir: StrPath | None = None): support.verbose = False - support.set_match_tests(match_tests, ignore_tests) + set_match_tests(match_tests) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index a125706927393c..fa1d4d575c8fd4 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -43,7 +43,10 @@ def log(self, line: str = '') -> None: def get_load_avg(self) -> float | None: if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] + try: + return os.getloadavg()[0] + except OSError: + pass if self.win_load_tracker is not None: return self.win_load_tracker.getloadavg() return None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cb60d5af732b43..5148d3070513e8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -5,21 +5,23 @@ import sys import sysconfig import time +import trace -from test import support -from test.support import os_helper, MS_WINDOWS +from test.support import (os_helper, MS_WINDOWS, flush_std_streams, + suppress_immortalization) from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases from .logger import Logger from .pgo import setup_pgo_tests -from .result import State +from .result import State, TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests from .utils import ( - StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, @@ -56,6 +58,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan # Test results self.results: TestResults = TestResults() @@ -72,32 +75,27 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts and ns._add_python_opts) # Select tests - if ns.match_tests: - self.match_tests: FilterTuple | None = tuple(ns.match_tests) - else: - self.match_tests = None - if ns.ignore_tests: - self.ignore_tests: FilterTuple | None = tuple(ns.ignore_tests) - else: - self.ignore_tests = None + self.match_tests: TestFilter = ns.match_tests self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start self.cmdline_args: TestList = ns.args # Workers - if ns.use_mp is None: - num_workers = 0 # run sequentially + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process elif ns.use_mp <= 0: - num_workers = -1 # use the number of CPUs + num_workers = -1 # run in parallel, use the number of CPUs else: - num_workers = ns.use_mp + num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers self.worker_json: StrJSON | None = ns.worker_json @@ -129,14 +127,19 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Randomize self.randomize: bool = ns.randomize - self.random_seed: int | None = ( - ns.random_seed - if ns.random_seed is not None - else random.getrandbits(32) - ) - if 'SOURCE_DATE_EPOCH' in os.environ: + if ('SOURCE_DATE_EPOCH' in os.environ + # don't use the variable if empty + and os.environ['SOURCE_DATE_EPOCH'] + ): self.randomize = False - self.random_seed = None + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed # tests self.first_runtests: RunTests | None = None @@ -184,6 +187,9 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) + if self.tsan: + setup_tsan_tests(self.cmdline_args) + exclude_tests = set() if self.exclude: for arg in self.cmdline_args: @@ -231,7 +237,7 @@ def list_tests(tests: TestTuple): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -241,7 +247,6 @@ def _rerun_failed_tests(self, runtests: RunTests): tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests - self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( tests=tests, rerun=True, @@ -251,7 +256,15 @@ def _rerun_failed_tests(self, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.logger.set_tests(runtests) - self._run_tests_mp(runtests, self.num_workers) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) return runtests def rerun_failed_tests(self, runtests: RunTests): @@ -274,6 +287,55 @@ def rerun_failed_tests(self, runtests: RunTests): self.display_result(rerun_runtests) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() + + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return + def display_result(self, runtests): # If running the test suite for PGO then no one cares about results. if runtests.pgo: @@ -286,7 +348,9 @@ def display_result(self, runtests): self.results.display_result(runtests.tests, self.quiet, self.print_slowest) - def run_test(self, test_name: TestName, runtests: RunTests, tracer): + def run_test( + self, test_name: TestName, runtests: RunTests, tracer: trace.Trace | None + ) -> TestResult: if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. @@ -294,6 +358,7 @@ def run_test(self, test_name: TestName, runtests: RunTests, tracer): namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] + result.covered_lines = list(tracer.counts) else: result = run_single_test(test_name, runtests) @@ -301,21 +366,20 @@ def run_test(self, test_name: TestName, runtests: RunTests, tracer): return result - def run_tests_sequentially(self, runtests): + def run_tests_sequentially(self, runtests) -> None: if self.coverage: - import trace tracer = trace.Trace(trace=False, count=True) else: tracer = None - save_modules = sys.modules.keys() + save_modules = set(sys.modules) jobs = runtests.get_jobs() if jobs is not None: tests = count(jobs, 'test') else: tests = 'tests' - msg = f"Run {tests} sequentially" + msg = f"Run {tests} sequentially in a single process" if runtests.timeout: msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) @@ -332,10 +396,18 @@ def run_tests_sequentially(self, runtests): result = self.run_test(test_name, runtests, tracer) - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) + # Unload the newly imported test modules (best effort finalization) + new_modules = [module for module in sys.modules + if module not in save_modules and + module.startswith(("test.", "test_"))] + for module in new_modules: + sys.modules.pop(module, None) + # Remove the attribute of the parent module. + parent, _, name = module.rpartition('.') + try: + delattr(sys.modules[parent], name) + except (KeyError, AttributeError): + pass if result.must_stop(self.fail_fast, self.fail_env_changed): break @@ -351,8 +423,6 @@ def run_tests_sequentially(self, runtests): if previous_test: print(previous_test) - return tracer - def get_state(self): state = self.results.get_state(self.fail_env_changed) if self.first_state: @@ -363,7 +433,7 @@ def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() - def finalize_tests(self, tracer): + def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -371,10 +441,11 @@ def finalize_tests(self, tracer): else: os.unlink(self.next_single_filename) - if tracer is not None: - results = tracer.results() - results.write_results(show_missing=True, summary=True, - coverdir=self.coverage_dir) + if coverage is not None: + # uses a new-in-Python 3.13 keyword argument that mypy doesn't know about yet: + coverage.write_results(show_missing=True, summary=True, # type: ignore[call-arg] + coverdir=self.coverage_dir, + ignore_missing_files=True) if self.want_run_leaks: os.system("leaks %d" % os.getpid()) @@ -384,7 +455,7 @@ def finalize_tests(self, tracer): def display_summary(self): duration = time.perf_counter() - self.logger.start_time - filtered = bool(self.match_tests) or bool(self.ignore_tests) + filtered = bool(self.match_tests) # Total duration print() @@ -402,7 +473,6 @@ def create_run_tests(self, tests: TestTuple): fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, match_tests_dict=None, rerun=False, forever=self.forever, @@ -415,13 +485,13 @@ def create_run_tests(self, tests: TestTuple): hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, use_resources=self.use_resources, python_cmd=self.python_cmd, randomize=self.randomize, random_seed=self.random_seed, - json_file=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: @@ -433,7 +503,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.num_workers < 0: # Use all CPUs + 2 extra worker processes for tests # that like to sleep - self.num_workers = (os.process_cpu_count() or 1) + 2 + # + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet + self.num_workers = (os.process_cpu_count() or 1) + 2 # type: ignore[attr-defined] # For a partial run, we do not need to clutter the output. if (self.want_header @@ -441,7 +514,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources, self.python_cmd) - print("Using random seed", self.random_seed) + print("Using random seed:", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests @@ -449,7 +522,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() - if self.hunt_refleak and not self.num_workers: + if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interfers with refleak check use_load_tracker = False else: @@ -461,20 +534,26 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: try: if self.num_workers: self._run_tests_mp(runtests, self.num_workers) - tracer = None else: - tracer = self.run_tests_sequentially(runtests) + # gh-117783: don't immortalize deferred objects when tracking + # refleaks. Only releveant for the free-threaded build. + with suppress_immortalization(runtests.hunt_refleak): + self.run_tests_sequentially(runtests) + coverage = self.results.get_coverage_results() self.display_result(runtests) if self.want_rerun and self.results.need_rerun(): self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) finally: if use_load_tracker: self.logger.stop_load_tracker() self.display_summary() - self.finalize_tests(tracer) + self.finalize_tests(coverage) return self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) @@ -528,7 +607,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) @@ -655,7 +734,6 @@ def main(self, tests: TestList | None = None): elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index fefc347728a701..22c7c7a9acef14 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -5,7 +5,7 @@ [mypy] files = Lib/test/libregrtest explicit_package_bases = True -python_version = 3.11 +python_version = 3.12 platform = linux pretty = True @@ -25,7 +25,7 @@ warn_return_any = False disable_error_code = return # Enable --strict-optional for these ASAP: -[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*,Lib.test.libregrtest.worker.*,Lib.test.libregrtest.single.*,Lib.test.libregrtest.results.*,Lib.test.libregrtest.utils.*] +[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*] strict_optional = False # Various internal modules that typeshed deliberately doesn't have stubs for: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index ada1a65b867ee6..20b05954c762ff 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import os import sys import warnings from inspect import isabstract @@ -5,6 +6,7 @@ from test import support from test.support import os_helper +from test.support import refleak_helper from .runtests import HuntRefleak from .utils import clear_caches @@ -22,6 +24,30 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + def runtest_refleak(test_name, test_func, hunt_refleak: HuntRefleak, quiet: bool): @@ -52,7 +78,8 @@ def runtest_refleak(test_name, test_func, except ImportError: zdc = None # Run unmodified on platforms without zipimport support else: - zdc = zipimport._zip_directory_cache.copy() + # private attribute that mypy doesn't know about: + zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined] abcs = {} for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): @@ -83,20 +110,30 @@ def get_pooled_int(value): getunicodeinternedsize = sys.getunicodeinternedsize fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = interned_before = 0 + rc_before = alloc_before = fd_before = interned_immortal_before = 0 if not quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) - - results = None + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) + + xml_filename = 'refleak-xml.tmp' + result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() for i in rep_range: - results = test_func() - + current = refleak_helper._hunting_for_refleaks + refleak_helper._hunting_for_refleaks = True + try: + result = test_func() + finally: + refleak_helper._hunting_for_refleaks = current + + save_support_xml(xml_filename) dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -104,22 +141,40 @@ def get_pooled_int(value): # Also, readjust the reference counts and alloc blocks by ignoring # any strings that might have been interned during test_func. These # strings will be deallocated at runtime shutdown - interned_after = getunicodeinternedsize() - alloc_after = getallocatedblocks() - interned_after - rc_after = gettotalrefcount() - interned_after * 2 + interned_immortal_after = getunicodeinternedsize( + # Use an internal-only keyword argument that mypy doesn't know yet + _only_immortal=True) # type: ignore[call-arg] + alloc_after = getallocatedblocks() - interned_immortal_after + rc_after = gettotalrefcount() - interned_immortal_after * 2 fd_after = fd_count() - if not quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after - interned_before = interned_after + interned_immortal_before = interned_immortal_after + + restore_support_xml(xml_filename) if not quiet: print(file=sys.stderr) @@ -151,15 +206,21 @@ def check_fd_deltas(deltas): ]: # ignore warmup runs deltas = deltas[warmups:] - if checker(deltas): + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(filename, "a", encoding="utf-8") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return (failed, results) + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) + return (failed, result) def dash_R_cleanup(fs, ps, pic, zdc, abcs): @@ -181,21 +242,24 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - # ignore deprecation warning for collections.abc.ByteString abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - for ref in abcs.get(obj, set()): - if ref() is not None: - obj.register(ref()) + refs = abcs.get(obj, None) + if refs is not None: + obj._abc_registry_clear() + for ref in refs: + subclass = ref() + if subclass is not None: + obj.register(subclass) obj._abc_caches_clear() # Clear caches clear_caches() - # Clear type cache at the end: previous function calls can modify types - sys._clear_type_cache() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() def warm_caches(): diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index d6b0d5ad383a5b..74eae40440435d 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -2,13 +2,35 @@ import json from typing import Any -from test.support import TestStats - from .utils import ( StrJSON, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed, + results.skipped) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + # Avoid enum.Enum to reduce the number of imports when tests are run class State: PASSED = "PASSED" @@ -56,6 +78,11 @@ def must_stop(state): } +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] + + @dataclasses.dataclass(slots=True) class TestResult: test_name: TestName @@ -69,6 +96,9 @@ class TestResult: errors: list[tuple[str, str]] | None = None failures: list[tuple[str, str]] | None = None + # partial coverage in a worker run; not used by sequential in-process runs + covered_lines: list[Location] | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -185,6 +215,10 @@ def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: data.pop('__test_result__') if data['stats'] is not None: data['stats'] = TestStats(**data['stats']) + if data['covered_lines'] is not None: + data['covered_lines'] = [ + tuple(loc) for loc in data['covered_lines'] + ] return TestResult(**data) else: return data diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3708078ff0bf3a..0e28435bc7d629 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,14 +1,14 @@ import sys -from test.support import TestStats +import trace from .runtests import RunTests -from .result import State, TestResult +from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) -# Python uses exit code 1 when an exception is not catched +# Python uses exit code 1 when an exception is not caught # argparse.ArgumentParser.error() uses exit code 2 EXITCODE_BAD_TEST = 2 EXITCODE_ENV_CHANGED = 3 @@ -18,7 +18,7 @@ class TestResults: - def __init__(self): + def __init__(self) -> None: self.bad: TestList = [] self.good: TestList = [] self.rerun_bad: TestList = [] @@ -34,24 +34,26 @@ def __init__(self): self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml - self.testsuite_xml: list[str] = [] + self.testsuite_xml: list = [] + # used by -T with -j + self.covered_lines: set[Location] = set() - def is_all_good(self): + def is_all_good(self) -> bool: return (not self.bad and not self.skipped and not self.interrupted and not self.worker_bug) - def get_executed(self): + def get_executed(self) -> set[TestName]: return (set(self.good) | set(self.bad) | set(self.skipped) | set(self.resource_denied) | set(self.env_changed) | set(self.run_no_tests)) - def no_tests_run(self): + def no_tests_run(self) -> bool: return not any((self.good, self.bad, self.skipped, self.interrupted, self.env_changed)) - def get_state(self, fail_env_changed): + def get_state(self, fail_env_changed: bool) -> str: state = [] if self.bad: state.append("FAILURE") @@ -115,20 +117,28 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): self.worker_bug = True if result.has_meaningful_duration() and not rerun: + if result.duration is None: + raise ValueError("result.duration is None") self.test_times.append((result.duration, test_name)) if result.stats is not None: self.stats.accumulate(result.stats) if rerun: self.rerun.append(test_name) - + if result.covered_lines: + # we don't care about trace counts so we don't have to sum them up + self.covered_lines.update(result.covered_lines) xml_data = result.xml_data if xml_data: self.add_junit(xml_data) + def get_coverage_results(self) -> trace.CoverageResults: + counts = {loc: 1 for loc in self.covered_lines} + return trace.CoverageResults(counts=counts) + def need_rerun(self): return bool(self.rerun_results) - def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: tests: TestList = [] match_tests_dict = {} for result in self.rerun_results: @@ -139,11 +149,12 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: if match_tests: match_tests_dict[result.test_name] = match_tests - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.env_changed.clear() - self.rerun_results.clear() + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() return (tuple(tests), match_tests_dict) @@ -193,7 +204,7 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): omitted = set(tests) - self.get_executed() # less important - all_tests.append((omitted, "test", "{} omitted:")) + all_tests.append((sorted(omitted), "test", "{} omitted:")) if not quiet: all_tests.append((self.skipped, "test", "{} skipped:")) all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 16f8331abd32f9..387ddf9614cf79 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,7 +10,7 @@ import threading import time import traceback -from typing import Literal, TextIO +from typing import Any, Literal, TextIO from test import support from test.support import os_helper, MS_WINDOWS @@ -18,11 +18,11 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural, get_signal_name) + format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -79,8 +79,12 @@ class MultiprocessResult: err_msg: str | None = None +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + ExcStr = str QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited class ExitThread(Exception): @@ -138,14 +142,20 @@ def _kill(self) -> None: return self._killed = True - if USE_PROCESS_GROUP: + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: what = f"{self} process group" else: what = f"{self} process" print(f"Kill {what}", file=sys.stderr, flush=True) try: - if USE_PROCESS_GROUP: + if use_killpg: os.killpg(popen.pid, signal.SIGKILL) else: popen.kill() @@ -162,7 +172,7 @@ def stop(self) -> None: self._stopped = True self._kill() - def _run_process(self, runtests: RunTests, output_fd: int, + def _run_process(self, runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> int | None: popen = create_worker_process(runtests, output_fd, tmp_dir) self._popen = popen @@ -209,7 +219,7 @@ def _run_process(self, runtests: RunTests, output_fd: int, self._popen = None def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: - """Create stdout temporay file (file descriptor).""" + """Create stdout temporary file (file descriptor).""" if MS_WINDOWS: # gh-95027: When stdout is not a TTY, Python uses the ANSI code @@ -243,41 +253,41 @@ def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextI json_fd = json_tmpfile.fileno() if MS_WINDOWS: - json_handle = msvcrt.get_osfhandle(json_fd) + # The msvcrt module is only available on Windows; + # we run mypy with `--platform=linux` in CI + json_handle: int = msvcrt.get_osfhandle(json_fd) # type: ignore[attr-defined] json_file = JsonFile(json_handle, JsonFileType.WINDOWS_HANDLE) else: json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) return (json_file, json_tmpfile) - def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> RunTests: - """Create the worker RunTests.""" - + def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: tests = (test_name,) if self.runtests.rerun: match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - kwargs = {} + kwargs: dict[str, Any] = {} if match_tests: - kwargs['match_tests'] = match_tests + kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False - return self.runtests.copy( + return self.runtests.create_worker_runtests( tests=tests, json_file=json_file, **kwargs) - def run_tmp_files(self, worker_runtests: RunTests, + def run_tmp_files(self, worker_runtests: WorkerRunTests, stdout_fd: int) -> tuple[int | None, list[StrPath]]: # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during # Python finalization: too late for libregrtest. if not support.is_wasi: # Don't check for leaked temporary files and directories if Python is - # run on WASI. WASI don't pass environment variables like TMPDIR to + # run on WASI. WASI doesn't pass environment variables like TMPDIR to # worker processes. tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) @@ -345,6 +355,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: json_file, json_tmpfile = self.create_json_file(stack) worker_runtests = self.create_worker_runtests(test_name, json_file) + retcode: str | int | None retcode, tmp_files = self.run_tmp_files(worker_runtests, stdout_file.fileno()) @@ -355,7 +366,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: err_msg=None, state=State.TIMEOUT) if retcode != 0: - name = get_signal_name(retcode) + name = support.get_signal_name(retcode) if name: retcode = f"{retcode} ({name})" raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, @@ -375,8 +386,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed - while not self._stopped: - try: + try: + while not self._stopped: try: test_name = next(self.pending) except StopIteration: @@ -395,11 +406,12 @@ def run(self) -> None: if mp_result.result.must_stop(fail_fast, fail_env_changed): break - except ExitThread: - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) def _wait_completed(self) -> None: popen = self._popen @@ -457,8 +469,9 @@ def __init__(self, num_workers: int, runtests: RunTests, self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.live_worker_count = 0 - self.output: queue.Queue[QueueOutput] = queue.Queue() + self.output: queue.Queue[QueueContent] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) self.timeout = runtests.timeout @@ -496,6 +509,7 @@ def start_workers(self) -> None: self.log(msg) for worker in self.workers: worker.start() + self.live_worker_count += 1 def stop_workers(self) -> None: start_time = time.monotonic() @@ -510,14 +524,18 @@ def _get_result(self) -> QueueOutput | None: # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. - while any(worker.is_alive() for worker in self.workers): + while self.live_worker_count > 0: if use_faulthandler: faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, exit=True) # wait for a thread try: - return self.output.get(timeout=PROGRESS_UPDATE) + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result except queue.Empty: pass @@ -527,12 +545,6 @@ def _get_result(self) -> QueueOutput | None: if running: self.log(running) - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None - def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result pgo = self.runtests.pgo diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 4da312db4cb02e..8e9779524c56a2 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,13 +2,15 @@ import dataclasses import json import os +import shlex import subprocess +import sys from typing import Any from test import support from .utils import ( - StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -33,7 +35,8 @@ def configure_subprocess(self, popen_kwargs: dict) -> None: popen_kwargs['pass_fds'] = [self.file] case JsonFileType.WINDOWS_HANDLE: # Windows handle - startupinfo = subprocess.STARTUPINFO() + # We run mypy with `--platform=linux` so it complains about this: + startupinfo = subprocess.STARTUPINFO() # type: ignore[attr-defined] startupinfo.lpAttributeList = {"handle_list": [self.file]} popen_kwargs['startupinfo'] = startupinfo @@ -66,14 +69,18 @@ class HuntRefleak: runs: int filename: StrPath + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple fail_fast: bool fail_env_changed: bool - match_tests: FilterTuple | None - ignore_tests: FilterTuple | None + match_tests: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool @@ -86,19 +93,24 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + coverage: bool memory_limit: str | None gc_threshold: int | None use_resources: tuple[str, ...] python_cmd: tuple[str, ...] | None randomize: bool - random_seed: int | None - json_file: JsonFile | None + random_seed: int | str - def copy(self, **override): + def copy(self, **override) -> 'RunTests': state = dataclasses.asdict(self) state.update(override) return RunTests(**state) + def create_worker_runtests(self, **override): + state = dataclasses.asdict(self) + state.update(override) + return WorkerRunTests(**state) + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests_dict is not None: return self.match_tests_dict.get(test_name, None) @@ -119,13 +131,6 @@ def iter_tests(self): else: yield from self.tests - def as_json(self) -> StrJSON: - return json.dumps(self, cls=_EncodeRunTests) - - @staticmethod - def from_json(worker_json: StrJSON) -> 'RunTests': - return json.loads(worker_json, object_hook=_decode_runtests) - def json_file_use_stdout(self) -> bool: # Use STDOUT in two cases: # @@ -139,10 +144,65 @@ def json_file_use_stdout(self) -> bool: or support.is_wasi ) + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + args.append(f"--randseed={self.random_seed}") + return args + + +@dataclasses.dataclass(slots=True, frozen=True) +class WorkerRunTests(RunTests): + json_file: JsonFile + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'WorkerRunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + class _EncodeRunTests(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, RunTests): + if isinstance(o, WorkerRunTests): result = dataclasses.asdict(o) result["__runtests__"] = True return result @@ -157,6 +217,6 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) if data['json_file']: data['json_file'] = JsonFile(**data['json_file']) - return RunTests(**data) + return WorkerRunTests(**data) else: return data diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 793347f60ad93c..9e9741493e9a5b 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -8,6 +8,7 @@ from test import support from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from .filter import set_match_tests from .runtests import RunTests from .utils import ( setup_unraisable_hook, setup_threading_excepthook, fix_umask, @@ -92,11 +93,11 @@ def setup_tests(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + set_match_tests(runtests.match_tests) if runtests.use_junit: support.junit_xml_list = [] - from test.support.testresult import RegressionTestResult + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None @@ -123,7 +124,8 @@ def setup_tests(runtests: RunTests): support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) if runtests.hunt_refleak: - unittest.BaseTestSuite._cleanup = False + # private attribute that mypy doesn't know about: + unittest.BaseTestSuite._cleanup = False # type: ignore[attr-defined] if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 0304f858edf42c..adc8f1f455579f 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -1,4 +1,3 @@ -import doctest import faulthandler import gc import importlib @@ -9,13 +8,14 @@ import unittest from test import support -from test.support import TestStats from test.support import threading_helper -from .result import State, TestResult +from .filter import match_test +from .result import State, TestResult, TestStats from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests +from .testresult import get_test_runner from .utils import ( TestName, clear_caches, remove_testfn, abs_module_name, print_warning) @@ -33,7 +33,50 @@ def run_unittest(test_mod): print(error, file=sys.stderr) if loader.errors: raise Exception("errors while loading tests") - return support.run_unittest(tests) + _filter_suite(tests, match_test) + return _run_suite(tests) + +def _filter_suite(suite, pred): + """Recursively filter test cases in a suite based on a predicate.""" + newtests = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _filter_suite(test, pred) + newtests.append(test) + else: + if pred(test): + newtests.append(test) + suite._tests = newtests + +def _run_suite(suite): + """Run tests from a unittest.TestSuite-derived class.""" + runner = get_test_runner(sys.stdout, + verbosity=support.verbose, + capture_output=(support.junit_xml_list is not None)) + + result = runner.run(suite) + + if support.junit_xml_list is not None: + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) + + if not result.testsRun and not result.skipped and not result.errors: + raise support.TestDidNotRun + if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "multiple errors occurred" + if not support.verbose: err += "; run in verbose mode for details" + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise support.TestFailedWithDetails(err, errors, failures, stats=stats) + return result def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: @@ -58,14 +101,18 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: stats = test_result case unittest.TestResult(): stats = TestStats.from_unittest(test_result) - case doctest.TestResults(): - stats = TestStats.from_doctest(test_result) case None: print_warning(f"{result.test_name} test runner returned None: {test_func}") stats = None case _: - print_warning(f"Unknown test result type: {type(test_result)}") - stats = None + # Don't import doctest at top level since only few tests return + # a doctest.TestResult instance. + import doctest + if isinstance(test_result, doctest.TestResults): + stats = TestStats.from_doctest(test_result) + else: + print_warning(f"Unknown test result type: {type(test_result)}") + stats = None result.stats = stats @@ -78,10 +125,6 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: # Load the test module and run the tests. test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) - - # Remove the module from sys.module to reload it if it was already imported - sys.modules.pop(module_name, None) - test_mod = importlib.import_module(module_name) if hasattr(test_mod, "test_main"): @@ -193,11 +236,11 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: output_on_failure = runtests.output_on_failure timeout = runtests.timeout - use_timeout = ( - timeout is not None and threading_helper.can_start_thread - ) - if use_timeout: + if timeout is not None and threading_helper.can_start_thread: + use_timeout = True faulthandler.dump_traceback_later(timeout, exit=True) + else: + use_timeout = False try: setup_tests(runtests) @@ -240,9 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: xml_list = support.junit_xml_list if xml_list: - import xml.etree.ElementTree as ET - result.xml_data = [ET.tostring(x).decode('us-ascii') - for x in xml_list] + result.xml_data = xml_list finally: if use_timeout: faulthandler.cancel_dump_traceback_later() @@ -263,7 +304,10 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: result = TestResult(test_name) pgo = runtests.pgo try: - _runtest(result, runtests) + # gh-117783: don't immortalize deferred objects when tracking + # refleaks. Only releveant for the free-threaded build. + with support.suppress_immortalization(runtests.hunt_refleak): + _runtest(result, runtests) except: if not pgo: msg = traceback.format_exc() diff --git a/Lib/test/support/testresult.py b/Lib/test/libregrtest/testresult.py similarity index 94% rename from Lib/test/support/testresult.py rename to Lib/test/libregrtest/testresult.py index de23fdd59ded95..1820f354572521 100644 --- a/Lib/test/support/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,6 +9,7 @@ import traceback import unittest from test import support +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args): if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-out').text = stdout + ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-err').text = stderr + ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue + e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: - e2.set(k2, str(v2)) + e2.set(k2, sanitize_xml(str(v2))) else: - e2.text = str(v2) + e2.text = sanitize_xml(str(v2)) else: - e2.text = str(v) + e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000000..dd18ae2584f5d8 --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,33 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', + 'test_code', + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_weakref', +] + + +def setup_tsan_tests(cmdline_args): + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index aac8395da87845..2a3449016fe951 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,6 +5,7 @@ import os.path import platform import random +import re import shlex import signal import subprocess @@ -12,7 +13,7 @@ import sysconfig import tempfile import textwrap -from collections.abc import Callable +from collections.abc import Callable, Iterable from test import support from test.support import os_helper @@ -52,6 +53,7 @@ TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] FilterTuple = tuple[TestName, ...] FilterDict = dict[TestName, FilterTuple] @@ -262,6 +264,12 @@ def clear_caches(): for f in typing._cleanups: f() + import inspect + abs_classes = filter(inspect.isabstract, typing.__dict__.values()) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + obj._abc_caches_clear() + try: fractions = sys.modules['fractions'] except KeyError: @@ -274,7 +282,16 @@ def clear_caches(): except KeyError: pass else: - inspect._shadowed_dict_from_mro_tuple.cache_clear() + inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() def get_build_info(): @@ -289,8 +306,8 @@ def get_build_info(): build = [] # --disable-gil - if sysconfig.get_config_var('Py_NOGIL'): - build.append("nogil") + if sysconfig.get_config_var('Py_GIL_DISABLED'): + build.append("free_threading") if hasattr(sys, 'gettotalrefcount'): # --with-pydebug @@ -339,6 +356,9 @@ def get_build_info(): # --with-undefined-behavior-sanitizer if support.check_sanitizer(ub=True): sanitizers.append("UBSAN") + # --with-thread-sanitizer + if support.check_sanitizer(thread=True): + sanitizers.append("TSAN") if sanitizers: build.append('+'.join(sanitizers)) @@ -376,10 +396,19 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: # Python out of the source tree, especially when the # source tree is read only. tmp_dir = sysconfig.get_config_var('srcdir') + if not tmp_dir: + raise RuntimeError( + "Could not determine the correct value for tmp_dir" + ) tmp_dir = os.path.join(tmp_dir, 'build') else: # WASI platform tmp_dir = sysconfig.get_config_var('projectbase') + if not tmp_dir: + raise RuntimeError( + "sysconfig.get_config_var('projectbase') " + f"unexpectedly returned {tmp_dir!r} on WASI" + ) tmp_dir = os.path.join(tmp_dir, 'build') # When get_temp_dir() is called in a worker process, @@ -409,7 +438,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # the tests. The name of the dir includes the pid to allow parallel # testing (see the -j option). # Emscripten and WASI have stubbed getpid(), Emscripten has only - # milisecond clock resolution. Use randint() instead. + # millisecond clock resolution. Use randint() instead. if support.is_emscripten or support.is_wasi: nounce = random.randint(0, 1_000_000) else: @@ -546,7 +575,7 @@ def is_cross_compiled(): return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: tuple[str, ...]): +def format_resources(use_resources: Iterable[str]): use_resources = set(use_resources) all_resources = set(ALL_RESOURCES) @@ -579,9 +608,10 @@ def display_header(use_resources: tuple[str, ...], print("== Python build:", ' '.join(get_build_info())) print("== cwd:", os.getcwd()) - cpu_count = os.cpu_count() + cpu_count: object = os.cpu_count() if cpu_count: - process_cpu_count = os.process_cpu_count() + # The function is new in Python 3.13; mypy doesn't know about it yet: + process_cpu_count = os.process_cpu_count() # type: ignore[attr-defined] if process_cpu_count and process_cpu_count != cpu_count: cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" print("== CPU count:", cpu_count) @@ -623,6 +653,7 @@ def display_header(use_resources: tuple[str, ...], asan = support.check_sanitizer(address=True) msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) + tsan = support.check_sanitizer(thread=True) sanitizers = [] if asan: sanitizers.append("address") @@ -630,12 +661,15 @@ def display_header(use_resources: tuple[str, ...], sanitizers.append("memory") if ubsan: sanitizers.append("undefined behavior") + if tsan: + sanitizers.append("thread") if sanitizers: print(f"== sanitizers: {', '.join(sanitizers)}") for sanitizer, env_var in ( (asan, "ASAN_OPTIONS"), (msan, "MSAN_OPTIONS"), (ubsan, "UBSAN_OPTIONS"), + (tsan, "TSAN_OPTIONS"), ): options= os.environ.get(env_var) if sanitizer and options is not None: @@ -657,23 +691,23 @@ def cleanup_temp_dir(tmp_dir: StrPath): print("Remove file: %s" % name) os_helper.unlink(name) -WINDOWS_STATUS = { - 0xC0000005: "STATUS_ACCESS_VIOLATION", - 0xC00000FD: "STATUS_STACK_OVERFLOW", - 0xC000013A: "STATUS_CONTROL_C_EXIT", -} - -def get_signal_name(exitcode): - if exitcode < 0: - signum = -exitcode - try: - return signal.Signals(signum).name - except ValueError: - pass - - try: - return WINDOWS_STATUS[exitcode] - except KeyError: - pass - return None +ILLEGAL_XML_CHARS_RE = re.compile( + '[' + # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal + '\x00-\x08\x0B\x0C\x0E-\x1F' + # Surrogate characters + '\uD800-\uDFFF' + # Special Unicode characters + '\uFFFE' + '\uFFFF' + # Match multiple sequential invalid characters for better effiency + ']+') + +def _sanitize_xml_replace(regs): + text = regs[0] + return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] + for ch in text) + +def sanitize_xml(text): + return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 5736cdfd3c7292..b51fde0af57d1f 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -24,6 +24,10 @@ class WindowsLoadTracker(): """ def __init__(self): + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + # Pre-flight test for access to the performance data; # `PermissionError` will be raised if not allowed winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index a9c8be0bb65d08..86cc30835fdbda 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -3,37 +3,28 @@ import os from typing import Any, NoReturn -from test import support -from test.support import os_helper +from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import WorkerRunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( - StrPath, StrJSON, FilterTuple, + StrPath, StrJSON, TestFilter, get_temp_dir, get_work_dir, exit_timeout) USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = { + 'test_ioctl', +} -def create_worker_process(runtests: RunTests, output_fd: int, +def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: - python_cmd = runtests.python_cmd worker_json = runtests.as_json() - python_opts = support.args_from_interpreter_flags() - if python_cmd is not None: - executable = python_cmd - # Remove -E option, since --python=COMMAND can set PYTHON environment - # variables, such as PYTHONPATH, in the worker process. - python_opts = [opt for opt in python_opts if opt != "-E"] - else: - executable = (sys.executable,) - cmd = [*executable, *python_opts, - '-u', # Unbuffered stdout and stderr - '-m', 'test.libregrtest.worker', - worker_json] + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) if tmp_dir is not None: @@ -59,7 +50,10 @@ def create_worker_process(runtests: RunTests, output_fd: int, close_fds=True, cwd=work_dir, ) - if USE_PROCESS_GROUP: + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: kwargs['start_new_session'] = True # Pass json_file to the worker process @@ -71,9 +65,9 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: - runtests = RunTests.from_json(worker_json) + runtests = WorkerRunTests.from_json(worker_json) test_name = runtests.tests[0] - match_tests: FilterTuple | None = runtests.match_tests + match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file setup_test_dir(runtests.test_dir) @@ -81,12 +75,24 @@ def worker_process(worker_json: StrJSON) -> NoReturn: if runtests.rerun: if match_tests: - matching = "matching: " + ", ".join(match_tests) + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) result = run_single_test(test_name, runtests) + if runtests.coverage: + if "test.cov" in sys.modules: # imported by -Xpresite= + result.covered_lines = list(sys.modules["test.cov"].coverage) + elif not Py_DEBUG: + print( + "Gathering coverage in worker processes requires --with-pydebug", + flush=True, + ) + else: + raise LookupError( + "`test.cov` not found in sys.modules but coverage wanted" + ) if json_file.file_type == JsonFileType.STDOUT: print() diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index d9ab21d4941cdb..dbc5ef4f9f2cd5 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) @@ -191,6 +191,14 @@ def test_setslice(self): self.assertRaises(TypeError, a.__setitem__) + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] @@ -562,3 +570,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index b4cfce19a7174e..ed89a81a6ea685 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +624,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt index a8b8a0a2148f00..0bc45603b8b18a 100644 --- a/Lib/test/mathdata/ieee754.txt +++ b/Lib/test/mathdata/ieee754.txt @@ -116,7 +116,7 @@ inf >>> 0 ** -1 Traceback (most recent call last): ... -ZeroDivisionError: 0.0 cannot be raised to a negative power +ZeroDivisionError: zero to a negative power >>> pow(0, NAN) nan diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index ddb180ef5ef825..9922591ce7114a 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1825,6 +1825,14 @@ def test_unicode_high_plane(self): t2 = self.loads(p) self.assert_is_copy(t, t2) + def test_unicode_memoization(self): + # Repeated str is re-used (even when escapes added). + for proto in protocols: + for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': + p = self.dumps((s, s), proto) + s1, s2 = self.loads(p) + self.assertIs(s1, s2) + def test_bytes(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1837,6 +1845,25 @@ def test_bytes(self): p = self.dumps(s, proto) self.assert_is_copy(s, self.loads(p)) + def test_bytes_memoization(self): + for proto in protocols: + for array_type in [bytes, ZeroCopyBytes]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + x, y = self.loads(p) + self.assertIs(x, y) + self.assert_is_copy((b, b), (x, y)) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1, b2 = array_type(s), array_type(s) + p = self.dumps((b1, b2), proto) + # Note that (b1, b2) = self.loads(p) might have identical + # components, i.e., b1 is b2, but this is not always the + # case if the content is large (equality still holds). + self.assert_is_copy((b1, b2), self.loads(p)) + def test_bytearray(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1856,13 +1883,31 @@ def test_bytearray(self): self.assertNotIn(b'bytearray', p) self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) - def test_bytearray_memoization_bug(self): + def test_bytearray_memoization(self): for proto in protocols: - for s in b'', b'xyz', b'xyz'*100: - b = bytearray(s) - p = self.dumps((b, b), proto) - b1, b2 = self.loads(p) - self.assertIs(b1, b2) + for array_type in [bytearray, ZeroCopyBytearray]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1a, b2a = array_type(s), array_type(s) + # Unlike bytes, equal but independent bytearray objects are + # never identical. + self.assertIsNot(b1a, b2a) + + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + self.assertIsNot(b1b, b2b) + + self.assertIsNot(b1a, b1b) + self.assert_is_copy(b1a, b1b) + + self.assertIsNot(b2a, b2b) + self.assert_is_copy(b2a, b2b) def test_ints(self): for proto in protocols: @@ -2429,7 +2474,7 @@ def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ x = BadGetattr() for proto in range(2): - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RuntimeError, self.dumps, x, proto) for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): s = self.dumps(x, proto) @@ -3514,6 +3559,84 @@ def __init__(self): pass self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) + def test_unpickler_bad_file(self): + # bpo-38384: Crash in _pickle if the read attribute raises an error. + def raises_oserror(self, *args, **kwargs): + raise OSError + @property + def bad_property(self): + 1/0 + + # File without read and readline + class F: + pass + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without read + class F: + readline = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without readline + class F: + read = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File with bad read + class F: + read = bad_property + readline = raises_oserror + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad readline + class F: + readline = bad_property + read = raises_oserror + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad readline, no read + class F: + readline = bad_property + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad read, no readline + class F: + read = bad_property + self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + + # File with bad peek + class F: + peek = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except ZeroDivisionError: + pass + + # File with bad readinto + class F: + readinto = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except ZeroDivisionError: + pass + + def test_pickler_bad_file(self): + # File without write + class F: + pass + self.assertRaises(TypeError, self.Pickler, F()) + + # File with bad write + class F: + @property + def write(self): + 1/0 + self.assertRaises(ZeroDivisionError, self.Pickler, F()) + def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to # check that dumps() and loads() redirect their arguments diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py index 19ccd62dead8ee..5535edbfa7795b 100644 --- a/Lib/test/pyclbr_input.py +++ b/Lib/test/pyclbr_input.py @@ -12,17 +12,19 @@ class B (object): def bm(self): pass class C (B): - foo = Other().foo - om = Other.om - d = 10 - # XXX: This causes test_pyclbr.py to fail, but only because the - # introspection-based is_method() code in the test can't - # distinguish between this and a genuine method function like m(). - # The pyclbr.py module gets this right as it parses the text. + # This one is correctly considered by both test_pyclbr.py and pyclbr.py + # as a non-method of C. + foo = Other().foo + + # This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). # - #f = f + # The pyclbr.py module gets this right as it parses the text. + om = Other.om + f = f def m(self): pass @@ -31,3 +33,53 @@ def sm(self): pass @classmethod def cm(self): pass + +# Check that mangling is correctly handled + +class a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class ___: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 4f3ebb12ed957d..d928e002ebda10 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -105,9 +105,13 @@ def collect_sys(info_add): ) copy_attributes(info_add, sys, 'sys.%s', attributes) - call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') - call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') - call_func(info_add, 'sys.getrecursionlimit', sys, 'getrecursionlimit') + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) encoding = sys.getfilesystemencoding() if hasattr(sys, 'getfilesystemencodeerrors'): @@ -179,6 +183,9 @@ def collect_platform(info_add): info_add(f'platform.freedesktop_os_release[{key}]', os_release[key]) + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + def collect_locale(info_add): import locale @@ -287,6 +294,7 @@ def format_groups(groups): "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", "LDSHARED", @@ -509,6 +517,7 @@ def collect_sysconfig(info_add): 'MACHDEP', 'MULTIARCH', 'OPT', + 'PGO_PROF_USE_FLAG', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', 'PY_CORE_LDFLAGS', @@ -517,9 +526,10 @@ def collect_sysconfig(info_add): 'PY_STDMODULE_CFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', - 'Py_NOGIL', + 'Py_GIL_DISABLED', 'SHELL', 'SOABI', + 'TEST_MODULES', 'abs_builddir', 'abs_srcdir', 'prefix', @@ -544,6 +554,7 @@ def collect_sysconfig(info_add): 'WITH_DOC_STRINGS', 'WITH_DTRACE', 'WITH_FREELISTS', + 'WITH_MIMALLOC', 'WITH_PYMALLOC', 'WITH_VALGRIND', ): @@ -864,26 +875,36 @@ def collect_subprocess(info_add): def collect_windows(info_add): - try: - import ctypes - except ImportError: + if sys.platform != "win32": + # Code specific to Windows return - if not hasattr(ctypes, 'WinDLL'): - return - - ntdll = ctypes.WinDLL('ntdll') - BOOLEAN = ctypes.c_ubyte - + # windows.RtlAreLongPathsEnabled: RtlAreLongPathsEnabled() + # windows.is_admin: IsUserAnAdmin() try: - RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled - except AttributeError: - res = '' + import ctypes + if not hasattr(ctypes, 'WinDLL'): + raise ImportError + except ImportError: + pass else: - RtlAreLongPathsEnabled.restype = BOOLEAN - RtlAreLongPathsEnabled.argtypes = () - res = bool(RtlAreLongPathsEnabled()) - info_add('windows.RtlAreLongPathsEnabled', res) + ntdll = ctypes.WinDLL('ntdll') + BOOLEAN = ctypes.c_ubyte + try: + RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + except AttributeError: + res = '' + else: + RtlAreLongPathsEnabled.restype = BOOLEAN + RtlAreLongPathsEnabled.argtypes = () + res = bool(RtlAreLongPathsEnabled()) + info_add('windows.RtlAreLongPathsEnabled', res) + + shell32 = ctypes.windll.shell32 + IsUserAnAdmin = shell32.IsUserAnAdmin + IsUserAnAdmin.restype = BOOLEAN + IsUserAnAdmin.argtypes = () + info_add('windows.is_admin', IsUserAnAdmin()) try: import _winapi @@ -892,6 +913,7 @@ def collect_windows(info_add): except (ImportError, AttributeError): pass + # windows.version_caption: "wmic os get Caption,Version /value" command import subprocess try: # When wmic.exe output is redirected to a pipe, @@ -918,12 +940,15 @@ def collect_windows(info_add): if line: info_add('windows.version', line) + # windows.ver: "ver" command try: proc = subprocess.Popen(["ver"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) output = proc.communicate()[0] + if proc.returncode == 0xc0000142: + return if proc.returncode: output = "" except OSError: @@ -934,6 +959,22 @@ def collect_windows(info_add): if line: info_add('windows.ver', line) + # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry + import winreg + try: + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") + subkey = "AllowDevelopmentWithoutDevLicense" + try: + value, value_type = winreg.QueryValueEx(key, subkey) + finally: + winreg.CloseKey(key) + except OSError: + pass + else: + info_add('windows.developer_mode', "enabled" if value else "disabled") + def collect_fips(info_add): try: diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py new file mode 100644 index 00000000000000..9c3d0c7cf4bfaa --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py new file mode 100644 index 00000000000000..3dfba253455ad2 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py @@ -0,0 +1,9 @@ +import sys +import unittest + +class Test(unittest.TestCase): + def test(self): + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIn('test_regrtest_b', sys.modules) + self.assertNotIn('test_regrtest_b.util', sys.modules) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py similarity index 100% rename from Lib/test/test_importlib/data/__init__.py rename to Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py new file mode 100644 index 00000000000000..de80769118d709 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index a41970d8f3f55a..719c9434a16820 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -426,6 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) + @support.suppress_immortalization() def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/shadowed_super.py b/Lib/test/shadowed_super.py deleted file mode 100644 index 2a62f667e93818..00000000000000 --- a/Lib/test/shadowed_super.py +++ /dev/null @@ -1,7 +0,0 @@ -class super: - msg = "truly super" - - -class C: - def method(self): - return super().msg diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py deleted file mode 100644 index 14a9d827ed57c5..00000000000000 --- a/Lib/test/sortperf.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Sort performance test. - -See main() for command line syntax. -See tabulate() for output format. - -""" - -import sys -import time -import random -import marshal -import tempfile -import os - -td = tempfile.gettempdir() - -def randfloats(n): - """Return a list of n random floats in [0, 1).""" - # Generating floats is expensive, so this writes them out to a file in - # a temp directory. If the file already exists, it just reads them - # back in and shuffles them a bit. - fn = os.path.join(td, "rr%06d" % n) - try: - fp = open(fn, "rb") - except OSError: - r = random.random - result = [r() for i in range(n)] - try: - try: - fp = open(fn, "wb") - marshal.dump(result, fp) - fp.close() - fp = None - finally: - if fp: - try: - os.unlink(fn) - except OSError: - pass - except OSError as msg: - print("can't write", fn, ":", msg) - else: - result = marshal.load(fp) - fp.close() - # Shuffle it a bit... - for i in range(10): - i = random.randrange(n) - temp = result[:i] - del result[:i] - temp.reverse() - result.extend(temp) - del temp - assert len(result) == n - return result - -def flush(): - sys.stdout.flush() - -def doit(L): - t0 = time.perf_counter() - L.sort() - t1 = time.perf_counter() - print("%6.2f" % (t1-t0), end=' ') - flush() - -def tabulate(r): - r"""Tabulate sort speed for lists of various sizes. - - The sizes are 2**i for i in r (the argument, a list). - - The output displays i, 2**i, and the time to sort arrays of 2**i - floating point numbers with the following properties: - - *sort: random data - \sort: descending data - /sort: ascending data - 3sort: ascending, then 3 random exchanges - +sort: ascending, then 10 random at the end - %sort: ascending, then randomly replace 1% of the elements w/ random values - ~sort: many duplicates - =sort: all equal - !sort: worst case scenario - - """ - cases = tuple([ch + "sort" for ch in r"*\/3+%~=!"]) - fmt = ("%2s %7s" + " %6s"*len(cases)) - print(fmt % (("i", "2**i") + cases)) - for i in r: - n = 1 << i - L = randfloats(n) - print("%2d %7d" % (i, n), end=' ') - flush() - doit(L) # *sort - L.reverse() - doit(L) # \sort - doit(L) # /sort - - # Do 3 random exchanges. - for dummy in range(3): - i1 = random.randrange(n) - i2 = random.randrange(n) - L[i1], L[i2] = L[i2], L[i1] - doit(L) # 3sort - - # Replace the last 10 with random floats. - if n >= 10: - L[-10:] = [random.random() for dummy in range(10)] - doit(L) # +sort - - # Replace 1% of the elements at random. - for dummy in range(n // 100): - L[random.randrange(n)] = random.random() - doit(L) # %sort - - # Arrange for lots of duplicates. - if n > 4: - del L[4:] - L = L * (n // 4) - # Force the elements to be distinct objects, else timings can be - # artificially low. - L = list(map(lambda x: --x, L)) - doit(L) # ~sort - del L - - # All equal. Again, force the elements to be distinct objects. - L = list(map(abs, [-0.5] * n)) - doit(L) # =sort - del L - - # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case - # for an older implementation of quicksort, which used the median - # of the first, last and middle elements as the pivot. - half = n // 2 - L = list(range(half - 1, -1, -1)) - L.extend(range(half)) - # Force to float, so that the timings are comparable. This is - # significantly faster if we leave them as ints. - L = list(map(float, L)) - doit(L) # !sort - print() - -def main(): - """Main program when invoked as a script. - - One argument: tabulate a single row. - Two arguments: tabulate a range (inclusive). - Extra arguments are used to seed the random generator. - - """ - # default range (inclusive) - k1 = 15 - k2 = 20 - if sys.argv[1:]: - # one argument: single point - k1 = k2 = int(sys.argv[1]) - if sys.argv[2:]: - # two arguments: specify range - k2 = int(sys.argv[2]) - if sys.argv[3:]: - # derive random seed from remaining arguments - x = 1 - for a in sys.argv[3:]: - x = 69069 * x + hash(a) - random.seed(x) - r = range(k1, k2+1) # include the end point - tabulate(r) - -if __name__ == '__main__': - main() diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index cecf309dca9194..9bb0ce7bb57f8b 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1503,19 +1503,19 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 21e8770ab31180..7f6579319589b4 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -4,9 +4,7 @@ raise ImportError('support must be imported from the test package') import contextlib -import dataclasses import functools -import getpass import _opcode import os import re @@ -19,8 +17,6 @@ import unittest import warnings -from .testresult import get_test_runner - __all__ = [ # globals @@ -29,12 +25,11 @@ "Error", "TestFailed", "TestDidNotRun", "ResourceDenied", # io "record_original_stdout", "get_original_stdout", "captured_stdout", - "captured_stdin", "captured_stderr", + "captured_stdin", "captured_stderr", "captured_output", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "run_unittest", "run_doctest", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", @@ -47,7 +42,7 @@ "requires_limited_api", "requires_specialization", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", # network @@ -60,9 +55,10 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", + "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", "skip_on_s390x", "without_optimizer", + "force_not_colorized" ] @@ -386,19 +382,20 @@ def wrapper(*args, **kw): def skip_if_buildbot(reason=None): """Decorator raising SkipTest if running on a buildbot.""" + import getpass if not reason: reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: + except (KeyError, OSError) as err: warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False): +def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): """Returns True if Python is compiled with sanitizer support""" - if not (address or memory or ub): - raise ValueError('At least one of address, memory, or ub must be True') + if not (address or memory or ub or thread): + raise ValueError('At least one of address, memory, ub or thread must be True') cflags = sysconfig.get_config_var('CFLAGS') or '' @@ -415,18 +412,23 @@ def check_sanitizer(*, address=False, memory=False, ub=False): '-fsanitize=undefined' in cflags or '--with-undefined-behavior-sanitizer' in config_args ) + thread_sanitizer = ( + '-fsanitize=thread' in cflags or + '--with-thread-sanitizer' in config_args + ) return ( (memory and memory_sanitizer) or (address and address_sanitizer) or - (ub and ub_sanitizer) + (ub and ub_sanitizer) or + (thread and thread_sanitizer) ) -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub) + skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) return unittest.skipIf(skip, reason) # gh-89363: True if fork() can hang if Python is built with Address Sanitizer @@ -435,7 +437,7 @@ def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): def set_sanitizer_env_var(env, option): - for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS', 'TSAN_OPTIONS'): if name in env: env[name] += f':{option}' else: @@ -513,14 +515,42 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) +@contextlib.contextmanager +def suppress_immortalization(suppress=True): + """Suppress immortalization of deferred objects.""" + try: + import _testinternalcapi + except ImportError: + yield + return + + if not suppress: + yield + return + + _testinternalcapi.suppress_immortalization(True) + try: + yield + finally: + _testinternalcapi.suppress_immortalization(False) + +def skip_if_suppress_immortalization(): + try: + import _testinternalcapi + except ImportError: + return + return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), + "requires immortalization of deferred objects") + + MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" -if sys.platform not in ('win32', 'vxworks'): +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -530,19 +560,44 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile + + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets @@ -770,6 +825,16 @@ def disable_gc(): if have_gc: gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) + def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -799,7 +864,22 @@ def check_cflags_pgo(): return any(option in cflags_nodist for option in pgo_options) -_header = 'nP' +Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + +def expected_failure_if_gil_disabled(): + """Expect test failure if the GIL is disabled.""" + if Py_GIL_DISABLED: + return unittest.expectedFailure + return lambda test_case: test_case + +if Py_GIL_DISABLED: + _header = 'PHBBInP' +else: + _header = 'nP' _align = '0n' _vheader = _header + 'n' @@ -1082,18 +1162,50 @@ def check_impl_detail(**guards): def no_tracing(func): """Decorator to temporarily turn off tracing for the duration of a test.""" - if not hasattr(sys, 'gettrace'): - return func - else: + trace_wrapper = func + if hasattr(sys, 'gettrace'): @functools.wraps(func) - def wrapper(*args, **kwargs): + def trace_wrapper(*args, **kwargs): original_trace = sys.gettrace() try: sys.settrace(None) return func(*args, **kwargs) finally: sys.settrace(original_trace) + + coverage_wrapper = trace_wrapper + if 'test.cov' in sys.modules: # -Xpresite=test.cov used + cov = sys.monitoring.COVERAGE_ID + @functools.wraps(func) + def coverage_wrapper(*args, **kwargs): + original_events = sys.monitoring.get_events(cov) + try: + sys.monitoring.set_events(cov, 0) + return trace_wrapper(*args, **kwargs) + finally: + sys.monitoring.set_events(cov, original_events) + + return coverage_wrapper + + +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + assert not isinstance(func, type), func + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True return wrapper + return deco def refcount_test(test): @@ -1109,185 +1221,21 @@ def refcount_test(test): def requires_limited_api(test): try: - import _testcapi + import _testcapi # noqa: F401 + import _testlimitedcapi # noqa: F401 except ImportError: - return unittest.skip('needs _testcapi module')(test) + return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) return test + +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' + def requires_specialization(test): return unittest.skipUnless( _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests - -@dataclasses.dataclass(slots=True) -class TestStats: - tests_run: int = 0 - failures: int = 0 - skipped: int = 0 - - @staticmethod - def from_unittest(result): - return TestStats(result.testsRun, - len(result.failures), - len(result.skipped)) - - @staticmethod - def from_doctest(results): - return TestStats(results.attempted, - results.failed, - results.skipped) - - def accumulate(self, stats): - self.tests_run += stats.tests_run - self.failures += stats.failures - self.skipped += stats.skipped - - -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - runner = get_test_runner(sys.stdout, - verbosity=verbose, - capture_output=(junit_xml_list is not None)) - - result = runner.run(suite) - - if junit_xml_list is not None: - junit_xml_list.append(result.get_xml_element()) - - if not result.testsRun and not result.skipped and not result.errors: - raise TestDidNotRun - if not result.wasSuccessful(): - stats = TestStats.from_unittest(result) - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - errors = [(str(tc), exc_str) for tc, exc_str in result.errors] - failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures, stats=stats) - return result - - -# By default, don't filter tests -_match_test_func = None - -_accept_test_patterns = None -_ignore_test_patterns = None - - -def match_test(test): - # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) - - -def _is_full_match_test(pattern): - # If a pattern contains at least one dot, it's considered - # as a full test identifier. - # Example: 'test.test_os.FileTests.test_access'. - # - # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, ignore 'test_access*'. - return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) - - -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) - - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function - - -def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): - # Simple case: all patterns are full test identifier. - # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ - else: - import fnmatch - regex = '|'.join(map(fnmatch.translate, patterns)) - # The search *is* case sensitive on purpose: - # don't use flags=re.IGNORECASE - regex_match = re.compile(regex).match - - def match_test_regex(test_id): - if regex_match(test_id): - # The regex matches the whole identifier, for example - # 'test.test_os.FileTests.test_access'. - return True - else: - # Try to match parts of the test identifier. - # For example, split 'test.test_os.FileTests.test_access' - # into: 'test', 'test_os', 'FileTests' and 'test_access'. - return any(map(regex_match, test_id.split("."))) - - func = match_test_regex - - return patterns, func - - -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - loader = unittest.TestLoader() - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(loader.loadTestsFromTestCase(cls)) - _filter_suite(suite, match_test) - return _run_suite(suite) #======================================================================= # Check for the presence of docstrings. @@ -1309,38 +1257,6 @@ def _check_docstrings(): "test requires docstrings") -#======================================================================= -# doctest driver. - -def run_doctest(module, verbosity=None, optionflags=0): - """Run doctest on the given module. Return (#failures, #tests). - - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). - """ - - import doctest - - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - results = doctest.testmod(module, - verbose=verbosity, - optionflags=optionflags) - if results.failed: - stats = TestStats.from_doctest(results) - raise TestFailed(f"{results.failed} of {results.attempted} " - f"doctests failed", - stats=stats) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, results.attempted)) - return results - - #======================================================================= # Support for saving and restoring the imported modules. @@ -1863,7 +1779,10 @@ def run_in_subinterp(code): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.run_in_subinterp(code) @@ -1873,11 +1792,25 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + elif not isinstance(gil, str): + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): @@ -1895,19 +1828,22 @@ def _check_tracemalloc(): def check_free_after_iterating(test, iter, cls, args=()): - class A(cls): - def __del__(self): - nonlocal done - done = True - try: - next(it) - except StopIteration: - pass - done = False - it = iter(A(*args)) - # Issue 26494: Shouldn't crash - test.assertRaises(StopIteration, next, it) + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + + wrapper() # The sequence should be deallocated just after the end of iterating gc_collect() test.assertTrue(done) @@ -1946,18 +1882,18 @@ def missing_compiler_executable(cmd_names=[]): return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) @@ -2032,8 +1968,19 @@ def restore(self): def with_pymalloc(): - import _testcapi - return _testcapi.WITH_PYMALLOC + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED + + +def with_mimalloc(): + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_MIMALLOC class _ALWAYS_EQ: @@ -2305,13 +2252,13 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" - if max_depth < 3: +def infinite_recursion(max_depth=None): + if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 + elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. @@ -2352,7 +2299,7 @@ def clear_ignored_deprecations(*tokens: object) -> None: def requires_venv_with_pip(): # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) try: - import zlib + import zlib # noqa: F401 except ImportError: return unittest.skipIf(True, "venv: ensurepip requires zlib") @@ -2372,7 +2319,9 @@ def _findwheel(pkgname): If set, the wheels are searched for in WHEEL_PKG_DIR (see ensurepip). Otherwise, they are searched for in the test directory. """ - wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or TEST_HOME_DIR + wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or os.path.join( + TEST_HOME_DIR, 'wheeldata', + ) filenames = os.listdir(wheel_dir) filenames = sorted(filenames, reverse=True) # approximate "newest" first for filename in filenames: @@ -2389,16 +2338,25 @@ def _findwheel(pkgname): # and returns the path to the venv directory and the path to the python executable @contextlib.contextmanager def setup_venv_with_pip_setuptools_wheel(venv_dir): + import shlex import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2413,10 +2371,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): '-m', 'pip', 'install', _findwheel('setuptools'), _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) yield python @@ -2539,6 +2494,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" @@ -2549,11 +2544,19 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 -# The default C recursion limit (from Include/cpython/pystate.h). -Py_C_RECURSION_LIMIT = 1500 +def get_c_recursion_limit(): + try: + import _testcapi + return _testcapi.Py_C_RECURSION_LIMIT + except ImportError: + raise unittest.SkipTest('requires _testcapi') + + +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 + #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', @@ -2564,17 +2567,17 @@ def adjust_int_max_str_digits(max_digits): # Decorator to disable optimizer while a function run def without_optimizer(func): try: - import _testinternalcapi + from _testinternalcapi import get_optimizer, set_optimizer except ImportError: return func @functools.wraps(func) def wrapper(*args, **kwargs): - save_opt = _testinternalcapi.get_optimizer() + save_opt = get_optimizer() try: - _testinternalcapi.set_optimizer(None) + set_optimizer(None) return func(*args, **kwargs) finally: - _testinternalcapi.set_optimizer(save_opt) + set_optimizer(save_opt) return wrapper @@ -2603,3 +2606,61 @@ def copy_python_src_ignore(path, names): 'build', } return ignored + + +def force_not_colorized(func): + """Force the terminal not to be colorized.""" + @functools.wraps(func) + def wrapper(*args, **kwargs): + import _colorize + original_fn = _colorize.can_colorize + variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + try: + for key in variables: + variables[key] = os.environ.pop(key, None) + _colorize.can_colorize = lambda: False + return func(*args, **kwargs) + finally: + _colorize.can_colorize = original_fn + for key, value in variables.items(): + if value is not None: + os.environ[key] = value + return wrapper + + +def initialized_with_pyrepl(): + """Detect whether PyREPL was used during Python initialization.""" + # If the main module has a __file__ attribute it's a Python module, which means PyREPL. + return hasattr(sys.modules["__main__"], "__file__") + + +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + import signal + + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + # Shell exit code (ex: WASI build) + if 128 < exitcode < 256: + signum = exitcode - 128 + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + + return None diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d1266773c8a..85bcd1f0f1cd4f 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,10 +3,26 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" @@ -53,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. - - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expcted + # instruction sequence and compare with the one given. - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -72,10 +86,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -90,54 +102,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index db93eea5e912e0..40f58a2f59c6c3 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -5,6 +5,13 @@ except ImportError: from . import _hypothesis_stubs as hypothesis else: + # Regrtest changes to use a tempdir as the working directory, so we have + # to tell Hypothesis to use the original in order to persist the database. + from .os_helper import SAVEDCWD + from hypothesis.configuration import set_hypothesis_home_dir + + set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) + # When using the real Hypothesis, we'll configure it to ignore occasional # slow tests (avoiding flakiness from random VM slowness in CI). hypothesis.settings.register_profile( diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 3d804f2b590108..edcd2b9a35bbd9 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -114,7 +114,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,6 +268,18 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py new file mode 100644 index 00000000000000..e067f259364d2a --- /dev/null +++ b/Lib/test/support/interpreters/__init__.py @@ -0,0 +1,258 @@ +"""Subinterpreters High Level Module.""" + +import threading +import weakref +import _interpreters + +# aliases: +from _interpreters import ( + InterpreterError, InterpreterNotFoundError, NotShareableError, + is_shareable, +) + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', +] + + +_queuemod = None + +def __getattr__(name): + if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): + global create_queue, Queue, QueueEmpty, QueueFull + ns = globals() + from .queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, + ) + return ns[name] + else: + raise AttributeError(name) + + +_EXEC_FAILURE_STR = """ +{superstr} + +Uncaught in the interpreter: + +{formatted} +""".strip() + +class ExecutionFailed(InterpreterError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return _EXEC_FAILURE_STR.format( + superstr=super().__str__(), + formatted=formatted, + ) + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(reqrefs=True) + return Interpreter(id, _ownsref=True) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] + + +def get_current(): + """Return the currently running interpreter.""" + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) + + +def get_main(): + """Return the main interpreter.""" + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter. + + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } + + def __new__(cls, id, /, _whence=None, _ownsref=None): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + self = super().__new__(cls) + _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters.decref(self._id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + # Everything past here is available only to interpreters created by + # interpreters.create(). + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in an InterpreterError. + """ + return _interpreters.destroy(self._id, restrict=True) + + def prepare_main(self, ns=None, /, **kwargs): + """Bind the given values into the interpreter's __main__. + + The values must be shareable. + """ + ns = dict(ns, **kwargs) if ns is not None else kwargs + _interpreters.set___main___attrs(self._id, ns, restrict=True) + + def exec(self, code, /): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. + + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ + def task(): + self.call(callable) + t = threading.Thread(target=task) + t.start() + return t diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters/channels.py similarity index 51% rename from Lib/test/support/interpreters.py rename to Lib/test/support/interpreters/channels.py index 9ba6862a9ee01a..fbae7e634cf34d 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters/channels.py @@ -1,119 +1,23 @@ -"""Subinterpreters High Level Module.""" +"""Cross-interpreter Channels High Level Module.""" import time -import _xxsubinterpreters as _interpreters -import _xxinterpchannels as _channels +import _interpchannels as _channels # aliases: -from _xxsubinterpreters import is_shareable -from _xxinterpchannels import ( +from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, ) __all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', + 'create', 'list_all', 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', +] -def create(*, isolated=True): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -class Interpreter: - """A single Python interpreter.""" - - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated - - def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - - @property - def id(self): - return self._id - - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - # XXX Rename "run" to "exec"? - def run(self, src_str, /, *, channels=None): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then a RunFailedError - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - _interpreters.exec(self._id, src_str, channels) - - -def create_channel(): +def create(): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. @@ -123,7 +27,7 @@ def create_channel(): return recv, send -def list_all_channels(): +def list_all(): """Return a list of (recv, send) for all open channels.""" return [(RecvChannel(cid), SendChannel(cid)) for cid in _channels.list_all()] @@ -134,7 +38,8 @@ class _ChannelEnd: _end = None - def __init__(self, cid): + def __new__(cls, cid): + self = super().__new__(cls) if self._end == 'send': cid = _channels._channel_id(cid, send=True, force=True) elif self._end == 'recv': @@ -142,6 +47,7 @@ def __init__(self, cid): else: raise NotImplementedError(self._end) self._id = cid + return self def __repr__(self): return f'{type(self).__name__}(id={int(self._id)})' @@ -157,10 +63,26 @@ def __eq__(self, other): return NotImplemented return other._id == self._id + # for pickling: + def __getnewargs__(self): + return (int(self._id),) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id + @property + def _info(self): + return _channels.get_info(self._id) + + @property + def is_closed(self): + return self._info.closed + _NOT_SET = object() @@ -170,15 +92,25 @@ class RecvChannel(_ChannelEnd): _end = 'recv' - def recv(self, *, _sentinel=object(), _delay=10 / 1000): # 10 milliseconds + def recv(self, timeout=None, *, + _sentinel=object(), + _delay=10 / 1000, # 10 milliseconds + ): """Return the next object from the channel. This blocks until an object has been sent, if none have been sent already. """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout obj = _channels.recv(self._id, _sentinel) while obj is _sentinel: time.sleep(_delay) + if timeout is not None and time.time() >= end: + raise TimeoutError obj = _channels.recv(self._id, _sentinel) return obj @@ -203,12 +135,17 @@ class SendChannel(_ChannelEnd): _end = 'send' - def send(self, obj): + @property + def is_closed(self): + info = self._info + return info.closed or info.closing + + def send(self, obj, timeout=None): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - _channels.send(self._id, obj, blocking=True) + _channels.send(self._id, obj, timeout=timeout, blocking=True) def send_nowait(self, obj): """Send the object to the channel's receiving end. @@ -221,12 +158,12 @@ def send_nowait(self, obj): # See bpo-32604 and gh-19829. return _channels.send(self._id, obj, blocking=False) - def send_buffer(self, obj): + def send_buffer(self, obj, timeout=None): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - _channels.send_buffer(self._id, obj, blocking=True) + _channels.send_buffer(self._id, obj, timeout=timeout, blocking=True) def send_buffer_nowait(self, obj): """Send the object's buffer to the channel's receiving end. @@ -241,4 +178,4 @@ def close(self): # XXX This is causing leaks (gh-110318): -#_channels._register_end_types(SendChannel, RecvChannel) +_channels._register_end_types(SendChannel, RecvChannel) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py new file mode 100644 index 00000000000000..1b9e7481f2e313 --- /dev/null +++ b/Lib/test/support/interpreters/queues.py @@ -0,0 +1,230 @@ +"""Cross-interpreter Queues High Level Module.""" + +import pickle +import queue +import time +import weakref +import _interpqueues as _queues + +# aliases: +from _interpqueues import ( + QueueError, QueueNotFoundError, +) + +__all__ = [ + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', +] + + +class QueueEmpty(QueueError, queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(QueueError, queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +_SHARED_ONLY = 0 +_PICKLED = 1 + +def create(maxsize=0, *, syncobj=False): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). + """ + fmt = _SHARED_ONLY if syncobj else _PICKLED + qid = _queues.create(maxsize, fmt) + return Queue(qid, _fmt=fmt) + + +def list_all(): + """Return a list of all open queues.""" + return [Queue(qid, _fmt=fmt) + for qid, fmt in _queues.list_all()] + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + def __new__(cls, id, /, *, _fmt=None): + # There is only one instance for any given ID. + if isinstance(id, int): + id = int(id) + else: + raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + _fmt, = _queues.get_queue_defaults(id) + try: + self = _known_queues[id] + except KeyError: + self = super().__new__(cls) + self._id = id + self._fmt = _fmt + _known_queues[id] = self + _queues.bind(id) + return self + + def __del__(self): + try: + _queues.release(self._id) + except QueueNotFoundError: + pass + try: + del _known_queues[self._id] + except KeyError: + pass + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + + @property + def id(self): + return self._id + + @property + def maxsize(self): + try: + return self._maxsize + except AttributeError: + self._maxsize = _queues.get_maxsize(self._id) + return self._maxsize + + def empty(self): + return self.qsize() == 0 + + def full(self): + return _queues.is_full(self._id) + + def qsize(self): + return _queues.get_count(self._id) + + def put(self, obj, timeout=None, *, + syncobj=None, + _delay=10 / 1000, # 10 milliseconds + ): + """Add the object to the queue. + + This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue().. + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. + """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) + while True: + try: + _queues.put(self._id, obj, fmt) + except QueueFull as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + + def put_nowait(self, obj, *, syncobj=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if fmt is _PICKLED: + obj = pickle.dumps(obj) + _queues.put(self._id, obj, fmt) + + def get(self, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + This blocks while the queue is empty. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + obj, fmt = _queues.get(self._id) + except QueueEmpty as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj + + def get_nowait(self): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + try: + obj, fmt = _queues.get(self._id) + except QueueEmpty as exc: + raise # re-raise + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj + + +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 821a4b1ffd5077..891405943b78c5 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -10,6 +10,8 @@ import unittest import warnings +from test import support + # Filename used for testing TESTFN_ASCII = '@test' @@ -20,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -46,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) @@ -196,6 +198,23 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) +_can_hardlink = None + +def can_hardlink(): + global _can_hardlink + if _can_hardlink is None: + # Android blocks hard links using SELinux + # (https://stackoverflow.com/q/32365690). + _can_hardlink = hasattr(os, "link") and not support.is_android + return _can_hardlink + + +def skip_unless_hardlink(test): + ok = can_hardlink() + msg = "requires hardlink support" + return test if ok else unittest.skip(msg)(test) + + _can_xattr = None @@ -245,15 +264,15 @@ def can_chmod(): global _can_chmod if _can_chmod is not None: return _can_chmod - if not hasattr(os, "chown"): + if not hasattr(os, "chmod"): _can_chmod = False return _can_chmod try: with open(TESTFN, "wb") as f: try: - os.chmod(TESTFN, 0o777) + os.chmod(TESTFN, 0o555) mode1 = os.stat(TESTFN).st_mode - os.chmod(TESTFN, 0o666) + os.chmod(TESTFN, 0o777) mode2 = os.stat(TESTFN).st_mode except OSError as e: can = False @@ -300,6 +319,10 @@ def can_dac_override(): else: _can_dac_override = True finally: + try: + os.chmod(TESTFN, 0o700) + except OSError: + pass unlink(TESTFN) return _can_dac_override @@ -589,11 +612,18 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): + fd_path = "/proc/self/fd" + elif sys.platform == "darwin": + fd_path = "/dev/fd" + else: + fd_path = None + + if fd_path is not None: try: - names = os.listdir("/proc/self/fd") + names = os.listdir(fd_path) # Subtract one because listdir() internally opens a file - # descriptor to list the content of the /proc/self/fd/ directory. + # descriptor to list the content of the directory. return len(names) - 1 except FileNotFoundError: pass @@ -602,7 +632,8 @@ def fd_count(): if hasattr(os, 'sysconf'): try: MAXFD = os.sysconf("SC_OPEN_MAX") - except OSError: + except (OSError, ValueError): + # gh-118201: ValueError is raised intermittently on iOS pass old_modes = None @@ -720,13 +751,16 @@ def __exit__(self, *ignore_exc): try: - import ctypes - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - - ERROR_FILE_NOT_FOUND = 2 - DDD_REMOVE_DEFINITION = 2 - DDD_EXACT_MATCH_ON_REMOVE = 4 - DDD_NO_BROADCAST_SYSTEM = 8 + if support.MS_WINDOWS: + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + ERROR_FILE_NOT_FOUND = 2 + DDD_REMOVE_DEFINITION = 2 + DDD_EXACT_MATCH_ON_REMOVE = 4 + DDD_NO_BROADCAST_SYSTEM = 8 + else: + raise AttributeError except (ImportError, AttributeError): def subst_drive(path): raise unittest.SkipTest('ctypes or kernel32 is not available') diff --git a/Lib/test/support/pty_helper.py b/Lib/test/support/pty_helper.py new file mode 100644 index 00000000000000..6587fd40333c51 --- /dev/null +++ b/Lib/test/support/pty_helper.py @@ -0,0 +1,80 @@ +""" +Helper to run a script in a pseudo-terminal. +""" +import os +import selectors +import subprocess +import sys +from contextlib import ExitStack +from errno import EIO + +from test.support.import_helper import import_module + +def run_pty(script, input=b"dummy input\r", env=None): + pty = import_module('pty') + output = bytearray() + [master, slave] = pty.openpty() + args = (sys.executable, '-c', script) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) + os.close(slave) + with ExitStack() as cleanup: + cleanup.enter_context(proc) + def terminate(proc): + try: + proc.terminate() + except ProcessLookupError: + # Workaround for Open/Net BSD bug (Issue 16762) + pass + cleanup.callback(terminate, proc) + cleanup.callback(os.close, master) + # Avoid using DefaultSelector and PollSelector. Kqueue() does not + # work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open + # BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4 + # either (Issue 20472). Hopefully the file descriptor is low enough + # to use with select(). + sel = cleanup.enter_context(selectors.SelectSelector()) + sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) + os.set_blocking(master, False) + while True: + for [_, events] in sel.select(): + if events & selectors.EVENT_READ: + try: + chunk = os.read(master, 0x10000) + except OSError as err: + # Linux raises EIO when slave is closed (Issue 5380) + if err.errno != EIO: + raise + chunk = b"" + if not chunk: + return output + output.extend(chunk) + if events & selectors.EVENT_WRITE: + try: + input = input[os.write(master, input):] + except OSError as err: + # Apparently EIO means the slave was closed + if err.errno != EIO: + raise + input = b"" # Stop writing + if not input: + sel.modify(master, selectors.EVENT_READ) + + +###################################################################### +## Fake stdin (for testing interactive debugging) +###################################################################### + +class FakeInput: + """ + A fake input stream for pdb's interactive debugger. Whenever a + line is read, print it (to simulate the user typing it), and then + return it. The set of lines to return is specified in the + constructor; they should not have trailing newlines. + """ + def __init__(self, lines): + self.lines = lines + + def readline(self): + line = self.lines.pop(0) + print(line) + return line + '\n' diff --git a/Lib/test/support/refleak_helper.py b/Lib/test/support/refleak_helper.py new file mode 100644 index 00000000000000..2f86c93a1e2e58 --- /dev/null +++ b/Lib/test/support/refleak_helper.py @@ -0,0 +1,8 @@ +""" +Utilities for changing test behaviour while hunting +for refleaks +""" + +_hunting_for_refleaks = False +def hunting_for_refleaks(): + return _hunting_for_refleaks diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index c2b43f4060eb55..d0be3179b0efa3 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -8,7 +8,6 @@ import os.path import subprocess import py_compile -import zipfile from importlib.util import source_from_cache from test import support @@ -64,42 +63,59 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] if len(err) > maxlen: err = b'(... truncated stderr ...)' + err[-maxlen:] - out = out.decode('ascii', 'replace').rstrip() - err = err.decode('ascii', 'replace').rstrip() - raise AssertionError("Process return code is %d\n" - "command line: %r\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s\n" - "---" - % (self.rc, cmd_line, - out, - err)) + out = out.decode('utf8', 'replace').rstrip() + err = err.decode('utf8', 'replace').rstrip() + + exitcode = self.rc + signame = support.get_signal_name(exitcode) + if signame: + exitcode = f"{exitcode} ({signame})" + raise AssertionError(f"Process return code is {exitcode}\n" + f"command line: {cmd_line!r}\n" + f"\n" + f"stdout:\n" + f"---\n" + f"{out}\n" + f"---\n" + f"\n" + f"stderr:\n" + f"---\n" + f"{err}\n" + f"---") # Executing the interpreter in a subprocess @support.requires_subprocess() def run_python_until_end(*args, **env_vars): + """Used to implement assert_python_*. + + *args are the command line flags to pass to the python interpreter. + **env_vars keyword arguments are environment variables to set on the process. + + If __run_using_command= is supplied, it must be a list of + command line arguments to prepend to the command line used. + Useful when you want to run another command that should launch the + python interpreter via its own arguments. ["/bin/echo", "--"] for + example could print the unquoted python command line instead of + run it. + """ env_required = interpreter_requires_environment() + run_using_command = env_vars.pop('__run_using_command', None) cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required cmd_line = [sys.executable, '-X', 'faulthandler'] + if run_using_command: + cmd_line = run_using_command + cmd_line if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path @@ -226,6 +242,7 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + import zipfile zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) with zipfile.ZipFile(zip_name, 'w') as zip_file: @@ -252,6 +269,7 @@ def make_pkg(pkg_dir, init_source=''): def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source, depth=1, compiled=False): + import zipfile unlink = [] init_name = make_script(zip_dir, '__init__', '') unlink.append(init_name) diff --git a/Lib/test/smtpd.py b/Lib/test/support/smtpd.py similarity index 98% rename from Lib/test/smtpd.py rename to Lib/test/support/smtpd.py index 6052232ec2b585..c2e17cad422861 100755 --- a/Lib/test/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -672,9 +672,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -685,7 +685,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 7f16050f32b9d1..afa25a76f63829 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -22,34 +22,37 @@ def threading_setup(): - return _thread._count(), threading._dangling.copy() + return _thread._count(), len(threading._dangling) def threading_cleanup(*original_values): - _MAX_COUNT = 100 - - for count in range(_MAX_COUNT): - values = _thread._count(), threading._dangling - if values == original_values: - break - - if not count: - # Display a warning at the first iteration - support.environment_altered = True - dangling_threads = values[1] - support.print_warning(f"threading_cleanup() failed to cleanup " - f"{values[0] - original_values[0]} threads " - f"(count: {values[0]}, " - f"dangling: {len(dangling_threads)})") - for thread in dangling_threads: - support.print_warning(f"Dangling thread: {thread!r}") - - # Don't hold references to threads - dangling_threads = None - values = None - - time.sleep(0.01) - support.gc_collect() + orig_count, orig_ndangling = original_values + + timeout = 1.0 + for _ in support.sleeping_retry(timeout, error=False): + # Copy the thread list to get a consistent output. threading._dangling + # is a WeakSet, its value changes when it's read. + dangling_threads = list(threading._dangling) + count = _thread._count() + + if count <= orig_count: + return + + # Timeout! + support.environment_altered = True + support.print_warning( + f"threading_cleanup() failed to clean up threads " + f"in {timeout:.1f} seconds\n" + f" before: thread count={orig_count}, dangling={orig_ndangling}\n" + f" after: thread count={count}, dangling={len(dangling_threads)}") + for thread in dangling_threads: + support.print_warning(f"Dangling thread: {thread!r}") + + # The warning happens when a test spawns threads and some of these threads + # are still running after the test completes. To fix this warning, join + # threads explicitly to wait until they complete. + # + # To make the warning more likely, reduce the timeout. def reap_threads(func): diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index c87cde4b3d1fab..e405056c8ffcb5 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,12 +3,6 @@ from test.support import warnings_helper import os import sys -import types - -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None if support.check_sanitizer(address=True, memory=True): @@ -36,17 +30,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( @@ -120,7 +103,7 @@ def test_all(self): # In case _socket fails to build, make this test fail more gracefully # than an AttributeError somewhere deep in concurrent.futures, email # or unittest. - import _socket + import _socket # noqa: F401 ignored = [] failed_imports = [] diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 00000000000000..d55b97ade68cef --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,59 @@ +import contextlib +import sys +import unittest +import unittest.mock +import _colorize +from test.support import force_not_colorized + +ORIGINAL_CAN_COLORIZE = _colorize.can_colorize + + +def setUpModule(): + _colorize.can_colorize = lambda: False + + +def tearDownModule(): + _colorize.can_colorize = ORIGINAL_CAN_COLORIZE + + +class TestColorizeFunction(unittest.TestCase): + @force_not_colorized + def test_colorized_detection_checks_for_environment_variables(self): + if sys.platform == "win32": + virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", + return_value=True) + else: + virtual_patching = contextlib.nullcontext() + with virtual_patching: + + flags = unittest.mock.MagicMock(ignore_environment=False) + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.flags", flags), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + isatty_mock.return_value = False + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__interpchannels.py similarity index 55% rename from Lib/test/test__xxinterpchannels.py rename to Lib/test/test__interpchannels.py index ff01a339c0008e..b76c58917c0b9c 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -8,14 +8,19 @@ from test.support import import_helper -from test.test__xxsubinterpreters import ( - interpreters, +from test.test__interpreters import ( + _interpreters, _run_output, clean_up_interpreters, ) -channels = import_helper.import_module('_xxinterpchannels') +_channels = import_helper.import_module('_interpchannels') + + +# Additional tests are found in Lib/test/test_interpreters/test_channels.py. +# New tests should be added there. +# XXX The tests here should be moved there. See the note under LowLevelTests. ################################## @@ -24,8 +29,8 @@ def recv_wait(cid): while True: try: - return channels.recv(cid) - except channels.ChannelEmptyError: + return _channels.recv(cid) + except _channels.ChannelEmptyError: time.sleep(0.1) #@contextmanager @@ -44,14 +49,15 @@ def run_interp(id, source, **shared): def _run_interp(id, source, shared, _mainns={}): source = dedent(source) - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if main == id: - if interpreters.get_current() != main: + cur, *_ = _interpreters.get_current() + if cur != main: raise RuntimeError # XXX Run a func? exec(source, _mainns) else: - interpreters.run_string(id, source, shared) + _interpreters.run_string(id, source, shared) class Interpreter(namedtuple('Interpreter', 'name id')): @@ -66,7 +72,7 @@ def from_raw(cls, raw): raise NotImplementedError def __new__(cls, name=None, id=None): - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if id == main: if not name: name = 'main' @@ -79,13 +85,12 @@ def __new__(cls, name=None, id=None): name = 'interp' elif name == 'main': raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) + assert isinstance(id, int), repr(id) elif not name or name == 'main': name = 'main' id = main else: - id = interpreters.create() + id = _interpreters.create() self = super().__new__(cls, name, id) return self @@ -96,7 +101,7 @@ def __new__(cls, name=None, id=None): def expect_channel_closed(): try: yield - except channels.ChannelClosedError: + except _channels.ChannelClosedError: pass else: assert False, 'channel not closed' @@ -183,7 +188,7 @@ def run_action(cid, action, end, state, *, hideclosed=True): try: result = _run_action(cid, action, end, state) - except channels.ChannelClosedError: + except _channels.ChannelClosedError: if not hideclosed and not expectfail: raise result = state.close() @@ -196,18 +201,18 @@ def run_action(cid, action, end, state, *, hideclosed=True): def _run_action(cid, action, end, state): if action == 'use': if end == 'send': - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) return state.incr() elif end == 'recv': if not state.pending: try: - channels.recv(cid) - except channels.ChannelEmptyError: + _channels.recv(cid) + except _channels.ChannelEmptyError: return state else: raise Exception('expected ChannelEmptyError') else: - channels.recv(cid) + _channels.recv(cid) return state.decr() else: raise ValueError(end) @@ -215,7 +220,7 @@ def _run_action(cid, action, end, state): kwargs = {} if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close() elif action == 'force-close': kwargs = { @@ -223,17 +228,17 @@ def _run_action(cid, action, end, state): } if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close(force=True) else: raise ValueError(action) def clean_up_channels(): - for cid in channels.list_all(): + for cid in _channels.list_all(): try: - channels.destroy(cid) - except channels.ChannelNotFoundError: + _channels.destroy(cid) + except _channels.ChannelNotFoundError: pass # already destroyed @@ -250,25 +255,25 @@ def tearDown(self): class ChannelIDTests(TestBase): def test_default_kwargs(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(int(cid), 10) self.assertEqual(cid.end, 'both') def test_with_kwargs(self): - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, send=True, recv=False, force=True) + cid = _channels._channel_id(10, send=True, recv=False, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, recv=True, send=False, force=True) + cid = _channels._channel_id(10, recv=True, send=False, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(cid.end, 'both') def test_coerce_id(self): @@ -276,47 +281,47 @@ class Int(str): def __index__(self): return 10 - cid = channels._channel_id(Int(), force=True) + cid = _channels._channel_id(Int(), force=True) self.assertEqual(int(cid), 10) def test_bad_id(self): - self.assertRaises(TypeError, channels._channel_id, object()) - self.assertRaises(TypeError, channels._channel_id, 10.0) - self.assertRaises(TypeError, channels._channel_id, '10') - self.assertRaises(TypeError, channels._channel_id, b'10') - self.assertRaises(ValueError, channels._channel_id, -1) - self.assertRaises(OverflowError, channels._channel_id, 2**64) + self.assertRaises(TypeError, _channels._channel_id, object()) + self.assertRaises(TypeError, _channels._channel_id, 10.0) + self.assertRaises(TypeError, _channels._channel_id, '10') + self.assertRaises(TypeError, _channels._channel_id, b'10') + self.assertRaises(ValueError, _channels._channel_id, -1) + self.assertRaises(OverflowError, _channels._channel_id, 2**64) def test_bad_kwargs(self): with self.assertRaises(ValueError): - channels._channel_id(10, send=False, recv=False) + _channels._channel_id(10, send=False, recv=False) def test_does_not_exist(self): - cid = channels.create() - with self.assertRaises(channels.ChannelNotFoundError): - channels._channel_id(int(cid) + 1) # unforced + cid = _channels.create() + with self.assertRaises(_channels.ChannelNotFoundError): + _channels._channel_id(int(cid) + 1) # unforced def test_str(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(str(cid), '10') def test_repr(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') def test_equality(self): - cid1 = channels.create() - cid2 = channels._channel_id(int(cid1)) - cid3 = channels.create() + cid1 = _channels.create() + cid2 = _channels._channel_id(int(cid1)) + cid3 = _channels.create() self.assertTrue(cid1 == cid1) self.assertTrue(cid1 == cid2) @@ -336,11 +341,11 @@ def test_equality(self): self.assertTrue(cid1 != cid3) def test_shareable(self): - chan = channels.create() + chan = _channels.create() - obj = channels.create() - channels.send(chan, obj, blocking=False) - got = channels.recv(chan) + obj = _channels.create() + _channels.send(chan, obj, blocking=False) + got = _channels.recv(chan) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -351,32 +356,32 @@ def test_shareable(self): class ChannelTests(TestBase): def test_create_cid(self): - cid = channels.create() - self.assertIsInstance(cid, channels.ChannelID) + cid = _channels.create() + self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = channels.list_all() - id1 = channels.create() - id2 = channels.create() - id3 = channels.create() - after = channels.list_all() + before = _channels.list_all() + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + after = _channels.list_all() self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels cid = _channels.create() print(cid) """)) cid1 = int(out.strip()) - id2 = interpreters.create() + id2 = _interpreters.create() out = _run_output(id2, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels cid = _channels.create() print(cid) """)) @@ -386,81 +391,81 @@ def test_ids_global(self): def test_channel_list_interpreters_none(self): """Test listing interpreters for a channel with no associations.""" - # Test for channel with no associated interpreters. - cid = channels.create() - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + # Test for channel with no associated _interpreters. + cid = _channels.create() + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, []) self.assertEqual(recv_interps, []) def test_channel_list_interpreters_basic(self): - """Test basic listing channel interpreters.""" - interp0 = interpreters.get_main() - cid = channels.create() - channels.send(cid, "send", blocking=False) + """Test basic listing channel _interpreters.""" + interp0, *_ = _interpreters.get_main() + cid = _channels.create() + _channels.send(cid, "send", blocking=False) # Test for a channel that has one end associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) - interp1 = interpreters.create() + interp1 = _interpreters.create() _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) # Test for channel that has both ends associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_multiple(self): """Test listing interpreters for a channel with many associations.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + interp3 = _interpreters.create() + cid = _channels.create() - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, "send", blocking=False) """)) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) _run_output(interp3, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(set(send_interps), {interp0, interp1}) self.assertEqual(set(recv_interps), {interp2, interp3}) def test_channel_list_interpreters_destroyed(self): """Test listing channel interpreters with a destroyed interpreter.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() - channels.send(cid, "send", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create() + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) # Should be one interpreter associated with each end. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) - interpreters.destroy(interp1) + _interpreters.destroy(interp1) # Destroyed interpreter should not be listed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) @@ -468,117 +473,172 @@ def test_channel_list_interpreters_released(self): """Test listing channel interpreters with a released channel.""" # Set up one channel with main interpreter on the send end and two # subinterpreters on the receive end. - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - cid = channels.create() - channels.send(cid, "data", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + cid = _channels.create() + _channels.send(cid, "data", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) - channels.send(cid, "data", blocking=False) + _channels.send(cid, "data", blocking=False) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) """)) # Check the setup. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 2) # Release the main interpreter from the send end. - channels.release(cid, send=True) - # Send end should have no associated interpreters. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + _channels.release(cid, send=True) + # Send end should have no associated _interpreters. + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(len(recv_interps), 2) # Release one of the subinterpreters from the receive end. _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.release({cid}) """)) # Receive end should have the released interpreter removed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_closed(self): """Test listing channel interpreters with a closed channel.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create() # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Force close the channel. - channels.close(cid, force=True) + _channels.close(cid, force=True) # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) def test_channel_list_interpreters_closed_send_end(self): """Test listing channel interpreters with a channel's send end closed.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create() # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Close the send end of the channel. - channels.close(cid, send=True) + _channels.close(cid, send=True) # Send end should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) # Receive end should not be closed (since channel is not empty). - recv_interps = channels.list_interpreters(cid, send=False) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(recv_interps), 0) # Close the receive end of the channel from a subinterpreter. _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.close({cid}, force=True) """)) return # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) + + def test_allowed_types(self): + cid = _channels.create() + objects = [ + None, + 'spam', + b'spam', + 42, + ] + for obj in objects: + with self.subTest(obj): + _channels.send(cid, obj, blocking=False) + got = _channels.recv(cid) + + self.assertEqual(got, obj) + self.assertIs(type(got), type(obj)) + # XXX Check the following? + #self.assertIsNot(got, obj) + # XXX What about between interpreters? + + def test_run_string_arg_unresolved(self): + cid = _channels.create() + interp = _interpreters.create() + + _interpreters.set___main___attrs(interp, dict(cid=cid.send)) + out = _run_output(interp, dedent(""" + import _interpchannels as _channels + print(cid.end) + _channels.send(cid, b'spam', blocking=False) + """)) + obj = _channels.recv(cid) - #################### + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + # XXX For now there is no high-level channel into which the + # sent channel ID can be converted... + # Note: this test caused crashes on some buildbots (bpo-33615). + @unittest.skip('disabled until high-level channels exist') + def test_run_string_arg_resolved(self): + cid = _channels.create() + cid = _channels._channel_id(cid, _resolve=True) + interp = _interpreters.create() + + out = _run_output(interp, dedent(""" + import _interpchannels as _channels + print(chan.id.end) + _channels.send(chan.id, b'spam', blocking=False) + """), + dict(chan=cid.send)) + obj = _channels.recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + #------------------- + # send/recv def test_send_recv_main(self): - cid = channels.create() + cid = _channels.create() orig = b'spam' - channels.send(cid, orig, blocking=False) - obj = channels.recv(cid) + _channels.send(cid, orig, blocking=False) + obj = _channels.recv(cid) self.assertEqual(obj, orig) self.assertIsNot(obj, orig) def test_send_recv_same_interpreter(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels cid = _channels.create() orig = b'spam' _channels.send(cid, orig, blocking=False) @@ -588,41 +648,41 @@ def test_send_recv_same_interpreter(self): """)) def test_send_recv_different_interpreters(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - obj = channels.recv(cid) + obj = _channels.recv(cid) self.assertEqual(obj, b'spam') def test_send_recv_different_threads(self): - cid = channels.create() + cid = _channels.create() def f(): obj = recv_wait(cid) - channels.send(cid, obj) + _channels.send(cid, obj) t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'spam') def test_send_recv_different_interpreters_and_threads(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create() + id1 = _interpreters.create() out = None def f(): nonlocal out out = _run_output(id1, dedent(f""" import time - import _xxinterpchannels as _channels + import _interpchannels as _channels while True: try: obj = _channels.recv({cid}) @@ -635,38 +695,38 @@ def f(): t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'eggs') def test_send_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.send(10, b'spam') + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.send(10, b'spam') def test_recv_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.recv(10) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.recv(10) def test_recv_empty(self): - cid = channels.create() - with self.assertRaises(channels.ChannelEmptyError): - channels.recv(cid) + cid = _channels.create() + with self.assertRaises(_channels.ChannelEmptyError): + _channels.recv(cid) def test_recv_default(self): default = object() - cid = channels.create() - obj1 = channels.recv(cid, default) - channels.send(cid, None, blocking=False) - channels.send(cid, 1, blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'eggs', blocking=False) - obj2 = channels.recv(cid, default) - obj3 = channels.recv(cid, default) - obj4 = channels.recv(cid) - obj5 = channels.recv(cid, default) - obj6 = channels.recv(cid, default) + cid = _channels.create() + obj1 = _channels.recv(cid, default) + _channels.send(cid, None, blocking=False) + _channels.send(cid, 1, blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'eggs', blocking=False) + obj2 = _channels.recv(cid, default) + obj3 = _channels.recv(cid, default) + obj4 = _channels.recv(cid) + obj5 = _channels.recv(cid, default) + obj6 = _channels.recv(cid, default) self.assertIs(obj1, default) self.assertIs(obj2, None) @@ -677,39 +737,42 @@ def test_recv_default(self): def test_recv_sending_interp_destroyed(self): with self.subTest('closed'): - cid1 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid1 = _channels.create() + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid1}, b'spam', blocking=False) """)) - interpreters.destroy(interp) + _interpreters.destroy(interp) with self.assertRaisesRegex(RuntimeError, f'channel {cid1} is closed'): - channels.recv(cid1) + _channels.recv(cid1) del cid1 with self.subTest('still open'): - cid2 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid2 = _channels.create() + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid2}, b'spam', blocking=False) """)) - channels.send(cid2, b'eggs', blocking=False) - interpreters.destroy(interp) + _channels.send(cid2, b'eggs', blocking=False) + _interpreters.destroy(interp) - channels.recv(cid2) + _channels.recv(cid2) with self.assertRaisesRegex(RuntimeError, f'channel {cid2} is empty'): - channels.recv(cid2) + _channels.recv(cid2) del cid2 + #------------------- + # send_buffer + def test_send_buffer(self): buf = bytearray(b'spamspamspam') - cid = channels.create() - channels.send_buffer(cid, buf, blocking=False) - obj = channels.recv(cid) + cid = _channels.create() + _channels.send_buffer(cid, buf, blocking=False) + obj = _channels.recv(cid) self.assertIsNot(obj, buf) self.assertIsInstance(obj, memoryview) @@ -720,105 +783,264 @@ def test_send_buffer(self): obj[4:8] = b'ham.' self.assertEqual(obj, buf) - def test_allowed_types(self): - cid = channels.create() - objects = [ - None, - 'spam', - b'spam', - 42, - ] - for obj in objects: - with self.subTest(obj): - channels.send(cid, obj, blocking=False) - got = channels.recv(cid) + #------------------- + # send with waiting - self.assertEqual(got, obj) - self.assertIs(type(got), type(obj)) - # XXX Check the following? - #self.assertIsNot(got, obj) - # XXX What about between interpreters? + def build_send_waiter(self, obj, *, buffer=False): + # We want a long enough sleep that send() actually has to wait. - def test_run_string_arg_unresolved(self): - cid = channels.create() - interp = interpreters.create() + if buffer: + send = _channels.send_buffer + else: + send = _channels.send - out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels - print(cid.end) - _channels.send(cid, b'spam', blocking=False) - """), - dict(cid=cid.send)) - obj = channels.recv(cid) + cid = _channels.create() + try: + started = time.monotonic() + send(cid, obj, blocking=False) + stopped = time.monotonic() + _channels.recv(cid) + finally: + _channels.destroy(cid) + delay = stopped - started # seconds + delay *= 3 + + def wait(): + time.sleep(delay) + return wait + + def test_send_blocking_waiting(self): + received = None + obj = b'spam' + wait = self.build_send_waiter(obj) + cid = _channels.create() + def f(): + nonlocal received + wait() + received = recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send(cid, obj, blocking=True) + t.join() - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') + self.assertEqual(received, obj) - # XXX For now there is no high-level channel into which the - # sent channel ID can be converted... - # Note: this test caused crashes on some buildbots (bpo-33615). - @unittest.skip('disabled until high-level channels exist') - def test_run_string_arg_resolved(self): - cid = channels.create() - cid = channels._channel_id(cid, _resolve=True) - interp = interpreters.create() + def test_send_buffer_blocking_waiting(self): + received = None + obj = bytearray(b'spam') + wait = self.build_send_waiter(obj, buffer=True) + cid = _channels.create() + def f(): + nonlocal received + wait() + received = recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send_buffer(cid, obj, blocking=True) + t.join() - out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels - print(chan.id.end) - _channels.send(chan.id, b'spam', blocking=False) - """), - dict(chan=cid.send)) - obj = channels.recv(cid) + self.assertEqual(received, obj) - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') + def test_send_blocking_no_wait(self): + received = None + obj = b'spam' + cid = _channels.create() + def f(): + nonlocal received + received = recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send(cid, obj, blocking=True) + t.join() + + self.assertEqual(received, obj) + def test_send_buffer_blocking_no_wait(self): + received = None + obj = bytearray(b'spam') + cid = _channels.create() + def f(): + nonlocal received + received = recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send_buffer(cid, obj, blocking=True) + t.join() + + self.assertEqual(received, obj) + + def test_send_timeout(self): + obj = b'spam' + + with self.subTest('non-blocking with timeout'): + cid = _channels.create() + with self.assertRaises(ValueError): + _channels.send(cid, obj, blocking=False, timeout=0.1) + + with self.subTest('timeout hit'): + cid = _channels.create() + with self.assertRaises(TimeoutError): + _channels.send(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = _channels.recv(cid) + print(repr(received)) + + with self.subTest('timeout not hit'): + cid = _channels.create() + def f(): + recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send(cid, obj, blocking=True, timeout=10) + t.join() + + def test_send_buffer_timeout(self): + try: + self._has_run_once_timeout + except AttributeError: + # At the moment, this test leaks a few references. + # It looks like the leak originates with the addition + # of _channels.send_buffer() (gh-110246), whereas the + # tests were added afterward. We want this test even + # if the refleak isn't fixed yet, so we skip here. + raise unittest.SkipTest('temporarily skipped due to refleaks') + else: + self._has_run_once_timeout = True + + obj = bytearray(b'spam') + + with self.subTest('non-blocking with timeout'): + cid = _channels.create() + with self.assertRaises(ValueError): + _channels.send_buffer(cid, obj, blocking=False, timeout=0.1) + + with self.subTest('timeout hit'): + cid = _channels.create() + with self.assertRaises(TimeoutError): + _channels.send_buffer(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = _channels.recv(cid) + print(repr(received)) + + with self.subTest('timeout not hit'): + cid = _channels.create() + def f(): + recv_wait(cid) + t = threading.Thread(target=f) + t.start() + _channels.send_buffer(cid, obj, blocking=True, timeout=10) + t.join() + + def test_send_closed_while_waiting(self): + obj = b'spam' + wait = self.build_send_waiter(obj) + + with self.subTest('without timeout'): + cid = _channels.create() + def f(): + wait() + _channels.close(cid, force=True) + t = threading.Thread(target=f) + t.start() + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True) + t.join() + + with self.subTest('with timeout'): + cid = _channels.create() + def f(): + wait() + _channels.close(cid, force=True) + t = threading.Thread(target=f) + t.start() + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True, timeout=30) + t.join() + + def test_send_buffer_closed_while_waiting(self): + try: + self._has_run_once_closed + except AttributeError: + # At the moment, this test leaks a few references. + # It looks like the leak originates with the addition + # of _channels.send_buffer() (gh-110246), whereas the + # tests were added afterward. We want this test even + # if the refleak isn't fixed yet, so we skip here. + raise unittest.SkipTest('temporarily skipped due to refleaks') + else: + self._has_run_once_closed = True + + obj = bytearray(b'spam') + wait = self.build_send_waiter(obj, buffer=True) + + with self.subTest('without timeout'): + cid = _channels.create() + def f(): + wait() + _channels.close(cid, force=True) + t = threading.Thread(target=f) + t.start() + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True) + t.join() + + with self.subTest('with timeout'): + cid = _channels.create() + def f(): + wait() + _channels.close(cid, force=True) + t = threading.Thread(target=f) + t.start() + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True, timeout=30) + t.join() + + #------------------- # close def test_close_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - interpreters.run_string(id2, dedent(f""" - import _xxinterpchannels as _channels + _interpreters.run_string(id2, dedent(f""" + import _interpchannels as _channels _channels.recv({cid}) """)) - channels.close(cid) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id1, dedent(f""" + _channels.close(cid) + + excsnap = _interpreters.run_string(id1, dedent(f""" _channels.send({cid}, b'spam') """)) - self.assertIn('ChannelClosedError', str(cm.exception)) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id2, dedent(f""" + self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') + + excsnap = _interpreters.run_string(id2, dedent(f""" _channels.send({cid}, b'spam') """)) - self.assertIn('ChannelClosedError', str(cm.exception)) + self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') def test_close_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_empty(self): tests = [ @@ -829,149 +1051,149 @@ def test_close_empty(self): ] for send, recv in tests: with self.subTest((send, recv)): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, send=send, recv=recv) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.close(cid, send=send, recv=recv) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_defaults_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid) + _channels.recv(cid) + _channels.send(cid, b'eggs', blocking=False) def test_close_recv_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True) + _channels.recv(cid) + _channels.send(cid, b'eggs', blocking=False) + _channels.recv(cid) + _channels.recv(cid) + _channels.close(cid, recv=True) def test_close_send_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - channels.recv(cid) - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + _channels.recv(cid) + _channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True, send=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True, send=True) + _channels.recv(cid) + _channels.send(cid, b'eggs', blocking=False) + _channels.recv(cid) + _channels.recv(cid) + _channels.close(cid, recv=True) def test_close_recv_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, recv=True, force=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_send_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, force=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, recv=True, force=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_never_used(self): - cid = channels.create() - channels.close(cid) + cid = _channels.create() + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.close({cid}, force=True) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, force=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.close(cid, force=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_channel_list_interpreters_invalid_channel(self): - cid = channels.create() + cid = _channels.create() # Test for invalid channel ID. - with self.assertRaises(channels.ChannelNotFoundError): - channels.list_interpreters(1000, send=True) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.list_interpreters(1000, send=True) - channels.close(cid) + _channels.close(cid) # Test for a channel that has been closed. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) def test_channel_list_interpreters_invalid_args(self): # Tests for invalid arguments passed to the API. - cid = channels.create() + cid = _channels.create() with self.assertRaises(TypeError): - channels.list_interpreters(cid) + _channels.list_interpreters(cid) class ChannelReleaseTests(TestBase): @@ -1018,125 +1240,125 @@ class ChannelReleaseTests(TestBase): """ def test_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) out = _run_output(id2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels obj = _channels.recv({cid}) _channels.release({cid}) print(repr(obj)) """)) - interpreters.run_string(id1, dedent(f""" + _interpreters.run_string(id1, dedent(f""" _channels.release({cid}) """)) self.assertEqual(out.strip(), "b'spam'") def test_no_kwargs(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.release(cid, send=True, recv=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.release(cid, send=True, recv=True) def test_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.release(cid, send=True, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_never_used(self): - cid = channels.create() - channels.release(cid) + cid = _channels.create() + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.release({cid}) """)) - obj = channels.recv(cid) - channels.release(cid) + obj = _channels.recv(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') self.assertEqual(obj, b'spam') def test_close_if_unassociated(self): # XXX Something's not right with this test... - cid = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create() + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels obj = _channels.send({cid}, b'spam', blocking=False) _channels.release({cid}) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_partially(self): # XXX Is partial close too weird/confusing? - cid = channels.create() - channels.send(cid, None, blocking=False) - channels.recv(cid) - channels.send(cid, b'spam', blocking=False) - channels.release(cid, send=True) - obj = channels.recv(cid) + cid = _channels.create() + _channels.send(cid, None, blocking=False) + _channels.recv(cid) + _channels.send(cid, b'spam', blocking=False) + _channels.release(cid, send=True) + obj = _channels.recv(cid) self.assertEqual(obj, b'spam') def test_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create() + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.recv(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) class ChannelCloseFixture(namedtuple('ChannelCloseFixture', @@ -1206,18 +1428,18 @@ def clean_up(self): def _new_channel(self, creator): if creator.name == 'main': - return channels.create() + return _channels.create() else: - ch = channels.create() + ch = _channels.create() run_interp(creator.id, f""" - import _xxsubinterpreters + import _interpreters cid = _xxsubchannels.create() # We purposefully send back an int to avoid tying the # channel to the other interpreter. _xxsubchannels.send({ch}, int(cid), blocking=False) - del _xxsubinterpreters + del _interpreters """) - self._cid = channels.recv(ch) + self._cid = _channels.recv(ch) return self._cid def _get_interpreter(self, interp): @@ -1242,13 +1464,13 @@ def _prep_interpreter(self, interp): if interp.name == 'main': return run_interp(interp.id, f""" - import _xxinterpchannels as channels - import test.test__xxinterpchannels as helpers + import _interpchannels as channels + import test.test__interpchannels as helpers ChannelState = helpers.ChannelState try: cid except NameError: - cid = channels._channel_id({self.cid}) + cid = _channels._channel_id({self.cid}) """) @@ -1435,7 +1657,7 @@ def run_action(self, fix, action, *, hideclosed=True): ) fix.record_action(action, result) else: - _cid = channels.create() + _cid = _channels.create() run_interp(interp.id, f""" result = helpers.run_action( {fix.cid}, @@ -1444,12 +1666,12 @@ def run_action(self, fix, action, *, hideclosed=True): {repr(fix.state)}, hideclosed={hideclosed}, ) - channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) - channels.send({_cid}, b'X' if result.closed else b'', blocking=False) + _channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) + _channels.send({_cid}, b'X' if result.closed else b'', blocking=False) """) result = ChannelState( - pending=int.from_bytes(channels.recv(_cid), 'little'), - closed=bool(channels.recv(_cid)), + pending=int.from_bytes(_channels.recv(_cid), 'little'), + closed=bool(_channels.recv(_cid)), ) fix.record_action(action, result) @@ -1472,42 +1694,42 @@ def _close(self, fix, *, force): if not fix.expect_closed_error(): self.run_action(fix, close, hideclosed=False) else: - with self.assertRaises(channels.ChannelClosedError): + with self.assertRaises(_channels.ChannelClosedError): self.run_action(fix, close, hideclosed=False) def _assert_closed_in_interp(self, fix, interp=None): if interp is None or interp.name == 'main': - with self.assertRaises(channels.ChannelClosedError): - channels.recv(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(fix.cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid, force=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(fix.cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid, force=True) else: run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.recv(cid) + _channels.recv(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid) + _channels.close(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid, force=True) + _channels.close(cid, force=True) """) def _assert_closed(self, fix): self.assertTrue(fix.state.closed) for _ in range(fix.state.pending): - channels.recv(fix.cid) + _channels.recv(fix.cid) self._assert_closed_in_interp(fix) for interp in ('same', 'other'): diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__interpreters.py similarity index 55% rename from Lib/test/test__xxsubinterpreters.py rename to Lib/test/test__interpreters.py index e3c917aa2eb19d..beeb280894ea99 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__interpreters.py @@ -7,13 +7,15 @@ import threading import unittest -import _testcapi from test import support from test.support import import_helper +from test.support import os_helper from test.support import script_helper -interpreters = import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_interpreters') +_testinternalcapi = import_helper.import_module('_testinternalcapi') +from _interpreters import InterpreterNotFoundError ################################## @@ -31,10 +33,10 @@ def _captured_script(script): return wrapped, open(r, encoding="utf-8") -def _run_output(interp, request, shared=None): +def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script, shared) + _interpreters.run_string(interp, script) return rpipe.read() @@ -45,7 +47,7 @@ def _wait_for_interp_to_run(interp, timeout=None): if timeout is None: timeout = support.SHORT_TIMEOUT for _ in support.sleeping_retry(timeout, error=False): - if interpreters.is_running(interp): + if _interpreters.is_running(interp): break else: raise RuntimeError('interp is not running') @@ -55,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None): def _running(interp): r, w = os.pipe() def run(): - interpreters.run_string(interp, dedent(f""" + _interpreters.run_string(interp, dedent(f""" # wait for "signal" with open({r}, encoding="utf-8") as rpipe: rpipe.read() @@ -73,12 +75,12 @@ def run(): def clean_up_interpreters(): - for id in interpreters.list_all(): + for id, *_ in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) - except RuntimeError: + _interpreters.destroy(id) + except _interpreters.InterpreterError: pass # already destroyed @@ -102,11 +104,15 @@ def test_default_shareables(self): 'spam', 10, -10, + True, + False, + 100.0, + (1, ('spam', 'eggs')), ] for obj in shareables: with self.subTest(obj): self.assertTrue( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) def test_not_shareable(self): class Cheese: @@ -120,8 +126,6 @@ class SubBytes(bytes): not_shareables = [ # singletons - True, - False, NotImplemented, ..., # builtin types and objects @@ -129,7 +133,6 @@ class SubBytes(bytes): object, object(), Exception(), - 100.0, # user-defined types and objects Cheese, Cheese('Wensleydale'), @@ -138,7 +141,7 @@ class SubBytes(bytes): for obj in not_shareables: with self.subTest(repr(obj)): self.assertFalse( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) class ShareableTypeTests(unittest.TestCase): @@ -146,8 +149,8 @@ class ShareableTypeTests(unittest.TestCase): def _assert_values(self, values): for obj in values: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -155,8 +158,8 @@ def _assert_values(self, values): def test_singletons(self): for obj in [None]: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) # XXX What about between interpreters? self.assertIs(got, obj) @@ -187,15 +190,48 @@ def test_non_shareable_int(self): for i in ints: with self.subTest(i): with self.assertRaises(OverflowError): - _testcapi.get_crossinterp_data(i) + _testinternalcapi.get_crossinterp_data(i) + + def test_bool(self): + self._assert_values([True, False]) + + def test_float(self): + self._assert_values([0.0, 1.1, -1.0, 0.12345678, -0.12345678]) + + def test_tuple(self): + self._assert_values([(), (1,), ("hello", "world", ), (1, True, "hello")]) + # Test nesting + self._assert_values([ + ((1,),), + ((1, 2), (3, 4)), + ((1, 2), (3, 4), (5, 6)), + ]) + + def test_tuples_containing_non_shareable_types(self): + non_shareables = [ + Exception(), + object(), + ] + for s in non_shareables: + value = tuple([0, 1.0, s]) + with self.subTest(repr(value)): + # XXX Assert the NotShareableError when it is exported + with self.assertRaises(ValueError): + _testinternalcapi.get_crossinterp_data(value) + # Check nested as well + value = tuple([0, 1., (s,)]) + with self.subTest("nested " + repr(value)): + # XXX Assert the NotShareableError when it is exported + with self.assertRaises(ValueError): + _testinternalcapi.get_crossinterp_data(value) class ModuleTests(TestBase): def test_import_in_interpreter(self): _run_output( - interpreters.create(), - 'import _xxsubinterpreters as _interpreters', + _interpreters.create(), + 'import _interpreters', ) @@ -205,45 +241,45 @@ def test_import_in_interpreter(self): class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main]) def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, first, second]) def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + _interpreters.destroy(first) + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, second]) class GetCurrentTests(TestBase): def test_main(self): - main = interpreters.get_main() - cur = interpreters.get_current() + main, *_ = _interpreters.get_main() + cur, *_ = _interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) + self.assertIsInstance(cur, int) def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() + main, *_ = _interpreters.get_main() + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() + import _interpreters + cur, *_ = _interpreters.get_current() print(cur) - assert isinstance(cur, _interpreters.InterpreterID) + assert isinstance(cur, int) """)) cur = int(out.strip()) - _, expected = interpreters.list_all() + _, expected = [id for id, *_ in _interpreters.list_all()] self.assertEqual(cur, expected) self.assertNotEqual(cur, main) @@ -251,19 +287,19 @@ def test_subinterpreter(self): class GetMainTests(TestBase): def test_from_main(self): - [expected] = interpreters.list_all() - main = interpreters.get_main() + [expected] = [id for id, *_ in _interpreters.list_all()] + main, *_ = _interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) + self.assertIsInstance(main, int) def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() + [expected] = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() + import _interpreters + main, *_ = _interpreters.get_main() print(main) - assert isinstance(main, _interpreters.InterpreterID) + assert isinstance(main, int) """)) main = int(out.strip()) self.assertEqual(main, expected) @@ -272,22 +308,22 @@ def test_from_subinterpreter(self): class IsRunningTests(TestBase): def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) + main, *_ = _interpreters.get_main() + self.assertTrue(_interpreters.is_running(main)) @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) + interp = _interpreters.create() + self.assertFalse(_interpreters.is_running(interp)) with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) + self.assertTrue(_interpreters.is_running(interp)) + self.assertFalse(_interpreters.is_running(interp)) def test_from_subinterpreter(self): - interp = interpreters.create() + interp = _interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -296,93 +332,35 @@ def test_from_subinterpreter(self): self.assertEqual(out.strip(), 'True') def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) - with self.assertRaises(RuntimeError): - interpreters.is_running(interp) + interp = _interpreters.create() + _interpreters.destroy(interp) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.is_running(interp) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.is_running(1_000_000) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.is_running(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.is_running(-1) - - -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - class Int(str): - def __index__(self): - return 10 - - id = interpreters.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - self.assertRaises(TypeError, interpreters.InterpreterID, object()) - self.assertRaises(TypeError, interpreters.InterpreterID, 10.0) - self.assertRaises(TypeError, interpreters.InterpreterID, '10') - self.assertRaises(TypeError, interpreters.InterpreterID, b'10') - self.assertRaises(ValueError, interpreters.InterpreterID, -1) - self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64) - - def test_does_not_exist(self): - id = interpreters.create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertTrue(int(id1) == id1) - self.assertTrue(id1 == float(int(id1))) - self.assertTrue(float(int(id1)) == id1) - self.assertFalse(id1 == float(int(id1)) + 0.1) - self.assertFalse(id1 == str(int(id1))) - self.assertFalse(id1 == 2**1000) - self.assertFalse(id1 == float('inf')) - self.assertFalse(id1 == 'spam') - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) + _interpreters.is_running(-1) class CreateTests(TestBase): def test_in_main(self): - id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) + id = _interpreters.create() + self.assertIsInstance(id, int) - self.assertIn(id, interpreters.list_all()) + after = [id for id, *_ in _interpreters.list_all()] + self.assertIn(id, after) @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) @@ -392,7 +370,7 @@ def test_in_thread(self): id = None def f(): nonlocal id - id = interpreters.create() + id = _interpreters.create() lock.acquire() lock.release() @@ -400,29 +378,31 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters.list_all()) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id, after) def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) - assert isinstance(id, _interpreters.InterpreterID) + assert isinstance(id, int) """)) id2 = int(out.strip()) - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() id2 = None def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) """)) @@ -432,144 +412,155 @@ def f(): t.start() t.join() - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. ids = [] for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.append(id) # Now destroy them. for id in ids: - interpreters.destroy(id) + _interpreters.destroy(id) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id}) def test_after_destroy_some(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) + _interpreters.destroy(id1) + _interpreters.destroy(id3) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id, id2}) class DestroyTests(TestBase): def test_one(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() + before = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id2, before) + + _interpreters.destroy(id2) + + after = set(id for id, *_ in _interpreters.list_all()) + self.assertNotIn(id2, after) + self.assertIn(id1, after) + self.assertIn(id3, after) def test_all(self): - before = set(interpreters.list_all()) + initial = set(id for id, *_ in _interpreters.list_all()) ids = set() for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) + before = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(before, initial | ids) for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) + _interpreters.destroy(id) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, initial) def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + main, = [id for id, *_ in _interpreters.list_all()] + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) def f(): - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) t = threading.Thread(target=f) t.start() t.join() def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) - with self.assertRaises(RuntimeError): - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.destroy(id) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.destroy(1_000_000) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.destroy(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.destroy(-1) + _interpreters.destroy(-1) def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters try: _interpreters.destroy({id}) - except RuntimeError: + except _interpreters.InterpreterError: pass """) - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) + _interpreters.run_string(id, script) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id}) def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() + id2 = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.destroy({id2}) """) - interpreters.run_string(id1, script) + _interpreters.run_string(id1, script) - self.assertEqual(set(interpreters.list_all()), {main, id1}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1}) def test_from_other_thread(self): - id = interpreters.create() + id = _interpreters.create() def f(): - interpreters.destroy(id) + _interpreters.destroy(id) t = threading.Thread(target=f) t.start() t.join() def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() with _running(interp): - self.assertTrue(interpreters.is_running(interp), + self.assertTrue(_interpreters.is_running(interp), msg=f"Interp {interp} should be running before destruction.") - with self.assertRaises(RuntimeError, + with self.assertRaises(_interpreters.InterpreterError, msg=f"Should not be able to destroy interp {interp} while it's still running."): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) + _interpreters.destroy(interp) + self.assertTrue(_interpreters.is_running(interp)) class RunStringTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): script, file = _captured_script('print("it worked!", end="")') with file: - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -578,7 +569,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) t = threading.Thread(target=f) t.start() @@ -588,7 +579,7 @@ def f(): self.assertEqual(out, 'it worked!') def test_create_thread(self): - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(""" import threading def f(): @@ -599,7 +590,7 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -607,7 +598,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = _interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -621,13 +612,13 @@ def f(): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = _interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): @@ -638,13 +629,13 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') def test_shareable_types(self): - interp = interpreters.create() + interp = _interpreters.create() objects = [ None, 'spam', @@ -653,15 +644,15 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): - interpreters.run_string( + _interpreters.set___main___attrs(interp, dict(obj=obj)) + _interpreters.run_string( interp, f'assert(obj == {obj!r})', - shared=dict(obj=obj), ) def test_os_exec(self): expected = 'spam spam spam spam spam' - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(f""" import os, sys try: @@ -670,7 +661,7 @@ def test_os_exec(self): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) @@ -691,7 +682,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) file.seek(0) content = file.read() @@ -699,68 +690,31 @@ def test_fork(self): def test_already_running(self): with _running(self.id): - with self.assertRaises(RuntimeError): - interpreters.run_string(self.id, 'print("spam")') + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.run_string(self.id, 'print("spam")') def test_does_not_exist(self): id = 0 - while id in interpreters.list_all(): + while id in set(id for id, *_ in _interpreters.list_all()): id += 1 - with self.assertRaises(RuntimeError): - interpreters.run_string(id, 'print("spam")') + with self.assertRaises(InterpreterNotFoundError): + _interpreters.run_string(id, 'print("spam")') def test_error_id(self): with self.assertRaises(ValueError): - interpreters.run_string(-1, 'print("spam")') + _interpreters.run_string(-1, 'print("spam")') def test_bad_id(self): with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') + _interpreters.run_string('spam', 'print("spam")') def test_bad_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) + _interpreters.run_string(self.id, 10) def test_bytes_for_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') - - @contextlib.contextmanager - def assert_run_failed(self, exctype, msg=None): - with self.assertRaises(interpreters.RunFailedError) as caught: - yield - if msg is None: - self.assertEqual(str(caught.exception).split(':')[0], - exctype.__name__) - else: - self.assertEqual(str(caught.exception), - "{}: {}".format(exctype.__name__, msg)) - - def test_invalid_syntax(self): - with self.assert_run_failed(SyntaxError): - # missing close paren - interpreters.run_string(self.id, 'print("spam"') - - def test_failure(self): - with self.assert_run_failed(Exception, 'spam'): - interpreters.run_string(self.id, 'raise Exception("spam")') - - def test_SystemExit(self): - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, 'raise SystemExit(42)') - - def test_sys_exit(self): - with self.assert_run_failed(SystemExit): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit() - """)) - - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit(42) - """)) + _interpreters.run_string(self.id, b'print("spam")') def test_with_shared(self): r, w = os.pipe() @@ -781,7 +735,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -791,7 +746,7 @@ def test_with_shared(self): self.assertIsNone(ns['cheddar']) def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" + _interpreters.run_string(self.id, dedent(""" spam = 'eggs' ns1 = dict(vars()) del ns1['__builtins__'] @@ -802,7 +757,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.run_string(self.id, script, shared) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -812,7 +768,7 @@ def test_shared_overwrites(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -833,7 +789,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -841,7 +798,7 @@ def test_shared_overwrites_default_vars(self): def test_main_reused(self): r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" spam = True ns = dict(vars()) @@ -855,7 +812,7 @@ def test_main_reused(self): ns1 = pickle.load(chan) r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" eggs = False ns = dict(vars()) @@ -884,7 +841,7 @@ def test_execution_namespace_is_main(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -905,7 +862,7 @@ def test_still_running_at_exit(self): script = dedent(""" from textwrap import dedent import threading - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() def f(): _interpreters.run_string(id, dedent(''' @@ -925,11 +882,168 @@ def f(): self.assertEqual(retcode, 0) +class RunFailedTests(TestBase): + + def setUp(self): + super().setUp() + self.id = _interpreters.create() + + def add_module(self, modname, text): + import tempfile + tempdir = tempfile.mkdtemp() + self.addCleanup(lambda: os_helper.rmtree(tempdir)) + _interpreters.run_string(self.id, dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + """)) + return script_helper.make_script(tempdir, modname, text) + + def run_script(self, text, *, fails=False): + r, w = os.pipe() + try: + script = dedent(f""" + import os, sys + os.write({w}, b'0') + + # This raises an exception: + {{}} + + # Nothing from here down should ever run. + os.write({w}, b'1') + class NeverError(Exception): pass + raise NeverError # never raised + """).format(dedent(text)) + if fails: + err = _interpreters.run_string(self.id, script) + self.assertIsNot(err, None) + return err + else: + err = _interpreters.run_string(self.id, script) + self.assertIs(err, None) + return None + except: + raise # re-raise + else: + msg = os.read(r, 100) + self.assertEqual(msg, b'0') + finally: + os.close(r) + os.close(w) + + def _assert_run_failed(self, exctype, msg, script): + if isinstance(exctype, str): + exctype_name = exctype + exctype = None + else: + exctype_name = exctype.__name__ + + # Run the script. + excinfo = self.run_script(script, fails=True) + + # Check the wrapper exception. + self.assertEqual(excinfo.type.__name__, exctype_name) + if msg is None: + self.assertEqual(excinfo.formatted.split(':')[0], + exctype_name) + else: + self.assertEqual(excinfo.formatted, + '{}: {}'.format(exctype_name, msg)) + + return excinfo + + def assert_run_failed(self, exctype, script): + self._assert_run_failed(exctype, None, script) + + def assert_run_failed_msg(self, exctype, msg, script): + self._assert_run_failed(exctype, msg, script) + + def test_exit(self): + with self.subTest('sys.exit(0)'): + # XXX Should an unhandled SystemExit(0) be handled as not-an-error? + self.assert_run_failed(SystemExit, """ + sys.exit(0) + """) + + with self.subTest('sys.exit()'): + self.assert_run_failed(SystemExit, """ + import sys + sys.exit() + """) + + with self.subTest('sys.exit(42)'): + self.assert_run_failed_msg(SystemExit, '42', """ + import sys + sys.exit(42) + """) + + with self.subTest('SystemExit'): + self.assert_run_failed_msg(SystemExit, '42', """ + raise SystemExit(42) + """) + + # XXX Also check os._exit() (via a subprocess)? + + def test_plain_exception(self): + self.assert_run_failed_msg(Exception, 'spam', """ + raise Exception("spam") + """) + + def test_invalid_syntax(self): + script = dedent(""" + x = 1 + 2 + y = 2 + 4 + z = 4 + 8 + + # missing close paren + print("spam" + + if x + y + z < 20: + ... + """) + + with self.subTest('script'): + self.assert_run_failed(SyntaxError, script) + + with self.subTest('module'): + modname = 'spam_spam_spam' + filename = self.add_module(modname, script) + self.assert_run_failed(SyntaxError, f""" + import {modname} + """) + + def test_NameError(self): + self.assert_run_failed(NameError, """ + res = spam + eggs + """) + # XXX check preserved suggestions + + def test_AttributeError(self): + self.assert_run_failed(AttributeError, """ + object().spam + """) + # XXX check preserved suggestions + + def test_ExceptionGroup(self): + self.assert_run_failed(ExceptionGroup, """ + raise ExceptionGroup('exceptions', [ + Exception('spam'), + ImportError('eggs'), + ]) + """) + + def test_user_defined_exception(self): + self.assert_run_failed_msg('MyError', 'spam', """ + class MyError(Exception): + pass + raise MyError('spam') + """) + + class RunFuncTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): r, w = os.pipe() @@ -939,7 +1053,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.run_func(self.id, script, shared=dict(w=w)) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -955,7 +1070,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.run_func(self.id, script, shared=dict(w=w)) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -975,7 +1091,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.run_func(self.id, code, shared=dict(w=w)) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -988,7 +1105,7 @@ def script(): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) # XXX This hasn't been fixed yet. @unittest.expectedFailure @@ -996,38 +1113,38 @@ def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) def test_args(self): with self.subTest('args'): def script(a, b=0): assert a == b with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('*args'): def script(*args): assert not args with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('**kwargs'): def script(**kwargs): assert not kwargs with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('kwonly'): def script(*, spam=True): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('posonly'): def script(spam, /): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) if __name__ == '__main__': diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index c1f612dc4a63cb..10f04b64dda40e 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -8,6 +8,14 @@ class OpListTests(unittest.TestCase): + def check_bool_function_result(self, func, ops, expected): + for op in ops: + if isinstance(op, str): + op = dis.opmap[op] + with self.subTest(opcode=op, func=func): + self.assertIsInstance(func(op), bool) + self.assertEqual(func(op), expected) + def test_invalid_opcodes(self): invalid = [-100, -1, 255, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) @@ -47,7 +55,7 @@ def check_function(self, func, expected): check_function(self, _opcode.has_exc, dis.hasexc) -class OpListTests(unittest.TestCase): +class StackEffectTests(unittest.TestCase): def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2e06f0d16fdd05..72232b670cdb89 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -1,14 +1,34 @@ """Unit tests for numbers.py.""" +import abc import math import operator import unittest -from numbers import Complex, Real, Rational, Integral +from numbers import Complex, Real, Rational, Integral, Number + + +def concretize(cls): + def not_implemented(*args, **kwargs): + raise NotImplementedError() + + for name in dir(cls): + try: + value = getattr(cls, name) + if value.__isabstractmethod__: + setattr(cls, name, not_implemented) + except AttributeError: + pass + abc.update_abstractmethods(cls) + return cls + class TestNumbers(unittest.TestCase): def test_int(self): self.assertTrue(issubclass(int, Integral)) + self.assertTrue(issubclass(int, Rational)) + self.assertTrue(issubclass(int, Real)) self.assertTrue(issubclass(int, Complex)) + self.assertTrue(issubclass(int, Number)) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -18,8 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): + self.assertFalse(issubclass(float, Integral)) self.assertFalse(issubclass(float, Rational)) self.assertTrue(issubclass(float, Real)) + self.assertTrue(issubclass(float, Complex)) + self.assertTrue(issubclass(float, Number)) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -27,8 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): + self.assertFalse(issubclass(complex, Integral)) + self.assertFalse(issubclass(complex, Rational)) self.assertFalse(issubclass(complex, Real)) self.assertTrue(issubclass(complex, Complex)) + self.assertTrue(issubclass(complex, Number)) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). @@ -40,5 +66,135 @@ def test_complex(self): self.assertRaises(TypeError, int, c1) +class TestNumbersDefaultMethods(unittest.TestCase): + def test_complex(self): + @concretize + class MyComplex(Complex): + def __init__(self, real, imag): + self.r = real + self.i = imag + + @property + def real(self): + return self.r + + @property + def imag(self): + return self.i + + def __add__(self, other): + if isinstance(other, Complex): + return MyComplex(self.imag + other.imag, + self.real + other.real) + raise NotImplementedError + + def __neg__(self): + return MyComplex(-self.real, -self.imag) + + def __eq__(self, other): + if isinstance(other, Complex): + return self.imag == other.imag and self.real == other.real + if isinstance(other, Number): + return self.imag == 0 and self.real == other.real + + # test __bool__ + self.assertTrue(bool(MyComplex(1, 1))) + self.assertTrue(bool(MyComplex(0, 1))) + self.assertTrue(bool(MyComplex(1, 0))) + self.assertFalse(bool(MyComplex(0, 0))) + + # test __sub__ + self.assertEqual(MyComplex(2, 3) - complex(1, 2), MyComplex(1, 1)) + + # test __rsub__ + self.assertEqual(complex(2, 3) - MyComplex(1, 2), MyComplex(1, 1)) + + def test_real(self): + @concretize + class MyReal(Real): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __float__(self): + return float(self.n) + + def __floordiv__(self, other): + return self.n // other + + def __rfloordiv__(self, other): + return other // self.n + + def __mod__(self, other): + return self.n % other + + def __rmod__(self, other): + return other % self.n + + # test __divmod__ + self.assertEqual(divmod(MyReal(3), 2), (1, 1)) + + # test __rdivmod__ + self.assertEqual(divmod(3, MyReal(2)), (1, 1)) + + # test __complex__ + self.assertEqual(complex(MyReal(1)), 1+0j) + + # test real + self.assertEqual(MyReal(3).real, 3) + + # test imag + self.assertEqual(MyReal(3).imag, 0) + + # test conjugate + self.assertEqual(MyReal(123).conjugate(), 123) + + + def test_rational(self): + @concretize + class MyRational(Rational): + def __init__(self, numerator, denominator): + self.n = numerator + self.d = denominator + + @property + def numerator(self): + return self.n + + @property + def denominator(self): + return self.d + + # test__float__ + self.assertEqual(float(MyRational(5, 2)), 2.5) + + + def test_integral(self): + @concretize + class MyIntegral(Integral): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __int__(self): + return self.n + + # test __index__ + self.assertEqual(operator.index(MyIntegral(123)), 123) + + # test __float__ + self.assertEqual(float(MyIntegral(123)), 123.0) + + # test numerator + self.assertEqual(MyIntegral(123).numerator, 123) + + # test denominator + self.assertEqual(MyIntegral(123).denominator, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..115882a4c281f6 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,332 @@ +import platform +import queue +import re +import subprocess +import sys +import unittest +from array import array +from contextlib import contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + + +# Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + Thread(target=logcat_thread).start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + def test_str(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + self.assertEqual("UTF-8", stream.encoding) + self.assertTrue(stream.line_buffering) + self.assertFalse(stream.write_through) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. + self.assertEqual("backslashreplace", stream.errors) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", + [("foobar" * 666) + "foob", # 4000 bytes + "ar" + ("foobar" * 33)]) # 200 bytes + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", + [s * 100, # 3000 bytes + s * 50]) # 1500 bytes + + s = "0123456789" + write(s * 200, []) + write(s * 150, []) + write(s * 51, [s * 350]) # 3500 bytes + write("\n", [s * 51]) # 510 bytes + + def test_bytes(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7c1f5d36999a3d..84fe74a1063fb6 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -765,49 +765,6 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 - parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) - class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -2190,7 +2147,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(argparse.ArgumentError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -2274,6 +2233,34 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_with_single_dash_option(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('-k', '--known', action='count', default=0) + parser.add_argument('-n', '--new', action='count', default=0) + self.assertEqual(parser.parse_known_args(['-k', '-u']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-u', '-k']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-ku']), + (NS(known=1, new=0), ['-u'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=u']) + self.assertEqual(parser.parse_known_args(['-uk']), + (NS(known=0, new=0), ['-uk'])) + self.assertEqual(parser.parse_known_args(['-u=k']), + (NS(known=0, new=0), ['-u=k'])) + self.assertEqual(parser.parse_known_args(['-kunknown']), + (NS(known=1, new=0), ['-unknown'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=unknown']) + self.assertEqual(parser.parse_known_args(['-ku=nknown']), + (NS(known=1, new=0), ['-u=nknown'])) + self.assertEqual(parser.parse_known_args(['-knew']), + (NS(known=1, new=1), ['-ew'])) + self.assertArgumentParserError(parser.parse_known_args, ['-kn=ew']) + self.assertArgumentParserError(parser.parse_known_args, ['-k-new']) + self.assertArgumentParserError(parser.parse_known_args, ['-kn-ew']) + self.assertEqual(parser.parse_known_args(['-kne-w']), + (NS(known=1, new=1), ['-e-w'])) + def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') @@ -2836,6 +2823,27 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): + self.maxDiff = None + parser = ErrorRaisingArgumentParser(prog='PROG') + commands = parser.add_subparsers(title="commands", dest="command") + cmd_foo = commands.add_parser("foo") + group = cmd_foo.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', help=argparse.SUPPRESS) + group.add_argument('--quiet', action='store_true', help=argparse.SUPPRESS) + longopt = '--' + 'long'*32 + longmeta = 'LONG'*32 + cmd_foo.add_argument(longopt) + expected = f'''\ + usage: PROG foo [-h] + [{longopt} {longmeta}] + + options: + -h, --help show this help message and exit + {longopt} {longmeta} + ''' + self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) + def test_empty_group(self): # See issue 26952 parser = argparse.ArgumentParser() @@ -3922,7 +3930,7 @@ class TestHelpUsageWithParentheses(HelpTestCase): options: -h, --help show this help message and exit - -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + -p, --optional {1 (option A), 2 (option B)} ''' version = '' @@ -4206,6 +4214,140 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + gg = g.add_mutually_exclusive_group() + gg.add_argument("--hax") + gg.add_argument("--hox", help=argparse.SUPPRESS) + gg.add_argument("--hex") + g.add_argument("--eggs") + parser.add_argument("--num") + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] + [--num NUM] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -4405,8 +4547,8 @@ class TestHelpAlternatePrefixChars(HelpTestCase): help = usage + '''\ options: - ^^foo foo help - ;b BAR, ;;bar BAR bar help + ^^foo foo help + ;b, ;;bar BAR bar help ''' version = '' @@ -5099,7 +5241,8 @@ def test_optional(self): string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " - "choices=[1, 2, 3], required=False, help='HELP', metavar='METAVAR')") + "choices=[1, 2, 3], required=False, help='HELP', " + "metavar='METAVAR', deprecated=False)") self.assertStringEqual(option, string) def test_argument(self): @@ -5116,7 +5259,8 @@ def test_argument(self): string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " - "required=True, help='H HH H', metavar='MV MV MV')" % float) + "required=True, help='H HH H', metavar='MV MV MV', " + "deprecated=False)" % float) self.assertStringEqual(argument, string) def test_namespace(self): @@ -5308,6 +5452,139 @@ def spam(string_to_convert): args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) + +# ============================================== +# Check that deprecated arguments output warning +# ============================================== + +class TestDeprecatedArguments(TestCase): + + def test_deprecated_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '-f', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '--foo', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_boolean_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', action=argparse.BooleanOptionalAction, deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', '--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='?', deprecated=True) + parser.add_argument('bar', nargs='?', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertRegex(stderr, "warning: argument 'bar' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_varargument(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='*', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_subparser(self): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.add_parser('foo', aliases=['baz'], deprecated=True) + subparsers.add_parser('bar') + + with captured_stderr() as stderr: + parser.parse_args(['bar']) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['baz']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'baz' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== @@ -5405,6 +5682,22 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) + def test_double_dash(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', nargs='*') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['--foo', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['-f', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) + self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + # =========================== # parse_intermixed_args tests @@ -5751,7 +6044,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -5762,6 +6056,44 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a219fa365e7f20..95383be9659eb9 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1014,6 +1014,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index fa96e259b8c978..55725ec36fd3a7 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,5 +1,6 @@ import ast import builtins +import copy import dis import enum import os @@ -8,18 +9,19 @@ import textwrap import types import unittest -import warnings import weakref -from functools import partial from textwrap import dedent +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: + if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: return t elif isinstance(t, list): return [to_tuple(e) for e in t] @@ -34,12 +36,13 @@ def to_tuple(t): result.append(to_tuple(getattr(t, f))) return tuple(result) +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) # These tests are compiled through "exec" # There should be at least one test per statement exec_tests = [ - # None - "None", # Module docstring "'module docstring'", # FunctionDef @@ -72,8 +75,11 @@ def to_tuple(t): "class C: 'docstring for class C'", # ClassDef, new style class "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", # Return "def f():return 1", + "def f():return", # Delete "del v", # Assign @@ -81,38 +87,72 @@ def to_tuple(t): "a,b = c", "(a,b) = c", "[a,b] = c", + "a[b] = c", # AnnAssign with unpacked types "x: tuple[*Ts]", "x: tuple[int, *Ts]", "x: tuple[int, *tuple[str, ...]]", # AugAssign "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", # For "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", # While "while v:pass", - # If + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else "if v:pass", - # If-Elif "if a:\n pass\nelif b:\n pass", - # If-Elif-Else + "if a:\n pass\nelse:\n pass", "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", # With + "with x: pass", + "with x, y: pass", "with x as y: pass", "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", # Raise + "raise", "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", # TryExcept "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", # TryFinally "try:\n pass\nfinally:\n pass", # TryStarExcept "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", # Assert "assert v", + # Assert with message + "assert v, 'message'", # Import "import sys", + "import foo as bar", # ImportFrom + "from sys import x as y", "from sys import v", # Global "global v", @@ -157,6 +197,9 @@ def to_tuple(t): # PEP 448: Additional Unpacking Generalizations "{**{1:2}, 2:3}", "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", # Asynchronous comprehensions "async def f():\n [i async for b in c]", # Decorated FunctionDef @@ -171,6 +214,10 @@ def to_tuple(t): "@a.b.c\ndef f(): pass", # Simple assignment expression "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", # Positional-only arguments "def f(a, /,): pass", "def f(a, /, c, d, e): pass", @@ -189,16 +236,23 @@ def to_tuple(t): "type X[T, *Ts, **P] = (T, Ts, P)", "type X[T: int, *Ts, **P] = (T, Ts, P)", "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", # Generic classes "class X[T]: pass", "class X[T, *Ts, **P]: pass", "class X[T: int, *Ts, **P]: pass", "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", # Generic functions "def f[T](): pass", "def f[T, *Ts, **P](): pass", "def f[T: int, *Ts, **P](): pass", "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", ] # These are compiled through "single" @@ -211,14 +265,34 @@ def to_tuple(t): # These are compiled through "eval" # It should test all expressions eval_tests = [ - # None + # Constant(value=None) "None", + # True + "True", + # False + "False", # BoolOp "a and b", + "a or b", # BinOp "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", # UnaryOp "not v", + "+v", + "-v", + "~v", # Lambda "lambda:None", # Dict @@ -233,10 +307,31 @@ def to_tuple(t): : 2 }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", # ListComp "[a for b in c if d]", # GeneratorExp "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", # Comprehensions with multiple for targets "[(a,b) for a,b in c]", "[(a,b) for (a,b) in c]", @@ -247,19 +342,33 @@ def to_tuple(t): "((a,b) for a,b in c)", "((a,b) for (a,b) in c)", "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # # Yield - yield expressions can't work outside a function # # Compare "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", # Call "f(1,2,c=3,*d,**e)", # Call with multi-character starred "f(*[0, 1])", # Call with a generator argument "f(a for a in b)", - # Num + # Constant(value=int()) "10", - # Str + # Complex num + "1j", + # Constant(value=str()) "'string'", # Attribute "a.b", @@ -279,10 +388,20 @@ def to_tuple(t): "()", # Combination "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", ] -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): maxDiff = None @@ -486,151 +605,59 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue # constructor has a different signature if name == 'Index': continue if self._is_ast_node(name, item): - x = item() + x = self._construct_ast_class(item) if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args + self.assertEqual(x.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) self.assertIsNone(x.vararg) x = ast.arguments(*range(1, 8)) self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): - x = ast.Constant() + x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_classattrs(self): - x = ast.Constant() + with self.assertWarns(DeprecationWarning): + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): @@ -645,7 +672,7 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Constant(lineno=2) + x = ast.Constant(lineno=2, value=3) self.assertEqual(x.lineno, 2) x = ast.Constant(42, lineno=0) @@ -656,8 +683,9 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) @@ -672,190 +700,6 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -882,8 +726,9 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too @@ -921,8 +766,9 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) def test_no_fields(self): @@ -930,14 +776,6 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) @@ -1024,6 +862,133 @@ def test_ast_asdl_signature(self): expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) + def test_compare_basics(self): + self.assertTrue(ast.compare(ast.parse("x = 10"), ast.parse("x = 10"))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse(""))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse("x"))) + self.assertFalse( + ast.compare(ast.parse("x = 10;y = 20"), ast.parse("class C:pass")) + ) + + def test_compare_modified_ast(self): + # The ast API is a bit underspecified. The objects are mutable, + # and even _fields and _attributes are mutable. The compare() does + # some simple things to accommodate mutability. + a = ast.parse("m * x + b", mode="eval") + b = ast.parse("m * x + b", mode="eval") + self.assertTrue(ast.compare(a, b)) + + a._fields = a._fields + ("spam",) + a.spam = "Spam" + self.assertNotEqual(a._fields, b._fields) + self.assertFalse(ast.compare(a, b)) + self.assertFalse(ast.compare(b, a)) + + b._fields = a._fields + b.spam = a.spam + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(b, a)) + + b._attributes = b._attributes + ("eggs",) + b.eggs = "eggs" + self.assertNotEqual(a._attributes, b._attributes) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + self.assertFalse(ast.compare(b, a, compare_attributes=True)) + + a._attributes = b._attributes + a.eggs = b.eggs + self.assertTrue(ast.compare(a, b, compare_attributes=True)) + self.assertTrue(ast.compare(b, a, compare_attributes=True)) + + def test_compare_literals(self): + constants = ( + -20, + 20, + 20.0, + 1, + 1.0, + True, + 0, + False, + frozenset(), + tuple(), + "ABCD", + "abcd", + "中文字", + 1e1000, + -1e1000, + ) + for next_index, constant in enumerate(constants[:-1], 1): + next_constant = constants[next_index] + with self.subTest(literal=constant, next_literal=next_constant): + self.assertTrue( + ast.compare(ast.Constant(constant), ast.Constant(constant)) + ) + self.assertFalse( + ast.compare( + ast.Constant(constant), ast.Constant(next_constant) + ) + ) + + same_looking_literal_cases = [ + {1, 1.0, True, 1 + 0j}, + {0, 0.0, False, 0 + 0j}, + ] + for same_looking_literals in same_looking_literal_cases: + for literal in same_looking_literals: + for same_looking_literal in same_looking_literals - {literal}: + self.assertFalse( + ast.compare( + ast.Constant(literal), + ast.Constant(same_looking_literal), + ) + ) + + def test_compare_fieldless(self): + self.assertTrue(ast.compare(ast.Add(), ast.Add())) + self.assertFalse(ast.compare(ast.Sub(), ast.Add())) + + # test that missing runtime fields is handled in ast.compare() + a1, a2 = ast.Name('a'), ast.Name('a') + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2)) + del a1.id + self.assertFalse(ast.compare(a1, a2)) + del a2.id + self.assertTrue(ast.compare(a1, a2)) + + def test_compare_modes(self): + for mode, sources in ( + ("exec", exec_tests), + ("eval", eval_tests), + ("single", single_tests), + ): + for source in sources: + a = ast.parse(source, mode=mode) + b = ast.parse(source, mode=mode) + self.assertTrue( + ast.compare(a, b), f"{ast.dump(a)} != {ast.dump(b)}" + ) + + def test_compare_attributes_option(self): + def parse(a, b): + return ast.parse(a), ast.parse(b) + + a, b = parse("2 + 2", "2+2") + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(a, b, compare_attributes=False)) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + + def test_compare_attributes_option_missing_attribute(self): + # test that missing runtime attributes is handled in ast.compare() + a1, a2 = ast.Name('a', lineno=1), ast.Name('a', lineno=1) + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + del a1.lineno + self.assertFalse(ast.compare(a1, a2, compare_attributes=True)) + del a2.lineno + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + def test_positional_only_feature_version(self): ast.parse('def foo(x, /): ...', feature_version=(3, 8)) ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) @@ -1039,19 +1004,15 @@ def test_positional_only_feature_version(self): with self.assertRaises(SyntaxError): ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - def test_assignment_expression_feature_version(self): ast.parse('(x := 0)', feature_version=(3, 8)) with self.assertRaises(SyntaxError): ast.parse('(x := 0)', feature_version=(3, 7)) + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) + def test_exception_groups_feature_version(self): code = dedent(''' try: ... @@ -1073,6 +1034,18 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) + def test_type_params_default_feature_version(self): + samples = [ + "type X[*Ts=int] = int", + "class X[T=int]: pass", + "def f[**P=int](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 12)) + def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): ast.parse('pass', feature_version=(2, 7)) @@ -1116,12 +1089,14 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) - @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT + fail_depth = support.exceeds_recursion_limit() crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.get_c_recursion_limit() * 0.8) + if _testinternalcapi is not None: + remaining = _testinternalcapi.get_c_recursion_remaining() + success_depth = min(success_depth, remaining) def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth @@ -1170,6 +1145,344 @@ def test_none_checks(self) -> None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + +class CopyTests(unittest.TestCase): + """Test copying and pickling AST nodes.""" + + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + + def test_pickling(self): + import pickle + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for code in exec_tests: + with self.subTest(code=code, protocol=protocol): + tree = compile(code, "?", "exec", 0x400) + ast2 = pickle.loads(pickle.dumps(tree, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(tree)) + + def test_copy_with_parents(self): + # gh-120108 + code = """ + ('',) + while i < n: + if ch == '': + ch = format[i] + if ch == '': + if freplace is None: + '' % getattr(object) + elif ch == '': + if zreplace is None: + if hasattr: + offset = object.utcoffset() + if offset is not None: + if offset.days < 0: + offset = -offset + h = divmod(timedelta(hours=0)) + if u: + zreplace = '' % (sign,) + elif s: + zreplace = '' % (sign,) + else: + zreplace = '' % (sign,) + elif ch == '': + if Zreplace is None: + Zreplace = '' + if hasattr(object): + s = object.tzname() + if s is not None: + Zreplace = s.replace('') + newformat.append(Zreplace) + else: + push('') + else: + push(ch) + + """ + tree = ast.parse(textwrap.dedent(code)) + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + try: + with support.infinite_recursion(200): + tree2 = copy.deepcopy(tree) + finally: + # Singletons like ast.Load() are shared; make sure we don't + # leave them mutated after this test. + for node in ast.walk(tree): + if hasattr(node, "parent"): + del node.parent + + for node in ast.walk(tree2): + for child in ast.iter_child_nodes(node): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): + self.assertEqual(to_tuple(child.parent), to_tuple(node)) + + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertTrue(hasattr(klass, '__replace__')) + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertTrue(hasattr(node, 'extra')) + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + class ASTHelpers_Test(unittest.TestCase): maxDiff = None @@ -1190,21 +1503,20 @@ def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" + "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) def test_dump_indent(self): @@ -1217,9 +1529,7 @@ def test_dump_indent(self): func=Name(id='spam', ctx=Load()), args=[ Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") + Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ @@ -1228,9 +1538,7 @@ def test_dump_indent(self): \t\t\t\tName('spam', Load()), \t\t\t\t[ \t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") +\t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( body=[ @@ -1257,7 +1565,6 @@ def test_dump_indent(self): col_offset=11, end_lineno=1, end_col_offset=23)], - keywords=[], lineno=1, col_offset=0, end_lineno=1, @@ -1265,8 +1572,7 @@ def test_dump_indent(self): lineno=1, col_offset=0, end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") + end_col_offset=24)])""") def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) @@ -1296,6 +1602,119 @@ def test_dump_incomplete(self): self.assertEqual(ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" ) + # Arguments: + node = ast.arguments(args=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments() + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments()", + ) + # Classes: + node = ast.ClassDef( + 'T', + [], + [ast.keyword('a', ast.Constant(None))], + [], + [ast.Name('dataclass', ctx=ast.Load())], + ) + self.assertEqual(ast.dump(node), + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + ) + + def test_dump_show_empty(self): + def check_node(node, empty, full, **kwargs): + with self.subTest(show_empty=False): + self.assertEqual( + ast.dump(node, show_empty=False, **kwargs), + empty, + ) + with self.subTest(show_empty=True): + self.assertEqual( + ast.dump(node, show_empty=True, **kwargs), + full, + ) + + def check_text(code, empty, full, **kwargs): + check_node(ast.parse(code), empty, full, **kwargs) + + check_node( + ast.arguments(), + empty="arguments()", + full="arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])", + ) + + check_node( + # Corner case: there are no real `Name` instances with `id=''`: + ast.Name(id='', ctx=ast.Load()), + empty="Name(id='', ctx=Load())", + full="Name(id='', ctx=Load())", + ) + + check_node( + ast.MatchSingleton(value=None), + empty="MatchSingleton(value=None)", + full="MatchSingleton(value=None)", + ) + + check_node( + ast.Constant(value=None), + empty="Constant(value=None)", + full="Constant(value=None)", + ) + + check_node( + ast.Constant(value=''), + empty="Constant(value='')", + full="Constant(value='')", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", + include_attributes=True, + ) + + check_text( + 'spam(eggs, "and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", + ) + + check_text( + 'spam(eggs, text="and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", + ) + + check_text( + "import _ast as ast; from module import sub", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + ) def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') @@ -1306,8 +1725,9 @@ def test_copy_location(self): 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) @@ -1323,14 +1743,13 @@ def test_fix_missing_locations(self): "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" + "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) def test_increment_lineno(self): @@ -1402,6 +1821,12 @@ def test_get_docstring(self): node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') + + node = ast.parse('x') + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + def test_get_docstring_none(self): self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') @@ -1426,6 +1851,9 @@ def test_get_docstring_none(self): node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n 42') + self.assertIsNone(ast.get_docstring(node.body[0])) + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): node = ast.parse( '"""line one\nline two"""\n\n' @@ -1566,15 +1994,15 @@ def test_level_as_none(self): self.assertIn('sleep', ns) def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e with self.assertRaises(RecursionError): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = f f.operand = e with self.assertRaises(RecursionError): @@ -1953,32 +2381,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") @@ -2018,31 +2420,17 @@ def test_list(self): def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) @@ -2684,69 +3072,8 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, x)) self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, tranformer_class, initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) @@ -2862,6 +3189,126 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + + def test_custom_subclass_with_no_fields(self): + class NoInit(ast.AST): + pass + + obj = NoInit() + self.assertIsInstance(obj, NoInit) + self.assertEqual(obj.__dict__, {}) + + def test_fields_but_no_field_types(self): + class Fields(ast.AST): + _fields = ('a',) + + obj = Fields() + with self.assertRaises(AttributeError): + obj.a + obj = Fields(a=1) + self.assertEqual(obj.a, 1) + + def test_fields_and_types(self): + class FieldsAndTypes(ast.AST): + _fields = ('a',) + _field_types = {'a': int | None} + a: int | None = None + + obj = FieldsAndTypes() + self.assertIs(obj.a, None) + obj = FieldsAndTypes(a=1) + self.assertEqual(obj.a, 1) + + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): + obj = MyAttrs(c=3) + + def test_fields_and_types_no_default(self): + class FieldsAndTypesNoDefault(ast.AST): + _fields = ('a',) + _field_types = {'a': int} + + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): + obj = FieldsAndTypesNoDefault() + with self.assertRaises(AttributeError): + obj.a + obj = FieldsAndTypesNoDefault(a=1) + self.assertEqual(obj.a, 1) + + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + + @support.cpython_only class ModuleStateTests(unittest.TestCase): # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. @@ -2980,7 +3427,6 @@ def main(): #### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), ('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), ('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), @@ -2998,29 +3444,63 @@ def main(): ('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), ('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), ('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), ('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), ('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), ('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), ('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), ('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), ('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), ('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), ('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), ('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), ('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), ('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), ('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), ('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), ('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), ('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), ('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), ('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), ('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), ('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), ('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), ('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), ('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), ('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), ('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), ('Module', [('Global', (1, 0, 1, 8), ['v'])], []), ('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), @@ -3040,6 +3520,8 @@ def main(): ('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), ('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), ('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), ('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), ('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), ('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), @@ -3047,6 +3529,8 @@ def main(): ('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), ('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), ('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), ('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), @@ -3058,34 +3542,62 @@ def main(): ('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), ('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), ] single_results = [ ('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), ] eval_results = [ ('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), ('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), ('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), ('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), ('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), ('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), ('Expression', ('Dict', (1, 0, 1, 2), [], [])), ('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), ('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), ('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), ('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), ('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), ('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), ('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), @@ -3096,10 +3608,20 @@ def main(): ('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), ('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), ('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), ('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), ('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), ('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), ('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), ('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), ('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), ('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), @@ -3110,5 +3632,14 @@ def main(): ('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), ('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), ('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), ] main() diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a49630112af510..4f2278bb263681 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -379,7 +379,10 @@ async def async_gen_wrapper(): def test_async_gen_exception_12(self): async def gen(): - await anext(me) + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'asend' of '{gen.__qualname__}' " + f"was never awaited"): + await anext(me) yield 123 me = gen() @@ -390,12 +393,161 @@ async def gen(): r'anext\(\): asynchronous generator is already running'): an.__next__() + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + def test_async_gen_3_arg_deprecation_warning(self): async def gen(): yield 123 with self.assertWarns(DeprecationWarning): - gen().athrow(GeneratorExit, GeneratorExit(), None) + x = gen().athrow(GeneratorExit, GeneratorExit(), None) + with self.assertRaises(GeneratorExit): + x.send(None) + del x + gc_collect() def test_async_gen_api_01(self): async def gen(): @@ -419,6 +571,54 @@ async def gen(): self.assertTrue(inspect.isawaitable(aclose)) aclose.close() + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + class AsyncGenAsyncioTest(unittest.TestCase): @@ -1564,6 +1764,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1617,6 +1819,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1654,6 +1857,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1701,6 +1960,14 @@ def test_asend(self): async def gen(): yield 1 + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" with self.assertWarnsRegex(RuntimeWarning, msg): g = gen() @@ -1727,7 +1994,67 @@ async def gen(): g.aclose() gc_collect() + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + del gen2 + gc_collect() # does not warn unawaited if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c2080977e9d587..c14a0bb180d79b 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -3,6 +3,7 @@ import concurrent.futures import errno import math +import platform import socket import sys import threading @@ -231,6 +232,22 @@ def test_set_default_executor_error(self): self.assertIsNone(self.loop._default_executor) + def test_shutdown_default_executor_timeout(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + time.sleep(0.1) + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + def test_call_soon(self): def cb(): pass @@ -1232,7 +1249,7 @@ def test_create_datagram_endpoint_wrong_sock(self): with sock: coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) with self.assertRaisesRegex(ValueError, - 'A UDP Socket was expected'): + 'A datagram socket was expected'): self.loop.run_until_complete(coro) def test_create_connection_no_host_port_sock(self): @@ -1414,6 +1431,10 @@ def test_create_connection_no_inet_pton(self, m_socket): self._test_create_connection_ip_addr(m_socket, False) @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) def test_create_connection_service_name(self, m_socket): m_socket.getaddrinfo = socket.getaddrinfo sock = m_socket.socket.return_value diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index b25c0975736e20..4dcf9f0e4037b6 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1125,12 +1125,16 @@ def test_create_server_ssl_match_failed(self): # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with mock.patch.object(self.loop, 'call_exception_handler'): with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + with self.assertRaisesRegex(ssl.CertificateError, regex): self.loop.run_until_complete(f_c) # close connection @@ -1374,6 +1378,80 @@ def test_create_datagram_endpoint_sock(self): tr.close() self.loop.run_until_complete(pr.done) + def test_datagram_send_to_non_listening_address(self): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages after + # sending a message to an address that wasn't listening. + loop = self.loop + + class Protocol(asyncio.DatagramProtocol): + + _received_datagram = None + + def datagram_received(self, data, addr): + self._received_datagram.set_result(data) + + async def wait_for_datagram_received(self): + self._received_datagram = loop.create_future() + result = await asyncio.wait_for(self._received_datagram, 10) + self._received_datagram = None + return result + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + transport_1, protocol_1 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_1) + ) + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + transport_2, protocol_2 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_2) + ) + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address that + # is not listening + socket_3 = create_socket() + transport_3, protocol_3 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_3) + ) + addr_3 = socket_3.getsockname() + transport_3.abort() + + transport_1.sendto(b'a', addr=addr_2) + self.assertEqual(loop.run_until_complete( + protocol_2.wait_for_datagram_received() + ), b'a') + + transport_2.sendto(b'b', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'b') + + # this should send to an address that isn't listening + transport_1.sendto(b'c', addr=addr_3) + loop.run_until_complete(asyncio.sleep(0)) + + # transport 1 should still be able to receive messages after sending to + # an address that wasn't listening + transport_2.sendto(b'd', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'd') + + transport_1.close() + transport_2.close() + def test_internal_fds(self): loop = self.create_event_loop() if not isinstance(loop, selector_events.BaseSelectorEventLoop): @@ -1815,6 +1893,7 @@ def check_killed(self, returncode): else: self.assertEqual(-signal.SIGKILL, returncode) + @support.requires_subprocess() def test_subprocess_exec(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1836,6 +1915,7 @@ def test_subprocess_exec(self): self.check_killed(proto.returncode) self.assertEqual(b'Python The Winner', proto.data[1]) + @support.requires_subprocess() def test_subprocess_interactive(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1863,6 +1943,7 @@ def test_subprocess_interactive(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_shell(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1879,6 +1960,7 @@ def test_subprocess_shell(self): self.assertEqual(proto.data[2], b'') transp.close() + @support.requires_subprocess() def test_subprocess_exitcode(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1890,6 +1972,7 @@ def test_subprocess_exitcode(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_close_after_finish(self): connect = self.loop.subprocess_shell( functools.partial(MySubprocessProtocol, self.loop), @@ -1904,6 +1987,7 @@ def test_subprocess_close_after_finish(self): self.assertEqual(7, proto.returncode) self.assertIsNone(transp.close()) + @support.requires_subprocess() def test_subprocess_kill(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1920,6 +2004,7 @@ def test_subprocess_kill(self): self.check_killed(proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_terminate(self): prog = os.path.join(os.path.dirname(__file__), 'echo.py') @@ -1937,6 +2022,7 @@ def test_subprocess_terminate(self): transp.close() @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() def test_subprocess_send_signal(self): # bpo-31034: Make sure that we get the default signal handler (killing # the process). The parent process may have decided to ignore SIGHUP, @@ -1961,6 +2047,7 @@ def test_subprocess_send_signal(self): finally: signal.signal(signal.SIGHUP, old_handler) + @support.requires_subprocess() def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -1982,6 +2069,7 @@ def test_subprocess_stderr(self): self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_stderr_redirect_to_stdout(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -2007,6 +2095,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): transp.close() self.assertEqual(0, proto.returncode) + @support.requires_subprocess() def test_subprocess_close_client_stream(self): prog = os.path.join(os.path.dirname(__file__), 'echo3.py') @@ -2041,6 +2130,7 @@ def test_subprocess_close_client_stream(self): self.loop.run_until_complete(proto.completed) self.check_killed(proto.returncode) + @support.requires_subprocess() def test_subprocess_wait_no_same_group(self): # start the new process in a new session connect = self.loop.subprocess_shell( @@ -2053,6 +2143,7 @@ def test_subprocess_wait_no_same_group(self): self.assertEqual(7, proto.returncode) transp.close() + @support.requires_subprocess() def test_subprocess_exec_invalid_args(self): async def connect(**kwds): await self.loop.subprocess_exec( @@ -2066,6 +2157,7 @@ async def connect(**kwds): with self.assertRaises(ValueError): self.loop.run_until_complete(connect(shell=True)) + @support.requires_subprocess() def test_subprocess_shell_invalid_args(self): async def connect(cmd=None, **kwds): @@ -2117,24 +2209,8 @@ def test_remove_fds_after_closing(self): else: import selectors - class UnixEventLoopTestsMixin(EventLoopTestsMixin): - def setUp(self): - super().setUp() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - - def tearDown(self): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() - - if hasattr(selectors, 'KqueueSelector'): - class KqueueEventLoopTests(UnixEventLoopTestsMixin, + class KqueueEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2159,7 +2235,7 @@ def test_write_pty(self): super().test_write_pty() if hasattr(selectors, 'EpollSelector'): - class EPollEventLoopTests(UnixEventLoopTestsMixin, + class EPollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2167,7 +2243,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.EpollSelector()) if hasattr(selectors, 'PollSelector'): - class PollEventLoopTests(UnixEventLoopTestsMixin, + class PollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2175,7 +2251,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.PollSelector()) # Should always exist. - class SelectEventLoopTests(UnixEventLoopTestsMixin, + class SelectEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2236,7 +2312,7 @@ def test_handle_repr(self): h = asyncio.Handle(noop, (1, 2), self.loop) filename, lineno = test_utils.get_function_source(noop) self.assertEqual(repr(h), - '' + '' % (filename, lineno)) # cancelled handle @@ -2254,14 +2330,14 @@ def test_handle_repr(self): # partial function cb = functools.partial(noop, 1, 2) h = asyncio.Handle(cb, (3,), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) # partial function with keyword args cb = functools.partial(noop, x=1) h = asyncio.Handle(cb, (2, 3), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) @@ -2272,7 +2348,7 @@ def test_handle_repr(self): h = asyncio.Handle(cb, (), self.loop) cb_regex = r'' - cb_regex = fr'functools.partialmethod\({cb_regex}, , \)\(\)' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' regex = fr'^$' self.assertRegex(repr(h), regex) @@ -2302,6 +2378,24 @@ def test_handle_repr_debug(self): '' % (filename, lineno, create_filename, create_lineno)) + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + def test_handle_source_traceback(self): loop = asyncio.get_event_loop_policy().new_event_loop() loop.set_debug(True) @@ -2606,9 +2700,6 @@ def test_event_loop_policy(self): self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) - self.assertRaises(NotImplementedError, policy.get_child_watcher) - self.assertRaises(NotImplementedError, policy.set_child_watcher, - object()) def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() @@ -2723,20 +2814,8 @@ def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - def tearDown(self): try: - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() finally: self.loop.close() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 2184b2091f84ee..458b70451a306a 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -5,6 +5,7 @@ import re import sys import threading +import traceback import unittest from unittest import mock from types import GenericAlias @@ -270,10 +271,6 @@ def test_exception(self): f = self._new_future(loop=self.loop) self.assertRaises(asyncio.InvalidStateError, f.exception) - # StopIteration cannot be raised into a Future - CPython issue26221 - self.assertRaisesRegex(TypeError, "StopIteration .* cannot be raised", - f.set_exception, StopIteration) - f.set_exception(exc) self.assertFalse(f.cancelled()) self.assertTrue(f.done()) @@ -283,6 +280,25 @@ def test_exception(self): self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) self.assertFalse(f.cancel()) + def test_stop_iteration_exception(self, stop_iteration_class=StopIteration): + exc = stop_iteration_class() + f = self._new_future(loop=self.loop) + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + exc = f.exception() + cause = exc.__cause__ + self.assertIsInstance(exc, RuntimeError) + self.assertRegex(str(exc), 'StopIteration .* cannot be raised') + self.assertIsInstance(cause, stop_iteration_class) + + def test_stop_iteration_subclass_exception(self): + class MyStopIteration(StopIteration): + pass + + self.test_stop_iteration_exception(MyStopIteration) + def test_exception_class(self): f = self._new_future(loop=self.loop) f.set_exception(RuntimeError) @@ -401,6 +417,24 @@ def test_copy_state(self): _copy_future_state(f_cancelled, newf_cancelled) self.assertTrue(newf_cancelled.cancelled()) + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + def test_iter(self): fut = self._new_future(loop=self.loop) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index f6c6a282429a21..34509717f2872a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -39,7 +39,7 @@ async def test_lock(self): with self.assertRaisesRegex( TypeError, - "object Lock can't be used in 'await' expression" + "'Lock' object can't be awaited" ): await lock @@ -77,7 +77,7 @@ async def test_lock_by_with_statement(self): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - r"object \w+ can't be used in 'await' expression" + r"'\w+' object can't be awaited" ): with await lock: pass @@ -758,6 +758,155 @@ async def test_timeout_in_block(self): with self.assertRaises(asyncio.TimeoutError): await asyncio.wait_for(condition.wait(), timeout=0.5) + async def test_cancelled_error_wakeup(self): + # Test that a cancelled error, received when awaiting wakeup, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition, cancel it there. + task.cancel(msg="foo") + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_error_re_aquire(self): + # Test that a cancelled error, received when re-aquiring lock, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition + await cond.acquire() + wake = True + cond.notify() + await asyncio.sleep(0) + # Task is now trying to re-acquire the lock, cancel it there. + task.cancel(msg="foo") + cond.release() + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] class SemaphoreTests(unittest.IsolatedAsyncioTestCase): @@ -792,7 +941,7 @@ async def test_semaphore(self): with self.assertRaisesRegex( TypeError, - "object Semaphore can't be used in 'await' expression", + "'Semaphore' object can't be awaited", ): await sem @@ -1044,6 +1193,62 @@ async def c3(result): await asyncio.gather(*tasks, return_exceptions=True) self.assertEqual([2, 3], result) + async def test_acquire_fifo_order_4(self): + # Test that a successfule `acquire()` will wake up multiple Tasks + # that were waiting in the Semaphore queue due to FIFO rules. + sem = asyncio.Semaphore(0) + result = [] + count = 0 + + async def c1(result): + # First task immediatlly waits for semaphore. It will be awoken by c2. + self.assertEqual(sem._value, 0) + await sem.acquire() + # We should have woken up all waiting tasks now. + self.assertEqual(sem._value, 0) + # Create a fourth task. It should run after c3, not c2. + nonlocal t4 + t4 = asyncio.create_task(c4(result)) + result.append(1) + return True + + async def c2(result): + # The second task begins by releasing semaphore three times, + # for c1, c2, and c3. + sem.release() + sem.release() + sem.release() + self.assertEqual(sem._value, 2) + # It is locked, because c1 hasn't woken up yet. + self.assertTrue(sem.locked()) + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + self.assertTrue(sem.locked()) + result.append(3) + return True + + async def c4(result): + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + t4 = None + + await asyncio.sleep(0) + # Three tasks are in the queue, the first hasn't woken up yet. + self.assertEqual(sem._value, 2) + self.assertEqual(len(sem._waiters), 3) + await asyncio.sleep(0) + + tasks = [t1, t2, t3, t4] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) class BarrierTests(unittest.IsolatedAsyncioTestCase): @@ -1065,7 +1270,7 @@ async def test_barrier(self): self.assertIn("filling", repr(barrier)) with self.assertRaisesRegex( TypeError, - "object Barrier can't be used in 'await' expression", + "'Barrier' object can't be awaited", ): await barrier @@ -1276,7 +1481,7 @@ async def coro(): # wait again only for rewait tasks await barrier.wait() else: - # wait for end of draining state` + # wait for end of draining state await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() @@ -1575,7 +1780,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) async def test_abort_barrier_when_exception_then_resetting(self): - # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index dc25a46985e349..033784bc7aec05 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -77,7 +77,7 @@ async def test(lock): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - "can't be used in 'await' expression" + "can't be awaited" ): with await lock: pass diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index c42856e578b8cc..4b3d551dd7b3a2 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -585,11 +585,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) - self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -628,6 +627,19 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + @mock.patch('asyncio.proactor_events.logger') def test_sendto_exception(self, m_log): data = b'data' @@ -1006,9 +1018,9 @@ def setUp(self): self.addCleanup(self.file.close) super().setUp() - def make_socket(self, cleanup=True): + def make_socket(self, cleanup=True, blocking=False): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(False) + sock.setblocking(blocking) sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) if cleanup: @@ -1070,6 +1082,11 @@ def test_sock_sendfile_not_regular_file(self): 0, None)) self.assertEqual(self.file.tell(), 0) + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c72..5019e9a293525d 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -522,5 +522,204 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa q_class = asyncio.PriorityQueue +class _QueueShutdownTestMixin: + q_class = None + + def assertRaisesShutdown(self, msg="Didn't appear to shut-down queue"): + return self.assertRaises(asyncio.QueueShutDown, msg=msg) + + async def test_format(self): + q = self.q_class() + q.shutdown() + self.assertEqual(q._format(), 'maxsize=0 shutdown') + + async def test_shutdown_empty(self): + # Test shutting down an empty queue + + # Setup empty queue, and join() and get() tasks + q = self.q_class() + loop = asyncio.get_running_loop() + get_task = loop.create_task(q.get()) + await asyncio.sleep(0) # want get task pending before shutdown + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 0 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 13493d3c806d6a..266f057f0776c3 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -495,6 +495,24 @@ async def coro(): self.assertEqual(1, policy.set_event_loop.call_count) runner.close() + def test_no_repr_is_call_on_the_task_result(self): + # See https://github.com/python/cpython/issues/112559. + class MyResult: + def __init__(self): + self.repr_count = 0 + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + async def coro(): + return MyResult() + + + with asyncio.Runner() as runner: + result = runner.run(coro()) + + self.assertEqual(0, result.repr_count) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c22b780b5edcb8..aaeda33dd0c677 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1280,11 +1280,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -1320,6 +1319,18 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_tryagain(self): data = b'data' diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 0198da21d77028..2509d4382cdebd 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -93,13 +93,10 @@ async def wait_closed(self): class SendfileBase: - # 256 KiB plus small unaligned to buffer chunk - # Newer versions of Windows seems to have increased its internal - # buffer and tries to send as much of the data as it can as it - # has some form of buffering for this which is less than 256KiB - # on newer server versions and Windows 11. - # So DATA should be larger than 256 KiB to make this test reliable. - DATA = b"x" * (1024 * 256 + 1) + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) # Reduce socket buffer size to test on relative small data sets. BUF_SIZE = 4 * 1024 # 4 KiB @@ -470,8 +467,11 @@ def test_sendfile_close_peer_in_the_middle_of_receiving(self): self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), srv_proto.nbytes) - self.assertTrue(1024 <= self.file.tell() < len(self.DATA), - self.file.tell()) + if not (sys.platform == 'win32' + and isinstance(self.loop, asyncio.ProactorEventLoop)): + # On Windows, Proactor uses transmitFile, which does not update tell() + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) self.assertTrue(cli_proto.transport.is_closing()) def test_sendfile_fallback_close_peer_in_the_middle_of_receiving(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 06d8b60f219f1a..4ca8a166a0f1a1 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -1,4 +1,6 @@ import asyncio +import os +import socket import time import threading import unittest @@ -122,29 +124,213 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): - async def test_wait_closed(self): - async def serve(*args): - pass + async def test_wait_closed_basic(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) - # active count = 0 + # active count = 0, not closed: should block task1 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) - self.assertTrue(task1.done()) + self.assertFalse(task1.done()) - # active count != 0 - srv._attach() + # active count != 0, not closed: should block + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) + self.assertFalse(task1.done()) self.assertFalse(task2.done()) srv.close() await asyncio.sleep(0) + # active count != 0, closed: should block + task3 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) self.assertFalse(task2.done()) + self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() + # active count == 0, closed: should unblock + await task1 await task2 + await task3 + await srv.wait_closed() # Return immediately + + async def test_wait_closed_race(self): + # Test a regression in 3.12.0, should be fixed in 3.12.1 + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + loop = asyncio.get_running_loop() + loop.call_soon(srv.close) + loop.call_soon(wr.close) + await srv.wait_closed() + + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # socket buffers are full on both server and client sockets and + # the kernel stops accepting more data + s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + +# Test the various corner cases of Unix server socket removal +class UnixServerCleanupTests(unittest.IsolatedAsyncioTestCase): + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_addr_cleanup(self): + # Default scenario + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_sock_cleanup(self): + # Using already bound socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_gone(self): + # Someone else has already cleaned up the socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + os.unlink(addr) + + srv.close() + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_replaced(self): + # Someone else has replaced the socket with their own + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + os.unlink(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv.close() + self.assertTrue(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_prevented(self): + # Automatic cleanup explicitly disabled + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr, cleanup_socket=False) + + srv.close() + self.assertTrue(os.path.exists(addr)) @unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index 075113cbe8e4a6..acef24a703ba38 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -555,12 +555,93 @@ class SelectEventLoopTests(BaseSockTestsMixin, def create_event_loop(self): return asyncio.SelectorEventLoop() + class ProactorEventLoopTests(BaseSockTestsMixin, test_utils.TestCase): def create_event_loop(self): return asyncio.ProactorEventLoop() + + async def _basetest_datagram_send_to_non_listening_address(self, + recvfrom): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages + # after sending a message to an address that wasn't listening. + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address + # that is not listening + socket_3 = create_socket() + addr_3 = socket_3.getsockname() + socket_3.shutdown(socket.SHUT_RDWR) + socket_3.close() + + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + socket_2_recv_task = self.loop.create_task(recvfrom(socket_2)) + await asyncio.sleep(0) + + await self.loop.sock_sendto(socket_1, b'a', addr_2) + self.assertEqual(await socket_2_recv_task, b'a') + + await self.loop.sock_sendto(socket_2, b'b', addr_1) + self.assertEqual(await socket_1_recv_task, b'b') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # this should send to an address that isn't listening + await self.loop.sock_sendto(socket_1, b'c', addr_3) + self.assertEqual(await socket_1_recv_task, b'') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # socket 1 should still be able to receive messages after sending + # to an address that wasn't listening + socket_2.sendto(b'd', addr_1) + self.assertEqual(await socket_1_recv_task, b'd') + + socket_1.shutdown(socket.SHUT_RDWR) + socket_1.close() + socket_2.shutdown(socket.SHUT_RDWR) + socket_2.close() + + + def test_datagram_send_to_non_listening_address_recvfrom(self): + async def recvfrom(socket): + data, _ = await self.loop.sock_recvfrom(socket, 4096) + return data + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom)) + + + def test_datagram_send_to_non_listening_address_recvfrom_into(self): + async def recvfrom_into(socket): + buf = bytearray(4096) + length, _ = await self.loop.sock_recvfrom_into(socket, buf, + 4096) + return buf[:length] + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom_into)) + else: import selectors diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index e9cc735613fb8e..e072ede29ee3c7 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1,3 +1,7 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import asyncio import contextlib import gc diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 37d015339761c6..f5f0afeab51c9e 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -47,6 +47,7 @@ def connection_made(self, ssl_proto, *, do_handshake=None): sslobj = mock.Mock() # emulate reading decompressed data sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError if do_handshake is not None: sslobj.do_handshake = do_handshake ssl_proto._sslobj = sslobj @@ -120,7 +121,19 @@ def test_close_during_handshake(self): test_utils.run_briefly(self.loop) ssl_proto._app_transport.close() - self.assertTrue(transport.abort.called) + self.assertTrue(transport._force_close.called) + + def test_close_during_ssl_over_ssl(self): + # gh-113214: passing exceptions from the inner wrapped SSL protocol to the + # shim transport provided by the outer SSL protocol should not raise + # attribute errors + outer = self.ssl_protocol(proto=self.ssl_protocol()) + self.connection_made(outer) + # Closing the outer app transport should not raise an exception + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + outer._app_transport.close() + self.assertEqual(messages, []) def test_get_extra_info_on_closed_connection(self): waiter = self.loop.create_future() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000000..e6e32f7dbbbcba --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,97 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 9c92e75886c593..d32b7ff251885d 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -10,7 +10,6 @@ import unittest from unittest import mock import warnings -from test.support import socket_helper try: import ssl except ImportError: @@ -18,6 +17,7 @@ import asyncio from test.test_asyncio import utils as test_utils +from test.support import requires_subprocess, socket_helper def tearDownModule(): @@ -37,8 +37,7 @@ def tearDown(self): # just in case if we have transport close callbacks test_utils.run_briefly(self.loop) - self.loop.close() - gc.collect() + # set_event_loop() takes care of closing self.loop in a safe way super().tearDown() def _basetest_open_connection(self, open_connection_fut): @@ -384,6 +383,10 @@ def test_readuntil_separator(self): stream = asyncio.StreamReader(loop=self.loop) with self.assertRaisesRegex(ValueError, 'Separator should be'): self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=(b'',))) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=())) def test_readuntil_multi_chunks(self): stream = asyncio.StreamReader(loop=self.loop) @@ -467,6 +470,55 @@ def test_readuntil_limit_found_sep(self): self.assertEqual(b'some dataAAA', stream._buffer) + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil((b'DEF', b'BCDE'))) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil((b'A', b'ome dataA'))) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil((b'A', b'long sep'))) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + + def test_readuntil_bytearray(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data\r\n') + data = self.loop.run_until_complete(stream.readuntil(bytearray(b'\r\n'))) + self.assertEqual(b'some data\r\n', data) + self.assertEqual(b'', stream._buffer) + def test_readexactly_zero_or_less(self): # Read exact number of bytes (zero or less). stream = asyncio.StreamReader(loop=self.loop) @@ -770,51 +822,6 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") - @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") - def test_read_all_from_pipe_reader(self): - # See asyncio issue 168. This test is derived from the example - # subprocess_attach_read_pipe.py, but we configure the - # StreamReader's limit so that twice it is less than the size - # of the data writer. Also we must explicitly attach a child - # watcher to the event loop. - - code = """\ -import os, sys -fd = int(sys.argv[1]) -os.write(fd, b'data') -os.close(fd) -""" - rfd, wfd = os.pipe() - args = [sys.executable, '-c', code, str(wfd)] - - pipe = open(rfd, 'rb', 0) - reader = asyncio.StreamReader(loop=self.loop, limit=1) - protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) - transport, _ = self.loop.run_until_complete( - self.loop.connect_read_pipe(lambda: protocol, pipe)) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(watcher) - create = asyncio.create_subprocess_exec( - *args, - pass_fds={wfd}, - ) - proc = self.loop.run_until_complete(create) - self.loop.run_until_complete(proc.wait()) - finally: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - - os.close(wfd) - data = self.loop.run_until_complete(reader.read(-1)) - self.assertEqual(data, b'data') - def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() @@ -1083,10 +1090,11 @@ async def inner(httpd): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() self.assertTrue(data.endswith(b'\r\n\r\nTest message')) - with self.assertWarns(ResourceWarning): + with self.assertWarns(ResourceWarning) as cm: del wr gc.collect() - + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed None: async def main() -> None: - cmd = f'''{sys.executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + executable = sys.executable + if sys.platform == "win32": + executable = f'"{executable}"' + cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' env = os.environ.copy() env["FOO"] = "bar" proc = await asyncio.create_subprocess_shell( @@ -867,60 +864,47 @@ async def main(): self.loop.run_until_complete(main()) + @unittest.skipIf(sys.platform != 'linux', "Linux only") + def test_subprocess_send_signal_race(self): + # See https://github.com/python/cpython/issues/87744 + async def main(): + for _ in range(10): + proc = await asyncio.create_subprocess_exec('sleep', '0.1') + await asyncio.sleep(0.1) + try: + proc.send_signal(signal.SIGUSR1) + except ProcessLookupError: + pass + self.assertNotEqual(await proc.wait(), 255) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix class SubprocessWatcherMixin(SubprocessMixin): - Watcher = None - def setUp(self): super().setUp() policy = asyncio.get_event_loop_policy() self.loop = policy.new_event_loop() self.set_event_loop(self.loop) - watcher = self._get_watcher() - watcher.attach_loop(self.loop) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - policy.set_child_watcher(watcher) + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) - def tearDown(self): - super().tearDown() - policy = asyncio.get_event_loop_policy() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - policy.set_child_watcher(None) - watcher.attach_loop(None) - watcher.close() class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - - def _get_watcher(self): - return unix_events.ThreadedChildWatcher() - - class SubprocessSafeWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.SafeChildWatcher() - - class MultiLoopChildWatcherTests(test_utils.TestCase): - - def test_warns(self): - with self.assertWarns(DeprecationWarning): - unix_events.MultiLoopChildWatcher() - - class SubprocessFastWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.FastChildWatcher() + def setUp(self): + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() @unittest.skipUnless( unix_events.can_use_pidfd(), @@ -929,65 +913,8 @@ def _get_watcher(self): class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - def _get_watcher(self): - return unix_events.PidfdChildWatcher() - - - class GenericWatcherTests(test_utils.TestCase): - - def test_create_subprocess_fails_with_inactive_watcher(self): - watcher = mock.create_autospec(asyncio.AbstractChildWatcher) - watcher.is_active.return_value = False - - async def execute(): - asyncio.set_child_watcher(watcher) - - with self.assertRaises(RuntimeError): - await subprocess.create_subprocess_exec( - os_helper.FakePath(sys.executable), '-c', 'pass') + pass - watcher.add_child_handler.assert_not_called() - - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(runner.run(execute())) - self.assertListEqual(watcher.mock_calls, [ - mock.call.__enter__(), - mock.call.is_active(), - mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), - ], watcher.mock_calls) - - - @unittest.skipUnless( - unix_events.can_use_pidfd(), - "operating system does not support pidfds", - ) - def test_create_subprocess_with_pidfd(self): - async def in_thread(): - proc = await asyncio.create_subprocess_exec( - *PROGRAM_CAT, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = await proc.communicate(b"some data") - return proc.returncode, stdout - - async def main(): - # asyncio.Runner did not call asyncio.set_event_loop() - with self.assertRaises(RuntimeError): - asyncio.get_event_loop_policy().get_event_loop() - return await asyncio.to_thread(asyncio.run, in_thread()) - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) - try: - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - returncode, stdout = runner.run(main()) - self.assertEqual(returncode, 0) - self.assertEqual(stdout, b'some data') - finally: - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 6a0231f2859a62..4852536defc93d 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -7,6 +7,9 @@ import contextlib from asyncio import taskgroups import unittest +import warnings + +from test.test_asyncio.utils import await_without_task # To prevent a warning "test altered the execution environment" @@ -736,10 +739,7 @@ async def coro2(g): await asyncio.sleep(1) except asyncio.CancelledError: with self.assertRaises(RuntimeError): - g.create_task(c1 := coro1()) - # We still have to await c1 to avoid a warning - with self.assertRaises(ZeroDivisionError): - await c1 + g.create_task(coro1()) with self.assertRaises(ExceptionGroup) as cm: async with taskgroups.TaskGroup() as g: @@ -779,6 +779,126 @@ async def main(): await asyncio.create_task(main()) + async def test_taskgroup_already_entered(self): + tg = taskgroups.TaskGroup() + async with tg: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_double_enter(self): + tg = taskgroups.TaskGroup() + async with tg: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_finished(self): + async def create_task_after_tg_finish(): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + # Make sure the coroutine was closed when submitted to the inactive tg + # (if not closed, a RuntimeWarning should have been raised) + with warnings.catch_warnings(record=True) as w: + await create_task_after_tg_finish() + self.assertEqual(len(w), 0) + + async def test_taskgroup_not_entered(self): + tg = taskgroups.TaskGroup() + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + async def test_taskgroup_without_parent_task(self): + tg = taskgroups.TaskGroup() + with self.assertRaisesRegex(RuntimeError, "parent task"): + await await_without_task(tg.__aenter__()) + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + def test_coro_closed_when_tg_closed(self): + async def run_coro_after_tg_closes(): + async with taskgroups.TaskGroup() as tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + loop = asyncio.get_event_loop() + loop.run_until_complete(run_coro_after_tg_closes()) + + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4dfaff847edb90..9b22fb942c6339 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,6 +1,7 @@ """Tests for tasks.py.""" import collections +import contextlib import contextvars import gc import io @@ -85,6 +86,7 @@ class BaseTaskTests: Task = None Future = None + all_tasks = None def new_task(self, loop, coro, name='TestTask', context=None): return self.__class__.Task(coro, loop=loop, name=name, context=context) @@ -683,6 +685,30 @@ def on_timeout(): finally: loop.close() + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + def test_cancel(self): def gen(): @@ -1409,12 +1435,6 @@ def gen(): yield 0.01 yield 0 - loop = self.new_test_loop(gen) - # disable "slow callback" warning - loop.slow_callback_duration = 1.0 - completed = set() - time_shifted = False - async def sleeper(dt, x): nonlocal time_shifted await asyncio.sleep(dt) @@ -1424,21 +1444,78 @@ async def sleeper(dt, x): loop.advance_time(0.14) return x - a = sleeper(0.01, 'a') - b = sleeper(0.01, 'b') - c = sleeper(0.15, 'c') + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values - async def foo(): + async def try_async_iterator(awaitables): values = [] - for f in asyncio.as_completed([b, c, a]): + async for f in asyncio.as_completed(awaitables): values.append(await f) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertAlmostEqual(0.15, loop.time()) - self.assertTrue('a' in res[:2]) - self.assertTrue('b' in res[:2]) - self.assertEqual(res[2], 'c') + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') def test_as_completed_with_timeout(self): @@ -1448,12 +1525,7 @@ def gen(): yield 0 yield 0.1 - loop = self.new_test_loop(gen) - - a = loop.create_task(asyncio.sleep(0.1, 'a')) - b = loop.create_task(asyncio.sleep(0.15, 'b')) - - async def foo(): + async def try_iterator(): values = [] for f in asyncio.as_completed([a, b], timeout=0.12): if values: @@ -1465,16 +1537,33 @@ async def foo(): values.append((2, exc)) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertEqual(len(res), 2, res) - self.assertEqual(res[0], (1, 'a')) - self.assertEqual(res[1][0], 2) - self.assertIsInstance(res[1][1], asyncio.TimeoutError) - self.assertAlmostEqual(0.12, loop.time()) + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values - # move forward to close generator - loop.advance_time(10) - loop.run_until_complete(asyncio.wait([a, b])) + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) def test_as_completed_with_unused_timeout(self): @@ -1483,19 +1572,75 @@ def gen(): yield 0 yield 0.01 - loop = self.new_test_loop(gen) - - a = asyncio.sleep(0.01, 'a') - - async def foo(): + async def try_iterator(): for f in asyncio.as_completed([a], timeout=1): v = await f self.assertEqual(v, 'a') - loop.run_until_complete(self.new_task(loop, foo())) + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') - def test_as_completed_reverse_wait(self): + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. def gen(): yield 0 yield 0.05 @@ -1522,7 +1667,8 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_concurrent(self): - + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. def gen(): when = yield self.assertAlmostEqual(0.05, when) @@ -1530,38 +1676,55 @@ def gen(): self.assertAlmostEqual(0.05, when) yield 0.05 - a = asyncio.sleep(0.05, 'a') - b = asyncio.sleep(0.05, 'b') - fs = {a, b} + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) - async def test(): - futs = list(asyncio.as_completed(fs)) - self.assertEqual(len(futs), 2) - done, pending = await asyncio.wait( - [asyncio.ensure_future(fut) for fut in futs] - ) - self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] - loop = self.new_test_loop(gen) - loop.run_until_complete(test()) + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) def test_as_completed_duplicate_coroutines(self): async def coro(s): return s - async def runner(): + async def try_iterator(): result = [] c = coro('ham') for f in asyncio.as_completed([c, c, coro('spam')]): result.append(await f) return result - fut = self.new_task(self.loop, runner()) - self.loop.run_until_complete(fut) - result = fut.result() - self.assertEqual(set(result), {'ham', 'spam'}) - self.assertEqual(len(result), 2) + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) def test_as_completed_coroutine_without_loop(self): async def coro(): @@ -1570,8 +1733,8 @@ async def coro(): a = coro() self.addCleanup(a.close) - futs = asyncio.as_completed([a]) with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) list(futs) def test_as_completed_coroutine_use_running_loop(self): @@ -2044,14 +2207,32 @@ async def coro(): self.assertEqual(res, 42) def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() - # as_completed() expects a list of futures, not a future instance - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(fut)) + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) coro = coroutine_function() - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(coro)) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) coro.close() def test_wait_invalid_args(self): @@ -2087,7 +2268,7 @@ async def kill_me(loop): coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) - self.assertEqual(asyncio.all_tasks(loop=self.loop), {task}) + self.assertEqual(self.all_tasks(loop=self.loop), {task}) asyncio.set_event_loop(None) @@ -2102,7 +2283,7 @@ async def kill_me(loop): # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(asyncio.all_tasks(loop=self.loop), set()) + self.assertEqual(self.all_tasks(loop=self.loop), set()) mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', @@ -2251,7 +2432,7 @@ async def coro(): message = m_log.error.call_args[0][0] self.assertIn('Task was destroyed but it is pending', message) - self.assertEqual(asyncio.all_tasks(self.loop), set()) + self.assertEqual(self.all_tasks(self.loop), set()) def test_create_task_with_noncoroutine(self): with self.assertRaisesRegex(TypeError, @@ -2551,6 +2732,7 @@ async def func(): # Add patched Task & Future back to the test case cls.Task = Task cls.Future = Future + cls.all_tasks = tasks.all_tasks # Add an extra unit-test cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture @@ -2624,6 +2806,7 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @support.refcount_test def test_refleaks_in_task___init__(self): @@ -2655,6 +2838,7 @@ class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2664,6 +2848,7 @@ class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2673,6 +2858,7 @@ class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): Future = getattr(futures, '_CFuture', None) Task = tasks._PyTask + all_tasks = tasks._py_all_tasks @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2681,6 +2867,7 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2689,6 +2876,7 @@ class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = tasks._PyTask Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, @@ -2696,6 +2884,7 @@ class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, Task = tasks._PyTask Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) @add_subclass_tests @@ -2735,6 +2924,7 @@ class BaseTaskIntrospectionTests: _unregister_task = None _enter_task = None _leave_task = None + all_tasks = None def test__register_task_1(self): class TaskLike: @@ -2748,9 +2938,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_2(self): @@ -2764,9 +2954,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_3(self): @@ -2780,9 +2970,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._unregister_task(task) def test__enter_task(self): @@ -2833,13 +3023,13 @@ def test__unregister_task(self): task.get_loop = lambda: loop self._register_task(task) self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) def test__unregister_task_not_registered(self): task = mock.Mock() loop = mock.Mock() self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): @@ -2847,6 +3037,7 @@ class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._py_unregister_task) _enter_task = staticmethod(tasks._py_enter_task) _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) @unittest.skipUnless(hasattr(tasks, '_c_register_task'), @@ -2857,6 +3048,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._c_unregister_task) _enter_task = staticmethod(tasks._c_enter_task) _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) else: _register_task = _unregister_task = _enter_task = _leave_task = None @@ -2923,14 +3115,14 @@ def test_asyncio_module_compiled(self): # fail on systems where C modules were successfully compiled # (hence the test for _functools etc), but _asyncio somehow didn't. try: - import _functools - import _json - import _pickle + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 except ImportError: self.skipTest('C modules are not available') else: try: - import _asyncio + import _asyncio # noqa: F401 except ImportError: self.fail('_asyncio module is missing') diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index e9b59b953518b3..1f7f9ee696a525 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -5,11 +5,12 @@ import asyncio +from test.test_asyncio.utils import await_without_task + def tearDownModule(): asyncio.set_event_loop_policy(None) - class TimeoutTests(unittest.IsolatedAsyncioTestCase): async def test_timeout_basic(self): @@ -115,15 +116,68 @@ async def test_foreign_exception_passed(self): raise KeyError self.assertFalse(cm.expired()) + async def test_timeout_exception_context(self): + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.01): + try: + 1/0 + finally: + await asyncio.sleep(1) + e = cm.exception + # Expect TimeoutError caused by CancelledError raised during handling + # of ZeroDivisionError. + e2 = e.__cause__ + self.assertIsInstance(e2, asyncio.CancelledError) + self.assertIs(e.__context__, e2) + self.assertIsNone(e2.__cause__) + self.assertIsInstance(e2.__context__, ZeroDivisionError) + async def test_foreign_exception_on_timeout(self): async def crash(): try: await asyncio.sleep(1) finally: 1/0 - with self.assertRaises(ZeroDivisionError): + with self.assertRaises(ZeroDivisionError) as cm: async with asyncio.timeout(0.01): await crash() + e = cm.exception + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIs(e2.__context__, e3) + + async def test_foreign_exception_on_timeout_2(self): + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + try: + try: + raise ValueError + finally: + await asyncio.sleep(1) + finally: + try: + raise KeyError + finally: + 1/0 + e = cm.exception + # Expect ZeroDivisionError raised during handling of KeyError + # raised during handling of TimeoutError caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, KeyError) + self.assertIsNone(e2.__cause__) + e3 = e2.__context__ + self.assertIsInstance(e3, TimeoutError) + e4 = e3.__cause__ + self.assertIsInstance(e4, asyncio.CancelledError) + self.assertIsNone(e4.__cause__) + self.assertIsInstance(e4.__context__, ValueError) + self.assertIs(e3.__context__, e4) async def test_foreign_cancel_doesnt_timeout_if_not_expired(self): with self.assertRaises(asyncio.CancelledError): @@ -218,14 +272,30 @@ async def test_repr_disabled(self): self.assertEqual(repr(cm), r"") async def test_nested_timeout_in_finally(self): - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm1: async with asyncio.timeout(0.01): try: await asyncio.sleep(1) finally: - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm2: async with asyncio.timeout(0.01): await asyncio.sleep(10) + e1 = cm1.exception + # Expect TimeoutError caused by CancelledError. + e12 = e1.__cause__ + self.assertIsInstance(e12, asyncio.CancelledError) + self.assertIsNone(e12.__cause__) + self.assertIsNone(e12.__context__) + self.assertIs(e1.__context__, e12) + e2 = cm2.exception + # Expect TimeoutError caused by CancelledError raised during + # handling of other CancelledError (which is the same as in + # the above chain). + e22 = e2.__cause__ + self.assertIsInstance(e22, asyncio.CancelledError) + self.assertIsNone(e22.__cause__) + self.assertIs(e22.__context__, e12) + self.assertIs(e2.__context__, e22) async def test_timeout_after_cancellation(self): try: @@ -234,7 +304,7 @@ async def test_timeout_after_cancellation(self): except asyncio.CancelledError: pass finally: - with self.assertRaises(TimeoutError): + with self.assertRaises(TimeoutError) as cm: async with asyncio.timeout(0.0): await asyncio.sleep(1) # some cleanup @@ -250,12 +320,91 @@ async def test_cancel_in_timeout_after_cancellation(self): asyncio.current_task().cancel() await asyncio.sleep(2) # some cleanup - async def test_timeout_exception_cause (self): - with self.assertRaises(asyncio.TimeoutError) as exc: - async with asyncio.timeout(0): + async def test_timeout_already_entered(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_double_enter(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_finished(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "finished"): + cm.reschedule(0.02) + + async def test_timeout_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expired"): + cm.reschedule(0.02) + + async def test_timeout_expiring(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaises(asyncio.CancelledError): await asyncio.sleep(1) - cause = exc.exception.__cause__ - assert isinstance(cause, asyncio.CancelledError) + with self.assertRaisesRegex(RuntimeError, "expiring"): + cm.reschedule(0.02) + + async def test_timeout_not_entered(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_without_task(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "task"): + await await_without_task(cm.__aenter__()) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_taskgroup(self): + async def task(): + try: + await asyncio.sleep(2) # Will be interrupted after 0.01 second + finally: + 1/0 # Crash in cleanup + + with self.assertRaises(ExceptionGroup) as cm: + async with asyncio.timeout(0.01): + async with asyncio.TaskGroup() as tg: + tg.create_task(task()) + try: + raise ValueError + finally: + await asyncio.sleep(1) + eg = cm.exception + # Expect ExceptionGroup raised during handling of TimeoutError caused + # by CancelledError raised during handling of ValueError. + self.assertIsNone(eg.__cause__) + e_1 = eg.__context__ + self.assertIsInstance(e_1, TimeoutError) + e_2 = e_1.__cause__ + self.assertIsInstance(e_2, asyncio.CancelledError) + self.assertIsNone(e_2.__cause__) + self.assertIsInstance(e_2.__context__, ValueError) + self.assertIs(e_1.__context__, e_2) + + self.assertEqual(len(eg.exceptions), 1, eg) + e1 = eg.exceptions[0] + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError (it is a different CancelledError). + self.assertIsInstance(e1, ZeroDivisionError) + self.assertIsNone(e1.__cause__) + e2 = e1.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIsNone(e3.__context__) + self.assertIsNone(e3.__cause__) + self.assertIs(e2.__context__, e3) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d2c8cba6acfa31..4966775acac7be 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -6,7 +6,6 @@ import multiprocessing from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests import os -import pathlib import signal import socket import stat @@ -304,20 +303,20 @@ def test_create_unix_server_existing_path_sock(self): self.loop.run_until_complete(srv.wait_closed()) @socket_helper.skip_unless_bind_unix_socket - def test_create_unix_server_pathlib(self): + def test_create_unix_server_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) srv_coro = self.loop.create_unix_server(lambda: None, path) srv = self.loop.run_until_complete(srv_coro) srv.close() self.loop.run_until_complete(srv.wait_closed()) - def test_create_unix_connection_pathlib(self): + def test_create_unix_connection_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) coro = self.loop.create_unix_connection(lambda: None, path) with self.assertRaises(FileNotFoundError): - # If pathlib.Path wasn't supported, the exception would be + # If path-like object weren't supported, the exception would be # different. self.loop.run_until_complete(coro) @@ -1113,697 +1112,6 @@ def test_write_eof_pending(self): self.assertFalse(self.protocol.connection_lost.called) -class AbstractChildWatcherTests(unittest.TestCase): - - def test_warns_on_subclassing(self): - with self.assertWarns(DeprecationWarning): - class MyWatcher(asyncio.AbstractChildWatcher): - pass - - def test_not_implemented(self): - f = mock.Mock() - watcher = asyncio.AbstractChildWatcher() - self.assertRaises( - NotImplementedError, watcher.add_child_handler, f, f) - self.assertRaises( - NotImplementedError, watcher.remove_child_handler, f) - self.assertRaises( - NotImplementedError, watcher.attach_loop, f) - self.assertRaises( - NotImplementedError, watcher.close) - self.assertRaises( - NotImplementedError, watcher.is_active) - self.assertRaises( - NotImplementedError, watcher.__enter__) - self.assertRaises( - NotImplementedError, watcher.__exit__, f, f, f) - - -class BaseChildWatcherTests(unittest.TestCase): - - def test_not_implemented(self): - f = mock.Mock() - watcher = unix_events.BaseChildWatcher() - self.assertRaises( - NotImplementedError, watcher._do_waitpid, f) - - -class ChildWatcherTestsMixin: - - ignore_warnings = mock.patch.object(log.logger, "warning") - - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.running = False - self.zombies = {} - - with mock.patch.object( - self.loop, "add_signal_handler") as self.m_add_signal_handler: - self.watcher = self.create_watcher() - self.watcher.attach_loop(self.loop) - - def waitpid(self, pid, flags): - if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1: - self.assertGreater(pid, 0) - try: - if pid < 0: - return self.zombies.popitem() - else: - return pid, self.zombies.pop(pid) - except KeyError: - pass - if self.running: - return 0, 0 - else: - raise ChildProcessError() - - def add_zombie(self, pid, status): - self.zombies[pid] = status - - def waitstatus_to_exitcode(self, status): - if status > 32768: - return status - 32768 - elif 32700 < status < 32768: - return status - 32768 - else: - return status - - def test_create_watcher(self): - self.m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - def waitpid_mocks(func): - def wrapped_func(self): - def patch(target, wrapper): - return mock.patch(target, wraps=wrapper, - new_callable=mock.Mock) - - with patch('asyncio.unix_events.waitstatus_to_exitcode', self.waitstatus_to_exitcode), \ - patch('os.waitpid', self.waitpid) as m_waitpid: - func(self, m_waitpid) - return wrapped_func - - @waitpid_mocks - def test_sigchld(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(42, callback, 9, 10, 14) - - self.assertFalse(callback.called) - - # child is running - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # child terminates (returncode 12) - self.running = False - self.add_zombie(42, EXITCODE(12)) - self.watcher._sig_chld() - - callback.assert_called_once_with(42, 12, 9, 10, 14) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(42, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_two_children(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(43, callback1, 7, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(44, callback2, 147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (signal 3) - self.add_zombie(43, SIGNAL(3)) - self.watcher._sig_chld() - - callback1.assert_called_once_with(43, -3, 7, 8) - self.assertFalse(callback2.called) - - callback1.reset_mock() - - # child 2 still running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 2 terminates (code 108) - self.add_zombie(44, EXITCODE(108)) - self.running = False - self.watcher._sig_chld() - - callback2.assert_called_once_with(44, 108, 147, 18) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(43, EXITCODE(14)) - self.add_zombie(44, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_two_children_terminating_together(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(45, callback1, 17, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(46, callback2, 1147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (code 78) - # child 2 terminates (signal 5) - self.add_zombie(45, EXITCODE(78)) - self.add_zombie(46, SIGNAL(5)) - self.running = False - self.watcher._sig_chld() - - callback1.assert_called_once_with(45, 78, 17, 8) - callback2.assert_called_once_with(46, -5, 1147, 18) - - callback1.reset_mock() - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(45, EXITCODE(14)) - self.add_zombie(46, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_race_condition(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - # child terminates before being registered - self.add_zombie(50, EXITCODE(4)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(50, callback, 1, 12) - - callback.assert_called_once_with(50, 4, 1, 12) - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(50, SIGNAL(1)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_replace_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(51, callback1, 19) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register the same child again - with self.watcher: - self.watcher.add_child_handler(51, callback2, 21) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child terminates (signal 8) - self.running = False - self.add_zombie(51, SIGNAL(8)) - self.watcher._sig_chld() - - callback2.assert_called_once_with(51, -8, 21) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(51, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_remove_handler(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(52, callback, 1984) - - self.assertFalse(callback.called) - - # unregister the child - self.watcher.remove_child_handler(52) - - self.assertFalse(callback.called) - - # child terminates (code 99) - self.running = False - self.add_zombie(52, EXITCODE(99)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_unknown_status(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(53, callback, -19) - - self.assertFalse(callback.called) - - # terminate with unknown status - self.zombies[53] = 1178 - self.running = False - self.watcher._sig_chld() - - callback.assert_called_once_with(53, 1178, -19) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(53, EXITCODE(101)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_remove_child_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - # register children - with self.watcher: - self.running = True - self.watcher.add_child_handler(54, callback1, 1) - self.watcher.add_child_handler(55, callback2, 2) - self.watcher.add_child_handler(56, callback3, 3) - - # remove child handler 1 - self.assertTrue(self.watcher.remove_child_handler(54)) - - # remove child handler 2 multiple times - self.assertTrue(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - - # all children terminate - self.add_zombie(54, EXITCODE(0)) - self.add_zombie(55, EXITCODE(1)) - self.add_zombie(56, EXITCODE(2)) - self.running = False - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(56, 2, 3) - - @waitpid_mocks - def test_sigchld_unhandled_exception(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(57, callback) - - # raise an exception - m_waitpid.side_effect = ValueError - - with mock.patch.object(log.logger, - 'error') as m_error: - - self.assertEqual(self.watcher._sig_chld(), None) - self.assertTrue(m_error.called) - - @waitpid_mocks - def test_sigchld_child_reaped_elsewhere(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(58, callback) - - self.assertFalse(callback.called) - - # child terminates - self.running = False - self.add_zombie(58, EXITCODE(4)) - - # waitpid is called elsewhere - os.waitpid(58, os.WNOHANG) - - m_waitpid.reset_mock() - - # sigchld - with self.ignore_warnings: - self.watcher._sig_chld() - - if isinstance(self.watcher, asyncio.FastChildWatcher): - # here the FastChildWatcher enters a deadlock - # (there is no way to prevent it) - self.assertFalse(callback.called) - else: - callback.assert_called_once_with(58, 255) - - @waitpid_mocks - def test_sigchld_unknown_pid_during_registration(self, m_waitpid): - # register two children - callback1 = mock.Mock() - callback2 = mock.Mock() - - with self.ignore_warnings, self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(591, EXITCODE(7)) - # an unknown child terminates - self.add_zombie(593, EXITCODE(17)) - - self.watcher._sig_chld() - - self.watcher.add_child_handler(591, callback1) - self.watcher.add_child_handler(592, callback2) - - callback1.assert_called_once_with(591, 7) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_set_loop(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(60, callback) - - # attach a new loop - old_loop = self.loop - self.loop = self.new_test_loop() - patch = mock.patch.object - - with patch(old_loop, "remove_signal_handler") as m_old_remove, \ - patch(self.loop, "add_signal_handler") as m_new_add: - - self.watcher.attach_loop(self.loop) - - m_old_remove.assert_called_once_with( - signal.SIGCHLD) - m_new_add.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - # child terminates - self.running = False - self.add_zombie(60, EXITCODE(9)) - self.watcher._sig_chld() - - callback.assert_called_once_with(60, 9) - - @waitpid_mocks - def test_set_loop_race_condition(self, m_waitpid): - # register 3 children - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(61, callback1) - self.watcher.add_child_handler(62, callback2) - self.watcher.add_child_handler(622, callback3) - - # detach the loop - old_loop = self.loop - self.loop = None - - with mock.patch.object( - old_loop, "remove_signal_handler") as m_remove_signal_handler: - - with self.assertWarnsRegex( - RuntimeWarning, 'A loop is being detached'): - self.watcher.attach_loop(None) - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - - # child 1 & 2 terminate - self.add_zombie(61, EXITCODE(11)) - self.add_zombie(62, SIGNAL(5)) - - # SIGCHLD was not caught - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - self.assertFalse(callback3.called) - - # attach a new loop - self.loop = self.new_test_loop() - - with mock.patch.object( - self.loop, "add_signal_handler") as m_add_signal_handler: - - self.watcher.attach_loop(self.loop) - - m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - callback1.assert_called_once_with(61, 11) # race condition! - callback2.assert_called_once_with(62, -5) # race condition! - self.assertFalse(callback3.called) - - callback1.reset_mock() - callback2.reset_mock() - - # child 3 terminates - self.running = False - self.add_zombie(622, EXITCODE(19)) - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(622, 19) - - @waitpid_mocks - def test_close(self, m_waitpid): - # register two children - callback1 = mock.Mock() - - with self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(63, EXITCODE(9)) - # other child terminates - self.add_zombie(65, EXITCODE(18)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(63, callback1) - self.watcher.add_child_handler(64, callback1) - - self.assertEqual(len(self.watcher._callbacks), 1) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertEqual(len(self.watcher._zombies), 1) - - with mock.patch.object( - self.loop, - "remove_signal_handler") as m_remove_signal_handler: - - self.watcher.close() - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - self.assertFalse(self.watcher._callbacks) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertFalse(self.watcher._zombies) - - -class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.SafeChildWatcher() - - -class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.FastChildWatcher() - - -class PolicyTests(unittest.TestCase): - - def create_policy(self): - return asyncio.DefaultEventLoopPolicy() - - @mock.patch('asyncio.unix_events.can_use_pidfd') - def test_get_default_child_watcher(self, m_can_use_pidfd): - m_can_use_pidfd.return_value = False - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - m_can_use_pidfd.return_value = True - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_after_set(self): - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.FastChildWatcher() - policy.set_child_watcher(watcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_thread(self): - - def f(): - policy.set_event_loop(policy.new_event_loop()) - - self.assertIsInstance(policy.get_event_loop(), - asyncio.AbstractEventLoop) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIsNone(watcher._loop) - - policy.get_event_loop().close() - - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - policy.set_child_watcher(asyncio.SafeChildWatcher()) - - th = threading.Thread(target=f) - th.start() - th.join() - - def test_child_watcher_replace_mainloop_existing(self): - policy = self.create_policy() - loop = policy.new_event_loop() - policy.set_event_loop(loop) - - # Explicitly setup SafeChildWatcher, - # default ThreadedChildWatcher has no _loop property - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - policy.set_child_watcher(watcher) - watcher.attach_loop(loop) - - self.assertIs(watcher._loop, loop) - - new_loop = policy.new_event_loop() - policy.set_event_loop(new_loop) - - self.assertIs(watcher._loop, new_loop) - - policy.set_event_loop(None) - - self.assertIs(watcher._loop, None) - - loop.close() - new_loop.close() - - class TestFunctional(unittest.TestCase): def setUp(self): @@ -1874,7 +1182,7 @@ async def runner(): wsock.close() -@unittest.skipUnless(hasattr(os, 'fork'), 'requires os.fork()') +@support.requires_fork() class TestFork(unittest.IsolatedAsyncioTestCase): async def test_fork_not_share_event_loop(self): diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d52f32534a0cfe..11a8eeeab37634 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -249,8 +249,8 @@ async def test_cancel_wait_for(self): await self._test_cancel_wait_for(60.0) async def test_wait_for_cancel_suppressed(self): - # GH-86296: Supressing CancelledError is discouraged - # but if a task subpresses CancelledError and returns a value, + # GH-86296: Suppressing CancelledError is discouraged + # but if a task suppresses CancelledError and returns a value, # `wait_for` should return the value instead of raising CancelledError. # This is the same behavior as `asyncio.timeout`. diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 6e6c90a247b291..0c128c599ba011 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -36,7 +36,23 @@ def data_received(self, data): self.trans.close() -class ProactorLoopCtrlC(test_utils.TestCase): +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): def test_ctrl_c(self): @@ -58,7 +74,7 @@ def SIGINT_after_delay(): thread.join() -class ProactorMultithreading(test_utils.TestCase): +class ProactorMultithreading(WindowsEventsTestCase): def test_run_from_nonmain_thread(self): finished = False @@ -79,7 +95,7 @@ def func(): self.assertTrue(finished) -class ProactorTests(test_utils.TestCase): +class ProactorTests(WindowsEventsTestCase): def setUp(self): super().setUp() @@ -283,8 +299,32 @@ async def probe(): return "done" - -class WinPolicyTests(test_utils.TestCase): + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index e1101bf42eb24e..dbb8d27c176950 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -37,9 +37,9 @@ # Use the maximum known clock resolution (gh-75191, gh-110088): Windows -# GetTickCount64() has a resolution of 15.6 ms. Use 20 ms to tolerate rounding +# GetTickCount64() has a resolution of 15.6 ms. Use 50 ms to tolerate rounding # issues. -CLOCK_RES = 0.020 +CLOCK_RES = 0.050 def data_file(*filename): @@ -546,20 +546,6 @@ def close_loop(loop): else: loop._default_executor.shutdown(wait=True) loop.close() - policy = support.maybe_get_event_loop_policy() - if policy is not None: - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - except NotImplementedError: - # watcher is not implemented by EventLoopPolicy, e.g. Windows - pass - else: - if isinstance(watcher, asyncio.ThreadedChildWatcher): - threads = list(watcher._threads.values()) - for thread in threads: - thread.join() def set_event_loop(self, loop, *, cleanup=True): if loop is None: @@ -612,3 +598,18 @@ def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, sock.family = family sock.gettimeout.return_value = 0.0 return sock + + +async def await_without_task(coro): + exc = None + def func(): + try: + for _ in coro.__await__(): + pass + except BaseException as err: + nonlocal exc + exc = err + asyncio.get_running_loop().call_soon(func) + await asyncio.sleep(0) + if exc is not None: + raise exc diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 47e5832d311bd1..7206307d8b0664 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -89,6 +89,7 @@ def test_excepthook(self): ) def test_unraisablehook(self): + import_helper.import_module("_testcapi") returncode, events, stderr = self.run_python("test_unraisablehook") if returncode: self.fail(stderr) @@ -139,6 +140,7 @@ def test_gc(self): ) + @support.requires_resource('network') def test_http(self): import_helper.import_module("http.client") returncode, events, stderr = self.run_python("test_http_client") @@ -209,6 +211,8 @@ def test_threading(self): expected = [ ("_thread.start_new_thread", "(, (), None)"), ("test.test_func", "()"), + ("_thread.start_joinable_thread", "(, 1, None)"), + ("test.test_func", "()"), ] self.assertEqual(actual, expected) @@ -288,6 +292,20 @@ def test_sys_monitoring_register_callback(self): self.assertEqual(actual, expected) + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index fa03fa1d61ceab..409c8c109e885f 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -545,6 +545,40 @@ def test_b85encode(self): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_z85encode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'www.python.org': b'CxXl-AcVLsz/dgCA+t', + bytes(range(255)): b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""", + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""": + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""", + b'no padding..': b'zF{UpvpS[.zF7NO', + b'zero compression\x00\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg00000', + b'zero compression\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg0000', + b"""Boundary:\x00\x00\x00\x00""": b"""lt}0:wmoI7iSGcW00""", + b'Space compr: ': b'q/DePwGUG3ze:IRarR^H', + b'\xff': b'@@', + b'\xff'*2: b'%nJ', + b'\xff'*3: b'%nS9', + b'\xff'*4: b'%nSc0', + } + + for data, res in tests.items(): + eq(base64.z85encode(data), res) + + self.check_other_types(base64.z85encode, b"www.python.org", + b'CxXl-AcVLsz/dgCA+t') + def test_a85decode(self): eq = self.assertEqual @@ -586,6 +620,7 @@ def test_a85decode(self): eq(base64.a85decode(b'y+', b"www.python.org") @@ -625,6 +660,41 @@ def test_b85decode(self): self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T', b"www.python.org") + def test_z85decode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'CxXl-AcVLsz/dgCA+t': b'www.python.org', + b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""": bytes(range(255)), + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""": + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""", + b'zF{UpvpS[.zF7NO': b'no padding..', + b'Ds.bnay/tbAb]JhB7]Mg00000': b'zero compression\x00\x00\x00\x00', + b'Ds.bnay/tbAb]JhB7]Mg0000': b'zero compression\x00\x00\x00', + b"""lt}0:wmoI7iSGcW00""": b"""Boundary:\x00\x00\x00\x00""", + b'q/DePwGUG3ze:IRarR^H': b'Space compr: ', + b'@@': b'\xff', + b'%nJ': b'\xff'*2, + b'%nS9': b'\xff'*3, + b'%nSc0': b'\xff'*4, + } + + for data, res in tests.items(): + eq(base64.z85decode(data), res) + eq(base64.z85decode(data.decode("ascii")), res) + + self.check_other_types(base64.z85decode, b'CxXl-AcVLsz/dgCA+t', + b'www.python.org') + def test_a85_padding(self): eq = self.assertEqual @@ -689,6 +759,8 @@ def test_a85decode_errors(self): self.assertRaises(ValueError, base64.a85decode, b's8W', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-"', adobe=False) + self.assertRaises(ValueError, base64.a85decode, b'aaaay', + foldspaces=True) def test_b85decode_errors(self): illegal = list(range(33)) + \ @@ -704,6 +776,21 @@ def test_b85decode_errors(self): self.assertRaises(ValueError, base64.b85decode, b'|NsC') self.assertRaises(ValueError, base64.b85decode, b'|NsC1') + def test_z85decode_errors(self): + illegal = list(range(33)) + \ + list(b'"\',;_`|\\~') + \ + list(range(128, 256)) + for c in illegal: + with self.assertRaises(ValueError, msg=bytes([c])): + base64.z85decode(b'0000' + bytes([c])) + + # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: + self.assertRaises(ValueError, base64.z85decode, b'%') + self.assertRaises(ValueError, base64.z85decode, b'%n') + self.assertRaises(ValueError, base64.z85decode, b'%nS') + self.assertRaises(ValueError, base64.z85decode, b'%nSc') + self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -711,7 +798,8 @@ def test_decode_nonascii_str(self): base64.b32decode, base64.b16decode, base64.b85decode, - base64.a85decode) + base64.a85decode, + base64.z85decode) for f in decode_funcs: self.assertRaises(ValueError, f, 'with non-ascii \xcb') diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 4c3cf0b964ae56..e599b02c17d9c0 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -78,6 +78,9 @@ def test_inheritance(self): last_depth = depth finally: inheritance_tree.close() + + # Underscore-prefixed (private) exceptions don't need to be documented + exc_set = set(e for e in exc_set if not e.startswith('_')) self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") @@ -129,7 +132,7 @@ class Value(str): d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now - # Exception.__setstate__ should aquire a strong reference of key and + # Exception.__setstate__ should acquire a strong reference of key and # value in the dict. Otherwise, Value()'s refcount would go below # zero in the tp_hash call in PyObject_SetAttr(), and it would cause # crash in GC. diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 568c88e326c087..10c58c04dfd25e 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -610,6 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -1033,8 +1046,9 @@ def main(): ('return', 1, ''), ('quit', ), ] import test_module_for_bdb + ns = {'test_module_for_bdb': test_module_for_bdb} with TracerRun(self) as tracer: - tracer.runeval('test_module_for_bdb.main()', globals(), locals()) + tracer.runeval('test_module_for_bdb.main()', ns, ns) class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index ef744f6b97259c..1f3b6746ce4a62 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -139,13 +139,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -157,8 +165,15 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') def test_base64errors(self): @@ -474,6 +489,12 @@ def test_base64_roundtrip(self, binary, newline): restored = binascii.a2b_base64(self.type2test(converted)) self.assertConversion(binary, converted, restored, newline=newline) + def test_c_contiguity(self): + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + binascii.b2a_hex(noncontig_writable) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 72a06d6af450e3..ae938d12c9401b 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -130,10 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) -# Format codes suppported by the memoryview object +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() -# Format codes suppported by array.array +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -168,7 +168,7 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) @@ -1918,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1950,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1996,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) @@ -4585,6 +4585,33 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + def test_inheritance(self): class A(bytearray): def __buffer__(self, flags): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index b7966f8f03875b..5818e96d61f480 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -46,6 +46,8 @@ x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +# used as proof of globals being used +A_GLOBAL_VALUE = 123 class Squares: @@ -308,14 +310,13 @@ class C3(C2): pass self.assertTrue(callable(c3)) def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -327,7 +328,11 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): self.assertTrue(not hasattr(builtins, "cmp")) @@ -611,6 +616,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): @@ -627,6 +640,11 @@ def __dir__(self): # test that object has a __dir__() self.assertEqual(sorted([].__dir__()), dir([])) + def test___ne__(self): + self.assertFalse(None.__ne__(None)) + self.assertIs(None.__ne__(0), NotImplemented) + self.assertIs(None.__ne__("abc"), NotImplemented) + def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) self.assertEqual(divmod(-12, 7), (-2, 2)) @@ -644,6 +662,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -668,6 +696,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -761,6 +794,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -832,6 +878,32 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -1696,6 +1768,11 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "sum accuracy not guaranteed on machines with double rounding") @@ -1703,6 +1780,10 @@ def __getitem__(self, index): def test_sum_accuracy(self): self.assertEqual(sum([0.1] * 10), 1.0) self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) def test_type(self): self.assertEqual(type(''), type('123')) @@ -2039,6 +2120,23 @@ def test_bytearray_extend_error(self): bad_iter = map(int, "X") self.assertRaises(ValueError, array.extend, bad_iter) + def test_bytearray_join_with_misbehaving_iterator(self): + # Issue #112625 + array = bytearray(b',') + def iterator(): + array.clear() + yield b'A' + yield b'B' + self.assertRaises(BufferError, array.join, iterator()) + + def test_bytearray_join_with_custom_iterator(self): + # Issue #112625 + array = bytearray(b',') + def iterator(): + yield b'A' + yield b'B' + self.assertEqual(bytearray(b'A,B'), array.join(iterator())) + def test_construct_singletons(self): for const in None, Ellipsis, NotImplemented: tp = type(const) @@ -2046,19 +2144,36 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): + not NotImplemented + + def test_singleton_attribute_access(self): + for singleton in (NotImplemented, Ellipsis): + with self.subTest(singleton): + self.assertIs(type(singleton), singleton.__class__) + self.assertIs(type(singleton).__class__, type) + + # Missing instance attributes: + with self.assertRaises(AttributeError): + singleton.prop = 1 + with self.assertRaises(AttributeError): + singleton.prop + + # Missing class attributes: + with self.assertRaises(TypeError): + type(singleton).prop = 1 + with self.assertRaises(AttributeError): + type(singleton).prop class TestBreakpoint(unittest.TestCase): @@ -2252,7 +2367,10 @@ def _run_child(self, child, terminal_input): return lines - def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): + def check_input_tty(self, prompt, terminal_input, stdio_encoding=None, *, + expected=None, + stdin_errors='surrogateescape', + stdout_errors='replace'): if not sys.stdin.isatty() or not sys.stdout.isatty(): self.skipTest("stdin and stdout must be ttys") def child(wpipe): @@ -2260,22 +2378,26 @@ def child(wpipe): if stdio_encoding: sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding=stdio_encoding, - errors='surrogateescape') + errors=stdin_errors) sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding=stdio_encoding, - errors='replace') + errors=stdout_errors) print("tty =", sys.stdin.isatty() and sys.stdout.isatty(), file=wpipe) - print(ascii(input(prompt)), file=wpipe) + try: + print(ascii(input(prompt)), file=wpipe) + except BaseException as e: + print(ascii(f'{e.__class__.__name__}: {e!s}'), file=wpipe) lines = self.run_child(child, terminal_input + b"\r\n") # Check we did exercise the GNU readline path self.assertIn(lines[0], {'tty = True', 'tty = False'}) if lines[0] != 'tty = True': self.skipTest("standard IO in should have been a tty") input_result = eval(lines[1]) # ascii() -> eval() roundtrip - if stdio_encoding: - expected = terminal_input.decode(stdio_encoding, 'surrogateescape') - else: - expected = terminal_input.decode(sys.stdin.encoding) # what else? + if expected is None: + if stdio_encoding: + expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + else: + expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) def test_input_tty(self): @@ -2296,13 +2418,32 @@ def skip_if_readline(self): def test_input_tty_non_ascii(self): self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") def test_input_tty_non_ascii_unicode_errors(self): self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") + def test_input_tty_null_in_prompt(self): + self.check_input_tty("prompt\0", b"", + expected='ValueError: input: prompt string cannot contain ' + 'null characters') + + def test_input_tty_nonencodable_prompt(self): + self.skip_if_readline() + self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', + expected="UnicodeEncodeError: 'ascii' codec can't encode " + "character '\\xe9' in position 6: ordinal not in " + "range(128)") + + def test_input_tty_nondecodable_input(self): + self.skip_if_readline() + self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', + expected="UnicodeDecodeError: 'ascii' codec can't decode " + "byte 0xe9 in position 4: ordinal not in " + "range(128)") + def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index afd506f07520d8..9e1985bb3a7639 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -46,6 +46,10 @@ def __index__(self): class BaseBytesTest: + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_basics(self): b = self.type2test() self.assertEqual(type(b), self.type2test) @@ -987,13 +991,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1023,36 +1027,63 @@ def test_buffer_is_readonly(self): self.assertRaises(TypeError, f.readinto, b"") def test_custom(self): - class A: - def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A()), b'abc') - class A: pass - self.assertRaises(TypeError, bytes, A()) - class A: - def __bytes__(self): - return None - self.assertRaises(TypeError, bytes, A()) - class A: + self.assertEqual(bytes(BytesSubclass(b'abc')), b'abc') + self.assertEqual(BytesSubclass(OtherBytesSubclass(b'abc')), + BytesSubclass(b'abc')) + self.assertEqual(bytes(WithBytes(b'abc')), b'abc') + self.assertEqual(BytesSubclass(WithBytes(b'abc')), BytesSubclass(b'abc')) + + class NoBytes: pass + self.assertRaises(TypeError, bytes, NoBytes()) + self.assertRaises(TypeError, bytes, WithBytes('abc')) + self.assertRaises(TypeError, bytes, WithBytes(None)) + class IndexWithBytes: def __bytes__(self): return b'a' def __index__(self): return 42 - self.assertEqual(bytes(A()), b'a') + self.assertEqual(bytes(IndexWithBytes()), b'a') # Issue #25766 - class A(str): + class StrWithBytes(str): + def __new__(cls, value): + self = str.__new__(cls, '\u20ac') + self.value = value + return self def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A('\u20ac')), b'abc') - self.assertEqual(bytes(A('\u20ac'), 'iso8859-15'), b'\xa4') + return self.value + self.assertEqual(bytes(StrWithBytes(b'abc')), b'abc') + self.assertEqual(bytes(StrWithBytes(b'abc'), 'iso8859-15'), b'\xa4') + self.assertEqual(bytes(StrWithBytes(BytesSubclass(b'abc'))), b'abc') + self.assertEqual(BytesSubclass(StrWithBytes(b'abc')), BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(b'abc'), 'iso8859-15'), + BytesSubclass(b'\xa4')) + self.assertEqual(BytesSubclass(StrWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Issue #24731 - class A: + self.assertTypedEqual(bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + + class BytesWithBytes(bytes): + def __new__(cls, value): + self = bytes.__new__(cls, b'\xa4') + self.value = value + return self def __bytes__(self): - return OtherBytesSubclass(b'abc') - self.assertEqual(bytes(A()), b'abc') - self.assertIs(type(bytes(A())), OtherBytesSubclass) - self.assertEqual(BytesSubclass(A()), b'abc') - self.assertIs(type(BytesSubclass(A())), BytesSubclass) + return self.value + self.assertTypedEqual(bytes(BytesWithBytes(b'abc')), b'abc') + self.assertTypedEqual(BytesSubclass(BytesWithBytes(b'abc')), + BytesSubclass(b'abc')) + self.assertTypedEqual(bytes(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Test PyBytes_FromFormat() def test_from_format(self): @@ -1225,7 +1256,7 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') def test_getitem_error(self): b = bytearray(b'python') @@ -1323,7 +1354,7 @@ def setitem_as_mapping(b, i, val): b[i] = val def setitem_as_sequence(b, i, val): - self._testcapi.sequence_setitem(b, i, val) + self._testlimitedcapi.sequence_setitem(b, i, val) def do_tests(setitem): b = bytearray([1, 2, 3]) @@ -1370,7 +1401,7 @@ def del_as_mapping(b, i): del b[i] def del_as_sequence(b, i): - self._testcapi.sequence_delitem(b, i) + self._testlimitedcapi.sequence_delitem(b, i) def do_tests(delete): b = bytearray(range(10)) @@ -1568,6 +1599,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') @@ -1772,7 +1810,7 @@ def __index__(self): with self.subTest("tp_as_sequence"): b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - self._testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) class AssortedBytesTest(unittest.TestCase): @@ -2069,6 +2107,12 @@ class BytesSubclass(bytes): class OtherBytesSubclass(bytes): pass +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class ByteArraySubclassTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclass diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc3698b4..e4d1381be5f340 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -537,26 +537,208 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" @@ -577,6 +759,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -592,6 +777,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 71f934756e26a1..e4b0b8c451fd45 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -26,17 +26,16 @@ TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform.startswith("linux"): + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" @@ -112,12 +111,16 @@ class EncodingDetails(_EncodingDetails): ]) @classmethod - def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars): + def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars): """Returns expected child process details for a given encoding""" _stream = stream_encoding + ":{}" - # stdin and stdout should use surrogateescape either because the - # coercion triggered, or because the C locale was detected - stream_info = 2*[_stream.format("surrogateescape")] + if stream_errors is None: + # stdin and stdout should use surrogateescape either because the + # coercion triggered, or because the C locale was detected + stream_errors = "surrogateescape" + + stream_info = [_stream.format(stream_errors)] * 2 + # stderr should always use backslashreplace stream_info.append(_stream.format("backslashreplace")) expected_lang = env_vars.get("LANG", "not set") @@ -210,6 +213,7 @@ def _check_child_encoding_details(self, env_vars, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, expected_warnings, coercion_expected): """Check the C locale handling for the given process environment @@ -225,6 +229,7 @@ def _check_child_encoding_details(self, coercion_expected, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, env_vars ) self.assertEqual(encoding_details, expected_details) @@ -257,6 +262,7 @@ def test_external_target_locale_configuration(self): "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } for env_var in ("LANG", "LC_CTYPE"): for locale_to_set in AVAILABLE_TARGETS: @@ -273,10 +279,43 @@ def test_external_target_locale_configuration(self): self._check_child_encoding_details(var_dict, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors=None, expected_warnings=None, coercion_expected=False) + def test_with_ioencoding(self): + # Explicitly setting a target locale should give the same behaviour as + # is seen when implicitly coercing to that target locale + self.maxDiff = None + expected_fs_encoding = "utf-8" + expected_stream_encoding = "utf-8" + + base_var_dict = { + "LANG": "", + "LC_CTYPE": "", + "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "UTF-8", + } + for env_var in ("LANG", "LC_CTYPE"): + for locale_to_set in AVAILABLE_TARGETS: + # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as + # expected, so skip that combination for now + # See https://bugs.python.org/issue30672 for discussion + if env_var == "LANG" and locale_to_set == "UTF-8": + continue + + with self.subTest(env_var=env_var, + configured_locale=locale_to_set): + var_dict = base_var_dict.copy() + var_dict[env_var] = locale_to_set + self._check_child_encoding_details(var_dict, + expected_fs_encoding, + expected_stream_encoding, + expected_stream_errors="strict", + expected_warnings=None, + coercion_expected=False) @support.cpython_only @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"), @@ -316,6 +355,7 @@ def _check_c_locale_coercion(self, "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } base_var_dict.update(extra_vars) if coerce_c_locale is not None: @@ -340,6 +380,7 @@ def _check_c_locale_coercion(self, self._check_child_encoding_details(base_var_dict, fs_encoding, stream_encoding, + None, _expected_warnings, _coercion_expected) @@ -348,13 +389,15 @@ def _check_c_locale_coercion(self, for env_var in ("LANG", "LC_CTYPE"): with self.subTest(env_var=env_var, nominal_locale=locale_to_set, - PYTHONCOERCECLOCALE=coerce_c_locale): + PYTHONCOERCECLOCALE=coerce_c_locale, + PYTHONIOENCODING=""): var_dict = base_var_dict.copy() var_dict[env_var] = locale_to_set # Check behaviour on successful coercion self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, + None, expected_warnings, coercion_expected) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index b1c78d7136fc9b..504f8800a00aa5 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,15 +1,19 @@ import unittest -from test.support import cpython_only, requires_limited_api, skip_on_s390x +from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, + set_recursion_limit, skip_on_s390x) try: import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc import contextlib -import sys import types @@ -41,11 +45,16 @@ def test_frames_are_popped_after_failed_calls(self): # recovering from failed calls: def f(): pass - for _ in range(1000): - try: - f(None) - except TypeError: + class C: + def m(self): pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass # BOOM! @@ -154,7 +163,7 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"'foo' is an invalid keyword argument for print\(\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) @@ -239,6 +248,7 @@ def test_module_not_callable_suggestion(self): self.assertRaisesRegex(TypeError, msg, mod) +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestCallingConventions(unittest.TestCase): """Test calling using various C calling conventions (METH_*) from Python @@ -436,6 +446,7 @@ def static_method(): NULL_OR_EMPTY = object() + class FastCallTests(unittest.TestCase): """Test calling using various callables from C """ @@ -479,42 +490,43 @@ class FastCallTests(unittest.TestCase): ] # Add all the calling conventions and variants of C callables - _instance = _testcapi.MethInstance() - for obj, expected_self in ( - (_testcapi, _testcapi), # module-level function - (_instance, _instance), # bound method - (_testcapi.MethClass, _testcapi.MethClass), # class method on class - (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. - (_testcapi.MethStatic, None), # static method - ): - CALLS_POSARGS.extend([ - (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), - (obj.meth_varargs_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), - (obj.meth_fastcall, (), (expected_self, ())), - (obj.meth_fastcall_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (), (expected_self, (), NULL_OR_EMPTY)), - (obj.meth_noargs, (), expected_self), - (obj.meth_o, (123, ), (expected_self, 123)), - ]) - - CALLS_KWARGS.extend([ - (obj.meth_varargs_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_varargs_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_varargs_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - ]) + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: @@ -522,6 +534,7 @@ def check_result(self, result, expected): expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): # Test PyObject_VectorcallDict() @@ -541,6 +554,7 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): # Test PyObject_Vectorcall() @@ -605,6 +619,7 @@ def testfunction_kw(self, *, kw): ADAPTIVE_WARMUP_DELAY = 2 +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): def test_method_descriptor_flag(self): @@ -836,12 +851,12 @@ def get_a(x): @requires_limited_api def test_vectorcall_limited_incoming(self): from _testcapi import pyobject_vectorcall - obj = _testcapi.LimitedVectorCallClass() + obj = _testlimitedcapi.LimitedVectorCallClass() self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") @requires_limited_api def test_vectorcall_limited_outgoing(self): - from _testcapi import call_vectorcall + from _testlimitedcapi import call_vectorcall args_captured = [] kwargs_captured = [] @@ -857,7 +872,7 @@ def f(*args, **kwargs): @requires_limited_api def test_vectorcall_limited_outgoing_method(self): - from _testcapi import call_vectorcall_method + from _testlimitedcapi import call_vectorcall_method args_captured = [] kwargs_captured = [] @@ -927,7 +942,7 @@ def check_suggestion_includes(self, message): self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) @contextlib.contextmanager - def check_suggestion_not_pressent(self): + def check_suggestion_not_present(self): with self.assertRaises(TypeError) as cm: yield self.assertNotIn("Did you mean", str(cm.exception)) @@ -945,7 +960,7 @@ def foo(blech=None, /, aaa=None, *args, late1=None): for keyword, suggestion in cases: with self.subTest(keyword): - ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() with ctx: foo(**{keyword:None}) @@ -986,10 +1001,38 @@ def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): with self.check_suggestion_includes(suggestion): func(bluch=None) + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + @cpython_only class TestRecursion(unittest.TestCase): @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_super_deep(self): def recurse(n): @@ -1010,9 +1053,7 @@ def c_py_recurse(m): if m: _testcapi.pyobject_vectorcall(py_recurse, (1000, m), ()) - depth = sys.getrecursionlimit() - sys.setrecursionlimit(100_000) - try: + with set_recursion_limit(100_000): recurse(90_000) with self.assertRaises(RecursionError): recurse(101_000) @@ -1022,8 +1063,6 @@ def c_py_recurse(m): c_py_recurse(90) with self.assertRaises(RecursionError): c_py_recurse(100_000) - finally: - sys.setrecursionlimit(depth) class TestFunctionWithManyArgs(unittest.TestCase): diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py index 4b16ecc31156a5..5a8ba6845399e0 100644 --- a/Lib/test/test_capi/__init__.py +++ b/Lib/test/test_capi/__init__.py @@ -1,5 +1,12 @@ import os +import unittest from test.support import load_package_tests +from test.support import TEST_MODULES_ENABLED + + +if not TEST_MODULES_ENABLED: + raise unittest.SkipTest("requires test modules") + def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py index eb99ae16f2b69e..bc4f3a0beb9c1e 100644 --- a/Lib/test/test_capi/check_config.py +++ b/Lib/test/test_capi/check_config.py @@ -10,7 +10,7 @@ def import_singlephase(): assert '_testsinglephase' not in sys.modules try: - import _testsinglephase + import _testsinglephase # noqa: F401 except ImportError: sys.modules.pop('_testsinglephase', None) return False diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index eeaef60a8b47b5..bc39036e90bf8b 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -1,10 +1,38 @@ import unittest from collections import OrderedDict -import _testcapi +from test import support +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None +class StrSubclass(str): + pass + +class BytesSubclass(bytes): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class TestObject: @property def evil(self): @@ -41,9 +69,71 @@ def gen(): class CAPITest(unittest.TestCase): + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_object_str(self): + # Test PyObject_Str() + object_str = _testlimitedcapi.object_str + self.assertTypedEqual(object_str(''), '') + self.assertTypedEqual(object_str('abc'), 'abc') + self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') + self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(object_str(WithRepr('')), '') + self.assertTypedEqual(object_str(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_str(NULL), '') + + def test_object_repr(self): + # Test PyObject_Repr() + object_repr = _testlimitedcapi.object_repr + self.assertTypedEqual(object_repr(''), "''") + self.assertTypedEqual(object_repr('abc'), "'abc'") + self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_repr(WithRepr('')), '') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertTypedEqual(object_repr(NULL), '') + + def test_object_ascii(self): + # Test PyObject_ASCII() + object_ascii = _testlimitedcapi.object_ascii + self.assertTypedEqual(object_ascii(''), "''") + self.assertTypedEqual(object_ascii('abc'), "'abc'") + self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_ascii(WithRepr('')), '') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(NULL), '') + + def test_object_bytes(self): + # Test PyObject_Bytes() + object_bytes = _testlimitedcapi.object_bytes + self.assertTypedEqual(object_bytes(b''), b'') + self.assertTypedEqual(object_bytes(b'abc'), b'abc') + self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') + self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') + self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') + self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') + self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) + self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) + self.assertRaises(TypeError, object_bytes, 3) + self.assertRaises(TypeError, object_bytes, 'abc') + self.assertRaises(TypeError, object_bytes, object()) + self.assertTypedEqual(object_bytes(NULL), b'') def test_object_getattr(self): - xgetattr = _testcapi.object_getattr + xgetattr = _testlimitedcapi.object_getattr obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -57,7 +147,7 @@ def test_object_getattr(self): # CRASHES xgetattr(NULL, 'a') def test_object_getattrstring(self): - getattrstring = _testcapi.object_getattrstring + getattrstring = _testlimitedcapi.object_getattrstring obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -99,7 +189,7 @@ def test_object_getoptionalattrstring(self): # CRASHES getoptionalattrstring(NULL, b'a') def test_object_hasattr(self): - xhasattr = _testcapi.object_hasattr + xhasattr = _testlimitedcapi.object_hasattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -107,13 +197,23 @@ def test_object_hasattr(self): self.assertFalse(xhasattr(obj, 'b')) self.assertTrue(xhasattr(obj, '\U0001f40d')) - self.assertFalse(xhasattr(obj, 'evil')) - self.assertFalse(xhasattr(obj, 1)) + with support.catch_unraisable_exception() as cm: + self.assertFalse(xhasattr(obj, 'evil')) + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'do not get evil') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(xhasattr(obj, 1)) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "attribute name must be string, not 'int'") + # CRASHES xhasattr(obj, NULL) # CRASHES xhasattr(NULL, 'a') def test_object_hasattrstring(self): - hasattrstring = _testcapi.object_hasattrstring + hasattrstring = _testlimitedcapi.object_hasattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -121,8 +221,18 @@ def test_object_hasattrstring(self): self.assertFalse(hasattrstring(obj, b'b')) self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) - self.assertFalse(hasattrstring(obj, b'evil')) - self.assertFalse(hasattrstring(obj, b'\xff')) + with support.catch_unraisable_exception() as cm: + self.assertFalse(hasattrstring(obj, b'evil')) + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'do not get evil') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(hasattrstring(obj, b'\xff')) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + # CRASHES hasattrstring(obj, NULL) # CRASHES hasattrstring(NULL, b'a') @@ -155,7 +265,7 @@ def test_object_hasattrstringwitherror(self): # CRASHES hasattrstring(NULL, b'a') def test_object_setattr(self): - xsetattr = _testcapi.object_setattr + xsetattr = _testlimitedcapi.object_setattr obj = TestObject() xsetattr(obj, 'a', 5) self.assertEqual(obj.a, 5) @@ -175,7 +285,7 @@ def test_object_setattr(self): # CRASHES xsetattr(NULL, 'a', 5) def test_object_setattrstring(self): - setattrstring = _testcapi.object_setattrstring + setattrstring = _testlimitedcapi.object_setattrstring obj = TestObject() setattrstring(obj, b'a', 5) self.assertEqual(obj.a, 5) @@ -196,7 +306,7 @@ def test_object_setattrstring(self): # CRASHES setattrstring(NULL, b'a', 5) def test_object_delattr(self): - xdelattr = _testcapi.object_delattr + xdelattr = _testlimitedcapi.object_delattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -213,7 +323,7 @@ def test_object_delattr(self): # CRASHES xdelattr(NULL, 'a') def test_object_delattrstring(self): - delattrstring = _testcapi.object_delattrstring + delattrstring = _testlimitedcapi.object_delattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -231,7 +341,7 @@ def test_object_delattrstring(self): def test_mapping_check(self): - check = _testcapi.mapping_check + check = _testlimitedcapi.mapping_check self.assertTrue(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -242,7 +352,7 @@ def test_mapping_check(self): self.assertFalse(check(NULL)) def test_mapping_size(self): - for size in _testcapi.mapping_size, _testcapi.mapping_length: + for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: self.assertEqual(size({1: 2}), 1) self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) @@ -254,7 +364,7 @@ def test_mapping_size(self): self.assertRaises(SystemError, size, NULL) def test_object_getitem(self): - getitem = _testcapi.object_getitem + getitem = _testlimitedcapi.object_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertRaises(KeyError, getitem, dct, 'b') @@ -274,7 +384,7 @@ def test_object_getitem(self): self.assertRaises(SystemError, getitem, NULL, 'a') def test_mapping_getitemstring(self): - getitemstring = _testcapi.mapping_getitemstring + getitemstring = _testlimitedcapi.mapping_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertRaises(KeyError, getitemstring, dct, b'b') @@ -328,7 +438,7 @@ def test_mapping_getoptionalitemstring(self): # CRASHES getitemstring(NULL, b'a') def test_mapping_haskey(self): - haskey = _testcapi.mapping_haskey + haskey = _testlimitedcapi.mapping_haskey dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -340,15 +450,44 @@ def test_mapping_haskey(self): self.assertTrue(haskey(['a', 'b', 'c'], 1)) - self.assertFalse(haskey(42, 'a')) - self.assertFalse(haskey({}, [])) # unhashable - self.assertFalse(haskey({}, NULL)) - self.assertFalse(haskey([], 1)) - self.assertFalse(haskey([], 'a')) - self.assertFalse(haskey(NULL, 'a')) + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey(42, 'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "'int' object is not subscriptable") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey({}, [])) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "unhashable type: 'list'") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey([], 1)) + self.assertEqual(cm.unraisable.exc_type, IndexError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list index out of range') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey([], 'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list indices must be integers or slices, not str') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey({}, NULL)) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + 'null argument to internal routine') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey(NULL, 'a')) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + 'null argument to internal routine') def test_mapping_haskeystring(self): - haskeystring = _testcapi.mapping_haskeystring + haskeystring = _testlimitedcapi.mapping_haskeystring dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -358,14 +497,38 @@ def test_mapping_haskeystring(self): self.assertTrue(haskeystring(dct2, b'a')) self.assertFalse(haskeystring(dct2, b'b')) - self.assertFalse(haskeystring(42, b'a')) - self.assertFalse(haskeystring({}, b'\xff')) - self.assertFalse(haskeystring({}, NULL)) - self.assertFalse(haskeystring([], b'a')) - self.assertFalse(haskeystring(NULL, b'a')) + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring(42, b'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "'int' object is not subscriptable") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring({}, b'\xff')) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring({}, NULL)) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + "null argument to internal routine") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring([], b'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list indices must be integers or slices, not str') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring(NULL, b'a')) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + "null argument to internal routine") def test_mapping_haskeywitherror(self): - haskey = _testcapi.mapping_haskeywitherror + haskey = _testlimitedcapi.mapping_haskeywitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -386,7 +549,7 @@ def test_mapping_haskeywitherror(self): # CRASHES haskey(NULL, 'a')) def test_mapping_haskeystringwitherror(self): - haskeystring = _testcapi.mapping_haskeystringwitherror + haskeystring = _testlimitedcapi.mapping_haskeystringwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -403,7 +566,7 @@ def test_mapping_haskeystringwitherror(self): # CRASHES haskeystring(NULL, b'a') def test_object_setitem(self): - setitem = _testcapi.object_setitem + setitem = _testlimitedcapi.object_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -429,7 +592,7 @@ def test_object_setitem(self): self.assertRaises(SystemError, setitem, NULL, 'a', 5) def test_mapping_setitemstring(self): - setitemstring = _testcapi.mapping_setitemstring + setitemstring = _testlimitedcapi.mapping_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -449,7 +612,7 @@ def test_mapping_setitemstring(self): self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) def test_object_delitem(self): - for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -475,7 +638,7 @@ def test_object_delitem(self): self.assertRaises(SystemError, delitem, NULL, 'a') def test_mapping_delitemstring(self): - delitemstring = _testcapi.mapping_delitemstring + delitemstring = _testlimitedcapi.mapping_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -515,23 +678,23 @@ def items(self): for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), dict_obj, OrderedDict(dict_obj), Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.mapping_keys(mapping), + self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), list(mapping.keys())) - self.assertListEqual(_testcapi.mapping_values(mapping), + self.assertListEqual(_testlimitedcapi.mapping_values(mapping), list(mapping.values())) - self.assertListEqual(_testcapi.mapping_items(mapping), + self.assertListEqual(_testlimitedcapi.mapping_items(mapping), list(mapping.items())) def test_mapping_keys_valuesitems_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) - self.assertRaises(AttributeError, _testcapi.mapping_values, object()) - self.assertRaises(AttributeError, _testcapi.mapping_items, object()) - self.assertRaises(AttributeError, _testcapi.mapping_keys, []) - self.assertRaises(AttributeError, _testcapi.mapping_values, []) - self.assertRaises(AttributeError, _testcapi.mapping_items, []) - self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) - self.assertRaises(SystemError, _testcapi.mapping_values, NULL) - self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) + self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) class BadMapping: def keys(self): @@ -541,12 +704,12 @@ def values(self): def items(self): return None bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) def test_sequence_check(self): - check = _testcapi.sequence_check + check = _testlimitedcapi.sequence_check self.assertFalse(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -557,7 +720,7 @@ def test_sequence_check(self): # CRASHES check(NULL) def test_sequence_size(self): - for size in _testcapi.sequence_size, _testcapi.sequence_length: + for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) self.assertEqual(size('abc'), 3) @@ -569,18 +732,20 @@ def test_sequence_size(self): self.assertRaises(SystemError, size, NULL) def test_sequence_getitem(self): - getitem = _testcapi.sequence_getitem + getitem = _testlimitedcapi.sequence_getitem lst = ['a', 'b', 'c'] self.assertEqual(getitem(lst, 1), 'b') self.assertEqual(getitem(lst, -1), 'c') self.assertRaises(IndexError, getitem, lst, 3) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) self.assertRaises(TypeError, getitem, 42, 1) self.assertRaises(TypeError, getitem, {}, 1) self.assertRaises(SystemError, getitem, NULL, 1) def test_sequence_concat(self): - concat = _testcapi.sequence_concat + concat = _testlimitedcapi.sequence_concat self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) @@ -593,18 +758,21 @@ def test_sequence_concat(self): self.assertRaises(SystemError, concat, NULL, []) def test_sequence_repeat(self): - repeat = _testcapi.sequence_repeat + repeat = _testlimitedcapi.sequence_repeat self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) self.assertEqual(repeat(['a', 'b'], 0), []) self.assertEqual(repeat(['a', 'b'], -1), []) + self.assertEqual(repeat(['a', 'b'], PY_SSIZE_T_MIN), []) + self.assertEqual(repeat([], PY_SSIZE_T_MAX), []) + self.assertRaises(MemoryError, repeat, ['a', 'b'], PY_SSIZE_T_MAX) self.assertRaises(TypeError, repeat, set(), 2) self.assertRaises(TypeError, repeat, 42, 2) self.assertRaises(SystemError, repeat, NULL, 2) def test_sequence_inplaceconcat(self): - inplaceconcat = _testcapi.sequence_inplaceconcat + inplaceconcat = _testlimitedcapi.sequence_inplaceconcat lst = ['a', 'b'] res = inplaceconcat(lst, [1, 2]) self.assertEqual(res, ['a', 'b', 1, 2]) @@ -623,7 +791,7 @@ def test_sequence_inplaceconcat(self): self.assertRaises(SystemError, inplaceconcat, NULL, []) def test_sequence_inplacerepeat(self): - inplacerepeat = _testcapi.sequence_inplacerepeat + inplacerepeat = _testlimitedcapi.sequence_inplacerepeat lst = ['a', 'b'] res = inplacerepeat(lst, 2) self.assertEqual(res, ['a', 'b', 'a', 'b']) @@ -631,13 +799,16 @@ def test_sequence_inplacerepeat(self): self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) self.assertEqual(inplacerepeat(['a', 'b'], 0), []) self.assertEqual(inplacerepeat(['a', 'b'], -1), []) + self.assertEqual(inplacerepeat(['a', 'b'], PY_SSIZE_T_MIN), []) + self.assertEqual(inplacerepeat([], PY_SSIZE_T_MAX), []) + self.assertRaises(MemoryError, inplacerepeat, ['a', 'b'], PY_SSIZE_T_MAX) self.assertRaises(TypeError, inplacerepeat, set(), 2) self.assertRaises(TypeError, inplacerepeat, 42, 2) self.assertRaises(SystemError, inplacerepeat, NULL, 2) def test_sequence_setitem(self): - setitem = _testcapi.sequence_setitem + setitem = _testlimitedcapi.sequence_setitem lst = ['a', 'b', 'c'] setitem(lst, 1, 'x') self.assertEqual(lst, ['a', 'x', 'c']) @@ -647,35 +818,45 @@ def test_sequence_setitem(self): setitem(lst, 0, NULL) self.assertEqual(lst, ['x', 'y']) self.assertRaises(IndexError, setitem, lst, 3, 'x') + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 'x') + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 'x') self.assertRaises(TypeError, setitem, 42, 1, 'x') self.assertRaises(TypeError, setitem, {}, 1, 'x') self.assertRaises(SystemError, setitem, NULL, 1, 'x') def test_sequence_delitem(self): - delitem = _testcapi.sequence_delitem + delitem = _testlimitedcapi.sequence_delitem lst = ['a', 'b', 'c'] delitem(lst, 1) self.assertEqual(lst, ['a', 'c']) delitem(lst, -1) self.assertEqual(lst, ['a']) self.assertRaises(IndexError, delitem, lst, 3) + self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MIN) self.assertRaises(TypeError, delitem, 42, 1) self.assertRaises(TypeError, delitem, {}, 1) self.assertRaises(SystemError, delitem, NULL, 1) def test_sequence_setslice(self): - setslice = _testcapi.sequence_setslice + setslice = _testlimitedcapi.sequence_setslice # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - setslice(data, 1, 3, [8, 9]) - data_copy[1:3] = [8, 9] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 8, 9, 4, 5]) + for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + setslice(data, start, stop, [8, 9]) + data_copy[start:stop] = [8, 9] + self.assertEqual(data, data_copy) + + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + setslice(data, start, stop, NULL) + del data_copy[start:stop] + self.assertEqual(data, data_copy) # Custom class: class Custom: @@ -701,21 +882,17 @@ def __setitem__(self, index, value): self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') - data_copy = data.copy() - setslice(data_copy, 1, 3, NULL) - self.assertEqual(data_copy, [1, 4, 5]) - def test_sequence_delslice(self): - delslice = _testcapi.sequence_delslice + delslice = _testlimitedcapi.sequence_delslice # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - delslice(data, 1, 3) - del data_copy[1:3] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 4, 5]) + for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + delslice(data, start, stop) + del data_copy[start:stop] + self.assertEqual(data, data_copy) # Custom class: class Custom: @@ -744,7 +921,7 @@ def __delitem__(self, index): self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) def test_sequence_count(self): - count = _testcapi.sequence_count + count = _testlimitedcapi.sequence_count lst = ['a', 'b', 'a'] self.assertEqual(count(lst, 'a'), 2) @@ -759,7 +936,7 @@ def test_sequence_count(self): self.assertRaises(SystemError, count, NULL, 'a') def test_sequence_contains(self): - contains = _testcapi.sequence_contains + contains = _testlimitedcapi.sequence_contains lst = ['a', 'b', 'a'] self.assertEqual(contains(lst, 'a'), 1) @@ -778,7 +955,7 @@ def test_sequence_contains(self): # CRASHES contains(NULL, 'a') def test_sequence_index(self): - index = _testcapi.sequence_index + index = _testlimitedcapi.sequence_index lst = ['a', 'b', 'a'] self.assertEqual(index(lst, 'a'), 0) @@ -798,7 +975,7 @@ def test_sequence_index(self): self.assertRaises(SystemError, index, NULL, 'a') def test_sequence_list(self): - xlist = _testcapi.sequence_list + xlist = _testlimitedcapi.sequence_list self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) @@ -808,7 +985,7 @@ def test_sequence_list(self): self.assertRaises(SystemError, xlist, NULL) def test_sequence_tuple(self): - xtuple = _testcapi.sequence_tuple + xtuple = _testlimitedcapi.sequence_tuple self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) @@ -817,6 +994,19 @@ def test_sequence_tuple(self): self.assertRaises(TypeError, xtuple, 42) self.assertRaises(SystemError, xtuple, NULL) + def test_number_check(self): + number_check = _testlimitedcapi.number_check + self.assertTrue(number_check(1 + 1j)) + self.assertTrue(number_check(1)) + self.assertTrue(number_check(0.5)) + self.assertFalse(number_check("1 + 1j")) + + def test_object_generichash(self): + # Test PyObject_GenericHash() + generichash = _testcapi.object_generichash + for obj in object(), 1, 'string', []: + self.assertEqual(generichash(obj), object.__hash__(obj)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py new file mode 100644 index 00000000000000..39099f6b82240f --- /dev/null +++ b/Lib/test/test_capi/test_bytearray.py @@ -0,0 +1,164 @@ +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + +NULL = None + +class ByteArraySubclass(bytearray): + pass + +class BytesLike: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyByteArray_Check() + check = _testlimitedcapi.bytearray_check + self.assertTrue(check(bytearray(b'abc'))) + self.assertFalse(check(b'abc')) + self.assertTrue(check(ByteArraySubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyByteArray_CheckExact() + check = _testlimitedcapi.bytearray_checkexact + self.assertTrue(check(bytearray(b'abc'))) + self.assertFalse(check(b'abc')) + self.assertFalse(check(ByteArraySubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_fromstringandsize(self): + # Test PyByteArray_FromStringAndSize() + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) + self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) + self.assertEqual(fromstringandsize(b'abc\0def'), bytearray(b'abc\0def')) + self.assertEqual(fromstringandsize(b'', 0), bytearray()) + self.assertEqual(fromstringandsize(NULL, 0), bytearray()) + self.assertEqual(len(fromstringandsize(NULL, 3)), 3) + self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + + def test_fromobject(self): + # Test PyByteArray_FromObject() + fromobject = _testlimitedcapi.bytearray_fromobject + + self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) + self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) + self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) + self.assertEqual(fromobject([97, 98, 99]), bytearray(b'abc')) + self.assertEqual(fromobject(3), bytearray(b'\0\0\0')) + self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) + self.assertRaises(TypeError, fromobject, 'abc') + self.assertRaises(TypeError, fromobject, object()) + + # CRASHES fromobject(NULL) + + def test_size(self): + # Test PyByteArray_Size() + size = _testlimitedcapi.bytearray_size + + self.assertEqual(size(bytearray(b'abc')), 3) + self.assertEqual(size(ByteArraySubclass(b'abc')), 3) + + # CRASHES size(b'abc') + # CRASHES size(object()) + # CRASHES size(NULL) + + def test_asstring(self): + """Test PyByteArray_AsString()""" + asstring = _testlimitedcapi.bytearray_asstring + + self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') + self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') + self.assertEqual(asstring(bytearray(b'abc\0def'), 8), b'abc\0def\0') + + # CRASHES asstring(b'abc', 0) + # CRASHES asstring(object()', 0) + # CRASHES asstring(NULL, 0) + + def test_concat(self): + """Test PyByteArray_Concat()""" + concat = _testlimitedcapi.bytearray_concat + + ba = bytearray(b'abc') + self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) + self.assertEqual(ba, b'abc') + + self.assertEqual(concat(b'abc', b'def'), bytearray(b'abcdef')) + self.assertEqual(concat(b'a\0b', b'c\0d'), bytearray(b'a\0bc\0d')) + self.assertEqual(concat(bytearray(b'abc'), b'def'), bytearray(b'abcdef')) + self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) + self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) + self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) + self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), + bytearray(b'abcdef')) + self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), + bytearray(b'abcdef')) + + self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') + self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, 'abc', 'def') + self.assertRaises(TypeError, concat, [], b'def') + self.assertRaises(TypeError, concat, b'abc', []) + self.assertRaises(TypeError, concat, [], []) + + # CRASHES concat(NULL, bytearray(b'def')) + # CRASHES concat(bytearray(b'abc'), NULL) + # CRASHES concat(NULL, object()) + # CRASHES concat(object(), NULL) + + def test_resize(self): + """Test PyByteArray_Resize()""" + resize = _testlimitedcapi.bytearray_resize + + ba = bytearray(b'abcdef') + self.assertEqual(resize(ba, 3), 0) + self.assertEqual(ba, bytearray(b'abc')) + self.assertEqual(resize(ba, 10), 0) + self.assertEqual(len(ba), 10) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertEqual(resize(ba, 2**20), 0) + self.assertEqual(len(ba), 2**20) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + + ba = ByteArraySubclass(b'abcdef') + self.assertEqual(resize(ba, 3), 0) + self.assertEqual(ba, bytearray(b'abc')) + + self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) + self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) + + # CRASHES resize(bytearray(b'abc'), -1) + # CRASHES resize(b'abc', 0) + # CRASHES resize(object(), 0) + # CRASHES resize(NULL, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py new file mode 100644 index 00000000000000..f14d5545c829e5 --- /dev/null +++ b/Lib/test/test_capi/test_bytes.py @@ -0,0 +1,252 @@ +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + +NULL = None + +class BytesSubclass(bytes): + pass + +class BytesLike: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyBytes_Check() + check = _testlimitedcapi.bytes_check + self.assertTrue(check(b'abc')) + self.assertFalse(check('abc')) + self.assertFalse(check(bytearray(b'abc'))) + self.assertTrue(check(BytesSubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyBytes_CheckExact() + check = _testlimitedcapi.bytes_checkexact + self.assertTrue(check(b'abc')) + self.assertFalse(check('abc')) + self.assertFalse(check(bytearray(b'abc'))) + self.assertFalse(check(BytesSubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_fromstringandsize(self): + # Test PyBytes_FromStringAndSize() + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), b'abc') + self.assertEqual(fromstringandsize(b'abc', 2), b'ab') + self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') + self.assertEqual(fromstringandsize(b'', 0), b'') + self.assertEqual(fromstringandsize(NULL, 0), b'') + self.assertEqual(len(fromstringandsize(NULL, 3)), 3) + self.assertRaises((MemoryError, OverflowError), + fromstringandsize, NULL, PY_SSIZE_T_MAX) + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + + def test_fromstring(self): + # Test PyBytes_FromString() + fromstring = _testlimitedcapi.bytes_fromstring + + self.assertEqual(fromstring(b'abc\0def'), b'abc') + self.assertEqual(fromstring(b''), b'') + + # CRASHES fromstring(NULL) + + def test_fromobject(self): + # Test PyBytes_FromObject() + fromobject = _testlimitedcapi.bytes_fromobject + + self.assertEqual(fromobject(b'abc'), b'abc') + self.assertEqual(fromobject(bytearray(b'abc')), b'abc') + self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') + self.assertEqual(fromobject([97, 98, 99]), b'abc') + self.assertRaises(TypeError, fromobject, 3) + self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) + self.assertRaises(TypeError, fromobject, 'abc') + self.assertRaises(TypeError, fromobject, object()) + self.assertRaises(SystemError, fromobject, NULL) + + def test_size(self): + # Test PyBytes_Size() + size = _testlimitedcapi.bytes_size + + self.assertEqual(size(b'abc'), 3) + self.assertEqual(size(BytesSubclass(b'abc')), 3) + self.assertRaises(TypeError, size, bytearray(b'abc')) + self.assertRaises(TypeError, size, 'abc') + self.assertRaises(TypeError, size, object()) + + # CRASHES size(NULL) + + def test_asstring(self): + """Test PyBytes_AsString()""" + asstring = _testlimitedcapi.bytes_asstring + + self.assertEqual(asstring(b'abc', 4), b'abc\0') + self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') + self.assertRaises(TypeError, asstring, 'abc', 0) + self.assertRaises(TypeError, asstring, object(), 0) + + # CRASHES asstring(NULL, 0) + + def test_asstringandsize(self): + """Test PyBytes_AsStringAndSize()""" + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null + + self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) + self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') + self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) + self.assertRaises(TypeError, asstringandsize, 'abc', 0) + self.assertRaises(TypeError, asstringandsize_null, 'abc', 0) + self.assertRaises(TypeError, asstringandsize, object(), 0) + self.assertRaises(TypeError, asstringandsize_null, object(), 0) + + # CRASHES asstringandsize(NULL, 0) + # CRASHES asstringandsize_null(NULL, 0) + + def test_repr(self): + # Test PyBytes_Repr() + bytes_repr = _testlimitedcapi.bytes_repr + + self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") + self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") + self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") + self.assertEqual(bytes_repr(b'''a'b"c"d''', 1), r"""b'a\'b"c"d'""") + self.assertEqual(bytes_repr(b'''a'b"c''', 0), r"""b'a\'b"c'""") + self.assertEqual(bytes_repr(b'''a'b"c''', 1), r"""b'a\'b"c'""") + self.assertEqual(bytes_repr(b'''a'b'c"d''', 0), r"""b'a\'b\'c"d'""") + self.assertEqual(bytes_repr(b'''a'b'c"d''', 1), r"""b'a\'b\'c"d'""") + self.assertEqual(bytes_repr(b'''a'b'c'd''', 0), r"""b'a\'b\'c\'d'""") + self.assertEqual(bytes_repr(b'''a'b'c'd''', 1), r'''b"a'b'c'd"''') + + self.assertEqual(bytes_repr(BytesSubclass(b'abc'), 0), r"""b'abc'""") + + # UDEFINED bytes_repr(object(), 0) + # CRASHES bytes_repr(NULL, 0) + + def test_concat(self, concat=None): + """Test PyBytes_Concat()""" + if concat is None: + concat = _testlimitedcapi.bytes_concat + + self.assertEqual(concat(b'abc', b'def'), b'abcdef') + self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') + self.assertEqual(concat(bytearray(b'abc'), b'def'), b'abcdef') + self.assertEqual(concat(b'abc', bytearray(b'def')), b'abcdef') + self.assertEqual(concat(bytearray(b'abc'), b''), b'abc') + self.assertEqual(concat(b'', bytearray(b'def')), b'def') + self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') + self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') + + self.assertEqual(concat(b'abc', b'def', True), b'abcdef') + self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') + # Check that it does not change the singleton + self.assertEqual(concat(bytes(), b'def', True), b'def') + self.assertEqual(len(bytes()), 0) + + self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') + self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, 'abc', 'def') + self.assertRaises(TypeError, concat, [], b'def') + self.assertRaises(TypeError, concat, b'abc', []) + self.assertRaises(TypeError, concat, [], []) + + self.assertEqual(concat(NULL, b'def'), NULL) + self.assertEqual(concat(b'abc', NULL), NULL) + self.assertEqual(concat(NULL, object()), NULL) + self.assertEqual(concat(object(), NULL), NULL) + + def test_concatanddel(self): + """Test PyBytes_ConcatAndDel()""" + self.test_concat(_testlimitedcapi.bytes_concatanddel) + + def test_decodeescape(self): + """Test PyBytes_DecodeEscape()""" + decodeescape = _testlimitedcapi.bytes_decodeescape + + self.assertEqual(decodeescape(b'abc'), b'abc') + self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), + b'''\t\n\r\v\f\0\\'"''') + self.assertEqual(decodeescape(b'\t\n\r\x0b\x0c\x00'), b'\t\n\r\v\f\0') + self.assertEqual(decodeescape(br'\xa1\xa2'), b'\xa1\xa2') + self.assertEqual(decodeescape(br'\2\24\241'), b'\x02\x14\xa1') + self.assertEqual(decodeescape(b'\xa1\xa2'), b'\xa1\xa2') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\u4f60'), br'\u4f60') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\z'), br'\z') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\541'), b'a') + + for b in b'\\', br'\x', br'\xa', br'\xz', br'\xaz': + self.assertRaises(ValueError, decodeescape, b) + self.assertRaises(ValueError, decodeescape, b, 'strict') + self.assertEqual(decodeescape(br'x\xa', 'replace'), b'x?') + self.assertEqual(decodeescape(br'x\xay', 'replace'), b'x?y') + self.assertEqual(decodeescape(br'x\xa\xy', 'replace'), b'x??y') + self.assertEqual(decodeescape(br'x\xa\xy', 'ignore'), b'xy') + self.assertRaises(ValueError, decodeescape, b'\\', 'spam') + self.assertEqual(decodeescape(NULL), b'') + self.assertRaises(OverflowError, decodeescape, b'abc', NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, decodeescape, NULL, NULL, PY_SSIZE_T_MAX) + + # CRASHES decodeescape(b'abc', NULL, -1) + # CRASHES decodeescape(NULL, NULL, 1) + + def test_resize(self): + """Test _PyBytes_Resize()""" + resize = _testcapi.bytes_resize + + for new in True, False: + self.assertEqual(resize(b'abc', 0, new), b'') + self.assertEqual(resize(b'abc', 1, new), b'a') + self.assertEqual(resize(b'abc', 2, new), b'ab') + self.assertEqual(resize(b'abc', 3, new), b'abc') + b = resize(b'abc', 4, new) + self.assertEqual(len(b), 4) + self.assertEqual(b[:3], b'abc') + + self.assertEqual(resize(b'a', 0, new), b'') + self.assertEqual(resize(b'a', 1, new), b'a') + b = resize(b'a', 2, new) + self.assertEqual(len(b), 2) + self.assertEqual(b[:1], b'a') + + self.assertEqual(resize(b'', 0, new), b'') + self.assertEqual(len(resize(b'', 1, new)), 1) + self.assertEqual(len(resize(b'', 2, new)), 2) + + self.assertRaises(SystemError, resize, b'abc', -1, False) + self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) + + # CRASHES resize(NULL, 0, False) + # CRASHES resize(NULL, 3, False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 682c56979c6dfa..bd521a509d07ec 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -2,7 +2,7 @@ import sys from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -27,7 +27,7 @@ class CAPITest(unittest.TestCase): def test_fromencodedobject(self): """Test PyUnicode_FromEncodedObject()""" - fromencodedobject = _testcapi.unicode_fromencodedobject + fromencodedobject = _testlimitedcapi.unicode_fromencodedobject self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') @@ -52,7 +52,7 @@ def test_fromencodedobject(self): def test_decode(self): """Test PyUnicode_Decode()""" - decode = _testcapi.unicode_decode + decode = _testlimitedcapi.unicode_decode self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') @@ -70,7 +70,7 @@ def test_decode(self): def test_asencodedstring(self): """Test PyUnicode_AsEncodedString()""" - asencodedstring = _testcapi.unicode_asencodedstring + asencodedstring = _testlimitedcapi.unicode_asencodedstring self.assertEqual(asencodedstring('abc', NULL), b'abc') self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') @@ -93,7 +93,7 @@ def test_asencodedstring(self): def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf8 = _testcapi.unicode_decodeutf8 + decodeutf8 = _testlimitedcapi.unicode_decodeutf8 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -113,7 +113,7 @@ def test_decodeutf8(self): def test_decodeutf8stateful(self): """Test PyUnicode_DecodeUTF8Stateful()""" - decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -136,7 +136,7 @@ def test_decodeutf8stateful(self): def test_asutf8string(self): """Test PyUnicode_AsUTF8String()""" - asutf8string = _testcapi.unicode_asutf8string + asutf8string = _testlimitedcapi.unicode_asutf8string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf8string(s), s.encode('utf-8')) @@ -148,7 +148,7 @@ def test_asutf8string(self): def test_decodeutf16(self): """Test PyUnicode_DecodeUTF16()""" - decodeutf16 = _testcapi.unicode_decodeutf16 + decodeutf16 = _testlimitedcapi.unicode_decodeutf16 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -192,7 +192,7 @@ def test_decodeutf16(self): def test_decodeutf16stateful(self): """Test PyUnicode_DecodeUTF16Stateful()""" - decodeutf16stateful = _testcapi.unicode_decodeutf16stateful + decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -238,7 +238,7 @@ def test_decodeutf16stateful(self): def test_asutf16string(self): """Test PyUnicode_AsUTF16String()""" - asutf16string = _testcapi.unicode_asutf16string + asutf16string = _testlimitedcapi.unicode_asutf16string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf16string(s), s.encode('utf-16')) @@ -250,7 +250,7 @@ def test_asutf16string(self): def test_decodeutf32(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf32 = _testcapi.unicode_decodeutf32 + decodeutf32 = _testlimitedcapi.unicode_decodeutf32 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -290,7 +290,7 @@ def test_decodeutf32(self): def test_decodeutf32stateful(self): """Test PyUnicode_DecodeUTF32Stateful()""" - decodeutf32stateful = _testcapi.unicode_decodeutf32stateful + decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -342,7 +342,7 @@ def test_decodeutf32stateful(self): def test_asutf32string(self): """Test PyUnicode_AsUTF32String()""" - asutf32string = _testcapi.unicode_asutf32string + asutf32string = _testlimitedcapi.unicode_asutf32string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf32string(s), s.encode('utf-32')) @@ -354,7 +354,7 @@ def test_asutf32string(self): def test_decodelatin1(self): """Test PyUnicode_DecodeLatin1()""" - decodelatin1 = _testcapi.unicode_decodelatin1 + decodelatin1 = _testlimitedcapi.unicode_decodelatin1 self.assertEqual(decodelatin1(b'abc'), 'abc') self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') @@ -365,7 +365,7 @@ def test_decodelatin1(self): def test_aslatin1string(self): """Test PyUnicode_AsLatin1String()""" - aslatin1string = _testcapi.unicode_aslatin1string + aslatin1string = _testlimitedcapi.unicode_aslatin1string self.assertEqual(aslatin1string('abc'), b'abc') self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') @@ -377,7 +377,7 @@ def test_aslatin1string(self): def test_decodeascii(self): """Test PyUnicode_DecodeASCII()""" - decodeascii = _testcapi.unicode_decodeascii + decodeascii = _testlimitedcapi.unicode_decodeascii self.assertEqual(decodeascii(b'abc'), 'abc') self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') @@ -392,7 +392,7 @@ def test_decodeascii(self): def test_asasciistring(self): """Test PyUnicode_AsASCIIString()""" - asasciistring = _testcapi.unicode_asasciistring + asasciistring = _testlimitedcapi.unicode_asasciistring self.assertEqual(asasciistring('abc'), b'abc') @@ -403,7 +403,7 @@ def test_asasciistring(self): def test_decodecharmap(self): """Test PyUnicode_DecodeCharmap()""" - decodecharmap = _testcapi.unicode_decodecharmap + decodecharmap = _testlimitedcapi.unicode_decodecharmap self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') @@ -426,7 +426,7 @@ def test_decodecharmap(self): def test_ascharmapstring(self): """Test PyUnicode_AsCharmapString()""" - ascharmapstring = _testcapi.unicode_ascharmapstring + ascharmapstring = _testlimitedcapi.unicode_ascharmapstring self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') @@ -443,7 +443,7 @@ def test_ascharmapstring(self): def test_decodeunicodeescape(self): """Test PyUnicode_DecodeUnicodeEscape()""" - decodeunicodeescape = _testcapi.unicode_decodeunicodeescape + decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape self.assertEqual(decodeunicodeescape(b'abc'), 'abc') self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') @@ -467,7 +467,7 @@ def test_decodeunicodeescape(self): def test_asunicodeescapestring(self): """Test PyUnicode_AsUnicodeEscapeString()""" - asunicodeescapestring = _testcapi.unicode_asunicodeescapestring + asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring self.assertEqual(asunicodeescapestring('abc'), b'abc') self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') @@ -481,7 +481,7 @@ def test_asunicodeescapestring(self): def test_decoderawunicodeescape(self): """Test PyUnicode_DecodeRawUnicodeEscape()""" - decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape + decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') @@ -503,7 +503,7 @@ def test_decoderawunicodeescape(self): def test_asrawunicodeescapestring(self): """Test PyUnicode_AsRawUnicodeEscapeString()""" - asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring + asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring self.assertEqual(asrawunicodeescapestring('abc'), b'abc') self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py new file mode 100644 index 00000000000000..328ea12f97462c --- /dev/null +++ b/Lib/test/test_capi/test_complex.py @@ -0,0 +1,251 @@ +from math import isnan +import errno +import unittest +import warnings + +from test.test_capi.test_getargs import (BadComplex, BadComplex2, Complex, + FloatSubclass, Float, BadFloat, + BadFloat2, ComplexSubclass) +from test.support import import_helper + + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +INF = float("inf") +NAN = float("nan") +DBL_MAX = _testcapi.DBL_MAX + + +class BadComplex3: + def __complex__(self): + raise RuntimeError + + +class CAPIComplexTest(unittest.TestCase): + def test_check(self): + # Test PyComplex_Check() + check = _testlimitedcapi.complex_check + + self.assertTrue(check(1+2j)) + self.assertTrue(check(ComplexSubclass(1+2j))) + self.assertFalse(check(Complex())) + self.assertFalse(check(3)) + self.assertFalse(check(3.0)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # PyComplex_CheckExact() + checkexact = _testlimitedcapi.complex_checkexact + + self.assertTrue(checkexact(1+2j)) + self.assertFalse(checkexact(ComplexSubclass(1+2j))) + self.assertFalse(checkexact(Complex())) + self.assertFalse(checkexact(3)) + self.assertFalse(checkexact(3.0)) + self.assertFalse(checkexact(object())) + + # CRASHES checkexact(NULL) + + def test_fromccomplex(self): + # Test PyComplex_FromCComplex() + fromccomplex = _testcapi.complex_fromccomplex + + self.assertEqual(fromccomplex(1+2j), 1.0+2.0j) + + def test_fromdoubles(self): + # Test PyComplex_FromDoubles() + fromdoubles = _testlimitedcapi.complex_fromdoubles + + self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) + + def test_realasdouble(self): + # Test PyComplex_RealAsDouble() + realasdouble = _testlimitedcapi.complex_realasdouble + + self.assertEqual(realasdouble(1+2j), 1.0) + self.assertEqual(realasdouble(-1+0j), -1.0) + self.assertEqual(realasdouble(4.25), 4.25) + self.assertEqual(realasdouble(-1.0), -1.0) + self.assertEqual(realasdouble(42), 42.) + self.assertEqual(realasdouble(-1), -1.0) + + # Test subclasses of complex/float + self.assertEqual(realasdouble(ComplexSubclass(1+2j)), 1.0) + self.assertEqual(realasdouble(FloatSubclass(4.25)), 4.25) + + # Test types with __complex__ dunder method + self.assertEqual(realasdouble(Complex()), 4.25) + self.assertRaises(TypeError, realasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(realasdouble(BadComplex2()), 4.25) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, realasdouble, BadComplex2()) + self.assertRaises(RuntimeError, realasdouble, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(realasdouble(Float()), 4.25) + self.assertRaises(TypeError, realasdouble, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(realasdouble(BadFloat2()), 4.25) + + self.assertRaises(TypeError, realasdouble, object()) + + # CRASHES realasdouble(NULL) + + def test_imagasdouble(self): + # Test PyComplex_ImagAsDouble() + imagasdouble = _testlimitedcapi.complex_imagasdouble + + self.assertEqual(imagasdouble(1+2j), 2.0) + self.assertEqual(imagasdouble(1-1j), -1.0) + self.assertEqual(imagasdouble(4.25), 0.0) + self.assertEqual(imagasdouble(42), 0.0) + + # Test subclasses of complex/float + self.assertEqual(imagasdouble(ComplexSubclass(1+2j)), 2.0) + self.assertEqual(imagasdouble(FloatSubclass(4.25)), 0.0) + + # Test types with __complex__ dunder method + self.assertEqual(imagasdouble(Complex()), 0.5) + self.assertRaises(TypeError, imagasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadComplex2()), 0.5) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, imagasdouble, BadComplex2()) + self.assertRaises(RuntimeError, imagasdouble, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(imagasdouble(Float()), 0.0) + self.assertRaises(TypeError, imagasdouble, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadFloat2()), 0.0) + + self.assertRaises(TypeError, imagasdouble, object()) + + # CRASHES imagasdouble(NULL) + + def test_asccomplex(self): + # Test PyComplex_AsCComplex() + asccomplex = _testcapi.complex_asccomplex + + self.assertEqual(asccomplex(1+2j), 1.0+2.0j) + self.assertEqual(asccomplex(-1+2j), -1.0+2.0j) + self.assertEqual(asccomplex(4.25), 4.25+0.0j) + self.assertEqual(asccomplex(-1.0), -1.0+0.0j) + self.assertEqual(asccomplex(42), 42+0j) + self.assertEqual(asccomplex(-1), -1.0+0.0j) + + # Test subclasses of complex/float + self.assertEqual(asccomplex(ComplexSubclass(1+2j)), 1.0+2.0j) + self.assertEqual(asccomplex(FloatSubclass(4.25)), 4.25+0.0j) + + # Test types with __complex__ dunder method + self.assertEqual(asccomplex(Complex()), 4.25+0.5j) + self.assertRaises(TypeError, asccomplex, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asccomplex(BadComplex2()), 4.25+0.5j) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, asccomplex, BadComplex2()) + self.assertRaises(RuntimeError, asccomplex, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(asccomplex(Float()), 4.25+0.0j) + self.assertRaises(TypeError, asccomplex, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asccomplex(BadFloat2()), 4.25+0.0j) + + self.assertRaises(TypeError, asccomplex, object()) + + # CRASHES asccomplex(NULL) + + def test_py_c_sum(self): + # Test _Py_c_sum() + _py_c_sum = _testcapi._py_c_sum + + self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + + def test_py_c_diff(self): + # Test _Py_c_diff() + _py_c_diff = _testcapi._py_c_diff + + self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + + def test_py_c_neg(self): + # Test _Py_c_neg() + _py_c_neg = _testcapi._py_c_neg + + self.assertEqual(_py_c_neg(1+1j), -1-1j) + + def test_py_c_prod(self): + # Test _Py_c_prod() + _py_c_prod = _testcapi._py_c_prod + + self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + + def test_py_c_quot(self): + # Test _Py_c_quot() + _py_c_quot = _testcapi._py_c_quot + + self.assertEqual(_py_c_quot(1, 1j), (-1j, 0)) + self.assertEqual(_py_c_quot(1, -1j), (1j, 0)) + self.assertEqual(_py_c_quot(1j, 2), (0.5j, 0)) + self.assertEqual(_py_c_quot(1j, -2), (-0.5j, 0)) + self.assertEqual(_py_c_quot(1, 2j), (-0.5j, 0)) + + z, e = _py_c_quot(NAN, 1j) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + z, e = _py_c_quot(1j, NAN) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + + def test_py_c_pow(self): + # Test _Py_c_pow() + _py_c_pow = _testcapi._py_c_pow + + self.assertEqual(_py_c_pow(1j, 0j), (1+0j, 0)) + self.assertEqual(_py_c_pow(1, 1j), (1+0j, 0)) + self.assertEqual(_py_c_pow(0j, 1), (0j, 0)) + self.assertAlmostEqual(_py_c_pow(1j, 2)[0], -1.0+0j) + + r, e = _py_c_pow(1+1j, -1) + self.assertAlmostEqual(r, 0.5-0.5j) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) + self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) + self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + + + def test_py_c_abs(self): + # Test _Py_c_abs() + _py_c_abs = _testcapi._py_c_abs + + self.assertEqual(_py_c_abs(-1), (1.0, 0)) + self.assertEqual(_py_c_abs(1j), (1.0, 0)) + + self.assertEqual(_py_c_abs(complex('+inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('-inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25+infj')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25-infj')), (INF, 0)) + + self.assertTrue(isnan(_py_c_abs(complex('1.25+nanj'))[0])) + self.assertTrue(isnan(_py_c_abs(complex('nan-1j'))[0])) + + self.assertEqual(_py_c_abs(complex(*[DBL_MAX]*2))[1], errno.ERANGE) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 11b2ca910707df..e726e3d813d888 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -1,7 +1,12 @@ import unittest from collections import OrderedDict, UserDict from types import MappingProxyType -import _testcapi +from test import support +from test.support import import_helper + + +_testcapi = import_helper.import_module("_testcapi") +_testlimitedcapi = import_helper.import_module("_testlimitedcapi") NULL = None @@ -24,25 +29,25 @@ def gen(): class CAPITest(unittest.TestCase): def test_dict_check(self): - check = _testcapi.dict_check + check = _testlimitedcapi.dict_check self.assertTrue(check({1: 2})) self.assertTrue(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) self.assertFalse(check([1, 2])) self.assertFalse(check(object())) - #self.assertFalse(check(NULL)) + # CRASHES check(NULL) def test_dict_checkexact(self): - check = _testcapi.dict_checkexact + check = _testlimitedcapi.dict_checkexact self.assertTrue(check({1: 2})) self.assertFalse(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) self.assertFalse(check([1, 2])) self.assertFalse(check(object())) - #self.assertFalse(check(NULL)) + # CRASHES check(NULL) def test_dict_new(self): - dict_new = _testcapi.dict_new + dict_new = _testlimitedcapi.dict_new dct = dict_new() self.assertEqual(dct, {}) self.assertIs(type(dct), dict) @@ -50,7 +55,7 @@ def test_dict_new(self): self.assertIsNot(dct2, dct) def test_dictproxy_new(self): - dictproxy_new = _testcapi.dictproxy_new + dictproxy_new = _testlimitedcapi.dictproxy_new for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): proxy = dictproxy_new(dct) self.assertIs(type(proxy), MappingProxyType) @@ -66,7 +71,7 @@ def test_dictproxy_new(self): # CRASHES dictproxy_new(NULL) def test_dict_copy(self): - copy = _testcapi.dict_copy + copy = _testlimitedcapi.dict_copy for dct in {1: 2}, OrderedDict({1: 2}): dct_copy = copy(dct) self.assertIs(type(dct_copy), dict) @@ -78,7 +83,7 @@ def test_dict_copy(self): self.assertRaises(SystemError, copy, NULL) def test_dict_clear(self): - clear = _testcapi.dict_clear + clear = _testlimitedcapi.dict_clear dct = {1: 2} clear(dct) self.assertEqual(dct, {}) @@ -97,7 +102,7 @@ def test_dict_clear(self): # CRASHES? clear(NULL) def test_dict_size(self): - size = _testcapi.dict_size + size = _testlimitedcapi.dict_size self.assertEqual(size({1: 2}), 1) self.assertEqual(size(OrderedDict({1: 2})), 1) @@ -108,7 +113,7 @@ def test_dict_size(self): self.assertRaises(SystemError, size, NULL) def test_dict_getitem(self): - getitem = _testcapi.dict_getitem + getitem = _testlimitedcapi.dict_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -118,14 +123,19 @@ def test_dict_getitem(self): self.assertEqual(getitem(dct2, 'a'), 1) self.assertIs(getitem(dct2, 'b'), KeyError) - self.assertIs(getitem({}, []), KeyError) # unhashable + with support.catch_unraisable_exception() as cm: + self.assertIs(getitem({}, []), KeyError) # unhashable + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "unhashable type: 'list'") + self.assertIs(getitem(42, 'a'), KeyError) self.assertIs(getitem([1], 0), KeyError) # CRASHES getitem({}, NULL) # CRASHES getitem(NULL, 'a') def test_dict_getitemstring(self): - getitemstring = _testcapi.dict_getitemstring + getitemstring = _testlimitedcapi.dict_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertIs(getitemstring(dct, b'b'), KeyError) @@ -135,7 +145,12 @@ def test_dict_getitemstring(self): self.assertEqual(getitemstring(dct2, b'a'), 1) self.assertIs(getitemstring(dct2, b'b'), KeyError) - self.assertIs(getitemstring({}, INVALID_UTF8), KeyError) + with support.catch_unraisable_exception() as cm: + self.assertIs(getitemstring({}, INVALID_UTF8), KeyError) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + self.assertIs(getitemstring(42, b'a'), KeyError) self.assertIs(getitemstring([], b'a'), KeyError) # CRASHES getitemstring({}, NULL) @@ -177,7 +192,7 @@ def test_dict_getitemstringref(self): # CRASHES getitemstring(NULL, b'a') def test_dict_getitemwitherror(self): - getitem = _testcapi.dict_getitemwitherror + getitem = _testlimitedcapi.dict_getitemwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -195,7 +210,7 @@ def test_dict_getitemwitherror(self): # CRASHES getitem(NULL, 'a') def test_dict_contains(self): - contains = _testcapi.dict_contains + contains = _testlimitedcapi.dict_contains dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(contains(dct, 'a')) self.assertFalse(contains(dct, 'b')) @@ -227,7 +242,7 @@ def test_dict_contains_string(self): # CRASHES contains(NULL, b'a') def test_dict_setitem(self): - setitem = _testcapi.dict_setitem + setitem = _testlimitedcapi.dict_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -247,7 +262,7 @@ def test_dict_setitem(self): # CRASHES setitem(NULL, 'a', 5) def test_dict_setitemstring(self): - setitemstring = _testcapi.dict_setitemstring + setitemstring = _testlimitedcapi.dict_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -266,7 +281,7 @@ def test_dict_setitemstring(self): # CRASHES setitemstring(NULL, b'a', 5) def test_dict_delitem(self): - delitem = _testcapi.dict_delitem + delitem = _testlimitedcapi.dict_delitem dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -287,7 +302,7 @@ def test_dict_delitem(self): # CRASHES delitem(NULL, 'a') def test_dict_delitemstring(self): - delitemstring = _testcapi.dict_delitemstring + delitemstring = _testlimitedcapi.dict_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -328,6 +343,28 @@ def test_dict_setdefault(self): # CRASHES setdefault({}, 'a', NULL) # CRASHES setdefault(NULL, 'a', 5) + def test_dict_setdefaultref(self): + setdefault = _testcapi.dict_setdefaultref + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + def test_mapping_keys_valuesitems(self): class BadMapping(dict): def keys(self): @@ -338,21 +375,21 @@ def items(self): return None dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: - self.assertListEqual(_testcapi.dict_keys(mapping), + self.assertListEqual(_testlimitedcapi.dict_keys(mapping), list(dict_obj.keys())) - self.assertListEqual(_testcapi.dict_values(mapping), + self.assertListEqual(_testlimitedcapi.dict_values(mapping), list(dict_obj.values())) - self.assertListEqual(_testcapi.dict_items(mapping), + self.assertListEqual(_testlimitedcapi.dict_items(mapping), list(dict_obj.items())) def test_dict_keys_valuesitems_bad_arg(self): for mapping in UserDict(), [], object(): - self.assertRaises(SystemError, _testcapi.dict_keys, mapping) - self.assertRaises(SystemError, _testcapi.dict_values, mapping) - self.assertRaises(SystemError, _testcapi.dict_items, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) def test_dict_next(self): - dict_next = _testcapi.dict_next + dict_next = _testlimitedcapi.dict_next self.assertIsNone(dict_next({}, 0)) dct = {'a': 1, 'b': 2, 'c': 3} pos = 0 @@ -369,7 +406,7 @@ def test_dict_next(self): # CRASHES dict_next(NULL, 0) def test_dict_update(self): - update = _testcapi.dict_update + update = _testlimitedcapi.dict_update for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -384,7 +421,7 @@ def test_dict_update(self): self.assertRaises(SystemError, update, NULL, {}) def test_dict_merge(self): - merge = _testcapi.dict_merge + merge = _testlimitedcapi.dict_merge for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -402,7 +439,7 @@ def test_dict_merge(self): self.assertRaises(SystemError, merge, NULL, {}, 0) def test_dict_mergefromseq2(self): - mergefromseq2 = _testcapi.dict_mergefromseq2 + mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 for cls1 in dict, DictSubclass: for cls2 in list, iter: dct = cls1({'a': 1, 'b': 2}) @@ -421,6 +458,93 @@ def test_dict_mergefromseq2(self): # CRASHES mergefromseq2({}, NULL, 0) # CRASHES mergefromseq2(NULL, {}, 0) + def test_dict_pop(self): + # Test PyDict_Pop() + dict_pop = _testcapi.dict_pop + dict_pop_null = _testcapi.dict_pop_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing, expect removed value; empty dict has a fast path + self.assertEqual(dict_pop({}, "key"), (0, NULL)) + self.assertEqual(dict_pop({"a": 1}, "key"), (0, NULL)) + + # key missing, ignored removed value; empty dict has a fast path + self.assertEqual(dict_pop_null({}, "key"), 0) + self.assertEqual(dict_pop_null({"a": 1}, "key"), 0) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_pop, not_dict, "key") + self.assertRaises(SystemError, dict_pop_null, not_dict, "key") + + # key error; don't hash key if dict is empty + not_hashable_key = ["list"] + self.assertEqual(dict_pop({}, not_hashable_key), (0, NULL)) + with self.assertRaises(TypeError): + dict_pop({'key': 1}, not_hashable_key) + dict_pop({}, NULL) # key is not checked if dict is empty + + # CRASHES dict_pop(NULL, "key") + # CRASHES dict_pop({"a": 1}, NULL) + + def test_dict_popstring(self): + # Test PyDict_PopString() + dict_popstring = _testcapi.dict_popstring + dict_popstring_null = _testcapi.dict_popstring_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing; empty dict has a fast path + self.assertEqual(dict_popstring({}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({}, "key"), 0) + self.assertEqual(dict_popstring({"a": 1}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({"a": 1}, "key"), 0) + + # non-ASCII key + non_ascii = '\U0001f40d' + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring(dct, '\U0001f40d'.encode()), (1, 123)) + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring_null(dct, '\U0001f40d'.encode()), 1) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_popstring, not_dict, "key") + self.assertRaises(SystemError, dict_popstring_null, not_dict, "key") + + # key error + self.assertRaises(UnicodeDecodeError, dict_popstring, {1: 2}, INVALID_UTF8) + self.assertRaises(UnicodeDecodeError, dict_popstring_null, {1: 2}, INVALID_UTF8) + + # CRASHES dict_popstring(NULL, "key") + # CRASHES dict_popstring({}, NULL) + # CRASHES dict_popstring({"a": 1}, NULL) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_eval_code_ex.py b/Lib/test/test_capi/test_eval_code_ex.py index 2d28e5289eff94..b298e5007e5e7d 100644 --- a/Lib/test/test_capi/test_eval_code_ex.py +++ b/Lib/test/test_capi/test_eval_code_ex.py @@ -1,11 +1,16 @@ import unittest +import builtins +from collections import UserDict from test.support import import_helper +from test.support import swap_attr # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class PyEval_EvalCodeExTests(unittest.TestCase): @@ -13,43 +18,108 @@ def test_simple(self): def f(): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, dict(a=1)), 1) - - # Need to force the compiler to use LOAD_NAME - # def test_custom_locals(self): - # def f(): - # return + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, dict(a=1)), 1) + + self.assertRaises(NameError, eval_code_ex, code, {}) + self.assertRaises(SystemError, eval_code_ex, code, UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, []) + self.assertRaises(SystemError, eval_code_ex, code, 1) + # CRASHES eval_code_ex(code, NULL) + # CRASHES eval_code_ex(1, {}) + # CRASHES eval_code_ex(NULL, {}) + + def test_custom_locals(self): + # Monkey-patch __build_class__ to get a class code object. + code = None + def build_class(func, name, /, *bases, **kwds): + nonlocal code + code = func.__code__ + + with swap_attr(builtins, '__build_class__', build_class): + class A: + # Uses LOAD_NAME for a + r[:] = [a] + + eval_code_ex = _testcapi.eval_code_ex + results = [] + g = dict(a=1, r=results) + self.assertIsNone(eval_code_ex(code, g)) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, dict(a=2))) + self.assertEqual(results, [2]) + self.assertIsNone(eval_code_ex(code, g, UserDict(a=3))) + self.assertEqual(results, [3]) + self.assertIsNone(eval_code_ex(code, g, {})) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, NULL)) + self.assertEqual(results, [1]) + + self.assertRaises(TypeError, eval_code_ex, code, g, []) + self.assertRaises(TypeError, eval_code_ex, code, g, 1) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), {}) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), NULL) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), []) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), 1) def test_with_args(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (1, 2, 3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (1, 2, 3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2, 3, 4)) def test_with_kwargs(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), dict(a=1, b=2, c=3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), dict(a=1, b=2, c=3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2, c=3, d=4)) def test_with_default(self): def f(a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (1,)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (1,)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, ()) def test_with_kwarg_default(self): def f(*, a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), dict(a=1)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), dict(a=1)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), {}) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), NULL) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), []) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), 1) def test_with_closure(self): a = 1 + b = 2 def f(): + b return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), {}, f.__closure__), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__), 1) + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__[::-1]), 2) + + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, ()), 1) + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, NULL), 1) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b96cc7922a0ee7..c475b6d78d0c56 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -3,11 +3,12 @@ import re import sys import unittest +import textwrap from test import support from test.support import import_helper from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin from .test_misc import decode_stderr @@ -17,6 +18,10 @@ NULL = None +class CustomError(Exception): + pass + + class Test_Exceptions(unittest.TestCase): def test_exception(self): @@ -64,6 +69,47 @@ def test_exc_info(self): else: self.assertTrue(False) + def test_warn_with_stacklevel(self): + code = textwrap.dedent('''\ + import _testcapi + + def foo(): + _testcapi.function_set_warning() + + foo() # line 6 + + + foo() # line 9 + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + self.assertEqual(warnings, [ + b':6: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 6', + b':9: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 9', + ]) + + def test_warn_during_finalization(self): + code = textwrap.dedent('''\ + import _testcapi + + class Foo: + def foo(self): + _testcapi.function_set_warning() + def __del__(self): + self.foo() + + ref = Foo() + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + # Due to the finalization of the interpreter, the source will be ommited + # because the ``warnings`` module cannot be imported at this time + self.assertEqual(warnings, [ + b':7: RuntimeWarning: Testing PyErr_WarnEx', + ]) + class Test_FatalError(unittest.TestCase): @@ -270,6 +316,104 @@ def test_setfromerrnowithfilename(self): (ENOENT, 'No such file or directory', 'file')) # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') + def test_err_writeunraisable(self): + # Test PyErr_WriteUnraisable() + writeunraisable = _testcapi.err_writeunraisable + firstline = self.test_err_writeunraisable.__code__.co_firstlineno + + with support.catch_unraisable_exception() as cm: + writeunraisable(CustomError('oops!'), hex) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 6) + self.assertIsNone(cm.unraisable.err_msg) + self.assertEqual(cm.unraisable.object, hex) + + with support.catch_unraisable_exception() as cm: + writeunraisable(CustomError('oops!'), NULL) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 15) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + writeunraisable(CustomError('oops!'), hex) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Exception ignored in: {hex!r}') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + writeunraisable(CustomError('oops!'), NULL) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + # CRASHES writeunraisable(NULL, hex) + # CRASHES writeunraisable(NULL, NULL) + + def test_err_formatunraisable(self): + # Test PyErr_FormatUnraisable() + formatunraisable = _testcapi.err_formatunraisable + firstline = self.test_err_formatunraisable.__code__.co_firstlineno + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), b'Error in %R', []) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 6) + self.assertEqual(cm.unraisable.err_msg, 'Error in []') + self.assertIsNone(cm.unraisable.object) + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), b'undecodable \xff') + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 15) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), NULL) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 24) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), b'Error in %R', []) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Error in []:') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), b'undecodable \xff') + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), NULL) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + # CRASHES formatunraisable(NULL, b'Error in %R', []) + # CRASHES formatunraisable(NULL, NULL) + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py new file mode 100644 index 00000000000000..92c987794142c9 --- /dev/null +++ b/Lib/test/test_capi/test_float.py @@ -0,0 +1,183 @@ +import math +import sys +import unittest +import warnings + +from test.test_capi.test_getargs import (Float, FloatSubclass, FloatSubclass2, + BadIndex2, BadFloat2, Index, BadIndex, + BadFloat) +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None + +# For PyFloat_Pack/Unpack* +BIG_ENDIAN = 0 +LITTLE_ENDIAN = 1 +EPSILON = { + 2: 2.0 ** -11, # binary16 + 4: 2.0 ** -24, # binary32 + 8: 2.0 ** -53, # binary64 +} + +HAVE_IEEE_754 = float.__getformat__("double").startswith("IEEE") +INF = float("inf") +NAN = float("nan") + + +class CAPIFloatTest(unittest.TestCase): + def test_check(self): + # Test PyFloat_Check() + check = _testlimitedcapi.float_check + + self.assertTrue(check(4.25)) + self.assertTrue(check(FloatSubclass(4.25))) + self.assertFalse(check(Float())) + self.assertFalse(check(3)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyFloat_CheckExact() + checkexact = _testlimitedcapi.float_checkexact + + self.assertTrue(checkexact(4.25)) + self.assertFalse(checkexact(FloatSubclass(4.25))) + self.assertFalse(checkexact(Float())) + self.assertFalse(checkexact(3)) + self.assertFalse(checkexact(object())) + + # CRASHES checkexact(NULL) + + def test_fromstring(self): + # Test PyFloat_FromString() + fromstring = _testlimitedcapi.float_fromstring + + self.assertEqual(fromstring("4.25"), 4.25) + self.assertEqual(fromstring(b"4.25"), 4.25) + self.assertRaises(ValueError, fromstring, "4.25\0") + self.assertRaises(ValueError, fromstring, b"4.25\0") + + self.assertEqual(fromstring(bytearray(b"4.25")), 4.25) + + self.assertEqual(fromstring(memoryview(b"4.25")), 4.25) + self.assertEqual(fromstring(memoryview(b"4.255")[:-1]), 4.25) + self.assertRaises(TypeError, fromstring, memoryview(b"4.25")[::2]) + + self.assertRaises(TypeError, fromstring, 4.25) + + # CRASHES fromstring(NULL) + + def test_fromdouble(self): + # Test PyFloat_FromDouble() + fromdouble = _testlimitedcapi.float_fromdouble + + self.assertEqual(fromdouble(4.25), 4.25) + + def test_asdouble(self): + # Test PyFloat_AsDouble() + asdouble = _testlimitedcapi.float_asdouble + + class BadFloat3: + def __float__(self): + raise RuntimeError + + self.assertEqual(asdouble(4.25), 4.25) + self.assertEqual(asdouble(-1.0), -1.0) + self.assertEqual(asdouble(42), 42.0) + self.assertEqual(asdouble(-1), -1.0) + self.assertEqual(asdouble(2**1000), float(2**1000)) + + self.assertEqual(asdouble(FloatSubclass(4.25)), 4.25) + self.assertEqual(asdouble(FloatSubclass2(4.25)), 4.25) + self.assertEqual(asdouble(Index()), 99.) + + self.assertRaises(TypeError, asdouble, BadIndex()) + self.assertRaises(TypeError, asdouble, BadFloat()) + self.assertRaises(RuntimeError, asdouble, BadFloat3()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asdouble(BadIndex2()), 1.) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asdouble(BadFloat2()), 4.25) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, asdouble, BadFloat2()) + self.assertRaises(TypeError, asdouble, object()) + self.assertRaises(TypeError, asdouble, NULL) + + def test_getinfo(self): + # Test PyFloat_GetInfo() + getinfo = _testlimitedcapi.float_getinfo + + self.assertEqual(getinfo(), sys.float_info) + + def test_getmax(self): + # Test PyFloat_GetMax() + getmax = _testlimitedcapi.float_getmax + + self.assertEqual(getmax(), sys.float_info.max) + + def test_getmin(self): + # Test PyFloat_GetMax() + getmin = _testlimitedcapi.float_getmin + + self.assertEqual(getmin(), sys.float_info.min) + + def test_pack(self): + # Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() + pack = _testcapi.float_pack + + self.assertEqual(pack(2, 1.5, BIG_ENDIAN), b'>\x00') + self.assertEqual(pack(4, 1.5, BIG_ENDIAN), b'?\xc0\x00\x00') + self.assertEqual(pack(8, 1.5, BIG_ENDIAN), + b'?\xf8\x00\x00\x00\x00\x00\x00') + self.assertEqual(pack(2, 1.5, LITTLE_ENDIAN), b'\x00>') + self.assertEqual(pack(4, 1.5, LITTLE_ENDIAN), b'\x00\x00\xc0?') + self.assertEqual(pack(8, 1.5, LITTLE_ENDIAN), + b'\x00\x00\x00\x00\x00\x00\xf8?') + + def test_unpack(self): + # Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() + unpack = _testcapi.float_unpack + + self.assertEqual(unpack(b'>\x00', BIG_ENDIAN), 1.5) + self.assertEqual(unpack(b'?\xc0\x00\x00', BIG_ENDIAN), 1.5) + self.assertEqual(unpack(b'?\xf8\x00\x00\x00\x00\x00\x00', BIG_ENDIAN), + 1.5) + self.assertEqual(unpack(b'\x00>', LITTLE_ENDIAN), 1.5) + self.assertEqual(unpack(b'\x00\x00\xc0?', LITTLE_ENDIAN), 1.5) + self.assertEqual(unpack(b'\x00\x00\x00\x00\x00\x00\xf8?', LITTLE_ENDIAN), + 1.5) + + def test_pack_unpack_roundtrip(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + large = 2.0 ** 100 + values = [1.0, 1.5, large, 1.0/7, math.pi] + if HAVE_IEEE_754: + values.extend((INF, NAN)) + for value in values: + for size in (2, 4, 8,): + if size == 2 and value == large: + # too large for 16-bit float + continue + rel_tol = EPSILON[size] + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(value=value, size=size, endian=endian): + data = pack(size, value, endian) + value2 = unpack(data, endian) + if math.isnan(value): + self.assertTrue(math.isnan(value2), (value, value2)) + elif size < 8: + self.assertTrue(math.isclose(value2, value, rel_tol=rel_tol), + (value, value2)) + else: + self.assertEqual(value2, value) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 96d34ab94c5981..232aa2a80025dc 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -4,11 +4,17 @@ import sys from test import support from test.support import import_helper +from test.support import script_helper from test.support import warnings_helper # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = NULL + # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. # > @@ -153,6 +159,8 @@ class TupleSubclass(tuple): class DictSubclass(dict): pass +NONCONTIG_WRITABLE = memoryview(bytearray(b'noncontig'))[::-2] +NONCONTIG_READONLY = memoryview(b'noncontig')[::-2] class Unsigned_TestCase(unittest.TestCase): def test_b(self): @@ -665,7 +673,7 @@ def test_invalid_keyword(self): try: getargs_keywords((1,2),3,arg5=10,arg666=666) except TypeError as err: - self.assertEqual(str(err), "'arg666' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument 'arg666'") else: self.fail('TypeError should have been raised') @@ -673,7 +681,7 @@ def test_surrogate_keyword(self): try: getargs_keywords((1,2), 3, (4,(5,6)), (7,8,9), **{'\uDC80': 10}) except TypeError as err: - self.assertEqual(str(err), "'\udc80' is an invalid keyword argument for this function") + self.assertEqual(str(err), "this function got an unexpected keyword argument '\udc80'") else: self.fail('TypeError should have been raised') @@ -740,12 +748,12 @@ def test_too_many_args(self): def test_invalid_keyword(self): # extraneous keyword arg with self.assertRaisesRegex(TypeError, - "'monster' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument 'monster'"): getargs_keyword_only(1, 2, monster=666) def test_surrogate_keyword(self): with self.assertRaisesRegex(TypeError, - "'\udc80' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument '\udc80'"): getargs_keyword_only(1, 2, **{'\uDC80': 10}) def test_weird_str_subclass(self): @@ -759,7 +767,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) def test_weird_str_subclass2(self): @@ -772,7 +780,7 @@ def __hash__(self): "invalid keyword argument for this function"): getargs_keyword_only(1, 2, **{BadStr("keyword_only"): 3}) with self.assertRaisesRegex(TypeError, - "invalid keyword argument for this function"): + "this function got an unexpected keyword argument"): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) @@ -805,7 +813,7 @@ def test_required_args(self): def test_empty_keyword(self): with self.assertRaisesRegex(TypeError, - "'' is an invalid keyword argument for this function"): + "this function got an unexpected keyword argument ''"): self.getargs(1, 2, **{'': 666}) @@ -837,6 +845,8 @@ def test_y_star(self): self.assertEqual(getargs_y_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_y_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_y_star, None) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_READONLY) def test_y_hash(self): from _testcapi import getargs_y_hash @@ -846,21 +856,39 @@ def test_y_hash(self): self.assertRaises(TypeError, getargs_y_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_y_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_y_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_READONLY) def test_w_star(self): # getargs_w_star() modifies first and last byte - from _testcapi import getargs_w_star - self.assertRaises(TypeError, getargs_w_star, 'abc\xe9') - self.assertRaises(TypeError, getargs_w_star, b'bytes') - self.assertRaises(TypeError, getargs_w_star, b'nul:\0') - self.assertRaises(TypeError, getargs_w_star, memoryview(b'bytes')) - buf = bytearray(b'bytearray') - self.assertEqual(getargs_w_star(buf), b'[ytearra]') - self.assertEqual(buf, bytearray(b'[ytearra]')) - buf = bytearray(b'memoryview') - self.assertEqual(getargs_w_star(memoryview(buf)), b'[emoryvie]') - self.assertEqual(buf, bytearray(b'[emoryvie]')) - self.assertRaises(TypeError, getargs_w_star, None) + # getargs_w_star_opt() takes additional optional args: with one + # argument it should behave the same as getargs_w_star + from _testcapi import getargs_w_star, getargs_w_star_opt + for func in (getargs_w_star, getargs_w_star_opt): + with self.subTest(func=func): + self.assertRaises(TypeError, func, 'abc\xe9') + self.assertRaises(TypeError, func, b'bytes') + self.assertRaises(TypeError, func, b'nul:\0') + self.assertRaises(TypeError, func, memoryview(b'bytes')) + buf = bytearray(b'bytearray') + self.assertEqual(func(buf), b'[ytearra]') + self.assertEqual(buf, bytearray(b'[ytearra]')) + buf = bytearray(b'memoryview') + self.assertEqual(func(memoryview(buf)), b'[emoryvie]') + self.assertEqual(buf, bytearray(b'[emoryvie]')) + self.assertRaises(TypeError, func, None) + self.assertRaises(TypeError, func, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, func, NONCONTIG_READONLY) + + def test_getargs_empty(self): + from _testcapi import getargs_empty + self.assertTrue(getargs_empty()) + self.assertRaises(TypeError, getargs_empty, 1) + self.assertRaises(TypeError, getargs_empty, 1, 2, 3) + self.assertRaises(TypeError, getargs_empty, a=1) + self.assertRaises(TypeError, getargs_empty, a=1, b=2) + self.assertRaises(TypeError, getargs_empty, 'x', 'y', 'z', a=1, b=2) class String_TestCase(unittest.TestCase): @@ -893,6 +921,8 @@ def test_s_star(self): self.assertEqual(getargs_s_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_s_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_s_star, None) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_READONLY) def test_s_hash(self): from _testcapi import getargs_s_hash @@ -902,6 +932,9 @@ def test_s_hash(self): self.assertRaises(TypeError, getargs_s_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_s_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_s_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_READONLY) def test_z(self): from _testcapi import getargs_z @@ -920,6 +953,8 @@ def test_z_star(self): self.assertEqual(getargs_z_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_z_star(memoryview(b'memoryview')), b'memoryview') self.assertIsNone(getargs_z_star(None)) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_READONLY) def test_z_hash(self): from _testcapi import getargs_z_hash @@ -929,6 +964,9 @@ def test_z_hash(self): self.assertRaises(TypeError, getargs_z_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_z_hash, memoryview(b'memoryview')) self.assertIsNone(getargs_z_hash(None)) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_READONLY) def test_es(self): from _testcapi import getargs_es @@ -1084,9 +1122,9 @@ def test_skipitem(self): c = chr(i) # skip parentheses, the error reporting is inconsistent about them - # skip 'e', it's always a two-character code + # skip 'e' and 'w', they're always two-character codes # skip '|' and '$', they don't represent arguments anyway - if c in '()e|$': + if c in '()ew|$': continue # test the format unit when not skipped @@ -1124,7 +1162,7 @@ def test_skipitem_with_suffix(self): dict_b = {'b':1} keywords = ["a", "b"] - supported = ('s#', 's*', 'z#', 'z*', 'y#', 'y*', 'w#', 'w*') + supported = ('s#', 's*', 'z#', 'z*', 'y#', 'y*', 'w*') for c in string.ascii_letters: for c2 in '#*': f = c + c2 @@ -1176,7 +1214,7 @@ def test_basic(self): "function missing required argument 'a'"): parse((), {}, 'O', ['a']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', ['a']) with self.assertRaisesRegex(TypeError, fr"argument for function given by name \('a'\) " @@ -1250,10 +1288,10 @@ def test_nonascii_keywords(self): fr"and position \(1\)"): parse((1,), {name: 2}, 'O|O', [name, 'b']) with self.assertRaisesRegex(TypeError, - f"'{name}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name}'"): parse((), {name: 1}, '|O', ['b']) with self.assertRaisesRegex(TypeError, - "'b' is an invalid keyword argument"): + "this function got an unexpected keyword argument 'b'"): parse((), {'b': 1}, '|O', [name]) invalid = name.encode() + (name.encode()[:-1] or b'\x80') @@ -1273,24 +1311,73 @@ def test_nonascii_keywords(self): for name2 in ('b', 'ë', 'ĉ', 'Ɐ', 'ð€'): with self.subTest(name2=name2): with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name2 = name.encode().decode('latin1') if name2 != name: with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1}, '|O', [name]) name3 = name + '3' with self.assertRaisesRegex(TypeError, - f"'{name2}' is an invalid keyword argument"): + f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) + def test_nested_tuple(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) + self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), + (1, 2, 3, 4)) + parse(((1, 2, 3),), {}, '(iii)', ['a']) -class Test_testcapi(unittest.TestCase): - locals().update((name, getattr(_testcapi, name)) - for name in dir(_testcapi) - if name.startswith('test_') and name.endswith('_code')) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 3"): + parse(((1, 2, 3),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 1"): + parse(((1,),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not int"): + parse((1,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not bytes"): + parse((b'ab',), {}, '(ii)', ['a']) + + for f in 'es', 'et', 'es#', 'et#': + with self.assertRaises(LookupError): # empty encoding "" + parse((('a',),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 1, not 0"): + parse(((),), {}, '(' + f + ')', ['a']) + + @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') + def test_gh_119213(self): + rc, out, err = script_helper.assert_python_ok("-c", """if True: + from test import support + script = '''if True: + import _testinternalcapi + _testinternalcapi.gh_119213_getargs(spam='eggs') + ''' + config = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + use_main_obmalloc=False, + gil=2, + check_multi_interp_extensions=True, + ) + rc = support.run_in_subinterp_with_config(script, **config) + assert rc == 0 + + # The crash is different if the interpreter was not destroyed first. + #interpid = _testinternalcapi.create_interpreter() + #rc = _testinternalcapi.exec_interpreter(interpid, script) + #assert rc == 0 + """) + self.assertEqual(rc, 0) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py new file mode 100644 index 00000000000000..8436da7c32df10 --- /dev/null +++ b/Lib/test/test_capi/test_hash.py @@ -0,0 +1,79 @@ +import sys +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P +SIZEOF_PY_HASH_T = SIZEOF_VOID_P + + +class CAPITest(unittest.TestCase): + def test_hash_getfuncdef(self): + # Test PyHash_GetFuncDef() + hash_getfuncdef = _testcapi.hash_getfuncdef + func_def = hash_getfuncdef() + + match func_def.name: + case "fnv": + self.assertEqual(func_def.hash_bits, 8 * SIZEOF_PY_HASH_T) + self.assertEqual(func_def.seed_bits, 16 * SIZEOF_PY_HASH_T) + case "siphash13": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case "siphash24": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case _: + self.fail(f"unknown function name: {func_def.name!r}") + + # compare with sys.hash_info + hash_info = sys.hash_info + self.assertEqual(func_def.name, hash_info.algorithm) + self.assertEqual(func_def.hash_bits, hash_info.hash_bits) + self.assertEqual(func_def.seed_bits, hash_info.seed_bits) + + def test_hash_pointer(self): + # Test Py_HashPointer() + hash_pointer = _testcapi.hash_pointer + + UHASH_T_MASK = ((2 ** (8 * SIZEOF_PY_HASH_T)) - 1) + HASH_T_MAX = (2 ** (8 * SIZEOF_PY_HASH_T - 1) - 1) + + def python_hash_pointer(x): + # Py_HashPointer() rotates the pointer bits by 4 bits to the right + x = (x >> 4) | ((x & 15) << (8 * SIZEOF_VOID_P - 4)) + + # Convert unsigned uintptr_t (Py_uhash_t) to signed Py_hash_t + if HASH_T_MAX < x: + x = (~x) + 1 + x &= UHASH_T_MASK + x = (~x) + 1 + return x + + if SIZEOF_VOID_P == 8: + values = ( + 0xABCDEF1234567890, + 0x1234567890ABCDEF, + 0xFEE4ABEDD1CECA5E, + ) + else: + values = ( + 0x12345678, + 0x1234ABCD, + 0xDEADCAFE, + ) + + for value in values: + expected = python_hash_pointer(value) + with self.subTest(value=value): + self.assertEqual(hash_pointer(value), expected, + f"hash_pointer({value:x}) = " + f"{hash_pointer(value):x} != {expected:x}") + + # Py_HashPointer(NULL) returns 0 + self.assertEqual(hash_pointer(0), 0) + + # Py_HashPointer((void*)(uintptr_t)-1) doesn't return -1 but -2 + VOID_P_MAX = -1 & (2 ** (8 * SIZEOF_VOID_P) - 1) + self.assertEqual(hash_pointer(VOID_P_MAX), -2) diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py new file mode 100644 index 00000000000000..83e41205bc9d12 --- /dev/null +++ b/Lib/test/test_capi/test_list.py @@ -0,0 +1,347 @@ +import unittest +from test.support import import_helper +from collections import UserList +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN +PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX + +class ListSubclass(list): + pass + + +class DelAppend: + def __init__(self, lst, item): + self.lst = lst + self.item = item + + def __del__(self): + self.lst.append(self.item) + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyList_Check() + check = _testlimitedcapi.list_check + self.assertTrue(check([1, 2])) + self.assertTrue(check([])) + self.assertTrue(check(ListSubclass([1, 2]))) + self.assertFalse(check({1: 2})) + self.assertFalse(check((1, 2))) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + + def test_list_check_exact(self): + # Test PyList_CheckExact() + check = _testlimitedcapi.list_check_exact + self.assertTrue(check([1])) + self.assertTrue(check([])) + self.assertFalse(check(ListSubclass([1]))) + self.assertFalse(check(UserList([1, 2]))) + self.assertFalse(check({1: 2})) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_list_new(self): + # Test PyList_New() + list_new = _testlimitedcapi.list_new + lst = list_new(0) + self.assertEqual(lst, []) + self.assertIs(type(lst), list) + lst2 = list_new(0) + self.assertIsNot(lst2, lst) + self.assertRaises(SystemError, list_new, NULL) + self.assertRaises(SystemError, list_new, -1) + + def test_list_size(self): + # Test PyList_Size() + size = _testlimitedcapi.list_size + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size(ListSubclass([1, 2])), 2) + self.assertRaises(SystemError, size, UserList()) + self.assertRaises(SystemError, size, {}) + self.assertRaises(SystemError, size, 23) + self.assertRaises(SystemError, size, object()) + # CRASHES size(NULL) + + def test_list_get_size(self): + # Test PyList_GET_SIZE() + size = _testcapi.list_get_size + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size(ListSubclass([1, 2])), 2) + # CRASHES size(object()) + # CRASHES size(23) + # CRASHES size({}) + # CRASHES size(UserList()) + # CRASHES size(NULL) + + def check_list_get_item(self, getitem, exctype): + # Common test cases for PyList_GetItem() and PyList_GetItemRef() + lst = [1, 2, 3] + self.assertEqual(getitem(lst, 0), 1) + self.assertEqual(getitem(lst, 2), 3) + self.assertRaises(IndexError, getitem, lst, 3) + self.assertRaises(IndexError, getitem, lst, -1) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(exctype, getitem, 42, 1) + self.assertRaises(exctype, getitem, (1, 2, 3), 1) + self.assertRaises(exctype, getitem, {1: 2}, 1) + # CRASHES getitem(NULL, 1) + + def test_list_getitem(self): + # Test PyList_GetItem() + self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) + + def test_list_get_item_ref(self): + # Test PyList_GetItemRef() + self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) + + def test_list_get_item(self): + # Test PyList_GET_ITEM() + get_item = _testcapi.list_get_item + lst = [1, 2, [1, 2, 3]] + self.assertEqual(get_item(lst, 0), 1) + self.assertEqual(get_item(lst, 2), [1, 2, 3]) + + # CRASHES for out of index: get_item(lst, 3) + # CRASHES for get_item(lst, PY_SSIZE_T_MIN) + # CRASHES for get_item(lst, PY_SSIZE_T_MAX) + # CRASHES get_item(21, 2) + # CRASHES get_item(NULL, 1) + + def test_list_setitem(self): + # Test PyList_SetItem() + setitem = _testlimitedcapi.list_setitem + lst = [1, 2, 3] + setitem(lst, 0, 10) + self.assertEqual(lst, [10, 2, 3]) + setitem(lst, 2, 12) + self.assertEqual(lst, [10, 2, 12]) + self.assertRaises(IndexError, setitem, lst, 3 , 5) + self.assertRaises(IndexError, setitem, lst, -1, 5) + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 5) + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 5) + self.assertRaises(SystemError, setitem, (1, 2, 3), 1, 5) + self.assertRaises(SystemError, setitem, {1: 2}, 1, 5) + + # CRASHES setitem(NULL, 'a', 5) + + def test_list_set_item(self): + # Test PyList_SET_ITEM() + set_item = _testcapi.list_set_item + lst = [1, 2, 3] + set_item(lst, 1, 10) + set_item(lst, 2, [1, 2, 3]) + self.assertEqual(lst, [1, 10, [1, 2, 3]]) + + # CRASHES for set_item([1], -1, 5) + # CRASHES for set_item([1], PY_SSIZE_T_MIN, 5) + # CRASHES for set_item([1], PY_SSIZE_T_MAX, 5) + # CRASHES for set_item([], 0, 1) + # CRASHES for set_item(NULL, 0, 1) + + + def test_list_insert(self): + # Test PyList_Insert() + insert = _testlimitedcapi.list_insert + lst = [1, 2, 3] + insert(lst, 0, 23) + self.assertEqual(lst, [23, 1, 2, 3]) + insert(lst, -1, 22) + self.assertEqual(lst, [23, 1, 2, 22, 3]) + insert(lst, PY_SSIZE_T_MIN, 1) + self.assertEqual(lst[0], 1) + insert(lst, len(lst), 123) + self.assertEqual(lst[-1], 123) + insert(lst, len(lst)-1, 124) + self.assertEqual(lst[-2], 124) + insert(lst, PY_SSIZE_T_MAX, 223) + self.assertEqual(lst[-1], 223) + + self.assertRaises(SystemError, insert, (1, 2, 3), 1, 5) + self.assertRaises(SystemError, insert, {1: 2}, 1, 5) + + # CRASHES insert(NULL, 1, 5) + + def test_list_append(self): + # Test PyList_Append() + append = _testlimitedcapi.list_append + lst = [1, 2, 3] + append(lst, 10) + self.assertEqual(lst, [1, 2, 3, 10]) + append(lst, [4, 5]) + self.assertEqual(lst, [1, 2, 3, 10, [4, 5]]) + self.assertRaises(SystemError, append, lst, NULL) + self.assertRaises(SystemError, append, (), 0) + self.assertRaises(SystemError, append, 42, 0) + # CRASHES append(NULL, 0) + + def test_list_getslice(self): + # Test PyList_GetSlice() + getslice = _testlimitedcapi.list_getslice + lst = [1, 2, 3] + + # empty + self.assertEqual(getslice(lst, PY_SSIZE_T_MIN, 0), []) + self.assertEqual(getslice(lst, -1, 0), []) + self.assertEqual(getslice(lst, 3, PY_SSIZE_T_MAX), []) + + # slice + self.assertEqual(getslice(lst, 1, 3), [2, 3]) + + # whole + self.assertEqual(getslice(lst, 0, len(lst)), lst) + self.assertEqual(getslice(lst, 0, 100), lst) + self.assertEqual(getslice(lst, -100, 100), lst) + + self.assertRaises(SystemError, getslice, (1, 2, 3), 0, 0) + self.assertRaises(SystemError, getslice, 'abc', 0, 0) + self.assertRaises(SystemError, getslice, 42, 0, 0) + + # CRASHES getslice(NULL, 0, 0) + + def test_list_setslice(self): + # Test PyList_SetSlice() + list_setslice = _testlimitedcapi.list_setslice + def set_slice(lst, low, high, value): + lst = lst.copy() + self.assertEqual(list_setslice(lst, low, high, value), 0) + return lst + + # insert items + self.assertEqual(set_slice([], 0, 0, list("abc")), list("abc")) + self.assertEqual(set_slice([], PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, list("abc")), list("abc")) + self.assertEqual(set_slice([], PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, list("abc")), list("abc")) + lst = list("abc") + self.assertEqual(set_slice(lst, 0, 0, ["X"]), list("Xabc")) + self.assertEqual(set_slice(lst, 1, 1, list("XY")), list("aXYbc")) + self.assertEqual(set_slice(lst, len(lst), len(lst), ["X"]), list("abcX")) + # self.assertEqual(set_slice(lst, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, ["X"]), list("abcX")) + + # replace items + lst = list("abc") + self.assertEqual(set_slice(lst, -100, 1, list("X")), list("Xbc")) + self.assertEqual(set_slice(lst, 1, 2, list("X")), list("aXc")) + self.assertEqual(set_slice(lst, 1, 3, list("XY")), list("aXY")) + self.assertEqual(set_slice(lst, 0, 3, list("XYZ")), list("XYZ")) + + # delete items + lst = list("abcdef") + self.assertEqual(set_slice(lst, 0, len(lst), []), []) + self.assertEqual(set_slice(lst, -100, 100, []), []) + self.assertEqual(set_slice(lst, 1, 5, []), list("af")) + self.assertEqual(set_slice(lst, 3, len(lst), []), list("abc")) + + # delete items with NULL + lst = list("abcdef") + self.assertEqual(set_slice(lst, 0, len(lst), NULL), []) + self.assertEqual(set_slice(lst, 3, len(lst), NULL), list("abc")) + + self.assertRaises(SystemError, list_setslice, (), 0, 0, []) + self.assertRaises(SystemError, list_setslice, 42, 0, 0, []) + + # Item finalizer modify the list (clear the list) + lst = [] + lst.append(DelAppend(lst, 'zombie')) + self.assertEqual(list_setslice(lst, 0, len(lst), NULL), 0) + self.assertEqual(lst, ['zombie']) + + # Item finalizer modify the list (remove an list item) + lst = [] + lst.append(DelAppend(lst, 'zombie')) + lst.extend("abc") + self.assertEqual(list_setslice(lst, 0, 1, NULL), 0) + self.assertEqual(lst, ['a', 'b', 'c', 'zombie']) + + # CRASHES setslice(NULL, 0, 0, []) + + def test_list_sort(self): + # Test PyList_Sort() + sort = _testlimitedcapi.list_sort + lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] + sort(lst) + self.assertEqual(lst, list(range(10))) + + lst2 = ListSubclass([4, 6, 7, 3, 1, 5, 9, 2, 0, 8]) + sort(lst2) + self.assertEqual(lst2, list(range(10))) + + self.assertRaises(SystemError, sort, ()) + self.assertRaises(SystemError, sort, object()) + self.assertRaises(SystemError, sort, NULL) + + + def test_list_reverse(self): + # Test PyList_Reverse() + reverse = _testlimitedcapi.list_reverse + def list_reverse(lst): + self.assertEqual(reverse(lst), 0) + return lst + + self.assertEqual(list_reverse([]), []) + self.assertEqual(list_reverse([2, 5, 10]), [10, 5, 2]) + + self.assertRaises(SystemError, reverse, ()) + self.assertRaises(SystemError, reverse, object()) + self.assertRaises(SystemError, reverse, NULL) + + def test_list_astuple(self): + # Test PyList_AsTuple() + astuple = _testlimitedcapi.list_astuple + self.assertEqual(astuple([]), ()) + self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) + + self.assertRaises(SystemError, astuple, ()) + self.assertRaises(SystemError, astuple, object()) + self.assertRaises(SystemError, astuple, NULL) + + def test_list_clear(self): + # Test PyList_Clear() + list_clear = _testcapi.list_clear + + lst = [1, 2, 3] + self.assertEqual(list_clear(lst), 0) + self.assertEqual(lst, []) + + lst = [] + self.assertEqual(list_clear(lst), 0) + self.assertEqual(lst, []) + + self.assertRaises(SystemError, list_clear, ()) + self.assertRaises(SystemError, list_clear, object()) + + # Item finalizer modify the list + lst = [] + lst.append(DelAppend(lst, 'zombie')) + list_clear(lst) + self.assertEqual(lst, ['zombie']) + + # CRASHES list_clear(NULL) + + def test_list_extend(self): + # Test PyList_Extend() + list_extend = _testcapi.list_extend + + for other_type in (list, tuple, str, iter): + lst = list("ab") + arg = other_type("def") + self.assertEqual(list_extend(lst, arg), 0) + self.assertEqual(lst, list("abdef")) + + # PyList_Extend(lst, lst) + lst = list("abc") + self.assertEqual(list_extend(lst, lst), 0) + self.assertEqual(lst, list("abcabc")) + + self.assertRaises(TypeError, list_extend, [], object()) + self.assertRaises(SystemError, list_extend, (), list("abc")) + + # CRASHES list_extend(NULL, []) + # CRASHES list_extend([], NULL) diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 8e3ef25d1ff86f..7e8d571ae234d1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -1,10 +1,12 @@ import unittest import sys +import test.support as support from test.support import import_helper -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi and _testlimitedcapi modules isn't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -55,7 +57,7 @@ def test_compact_known(self): def test_long_check(self): # Test PyLong_Check() - check = _testcapi.pylong_check + check = _testlimitedcapi.pylong_check self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -67,7 +69,7 @@ def test_long_check(self): def test_long_checkexact(self): # Test PyLong_CheckExact() - check = _testcapi.pylong_checkexact + check = _testlimitedcapi.pylong_checkexact self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -79,7 +81,7 @@ def test_long_checkexact(self): def test_long_fromdouble(self): # Test PyLong_FromDouble() - fromdouble = _testcapi.pylong_fromdouble + fromdouble = _testlimitedcapi.pylong_fromdouble float_max = sys.float_info.max for value in (5.0, 5.1, 5.9, -5.1, -5.9, 0.0, -0.0, float_max, -float_max): with self.subTest(value=value): @@ -90,7 +92,7 @@ def test_long_fromdouble(self): def test_long_fromvoidptr(self): # Test PyLong_FromVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -102,7 +104,7 @@ def test_long_fromvoidptr(self): def test_long_fromstring(self): # Test PyLong_FromString() - fromstring = _testcapi.pylong_fromstring + fromstring = _testlimitedcapi.pylong_fromstring self.assertEqual(fromstring(b'123', 10), (123, 3)) self.assertEqual(fromstring(b'cafe', 16), (0xcafe, 4)) self.assertEqual(fromstring(b'xyz', 36), (44027, 3)) @@ -162,7 +164,7 @@ def test_long_fromunicodeobject(self): def test_long_asint(self): # Test PyLong_AsInt() - PyLong_AsInt = _testcapi.PyLong_AsInt + PyLong_AsInt = _testlimitedcapi.PyLong_AsInt from _testcapi import INT_MIN, INT_MAX # round trip (object -> int -> object) @@ -185,7 +187,7 @@ def test_long_asint(self): def test_long_aslong(self): # Test PyLong_AsLong() and PyLong_FromLong() - aslong = _testcapi.pylong_aslong + aslong = _testlimitedcapi.pylong_aslong from _testcapi import LONG_MIN, LONG_MAX # round trip (object -> long -> object) for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): @@ -205,7 +207,7 @@ def test_long_aslong(self): def test_long_aslongandoverflow(self): # Test PyLong_AsLongAndOverflow() - aslongandoverflow = _testcapi.pylong_aslongandoverflow + aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow from _testcapi import LONG_MIN, LONG_MAX # round trip (object -> long -> object) for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): @@ -223,7 +225,7 @@ def test_long_aslongandoverflow(self): def test_long_asunsignedlong(self): # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() - asunsignedlong = _testcapi.pylong_asunsignedlong + asunsignedlong = _testlimitedcapi.pylong_asunsignedlong from _testcapi import ULONG_MAX # round trip (object -> unsigned long -> object) for value in (ULONG_MAX, 0, 1, 1234): @@ -243,7 +245,7 @@ def test_long_asunsignedlong(self): def test_long_asunsignedlongmask(self): # Test PyLong_AsUnsignedLongMask() - asunsignedlongmask = _testcapi.pylong_asunsignedlongmask + asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask from _testcapi import ULONG_MAX # round trip (object -> unsigned long -> object) for value in (ULONG_MAX, 0, 1, 1234): @@ -263,7 +265,7 @@ def test_long_asunsignedlongmask(self): def test_long_aslonglong(self): # Test PyLong_AsLongLong() and PyLong_FromLongLong() - aslonglong = _testcapi.pylong_aslonglong + aslonglong = _testlimitedcapi.pylong_aslonglong from _testcapi import LLONG_MIN, LLONG_MAX # round trip (object -> long long -> object) for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): @@ -283,7 +285,7 @@ def test_long_aslonglong(self): def test_long_aslonglongandoverflow(self): # Test PyLong_AsLongLongAndOverflow() - aslonglongandoverflow = _testcapi.pylong_aslonglongandoverflow + aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow from _testcapi import LLONG_MIN, LLONG_MAX # round trip (object -> long long -> object) for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): @@ -301,7 +303,7 @@ def test_long_aslonglongandoverflow(self): def test_long_asunsignedlonglong(self): # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() - asunsignedlonglong = _testcapi.pylong_asunsignedlonglong + asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong from _testcapi import ULLONG_MAX # round trip (object -> unsigned long long -> object) for value in (ULLONG_MAX, 0, 1, 1234): @@ -321,7 +323,7 @@ def test_long_asunsignedlonglong(self): def test_long_asunsignedlonglongmask(self): # Test PyLong_AsUnsignedLongLongMask() - asunsignedlonglongmask = _testcapi.pylong_asunsignedlonglongmask + asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask from _testcapi import ULLONG_MAX # round trip (object -> unsigned long long -> object) for value in (ULLONG_MAX, 0, 1, 1234): @@ -341,7 +343,7 @@ def test_long_asunsignedlonglongmask(self): def test_long_as_ssize_t(self): # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() - as_ssize_t = _testcapi.pylong_as_ssize_t + as_ssize_t = _testlimitedcapi.pylong_as_ssize_t from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX # round trip (object -> Py_ssize_t -> object) for value in (PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1, 0, 1, 1234): @@ -361,7 +363,7 @@ def test_long_as_ssize_t(self): def test_long_as_size_t(self): # Test PyLong_AsSize_t() and PyLong_FromSize_t() - as_size_t = _testcapi.pylong_as_size_t + as_size_t = _testlimitedcapi.pylong_as_size_t from _testcapi import SIZE_MAX # round trip (object -> size_t -> object) for value in (SIZE_MAX, 0, 1, 1234): @@ -381,7 +383,7 @@ def test_long_as_size_t(self): def test_long_asdouble(self): # Test PyLong_AsDouble() - asdouble = _testcapi.pylong_asdouble + asdouble = _testlimitedcapi.pylong_asdouble MAX = int(sys.float_info.max) for value in (-MAX, MAX, -1, 0, 1, 1234): with self.subTest(value=value): @@ -401,8 +403,8 @@ def test_long_asdouble(self): def test_long_asvoidptr(self): # Test PyLong_AsVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr - asvoidptr = _testcapi.pylong_asvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + asvoidptr = _testlimitedcapi.pylong_asvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -423,6 +425,325 @@ def test_long_asvoidptr(self): self.assertRaises(OverflowError, asvoidptr, -2**1000) # CRASHES asvoidptr(NULL) + def _test_long_aspid(self, aspid): + # Test PyLong_AsPid() + from _testcapi import SIZEOF_PID_T + bits = 8 * SIZEOF_PID_T + PID_T_MIN = -2**(bits-1) + PID_T_MAX = 2**(bits-1) - 1 + # round trip (object -> long -> object) + for value in (PID_T_MIN, PID_T_MAX, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(aspid(value), value) + + self.assertEqual(aspid(IntSubclass(42)), 42) + self.assertEqual(aspid(Index(42)), 42) + self.assertEqual(aspid(MyIndexAndInt()), 10) + + self.assertRaises(OverflowError, aspid, PID_T_MIN - 1) + self.assertRaises(OverflowError, aspid, PID_T_MAX + 1) + self.assertRaises(TypeError, aspid, 1.0) + self.assertRaises(TypeError, aspid, b'2') + self.assertRaises(TypeError, aspid, '3') + self.assertRaises(SystemError, aspid, NULL) + + def test_long_aspid(self): + self._test_long_aspid(_testcapi.pylong_aspid) + + def test_long_aspid_limited(self): + self._test_long_aspid(_testlimitedcapi.pylong_aspid) + + def test_long_asnativebytes(self): + import math + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + if support.verbose: + print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") + + # These tests check that the requested buffer size is correct. + # This matches our current implementation: We only specify that the + # return value is a size *sufficient* to hold the result when queried + # using n_bytes=0. If our implementation changes, feel free to update + # the expectations here -- or loosen them to be range checks. + # (i.e. 0 *could* be stored in 1 byte and 512 in 2) + for v, expect in [ + (0, SZ), + (512, SZ), + (-512, SZ), + (MAX_SSIZE, SZ), + (MAX_USIZE, SZ + 1), + (-MAX_SSIZE, SZ), + (-MAX_USIZE, SZ + 1), + (2**255-1, 32), + (-(2**255-1), 32), + (2**255, 33), + (-(2**255), 33), # if you ask, we'll say 33, but 32 would do + (2**256-1, 33), + (-(2**256-1), 33), + (2**256, 33), + (-(2**256), 33), + ]: + with self.subTest(f"sizeof-{v:X}"): + buffer = bytearray(b"\x5a") + self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), + "PyLong_AsNativeBytes(v, , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), + "PyLong_AsNativeBytes(Index(v), , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + + # Test that we populate n=2 bytes but do not overwrite more. + buffer = bytearray(b"\x99"*3) + self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") + self.assertEqual(buffer, b"\x00\x04\x99") + self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") + self.assertEqual(buffer, b"\x04\x00\x99") + + # We request as many bytes as `expect_be` contains, and always check + # the result (both big and little endian). We check the return value + # independently, since the buffer should always be filled correctly even + # if we need more bytes + for v, expect_be, expect_n in [ + (0, b'\x00', 1), + (0, b'\x00' * 2, 2), + (0, b'\x00' * 8, min(8, SZ)), + (1, b'\x01', 1), + (1, b'\x00' * 10 + b'\x01', min(11, SZ)), + (42, b'\x2a', 1), + (42, b'\x00' * 10 + b'\x2a', min(11, SZ)), + (-1, b'\xff', 1), + (-1, b'\xff' * 10, min(11, SZ)), + (-42, b'\xd6', 1), + (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), + # Extracts 255 into a single byte, but requests 2 + # (this is currently a special case, and "should" request SZ) + (255, b'\xff', 2), + (255, b'\x00\xff', 2), + (256, b'\x01\x00', 2), + (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), + # Extracts successfully (unsigned), but requests 9 bytes + (2**63, b'\x80' + b'\x00' * 7, 9), + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + # Extracts into 8 bytes, but if you provide 9 we'll say 9 + (-2**63, b'\x80' + b'\x00' * 7, 8), + (-2**63, b'\xff\x80' + b'\x00' * 7, 9), + + (2**255-1, b'\x7f' + b'\xff' * 31, 32), + (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32), + # Request extra bytes, but result says we only needed 32 + (-(2**255-1), b'\xff\x80' + b'\x00' * 30 + b'\x01', 32), + (-(2**255-1), b'\xff\xff\x80' + b'\x00' * 30 + b'\x01', 32), + + # Extracting 256 bits of integer will request 33 bytes, but still + # copy as many bits as possible into the buffer. So we *can* copy + # into a 32-byte buffer, though negative number may be unrecoverable + (2**256-1, b'\xff' * 32, 33), + (2**256-1, b'\x00' + b'\xff' * 32, 33), + (-(2**256-1), b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + # However, -2**255 precisely will extract into 32 bytes and return + # success. For bigger buffers, it will still succeed, but will + # return 33 + (-(2**255), b'\x80' + b'\x00' * 31, 32), + (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), + + # The classic "Windows HRESULT as negative number" case + # HRESULT hr; + # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) + # assert(hr == E_FAIL) + (-2147467259, b'\x80\x00\x40\x05', 4), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes"): + n = len(expect_be) + # Fill the buffer with dummy data to ensure all bytes + # are overwritten. + buffer = bytearray(b"\xa5"*n) + expect_le = expect_be[::-1] + + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_be, buffer[:n], "") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 1), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_le, buffer[:n], "") + + # Test cases that do not request size for a sign bit when we pass the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag + for v, expect_be, expect_n in [ + (255, b'\xff', 1), + # We pass a 2 byte buffer so it just uses the whole thing + (255, b'\x00\xff', 2), + + (2**63, b'\x80' + b'\x00' * 7, 8), + # We pass a 9 byte buffer so it uses the whole thing + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + + (2**256-1, b'\xff' * 32, 32), + # We pass a 33 byte buffer so it uses the whole thing + (2**256-1, b'\x00' + b'\xff' * 32, 33), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): + n = len(expect_be) + buffer = bytearray(b"\xa5"*n) + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + + # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value + with self.assertRaises(ValueError): + asnativebytes(-1, buffer, 0, 8) + + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + + # Check a few error conditions. These are validated in code, but are + # unspecified in docs, so if we make changes to the implementation, it's + # fine to just update these tests rather than preserve the behaviour. + with self.assertRaises(TypeError): + asnativebytes('not a number', buffer, 0, -1) + + def test_long_asnativebytes_fuzz(self): + import math + from random import Random + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + + rng = Random() + # Allocate bigger buffer than actual values are going to be + buffer = bytearray(260) + + for _ in range(1000): + n = rng.randrange(1, 256) + bytes_be = bytes([ + # Ensure the most significant byte is nonzero + rng.randrange(1, 256), + *[rng.randrange(256) for _ in range(n - 1)] + ]) + bytes_le = bytes_be[::-1] + v = int.from_bytes(bytes_le, 'little') + + expect_1 = expect_2 = (SZ, n) + if bytes_be[0] & 0x80: + # All values are positive, so if MSB is set, expect extra bit + # when we request the size or have a large enough buffer + expect_1 = (SZ, n + 1) + # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the + # return to be exactly the right size. + expect_2 = (n,) + + try: + actual = asnativebytes(v, buffer, 0, -1) + self.assertIn(actual, expect_1) + + actual = asnativebytes(v, buffer, len(buffer), 0) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_be, buffer[-n:]) + + actual = asnativebytes(v, buffer, len(buffer), 1) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_le, buffer[:n]) + + actual = asnativebytes(v, buffer, n, 4) + self.assertIn(actual, expect_2, bytes_be.hex()) + actual = asnativebytes(v, buffer, n, 5) + self.assertIn(actual, expect_2, bytes_be.hex()) + except AssertionError as ex: + value_hex = ''.join(reversed([ + f'{b:02X}{"" if i % 8 else "_"}' + for i, b in enumerate(bytes_le, start=1) + ])).strip('_') + if support.verbose: + print() + print(n, 'bytes') + print('hex =', value_hex) + print('int =', v) + raise + raise AssertionError(f"Value: 0x{value_hex}") from ex + + def test_long_fromnativebytes(self): + import math + from _testcapi import ( + pylong_fromnativebytes as fromnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + + for v_be, expect_s, expect_u in [ + (b'\x00', 0, 0), + (b'\x01', 1, 1), + (b'\xff', -1, 255), + (b'\x00\xff', 255, 255), + (b'\xff\xff', -1, 65535), + ]: + with self.subTest(f"{expect_s}-{expect_u:X}-{len(v_be)}bytes"): + n = len(v_be) + v_le = v_be[::-1] + + self.assertEqual(expect_s, fromnativebytes(v_be, n, 0, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_s, fromnativebytes(v_le, n, 1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, 0, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_le, n, 1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Check native endian when the result would be the same either + # way and we can test it. + if v_be == v_le: + self.assertEqual(expect_s, fromnativebytes(v_be, n, -1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Swap the unsigned request for tests and use the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead + self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + + def test_long_getsign(self): + # Test PyLong_GetSign() + getsign = _testcapi.pylong_getsign + self.assertEqual(getsign(1), 1) + self.assertEqual(getsign(123456), 1) + self.assertEqual(getsign(-2), -1) + self.assertEqual(getsign(0), 0) + self.assertEqual(getsign(True), 1) + self.assertEqual(getsign(IntSubclass(-11)), -1) + self.assertEqual(getsign(False), 0) + + self.assertRaises(TypeError, getsign, 1.0) + self.assertRaises(TypeError, getsign, Index(123)) + + # CRASHES getsign(NULL) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 72f23b1a34080e..6ab7b685c2e18b 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -148,10 +148,12 @@ class C(): pass self.assertIn(b'MemoryError', out) *_, count = line.split(b' ') count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) +# free-threading requires mimalloc (not malloc) +@support.requires_gil_enabled() class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' @@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'pymalloc_debug' +@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') +class PyMemMimallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'mimalloc_debug' + + @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') class PyMemDefaultTests(PyMemDebugTests): # test default allocator of Python compiled in debug mode diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index b3f32d860eee03..9de97c0c2c776a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -7,7 +7,6 @@ import importlib.machinery import importlib.util import json -import opcode import os import pickle import queue @@ -18,7 +17,6 @@ import time import types import unittest -import warnings import weakref import operator from test import support @@ -27,6 +25,9 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api +from test.support import suppress_immortalization +from test.support import expected_failure_if_gil_disabled +from test.support import Py_GIL_DISABLED from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess @@ -41,13 +42,14 @@ except ImportError: _testsinglephase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +import _testlimitedcapi import _testinternalcapi @@ -470,6 +472,8 @@ def __del__(self): del L self.assertEqual(PyList.num, 0) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_heap_ctype_doc_and_text_signature(self): self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") @@ -477,6 +481,7 @@ def test_heap_ctype_doc_and_text_signature(self): def test_null_type_doc(self): self.assertEqual(_testcapi.NullTpDocType.__doc__, None) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): class HeapGcCTypeSubclass(_testcapi.HeapGcCType): def __init__(self): @@ -494,6 +499,7 @@ def __init__(self): del subclass_instance self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass)) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): class A(_testcapi.HeapGcCType): def __init__(self): @@ -770,33 +776,11 @@ def test_pytype_fromspec_with_repeated_slots(self): with self.assertRaises(SystemError): _testcapi.create_type_from_repeated_slots(variant) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_immutable_type_with_mutable_base(self): - # Add deprecation warning here so it's removed in 3.14 - warnings._deprecated( - 'creating immutable classes with mutable bases', remove=(3, 14)) - - class MutableBase: - def meth(self): - return 'original' - - with self.assertWarns(DeprecationWarning): - ImmutableSubclass = _testcapi.make_immutable_type_with_base( - MutableBase) - instance = ImmutableSubclass() - - self.assertEqual(instance.meth(), 'original') - - # Cannot override the static type's method - with self.assertRaisesRegex( - TypeError, - "cannot set 'meth' attribute of immutable type"): - ImmutableSubclass.meth = lambda self: 'overridden' - self.assertEqual(instance.meth(), 'original') + class MutableBase: ... - # Can change the method on the mutable base - MutableBase.meth = lambda self: 'changed' - self.assertEqual(instance.meth(), 'changed') + with self.assertRaisesRegex(TypeError, 'Creating immutable type'): + _testcapi.make_immutable_type_with_base(MutableBase) def test_pynumber_tobase(self): from _testcapi import pynumber_tobase @@ -1098,45 +1082,80 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) - def test_sys_getobject(self): - getobject = _testcapi.sys_getobject - - self.assertIs(getobject(b'stdout'), sys.stdout) - with support.swap_attr(sys, '\U0001f40d', 42): - self.assertEqual(getobject('\U0001f40d'.encode()), 42) - - self.assertIs(getobject(b'nonexisting'), AttributeError) - self.assertIs(getobject(b'\xff'), AttributeError) - # CRASHES getobject(NULL) + def test_get_type_name(self): + class MyType: + pass - def test_sys_setobject(self): - setobject = _testcapi.sys_setobject + from _testcapi import ( + get_type_name, get_type_qualname, + get_type_fullyqualname, get_type_module_name) + + from collections import OrderedDict + ht = _testcapi.get_heaptype_for_name() + for cls, fullname, modname, qualname, name in ( + (int, + 'int', + 'builtins', + 'int', + 'int'), + (OrderedDict, + 'collections.OrderedDict', + 'collections', + 'OrderedDict', + 'OrderedDict'), + (ht, + '_testcapi.HeapTypeNameType', + '_testcapi', + 'HeapTypeNameType', + 'HeapTypeNameType'), + (MyType, + f'{__name__}.CAPITest.test_get_type_name..MyType', + __name__, + 'CAPITest.test_get_type_name..MyType', + 'MyType'), + ): + with self.subTest(cls=repr(cls)): + self.assertEqual(get_type_fullyqualname(cls), fullname) + self.assertEqual(get_type_module_name(cls), modname) + self.assertEqual(get_type_qualname(cls), qualname) + self.assertEqual(get_type_name(cls), name) + + # override __module__ + ht.__module__ = 'test_module' + self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') + self.assertEqual(get_type_module_name(ht), 'test_module') + self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') + self.assertEqual(get_type_name(ht), 'HeapTypeNameType') + + # override __name__ and __qualname__ + MyType.__name__ = 'my_name' + MyType.__qualname__ = 'my_qualname' + self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') + self.assertEqual(get_type_module_name(MyType), __name__) + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # override also __module__ + MyType.__module__ = 'my_module' + self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') + self.assertEqual(get_type_module_name(MyType), 'my_module') + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" + # or "__main__" of it is not a string + MyType.__module__ = 'builtins' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = '__main__' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = 123 + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') - value = ['value'] - value2 = ['value2'] - try: - self.assertEqual(setobject(b'newattr', value), 0) - self.assertIs(sys.newattr, value) - self.assertEqual(setobject(b'newattr', value2), 0) - self.assertIs(sys.newattr, value2) - self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertFalse(hasattr(sys, 'newattr')) - self.assertEqual(setobject(b'newattr', NULL), 0) - finally: - with contextlib.suppress(AttributeError): - del sys.newattr - try: - self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) - self.assertIs(getattr(sys, '\U0001f40d'), value) - self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertFalse(hasattr(sys, '\U0001f40d')) - finally: - with contextlib.suppress(AttributeError): - delattr(sys, '\U0001f40d') - with self.assertRaises(UnicodeDecodeError): - setobject(b'\xff', value) - # CRASHES setobject(NULL, value) + def test_gen_get_code(self): + def genf(): yield + gen = genf() + self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) @requires_limited_api @@ -1148,7 +1167,7 @@ def test_heaptype_relative_sizes(self): # Test subclassing using "relative" basicsize, see PEP 697 def check(extra_base_size, extra_size): Base, Sub, instance, data_ptr, data_offset, data_size = ( - _testcapi.make_sized_heaptypes( + _testlimitedcapi.make_sized_heaptypes( extra_base_size, -extra_size)) # no alignment shenanigans when inheriting directly @@ -1176,11 +1195,11 @@ def check(extra_base_size, extra_size): # we don't reserve (requested + alignment) or more data self.assertLess(data_size - extra_size, - _testcapi.ALIGNOF_MAX_ALIGN_T) + _testlimitedcapi.ALIGNOF_MAX_ALIGN_T) # The offsets/sizes we calculated should be aligned. - self.assertEqual(data_offset % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) - self.assertEqual(data_size % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_offset % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, object.__basicsize__, @@ -1206,7 +1225,7 @@ def test_heaptype_inherit_itemsize(self): object.__basicsize__+1}) for extra_size in sizes: with self.subTest(extra_size=extra_size): - Sub = _testcapi.subclass_var_heaptype( + Sub = _testlimitedcapi.subclass_var_heaptype( _testcapi.HeapCCollection, -extra_size, 0, 0) collection = Sub(1, 2, 3) collection.set_data_to_3s() @@ -1220,7 +1239,7 @@ def test_heaptype_invalid_inheritance(self): with self.assertRaises(SystemError, msg="Cannot extend variable-size class without " + "Py_TPFLAGS_ITEMS_AT_END"): - _testcapi.subclass_heaptype(int, -8, 0) + _testlimitedcapi.subclass_heaptype(int, -8, 0) def test_heaptype_relative_members(self): """Test HeapCCollection subclasses work properly""" @@ -1233,7 +1252,7 @@ def test_heaptype_relative_members(self): for offset in sizes: with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): if offset < extra_size: - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) Base = Sub.mro()[1] instance = Sub() @@ -1252,29 +1271,29 @@ def test_heaptype_relative_members(self): instance.set_memb_relative(0) else: with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, extra_size, offset, True) with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, -1, True) def test_heaptype_relative_members_errors(self): with self.assertRaisesRegex( SystemError, r"With Py_RELATIVE_OFFSET, basicsize must be negative"): - _testcapi.make_heaptype_with_member(0, 1234, 0, True) + _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, 1234, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, -1, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) - Sub = _testcapi.make_heaptype_with_member(0, -8, 0, True) + Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) instance = Sub() with self.assertRaisesRegex( SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): @@ -1294,13 +1313,132 @@ def test_pyobject_getitemdata_error(self): _testcapi.pyobject_getitemdata(0) + def test_function_get_closure(self): + from types import CellType + + def regular_function(): ... + def unused_one_level(arg1): + def inner(arg2, arg3): ... + return inner + def unused_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): ... + return inner + return decorator + def with_one_level(arg1): + def inner(arg2, arg3): + return arg1 + arg2 + arg3 + return inner + def with_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): + return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + return inner + return decorator + + # Functions without closures: + self.assertIsNone(_testcapi.function_get_closure(regular_function)) + self.assertIsNone(regular_function.__closure__) + + func = unused_one_level(1) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + func = unused_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + # Functions with closures: + func = with_one_level(5) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 1) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertTrue(closure[0].cell_contents, 5) + + func = with_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 4) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertEqual([cell.cell_contents for cell in closure], + [1, 2, 3, 4]) + + def test_function_get_closure_error(self): + with self.assertRaises(SystemError): + _testcapi.function_get_closure(1) + with self.assertRaises(SystemError): + _testcapi.function_get_closure(None) + + def test_function_set_closure(self): + from types import CellType + + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + func = function_without_closure + _testcapi.function_set_closure(func, (CellType(1), CellType(1))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 1]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) + + func = function_with_closure(1) + _testcapi.function_set_closure(func, + (CellType(1), CellType(2), CellType(3))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) + + def test_function_set_closure_none(self): + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + _testcapi.function_set_closure(function_without_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_without_closure)) + self.assertIsNone(function_without_closure.__closure__) + + _testcapi.function_set_closure(function_with_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_with_closure)) + self.assertIsNone(function_with_closure.__closure__) + + def test_function_set_closure_errors(self): + def function_without_closure(): ... + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(None, ()) # not a function + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(function_without_closure, 1) + self.assertIsNone(function_without_closure.__closure__) # no change + + # NOTE: this works, but goes against the docs: + _testcapi.function_set_closure(function_without_closure, (1, 2)) + self.assertEqual( + _testcapi.function_get_closure(function_without_closure), (1, 2)) + self.assertEqual(function_without_closure.__closure__, (1, 2)) + + class TestPendingCalls(unittest.TestCase): # See the comment in ceval.c (at the "handle_eval_breaker" label) # about when pending calls get run. This is especially relevant # here for creating deterministic tests. - def pendingcalls_submit(self, l, n): + def main_pendingcalls_submit(self, l, n): def callback(): #this function can be interrupted by thread switching so let's #use an atomic operation @@ -1315,12 +1453,27 @@ def callback(): if _testcapi._pending_threadfunc(callback): break - def pendingcalls_wait(self, l, n, context = None): + def pendingcalls_submit(self, l, n, *, main=True, ensure=False): + def callback(): + #this function can be interrupted by thread switching so let's + #use an atomic operation + l.append(None) + + if main: + return _testcapi._pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + else: + return _testinternalcapi.pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + + def pendingcalls_wait(self, l, numadded, context = None): #now, stick around until l[0] has grown to 10 count = 0 - while len(l) != n: + while len(l) != numadded: #this busy loop is where we expect to be interrupted to - #run our callbacks. Note that callbacks are only run on the + #run our callbacks. Note that some callbacks are only run on the #main thread if False and support.verbose: print("(%i)"%(len(l),),) @@ -1330,12 +1483,12 @@ def pendingcalls_wait(self, l, n, context = None): continue count += 1 self.assertTrue(count < 10000, - "timeout waiting for %i callbacks, got %i"%(n, len(l))) + "timeout waiting for %i callbacks, got %i"%(numadded, len(l))) if False and support.verbose: print("(%i)"%(len(l),)) @threading_helper.requires_working_threading() - def test_pendingcalls_threaded(self): + def test_main_pendingcalls_threaded(self): #do every callback on a separate thread n = 32 #total callbacks @@ -1349,15 +1502,15 @@ class foo(object):pass context.lock = threading.Lock() context.event = threading.Event() - threads = [threading.Thread(target=self.pendingcalls_thread, + threads = [threading.Thread(target=self.main_pendingcalls_thread, args=(context,)) for i in range(context.nThreads)] with threading_helper.start_threads(threads): self.pendingcalls_wait(context.l, n, context) - def pendingcalls_thread(self, context): + def main_pendingcalls_thread(self, context): try: - self.pendingcalls_submit(context.l, context.n) + self.main_pendingcalls_submit(context.l, context.n) finally: with context.lock: context.nFinished += 1 @@ -1367,20 +1520,54 @@ def pendingcalls_thread(self, context): if nFinished == context.nThreads: context.event.set() - def test_pendingcalls_non_threaded(self): + def test_main_pendingcalls_non_threaded(self): #again, just using the main thread, likely they will all be dispatched at #once. It is ok to ask for too many, because we loop until we find a slot. #the loop can be interrupted to dispatch. #there are only 32 dispatch slots, so we go for twice that! l = [] n = 64 - self.pendingcalls_submit(l, n) + self.main_pendingcalls_submit(l, n) self.pendingcalls_wait(l, n) - def test_gen_get_code(self): - def genf(): yield - gen = genf() - self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + def test_max_pending(self): + with self.subTest('main-only'): + maxpending = 32 + + l = [] + added = self.pendingcalls_submit(l, 1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + with self.subTest('not main-only'): + # Per-interpreter pending calls has a much higher limit + # on how many may be pending at a time. + maxpending = 300 + + l = [] + added = self.pendingcalls_submit(l, 1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) class PendingTask(types.SimpleNamespace): @@ -1552,6 +1739,7 @@ def test_isolated_subinterpreter(self): maxtext = 250 main_interpid = 0 interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) _interpreters.run_string(interpid, f"""if True: import json import os @@ -1819,6 +2007,7 @@ def test_py_config_isoloated_per_interpreter(self): # double checked at the time this test was written. config = _testinternalcapi.get_config() config['int_max_str_digits'] = 55555 + config['parse_argv'] = 0 _testinternalcapi.set_config(config) sub_value = _testinternalcapi.get_config()['int_max_str_digits'] assert sub_value == 55555, sub_value @@ -1871,15 +2060,30 @@ def test_configured_settings(self): kwlist[-2] = 'check_multi_interp_extensions' kwlist[-1] = 'own_gil' - # expected to work - for config, expected in { + expected_to_work = { (True, True, True, True, True, True, True): (ALL_FLAGS, True), (True, False, False, False, False, False, False): (OBMALLOC, False), (False, False, False, True, False, True, False): (THREADS | EXTENSIONS, False), - }.items(): + } + + expected_to_fail = { + (False, False, False, False, False, False, False), + } + + # gh-117649: The free-threaded build does not currently allow + # setting check_multi_interp_extensions to False. + if Py_GIL_DISABLED: + for config in list(expected_to_work.keys()): + kwargs = dict(zip(kwlist, config)) + if not kwargs['check_multi_interp_extensions']: + del expected_to_work[config] + expected_to_fail.add(config) + + # expected to work + for config, expected in expected_to_work.items(): kwargs = dict(zip(kwlist, config)) exp_flags, exp_gil = expected expected = { @@ -1903,9 +2107,7 @@ def test_configured_settings(self): self.assertEqual(settings, expected) # expected to fail - for config in [ - (False, False, False, False, False, False, False), - ]: + for config in expected_to_fail: kwargs = dict(zip(kwlist, config)) with self.subTest(config): script = textwrap.dedent(f''' @@ -1913,11 +2115,14 @@ def test_configured_settings(self): _testinternalcapi.get_interp_settings() raise NotImplementedError('unreachable') ''') - with self.assertRaises(RuntimeError): + with self.assertRaises(_interpreters.InterpreterError): support.run_in_subinterp_with_config(script, **kwargs) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + # gh-117649: The free-threaded build does not currently allow overriding + # the check_multi_interp_extensions setting. + @expected_failure_if_gil_disabled() def test_overridden_setting_extensions_subinterp_check(self): """ PyInterpreterConfig.check_multi_interp_extensions can be overridden @@ -1969,6 +2174,9 @@ def check(enabled, override): } r, w = os.pipe() + if Py_GIL_DISABLED: + # gh-117649: The test fails before `w` is closed + self.addCleanup(os.close, w) script = textwrap.dedent(f''' from test.test_capi.check_config import run_singlephase_check run_singlephase_check({override}, {w}) @@ -2013,6 +2221,9 @@ def test_mutate_exception(self): self.assertFalse(hasattr(binascii.Error, "foobar")) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + # gh-117649: The free-threaded build does not currently support sharing + # extension module state between interpreters. + @expected_failure_if_gil_disabled() def test_module_state_shared_in_global(self): """ bpo-44050: Extension module state should be shared between interpreters @@ -2022,6 +2233,13 @@ def test_module_state_shared_in_global(self): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2029,7 +2247,7 @@ def test_module_state_shared_in_global(self): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2045,6 +2263,516 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _interpreters.new_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _interpreters.new_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + if Py_GIL_DISABLED: + invalid.append(dict(check_multi_interp_extensions=False)) + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(_interpreters.InterpreterError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _interpreters.create(config, reqrefs=False) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + with new_interp('isolated') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + with new_interp('legacy') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + ) + with new_interp(orig) as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + def add_interp_cleanup(self, interpid): + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + + def new_interpreter(self): + id = _interpreters.create() + self.add_interp_cleanup(id) + return id + + def test_conversion_int(self): + convert = _testinternalcapi.normalize_interp_id + interpid = convert(10) + self.assertEqual(interpid, 10) + + def test_conversion_coerced(self): + convert = _testinternalcapi.normalize_interp_id + class MyInt(str): + def __index__(self): + return 10 + interpid = convert(MyInt()) + self.assertEqual(interpid, 10) + + def test_conversion_from_interpreter(self): + convert = _testinternalcapi.normalize_interp_id + interpid = self.new_interpreter() + converted = convert(interpid) + self.assertEqual(converted, interpid) + + def test_conversion_bad(self): + convert = _testinternalcapi.normalize_interp_id + + for badid in [ + object(), + 10.0, + '10', + b'10', + ]: + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(TypeError): + convert(badid) + + badid = -1 + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(ValueError): + convert(badid) + + badid = 2**64 + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(OverflowError): + convert(badid) + + def test_lookup_exists(self): + interpid = self.new_interpreter() + self.assertTrue( + _testinternalcapi.interpreter_exists(interpid)) + + def test_lookup_does_not_exist(self): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def test_lookup_destroyed(self): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + + def test_linked_lifecycle_does_not_exist(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + with self.subTest('never existed'): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + with self.subTest('destroyed'): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + def test_linked_lifecycle_initial(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, _, _ = self.get_refcount_helpers() + + # A new interpreter will start out not linked, with a refcount of 0. + interpid = self.new_interpreter() + linked = is_linked(interpid) + refcount = get_refcount(interpid) + + self.assertFalse(linked) + self.assertEqual(refcount, 0) + + def test_linked_lifecycle_never_linked(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Incref will not automatically link it. + incref(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # It isn't linked so it isn't destroyed. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + def test_linked_lifecycle_link_unlink(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + + interpid = self.new_interpreter() + + # Linking at refcount 0 does not destroy the interpreter. + link(interpid) + self.assertTrue( + exists(interpid)) + self.assertTrue( + is_linked(interpid)) + + # Unlinking at refcount 0 does not destroy the interpreter. + unlink(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + + def test_linked_lifecycle_link_incref_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Linking it will not change the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + # Decref with a refcount of 0 is not allowed. + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # When linked, decref back to 0 destroys the interpreter. + decref(interpid) + self.assertFalse( + exists(interpid)) + + def test_linked_lifecycle_incref_link(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, _ = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # Linking it will not reset the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + def test_linked_lifecycle_link_incref_unlink_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + link(interpid) + self.assertTrue( + is_linked(interpid)) + + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinking it will not change the refcount. + unlink(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinked: decref back to 0 does not destroys the interpreter. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + class BuiltinStaticTypesTests(unittest.TestCase): TYPES = [ @@ -2137,6 +2865,22 @@ def callback(): t.start() t.join() + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_gilstate_in_clear(self): + # See https://github.com/python/cpython/issues/119585 + class C: + def __del__(self): + _testcapi.gilstate_ensure_release() + + # Thread-local variables are destroyed in `PyThreadState_Clear()`. + local_var = threading.local() + + def callback(): + local_var.x = C() + + _testcapi._test_thread_state(callback) + @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_gilstate_ensure_no_deadlock(self): @@ -2156,25 +2900,36 @@ def test_gilstate_matches_current(self): _testcapi.test_current_tstate_matches() +def get_test_funcs(mod, exclude_prefix=None): + funcs = {} + for name in dir(mod): + if not name.startswith('test_'): + continue + if exclude_prefix is not None and name.startswith(exclude_prefix): + continue + funcs[name] = getattr(mod, name) + return funcs + + class Test_testcapi(unittest.TestCase): - locals().update((name, getattr(_testcapi, name)) - for name in dir(_testcapi) - if name.startswith('test_') and not name.endswith('_code')) + locals().update(get_test_funcs(_testcapi)) # Suppress warning from PyUnicode_FromUnicode(). @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_widechar(self): - _testcapi.test_widechar() + _testlimitedcapi.test_widechar() def test_version_api_data(self): self.assertEqual(_testcapi.Py_Version, sys.hexversion) +class Test_testlimitedcapi(unittest.TestCase): + locals().update(get_test_funcs(_testlimitedcapi)) + + class Test_testinternalcapi(unittest.TestCase): - locals().update((name, getattr(_testinternalcapi, name)) - for name in dir(_testinternalcapi) - if name.startswith('test_') - and not name.startswith('test_lock_')) + locals().update(get_test_funcs(_testinternalcapi, + exclude_prefix='test_lock_')) @threading_helper.requires_working_threading() @@ -2195,7 +2950,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -2376,416 +3136,59 @@ def func(): self.do_test(func, names) -@contextlib.contextmanager -def temporary_optimizer(opt): - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(opt) - try: - yield - finally: - _testinternalcapi.set_optimizer(old_opt) +@unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') +class TestPyThreadId(unittest.TestCase): + def test_py_thread_id(self): + # gh-112535: Test _Py_ThreadId(): make sure that thread identifiers + # in a few threads are unique + py_thread_id = _testinternalcapi.py_thread_id + short_sleep = 0.010 - -@contextlib.contextmanager -def clear_executors(func): - # Clear executors in func before and after running a block - func.__code__ = func.__code__.replace() - try: - yield - finally: - func.__code__ = func.__code__.replace() - - -class TestOptimizerAPI(unittest.TestCase): - - def test_get_counter_optimizer_dealloc(self): - # See gh-108727 - def f(): - _testinternalcapi.get_counter_optimizer() - - f() - - def test_get_set_optimizer(self): - old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() - try: - _testinternalcapi.set_optimizer(opt) - self.assertEqual(_testinternalcapi.get_optimizer(), opt) - _testinternalcapi.set_optimizer(None) - self.assertEqual(_testinternalcapi.get_optimizer(), None) - finally: - _testinternalcapi.set_optimizer(old) - - - def test_counter_optimizer(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def loop(): - for _ in range(1000): - pass - """), ns, ns) - loop = ns['loop'] - - for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - with clear_executors(loop): - loop() - self.assertEqual(opt.get_count(), 1000) - - def test_long_loop(self): - "Check that we aren't confused by EXTENDED_ARG" - - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def nop(): - pass - - def long_loop(): - for _ in range(10): - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - """), ns, ns) - long_loop = ns['long_loop'] - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - long_loop() - self.assertEqual(opt.get_count(), 10) - - def test_code_restore_for_ENTER_EXECUTOR(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - code, replace_code = testfunc.__code__, testfunc.__code__.replace() - self.assertEqual(code, replace_code) - self.assertEqual(hash(code), hash(replace_code)) - - -def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] - for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass - return None - - -class TestUops(unittest.TestCase): - - def test_basic_loop(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_SET_IP", uops) - self.assertIn("LOAD_FAST", uops) - - def test_extended_arg(self): - "Check EXTENDED_ARG handling in superblock creation" - ns = {} - exec(textwrap.dedent(""" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 - """), ns, ns) - many_vars = ns["many_vars"] - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - ex = get_first_executor(many_vars) - self.assertIsNone(ex) - many_vars() - - ex = get_first_executor(many_vars) - self.assertIsNotNone(ex) - self.assertIn(("LOAD_FAST", 259, 0), list(ex)) - - def test_unspecialized_unpack(self): - # An example of an unspecialized opcode - def testfunc(x): - i = 0 - while i < x: - i += 1 - a, b = {1: 2, 3: 3} - assert a == 1 and b == 3 - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("UNPACK_SEQUENCE", uops) - - def test_pop_jump_if_false(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_FALSE", uops) - - def test_pop_jump_if_none(self): - def testfunc(a): - for x in a: - if x is None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) - - def test_pop_jump_if_not_none(self): - def testfunc(a): - for x in a: - x = None - if x is not None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_FALSE", uops) - - def test_pop_jump_if_true(self): - def testfunc(n): - i = 0 - while not i >= n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) - - def test_jump_backward(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_JUMP_TO_TOP", uops) - - def test_jump_forward(self): - def testfunc(n): - a = 0 - while a < n: - if a < 0: - a = -a - else: - a = +a - a += 1 - return a - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - # Since there is no JUMP_FORWARD instruction, - # look for indirect evidence: the += operator - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_for_iter_range(self): - def testfunc(n): - total = 0 - for i in range(n): - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - total = testfunc(20) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_IS_ITER_EXHAUSTED_RANGE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_list(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = list(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_IS_ITER_EXHAUSTED_LIST", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_tuple(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = tuple(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_IS_ITER_EXHAUSTED_TUPLE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_list_edge_case(self): - def testfunc(it): - for x in it: - pass - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = [1, 2, 3] - it = iter(a) - testfunc(it) - a.append(4) - with self.assertRaises(StopIteration): - next(it) - - def test_call_py_exact_args(self): - def testfunc(n): - def dummy(x): - return x+1 - for i in range(n): - dummy(i) - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_PUSH_FRAME", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_branch_taken(self): - def testfunc(n): - for i in range(n): - if i < 0: - i = 0 - else: - i = 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_POP_JUMP_IF_TRUE", uops) + class GetThreadId(threading.Thread): + def __init__(self): + super().__init__() + self.get_lock = threading.Lock() + self.get_lock.acquire() + self.started_lock = threading.Event() + self.py_tid = None + + def run(self): + self.started_lock.set() + self.get_lock.acquire() + self.py_tid = py_thread_id() + time.sleep(short_sleep) + self.py_tid2 = py_thread_id() + + nthread = 5 + threads = [GetThreadId() for _ in range(nthread)] + + # first make run sure that all threads are running + for thread in threads: + thread.start() + for thread in threads: + thread.started_lock.wait() + + # call _Py_ThreadId() in the main thread + py_thread_ids = [py_thread_id()] + + # now call _Py_ThreadId() in each thread + for thread in threads: + thread.get_lock.release() + + # call _Py_ThreadId() in each thread and wait until threads complete + for thread in threads: + thread.join() + py_thread_ids.append(thread.py_tid) + # _PyThread_Id() should not change for a given thread. + # For example, it should remain the same after a short sleep. + self.assertEqual(thread.py_tid2, thread.py_tid) + + # make sure that all _Py_ThreadId() are unique + for tid in py_thread_ids: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), + py_thread_ids) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py new file mode 100644 index 00000000000000..cc9c9b688f00e2 --- /dev/null +++ b/Lib/test/test_capi/test_object.py @@ -0,0 +1,135 @@ +import enum +import unittest +from test.support import import_helper +from test.support import os_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') + + +class Constant(enum.IntEnum): + Py_CONSTANT_NONE = 0 + Py_CONSTANT_FALSE = 1 + Py_CONSTANT_TRUE = 2 + Py_CONSTANT_ELLIPSIS = 3 + Py_CONSTANT_NOT_IMPLEMENTED = 4 + Py_CONSTANT_ZERO = 5 + Py_CONSTANT_ONE = 6 + Py_CONSTANT_EMPTY_STR = 7 + Py_CONSTANT_EMPTY_BYTES = 8 + Py_CONSTANT_EMPTY_TUPLE = 9 + + INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 + + +class GetConstantTest(unittest.TestCase): + def check_get_constant(self, get_constant): + self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) + self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) + self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) + self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) + self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) + + for constant_id, constant_type, value in ( + (Constant.Py_CONSTANT_ZERO, int, 0), + (Constant.Py_CONSTANT_ONE, int, 1), + (Constant.Py_CONSTANT_EMPTY_STR, str, ""), + (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), + (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), + ): + with self.subTest(constant_id=constant_id): + obj = get_constant(constant_id) + self.assertEqual(type(obj), constant_type, obj) + self.assertEqual(obj, value) + + with self.assertRaises(SystemError): + get_constant(Constant.INVALID_CONSTANT) + + def test_get_constant(self): + self.check_get_constant(_testlimitedcapi.get_constant) + + def test_get_constant_borrowed(self): + self.check_get_constant(_testlimitedcapi.get_constant_borrowed) + + +class PrintTest(unittest.TestCase): + def testPyObjectPrintObject(self): + + class PrintableObject: + + def __repr__(self): + return "spam spam spam" + + def __str__(self): + return "egg egg egg" + + obj = PrintableObject() + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.call_pyobject_print(obj, output_filename, False) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), repr(obj)) + + # Test str printing + _testcapi.call_pyobject_print(obj, output_filename, True) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), str(obj)) + + def testPyObjectPrintNULL(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.pyobject_print_null(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), '') + + def testPyObjectPrintNoRefObject(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + correct_output = _testcapi.pyobject_print_noref_object(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), correct_output) + + def testPyObjectPrintOSError(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + open(output_filename, "w+").close() + with self.assertRaises(OSError): + _testcapi.pyobject_print_os_error(output_filename) + + +class ClearWeakRefsNoCallbacksTest(unittest.TestCase): + """Test PyUnstable_Object_ClearWeakRefsNoCallbacks""" + def test_ClearWeakRefsNoCallbacks(self): + """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works""" + import weakref + import gc + class C: + pass + obj = C() + messages = [] + ref = weakref.ref(obj, lambda: messages.append("don't add this")) + self.assertIs(ref(), obj) + self.assertFalse(messages) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + self.assertIsNone(ref()) + gc.collect() + self.assertFalse(messages) + + def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): + """Don't fail on objects that don't support weakrefs""" + import weakref + obj = object() + with self.assertRaises(TypeError): + ref = weakref.ref(obj) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py new file mode 100644 index 00000000000000..328b6424772061 --- /dev/null +++ b/Lib/test/test_capi/test_opt.py @@ -0,0 +1,1485 @@ +import contextlib +import sys +import textwrap +import unittest +import gc +import os + +import _opcode + +from test.support import script_helper, requires_specialization, import_helper + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + +from _testinternalcapi import TIER2_THRESHOLD + +@contextlib.contextmanager +def temporary_optimizer(opt): + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(opt) + try: + yield + finally: + _testinternalcapi.set_optimizer(old_opt) + + +@contextlib.contextmanager +def clear_executors(func): + # Clear executors in func before and after running a block + func.__code__ = func.__code__.replace() + try: + yield + finally: + func.__code__ = func.__code__.replace() + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +class TestOptimizerAPI(unittest.TestCase): + + def test_new_counter_optimizer_dealloc(self): + # See gh-108727 + def f(): + _testinternalcapi.new_counter_optimizer() + + f() + + def test_get_set_optimizer(self): + old = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + try: + _testinternalcapi.set_optimizer(opt) + self.assertEqual(_testinternalcapi.get_optimizer(), opt) + _testinternalcapi.set_optimizer(None) + self.assertEqual(_testinternalcapi.get_optimizer(), None) + finally: + _testinternalcapi.set_optimizer(old) + + + def test_counter_optimizer(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def loop(): + for _ in range(1000): + pass + """), ns, ns) + loop = ns['loop'] + + for repeat in range(5): + opt = _testinternalcapi.new_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + with clear_executors(loop): + loop() + # Subtract because optimizer doesn't kick in sooner + self.assertEqual(opt.get_count(), 1000 - TIER2_THRESHOLD) + + def test_long_loop(self): + "Check that we aren't confused by EXTENDED_ARG" + + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def nop(): + pass + + def long_loop(): + for _ in range(20): + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + """), ns, ns) + long_loop = ns['long_loop'] + + opt = _testinternalcapi.new_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + long_loop() + self.assertEqual(opt.get_count(), 20 - TIER2_THRESHOLD) # Need iterations to warm up + + def test_code_restore_for_ENTER_EXECUTOR(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.new_counter_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + code, replace_code = testfunc.__code__, testfunc.__code__.replace() + self.assertEqual(code, replace_code) + self.assertEqual(hash(code), hash(replace_code)) + + +def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + +def iter_opnames(ex): + for item in ex: + yield item[0] + + +def get_opnames(ex): + return list(iter_opnames(ex)) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +class TestExecutorInvalidation(unittest.TestCase): + + def setUp(self): + self.old = _testinternalcapi.get_optimizer() + self.opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(self.opt) + + def tearDown(self): + _testinternalcapi.set_optimizer(self.old) + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor mutliple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUops(unittest.TestCase): + + def test_basic_loop(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_JUMP_TO_TOP", uops) + self.assertIn("_LOAD_FAST_0", uops) + + def test_extended_arg(self): + "Check EXTENDED_ARG handling in superblock creation" + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + ex = get_first_executor(many_vars) + self.assertIsNone(ex) + many_vars() + + ex = get_first_executor(many_vars) + self.assertIsNotNone(ex) + self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST", 259, 0) + for opcode, oparg, _, operand in list(ex))) + + def test_unspecialized_unpack(self): + # An example of an unspecialized opcode + def testfunc(x): + i = 0 + while i < x: + i += 1 + a, b = {1: 2, 3: 3} + assert a == 1 and b == 3 + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.new_uop_optimizer() + + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNPACK_SEQUENCE", uops) + + def test_pop_jump_if_false(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_pop_jump_if_none(self): + def testfunc(a): + for x in a: + if x is None: + x = 0 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_not_none(self): + def testfunc(a): + for x in a: + x = None + if x is not None: + x = 0 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_true(self): + def testfunc(n): + i = 0 + while not i >= n: + i += 1 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_jump_backward(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_JUMP_TO_TOP", uops) + + def test_jump_forward(self): + def testfunc(n): + a = 0 + while a < n: + if a < 0: + a = -a + else: + a = +a + a += 1 + return a + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Since there is no JUMP_FORWARD instruction, + # look for indirect evidence: the += operator + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_for_iter_range(self): + def testfunc(n): + total = 0 + for i in range(n): + total += i + return total + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + total = testfunc(20) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_list(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + a = list(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_tuple(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + a = tuple(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_list_edge_case(self): + def testfunc(it): + for x in it: + pass + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_branch_taken(self): + def testfunc(n): + for i in range(n): + if i < 0: + i = 0 + else: + i = 1 + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_for_iter_tier_two(self): + class MyIter: + def __init__(self, n): + self.n = n + def __iter__(self): + return self + def __next__(self): + self.n -= 1 + if self.n < 0: + raise StopIteration + return self.n + + def testfunc(n, m): + x = 0 + for i in range(m): + for j in MyIter(n): + x += 1000*i + j + return x + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(10, 10) + + self.assertEqual(x, sum(range(10)) * 10010) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + + def test_confidence_score(self): + def testfunc(n): + bits = 0 + for i in range(n): + if i & 0x01: + bits += 1 + if i & 0x02: + bits += 1 + if i&0x04: + bits += 1 + if i&0x08: + bits += 1 + if i&0x10: + bits += 1 + if i&0x20: + bits += 1 + return bits + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(20) + + self.assertEqual(x, 40) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + ops = list(iter_opnames(ex)) + #Since branch is 50/50 the trace could go either way. + count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") + self.assertLessEqual(count, 2) + + +@requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUopsOptimization(unittest.TestCase): + + def _run_with_optimizer(self, testfunc, arg): + res = None + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + res = testfunc(arg) + + ex = get_first_executor(testfunc) + return res, ex + + + def test_int_type_propagation(self): + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = x + 1 + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + self.assertEqual(res, 63) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_through_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = double(x) + num += 1 + return a + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_type_propagation_from_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + a = double(num) + x = a + a + num += 1 + return x + + opt = _testinternalcapi.new_uop_optimizer() + res = None + with temporary_optimizer(opt): + res = testfunc(32) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, 124) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_both_int_count), 1) + + def test_int_impure_region(self): + def testfunc(loops): + num = 0 + while num < loops: + x = num + num + y = 1 + x // 2 + a = x + y + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, 64) + self.assertIsNotNone(ex) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + self.assertGreaterEqual(len(binop_count), 3) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_CHECK_PEP_523", uops) + + def test_int_type_propagate_through_range(self): + def testfunc(n): + + for i in range(n): + x = i + i + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 62) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + + def test_int_value_numbering(self): + def testfunc(n): + + y = 1 + for i in range(n): + x = y + z = x + a = z + b = a + res = x + z + a + b + return res + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_BOTH_INT", uops) + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertEqual(len(guard_count), 1) + + def test_comprehension(self): + def testfunc(n): + for _ in range(n): + return [i for i in range(n)] + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, list(range(32))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_BINARY_OP_ADD_INT", uops) + + def test_call_py_exact_args_disappearing(self): + def dummy(x): + return x+1 + + def testfunc(n): + for i in range(n): + dummy(i) + + opt = _testinternalcapi.new_uop_optimizer() + # Trigger specialization + testfunc(8) + with temporary_optimizer(opt): + del dummy + gc.collect() + + def dummy(x): + return x + 2 + testfunc(32) + + ex = get_first_executor(testfunc) + # Honestly as long as it doesn't crash it's fine. + # Whether we get an executor or not is non-deterministic, + # because it's decided by when the function is freed. + # This test is a little implementation specific. + + def test_promote_globals_to_constants(self): + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import _testinternalcapi + import opcode + import _opcode + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def get_opnames(ex): + return {item[0] for item in ex} + + def testfunc(n): + for i in range(n): + x = range(i) + return x + + opt = _testinternalcapi.new_uop_optimizer() + _testinternalcapi.set_optimizer(opt) + testfunc(64) + + ex = get_first_executor(testfunc) + assert ex is not None + uops = get_opnames(ex) + assert "_LOAD_GLOBAL_BUILTINS" not in uops + assert "_LOAD_CONST_INLINE_BORROW_WITH_NULL" in uops + """)) + self.assertEqual(result[0].rc, 0, result) + + def test_float_add_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 33.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + + def test_float_subtract_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, -31.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + + def test_float_multiply_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 1.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + + def test_add_unicode_propagation(self): + def testfunc(n): + a = "" + for _ in range(n): + a + a + a + a + a + a + a + a + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, "") + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_unicode_count), 1) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + + def test_compare_op_type_propagation_float(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_int(self): + def testfunc(n): + a = 1 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_int_count), 1) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_int_partial(self): + def testfunc(n): + a = 1 + for _ in range(n): + if a > 2: + x = 0 + if a < 2: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_left_int_count), 1) + self.assertEqual(len(guard_both_int_count), 0) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_float_partial(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + if a > 2.0: + x = 0 + if a < 2.0: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_left_float_count), 1) + self.assertEqual(len(guard_both_float_count), 0) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_unicode(self): + def testfunc(n): + a = "" + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_STR", uops) + + def test_type_inconsistency(self): + ns = {} + src = textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """) + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + ns['_test_global'] = 1 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + # Try again, but between the runs, set the global to a float. + # This should result in no executor the second time. + ns = {} + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + ns['_test_global'] = 3.14 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(ex) + + def test_combine_stack_space_checks_sequential(self): + def dummy12(x): + return x - 1 + def dummy13(y): + z = y + 2 + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(7) + c, d = dummy13(9) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 832) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # sequential calls: max(12, 13) == 13 + largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_nested(self): + def dummy12(x): + return x + 3 + def dummy15(y): + z = dummy12(y) + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b, c = dummy15(2) + a += b + c + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 224) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # nested calls: 15 + 12 == 27 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy12.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls(self): + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(5) + c, d, e = dummy18(2) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(12, 18 + max(12, 13)) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls_different_order(self): + # same as `several_calls` but with top-level calls reversed + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + c, d, e = dummy18(2) + b = dummy12(5) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(18 + max(12, 13), 12) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_complex(self): + def dummy0(x): + return x + def dummy1(x): + return dummy0(x) + def dummy2(x): + return dummy1(x) + def dummy3(x): + return dummy0(x) + def dummy4(x): + y = dummy0(x) + return dummy3(y) + def dummy5(x): + return dummy2(x) + def dummy6(x): + y = dummy5(x) + z = dummy0(y) + return dummy4(z) + def testfunc(n): + a = 0; + for _ in range(32): + b = dummy5(1) + c = dummy0(1) + d = dummy6(1) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 96) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 15) + + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy6.__code__) + + _testinternalcapi.get_co_framesize(dummy5.__code__) + + _testinternalcapi.get_co_framesize(dummy2.__code__) + + _testinternalcapi.get_co_framesize(dummy1.__code__) + + _testinternalcapi.get_co_framesize(dummy0.__code__) + ) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_large_framesize(self): + # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is + # actually doing its job. Note that the resulting trace hits + # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this + # test is still meaningful. + repetitions = 10000 + ns = {} + header = """ + def dummy_large(a0): + """ + body = "".join([f""" + a{n+1} = a{n} + 1 + """ for n in range(repetitions)]) + return_ = f""" + return a{repetitions-1} + """ + exec(textwrap.dedent(header + body + return_), ns, ns) + dummy_large = ns['dummy_large'] + + # this is something like: + # + # def dummy_large(a0): + # a1 = a0 + 1 + # a2 = a1 + 1 + # .... + # a9999 = a9998 + 1 + # return a9999 + + def dummy15(z): + y = dummy_large(z) + return y + 3 + + def testfunc(n): + b = 0 + for _ in range(n): + b += dummy15(7) + return b + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * (repetitions + 9)) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + + # this hits a different case during trace projection in refcount test runs only, + # so we need to account for both possibilities + self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) + if uop_names.count("_CHECK_STACK_SPACE") == 0: + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy_large.__code__) + ) + else: + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_recursion(self): + def dummy15(x): + while x > 0: + return dummy15(x - 1) + return 42 + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy15(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 42 * 32) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_many_nested(self): + # overflow the trace_stack + def dummy_a(x): + return x + def dummy_b(x): + return dummy_a(x) + def dummy_c(x): + return dummy_b(x) + def dummy_d(x): + return dummy_c(x) + def dummy_e(x): + return dummy_d(x) + def dummy_f(x): + return dummy_e(x) + def dummy_g(x): + return dummy_f(x) + def dummy_h(x): + return dummy_g(x) + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy_h(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * 32) + self.assertIsNone(ex) + + def test_return_generator(self): + def gen(): + yield None + def testfunc(n): + for i in range(n): + gen() + return i + res, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + + def test_for_iter_gen(self): + def gen(n): + for i in range(n): + yield i + def testfunc(n): + g = gen(n) + s = 0 + for i in g: + s += i + return s + res, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(res, 190) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_not_removed(self): + """ + Verify that the guard type version is not removed if we modify the class + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # for the first 90 iterations we set the attribute on this dummy function which shouldn't + # trigger the type watcher + # then after 90 it should trigger it and stop optimizing + # Note that the code needs to be in this weird form so it's optimized inline without any control flow + setattr((Foo, Bar)[i < 90], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + + self.assertIsNotNone(ex) + self.assertEqual(res, 219) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 2) + + + @unittest.expectedFailure + def test_guard_type_version_not_removed_escaping(self): + """ + Verify that the guard type version is not removed if have an escaping function + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # eval should be escaping and so should cause optimization to stop and preserve both type versions + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + # Note: This will actually be 1 for noe + # https://github.com/python/cpython/pull/119365#discussion_r1626220129 + self.assertEqual(guard_type_version_count, 2) + + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, 200) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py new file mode 100644 index 00000000000000..894f66b437a39c --- /dev/null +++ b/Lib/test/test_capi/test_run.py @@ -0,0 +1,118 @@ +import os +import unittest +from collections import UserDict +from test.support import import_helper +from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE + +NULL = None +_testcapi = import_helper.import_module('_testcapi') +Py_single_input = _testcapi.Py_single_input +Py_file_input = _testcapi.Py_file_input +Py_eval_input = _testcapi.Py_eval_input + + +class DictSubclass(dict): + pass + + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyRun_SimpleStringFlags + # PyRun_AnyFileExFlags + # PyRun_SimpleFileExFlags + # PyRun_InteractiveOneFlags + # PyRun_InteractiveOneObject + # PyRun_InteractiveLoopFlags + # PyRun_String (may be a macro) + # PyRun_AnyFile (may be a macro) + # PyRun_AnyFileEx (may be a macro) + # PyRun_AnyFileFlags (may be a macro) + # PyRun_SimpleString (may be a macro) + # PyRun_SimpleFile (may be a macro) + # PyRun_SimpleFileEx (may be a macro) + # PyRun_InteractiveOne (may be a macro) + # PyRun_InteractiveLoop (may be a macro) + # PyRun_File (may be a macro) + # PyRun_FileEx (may be a macro) + # PyRun_FileFlags (may be a macro) + + def test_run_stringflags(self): + # Test PyRun_StringFlags(). + # XXX: fopen() uses different path encoding than Python on Windows. + def run(s, *args): + return _testcapi.run_stringflags(s, Py_file_input, *args) + source = b'a\n' + + self.assertIsNone(run(b'a\n', dict(a=1))) + self.assertIsNone(run(b'a\n', dict(a=1), {})) + self.assertIsNone(run(b'a\n', {}, dict(a=1))) + self.assertIsNone(run(b'a\n', {}, UserDict(a=1))) + + self.assertRaises(NameError, run, b'a\n', {}) + self.assertRaises(NameError, run, b'a\n', {}, {}) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) + self.assertRaises(SyntaxError, run, b'\xe4\n', {}) + + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) + + # CRASHES run(NULL, {}) + + def test_run_fileexflags(self): + # Test PyRun_FileExFlags(). + filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) + with open(filename, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, filename) + def run(*args): + return _testcapi.run_fileexflags(filename, Py_file_input, *args) + + self.assertIsNone(run(dict(a=1))) + self.assertIsNone(run(dict(a=1), {})) + self.assertIsNone(run({}, dict(a=1))) + self.assertIsNone(run({}, UserDict(a=1))) + self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True + + self.assertRaises(NameError, run, {}) + self.assertRaises(NameError, run, {}, {}) + self.assertRaises(TypeError, run, dict(a=1), []) + self.assertRaises(TypeError, run, dict(a=1), 1) + + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) + + @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') + @unittest.skipIf(os.name == 'nt', 'does not work on Windows') + def test_run_fileexflags_with_undecodable_filename(self): + run = _testcapi.run_fileexflags + try: + with open(TESTFN_UNDECODABLE, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, TESTFN_UNDECODABLE) + except OSError: + self.skipTest('undecodable paths are not supported') + self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py index 5235f81543e0b6..499a5148d782ab 100644 --- a/Lib/test/test_capi/test_set.py +++ b/Lib/test/test_capi/test_set.py @@ -2,8 +2,10 @@ from test.support import import_helper -# Skip this test if the _testcapi or _testinternalcapi modules aren't available. +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') class set_subclass(set): @@ -23,7 +25,7 @@ def assertImmutable(self, action, *args): class TestSetCAPI(BaseSetTests, unittest.TestCase): def test_set_check(self): - check = _testcapi.set_check + check = _testlimitedcapi.set_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -33,7 +35,7 @@ def test_set_check(self): # CRASHES: check(NULL) def test_set_check_exact(self): - check = _testcapi.set_checkexact + check = _testlimitedcapi.set_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -43,7 +45,7 @@ def test_set_check_exact(self): # CRASHES: check(NULL) def test_frozenset_check(self): - check = _testcapi.frozenset_check + check = _testlimitedcapi.frozenset_check self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -53,7 +55,7 @@ def test_frozenset_check(self): # CRASHES: check(NULL) def test_frozenset_check_exact(self): - check = _testcapi.frozenset_checkexact + check = _testlimitedcapi.frozenset_checkexact self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -63,7 +65,7 @@ def test_frozenset_check_exact(self): # CRASHES: check(NULL) def test_anyset_check(self): - check = _testcapi.anyset_check + check = _testlimitedcapi.anyset_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -74,7 +76,7 @@ def test_anyset_check(self): # CRASHES: check(NULL) def test_anyset_check_exact(self): - check = _testcapi.anyset_checkexact + check = _testlimitedcapi.anyset_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -85,7 +87,7 @@ def test_anyset_check_exact(self): # CRASHES: check(NULL) def test_set_new(self): - set_new = _testcapi.set_new + set_new = _testlimitedcapi.set_new self.assertEqual(set_new().__class__, set) self.assertEqual(set_new(), set()) self.assertEqual(set_new((1, 1, 2)), {1, 2}) @@ -98,7 +100,7 @@ def test_set_new(self): set_new((1, {})) def test_frozenset_new(self): - frozenset_new = _testcapi.frozenset_new + frozenset_new = _testlimitedcapi.frozenset_new self.assertEqual(frozenset_new().__class__, frozenset) self.assertEqual(frozenset_new(), frozenset()) self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) @@ -111,7 +113,7 @@ def test_frozenset_new(self): frozenset_new((1, {})) def test_set_size(self): - get_size = _testcapi.set_size + get_size = _testlimitedcapi.set_size self.assertEqual(get_size(set()), 0) self.assertEqual(get_size(frozenset()), 0) self.assertEqual(get_size({1, 1, 2}), 2) @@ -134,7 +136,7 @@ def test_set_get_size(self): # CRASHES: get_size(object()) def test_set_contains(self): - contains = _testcapi.set_contains + contains = _testlimitedcapi.set_contains for cls in (set, frozenset, set_subclass, frozenset_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -147,7 +149,7 @@ def test_set_contains(self): # CRASHES: contains(NULL, NULL) def test_add(self): - add = _testcapi.set_add + add = _testlimitedcapi.set_add for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -165,7 +167,7 @@ def test_add(self): # CRASHES: add(NULL, NULL) def test_discard(self): - discard = _testcapi.set_discard + discard = _testlimitedcapi.set_discard for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -187,7 +189,7 @@ def test_discard(self): # CRASHES: discard(NULL, NULL) def test_pop(self): - pop = _testcapi.set_pop + pop = _testlimitedcapi.set_pop orig = (1, 2) for cls in (set, set_subclass): with self.subTest(cls=cls): @@ -204,7 +206,7 @@ def test_pop(self): # CRASHES: pop(NULL) def test_clear(self): - clear = _testcapi.set_clear + clear = _testlimitedcapi.set_clear for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 2cf46b203478dc..6b27dc512a7d15 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -1,6 +1,5 @@ import unittest from test.support import import_helper -from test.support import warnings_helper # Skip this test if the _testcapi module isn't available. import_helper.import_module('_testcapi') @@ -14,6 +13,13 @@ PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, ) + +class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # There are two classes: one using and another using # `Py_`-prefixed API. They should behave the same in Python @@ -38,75 +44,99 @@ class ReadWriteTests: def setUp(self): self.ts = _make_test_object(self.cls) + def _test_write(self, name, value, expected=None): + if expected is None: + expected = value + ts = self.ts + setattr(ts, name, value) + self.assertEqual(getattr(ts, name), expected) + + def _test_warn(self, name, value, expected=None): + ts = self.ts + self.assertWarns(RuntimeWarning, setattr, ts, name, value) + if expected is not None: + self.assertEqual(getattr(ts, name), expected) + + def _test_overflow(self, name, value): + ts = self.ts + self.assertRaises(OverflowError, setattr, ts, name, value) + + def _test_int_range(self, name, minval, maxval, *, hardlimit=None, + indexlimit=None): + if hardlimit is None: + hardlimit = (minval, maxval) + ts = self.ts + self._test_write(name, minval) + self._test_write(name, maxval) + hardminval, hardmaxval = hardlimit + self._test_overflow(name, hardminval-1) + self._test_overflow(name, hardmaxval+1) + self._test_overflow(name, 2**1000) + self._test_overflow(name, -2**1000) + if hardminval < minval: + self._test_warn(name, hardminval) + self._test_warn(name, minval-1, maxval) + if maxval < hardmaxval: + self._test_warn(name, maxval+1, minval) + self._test_warn(name, hardmaxval) + + if indexlimit is False: + self.assertRaises(TypeError, setattr, ts, name, Index(minval)) + self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) + else: + self._test_write(name, Index(minval), minval) + self._test_write(name, Index(maxval), maxval) + self._test_overflow(name, Index(hardminval-1)) + self._test_overflow(name, Index(hardmaxval+1)) + self._test_overflow(name, Index(2**1000)) + self._test_overflow(name, Index(-2**1000)) + if hardminval < minval: + self._test_warn(name, Index(hardminval)) + self._test_warn(name, Index(minval-1), maxval) + if maxval < hardmaxval: + self._test_warn(name, Index(maxval+1), minval) + self._test_warn(name, Index(hardmaxval)) + def test_bool(self): ts = self.ts ts.T_BOOL = True - self.assertEqual(ts.T_BOOL, True) + self.assertIs(ts.T_BOOL, True) ts.T_BOOL = False - self.assertEqual(ts.T_BOOL, False) + self.assertIs(ts.T_BOOL, False) self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) def test_byte(self): - ts = self.ts - ts.T_BYTE = CHAR_MAX - self.assertEqual(ts.T_BYTE, CHAR_MAX) - ts.T_BYTE = CHAR_MIN - self.assertEqual(ts.T_BYTE, CHAR_MIN) - ts.T_UBYTE = UCHAR_MAX - self.assertEqual(ts.T_UBYTE, UCHAR_MAX) + self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UBYTE', 0, UCHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_short(self): - ts = self.ts - ts.T_SHORT = SHRT_MAX - self.assertEqual(ts.T_SHORT, SHRT_MAX) - ts.T_SHORT = SHRT_MIN - self.assertEqual(ts.T_SHORT, SHRT_MIN) - ts.T_USHORT = USHRT_MAX - self.assertEqual(ts.T_USHORT, USHRT_MAX) + self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_USHORT', 0, USHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) def test_int(self): - ts = self.ts - ts.T_INT = INT_MAX - self.assertEqual(ts.T_INT, INT_MAX) - ts.T_INT = INT_MIN - self.assertEqual(ts.T_INT, INT_MIN) - ts.T_UINT = UINT_MAX - self.assertEqual(ts.T_UINT, UINT_MAX) + self._test_int_range('T_INT', INT_MIN, INT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UINT', 0, UINT_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) def test_long(self): - ts = self.ts - ts.T_LONG = LONG_MAX - self.assertEqual(ts.T_LONG, LONG_MAX) - ts.T_LONG = LONG_MIN - self.assertEqual(ts.T_LONG, LONG_MIN) - ts.T_ULONG = ULONG_MAX - self.assertEqual(ts.T_ULONG, ULONG_MAX) + self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) + self._test_int_range('T_ULONG', 0, ULONG_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) def test_py_ssize_t(self): - ts = self.ts - ts.T_PYSSIZET = PY_SSIZE_T_MAX - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MAX) - ts.T_PYSSIZET = PY_SSIZE_T_MIN - self.assertEqual(ts.T_PYSSIZET, PY_SSIZE_T_MIN) + self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) def test_longlong(self): - ts = self.ts - if not hasattr(ts, "T_LONGLONG"): - self.skipTest("long long not present") - - ts.T_LONGLONG = LLONG_MAX - self.assertEqual(ts.T_LONGLONG, LLONG_MAX) - ts.T_LONGLONG = LLONG_MIN - self.assertEqual(ts.T_LONGLONG, LLONG_MIN) - - ts.T_ULONGLONG = ULLONG_MAX - self.assertEqual(ts.T_ULONGLONG, ULLONG_MAX) - - ## make sure these will accept a plain int as well as a long - ts.T_LONGLONG = 3 - self.assertEqual(ts.T_LONGLONG, 3) - ts.T_ULONGLONG = 4 - self.assertEqual(ts.T_ULONGLONG, 4) + self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) + self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, + hardlimit=(LONG_MIN, ULLONG_MAX)) def test_bad_assignments(self): ts = self.ts @@ -116,10 +146,9 @@ def test_bad_assignments(self): 'T_SHORT', 'T_USHORT', 'T_INT', 'T_UINT', 'T_LONG', 'T_ULONG', + 'T_LONGLONG', 'T_ULONGLONG', 'T_PYSSIZET' ] - if hasattr(ts, 'T_LONGLONG'): - integer_attributes.extend(['T_LONGLONG', 'T_ULONGLONG']) # issue8014: this produced 'bad argument to internal function' # internal error @@ -139,46 +168,6 @@ class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_NewAPI -class TestWarnings: - def setUp(self): - self.ts = _make_test_object(self.cls) - - def test_byte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MAX+1 - - def test_byte_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_BYTE = CHAR_MIN-1 - - def test_ubyte_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_UBYTE = UCHAR_MAX+1 - - def test_short_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MAX+1 - - def test_short_min(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_SHORT = SHRT_MIN-1 - - def test_ushort_max(self): - ts = self.ts - with warnings_helper.check_warnings(('', RuntimeWarning)): - ts.T_USHORT = USHRT_MAX+1 - -class TestWarnings_OldAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_OldAPI - -class TestWarnings_NewAPI(TestWarnings, unittest.TestCase): - cls = _test_structmembersType_NewAPI - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py new file mode 100644 index 00000000000000..54a8e026d883d4 --- /dev/null +++ b/Lib/test/test_capi/test_sys.py @@ -0,0 +1,154 @@ +import unittest +import contextlib +import sys +from test import support +from test.support import import_helper + +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None + +NULL = None + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PySys_Audit() + # PySys_AuditTuple() + + maxDiff = None + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getobject(self): + # Test PySys_GetObject() + getobject = _testlimitedcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + with support.catch_unraisable_exception() as cm: + self.assertIs(getobject(b'\xff'), AttributeError) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + # CRASHES getobject(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_setobject(self): + # Test PySys_SetObject() + setobject = _testlimitedcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getxoptions(self): + # Test PySys_GetXOptions() + getxoptions = _testlimitedcapi.sys_getxoptions + + self.assertIs(getxoptions(), sys._xoptions) + + xoptions = sys._xoptions + try: + sys._xoptions = 'non-dict' + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + + del sys._xoptions + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + finally: + sys._xoptions = xoptions + self.assertIs(getxoptions(), sys._xoptions) + + def _test_sys_formatstream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p, py_object + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # Supports Python objects. + with support.captured_output(streamname) as stream: + func(b'Hello, %R!', py_object('world')) + self.assertEqual(stream.getvalue(), "Hello, 'world'!") + + # The total length is not limited. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*200 + '!') + + def test_sys_formatstdout(self): + # Test PySys_FormatStdout() + self._test_sys_formatstream('PySys_FormatStdout', 'stdout') + + def test_sys_formatstderr(self): + # Test PySys_FormatStderr() + self._test_sys_formatstream('PySys_FormatStderr', 'stderr') + + def _test_sys_writestream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # There is a limit on the total length. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*100)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*100 + '!') + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + out = stream.getvalue() + self.assertEqual(out[:20], 'Hello, worldworldwor') + self.assertEqual(out[-13:], '... truncated') + self.assertGreater(len(out), 1000) + + def test_sys_writestdout(self): + # Test PySys_WriteStdout() + self._test_sys_writestream('PySys_WriteStdout', 'stdout') + + def test_sys_writestderr(self): + # Test PySys_WriteStderr() + self._test_sys_writestream('PySys_WriteStderr', 'stderr') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py new file mode 100644 index 00000000000000..17ebd7c1962d48 --- /dev/null +++ b/Lib/test/test_capi/test_time.py @@ -0,0 +1,74 @@ +import time +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +PyTime_MIN = _testcapi.PyTime_MIN +PyTime_MAX = _testcapi.PyTime_MAX +SEC_TO_NS = 10 ** 9 +DAY_TO_SEC = (24 * 60 * 60) +# Worst clock resolution: maximum delta between two clock reads. +CLOCK_RES = 0.050 + + +class CAPITest(unittest.TestCase): + def test_min_max(self): + # PyTime_t is just int64_t + self.assertEqual(PyTime_MIN, -2**63) + self.assertEqual(PyTime_MAX, 2**63 - 1) + + def test_assecondsdouble(self): + # Test PyTime_AsSecondsDouble() + def ns_to_sec(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + seconds = ( + 0, + 1, + DAY_TO_SEC, + 365 * DAY_TO_SEC, + ) + values = { + PyTime_MIN, + PyTime_MIN + 1, + PyTime_MAX - 1, + PyTime_MAX, + } + for second in seconds: + ns = second * SEC_TO_NS + values.add(ns) + # test nanosecond before/after to test rounding + values.add(ns - 1) + values.add(ns + 1) + for ns in list(values): + if (-ns) > PyTime_MAX: + continue + values.add(-ns) + for ns in sorted(values): + with self.subTest(ns=ns): + self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), + ns_to_sec(ns)) + + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + + def test_monotonic(self): + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() + self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) + + def test_perf_counter(self): + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() + self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) + + def test_time(self): + # Test PyTime_Time() and PyTime_TimeRaw() + self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index a73e669dda7ddc..9ef476a02de47d 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -5,12 +5,21 @@ try: import _testcapi + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None try: import _testinternalcapi except ImportError: _testinternalcapi = None +try: + import ctypes +except ImportError: + ctypes = None NULL = None @@ -30,9 +39,17 @@ def test_new(self): for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: self.assertEqual(new(0, maxchar), '') self.assertEqual(new(5, maxchar), chr(maxchar)*5) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar) self.assertEqual(new(0, 0x110000), '') + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600) self.assertRaises(SystemError, new, 5, 0x110000) self.assertRaises(SystemError, new, -1, 0) + self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') @@ -53,8 +70,8 @@ def test_fill(self): for to in strings[:idx]: self.assertRaises(ValueError, fill, to, 0, 0, fill_char) for to in strings[idx:]: - for start in range(7): - for length in range(-1, 7 - start): + for start in [*range(7), PY_SSIZE_T_MAX]: + for length in [*range(-1, 7 - start), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: filled = max(min(length, 5 - start), 0) if filled == 5 and to != strings[idx]: # narrow -> wide @@ -66,6 +83,7 @@ def test_fill(self): s = strings[0] self.assertRaises(IndexError, fill, s, -1, 0, 0x78) + self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) self.assertRaises(SystemError, fill, [], 0, 0, 0x78) @@ -74,10 +92,10 @@ def test_fill(self): # TODO: Test PyUnicode_Fill() with non-modifiable unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_writechar(self): - """Test PyUnicode_ReadChar()""" - from _testcapi import unicode_writechar as writechar + """Test PyUnicode_WriteChar()""" + from _testlimitedcapi import unicode_writechar as writechar strings = [ # one string for every kind @@ -96,17 +114,19 @@ def test_writechar(self): self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) + self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MAX, 0x78) + self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MIN, 0x78) self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) self.assertRaises(TypeError, writechar, [], 0, 0x78) # CRASHES writechar(NULL, 0, 0x78) - # TODO: Test PyUnicode_CopyCharacters() with non-modifiable and legacy + # TODO: Test PyUnicode_WriteChar() with non-modifiable and legacy # unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_resize(self): """Test PyUnicode_Resize()""" - from _testcapi import unicode_resize as resize + from _testlimitedcapi import unicode_resize as resize strings = [ # all strings have exactly 3 characters @@ -117,7 +137,11 @@ def test_resize(self): self.assertEqual(resize(s, 3), (s, 0)) self.assertEqual(resize(s, 2), (s[:2], 0)) self.assertEqual(resize(s, 4), (s + '\0', 0)) + self.assertEqual(resize(s, 10), (s + '\0'*7, 0)) self.assertEqual(resize(s, 0), ('', 0)) + self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, resize, s, -1) + self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN) self.assertRaises(SystemError, resize, b'abc', 0) self.assertRaises(SystemError, resize, [], 0) self.assertRaises(SystemError, resize, NULL, 0) @@ -125,10 +149,10 @@ def test_resize(self): # and with NULL as the address. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_append(self): """Test PyUnicode_Append()""" - from _testcapi import unicode_append as append + from _testlimitedcapi import unicode_append as append strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -153,10 +177,10 @@ def test_append(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_appendanddel(self): """Test PyUnicode_AppendAndDel()""" - from _testcapi import unicode_appendanddel as appendanddel + from _testlimitedcapi import unicode_appendanddel as appendanddel strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -180,10 +204,10 @@ def test_appendanddel(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstringandsize(self): """Test PyUnicode_FromStringAndSize()""" - from _testcapi import unicode_fromstringandsize as fromstringandsize + from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize self.assertEqual(fromstringandsize(b'abc'), 'abc') self.assertEqual(fromstringandsize(b'abc', 2), 'ab') @@ -196,14 +220,19 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'', 0), '') self.assertEqual(fromstringandsize(NULL, 0), '') + self.assertRaises(MemoryError, fromstringandsize, b'abc', PY_SSIZE_T_MAX) self.assertRaises(SystemError, fromstringandsize, b'abc', -1) - # TODO: Test PyUnicode_FromStringAndSize(NULL, size) for size != 0 + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, 3) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstring(self): """Test PyUnicode_FromString()""" - from _testcapi import unicode_fromstring as fromstring + from _testlimitedcapi import unicode_fromstring as fromstring self.assertEqual(fromstring(b'abc'), 'abc') self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') @@ -245,15 +274,17 @@ def test_fromkindanddata(self): for kind in -1, 0, 3, 5, 8: self.assertRaises(SystemError, fromkindanddata, kind, b'') self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) + self.assertRaises(ValueError, fromkindanddata, 1, b'abc', PY_SSIZE_T_MIN) self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) + self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN) # CRASHES fromkindanddata(1, NULL, 1) # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_substring(self): """Test PyUnicode_Substring()""" - from _testcapi import unicode_substring as substring + from _testlimitedcapi import unicode_substring as substring strings = [ 'ab', 'ab\xa1\xa2', @@ -261,21 +292,23 @@ def test_substring(self): 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' ] for s in strings: - for start in range(0, len(s) + 2): - for end in range(max(start-1, 0), len(s) + 2): + for start in [*range(0, len(s) + 2), PY_SSIZE_T_MAX]: + for end in [*range(max(start-1, 0), len(s) + 2), PY_SSIZE_T_MAX]: self.assertEqual(substring(s, start, end), s[start:end]) self.assertRaises(IndexError, substring, 'abc', -1, 0) + self.assertRaises(IndexError, substring, 'abc', PY_SSIZE_T_MIN, 0) self.assertRaises(IndexError, substring, 'abc', 0, -1) + self.assertRaises(IndexError, substring, 'abc', 0, PY_SSIZE_T_MIN) # CRASHES substring(b'abc', 0, 0) # CRASHES substring([], 0, 0) # CRASHES substring(NULL, 0, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getlength(self): """Test PyUnicode_GetLength()""" - from _testcapi import unicode_getlength as getlength + from _testlimitedcapi import unicode_getlength as getlength for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -286,27 +319,29 @@ def test_getlength(self): # CRASHES getlength(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_readchar(self): """Test PyUnicode_ReadChar()""" - from _testcapi import unicode_readchar as readchar + from _testlimitedcapi import unicode_readchar as readchar for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: for i, c in enumerate(s): self.assertEqual(readchar(s, i), ord(c)) self.assertRaises(IndexError, readchar, s, len(s)) + self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MAX) self.assertRaises(IndexError, readchar, s, -1) + self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MIN) self.assertRaises(TypeError, readchar, b'abc', 0) self.assertRaises(TypeError, readchar, [], 0) # CRASHES readchar(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromobject(self): """Test PyUnicode_FromObject()""" - from _testcapi import unicode_fromobject as fromobject + from _testlimitedcapi import unicode_fromobject as fromobject for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -321,13 +356,13 @@ def test_fromobject(self): self.assertRaises(TypeError, fromobject, []) # CRASHES fromobject(NULL) + @unittest.skipIf(ctypes is None, 'need ctypes') def test_from_format(self): """Test PyUnicode_FromFormat()""" # Length modifiers "j" and "t" are not tested here because ctypes does # not expose types for intmax_t and ptrdiff_t. - # _testcapi.test_string_from_format() has a wider coverage of all + # _testlimitedcapi.test_string_from_format() has a wider coverage of all # formats. - import_helper.import_module('ctypes') from ctypes import ( c_char_p, pythonapi, py_object, sizeof, @@ -384,8 +419,29 @@ def check_format(expected, format, *args): # truncated string check_format('abc', b'%.3s', b'abcdef') + check_format('abc[', + b'%.6s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\ufffd', + b'%.5s', b'abc[\xff]') + check_format('abc[', + b'%.6s', b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7s', b'abc[\xe2\x82]') check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) + b'%.7s', b'abc[\xe2\x82\0') + check_format(' abc[', + b'%10.6s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5s', b'abc[\xff]') + check_format(' abc[', + b'%10.6s', b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7s', b'abc[\xe2\x82]') + check_format("'\\u20acABC'", b'%A', '\u20acABC') check_format("'\\u20", @@ -398,10 +454,31 @@ def check_format(expected, format, *args): b'%.3S', '\u20acABCDEF') check_format('\u20acAB', b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', b'%.3V', '\u20acABCDEF', None) + check_format('abc[', + b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + b'%.5V', None, b'abc[\xff]') + check_format('abc[', + b'%.6V', None, b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7V', None, b'abc[\xe2\x82]') + check_format(' abc[', + b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5V', None, b'abc[\xff]') + check_format(' abc[', + b'%10.6V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd', + b'%10.7V', None, b'abc[\xe2\x82\0') # following tests comes from #7330 # test width modifier and precision modifier with %S @@ -619,6 +696,40 @@ def check_format(expected, format, *args): check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %#T', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %#N', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + # test variable width and precision check_format(' abc', b'%*s', c_int(5), b'abc') check_format('ab', b'%.*s', c_int(2), b'abc') @@ -680,10 +791,10 @@ def check_format(expected, format, *args): PyUnicode_FromFormat, b'%+i', c_int(10)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_interninplace(self): """Test PyUnicode_InternInPlace()""" - from _testcapi import unicode_interninplace as interninplace + from _testlimitedcapi import unicode_interninplace as interninplace s = b'abc'.decode() r = interninplace(s) @@ -693,10 +804,10 @@ def test_interninplace(self): # CRASHES interninplace(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_internfromstring(self): """Test PyUnicode_InternFromString()""" - from _testcapi import unicode_internfromstring as internfromstring + from _testlimitedcapi import unicode_internfromstring as internfromstring self.assertEqual(internfromstring(b'abc'), 'abc') self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') @@ -707,10 +818,10 @@ def test_internfromstring(self): # CRASHES internfromstring(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromwidechar(self): """Test PyUnicode_FromWideChar()""" - from _testcapi import unicode_fromwidechar as fromwidechar + from _testlimitedcapi import unicode_fromwidechar as fromwidechar from _testcapi import SIZEOF_WCHAR_T if SIZEOF_WCHAR_T == 2: @@ -731,17 +842,22 @@ def test_fromwidechar(self): if SIZEOF_WCHAR_T == 2: self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') + self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) + self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) self.assertEqual(fromwidechar(NULL, 0), '') self.assertRaises(SystemError, fromwidechar, NULL, 1) + self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) self.assertRaises(SystemError, fromwidechar, NULL, -1) + self.assertRaises(SystemError, fromwidechar, NULL, -2) + self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" - from _testcapi import unicode_aswidechar - from _testcapi import unicode_aswidechar_null + from _testlimitedcapi import unicode_aswidechar + from _testlimitedcapi import unicode_aswidechar_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) @@ -785,11 +901,11 @@ def test_aswidechar(self): self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" - from _testcapi import unicode_aswidecharstring - from _testcapi import unicode_aswidecharstring_null + from _testlimitedcapi import unicode_aswidecharstring + from _testlimitedcapi import unicode_aswidecharstring_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') @@ -861,10 +977,10 @@ def test_asucs4copy(self): # CRASHES asucs4copy(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromordinal(self): """Test PyUnicode_FromOrdinal()""" - from _testcapi import unicode_fromordinal as fromordinal + from _testlimitedcapi import unicode_fromordinal as fromordinal self.assertEqual(fromordinal(0x61), 'a') self.assertEqual(fromordinal(0x20ac), '\u20ac') @@ -890,11 +1006,11 @@ def test_asutf8(self): # CRASHES unicode_asutf8(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" - from _testcapi import unicode_asutf8andsize - from _testcapi import unicode_asutf8andsize_null + from _testlimitedcapi import unicode_asutf8andsize + from _testlimitedcapi import unicode_asutf8andsize_null self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) @@ -906,13 +1022,17 @@ def test_asutf8andsize(self): self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) # CRASHES unicode_asutf8andsize(NULL, 0) + # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getdefaultencoding(self): """Test PyUnicode_GetDefaultEncoding()""" - from _testcapi import unicode_getdefaultencoding as getdefaultencoding + from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding self.assertEqual(getdefaultencoding(), b'utf-8') @@ -937,10 +1057,10 @@ def test_transform_decimal_and_space(self): # CRASHES transform_decimal(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_concat(self): """Test PyUnicode_Concat()""" - from _testcapi import unicode_concat as concat + from _testlimitedcapi import unicode_concat as concat self.assertEqual(concat('abc', 'def'), 'abcdef') self.assertEqual(concat('abc', 'где'), 'abcгде') @@ -958,13 +1078,18 @@ def test_concat(self): # CRASHES concat('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_split(self): """Test PyUnicode_Split()""" - from _testcapi import unicode_split as split + from _testlimitedcapi import unicode_split as split self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) + self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MAX), + ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MIN), + ['a', 'b', 'c', 'd']) self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d']) self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd']) self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) @@ -981,13 +1106,18 @@ def test_split(self): # CRASHES split(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rsplit(self): """Test PyUnicode_RSplit()""" - from _testcapi import unicode_rsplit as rsplit + from _testlimitedcapi import unicode_rsplit as rsplit self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MAX), + ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MIN), + ['a', 'b', 'c', 'd']) self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d']) self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd']) self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) @@ -1005,10 +1135,10 @@ def test_rsplit(self): # CRASHES rsplit(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_partition(self): """Test PyUnicode_Partition()""" - from _testcapi import unicode_partition as partition + from _testlimitedcapi import unicode_partition as partition self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) @@ -1025,10 +1155,10 @@ def test_partition(self): # CRASHES partition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rpartition(self): """Test PyUnicode_RPartition()""" - from _testcapi import unicode_rpartition as rpartition + from _testlimitedcapi import unicode_rpartition as rpartition self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) @@ -1045,10 +1175,10 @@ def test_rpartition(self): # CRASHES rpartition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_splitlines(self): """Test PyUnicode_SplitLines()""" - from _testcapi import unicode_splitlines as splitlines + from _testlimitedcapi import unicode_splitlines as splitlines self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) self.assertEqual(splitlines('a\nb\rc\r\nd', True), @@ -1063,10 +1193,10 @@ def test_splitlines(self): # CRASHES splitlines(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_translate(self): """Test PyUnicode_Translate()""" - from _testcapi import unicode_translate as translate + from _testlimitedcapi import unicode_translate as translate self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') self.assertEqual(translate('абвг', {ord('а'): 'Ð', ord('б'): ord('Б'), ord('в'): '<>'}), 'ÐБ<>г') @@ -1088,10 +1218,10 @@ def test_translate(self): # CRASHES translate(NULL, []) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_join(self): """Test PyUnicode_Join()""" - from _testcapi import unicode_join as join + from _testlimitedcapi import unicode_join as join self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') self.assertEqual(join('|', ['a', '', 'c']), 'a||c') self.assertEqual(join('', ['a', 'b', 'c']), 'abc') @@ -1106,10 +1236,10 @@ def test_join(self): self.assertRaises(SystemError, join, '|', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_count(self): """Test PyUnicode_Count()""" - from _testcapi import unicode_count + from _testlimitedcapi import unicode_count for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1120,11 +1250,14 @@ def test_count(self): self.assertEqual(unicode_count(str, '', 0, len(str)), len(str)+1) # start < end self.assertEqual(unicode_count(str, '!', 1, len(str)+1), 1) + self.assertEqual(unicode_count(str, '!', 1, PY_SSIZE_T_MAX), 1) # start >= end self.assertEqual(unicode_count(str, '!', 0, 0), 0) self.assertEqual(unicode_count(str, '!', len(str), 0), 0) # negative self.assertEqual(unicode_count(str, '!', -len(str), -1), 1) + self.assertEqual(unicode_count(str, '!', -len(str)-1, -1), 1) + self.assertEqual(unicode_count(str, '!', PY_SSIZE_T_MIN, -1), 1) # bad arguments self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str)) self.assertRaises(TypeError, unicode_count, b"!>_= end self.assertEqual(find(str, '!', 0, 0, 1), -1) + self.assertEqual(find(str, '!', 0, 0, -1), -1) self.assertEqual(find(str, '!', len(str), 0, 1), -1) + self.assertEqual(find(str, '!', len(str), 0, -1), -1) # negative self.assertEqual(find(str, '!', -len(str), -1, 1), 0) self.assertEqual(find(str, '!', -len(str), -1, -1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, 1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, -1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) # bad arguments self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1) self.assertRaises(TypeError, find, b"!>_= end self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, -1), -1) self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, -1), -1) # negative self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, -1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) # bad arguments # CRASHES unicode_findchar(b"!>_") + print(f"extra_compile_args: {cflags!r}") + + ext = Extension( + module_name, + sources=[SOURCE], + extra_compile_args=cflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 1531aad4f1f779..655d53b8d5bb6a 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -2,7 +2,6 @@ import unittest - testmeths = [ # Binary operations @@ -448,15 +447,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): @@ -771,6 +770,137 @@ def add_one_level(): with self.assertRaises(RecursionError): add_one_level() + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) + + +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + def test_bug_117750(self): + "Aborted on 3.13a6" + class C: + def __init__(self): + self.__dict__.clear() + + obj = C() + self.assertEqual(obj.__dict__, {}) + obj.foo = None # Aborted here + self.assertEqual(obj.__dict__, {"foo":None}) + + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsisent state + C.a = X() + C.a = X() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 627a329bb738a2..f3fd610414cd8a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -5,10 +5,9 @@ from functools import partial from test import support, test_tools from test.support import os_helper -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent from unittest import TestCase -import contextlib import inspect import os.path import re @@ -17,14 +16,27 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): + import libclinic + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter + from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) import clinic - from clinic import DSLParser - - -def _make_clinic(*, filename='clinic_tests'): - clang = clinic.CLanguage(None) - c = clinic.Clinic(clang, filename=filename, limited_capi=False) - c.block_parser = clinic.BlockParser('', clang) + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic + + +def _make_clinic(*, filename='clinic_tests', limited_capi=False): + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) return c @@ -43,7 +55,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, if strip: code = code.strip() errmsg = re.escape(errmsg) - with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: parser(code) if filename is not None: tc.assertEqual(cm.exception.filename, filename) @@ -52,6 +64,20 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) + + class ClinicWholeFileTest(TestCase): maxDiff = None @@ -60,6 +86,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): + save_restore_converters(self) self.clinic = _make_clinic(filename="test.c") def test_eol(self): @@ -121,11 +148,11 @@ def test_whitespace_before_stop_line(self): self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): - clang = clinic.CLanguage(None) + clang = CLanguage(None) clang.body_prefix = "//" clang.start_line = "//[{dsl_name} start]" clang.stop_line = "//[{dsl_name} stop]" - cl = clinic.Clinic(clang, filename="test.c", limited_capi=False) + cl = Clinic(clang, filename="test.c", limited_capi=False) raw = dedent(""" //[clinic start] //module test @@ -264,70 +291,6 @@ def converter_init(self): ) self.expect_failure(raw, err) - @staticmethod - @contextlib.contextmanager - def _clinic_version(new_version): - """Helper for test_version_*() tests""" - _saved = clinic.version - clinic.version = new_version - try: - yield - finally: - clinic.version = _saved - - def test_version_directive(self): - dataset = ( - # (clinic version, required version) - ('3', '2'), # required version < clinic version - ('3.1', '3.0'), # required version < clinic version - ('1.2b0', '1.2a7'), # required version < clinic version - ('5', '5'), # required version == clinic version - ('6.1', '6.1'), # required version == clinic version - ('1.2b3', '1.2b3'), # required version == clinic version - ) - for clinic_version, required_version in dataset: - with self.subTest(clinic_version=clinic_version, - required_version=required_version): - with self._clinic_version(clinic_version): - block = dedent(f""" - /*[clinic input] - version {required_version} - [clinic start generated code]*/ - """) - self.clinic.parse(block) - - def test_version_directive_insufficient_version(self): - with self._clinic_version('4'): - err = ( - "Insufficient Clinic version!\n" - " Version: 4\n" - " Required: 5" - ) - block = """ - /*[clinic input] - version 5 - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - - def test_version_directive_illegal_char(self): - err = "Illegal character 'v' in version string 'v5'" - block = """ - /*[clinic input] - version v5 - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - - def test_version_directive_unsupported_string(self): - err = "Unsupported version string: '.-'" - block = """ - /*[clinic input] - version .- - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - def test_clone_mismatch(self): err = "'kind' of function and cloned function don't match!" block = """ @@ -359,6 +322,20 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) + def test_multiple_star_in_args(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + *args: object + * + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + def test_module_already_got_one(self): err = "Already defined module 'm'!" block = """ @@ -624,7 +601,7 @@ class C "void *" "" C.__init__ = C.meth [clinic start generated code]*/ """ - err = "'__init__' must be a normal method, not a class or static method" + err = "'__init__' must be a normal method; got 'FunctionKind.CLASS_METHOD'!" self.expect_failure(block, err, lineno=8) def test_validate_cloned_new(self): @@ -685,14 +662,69 @@ class C "void *" "" err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) - with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename, limited_capi=False) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -713,7 +745,7 @@ def test_parse_file_strange_extension(self) -> None: class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): - computed = clinic.permute_optional_groups(l, m, r) + computed = permute_optional_groups(l, m, r) self.assertEqual(output, computed) def test_range(self): @@ -755,13 +787,13 @@ def test_right_only(self): def test_have_left_options_but_required_is_empty(self): def fn(): - clinic.permute_optional_groups(['a'], [], []) + permute_optional_groups(['a'], [], []) self.assertRaises(ValueError, fn) class ClinicLinearFormatTest(TestCase): def _test(self, input, output, **kwargs): - computed = clinic.linear_format(input, **kwargs) + computed = libclinic.linear_format(input, **kwargs) self.assertEqual(output, computed) def test_empty_strings(self): @@ -811,6 +843,19 @@ def test_multiline_substitution(self): def """, name='bingle\nbungle\n') + def test_text_before_block_marker(self): + regex = re.escape("found before '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("no text before marker for you! {marker}", + marker="not allowed!") + + def test_text_after_block_marker(self): + regex = re.escape("found after '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("{marker} no text after marker for you!", + marker="not allowed!") + + class InertParser: def __init__(self, clinic): pass @@ -828,13 +873,12 @@ def parse(self, block): class ClinicBlockParserTest(TestCase): def _test(self, input, output): - language = clinic.CLanguage(None) + language = CLanguage(None) - blocks = list(clinic.BlockParser(input, language)) - writer = clinic.BlockPrinter(language) - c = _make_clinic() + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) for block in blocks: - writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes) + writer.print_block(block) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) @@ -859,8 +903,8 @@ def test_round_trip_2(self): """) def _test_clinic(self, input, output): - language = clinic.CLanguage(None) - c = clinic.Clinic(language, filename="file", limited_capi=False) + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) computed = c.parse(input) @@ -893,7 +937,7 @@ class ClinicParserTest(TestCase): def parse(self, text): c = _make_clinic() parser = DSLParser(c) - block = clinic.Block(text) + block = Block(text) parser.parse(block) return block @@ -901,8 +945,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): block = self.parse(text) s = block.signatures self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) return s[function_index] def expect_failure(self, block, err, *, @@ -917,7 +961,7 @@ def checkDocstring(self, fn, expected): def test_trivial(self): parser = DSLParser(_make_clinic()) - block = clinic.Block(""" + block = Block(""" module os os.access """) @@ -946,7 +990,7 @@ def test_param(self): self.assertEqual(2, len(function.parameters)) p = function.parameters['path'] self.assertEqual('path', p.name) - self.assertIsInstance(p.converter, clinic.int_converter) + self.assertIsInstance(p.converter, int_converter) def test_param_default(self): function = self.parse_function(""" @@ -1045,7 +1089,7 @@ def test_param_no_docstring(self): """) self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_param_default_parameters_out_of_order(self): err = ( @@ -1206,7 +1250,7 @@ def test_base_invalid_syntax(self): Function 'stat' has an invalid parameter declaration: \s+'invalid syntax: int = 42' """).strip() - with self.assertRaisesRegex(clinic.ClinicError, err): + with self.assertRaisesRegex(ClinicError, err): self.parse_function(block) def test_param_default_invalid_syntax(self): @@ -1238,7 +1282,7 @@ def test_return_converter(self): module os os.stat -> int """) - self.assertIsInstance(function.return_converter, clinic.int_return_converter) + self.assertIsInstance(function.return_converter, int_return_converter) def test_return_converter_invalid_syntax(self): block = """ @@ -1950,7 +1994,7 @@ def test_parameters_no_more_than_one_vararg(self): *vararg1: object *vararg2: object """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -2054,7 +2098,7 @@ def test_directive(self): parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) - block = clinic.Block("setflag") + block = Block("setflag") parser.parse(block) self.assertTrue(parser.flag) @@ -2062,7 +2106,7 @@ def test_legacy_converters(self): block = self.parse('module os\nos.access\n path: "s"') module, function = block.signatures conv = (function.parameters['path']).converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_legacy_converters_non_string_constant_annotation(self): err = "Annotations must be either a name, a function call, or a string" @@ -2166,14 +2210,108 @@ class Foo "" "" self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - err = "'__init__' must be a normal method, not a class or static method!" + err_template = "'__init__' must be a normal method; got 'FunctionKind.{}'!" + annotations = { + "@classmethod": "CLASS_METHOD", + "@staticmethod": "STATIC_METHOD", + "@getter": "GETTER", + } + for annotation, invalid_kind in annotations.items(): + with self.subTest(annotation=annotation, invalid_kind=invalid_kind): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.__init__ + """ + expected_error = err_template.format(invalid_kind) + self.expect_failure(block, expected_error, lineno=3) + + def test_init_cannot_define_a_return_type(self): + block = """ + class Foo "" "" + Foo.__init__ -> long + """ + expected_error = "__init__ methods cannot define a return type" + self.expect_failure(block, expected_error, lineno=1) + + def test_invalid_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property -> int + """ + expected_error = f"{annotation} method cannot define a return type" + self.expect_failure(block, expected_error, lineno=3) + + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property + obj: int + / + """ + expected_error = f"{annotation} methods cannot define parameters" + self.expect_failure(block, expected_error) + + def test_setter_docstring(self): block = """ module foo class Foo "" "" - @classmethod - Foo.__init__ + @setter + Foo.property + + foo + + bar + [clinic start generated code]*/ """ - self.expect_failure(block, err, lineno=3) + expected_error = "docstrings are only supported for @getter, not @setter" + self.expect_failure(block, expected_error) + + def test_duplicate_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + {annotation} + Foo.property -> int + """ + expected_error = f"Cannot apply {annotation} twice to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getter_and_setter_disallowed_on_same_function(self): + dup_annotations = [("@getter", "@setter"), ("@setter", "@getter")] + for dup in dup_annotations: + with self.subTest(dup=dup): + block = f""" + module foo + class Foo "" "" + {dup[0]} + {dup[1]} + Foo.property -> int + """ + expected_error = "Cannot apply both @getter and @setter to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getset_no_class(self): + for annotation in "@getter", "@setter": + with self.subTest(annotation=annotation): + block = f""" + module m + {annotation} + m.func + """ + expected_error = "@getter and @setter must be methods" + self.expect_failure(block, expected_error, lineno=2) def test_duplicate_coexist(self): err = "Called @coexist twice" @@ -2225,14 +2363,14 @@ def test_unused_param(self): def test_scaffolding(self): # test repr on special values - self.assertEqual(repr(clinic.unspecified), '') - self.assertEqual(repr(clinic.NULL), '') + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') # test that fail fails with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' - with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: - clinic.fail(errmsg, filename='clown.txt', line_number=69) + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) @@ -2250,10 +2388,10 @@ def test_non_ascii_character_in_docstring(self): self.parse(block) # The line numbers are off; this is a known limitation. expected = dedent("""\ - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'á' - Warning in file 'clinic_tests' on line 0: + Warning: Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' """) @@ -2354,7 +2492,7 @@ def test_state_func_docstring_no_summary(self): docstring1 docstring2 """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=3) def test_state_func_docstring_only_one_param_template(self): err = "You may not specify {parameters} more than once in a docstring!" @@ -2368,12 +2506,34 @@ def test_state_func_docstring_only_one_param_template(self): these are the params again: {parameters} """ - self.expect_failure(block, err, lineno=0) + self.expect_failure(block, err, lineno=7) + + def test_kind_defining_class(self): + function = self.parse_function(""" + module m + class m.C "PyObject *" "" + m.C.meth + cls: defining_class + """, signatures_in_block=3, function_index=2) + p = function.parameters['cls'] + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + def test_disallow_defining_class_at_module_level(self): + err = "A 'defining_class' parameter cannot be defined at module level." + block = """ + module m + m.func + cls: defining_class + """ + self.expect_failure(block, err, lineno=2) class ClinicExternalTest(TestCase): maxDiff = None + def setUp(self): + save_restore_converters(self) + def run_clinic(self, *args): with ( support.captured_stdout() as out, @@ -2464,7 +2624,7 @@ def test_cli_force(self): # Verify by checking the checksum. checksum = ( "/*[clinic end generated code: " - "output=c16447c01510dfb3 input=9543a8d2da235301]*/\n" + "output=0acbef4794cb933e input=9543a8d2da235301]*/\n" ) with open(fn, encoding='utf-8') as f: generated = f.read() @@ -2598,9 +2758,9 @@ def test_cli_converters(self): bool() double() float() - init() int() long() + object() Py_ssize_t() size_t() unsigned_int() @@ -3154,6 +3314,10 @@ def test_posonly_vararg(self): self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) + with self.assertRaises(TypeError): + ac_tester.posonly_vararg(b=4) + with self.assertRaises(TypeError): + ac_tester.posonly_vararg(1, 2, 3, b=4) def test_vararg_and_posonly(self): with self.assertRaises(TypeError): @@ -3204,6 +3368,37 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, err): ac_tester.gh_99240_double_free('a', '\0b') + def test_null_or_tuple_for_varargs(self): + # All of these should not crash: + valid_args_for_test = [ + (('a',), {}, + ('a', (), False)), + (('a', 1, 2, 3), {'covariant': True}, + ('a', (1, 2, 3), True)), + ((), {'name': 'a'}, + ('a', (), False)), + ((), {'name': 'a', 'covariant': True}, + ('a', (), True)), + ((), {'covariant': True, 'name': 'a'}, + ('a', (), True)), + ] + for args, kwargs, expected in valid_args_for_test: + with self.subTest(args=args, kwargs=kwargs): + self.assertEqual( + ac_tester.null_or_tuple_for_varargs(*args, **kwargs), + expected, + ) + + def test_null_or_tuple_for_varargs_error(self): + with self.assertRaises(TypeError): + ac_tester.null_or_tuple_for_varargs(covariant=True) + with self.assertRaises(TypeError): + ac_tester.null_or_tuple_for_varargs(1, name='a') + with self.assertRaises(TypeError): + ac_tester.null_or_tuple_for_varargs(1, 2, 3, name='a', covariant=True) + with self.assertRaises(TypeError): + ac_tester.null_or_tuple_for_varargs(1, 2, 3, covariant=True, name='a') + def test_cloned_func_exception_message(self): incorrect_arg = -1 # f1() and f2() accept a single str with self.assertRaisesRegex(TypeError, "clone_f1"): @@ -3217,6 +3412,50 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_get_defining_class(self): + obj = ac_tester.TestClass() + meth = obj.get_defining_class + self.assertIs(obj.get_defining_class(), ac_tester.TestClass) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_get_defining_class_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.get_defining_class + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + self.assertIs(pyobject_vectorcall(meth, (), ()), ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + + def test_get_defining_class_arg(self): + obj = ac_tester.TestClass() + self.assertEqual(obj.get_defining_class_arg("arg"), + (ac_tester.TestClass, "arg")) + self.assertEqual(obj.get_defining_class_arg(arg=123), + (ac_tester.TestClass, 123)) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass, arg="arg") + + # wrong number of arguments + with self.assertRaises(TypeError): + obj.get_defining_class_arg() + with self.assertRaises(TypeError): + obj.get_defining_class_arg("arg1", "arg2") + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() @@ -3503,6 +3742,46 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + try: import _testclinic_limited except ImportError: @@ -3533,6 +3812,53 @@ def test_my_int_sum(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_sum(1, "str") + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) class PermutationTests(unittest.TestCase): @@ -3546,7 +3872,7 @@ def test_permute_left_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_left_option_groups(data)) + actual = tuple(permute_left_option_groups(data)) self.assertEqual(actual, expected) def test_permute_right_option_groups(self): @@ -3557,7 +3883,7 @@ def test_permute_right_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_right_option_groups(data)) + actual = tuple(permute_right_option_groups(data)) self.assertEqual(actual, expected) def test_permute_optional_groups(self): @@ -3636,7 +3962,7 @@ def test_permute_optional_groups(self): for params in dataset: with self.subTest(**params): left, required, right, expected = params.values() - permutations = clinic.permute_optional_groups(left, required, right) + permutations = permute_optional_groups(left, required, right) actual = tuple(permutations) self.assertEqual(actual, expected) @@ -3659,7 +3985,7 @@ def test_strip_leading_and_trailing_blank_lines(self): ) for lines, expected in dataset: with self.subTest(lines=lines, expected=expected): - out = clinic.strip_leading_and_trailing_blank_lines(lines) + out = libclinic.normalize_snippet(lines) self.assertEqual(out, expected) def test_normalize_snippet(self): @@ -3688,63 +4014,33 @@ def test_normalize_snippet(self): expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} for indent, expected in expected_outputs.items(): with self.subTest(indent=indent): - actual = clinic.normalize_snippet(snippet, indent=indent) + actual = libclinic.normalize_snippet(snippet, indent=indent) self.assertEqual(actual, expected) - def test_accumulator(self): - acc = clinic.text_accumulator() - self.assertEqual(acc.output(), "") - acc.append("a") - self.assertEqual(acc.output(), "a") - self.assertEqual(acc.output(), "") - acc.append("b") - self.assertEqual(acc.output(), "b") - self.assertEqual(acc.output(), "") - acc.append("c") - acc.append("d") - self.assertEqual(acc.output(), "cd") - self.assertEqual(acc.output(), "") - - def test_quoted_for_c_string(self): + def test_escaped_docstring(self): dataset = ( # input, expected - (r"abc", r"abc"), - (r"\abc", r"\\abc"), - (r"\a\bc", r"\\a\\bc"), - (r"\a\\bc", r"\\a\\\\bc"), - (r'"abc"', r'\"abc\"'), - (r"'a'", r"\'a\'"), + (r"abc", r'"abc"'), + (r"\abc", r'"\\abc"'), + (r"\a\bc", r'"\\a\\bc"'), + (r"\a\\bc", r'"\\a\\\\bc"'), + (r'"abc"', r'"\"abc\""'), + (r"'a'", r'"\'a\'"'), ) for line, expected in dataset: with self.subTest(line=line, expected=expected): - out = clinic.quoted_for_c_string(line) + out = libclinic.docstring_for_c_string(line) self.assertEqual(out, expected) - def test_rstrip_lines(self): - lines = ( - "a \n" - "b\n" - " c\n" - " d \n" - ) - expected = ( - "a\n" - "b\n" - " c\n" - " d\n" - ) - out = clinic.rstrip_lines(lines) - self.assertEqual(out, expected) - def test_format_escape(self): line = "{}, {a}" expected = "{{}}, {{a}}" - out = clinic.format_escape(line) + out = libclinic.format_escape(line) self.assertEqual(out, expected) def test_indent_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.indent_all_lines("", prefix="bar"), "") + self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") lines = ( "one\n" @@ -3754,7 +4050,7 @@ def test_indent_all_lines(self): "barone\n" "bartwo" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3770,12 +4066,12 @@ def test_indent_all_lines(self): "bartwo\n" "" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) def test_suffix_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.suffix_all_lines("", suffix="foo"), "") + self.assertEqual(libclinic.suffix_all_lines("", suffix="foo"), "") lines = ( "one\n" @@ -3785,7 +4081,7 @@ def test_suffix_all_lines(self): "onefoo\n" "twofoo" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3801,21 +4097,21 @@ def test_suffix_all_lines(self): "twofoo\n" "" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) class ClinicReprTests(unittest.TestCase): def test_Block_repr(self): - block = clinic.Block("foo") + block = Block("foo") expected_repr = "" self.assertEqual(repr(block), expected_repr) - block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + block2 = Block("bar", "baz", [], "eggs", "spam") expected_repr_2 = "" self.assertEqual(repr(block2), expected_repr_2) - block3 = clinic.Block( + block3 = Block( input="longboi_" * 100, dsl_name="wow_so_long", signatures=[], @@ -3830,47 +4126,44 @@ def test_Block_repr(self): def test_Destination_repr(self): c = _make_clinic() - destination = clinic.Destination( + destination = Destination( "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=c) + destination2 = Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", _make_clinic()) + module = Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): self.assertEqual( - repr(clinic.FunctionKind.INVALID), "" - ) - self.assertEqual( - repr(clinic.FunctionKind.CLASS_METHOD), "" + repr(FunctionKind.CLASS_METHOD), "" ) def test_Function_and_Parameter_reprs(self): - function = clinic.Function( + function = Function( name='foo', module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', - return_converter=clinic.init_return_converter(), - kind=clinic.FunctionKind.METHOD_INIT, + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, coexist=False ) self.assertEqual(repr(function), "") - converter = clinic.self_converter('bar', 'bar', function) - parameter = clinic.Parameter( + converter = self_converter('bar', 'bar', function) + parameter = Parameter( "bar", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, function=function, @@ -3879,7 +4172,7 @@ def test_Function_and_Parameter_reprs(self): self.assertEqual(repr(parameter), "") def test_Monitor_repr(self): - monitor = clinic.cpp.Monitor() + monitor = libclinic.cpp.Monitor("test.c") self.assertRegex(repr(monitor), r"") monitor.line_number = 42 diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py index 319801c71f776b..46ec82b704963d 100644 --- a/Lib/test/test_cmd.py +++ b/Lib/test/test_cmd.py @@ -9,7 +9,10 @@ import doctest import unittest import io +import textwrap from test import support +from test.support.import_helper import import_module +from test.support.pty_helper import run_pty class samplecmdclass(cmd.Cmd): """ @@ -244,23 +247,55 @@ def test_input_reset_at_EOF(self): "(Cmd) *** Unknown syntax: EOF\n")) +class CmdPrintExceptionClass(cmd.Cmd): + """ + GH-80731 + cmd.Cmd should print the correct exception in default() + >>> mycmd = CmdPrintExceptionClass() + >>> try: + ... raise ValueError("test") + ... except ValueError: + ... mycmd.onecmd("not important") + (, ValueError('test')) + """ + + def default(self, line): + print(sys.exc_info()[:2]) + + +@support.requires_subprocess() +class CmdTestReadline(unittest.TestCase): + def setUpClass(): + # Ensure that the readline module is loaded + # If this fails, the test is skipped because SkipTest will be raised + readline = import_module('readline') + + def test_basic_completion(self): + script = textwrap.dedent(""" + import cmd + class simplecmd(cmd.Cmd): + def do_tab_completion_test(self, args): + print('tab completion success') + return True + + simplecmd().cmdloop() + """) + + # 't' and complete 'ab_completion_test' to 'tab_completion_test' + input = b"t\t\n" + + output = run_pty(script, input) + + self.assertIn(b'ab_completion_test', output) + self.assertIn(b'tab completion success', output) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests -def test_coverage(coverdir): - trace = support.import_module('trace') - tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], - trace=0, count=1) - tracer.run('import importlib; importlib.reload(cmd); test_main()') - r=tracer.results() - print("Writing coverage results...") - r.write_results(show_missing=True, summary=True, coverdir=coverdir) if __name__ == "__main__": - if "-c" in sys.argv: - test_coverage('/tmp/cmd.cover') - elif "-i" in sys.argv: + if "-i" in sys.argv: samplecmdclass().cmdloop() else: unittest.main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index eaf19aa160e860..ac99dc4f915f7d 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,11 +5,13 @@ import os import subprocess import sys +import sysconfig import tempfile import textwrap import unittest from test import support from test.support import os_helper +from test.support import force_not_colorized from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, interpreter_requires_environment @@ -38,6 +40,7 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out + @support.cpython_only def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -48,14 +51,17 @@ def test_help(self): self.assertNotIn(b'-X dev', out) self.assertLess(len(lines), 50) + @support.cpython_only def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + @support.cpython_only def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -74,6 +80,7 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') + @support.cpython_only def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -139,6 +146,7 @@ def run_python(*args): else: self.assertEqual(err, b'') + @support.cpython_only def test_xoption_frozen_modules(self): tests = { ('=on', 'FrozenImporter'), @@ -153,6 +161,18 @@ def test_xoption_frozen_modules(self): res = assert_python_ok(*cmd) self.assertRegex(res.out.decode('utf-8'), expected) + @support.cpython_only + def test_env_var_frozen_modules(self): + tests = { + ('on', 'FrozenImporter'), + ('off', 'SourceFileLoader'), + } + for raw, expected in tests: + cmd = ['-c', 'import os; print(os.__spec__.loader, end="")'] + with self.subTest(raw): + res = assert_python_ok(*cmd, PYTHON_FROZEN_MODULES=raw) + self.assertRegex(res.out.decode('utf-8'), expected) + def test_run_module(self): # Test expected operation of the '-m' switch # Switch needs an argument @@ -468,8 +488,9 @@ def test_stdout_flush_at_shutdown(self): rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) self.assertEqual(120, rc) - self.assertRegex(err.decode('ascii', 'ignore'), - 'Exception ignored in.*\nOSError: .*') + self.assertIn(b'Exception ignored on flushing sys.stdout:\n' + b'OSError: '.replace(b'\n', os.linesep.encode()), + err) def test_closed_stdout(self): # Issue #13444: if stdout has been explicitly closed, we should @@ -567,6 +588,7 @@ def test_del___main__(self): print("del sys.modules['__main__']", file=script) assert_python_ok(filename) + @support.cpython_only def test_unknown_options(self): rc, out, err = assert_python_failure('-E', '-z') self.assertIn(b'Unknown option: -z', err) @@ -622,15 +644,13 @@ def test_sys_flags_set(self): PYTHONDONTWRITEBYTECODE=value, PYTHONVERBOSE=value, ) - dont_write_bytecode = int(bool(value)) + expected_bool = int(bool(value)) code = ( "import sys; " "sys.stderr.write(str(sys.flags)); " f"""sys.exit(not ( - sys.flags.debug == sys.flags.optimize == - sys.flags.verbose == - {expected} - and sys.flags.dont_write_bytecode == {dont_write_bytecode} + sys.flags.optimize == sys.flags.verbose == {expected} + and sys.flags.debug == sys.flags.dont_write_bytecode == {expected_bool} ))""" ) with self.subTest(envar_value=value): @@ -681,6 +701,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() + @support.cpython_only def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -717,7 +738,7 @@ def test_xdev(self): # Memory allocator debug hooks try: - import _testinternalcapi + import _testinternalcapi # noqa: F401 except ImportError: pass else: @@ -726,13 +747,15 @@ def test_xdev(self): out = self.run_xdev("-c", code, check_exitcode=False) if support.with_pymalloc(): alloc_name = "pymalloc_debug" + elif support.Py_GIL_DISABLED: + alloc_name = "mimalloc_debug" else: alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) # Faulthandler try: - import faulthandler + import faulthandler # noqa: F401 except ImportError: pass else: @@ -802,8 +825,13 @@ def check_pythonmalloc(self, env_var, name): @support.cpython_only def test_pythonmalloc(self): # Test the PYTHONMALLOC environment variable + malloc = not support.Py_GIL_DISABLED pymalloc = support.with_pymalloc() - if pymalloc: + mimalloc = support.with_mimalloc() + if support.Py_GIL_DISABLED: + default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc' + default_name_debug = 'mimalloc_debug' + elif pymalloc: default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc' default_name_debug = 'pymalloc_debug' else: @@ -813,14 +841,22 @@ def test_pythonmalloc(self): tests = [ (None, default_name), ('debug', default_name_debug), - ('malloc', 'malloc'), - ('malloc_debug', 'malloc_debug'), ] + if malloc: + tests.extend([ + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ]) if pymalloc: tests.extend(( ('pymalloc', 'pymalloc'), ('pymalloc_debug', 'pymalloc_debug'), )) + if mimalloc: + tests.extend(( + ('mimalloc', 'mimalloc'), + ('mimalloc_debug', 'mimalloc_debug'), + )) for env_var, name in tests: with self.subTest(env_var=env_var, name=name): @@ -844,6 +880,108 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds") + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', None, '1', "PYTHON_GIL=1"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + (None, '1', '1', "-X gil=1"), + ] + + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + + def test_python_asyncio_debug(self): + code = "import asyncio; print(asyncio.get_event_loop().get_debug())" + rc, out, err = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1') + self.assertIn(b'True', out) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs(self): + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + self.assertEqual(rc, 0) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs_file(self): + with tempfile.NamedTemporaryFile() as dump_file: + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + self.assertEqual(rc, 0) + with open(dump_file.name, 'r') as file: + contents = file.read() + self.assertIn('Remaining objects', contents) + + @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS') + def test_python_executable(self): + code = 'import sys; print(sys.executable)' + expected = "/busr/bbin/bpython" + rc, out, err = assert_python_ok('-c', code, PYTHONEXECUTABLE=expected) + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_fs_encoding(self): + code = "import sys; print(sys.getfilesystemencoding())" + expected = 'mbcs' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSFSENCODING='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio(self): + code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" + expected = 'cp' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') + self.assertIn(expected.encode(), out) + + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), + "PYTHONMALLOCSTATS doesn't work with ASAN") + def test_python_malloc_stats(self): + code = "pass" + rc, out, err = assert_python_ok('-c', code, PYTHONMALLOCSTATS='1') + self.assertIn(b'Small block threshold', err) + + def test_python_user_base(self): + code = "import site; print(site.USER_BASE)" + expected = "/custom/userbase" + rc, out, err = assert_python_ok('-c', code, PYTHONUSERBASE=expected) + self.assertIn(expected.encode(), out) + + def test_python_basic_repl(self): + # Currently this only tests that the env var is set. See test_pyrepl.test_python_basic_repl. + code = "import os; print('PYTHON_BASIC_REPL' in os.environ)" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_BASIC_REPL='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(sysconfig.get_config_var('HAVE_PERF_TRAMPOLINE'), "Requires HAVE_PERF_TRAMPOLINE support") + def test_python_perf_jit_support(self): + code = "import sys; print(sys.is_stack_trampoline_active())" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_PERF_JIT_SUPPORT='1') + self.assertIn(expected.encode(), out) + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): @@ -856,6 +994,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.returncode, 0, proc) self.assertEqual(proc.stdout.strip(), b'0') + @support.cpython_only def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, @@ -912,7 +1051,7 @@ def test_cpu_count_default(self): self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): @@ -959,6 +1098,7 @@ def test_sys_flags_not_set(self): class SyntaxErrorTests(unittest.TestCase): + @force_not_colorized def check_string(self, code): proc = subprocess.run([sys.executable, "-"], input=code, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 614c6b3c3b5299..3a5a8abf81e43d 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -14,8 +14,7 @@ import textwrap from test import support -from test.support import import_helper -from test.support import os_helper +from test.support import import_helper, is_apple, os_helper from test.support.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, assert_python_ok, assert_python_failure, spawn_python, kill_python) @@ -557,12 +556,17 @@ def test_pep_409_verbiage(self): self.assertTrue(text[3].startswith('NameError')) def test_non_ascii(self): - # Mac OS X denies the creation of a file with an invalid UTF-8 name. + # Apple platforms deny the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. - # WASI does not permit invalid UTF-8 names. - if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + # Emscripten/WASI does not permit invalid UTF-8 names. + if ( + os_helper.TESTFN_UNDECODABLE + and sys.platform not in { + "win32", "emscripten", "wasi" + } + and not is_apple + ): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII @@ -684,6 +688,27 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) + def test_source_lines_are_shown_when_running_source(self): + _, _, stderr = assert_python_failure("-c", "1/0") + expected_lines = [ + b'Traceback (most recent call last):', + b' File "", line 1, in ', + b' 1/0', + b' ~^~', + b'ZeroDivisionError: division by zero'] + self.assertEqual(stderr.splitlines(), expected_lines) + + def test_syntaxerror_does_not_crash(self): + script = "nonlocal x\n" + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script', script) + exitcode, stdout, stderr = assert_python_failure(script_name) + text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() + # It used to crash in https://github.com/python/cpython/issues/111132 + self.assertTrue(text.endswith( + 'SyntaxError: nonlocal declaration not allowed at module level\n', + ), text) + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index a961ddbe17a3d3..ba77e1c5341db8 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -141,9 +141,12 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect) + gc_collect, Py_GIL_DISABLED, + suppress_immortalization, + skip_if_suppress_immortalization) from test.support.script_helper import assert_python_ok -from test.support import threading_helper +from test.support import threading_helper, import_helper +from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -174,7 +177,7 @@ class CodeTest(unittest.TestCase): @cpython_only def test_newempty(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") @@ -384,10 +387,8 @@ def test_co_positions_artificial_instructions(self): code = traceback.tb_frame.f_code artificial_instructions = [] - for instr, positions in zip( - dis.get_instructions(code, show_caches=True), - code.co_positions(), - strict=True + for instr, positions in instructions_with_positions( + dis.get_instructions(code), code.co_positions() ): # If any of the positions is None, then all have to # be None as well for the case above. There are still @@ -570,14 +571,35 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + @skip_if_suppress_immortalization() + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + class CodeWeakRefTest(unittest.TestCase): + @suppress_immortalization() def test_basic(self): # Create a code object in a clean environment so that we know we have # the only reference to it left. @@ -761,7 +783,7 @@ def f(): co_code=bytes( [ dis.opmap["RESUME"], 0, - dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["LOAD_COMMON_CONSTANT"], 0, dis.opmap["RAISE_VARARGS"], 1, ] ), @@ -828,6 +850,7 @@ def test_bad_index(self): self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, ctypes.c_voidp(100)), 0) + @suppress_immortalization() def test_free_called(self): # Verify that the provided free function gets invoked # when the code object is cleaned up. @@ -835,6 +858,7 @@ def test_free_called(self): SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) del f + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 100) def test_get_set(self): @@ -854,6 +878,7 @@ def test_get_set(self): del f @threading_helper.requires_working_threading() + @suppress_immortalization() def test_free_different_thread(self): # Freeing a code object on a different thread then # where the co_extra was set should be safe. @@ -865,13 +890,19 @@ def __init__(self, f, test): self.test = test def run(self): del self.f - self.test.assertEqual(LAST_FREED, 500) + gc_collect() + # gh-117683: In the free-threaded build, the code object's + # destructor may still be running concurrently in the main + # thread. + if not Py_GIL_DISABLED: + self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) tt = ThreadTest(f, self) del f tt.start() tt.join() + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 500) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 226bc3a853b7b9..259778a5cade98 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -10,11 +10,7 @@ code = import_helper.import_module('code') -class TestInteractiveConsole(unittest.TestCase): - - def setUp(self): - self.console = code.InteractiveConsole() - self.mock_sys() +class MockSys: def mock_sys(self): "Mock system environment for InteractiveConsole" @@ -32,6 +28,13 @@ def mock_sys(self): del self.sysmod.ps1 del self.sysmod.ps2 + +class TestInteractiveConsole(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole() + self.mock_sys() + def test_ps1(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() @@ -151,5 +154,22 @@ def test_context_tb(self): self.assertIn(expected, output) +class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole(local_exit=True) + self.mock_sys() + + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") + def test_exit(self): + # default exit message + self.infunc.side_effect = ["exit()"] + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 00ea1c39dd6fb6..027dbecc6134df 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -24,6 +24,52 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): (b'ab\x1BNdef', 'replace', 'abdef'), ) +class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_3' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(O\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(O\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'), + ('ab\u4FF1def', 'replace', b'ab?def'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2004' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(Q\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(Q\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'), + ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩' + ) + class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_kr' tstring = multibytecodec_support.load_teststring('iso2022_kr') diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index ff511a625a0194..428036e1765b8f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -13,9 +13,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -482,11 +482,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -666,11 +666,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -1356,13 +1356,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1532,7 +1548,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1542,11 +1558,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("ã‚ã•.\u034f", UnicodeEncodeError("idna", "ã‚ã•.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1554,10 +1605,21 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1595,6 +1657,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1623,6 +1718,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("ã‚ã•.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -2112,14 +2224,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2128,7 +2240,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2139,12 +2251,12 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, @@ -2846,7 +2958,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index e3c382266fa058..787bd1b6a79e20 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -5,6 +5,7 @@ import unittest import warnings from test.support import warnings_helper +from textwrap import dedent from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT @@ -222,6 +223,9 @@ def test_incomplete(self): ai("(x for x in") ai("(x for x in (") + ai('a = f"""') + ai('a = \\') + def test_invalid(self): ai = self.assertInvalid ai("a b") @@ -308,6 +312,19 @@ def test_invalid_warning(self): self.assertRegex(str(w[0].message), 'invalid escape sequence') self.assertEqual(w[0].filename, '') + def assertSyntaxErrorMatches(self, code, message): + with self.subTest(code): + with self.assertRaisesRegex(SyntaxError, message): + compile_command(code, symbol='exec') + + def test_syntax_errors(self): + self.assertSyntaxErrorMatches( + dedent("""\ + def foo(x,x): + pass + """), "duplicate argument 'x' in function definition") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index bb8b352518ef3e..a24d3e3ea142b7 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,5 +1,6 @@ """Unit tests for collections.py.""" +import array import collections import copy import doctest @@ -25,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -488,12 +489,8 @@ def test_instance(self): self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method - try: + with self.assertRaises(TypeError): p._replace(x=1, error=2) - except ValueError: - pass - else: - self._fail('Did not detect an incorrect fieldname') # verify that field string can have commas Point = namedtuple('Point', 'x, y') @@ -545,7 +542,7 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = support.EXCEEDS_RECURSION_LIMIT + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) @@ -1938,28 +1935,6 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) - def test_ByteString(self): - for sample in [bytes, bytearray]: - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(sample(), ByteString) - self.assertTrue(issubclass(sample, ByteString)) - for sample in [str, list, tuple]: - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(sample(), ByteString) - self.assertFalse(issubclass(sample, ByteString)) - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(memoryview(b""), ByteString) - self.assertFalse(issubclass(memoryview, ByteString)) - with self.assertWarns(DeprecationWarning): - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') - - with self.assertWarns(DeprecationWarning): - class X(ByteString): pass - - with self.assertWarns(DeprecationWarning): - # No metaclass conflict - class Z(ByteString, Awaitable): pass - def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1976,6 +1951,7 @@ def test_MutableSequence(self): for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) self.assertTrue(issubclass(sample, MutableSequence)) + self.assertTrue(issubclass(array.array, MutableSequence)) self.assertFalse(issubclass(str, MutableSequence)) self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index c4452e38934cf8..9def47e101b496 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,5 +1,8 @@ +import contextlib import dis +import io import math +import opcode import os import unittest import sys @@ -9,9 +12,12 @@ import types import textwrap import warnings +import _testinternalcapi + from test import support -from test.support import (script_helper, requires_debug_ranges, - requires_specialization, Py_C_RECURSION_LIMIT) +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization, get_c_recursion_limit) +from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -111,7 +117,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = int(Py_C_RECURSION_LIMIT * 0.9) + repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -435,14 +441,40 @@ class A: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) self.assertIn("__package__", A.f.__code__.co_varnames) + def test_condition_expression_with_dead_blocks_compiles(self): + # See gh-113054 + compile('if (5 if 5 else T): 0', '', 'exec') + + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054, gh-114083 + exprs = [ + 'if 9<9<9and 9or 9:9', + 'if 9<9<9and 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9or 9:9', + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertWarns(SyntaxWarning): + compile(expr, '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -470,6 +502,58 @@ def test_compile_invalid_namedexpr(self): with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): compile(ast.fix_missing_locations(m), "", "exec") + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -498,11 +582,11 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') def test_compile_invalid_typealias(self): @@ -604,12 +688,11 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is Py_C_RECURSION_LIMIT * 2 - # Duplicating the limit here is a little ugly. - # Perhaps it should be exposed somewhere... - fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 - crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 1.8) + # Expected limit is Py_C_RECURSION_LIMIT + limit = get_c_recursion_limit() + fail_depth = limit + 1 + crash_depth = limit * 100 + success_depth = int(limit * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth @@ -722,7 +805,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - ("docstring", "used")) + (f.__doc__, "used")) @support.cpython_only def test_remove_unused_consts_no_docstring(self): @@ -767,7 +850,7 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__, 42)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -787,6 +870,30 @@ def unused_code_at_end(): 'RETURN_CONST', list(dis.get_instructions(unused_code_at_end))[-1].opname) + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP' , output.getvalue()) + def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -996,6 +1103,8 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) @@ -1079,19 +1188,72 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) - def test_lineno_of_backward_jump(self): + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + + def test_line_number_synthetic_jump_multiple_predecessors(self): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): # Issue gh-107901 def f(): for i in x: if y: pass - linenos = list(inst.positions.lineno - for inst in dis.get_instructions(f.__code__) - if inst.opname == 'JUMP_BACKWARD') - - self.assertTrue(len(linenos) > 0) - self.assertTrue(all(l is not None for l in linenos)) + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles @@ -1283,6 +1445,33 @@ def test_remove_redundant_nop_edge_case(self): def f(): a if (1 if b else c) else d + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers @@ -1329,8 +1518,8 @@ def generic_visit(self, node): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 @@ -1403,6 +1592,7 @@ def test_multiline_boolean_expression(self): self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', line=4, end_line=4, column=8, end_column=13, occurrence=2) + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and @@ -1410,7 +1600,7 @@ def test_multiline_assert(self): ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', @@ -1830,11 +2020,132 @@ def test_column_offset_deduplication(self): def test_load_super_attr(self): source = "class C:\n def __init__(self):\n super().__init__()" - code = compile(source, "", "exec").co_consts[0].co_consts[1] + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break self.assertOpcodeSourcePositionIs( code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 ) + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect three load None instructions for the no-exception __exit__ call, + # and one RETURN_VALUE. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 3) + self.assertEqual(len(return_value), 1) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestExpectedAttributes(unittest.TestCase): + + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -2236,6 +2547,49 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +class TestInstructionSequence(unittest.TestCase): + def compare_instructions(self, seq, expected): + self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], + expected) + + def test_basics(self): + seq = _testinternalcapi.new_instruction_sequence() + + def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): + seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) + + add_op(seq, 'LOAD_CONST', 1, 1) + add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) + add_op(seq, 'LOAD_CONST', 1, 3) + add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) + seq.use_label(lbl1) + add_op(seq, 'LOAD_CONST', 2, 4) + seq.use_label(lbl2) + add_op(seq, 'RETURN_VALUE', 0, 3) + + expected = [('LOAD_CONST', 1, 1), + ('JUMP', 4, 2), + ('LOAD_CONST', 1, 3), + ('JUMP', 5, 4), + ('LOAD_CONST', 2, 4), + ('RETURN_VALUE', None, 3), + ] + + self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) + + def test_nested(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + nested = _testinternalcapi.new_instruction_sequence() + nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) + + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + seq.add_nested(nested) + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 9cd92ad365c5a9..3a34c6822bc079 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -4,7 +4,6 @@ import importlib.util import io import os -import pathlib import py_compile import shutil import struct @@ -19,7 +18,7 @@ # compileall relies on ProcessPoolExecutor if ProcessPoolExecutor exists # and it can function. from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests - from concurrent.futures import ProcessPoolExecutor + from concurrent.futures import ProcessPoolExecutor # noqa: F401 from concurrent.futures.process import _check_system_limits _check_system_limits() _have_multiprocessing = True @@ -31,6 +30,7 @@ from test.support import script_helper from test.test_py_compile import without_source_date_epoch from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import FakePath def get_pyc(script, opt): @@ -156,28 +156,28 @@ def test_compile_file_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) # we should also test the output with support.captured_stdout() as stdout: - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) + self.assertTrue(compileall.compile_file(FakePath(self.source_path))) self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_ddir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - ddir=pathlib.Path('ddir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + ddir=FakePath('ddir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -228,22 +228,22 @@ def test_optimize(self): def test_compile_dir_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) with support.captured_stdout() as stdout: - compileall.compile_dir(pathlib.Path(self.directory)) + compileall.compile_dir(FakePath(self.directory)) line = stdout.getvalue().splitlines()[0] self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -362,6 +362,31 @@ def test_strip_only(self): str(err, encoding=sys.getdefaultencoding()) ) + def test_strip_only_invalid(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + stripdir = os.path.join(self.directory, *(fullpath[:2] + ['fake'])) + with support.captured_stdout() as out: + compileall.compile_dir(path, quiet=True, stripdir=stripdir) + self.assertIn("not a valid prefix", out.getvalue()) + rc, out, err = script_helper.assert_python_failure(bc) + expected_not_in = os.path.join(self.directory, *fullpath[2:]) + self.assertIn( + path, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + expected_not_in, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + stripdir, + str(err, encoding=sys.getdefaultencoding()) + ) + def test_prepend_only(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -477,19 +502,25 @@ def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') with open(self.source_path, 'w', encoding='utf-8') as file: - file.write('# -*- coding: utf-8 -*-\n') - file.write('print u"\u20ac"\n') + # Intentional syntax error: bytes can only contain + # ASCII literal characters. + file.write('b"\u20ac"') def tearDown(self): shutil.rmtree(self.directory) def test_error(self): - try: - orig_stdout = sys.stdout - sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') - compileall.compile_dir(self.directory) - finally: - sys.stdout = orig_stdout + buffer = io.TextIOWrapper(io.BytesIO(), encoding='ascii') + with contextlib.redirect_stdout(buffer): + compiled = compileall.compile_dir(self.directory) + self.assertFalse(compiled) # should not be successful + buffer.seek(0) + res = buffer.read() + self.assertIn( + 'SyntaxError: bytes can only contain ASCII literal characters', + res, + ) + self.assertNotIn('UnicodeEncodeError', res) class CommandLineTestsBase: @@ -952,7 +983,7 @@ class CommandLineTestsNoSourceEpoch(CommandLineTestsBase, -@unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') +@os_helper.skip_unless_hardlink class HardlinkDedupTestsBase: # Test hardlink_dupes parameter of compileall.compile_dir() diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 5696433e529d0a..1b98b0d97ed8a5 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -1,3 +1,6 @@ +import dis +import io +import textwrap import types from test.support.bytecode_helper import AssemblerTestCase @@ -22,11 +25,13 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault('filename', filename) return metadata - def assemble_test(self, insts, metadata, expected): + def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) - insts = self.complete_insts_info(insts) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) - co = self.get_code_object(metadata['filename'], insts, metadata) + def assemble_test(self, insts, metadata, expected): + co = self.insts_to_code_object(insts, metadata) self.assertIsInstance(co, types.CodeType) expected_metadata = {} @@ -66,7 +71,7 @@ def test_simple_expr(self): ('BINARY_OP', 0, 1), # '+' ('LOAD_CONST', 0, 1), # 2 ('BINARY_OP', 11, 1), # '/' - ('RETURN_VALUE', 1), + ('RETURN_VALUE', None, 1), ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) @@ -97,14 +102,46 @@ def inner(): ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), - ('MAKE_FUNCTION', 0, 2), + ('MAKE_FUNCTION', None, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), - ('PUSH_NULL', 0, 1), + ('PUSH_NULL', None, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % - ('RETURN_VALUE', 0, 2) + ('RETURN_VALUE', None, 2) ] expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} self.assemble_test(instructions, metadata, expected) + + + def test_exception_table(self): + metadata = { + 'filename' : 'exc.py', + 'name' : 'exc', + 'consts' : {2 : 0}, + } + + # code for "try: pass\n except: pass" + insts = [ + ('RESUME', 0), + ('SETUP_FINALLY', 3), + ('RETURN_CONST', 0), + ('SETUP_CLEANUP', 8), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), + ('RETURN_CONST', 0), + ('COPY', 3), + ('POP_EXCEPT', None), + ('RERAISE', 1), + ] + co = self.insts_to_code_object(insts, metadata) + output = io.StringIO() + dis.dis(co, file=output) + exc_table = textwrap.dedent(""" + ExceptionTable: + L1 to L2 -> L2 [0] + L2 to L3 -> L3 [1] lasti + """) + self.assertTrue(output.getvalue().endswith(exc_table)) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 6d7731ddba02c5..d82fb85ed259ab 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -1,4 +1,5 @@ +import textwrap from test.support.bytecode_helper import CodegenTestCase # Tests for the code-generation stage of the compiler. @@ -6,11 +7,19 @@ class IsolatedCodeGenTests(CodegenTestCase): + def assertInstructionsMatch_recursive(self, insts, expected_insts): + expected_nested = [i for i in expected_insts if isinstance(i, list)] + expected_insts = [i for i in expected_insts if not isinstance(i, list)] + self.assertInstructionsMatch(insts, expected_insts) + self.assertEqual(len(insts.get_nested()), len(expected_nested)) + for n_insts, n_expected in zip(insts.get_nested(), expected_nested): + self.assertInstructionsMatch_recursive(n_insts, n_expected) + def codegen_test(self, snippet, expected_insts): import ast - a = ast.parse(snippet, "my_file.py", "exec"); + a = ast.parse(snippet, "my_file.py", "exec") insts = self.generate_code(a) - self.assertInstructionsMatch(insts, expected_insts) + self.assertInstructionsMatch_recursive(insts, expected_insts) def test_if_expression(self): snippet = "42 if True else 24" @@ -21,7 +30,7 @@ def test_if_expression(self): ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), ('LOAD_CONST', 1, 1), - ('JUMP', exit_lbl := self.Label()), + ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, ('LOAD_CONST', 2, 1), exit_lbl, @@ -40,6 +49,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), @@ -49,7 +59,98 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('POP_TOP', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) + + def test_function(self): + snippet = textwrap.dedent(""" + def f(x): + return x + 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_FAST', 0), + ('LOAD_CONST', 1), + ('BINARY_OP', 0), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + ] + self.codegen_test(snippet, expected) + + def test_nested_functions(self): + snippet = textwrap.dedent(""" + def f(): + def h(): + return 12 + def g(): + x = 1 + y = 2 + z = 3 + u = 4 + return 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_CONST', 1), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 1), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + [ + ('RESUME', 0), + ('NOP', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ], + [ + ('RESUME', 0), + ('LOAD_CONST', 1), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('STORE_FAST', 1), + ('LOAD_CONST', 3), + ('STORE_FAST', 2), + ('LOAD_CONST', 4), + ('STORE_FAST', 3), + ('NOP', None), + ('LOAD_CONST', 5), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ], + ], + ] + self.codegen_test(snippet, expected) + + def test_syntax_error__return_not_in_function(self): + snippet = "return 42" + with self.assertRaisesRegex(SyntaxError, "'return' outside function"): + self.codegen_test(snippet, None) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 873e5c079f0c09..e3a2205c43d747 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -5,57 +5,53 @@ INVALID_UNDERSCORE_LITERALS) from random import random -from math import atan2, isnan, copysign +from math import isnan, copysign import operator INF = float("inf") NAN = float("nan") +DBL_MAX = sys.float_info.max +# These tests ensure that complex math does the right thing -class ComplexSubclass(complex): - pass - -class OtherComplexSubclass(complex): - pass +ZERO_DIVISION = ( + (1+1j, 0+0j), + (1+1j, 0.0), + (1+1j, 0), + (1.0, 0+0j), + (1, 0+0j), +) -class MyIndex: +class WithIndex: def __init__(self, value): self.value = value - def __index__(self): return self.value -class MyInt: +class WithFloat: def __init__(self, value): self.value = value - - def __int__(self): + def __float__(self): return self.value -class FloatLike: +class ComplexSubclass(complex): + pass + +class OtherComplexSubclass(complex): + pass + +class MyInt: def __init__(self, value): self.value = value - def __float__(self): + def __int__(self): return self.value -class ComplexLike: +class WithComplex: def __init__(self, value): self.value = value - def __complex__(self): return self.value - -# These tests ensure that complex math does the right thing - -ZERO_DIVISION = ( - (1+1j, 0+0j), - (1+1j, 0.0), - (1+1j, 0), - (1.0, 0+0j), - (1, 0+0j), -) - class ComplexTest(unittest.TestCase): def assertAlmostEqual(self, a, b): @@ -108,6 +104,10 @@ def assertFloatsAreIdentical(self, x, y): msg += ': zeros have different signs' self.fail(msg.format(x, y)) + def assertComplexesAreIdentical(self, x, y): + self.assertFloatsAreIdentical(x.real, y.real) + self.assertFloatsAreIdentical(x.imag, y.imag) + def assertClose(self, x, y, eps=1e-9): """Return true iff complexes x and y "are close".""" self.assertCloseAbs(x.real, y.real, eps) @@ -145,12 +145,41 @@ def test_truediv(self): complex(random(), random())) self.assertAlmostEqual(complex.__truediv__(2+0j, 1+1j), 1-1j) + self.assertRaises(TypeError, operator.truediv, 1j, None) + self.assertRaises(TypeError, operator.truediv, None, 1j) for denom_real, denom_imag in [(0, NAN), (NAN, 0), (NAN, NAN)]: z = complex(0, 0) / complex(denom_real, denom_imag) self.assertTrue(isnan(z.real)) self.assertTrue(isnan(z.imag)) + self.assertComplexesAreIdentical(complex(INF, 1)/(0.0+1j), + complex(NAN, -INF)) + + # test recover of infs if numerator has infs and denominator is finite + self.assertComplexesAreIdentical(complex(INF, -INF)/(1+0j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(INF, INF)/(0.0+1j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(NAN, INF)/complex(2**1000, 2**-1000), + complex(INF, INF)) + self.assertComplexesAreIdentical(complex(INF, NAN)/complex(2**1000, 2**-1000), + complex(INF, -INF)) + + # test recover of zeros if denominator is infinite + self.assertComplexesAreIdentical((1+1j)/complex(INF, INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(INF, -INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, INF), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, -INF), + complex(-0.0, 0)) + self.assertComplexesAreIdentical((INF+1j)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(1, INF)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(INF, 1)/complex(1, INF), + complex(NAN, NAN)) + def test_truediv_zero_division(self): for a, b in ZERO_DIVISION: with self.assertRaises(ZeroDivisionError): @@ -176,6 +205,7 @@ def test_floordiv_zero_division(self): def test_richcompare(self): self.assertIs(complex.__eq__(1+1j, 1<<10000), False) self.assertIs(complex.__lt__(1+1j, None), NotImplemented) + self.assertIs(complex.__eq__(1+1j, None), NotImplemented) self.assertIs(complex.__eq__(1+1j, 1+1j), True) self.assertIs(complex.__eq__(1+1j, 2+2j), False) self.assertIs(complex.__ne__(1+1j, 1+1j), False) @@ -198,6 +228,7 @@ def test_richcompare(self): self.assertIs(operator.eq(1+1j, 2+2j), False) self.assertIs(operator.ne(1+1j, 1+1j), False) self.assertIs(operator.ne(1+1j, 2+2j), True) + self.assertIs(operator.eq(1+1j, 2.0), False) def test_richcompare_boundaries(self): def check(n, deltas, is_equal, imag = 0.0): @@ -216,6 +247,27 @@ def check(n, deltas, is_equal, imag = 0.0): check(2 ** pow, range(1, 101), lambda delta: False, float(i)) check(2 ** 53, range(-100, 0), lambda delta: True) + def test_add(self): + self.assertEqual(1j + int(+1), complex(+1, 1)) + self.assertEqual(1j + int(-1), complex(-1, 1)) + self.assertRaises(OverflowError, operator.add, 1j, 10**1000) + self.assertRaises(TypeError, operator.add, 1j, None) + self.assertRaises(TypeError, operator.add, None, 1j) + + def test_sub(self): + self.assertEqual(1j - int(+1), complex(-1, 1)) + self.assertEqual(1j - int(-1), complex(1, 1)) + self.assertRaises(OverflowError, operator.sub, 1j, 10**1000) + self.assertRaises(TypeError, operator.sub, 1j, None) + self.assertRaises(TypeError, operator.sub, None, 1j) + + def test_mul(self): + self.assertEqual(1j * int(20), complex(0, 20)) + self.assertEqual(1j * int(-1), complex(0, -1)) + self.assertRaises(OverflowError, operator.mul, 1j, 10**1000) + self.assertRaises(TypeError, operator.mul, 1j, None) + self.assertRaises(TypeError, operator.mul, None, 1j) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -248,11 +300,18 @@ def test_divmod_zero_division(self): def test_pow(self): self.assertAlmostEqual(pow(1+1j, 0+0j), 1.0) self.assertAlmostEqual(pow(0+0j, 2+0j), 0.0) + self.assertEqual(pow(0+0j, 2000+0j), 0.0) + self.assertEqual(pow(0, 0+0j), 1.0) + self.assertEqual(pow(-1, 0+0j), 1.0) self.assertRaises(ZeroDivisionError, pow, 0+0j, 1j) + self.assertRaises(ZeroDivisionError, pow, 0+0j, -1000) self.assertAlmostEqual(pow(1j, -1), 1/1j) self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(TypeError, pow, 1j, None) + self.assertRaises(TypeError, pow, None, 1j) + self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) a = 3.33+4.43j self.assertEqual(a ** 0j, 1) @@ -337,134 +396,124 @@ def test_boolcontext(self): for i in range(100): self.assertTrue(complex(random() + 1e-6, random() + 1e-6)) self.assertTrue(not complex(0.0, 0.0)) + self.assertTrue(1j) def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): - self.assertEqual(complex(ComplexLike(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, ComplexLike(None)) - self.assertRaises(TypeError, complex, {}) - self.assertRaises(TypeError, complex, ComplexLike(1.5)) - self.assertRaises(TypeError, complex, ComplexLike(1)) - - self.assertAlmostEqual(complex("1+10j"), 1+10j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10.0), 10+0j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10+0j), 10+0j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10.0), 1+10j) - self.assertAlmostEqual(complex(3.14+0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14), 3.14+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(3.14+0j, 0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14, 0.0), 3.14+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(0j, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0.0, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0j, 3.14), 3.14j) - self.assertAlmostEqual(complex(0.0, 3.14), 3.14j) - self.assertAlmostEqual(complex("1"), 1+0j) - self.assertAlmostEqual(complex("1j"), 1j) - self.assertAlmostEqual(complex(), 0) - self.assertAlmostEqual(complex("-1"), -1) - self.assertAlmostEqual(complex("+1"), +1) - self.assertAlmostEqual(complex("(1+2j)"), 1+2j) - self.assertAlmostEqual(complex("(1.3+2.2j)"), 1.3+2.2j) - self.assertAlmostEqual(complex("3.14+1J"), 3.14+1j) - self.assertAlmostEqual(complex(" ( +3.14-6J )"), 3.14-6j) - self.assertAlmostEqual(complex(" ( +3.14-J )"), 3.14-1j) - self.assertAlmostEqual(complex(" ( +3.14+j )"), 3.14+1j) - self.assertAlmostEqual(complex("J"), 1j) - self.assertAlmostEqual(complex("( j )"), 1j) - self.assertAlmostEqual(complex("+J"), 1j) - self.assertAlmostEqual(complex("( -j)"), -1j) - self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) - self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) - self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) - - self.assertAlmostEqual(complex(ComplexSubclass(1+1j)), 1+1j) - self.assertAlmostEqual(complex(real=17, imag=23), 17+23j) - self.assertAlmostEqual(complex(real=17+23j), 17+23j) - self.assertAlmostEqual(complex(real=17+23j, imag=23), 17+46j) - self.assertAlmostEqual(complex(real=1+2j, imag=3+4j), -3+5j) + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex(), 0.0, 0.0) + check(complex(10), 10.0, 0.0) + check(complex(4.25), 4.25, 0.0) + check(complex(4.25+0j), 4.25, 0.0) + check(complex(4.25+0.5j), 4.25, 0.5) + check(complex(ComplexSubclass(4.25+0.5j)), 4.25, 0.5) + check(complex(WithComplex(4.25+0.5j)), 4.25, 0.5) + + check(complex(1, 10), 1.0, 10.0) + check(complex(1, 10.0), 1.0, 10.0) + check(complex(1, 4.25), 1.0, 4.25) + check(complex(1.0, 10), 1.0, 10.0) + check(complex(4.25, 10), 4.25, 10.0) + check(complex(1.0, 10.0), 1.0, 10.0) + check(complex(4.25, 0.5), 4.25, 0.5) + + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*ComplexSubclass"): + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*WithComplex"): + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not .*ComplexSubclass"): + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not .*WithComplex"): + complex(0, WithComplex(4.25+0j)) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0.0, 4.25j), -4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25j), -4.25, 0.0) + + check(complex(real=4.25), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+1.5j), 4.25, 1.5) + check(complex(imag=1.5), 0.0, 1.5) + check(complex(real=4.25, imag=1.5), 4.25, 1.5) + check(complex(4.25, imag=1.5), 4.25, 1.5) # check that the sign of a zero in the real or imaginary part - # is preserved when constructing from two floats. (These checks - # are harmless on systems without support for signed zeros.) - def split_zeros(x): - """Function that produces different results for 0. and -0.""" - return atan2(x, -1.) - - self.assertEqual(split_zeros(complex(1., 0.).imag), split_zeros(0.)) - self.assertEqual(split_zeros(complex(1., -0.).imag), split_zeros(-0.)) - self.assertEqual(split_zeros(complex(0., 1.).real), split_zeros(0.)) - self.assertEqual(split_zeros(complex(-0., 1.).real), split_zeros(-0.)) - - c = 3.14 + 1j - self.assertTrue(complex(c) is c) - del c - - self.assertRaises(TypeError, complex, "1", "1") - self.assertRaises(TypeError, complex, 1, "1") - - # SF bug 543840: complex(string) accepts strings with \0 - # Fixed in 2.3. - self.assertRaises(ValueError, complex, '1+1j\0j') - - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, float, 5+3j) - self.assertRaises(ValueError, complex, "") - self.assertRaises(TypeError, complex, None) - self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None) - self.assertRaises(ValueError, complex, "\0") - self.assertRaises(ValueError, complex, "3\09") - self.assertRaises(TypeError, complex, "1", "2") - self.assertRaises(TypeError, complex, "1", 42) - self.assertRaises(TypeError, complex, 1, "2") - self.assertRaises(ValueError, complex, "1+") - self.assertRaises(ValueError, complex, "1+1j+1j") - self.assertRaises(ValueError, complex, "--") - self.assertRaises(ValueError, complex, "(1+2j") - self.assertRaises(ValueError, complex, "1+2j)") - self.assertRaises(ValueError, complex, "1+(2j)") - self.assertRaises(ValueError, complex, "(1+2j)123") - self.assertRaises(ValueError, complex, "x") - self.assertRaises(ValueError, complex, "1j+2") - self.assertRaises(ValueError, complex, "1e1ej") - self.assertRaises(ValueError, complex, "1e++1ej") - self.assertRaises(ValueError, complex, ")1+2j(") - self.assertRaisesRegex( - TypeError, - "first argument must be a string or a number, not 'dict'", - complex, {1:2}, 1) - self.assertRaisesRegex( - TypeError, - "second argument must be a number, not 'dict'", - complex, 1, {1:2}) - # the following three are accepted by Python 2.6 - self.assertRaises(ValueError, complex, "1..1j") - self.assertRaises(ValueError, complex, "1.11.1j") - self.assertRaises(ValueError, complex, "1e1.1j") - - # check that complex accepts long unicode strings - self.assertEqual(type(complex("1"*500)), complex) - # check whitespace processing - self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) - # Invalid unicode string - # See bpo-34087 - self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + # is preserved when constructing from two floats. + for x in 1.0, -1.0: + for y in 0.0, -0.0: + check(complex(x, y), x, y) + check(complex(y, x), y, x) + + c = complex(4.25, 1.5) + self.assertIs(complex(c), c) + c2 = ComplexSubclass(c) + self.assertEqual(c2, c) + self.assertIs(type(c2), ComplexSubclass) + del c, c2 + + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not dict", + complex, {}) + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not NoneType", + complex, None) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not dict", + complex, {1:2}, 0) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not str", + complex, '1', 0) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not dict", + complex, 0, {1:2}) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not str", + complex, 0, '1') + + self.assertRaises(TypeError, complex, WithComplex(1.5)) + self.assertRaises(TypeError, complex, WithComplex(1)) + self.assertRaises(TypeError, complex, WithComplex(None)) + self.assertRaises(TypeError, complex, WithComplex(4.25+0j), object()) + self.assertRaises(TypeError, complex, WithComplex(1.5), object()) + self.assertRaises(TypeError, complex, WithComplex(1), object()) + self.assertRaises(TypeError, complex, WithComplex(None), object()) class EvilExc(Exception): pass @@ -475,17 +524,33 @@ def __complex__(self): self.assertRaises(EvilExc, complex, evilcomplex()) - self.assertAlmostEqual(complex(FloatLike(42.)), 42) - self.assertAlmostEqual(complex(real=FloatLike(17.), imag=FloatLike(23.)), 17+23j) - self.assertRaises(TypeError, complex, FloatLike(None)) - - self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) - self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) - self.assertRaises(OverflowError, complex, MyIndex(2**2000)) - self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) - - self.assertRaises(TypeError, complex, MyInt(42)) - self.assertRaises(TypeError, complex, 123, MyInt(42)) + check(complex(WithFloat(4.25)), 4.25, 0.0) + check(complex(WithFloat(4.25), 1.5), 4.25, 1.5) + check(complex(1.5, WithFloat(4.25)), 1.5, 4.25) + self.assertRaises(TypeError, complex, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(42), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(None)) + self.assertRaises(TypeError, complex, WithFloat(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(None)) + + check(complex(WithIndex(42)), 42.0, 0.0) + check(complex(WithIndex(42), 1.5), 42.0, 1.5) + check(complex(1.5, WithIndex(42)), 1.5, 42.0) + self.assertRaises(OverflowError, complex, WithIndex(2**2000)) + self.assertRaises(OverflowError, complex, WithIndex(2**2000), 1.5) + self.assertRaises(OverflowError, complex, 1.5, WithIndex(2**2000)) + self.assertRaises(TypeError, complex, WithIndex(None)) + self.assertRaises(TypeError, complex, WithIndex(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithIndex(None)) + + class MyInt: + def __int__(self): + return 42 + + self.assertRaises(TypeError, complex, MyInt()) + self.assertRaises(TypeError, complex, MyInt(), 1.5) + self.assertRaises(TypeError, complex, 1.5, MyInt()) class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" @@ -505,9 +570,9 @@ class complex2(complex): def __complex__(self): return None - self.assertEqual(complex(complex0(1j)), 42j) + check(complex(complex0(1j)), 0.0, 42.0) with self.assertWarns(DeprecationWarning): - self.assertEqual(complex(complex1(1j)), 2j) + check(complex(complex1(1j)), 0.0, 2.0) self.assertRaises(TypeError, complex, complex2(1j)) def test___complex__(self): @@ -515,10 +580,7 @@ def test___complex__(self): self.assertEqual(z.__complex__(), z) self.assertEqual(type(z.__complex__()), complex) - class complex_subclass(complex): - pass - - z = complex_subclass(3 + 4j) + z = ComplexSubclass(3 + 4j) self.assertEqual(z.__complex__(), 3 + 4j) self.assertEqual(type(z.__complex__()), complex) @@ -543,6 +605,68 @@ def test_constructor_special_numbers(self): self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_from_string(self): + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex("1"), 1.0, 0.0) + check(complex("1j"), 0.0, 1.0) + check(complex("-1"), -1.0, 0.0) + check(complex("+1"), 1.0, 0.0) + check(complex("1+2j"), 1.0, 2.0) + check(complex("(1+2j)"), 1.0, 2.0) + check(complex("(1.5+4.25j)"), 1.5, 4.25) + check(complex("4.25+1J"), 4.25, 1.0) + check(complex(" ( +4.25-6J )"), 4.25, -6.0) + check(complex(" ( +4.25-J )"), 4.25, -1.0) + check(complex(" ( +4.25+j )"), 4.25, 1.0) + check(complex("J"), 0.0, 1.0) + check(complex("( j )"), 0.0, 1.0) + check(complex("+J"), 0.0, 1.0) + check(complex("( -j)"), 0.0, -1.0) + check(complex('1-1j'), 1.0, -1.0) + check(complex('1J'), 0.0, 1.0) + + check(complex('1e-500'), 0.0, 0.0) + check(complex('-1e-500j'), 0.0, -0.0) + check(complex('1e-500+1e-500j'), 0.0, 0.0) + check(complex('-1e-500+1e-500j'), -0.0, 0.0) + check(complex('1e-500-1e-500j'), 0.0, -0.0) + check(complex('-1e-500-1e-500j'), -0.0, -0.0) + + # SF bug 543840: complex(string) accepts strings with \0 + # Fixed in 2.3. + self.assertRaises(ValueError, complex, '1+1j\0j') + self.assertRaises(ValueError, complex, "") + self.assertRaises(ValueError, complex, "\0") + self.assertRaises(ValueError, complex, "3\09") + self.assertRaises(ValueError, complex, "1+") + self.assertRaises(ValueError, complex, "1+1j+1j") + self.assertRaises(ValueError, complex, "--") + self.assertRaises(ValueError, complex, "(1+2j") + self.assertRaises(ValueError, complex, "1+2j)") + self.assertRaises(ValueError, complex, "1+(2j)") + self.assertRaises(ValueError, complex, "(1+2j)123") + self.assertRaises(ValueError, complex, "x") + self.assertRaises(ValueError, complex, "1j+2") + self.assertRaises(ValueError, complex, "1e1ej") + self.assertRaises(ValueError, complex, "1e++1ej") + self.assertRaises(ValueError, complex, ")1+2j(") + # the following three are accepted by Python 2.6 + self.assertRaises(ValueError, complex, "1..1j") + self.assertRaises(ValueError, complex, "1.11.1j") + self.assertRaises(ValueError, complex, "1e1.1j") + + # check that complex accepts long unicode strings + self.assertIs(type(complex("1"*500)), complex) + # check whitespace processing + self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) + # Invalid unicode string + # See bpo-34087 + self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + def test_constructor_negative_nans_from_string(self): self.assertEqual(copysign(1., complex("-nan").real), -1.) self.assertEqual(copysign(1., complex("-nanj").imag), -1.) @@ -570,9 +694,9 @@ def eq(actual, expected): eq(cls.from_number(3.14j), 3.14j) eq(cls.from_number(314), 314.0+0j) eq(cls.from_number(OtherComplexSubclass(3.14, 2.72)), 3.14+2.72j) - eq(cls.from_number(ComplexLike(3.14+2.72j)), 3.14+2.72j) - eq(cls.from_number(FloatLike(3.14)), 3.14+0j) - eq(cls.from_number(MyIndex(314)), 314.0+0j) + eq(cls.from_number(WithComplex(3.14+2.72j)), 3.14+2.72j) + eq(cls.from_number(WithFloat(3.14)), 3.14+0j) + eq(cls.from_number(WithIndex(314)), 314.0+0j) cNAN = complex(NAN, NAN) x = cls.from_number(cNAN) @@ -596,11 +720,15 @@ def test_hash(self): x /= 3.0 # now check against floating point self.assertEqual(hash(x), hash(complex(x, 0.))) + self.assertNotEqual(hash(2000005 - 1j), -1) + def test_abs(self): nums = [complex(x/3., y/7.) for x in range(-9,9) for y in range(-9,9)] for num in nums: self.assertAlmostEqual((num.real**2 + num.imag**2) ** 0.5, abs(num)) + self.assertRaises(OverflowError, abs, complex(DBL_MAX, DBL_MAX)) + def test_repr_str(self): def test(v, expected, test_fn=self.assertEqual): test_fn(repr(v), expected) @@ -645,6 +773,11 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., 0.), "(-0+0j)") test(complex(-0., -0.), "(-0-0j)") + def test_pos(self): + self.assertEqual(+(1+6j), 1+6j) + self.assertEqual(+ComplexSubclass(1, 6), 1+6j) + self.assertIs(type(+ComplexSubclass(1, 6)), complex) + def test_neg(self): self.assertEqual(-(1+6j), -1-6j) @@ -660,8 +793,8 @@ def test_getnewargs(self): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified z1, z2 = 0j, -0j - self.assertEqual(atan2(z1.imag, -1.), atan2(0., -1.)) - self.assertEqual(atan2(z2.imag, -1.), atan2(-0., -1.)) + self.assertFloatsAreIdentical(z1.imag, 0.0) + self.assertFloatsAreIdentical(z2.imag, -0.0) @support.requires_IEEE_754 def test_negated_imaginary_literal(self): diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py index 430fa93aa456a2..b38bd38d338f0e 100644 --- a/Lib/test/test_concurrent_futures/__init__.py +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -1,10 +1,12 @@ import os.path import unittest from test import support -from test.support import import_helper +from test.support import threading_helper + + +# Adjust if we ever have a platform with processes but not threads. +threading_helper.requires_working_threading(module=True) -# Skip tests if _multiprocessing wasn't built. -import_helper.import_module('_multiprocessing') if support.check_sanitizer(address=True, memory=True): # gh-90791: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 1e7d4344740943..4160656cb133ab 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -3,6 +3,7 @@ import weakref from concurrent import futures from test import support +from test.support import Py_GIL_DISABLED def mul(x, y): @@ -81,13 +82,34 @@ def test_no_stale_references(self): # references. my_object = MyObject() my_object_collected = threading.Event() - my_object_callback = weakref.ref( - my_object, lambda obj: my_object_collected.set()) + def set_event(): + if Py_GIL_DISABLED: + # gh-117688 Avoid deadlock by setting the event in a + # background thread. The current thread may be in the middle + # of the my_object_collected.wait() call, which holds locks + # needed by my_object_collected.set(). + threading.Thread(target=my_object_collected.set).start() + else: + my_object_collected.set() + my_object_callback = weakref.ref(my_object, lambda obj: set_event()) # Deliberately discarding the future. self.executor.submit(my_object.my_method) del my_object - collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) + if Py_GIL_DISABLED: + # Due to biased reference counting, my_object might only be + # deallocated while the thread that created it runs -- if the + # thread is paused waiting on an event, it may not merge the + # refcount of the queued object. For that reason, we alternate + # between running the GC and waiting for the event. + wait_time = 0 + collected = False + while not collected and wait_time <= support.SHORT_TIMEOUT: + support.gc_collect() + collected = my_object_collected.wait(timeout=1.0) + wait_time += 1.0 + else: + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index ce01e0ff0f287a..df640929309318 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -3,7 +3,11 @@ import queue import time import unittest +import sys +import io from concurrent.futures._base import BrokenExecutor +from concurrent.futures.process import _check_system_limits + from logging.handlers import QueueHandler from test import support @@ -109,6 +113,37 @@ def _assert_logged(self, msg): create_executor_tests(globals(), FailingInitializerMixin) +@unittest.skipIf(sys.platform == "win32", "Resource Tracker doesn't run on Windows") +class FailingInitializerResourcesTest(unittest.TestCase): + """ + Source: https://github.com/python/cpython/issues/104090 + """ + + def _test(self, test_class): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + + runner = unittest.TextTestRunner(stream=io.StringIO()) + runner.run(test_class('test_initializer')) + + # GH-104090: + # Stop resource tracker manually now, so we can verify there are not leaked resources by checking + # the process exit code + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker._stop() + + self.assertEqual(_resource_tracker._exitcode, 0) + + def test_spawn(self): + self._test(ProcessPoolSpawnFailingInitializerTest) + + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_forkserver(self): + self._test(ProcessPoolForkserverFailingInitializerTest) + + def setUpModule(): setup_module() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index c73c2da1a01088..8b1bdaa33d8f5c 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -98,6 +98,7 @@ def test_ressources_gced_in_workers(self): # explicitly destroy the object to ensure that EventfulGCObj.__del__() # is called while manager is still running. + support.gc_collect() obj = None support.gc_collect() @@ -115,6 +116,7 @@ def test_saturation(self): for _ in range(job_count): sem.release() + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 @@ -194,21 +196,21 @@ def test_python_finalization_error(self): context = self.get_context() - # gh-109047: Mock the threading.start_new_thread() function to inject + # gh-109047: Mock the threading.start_joinable_thread() function to inject # RuntimeError: simulate the error raised during Python finalization. # Block the second creation: create _ExecutorManagerThread, but block # QueueFeederThread. - orig_start_new_thread = threading._start_new_thread + orig_start_new_thread = threading._start_joinable_thread nthread = 0 - def mock_start_new_thread(func, *args): + def mock_start_new_thread(func, *args, **kwargs): nonlocal nthread if nthread >= 1: raise RuntimeError("can't create new thread at " "interpreter shutdown") nthread += 1 - return orig_start_new_thread(func, *args) + return orig_start_new_thread(func, *args, **kwargs) - with support.swap_attr(threading, '_start_new_thread', + with support.swap_attr(threading, '_start_joinable_thread', mock_start_new_thread): executor = self.executor_type(max_workers=2, mp_context=context) with executor: diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 45dab7a75fdd50..7a4065afd46fc8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -247,7 +247,9 @@ def test_cancel_futures_wait_false(self): # Errors in atexit hooks don't change the process exit code, check # stderr manually. self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) class ProcessPoolShutdownTest(ExecutorShutdownTest): diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 5926a632aa4bec..2b5bea9f4055a2 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -41,6 +41,7 @@ def acquire_lock(lock): sem.release() executor.shutdown(wait=True) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() executor.submit(mul, 21, 2).result() @@ -49,6 +50,7 @@ def test_idle_thread_reuse(self): self.assertEqual(len(executor._threads), 1) executor.shutdown(wait=True) + @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') @support.requires_resource('cpu') def test_hang_global_shutdown_lock(self): diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index ff486202092c81..108cf54bf79e6f 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -142,7 +142,7 @@ def test_pending_calls_race(self): def future_func(): event.wait() oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) try: fs = {self.executor.submit(future_func) for i in range(100)} event.set() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index dc48bec796b87f..3b8ec3e205d5aa 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -85,6 +85,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,6 +113,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -136,6 +140,12 @@ def strip_mixin(name): def setup_module(): - unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + try: + _check_system_limits() + except NotImplementedError: + pass + else: + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 53163d7528b64a..a934e493a76391 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,6 @@ import configparser import io import os -import pathlib import textwrap import unittest @@ -543,7 +542,7 @@ def test_parse_errors(self): "[Foo]\n wrong-indent\n") self.assertEqual(e.args, ('',)) # read_file on a real file - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") if self.delimiters[0] == '=': error = configparser.ParsingError expected = (tricky,) @@ -646,6 +645,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" @@ -717,7 +731,7 @@ class mystr(str): def test_read_returns_file_list(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1 = support.findfile("cfgparser.1") + file1 = support.findfile("cfgparser.1", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8") @@ -730,12 +744,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: @@ -750,7 +764,7 @@ def test_read_returns_file_list(self): def test_read_returns_file_list_with_bytestring_path(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1_bytestring = support.findfile("cfgparser.1").encode() + file1_bytestring = support.findfile("cfgparser.1", subdir="configdata").encode() # check when passing an existing bytestring path cf = self.newconfig() parsed_files = cf.read(file1_bytestring, encoding="utf-8") @@ -1126,7 +1140,7 @@ class RawConfigParserTestSambaConf(CfgParserTestCaseClass, unittest.TestCase): empty_lines_in_values = False def test_reading(self): - smbconf = support.findfile("cfgparser.2") + smbconf = support.findfile("cfgparser.2", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([smbconf, "nonexistent-file"], encoding='utf-8') @@ -1321,7 +1335,7 @@ class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase): allow_no_value = True def test_cfgparser_dot_3(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() self.assertEqual(len(cf.read(tricky, encoding='utf-8')), 1) self.assertEqual(cf.sections(), ['strange', @@ -1353,7 +1367,7 @@ def test_cfgparser_dot_3(self): self.assertEqual(cf.get('more interpolation', 'lets'), 'go shopping') def test_unicode_failure(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() with self.assertRaises(UnicodeDecodeError): cf.read(tricky, encoding='ascii') @@ -1454,7 +1468,7 @@ def fromstring(self, string, defaults=None): class FakeFile: def __init__(self): - file_path = support.findfile("cfgparser.1") + file_path = support.findfile("cfgparser.1", subdir="configdata") with open(file_path, encoding="utf-8") as f: self.lines = f.readlines() self.lines.reverse() @@ -1475,7 +1489,7 @@ def readline_generator(f): class ReadFileTestCase(unittest.TestCase): def test_file(self): - file_paths = [support.findfile("cfgparser.1")] + file_paths = [support.findfile("cfgparser.1", subdir="configdata")] try: file_paths.append(file_paths[0].encode('utf8')) except UnicodeEncodeError: @@ -1555,6 +1569,30 @@ def test_source_as_bytes(self): "'[badbad'" ) + def test_keys_without_value_with_extra_whitespace(self): + lines = [ + '[SECT]\n', + 'KEY1\n', + ' KEY2 = VAL2\n', # note the Space before the key! + ] + parser = configparser.ConfigParser( + comment_prefixes="", + allow_no_value=True, + strict=False, + delimiters=('=',), + interpolation=None, + ) + with self.assertRaises(configparser.MultilineContinuationError) as dse: + parser.read_file(lines) + self.assertEqual( + str(dse.exception), + "Key without value continued with an indented line.\n" + "file: '', line: 3\n" + "' KEY2 = VAL2\\n'" + ) + + + class CoverageOneHundredTestCase(unittest.TestCase): """Covers edge cases in the codebase.""" @@ -2076,6 +2114,54 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 471d04a76ca4e4..259c954aa1d616 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -24,8 +24,11 @@ def test_common_tests(self): self.assertNotIn(0, b) self.assertIn(1, c) self.assertNotIn(0, c) - self.assertRaises(TypeError, lambda: 1 in a) - self.assertRaises(TypeError, lambda: 1 not in a) + msg = "argument of type 'base_set' is not a container or iterable" + with self.assertRaisesRegex(TypeError, msg): + 1 in a + with self.assertRaisesRegex(TypeError, msg): + 1 not in a # test char in string self.assertIn('c', 'abc') diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 5d94ec7cae4706..36c3abca80f894 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -173,6 +173,15 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_trap_no_yield(self): + @contextmanager + def whoo(): + if False: + yield + ctx = whoo() + with self.assertRaises(RuntimeError): + ctx.__enter__() + def test_contextmanager_trap_second_yield(self): @contextmanager def whoo(): @@ -186,6 +195,19 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_non_normalised(self): + @contextmanager + def whoo(): + try: + yield + except RuntimeError: + raise SyntaxError + + ctx = whoo() + ctx.__enter__() + with self.assertRaises(SyntaxError): + ctx.__exit__(RuntimeError, None, None) + def test_contextmanager_except(self): state = [] @contextmanager @@ -265,6 +287,25 @@ def test_issue29692(): self.assertEqual(ex.args[0], 'issue29692:Unchained') self.assertIsNone(ex.__cause__) + def test_contextmanager_wrap_runtimeerror(self): + @contextmanager + def woohoo(): + try: + yield + except Exception as exc: + raise RuntimeError(f'caught {exc}') from exc + + with self.assertRaises(RuntimeError): + with woohoo(): + 1 / 0 + + # If the context manager wrapped StopIteration in a RuntimeError, + # we also unwrap it, because we can't tell whether the wrapping was + # done by the generator machinery or by the generator itself. + with self.assertRaises(StopIteration): + with woohoo(): + raise StopIteration + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): @@ -276,6 +317,7 @@ def decorate(func): @attribs(foo='bar') def baz(spam): """Whee!""" + yield return baz def test_contextmanager_attribs(self): @@ -332,8 +374,11 @@ def woohoo(a, *, b): def test_recursive(self): depth = 0 + ncols = 0 @contextmanager def woohoo(): + nonlocal ncols + ncols += 1 nonlocal depth before = depth depth += 1 @@ -347,6 +392,7 @@ def recursive(): recursive() recursive() + self.assertEqual(ncols, 10) self.assertEqual(depth, 0) @@ -1251,6 +1297,24 @@ def test_exception_groups(self): [KeyError("ke1"), KeyError("ke2")], ), ) + # Check handling of BaseExceptionGroup, using GeneratorExit so that + # we don't accidentally discard a ctrl-c with KeyboardInterrupt. + with suppress(GeneratorExit): + raise BaseExceptionGroup("message", [GeneratorExit()]) + # If we raise a BaseException group, we can still suppress parts + with self.assertRaises(BaseExceptionGroup) as eg1: + with suppress(KeyError): + raise BaseExceptionGroup("message", [GeneratorExit("g"), KeyError("k")]) + self.assertExceptionIsLike( + eg1.exception, BaseExceptionGroup("message", [GeneratorExit("g")]), + ) + # If we suppress all the leaf BaseExceptions, we get a non-base ExceptionGroup + with self.assertRaises(ExceptionGroup) as eg1: + with suppress(GeneratorExit): + raise BaseExceptionGroup("message", [GeneratorExit("g"), KeyError("k")]) + self.assertExceptionIsLike( + eg1.exception, ExceptionGroup("message", [KeyError("k")]), + ) class TestChdir(unittest.TestCase): @@ -1272,7 +1336,7 @@ def test_simple(self): def test_reentrant(self): old_cwd = os.getcwd() target1 = self.make_relative_path('data') - target2 = self.make_relative_path('ziptestdata') + target2 = self.make_relative_path('archivetestdata') self.assertNotIn(old_cwd, (target1, target2)) chdir1, chdir2 = chdir(target1), chdir(target2) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 60735ba89a80ee..3dec64cc9a2414 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -952,7 +952,7 @@ class PointFromClass(NamedTuple): self.assertEqual(copy.replace(p, x=1), (1, 22)) self.assertEqual(copy.replace(p, y=2), (11, 2)) self.assertEqual(copy.replace(p, x=1, y=2), (1, 2)) - with self.assertRaisesRegex(ValueError, 'unexpected field name'): + with self.assertRaisesRegex(TypeError, 'unexpected field name'): copy.replace(p, x=1, error=2) def test_dataclass(self): @@ -972,6 +972,10 @@ class C: copy.replace(c, x=1, error=2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, copy, not_exported={"dispatch_table", "error"}) + def global_foo(x, y): return x+y diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 47145782c0f04f..a677301c62becc 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -11,6 +11,10 @@ from test.support import import_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok +try: + import _testcapi +except ImportError: + _testcapi = None class AsyncYieldFrom: @@ -953,11 +957,12 @@ async def b(): def test_corotype_1(self): ct = types.CoroutineType - self.assertIn('into coroutine', ct.send.__doc__) - self.assertIn('inside coroutine', ct.close.__doc__) - self.assertIn('in coroutine', ct.throw.__doc__) - self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) - self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('into coroutine', ct.send.__doc__) + self.assertIn('inside coroutine', ct.close.__doc__) + self.assertIn('in coroutine', ct.throw.__doc__) + self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) + self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) self.assertEqual(ct.__name__, 'coroutine') async def f(): pass @@ -969,13 +974,13 @@ def test_await_1(self): async def foo(): await 1 - with self.assertRaisesRegex(TypeError, "object int can.t.*await"): + with self.assertRaisesRegex(TypeError, "'int' object can.t be awaited"): run_async(foo()) def test_await_2(self): async def foo(): await [] - with self.assertRaisesRegex(TypeError, "object list can.t.*await"): + with self.assertRaisesRegex(TypeError, "'list' object can.t be awaited"): run_async(foo()) def test_await_3(self): @@ -1035,7 +1040,7 @@ class Awaitable: async def foo(): return await Awaitable() with self.assertRaisesRegex( - TypeError, "object Awaitable can't be used in 'await' expression"): + TypeError, "'Awaitable' object can't be awaited"): run_async(foo()) @@ -2216,6 +2221,14 @@ async def f(): gen.cr_frame.clear() gen.close() + def test_cr_frame_after_close(self): + async def f(): + pass + gen = f() + self.assertIsNotNone(gen.cr_frame) + gen.close() + self.assertIsNone(gen.cr_frame) + def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 async def a(): @@ -2436,6 +2449,7 @@ def test_unawaited_warning_during_shutdown(self): @support.cpython_only +@unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 4d9ee3cb2228ae..efd79448c66104 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,11 +1,10 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import shlex import shutil -import sys -import unittest import subprocess -import sysconfig +import unittest from test import support @@ -13,36 +12,41 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# gh-110119: pip does not currently support 't' in the ABI flag use by -# --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(sysconfig.get_config_var('Py_NOGIL') == 1, - 'test does not work with --disable-gil') +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() @support.requires_subprocess() +@support.requires_resource('cpu') class TestCPPExt(unittest.TestCase): - @support.requires_resource('cpu') - def test_build_cpp11(self): - self.check_build(False, '_testcpp11ext') + def test_build(self): + self.check_build('_testcppext') - @support.requires_resource('cpu') def test_build_cpp03(self): - self.check_build(True, '_testcpp03ext') + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') - # With MSVC, the linker fails with: cannot open file 'python311.lib' - # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') - # Building and running an extension in clang sanitizing mode is not - # straightforward - @unittest.skipIf( - '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), - 'test does not work with analyzing builds') - # the test uses venv+pip: skip if it's not available - @support.requires_venv_with_pip() - def check_build(self, std_cpp03, extension_name): + def check_build(self, extension_name, std=None): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(std_cpp03, extension_name, python_exe) + self._check_build(extension_name, python_exe, std=std) - def _check_build(self, std_cpp03, extension_name, python_exe): + def _check_build(self, extension_name, python_exe, std): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -50,10 +54,11 @@ def _check_build(self, std_cpp03, extension_name, python_exe): def run_cmd(operation, cmd): env = os.environ.copy() - env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' + if std: + env['CPYTHON_TEST_CPP_STD'] = std env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: - print('Run:', ' '.join(cmd)) + print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) else: proc = subprocess.run(cmd, @@ -62,6 +67,7 @@ def run_cmd(operation, cmd): stderr=subprocess.STDOUT, text=True) if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) print(proc.stdout, end='') self.fail( f"{operation} failed with exit code {proc.returncode}") @@ -70,6 +76,8 @@ def run_cmd(operation, cmd): cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] + if support.verbose: + cmd.append('-v') run_cmd('Install', cmd) # Do a reference run. Until we test that running python diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 90669b10cb2c6d..ab485b629b7788 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -8,10 +8,8 @@ #include "Python.h" -#if __cplusplus >= 201103 -# define NAME _testcpp11ext -#else -# define NAME _testcpp03ext +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" #endif #define _STR(NAME) #NAME @@ -160,7 +158,7 @@ PyType_Slot VirtualPyObject_Slots[] = { }; PyType_Spec VirtualPyObject_Spec = { - /* .name */ STR(NAME) ".VirtualPyObject", + /* .name */ STR(MODULE_NAME) ".VirtualPyObject", /* .basicsize */ sizeof(VirtualPyObject), /* .itemsize */ 0, /* .flags */ Py_TPFLAGS_DEFAULT, @@ -227,6 +225,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } @@ -240,7 +242,7 @@ PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); static struct PyModuleDef _testcppext_module = { PyModuleDef_HEAD_INIT, // m_base - STR(NAME), // m_name + STR(MODULE_NAME), // m_name _testcppext_doc, // m_doc 0, // m_size _testcppext_methods, // m_methods @@ -254,7 +256,7 @@ static struct PyModuleDef _testcppext_module = { #define FUNC_NAME(NAME) _FUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(NAME)(void) +FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcppext_module); } diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index c7ba1efb4dd05a..f1848f2fd42a46 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,8 +1,8 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import platform import shlex -import sys import sysconfig from test import support @@ -10,6 +10,7 @@ SOURCE = 'extension.cpp' + if not support.MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ @@ -19,34 +20,75 @@ '-Werror', ] else: - # Don't pass any compiler flag to MSVC - CPPFLAGS = [] + # MSVC compiler flags + CPPFLAGS = [ + # Treat all compiler warnings as compiler errors + '/WX', + ] def main(): cppflags = list(CPPFLAGS) - std = os.environ["CPYTHON_TEST_CPP_STD"] - name = os.environ["CPYTHON_TEST_EXT_NAME"] + std = os.environ.get("CPYTHON_TEST_CPP_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] - cppflags = [*CPPFLAGS, f'-std={std}'] + cppflags = list(CPPFLAGS) + cppflags.append(f'-DMODULE_NAME={module_name}') + + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cppflags.append(f'/std:{std}') + else: + cppflags.append(f'-std={std}') # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 # option emits a C++ compiler warning. Remove "-std11" option from the # CC command. cmd = (sysconfig.get_config_var('CC') or '') if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] cmd = shlex.join(cmd) # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd - cpp_ext = Extension( - name, + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cppflags!r}") + + ext = Extension( + module_name, sources=[SOURCE], language='c++', - extra_compile_args=cppflags) - setup(name='internal' + name, version='0.0', ext_modules=[cpp_ext]) + extra_compile_args=cppflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) if __name__ == "__main__": diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 3056fe84dac5dd..27e8a767903777 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -83,8 +83,8 @@ def test_throw(self): for func, (cc, nc, _, _, _) in pr.stats.items(): if func[2] == "": - self.assertEqual(cc, 2) - self.assertEqual(nc, 2) + self.assertEqual(cc, 1) + self.assertEqual(nc, 1) class TestCommandLine(unittest.TestCase): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 97b9bba24bcbca..d74ab7e016f78c 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -28,14 +28,20 @@ class Test_Csv(unittest.TestCase): in TestDialectRegistry. """ def _test_arg_valid(self, ctor, arg): + ctor(arg) self.assertRaises(TypeError, ctor) self.assertRaises(TypeError, ctor, None) - self.assertRaises(TypeError, ctor, arg, bad_attr = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 'XX') + self.assertRaises(TypeError, ctor, arg, bad_attr=0) + self.assertRaises(TypeError, ctor, arg, delimiter='') + self.assertRaises(TypeError, ctor, arg, escapechar='') + self.assertRaises(TypeError, ctor, arg, quotechar='') + self.assertRaises(TypeError, ctor, arg, delimiter='^^') + self.assertRaises(TypeError, ctor, arg, escapechar='^^') + self.assertRaises(TypeError, ctor, arg, quotechar='^^') self.assertRaises(csv.Error, ctor, arg, 'foo') self.assertRaises(TypeError, ctor, arg, delimiter=None) self.assertRaises(TypeError, ctor, arg, delimiter=1) + self.assertRaises(TypeError, ctor, arg, escapechar=1) self.assertRaises(TypeError, ctor, arg, quotechar=1) self.assertRaises(TypeError, ctor, arg, lineterminator=None) self.assertRaises(TypeError, ctor, arg, lineterminator=1) @@ -46,6 +52,39 @@ def _test_arg_valid(self, ctor, arg): quoting=csv.QUOTE_ALL, quotechar=None) self.assertRaises(TypeError, ctor, arg, quoting=csv.QUOTE_NONE, quotechar='') + self.assertRaises(ValueError, ctor, arg, delimiter='\n') + self.assertRaises(ValueError, ctor, arg, escapechar='\n') + self.assertRaises(ValueError, ctor, arg, quotechar='\n') + self.assertRaises(ValueError, ctor, arg, delimiter='\r') + self.assertRaises(ValueError, ctor, arg, escapechar='\r') + self.assertRaises(ValueError, ctor, arg, quotechar='\r') + ctor(arg, delimiter=' ') + ctor(arg, escapechar=' ') + ctor(arg, quotechar=' ') + ctor(arg, delimiter='\t', skipinitialspace=True) + ctor(arg, escapechar='\t', skipinitialspace=True) + ctor(arg, quotechar='\t', skipinitialspace=True) + ctor(arg, delimiter=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + escapechar=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + quotechar=' ', skipinitialspace=True) + ctor(arg, delimiter='^') + ctor(arg, escapechar='^') + ctor(arg, quotechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', escapechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', quotechar='^') + self.assertRaises(ValueError, ctor, arg, escapechar='^', quotechar='^') + ctor(arg, delimiter='\x85') + ctor(arg, escapechar='\x85') + ctor(arg, quotechar='\x85') + ctor(arg, lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + delimiter='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + escapechar='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + quotechar='\x85', lineterminator='\x85') def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) @@ -152,9 +191,6 @@ def _write_error_test(self, exc, fields, **kwargs): def test_write_arg_valid(self): self._write_error_test(csv.Error, None) - self._write_test((), '') - self._write_test([None], '""') - self._write_error_test(csv.Error, [None], quoting = csv.QUOTE_NONE) # Check that exceptions are passed up the chain self._write_error_test(OSError, BadIterable()) class BadList: @@ -168,7 +204,6 @@ class BadItem: def __str__(self): raise OSError self._write_error_test(OSError, [BadItem()]) - def test_write_bigfield(self): # This exercises the buffer realloc functionality bigstring = 'X' * 50000 @@ -230,9 +265,11 @@ def test_write_lineterminator(self): writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' - f'1,2{lineterminator}') + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') @@ -275,6 +312,49 @@ def test_writerows_with_none(self): fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') + + def test_write_empty_fields(self): + self._write_test((), '') + self._write_test([''], '""') + self._write_error_test(csv.Error, [''], quoting=csv.QUOTE_NONE) + self._write_test([''], '""', quoting=csv.QUOTE_STRINGS) + self._write_test([''], '""', quoting=csv.QUOTE_NOTNULL) + self._write_test([None], '""') + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_STRINGS) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NOTNULL) + self._write_test(['', ''], ',') + self._write_test([None, None], ',') + + def test_write_empty_fields_space_delimiter(self): + self._write_test([''], '""', delimiter=' ', skipinitialspace=False) + self._write_test([''], '""', delimiter=' ', skipinitialspace=True) + self._write_test([None], '""', delimiter=' ', skipinitialspace=False) + self._write_test([None], '""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True) + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False) + self._write_test([None, None], '"" ""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False, + quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, ['', ''], + delimiter=' ', skipinitialspace=True, + quoting=csv.QUOTE_NONE) + for quoting in csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True, + quoting=quoting) + + for quoting in csv.QUOTE_NONE, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_error_test(csv.Error, [None, None], + delimiter=' ', skipinitialspace=True, + quoting=quoting) + def test_writerows_errors(self): with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) @@ -297,13 +377,18 @@ def test_read_oddinputs(self): [b'abc'], None) def test_read_eol(self): - self._read_test(['a,b'], [['a','b']]) - self._read_test(['a,b\n'], [['a','b']]) - self._read_test(['a,b\r\n'], [['a','b']]) - self._read_test(['a,b\r'], [['a','b']]) - self.assertRaises(csv.Error, self._read_test, ['a,b\rc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\nc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\r\nc,d'], []) + self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']]) + self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r\n', 'c,d\r\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r', 'c,d\r'], [['a','b'], ['c','d']]) + + errmsg = "with newline=''" + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\rc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\nc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\r\nc,d'])) def test_read_eof(self): self._read_test(['a,"'], [['a', '']]) @@ -347,10 +432,26 @@ def test_read_quoting(self): # will this fail where locale uses comma for decimals? self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]], + quoting=csv.QUOTE_STRINGS) + + self._read_test([',,"",'], [['', '', '', '']]) + self._read_test([',,"",'], [['', '', '', '']], + quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_STRINGS) + self._read_test(['"a\nb", 7'], [['a\nb', ' 7']]) self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self.assertRaises(ValueError, self._read_test, + ['abc,3'], [[]], + quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') @@ -358,6 +459,23 @@ def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], skipinitialspace=True) + self._read_test([' , , '], + [['', '', '']], + skipinitialspace=True) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_NOTNULL) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_STRINGS) + + def test_read_space_delimiter(self): + self._read_test(['a b', ' a ', ' ', ''], + [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], + delimiter=' ', skipinitialspace=False) + self._read_test(['a b', ' a ', ' ', ''], + [['a', 'b'], ['a', ''], [''], []], + delimiter=' ', skipinitialspace=True) def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size @@ -391,22 +509,44 @@ def test_read_linenum(self): self.assertEqual(r.line_num, 3) def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj)): - self.assertEqual(row, rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator) + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj)): + self.assertEqual(row, rows[i]) def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): - self.assertEqual(row,rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator, + quoting=csv.QUOTE_NONE, escapechar="\\") + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj, + quoting=csv.QUOTE_NONE, + escapechar="\\")): + self.assertEqual(row, rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): @@ -485,10 +625,10 @@ class space(csv.excel): escapechar = "\\" with TemporaryFile("w+", encoding="utf-8") as fileobj: - fileobj.write("abc def\nc1ccccc1 benzene\n") + fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) - self.assertEqual(next(reader), ["abc", "def"]) + self.assertEqual(next(reader), ["abc", "", "", "def"]) self.assertEqual(next(reader), ["c1ccccc1", "benzene"]) def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): @@ -530,14 +670,6 @@ class unspecified(): finally: csv.unregister_dialect('testC') - def test_bad_dialect(self): - # Unknown parameter - self.assertRaises(TypeError, csv.reader, [], bad_attr = 0) - # Bad values - self.assertRaises(TypeError, csv.reader, [], delimiter = None) - self.assertRaises(TypeError, csv.reader, [], quoting = -1) - self.assertRaises(TypeError, csv.reader, [], quoting = 100) - def test_copy(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) @@ -1083,10 +1215,15 @@ class mydialect(csv.Dialect): '"lineterminator" must be a string') def test_invalid_chars(self): - def create_invalid(field_name, value): + def create_invalid(field_name, value, **kwargs): class mydialect(csv.Dialect): - pass + delimiter = ',' + quoting = csv.QUOTE_ALL + quotechar = '"' + lineterminator = '\r\n' setattr(mydialect, field_name, value) + for field_name, value in kwargs.items(): + setattr(mydialect, field_name, value) d = mydialect() for field_name in ("delimiter", "escapechar", "quotechar"): @@ -1095,6 +1232,11 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, "abc") self.assertRaises(csv.Error, create_invalid, field_name, b'x') self.assertRaises(csv.Error, create_invalid, field_name, 5) + self.assertRaises(ValueError, create_invalid, field_name, "\n") + self.assertRaises(ValueError, create_invalid, field_name, "\r") + if field_name != "delimiter": + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): @@ -1411,8 +1553,7 @@ def test_ordered_dict_reader(self): class MiscTestCase(unittest.TestCase): def test__all__(self): - extra = {'__doc__', '__version__'} - support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + support.check__all__(self, csv, ('csv', '_csv')) def test_subclassable(self): # issue 44089 diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py new file mode 100644 index 00000000000000..e4c2b33825ae8f --- /dev/null +++ b/Lib/test/test_ctypes/_support.py @@ -0,0 +1,24 @@ +# Some classes and types are not export to _ctypes module directly. + +import ctypes +from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr + + +_CData = Structure.__base__ +assert _CData.__name__ == "_CData" + +class _X(Structure): + _fields_ = [("x", ctypes.c_int)] +CField = type(_X.x) + +# metaclasses +PyCStructType = type(Structure) +UnionType = type(Union) +PyCPointerType = type(_Pointer) +PyCArrayType = type(Array) +PyCSimpleType = type(_SimpleCData) +PyCFuncPtrType = type(CFuncPtr) + +# type flags +Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 +Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py new file mode 100644 index 00000000000000..a208fb9a00966a --- /dev/null +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -0,0 +1,286 @@ +from ctypes import ( + c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, + BigEndianStructure, LittleEndianStructure, + BigEndianUnion, LittleEndianUnion, +) +import struct +import unittest + + +class TestAlignedStructures(unittest.TestCase): + def test_aligned_string(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] + + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 7) + self.assertEqual(main.string.value, b'hello world!') + self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0') + self.assertEqual(Main.string.offset, 16) + self.assertEqual(Main.string.size, 16) + self.assertEqual(alignment(main.string), 16) + self.assertEqual(alignment(main), 16) + + def test_aligned_structures(self): + for base, data in ( + (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ): + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(SomeBools), 4) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.y), 4) + self.assertEqual(Main.x.size, 1) + self.assertEqual(Main.y.offset, 4) + self.assertEqual(Main.y.size, 4) + self.assertEqual(main.y.bool1, True) + self.assertEqual(main.y.bool2, False) + self.assertEqual(Main.z.offset, 8) + self.assertEqual(main.z, 7) + + def test_oversized_structure(self): + data = bytearray(b"\0" * 8) + for base in (LittleEndianStructure, BigEndianStructure): + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] + with self.assertRaises(ValueError) as ctx: + Main.from_buffer(data) + self.assertEqual( + ctx.exception.args[0], + 'Buffer size too small (4 instead of at least 8 bytes)' + ) + + def test_aligned_subclasses(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] + + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] + + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main.b), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(sizeof(main), 16) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 3) + self.assertEqual(main.b.y, 4) + self.assertEqual(Main.b.offset, 8) + self.assertEqual(Main.b.size, 8) + + def test_aligned_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class AlignedUnion(ubase): + _align_ = 8 + _fields_ = [ + ("a", c_uint32), + ("b", c_ubyte * 7), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(bytes(main.union.b), data[8:-1]) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + + def test_aligned_struct_in_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] + + class MainUnion(ubase): + _fields_ = [ + ("a", c_uint32), + ("b", Sub), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(Main.first.size, 4) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(main.union.b.x, 3) + self.assertEqual(main.union.b.y, 4) + + def test_smaller_aligned_subclassed_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + class SubUnion(ubase): + _align_ = 2 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class MainUnion(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.union.num, 0xD60102D7) + self.assertEqual(main.union.unsigned, data[4]) + self.assertEqual(main.union.signed, data[4] - 256) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.union), 4) + self.assertEqual(Main.union.offset, 4) + self.assertEqual(Main.union.size, 4) + self.assertEqual(Main.first.size, 2) + + def test_larger_aligned_subclassed_union(self): + for ubase, e in ( + (LittleEndianUnion, "<"), + (BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + class SubUnion(ubase): + _align_ = 8 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class Main(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main), 8) + self.assertEqual(main.num, 0xD60102D6) + self.assertEqual(main.unsigned, 0xD6) + self.assertEqual(main.signed, -42) + + def test_aligned_packed_structures(self): + for sbase, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) + + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] + + class Main(sbase): + _pack_ = 1 + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(sizeof(main), 10) + self.assertEqual(Main.b.offset, 1) + # Alignment == 8 because _pack_ wins out. + self.assertEqual(alignment(main.b), 8) + # Size is still 8 though since inside this Structure, it will have + # effect. + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(Main.c.offset, 9) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 2) + self.assertEqual(main.b.y, 3) + self.assertEqual(main.c, 4) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index f7a4b5df514e5b..3568cf97f40b50 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -7,6 +7,8 @@ c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) from test.support import bigmemtest, _2G +from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) formats = "bBhHiIlLqQfd" @@ -23,6 +25,34 @@ def ARRAY(*args): class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Array.mro(), [Array, _CData, object]) + + self.assertEqual(PyCArrayType.__name__, "PyCArrayType") + self.assertEqual(type(PyCArrayType), type) + + def test_type_flags(self): + for cls in Array, PyCArrayType: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -189,10 +219,10 @@ def test_bad_subclass(self): class T(Array): pass with self.assertRaises(AttributeError): - class T(Array): + class T2(Array): _type_ = c_int with self.assertRaises(AttributeError): - class T(Array): + class T3(Array): _length_ = 13 def test_bad_length(self): @@ -201,15 +231,15 @@ class T(Array): _type_ = c_int _length_ = - sys.maxsize * 2 with self.assertRaises(ValueError): - class T(Array): + class T2(Array): _type_ = c_int _length_ = -1 with self.assertRaises(TypeError): - class T(Array): + class T3(Array): _type_ = c_int _length_ = 1.87 with self.assertRaises(OverflowError): - class T(Array): + class T4(Array): _type_ = c_int _length_ = sys.maxsize * 2 diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index a1a8745e737fa2..cc62b1a22a3b06 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (Structure, CDLL, CFUNCTYPE, @@ -6,6 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) @@ -221,5 +222,16 @@ class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index d43c56ad371fbd..e6509e6bf89e1d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,12 +1,14 @@ -import _ctypes_test import os +import sys import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, + c_uint8, c_uint16, c_uint32, c_uint64, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class BITS(Structure): @@ -32,27 +34,88 @@ class BITS(Structure): func.argtypes = POINTER(BITS), c_char +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) @@ -86,35 +149,41 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): return self.get_except(type(Structure), "X", (), @@ -148,22 +217,28 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): @@ -235,6 +310,161 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + + class GoodA(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index b97b57646ecd71..78eff0392c4548 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -363,6 +363,24 @@ class TestUnion(parent): self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) + def test_build_struct_union_opposite_system_byteorder(self): + # gh-105102 + if sys.byteorder == "little": + _Structure = BigEndianStructure + _Union = BigEndianUnion + else: + _Structure = LittleEndianStructure + _Union = LittleEndianUnion + + class S1(_Structure): + _fields_ = [("a", c_byte), ("b", c_byte)] + + class U1(_Union): + _fields_ = [("s1", S1), ("ab", c_short)] + + class S2(_Structure): + _fields_ = [("u1", U1), ("c", c_byte)] + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 6fe3e119672409..8f483dfe1db801 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import functools import gc @@ -14,6 +13,8 @@ c_float, c_double, c_longdouble, py_object) from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class Callbacks(unittest.TestCase): @@ -106,7 +107,7 @@ def test_pyobject(self): def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) @@ -148,9 +149,10 @@ def callback(a, b): print(f"a={a}, b={b}, c={c}") return c dll = cdll[_ctypes_test.__file__] - # With no fix for i38748, the next line will raise OSError and cause the test to fail. - self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) - + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") if hasattr(ctypes, 'WINFUNCTYPE'): class StdcallCallbacks(Callbacks): @@ -322,9 +324,9 @@ def func(): self.assertIsInstance(cm.unraisable.exc_value, TypeError) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on converting result " - "of ctypes callback function") - self.assertIs(cm.unraisable.object, func) + f"Exception ignored on converting result " + f"of ctypes callback function {func!r}") + self.assertIsNone(cm.unraisable.object) if __name__ == '__main__': diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 6ff0878a35da2f..48330c4b0a763b 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, @@ -6,6 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CFunctions(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py index 5dc9e25aa38226..9d6bfdb845e6c7 100644 --- a/Lib/test/test_ctypes/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,8 @@ -import _ctypes_test import ctypes import unittest from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CHECKED(c_int): diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 66ff23e72b5e10..85b28617d2d754 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -125,6 +125,32 @@ def test_find_library_with_ld(self): unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): self.assertNotEqual(find_library('c'), None) + def test_gh114257(self): + self.assertIsNone(find_library("libc")) + + +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 2ad40647e0cfbb..8362fb16d94dcd 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,8 +1,11 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) try: @@ -15,6 +18,24 @@ class CFuncPtrTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) + + self.assertEqual(PyCFuncPtrType.__name__, "PyCFuncPtrType") + self.assertEqual(type(PyCFuncPtrType), type) + + def test_type_flags(self): + for cls in _CFuncPtr, PyCFuncPtrType: + with self.subTest(cls=cls): + self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 08eecbc9ea4442..3454b83d43e1e7 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,12 +1,13 @@ -import _ctypes_test import ctypes import sys import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, - byref, POINTER, pointer, ArgumentError, + byref, POINTER, pointer, ArgumentError, sizeof, c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from _ctypes import _Pointer, _SimpleCData @@ -46,15 +47,15 @@ class X(object, Array): _type_ = "i" with self.assertRaises(TypeError): - class X(object, _Pointer): + class X2(object, _Pointer): pass with self.assertRaises(TypeError): - class X(object, _SimpleCData): + class X3(object, _SimpleCData): _type_ = "i" with self.assertRaises(TypeError): - class X(object, Structure): + class X4(object, Structure): _fields_ = [] def test_c_char_parm(self): @@ -71,7 +72,8 @@ def callback(*args): self.assertEqual(str(cm.exception), "argument 1: TypeError: one character bytes, " - "bytearray or integer expected") + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") def test_wchar_parm(self): f = dll._testfunc_i_bhilfd @@ -83,14 +85,27 @@ def test_wchar_parm(self): with self.assertRaises(ArgumentError) as cm: f(1, 2, 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: unicode string expected " - "instead of int instance") + "argument 2: TypeError: a unicode character expected, " + "not instance of int") with self.assertRaises(ArgumentError) as cm: f(1, "abc", 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: one character unicode string " - "expected") + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") def test_c_char_p_parm(self): """Test the error message when converting an incompatible type to c_char_p.""" diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000000..cbd73c4e911e4e --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,717 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper +import re +from dataclasses import dataclass +from functools import cached_property + +import ctypes +from ctypes import Structure, Union +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + + +@register() +class Packed4(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits zero. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms' or pack: + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + @cached_property + def descriptor(self): + return getattr(self.parent_type, self.name) + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output(""" + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value = {{0}}; + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 94c9a86c2d06df..778da6573da975 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -28,7 +28,7 @@ def test_ints(self): self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" + s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) cs = c_char_p(s) self.assertEqual(refcnt + 1, sys.getrefcount(s)) diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py index 09c76db0bd0b17..cab3cc9f46003a 100644 --- a/Lib/test/test_ctypes/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,8 +1,10 @@ -import _ctypes_test +import ctypes import math import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) @@ -20,6 +22,31 @@ def test_sqrt(self): self.assertEqual(lib.my_sqrt(4.0), 2.0) self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_csqrt(self): + lib.my_csqrt.argtypes = ctypes.c_double_complex, + lib.my_csqrt.restype = ctypes.c_double_complex + self.assertEqual(lib.my_csqrt(4), 2+0j) + self.assertAlmostEqual(lib.my_csqrt(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrt(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtf.argtypes = ctypes.c_float_complex, + lib.my_csqrtf.restype = ctypes.c_float_complex + self.assertAlmostEqual(lib.my_csqrtf(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtf(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtl.argtypes = ctypes.c_longdouble_complex, + lib.my_csqrtl.restype = ctypes.c_longdouble_complex + self.assertAlmostEqual(lib.my_csqrtl(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtl(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + def test_qsort(self): comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 22db97b818c17e..fc1eecb77e17e3 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,5 +1,4 @@ import _ctypes -import _ctypes_test import ctypes import os import shutil @@ -10,6 +9,7 @@ from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") libc_name = None @@ -42,10 +42,7 @@ def test_load(self): self.skipTest('could not find library to load') CDLL(test_lib) CDLL(os.path.basename(test_lib)) - class CTypesTestPathLikeCls: - def __fspath__(self): - return test_lib - CDLL(CTypesTestPathLikeCls()) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): @@ -59,11 +56,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') @@ -141,7 +142,7 @@ def test_load_hasattr(self): def test_load_dll_with_flags(self): _sqlite3 = import_helper.import_module("_sqlite3") src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): + if os.path.basename(src).partition(".")[0].lower().endswith("_d"): ext = "_d.dll" else: ext = ".dll" diff --git a/Lib/test/test_ctypes/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py index 29108a28ec16e1..1dd3f2a234b1ee 100644 --- a/Lib/test/test_ctypes/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,7 +1,10 @@ import array +import ctypes import struct import sys import unittest +from itertools import combinations +from math import copysign, isnan from operator import truth from ctypes import (byref, sizeof, alignment, c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, @@ -38,8 +41,55 @@ def valid_ranges(*types): signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] +class IntLike: + def __int__(self): + return 2 + +class IndexLike: + def __index__(self): + return 2 + +class FloatLike: + def __float__(self): + return 2.0 + +class ComplexLike: + def __complex__(self): + return 1+1j + + +INF = float("inf") +NAN = float("nan") + class NumberTestCase(unittest.TestCase): + # from Lib/test/test_complex.py + def assertFloatsAreIdentical(self, x, y): + """assert that floats x and y are identical, in the sense that: + (1) both x and y are nans, or + (2) both x and y are infinities, with the same sign, or + (3) both x and y are zeros, with the same sign, or + (4) x and y are both finite and nonzero, and x == y + + """ + msg = 'floats {!r} and {!r} are not identical' + + if isnan(x) or isnan(y): + if isnan(x) and isnan(y): + return + elif x == y: + if x != 0.0: + return + # both zero; check that signs match + elif copysign(1.0, x) == copysign(1.0, y): + return + else: + msg += ': zeros have different signs' + self.fail(msg.format(x, y)) + + def assertComplexesAreIdentical(self, x, y): + self.assertFloatsAreIdentical(x.real, y.real) + self.assertFloatsAreIdentical(x.imag, y.imag) def test_default_init(self): # default values are set to zero @@ -86,9 +136,6 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() for t in float_types: self.assertEqual(t(2.0).value, 2.0) @@ -96,18 +143,34 @@ def __float__(self): self.assertEqual(t(2).value, 2.0) self.assertEqual(t(f).value, 2.0) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex(self): + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + self.assertEqual(t(1).value, 1+0j) + self.assertEqual(t(1.0).value, 1+0j) + self.assertEqual(t(1+0.125j).value, 1+0.125j) + self.assertEqual(t(IndexLike()).value, 2+0j) + self.assertEqual(t(FloatLike()).value, 2+0j) + self.assertEqual(t(ComplexLike()).value, 1+1j) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex_round_trip(self): + # Ensure complexes transformed exactly. The CMPLX macro should + # preserve special components (like inf/nan or signed zero). + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + with self.subTest(z=z, type=t): + self.assertComplexesAreIdentical(z, t(z).value) + def test_integers(self): - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() - class IntLike: - def __int__(self): - return 2 d = IntLike() - class IndexLike: - def __index__(self): - return 2 i = IndexLike() # integers cannot be constructed from floats, # but from integer-like objects diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index d1eeee6b0306fe..f89521cf8b3a67 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,10 +1,9 @@ -import _ctypes_test import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, - POINTER, pointer, byref, + POINTER, pointer, byref, sizeof, c_void_p, c_char_p, c_wchar_p, py_object, c_bool, c_char, c_wchar, @@ -14,6 +13,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SimpleTypesTestCase(unittest.TestCase): @@ -86,19 +87,33 @@ def test_c_char(self): with self.assertRaises(TypeError) as cm: c_char.from_param(b"abc") self.assertEqual(str(cm.exception), - "one character bytes, bytearray or integer expected") + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") def test_c_wchar(self): with self.assertRaises(TypeError) as cm: c_wchar.from_param("abc") self.assertEqual(str(cm.exception), - "one character unicode string expected") + "a unicode character expected, not a string of length 3") + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") with self.assertRaises(TypeError) as cm: c_wchar.from_param(123) self.assertEqual(str(cm.exception), - "unicode string expected instead of int instance") + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + def test_int_pointers(self): LPINT = POINTER(c_int) diff --git a/Lib/test/test_ctypes/test_pickling.py b/Lib/test/test_ctypes/test_pickling.py index 0ca42a68f0675f..9d433fc69de391 100644 --- a/Lib/test/test_ctypes/test_pickling.py +++ b/Lib/test/test_ctypes/test_pickling.py @@ -1,9 +1,10 @@ -import _ctypes_test import pickle import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, pointer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_int, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 8410174358c19d..fc558e10ba40c5 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,4 +1,3 @@ -import _ctypes_test import array import ctypes import sys @@ -10,6 +9,10 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, @@ -19,6 +22,23 @@ class PointersTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_Pointer.mro(), [_Pointer, _CData, object]) + + self.assertEqual(PyCPointerType.__name__, "PyCPointerType") + self.assertEqual(type(PyCPointerType), type) + + def test_type_flags(self): + for cls in _Pointer, PyCPointerType: + with self.subTest(cls=cls): + self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCPointerType.__init__(POINTER(c_byte), 'ptr', (), {}) + def test_pointer_crash(self): class A(POINTER(c_ulong)): diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 81eb4562c740fd..63ae799ea86ab2 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -18,12 +18,13 @@ # # In this case, there would have to be an additional reference to the argument... -import _ctypes_test import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, ArgumentError, pointer, byref, sizeof, addressof, create_string_buffer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_short, c_int, c_long, c_longlong, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") testdll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 77da35855928a4..1072a109833261 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -47,7 +47,7 @@ def test_PyLong_Long(self): @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" + s = object() ref = sys.getrefcount(s) # id(python-object) is the address pyobj = _ctypes.PyObj_FromPtr(id(s)) diff --git a/Lib/test/test_ctypes/test_random_things.py b/Lib/test/test_ctypes/test_random_things.py index 65eb53f86475d3..630f6ed9489eba 100644 --- a/Lib/test/test_ctypes/test_random_things.py +++ b/Lib/test/test_ctypes/test_random_things.py @@ -51,9 +51,9 @@ def expect_unraisable(self, exc_type, exc_msg=None): if exc_msg is not None: self.assertEqual(str(cm.unraisable.exc_value), exc_msg) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on calling ctypes " - "callback function") - self.assertIs(cm.unraisable.object, callback_func) + f"Exception ignored on calling ctypes " + f"callback function {callback_func!r}") + self.assertIsNone(cm.unraisable.object) def test_ValueError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index a90588ca9bb1b6..012722d8486218 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,9 +1,11 @@ -import _ctypes_test import ctypes import gc import sys import unittest from test import support +from test.support import import_helper +from test.support import script_helper +_ctypes_test = import_helper.import_module("_ctypes_test") MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) @@ -109,5 +111,18 @@ def func(): func() +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_returnfuncptrs.py b/Lib/test/test_ctypes/test_returnfuncptrs.py index 4010e511e75ade..337801b226ab06 100644 --- a/Lib/test/test_ctypes/test_returnfuncptrs.py +++ b/Lib/test/test_ctypes/test_returnfuncptrs.py @@ -1,6 +1,7 @@ -import _ctypes_test import unittest from ctypes import CDLL, CFUNCTYPE, ArgumentError, c_char_p, c_void_p, c_char +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class ReturnFuncPtrTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_simplesubclasses.py b/Lib/test/test_ctypes/test_simplesubclasses.py index 6072b62de5d53a..4e4bef3690f66a 100644 --- a/Lib/test/test_ctypes/test_simplesubclasses.py +++ b/Lib/test/test_ctypes/test_simplesubclasses.py @@ -1,5 +1,7 @@ import unittest -from ctypes import Structure, CFUNCTYPE, c_int +from ctypes import Structure, CFUNCTYPE, c_int, _SimpleCData +from ._support import (_CData, PyCSimpleType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class MyInt(c_int): @@ -10,6 +12,42 @@ def __eq__(self, other): class Test(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_SimpleCData.mro(), [_SimpleCData, _CData, object]) + + self.assertEqual(PyCSimpleType.__name__, "PyCSimpleType") + self.assertEqual(type(PyCSimpleType), type) + + self.assertEqual(c_int.mro(), [c_int, _SimpleCData, _CData, object]) + + def test_type_flags(self): + for cls in _SimpleCData, PyCSimpleType: + with self.subTest(cls=cls): + self.assertTrue(_SimpleCData.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_SimpleCData.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewT = PyCSimpleType.__new__(PyCSimpleType, 'NewT', (_SimpleCData,), {}) + for cls in _SimpleCData, NewT: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(_SimpleCData): + _type_ = "i" + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCSimpleType.__init__(T, 'ptr', (), {}) + + def test_swapped_type_creation(self): + cls = PyCSimpleType.__new__(PyCSimpleType, '', (), {'_type_': 'i'}) + with self.assertRaises(TypeError): + PyCSimpleType.__init__(cls) + PyCSimpleType.__init__(cls, '', (), {'_type_': 'i'}) + self.assertEqual(cls.__ctype_le__.__dict__.get('_type_'), 'i') + self.assertEqual(cls.__ctype_be__.__dict__.get('_type_'), 'i') def test_compare(self): self.assertEqual(MyInt(3), MyInt(3)) diff --git a/Lib/test/test_ctypes/test_slicing.py b/Lib/test/test_ctypes/test_slicing.py index a592d911cbe6ca..66f9e530104cac 100644 --- a/Lib/test/test_ctypes/test_slicing.py +++ b/Lib/test/test_ctypes/test_slicing.py @@ -1,7 +1,8 @@ -import _ctypes_test import unittest from ctypes import (CDLL, POINTER, sizeof, c_byte, c_short, c_int, c_long, c_char, c_wchar, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SlicesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_stringptr.py b/Lib/test/test_ctypes/test_stringptr.py index 67c61c6c3e17e6..bb6045b250ffce 100644 --- a/Lib/test/test_ctypes/test_stringptr.py +++ b/Lib/test/test_ctypes/test_stringptr.py @@ -1,9 +1,10 @@ -import _ctypes_test import sys import unittest from test import support from ctypes import (CDLL, Structure, POINTER, create_string_buffer, c_char, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index f60dfe5b42ef65..7adab794809def 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -1,5 +1,7 @@ import unittest from ctypes import Structure, Union, sizeof, c_char, c_int +from ._support import (CField, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class StructFieldsTestCase(unittest.TestCase): @@ -12,7 +14,6 @@ class StructFieldsTestCase(unittest.TestCase): # 4. The type is subclassed # # When they are finalized, assigning _fields_ is no longer allowed. - def test_1_A(self): class X(Structure): pass @@ -56,15 +57,19 @@ class X(Structure): self.assertEqual(bytes(x), b'a\x00###') def test_6(self): - class X(Structure): - _fields_ = [("x", c_int)] - CField = type(X.x) self.assertRaises(TypeError, CField) + def test_cfield_type_flags(self): + self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + + def test_cfield_inheritance_hierarchy(self): + self.assertEqual(CField.mro(), [CField, object]) + def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): - cls._fields_ = [] # This line will fail, `stgdict` is not ready + cls._fields_ = [] # This line will fail, `stginfo` is not ready with self.assertRaisesRegex(TypeError, 'ctypes state is not initialized'): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index f05ee5e491a41e..6cc09c8f2b5b59 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,4 +1,4 @@ -import _ctypes_test +from platform import architecture as _architecture import struct import sys import unittest @@ -7,8 +7,14 @@ c_uint8, c_uint16, c_uint32, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ctypes.util import find_library from struct import calcsize +from collections import namedtuple from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) class SubclassesTest(unittest.TestCase): @@ -67,6 +73,36 @@ class StructureTestCase(unittest.TestCase): "d": c_double, } + def test_inheritance_hierarchy(self): + self.assertEqual(Structure.mro(), [Structure, _CData, object]) + + self.assertEqual(PyCStructType.__name__, "PyCStructType") + self.assertEqual(type(PyCStructType), type) + + + def test_type_flags(self): + for cls in Structure, PyCStructType: + with self.subTest(cls=cls): + self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewStructure = PyCStructType.__new__(PyCStructType, 'NewStructure', + (Structure,), {}) + for cls in Structure, NewStructure: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Structure): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCStructType.__init__(T, 'ptr', (), {}) + def test_simple_structs(self): for code, tp in self.formats.items(): class X(Structure): @@ -470,37 +506,116 @@ class X(Structure): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + def _test_issue18060(self, Vector): + # The call to atan2() should succeed if the + # class fields were correctly cloned in the + # subclasses. Otherwise, it will segfault. + if sys.platform == 'win32': + libm = CDLL(find_library('msvcrt.dll')) + else: + libm = CDLL(find_library('m')) + + libm.atan2.argtypes = [Vector] + libm.atan2.restype = c_double + + arg = Vector(y=0.0, x=-1.0) + self.assertAlmostEqual(libm.atan2(arg), 3.141592653589793) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_a(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment, and PyCStgInfo_clone() + # for the Mid and Vector class definitions. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + pass + Mid._fields_ = [] + class Vector(Mid): pass + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_b(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [] + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_c(self): + # This test case calls + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [('x', c_double)] + self._test_issue18060(Vector) + def test_array_in_struct(self): # See bpo-22273 + # Load the shared library + dll = CDLL(_ctypes_test.__file__) + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] - class Test3(Structure): + class Test3AParent(Structure): + _fields_ = [ + ('data', c_float * 2), + ] + + class Test3A(Test3AParent): + _fields_ = [ + ('more_data', c_float * 2), + ] + + class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] - class Test3A(Structure): + class Test3C(Structure): _fields_ = [ - ('data', c_float * 2), + ("data", c_double * 4) ] - class Test3B(Test3A): + class Test3D(Structure): _fields_ = [ - ('more_data', c_float * 2), + ("data", c_double * 8) + ] + + class Test3E(Structure): + _fields_ = [ + ("data", c_double * 9) ] + + # Tests for struct Test2 s = Test2() expected = 0 for i in range(16): s.data[i] = i expected += i - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_array_in_struct1 + func = dll._testfunc_array_in_struct2 func.restype = c_int func.argtypes = (Test2,) result = func(s) @@ -509,29 +624,16 @@ class Test3B(Test3A): for i in range(16): self.assertEqual(s.data[i], i) - s = Test3() - s.data[0] = 3.14159 - s.data[1] = 2.71828 - expected = 3.14159 + 2.71828 - func = dll._testfunc_array_in_struct2 - func.restype = c_double - func.argtypes = (Test3,) - result = func(s) - self.assertEqual(result, expected) - # check the passed-in struct hasn't changed - self.assertEqual(s.data[0], 3.14159) - self.assertEqual(s.data[1], 2.71828) - - s = Test3B() + # Tests for struct Test3A + s = Test3A() s.data[0] = 3.14159 s.data[1] = 2.71828 s.more_data[0] = -3.0 s.more_data[1] = -2.0 - - expected = 3.14159 + 2.71828 - 5.0 - func = dll._testfunc_array_in_struct2a + expected = 3.14159 + 2.71828 - 3.0 - 2.0 + func = dll._testfunc_array_in_struct3A func.restype = c_double - func.argtypes = (Test3B,) + func.argtypes = (Test3A,) result = func(s) self.assertAlmostEqual(result, expected, places=6) # check the passed-in struct hasn't changed @@ -540,6 +642,61 @@ class Test3B(Test3A): self.assertAlmostEqual(s.more_data[0], -3.0, places=6) self.assertAlmostEqual(s.more_data[1], -2.0, places=6) + # Test3B, Test3C, Test3D, Test3E have the same logic with different + # sizes hence putting them in a loop. + StructCtype = namedtuple( + "StructCtype", + ["cls", "cfunc1", "cfunc2", "items"] + ) + structs_to_test = [ + StructCtype( + Test3B, + dll._testfunc_array_in_struct3B, + dll._testfunc_array_in_struct3B_set_defaults, + 2), + StructCtype( + Test3C, + dll._testfunc_array_in_struct3C, + dll._testfunc_array_in_struct3C_set_defaults, + 4), + StructCtype( + Test3D, + dll._testfunc_array_in_struct3D, + dll._testfunc_array_in_struct3D_set_defaults, + 8), + StructCtype( + Test3E, + dll._testfunc_array_in_struct3E, + dll._testfunc_array_in_struct3E_set_defaults, + 9), + ] + + for sut in structs_to_test: + s = sut.cls() + + # Test for cfunc1 + expected = 0 + for i in range(sut.items): + float_i = float(i) + s.data[i] = float_i + expected += float_i + func = sut.cfunc1 + func.restype = c_double + func.argtypes = (sut.cls,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + for i in range(sut.items): + self.assertEqual(s.data[i], float(i)) + + # Test for cfunc2 + func = sut.cfunc2 + func.restype = sut.cls + result = func() + # check if the default values have been set correctly + for i in range(sut.items): + self.assertEqual(result.data[i], float(i+1)) + def test_38368(self): class U(Union): _fields_ = [ diff --git a/Lib/test/test_ctypes/test_unicode.py b/Lib/test/test_ctypes/test_unicode.py index 2ddc7c56544e35..d9e17371d13572 100644 --- a/Lib/test/test_ctypes/test_unicode.py +++ b/Lib/test/test_ctypes/test_unicode.py @@ -1,6 +1,7 @@ -import _ctypes_test import ctypes import unittest +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class UnicodeTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py new file mode 100644 index 00000000000000..e2dff0f22a9213 --- /dev/null +++ b/Lib/test/test_ctypes/test_unions.py @@ -0,0 +1,35 @@ +import unittest +from ctypes import Union, c_char +from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) + + +class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Union.mro(), [Union, _CData, object]) + + self.assertEqual(UnionType.__name__, "UnionType") + self.assertEqual(type(UnionType), type) + + def test_type_flags(self): + for cls in Union, UnionType: + with self.subTest(cls=Union): + self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewUnion = UnionType.__new__(UnionType, 'NewUnion', + (Union,), {}) + for cls in Union, NewUnion: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Union): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + UnionType.__init__(T, 'ptr', (), {}) diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index d0b4803dff8529..1b757e020d5ce2 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -2,7 +2,6 @@ A testcase which accesses *values* in a dll. """ -import _ctypes_test import _imp import importlib.util import sys @@ -15,10 +14,14 @@ class ValuesTestCase(unittest.TestCase): + def setUp(self): + _ctypes_test = import_helper.import_module("_ctypes_test") + self.ctdll = CDLL(_ctypes_test.__file__) + def test_an_integer(self): # This test checks and changes an integer stored inside the # _ctypes_test dll/shared lib. - ctdll = CDLL(_ctypes_test.__file__) + ctdll = self.ctdll an_integer = c_int.in_dll(ctdll, "an_integer") x = an_integer.value self.assertEqual(x, ctdll.get_an_integer()) @@ -30,8 +33,7 @@ def test_an_integer(self): self.assertEqual(x, ctdll.get_an_integer()) def test_undefined(self): - ctdll = CDLL(_ctypes_test.__file__) - self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + self.assertRaises(ValueError, c_int.in_dll, self.ctdll, "Undefined_Symbol") class PythonValuesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 01e624f76f0685..31919118670613 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -1,6 +1,5 @@ # Windows specific tests -import _ctypes_test import ctypes import errno import sys @@ -9,6 +8,8 @@ _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support +from test.support import import_helper +from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') @@ -35,6 +36,7 @@ def test_noargs(self): @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class ReturnStructSizesTestCase(unittest.TestCase): def test_sizes(self): + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) for i in range(1, 11): fields = [ (f"f{f}", c_char) for f in range(1, i + 1)] @@ -73,6 +75,11 @@ def test_COMError(self): self.assertEqual(ex.text, "text") self.assertEqual(ex.details, ("details",)) + self.assertEqual(COMError.mro(), + [COMError, Exception, BaseException, object]) + self.assertFalse(COMError.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + self.assertTrue(COMError.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWinError(unittest.TestCase): @@ -110,6 +117,7 @@ class RECT(Structure): ("right", c_long), ("bottom", c_long)] + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) pt = POINT(15, 25) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 31bc108e7712ea..83d10dd8579074 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock from test.support import (requires, verbose, SaveSignals, cpython_only, - check_disallow_instantiation) + check_disallow_instantiation, MISSING_C_DOCSTRINGS) from test.support.import_helper import import_module # Optionally test curses module. This currently requires that the @@ -1142,6 +1142,8 @@ def test_encoding(self): with self.assertRaises(TypeError): del stdscr.encoding + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_issue21088(self): stdscr = self.stdscr # diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 272d427875ae40..ffb8bbe75c504f 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -22,6 +22,8 @@ import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. +from test import support + # Just any custom exception we can catch. class CustomError(Exception): pass @@ -1315,6 +1317,29 @@ def __post_init__(self, init_base, init_derived): c = C(10, 11, 50, 51) self.assertEqual(vars(c), {'x': 21, 'y': 101}) + def test_init_var_name_shadowing(self): + # Because dataclasses rely exclusively on `__annotations__` for + # handling InitVar and `__annotations__` preserves shadowed definitions, + # you can actually shadow an InitVar with a method or property. + # + # This only works when there is no default value; `dataclasses` uses the + # actual name (which will be bound to the shadowing method) for default + # values. + @dataclass + class C: + shadowed: InitVar[int] + _shadowed: int = field(init=False) + + def __post_init__(self, shadowed): + self._shadowed = shadowed * 2 + + @property + def shadowed(self): + return self._shadowed * 3 + + c = C(5) + self.assertEqual(c.shadowed, 30) + def test_default_factory(self): # Test a factory that returns a new list. @dataclass @@ -1522,6 +1547,24 @@ class A(types.GenericAlias): self.assertTrue(is_dataclass(type(a))) self.assertTrue(is_dataclass(a)) + def test_is_dataclass_inheritance(self): + @dataclass + class X: + y: int + + class Z(X): + pass + + self.assertTrue(is_dataclass(X), "X should be a dataclass") + self.assertTrue( + is_dataclass(Z), + "Z should be a dataclass because it inherits from X", + ) + z_instance = Z(y=5) + self.assertTrue( + is_dataclass(z_instance), + "z_instance should be a dataclass because it is an instance of Z", + ) def test_helper_fields_with_class_instance(self): # Check that we can call fields() on either a class or instance, @@ -2216,6 +2259,7 @@ def assertDocStrEqual(self, a, b): # whitespace stripped. self.assertEqual(a.replace(' ', ''), b.replace(' ', '')) + @support.requires_docstrings def test_existing_docstring_not_overridden(self): @dataclass class C: @@ -2468,6 +2512,15 @@ def __repr__(self): class TestEq(unittest.TestCase): + def test_recursive_eq(self): + # Test a class with recursive child + @dataclass + class C: + recursive: object = ... + c = C() + c.recursive = c + self.assertEqual(c, c) + def test_no_eq(self): # Test a class with no __eq__ and eq=False. @dataclass(eq=False) @@ -3495,6 +3548,123 @@ class A(Base): self.assertIs(a.__weakref__, a_ref) + def test_dataclass_derived_weakref_slot(self): + class A: + pass + + @dataclass(slots=True, weakref_slot=True) + class B(A): + pass + + self.assertEqual(B.__slots__, ()) + B() + + def test_dataclass_derived_generic(self): + T = typing.TypeVar('T') + + @dataclass(slots=True, weakref_slot=True) + class A(typing.Generic[T]): + pass + self.assertEqual(A.__slots__, ('__weakref__',)) + self.assertTrue(A.__weakref__) + A() + + @dataclass(slots=True, weakref_slot=True) + class B[T2]: + pass + self.assertEqual(B.__slots__, ('__weakref__',)) + self.assertTrue(B.__weakref__) + B() + + def test_dataclass_derived_generic_from_base(self): + T = typing.TypeVar('T') + + class RawBase: ... + + @dataclass(slots=True, weakref_slot=True) + class C1(typing.Generic[T], RawBase): + pass + self.assertEqual(C1.__slots__, ()) + self.assertTrue(C1.__weakref__) + C1() + @dataclass(slots=True, weakref_slot=True) + class C2(RawBase, typing.Generic[T]): + pass + self.assertEqual(C2.__slots__, ()) + self.assertTrue(C2.__weakref__) + C2() + + @dataclass(slots=True, weakref_slot=True) + class D[T2](RawBase): + pass + self.assertEqual(D.__slots__, ()) + self.assertTrue(D.__weakref__) + D() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithSlots: + __slots__ = ('a', 'b') + + @dataclass(slots=True, weakref_slot=True) + class E1(WithSlots, Generic[T]): + pass + self.assertEqual(E1.__slots__, ('__weakref__',)) + self.assertTrue(E1.__weakref__) + E1() + @dataclass(slots=True, weakref_slot=True) + class E2(Generic[T], WithSlots): + pass + self.assertEqual(E2.__slots__, ('__weakref__',)) + self.assertTrue(E2.__weakref__) + E2() + + @dataclass(slots=True, weakref_slot=True) + class F[T2](WithSlots): + pass + self.assertEqual(F.__slots__, ('__weakref__',)) + self.assertTrue(F.__weakref__) + F() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithWeakrefSlot: + __slots__ = ('__weakref__',) + + @dataclass(slots=True, weakref_slot=True) + class G1(WithWeakrefSlot, Generic[T]): + pass + self.assertEqual(G1.__slots__, ()) + self.assertTrue(G1.__weakref__) + G1() + @dataclass(slots=True, weakref_slot=True) + class G2(Generic[T], WithWeakrefSlot): + pass + self.assertEqual(G2.__slots__, ()) + self.assertTrue(G2.__weakref__) + G2() + + @dataclass(slots=True, weakref_slot=True) + class H[T2](WithWeakrefSlot): + pass + self.assertEqual(H.__slots__, ()) + self.assertTrue(H.__weakref__) + H() + + def test_dataclass_slot_dict(self): + class WithDictSlot: + __slots__ = ('__dict__',) + + @dataclass(slots=True) + class A(WithDictSlot): ... + + self.assertEqual(A.__slots__, ()) + self.assertEqual(A().__dict__, {}) + A() + + class TestDescriptors(unittest.TestCase): def test_set_name(self): # See bpo-33141. diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 3859733a4fe65b..005187f13e665f 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -1,5 +1,6 @@ import unittest import sys +import functools from test.support.import_helper import import_fresh_module @@ -39,21 +40,26 @@ def load_tests(loader, tests, pattern): for cls in test_classes: cls.__name__ += suffix cls.__qualname__ += suffix - @classmethod - def setUpClass(cls_, module=module): - cls_._save_sys_modules = sys.modules.copy() - sys.modules[TESTS] = module - sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime - sys.modules['_strptime'] = module._strptime - @classmethod - def tearDownClass(cls_): - sys.modules.clear() - sys.modules.update(cls_._save_sys_modules) - cls.setUpClass = setUpClass - cls.tearDownClass = tearDownClass - tests.addTests(loader.loadTestsFromTestCase(cls)) + + @functools.wraps(cls, updated=()) + class Wrapper(cls): + @classmethod + def setUpClass(cls_, module=module): + cls_._save_sys_modules = sys.modules.copy() + sys.modules[TESTS] = module + sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_strptime'] = module._strptime + super().setUpClass() + + @classmethod + def tearDownClass(cls_): + super().tearDownClass() + sys.modules.clear() + sys.modules.update(cls_._save_sys_modules) + + tests.addTests(loader.loadTestsFromTestCase(Wrapper)) return tests diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e3924d8ec8b5c1..4be7c5649da68a 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -6,6 +6,13 @@ from test.support import import_helper from test.support import os_helper + +try: + from dbm import sqlite3 as dbm_sqlite3 +except ImportError: + dbm_sqlite3 = None + + try: from dbm import ndbm except ImportError: @@ -213,6 +220,27 @@ def test_whichdb_ndbm(self): for path in fnames: self.assertIsNone(self.dbm.whichdb(path)) + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3(self): + # Databases created by dbm.sqlite3 are detected correctly. + with dbm_sqlite3.open(_fname, "c") as db: + db["key"] = "value" + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + @unittest.skipUnless(dbm_sqlite3, reason='Test requires dbm.sqlite3') + def test_whichdb_sqlite3_existing_db(self): + # Existing sqlite3 databases are detected correctly. + sqlite3 = import_helper.import_module("sqlite3") + try: + # Create an empty database. + with sqlite3.connect(_fname) as cx: + cx.execute("CREATE TABLE dummy(database)") + cx.commit() + finally: + cx.close() + self.assertEqual(self.dbm.whichdb(_fname), "dbm.sqlite3") + + def setUp(self): self.addCleanup(cleaunup_test_dir) setup_test_dir() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index a481175b3bfdbd..672f9092207cf6 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -246,9 +246,27 @@ def test_missing_data(self): _delete_files() with self.assertRaises(FileNotFoundError): dumbdbm.open(_fname, value) + self.assertFalse(os.path.exists(_fname + '.dat')) self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + for value in ('c', 'n'): + _delete_files() + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dat')) + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + def test_missing_index(self): with dumbdbm.open(_fname, 'n') as f: pass @@ -259,6 +277,60 @@ def test_missing_index(self): self.assertFalse(os.path.exists(_fname + '.dir')) self.assertFalse(os.path.exists(_fname + '.bak')) + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + + def test_sync_empty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + + def test_sync_nonempty_unmodified(self): + with dumbdbm.open(_fname, 'n') as f: + pass + os.unlink(_fname + '.dir') + for value in ('c', 'n'): + with dumbdbm.open(_fname, value) as f: + f['key'] = 'value' + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + f.sync() + self.assertTrue(os.path.exists(_fname + '.dir')) + self.assertTrue(os.path.exists(_fname + '.bak')) + os.unlink(_fname + '.dir') + os.unlink(_fname + '.bak') + f.sync() + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + self.assertFalse(os.path.exists(_fname + '.dir')) + self.assertFalse(os.path.exists(_fname + '.bak')) + def test_invalid_flag(self): for flag in ('x', 'rf', None): with self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py new file mode 100644 index 00000000000000..2e1f2d32924bad --- /dev/null +++ b/Lib/test/test_dbm_sqlite3.py @@ -0,0 +1,310 @@ +import sys +import unittest +from contextlib import closing +from functools import partial +from pathlib import Path +from test.support import import_helper, os_helper + +dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") +# N.B. The test will fail on some platforms without sqlite3 +# if the sqlite3 import is above the import of dbm.sqlite3. +# This is deliberate: if the import helper managed to import dbm.sqlite3, +# we must inevitably be able to import sqlite3. Else, we have a problem. +import sqlite3 +from dbm.sqlite3 import _normalize_uri + + +class _SQLiteDbmTests(unittest.TestCase): + + def setUp(self): + self.filename = os_helper.TESTFN + db = dbm_sqlite3.open(self.filename, "c") + db.close() + + def tearDown(self): + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) + + +class URI(unittest.TestCase): + + def test_uri_substitutions(self): + dataset = ( + ("/absolute/////b/c", "/absolute/b/c"), + ("PRE#MID##END", "PRE%23MID%23%23END"), + ("%#?%%#", "%25%23%3F%25%25%23"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_uri_windows(self): + dataset = ( + # Relative subdir. + (r"2018\January.xlsx", + "2018/January.xlsx"), + # Absolute with drive letter. + (r"C:\Projects\apilibrary\apilibrary.sln", + "/C:/Projects/apilibrary/apilibrary.sln"), + # Relative with drive letter. + (r"C:Projects\apilibrary\apilibrary.sln", + "/C:Projects/apilibrary/apilibrary.sln"), + ) + for path, normalized in dataset: + with self.subTest(path=path, normalized=normalized): + if not Path(path).is_absolute(): + self.skipTest(f"skipping relative path: {path!r}") + self.assertTrue(_normalize_uri(path).endswith(normalized)) + + +class ReadOnly(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + with dbm_sqlite3.open(self.filename, "w") as db: + db[b"key1"] = "value1" + db[b"key2"] = "value2" + self.db = dbm_sqlite3.open(self.filename, "r") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_readonly_read(self): + self.assertEqual(self.db[b"key1"], b"value1") + self.assertEqual(self.db[b"key2"], b"value2") + + def test_readonly_write(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"new"] = "value" + + def test_readonly_delete(self): + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"key1"] + + def test_readonly_keys(self): + self.assertEqual(self.db.keys(), [b"key1", b"key2"]) + + def test_readonly_iter(self): + self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + + +class ReadWrite(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def db_content(self): + with closing(sqlite3.connect(self.filename)) as cx: + keys = [r[0] for r in cx.execute("SELECT key FROM Dict")] + vals = [r[0] for r in cx.execute("SELECT value FROM Dict")] + return keys, vals + + def test_readwrite_unique_key(self): + self.db["key"] = "value" + self.db["key"] = "other" + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"other"]) + + def test_readwrite_delete(self): + self.db["key"] = "value" + self.db["new"] = "other" + + del self.db[b"new"] + keys, vals = self.db_content() + self.assertEqual(keys, [b"key"]) + self.assertEqual(vals, [b"value"]) + + del self.db[b"key"] + keys, vals = self.db_content() + self.assertEqual(keys, []) + self.assertEqual(vals, []) + + def test_readwrite_null_key(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[None] = "value" + + def test_readwrite_null_value(self): + with self.assertRaises(dbm_sqlite3.error): + self.db[b"key"] = None + + +class Misuse(_SQLiteDbmTests): + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_misuse_double_create(self): + self.db["key"] = "value" + with dbm_sqlite3.open(self.filename, "c") as db: + self.assertEqual(db[b"key"], b"value") + + def test_misuse_double_close(self): + self.db.close() + + def test_misuse_invalid_flag(self): + regex = "must be.*'r'.*'w'.*'c'.*'n', not 'invalid'" + with self.assertRaisesRegex(ValueError, regex): + dbm_sqlite3.open(self.filename, flag="invalid") + + def test_misuse_double_delete(self): + self.db["key"] = "value" + del self.db[b"key"] + with self.assertRaises(KeyError): + del self.db[b"key"] + + def test_misuse_invalid_key(self): + with self.assertRaises(KeyError): + self.db[b"key"] + + def test_misuse_iter_close1(self): + self.db["1"] = 1 + it = iter(self.db) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_iter_close2(self): + self.db["1"] = 1 + self.db["2"] = 2 + it = iter(self.db) + next(it) + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + next(it) + + def test_misuse_use_after_close(self): + self.db.close() + with self.assertRaises(dbm_sqlite3.error): + self.db[b"read"] + with self.assertRaises(dbm_sqlite3.error): + self.db[b"write"] = "value" + with self.assertRaises(dbm_sqlite3.error): + del self.db[b"del"] + with self.assertRaises(dbm_sqlite3.error): + len(self.db) + with self.assertRaises(dbm_sqlite3.error): + self.db.keys() + + def test_misuse_reinit(self): + with self.assertRaises(dbm_sqlite3.error): + self.db.__init__("new.db", flag="n", mode=0o666) + + def test_misuse_empty_filename(self): + for flag in "r", "w", "c", "n": + with self.assertRaises(dbm_sqlite3.error): + db = dbm_sqlite3.open("", flag="c") + + +class DataTypes(_SQLiteDbmTests): + + dataset = ( + # (raw, coerced) + (42, b"42"), + (3.14, b"3.14"), + ("string", b"string"), + (b"bytes", b"bytes"), + ) + + def setUp(self): + super().setUp() + self.db = dbm_sqlite3.open(self.filename, "w") + + def tearDown(self): + self.db.close() + super().tearDown() + + def test_datatypes_values(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db["key"] = raw + self.assertEqual(self.db[b"key"], coerced) + + def test_datatypes_keys(self): + for raw, coerced in self.dataset: + with self.subTest(raw=raw, coerced=coerced): + self.db[raw] = "value" + self.assertEqual(self.db[coerced], b"value") + # Raw keys are silently coerced to bytes. + self.assertEqual(self.db[raw], b"value") + del self.db[raw] + + def test_datatypes_replace_coerced(self): + self.db["10"] = "value" + self.db[b"10"] = "value" + self.db[10] = "value" + self.assertEqual(self.db.keys(), [b"10"]) + + +class CorruptDatabase(_SQLiteDbmTests): + """Verify that database exceptions are raised as dbm.sqlite3.error.""" + + def setUp(self): + super().setUp() + with closing(sqlite3.connect(self.filename)) as cx: + with cx: + cx.execute("DROP TABLE IF EXISTS Dict") + cx.execute("CREATE TABLE Dict (invalid_schema)") + + def check(self, flag, fn, should_succeed=False): + with closing(dbm_sqlite3.open(self.filename, flag)) as db: + with self.assertRaises(dbm_sqlite3.error): + fn(db) + + @staticmethod + def read(db): + return db["key"] + + @staticmethod + def write(db): + db["key"] = "value" + + @staticmethod + def iter(db): + next(iter(db)) + + @staticmethod + def keys(db): + db.keys() + + @staticmethod + def del_(db): + del db["key"] + + @staticmethod + def len_(db): + len(db) + + def test_corrupt_readwrite(self): + for flag in "r", "w", "c": + with self.subTest(flag=flag): + check = partial(self.check, flag=flag) + check(fn=self.read) + check(fn=self.write) + check(fn=self.iter) + check(fn=self.keys) + check(fn=self.del_) + check(fn=self.len_) + + def test_corrupt_force_new(self): + with closing(dbm_sqlite3.open(self.filename, "n")) as db: + db["foo"] = "write" + _ = db[b"foo"] + next(iter(db)) + del db[b"foo"] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7a5fe62b467372..46755107de0102 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,13 +34,13 @@ import locale from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, - check_sanitizer, check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, - darwin_malloc_err_warning, is_emscripten) + darwin_malloc_err_warning) from test.support.import_helper import import_fresh_module from test.support import threading_helper +from test.support import warnings_helper import random import inspect import threading @@ -1109,6 +1109,13 @@ def test_formatting(self): ('z>z6.1f', '-0.', 'zzz0.0'), ('x>z6.1f', '-0.', 'xxx0.0'), ('🖤>z6.1f', '-0.', '🖤🖤🖤0.0'), # multi-byte fill char + ('\x00>z6.1f', '-0.', '\x00\x00\x000.0'), # null fill char + + # issue 114563 ('z' format on F type in cdecimal) + ('z3,.10F', '-6.24E-323', '0.0000000000'), + + # issue 91060 ('#' format in cdecimal) + ('#', '0', '0.'), # issue 6850 ('a=-7.0', '0.12345', 'aaaa0.1'), @@ -1237,7 +1244,14 @@ def test_deprecated_N_format(self): else: self.assertRaises(ValueError, format, h, 'N') self.assertRaises(ValueError, format, h, '010.3N') - + with warnings_helper.check_no_warnings(self): + self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34') + self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34') + self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34') + self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000') + self.assertRaises(ValueError, format, h, '>Nf') + self.assertRaises(ValueError, format, h, '10Nf') + self.assertRaises(ValueError, format, h, 'Nx') @run_with_locale('LC_ALL', 'ps_AF') def test_wide_char_separator_decimal_point(self): @@ -4702,9 +4716,33 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**(MAX_EMAX + 1). + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5628,47 +5666,6 @@ def __abs__(self): self.assertEqual(Decimal.from_float(cls(101.1)), Decimal.from_float(101.1)) - # Issue 41540: - @unittest.skipIf(sys.platform.startswith("aix"), - "AIX: default ulimit: test is flaky because of extreme over-allocation") - @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten") - @unittest.skipIf(check_sanitizer(address=True, memory=True), - "ASAN/MSAN sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_maxcontext_exact_arith(self): - - # Make sure that exact operations do not raise MemoryError due - # to huge intermediate values when the context precision is very - # large. - - # The following functions fill the available precision and are - # therefore not suitable for large precisions (by design of the - # specification). - MaxContextSkip = ['logical_invert', 'next_minus', 'next_plus', - 'logical_and', 'logical_or', 'logical_xor', - 'next_toward', 'rotate', 'shift'] - - Decimal = C.Decimal - Context = C.Context - localcontext = C.localcontext - - # Here only some functions that are likely candidates for triggering a - # MemoryError are tested. deccheck.py has an exhaustive test. - maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX) - with localcontext(maxcontext): - self.assertEqual(Decimal(0).exp(), 1) - self.assertEqual(Decimal(1).ln(), 0) - self.assertEqual(Decimal(1).log10(), 0) - self.assertEqual(Decimal(10**2).log10(), 2) - self.assertEqual(Decimal(10**223).log10(), 223) - self.assertEqual(Decimal(10**19).logb(), 19) - self.assertEqual(Decimal(4).sqrt(), 2) - self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5')) - self.assertEqual(divmod(Decimal(10), 3), (3, 1)) - self.assertEqual(Decimal(10) // 3, 3) - self.assertEqual(Decimal(4) / 2, 2) - self.assertEqual(Decimal(400) ** -1, Decimal('0.0025')) - def test_c_immutable_types(self): SignalDict = type(C.Context().flags) SignalDictMixin = SignalDict.__bases__[0] @@ -5718,6 +5715,20 @@ def test_c_signaldict_segfault(self): with self.assertRaisesRegex(ValueError, err_msg): sd.copy() + def test_format_fallback_capitals(self): + # Fallback to _pydecimal formatting (triggered by `#` format which + # is unsupported by mpdecimal) should honor the current context. + x = C.Decimal('6.09e+23') + self.assertEqual(format(x, '#'), '6.09E+23') + with C.localcontext(capitals=0): + self.assertEqual(format(x, '#'), '6.09e+23') + + def test_format_fallback_rounding(self): + y = C.Decimal('6.09') + self.assertEqual(format(y, '#.1f'), '6.1') + with C.localcontext(rounding=C.ROUND_DOWN): + self.assertEqual(format(y, '#.1f'), '6.0') + @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): @@ -5881,13 +5892,17 @@ def load_tests(loader, tests, pattern): if TODO_TESTS is None: from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL + orig_context = orig_sys_decimal.getcontext().copy() for mod in C, P: if not mod: continue def setUp(slf, mod=mod): sys.modules['decimal'] = mod - def tearDown(slf): + init(mod) + def tearDown(slf, mod=mod): sys.modules['decimal'] = orig_sys_decimal + mod.setcontext(ORIGINAL_CONTEXT[mod].copy()) + orig_sys_decimal.setcontext(orig_context.copy()) optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0 sys.modules['decimal'] = mod tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown, @@ -5902,8 +5917,8 @@ def setUpModule(): TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal') def tearDownModule(): - if C: C.setcontext(ORIGINAL_CONTEXT[C]) - P.setcontext(ORIGINAL_CONTEXT[P]) + if C: C.setcontext(ORIGINAL_CONTEXT[C].copy()) + P.setcontext(ORIGINAL_CONTEXT[P].copy()) if not C: warnings.warn('C tests skipped: no module named _decimal.', UserWarning) diff --git a/Lib/test/test_decorators.py b/Lib/test/test_decorators.py index 4b492178c1581f..78361be9fa1e61 100644 --- a/Lib/test/test_decorators.py +++ b/Lib/test/test_decorators.py @@ -1,5 +1,5 @@ import unittest -from types import MethodType + def funcattrs(**kwds): def decorate(func): @@ -291,44 +291,6 @@ def bar(): return 42 self.assertEqual(bar(), 42) self.assertEqual(actions, expected_actions) - def test_wrapped_descriptor_inside_classmethod(self): - class BoundWrapper: - def __init__(self, wrapped): - self.__wrapped__ = wrapped - - def __call__(self, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) - - class Wrapper: - def __init__(self, wrapped): - self.__wrapped__ = wrapped - - def __get__(self, instance, owner): - bound_function = self.__wrapped__.__get__(instance, owner) - return BoundWrapper(bound_function) - - def decorator(wrapped): - return Wrapper(wrapped) - - class Class: - @decorator - @classmethod - def inner(cls): - # This should already work. - return 'spam' - - @classmethod - @decorator - def outer(cls): - # Raised TypeError with a message saying that the 'Wrapper' - # object is not callable. - return 'eggs' - - self.assertEqual(Class.inner(), 'spam') - self.assertEqual(Class.outer(), 'eggs') - self.assertEqual(Class().inner(), 'spam') - self.assertEqual(Class().outer(), 'eggs') - def test_bound_function_inside_classmethod(self): class A: def foo(self, cls): @@ -339,91 +301,6 @@ class B: self.assertEqual(B.bar(), 'spam') - def test_wrapped_classmethod_inside_classmethod(self): - class MyClassMethod1: - def __init__(self, func): - self.func = func - - def __call__(self, cls): - if hasattr(self.func, '__get__'): - return self.func.__get__(cls, cls)() - return self.func(cls) - - def __get__(self, instance, owner=None): - if owner is None: - owner = type(instance) - return MethodType(self, owner) - - class MyClassMethod2: - def __init__(self, func): - if isinstance(func, classmethod): - func = func.__func__ - self.func = func - - def __call__(self, cls): - return self.func(cls) - - def __get__(self, instance, owner=None): - if owner is None: - owner = type(instance) - return MethodType(self, owner) - - for myclassmethod in [MyClassMethod1, MyClassMethod2]: - class A: - @myclassmethod - def f1(cls): - return cls - - @classmethod - @myclassmethod - def f2(cls): - return cls - - @myclassmethod - @classmethod - def f3(cls): - return cls - - @classmethod - @classmethod - def f4(cls): - return cls - - @myclassmethod - @MyClassMethod1 - def f5(cls): - return cls - - @myclassmethod - @MyClassMethod2 - def f6(cls): - return cls - - self.assertIs(A.f1(), A) - self.assertIs(A.f2(), A) - self.assertIs(A.f3(), A) - self.assertIs(A.f4(), A) - self.assertIs(A.f5(), A) - self.assertIs(A.f6(), A) - a = A() - self.assertIs(a.f1(), A) - self.assertIs(a.f2(), A) - self.assertIs(a.f3(), A) - self.assertIs(a.f4(), A) - self.assertIs(a.f5(), A) - self.assertIs(a.f6(), A) - - def f(cls): - return cls - - self.assertIs(myclassmethod(f).__get__(a)(), A) - self.assertIs(myclassmethod(f).__get__(a, A)(), A) - self.assertIs(myclassmethod(f).__get__(A, A)(), A) - self.assertIs(myclassmethod(f).__get__(A)(), type(A)) - self.assertIs(classmethod(f).__get__(a)(), A) - self.assertIs(classmethod(f).__get__(a, A)(), A) - self.assertIs(classmethod(f).__get__(A, A)(), A) - self.assertIs(classmethod(f).__get__(A)(), type(A)) class TestClassDecorators(unittest.TestCase): diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index ae1dfacd7262e4..4679f297fd7f4a 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -166,7 +166,7 @@ def test_contains(self): with self.assertRaises(RuntimeError): n in d - def test_contains_count_stop_crashes(self): + def test_contains_count_index_stop_crashes(self): class A: def __eq__(self, other): d.clear() @@ -178,6 +178,10 @@ def __eq__(self, other): with self.assertRaises(RuntimeError): _ = d.count(3) + d = deque([A()]) + with self.assertRaises(RuntimeError): + d.index(0) + def test_extend(self): d = deque('a') self.assertRaises(TypeError, d.extend, 1) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 4a3db80ca43c27..b0f86317bfecf6 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1314,7 +1314,7 @@ class X(object): # Inherit from object on purpose to check some backwards compatibility paths class X(object): __slots__ = "a" - with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): + with self.assertRaisesRegex(AttributeError, "'test.test_descr.ClassPropertiesAndMethods.test_slots..X' object has no attribute 'a'"): X().a # Test string subclass in `__slots__`, see gh-98783 @@ -1593,8 +1593,11 @@ def f(cls, arg): self.fail("classmethod shouldn't accept keyword args") cm = classmethod(f) - cm_dict = {'__annotations__': {}, - '__doc__': "f docstring", + cm_dict = {'__doc__': ( + "f docstring" + if support.HAVE_DOCSTRINGS + else None + ), '__module__': __name__, '__name__': 'f', '__qualname__': f.__qualname__} @@ -1606,6 +1609,41 @@ def f(cls, arg): del cm.x self.assertNotHasAttr(cm, "x") + def test_classmethod_staticmethod_annotations(self): + for deco in (classmethod, staticmethod): + @deco + def unannotated(cls): pass + @deco + def annotated(cls) -> int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') @@ -1683,10 +1721,10 @@ class D(C): self.assertEqual(d.foo(1), (d, 1)) self.assertEqual(D.foo(d, 1), (d, 1)) sm = staticmethod(None) - self.assertEqual(sm.__dict__, {'__doc__': None}) + self.assertEqual(sm.__dict__, {'__doc__': None.__doc__}) sm.x = 42 self.assertEqual(sm.x, 42) - self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None}) + self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None.__doc__}) del sm.x self.assertNotHasAttr(sm, "x") @@ -4590,18 +4628,16 @@ def test_special_unbound_method_types(self): def test_not_implemented(self): # Testing NotImplemented... # all binary methods should be able to return a NotImplemented - import operator def specialmethod(self, other): return NotImplemented def check(expr, x, y): - try: - exec(expr, {'x': x, 'y': y, 'operator': operator}) - except TypeError: - pass - else: - self.fail("no TypeError from %r" % (expr,)) + with ( + self.subTest(expr=expr, x=x, y=y), + self.assertRaises(TypeError), + ): + exec(expr, {'x': x, 'y': y}) N1 = sys.maxsize + 1 # might trigger OverflowErrors instead of # TypeErrors @@ -4622,12 +4658,23 @@ def check(expr, x, y): ('__and__', 'x & y', 'x &= y'), ('__or__', 'x | y', 'x |= y'), ('__xor__', 'x ^ y', 'x ^= y')]: - rname = '__r' + name[2:] + # Defines 'left' magic method: A = type('A', (), {name: specialmethod}) a = A() check(expr, a, a) check(expr, a, N1) check(expr, a, N2) + # Defines 'right' magic method: + rname = '__r' + name[2:] + B = type('B', (), {rname: specialmethod}) + b = B() + check(expr, b, b) + check(expr, a, b) + check(expr, b, a) + check(expr, b, N1) + check(expr, b, N2) + check(expr, N1, b) + check(expr, N2, b) if iexpr: check(iexpr, a, a) check(iexpr, a, N1) @@ -4734,6 +4781,20 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ + def test_gh55664(self): + # gh-55664: issue a warning when the + # __dict__ of a class contains non-string keys + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = type('MyClass', (), {1: 2}) + + class meta(type): + def __new__(mcls, name, bases, ns): + ns[1] = 2 + return super().__new__(mcls, name, bases, ns) + + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = meta('MyClass', (), {}) + def test_proxy_call(self): class FakeStr: __class__ = str @@ -4987,6 +5048,7 @@ def __new__(cls): cls.lst = [2**i for i in range(10000)] X.descr + @support.suppress_immortalization() def test_remove_subclass(self): # bpo-46417: when the last subclass of a type is deleted, # remove_subclass() clears the internal dictionary of subclasses: @@ -5004,6 +5066,21 @@ class Child(Parent): gc.collect() self.assertEqual(Parent.__subclasses__(), []) + def test_instance_method_get_behavior(self): + # test case for gh-113157 + + class A: + def meth(self): + return self + + class B: + pass + + a = A() + b = B() + b.meth = a.meth.__get__(b, B) + self.assertEqual(b.meth(), a) + def test_attr_raise_through_property(self): # test case for gh-103272 class A: @@ -5046,8 +5123,10 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5056,7 +5135,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 5) + self.assertEqual(len(values), 7) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5066,8 +5145,10 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... @@ -5136,7 +5217,8 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - X = type('X', (Base,), {MyKey(): 5}) + with self.assertWarnsRegex(RuntimeWarning, 'X'): + X = type('X', (Base,), {MyKey(): 5}) # mykey is read from Base self.assertEqual(X.mykey, 'from Base') # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 13e3ea41bdb76c..828440a993a975 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -8,7 +8,7 @@ # of much interest anymore), and a few were fiddled to make the output # deterministic. -from test.support import sortdict +from test.support import sortdict # noqa: F401 import doctest import unittest @@ -39,16 +39,16 @@ def merge(self, other): Here's the new type at work: >>> print(defaultdict) # show our type - + >>> print(type(defaultdict)) # its metatype >>> a = defaultdict(default=0.0) # create an instance >>> print(a) # show the instance {} >>> print(type(a)) # show its type - + >>> print(a.__class__) # show its class - + >>> print(type(a) is a.__class__) # its type is its class True >>> a[1] = 3.25 # modify the instance @@ -99,7 +99,7 @@ def merge(self, other): >>> print(sortdict(a.__dict__)) {'default': -1000, 'x1': 100, 'x2': 200} >>> -""" +""" % {'modname': __name__} class defaultdict2(dict): __slots__ = ['default'] @@ -264,19 +264,19 @@ def merge(self, other): ... print("classmethod", cls, y) >>> C.foo(1) - classmethod 1 + classmethod 1 >>> c = C() >>> c.foo(1) - classmethod 1 + classmethod 1 >>> class D(C): ... pass >>> D.foo(1) - classmethod 1 + classmethod 1 >>> d = D() >>> d.foo(1) - classmethod 1 + classmethod 1 This prints "classmethod __main__.D 1" both times; in other words, the class passed as the first argument of foo() is the class involved in the @@ -292,18 +292,18 @@ class passed as the first argument of foo() is the class involved in the >>> E.foo(1) E.foo() called - classmethod 1 + classmethod 1 >>> e = E() >>> e.foo(1) E.foo() called - classmethod 1 + classmethod 1 In this example, the call to C.foo() from E.foo() will see class C as its first argument, not class E. This is to be expected, since the call specifies the class C. But it stresses the difference between these class methods and methods defined in metaclasses (where an upcall to a metamethod would pass the target class as an explicit first argument). -""" +""" % {'modname': __name__} test_5 = """ diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 620d0ca4f4c2da..e5dba7cdc570a8 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, Py_C_RECURSION_LIMIT +from test.support import import_helper, get_c_recursion_limit class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index cad568b6ac4c2d..d9881611c19c43 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class DictSetTest(unittest.TestCase): @@ -279,7 +279,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(Py_C_RECURSION_LIMIT//2 + 100): + for i in range(get_c_recursion_limit()//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6afd90af8442ad..9e217249be7332 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -272,10 +272,30 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) +class TestDiffer(unittest.TestCase): + def test_close_matches_aligned(self): + # Of the 4 closely matching pairs, we want 1 to match with 3, + # and 2 with 4, to align with a "top to bottom" mental model. + a = ["cat\n", "dog\n", "close match 1\n", "close match 2\n"] + b = ["close match 3\n", "close match 4\n", "kitten\n", "puppy\n"] + m = difflib.Differ().compare(a, b) + self.assertEqual(list(m), + ['- cat\n', + '- dog\n', + '- close match 1\n', + '? ^\n', + '+ close match 3\n', + '? ^\n', + '- close match 2\n', + '? ^\n', + '+ close match 4\n', + '? ^\n', + '+ kitten\n', + '+ puppy\n']) class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -287,7 +307,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -427,6 +447,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -475,10 +517,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8ab0e1ecbc4a7f..ab1c48f9b25361 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -2,6 +2,7 @@ import contextlib import dis +import functools import io import re import sys @@ -13,6 +14,7 @@ import opcode +CACHE = dis.opmap["CACHE"] def get_tb(): def _error(): @@ -42,45 +44,45 @@ def cm(cls, x): cls.x = x == 1 dis_c_instance_method = """\ -%3d RESUME 0 - -%3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) - COMPARE_OP 72 (==) - LOAD_FAST 0 (self) - STORE_ATTR 0 (x) - RETURN_CONST 0 (None) +%3d RESUME 0 + +%3d LOAD_FAST 1 (x) + LOAD_CONST 1 (1) + COMPARE_OP 72 (==) + LOAD_FAST 0 (self) + STORE_ATTR 0 (x) + RETURN_CONST 0 (None) """ % (_C.__init__.__code__.co_firstlineno, _C.__init__.__code__.co_firstlineno + 1,) dis_c_instance_method_bytes = """\ - RESUME 0 - LOAD_FAST 1 - LOAD_CONST 1 - COMPARE_OP 72 (==) - LOAD_FAST 0 - STORE_ATTR 0 - RETURN_CONST 0 + RESUME 0 + LOAD_FAST 1 + LOAD_CONST 1 + COMPARE_OP 72 (==) + LOAD_FAST 0 + STORE_ATTR 0 + RETURN_CONST 0 """ dis_c_class_method = """\ -%3d RESUME 0 - -%3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) - COMPARE_OP 72 (==) - LOAD_FAST 0 (cls) - STORE_ATTR 0 (x) - RETURN_CONST 0 (None) +%3d RESUME 0 + +%3d LOAD_FAST 1 (x) + LOAD_CONST 1 (1) + COMPARE_OP 72 (==) + LOAD_FAST 0 (cls) + STORE_ATTR 0 (x) + RETURN_CONST 0 (None) """ % (_C.cm.__code__.co_firstlineno, _C.cm.__code__.co_firstlineno + 2,) dis_c_static_method = """\ -%3d RESUME 0 +%3d RESUME 0 -%3d LOAD_FAST 0 (x) - LOAD_CONST 1 (1) - COMPARE_OP 72 (==) - STORE_FAST 0 (x) - RETURN_CONST 0 (None) +%3d LOAD_FAST 0 (x) + LOAD_CONST 1 (1) + COMPARE_OP 72 (==) + STORE_FAST 0 (x) + RETURN_CONST 0 (None) """ % (_C.sm.__code__.co_firstlineno, _C.sm.__code__.co_firstlineno + 2,) # Class disassembling info has an extra newline at end. @@ -100,51 +102,64 @@ def _f(a): return 1 dis_f = """\ -%3d RESUME 0 +%3d RESUME 0 -%3d LOAD_GLOBAL 1 (print + NULL) - LOAD_FAST 0 (a) - CALL 1 - POP_TOP +%3d LOAD_GLOBAL 1 (print + NULL) + LOAD_FAST 0 (a) + CALL 1 + POP_TOP -%3d RETURN_CONST 1 (1) +%3d RETURN_CONST 1 (1) +""" % (_f.__code__.co_firstlineno, + _f.__code__.co_firstlineno + 1, + _f.__code__.co_firstlineno + 2) + +dis_f_with_offsets = """\ +%3d 0 RESUME 0 + +%3d 2 LOAD_GLOBAL 1 (print + NULL) + 12 LOAD_FAST 0 (a) + 14 CALL 1 + 22 POP_TOP + +%3d 24 RETURN_CONST 1 (1) """ % (_f.__code__.co_firstlineno, _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) dis_f_co_code = """\ - RESUME 0 - LOAD_GLOBAL 1 - LOAD_FAST 0 - CALL 1 - POP_TOP - RETURN_CONST 1 + RESUME 0 + LOAD_GLOBAL 1 + LOAD_FAST 0 + CALL 1 + POP_TOP + RETURN_CONST 1 """ - def bug708901(): for res in range(1, 10): pass dis_bug708901 = """\ -%3d RESUME 0 +%3d RESUME 0 -%3d LOAD_GLOBAL 1 (range + NULL) - LOAD_CONST 1 (1) +%3d LOAD_GLOBAL 1 (range + NULL) + LOAD_CONST 1 (1) -%3d LOAD_CONST 2 (10) +%3d LOAD_CONST 2 (10) -%3d CALL 2 - GET_ITER - >> FOR_ITER 3 (to 36) - STORE_FAST 0 (res) +%3d CALL 2 + GET_ITER + L1: FOR_ITER 3 (to L2) + STORE_FAST 0 (res) -%3d JUMP_BACKWARD 5 (to 26) +%3d JUMP_BACKWARD 5 (to L1) -%3d >> END_FOR - RETURN_CONST 0 (None) +%3d L2: END_FOR + POP_TOP + RETURN_CONST 0 (None) """ % (bug708901.__code__.co_firstlineno, bug708901.__code__.co_firstlineno + 1, bug708901.__code__.co_firstlineno + 2, @@ -159,20 +174,20 @@ def bug1333982(x=[]): pass dis_bug1333982 = """\ -%3d RESUME 0 +%3d RESUME 0 -%3d LOAD_ASSERTION_ERROR - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) - MAKE_FUNCTION - LOAD_FAST 0 (x) - GET_ITER - CALL 0 +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + MAKE_FUNCTION + LOAD_FAST 0 (x) + GET_ITER + CALL 0 -%3d LOAD_CONST 2 (1) +%3d LOAD_CONST 2 (1) -%3d BINARY_OP 0 (+) - CALL 0 - RAISE_VARARGS 1 +%3d BINARY_OP 0 (+) + CALL 0 + RAISE_VARARGS 1 """ % (bug1333982.__code__.co_firstlineno, bug1333982.__code__.co_firstlineno + 1, __file__, @@ -190,8 +205,8 @@ def bug42562(): dis_bug42562 = """\ - RESUME 0 - RETURN_CONST 0 (None) + RESUME 0 + RETURN_CONST 0 (None) """ # Extended arg followed by NOP @@ -204,11 +219,11 @@ def bug42562(): ]) dis_bug_45757 = """\ - EXTENDED_ARG 1 - NOP - EXTENDED_ARG 1 - LOAD_CONST 297 - RETURN_VALUE + EXTENDED_ARG 1 + NOP + EXTENDED_ARG 1 + LOAD_CONST 297 + RETURN_VALUE """ # [255, 255, 255, 252] is -4 in a 4 byte signed integer @@ -221,10 +236,10 @@ def bug42562(): dis_bug46724 = """\ - >> EXTENDED_ARG 255 - EXTENDED_ARG 65535 - EXTENDED_ARG 16777215 - JUMP_FORWARD -4 (to 0) + L1: EXTENDED_ARG 255 + EXTENDED_ARG 65535 + EXTENDED_ARG 16777215 + JUMP_FORWARD -4 (to L1) """ def func_w_kwargs(a, b, **c): @@ -234,96 +249,96 @@ def wrap_func_w_kwargs(): func_w_kwargs(1, 2, c=5) dis_kw_names = """\ -%3d RESUME 0 - -%3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) - LOAD_CONST 1 (1) - LOAD_CONST 2 (2) - LOAD_CONST 3 (5) - LOAD_CONST 4 (('c',)) - CALL_KW 3 - POP_TOP - RETURN_CONST 0 (None) +%3d RESUME 0 + +%3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) + LOAD_CONST 1 (1) + LOAD_CONST 2 (2) + LOAD_CONST 3 (5) + LOAD_CONST 4 (('c',)) + CALL_KW 3 + POP_TOP + RETURN_CONST 0 (None) """ % (wrap_func_w_kwargs.__code__.co_firstlineno, wrap_func_w_kwargs.__code__.co_firstlineno + 1) dis_intrinsic_1_2 = """\ - 0 RESUME 0 - - 1 LOAD_CONST 0 (0) - LOAD_CONST 1 (('*',)) - IMPORT_NAME 0 (math) - CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) - POP_TOP - RETURN_CONST 2 (None) + 0 RESUME 0 + + 1 LOAD_CONST 0 (0) + LOAD_CONST 1 (('*',)) + IMPORT_NAME 0 (math) + CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) + POP_TOP + RETURN_CONST 2 (None) """ dis_intrinsic_1_5 = """\ - 0 RESUME 0 + 0 RESUME 0 - 1 LOAD_NAME 0 (a) - CALL_INTRINSIC_1 5 (INTRINSIC_UNARY_POSITIVE) - RETURN_VALUE + 1 LOAD_NAME 0 (a) + CALL_INTRINSIC_1 5 (INTRINSIC_UNARY_POSITIVE) + RETURN_VALUE """ dis_intrinsic_1_6 = """\ - 0 RESUME 0 + 0 RESUME 0 - 1 BUILD_LIST 0 - LOAD_NAME 0 (a) - LIST_EXTEND 1 - CALL_INTRINSIC_1 6 (INTRINSIC_LIST_TO_TUPLE) - RETURN_VALUE + 1 BUILD_LIST 0 + LOAD_NAME 0 (a) + LIST_EXTEND 1 + CALL_INTRINSIC_1 6 (INTRINSIC_LIST_TO_TUPLE) + RETURN_VALUE """ _BIG_LINENO_FORMAT = """\ - 1 RESUME 0 + 1 RESUME 0 -%3d LOAD_GLOBAL 0 (spam) - POP_TOP - RETURN_CONST 0 (None) +%3d LOAD_GLOBAL 0 (spam) + POP_TOP + RETURN_CONST 0 (None) """ _BIG_LINENO_FORMAT2 = """\ - 1 RESUME 0 + 1 RESUME 0 -%4d LOAD_GLOBAL 0 (spam) - POP_TOP - RETURN_CONST 0 (None) +%4d LOAD_GLOBAL 0 (spam) + POP_TOP + RETURN_CONST 0 (None) """ dis_module_expected_results = """\ Disassembly of f: - 4 RESUME 0 - RETURN_CONST 0 (None) + 4 RESUME 0 + RETURN_CONST 0 (None) Disassembly of g: - 5 RESUME 0 - RETURN_CONST 0 (None) + 5 RESUME 0 + RETURN_CONST 0 (None) """ expr_str = "x + 1" dis_expr_str = """\ - 0 RESUME 0 + 0 RESUME 0 - 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) - BINARY_OP 0 (+) - RETURN_VALUE + 1 LOAD_NAME 0 (x) + LOAD_CONST 0 (1) + BINARY_OP 0 (+) + RETURN_VALUE """ simple_stmt_str = "x = x + 1" dis_simple_stmt_str = """\ - 0 RESUME 0 + 0 RESUME 0 - 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) - BINARY_OP 0 (+) - STORE_NAME 0 (x) - RETURN_CONST 1 (None) + 1 LOAD_NAME 0 (x) + LOAD_CONST 0 (1) + BINARY_OP 0 (+) + STORE_NAME 0 (x) + RETURN_CONST 1 (None) """ annot_stmt_str = """\ @@ -335,34 +350,23 @@ def wrap_func_w_kwargs(): # leading newline is for a reason (tests lineno) dis_annot_stmt_str = """\ - 0 RESUME 0 - - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) - STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - - 3 LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 0 (1) - CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') - STORE_SUBSCR - - 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 3 (0) - CALL 1 - STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + 0 RESUME 0 + + 2 LOAD_CONST 0 (1) + STORE_NAME 0 (x) + + 4 LOAD_CONST 0 (1) + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) + PUSH_NULL + LOAD_CONST 1 (0) + CALL 1 + STORE_SUBSCR + + 2 LOAD_CONST 2 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + RETURN_CONST 3 (None) """ compound_stmt_str = """\ @@ -372,64 +376,67 @@ def wrap_func_w_kwargs(): # Trailing newline has been deliberately omitted dis_compound_stmt_str = """\ - 0 RESUME 0 + 0 RESUME 0 - 1 LOAD_CONST 0 (0) - STORE_NAME 0 (x) + 1 LOAD_CONST 0 (0) + STORE_NAME 0 (x) - 2 NOP + 2 NOP - 3 >> LOAD_NAME 0 (x) - LOAD_CONST 1 (1) - BINARY_OP 13 (+=) - STORE_NAME 0 (x) + 3 L1: LOAD_NAME 0 (x) + LOAD_CONST 1 (1) + BINARY_OP 13 (+=) + STORE_NAME 0 (x) - 2 JUMP_BACKWARD 7 (to 8) + 2 JUMP_BACKWARD 7 (to L1) """ dis_traceback = """\ -%4d RESUME 0 +%4d RESUME 0 -%4d NOP +%4d NOP -%4d LOAD_CONST 1 (1) - LOAD_CONST 2 (0) - --> BINARY_OP 11 (/) - POP_TOP +%4d L1: LOAD_CONST 1 (1) + LOAD_CONST 2 (0) + --> BINARY_OP 11 (/) + POP_TOP -%4d LOAD_FAST_CHECK 1 (tb) - RETURN_VALUE +%4d L2: LOAD_FAST_CHECK 1 (tb) + RETURN_VALUE -None >> PUSH_EXC_INFO + -- L3: PUSH_EXC_INFO -%4d LOAD_GLOBAL 0 (Exception) - CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 82) - STORE_FAST 0 (e) +%4d LOAD_GLOBAL 0 (Exception) + CHECK_EXC_MATCH + POP_JUMP_IF_FALSE 23 (to L7) + STORE_FAST 0 (e) -%4d LOAD_FAST 0 (e) - LOAD_ATTR 2 (__traceback__) - STORE_FAST 1 (tb) - POP_EXCEPT - LOAD_CONST 0 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) +%4d L4: LOAD_FAST 0 (e) + LOAD_ATTR 2 (__traceback__) + STORE_FAST 1 (tb) + L5: POP_EXCEPT + LOAD_CONST 0 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) -%4d LOAD_FAST 1 (tb) - RETURN_VALUE +%4d LOAD_FAST 1 (tb) + RETURN_VALUE -None >> LOAD_CONST 0 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) - RERAISE 1 + -- L6: LOAD_CONST 0 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + RERAISE 1 -%4d >> RERAISE 0 +%4d L7: RERAISE 0 -None >> COPY 3 - POP_EXCEPT - RERAISE 1 + -- L8: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: -4 rows + L1 to L2 -> L3 [0] + L3 to L4 -> L8 [1] lasti + L4 to L5 -> L6 [1] lasti + L6 to L8 -> L8 [1] lasti """ % (TRACEBACK_CODE.co_firstlineno, TRACEBACK_CODE.co_firstlineno + 1, TRACEBACK_CODE.co_firstlineno + 2, @@ -443,25 +450,25 @@ def _fstring(a, b, c, d): return f'{a} {b:4} {c!r} {d!r:4}' dis_fstring = """\ -%3d RESUME 0 - -%3d LOAD_FAST 0 (a) - FORMAT_SIMPLE - LOAD_CONST 1 (' ') - LOAD_FAST 1 (b) - LOAD_CONST 2 ('4') - FORMAT_WITH_SPEC - LOAD_CONST 1 (' ') - LOAD_FAST 2 (c) - CONVERT_VALUE 2 (repr) - FORMAT_SIMPLE - LOAD_CONST 1 (' ') - LOAD_FAST 3 (d) - CONVERT_VALUE 2 (repr) - LOAD_CONST 2 ('4') - FORMAT_WITH_SPEC - BUILD_STRING 7 - RETURN_VALUE +%3d RESUME 0 + +%3d LOAD_FAST 0 (a) + FORMAT_SIMPLE + LOAD_CONST 1 (' ') + LOAD_FAST 1 (b) + LOAD_CONST 2 ('4') + FORMAT_WITH_SPEC + LOAD_CONST 1 (' ') + LOAD_FAST 2 (c) + CONVERT_VALUE 2 (repr) + FORMAT_SIMPLE + LOAD_CONST 1 (' ') + LOAD_FAST 3 (d) + CONVERT_VALUE 2 (repr) + LOAD_CONST 2 ('4') + FORMAT_WITH_SPEC + BUILD_STRING 7 + RETURN_VALUE """ % (_fstring.__code__.co_firstlineno, _fstring.__code__.co_firstlineno + 1) def _with(c): @@ -470,44 +477,51 @@ def _with(c): y = 2 dis_with = """\ -%4d RESUME 0 - -%4d LOAD_FAST 0 (c) - BEFORE_WITH - POP_TOP - -%4d LOAD_CONST 1 (1) - STORE_FAST 1 (x) - -%4d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - POP_TOP - -%4d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - -%4d >> PUSH_EXC_INFO - WITH_EXCEPT_START - TO_BOOL - POP_JUMP_IF_TRUE 1 (to 52) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP - -%4d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - -None >> COPY 3 - POP_EXCEPT - RERAISE 1 +%4d RESUME 0 + +%4d LOAD_FAST 0 (c) + COPY 1 + LOAD_SPECIAL 1 (__exit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 0 (__enter__) + CALL 0 + L1: POP_TOP + +%4d LOAD_CONST 1 (1) + STORE_FAST 1 (x) + +%4d L2: LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 3 + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + +%4d L3: PUSH_EXC_INFO + WITH_EXCEPT_START + TO_BOOL + POP_JUMP_IF_TRUE 1 (to L4) + RERAISE 2 + L4: POP_TOP + L5: POP_EXCEPT + POP_TOP + POP_TOP + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + + -- L6: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: -2 rows + L1 to L2 -> L3 [2] lasti + L3 to L5 -> L6 [4] lasti """ % (_with.__code__.co_firstlineno, _with.__code__.co_firstlineno + 1, _with.__code__.co_firstlineno + 2, @@ -523,86 +537,97 @@ async def _asyncwith(c): y = 2 dis_asyncwith = """\ -%4d RETURN_GENERATOR - POP_TOP - RESUME 0 - -%4d LOAD_FAST 0 (c) - BEFORE_ASYNC_WITH - GET_AWAITABLE 1 - LOAD_CONST 0 (None) - >> SEND 3 (to 24) - YIELD_VALUE 2 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) - >> END_SEND - POP_TOP - -%4d LOAD_CONST 1 (1) - STORE_FAST 1 (x) - -%4d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 3 (to 60) - YIELD_VALUE 2 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 50) - >> END_SEND - POP_TOP - -%4d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - -%4d >> CLEANUP_THROW - -None JUMP_BACKWARD 26 (to 24) - -%4d >> CLEANUP_THROW - -None JUMP_BACKWARD 11 (to 60) - -%4d >> PUSH_EXC_INFO - WITH_EXCEPT_START - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 4 (to 102) - YIELD_VALUE 3 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) - >> CLEANUP_THROW - >> END_SEND - TO_BOOL - POP_JUMP_IF_TRUE 1 (to 118) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP - -%4d LOAD_CONST 2 (2) - STORE_FAST 2 (y) - RETURN_CONST 0 (None) - -None >> COPY 3 - POP_EXCEPT - RERAISE 1 - >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) - RERAISE 1 +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + +%4d LOAD_FAST 0 (c) + COPY 1 + LOAD_SPECIAL 3 (__aexit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 2 (__aenter__) + CALL 0 + GET_AWAITABLE 1 + LOAD_CONST 0 (None) + L2: SEND 3 (to L5) + L3: YIELD_VALUE 1 + L4: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L2) + L5: END_SEND + L6: POP_TOP + +%4d LOAD_CONST 1 (1) + STORE_FAST 1 (x) + +%4d L7: LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 3 + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + L8: SEND 3 (to L11) + L9: YIELD_VALUE 1 + L10: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L8) + L11: END_SEND + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + +%4d L12: CLEANUP_THROW + L13: JUMP_BACKWARD_NO_INTERRUPT 25 (to L5) + L14: CLEANUP_THROW + L15: JUMP_BACKWARD_NO_INTERRUPT 9 (to L11) + L16: PUSH_EXC_INFO + WITH_EXCEPT_START + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + L17: SEND 4 (to L21) + L18: YIELD_VALUE 1 + L19: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 5 (to L17) + L20: CLEANUP_THROW + L21: END_SEND + TO_BOOL + POP_JUMP_IF_TRUE 1 (to L22) + RERAISE 2 + L22: POP_TOP + L23: POP_EXCEPT + POP_TOP + POP_TOP + POP_TOP + +%4d LOAD_CONST 2 (2) + STORE_FAST 2 (y) + RETURN_CONST 0 (None) + + -- L24: COPY 3 + POP_EXCEPT + RERAISE 1 + L25: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 ExceptionTable: -12 rows + L1 to L3 -> L25 [0] lasti + L3 to L4 -> L12 [4] + L4 to L6 -> L25 [0] lasti + L6 to L7 -> L16 [2] lasti + L7 to L9 -> L25 [0] lasti + L9 to L10 -> L14 [2] + L10 to L13 -> L25 [0] lasti + L14 to L15 -> L25 [0] lasti + L16 to L18 -> L24 [4] lasti + L18 to L19 -> L20 [7] + L19 to L23 -> L24 [4] lasti + L23 to L25 -> L25 [0] lasti """ % (_asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 2, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 3, ) @@ -620,31 +645,32 @@ def _tryfinallyconst(b): b() dis_tryfinally = """\ -%4d RESUME 0 +%4d RESUME 0 -%4d NOP +%4d NOP -%4d LOAD_FAST 0 (a) +%4d L1: LOAD_FAST 0 (a) -%4d LOAD_FAST 1 (b) - PUSH_NULL - CALL 0 - POP_TOP - RETURN_VALUE +%4d L2: LOAD_FAST 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RETURN_VALUE -None >> PUSH_EXC_INFO + -- L3: PUSH_EXC_INFO -%4d LOAD_FAST 1 (b) - PUSH_NULL - CALL 0 - POP_TOP - RERAISE 0 +%4d LOAD_FAST 1 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 -None >> COPY 3 - POP_EXCEPT - RERAISE 1 + -- L4: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: -2 rows + L1 to L2 -> L3 [0] + L3 to L4 -> L4 [1] lasti """ % (_tryfinally.__code__.co_firstlineno, _tryfinally.__code__.co_firstlineno + 1, _tryfinally.__code__.co_firstlineno + 2, @@ -653,31 +679,31 @@ def _tryfinallyconst(b): ) dis_tryfinallyconst = """\ -%4d RESUME 0 +%4d RESUME 0 -%4d NOP +%4d NOP -%4d NOP +%4d NOP -%4d LOAD_FAST 0 (b) - PUSH_NULL - CALL 0 - POP_TOP - RETURN_CONST 1 (1) +%4d LOAD_FAST 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + RETURN_CONST 1 (1) -None PUSH_EXC_INFO + -- L1: PUSH_EXC_INFO -%4d LOAD_FAST 0 (b) - PUSH_NULL - CALL 0 - POP_TOP - RERAISE 0 +%4d LOAD_FAST 0 (b) + PUSH_NULL + CALL 0 + POP_TOP + RERAISE 0 -None >> COPY 3 - POP_EXCEPT - RERAISE 1 + -- L2: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: -1 row + L1 to L2 -> L2 [1] lasti """ % (_tryfinallyconst.__code__.co_firstlineno, _tryfinallyconst.__code__.co_firstlineno + 1, _tryfinallyconst.__code__.co_firstlineno + 2, @@ -702,19 +728,19 @@ def foo(x): return foo dis_nested_0 = """\ -None MAKE_CELL 0 (y) + -- MAKE_CELL 0 (y) -%4d RESUME 0 +%4d RESUME 0 -%4d LOAD_FAST 0 (y) - BUILD_TUPLE 1 - LOAD_CONST 1 () - MAKE_FUNCTION - SET_FUNCTION_ATTRIBUTE 8 (closure) - STORE_FAST 1 (foo) +%4d LOAD_FAST 0 (y) + BUILD_TUPLE 1 + LOAD_CONST 1 () + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + STORE_FAST 1 (foo) -%4d LOAD_FAST 1 (foo) - RETURN_VALUE +%4d LOAD_FAST 1 (foo) + RETURN_VALUE """ % (_h.__code__.co_firstlineno, _h.__code__.co_firstlineno + 1, __file__, @@ -724,22 +750,22 @@ def foo(x): dis_nested_1 = """%s Disassembly of : -None COPY_FREE_VARS 1 - MAKE_CELL 0 (x) - -%4d RESUME 0 - -%4d LOAD_GLOBAL 1 (list + NULL) - LOAD_FAST 0 (x) - BUILD_TUPLE 1 - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) - MAKE_FUNCTION - SET_FUNCTION_ATTRIBUTE 8 (closure) - LOAD_DEREF 1 (y) - GET_ITER - CALL 0 - CALL 1 - RETURN_VALUE + -- COPY_FREE_VARS 1 + MAKE_CELL 0 (x) + +%4d RESUME 0 + +%4d LOAD_GLOBAL 1 (list + NULL) + LOAD_FAST 0 (x) + BUILD_TUPLE 1 + LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 8 (closure) + LOAD_DEREF 1 (y) + GET_ITER + CALL 0 + CALL 1 + RETURN_VALUE """ % (dis_nested_0, __file__, _h.__code__.co_firstlineno + 1, @@ -751,28 +777,29 @@ def foo(x): dis_nested_2 = """%s Disassembly of at 0x..., file "%s", line %d>: -None COPY_FREE_VARS 1 - -%4d RETURN_GENERATOR - POP_TOP - RESUME 0 - LOAD_FAST 0 (.0) - >> FOR_ITER 10 (to 34) - STORE_FAST 1 (z) - LOAD_DEREF 2 (x) - LOAD_FAST 1 (z) - BINARY_OP 0 (+) - YIELD_VALUE 1 - RESUME 1 - POP_TOP - JUMP_BACKWARD 12 (to 10) - >> END_FOR - RETURN_CONST 0 (None) - -None >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) - RERAISE 1 + -- COPY_FREE_VARS 1 + +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + LOAD_FAST 0 (.0) + L2: FOR_ITER 10 (to L3) + STORE_FAST 1 (z) + LOAD_DEREF 2 (x) + LOAD_FAST 1 (z) + BINARY_OP 0 (+) + YIELD_VALUE 0 + RESUME 5 + POP_TOP + JUMP_BACKWARD 12 (to L2) + L3: END_FOR + POP_TOP + RETURN_CONST 0 (None) + + -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 ExceptionTable: -1 row + L1 to L4 -> L4 [0] lasti """ % (dis_nested_1, __file__, _h.__code__.co_firstlineno + 3, @@ -784,14 +811,14 @@ def load_test(x, y=0): return a, b dis_load_test_quickened_code = """\ -%3d 0 RESUME_CHECK 0 +%3d RESUME_CHECK 0 -%3d 2 LOAD_FAST_LOAD_FAST 1 (x, y) - 4 STORE_FAST_STORE_FAST 50 (b, a) +%3d LOAD_FAST_LOAD_FAST 1 (x, y) + STORE_FAST_STORE_FAST 50 (b, a) -%3d 6 LOAD_FAST_LOAD_FAST 35 (a, b) - 8 BUILD_TUPLE 2 - 10 RETURN_VALUE +%3d LOAD_FAST_LOAD_FAST 35 (a, b) + BUILD_TUPLE 2 + RETURN_VALUE """ % (load_test.__code__.co_firstlineno, load_test.__code__.co_firstlineno + 1, load_test.__code__.co_firstlineno + 2) @@ -801,25 +828,26 @@ def loop_test(): load_test(i) dis_loop_test_quickened_code = """\ -%3d RESUME_CHECK 0 - -%3d BUILD_LIST 0 - LOAD_CONST 1 ((1, 2, 3)) - LIST_EXTEND 1 - LOAD_CONST 2 (3) - BINARY_OP 5 (*) - GET_ITER - >> FOR_ITER_LIST 14 (to 48) - STORE_FAST 0 (i) - -%3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) - LOAD_FAST 0 (i) - CALL_PY_WITH_DEFAULTS 1 - POP_TOP - JUMP_BACKWARD 16 (to 16) - -%3d >> END_FOR - RETURN_CONST 0 (None) +%3d RESUME_CHECK 0 + +%3d BUILD_LIST 0 + LOAD_CONST 1 ((1, 2, 3)) + LIST_EXTEND 1 + LOAD_CONST 2 (3) + BINARY_OP 5 (*) + GET_ITER + L1: FOR_ITER_LIST 14 (to L2) + STORE_FAST 0 (i) + +%3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) + LOAD_FAST 0 (i) + CALL_PY_GENERAL 1 + POP_TOP + JUMP_BACKWARD 16 (to L1) + +%3d L2: END_FOR + POP_TOP + RETURN_CONST 0 (None) """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, loop_test.__code__.co_firstlineno + 2, @@ -829,14 +857,14 @@ def extended_arg_quick(): *_, _ = ... dis_extended_arg_quick_code = """\ -%3d 0 RESUME 0 - -%3d 2 LOAD_CONST 1 (Ellipsis) - 4 EXTENDED_ARG 1 - 6 UNPACK_EX 256 - 8 POP_TOP - 10 STORE_FAST 0 (_) - 12 RETURN_CONST 0 (None) +%3d RESUME 0 + +%3d LOAD_CONST 1 (Ellipsis) + EXTENDED_ARG 1 + UNPACK_EX 256 + POP_TOP + STORE_FAST 0 (_) + RETURN_CONST 0 (None) """% (extended_arg_quick.__code__.co_firstlineno, extended_arg_quick.__code__.co_firstlineno + 1,) @@ -848,74 +876,19 @@ class DisTestBase(unittest.TestCase): def strip_addresses(self, text): return re.sub(r'\b0x[0-9A-Fa-f]+\b', '0x...', text) - def find_offset_column(self, lines): - for line in lines: - if line and not line.startswith("Disassembly"): - break - else: - return 0, 0 - offset = 5 - while (line[offset] == " "): - offset += 1 - if (line[offset] == ">"): - offset += 2 - while (line[offset] == " "): - offset += 1 - end = offset - while line[end] in "0123456789": - end += 1 - return end-5, end - - def assert_offsets_increasing(self, text, delta): - expected_offset = 0 - lines = text.splitlines() - start, end = self.find_offset_column(lines) - for line in lines: - if not line: - continue - if line.startswith("Disassembly"): - expected_offset = 0 - continue - if line.startswith("Exception"): - break - offset = int(line[start:end]) - self.assertGreaterEqual(offset, expected_offset, line) - expected_offset = offset + delta - def assert_exception_table_increasing(self, lines): prev_start, prev_end = -1, -1 count = 0 for line in lines: - m = re.match(r' (\d+) to (\d+) -> \d+ \[\d+\]', line) + m = re.match(r' L(\d+) to L(\d+) -> L\d+ \[\d+\]', line) start, end = [int(g) for g in m.groups()] self.assertGreaterEqual(end, start) - self.assertGreater(start, prev_end) + self.assertGreaterEqual(start, prev_end) prev_start, prev_end = start, end count += 1 return count - def strip_offsets(self, text): - lines = text.splitlines(True) - start, end = self.find_offset_column(lines) - res = [] - lines = iter(lines) - for line in lines: - if line.startswith("Exception"): - res.append(line) - break - if not line or line.startswith("Disassembly"): - res.append(line) - else: - res.append(line[:start] + line[end:]) - num_rows = self.assert_exception_table_increasing(lines) - if num_rows: - res.append(f"{num_rows} row{'s' if num_rows > 1 else ''}\n") - return "".join(res) - - def do_disassembly_compare(self, got, expected, with_offsets=False): - if not with_offsets: - self.assert_offsets_increasing(got, 2) - got = self.strip_offsets(got) + def do_disassembly_compare(self, got, expected): if got != expected: got = self.strip_addresses(got) self.assertEqual(got, expected) @@ -938,17 +911,16 @@ def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): def get_disassemble_as_string(self, func, lasti=-1): return self.get_disassembly(func, lasti, False) - def do_disassembly_test(self, func, expected, with_offsets=False): + def do_disassembly_test(self, func, expected, **kwargs): self.maxDiff = None - got = self.get_disassembly(func, depth=0) - self.do_disassembly_compare(got, expected, with_offsets) + got = self.get_disassembly(func, depth=0, **kwargs) + self.do_disassembly_compare(got, expected) # Add checks for dis.disco if hasattr(func, '__code__'): got_disco = io.StringIO() with contextlib.redirect_stdout(got_disco): - dis.disco(func.__code__) - self.do_disassembly_compare(got_disco.getvalue(), expected, - with_offsets) + dis.disco(func.__code__, **kwargs) + self.do_disassembly_compare(got_disco.getvalue(), expected) def test_opmap(self): self.assertEqual(dis.opmap["CACHE"], 0) @@ -976,6 +948,9 @@ def test_widths(self): def test_dis(self): self.do_disassembly_test(_f, dis_f) + def test_dis_with_offsets(self): + self.do_disassembly_test(_f, dis_f_with_offsets, show_offsets=True) + def test_bug_708901(self): self.do_disassembly_test(bug708901, dis_bug708901) @@ -1036,41 +1011,6 @@ def func(count): from test import dis_module self.do_disassembly_test(dis_module, dis_module_expected_results) - def test_big_offsets(self): - self.maxDiff = None - def func(count): - namespace = {} - func = "def foo(x):\n " + ";".join(["x = x + 1"] * count) + "\n return x" - exec(func, namespace) - return namespace['foo'] - - def expected(count, w): - s = ['''\ - 1 %*d RESUME 0 - - 2 %*d LOAD_FAST 0 (x) - %*d LOAD_CONST 1 (1) - %*d BINARY_OP 0 (+) -''' % (w, 0, w, 2, w, 4, w, 6)] - s += ['''\ - %*d STORE_FAST_LOAD_FAST 0 (x, x) - %*d LOAD_CONST 1 (1) - %*d BINARY_OP 0 (+) -''' % (w, 8*i + 10, w, 8*i + 12, w, 8*i + 14) - for i in range(count-1)] - s += ['''\ - %*d STORE_FAST 0 (x) - - 3 %*d LOAD_FAST 0 (x) - %*d RETURN_VALUE -''' % (w, 8*count + 2, w, 8*count + 4, w, 8*count + 6)] - return ''.join(s) - - for i in range(1, 5): - self.do_disassembly_test(func(i), expected(i, 4), True) - self.do_disassembly_test(func(1200), expected(1200, 4), True) - self.do_disassembly_test(func(1300), expected(1300, 5), True) - def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) @@ -1151,7 +1091,7 @@ def test_dis_traceback(self): sys.last_exc = e tb_dis = self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) - self.do_disassembly_test(None, tb_dis, True) + self.do_disassembly_test(None, tb_dis) def test_dis_object(self): self.assertRaises(TypeError, dis.dis, object()) @@ -1160,7 +1100,6 @@ def test_disassemble_recursive(self): def check(expected, **kwargs): dis = self.get_disassembly(_h, **kwargs) dis = self.strip_addresses(dis) - dis = self.strip_offsets(dis) self.assertEqual(dis, expected) check(dis_nested_0, depth=0) @@ -1187,73 +1126,73 @@ def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY): def test_super_instructions(self): self.code_quicken(lambda: load_test(0, 0)) got = self.get_disassembly(load_test, adaptive=True) - self.do_disassembly_compare(got, dis_load_test_quickened_code, True) + self.do_disassembly_compare(got, dis_load_test_quickened_code) @cpython_only @requires_specialization def test_binary_specialize(self): binary_op_quicken = """\ - 0 0 RESUME_CHECK 0 + 0 RESUME_CHECK 0 - 1 2 LOAD_NAME 0 (a) - 4 LOAD_NAME 1 (b) - 6 %s - 10 RETURN_VALUE + 1 LOAD_NAME 0 (a) + LOAD_NAME 1 (b) + %s + RETURN_VALUE """ co_int = compile('a + b', "", "eval") self.code_quicken(lambda: exec(co_int, {}, {'a': 1, 'b': 2})) got = self.get_disassembly(co_int, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)", True) + self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)") co_unicode = compile('a + b', "", "eval") self.code_quicken(lambda: exec(co_unicode, {}, {'a': 'a', 'b': 'b'})) got = self.get_disassembly(co_unicode, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)", True) + self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)") binary_subscr_quicken = """\ - 0 0 RESUME_CHECK 0 + 0 RESUME_CHECK 0 - 1 2 LOAD_NAME 0 (a) - 4 LOAD_CONST 0 (0) - 6 %s - 10 RETURN_VALUE + 1 LOAD_NAME 0 (a) + LOAD_CONST 0 (0) + %s + RETURN_VALUE """ co_list = compile('a[0]', "", "eval") self.code_quicken(lambda: exec(co_list, {}, {'a': [0]})) got = self.get_disassembly(co_list, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT", True) + self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT") co_dict = compile('a[0]', "", "eval") self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}})) got = self.get_disassembly(co_dict, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT", True) + self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT") @cpython_only @requires_specialization def test_load_attr_specialize(self): load_attr_quicken = """\ - 0 0 RESUME_CHECK 0 + 0 RESUME_CHECK 0 - 1 2 LOAD_CONST 0 ('a') - 4 LOAD_ATTR_SLOT 0 (__class__) - 24 RETURN_VALUE + 1 LOAD_CONST 0 ('a') + LOAD_ATTR_SLOT 0 (__class__) + RETURN_VALUE """ co = compile("'a'.__class__", "", "eval") self.code_quicken(lambda: exec(co, {}, {})) got = self.get_disassembly(co, adaptive=True) - self.do_disassembly_compare(got, load_attr_quicken, True) + self.do_disassembly_compare(got, load_attr_quicken) @cpython_only @requires_specialization def test_call_specialize(self): call_quicken = """\ - 0 RESUME_CHECK 0 + 0 RESUME_CHECK 0 - 1 LOAD_NAME 0 (str) - PUSH_NULL - LOAD_CONST 0 (1) - CALL_STR_1 1 - RETURN_VALUE + 1 LOAD_NAME 0 (str) + PUSH_NULL + LOAD_CONST 0 (1) + CALL_STR_1 1 + RETURN_VALUE """ co = compile("str(1)", "", "eval") self.code_quicken(lambda: exec(co, {}, {})) @@ -1263,20 +1202,46 @@ def test_call_specialize(self): @cpython_only @requires_specialization def test_loop_quicken(self): - import _testinternalcapi # Loop can trigger a quicken where the loop is located - self.code_quicken(loop_test, 1) + self.code_quicken(loop_test, 4) got = self.get_disassembly(loop_test, adaptive=True) expected = dis_loop_test_quickened_code - if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly - got = got.replace("ENTER_EXECUTOR", "JUMP_BACKWARD ") self.do_disassembly_compare(got, expected) + @cpython_only + @requires_specialization + def test_loop_with_conditional_at_end_is_quickened(self): + def for_loop_true(x): + for i in range(10): + if x: + pass + + for_loop_true(True) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_true, adaptive=True)) + + def for_loop_false(x): + for i in range(10): + if x: + pass + + for_loop_false(False) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_false, adaptive=True)) + + def while_loop(): + i = 0 + while i < 10: + i += 1 + + while_loop() + self.assertIn('COMPARE_OP_INT', + self.get_disassembly(while_loop, adaptive=True)) + @cpython_only def test_extended_arg_quick(self): got = self.get_disassembly(extended_arg_quick) - self.do_disassembly_compare(got, dis_extended_arg_quick_code, True) + self.do_disassembly_compare(got, dis_extended_arg_quick_code) def get_cached_values(self, quickened, adaptive): def f(): @@ -1288,9 +1253,9 @@ def f(): else: # "copy" the code to un-quicken it: f.__code__ = f.__code__.replace() - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( f, show_caches=True, adaptive=adaptive - ): + ), show_caches=True): if instruction.opname == "CACHE": yield instruction.argrepr @@ -1323,19 +1288,22 @@ def f(): # However, this might change in the future. So we explicitly try to find # a CACHE entry in the instructions. If we can't do that, fail the test - for inst in dis.get_instructions(f, show_caches=True): + for inst in _unroll_caches_as_Instructions( + dis.get_instructions(f, show_caches=True), show_caches=True): if inst.opname == "CACHE": op_offset = inst.offset - 2 cache_offset = inst.offset break + else: + opname = inst.opname else: self.fail("Can't find a CACHE entry in the function provided to do the test") assem_op = self.get_disassembly(f.__code__, lasti=op_offset, wrapper=False) assem_cache = self.get_disassembly(f.__code__, lasti=cache_offset, wrapper=False) - # Make sure --> exists and points to the correct offset - self.assertRegex(assem_op, fr"-->\s+{op_offset}") + # Make sure --> exists and points to the correct op + self.assertRegex(assem_op, fr"--> {opname}") # Make sure when lasti points to cache, it shows the same disassembly self.assertEqual(assem_op, assem_cache) @@ -1637,205 +1605,212 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=47, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=48, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=44, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=45, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=49, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=88, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=59, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=86, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to 88', offset=24, start_offset=24, starts_line=False, line_number=3, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to 68', offset=60, start_offset=60, starts_line=False, line_number=5, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to 24', offset=64, start_offset=64, starts_line=True, line_number=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to 84', offset=76, start_offset=76, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to 24', offset=80, start_offset=80, starts_line=False, line_number=7, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=12, argval=112, argrepr='to 112', offset=86, start_offset=86, starts_line=False, line_number=8, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=90, start_offset=90, starts_line=True, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=100, start_offset=100, starts_line=False, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=102, start_offset=102, starts_line=False, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=112, start_offset=112, starts_line=True, line_number=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=114, start_offset=114, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=206, argrepr='to 206', offset=122, start_offset=122, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=126, start_offset=126, starts_line=True, line_number=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=136, start_offset=136, starts_line=False, line_number=12, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=138, start_offset=138, starts_line=False, line_number=12, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=148, start_offset=148, starts_line=True, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=150, start_offset=150, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=152, start_offset=152, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=False, line_number=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=160, start_offset=160, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=162, start_offset=162, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=174, argrepr='to 174', offset=166, start_offset=166, starts_line=False, line_number=14, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=112, argrepr='to 112', offset=170, start_offset=170, starts_line=True, line_number=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=176, start_offset=176, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=178, start_offset=178, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=188, argrepr='to 188', offset=182, start_offset=182, starts_line=False, line_number=16, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=228, argrepr='to 228', offset=186, start_offset=186, starts_line=True, line_number=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=188, start_offset=188, starts_line=True, line_number=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=190, start_offset=190, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=206, argrepr='to 206', offset=198, start_offset=198, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=126, argrepr='to 126', offset=202, start_offset=202, starts_line=False, line_number=11, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=206, start_offset=206, starts_line=True, line_number=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=216, start_offset=216, starts_line=False, line_number=19, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=218, start_offset=218, starts_line=False, line_number=19, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=True, line_number=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=230, start_offset=230, starts_line=True, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=232, start_offset=232, starts_line=False, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=234, start_offset=234, starts_line=False, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=240, start_offset=240, starts_line=True, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=242, start_offset=242, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=244, start_offset=244, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=246, start_offset=246, starts_line=True, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=256, start_offset=256, starts_line=False, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=268, start_offset=268, starts_line=True, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=274, start_offset=274, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=282, start_offset=282, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=284, start_offset=284, starts_line=True, line_number=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=294, start_offset=294, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=306, start_offset=306, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=308, start_offset=308, starts_line=True, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=326, argrepr='to 326', offset=320, start_offset=320, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=324, start_offset=324, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=27, argval=284, argrepr='to 284', offset=334, start_offset=334, starts_line=False, line_number=25, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=392, argrepr='to 392', offset=358, start_offset=358, starts_line=False, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=54, argval=284, argrepr='to 284', offset=388, start_offset=388, starts_line=False, line_number=23, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=394, start_offset=394, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=402, start_offset=402, starts_line=True, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=412, start_offset=412, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=28, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=69, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=42, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=42, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=1, argval=1, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=1, argval=1, argrepr='__exit__', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=91, arg=0, argval=0, argrepr='__enter__', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=0, argval=0, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=264, start_offset=264, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=274, start_offset=274, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=3, argval=3, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=302, start_offset=302, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=41, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=1, argval=344, argrepr='to L12', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='RERAISE', opcode=101, arg=2, argval=2, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=27, argval=302, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=356, start_offset=356, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=364, start_offset=364, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=14, argval=408, argrepr='to L13', offset=376, start_offset=376, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=382, start_offset=382, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=53, argval=302, argrepr='to L11', offset=406, start_offset=406, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=416, start_offset=416, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=418, start_offset=418, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=438, start_offset=438, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=442, start_offset=442, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=444, start_offset=444, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, is_jump_target=False), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), + Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] class InstructionTestCase(BytecodeTestCase): def assertInstructionsEqual(self, instrs_1, instrs_2, /): - instrs_1 = [instr_1._replace(positions=None) for instr_1 in instrs_1] - instrs_2 = [instr_2._replace(positions=None) for instr_2 in instrs_2] + instrs_1 = [instr_1._replace(positions=None, cache_info=None) for instr_1 in instrs_1] + instrs_2 = [instr_2._replace(positions=None, cache_info=None) for instr_2 in instrs_2] self.assertEqual(instrs_1, instrs_2) class InstructionTests(InstructionTestCase): @@ -1844,6 +1819,12 @@ def __init__(self, *args): super().__init__(*args) self.maxDiff = None + def test_instruction_str(self): + # smoke test for __str__ + instrs = dis.get_instructions(simple) + for instr in instrs: + str(instr) + def test_default_first_line(self): actual = dis.get_instructions(simple) self.assertInstructionsEqual(list(actual), expected_opinfo_simple) @@ -1943,23 +1924,35 @@ def roots(a, b, c): instruction.positions.col_offset, instruction.positions.end_col_offset, ) - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( code, adaptive=adaptive, show_caches=show_caches - ) + ), show_caches=show_caches) ] self.assertEqual(co_positions, dis_positions) def test_oparg_alias(self): instruction = Instruction(opname="NOP", opcode=dis.opmap["NOP"], arg=None, argval=None, - argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, positions=None) self.assertEqual(instruction.arg, instruction.oparg) + def test_show_caches_with_label(self): + def f(x, y, z): + if x: + res = y + else: + res = z + return res + + output = io.StringIO() + dis.dis(f.__code__, file=output, show_caches=True) + self.assertIn("L1:", output.getvalue()) + def test_baseopname_and_baseopcode(self): # Standard instructions for name, code in dis.opmap.items(): instruction = Instruction(opname=name, opcode=code, arg=None, argval=None, argrepr='', offset=0, - start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None) + start_offset=0, starts_line=True, line_number=1, label=None, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode self.assertIsNotNone(baseopname) @@ -1970,7 +1963,7 @@ def test_baseopname_and_baseopcode(self): # Specialized instructions for name in opcode._specialized_opmap: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', - offset=0, start_offset=0, starts_line=True, line_number=1, is_jump_target=False, positions=None) + offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode self.assertIn(name, opcode._specializations[baseopname]) @@ -1979,41 +1972,85 @@ def test_baseopname_and_baseopcode(self): def test_jump_target(self): # Non-jump instructions should return None instruction = Instruction(opname="NOP", opcode=dis.opmap["NOP"], arg=None, argval=None, - argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, positions=None) self.assertIsNone(instruction.jump_target) delta = 100 instruction = Instruction(opname="JUMP_FORWARD", opcode=dis.opmap["JUMP_FORWARD"], arg=delta, argval=delta, - argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, positions=None) self.assertEqual(10 + 2 + 100*2, instruction.jump_target) # Test negative deltas instruction = Instruction(opname="JUMP_BACKWARD", opcode=dis.opmap["JUMP_BACKWARD"], arg=delta, argval=delta, - argrepr='', offset=200, start_offset=200, starts_line=True, line_number=1, is_jump_target=False, + argrepr='', offset=200, start_offset=200, starts_line=True, line_number=1, label=None, positions=None) self.assertEqual(200 + 2 - 100*2 + 2*1, instruction.jump_target) # Make sure cache entries are handled instruction = Instruction(opname="SEND", opcode=dis.opmap["SEND"], arg=delta, argval=delta, - argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, is_jump_target=False, + argrepr='', offset=10, start_offset=10, starts_line=True, line_number=1, label=None, positions=None) self.assertEqual(10 + 2 + 1*2 + 100*2, instruction.jump_target) + def test_argval_argrepr(self): + def f(opcode, oparg, offset, *init_args): + arg_resolver = dis.ArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + + offset = 42 + co_consts = (0, 1, 2, 3) + names = {1: 'a', 2: 'b'} + varname_from_oparg = lambda i : names[i] + labels_map = {24: 1} + args = (offset, co_consts, names, varname_from_oparg, labels_map) + self.assertEqual(f(opcode.opmap["POP_TOP"], None, *args), (None, '')) + self.assertEqual(f(opcode.opmap["LOAD_CONST"], 1, *args), (1, '1')) + self.assertEqual(f(opcode.opmap["LOAD_GLOBAL"], 2, *args), ('a', 'a')) + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 11, *args), (24, 'to L1')) + self.assertEqual(f(opcode.opmap["COMPARE_OP"], 3, *args), ('<', '<')) + self.assertEqual(f(opcode.opmap["SET_FUNCTION_ATTRIBUTE"], 2, *args), (2, 'kwdefaults')) + self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) + self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + + def test_custom_arg_resolver(self): + class MyArgResolver(dis.ArgResolver): + def offset_from_jump_arg(self, op, arg, offset): + return arg + 1 + + def get_label_for_offset(self, offset): + return 2 * offset + + def f(opcode, oparg, offset, *init_args): + arg_resolver = MyArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + offset = 42 + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to L4')) + self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to L6')) + + + def get_instructions(self, code): + return dis._get_instructions_bytes(code) + def test_start_offset(self): # When no extended args are present, # start_offset should be equal to offset + instructions = list(dis.Bytecode(_f)) for instruction in instructions: self.assertEqual(instruction.offset, instruction.start_offset) + def last_item(iterable): + return functools.reduce(lambda a, b : b, iterable) + code = bytes([ opcode.opmap["LOAD_FAST"], 0x00, opcode.opmap["EXTENDED_ARG"], 0x01, opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + labels_map = dis._make_labels_map(code) + jump = last_item(self.get_instructions(code)) self.assertEqual(4, jump.offset) self.assertEqual(2, jump.start_offset) @@ -2025,7 +2062,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + jump = last_item(self.get_instructions(code)) self.assertEqual(8, jump.offset) self.assertEqual(2, jump.start_offset) @@ -2040,7 +2077,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) # 1st jump self.assertEqual(4, instructions[2].offset) self.assertEqual(2, instructions[2].start_offset) @@ -2061,7 +2098,7 @@ def test_cache_offset_and_end_offset(self): opcode.opmap["CACHE"], 0x00, opcode.opmap["CACHE"], 0x00 ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) self.assertEqual(2, instructions[0].cache_offset) self.assertEqual(10, instructions[0].end_offset) self.assertEqual(12, instructions[1].cache_offset) @@ -2132,7 +2169,7 @@ def test_from_traceback_dis(self): self.maxDiff = None tb = get_tb() b = dis.Bytecode.from_traceback(tb) - self.assertEqual(self.strip_offsets(b.dis()), dis_traceback) + self.assertEqual(b.dis(), dis_traceback) @requires_debug_ranges() def test_bytecode_co_positions(self): @@ -2254,6 +2291,31 @@ def get_disassembly(self, tb): dis.distb(tb, file=output) return output.getvalue() +def _unroll_caches_as_Instructions(instrs, show_caches=False): + # Cache entries are no longer reported by dis as fake instructions, + # but some tests assume that do. We should rewrite the tests to assume + # the new API, but it will be clearer to keep the tests working as + # before and do that in a separate PR. + + for instr in instrs: + yield instr + if not show_caches: + continue + + offset = instr.offset + for name, size, data in (instr.cache_info or ()): + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + + yield Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_doctest/__init__.py b/Lib/test/test_doctest/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_doctest/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_doctest/decorator_mod.py b/Lib/test/test_doctest/decorator_mod.py new file mode 100644 index 00000000000000..9f106888411202 --- /dev/null +++ b/Lib/test/test_doctest/decorator_mod.py @@ -0,0 +1,10 @@ +# This module is used in `doctest_lineno.py`. +import functools + + +def decorator(f): + @functools.wraps(f) + def inner(): + return f() + + return inner diff --git a/Lib/test/doctest_aliases.py b/Lib/test/test_doctest/doctest_aliases.py similarity index 100% rename from Lib/test/doctest_aliases.py rename to Lib/test/test_doctest/doctest_aliases.py diff --git a/Lib/test/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py similarity index 62% rename from Lib/test/doctest_lineno.py rename to Lib/test/test_doctest/doctest_lineno.py index 729a68aceaa990..0dbcd9a11eaba2 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -49,5 +49,30 @@ def method_with_doctest(self): 'method_with_doctest' """ + @classmethod + def classmethod_with_doctest(cls): + """ + This has a doctest! + >>> MethodWrapper.classmethod_with_doctest.__name__ + 'classmethod_with_doctest' + """ + + @property + def property_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.property_with_doctest.__name__ + 'property_with_doctest' + """ + # https://github.com/python/cpython/issues/99433 str_wrapper = object().__str__ + + +# https://github.com/python/cpython/issues/115392 +from test.test_doctest.decorator_mod import decorator + +@decorator +@decorator +def func_with_docstring_wrapped(): + """Some unrelated info.""" diff --git a/Lib/test/sample_doctest.py b/Lib/test/test_doctest/sample_doctest.py similarity index 91% rename from Lib/test/sample_doctest.py rename to Lib/test/test_doctest/sample_doctest.py index 89eb5cb7cf1d97..049f737a0a44ac 100644 --- a/Lib/test/sample_doctest.py +++ b/Lib/test/test_doctest/sample_doctest.py @@ -32,8 +32,8 @@ def bar(): def test_silly_setup(): """ - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup True """ diff --git a/Lib/test/sample_doctest_no_docstrings.py b/Lib/test/test_doctest/sample_doctest_no_docstrings.py similarity index 100% rename from Lib/test/sample_doctest_no_docstrings.py rename to Lib/test/test_doctest/sample_doctest_no_docstrings.py diff --git a/Lib/test/sample_doctest_no_doctests.py b/Lib/test/test_doctest/sample_doctest_no_doctests.py similarity index 100% rename from Lib/test/sample_doctest_no_doctests.py rename to Lib/test/test_doctest/sample_doctest_no_doctests.py diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000000..1b83dec1f8c4dc --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest/test_doctest.py similarity index 86% rename from Lib/test/test_doctest.py rename to Lib/test/test_doctest/test_doctest.py index 6e12e82a7a0084..3e93a3bba283c2 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -17,8 +17,12 @@ import contextlib -if not support.has_subprocess_support: - raise unittest.SkipTest("test_CLI requires subprocess support.") +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator # NOTE: There are some additional tests relating to interaction with @@ -77,6 +81,15 @@ def get(self): """ return self.val + def setter(self, val): + """ + >>> s = SampleClass(-5) + >>> s.setter(1) + >>> print(s.val) + 1 + """ + self.val = val + def a_staticmethod(v): """ >>> print(SampleClass.a_staticmethod(10)) @@ -95,22 +108,13 @@ def a_classmethod(cls, v): return v+2 a_classmethod = classmethod(a_classmethod) - a_property = property(get, doc=""" + a_property = property(get, setter, doc=""" >>> print(SampleClass(22).a_property) 22 """) a_class_attribute = 42 - @classmethod - @property - def a_classmethod_property(cls): - """ - >>> print(SampleClass.a_classmethod_property) - 42 - """ - return cls.a_class_attribute - @functools.cached_property def a_cached_property(self): """ @@ -165,25 +169,6 @@ def get(self): """ return self.val -###################################################################### -## Fake stdin (for testing interactive debugging) -###################################################################### - -class _FakeInput: - """ - A fake input stream for pdb's interactive debugger. Whenever a - line is read, print it (to simulate the user typing it), and then - return it. The set of lines to return is specified in the - constructor; they should not have trailing newlines. - """ - def __init__(self, lines): - self.lines = lines - - def readline(self): - line = self.lines.pop(0) - print(line) - return line+'\n' - ###################################################################### ## Test Cases ###################################################################### @@ -422,6 +407,23 @@ def test_DocTest(): r""" False >>> test != other_test True + >>> test < other_test + False + >>> other_test < test + True + +Test comparison with lineno None on one side + + >>> no_lineno = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', None) + >>> test.lineno is None + False + >>> no_lineno.lineno is None + True + >>> test < no_lineno + False + >>> no_lineno < test + True Compare `DocTestCase`: @@ -460,14 +462,14 @@ def basics(): r""" We'll simulate a __file__ attr that ends in pyc: - >>> import test.test_doctest - >>> old = test.test_doctest.__file__ - >>> test.test_doctest.__file__ = 'test_doctest.pyc' + >>> from test.test_doctest import test_doctest + >>> old = test_doctest.__file__ + >>> test_doctest.__file__ = 'test_doctest.pyc' >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -475,7 +477,7 @@ def basics(): r""" >>> tests[0].filename # doctest: +ELLIPSIS '...test_doctest.py' - >>> test.test_doctest.__file__ = old + >>> test_doctest.__file__ = old >>> e = tests[0].examples[0] @@ -525,11 +527,11 @@ def basics(): r""" 1 SampleClass.__init__ 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod - 1 SampleClass.a_classmethod_property 1 SampleClass.a_property 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter New-style classes are also supported: @@ -569,10 +571,10 @@ def basics(): r""" ... 'c': triple}}) >>> finder = doctest.DocTestFinder() - >>> # Use module=test.test_doctest, to prevent doctest from + >>> # Use module=test_doctest, to prevent doctest from >>> # ignoring the objects since they weren't defined in m. - >>> import test.test_doctest - >>> tests = finder.find(m, module=test.test_doctest) + >>> from test.test_doctest import test_doctest + >>> tests = finder.find(m, module=test_doctest) >>> for t in tests: ... print('%2s %s' % (len(t.examples), t.name)) 1 some_module @@ -582,28 +584,42 @@ def basics(): r""" 1 some_module.SampleClass.__init__ 1 some_module.SampleClass.a_cached_property 2 some_module.SampleClass.a_classmethod - 1 some_module.SampleClass.a_classmethod_property 1 some_module.SampleClass.a_property 1 some_module.SampleClass.a_staticmethod 1 some_module.SampleClass.double 1 some_module.SampleClass.get + 3 some_module.SampleClass.setter 1 some_module.__test__.c 2 some_module.__test__.d 1 some_module.sample_func +However, doctest will ignore imported objects from other modules +(without proper `module=`): + + >>> import types + >>> m = types.ModuleType('poluted_namespace') + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... }) + + >>> finder = doctest.DocTestFinder() + >>> finder.find(m) + [] + Duplicate Removal ~~~~~~~~~~~~~~~~~ If a single object is listed twice (under different names), then tests will only be generated for it once: - >>> from test import doctest_aliases + >>> from test.test_doctest import doctest_aliases >>> assert doctest_aliases.TwoNames.f >>> assert doctest_aliases.TwoNames.g >>> tests = excl_empty_finder.find(doctest_aliases) >>> print(len(tests)) 2 >>> print(tests[0].name) - test.doctest_aliases.TwoNames + test.test_doctest.doctest_aliases.TwoNames TwoNames.f and TwoNames.g are bound to the same object. We can't guess which will be found in doctest's traversal of @@ -625,11 +641,11 @@ def basics(): r""" 1 SampleClass.__init__ 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod - 1 SampleClass.a_classmethod_property 1 SampleClass.a_property 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter By default, that excluded objects with no doctests. exclude_empty=False tells it to include (empty) tests for objects with no doctests. This feature @@ -647,31 +663,34 @@ def basics(): r""" 1 SampleClass.__init__ 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod - 1 SampleClass.a_classmethod_property 1 SampleClass.a_property 1 SampleClass.a_staticmethod 1 SampleClass.double 1 SampleClass.get + 3 SampleClass.setter When used with `exclude_empty=False` we are also interested in line numbers of doctests that are empty. It used to be broken for quite some time until `bpo-28249`. - >>> from test import doctest_lineno + >>> from test.test_doctest import doctest_lineno >>> tests = doctest.DocTestFinder(exclude_empty=False).find(doctest_lineno) >>> for t in tests: ... print('%5s %s' % (t.lineno, t.name)) - None test.doctest_lineno - 22 test.doctest_lineno.ClassWithDocstring - 30 test.doctest_lineno.ClassWithDoctest - None test.doctest_lineno.ClassWithoutDocstring - None test.doctest_lineno.MethodWrapper - 39 test.doctest_lineno.MethodWrapper.method_with_docstring - 45 test.doctest_lineno.MethodWrapper.method_with_doctest - None test.doctest_lineno.MethodWrapper.method_without_docstring - 4 test.doctest_lineno.func_with_docstring - 12 test.doctest_lineno.func_with_doctest - None test.doctest_lineno.func_without_docstring + None test.test_doctest.doctest_lineno + 22 test.test_doctest.doctest_lineno.ClassWithDocstring + 30 test.test_doctest.doctest_lineno.ClassWithDoctest + None test.test_doctest.doctest_lineno.ClassWithoutDocstring + None test.test_doctest.doctest_lineno.MethodWrapper + 53 test.test_doctest.doctest_lineno.MethodWrapper.classmethod_with_doctest + 39 test.test_doctest.doctest_lineno.MethodWrapper.method_with_docstring + 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest + None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 4 test.test_doctest.doctest_lineno.func_with_docstring + 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped + 12 test.test_doctest.doctest_lineno.func_with_doctest + None test.test_doctest.doctest_lineno.func_without_docstring Turning off Recursion ~~~~~~~~~~~~~~~~~~~~~ @@ -872,6 +891,10 @@ def basics(): r""" DocTestRunner is used to run DocTest test cases, and to accumulate statistics. Here's a simple DocTest case we can use: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 @@ -926,6 +949,8 @@ def basics(): r""" 6 ok TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def verbose_flag(): r""" The `verbose` flag makes the test runner generate more detailed @@ -1001,12 +1026,16 @@ def exceptions(): r""" lines between the first line and the type/value may be omitted or replaced with any other string: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 ... >>> print(x//0) ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1023,7 +1052,7 @@ def exceptions(): r""" ... >>> print('pre-exception output', x//0) ... pre-exception output ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1034,7 +1063,7 @@ def exceptions(): r""" print('pre-exception output', x//0) Exception raised: ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=2) Exception messages may contain newlines: @@ -1229,8 +1258,10 @@ def exceptions(): r""" Exception raised: Traceback (most recent call last): ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ def displayhook(): r""" Test that changing sys.displayhook doesn't matter for doctest. @@ -1272,6 +1303,10 @@ def optionflags(): r""" The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False and 1/0: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... '>>> True\n1\n' @@ -1691,6 +1726,7 @@ def optionflags(): r""" Clean up. >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] + >>> _colorize.COLORIZE = save_colorize """ @@ -1701,6 +1737,10 @@ def option_directives(): r""" single example. To turn an option on for an example, follow that example with a comment of the form ``# doctest: +OPTION``: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): r''' ... >>> print(list(range(10))) # should fail: no ellipsis ... [0, 1, ..., 9] @@ -1908,6 +1948,8 @@ def option_directives(): r""" >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) Traceback (most recent call last): ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' + + >>> _colorize.COLORIZE = save_colorize """ def test_testsource(): r""" @@ -1918,9 +1960,9 @@ def test_testsource(): r""" example code is converted to regular Python code. The surrounding words and expected output are converted to comments: - >>> import test.test_doctest - >>> name = 'test.test_doctest.sample_func' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> from test.test_doctest import test_doctest + >>> name = 'test.test_doctest.test_doctest.sample_func' + >>> print(doctest.testsource(test_doctest, name)) # Blah blah # print(sample_func(22)) @@ -1930,8 +1972,8 @@ def test_testsource(): r""" # Yee ha! - >>> name = 'test.test_doctest.SampleNewStyleClass' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> name = 'test.test_doctest.test_doctest.SampleNewStyleClass' + >>> print(doctest.testsource(test_doctest, name)) print('1\n2\n3') # Expected: ## 1 @@ -1939,8 +1981,8 @@ def test_testsource(): r""" ## 3 - >>> name = 'test.test_doctest.SampleClass.a_classmethod' - >>> print(doctest.testsource(test.test_doctest, name)) + >>> name = 'test.test_doctest.test_doctest.SampleClass.a_classmethod' + >>> print(doctest.testsource(test_doctest, name)) print(SampleClass.a_classmethod(10)) # Expected: ## 12 @@ -1962,8 +2004,9 @@ def test_debug(): r""" Create some fake stdin input, to feed to the debugger: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput(['next', 'print(x)', 'continue']) + >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) Run the debugger on the docstring, and then restore sys.stdin. @@ -1991,6 +2034,10 @@ def test_pdb_set_trace(): with a version that restores stdout. This is necessary for you to see debugger output. + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> x = 42 ... >>> raise Exception('clé') @@ -2005,16 +2052,16 @@ def test_pdb_set_trace(): To demonstrate this, we'll create a fake standard input that captures our debugger input: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'print(x)', # print data defined by the example ... 'continue', # stop debugging ... '']) >>> try: runner.run(test) ... finally: sys.stdin = real_stdin - --Return-- - > (1)()->None + > (1)() -> import pdb; pdb.set_trace() (Pdb) print(x) 42 @@ -2033,7 +2080,7 @@ def test_pdb_set_trace(): ... ''' >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ ... 'print(y)', # print data defined in the function ... 'up', # out of function ... 'print(x)', # print data defined by the example @@ -2044,8 +2091,7 @@ def test_pdb_set_trace(): ... runner.run(test) ... finally: ... sys.stdin = real_stdin - --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace() -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2070,7 +2116,8 @@ def test_pdb_set_trace(): ... ''' >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ + ... 'step', # return event of g ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2081,6 +2128,9 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step --Return-- > (3)g()->None -> import pdb; pdb.set_trace() @@ -2113,6 +2163,8 @@ def test_pdb_set_trace(): Got: 9 TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def test_pdb_set_trace_nested(): @@ -2141,8 +2193,10 @@ def test_pdb_set_trace_nested(): >>> parser = doctest.DocTestParser() >>> runner = doctest.DocTestRunner(verbose=False) >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ + >>> sys.stdin = FakeInput([ + ... 'step', ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2156,39 +2210,42 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF - > (5)calls_set_trace() + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step + > (5)calls_set_trace() -> self.f1() (Pdb) print(y) 1 (Pdb) step --Call-- - > (7)f1() + > (7)f1() -> def f1(self): (Pdb) step - > (8)f1() + > (8)f1() -> x = 1 (Pdb) step - > (9)f1() + > (9)f1() -> self.f2() (Pdb) step --Call-- - > (11)f2() + > (11)f2() -> def f2(self): (Pdb) step - > (12)f2() + > (12)f2() -> z = 1 (Pdb) step - > (13)f2() + > (13)f2() -> z = 2 (Pdb) print(z) 1 (Pdb) up - > (9)f1() + > (9)f1() -> self.f2() (Pdb) print(x) 1 (Pdb) up - > (5)calls_set_trace() + > (5)calls_set_trace() -> self.f1() (Pdb) print(y) 1 @@ -2208,39 +2265,49 @@ def test_DocTestSuite(): by passing a module object: >>> import unittest - >>> import test.sample_doctest - >>> suite = doctest.DocTestSuite(test.sample_doctest) + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) We can also supply the module by name: - >>> suite = doctest.DocTestSuite('test.sample_doctest') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') >>> suite.run(unittest.TestResult()) The module need not contain any doctest examples: - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_doctests') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_doctests') >>> suite.run(unittest.TestResult()) The module need not contain any docstrings either: - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_docstrings') + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings') >>> suite.run(unittest.TestResult()) + If all examples in a docstring are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 2 + We can use the current module: - >>> suite = test.sample_doctest.test_suite() + >>> suite = test.test_doctest.sample_doctest.test_suite() >>> suite.run(unittest.TestResult()) We can also provide a DocTestFinder: >>> finder = doctest.DocTestFinder() - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... test_finder=finder) >>> suite.run(unittest.TestResult()) @@ -2248,7 +2315,7 @@ def test_DocTestSuite(): The DocTestFinder need not return any tests: >>> finder = doctest.DocTestFinder() - >>> suite = doctest.DocTestSuite('test.sample_doctest_no_docstrings', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings', ... test_finder=finder) >>> suite.run(unittest.TestResult()) @@ -2257,14 +2324,14 @@ def test_DocTestSuite(): used instead of the module globals. Here we'll pass an empty globals, triggering an extra error: - >>> suite = doctest.DocTestSuite('test.sample_doctest', globs={}) + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) >>> suite.run(unittest.TestResult()) Alternatively, we can provide extra globals. Here we'll make an error go away by providing an extra global variable: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... extraglobs={'y': 1}) >>> suite.run(unittest.TestResult()) @@ -2272,7 +2339,7 @@ def test_DocTestSuite(): You can pass option flags. Here we'll cause an extra error by disabling the blank-line feature: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> suite.run(unittest.TestResult()) @@ -2280,27 +2347,27 @@ def test_DocTestSuite(): You can supply setUp and tearDown functions: >>> def setUp(t): - ... import test.test_doctest - ... test.test_doctest.sillySetup = True + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True >>> def tearDown(t): - ... import test.test_doctest - ... del test.test_doctest.sillySetup + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup Here, we installed a silly variable that the test expects: - >>> suite = doctest.DocTestSuite('test.sample_doctest', + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) But the tearDown restores sanity: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup Traceback (most recent call last): ... - AttributeError: module 'test.test_doctest' has no attribute 'sillySetup' + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' The setUp and tearDown functions are passed test objects. Here we'll use the setUp function to supply the missing variable y: @@ -2308,7 +2375,7 @@ def test_DocTestSuite(): >>> def setUp(test): ... test.globs['y'] = 1 - >>> suite = doctest.DocTestSuite('test.sample_doctest', setUp=setUp) + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) >>> suite.run(unittest.TestResult()) @@ -2339,7 +2406,7 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest2.txt', ... 'test_doctest4.txt', - ... package='test') + ... package='test.test_doctest') >>> suite.run(unittest.TestResult()) @@ -2355,7 +2422,7 @@ def test_DocFileSuite(): ... suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest2.txt', ... 'test_doctest4.txt', - ... package='test') + ... package='test.test_doctest') ... suite.run(unittest.TestResult()) ... finally: ... if added_loader: @@ -2365,16 +2432,17 @@ def test_DocFileSuite(): '/' should be used as a path separator. It will be converted to a native separator at run time: - >>> suite = doctest.DocFileSuite('../test/test_doctest.txt') + >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') >>> suite.run(unittest.TestResult()) If DocFileSuite is used from an interactive session, then files are resolved relative to the directory of sys.argv[0]: - >>> import types, os.path, test.test_doctest + >>> import types, os.path + >>> from test.test_doctest import test_doctest >>> save_argv = sys.argv - >>> sys.argv = [test.test_doctest.__file__] + >>> sys.argv = [test_doctest.__file__] >>> suite = doctest.DocFileSuite('test_doctest.txt', ... package=types.ModuleType('__main__')) >>> sys.argv = save_argv @@ -2384,7 +2452,7 @@ def test_DocFileSuite(): working directory): >>> # Get the absolute path of the test package. - >>> test_doctest_path = os.path.abspath(test.test_doctest.__file__) + >>> test_doctest_path = os.path.abspath(test_doctest.__file__) >>> test_pkg_path = os.path.split(test_doctest_path)[0] >>> # Use it to find the absolute path of test_doctest.txt. @@ -2401,6 +2469,18 @@ def test_DocFileSuite(): Traceback (most recent call last): ValueError: Package may only be specified for module-relative paths. + If all examples in a file are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest4.txt', + ... 'test_doctest_skip.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 1 + You can specify initial global variables: >>> suite = doctest.DocFileSuite('test_doctest.txt', @@ -2424,12 +2504,12 @@ def test_DocFileSuite(): And, you can provide setUp and tearDown functions: >>> def setUp(t): - ... import test.test_doctest - ... test.test_doctest.sillySetup = True + ... from test.test_doctest import test_doctest + ... test_doctest.sillySetup = True >>> def tearDown(t): - ... import test.test_doctest - ... del test.test_doctest.sillySetup + ... from test.test_doctest import test_doctest + ... del test_doctest.sillySetup Here, we installed a silly variable that the test expects: @@ -2442,11 +2522,11 @@ def test_DocFileSuite(): But the tearDown restores sanity: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> from test.test_doctest import test_doctest + >>> test_doctest.sillySetup Traceback (most recent call last): ... - AttributeError: module 'test.test_doctest' has no attribute 'sillySetup' + AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' The setUp and tearDown functions are passed test objects. Here, we'll use a setUp function to set the favorite color in @@ -2502,7 +2582,7 @@ def __call__(self, *args, **kwargs): self.func(*args, **kwargs) @Wrapper -def test_look_in_unwrapped(): +def wrapped(): """ Docstrings in wrapped functions must be detected as well. @@ -2510,6 +2590,35 @@ def test_look_in_unwrapped(): 'one other test' """ +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + def test_unittest_reportflags(): """Default unittest reporting flags can be set to control reporting @@ -2595,7 +2704,11 @@ def test_testfile(): r""" called with the name of a file, which is taken to be relative to the calling module. The return value is (#failures, #tests). -We don't want `-v` in sys.argv for these tests. +We don't want color or `-v` in sys.argv for these tests. + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> save_argv = sys.argv >>> if '-v' in sys.argv: @@ -2611,9 +2724,9 @@ def test_testfile(): r""" ... NameError: name 'favorite_color' is not defined ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2640,9 +2753,9 @@ def test_testfile(): r""" Got: 'red' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2672,10 +2785,10 @@ def test_testfile(): r""" b ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. @@ -2732,7 +2845,7 @@ def test_testfile(): r""" ********************************************************************** ... ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 2 in test_doctest4.txt ***Test Failed*** 2 failures. TestResults(failed=2, attempted=2) @@ -2755,14 +2868,15 @@ def test_testfile(): r""" Expecting: 'b\u0105r' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest4.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv + >>> _colorize.COLORIZE = save_colorize """ class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): @@ -2900,6 +3014,10 @@ def test_testmod(): r""" def test_unicode(): """ Check doctest with a non-ascii filename: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> raise Exception('clé') ... ''' @@ -2918,12 +3036,19 @@ def test_unicode(): """ Traceback (most recent call last): File ... exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in raise Exception('clé') Exception: clé TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ + +@doctest_skip_if(not support.has_subprocess_support) def test_CLI(): r""" The doctest module can be used to run doctests against an arbitrary file. These tests test this CLI functionality. @@ -2977,10 +3102,10 @@ def test_CLI(): r""" Expecting: 'a' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile.doc - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. Now we'll write a couple files, one with three tests, the other a python module @@ -3054,7 +3179,7 @@ def test_CLI(): r""" Got: 'ajkml' ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 3 in myfile.doc ***Test Failed*** 2 failures. @@ -3081,9 +3206,9 @@ def test_CLI(): r""" Got: 'abcdef' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in myfile.doc - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. The fifth test uses verbose with the two options, so we should get verbose success output for the tests in both files: @@ -3106,10 +3231,10 @@ def test_CLI(): r""" Expecting: 'a...l' ok - 1 items passed all tests: + 1 item passed all tests: 3 tests in myfile.doc - 3 tests in 1 items. - 3 passed and 0 failed. + 3 tests in 1 item. + 3 passed. Test passed. Trying: 1 + 1 @@ -3121,12 +3246,12 @@ def test_CLI(): r""" Expecting: 'abc def' ok - 1 items had no tests: + 1 item had no tests: myfile2 - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile2.test_func 2 tests in 2 items. - 2 passed and 0 failed. + 2 passed. Test passed. We should also check some typical error cases. @@ -3203,8 +3328,8 @@ def test_run_doctestsuite_multiple_times(): http://bugs.python.org/issue9736 >>> import unittest - >>> import test.sample_doctest - >>> suite = doctest.DocTestSuite(test.sample_doctest) + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) >>> suite.run(unittest.TestResult()) @@ -3212,25 +3337,185 @@ def test_run_doctestsuite_multiple_times(): """ +def test_exception_with_note(note): + """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> test_exception_with_note('Note') + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('Note') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('''Note + ... multiline + ... example''') + Traceback (most recent call last): + ValueError: Text + Note + multiline + example + + Different note will fail the test: + + >>> def f(x): + ... r''' + ... >>> exc = ValueError('message') + ... >>> exc.add_note('note') + ... >>> raise exc + ... Traceback (most recent call last): + ... ValueError: message + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 5, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + ValueError: message + wrong note + Got: + Traceback (most recent call last): + ... + ValueError: message + note + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + exc = ValueError('Text') + exc.add_note(note) + raise exc + + +def test_exception_with_multiple_notes(): + """ + >>> test_exception_with_multiple_notes() + Traceback (most recent call last): + ... + ValueError: Text + One + Two + """ + exc = ValueError('Text') + exc.add_note('One') + exc.add_note('Two') + raise exc + + +def test_syntax_error_with_note(cls, multiline=False): + """ + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + File "x.py", line 23 + bad syntax + SyntaxError: error + Note + + >>> test_syntax_error_with_note(IndentationError) + Traceback (most recent call last): + ... + IndentationError: error + Note + + >>> test_syntax_error_with_note(TabError, multiline=True) + Traceback (most recent call last): + ... + TabError: error + Note + Line + """ + exc = cls("error", ("x.py", 23, None, "bad syntax")) + exc.add_note('Note\nLine' if multiline else 'Note') + raise exc + + +def test_syntax_error_subclass_from_stdlib(): + """ + `ParseError` is a subclass of `SyntaxError`, but it is not a builtin: + + >>> test_syntax_error_subclass_from_stdlib() + Traceback (most recent call last): + ... + xml.etree.ElementTree.ParseError: error + error + Note + Line + """ + from xml.etree.ElementTree import ParseError + exc = ParseError("error\nerror") + exc.add_note('Note\nLine') + raise exc + + +def test_syntax_error_with_incorrect_expected_note(): + """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... r''' + ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + ... >>> exc.add_note('note1') + ... >>> exc.add_note('note2') + ... >>> raise exc + ... Traceback (most recent call last): + ... SyntaxError: error + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + SyntaxError: error + wrong note + Got: + Traceback (most recent call last): + ... + SyntaxError: error + note1 + note2 + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(doctest)) tests.addTest(doctest.DocTestSuite()) return tests -def test_coverage(coverdir): - trace = import_helper.import_module('trace') - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], - trace=0, count=1) - tracer.run('test_main()') - r = tracer.results() - print('Writing coverage results...') - r.write_results(show_missing=True, summary=True, - coverdir=coverdir) - - if __name__ == '__main__': - if '-c' in sys.argv: - test_coverage('/tmp/doctest.cover') - else: - unittest.main() + unittest.main(module='test.test_doctest.test_doctest') diff --git a/Lib/test/test_doctest.txt b/Lib/test/test_doctest/test_doctest.txt similarity index 100% rename from Lib/test/test_doctest.txt rename to Lib/test/test_doctest/test_doctest.txt diff --git a/Lib/test/test_doctest2.py b/Lib/test/test_doctest/test_doctest2.py similarity index 100% rename from Lib/test/test_doctest2.py rename to Lib/test/test_doctest/test_doctest2.py diff --git a/Lib/test/test_doctest2.txt b/Lib/test/test_doctest/test_doctest2.txt similarity index 77% rename from Lib/test/test_doctest2.txt rename to Lib/test/test_doctest/test_doctest2.txt index 2e14856c27d8b3..76dab94a9c0470 100644 --- a/Lib/test/test_doctest2.txt +++ b/Lib/test/test_doctest/test_doctest2.txt @@ -2,8 +2,8 @@ This is a sample doctest in a text file. In this example, we'll rely on some silly setup: - >>> import test.test_doctest - >>> test.test_doctest.sillySetup + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup True This test also has some (random) encoded (utf-8) unicode text: diff --git a/Lib/test/test_doctest3.txt b/Lib/test/test_doctest/test_doctest3.txt similarity index 100% rename from Lib/test/test_doctest3.txt rename to Lib/test/test_doctest/test_doctest3.txt diff --git a/Lib/test/test_doctest4.txt b/Lib/test/test_doctest/test_doctest4.txt similarity index 100% rename from Lib/test/test_doctest4.txt rename to Lib/test/test_doctest/test_doctest4.txt diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000000..f340e2b8141253 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,4 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 0aa3be6a1bde6a..21bece26b893c6 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import swap_item, swap_attr +from test.support import swap_item, swap_attr, is_wasi, Py_DEBUG class RebindBuiltinsTests(unittest.TestCase): @@ -134,6 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 854f2ff009c618..5413319a414a62 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -801,6 +801,10 @@ def test_get_quoted_string_header_ends_in_qcontent(self): self.assertEqual(qs.content, 'bob') self.assertEqual(qs.quoted_value, ' "bob"') + def test_get_quoted_string_cfws_only_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_quoted_string(' (foo) ') + def test_get_quoted_string_no_quoted_string(self): with self.assertRaises(errors.HeaderParseError): parser.get_quoted_string(' (ab) xyz') @@ -1135,6 +1139,10 @@ def test_get_local_part_complex_obsolete_invalid(self): '@python.org') self.assertEqual(local_part.local_part, 'Fred.A.Johnson and dogs') + def test_get_local_part_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part('') + def test_get_local_part_no_part_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_local_part(' (foo) ') @@ -1387,6 +1395,10 @@ def test_get_domain_obsolete(self): '') self.assertEqual(domain.domain, 'example.com') + def test_get_domain_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain("") + def test_get_domain_no_non_cfws_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_domain(" (foo)\t") @@ -1512,6 +1524,10 @@ def test_get_obs_route_no_route_before_end_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com,') + def test_get_obs_route_no_route_before_end_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com, (foo) ') + def test_get_obs_route_no_route_before_special_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) [abc],') @@ -1520,6 +1536,14 @@ def test_get_obs_route_no_route_before_special_raises2(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com [abc],') + def test_get_obs_route_no_domain_after_at_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@') + + def test_get_obs_route_no_domain_after_at_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@example.com, @') + # get_angle_addr def test_get_angle_addr_simple(self): @@ -1646,6 +1670,14 @@ def test_get_angle_addr_ends_at_special(self): self.assertIsNone(angle_addr.route) self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + def test_get_angle_addr_empty_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('') + + def test_get_angle_addr_left_angle_only_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('<') + def test_get_angle_addr_no_angle_raise(self): with self.assertRaises(errors.HeaderParseError): parser.get_angle_addr('(foo) ') @@ -1805,6 +1837,32 @@ def test_get_name_addr_qs_name(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_ending_with_dot_without_space(self): + name_addr = self._test_get_x(parser.get_name_addr, + 'John X.', + 'John X.', + '"John X."', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, 'John X.') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + + def test_get_name_addr_starting_with_dot(self): + name_addr = self._test_get_x(parser.get_name_addr, + '. Doe ', + '. Doe ', + '". Doe" ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, '. Doe') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + def test_get_name_addr_with_route(self): name_addr = self._test_get_x(parser.get_name_addr, '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', @@ -1831,6 +1889,10 @@ def test_get_name_addr_ends_at_special(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr('') + def test_get_name_addr_no_content_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_name_addr(' (foo) ') @@ -2698,6 +2760,35 @@ def test_get_msg_id_no_angle_end(self): ) self.assertEqual(msg_id.token_type, 'msg-id') + def test_get_msg_id_empty_id_left(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("<@domain>") + + def test_get_msg_id_empty_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_no_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_ws_only_local(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "< @domain>", + "< @domain>", + "< @domain>", + [errors.ObsoleteHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + @parameterize @@ -2915,6 +3006,45 @@ def test_ews_combined_before_wrap(self): "mich. And that's\n" " all I'm sayin.\n") + def test_unicode_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=\xa4"), + "=?unknown-8bit?q?=A4?==?utf-8?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=\xa4"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?=C2=A4?=\n") + + def test_ascii_after_unknown_not_combined(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=A4?=abc"), + "=?unknown-8bit?q?=A4?=abc\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=A4?=abc"), + prefix + "=?unknown-8bit?q?=A4?=\n =?utf-8?q?abc?=\n") + + def test_unknown_after_unicode_not_combined(self): + self._test(parser.get_unstructured("\xa4" + "=?unknown-8bit?q?=A4?="), + "=?utf-8?q?=C2=A4?==?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "\xa4=?unknown-8bit?q?=A4?="), + prefix + "=?utf-8?q?=C2=A4?=\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_ascii_not_combined(self): + self._test(parser.get_unstructured("abc" + "=?unknown-8bit?q?=A4?="), + "abc=?unknown-8bit?q?=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "abcd=?unknown-8bit?q?=A4?="), + prefix + "abcd\n =?unknown-8bit?q?=A4?=\n") + + def test_unknown_after_unknown(self): + self._test(parser.get_unstructured("=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + "=?unknown-8bit?q?=C2=A4?=\n") + prefix = "0123456789 "*5 + self._test(parser.get_unstructured(prefix + "=?unknown-8bit?q?=C2?=" + "=?unknown-8bit?q?=A4?="), + prefix + "=?unknown-8bit?q?=C2?=\n =?unknown-8bit?q?=A4?=\n") + # XXX Need test of an encoded word so long that it needs to be wrapped def test_simple_address(self): @@ -2946,6 +3076,19 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + def test_address_list_with_list_separator_after_fold(self): + a = 'x' * 66 + '@example.com' + to = f'{a}, "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus \n') + + a = '.' * 79 + to = f'"{a}" , "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a}\n' + ' , =?utf-8?q?H=C3=BCbsch?= Kaktus ' + '\n') + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 512464f87162cd..65ddbabcaa1997 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -16,6 +16,7 @@ import email import email.policy +import email.utils from email.charset import Charset from email.generator import Generator, DecodedGenerator, BytesGenerator @@ -336,6 +337,21 @@ def test_nonascii_as_string_without_cte(self): msg = email.message_from_bytes(source) self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_with_ascii_charset(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 8bit + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = source.decode('ascii', 'replace') + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_without_content_type_and_cte(self): m = textwrap.dedent("""\ MIME-Version: 1.0 @@ -3337,15 +3353,137 @@ def test_getaddresses_comma_in_name(self): ], ) + def test_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056""" + alice = 'alice@example.org' + bob = 'bob@example.com' + empty = ('', '') + + # Test utils.getaddresses() and utils.parseaddr() on malformed email + # addresses: default behavior (strict=True) rejects malformed address, + # and strict=False which tolerates malformed address. + for invalid_separator, expected_non_strict in ( + ('(', [(f'<{bob}>', alice)]), + (')', [('', alice), empty, ('', bob)]), + ('<', [('', alice), empty, ('', bob), empty]), + ('>', [('', alice), empty, ('', bob)]), + ('[', [('', f'{alice}[<{bob}>]')]), + (']', [('', alice), empty, ('', bob)]), + ('@', [empty, empty, ('', bob)]), + (';', [('', alice), empty, ('', bob)]), + (':', [('', alice), ('', bob)]), + ('.', [('', alice + '.'), ('', bob)]), + ('"', [('', alice), ('', f'<{bob}>')]), + ): + address = f'{alice}{invalid_separator}<{bob}>' + with self.subTest(address=address): + self.assertEqual(utils.getaddresses([address]), + [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + expected_non_strict) + + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Comma (',') is treated differently depending on strict parameter. + # Comma without quotes. + address = f'{alice},<{bob}>' + self.assertEqual(utils.getaddresses([address]), + [('', alice), ('', bob)]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', alice), ('', bob)]) + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Real name between quotes containing comma. + address = '"Alice, alice@example.org" ' + expected_strict = ('Alice, alice@example.org', 'bob@example.com') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Valid parenthesis in comments. + address = 'alice@example.org (Alice)' + expected_strict = ('Alice', 'alice@example.org') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Invalid parenthesis in comments. + address = 'alice@example.org )Alice(' + self.assertEqual(utils.getaddresses([address]), [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', 'alice@example.org'), ('', ''), ('', 'Alice')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Two addresses with quotes separated by comma. + address = '"Jane Doe" , "John Doe" ' + self.assertEqual(utils.getaddresses([address]), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Test email.utils.supports_strict_parsing attribute + self.assertEqual(email.utils.supports_strict_parsing, True) + def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(utils.getaddresses(['foo: ;']), [('', '')]) - eq(utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" ']), - [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) + for addresses, expected in ( + (['"Sürname, Firstname" '], + [('Sürname, Firstname', 'to@example.com')]), + + (['foo: ;'], + [('', '')]), + + (['foo: ;', '"Jason R. Mastaler" '], + [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]), + + ([r'Pete(A nice \) chap) '], + [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]), + + (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'], + [('', '')]), + + (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'], + [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]), + + (['John Doe '], + [('John Doe (comment)', 'jdoe@machine.example')]), + + (['"Mary Smith: Personal Account" '], + [('Mary Smith: Personal Account', 'smith@home.example')]), + + (['Undisclosed recipients:;'], + [('', '')]), + + ([r', "Giant; \"Big\" Box" '], + [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]), + ): + with self.subTest(addresses=addresses): + self.assertEqual(utils.getaddresses(addresses), + expected) + self.assertEqual(utils.getaddresses(addresses, strict=False), + expected) + + addresses = ['[]*-- =~$'] + self.assertEqual(utils.getaddresses(addresses), + [('', '')]) + self.assertEqual(utils.getaddresses(addresses, strict=False), + [('', ''), ('', ''), ('', '*--')]) def test_getaddresses_embedded_comment(self): """Test proper handling of a nested comment""" @@ -3536,6 +3674,54 @@ def test_mime_classes_policy_argument(self): m = cls(*constructor, policy=email.policy.default) self.assertIs(m.policy, email.policy.default) + def test_iter_escaped_chars(self): + self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')), + [(0, 'a'), + (2, '\\\\'), + (3, 'b'), + (5, '\\"'), + (6, 'c'), + (8, '\\\\'), + (9, '"'), + (10, 'd')]) + self.assertEqual(list(utils._iter_escaped_chars('a\\')), + [(0, 'a'), (1, '\\')]) + + def test_strip_quoted_realnames(self): + def check(addr, expected): + self.assertEqual(utils._strip_quoted_realnames(addr), expected) + + check('"Jane Doe" , "John Doe" ', + ' , ') + check(r'"Jane \"Doe\"." ', + ' ') + + # special cases + check(r'before"name"after', 'beforeafter') + check(r'before"name"', 'before') + check(r'b"name"', 'b') # single char + check(r'"name"after', 'after') + check(r'"name"a', 'a') # single char + check(r'"name"', '') + + # no change + for addr in ( + 'Jane Doe , John Doe ', + 'lone " quote', + ): + self.assertEqual(utils._strip_quoted_realnames(addr), addr) + + + def test_check_parenthesis(self): + addr = 'alice@example.net' + self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)')) + self.assertFalse(utils._check_parenthesis(f'{addr} )Alice(')) + self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))')) + self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)')) + + # Ignore real name between quotes + self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}')) + # Test the iterator/generators class TestIterators(TestEmailBase): @@ -3995,6 +4181,21 @@ def test_8bit_in_uuencode_body(self): self.assertEqual(msg.get_payload(decode=True), '<,.V', + [errors.ObsoleteHeaderDefect], + '"John X." ', + 'John X.', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + 'name_starting_with_dot': + ('. Doe ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '". Doe" ', + '. Doe', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + } # XXX: Need many more examples, and in particular some with names in @@ -1628,7 +1648,7 @@ def test_address_display_names(self): 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' - '=C3=B4nsectetuer?=\n =?utf-8?q?adipiscing=2E_Suspendisse' + '=C3=B4nsectetuer?=\n =?utf-8?q?_adipiscing=2E_Suspendisse' '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' '?q?p=C3=B4tenti=2E?=', ), diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index d3f396f02e7a72..034f7626c1fc7c 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -748,6 +748,35 @@ def test_iter_attachments_mutation(self): self.assertEqual(len(list(m.iter_attachments())), 2) self.assertEqual(m.get_payload(), orig) + get_payload_surrogate_params = { + + 'good_surrogateescape': ( + "String that can be encod\udcc3\udcabd with surrogateescape", + b'String that can be encod\xc3\xabd with surrogateescape' + ), + + 'string_with_utf8': ( + "String with utf-8 charactër", + b'String with utf-8 charact\xebr' + ), + + 'surrogate_and_utf8': ( + "String that cannot be ëncod\udcc3\udcabd with surrogateescape", + b'String that cannot be \xebncod\\udcc3\\udcabd with surrogateescape' + ), + + 'out_of_range_surrogate': ( + "String with \udfff cannot be encoded with surrogateescape", + b'String with \\udfff cannot be encoded with surrogateescape' + ), + } + + def get_payload_surrogate_as_gh_94606(self, msg, expected): + """test for GH issue 94606""" + m = self._str_msg(msg) + payload = m.get_payload(decode=True) + self.assertEqual(expected, payload) + class TestEmailMessage(TestEmailMessageBase, TestEmailBase): message = EmailMessage diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index e87c275549406d..c6b9c80efe1b54 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -135,6 +135,23 @@ def test_policy_addition(self): for attr, value in expected.items(): self.assertEqual(getattr(added, attr), value) + def test_fold_utf8(self): + expected_ascii = 'Subject: =?utf-8?q?=C3=A1?=\n' + expected_utf8 = 'Subject: á\n' + + msg = email.message.EmailMessage() + s = 'á' + msg['Subject'] = s + + p_ascii = email.policy.default.clone() + p_utf8 = email.policy.default.clone(utf8=True) + + self.assertEqual(p_ascii.fold('Subject', msg['Subject']), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', msg['Subject']), expected_utf8) + + self.assertEqual(p_ascii.fold('Subject', s), expected_ascii) + self.assertEqual(p_utf8.fold('Subject', s), expected_utf8) + def test_fold_zero_max_line_length(self): expected = 'Subject: =?utf-8?q?=C3=A1?=\n' diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index 25fa48c5ee217b..4e6201e13c87f9 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -3,8 +3,7 @@ import test.support import time import unittest -import sys -import os.path + class DateTimeTests(unittest.TestCase): @@ -142,25 +141,17 @@ def test_localtime_epoch_notz_daylight_false(self): t2 = utils.localtime(t0.replace(tzinfo=None)) self.assertEqual(t1, t2) - # XXX: Need a more robust test for Olson's tzdata - @unittest.skipIf(sys.platform.startswith('win'), - "Windows does not use Olson's TZ database") - @unittest.skipUnless(os.path.exists('/usr/share/zoneinfo') or - os.path.exists('/usr/lib/zoneinfo'), - "Can't find the Olson's TZ database") - @test.support.run_with_tz('Europe/Kiev') + @test.support.run_with_tz('Europe/Kyiv') def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) + if t1.tzname() in ('Europe', 'UTC'): + self.skipTest("Can't find a Kyiv timezone database") self.assertEqual(t1.tzname(), 'MSK') t0 = datetime.datetime(1994, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) self.assertEqual(t1.tzname(), 'EET') - def test_isdst_deprecation(self): - with self.assertWarns(DeprecationWarning): - t0 = datetime.datetime(1990, 1, 1) - t1 = utils.localtime(t0, isdst=True) # Issue #24836: The timezone files are out of date (pre 2011k) # on Mac OS X Snow Leopard. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5a8690a4836dd6..30dab1fbaa48b2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -19,10 +19,23 @@ if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + + MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 PYMEM_ALLOCATOR_MALLOC = 3 +PYMEM_ALLOCATOR_MIMALLOC = 7 +if support.Py_GIL_DISABLED: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC +else: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC + Py_STATS = hasattr(sys, '_stats_on') # _PyCoreConfig_InitCompatConfig() @@ -35,6 +48,8 @@ INIT_LOOPS = 4 MAX_HASH_SEED = 4294967295 +ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' + # If we are running from a build dir, but the stdlib has been installed, # some tests need to expect different results. @@ -346,6 +361,7 @@ def test_simple_initialization_api(self): self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) @support.requires_specialization + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 @@ -390,6 +406,17 @@ def test_ucnhash_capi_reset(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '9\n' * INIT_LOOPS) + def test_datetime_reset_strptime(self): + code = ( + "import datetime;" + "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');" + "print(d.strftime('%Y%m%d'))" + ) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '20000101\n' * INIT_LOOPS) + + +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') @@ -437,31 +464,31 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT = { '_config_init': API_COMPAT, - 'isolated': 0, - 'use_environment': 1, - 'dev_mode': 0, + 'isolated': False, + 'use_environment': True, + 'dev_mode': False, - 'install_signal_handlers': 1, - 'use_hash_seed': 0, + 'install_signal_handlers': True, + 'use_hash_seed': False, 'hash_seed': 0, 'int_max_str_digits': sys.int_info.default_max_str_digits, 'cpu_count': -1, - 'faulthandler': 0, + 'faulthandler': False, 'tracemalloc': 0, - 'perf_profiling': 0, - 'import_time': 0, - 'code_debug_ranges': 1, - 'show_ref_count': 0, - 'dump_refs': 0, + 'perf_profiling': False, + 'import_time': False, + 'code_debug_ranges': True, + 'show_ref_count': False, + 'dump_refs': False, 'dump_refs_file': None, - 'malloc_stats': 0, + 'malloc_stats': False, 'filesystem_encoding': GET_DEFAULT_CONFIG, 'filesystem_errors': GET_DEFAULT_CONFIG, 'pycache_prefix': None, 'program_name': GET_DEFAULT_CONFIG, - 'parse_argv': 0, + 'parse_argv': False, 'argv': [""], 'orig_argv': [], @@ -478,43 +505,47 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, 'module_search_paths': GET_DEFAULT_CONFIG, - 'module_search_paths_set': 1, + 'module_search_paths_set': True, 'platlibdir': sys.platlibdir, 'stdlib_dir': GET_DEFAULT_CONFIG, - 'site_import': 1, + 'site_import': True, 'bytes_warning': 0, - 'warn_default_encoding': 0, - 'inspect': 0, - 'interactive': 0, + 'warn_default_encoding': False, + 'inspect': False, + 'interactive': False, 'optimization_level': 0, - 'parser_debug': 0, - 'write_bytecode': 1, + 'parser_debug': False, + 'write_bytecode': True, 'verbose': 0, - 'quiet': 0, - 'user_site_directory': 1, - 'configure_c_stdio': 0, - 'buffered_stdio': 1, + 'quiet': False, + 'user_site_directory': True, + 'configure_c_stdio': False, + 'buffered_stdio': True, 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, - 'skip_source_first_line': 0, + 'skip_source_first_line': False, 'run_command': None, 'run_module': None, 'run_filename': None, 'sys_path_0': None, - '_install_importlib': 1, + '_install_importlib': True, 'check_hash_pycs_mode': 'default', - 'pathconfig_warnings': 1, - '_init_main': 1, + 'pathconfig_warnings': True, + '_init_main': True, 'use_frozen_modules': not support.Py_DEBUG, - 'safe_path': 0, + 'safe_path': False, '_is_python_build': IGNORE_CONFIG, } if Py_STATS: CONFIG_COMPAT['_pystats'] = 0 + if support.Py_DEBUG: + CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, @@ -522,22 +553,22 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, - configure_c_stdio=1, - parse_argv=2, + configure_c_stdio=True, + parse_argv=True, ) CONFIG_ISOLATED = dict(CONFIG_COMPAT, _config_init=API_ISOLATED, - isolated=1, - use_environment=0, - user_site_directory=0, - safe_path=1, - dev_mode=0, - install_signal_handlers=0, - use_hash_seed=0, - faulthandler=0, + isolated=True, + use_environment=False, + user_site_directory=False, + safe_path=True, + dev_mode=False, + install_signal_handlers=False, + use_hash_seed=False, + faulthandler=False, tracemalloc=0, - perf_profiling=0, - pathconfig_warnings=0, + perf_profiling=False, + pathconfig_warnings=False, ) if MS_WINDOWS: CONFIG_ISOLATED['legacy_windows_stdio'] = 0 @@ -727,6 +758,9 @@ def check_config(self, configs, expected): if value is self.IGNORE_CONFIG: config.pop(key, None) del expected[key] + # Resolve bool/int mismatches to reduce noise in diffs + if isinstance(value, (bool, int)) and isinstance(config.get(key), (bool, int)): + expected[key] = type(config[key])(expected[key]) self.assertEqual(config, expected) def check_global_config(self, configs): @@ -839,19 +873,19 @@ def test_init_global_config(self): def test_init_from_config(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { - 'install_signal_handlers': 0, - 'use_hash_seed': 1, + 'install_signal_handlers': False, + 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, - 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'show_ref_count': 1, - 'malloc_stats': 1, + 'perf_profiling': False, + 'import_time': True, + 'code_debug_ranges': False, + 'show_ref_count': True, + 'malloc_stats': True, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', @@ -864,7 +898,7 @@ def test_init_from_config(self): '-X', 'cmdline_xoption', '-c', 'pass', 'arg2'], - 'parse_argv': 2, + 'parse_argv': True, 'xoptions': [ 'config_xoption1=3', 'config_xoption2=', @@ -878,26 +912,26 @@ def test_init_from_config(self): ], 'run_command': 'pass\n', - 'site_import': 0, + 'site_import': False, 'bytes_warning': 1, - 'inspect': 1, - 'interactive': 1, + 'inspect': True, + 'interactive': True, 'optimization_level': 2, - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'quiet': 1, - 'configure_c_stdio': 1, - 'buffered_stdio': 0, - 'user_site_directory': 0, - 'faulthandler': 1, + 'quiet': True, + 'configure_c_stdio': True, + 'buffered_stdio': False, + 'user_site_directory': False, + 'faulthandler': True, 'platlibdir': 'my_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 31337, 'cpu_count': 4321, 'check_hash_pycs_mode': 'always', - 'pathconfig_warnings': 0, + 'pathconfig_warnings': False, } if Py_STATS: config['_pystats'] = 1 @@ -906,31 +940,31 @@ def test_init_from_config(self): def test_init_compat_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'perf_profiling': False, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, } if Py_STATS: @@ -940,36 +974,36 @@ def test_init_compat_env(self): def test_init_python_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'perf_profiling': False, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, } if Py_STATS: - config['_pystats'] = 1 + config['_pystats'] = True self.check_all_configs("test_init_python_env", config, preconfig, api=API_PYTHON) @@ -982,7 +1016,7 @@ def test_init_env_dev_mode(self): api=API_COMPAT) def test_init_env_dev_mode_alloc(self): - preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC) + preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG) config = dict(dev_mode=1, faulthandler=1, warnoptions=['default']) @@ -994,8 +1028,8 @@ def test_init_dev_mode(self): 'allocator': PYMEM_ALLOCATOR_DEBUG, } config = { - 'faulthandler': 1, - 'dev_mode': 1, + 'faulthandler': True, + 'dev_mode': True, 'warnoptions': ['default'], } self.check_all_configs("test_init_dev_mode", config, preconfig, @@ -1011,11 +1045,11 @@ def test_preinit_parse_argv(self): 'argv': ['script.py'], 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'], 'run_filename': os.path.abspath('script.py'), - 'dev_mode': 1, - 'faulthandler': 1, + 'dev_mode': True, + 'faulthandler': True, 'warnoptions': ['default'], 'xoptions': ['dev'], - 'safe_path': 1, + 'safe_path': True, } self.check_all_configs("test_preinit_parse_argv", config, preconfig, api=API_PYTHON) @@ -1040,30 +1074,30 @@ def test_preinit_dont_parse_argv(self): def test_init_isolated_flag(self): config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) def test_preinit_isolated1(self): # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) def test_preinit_isolated2(self): # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) @@ -1131,7 +1165,7 @@ def test_init_run_main(self): 'orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 2, + 'parse_argv': True, 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1146,7 +1180,7 @@ def test_init_main(self): 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 2, + 'parse_argv': True, '_init_main': 0, 'sys_path_0': '', } @@ -1156,12 +1190,12 @@ def test_init_main(self): def test_init_parse_argv(self): config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 0, + 'use_environment': False, } self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) @@ -1170,7 +1204,7 @@ def test_init_dont_parse_argv(self): 'parse_argv': 0, } config = { - 'parse_argv': 0, + 'parse_argv': False, 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', @@ -1253,11 +1287,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None): ver = sys.version_info return [ os.path.join(prefix, sys.platlibdir, - f'python{ver.major}{ver.minor}.zip'), + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), os.path.join(prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), os.path.join(exec_prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}', 'lib-dynload'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), ] @contextlib.contextmanager @@ -1311,7 +1345,7 @@ def test_init_setpythonhome(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1352,7 +1386,7 @@ def test_init_is_python_build_with_home(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1483,7 +1517,7 @@ def test_init_pyvenv_cfg(self): if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, sys.platlibdir, - f'python{ver.major}.{ver.minor}', + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload') os.makedirs(lib_dynload) else: @@ -1578,7 +1612,6 @@ def test_global_pathconfig(self): # The global path configuration (_Py_path_config) must be a copy # of the path configuration of PyInterpreter.config (PyConfig). ctypes = import_helper.import_module('ctypes') - _testinternalcapi = import_helper.import_module('_testinternalcapi') def get_func(name): func = getattr(ctypes.pythonapi, name) @@ -1645,20 +1678,20 @@ def test_get_argc_argv(self): def test_init_use_frozen_modules(self): tests = { - ('=on', 1), - ('=off', 0), - ('=', 1), - ('', 1), + ('=on', True), + ('=off', False), + ('=', True), + ('', True), } for raw, expected in tests: optval = f'frozen_modules{raw}' config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c'], 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 1, + 'use_environment': True, 'xoptions': [optval], 'use_frozen_modules': expected, } @@ -1774,6 +1807,7 @@ def test_unicode_id_init(self): # See bpo-44133 @unittest.skipIf(os.name == 'nt', 'Py_FrozenMain is not exported on Windows') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozenmain(self): env = dict(os.environ) env['PYTHONUNBUFFERED'] = '1' @@ -1784,9 +1818,9 @@ def test_frozenmain(self): sys.argv ['./argv0', '-E', 'arg1', 'arg2'] config program_name: ./argv0 config executable: {executable} - config use_environment: 1 - config configure_c_stdio: 1 - config buffered_stdio: 0 + config use_environment: True + config configure_c_stdio: True + config buffered_stdio: False """).lstrip() self.assertEqual(out, expected) @@ -1818,6 +1852,22 @@ def test_no_memleak(self): self.assertEqual(refs, 0, out) self.assertEqual(blocks, 0, out) + @unittest.skipUnless(support.Py_DEBUG, + '-X presite requires a Python debug build') + def test_presite(self): + cmd = [sys.executable, "-I", "-X", "presite=test.reperf", "-c", "print('cmd')"] + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.assertEqual(proc.returncode, 0) + out = proc.stdout.strip() + self.assertIn("10 times sub", out) + self.assertIn("CPU seconds", out) + self.assertIn("cmd", out) + class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): # Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr(): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 69ab2a4feaa938..6d3c91b0b6d9f9 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,6 +6,7 @@ import test.support import unittest import unittest.mock +from pathlib import Path import ensurepip import ensurepip._uninstall @@ -20,41 +21,35 @@ def test_version(self): # Test version() with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): self.assertEqual(ensurepip.version(), '1.2.3b1') - def test_get_packages_no_dir(self): - # Test _get_packages() without a wheel package directory - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): - packages = ensurepip._get_packages() - - # when bundled wheel packages are used, we get _PIP_VERSION + def test_version_no_dir(self): + # Test version() without a wheel package directory + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) - # use bundled wheel packages - self.assertIsNotNone(packages['pip'].wheel_name) + def test_selected_wheel_path_no_dir(self): + pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) - def test_get_packages_with_dir(self): - # Test _get_packages() with a wheel package directory + def test_selected_wheel_path_with_dir(self): + # Test _get_pip_whl_path_ctx() with a wheel package directory pip_filename = "pip-20.2.2-py2.py3-none-any.whl" with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, pip_filename) - # not used, make sure that it's ignored + # not used, make sure that they're ignored + self.touch(tmpdir, "pip-1.2.3-py2.py3-none-any.whl") self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + self.touch(tmpdir, "pip-script.py") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): - packages = ensurepip._get_packages() - - self.assertEqual(packages['pip'].version, '20.2.2') - self.assertEqual(packages['pip'].wheel_path, - os.path.join(tmpdir, pip_filename)) - - # wheel package is ignored - self.assertEqual(sorted(packages), ['pip']) + with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)): + with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path: + self.assertEqual(pip_filename, bundled_wheel_path.name) class EnsurepipMixin: @@ -69,7 +64,7 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() - # But expose os.listdir() used by _find_packages() + # But expose os.listdir() used by _find_wheel_pkg_dir_pip() patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 8c1f285f7b3bc2..99fd16ba361e6f 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -18,7 +18,7 @@ from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support -from test.support import ALWAYS_EQ +from test.support import ALWAYS_EQ, REPO_ROOT from test.support import threading_helper from datetime import timedelta @@ -26,14 +26,19 @@ def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(enum)) - if os.path.exists('Doc/library/enum.rst'): + + lib_tests = os.path.join(REPO_ROOT, 'Doc/library/enum.rst') + if os.path.exists(lib_tests): tests.addTests(doctest.DocFileSuite( - '../../Doc/library/enum.rst', + lib_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) - if os.path.exists('Doc/howto/enum.rst'): + howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') + if os.path.exists(howto_tests): tests.addTests(doctest.DocFileSuite( - '../../Doc/howto/enum.rst', + howto_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) return tests @@ -171,7 +176,7 @@ class TestHelpers(unittest.TestCase): sunder_names = '_bad_', '_good_', '_what_ho_' dunder_names = '__mal__', '__bien__', '__que_que__' - private_names = '_MyEnum__private', '_MyEnum__still_private' + private_names = '_MyEnum__private', '_MyEnum__still_private', '_MyEnum___triple_private' private_and_sunder_names = '_MyEnum__private_', '_MyEnum__also_private_' random_names = 'okay', '_semi_private', '_weird__', '_MyEnum__' @@ -442,7 +447,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -509,6 +514,7 @@ def test_contains_tf(self): self.assertFalse('first' in MainEnum) val = MainEnum.dupe self.assertIn(val, MainEnum) + self.assertNotIn(float('nan'), MainEnum) # class OtherEnum(Enum): one = auto() @@ -1042,6 +1048,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -1473,6 +1495,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -2338,6 +2381,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () @@ -3196,6 +3273,37 @@ class NTEnum(Enum): [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], ) + self.assertRaises(AttributeError, getattr, NTEnum.NONE, 'id') + # + class NTCEnum(TTuple, Enum): + NONE = 0, 0, [] + A = 1, 2, [4] + B = 2, 4, [0, 1, 2] + self.assertEqual(repr(NTCEnum.NONE), "") + self.assertEqual(NTCEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual(NTCEnum.NONE.id, 0) + self.assertEqual(NTCEnum.A.a, 2) + self.assertEqual(NTCEnum.B.blist, [0, 1 ,2]) + self.assertEqual( + [x.value for x in NTCEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) + # + class NTDEnum(Enum): + def __new__(cls, id, a, blist): + member = object.__new__(cls) + member.id = id + member.a = a + member.blist = blist + return member + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTDEnum.NONE), "") + self.assertEqual(NTDEnum.NONE.id, 0) + self.assertEqual(NTDEnum.A.a, 2) + self.assertEqual(NTDEnum.B.blist, [0, 1 ,2]) + def test_flag_with_custom_new(self): class FlagFromChar(IntFlag): def __new__(cls, c): @@ -3263,6 +3371,95 @@ def __new__(cls, value): member._value_ = Base(value) return member + def test_extra_member_creation(self): + class IDEnumMeta(EnumMeta): + def __new__(metacls, cls, bases, classdict, **kwds): + # add new entries to classdict + for name in classdict.member_names: + classdict[f'{name}_DESC'] = f'-{classdict[name]}' + return super().__new__(metacls, cls, bases, classdict, **kwds) + class IDEnum(StrEnum, metaclass=IDEnumMeta): + pass + class MyEnum(IDEnum): + ID = 'id' + NAME = 'name' + self.assertEqual(list(MyEnum), [MyEnum.ID, MyEnum.NAME, MyEnum.ID_DESC, MyEnum.NAME_DESC]) + + def test_add_alias(self): + class mixin: + @property + def ORG(self): + return 'huh' + class Color(mixin, Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_alias_('ROJO') + self.assertIs(Color.RED, Color['ROJO']) + self.assertIs(Color.RED, Color.ROJO) + Color.BLUE._add_alias_('ORG') + self.assertIs(Color.BLUE, Color['ORG']) + self.assertIs(Color.BLUE, Color.ORG) + self.assertEqual(Color.RED.ORG, 'huh') + self.assertEqual(Color.GREEN.ORG, 'huh') + self.assertEqual(Color.BLUE.ORG, 'huh') + self.assertEqual(Color.ORG.ORG, 'huh') + + def test_add_value_alias_after_creation(self): + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_value_alias_(5) + self.assertIs(Color.RED, Color(5)) + + def test_add_value_alias_during_creation(self): + class Types(Enum): + Unknown = 0, + Source = 1, 'src' + NetList = 2, 'nl' + def __new__(cls, int_value, *value_aliases): + member = object.__new__(cls) + member._value_ = int_value + for alias in value_aliases: + member._add_value_alias_(alias) + return member + self.assertIs(Types(0), Types.Unknown) + self.assertIs(Types(1), Types.Source) + self.assertIs(Types('src'), Types.Source) + self.assertIs(Types(2), Types.NetList) + self.assertIs(Types('nl'), Types.NetList) + + def test_second_tuple_item_is_falsey(self): + class Cardinal(Enum): + RIGHT = (1, 0) + UP = (0, 1) + LEFT = (-1, 0) + DOWN = (0, -1) + self.assertIs(Cardinal(1, 0), Cardinal.RIGHT) + self.assertIs(Cardinal(-1, 0), Cardinal.LEFT) + + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) + + def test_empty_names(self): + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -3887,6 +4084,8 @@ def test_global_repr_conform1(self): @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') @@ -4721,22 +4920,22 @@ class Color(enum.Enum) | The value of the Enum member. | | ---------------------------------------------------------------------- - | Methods inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: | - | __contains__(value) from enum.EnumType + | __contains__(value) | Return True if `value` is in `cls`. | | `value` is in `cls` if: | 1) `value` is a member of `cls`, or | 2) `value` is the value of one of the `cls`'s members. | - | __getitem__(name) from enum.EnumType + | __getitem__(name) | Return the member matching `name`. | - | __iter__() from enum.EnumType + | __iter__() | Return members in definition order. | - | __len__() from enum.EnumType + | __len__() | Return the number of members (no aliases) | | ---------------------------------------------------------------------- @@ -4761,11 +4960,11 @@ class Color(enum.Enum) | | Data and other attributes defined here: | - | YELLOW = + | CYAN = | | MAGENTA = | - | CYAN = + | YELLOW = | | ---------------------------------------------------------------------- | Data descriptors inherited from enum.Enum: @@ -4775,7 +4974,18 @@ class Color(enum.Enum) | value | | ---------------------------------------------------------------------- - | Data descriptors inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: + | + | __contains__(value) + | + | __getitem__(name) + | + | __iter__() + | + | __len__() + | + | ---------------------------------------------------------------------- + | Readonly properties inherited from enum.EnumType: | | __members__""" @@ -4936,12 +5146,14 @@ class CheckedColor(Enum): @bltns.property def zeroth(self): return 'zeroed %s' % self.name - self.assertTrue(_test_simple_enum(CheckedColor, SimpleColor) is None) + _test_simple_enum(CheckedColor, SimpleColor) SimpleColor.MAGENTA._value_ = 9 self.assertRaisesRegex( TypeError, "enum mismatch", _test_simple_enum, CheckedColor, SimpleColor, ) + # + # class CheckedMissing(IntFlag, boundary=KEEP): SIXTY_FOUR = 64 ONE_TWENTY_EIGHT = 128 @@ -4958,8 +5170,78 @@ class Missing: ALL = 2048 + 128 + 64 + 12 M = Missing self.assertEqual(list(CheckedMissing), [M.SIXTY_FOUR, M.ONE_TWENTY_EIGHT, M.TWENTY_FORTY_EIGHT]) - # _test_simple_enum(CheckedMissing, Missing) + # + # + class CheckedUnhashable(Enum): + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), CheckedUnhashable) + self.assertIn('python', CheckedUnhashable) + self.assertEqual(CheckedUnhashable.name.value, 'python') + self.assertEqual(CheckedUnhashable.name.name, 'name') + # + @_simple_enum() + class Unhashable: + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), Unhashable) + self.assertIn('python', Unhashable) + self.assertEqual(Unhashable.name.value, 'python') + self.assertEqual(Unhashable.name.name, 'name') + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): @@ -5166,7 +5448,7 @@ def member_dir(member): allowed.add(name) else: allowed.discard(name) - else: + elif name not in member._member_map_: allowed.add(name) return sorted(allowed) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 20122679223843..b4fc290b1f32b6 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index eafa7d84638b76..e4f2e3a97b8bb8 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -12,12 +12,20 @@ from test.support import (captured_stderr, check_impl_detail, cpython_only, gc_collect, no_tracing, script_helper, - SuppressCrashReport) + SuppressCrashReport, + force_not_colorized) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink from test.support.warnings_helper import check_warnings from test import support +try: + import _testcapi + from _testcapi import INT_MAX +except ImportError: + _testcapi = None + INT_MAX = 2**31 - 1 + class NaiveException(Exception): def __init__(self, x): @@ -34,6 +42,7 @@ def __str__(self): # XXX This is not really enough, each *operation* should be tested! + class ExceptionTests(unittest.TestCase): def raise_catch(self, exc, excname): @@ -295,6 +304,7 @@ def baz(): { 6 0="""''', 5, 13) + check('b"fooжжж"'.encode(), 1, 1, 1, 10) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) @@ -318,11 +328,13 @@ def baz(): check('(yield i) = 2', 1, 2) check('def f(*):\n pass', 1, 7) + @unittest.skipIf(INT_MAX >= sys.maxsize, "Downcasting to int is safe for col_offset") @support.requires_resource('cpu') - @support.bigmemtest(support._2G, memuse=1.5) - def testMemoryErrorBigSource(self, _size): - with self.assertRaises(OverflowError): - exec(f"if True:\n {' ' * 2**31}print('hello world')") + @support.bigmemtest(INT_MAX, memuse=2, dry_run=False) + def testMemoryErrorBigSource(self, size): + src = b"if True:\n%*s" % (size, b"pass") + with self.assertRaisesRegex(OverflowError, "Parser column offset overflow"): + compile(src, '', 'exec') @cpython_only def testSettingException(self): @@ -336,8 +348,8 @@ def __init__(self_): class InvalidException: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi1(): - import _testcapi try: _testcapi.raise_exception(BadException, 1) except TypeError as err: @@ -347,8 +359,8 @@ def test_capi1(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi2(): - import _testcapi try: _testcapi.raise_exception(BadException, 0) except RuntimeError as err: @@ -361,8 +373,8 @@ def test_capi2(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi3(): - import _testcapi self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) @@ -1372,6 +1384,7 @@ def foo(): @cpython_only def test_recursion_normalizing_exception(self): + import_module("_testinternalcapi") # Issue #22898. # Test that a RecursionError is raised when tstate->recursion_depth is # equal to recursion_limit in PyErr_NormalizeException() and check @@ -1413,7 +1426,7 @@ def gen(): next(generator) recursionlimit = sys.getrecursionlimit() try: - recurse(support.EXCEEDS_RECURSION_LIMIT) + recurse(support.exceeds_recursion_limit()) finally: sys.setrecursionlimit(recursionlimit) print('Done.') @@ -1426,6 +1439,7 @@ def gen(): self.assertIn(b'Done.', out) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1439,7 +1453,8 @@ def test_recursion_normalizing_infinite_exception(self): """ rc, out, err = script_helper.assert_python_failure("-c", code) self.assertEqual(rc, 1) - self.assertIn(b'RecursionError: maximum recursion depth exceeded', err) + expected = b'RecursionError: maximum recursion depth exceeded' + self.assertTrue(expected in err, msg=f"{expected!r} not found in {err[:3_000]!r}... (truncated)") self.assertIn(b'Done.', out) @@ -1494,6 +1509,7 @@ def recurse_in_body_and_except(): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1516,6 +1532,7 @@ def recurse(cnt): self.assertIn(b'MemoryError', err) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_MemoryError(self): # PyErr_NoMemory always raises the same exception instance. # Check that the traceback is not doubled. @@ -1535,8 +1552,8 @@ def raiseMemError(): self.assertEqual(tb1, tb2) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_exception_with_doc(self): - import _testcapi doc2 = "This is a test docstring." doc4 = "This is another test docstring." @@ -1575,6 +1592,7 @@ class C(object): self.assertEqual(error5.__doc__, "") @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_cleanup(self): # Issue #5437: preallocated MemoryError instances should not keep # traceback objects alive. @@ -1665,6 +1683,7 @@ def test_unhandled(self): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1783,6 +1802,44 @@ class TestException(MemoryError): gc_collect() + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_in_subinterp(self): + # gh-109894: subinterpreters shouldn't count on last resort memory error + # when MemoryError is raised through PyErr_NoMemory() call, + # and should preallocate memory errors as does the main interpreter. + # interp.static_objects.last_resort_memory_error.args + # should be initialized to empty tuple to avoid crash on attempt to print it. + code = f"""if 1: + import _testcapi + _testcapi.run_in_subinterp(\"[0]*{sys.maxsize}\") + exit(0) + """ + rc, _, err = script_helper.assert_python_ok("-c", code) + self.assertIn(b'MemoryError', err) + + def test_keyerror_context(self): + # Make sure that _PyErr_SetKeyError() chains exceptions + try: + err1 = None + err2 = None + try: + d = {} + try: + raise ValueError("bug") + except Exception as exc: + err1 = exc + d[1] + except Exception as exc: + err2 = exc + + self.assertIsInstance(err1, ValueError) + self.assertIsInstance(err2, KeyError) + self.assertEqual(err2.__context__, err1) + finally: + # Break any potential reference cycle + exc1 = None + exc2 = None + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): @@ -1802,6 +1859,8 @@ def f(): except self.failureException: with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) + else: + self.fail("assertRaisesRegex should have failed.") self.assertIn("aab", err.getvalue()) @@ -1822,6 +1881,13 @@ def f(): self.assertIn("nonsense", err.getvalue()) self.assertIn("ZeroDivisionError", err.getvalue()) + def test_gh_111654(self): + def f(): + class TestClass: + TestClass + + self.assertRaises(NameError, f) + # Note: name suggestion tests live in `test_traceback`. @@ -1888,7 +1954,7 @@ def test_attributes(self): self.assertEqual(exc.name, 'somename') self.assertEqual(exc.path, 'somepath') - msg = "'invalid' is an invalid keyword argument for ImportError" + msg = r"ImportError\(\) got an unexpected keyword argument 'invalid'" with self.assertRaisesRegex(TypeError, msg): ImportError('test', invalid='keyword') @@ -1955,6 +2021,7 @@ def write_source(self, source): _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) return err.decode('utf-8').splitlines() + @force_not_colorized def test_assertion_error_location(self): cases = [ ('assert None', @@ -2031,6 +2098,7 @@ def test_assertion_error_location(self): result = self.write_source(source) self.assertEqual(result[-3:], expected) + @force_not_colorized def test_multiline_not_highlighted(self): cases = [ (""" @@ -2051,6 +2119,7 @@ def test_multiline_not_highlighted(self): """, [ ' 1 < 2 and', + ' 3 > 4', 'AssertionError', ], ), @@ -2058,10 +2127,11 @@ def test_multiline_not_highlighted(self): for source, expected in cases: with self.subTest(source): result = self.write_source(source) - self.assertEqual(result[-2:], expected) + self.assertEqual(result[-len(expected):], expected) class SyntaxErrorTests(unittest.TestCase): + @force_not_colorized def test_range_of_offsets(self): cases = [ # Basic range from 2->7 diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py new file mode 100644 index 00000000000000..d896fec73d1971 --- /dev/null +++ b/Lib/test/test_external_inspection.py @@ -0,0 +1,84 @@ +import unittest +import os +import textwrap +import importlib +import sys +from test.support import os_helper, SHORT_TIMEOUT +from test.support.script_helper import make_script + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _testexternalinspection import get_stack_trace +except ImportError: + raise unittest.SkipTest("Test only runs when _testexternalinspection is available") + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class TestGetStackTrace(unittest.TestCase): + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import time, sys, os + def bar(): + for x in range(100): + if x == 50: + baz() + def baz(): + foo() + + def foo(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(1000) + + bar() + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + 'foo', + 'baz', + 'bar', + '' + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + self.assertEqual(stack_trace[0], "test_self_trace") + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d0473500a17735..61ec8fe3151af1 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -266,6 +266,7 @@ def test_sigill(self): 5, 'Illegal instruction') + @unittest.skipIf(_testcapi is None, 'need _testcapi') def check_fatal_error_func(self, release_gil): # Test that Py_FatalError() dumps a traceback with support.SuppressCrashReport(): @@ -574,10 +575,12 @@ def run(self): lineno = 8 else: lineno = 10 + # When the traceback is dumped, the waiter thread may be in the + # `self.running.set()` call or in `self.stop.wait()`. regex = r""" ^Thread 0x[0-9a-f]+ \(most recent call first\): (?: File ".*threading.py", line [0-9]+ in [_a-z]+ - ){{1,3}} File "", line 23 in run + ){{1,3}} File "", line (?:22|23) in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 203dd6fe57dcd9..bb784cbe2ce036 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -6,7 +6,9 @@ import struct import sys import unittest -from test.support import verbose, cpython_only, get_pagesize +from test.support import ( + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose +) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -56,8 +58,10 @@ def get_lockdata(): else: start_len = "qq" - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): + if ( + sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or is_apple + ): if struct.calcsize('l') == 8: off_t = 'l' pid_t = 'i' @@ -113,7 +117,9 @@ def test_fcntl_bad_file(self): @cpython_only def test_fcntl_bad_file_overflow(self): - from _testcapi import INT_MAX, INT_MIN + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX + INT_MIN = _testcapi.INT_MIN # Issue 15989 with self.assertRaises(OverflowError): fcntl.fcntl(INT_MAX + 1, fcntl.F_SETFL, os.O_NONBLOCK) @@ -125,8 +131,9 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - platform.machine().startswith('arm') and platform.system() == 'Linux', - "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") + (platform.machine().startswith("arm") and platform.system() == "Linux") + or platform.system() == "Android", + "this platform returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a # C 'long' but not in a C 'int'. @@ -157,6 +164,7 @@ def test_flock(self): self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_EX | fcntl.LOCK_NB @@ -169,6 +177,7 @@ def test_lockf_exclusive(self): self.assertEqual(p.exitcode, 0) @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + @requires_subprocess() def test_lockf_share(self): self.f = open(TESTFN, 'wb+') cmd = fcntl.LOCK_SH | fcntl.LOCK_NB @@ -182,7 +191,7 @@ def test_lockf_share(self): @cpython_only def test_flock_overflow(self): - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(OverflowError, fcntl.flock, _testcapi.INT_MAX+1, fcntl.LOCK_SH) diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_file_eintr.py index f9236f45ca4be8..466f94e389f986 100644 --- a/Lib/test/test_file_eintr.py +++ b/Lib/test/test_file_eintr.py @@ -21,8 +21,8 @@ raise unittest.SkipTest("test module requires subprocess") # Test import all of the things we're about to try testing up front. -import _io -import _pyio +import _io # noqa: F401 +import _pyio # noqa: F401 @unittest.skipUnless(os.name == 'posix', 'tests requires a posix system.') class TestFileIOSignalInterrupt: diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 9b5ac12bccc58f..1fb47163719ede 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -8,11 +9,24 @@ from test.support import os_helper +def _create_file_shallow_equal(template_path, new_path): + """create a file with the same size and mtime but different content.""" + shutil.copy2(template_path, new_path) + with open(new_path, 'r+b') as f: + next_char = bytearray(f.read(1)) + next_char[0] = (next_char[0] + 1) % 256 + f.seek(0) + f.write(next_char) + shutil.copystat(template_path, new_path) + assert os.stat(new_path).st_size == os.stat(template_path).st_size + assert os.stat(new_path).st_mtime == os.stat(template_path).st_mtime + class FileCompareTestCase(unittest.TestCase): def setUp(self): self.name = os_helper.TESTFN self.name_same = os_helper.TESTFN + '-same' self.name_diff = os_helper.TESTFN + '-diff' + self.name_same_shallow = os_helper.TESTFN + '-same-shallow' data = 'Contents of file go here.\n' for name in [self.name, self.name_same, self.name_diff]: with open(name, 'w', encoding="utf-8") as output: @@ -20,12 +34,19 @@ def setUp(self): with open(self.name_diff, 'a+', encoding="utf-8") as output: output.write('An extra line.\n') + + for name in [self.name_same, self.name_diff]: + shutil.copystat(self.name, name) + + _create_file_shallow_equal(self.name, self.name_same_shallow) + self.dir = tempfile.gettempdir() def tearDown(self): os.unlink(self.name) os.unlink(self.name_same) os.unlink(self.name_diff) + os.unlink(self.name_same_shallow) def test_matching(self): self.assertTrue(filecmp.cmp(self.name, self.name), @@ -36,12 +57,17 @@ def test_matching(self): "Comparing file to identical file fails") self.assertTrue(filecmp.cmp(self.name, self.name_same, shallow=False), "Comparing file to identical file fails") + self.assertTrue(filecmp.cmp(self.name, self.name_same_shallow), + "Shallow identical files should be considered equal") def test_different(self): self.assertFalse(filecmp.cmp(self.name, self.name_diff), "Mismatched files compare as equal") self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + self.assertFalse(filecmp.cmp(self.name, self.name_same_shallow, + shallow=False), + "Mismatched file to shallow identical file compares as equal") def test_cache_clear(self): first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) @@ -56,6 +82,8 @@ def setUp(self): self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + self.dir_diff_file = os.path.join(tmpdir, 'dir-diff-file') + self.dir_same_shallow = os.path.join(tmpdir, 'dir-same-shallow') # Another dir is created under dir_same, but it has a name from the # ignored list so it should not affect testing results. @@ -63,7 +91,17 @@ def setUp(self): self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): + + shutil.rmtree(self.dir, True) + os.mkdir(self.dir) + subdir_path = os.path.join(self.dir, 'subdir') + os.mkdir(subdir_path) + dir_file_path = os.path.join(self.dir, "file") + with open(dir_file_path, 'w', encoding="utf-8") as output: + output.write(data) + + for dir in (self.dir_same, self.dir_same_shallow, + self.dir_diff, self.dir_diff_file): shutil.rmtree(dir, True) os.mkdir(dir) subdir_path = os.path.join(dir, 'subdir') @@ -72,14 +110,25 @@ def setUp(self): fn = 'FiLe' # Verify case-insensitive comparison else: fn = 'file' - with open(os.path.join(dir, fn), 'w', encoding="utf-8") as output: - output.write(data) + + file_path = os.path.join(dir, fn) + + if dir is self.dir_same_shallow: + _create_file_shallow_equal(dir_file_path, file_path) + else: + shutil.copy2(dir_file_path, file_path) with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: output.write('An extra file.\n') + # Add different file2 with respect to dir_diff + with open(os.path.join(self.dir_diff_file, 'file2'), 'w', encoding="utf-8") as output: + output.write('Different contents.\n') + + def tearDown(self): - for dir in (self.dir, self.dir_same, self.dir_diff): + for dir in (self.dir, self.dir_same, self.dir_diff, + self.dir_same_shallow, self.dir_diff_file): shutil.rmtree(dir) def test_default_ignores(self): @@ -102,11 +151,7 @@ def test_cmpfiles(self): shallow=False), "Comparing directory to same fails") - # Add different file2 - with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - - self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_same, + self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_diff_file, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails") @@ -116,11 +161,22 @@ def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" self.assertEqual(sorted(actual), sorted(expected)) + def test_dircmp_identical_directories(self): + self._assert_dircmp_identical_directories() + self._assert_dircmp_identical_directories(shallow=False) + + def test_dircmp_different_file(self): + self._assert_dircmp_different_file() + self._assert_dircmp_different_file(shallow=False) + + def test_dircmp_different_directories(self): + self._assert_dircmp_different_directories() + self._assert_dircmp_different_directories(shallow=False) - def test_dircmp(self): + def _assert_dircmp_identical_directories(self, **options): # Check attributes for comparison of two identical directories left_dir, right_dir = self.dir, self.dir_same - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) if self.caseinsensitive: @@ -142,9 +198,10 @@ def test_dircmp(self): ] self._assert_report(d.report, expected_report) + def _assert_dircmp_different_directories(self, **options): # Check attributes for comparison of two different directories (right) left_dir, right_dir = self.dir, self.dir_diff - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'subdir']) @@ -164,12 +221,8 @@ def test_dircmp(self): self._assert_report(d.report, expected_report) # Check attributes for comparison of two different directories (left) - left_dir, right_dir = self.dir, self.dir_diff - shutil.move( - os.path.join(self.dir_diff, 'file2'), - os.path.join(self.dir, 'file2') - ) - d = filecmp.dircmp(left_dir, right_dir) + left_dir, right_dir = self.dir_diff, self.dir + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'file2', 'subdir']) @@ -180,27 +233,62 @@ def test_dircmp(self): self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, []) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), - "Only in {} : ['file2']".format(self.dir), + "diff {} {}".format(self.dir_diff, self.dir), + "Only in {} : ['file2']".format(self.dir_diff), "Identical files : ['file']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) - # Add different file2 - with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - d = filecmp.dircmp(self.dir, self.dir_diff) + + def _assert_dircmp_different_file(self, **options): + # A different file2 + d = filecmp.dircmp(self.dir_diff, self.dir_diff_file, **options) self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, ['file2']) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), + "diff {} {}".format(self.dir_diff, self.dir_diff_file), "Identical files : ['file']", "Differing files : ['file2']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) + def test_dircmp_no_shallow_different_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow, shallow=False) + self.assertEqual(d.same_files, []) + self.assertEqual(d.diff_files, ['file']) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Differing files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_same_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow) + self.assertEqual(d.same_files, ['file']) + self.assertEqual(d.diff_files, []) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Identical files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 786d9186634305..b340ef7ed1621c 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -23,10 +23,9 @@ from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from pathlib import Path from test.support import verbose -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, FakePath from test.support.os_helper import unlink as safe_unlink from test.support import os_helper from test import support @@ -151,7 +150,7 @@ def test_buffer_sizes(self): print('6. Inplace') savestdout = sys.stdout try: - fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8") + fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8") for line in fi: line = line[:-1].upper() print(line) @@ -256,7 +255,7 @@ def test_detached_stdin_binary_mode(self): def test_file_opening_hook(self): try: # cannot use openhook and inplace mode - fi = FileInput(inplace=1, openhook=lambda f, m: None) + fi = FileInput(inplace=True, openhook=lambda f, m: None) self.fail("FileInput should raise if both inplace " "and openhook arguments are given") except ValueError: @@ -478,23 +477,23 @@ def test_iteration_buffering(self): self.assertRaises(StopIteration, next, fi) self.assertEqual(src.linesread, []) - def test_pathlib_file(self): - t1 = Path(self.writeTmp("Pathlib file.")) + def test_pathlike_file(self): + t1 = FakePath(self.writeTmp("Path-like file.")) with FileInput(t1, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1)) - def test_pathlib_file_inplace(self): - t1 = Path(self.writeTmp('Pathlib file.')) + def test_pathlike_file_inplace(self): + t1 = FakePath(self.writeTmp('Path-like file.')) with FileInput(t1, inplace=True, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') print('Modified %s' % line) with open(t1, encoding="utf-8") as f: - self.assertEqual(f.read(), 'Modified Pathlib file.\n') + self.assertEqual(f.read(), 'Modified Path-like file.\n') class MockFileInput: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index ebfcffd1829174..0611d1749f41c1 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,12 +10,14 @@ from functools import wraps from test.support import ( - cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi + cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, + infinite_recursion, ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, ) from test.support.warnings_helper import check_warnings +from test.support.import_helper import import_module from collections import UserList import _io # C implementation of io @@ -173,6 +175,16 @@ def testRepr(self): self.assertEqual(repr(self.f), "<%s.FileIO [closed]>" % (self.modulename,)) + def test_subclass_repr(self): + class TestSubclass(self.FileIO): + pass + + f = TestSubclass(TESTFN) + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + def testReprNoCloseFD(self): fd = os.open(TESTFN, os.O_RDONLY) try: @@ -183,6 +195,7 @@ def testReprNoCloseFD(self): finally: os.close(fd) + @infinite_recursion(25) def testRecursiveRepr(self): # Issue #25455 with swap_attr(self.f, 'name', self.f): @@ -472,6 +485,14 @@ def testInvalidFd(self): import msvcrt self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd()) + def testBooleanFd(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + f = self.FileIO(fd, closefd=False) + f.close() + self.assertEqual(cm.filename, __file__) + def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" @@ -577,7 +598,7 @@ class COtherFileTests(OtherFileTests, unittest.TestCase): @cpython_only def testInvalidFd_overflow(self): # Issue 15989 - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 1d134430909d84..42871f8a09b16b 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -13,7 +13,7 @@ def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C try: @@ -22,7 +22,7 @@ def __new__(cls, *args, **kwargs): def without_gc(cls): class C: def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.without_gc') + raise unittest.SkipTest('requires _testcapi.without_gc') return C from test import support diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 08d56e77aa6393..36ba67ac7415e7 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -18,7 +18,6 @@ except ImportError: _testcapi = None -HAVE_IEEE_754 = float.__getformat__("double").startswith("IEEE") INF = float("inf") NAN = float("nan") @@ -983,6 +982,13 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1.0).__round__(None), + round(1.0), + round(1.0, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) # Beginning with Python 2.6 float has cross platform compatible # ways to create and represent inf and nan @@ -1538,69 +1544,5 @@ def __init__(self, value): self.assertEqual(getattr(f, 'foo', 'none'), 'bar') -# Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() -# Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() -BIG_ENDIAN = 0 -LITTLE_ENDIAN = 1 -EPSILON = { - 2: 2.0 ** -11, # binary16 - 4: 2.0 ** -24, # binary32 - 8: 2.0 ** -53, # binary64 -} - -@unittest.skipIf(_testcapi is None, 'needs _testcapi') -class PackTests(unittest.TestCase): - def test_pack(self): - self.assertEqual(_testcapi.float_pack(2, 1.5, BIG_ENDIAN), - b'>\x00') - self.assertEqual(_testcapi.float_pack(4, 1.5, BIG_ENDIAN), - b'?\xc0\x00\x00') - self.assertEqual(_testcapi.float_pack(8, 1.5, BIG_ENDIAN), - b'?\xf8\x00\x00\x00\x00\x00\x00') - self.assertEqual(_testcapi.float_pack(2, 1.5, LITTLE_ENDIAN), - b'\x00>') - self.assertEqual(_testcapi.float_pack(4, 1.5, LITTLE_ENDIAN), - b'\x00\x00\xc0?') - self.assertEqual(_testcapi.float_pack(8, 1.5, LITTLE_ENDIAN), - b'\x00\x00\x00\x00\x00\x00\xf8?') - - def test_unpack(self): - self.assertEqual(_testcapi.float_unpack(b'>\x00', BIG_ENDIAN), - 1.5) - self.assertEqual(_testcapi.float_unpack(b'?\xc0\x00\x00', BIG_ENDIAN), - 1.5) - self.assertEqual(_testcapi.float_unpack(b'?\xf8\x00\x00\x00\x00\x00\x00', BIG_ENDIAN), - 1.5) - self.assertEqual(_testcapi.float_unpack(b'\x00>', LITTLE_ENDIAN), - 1.5) - self.assertEqual(_testcapi.float_unpack(b'\x00\x00\xc0?', LITTLE_ENDIAN), - 1.5) - self.assertEqual(_testcapi.float_unpack(b'\x00\x00\x00\x00\x00\x00\xf8?', LITTLE_ENDIAN), - 1.5) - - def test_roundtrip(self): - large = 2.0 ** 100 - values = [1.0, 1.5, large, 1.0/7, math.pi] - if HAVE_IEEE_754: - values.extend((INF, NAN)) - for value in values: - for size in (2, 4, 8,): - if size == 2 and value == large: - # too large for 16-bit float - continue - rel_tol = EPSILON[size] - for endian in (BIG_ENDIAN, LITTLE_ENDIAN): - with self.subTest(value=value, size=size, endian=endian): - data = _testcapi.float_pack(size, value, endian) - value2 = _testcapi.float_unpack(data, endian) - if isnan(value): - self.assertTrue(isnan(value2), (value, value2)) - elif size < 8: - self.assertTrue(math.isclose(value2, value, rel_tol=rel_tol), - (value, value2)) - else: - self.assertEqual(value2, value) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 6fa49dbc0b730c..d2026152d8e747 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -4,6 +4,7 @@ import re import test.support as support import unittest +from test.support.import_helper import import_module maxsize = support.MAX_Py_ssize_t @@ -303,9 +304,9 @@ def test_str_format(self): test_exc('%c', sys.maxunicode+1, OverflowError, "%c arg not in range(0x110000)") #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") - test_exc('%c', 3.14, TypeError, "%c requires int or char") - test_exc('%c', 'ab', TypeError, "%c requires int or char") - test_exc('%c', b'x', TypeError, "%c requires int or char") + test_exc('%c', 3.14, TypeError, "%c requires an int or a unicode character, not float") + test_exc('%c', 'ab', TypeError, "%c requires an int or a unicode character, not a string of length 2") + test_exc('%c', b'x', TypeError, "%c requires an int or a unicode character, not bytes") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -369,11 +370,11 @@ def __bytes__(self): test_exc(b"%c", 2**128, OverflowError, "%c arg not in range(256)") test_exc(b"%c", b"Za", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not a bytes object of length 2") test_exc(b"%c", "Y", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not str") test_exc(b"%c", 3.14, TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not float") test_exc(b"%b", "Xc", TypeError, "%b requires a bytes-like object, " "or an object that implements __bytes__, not 'str'") @@ -478,7 +479,8 @@ def test_precision(self): @support.cpython_only def test_precision_c_limits(self): - from _testcapi import INT_MAX + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX f = 1.2 with self.assertRaises(ValueError) as cm: diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 499e3b6e656faa..589669298e22e2 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -91,6 +91,197 @@ class DummyFraction(fractions.Fraction): def _components(r): return (r.numerator, r.denominator) +def typed_approx_eq(a, b): + return type(a) == type(b) and (a == b or math.isclose(a, b)) + +class Symbolic: + """Simple non-numeric class for testing mixed arithmetic. + It is not Integral, Rational, Real or Complex, and cannot be conveted + to int, float or complex. but it supports some arithmetic operations. + """ + def __init__(self, value): + self.value = value + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} * {other}') + def __rmul__(self, other): + return self.__class__(f'{other} * {self}') + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} / {other}') + def __rtruediv__(self, other): + return self.__class__(f'{other} / {self}') + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} % {other}') + def __rmod__(self, other): + return self.__class__(f'{other} % {self}') + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} ** {other}') + def __rpow__(self, other): + return self.__class__(f'{other} ** {self}') + def __eq__(self, other): + if other.__class__ != self.__class__: + return NotImplemented + return self.value == other.value + def __str__(self): + return f'{self.value}' + def __repr__(self): + return f'{self.__class__.__name__}({self.value!r})' + +class SymbolicReal(Symbolic): + pass +numbers.Real.register(SymbolicReal) + +class SymbolicComplex(Symbolic): + pass +numbers.Complex.register(SymbolicComplex) + +class Rat: + """Simple Rational class for testing mixed arithmetic.""" + def __init__(self, n, d): + self.numerator = n + self.denominator = d + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.numerator, + self.denominator * other.denominator) + def __rmul__(self, other): + return self.__class__(other.numerator * self.numerator, + other.denominator * self.denominator) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.denominator, + self.denominator * other.numerator) + def __rtruediv__(self, other): + return self.__class__(other.numerator * self.denominator, + other.denominator * self.numerator) + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + d = self.denominator * other.numerator + return self.__class__(self.numerator * other.denominator % d, d) + def __rmod__(self, other): + d = other.denominator * self.numerator + return self.__class__(other.numerator * self.denominator % d, d) + + return self.__class__(other.numerator / self.numerator, + other.denominator / self.denominator) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator ** other, + self.denominator ** other) + def __float__(self): + return self.numerator / self.denominator + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return (typed_approx_eq(self.numerator, other.numerator) and + typed_approx_eq(self.denominator, other.denominator)) + def __repr__(self): + return f'{self.__class__.__name__}({self.numerator!r}, {self.denominator!r})' +numbers.Rational.register(Rat) + +class Root: + """Simple Real class for testing mixed arithmetic.""" + def __init__(self, v, n=F(2)): + self.base = v + self.degree = n + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base * other**self.degree, self.degree) + def __rmul__(self, other): + return self.__class__(other**self.degree * self.base, self.degree) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base / other**self.degree, self.degree) + def __rtruediv__(self, other): + return self.__class__(other**self.degree / self.base, self.degree) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base, self.degree / other) + def __float__(self): + return float(self.base) ** (1 / float(self.degree)) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.base, other.base) and typed_approx_eq(self.degree, other.degree) + def __repr__(self): + return f'{self.__class__.__name__}({self.base!r}, {self.degree!r})' +numbers.Real.register(Root) + +class Polar: + """Simple Complex class for testing mixed arithmetic.""" + def __init__(self, r, phi): + self.r = r + self.phi = phi + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r * other, self.phi) + def __rmul__(self, other): + return self.__class__(other * self.r, self.phi) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r / other, self.phi) + def __rtruediv__(self, other): + return self.__class__(other / self.r, -self.phi) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r ** other, self.phi * other) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.r, other.r) and typed_approx_eq(self.phi, other.phi) + def __repr__(self): + return f'{self.__class__.__name__}({self.r!r}, {self.phi!r})' +numbers.Complex.register(Polar) + +class Rect: + """Other simple Complex class for testing mixed arithmetic.""" + def __init__(self, x, y): + self.x = x + self.y = y + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x * other, self.y * other) + def __rmul__(self, other): + return self.__class__(other * self.x, other * self.y) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x / other, self.y / other) + def __rtruediv__(self, other): + r = self.x * self.x + self.y * self.y + return self.__class__(other * (self.x / r), other * (self.y / r)) + def __rpow__(self, other): + return Polar(other ** self.x, math.log(other) * self.y) + def __complex__(self): + return complex(self.x, self.y) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.x, other.x) and typed_approx_eq(self.y, other.y) + def __repr__(self): + return f'{self.__class__.__name__}({self.x!r}, {self.y!r})' +numbers.Complex.register(Rect) + +class RectComplex(Rect, complex): + pass class FractionTest(unittest.TestCase): @@ -261,6 +452,30 @@ def testFromString(self): self.assertRaisesMessage( ValueError, "Invalid literal for Fraction: '1.1e+1__1'", F, "1.1e+1__1") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '123.dd'", + F, "123.dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '123.5_dd'", + F, "123.5_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: 'dd.5'", + F, "dd.5") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '7_dd'", + F, "7_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '1/dd'", + F, "1/dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '1/123_dd'", + F, "1/123_dd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '789edd'", + F, "789edd") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '789e2_dd'", + F, "789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" self.assertRaisesMessage( @@ -569,6 +784,7 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.9, 1.0 - F(1, 10)) self.assertTypedEquals(0.9 + 0j, (1.0 + 0j) - F(1, 10)) + def testMixedMultiplication(self): self.assertTypedEquals(F(1, 10), F(1, 10) * 1) self.assertTypedEquals(0.1, F(1, 10) * 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) * (1.0 + 0j)) @@ -576,6 +792,29 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.1, 1.0 * F(1, 10)) self.assertTypedEquals(0.1 + 0j, (1.0 + 0j) * F(1, 10)) + self.assertTypedEquals(F(3, 2) * DummyFraction(5, 3), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) * F(3, 2), F(5, 2)) + self.assertTypedEquals(F(3, 2) * Rat(5, 3), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) * F(3, 2), F(5, 2)) + + self.assertTypedEquals(F(3, 2) * Root(4), Root(F(9, 1))) + self.assertTypedEquals(Root(4) * F(3, 2), 3.0) + self.assertEqual(F(3, 2) * SymbolicReal('X'), SymbolicReal('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) + self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) + self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0, 4.5)) + self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) + self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) + self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) * Symbolic('X'), Symbolic('3/2 * X')) + self.assertRaises(TypeError, operator.mul, Symbolic('X'), F(3, 2)) + + def testMixedDivision(self): self.assertTypedEquals(F(1, 10), F(1, 10) / 1) self.assertTypedEquals(0.1, F(1, 10) / 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) / (1.0 + 0j)) @@ -583,6 +822,28 @@ def testMixedArithmetic(self): self.assertTypedEquals(10.0, 1.0 / F(1, 10)) self.assertTypedEquals(10.0 + 0j, (1.0 + 0j) / F(1, 10)) + self.assertTypedEquals(F(3, 2) / DummyFraction(3, 5), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) / F(2, 3), F(5, 2)) + self.assertTypedEquals(F(3, 2) / Rat(3, 5), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) / F(2, 3), F(5, 2)) + + self.assertTypedEquals(F(2, 3) / Root(4), Root(F(1, 9))) + self.assertTypedEquals(Root(4) / F(2, 3), 3.0) + self.assertEqual(F(3, 2) / SymbolicReal('X'), SymbolicReal('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) / Polar(4, 2), Polar(F(3, 8), -2)) + self.assertTypedEquals(F(3, 2) / Polar(4.0, 2), Polar(0.375, -2)) + self.assertTypedEquals(F(3, 2) / Rect(4, 3), Rect(0.24, 0.18)) + self.assertRaises(TypeError, operator.truediv, Polar(4, 2), F(2, 3)) + self.assertTypedEquals(Rect(4, 3) / F(2, 3), 6.0 + 4.5j) + self.assertEqual(F(3, 2) / SymbolicComplex('X'), SymbolicComplex('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) / Symbolic('X'), Symbolic('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, Symbolic('X'), F(2, 3)) + + def testMixedIntegerDivision(self): self.assertTypedEquals(0, F(1, 10) // 1) self.assertTypedEquals(0.0, F(1, 10) // 1.0) self.assertTypedEquals(10, 1 // F(1, 10)) @@ -607,6 +868,26 @@ def testMixedArithmetic(self): self.assertTypedTupleEquals(divmod(-0.1, float('inf')), divmod(F(-1, 10), float('inf'))) self.assertTypedTupleEquals(divmod(-0.1, float('-inf')), divmod(F(-1, 10), float('-inf'))) + self.assertTypedEquals(F(3, 2) % DummyFraction(3, 5), F(3, 10)) + self.assertTypedEquals(DummyFraction(5, 3) % F(2, 3), F(1, 3)) + self.assertTypedEquals(F(3, 2) % Rat(3, 5), Rat(3, 6)) + self.assertTypedEquals(Rat(5, 3) % F(2, 3), F(1, 3)) + + self.assertRaises(TypeError, operator.mod, F(2, 3), Root(4)) + self.assertTypedEquals(Root(4) % F(3, 2), 0.5) + self.assertEqual(F(3, 2) % SymbolicReal('X'), SymbolicReal('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicReal('X'), F(3, 2)) + + self.assertRaises(TypeError, operator.mod, F(3, 2), Polar(4, 2)) + self.assertRaises(TypeError, operator.mod, F(3, 2), RectComplex(4, 3)) + self.assertRaises(TypeError, operator.mod, Rect(4, 3), F(2, 3)) + self.assertEqual(F(3, 2) % SymbolicComplex('X'), SymbolicComplex('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) % Symbolic('X'), Symbolic('3/2 % X')) + self.assertRaises(TypeError, operator.mod, Symbolic('X'), F(2, 3)) + + def testMixedPower(self): # ** has more interesting conversion rules. self.assertTypedEquals(F(100, 1), F(1, 10) ** -2) self.assertTypedEquals(F(100, 1), F(10, 1) ** 2) @@ -623,6 +904,40 @@ def testMixedArithmetic(self): self.assertRaises(ZeroDivisionError, operator.pow, F(0, 1), -2) + self.assertTypedEquals(F(3, 2) ** Rat(3, 1), F(27, 8)) + self.assertTypedEquals(F(3, 2) ** Rat(-3, 1), F(8, 27)) + self.assertTypedEquals(F(-3, 2) ** Rat(-3, 1), F(-8, 27)) + self.assertTypedEquals(F(9, 4) ** Rat(3, 2), 3.375) + self.assertIsInstance(F(4, 9) ** Rat(-3, 2), float) + self.assertAlmostEqual(F(4, 9) ** Rat(-3, 2), 3.375) + self.assertAlmostEqual(F(-4, 9) ** Rat(-3, 2), 3.375j) + self.assertTypedEquals(Rat(9, 4) ** F(3, 2), 3.375) + self.assertTypedEquals(Rat(3, 2) ** F(3, 1), Rat(27, 8)) + self.assertTypedEquals(Rat(3, 2) ** F(-3, 1), F(8, 27)) + self.assertIsInstance(Rat(4, 9) ** F(-3, 2), float) + self.assertAlmostEqual(Rat(4, 9) ** F(-3, 2), 3.375) + + self.assertTypedEquals(Root(4) ** F(2, 3), Root(4, 3.0)) + self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) + self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) + + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) + self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) + self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) + self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) + + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) + self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) + def testMixingWithDecimal(self): # Decimal refuses mixed arithmetic (but not mixed comparisons) self.assertRaises(TypeError, operator.add, @@ -849,12 +1164,50 @@ def denominator(self): self.assertEqual(type(f.denominator), myint) def test_format_no_presentation_type(self): - # Triples (fraction, specification, expected_result) + # Triples (fraction, specification, expected_result). testcases = [ - (F(1, 3), '', '1/3'), - (F(-1, 3), '', '-1/3'), - (F(3), '', '3'), - (F(-3), '', '-3'), + # Explicit sign handling + (F(2, 3), '+', '+2/3'), + (F(-2, 3), '+', '-2/3'), + (F(3), '+', '+3'), + (F(-3), '+', '-3'), + (F(2, 3), ' ', ' 2/3'), + (F(-2, 3), ' ', '-2/3'), + (F(3), ' ', ' 3'), + (F(-3), ' ', '-3'), + (F(2, 3), '-', '2/3'), + (F(-2, 3), '-', '-2/3'), + (F(3), '-', '3'), + (F(-3), '-', '-3'), + # Padding + (F(0), '5', ' 0'), + (F(2, 3), '5', ' 2/3'), + (F(-2, 3), '5', ' -2/3'), + (F(2, 3), '0', '2/3'), + (F(2, 3), '1', '2/3'), + (F(2, 3), '2', '2/3'), + # Alignment + (F(2, 3), '<5', '2/3 '), + (F(2, 3), '>5', ' 2/3'), + (F(2, 3), '^5', ' 2/3 '), + (F(2, 3), '=5', ' 2/3'), + (F(-2, 3), '<5', '-2/3 '), + (F(-2, 3), '>5', ' -2/3'), + (F(-2, 3), '^5', '-2/3 '), + (F(-2, 3), '=5', '- 2/3'), + # Fill + (F(2, 3), 'X>5', 'XX2/3'), + (F(-2, 3), '.<5', '-2/3.'), + (F(-2, 3), '\n^6', '\n-2/3\n'), + # Thousands separators + (F(1234, 5679), ',', '1,234/5,679'), + (F(-1234, 5679), '_', '-1_234/5_679'), + (F(1234567), '_', '1_234_567'), + (F(-1234567), ',', '-1,234,567'), + # Alternate form forces a slash in the output + (F(123), '#', '123/1'), + (F(-123), '#', '-123/1'), + (F(0), '#', '0/1'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1218,6 +1571,10 @@ def test_invalid_formats(self): '.%', # Z instead of z for negative zero suppression 'Z.2f' + # z flag not supported for general formatting + 'z', + # zero padding not supported for general formatting + '05', ] for spec in invalid_specs: with self.subTest(spec=spec): @@ -1248,6 +1605,39 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + + def test_three_argument_pow(self): + message = "unsupported operand type(s) for ** or pow(): '%s', '%s', '%s'" + self.assertRaisesMessage(TypeError, + message % ("Fraction", "int", "int"), + pow, F(3), 4, 5) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 9491c7facdf077..b7ef6cefaabbc0 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,10 +1,9 @@ -import gc +import copy import operator import re import sys import textwrap import threading -import types import unittest import weakref try: @@ -12,8 +11,9 @@ except ImportError: _testcapi = None +from collections.abc import Mapping from test import support -from test.support import threading_helper +from test.support import import_helper, threading_helper from test.support.script_helper import assert_python_ok @@ -55,6 +55,27 @@ class C: # The reference was released by .clear() self.assertIs(None, wr()) + def test_clear_locals_after_f_locals_access(self): + # see gh-113939 + class C: + pass + + wr = None + def inner(): + nonlocal wr + c = C() + wr = weakref.ref(c) + 1/0 + + try: + inner() + except ZeroDivisionError as exc: + support.gc_collect() + self.assertIsNotNone(wr()) + exc.__traceback__.tb_next.tb_frame.clear() + support.gc_collect() + self.assertIsNone(wr()) + def test_clear_does_not_clear_specials(self): class C: pass @@ -80,9 +101,11 @@ def g(): gen = g() next(gen) self.assertFalse(endly) - # Clearing the frame closes the generator - gen.gi_frame.clear() - self.assertTrue(endly) + + # Cannot clear a suspended frame + with self.assertRaisesRegex(RuntimeError, r'suspended frame'): + gen.gi_frame.clear() + self.assertFalse(endly) def test_clear_executing(self): # Attempting to clear an executing frame is forbidden. @@ -114,9 +137,10 @@ def g(): gen = g() f = next(gen) self.assertFalse(endly) - # Clearing the frame closes the generator - f.clear() - self.assertTrue(endly) + # Cannot clear a suspended frame + with self.assertRaisesRegex(RuntimeError, 'suspended frame'): + f.clear() + self.assertFalse(endly) def test_lineno_with_tracing(self): def record_line(): @@ -174,14 +198,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -193,8 +209,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -245,6 +261,209 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + with self.assertRaises(KeyError): + d['non_exist'] + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + + def test_local_objects(self): + o = object() + k = '.'.join(['a', 'b', 'c']) + f_locals = sys._getframe().f_locals + f_locals['o'] = f_locals['k'] + self.assertEqual(o, 'a.b.c') + + def test_copy(self): + x = 0 + d = sys._getframe().f_locals + d_copy = d.copy() + self.assertIsInstance(d_copy, dict) + self.assertEqual(d_copy['x'], 0) + d_copy['x'] = 1 + self.assertEqual(x, 0) + + def test_update_with_self(self): + def f(): + f_locals = sys._getframe().f_locals + f_locals.update(f_locals) + f_locals.update(f_locals) + f_locals.update(f_locals) + f() + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + del d['x'] + + with self.assertRaises(AttributeError): + d.clear() + + with self.assertRaises(AttributeError): + d.pop('x') + + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + def test_is_mapping(self): + x = 1 + d = sys._getframe().f_locals + self.assertIsInstance(d, Mapping) + match d: + case {"x": value}: + self.assertEqual(value, 1) + kind = "mapping" + case _: + kind = "other" + self.assertEqual(kind, "mapping") + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): @@ -269,69 +488,6 @@ def gen(): """) assert_python_ok("-c", code) - @support.cpython_only - def test_sneaky_frame_object(self): - - def trace(frame, event, arg): - """ - Don't actually do anything, just force a frame object to be created. - """ - - def callback(phase, info): - """ - Yo dawg, I heard you like frames, so I'm allocating a frame while - you're allocating a frame, so you can have a frame while you have a - frame! - """ - nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back.f_back - # We're done here: - gc.callbacks.remove(callback) - - def f(): - while True: - yield - - old_threshold = gc.get_threshold() - old_callbacks = gc.callbacks[:] - old_enabled = gc.isenabled() - old_trace = sys.gettrace() - try: - # Stop the GC for a second while we set things up: - gc.disable() - # Create a paused generator: - g = f() - next(g) - # Move all objects to the oldest generation, and tell the GC to run - # on the *very next* allocation: - gc.collect() - gc.set_threshold(1, 0, 0) - # Okay, so here's the nightmare scenario: - # - We're tracing the resumption of a generator, which creates a new - # frame object. - # - The allocation of this frame object triggers a collection - # *before* the frame object is actually created. - # - During the collection, we request the exact same frame object. - # This test does it with a GC callback, but in real code it would - # likely be a trace function, weakref callback, or finalizer. - # - The collection finishes, and the original frame object is - # created. We now have two frame objects fighting over ownership - # of the same interpreter frame! - sys.settrace(trace) - gc.callbacks.append(callback) - sneaky_frame_object = None - gc.enable() - next(g) - # g.gi_frame should be the frame object from the callback (the - # one that was *requested* second, but *created* first): - self.assertIs(g.gi_frame, sneaky_frame_object) - finally: - gc.set_threshold(*old_threshold) - gc.callbacks[:] = old_callbacks - sys.settrace(old_trace) - if old_enabled: - gc.enable() - @support.cpython_only @threading_helper.requires_working_threading() def test_sneaky_frame_object_teardown(self): diff --git a/Lib/test/test_free_threading/__init__.py b/Lib/test/test_free_threading/__init__.py new file mode 100644 index 00000000000000..34e1ad30e11097 --- /dev/null +++ b/Lib/test/test_free_threading/__init__.py @@ -0,0 +1,11 @@ +import os +import unittest + +from test import support + + +if not support.Py_GIL_DISABLED: + raise unittest.SkipTest("GIL enabled") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py new file mode 100644 index 00000000000000..3126458e08e50a --- /dev/null +++ b/Lib/test/test_free_threading/test_dict.py @@ -0,0 +1,182 @@ +import gc +import time +import unittest +import weakref + +from ast import Or +from functools import partial +from threading import Thread +from unittest import TestCase + +try: + import _testcapi +except ImportError: + _testcapi = None + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestDict(TestCase): + def test_racing_creation_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with shared keys""" + class C(int): + pass + + self.racing_creation(C) + + def test_racing_creation_no_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with an ordinary dict""" + self.racing_creation(Or) + + def test_racing_creation_inline_values_invalid(self): + """Verify that re-creating a dict after we have invalid inline values + is thread safe""" + class C: + pass + + def make_obj(): + a = C() + # Make object, make inline values invalid, and then delete dict + a.__dict__ = {} + del a.__dict__ + return a + + self.racing_creation(make_obj) + + def test_racing_creation_nonmanaged_dict(self): + """Verify that explicit creation of an unmanaged dict is thread safe + outside of the normal attribute setting code path""" + def make_obj(): + def f(): pass + return f + + def set(func, name, val): + # Force creation of the dict via PyObject_GenericGetDict + func.__dict__[name] = val + + self.racing_creation(make_obj, set) + + def racing_creation(self, cls, set=setattr): + objects = [] + processed = [] + + OBJECT_COUNT = 100 + THREAD_COUNT = 10 + CUR = 0 + + for i in range(OBJECT_COUNT): + objects.append(cls()) + + def writer_func(name): + last = -1 + while True: + if CUR == last: + continue + elif CUR == OBJECT_COUNT: + break + + obj = objects[CUR] + set(obj, name, name) + last = CUR + processed.append(name) + + writers = [] + for x in range(THREAD_COUNT): + writer = Thread(target=partial(writer_func, f"a{x:02}")) + writers.append(writer) + writer.start() + + for i in range(OBJECT_COUNT): + CUR = i + while len(processed) != THREAD_COUNT: + time.sleep(0.001) + processed.clear() + + CUR = OBJECT_COUNT + + for writer in writers: + writer.join() + + for obj_idx, obj in enumerate(objects): + assert ( + len(obj.__dict__) == THREAD_COUNT + ), f"{len(obj.__dict__)} {obj.__dict__!r} {obj_idx}" + for i in range(THREAD_COUNT): + assert f"a{i:02}" in obj.__dict__, f"a{i:02} missing at {obj_idx}" + + def test_racing_set_dict(self): + """Races assigning to __dict__ should be thread safe""" + + def f(): pass + l = [] + THREAD_COUNT = 10 + class MyDict(dict): pass + + def writer_func(l): + for i in range(1000): + d = MyDict() + l.append(weakref.ref(d)) + f.__dict__ = d + + lists = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + f.__dict__ = {} + gc.collect() + + for thread_list in lists: + for ref in thread_list: + self.assertIsNone(ref()) + + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_dict_version(self): + dict_version = _testcapi.dict_version + THREAD_COUNT = 10 + DICT_COUNT = 10000 + lists = [] + writers = [] + + def writer_func(thread_list): + for i in range(DICT_COUNT): + thread_list.append(dict_version({})) + + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + total_len = 0 + values = set() + for thread_list in lists: + for v in thread_list: + if v in values: + print('dup', v, (v/4096)%256) + values.add(v) + total_len += len(thread_list) + versions = set(dict_version for thread_list in lists for dict_version in thread_list) + self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py new file mode 100644 index 00000000000000..6ad806d67a80ed --- /dev/null +++ b/Lib/test/test_free_threading/test_list.py @@ -0,0 +1,80 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + +class C: + def __init__(self, v): + self.v = v + + +@threading_helper.requires_working_threading() +class TestList(TestCase): + def test_racing_iter_append(self): + + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.append(C(i + OBJECT_COUNT)) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_iter_extend(self): + iters = [ + lambda x: [x], + ] + for iter_case in iters: + with self.subTest(iter=iter_case): + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend(iter_case(C(i + OBJECT_COUNT))) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py new file mode 100644 index 00000000000000..3a3f1ba3b605d6 --- /dev/null +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -0,0 +1,245 @@ +"""Tests monitoring, sys.settrace, and sys.setprofile in a multi-threaded +environmenet to verify things are thread-safe in a free-threaded build""" + +import sys +import time +import unittest +import weakref + +from sys import monitoring +from test.support import threading_helper +from threading import Thread, _PyRLock +from unittest import TestCase + + +class InstrumentationMultiThreadedMixin: + thread_count = 10 + func_count = 200 + fib = 12 + + def after_threads(self): + """Runs once after all the threads have started""" + pass + + def during_threads(self): + """Runs repeatedly while the threads are still running""" + pass + + def work(self, n, funcs): + """Fibonacci function which also calls a bunch of random functions""" + for func in funcs: + func() + if n < 2: + return n + return self.work(n - 1, funcs) + self.work(n - 2, funcs) + + def start_work(self, n, funcs): + # With the GIL builds we need to make sure that the hooks have + # a chance to run as it's possible to run w/o releasing the GIL. + time.sleep(1) + self.work(n, funcs) + + def after_test(self): + """Runs once after the test is done""" + pass + + def test_instrumentation(self): + # Setup a bunch of functions which will need instrumentation... + funcs = [] + for i in range(self.func_count): + x = {} + exec("def f(): pass", x) + funcs.append(x["f"]) + + threads = [] + for i in range(self.thread_count): + # Each thread gets a copy of the func list to avoid contention + t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t.start() + threads.append(t) + + self.after_threads() + + while True: + any_alive = False + for t in threads: + if t.is_alive(): + any_alive = True + break + + if not any_alive: + break + + self.during_threads() + + self.after_test() + + +class MonitoringTestMixin: + def setUp(self): + for i in range(6): + if monitoring.get_tool(i) is None: + self.tool_id = i + monitoring.use_tool_id(i, self.__class__.__name__) + break + + def tearDown(self): + monitoring.free_tool_id(self.tool_id) + + +@threading_helper.requires_working_threading() +class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Sets tracing one time after the threads have started""" + + def setUp(self): + super().setUp() + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def after_threads(self): + sys.settrace(self.trace_func) + + +@threading_helper.requires_working_threading() +class MonitoringMultiThreaded( + MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase +): + """Uses sys.monitoring and repeatedly toggles instrumentation on and off""" + + def setUp(self): + super().setUp() + self.set = False + self.called = False + monitoring.register_callback( + self.tool_id, monitoring.events.LINE, self.callback + ) + + def tearDown(self): + monitoring.set_events(self.tool_id, 0) + super().tearDown() + + def callback(self, *args): + self.called = True + + def after_test(self): + self.assertTrue(self.called) + + def during_threads(self): + if self.set: + monitoring.set_events( + self.tool_id, monitoring.events.CALL | monitoring.events.LINE + ) + else: + monitoring.set_events(self.tool_id, 0) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.settrace and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.settrace(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.settrace(self.trace_func) + else: + sys.settrace(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.setprofile(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.setprofile(self.trace_func) + else: + sys.setprofile(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class MonitoringMisc(MonitoringTestMixin, TestCase): + def register_callback(self): + def callback(*args): + pass + + for i in range(200): + monitoring.register_callback(self.tool_id, monitoring.events.LINE, callback) + + self.refs.append(weakref.ref(callback)) + + def test_register_callback(self): + self.refs = [] + threads = [] + for i in range(50): + t = Thread(target=self.register_callback) + t.start() + threads.append(t) + + for thread in threads: + thread.join() + + monitoring.register_callback(self.tool_id, monitoring.events.LINE, None) + for ref in self.refs: + self.assertEqual(ref(), None) + + def test_set_local_trace_opcodes(self): + def trace(frame, event, arg): + frame.f_trace_opcodes = True + return trace + + sys.settrace(trace) + try: + l = _PyRLock() + + def f(): + for i in range(3000): + with l: + pass + + t = Thread(target=f) + t.start() + for i in range(3000): + with l: + pass + t.join() + finally: + sys.settrace(None) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py new file mode 100644 index 00000000000000..758f74f54d0b56 --- /dev/null +++ b/Lib/test/test_free_threading/test_slots.py @@ -0,0 +1,43 @@ +import threading +from test.support import threading_helper +from unittest import TestCase + + +def run_in_threads(targets): + """Run `targets` in separate threads""" + threads = [ + threading.Thread(target=target) + for target in targets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestSlots(TestCase): + + def test_object(self): + class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + spam = Spam(0) + iters = 20_000 + + def writer(): + for _ in range(iters): + spam.eggs += 1 + + def reader(): + for _ in range(iters): + eggs = spam.eggs + assert type(eggs) is int + assert 0 <= eggs <= iters + + run_in_threads([writer, reader, reader, reader]) diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py new file mode 100644 index 00000000000000..72044e979b0f48 --- /dev/null +++ b/Lib/test/test_free_threading/test_str.py @@ -0,0 +1,74 @@ +import unittest + +from itertools import cycle +from threading import Event, Thread +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestStr(TestCase): + def test_racing_join_extend(self): + '''Test joining a string being extended by another thread''' + l = [] + ITERS = 100 + READERS = 10 + done_event = Event() + def writer_func(): + for i in range(ITERS): + l.extend(map(str, range(i))) + l.clear() + done_event.set() + def reader_func(): + while not done_event.is_set(): + ''.join(l) + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_join_replace(self): + ''' + Test joining a string of characters being replaced with ephemeral + strings by another thread. + ''' + l = [*'abcdefg'] + MAX_ORDINAL = 1_000 + READERS = 10 + done_event = Event() + + def writer_func(): + for i, c in zip(cycle(range(len(l))), + map(chr, range(128, MAX_ORDINAL))): + l[i] = c + done_event.set() + + def reader_func(): + while not done_event.is_set(): + ''.join(l) + ''.join(l) + ''.join(l) + ''.join(l) + + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py new file mode 100644 index 00000000000000..649676db9c08a5 --- /dev/null +++ b/Lib/test/test_free_threading/test_type.py @@ -0,0 +1,139 @@ +import threading +import unittest + +from concurrent.futures import ThreadPoolExecutor +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + + +NTHREADS = 6 +BOTTOM = 0 +TOP = 1000 +ITERS = 100 + +class A: + attr = 1 + +@threading_helper.requires_working_threading() +class TestType(TestCase): + def test_attr_cache(self): + def read(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + A.attr + + def write(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + # Make _PyType_Lookup cache hot first + A.attr + A.attr + x = A.attr + x += 1 + A.attr = x + + + with ThreadPoolExecutor(NTHREADS) as pool: + pool.submit(read, (1,)) + pool.submit(write, (1,)) + pool.shutdown(wait=True) + + def test_attr_cache_consistency(self): + class C: + x = 0 + + DONE = False + def writer_func(): + for i in range(3000): + C.x + C.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = C.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test_attr_cache_consistency_subclass(self): + class C: + x = 0 + + class D(C): + pass + + DONE = False + def writer_func(): + for i in range(3000): + D.x + D.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = D.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test___class___modification(self): + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(5000): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for i in range(NTHREADS): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + def run_one(self, writer_func, reader_func): + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index dd8c2dd628ee13..49c6f761e5b4f0 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,12 +8,12 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import dis import os import re import types import decimal import unittest -import warnings from test import support from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok @@ -1627,6 +1627,9 @@ def __repr__(self): self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') self.assertEqual(f"sadsd {1 + 1 = :{1 + 1:1d}f}", "sadsd 1 + 1 = 2.000000") + self.assertEqual(f"{1+2 = # my comment + }", '1+2 = \n 3') + # These next lines contains tabs. Backslash escapes don't # work in f-strings. # patchcheck doesn't like these tabs. So the only way to test @@ -1735,5 +1738,14 @@ def test_syntax_warning_infinite_recursion_in_file(self): self.assertIn(rb'\1', stdout) self.assertEqual(len(stderr.strip().splitlines()), 2) + def test_fstring_without_formatting_bytecode(self): + # f-string without any formatting should emit the same bytecode + # as a normal string. See gh-99606. + def get_code(s): + return [(i.opname, i.oparg) for i in dis.get_instructions(s)] + + for s in ["", "some string"]: + self.assertEqual(get_code(f"'{s}'"), get_code(f"f'{s}'")) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2f191ea7a44c16..bed0e6d973b8da 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -542,8 +543,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') @@ -900,6 +901,7 @@ def retr(): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -916,6 +918,7 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index 35b473d5e9a0b4..b3fc5ad42e7fde 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -2,6 +2,7 @@ import types import typing import unittest +import warnings def global_function(): @@ -70,6 +71,27 @@ def test(): pass test.__code__ = self.b.__code__ self.assertEqual(test(), 3) # self.b always returns 3, arbitrarily + def test_invalid___code___assignment(self): + def A(): pass + def B(): yield + async def C(): yield + async def D(x): await x + + for src in [A, B, C, D]: + for dst in [A, B, C, D]: + if src == dst: + continue + + assert src.__code__.co_flags != dst.__code__.co_flags + prev = dst.__code__ + try: + with self.assertWarnsRegex(DeprecationWarning, 'code object of non-matching type'): + dst.__code__ = src.__code__ + finally: + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + dst.__code__ = prev + def test___globals__(self): self.assertIs(self.b.__globals__, globals()) self.cannot_set_attr(self.b, '__globals__', 2, diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index e4de2c5ede15f1..492a16a8c7ff45 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -31,10 +31,6 @@ decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal']) -_partial_types = [py_functools.partial] -if c_functools: - _partial_types.append(c_functools.partial) - @contextlib.contextmanager def replaced_module(name, replacement): @@ -189,6 +185,19 @@ def test_nested_optimization(self): flat = partial(signature, 'asdf', bar=True) self.assertEqual(signature(nested), signature(flat)) + def test_nested_optimization_bug(self): + partial = self.partial + class Builder: + def __call__(self, tag, *children, **attrib): + return (tag, children, attrib) + + def __getattr__(self, tag): + return partial(self, tag) + + B = Builder() + m = B.m + assert m(1, 2, a=2) == ('m', (1, 2), dict(a=2)) + def test_nested_partial_with_attribute(self): # see issue 25137 partial = self.partial @@ -207,10 +216,7 @@ def test_repr(self): kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) self.assertEqual(f'{name}({capture!r})', repr(f)) @@ -229,10 +235,7 @@ def test_repr(self): for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) f.__setstate__((f, (), {}, {})) @@ -344,8 +347,10 @@ def test_recursive_pickle(self): f.__setstate__((f, (), {}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) + # gh-117008: Small limit since pickle uses C stack memory + with support.infinite_recursion(100): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) finally: f.__setstate__((capture, (), {}, {})) @@ -390,6 +395,21 @@ def __getitem__(self, key): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) + def test_partial_as_method(self): + class A: + meth = self.partial(capture, 1, a=2) + cmeth = classmethod(self.partial(capture, 1, a=2)) + smeth = staticmethod(self.partial(capture, 1, a=2)) + + a = A() + self.assertEqual(A.meth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.meth(3, b=4), ((1, a, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: @@ -564,6 +584,14 @@ class B: method = functools.partialmethod(func=capture, a=1) def test_repr(self): + self.assertEqual(repr(vars(self.A)['nothing']), + 'functools.partialmethod({})'.format(capture)) + self.assertEqual(repr(vars(self.A)['positional']), + 'functools.partialmethod({}, 1)'.format(capture)) + self.assertEqual(repr(vars(self.A)['keywords']), + 'functools.partialmethod({}, a=2)'.format(capture)) + self.assertEqual(repr(vars(self.A)['spec_keywords']), + 'functools.partialmethod({}, self=1, func=2)'.format(capture)) self.assertEqual(repr(vars(self.A)['both']), 'functools.partialmethod({}, 3, b=4)'.format(capture)) @@ -705,6 +733,14 @@ def wrapper(): self.assertTrue(wrapper.__doc__.startswith('max(')) self.assertEqual(wrapper.__annotations__, {}) + def test_update_type_wrapper(self): + def wrapper(*args): pass + + functools.update_wrapper(wrapper, type) + self.assertEqual(wrapper.__name__, 'type') + self.assertEqual(wrapper.__annotations__, {}) + self.assertEqual(wrapper.__type_params__, ()) + class TestWraps(TestUpdateWrapper): @@ -939,9 +975,16 @@ def mycmp(x, y): self.assertRaises(TypeError, hash, k) self.assertNotIsInstance(k, collections.abc.Hashable) + @unittest.skipIf(support.MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): - self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), - '(mycmp)') + sig = Signature.from_callable(self.cmp_to_key) + self.assertEqual(str(sig), '(mycmp)') + def mycmp(x, y): + return y - x + sig = Signature.from_callable(self.cmp_to_key(mycmp)) + self.assertEqual(str(sig), '(obj)') + @unittest.skipUnless(c_functools, 'requires the C _functools module') @@ -1826,6 +1869,7 @@ def f(): return 1 self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True}) + @support.suppress_immortalization() def test_lru_cache_weakrefable(self): @self.module.lru_cache def test_function(x): @@ -1856,12 +1900,33 @@ def test_staticmethod(x): self.assertIsNone(ref()) def test_common_signatures(self): - def orig(): ... + def orig(a, /, b, c=True): ... lru = self.module.lru_cache(1)(orig) + self.assertEqual(str(Signature.from_callable(lru)), '(a, /, b, c=True)') self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + @support.skip_on_s390x + @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + def test_lru_recursion(self): + + @self.module.lru_cache + def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + + if not support.Py_DEBUG: + depth = support.get_c_recursion_limit()*2//7 + with support.infinite_recursion(): + fib(depth) + if self.module == c_functools: + fib.cache_clear() + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + fib(10000) + @py_functools.lru_cache() def py_cached_func(x, y): @@ -2684,7 +2749,10 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual(A.func.__name__, 'func') @@ -2773,7 +2841,10 @@ def decorated_classmethod(cls, arg: int) -> str: WithSingleDispatch().decorated_classmethod ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual( @@ -2845,11 +2916,26 @@ def _(arg: typing.Union[int, typing.Iterable[str]]): def test_invalid_positional_argument(self): @functools.singledispatch - def f(*args): + def f(*args, **kwargs): pass msg = 'f requires at least 1 positional argument' with self.assertRaisesRegex(TypeError, msg): f() + msg = 'f requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + f(a=1) + + def test_invalid_positional_argument_singledispatchmethod(self): + class A: + @functools.singledispatchmethod + def t(self, *args, **kwargs): + pass + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t() + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t(a=1) def test_union(self): @functools.singledispatch @@ -3101,7 +3187,10 @@ def test_access_from_class(self): self.assertIsInstance(CachedCostItem.cost, py_functools.cached_property) def test_doc(self): - self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + self.assertEqual(CachedCostItem.cost.__doc__, + ("The cost of the item." + if support.HAVE_DOCSTRINGS + else None)) def test_module(self): self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) diff --git a/Lib/test/test_future_stmt/nested_scope.py b/Lib/test/test_future_stmt/nested_scope.py index 3d7fc860a37655..a8433a42cbb6b0 100644 --- a/Lib/test/test_future_stmt/nested_scope.py +++ b/Lib/test/test_future_stmt/nested_scope.py @@ -1,6 +1,6 @@ """This is a test""" -from __future__ import nested_scopes; import site +from __future__ import nested_scopes; import site # noqa: F401 def f(x): def g(y): diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 2c8ceb664cb362..44512e0101dac0 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -67,19 +67,19 @@ def test_future_single_import(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_single_import', ): - from test.test_future_stmt import test_future_single_import + from test.test_future_stmt import test_future_single_import # noqa: F401 def test_future_multiple_imports(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_multiple_imports', ): - from test.test_future_stmt import test_future_multiple_imports + from test.test_future_stmt import test_future_multiple_imports # noqa: F401 def test_future_multiple_features(self): with import_helper.CleanImport( "test.test_future_stmt.test_future_multiple_features", ): - from test.test_future_stmt import test_future_multiple_features + from test.test_future_stmt import test_future_multiple_features # noqa: F401 def test_unknown_future_flag(self): code = """ @@ -153,7 +153,7 @@ def test_future_import_braces(self): def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future + from test.test_future_stmt import badsyntax_future # noqa: F401 self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): @@ -171,26 +171,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -203,6 +183,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index db7cb9ace6e5f3..906f9884d6792f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,11 +1,14 @@ import unittest import unittest.mock +from test import support from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess) + cpython_only, requires_subprocess, + requires_gil_enabled, suppress_immortalization, + Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script -from test.support import threading_helper +from test.support import threading_helper, gc_threshold import gc import sys @@ -16,17 +19,16 @@ import weakref try: + import _testcapi from _testcapi import with_tp_del + from _testcapi import ContainerNoGC except ImportError: + _testcapi = None def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C - -try: - from _testcapi import ContainerNoGC -except ImportError: ContainerNoGC = None ### Support code @@ -108,6 +110,7 @@ def test_tuple(self): del l self.assertEqual(gc.collect(), 2) + @suppress_immortalization() def test_class(self): class A: pass @@ -116,6 +119,7 @@ class A: del A self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newstyleclass(self): class A(object): pass @@ -132,6 +136,7 @@ class A: del a self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newinstance(self): class A(object): pass @@ -218,6 +223,7 @@ class B(object): self.fail("didn't find obj in garbage (finalizer)") gc.garbage.remove(obj) + @suppress_immortalization() def test_function(self): # Tricky: f -> d -> f, code should call d.clear() after the exec to # break the cycle. @@ -225,7 +231,9 @@ def test_function(self): exec("def f(): pass\n", d) gc.collect() del d - self.assertEqual(gc.collect(), 2) + # In the free-threaded build, the count returned by `gc.collect()` + # is 3 because it includes f's code object. + self.assertIn(gc.collect(), (2, 3)) def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 @@ -363,6 +371,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test + @requires_gil_enabled('needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -383,19 +392,11 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in gen 1 + # x is now in the old gen a, b, c = gc.get_count() - gc.collect(1) - # x is now in gen 2 - d, e, f = gc.get_count() - gc.collect(2) - # x is now in gen 3 - g, h, i = gc.get_count() - # We don't check a, d, g since their exact values depends on + # We don't check a since its exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) - self.assertEqual((e, f), (0, 1)) - self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -477,7 +478,7 @@ def run_thread(): make_nested() old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) try: exit = [] threads = [] @@ -565,6 +566,7 @@ def test_get_referents(self): self.assertEqual(gc.get_referents(1, 'a', 4j), []) + @suppress_immortalization() def test_is_tracked(self): # Atomic built-in types are not tracked, user-defined objects and # mutable containers are. @@ -602,7 +604,9 @@ class UserFloatSlots(float): class UserIntSlots(int): __slots__ = () - self.assertTrue(gc.is_tracked(gc)) + if not Py_GIL_DISABLED: + # gh-117783: modules may be immortalized in free-threaded build + self.assertTrue(gc.is_tracked(gc)) self.assertTrue(gc.is_tracked(UserClass)) self.assertTrue(gc.is_tracked(UserClass())) self.assertTrue(gc.is_tracked(UserInt())) @@ -688,6 +692,7 @@ def do_work(): @cpython_only @requires_subprocess() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_garbage_at_shutdown(self): import subprocess code = """if 1: @@ -819,44 +824,21 @@ def test_get_objects(self): l = [] l.append(l) self.assertTrue( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=0) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) + any(l is element for element in gc.get_objects()) ) + + @requires_gil_enabled('need generational GC') + def test_get_objects_generations(self): + gc.collect() + l = [] + l.append(l) self.assertTrue( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=1) - self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=2) + gc.collect() self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) del l gc.collect() @@ -1067,6 +1049,71 @@ class Z: gc.enable() +class IncrementalGCTests(unittest.TestCase): + + def setUp(self): + # Reenable GC as it is disabled module-wide + gc.enable() + + def tearDown(self): + gc.disable() + + @requires_gil_enabled("Free threading does not support incremental GC") + # Use small increments to emulate longer running process in a shorter time + @gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + count = 1000 + + # There will be some objects we aren't counting, + # e.g. the gc stats dicts. This test checks + # that the counts don't grow, so we try to + # correct for the uncounted objects + # This is just an estimate. + CORRECTION = 20 + + enabled = gc.isenabled() + gc.enable() + olds = [] + for i in range(20_000): + newhead = make_ll(20) + count += 20 + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + stats = gc.get_stats() + young = stats[0] + incremental = stats[1] + old = stats[2] + collected = young['collected'] + incremental['collected'] + old['collected'] + count += CORRECTION + live = count - collected + self.assertLess(live, 25000) + del olds[:] + if not enabled: + gc.disable() + + class GCCallbackTests(unittest.TestCase): def setUp(self): # Save gc state and disable it. @@ -1188,6 +1235,7 @@ def test_collect_garbage(self): self.assertEqual(len(gc.garbage), 0) + @requires_subprocess() @unittest.skipIf(BUILD_WITH_NDEBUG, 'built with -NDEBUG') def test_refcount_errors(self): @@ -1225,7 +1273,7 @@ def test_refcount_errors(self): p.stderr.close() # Verify that stderr has a useful error message: self.assertRegex(stderr, - br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.') + br'gc.*\.c:[0-9]+: .*: Assertion "gc_get_refs\(.+\) .*" failed.') self.assertRegex(stderr, br'refcount is too small') # "address : 0x7fb5062efc18" @@ -1307,6 +1355,10 @@ def callback(ignored): junk = [] i = 0 detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1320,6 +1372,7 @@ def callback(ignored): # with an empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1374,6 +1427,10 @@ def __del__(self): detector = GC_Detector() junk = [] i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1387,6 +1444,32 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) + def test_indirect_calls_with_gc_disabled(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + junk.append([]) # this will eventually trigger gc + + try: + gc.disable() + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 10001) + finally: + gc.enable() + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index c41e7cb7c210de..714853c7b4732d 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -49,6 +49,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_gil_enabled() @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 7f4e3cba3534bd..8fe9cfc543395e 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -1,6 +1,7 @@ import os import re import shlex +import shutil import subprocess import sys import sysconfig @@ -8,6 +9,8 @@ from test import support +GDB_PROGRAM = shutil.which('gdb') or 'gdb' + # Location of custom hooks file in a repository checkout. CHECKOUT_HOOK_PATH = os.path.join(os.path.dirname(sys.executable), 'python-gdb.py') @@ -27,7 +30,7 @@ def clean_environment(): # Temporary value until it's initialized by get_gdb_version() below GDB_VERSION = (0, 0) -def run_gdb(*args, exitcode=0, **env_vars): +def run_gdb(*args, exitcode=0, check=True, **env_vars): """Runs gdb in --batch mode with the additional arguments given by *args. Returns its (stdout, stderr) decoded from utf-8 using the replace handler. @@ -36,7 +39,7 @@ def run_gdb(*args, exitcode=0, **env_vars): if env_vars: env.update(env_vars) - cmd = ['gdb', + cmd = [GDB_PROGRAM, # Batch mode: Exit after processing all the command files # specified with -x/--command '--batch', @@ -59,7 +62,7 @@ def run_gdb(*args, exitcode=0, **env_vars): stdout = proc.stdout stderr = proc.stderr - if proc.returncode != exitcode: + if check and proc.returncode != exitcode: cmd_text = shlex.join(cmd) raise Exception(f"{cmd_text} failed with exit code {proc.returncode}, " f"expected exit code {exitcode}:\n" @@ -72,10 +75,10 @@ def run_gdb(*args, exitcode=0, **env_vars): def get_gdb_version(): try: stdout, stderr = run_gdb('--version') - except OSError: + except OSError as exc: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. - raise unittest.SkipTest("Couldn't find gdb program on the path") + raise unittest.SkipTest(f"Couldn't find gdb program on the path: {exc}") # Regex to parse: # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7 @@ -106,7 +109,8 @@ def check_usable_gdb(): # disallow this without a customized .gdbinit. stdout, stderr = run_gdb( '--eval-command=python import sys; print(sys.version_info)', - '--args', sys.executable) + '--args', sys.executable, + check=False) if "auto-loading has been declined" in stderr: raise unittest.SkipTest( @@ -144,6 +148,7 @@ def setup_module(): print(f"gdb version {GDB_VERSION[0]}.{GDB_VERSION[1]}:") for line in GDB_VERSION_TEXT.splitlines(): print(" " * 4 + line) + print(f" path: {GDB_PROGRAM}") print() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 790e6b1b1b91e6..00def509a219c3 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -9,29 +9,31 @@ def skip_if_different_mount_drives(): - if sys.platform != 'win32': + if sys.platform != "win32": return ROOT = os.path.dirname(os.path.dirname(__file__)) root_drive = os.path.splitroot(ROOT)[0] cwd_drive = os.path.splitroot(os.getcwd())[0] if root_drive != cwd_drive: - # generate_cases.py uses relpath() which raises ValueError if ROOT - # and the current working different have different mount drives - # (on Windows). + # May raise ValueError if ROOT and the current working + # different have different mount drives (on Windows). raise unittest.SkipTest( f"the current working directory and the Python source code " f"directory have different mount drives " f"({cwd_drive} and {root_drive})" ) + + skip_if_different_mount_drives() -test_tools.skip_if_missing('cases_generator') -with test_tools.imports_under_tool('cases_generator'): - import generate_cases - import analysis - import formatting - from parsing import StackEffect +test_tools.skip_if_missing("cases_generator") +with test_tools.imports_under_tool("cases_generator"): + from analyzer import StackItem + import parser + from stack import Stack + import tier1_generator + import optimizer_generator def handle_stderr(): @@ -40,39 +42,27 @@ def handle_stderr(): else: return support.captured_stderr() + class TestEffects(unittest.TestCase): def test_effect_sizes(self): - input_effects = [ - x := StackEffect("x", "", "", ""), - y := StackEffect("y", "", "", "oparg"), - z := StackEffect("z", "", "", "oparg*2"), - ] - output_effects = [ - StackEffect("a", "", "", ""), - StackEffect("b", "", "", "oparg*4"), - StackEffect("c", "", "", ""), + stack = Stack() + inputs = [ + x := StackItem("x", None, "", "1"), + y := StackItem("y", None, "", "oparg"), + z := StackItem("z", None, "", "oparg*2"), ] - other_effects = [ - StackEffect("p", "", "", "oparg<<1"), - StackEffect("q", "", "", ""), - StackEffect("r", "", "", ""), + outputs = [ + StackItem("x", None, "", "1"), + StackItem("b", None, "", "oparg*4"), + StackItem("c", None, "", "1"), ] - self.assertEqual(formatting.effect_size(x), (1, "")) - self.assertEqual(formatting.effect_size(y), (0, "oparg")) - self.assertEqual(formatting.effect_size(z), (0, "oparg*2")) - - self.assertEqual( - formatting.list_effect_size(input_effects), - (1, "oparg + oparg*2"), - ) - self.assertEqual( - formatting.list_effect_size(output_effects), - (2, "oparg*4"), - ) - self.assertEqual( - formatting.list_effect_size(other_effects), - (2, "(oparg<<1)"), - ) + stack.pop(z) + stack.pop(y) + stack.pop(x) + for out in outputs: + stack.push(out) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg*2 - oparg") + self.assertEqual(stack.top_offset.to_c(), "1 - oparg*2 - oparg + oparg*4") class TestGeneratedCases(unittest.TestCase): @@ -103,32 +93,31 @@ def tearDown(self) -> None: def run_cases_test(self, input: str, expected: str): with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(analysis.BEGIN_MARKER) + temp_input.write(parser.BEGIN_MARKER) temp_input.write(input) - temp_input.write(analysis.END_MARKER) + temp_input.write(parser.END_MARKER) temp_input.flush() - a = generate_cases.Generator([self.temp_input_filename]) with handle_stderr(): - a.parse() - a.analyze() - if a.errors: - raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions(self.temp_output_filename, False) + tier1_generator.generate_tier1_from_files( + [self.temp_input_filename], self.temp_output_filename, False + ) with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() - while lines and lines[0].startswith("// "): + while lines and lines[0].startswith(("// ", "#", " #", "\n")): lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) actual = "".join(lines) - # if actual.rstrip() != expected.rstrip(): + # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) # print("Expected:") # print(expected) # print("End") - self.assertEqual(actual.rstrip(), expected.rstrip()) + self.assertEqual(actual.strip(), expected.strip()) def test_inst_no_args(self): input = """ @@ -138,6 +127,9 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); spam(); DISPATCH(); } @@ -152,10 +144,14 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; value = stack_pointer[-1]; spam(); - STACK_SHRINK(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -169,10 +165,14 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef res; spam(); - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -186,8 +186,11 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; spam(); stack_pointer[-1] = res; @@ -204,14 +207,18 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - PyObject *right; - PyObject *left; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; spam(); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -225,9 +232,12 @@ def test_overlap(self): """ output = """ TARGET(OP) { - PyObject *right; - PyObject *left; - PyObject *result; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef right; + _PyStackRef left; + _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; spam(); @@ -249,18 +259,24 @@ def test_predictions_and_eval_breaker(self): """ output = """ TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); PREDICTED(OP1); - static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - PyObject *arg; - PyObject *rest; + _PyStackRef arg; + _PyStackRef rest; arg = stack_pointer[-1]; stack_pointer[-1] = rest; DISPATCH(); } TARGET(OP3) { - PyObject *arg; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef res; arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); stack_pointer[-1] = res; @@ -278,6 +294,9 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; DISPATCH(); } @@ -292,7 +311,11 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; + // Comment is ok DISPATCH(); } """ @@ -306,14 +329,18 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - PyObject *right; - PyObject *left; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (cond) goto pop_2_label; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -326,12 +353,18 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - PyObject *value; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + _PyStackRef value; value = stack_pointer[-1]; - uint16_t counter = read_u16(&next_instr[0].cache); - uint32_t extra = read_u32(&next_instr[1].cache); - STACK_SHRINK(1); - next_instr += 3; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -345,6 +378,9 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); goto somewhere; } """ @@ -365,55 +401,128 @@ def test_macro_instruction(self): family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ output = """ - TARGET(OP1) { - PyObject *right; - PyObject *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&next_instr[0].cache); - op1(left, right); - next_instr += 1; - DISPATCH(); - } - TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); PREDICTED(OP); - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; - // OP1 + _Py_CODEUNIT *this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 right = stack_pointer[-1]; left = stack_pointer[-2]; { - uint16_t counter = read_u16(&next_instr[0].cache); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; op1(left, right); } + /* Skip 2 cache entries */ // OP2 arg2 = stack_pointer[-3]; { - uint32_t extra = read_u32(&next_instr[3].cache); + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; res = op2(arg2, left, right); } - STACK_SHRINK(2); - stack_pointer[-1] = res; - next_instr += 5; + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(OP1) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + op1(left, right); DISPATCH(); } TARGET(OP3) { - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef arg2; + _PyStackRef res; + /* Skip 5 cache entries */ right = stack_pointer[-1]; left = stack_pointer[-2]; arg2 = stack_pointer[-3]; res = op3(arg2, left, right); - STACK_SHRINK(2); - stack_pointer[-1] = res; - next_instr += 5; + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_caches(self): + input = """ + inst(OP, (unused/1, unused/2 --)) { + body(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + body(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_no_flags(self): + input = """ + pseudo(OP, (in -- out1, out2)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_with_flags(self): + input = """ + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); DISPATCH(); } """ @@ -427,15 +536,18 @@ def test_array_input(self): """ output = """ TARGET(OP) { - PyObject *above; - PyObject **values; - PyObject *below; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef above; + _PyStackRef *values; + _PyStackRef below; above = stack_pointer[-1]; - values = stack_pointer - 1 - oparg*2; + values = &stack_pointer[-1 - oparg*2]; below = stack_pointer[-2 - oparg*2]; spam(); - STACK_SHRINK(oparg*2); - STACK_SHRINK(2); + stack_pointer += -2 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -449,14 +561,18 @@ def test_array_output(self): """ output = """ TARGET(OP) { - PyObject *below; - PyObject **values; - PyObject *above; - values = stack_pointer - 1; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-1]; spam(values, oparg); - STACK_GROW(oparg*3); - stack_pointer[-2 - oparg*3] = below; - stack_pointer[-1] = above; + stack_pointer[-2] = below; + stack_pointer[-1 + oparg*3] = above; + stack_pointer += oparg*3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -470,12 +586,16 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - PyObject **values; - PyObject *above; - values = stack_pointer - oparg; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-oparg]; spam(values, oparg); - STACK_GROW(1); - stack_pointer[-1] = above; + stack_pointer[0] = above; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -489,13 +609,16 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - PyObject **values; - PyObject *extra; - values = stack_pointer - oparg; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *values; + _PyStackRef extra; + values = &stack_pointer[-oparg]; extra = stack_pointer[-1 - oparg]; - if (oparg == 0) { STACK_SHRINK(oparg); goto pop_1_somewhere; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); + if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -509,21 +632,24 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { - PyObject *cc; - PyObject *input = NULL; - PyObject *aa; - PyObject *xx; - PyObject *output = NULL; - PyObject *zz; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef cc; + _PyStackRef input = PyStackRef_NULL; + _PyStackRef aa; + _PyStackRef xx; + _PyStackRef output = PyStackRef_NULL; + _PyStackRef zz; cc = stack_pointer[-1]; - if ((oparg & 1) == 1) { input = stack_pointer[-1 - ((oparg & 1) == 1 ? 1 : 0)]; } - aa = stack_pointer[-2 - ((oparg & 1) == 1 ? 1 : 0)]; + if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } + aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; output = spam(oparg, input); - STACK_SHRINK((((oparg & 1) == 1) ? 1 : 0)); - STACK_GROW(((oparg & 2) ? 1 : 0)); - stack_pointer[-2 - (oparg & 2 ? 1 : 0)] = xx; - if (oparg & 2) { stack_pointer[-1 - (oparg & 2 ? 1 : 0)] = output; } - stack_pointer[-1] = zz; + stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; + if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; + stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; + stack_pointer += -(((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -541,12 +667,15 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { - PyObject *right; - PyObject *middle; - PyObject *left; - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + _PyStackRef right; + _PyStackRef middle; + _PyStackRef left; + _PyStackRef deep; + _PyStackRef extra = PyStackRef_NULL; + _PyStackRef res; // A right = stack_pointer[-1]; middle = stack_pointer[-2]; @@ -558,11 +687,11 @@ def test_macro_cond_effect(self): { # Body of B } - STACK_SHRINK(1); - STACK_GROW((oparg ? 1 : 0)); - stack_pointer[-2 - (oparg ? 1 : 0)] = deep; - if (oparg) { stack_pointer[-1 - (oparg ? 1 : 0)] = extra; } - stack_pointer[-1] = res; + stack_pointer[-3] = deep; + if (oparg) stack_pointer[-2] = extra; + stack_pointer[-2 + ((oparg) ? 1 : 0)] = res; + stack_pointer += -1 + ((oparg) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -580,8 +709,11 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { - PyObject *val1; - PyObject *val2; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + _PyStackRef val1; + _PyStackRef val2; // A { val1 = spam(); @@ -590,9 +722,10 @@ def test_macro_push_push(self): { val2 = spam(); } - STACK_GROW(2); - stack_pointer[-2] = val1; - stack_pointer[-1] = val2; + stack_pointer[0] = val1; + stack_pointer[1] = val2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -609,6 +742,9 @@ def test_override_inst(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); ham(); DISPATCH(); } @@ -627,12 +763,261 @@ def test_override_op(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + ham(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_inst(self): + input = """ + pure inst(OP, (--)) { + ham(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); ham(); DISPATCH(); } """ self.run_cases_test(input, output) + def test_annotated_op(self): + input = """ + pure op(OP, (--)) { + spam(); + } + macro(M) = OP; + """ + output = """ + TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + spam(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + input = """ + pure register specializing op(OP, (--)) { + spam(); + } + macro(M) = OP; + """ + self.run_cases_test(input, output) + + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(Exception): + self.run_cases_test(input, output) + + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pointer_to_stackref(self): + input = """ + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; + stack_pointer[-1] = out; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + spam(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + op(OP2, (arg1 -- out)) { + eggs(); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + eggs(); + } + """ + output = """ + case OP: { + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; + arg1 = stack_pointer[-1]; + eggs(); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + } + """ + output = """ + case OP: { + _Py_UopsSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; + arg1 = stack_pointer[-1]; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + spam(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d48f0d47ba1962..34f79dafbe9851 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -6,6 +6,7 @@ import unittest import weakref import inspect +import types from test import support @@ -89,9 +90,12 @@ def gen(): self.assertEqual(gc.garbage, old_garbage) def test_lambda_generator(self): - # Issue #23192: Test that a lambda returning a generator behaves + # bpo-23192, gh-119897: Test that a lambda returning a generator behaves # like the equivalent function f = lambda: (yield 1) + self.assertIsInstance(f(), types.GeneratorType) + self.assertEqual(next(f()), 1) + def g(): return (yield 1) # test 'yield from' @@ -532,6 +536,26 @@ def f(): with self.assertRaises(RuntimeError): gen.close() + def test_close_releases_frame_locals(self): + # See gh-118272 + + class Foo: + pass + + f = Foo() + f_wr = weakref.ref(f) + + def genfn(): + a = f + yield + + g = genfn() + next(g) + del f + g.close() + support.gc_collect() + self.assertIsNone(f_wr()) + class GeneratorThrowTest(unittest.TestCase): @@ -887,7 +911,7 @@ def b(): File "", line 1, in ? File "", line 2, in g File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(k) # and the generator cannot be resumed Traceback (most recent call last): File "", line 1, in ? @@ -2223,6 +2247,11 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function +>>> yield from [1,2] +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + >>> def f(): x = yield = y Traceback (most recent call last): ... diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index bf600a0f4d1834..04122fbdd0ae80 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -28,7 +28,7 @@ from itertools import chain from http.cookies import Morsel try: - from multiprocessing.managers import ValueProxy + from multiprocessing.managers import ValueProxy, DictProxy, ListProxy from multiprocessing.pool import ApplyResult from multiprocessing.queues import SimpleQueue as MPSimpleQueue from multiprocessing.queues import Queue as MPQueue @@ -36,6 +36,8 @@ except ImportError: # _multiprocessing module is optional ValueProxy = None + DictProxy = None + ListProxy = None ApplyResult = None MPSimpleQueue = None MPQueue = None @@ -47,7 +49,7 @@ ShareableList = None from os import DirEntry from re import Pattern, Match -from types import GenericAlias, MappingProxyType, AsyncGeneratorType +from types import GenericAlias, MappingProxyType, AsyncGeneratorType, CoroutineType, GeneratorType from tempfile import TemporaryDirectory, SpooledTemporaryFile from urllib.parse import SplitResult, ParseResult from unittest.case import _AssertRaisesContext @@ -118,6 +120,7 @@ class BaseTest(unittest.TestCase): KeysView, ItemsView, ValuesView, Sequence, MutableSequence, MappingProxyType, AsyncGeneratorType, + GeneratorType, CoroutineType, DirEntry, chain, LoggerAdapter, StreamHandler, @@ -134,7 +137,7 @@ class BaseTest(unittest.TestCase): if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader)) if ValueProxy is not None: - generic_types.extend((ValueProxy, ApplyResult, + generic_types.extend((ValueProxy, DictProxy, ListProxy, ApplyResult, MPSimpleQueue, MPQueue, MPJoinableQueue)) def test_subscriptable(self): diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index d8bb37f69e18a1..e530b463966819 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,5 +1,6 @@ import unittest from test import support +from test.support.import_helper import import_module class TestMROEntry(unittest.TestCase): @@ -98,7 +99,7 @@ def __mro_entries__(self): return () d = C_too_few() with self.assertRaises(TypeError): - class D(d): ... + class E(d): ... def test_mro_entry_errors_2(self): class C_not_callable: @@ -111,7 +112,7 @@ def __mro_entries__(self): return object c = C_not_tuple() with self.assertRaises(TypeError): - class D(c): ... + class E(c): ... def test_mro_entry_metaclass(self): meta_args = [] @@ -277,7 +278,9 @@ def __class_getitem__(cls, item): class CAPITest(unittest.TestCase): def test_c_class(self): - from _testcapi import Generic, GenericAlias + _testcapi = import_module("_testcapi") + Generic = _testcapi.Generic + GenericAlias = _testcapi.GenericAlias self.assertIsInstance(Generic.__class_getitem__(int), GenericAlias) IntGeneric = Generic[int] diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 4f311c2d498e9f..bf04b3fecf7057 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import is_emscripten -from test.support import os_helper -from test.support import warnings_helper +from test.support import ( + is_apple, is_emscripten, os_helper, warnings_helper +) from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -135,6 +135,9 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename), False) self.assertIs(self.pathmodule.exists(bfilename), False) + self.assertIs(self.pathmodule.lexists(filename), False) + self.assertIs(self.pathmodule.lexists(bfilename), False) + create_file(filename) self.assertIs(self.pathmodule.exists(filename), True) @@ -145,14 +148,13 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename + '\x00'), False) self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False) - if self.pathmodule is not genericpath: - self.assertIs(self.pathmodule.lexists(filename), True) - self.assertIs(self.pathmodule.lexists(bfilename), True) + self.assertIs(self.pathmodule.lexists(filename), True) + self.assertIs(self.pathmodule.lexists(bfilename), True) - self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) - self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) + self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") @@ -165,6 +167,12 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) + def test_exists_bool(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor'): + self.pathmodule.exists(fd) + def test_isdir(self): filename = os_helper.TESTFN bfilename = os.fsencode(filename) @@ -483,12 +491,16 @@ def test_abspath_issue3426(self): self.assertIsInstance(abspath(path), str) def test_nonascii_abspath(self): - if (os_helper.TESTFN_UNDECODABLE - # macOS and Emscripten deny the creation of a directory with an - # invalid UTF-8 name. Windows allows creating a directory with an - # arbitrary bytes name, but fails to enter this directory - # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + if ( + os_helper.TESTFN_UNDECODABLE + # Apple platforms and Emscripten/WASI deny the creation of a + # directory with an invalid UTF-8 name. Windows allows creating a + # directory with an arbitrary bytes name, but fails to enter this + # directory (when the bytes name is used). + and sys.platform not in { + "win32", "emscripten", "wasi" + } and not is_apple + ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 4f2d3cdcc7943e..7fb58a67368576 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -223,7 +223,7 @@ next(g) File "", line 1, in g = (10 // i for i in (5, 0, 2)) - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(g) Traceback (most recent call last): File "", line 1, in -toplevel- diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 98ecec94336e32..80dda2caaa3331 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -26,7 +26,7 @@ def test_username_priorities_of_env_values(self, environ): environ.get.return_value = None try: getpass.getuser() - except ImportError: # in case there's no pwd module + except OSError: # in case there's no pwd module pass except KeyError: # current user has no pwd entry @@ -47,7 +47,7 @@ def test_username_falls_back_to_pwd(self, environ): getpass.getuser()) getpw.assert_called_once_with(42) else: - self.assertRaises(ImportError, getpass.getuser) + self.assertRaises(OSError, getpass.getuser) class GetpassRawinputTest(unittest.TestCase): diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index b9cbe1d92c436f..6c86c3d1c8c57e 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -818,6 +818,20 @@ def test_symlink_buildpath_macos(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_explicitly_set_stdlib_dir(self): + """Test the explicitly set stdlib_dir in the config is respected.""" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="python", + ENV_PATH="/usr/bin", + ) + ns["config"]["stdlib_dir"] = "/custom_stdlib_dir" + expected = dict( + stdlib_dir="/custom_stdlib_dir", + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + # ****************************************************************************** @@ -830,6 +844,7 @@ def test_symlink_buildpath_macos(self): PYDEBUGEXT="", VERSION_MAJOR=9, # fixed version number for ease VERSION_MINOR=8, # of testing + ABI_THREAD="", PYWINVER=None, EXE_SUFFIX=None, diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index f4b5821f408cb4..b72640bd871ba6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -1,9 +1,12 @@ import glob import os +import re import shutil import sys import unittest +import warnings +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -40,6 +43,11 @@ def setUp(self): os.symlink(self.norm('broken'), self.norm('sym1')) os.symlink('broken', self.norm('sym2')) os.symlink(os.path.join('a', 'bcd'), self.norm('sym3')) + self.open_dirfd() + + def open_dirfd(self): + if self.dir_fd is not None: + os.close(self.dir_fd) if {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd: self.dir_fd = os.open(self.tempdir, os.O_RDONLY | os.O_DIRECTORY) else: @@ -332,6 +340,35 @@ def test_recursive_glob(self): eq(glob.glob('**', recursive=True, include_hidden=True), [join(*i) for i in full+rec]) + def test_glob_non_directory(self): + eq = self.assertSequencesEqual_noorder + eq(self.rglob('EF'), self.joins(('EF',))) + eq(self.rglob('EF', ''), []) + eq(self.rglob('EF', '*'), []) + eq(self.rglob('EF', '**'), []) + eq(self.rglob('nonexistent'), []) + eq(self.rglob('nonexistent', ''), []) + eq(self.rglob('nonexistent', '*'), []) + eq(self.rglob('nonexistent', '**'), []) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_glob_named_pipe(self): + path = os.path.join(self.tempdir, 'mypipe') + os.mkfifo(path) + + # gh-117127: Reopen self.dir_fd to pick up directory changes + self.open_dirfd() + + self.assertEqual(self.rglob('mypipe'), [path]) + self.assertEqual(self.rglob('mypipe*'), [path]) + self.assertEqual(self.rglob('mypipe', ''), []) + self.assertEqual(self.rglob('mypipe', 'sub'), []) + self.assertEqual(self.rglob('mypipe', '*'), []) + + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') @@ -349,6 +386,126 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) + def test_glob0(self): + with self.assertWarns(DeprecationWarning): + glob.glob0(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob0(self.tempdir, 'a'), ['a']) + eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob0(self.tempdir, '.b*'), []) + eq(glob.glob0(self.tempdir, 'b'), []) + eq(glob.glob0(self.tempdir, '?'), []) + eq(glob.glob0(self.tempdir, '*a'), []) + eq(glob.glob0(self.tempdir, 'a*'), []) + + def test_glob1(self): + with self.assertWarns(DeprecationWarning): + glob.glob1(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob1(self.tempdir, 'a'), ['a']) + eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) + eq(glob.glob1(self.tempdir, 'b'), []) + eq(glob.glob1(self.tempdir, '?'), ['a']) + eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) + eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) + + def test_translate_matching(self): + match = re.compile(glob.translate('*')).match + self.assertIsNotNone(match('foo')) + self.assertIsNotNone(match('foo.bar')) + self.assertIsNone(match('.foo')) + match = re.compile(glob.translate('.*')).match + self.assertIsNotNone(match('.foo')) + match = re.compile(glob.translate('**', recursive=True)).match + self.assertIsNotNone(match('foo')) + self.assertIsNone(match('.foo')) + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/*', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('*/**', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', 'bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/.bar', recursive=True)).match + self.assertIsNotNone(match(os.path.join('foo', '.bar'))) + self.assertIsNone(match(os.path.join('.foo', '.bar'))) + match = re.compile(glob.translate('**/*.*', recursive=True)).match + self.assertIsNone(match(os.path.join('foo', 'bar'))) + self.assertIsNone(match(os.path.join('foo', '.bar'))) + self.assertIsNotNone(match(os.path.join('foo', 'bar.txt'))) + self.assertIsNone(match(os.path.join('foo', '.bar.txt'))) + + def test_translate(self): + def fn(pat): + return glob.translate(pat, seps='/') + self.assertEqual(fn('foo'), r'(?s:foo)\Z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') + self.assertEqual(fn('*'), r'(?s:[^/.][^/]*)\Z') + self.assertEqual(fn('?'), r'(?s:(?!\.)[^/])\Z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('*a'), r'(?s:(?!\.)[^/]*a)\Z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') + self.assertEqual(fn('?aa'), r'(?s:(?!\.)[^/]aa)\Z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') + self.assertEqual(fn('**'), r'(?s:(?!\.)[^/]*)\Z') + self.assertEqual(fn('***'), r'(?s:(?!\.)[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:(?!\.)[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), + r'(?s:/(?!\.)[^/]*/[^/.][^/]*/(?!\.)[^/]*\.[^/]*/(?!\.)[^/]*)\Z') + + def test_translate_include_hidden(self): + def fn(pat): + return glob.translate(pat, include_hidden=True, seps='/') + self.assertEqual(fn('foo'), r'(?s:foo)\Z') + self.assertEqual(fn('foo/bar'), r'(?s:foo/bar)\Z') + self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') + self.assertEqual(fn('?'), r'(?s:[^/])\Z') + self.assertEqual(fn('a*'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('*a'), r'(?s:[^/]*a)\Z') + self.assertEqual(fn('.*'), r'(?s:\.[^/]*)\Z') + self.assertEqual(fn('?aa'), r'(?s:[^/]aa)\Z') + self.assertEqual(fn('aa?'), r'(?s:aa[^/])\Z') + self.assertEqual(fn('aa[ab]'), r'(?s:aa[ab])\Z') + self.assertEqual(fn('**'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/[^/]*/[^/]+/[^/]*\.[^/]*/[^/]*)\Z') + + def test_translate_recursive(self): + def fn(pat): + return glob.translate(pat, recursive=True, include_hidden=True, seps='/') + self.assertEqual(fn('*'), r'(?s:[^/]+)\Z') + self.assertEqual(fn('?'), r'(?s:[^/])\Z') + self.assertEqual(fn('**'), r'(?s:.*)\Z') + self.assertEqual(fn('**/**'), r'(?s:.*)\Z') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') + self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') + + def test_translate_seps(self): + def fn(pat): + return glob.translate(pat, recursive=True, include_hidden=True, seps=['/', '\\']) + self.assertEqual(fn('foo/bar\\baz'), r'(?s:foo[/\\]bar[/\\]baz)\Z') + self.assertEqual(fn('**/*'), r'(?s:(?:.+[/\\])?[^/\\]+)\Z') + @skip_unless_symlink class SymlinkLoopGlobTests(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8501006b799262..5b7a639c025a0f 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -164,7 +164,7 @@ def test_floats(self): x = 3.14 x = 314. x = 0.314 - # XXX x = 000.314 + x = 000.314 x = .314 x = 3e14 x = 3E14 @@ -306,16 +306,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +436,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +456,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](1), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 128f933787a3f6..ae384c3849d49e 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,7 +5,6 @@ import functools import io import os -import pathlib import struct import sys import unittest @@ -79,16 +78,18 @@ def test_write(self): f.close() def test_write_read_with_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.GzipFile(filename, 'w') as f: f.write(data1 * 50) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) with gzip.GzipFile(filename, 'a') as f: f.write(data1) with gzip.GzipFile(filename) as f: d = f.read() self.assertEqual(d, data1 * 51) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input @@ -472,15 +473,122 @@ def test_textio_readlines(self): with io.TextIOWrapper(f, encoding="ascii") as t: self.assertEqual(t.readlines(), lines) + def test_fileobj_with_name(self): + with open(self.filename, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + def test_fileobj_from_fdopen(self): # Issue #13781: Opening a GzipFile for writing fails when using a # fileobj created with os.fdopen(). - fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT) - with os.fdopen(fd, "wb") as f: - with gzip.GzipFile(fileobj=f, mode="w") as g: - pass + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with os.fdopen(fd, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with os.fdopen(fd, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): + self.assertEqual(gzip.READ, 'rb') + self.assertEqual(gzip.WRITE, 'wb') gzip.GzipFile(self.filename, "wb").close() with open(self.filename, "r+b") as f: with gzip.GzipFile(fileobj=f, mode='r') as g: @@ -508,17 +616,69 @@ def test_fileobj_mode(self): def test_bytes_filename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with gzip.GzipFile(bytes_filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, bytes_filename) with gzip.GzipFile(bytes_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with gzip.GzipFile(str_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, str_filename) + + def test_fileobj_without_name(self): + bio = io.BytesIO() + with gzip.GzipFile(fileobj=bio, mode='wb') as f: + f.write(data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + bio.seek(0) + with gzip.GzipFile(fileobj=bio, mode='rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + + def test_fileobj_and_filename(self): + filename2 = self.filename + 'new' + with (open(self.filename, 'wb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f): + f.write(data1 * 50) + self.assertEqual(f.name, filename2) + with (open(self.filename, 'rb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f): + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, filename2) + # Sanity check that we are actually operating on the right file. + with gzip.GzipFile(self.filename, 'rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, self.filename) def test_decompress_limited(self): """Decompressed data buffering should be limited""" @@ -554,7 +714,6 @@ def test_compress_mtime(self): self.assertEqual(f.mtime, mtime) def test_compress_correct_level(self): - # gzip.compress calls with mtime == 0 take a different code path. for mtime in (0, 42): with self.subTest(mtime=mtime): nocompress = gzip.compress(data1, compresslevel=0, mtime=mtime) @@ -562,6 +721,17 @@ def test_compress_correct_level(self): self.assertIn(data1, nocompress) self.assertNotIn(data1, yescompress) + def test_issue112346(self): + # The OS byte should be 255, this should not change between Python versions. + for mtime in (0, 42): + with self.subTest(mtime=mtime): + compress = gzip.compress(data1, compresslevel=1, mtime=mtime) + self.assertEqual( + struct.unpack(" TreeMaker: @@ -51,12 +50,16 @@ def build( ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): create(contents, _ensure_tree_maker(prefix) / name) @@ -79,8 +82,8 @@ def _(content: str, path): @create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') +def _(content: Symlink, path): + path.symlink_to(content) class Recording: @@ -107,3 +110,6 @@ def write_text(self, content, **kwargs): def mkdir(self, **kwargs): return + + def symlink_to(self, target): + pass diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3.6.egg rename to Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000000..ba73b743394169 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000000..479488a0348186 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000000..de645c2e8bc75b --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000000..011f4751fb9e32 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py similarity index 73% rename from Lib/test/test_importlib/fixtures.py rename to Lib/test/test_importlib/metadata/fixtures.py index 73e5da2ba92279..826b1b3259b4cd 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,14 +1,14 @@ -import os import sys import copy +import json import shutil import pathlib -import tempfile import textwrap import functools import contextlib -from test.support.os_helper import FS_NONASCII +from test.support import import_helper +from test.support import os_helper from test.support import requires_zlib from . import _path @@ -25,29 +25,12 @@ @contextlib.contextmanager -def tempdir(): - tmpdir = tempfile.mkdtemp() - try: - yield pathlib.Path(tmpdir) - finally: - shutil.rmtree(tmpdir) - - -@contextlib.contextmanager -def save_cwd(): - orig = os.getcwd() - try: - yield - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tempdir_as_cwd(): - with tempdir() as tmp: - with save_cwd(): - os.chdir(str(tmp)) - yield tmp +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) @contextlib.contextmanager @@ -68,7 +51,7 @@ def setUp(self): class SiteDir(Fixtures): def setUp(self): super().setUp() - self.site_dir = self.fixtures.enter_context(tempdir()) + self.site_dir = self.fixtures.enter_context(tmp_path()) class OnSysPath(Fixtures): @@ -84,9 +67,18 @@ def add_sys_path(dir): def setUp(self): super().setUp() self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) + + +class SiteBuilder(SiteDir): + def setUp(self): + super().setUp() + for cls in self.__class__.mro(): + with contextlib.suppress(AttributeError): + build_files(cls.files, prefix=self.site_dir) -class DistInfoPkg(OnSysPath, SiteDir): +class DistInfoPkg(OnSysPath, SiteBuilder): files: FilesSpec = { "distinfo_pkg-1.0.0.dist-info": { "METADATA": """ @@ -113,10 +105,6 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - def make_uppercase(self): """ Rewrite metadata with everything uppercase. @@ -128,7 +116,26 @@ def make_uppercase(self): build_files(files, self.site_dir) -class DistInfoPkgWithDot(OnSysPath, SiteDir): +class DistInfoPkgEditable(DistInfoPkg): + """ + Package with a PEP 660 direct_url.json. + """ + + some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' + files: FilesSpec = { + 'distinfo_pkg-1.0.0.dist-info': { + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) + }, + } + + +class DistInfoPkgWithDot(OnSysPath, SiteBuilder): files: FilesSpec = { "pkg_dot-1.0.0.dist-info": { "METADATA": """ @@ -138,12 +145,8 @@ class DistInfoPkgWithDot(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDot.files, self.site_dir) - -class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): +class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): files: FilesSpec = { "pkg.dot-1.0.0.dist-info": { "METADATA": """ @@ -159,18 +162,12 @@ class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDotLegacy.files, self.site_dir) - -class DistInfoPkgOffPath(SiteDir): - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) +class DistInfoPkgOffPath(SiteBuilder): + files = DistInfoPkg.files -class EggInfoPkg(OnSysPath, SiteDir): +class EggInfoPkg(OnSysPath, SiteBuilder): files: FilesSpec = { "egginfo_pkg.egg-info": { "PKG-INFO": """ @@ -205,12 +202,8 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkg.files, prefix=self.site_dir) - -class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteDir): +class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_module_pkg.egg-info": { "PKG-INFO": "Name: egg_with_module-pkg", @@ -240,12 +233,42 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoToplevel.files, prefix=self.site_dir) + +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } -class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): +class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_no_modules_pkg.egg-info": { "PKG-INFO": "Name: egg_with_no_modules-pkg", @@ -270,12 +293,8 @@ class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoModules.files, prefix=self.site_dir) - -class EggInfoPkgSourcesFallback(OnSysPath, SiteDir): +class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): files: FilesSpec = { "sources_fallback_pkg.egg-info": { "PKG-INFO": "Name: sources_fallback-pkg", @@ -296,12 +315,8 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgSourcesFallback.files, prefix=self.site_dir) - -class EggInfoFile(OnSysPath, SiteDir): +class EggInfoFile(OnSysPath, SiteBuilder): files: FilesSpec = { "egginfo_file.egg-info": """ Metadata-Version: 1.0 @@ -317,10 +332,6 @@ class EggInfoFile(OnSysPath, SiteDir): """, } - def setUp(self): - super().setUp() - build_files(EggInfoFile.files, prefix=self.site_dir) - # dedent all text strings before writing orig = _path.create.registry[str] @@ -342,7 +353,9 @@ def record_names(file_defs): class FileBuilder: def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) def DALS(str): @@ -352,7 +365,7 @@ def DALS(str): @requires_zlib() class ZipFixtures: - root = 'test.test_importlib.data' + root = 'test.test_importlib.metadata.data' def _fixture_on_path(self, filename): pkg_file = resources.files(self.root).joinpath(filename) diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/metadata/stubs.py similarity index 100% rename from Lib/test/test_importlib/stubs.py rename to Lib/test/test_importlib/metadata/stubs.py diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/metadata/test_api.py similarity index 98% rename from Lib/test/test_importlib/test_metadata_api.py rename to Lib/test/test_importlib/metadata/test_api.py index 33c6e85ee94753..2256e0c502e46f 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -29,6 +29,7 @@ class APITests( fixtures.EggInfoPkg, fixtures.EggInfoPkgPipInstalledNoToplevel, fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, fixtures.EggInfoPkgSourcesFallback, fixtures.DistInfoPkg, fixtures.DistInfoPkgWithDot, @@ -109,7 +110,7 @@ def test_entry_points_unique_packages_normalized(self): Entry points should only be exposed for the first package on sys.path with a given name (even when normalized). """ - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) alt_pkg = { "DistInfo_pkg-1.1.0.dist-info": { diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/metadata/test_main.py similarity index 86% rename from Lib/test/test_importlib/test_main.py rename to Lib/test/test_importlib/metadata/test_main.py index 3b49227255eb58..e4218076f8cb0e 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -2,8 +2,10 @@ import pickle import unittest import warnings +import importlib import importlib.metadata import contextlib +from test.support import os_helper try: import pyfakefs.fake_filesystem_unittest as ffs @@ -12,6 +14,7 @@ from . import fixtures from ._context import suppress +from ._path import Symlink from importlib.metadata import ( Distribution, EntryPoint, @@ -68,7 +71,7 @@ def test_abc_enforced(self): dict(name=''), ) def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(ValueError): + with self.assertRaises(Exception): Distribution.from_name(name) @@ -135,7 +138,7 @@ def test_unique_distributions(self): fixtures.build_files(self.make_pkg('abc'), self.site_dir) before = list(_unique(distributions())) - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) after = list(_unique(distributions())) @@ -207,6 +210,20 @@ def test_invalid_usage(self): with self.assertRaises(ValueError): list(distributions(context='something', name='else')) + def test_interleaved_discovery(self): + """ + Ensure interleaved searches are safe. + + When the search is cached, it is possible for searches to be + interleaved, so make sure those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('egginfo-pkg') + next(dists) + class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): def test_egg_info(self): @@ -292,12 +309,10 @@ def test_sortable(self): """ EntryPoint objects are sortable, but result is undefined. """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) class FileSystem( @@ -364,18 +379,16 @@ def test_packages_distributions_all_module_types(self): 'all_distributions-1.0.0.dist-info': metadata, } for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) metadata.update(RECORD=fixtures.build_record(files)) fixtures.build_files(files, prefix=self.site_dir) @@ -388,6 +401,28 @@ def test_packages_distributions_all_module_types(self): assert not any(name.endswith('.dist-info') for name in distributions) + @os_helper.skip_unless_symlink + def test_packages_distributions_symlinked_top_level(self) -> None: + """ + Distribution is resolvable from a simple top-level symlink in RECORD. + See #452. + """ + + files: fixtures.FilesSpec = { + "symlinked_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: symlinked-pkg + Version: 1.0.0 + """, + "RECORD": "symlinked,,\n", + }, + ".symlink.target": {}, + "symlinked": Symlink(".symlink.target"), + } + + fixtures.build_files(files, self.site_dir) + assert packages_distributions()['symlinked'] == ['symlinked-pkg'] + class PackagesDistributionsEggTest( fixtures.EggInfoPkg, @@ -424,3 +459,10 @@ def import_names_from_package(package_name): # sources_fallback-pkg has one import ('sources_fallback') inferred from # SOURCES.txt (top_level.txt and installed-files.txt is missing) assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} + + +class EditableDistributionTest(fixtures.DistInfoPkgEditable, unittest.TestCase): + def test_origin(self): + dist = Distribution.from_name('distinfo-pkg') + assert dist.origin.url.endswith('.whl') + assert dist.origin.archive_info.hashes.sha256 diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py similarity index 100% rename from Lib/test/test_importlib/test_zip.py rename to Lib/test/test_importlib/metadata/test_zip.py diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file index eaf36c1daccfdf..5bd8bb897b1322 100644 Binary files a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file and b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file b/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file new file mode 100644 index 00000000000000..100f50643d8d21 --- /dev/null +++ b/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file @@ -0,0 +1 @@ +  \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86f03..beab67ccc21680 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -31,8 +31,8 @@ class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): class ContentsNamespaceTests(ContentsTests, unittest.TestCase): expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a2761b..640f90fc0dd91a 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310926a..7df6d03ead7480 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,3 @@ -import typing import textwrap import unittest import warnings @@ -32,13 +31,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -64,13 +64,17 @@ def setUp(self): self.data = namespacedata01 +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + class SiteDir: def setUp(self): self.fixtures = contextlib.ExitStack() self.addCleanup(self.fixtures.close) self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) + self.fixtures.enter_context(import_helper.isolated_modules()) class ModulesFilesTests(SiteDir, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000000..d60a2bedd89798 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,225 @@ +import unittest +import os + +from test.support.warnings_helper import ignore_warnings, check_warnings + +import importlib.resources as resources + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'test.test_importlib.resources.data01' + anchor02 = 'test.test_importlib.resources.data02' + + +class ModuleAnchorMixin: + from . import data01 as anchor01 + from . import data02 as anchor02 + + +class FunctionalAPIBase: + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest + """ + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def assertEndsWith(self, string, suffix): + """Assert that `string` ends with `suffix`. + + Used to ignore an architecture-specific UTF-16 byte-order mark.""" + self.assertEqual(string[-len(suffix):], suffix) + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, 'subdirectory', 'subsubdir', 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, *path_parts, encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, 'binary.file', encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, 'binary.file', encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path)) as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file')) as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + is_resource = resources.is_resource + with check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with ( + self.assertRaises(OSError), + check_warnings((".*contents.*", DeprecationWarning)), + ): + list(resources.contents(self.anchor01, 'utf-8.file')) + for path_parts in self._gen_resourcetxt_path_parts(): + with ( + self.assertRaises(OSError), + check_warnings((".*contents.*", DeprecationWarning)), + ): + list(resources.contents(self.anchor01, *path_parts)) + with check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, 'subdirectory', + 'subsubdir', 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + unittest.TestCase, FunctionalAPIBase, StringAnchorMixin, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + unittest.TestCase, FunctionalAPIBase, ModuleAnchorMixin, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaad37..3b6b2142ef47b1 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -24,7 +24,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -81,5 +81,9 @@ class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58b91..90b22905ab8692 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,4 +1,5 @@ import io +import pathlib import unittest from importlib import resources @@ -15,18 +16,13 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: + self.assertIsInstance(path, pathlib.Path) self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) class PathDiskTests(PathTests, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8b0c..984feecbb9ed69 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -57,17 +57,15 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) class ReadNamespaceTests(ReadTests, unittest.TestCase): @@ -77,5 +75,22 @@ def setUp(self): self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334585..dac9c2a892ffd2 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -10,8 +10,7 @@ class MultiplexedPathTest(unittest.TestCase): @classmethod def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + cls.folder = pathlib.Path(__file__).parent / 'namespacedata01' def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +18,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,10 +26,12 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) + data01 = pathlib.Path(__file__).parent.joinpath('data01') contents = { path.name for path in MultiplexedPath(self.folder, data01).iterdir() } @@ -60,17 +61,17 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + prefix = str(data01.parent) path = MultiplexedPath(self.folder, data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,9 +83,9 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + data02 = pathlib.Path(__file__).parent.joinpath('data02') + prefix = str(data01.parent) path = MultiplexedPath(data01, data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03d02..d1d45d9b4617f3 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,10 @@ -import contextlib import sys import unittest -import uuid import pathlib from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -89,34 +84,32 @@ def test_package_has_no_reader_fallback(self): class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore + ZIP_MODULE = 'data01' def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) @@ -124,7 +117,7 @@ def test_as_file_directory(self): class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore + ZIP_MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +125,48 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetupBase, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +185,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +195,45 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests(ResourceFromNamespaceTests, unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) + + +class ResourceFromNamespaceZipTests( + util.ZipSetupBase, + ResourceFromNamespaceTests, + unittest.TestCase, +): + ZIP_MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476699..d4bf3e6cc5dfdc 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ from importlib.machinery import ModuleSpec @@ -141,39 +142,23 @@ def test_useless_loader(self): class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass + ZIP_MODULE = 'data01' def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + src_path = pathlib.Path(__file__).parent.joinpath(self.ZIP_MODULE) + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(src_path, modules))) + ) + + self.data = importlib.import_module(self.ZIP_MODULE) class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100755 index 00000000000000..4dcf6facc770cb --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,30 @@ +""" +Generate zip test data files. +""" + +import contextlib +import os +import pathlib +import zipfile + + +def make_zip_file(src, dst): + """ + Zip the files in src into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for src_path, rel in walk(src): + dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) + zf.write(src_path, dst_name) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(datapath): + for dirpath, dirnames, filenames in os.walk(datapath): + with contextlib.suppress(ValueError): + dirnames.remove('__pycache__') + for filename in filenames: + res = pathlib.Path(dirpath) / filename + rel = res.relative_to(datapath) + yield res, rel diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 2a35f3dcb7210c..973237c0791a3e 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,6 +6,7 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper import types @@ -437,5 +438,44 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333e355a..5c6e0303528906 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -140,6 +147,83 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 9b3bef02c66820..cbbdada3b010a7 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -80,7 +80,7 @@ def test_cant_import_other(self): def test_simple_repr(self): import foo.one - assert repr(foo).startswith(" None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class F[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + + return SimpleNamespace( + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 31ac6b1070a0a2..d39c3ccdc847bd 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4,6 +4,7 @@ import copy import datetime import functools +import gc import importlib import inspect import io @@ -16,14 +17,17 @@ import shutil import stat import sys +import subprocess import time import types import tempfile import textwrap +from typing import Unpack import unicodedata import unittest import unittest.mock import warnings +import weakref try: @@ -31,27 +35,28 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only +from test.support import cpython_only, import_helper, suppress_immortalization from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.os_helper import TESTFN, temp_cwd +from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python +from test.support import has_subprocess_support from test import support -from . import inspect_fodder as mod -from . import inspect_fodder2 as mod2 -from . import inspect_stock_annotations -from . import inspect_stringized_annotations -from . import inspect_stringized_annotations_2 +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 # Functions tested in this suite: # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargvalues, formatargvalues, -# currentframe, stack, trace, isdatadescriptor, -# ismethodwrapper +# getclasstree, getargvalues, formatargvalues, currentframe, +# stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -173,6 +178,7 @@ def test_excluding_predicates(self): self.istest(inspect.ismethod, 'git.argue') self.istest(inspect.ismethod, 'mod.custom_method') self.istest(inspect.ismodule, 'mod') + self.istest(inspect.ismethoddescriptor, 'int.__add__') self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') self.istest(inspect.isgenerator, '(x for x in range(2))') self.istest(inspect.isgeneratorfunction, 'generator_function_example') @@ -204,12 +210,34 @@ def test_iscoroutine(self): gen_coro = gen_coroutine_function_example(1) coro = coroutine_function_example(1) + class PMClass: + async_generator_partialmethod_example = functools.partialmethod( + async_generator_function_example) + coroutine_partialmethod_example = functools.partialmethod( + coroutine_function_example) + gen_coroutine_partialmethod_example = functools.partialmethod( + gen_coroutine_function_example) + + # partialmethods on the class, bound to an instance + pm_instance = PMClass() + async_gen_coro_pmi = pm_instance.async_generator_partialmethod_example + gen_coro_pmi = pm_instance.gen_coroutine_partialmethod_example + coro_pmi = pm_instance.coroutine_partialmethod_example + + # partialmethods on the class, unbound but accessed via the class + async_gen_coro_pmc = PMClass.async_generator_partialmethod_example + gen_coro_pmc = PMClass.gen_coroutine_partialmethod_example + coro_pmc = PMClass.coroutine_partialmethod_example + self.assertFalse( inspect.iscoroutinefunction(gen_coroutine_function_example)) self.assertFalse( inspect.iscoroutinefunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) self.assertFalse(inspect.iscoroutine(gen_coro)) self.assertTrue( @@ -218,6 +246,8 @@ def test_iscoroutine(self): inspect.isgeneratorfunction( functools.partial(functools.partial( gen_coroutine_function_example)))) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmi)) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmc)) self.assertTrue(inspect.isgenerator(gen_coro)) async def _fn3(): @@ -283,6 +313,8 @@ def do_something_static(): inspect.iscoroutinefunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertTrue(inspect.iscoroutinefunction(coro_pmi)) + self.assertTrue(inspect.iscoroutinefunction(coro_pmc)) self.assertTrue(inspect.iscoroutine(coro)) self.assertFalse( @@ -295,6 +327,8 @@ def do_something_static(): inspect.isgeneratorfunction( functools.partial(functools.partial( coroutine_function_example)))) + self.assertFalse(inspect.isgeneratorfunction(coro_pmi)) + self.assertFalse(inspect.isgeneratorfunction(coro_pmc)) self.assertFalse(inspect.isgenerator(coro)) self.assertFalse( @@ -309,6 +343,8 @@ def do_something_static(): inspect.isasyncgenfunction( functools.partial(functools.partial( async_generator_function_example)))) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmi)) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmc)) self.assertTrue(inspect.isasyncgen(async_gen_coro)) coro.close(); gen_coro.close(); # silence warnings @@ -637,7 +673,10 @@ def test_cleandoc(self): @cpython_only def test_c_cleandoc(self): - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") func = _testinternalcapi.compiler_cleandoc for i, (input, expected) in enumerate(self.cleandoc_testdata): with self.subTest(i=i): @@ -699,6 +738,18 @@ def test_getsourcefile(self): finally: del linecache.cache[co.co_filename] + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + def test_getfile(self): self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) @@ -720,6 +771,7 @@ def test_getfile_builtin_function_or_method(self): inspect.getfile(list.append) self.assertIn('expected, got', str(e_append.exception)) + @suppress_immortalization() def test_getfile_class_without_module(self): class CM(type): @property @@ -769,6 +821,21 @@ def monkey(filename, module_globals=None): def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) + def test_getsource_on_generated_class(self): + A = type('A', (), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -940,6 +1007,9 @@ def test_findsource_with_out_of_bounds_lineno(self): def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + def test_nested_func(self): self.assertSourceEqual(mod2.cls135.func136, 136, 139) @@ -960,7 +1030,11 @@ def test_class_decorator(self): self.assertSourceEqual(mod2.cls196, 194, 201) self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + @support.requires_docstrings def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. self.assertSourceEqual(mod2.cls238.cls239, 239, 240) def test_multiple_children_classes(self): @@ -1170,7 +1244,7 @@ def test_getfullargspec_builtin_methods(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_with_signature_with_defaults spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') @@ -1179,7 +1253,7 @@ def test_getfullargspec_builtin_func(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_no_signature with self.assertRaises(TypeError): inspect.getfullargspec(builtin) @@ -1213,7 +1287,7 @@ def test_getfullargspec_builtin_func_no_signature(self): (dict.__class_getitem__, meth_type_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -1620,6 +1694,117 @@ def wrapper(a, b): self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = inspect.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = inspect.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = inspect.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): + self.assertEqual( + inspect.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), + set() + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__) + ) + class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): @@ -1628,6 +1813,121 @@ def test_typing_replacement(self): self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') +class TestIsMethodDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class MethodDescriptor: + def __get__(self, *_): pass + class MethodDescriptorSub(MethodDescriptor): + pass + class DataDescriptorWithNoGet: + def __set__(self, *_): pass + class DataDescriptorWithGetSet: + def __get__(self, *_): pass + def __set__(self, *_): pass + class DataDescriptorWithGetDelete: + def __get__(self, *_): pass + def __delete__(self, *_): pass + class DataDescriptorSub(DataDescriptorWithNoGet, + DataDescriptorWithGetDelete): + pass + + # Custom method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptor()), + '__get__ and no __set__/__delete__ => method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptorSub()), + '__get__ (inherited) and no __set__/__delete__' + ' => method descriptor') + + # Custom data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithNoGet()), + '__set__ (and no __get__) => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetSet()), + '__get__ and __set__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetDelete()), + '__get__ and __delete__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorSub()), + '__get__, __set__ and __delete__ => not a method descriptor') + + # Classes of descriptors (are *not* descriptors themselves): + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptor)) + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) + self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) + + def test_builtin_descriptors(self): + builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. + class Owner: + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + @property + def a_property(self): pass + class Slotermeyer: + __slots__ = 'a_slot', + def function(): + pass + a_lambda = lambda: None + + # Example builtin method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(builtin_slot_wrapper), + 'a builtin slot wrapper is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['class_method']), + 'a classmethod object is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['static_method']), + 'a staticmethod object is a method descriptor') + + # Example builtin data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(Owner.__dict__['a_property']), + 'a property is not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(Slotermeyer.__dict__['a_slot']), + 'a slot is not a method descriptor') + + # `types.MethodType`/`types.FunctionType` instances (they *are* + # method descriptors, but `ismethoddescriptor()` explicitly + # excludes them): + self.assertFalse(inspect.ismethoddescriptor(Owner().instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().static_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) + self.assertFalse(inspect.ismethoddescriptor(function)) + self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + + def test_descriptor_being_a_class(self): + class MethodDescriptorMeta(type): + def __get__(self, *_): pass + class ClassBeingMethodDescriptor(metaclass=MethodDescriptorMeta): + pass + # `ClassBeingMethodDescriptor` itself *is* a method descriptor, + # but it is *also* a class, and `ismethoddescriptor()` explicitly + # excludes classes. + self.assertFalse( + inspect.ismethoddescriptor(ClassBeingMethodDescriptor), + 'classes (instances of type) are explicitly excluded') + + def test_non_descriptors(self): + class Test: + pass + self.assertFalse(inspect.ismethoddescriptor(Test())) + self.assertFalse(inspect.ismethoddescriptor(Test)) + self.assertFalse(inspect.ismethoddescriptor([42])) + self.assertFalse(inspect.ismethoddescriptor(42)) + + class TestIsDataDescriptor(unittest.TestCase): def test_custom_descriptors(self): @@ -2249,6 +2549,13 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + class Bar(Foo): pass + + bar = Bar() + bar.a = 5 + self.assertEqual(inspect.getattr_static(bar, 'a'), 3) + self.assertFalse(test.called) + def test_mutated_mro(self): test = self test.called = False @@ -2353,6 +2660,22 @@ def __getattribute__(self, attr): self.assertFalse(test.called) + @suppress_immortalization() + def test_cache_does_not_cause_classes_to_persist(self): + # regression test for gh-118013: + # check that the internal _shadowed_dict cache does not cause + # dynamically created classes to have extended lifetimes even + # when no other strong references to those classes remain. + # Since these classes can themselves hold strong references to + # other objects, this can cause unexpected memory consumption. + class Foo: pass + Foo.instance = Foo() + weakref_to_class = weakref.ref(Foo) + inspect.getattr_static(Foo.instance, 'whatever', 'irrelevant') + del Foo + gc.collect() + self.assertIsNone(weakref_to_class()) + class TestGetGeneratorState(unittest.TestCase): @@ -2382,6 +2705,10 @@ def test_closed_after_immediate_exception(self): self.generator.throw(RuntimeError) self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + def test_closed_after_close(self): + self.generator.close() + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + def test_running(self): # As mentioned on issue #10220, checking for the RUNNING state only # makes sense inside the generator itself. @@ -2491,6 +2818,10 @@ def test_closed_after_immediate_exception(self): self.coroutine.throw(RuntimeError) self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + def test_closed_after_close(self): + self.coroutine.close() + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + def test_easy_debugging(self): # repr() and str() of a coroutine state should contain the state name names = 'CORO_CREATED CORO_RUNNING CORO_SUSPENDED CORO_CLOSED'.split() @@ -2832,7 +3163,7 @@ def test_staticmethod(*args): # NOQA @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -2889,9 +3220,12 @@ def p(name): return signature.parameters[name].default # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "no signature found"): - class ThisWorksNow: - __call__ = type test_callable(ThisWorksNow()) # Regression test for issue #20786 @@ -2910,7 +3244,7 @@ class ThisWorksNow: @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") func = _testcapi.docstring_with_signature_with_defaults def decorator(func): @@ -2931,7 +3265,7 @@ def wrapper_like(*args, **kwargs) -> int: pass @cpython_only def test_signature_on_builtins_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") with self.assertRaisesRegex(ValueError, 'no signature found for builtin'): inspect.signature(_testcapi.docstring_no_signature) @@ -2969,7 +3303,7 @@ def test_signature_on_builtins_no_signature(self): (dict.__class_getitem__, meth_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -2981,6 +3315,13 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) @@ -3098,6 +3439,10 @@ def m1d(*args, **kwargs): int)) def test_signature_on_classmethod(self): + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @classmethod def foo(cls, arg1, *, arg2=1): @@ -3116,6 +3461,10 @@ def foo(cls, arg1, *, arg2=1): ...)) def test_signature_on_staticmethod(self): + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @staticmethod def foo(cls, *, arg): @@ -3379,7 +3728,7 @@ def test(self: 'anno', x): def test_signature_on_fake_partialmethod(self): def foo(a): pass - foo._partialmethod = 'spam' + foo.__partialmethod__ = 'spam' self.assertEqual(str(inspect.signature(foo)), '(a)') def test_signature_on_decorated(self): @@ -3474,6 +3823,98 @@ def __init__(self, b): ((('a', ..., ..., "positional_or_keyword"),), ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + class C(metaclass=CM): + def __init__(self, c): + pass + + self.assertEqual(C(1), (2, C, 1)) + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + class CM(type): def __new__(mcls, name, bases, dct, *, foo=1): return super().__new__(mcls, name, bases, dct) @@ -3535,6 +3976,169 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a, b: None, 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + def test_signature_on_subclass(self): class A: def __new__(cls, a=1, *args, **kwargs): @@ -3588,8 +4192,11 @@ class D(C): pass # Test meta-classes without user-defined __init__ or __new__ class C(type): pass class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) @@ -3639,16 +4246,140 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) - class Wrapped: - pass - Wrapped.__wrapped__ = lambda a: None - self.assertEqual(self.signature(Wrapped), + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + + c = C() + self.assertEqual(c(1), (2, c, 1)) + self.assertEqual(self.signature(C()), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), ((('a', ..., ..., "positional_or_keyword"),), ...)) # wrapper loop: - Wrapped.__wrapped__ = Wrapped + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper with self.assertRaisesRegex(ValueError, 'wrapper loop'): - self.signature(Wrapped) + self.signature(wrapper) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -3786,26 +4517,36 @@ def foo(a:int=1, *, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), '(a: int = 1, *, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(a:int=1, *args, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), '(a: int = 1, *args, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(): pass self.assertEqual(str(inspect.signature(foo)), '()') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(a: list[str]) -> tuple[str, float]: pass self.assertEqual(str(inspect.signature(foo)), '(a: list[str]) -> tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) from typing import Tuple def foo(a: list[str]) -> Tuple[str, float]: pass self.assertEqual(str(inspect.signature(foo)), '(a: list[str]) -> Tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def test_signature_str_positional_only(self): P = inspect.Parameter @@ -3816,19 +4557,85 @@ def test(a_po, /, *, b, **kwargs): self.assertEqual(str(inspect.signature(test)), '(a_po, /, *, b, **kwargs)') + self.assertEqual(str(inspect.signature(test)), + inspect.signature(test).format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY)]) + self.assertEqual(str(test), '(foo, /)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_KEYWORD)]) + self.assertEqual(str(test), '(foo, /, **bar)') + self.assertEqual(str(test), test.format()) - self.assertEqual(str(S(parameters=[P('foo', P.POSITIONAL_ONLY)])), - '(foo, /)') + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_POSITIONAL)]) + self.assertEqual(str(test), '(foo, /, *bar)') + self.assertEqual(str(test), test.format()) + + def test_signature_format(self): + from typing import Annotated, Literal + + def func(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString'): + pass + + expected_singleline = "(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString')" + expected_multiline = """( + x: Annotated[int, 'meta'], + y: Literal['a', 'b'], + z: 'LiteralString' +)""" + self.assertEqual( + inspect.signature(func).format(), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=None), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline)), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline) - 1), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=0), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) - self.assertEqual(str(S(parameters=[ - P('foo', P.POSITIONAL_ONLY), - P('bar', P.VAR_KEYWORD)])), - '(foo, /, **bar)') + def test_signature_format_all_arg_types(self): + from typing import Annotated, Literal + + def func( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object, + ) -> None: + pass - self.assertEqual(str(S(parameters=[ - P('foo', P.POSITIONAL_ONLY), - P('bar', P.VAR_POSITIONAL)])), - '(foo, /, *bar)') + expected_multiline = """( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object +) -> None""" + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) def test_signature_replace_parameters(self): def test(a, b) -> 42: @@ -3904,6 +4711,8 @@ def foo(a, *, b:1): pass foo_sig = MySignature.from_callable(foo) self.assertIsInstance(foo_sig, MySignature) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signature_from_callable_class(self): # A regression test for a class inheriting its signature from `object`. class MySignature(inspect.Signature): pass @@ -3994,7 +4803,8 @@ def test_signature_eval_str(self): par('c', PORK, annotation="'MyClass'"), ))) - self.assertEqual(signature_func(isa.UnannotatedClass), sig()) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(signature_func(isa.UnannotatedClass), sig()) self.assertEqual(signature_func(isa.unannotated_function), sig( parameters=( @@ -4134,6 +4944,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): @@ -4499,15 +5319,30 @@ def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), (1, 2, 3, 4, 5, {})) - with self.assertRaisesRegex(TypeError, "but was passed as a keyword"): - self.call(test, 1, 2, foo=4, bar=5, c_po=10) + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) + + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): - self.call(test, 1, 2, c_po=4) + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) + + with self.assertRaisesRegex(TypeError, "missing 2 required positional arguments"): self.call(test, a_po=1, b_po=2) + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + def test_signature_bind_with_self_arg(self): # Issue #17071: one of the parameters is named "self def test(a, self, b): @@ -4711,10 +5546,17 @@ class TestSignatureDefinitions(unittest.TestCase): # This test case provides a home for checking that particular APIs # have signatures available for introspection + @staticmethod + def is_public(name): + return not name.startswith('_') or name.startswith('__') and name.endswith('__') + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - def test_builtins_have_signatures(self): + def _test_module_has_signatures(self, module, + no_signature=(), unsupported_signature=(), + methods_no_signature={}, methods_unsupported_signature={}, + good_exceptions=()): # This checks all builtin callables in CPython have signatures # A few have signatures Signature can't yet handle, so we skip those # since they will have to wait until PEP 457 adds the required @@ -4723,48 +5565,273 @@ def test_builtins_have_signatures(self): # reasons, so we also skip those for the time being, but design # the test to fail in order to indicate when it needs to be # updated. - no_signature = set() - # These need PEP 457 groups - needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} - no_signature |= needs_groups - # These have unrepresentable parameter default values of NULL - needs_null = {"anext"} - no_signature |= needs_null - # These need PEP 457 groups or a signature change to accept None - needs_semantic_update = {"round"} - no_signature |= needs_semantic_update - # These need *args support in Argument Clinic - needs_varargs = {"breakpoint", "min", "max", "print", - "__build_class__"} - no_signature |= needs_varargs - # These simply weren't covered in the initial AC conversion - # for builtin callables - not_converted_yet = {"open", "__import__"} - no_signature |= not_converted_yet - # These builtin types are expected to provide introspection info - types_with_signatures = set() + no_signature = no_signature or set() # Check the signatures we expect to be there - ns = vars(builtins) + ns = vars(module) + try: + names = set(module.__all__) + except AttributeError: + names = set(name for name in ns if self.is_public(name)) for name, obj in sorted(ns.items()): + if name not in names: + continue if not callable(obj): continue - # The builtin types haven't been converted to AC yet - if isinstance(obj, type) and (name not in types_with_signatures): - # Note that this also skips all the exception types + if (isinstance(obj, type) and + issubclass(obj, BaseException) and + name not in good_exceptions): no_signature.add(name) - if (name in no_signature): - # Not yet converted - continue - with self.subTest(builtin=name): - self.assertIsNotNone(inspect.signature(obj)) + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', builtin=name): + self.assertIsNotNone(inspect.signature(obj)) + if isinstance(obj, type): + with self.subTest(type=name): + self._test_builtin_methods_have_signatures(obj, + methods_no_signature.get(name, ()), + methods_unsupported_signature.get(name, ())) # Check callables that haven't been converted don't claim a signature # This ensures this test will start failing as more signatures are # added, so the affected items can be moved into the scope of the # regression test above for name in no_signature: - with self.subTest(builtin=name): + with self.subTest('none', builtin=name): + obj = ns[name] self.assertIsNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + for name in unsupported_signature: + with self.subTest('unsupported', builtin=name): + obj = ns[name] + self.assertIsNotNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + + def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_signature): + ns = vars(cls) + for name in ns: + obj = getattr(cls, name, None) + if not callable(obj) or isinstance(obj, type): + continue + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', method=name): + self.assertIsNotNone(inspect.signature(obj)) + for name in no_signature: + with self.subTest('none', method=name): + self.assertIsNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + for name in unsupported_signature: + with self.subTest('unsupported', method=name): + self.assertIsNotNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + + def test_builtins_have_signatures(self): + no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + # These need PEP 457 groups + needs_groups = {"range", "slice", "dir", "getattr", + "next", "iter", "vars"} + no_signature |= needs_groups + # These have unrepresentable parameter default values of NULL + unsupported_signature = {"anext"} + # These need *args support in Argument Clinic + needs_varargs = {"min", "max", "__build_class__"} + no_signature |= needs_varargs + + methods_no_signature = { + 'dict': {'update'}, + 'object': {'__class__'}, + } + methods_unsupported_signature = { + 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'dict': {'pop'}, + 'memoryview': {'cast', 'hex'}, + 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, + } + self._test_module_has_signatures(builtins, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_types_module_has_signatures(self): + unsupported_signature = {'CellType'} + methods_no_signature = { + 'AsyncGeneratorType': {'athrow'}, + 'CoroutineType': {'throw'}, + 'GeneratorType': {'throw'}, + } + self._test_module_has_signatures(types, + unsupported_signature=unsupported_signature, + methods_no_signature=methods_no_signature) + + def test_sys_module_has_signatures(self): + no_signature = {'getsizeof', 'set_asyncgen_hooks'} + no_signature |= {name for name in ['getobjects'] + if hasattr(sys, name)} + self._test_module_has_signatures(sys, no_signature) + + def test_abc_module_has_signatures(self): + import abc + self._test_module_has_signatures(abc) + + def test_atexit_module_has_signatures(self): + import atexit + self._test_module_has_signatures(atexit) + + def test_codecs_module_has_signatures(self): + import codecs + methods_no_signature = {'StreamReader': {'charbuffertype'}} + self._test_module_has_signatures(codecs, + methods_no_signature=methods_no_signature) + + def test_collections_module_has_signatures(self): + no_signature = {'OrderedDict', 'defaultdict'} + unsupported_signature = {'deque'} + methods_no_signature = { + 'OrderedDict': {'update'}, + } + methods_unsupported_signature = { + 'deque': {'index'}, + 'OrderedDict': {'pop'}, + 'UserString': {'maketrans'}, + } + self._test_module_has_signatures(collections, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_collections_abc_module_has_signatures(self): + import collections.abc + self._test_module_has_signatures(collections.abc) + + def test_errno_module_has_signatures(self): + import errno + self._test_module_has_signatures(errno) + + def test_faulthandler_module_has_signatures(self): + import faulthandler + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable'} + unsupported_signature |= {name for name in ['register'] + if hasattr(faulthandler, name)} + self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) + + def test_functools_module_has_signatures(self): + no_signature = {'reduce'} + self._test_module_has_signatures(functools, no_signature) + + def test_gc_module_has_signatures(self): + import gc + no_signature = {'set_threshold'} + self._test_module_has_signatures(gc, no_signature) + + def test_io_module_has_signatures(self): + methods_no_signature = { + 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, + } + self._test_module_has_signatures(io, + methods_no_signature=methods_no_signature) + + def test_itertools_module_has_signatures(self): + import itertools + no_signature = {'islice', 'repeat'} + self._test_module_has_signatures(itertools, no_signature) + + def test_locale_module_has_signatures(self): + import locale + self._test_module_has_signatures(locale) + + def test_marshal_module_has_signatures(self): + import marshal + self._test_module_has_signatures(marshal) + + def test_operator_module_has_signatures(self): + import operator + self._test_module_has_signatures(operator) + + def test_os_module_has_signatures(self): + unsupported_signature = {'chmod', 'utime'} + unsupported_signature |= {name for name in + ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + 'register_at_fork', 'startfile'] + if hasattr(os, name)} + self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) + + def test_pwd_module_has_signatures(self): + pwd = import_helper.import_module('pwd') + self._test_module_has_signatures(pwd) + + def test_re_module_has_signatures(self): + import re + methods_no_signature = {'Match': {'group'}} + self._test_module_has_signatures(re, + methods_no_signature=methods_no_signature, + good_exceptions={'error', 'PatternError'}) + + def test_signal_module_has_signatures(self): + import signal + self._test_module_has_signatures(signal) + + def test_stat_module_has_signatures(self): + import stat + self._test_module_has_signatures(stat) + + def test_string_module_has_signatures(self): + import string + self._test_module_has_signatures(string) + + def test_symtable_module_has_signatures(self): + import symtable + self._test_module_has_signatures(symtable) + + def test_sysconfig_module_has_signatures(self): + import sysconfig + self._test_module_has_signatures(sysconfig) + + def test_threading_module_has_signatures(self): + import threading + self._test_module_has_signatures(threading) + + def test_thread_module_has_signatures(self): + import _thread + no_signature = {'RLock'} + self._test_module_has_signatures(_thread, no_signature) + + def test_time_module_has_signatures(self): + no_signature = { + 'asctime', 'ctime', 'get_clock_info', 'gmtime', 'localtime', + 'strftime', 'strptime' + } + no_signature |= {name for name in + ['clock_getres', 'clock_settime', 'clock_settime_ns', + 'pthread_getcpuclockid'] + if hasattr(time, name)} + self._test_module_has_signatures(time, no_signature) + + def test_tokenize_module_has_signatures(self): + import tokenize + self._test_module_has_signatures(tokenize) + + def test_tracemalloc_module_has_signatures(self): + import tracemalloc + self._test_module_has_signatures(tracemalloc) + + def test_typing_module_has_signatures(self): + import typing + no_signature = {'ParamSpec', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Text', 'TypeAliasType', 'TypeVar', 'TypeVarTuple'} + methods_no_signature = { + 'Generic': {'__class_getitem__', '__init_subclass__'}, + } + methods_unsupported_signature = { + 'Text': {'count', 'find', 'index', 'rfind', 'rindex', 'startswith', 'endswith', 'maketrans'}, + } + self._test_module_has_signatures(typing, no_signature, + methods_no_signature=methods_no_signature, + methods_unsupported_signature=methods_unsupported_signature) + + def test_warnings_module_has_signatures(self): + unsupported_signature = {'warn', 'warn_explicit'} + self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) + + def test_weakref_module_has_signatures(self): + import weakref + no_signature = {'ReferenceType', 'ref'} + self._test_module_has_signatures(weakref, no_signature) def test_python_function_override_signature(self): def func(*args, **kwargs): @@ -4796,6 +5863,7 @@ def func(*args, **kwargs): with self.assertRaises(ValueError): inspect.signature(func) + @support.requires_docstrings def test_base_class_have_text_signature(self): # see issue 43118 from test.typinganndata.ann_module7 import BufferedReader @@ -4881,6 +5949,14 @@ def test_recursion_limit(self): with self.assertRaisesRegex(ValueError, 'wrapper loop'): inspect.unwrap(obj) + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') @@ -4961,5 +6037,66 @@ def test_getsource_reload(self): self.assertInspectEqual(path, module) +class TestRepl(unittest.TestCase): + + def spawn_repl(self, *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): + """Run the Python REPL with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ + + # To run the REPL without using a terminal, spawn python with the command + # line option '-i' and the process name set to ''. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + + def run_on_interactive_mode(self, source): + """Spawn a new Python interpreter, pass the given + input source code from the stdin and return the + result back. If the interpreter exits non-zero, it + raises a ValueError.""" + + process = self.spawn_repl() + process.stdin.write(source) + output = kill_python(process) + + if process.returncode != 0: + raise ValueError("Process didn't exit properly.") + return output + + @unittest.skipIf(not has_subprocess_support, "test requires subprocess") + def test_getsource(self): + output = self.run_on_interactive_mode(textwrap.dedent("""\ + def f(): + print(0) + return 1 + 2 + + import inspect + print(f"The source is: <<<{inspect.getsource(f)}>>>") + """)) + + expected = "The source is: <<>>" + self.assertIn(expected, output) + + + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 5545ee39d8e942..8870d7aa5d663d 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,5 +1,4 @@ import sys -import time import unittest from unittest import mock @@ -12,6 +11,11 @@ except ImportError: _pylong = None +try: + import _decimal +except ImportError: + _decimal = None + L = [ ('0', 0), ('1', 1), @@ -90,6 +94,7 @@ def test_basic(self): self.assertRaises(TypeError, int, 1, 12) + self.assertRaises(TypeError, int, "10", 2, 1) self.assertEqual(int('0o123', 0), 83) self.assertEqual(int('0x123', 16), 291) @@ -396,68 +401,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError), \ - self.assertWarns(DeprecationWarning): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - with self.assertWarns(DeprecationWarning): - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError), \ - self.assertWarns(DeprecationWarning): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -508,18 +453,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -543,26 +476,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - with self.assertWarns(DeprecationWarning): - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - with self.assertWarns(DeprecationWarning): - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, @@ -603,6 +516,13 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) class IntStrDigitLimitsTests(unittest.TestCase): @@ -663,84 +583,78 @@ def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 50_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. digits = 78_268 - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): huge_decimal = str(huge_int) - seconds_to_convert = get_time() - start self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') # We test with the limit almost at the size needed to check performance. # The performant limit check is slightly fuzzy, give it a some room. with support.adjust_int_max_str_digits(int(.995 * digits)): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): str(huge_int) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 100_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic digits = 133700 huge = '8'*digits - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): int(huge) - seconds_to_convert = get_time() - start # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') with support.adjust_int_max_str_digits(digits - 1): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): int(huge) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge = '7'*1_200_000 - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_power_of_two_bases_unlimited(self): """The limit does not apply to power of 2 bases.""" @@ -834,17 +748,28 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_limit) super().tearDown() - def test_pylong_int_to_decimal(self): - n = (1 << 100_000) - 1 - suffix = '9883109375' + def _test_pylong_int_to_decimal(self, n, suffix): s = str(n) - assert s[-10:] == suffix - s = str(-n) - assert s[-10:] == suffix - s = '%d' % n - assert s[-10:] == suffix - s = b'%d' % n - assert s[-10:] == suffix.encode('ascii') + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') def test_pylong_int_divmod(self): n = (1 << 100_000) @@ -900,6 +825,100 @@ def test_pylong_misbehavior_error_path_from_str( with self.assertRaises(RuntimeError): int(big_value) + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 + + @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py deleted file mode 100644 index 0910b51bfe5dbd..00000000000000 --- a/Lib/test/test_interpreters.py +++ /dev/null @@ -1,1112 +0,0 @@ -import contextlib -import json -import os -import os.path -import sys -import threading -from textwrap import dedent -import unittest -import time - -from test import support -from test.support import import_helper -from test.support import threading_helper -from test.support import os_helper -_interpreters = import_helper.import_module('_xxsubinterpreters') -_channels = import_helper.import_module('_xxinterpchannels') -from test.support import interpreters - - -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): - {indented} - """) - return wrapped, open(r, encoding='utf-8') - - -def clean_up_interpreters(): - for interp in interpreters.list_all(): - if interp.id == 0: # main - continue - try: - interp.close() - except RuntimeError: - pass # already destroyed - - -def _run_output(interp, request, channels=None): - script, rpipe = _captured_script(request) - with rpipe: - interp.run(script, channels=channels) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interp.run(dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -class TestBase(unittest.TestCase): - - def pipe(self): - def ensure_closed(fd): - try: - os.close(fd) - except OSError: - pass - r, w = os.pipe() - self.addCleanup(lambda: ensure_closed(r)) - self.addCleanup(lambda: ensure_closed(w)) - return r, w - - def tearDown(self): - clean_up_interpreters() - - -class CreateTests(TestBase): - - def test_in_main(self): - interp = interpreters.create() - self.assertIsInstance(interp, interpreters.Interpreter) - self.assertIn(interp, interpreters.list_all()) - - def test_in_thread(self): - lock = threading.Lock() - interp = None - def f(): - nonlocal interp - interp = interpreters.create() - lock.acquire() - lock.release() - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(interp, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - interp = interpreters.create() - print(interp.id) - """)) - interp2 = interpreters.Interpreter(int(out)) - self.assertEqual(interpreters.list_all(), [main, interp, interp2]) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp_lst = [] - for _ in range(3): - interps = interpreters.create() - interp_lst.append(interps) - # Now destroy them. - for interp in interp_lst: - interp.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - # Now destroy 2 of them. - interp1.close() - interp2.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) - - -class GetCurrentTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - current = interpreters.get_current() - self.assertEqual(current, main) - - def test_subinterpreter(self): - main = _interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(cur.id) - """)) - current = interpreters.Interpreter(int(out)) - self.assertNotEqual(current, main) - - -class ListAllTests(TestBase): - - def test_initial(self): - interps = interpreters.list_all() - self.assertEqual(1, len(interps)) - - def test_after_creating(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, first.id, second.id]) - - def test_after_destroying(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - first.close() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, second.id]) - - -class TestInterpreterAttrs(TestBase): - - def test_id_type(self): - main = interpreters.get_main() - current = interpreters.get_current() - interp = interpreters.create() - self.assertIsInstance(main.id, _interpreters.InterpreterID) - self.assertIsInstance(current.id, _interpreters.InterpreterID) - self.assertIsInstance(interp.id, _interpreters.InterpreterID) - - def test_main_id(self): - main = interpreters.get_main() - self.assertEqual(main.id, 0) - - def test_custom_id(self): - interp = interpreters.Interpreter(1) - self.assertEqual(interp.id, 1) - - with self.assertRaises(TypeError): - interpreters.Interpreter('1') - - def test_id_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.id = 2 - - @unittest.skip('not ready yet (see bpo-32604)') - def test_main_isolated(self): - main = interpreters.get_main() - self.assertFalse(main.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_subinterpreter_isolated_default(self): - interp = interpreters.create() - self.assertFalse(interp.isolated) - - def test_subinterpreter_isolated_explicit(self): - interp1 = interpreters.create(isolated=True) - interp2 = interpreters.create(isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_custom_isolated_default(self): - interp = interpreters.Interpreter(1) - self.assertFalse(interp.isolated) - - def test_custom_isolated_explicit(self): - interp1 = interpreters.Interpreter(1, isolated=True) - interp2 = interpreters.Interpreter(1, isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - def test_isolated_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.isolated = True - - def test_equality(self): - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(interp1, interp1) - self.assertNotEqual(interp1, interp2) - - -class TestInterpreterIsRunning(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(main.is_running()) - - @unittest.skip('Fails on FreeBSD') - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interp.is_running()) - - with _running(interp): - self.assertTrue(interp.is_running()) - self.assertFalse(interp.is_running()) - - def test_finished(self): - r, w = self.pipe() - interp = interpreters.create() - interp.run(f"""if True: - import os - os.write({w}, b'x') - """) - self.assertFalse(interp.is_running()) - self.assertEqual(os.read(r, 1), b'x') - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp.id}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.is_running() - - def test_with_only_background_threads(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - self.assertFalse(interp.is_running()) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterClose(TestBase): - - def test_basic(self): - main = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2, interp3}) - interp2.close() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp3}) - - def test_all(self): - before = set(interpreters.list_all()) - interps = set() - for _ in range(3): - interp = interpreters.create() - interps.add(interp) - self.assertEqual(set(interpreters.list_all()), before | interps) - for interp in interps: - interp.close() - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - main.close() - - def f(): - with self.assertRaises(RuntimeError): - main.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.close() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.close() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.close() - - def test_from_current(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - from test.support import interpreters - interp = interpreters.Interpreter({int(interp.id)}) - try: - interp.close() - except RuntimeError: - print('failed') - """)) - self.assertEqual(out.strip(), 'failed') - self.assertEqual(set(interpreters.list_all()), {main, interp}) - - def test_from_sibling(self): - main, = interpreters.list_all() - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2}) - interp1.run(dedent(f""" - from test.support import interpreters - interp2 = interpreters.Interpreter(int({interp2.id})) - interp2.close() - interp3 = interpreters.create() - interp3.close() - """)) - self.assertEqual(set(interpreters.list_all()), {main, interp1}) - - def test_from_other_thread(self): - interp = interpreters.create() - def f(): - interp.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - @unittest.skip('Fails on FreeBSD') - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.close() - self.assertTrue(interp.is_running()) - - def test_subthreads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - import time - - done = False - - def notify_fini(): - global done - done = True - t.join() - threading._register_atexit(notify_fini) - - def task(): - while not done: - time.sleep(0.1) - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - interp.close() - - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - interp.run(script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_in_thread(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - def f(): - interp.run(script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - @support.requires_fork() - def test_fork(self): - interp = interpreters.create() - import tempfile - with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w', encoding='utf-8') as out: - out.write('{expected}') - """) - interp.run(script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - @unittest.skip('Fails on FreeBSD') - def test_already_running(self): - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.run('print("spam")') - - def test_bad_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(10) - - def test_bytes_for_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(b'print("spam")') - - def test_with_background_threads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - RAN = b'R' - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - os.write({w_interp}, {RAN!r}) - """) - interp.run(f"""if True: - os.write({w_interp}, {RAN!r}) - """) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), FINISHED) - - # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. - - -class StressTests(TestBase): - - # In these tests we generally want a lot of interpreters, - # but not so many that any test takes too long. - - @support.requires_resource('cpu') - def test_create_many_sequential(self): - alive = [] - for _ in range(100): - interp = interpreters.create() - alive.append(interp) - - @support.requires_resource('cpu') - def test_create_many_threaded(self): - alive = [] - def task(): - interp = interpreters.create() - alive.append(interp) - threads = (threading.Thread(target=task) for _ in range(200)) - with threading_helper.start_threads(threads): - pass - - -class StartupTests(TestBase): - - # We want to ensure the initial state of subinterpreters - # matches expectations. - - _subtest_count = 0 - - @contextlib.contextmanager - def subTest(self, *args): - with super().subTest(*args) as ctx: - self._subtest_count += 1 - try: - yield ctx - finally: - if self._debugged_in_subtest: - if self._subtest_count == 1: - # The first subtest adds a leading newline, so we - # compensate here by not printing a trailing newline. - print('### end subtest debug ###', end='') - else: - print('### end subtest debug ###') - self._debugged_in_subtest = False - - def debug(self, msg, *, header=None): - if header: - self._debug(f'--- {header} ---') - if msg: - if msg.endswith(os.linesep): - self._debug(msg[:-len(os.linesep)]) - else: - self._debug(msg) - self._debug('') - self._debug('------') - else: - self._debug(msg) - - _debugged = False - _debugged_in_subtest = False - def _debug(self, msg): - if not self._debugged: - print() - self._debugged = True - if self._subtest is not None: - if True: - if not self._debugged_in_subtest: - self._debugged_in_subtest = True - print('### start subtest debug ###') - print(msg) - else: - print(msg) - - def create_temp_dir(self): - import tempfile - tmp = tempfile.mkdtemp(prefix='test_interpreters_') - tmp = os.path.realpath(tmp) - self.addCleanup(os_helper.rmtree, tmp) - return tmp - - def write_script(self, *path, text): - filename = os.path.join(*path) - dirname = os.path.dirname(filename) - if dirname: - os.makedirs(dirname, exist_ok=True) - with open(filename, 'w', encoding='utf-8') as outfile: - outfile.write(dedent(text)) - return filename - - @support.requires_subprocess() - def run_python(self, argv, *, cwd=None): - # This method is inspired by - # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. - import shlex - import subprocess - if isinstance(argv, str): - argv = shlex.split(argv) - argv = [sys.executable, *argv] - try: - proc = subprocess.run( - argv, - cwd=cwd, - capture_output=True, - text=True, - ) - except Exception as exc: - self.debug(f'# cmd: {shlex.join(argv)}') - if isinstance(exc, FileNotFoundError) and not exc.filename: - if os.path.exists(argv[0]): - exists = 'exists' - else: - exists = 'does not exist' - self.debug(f'{argv[0]} {exists}') - raise # re-raise - assert proc.stderr == '' or proc.returncode != 0, proc.stderr - if proc.returncode != 0 and support.verbose: - self.debug(f'# python3 {shlex.join(argv[1:])} failed:') - self.debug(proc.stdout, header='stdout') - self.debug(proc.stderr, header='stderr') - self.assertEqual(proc.returncode, 0) - self.assertEqual(proc.stderr, '') - return proc.stdout - - def test_sys_path_0(self): - # The main interpreter's sys.path[0] should be used by subinterpreters. - script = ''' - import sys - from test.support import interpreters - - orig = sys.path[0] - - interp = interpreters.create() - interp.run(f"""if True: - import json - import sys - print(json.dumps({{ - 'main': {orig!r}, - 'sub': sys.path[0], - }}, indent=4), flush=True) - """) - ''' - # / - # pkg/ - # __init__.py - # __main__.py - # script.py - # script.py - cwd = self.create_temp_dir() - self.write_script(cwd, 'pkg', '__init__.py', text='') - self.write_script(cwd, 'pkg', '__main__.py', text=script) - self.write_script(cwd, 'pkg', 'script.py', text=script) - self.write_script(cwd, 'script.py', text=script) - - cases = [ - ('script.py', cwd), - ('-m script', cwd), - ('-m pkg', cwd), - ('-m pkg.script', cwd), - ('-c "import script"', ''), - ] - for argv, expected in cases: - with self.subTest(f'python3 {argv}'): - out = self.run_python(argv, cwd=cwd) - data = json.loads(out) - sp0_main, sp0_sub = data['main'], data['sub'] - self.assertEqual(sp0_sub, sp0_main) - self.assertEqual(sp0_sub, expected) - # XXX Also check them all with the -P cmdline flag? - - -class FinalizationTests(TestBase): - - def test_gh_109793(self): - import subprocess - argv = [sys.executable, '-c', '''if True: - import _xxsubinterpreters as _interpreters - interpid = _interpreters.create() - raise Exception - '''] - proc = subprocess.run(argv, capture_output=True, text=True) - self.assertIn('Traceback', proc.stderr) - if proc.returncode == 0 and support.verbose: - print() - print("--- cmd unexpected succeeded ---") - print(f"stdout:\n{proc.stdout}") - print(f"stderr:\n{proc.stderr}") - print("------") - self.assertEqual(proc.returncode, 1) - - -class TestIsShareable(TestBase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - ] - for obj in shareables: - with self.subTest(obj): - shareable = interpreters.is_shareable(obj) - self.assertTrue(shareable) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - True, - False, - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - 100.0, - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class TestChannels(TestBase): - - def test_create(self): - r, s = interpreters.create_channel() - self.assertIsInstance(r, interpreters.RecvChannel) - self.assertIsInstance(s, interpreters.SendChannel) - - def test_list_all(self): - self.assertEqual(interpreters.list_all_channels(), []) - created = set() - for _ in range(3): - ch = interpreters.create_channel() - created.add(ch) - after = set(interpreters.list_all_channels()) - self.assertEqual(after, created) - - @unittest.expectedFailure # See gh-110318: - def test_shareable(self): - rch, sch = interpreters.create_channel() - - self.assertTrue( - interpreters.is_shareable(rch)) - self.assertTrue( - interpreters.is_shareable(sch)) - - sch.send_nowait(rch) - sch.send_nowait(sch) - rch2 = rch.recv() - sch2 = rch.recv() - - self.assertEqual(rch2, rch) - self.assertEqual(sch2, sch) - - -class TestRecvChannelAttrs(TestBase): - - def test_id_type(self): - rch, _ = interpreters.create_channel() - self.assertIsInstance(rch.id, _channels.ChannelID) - - def test_custom_id(self): - rch = interpreters.RecvChannel(1) - self.assertEqual(rch.id, 1) - - with self.assertRaises(TypeError): - interpreters.RecvChannel('1') - - def test_id_readonly(self): - rch = interpreters.RecvChannel(1) - with self.assertRaises(AttributeError): - rch.id = 2 - - def test_equality(self): - ch1, _ = interpreters.create_channel() - ch2, _ = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendChannelAttrs(TestBase): - - def test_id_type(self): - _, sch = interpreters.create_channel() - self.assertIsInstance(sch.id, _channels.ChannelID) - - def test_custom_id(self): - sch = interpreters.SendChannel(1) - self.assertEqual(sch.id, 1) - - with self.assertRaises(TypeError): - interpreters.SendChannel('1') - - def test_id_readonly(self): - sch = interpreters.SendChannel(1) - with self.assertRaises(AttributeError): - sch.id = 2 - - def test_equality(self): - _, ch1 = interpreters.create_channel() - _, ch2 = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendRecv(TestBase): - - def test_send_recv_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_send_recv_different_threads(self): - r, s = interpreters.create_channel() - - def f(): - while True: - try: - obj = r.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - s.send(obj) - t = threading.Thread(target=f) - t.start() - - orig = b'spam' - s.send(orig) - obj = r.recv() - t.join() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main_with_default(self): - r, _ = interpreters.create_channel() - obj = r.recv_nowait(None) - - self.assertIsNone(obj) - - def test_send_recv_nowait_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - assert obj == orig, 'expected: obj == orig' - # When going back to the same interpreter we get the same object. - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_nowait_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv_nowait() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv_nowait() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_recv_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv() - - def test_send_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send(b'spam') - - def test_recv_nowait_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv_nowait() - - def test_send_nowait_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send_nowait(b'spam') - - def test_recv_nowait_empty(self): - ch, _ = interpreters.create_channel() - with self.assertRaises(interpreters.ChannelEmptyError): - ch.recv_nowait() - - def test_recv_nowait_default(self): - default = object() - rch, sch = interpreters.create_channel() - obj1 = rch.recv_nowait(default) - sch.send_nowait(None) - sch.send_nowait(1) - sch.send_nowait(b'spam') - sch.send_nowait(b'eggs') - obj2 = rch.recv_nowait(default) - obj3 = rch.recv_nowait(default) - obj4 = rch.recv_nowait() - obj5 = rch.recv_nowait(default) - obj6 = rch.recv_nowait(default) - - self.assertIs(obj1, default) - self.assertIs(obj2, None) - self.assertEqual(obj3, 1) - self.assertEqual(obj4, b'spam') - self.assertEqual(obj5, b'eggs') - self.assertIs(obj6, default) - - def test_send_buffer(self): - buf = bytearray(b'spamspamspam') - obj = None - rch, sch = interpreters.create_channel() - - def f(): - nonlocal obj - while True: - try: - obj = rch.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - t = threading.Thread(target=f) - t.start() - - sch.send_buffer(buf) - t.join() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) - - def test_send_buffer_nowait(self): - buf = bytearray(b'spamspamspam') - rch, sch = interpreters.create_channel() - sch.send_buffer_nowait(buf) - obj = rch.recv() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py new file mode 100644 index 00000000000000..e3d189c4efcd27 --- /dev/null +++ b/Lib/test/test_interpreters/__init__.py @@ -0,0 +1,9 @@ +import os +from test.support import load_package_tests, Py_GIL_DISABLED +import unittest + +if Py_GIL_DISABLED: + raise unittest.SkipTest("GIL disabled") + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py new file mode 100644 index 00000000000000..40a23a297ec2b4 --- /dev/null +++ b/Lib/test/test_interpreters/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py new file mode 100644 index 00000000000000..5e3d7a052bae91 --- /dev/null +++ b/Lib/test/test_interpreters/test_api.py @@ -0,0 +1,1664 @@ +import os +import pickle +from textwrap import dedent +import threading +import types +import unittest + +from test import support +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_interpreters = import_helper.import_module('_interpreters') +from test.support import Py_GIL_DISABLED +from test.support import interpreters +from test.support import force_not_colorized +from test.support.interpreters import ( + InterpreterError, InterpreterNotFoundError, ExecutionFailed, +) +from .utils import ( + _captured_script, _run_output, _running, TestBase, + requires_test_modules, _testinternalcapi, +) + + +WHENCE_STR_UNKNOWN = 'unknown' +WHENCE_STR_RUNTIME = 'runtime init' +WHENCE_STR_LEGACY_CAPI = 'legacy C-API' +WHENCE_STR_CAPI = 'C-API' +WHENCE_STR_XI = 'cross-interpreter C-API' +WHENCE_STR_STDLIB = '_interpreters module' + + +class ModuleTests(TestBase): + + def test_queue_aliases(self): + first = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + second = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + self.assertEqual(second, first) + + +class CreateTests(TestBase): + + def test_in_main(self): + interp = interpreters.create() + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertIn(interp, interpreters.list_all()) + + def test_in_thread(self): + lock = threading.Lock() + interp = None + def f(): + nonlocal interp + interp = interpreters.create() + lock.acquire() + lock.release() + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(interp, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + interp = interpreters.create() + print(interp.id) + """)) + interp2 = interpreters.Interpreter(int(out)) + self.assertEqual(interpreters.list_all(), [main, interp, interp2]) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.close() + interp2.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) + + +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertEqual(current, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(cur.id) + """)) + current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) + self.assertNotEqual(current, main) + + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = id(interp) + self.assertNotEqual(id1, id2) + + @requires_test_modules + def test_created_with_capi(self): + expected = _testinternalcapi.next_interpreter_id() + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print((interp.id, interp.whence)) + """) + interpid, whence = eval(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, WHENCE_STR_CAPI) + + +class ListAllTests(TestBase): + + def test_initial(self): + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) + + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.close() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + def test_created_with_capi(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected = [ + (mainid, WHENCE_STR_RUNTIME), + (interpid1, WHENCE_STR_STDLIB), + (interpid2, WHENCE_STR_STDLIB), + (interpid3, WHENCE_STR_STDLIB), + (interpid4, WHENCE_STR_CAPI), + (interpid5, WHENCE_STR_STDLIB), + ] + expected2 = expected[:-2] + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.create() + print( + [(i.id, i.whence) for i in interpreters.list_all()]) + """) + res = eval(text) + res2 = [(i.id, i.whence) for i in interpreters.list_all()] + self.assertEqual(res, expected) + self.assertEqual(res2, expected2) + + +class InterpreterObjectTests(TestBase): + + def test_init_int(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_init_interpreter_id(self): + interpid = interpreters.get_current()._id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp._id, interpid) + + def test_init_unsupported(self): + actualid = interpreters.get_current().id + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(main.id) + self.assertIs(interp, main) + + def test_init_does_not_exist(self): + with self.assertRaises(InterpreterNotFoundError): + interpreters.Interpreter(1_000_000) + + def test_init_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + + def test_id_type(self): + main = interpreters.get_main() + current = interpreters.get_current() + interp = interpreters.create() + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) + + def test_id_readonly(self): + interp = interpreters.create() + with self.assertRaises(AttributeError): + interp.id = 1_000_000 + + def test_whence(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertEqual(main.whence, WHENCE_STR_RUNTIME) + + with self.subTest('from _interpreters'): + self.assertEqual(interp.whence, WHENCE_STR_STDLIB) + + with self.subTest('from C-API'): + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(repr(interp.whence)) + """) + whence = eval(text) + self.assertEqual(whence, WHENCE_STR_CAPI) + + with self.subTest('readonly'): + for value in [ + None, + WHENCE_STR_UNKNOWN, + WHENCE_STR_RUNTIME, + WHENCE_STR_STDLIB, + WHENCE_STR_CAPI, + ]: + with self.assertRaises(AttributeError): + interp.whence = value + with self.assertRaises(AttributeError): + main.whence = value + + def test_hashable(self): + interp = interpreters.create() + expected = hash(interp.id) + actual = hash(interp) + self.assertEqual(actual, expected) + + def test_equality(self): + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(interp1, interp1) + self.assertNotEqual(interp1, interp2) + + def test_pickle(self): + interp = interpreters.create() + data = pickle.dumps(interp) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + + +class TestInterpreterIsRunning(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(main.is_running()) + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_finished(self): + r, w = self.pipe() + interp = interpreters.create() + interp.exec(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.is_running() + + def test_with_only_background_threads(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.exec('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.is_running()) + """) + def parse_results(text): + self.assertNotEqual(text, "") + try: + return eval(text) + except Exception: + raise Exception(repr(text)) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + text = self.run_from_capi(interpid, script, main=True) + running = parse_results(text) + self.assertTrue(running) + + with self.subTest('running, but not __main__ (from self)'): + text = self.run_temp_from_capi(script) + running = parse_results(text) + self.assertFalse(running) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=True): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertTrue(during) + self.assertFalse(after) + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=False): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertFalse(during) + self.assertFalse(after) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, _): + running = interp.is_running() + self.assertFalse(running) + + +class TestInterpreterClose(TestBase): + + def test_basic(self): + main = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2, interp3}) + interp2.close() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp3}) + + def test_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(set(interpreters.list_all()), before | interps) + for interp in interps: + interp.close() + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(InterpreterError): + main.close() + + def f(): + with self.assertRaises(InterpreterError): + main.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.close() + + def test_from_current(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + from test.support import interpreters + interp = interpreters.Interpreter({interp.id}) + try: + interp.close() + except interpreters.InterpreterError: + print('failed') + """)) + self.assertEqual(out.strip(), 'failed') + self.assertEqual(set(interpreters.list_all()), {main, interp}) + + def test_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2}) + interp1.exec(dedent(f""" + from test.support import interpreters + interp2 = interpreters.Interpreter({interp2.id}) + interp2.close() + interp3 = interpreters.create() + interp3.close() + """)) + self.assertEqual(set(interpreters.list_all()), {main, interp1}) + + def test_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(InterpreterError): + interp.close() + self.assertTrue(interp.is_running()) + + def test_subthreads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + interp.close() + """) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_from_capi(interpid, script, main=True) + + with self.subTest('running, but not __main__ (from self)'): + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_temp_from_capi(script) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=True): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + # The rest would be skipped until we deal with running threads when + # interp.close() is called. However, the "whence" restrictions + # trigger first. + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=False): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + self.assertTrue( + self.interp_exists(interpid)) + + +class TestInterpreterPrepareMain(TestBase): + + def test_empty(self): + interp = interpreters.create() + with self.assertRaises(ValueError): + interp.prepare_main() + + def test_dict(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_tuple(self): + values = {'spam': 42, 'eggs': 'ham'} + values = tuple(values.items()) + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(**values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_dict_and_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values, foo='bar') + out = _run_output(interp, dedent(""" + print(spam, eggs, foo) + """)) + self.assertEqual(out.strip(), '42 ham bar') + + def test_not_shareable(self): + interp = interpreters.create() + # XXX TypeError? + with self.assertRaises(ValueError): + interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) + + # Make sure neither was actually bound. + with self.assertRaises(ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(ExecutionFailed): + interp.exec('print(spam)') + + def test_running(self): + interp = interpreters.create() + interp.prepare_main({'spam': True}) + with self.running(interp): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.prepare_main({'spam': False}) + interp.exec('assert spam is True') + + @requires_test_modules + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.prepare_main({'spam': True}) + with self.assertRaisesRegex(ExecutionFailed, 'NameError'): + self.run_from_capi(interpid, 'assert spam is True') + + +class TestInterpreterExec(TestBase): + + def test_success(self): + interp = interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + interp.exec(script) + results = results.final() + results.raise_if_failed() + out = results.stdout + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.exec('raise Exception') + + @force_not_colorized + def test_display_preserved_exception(self): + tempdir = self.temp_dir() + modfile = self.make_module('spam', tempdir, text=""" + def ham(): + raise RuntimeError('uh-oh!') + + def eggs(): + ham() + """) + scriptfile = self.make_script('script.py', tempdir, text=""" + from test.support import interpreters + + def script(): + import spam + spam.eggs() + + interp = interpreters.create() + interp.exec(script) + """) + + stdout, stderr = self.assert_python_failure(scriptfile) + self.maxDiff = None + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec + self.assertEqual(stderr, dedent(f"""\ + Traceback (most recent call last): + File "{scriptfile}", line 9, in + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ + {interpmod_line.strip()} + raise ExecutionFailed(excinfo) + test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + + Uncaught in the interpreter: + + Traceback (most recent call last): + File "{scriptfile}", line 6, in script + spam.eggs() + ~~~~~~~~~^^ + File "{modfile}", line 6, in eggs + ham() + ~~~^^ + File "{modfile}", line 3, in ham + raise RuntimeError('uh-oh!') + RuntimeError: uh-oh! + """)) + self.assertEqual(stdout, '') + + def test_in_thread(self): + interp = interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + def f(): + interp.exec(script) + + t = threading.Thread(target=f) + t.start() + t.join() + results = results.final() + results.raise_if_failed() + out = results.stdout + + self.assertEqual(out, 'it worked!') + + @support.requires_fork() + def test_fork(self): + interp = interpreters.create() + import tempfile + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w', encoding='utf-8') as out: + out.write('{expected}') + """) + interp.exec(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_already_running(self): + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.exec('print("spam")') + + def test_bad_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec(10) + + def test_bytes_for_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec(b'print("spam")') + + def test_with_background_threads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.exec(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.exec('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, _): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.exec('raise Exception("it worked!")') + + # test__interpreters covers the remaining + # Interpreter.exec() behavior. + + +def call_func_noop(): + pass + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_not_shareable(): + return [1, 2, 3] + + +def call_func_failure(): + raise Exception('spam!') + + +def call_func_ident(value): + return value + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + def test_call(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable) + self.assertIs(res, None) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.assertRaises(Exception): + if args or kwargs: + raise Exception((args, kwargs)) + interp.call(callable) + + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + def test_call_in_thread(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + if args or kwargs: + continue + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) + t.join() + self.assertIsNotNone(ctx.caught) + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + True, + False, + 100.0, + (), + (1, ('spam', 'eggs'), True), + ] + for obj in shareables: + with self.subTest(obj): + shareable = interpreters.is_shareable(obj) + self.assertTrue(shareable) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_main(self): + interpid, whence = _interpreters.get_main() + self.assertEqual(interpid, 0) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_RUNTIME) + + def test_get_current(self): + with self.subTest('main'): + main, *_ = _interpreters.get_main() + interpid, whence = _interpreters.get_current() + self.assertEqual(interpid, main) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + script = f""" + import _interpreters + interpid, whence = _interpreters.get_current() + print((interpid, whence)) + """ + def parse_stdout(text): + interpid, whence = eval(text) + return interpid, whence + + with self.subTest('from _interpreters'): + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, orig) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + with self.subTest('from C-API'): + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = last + 1 + text = self.run_temp_from_capi(script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + def test_list_all(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + expected = [ + (mainid, _interpreters.WHENCE_RUNTIME), + (interpid1, _interpreters.WHENCE_STDLIB), + (interpid2, _interpreters.WHENCE_STDLIB), + (interpid3, _interpreters.WHENCE_STDLIB), + ] + + with self.subTest('main'): + res = _interpreters.list_all() + self.assertEqual(res, expected) + + with self.subTest('via interp from _interpreters'): + text = self.run_and_capture(interpid2, f""" + import _interpreters + print( + _interpreters.list_all()) + """) + + res = eval(text) + self.assertEqual(res, expected) + + with self.subTest('via interp from C-API'): + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected2 = expected + [ + (interpid4, _interpreters.WHENCE_CAPI), + (interpid5, _interpreters.WHENCE_STDLIB), + ] + expected3 = expected + [ + (interpid5, _interpreters.WHENCE_STDLIB), + ] + text = self.run_temp_from_capi(f""" + import _interpreters + _interpreters.create() + print( + _interpreters.list_all()) + """) + res2 = eval(text) + res3 = _interpreters.list_all() + self.assertEqual(res2, expected2) + self.assertEqual(res3, expected3) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no args'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: \'empty\''): + with self.assertRaises(InterpreterError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.check_multi_interp_extensions = bool(Py_GIL_DISABLED) + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('whence'): + interpid = _interpreters.create() + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_STDLIB) + + @requires_test_modules + def test_destroy(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before = [id for id, *_ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, *_ in _interpreters.list_all()] + + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + self.assertFalse( + self.interp_exists(interpid)) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + with self.assertRaises(InterpreterError): + # It is the current interpreter. + _interpreters.destroy(interpid) + + with self.subTest('from C-API'): + interpid = _testinternalcapi.create_interpreter() + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.destroy(interpid, restrict=True) + self.assertTrue( + self.interp_exists(interpid)) + _interpreters.destroy(interpid) + self.assertFalse( + self.interp_exists(interpid)) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('from C-API'): + orig = _interpreters.new_config('isolated') + with self.interpreter_from_capi(orig) as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.get_config(interpid, restrict=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_test_modules + def test_whence(self): + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + with self.subTest('stdlib'): + interpid = _interpreters.create() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + for orig, name in { + _interpreters.WHENCE_UNKNOWN: 'not ready', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + }.items(): + with self.subTest(f'from C-API ({orig}: {name})'): + with self.interpreter_from_capi(whence=orig) as interpid: + whence = _interpreters.whence(interpid) + self.assertEqual(whence, orig) + + with self.subTest('from C-API, running'): + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=True) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + with self.subTest('from legacy C-API, running'): + ... + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=False) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + + def test_is_running(self): + def check(interpid, expected): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.is_running(interpid, restrict=True) + running = _interpreters.is_running(interpid) + self.assertIs(running, expected) + + with self.subTest('from _interpreters (running)'): + interpid = _interpreters.create() + with self.running(interpid): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('from _interpreters (not running)'): + interpid = _interpreters.create() + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + check(interpid, True) + + with self.subTest('from C-API (running __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=True): + check(interpid, True) + + with self.subTest('from C-API (running, but not __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=False): + check(interpid, False) + + with self.subTest('from C-API (not running)'): + with self.interpreter_from_capi() as interpid: + check(interpid, False) + + def test_exec(self): + with self.subTest('run script'): + interpid = _interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + exc = _interpreters.exec(interpid, script) + results = results.final() + results.raise_if_failed() + out = results.stdout + self.assertEqual(out, 'it worked!') + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + script, results = _captured_script(""" + raise Exception('uh-oh!') + print("it worked!", end="") + """) + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + self.assertEqual(out, '') + self.assert_ns_equal(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='uh-oh!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.exec(interpid, 'raise Exception("it worked!")', + restrict=True) + exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') + self.assertIsNot(exc, None) + self.assertEqual(exc.msg, 'it worked!') + + def test_call(self): + with self.subTest('no args'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_failure) + self.assertEqual(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='spam!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + @requires_test_modules + def test_set___main___attrs(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.set___main___attrs(interpid, {'spam': True}, + restrict=True) + _interpreters.set___main___attrs(interpid, {'spam': True}) + rc = _testinternalcapi.exec_interpreter( + interpid, + 'assert spam is True', + ) + self.assertEqual(rc, 0) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py new file mode 100644 index 00000000000000..6c37754142e361 --- /dev/null +++ b/Lib/test/test_interpreters/test_channels.py @@ -0,0 +1,378 @@ +import importlib +import pickle +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_channels = import_helper.import_module('_interpchannels') +from test.support import interpreters +from test.support.interpreters import channels +from .utils import _run_output, TestBase + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + # Additional tests are found in Lib/test/test__interpchannels.py. + # XXX Those should be either moved to LowLevelTests or eliminated + # in favor of high-level tests in this file. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(channels) + + +class TestChannels(TestBase): + + def test_create(self): + r, s = channels.create() + self.assertIsInstance(r, channels.RecvChannel) + self.assertIsInstance(s, channels.SendChannel) + + def test_list_all(self): + self.assertEqual(channels.list_all(), []) + created = set() + for _ in range(3): + ch = channels.create() + created.add(ch) + after = set(channels.list_all()) + self.assertEqual(after, created) + + def test_shareable(self): + interp = interpreters.create() + rch, sch = channels.create() + + self.assertTrue( + interpreters.is_shareable(rch)) + self.assertTrue( + interpreters.is_shareable(sch)) + + sch.send_nowait(rch) + sch.send_nowait(sch) + rch2 = rch.recv() + sch2 = rch.recv() + + interp.prepare_main(rch=rch, sch=sch) + sch.send_nowait(rch) + sch.send_nowait(sch) + interp.exec(dedent(""" + rch2 = rch.recv() + sch2 = rch.recv() + assert rch2 == rch + assert sch2 == sch + + sch.send_nowait(rch2) + sch.send_nowait(sch2) + """)) + rch3 = rch.recv() + sch3 = rch.recv() + + self.assertEqual(rch2, rch) + self.assertEqual(sch2, sch) + self.assertEqual(rch3, rch) + self.assertEqual(sch3, sch) + + def test_is_closed(self): + rch, sch = channels.create() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + + +class TestRecvChannelAttrs(TestBase): + + def test_id_type(self): + rch, _ = channels.create() + self.assertIsInstance(rch.id, _channels.ChannelID) + + def test_custom_id(self): + rch = channels.RecvChannel(1) + self.assertEqual(rch.id, 1) + + with self.assertRaises(TypeError): + channels.RecvChannel('1') + + def test_id_readonly(self): + rch = channels.RecvChannel(1) + with self.assertRaises(AttributeError): + rch.id = 2 + + def test_equality(self): + ch1, _ = channels.create() + ch2, _ = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + def test_pickle(self): + ch, _ = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + + +class TestSendChannelAttrs(TestBase): + + def test_id_type(self): + _, sch = channels.create() + self.assertIsInstance(sch.id, _channels.ChannelID) + + def test_custom_id(self): + sch = channels.SendChannel(1) + self.assertEqual(sch.id, 1) + + with self.assertRaises(TypeError): + channels.SendChannel('1') + + def test_id_readonly(self): + sch = channels.SendChannel(1) + with self.assertRaises(AttributeError): + sch.id = 2 + + def test_equality(self): + _, ch1 = channels.create() + _, ch2 = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + def test_pickle(self): + _, ch = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + + +class TestSendRecv(TestBase): + + def test_send_recv_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_send_recv_different_threads(self): + r, s = channels.create() + + def f(): + while True: + try: + obj = r.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + s.send(orig) + obj = r.recv() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main_with_default(self): + r, _ = channels.create() + obj = r.recv_nowait(None) + + self.assertIsNone(obj) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + assert obj == orig, 'expected: obj == orig' + # When going back to the same interpreter we get the same object. + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_nowait_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv_nowait() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv_nowait() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_recv_timeout(self): + r, _ = channels.create() + with self.assertRaises(TimeoutError): + r.recv(timeout=1) + + def test_recv_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv() + + def test_send_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send(b'spam') + + def test_recv_nowait_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv_nowait() + + def test_send_nowait_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send_nowait(b'spam') + + def test_recv_nowait_empty(self): + ch, _ = channels.create() + with self.assertRaises(channels.ChannelEmptyError): + ch.recv_nowait() + + def test_recv_nowait_default(self): + default = object() + rch, sch = channels.create() + obj1 = rch.recv_nowait(default) + sch.send_nowait(None) + sch.send_nowait(1) + sch.send_nowait(b'spam') + sch.send_nowait(b'eggs') + obj2 = rch.recv_nowait(default) + obj3 = rch.recv_nowait(default) + obj4 = rch.recv_nowait() + obj5 = rch.recv_nowait(default) + obj6 = rch.recv_nowait(default) + + self.assertIs(obj1, default) + self.assertIs(obj2, None) + self.assertEqual(obj3, 1) + self.assertEqual(obj4, b'spam') + self.assertEqual(obj5, b'eggs') + self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = channels.create() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = channels.create() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py new file mode 100644 index 00000000000000..ac24f6568acd95 --- /dev/null +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -0,0 +1,190 @@ +import contextlib +import json +import os +import os.path +import sys +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_interpreters') +from .utils import TestBase + + +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.exec(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + +class FinalizationTests(TestBase): + + @support.requires_subprocess() + def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. + import subprocess + argv = [sys.executable, '-c', '''if True: + from test.support import interpreters + interp = interpreters.create() + raise Exception + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py new file mode 100644 index 00000000000000..9ee7647ed79b4e --- /dev/null +++ b/Lib/test/test_interpreters/test_queues.py @@ -0,0 +1,477 @@ +import importlib +import pickle +import threading +from textwrap import dedent +import unittest + +from test.support import import_helper, Py_DEBUG +# Raise SkipTest if subinterpreters not supported. +_queues = import_helper.import_module('_interpqueues') +from test.support import interpreters +from test.support.interpreters import queues +from .utils import _run_output, TestBase as _TestBase + + +def get_num_queues(): + return len(_queues.list_all()) + + +class TestBase(_TestBase): + def tearDown(self): + for qid, _ in _queues.list_all(): + try: + _queues.destroy(qid) + except Exception: + pass + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(queues) + + def test_create_destroy(self): + qid = _queues.create(2, 0) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + with self.assertRaises(queues.QueueNotFoundError): + _queues.get(qid) + with self.assertRaises(queues.QueueNotFoundError): + _queues.destroy(qid) + + def test_not_destroyed(self): + # It should have cleaned up any remaining queues. + stdout, stderr = self.assert_python_ok( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, 0) + """), + ) + self.assertEqual(stdout, '') + if Py_DEBUG: + self.assertNotEqual(stderr, '') + else: + self.assertEqual(stderr, '') + + def test_bind_release(self): + with self.subTest('typical'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('bind too much'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('nested'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('release without binding'): + qid = _queues.create(2, 0) + with self.assertRaises(queues.QueueError): + _queues.release(qid) + + +class QueueTests(TestBase): + + def test_create(self): + with self.subTest('vanilla'): + queue = queues.create() + self.assertEqual(queue.maxsize, 0) + + with self.subTest('small maxsize'): + queue = queues.create(3) + self.assertEqual(queue.maxsize, 3) + + with self.subTest('big maxsize'): + queue = queues.create(100) + self.assertEqual(queue.maxsize, 100) + + with self.subTest('no maxsize'): + queue = queues.create(0) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('negative maxsize'): + queue = queues.create(-10) + self.assertEqual(queue.maxsize, -10) + + with self.subTest('bad maxsize'): + with self.assertRaises(TypeError): + queues.create('1') + + def test_shareable(self): + queue1 = queues.create() + + interp = interpreters.create() + interp.exec(dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + """)); + + with self.subTest('same interpreter'): + queue2 = queues.create() + queue1.put(queue2, syncobj=True) + queue3 = queue1.get() + self.assertIs(queue3, queue2) + + with self.subTest('from current interpreter'): + queue4 = queues.create() + queue1.put(queue4, syncobj=True) + out = _run_output(interp, dedent(""" + queue4 = queue1.get() + print(queue4.id) + """)) + qid = int(out) + self.assertEqual(qid, queue4.id) + + with self.subTest('from subinterpreter'): + out = _run_output(interp, dedent(""" + queue5 = queues.create() + queue1.put(queue5, syncobj=True) + print(queue5.id) + """)) + qid = int(out) + queue5 = queue1.get() + self.assertEqual(queue5.id, qid) + + def test_id_type(self): + queue = queues.create() + self.assertIsInstance(queue.id, int) + + def test_custom_id(self): + with self.assertRaises(queues.QueueNotFoundError): + queues.Queue(1_000_000) + + def test_id_readonly(self): + queue = queues.create() + with self.assertRaises(AttributeError): + queue.id = 1_000_000 + + def test_maxsize_readonly(self): + queue = queues.create(10) + with self.assertRaises(AttributeError): + queue.maxsize = 1_000_000 + + def test_hashable(self): + queue = queues.create() + expected = hash(queue.id) + actual = hash(queue) + self.assertEqual(actual, expected) + + def test_equality(self): + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(queue1, queue1) + self.assertNotEqual(queue1, queue2) + + def test_pickle(self): + queue = queues.create() + data = pickle.dumps(queue) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) + + +class TestQueueOps(TestBase): + + def test_empty(self): + queue = queues.create() + before = queue.empty() + queue.put(None, syncobj=True) + during = queue.empty() + queue.get() + after = queue.empty() + + self.assertIs(before, True) + self.assertIs(during, False) + self.assertIs(after, True) + + def test_full(self): + expected = [False, False, False, True, False, False, False] + actual = [] + queue = queues.create(3) + for _ in range(3): + actual.append(queue.full()) + queue.put(None, syncobj=True) + actual.append(queue.full()) + for _ in range(3): + queue.get() + actual.append(queue.full()) + + self.assertEqual(actual, expected) + + def test_qsize(self): + expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] + actual = [] + queue = queues.create() + for _ in range(3): + actual.append(queue.qsize()) + queue.put(None, syncobj=True) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + queue.put(None, syncobj=True) + actual.append(queue.qsize()) + for _ in range(3): + queue.get() + actual.append(queue.qsize()) + queue.put(None, syncobj=True) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + + self.assertEqual(actual, expected) + + def test_put_get_main(self): + expected = list(range(20)) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create() + for i in range(20): + queue.put(i, **kwds) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + def test_put_timeout(self): + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put(None, **kwds) + queue.put(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1, **kwds) + queue.get() + queue.put(None, **kwds) + + def test_put_nowait(self): + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put_nowait(None, **kwds) + queue.put_nowait(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None, **kwds) + queue.get() + queue.put_nowait(None, **kwds) + + def test_put_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=True) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=True) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) + + for obj in [ + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj, syncobj=True) + + def test_put_not_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=False) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=False) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) + + def test_get_timeout(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0.1) + + def test_get_nowait(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_get_default_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=True) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj) + + def test_put_get_default_not_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=False) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + + def test_put_get_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from test.support.interpreters import queues + queue = queues.create() + """)) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + interp.exec(dedent(f""" + orig = b'spam' + queue.put(orig, syncobj=True) + obj = queue.{methname}() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + def test_put_get_different_interpreters(self): + interp = interpreters.create() + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(len(queues.list_all()), 2) + + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + obj1 = b'spam' + queue1.put(obj1, syncobj=True) + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' + obj = queue1.{methname}() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' + queue2.put(obj2, syncobj=True) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + """)) + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + + get = getattr(queue2, methname) + obj2 = get() + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_put_cleared_with_subinterpreter(self): + interp = interpreters.create() + queue = queues.create() + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = b'spam' + obj2 = b'eggs' + queue.put(obj1, syncobj=True) + queue.put(obj2, syncobj=True) + """)) + self.assertEqual(queue.qsize(), 2) + + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), 1) + + del interp + self.assertEqual(queue.qsize(), 0) + + def test_put_get_different_threads(self): + queue1 = queues.create() + queue2 = queues.create() + + def f(): + while True: + try: + obj = queue1.get(timeout=0.1) + break + except queues.QueueEmpty: + continue + queue2.put(obj, syncobj=True) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + queue1.put(orig, syncobj=True) + obj = queue2.get() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py new file mode 100644 index 00000000000000..e400535b2a0e4e --- /dev/null +++ b/Lib/test/test_interpreters/test_stress.py @@ -0,0 +1,38 @@ +import threading +import unittest + +from test import support +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_interpreters') +from test.support import interpreters +from .utils import TestBase + + +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + @support.requires_resource('cpu') + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + + @support.requires_resource('cpu') + def test_create_many_threaded(self): + alive = [] + def task(): + interp = interpreters.create() + alive.append(interp) + threads = (threading.Thread(target=task) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py new file mode 100644 index 00000000000000..3cab76d0f279e0 --- /dev/null +++ b/Lib/test/test_interpreters/utils.py @@ -0,0 +1,686 @@ +from collections import namedtuple +import contextlib +import json +import os +import os.path +#import select +import subprocess +import sys +import tempfile +from textwrap import dedent +import threading +import types +import unittest +import warnings + +from test import support + +# We would use test.support.import_helper.import_module(), +# but the indirect import of test.support.os_helper causes refleaks. +try: + import _interpreters +except ImportError as exc: + raise unittest.SkipTest(str(exc)) +from test.support import interpreters + + +try: + import _testinternalcapi + import _testcapi +except ImportError: + _testinternalcapi = None + _testcapi = None + +def requires_test_modules(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + +def _dump_script(text): + lines = text.splitlines() + print() + print('-' * 20) + for i, line in enumerate(lines, 1): + print(f' {i:>{len(str(len(lines)))}} {line}') + print('-' * 20) + + +def _close_file(file): + try: + if hasattr(file, 'close'): + file.close() + else: + os.close(file) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + + +def pack_exception(exc=None): + captured = _interpreters.capture_exception(exc) + data = dict(captured.__dict__) + data['type'] = dict(captured.type.__dict__) + return json.dumps(data) + + +def unpack_exception(packed): + try: + data = json.loads(packed) + except json.decoder.JSONDecodeError: + warnings.warn('incomplete exception data', RuntimeWarning) + print(packed if isinstance(packed, str) else packed.decode('utf-8')) + return None + exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return exc; + + +class CapturingResults: + + STDIO = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: + _captured_std{stream} = io.StringIO() + with contextlib.redirect_std{stream}(_captured_std{stream}): + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + text = _captured_std{stream}.getvalue() + _spipe_{stream}.write(text.encode('utf-8')) + """)[:-1] + EXC = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: + try: + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + except Exception as exc: + text = _interp_utils.pack_exception(exc) + _spipe_exc.write(text.encode('utf-8')) + """)[:-1] + + @classmethod + def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): + script = dedent(script).strip(os.linesep) + imports = [ + f'import {__name__} as _interp_utils', + ] + wrapped = script + + # Handle exc. + if exc: + exc = os.pipe() + r_exc, w_exc = exc + indented = wrapped.replace('\n', '\n ') + wrapped = cls.EXC.format( + w_pipe=w_exc, + indented=indented, + ) + else: + exc = None + + # Handle stdout. + if stdout: + imports.extend([ + 'import contextlib, io', + ]) + stdout = os.pipe() + r_out, w_out = stdout + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_out, + indented=indented, + stream='out', + ) + else: + stdout = None + + # Handle stderr. + if stderr == 'stdout': + stderr = None + elif stderr: + if not stdout: + imports.extend([ + 'import contextlib, io', + ]) + stderr = os.pipe() + r_err, w_err = stderr + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_err, + indented=indented, + stream='err', + ) + else: + stderr = None + + if wrapped == script: + raise NotImplementedError + else: + for line in imports: + wrapped = f'{line}{os.linesep}{wrapped}' + + results = cls(stdout, stderr, exc) + return wrapped, results + + def __init__(self, out, err, exc): + self._rf_out = None + self._rf_err = None + self._rf_exc = None + self._w_out = None + self._w_err = None + self._w_exc = None + + if out is not None: + r_out, w_out = out + self._rf_out = open(r_out, 'rb', buffering=0) + self._w_out = w_out + + if err is not None: + r_err, w_err = err + self._rf_err = open(r_err, 'rb', buffering=0) + self._w_err = w_err + + if exc is not None: + r_exc, w_exc = exc + self._rf_exc = open(r_exc, 'rb', buffering=0) + self._w_exc = w_exc + + self._buf_out = b'' + self._buf_err = b'' + self._buf_exc = b'' + self._exc = None + + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def closed(self): + return self._closed + + def close(self): + if self._closed: + return + self._closed = True + + if self._w_out is not None: + _close_file(self._w_out) + self._w_out = None + if self._w_err is not None: + _close_file(self._w_err) + self._w_err = None + if self._w_exc is not None: + _close_file(self._w_exc) + self._w_exc = None + + self._capture() + + if self._rf_out is not None: + _close_file(self._rf_out) + self._rf_out = None + if self._rf_err is not None: + _close_file(self._rf_err) + self._rf_err = None + if self._rf_exc is not None: + _close_file(self._rf_exc) + self._rf_exc = None + + def _capture(self): + # Ideally this is called only after the script finishes + # (and thus has closed the write end of the pipe. + if self._rf_out is not None: + chunk = self._rf_out.read(100) + while chunk: + self._buf_out += chunk + chunk = self._rf_out.read(100) + if self._rf_err is not None: + chunk = self._rf_err.read(100) + while chunk: + self._buf_err += chunk + chunk = self._rf_err.read(100) + if self._rf_exc is not None: + chunk = self._rf_exc.read(100) + while chunk: + self._buf_exc += chunk + chunk = self._rf_exc.read(100) + + def _unpack_stdout(self): + return self._buf_out.decode('utf-8') + + def _unpack_stderr(self): + return self._buf_err.decode('utf-8') + + def _unpack_exc(self): + if self._exc is not None: + return self._exc + if not self._buf_exc: + return None + self._exc = unpack_exception(self._buf_exc) + return self._exc + + def stdout(self): + if self.closed: + return self.final().stdout + self._capture() + return self._unpack_stdout() + + def stderr(self): + if self.closed: + return self.final().stderr + self._capture() + return self._unpack_stderr() + + def exc(self): + if self.closed: + return self.final().exc + self._capture() + return self._unpack_exc() + + def final(self, *, force=False): + try: + return self._final + except AttributeError: + if not self._closed: + if not force: + raise Exception('no final results available yet') + else: + return CapturedResults.Proxy(self) + self._final = CapturedResults( + self._unpack_stdout(), + self._unpack_stderr(), + self._unpack_exc(), + ) + return self._final + + +class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')): + + class Proxy: + def __init__(self, capturing): + self._capturing = capturing + def _finish(self): + if self._capturing is None: + return + self._final = self._capturing.final() + self._capturing = None + def __iter__(self): + self._finish() + yield from self._final + def __len__(self): + self._finish() + return len(self._final) + def __getattr__(self, name): + self._finish() + if name.startswith('_'): + raise AttributeError(name) + return getattr(self._final, name) + + def raise_if_failed(self): + if self.exc is not None: + raise interpreters.ExecutionFailed(self.exc) + + +def _captured_script(script, *, stdout=True, stderr=False, exc=False): + return CapturingResults.wrap_script( + script, + stdout=stdout, + stderr=stderr, + exc=exc, + ) + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue + try: + interp.close() + except _interpreters.InterpreterError: + pass # already destroyed + + +def _run_output(interp, request, init=None): + script, results = _captured_script(request) + with results: + if init: + interp.prepare_main(init) + interp.exec(script) + return results.stdout() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +class TestBase(unittest.TestCase): + + def tearDown(self): + clean_up_interpreters() + + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + + def temp_dir(self): + tempdir = tempfile.mkdtemp() + tempdir = os.path.realpath(tempdir) + from test.support import os_helper + self.addCleanup(lambda: os_helper.rmtree(tempdir)) + return tempdir + + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + + def make_script(self, filename, dirname=None, text=None): + if text: + text = dedent(text) + if dirname is None: + dirname = self.temp_dir() + filename = os.path.join(dirname, filename) + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + def make_module(self, name, pathentry=None, text=None): + if text: + text = dedent(text) + if pathentry is None: + pathentry = self.temp_dir() + else: + os.makedirs(pathentry, exist_ok=True) + *subnames, basename = name.split('.') + + dirname = pathentry + for subname in subnames: + dirname = os.path.join(dirname, subname) + if os.path.isdir(dirname): + pass + elif os.path.exists(dirname): + raise Exception(dirname) + else: + os.mkdir(dirname) + initfile = os.path.join(dirname, '__init__.py') + if not os.path.exists(initfile): + with open(initfile, 'w'): + pass + filename = os.path.join(dirname, basename + '.py') + + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + @support.requires_subprocess() + def run_python(self, *argv): + proc = subprocess.run( + [sys.executable, *argv], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout, proc.stderr + + def assert_python_ok(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 1) + return stdout, stderr + + def assert_python_failure(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 0) + return stdout, stderr + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def _run_string(self, interp, script): + wrapped, results = _captured_script(script, exc=False) + #_dump_script(wrapped) + with results: + if isinstance(interp, interpreters.Interpreter): + interp.exec(script) + else: + err = _interpreters.run_string(interp, wrapped) + if err is not None: + return None, err + return results.stdout(), None + + def run_and_capture(self, interp, script): + text, err = self._run_string(interp, script) + if err is not None: + raise interpreters.ExecutionFailed(err) + else: + return text + + def interp_exists(self, interpid): + try: + _interpreters.whence(interpid) + except _interpreters.InterpreterNotFoundError: + return False + else: + return True + + @requires_test_modules + @contextlib.contextmanager + def interpreter_from_capi(self, config=None, whence=None): + if config is False: + if whence is None: + whence = _interpreters.WHENCE_LEGACY_CAPI + else: + assert whence in (_interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN), repr(whence) + config = None + elif config is True: + config = _interpreters.new_config('default') + elif config is None: + if whence not in ( + _interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN, + ): + config = _interpreters.new_config('legacy') + elif isinstance(config, str): + config = _interpreters.new_config(config) + + if whence is None: + whence = _interpreters.WHENCE_XI + + interpid = _testinternalcapi.create_interpreter(config, whence=whence) + try: + yield interpid + finally: + try: + _testinternalcapi.destroy_interpreter(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + @contextlib.contextmanager + def interpreter_obj_from_capi(self, config='legacy'): + with self.interpreter_from_capi(config) as interpid: + interp = interpreters.Interpreter( + interpid, + _whence=_interpreters.WHENCE_CAPI, + _ownsref=False, + ) + yield interp, interpid + + @contextlib.contextmanager + def capturing(self, script): + wrapped, capturing = _captured_script(script, stdout=True, exc=True) + #_dump_script(wrapped) + with capturing: + yield wrapped, capturing.final(force=True) + + @requires_test_modules + def run_from_capi(self, interpid, script, *, main=False): + with self.capturing(script) as (wrapped, results): + rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout + + @contextlib.contextmanager + def _running(self, run_interp, exec_interp): + token = b'\0' + r_in, w_in = self.pipe() + r_out, w_out = self.pipe() + + def close(): + _close_file(r_in) + _close_file(w_in) + _close_file(r_out) + _close_file(w_out) + + # Start running (and wait). + script = dedent(f""" + import os + try: + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + except BrokenPipeError: + pass + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + """) + failed = None + def run(): + nonlocal failed + try: + run_interp(script) + except Exception as exc: + failed = exc + close() + t = threading.Thread(target=run) + t.start() + + # handshake + try: + os.write(w_in, token) + token2 = os.read(r_out, 1) + assert token2 == token, (token2, token) + except OSError: + t.join() + if failed is not None: + raise failed + + # CM __exit__() + try: + try: + yield + finally: + # Send "done". + os.write(w_in, b'\0') + finally: + close() + t.join() + if failed is not None: + raise failed + + @contextlib.contextmanager + def running(self, interp): + if isinstance(interp, int): + interpid = interp + def exec_interp(script): + exc = _interpreters.exec(interpid, script) + assert exc is None, exc + run_interp = exec_interp + else: + def run_interp(script): + text = self.run_and_capture(interp, script) + assert text == '', repr(text) + def exec_interp(script): + interp.exec(script) + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + @contextlib.contextmanager + def running_from_capi(self, interpid, *, main=False): + def run_interp(script): + text = self.run_from_capi(interpid, script, main=main) + assert text == '', repr(text) + def exec_interp(script): + rc = _testinternalcapi.exec_interpreter(interpid, script) + assert rc == 0, rc + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + def run_temp_from_capi(self, script, config='legacy'): + if config is False: + # Force using Py_NewInterpreter(). + run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s)) + config = None + else: + run_in_interp = _testinternalcapi.run_in_subinterp_with_config + if config is True: + config = 'default' + if isinstance(config, str): + config = _interpreters.new_config(config) + with self.capturing(script) as (wrapped, results): + rc = run_in_interp(wrapped, config) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 022cf21a4709a2..1ca3edac8c8dc9 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) from test.support.os_helper import FakePath import codecs @@ -77,6 +75,10 @@ def _default_chunk_size(): ) +class BadIndex: + def __index__(self): + 1/0 + class MockRawIOWithoutRead: """A RawIO implementation without read(), so as to exercise the default RawIO.read() which calls readinto().""" @@ -255,6 +257,27 @@ class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): UnsupportedOperation = pyio.UnsupportedOperation +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + class MockNonBlockWriterIO: def __init__(self): @@ -602,10 +625,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -1137,7 +1160,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1230,11 +1253,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() b = self.tp(raw) - with support.swap_attr(raw, 'name', b): - try: + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(b) # Should not crash - except RuntimeError: - pass def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush @@ -1648,22 +1669,34 @@ def test_truncate_on_read_only(self): self.assertRaises(self.UnsupportedOperation, bufio.truncate) self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -2019,18 +2052,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2495,6 +2516,28 @@ def test_interleaved_read_write(self): f.flush() self.assertEqual(raw.getvalue(), b'a2c') + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + def test_interleaved_readline_write(self): with self.BytesIO(b'ab\ncdef\ng\n') as raw: with self.tp(raw) as f: @@ -2518,18 +2561,6 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2709,8 +2740,29 @@ def test_constructor(self): self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) - self.assertRaises(TypeError, t.__init__, b, encoding="utf-8", newline=42) - self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') + invalid_type = TypeError if self.is_C else ValueError + with self.assertRaises(invalid_type): + t.__init__(b, encoding=42) + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding='utf-8\0') + with self.assertRaises(invalid_type): + t.__init__(b, encoding="utf-8", errors=42) + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding="utf-8", errors='\udcfe') + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", errors='replace\0') + with self.assertRaises(TypeError): + t.__init__(b, encoding="utf-8", newline=42) + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\n\0') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='xyzzy') def test_uninitialized(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) @@ -2776,11 +2828,16 @@ def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t): - try: + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(t) # Should not crash - except RuntimeError: - pass + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) def test_line_buffering(self): r = self.BytesIO() @@ -3621,10 +3678,8 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) @@ -3756,6 +3811,59 @@ def test_reconfigure_defaults(self): self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') + def test_reconfigure_errors(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') + with self.assertRaises(TypeError): # there was a crash + txt.reconfigure(encoding=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(encoding='\udcfe') + with self.assertRaises(LookupError): + txt.reconfigure(encoding='locale\0') + # TODO: txt.reconfigure(encoding='utf-8\0') + # TODO: txt.reconfigure(encoding='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(errors=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(errors='\udcfe') + # TODO: txt.reconfigure(errors='ignore\0') + # TODO: txt.reconfigure(errors='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(newline=42) + with self.assertRaises(ValueError): + txt.reconfigure(newline='\udcfe') + with self.assertRaises(ValueError): + txt.reconfigure(newline='xyz') + if not self.is_C: + # TODO: Should fail in C too. + with self.assertRaises(ValueError): + txt.reconfigure(newline='\n\0') + if self.is_C: + # TODO: Use __bool__(), not __index__(). + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(line_buffering=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(line_buffering=2**1000) + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(write_through=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(write_through=2**1000) + with self.assertRaises(ZeroDivisionError): # there was a crash + txt.reconfigure(line_buffering=BadIndex(), + write_through=BadIndex()) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + self.assertIs(txt.line_buffering, False) + self.assertIs(txt.write_through, False) + + txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', + line_buffering=True, write_through=True) + self.assertEqual(txt.encoding, 'latin1') + self.assertEqual(txt.errors, 'ignore') + self.assertIs(txt.line_buffering, True) + self.assertIs(txt.write_through, True) + def test_reconfigure_newline(self): raw = self.BytesIO(b'CR\rEOF') txt = self.TextIOWrapper(raw, 'ascii', newline='\n') @@ -3803,6 +3911,14 @@ def test_issue25862(self): t.write('x') t.tell() + def test_issue35928(self): + p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) + f = self.TextIOWrapper(p) + res = f.readline() + self.assertEqual(res, 'foo\n') + f.write(res) + self.assertEqual(res + f.readline(), 'foo\nbar\n') + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -3900,6 +4016,28 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio @@ -4042,19 +4180,18 @@ class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): class MiscIOTest(unittest.TestCase): + # for test__all__, actual values are set in subclasses + name_of_module = None + extra_exported = () + not_exported = () + def tearDown(self): os_helper.unlink(os_helper.TESTFN) def test___all__(self): - for name in self.io.__all__: - obj = getattr(self.io, name, None) - self.assertIsNotNone(obj, name) - if name in ("open", "open_code"): - continue - elif "error" in name.lower() or name == "UnsupportedOperation": - self.assertTrue(issubclass(obj, Exception), name) - elif not name.startswith("SEEK_"): - self.assertTrue(issubclass(obj, self.IOBase)) + support.check__all__(self, self.io, self.name_of_module, + extra=self.extra_exported, + not_exported=self.not_exported) def test_attributes(self): f = self.open(os_helper.TESTFN, "wb", buffering=0) @@ -4396,11 +4533,11 @@ def test_check_encoding_warning(self): ''') proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2) + self.assertEqual(len(warnings), 4) self.assertTrue( warnings[0].startswith(b":5: EncodingWarning: ")) self.assertTrue( - warnings[1].startswith(b":8: EncodingWarning: ")) + warnings[2].startswith(b":8: EncodingWarning: ")) def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" @@ -4416,6 +4553,8 @@ def test_text_encoding(self): class CMiscIOTest(MiscIOTest): io = io + name_of_module = "io", "_io" + extra_exported = "BlockingIOError", def test_readinto_buffer_overflow(self): # Issue #18025 @@ -4480,6 +4619,9 @@ def test_daemon_threads_shutdown_stderr_deadlock(self): class PyMiscIOTest(MiscIOTest): io = pyio + name_of_module = "_pyio", "io" + extra_exported = "BlockingIOError", "open_code", + not_exported = "valid_seek_flags", @unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') @@ -4766,8 +4908,8 @@ def load_tests(loader, tests, pattern): # classes in the __dict__ of each test. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO) - all_members = io.__all__ + ["IncrementalNewlineDecoder"] + SlowFlushRawIO, MockCharPseudoDevFileIO) + all_members = io.__all__ c_io_ns = {name : getattr(io, name) for name in all_members} py_io_ns = {name : getattr(pyio, name) for name in all_members} globs = globals() @@ -4777,9 +4919,11 @@ def load_tests(loader, tests, pattern): if test.__name__.startswith("C"): for name, obj in c_io_ns.items(): setattr(test, name, obj) + test.is_C = True elif test.__name__.startswith("Py"): for name, obj in py_io_ns.items(): setattr(test, name, obj) + test.is_C = False suite = loader.suiteClass() for test in tests: diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7b7067eb7b61d4..04934dfa16a5f0 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -66,23 +66,15 @@ def test_ioctl_mutate_2048(self): # Test with a larger buffer, just for the record. self._check_ioctl_mutate_len(2048) - def test_ioctl_signed_unsigned_code_param(self): - if not pty: - raise unittest.SkipTest('pty module required') + @unittest.skipIf(pty is None, 'pty module required') + def test_ioctl_set_window_size(self): mfd, sfd = pty.openpty() try: - if termios.TIOCSWINSZ < 0: - set_winsz_opcode_maybe_neg = termios.TIOCSWINSZ - set_winsz_opcode_pos = termios.TIOCSWINSZ & 0xffffffff - else: - set_winsz_opcode_pos = termios.TIOCSWINSZ - set_winsz_opcode_maybe_neg, = struct.unpack("i", - struct.pack("I", termios.TIOCSWINSZ)) - - our_winsz = struct.pack("HHHH",80,25,0,0) - # test both with a positive and potentially negative ioctl code - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_pos, our_winsz) - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_maybe_neg, our_winsz) + # (rows, columns, xpixel, ypixel) + our_winsz = struct.pack("HHHH", 20, 40, 0, 0) + result = fcntl.ioctl(mfd, termios.TIOCSWINSZ, our_winsz) + new_winsz = struct.unpack("HHHH", result) + self.assertEqual(new_winsz[:2], (20, 40)) finally: os.close(mfd) os.close(sfd) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 6f204948c9fc48..c3ecf2a742941a 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -4,6 +4,7 @@ """Unittest for ipaddress module.""" +import copy import unittest import re import contextlib @@ -302,6 +303,14 @@ def test_pickle(self): def test_weakref(self): weakref.ref(self.factory('192.0.2.1')) + def test_ipv6_mapped(self): + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped, + ipaddress.IPv6Address('::ffff:192.168.1.1')) + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped, + ipaddress.IPv6Address('::ffff:c0a8:101')) + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped.ipv4_mapped, + ipaddress.IPv4Address('192.168.1.1')) + class AddressTestCase_v6(BaseTestCase, CommonTestMixin_v6): factory = ipaddress.IPv6Address @@ -542,11 +551,17 @@ def assertBadPart(addr, part): def test_pickle(self): self.pickle_test('2001:db8::') + self.pickle_test('2001:db8::%scope') def test_weakref(self): weakref.ref(self.factory('2001:db8::')) weakref.ref(self.factory('2001:db8::%scope')) + def test_copy(self): + addr = self.factory('2001:db8::%scope') + self.assertEqual(addr, copy.copy(addr)) + self.assertEqual(addr, copy.deepcopy(addr)) + class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" @@ -2273,6 +2288,10 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_address( '172.31.255.255').is_private) self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) + self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) + self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) self.assertEqual(True, ipaddress.ip_address('169.254.100.200').is_link_local) @@ -2298,6 +2317,7 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) @@ -2314,8 +2334,8 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("::/128").is_private) self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) - self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) @@ -2394,6 +2414,20 @@ def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) + self.assertFalse(ipaddress.ip_address('2001::').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) + self.assertFalse(ipaddress.ip_address('2001:2::').is_global) + self.assertTrue(ipaddress.ip_address('2001:3::').is_global) + self.assertFalse(ipaddress.ip_address('2001:4::').is_global) + self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) + self.assertFalse(ipaddress.ip_address('2001:10::').is_global) + self.assertTrue(ipaddress.ip_address('2001:20::').is_global) + self.assertTrue(ipaddress.ip_address('2001:30::').is_global) + self.assertFalse(ipaddress.ip_address('2001:40::').is_global) + self.assertFalse(ipaddress.ip_address('2002::').is_global) + # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) @@ -2412,6 +2446,22 @@ def testIpv4MappedPrivateCheck(self): self.assertEqual( False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testIpv4MappedLoopbackCheck(self): + # test networks + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.100.200.254/128').is_loopback) + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.42.0.0/112').is_loopback) + self.assertEqual(False, ipaddress.ip_network( + '::ffff:128.0.0.0').is_loopback) + # test addresses + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.100.200.254').is_loopback) + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.42.0.0').is_loopback) + self.assertEqual(False, ipaddress.ip_address( + '::ffff:128.0.0.0').is_loopback) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index bf9332e40aeaf2..95a119ba683e09 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -310,7 +310,7 @@ class X: @property def __bases__(self): return self.__bases__ - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) @@ -344,7 +344,7 @@ class B: pass A.__getattr__ = B.__getattr__ = X.__getattr__ return (A(), B()) - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) @@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 30aedb0db3bb3d..ec2b68acb90785 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -10,6 +10,7 @@ import functools import contextlib import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -302,7 +303,7 @@ def __eq__(self, other): # listiter_reduce_general self.assertEqual( run("reversed", orig["reversed"](list(range(8)))), - (iter, ([],)) + (reversed, ([],)) ) for case in types: @@ -1143,6 +1144,51 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + class Iter: + def __init__(self, init_raises=False, next_raises=False): + if init_raises: + 1/0 + self.next_raises = next_raises + + def __next__(self): + if self.next_raises: + 1/0 + + def __iter__(self): + return self + + def init_raises(): + try: + for x in Iter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in Iter(next_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "Iter(init_raises=True)"), + (next_raises, "Iter(next_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 512745e45350d1..5fd6ecf37427f7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,7 @@ import doctest import unittest from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -144,7 +144,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -171,9 +170,6 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) @@ -187,7 +183,11 @@ def test_batched(self): [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) + self.assertEqual(list(batched('ABCDEF', 2, strict=True)), + [('A', 'B'), ('C', 'D'), ('E', 'F')]) + with self.assertRaises(ValueError): # Incomplete batch when strict + list(batched('ABCDEFG', 3, strict=True)) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) with self.assertRaises(TypeError): @@ -241,58 +241,12 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @pickle_deprecated - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - @pickle_deprecated - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -346,9 +300,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -360,7 +311,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -368,15 +318,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -434,22 +375,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -510,9 +447,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -556,7 +490,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -573,24 +506,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -628,19 +543,10 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @pickle_deprecated - def test_count_with_stride(self): + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -683,17 +589,28 @@ def test_count_with_stride(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) + + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) @@ -702,113 +619,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - @pickle_deprecated - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - @pickle_deprecated - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - @pickle_deprecated - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -827,15 +637,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -846,18 +647,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -877,13 +666,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -966,7 +748,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -977,8 +758,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -997,33 +776,12 @@ def test_zip(self): lzip('abc', 'def')) @support.impl_detail("tuple reuse is specific to CPython") - @pickle_deprecated def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1073,14 +831,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @pickle_deprecated - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1152,6 +902,78 @@ def test_pairwise(self): with self.assertRaises(TypeError): pairwise(None) # non-iterable argument + def test_pairwise_reenter(self): + def check(reenter_at, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + self.count +=1 + if self.count in reenter_at: + return next(it) + return [self.count] # new object + + it = pairwise(I()) + for item in expected: + self.assertEqual(next(it), item) + + check({1}, [ + (([2], [3]), [4]), + ([4], [5]), + ]) + check({2}, [ + ([1], ([1], [3])), + (([1], [3]), [4]), + ([4], [5]), + ]) + check({3}, [ + ([1], [2]), + ([2], ([2], [4])), + (([2], [4]), [5]), + ([5], [6]), + ]) + check({1, 2}, [ + ((([3], [4]), [5]), [6]), + ([6], [7]), + ]) + check({1, 3}, [ + (([2], ([2], [4])), [5]), + ([5], [6]), + ]) + check({1, 4}, [ + (([2], [3]), (([2], [3]), [5])), + ((([2], [3]), [5]), [6]), + ([6], [7]), + ]) + check({2, 3}, [ + ([1], ([1], ([1], [4]))), + (([1], ([1], [4])), [5]), + ([5], [6]), + ]) + + def test_pairwise_reenter2(self): + def check(maxcount, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + if self.count >= maxcount: + raise StopIteration + self.count +=1 + if self.count == 1: + return next(it, None) + return [self.count] # new object + + it = pairwise(I()) + self.assertEqual(list(it), expected) + + check(1, []) + check(2, []) + check(3, []) + check(4, [(([2], [3]), [4])]) + def test_product(self): for args, result in [ ([], [()]), # zero iterables @@ -1220,34 +1042,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @pickle_deprecated - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - @pickle_deprecated - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - - @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1266,21 +1060,12 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @pickle_deprecated def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1298,20 +1083,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - - @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1326,20 +1097,6 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - @pickle_deprecated def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1396,21 +1153,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1434,7 +1176,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1448,14 +1189,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - @pickle_deprecated def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1466,14 +1199,6 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - @pickle_deprecated def test_tee(self): n = 200 @@ -1588,40 +1313,13 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): @@ -1737,6 +1435,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject @@ -1775,33 +1480,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @pickle_deprecated - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - @pickle_deprecated - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') @@ -1909,27 +1587,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -1949,6 +1769,161 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + iterator = iter(iterable) + shared_link = [None, None] + return tuple(_tee(iterator, shared_link) for _ in range(n)) + + def _tee(iterator, link): + try: + while True: + if link[1] is None: + link[0] = next(iterator) + link[1] = [None, None] + value, link = link + yield value + except StopIteration: + return + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # Tests not applicable to the tee() recipe + if False: + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # Tests not applicable to the tee() recipe + if False: + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + + # Tests not applicable to the tee() recipe + if False: + # tee pass-through to copyable iterator + a, b = tee('abc') + c, d = tee(a) + self.assertTrue(a is c) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 124045b13184b3..79fb239b35d3f2 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -8,14 +8,34 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = self.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) + self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + def test_bytes(self): + self.assertEqual(self.loads(b"1"), 1) + + def test_parse_constant(self): + for constant, expected in [ + ("Infinity", "INFINITY"), + ("-Infinity", "-INFINITY"), + ("NaN", "NAN"), + ]: + self.assertEqual( + self.loads(constant, parse_constant=str.upper), expected + ) + + def test_constant_invalid_case(self): + for constant in [ + "nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy" + ]: + with self.assertRaises(self.JSONDecodeError): + self.loads(constant) + def test_empty_objects(self): self.assertEqual(self.loads('{}'), {}) self.assertEqual(self.loads('[]'), []) @@ -88,7 +108,8 @@ def test_string_with_utf8_bom(self): self.json.load(StringIO(bom_json)) self.assertIn('BOM', str(cm.exception)) # make sure that the BOM is not detected in the middle of a string - bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + bom = ''.encode('utf-8-sig').decode('utf-8') + bom_in_str = f'"{bom}"' self.assertEqual(self.loads(bom_in_str), '\ufeff') self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 4bbc6c71489a83..6a39b72a09df35 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -23,8 +23,7 @@ def test_encode_basestring_ascii(self): for input_string, expect in CASES: result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, - '{0!r} != {1!r} for {2}({3!r})'.format( - result, expect, fname, input_string)) + f'{result!r} != {expect!r} for {fname}({input_string!r})') def test_ordered_dict(self): # See issue 6105 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index efc982e8b0eb04..a74240f1107de3 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -89,7 +89,7 @@ def test_failures(self): except self.JSONDecodeError: pass else: - self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + self.fail(f"Expected failure for fail{idx}.json: {doc!r}") def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} @@ -143,11 +143,11 @@ def test_unexpected_data(self): ('{"spam":[}', 'Expecting value', 9), ('[42:', "Expecting ',' delimiter", 3), ('[42 "spam"', "Expecting ',' delimiter", 4), - ('[42,]', 'Expecting value', 4), + ('[42,]', "Illegal trailing comma before end of array", 3), ('{"spam":[42}', "Expecting ',' delimiter", 11), ('["]', 'Unterminated string starting at', 1), ('["spam":', "Expecting ',' delimiter", 7), - ('["spam",]', 'Expecting value', 8), + ('["spam",]', "Illegal trailing comma before end of array", 7), ('{:', 'Expecting property name enclosed in double quotes', 1), ('{,', 'Expecting property name enclosed in double quotes', 1), ('{42', 'Expecting property name enclosed in double quotes', 1), @@ -159,7 +159,9 @@ def test_unexpected_data(self): ('[{"spam":]', 'Expecting value', 9), ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), ('[{"spam":42]', "Expecting ',' delimiter", 11), - ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ('{"spam":42,}', "Illegal trailing comma before end of object", 10), + ('{"spam":42 , }', "Illegal trailing comma before end of object", 11), + ('[123 , ]', "Illegal trailing comma before end of array", 6), ] for data, msg, idx in test_cases: with self.assertRaises(self.JSONDecodeError) as cm: diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54ef7..164ff2013eb552 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self): for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +99,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 2e8bba2775256a..68629cceeb9be9 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -20,12 +20,17 @@ def test_encoding4(self): def test_encoding5(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps(u, ensure_ascii=False) - self.assertEqual(j, '"{0}"'.format(u)) + self.assertEqual(j, f'"{u}"') def test_encoding6(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps([u], ensure_ascii=False) - self.assertEqual(j, '["{0}"]'.format(u)) + self.assertEqual(j, f'["{u}"]') + + def test_encoding7(self): + u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + j = self.dumps(u + "\n", ensure_ascii=False) + self.assertEqual(j, f'"{u}\\n"') def test_big_unicode_encode(self): u = '\U0001d120' @@ -34,13 +39,13 @@ def test_big_unicode_encode(self): def test_big_unicode_decode(self): u = 'z\U0001d120x' - self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads(f'"{u}"'), u) self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = chr(i) - s = '"\\u{0:04x}"'.format(i) + s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) def test_unicode_preservation(self): diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index df82f677a00a48..918f953cae5702 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -170,7 +170,7 @@ def f(v=a, x=b, *, y=c, z=d): pass self.assertEqual(str(err.exception), "name 'b' is not defined") with self.assertRaises(NameError) as err: - f = lambda v=a, x=b, *, y=c, z=d: None + g = lambda v=a, x=b, *, y=c, z=d: None self.assertEqual(str(err.exception), "name 'b' is not defined") diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 998fd9d46496bb..e94edcbc107ba9 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -5,6 +5,7 @@ import os import select import socket +from test import support import time import unittest @@ -256,6 +257,23 @@ def test_fd_non_inheritable(self): self.addCleanup(kqueue.close) self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + @support.requires_fork() + def test_fork(self): + # gh-110395: kqueue objects must be closed after fork + kqueue = select.kqueue() + if (pid := os.fork()) == 0: + try: + self.assertTrue(kqueue.closed) + with self.assertRaisesRegex(ValueError, "closed kqueue"): + kqueue.fileno() + except: + os._exit(1) + finally: + os._exit(0) + else: + support.wait_process(pid, exitcode=0) + self.assertFalse(kqueue.closed) # child done, we're still open. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3b0930fe69e30e..849b6cb3e50a19 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -2,7 +2,6 @@ """ import os -import stat import sys import unittest import socket @@ -29,7 +28,7 @@ def setUp(self): mode = 'w+b' with self.open(TESTFN, mode) as f: - current_size = os.fstat(f.fileno())[stat.ST_SIZE] + current_size = os.fstat(f.fileno()).st_size if current_size == size+1: return @@ -40,13 +39,13 @@ def setUp(self): f.seek(size) f.write(b'a') f.flush() - self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + self.assertEqual(os.fstat(f.fileno()).st_size, size+1) @classmethod def tearDownClass(cls): with cls.open(TESTFN, 'wb'): pass - if not os.stat(TESTFN)[stat.ST_SIZE] == 0: + if not os.stat(TESTFN).st_size == 0: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') unlink(TESTFN2) @@ -67,7 +66,7 @@ def test_large_read(self, _size): self.assertEqual(f.tell(), size + 1) def test_osstat(self): - self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) + self.assertEqual(os.stat(TESTFN).st_size, size+1) def test_seek_read(self): with self.open(TESTFN, 'rb') as f: diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index bcd4ed63bf25a0..58baae25df3df7 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -19,8 +19,10 @@ PY_EXE = "py.exe" +DEBUG_BUILD = False if sys.executable.casefold().endswith("_d.exe".casefold()): PY_EXE = "py_d.exe" + DEBUG_BUILD = True # Registry data to create. On removal, everything beneath top-level names will # be deleted. @@ -88,6 +90,12 @@ "test-command=TEST_EXE.exe", ]) + +def quote(s): + s = str(s) + return f'"{s}"' if " " in s else s + + def create_registry_data(root, data): def _create_registry_data(root, key, value): if isinstance(value, dict): @@ -232,7 +240,7 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non p.stdin.close() p.wait(10) out = p.stdout.read().decode("utf-8", "replace") - err = p.stderr.read().decode("ascii", "replace") + err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") if p.returncode != expect_returncode and support.verbose and not allow_fail: print("++ COMMAND ++") print([self.py_exe, *args]) @@ -273,7 +281,7 @@ def script(self, content, encoding="utf-8"): def fake_venv(self): venv = Path.cwd() / "Scripts" venv.mkdir(exist_ok=True, parents=True) - venv_exe = (venv / Path(sys.executable).name) + venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) venv_exe.touch() try: yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} @@ -521,6 +529,9 @@ def test_virtualenv_in_list(self): self.assertEqual(str(venv_exe), m.group(1)) break else: + if support.verbose: + print(data["stdout"]) + print(data["stderr"]) self.fail("did not find active venv path") data = self.run_py(["-0"], env=env) @@ -537,10 +548,10 @@ def test_virtualenv_with_env(self): data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) # Compare stdout, because stderr goes via ascii - self.assertEqual(data1["stdout"].strip(), str(venv_exe)) + self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") # Ensure passing the argument doesn't trigger the same behaviour - self.assertNotEqual(data2["stdout"].strip(), str(venv_exe)) + self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") def test_py_shebang(self): @@ -549,7 +560,7 @@ def test_py_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_python_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -557,7 +568,7 @@ def test_python_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) def test_py2_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -565,7 +576,8 @@ def test_py2_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -573,7 +585,8 @@ def test_py3_shebang(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -581,7 +594,8 @@ def test_py_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py2_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -589,7 +603,8 @@ def test_py2_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py3_shebang_nl(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -597,7 +612,8 @@ def test_py3_shebang_nl(self): data = self.run_py([script, "-postarg"]) self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_py_shebang_short_argv0(self): with self.py_ini(TEST_PY_DEFAULTS): @@ -616,25 +632,31 @@ def test_py_handle_64_in_ini(self): self.assertEqual("True", data["SearchInfo.oldStyleTag"]) def test_search_path(self): - stem = Path(sys.executable).stem + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {stem} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_search_path_exe(self): # Leave the .exe on the name to ensure we don't add it a second time - name = Path(sys.executable).name + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {name} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) def test_recursive_search_path(self): stem = self.get_py_exe().stem @@ -645,7 +667,7 @@ def test_recursive_search_path(self): env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, ) # The recursive search is ignored and we get normal "py" behavior - self.assertEqual(f"X.Y.exe {script}", data["stdout"].strip()) + self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) def test_install(self): data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) @@ -665,7 +687,7 @@ def test_literal_shebang_absolute(self): with self.script("#! C:/some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"C:\\some_random_app -witharg {script}", + f"C:\\some_random_app -witharg {quote(script)}", data["stdout"].strip(), ) @@ -673,7 +695,7 @@ def test_literal_shebang_relative(self): with self.script("#! ..\\some_random_app -witharg") as script: data = self.run_py([script]) self.assertEqual( - f"{script.parent.parent}\\some_random_app -witharg {script}", + f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -681,14 +703,14 @@ def test_literal_shebang_quoted(self): with self.script('#! "some random app" -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) with self.script('#! some" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some random app" -witharg {script}', + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -696,7 +718,7 @@ def test_literal_shebang_quoted_escape(self): with self.script('#! some\\" random "app -witharg') as script: data = self.run_py([script]) self.assertEqual( - f'"{script.parent}\\some\\ random app" -witharg {script}', + f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", data["stdout"].strip(), ) @@ -705,7 +727,7 @@ def test_literal_shebang_command(self): with self.script('#! test-command arg1') as script: data = self.run_py([script]) self.assertEqual( - f"TEST_EXE.exe arg1 {script}", + f"TEST_EXE.exe arg1 {quote(script)}", data["stdout"].strip(), ) @@ -714,7 +736,7 @@ def test_literal_shebang_invalid_template(self): data = self.run_py([script]) expect = script.parent / "/usr/bin/not-python" self.assertEqual( - f"{expect} arg1 {script}", + f"{quote(expect)} arg1 {quote(script)}", data["stdout"].strip(), ) @@ -727,15 +749,26 @@ def test_shebang_command_in_venv(self): data = self.run_py([script], expect_returncode=103) with self.fake_venv() as (venv_exe, env): - # Put a real Python (ourselves) on PATH as a distraction. + # Put a "normal" Python on PATH as a distraction. # The active VIRTUAL_ENV should be preferred when the name isn't an # exact match. - env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}" + exe = Path(Path(venv_exe).name).absolute() + exe.touch() + self.addCleanup(exe.unlink) + env["PATH"] = f"{exe.parent};{os.environ['PATH']}" with self.script(f'#! /usr/bin/env {stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") - with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script: + with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 72dd40136cfdb2..8ac521d72ef13e 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -5,6 +5,7 @@ import os.path import tempfile import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper @@ -82,6 +83,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] @@ -97,6 +102,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -238,6 +253,33 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 2969c6e2f98a23..ad7accf2099f43 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -96,6 +96,11 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -229,6 +234,31 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + + def test_list_index_modifing_operand(self): + # See gh-120384 + class evil: + def __init__(self, lst): + self.lst = lst + def __iter__(self): + yield from self.lst + self.lst.clear() + + lst = list(range(5)) + operand = evil(lst) + with self.assertRaises(ValueError): + lst[::-1] = operand + @cpython_only def test_preallocation(self): iterable = [0] * 10 @@ -269,6 +299,15 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 12f7bbd123b30c..58b076e9ea5d8a 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -1,5 +1,6 @@ import doctest import textwrap +import types import unittest @@ -92,7 +93,8 @@ class ListComprehensionTest(unittest.TestCase): - def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=()): + def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=(), + exec_func=exec): code = textwrap.dedent(code) scopes = scopes or ["module", "class", "function"] for scope in scopes: @@ -119,7 +121,7 @@ def get_output(moddict, name): return moddict[name] newns = ns.copy() if ns else {} try: - exec(newcode, newns) + exec_func(newcode, newns) except raises as e: # We care about e.g. NameError vs UnboundLocalError self.assertIs(type(e), raises) @@ -154,6 +156,43 @@ def method(self): self.assertEqual(C.y, [4, 4, 4, 4, 4]) self.assertIs(C().method(), C) + def test_references_super(self): + code = """ + res = [super for x in [1]] + """ + self._check_in_scopes(code, outputs={"res": [super]}) + + def test_references___class__(self): + code = """ + res = [__class__ for x in [1]] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] @@ -608,11 +647,69 @@ def test_exception_in_post_comp_call(self): def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + def _recursive_replace(self, maybe_code): + if not isinstance(maybe_code, types.CodeType): + return maybe_code + return maybe_code.replace(co_consts=tuple( + self._recursive_replace(c) for c in maybe_code.co_consts + )) + + def _replacing_exec(self, code_string, ns): + co = compile(code_string, "", "exec") + co = self._recursive_replace(co) + exec(co, ns) + + def test_code_replace(self): + code = """ + x = 3 + [x for x in (1, 2)] + dir() + y = [x] + """ + self._check_in_scopes(code, {"y": [3], "x": 3}) + self._check_in_scopes(code, {"y": [3], "x": 3}, exec_func=self._replacing_exec) + + def test_code_replace_extended_arg(self): + num_names = 300 + assignments = "; ".join(f"x{i} = {i}" for i in range(num_names)) + name_list = ", ".join(f"x{i}" for i in range(num_names)) + expected = { + "y": list(range(num_names)), + **{f"x{i}": i for i in range(num_names)} + } + code = f""" + {assignments} + [({name_list}) for {name_list} in (range(300),)] + dir() + y = [{name_list}] + """ + self._check_in_scopes(code, expected) + self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index ab969ce26a69e7..e6daea2333b206 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -43,11 +43,13 @@ import tempfile from test.support.script_helper import assert_python_ok, assert_python_failure from test import support +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +from test.support import smtpd from test.support.logging_helper import TestHandler import textwrap import threading @@ -58,13 +60,11 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler +from unittest.mock import patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) -with warnings.catch_warnings(): - from . import smtpd - try: import win32evtlog, win32evtlogutil, pywintypes except ImportError: @@ -81,6 +81,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -604,7 +607,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -654,21 +657,21 @@ def test_builtin_handlers(self): self.assertFalse(h.shouldFlush(r)) h.close() - def test_path_objects(self): + def test_pathlike_objects(self): """ - Test that Path objects are accepted as filename arguments to handlers. + Test that path-like objects are accepted as filename arguments to handlers. See Issue #27493. """ fn = make_temp_file() os.unlink(fn) - pfn = pathlib.Path(fn) + pfn = os_helper.FakePath(fn) cases = ( (logging.FileHandler, (pfn, 'w')), (logging.handlers.RotatingFileHandler, (pfn, 'a')), (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), ) - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") @@ -732,6 +735,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): @@ -1034,6 +1038,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True + allow_reuse_port = True def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -2188,7 +2193,8 @@ def test_output(self): self.handled.clear() msg = "sp\xe4m" logger.error(msg) - self.handled.wait() + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") self.assertEqual(self.log_data.path, '/frob') self.assertEqual(self.command, method) if method == 'GET': @@ -2998,6 +3004,63 @@ class ConfigDictTest(BaseTest): }, } + class CustomFormatter(logging.Formatter): + custom_property = "." + + def format(self, record): + return super().format(record) + + config17 = { + 'version': 1, + 'formatters': { + "custom": { + "()": CustomFormatter, + "style": "{", + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": "{message}", # <-- to force an exception when configuring + ".": { + "custom_property": "value" + } + } + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'custom', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + config18 = { + "version": 1, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "buffering": { + "class": "logging.handlers.MemoryHandler", + "capacity": 5, + "target": "console", + "level": "DEBUG", + "flushLevel": "ERROR" + } + }, + "loggers": { + "mymodule": { + "level": "DEBUG", + "handlers": ["buffering"], + "propagate": "true" + } + } + } + bad_format = { "version": 1, "formatters": { @@ -3479,7 +3542,15 @@ def test_config16_ok(self): {'msg': 'Hello'})) self.assertEqual(result, 'Hello ++ defaultvalue') + def test_config17_ok(self): + self.apply_config(self.config17) + h = logging._handlers['hand1'] + self.assertEqual(h.formatter.custom_property, 'value') + def test_config18_ok(self): + self.apply_config(self.config18) + handler = logging.getLogger('mymodule').handlers[0] + self.assertEqual(handler.flushLevel, logging.ERROR) def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") @@ -3828,6 +3899,7 @@ def do_queuehandler_configuration(self, qspec, lspec): self.addCleanup(os.remove, fn) @threading_helper.requires_working_threading() + @support.requires_subprocess() def test_config_queue_handler(self): q = CustomQueue() dq = { @@ -3856,6 +3928,78 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + @threading_helper.requires_working_threading() + @support.requires_subprocess() + @patch("multiprocessing.Manager") + def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): + # gh-120868 + + from multiprocessing import Queue as MQ + + q1 = {"()": "queue.Queue", "maxsize": -1} + q2 = MQ() + q3 = queue.Queue() + + for qspec in (q1, q2, q3): + self.apply_config( + { + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + } + ) + manager.assert_not_called() + + @patch("multiprocessing.Manager") + def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): + # gh-120868 + + with self.assertRaises(ValueError): + self.apply_config( + { + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": object(), + }, + }, + } + ) + manager.assert_not_called() + + @support.requires_subprocess() + def test_multiprocessing_queues(self): + # See gh-119819 + + cd = copy.deepcopy(self.config_queue_handler) + from multiprocessing import Queue as MQ, Manager as MM + q1 = MQ() # this can't be pickled + q2 = MM().Queue() # a proxy queue for use when pickling is needed + q3 = MM().JoinableQueue() # a joinable proxy queue + for qspec in (q1, q2, q3): + fn = make_temp_file('.log', 'test_logging-cmpqh-') + cd['handlers']['h1']['filename'] = fn + cd['handlers']['ah']['queue'] = qspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + self.assertIs(qh.queue, qspec) + self.assertIs(qh.listener.queue, qspec) + finally: + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + def test_90195(self): # See gh-90195 config = { @@ -3886,6 +4030,55 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) + def test_111615(self): + # See gh-111615 + import_helper.import_module('_multiprocessing') # see gh-113692 + mp = import_helper.import_module('multiprocessing') + + config = { + 'version': 1, + 'handlers': { + 'sink': { + 'class': 'logging.handlers.QueueHandler', + 'queue': mp.get_context('spawn').Queue(), + }, + }, + 'root': { + 'handlers': ['sink'], + 'level': 'DEBUG', + }, + } + logging.config.dictConfig(config) + + # gh-118868: check if kwargs are passed to logging QueueHandler + def test_kwargs_passing(self): + class CustomQueueHandler(logging.handlers.QueueHandler): + def __init__(self, *args, **kwargs): + super().__init__(queue.Queue()) + self.custom_kwargs = kwargs + + custom_kwargs = {'foo': 'bar'} + + config = { + 'version': 1, + 'handlers': { + 'custom': { + 'class': CustomQueueHandler, + **custom_kwargs + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['custom'] + } + } + + logging.config.dictConfig(config) + + handler = logging.getHandlerByName('custom') + self.assertEqual(handler.custom_kwargs, custom_kwargs) + + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] @@ -4034,6 +4227,7 @@ def test_queue_listener(self): self.que_logger.critical(self.next_message()) finally: listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) @@ -4491,6 +4685,77 @@ def test_issue_89047(self): s = f.format(r) self.assertNotIn('.1000', s) + def test_msecs_has_no_floating_point_precision_loss(self): + # See issue gh-102402 + tests = ( + # time_ns is approx. 2023-03-04 04:25:20 UTC + # (time_ns, expected_msecs_value) + (1_677_902_297_100_000_000, 100.0), # exactly 100ms + (1_677_903_920_999_998_503, 999.0), # check truncating doesn't round + (1_677_903_920_000_998_503, 0.0), # check truncating doesn't round + (1_677_903_920_999_999_900, 0.0), # check rounding up + ) + for ns, want in tests: + with patch('time.time_ns') as patched_ns: + patched_ns.return_value = ns + record = logging.makeLogRecord({'msg': 'test'}) + with self.subTest(ns): + self.assertEqual(record.msecs, want) + self.assertEqual(record.created, ns / 1e9) + self.assertAlmostEqual(record.created - int(record.created), + record.msecs / 1e3, + delta=1e-3) + + def test_relativeCreated_has_higher_precision(self): + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging + + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) + # After PR gh-102412, precision (places) increases from 3 to 7 + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): @@ -5422,6 +5687,7 @@ def test_critical(self): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, msg) self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_critical') def test_is_enabled_for(self): old_disable = self.adapter.logger.manager.disable @@ -5440,15 +5706,9 @@ def test_has_handlers(self): self.assertFalse(self.adapter.hasHandlers()) def test_nested(self): - class Adapter(logging.LoggerAdapter): - prefix = 'Adapter' - - def process(self, msg, kwargs): - return f"{self.prefix} {msg}", kwargs - msg = 'Adapters can be nested, yo.' - adapter = Adapter(logger=self.logger, extra=None) - adapter_adapter = Adapter(logger=adapter, extra=None) + adapter = PrefixAdapter(logger=self.logger, extra=None) + adapter_adapter = PrefixAdapter(logger=adapter, extra=None) adapter_adapter.prefix = 'AdapterAdapter' self.assertEqual(repr(adapter), repr(adapter_adapter)) adapter_adapter.log(logging.CRITICAL, msg, self.recording) @@ -5457,6 +5717,7 @@ def process(self, msg, kwargs): self.assertEqual(record.levelno, logging.CRITICAL) self.assertEqual(record.msg, f"Adapter AdapterAdapter {msg}") self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_nested') orig_manager = adapter_adapter.manager self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) @@ -5472,6 +5733,61 @@ def process(self, msg, kwargs): self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) + def test_styled_adapter(self): + # Test an example from the Cookbook. + records = self.recording.records + adapter = StyleAdapter(self.logger) + adapter.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), 'Hello, world!') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + adapter.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), 'Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + + def test_nested_styled_adapter(self): + records = self.recording.records + adapter = PrefixAdapter(self.logger) + adapter.prefix = '{}' + adapter2 = StyleAdapter(adapter) + adapter2.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), '{} Hello, world!') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + adapter2.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), '{} Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + + def test_find_caller_with_stacklevel(self): + the_level = 1 + trigger = self.adapter.warning + + def innermost(): + trigger('test', stacklevel=the_level) + + def inner(): + innermost() + + def outer(): + inner() + + records = self.recording.records + outer() + self.assertEqual(records[-1].funcName, 'innermost') + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'inner') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'outer') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'test_find_caller_with_stacklevel') + self.assertGreater(records[-1].lineno, lineno) + def test_extra_in_records(self): self.adapter = logging.LoggerAdapter(logger=self.logger, extra={'foo': '1'}) @@ -5513,6 +5829,30 @@ def test_extra_merged_log_call_has_precedence(self): self.assertEqual(record.foo, '2') +class PrefixAdapter(logging.LoggerAdapter): + prefix = 'Adapter' + + def process(self, msg, kwargs): + return f"{self.prefix} {msg}", kwargs + + +class Message: + def __init__(self, fmt, args): + self.fmt = fmt + self.args = args + + def __str__(self): + return self.fmt.format(*self.args) + + +class StyleAdapter(logging.LoggerAdapter): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) + + class LoggerTest(BaseTest, AssertErrorMessage): def setUp(self): @@ -5949,6 +6289,52 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) + now = datetime.datetime.now() + atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) + fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' + for i in range(3): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) + fh.emit(r2) + fh.close() + self.assertLogFile(self.fn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + os.utime(self.fn, (now.timestamp() - 1,)*2) + for i in range(2): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) + fh.emit(r2) + fh.close() + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + self.assertLogFile(otherfn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing2 {i}', line) + with open(otherfn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, @@ -5958,22 +6344,47 @@ def test_invalid(self): assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, self.fn, 'W7', encoding="utf-8", delay=True) + # TODO: Test for utc=False. def test_compute_rollover_daily_attime(self): currentTime = 0 + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', + utc=True, atTime=None) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + + actual = rh.computeRollover(currentTime + 25 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + finally: + rh.close() + atTime = datetime.time(12, 0, 0) rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', interval=1, backupCount=0, + self.fn, encoding="utf-8", when='MIDNIGHT', utc=True, atTime=atTime) try: actual = rh.computeRollover(currentTime) self.assertEqual(actual, currentTime + 12 * 60 * 60) + actual = rh.computeRollover(currentTime + 12 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 12 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + actual = rh.computeRollover(currentTime + 13 * 60 * 60) self.assertEqual(actual, currentTime + 36 * 60 * 60) finally: rh.close() - #@unittest.skipIf(True, 'Temporarily skipped while failures investigated.') + # TODO: Test for utc=False. def test_compute_rollover_weekly_attime(self): currentTime = int(time.time()) today = currentTime - currentTime % 86400 @@ -5998,14 +6409,28 @@ def test_compute_rollover_weekly_attime(self): expected += 12 * 60 * 60 # Add in adjustment for today expected += today + actual = rh.computeRollover(today) if actual != expected: print('failed in timezone: %d' % time.timezone) print('local vars: %s' % locals()) self.assertEqual(actual, expected) + + actual = rh.computeRollover(today + 12 * 60 * 60 - 1) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + if day == wday: # goes into following week expected += 7 * 24 * 60 * 60 + actual = rh.computeRollover(today + 12 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + actual = rh.computeRollover(today + 13 * 60 * 60) if actual != expected: print('failed in timezone: %d' % time.timezone) @@ -6023,7 +6448,7 @@ def test_compute_files_to_delete(self): for i in range(10): times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) dt += datetime.timedelta(seconds=5) - prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f') + prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f', 'g') files = [] rotators = [] for prefix in prefixes: @@ -6036,10 +6461,22 @@ def test_compute_files_to_delete(self): if prefix.startswith('a.b'): for t in times: files.append('%s.log.%s' % (prefix, t)) - else: - rotator.namer = lambda name: name.replace('.log', '') + '.log' + elif prefix.startswith('d.e'): + def namer(filename): + dirname, basename = os.path.split(filename) + basename = basename.replace('.log', '') + '.log' + return os.path.join(dirname, basename) + rotator.namer = namer for t in times: files.append('%s.%s.log' % (prefix, t)) + elif prefix == 'g': + def namer(filename): + dirname, basename = os.path.split(filename) + basename = 'g' + basename[6:] + '.oldlog' + return os.path.join(dirname, basename) + rotator.namer = namer + for t in times: + files.append('g%s.oldlog' % t) # Create empty files for fn in files: p = os.path.join(wd, fn) @@ -6049,18 +6486,423 @@ def test_compute_files_to_delete(self): for i, prefix in enumerate(prefixes): rotator = rotators[i] candidates = rotator.getFilesToDelete() - self.assertEqual(len(candidates), 3) + self.assertEqual(len(candidates), 3, candidates) if prefix.startswith('a.b'): p = '%s.log.' % prefix for c in candidates: d, fn = os.path.split(c) self.assertTrue(fn.startswith(p)) - else: + elif prefix.startswith('d.e'): for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.log')) + self.assertTrue(fn.endswith('.log'), fn) self.assertTrue(fn.startswith(prefix + '.') and fn[len(prefix) + 2].isdigit()) + elif prefix == 'g': + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.endswith('.oldlog')) + self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + + def test_compute_files_to_delete_same_filename_different_extensions(self): + # See GH-93205 for background + wd = pathlib.Path(tempfile.mkdtemp(prefix='test_logging_')) + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + n_files = 10 + for _ in range(n_files): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.log', 'a.log.b') + files = [] + rotators = [] + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = logging.handlers.TimedRotatingFileHandler(wd / prefix, when='s', + interval=5, + backupCount=backupCount, + delay=True) + rotators.append(rotator) + for t in times: + files.append('%s.%s' % (prefix, t)) + for t in times: + files.append('a.log.%s.c' % t) + # Create empty files + for f in files: + (wd / f).touch() + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), n_files - backupCount, candidates) + matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(prefix+'.')) + suffix = fn[(len(prefix)+1):] + self.assertRegex(suffix, matcher) + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 10, 1, 59, 59), DT(2012, 3, 10, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 12, 2, 0)) + + test(DT(2012, 11, 3, 1, 59, 59), DT(2012, 11, 3, 2, 0)) + test(DT(2012, 11, 3, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 5, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 10, 2, 29, 59), DT(2012, 3, 10, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 2, 30)) + + test(DT(2012, 11, 3, 2, 29, 59), DT(2012, 11, 3, 2, 30)) + test(DT(2012, 11, 3, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 12, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 5, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-5 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 5, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False) + + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 18, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 11, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 4, 1, 59, 59), DT(2012, 3, 4, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 18, 2, 0)) + + test(DT(2012, 10, 28, 1, 59, 59), DT(2012, 10, 28, 2, 0)) + test(DT(2012, 10, 28, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 11, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 4, 2, 29, 59), DT(2012, 3, 4, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 2, 30)) + + test(DT(2012, 10, 28, 2, 29, 59), DT(2012, 10, 28, 2, 30)) + test(DT(2012, 10, 28, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 18, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 11, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-11 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 11, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3) + + test(DT(2012, 3, 8, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 9, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 9, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 13, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 14, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 14, 0, 0)) + + test(DT(2012, 11, 1, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 2, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 2, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 6, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 7, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 7, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 8, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 8, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 8, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 12, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 13, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 13, 12, 0)) + + test(DT(2012, 11, 1, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 1, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 1, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 5, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 6, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 6, 12, 0)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3) + + test(DT(2012, 2, 19, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 2, 20, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 2, 20, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 19, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 26, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 26, 0, 0)) + + test(DT(2012, 10, 14, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 15, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 15, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 11, 12, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 19, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 19, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 2, 25, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 2, 26, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 2, 26, 1, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 25, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 4, 1, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 4, 1, 0, 0)) + + test(DT(2012, 10, 20, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 10, 21, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 10, 21, 1, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 18, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 25, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 25, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 2, 18, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 2, 19, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 2, 19, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 18, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 25, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 25, 12, 0)) + + test(DT(2012, 10, 14, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 14, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 14, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 11, 11, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 18, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 18, 12, 0)) + + fh.close() def secs(**kw): @@ -6074,40 +6916,49 @@ def secs(**kw): # current time (epoch start) is a Thursday, W0 means Monday ('W0', secs(days=4, hours=24)), ): - def test_compute_rollover(self, when=when, exp=exp): - rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=when, interval=1, backupCount=0, utc=True) - currentTime = 0.0 - actual = rh.computeRollover(currentTime) - if exp != actual: - # Failures occur on some systems for MIDNIGHT and W0. - # Print detailed calculation for MIDNIGHT so we can try to see - # what's going on - if when == 'MIDNIGHT': - try: - if rh.utc: - t = time.gmtime(currentTime) - else: - t = time.localtime(currentTime) - currentHour = t[3] - currentMinute = t[4] - currentSecond = t[5] - # r is the number of seconds left between now and midnight - r = logging.handlers._MIDNIGHT - ((currentHour * 60 + - currentMinute) * 60 + - currentSecond) - result = currentTime + r - print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) - print('currentHour: %s' % currentHour, file=sys.stderr) - print('currentMinute: %s' % currentMinute, file=sys.stderr) - print('currentSecond: %s' % currentSecond, file=sys.stderr) - print('r: %s' % r, file=sys.stderr) - print('result: %s' % result, file=sys.stderr) - except Exception as e: - print('exception in diagnostic code: %s' % e, file=sys.stderr) - self.assertEqual(exp, actual) - rh.close() - setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + for interval in 1, 3: + def test_compute_rollover(self, when=when, interval=interval, exp=exp): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, interval=interval, backupCount=0, utc=True) + currentTime = 0.0 + actual = rh.computeRollover(currentTime) + if when.startswith('W'): + exp += secs(days=7*(interval-1)) + else: + exp *= interval + if exp != actual: + # Failures occur on some systems for MIDNIGHT and W0. + # Print detailed calculation for MIDNIGHT so we can try to see + # what's going on + if when == 'MIDNIGHT': + try: + if rh.utc: + t = time.gmtime(currentTime) + else: + t = time.localtime(currentTime) + currentHour = t[3] + currentMinute = t[4] + currentSecond = t[5] + # r is the number of seconds left between now and midnight + r = logging.handlers._MIDNIGHT - ((currentHour * 60 + + currentMinute) * 60 + + currentSecond) + result = currentTime + r + print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) + print('currentHour: %s' % currentHour, file=sys.stderr) + print('currentMinute: %s' % currentMinute, file=sys.stderr) + print('currentSecond: %s' % currentSecond, file=sys.stderr) + print('r: %s' % r, file=sys.stderr) + print('result: %s' % result, file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) + self.assertEqual(exp, actual) + rh.close() + name = "test_compute_rollover_%s" % when + if interval > 1: + name += "_interval" + test_compute_rollover.__name__ = name + setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index d299c34cec076d..3b2e7c4e71d10d 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -386,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. @@ -1639,6 +1630,8 @@ class MyInt(int): MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits ) + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 13b200912f6abd..22478c14fb4a65 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2,7 +2,6 @@ import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os -import pathlib import pickle import random import sys @@ -12,7 +11,7 @@ from test.support import _4G, bigmemtest from test.support.import_helper import import_module from test.support.os_helper import ( - TESTFN, unlink + TESTFN, unlink, FakePath ) lzma = import_module("lzma") @@ -548,23 +547,29 @@ def test_init(self): pass def test_init_with_PathLike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename, "a") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) def test_init_with_filename(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') with LZMAFile(TESTFN, "w") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') def test_init_mode(self): with TempFile(TESTFN): @@ -585,11 +590,11 @@ def test_init_with_x_mode(self): self.addCleanup(unlink, TESTFN) for mode in ("x", "xb"): unlink(TESTFN) - with LZMAFile(TESTFN, mode): + with LZMAFile(TESTFN, mode) as f: pass + self.assertEqual(f.mode, 'wb') with self.assertRaises(FileExistsError): - with LZMAFile(TESTFN, mode): - pass + LZMAFile(TESTFN, mode) def test_init_bad_mode(self): with self.assertRaises(ValueError): @@ -867,16 +872,74 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_from_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) + + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_incomplete(self): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: @@ -1005,6 +1068,8 @@ def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(INPUT) + with self.assertRaises(AttributeError): + f.name expected = lzma.compress(INPUT) self.assertEqual(dst.getvalue(), expected) with BytesIO() as dst: @@ -1041,16 +1106,35 @@ def test_write_append(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') self.assertEqual(dst.getvalue(), expected) def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1058,13 +1142,66 @@ def test_write_to_file(self): unlink(TESTFN) def test_write_to_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) + self.assertEqual(f.name, bytes_filename) + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1079,10 +1216,13 @@ def test_write_append_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) finally: @@ -1276,15 +1416,17 @@ def test_filename(self): self.assertEqual(f.read(), INPUT * 2) def test_with_pathlike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with open(filename, "rb") as f: file_data = lzma.decompress(f.read()) self.assertEqual(file_data, INPUT) with lzma.open(filename, "rb") as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) def test_bad_params(self): # Test invalid parameter combinations. @@ -1401,6 +1543,14 @@ def test__decode_filter_properties(self): self.assertEqual(filterspec["lc"], 3) self.assertEqual(filterspec["dict_size"], 8 << 20) + # see gh-104282 + filters = [lzma.FILTER_X86, lzma.FILTER_POWERPC, + lzma.FILTER_IA64, lzma.FILTER_ARM, + lzma.FILTER_ARMTHUMB, lzma.FILTER_SPARC] + for f in filters: + filterspec = lzma._decode_filter_properties(f, b"") + self.assertEqual(filterspec, {"id": f}) + def test_filter_properties_roundtrip(self): spec1 = lzma._decode_filter_properties( lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 4977a9369ddf88..3ecb5eab26d4b9 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1,7 +1,6 @@ import os import sys import time -import stat import socket import email import email.message @@ -9,7 +8,9 @@ import io import tempfile from test import support +from test.support import import_helper from test.support import os_helper +from test.support import refleak_helper from test.support import socket_helper import unittest import textwrap @@ -681,14 +682,27 @@ def test_initialize_existing(self): self._box = mailbox.Maildir(self._path) self._check_basics() + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + def _check_basics(self, factory=None): # (Used by test_open_new() and test_open_existing().) self.assertEqual(self._box._path, os.path.abspath(self._path)) self.assertEqual(self._box._factory, factory) for subdir in '', 'tmp', 'new', 'cur': path = os.path.join(self._path, subdir) - mode = os.stat(path)[stat.ST_MODE] - self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") def test_list_folders(self): # List folders @@ -847,6 +861,92 @@ def test_lock_unlock(self): self._box.lock() self._box.unlock() + def test_get_info(self): + # Test getting message info from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_info(key), '') + msg.set_info('OurTestInfo') + self._box[key] = msg + self.assertEqual(self._box.get_info(key), 'OurTestInfo') + + def test_set_info(self): + # Test setting message info from Maildir, not the message. + # This should immediately rename the message file. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + def check_info(oldinfo, newinfo): + oldfilename = os.path.join(self._box._path, self._box._lookup(key)) + newsubpath = self._box._lookup(key).split(self._box.colon)[0] + if newinfo: + newsubpath += self._box.colon + newinfo + newfilename = os.path.join(self._box._path, newsubpath) + # assert initial conditions + self.assertEqual(self._box.get_info(key), oldinfo) + if not oldinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + self.assertTrue(os.path.exists(oldfilename)) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(newfilename)) + # do the rename + self._box.set_info(key, newinfo) + # assert post conditions + if not newinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(oldfilename)) + self.assertTrue(os.path.exists(newfilename)) + self.assertEqual(self._box.get_info(key), newinfo) + # none -> has info + check_info('', 'info1') + # has info -> same info + check_info('info1', 'info1') + # has info -> different info + check_info('info1', 'info2') + # has info -> none + check_info('info2', '') + # none -> none + check_info('', '') + + def test_get_flags(self): + # Test getting message flags from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + msg.set_flags('T') + self._box[key] = msg + self.assertEqual(self._box.get_flags(key), 'T') + + def test_set_flags(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.set_flags(key, 'S') + self.assertEqual(self._box.get_flags(key), 'S') + + def test_add_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'AC') + self.assertEqual(self._box.get_flags(key), 'ABC') + + def test_remove_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self._box.set_flags(key, 'abc') + self.assertEqual(self._box.get_flags(key), 'abc') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'ac') + self.assertEqual(self._box.get_flags(key), '') + def test_folder (self): # Test for bug #1569790: verify that folders returned by .get_folder() # use the same factory function. @@ -981,6 +1081,47 @@ def test_permissions_after_flush(self): self.assertEqual(os.stat(self._path).st_mode, mode) + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + class _TestMboxMMDF(_TestSingleFile): @@ -1027,12 +1168,14 @@ def test_add_from_string(self): # Add a string starting with 'From ' to the mailbox key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_from_bytes(self): # Add a byte string starting with 'From ' to the mailbox key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') self.assertEqual(self._box[key].get_payload(), '0\n') def test_add_mbox_or_mmdf_message(self): @@ -1247,6 +1390,28 @@ def test_sequences(self): self._box.remove(key1) self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_lock_unlock_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + box.lock() + box.unlock() + self.assertEqual(os.listdir(path), ['.mh_sequences']) + def test_issue2625(self): msg0 = mailbox.MHMessage(self._template % 0) msg0.add_sequence('foo') @@ -1545,18 +1710,23 @@ def test_initialize_with_unixfrom(self): msg = mailbox.Message(_sample_message) msg.set_unixfrom('From foo@bar blah') msg = mailbox.mboxMessage(msg) - self.assertEqual(msg.get_from(), 'foo@bar blah', msg.get_from()) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') def test_from(self): # Get and set "From " line msg = mailbox.mboxMessage(_sample_message) self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo bar') self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('foo@bar', True) self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) msg.set_from('blah@temp', time.localtime()) self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) def test_flags(self): # Use get_flags(), set_flags(), add_flag(), remove_flag() @@ -1744,6 +1914,7 @@ def test_maildir_to_mboxmmdf(self): self.assertEqual(msg.get_flags(), result) self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) msg_maildir.set_subdir('cur') self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') @@ -1792,10 +1963,12 @@ def test_mboxmmdf_to_mboxmmdf(self): msg_mboxMMDF = class_(_sample_message) msg_mboxMMDF.set_flags('RODFA') msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): msg2 = class2_(msg_mboxMMDF) self.assertEqual(msg2.get_flags(), 'RODFA') self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) def test_mboxmmdf_to_mh(self): # Convert mboxMessage and MMDFMessage to MHMessage @@ -2311,6 +2484,9 @@ def test__all__(self): def tearDownModule(): support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() if __name__ == '__main__': diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 3d9d6d5d0aca34..64ee1ba867d592 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges from test.support.script_helper import assert_python_ok import array import io @@ -118,7 +118,7 @@ def test_code(self): def test_many_codeobjects(self): # Issue2957: bad recursion count on code objects # more than MAX_MARSHAL_STACK_DEPTH - count = support.EXCEEDS_RECURSION_LIMIT + count = support.exceeds_recursion_limit() codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) @@ -129,6 +129,32 @@ def test_different_filenames(self): self.assertEqual(co1.co_filename, "f1") self.assertEqual(co2.co_filename, "f2") + def test_no_allow_code(self): + data = {'a': [({0},)]} + dump = marshal.dumps(data, allow_code=False) + self.assertEqual(marshal.loads(dump, allow_code=False), data) + + f = io.BytesIO() + marshal.dump(data, f, allow_code=False) + f.seek(0) + self.assertEqual(marshal.load(f, allow_code=False), data) + + co = ExceptionTestCase.test_exceptions.__code__ + data = {'a': [({co, 0},)]} + dump = marshal.dumps(data, allow_code=True) + self.assertEqual(marshal.loads(dump, allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dumps(data, allow_code=False) + with self.assertRaises(ValueError): + marshal.loads(dump, allow_code=False) + + marshal.dump(data, io.BytesIO(), allow_code=True) + self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dump(data, io.BytesIO(), allow_code=False) + with self.assertRaises(ValueError): + marshal.load(io.BytesIO(dump), allow_code=False) + @requires_debug_ranges() def test_minimal_linetable_with_no_debug_ranges(self): # Make sure when demarshalling objects with `-X no_debug_ranges` @@ -260,7 +286,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi': + elif sys.platform == 'wasi' or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index d5d2197c36b254..b68c442013c09f 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -657,7 +657,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -685,6 +685,7 @@ def msum(iterable): ([], 0.0), ([0.0], 0.0), ([1e100, 1.0, -1e100, 1e-100, 1e50, -1.0, -1e50], 1e-100), + ([1e100, 1.0, -1e100, 1e-100, 1e50, -1, -1e50], 1e-100), ([2.0**53, -0.5, -2.0**-54], 2.0**53-1.0), ([2.0**53, 1.0, 2.0**-100], 2.0**53+2.0), ([2.0**53+10.0, 1.0, 2.0**-100], 2.0**53+12.0), @@ -733,9 +734,20 @@ def msum(iterable): self.assertEqual(msum(vals), math.fsum(vals)) self.assertEqual(math.fsum([1.0, math.inf]), math.inf) + self.assertTrue(math.isnan(math.fsum([math.nan, 1.0]))) + self.assertEqual(math.fsum([1e100, FloatLike(1.0), -1e100, 1e-100, + 1e50, FloatLike(-1.0), -1e50]), 1e-100) self.assertRaises(OverflowError, math.fsum, [1e+308, 1e+308]) self.assertRaises(ValueError, math.fsum, [math.inf, -math.inf]) self.assertRaises(TypeError, math.fsum, ['spam']) + self.assertRaises(TypeError, math.fsum, 1) + self.assertRaises(OverflowError, math.fsum, [10**1000]) + + def bad_iter(): + yield 1.0 + raise ZeroDivisionError + + self.assertRaises(ZeroDivisionError, math.fsum, bad_iter()) def testGcd(self): gcd = math.gcd @@ -797,6 +809,8 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) + self.assertEqual(hypot(1, -1), math.sqrt(2)) + self.assertEqual(hypot(1, FloatLike(-1.)), math.sqrt(2)) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) @@ -948,6 +962,10 @@ def testDist(self): # Test allowable types (those with __float__) self.assertEqual(dist((14.0, 1.0), (2.0, -4.0)), 13.0) self.assertEqual(dist((14, 1), (2, -4)), 13) + self.assertEqual(dist((FloatLike(14.), 1), (2, -4)), 13) + self.assertEqual(dist((11, 1), (FloatLike(-1.), -4)), 13) + self.assertEqual(dist((14, FloatLike(-1.)), (2, -6)), 13) + self.assertEqual(dist((14, -1), (2, -6)), 13) self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) @@ -1005,6 +1023,12 @@ class T(tuple): with self.assertRaises(TypeError): dist([1], 2) + class BadFloat: + __float__ = BadDescr() + + with self.assertRaises(ValueError): + dist([1], [BadFloat()]) + # Verify that the one dimensional case is equivalent to abs() for i in range(20): p, q = random.random(), random.random() @@ -1175,6 +1199,7 @@ def testLdexp(self): def testLog(self): self.assertRaises(TypeError, math.log) + self.assertRaises(TypeError, math.log, 1, 2, 3) self.ftest('log(1/e)', math.log(1/math.e), -1) self.ftest('log(1)', math.log(1), 0) self.ftest('log(e)', math.log(math.e), 1) @@ -1245,6 +1270,8 @@ def testSumProd(self): self.assertEqual(sumprod(iter([10, 20, 30]), (1, 2, 3)), 140) self.assertEqual(sumprod([1.5, 2.5], [3.5, 4.5]), 16.5) self.assertEqual(sumprod([], []), 0) + self.assertEqual(sumprod([-1], [1.]), -1) + self.assertEqual(sumprod([1.], [-1]), -1) # Type preservation and coercion for v in [ @@ -1270,11 +1297,20 @@ def testSumProd(self): self.assertRaises(TypeError, sumprod, [], [], []) # Three args self.assertRaises(TypeError, sumprod, None, [10]) # Non-iterable self.assertRaises(TypeError, sumprod, [10], None) # Non-iterable + self.assertRaises(TypeError, sumprod, ['x'], [1.0]) # Uneven lengths self.assertRaises(ValueError, sumprod, [10, 20], [30]) self.assertRaises(ValueError, sumprod, [10], [20, 30]) + # Overflows + self.assertEqual(sumprod([10**20], [1]), 10**20) + self.assertEqual(sumprod([1], [10**20]), 10**20) + self.assertEqual(sumprod([10**10], [10**10]), 10**20) + self.assertEqual(sumprod([10**7]*10**5, [10**7]*10**5), 10**19) + self.assertRaises(OverflowError, sumprod, [10**1000], [1.0]) + self.assertRaises(OverflowError, sumprod, [1.0], [10**1000]) + # Error in iterator def raise_after(n): for i in range(n): @@ -1285,6 +1321,11 @@ def raise_after(n): with self.assertRaises(RuntimeError): sumprod(raise_after(5), range(10)) + from test.test_iter import BasicIterClass + + self.assertEqual(sumprod(BasicIterClass(1), [1]), 0) + self.assertEqual(sumprod([1], BasicIterClass(1)), 0) + # Error in multiplication class BadMultiply: def __mul__(self, other): @@ -1325,6 +1366,7 @@ def test_sumprod_accuracy(self): sumprod = math.sumprod self.assertEqual(sumprod([0.1] * 10, [1]*10), 1.0) self.assertEqual(sumprod([0.1] * 20, [True, False] * 10), 1.0) + self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) @support.requires_resource('cpu') @@ -1523,6 +1565,7 @@ def testPow(self): self.assertTrue(math.isnan(math.pow(2, NAN))) self.assertTrue(math.isnan(math.pow(0, NAN))) self.assertEqual(math.pow(1, NAN), 1) + self.assertRaises(OverflowError, math.pow, 1e+100, 1e+100) # pow(0., x) self.assertEqual(math.pow(0., INF), 0.) @@ -1879,6 +1922,8 @@ def __trunc__(self): return 23 class TestNoTrunc: pass + class TestBadTrunc: + __trunc__ = BadDescr() self.assertEqual(math.trunc(TestTrunc()), 23) self.assertEqual(math.trunc(FloatTrunc()), 23) @@ -1887,6 +1932,7 @@ class TestNoTrunc: self.assertRaises(TypeError, math.trunc, 1, 2) self.assertRaises(TypeError, math.trunc, FloatLike(23.5)) self.assertRaises(TypeError, math.trunc, TestNoTrunc()) + self.assertRaises(ValueError, math.trunc, TestBadTrunc()) def testIsfinite(self): self.assertTrue(math.isfinite(0.0)) @@ -2087,6 +2133,8 @@ def test_mtestfile(self): '\n '.join(failures)) def test_prod(self): + from fractions import Fraction as F + prod = math.prod self.assertEqual(prod([]), 1) self.assertEqual(prod([], start=5), 5) @@ -2098,6 +2146,14 @@ def test_prod(self): self.assertEqual(prod([1.0, 2.0, 3.0, 4.0, 5.0]), 120.0) self.assertEqual(prod([1, 2, 3, 4.0, 5.0]), 120.0) self.assertEqual(prod([1.0, 2.0, 3.0, 4, 5]), 120.0) + self.assertEqual(prod([1., F(3, 2)]), 1.5) + + # Error in multiplication + class BadMultiply: + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + prod([10., BadMultiply()]) # Test overflow in fast-path for integers self.assertEqual(prod([1, 1, 2**32, 1, 1]), 2**32) @@ -2557,6 +2613,244 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi")) + or (sys.platform == "android" and platform.machine() == "x86_64"), + f"this platform doesn't implement IEE 754-2008 properly") + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index cd2faba1791c77..95629ed862d6eb 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -6,10 +6,12 @@ import unittest from test import support +import gc import io import _pyio as pyio import pickle import sys +import weakref class IntLike: def __init__(self, num): @@ -463,6 +465,39 @@ def test_getbuffer(self): memio.close() self.assertRaises(ValueError, memio.getbuffer) + def test_getbuffer_empty(self): + memio = self.ioclass() + buf = memio.getbuffer() + self.assertEqual(bytes(buf), b"") + # Trying to change the size of the BytesIO while a buffer is exported + # raises a BufferError. + self.assertRaises(BufferError, memio.write, b'x') + buf2 = memio.getbuffer() + self.assertRaises(BufferError, memio.write, b'x') + buf.release() + self.assertRaises(BufferError, memio.write, b'x') + buf2.release() + memio.write(b'x') + + def test_getbuffer_gc_collect(self): + memio = self.ioclass(b"1234567890") + buf = memio.getbuffer() + memiowr = weakref.ref(memio) + bufwr = weakref.ref(buf) + # Create a reference loop. + a = [buf] + a.append(a) + # The Python implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + del memio + del buf + del a + # The C implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + gc.collect() + self.assertIsNone(memiowr()) + self.assertIsNone(bufwr()) + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) @@ -766,7 +801,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 36e8ab4cda3dad..b37b7defe84d1c 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -164,9 +164,11 @@ ... d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 + d['__static_attributes__'] = () >>> Use a metaclass that doesn't derive from type. @@ -182,12 +184,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -205,11 +207,13 @@ prepare: C () [('other', 'booh')] d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['a'] = 1 d['a'] = 2 d['b'] = 3 + d['__static_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index d64aee71fc48b1..58f6a4dfae08ba 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,6 +1,6 @@ import io import mimetypes -import pathlib +import os import sys import unittest.mock @@ -35,20 +35,28 @@ def setUp(self): def test_case_sensitivity(self): eq = self.assertEqual - eq(self.db.guess_type("foobar.HTML"), self.db.guess_type("foobar.html")) - eq(self.db.guess_type("foobar.TGZ"), self.db.guess_type("foobar.tgz")) - eq(self.db.guess_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -62,7 +70,7 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") @@ -74,11 +82,19 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding="utf-8") + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. @@ -94,16 +110,14 @@ def test_read_mime_types(self): def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) - eq(self.db.guess_extension('image/webp', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') - eq(self.db.guess_extension('image/webp', strict=False), '.webp') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -111,15 +125,51 @@ def test_filename_with_url_delimiters(self): # compared to when interpreted as filename because of the semicolon. eq = self.assertEqual gzip_expected = ('application/x-tar', 'gzip') - eq(self.db.guess_type(";1.tar.gz"), gzip_expected) - eq(self.db.guess_type("?1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) - eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + def test_guess_all_types(self): # First try strict. Use a set here for testing the results because if # test_urllib2 is run before test_mimetypes, global state is modified @@ -183,11 +233,14 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') + self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') self.assertEqual(mimetypes.guess_extension('text/html'), '.html') self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') + self.assertEqual(mimetypes.guess_extension('text/rtf'), '.rtf') + self.assertEqual(mimetypes.guess_extension('text/x-rst'), '.rst') self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') @@ -215,22 +268,38 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index dfcf3039422af5..a1cf5384ada5b5 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,9 +1,11 @@ from test.support import ( - requires, _2G, _4G, gc_collect, cpython_only, is_emscripten + requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok import unittest +import errno import os import re import itertools @@ -11,6 +13,7 @@ import socket import string import sys +import textwrap import weakref # Skip test if we can't import mmap. @@ -93,11 +96,12 @@ def test_basic(self): self.assertEqual(end, PAGESIZE + 6) # test seeking around (try to overflow the seek implementation) - m.seek(0,0) + self.assertTrue(m.seekable()) + self.assertEqual(m.seek(0, 0), 0) self.assertEqual(m.tell(), 0) - m.seek(42,1) + self.assertEqual(m.seek(42, 1), 42) self.assertEqual(m.tell(), 42) - m.seek(0,2) + self.assertEqual(m.seek(0, 2), len(m)) self.assertEqual(m.tell(), len(m)) # Try to seek to negative position... @@ -162,7 +166,7 @@ def test_access_parameter(self): # Ensuring that readonly mmap can't be write() to try: - m.seek(0,0) + m.seek(0, 0) m.write(b'abc') except TypeError: pass @@ -171,7 +175,7 @@ def test_access_parameter(self): # Ensuring that readonly mmap can't be write_byte() to try: - m.seek(0,0) + m.seek(0, 0) m.write_byte(b'd') except TypeError: pass @@ -265,6 +269,62 @@ def test_access_parameter(self): self.assertRaises(TypeError, m.write_byte, 0) m.close() + @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') + def test_trackfd_parameter(self): + size = 64 + with open(TESTFN, "wb") as f: + f.write(b"a"*size) + for close_original_fd in True, False: + with self.subTest(close_original_fd=close_original_fd): + with open(TESTFN, "r+b") as f: + with mmap.mmap(f.fileno(), size, trackfd=False) as m: + if close_original_fd: + f.close() + self.assertEqual(len(m), size) + with self.assertRaises(OSError) as err_cm: + m.size() + self.assertEqual(err_cm.exception.errno, errno.EBADF) + with self.assertRaises(ValueError): + m.resize(size * 2) + with self.assertRaises(ValueError): + m.resize(size // 2) + self.assertEqual(m.closed, False) + + # Smoke-test other API + m.write_byte(ord('X')) + m[2] = ord('Y') + m.flush() + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(4), b'XaYa') + self.assertEqual(m.tell(), 1) + m.seek(0) + self.assertEqual(m.tell(), 0) + self.assertEqual(m.read_byte(), ord('X')) + + self.assertEqual(m.closed, True) + self.assertEqual(os.stat(TESTFN).st_size, size) + + @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') + def test_trackfd_neg1(self): + size = 64 + with mmap.mmap(-1, size, trackfd=False) as m: + with self.assertRaises(OSError): + m.size() + with self.assertRaises(ValueError): + m.resize(size // 2) + self.assertEqual(len(m), size) + m[0] = ord('a') + assert m[0] == ord('a') + + @unittest.skipIf(os.name != 'nt', 'trackfd only fails on Windows') + def test_no_trackfd_parameter_on_windows(self): + # 'trackffd' is an invalid keyword argument for this function + size = 64 + with self.assertRaises(TypeError): + mmap.mmap(-1, size, trackfd=True) + with self.assertRaises(TypeError): + mmap.mmap(-1, size, trackfd=False) + def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) @@ -671,14 +731,16 @@ def test_tagname(self): m2.close() m1.close() + with self.assertRaisesRegex(TypeError, 'tagname'): + mmap.mmap(-1, 8, tagname=1) + @cpython_only @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_sizeof(self): m1 = mmap.mmap(-1, 100) tagname = random_tagname() m2 = mmap.mmap(-1, 100, tagname=tagname) - self.assertEqual(sys.getsizeof(m2), - sys.getsizeof(m1) + len(tagname) + 1) + self.assertGreater(sys.getsizeof(m2), sys.getsizeof(m1)) @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_crasher_on_windows(self): @@ -777,7 +839,7 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith('linux'): + if sys.platform.startswith(('linux', 'android')): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) @@ -998,6 +1060,81 @@ def __exit__(self, exc_type, exc_value, traceback): with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): m.write_byte(X()) + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build') + def test_access_violations(self): + from test.support.os_helper import TESTFN + + code = textwrap.dedent(""" + import faulthandler + import mmap + import os + import sys + from contextlib import suppress + + # Prevent logging access violations to stderr. + faulthandler.disable() + + PAGESIZE = mmap.PAGESIZE + PAGE_NOACCESS = 0x01 + + with open(sys.argv[1], 'bw+') as f: + f.write(b'A'* PAGESIZE) + f.flush() + + m = mmap.mmap(f.fileno(), PAGESIZE) + m._protect(PAGE_NOACCESS, 0, PAGESIZE) + with suppress(OSError): + m.read(PAGESIZE) + assert False, 'mmap.read() did not raise' + with suppress(OSError): + m.read_byte() + assert False, 'mmap.read_byte() did not raise' + with suppress(OSError): + m.readline() + assert False, 'mmap.readline() did not raise' + with suppress(OSError): + m.write(b'A'* PAGESIZE) + assert False, 'mmap.write() did not raise' + with suppress(OSError): + m.write_byte(0) + assert False, 'mmap.write_byte() did not raise' + with suppress(OSError): + m[0] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10:2] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0] = 1 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10] = b'A'* 10 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10:2] = b'A'* 5 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m.move(0, 10, 1) + assert False, 'mmap.move() did not raise' + with suppress(OSError): + list(m) # test mmap_item + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m.find(b'A') + assert False, 'mmap.find() did not raise' + with suppress(OSError): + m.rfind(b'A') + assert False, 'mmap.rfind() did not raise' + """) + rt, stdout, stderr = assert_python_ok("-c", code, TESTFN) + self.assertEqual(stdout.strip(), b'') + self.assertEqual(stderr.strip(), b'') + + class LargeMmapTests(unittest.TestCase): def setUp(self): @@ -1007,7 +1144,7 @@ def tearDown(self): unlink(TESTFN) def _make_test_file(self, num_zeroes, tail): - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + if sys.platform[:3] == 'win' or is_apple: requires('largefile', 'test requires %s bytes and a long time to run' % str(0x180000000)) f = open(TESTFN, 'w+b') diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 2524e6c87cb459..56edd0c637f376 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -1,8 +1,10 @@ # Test the module type +import importlib.machinery import unittest import weakref from test.support import gc_collect from test.support import import_helper +from test.support import suppress_immortalization from test.support.script_helper import assert_python_ok import sys @@ -29,7 +31,7 @@ def test_uninitialized(self): self.fail("__name__ = %s" % repr(s)) except AttributeError: pass - self.assertEqual(foo.__doc__, ModuleType.__doc__) + self.assertEqual(foo.__doc__, ModuleType.__doc__ or '') def test_uninitialized_missing_getattr(self): # Issue 8297 @@ -102,6 +104,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) + @suppress_immortalization() def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") @@ -117,6 +120,7 @@ def __del__(self): gc_collect() self.assertEqual(destroyed, [1]) + @suppress_immortalization() def test_weakref(self): m = ModuleType("foo") wr = weakref.ref(m) @@ -264,9 +268,38 @@ def test_module_repr_source(self): self.assertEqual(r[-len(ends_with):], ends_with, '{!r} does not end with {!r}'.format(r, ends_with)) + def test_module_repr_with_namespace_package(self): + m = ModuleType('foo') + loader = importlib.machinery.NamespaceLoader('foo', ['bar'], 'baz') + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + self.assertEqual(repr(m), "") + + def test_module_repr_with_namespace_package_and_custom_loader(self): + m = ModuleType('foo') + loader = BareLoader() + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + + def test_module_repr_with_fake_namespace_package(self): + m = ModuleType('foo') + loader = BareLoader() + loader._path = ['spam'] + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + def test_module_finalization_at_shutdown(self): # Module globals and builtins should still be available during shutdown - rc, out, err = assert_python_ok("-c", "from test import final_a") + rc, out, err = assert_python_ok("-c", "from test.test_module import final_a") self.assertFalse(err) lines = out.splitlines() self.assertEqual(set(lines), { @@ -327,6 +360,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/final_a.py b/Lib/test/test_module/final_a.py similarity index 79% rename from Lib/test/final_a.py rename to Lib/test/test_module/final_a.py index 390ee8895a8a9e..a983f3111248e0 100644 --- a/Lib/test/final_a.py +++ b/Lib/test/test_module/final_a.py @@ -3,7 +3,7 @@ """ import shutil -import test.final_b +import test.test_module.final_b x = 'a' @@ -11,7 +11,7 @@ class C: def __del__(self): # Inspect module globals and builtins print("x =", x) - print("final_b.x =", test.final_b.x) + print("final_b.x =", test.test_module.final_b.x) print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) print("len =", getattr(len, '__name__', None)) diff --git a/Lib/test/final_b.py b/Lib/test/test_module/final_b.py similarity index 79% rename from Lib/test/final_b.py rename to Lib/test/test_module/final_b.py index 7228d82b880156..f3e8d5594904f2 100644 --- a/Lib/test/final_b.py +++ b/Lib/test/test_module/final_b.py @@ -3,7 +3,7 @@ """ import shutil -import test.final_a +import test.test_module.final_a x = 'b' @@ -11,7 +11,7 @@ class C: def __del__(self): # Inspect module globals and builtins print("x =", x) - print("final_a.x =", test.final_a.x) + print("final_a.x =", test.test_module.final_a.x) print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) print("len =", getattr(len, '__name__', None)) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 3b8ecb765c23f0..a07be306986b43 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,12 +3,18 @@ import collections import dis import functools +import math import operator import sys import textwrap import types import unittest -import asyncio + +import test.support +from test.support import requires_specialization, script_helper +from test.support.import_helper import import_module + +_testcapi = test.support.import_helper.import_module("_testcapi") PAIR = (0,1) @@ -35,6 +41,9 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + class MonitoringBasicTest(unittest.TestCase): def test_has_objects(self): @@ -529,8 +538,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno - self.assertEqual(events, [start+7, 16, start+8]) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -547,8 +556,13 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 23, 24, 23, 24, 23, start+8]) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -569,8 +583,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno - expected = [start+10, 16, start+11] + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: @@ -641,6 +655,17 @@ def func2(): self.check_lines(func2, [1,2,3,4,5,6]) + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + class TestDisable(MonitoringTestBase, unittest.TestCase): def gen(self, cond): @@ -750,7 +775,7 @@ class UnwindRecorder(ExceptionRecorder): event_type = E.PY_UNWIND def __call__(self, code, offset, exc): - self.events.append(("unwind", type(exc))) + self.events.append(("unwind", type(exc), code.co_name)) class ExceptionHandledRecorder(ExceptionRecorder): @@ -766,8 +791,27 @@ class ThrowRecorder(ExceptionRecorder): def __call__(self, code, offset, exc): self.events.append(("throw", type(exc))) -class ExceptionMonitoringTest(CheckEvents): +class CallRecorder: + + event_type = E.CALL + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, val): + self.events.append(("return", code.co_name, val)) + +class ExceptionMonitoringTest(CheckEvents): exception_recorders = ( ExceptionRecorder, @@ -788,6 +832,9 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) + # gh-116090: This test doesn't really require specialization, but running + # it without specialization exposes a monitoring bug. + @requires_specialization def test_implicit_stop_iteration(self): def gen(): @@ -936,26 +983,49 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) -class LineRecorder: + @requires_specialization + def test_no_unwind_for_shim_frame(self): - event_type = E.LINE + class B: + def __init__(self): + raise ValueError() + def f(): + try: + return B() + except ValueError: + pass - def __init__(self, events): - self.events = events + for _ in range(100): + f() + recorders = ( + ReturnRecorder, + UnwindRecorder + ) + events = self.get_events(f, TEST_TOOL, recorders) + adaptive_insts = dis.get_instructions(f, adaptive=True) + self.assertIn( + "CALL_ALLOC_AND_ENTER_INIT", + [i.opname for i in adaptive_insts] + ) + #There should be only one unwind event + expected = [ + ('unwind', ValueError, '__init__'), + ('return', 'f', None), + ] - def __call__(self, code, line): - self.events.append(("line", code.co_name, line - code.co_firstlineno)) + self.assertEqual(events, expected) -class CallRecorder: +class LineRecorder: + + event_type = E.LINE - event_type = E.CALL def __init__(self, events): self.events = events - def __call__(self, code, offset, func, arg): - self.events.append(("call", func.__name__, arg)) + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) class CEventRecorder: @@ -1351,15 +1421,6 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" -class ReturnRecorder: - - event_type = E.PY_RETURN - - def __init__(self, events): - self.events = events - - def __call__(self, code, offset, val): - self.events.append(("return", val)) JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder @@ -1378,15 +1439,16 @@ def func(): x = 4 else: x = 6 + 7 self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('jump', 'func', 6, 2), ('branch', 'func', 2, 2), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('jump', 'func', 4, 2), - ('branch', 'func', 2, 2)]) + ('branch', 'func', 2, 7)]) self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ ('line', 'get_events', 10), @@ -1394,17 +1456,18 @@ def func(): ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('line', 'func', 6), ('jump', 'func', 6, 2), ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('line', 'func', 4), ('jump', 'func', 4, 2), ('line', 'func', 2), - ('branch', 'func', 2, 2), + ('branch', 'func', 2, 7), + ('line', 'func', 7), ('line', 'get_events', 11)]) def test_except_star(self): @@ -1432,9 +1495,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1447,11 +1509,10 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('return', None), - ('jump', 'func', 5, 5), - ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), - ('return', None), + ('return', 'meth', None), + ('jump', 'func', 5, '[offset=118]'), + ('branch', 'func', '[offset=122]', '[offset=126]'), + ('return', 'func', None), ('line', 'get_events', 11)]) class TestLoadSuperAttr(CheckEvents): @@ -1761,20 +1822,40 @@ def test_gh108976(self): sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) sys.monitoring.set_events(0, 0) + def test_call_function_ex(self): + def f(a=1, b=2): + return a + b + args = (1, 2) + empty_args = [] + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + f(*empty_args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): - import _testinternalcapi - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() - _testinternalcapi.set_optimizer(opt) + _testinternalcapi = import_module("_testinternalcapi") + if hasattr(_testinternalcapi, "get_optimizer"): + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() def tearDown(self): - import _testinternalcapi super(TestOptimizer, self).tearDown() - _testinternalcapi.set_optimizer(self.old_opt) + import _testinternalcapi + if hasattr(_testinternalcapi, "get_optimizer"): + _testinternalcapi.set_optimizer(self.old_opt) def test_for_loop(self): def test_func(x): @@ -1788,3 +1869,233 @@ def test_func(x): test_func(1000) sys.monitoring.set_local_events(TEST_TOOL, code, 0) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + +class TestTier2Optimizer(CheckEvents): + + def test_monitoring_already_opimized_loop(self): + def test_func(recorder): + set_events = sys.monitoring.set_events + line = E.LINE + i = 0 + for i in range(551): + # Turn on events without branching once i reaches 500. + set_events(TEST_TOOL, line * int(i >= 500)) + pass + pass + pass + + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = LineRecorder(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + try: + test_func(recorder) + finally: + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertGreater(len(events), 250) + +class TestMonitoringAtShutdown(unittest.TestCase): + + def test_monitoring_live_at_shutdown(self): + # gh-115832: An object destructor running during the final GC of + # interpreter shutdown triggered an infinite loop in the + # instrumentation code. + script = test.support.findfile("_test_monitoring_shutdown.py") + script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), + ] + + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] + + + def check_event_count(self, event, func, args, expected, callback_raises=None): + class Counter: + def __init__(self, callback_raises): + self.callback_raises = callback_raises + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc + + try: + counter = Counter(callback_raises) + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if event not in self.EXPECT_RAISED_EXCEPTION: + continue + assert args and isinstance(args[-1], BaseException) + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 6451df14696933..1b55f1e70b32f5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -12,6 +12,7 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN +from test.support.import_helper import import_module ALL_CJKENCODINGS = [ # _codecs_cn @@ -212,7 +213,7 @@ def test_issue5640(self): @support.cpython_only def test_subinterp(self): # bpo-42846: Test a CJK codec in a subinterpreter - import _testcapi + _testcapi = import_module("_testcapi") encoding = 'cp932' text = "Python ã®é–‹ç™ºã¯ã€1990 å¹´ã”ã‚ã‹ã‚‰é–‹å§‹ã•ã‚Œã¦ã„ã¾ã™ã€‚" code = textwrap.dedent(""" @@ -303,7 +304,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) class Test_StreamReader(unittest.TestCase): def test_bug1728403(self): diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py index aa1fff50b28f5f..b35e82879d7fe2 100644 --- a/Lib/test/test_multiprocessing_fork/__init__.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -12,5 +12,8 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 7b2fa844827ae9..cf44080670dc2e 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,82 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_rebinding_dict_comprehension_iteration_variable(self): + cases = [ + ("Key reuse", 'i', "{(i := 0): 1 for i in range(5)}"), + ("Value reuse", 'i', "{1: (i := 0) for i in range(5)}"), + ("Both reuse", 'i', "{(i := 0): (i := 0) for i in range(5)}"), + ("Nested reuse", 'j', "{{(j := 0): 1 for i in range(5)} for j in range(5)}"), + ("Reuse inner loop target", 'j', "{(j := 0): 1 for i in range(5) for j in range(5)}"), + ("Unpacking key reuse", 'i', "{(i := 0): 1 for i, j in {(0, 1)}}"), + ("Unpacking value reuse", 'i', "{1: (i := 0) for i, j in {(0, 1)}}"), + ("Reuse in loop condition", 'i', "{i+1: 1 for i in range(5) if (i := 0)}"), + ("Unreachable reuse", 'i', "{(False or (i:=0)): 1 for i in range(5)}"), + ("Unreachable nested reuse", 'i', + "{i: j for i in range(5) for j in range(5) if True or (i:=10)}"), + # Regression tests from https://github.com/python/cpython/issues/87447 + ("Complex expression: a", "a", + "{(a := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ("Complex expression: b", "b", + "{(b := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ] + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_rebinding_dict_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j in range(5)}"), + ("Inner unpacking reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j, k in {(0, 1)}}"), + ] + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_dict_comprehension_iterable_expression(self): + cases = [ + ("Top level", "{i: 1 for i in (i := range(5))}"), + ("Inside tuple", "{i: 1 for i in (2, 3, i := range(5))}"), + ("Inside list", "{i: 1 for i in [2, 3, i := range(5)]}"), + ("Different name", "{i: 1 for i in (j := range(5))}"), + ("Lambda expression", "{i: 1 for i in (lambda:(j := range(5)))()}"), + ("Inner loop", "{i: 1 for i in range(5) for j in (i := range(5))}"), + ("Nested comprehension", "{i: 1 for i in {j: 2 for j in (k := range(5))}}"), + ("Nested comprehension condition", "{i: 1 for i in {j: 2 for j in range(5) if (j := True)}}"), + ("Nested comprehension body", "{i: 1 for i in {(j := True) for j in range(5)}}"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_mangled_class_variables(self): + code = """class Foo: + def bar(self): + [[(__x:=2) for _ in range(2)] for __x in range(2)] + """ + + with self.assertRaisesRegex(SyntaxError, + "assignment expression cannot rebind comprehension iteration variable '__x'"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): @@ -351,7 +427,7 @@ def test_named_expression_assignment_09(self): def test_named_expression_assignment_10(self): if (match := 10) == 10: - pass + self.assertEqual(match, 10) else: self.fail("variable was not assigned using named expression") def test_named_expression_assignment_11(self): @@ -393,7 +469,7 @@ def test_named_expression_assignment_14(self): def test_named_expression_assignment_15(self): while a := False: - pass # This will not run + self.fail("While body executed") # This will not run self.assertEqual(a, False) @@ -674,6 +750,18 @@ def test_named_expression_scope_in_genexp(self): for idx, elem in enumerate(genexp): self.assertEqual(elem, b[idx] + a) + def test_named_expression_scope_mangled_names(self): + class Foo: + def f(self_): + global __x1 + __x1 = 0 + [_Foo__x1 := 1 for a in [2]] + self.assertEqual(__x1, 1) + [__x1 := 2 for a in [3]] + self.assertEqual(__x1, 2) + + Foo().f() + self.assertEqual(_Foo__x1, 2) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 3e710d1c6dabe4..64cbfaaaaa0690 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -227,10 +227,18 @@ def test_split(self): tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) def test_isabs(self): + tester('ntpath.isabs("foo\\bar")', 0) + tester('ntpath.isabs("foo/bar")', 0) tester('ntpath.isabs("c:\\")', 1) + tester('ntpath.isabs("c:\\foo\\bar")', 1) + tester('ntpath.isabs("c:/foo/bar")', 1) tester('ntpath.isabs("\\\\conky\\mountpoint\\")', 1) - tester('ntpath.isabs("\\foo")', 1) - tester('ntpath.isabs("\\foo\\bar")', 1) + + # gh-44626: paths with only a drive or root are not absolute. + tester('ntpath.isabs("\\foo\\bar")', 0) + tester('ntpath.isabs("/foo/bar")', 0) + tester('ntpath.isabs("c:foo\\bar")', 0) + tester('ntpath.isabs("c:foo/bar")', 0) # gh-96290: normal UNC paths and device paths without trailing backslashes tester('ntpath.isabs("\\\\conky\\mountpoint")', 1) @@ -256,6 +264,7 @@ def test_join(self): tester('ntpath.join("a", "b", "c")', 'a\\b\\c') tester('ntpath.join("a\\", "b", "c")', 'a\\b\\c') tester('ntpath.join("a", "b\\", "c")', 'a\\b\\c') + tester('ntpath.join("a", "b", "c\\")', 'a\\b\\c\\') tester('ntpath.join("a", "b", "\\c")', '\\c') tester('ntpath.join("d:\\", "\\pleep")', 'd:\\pleep') tester('ntpath.join("d:\\", "a", "b")', 'd:\\a\\b') @@ -313,6 +322,16 @@ def test_join(self): tester("ntpath.join('\\\\computer\\', 'share')", '\\\\computer\\share') tester("ntpath.join('\\\\computer\\share\\', 'a')", '\\\\computer\\share\\a') tester("ntpath.join('\\\\computer\\share\\a\\', 'b')", '\\\\computer\\share\\a\\b') + # Second part is anchored, so that the first part is ignored. + tester("ntpath.join('a', 'Z:b', 'c')", 'Z:b\\c') + tester("ntpath.join('a', 'Z:\\b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('a', '\\\\b\\c', 'd')", '\\\\b\\c\\d') + # Second part has a root but not drive. + tester("ntpath.join('a', '\\b', 'c')", '\\b\\c') + tester("ntpath.join('Z:/a', '/b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('//?/Z:/a', '/b', 'c')", '\\\\?\\Z:\\b\\c') + tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') + tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') @@ -355,6 +374,7 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) @@ -847,43 +867,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) - + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) + + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -897,20 +921,16 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") def test_sameopenfile(self): @@ -962,6 +982,62 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_isreserved(self): + self.assertFalse(ntpath.isreserved('')) + self.assertFalse(ntpath.isreserved('.')) + self.assertFalse(ntpath.isreserved('..')) + self.assertFalse(ntpath.isreserved('/')) + self.assertFalse(ntpath.isreserved('/foo/bar')) + # A name that ends with a space or dot is reserved. + self.assertTrue(ntpath.isreserved('foo.')) + self.assertTrue(ntpath.isreserved('foo ')) + # ASCII control characters are reserved. + self.assertTrue(ntpath.isreserved('\foo')) + # Wildcard characters, colon, and pipe are reserved. + self.assertTrue(ntpath.isreserved('foo*bar')) + self.assertTrue(ntpath.isreserved('foo?bar')) + self.assertTrue(ntpath.isreserved('foo"bar')) + self.assertTrue(ntpath.isreserved('foobar')) + self.assertTrue(ntpath.isreserved('foo:bar')) + self.assertTrue(ntpath.isreserved('foo|bar')) + # Case-insensitive DOS-device names are reserved. + self.assertTrue(ntpath.isreserved('nul')) + self.assertTrue(ntpath.isreserved('aux')) + self.assertTrue(ntpath.isreserved('prn')) + self.assertTrue(ntpath.isreserved('con')) + self.assertTrue(ntpath.isreserved('conin$')) + self.assertTrue(ntpath.isreserved('conout$')) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertTrue(ntpath.isreserved('COM1')) + self.assertTrue(ntpath.isreserved('LPT9')) + self.assertTrue(ntpath.isreserved('com\xb9')) + self.assertTrue(ntpath.isreserved('com\xb2')) + self.assertTrue(ntpath.isreserved('lpt\xb3')) + # DOS-device name matching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertTrue(ntpath.isreserved('NUL.txt')) + self.assertTrue(ntpath.isreserved('PRN ')) + self.assertTrue(ntpath.isreserved('AUX .txt')) + self.assertTrue(ntpath.isreserved('COM1:bar')) + self.assertTrue(ntpath.isreserved('LPT9 :bar')) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertFalse(ntpath.isreserved('bar.com9')) + self.assertFalse(ntpath.isreserved('bar.lpt9')) + # The entire path is checked, except for the drive. + self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL')) + self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz')) + self.assertFalse(ntpath.isreserved('//./NUL')) + # Bytes are supported. + self.assertFalse(ntpath.isreserved(b'')) + self.assertFalse(ntpath.isreserved(b'.')) + self.assertFalse(ntpath.isreserved(b'..')) + self.assertFalse(ntpath.isreserved(b'/')) + self.assertFalse(ntpath.isreserved(b'/foo/bar')) + self.assertTrue(ntpath.isreserved(b'foo.')) + self.assertTrue(ntpath.isreserved(b'nul')) + def assertEqualCI(self, s1, s2): """Assert that two strings are equal ignoring case differences.""" self.assertEqual(s1.lower(), s2.lower()) @@ -1019,6 +1095,27 @@ def test_isfile_driveletter(self): raise unittest.SkipTest('SystemDrive is not defined or malformed') self.assertFalse(os.path.isfile('\\\\.\\' + drive)) + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + @unittest.skipIf(sys.platform != 'win32', "windows only") def test_con_device(self): self.assertFalse(os.path.isfile(r"\\.\CON")) @@ -1032,14 +1129,22 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) self.assertFalse(inspect.isfunction(os.path.isfile)) self.assertTrue(os.path.islink is nt._path_islink) self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 2b2783d57be8f4..92a34113bc0383 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,17 +4,20 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail +from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): raise unittest.SkipTest('implementation detail specific to cpython') -import _testinternalcapi +_testinternalcapi = import_module("_testinternalcapi") def disabling_optimizer(func): def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) old_opt = _testinternalcapi.get_optimizer() _testinternalcapi.set_optimizer(None) try: @@ -506,6 +509,7 @@ def f(x, y): @threading_helper.requires_working_threading() +@requires_specialization class TestRacesDoNotCrash(unittest.TestCase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: @@ -1021,6 +1025,7 @@ def write(items): class C: pass +@requires_specialization class TestInstanceDict(unittest.TestCase): def setUp(self): @@ -1042,20 +1047,13 @@ def test_dict_materialization(self): c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) + self.assertEqual(c.__dict__, {"a":1, "b": 2}) def test_dict_dematerialization(self): c = C() c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) for _ in range(100): c.a self.assertEqual( @@ -1070,10 +1068,6 @@ def test_dict_dematerialization_multiple_refs(self): d = c.__dict__ for _ in range(100): c.a - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) self.assertIs(c.__dict__, d) def test_dict_dematerialization_copy(self): diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 1db738d228b1b9..f8eac8dc002636 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,6 +1,9 @@ import unittest +import inspect import pickle import sys +from decimal import Decimal +from fractions import Fraction from test import support from test.support import import_helper @@ -508,6 +511,44 @@ def __getitem__(self, other): return 5 # so that C is a sequence self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def test_iconcat_without_getitem(self): + operator = self.module + + msg = "'int' object can't be concatenated" + with self.assertRaisesRegex(TypeError, msg): + operator.iconcat(1, 0.5) + + def test_index(self): + operator = self.module + class X: + def __index__(self): + return 1 + + self.assertEqual(operator.index(X()), 1) + self.assertEqual(operator.index(0), 0) + self.assertEqual(operator.index(1), 1) + self.assertEqual(operator.index(2), 2) + with self.assertRaises((AttributeError, TypeError)): + operator.index(1.5) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Fraction(3, 7)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Decimal(1)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(None) + + def test_not_(self): + operator = self.module + class C: + def __bool__(self): + raise SyntaxError + self.assertRaises(TypeError, operator.not_) + self.assertRaises(SyntaxError, operator.not_, C()) + self.assertFalse(operator.not_(5)) + self.assertFalse(operator.not_([0])) + self.assertTrue(operator.not_(0)) + self.assertTrue(operator.not_([])) + def test_length_hint(self): operator = self.module class X(object): @@ -533,6 +574,13 @@ def __length_hint__(self): with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) + class Y: pass + + msg = "'str' object cannot be interpreted as an integer" + with self.assertRaisesRegex(TypeError, msg): + operator.length_hint(X(2), "abc") + self.assertEqual(operator.length_hint(Y(), 10), 10) + def test_call(self): operator = self.module @@ -555,6 +603,28 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py new file mode 100644 index 00000000000000..fac4d1a4ab44c4 --- /dev/null +++ b/Lib/test/test_optimizer.py @@ -0,0 +1,90 @@ +import unittest +import types +from test.support import import_helper + + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +class TestRareEventCounters(unittest.TestCase): + def setUp(self): + _testinternalcapi.reset_rare_event_counters() + + def test_set_class(self): + class A: + pass + class B: + pass + a = A() + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"] + a.__class__ = B + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_class"] + ) + + def test_set_bases(self): + class A: + pass + class B: + pass + class C(B): + pass + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"] + C.__bases__ = (A,) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_bases"] + ) + + def test_set_eval_frame_func(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + _testinternalcapi.set_eval_frame_record([]) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + ) + _testinternalcapi.set_eval_frame_default() + + def test_builtin_dict(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["builtin_dict"] + if isinstance(__builtins__, types.ModuleType): + builtins = __builtins__.__dict__ + else: + builtins = __builtins__ + builtins["FOO"] = 42 + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["builtin_dict"] + ) + del builtins["FOO"] + + def test_func_modification(self): + def func(x=0): + pass + + for attribute in ( + "__code__", + "__defaults__", + "__kwdefaults__" + ): + orig_counter = _testinternalcapi.get_rare_event_counters()["func_modification"] + setattr(func, attribute, getattr(func, attribute)) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["func_modification"] + ) + + +class TestOptimizerSymbols(unittest.TestCase): + + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4571b23dfe7c1a..06a0e81227188c 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -10,7 +10,7 @@ import weakref from collections.abc import MutableMapping from test import mapping_tests, support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization py_coll = import_helper.import_fresh_module('collections', @@ -667,6 +667,7 @@ def test_dict_update(self): dict.update(od, [('spam', 1)]) self.assertNotIn('NULL', repr(od)) + @suppress_immortalization() def test_reference_loop(self): # Issue 25935 OrderedDict = self.OrderedDict diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c23de4dbe2b636..f93937fb587386 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,6 +13,7 @@ import locale import os import pickle +import platform import select import selectors import shutil @@ -34,7 +35,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -56,8 +57,10 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: @@ -1150,9 +1153,12 @@ def test_putenv_unsetenv(self): def test_putenv_unsetenv_error(self): # Empty variable name is invalid. # "=" and null character are not allowed in a variable name. - for name in ('', '=name', 'na=me', 'name=', 'name\0', 'na\0me'): + for name in ('', '=name', 'na=me', 'name='): self.assertRaises((OSError, ValueError), os.putenv, name, "value") self.assertRaises((OSError, ValueError), os.unsetenv, name) + for name in ('name\0', 'na\0me'): + self.assertRaises(ValueError, os.putenv, name, "value") + self.assertRaises(ValueError, os.unsetenv, name) if sys.platform == "win32": # On Windows, an environment variable string ("name=value" string) @@ -1292,9 +1298,56 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_refresh(self): + # Test os.environ.refresh() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.environ.refresh() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.environ.refresh() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test os.environb.refresh() with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.environb.refresh() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test os.environb.refresh() with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.environb.refresh() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1329,14 +1382,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1346,13 +1399,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1448,6 +1501,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1469,6 +1527,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1493,7 +1618,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] @@ -1534,6 +1659,7 @@ def test_walk_above_recursion_limit(self): @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): @@ -1589,6 +1715,9 @@ def test_yields_correct_dir_fd(self): @unittest.skipIf( support.is_emscripten, "Cannot dup stdout on Emscripten" ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -1602,10 +1731,29 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_emscripten, "Cannot dup stdout on Emscripten" + ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): @@ -1728,6 +1876,19 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] + self.assertEqual( + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -1740,7 +1901,7 @@ def tearDown(self): os.removedirs(path) -@os_helper.skip_unless_working_chmod +@unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") class ChownFileTests(unittest.TestCase): @classmethod @@ -2192,12 +2353,15 @@ def test_chmod(self): class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir", "fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: locals()["test_"+f] = get_single(f) @@ -2211,8 +2375,16 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2271,14 +2443,18 @@ def test_fchown(self): support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): @@ -2475,8 +2651,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -3082,6 +3260,64 @@ def test_stat_unlink_race(self): except subprocess.TimeoutExpired: proc.terminate() + @support.requires_subprocess() + def test_stat_inaccessible_file(self): + filename = os_helper.TESTFN + ICACLS = os.path.expandvars(r"%SystemRoot%\System32\icacls.exe") + + with open(filename, "wb") as f: + f.write(b'Test data') + + stat1 = os.stat(filename) + + try: + # Remove all permissions from the file + subprocess.check_output([ICACLS, filename, "/inheritance:r"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as ex: + if support.verbose: + print(ICACLS, filename, "/inheritance:r", "failed.") + print(ex.stdout.decode("oem", "replace").rstrip()) + try: + os.unlink(filename) + except OSError: + pass + self.skipTest("Unable to create inaccessible file") + + def cleanup(): + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], + stderr=subprocess.STDOUT) + os.unlink(filename) + + self.addCleanup(cleanup) + + if support.verbose: + print("File:", filename) + print("stat with access:", stat1) + + # First test - we shouldn't raise here, because we still have access to + # the directory and can extract enough information from its metadata. + stat2 = os.stat(filename) + + if support.verbose: + print(" without access:", stat2) + + # We may not get st_dev/st_ino, so ensure those are 0 or match + self.assertIn(stat2.st_dev, (0, stat1.st_dev)) + self.assertIn(stat2.st_ino, (0, stat1.st_ino)) + + # st_mode and st_size should match (for a normal file, at least) + self.assertEqual(stat1.st_mode, stat2.st_mode) + self.assertEqual(stat1.st_size, stat2.st_size) + + # st_ctime and st_mtime should be the same + self.assertEqual(stat1.st_ctime, stat2.st_ctime) + self.assertEqual(stat1.st_mtime, stat2.st_mtime) + + # st_atime should be the same or later + self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) + @os_helper.skip_unless_symlink class NonLocalSymlinkTests(unittest.TestCase): @@ -3430,31 +3666,30 @@ class ProgramPriorityTests(unittest.TestCase): """Tests for os.getpriority() and os.setpriority().""" def test_set_get_priority(self): - base = os.getpriority(os.PRIO_PROCESS, os.getpid()) - os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) - try: - new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) - if base >= 19 and new_prio <= 19: - raise unittest.SkipTest("unable to reliably test setpriority " - "at current nice level of %s" % base) - else: - self.assertEqual(new_prio, base + 1) - finally: - try: - os.setpriority(os.PRIO_PROCESS, os.getpid(), base) - except OSError as err: - if err.errno != errno.EACCES: - raise + code = f"""if 1: + import os + os.setpriority(os.PRIO_PROCESS, os.getpid(), {base} + 1) + print(os.getpriority(os.PRIO_PROCESS, os.getpid())) + """ + + # Subprocess inherits the current process' priority. + _, out, _ = assert_python_ok("-c", code) + new_prio = int(out) + # nice value cap is 19 for linux and 20 for FreeBSD + if base >= 19 and new_prio <= base: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -3777,7 +4012,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") @@ -3786,6 +4026,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -3930,9 +4171,15 @@ def test_eventfd_select(self): @unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # Tolerate a difference of 1 ms - CLOCK_RES_NS = 1_000_000 - CLOCK_RES = CLOCK_RES_NS * 1e-9 + # 1 ms accuracy is reliably achievable on every platform except Android + # emulators, where we allow 10 ms (gh-108277). + if sys.platform == "android" and platform.android_ver().is_emulator: + CLOCK_RES_PLACES = 2 + else: + CLOCK_RES_PLACES = 3 + + CLOCK_RES = 10 ** -CLOCK_RES_PLACES + CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) def timerfd_create(self, *args, **kwargs): fd = os.timerfd_create(*args, **kwargs) @@ -3954,18 +4201,18 @@ def test_timerfd_initval(self): # 1st call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, 0.0, places=3) - self.assertAlmostEqual(next_expiration, 0.0, places=3) + self.assertAlmostEqual(interval2, 0.0, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, 0.0, places=self.CLOCK_RES_PLACES) # 2nd call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) def test_timerfd_non_blocking(self): fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) @@ -4019,8 +4266,8 @@ def test_timerfd_interval(self): # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) count = 3 t = time.perf_counter() @@ -4029,7 +4276,7 @@ def test_timerfd_interval(self): t = time.perf_counter() - t total_time = initial_expiration + interval * (count - 1) - self.assertGreater(t, total_time) + self.assertGreater(t, total_time - self.CLOCK_RES) # wait 3.5 time of interval time.sleep( (count+0.5) * interval) @@ -4051,8 +4298,8 @@ def test_timerfd_TFD_TIMER_ABSTIME(self): # timerfd_gettime # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, offset, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, offset, places=self.CLOCK_RES_PLACES) t = time.perf_counter() count_signaled = self.read_count_signaled(fd) @@ -4083,7 +4330,7 @@ def test_timerfd_select(self): t = time.perf_counter() - t total_time = initial_expiration + interval * (count - 1) - self.assertGreater(t, total_time) + self.assertGreater(t, total_time - self.CLOCK_RES) def check_timerfd_poll(self, nanoseconds): fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) @@ -4129,10 +4376,10 @@ def check_timerfd_poll(self, nanoseconds): total_time = initial_expiration_ns + interval_ns * (count - 1) if nanoseconds: dt = time.perf_counter_ns() - t - self.assertGreater(dt, total_time) + self.assertGreater(dt, total_time - self.CLOCK_RES_NS) else: dt = time.perf_counter() - t - self.assertGreater(dt, total_time / sec_to_nsec) + self.assertGreater(dt, total_time / sec_to_nsec - self.CLOCK_RES) selector.unregister(fd) def test_timerfd_poll(self): @@ -4189,7 +4436,7 @@ def test_timerfd_ns_interval(self): t = time.perf_counter_ns() - t total_time_ns = initial_expiration_ns + interval_ns * (count - 1) - self.assertGreater(t, total_time_ns) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) # wait 3.5 time of interval time.sleep( (count+0.5) * interval_ns / one_sec_in_nsec) @@ -4248,7 +4495,7 @@ def test_timerfd_ns_select(self): t = time.perf_counter_ns() - t total_time_ns = initial_expiration_ns + interval_ns * (count - 1) - self.assertGreater(t, total_time_ns) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) class OSErrorTests(unittest.TestCase): def setUp(self): @@ -4342,8 +4589,8 @@ def test_process_cpu_count(self): @unittest.skipUnless(hasattr(os, 'sched_setaffinity'), "don't have sched affinity support") def test_process_cpu_count_affinity(self): - ncpu = os.cpu_count() - if ncpu is None: + affinity1 = os.process_cpu_count() + if affinity1 is None: self.skipTest("Could not determine the number of CPUs") # Disable one CPU @@ -4356,8 +4603,8 @@ def test_process_cpu_count_affinity(self): os.sched_setaffinity(0, mask) # test process_cpu_count() - affinity = os.process_cpu_count() - self.assertEqual(affinity, ncpu - 1) + affinity2 = os.process_cpu_count() + self.assertEqual(affinity2, affinity1 - 1) # FD inheritance check is only useful for systems with process support. @@ -4474,13 +4721,105 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd + def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() + def test_pipe_spawnl(self): + # gh-77046: On Windows, os.pipe() file descriptors must be created with + # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to + # get (_osfile(fd) & _O_NOINHERIT), so use a functional test. + # + # Make sure that fd is not inherited by a child process created by + # os.spawnl(): get_osfhandle() and dup() must fail with EBADF. + + fd, fd2 = os.pipe() + self.addCleanup(os.close, fd) + self.addCleanup(os.close, fd2) + + code = textwrap.dedent(f""" + import errno + import os + import test.support + try: + import msvcrt + except ImportError: + msvcrt = None + + fd = {fd} + + with test.support.SuppressCrashReport(): + if msvcrt is not None: + try: + handle = msvcrt.get_osfhandle(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # get_osfhandle(fd) failed with EBADF as expected + else: + raise Exception("get_osfhandle() must fail") + + try: + fd3 = os.dup(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # os.dup(fd) failed with EBADF as expected + else: + os.close(fd3) + raise Exception("dup must fail") + """) + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(filename, "w") as fp: + print(code, file=fp, end="") + + executable = sys.executable + cmd = [executable, filename] + if os.name == "nt" and " " in cmd[0]: + cmd[0] = f'"{cmd[0]}"' + exitcode = os.spawnl(os.P_WAIT, executable, *cmd) + self.assertEqual(exitcode, 0) class PathTConverterTests(unittest.TestCase): @@ -4669,7 +5008,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): os.name == 'nt') def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") @@ -5077,10 +5416,14 @@ def test_fork(self): support.wait_process(pid, exitcode=0) """ assert_python_ok("-c", code) - assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + if support.Py_GIL_DISABLED: + assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug") + else: + assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings @@ -5107,20 +5450,21 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_pathlib/__init__.py b/Lib/test/test_pathlib/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_pathlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py new file mode 100644 index 00000000000000..1328a8695b0cca --- /dev/null +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -0,0 +1,1676 @@ +import io +import os +import sys +import errno +import ntpath +import pathlib +import pickle +import posixpath +import socket +import stat +import tempfile +import unittest +from unittest import mock +from urllib.request import pathname2url + +from test.support import import_helper +from test.support import is_emscripten, is_wasi +from test.support import infinite_recursion +from test.support import os_helper +from test.support.os_helper import TESTFN, FakePath +from test.test_pathlib import test_pathlib_abc +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks + +try: + import grp, pwd +except ImportError: + grp = pwd = None + + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + +# +# Tests for the pure classes. +# + +class PurePathTest(test_pathlib_abc.DummyPurePathTest): + cls = pathlib.PurePath + + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) + + # Keys are canonical paths, values are list of tuples of arguments + # supposed to produce equal paths. + equivalences = { + 'a/b': [ + ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), + ('a/b/',), ('a//b',), ('a//b//',), + # Empty components get removed. + ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), + ], + '/b/c/d': [ + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), + # Empty components get removed. + ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), + ], + } + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_concrete_parser(self): + if self.cls is pathlib.PurePosixPath: + expected = posixpath + elif self.cls is pathlib.PureWindowsPath: + expected = ntpath + else: + expected = os.path + p = self.cls('a') + self.assertIs(p.parser, expected) + + def test_different_parsers_unequal(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_parsers_unordered(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + + def test_constructor_nested(self): + P = self.cls + P(FakePath("a/b/c")) + self.assertEqual(P(P('a')), P('a')) + self.assertEqual(P(P('a'), 'b'), P('a/b')) + self.assertEqual(P(P('a'), P('b')), P('a/b')) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) + + def _check_parse_path(self, raw_path, *expected): + sep = self.parser.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.parser.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.parser.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') + # Special case for the empty path. + self._check_str('.', ('',)) + + def test_parts_interning(self): + P = self.cls + p = P('/usr/bin/foo') + q = P('/usr/local/bin') + # 'usr' + self.assertIs(p.parts[1], q.parts[1]) + # 'bin' + self.assertIs(p.parts[2], q.parts[3]) + + def test_join_nested(self): + P = self.cls + p = P('a/b').joinpath(P('c')) + self.assertEqual(p, P('a/b/c')) + + def test_div_nested(self): + P = self.cls + p = P('a/b') / P('c') + self.assertEqual(p, P('a/b/c')) + + def test_pickling_common(self): + P = self.cls + for pathstr in ('a', 'a/', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c', 'a/b/c/'): + with self.subTest(pathstr=pathstr): + p = P(pathstr) + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertIs(pp.__class__, p.__class__) + self.assertEqual(pp, p) + self.assertEqual(hash(pp), hash(p)) + self.assertEqual(str(pp), str(p)) + + def test_repr_common(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_fspath_common(self): + P = self.cls + p = P('a/b') + self._check_str(p.__fspath__(), ('a/b',)) + self._check_str(os.fspath(p), ('a/b',)) + + def test_bytes_exc_message(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaisesRegex(TypeError, message): + P(b'a', 'b') + with self.assertRaisesRegex(TypeError, message): + P('a', b'b') + + def test_as_bytes_common(self): + sep = os.fsencode(self.sep) + P = self.cls + self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + + def test_eq_common(self): + P = self.cls + self.assertEqual(P('a/b'), P('a/b')) + self.assertEqual(P('a/b'), P('a', 'b')) + self.assertNotEqual(P('a/b'), P('a')) + self.assertNotEqual(P('a/b'), P('/a/b')) + self.assertNotEqual(P('a/b'), P()) + self.assertNotEqual(P('/a/b'), P('/')) + self.assertNotEqual(P(), P('/')) + self.assertNotEqual(P(), "") + self.assertNotEqual(P(), {}) + self.assertNotEqual(P(), int) + + def test_equivalences(self, equivalences=None): + if equivalences is None: + equivalences = self.equivalences + for k, tuples in equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + + def test_ordering_common(self): + # Ordering is tuple-alike. + def assertLess(a, b): + self.assertLess(a, b) + self.assertGreater(b, a) + P = self.cls + a = P('a') + b = P('a/b') + c = P('abc') + d = P('b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + P = self.cls + a = P('/a') + b = P('/a/b') + c = P('/abc') + d = P('/b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + with self.assertRaises(TypeError): + P() < {} + + def test_as_uri_common(self): + P = self.cls + with self.assertRaises(ValueError): + P('a').as_uri() + with self.assertRaises(ValueError): + P().as_uri() + + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) + + def test_name_empty(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('.').name, '') + self.assertEqual(P('/a/b/.').name, 'b') + + def test_stem_empty(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '') + + def test_with_name_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_name, 'd.xml') + self.assertRaises(ValueError, P('.').with_name, 'd.xml') + self.assertRaises(ValueError, P('/').with_name, 'd.xml') + self.assertRaises(ValueError, P('a/b').with_name, '') + self.assertRaises(ValueError, P('a/b').with_name, '.') + + def test_with_stem_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_stem, 'd') + self.assertRaises(ValueError, P('.').with_stem, 'd') + self.assertRaises(ValueError, P('/').with_stem, 'd') + self.assertRaises(ValueError, P('a/b').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '.') + + def test_is_reserved_deprecated(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.is_reserved() + + def test_match_empty(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + + @needs_posix + def test_parse_path_posix(self): + check = self._check_parse_path + # Collapsing of excess leading slashes, except for the double-slash + # special case. + check('//a/b', '', '//', ['a', 'b']) + check('///a/b', '', '/', ['a', 'b']) + check('////a/b', '', '/', ['a', 'b']) + # Paths which look like NT paths aren't treated specially. + check('c:a', '', '', ['c:a',]) + check('c:\\a', '', '', ['c:\\a',]) + check('\\a', '', '', ['\\a',]) + + @needs_posix + def test_eq_posix(self): + P = self.cls + self.assertNotEqual(P('a/b'), P('A/b')) + self.assertEqual(P('/a'), P('///a')) + self.assertNotEqual(P('/a'), P('//a')) + + @needs_posix + def test_as_uri_posix(self): + P = self.cls + self.assertEqual(P('/').as_uri(), 'file:///') + self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') + self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') + + @needs_posix + def test_as_uri_non_ascii(self): + from urllib.parse import quote_from_bytes + P = self.cls + try: + os.fsencode('\xe9') + except UnicodeEncodeError: + self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") + self.assertEqual(P('/a/b\xe9').as_uri(), + 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) + + @needs_posix + def test_parse_windows_path(self): + P = self.cls + p = P('c:', 'a', 'b') + pp = P(pathlib.PureWindowsPath('c:\\a\\b')) + self.assertEqual(p, pp) + + windows_equivalences = { + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], + 'c:/a': [ + ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), + ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), + ], + '//a/b/': [ ('//a/b',) ], + '//a/b/c': [ + ('//a/b', 'c'), ('//a/b/', 'c'), + ], + } + + @needs_windows + def test_equivalences_windows(self): + self.test_equivalences(self.windows_equivalences) + + @needs_windows + def test_parse_path_windows(self): + check = self._check_parse_path + # First part is anchored. + check('c:', 'c:', '', []) + check('c:/', 'c:', '\\', []) + check('/', '', '\\', []) + check('c:a', 'c:', '', ['a']) + check('c:/a', 'c:', '\\', ['a']) + check('/a', '', '\\', ['a']) + # UNC paths. + check('//', '\\\\', '', []) + check('//a', '\\\\a', '', []) + check('//a/', '\\\\a\\', '', []) + check('//a/b', '\\\\a\\b', '\\', []) + check('//a/b/', '\\\\a\\b', '\\', []) + check('//a/b/c', '\\\\a\\b', '\\', ['c']) + # Collapsing and stripping excess slashes. + check('Z://b//c/d/', 'Z:', '\\', ['b', 'c', 'd']) + # UNC paths. + check('//b/c//d', '\\\\b\\c', '\\', ['d']) + # Extended paths. + check('//./c:', '\\\\.\\c:', '', []) + check('//?/c:/', '\\\\?\\c:', '\\', []) + check('//?/c:/a', '\\\\?\\c:', '\\', ['a']) + # Extended UNC paths (format is "\\?\UNC\server\share"). + check('//?', '\\\\?', '', []) + check('//?/', '\\\\?\\', '', []) + check('//?/UNC', '\\\\?\\UNC', '', []) + check('//?/UNC/', '\\\\?\\UNC\\', '', []) + check('//?/UNC/b', '\\\\?\\UNC\\b', '', []) + check('//?/UNC/b/', '\\\\?\\UNC\\b\\', '', []) + check('//?/UNC/b/c', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/d', '\\\\?\\UNC\\b\\c', '\\', ['d']) + # UNC device paths + check('//./BootPartition/', '\\\\.\\BootPartition', '\\', []) + check('//?/BootPartition/', '\\\\?\\BootPartition', '\\', []) + check('//./PhysicalDrive0', '\\\\.\\PhysicalDrive0', '', []) + check('//?/Volume{}/', '\\\\?\\Volume{}', '\\', []) + check('//./nul', '\\\\.\\nul', '', []) + # Paths to files with NTFS alternate data streams + check('./c:s', '', '', ['c:s']) + check('cc:s', '', '', ['cc:s']) + check('C:c:s', 'C:', '', ['c:s']) + check('C:/c:s', 'C:', '\\', ['c:s']) + check('D:a/c:b', 'D:', '', ['a', 'c:b']) + check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) + + @needs_windows + def test_eq_windows(self): + P = self.cls + self.assertEqual(P('c:a/b'), P('c:a/b')) + self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) + self.assertNotEqual(P('c:a/b'), P('d:a/b')) + self.assertNotEqual(P('c:a/b'), P('c:/a/b')) + self.assertNotEqual(P('/a/b'), P('c:/a/b')) + # Case-insensitivity. + self.assertEqual(P('a/B'), P('A/b')) + self.assertEqual(P('C:a/B'), P('c:A/b')) + self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) + self.assertEqual(P('\u0130'), P('i\u0307')) + + @needs_windows + def test_as_uri_windows(self): + P = self.cls + with self.assertRaises(ValueError): + P('/a/b').as_uri() + with self.assertRaises(ValueError): + P('c:a/b').as_uri() + self.assertEqual(P('c:/').as_uri(), 'file:///c:/') + self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') + self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') + self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') + self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') + self.assertEqual(P('//some/share/a/b.c').as_uri(), + 'file://some/share/a/b.c') + self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), + 'file://some/share/a/b%25%23c%C3%A9') + + @needs_windows + def test_ordering_windows(self): + # Case-insensitivity. + def assertOrderedEqual(a, b): + self.assertLessEqual(a, b) + self.assertGreaterEqual(b, a) + P = self.cls + p = P('c:A/b') + q = P('C:a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + p = P('//some/Share/A/b') + q = P('//Some/SHARE/a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + + +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath + + +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath + + +class PurePathSubclassTest(PurePathTest): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +# +# Tests for the concrete classes. +# + +class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): + """Tests for the FS-accessing functionalities of the Path classes.""" + cls = pathlib.Path + can_symlink = os_helper.can_symlink() + + def setUp(self): + super().setUp() + os.chmod(self.parser.join(self.base, 'dirE'), 0) + + def tearDown(self): + os.chmod(self.parser.join(self.base, 'dirE'), 0o777) + os_helper.rmtree(self.base) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + + def test_matches_pathbase_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + our_names.remove('is_reserved') # only present in PurePath + path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but PathBase.parser is + # posixpath, and so their docstrings differ. + continue + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib._abc.PathBase, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + def test_concrete_class(self): + if self.cls is pathlib.Path: + expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") + else: + self.assertRaises(pathlib.UnsupportedOperation, self.cls) + + def _test_cwd(self, p): + q = self.cls(os.getcwd()) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + def test_cwd(self): + p = self.cls.cwd() + self._test_cwd(p) + + def test_absolute_common(self): + P = self.cls + + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = self.base + + # Simple relative paths. + self.assertEqual(str(P().absolute()), self.base) + self.assertEqual(str(P('.').absolute()), self.base) + self.assertEqual(str(P('a').absolute()), os.path.join(self.base, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(self.base, 'a', 'b', 'c')) + + # Symlinks should not be resolved. + self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(self.base, 'linkB', 'fileB')) + self.assertEqual(str(P('brokenLink').absolute()), os.path.join(self.base, 'brokenLink')) + self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(self.base, 'brokenLinkLoop')) + + # '..' entries should be preserved and not normalised. + self.assertEqual(str(P('..').absolute()), os.path.join(self.base, '..')) + self.assertEqual(str(P('a', '..').absolute()), os.path.join(self.base, 'a', '..')) + self.assertEqual(str(P('..', 'b').absolute()), os.path.join(self.base, '..', 'b')) + + def _test_home(self, p): + q = self.cls(os.path.expanduser('~')) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + @unittest.skipIf( + pwd is None, reason="Test requires pwd module to get homedir." + ) + def test_home(self): + with os_helper.EnvironmentVarGuard() as env: + self._test_home(self.cls.home()) + + env.clear() + env['USERPROFILE'] = os.path.join(self.base, 'userprofile') + self._test_home(self.cls.home()) + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = os.path.join(self.base, 'home') + self._test_home(self.cls.home()) + + @unittest.skipIf(is_wasi, "WASI has no user accounts.") + def test_expanduser_common(self): + P = self.cls + p = P('~') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) + p = P('foo') + self.assertEqual(p.expanduser(), p) + p = P('/~') + self.assertEqual(p.expanduser(), p) + p = P('../~') + self.assertEqual(p.expanduser(), p) + p = P(P('').absolute().anchor) / '~' + self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) + + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P(self.base, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if self.can_symlink: + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + + def test_open_unbuffered(self): + p = self.cls(self.base) + with (p / 'fileA').open('rb', buffering=0) as f: + self.assertIsInstance(f, io.RawIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_copy_file_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'fileA' + if hasattr(os, 'chmod'): + os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source, stat.UF_NODUMP) + source_st = source.stat() + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + target_st = target.stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + + @needs_symlinks + def test_copy_link_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'linkA' + if hasattr(os, 'lchmod'): + os.lchmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(source, stat.UF_NODUMP) + source_st = source.lstat() + target = base / 'copyA' + source.copy(target, follow_symlinks=False, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + target_st = target.lstat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") + @unittest.skipIf(root_in_posix, "test fails with root privilege") + def test_copytree_no_read_permission(self): + base = self.cls(self.base) + source = base / 'dirE' + target = base / 'copyE' + self.assertRaises(PermissionError, source.copytree, target) + self.assertFalse(target.exists()) + errors = [] + source.copytree(target, on_error=errors.append) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], PermissionError) + self.assertFalse(target.exists()) + + def test_resolve_nonexist_relative_issue38671(self): + p = self.cls('non', 'exist') + + old_cwd = os.getcwd() + os.chdir(self.base) + try: + self.assertEqual(p.resolve(), self.cls(self.base, p)) + finally: + os.chdir(old_cwd) + + @os_helper.skip_unless_working_chmod + def test_chmod(self): + p = self.cls(self.base) / 'fileA' + mode = p.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + # Set writable bit. + new_mode = mode | 0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + + # On Windows, os.chmod does not follow symlinks (issue #15411) + @needs_posix + @os_helper.skip_unless_working_chmod + def test_chmod_follow_symlinks_true(self): + p = self.cls(self.base) / 'linkA' + q = p.resolve() + mode = q.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + # Set writable bit + new_mode = mode | 0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + + # XXX also need a test for lchmod. + + def _get_pw_name_or_skip_test(self, uid): + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + self.skipTest( + "user %d doesn't have an entry in the system database" % uid) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + def test_owner(self): + p = self.cls(self.base) / 'fileA' + expected_uid = p.stat().st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_name, p.owner()) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_owner_no_follow_symlinks(self): + all_users = [u.pw_uid for u in pwd.getpwall()] + if len(all_users) < 2: + self.skipTest("test needs more than one user") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + uid_1, uid_2 = all_users[:2] + os.chown(target, uid_1, -1) + os.chown(link, uid_2, -1, follow_symlinks=False) + + expected_uid = link.stat(follow_symlinks=False).st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_uid, uid_2) + self.assertEqual(expected_name, link.owner(follow_symlinks=False)) + + def _get_gr_name_or_skip_test(self, gid): + try: + return grp.getgrgid(gid).gr_name + except KeyError: + self.skipTest( + "group %d doesn't have an entry in the system database" % gid) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + def test_group(self): + p = self.cls(self.base) / 'fileA' + expected_gid = p.stat().st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_name, p.group()) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_group_no_follow_symlinks(self): + all_groups = [g.gr_gid for g in grp.getgrall()] + if len(all_groups) < 2: + self.skipTest("test needs more than one group") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + gid_1, gid_2 = all_groups[:2] + os.chown(target, -1, gid_1) + os.chown(link, -1, gid_2, follow_symlinks=False) + + expected_gid = link.stat(follow_symlinks=False).st_gid + expected_name = self._get_pw_name_or_skip_test(expected_gid) + + self.assertEqual(expected_gid, gid_2) + self.assertEqual(expected_name, link.group(follow_symlinks=False)) + + @os_helper.skip_unless_hardlink + def test_hardlink_to(self): + P = self.cls(self.base) + target = P / 'fileA' + size = target.stat().st_size + # linking to another path. + link = P / 'dirA' / 'fileAA' + link.hardlink_to(target) + self.assertEqual(link.stat().st_size, size) + self.assertTrue(os.path.samefile(target, link)) + self.assertTrue(target.exists()) + # Linking to a str of a relative path. + link2 = P / 'dirA' / 'fileAAA' + target2 = self.parser.join(TESTFN, 'fileA') + link2.hardlink_to(target2) + self.assertEqual(os.stat(target2).st_size, size) + self.assertTrue(link2.exists()) + + @unittest.skipIf(hasattr(os, "link"), "os.link() is present") + def test_hardlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.hardlink_to(p) + + def test_rename(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Renaming to another path. + q = P / 'dirA' / 'fileAA' + renamed_p = p.rename(q) + self.assertEqual(renamed_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Renaming to a str of a relative path. + r = self.parser.join(TESTFN, 'fileAAA') + renamed_q = q.rename(r) + self.assertEqual(renamed_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_replace(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Replacing a non-existing path. + q = P / 'dirA' / 'fileAA' + replaced_p = p.replace(q) + self.assertEqual(replaced_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Replacing another (existing) path. + r = self.parser.join(TESTFN, 'dirB', 'fileB') + replaced_q = q.replace(r) + self.assertEqual(replaced_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_touch_common(self): + P = self.cls(self.base) + p = P / 'newfileA' + self.assertFalse(p.exists()) + p.touch() + self.assertTrue(p.exists()) + st = p.stat() + old_mtime = st.st_mtime + old_mtime_ns = st.st_mtime_ns + # Rewind the mtime sufficiently far in the past to work around + # filesystem-specific timestamp granularity. + os.utime(str(p), (old_mtime - 10, old_mtime - 10)) + # The file mtime should be refreshed by calling touch() again. + p.touch() + st = p.stat() + self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) + self.assertGreaterEqual(st.st_mtime, old_mtime) + # Now with exist_ok=False. + p = P / 'newfileB' + self.assertFalse(p.exists()) + p.touch(mode=0o700, exist_ok=False) + self.assertTrue(p.exists()) + self.assertRaises(OSError, p.touch, exist_ok=False) + + def test_touch_nochange(self): + P = self.cls(self.base) + p = P / 'fileA' + p.touch() + with p.open('rb') as f: + self.assertEqual(f.read().strip(), b"this is file A") + + def test_mkdir(self): + P = self.cls(self.base) + p = P / 'newdirA' + self.assertFalse(p.exists()) + p.mkdir() + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_parents(self): + # Creating a chain of directories. + p = self.cls(self.base, 'newdirB', 'newdirC') + self.assertFalse(p.exists()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.ENOENT) + p.mkdir(parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + # Test `mode` arg. + mode = stat.S_IMODE(p.stat().st_mode) # Default mode. + p = self.cls(self.base, 'newdirD', 'newdirE') + p.mkdir(0o555, parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + if os.name != 'nt': + # The directory's permissions follow the mode argument. + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) + # The parent's permissions follow the default process settings. + self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) + + def test_mkdir_exist_ok(self): + p = self.cls(self.base, 'dirB') + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + def test_mkdir_exist_ok_with_parent(self): + p = self.cls(self.base, 'dirC') + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p = p / 'newdirC' + p.mkdir(parents=True) + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(parents=True, exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows. + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + + @needs_windows # XXX: not sure how to test this on POSIX. + def test_mkdir_with_unknown_drive(self): + for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': + p = self.cls(d + ':\\') + if not p.is_dir(): + break + else: + self.skipTest("cannot find a drive that doesn't exist") + with self.assertRaises(OSError): + (p / 'child' / 'path').mkdir(parents=True) + + def test_mkdir_with_child_file(self): + p = self.cls(self.base, 'dirB', 'fileB') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True, exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_no_parents_file(self): + p = self.cls(self.base, 'fileA') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(self.base, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + real_mkdir = os.mkdir + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + real_mkdir(path, mode) # From another process. + concurrently_created.add(path) + real_mkdir(path, mode) # Our real call. + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("os.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + + @needs_symlinks + def test_symlink_to(self): + P = self.cls(self.base) + target = P / 'fileA' + # Symlinking a path target. + link = P / 'dirA' / 'linkAA' + link.symlink_to(target) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + # Symlinking a str target. + link = P / 'dirA' / 'linkAAA' + link.symlink_to(str(target)) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertFalse(link.is_dir()) + # Symlinking to a directory. + target = P / 'dirB' + link = P / 'dirA' / 'linkAAAA' + link.symlink_to(target, target_is_directory=True) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertTrue(link.is_dir()) + self.assertTrue(list(link.iterdir())) + + @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") + def test_symlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.symlink_to(p) + + def test_is_junction(self): + P = self.cls(self.base) + + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_is_fifo_true(self): + P = self.cls(self.base, 'myfifo') + try: + os.mkfifo(str(P)) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + self.assertTrue(P.is_fifo()) + self.assertFalse(P.is_socket()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") + @unittest.skipIf( + is_emscripten, "Unix sockets are not implemented on Emscripten." + ) + @unittest.skipIf( + is_wasi, "Cannot create socket on WASI." + ) + def test_is_socket_true(self): + P = self.cls(self.base, 'mysock') + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.addCleanup(sock.close) + try: + sock.bind(str(P)) + except OSError as e: + if (isinstance(e, PermissionError) or + "AF_UNIX path too long" in str(e)): + self.skipTest("cannot bind Unix socket: " + str(e)) + self.assertTrue(P.is_socket()) + self.assertFalse(P.is_fifo()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + + def test_is_char_device_true(self): + # os.devnull should generally be a char device. + P = self.cls(os.devnull) + if not P.exists(): + self.skipTest("null device required") + self.assertTrue(P.is_char_device()) + self.assertFalse(P.is_block_device()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) + + def test_is_mount_root(self): + if os.name == 'nt': + R = self.cls('c:\\') + else: + R = self.cls('/') + self.assertTrue(R.is_mount()) + self.assertFalse((R / '\udfff').is_mount()) + + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): + self.cls(foo="bar") + + def setUpWalk(self): + super().setUpWalk() + sub21_path= self.sub2_path / "SUB21" + tmp5_path = sub21_path / "tmp3" + broken_link3_path = self.sub2_path / "broken_link3" + + os.makedirs(sub21_path) + tmp5_path.write_text("I am tmp5, blame test_pathlib.") + if self.can_symlink: + os.symlink(tmp5_path, broken_link3_path) + self.sub2_tree[2].append('broken_link3') + self.sub2_tree[2].sort() + if not is_emscripten: + # Emscripten fails with inaccessible directories. + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.sub2_tree[1].append('SUB21') + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + + def test_walk_bad_dir(self): + self.setUpWalk() + errors = [] + walk_it = self.walk_path.walk(on_error=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = root / dir1 + path1new = (root / dir1).with_suffix(".new") + path1.rename(path1new) + try: + roots = [r for r, _, _ in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(root / dir2, roots) + finally: + path1new.rename(path1) + + def test_walk_many_open_files(self): + depth = 30 + base = self.cls(self.base, 'deep') + path = self.cls(base, *(['d']*depth)) + path.mkdir(parents=True) + + iters = [base.walk(top_down=False) for _ in range(100)] + for i in range(depth + 1): + expected = (path, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path.parent + + iters = [base.walk(top_down=True) for _ in range(100)] + path = base + for i in range(depth + 1): + expected = (path, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path / 'd' + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + def test_glob_empty_pattern(self): + p = self.cls('') + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('.')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('./')) + + def test_glob_many_open_files(self): + depth = 30 + P = self.cls + p = base = P(self.base) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' + for it in iters: + self.assertEqual(next(it), p) + + def test_glob_above_recursion_limit(self): + recursion_limit = 50 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.glob('**/')) + + def test_glob_pathlike(self): + P = self.cls + p = P(self.base) + pattern = "dir*/file*" + expect = {p / "dirB/fileB", p / "dirC/fileC"} + self.assertEqual(expect, set(p.glob(P(pattern)))) + self.assertEqual(expect, set(p.glob(FakePath(pattern)))) + + @needs_symlinks + def test_glob_dot(self): + P = self.cls + with os_helper.change_cwd(P(self.base, "dirC")): + self.assertEqual( + set(P('.').glob('*')), {P("fileC"), P("novel.txt"), P("dirD")}) + self.assertEqual( + set(P('.').glob('**')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD"), P(".")}) + self.assertEqual( + set(P('.').glob('**/*')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD")}) + self.assertEqual( + set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + + def test_glob_inaccessible(self): + P = self.cls + p = P(self.base, "mydir1", "mydir2") + p.mkdir(parents=True) + p.parent.chmod(0) + self.assertEqual(set(p.glob('*')), set()) + + def test_rglob_pathlike(self): + P = self.cls + p = P(self.base, "dirC") + pattern = "**/file*" + expect = {p / "fileC", p / "dirD/fileD"} + self.assertEqual(expect, set(p.rglob(P(pattern)))) + self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + + @needs_posix + def test_absolute_posix(self): + P = self.cls + self.assertEqual(str(P('/').absolute()), '/') + self.assertEqual(str(P('/a').absolute()), '/a') + self.assertEqual(str(P('/a/b').absolute()), '/a/b') + + # '//'-prefixed absolute path (supported by POSIX). + self.assertEqual(str(P('//').absolute()), '//') + self.assertEqual(str(P('//a').absolute()), '//a') + self.assertEqual(str(P('//a/b').absolute()), '//a/b') + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @needs_posix + def test_open_mode(self): + old_mask = os.umask(0) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + with (p / 'new_file').open('wb'): + pass + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) + os.umask(0o022) + with (p / 'other_new_file').open('wb'): + pass + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + + @needs_posix + def test_resolve_root(self): + current_directory = os.getcwd() + try: + os.chdir('/') + p = self.cls('spam') + self.assertEqual(str(p.resolve()), '/spam') + finally: + os.chdir(current_directory) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @needs_posix + def test_touch_mode(self): + old_mask = os.umask(0) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + (p / 'new_file').touch() + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) + os.umask(0o022) + (p / 'other_new_file').touch() + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + (p / 'masked_new_file').touch(mode=0o750) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) + + @unittest.skipUnless(hasattr(pwd, 'getpwall'), + 'pwd module does not expose getpwall()') + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + @needs_posix + def test_expanduser_posix(self): + P = self.cls + import_helper.import_module('pwd') + import pwd + pwdent = pwd.getpwuid(os.getuid()) + username = pwdent.pw_name + userhome = pwdent.pw_dir.rstrip('/') or '/' + # Find arbitrary different user (if exists). + for pwdent in pwd.getpwall(): + othername = pwdent.pw_name + otherhome = pwdent.pw_dir.rstrip('/') + if othername != username and otherhome: + break + else: + othername = username + otherhome = userhome + + fakename = 'fakeuser' + # This user can theoretically exist on a test runner. Create unique name: + try: + while pwd.getpwnam(fakename): + fakename += '1' + except KeyError: + pass # Non-existent name found + + p1 = P('~/Documents') + p2 = P(f'~{username}/Documents') + p3 = P(f'~{othername}/Documents') + p4 = P(f'../~{username}/Documents') + p5 = P(f'/~{username}/Documents') + p6 = P('') + p7 = P(f'~{fakename}/Documents') + + with os_helper.EnvironmentVarGuard() as env: + env.pop('HOME', None) + + self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + env['HOME'] = '/tmp' + self.assertEqual(p1.expanduser(), P('/tmp/Documents')) + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + @unittest.skipIf(sys.platform != "darwin", + "Bad file descriptor in /dev/fd affects only macOS") + @needs_posix + def test_handling_bad_descriptor(self): + try: + file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] + if not file_descriptors: + self.skipTest("no file descriptors - issue was not reproduced") + # Checking all file descriptors because there is no guarantee + # which one will fail. + for f in file_descriptors: + f.exists() + f.is_dir() + f.is_file() + f.is_symlink() + f.is_block_device() + f.is_char_device() + f.is_fifo() + f.is_socket() + except OSError as e: + if e.errno == errno.EBADF: + self.fail("Bad file descriptor not handled.") + raise + + @needs_posix + def test_from_uri_posix(self): + P = self.cls + self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, '/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + @needs_posix + def test_from_uri_pathname2url_posix(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) + self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) + + @needs_windows + def test_absolute_windows(self): + P = self.cls + + # Simple absolute paths. + self.assertEqual(str(P('c:\\').absolute()), 'c:\\') + self.assertEqual(str(P('c:\\a').absolute()), 'c:\\a') + self.assertEqual(str(P('c:\\a\\b').absolute()), 'c:\\a\\b') + + # UNC absolute paths. + share = '\\\\server\\share\\' + self.assertEqual(str(P(share).absolute()), share) + self.assertEqual(str(P(share + 'a').absolute()), share + 'a') + self.assertEqual(str(P(share + 'a\\b').absolute()), share + 'a\\b') + + # UNC relative paths. + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = share + + self.assertEqual(str(P().absolute()), share) + self.assertEqual(str(P('.').absolute()), share) + self.assertEqual(str(P('a').absolute()), os.path.join(share, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), + os.path.join(share, 'a', 'b', 'c')) + + drive = os.path.splitdrive(self.base)[0] + with os_helper.change_cwd(self.base): + # Relative path with root + self.assertEqual(str(P('\\').absolute()), drive + '\\') + self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') + + # Relative path on current drive + self.assertEqual(str(P(drive).absolute()), self.base) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(self.base, 'foo')) + + with os_helper.subst_drive(self.base) as other_drive: + # Set the working directory on the substitute drive + saved_cwd = os.getcwd() + other_cwd = f'{other_drive}\\dirA' + os.chdir(other_cwd) + os.chdir(saved_cwd) + + # Relative path on another drive + self.assertEqual(str(P(other_drive).absolute()), other_cwd) + self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') + + @needs_windows + def test_expanduser_windows(self): + P = self.cls + with os_helper.EnvironmentVarGuard() as env: + env.pop('HOME', None) + env.pop('USERPROFILE', None) + env.pop('HOMEPATH', None) + env.pop('HOMEDRIVE', None) + env['USERNAME'] = 'alice' + + # test that the path returns unchanged + p1 = P('~/My Documents') + p2 = P('~alice/My Documents') + p3 = P('~bob/My Documents') + p4 = P('/~/My Documents') + p5 = P('d:~/My Documents') + p6 = P('') + self.assertRaises(RuntimeError, p1.expanduser) + self.assertRaises(RuntimeError, p2.expanduser) + self.assertRaises(RuntimeError, p3.expanduser) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + def check(): + env.pop('USERNAME', None) + self.assertEqual(p1.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertRaises(RuntimeError, p2.expanduser) + env['USERNAME'] = 'alice' + self.assertEqual(p2.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertEqual(p3.expanduser(), + P('C:/Users/bob/My Documents')) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + env['HOMEPATH'] = 'C:\\Users\\alice' + check() + + env['HOMEDRIVE'] = 'C:\\' + env['HOMEPATH'] = 'Users\\alice' + check() + + env.pop('HOMEDRIVE', None) + env.pop('HOMEPATH', None) + env['USERPROFILE'] = 'C:\\Users\\alice' + check() + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = 'C:\\Users\\eve' + check() + + @needs_windows + def test_from_uri_windows(self): + P = self.cls + # DOS drive paths + self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) + # UNC paths + self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) + # Localhost paths + self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) + # Invalid paths + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + @needs_windows + def test_from_uri_pathname2url_windows(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) + + @needs_windows + def test_owner_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').owner() + + @needs_windows + def test_group_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').group() + + +@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + +@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath + + +class PathSubclassTest(PathTest): + class cls(pathlib.Path): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +class CompatiblePathTest(unittest.TestCase): + """ + Test that a type can be made compatible with PurePath + derivatives by implementing division operator overloads. + """ + + class CompatPath: + """ + Minimum viable class to test PurePath compatibility. + Simply uses the division operator to join a given + string and the string value of another object with + a forward slash. + """ + def __init__(self, string): + self.string = string + + def __truediv__(self, other): + return type(self)(f"{self.string}/{other}") + + def __rtruediv__(self, other): + return type(self)(f"{other}/{self.string}") + + def test_truediv(self): + result = pathlib.PurePath("test") / self.CompatPath("right") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "test/right") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + pathlib.PurePath("test") / 10 + + def test_rtruediv(self): + result = self.CompatPath("left") / pathlib.PurePath("test") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "left/test") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + 10 / pathlib.PurePath("test") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib_abc.py similarity index 50% rename from Lib/test/test_pathlib.py rename to Lib/test/test_pathlib/test_pathlib_abc.py index 2781e019220353..28c9664cc90fe1 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,75 +1,153 @@ -import collections.abc +import collections import io import os -import sys import errno -import pathlib -import pickle -import posixpath -import socket import stat -import tempfile import unittest -from unittest import mock -from urllib.request import pathname2url -from test.support import import_helper -from test.support import set_recursion_limit -from test.support import is_emscripten, is_wasi -from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath +from pathlib._os import UnsupportedOperation +from pathlib._abc import ParserBase, PurePathBase, PathBase +import posixpath + +from test.support import is_wasi +from test.support.os_helper import TESTFN + + +_tests_needing_posix = set() +_tests_needing_windows = set() +_tests_needing_symlinks = set() + + +def needs_posix(fn): + """Decorator that marks a test as requiring a POSIX-flavoured path class.""" + _tests_needing_posix.add(fn.__name__) + return fn + +def needs_windows(fn): + """Decorator that marks a test as requiring a Windows-flavoured path class.""" + _tests_needing_windows.add(fn.__name__) + return fn -try: - import grp, pwd -except ImportError: - grp = pwd = None +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) -rel_join = lambda *x: os.path.join(TESTFN, *x) - -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') +class ParserBaseTest(unittest.TestCase): + cls = ParserBase + def test_unsupported_operation(self): + m = self.cls() + e = UnsupportedOperation + with self.assertRaises(e): + m.sep + self.assertRaises(e, m.join, 'foo') + self.assertRaises(e, m.split, 'foo') + self.assertRaises(e, m.splitdrive, 'foo') + self.assertRaises(e, m.splitext, 'foo') + self.assertRaises(e, m.normcase, 'foo') + self.assertRaises(e, m.isabs, 'foo') # # Tests for the pure classes. # -class PurePathTest(unittest.TestCase): - cls = pathlib.PurePath - - # Keys are canonical paths, values are list of tuples of arguments - # supposed to produce equal paths. - equivalences = { - 'a/b': [ - ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), - ('a/b/',), ('a//b',), ('a//b//',), - # Empty components get removed. - ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), - ], - '/b/c/d': [ - ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), - # Empty components get removed. - ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), - ], - } + +class PurePathBaseTest(unittest.TestCase): + cls = PurePathBase + + def test_unsupported_operation_pure(self): + p = self.cls('foo') + e = UnsupportedOperation + with self.assertRaises(e): + p.drive + with self.assertRaises(e): + p.root + with self.assertRaises(e): + p.anchor + with self.assertRaises(e): + p.parts + with self.assertRaises(e): + p.parent + with self.assertRaises(e): + p.parents + with self.assertRaises(e): + p.name + with self.assertRaises(e): + p.stem + with self.assertRaises(e): + p.suffix + with self.assertRaises(e): + p.suffixes + with self.assertRaises(e): + p / 'bar' + with self.assertRaises(e): + 'bar' / p + self.assertRaises(e, p.joinpath, 'bar') + self.assertRaises(e, p.with_name, 'bar') + self.assertRaises(e, p.with_stem, 'bar') + self.assertRaises(e, p.with_suffix, '.txt') + self.assertRaises(e, p.relative_to, '') + self.assertRaises(e, p.is_relative_to, '') + self.assertRaises(e, p.is_absolute) + self.assertRaises(e, p.match, '*') + + def test_magic_methods(self): + P = self.cls + self.assertFalse(hasattr(P, '__fspath__')) + self.assertFalse(hasattr(P, '__bytes__')) + self.assertIs(P.__reduce__, object.__reduce__) + self.assertIs(P.__repr__, object.__repr__) + self.assertIs(P.__hash__, object.__hash__) + self.assertIs(P.__eq__, object.__eq__) + self.assertIs(P.__lt__, object.__lt__) + self.assertIs(P.__le__, object.__le__) + self.assertIs(P.__gt__, object.__gt__) + self.assertIs(P.__ge__, object.__ge__) + + def test_parser(self): + self.assertIsInstance(self.cls.parser, ParserBase) + + +class DummyPurePath(PurePathBase): + __slots__ = () + parser = posixpath + + def __eq__(self, other): + if not isinstance(other, DummyPurePath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + +class DummyPurePathTest(unittest.TestCase): + cls = DummyPurePath + + # Use a base path that's unrelated to any real filesystem path. + base = f'/this/path/kills/fascists/{TESTFN}' def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_posix and self.cls.parser is not posixpath: + self.skipTest('requires POSIX-flavoured path class') + if name in _tests_needing_windows and self.cls.parser is posixpath: + self.skipTest('requires Windows-flavoured path class') p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep def test_constructor_common(self): P = self.cls @@ -79,53 +157,14 @@ def test_constructor_common(self): P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(FakePath("a/b/c")) - self.assertEqual(P(P('a')), P('a')) - self.assertEqual(P(P('a'), 'b'), P('a/b')) - self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) - self.assertEqual(P(P('./a:b')), P('./a:b')) - - def test_concrete_class(self): - if self.cls is pathlib.PurePath: - expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_different_pathmods_unequal(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - self.assertNotEqual(p, q) - - def test_different_pathmods_unordered(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - with self.assertRaises(TypeError): - p < q - with self.assertRaises(TypeError): - p <= q - with self.assertRaises(TypeError): - p > q - with self.assertRaises(TypeError): - p >= q def test_bytes(self): P = self.cls - message = (r"argument should be a str or an os\.PathLike object " - r"where __fspath__ returns a str, not 'bytes'") - with self.assertRaisesRegex(TypeError, message): + with self.assertRaises(TypeError): P(b'a') - with self.assertRaisesRegex(TypeError, message): + with self.assertRaises(TypeError): P(b'a', 'b') - with self.assertRaisesRegex(TypeError, message): + with self.assertRaises(TypeError): P('a', b'b') with self.assertRaises(TypeError): P('a').joinpath(b'b') @@ -163,6 +202,19 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') + @needs_windows + def test_str_subclass_windows(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') + def test_with_segments_common(self): class P(self.cls): def __init__(self, *pathsegments, session_id): @@ -184,46 +236,6 @@ def with_segments(self, *pathsegments): for parent in p.parents: self.assertEqual(42, parent.session_id) - def _get_drive_root_parts(self, parts): - path = self.cls(*parts) - return path.drive, path.root, path.parts - - def _check_drive_root_parts(self, arg, *expected): - sep = self.pathmod.sep - actual = self._get_drive_root_parts([x.replace('/', sep) for x in arg]) - self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: - actual = self._get_drive_root_parts([x.replace('/', altsep) for x in arg]) - self.assertEqual(actual, expected) - - def test_drive_root_parts_common(self): - check = self._check_drive_root_parts - sep = self.pathmod.sep - # Unanchored parts. - check((), '', '', ()) - check(('a',), '', '', ('a',)) - check(('a/',), '', '', ('a',)) - check(('a', 'b'), '', '', ('a', 'b')) - # Expansion. - check(('a/b',), '', '', ('a', 'b')) - check(('a/b/',), '', '', ('a', 'b')) - check(('a', 'b/c', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Collapsing and stripping excess slashes. - check(('a', 'b//c', 'd'), '', '', ('a', 'b', 'c', 'd')) - check(('a', 'b/c/', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Eliminating standalone dots. - check(('.',), '', '', ()) - check(('.', '.', 'b'), '', '', ('b',)) - check(('a', '.', 'b'), '', '', ('a', 'b')) - check(('a', '.', '.'), '', '', ('a',)) - # The first part is anchored. - check(('/a/b',), '', sep, (sep, 'a', 'b')) - check(('/a', 'b'), '', sep, (sep, 'a', 'b')) - check(('/a/', 'b'), '', sep, (sep, 'a', 'b')) - # Ignoring parts before an anchored part. - check(('a', '/b', 'c'), '', sep, (sep, 'b', 'c')) - check(('a', '/b', '/c'), '', sep, (sep, 'c')) - def test_join_common(self): P = self.cls p = P('a/b') @@ -232,11 +244,58 @@ def test_join_common(self): self.assertIs(type(pp), type(p)) pp = p.joinpath('c', 'd') self.assertEqual(pp, P('a/b/c/d')) - pp = p.joinpath(P('c')) - self.assertEqual(pp, P('a/b/c')) pp = p.joinpath('/c') self.assertEqual(pp, P('/c')) + @needs_posix + def test_join_posix(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_join_windows(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//').joinpath('server') + self.assertEqual(pp, P('//server')) + pp = P('//server').joinpath('share') + self.assertEqual(pp, P('//server/share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P('//./BootPartition/Windows')) + def test_div_common(self): # Basically the same as joinpath(). P = self.cls @@ -250,11 +309,47 @@ def test_div_common(self): self.assertEqual(pp, P('a/b/c/d')) pp = 'c' / p / 'd' self.assertEqual(pp, P('c/a/b/d')) - pp = p / P('c') - self.assertEqual(pp, P('a/b/c')) pp = p/ '/c' self.assertEqual(pp, P('/c')) + @needs_posix + def test_div_posix(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + @needs_windows + def test_div_windows(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) + def _check_str(self, expected, args): p = self.cls(*args) self.assertEqual(str(p), expected.replace('/', self.sep)) @@ -263,68 +358,33 @@ def test_str_common(self): # Canonicalized paths roundtrip. for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): self._check_str(pathstr, (pathstr,)) - # Special case for the empty path. - self._check_str('.', ('',)) # Other tests for str() are in test_equivalences(). + @needs_windows + def test_str_windows(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + def test_as_posix_common(self): P = self.cls for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): self.assertEqual(P(pathstr).as_posix(), pathstr) # Other tests for as_posix() are in test_equivalences(). - def test_as_bytes_common(self): - sep = os.fsencode(self.sep) - P = self.cls - self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') - - def test_as_uri_common(self): + def test_match_empty(self): P = self.cls - with self.assertRaises(ValueError): - P('a').as_uri() - with self.assertRaises(ValueError): - P().as_uri() - - def test_repr_common(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - - def test_repr_roundtrips(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - r = repr(p) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) - - def test_eq_common(self): - P = self.cls - self.assertEqual(P('a/b'), P('a/b')) - self.assertEqual(P('a/b'), P('a', 'b')) - self.assertNotEqual(P('a/b'), P('a')) - self.assertNotEqual(P('a/b'), P('/a/b')) - self.assertNotEqual(P('a/b'), P()) - self.assertNotEqual(P('/a/b'), P('/')) - self.assertNotEqual(P(), P('/')) - self.assertNotEqual(P(), "") - self.assertNotEqual(P(), {}) - self.assertNotEqual(P(), int) + self.assertRaises(ValueError, P('a').match, '') def test_match_common(self): P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') # Simple relative pattern. self.assertTrue(P('b.py').match('b.py')) self.assertTrue(P('a/b.py').match('b.py')) @@ -356,67 +416,121 @@ def test_match_common(self): self.assertFalse(P('/ab.py').match('/a/*.py')) self.assertFalse(P('/a/b/c.py').match('/a/*.py')) # Multi-part glob-style pattern. - self.assertTrue(P('a').match('**')) - self.assertTrue(P('c.py').match('**')) - self.assertTrue(P('a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('/**')) - self.assertTrue(P('/a/b/c.py').match('**/')) - self.assertTrue(P('/a/b/c.py').match('/a/**')) - self.assertTrue(P('/a/b/c.py').match('**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/*.py')) + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) - self.assertFalse(P('c.py').match('**/a.py')) - self.assertFalse(P('c.py').match('c/**')) - self.assertFalse(P('a/b/c.py').match('**/a')) - self.assertFalse(P('a/b/c.py').match('**/a/b')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) - self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').match, '**a/b/c') - self.assertRaises(ValueError, P('a').match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) # Matching against empty path - self.assertFalse(P().match('*')) - self.assertTrue(P().match('**')) - self.assertFalse(P().match('**/*')) - - def test_ordering_common(self): - # Ordering is tuple-alike. - def assertLess(a, b): - self.assertLess(a, b) - self.assertGreater(b, a) - P = self.cls - a = P('a') - b = P('a/b') - c = P('abc') - d = P('b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - P = self.cls - a = P('/a') - b = P('/a/b') - c = P('/abc') - d = P('/b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - with self.assertRaises(TypeError): - P() < {} + self.assertFalse(P('').match('*')) + self.assertFalse(P('').match('**')) + self.assertFalse(P('').match('**/*')) + + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + def test_full_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').full_match('b.py')) + self.assertFalse(P('a/b.py').full_match('b.py')) + self.assertFalse(P('/a/b.py').full_match('b.py')) + self.assertFalse(P('a.py').full_match('b.py')) + self.assertFalse(P('b/py').full_match('b.py')) + self.assertFalse(P('/a.py').full_match('b.py')) + self.assertFalse(P('b.py/c').full_match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').full_match('*.py')) + self.assertFalse(P('a/b.py').full_match('*.py')) + self.assertFalse(P('/a/b.py').full_match('*.py')) + self.assertFalse(P('b.pyc').full_match('*.py')) + self.assertFalse(P('b./py').full_match('*.py')) + self.assertFalse(P('b.py/c').full_match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('/d/ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('a.py').full_match('a*/*.py')) + self.assertFalse(P('/dab/c.py').full_match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').full_match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').full_match('/*.py')) + self.assertFalse(P('b.py').full_match('/*.py')) + self.assertFalse(P('a/b.py').full_match('/*.py')) + self.assertFalse(P('/a/b.py').full_match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').full_match('/a/*.py')) + self.assertFalse(P('/ab.py').full_match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').full_match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').full_match('**')) + self.assertTrue(P('c.py').full_match('**')) + self.assertTrue(P('a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('/**')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**')) + self.assertTrue(P('/a/b/c.py').full_match('**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').full_match('**/a.py')) + self.assertFalse(P('c.py').full_match('c/**')) + self.assertFalse(P('a/b/c.py').full_match('**/a')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) + # Case-sensitive flag + self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').full_match('*')) + self.assertTrue(P('').full_match('**')) + self.assertFalse(P('').full_match('**/*')) + # Matching with empty pattern + self.assertTrue(P('').full_match('')) + self.assertTrue(P('.').full_match('.')) + self.assertFalse(P('/').full_match('')) + self.assertFalse(P('/').full_match('.')) + self.assertFalse(P('foo').full_match('')) + self.assertFalse(P('foo').full_match('.')) def test_parts_common(self): # `parts` returns a tuple. @@ -430,29 +544,18 @@ def test_parts_common(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) - def test_fspath_common(self): + @needs_windows + def test_parts_windows(self): P = self.cls - p = P('a/b') - self._check_str(p.__fspath__(), ('a/b',)) - self._check_str(os.fspath(p), ('a/b',)) - - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) + p = P('c:a/b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P('c:/a/b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P('//a/b/c/d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) def test_parent_common(self): # Relative @@ -460,8 +563,8 @@ def test_parent_common(self): p = P('a/b/c') self.assertEqual(p.parent, P('a/b')) self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) + self.assertEqual(p.parent.parent.parent, P('')) + self.assertEqual(p.parent.parent.parent.parent, P('')) # Anchored p = P('/a/b/c') self.assertEqual(p.parent, P('/a/b')) @@ -469,6 +572,25 @@ def test_parent_common(self): self.assertEqual(p.parent.parent.parent, P('/')) self.assertEqual(p.parent.parent.parent.parent, P('/')) + @needs_windows + def test_parent_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b')) + self.assertEqual(p.parent.parent.parent, P('//a/b')) + def test_parents_common(self): # Relative P = self.cls @@ -477,17 +599,17 @@ def test_parents_common(self): self.assertEqual(len(par), 3) self.assertEqual(par[0], P('a/b')) self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) + self.assertEqual(par[2], P('')) + self.assertEqual(par[-1], P('')) self.assertEqual(par[-2], P('a')) self.assertEqual(par[-3], P('a/b')) self.assertEqual(par[0:1], (P('a/b'),)) self.assertEqual(par[:2], (P('a/b'), P('a'))) self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) + self.assertEqual(par[1:], (P('a'), P(''))) + self.assertEqual(par[::2], (P('a/b'), P(''))) + self.assertEqual(par[::-1], (P(''), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('')]) with self.assertRaises(IndexError): par[-4] with self.assertRaises(IndexError): @@ -516,13 +638,72 @@ def test_parents_common(self): with self.assertRaises(IndexError): par[3] + @needs_windows + def test_parents_windows(self): + # Anchored + P = self.cls + p = P('z:a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) + self.assertEqual(par[1:], (P('//a/b'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) + with self.assertRaises(IndexError): + par[2] + def test_drive_common(self): P = self.cls self.assertEqual(P('a/b').drive, '') self.assertEqual(P('/a/b').drive, '') self.assertEqual(P('').drive, '') - def test_root_common(self): + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + def test_root_common(self): P = self.cls sep = self.sep self.assertEqual(P('').root, '') @@ -530,6 +711,24 @@ def test_root_common(self): self.assertEqual(P('/').root, sep) self.assertEqual(P('/a/b').root, sep) + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_anchor_common(self): P = self.cls sep = self.sep @@ -538,17 +737,43 @@ def test_anchor_common(self): self.assertEqual(P('/').anchor, sep) self.assertEqual(P('/a/b').anchor, sep) - def test_name_common(self): + @needs_windows + def test_anchor_windows(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:/').anchor, 'c:\\') + self.assertEqual(P('c:/a/b/').anchor, 'c:\\') + self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') + + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') - self.assertEqual(P('.').name, '') + self.assertEqual(P('.').name, '.') + self.assertEqual(P('/a/b/.').name, '.') + + def test_name_common(self): + P = self.cls self.assertEqual(P('/').name, '') self.assertEqual(P('a/b').name, 'b') self.assertEqual(P('/a/b').name, 'b') - self.assertEqual(P('/a/b/.').name, 'b') self.assertEqual(P('a/b.py').name, 'b.py') self.assertEqual(P('/a/b.py').name, 'b.py') + @needs_windows + def test_name_windows(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + def test_suffix_common(self): P = self.cls self.assertEqual(P('').suffix, '') @@ -566,617 +791,15 @@ def test_suffix_common(self): self.assertEqual(P('/a/.hg.rc').suffix, '.rc') self.assertEqual(P('a/b.tar.gz').suffix, '.gz') self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') - - def test_suffixes_common(self): - P = self.cls - self.assertEqual(P('').suffixes, []) - self.assertEqual(P('.').suffixes, []) - self.assertEqual(P('/').suffixes, []) - self.assertEqual(P('a/b').suffixes, []) - self.assertEqual(P('/a/b').suffixes, []) - self.assertEqual(P('/a/b/.').suffixes, []) - self.assertEqual(P('a/b.py').suffixes, ['.py']) - self.assertEqual(P('/a/b.py').suffixes, ['.py']) - self.assertEqual(P('a/.hgrc').suffixes, []) - self.assertEqual(P('/a/.hgrc').suffixes, []) - self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem_common(self): - P = self.cls - self.assertEqual(P('').stem, '') - self.assertEqual(P('.').stem, '') - self.assertEqual(P('..').stem, '..') - self.assertEqual(P('/').stem, '') - self.assertEqual(P('a/b').stem, 'b') - self.assertEqual(P('a/b.py').stem, 'b') - self.assertEqual(P('a/.hgrc').stem, '.hgrc') - self.assertEqual(P('a/.hg.rc').stem, '.hg') - self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name_common(self): - P = self.cls - self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - self.assertRaises(ValueError, P('').with_name, 'd.xml') - self.assertRaises(ValueError, P('.').with_name, 'd.xml') - self.assertRaises(ValueError, P('/').with_name, 'd.xml') - self.assertRaises(ValueError, P('a/b').with_name, '') - self.assertRaises(ValueError, P('a/b').with_name, '.') - self.assertRaises(ValueError, P('a/b').with_name, '/c') - self.assertRaises(ValueError, P('a/b').with_name, 'c/') - self.assertRaises(ValueError, P('a/b').with_name, 'c/d') - - def test_with_stem_common(self): - P = self.cls - self.assertEqual(P('a/b').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) - self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) - self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) - self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) - self.assertRaises(ValueError, P('').with_stem, 'd') - self.assertRaises(ValueError, P('.').with_stem, 'd') - self.assertRaises(ValueError, P('/').with_stem, 'd') - self.assertRaises(ValueError, P('a/b').with_stem, '') - self.assertRaises(ValueError, P('a/b').with_stem, '.') - self.assertRaises(ValueError, P('a/b').with_stem, '/c') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') - - def test_with_suffix_common(self): - P = self.cls - self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) - self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) - # Stripping suffix. - self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) - self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') - self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') - self.assertRaises(ValueError, P('a/b').with_suffix, './.d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - self.assertRaises(ValueError, P('a/b').with_suffix, - (self.pathmod.sep, 'd')) - - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - def test_pickling_common(self): - P = self.cls - p = P('/a/b') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertIs(pp.__class__, p.__class__) - self.assertEqual(pp, p) - self.assertEqual(hash(pp), hash(p)) - self.assertEqual(str(pp), str(p)) - - -class PurePosixPathTest(PurePathTest): - cls = pathlib.PurePosixPath - - def test_drive_root_parts(self): - check = self._check_drive_root_parts - # Collapsing of excess leading slashes, except for the double-slash - # special case. - check(('//a', 'b'), '', '//', ('//', 'a', 'b')) - check(('///a', 'b'), '', '/', ('/', 'a', 'b')) - check(('////a', 'b'), '', '/', ('/', 'a', 'b')) - # Paths which look like NT paths aren't treated specially. - check(('c:a',), '', '', ('c:a',)) - check(('c:\\a',), '', '', ('c:\\a',)) - check(('\\a',), '', '', ('\\a',)) - - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): - P = self.cls - self.assertNotEqual(P('a/b'), P('A/b')) - self.assertEqual(P('/a'), P('///a')) - self.assertNotEqual(P('/a'), P('//a')) - - def test_as_uri(self): - P = self.cls - self.assertEqual(P('/').as_uri(), 'file:///') - self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') - self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') - - def test_as_uri_non_ascii(self): - from urllib.parse import quote_from_bytes - P = self.cls - try: - os.fsencode('\xe9') - except UnicodeEncodeError: - self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") - self.assertEqual(P('/a/b\xe9').as_uri(), - 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - - def test_parse_windows_path(self): - P = self.cls - p = P('c:', 'a', 'b') - pp = P(pathlib.PureWindowsPath('c:\\a\\b')) - self.assertEqual(p, pp) - - -class PureWindowsPathTest(PurePathTest): - cls = pathlib.PureWindowsPath - - equivalences = PurePathTest.equivalences.copy() - equivalences.update({ - './a:b': [ ('./a:b',) ], - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], - 'c:/a': [ - ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), - ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), - ], - '//a/b/': [ ('//a/b',) ], - '//a/b/c': [ - ('//a/b', 'c'), ('//a/b/', 'c'), - ], - }) - - def test_drive_root_parts(self): - check = self._check_drive_root_parts - # First part is anchored. - check(('c:',), 'c:', '', ('c:',)) - check(('c:/',), 'c:', '\\', ('c:\\',)) - check(('/',), '', '\\', ('\\',)) - check(('c:a',), 'c:', '', ('c:', 'a')) - check(('c:/a',), 'c:', '\\', ('c:\\', 'a')) - check(('/a',), '', '\\', ('\\', 'a')) - # UNC paths. - check(('//',), '\\\\', '', ('\\\\',)) - check(('//a',), '\\\\a', '', ('\\\\a',)) - check(('//a/',), '\\\\a\\', '', ('\\\\a\\',)) - check(('//a/b',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/c',), '\\\\a\\b', '\\', ('\\\\a\\b\\', 'c')) - # Second part is anchored, so that the first part is ignored. - check(('a', 'Z:b', 'c'), 'Z:', '', ('Z:', 'b', 'c')) - check(('a', 'Z:/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) - # UNC paths. - check(('a', '//b/c', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) - # Collapsing and stripping excess slashes. - check(('a', 'Z://b//c/', 'd/'), 'Z:', '\\', ('Z:\\', 'b', 'c', 'd')) - # UNC paths. - check(('a', '//b/c//', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) - # Extended paths. - check(('//./c:',), '\\\\.\\c:', '', ('\\\\.\\c:',)) - check(('//?/c:/',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\',)) - check(('//?/c:/a',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'a')) - check(('//?/c:/a', '/b'), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'b')) - # Extended UNC paths (format is "\\?\UNC\server\share"). - check(('//?',), '\\\\?', '', ('\\\\?',)) - check(('//?/',), '\\\\?\\', '', ('\\\\?\\',)) - check(('//?/UNC',), '\\\\?\\UNC', '', ('\\\\?\\UNC',)) - check(('//?/UNC/',), '\\\\?\\UNC\\', '', ('\\\\?\\UNC\\',)) - check(('//?/UNC/b',), '\\\\?\\UNC\\b', '', ('\\\\?\\UNC\\b',)) - check(('//?/UNC/b/',), '\\\\?\\UNC\\b\\', '', ('\\\\?\\UNC\\b\\',)) - check(('//?/UNC/b/c',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/d',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\', 'd')) - # UNC device paths - check(('//./BootPartition/',), '\\\\.\\BootPartition', '\\', ('\\\\.\\BootPartition\\',)) - check(('//?/BootPartition/',), '\\\\?\\BootPartition', '\\', ('\\\\?\\BootPartition\\',)) - check(('//./PhysicalDrive0',), '\\\\.\\PhysicalDrive0', '', ('\\\\.\\PhysicalDrive0',)) - check(('//?/Volume{}/',), '\\\\?\\Volume{}', '\\', ('\\\\?\\Volume{}\\',)) - check(('//./nul',), '\\\\.\\nul', '', ('\\\\.\\nul',)) - # Second part has a root but not drive. - check(('a', '/b', 'c'), '', '\\', ('\\', 'b', 'c')) - check(('Z:/a', '/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) - check(('//?/Z:/a', '/b', 'c'), '\\\\?\\Z:', '\\', ('\\\\?\\Z:\\', 'b', 'c')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - check(('c:/a/b', 'c:x/y'), 'c:', '\\', ('c:\\', 'a', 'b', 'x', 'y')) - check(('c:/a/b', 'c:/x/y'), 'c:', '\\', ('c:\\', 'x', 'y')) - # Paths to files with NTFS alternate data streams - check(('./c:s',), '', '', ('c:s',)) - check(('cc:s',), '', '', ('cc:s',)) - check(('C:c:s',), 'C:', '', ('C:', 'c:s')) - check(('C:/c:s',), 'C:', '\\', ('C:\\', 'c:s')) - check(('D:a', './c:b'), 'D:', '', ('D:', 'a', 'c:b')) - check(('D:/a', './c:b'), 'D:', '\\', ('D:\\', 'a', 'c:b')) - - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') - - def test_str_subclass(self): - self._check_str_subclass('.\\a:b') - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): - P = self.cls - self.assertEqual(P('c:a/b'), P('c:a/b')) - self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) - self.assertNotEqual(P('c:a/b'), P('d:a/b')) - self.assertNotEqual(P('c:a/b'), P('c:/a/b')) - self.assertNotEqual(P('/a/b'), P('c:/a/b')) - # Case-insensitivity. - self.assertEqual(P('a/B'), P('A/b')) - self.assertEqual(P('C:a/B'), P('c:A/b')) - self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) - self.assertEqual(P('\u0130'), P('i\u0307')) - - def test_as_uri(self): - P = self.cls - with self.assertRaises(ValueError): - P('/a/b').as_uri() - with self.assertRaises(ValueError): - P('c:a/b').as_uri() - self.assertEqual(P('c:/').as_uri(), 'file:///c:/') - self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') - self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') - self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') - self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') - self.assertEqual(P('//some/share/a/b.c').as_uri(), - 'file://some/share/a/b.c') - self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), - 'file://some/share/a/b%25%23c%C3%A9') - - def test_match(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - - def test_ordering_common(self): - # Case-insensitivity. - def assertOrderedEqual(a, b): - self.assertLessEqual(a, b) - self.assertGreaterEqual(b, a) - P = self.cls - p = P('c:A/b') - q = P('C:a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - p = P('//some/Share/A/b') - q = P('//Some/SHARE/a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') - def test_suffix(self): + @needs_windows + def test_suffix_windows(self): P = self.cls self.assertEqual(P('c:').suffix, '') self.assertEqual(P('c:/').suffix, '') @@ -1190,12 +813,36 @@ def test_suffix(self): self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') self.assertEqual(P('//My.py/Share.php').suffix, '') self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - def test_suffixes(self): + def test_suffixes_common(self): + P = self.cls + self.assertEqual(P('').suffixes, []) + self.assertEqual(P('.').suffixes, []) + self.assertEqual(P('/').suffixes, []) + self.assertEqual(P('a/b').suffixes, []) + self.assertEqual(P('/a/b').suffixes, []) + self.assertEqual(P('/a/b/.').suffixes, []) + self.assertEqual(P('a/b.py').suffixes, ['.py']) + self.assertEqual(P('/a/b.py').suffixes, ['.py']) + self.assertEqual(P('a/.hgrc').suffixes, []) + self.assertEqual(P('/a/.hgrc').suffixes, []) + self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) + + @needs_windows + def test_suffixes_windows(self): P = self.cls self.assertEqual(P('c:').suffixes, []) self.assertEqual(P('c:/').suffixes, []) @@ -1211,10 +858,31 @@ def test_suffixes(self): self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('//My.py/Share.php').suffixes, []) self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) + + def test_stem_empty(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '.') + + def test_stem_common(self): + P = self.cls + self.assertEqual(P('..').stem, '..') + self.assertEqual(P('/').stem, '') + self.assertEqual(P('a/b').stem, 'b') + self.assertEqual(P('a/b.py').stem, 'b') + self.assertEqual(P('a/.hgrc').stem, '.hgrc') + self.assertEqual(P('a/.hg.rc').stem, '.hg') + self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') - def test_stem(self): + @needs_windows + def test_stem_windows(self): P = self.cls self.assertEqual(P('c:').stem, '') self.assertEqual(P('c:.').stem, '') @@ -1225,10 +893,19 @@ def test_stem(self): self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') self.assertEqual(P('c:a/.hg.rc').stem, '.hg') self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + + def test_with_name_common(self): + P = self.cls + self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - def test_with_name(self): + @needs_windows + def test_with_name_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) @@ -1244,12 +921,37 @@ def test_with_name(self): self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') - def test_with_stem(self): + def test_with_name_empty(self): + P = self.cls + self.assertEqual(P('').with_name('d.xml'), P('d.xml')) + self.assertEqual(P('.').with_name('d.xml'), P('d.xml')) + self.assertEqual(P('/').with_name('d.xml'), P('/d.xml')) + self.assertEqual(P('a/b').with_name(''), P('a/')) + self.assertEqual(P('a/b').with_name('.'), P('a/.')) + + def test_with_name_seps(self): + P = self.cls + self.assertRaises(ValueError, P('a/b').with_name, '/c') + self.assertRaises(ValueError, P('a/b').with_name, 'c/') + self.assertRaises(ValueError, P('a/b').with_name, 'c/d') + + def test_with_stem_common(self): + P = self.cls + self.assertEqual(P('a/b').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) + self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) + self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) + + @needs_windows + def test_with_stem_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) self.assertRaises(ValueError, P('c:').with_stem, 'd') self.assertRaises(ValueError, P('c:/').with_stem, 'd') self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') @@ -1260,7 +962,39 @@ def test_with_stem(self): self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - def test_with_suffix(self): + def test_with_stem_empty(self): + P = self.cls + self.assertEqual(P('').with_stem('d'), P('d')) + self.assertEqual(P('.').with_stem('d'), P('d')) + self.assertEqual(P('/').with_stem('d'), P('/d')) + self.assertEqual(P('a/b').with_stem(''), P('a/')) + self.assertEqual(P('a/b').with_stem('.'), P('a/.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') + + def test_with_stem_seps(self): + P = self.cls + self.assertRaises(ValueError, P('a/b').with_stem, '/c') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') + + def test_with_suffix_common(self): + P = self.cls + self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) + self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) + # Stripping suffix. + self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) + self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) + + @needs_windows + def test_with_suffix_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) @@ -1283,8 +1017,103 @@ def test_with_suffix(self): self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) + + def test_with_suffix_empty(self): + P = self.cls + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + + def test_with_suffix_invalid(self): + P = self.cls + # Invalid suffix. + self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('a/b').with_suffix, '/') + self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') + self.assertRaises(ValueError, P('a/b').with_suffix, './.d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + self.assertRaises(TypeError, P('a/b').with_suffix, None) + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - def test_relative_to(self): + @needs_windows + def test_relative_to_windows(self): P = self.cls p = P('C:Foo/Bar') self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) @@ -1389,7 +1218,40 @@ def test_relative_to(self): self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - def test_is_relative_to(self): + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): P = self.cls p = P('C:Foo/Bar') self.assertTrue(p.is_relative_to(P('c:'))) @@ -1442,7 +1304,20 @@ def test_is_relative_to(self): self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - def test_is_absolute(self): + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): P = self.cls # Under NT, only paths with both a drive and a root are absolute. self.assertFalse(P().is_absolute()) @@ -1456,149 +1331,29 @@ def test_is_absolute(self): self.assertFalse(P('c:a/b/').is_absolute()) self.assertTrue(P('c:/').is_absolute()) self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - pp = p.joinpath(P('./d:s')) - self.assertEqual(pp, P('C:/a/b/d:s')) - pp = p.joinpath(P('./dd:s')) - self.assertEqual(pp, P('C:/a/b/dd:s')) - pp = p.joinpath(P('E:d:s')) - self.assertEqual(pp, P('E:d:s')) - # Joining onto a UNC path with no root - pp = P('//').joinpath('server') - self.assertEqual(pp, P('//server')) - pp = P('//server').joinpath('share') - self.assertEqual(pp, P('//server/share')) - pp = P('//./BootPartition').joinpath('Windows') - self.assertEqual(pp, P('//./BootPartition/Windows')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) - self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) - self.assertEqual(p / P('E:d:s'), P('E:d:s')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) - - -class PurePathSubclassTest(PurePathTest): - class cls(pathlib.PurePath): - pass - - # repr() roundtripping is not supported in custom subclass. - test_repr_roundtrips = None - - -@only_posix -class PosixPathAsPureTest(PurePosixPathTest): - cls = pathlib.PosixPath - -@only_nt -class WindowsPathAsPureTest(PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_owner(self): - P = self.cls - with self.assertRaises(pathlib.UnsupportedOperation): - P('c:/').owner() - - def test_group(self): - P = self.cls - with self.assertRaises(pathlib.UnsupportedOperation): - P('c:/').group() + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) # # Tests for the virtual classes. # -class PathBaseTest(PurePathTest): - cls = pathlib._PathBase +class PathBaseTest(PurePathBaseTest): + cls = PathBase def test_unsupported_operation(self): P = self.cls - p = self.cls() - e = pathlib.UnsupportedOperation + p = self.cls('') + e = UnsupportedOperation self.assertRaises(e, p.stat) self.assertRaises(e, p.lstat) self.assertRaises(e, p.exists) @@ -1640,23 +1395,14 @@ def test_unsupported_operation(self): self.assertRaises(e, p.as_uri) def test_as_uri_common(self): - e = pathlib.UnsupportedOperation - self.assertRaises(e, self.cls().as_uri) + e = UnsupportedOperation + self.assertRaises(e, self.cls('').as_uri) def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls()) + self.assertRaises(TypeError, os.fspath, self.cls('')) def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls()) - - def test_matches_path_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: - our_attr = getattr(self.cls, attr_name) - path_attr = getattr(pathlib.Path, attr_name) - self.assertEqual(our_attr.__doc__, path_attr.__doc__) + self.assertRaises(TypeError, bytes, self.cls('')) class DummyPathIO(io.BytesIO): @@ -1674,20 +1420,39 @@ def close(self): super().close() -class DummyPath(pathlib._PathBase): +DummyPathStatResult = collections.namedtuple( + 'DummyPathStatResult', + 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') + + +class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in memory. """ + __slots__ = () + parser = posixpath + _files = {} _directories = {} _symlinks = {} + def __eq__(self, other): + if not isinstance(other, DummyPath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def stat(self, *, follow_symlinks=True): - if follow_symlinks: - path = str(self.resolve()) + if follow_symlinks or self.name in ('', '.', '..'): + path = str(self.resolve(strict=True)) else: - path = str(self.parent.resolve() / self.name) + path = str(self.parent.resolve(strict=True) / self.name) if path in self._files: st_mode = stat.S_IFREG elif path in self._directories: @@ -1696,7 +1461,7 @@ def stat(self, *, follow_symlinks=True): st_mode = stat.S_IFLNK else: raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): @@ -1732,7 +1497,7 @@ def iterdir(self): if path in self._files: raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) elif path in self._directories: - return (self / name for name in self._directories[path]) + return iter([self / name for name in self._directories[path]]) else: raise FileNotFoundError(errno.ENOENT, "File not found", path) @@ -1753,14 +1518,45 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) + def unlink(self, missing_ok=False): + path_obj = self.parent.resolve(strict=True) / self.name + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + elif path in self._files: + self._directories[parent].remove(name) + del self._files[path] + elif path in self._symlinks: + self._directories[parent].remove(name) + del self._symlinks[path] + elif not missing_ok: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def rmdir(self): + path_obj = self.parent.resolve(strict=True) / self.name + path = str(path_obj) + if path in self._files or path in self._symlinks: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + elif self._directories[path]: + raise OSError(errno.ENOTEMPTY, "Directory not empty", path) + else: + name = path_obj.name + parent = str(path_obj.parent) + self._directories[parent].remove(name) + del self._directories[path] + -class DummyPathTest(unittest.TestCase): +class DummyPathTest(DummyPurePathTest): """Tests for PathBase methods that use stat(), open() and iterdir().""" cls = DummyPath can_symlink = False - # (BASE) + # (self.base) # | # |-- brokenLink -> non-existing # |-- dirA @@ -1781,8 +1577,12 @@ class DummyPathTest(unittest.TestCase): # def setUp(self): - pathmod = self.cls.pathmod - p = self.cls(BASE) + super().setUp() + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + parser = self.cls.parser + p = self.cls(self.base) p.mkdir(parents=True) p.joinpath('dirA').mkdir() p.joinpath('dirB').mkdir() @@ -1803,8 +1603,8 @@ def setUp(self): p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('dirA', 'linkC').symlink_to(parser.join('..', 'dirB')) + p.joinpath('dirB', 'linkD').symlink_to(parser.join('..', 'dirB')) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') def tearDown(self): @@ -1814,7 +1614,7 @@ def tearDown(self): cls._symlinks.clear() def tempdir(self): - path = self.cls(BASE).with_name('tmp-dirD') + path = self.cls(self.base).with_name('tmp-dirD') path.mkdir() return path @@ -1824,11 +1624,13 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) + normcase = self.parser.normcase + self.assertEqual(normcase(path_a), normcase(path_b)) def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') p = self.cls(fileA_path) pp = self.cls(fileA_path) q = self.cls(fileB_path) @@ -1837,7 +1639,7 @@ def test_samefile(self): self.assertFalse(p.samefile(fileB_path)) self.assertFalse(p.samefile(q)) # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') + non_existent = parser.join(self.base, 'foo') r = self.cls(non_existent) self.assertRaises(FileNotFoundError, p.samefile, r) self.assertRaises(FileNotFoundError, p.samefile, non_existent) @@ -1846,14 +1648,9 @@ def test_samefile(self): self.assertRaises(FileNotFoundError, r.samefile, r) self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(str(p), '.') - def test_exists(self): P = self.cls - p = P(BASE) + p = P(self.base) self.assertIs(True, p.exists()) self.assertIs(True, (p / 'dirA').exists()) self.assertIs(True, (p / 'fileA').exists()) @@ -1867,11 +1664,11 @@ def test_exists(self): self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) self.assertIs(False, (p / 'foo').exists()) self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) def test_open_common(self): - p = self.cls(BASE) + p = self.cls(self.base) with (p / 'fileA').open('r') as f: self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.read(), "this is file A\n") @@ -1880,7 +1677,7 @@ def test_open_common(self): self.assertEqual(f.read().strip(), b"this is file A") def test_read_write_bytes(self): - p = self.cls(BASE) + p = self.cls(self.base) (p / 'fileA').write_bytes(b'abcdefg') self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') # Check that trying to write str does not truncate the file. @@ -1888,7 +1685,7 @@ def test_read_write_bytes(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') def test_read_write_text(self): - p = self.cls(BASE) + p = self.cls(self.base) (p / 'fileA').write_text('äbcdefg', encoding='latin-1') self.assertEqual((p / 'fileA').read_text( encoding='utf-8', errors='ignore'), 'bcdefg') @@ -1896,8 +1693,23 @@ def test_read_write_text(self): self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') + def test_read_text_with_newlines(self): + p = self.cls(self.base) + # Check that `\n` character change nothing + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\n'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r\n'), + 'abcde\r\nfghlk\n\rmnopq') + def test_write_text_with_newlines(self): - p = self.cls(BASE) + p = self.cls(self.base) # Check that `\n` character change nothing (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') self.assertEqual((p / 'fileA').read_bytes(), @@ -1916,29 +1728,280 @@ def test_write_text_with_newlines(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + def test_copy_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_directory(self): + base = self.cls(self.base) + source = base / 'dirA' + target = base / 'copyA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + source.copy(target, follow_symlinks=False) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + source.copy(target, follow_symlinks=False) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + def test_copy_to_existing_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' / 'fileB' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + source.copy(target, follow_symlinks=False) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + def test_copy_empty(self): + base = self.cls(self.base) + source = base / 'empty' + target = base / 'copyA' + source.write_bytes(b'') + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(target.read_bytes(), b'') + + def test_copytree_simple(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + source.copytree(target) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copytree_complex(self, follow_symlinks=True): + def ordered_walk(path): + for dirpath, dirnames, filenames in path.walk(follow_symlinks=follow_symlinks): + dirnames.sort() + filenames.sort() + yield dirpath, dirnames, filenames + base = self.cls(self.base) + source = base / 'dirC' + + if self.can_symlink: + # Add some symlinks + source.joinpath('linkC').symlink_to('fileC') + source.joinpath('linkD').symlink_to('dirD') + + # Perform the copy + target = base / 'copyC' + source.copytree(target, follow_symlinks=follow_symlinks) + + # Compare the source and target trees + source_walk = ordered_walk(source) + target_walk = ordered_walk(target) + for source_item, target_item in zip(source_walk, target_walk, strict=True): + self.assertEqual(source_item[0].relative_to(source), + target_item[0].relative_to(target)) # dirpath + self.assertEqual(source_item[1], target_item[1]) # dirnames + self.assertEqual(source_item[2], target_item[2]) # filenames + # Compare files and symlinks + for filename in source_item[2]: + source_file = source_item[0].joinpath(filename) + target_file = target_item[0].joinpath(filename) + if follow_symlinks or not source_file.is_symlink(): + # Regular file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + elif source_file.is_dir(): + # Symlink to directory. + self.assertTrue(target_file.is_dir()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + else: + # Symlink to file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + + def test_copytree_complex_follow_symlinks_false(self): + self.test_copytree_complex(follow_symlinks=False) + + def test_copytree_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + self.assertRaises(FileExistsError, source.copytree, target) + + def test_copytree_to_existing_directory_dirs_exist_ok(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + source.copytree(target, dirs_exist_ok=True) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copytree_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + self.assertRaises(NotADirectoryError, source.copytree, target) + + def test_copytree_file_on_error(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + errors = [] + source.copytree(target, on_error=errors.append) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], NotADirectoryError) + + def test_copytree_ignore_false(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + ignores = [] + def ignore_false(path): + ignores.append(path) + return False + source.copytree(target, ignore=ignore_false) + self.assertEqual(set(ignores), { + source / 'dirD', + source / 'dirD' / 'fileD', + source / 'fileC', + source / 'novel.txt', + }) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copytree_ignore_true(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + ignores = [] + def ignore_true(path): + ignores.append(path) + return True + source.copytree(target, ignore=ignore_true) + self.assertEqual(set(ignores), { + source / 'dirD', + source / 'fileC', + source / 'novel.txt', + }) + self.assertTrue(target.is_dir()) + self.assertFalse(target.joinpath('dirD').exists()) + self.assertFalse(target.joinpath('fileC').exists()) + self.assertFalse(target.joinpath('novel.txt').exists()) + + @needs_symlinks + def test_copytree_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copytree, target) + + target2 = base / 'target2' + source.copytree(target2, follow_symlinks=False) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + def test_iterdir(self): P = self.cls - p = P(BASE) + p = P(self.base) it = p.iterdir() paths = set(it) expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] if self.can_symlink: expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) + self.assertEqual(paths, { P(self.base, q) for q in expected }) + @needs_symlinks def test_iterdir_symlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") # __iter__ on a symlink to a directory. P = self.cls - p = P(BASE, 'linkB') + p = P(self.base, 'linkB') paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } self.assertEqual(paths, expected) def test_iterdir_nodir(self): # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') + p = self.cls(self.base, 'fileA') with self.assertRaises(OSError) as cm: p.iterdir() # ENOENT or EINVAL under Windows, ENOTDIR otherwise @@ -1948,9 +2011,9 @@ def test_iterdir_nodir(self): def test_glob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(set(glob), { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) it = p.glob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) @@ -1973,182 +2036,180 @@ def _check(glob, expected): _check(p.glob("brokenLink"), ['brokenLink']) if not self.can_symlink: - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE"]) + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/"]) else: - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE", "linkB"]) + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + + @needs_posix + def test_glob_posix(self): + P = self.cls + p = P(self.base) + q = p / "FILEa" + given = set(p.glob("FILEa")) + expect = {q} if q.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) + + @needs_windows + def test_glob_windows(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) + P = self.cls + p = P(self.base) + self.assertEqual(list(p.glob("")), [p]) + self.assertEqual(list(p.glob(".")), [p / "."]) + self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls def _check(path, pattern, case_sensitive, expected): actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} - expected = {str(P(BASE, q)) for q in expected} + expected = {str(P(self.base, q)) for q in expected} self.assertEqual(actual, expected) - path = P(BASE) + path = P(self.base) _check(path, "DIRB/FILE*", True, []) _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) - def test_glob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") + @needs_symlinks + def test_glob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) - if "linkD" not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) _check(p, "fileB", []) _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) _check(p, "*A", ["dirA", "fileA", "linkA"]) _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirE", "linkB"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) - _check(p, "dir*/**/", ["dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirC/dirD", "dirE"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirA/linkC", "dirA/linkC/linkD", "dirB/linkD", "dirC/dirD"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD"]) - _check(p, "*/dirD/**/", ["dirC/dirD"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) - def test_glob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") + def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirE"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/", ["dirA", "dirB", "dirC", "dirC/dirD", "dirE"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirC/dirD"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD"]) - _check(p, "*/dirD/**/", ["dirC/dirD"]) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) + actual = set(path.rglob(glob, recurse_symlinks=False)) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) + p = P(self.base) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not self.can_symlink: - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not self.can_symlink: - _check(p.rglob("*/"), [ - "dirA", "dirB", "dirC", "dirC/dirD", "dirE", - ]) - else: - _check(p.rglob("*/"), [ - "dirA", "dirA/linkC", "dirB", "dirB/linkD", "dirC", - "dirC/dirD", "dirE", "linkB", - ]) - _check(p.rglob(""), ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) - - p = P(BASE, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**/"), ["dirC/dirD"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD"]) - _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**/"), ["dirC", "dirC/dirD"]) - # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) - - def test_rglob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) - if 'linkD' not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "fileA", ["fileA"]) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "**/fileB", ["dirB/fileB"]) _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirC/dirD", "dirE", "linkB", "linkB/linkD"]) - _check(p, "", ["", "dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirE", "dirC/dirD", "linkB", "linkB/linkD"]) - p = P(BASE, "dirC") + if self.can_symlink: + _check(p, "*/fileB", ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p, "*/", [ + "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", + "dirC/dirD/", "dirE/", "linkB/"]) + else: + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + p = P(self.base, "dirC") _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) + "dirC/dirD", "dirC/dirD/fileD"]) _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "**/file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "dir*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/**/", ["dirC/dirD/"]) _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD"]) - _check(p, "", ["dirC", "dirC/dirD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + _check(p, "**", ["dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + _check(p, "**/", ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) - def test_rglob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") + @needs_posix + def test_rglob_posix(self): + P = self.cls + p = P(self.base, "dirC") + q = p / "dirD" / "FILEd" + given = set(p.rglob("FILEd")) + expect = {q} if q.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + + @needs_windows + def test_rglob_windows(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) + + @needs_symlinks + def test_rglob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB"]) + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirC/dirD", "dirE"]) - _check(p, "", ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - p = P(BASE, "dirC") + p = P(self.base, "dirC") _check(p, "*", ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD"]) - _check(p, "", ["dirC", "dirC/dirD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) + @needs_symlinks def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls - p = P(BASE) - given = set(p.rglob('*')) + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) expect = {'brokenLink', 'dirA', 'dirA/linkC', 'dirB', 'dirB/fileB', 'dirB/linkD', @@ -2162,96 +2223,56 @@ def test_rglob_symlink_loop(self): } self.assertEqual(given, {p / x for x in expect}) - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - p = base = P(BASE) / 'deep' - p.mkdir() - for _ in range(depth): - p /= 'd' - p.mkdir() - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") def test_glob_dotdot(self): # ".." is not special in globs. P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) self.assertEqual(set(p.glob("dirA/../file*/..")), set()) self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + @needs_symlinks def test_glob_permissions(self): # See bpo-38894 - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls - base = P(BASE) / 'permissions' + base = P(self.base) / 'permissions' base.mkdir() for i in range(100): link = base / f"link{i}" if i % 2: - link.symlink_to(P(BASE, "dirE", "nonexistent")) + link.symlink_to(P(self.base, "dirE", "nonexistent")) else: - link.symlink_to(P(BASE, "dirC")) + link.symlink_to(P(self.base, "dirC")) self.assertEqual(len(set(base.glob("*"))), 100) self.assertEqual(len(set(base.glob("*/"))), 50) self.assertEqual(len(set(base.glob("*/fileC"))), 50) self.assertEqual(len(set(base.glob("*/file*"))), 50) + @needs_symlinks def test_glob_long_symlink(self): # See gh-87695 - if not self.can_symlink: - self.skipTest("symlinks required") - base = self.cls(BASE) / 'long_symlink' + base = self.cls(self.base) / 'long_symlink' base.mkdir() bad_link = base / 'bad_link' bad_link.symlink_to("bad" * 200) self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - def test_glob_above_recursion_limit(self): - recursion_limit = 50 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') - path = self.cls(base, *(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.glob('**/')) - - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(BASE) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - - + @needs_symlinks def test_readlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls(BASE) + P = self.cls(self.base) self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) self.assertEqual((P / 'brokenLink').readlink(), self.cls('non-existing')) @@ -2262,9 +2283,9 @@ def test_readlink(self): @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") def test_readlink_unsupported(self): - P = self.cls(BASE) + P = self.cls(self.base) p = P / 'fileA' - with self.assertRaises(pathlib.UnsupportedOperation): + with self.assertRaises(UnsupportedOperation): q.readlink(p) def _check_resolve(self, p, expected, strict=True): @@ -2274,74 +2295,74 @@ def _check_resolve(self, p, expected, strict=True): # This can be used to check both relative and absolute resolutions. _check_resolve_relative = _check_resolve_absolute = _check_resolve + @needs_symlinks def test_resolve_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") P = self.cls - p = P(BASE, 'foo') + p = P(self.base, 'foo') with self.assertRaises(OSError) as cm: p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict + parser = self.parser self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') + p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) # Now create absolute symlinks. d = self.tempdir() - P(BASE, 'dirA', 'linkX').symlink_to(d) - P(BASE, str(d), 'linkY').symlink_to(join('dirB')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) + parser = self.parser + p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) q = p / '2' self.assertEqual(q.resolve(strict=True), p) r = q / '3' / '4' @@ -2355,36 +2376,34 @@ def _check_symlink_loop(self, *args): path.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ELOOP) + @needs_posix + @needs_symlinks def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - if os.name == 'nt' and issubclass(self.cls, pathlib.Path): - self.skipTest("symlink loops work differently with concrete Windows paths") # Loops with relative symlinks. - self.cls(BASE, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(BASE, 'linkX') - self.cls(BASE, 'linkY').symlink_to('linkY') - self._check_symlink_loop(BASE, 'linkY') - self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(BASE, 'linkZ') + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') + p = self.cls(self.base, 'linkZ', 'foo') self.assertEqual(p.resolve(strict=False), p) # Loops with absolute symlinks. - self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) - self._check_symlink_loop(BASE, 'linkU') - self.cls(BASE, 'linkV').symlink_to(join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) - self._check_symlink_loop(BASE, 'linkW') + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') # Non-strict - q = self.cls(BASE, 'linkW', 'foo') + q = self.cls(self.base, 'linkW', 'foo') self.assertEqual(q.resolve(strict=False), q) def test_stat(self): - statA = self.cls(BASE).joinpath('fileA').stat() - statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() - statC = self.cls(BASE).joinpath('dirC').stat() + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() # st_mode: files are the same, directory differs. self.assertIsInstance(statA.st_mode, int) self.assertEqual(statA.st_mode, statB.st_mode) @@ -2401,32 +2420,30 @@ def test_stat(self): self.assertEqual(statA.st_dev, statC.st_dev) # other attributes not used by pathlib. + @needs_symlinks def test_stat_no_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) / 'linkA' + p = self.cls(self.base) / 'linkA' st = p.stat() self.assertNotEqual(st, p.stat(follow_symlinks=False)) def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) + @needs_symlinks def test_lstat(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE)/ 'linkA' + p = self.cls(self.base)/ 'linkA' st = p.stat() self.assertNotEqual(st, p.lstat()) def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' st = p.stat() self.assertEqual(st, p.lstat()) def test_is_dir(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) self.assertFalse((P / 'fileA').is_dir()) self.assertFalse((P / 'non-existing').is_dir()) @@ -2439,7 +2456,7 @@ def test_is_dir(self): self.assertFalse((P / 'dirA\x00').is_dir()) def test_is_dir_no_follow_symlinks(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) @@ -2452,7 +2469,7 @@ def test_is_dir_no_follow_symlinks(self): self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) def test_is_file(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'fileA').is_file()) self.assertFalse((P / 'dirA').is_file()) self.assertFalse((P / 'non-existing').is_file()) @@ -2465,7 +2482,7 @@ def test_is_file(self): self.assertFalse((P / 'fileA\x00').is_file()) def test_is_file_no_follow_symlinks(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) @@ -2478,7 +2495,7 @@ def test_is_file_no_follow_symlinks(self): self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) def test_is_mount(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_mount()) self.assertFalse((P / 'dirA').is_mount()) self.assertFalse((P / 'non-existing').is_mount()) @@ -2487,7 +2504,7 @@ def test_is_mount(self): self.assertFalse((P / 'linkA').is_mount()) def test_is_symlink(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) self.assertFalse((P / 'dirA').is_symlink()) self.assertFalse((P / 'non-existing').is_symlink()) @@ -2503,7 +2520,7 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\x00').is_file(), False) def test_is_junction_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_junction()) self.assertFalse((P / 'dirA').is_junction()) self.assertFalse((P / 'non-existing').is_junction()) @@ -2512,7 +2529,7 @@ def test_is_junction_false(self): self.assertFalse((P / 'fileA\x00').is_junction()) def test_is_fifo_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_fifo()) self.assertFalse((P / 'dirA').is_fifo()) self.assertFalse((P / 'non-existing').is_fifo()) @@ -2521,7 +2538,7 @@ def test_is_fifo_false(self): self.assertIs((P / 'fileA\x00').is_fifo(), False) def test_is_socket_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_socket()) self.assertFalse((P / 'dirA').is_socket()) self.assertFalse((P / 'non-existing').is_socket()) @@ -2530,7 +2547,7 @@ def test_is_socket_false(self): self.assertIs((P / 'fileA\x00').is_socket(), False) def test_is_block_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_block_device()) self.assertFalse((P / 'dirA').is_block_device()) self.assertFalse((P / 'non-existing').is_block_device()) @@ -2539,7 +2556,7 @@ def test_is_block_device_false(self): self.assertIs((P / 'fileA\x00').is_block_device(), False) def test_is_char_device_false(self): - P = self.cls(BASE) + P = self.cls(self.base) self.assertFalse((P / 'fileA').is_char_device()) self.assertFalse((P / 'dirA').is_char_device()) self.assertFalse((P / 'non-existing').is_char_device()) @@ -2547,658 +2564,84 @@ def test_is_char_device_false(self): self.assertIs((P / 'fileA\udfff').is_char_device(), False) self.assertIs((P / 'fileA\x00').is_char_device(), False) - def test_pickling_common(self): - p = self.cls(BASE, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - def _check_complex_symlinks(self, link0_target): - if not self.can_symlink: - self.skipTest("symlinks required") - # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) - P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) + parser = self.parser + P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) # Resolve absolute paths. p = (P / 'link0').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link1').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link2').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = (P / 'link3').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) # Resolve relative paths. try: - self.cls().absolute() - except pathlib.UnsupportedOperation: + self.cls('').absolute() + except UnsupportedOperation: return old_path = os.getcwd() - os.chdir(BASE) + os.chdir(self.base) try: p = self.cls('link0').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link1').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link2').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) p = self.cls('link3').resolve() self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) + self.assertEqualNormCase(str(p), self.base) finally: os.chdir(old_path) + @needs_symlinks def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) + self._check_complex_symlinks(self.base) + @needs_symlinks def test_complex_symlinks_relative(self): self._check_complex_symlinks('.') + @needs_symlinks def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) - - -class DummyPathWithSymlinks(DummyPath): - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - self._directories[str(self.parent)].add(self.name) - self._symlinks[str(self)] = str(target) - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True - - -# -# Tests for the concrete classes. -# - -class PathTest(DummyPathTest): - """Tests for the FS-accessing functionalities of the Path classes.""" - cls = pathlib.Path - can_symlink = os_helper.can_symlink() - - def setUp(self): - super().setUp() - os.chmod(join('dirE'), 0) - - def tearDown(self): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - - def tempdir(self): - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) - return d - - def test_concrete_class(self): - if self.cls is pathlib.Path: - expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_unsupported_pathmod(self): - if self.cls.pathmod is os.path: - self.skipTest("path flavour is supported") - else: - self.assertRaises(pathlib.UnsupportedOperation, self.cls) - - def _test_cwd(self, p): - q = self.cls(os.getcwd()) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - def test_cwd(self): - p = self.cls.cwd() - self._test_cwd(p) - - def test_absolute_common(self): - P = self.cls - - with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = BASE - - # Simple relative paths. - self.assertEqual(str(P().absolute()), BASE) - self.assertEqual(str(P('.').absolute()), BASE) - self.assertEqual(str(P('a').absolute()), os.path.join(BASE, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(BASE, 'a', 'b', 'c')) - - # Symlinks should not be resolved. - self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(BASE, 'linkB', 'fileB')) - self.assertEqual(str(P('brokenLink').absolute()), os.path.join(BASE, 'brokenLink')) - self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(BASE, 'brokenLinkLoop')) - - # '..' entries should be preserved and not normalised. - self.assertEqual(str(P('..').absolute()), os.path.join(BASE, '..')) - self.assertEqual(str(P('a', '..').absolute()), os.path.join(BASE, 'a', '..')) - self.assertEqual(str(P('..', 'b').absolute()), os.path.join(BASE, '..', 'b')) - - def _test_home(self, p): - q = self.cls(os.path.expanduser('~')) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - @unittest.skipIf( - pwd is None, reason="Test requires pwd module to get homedir." - ) - def test_home(self): - with os_helper.EnvironmentVarGuard() as env: - self._test_home(self.cls.home()) - - env.clear() - env['USERPROFILE'] = os.path.join(BASE, 'userprofile') - self._test_home(self.cls.home()) - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = os.path.join(BASE, 'home') - self._test_home(self.cls.home()) - - @unittest.skipIf(is_wasi, "WASI has no user accounts.") - def test_expanduser_common(self): - P = self.cls - p = P('~') - self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) - p = P('foo') - self.assertEqual(p.expanduser(), p) - p = P('/~') - self.assertEqual(p.expanduser(), p) - p = P('../~') - self.assertEqual(p.expanduser(), p) - p = P(P('').absolute().anchor) / '~' - self.assertEqual(p.expanduser(), p) - p = P('~/a:b') - self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) - - def test_with_segments(self): - class P(self.cls): - def __init__(self, *pathsegments, session_id): - super().__init__(*pathsegments) - self.session_id = session_id - - def with_segments(self, *pathsegments): - return type(self)(*pathsegments, session_id=self.session_id) - p = P(BASE, session_id=42) - self.assertEqual(42, p.absolute().session_id) - self.assertEqual(42, p.resolve().session_id) - if not is_wasi: # WASI has no user accounts. - self.assertEqual(42, p.with_segments('~').expanduser().session_id) - self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) - self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) - if self.can_symlink: - self.assertEqual(42, (p / 'linkA').readlink().session_id) - for path in p.iterdir(): - self.assertEqual(42, path.session_id) - for path in p.glob('*'): - self.assertEqual(42, path.session_id) - for path in p.rglob('*'): - self.assertEqual(42, path.session_id) - for dirpath, dirnames, filenames in p.walk(): - self.assertEqual(42, dirpath.session_id) - - def test_open_unbuffered(self): - p = self.cls(BASE) - with (p / 'fileA').open('rb', buffering=0) as f: - self.assertIsInstance(f, io.RawIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - - def test_resolve_nonexist_relative_issue38671(self): - p = self.cls('non', 'exist') - - old_cwd = os.getcwd() - os.chdir(BASE) - try: - self.assertEqual(p.resolve(), self.cls(BASE, p)) - finally: - os.chdir(old_cwd) - - @os_helper.skip_unless_working_chmod - def test_chmod(self): - p = self.cls(BASE) / 'fileA' - mode = p.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - # Set writable bit. - new_mode = mode | 0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - - # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix - @os_helper.skip_unless_working_chmod - def test_chmod_follow_symlinks_true(self): - p = self.cls(BASE) / 'linkA' - q = p.resolve() - mode = q.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - # Set writable bit - new_mode = mode | 0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - - # XXX also need a test for lchmod. - - @unittest.skipUnless(pwd, "the pwd module is needed for this test") - def test_owner(self): - p = self.cls(BASE) / 'fileA' - uid = p.stat().st_uid - try: - name = pwd.getpwuid(uid).pw_name - except KeyError: - self.skipTest( - "user %d doesn't have an entry in the system database" % uid) - self.assertEqual(name, p.owner()) - - @unittest.skipUnless(grp, "the grp module is needed for this test") - def test_group(self): - p = self.cls(BASE) / 'fileA' - gid = p.stat().st_gid - try: - name = grp.getgrgid(gid).gr_name - except KeyError: - self.skipTest( - "group %d doesn't have an entry in the system database" % gid) - self.assertEqual(name, p.group()) + self._check_complex_symlinks(self.parser.join('dirA', '..')) def test_unlink(self): - p = self.cls(BASE) / 'fileA' + p = self.cls(self.base) / 'fileA' p.unlink() self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) def test_unlink_missing_ok(self): - p = self.cls(BASE) / 'fileAAA' + p = self.cls(self.base) / 'fileAAA' self.assertFileNotFound(p.unlink) p.unlink(missing_ok=True) def test_rmdir(self): - p = self.cls(BASE) / 'dirA' + p = self.cls(self.base) / 'dirA' for q in p.iterdir(): q.unlink() p.rmdir() self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") - def test_hardlink_to(self): - P = self.cls(BASE) - target = P / 'fileA' - size = target.stat().st_size - # linking to another path. - link = P / 'dirA' / 'fileAA' - link.hardlink_to(target) - self.assertEqual(link.stat().st_size, size) - self.assertTrue(os.path.samefile(target, link)) - self.assertTrue(target.exists()) - # Linking to a str of a relative path. - link2 = P / 'dirA' / 'fileAAA' - target2 = rel_join('fileA') - link2.hardlink_to(target2) - self.assertEqual(os.stat(target2).st_size, size) - self.assertTrue(link2.exists()) - - @unittest.skipIf(hasattr(os, "link"), "os.link() is present") - def test_hardlink_to_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - # linking to another path. - q = P / 'dirA' / 'fileAA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.hardlink_to(p) - - def test_rename(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Renaming to another path. - q = P / 'dirA' / 'fileAA' - renamed_p = p.rename(q) - self.assertEqual(renamed_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Renaming to a str of a relative path. - r = rel_join('fileAAA') - renamed_q = q.rename(r) - self.assertEqual(renamed_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - def test_replace(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Replacing a non-existing path. - q = P / 'dirA' / 'fileAA' - replaced_p = p.replace(q) - self.assertEqual(replaced_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Replacing another (existing) path. - r = rel_join('dirB', 'fileB') - replaced_q = q.replace(r) - self.assertEqual(replaced_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - def test_touch_common(self): - P = self.cls(BASE) - p = P / 'newfileA' - self.assertFalse(p.exists()) - p.touch() - self.assertTrue(p.exists()) - st = p.stat() - old_mtime = st.st_mtime - old_mtime_ns = st.st_mtime_ns - # Rewind the mtime sufficiently far in the past to work around - # filesystem-specific timestamp granularity. - os.utime(str(p), (old_mtime - 10, old_mtime - 10)) - # The file mtime should be refreshed by calling touch() again. - p.touch() - st = p.stat() - self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) - self.assertGreaterEqual(st.st_mtime, old_mtime) - # Now with exist_ok=False. - p = P / 'newfileB' - self.assertFalse(p.exists()) - p.touch(mode=0o700, exist_ok=False) - self.assertTrue(p.exists()) - self.assertRaises(OSError, p.touch, exist_ok=False) - - def test_touch_nochange(self): - P = self.cls(BASE) - p = P / 'fileA' - p.touch() - with p.open('rb') as f: - self.assertEqual(f.read().strip(), b"this is file A") - - def test_mkdir(self): - P = self.cls(BASE) - p = P / 'newdirA' - self.assertFalse(p.exists()) - p.mkdir() - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_parents(self): - # Creating a chain of directories. - p = self.cls(BASE, 'newdirB', 'newdirC') - self.assertFalse(p.exists()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.ENOENT) - p.mkdir(parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - # Test `mode` arg. - mode = stat.S_IMODE(p.stat().st_mode) # Default mode. - p = self.cls(BASE, 'newdirD', 'newdirE') - p.mkdir(0o555, parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - if os.name != 'nt': - # The directory's permissions follow the mode argument. - self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) - # The parent's permissions follow the default process settings. - self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) - - def test_mkdir_exist_ok(self): - p = self.cls(BASE, 'dirB') - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - def test_mkdir_exist_ok_with_parent(self): - p = self.cls(BASE, 'dirC') - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p = p / 'newdirC' - p.mkdir(parents=True) - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(parents=True, exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") - def test_mkdir_exist_ok_root(self): - # Issue #25803: A drive root could raise PermissionError on Windows. - self.cls('/').resolve().mkdir(exist_ok=True) - self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - - @only_nt # XXX: not sure how to test this on POSIX. - def test_mkdir_with_unknown_drive(self): - for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': - p = self.cls(d + ':\\') - if not p.is_dir(): - break - else: - self.skipTest("cannot find a drive that doesn't exist") - with self.assertRaises(OSError): - (p / 'child' / 'path').mkdir(parents=True) - - def test_mkdir_with_child_file(self): - p = self.cls(BASE, 'dirB', 'fileB') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True, exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_no_parents_file(self): - p = self.cls(BASE, 'fileA') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_concurrent_parent_creation(self): - for pattern_num in range(32): - p = self.cls(BASE, 'dirCPC%d' % pattern_num) - self.assertFalse(p.exists()) - - real_mkdir = os.mkdir - def my_mkdir(path, mode=0o777): - path = str(path) - # Emulate another process that would create the directory - # just before we try to create it ourselves. We do it - # in all possible pattern combinations, assuming that this - # function is called at most 5 times (dirCPC/dir1/dir2, - # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). - if pattern.pop(): - real_mkdir(path, mode) # From another process. - concurrently_created.add(path) - real_mkdir(path, mode) # Our real call. - - pattern = [bool(pattern_num & (1 << n)) for n in range(5)] - concurrently_created = set() - p12 = p / 'dir1' / 'dir2' - try: - with mock.patch("os.mkdir", my_mkdir): - p12.mkdir(parents=True, exist_ok=False) - except FileExistsError: - self.assertIn(str(p12), concurrently_created) - else: - self.assertNotIn(str(p12), concurrently_created) - self.assertTrue(p.exists()) - - def test_symlink_to(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls(BASE) - target = P / 'fileA' - # Symlinking a path target. - link = P / 'dirA' / 'linkAA' - link.symlink_to(target) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - # Symlinking a str target. - link = P / 'dirA' / 'linkAAA' - link.symlink_to(str(target)) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertFalse(link.is_dir()) - # Symlinking to a directory. - target = P / 'dirB' - link = P / 'dirA' / 'linkAAAA' - link.symlink_to(target, target_is_directory=True) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertTrue(link.is_dir()) - self.assertTrue(list(link.iterdir())) - - @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") - def test_symlink_to_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - # linking to another path. - q = P / 'dirA' / 'fileAA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.symlink_to(p) - - def test_is_junction(self): - P = self.cls(BASE) - - with mock.patch.object(P.pathmod, 'isjunction'): - self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) - P.pathmod.isjunction.assert_called_once_with(P) - - @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") - @unittest.skipIf(sys.platform == "vxworks", - "fifo requires special path on VxWorks") - def test_is_fifo_true(self): - P = self.cls(BASE, 'myfifo') - try: - os.mkfifo(str(P)) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) - self.assertTrue(P.is_fifo()) - self.assertFalse(P.is_socket()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'myfifo\udfff').is_fifo(), False) - self.assertIs(self.cls(BASE, 'myfifo\x00').is_fifo(), False) - - @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") - @unittest.skipIf( - is_emscripten, "Unix sockets are not implemented on Emscripten." - ) - @unittest.skipIf( - is_wasi, "Cannot create socket on WASI." - ) - def test_is_socket_true(self): - P = self.cls(BASE, 'mysock') - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.addCleanup(sock.close) - try: - sock.bind(str(P)) - except OSError as e: - if (isinstance(e, PermissionError) or - "AF_UNIX path too long" in str(e)): - self.skipTest("cannot bind Unix socket: " + str(e)) - self.assertTrue(P.is_socket()) - self.assertFalse(P.is_fifo()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'mysock\udfff').is_socket(), False) - self.assertIs(self.cls(BASE, 'mysock\x00').is_socket(), False) - - def test_is_char_device_true(self): - # Under Unix, /dev/null should generally be a char device. - P = self.cls('/dev/null') - if not P.exists(): - self.skipTest("/dev/null required") - self.assertTrue(P.is_char_device()) - self.assertFalse(P.is_block_device()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) - self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) - - def test_is_mount_root(self): - if os.name == 'nt': - R = self.cls('c:\\') - else: - R = self.cls('/') - self.assertTrue(R.is_mount()) - self.assertFalse((R / '\udfff').is_mount()) - - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): - self.cls(foo="bar") - - -class WalkTests(unittest.TestCase): - - def setUp(self): - self.addCleanup(os_helper.rmtree, os_helper.TESTFN) - + def setUpWalk(self): # Build: # TESTFN/ # TEST1/ a file kid and two directory kids @@ -3208,64 +2651,42 @@ def setUp(self): # SUB11/ no kids # SUB2/ a file kid and a dirsymlink kid # tmp3 - # SUB21/ not readable - # tmp5 # link/ a symlink to TEST2 # broken_link # broken_link2 - # broken_link3 # TEST2/ # tmp4 a lone file - self.walk_path = pathlib.Path(os_helper.TESTFN, "TEST1") + self.walk_path = self.cls(self.base, "TEST1") self.sub1_path = self.walk_path / "SUB1" self.sub11_path = self.sub1_path / "SUB11" self.sub2_path = self.walk_path / "SUB2" - sub21_path= self.sub2_path / "SUB21" tmp1_path = self.walk_path / "tmp1" tmp2_path = self.sub1_path / "tmp2" tmp3_path = self.sub2_path / "tmp3" - tmp5_path = sub21_path / "tmp3" self.link_path = self.sub2_path / "link" - t2_path = pathlib.Path(os_helper.TESTFN, "TEST2") - tmp4_path = pathlib.Path(os_helper.TESTFN, "TEST2", "tmp4") + t2_path = self.cls(self.base, "TEST2") + tmp4_path = self.cls(self.base, "TEST2", "tmp4") broken_link_path = self.sub2_path / "broken_link" broken_link2_path = self.sub2_path / "broken_link2" - broken_link3_path = self.sub2_path / "broken_link3" - os.makedirs(self.sub11_path) - os.makedirs(self.sub2_path) - os.makedirs(sub21_path) - os.makedirs(t2_path) + self.sub11_path.mkdir(parents=True) + self.sub2_path.mkdir(parents=True) + t2_path.mkdir(parents=True) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: - with open(path, "x", encoding='utf-8') as f: + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with path.open("w", encoding='utf-8') as f: f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") - if os_helper.can_symlink(): - os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) - os.symlink(pathlib.Path('tmp3', 'broken'), broken_link2_path, True) - os.symlink(pathlib.Path('SUB21', 'tmp5'), broken_link3_path, True) - self.sub2_tree = (self.sub2_path, ["SUB21"], - ["broken_link", "broken_link2", "broken_link3", - "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, ["SUB21"], ["tmp3"]) - - if not is_emscripten: - # Emscripten fails with inaccessible directories. - os.chmod(sub21_path, 0) - try: - os.listdir(sub21_path) - except PermissionError: - self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU) + if self.can_symlink: + self.link_path.symlink_to(t2_path) + broken_link_path.symlink_to('broken') + broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) else: - os.chmod(sub21_path, stat.S_IRWXU) - os.unlink(tmp5_path) - os.rmdir(sub21_path) - del self.sub2_tree[1][:1] + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) def test_walk_topdown(self): + self.setUpWalk() walker = self.walk_path.walk() entry = next(walker) entry[1].sort() # Ensure we visit SUB1 before SUB2 @@ -3281,12 +2702,11 @@ def test_walk_topdown(self): with self.assertRaises(StopIteration): next(walker) - def test_walk_prune(self, walk_path=None): - if walk_path is None: - walk_path = self.walk_path + def test_walk_prune(self): + self.setUpWalk() # Prune the search. all = [] - for root, dirs, files in walk_path.walk(): + for root, dirs, files in self.walk_path.walk(): all.append((root, dirs, files)) if 'SUB1' in dirs: # Note that this also mutates the dirs we appended to all! @@ -3299,10 +2719,8 @@ def test_walk_prune(self, walk_path=None): all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) - def test_file_like_path(self): - self.test_walk_prune(FakePath(self.walk_path).__fspath__()) - def test_walk_bottom_up(self): + self.setUpWalk() seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False for path, dirnames, filenames in self.walk_path.walk(top_down=False): if path == self.walk_path: @@ -3335,8 +2753,9 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) - @os_helper.skip_unless_symlink + @needs_symlinks def test_walk_follow_symlinks(self): + self.setUpWalk() walk_it = self.walk_path.walk(follow_symlinks=True) for root, dirs, files in walk_it: if root == self.link_path: @@ -3346,8 +2765,9 @@ def test_walk_follow_symlinks(self): else: self.fail("Didn't follow symlink with follow_symlinks=True") - @os_helper.skip_unless_symlink + @needs_symlinks def test_walk_symlink_location(self): + self.setUpWalk() # Tests whether symlinks end up in filenames or dirnames depending # on the `follow_symlinks` argument. walk_it = self.walk_path.walk(follow_symlinks=False) @@ -3363,429 +2783,33 @@ def test_walk_symlink_location(self): if root == self.sub2_path: self.assertIn("link", dirs) break - - def test_walk_bad_dir(self): - errors = [] - walk_it = self.walk_path.walk(on_error=errors.append) - root, dirs, files = next(walk_it) - self.assertEqual(errors, []) - dir1 = 'SUB1' - path1 = root / dir1 - path1new = (root / dir1).with_suffix(".new") - path1.rename(path1new) - try: - roots = [r for r, _, _ in walk_it] - self.assertTrue(errors) - self.assertNotIn(path1, roots) - self.assertNotIn(path1new, roots) - for dir2 in dirs: - if dir2 != dir1: - self.assertIn(root / dir2, roots) - finally: - path1new.rename(path1) - - def test_walk_many_open_files(self): - depth = 30 - base = pathlib.Path(os_helper.TESTFN, 'deep') - path = pathlib.Path(base, *(['d']*depth)) - path.mkdir(parents=True) - - iters = [base.walk(top_down=False) for _ in range(100)] - for i in range(depth + 1): - expected = (path, ['d'] if i else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path.parent - - iters = [base.walk(top_down=True) for _ in range(100)] - path = base - for i in range(depth + 1): - expected = (path, ['d'] if i < depth else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path / 'd' - - def test_walk_above_recursion_limit(self): - recursion_limit = 40 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = pathlib.Path(os_helper.TESTFN, 'deep') - path = pathlib.Path(base, *(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) - - -@only_posix -class PosixPathTest(PathTest): - cls = pathlib.PosixPath - - def test_absolute(self): - P = self.cls - self.assertEqual(str(P('/').absolute()), '/') - self.assertEqual(str(P('/a').absolute()), '/a') - self.assertEqual(str(P('/a/b').absolute()), '/a/b') - - # '//'-prefixed absolute path (supported by POSIX). - self.assertEqual(str(P('//').absolute()), '//') - self.assertEqual(str(P('//a').absolute()), '//a') - self.assertEqual(str(P('//a/b').absolute()), '//a/b') - - @unittest.skipIf( - is_emscripten or is_wasi, - "umask is not implemented on Emscripten/WASI." - ) - def test_open_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - with (p / 'new_file').open('wb'): - pass - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - with (p / 'other_new_file').open('wb'): - pass - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - - def test_resolve_root(self): - current_directory = os.getcwd() - try: - os.chdir('/') - p = self.cls('spam') - self.assertEqual(str(p.resolve()), '/spam') - finally: - os.chdir(current_directory) - - @unittest.skipIf( - is_emscripten or is_wasi, - "umask is not implemented on Emscripten/WASI." - ) - def test_touch_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - (p / 'new_file').touch() - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - (p / 'other_new_file').touch() - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(join('masked_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - - def test_glob(self): - P = self.cls - p = P(BASE) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - - @unittest.skipUnless(hasattr(pwd, 'getpwall'), - 'pwd module does not expose getpwall()') - @unittest.skipIf(sys.platform == "vxworks", - "no home directory on VxWorks") - def test_expanduser(self): - P = self.cls - import_helper.import_module('pwd') - import pwd - pwdent = pwd.getpwuid(os.getuid()) - username = pwdent.pw_name - userhome = pwdent.pw_dir.rstrip('/') or '/' - # Find arbitrary different user (if exists). - for pwdent in pwd.getpwall(): - othername = pwdent.pw_name - otherhome = pwdent.pw_dir.rstrip('/') - if othername != username and otherhome: - break else: - othername = username - otherhome = userhome - - fakename = 'fakeuser' - # This user can theoretically exist on a test runner. Create unique name: - try: - while pwd.getpwnam(fakename): - fakename += '1' - except KeyError: - pass # Non-existent name found - - p1 = P('~/Documents') - p2 = P(f'~{username}/Documents') - p3 = P(f'~{othername}/Documents') - p4 = P(f'../~{username}/Documents') - p5 = P(f'/~{username}/Documents') - p6 = P('') - p7 = P(f'~{fakename}/Documents') - - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - - self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - env['HOME'] = '/tmp' - self.assertEqual(p1.expanduser(), P('/tmp/Documents')) - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - @unittest.skipIf(sys.platform != "darwin", - "Bad file descriptor in /dev/fd affects only macOS") - def test_handling_bad_descriptor(self): - try: - file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] - if not file_descriptors: - self.skipTest("no file descriptors - issue was not reproduced") - # Checking all file descriptors because there is no guarantee - # which one will fail. - for f in file_descriptors: - f.exists() - f.is_dir() - f.is_file() - f.is_symlink() - f.is_block_device() - f.is_char_device() - f.is_fifo() - f.is_socket() - except OSError as e: - if e.errno == errno.EBADF: - self.fail("Bad file descriptor not handled.") - raise - - def test_from_uri(self): - P = self.cls - self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) - self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) - self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) - self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) - self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) - self.assertRaises(ValueError, P.from_uri, 'foo/bar') - self.assertRaises(ValueError, P.from_uri, '/foo/bar') - self.assertRaises(ValueError, P.from_uri, '//foo/bar') - self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') - self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - - def test_from_uri_pathname2url(self): - P = self.cls - self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) - self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) - - -@only_nt -class WindowsPathTest(PathTest): - cls = pathlib.WindowsPath - - def test_absolute(self): - P = self.cls - - # Simple absolute paths. - self.assertEqual(str(P('c:\\').absolute()), 'c:\\') - self.assertEqual(str(P('c:\\a').absolute()), 'c:\\a') - self.assertEqual(str(P('c:\\a\\b').absolute()), 'c:\\a\\b') - - # UNC absolute paths. - share = '\\\\server\\share\\' - self.assertEqual(str(P(share).absolute()), share) - self.assertEqual(str(P(share + 'a').absolute()), share + 'a') - self.assertEqual(str(P(share + 'a\\b').absolute()), share + 'a\\b') - - # UNC relative paths. - with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = share - - self.assertEqual(str(P().absolute()), share) - self.assertEqual(str(P('.').absolute()), share) - self.assertEqual(str(P('a').absolute()), os.path.join(share, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), - os.path.join(share, 'a', 'b', 'c')) - - drive = os.path.splitdrive(BASE)[0] - with os_helper.change_cwd(BASE): - # Relative path with root - self.assertEqual(str(P('\\').absolute()), drive + '\\') - self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') - - # Relative path on current drive - self.assertEqual(str(P(drive).absolute()), BASE) - self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) - - with os_helper.subst_drive(BASE) as other_drive: - # Set the working directory on the substitute drive - saved_cwd = os.getcwd() - other_cwd = f'{other_drive}\\dirA' - os.chdir(other_cwd) - os.chdir(saved_cwd) - - # Relative path on another drive - self.assertEqual(str(P(other_drive).absolute()), other_cwd) - self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') - - def test_glob(self): - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA") }) - self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - def test_expanduser(self): - P = self.cls - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - env.pop('USERPROFILE', None) - env.pop('HOMEPATH', None) - env.pop('HOMEDRIVE', None) - env['USERNAME'] = 'alice' - - # test that the path returns unchanged - p1 = P('~/My Documents') - p2 = P('~alice/My Documents') - p3 = P('~bob/My Documents') - p4 = P('/~/My Documents') - p5 = P('d:~/My Documents') - p6 = P('') - self.assertRaises(RuntimeError, p1.expanduser) - self.assertRaises(RuntimeError, p2.expanduser) - self.assertRaises(RuntimeError, p3.expanduser) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - def check(): - env.pop('USERNAME', None) - self.assertEqual(p1.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertRaises(RuntimeError, p2.expanduser) - env['USERNAME'] = 'alice' - self.assertEqual(p2.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertEqual(p3.expanduser(), - P('C:/Users/bob/My Documents')) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - env['HOMEPATH'] = 'C:\\Users\\alice' - check() - - env['HOMEDRIVE'] = 'C:\\' - env['HOMEPATH'] = 'Users\\alice' - check() - - env.pop('HOMEDRIVE', None) - env.pop('HOMEPATH', None) - env['USERPROFILE'] = 'C:\\Users\\alice' - check() - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = 'C:\\Users\\eve' - check() - - def test_from_uri(self): - P = self.cls - # DOS drive paths - self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) - # UNC paths - self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) - self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) - self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) - # Localhost paths - self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) - # Invalid paths - self.assertRaises(ValueError, P.from_uri, 'foo/bar') - self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') - self.assertRaises(ValueError, P.from_uri, '//foo/bar') - self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') - self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - - def test_from_uri_pathname2url(self): - P = self.cls - self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) - - -class PathSubclassTest(PathTest): - class cls(pathlib.Path): - pass - - # repr() roundtripping is not supported in custom subclass. - test_repr_roundtrips = None - - -class CompatiblePathTest(unittest.TestCase): - """ - Test that a type can be made compatible with PurePath - derivatives by implementing division operator overloads. - """ + self.fail("symlink not found") - class CompatPath: - """ - Minimum viable class to test PurePath compatibility. - Simply uses the division operator to join a given - string and the string value of another object with - a forward slash. - """ - def __init__(self, string): - self.string = string - def __truediv__(self, other): - return type(self)(f"{self.string}/{other}") +class DummyPathWithSymlinks(DummyPath): + __slots__ = () - def __rtruediv__(self, other): - return type(self)(f"{other}/{self.string}") + # Reduce symlink traversal limit to make tests run faster. + _max_symlinks = 20 - def test_truediv(self): - result = pathlib.PurePath("test") / self.CompatPath("right") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "test/right") + def readlink(self): + path = str(self.parent.resolve() / self.name) + if path in self._symlinks: + return self.with_segments(self._symlinks[path]) + elif path in self._files or path in self._directories: + raise OSError(errno.EINVAL, "Not a symlink", path) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - pathlib.PurePath("test") / 10 + def symlink_to(self, target, target_is_directory=False): + self._directories[str(self.parent)].add(self.name) + self._symlinks[str(self)] = str(target) - def test_rtruediv(self): - result = self.CompatPath("left") / pathlib.PurePath("test") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "left/test") - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - 10 / pathlib.PurePath("test") +class DummyPathWithSymlinksTest(DummyPathTest): + cls = DummyPathWithSymlinks + can_symlink = True if __name__ == "__main__": diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index dedbc828784184..1bdab125dc6ef0 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2760,6 +2760,132 @@ def test_patma_255(self): self.assertEqual(y, 1) self.assertIs(z, x) + def test_patma_runtime_checkable_protocol(self): + # Runtime-checkable protocol + from typing import Protocol, runtime_checkable + + @runtime_checkable + class P(Protocol): + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + + def test_patma_generic_protocol(self): + # Runtime-checkable generic protocol + from typing import Generic, TypeVar, Protocol, runtime_checkable + + T = TypeVar('T') # not using PEP695 to be able to backport changes + + @runtime_checkable + class P(Protocol[T]): + a: T + b: T + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class G(Generic[T]): + def __init__(self, x: T, y: T): + self.x = x + self.y = y + + for cls in (A, G): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P(): + w = 1 + self.assertEqual(w, 0) + + def test_patma_protocol_with_match_args(self): + # Runtime-checkable protocol with `__match_args__` + from typing import Protocol, runtime_checkable + + # Used to fail before + # https://github.com/python/cpython/issues/110682 + @runtime_checkable + class P(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + j = 0 + match inst: + case P(x=1, y=2): + j = 1 + self.assertEqual(j, 1) + + g = 0 + match inst: + case P(x, y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + g = 1 + self.assertEqual(g, 1) + + h = 0 + match inst: + case P(1, 2): + h = 1 + self.assertEqual(h, 1) + class TestSyntaxErrors(unittest.TestCase): @@ -2831,6 +2957,14 @@ def test_invalid_syntax_3(self): pass """) + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): self.assert_syntax_error(""" match ...: @@ -3198,6 +3332,35 @@ def test_class_pattern_not_type(self): w = 0 self.assertIsNone(w) + def test_regular_protocol(self): + from typing import Protocol + class P(Protocol): ... + msg = ( + 'Instance and class checks can only be used ' + 'with @runtime_checkable protocols' + ) + w = None + with self.assertRaisesRegex(TypeError, msg): + match 1: + case P(): + w = 0 + self.assertIsNone(w) + + def test_positional_patterns_with_regular_protocol(self): + from typing import Protocol + class P(Protocol): + x: int # no `__match_args__` + y: int + class A: + x = 1 + y = 2 + w = None + with self.assertRaises(TypeError): + match A(): + case P(x, y): + w = 0 + self.assertIsNone(w) + class TestValueErrors(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index ff677aeb795442..f71856d3603d73 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -10,15 +10,19 @@ import subprocess import textwrap import linecache +import zipapp from contextlib import ExitStack, redirect_stdout from io import StringIO from test import support -from test.support import os_helper -# This little helper class is essential for testing pdb under doctest. -from test.test_doctest import _FakeInput +from test.support import force_not_colorized, os_helper +from test.support.import_helper import import_module +from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch +# gh-114275: WASI fails to run asyncio tests, similar skip than test_asyncio. +SKIP_ASYNCIO_TESTS = (not support.has_socket_support) + class PdbTestInput(object): """Context manager that makes testing Pdb in doctests easier.""" @@ -28,7 +32,7 @@ def __init__(self, input): def __enter__(self): self.real_stdin = sys.stdin - sys.stdin = _FakeInput(self.input) + sys.stdin = FakeInput(self.input) self.orig_trace = sys.gettrace() if hasattr(sys, 'gettrace') else None def __exit__(self, *exc): @@ -42,7 +46,6 @@ def test_pdb_displayhook(): >>> def test_function(foo, bar): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'foo', @@ -51,8 +54,8 @@ def test_pdb_displayhook(): ... 'continue', ... ]): ... test_function(1, None) - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) foo 1 (Pdb) bar @@ -94,6 +97,7 @@ def test_pdb_basic_commands(): ... print(ret) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to line ret = test_function_2('baz') ... 'step', # entering the function call ... 'args', # display function args ... 'list', # list function source @@ -118,6 +122,9 @@ def test_pdb_basic_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) step @@ -141,7 +148,7 @@ def test_pdb_basic_commands(): [EOF] (Pdb) bt ... - (25)() + (26)() -> test_function() (3)test_function() -> ret = test_function_2('baz') @@ -251,6 +258,8 @@ def test_pdb_breakpoint_commands(): ... 'clear 3', ... 'break', ... 'condition 1', + ... 'commands 1', + ... 'EOF', # Simulate Ctrl-D/Ctrl-Z from user, should end input ... 'enable 1', ... 'clear 1', ... 'commands 2', @@ -271,8 +280,8 @@ def test_pdb_breakpoint_commands(): ... 'continue', ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) break 4, + @@ -306,6 +315,9 @@ def test_pdb_breakpoint_commands(): 2 breakpoint keep yes at :4 (Pdb) condition 1 Breakpoint 1 is now unconditional. + (Pdb) commands 1 + (com) EOF + (Pdb) enable 1 Enabled breakpoint 1 at :3 (Pdb) clear 1 @@ -351,6 +363,46 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_breakpoint_with_filename(): + """Breakpoints with filename:lineno + + >>> def test_function(): + ... # inspect_fodder2 is a great module as the line number is stable + ... from test.test_inspect import inspect_fodder2 as mod2 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... mod2.func88() + ... mod2.func114() + ... # Be a good citizen and clean up the mess + ... reset_Breakpoint() + + First, need to clear bdb state that might be left over from previous tests. + Otherwise, the new breakpoints might get assigned different numbers. + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS + ... 'break test.test_inspect.inspect_fodder2:90', + ... 'continue', # will stop at func88 + ... 'break test/test_inspect/inspect_fodder2.py:115', + ... 'continue', # will stop at func114 + ... 'continue', + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break test.test_inspect.inspect_fodder2:90 + Breakpoint 1 at ...inspect_fodder2.py:90 + (Pdb) continue + > ...inspect_fodder2.py(90)func88() + -> return 90 + (Pdb) break test/test_inspect/inspect_fodder2.py:115 + Breakpoint 2 at ...inspect_fodder2.py:115 + (Pdb) continue + > ...inspect_fodder2.py(115)func114() + -> return 115 + (Pdb) continue + """ + def test_pdb_breakpoints_preserved_across_interactive_sessions(): """Breakpoints are remembered between interactive sessions @@ -435,8 +487,7 @@ def test_pdb_pp_repr_exc(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p obj *** Exception: repr_exc @@ -445,6 +496,37 @@ def test_pdb_pp_repr_exc(): (Pdb) continue """ +def test_pdb_empty_line(): + """Test that empty line repeats the last command. + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... y = 2 + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'p x', + ... '', # Should repeat p x + ... 'n ;; p 0 ;; p x', # Fill cmdqueue with multiple commands + ... '', # Should still repeat p x + ... 'continue', + ... ]): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p x + 1 + (Pdb) + 1 + (Pdb) n ;; p 0 ;; p x + 0 + 1 + > (4)test_function() + -> y = 2 + (Pdb) + 1 + (Pdb) continue + """ def do_nothing(): pass @@ -476,6 +558,7 @@ def test_list_commands(): ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to the test function line ... 'list', # list first function ... 'step', # step into second function ... 'list', # list second function @@ -491,6 +574,9 @@ def test_list_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) list @@ -571,8 +657,7 @@ def test_pdb_whatis_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) whatis myvar @@ -601,6 +686,7 @@ def test_pdb_display_command(): ... a = 4 >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'display +', ... 'display', ... 'display a', @@ -616,6 +702,9 @@ def test_pdb_display_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> a = 1 (Pdb) display + @@ -664,6 +753,7 @@ def test_pdb_alias_command(): ... o.method() >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'alias pi', ... 'alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}")', ... 'alias ps pi self', @@ -671,9 +761,20 @@ def test_pdb_alias_command(): ... 'pi o', ... 's', ... 'ps', + ... 'alias myp p %2', + ... 'alias myp', + ... 'alias myp p %1', + ... 'myp', + ... 'myp 1', + ... 'myp 1 2', + ... 'alias repeat_second_arg p "%* %2"', + ... 'repeat_second_arg 1 2 3', ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> o.method() (Pdb) alias pi @@ -692,6 +793,20 @@ def test_pdb_alias_command(): (Pdb) ps self.attr1 = 10 self.attr2 = str + (Pdb) alias myp p %2 + *** Replaceable parameters must be consecutive + (Pdb) alias myp + *** Unknown alias 'myp' + (Pdb) alias myp p %1 + (Pdb) myp + *** Not enough arguments for alias 'myp' + (Pdb) myp 1 + 1 + (Pdb) myp 1 2 + *** Too many arguments for alias 'myp' + (Pdb) alias repeat_second_arg p "%* %2" + (Pdb) repeat_second_arg 1 2 3 + '1 2 3 2' (Pdb) continue """ @@ -702,7 +817,7 @@ def test_pdb_where_command(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() >>> def f(): - ... g(); + ... g() >>> def test_function(): ... f() @@ -710,50 +825,138 @@ def test_pdb_where_command(): >>> with PdbTestInput([ # doctest: +ELLIPSIS ... 'w', ... 'where', + ... 'w 1', + ... 'w invalid', ... 'u', ... 'w', + ... 'w 0', + ... 'w 100', + ... 'w -100', ... 'continue', ... ]): ... test_function() - --Return-- - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 1 + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w invalid + *** Invalid count (invalid) (Pdb) u > (2)f() - -> g(); + -> g() (Pdb) w ... - (8)() + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 0 + > (2)f() + -> g() + (Pdb) w 100 + ... + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w -100 + ... + (13)() -> test_function() (2)test_function() -> f() > (2)f() - -> g(); - (2)g()->None + -> g() + (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ + +# skip this test if sys.flags.no_site = True; +# exit() isn't defined unless there's a site module. +if not sys.flags.no_site: + def test_pdb_interact_command(): + """Test interact command + + >>> g = 0 + >>> dict_g = {} + + >>> def test_function(): + ... x = 1 + ... lst_local = [] + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'interact', + ... 'x', + ... 'g', + ... 'x = 2', + ... 'g = 3', + ... 'dict_g["a"] = True', + ... 'lst_local.append(x)', + ... 'exit()', + ... 'p x', + ... 'p g', + ... 'p dict_g', + ... 'p lst_local', + ... 'continue', + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) interact + *pdb interact start* + ... x + 1 + ... g + 0 + ... x = 2 + ... g = 3 + ... dict_g["a"] = True + ... lst_local.append(x) + ... exit() + *exit from pdb interact command* + (Pdb) p x + 1 + (Pdb) p g + 0 + (Pdb) p dict_g + {'a': True} + (Pdb) p lst_local + [2] + (Pdb) continue + """ + def test_convenience_variables(): """Test convenience variables @@ -769,10 +972,14 @@ def test_convenience_variables(): ... util_function() >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # Step to try statement ... '$_frame.f_lineno', # Check frame convenience variable + ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable ... '$a', # Print its value + ... 'p "$a"', # Print the string $a ... 'p $a + 2', # Do some calculation + ... 'p f"$a = {$a}"', # Make sure $ in string is not converted and f-string works ... 'u', # Switch frame ... '$_frame.f_lineno', # Make sure the frame changed ... '$a', # Make sure the value persists @@ -788,15 +995,24 @@ def test_convenience_variables(): ... 'continue', ... ]): ... test_function() + > (2)util_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)util_function() -> try: (Pdb) $_frame.f_lineno 3 + (Pdb) $ _frame + *** SyntaxError: invalid syntax (Pdb) $a = 10 (Pdb) $a 10 + (Pdb) p "$a" + '$a' (Pdb) p $a + 2 12 + (Pdb) p f"$a = {$a}" + '$a = 10' (Pdb) u > (2)test_function() -> util_function() @@ -1255,6 +1471,7 @@ def test_post_mortem(): ... print('Not reached.') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace ... 'list', # list code of test_function() @@ -1266,6 +1483,9 @@ def test_post_mortem(): ... test_function() ... except ZeroDivisionError: ... print('Correctly reraised.') + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> test_function_2() (Pdb) next @@ -1275,7 +1495,7 @@ def test_post_mortem(): -> test_function_2() (Pdb) bt ... - (10)() + (11)() -> test_function() > (3)test_function() -> test_function_2() @@ -1302,6 +1522,58 @@ def test_post_mortem(): """ +def test_pdb_return_to_different_file(): + """When pdb returns to a different file, it should not skip if f_trace is + not already set + + >>> import pprint + + >>> class A: + ... def __repr__(self): + ... return 'A' + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pprint.pprint(A()) + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'b A.__repr__', + ... 'continue', + ... 'return', + ... 'next', + ... 'return', + ... 'return', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b A.__repr__ + Breakpoint 1 at :3 + (Pdb) continue + > (3)__repr__() + -> return 'A' + (Pdb) return + --Return-- + > (3)__repr__()->'A' + -> return 'A' + (Pdb) next + > ...pprint.py..._safe_repr() + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py..._safe_repr()->('A'...) + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py...format()->('A'...) + -> return... + (Pdb) continue + A + """ + + def test_pdb_skip_modules(): """This illustrates the simple case of module skipping. @@ -1312,9 +1584,13 @@ def test_pdb_skip_modules(): >>> with PdbTestInput([ ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() + > (3)skip_module() + -> import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (4)skip_module() -> string.capwords('FOO') (Pdb) step @@ -1329,7 +1605,6 @@ def test_pdb_invalid_arg(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'a = 3', @@ -1338,8 +1613,8 @@ def test_pdb_invalid_arg(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) a = 3 *** Invalid argument: = 3 Usage: a(rgs) @@ -1373,10 +1648,14 @@ def test_pdb_skip_modules_with_callback(): ... 'step', ... 'step', ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() ... pass # provides something to "step" to + > (4)skip_module() + -> import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (5)skip_module() -> mod.foo_pony(callback) (Pdb) step @@ -1395,7 +1674,7 @@ def test_pdb_skip_modules_with_callback(): > (5)skip_module()->None -> mod.foo_pony(callback) (Pdb) step - > (10)() + > (11)() -> pass # provides something to "step" to (Pdb) continue """ @@ -1414,6 +1693,7 @@ def test_pdb_continue_in_bottomframe(): ... print(4) >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'step', ... 'next', ... 'break 7', ... 'continue', @@ -1422,6 +1702,9 @@ def test_pdb_continue_in_bottomframe(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> inst.set_trace() + (Pdb) step > (4)test_function() -> inst.botframe = sys._getframe() # hackery to get the right botframe (Pdb) next @@ -1513,10 +1796,10 @@ def test_next_until_return_at_return_event(): ... 'return', ... 'continue']): ... test_function() - > (3)test_function() - -> test_function_2() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_function_2 - Breakpoint 1 at :1 + Breakpoint 1 at :2 (Pdb) continue > (2)test_function_2() -> x = 1 @@ -1573,12 +1856,16 @@ def test_pdb_next_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1609,122 +1896,131 @@ def test_pdb_next_command_for_generator(): finished """ -def test_pdb_next_command_for_coroutine(): - """Testing skip unwindng stack on yield for coroutines for "next" command - - >>> import asyncio - - >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'next', - ... 'next', - ... 'step', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (3)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (4)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - Internal StopIteration - > (3)test_main() - -> await test_coro() - (Pdb) step - --Return-- - > (3)test_main()->None - -> await test_coro() - (Pdb) continue - finished - """ - -def test_pdb_next_command_for_asyncgen(): - """Testing skip unwindng stack on yield for coroutines for "next" command - - >>> import asyncio - - >>> async def agen(): - ... yield 1 - ... await asyncio.sleep(0) - ... yield 2 - - >>> async def test_coro(): - ... async for x in agen(): - ... print(x) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") +if not SKIP_ASYNCIO_TESTS: + def test_pdb_next_command_for_coroutine(): + """Testing skip unwindng stack on yield for coroutines for "next" command + + >>> import asyncio + + >>> async def test_coro(): + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'step', + ... 'next', + ... 'next', + ... 'next', + ... 'step', + ... 'continue']): + ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (3)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (4)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + Internal StopIteration + > (3)test_main() + -> await test_coro() + (Pdb) step + --Return-- + > (3)test_main()->None + -> await test_coro() + (Pdb) continue + finished + """ - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'next', - ... 'step', - ... 'next', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> async for x in agen(): - (Pdb) next - > (3)test_coro() - -> print(x) - (Pdb) next - 1 - > (2)test_coro() - -> async for x in agen(): - (Pdb) step - --Call-- - > (2)agen() - -> yield 1 - (Pdb) next - > (3)agen() - -> await asyncio.sleep(0) - (Pdb) continue - 2 - finished - """ + def test_pdb_next_command_for_asyncgen(): + """Testing skip unwindng stack on yield for coroutines for "next" command + + >>> import asyncio + + >>> async def agen(): + ... yield 1 + ... await asyncio.sleep(0) + ... yield 2 + + >>> async def test_coro(): + ... async for x in agen(): + ... print(x) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'step', + ... 'next', + ... 'next', + ... 'step', + ... 'next', + ... 'continue']): + ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> async for x in agen(): + (Pdb) next + > (3)test_coro() + -> print(x) + (Pdb) next + 1 + > (2)test_coro() + -> async for x in agen(): + (Pdb) step + --Call-- + > (2)agen() + -> yield 1 + (Pdb) next + > (3)agen() + -> await asyncio.sleep(0) + (Pdb) continue + 2 + finished + """ def test_pdb_return_command_for_generator(): """Testing no unwindng stack on yield for generators @@ -1750,11 +2046,15 @@ def test_pdb_return_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'return', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1781,47 +2081,52 @@ def test_pdb_return_command_for_generator(): finished """ -def test_pdb_return_command_for_coroutine(): - """Testing no unwindng stack on yield for coroutines for "return" command - - >>> import asyncio - - >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'step', - ... 'next', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) step - > (2)test_coro() - -> await asyncio.sleep(0) - (Pdb) next - > (3)test_coro() - -> await asyncio.sleep(0) - (Pdb) continue - finished - """ +if not SKIP_ASYNCIO_TESTS: + def test_pdb_return_command_for_coroutine(): + """Testing no unwindng stack on yield for coroutines for "return" command + + >>> import asyncio + + >>> async def test_coro(): + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + ... await asyncio.sleep(0) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'step', + ... 'next', + ... 'continue']): + ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) step + > (2)test_coro() + -> await asyncio.sleep(0) + (Pdb) next + > (3)test_coro() + -> await asyncio.sleep(0) + (Pdb) continue + finished + """ def test_pdb_until_command_for_generator(): """Testing no unwindng stack on yield for generators @@ -1839,11 +2144,15 @@ def test_pdb_until_command_for_generator(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 4', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -1867,52 +2176,57 @@ def test_pdb_until_command_for_generator(): finished """ -def test_pdb_until_command_for_coroutine(): - """Testing no unwindng stack for coroutines - for "until" command if target breakpoint is not reached - - >>> import asyncio - - >>> async def test_coro(): - ... print(0) - ... await asyncio.sleep(0) - ... print(1) - ... await asyncio.sleep(0) - ... print(2) - ... await asyncio.sleep(0) - ... print(3) - - >>> async def test_main(): - ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... await test_coro() - - >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) - ... print("finished") - - >>> with PdbTestInput(['step', - ... 'until 8', - ... 'continue']): - ... test_function() - > (3)test_main() - -> await test_coro() - (Pdb) step - --Call-- - > (1)test_coro() - -> async def test_coro(): - (Pdb) until 8 - 0 - 1 - 2 - > (8)test_coro() - -> print(3) - (Pdb) continue - 3 - finished - """ +if not SKIP_ASYNCIO_TESTS: + def test_pdb_until_command_for_coroutine(): + """Testing no unwindng stack for coroutines + for "until" command if target breakpoint is not reached + + >>> import asyncio + + >>> async def test_coro(): + ... print(0) + ... await asyncio.sleep(0) + ... print(1) + ... await asyncio.sleep(0) + ... print(2) + ... await asyncio.sleep(0) + ... print(3) + + >>> async def test_main(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... await test_coro() + + >>> def test_function(): + ... loop = asyncio.new_event_loop() + ... loop.run_until_complete(test_main()) + ... loop.close() + ... asyncio.set_event_loop_policy(None) + ... print("finished") + + >>> with PdbTestInput(['step', + ... 'step', + ... 'until 8', + ... 'continue']): + ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step + > (3)test_main() + -> await test_coro() + (Pdb) step + --Call-- + > (1)test_coro() + -> async def test_coro(): + (Pdb) until 8 + 0 + 1 + 2 + > (8)test_coro() + -> print(3) + (Pdb) continue + 3 + finished + """ def test_pdb_next_command_in_generator_for_loop(): """The next command on returning from a generator controlled by a for loop. @@ -1935,10 +2249,10 @@ def test_pdb_next_command_in_generator_for_loop(): ... 'next', ... 'continue']): ... test_function() - > (3)test_function() - -> for i in test_gen(): + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_gen - Breakpoint 1 at :1 + Breakpoint 1 at :2 (Pdb) continue > (2)test_gen() -> yield 0 @@ -1975,11 +2289,15 @@ def test_pdb_next_command_subiterator(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2008,7 +2326,6 @@ def test_pdb_multiline_statement(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'def f(x):', @@ -2018,8 +2335,8 @@ def test_pdb_multiline_statement(): ... 'c' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) def f(x): ... return x * 2 ... @@ -2028,29 +2345,91 @@ def test_pdb_multiline_statement(): (Pdb) c """ -def test_pdb_show_attribute_and_item(): - """Test for multiline statement +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + >>> k = 0 + >>> g = 1 >>> def test_function(): - ... n = lambda x: x - ... c = {"a": 1} + ... x = 2 + ... g = 3 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE - ... 'c["a"]', - ... 'c.get("a")', - ... 'n(1)', - ... 'j=1', - ... 'j+1', - ... 'r"a"', - ... 'next(iter([1]))', + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... 'global g; (lambda: g)()', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) global g; (lambda: g)() + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 1 + (Pdb) c + """ + +def test_pdb_show_attribute_and_item(): + """Test for expressions with command prefix + + >>> def test_function(): + ... n = lambda x: x + ... c = {"a": 1} + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'c["a"]', + ... 'c.get("a")', + ... 'n(1)', + ... 'j=1', + ... 'j+1', + ... 'r"a"', + ... 'next(iter([1]))', ... 'list((0, 1))', ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) c["a"] 1 (Pdb) c.get("a") @@ -2069,6 +2448,49 @@ def test_pdb_show_attribute_and_item(): (Pdb) c """ +# doctest will modify pdb.set_trace during the test, so we need to backup +# the original function to use it in the test +original_pdb_settrace = pdb.set_trace + +def test_pdb_with_inline_breakpoint(): + """Hard-coded breakpoint() calls should invoke the same debugger instance + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb().set_trace() + ... original_pdb_settrace() + ... x = 2 + + >>> with PdbTestInput(['display x', + ... 'n', + ... 'n', + ... 'n', + ... 'n', + ... 'undisplay', + ... 'c']): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb().set_trace() + (Pdb) display x + display x: 1 + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (5)test_function() + -> x = 2 + (Pdb) n + --Return-- + > (5)test_function()->None + -> x = 2 + display x: 2 [old: 1] + (Pdb) undisplay + (Pdb) c + """ + def test_pdb_issue_20766(): """Test for reference leaks when the SIGINT handler is set. @@ -2084,12 +2506,12 @@ def test_pdb_issue_20766(): >>> with PdbTestInput(['continue', ... 'continue']): ... test_function() - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 1: - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 2: """ @@ -2110,8 +2532,8 @@ def test_pdb_issue_43318(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) clear :3 @@ -2143,12 +2565,16 @@ def test_pdb_issue_gh_91742(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 5', ... 'continue' ... ]): ... test_function() + > (11)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (12)test_function() -> about() (Pdb) step @@ -2186,6 +2612,7 @@ def test_pdb_issue_gh_94215(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 3', @@ -2198,6 +2625,9 @@ def test_pdb_issue_gh_94215(): ... 'continue' ... ]): ... test_function() + > (8)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (9)test_function() -> func() (Pdb) step @@ -2237,7 +2667,7 @@ def test_pdb_issue_gh_94215(): def test_pdb_issue_gh_101673(): """See GH-101673 - Make sure ll won't revert local variable assignment + Make sure ll and switching frames won't revert local variable assignment >>> def test_function(): ... a = 1 @@ -2247,11 +2677,14 @@ def test_pdb_issue_gh_101673(): ... '!a = 2', ... 'll', ... 'p a', + ... 'u', + ... 'p a', + ... 'd', + ... 'p a', ... 'continue' ... ]): ... test_function() - --Return-- - > (3)test_function()->None + > (3)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) !a = 2 (Pdb) ll @@ -2260,6 +2693,16 @@ def test_pdb_issue_gh_101673(): 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p a 2 + (Pdb) u + > (11)() + -> test_function() + (Pdb) p a + *** NameError: name 'a' is not defined + (Pdb) d + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 (Pdb) continue """ @@ -2275,16 +2718,16 @@ def test_pdb_issue_gh_103225(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... b = 2 - > (7)() - -> b = 2 + > (6)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) longlist 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE 2 'longlist', 3 'continue' 4 ]): 5 a = 1 - 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - 7 -> b = 2 + 6 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 b = 2 (Pdb) continue """ @@ -2303,9 +2746,8 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - --Return-- - > (None)test_function()->None - -> Warning: lineno is None + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2321,18 +2763,42 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - bdb.Bdb.dispatch: unknown debugging event: 'opcode' - > (5)test_function() - -> a = 1 + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ + +def test_pdb_issue_gh_80731(): + """See GH-80731 + + pdb should correctly print exception info if in an except block. + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'import sys', + ... 'sys.exc_info()', + ... 'continue' + ... ]): + ... try: + ... raise ValueError('Correct') + ... except ValueError: + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + > (9)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) import sys + (Pdb) sys.exc_info() + (, ValueError('Correct'), ) + (Pdb) continue + """ + + def test_pdb_ambiguous_statements(): """See GH-104301 Make sure that ambiguous statements prefixed by '!' are properly disambiguated >>> with PdbTestInput([ + ... 's', # step to the print line ... '! n = 42', # disambiguated statement: reassign the name n ... 'n', # advance the debugger into the print() ... 'continue' @@ -2341,6 +2807,9 @@ def test_pdb_ambiguous_statements(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(f"The value of n is {n}") > (8)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s + > (9)() -> print(f"The value of n is {n}") (Pdb) ! n = 42 (Pdb) n @@ -2350,6 +2819,71 @@ def test_pdb_ambiguous_statements(): (Pdb) continue """ +def test_pdb_f_trace_lines(): + """GH-80675 + + pdb should work even if f_trace_lines is set to False on some frames. + + >>> reset_Breakpoint() + + >>> def test_function(): + ... import sys + ... frame = sys._getframe() + ... frame.f_trace_lines = False + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... if frame.f_trace_lines != False: + ... print("f_trace_lines is not reset after continue!") + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue' + ... ]): + ... test_function() + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) continue + """ + +def test_pdb_function_break(): + """Testing the line number of break on function + + >>> def foo(): pass + + >>> def bar(): + ... + ... pass + + >>> def boo(): + ... # comments + ... global x + ... x = 1 + + >>> def gen(): + ... yield 42 + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ... 'break foo', + ... 'break bar', + ... 'break boo', + ... 'break gen', + ... 'continue' + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break foo + Breakpoint ... at :1 + (Pdb) break bar + Breakpoint ... at :3 + (Pdb) break boo + Breakpoint ... at :4 + (Pdb) break gen + Breakpoint ... at :2 + (Pdb) continue + """ + def test_pdb_issue_gh_65052(): """See GH-65052 @@ -2368,6 +2902,7 @@ def test_pdb_issue_gh_65052(): ... A() >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 's', + ... 's', ... 'retval', ... 'continue', ... 'args', @@ -2376,6 +2911,9 @@ def test_pdb_issue_gh_65052(): ... 'continue', ... ]): ... test_function() + > (3)__new__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)__new__() -> return object.__new__(cls) (Pdb) s @@ -2385,8 +2923,8 @@ def test_pdb_issue_gh_65052(): (Pdb) retval *** repr(retval) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) continue - > (7)__init__() - -> self.a = 1 + > (6)__init__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) args self = *** repr(self) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) display self @@ -2405,19 +2943,25 @@ def tearDown(self): @unittest.skipIf(sys.flags.safe_path, 'PYTHONSAFEPATH changes default sys.path') - def _run_pdb(self, pdb_args, commands, expected_returncode=0): + def _run_pdb(self, pdb_args, commands, + expected_returncode=0, + extra_env=None): self.addCleanup(os_helper.rmtree, '__pycache__') cmd = [sys.executable, '-m', 'pdb'] + pdb_args + if extra_env is not None: + env = os.environ | extra_env + else: + env = os.environ with subprocess.Popen( cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, - env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} + stderr=subprocess.PIPE, + env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) - stdout = stdout and bytes.decode(stdout) - stderr = stderr and bytes.decode(stderr) + stdout = bytes.decode(stdout) if isinstance(stdout, bytes) else stdout + stderr = bytes.decode(stderr) if isinstance(stderr, bytes) else stderr self.assertEqual( proc.returncode, expected_returncode, @@ -2425,13 +2969,31 @@ def _run_pdb(self, pdb_args, commands, expected_returncode=0): ) return stdout, stderr - def run_pdb_script(self, script, commands, expected_returncode=0): + def run_pdb_script(self, script, commands, + expected_returncode=0, + extra_env=None, + pdbrc=None, + remove_home=False): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) + + if pdbrc is not None: + with open('.pdbrc', 'w') as f: + f.write(textwrap.dedent(pdbrc)) + self.addCleanup(os_helper.unlink, '.pdbrc') self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode) + + homesave = None + if remove_home: + homesave = os.environ.pop('HOME', None) + try: + stdout, stderr = self._run_pdb([filename], commands, expected_returncode, extra_env) + finally: + if homesave is not None: + os.environ['HOME'] = homesave + return stdout, stderr def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -2472,7 +3034,7 @@ def quux(): pass """.encode(), 'bÅ“r', - ('bÅ“r', 4), + ('bÅ“r', 5), ) def test_find_function_found_with_encoding_cookie(self): @@ -2489,7 +3051,7 @@ def quux(): pass """.encode('iso-8859-15'), 'bÅ“r', - ('bÅ“r', 5), + ('bÅ“r', 6), ) def test_find_function_found_with_bom(self): @@ -2499,9 +3061,46 @@ def bÅ“r(): pass """.encode(), 'bÅ“r', - ('bÅ“r', 1), + ('bÅ“r', 2), ) + def test_spec(self): + # Test that __main__.__spec__ is set to None when running a script + script = """ + import __main__ + print(__main__.__spec__) + """ + + commands = "continue" + + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('None', stdout) + + def test_find_function_first_executable_line(self): + code = textwrap.dedent("""\ + def foo(): pass + + def bar(): + pass # line 4 + + def baz(): + # comment + pass # line 8 + + def mul(): + # code on multiple lines + code = compile( # line 12 + 'def f()', + '', + 'exec', + ) + """).encode() + + self._assert_find_function(code, 'foo', ('foo', 1)) + self._assert_find_function(code, 'bar', ('bar', 4)) + self._assert_find_function(code, 'baz', ('baz', 8)) + self._assert_find_function(code, 'mul', ('mul', 12)) + def test_issue7964(self): # open the file as binary so we can force \r\n newline with open(os_helper.TESTFN, 'wb') as f: @@ -2510,7 +3109,7 @@ def test_issue7964(self): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, ) self.addCleanup(proc.stdout.close) stdout, stderr = proc.communicate(b'quit\n') @@ -2594,7 +3193,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env={**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2624,7 +3223,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2632,18 +3231,36 @@ def start_pdb(): self.assertNotIn(b'Error', stdout, "Got an error running test script under PDB") + @force_not_colorized def test_issue16180(self): # A syntax error in the debuggee. script = "def f: pass\n" commands = '' expected = "SyntaxError:" stdout, stderr = self.run_pdb_script( - script, commands, expected_returncode=1 + script, commands ) - self.assertIn(expected, stdout, + self.assertIn(expected, stderr, '\n\nExpected:\n{}\nGot:\n{}\n' 'Fail to handle a syntax error in the debuggee.' - .format(expected, stdout)) + .format(expected, stderr)) + + @force_not_colorized + def test_issue84583(self): + # A syntax error from ast.literal_eval should not make pdb exit. + script = "import ast; ast.literal_eval('')\n" + commands = """ + continue + where + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + # The code should appear 3 times in the stdout/stderr: + # 1. when pdb starts (stdout) + # 2. when the exception is raised, in trackback (stderr) + # 3. in where command (stdout) + self.assertEqual(stdout.count("ast.literal_eval('')"), 2) + self.assertEqual(stderr.count("ast.literal_eval('')"), 1) def test_issue26053(self): # run command of pdb prompt echoes the correct args @@ -2659,37 +3276,100 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") - def test_readrc_kwarg(self): + def test_pdbrc_basic(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb(readrc=False).set_trace() + a = 1 + b = 2 + """) - print('hello') + pdbrc = textwrap.dedent(""" + # Comments should be fine + n + p f"{a+8=}" """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w') as f: - f.write("invalid\n") - - with open('main.py', 'w') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - with proc: - stdout, stderr = proc.communicate(b'q\n') - self.assertNotIn(b"NameError: name 'invalid' is not defined", - stdout) + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertNotIn("SyntaxError", stdout) + self.assertIn("a+8=9", stdout) + self.assertIn("-> b = 2", stdout) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + def test_pdbrc_empty_line(self): + """Test that empty lines in .pdbrc are ignored.""" + + script = textwrap.dedent(""" + a = 1 + b = 2 + c = 3 + """) + + pdbrc = textwrap.dedent(""" + n + + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("b = 2", stdout) + self.assertNotIn("c = 3", stdout) + + def test_pdbrc_alias(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + until 6 + pi a + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a.attr = 1", stdout) + + def test_pdbrc_semicolon(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 5;;c;;n + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("-> b = 2", stdout) + + def test_pdbrc_commands(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 6 + commands 1 ;; p a;; end + c + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("<__main__.A object at", stdout) + + def test_readrc_kwarg(self): + script = textwrap.dedent(""" + print('hello') + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc='invalid', remove_home=True) + self.assertIn("NameError: name 'invalid' is not defined", stdout) def test_readrc_homedir(self): save_home = os.environ.pop("HOME", None) @@ -2704,46 +3384,17 @@ def test_readrc_homedir(self): if save_home is not None: os.environ["HOME"] = save_home - def test_read_pdbrc_with_ascii_encoding(self): - script = textwrap.dedent(""" - import pdb; pdb.Pdb().set_trace() - print('hello') - """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w', encoding='utf-8') as f: - f.write("Fran\u00E7ais") - - with open('main.py', 'w', encoding='utf-8') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - env = {'PYTHONIOENCODING': 'ascii'} - if sys.platform == 'win32': - env['PYTHONLEGACYWINDOWSSTDIO'] = 'non-empty-string' - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ, **env} - ) - with proc: - stdout, stderr = proc.communicate(b'c\n') - self.assertIn(b"UnicodeEncodeError: \'ascii\' codec can\'t encode character " - b"\'\\xe7\' in position 21: ordinal not in range(128)", stderr) - - finally: - if save_home is not None: - os.environ['HOME'] = save_home - def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' with ExitStack() as resources: resources.enter_context(patch('sys.stdout', stdout)) + # patch pdb.Pdb.set_trace() to avoid entering the debugger resources.enter_context(patch.object(pdb.Pdb, 'set_trace')) + # We need to manually clear pdb.Pdb._last_pdb_instance so a + # new instance with stdout redirected could be created when + # pdb.set_trace() is called. + pdb.Pdb._last_pdb_instance = None pdb.set_trace(header=header) self.assertEqual(stdout.getvalue(), header + '\n') @@ -2768,6 +3419,15 @@ def test_module_is_run_as_main(self): stdout, stderr = self.run_pdb_module(script, commands) self.assertTrue(any("SUCCESS" in l for l in stdout.splitlines()), stdout) + def test_run_module_with_args(self): + commands = """ + continue + """ + self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + + stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) + self.assertIn("December", stdout) + def test_breakpoint(self): script = """ if __name__ == '__main__': @@ -2805,8 +3465,7 @@ def test_module_without_a_main(self): stdout, stderr = self._run_pdb( ['-m', module_name], "", expected_returncode=1 ) - self.assertIn("ImportError: No module named t_main.__main__", - stdout.splitlines()) + self.assertIn("ImportError: No module named t_main.__main__;", stdout) def test_package_without_a_main(self): pkg_name = 't_pkg' @@ -2824,6 +3483,22 @@ def test_package_without_a_main(self): "'t_pkg.t_main' is a package and cannot be directly executed", stdout) + def test_nonexistent_module(self): + assert not os.path.exists(os_helper.TESTFN) + stdout, stderr = self._run_pdb(["-m", os_helper.TESTFN], "", expected_returncode=1) + self.assertIn(f"ImportError: No module named {os_helper.TESTFN}", stdout) + + def test_dir_as_script(self): + with os_helper.temp_dir() as temp_dir: + stdout, stderr = self._run_pdb([temp_dir], "", expected_returncode=1) + self.assertIn(f"Error: {temp_dir} is a directory", stdout) + + def test_invalid_cmd_line_options(self): + stdout, stderr = self._run_pdb(["-c"], "", expected_returncode=2) + self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stderr.split('\n')[1]) + stdout, stderr = self._run_pdb(["--spam", "-m", "pdb"], "", expected_returncode=2) + self.assertIn(f"pdb: error: unrecognized arguments: --spam", stderr.split('\n')[1]) + def test_blocks_at_first_code_line(self): script = """ #This is a comment, on line 2 @@ -2837,6 +3512,104 @@ def test_blocks_at_first_code_line(self): self.assertTrue(any("__main__.py(4)()" in l for l in stdout.splitlines()), stdout) + def test_file_modified_after_execution(self): + script = """ + print("hello") + """ + + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + f.close() + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_multiple_instances(self): + script = """ + import pdb; pdb.Pdb().set_trace() + with open(__file__, "w") as f: + f.write("print('goodbye')\\n" * 5) + import pdb; pdb.Pdb().set_trace() + """ + + commands = """ + continue + continue + """ + + filename = 'main.py' + with open(filename, 'w') as f: + f.write(textwrap.dedent(script)) + self.addCleanup(os_helper.unlink, filename) + self.addCleanup(os_helper.rmtree, '__pycache__') + cmd = [sys.executable, filename] + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, + ) as proc: + stdout, _ = proc.communicate(str.encode(commands)) + stdout = stdout and bytes.decode(stdout) + + self.assertEqual(proc.returncode, 0) + self.assertIn("WARNING:", stdout) + self.assertIn("was edited", stdout) + + def test_file_modified_after_execution_with_restart(self): + script = """ + import random + # Any code with a source to step into so this script is not checked + # for changes when it's being changed + random.randint(1, 4) + print("hello") + """ + + commands = """ + ll + n + s + filename = $_frame.f_back.f_code.co_filename + def change_file(content, filename): + with open(filename, "w") as f: + f.write(f"print({content})") + + change_file('world', filename) + restart + ll + """ + + stdout, stderr = self.run_pdb_script(script, commands) + # Make sure the code is running correctly and the file is edited + self.assertIn("hello", stdout) + self.assertIn("world", stdout) + # The file was edited, but restart should clear the state and consider + # the file as up to date + self.assertNotIn("WARNING:", stdout) + + def test_post_mortem_restart(self): + script = """ + def foo(): + raise ValueError("foo") + foo() + """ + + commands = """ + continue + restart + continue + quit + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("Restarting", stdout) + def test_relative_imports(self): self.module_name = 't_main' os_helper.rmtree(self.module_name) @@ -2986,6 +3759,23 @@ def test_issue42384_symlink(self): self.assertEqual(stdout.split('\n')[2].rstrip('\r'), expected) + def test_safe_path(self): + """ With safe_path set, pdb should not mangle sys.path[0]""" + + script = textwrap.dedent(""" + import sys + import random + print('sys.path[0] is', sys.path[0]) + """) + commands = 'c\n' + + + with os_helper.temp_cwd() as cwd: + stdout, _ = self.run_pdb_script(script, commands, extra_env={'PYTHONSAFEPATH': '1'}) + + unexpected = f'sys.path[0] is {os.path.realpath(cwd)}' + self.assertNotIn(unexpected, stdout) + def test_issue42383(self): with os_helper.temp_cwd() as cwd: with open('foo.py', 'w') as f: @@ -3080,6 +3870,30 @@ def test_non_utf8_encoding(self): if filename.endswith(".py"): self._run_pdb([os.path.join(script_dir, filename)], 'q') + def test_zipapp(self): + with os_helper.temp_dir() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'source')) + script = textwrap.dedent( + """ + def f(x): + return x + 1 + f(21 + 21) + """ + ) + with open(os.path.join(temp_dir, 'source', '__main__.py'), 'w') as f: + f.write(script) + zipapp.create_archive(os.path.join(temp_dir, 'source'), + os.path.join(temp_dir, 'zipapp.pyz')) + stdout, _ = self._run_pdb([os.path.join(temp_dir, 'zipapp.pyz')], '\n'.join([ + 'b f', + 'c', + 'p x', + 'q' + ])) + self.assertIn('42', stdout) + self.assertIn('return x + 1', stdout) + + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() @@ -3117,6 +3931,106 @@ def test_checkline_is_not_executable(self): self.assertFalse(db.checkline(os_helper.TESTFN, lineno)) +@support.requires_subprocess() +class PdbTestReadline(unittest.TestCase): + def setUpClass(): + # Ensure that the readline module is loaded + # If this fails, the test is skipped because SkipTest will be raised + readline = import_module('readline') + if readline.backend == "editline": + raise unittest.SkipTest("libedit readline is not supported for pdb") + + def test_basic_completion(self): + script = textwrap.dedent(""" + import pdb; pdb.Pdb().set_trace() + # Concatenate strings so that the output doesn't appear in the source + print('hello' + '!') + """) + + # List everything starting with 'co', there should be multiple matches + # then add ntin and complete 'contin' to 'continue' + input = b"co\t\tntin\t\n" + + output = run_pty(script, input) + + self.assertIn(b'commands', output) + self.assertIn(b'condition', output) + self.assertIn(b'continue', output) + self.assertIn(b'hello!', output) + + def test_expression_completion(self): + script = textwrap.dedent(""" + value = "speci" + import pdb; pdb.Pdb().set_trace() + """) + + # Complete: value + 'al' + input = b"val\t + 'al'\n" + # Complete: p value + 'es' + input += b"p val\t + 'es'\n" + # Complete: $_frame + input += b"$_fra\t\n" + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'special', output) + self.assertIn(b'species', output) + self.assertIn(b'$_frame', output) + + def test_builtin_completion(self): + script = textwrap.dedent(""" + value = "speci" + import pdb; pdb.Pdb().set_trace() + """) + + # Complete: print(value + 'al') + input = b"pri\tval\t + 'al')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'special', output) + + def test_local_namespace(self): + script = textwrap.dedent(""" + def f(): + original = "I live Pythin" + import pdb; pdb.Pdb().set_trace() + f() + """) + + # Complete: original.replace('i', 'o') + input = b"orig\t.repl\t('i', 'o')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'I love Python', output) + + def test_multiline_completion(self): + script = textwrap.dedent(""" + import pdb; pdb.Pdb().set_trace() + """) + + input = b"def func():\n" + # Complete: \treturn 40 + 2 + input += b"\tret\t 40 + 2\n" + input += b"\n" + # Complete: func() + input += b"fun\t()\n" + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'42', output) + + def load_tests(loader, tests, pattern): from test import test_pdb tests.addTest(doctest.DocTestSuite(test_pdb)) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index b825b27060e86b..dd3eaeb39e7fe3 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -932,23 +932,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): @@ -981,10 +964,15 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + if expected_consts is None: expected_consts = consts - opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) - expected_insts = self.normalize_insts(expected_insts) + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -993,10 +981,10 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ ('LOAD_NAME', 1, 11), @@ -1020,11 +1008,11 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ - ('NOP', 11), - ('NOP', 12), + ('NOP', None, 11), + ('NOP', None, 12), ('RETURN_CONST', 1, 14), ] self.cfg_optimization_test(insts, @@ -1038,14 +1026,14 @@ def test_conditional_jump_backward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1056,15 +1044,31 @@ def test_conditional_jump_backward_const_condition(self): ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected_insts = [ lbl := self.Label(), - ('NOP', 11), + ('NOP', None, 11), ('JUMP', lbl, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_except_handler_label(self): + insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('POP_BLOCK', None, -1), + ('RETURN_CONST', 1, 11), + handler, + ('RETURN_CONST', 2, 12), + ] + expected_insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('RETURN_CONST', 1, 11), + handler, + ('RETURN_CONST', 2, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_no_unsafe_static_swap(self): # We can't change order of two stores to the same location insts = [ @@ -1074,16 +1078,16 @@ def test_no_unsafe_static_swap(self): ('SWAP', 3, 4), ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), + ('NOP', None, 3), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('RETURN_CONST', 0) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1097,13 +1101,13 @@ def test_dead_store_elimination_in_same_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), - ('POP_TOP', 0, 4), + ('NOP', None, 3), + ('POP_TOP', None, 4), ('STORE_FAST', 1, 4), ('RETURN_CONST', 0, 5) ] @@ -1118,7 +1122,7 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), @@ -1131,6 +1135,47 @@ def test_no_dead_store_elimination_in_different_lineno(self): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + def test_unconditional_jump_threading(self): + + def get_insts(lno1, lno2, op1, op2): + return [ + lbl2 := self.Label(), + ('LOAD_NAME', 0, 10), + (op1, lbl1 := self.Label(), lno1), + ('LOAD_NAME', 1, 20), + lbl1, + (op2, lbl2, lno2), + ] + + + for op1 in ('JUMP', 'JUMP_NO_INTERRUPT'): + for op2 in ('JUMP', 'JUMP_NO_INTERRUPT'): + # different lines + lno1, lno2 = (4, 5) + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('NOP', None, 4), + (op, 0, 5), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + # Threading + for lno1, lno2 in [(-1, -1), (-1, 5), (6, -1), (7, 7)]: + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + lno = lno1 if lno1 != -1 else lno2 + if lno == -1: + lno = 10 # Propagated from the line before + + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + (op, 0, lno), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 9e273e99e387a4..1411e55dd0f293 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -13,9 +13,7 @@ from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok -_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") -_pgo_flag = sysconfig.get_config_var("PGO_PROF_USE_FLAG") -if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist: +if support.check_cflags_pgo(): raise unittest.SkipTest("peg_generator test disabled under PGO build") test_tools.skip_if_missing("peg_generator") diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index 72c3d2054fa8f9..c7f20e1de802ce 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,7 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -49,3 +49,13 @@ def test_rule_with_collision_after_some_other_rules(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index fe8707a156e9dc..ac1911ca24eafe 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -76,14 +76,27 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) def test_trampoline_works_with_forks(self): code = """if 1: @@ -212,11 +225,11 @@ def test_sys_api_get_status(self): assert_python_ok("-c", code) -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False - return "no-omit-frame-pointer" in cflags + return "no-omit-frame-pointer" in cflags and "_Py_JIT" not in cflags def perf_command_works(): @@ -259,14 +272,27 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): if env_vars: env = os.environ.copy() env.update(env_vars) else: env = None output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +300,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +328,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +349,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +373,149 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + + def test_pre_fork_compile(self): + code = """if 1: + import sys + import os + import sysconfig + from _testinternalcapi import ( + compile_perf_trampoline_entry, + perf_trampoline_set_persist_after_fork, + ) + + def foo_fork(): + pass + + def bar_fork(): + foo_fork() + + def foo(): + import time; time.sleep(1) + + def bar(): + foo() + + def compile_trampolines_for_all_functions(): + perf_trampoline_set_persist_after_fork(1) + for _, obj in globals().items(): + if callable(obj) and hasattr(obj, '__code__'): + compile_perf_trampoline_entry(obj.__code__) + + if __name__ == "__main__": + compile_trampolines_for_all_functions() + pid = os.fork() + if pid == 0: + print(os.getpid()) + bar_fork() + else: + bar() + """ + + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + with subprocess.Popen( + [sys.executable, "-Xperf", script], + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0) + self.assertNotIn("Error:", stderr) + child_pid = int(stdout.strip()) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + perf_child_file = pathlib.Path(f"/tmp/perf-{child_pid}.map") + self.assertTrue(perf_file.exists()) + self.assertTrue(perf_child_file.exists()) + + perf_file_contents = perf_file.read_text() + self.assertIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", perf_file_contents) + + child_perf_file_contents = perf_child_file.read_text() + self.assertIn(f"py::foo_fork:{script}", child_perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) + + # Pre-compiled perf-map entries of a forked process must be + # identical in both the parent and child perf-map files. + perf_file_lines = perf_file_contents.split("\n") + for line in perf_file_lines: + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: + self.assertIn(line, child_perf_file_contents) + + +def _is_perf_vesion_at_least(major, minor): + # The output of perf --version looks like "perf version 6.7-3" but + # it can also be perf version "perf version 5.15.143" + try: + output = subprocess.check_output(["perf", "--version"], text=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return False + version = output.split()[2] + version = version.split("-")[0] + version = version.split(".") + version = tuple(map(int, version)) + return version >= (major, minor) + + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless(_is_perf_vesion_at_least(6, 6), "perf command may not work due to a perf bug") +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperf_jit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index a17adb89f55360..d4c6fe0124af18 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -2,7 +2,11 @@ import sys import unittest -from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +try: + from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") + if sys.platform != 'linux': raise unittest.SkipTest('Linux only') diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 1a55da39bdc58d..49aa4b386039ec 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -122,6 +122,7 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + persistent_load_error = pickle.UnpicklingError @support.cpython_only def test_pickler_reference_cycle(self): @@ -176,7 +177,6 @@ class DispatchTable: support.gc_collect() self.assertIsNone(table_ref()) - @support.cpython_only def test_unpickler_reference_cycle(self): def check(Unpickler): @@ -206,6 +206,28 @@ def persistent_load(pid): return pid check(PersUnpickler) + def test_pickler_super(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + pickler.dump('abc') + self.assertEqual(self.loads(f.getvalue()), 'abc') + + def test_unpickler_super(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + self.assertEqual(unpickler.load(), 'abc') class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -256,6 +278,7 @@ class CPersPicklerTests(PyPersPicklerTests): class CIdPersPicklerTests(PyIdPersPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + persistent_load_error = _pickle.UnpicklingError class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler @@ -326,7 +349,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('7P2n3i2n3i2P') + basesize = support.calcobjsize('6P2n3i2n3i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -343,7 +366,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2nP 2P2n2i5P 2P3n8P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -379,7 +402,9 @@ def recurse(deep): check_unpickler(recurse(1), 32, 20) check_unpickler(recurse(20), 32, 20) check_unpickler(recurse(50), 64, 60) - check_unpickler(recurse(100), 128, 140) + if not (support.is_wasi and support.Py_DEBUG): + # stack depth too shallow in pydebug WASI. + check_unpickler(recurse(100), 128, 140) u = unpickler(io.BytesIO(pickle.dumps('a', 0)), encoding='ASCII', errors='strict') @@ -539,10 +564,12 @@ def test_exceptions(self): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, + PythonFinalizationError, RecursionError, EncodingWarning, BaseExceptionGroup, - ExceptionGroup): + ExceptionGroup, + _IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index eed0fd1c6b73fa..a7a1c2affbe1fb 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -94,7 +94,7 @@ def mkhier(self, descr): def test_1(self): hier = [("t1", None), ("t1 __init__.py", "")] self.mkhier(hier) - import t1 + import t1 # noqa: F401 def test_2(self): hier = [ @@ -124,7 +124,7 @@ def test_2(self): from t2 import sub from t2.sub import subsub - from t2.sub.subsub import spam + from t2.sub.subsub import spam # noqa: F401 self.assertEqual(sub.__name__, "t2.sub") self.assertEqual(subsub.__name__, "t2.sub.subsub") self.assertEqual(sub.subsub.__name__, "t2.sub.subsub") diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac..d095f440a99f63 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,10 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -118,7 +122,7 @@ def test_issue44061_iter_modules(self): # make sure iter_modules accepts Path objects names = [] - for moduleinfo in pkgutil.iter_modules([Path(zip_file)]): + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) names.append(moduleinfo.name) self.assertEqual(names, [pkg]) @@ -318,6 +322,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 216973350319fe..40d5fb338ce563 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -219,6 +227,30 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") def test_uname_win32_without_wmi(self): def raises_oserror(*a): @@ -318,12 +350,44 @@ def raises_oserror(*a): platform._uname_cache = None def test_java_ver(self): - res = platform.java_ver() - if sys.platform == 'java': # Is never actually checked in CI - self.assertTrue(all(res)) + import re + msg = re.escape( + "'java_ver' is deprecated and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, msg): + res = platform.java_ver() + self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() @@ -377,6 +441,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception @@ -426,6 +540,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V @@ -472,7 +623,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index b08ababa341cfe..b231b05f864ab9 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -13,6 +13,7 @@ import subprocess import binascii import collections +import zoneinfo from test import support from test.support import os_helper from io import BytesIO @@ -508,6 +509,19 @@ def test_bytes(self): data2 = plistlib.dumps(pl2) self.assertEqual(data, data2) + def test_loads_str_with_xml_fmt(self): + pl = self._create() + b = plistlib.dumps(pl) + s = b.decode() + self.assertIsInstance(s, str) + pl2 = plistlib.loads(s) + self.assertEqual(pl, pl2) + + def test_loads_str_with_binary_fmt(self): + msg = "value must be bytes-like object when fmt is FMT_BINARY" + with self.assertRaisesRegex(TypeError, msg): + plistlib.loads('test', fmt=plistlib.FMT_BINARY) + def test_indentation_array(self): data = [[[[[[[[{'test': b'aaaaaa'}]]]]]]]] self.assertEqual(plistlib.loads(plistlib.dumps(data)), data) @@ -838,6 +852,54 @@ def test_xml_plist_with_entity_decl(self): "XML entity declarations are not supported"): plistlib.loads(XML_PLIST_WITH_ENTITY, fmt=plistlib.FMT_XML) + def test_load_aware_datetime(self): + dt = plistlib.loads(b"2023-12-10T08:03:30Z", + aware_datetime=True) + self.assertEqual(dt.tzinfo, datetime.UTC) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + def test_dump_utc_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, tzinfo=datetime.UTC) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_naive_datetime_with_aware_datetime_option(self): + # Save a naive datetime with aware_datetime set to true. This will lead + # to having different time as compared to the current machine's + # timezone, which is UTC. + dt = datetime.datetime(2003, 6, 7, 8, tzinfo=None) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + parsed = plistlib.loads(s, aware_datetime=False) + expected = dt.astimezone(datetime.UTC).replace(tzinfo=None) + self.assertEqual(parsed, expected) + class TestBinaryPlistlib(unittest.TestCase): @@ -908,12 +970,12 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - for N in [300, 100000]: + for N in [50, 300, 100_000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) except RecursionError: - pass + self.assertGreater(N, sys.getrecursionlimit()) else: for i in range(N): self.assertIsInstance(result, list) @@ -962,6 +1024,28 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_load_aware_datetime(self): + data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' + b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11') + self.assertEqual(plistlib.loads(data, aware_datetime=True), + datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC)) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + class TestKeyedArchive(unittest.TestCase): def test_keyed_archive_data(self): @@ -1072,5 +1156,6 @@ def test_octal_and_hex(self): self.assertEqual(p.get("HexType"), 16777228) self.assertEqual(p.get("IntType"), 83) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 1847ae95db9292..5675db8d1cab6e 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -172,7 +172,10 @@ def test_poll3(self): @cpython_only def test_poll_c_limits(self): - from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + try: + from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + except ImportError: + raise unittest.SkipTest("requires _testcapi") pollster = select.poll() pollster.register(1) diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9d72dba159c6be..908354cb8574d1 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,14 +1,11 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok -# Skip these tests if there is no posix module. -posix = import_helper.import_module('posix') - import errno import sys import signal @@ -22,6 +19,11 @@ import textwrap from contextlib import contextmanager +try: + import posix +except ImportError: + import nt as posix + try: import pwd except ImportError: @@ -562,6 +564,7 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): self.assertRaises(ValueError, posix.confstr, "CS_garbage") self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) @@ -701,7 +704,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -709,13 +713,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux': + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" @@ -779,9 +793,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -789,7 +804,7 @@ def check_stat(uid, gid): self.assertRaises(TypeError, chown_func, first_param, uid, t(gid)) check_stat(uid, gid) - @os_helper.skip_unless_working_chmod + @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @unittest.skipIf(support.is_emscripten, "getgid() is a stub") def test_chown(self): # raise an OSError if the file does not exist @@ -933,6 +948,131 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (int(now), int(now))) posix.utime(os_helper.TESTFN, (now, now)) + def check_chmod(self, chmod_func, target, **kwargs): + closefd = not isinstance(target, int) + mode = os.stat(target).st_mode + try: + new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + try: + with open(target, 'wb+', closefd=closefd): + pass + except PermissionError: + pass + new_mode = mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + with open(target, 'wb+', closefd=closefd): + pass + finally: + chmod_func(target, mode) + + @os_helper.skip_unless_working_chmod + def test_chmod_file(self): + self.check_chmod(posix.chmod, os_helper.TESTFN) + + def tempdir(self): + target = os_helper.TESTFN + 'd' + posix.mkdir(target) + self.addCleanup(posix.rmdir, target) + return target + + @os_helper.skip_unless_working_chmod + def test_chmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.chmod, target) + + @os_helper.skip_unless_working_chmod + def test_fchmod_file(self): + with open(os_helper.TESTFN, 'wb+') as f: + self.check_chmod(posix.fchmod, f.fileno()) + self.check_chmod(posix.chmod, f.fileno()) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_file(self): + self.check_chmod(posix.lchmod, os_helper.TESTFN) + self.check_chmod(posix.chmod, os_helper.TESTFN, follow_symlinks=False) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.lchmod, target) + self.check_chmod(posix.chmod, target, follow_symlinks=False) + + def check_chmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + try: + new_mode = target_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + new_mode = target_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + finally: + posix.chmod(target, target_mode) + + def check_lchmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + new_mode = link_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + new_mode = link_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + + @os_helper.skip_unless_symlink + def test_chmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @os_helper.skip_unless_symlink + def test_chmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link, target_is_directory=True) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) self.assertTrue(hasattr(st, 'st_flags')) @@ -1012,16 +1152,17 @@ def test_environ(self): def test_putenv(self): with self.assertRaises(ValueError): os.putenv('FRUIT\0VEGETABLE', 'cabbage') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') with self.assertRaises(ValueError): os.putenv('FRUIT', 'orange\0VEGETABLE=cabbage') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') with self.assertRaises(ValueError): os.putenv('FRUIT=ORANGE', 'lemon') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT=ORANGE', b'lemon') + if os.name == 'posix': + with self.assertRaises(ValueError): + os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT=ORANGE', b'lemon') @unittest.skipUnless(hasattr(posix, 'getcwd'), 'test needs posix.getcwd()') def test_getcwd_long_pathnames(self): @@ -1128,8 +1269,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple plaforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1141,9 +1282,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1158,9 +1300,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) @@ -1206,12 +1347,21 @@ def test_sched_getaffinity(self): def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) self.addCleanup(posix.sched_setaffinity, 0, list(mask)) + if len(mask) > 1: # Empty masks are forbidden mask.pop() posix.sched_setaffinity(0, mask) self.assertEqual(posix.sched_getaffinity(0), mask) - self.assertRaises(OSError, posix.sched_setaffinity, 0, []) + + try: + posix.sched_setaffinity(0, []) + # gh-117061: On RHEL9, sched_setaffinity(0, []) does not fail + except OSError: + # sched_setaffinity() manual page documents EINVAL error + # when the mask is empty. + pass + self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) @@ -1220,6 +1370,7 @@ def test_sched_setaffinity(self): self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) @unittest.skipIf(support.is_wasi, "No dynamic linking on WASI") + @unittest.skipUnless(os.name == 'posix', "POSIX-only test") def test_rtld_constants(self): # check presence of major RTLD_* constants posix.RTLD_LAZY @@ -1384,6 +1535,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -1899,11 +2057,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 9be4640f970aef..fb714fd90ae2b3 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,3 +1,4 @@ +import inspect import os import posixpath import sys @@ -5,7 +6,7 @@ from posixpath import realpath, abspath, dirname, basename from test import test_genericpath from test.support import import_helper -from test.support import os_helper +from test.support import cpython_only, os_helper from test.support.os_helper import FakePath from unittest import mock @@ -47,18 +48,28 @@ def tearDown(self): safe_rmdir(os_helper.TESTFN + suffix) def test_join(self): - self.assertEqual(posixpath.join("/foo", "bar", "/bar", "baz"), - "/bar/baz") - self.assertEqual(posixpath.join("/foo", "bar", "baz"), "/foo/bar/baz") - self.assertEqual(posixpath.join("/foo/", "bar/", "baz/"), - "/foo/bar/baz/") - - self.assertEqual(posixpath.join(b"/foo", b"bar", b"/bar", b"baz"), - b"/bar/baz") - self.assertEqual(posixpath.join(b"/foo", b"bar", b"baz"), - b"/foo/bar/baz") - self.assertEqual(posixpath.join(b"/foo/", b"bar/", b"baz/"), - b"/foo/bar/baz/") + fn = posixpath.join + self.assertEqual(fn("/foo", "bar", "/bar", "baz"), "/bar/baz") + self.assertEqual(fn("/foo", "bar", "baz"), "/foo/bar/baz") + self.assertEqual(fn("/foo/", "bar/", "baz/"), "/foo/bar/baz/") + + self.assertEqual(fn(b"/foo", b"bar", b"/bar", b"baz"), b"/bar/baz") + self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") + self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + + self.assertEqual(fn("a", ""), "a/") + self.assertEqual(fn("a", "", ""), "a/") + self.assertEqual(fn("a", "b"), "a/b") + self.assertEqual(fn("a", "b/"), "a/b/") + self.assertEqual(fn("a/", "b"), "a/b") + self.assertEqual(fn("a/", "b/"), "a/b/") + self.assertEqual(fn("a", "b/c", "d"), "a/b/c/d") + self.assertEqual(fn("a", "b//c", "d"), "a/b//c/d") + self.assertEqual(fn("a", "b/c/", "d"), "a/b/c/d") + self.assertEqual(fn("/a", "b"), "/a/b") + self.assertEqual(fn("/a/", "b"), "/a/b") + self.assertEqual(fn("a", "/b", "c"), "/b/c") + self.assertEqual(fn("a", "/b", "/c"), "/c") def test_split(self): self.assertEqual(posixpath.split("/foo/bar"), ("/foo", "bar")) @@ -273,6 +284,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") @@ -334,6 +355,24 @@ def test_expanduser_pwd(self): for path in ('~', '~/.local', '~vstinner/'): self.assertEqual(posixpath.expanduser(path), path) + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser_pwd2(self): + pwd = import_helper.import_module('pwd') + for all_entry in pwd.getpwall(): + name = all_entry.pw_name + + # gh-121200: pw_dir can be different between getpwall() and + # getpwnam(), so use getpwnam() pw_dir as expanduser() does. + entry = pwd.getpwnam(name) + home = entry.pw_dir + home = home.rstrip('/') or '/' + + with self.subTest(all_entry=all_entry, entry=entry): + self.assertEqual(posixpath.expanduser('~' + name), home) + self.assertEqual(posixpath.expanduser(os.fsencode('~' + name)), + os.fsencode(home)) + NORMPATH_CASES = [ ("", "."), ("/", "/"), @@ -448,6 +487,15 @@ def test_realpath_relative(self): finally: os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_missing_pardir(self): + try: + os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) + self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + finally: + os_helper.unlink(os_helper.TESTFN) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): @@ -467,7 +515,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") @@ -629,10 +677,29 @@ def test_realpath_resolve_first(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + with self.assertRaises(PermissionError): + realpath(ABSTFN, strict=True) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") @@ -695,7 +762,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 6ea7e7db2ce113..dfbc2a06e7346f 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -7,8 +7,7 @@ import itertools import pprint import random -import test.support -import test.test_set +import re import types import unittest @@ -535,7 +534,10 @@ def test_dataclass_with_repr(self): def test_dataclass_no_repr(self): dc = dataclass3() formatted = pprint.pformat(dc, width=10) - self.assertRegex(formatted, r"") + self.assertRegex( + formatted, + fr"<{re.escape(__name__)}.dataclass3 object at \w+>", + ) def test_recursive_dataclass(self): dc = dataclass4(None) @@ -619,9 +621,6 @@ def test_set_reprs(self): self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), 'frozenset3({0, 1, 2, 3, 4, 5, 6})') - @unittest.expectedFailure - #See http://bugs.python.org/issue13907 - @test.support.cpython_only def test_set_of_sets_reprs(self): # This test creates a complex arrangement of frozensets and # compares the pretty-printed repr against a string hard-coded in @@ -632,204 +631,106 @@ def test_set_of_sets_reprs(self): # partial ordering (subset relationships), the output of the # list.sort() method is undefined for lists of sets." # - # In a nutshell, the test assumes frozenset({0}) will always - # sort before frozenset({1}), but: - # # >>> frozenset({0}) < frozenset({1}) # False # >>> frozenset({1}) < frozenset({0}) # False # - # Consequently, this test is fragile and - # implementation-dependent. Small changes to Python's sort - # algorithm cause the test to fail when it should pass. - # XXX Or changes to the dictionary implementation... - - cube_repr_tgt = """\ -{frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}), - frozenset({0}): frozenset({frozenset(), - frozenset({0, 2}), - frozenset({0, 1})}), - frozenset({1}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 1})}), - frozenset({2}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 2})}), - frozenset({1, 2}): frozenset({frozenset({2}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 2}): frozenset({frozenset({2}), - frozenset({0}), - frozenset({0, 1, 2})}), - frozenset({0, 1}): frozenset({frozenset({0}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 1, 2}): frozenset({frozenset({1, 2}), - frozenset({0, 2}), - frozenset({0, 1})})}""" - cube = test.test_set.cube(3) - self.assertEqual(pprint.pformat(cube), cube_repr_tgt) - cubo_repr_tgt = """\ -{frozenset({frozenset({0, 2}), frozenset({0})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1}), frozenset({1})}): frozenset({frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})})}), - frozenset({frozenset({1, 2}), frozenset({1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({1, 2}), frozenset({2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({0})}): frozenset({frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset()}): frozenset({frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({0}), frozenset({0, 1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({1, 2}), frozenset({0, 1, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})})})}""" - - cubo = test.test_set.linegraph(cube) - self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt) + # In this test we list all possible invariants of the result + # for unordered frozensets. + # + # This test has a long history, see: + # - https://github.com/python/cpython/commit/969fe57baa0eb80332990f9cda936a33e13fabef + # - https://github.com/python/cpython/issues/58115 + # - https://github.com/python/cpython/issues/111147 + + import textwrap + + # Single-line, always ordered: + fs0 = frozenset() + fs1 = frozenset(('abc', 'xyz')) + data = frozenset((fs0, fs1)) + self.assertEqual(pprint.pformat(data), + 'frozenset({%r, %r})' % (fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + fs2 = frozenset(('one', 'two')) + data = {fs2: frozenset((fs0, fs1))} + self.assertEqual(pprint.pformat(data), + "{%r: frozenset({%r, %r})}" % (fs2, fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + # Single-line, unordered: + fs1 = frozenset(("xyz", "qwerty")) + fs2 = frozenset(("abcd", "spam")) + fs = frozenset((fs1, fs2)) + self.assertEqual(pprint.pformat(fs), repr(fs)) + + # Multiline, unordered: + def check(res, invariants): + self.assertIn(res, [textwrap.dedent(i).strip() for i in invariants]) + + # Inner-most frozensets are singleline, result is multiline, unordered: + fs1 = frozenset(('regular string', 'other string')) + fs2 = frozenset(('third string', 'one more string')) + check( + pprint.pformat(frozenset((fs1, fs2))), + [ + """ + frozenset({%r, + %r}) + """ % (fs1, fs2), + """ + frozenset({%r, + %r}) + """ % (fs2, fs1), + ], + ) + + # Everything is multiline, unordered: + check( + pprint.pformat( + frozenset(( + frozenset(( + "xyz very-very long string", + "qwerty is also absurdly long", + )), + frozenset(( + "abcd is even longer that before", + "spam is not so long", + )), + )), + ), + [ + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'})}) + """, + + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'xyz very-very long string', + 'qwerty is also absurdly long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'abcd is even longer that before', + 'spam is not so long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'spam is not so long', + 'abcd is even longer that before'})}) + """, + ], + ) def test_depth(self): nested_tuple = (1, (2, (3, (4, (5, 6))))) diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index a1dfc9abbb8ef7..0f16b92334999c 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -7,7 +7,7 @@ from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stdout import profile from test.profilee import testfunc, timer @@ -92,6 +92,11 @@ def test_run(self): self.profilermodule.run("int('1')", filename=TESTFN) self.assertTrue(os.path.exists(TESTFN)) + def test_run_with_sort_by_values(self): + with redirect_stdout(StringIO()) as f: + self.profilermodule.run("int('1')", sort=('tottime', 'stdname')) + self.assertIn("Ordered by: internal time, standard name", f.getvalue()) + def test_runctx(self): with silent(): self.profilermodule.runctx("testfunc()", globals(), locals()) diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 45aa9e51c06de0..b7a2219b96149a 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -183,26 +183,76 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property(self): - class A: - @classmethod - @property - def __doc__(cls): - return 'A doc for %r' % cls.__name__ - self.assertEqual(A.__doc__, "A doc for 'A'") + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property_override(self): class A: - """First""" - @classmethod @property - def __doc__(cls): - return 'Second' - self.assertEqual(A.__doc__, 'Second') + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') def test_property_set_name_incorrect_args(self): p = property() @@ -245,6 +295,7 @@ class PropertySubSlots(property): class PropertySubclassTests(unittest.TestCase): + @support.requires_docstrings def test_slots_docstring_copy_exception(self): # A special case error that we preserve despite the GH-98963 behavior # that would otherwise silently ignore this error. @@ -412,6 +463,40 @@ def getter3(self): self.assertEqual(p.__doc__, "user") self.assertEqual(p2.__doc__, "user") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): diff --git a/Lib/test/test_pstats.py b/Lib/test/test_pstats.py index acc2fa5385d923..d5a5a9738c2498 100644 --- a/Lib/test/test_pstats.py +++ b/Lib/test/test_pstats.py @@ -5,7 +5,9 @@ from pstats import SortKey from enum import StrEnum, _test_simple_enum +import os import pstats +import tempfile import cProfile class AddCallersTestCase(unittest.TestCase): @@ -36,6 +38,33 @@ def test_add(self): stats = pstats.Stats(stream=stream) stats.add(self.stats, self.stats) + def test_dump_and_load_works_correctly(self): + temp_storage_new = tempfile.NamedTemporaryFile(delete=False) + try: + self.stats.dump_stats(filename=temp_storage_new.name) + tmp_stats = pstats.Stats(temp_storage_new.name) + self.assertEqual(self.stats.stats, tmp_stats.stats) + finally: + temp_storage_new.close() + os.remove(temp_storage_new.name) + + def test_load_equivalent_to_init(self): + stats = pstats.Stats() + self.temp_storage = tempfile.NamedTemporaryFile(delete=False) + try: + cProfile.run('import os', filename=self.temp_storage.name) + stats.load_stats(self.temp_storage.name) + created = pstats.Stats(self.temp_storage.name) + self.assertEqual(stats.stats, created.stats) + finally: + self.temp_storage.close() + os.remove(self.temp_storage.name) + + def test_loading_wrong_types(self): + stats = pstats.Stats() + with self.assertRaises(TypeError): + stats.load_stats(42) + def test_sort_stats_int(self): valid_args = {-1: 'stdname', 0: 'calls', diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index f31a68c5d84e03..dee94533c74549 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,11 +1,15 @@ -from test.support import verbose, reap_children +import unittest +from test.support import ( + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink -# Skip these tests if termios or fcntl are not available +# Skip these tests if termios is not available import_module('termios') -# fcntl is a proxy for not being one of the wasm32 platforms even though we -# don't use this module... a proper check for what crashes those is needed. -import_module("fcntl") + +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") import errno import os @@ -16,7 +20,6 @@ import signal import socket import io # readline -import unittest import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -292,7 +295,26 @@ def test_master_read(self): self.assertEqual(data, b"") def test_spawn_doesnt_hang(self): - pty.spawn([sys.executable, '-c', 'print("hi there")']) + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f: + STDOUT_FILENO = 1 + dup_stdout = os.dup(STDOUT_FILENO) + os.dup2(f.fileno(), STDOUT_FILENO) + buf = b'' + def master_read(fd): + nonlocal buf + data = os.read(fd, 1024) + buf += data + return data + try: + pty.spawn([sys.executable, '-c', 'print("hi there")'], + master_read) + finally: + os.dup2(dup_stdout, STDOUT_FILENO) + os.close(dup_stdout) + self.assertEqual(buf, b'hi there\r\n') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'hi there\r\n') class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index c4e6551f605782..64387296e84621 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -227,7 +227,8 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') @@ -250,6 +251,7 @@ def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index c7c5419ffe3e37..4bf0576586cca5 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -78,7 +78,8 @@ def ismethod(oclass, obj, name): objname = obj.__name__ if objname.startswith("__") and not objname.endswith("__"): - objname = "_%s%s" % (oclass.__name__, objname) + if stripped_typename := oclass.__name__.lstrip('_'): + objname = f"_{stripped_typename}{objname}" return objname == name # Make sure the toplevel functions and classes are the same. @@ -109,14 +110,20 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) - foundMethods = [] - for m in value.methods.keys(): - if m[:2] == '__' and m[-2:] != '__': - foundMethods.append('_'+name+m) - else: - foundMethods.append(m) + + if stripped_typename := name.lstrip('_'): + foundMethods = [] + for m in value.methods.keys(): + if m.startswith('__') and not m.endswith('__'): + foundMethods.append(f"_{stripped_typename}{m}") + else: + foundMethods.append(m) + else: + foundMethods = list(value.methods.keys()) try: self.assertListEq(foundMethods, actualMethods, ignore) @@ -150,8 +157,9 @@ def test_easy(self): "DocTestCase", '_DocTestSuite')) self.checkModule('difflib', ignore=("Match",)) - def test_decorators(self): - self.checkModule('test.pyclbr_input', ignore=['om']) + def test_cases(self): + # see test.pyclbr_input for the rationale behind the ignored symbols + self.checkModule('test.pyclbr_input', ignore=['om', 'f']) def test_nested(self): mb = pyclbr @@ -226,7 +234,7 @@ def test_others(self): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget'), + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pydoc/__init__.py b/Lib/test/test_pydoc/__init__.py new file mode 100644 index 00000000000000..f2a39a3fe29c7f --- /dev/null +++ b/Lib/test/test_pydoc/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/pydoc_mod.py b/Lib/test/test_pydoc/pydoc_mod.py similarity index 100% rename from Lib/test/pydoc_mod.py rename to Lib/test/test_pydoc/pydoc_mod.py diff --git a/Lib/test/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py similarity index 69% rename from Lib/test/pydocfodder.py rename to Lib/test/test_pydoc/pydocfodder.py index d0750e5a43c0c0..3cc2d5bd57fe5b 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -2,8 +2,14 @@ import types -class A_new: - "A new-style class." +def global_func(x, y): + """Module global function""" + +def global_func2(x, y): + """Module global function 2""" + +class A: + "A class." def A_method(self): "Method defined in A." @@ -26,7 +32,7 @@ def A_classmethod(cls, x): "A class method defined in A." A_classmethod = classmethod(A_classmethod) - def A_staticmethod(): + def A_staticmethod(x, y): "A static method defined in A." A_staticmethod = staticmethod(A_staticmethod) @@ -41,8 +47,8 @@ def _delx(self): A_int_alias = int -class B_new(A_new): - "A new-style class, derived from A_new." +class B(A): + "A class, derived from A." def AB_method(self): "Method defined in A and B." @@ -61,8 +67,32 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." -class C_new(A_new): - "A new-style class, derived from A_new." + @classmethod + def B_classmethod(cls, x): + "A class method defined in B." + + global_func = global_func # same name + global_func_alias = global_func + global_func2_alias = global_func2 + B_classmethod_alias = B_classmethod + A_classmethod_ref = A.A_classmethod + A_staticmethod = A.A_staticmethod # same name + A_staticmethod_alias = A.A_staticmethod + A_method_ref = A().A_method + A_method_alias = A.A_method + B_method_alias = B_method + count = list.count # same name + list_count = list.count + __repr__ = object.__repr__ # same name + object_repr = object.__repr__ + get = {}.get # same name + dict_get = {}.get + +B.B_classmethod_ref = B.B_classmethod + + +class C(A): + "A class, derived from A." def AC_method(self): "Method defined in A and C." @@ -81,8 +111,8 @@ def C_method(self): def CD_method(self): "Method defined in C and D." -class D_new(B_new, C_new): - """A new-style class, derived from B_new and C_new. +class D(B, C): + """A class, derived from B and C. """ def AD_method(self): @@ -136,3 +166,23 @@ def __call__(self, inst): submodule = types.ModuleType(__name__ + '.submodule', """A submodule, which should appear in its parent's summary""") + +global_func_alias = global_func +A_classmethod = A.A_classmethod # same name +A_classmethod2 = A.A_classmethod +A_classmethod3 = B.A_classmethod +A_staticmethod = A.A_staticmethod # same name +A_staticmethod_alias = A.A_staticmethod +A_staticmethod_ref = A().A_staticmethod +A_staticmethod_ref2 = B().A_staticmethod +A_method = A().A_method # same name +A_method2 = A().A_method +A_method3 = B().A_method +B_method = B.B_method # same name +B_method2 = B.B_method +count = list.count # same name +list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ +get = {}.get # same name +dict_get = {}.get diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py similarity index 69% rename from Lib/test/test_pydoc.py rename to Lib/test/test_pydoc/test_pydoc.py index 70c5ebd694ca88..40b3aca25e0da8 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -17,6 +17,7 @@ import types import typing import unittest +import unittest.mock import urllib.parse import xml.etree import xml.etree.ElementTree @@ -30,11 +31,12 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, spawn_python) from test.support import threading_helper -from test.support import (reap_children, captured_output, captured_stdout, +from test.support import (reap_children, captured_stdout, captured_stderr, is_emscripten, is_wasi, - requires_docstrings) + requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) -from test import pydoc_mod +from test.test_pydoc import pydoc_mod +from test.test_pydoc import pydocfodder class nonascii: @@ -43,15 +45,15 @@ class nonascii: if test.support.HAVE_DOCSTRINGS: expected_data_docstrings = ( - 'dictionary for instance variables (if defined)', - 'list of weak references to the object (if defined)', + 'dictionary for instance variables', + 'list of weak references to the object', ) * 2 else: expected_data_docstrings = ('', '', '', '') expected_text_pattern = """ NAME - test.pydoc_mod - This is a test module for test_pydoc + test.test_pydoc.pydoc_mod - This is a test module for test_pydoc %s CLASSES builtins.object @@ -75,6 +77,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(...) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -85,8 +92,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -102,16 +107,16 @@ class C(builtins.object) | ---------------------------------------------------------------------- | Class methods defined here: | - | __class_getitem__(item) from builtins.type + | __class_getitem__(item) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ - | dictionary for instance variables (if defined) + | dictionary for instance variables | | __weakref__ - | list of weak references to the object (if defined) + | list of weak references to the object FUNCTIONS doc_func() @@ -124,7 +129,7 @@ class C(builtins.object) DATA __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -147,7 +152,7 @@ class C(builtins.object) for s in expected_data_docstrings) html2text_of_expected = """ -test.pydoc_mod (version 1.2.3.4) +test.test_pydoc.pydoc_mod (version 1.2.3.4) This is a test module for test_pydoc Modules @@ -166,22 +171,25 @@ class A(builtins.object) Methods defined here: __init__() Wow, I have no function! - + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ - dictionary for instance variables (if defined) + dictionary for instance variables __weakref__ - list of weak references to the object (if defined) + list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(...) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ - dictionary for instance variables (if defined) + dictionary for instance variables __weakref__ - list of weak references to the object (if defined) + list of weak references to the object + ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) @@ -191,13 +199,15 @@ class C(builtins.object) is_it_true(self) Return self.get_answer() say_no(self) + ---------------------------------------------------------------------- Class methods defined here: - __class_getitem__(item) from builtins.type + __class_getitem__(item) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ - dictionary for instance variables (if defined) + dictionary for instance variables __weakref__ - list of weak references to the object (if defined) + list of weak references to the object Functions doc_func() @@ -209,7 +219,7 @@ class C(builtins.object) Data __xyz__ = 'X, Y and Z' - c_alias = test.pydoc_mod.C[int] + c_alias = test.test_pydoc.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] type_union1 = typing.Union[int, str] @@ -330,11 +340,15 @@ def get_pydoc_html(module): loc = "
    Module Docs" return output.strip(), loc +def clean_text(doc): + # clean up the extra text formatting that pydoc performs + return re.sub('\b.', '', doc) + def get_pydoc_link(module): "Returns a documentation web link of a module" abspath = os.path.abspath dirname = os.path.dirname - basedir = dirname(dirname(abspath(__file__))) + basedir = dirname(dirname(dirname(abspath(__file__)))) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc @@ -347,10 +361,7 @@ def get_pydoc_text(module): loc = "\nMODULE DOCS\n " + loc + "\n" output = doc.docmodule(module) - - # clean up the extra text formatting that pydoc performs - patt = re.compile('\b.') - output = patt.sub('', output) + output = clean_text(output) return output.strip(), loc def get_html_title(text): @@ -367,12 +378,18 @@ def html2text(html): Tailored for pydoc tests only. """ html = html.replace("
    ", "\n") + html = html.replace("
    ", "-"*70) html = re.sub("<.*?>", "", html) html = pydoc.replace(html, " ", " ", ">", ">", "<", "<") return html class PydocBaseTest(unittest.TestCase): + def tearDown(self): + # Self-testing. Mocking only works if sys.modules['pydoc'] and pydoc + # are the same. But some pydoc functions reload the module and change + # sys.modules, so check that it was restored. + self.assertIs(sys.modules['pydoc'], pydoc) def _restricted_walk_packages(self, walk_packages, path=None): """ @@ -404,6 +421,8 @@ def call_url_handler(self, url, expected_title): class PydocDocTest(unittest.TestCase): maxDiff = None + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @@ -483,7 +502,7 @@ def test_not_here(self): @requires_docstrings def test_not_ascii(self): - result = run_pydoc('test.test_pydoc.nonascii', PYTHONIOENCODING='ascii') + result = run_pydoc('test.test_pydoc.test_pydoc.nonascii', PYTHONIOENCODING='ascii') encoded = nonascii.__doc__.encode('ascii', 'backslashreplace') self.assertIn(encoded, result) @@ -652,40 +671,188 @@ def test_fail_help_output_redirect(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') @requires_docstrings - def test_help_output_redirect(self): + def test_help_output_redirect(self, pager_mock): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected - getpager_old = pydoc.getpager - getpager_new = lambda: (lambda x: x) self.maxDiff = None - buf = StringIO() - helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) - module = "test.pydoc_mod" + module = "test.test_pydoc.pydoc_mod" help_header = """ - Help on module test.pydoc_mod in test: + Help on module test.test_pydoc.pydoc_mod in test.test_pydoc: """.lstrip() help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - pydoc.getpager = getpager_new - try: - with captured_output('stdout') as output, \ - captured_output('stderr') as err: - helper.help(module) + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(request) result = buf.getvalue().strip() - expected_text = expected_help_pattern % ( - (doc_loc,) + - expected_text_data_docstrings + - (inspect.getabsfile(pydoc_mod),)) - self.assertEqual('', output.getvalue()) + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) self.assertEqual('', err.getvalue()) - self.assertEqual(expected_text, result) - finally: - pydoc.getpager = getpager_old + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', 'Help on method_descriptor in str:') + run_pydoc_pager(str.upper, 'str.upper', 'Help on method_descriptor:') + run_pydoc_pager(str.__add__, 'str.__add__', 'Help on wrapper_descriptor:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() + + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) def test_namedtuple_fields(self): Person = namedtuple('Person', ['nickname', 'firstname']) @@ -745,14 +912,18 @@ def test_splitdoc_with_description(self): def test_is_package_when_not_package(self): with os_helper.temp_cwd() as test_dir: - self.assertFalse(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertFalse(pydoc.ispackage(test_dir)) + self.assertEqual(cm.filename, __file__) def test_is_package_when_is_package(self): with os_helper.temp_cwd() as test_dir: init_path = os.path.join(test_dir, '__init__.py') open(init_path, 'w').close() - self.assertTrue(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertTrue(pydoc.ispackage(test_dir)) os.remove(init_path) + self.assertEqual(cm.filename, __file__) def test_allmethods(self): # issue 17476: allmethods was no longer returning unbound methods. @@ -794,8 +965,7 @@ def itemconfigure(self, tagOrId, cnf=None, **kw): b_size = A.a_size doc = pydoc.render_doc(B) - # clean up the extra text formatting that pydoc performs - doc = re.sub('\b.', '', doc) + doc = clean_text(doc) self.assertEqual(doc, '''\ Python Library Documentation: class B in module %s @@ -829,10 +999,10 @@ class B(A) | Data descriptors inherited from A: | | __dict__ - | dictionary for instance variables (if defined) + | dictionary for instance variables | | __weakref__ - | list of weak references to the object (if defined) + | list of weak references to the object ''' % __name__) doc = pydoc.render_doc(B, renderer=pydoc.HTMLDoc()) @@ -861,15 +1031,101 @@ class B(A) Data descriptors inherited from A: __dict__ - dictionary for instance variables (if defined) + dictionary for instance variables __weakref__ - list of weak references to the object (if defined) + list of weak references to the object """ as_text = html2text(doc) expected_lines = [line.strip() for line in expected_text.split("\n") if line] for expected_line in expected_lines: self.assertIn(expected_line, as_text) + def test_long_signatures(self): + from collections.abc import Callable + from typing import Literal, Annotated + + class A: + def __init__(self, + arg1: Callable[[int, int, int], str], + arg2: Literal['some value', 'other value'], + arg3: Annotated[int, 'some docs about this type'], + ) -> None: + ... + + doc = pydoc.render_doc(A) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: class A in module %s + +class A(builtins.object) + | A( + | arg1: collections.abc.Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | Methods defined here: + | + | __init__( + | self, + | arg1: collections.abc.Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s +''' % (__name__, + '' if MISSING_C_DOCSTRINGS else '\n | dictionary for instance variables', + '' if MISSING_C_DOCSTRINGS else '\n | list of weak references to the object', + )) + + def func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8], + ) -> Annotated[int, 'Some other']: + ... + + doc = pydoc.render_doc(func) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function func in module %s + +func( + arg1: collections.abc.Callable[[typing.Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] +) -> Annotated[int, 'Some other'] +''' % __name__) + + def function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str, + ): + ... + + doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s + +function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str +) +''' % __name__) + + does_not_have_name = lambda \ + very_long_parameter_name_that_should_not_fit_into_a_single_line, \ + second_very_long_parameter_name: ... + + doc = pydoc.render_doc(does_not_have_name) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function in module %s + + lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name +''' % __name__) + def test__future__imports(self): # __future__ features are excluded from module help, # except when it's the __future__ module itself @@ -983,7 +1239,8 @@ def test_url_search_package_error(self): sys.path.insert(0, TESTFN) try: with self.assertRaisesRegex(ValueError, "ouch"): - import test_error_package # Sanity check + # Sanity check + import test_error_package # noqa: F401 text = self.call_url_handler("search?key=test_error_package", "Pydoc: Search Results") @@ -1035,19 +1292,34 @@ def test_modules_search_builtin(self): self.assertTrue(result.startswith(expected)) def test_importfile(self): - loaded_pydoc = pydoc.importfile(pydoc.__file__) + try: + loaded_pydoc = pydoc.importfile(pydoc.__file__) + + self.assertIsNot(loaded_pydoc, pydoc) + self.assertEqual(loaded_pydoc.__name__, 'pydoc') + self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) + self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + finally: + sys.modules['pydoc'] = pydoc + + +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h - self.assertIsNot(loaded_pydoc, pydoc) - self.assertEqual(loaded_pydoc.__name__, 'pydoc') - self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) - self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + +class Square(Rect): + area = property(lambda self: self.side**2) class TestDescriptions(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_module(self): # Check that pydocfodder module can be described - from test import pydocfodder doc = pydoc.render_doc(pydocfodder) self.assertIn("pydocfodder", doc) @@ -1065,13 +1337,15 @@ def test_generic_alias(self): doc = pydoc.render_doc(typing.List[int], renderer=pydoc.plaintext) self.assertIn('_GenericAlias in module typing', doc) self.assertIn('List = class list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) self.assertEqual(pydoc.describe(list[int]), 'GenericAlias') doc = pydoc.render_doc(list[int], renderer=pydoc.plaintext) self.assertIn('GenericAlias in module builtins', doc) self.assertIn('\nclass list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) def test_union_type(self): self.assertEqual(pydoc.describe(typing.Union[int, str]), '_UnionGenericAlias') @@ -1085,7 +1359,8 @@ def test_union_type(self): doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) self.assertIn('UnionType in module types object', doc) self.assertIn('\nclass UnionType(builtins.object)', doc) - self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) def test_special_form(self): self.assertEqual(pydoc.describe(typing.NoReturn), '_SpecialForm') @@ -1147,7 +1422,7 @@ def test_unbound_python_method(self): @requires_docstrings def test_unbound_builtin_method(self): self.assertEqual(self._get_summary_line(_pickle.Pickler.dump), - "dump(self, obj, /)") + "dump(self, obj, /) unbound _pickle.Pickler method") # these no longer include "self" def test_bound_python_method(self): @@ -1199,7 +1474,7 @@ def test_module_level_callable_o(self): def test_unbound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(str.lower), - "lower(self, /)") + "lower(self, /) unbound builtins.str method") def test_bound_builtin_method_noargs(self): self.assertEqual(self._get_summary_line(''.lower), @@ -1207,7 +1482,7 @@ def test_bound_builtin_method_noargs(self): def test_unbound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set.add), - "add(self, object, /)") + "add(self, object, /) unbound builtins.set method") def test_bound_builtin_method_o(self): self.assertEqual(self._get_summary_line(set().add), @@ -1215,7 +1490,7 @@ def test_bound_builtin_method_o(self): def test_unbound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set.__contains__), - "__contains__(self, object, /)") + "__contains__(self, object, /) unbound builtins.set method") def test_bound_builtin_method_coexist_o(self): self.assertEqual(self._get_summary_line(set().__contains__), @@ -1223,70 +1498,80 @@ def test_bound_builtin_method_coexist_o(self): def test_unbound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), - "utcnow(type, /)") + "utcnow(type, /) unbound datetime.datetime method") def test_bound_builtin_classmethod_noargs(self): self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), - "utcnow() method of builtins.type instance") + "utcnow() class method of datetime.datetime") def test_unbound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), - "__class_getitem__(type, object, /)") + "__class_getitem__(type, object, /) unbound builtins.dict method") def test_bound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__class_getitem__), - "__class_getitem__(object, /) method of builtins.type instance") + "__class_getitem__(object, /) class method of builtins.dict") @support.cpython_only + @requires_docstrings def test_module_level_callable_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.func_with_unrepresentable_signature self.assertEqual(self._get_summary_line(builtin), "func_with_unrepresentable_signature(a, b=)") @support.cpython_only + @requires_docstrings def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.staticmeth), "staticmeth(a, b=)") @support.cpython_only + @requires_docstrings def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), - "pop(self, key, default=, /)") - import _testcapi + "pop(self, key, default=, /) " + "unbound builtins.dict method") + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.meth), - "meth(self, /, a, b=)") + "meth(self, /, a, b=) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only + @requires_docstrings def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " "method of builtins.dict instance") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") obj = _testcapi.DocStringUnrepresentableSignatureTest() self.assertEqual(self._get_summary_line(obj.meth), "meth(a, b=) " "method of _testcapi.DocStringUnrepresentableSignatureTest instance") @support.cpython_only + @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest descr = cls.__dict__['classmeth'] self.assertEqual(self._get_summary_line(descr), - "classmeth(type, /, a, b=)") + "classmeth(type, /, a, b=) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") @support.cpython_only + @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.classmeth), - "classmeth(a, b=) method of builtins.type instance") + "classmeth(a, b=) class method of " + "_testcapi.DocStringUnrepresentableSignatureTest") def test_overridden_text_signature(self): class C: @@ -1317,10 +1602,10 @@ def smeth(*args, **kwargs): self.assertEqual(self._get_summary_line(C.meth), "meth" + unbound) self.assertEqual(self._get_summary_line(C().meth), - "meth" + bound + " method of test.test_pydoc.C instance") + "meth" + bound + " method of test.test_pydoc.test_pydoc.C instance") C.cmeth.__func__.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.cmeth), - "cmeth" + bound + " method of builtins.type instance") + "cmeth" + bound + " class method of test.test_pydoc.test_pydoc.C") C.smeth.__text_signature__ = text_signature self.assertEqual(self._get_summary_line(C.smeth), "smeth" + unbound) @@ -1357,13 +1642,13 @@ def cm(cls, x): 'cm(...)\n' ' A class method\n') self.assertEqual(self._get_summary_lines(X.cm), """\ -cm(x) method of builtins.type instance +cm(x) class method of test.test_pydoc.test_pydoc.X A class method """) self.assertIn(""" | Class methods defined here: | - | cm(x) from builtins.type + | cm(x) | A class method """, pydoc.plain(pydoc.render_doc(X))) @@ -1419,13 +1704,13 @@ def test_namedtuple_field_descriptor(self): @requires_docstrings def test_property(self): - class Rect: - @property - def area(self): - '''Area of the rect''' - return self.w * self.h - self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area Area of the rect """) self.assertIn(""" @@ -1520,12 +1805,150 @@ def a_fn_with_https_link(): ) +class PydocFodderTest(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def getsection(self, text, beginline, endline): + lines = text.splitlines() + beginindex, endindex = 0, None + if beginline is not None: + beginindex = lines.index(beginline) + if endline is not None: + endindex = lines.index(endline, beginindex) + return lines[beginindex:endindex] + + def test_text_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.TextDoc() + result = doc.docclass(cls) + result = clean_text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f' | Methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_method_alias = A_method(self)', lines) + self.assertIn(' | B_method_alias = B_method(self)', lines) + self.assertIn(' | A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn(' | A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | global_func_alias = global_func(x, y)', lines) + self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) + self.assertIn(' | __repr__(self, /) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f' | Static methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn(' | B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn(' | A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + + lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) + self.assertIn(' | B_classmethod(x)', lines) + self.assertIn(' | B_classmethod_alias = B_classmethod(x)', lines) + + def test_html_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.HTMLDoc() + result = doc.docclass(cls) + result = html2text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f'Methods {where}:', '-'*70) + self.assertIn('A_method_alias = A_method(self)', lines) + self.assertIn('B_method_alias = B_method(self)', lines) + self.assertIn('A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn('A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('global_func_alias = global_func(x, y)', lines) + self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) + self.assertIn('__repr__(self, /) from builtins.object', lines) + self.assertIn('object_repr = __repr__(self, /)', lines) + + lines = self.getsection(result, f'Static methods {where}:', '-'*70) + self.assertIn('A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn('B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn('A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + + lines = self.getsection(result, f'Class methods {where}:', '-'*70) + self.assertIn('B_classmethod(x)', lines) + self.assertIn('B_classmethod_alias = B_classmethod(x)', lines) + + def test_text_doc_inherited_routines_in_class(self): + self.test_text_doc_routines_in_class(pydocfodder.D) + + def test_html_doc_inherited_routines_in_class(self): + self.test_html_doc_routines_in_class(pydocfodder.D) + + def test_text_doc_routines_in_module(self): + doc = pydoc.TextDoc() + result = doc.docmodule(pydocfodder) + result = clean_text(result) + lines = self.getsection(result, 'FUNCTIONS', 'FILE') + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn(' A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + def test_html_doc_routines_in_module(self): + doc = pydoc.HTMLDoc() + result = doc.docmodule(pydocfodder) + result = html2text(result) + lines = self.getsection(result, ' Functions', None) + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn('A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + + @unittest.skipIf( is_emscripten or is_wasi, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): """Tests for pydoc._start_server""" + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_server(self): # Minimal test that starts the server, checks that it works, then stops @@ -1588,9 +2011,14 @@ def test_url_requests(self): ("foobar", "Pydoc: Error - foobar"), ] - with self.restrict_walk_packages(): - for url, title in requests: - self.call_url_handler(url, title) + self.assertIs(sys.modules['pydoc'], pydoc) + try: + with self.restrict_walk_packages(): + for url, title in requests: + self.call_url_handler(url, title) + finally: + # Some requests reload the module and change sys.modules. + sys.modules['pydoc'] = pydoc class TestHelper(unittest.TestCase): @@ -1600,6 +2028,9 @@ def test_keywords(self): class PydocWithMetaClasses(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @requires_docstrings diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index a542abaf1f35aa..1d56ccd71cf962 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -544,7 +544,7 @@ def handler(text): parser = expat.ParserCreate() parser.CharacterDataHandler = handler - self.assertRaises(Exception, parser.Parse, xml.encode('iso8859')) + self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859')) class ChardataBufferTest(unittest.TestCase): """ @@ -755,5 +755,59 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ReparseDeferralTest(unittest.TestCase): + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py new file mode 100644 index 00000000000000..8359d9844623c2 --- /dev/null +++ b/Lib/test/test_pyrepl/__init__.py @@ -0,0 +1,15 @@ +import os +import sys +from test.support import requires, load_package_tests +from test.support.import_helper import import_module + +if sys.platform != "win32": + # On non-Windows platforms, testing pyrepl currently requires that the + # 'curses' resource be given on the regrtest command line using the -u + # option. Additionally, we need to attempt to import curses and readline. + requires("curses") + curses = import_module("curses") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pyrepl/__main__.py b/Lib/test/test_pyrepl/__main__.py new file mode 100644 index 00000000000000..cbe9e01d0df820 --- /dev/null +++ b/Lib/test/test_pyrepl/__main__.py @@ -0,0 +1,4 @@ +import unittest +from test.test_pyrepl import load_tests + +unittest.main() diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py new file mode 100644 index 00000000000000..58b1a92d68b184 --- /dev/null +++ b/Lib/test/test_pyrepl/support.py @@ -0,0 +1,157 @@ +from code import InteractiveConsole +from functools import partial +from typing import Iterable +from unittest.mock import MagicMock + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent + + +def multiline_input(reader: ReadlineAlikeReader, namespace: dict | None = None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def more_lines(text: str, namespace: dict | None = None): + if namespace is None: + namespace = {} + src = _strip_final_indent(text) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def code_to_events(code: str): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def clean_screen(screen: Iterable[str]): + """Cleans color and console characters out of a screen output. + + This is useful for screen testing, it increases the test readability since + it strips out all the unreadable side of the screen. + """ + output = [] + for line in screen: + if line.startswith(">>>") or line.startswith("..."): + line = line[3:] + output.append(line) + return "\n".join(output).strip() + + +def prepare_reader(console: Console, **kwargs): + config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def handle_all_events( + events, prepare_console=prepare_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + except KeyboardInterrupt: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_console, width=10), +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8"): + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self) -> None: + pass + + def repaint(self) -> None: + pass diff --git a/Lib/test/test_pyrepl/test_input.py b/Lib/test/test_pyrepl/test_input.py new file mode 100644 index 00000000000000..c78c876c2c4c2a --- /dev/null +++ b/Lib/test/test_pyrepl/test_input.py @@ -0,0 +1,102 @@ +import unittest + +from _pyrepl.console import Event +from _pyrepl.input import KeymapTranslator + + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py new file mode 100644 index 00000000000000..31f08cdb25e078 --- /dev/null +++ b/Lib/test/test_pyrepl/test_interact.py @@ -0,0 +1,113 @@ +import contextlib +import io +import unittest +from unittest.mock import patch +from textwrap import dedent + +from test.support import force_not_colorized + +from _pyrepl.console import InteractiveColoredConsole + + +class TestSimpleInteract(unittest.TestCase): + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + class A: + def foo(self): + + + pass + + class B: + def bar(self): + pass + + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with ( + patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, + patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), + ): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + showsyntaxerror.assert_not_called() + + + def test_multiple_statements_output(self): + namespace = {} + code = dedent("""\ + b = 1 + b + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "1\n") + + def test_empty(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "") + + def test_runsource_compiles_and_runs_code(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + with patch.object(console, "runcode") as mock_runcode: + console.runsource(source) + mock_runcode.assert_called_once() + + def test_runsource_returns_false_for_successful_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + + @force_not_colorized + def test_runsource_returns_false_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn('SyntaxError', f.getvalue()) + + def test_runsource_shows_syntax_error_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + source = dedent("""\ + match 1: + case {0: _, 0j: _}: + pass + """) + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + + def test_no_active_future(self): + console = InteractiveColoredConsole() + source = "x: int = 1; print(__annotate__(1))" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': }\n") diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py new file mode 100644 index 00000000000000..2c97066b2c7043 --- /dev/null +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -0,0 +1,120 @@ +import string +import unittest + +from _pyrepl.keymap import _keynames, _escapes, parse_keys, compile_keymap, KeySpecError + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + """Ensure that single ascii characters or single digits are parsed as single characters.""" + test_cases = [(key, [key]) for key in string.ascii_letters + string.digits] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_keynames(self): + """Ensure that keynames are parsed to their corresponding mapping. + + A keyname is expected to be of the following form: \\ such as \\ + which would get parsed as "left". + """ + test_cases = [(f"\\<{keyname}>", [parsed_keyname]) for keyname, parsed_keyname in _keynames.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_escape_sequences(self): + """Ensure that escaping sequences are parsed to their corresponding mapping.""" + test_cases = [(f"\\{escape}", [parsed_escape]) for escape, parsed_escape in _escapes.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_control_sequences(self): + """Ensure that supported control sequences are parsed successfully.""" + keys = ["@", "[", "]", "\\", "^", "_", "\\", "\\"] + keys.extend(string.ascii_letters) + test_cases = [(f"\\C-{key}", chr(ord(key) & 0x1F)) for key in []] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + def test_keyspec_errors(self): + cases = [ + ("\\Ca", "\\C must be followed by `-'"), + ("\\ca", "\\C must be followed by `-'"), + ("\\C-\\C-", "doubled \\C-"), + ("\\Ma", "\\M must be followed by `-'"), + ("\\ma", "\\M must be followed by `-'"), + ("\\M-\\M-", "doubled \\M-"), + ("\\", "unrecognised keyname"), + ("\\大", "unknown backslash escape"), + ("\\C-\\", "\\C- followed by invalid key") + ] + for test_keys, expected_err in cases: + with self.subTest(f"{test_keys} should give error {expected_err}"): + with self.assertRaises(KeySpecError) as e: + parse_keys(test_keys) + self.assertIn(expected_err, str(e.exception)) + + def test_index_errors(self): + test_cases = ["\\", "\\C", "\\C-\\C"] + for test_keys in test_cases: + with self.assertRaises(IndexError): + parse_keys(test_keys) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + def test_clashing_definitions(self): + km = {b'a': 'c', b'a' + b'b': 'd'} + with self.assertRaises(KeySpecError): + compile_keymap(km) + + def test_non_bytes_key(self): + with self.assertRaises(TypeError): + compile_keymap({123: 'a'}) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py new file mode 100644 index 00000000000000..8fff372da97a6a --- /dev/null +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -0,0 +1,1006 @@ +import io +import itertools +import os +import pathlib +import rlcompleter +import select +import subprocess +import sys +import tempfile +from unittest import TestCase, skipUnless +from unittest.mock import patch +from test.support import force_not_colorized +from test.support import SHORT_TIMEOUT +from test.support.import_helper import import_module +from test.support.os_helper import unlink + +from .support import ( + FakeConsole, + handle_all_events, + handle_events_narrow_console, + more_lines, + multiline_input, + code_to_events, + clean_screen +) +from _pyrepl.console import Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.readline import multiline_input as readline_multiline_input + +try: + import pty +except ImportError: + pty = None + +class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_up_arrow_simple(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' å¯å£å¯ä¹; å¯å£å¯æ¨‚'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' å¯å£å¯ä¹; å¯å£å¯æ¨‚'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' å¯å£å¯ä¹; å¯å£å¯æ¨‚'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' å¯å£å¯ä¹; å¯å£å¯æ¨‚'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = 'å¯å£å¯ä¹; å¯å£å¯æ¨‚'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = 'å¯å£å¯ä¹; å¯å£å¯æ¨‚' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplAutoindent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_auto_indent_default(self): + # fmt: off + input_code = ( + "def f():\n" + "pass\n\n" + ) + + output_code = ( + "def f():\n" + " pass\n" + " " + ) + # fmt: on + + def test_auto_indent_continuation(self): + # auto indenting according to previous user indentation + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_prev_block(self): + # auto indenting according to indentation in different block + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + code_to_events( + "def g():\n" + "pass\n\n" + ), + ) + + output_code = ( + "def g():\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output1 = multiline_input(reader) + output2 = multiline_input(reader) + self.assertEqual(output2, output_code) + + def test_auto_indent_multiline(self): + # fmt: off + events = itertools.chain( + code_to_events( + "def f():\n" + "pass" + ), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new line should be autoindented + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + code_to_events( + "pass" + ), + [ + # go to end of last line + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # double newline to terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_comment(self): + # fmt: off + events = code_to_events( + "def f(): # foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): # foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_ignore_comments(self): + # fmt: off + events = code_to_events( + "pass #:\n" + ) + + output_code = ( + "pass #:" + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.can_colorize = False + return reader + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="delete", raw=bytearray(b"\x7F")), + Event(evt="key", data="right", raw=bytearray(b"g")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="a", raw=bytearray(b"a")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + self.assertEqual(clean_screen(reader.screen), "def f():\n ...") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n pass\n ") + self.assertEqual(clean_screen(reader.screen), "def g():\n pass") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_with_multiline_entries(self): + code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" + events = list(itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + )) + + reader = self.prepare_reader(events) + output = multiline_input(reader) + output = multiline_input(reader) + output = multiline_input(reader) + self.assertEqual( + clean_screen(reader.screen), + 'def foo():\n x = 1\n y = 2\n z = 3' + ) + self.assertEqual(output, "def foo():\n x = 1\n y = 2\n z = 3\n ") + + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + self.assertEqual(clean_screen(reader.screen), "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + self.assertEqual(clean_screen(reader.screen), "c") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + @patch("rlcompleter._readline_available", False) + def test_simple_completion(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + def test_updown_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + + @patch("_pyrepl.readline._ReadlineWrapper.get_reader") + @patch("sys.stderr", new_callable=io.StringIO) + def test_completion_with_warnings(self, mock_stderr, mock_get_reader): + class Dummy: + @property + def test_func(self): + import warnings + + warnings.warn("warnings\n") + return None + + dummy = Dummy() + events = code_to_events("dummy.test_func.\t\n\n") + namespace = {"dummy": dummy} + reader = self.prepare_reader(events, namespace) + mock_get_reader.return_value = reader + output = readline_multiline_input(more_lines, ">>>", "...") + self.assertEqual(output, "dummy.test_func.__") + self.assertEqual(mock_stderr.getvalue(), "") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:\n" + " pass\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + "def f():\n" + " x = y\n" + " \n" + " y = z\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + "def f():\n" + "x = y\n" + "\n" + "y = z\n\n" + ) + + expected = ( + "def f():\n" + " x = y\n" + " " + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + "def a():\n" + "for x in range(10):\n" + "if x%2:\n" + "print(x)\n" + "else:\n" + "pass\n\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:" + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +@skipUnless(pty, "requires pty") +class TestMain(TestCase): + @force_not_colorized + def test_exposed_globals_in_repl(self): + pre = "['__annotations__', '__builtins__'" + post = "'__loader__', '__name__', '__package__', '__spec__']" + output, exit_code = self.run_repl(["sorted(dir())", "exit"]) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + + # if `__main__` is not a file (impossible with pyrepl) + case1 = f"{pre}, '__doc__', {post}" in output + + # if `__main__` is an uncached .py file (no .pyc) + case2 = f"{pre}, '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file and the .py source exists + case3 = f"{pre}, '__cached__', '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file but there's no .py source file + case4 = f"{pre}, '__cached__', '__doc__', {post}" in output + + self.assertTrue(case1 or case2 or case3 or case4, output) + + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can\'t use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + @force_not_colorized + def test_python_basic_repl(self): + env = os.environ.copy() + commands = ("from test.support import initialized_with_pyrepl\n" + "initialized_with_pyrepl()\n" + "exit()\n") + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + self.assertIn("True", output) + self.assertNotIn("False", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def test_not_wiping_history_file(self): + # skip, if readline module is not available + import_module('readline') + + hfile = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + commands = "123\nspam\nexit()\n" + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + hfile.file.truncate() + hfile.close() + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tuple[str, int]: + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + + os.close(master_fd) + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + return "\n".join(output), exit_code diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py new file mode 100644 index 00000000000000..e82c3ca0bb5cc2 --- /dev/null +++ b/Lib/test/test_pyrepl/test_reader.py @@ -0,0 +1,283 @@ +import itertools +import functools +import rlcompleter +from unittest import TestCase +from unittest.mock import MagicMock + +from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader +from _pyrepl.console import Event +from _pyrepl.reader import Reader + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.screen + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_control_characters(self): + code = 'flag = "ðŸ³ï¸â€ðŸŒˆ"' + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, 'flag = "ðŸ³ï¸\\u200d🌈"') + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") + + def test_newline_within_block_trailing_whitespace(self): + # fmt: off + code = ( + "def foo():\n" + "a = 1\n" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new lines in-block shouldn't terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + # end of line 2 + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # a double new line in-block should terminate the block + # even if its followed by whitespace + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + no_paste_reader = functools.partial(prepare_reader, paste_mode=False) + reader, _ = handle_all_events(events, prepare_reader=no_paste_reader) + + expected = ( + "def foo():\n" + " \n" + " \n" + " a = 1\n" + " \n" + " " # HistoricalReader will trim trailing whitespace + ) + self.assert_screen_equals(reader, expected) + self.assertTrue(reader.finished) + + def test_input_hook_is_called_if_set(self): + input_hook = MagicMock() + def _prepare_console(events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + console.input_hook = input_hook + return console + + events = code_to_events("a") + reader, _ = handle_all_events(events, prepare_console=_prepare_console) + + self.assertEqual(len(input_hook.mock_calls), 4) + + def test_keyboard_interrupt_clears_screen(self): + namespace = {"itertools": itertools} + code = "import itertools\nitertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + Event(evt='key', data='\x03', raw=bytearray(b'\x03')), # Ctrl-C + ]) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + self.assertEqual(reader.calc_screen(), code.split("\n")) + + def test_prompt_length(self): + # Handles simple ASCII prompt + ps1 = ">>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 4) + + # Handles ANSI escape sequences + ps1 = "\033[0;32m>>> \033[0m" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles ANSI escape sequences bracketed in \001 .. \002 + ps1 = "\001\033[0;32m\002>>> \001\033[0m\002" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles wide characters in prompt + ps1 = "樂>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 5) + + # Handles wide characters AND ANSI sequences together + ps1 = "\001\033[0;32m\002樂>\001\033[0m\002> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") + self.assertEqual(l, 5) + + def test_completions_updated_on_key_press(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + actual = reader.screen + self.assertEqual(len(actual), 2) + self.assertEqual(actual[0].rstrip(), "itertools.accumulate(") + self.assertEqual(actual[1], f"{code}a") + + def test_key_press_on_tab_press_once(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + self.assert_screen_equals(reader, f"{code}a") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py new file mode 100644 index 00000000000000..e3bbabcb0089fb --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -0,0 +1,314 @@ +import itertools +import sys +import unittest +from functools import partial +from unittest import TestCase +from unittest.mock import MagicMock, call, patch, ANY + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event + from _pyrepl.unix_console import UnixConsole +except ImportError: + pass + + +def unix_console(events, **kwargs): + console = UnixConsole() + console.get_event = MagicMock(side_effect=events) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console), +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +TERM_CAPABILITIES = { + "bel": b"\x07", + "civis": b"\x1b[?25l", + "clear": b"\x1b[H\x1b[2J", + "cnorm": b"\x1b[?12l\x1b[?25h", + "cub": b"\x1b[%p1%dD", + "cub1": b"\x08", + "cud": b"\x1b[%p1%dB", + "cud1": b"\n", + "cuf": b"\x1b[%p1%dC", + "cuf1": b"\x1b[C", + "cup": b"\x1b[%i%p1%d;%p2%dH", + "cuu": b"\x1b[%p1%dA", + "cuu1": b"\x1b[A", + "dch1": b"\x1b[P", + "dch": b"\x1b[%p1%dP", + "el": b"\x1b[K", + "hpa": b"\x1b[%i%p1%dG", + "ich": b"\x1b[%p1%d@", + "ich1": None, + "ind": b"\n", + "pad": None, + "ri": b"\x1bM", + "rmkx": b"\x1b[?1l\x1b>", + "smkx": b"\x1b[?1h\x1b=", +} + + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) +@patch( + "_pyrepl.curses.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch("_pyrepl.curses.setupterm", lambda a, b: None) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +class TestConsole(TestCase): + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + con.restore() + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) + console.restore() + con.restore() diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py new file mode 100644 index 00000000000000..301f79927a741f --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -0,0 +1,117 @@ +import tempfile +import unittest +import sys +from unittest.mock import patch + +try: + from _pyrepl.console import Event + from _pyrepl.unix_eventqueue import EventQueue +except ImportError: + pass + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnixEventQueue(unittest.TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def test_get(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(self.file.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py new file mode 100644 index 00000000000000..4a3b2baf64a944 --- /dev/null +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -0,0 +1,334 @@ +import sys +import unittest + +if sys.platform != "win32": + raise unittest.SkipTest("test only relevant on win32") + + +import itertools +from functools import partial +from typing import Iterable +from unittest import TestCase +from unittest.mock import MagicMock, call + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event, Console + from _pyrepl.windows_console import ( + WindowsConsole, + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, + ) +except ImportError: + pass + + +class WindowsConsoleTests(TestCase): + def console(self, events, **kwargs) -> Console: + console = WindowsConsole() + console.get_event = MagicMock(side_effect=events) + console._scroll = MagicMock() + console._hide_cursor = MagicMock() + console._show_cursor = MagicMock() + console._getscrollbacksize = MagicMock(42) + console.out = MagicMock() + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + def handle_events(self, events: Iterable[Event], **kwargs): + return handle_all_events(events, partial(self.console, **kwargs)) + + def handle_events_narrow(self, events): + return self.handle_events(events, width=5) + + def handle_events_short(self, events): + return self.handle_events(events, height=1) + + def handle_events_height_3(self, events): + return self.handle_events(events, height=3) + + def test_simple_addition(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_wrap(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events_narrow(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"\\") + con.out.write.assert_any_call(b"\n") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_resize_wider(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events_narrow(events) + + console.height = 20 + console.width = 80 + console.getheightwidth = MagicMock(lambda _: (20, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(self.move_right(2)) + con.out.write.assert_any_call(self.move_up(2)) + con.out.write.assert_any_call(b"567890") + + con.restore() + + def test_resize_narrower(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events(events) + + console.height = 20 + console.width = 4 + console.getheightwidth = MagicMock(lambda _: (20, 4)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(b"456\\") + con.out.write.assert_any_call(b"789\\") + + con.restore() + + def test_cursor_left(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.restore() + + def test_cursor_left_right(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(self.move_right()) + con.restore() + + def test_cursor_up(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_cursor_up_down(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.out.write.assert_any_call(self.move_down()) + con.restore() + + def test_cursor_back_write(self): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(b"21") + con.restore() + + def test_multiline_function_move_up_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(5)) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(8)) + con.out.write.assert_any_call(self.erase_in_line()) + con.restore() + + def test_resize_bigger_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_short(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(b"def f():"), + call(self.move_left(3)), + call(self.move_down()), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(self.erase_in_line()), + call(b" foo"), + ] + ) + console.restore() + con.restore() + + def move_up(self, lines=1): + return MOVE_UP.format(lines).encode("utf8") + + def move_down(self, lines=1): + return MOVE_DOWN.format(lines).encode("utf8") + + def move_left(self, cols=1): + return MOVE_LEFT.format(cols).encode("utf8") + + def move_right(self, cols=1): + return MOVE_RIGHT.format(cols).encode("utf8") + + def erase_in_line(self): + return ERASE_IN_LINE.encode("utf8") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 33113a72e6b6a9..6dced7df0064d7 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -241,6 +241,418 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): + try: + q.put((i, "YDLO")) + except self.queue.ShutDown: + results.append(False) + break + + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: + try: + q.get(False) + q.task_done() + except self.queue.ShutDown: + results.append(True) + break + except self.queue.Empty: + pass + + def _shutdown_thread(self, q, results, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + results.append(q.qsize() == 0) + + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. + q = self.type2test() + ps = [] + res_puts = [] + res_gets = [] + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + for thread in ps: + thread.join() + + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads shoud raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, 1) + + q.shutdown(immediate) + go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + + class QueueTest(BaseQueueTestMixin): def setUp(self): diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 5936d7535edd5f..6d26a61bee4292 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,6 +185,20 @@ def test_class_cause(self): else: self.fail("No exception raised") + def test_class_cause_nonexception_result(self): + class ConstructsNone(BaseException): + @classmethod + def __new__(*args, **kwargs): + return None + try: + raise IndexError from ConstructsNone + except TypeError as e: + self.assertIn("should have returned an instance of BaseException", str(e)) + except IndexError: + self.fail("Wrong kind of exception raised") + else: + self.fail("No exception raised") + def test_instance_cause(self): cause = KeyError() try: diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 50bea7be6d54c7..9a44ab1768656a 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -1081,6 +1082,7 @@ def test_binomialvariate(self): B(n=1, p=-0.5) # Negative p with self.assertRaises(ValueError): B(n=1, p=1.5) # p > 1.0 + self.assertEqual(B(0, 0.5), 0) # n == 0 self.assertEqual(B(10, 0.0), 0) # p == 0.0 self.assertEqual(B(10, 1.0), 10) # p == 1.0 self.assertTrue(B(1, 0.3) in {0, 1}) # n == 1 fast path @@ -1088,6 +1090,9 @@ def test_binomialvariate(self): self.assertTrue(B(1, 0.0) in {0}) # n == 1 fast path self.assertTrue(B(1, 1.0) in {1}) # n == 1 fast path + # BG method very small p + self.assertEqual(B(5, 1e-18), 0) + # BG method p <= 0.5 and n*p=1.25 self.assertTrue(B(5, 0.25) in set(range(6))) @@ -1393,5 +1398,47 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertTrue(help_text.startswith("usage: ")) + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.266632777287572), + ("2.5", 2.266632777287572), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 1eca22f45378df..a93c2aef170fc8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,12 +1,11 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch, requires_resource) import locale import re import string import sys -import time import unittest import warnings from re import Scanner @@ -14,7 +13,7 @@ # some platforms lack working multiprocessing try: - import _multiprocessing + import _multiprocessing # noqa: F401 except ImportError: multiprocessing = None else: @@ -47,7 +46,7 @@ def recurse(actual, expect): recurse(actual, expect) def checkPatternError(self, pattern, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(pattern) with self.subTest(pattern=pattern): err = cm.exception @@ -56,7 +55,7 @@ def checkPatternError(self, pattern, errmsg, pos=None): self.assertEqual(err.pos, pos) def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.sub(pattern, repl, string) with self.subTest(pattern=pattern, repl=repl): err = cm.exception @@ -64,6 +63,9 @@ def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): if pos is not None: self.assertEqual(err.pos, pos) + def test_error_is_PatternError_alias(self): + assert re.error is re.PatternError + def test_keep_buffer(self): # See bug 14212 b = bytearray(b'x') @@ -154,7 +156,7 @@ def test_basic_re_sub(self): (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': with self.subTest(c): - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') @@ -836,10 +838,10 @@ def test_other_escapes(self): re.purge() # for warnings for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - self.assertRaises(re.error, re.compile, '\\%c' % c) + self.assertRaises(re.PatternError, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - self.assertRaises(re.error, re.compile, '[\\%c]' % c) + self.assertRaises(re.PatternError, re.compile, '[\\%c]' % c) def test_named_unicode_escapes(self): # test individual Unicode named escapes @@ -970,14 +972,14 @@ def test_lookbehind(self): self.assertIsNone(re.match(r'(?:(a)|(x))b(?<=(?(1)c|x))c', 'abc')) self.assertTrue(re.match(r'(?:(a)|(x))b(?<=(?(1)b|x))c', 'abc')) # Group used before defined. - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?(2)b|x))(c)') self.assertIsNone(re.match(r'(a)b(?<=(?(1)c|x))(c)', 'abc')) self.assertTrue(re.match(r'(a)b(?<=(?(1)b|x))(c)', 'abc')) # Group defined in the same lookbehind pattern - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)\2)(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)\2)(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") @@ -1226,7 +1228,7 @@ def test_pickling(self): newpat = pickle.loads(pickled) self.assertEqual(newpat, oldpat) # current pickle expects the _compile() reconstructor in re module - from re import _compile + from re import _compile # noqa: F401 def test_copying(self): import copy @@ -1318,8 +1320,8 @@ def test_sre_byte_literals(self): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - self.assertRaises(re.error, re.compile, br"\u1234") - self.assertRaises(re.error, re.compile, br"\U00012345") + self.assertRaises(re.PatternError, re.compile, br"\u1234") + self.assertRaises(re.PatternError, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1341,8 +1343,8 @@ def test_sre_byte_class_literals(self): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - self.assertRaises(re.error, re.compile, br"[\u1234]") - self.assertRaises(re.error, re.compile, br"[\U00012345]") + self.assertRaises(re.PatternError, re.compile, br"[\u1234]") + self.assertRaises(re.PatternError, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1675,11 +1677,11 @@ def test_ascii_and_unicode_flag(self): self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) - self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(re.PatternError, re.compile, br'(?u)\w') self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) - self.assertRaises(re.error, re.compile, r'(?au)\w') + self.assertRaises(re.PatternError, re.compile, r'(?au)\w') def test_locale_flag(self): enc = locale.getpreferredencoding() @@ -1720,11 +1722,11 @@ def test_locale_flag(self): self.assertIsNone(pat.match(bletter)) # Incompatibilities self.assertRaises(ValueError, re.compile, '', re.LOCALE) - self.assertRaises(re.error, re.compile, '(?L)') + self.assertRaises(re.PatternError, re.compile, '(?L)') self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) - self.assertRaises(re.error, re.compile, b'(?aL)') + self.assertRaises(re.PatternError, re.compile, b'(?aL)') def test_scoped_flags(self): self.assertTrue(re.match(r'(?i:a)b', 'Ab')) @@ -2060,7 +2062,7 @@ def test_locale_compiled(self): self.assertIsNone(p4.match(b'\xc5\xc5')) def test_error(self): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile('(\u20ac))') err = cm.exception self.assertIsInstance(err.pattern, str) @@ -2072,14 +2074,14 @@ def test_error(self): self.assertIn(' at position 3', str(err)) self.assertNotIn(' at position 3', err.msg) # Bytes pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(b'(\xa4))') err = cm.exception self.assertIsInstance(err.pattern, bytes) self.assertEqual(err.pattern, b'(\xa4))') self.assertEqual(err.pos, 3) # Multiline pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(""" ( abc @@ -2279,19 +2281,21 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with CPUStopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.1) + self.assertLess(stopwatch.seconds, 0.1) def test_possessive_quantifiers(self): """Test Possessive Quantifiers @@ -2469,6 +2473,24 @@ def test_regression_gh94675(self): def test_fail(self): self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + def test_character_set_any(self): + # The union of complementary character sets mathes any character + # and is equivalent to "(?s:.)". + s = '1x\n' + for p in r'[\s\S]', r'[\d\D]', r'[\w\W]', r'[\S\s]', r'\s|\S': + with self.subTest(pattern=p): + self.assertEqual(re.findall(p, s), list(s)) + self.assertEqual(re.fullmatch('(?:' + p + ')+', s).group(), s) + + def test_character_set_none(self): + # Negation of the union of complementary character sets does not match + # any character. + s = '1x\n' + for p in r'[^\s\S]', r'[^\d\D]', r'[^\w\W]', r'[^\S\s]': + with self.subTest(pattern=p): + self.assertIsNone(re.search(p, s)) + self.assertIsNone(re.search('(?s:.)' + p, s)) + def get_debug_out(pat): with captured_stdout() as out: @@ -2820,7 +2842,7 @@ def test_re_tests(self): with self.subTest(pattern=pattern, string=s): if outcome == SYNTAX_ERROR: # Expected a syntax error - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): re.compile(pattern) continue diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 59dbef90380053..5e0e6f8dfac651 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -1,18 +1,16 @@ """ Very minimal unittests for parts of the readline module. """ -from contextlib import ExitStack -from errno import EIO import locale import os -import selectors -import subprocess import sys import tempfile +import textwrap import unittest from test.support import verbose from test.support.import_helper import import_module from test.support.os_helper import unlink, temp_dir, TESTFN +from test.support.pty_helper import run_pty from test.support.script_helper import assert_python_ok # Skip tests if there is no readline module @@ -21,7 +19,7 @@ if hasattr(readline, "_READLINE_LIBRARY_VERSION"): is_editline = ("EditLine wrapper" in readline._READLINE_LIBRARY_VERSION) else: - is_editline = (readline.__doc__ and "libedit" in readline.__doc__) + is_editline = readline.backend == "editline" def setUpModule(): @@ -147,6 +145,9 @@ def test_init(self): TERM='xterm-256color') self.assertEqual(stdout, b'') + def test_backend(self): + self.assertIn(readline.backend, ("readline", "editline")) + auto_history_script = """\ import readline readline.set_auto_history({}) @@ -166,6 +167,25 @@ def test_auto_history_disabled(self): # end, so don't expect it in the output. self.assertIn(b"History length: 0", output) + def test_set_complete_delims(self): + script = textwrap.dedent(""" + import readline + def complete(text, state): + if state == 0 and text == "$": + return "$complete" + return None + if readline.backend == "editline": + readline.parse_and_bind(r'bind "\\t" rl_complete') + else: + readline.parse_and_bind(r'"\\t": complete') + readline.set_completer_delims(" \\t\\n") + readline.set_completer(complete) + print(input()) + """) + + output = run_pty(script, input=b"$\t\n") + self.assertIn(b"$complete", output) + def test_nonascii(self): loc = locale.setlocale(locale.LC_CTYPE, None) if loc in ('C', 'POSIX'): @@ -181,7 +201,7 @@ def test_nonascii(self): script = r"""import readline -is_editline = readline.__doc__ and "libedit" in readline.__doc__ +is_editline = readline.backend == "editline" inserted = "[\xEFnserted]" macro = "|t\xEB[after]" set_pre_input_hook = getattr(readline, "set_pre_input_hook", None) @@ -304,55 +324,5 @@ def test_history_size(self): self.assertEqual(lines[-1].strip(), b"last input") -def run_pty(script, input=b"dummy input\r", env=None): - pty = import_module('pty') - output = bytearray() - [master, slave] = pty.openpty() - args = (sys.executable, '-c', script) - proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) - os.close(slave) - with ExitStack() as cleanup: - cleanup.enter_context(proc) - def terminate(proc): - try: - proc.terminate() - except ProcessLookupError: - # Workaround for Open/Net BSD bug (Issue 16762) - pass - cleanup.callback(terminate, proc) - cleanup.callback(os.close, master) - # Avoid using DefaultSelector and PollSelector. Kqueue() does not - # work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open - # BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4 - # either (Issue 20472). Hopefully the file descriptor is low enough - # to use with select(). - sel = cleanup.enter_context(selectors.SelectSelector()) - sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) - os.set_blocking(master, False) - while True: - for [_, events] in sel.select(): - if events & selectors.EVENT_READ: - try: - chunk = os.read(master, 0x10000) - except OSError as err: - # Linux raises EIO when slave is closed (Issue 5380) - if err.errno != EIO: - raise - chunk = b"" - if not chunk: - return output - output.extend(chunk) - if events & selectors.EVENT_WRITE: - try: - input = input[os.write(master, input):] - except OSError as err: - # Apparently EIO means the slave was closed - if err.errno != EIO: - raise - input = b"" # Stop writing - if not input: - sel.modify(master, selectors.EVENT_READ) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 26c0a670241f58..d4f4a69a7a38c1 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -21,12 +21,17 @@ import tempfile import textwrap import unittest +from xml.etree import ElementTree + from test import support -from test.support import os_helper, TestStats, without_optimizer +from test.support import import_helper +from test.support import os_helper from test.libregrtest import cmdline from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test +from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -143,18 +148,26 @@ def test_header(self): self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) with os_helper.EnvironmentVarGuard() as env: - env['SOURCE_DATE_EPOCH'] = '1' - + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' ns = self.parse_args(['--randomize']) regrtest = main.Regrtest(ns) self.assertFalse(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) @@ -184,34 +197,27 @@ def test_single(self): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = self.parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -219,10 +225,13 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) + self.assertEqual(ns.match_tests, + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': @@ -300,13 +309,23 @@ def test_multiprocess(self): self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') - self.checkError([opt, '2', '-T'], "don't go together") - self.checkError([opt, '0', '-T'], "don't go together") - def test_coverage(self): + def test_coverage_sequential(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = self.parse_args([opt]) + with support.captured_stderr() as stderr: + ns = self.parse_args([opt]) + self.assertTrue(ns.trace) + self.assertIn( + "collecting coverage without -j is imprecise", + stderr.getvalue(), + ) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_coverage_mp(self): + for opt in '-T', '--coverage': + with self.subTest(opt=opt): + ns = self.parse_args([opt, '-j1']) self.assertTrue(ns.trace) def test_coverdir(self): @@ -383,12 +402,22 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') - def check_ci_mode(self, args, use_resources, rerun=True): + def create_regrtest(self, args): ns = cmdline._parse_args(args) # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API - regrtest = main.Regrtest(ns) + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + if 'SOURCE_DATE_EPOCH' in env: + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + + return regrtest + + def check_ci_mode(self, args, use_resources, rerun=True): + regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -415,9 +444,11 @@ def test_fast_ci_python_cmd(self): self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) def test_fast_ci_resource(self): - # it should be possible to override resources - args = ['--fast-ci', '-u', 'network'] - use_resources = ['network'] + # it should be possible to override resources individually + args = ['--fast-ci', '-u-network'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + use_resources.remove('network') self.check_ci_mode(args, use_resources) def test_slow_ci(self): @@ -431,6 +462,33 @@ def test_dont_add_python_opts(self): ns = cmdline._parse_args(args) self.assertFalse(ns._add_python_opts) + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + + def test_verbose3_huntrleaks(self): + args = ['-R', '3:10', '--verbose3'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertIsNotNone(regrtest.hunt_refleak) + self.assertEqual(regrtest.hunt_refleak.warmups, 3) + self.assertEqual(regrtest.hunt_refleak.runs, 10) + self.assertFalse(regrtest.output_on_failure) + + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + @dataclasses.dataclass(slots=True) class Rerun: @@ -659,21 +717,26 @@ def list_regex(line_format, tests): state = f'{state} then {new_state}' self.check_line(output, f'Result: {state}', full=True) - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed, randseed) - return randseed + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" @@ -749,7 +812,9 @@ def setUp(self): self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) @@ -814,6 +879,8 @@ def test_tools_buildbot_test(self): test_args.append('-x64') # 64-bit build if not support.Py_DEBUG: test_args.append('+d') # Release build, use python.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + test_args.append('--disable-gil') self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -831,6 +898,8 @@ def test_pcbuild_rt(self): rt_args.append('-x64') # 64-bit build if support.Py_DEBUG: rt_args.append('-d') # Debug build, use python_d.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + rt_args.append('--disable-gil') self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) @@ -940,7 +1009,7 @@ def test_random(self): test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test, + output = self.run_tests('-r', f'--randseed={randseed}', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -951,7 +1020,32 @@ def test_random(self): # check that random.seed is used by default output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) - self.assertIsInstance(self.parse_random_seed(output), int) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string): ignore the env var + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) def test_fromfile(self): # test --fromfile @@ -1087,7 +1181,7 @@ def test_run(self): stats=TestStats(4, 1), forever=True) - @without_optimizer + @support.without_optimizer def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) @@ -1102,8 +1196,8 @@ def check_leak(self, code, what, *, run_workers=False): stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -1132,6 +1226,47 @@ def test_huntrleaks(self): def test_huntrleaks_mp(self): self.check_huntrleaks(run_workers=True) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak @@ -1614,6 +1749,9 @@ def test_other_bug(self): @support.cpython_only def test_uncollectable(self): + # Skip test if _testcapi is missing + import_helper.import_module('_testcapi') + code = textwrap.dedent(r""" import _testcapi import gc @@ -1975,8 +2113,31 @@ def test_dev_mode(self): self.check_executed_tests(output, tests, stats=len(tests), parallel=True) + def test_unload_tests(self): + # Test that unloading test modules does not break tests + # that import from other tests. + # The test execution order matters for this test. + # Both test_regrtest_a and test_regrtest_c which are executed before + # and after test_regrtest_b import a submodule from the test_regrtest_b + # package and use it in testing. test_regrtest_b itself does not import + # that submodule. + # Previously test_regrtest_c failed because test_regrtest_b.util in + # sys.modules was left after test_regrtest_a (making the import + # statement no-op), but new test_regrtest_b without the util attribute + # was imported for test_regrtest_b. + testdir = os.path.join(os.path.dirname(__file__), + 'regrtestdata', 'import_from_tests') + tests = [f'test_regrtest_{name}' for name in ('a', 'b', 'c')] + args = ['-Wd', '-E', '-bb', '-m', 'test', '--testdir=%s' % testdir, *tests] + output = self.run_python(args) + self.check_executed_tests(output, tests, stats=3) + def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + + # Skip test if _testinternalcapi is missing + import_helper.import_module('_testinternalcapi') + code = textwrap.dedent(r""" import sys import unittest @@ -2039,10 +2200,8 @@ def test_add_python_opts(self): @unittest.skipIf(support.is_android, 'raising SIGSEGV on Android is unreliable') def test_worker_output_on_failure(self): - try: - from faulthandler import _sigsegv - except ImportError: - self.skipTest("need faulthandler._sigsegv") + # Skip test if faulthandler is missing + import_helper.import_module('faulthandler') code = textwrap.dedent(r""" import faulthandler @@ -2074,6 +2233,67 @@ def test_crash(self): self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) self.check_line(output, "just before crash!", full=True, regex=False) + def test_verbose3(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_pass(self): + print("SPAM SPAM SPAM") + """) + testname = self.create_test(code=code) + + # Run sequentially + output = self.run_tests("--verbose3", testname) + self.check_executed_tests(output, testname, stats=1) + self.assertNotIn('SPAM SPAM SPAM', output) + + # -R option needs a debug build + if support.Py_DEBUG: + # Check for reference leaks, run in parallel + output = self.run_tests("-R", "3:3", "-j1", "--verbose3", testname) + self.check_executed_tests(output, testname, stats=1, parallel=True) + self.assertNotIn('SPAM SPAM SPAM', output) + + def test_xml(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_failed(self): + print("abc \x1b def") + self.fail() + """) + testname = self.create_test(code=code) + + # Run sequentially + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + output = self.run_tests(testname, "--junit-xml", filename, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=testname, + stats=TestStats(1, 1, 0)) + + # Test generated XML + with open(filename, encoding="utf8") as fp: + content = fp.read() + + testsuite = ElementTree.fromstring(content) + self.assertEqual(int(testsuite.get('tests')), 1) + self.assertEqual(int(testsuite.get('errors')), 0) + self.assertEqual(int(testsuite.get('failures')), 1) + + testcase = testsuite[0][0] + self.assertEqual(testcase.get('status'), 'run') + self.assertEqual(testcase.get('result'), 'completed') + self.assertGreater(float(testcase.get('time')), 0) + for out in testcase.iter('system-out'): + self.assertEqual(out.text, r"abc \x1b def") + class TestUtils(unittest.TestCase): def test_format_duration(self): @@ -2109,15 +2329,6 @@ def test_normalize_test_name(self): self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) - def test_get_signal_name(self): - for exitcode, expected in ( - (-int(signal.SIGINT), 'SIGINT'), - (-int(signal.SIGSEGV), 'SIGSEGV'), - (3221225477, "STATUS_ACCESS_VIOLATION"), - (0xC00000FD, "STATUS_STACK_OVERFLOW"), - ): - self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) - def test_format_resources(self): format_resources = utils.format_resources ALL_RESOURCES = utils.ALL_RESOURCES @@ -2138,6 +2349,143 @@ def test_format_resources(self): format_resources((*ALL_RESOURCES, "tzdata")), 'resources: all,tzdata') + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') + + # Test acceptance + with support.swap_attr(support, '_test_matchers', ()): + # match all + set_match_tests([]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match all using None + set_match_tests(None) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the full test identifier + set_match_tests([(test_access.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', True)]) + self.assertFalse(match_test(test_access)) + set_match_tests([('FileTests', True)]) + self.assertTrue(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + set_match_tests([('test_access', True), ('DONTMATCH', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test rejection + with support.swap_attr(support, '_test_matchers', ()): + # match the full test identifier + set_match_tests([(test_access.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', False)]) + self.assertTrue(match_test(test_access)) + set_match_tests([('FileTests', False)]) + self.assertFalse(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('test_access', False), ('DONTMATCH', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('*test_os', True), ('test_access', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + def test_sanitize_xml(self): + sanitize_xml = utils.sanitize_xml + + # escape invalid XML characters + self.assertEqual(sanitize_xml('abc \x1b\x1f def'), + r'abc \x1b\x1f def') + self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'), + r'nul:\x00, bell:\x07') + self.assertEqual(sanitize_xml('surrogate:\uDC80'), + r'surrogate:\udc80') + self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'), + r'illegal \ufffe and \uffff') + + # no escape for valid XML characters + self.assertEqual(sanitize_xml('a\n\tb'), + 'a\n\tb') + self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), + 'valid t\xe9xt \u20ac') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 2ee5117bcd7bd4..0e73fb6249c787 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -1,13 +1,14 @@ """Test the interactive interpreter.""" -import sys import os -import unittest import subprocess +import sys +import unittest from textwrap import dedent from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport -from test.support.script_helper import kill_python +from test.support.script_helper import assert_python_failure, kill_python, assert_python_ok +from test.support.import_helper import import_module if not has_subprocess_support: @@ -64,6 +65,7 @@ class TestInteractiveInterpreter(unittest.TestCase): # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_no_memory(self): + import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -161,10 +163,11 @@ def foo(x): output = kill_python(p) self.assertEqual(p.returncode, 0) - traceback_lines = output.splitlines()[-7:-1] + traceback_lines = output.splitlines()[-8:-1] expected_lines = [ ' File "", line 1, in ', ' foo(0)', + ' ~~~^^^', ' File "", line 2, in foo', ' 1 / x', ' ~~^~~', @@ -184,7 +187,7 @@ def bar(x): p.stdin.write(user_input) user_input2 = dedent(""" import linecache - print(linecache.cache['']) + print(linecache.cache['-1']) """) p.stdin.write(user_input2) output = kill_python(p) @@ -192,6 +195,8 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_no_tty_fails(self): + assert assert_python_failure("-m", "asyncio") class TestInteractiveModeSyntaxErrors(unittest.TestCase): diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 317e7ca8f8c853..d23d3623235f38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -138,7 +138,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 58729a9fea62fa..5f449cdc05c6ba 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -221,6 +221,7 @@ def do(bad): self.assertRaises(Exc, func, Bad()) @support.no_tracing + @support.infinite_recursion(25) def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 7347fca71be2fe..1cff6a218f8d75 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -2,6 +2,7 @@ from unittest.mock import patch import builtins import rlcompleter +from test.support import MISSING_C_DOCSTRINGS class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -40,12 +41,12 @@ def test_global_matches(self): # test with a customized namespace self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) # XXX: see issue5256 self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) def test_attr_matches(self): # test with builtins namespace @@ -54,7 +55,7 @@ def test_attr_matches(self): if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) expected = sorted({'None.%s%s' % (x, - '()' if x == '__init_subclass__' + '()' if x in ('__init_subclass__', '__class__') else '' if x == '__doc__' else '(') for x in dir(None)}) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 628c8cae38a751..b64383f6546f31 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,9 +12,10 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose, requires_subprocess, requires_resource +from test.support import (infinite_recursion, no_tracing, verbose, + requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -656,13 +657,14 @@ def test_basic_script(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "", script_name, - script_name, expect_spec=False) + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "", + script_name, + script_name, + expect_spec=False) def test_basic_script_no_suffix(self): with temp_dir() as script_dir: @@ -741,7 +743,8 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) def test_encoding(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index eda4e6a46df437..0d0f86c145b499 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -16,15 +16,17 @@ from xml.sax.handler import (feature_namespaces, feature_external_ges, LexicalHandler) from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax from io import BytesIO, StringIO import codecs import os.path +import pyexpat import shutil import sys from urllib.error import URLError import urllib.request from test.support import os_helper -from test.support import findfile +from test.support import findfile, check__all__ from test.support.os_helper import FakePath, TESTFN @@ -1214,6 +1216,56 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + self.assertEqual(result.getvalue(), start) # i.e. no elements started + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + # ===== Locator support def test_expat_locator_noinfo(self): @@ -1506,5 +1558,20 @@ def characters(self, content): self.assertEqual(self.char_index, 2) +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 677349c2bfca93..643775597c56c6 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -526,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d231e66b7b889f..02ef172613bf94 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -208,8 +208,7 @@ def test_rmtree_fails_on_symlink_onerror(self): errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(link, onerror=onerror) + shutil.rmtree(link, onerror=onerror) self.assertEqual(len(errors), 1) self.assertIs(errors[0][0], os.path.islink) self.assertEqual(errors[0][1], link) @@ -270,8 +269,7 @@ def test_rmtree_fails_on_junctions_onerror(self): errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(link, onerror=onerror) + shutil.rmtree(link, onerror=onerror) self.assertEqual(len(errors), 1) self.assertIs(errors[0][0], os.path.islink) self.assertEqual(errors[0][1], link) @@ -319,7 +317,7 @@ def test_rmtree_works_on_junctions(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - def test_rmtree_errors_onerror(self): + def test_rmtree_errors(self): # filename is guaranteed not to exist filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) @@ -328,8 +326,8 @@ def test_rmtree_errors_onerror(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + write_file(filename, "") with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -337,11 +335,23 @@ def test_rmtree_errors_onerror(self): # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) self.assertTrue(os.path.exists(filename)) + + self.assertRaises(TypeError, shutil.rmtree, None) + self.assertRaises(TypeError, shutil.rmtree, None, ignore_errors=True) + exc = TypeError if shutil.rmtree.avoids_symlink_attacks else NotImplementedError + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid') + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid', ignore_errors=True) + + def test_rmtree_errors_onerror(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + write_file(filename, "") errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(filename, onerror=onerror) + shutil.rmtree(filename, onerror=onerror) self.assertEqual(len(errors), 2) self.assertIs(errors[0][0], os.scandir) self.assertEqual(errors[0][1], filename) @@ -353,23 +363,9 @@ def onerror(*args): self.assertEqual(errors[1][2][1].filename, filename) def test_rmtree_errors_onexc(self): - # filename is guaranteed not to exist - filename = tempfile.mktemp(dir=self.mkdtemp()) - self.assertRaises(FileNotFoundError, shutil.rmtree, filename) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - - # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") - with self.assertRaises(NotADirectoryError) as cm: - shutil.rmtree(filename) - self.assertEqual(cm.exception.filename, filename) - self.assertTrue(os.path.exists(filename)) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - self.assertTrue(os.path.exists(filename)) + write_file(filename, "") errors = [] def onexc(*args): errors.append(args) @@ -410,8 +406,7 @@ def test_on_error(self): self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) + shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) # Test whether onerror has actually been called. self.assertEqual(self.errorState, 3, "Expected call to onerror function did not happen.") @@ -537,8 +532,7 @@ def onexc(*args): self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) + shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) self.assertTrue(onexc_called) self.assertFalse(onerror_called) @@ -564,25 +558,58 @@ def test_rmtree_uses_safe_fd_version_if_available(self): os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) self.assertTrue(shutil.rmtree.avoids_symlink_attacks) tmp_dir = self.mkdtemp() d = os.path.join(tmp_dir, 'a') os.mkdir(d) try: - real_rmtree = shutil._rmtree_safe_fd + real_open = os.open class Called(Exception): pass def _raiser(*args, **kwargs): raise Called - shutil._rmtree_safe_fd = _raiser + os.open = _raiser self.assertRaises(Called, shutil.rmtree, d) finally: - shutil._rmtree_safe_fd = real_rmtree + os.open = real_open else: - self.assertFalse(shutil._use_fd_functions) self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "requires safe rmtree") + def test_rmtree_fails_on_close(self): + # Test that the error handler is called for failed os.close() and that + # os.close() is only called once for a file descriptor. + tmp = self.mkdtemp() + dir1 = os.path.join(tmp, 'dir1') + os.mkdir(dir1) + dir2 = os.path.join(dir1, 'dir2') + os.mkdir(dir2) + def close(fd): + orig_close(fd) + nonlocal close_count + close_count += 1 + raise OSError + + close_count = 0 + with support.swap_attr(os, 'close', close) as orig_close: + with self.assertRaises(OSError): + shutil.rmtree(dir1) + self.assertTrue(os.path.isdir(dir2)) + self.assertEqual(close_count, 2) + + close_count = 0 + errors = [] + def onexc(*args): + errors.append(args) + with support.swap_attr(os, 'close', close) as orig_close: + shutil.rmtree(dir1, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], close) + self.assertEqual(errors[0][1], dir2) + self.assertIs(errors[1][0], close) + self.assertEqual(errors[1][1], dir1) + self.assertEqual(close_count, 2) + + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "dir_fd is not supported") def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() victim = 'killme' @@ -596,7 +623,7 @@ def test_rmtree_with_dir_fd(self): shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) - @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported") + @unittest.skipIf(shutil.rmtree.avoids_symlink_attacks, "dir_fd is supported") def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() with self.assertRaises(NotImplementedError): @@ -638,6 +665,90 @@ def test_rmtree_on_junction(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def _onexc(fn, path, exc): + assert fn in (os.rmdir, os.unlink) + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + os.chmod(p, mode) + # Remove other dirs except one. + keep = next(p for p in dirs if p != path) + for p in dirs: + if p != keep: + os.rmdir(p) + # Remove other files except one. + keep = next(p for p in files if p != path) + for p in files: + if p != keep: + os.unlink(p) + + os.mkdir(TESTFN) + paths = [TESTFN] + [os.path.join(TESTFN, f'child{i}') + for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + os.mkdir(path) + for path in files: + write_file(path, '') + + old_modes = [os.stat(path).st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD|stat.S_IEXEC + for path in reversed(paths): + os.chmod(path, new_mode) + + try: + shutil.rmtree(TESTFN, onexc=_onexc) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + os.chmod(path, mode) + except OSError: + pass + shutil.rmtree(TESTFN) + raise + + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): @@ -811,7 +922,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -1015,19 +1126,18 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src, dst) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - if os.name != 'nt': - # follow src link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow dst link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow both links - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow src link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow dst link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow both links + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') @os_helper.skip_unless_symlink @@ -1046,10 +1156,11 @@ def test_copymode_symlink_to_symlink(self): os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) # link to link os.lchmod(dst_link, stat.S_IRWXO) + old_mode = os.stat(dst).st_mode shutil.copymode(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.lstat(src_link).st_mode, os.lstat(dst_link).st_mode) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + self.assertEqual(os.stat(dst).st_mode, old_mode) # src link - use chmod os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src_link, dst, follow_symlinks=False) @@ -1512,42 +1623,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1568,6 +1643,92 @@ def _create_files(self, base_dir='dist'): write_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1584,6 +1745,17 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(root_dir, 'archive2.tar') tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir] + if sys.platform == 'darwin': + # macOS tar can include extended attributes, + # ACLs and other mac specific metadata into the + # archive (an recentish version of the OS). + # + # This feature can be disabled with the + # '--no-mac-metadata' option on macOS 11 or + # later. + import platform + if int(platform.mac_ver()[0].split('.')[0]) >= 11: + tar_cmd.insert(1, '--no-mac-metadata') subprocess.check_call(tar_cmd, cwd=root_dir, stdout=subprocess.DEVNULL) @@ -1606,40 +1778,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1809,17 +2030,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1840,10 +2063,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1852,6 +2076,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1862,10 +2087,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1874,6 +2100,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1888,7 +2115,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -1993,7 +2220,9 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) + linkname = os.path.join(dirname, "chown_link") write_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -2014,7 +2243,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -2050,7 +2279,38 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, user, group) check_chown(dirname, uid, gid) + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) + + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) + + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) + +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): @@ -2420,12 +2680,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename @@ -2591,6 +2851,35 @@ def test_move_dir_caseinsensitive(self): finally: os.rmdir(dst_dir) + # bpo-26791: Check that a symlink to a directory can + # be moved into that directory. + @mock_rename + def _test_move_symlink_to_dir_into_dir(self, dst): + src = os.path.join(self.src_dir, 'linktodir') + dst_link = os.path.join(self.dst_dir, 'linktodir') + os.symlink(self.dst_dir, src, target_is_directory=True) + shutil.move(src, dst) + self.assertTrue(os.path.islink(dst_link)) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertFalse(os.path.exists(src)) + + # Repeat the move operation with the destination + # symlink already in place (should raise shutil.Error). + os.symlink(self.dst_dir, src, target_is_directory=True) + with self.assertRaises(shutil.Error): + shutil.move(src, dst) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertTrue(os.path.exists(src)) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_dir(self): + self._test_move_symlink_to_dir_into_dir(self.dst_dir) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_symlink_to_dir(self): + dst = os.path.join(self.src_dir, 'otherlinktodir') + os.symlink(self.dst_dir, dst, target_is_directory=True) + self._test_move_symlink_to_dir_into_dir(dst) @os_helper.skip_unless_dac_override @unittest.skipUnless(hasattr(os, 'lchflags') @@ -3055,6 +3344,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index f2ae28c38dd72d..7f8fe34bb315b2 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,5 +1,6 @@ import enum import errno +import functools import inspect import os import random @@ -12,9 +13,10 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python -from test.support import threading_helper try: import _testcapi except ImportError: @@ -76,6 +78,9 @@ class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass + def create_handler_with_partial(self, argument): + return functools.partial(self.trivial_signal_handler, argument) + def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.getsignal, 4242) @@ -96,6 +101,28 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + def test_no_repr_is_called_on_signal_handler(self): + # See https://github.com/python/cpython/issues/112559. + + class MyArgument: + def __init__(self): + self.repr_count = 0 + + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + argument = MyArgument() + self.assertEqual(0, argument.repr_count) + + handler = self.create_handler_with_partial(argument) + hup = signal.signal(signal.SIGHUP, handler) + self.assertIsInstance(hup, signal.Handlers) + self.assertEqual(signal.getsignal(signal.SIGHUP), handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + self.assertEqual(0, argument.repr_count) + def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) @@ -671,7 +698,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -719,7 +746,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -750,7 +777,7 @@ def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) @@ -806,7 +833,7 @@ def test_itimer_real(self): self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL @@ -1318,6 +1345,8 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index e8ec3b35881fec..035913cdd05f34 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,6 +7,7 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -18,6 +19,7 @@ import os import re import shutil +import stat import subprocess import sys import sysconfig @@ -194,6 +196,45 @@ def test_addsitedir(self): finally: pth_file.cleanup() + def test_addsitedir_dotfile(self): + pth_file = PthFile('.dotfile') + pth_file.cleanup(prep=True) + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') + def test_addsitedir_hidden_flags(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + st = os.stat(pth_file.file_path) + os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') + @support.requires_subprocess() + def test_addsitedir_hidden_file_attribute(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + subprocess.check_call(['attrib', '+H', pth_file.file_path]) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase def test__getuserbase(self): @@ -287,13 +328,13 @@ def test_getsitepackages(self): if sys.platlibdir != "lib": self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -338,6 +379,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -459,12 +513,50 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: self.fail("sitecustomize not imported automatically") + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -603,10 +695,24 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): sys_path.append(abs_path) return sys_path + def _get_pth_lines(self, libpath: str, *, import_site: bool): + pth_lines = ['fake-path-name'] + # include 200 lines of `libpath` in _pth lines (or fewer + # if the `libpath` is long enough to get close to 32KB + # see https://github.com/python/cpython/issues/113628) + encoded_libpath_length = len(libpath.encode("utf-8")) + repetitions = min(200, 30000 // encoded_libpath_length) + if repetitions <= 2: + self.skipTest( + f"Python stdlib path is too long ({encoded_libpath_length:,} bytes)") + pth_lines.extend(libpath for _ in range(repetitions)) + pth_lines.extend(['', '# comment']) + if import_site: + pth_lines.append('import site') + return pth_lines + @support.requires_subprocess() def test_underpth_basic(self): - libpath = test.support.STDLIB_DIR - exe_prefix = os.path.dirname(sys.executable) pth_lines = ['#.', '# ..', *sys.path, '.', '..'] exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( @@ -628,12 +734,7 @@ def test_underpth_basic(self): def test_underpth_nosite_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - pth_lines = [ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - ] + pth_lines = self._get_pth_lines(libpath, import_site=False) exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( os.path.dirname(exe_file), @@ -657,13 +758,8 @@ def test_underpth_nosite_file(self): def test_underpth_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ]) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True)) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' @@ -682,13 +778,8 @@ def test_underpth_file(self): def test_underpth_dll_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ], exe_pth=False) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True), exe_pth=False) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index f2e02dab1c3ca5..4c9fc14bd43f54 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -22,10 +22,9 @@ from test.support import socket_helper from test.support import threading_helper from test.support import asyncore +from test.support import smtpd from unittest.mock import Mock -from . import smtpd - support.requires_working_socket(module=True) diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 2e0dc1aa276f35..d765746987bc4b 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -2,6 +2,7 @@ from test import support from test.support import import_helper from test.support import socket_helper +import os import smtplib import socket @@ -9,6 +10,8 @@ support.requires("network") +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + def check_ssl_verifiy(host, port): context = ssl.create_default_context() with socket.create_connection((host, port)) as sock: @@ -22,7 +25,7 @@ def check_ssl_verifiy(host, port): class SmtpTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 587 def test_connect_starttls(self): @@ -44,7 +47,7 @@ def test_connect_starttls(self): class SmtpSSLTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 465 def test_connect(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 86701caf05399e..ce0f64b43ed49f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,9 +1,8 @@ import unittest from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper - +from test.support import ( + is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) import _thread as thread import array import contextlib @@ -37,6 +36,10 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None support.requires_working_socket(module=True) @@ -46,12 +49,42 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket except ImportError: _socket = None +def skipForRefleakHuntinIf(condition, issueref): + if not condition: + def decorator(f): + f.client_skip = lambda f: f + return f + + else: + def decorator(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + raise unittest.SkipTest(f"ignore while hunting for refleaks, see {issueref}") + + return f(*args, **kwds) + + def client_skip(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + return + + return f(*args, **kwds) + + return wrapper + wrapper.client_skip = client_skip + return wrapper + + return decorator + def get_cid(): if fcntl is None: return None @@ -127,8 +160,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -179,7 +212,10 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -481,10 +517,9 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -497,6 +532,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) @@ -505,6 +541,9 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): @@ -1082,7 +1121,20 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') def testInvalidInterfaceIndexToName(self): self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OverflowError, socket.if_indextoname, -1) + self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + if hasattr(socket, 'if_nameindex'): + indices = dict(socket.if_nameindex()) + for index in indices: + index2 = index + 2**32 + if index2 not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index2) + for index in 2**32-1, 2**64-1: + if index not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index) @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') @@ -1125,6 +1177,7 @@ def testNtoH(self): self.assertRaises(OverflowError, func, 1<<34) @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testNtoHErrors(self): import _testcapi s_good_values = [0, 1, 2, 0xffff] @@ -1153,8 +1206,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -1169,9 +1225,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1182,8 +1237,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1528,9 +1584,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -1588,6 +1643,7 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1781,6 +1837,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -2459,6 +2516,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID @@ -2662,22 +2720,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -3148,7 +3213,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -3665,7 +3730,7 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3676,7 +3741,7 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3689,7 +3754,7 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3703,7 +3768,7 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3727,7 +3792,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. @@ -3801,6 +3866,7 @@ def checkTruncatedHeader(self, result, ignoreflags=0): self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, ignore=ignoreflags) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncNoBufSize(self): # Check that no ancillary data is received when no buffer size # is specified. @@ -3810,26 +3876,32 @@ def testCmsgTruncNoBufSize(self): # received. ignoreflags=socket.MSG_CTRUNC) + @testCmsgTruncNoBufSize.client_skip def _testCmsgTruncNoBufSize(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc0(self): # Check that no ancillary data is received when buffer size is 0. self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), ignoreflags=socket.MSG_CTRUNC) + @testCmsgTrunc0.client_skip def _testCmsgTrunc0(self): self.createAndSendFDs(1) # Check that no ancillary data is returned for various non-zero # (but still too small) buffer sizes. + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + @testCmsgTrunc1.client_skip def _testCmsgTrunc1(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -3837,13 +3909,16 @@ def testCmsgTrunc2Int(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), SIZEOF_INT * 2)) + @testCmsgTrunc2Int.client_skip def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) + @testCmsgTruncLen0Minus1.client_skip def _testCmsgTruncLen0Minus1(self): self.createAndSendFDs(1) @@ -3874,29 +3949,38 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) self.checkFDs(fds) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + @testCmsgTruncLen0.client_skip def _testCmsgTruncLen0(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + @testCmsgTruncLen0Plus1.client_skip def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) + @testCmsgTruncLen1.client_skip def _testCmsgTruncLen1(self): self.createAndSendFDs(2) + + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) + @testCmsgTruncLen2Minus1.client_skip def _testCmsgTruncLen2Minus1(self): self.createAndSendFDs(2) @@ -4815,6 +4899,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi @@ -5567,7 +5652,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5692,7 +5777,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index 3b6ad4d17b0416..2a7cfb7affaa21 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index f3efe0f52f4fd7..488b401fb0054d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -28,14 +28,14 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_emscripten, is_wasi + is_apple, is_emscripten, is_wasi ) from test.support import gc_collect -from test.support import threading_helper -from _testcapi import INT_MAX, ULLONG_MAX +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath @@ -48,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % @@ -590,6 +579,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): @@ -667,7 +661,7 @@ def test_open_with_path_like_object(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): @@ -713,7 +707,7 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipIf(is_apple, "skipped on Apple platforms") @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): @@ -884,9 +878,21 @@ def test_execute_named_param_and_sequence(self): msg = "Binding.*is a named parameter" for query, params in dataset: with self.subTest(query=query, params=params): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: self.cu.execute(query, params) - self.assertEqual(cm.filename, __file__) + + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER @@ -1202,7 +1208,6 @@ def test_blob_seek_and_tell(self): def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1214,12 +1219,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1379,14 +1387,17 @@ def test_blob_get_item_error(self): with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" @@ -1423,7 +1434,7 @@ def test_blob_sequence_not_supported(self): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 14a18c1ad37102..d508f238f84fb5 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -20,7 +20,8 @@ def test_table_dump(self): , "CREATE TABLE t1(id integer primary key, s1 text, " \ "t1_i1 integer not null, i2 integer, unique (s1), " \ - "constraint t1_idx1 unique (i2));" + "constraint t1_idx1 unique (i2), " \ + "constraint t1_i1_idx1 unique (t1_i1));" , "INSERT INTO \"t1\" VALUES(1,'foo',10,20);" , @@ -30,6 +31,9 @@ def test_table_dump(self): "t2_i2 integer, primary key (id)," \ "foreign key(t2_i1) references t1(t1_i1));" , + # Foreign key violation. + "INSERT INTO \"t2\" VALUES(1,2,3);" + , "CREATE TRIGGER trigger_1 update of t1_i1 on t1 " \ "begin " \ "update t2 set t2_i1 = new.t1_i1 where t2_i1 = old.t1_i1; " \ @@ -41,11 +45,85 @@ def test_table_dump(self): [self.cu.execute(s) for s in expected_sqls] i = self.cx.iterdump() actual_sqls = [s for s in i] - expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ - ['COMMIT;'] + expected_sqls = [ + "PRAGMA foreign_keys=OFF;", + "BEGIN TRANSACTION;", + *expected_sqls, + "COMMIT;", + ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', @@ -112,6 +190,21 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + def test_dump_virtual_tables(self): # gh-64662 expected = [ diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 06304dcb4ec7b8..6ec010d13f9e7e 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,6 +10,7 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import array import re import socket import select @@ -37,7 +38,7 @@ ssl = import_helper.import_module("ssl") import _ssl -from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType +from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' @@ -86,9 +87,9 @@ def data_file(*name): (('localityName', 'Castle Anthrax'),), (('organizationName', 'Python Software Foundation'),), (('commonName', 'localhost'),)), - 'notAfter': 'Aug 26 14:23:15 2028 GMT', - 'notBefore': 'Aug 29 14:23:15 2018 GMT', - 'serialNumber': '98A7CF88C74A32ED', + 'notAfter': 'Jan 24 04:21:36 2043 GMT', + 'notBefore': 'Nov 25 04:21:36 2023 GMT', + 'serialNumber': '53E14833F7546C29256DD0F034F776C5E983004C', 'subject': ((('countryName', 'XY'),), (('localityName', 'Castle Anthrax'),), (('organizationName', 'Python Software Foundation'),), @@ -127,6 +128,13 @@ def data_file(*name): SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' +# A custom testcase, extracted from `rfc5280::aki::leaf-missing-aki` in x509-limbo: +# The leaf (server) certificate has no AKI, which is forbidden under RFC 5280. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +LEAF_MISSING_AKI_CERTFILE = data_file("leaf-missing-aki.keycert.pem") +LEAF_MISSING_AKI_CERTFILE_HOSTNAME = "example.com" +LEAF_MISSING_AKI_CA = data_file("leaf-missing-aki.ca.pem") + # Same certificate as pycacert.pem, but without extra text in file SIGNING_CA = data_file("capath", "ceff1710.0") # cert with all kinds of subject alt names @@ -543,7 +551,7 @@ def test_openssl_version(self): else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" self.assertTrue( - s.startswith((openssl_ver, libressl_ver)), + s.startswith((openssl_ver, libressl_ver, "AWS-LC")), (s, t, hex(n)) ) @@ -1161,24 +1169,30 @@ def test_load_cert_chain(self): with self.assertRaises(OSError) as cm: ctx.load_cert_chain(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(BADCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(EMPTYCERT) # Separate key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_cert_chain(ONLYCERT, ONLYKEY) ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) # Mismatching key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) @@ -1246,7 +1260,7 @@ def test_load_verify_locations(self): with self.assertRaises(OSError) as cm: ctx.load_verify_locations(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_verify_locations(BADCERT) ctx.load_verify_locations(CERTFILE, CAPATH) ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) @@ -1496,6 +1510,10 @@ def test_create_default_context(self): self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_PARTIAL_CHAIN, + ssl.VERIFY_X509_PARTIAL_CHAIN) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_STRICT, + ssl.VERIFY_X509_STRICT) self.assertTrue(ctx.check_hostname) self._assert_context_options(ctx) @@ -1650,9 +1668,10 @@ def test_lib_reason(self): with self.assertRaises(ssl.SSLError) as cm: ctx.load_dh_params(CERTFILE) self.assertEqual(cm.exception.library, 'PEM') - self.assertEqual(cm.exception.reason, 'NO_START_LINE') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + self.assertTrue("NO_START_LINE" in s, s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1739,6 +1758,10 @@ def test_buffer_types(self): self.assertEqual(bio.read(), b'bar') bio.write(memoryview(b'baz')) self.assertEqual(bio.read(), b'baz') + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + bio.write(memoryview(noncontig_writable)) def test_error_types(self): bio = ssl.MemoryBIO() @@ -1828,7 +1851,13 @@ def test_connect_fail(self): s = test_wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_ex(self): @@ -1896,7 +1925,13 @@ def test_connect_with_context_fail(self): server_hostname=SIGNED_CERTFILE_HOSTNAME ) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_capath(self): @@ -2113,14 +2148,16 @@ def test_bio_handshake(self): self.assertIsNone(sslobj.version()) self.assertIsNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertIsNone(sslobj.get_channel_binding('tls-unique')) self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) self.assertTrue(sslobj.cipher()) self.assertIsNone(sslobj.shared_ciphers()) self.assertIsNotNone(sslobj.version()) self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertTrue(sslobj.get_channel_binding('tls-unique')) try: self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) @@ -2201,14 +2238,15 @@ def _test_get_server_certificate(test, host, port, cert=None): sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) def _test_get_server_certificate_fail(test, host, port): - try: - pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) from test.ssl_servers import make_https_server @@ -2391,16 +2429,18 @@ def run(self): self.write(msg.lower()) except OSError as e: # handles SSLError and socket errors + if isinstance(e, ConnectionError): + # OpenSSL 1.1.1 sometimes raises + # ConnectionResetError when connection is not + # shut down gracefully. + if self.server.chatty and support.verbose: + print(f" Connection reset by peer: {self.addr}") + + self.close() + self.running = False + return if self.server.chatty and support.verbose: - if isinstance(e, ConnectionError): - # OpenSSL 1.1.1 sometimes raises - # ConnectionResetError when connection is not - # shut down gracefully. - print( - f" Connection reset by peer: {self.addr}" - ) - else: - handle_error("Test server failure:\n") + handle_error("Test server failure:\n") try: self.write(b"ERROR\n") except OSError: @@ -2844,11 +2884,16 @@ def test_crl_check(self): client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname=hostname) as s: - with self.assertRaisesRegex(ssl.SSLError, - "certificate verify failed"): + with self.assertRaisesRegex(ssl.SSLError, regex): s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. @@ -2879,12 +2924,16 @@ def test_check_hostname(self): # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: - with self.assertRaisesRegex( - ssl.CertificateError, - "Hostname mismatch, certificate is not valid for 'invalid'."): + with self.assertRaisesRegex(ssl.CertificateError, regex): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too @@ -2940,6 +2989,38 @@ def test_ecc_cert(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + @unittest.skipUnless(IS_OPENSSL_3_0_0, + "test requires RFC 5280 check added in OpenSSL 3.0+") + def test_verify_strict(self): + # verification fails by default, since the server cert is non-conforming + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + hostname = LEAF_MISSING_AKI_CERTFILE_HOSTNAME + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # explicitly disabling VERIFY_X509_STRICT allows it to succeed + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + client_context.verify_flags &= ~ssl.VERIFY_X509_STRICT + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + def test_dual_rsa_ecc(self): client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.load_verify_locations(SIGNING_CA) @@ -3021,6 +3102,16 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeError): + context.wrap_socket(socket.socket(), + server_hostname='.pythontest.net') + with ThreadedEchoServer(context=server_context, chatty=True) as server: + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(UnicodeDecodeError): + context.wrap_socket(socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net') def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -3077,8 +3168,10 @@ def test_wrong_cert_tls13(self): suppress_ragged_eofs=False) as s: s.connect((HOST, server.port)) with self.assertRaisesRegex( - ssl.SSLError, - 'alert unknown ca|EOF occurred' + OSError, + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') @@ -3142,13 +3235,21 @@ def test_ssl_cert_verify_error(self): server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: try: s.connect((HOST, server.port)) + self.fail("Expected connection failure") except ssl.SSLError as e: msg = 'unable to get local issuer certificate' self.assertIsInstance(e, ssl.SSLCertVerificationError) self.assertEqual(e.verify_code, 20) self.assertEqual(e.verify_message, msg) - self.assertIn(msg, repr(e)) - self.assertIn('certificate verify failed', repr(e)) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) def test_PROTOCOL_TLS(self): """Connecting to an SSLv23 server with various client options""" @@ -3513,6 +3614,27 @@ def test_recv_zero(self): self.assertEqual(s.recv(0), b"") self.assertEqual(s.recv_into(bytearray()), 0) + def test_recv_into_buffer_protocol_len(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + s.send(b"data") + buf = array.array('I', [0, 0]) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf)[:4], b"data") + + class B(bytearray): + def __len__(self): + 1/0 + s.send(b"data") + buf = B(6) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf), b"data\0\0") + def test_nonblocking_send(self): server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, @@ -3659,7 +3781,7 @@ def test_no_shared_ciphers(self): server_hostname=hostname) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", server.conn_errors[0]) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) def test_version_basic(self): """ @@ -3747,7 +3869,7 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) - self.assertIn("alert", str(e.exception)) + self.assertRegex(str(e.exception), "(alert|ALERT)") @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): @@ -3789,6 +3911,10 @@ def test_tls_unique_channel_binding(self): client_context, server_context, hostname = testing_context() + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server = ThreadedEchoServer(context=server_context, chatty=True, connectionchatty=False) @@ -3889,7 +4015,7 @@ def test_dh_params(self): cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: - self.fail("Non-DH cipher: " + cipher[0]) + self.fail("Non-DH key exchange: " + cipher[0]) def test_ecdh_curve(self): # server secp384r1, client auto @@ -4056,8 +4182,9 @@ def cb_raising(ssl_sock, server_name, initial_context): chatty=False, sni_name='supermessage') - self.assertEqual(cm.exception.reason, - 'SSLV3_ALERT_HANDSHAKE_FAILURE') + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(cm.exception.reason, regex) self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): @@ -4232,6 +4359,107 @@ def test_session_handling(self): self.assertEqual(str(e.exception), 'Session refers to a different SSLContext.') + @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk(self): + psk = bytes.fromhex('deadbeef') + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(lambda hint: (None, psk)) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(lambda identity: psk) + + # correct PSK should connect + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # incorrect PSK should fail + incorrect_psk = bytes.fromhex('cafebabe') + client_context.set_psk_client_callback(lambda hint: (None, incorrect_psk)) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # identity_hint and client_identity should be sent to the other side + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + self.assertEqual(hint, identity_hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # adding client callback to server or vice versa raises an exception + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK server callback'): + client_context.set_psk_server_callback(server_callback, identity_hint) + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK client callback'): + server_context.set_psk_client_callback(client_callback) + + # test with UTF-8 identities + identity_hint = '身份暗示' # Translation: "Identity hint" + client_identity = '客户身份' # Translation: "Customer identity" + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk_tls1_3(self): + psk = bytes.fromhex('deadbeef') + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + # identity_hint is not sent to the client in TLS 1.3 + self.assertIsNone(hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(client_callback) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(server_callback, identity_hint) + + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + @unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") class TestPostHandshakeAuth(unittest.TestCase): @@ -4308,8 +4536,12 @@ def msg_cb(conn, direction, version, content_type, msg_type, data): # test sometimes fails with EOF error. Test passes as long as # server aborts connection with an error. with self.assertRaisesRegex( - ssl.SSLError, - '(certificate required|EOF occurred)' + OSError, + ('certificate required' + '|EOF occurred' + '|closed by the remote host' + '|Connection reset by peer' + '|Broken pipe') ): # receive CertificateRequest data = s.recv(1024) @@ -4733,7 +4965,7 @@ def run(self): pass # closed, protocol error, etc. def non_linux_skip_if_other_okay_error(self, err): - if sys.platform == "linux": + if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or @@ -4856,7 +5088,8 @@ def call_after_accept(conn_to_client): self.assertIsNone(wrap_error.library, msg="attr must exist") finally: # gh-108342: Explicitly break the reference cycle - wrap_error = None + with warnings_helper.check_no_resource_warning(self): + wrap_error = None server = None def test_https_client_non_tls_response_ignored(self): @@ -4905,7 +5138,8 @@ def call_after_accept(conn_to_client): # socket; that fails if the connection is broken. It may seem pointless # to test this. It serves as an illustration of something that we never # want to happen... properly not happening. - with self.assertRaises(OSError): + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): connection.request("HEAD", "/test", headers={"Host": "localhost"}) response = connection.getresponse() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e06f9cabf4366b..d1d8a967dbe62f 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -6,9 +6,19 @@ import sys import unittest from test.support.import_helper import import_module -from _testcapi import get_feature_macros +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -217,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", @@ -364,6 +377,7 @@ def test_windows_feature_macros(self): "PyList_Append", "PyList_AsTuple", "PyList_GetItem", + "PyList_GetItemRef", "PyList_GetSlice", "PyList_Insert", "PyList_New", @@ -418,6 +432,10 @@ def test_windows_feature_macros(self): "PyMem_Calloc", "PyMem_Free", "PyMem_Malloc", + "PyMem_RawCalloc", + "PyMem_RawFree", + "PyMem_RawMalloc", + "PyMem_RawRealloc", "PyMem_Realloc", "PyMemberDescr_Type", "PyMember_GetOne", @@ -437,7 +455,9 @@ def test_windows_feature_macros(self): "PyModule_AddObjectRef", "PyModule_AddStringConstant", "PyModule_AddType", + "PyModule_Create2", "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -622,6 +642,8 @@ def test_windows_feature_macros(self): "PySys_AddWarnOption", "PySys_AddWarnOptionUnicode", "PySys_AddXOption", + "PySys_Audit", + "PySys_AuditTuple", "PySys_FormatStderr", "PySys_FormatStdout", "PySys_GetObject", @@ -689,7 +711,10 @@ def test_windows_feature_macros(self): "PyType_GenericAlloc", "PyType_GenericNew", "PyType_GetFlags", + "PyType_GetFullyQualifiedName", "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", "PyType_GetQualName", @@ -837,6 +862,8 @@ def test_windows_feature_macros(self): "Py_GetArgcArgv", "Py_GetBuildInfo", "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", "Py_GetCopyright", "Py_GetExecPrefix", "Py_GetPath", @@ -868,6 +895,7 @@ def test_windows_feature_macros(self): "Py_SetProgramName", "Py_SetPythonHome", "Py_SetRecursionLimit", + "Py_TYPE", "Py_UTF8Mode", "Py_VaBuildValue", "Py_Version", @@ -900,10 +928,18 @@ def test_windows_feature_macros(self): "_Py_IncRef", "_Py_NoneStruct", "_Py_NotImplementedStruct", + "_Py_SetRefcnt", "_Py_SwappedOp", "_Py_TrueStruct", "_Py_VaBuildValue_SizeT", ) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) if feature_macros['MS_WINDOWS']: SYMBOL_NAMES += ( 'PyErr_SetExcFromWindowsErr', @@ -919,17 +955,6 @@ def test_windows_feature_macros(self): 'PyUnicode_DecodeMBCSStateful', 'PyUnicode_EncodeCodePage', ) -if feature_macros['HAVE_FORK']: - SYMBOL_NAMES += ( - 'PyOS_AfterFork', - 'PyOS_AfterFork_Child', - 'PyOS_AfterFork_Parent', - 'PyOS_BeforeFork', - ) -if feature_macros['USE_STACKCHECK']: - SYMBOL_NAMES += ( - 'PyOS_CheckStack', - ) if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: SYMBOL_NAMES += ( 'PyThread_get_thread_native_id', @@ -939,14 +964,23 @@ def test_windows_feature_macros(self): '_Py_NegativeRefcount', '_Py_RefTotal', ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', 'MS_WINDOWS', 'PY_HAVE_THREAD_NATIVE_ID', 'Py_REF_DEBUG', + 'Py_TRACE_REFS', 'USE_STACKCHECK']) WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, 'MS_WINDOWS': True, 'PY_HAVE_THREAD_NATIVE_ID': True, 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 0eced2fcf98376..49013a4bcd8af6 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -15,8 +14,10 @@ class TestFilemode: statmod = None file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', - 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', - 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + 'SF_SNAPSHOT', 'SF_SETTABLE', 'SF_RESTRICTED', 'SF_FIRMLINK', + 'SF_DATALESS', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE', + 'UF_SETTABLE', 'UF_TRACKED', 'UF_DATAVAULT'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', 'S_IFREG', 'S_IFSOCK', 'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} @@ -148,12 +149,19 @@ def test_mode(self): self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: + os.chmod(TESTFN, 0o500) + st_mode, modestr = self.get_mode() + self.assertEqual(modestr[:3], '-r-') + self.assertS_IS("REG", st_mode) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() self.assertEqual(modestr[:3], '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) @os_helper.skip_unless_working_chmod def test_directory(self): @@ -232,6 +240,18 @@ def test_module_attributes(self): self.assertTrue(callable(func)) self.assertEqual(func(0), 0) + def test_flags_consistent(self): + self.assertFalse(self.statmod.UF_SETTABLE & self.statmod.SF_SETTABLE) + + for flag in self.file_flags: + if flag.startswith("UF"): + self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': + self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") + self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") + else: + self.assertTrue(getattr(self.statmod, flag) & self.statmod.SF_SETTABLE, f"{flag} notin SF_SETTABLE") + @unittest.skipUnless(sys.platform == "win32", "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): @@ -240,6 +260,66 @@ def test_file_attribute_constants(self): modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) + @unittest.skipUnless(sys.platform == "darwin", "macOS system check") + def test_macosx_attribute_values(self): + self.assertEqual(self.statmod.UF_SETTABLE, 0x0000ffff) + self.assertEqual(self.statmod.UF_NODUMP, 0x00000001) + self.assertEqual(self.statmod.UF_IMMUTABLE, 0x00000002) + self.assertEqual(self.statmod.UF_APPEND, 0x00000004) + self.assertEqual(self.statmod.UF_OPAQUE, 0x00000008) + self.assertEqual(self.statmod.UF_COMPRESSED, 0x00000020) + self.assertEqual(self.statmod.UF_TRACKED, 0x00000040) + self.assertEqual(self.statmod.UF_DATAVAULT, 0x00000080) + self.assertEqual(self.statmod.UF_HIDDEN, 0x00008000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.SF_SUPPORTED, 0x009f0000) + self.assertEqual(self.statmod.SF_SETTABLE, 0x3fff0000) + self.assertEqual(self.statmod.SF_SYNTHETIC, 0xc0000000) + else: + self.assertEqual(self.statmod.SF_SETTABLE, 0xffff0000) + self.assertEqual(self.statmod.SF_ARCHIVED, 0x00010000) + self.assertEqual(self.statmod.SF_IMMUTABLE, 0x00020000) + self.assertEqual(self.statmod.SF_APPEND, 0x00040000) + self.assertEqual(self.statmod.SF_RESTRICTED, 0x00080000) + self.assertEqual(self.statmod.SF_NOUNLINK, 0x00100000) + self.assertEqual(self.statmod.SF_FIRMLINK, 0x00800000) + self.assertEqual(self.statmod.SF_DATALESS, 0x40000000) + + self.assertFalse(isinstance(self.statmod.S_IFMT, int)) + self.assertEqual(self.statmod.S_IFIFO, 0o010000) + self.assertEqual(self.statmod.S_IFCHR, 0o020000) + self.assertEqual(self.statmod.S_IFDIR, 0o040000) + self.assertEqual(self.statmod.S_IFBLK, 0o060000) + self.assertEqual(self.statmod.S_IFREG, 0o100000) + self.assertEqual(self.statmod.S_IFLNK, 0o120000) + self.assertEqual(self.statmod.S_IFSOCK, 0o140000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.S_IFWHT, 0o160000) + + self.assertEqual(self.statmod.S_IRWXU, 0o000700) + self.assertEqual(self.statmod.S_IRUSR, 0o000400) + self.assertEqual(self.statmod.S_IWUSR, 0o000200) + self.assertEqual(self.statmod.S_IXUSR, 0o000100) + self.assertEqual(self.statmod.S_IRWXG, 0o000070) + self.assertEqual(self.statmod.S_IRGRP, 0o000040) + self.assertEqual(self.statmod.S_IWGRP, 0o000020) + self.assertEqual(self.statmod.S_IXGRP, 0o000010) + self.assertEqual(self.statmod.S_IRWXO, 0o000007) + self.assertEqual(self.statmod.S_IROTH, 0o000004) + self.assertEqual(self.statmod.S_IWOTH, 0o000002) + self.assertEqual(self.statmod.S_IXOTH, 0o000001) + self.assertEqual(self.statmod.S_ISUID, 0o004000) + self.assertEqual(self.statmod.S_ISGID, 0o002000) + self.assertEqual(self.statmod.S_ISVTX, 0o001000) + + self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) + self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) + self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) + + @unittest.skipIf(c_stat is None, 'need _stat extension') class TestFilemodeCStat(TestFilemode, unittest.TestCase): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index b24fc3c3d077fe..c374c947e02a6b 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2302,10 +2302,12 @@ def test_error_cases(self): StatisticsError = statistics.StatisticsError with self.assertRaises(StatisticsError): geometric_mean([]) # empty input - with self.assertRaises(StatisticsError): - geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): geometric_mean([3.5, -4.0, 5.25]) # negative input + with self.assertRaises(StatisticsError): + geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + with self.assertRaises(StatisticsError): + geometric_mean([3.5, -math.inf, 5.25]) # negative infinity with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2328,6 +2330,12 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) + # Cases with zero + self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero + self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed + self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero + self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN + def test_mixed_int_and_float(self): # Regression test for b.p.o. issue #28327 geometric_mean = statistics.geometric_mean @@ -2345,6 +2353,183 @@ def test_mixed_int_and_float(self): self.assertAlmostEqual(actual_mean, expected_mean, places=5) +class TestKDE(unittest.TestCase): + + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=-1.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() + kernel_specs = statistics._kernel_specs + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] + for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] + invcdf = spec['invcdf'] + with self.subTest(kernel=kernel): + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + + class TestQuantiles(unittest.TestCase): def test_specific_cases(self): diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 814ef111c5bec8..7bdd2881904548 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -55,6 +55,21 @@ def duplicate_string(text): class StrSubclass(str): pass +class OtherStrSubclass(str): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + class StrTest(string_tests.StringLikeTest, string_tests.MixinStrUnicodeTest, unittest.TestCase): @@ -83,6 +98,10 @@ def __repr__(self): self.assertEqual(realresult, result) self.assertTrue(object is not realresult) + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_literals(self): self.assertEqual('\xff', '\u00ff') self.assertEqual('\uffff', '\U0000ffff') @@ -127,10 +146,13 @@ def test_ascii(self): self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), ascii("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) + self.assertTypedEqual(ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(ascii(WithRepr('')), '') + self.assertTypedEqual(ascii(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertRaises(TypeError, ascii, WithRepr(b'byte-repr')) def test_repr(self): # Test basic sanity of repr() @@ -168,10 +190,13 @@ def test_repr(self): self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), repr("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) + self.assertTypedEqual(repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(repr(WithRepr('')), '') + self.assertTypedEqual(repr(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertRaises(TypeError, repr, WithRepr(b'byte-repr')) def test_iterators(self): # Make sure unicode objects have an __iter__ method @@ -1553,7 +1578,7 @@ def __int__(self): self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) - self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + self.assertRaisesRegex(TypeError, r'%c requires an int or a unicode character, not .*\.PseudoFloat', operator.mod, '%c', pi) class RaisingNumber: def __int__(self): @@ -2367,28 +2392,37 @@ def test_ucs4(self): def test_conversion(self): # Make sure __str__() works properly - class ObjectToStr: - def __str__(self): - return "foo" - - class StrSubclassToStr(str): - def __str__(self): - return "foo" - - class StrSubclassToStrSubclass(str): - def __new__(cls, content=""): - return str.__new__(cls, 2*content) - def __str__(self): + class StrWithStr(str): + def __new__(cls, value): + self = str.__new__(cls, "") + self.value = value return self + def __str__(self): + return self.value - self.assertEqual(str(ObjectToStr()), "foo") - self.assertEqual(str(StrSubclassToStr("bar")), "foo") - s = str(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclassToStrSubclass) - s = StrSubclass(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclass) + self.assertTypedEqual(str(WithStr('abc')), 'abc') + self.assertTypedEqual(str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(StrWithStr('abc')), 'abc') + self.assertTypedEqual(str(StrWithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(WithRepr('')), '') + self.assertTypedEqual(str(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr('')), StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr(StrSubclass(''))), + StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr(OtherStrSubclass(''))), + StrSubclass('')) def test_unicode_repr(self): class s1: @@ -2617,6 +2651,24 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) + def test_str_invalid_call(self): + check = lambda *a, **kw: self.assertRaises(TypeError, str, *a, **kw) + + # too many args + check(1, "", "", 1) + + # no such kw arg + check(test=1) + + # 'encoding' must be str + check(1, encoding=1) + check(1, 1) + + # 'errors' must be str + check(1, errors=1) + check(1, "", errors=1) + check(1, 1, 1) + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 9b663c00223d1b..371e8193b3544d 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -131,6 +131,18 @@ def test_eval_str_invalid_escape(self): self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) + # Check that the warning is raised ony once if there are syntax errors + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'\\e' $") + exc = cm.exception + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, SyntaxWarning) + self.assertRegex(str(w[0].message), 'invalid escape sequence') + self.assertEqual(w[0].filename, '') + def test_eval_str_invalid_octal_escape(self): for i in range(0o400, 0o1000): with self.assertWarns(SyntaxWarning): diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 810c5a36e02f41..038746e26c24ad 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -7,7 +7,7 @@ import os import sys from test import support -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, warnings_helper from datetime import date as datetime_date import _strptime @@ -120,7 +120,7 @@ def setUp(self): def test_pattern(self): # Test TimeRE.pattern - pattern_string = self.time_re.pattern(r"%a %A %d") + pattern_string = self.time_re.pattern(r"%a %A %d %Y") self.assertTrue(pattern_string.find(self.locale_time.a_weekday[2]) != -1, "did not find abbreviated weekday in pattern string '%s'" % pattern_string) @@ -160,10 +160,11 @@ def test_compile(self): found.group('b'))) for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', 'S','u','U','V','w','W','x','X','y','Y','Z','%'): - compiled = self.time_re.compile("%" + directive) - found = compiled.match(time.strftime("%" + directive)) + fmt = "%d %Y" if directive == 'd' else "%" + directive + compiled = self.time_re.compile(fmt) + found = compiled.match(time.strftime(fmt)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % - (time.strftime("%" + directive), + (time.strftime(fmt), compiled.pattern)) def test_blankpattern(self): @@ -224,35 +225,55 @@ def test_ValueError(self): else: self.fail("'%s' did not raise ValueError" % bad_format) + msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ + r"the ISO year directive '%G' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + msg_week_not_compatible = r"ISO week directive '%V' is incompatible with " \ + r"the year directive '%Y'. Use the ISO year '%G' instead." + msg_julian_not_compatible = r"Day of the year directive '%j' is not " \ + r"compatible with ISO year directive '%G'. Use '%Y' instead." + msg_year_no_week_or_weekday = r"ISO year directive '%G' must be used with " \ + r"the ISO week directive '%V' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + + locale_time = _strptime.LocaleTime() + # Ambiguous or incomplete cases using ISO year/week/weekday directives - # 1. ISO week (%V) is specified, but the year is specified with %Y - # instead of %G - with self.assertRaises(ValueError): - _strptime._strptime("1999 50", "%Y %V") - # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not - with self.assertRaises(ValueError): - _strptime._strptime("1999 51", "%G %V") - # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not - for w in ('A', 'a', 'w', 'u'): - with self.assertRaises(ValueError): - _strptime._strptime("1999 51","%G %{}".format(w)) - # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) - with self.assertRaises(ValueError): - _strptime._strptime("2015", "%G") - # 5. Julian/ordinal day (%j) is specified with %G, but not %Y - with self.assertRaises(ValueError): - _strptime._strptime("1999 256", "%G %j") - # 6. Invalid ISO weeks - invalid_iso_weeks = [ - "2019-00-1", - "2019-54-1", - "2021-53-1", + subtests = [ + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + ("1999 50", "%Y %V", msg_week_no_year_or_weekday), + ("1999 50 5", "%Y %V %u", msg_week_not_compatible), + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + ("1999 51", "%G %V", msg_year_no_week_or_weekday), + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + ("1999 {}".format(locale_time.f_weekday[5]), "%G %A", + msg_year_no_week_or_weekday), + ("1999 {}".format(locale_time.a_weekday[5]), "%G %a", + msg_year_no_week_or_weekday), + ("1999 5", "%G %w", msg_year_no_week_or_weekday), + ("1999 5", "%G %u", msg_year_no_week_or_weekday), + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + ("2015", "%G", msg_year_no_week_or_weekday), + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + ("1999 256", "%G %j", msg_julian_not_compatible), + ("1999 50 5 256", "%G %V %u %j", msg_julian_not_compatible), + # ISO week specified alone + ("50", "%V", msg_week_no_year_or_weekday), + # ISO year is unspecified, falling back to year + ("50 5", "%V %u", msg_week_no_year_or_weekday), + # 6. Invalid ISO weeks + ("2019-00-1", "%G-%V-%u", + "time data '2019-00-1' does not match format '%G-%V-%u'"), + ("2019-54-1", "%G-%V-%u", + "time data '2019-54-1' does not match format '%G-%V-%u'"), + ("2021-53-1", "%G-%V-%u", "Invalid week: 53"), ] - for invalid_iso_dtstr in invalid_iso_weeks: - with self.subTest(invalid_iso_dtstr): - with self.assertRaises(ValueError): - _strptime._strptime(invalid_iso_dtstr, "%G-%V-%u") + for (data_string, format, message) in subtests: + with self.subTest(data_string=data_string, format=format): + with self.assertRaisesRegex(ValueError, message): + _strptime._strptime(data_string, format) def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) @@ -270,8 +291,9 @@ def test_unconverteddata(self): def helper(self, directive, position): """Helper fxn in testing.""" - strf_output = time.strftime("%" + directive, self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%" + directive) + fmt = "%d %Y" if directive == 'd' else "%" + directive + strf_output = time.strftime(fmt, self.time_tuple) + strp_output = _strptime._strptime_time(strf_output, fmt) self.assertTrue(strp_output[position] == self.time_tuple[position], "testing of '%s' directive failed; '%s' -> %s != %s" % (directive, strf_output, strp_output[position], @@ -477,9 +499,11 @@ def test_escaping(self): need_escaping = r".^$*+?{}\[]|)(" self.assertTrue(_strptime._strptime_time(need_escaping, need_escaping)) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_feb29_on_leap_year_without_year(self): time.strptime("Feb 29", "%b %d") + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_mar1_comes_after_feb29_even_when_omitting_the_year(self): self.assertLess( time.strptime("Feb 29", "%b %d"), @@ -659,25 +683,25 @@ class CacheTests(unittest.TestCase): def test_time_re_recreation(self): # Make sure cache is recreated when current locale does not match what # cached object was created with. - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") _strptime._strptime_time("2005", "%Y") _strptime._TimeRE_cache.locale_time.lang = "Ni" original_time_re = _strptime._TimeRE_cache - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(original_time_re, _strptime._TimeRE_cache) self.assertEqual(len(_strptime._regex_cache), 1) def test_regex_cleanup(self): # Make sure cached regexes are discarded when cache becomes "full". try: - del _strptime._regex_cache['%d'] + del _strptime._regex_cache['%d %Y'] except KeyError: pass bogus_key = 0 while len(_strptime._regex_cache) <= _strptime._CACHE_MAX_SIZE: _strptime._regex_cache[bogus_key] = None bogus_key += 1 - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertEqual(len(_strptime._regex_cache), 1) def test_new_localetime(self): @@ -685,7 +709,7 @@ def test_new_localetime(self): # is created. locale_time_id = _strptime._TimeRE_cache.locale_time _strptime._TimeRE_cache.locale_time.lang = "Ni" - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) def test_TimeRE_recreation_locale(self): @@ -696,13 +720,13 @@ def test_TimeRE_recreation_locale(self): except locale.Error: self.skipTest('test needs en_US.UTF8 locale') try: - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get id of current cache object. first_time_re = _strptime._TimeRE_cache try: # Change the locale and force a recreation of the cache. locale.setlocale(locale.LC_TIME, ('de_DE', 'UTF8')) - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get the new cache object's id. second_time_re = _strptime._TimeRE_cache # They should not be equal. diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index c76649cdcd9cce..5508cc3eec85c8 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -9,7 +9,7 @@ import weakref from test import support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization from test.support.script_helper import assert_python_ok ISBIGENDIAN = sys.byteorder == "big" @@ -674,6 +674,7 @@ def __del__(self): self.assertIn(b"Exception ignored in:", stderr) self.assertIn(b"C.__del__", stderr) + @suppress_immortalization() def test__struct_reference_cycle_cleaned_up(self): # Regression test for python/cpython#94207. @@ -700,20 +701,6 @@ def test__struct_types_immutable(self): with self.assertRaises(TypeError): cls.x = 1 - @support.cpython_only - def test__struct_Struct__new__initialized(self): - # See https://github.com/python/cpython/issues/78724 - - s = struct.Struct.__new__(struct.Struct, "b") - s.unpack_from(b"abcd") - - @support.cpython_only - def test__struct_Struct_subclassing(self): - class Bob(struct.Struct): - pass - - s = Bob("b") - s.unpack_from(b"abcd") def test_issue35714(self): # Embedded null characters should not be allowed in format strings. @@ -774,6 +761,15 @@ def test_error_propagation(fmt_str): test_error_propagation('N') test_error_propagation('n') + def test_struct_subclass_instantiation(self): + # Regression test for https://github.com/python/cpython/issues/112358 + class MyStruct(struct.Struct): + def __init__(self): + super().__init__('>h') + + my_struct = MyStruct() + self.assertEqual(my_struct.pack(12345), b'\x30\x39') + def test_repr(self): s = struct.Struct('=i2H') self.assertEqual(repr(s), f'Struct({s.format!r})') diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py index 310473a4a2fe58..0d32aa509bd25c 100644 --- a/Lib/test/test_subclassinit.py +++ b/Lib/test/test_subclassinit.py @@ -230,7 +230,7 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) with self.assertRaises(TypeError): - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass class MyMeta(type): @@ -241,10 +241,10 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) self.otherarg = otherarg - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass3(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass3.otherarg, 1) def test_errors_changed_pep487(self): # These tests failed before Python 3.6, PEP 487 @@ -263,10 +263,10 @@ def __new__(cls, name, bases, namespace, otherarg): self.otherarg = otherarg return self - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass2.otherarg, 1) def test_type(self): t = type('NewClass', (object,), {}) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index a865df1082fba3..9412a2d737bb2e 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -25,7 +25,6 @@ import gc import textwrap import json -import pathlib from test.support.os_helper import FakePath try: @@ -791,6 +790,19 @@ def test_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_duplicate_envs(self): + newenv = os.environ.copy() + newenv["fRUit"] = "cherry" + newenv["fruit"] = "lemon" + newenv["FRUIT"] = "orange" + newenv["frUit"] = "banana" + with subprocess.Popen(["CMD", "/c", "SET", "fruit"], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, _ = p.communicate() + self.assertEqual(stdout.strip(), b"frUit=banana") + # Windows requires at least the SYSTEMROOT environment variable to start # Python @unittest.skipIf(sys.platform == 'win32', @@ -822,6 +834,26 @@ def is_env_var_to_ignore(n): if not is_env_var_to_ignore(k)] self.assertEqual(child_env_names, []) + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'The Python shared library cannot be loaded ' + 'without some system environments.') + @unittest.skipIf(check_sanitizer(address=True), + 'AddressSanitizer adds to the environment.') + def test_one_environment_variable(self): + newenv = {'fruit': 'orange'} + cmd = [sys.executable, '-c', + 'import sys,os;' + 'sys.stdout.write("fruit="+os.getenv("fruit"))'] + if sys.platform == "win32": + cmd = ["CMD", "/c", "SET", "fruit"] + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=newenv) as p: + stdout, stderr = p.communicate() + if p.returncode and support.verbose: + print("STDOUT:", stdout.decode("ascii", "replace")) + print("STDERR:", stderr.decode("ascii", "replace")) + self.assertEqual(p.returncode, 0) + self.assertEqual(stdout.strip(), b"fruit=orange") + def test_invalid_cmd(self): # null character in the command name cmd = sys.executable + '\0' @@ -862,6 +894,19 @@ def test_invalid_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange=lemon") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_invalid_env(self): + # '=' in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + newenv = os.environ.copy() + newenv["==FRUIT"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' @@ -1362,7 +1407,7 @@ def open_fds(): t = threading.Thread(target=open_fds) t.start() try: - with self.assertRaises(EnvironmentError): + with self.assertRaises(OSError): subprocess.Popen(NONEXISTING_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -1476,9 +1521,6 @@ def test_communicate_epipe(self): p.communicate(b"x" * 2**20) def test_repr(self): - path_cmd = pathlib.Path("my-tool.py") - pathlib_cls = path_cmd.__class__.__name__ - cases = [ ("ls", True, 123, ""), ('a' * 100, True, 0, @@ -1486,7 +1528,8 @@ def test_repr(self): (["ls"], False, None, ""), (["ls", '--my-opts', 'a' * 100], False, None, ""), - (path_cmd, False, 7, f"") + (os_helper.FakePath("my-tool.py"), False, 7, + ">") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: @@ -1561,20 +1604,21 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) + @unittest.skipUnless(hasattr(subprocess, '_winapi'), + 'need subprocess._winapi') + def test_wait_negative_timeout(self): + proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc: + patch = mock.patch.object( + subprocess._winapi, + 'WaitForSingleObject', + return_value=subprocess._winapi.WAIT_OBJECT_0) + with patch as mock_wait: + proc.wait(-1) # negative timeout + mock_wait.assert_called_once_with(proc._handle, 0) + proc.returncode = None + + self.assertEqual(proc.wait(), 0) class RunFuncTestCase(BaseTestCase): @@ -1716,6 +1760,13 @@ def test_capture_output(self): self.assertIn(b'BDFL', cp.stdout) self.assertIn(b'FLUFL', cp.stderr) + def test_stdout_stdout(self): + # run() refuses to accept stdout=STDOUT + with self.assertRaises(ValueError, + msg=("STDOUT can only be used for stderr")): + self.run_python("print('will not be run')", + stdout=subprocess.STDOUT) + def test_stdout_with_capture_output_arg(self): # run() refuses to accept 'stdout' with 'capture_output' tf = tempfile.TemporaryFile() @@ -1769,9 +1820,9 @@ def test_encoding_warning(self): cp = subprocess.run([sys.executable, "-Xwarn_default_encoding", "-c", code], capture_output=True) lines = cp.stderr.splitlines() - self.assertEqual(len(lines), 2, lines) + self.assertEqual(len(lines), 4, lines) self.assertTrue(lines[0].startswith(b":2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b":3: EncodingWarning: ")) + self.assertTrue(lines[2].startswith(b":3: EncodingWarning: ")) def _get_test_grp_name(): @@ -1960,9 +2011,9 @@ def test_process_group_0(self): @unittest.skipUnless(hasattr(os, 'setreuid'), 'no setreuid on platform') def test_user(self): - # For code coverage of the user parameter. We don't care if we get an - # EPERM error from it depending on the test execution environment, that - # still indicates that it was called. + # For code coverage of the user parameter. We don't care if we get a + # permission error from it depending on the test execution environment, + # that still indicates that it was called. uid = os.geteuid() test_users = [65534 if uid != 65534 else 65533, uid] @@ -1986,11 +2037,11 @@ def test_user(self): "import os; print(os.getuid())"], user=user, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass - except OSError as e: - if e.errno not in (errno.EACCES, errno.EPERM): - raise + except PermissionError as e: # (EACCES, EPERM) + if e.errno == errno.EACCES: + self.assertEqual(e.filename, sys.executable) + else: + self.assertIsNone(e.filename) else: if isinstance(user, str): user_uid = pwd.getpwnam(user).pw_uid @@ -2034,8 +2085,8 @@ def test_group(self): "import os; print(os.getgid())"], group=group, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass + except PermissionError as e: # (EACCES, EPERM) + self.assertIsNone(e.filename) else: if isinstance(group, str): group_gid = grp.getgrnam(group).gr_gid @@ -2066,8 +2117,14 @@ def test_group_error(self): def test_extra_groups(self): gid = os.getegid() group_list = [65534 if gid != 65534 else 65533] + self._test_extra_groups_impl(gid=gid, group_list=group_list) + + @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') + def test_extra_groups_empty_list(self): + self._test_extra_groups_impl(gid=os.getegid(), group_list=[]) + + def _test_extra_groups_impl(self, *, gid, group_list): name_group = _get_test_grp_name() - perm_error = False if grp is not None: group_list.append(name_group) @@ -2077,11 +2134,9 @@ def test_extra_groups(self): [sys.executable, "-c", "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"], extra_groups=group_list) - except OSError as ex: - if ex.errno != errno.EPERM: - raise - perm_error = True - + except PermissionError as e: + self.assertIsNone(e.filename) + self.skipTest("setgroup() EPERM; this test may require root.") else: parent_groups = os.getgroups() child_groups = json.loads(output) @@ -2092,12 +2147,15 @@ def test_extra_groups(self): else: desired_gids = group_list - if perm_error: - self.assertEqual(set(child_groups), set(parent_groups)) - else: - self.assertEqual(set(desired_gids), set(child_groups)) + self.assertEqual(set(desired_gids), set(child_groups)) - # make sure we bomb on negative values + if grp is None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, + extra_groups=[name_group]) + + # No skip necessary, this test won't make it to a setgroup() call. + def test_extra_groups_invalid_gid_t_values(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) @@ -2106,16 +2164,6 @@ def test_extra_groups(self): cwd=os.curdir, env=os.environ, extra_groups=[2**64]) - if grp is None: - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, - extra_groups=[name_group]) - - @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform') - def test_extra_groups_error(self): - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[]) - @unittest.skipIf(mswindows or not hasattr(os, 'umask'), 'POSIX umask() is not available.') def test_umask(self): @@ -3354,16 +3402,105 @@ def test_preexec_at_exit(self): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @mock.patch("subprocess._fork_exec") + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test__use_vfork(self, mock_fork_exec): + self.assertTrue(subprocess._USE_VFORK) # The default value regardless. + mock_fork_exec.side_effect = RuntimeError("just testing args") + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + mock_fork_exec.assert_called_once() + # NOTE: These assertions are *ugly* as they require the last arg + # to remain the have_vfork boolean. We really need to refactor away + # from the giant "wall of args" internal C extension API. + self.assertTrue(mock_fork_exec.call_args.args[-1]) + with mock.patch.object(subprocess, '_USE_VFORK', False): + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) + + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test_vfork_used_when_expected(self): + # This is a performance regression test to ensure we default to using + # vfork() when possible. + # Technically this test could pass when posix_spawn is used as well + # because libc tends to implement that internally using vfork. But + # that'd just be testing a libc+kernel implementation detail. + strace_binary = "/usr/bin/strace" + # The only system calls we are interested in. + strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" + true_binary = "/bin/true" + strace_command = [strace_binary, strace_filter] + + try: + does_strace_work_process = subprocess.run( + strace_command + [true_binary], + stderr=subprocess.PIPE, + stdout=subprocess.DEVNULL, + ) + rc = does_strace_work_process.returncode + stderr = does_strace_work_process.stderr + except OSError: + rc = -1 + stderr = "" + if rc or (b"+++ exited with 0 +++" not in stderr): + self.skipTest("strace not found or not working as expected.") + + with self.subTest(name="default_is_vfork"): + vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])"""), + __run_using_command=strace_command, + ) + # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) + self.assertRegex(vfork_result.err, br"(?i)vfork") + # Do NOT check that fork() or other clones did not happen. + # If the OS denys the vfork it'll fallback to plain fork(). + + # Test that each individual thing that would disable the use of vfork + # actually disables it. + for sub_name, preamble, sp_kwarg, expect_permission_error in ( + ("!use_vfork", "subprocess._USE_VFORK = False", "", False), + ("preexec", "", "preexec_fn=lambda: None", False), + ("setgid", "", f"group={os.getgid()}", True), + ("setuid", "", f"user={os.getuid()}", True), + ("setgroups", "", "extra_groups=[]", True), + ): + with self.subTest(name=sub_name): + non_vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + {preamble} + try: + subprocess.check_call( + [{true_binary!r}], **dict({sp_kwarg})) + except PermissionError: + if not {expect_permission_error}: + raise"""), + __run_using_command=strace_command, + ) + # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). + self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index f4a8d434ed1b8c..d6d08ee53f821c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -18,10 +18,11 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import html.entities + import html.entities # noqa: F401 try: - import tty # Not available on Windows + # Not available on Windows + import tty # noqa: F401 except ImportError: if support.verbose: print("skipping tty") diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 43162c540b55ae..3ffbe03f0c2f11 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,8 +1,10 @@ """Unit tests for zero-argument super() & related machinery.""" +import textwrap +import threading import unittest from unittest.mock import patch -from test import shadowed_super +from test.support import import_helper, threading_helper ADAPTIVE_WARMUP_DELAY = 2 @@ -342,7 +344,20 @@ def test_super_argtype(self): super(1, int) def test_shadowed_global(self): + source = textwrap.dedent( + """ + class super: + msg = "truly super" + + class C: + def method(self): + return super().msg + """, + ) + with import_helper.ready_to_import(name="shadowed_super", source=source): + import shadowed_super self.assertEqual(shadowed_super.C().method(), "truly super") + import_helper.unload("shadowed_super") def test_shadowed_local(self): class super: @@ -396,6 +411,33 @@ def method(self): with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): C().method() + def test_supercheck_fail(self): + class C: + def method(self, type_, obj): + return super(type_, obj).method() + + c = C() + err_msg = ( + r"super\(type, obj\): obj \({} {}\) is not " + r"an instance or subtype of type \({}\)." + ) + + cases = ( + (int, c, int.__name__, C.__name__, "instance of"), + # obj is instance of type + (C, list(), C.__name__, list.__name__, "instance of"), + # obj is type itself + (C, list, C.__name__, list.__name__, "type"), + ) + + for case in cases: + with self.subTest(case=case): + type_, obj, type_str, obj_str, instance_or_type = case + regex = err_msg.format(instance_or_type, obj_str, type_str) + + with self.assertRaisesRegex(TypeError, regex): + c.method(type_, obj) + def test_super___class__(self): class C: def method(self): @@ -464,6 +506,38 @@ def some(cls): for _ in range(ADAPTIVE_WARMUP_DELAY): C.some(C) + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(5000): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 97de81677b10bc..e60e5477d32e1f 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -3,6 +3,7 @@ import io import os import shutil +import signal import socket import stat import subprocess @@ -70,7 +71,7 @@ def test_get_original_stdout(self): self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): - import sched + import sched # noqa: F401 self.assertIn("sched", sys.modules) import_helper.unload("sched") self.assertNotIn("sched", sys.modules) @@ -547,111 +548,6 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') - def test_match_test(self): - class Test: - def __init__(self, test_id): - self.test_id = test_id - - def id(self): - return self.test_id - - test_access = Test('test.test_os.FileTests.test_access') - test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') - - # Test acceptance - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests([]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests([test_access.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(['test_os'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(['test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(['filetests'], None) - self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) - self.assertTrue(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - support.set_match_tests(['test_access', 'DONTMATCH'], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests(None, [test_access.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(None, ['test_os']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(None, ['test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(None, ['filetests']) - self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) - self.assertFalse(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - support.set_match_tests(None, ['test_access', 'DONTMATCH']) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): @@ -735,7 +631,7 @@ def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() @@ -837,6 +733,17 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(support.get_signal_name(exitcode), expected, + exitcode) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled @@ -852,7 +759,6 @@ def test_copy_python_src_ignore(self): # precisionbigmemtest # bigaddrspacetest # requires_resource - # run_doctest # threading_cleanup # reap_threads # can_symlink diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 82c1d7c856a1e5..0cc192655931ba 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -4,6 +4,8 @@ import symtable import unittest +from test import support +from test.support import os_helper TEST_CODE = """ @@ -11,7 +13,7 @@ glob = 42 some_var = 12 -some_non_assigned_global_var = 11 +some_non_assigned_global_var: int some_assigned_global_var = 11 class Mine: @@ -47,10 +49,124 @@ def namespace_test(): pass def generic_spam[T](a): pass -class GenericMine[T: int]: +class GenericMine[T: int, U: (int, str) = int]: pass """ +TEST_COMPLEX_CLASS_CODE = """ +# The following symbols are defined in ComplexClass +# without being introduced by a 'global' statement. +glob_unassigned_meth: Any +glob_unassigned_meth_pep_695: Any + +glob_unassigned_async_meth: Any +glob_unassigned_async_meth_pep_695: Any + +def glob_assigned_meth(): pass +def glob_assigned_meth_pep_695[T](): pass + +async def glob_assigned_async_meth(): pass +async def glob_assigned_async_meth_pep_695[T](): pass + +# The following symbols are defined in ComplexClass after +# being introduced by a 'global' statement (and therefore +# are not considered as local symbols of ComplexClass). +glob_unassigned_meth_ignore: Any +glob_unassigned_meth_pep_695_ignore: Any + +glob_unassigned_async_meth_ignore: Any +glob_unassigned_async_meth_pep_695_ignore: Any + +def glob_assigned_meth_ignore(): pass +def glob_assigned_meth_pep_695_ignore[T](): pass + +async def glob_assigned_async_meth_ignore(): pass +async def glob_assigned_async_meth_pep_695_ignore[T](): pass + +class ComplexClass: + a_var = 1234 + a_genexpr = (x for x in []) + a_lambda = lambda x: x + + type a_type_alias = int + type a_type_alias_pep_695[T] = list[T] + + class a_class: pass + class a_class_pep_695[T]: pass + + def a_method(self): pass + def a_method_pep_695[T](self): pass + + async def an_async_method(self): pass + async def an_async_method_pep_695[T](self): pass + + @classmethod + def a_classmethod(cls): pass + @classmethod + def a_classmethod_pep_695[T](self): pass + + @classmethod + async def an_async_classmethod(cls): pass + @classmethod + async def an_async_classmethod_pep_695[T](self): pass + + @staticmethod + def a_staticmethod(): pass + @staticmethod + def a_staticmethod_pep_695[T](self): pass + + @staticmethod + async def an_async_staticmethod(): pass + @staticmethod + async def an_async_staticmethod_pep_695[T](self): pass + + # These ones will be considered as methods because of the 'def' although + # they are *not* valid methods at runtime since they are not decorated + # with @staticmethod. + def a_fakemethod(): pass + def a_fakemethod_pep_695[T](): pass + + async def an_async_fakemethod(): pass + async def an_async_fakemethod_pep_695[T](): pass + + # Check that those are still considered as methods + # since they are not using the 'global' keyword. + def glob_unassigned_meth(): pass + def glob_unassigned_meth_pep_695[T](): pass + + async def glob_unassigned_async_meth(): pass + async def glob_unassigned_async_meth_pep_695[T](): pass + + def glob_assigned_meth(): pass + def glob_assigned_meth_pep_695[T](): pass + + async def glob_assigned_async_meth(): pass + async def glob_assigned_async_meth_pep_695[T](): pass + + # The following are not picked as local symbols because they are not + # visible by the class at runtime (this is equivalent to having the + # definitions outside of the class). + global glob_unassigned_meth_ignore + def glob_unassigned_meth_ignore(): pass + global glob_unassigned_meth_pep_695_ignore + def glob_unassigned_meth_pep_695_ignore[T](): pass + + global glob_unassigned_async_meth_ignore + async def glob_unassigned_async_meth_ignore(): pass + global glob_unassigned_async_meth_pep_695_ignore + async def glob_unassigned_async_meth_pep_695_ignore[T](): pass + + global glob_assigned_meth_ignore + def glob_assigned_meth_ignore(): pass + global glob_assigned_meth_pep_695_ignore + def glob_assigned_meth_pep_695_ignore[T](): pass + + global glob_assigned_async_meth_ignore + async def glob_assigned_async_meth_ignore(): pass + global glob_assigned_async_meth_pep_695_ignore + async def glob_assigned_async_meth_pep_695_ignore[T](): pass +""" + def find_block(block, name): for ch in block.get_children(): @@ -63,6 +179,7 @@ class SymtableTest(unittest.TestCase): top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") @@ -76,6 +193,7 @@ class SymtableTest(unittest.TestCase): GenericMine = find_block(top, "GenericMine") GenericMine_inner = find_block(GenericMine, "GenericMine") T = find_block(GenericMine, "T") + U = find_block(GenericMine, "U") def test_type(self): self.assertEqual(self.top.get_type(), "module") @@ -85,13 +203,14 @@ def test_type(self): self.assertEqual(self.internal.get_type(), "function") self.assertEqual(self.foo.get_type(), "function") self.assertEqual(self.Alias.get_type(), "type alias") - self.assertEqual(self.GenericAlias.get_type(), "type parameter") + self.assertEqual(self.GenericAlias.get_type(), "type parameters") self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") - self.assertEqual(self.generic_spam.get_type(), "type parameter") + self.assertEqual(self.generic_spam.get_type(), "type parameters") self.assertEqual(self.generic_spam_inner.get_type(), "function") - self.assertEqual(self.GenericMine.get_type(), "type parameter") + self.assertEqual(self.GenericMine.get_type(), "type parameters") self.assertEqual(self.GenericMine_inner.get_type(), "class") - self.assertEqual(self.T.get_type(), "TypeVar bound") + self.assertEqual(self.T.get_type(), "type variable") + self.assertEqual(self.U.get_type(), "type variable") def test_id(self): self.assertGreater(self.top.get_id(), 0) @@ -203,12 +322,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) @@ -238,6 +359,24 @@ def test_name(self): def test_class_info(self): self.assertEqual(self.Mine.get_methods(), ('a_method',)) + top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") + this = find_block(top, "ComplexClass") + + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised ### while parsing or building symbol table. @@ -282,10 +421,84 @@ def test_symtable_repr(self): self.assertEqual(str(self.top), "") self.assertEqual(str(self.spam), "") + def test_symbol_repr(self): + self.assertEqual(repr(self.spam.lookup("glob")), + "") + self.assertEqual(repr(self.spam.lookup("bar")), + "") + self.assertEqual(repr(self.spam.lookup("a")), + "") + self.assertEqual(repr(self.spam.lookup("internal")), + "") + self.assertEqual(repr(self.spam.lookup("other_internal")), + "") + self.assertEqual(repr(self.internal.lookup("x")), + "") + self.assertEqual(repr(self.other_internal.lookup("some_var")), + "") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "") + + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "") + def test_symtable_entry_repr(self): expected = f"" self.assertEqual(repr(self.top._table), expected) +class CommandLineTest(unittest.TestCase): + maxDiff = None + + def test_file(self): + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + with open(filename, 'w') as f: + f.write(TEST_CODE) + with support.captured_stdout() as stdout: + symtable.main([filename]) + out = stdout.getvalue() + self.assertIn('\n\n', out) + self.assertNotIn('\n\n\n', out) + lines = out.splitlines() + self.assertIn(f"symbol table for module from file {filename!r}:", lines) + self.assertIn(" local symbol 'glob': def_local", lines) + self.assertIn(" global_implicit symbol 'glob': use", lines) + self.assertIn(" local symbol 'spam': def_local", lines) + self.assertIn(" symbol table for function 'spam':", lines) + + def test_stdin(self): + with support.captured_stdin() as stdin: + stdin.write(TEST_CODE) + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main([]) + out = stdout.getvalue() + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main(['-']) + self.assertEqual(stdout.getvalue(), out) + lines = out.splitlines() + self.assertIn("symbol table for module from file '':", lines) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 52fc3573ddb7e0..cdeb26adf34d89 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -1004,11 +1034,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -1168,6 +1213,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1446,11 +1507,21 @@ Traceback (most recent call last): IndentationError: expected an indented block after function definition on line 1 + >>> def foo[T](x, /, y, *, z=2): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after function definition on line 1 + >>> class Blech(A): ... pass Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class Blech[T](A): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after class definition on line 1 + >>> match something: ... pass Traceback (most recent call last): @@ -1644,6 +1715,18 @@ Traceback (most recent call last): SyntaxError: invalid syntax +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + >>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated @@ -1657,6 +1740,49 @@ Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1791,22 +1917,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1817,7 +1943,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1895,16 +2021,116 @@ def f(x: *b) Invalid expressions in type scopes: + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + >>> type A[T: (x:=3)] = int Traceback (most recent call last): ... SyntaxError: named expression cannot be used within a TypeVar bound + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A[T: (yield 3)] = int Traceback (most recent call last): ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + >>> type A[T: (await 3)] = int Traceback (most recent call last): ... @@ -1915,6 +2141,31 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A = (x := 3) Traceback (most recent call last): ... @@ -1961,15 +2212,28 @@ def f(x: *b) >>> f(**x, *) Traceback (most recent call last): - SyntaxError: iterable argument unpacking follows keyword argument unpacking + SyntaxError: Invalid star expression >>> f(x, *:) Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re import doctest +import textwrap import unittest from test import support @@ -2216,6 +2480,58 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2271,15 +2587,28 @@ def test_error_parenthesis(self): """ self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + def test_error_string_literal(self): - self._check_error("'blech", "unterminated string literal") - self._check_error('"blech', "unterminated string literal") + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) self._check_error("'''blech", "unterminated triple-quoted string literal") self._check_error('"""blech', "unterminated triple-quoted string literal") def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ @@ -2337,7 +2666,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index da213506151594..69dccf9a9322f6 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -16,6 +16,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import import_helper +from test.support import force_not_colorized try: from test.support import interpreters except ImportError: @@ -145,6 +146,7 @@ def f(): class ExceptHookTest(unittest.TestCase): + @force_not_colorized def test_original_excepthook(self): try: raise ValueError(42) @@ -156,6 +158,7 @@ def test_original_excepthook(self): self.assertRaises(TypeError, sys.__excepthook__) + @force_not_colorized def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -391,10 +394,15 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) @@ -562,7 +570,8 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertTrue((sourceline.startswith("if leave_g.wait(") or + sourceline.startswith("g_raised.set()"))) finally: # Reap the spawned thread. leave_g.set() @@ -668,7 +677,7 @@ def test_thread_info(self): self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -691,11 +700,23 @@ def test_43581(self): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): + has_is_interned = (test.support.check_impl_detail(cpython=True) + or hasattr(sys, '_is_interned')) self.assertRaises(TypeError, sys.intern) + self.assertRaises(TypeError, sys.intern, b'abc') + if has_is_interned: + self.assertRaises(TypeError, sys._is_interned) + self.assertRaises(TypeError, sys._is_interned, b'abc') s = "never interned before" + str(random.randrange(0, 10**9)) self.assertTrue(sys.intern(s) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s), True) s2 = s.swapcase().swapcase() + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) self.assertTrue(sys.intern(s2) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) # Subclasses of string can't be interned, because they # provide too much opportunity for insane things to happen. @@ -707,34 +728,73 @@ def __hash__(self): return 123 self.assertRaises(TypeError, sys.intern, S("abc")) + if has_is_interned: + self.assertIs(sys._is_interned(S("abc")), False) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): + # Implementation detail: Dynamically allocated strings + # are distinct between interpreters s = "never interned before" + str(random.randrange(0, 10**9)) t = sys.intern(s) self.assertIs(t, s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys - t = sys.intern({s!r}) + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) != {id(s)}, (id(t), {id(s)}) assert id(t) != {id(t)}, (id(t), {id(t)}) ''')) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_statically_allocated(self): + # Implementation detail: Statically allocated strings are shared + # between interpreters. # See Tools/build/generate_global_objects.py for the list # of strings that are always statically allocated. - s = '__init__' - t = sys.intern(s) + for s in ('__init__', 'CANCELLED', '', 'utf-8', + '{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff', + ): + with self.subTest(s=s): + t = sys.intern(s) - interp = interpreters.create() - interp.run(textwrap.dedent(f''' - import sys - t = sys.intern({s!r}) - assert id(t) == {id(t)}, (id(t), {id(t)}) - ''')) + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) == {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_singleton(self): + # Implementation detail: singletons are used for 0- and 1-character + # latin1 strings. + for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff': + with self.subTest(s=s): + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + assert id(s) == {id(s)} + t = sys.intern(s) + ''')) + self.assertTrue(sys._is_interned(s)) def test_sys_flags(self): self.assertTrue(sys.flags) @@ -778,6 +838,7 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @force_not_colorized @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -1034,6 +1095,12 @@ def test_getallocatedblocks(self): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1087,13 +1154,13 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) self.assertGreater(level, 0) + @force_not_colorized @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: @@ -1114,14 +1181,20 @@ def check(tracebacklimit, expected): traceback = [ b'Traceback (most recent call last):', b' File "", line 8, in ', + b' f2()', + b' ~~^^', b' File "", line 6, in f2', + b' f1()', + b' ~~^^', b' File "", line 4, in f1', + b' 1 / 0', + b' ~~^~~', b'ZeroDivisionError: division by zero' ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[2:]) - check(1, traceback[:1] + traceback[3:]) + check(2, traceback[:1] + traceback[4:]) + check(1, traceback[:1] + traceback[7:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) @@ -1204,45 +1277,43 @@ def test_pystats(self): @test.support.cpython_only @unittest.skipUnless(hasattr(sys, 'abiflags'), 'need sys.abiflags') def test_disable_gil_abi(self): - abi_threaded = 't' in sys.abiflags - py_nogil = (sysconfig.get_config_var('Py_NOGIL') == 1) - self.assertEqual(py_nogil, abi_threaded) + self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): - def write_unraisable_exc(self, exc, err_msg, obj): - import _testinternalcapi - import types - err_msg2 = f"Exception ignored {err_msg}" - try: - _testinternalcapi.write_unraisable_exc(exc, err_msg, obj) - return types.SimpleNamespace(exc_type=type(exc), - exc_value=exc, - exc_traceback=exc.__traceback__, - err_msg=err_msg2, - object=obj) - finally: - # Explicitly break any reference cycle - exc = None - def test_original_unraisablehook(self): - for err_msg in (None, "original hook"): - with self.subTest(err_msg=err_msg): - obj = "an object" - - with test.support.captured_output("stderr") as stderr: - with test.support.swap_attr(sys, 'unraisablehook', - sys.__unraisablehook__): - self.write_unraisable_exc(ValueError(42), err_msg, obj) - - err = stderr.getvalue() - if err_msg is not None: - self.assertIn(f'Exception ignored {err_msg}: {obj!r}\n', err) - else: - self.assertIn(f'Exception ignored in: {obj!r}\n', err) - self.assertIn('Traceback (most recent call last):\n', err) - self.assertIn('ValueError: 42\n', err) + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable + obj = hex + + with support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Exception ignored in: {obj!r}') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), 'Error in %R', obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Error in {obj!r}:') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') def test_original_unraisablehook_err(self): # bpo-22836: PyErr_WriteUnraisable() should give sensible reports @@ -1289,6 +1360,8 @@ def test_original_unraisablehook_exception_qualname(self): # Check that the exception is printed with its qualified name # rather than just classname, and the module names appears # unless it is one of the hard-coded exclusions. + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable class A: class B: class X(Exception): @@ -1300,9 +1373,7 @@ class X(Exception): with test.support.captured_stderr() as stderr, test.support.swap_attr( sys, 'unraisablehook', sys.__unraisablehook__ ): - expected = self.write_unraisable_exc( - A.B.X(), "msg", "obj" - ) + err_writeunraisable(A.B.X(), "obj") report = stderr.getvalue() self.assertIn(A.B.X.__qualname__, report) if moduleName in ['builtins', '__main__']: @@ -1318,34 +1389,45 @@ def test_original_unraisablehook_wrong_type(self): sys.unraisablehook(exc) def test_custom_unraisablehook(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable hook_args = None def hook_func(args): nonlocal hook_args hook_args = args - obj = object() + obj = hex try: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - expected = self.write_unraisable_exc(ValueError(42), - "custom hook", obj) - for attr in "exc_type exc_value exc_traceback err_msg object".split(): - self.assertEqual(getattr(hook_args, attr), - getattr(expected, attr), - (hook_args, expected)) + exc = ValueError(42) + err_writeunraisable(exc, obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertIsNone(hook_args.err_msg) + self.assertEqual(hook_args.object, obj) + + err_formatunraisable(exc, "custom hook %R", obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertEqual(hook_args.err_msg, f'custom hook {obj!r}') + self.assertIsNone(hook_args.object) finally: # expected and hook_args contain an exception: break reference cycle expected = None hook_args = None def test_custom_unraisablehook_fail(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable def hook_func(*args): raise Exception("hook_func failed") with test.support.captured_output("stderr") as stderr: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - self.write_unraisable_exc(ValueError(42), - "custom hook fail", None) + err_writeunraisable(ValueError(42), "custom hook fail") err = stderr.getvalue() self.assertIn(f'Exception ignored in sys.unraisablehook: ' @@ -1361,8 +1443,9 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1398,7 +1481,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize*2) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1520,10 +1603,10 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass - check(func, size('15Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): @@ -1565,7 +1648,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1621,7 +1707,7 @@ def delx(self): del self.__x # type # static type: PyTypeObject fmt = 'P2nPI13Pl4Pn9Pn12PIPc' - s = vsize('2P' + fmt) + s = vsize(fmt) check(int, s) # class s = vsize(fmt + # PyTypeObject @@ -1678,11 +1764,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) @@ -1740,6 +1830,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1747,7 +1852,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 9e8936630de920..32e03d7cd25dbe 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -30,9 +30,9 @@ def callback(self, frame, event, arg): if (event == "call" or event == "return" or event == "exception"): - self.add_event(event, frame) + self.add_event(event, frame, arg) - def add_event(self, event, frame=None): + def add_event(self, event, frame=None, arg=None): """Add an event to the log.""" if frame is None: frame = sys._getframe(1) @@ -43,7 +43,7 @@ def add_event(self, event, frame=None): frameno = len(self.frames) self.frames.append(frame) - self.events.append((frameno, event, ident(frame))) + self.events.append((frameno, event, ident(frame), arg)) def get_events(self): """Remove calls to add_event().""" @@ -89,11 +89,16 @@ def trace_pass(self, frame): class TestCaseBase(unittest.TestCase): - def check_events(self, callable, expected): + def check_events(self, callable, expected, check_args=False): events = capture_events(callable, self.new_watcher()) - if events != expected: - self.fail("Expected events:\n%s\nReceived events:\n%s" - % (pprint.pformat(expected), pprint.pformat(events))) + if check_args: + if events != expected: + self.fail("Expected events:\n%s\nReceived events:\n%s" + % (pprint.pformat(expected), pprint.pformat(events))) + else: + if [(frameno, event, ident) for frameno, event, ident, arg in events] != expected: + self.fail("Expected events:\n%s\nReceived events:\n%s" + % (pprint.pformat(expected), pprint.pformat(events))) class ProfileHookTestCase(TestCaseBase): @@ -264,15 +269,11 @@ def g(p): f_ident = ident(f) g_ident = ident(g) - self.check_events(g, [(1, 'call', g_ident), - (2, 'call', f_ident), - (2, 'return', f_ident), - # once more; the generator is being garbage collected - # and it will do a PY_THROW - (2, 'call', f_ident), - (2, 'return', f_ident), - (1, 'return', g_ident), - ]) + self.check_events(g, [(1, 'call', g_ident, None), + (2, 'call', f_ident, None), + (2, 'return', f_ident, 0), + (1, 'return', g_ident, None), + ], check_args=True) def test_stop_iteration(self): def f(): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index f02169602e4925..c622fd9ce7c466 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,8 +7,17 @@ import gc from functools import wraps import asyncio -from test.support import import_helper +from test.support import import_helper, requires_subprocess import contextlib +import os +import tempfile +import textwrap +import subprocess +import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None support.requires_working_socket(module=True) @@ -1641,15 +1650,15 @@ def func(): EXPECTED_EVENTS = [ (0, 'call'), (2, 'line'), - (1, 'line'), (-3, 'call'), (-2, 'line'), (-2, 'return'), - (4, 'line'), (1, 'line'), + (4, 'line'), + (2, 'line'), (-2, 'call'), (-2, 'return'), - (1, 'return'), + (2, 'return'), ] # C level events should be the same as expected and the same as Python level. @@ -1801,6 +1810,40 @@ def compare_events(self, line_offset, events, expected_events): def make_tracer(): return Tracer(trace_opcode_events=True) + @requires_subprocess() + def test_trace_opcodes_after_settrace(self): + """Make sure setting f_trace_opcodes after starting trace works even + if it's the first time f_trace_opcodes is being set. GH-103615""" + + code = textwrap.dedent(""" + import sys + + def opcode_trace_func(frame, event, arg): + if event == "opcode": + print("opcode trace triggered") + return opcode_trace_func + + sys.settrace(opcode_trace_func) + sys._getframe().f_trace = opcode_trace_func + sys._getframe().f_trace_opcodes = True + a = 1 + """) + + # We can't use context manager because Windows can't execute a file while + # it's being written + tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.py') + tmp.write(code.encode('utf-8')) + tmp.close() + try: + p = subprocess.Popen([sys.executable, tmp.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + out = p.stdout.read() + finally: + os.remove(tmp.name) + p.stdout.close() + p.stderr.close() + self.assertIn(b"opcode trace triggered", out) + class RaisingTraceFuncTestCase(unittest.TestCase): def setUp(self): @@ -2001,6 +2044,9 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, stack.enter_context(self.assertRaisesRegex(*error)) if warning is not None: stack.enter_context(self.assertWarnsRegex(*warning)) + else: + stack.enter_context(warnings.catch_warnings()) + warnings.simplefilter('error') func(output) sys.settrace(None) @@ -2064,6 +2110,40 @@ def test_jump_simple_backwards(output): output.append(1) output.append(2) + @jump_test(1, 4, [5], warning=(RuntimeWarning, unbound_locals)) + def test_jump_is_none_forwards(output): + x = None + if x is None: + output.append(3) + else: + output.append(5) + + @jump_test(6, 5, [3, 5, 6]) + def test_jump_is_none_backwards(output): + x = None + if x is None: + output.append(3) + else: + output.append(5) + output.append(6) + + @jump_test(2, 4, [5]) + def test_jump_is_not_none_forwards(output): + x = None + if x is not None: + output.append(3) + else: + output.append(5) + + @jump_test(6, 5, [5, 5, 6]) + def test_jump_is_not_none_backwards(output): + x = None + if x is not None: + output.append(3) + else: + output.append(5) + output.append(6) + @jump_test(3, 5, [2, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_out_of_block_forwards(output): for i in 1, 2: @@ -2958,16 +3038,19 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): + + count = min(1000, int(support.get_c_recursion_limit() * 0.8)) + code = """if 1: def f(): return ( {} ) - """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) - ns = {f"var{i}": i for i in range(1000)} + """.format("\n+\n".join(f"var{i}\n" for i in range(count))) + ns = {f"var{i}": i for i in range(count)} exec(code, ns) counts = self.count_traces(ns["f"]) - self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1}) class TestEdgeCases(unittest.TestCase): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2a6813f00bccc6..37cee927686ba3 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,3 +1,5 @@ +import platform +import re import unittest import sys import os @@ -6,7 +8,11 @@ from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, @@ -43,6 +49,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -66,6 +73,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -139,7 +147,7 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) @@ -149,20 +157,24 @@ def test_posix_venv_scheme(self): binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) def test_nt_venv_scheme(self): @@ -172,14 +184,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -215,6 +232,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -330,9 +352,26 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -410,10 +449,16 @@ def test_library(self): self.assertTrue(library.startswith(f'python{major}{minor}')) self.assertTrue(library.endswith('.dll')) self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") else: self.assertTrue(library.startswith(f'libpython{major}.{minor}')) self.assertTrue(library.endswith('.a')) - self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() @@ -460,6 +505,8 @@ def test_platform_in_subprocess(self): self.assertEqual(my_platform, test_platform) @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -472,11 +519,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # /PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # /PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # /pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already @@ -498,12 +549,9 @@ def test_EXT_SUFFIX_in_vars(self): vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -516,6 +564,19 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') @@ -526,6 +587,8 @@ class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index cc122cafc7985c..30dcb3e3c4f4f9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: - stderr = f"{file_path!r}: Token Error: " + stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index cc26da05daeafc..f715940de1d584 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -15,6 +15,7 @@ import unittest.mock import tarfile +from test import archiver_tests from test import support from test.support import os_helper from test.support import script_helper @@ -43,7 +44,7 @@ def sha256sum(data): TEMPDIR = os.path.abspath(os_helper.TESTFN) + "-tardir" tarextdir = TEMPDIR + '-extract-test' -tarname = support.findfile("testtar.tar") +tarname = support.findfile("testtar.tar", subdir="archivetestdata") gzipname = os.path.join(TEMPDIR, "testtar.tar.gz") bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2") xzname = os.path.join(TEMPDIR, "testtar.tar.xz") @@ -323,11 +324,23 @@ def test_list_verbose(self): # accessories if verbose flag is being used # ... # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype - # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # -rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # drwxr-xr-x tarfile/tarfile 0 2003-01-05 15:19:43 ustar/dirtype/ # ... - self.assertRegex(out, (br'\?rw-r--r-- tarfile/tarfile\s+7011 ' - br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' - br'ustar/\w+type ?\r?\n') * 2) + # + # Array of values to modify the regex below: + # ((file_type, file_permissions, file_length), ...) + type_perm_lengths = ( + (br'\?', b'rw-r--r--', b'7011'), (b'-', b'rw-r--r--', b'7011'), + (b'd', b'rwxr-xr-x', b'0'), (b'd', b'rwxr-xr-x', b'255'), + (br'\?', b'rw-r--r--', b'0'), (b'l', b'rwxrwxrwx', b'0'), + (b'b', b'rw-rw----', b'3,0'), (b'c', b'rw-rw-rw-', b'1,3'), + (b'p', b'rw-r--r--', b'0')) + self.assertRegex(out, b''.join( + [(tp + (br'%s tarfile/tarfile\s+%s ' % (perm, ln) + + br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' + br'ustar/\w+type[/>\sa-z-]*\n')) for tp, perm, ln + in type_perm_lengths])) # Make sure it prints the source of link with verbose flag self.assertIn(b'ustar/symtype -> regtype', out) self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out) @@ -373,7 +386,7 @@ def test_is_tarfile_erroneous(self): self.assertFalse(tarfile.is_tarfile(tmpname)) # is_tarfile works on path-like objects - self.assertFalse(tarfile.is_tarfile(pathlib.Path(tmpname))) + self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname))) # is_tarfile works on file objects with open(tmpname, "rb") as fobj: @@ -387,7 +400,7 @@ def test_is_tarfile_valid(self): self.assertTrue(tarfile.is_tarfile(self.tarname)) # is_tarfile works on path-like objects - self.assertTrue(tarfile.is_tarfile(pathlib.Path(self.tarname))) + self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname))) # is_tarfile works on file objects with open(self.tarname, "rb") as fobj: @@ -491,22 +504,39 @@ def test_length_zero_header(self): # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail # with an exception with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"): - with tarfile.open(support.findfile('recursion.tar')) as tar: + with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass - def test_extractfile_name(self): + def test_extractfile_attrs(self): # gh-74468: TarFile.name must name a file, not a parent archive. file = self.tar.getmember('ustar/regtype') with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) class MiscReadTestBase(CommonReadTest): - def requires_name_attribute(self): - pass + is_stream = False def test_no_name_argument(self): - self.requires_name_attribute() with open(self.tarname, "rb") as fobj: self.assertIsInstance(fobj.name, str) with tarfile.open(fileobj=fobj, mode=self.mode) as tar: @@ -539,7 +569,6 @@ def test_int_name_attribute(self): self.assertIsNone(tar.name) def test_bytes_name_attribute(self): - self.requires_name_attribute() tarname = os.fsencode(self.tarname) with open(tarname, 'rb') as fobj: self.assertIsInstance(fobj.name, bytes) @@ -547,21 +576,23 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) - def test_pathlike_name(self): - tarname = pathlib.Path(self.tarname) + def test_pathlike_name(self, tarname=None): + if tarname is None: + tarname = self.tarname + expected = os.path.abspath(tarname) + tarname = os_helper.FakePath(tarname) with tarfile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with self.taropen(tarname) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with tarfile.TarFile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) if self.suffix == '': with tarfile.TarFile(tarname, mode='r') as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) + + def test_pathlike_bytes_name(self): + self.test_pathlike_name(os.fsencode(self.tarname)) def test_illegal_mode_arg(self): with open(tmpname, 'wb'): @@ -707,24 +738,49 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - def test_extractall_pathlike_name(self): + def test_deprecation_if_no_filter_passed_to_extractall(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar + ): + directories = [t for t in tar if t.isdir()] + with self.assertWarnsRegex(DeprecationWarning, "Use the filter argument") as cm: + tar.extractall(DIR, directories) + # check that the stacklevel of the deprecation warning is correct: + self.assertEqual(cm.filename, __file__) + + def test_deprecation_if_no_filter_passed_to_extract(self): + dirtype = "ustar/dirtype" DIR = pathlib.Path(TEMPDIR) / "extractall" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar + ): + tarinfo = tar.getmember(dirtype) + with self.assertWarnsRegex(DeprecationWarning, "Use the filter argument") as cm: + tar.extract(tarinfo, path=DIR) + # check that the stacklevel of the deprecation warning is correct: + self.assertEqual(cm.filename, __file__) + + def test_extractall_pathlike_dir(self): + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories, filter='fully_trusted') + tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted') for tarinfo in directories: - path = DIR / tarinfo.name + path = os.path.join(DIR, tarinfo.name) self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - def test_extract_pathlike_name(self): + def test_extract_pathlike_dir(self): dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR, filter='fully_trusted') - extracted = DIR / dirtype + tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted') + extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) def test_init_close_fobj(self): @@ -783,17 +839,16 @@ class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("BZ2File have no name attribute") + pass class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("LZMAFile have no name attribute") + pass class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method @@ -1154,7 +1209,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -1337,11 +1392,11 @@ def test_ordered_recursion(self): def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: - path = pathlib.Path(TEMPDIR) / "file" + path = os.path.join(TEMPDIR, "file") with open(path, "wb") as fobj: fobj.write(b"aaa") - tarinfo = tar.gettarinfo(path) - tarinfo2 = tar.gettarinfo(os.fspath(path)) + tarinfo = tar.gettarinfo(os_helper.FakePath(path)) + tarinfo2 = tar.gettarinfo(path) self.assertIsInstance(tarinfo.name, str) self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) @@ -1555,6 +1610,12 @@ def write(self, data): pax_headers={'non': 'empty'}) self.assertFalse(f.closed) + def test_missing_fileobj(self): + with tarfile.open(tmpname, self.mode) as tar: + tarinfo = tar.gettarinfo(tarname) + with self.assertRaises(ValueError): + tar.addfile(tarinfo) + class GzipWriteTest(GzipTest, WriteTest): pass @@ -1888,10 +1949,10 @@ def test_create_existing_taropen(self): self.assertIn("spameggs42", names[0]) def test_create_pathlike_name(self): - with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1902,10 +1963,10 @@ def test_create_pathlike_name(self): self.assertIn('spameggs42', names[0]) def test_create_taropen_pathlike_name(self): - with self.taropen(pathlib.Path(tmpname), "x") as tobj: + with self.taropen(os_helper.FakePath(tmpname), "x") as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -2565,7 +2626,7 @@ def test__all__(self): support.check__all__(self, tarfile, not_exported=not_exported) def test_useful_error_message_when_modules_missing(self): - fname = os.path.join(os.path.dirname(__file__), 'testtar.tar.xz') + fname = os.path.join(os.path.dirname(__file__), 'archivetestdata', 'testtar.tar.xz') with self.assertRaises(tarfile.ReadError) as excinfo: error = tarfile.CompressionError('lzma module is not available'), with unittest.mock.patch.object(tarfile.TarFile, 'xzopen', side_effect=error): @@ -2630,7 +2691,7 @@ def test_test_command_verbose(self): self.assertIn(b'is a tar archive.\n', out) def test_test_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') rc, out, err = self.tarfilecmd_failure('-t', zipname) self.assertIn(b' is not a tar archive.', err) self.assertEqual(out, b'') @@ -2672,7 +2733,7 @@ def test_list_command_verbose(self): self.assertEqual(out, expected) def test_list_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') rc, out, err = self.tarfilecmd_failure('-l', zipname) self.assertIn(b' is not a tar archive.', err) self.assertEqual(out, b'') @@ -2797,7 +2858,7 @@ def test_extract_command_different_directory(self): os_helper.rmtree(tarextdir) def test_extract_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') with os_helper.temp_cwd(tarextdir): rc, out, err = self.tarfilecmd_failure('-e', zipname) self.assertIn(b' is not a tar archive.', err) @@ -3226,7 +3287,8 @@ def test_add(self): tar = tarfile.open(fileobj=bio, mode='w', format=tarformat) tarinfo = tar.gettarinfo(tarname) try: - tar.addfile(tarinfo) + with open(tarname, 'rb') as f: + tar.addfile(tarinfo, f) except Exception: if tarformat == tarfile.USTAR_FORMAT: # In the old, limited format, adding might fail for @@ -3241,7 +3303,8 @@ def test_add(self): replaced = tarinfo.replace(**{attr_name: None}) with self.assertRaisesRegex(ValueError, f"{attr_name}"): - tar.addfile(replaced) + with open(tarname, 'rb') as f: + tar.addfile(replaced, f) def test_list(self): # Change some metadata to None, then compare list() output @@ -3452,7 +3515,7 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, path = pathlib.Path(os.path.normpath(self.destdir / name)) self.assertIn(path, self.expected_paths) self.expected_paths.remove(path) - if mode is not None and os_helper.can_chmod(): + if mode is not None and os_helper.can_chmod() and os.name != 'nt': got = stat.filemode(stat.S_IMODE(path.stat().st_mode)) self.assertEqual(got, mode) if type is None and isinstance(name, str) and name.endswith('/'): @@ -4123,6 +4186,38 @@ def valueerror_filter(tarinfo, path): self.expect_exception(TypeError) # errorlevel is not int +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = os.path.join(TEMPDIR, "testoverwrite") + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + tar.addfile(tar.gettarinfo(os.curdir, 'test')) + + p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar') + cls.ar_with_implicit_dir = p + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test/file') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + def open(self, path): + return tarfile.open(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir, filter='fully_trusted') + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ebdb58f91d3d8a..d479f7d7515d9b 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -51,7 +51,7 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertIn(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') def testEvalException(self): tcl = self.interp @@ -61,11 +61,30 @@ def testEvalException2(self): tcl = self.interp self.assertRaises(TclError,tcl.eval,'this is wrong') + def test_eval_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.eval('set a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + self.assertEqual(a, expected) + def testCall(self): tcl = self.interp tcl.call('set','a','1') self.assertEqual(tcl.call('set','a'),'1') + def test_call_passing_null(self): + tcl = self.interp + tcl.call('set', 'a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.call('set', 'a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testCallException(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'set','a') @@ -74,11 +93,35 @@ def testCallException2(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'this','is','wrong') + def test_call_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.call('set', 'a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testSetVar(self): tcl = self.interp tcl.setvar('a','1') self.assertEqual(tcl.eval('set a'),'1') + def test_setvar_passing_null(self): + tcl = self.interp + tcl.setvar('a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.setvar('a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testSetVarArray(self): tcl = self.interp tcl.setvar('a(1)','1') @@ -102,6 +145,18 @@ def testGetVarArrayException(self): tcl = self.interp self.assertRaises(TclError,tcl.getvar,'a(1)') + def test_getvar_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.getvar('a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testUnsetVar(self): tcl = self.interp tcl.setvar('a',1) @@ -219,10 +274,18 @@ def test_evalfile_surrogates_in_result(self): with open(filename, 'wb') as f: f.write(b""" set a "<\xed\xa0\xbd\xed\xb2\xbb>" + """) + if tcl_version >= (9, 0): + self.assertRaises(TclError, tcl.evalfile, filename) + else: + tcl.evalfile(filename) + self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') + + with open(filename, 'wb') as f: + f.write(b""" set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') def testEvalFileException(self): @@ -482,29 +545,36 @@ def testfunc(arg): return arg self.interp.createcommand('testfunc', testfunc) self.addCleanup(self.interp.tk.deletecommand, 'testfunc') - def check(value, expected=None, *, eq=self.assertEqual): - if expected is None: - expected = value + def check(value, expected1=None, expected2=None, *, eq=self.assertEqual): + expected = value + if self.wantobjects >= 2: + if expected2 is not None: + expected = expected2 + expected_type = type(expected) + else: + if expected1 is not None: + expected = expected1 + expected_type = str nonlocal result result = None r = self.interp.call('testfunc', value) - self.assertIsInstance(result, str) + self.assertIsInstance(result, expected_type) eq(result, expected) - self.assertIsInstance(r, str) + self.assertIsInstance(r, expected_type) eq(r, expected) def float_eq(actual, expected): self.assertAlmostEqual(float(actual), expected, delta=abs(expected) * 1e-10) - check(True, '1') - check(False, '0') + check(True, '1', 1) + check(False, '0', 0) check('string') check('string\xbd') check('string\u20ac') check('string\U0001f4bb') if sys.platform != 'win32': - check('<\udce2\udc82\udcac>', '<\u20ac>') - check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>') + check('<\udce2\udc82\udcac>', '<\u20ac>', '<\u20ac>') + check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>', '<\U0001f4bb>') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -526,9 +596,31 @@ def float_eq(actual, expected): check(float('inf'), eq=float_eq) check(-float('inf'), eq=float_eq) # XXX NaN representation can be not parsable by float() - check((), '') - check((1, (2,), (3, 4), '5 6', ()), '1 2 {3 4} {5 6} {}') - check([1, [2,], [3, 4], '5 6', []], '1 2 {3 4} {5 6} {}') + check((), '', '') + check((1, (2,), (3, 4), '5 6', ()), + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + check([1, [2,], [3, 4], '5 6', []], + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + + def test_passing_tcl_obj(self): + tcl = self.interp.tk + a = None + def testfunc(arg): + nonlocal a + a = arg + self.interp.createcommand('testfunc', testfunc) + self.addCleanup(self.interp.tk.deletecommand, 'testfunc') + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + tcl.eval(r'testfunc $a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects >= 2: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) def test_splitlist(self): splitlist = self.interp.tk.splitlist @@ -654,6 +746,7 @@ def test_new_tcl_obj(self): support.check_disallow_instantiation(self, _tkinter.TkttType) support.check_disallow_instantiation(self, _tkinter.TkappType) + class BigmemTclTest(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 1673507e2f7c91..a5e182cef23dc5 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -13,6 +13,7 @@ import weakref import gc import shutil +import subprocess from unittest import mock import unittest @@ -62,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -442,7 +437,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) @@ -680,7 +675,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -781,7 +776,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) @@ -803,6 +798,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists @@ -1641,6 +1663,28 @@ def test_explicit_cleanup_ignore_errors(self): temp_path.exists(), f"TemporaryDirectory {temp_path!s} exists after cleanup") + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_explicit_cleanup_correct_error(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + with open(os.path.join(temp_dir.name, "example.txt"), 'wb'): + # Previously raised NotADirectoryError on some OSes + # (e.g. Windows). See bpo-43153. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_cleanup_with_used_directory(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + subdir = os.path.join(temp_dir.name, "subdir") + os.mkdir(subdir) + with os_helper.change_cwd(subdir): + # Previously raised RecursionError on some OSes + # (e.g. Windows). See bpo-35144. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + @os_helper.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) @@ -1662,6 +1706,103 @@ def test_cleanup_with_symlink_to_a_directory(self): "were deleted") d2.cleanup() + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_modes(self): + # cleanup() should not follow symlinks when fixing mode bits (#91133) + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chmod(symlink, mode, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chmod(symlink, mode) + except FileNotFoundError: + pass + os.chmod(d1.name, mode) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chmod(file1, mode) + old_mode = os.stat(file1).st_mode + test(file1, target_is_directory=False) + new_mode = os.stat(file1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + with self.subTest('existing dir'): + os.chmod(dir1, mode) + old_mode = os.stat(dir1).st_mode + test(dir1, target_is_directory=True) + new_mode = os.stat(dir1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_flags(self): + # cleanup() should not follow symlinks when fixing flags (#91133) + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chflags(symlink, flags, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chflags(symlink, flags) + except FileNotFoundError: + pass + os.chflags(d1.name, flags) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chflags(file1, flags) + old_flags = os.stat(file1).st_flags + test(file1, target_is_directory=False) + new_flags = os.stat(file1).st_flags + self.assertEqual(new_flags, old_flags) + + with self.subTest('existing dir'): + os.chflags(dir1, flags) + old_flags = os.stat(dir1).st_flags + test(dir1, target_is_directory=True) + new_flags = os.stat(dir1).st_flags + self.assertEqual(new_flags, old_flags) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -1834,10 +1975,7 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) - @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') - def test_flags(self): - flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK - + def check_flags(self, flags): # skip the test if these flags are not supported (ex: FreeBSD 13) filename = os_helper.TESTFN try: @@ -1846,13 +1984,18 @@ def test_flags(self): os.chflags(filename, flags) except OSError as exc: # "OSError: [Errno 45] Operation not supported" - self.skipTest(f"chflags() doesn't support " - f"UF_IMMUTABLE|UF_NOUNLINK: {exc}") + self.skipTest(f"chflags() doesn't support flags " + f"{flags:#b}: {exc}") else: os.chflags(filename, 0) finally: os_helper.unlink(filename) + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + def test_flags(self): + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + d = self.do_create(recurse=3, dirs=2, files=2) with d: # Change files and directories flags recursively. diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index 58698ffac2d981..22e397c7a409c4 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -211,6 +211,15 @@ def test_constants(self): self.assertLess(termios.VTIME, termios.NCCS) self.assertLess(termios.VMIN, termios.NCCS) + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + def test_exception(self): self.assertTrue(issubclass(termios.error, Exception)) self.assertFalse(issubclass(termios.error, OSError)) diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 8656fbdd83e9c7..d94e04250c9307 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -155,11 +155,188 @@ def task(): started.acquire() self.assertEqual(str(cm.unraisable.exc_value), "task failed") - self.assertIs(cm.unraisable.object, task) + self.assertIsNone(cm.unraisable.object) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored in thread started by") + f"Exception ignored in thread started by {task!r}") self.assertIsNotNone(cm.unraisable.exc_traceback) + def test_join_thread(self): + finished = [] + + def task(): + time.sleep(0.05) + finished.append(thread.get_ident()) + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(task) + handle.join() + self.assertEqual(len(finished), 1) + self.assertEqual(handle.ident, finished[0]) + + def test_join_thread_already_exited(self): + def task(): + pass + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(task) + time.sleep(0.05) + handle.join() + + def test_join_several_times(self): + def task(): + pass + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(task) + handle.join() + # Subsequent join() calls should succeed + handle.join() + + def test_joinable_not_joined(self): + handle_destroyed = thread.allocate_lock() + handle_destroyed.acquire() + + def task(): + handle_destroyed.acquire() + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(task) + del handle + handle_destroyed.release() + + def test_join_from_self(self): + errors = [] + handles = [] + start_joinable_thread_returned = thread.allocate_lock() + start_joinable_thread_returned.acquire() + task_tried_to_join = thread.allocate_lock() + task_tried_to_join.acquire() + + def task(): + start_joinable_thread_returned.acquire() + try: + handles[0].join() + except Exception as e: + errors.append(e) + finally: + task_tried_to_join.release() + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(task) + handles.append(handle) + start_joinable_thread_returned.release() + # Can still join after joining failed in other thread + task_tried_to_join.acquire() + handle.join() + + assert len(errors) == 1 + with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): + raise errors[0] + + def test_join_then_self_join(self): + # make sure we can't deadlock in the following scenario with + # threads t0 and t1 (see comment in `ThreadHandle_join()` for more + # details): + # + # - t0 joins t1 + # - t1 self joins + def make_lock(): + lock = thread.allocate_lock() + lock.acquire() + return lock + + error = None + self_joiner_handle = None + self_joiner_started = make_lock() + self_joiner_barrier = make_lock() + def self_joiner(): + nonlocal error + + self_joiner_started.release() + self_joiner_barrier.acquire() + + try: + self_joiner_handle.join() + except Exception as e: + error = e + + joiner_started = make_lock() + def joiner(): + joiner_started.release() + self_joiner_handle.join() + + with threading_helper.wait_threads_exit(): + self_joiner_handle = thread.start_joinable_thread(self_joiner) + # Wait for the self-joining thread to start + self_joiner_started.acquire() + + # Start the thread that joins the self-joiner + joiner_handle = thread.start_joinable_thread(joiner) + + # Wait for the joiner to start + joiner_started.acquire() + + # Not great, but I don't think there's a deterministic way to make + # sure that the self-joining thread has been joined. + time.sleep(0.1) + + # Unblock the self-joiner + self_joiner_barrier.release() + + self_joiner_handle.join() + joiner_handle.join() + + with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): + raise error + + def test_join_with_timeout(self): + lock = thread.allocate_lock() + lock.acquire() + + def thr(): + lock.acquire() + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(thr) + handle.join(0.1) + self.assertFalse(handle.is_done()) + lock.release() + handle.join() + self.assertTrue(handle.is_done()) + + def test_join_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle.join() + + def test_set_done_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle._set_done() + + def test_start_duplicate_handle(self): + lock = thread.allocate_lock() + lock.acquire() + + def func(): + lock.acquire() + + handle = thread._ThreadHandle() + with threading_helper.wait_threads_exit(): + thread.start_joinable_thread(func, handle=handle) + with self.assertRaisesRegex(RuntimeError, "thread already started"): + thread.start_joinable_thread(func, handle=handle) + lock.release() + handle.join() + + def test_start_with_none_handle(self): + def func(): + pass + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(func, handle=None) + handle.join() + class Barrier: def __init__(self, num_threads): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 00a64372b394dc..329767aa82e336 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,11 @@ """ import test.support -from test.support import threading_helper, requires_subprocess +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import force_not_colorized import random import sys @@ -28,7 +29,7 @@ try: from test.support import interpreters -except ModuleNotFoundError: +except ImportError: interpreters = None threading_helper.requires_working_threading(module=True) @@ -47,6 +48,8 @@ def skip_unless_reliable_fork(test): return unittest.skip("due to known OS bug related to thread+fork")(test) if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork")(test) return test @@ -115,6 +118,7 @@ def tearDown(self): class ThreadTests(BaseTestCase): + maxDiff = 9999 @cpython_only def test_name(self): @@ -170,11 +174,21 @@ def test_args_argument(self): t.start() t.join() - @cpython_only - def test_disallow_instantiation(self): - # Ensure that the type disallows instantiation (bpo-43916) - lock = threading.Lock() - test.support.check_disallow_instantiation(self, type(lock)) + def test_lock_no_args(self): + threading.Lock() # works + self.assertRaises(TypeError, threading.Lock, 1) + self.assertRaises(TypeError, threading.Lock, a=1) + self.assertRaises(TypeError, threading.Lock, 1, 2, a=1, b=2) + + def test_lock_no_subclass(self): + # Intentionally disallow subclasses of threading.Lock because they have + # never been allowed, so why start now just because the type is public? + with self.assertRaises(TypeError): + class MyLock(threading.Lock): pass + + def test_lock_or_none(self): + import types + self.assertIsInstance(threading.Lock | None, types.UnionType) # Create a bunch of threads, let each do some work, wait until all are # done. @@ -226,8 +240,6 @@ def f(): tid = _thread.start_new_thread(f, ()) done.wait() self.assertEqual(ident[0], tid) - # Kill the "immortal" _DummyThread - del threading._active[ident[0]] # run with a small(ish) thread stack size (256 KiB) def test_various_ops_small_stack(self): @@ -255,11 +267,29 @@ def test_various_ops_large_stack(self): def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. + dummy_thread = None + error = None def f(mutex): - # Calling current_thread() forces an entry for the foreign - # thread to get made in the threading._active map. - threading.current_thread() - mutex.release() + try: + nonlocal dummy_thread + nonlocal error + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + dummy_thread = threading.current_thread() + tid = dummy_thread.ident + self.assertIn(tid, threading._active) + self.assertIsInstance(dummy_thread, threading._DummyThread) + self.assertIs(threading._active.get(tid), dummy_thread) + # gh-29376 + self.assertTrue( + dummy_thread.is_alive(), + 'Expected _DummyThread to be considered alive.' + ) + self.assertIn('_DummyThread', repr(dummy_thread)) + except BaseException as e: + error = e + finally: + mutex.release() mutex = threading.Lock() mutex.acquire() @@ -267,20 +297,25 @@ def f(mutex): tid = _thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() - self.assertIn(tid, threading._active) - self.assertIsInstance(threading._active[tid], threading._DummyThread) - #Issue 29376 - self.assertTrue(threading._active[tid].is_alive()) - self.assertRegex(repr(threading._active[tid]), '_DummyThread') - + if error is not None: + raise error + self.assertEqual(tid, dummy_thread.ident) # Issue gh-106236: with self.assertRaises(RuntimeError): - threading._active[tid].join() - threading._active[tid]._started.clear() + dummy_thread.join() + dummy_thread._started.clear() with self.assertRaises(RuntimeError): - threading._active[tid].is_alive() - - del threading._active[tid] + dummy_thread.is_alive() + # Busy wait for the following condition: after the thread dies, the + # related dummy thread must be removed from threading._active. + timeout = 5 + timeout_at = time.monotonic() + timeout + while time.monotonic() < timeout_at: + if threading._active.get(dummy_thread.ident) is not dummy_thread: + break + time.sleep(.1) + else: + self.fail('It was expected that the created threading._DummyThread was removed from threading._active.') # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. @@ -374,10 +409,10 @@ def run(self): def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() - _start_new_thread = threading._start_new_thread - threading._start_new_thread = fail_new_thread + _start_joinable_thread = threading._start_joinable_thread + threading._start_joinable_thread = fail_new_thread try: t = threading.Thread(target=lambda: None) self.assertRaises(threading.ThreadError, t.start) @@ -385,12 +420,16 @@ def fail_new_thread(*args): t in threading._limbo, "Failed to cleanup _limbo map on failure of Thread.start().") finally: - threading._start_new_thread = _start_new_thread + threading._start_joinable_thread = _start_joinable_thread def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -423,6 +462,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -472,7 +516,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() @@ -482,6 +526,47 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) + def test_join_from_multiple_threads(self): + # Thread.join() should be thread-safe + errors = [] + + def worker(): + time.sleep(0.005) + + def joiner(thread): + try: + thread.join() + except Exception as e: + errors.append(e) + + for N in range(2, 20): + threads = [threading.Thread(target=worker)] + for i in range(N): + threads.append(threading.Thread(target=joiner, + args=(threads[0],))) + for t in threads: + t.start() + time.sleep(0.01) + for t in threads: + t.join() + if errors: + raise errors[0] + + def test_join_with_timeout(self): + lock = _thread.allocate_lock() + lock.acquire() + + def worker(): + lock.acquire() + + thread = threading.Thread(target=worker) + thread.start() + thread.join(timeout=0.01) + assert thread.is_alive() + lock.release() + thread.join() + assert not thread.is_alive() + def test_no_refcycle_through_target(self): class RunSelfFunction(object): def __init__(self, should_raise): @@ -635,19 +720,25 @@ def test_main_thread_after_fork(self): import os, threading from test import support + ident = threading.get_ident() pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) else: support.wait_process(pid, exitcode=0) """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err, b"") - self.assertEqual(data, "MainThread\nTrue\nTrue\n") + self.assertEqual(data, + "current ident True\n" + "main MainThread\n" + "main ident True\n" + "current is main True\n") @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") @@ -657,15 +748,17 @@ def test_main_thread_after_fork_from_nonmain_thread(self): from test import support def func(): + ident = threading.get_ident() with warnings.catch_warnings(record=True) as ws: warnings.filterwarnings( "always", category=DeprecationWarning) pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) # stdout is fully buffered because not a tty, # we have to flush before exit. sys.stdout.flush() @@ -681,7 +774,79 @@ def func(): _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err.decode('utf-8'), "") - self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") + self.assertEqual(data, + "current ident True\n" + "main Thread-1 (func) Thread\n" + "main ident True\n" + "current is main True\n" + ) + + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): + code = """if 1: + import os, threading, sys, traceback, _thread + from test import support + + def func(lock): + ident = threading.get_ident() + if %s: + # call current_thread() before fork to allocate DummyThread + current = threading.current_thread() + print("current", current.name, type(current).__name__) + print("ident in _active", ident in threading._active) + # flush before fork, so child won't flush it again + sys.stdout.flush() + pid = os.fork() + if pid == 0: + print("current ident", threading.get_ident() == ident) + main = threading.main_thread() + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + print("_dangling", [t.name for t in list(threading._dangling)]) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + try: + threading._shutdown() + os._exit(0) + except: + traceback.print_exc() + sys.stderr.flush() + os._exit(1) + else: + try: + support.wait_process(pid, exitcode=0) + except Exception: + # avoid 'could not acquire lock for + # <_io.BufferedWriter name=''> at interpreter shutdown,' + traceback.print_exc() + sys.stderr.flush() + finally: + lock.release() + + join_lock = _thread.allocate_lock() + join_lock.acquire() + th = _thread.start_new_thread(func, (join_lock,)) + join_lock.acquire() + """ % create_dummy + # "DeprecationWarning: This process is multi-threaded, use of fork() + # may lead to deadlocks in the child" + _, out, err = assert_python_ok("-W", "ignore::DeprecationWarning", "-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err.decode(), "") + self.assertEqual(data, + ("current Dummy-1 _DummyThread\n" if create_dummy else "") + + f"ident in _active {create_dummy!s}\n" + + "current ident True\n" + "main MainThread _MainThread\n" + "main ident True\n" + "current is main True\n" + "_dangling ['MainThread']\n") + + def test_main_thread_after_fork_from_dummy_thread(self, create_dummy=False): + self.test_main_thread_after_fork_from_foreign_thread(create_dummy=True) def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread @@ -747,41 +912,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -829,6 +959,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -856,7 +987,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -947,30 +1077,6 @@ def checker(): self.assertEqual(threading.getprofile(), old_profile) self.assertEqual(sys.getprofile(), old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) - - # unblock the thread and join it - event.set() - thread.join() - - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) - def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called @@ -1048,21 +1154,21 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) class ThreadJoinOnShutdown(BaseTestCase): @@ -1149,6 +1255,11 @@ def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random @@ -1186,6 +1297,30 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with @@ -1324,7 +1459,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.run(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -1355,6 +1490,7 @@ def task(): @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1381,6 +1517,7 @@ def _check_allowed(self, before_start='', *, daemon_allowed=True, daemon=False, ): + import_module("_testinternalcapi") subinterp_code = textwrap.dedent(f""" import test.support import threading @@ -1390,6 +1527,7 @@ def func(): {before_start} t.start() """) + check_multi_interp_extensions = bool(support.Py_GIL_DISABLED) script = textwrap.dedent(f""" import test.support test.support.run_in_subinterp_with_config( @@ -1399,7 +1537,7 @@ def func(): allow_exec=True, allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, - check_multi_interp_extensions=False, + check_multi_interp_extensions={check_multi_interp_extensions}, own_gil=False, ) """) @@ -1655,6 +1793,7 @@ def setUp(self): restore_default_excepthook(self) super().setUp() + @force_not_colorized def test_excepthook(self): with support.captured_output("stderr") as stderr: thread = ThreadRunFail(name="excepthook thread") @@ -1668,6 +1807,7 @@ def test_excepthook(self): self.assertIn('ValueError: run failed', stderr) @support.cpython_only + @force_not_colorized def test_excepthook_thread_None(self): # threading.excepthook called with thread=None: log the thread # identifier in this case. @@ -1884,6 +2024,7 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index 6a53d655015cdb..bf241ada90ea35 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -32,39 +32,28 @@ def handle_signals(sig,frame): # a function that will be spawned as a separate thread. def send_signals(): - os.kill(process_pid, signal.SIGUSR1) - os.kill(process_pid, signal.SIGUSR2) + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) signalled_all.release() @threading_helper.requires_working_threading() -@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): with threading_helper.wait_threads_exit(): # Test signal handling semantics of threads. - # We spawn a thread, have the thread send two signals, and + # We spawn a thread, have the thread send itself two signals, and # wait for it to finish. Check that we got both signals # and that they were run by the main thread. signalled_all.acquire() self.spawnSignallingThread() signalled_all.acquire() - # the signals that we asked the kernel to send - # will come back, but we don't know when. - # (it might even be after the thread exits - # and might be out of order.) If we haven't seen - # the signals yet, send yet another signal and - # wait for it return. - if signal_blackboard[signal.SIGUSR1]['tripped'] == 0 \ - or signal_blackboard[signal.SIGUSR2]['tripped'] == 0: - try: - signal.alarm(1) - signal.pause() - finally: - signal.alarm(0) - self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], thread.get_ident()) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 3b5640abdb6b89..293799ff68ea05 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -43,8 +43,8 @@ class _PyTime(enum.IntEnum): ROUND_UP = 3 # _PyTime_t is int64_t -_PyTime_MIN = -2 ** 63 -_PyTime_MAX = 2 ** 63 - 1 +PyTime_MIN = -2 ** 63 +PyTime_MAX = 2 ** 63 - 1 # Rounding modes supported by PyTime ROUNDING_MODES = ( @@ -277,6 +277,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -299,6 +301,12 @@ def test_strptime_exception_context(self): time.strptime('19', '%Y %') self.assertIs(e.exception.__suppress_context__, True) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') + def test_asctime(self): time.asctime(time.gmtime(self.t)) @@ -511,7 +519,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: @@ -934,7 +942,7 @@ def test_FromSecondsObject(self): _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): - from _testinternalcapi import _PyTime_AsSecondsDouble + from _testcapi import PyTime_AsSecondsDouble def float_converter(ns): if abs(ns) % SEC_TO_NS == 0: @@ -942,15 +950,10 @@ def float_converter(ns): else: return float(ns) / SEC_TO_NS - self.check_int_rounding(lambda ns, rnd: _PyTime_AsSecondsDouble(ns), + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), float_converter, NS_TO_SEC) - # test nan - for time_rnd, _ in ROUNDING_MODES: - with self.assertRaises(TypeError): - _PyTime_AsSecondsDouble(float('nan')) - def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -1009,7 +1012,7 @@ def test_AsTimeval_clamp(self): tv_sec_max = self.time_t_max tv_sec_min = self.time_t_min - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) with decimal.localcontext() as context: context.rounding = decimal.ROUND_CEILING @@ -1028,7 +1031,7 @@ def test_AsTimeval_clamp(self): def test_AsTimespec_clamp(self): from _testinternalcapi import _PyTime_AsTimespec_clamp - for t in (_PyTime_MIN, _PyTime_MAX): + for t in (PyTime_MIN, PyTime_MAX): ts = _PyTime_AsTimespec_clamp(t) tv_sec, tv_nsec = divmod(t, NS_TO_SEC) if self.time_t_max < tv_sec: diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index a37705f0ae6feb..ebb9e00ff91bf0 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -14,7 +14,7 @@ def setUpClass(cls): # Some window managers can maximize new windows. cls.root.wm_state('normal') try: - cls.root.wm_attributes('-zoomed', False) + cls.root.wm_attributes(zoomed=False) except tkinter.TclError: pass diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index 59fe592b492adc..d71a634a767310 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -10,6 +10,11 @@ requires('gui') +EXPECTED_FLOAT_ERRMSG = 'expected floating-point number but got "{}"' +EXPECTED_FLOAT_OR_EMPTY_ERRMSG = 'expected floating-point number (or "" )?but got "{}"' +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + class PackTest(AbstractWidgetTest, unittest.TestCase): test_keys = None @@ -317,7 +322,8 @@ def test_place_configure_x(self): self.assertEqual(f2.place_info()['x'], '-10') self.root.update() self.assertEqual(f2.winfo_x(), 190) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, x='spam') def test_place_configure_y(self): @@ -334,7 +340,8 @@ def test_place_configure_y(self): self.assertEqual(f2.place_info()['y'], '-10') self.root.update() self.assertEqual(f2.winfo_y(), 110) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, y='spam') def test_place_configure_relx(self): @@ -351,8 +358,7 @@ def test_place_configure_relx(self): self.assertEqual(f2.place_info()['relx'], '1') self.root.update() self.assertEqual(f2.winfo_x(), 200) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, relx='spam') def test_place_configure_rely(self): @@ -369,8 +375,7 @@ def test_place_configure_rely(self): self.assertEqual(f2.place_info()['rely'], '1') self.root.update() self.assertEqual(f2.winfo_y(), 120) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, rely='spam') def test_place_configure_anchor(self): @@ -391,7 +396,8 @@ def test_place_configure_width(self): f2.place_configure(width='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(width='abcd') def test_place_configure_height(self): @@ -402,7 +408,8 @@ def test_place_configure_height(self): f2.place_configure(height='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(height='abcd') def test_place_configure_relwidth(self): @@ -413,8 +420,7 @@ def test_place_configure_relwidth(self): f2.place_configure(relwidth='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relwidth='abcd') def test_place_configure_relheight(self): @@ -425,8 +431,7 @@ def test_place_configure_relheight(self): f2.place_configure(relheight='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relheight='abcd') def test_place_configure_bordermode(self): @@ -629,7 +634,8 @@ def test_grid_columnconfigure(self): self.assertEqual(self.root.grid_columnconfigure(0, 'weight'), 4) def test_grid_columnconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, minsize='foo') self.root.grid_columnconfigure(0, minsize=10) self.assertEqual(self.root.grid_columnconfigure(0, 'minsize'), 10) @@ -646,7 +652,8 @@ def test_grid_columnconfigure_weight(self): self.assertEqual(self.root.grid_columnconfigure(0)['weight'], 3) def test_grid_columnconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -683,7 +690,8 @@ def test_grid_rowconfigure(self): self.assertEqual(self.root.grid_rowconfigure(0, 'weight'), 4) def test_grid_rowconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, minsize='foo') self.root.grid_rowconfigure(0, minsize=10) self.assertEqual(self.root.grid_rowconfigure(0, 'minsize'), 10) @@ -700,7 +708,8 @@ def test_grid_rowconfigure_weight(self): self.assertEqual(self.root.grid_rowconfigure(0)['weight'], 3) def test_grid_rowconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -818,9 +827,11 @@ def test_grid_location(self): self.root.grid_location(0) with self.assertRaises(TypeError): self.root.grid_location(0, 0, 0) - with self.assertRaisesRegex(TclError, 'bad screen distance "x"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('x')): self.root.grid_location('x', 'y') - with self.assertRaisesRegex(TclError, 'bad screen distance "y"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('y')): self.root.grid_location('1c', 'y') t = self.root # de-maximize @@ -893,9 +904,5 @@ def test_grid_slaves(self): self.assertEqual(self.root.grid_slaves(row=1, column=1), [d, c]) -tests_gui = ( - PackTest, PlaceTest, GridTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index ef1c99f57c6f47..38371fe00d6eb5 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -302,7 +302,37 @@ def test_copy(self): image2 = image.copy() self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) - self.assertEqual(image.get(4, 6), image.get(4, 6)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = image.copy(from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(3, 2), image.get(3*2+2, 2*2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) def test_subsample(self): image = self.create() @@ -316,6 +346,13 @@ def test_subsample(self): self.assertEqual(image2.height(), 8) self.assertEqual(image2.get(2, 3), image.get(4, 6)) + image2 = image.subsample(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + def test_zoom(self): image = self.create() image2 = image.zoom(2, 3) @@ -330,6 +367,118 @@ def test_zoom(self): self.assertEqual(image2.get(8, 12), image.get(4, 6)) self.assertEqual(image2.get(9, 13), image.get(4, 6)) + image2 = image.zoom(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + def test_copy_replace(self): + image = self.create() + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(0, 0, 100, 50)) + self.assertEqual(image2.width(), 100) + self.assertEqual(image2.height(), 50) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12, 4+8), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12*2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2, 4+8*3), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(3*3, 2*3), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(3*3+2, 2*3+2), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) + + def checkImgTrans(self, image, expected): + actual = {(x, y) + for x in range(image.width()) + for y in range(image.height()) + if image.transparency_get(x, y)} + self.assertEqual(actual, expected) + + def test_copy_replace_compositingrule(self): + image1 = tkinter.PhotoImage(master=self.root, width=2, height=2) + image1.blank() + image1.put('black', to=(0, 0, 2, 2)) + image1.transparency_set(0, 0, True) + + # default compositingrule + image2 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image2.blank() + image2.put('white', to=(0, 0, 2, 2)) + image2.copy_replace(image1, to=(1, 1)) + self.checkImgTrans(image2, {(0, 2), (2, 0)}) + + image3 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image3.blank() + image3.put('white', to=(0, 0, 2, 2)) + image3.copy_replace(image1, to=(1, 1), compositingrule='overlay') + self.checkImgTrans(image3, {(0, 2), (2, 0)}) + + image4 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image4.blank() + image4.put('white', to=(0, 0, 2, 2)) + image4.copy_replace(image1, to=(1, 1), compositingrule='set') + self.checkImgTrans(image4, {(0, 2), (1, 1), (2, 0)}) + def test_put(self): image = self.create() image.put('{red green} {blue yellow}', to=(4, 6)) @@ -356,6 +505,50 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + def test_read(self): + # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 + # the -from option does not work correctly for GIF and PNG files. + # Use the PPM file for this test. + testfile = support.findfile('python.ppm', subdir='tkinterdata') + image = tkinter.PhotoImage(master=self.root, file=testfile) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + self.assertRaises(tkinter.TclError, image2.read, self.testfile, 'ppm') + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root, file=testfile) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + image2.read(testfile, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + def test_write(self): filename = os_helper.TESTFN import locale @@ -367,19 +560,17 @@ def test_write(self): image.write(filename) image2 = tkinter.PhotoImage('::img::test2', master=self.root, - format='ppm', - file=filename) + format='ppm', file=filename) self.assertEqual(str(image2), '::img::test2') self.assertEqual(image2.type(), 'photo') self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) self.assertEqual(image2.get(0, 0), image.get(0, 0)) - self.assertEqual(image2.get(15, 8), image.get(15, 8)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) image.write(filename, format='gif', from_coords=(4, 6, 6, 9)) image3 = tkinter.PhotoImage('::img::test3', master=self.root, - format='gif', - file=filename) + format='gif', file=filename) self.assertEqual(str(image3), '::img::test3') self.assertEqual(image3.type(), 'photo') self.assertEqual(image3.width(), 2) @@ -387,6 +578,67 @@ def test_write(self): self.assertEqual(image3.get(0, 0), image.get(4, 6)) self.assertEqual(image3.get(1, 2), image.get(5, 8)) + image.write(filename, background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', file=filename) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + image.write(filename, grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', file=filename) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_data(self): + image = self.create() + + data = image.data() + self.assertIsInstance(data, tuple) + for row in data: + self.assertIsInstance(row, str) + c = image.get(4, 6) + if not self.wantobjects: + c = tuple(map(int, c.split())) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % c) + + data = image.data('ppm') + image2 = tkinter.PhotoImage('::img::test2', master=self.root, + format='ppm', data=data) + self.assertEqual(str(image2), '::img::test2') + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + data = image.data(format='gif', from_coords=(4, 6, 6, 9)) + image3 = tkinter.PhotoImage('::img::test3', master=self.root, + format='gif', data=data) + self.assertEqual(str(image3), '::img::test3') + self.assertEqual(image3.type(), 'photo') + self.assertEqual(image3.width(), 2) + self.assertEqual(image3.height(), 3) + self.assertEqual(image3.get(0, 0), image.get(4, 6)) + self.assertEqual(image3.get(1, 2), image.get(5, 8)) + + data = image.data('ppm', background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', data=data) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + data = image.data('ppm', grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', data=data) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_transparency(self): image = self.create() self.assertEqual(image.transparency_get(0, 0), True) diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 0ae27610be6078..b0b9ed60040443 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -232,6 +232,46 @@ def callback(): with self.assertRaises(tkinter.TclError): root.tk.call('after', 'info', idle1) + def test_after_info(self): + root = self.root + + # No events. + self.assertEqual(root.after_info(), ()) + + # Add timer. + timer = root.after(1, lambda: 'break') + + # With no parameter, it returns a tuple of the event handler ids. + self.assertEqual(root.after_info(), (timer, )) + root.after_cancel(timer) + + timer1 = root.after(5000, lambda: 'break') + timer2 = root.after(5000, lambda: 'break') + idle1 = root.after_idle(lambda: 'break') + # Only contains new events and not 'timer'. + self.assertEqual(root.after_info(), (idle1, timer2, timer1)) + + # With a parameter returns a tuple of (script, type). + timer1_info = root.after_info(timer1) + self.assertEqual(len(timer1_info), 2) + self.assertEqual(timer1_info[1], 'timer') + idle1_info = root.after_info(idle1) + self.assertEqual(len(idle1_info), 2) + self.assertEqual(idle1_info[1], 'idle') + + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.after_info(timer1) + root.after_cancel(timer2) + with self.assertRaises(tkinter.TclError): + root.after_info(timer2) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.after_info(idle1) + + # No events. + self.assertEqual(root.after_info(), ()) + def test_clipboard(self): root = self.root root.clipboard_clear() @@ -281,6 +321,18 @@ def assertApprox(col1, col2): with self.assertRaises(tkinter.TclError): rgb((111, 78, 55)) + def test_winfo_pathname(self): + t = tkinter.Toplevel(self.root) + w = tkinter.Button(t) + wid = w.winfo_id() + self.assertIsInstance(wid, int) + self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) + self.assertEqual(self.root.winfo_pathname(wid), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + def test_event_repr_defaults(self): e = tkinter.Event() e.serial = 12345 @@ -424,6 +476,766 @@ def test_info_patchlevel(self): self.assertEqual(vi.micro, 0) self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + def test_embedded_null(self): + widget = tkinter.Entry(self.root) + widget.insert(0, 'abc\0def') # ASCII-only + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), 'abc\x00def') + widget.insert(0, '\u20ac\0') # non-ASCII + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), '\u20ac\0abc\x00def') + + +class WmTest(AbstractTkTest, unittest.TestCase): + + def test_wm_attribute(self): + w = self.root + attributes = w.wm_attributes(return_python_dict=True) + self.assertIsInstance(attributes, dict) + attributes2 = w.wm_attributes() + self.assertIsInstance(attributes2, tuple) + self.assertEqual(attributes2[::2], + tuple('-' + k for k in attributes)) + self.assertEqual(attributes2[1::2], tuple(attributes.values())) + # silently deprecated + attributes3 = w.wm_attributes(None) + if self.wantobjects: + self.assertEqual(attributes3, attributes2) + else: + self.assertIsInstance(attributes3, str) + + for name in attributes: + self.assertEqual(w.wm_attributes(name), attributes[name]) + # silently deprecated + for name in attributes: + self.assertEqual(w.wm_attributes('-' + name), attributes[name]) + + self.assertIn('alpha', attributes) + self.assertIn('fullscreen', attributes) + self.assertIn('topmost', attributes) + if w._windowingsystem == "win32": + self.assertIn('disabled', attributes) + self.assertIn('toolwindow', attributes) + self.assertIn('transparentcolor', attributes) + if w._windowingsystem == "aqua": + self.assertIn('modified', attributes) + self.assertIn('notify', attributes) + self.assertIn('titlepath', attributes) + self.assertIn('transparent', attributes) + if w._windowingsystem == "x11": + self.assertIn('type', attributes) + self.assertIn('zoomed', attributes) + + w.wm_attributes(alpha=0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + # silently deprecated + w.wm_attributes('-alpha', 0.5) + self.assertEqual(w.wm_attributes('alpha'), + 0.5 if self.wantobjects else '0.5') + w.wm_attributes(alpha=1.0) + self.assertEqual(w.wm_attributes('alpha'), + 1.0 if self.wantobjects else '1.0') + + +class EventTest(AbstractTkTest, unittest.TestCase): + + def test_focus(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.focus_force() + self.root.update() + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.FocusIn) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, '??') + self.assertEqual(e.y, '??') + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_configure(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.configure(height=120, borderwidth=10) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Configure) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, 150) + self.assertEqual(e.height, 100) + self.assertEqual(e.x, 0) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_key_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('') + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.KeyPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertIsInstance(e.state, int) + self.assertNotEqual(e.state, 0) + self.assertEqual(e.char, 'z') + self.assertIsInstance(e.keycode, int) + self.assertNotEqual(e.keycode, 0) + self.assertEqual(e.keysym, 'z') + self.assertEqual(e.keysym_num, ord('z')) + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, -1 - f.winfo_rootx()) + self.assertEqual(e.y, -1 - f.winfo_rooty()) + self.assertEqual(e.x_root, -1) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + def test_event_generate_enter(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Enter) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertIs(e.focus, False) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, 100 + f.winfo_rootx()) + self.assertEqual(e.y_root, 50 + f.winfo_rooty()) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_button_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.ButtonPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, 1) + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_motion(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Motion) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0x100) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_mouse_wheel(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50, delta=-5) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.MouseWheel) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.time, 0) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, -5) + self.assertEqual(repr(e), '') + + def test_generate_event_virtual_event(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('<>', events.append) + f.focus_force() + + f.event_generate('<>', x=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.VirtualEvent) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 50) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, f.winfo_rootx() + 50) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + +class BindTest(AbstractTkTest, unittest.TestCase): + + def setUp(self): + super().setUp() + root = self.root + self.frame = tkinter.Frame(self.root, class_='Test', + width=150, height=100) + self.frame.pack() + + def assertCommandExist(self, funcid): + self.assertEqual(_info_commands(self.root, funcid), (funcid,)) + + def assertCommandNotExist(self, funcid): + self.assertEqual(_info_commands(self.root, funcid), ()) + + def test_bind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + + funcid = f.bind(event, test1) + self.assertEqual(f.bind(), (event,)) + script = f.bind(event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = f.bind(event, test2, add=True) + script = f.bind(event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + + self.assertRaises(TypeError, f.unbind) + f.unbind(event) + self.assertEqual(f.bind(event), '') + self.assertEqual(f.bind(), ()) + + def test_unbind2(self): + f = self.frame + f.wait_visibility() + f.focus_force() + f.update_idletasks() + event = '' + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): events.append('a') + def test2(e): events.append('b') + def test3(e): events.append('c') + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + funcid3 = f.bind(event, test3, add=True) + events = [] + f.event_generate(event) + self.assertEqual(events, ['a', 'b', 'c']) + + f.unbind(event, funcid2) + script = f.bind(event) + self.assertNotIn(funcid2, script) + self.assertIn(funcid, script) + self.assertIn(funcid3, script) + self.assertEqual(f.bind(), (event,)) + self.assertCommandNotExist(funcid2) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid3) + events = [] + f.event_generate(event) + self.assertEqual(events, ['a', 'c']) + + f.unbind(event, funcid) + f.unbind(event, funcid3) + self.assertEqual(f.bind(event), '') + self.assertEqual(f.bind(), ()) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) + self.assertCommandNotExist(funcid3) + events = [] + f.event_generate(event) + self.assertEqual(events, []) + + # non-idempotent + self.assertRaises(tkinter.TclError, f.unbind, event, funcid2) + + def test_bind_rebind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + script = f.bind(event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = f.bind(event, test3) + script = f.bind(event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_bind_class(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertRaises(TypeError, bind_class) + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + + funcid = bind_class('Test', event, test1) + self.assertEqual(bind_class('Test'), (event,)) + script = bind_class('Test', event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = bind_class('Test', event, test2, add=True) + script = bind_class('Test', event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind_class(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + + funcid = bind_class('Test', event, test1) + funcid2 = bind_class('Test', event, test2, add=True) + + self.assertRaises(TypeError, unbind_class) + self.assertRaises(TypeError, unbind_class, 'Test') + unbind_class('Test', event) + self.assertEqual(bind_class('Test', event), '') + self.assertEqual(bind_class('Test'), ()) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + unbind_class('Test', event) # idempotent + + def test_bind_class_rebind(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = bind_class('Test', event, test1) + funcid2 = bind_class('Test', event, test2, add=True) + script = bind_class('Test', event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = bind_class('Test', event, test3) + script = bind_class('Test', event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + self.assertCommandExist(funcid3) + + def test_bind_all(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + + funcid = bind_all(event, test1) + self.assertIn(event, bind_all()) + script = bind_all(event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = bind_all(event, test2, add=True) + script = bind_all(event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind_all(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + + funcid = bind_all(event, test1) + funcid2 = bind_all(event, test2, add=True) + + unbind_all(event) + self.assertEqual(bind_all(event), '') + self.assertNotIn(event, bind_all()) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + unbind_all(event) # idempotent + + def test_bind_all_rebind(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = bind_all(event, test1) + funcid2 = bind_all(event, test2, add=True) + script = bind_all(event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = bind_all(event, test3) + script = bind_all(event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + self.assertCommandExist(funcid3) + + def _test_tag_bind(self, w): + tag = 'sel' + event = '' + w.pack() + self.assertRaises(TypeError, w.tag_bind) + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + if isinstance(w, tkinter.Text): + self.assertRaises(TypeError, w.tag_bind, tag) + self.assertRaises(TypeError, w.tag_bind, tag, event) + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + self.assertEqual(tag_bind(tag), (event,)) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def _test_tag_unbind(self, w): + tag = 'sel' + event = '' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + + self.assertRaises(TypeError, w.tag_unbind, tag) + w.tag_unbind(tag, event) + self.assertEqual(tag_bind(tag, event), '') + self.assertEqual(tag_bind(tag), ()) + + def _test_tag_bind_rebind(self, w): + tag = 'sel' + event = '' + w.pack() + tag_bind = w._tag_bind if isinstance(w, tkinter.Text) else w.tag_bind + self.assertEqual(tag_bind(tag), ()) + self.assertEqual(tag_bind(tag, event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = w.tag_bind(tag, event, test1) + funcid2 = w.tag_bind(tag, event, test2, add=True) + script = tag_bind(tag, event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = w.tag_bind(tag, event, test3) + script = tag_bind(tag, event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_canvas_tag_bind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind(c) + + def test_canvas_tag_unbind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_unbind(c) + + def test_canvas_tag_bind_rebind(self): + c = tkinter.Canvas(self.frame) + self._test_tag_bind_rebind(c) + + def test_text_tag_bind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind(t) + + def test_text_tag_unbind(self): + t = tkinter.Text(self.frame) + self._test_tag_unbind(t) + + def test_text_tag_bind_rebind(self): + t = tkinter.Text(self.frame) + self._test_tag_bind_rebind(t) + + def test_bindtags(self): + f = self.frame + self.assertEqual(self.root.bindtags(), ('.', 'Tk', 'all')) + self.assertEqual(f.bindtags(), (str(f), 'Test', '.', 'all')) + f.bindtags(('a', 'b c')) + self.assertEqual(f.bindtags(), ('a', 'b c')) + + def test_bind_events(self): + event = '' + root = self.root + t = tkinter.Toplevel(root) + f = tkinter.Frame(t, class_='Test', width=150, height=100) + f.pack() + root.wait_visibility() # needed on Windows + root.update_idletasks() + self.addCleanup(root.unbind_class, 'Test', event) + self.addCleanup(root.unbind_class, 'Toplevel', event) + self.addCleanup(root.unbind_class, 'tag', event) + self.addCleanup(root.unbind_class, 'tag2', event) + self.addCleanup(root.unbind_all, event) + def test(what): + return lambda e: events.append((what, e.widget)) + + root.bind_all(event, test('all')) + root.bind_class('Test', event, test('frame class')) + root.bind_class('Toplevel', event, test('toplevel class')) + root.bind_class('tag', event, test('tag')) + root.bind_class('tag2', event, test('tag2')) + f.bind(event, test('frame')) + t.bind(event, test('toplevel')) + + events = [] + f.event_generate(event) + self.assertEqual(events, [ + ('frame', f), + ('frame class', f), + ('toplevel', f), + ('all', f), + ]) + + events = [] + t.event_generate(event) + self.assertEqual(events, [ + ('toplevel', t), + ('toplevel class', t), + ('all', t), + ]) + + f.bindtags(('tag', 'tag3')) + events = [] + f.event_generate(event) + self.assertEqual(events, [('tag', f)]) + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): @@ -480,5 +1292,9 @@ def test_mainloop(self): self.assertRaises(RuntimeError, tkinter.mainloop) +def _info_commands(widget, pattern=None): + return widget.tk.splitlist(widget.tk.call('info', 'commands', pattern)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index 328e4256ce0711..b26956930d3402 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -10,6 +10,7 @@ class TextTest(AbstractTkTest, unittest.TestCase): def setUp(self): super().setUp() self.text = tkinter.Text(self.root) + self.text.pack() def test_debug(self): text = self.text @@ -41,8 +42,6 @@ def test_search(self): self.assertEqual(text.search('test', '1.0', 'end'), '1.3') def test_count(self): - # XXX Some assertions do not check against the intended result, - # but instead check the current result to prevent regression. text = self.text text.insert('1.0', 'Lorem ipsum dolor sit amet,\n' @@ -53,44 +52,47 @@ def test_count(self): options = ('chars', 'indices', 'lines', 'displaychars', 'displayindices', 'displaylines', 'xpixels', 'ypixels') - if self.wantobjects: - self.assertEqual(len(text.count('1.0', 'end', *options)), 8) - else: - text.count('1.0', 'end', *options) - self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4) - if self.wantobjects else '124 4') - self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3) - if self.wantobjects else '92 3') - self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3) - if self.wantobjects else '-92 -3') - self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0) - if self.wantobjects else '0 0') - self.assertEqual(text.count('1.0', 'end', 'lines'), (4,) - if self.wantobjects else ('4',)) - self.assertEqual(text.count('end', '1.0', 'lines'), (-4,) - if self.wantobjects else ('-4',)) - self.assertEqual(text.count('1.3', '1.5', 'lines'), None - if self.wantobjects else ('0',)) - self.assertEqual(text.count('1.3', '1.3', 'lines'), None - if self.wantobjects else ('0',)) - self.assertEqual(text.count('1.0', 'end'), (124,) # 'indices' by default - if self.wantobjects else ('124',)) + self.assertEqual(len(text.count('1.0', 'end', *options, return_ints=True)), 8) + self.assertEqual(len(text.count('1.0', 'end', *options)), 8) + self.assertEqual(text.count('1.0', 'end', 'chars', 'lines', return_ints=True), + (124, 4)) + self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3)) + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines', return_ints=True), + (-92, -3)) + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3)) + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines', return_ints=True), + (0, 0)) + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0)) + self.assertEqual(text.count('1.0', 'end', 'lines', return_ints=True), 4) + self.assertEqual(text.count('1.0', 'end', 'lines'), (4,)) + self.assertEqual(text.count('end', '1.0', 'lines', return_ints=True), -4) + self.assertEqual(text.count('end', '1.0', 'lines'), (-4,)) + self.assertEqual(text.count('1.3', '1.5', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.5', 'lines'), None) + self.assertEqual(text.count('1.3', '1.3', 'lines', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'lines'), None) + # Count 'indices' by default. + self.assertEqual(text.count('1.0', 'end', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end'), (124,)) + self.assertEqual(text.count('1.0', 'end', 'indices', return_ints=True), 124) + self.assertEqual(text.count('1.0', 'end', 'indices'), (124,)) self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels', return_ints=True), int) self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) - self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int - if self.wantobjects else str) - self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None - if self.wantobjects else '0') - self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2 - if self.wantobjects else '2') - self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None - if self.wantobjects else '0') - self.assertEqual(text.count('1.3', '1.5', 'update'), (2,) - if self.wantobjects else ('2',)) - self.assertEqual(text.count('1.3', '1.3', 'update'), None - if self.wantobjects else ('0',)) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels', return_ints=True), int) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices', return_ints=True), 2) + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None) + self.assertEqual(text.count('1.3', '1.5', 'update', return_ints=True), 2) + self.assertEqual(text.count('1.3', '1.5', 'update'), (2,)) + self.assertEqual(text.count('1.3', '1.3', 'update', return_ints=True), 0) + self.assertEqual(text.count('1.3', '1.3', 'update'), None) if __name__ == "__main__": diff --git a/Lib/test/test_tkinter/test_variables.py b/Lib/test/test_tkinter/test_variables.py index c1d232e2febc7a..def7aec077e800 100644 --- a/Lib/test/test_tkinter/test_variables.py +++ b/Lib/test/test_tkinter/test_variables.py @@ -6,7 +6,7 @@ from tkinter import (Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tcl, TclError) from test.support import ALWAYS_EQ -from test.test_tkinter.support import AbstractDefaultRootTest +from test.test_tkinter.support import AbstractDefaultRootTest, tcl_version class Var(Variable): @@ -112,6 +112,8 @@ def test_initialize(self): self.assertTrue(v.side_effect) def test_trace_old(self): + if tcl_version >= (9, 0): + self.skipTest('requires Tcl version < 9.0') # Old interface v = Variable(self.root) vname = str(v) diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index d3f942db7baf9a..9ea764ca2a39d8 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tk, +from test.test_tkinter.support import (requires_tk, tk_version, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -14,6 +14,9 @@ requires('gui') +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + def float_round(x): return float(round(x)) @@ -58,11 +61,11 @@ def test_configure_visual(self): @add_standard_options(StandardOptionsTests) class ToplevelTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', 'menu', 'padx', 'pady', 'relief', 'screen', - 'takefocus', 'use', 'visual', 'width', + 'takefocus', 'tile', 'use', 'visual', 'width', ) def create(self, **kwargs): @@ -101,10 +104,10 @@ def test_configure_use(self): @add_standard_options(StandardOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', - 'padx', 'pady', 'relief', 'takefocus', 'visual', 'width', + 'padx', 'pady', 'relief', 'takefocus', 'tile', 'visual', 'width', ) def create(self, **kwargs): @@ -141,11 +144,9 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest, IntegerSizeTests): _conv_pixels = False - - def test_configure_highlightthickness(self): - widget = self.create() - self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, -2, '10p') + _clip_highlightthickness = tk_version >= (8, 7) + _clip_pad = tk_version >= (8, 7) + _clip_borderwidth = tk_version >= (8, 7) @add_standard_options(StandardOptionsTests) @@ -277,6 +278,9 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _conv_pixels = round + _clip_highlightthickness = True + _clip_pad = True + _clip_borderwidth = False def create(self, **kwargs): return tkinter.Menubutton(self.root, **kwargs) @@ -290,9 +294,6 @@ def test_configure_height(self): widget = self.create() self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) - test_configure_highlightthickness = \ - StandardOptionsTests.test_configure_highlightthickness - def test_configure_image(self): widget = self.create() image = tkinter.PhotoImage(master=self.root, name='image1') @@ -313,16 +314,6 @@ def test_configure_menu(self): self.checkParam(widget, 'menu', menu, eq=widget_eq) menu.destroy() - def test_configure_padx(self): - widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'padx', -2, expected=0) - - def test_configure_pady(self): - widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'pady', -2, expected=0) - def test_configure_width(self): widget = self.create() self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) @@ -347,7 +338,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'highlightbackground', 'highlightcolor', 'highlightthickness', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'readonlybackground', 'relief', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'readonlybackground', 'relief', 'selectbackground', 'selectborderwidth', 'selectforeground', 'show', 'state', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', @@ -441,8 +433,8 @@ class SpinboxTest(EntryTest, unittest.TestCase): 'increment', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'relief', 'readonlybackground', - 'repeatdelay', 'repeatinterval', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'relief', 'readonlybackground', 'repeatdelay', 'repeatinterval', 'selectbackground', 'selectborderwidth', 'selectforeground', 'state', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', @@ -489,8 +481,12 @@ def test_configure_from(self): widget = self.create() self.checkParam(widget, 'to', 100.0) self.checkFloatParam(widget, 'from', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'from', 200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'from', 200, expected=100) + else: + self.checkInvalidParam( + widget, 'from', 200, + errmsg='-to value must be greater than -from value') def test_configure_increment(self): widget = self.create() @@ -500,8 +496,12 @@ def test_configure_to(self): widget = self.create() self.checkParam(widget, 'from', -100.0) self.checkFloatParam(widget, 'to', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'to', -200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'to', -200, expected=-100) + else: + self.checkInvalidParam( + widget, 'to', -200, + errmsg='-to value must be greater than -from value') def test_configure_values(self): # XXX @@ -660,11 +660,13 @@ def test_configure_tabs(self): widget = self.create() self.checkParam(widget, 'tabs', (10.2, 20.7, '1i', '2i')) self.checkParam(widget, 'tabs', '10.2 20.7 1i 2i', - expected=('10.2', '20.7', '1i', '2i')) + expected=(10.2, 20.7, '1i', '2i') + if get_tk_patchlevel(self.root) >= (8, 6, 14) + else ('10.2', '20.7', '1i', '2i')) self.checkParam(widget, 'tabs', '2c left 4c 6c center', expected=('2c', 'left', '4c', '6c', 'center')) self.checkInvalidParam(widget, 'tabs', 'spam', - errmsg='bad screen distance "spam"') + errmsg=EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')) def test_configure_tabstyle(self): widget = self.create() @@ -858,24 +860,27 @@ def test_create_line(self): def test_create_polygon(self): c = self.create() - i1 = c.create_polygon(20, 30, 40, 50, 60, 10) + tk87 = tk_version >= (8, 7) + # In Tk < 8.7 polygons are filled, but has no outline by default. + # This affects its size, so always explicitly specify outline. + i1 = c.create_polygon(20, 30, 40, 50, 60, 10, outline='red') self.assertEqual(c.coords(i1), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) - self.assertEqual(c.bbox(i1), (19, 9, 61, 51)) + self.assertEqual(c.bbox(i1), (18, 8, 62, 52)) self.assertEqual(c.itemcget(i1, 'joinstyle'), 'round') self.assertEqual(c.itemcget(i1, 'smooth'), '0') self.assertEqual(c.itemcget(i1, 'splinestep'), '12') - i2 = c.create_polygon([21, 31, 41, 51, 61, 11]) + i2 = c.create_polygon([21, 31, 41, 51, 61, 11], outline='red') self.assertEqual(c.coords(i2), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) - self.assertEqual(c.bbox(i2), (20, 10, 62, 52)) + self.assertEqual(c.bbox(i2), (19, 9, 63, 53)) - i3 = c.create_polygon((22, 32), (42, 52), (62, 12)) + i3 = c.create_polygon((22, 32), (42, 52), (62, 12), outline='red') self.assertEqual(c.coords(i3), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) - self.assertEqual(c.bbox(i3), (21, 11, 63, 53)) + self.assertEqual(c.bbox(i3), (20, 10, 64, 54)) - i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)]) + i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)], outline='red') self.assertEqual(c.coords(i4), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) - self.assertEqual(c.bbox(i4), (22, 12, 64, 54)) + self.assertEqual(c.bbox(i4), (21, 11, 65, 55)) self.assertRaises(TclError, c.create_polygon, 20, 30, 60) self.assertRaises(TclError, c.create_polygon, [20, 30, 60]) @@ -999,12 +1004,16 @@ def test_itemconfigure(self): widget.itemconfigure() with self.assertRaisesRegex(TclError, 'bad listbox index "red"'): widget.itemconfigure('red') + if get_tk_patchlevel(self.root) >= (8, 6, 14): + prefix = ('background', '', '', '') + else: + prefix = ('background', 'background', 'Background', '') self.assertEqual(widget.itemconfigure(0, 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) self.assertEqual(widget.itemconfigure('end', 'background'), - ('background', 'background', 'Background', '', 'violet')) + (*prefix, 'violet')) self.assertEqual(widget.itemconfigure('@0,0', 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) d = widget.itemconfigure(0) self.assertIsInstance(d, dict) @@ -1168,18 +1177,16 @@ class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): def create(self, **kwargs): return tkinter.Scrollbar(self.root, **kwargs) - def test_configure_activerelief(self): - widget = self.create() - self.checkReliefParam(widget, 'activerelief') - def test_configure_elementborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, -2, '1m') + self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, '1m') + expected = self._default_pixels if tk_version >= (8, 7) else -2 + self.checkParam(widget, 'elementborderwidth', -2, expected=expected) def test_configure_orient(self): widget = self.create() self.checkEnumParam(widget, 'orient', 'vertical', 'horizontal', - errmsg='bad orientation "{}": must be vertical or horizontal') + fullname='orientation', allow_empty=True) def test_activate(self): sb = self.create() @@ -1250,7 +1257,8 @@ def test_configure_proxyborderwidth(self): @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() - self.checkReliefParam(widget, 'proxyrelief') + self.checkReliefParam(widget, 'proxyrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_sashcursor(self): widget = self.create() @@ -1323,7 +1331,7 @@ def test_paneconfigure_height(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'height', 10, 10) self.check_paneconfigure_bad(p, b, 'height', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) def test_paneconfigure_hide(self): p, b, c = self.create2() @@ -1335,19 +1343,19 @@ def test_paneconfigure_minsize(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'minsize', 10, 10) self.check_paneconfigure_bad(p, b, 'minsize', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_padx(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'padx', 1.3, 1) self.check_paneconfigure_bad(p, b, 'padx', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_pady(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'pady', 1.3, 1) self.check_paneconfigure_bad(p, b, 'pady', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_sticky(self): p, b, c = self.create2() @@ -1368,13 +1376,14 @@ def test_paneconfigure_width(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'width', 10, 10) self.check_paneconfigure_bad(p, b, 'width', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) @add_standard_options(StandardOptionsTests) class MenuTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeborderwidth', 'activeforeground', + 'activerelief', 'background', 'borderwidth', 'cursor', 'disabledforeground', 'font', 'foreground', 'postcommand', 'relief', 'selectcolor', 'takefocus', @@ -1390,6 +1399,8 @@ def test_indexcommand_none(self): i = widget.index('none') self.assertIsNone(i) + test_configure_activerelief = requires_tk(8, 7)(StandardOptionsTests.test_configure_activerelief) + def test_configure_postcommand(self): widget = self.create() self.checkCommandParam(widget, 'postcommand') @@ -1408,14 +1419,10 @@ def test_configure_title(self): def test_configure_type(self): widget = self.create() - opts = ('normal, tearoff, or menubar' - if widget.info_patchlevel() < (8, 7) else - 'menubar, normal, or tearoff') - self.checkEnumParam( - widget, 'type', - 'normal', 'tearoff', 'menubar', - errmsg='bad type "{}": must be ' + opts, - ) + values = ('normal', 'tearoff', 'menubar') + self.checkEnumParam(widget, 'type', *values, + allow_empty=tk_version < (8, 7), + sort=tk_version >= (8, 7)) def test_entryconfigure(self): m1 = self.create() @@ -1461,6 +1468,10 @@ class MessageTest(AbstractWidgetTest, unittest.TestCase): 'takefocus', 'text', 'textvariable', 'width', ) _conv_pad_pixels = False + if tk_version >= (8, 7): + _conv_pixels = False + _clip_pad = tk_version >= (8, 7) + _clip_borderwidth = tk_version >= (8, 7) def create(self, **kwargs): return tkinter.Message(self.root, **kwargs) @@ -1469,6 +1480,26 @@ def test_configure_aspect(self): widget = self.create() self.checkIntegerParam(widget, 'aspect', 250, 0, -300) + def test_configure_padx(self): + widget = self.create() + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', + conv=self._conv_pad_pixels) + expected = self._default_pixels if self._clip_pad else -2 + self.checkParam(widget, 'padx', -2, expected=expected) + + def test_configure_pady(self): + widget = self.create() + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', + conv=self._conv_pad_pixels) + expected = self._default_pixels if self._clip_pad else -2 + self.checkParam(widget, 'pady', -2, expected=expected) + + def test_configure_width(self): + widget = self.create() + self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, 0, '5i') + expected = 0 if tk_version >= (8, 7) else -402 + self.checkParam(widget, 'width', -402, expected=expected) + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): @@ -1479,13 +1510,5 @@ def test_label(self): self._test_widget(tkinter.Label) -tests_gui = ( - ButtonTest, CanvasTest, CheckbuttonTest, EntryTest, - FrameTest, LabelFrameTest,LabelTest, ListboxTest, - MenubuttonTest, MenuTest, MessageTest, OptionMenuTest, - PanedWindowTest, RadiobuttonTest, ScaleTest, ScrollbarTest, - SpinboxTest, TextTest, ToplevelTest, DefaultRootTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index 31f82f459beefd..8ab2f74245095d 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,7 +1,8 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py +import re import tkinter -from test.test_tkinter.support import (AbstractTkTest, tk_version, +from test.test_tkinter.support import (AbstractTkTest, requires_tk, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -9,9 +10,14 @@ _sentinel = object() class AbstractWidgetTest(AbstractTkTest): + _default_pixels = '' if tk_version >= (9, 0) else -1 if tk_version >= (8, 7) else '' _conv_pixels = round _conv_pad_pixels = None _stringify = False + _clip_highlightthickness = True + _clip_pad = False + _clip_borderwidth = False + _allow_empty_justify = False @property def scaling(self): @@ -56,16 +62,13 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, def checkInvalidParam(self, widget, name, value, errmsg=None): orig = widget[name] if errmsg is not None: - errmsg = errmsg.format(value) - with self.assertRaises(tkinter.TclError) as cm: + errmsg = errmsg.format(re.escape(str(value))) + errmsg = fr'\A{errmsg}\Z' + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget[name] = value - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) - with self.assertRaises(tkinter.TclError) as cm: + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget.configure({name: value}) - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) def checkParams(self, widget, name, *values, **kwargs): @@ -74,30 +77,26 @@ def checkParams(self, widget, name, *values, **kwargs): def checkIntegerParam(self, widget, name, *values, **kwargs): self.checkParams(widget, name, *values, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected integer but got ""') - self.checkInvalidParam(widget, name, '10p', - errmsg='expected integer but got "10p"') - self.checkInvalidParam(widget, name, 3.2, - errmsg='expected integer but got "3.2"') + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, '10p', errmsg=errmsg) + self.checkInvalidParam(widget, name, 3.2, errmsg=errmsg) def checkFloatParam(self, widget, name, *values, conv=float, **kwargs): for value in values: self.checkParam(widget, name, value, conv=conv, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected floating-point number but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected floating-point number but got "spam"') + errmsg = 'expected floating-point number but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkBooleanParam(self, widget, name): for value in (False, 0, 'false', 'no', 'off'): self.checkParam(widget, name, value, expected=0) for value in (True, 1, 'true', 'yes', 'on'): self.checkParam(widget, name, value, expected=1) - self.checkInvalidParam(widget, name, '', - errmsg='expected boolean value but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected boolean value but got "spam"') + errmsg = 'expected boolean value but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkColorParam(self, widget, name, *, allow_empty=None, **kwargs): self.checkParams(widget, name, @@ -120,16 +119,24 @@ def command(*args): self.assertTrue(widget[name]) self.checkParams(widget, name, '') - def checkEnumParam(self, widget, name, *values, errmsg=None, **kwargs): + def checkEnumParam(self, widget, name, *values, + errmsg=None, allow_empty=False, fullname=None, + sort=False, **kwargs): self.checkParams(widget, name, *values, **kwargs) if errmsg is None: + if sort: + if values[-1]: + values = tuple(sorted(values)) + else: + values = tuple(sorted(values[:-1])) + ('',) errmsg2 = ' %s "{}": must be %s%s or %s' % ( - name, + fullname or name, ', '.join(values[:-1]), ',' if len(values) > 2 else '', - values[-1]) - self.checkInvalidParam(widget, name, '', - errmsg='ambiguous' + errmsg2) + values[-1] or '""') + if '' not in values and not allow_empty: + self.checkInvalidParam(widget, name, '', + errmsg='ambiguous' + errmsg2) errmsg = 'bad' + errmsg2 self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) @@ -146,20 +153,21 @@ def checkPixelsParam(self, widget, name, *values, conv1 = round self.checkParam(widget, name, value, expected=expected, conv=conv1, **kwargs) - self.checkInvalidParam(widget, name, '6x', - errmsg='bad screen distance "6x"') - self.checkInvalidParam(widget, name, 'spam', - errmsg='bad screen distance "spam"') + errmsg = '(bad|expected) screen distance ((or "" )?but got )?"{}"' + self.checkInvalidParam(widget, name, '6x', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) - def checkReliefParam(self, widget, name): - self.checkParams(widget, name, - 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') - errmsg='bad relief "spam": must be '\ - 'flat, groove, raised, ridge, solid, or sunken' + def checkReliefParam(self, widget, name, *, allow_empty=False): + values = ('flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') + if allow_empty: + values += ('',) + self.checkParams(widget, name, *values) + errmsg = 'bad relief "{}": must be %s, or %s' % ( + ', '.join(values[:-1]), + values[-1] or '""') if tk_version < (8, 6): errmsg = None - self.checkInvalidParam(widget, name, 'spam', - errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -193,6 +201,7 @@ def test_keys(self): aliases = { 'bd': 'borderwidth', 'bg': 'background', + 'bgimg': 'backgroundimage', 'fg': 'foreground', 'invcmd': 'invalidcommand', 'vcmd': 'validatecommand', @@ -235,6 +244,10 @@ def test_configure_activeforeground(self): widget = self.create() self.checkColorParam(widget, 'activeforeground') + def test_configure_activerelief(self): + widget = self.create() + self.checkReliefParam(widget, 'activerelief') + def test_configure_anchor(self): widget = self.create() self.checkEnumParam(widget, 'anchor', @@ -246,6 +259,11 @@ def test_configure_background(self): if 'bg' in self.OPTIONS: self.checkColorParam(widget, 'bg') + @requires_tk(8, 7) + def test_configure_backgroundimage(self): + widget = self.create() + self.checkImageParam(widget, 'backgroundimage') + def test_configure_bitmap(self): widget = self.create() self.checkParam(widget, 'bitmap', 'questhead') @@ -262,9 +280,14 @@ def test_configure_bitmap(self): def test_configure_borderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'borderwidth', - 0, 1.3, 2.6, 6, -2, '10p') + 0, 1.3, 2.6, 6, '10p') + expected = 0 if self._clip_borderwidth else -2 + self.checkParam(widget, 'borderwidth', -2, expected=expected, + conv=self._conv_pixels) if 'bd' in self.OPTIONS: - self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, -2, '10p') + self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'bd', -2, expected=expected, + conv=self._conv_pixels) def test_configure_compound(self): widget = self.create() @@ -287,8 +310,10 @@ def test_configure_font(self): widget = self.create() self.checkParam(widget, 'font', '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') - self.checkInvalidParam(widget, 'font', '', - errmsg='font "" doesn\'t exist') + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + if not is_ttk: + self.checkInvalidParam(widget, 'font', '', + errmsg='font "" doesn\'t exist') def test_configure_foreground(self): widget = self.create() @@ -308,7 +333,8 @@ def test_configure_highlightthickness(self): widget = self.create() self.checkPixelsParam(widget, 'highlightthickness', 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'highlightthickness', -2, expected=0, + expected = 0 if self._clip_highlightthickness else -2 + self.checkParam(widget, 'highlightthickness', -2, expected=expected, conv=self._conv_pixels) def test_configure_image(self): @@ -342,12 +368,11 @@ def test_configure_jump(self): def test_configure_justify(self): widget = self.create() - self.checkEnumParam(widget, 'justify', 'left', 'right', 'center', - errmsg='bad justification "{}": must be ' - 'left, right, or center') - self.checkInvalidParam(widget, 'justify', '', - errmsg='ambiguous justification "": must be ' - 'left, right, or center') + values = ('left', 'right', 'center') + if self._allow_empty_justify: + values += ('',) + self.checkEnumParam(widget, 'justify', *values, + fullname='justification') def test_configure_orient(self): widget = self.create() @@ -356,13 +381,29 @@ def test_configure_orient(self): def test_configure_padx(self): widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, -2, '12m', + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m', conv=self._conv_pad_pixels) + expected = 0 if self._clip_pad else -2 + self.checkParam(widget, 'padx', -2, expected=expected, + conv=self._conv_pad_pixels) def test_configure_pady(self): widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, -2, '12m', + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m', conv=self._conv_pad_pixels) + expected = 0 if self._clip_pad else -2 + self.checkParam(widget, 'pady', -2, expected=expected, + conv=self._conv_pad_pixels) + + @requires_tk(8, 7) + def test_configure_placeholder(self): + widget = self.create() + self.checkParam(widget, 'placeholder', 'xxx') + + @requires_tk(8, 7) + def test_configure_placeholderforeground(self): + widget = self.create() + self.checkColorParam(widget, 'placeholderforeground') def test_configure_relief(self): widget = self.create() @@ -409,13 +450,35 @@ def test_configure_textvariable(self): var = tkinter.StringVar(self.root) self.checkVariableParam(widget, 'textvariable', var) + @requires_tk(8, 7) + def test_configure_tile(self): + widget = self.create() + self.checkBooleanParam(widget, 'tile') + def test_configure_troughcolor(self): widget = self.create() self.checkColorParam(widget, 'troughcolor') def test_configure_underline(self): widget = self.create() - self.checkIntegerParam(widget, 'underline', 0, 1, 10) + self.checkParams(widget, 'underline', 0, 1, 10) + if tk_version >= (8, 7): + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + self.checkParam(widget, 'underline', '', + expected='' if is_ttk else self._default_pixels) + self.checkParam(widget, 'underline', '5+2', + expected='5+2' if is_ttk else 7) + self.checkParam(widget, 'underline', '5-2', + expected='5-2' if is_ttk else 3) + self.checkParam(widget, 'underline', 'end', expected='end') + self.checkParam(widget, 'underline', 'end-2', expected='end-2') + errmsg = (r'bad index "{}": must be integer\?\[\+-\]integer\?, ' + r'end\?\[\+-\]integer\?, or ""') + else: + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, 'underline', '', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', '10p', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', 3.2, errmsg=errmsg) def test_configure_wraplength(self): widget = self.create() @@ -445,7 +508,8 @@ def test_configure_offrelief(self): def test_configure_overrelief(self): widget = self.create() - self.checkReliefParam(widget, 'overrelief') + self.checkReliefParam(widget, 'overrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_selectcolor(self): widget = self.create() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 41b9ebe3374d62..51aeb35f01065a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -615,6 +615,16 @@ def test_string(self): OP '}' (3, 0) (3, 1) FSTRING_MIDDLE '__' (3, 1) (3, 3) FSTRING_END "'" (3, 3) (3, 4) + """) + + self.check_tokenize("""\ + '''Autorzy, którzy tÄ… jednostkÄ™ majÄ… wpisani jako AKTUALNA -- czyli + aktualni pracownicy, obecni pracownicy''' +""", """\ + INDENT ' ' (1, 0) (1, 4) + STRING "'''Autorzy, którzy tÄ… jednostkÄ™ majÄ… wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45) + NEWLINE '\\n' (2, 45) (2, 46) + DEDENT '' (3, 0) (3, 0) """) def test_function(self): @@ -1189,6 +1199,31 @@ def test_closing_parenthesis_from_different_line(self): NAME 'x' (1, 3) (1, 4) """) + def test_multiline_non_ascii_fstring(self): + self.check_tokenize("""\ +a = f''' + Autorzy, którzy tÄ… jednostkÄ™ majÄ… wpisani jako AKTUALNA -- czyli'''""", """\ + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + FSTRING_START "f\'\'\'" (1, 4) (1, 8) + FSTRING_MIDDLE '\\n Autorzy, którzy tÄ… jednostkÄ™ majÄ… wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68) + FSTRING_END "\'\'\'" (2, 68) (2, 71) + """) + + def test_multiline_non_ascii_fstring_with_expr(self): + self.check_tokenize("""\ +f''' + 🔗 This is a test {test_arg1}🔗 +🔗'''""", """\ + FSTRING_START "f\'\'\'" (1, 0) (1, 4) + FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21) + OP '{' (2, 21) (2, 22) + NAME 'test_arg1' (2, 22) (2, 31) + OP '}' (2, 31) (2, 32) + FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1) + FSTRING_END "\'\'\'" (3, 1) (3, 4) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -1867,6 +1902,43 @@ def test_roundtrip(self): " print('Can not import' # comment2\n)" "else: print('Loaded')\n") + self.check_roundtrip("f'\\N{EXCLAMATION MARK}'") + self.check_roundtrip(r"f'\\N{SNAKE}'") + self.check_roundtrip(r"f'\\N{{SNAKE}}'") + self.check_roundtrip(r"f'\N{SNAKE}'") + self.check_roundtrip(r"f'\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'") + + self.check_roundtrip(r"f'\\N{1}'") + self.check_roundtrip(r"f'\\\\N{2}'") + self.check_roundtrip(r"f'\\\\\\N{3}'") + self.check_roundtrip(r"f'\\\\\\\\N{4}'") + + self.check_roundtrip(r"f'\\N{{'") + self.check_roundtrip(r"f'\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\\\N{{'") + cases = [ + """ +if 1: + "foo" +"bar" +""", + """ +if 1: + ("foo" + "bar") +""", + """ +if 1: + "foo" + "bar" +""" ] + for case in cases: + self.check_roundtrip(case) + + def test_continuation(self): # Balancing continuation self.check_roundtrip("a = (3,4, \n" @@ -1901,19 +1973,6 @@ def test_random_files(self): tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # Tokenize is broken on test_pep3131.py because regular expressions are - # broken on the obscure unicode identifiers in it. *sigh* - # With roundtrip extended to test the 5-tuple mode of untokenize, - # 7 more testfiles fail. Remove them also until the failure is diagnosed. - - testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - - # TODO: Remove this once we can untokenize PEP 701 syntax - testfiles.remove(os.path.join(tempdir, "test_fstring.py")) - - for f in ('buffer', 'builtin', 'fileio', 'os', 'platform', 'sys'): - testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 671ec2961e7f8f..0e7ed67de71067 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -14,6 +14,8 @@ @support.requires_zlib() @unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows') +@unittest.skipIf(sys.platform == 'darwin' and sys._framework, + 'not supported for frameworks builds on macOS') @support.skip_if_buildbot('not all buildbots have enough space') # gh-103053: Skip test if Python is built with Profile Guided Optimization # (PGO), since the test is just too slow in this case. diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 00000000000000..df95e6d0068516 --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,78 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support +import sysconfig + +MAKEFILE = sysconfig.get_makefile_filename() + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + + +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result + + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = set([idle_test]) + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all(filename.startswith('.') for filename in files): + continue + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) + used.add(relpath) + + # Don't check the wheel dir when Python is built --with-wheel-pkg-dir + if sysconfig.get_config_var('WHEEL_PKG_DIR'): + test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertSetEqual(unique_test_dirs, used) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) diff --git a/Lib/test/test_tools/test_makeunicodedata.py b/Lib/test/test_tools/test_makeunicodedata.py new file mode 100644 index 00000000000000..f31375117e2e92 --- /dev/null +++ b/Lib/test/test_tools/test_makeunicodedata.py @@ -0,0 +1,122 @@ +import unittest +from test.test_tools import skip_if_missing, imports_under_tool +from test import support +from test.support.hypothesis_helper import hypothesis + +st = hypothesis.strategies +given = hypothesis.given +example = hypothesis.example + + +skip_if_missing("unicode") +with imports_under_tool("unicode"): + from dawg import Dawg, build_compression_dawg, lookup, inverse_lookup + + +@st.composite +def char_name_db(draw, min_length=1, max_length=30): + m = draw(st.integers(min_value=min_length, max_value=max_length)) + names = draw( + st.sets(st.text("abcd", min_size=1, max_size=10), min_size=m, max_size=m) + ) + characters = draw(st.sets(st.characters(), min_size=m, max_size=m)) + return list(zip(names, characters)) + + +class TestDawg(unittest.TestCase): + """Tests for the directed acyclic word graph data structure that is used + to store the unicode character names in unicodedata. Tests ported from PyPy + """ + + def test_dawg_direct_simple(self): + dawg = Dawg() + dawg.insert("a", -4) + dawg.insert("c", -2) + dawg.insert("cat", -1) + dawg.insert("catarr", 0) + dawg.insert("catnip", 1) + dawg.insert("zcatnip", 5) + packed, data, inverse = dawg.finish() + + self.assertEqual(lookup(packed, data, b"a"), -4) + self.assertEqual(lookup(packed, data, b"c"), -2) + self.assertEqual(lookup(packed, data, b"cat"), -1) + self.assertEqual(lookup(packed, data, b"catarr"), 0) + self.assertEqual(lookup(packed, data, b"catnip"), 1) + self.assertEqual(lookup(packed, data, b"zcatnip"), 5) + self.assertRaises(KeyError, lookup, packed, data, b"b") + self.assertRaises(KeyError, lookup, packed, data, b"catni") + self.assertRaises(KeyError, lookup, packed, data, b"catnipp") + + self.assertEqual(inverse_lookup(packed, inverse, -4), b"a") + self.assertEqual(inverse_lookup(packed, inverse, -2), b"c") + self.assertEqual(inverse_lookup(packed, inverse, -1), b"cat") + self.assertEqual(inverse_lookup(packed, inverse, 0), b"catarr") + self.assertEqual(inverse_lookup(packed, inverse, 1), b"catnip") + self.assertEqual(inverse_lookup(packed, inverse, 5), b"zcatnip") + self.assertRaises(KeyError, inverse_lookup, packed, inverse, 12) + + def test_forbid_empty_dawg(self): + dawg = Dawg() + self.assertRaises(ValueError, dawg.finish) + + @given(char_name_db()) + @example([("abc", "a"), ("abd", "b")]) + @example( + [ + ("bab", "1"), + ("a", ":"), + ("ad", "@"), + ("b", "<"), + ("aacc", "?"), + ("dab", "D"), + ("aa", "0"), + ("ab", "F"), + ("aaa", "7"), + ("cbd", "="), + ("abad", ";"), + ("ac", "B"), + ("abb", "4"), + ("bb", "2"), + ("aab", "9"), + ("caaaaba", "E"), + ("ca", ">"), + ("bbaaa", "5"), + ("d", "3"), + ("baac", "8"), + ("c", "6"), + ("ba", "A"), + ] + ) + @example( + [ + ("bcdac", "9"), + ("acc", "g"), + ("d", "d"), + ("daabdda", "0"), + ("aba", ";"), + ("c", "6"), + ("aa", "7"), + ("abbd", "c"), + ("badbd", "?"), + ("bbd", "f"), + ("cc", "@"), + ("bb", "8"), + ("daca", ">"), + ("ba", ":"), + ("baac", "3"), + ("dbdddac", "a"), + ("a", "2"), + ("cabd", "b"), + ("b", "="), + ("abd", "4"), + ("adcbd", "5"), + ("abc", "e"), + ("ab", "1"), + ] + ) + def test_dawg(self, data): + # suppress debug prints + with support.captured_stdout() as output: + # it's enough to build it, building will also check the result + build_compression_dawg(data) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index c1e289bcaff9e5..7ff3fe4091dfa4 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,12 @@ import os from pickle import dump import sys -from test.support import captured_stdout, requires_resource +from test.support import captured_stdout, requires_resource, requires_gil_enabled from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -300,6 +301,7 @@ def test_loop_caller_importing(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -333,6 +335,7 @@ def setUp(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -559,5 +562,29 @@ def test_run_as_module(self): assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + def test_no_source_file(self): + filename = "" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 9bb1786c5472f5..1895c88d23b70d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -8,6 +8,7 @@ import inspect import builtins import unittest +import unittest.mock import re import tempfile import random @@ -20,12 +21,14 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok, assert_python_failure from test.support.import_helper import forget +from test.support import force_not_colorized import json import textwrap import traceback from functools import partial from pathlib import Path +import _colorize MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' @@ -41,6 +44,14 @@ class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. + def setUp(self): + super().setUp() + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False + + def tearDown(self): + super().tearDown() + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -114,6 +125,7 @@ def test_nocaret(self): self.assertEqual(len(err), 3) self.assertEqual(err[1].strip(), "bad syntax") + @force_not_colorized def test_no_caret_with_no_debug_ranges_flag(self): # Make sure that if `-X no_debug_ranges` is used, there are no carets # in the traceback. @@ -215,7 +227,157 @@ def __str__(self): str_name = '.'.join([X.__module__, X.__qualname__]) self.assertEqual(err[0], "%s: %s\n" % (str_name, str_value)) + def test_format_exception_group_without_show_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg) + self.assertEqual(err, ['ExceptionGroup: A (1 sub-exception)\n']) + + def test_format_exception_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + def test_format_base_exception_group(self): + eg = BaseExceptionGroup('A', [BaseException('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'BaseExceptionGroup: A (1 sub-exception)\n', + ' BaseException: B\n', + ]) + + def test_format_exception_group_with_note(self): + exc = ValueError('B') + exc.add_note('Note') + eg = ExceptionGroup('A', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ' Note\n', + ]) + + def test_format_exception_group_explicit_class(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(ExceptionGroup, eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + def test_format_exception_group_multiple_exceptions(self): + eg = ExceptionGroup('A', [ValueError('B'), TypeError('C')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ValueError: B\n', + ' TypeError: C\n', + ]) + + def test_format_exception_group_multiline_messages(self): + eg = ExceptionGroup('A\n1', [ValueError('B\n2')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' ValueError: B\n', + ' 2\n', + ]) + + def test_format_exception_group_multiline2_messages(self): + exc = ValueError('B\n\n2\n') + exc.add_note('\nC\n\n3') + eg = ExceptionGroup('A\n\n1\n', [exc, IndexError('D')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n\n1\n (2 sub-exceptions)\n', + ' ValueError: B\n', + ' \n', + ' 2\n', + ' \n', + ' \n', # first char of `note` + ' C\n', + ' \n', + ' 3\n', # note ends + ' IndexError: D\n', + ]) + + def test_format_exception_group_syntax_error(self): + exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + eg = ExceptionGroup('A\n1', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' File "x.py", line 23\n', + ' bad syntax\n', + ' SyntaxError: error\n', + ]) + + def test_format_exception_group_nested_with_notes(self): + exc = IndexError('D') + exc.add_note('Note\nmultiline') + eg = ExceptionGroup('A', [ + ValueError('B'), + ExceptionGroup('C', [exc, LookupError('E')]), + TypeError('F'), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (3 sub-exceptions)\n', + ' ValueError: B\n', + ' ExceptionGroup: C (2 sub-exceptions)\n', + ' IndexError: D\n', + ' Note\n', + ' multiline\n', + ' LookupError: E\n', + ' TypeError: F\n', + ]) + + def test_format_exception_group_with_tracebacks(self): + def f(): + try: + 1 / 0 + except ZeroDivisionError as e: + return e + + def g(): + try: + raise TypeError('g') + except TypeError as e: + return e + + eg = ExceptionGroup('A', [ + f(), + ExceptionGroup('B', [g()]), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ZeroDivisionError: division by zero\n', + ' ExceptionGroup: B (1 sub-exception)\n', + ' TypeError: g\n', + ]) + + def test_format_exception_group_with_cause(self): + def f(): + try: + try: + 1 / 0 + except ZeroDivisionError: + raise ValueError(0) + except Exception as e: + return e + + eg = ExceptionGroup('A', [f()]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: 0\n', + ]) + @requires_subprocess() + @force_not_colorized def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) @@ -313,6 +475,8 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) expected = [b'Traceback (most recent call last):', b' File "", line 8, in __init__', + b' x = 1 / 0', + b' ^^^^^', b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) @@ -336,7 +500,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -370,16 +534,16 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), ('(exc, /, value=, tb=, ' - 'limit=None, file=None, chain=True)')) + 'limit=None, file=None, chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=)') + '(exc, /, value=, *, show_group=False, **kwargs)') class PurePythonExceptionFormattingMixin: @@ -427,6 +591,7 @@ def f(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' if True: raise ValueError("basic caret tests")\n' ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -445,6 +610,7 @@ def f_with_unicode(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f_with_unicode\n' ' if True: raise ValueError("Ĥellö Wörld")\n' ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -456,13 +622,17 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) @@ -482,9 +652,14 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f_with_multiline\n' ' if True: raise ValueError(\n' - ' ^^^^^^^^^^^^^^^^^' + ' ^^^^^^^^^^^^^^^^^\n' + ' "error over multiple lines"\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ' )\n' + ' ^' ) result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) @@ -513,9 +688,9 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -538,9 +713,12 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' 2 + 1 /\n' - ' ^^^' + ' ~~^\n' + ' 0\n' + ' ~' ) result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) @@ -555,6 +733,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + divisor / 0 + 30\n' ' ~~~~~~~~^~~\n' @@ -572,6 +751,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + áóí / 0 + 30\n' ' ~~~~^~~\n' @@ -589,6 +769,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + divisor // 0 + 30\n' ' ~~~~~~~~^^~~\n' @@ -599,17 +780,103 @@ def f_with_binary_operator(): def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 - b = "" - return ( a ) + b + b = c = "" + return ( a ) +b + c lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) + b\n' - ' ~~~~~~~~~~^~~\n' + ' return ( a ) +b + c\n' + ' ~~~~~~~~~~^~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = b \ + +\ + c # test + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = b \\\n' + ' ~~~~~~\n' + ' +\\\n' + ' ^~\n' + ' c # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_two_char(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = ( + (b # test + + ) \ + # + + << (c # test + \ + ) # test + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_binary_operator\n' + ' (b # test +\n' + ' ~~~~~~~~~~~~\n' + ' ) \\\n' + ' ~~~~\n' + ' # +\n' + ' ~~~\n' + ' << (c # test\n' + ' ^^~~~~~~~~~~~\n' + ' \\\n' + ' ~\n' + ' ) # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_with_unicode(self): + def f_with_binary_operator(): + b = 1 + a = ("ááá" + + "áá") + b + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' a = ("ááá" +\n' + ' ~~~~~~~~\n' + ' "áá") + b\n' + ' ~~~~~~^~~\n' ) result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) @@ -624,6 +891,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " return some_dict['x']['y']['z']\n" ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' @@ -641,6 +909,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " return some_dict['ó']['á']['í']['beta']\n" ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^\n' @@ -659,6 +928,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' ' return b [ a ] + c\n' ' ~~~~~~^^^^^^^^^\n' @@ -666,6 +936,226 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript_multiline(self): + def f_with_subscript(): + bbbbb = {} + ccc = 1 + ddd = 2 + b = bbbbb \ + [ ccc # test + + + ddd \ + + ] # test + return b + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_subscript\n' + ' b = bbbbb \\\n' + ' ~~~~~~~\n' + ' [ ccc # test\n' + ' ^^^^^^^^^^^^^\n' + ' \n' + ' \n' + ' + ddd \\\n' + ' ^^^^^^^^\n' + ' \n' + ' \n' + ' ] # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("x")("y")("z") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("x")("y")("z")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_unicode(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("ó")("á") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("ó")("á")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + def f(a): + raise RuntimeError("fail") + return f ( "x" ) + 2 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return f ( "x" ) + 2\n' + ' ~~~~~~^^^^^^^^^^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_multiline(self): + def f_with_call(): + class C: + def y(self, a): + def f(b): + raise RuntimeError("fail") + return f + def g(x): + return C() + a = (g(1).y)( + 2 + )(3)(4) + return a + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+8}, in f_with_call\n' + ' a = (g(1).y)(\n' + ' ~~~~~~~~~\n' + ' 2\n' + ' ~\n' + ' )(3)(4)\n' + ' ~^^^\n' + f' File "{__file__}", line {lineno_f+4}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines(self): + def f(): + x = 1 + if True: x += ( + "a" + + "a" + ) # test + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' if True: x += (\n' + ' ^^^^^^\n' + ' ...<2 lines>...\n' + ' ) # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_no_caret(self): + def f(): + x = 1 + x += ( + "a" + + "a" + ) + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' x += (\n' + ' ...<2 lines>...\n' + ' )\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_binary_op(self): + def f_with_binary_operator(): + b = 1 + c = "a" + a = ( + b + + b + ) + ( + c + + c + + c + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = (\n' + ' ~\n' + ' b +\n' + ' ~~~\n' + ' b\n' + ' ~\n' + ' ) + (\n' + ' ~~^~~\n' + ' c +\n' + ' ~~~\n' + ' ...<2 lines>...\n' + ' )\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") @@ -682,6 +1172,7 @@ def test_traceback_specialization_with_syntax_error(self): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{TESTFN}", line {lineno_f}, in \n' " 1 $ 0 / 1 / 2\n" ' ^^^^^\n' @@ -704,6 +1195,7 @@ def test_traceback_very_long_line(self): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{TESTFN}", line {lineno_f}, in \n' f' {source}\n' f' {" "*len("if True: ") + "^"*256}\n' @@ -721,6 +1213,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " some_dict['x']['y']['z']\n" ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' @@ -740,6 +1233,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | callable()\n' + f' | ~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' f' | if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -805,6 +1299,7 @@ def g(): pass 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_applydescs + 1}, in applydecs\n' ' @dec_error\n' ' ^^^^^^^^^\n' @@ -823,6 +1318,7 @@ class A: pass 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_applydescs_class + 1}, in applydecs_class\n' ' @dec_error\n' ' ^^^^^^^^^\n' @@ -841,6 +1337,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " .method", " ^^^^^^", @@ -857,6 +1354,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " method", ] @@ -872,6 +1370,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " . method", " ^^^^^^", @@ -887,6 +1386,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", " width", ] @@ -903,6 +1403,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " raise ValueError(width)", ] @@ -921,12 +1422,263 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", - " print(1, ï½—ï½—ï½—(", - " ^^^^", + f" print(1, ï½—ï½—ï½—(", + f" ~~~~~~^", + f" th))", + f" ^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_wide_characters_term_highlight(self): + def f(): + 说明说明 = 1 + şçöğıĤellö = 0 # not wide but still non-ascii + return 说明说明 / şçöğıĤellö + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 3}, in f", + f" return 说明说明 / şçöğıĤellö", + f" ~~~~~~~~~^~~~~~~~~~~~", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_emojis_term_highlight(self): + def f(): + return "✨ðŸ" + func_说明说明("📗🚛", + "📗🚛") + "ðŸ" + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + f' return "✨ðŸ" + func_说明说明("📗🚛",', + f" ^^^^^^^^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_wide_chars_subscript(self): + def f(): + my_dct = { + "✨🚛✨": { + "说明": { + "ðŸðŸðŸ": None + } + } + } + return my_dct["✨🚛✨"]["说明"]["ðŸ"]["说明"]["ðŸðŸ"] + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 8}, in f", + f' return my_dct["✨🚛✨"]["说明"]["ðŸ"]["说明"]["ðŸðŸ"]', + f" ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^", ] self.assertEqual(actual, expected) + def test_memory_error(self): + def f(): + raise MemoryError() + + actual = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f' File "{__file__}", line {self.callable_line}, in get_exception', + ' callable()', + ' ~~~~~~~~^^', + f' File "{__file__}", line {f.__code__.co_firstlineno + 1}, in f', + ' raise MemoryError()'] + self.assertEqual(actual, expected) + + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) @requires_debug_ranges() @@ -970,6 +1722,14 @@ class TracebackFormatMixin: def some_exception(self): raise KeyError('blah') + def _filter_debug_ranges(self, expected): + return [line for line in expected if not set(line.strip()) <= set("^~")] + + def _maybe_filter_debug_ranges(self, expected): + if not self.DEBUG_RANGES: + return self._filter_debug_ranges(expected) + return expected + @cpython_only def check_traceback_format(self, cleanup_func=None): from _testcapi import traceback_print @@ -982,6 +1742,11 @@ def check_traceback_format(self, cleanup_func=None): cleanup_func(tb.tb_next) traceback_fmt = 'Traceback (most recent call last):\n' + \ ''.join(traceback.format_tb(tb)) + # clear caret lines from traceback_fmt since internal API does + # not emit them + traceback_fmt = "\n".join( + self._filter_debug_ranges(traceback_fmt.splitlines()) + ) + "\n" file_ = StringIO() traceback_print(tb, file_) python_fmt = file_.getvalue() @@ -1074,12 +1839,16 @@ def f(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' # XXX: The following line changes depending on whether the tests # are run through the interactive interpreter or with -m # It also varies depending on the platform (stack size) @@ -1088,7 +1857,7 @@ def f(): 'RecursionError: maximum recursion depth exceeded\n' ) - expected = result_f.splitlines() + expected = self._maybe_filter_debug_ranges(result_f.splitlines()) actual = stderr_f.getvalue().splitlines() # Check the output text matches expectations @@ -1105,7 +1874,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -1119,14 +1888,14 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1136,11 +1905,10 @@ def g(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' ' g()\n' + ' ~^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check 2 different repetitive sections @@ -1162,23 +1930,20 @@ def h(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_h+7}, in _check_recursive_traceback_display\n' ' h()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' + ' ~^^\n' ) - expected = (result_h + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((result_h + result_g).splitlines()) actual = stderr_h.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check the boundary conditions. First, test just below the cutoff. @@ -1191,27 +1956,26 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' 'ValueError\n' ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+81}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Second, test just above the cutoff. @@ -1224,14 +1988,14 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1239,13 +2003,12 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+114}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) @requires_debug_ranges() @@ -1626,6 +2389,20 @@ def __repr__(self): err_msg = "b'please do not show me as numbers'" self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + # an exception with a broken __getattr__ raising a non expected error + class BrokenException(Exception): + broken = False + def __getattr__(self, name): + if self.broken: + raise ValueError(f'no {name}') + + e = BrokenException(123) + vanilla = self.get_report(e) + e.broken = True + self.assertEqual( + self.get_report(e), + vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") + def test_exception_with_multiple_notes(self): for e in [ValueError(42), SyntaxError('bad syntax')]: with self.subTest(e=e): @@ -1725,6 +2502,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' f' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' f' | ExceptionGroup: eg (2 sub-exceptions)\n' @@ -1760,6 +2538,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' f' | raise EG("eg2", [ValueError(3), TypeError(4)]) from e\n' f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' @@ -1811,6 +2590,7 @@ def exc(): f'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' f' exception_or_callable()\n' + f' ~~~~~~~~~~~~~~~~~~~~~^^\n' f' File "{__file__}", line {exc.__code__.co_firstlineno + 8}, in exc\n' f' raise ImportError(5)\n' f'ImportError: 5\n') @@ -1857,6 +2637,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 11}, in exc\n' f' | raise EG("top", [VE(5)])\n' f' | ExceptionGroup: top (1 sub-exception)\n' @@ -2016,6 +2797,7 @@ def exc(): expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' f' | raise ExceptionGroup("nested", excs)\n' f' | ExceptionGroup: nested (2 sub-exceptions)\n' @@ -2067,6 +2849,7 @@ def exc(): expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 10}, in exc\n' f' | raise ExceptionGroup("nested", excs)\n' f' | ExceptionGroup: nested (2 sub-exceptions)\n' @@ -2335,7 +3118,7 @@ def test_basics(self): def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) - self.assertEqual(None, f._line) + self.assertEqual(None, f._lines) linecache.lazycache("f", globals()) self.assertEqual( '"""Test cases for traceback module"""', @@ -2452,7 +3235,7 @@ def some_inner(k, v): def test_custom_format_frame(self): class CustomStackSummary(traceback.StackSummary): - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, colorize=False): return f'{frame_summary.filename}:{frame_summary.lineno}' def some_inner(): @@ -2477,7 +3260,7 @@ def g(): tb = g() class Skip_G(traceback.StackSummary): - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, colorize=False): if frame_summary.name == 'g': return None return super().format_frame_summary(frame_summary) @@ -2497,10 +3280,9 @@ def __repr__(self) -> str: raise Exception("Unrepresentable") class TestTracebackException(unittest.TestCase): - - def test_smoke(self): + def do_test_smoke(self, exc, expected_type_str): try: - 1/0 + raise exc except Exception as e: exc_obj = e exc = traceback.TracebackException.from_exception(e) @@ -2510,9 +3292,26 @@ def test_smoke(self): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(expected_type_str, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_smoke_builtin(self): + self.do_test_smoke(ValueError(42), 'ValueError') + + def test_smoke_user_exception(self): + class MyException(Exception): + pass + + if __name__ == '__main__': + expected = ('TestTracebackException.' + 'test_smoke_user_exception..MyException') + else: + expected = ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception..MyException') + self.do_test_smoke(MyException('bad things happened'), expected) + def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -2533,7 +3332,9 @@ def foo(): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_cause(self): @@ -2555,7 +3356,9 @@ def test_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_context(self): @@ -2575,7 +3378,9 @@ def test_context(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_long_context_chain(self): @@ -2620,7 +3425,9 @@ def test_compact_with_cause(self): self.assertEqual(None, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_compact_no_cause(self): @@ -2640,9 +3447,22 @@ def test_compact_no_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_no_save_exc_type(self): + try: + 1/0 + except Exception as e: + exc = e + + te = traceback.TracebackException.from_exception( + exc, save_exc_type=False) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(te.exc_type) + def test_no_refs_to_exception_and_traceback_objects(self): try: 1/0 @@ -2891,6 +3711,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+9}, in _get_exception_group', f' | f()', + f' | ~^^', f' | File "{__file__}", line {lno_f+1}, in f', f' | 1/0', f' | ~^~', @@ -2899,6 +3720,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+13}, in _get_exception_group', f' | g(42)', + f' | ~^^^^', f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ValueError: 42', @@ -2907,6 +3729,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+20}, in _get_exception_group', f' | g(24)', + f' | ~^^^^', f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ValueError: 24', @@ -3062,6 +3885,27 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -3254,6 +4098,17 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -3628,6 +4483,92 @@ def test_levenshtein_distance_short_circuit(self): res3 = traceback._levenshtein_distance(a, b, threshold) self.assertGreater(res3, threshold, msg=(a, b, threshold)) +class TestColorizedTraceback(unittest.TestCase): + def test_colorized_traceback(self): + def foo(*args): + x = {'a':{'b': None}} + y = x['a']['b']['c'] + + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) + + def bar(): + return baz1(1, + 2,3 + ,4) + try: + bar() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + lines = "".join(exc.format(colorize=True)) + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) + self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + + def test_colorized_syntax_error(self): + try: + compile("a $ b", "", "exec") + except SyntaxError as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + actual = "".join(exc.format(colorize=True)) + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + expected = "".join([ + f' File {magenta}""{reset}, line {magenta}1{reset}\n', + f' a {boldr}${reset} b\n', + f' {boldr}^{reset}\n', + f'{boldm}SyntaxError{reset}: {magenta}invalid syntax{reset}\n'] + ) + self.assertIn(expected, actual) + + def test_colorized_traceback_is_the_default(self): + def foo(): + 1/0 + + from _testcapi import exception_print + try: + foo() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + with unittest.mock.patch('_colorize.can_colorize', return_value=True): + exception_print(e) + actual = tbstderr.getvalue().splitlines() + + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + expected = ['Traceback (most recent call last):', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+5}{reset}, in {magenta}test_colorized_traceback_is_the_default{reset}', + f' {red}foo{reset+boldr}(){reset}', + f' {red}~~~{reset+boldr}^^{reset}', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+1}{reset}, in {magenta}foo{reset}', + f' {red}1{reset+boldr}/{reset+red}0{reset}', + f' {red}~{reset+boldr}^{reset+red}~{reset}', + f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] + self.assertEqual(actual, expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index bea124521032d1..5755f7697de91a 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -8,6 +8,7 @@ interpreter_requires_environment) from test import support from test.support import os_helper +from test.support import force_not_colorized try: import _testcapi @@ -938,6 +939,7 @@ def test_env_limit(self): stdout = stdout.rstrip() self.assertEqual(stdout, b'10') + @force_not_colorized def check_env_var_invalid(self, nframe): with support.SuppressCrashReport(): ok, stdout, stderr = assert_python_failure( diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index f9c56ec2357451..9a04a95dc40d65 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -2,6 +2,7 @@ import sys import tkinter from tkinter import ttk +from tkinter import TclError from test import support from test.support import requires from test.test_tkinter.support import AbstractTkTest, get_tk_patchlevel @@ -122,7 +123,6 @@ def test_theme_use(self): self.style.theme_use(curr_theme) - def test_configure_custom_copy(self): style = self.style @@ -176,6 +176,270 @@ def test_map_custom_copy(self): for key, value in default.items(): self.assertEqual(style.map(newname, key), value) + def test_element_options(self): + style = self.style + element_names = style.element_names() + self.assertNotIsInstance(element_names, str) + for name in element_names: + self.assertIsInstance(name, str) + element_options = style.element_options(name) + self.assertNotIsInstance(element_options, str) + for optname in element_options: + self.assertIsInstance(optname, str) + + def test_element_create_errors(self): + style = self.style + with self.assertRaises(TypeError): + style.element_create('plain.newelem') + with self.assertRaisesRegex(TclError, 'No such element type spam'): + style.element_create('plain.newelem', 'spam') + + def test_element_create_from(self): + style = self.style + style.element_create('plain.background', 'from', 'default') + self.assertIn('plain.background', style.element_names()) + style.element_create('plain.arrow', 'from', 'default', 'rightarrow') + self.assertIn('plain.arrow', style.element_names()) + + def test_element_create_from_errors(self): + style = self.style + with self.assertRaises(IndexError): + style.element_create('plain.newelem', 'from') + with self.assertRaisesRegex(TclError, 'theme "spam" doesn\'t exist'): + style.element_create('plain.newelem', 'from', 'spam') + + def test_element_create_image(self): + style = self.style + image = tkinter.PhotoImage(master=self.root, width=12, height=10) + style.element_create('block', 'image', image) + self.assertIn('block', style.element_names()) + + style.layout('TestLabel1', [('block', {'sticky': 'news'})]) + a = ttk.Label(self.root, style='TestLabel1') + a.pack(expand=True, fill='both') + self.assertEqual(a.winfo_reqwidth(), 12) + self.assertEqual(a.winfo_reqheight(), 10) + + imgfile = support.findfile('python.xbm', subdir='tkinterdata') + img1 = tkinter.BitmapImage(master=self.root, file=imgfile, + foreground='yellow', background='blue') + img2 = tkinter.BitmapImage(master=self.root, file=imgfile, + foreground='blue', background='yellow') + img3 = tkinter.BitmapImage(master=self.root, file=imgfile, + foreground='white', background='black') + style.element_create('Button.button', 'image', + img1, ('pressed', img2), ('active', img3), + border=(2, 4), sticky='we') + self.assertIn('Button.button', style.element_names()) + + style.layout('Button', [('Button.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='Button') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 16) + self.assertEqual(b.winfo_reqheight(), 16) + + def test_element_create_image_errors(self): + style = self.style + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + with self.assertRaises(IndexError): + style.element_create('block2', 'image') + with self.assertRaises(TypeError): + style.element_create('block2', 'image', image, 1) + with self.assertRaises(ValueError): + style.element_create('block2', 'image', image, ()) + with self.assertRaisesRegex(TclError, 'Invalid state name'): + style.element_create('block2', 'image', image, ('spam', image)) + with self.assertRaisesRegex(TclError, 'Invalid state name'): + style.element_create('block2', 'image', image, (1, image)) + with self.assertRaises(TypeError): + style.element_create('block2', 'image', image, ('pressed', 1, image)) + with self.assertRaises(TypeError): + style.element_create('block2', 'image', image, (1, 'selected', image)) + with self.assertRaisesRegex(TclError, 'bad option'): + style.element_create('block2', 'image', image, spam=1) + + def test_element_create_vsapi_1(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('smallclose', 'vsapi', 'WINDOW', 19, [ + ('disabled', 4), + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('CloseButton', + [('CloseButton.smallclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 13) + self.assertEqual(b.winfo_reqheight(), 13) + + def test_element_create_vsapi_2(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + def test_element_create_vsapi_3(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('headerclose', 'vsapi', 'EXPLORERBAR', 2, [ + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('Explorer.CloseButton', + [('Explorer.CloseButton.headerclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='Explorer.CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 16) + self.assertEqual(b.winfo_reqheight(), 16) + + def test_theme_create(self): + style = self.style + curr_theme = style.theme_use() + curr_layout = style.layout('TLabel') + style.theme_create('testtheme1') + self.assertIn('testtheme1', style.theme_names()) + + style.theme_create('testtheme2', settings={ + 'elem' : {'element create': ['from', 'default'],}, + 'TLabel' : { + 'configure': {'padding': 10}, + 'layout': [('elem', {'sticky': 'we'})], + }, + }) + self.assertIn('testtheme2', style.theme_names()) + + style.theme_create('testtheme3', 'testtheme2') + self.assertIn('testtheme3', style.theme_names()) + + style.theme_use('testtheme1') + self.assertEqual(style.element_names(), ()) + self.assertEqual(style.layout('TLabel'), curr_layout) + + style.theme_use('testtheme2') + self.assertEqual(style.element_names(), ('elem',)) + self.assertEqual(style.lookup('TLabel', 'padding'), '10') + self.assertEqual(style.layout('TLabel'), [('elem', {'sticky': 'we'})]) + + style.theme_use('testtheme3') + self.assertEqual(style.element_names(), ()) + self.assertEqual(style.lookup('TLabel', 'padding'), '') + self.assertEqual(style.layout('TLabel'), [('elem', {'sticky': 'we'})]) + + style.theme_use(curr_theme) + + def test_theme_create_image(self): + style = self.style + curr_theme = style.theme_use() + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + new_theme = 'testtheme4' + style.theme_create(new_theme, settings={ + 'block' : { + 'element create': ['image', image, {'width': 120, 'height': 100}], + }, + 'TestWidget.block2' : { + 'element create': ['image', image], + }, + 'TestWidget' : { + 'configure': { + 'anchor': 'left', + 'padding': (3, 0, 0, 2), + 'foreground': 'yellow', + }, + 'map': { + 'foreground': [ + ('pressed', 'red'), + ('active', 'disabled', 'blue'), + ], + }, + 'layout': [ + ('TestWidget.block', {'sticky': 'we', 'side': 'left'}), + ('TestWidget.border', { + 'sticky': 'nsw', + 'border': 1, + 'children': [ + ('TestWidget.block2', {'sticky': 'nswe'}) + ] + }) + ], + }, + }) + + style.theme_use(new_theme) + self.assertIn('block', style.element_names()) + self.assertEqual(style.lookup('TestWidget', 'anchor'), 'left') + self.assertEqual(style.lookup('TestWidget', 'padding'), '3 0 0 2') + self.assertEqual(style.lookup('TestWidget', 'foreground'), 'yellow') + self.assertEqual(style.lookup('TestWidget', 'foreground', + ['active']), 'yellow') + self.assertEqual(style.lookup('TestWidget', 'foreground', + ['active', 'pressed']), 'red') + self.assertEqual(style.lookup('TestWidget', 'foreground', + ['active', 'disabled']), 'blue') + self.assertEqual(style.layout('TestWidget'), + [ + ('TestWidget.block', {'side': 'left', 'sticky': 'we'}), + ('TestWidget.border', { + 'sticky': 'nsw', + 'border': '1', + 'children': [('TestWidget.block2', {'sticky': 'nswe'})] + }) + ]) + + b = ttk.Label(self.root, style='TestWidget') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 134) + self.assertEqual(b.winfo_reqheight(), 100) + + style.theme_use(curr_theme) + + def test_theme_create_vsapi(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + curr_theme = style.theme_use() + new_theme = 'testtheme5' + style.theme_create(new_theme, settings={ + 'pin' : { + 'element create': ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]], + }, + 'Explorer.Pin' : { + 'layout': [('Explorer.Pin.pin', {'sticky': 'news'})], + }, + }) + + style.theme_use(new_theme) + self.assertIn('pin', style.element_names()) + self.assertEqual(style.layout('Explorer.Pin'), + [('Explorer.Pin.pin', {'sticky': 'nswe'})]) + + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + style.theme_use(curr_theme) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index fd1a748a498ac5..cb210b7d2fc960 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,8 +5,9 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, - simulate_mouse_click, AbstractDefaultRootTest) +from test.test_tkinter.support import ( + AbstractTkTest, requires_tk, tk_version, get_tk_patchlevel, + simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -27,16 +28,27 @@ def test_configure_class(self): def test_configure_padding(self): widget = self.create() - self.checkParam(widget, 'padding', 0, expected=('0',)) - self.checkParam(widget, 'padding', 5, expected=('5',)) - self.checkParam(widget, 'padding', (5, 6), expected=('5', '6')) + if get_tk_patchlevel(self.root) < (8, 6, 14): + def padding_conv(value): + self.assertIsInstance(value, tuple) + return tuple(map(str, value)) + else: + padding_conv = None + self.checkParam(widget, 'padding', 0, expected=(0,), conv=padding_conv) + self.checkParam(widget, 'padding', 5, expected=(5,), conv=padding_conv) + self.checkParam(widget, 'padding', (5, 6), + expected=(5, 6), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7), - expected=('5', '6', '7')) + expected=(5, 6, 7), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7, 8), - expected=('5', '6', '7', '8')) + expected=(5, 6, 7, 8), conv=padding_conv) self.checkParam(widget, 'padding', ('5p', '6p', '7p', '8p')) self.checkParam(widget, 'padding', (), expected='') + def test_configure_state(self): + widget = self.create() + self.checkParams(widget, 'state', 'active', 'disabled', 'readonly') + def test_configure_style(self): widget = self.create() self.assertEqual(widget['style'], '') @@ -50,6 +62,11 @@ def test_configure_style(self): self.assertEqual(widget2['class'], 'Foo') # XXX + def test_configure_relief(self): + widget = self.create() + self.checkReliefParam(widget, 'relief', + allow_empty=(tk_version >= (8, 7))) + class WidgetTest(AbstractTkTest, unittest.TestCase): """Tests methods available in every ttk widget.""" @@ -150,6 +167,7 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest): + _allow_empty_justify = True def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -165,17 +183,13 @@ def checkImageParam(self, widget, name): errmsg='image "spam" doesn\'t exist') def test_configure_compound(self): - options = 'none text image center top bottom left right'.split() - errmsg = ( - 'bad compound "{}": must be' - f' {", ".join(options[:-1])}, or {options[-1]}' - ) + values = ('none', 'text', 'image', 'center', 'top', 'bottom', 'left', 'right') + if tk_version >= (8, 7): + values += ('',) widget = self.create() - self.checkEnumParam(widget, 'compound', *options, errmsg=errmsg) + self.checkEnumParam(widget, 'compound', *values, allow_empty=True) - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', 'active', 'disabled', 'normal') + test_configure_justify = requires_tk(8, 7)(StandardOptionsTests.test_configure_justify) def test_configure_width(self): widget = self.create() @@ -192,21 +206,19 @@ class LabelTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _conv_pixels = False + _allow_empty_justify = tk_version >= (8, 7) def create(self, **kwargs): return ttk.Label(self.root, **kwargs) - def test_configure_font(self): - widget = self.create() - self.checkParam(widget, 'font', - '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') + test_configure_justify = StandardOptionsTests.test_configure_justify @add_standard_options(StandardTtkOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', 'default', - 'image', 'padding', 'state', 'style', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -216,7 +228,9 @@ def create(self, **kwargs): def test_configure_default(self): widget = self.create() - self.checkEnumParam(widget, 'default', 'normal', 'active', 'disabled') + values = ('normal', 'active', 'disabled') + self.checkEnumParam(widget, 'default', *values, + sort=tk_version >= (8, 7)) def test_invoke(self): success = [] @@ -229,7 +243,7 @@ def test_invoke(self): class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'offvalue', 'onvalue', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', @@ -268,7 +282,10 @@ def cb_test(): cbtn['command'] = '' res = cbtn.invoke() - self.assertFalse(str(res)) + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) @@ -285,9 +302,29 @@ def test_unique_variables(self): b.pack() buttons.append(b) variables = [str(b['variable']) for b in buttons] - print(variables) self.assertEqual(len(set(variables)), 4, variables) + def test_unique_variables2(self): + buttons = [] + f = ttk.Frame(self.root) + f.pack() + f = ttk.Frame(self.root) + f.pack() + for j in 'AB': + b = tkinter.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + # Should be larger than the number of all previously created + # tkinter.Checkbutton widgets: + for j in range(100): + b = ttk.Checkbutton(f, text=str(j)) + b.pack() + buttons.append(b) + names = [str(b) for b in buttons] + self.assertEqual(len(set(names)), len(buttons), names) + variables = [str(b['variable']) for b in buttons] + self.assertEqual(len(set(variables)), len(buttons), variables) + @add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): @@ -295,6 +332,7 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', ) @@ -317,11 +355,6 @@ def test_configure_show(self): self.checkParam(widget, 'show', '') self.checkParam(widget, 'show', ' ') - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', - 'disabled', 'normal', 'readonly') - def test_configure_validate(self): widget = self.create() self.checkEnumParam(widget, 'validate', @@ -422,7 +455,8 @@ class ComboboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'height', 'invalidcommand', - 'justify', 'postcommand', 'show', 'state', 'style', + 'justify', 'placeholder', 'placeholderforeground', 'postcommand', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'values', 'width', 'xscrollcommand', @@ -486,7 +520,7 @@ def check_get_current(getval, currval): self.assertEqual(self.combo.get(), getval) self.assertEqual(self.combo.current(), currval) - self.assertEqual(self.combo['values'], '') + self.assertIn(self.combo['values'], ((), '')) check_get_current('', -1) self.checkParam(self.combo, 'values', 'mon tue wed thur', @@ -611,8 +645,14 @@ def test_insert(self): child2 = ttk.Label(self.root) child3 = ttk.Label(self.root) - self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + if tk_version >= (8, 7): + self.paned.insert(0, child) + self.assertEqual(self.paned.panes(), (str(child),)) + self.paned.forget(0) + else: + self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + self.assertEqual(self.paned.panes(), ()) self.paned.insert('end', child2) self.paned.insert(0, child) self.assertEqual(self.paned.panes(), (str(child), str(child2))) @@ -676,7 +716,7 @@ def test_sashpos(self): class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'value', 'variable', 'width', @@ -715,7 +755,10 @@ def cb_test(): cbtn2['command'] = '' res = cbtn2.invoke() - self.assertEqual(str(res), '') + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(conv(cbtn2['value']), myvar.get()) self.assertEqual(myvar.get(), @@ -727,7 +770,7 @@ def cb_test(): class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'compound', 'cursor', 'direction', - 'image', 'menu', 'padding', 'state', 'style', + 'image', 'justify', 'menu', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -735,10 +778,11 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): def create(self, **kwargs): return ttk.Menubutton(self.root, **kwargs) - def test_direction(self): + def test_configure_direction(self): widget = self.create() - self.checkEnumParam(widget, 'direction', - 'above', 'below', 'left', 'right', 'flush') + values = ('above', 'below', 'left', 'right', 'flush') + self.checkEnumParam(widget, 'direction', *values, + sort=tk_version >= (8, 7)) def test_configure_menu(self): widget = self.create() @@ -751,7 +795,7 @@ def test_configure_menu(self): class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'from', 'length', - 'orient', 'style', 'takefocus', 'to', 'value', 'variable', + 'orient', 'state', 'style', 'takefocus', 'to', 'value', 'variable', ) _conv_pixels = False default_orient = 'horizontal' @@ -773,6 +817,8 @@ def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 130, 131.2, 135.6, '5i') + test_configure_state = requires_tk(8, 6, 9)(StandardTtkOptionsTests.test_configure_state) + def test_configure_to(self): widget = self.create() self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=False) @@ -856,16 +902,28 @@ def test_set(self): @add_standard_options(StandardTtkOptionsTests) class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( - 'class', 'cursor', 'orient', 'length', - 'mode', 'maximum', 'phase', + 'anchor', 'class', 'cursor', 'font', 'foreground', 'justify', + 'orient', 'length', + 'mode', 'maximum', 'phase', 'text', 'wraplength', 'style', 'takefocus', 'value', 'variable', ) _conv_pixels = False + _allow_empty_justify = True default_orient = 'horizontal' def create(self, **kwargs): return ttk.Progressbar(self.root, **kwargs) + @requires_tk(8, 7) + def test_configure_anchor(self): + widget = self.create() + self.checkEnumParam(widget, 'anchor', + 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center', '') + + test_configure_font = requires_tk(8, 7)(StandardOptionsTests.test_configure_font) + test_configure_foreground = requires_tk(8, 7)(StandardOptionsTests.test_configure_foreground) + test_configure_justify = requires_tk(8, 7)(StandardTtkOptionsTests.test_configure_justify) + def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 100.1, 56.7, '2i') @@ -882,11 +940,15 @@ def test_configure_phase(self): # XXX pass + test_configure_text = requires_tk(8, 7)(StandardOptionsTests.test_configure_text) + def test_configure_value(self): widget = self.create() self.checkFloatParam(widget, 'value', 150.2, 77.7, 0, -10, conv=False) + test_configure_wraplength = requires_tk(8, 7)(StandardOptionsTests.test_configure_wraplength) + @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') @@ -901,11 +963,14 @@ def create(self, **kwargs): return ttk.Scrollbar(self.root, **kwargs) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_standard_options(PixelSizeTests if tk_version >= (8, 7) else IntegerSizeTests, + StandardTtkOptionsTests) class NotebookTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', ) + if tk_version >= (8, 7): + _conv_pixels = False def setUp(self): super().setUp() @@ -1024,7 +1089,11 @@ def test_insert(self): self.nb.insert(self.child1, child3) self.assertEqual(self.nb.tabs(), (str(child3), ) + tabs) self.nb.forget(child3) - self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) + if tk_version >= (8, 7): + self.nb.insert(2, child3) + self.assertEqual(self.nb.tabs(), (*tabs, str(child3))) + else: + self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) self.assertRaises(tkinter.TclError, self.nb.insert, -1, child3) # bad inserts @@ -1116,7 +1185,9 @@ class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'command', 'cursor', 'exportselection', 'font', 'foreground', 'format', 'from', 'increment', - 'invalidcommand', 'justify', 'show', 'state', 'style', + 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', 'width', 'wrap', 'xscrollcommand', ) @@ -1290,8 +1361,9 @@ def test_configure_values(self): class TreeviewTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'columns', 'cursor', 'displaycolumns', - 'height', 'padding', 'selectmode', 'show', - 'style', 'takefocus', 'xscrollcommand', 'yscrollcommand', + 'height', 'padding', 'selectmode', 'selecttype', 'show', 'striped', + 'style', 'takefocus', 'titlecolumns', 'titleitems', + 'xscrollcommand', 'yscrollcommand', ) def setUp(self): @@ -1306,7 +1378,8 @@ def test_configure_columns(self): self.checkParam(widget, 'columns', 'a b c', expected=('a', 'b', 'c')) self.checkParam(widget, 'columns', ('a', 'b', 'c')) - self.checkParam(widget, 'columns', '') + self.checkParam(widget, 'columns', '', + expected=() if tk_version >= (8, 7) else '') def test_configure_displaycolumns(self): widget = self.create() @@ -1318,11 +1391,12 @@ def test_configure_displaycolumns(self): expected=('#all',)) self.checkParam(widget, 'displaycolumns', (2, 1, 0)) self.checkInvalidParam(widget, 'displaycolumns', ('a', 'b', 'd'), - errmsg='Invalid column index d') + errmsg='Invalid column index "?d"?') + errmsg = 'Column index "?{}"? out of bounds' self.checkInvalidParam(widget, 'displaycolumns', (1, 2, 3), - errmsg='Column index 3 out of bounds') + errmsg=errmsg.format(3)) self.checkInvalidParam(widget, 'displaycolumns', (1, -2), - errmsg='Column index -2 out of bounds') + errmsg=errmsg.format(-2)) def test_configure_height(self): widget = self.create() @@ -1334,6 +1408,11 @@ def test_configure_selectmode(self): self.checkEnumParam(widget, 'selectmode', 'none', 'browse', 'extended') + @requires_tk(8, 7) + def test_configure_selecttype(self): + widget = self.create() + self.checkEnumParam(widget, 'selecttype', 'item', 'cell') + def test_configure_show(self): widget = self.create() self.checkParam(widget, 'show', 'tree headings', @@ -1343,6 +1422,23 @@ def test_configure_show(self): self.checkParam(widget, 'show', 'tree', expected=('tree',)) self.checkParam(widget, 'show', 'headings', expected=('headings',)) + @requires_tk(8, 7) + def test_configure_striped(self): + widget = self.create() + self.checkBooleanParam(widget, 'striped') + + @requires_tk(8, 7) + def test_configure_titlecolumns(self): + widget = self.create() + self.checkIntegerParam(widget, 'titlecolumns', 0, 1, 5) + self.checkInvalidParam(widget, 'titlecolumns', -2) + + @requires_tk(8, 7) + def test_configure_titleitems(self): + widget = self.create() + self.checkIntegerParam(widget, 'titleitems', 0, 1, 5) + self.checkInvalidParam(widget, 'titleitems', -2) + def test_bbox(self): self.tv.pack() self.assertEqual(self.tv.bbox(''), '') @@ -1859,13 +1955,5 @@ def test_label(self): self._test_widget(ttk.Label) -tests_gui = ( - ButtonTest, CheckbuttonTest, ComboboxTest, EntryTest, - FrameTest, LabelFrameTest, LabelTest, MenubuttonTest, - NotebookTest, PanedWindowTest, ProgressbarTest, - RadiobuttonTest, ScaleTest, ScrollbarTest, SeparatorTest, - SizegripTest, SpinboxTest, TreeviewTest, WidgetTest, DefaultRootTest, - ) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 96dc179a69ecac..e6525c4d6c6982 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -179,7 +179,7 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), - ("test ", ())) + ("test", ())) # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) @@ -203,17 +203,19 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for vsapi self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b'), - ("a b ", ())) + ('a', 'b', ('', 1), ())) # now with a state spec with multiple states self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b', 'c')), ("a b {a b} c", ())) + [('a', 'b', 'c')]), ('a', 'b', ('a b', 'c'), ())) # state spec and option self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b'), opt='x'), ("a b a b", ("-opt", "x"))) + [('a', 'b')], opt='x'), ('a', 'b', ('a', 'b'), ("-opt", "x"))) # format returned values as a tcl script # state spec with a multivalue and an option self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', - ('a', 'b', [1, 2]), opt='x'), ("{a b {a b} {1 2}}", "-opt x")) + opt='x'), ("a b {{} 1}", "-opt x")) + self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', + [('a', 'b', [1, 2])], opt='x'), ("a b {{a b} {1 2}}", "-opt x")) # Testing type = from # from type expects at least a type name @@ -222,9 +224,9 @@ def test_format_elemcreate(self): self.assertEqual(ttk._format_elemcreate('from', False, 'a'), ('a', ())) self.assertEqual(ttk._format_elemcreate('from', False, 'a', 'b'), - ('a', ('b', ))) + ('a', ('b',))) self.assertEqual(ttk._format_elemcreate('from', True, 'a', 'b'), - ('{a}', 'b')) + ('a', 'b')) def test_format_layoutlist(self): @@ -326,6 +328,22 @@ def test_script_from_settings(self): "ttk::style element create thing image {name {state1 state2} val} " "-opt {3 2m}") + vsapi = {'pin': {'element create': + ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]]}} + self.assertEqual(ttk._script_from_settings(vsapi), + "ttk::style element create pin vsapi EXPLORERBAR 3 {" + "{pressed !selected} 3 " + "{active !selected} 2 " + "{pressed selected} 6 " + "{active selected} 5 " + "selected 4 " + "{} 1} ") def test_tclobj_to_py(self): self.assertEqual( diff --git a/Lib/test/test_tty.py b/Lib/test/test_tty.py index af20864aac361e..4cb730c226f134 100644 --- a/Lib/test/test_tty.py +++ b/Lib/test/test_tty.py @@ -19,7 +19,6 @@ def setUp(self): self.addCleanup(termios.tcsetattr, self.fd, termios.TCSAFLUSH, self.mode) def check_cbreak(self, mode): - self.assertEqual(mode[0] & termios.ICRNL, 0) self.assertEqual(mode[3] & termios.ECHO, 0) self.assertEqual(mode[3] & termios.ICANON, 0) self.assertEqual(mode[6][termios.VMIN], 1) @@ -56,6 +55,14 @@ def test_cfmakecbreak(self): self.assertEqual(mode[2], self.mode[2]) self.assertEqual(mode[4], self.mode[4]) self.assertEqual(mode[5], self.mode[5]) + mode[tty.IFLAG] |= termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, termios.ICRNL, + msg="ICRNL should not be cleared by cbreak") + mode[tty.IFLAG] &= ~termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, 0, + msg="ICRNL should not be set by cbreak") def test_setraw(self): mode0 = termios.tcgetattr(self.fd) @@ -74,6 +81,9 @@ def test_setcbreak(self): self.assertEqual(mode1, mode0) mode2 = termios.tcgetattr(self.fd) self.check_cbreak(mode2) + ICRNL = termios.ICRNL + self.assertEqual(mode2[tty.IFLAG] & ICRNL, mode0[tty.IFLAG] & ICRNL, + msg="ICRNL should not be altered by cbreak") mode3 = tty.setcbreak(self.fd, termios.TCSANOW) self.assertEqual(mode3, mode2) tty.setcbreak(self.stream) diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 8f0a998e1f3dc1..49d6aa810304fb 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -2,7 +2,7 @@ import types import unittest from test.support import check_syntax_error, run_code -from test import mod_generics_cache +from test.typinganndata import mod_generics_cache from typing import Callable, TypeAliasType, TypeVar, get_args @@ -328,3 +328,22 @@ def test_pickling_local(self): with self.subTest(thing=thing, proto=proto): with self.assertRaises(pickle.PickleError): pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..a9be1f5aa84681 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,6 +1,12 @@ import textwrap +import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + +VALUE = 1 +FORWARDREF = 2 +SOURCE = 3 + class TypeAnnotationTests(unittest.TestCase): @@ -48,6 +54,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -105,6 +112,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -112,11 +126,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -212,3 +225,189 @@ def test_match(self): case 0: x: int = 1 """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(2) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(SOURCE) + with self.assertRaises(NotImplementedError): + annotate(None) + self.assertEqual(annotate(VALUE), {"x": int}) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 72587ecc11b6f3..89632a3abebfb5 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,15 +1,20 @@ """ Tests for the internal type cache in CPython. """ import unittest +import dis from test import support -from test.support import import_helper +from test.support import import_helper, requires_specialization try: from sys import _clear_type_cache except ImportError: _clear_type_cache = None # Skip this test if the _testcapi module isn't available. -type_get_version = import_helper.import_module('_testcapi').type_get_version -type_assign_version = import_helper.import_module('_testcapi').type_assign_version +_testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") +type_get_version = _testcapi.type_get_version +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe +type_assign_version = _testcapi.type_assign_version +type_modified = _testcapi.type_modified @support.cpython_only @@ -56,6 +61,214 @@ class C: self.assertNotEqual(type_get_version(C), 0) self.assertNotEqual(type_get_version(C), c_ver) + def test_type_assign_specific_version(self): + """meta-test for type_assign_specific_version_unsafe""" + class C: + pass + + type_assign_version(C) + orig_version = type_get_version(C) + if orig_version == 0: + self.skipTest("Could not assign a valid type version") + + type_modified(C) + type_assign_specific_version_unsafe(C, orig_version + 5) + type_assign_version(C) # this should do nothing + + new_version = type_get_version(C) + self.assertEqual(new_version, orig_version + 5) + + _clear_type_cache() + + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + + def test_119462(self): + + class Holder: + value = None + + @classmethod + def set_value(cls): + cls.value = object() + + class HolderSub(Holder): + pass + + for _ in range(1050): + Holder.set_value() + HolderSub.value + +@support.cpython_only +@requires_specialization +class TypeCacheWithSpecializationTests(unittest.TestCase): + def tearDown(self): + _clear_type_cache() + + def _assign_valid_version_or_skip(self, type_): + type_modified(type_) + type_assign_version(type_) + if type_get_version(type_) == 0: + self.skipTest("Could not assign valid type version") + + def _no_more_versions(self, user_type): + type_modified(user_type) + for _ in range(1001): + type_assign_specific_version_unsafe(user_type, 1000_000_000) + type_assign_specific_version_unsafe(user_type, 0) + self.assertEqual(type_get_version(user_type), 0) + + def _all_opnames(self, func): + return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) + + def _check_specialization(self, func, arg, opname, *, should_specialize): + for _ in range(100): + func(arg) + + if should_specialize: + self.assertNotIn(opname, self._all_opnames(func)) + else: + self.assertIn(opname, self._all_opnames(func)) + + def test_class_load_attr_specialization_user_type(self): + class A: + def foo(self): + pass + + self._assign_valid_version_or_skip(A) + + def load_foo_1(type_): + type_.foo + + self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) + del load_foo_1 + + self._no_more_versions(A) + + def load_foo_2(type_): + return type_.foo + + self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + + def test_class_load_attr_specialization_static_type(self): + self._assign_valid_version_or_skip(str) + self._assign_valid_version_or_skip(bytes) + + def get_capitalize_1(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) + self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + del get_capitalize_1 + + # Permanently overflow the static type version counter, and force str and bytes + # to have tp_version_tag == 0 + for _ in range(2**16): + type_modified(str) + type_assign_version(str) + type_modified(bytes) + type_assign_version(bytes) + + self.assertEqual(type_get_version(str), 0) + self.assertEqual(type_get_version(bytes), 0) + + def get_capitalize_2(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_2, str, "LOAD_ATTR", should_specialize=False) + self.assertEqual(get_capitalize_2(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_2(bytes)(b'hello'), b'Hello') + + def test_property_load_attr_specialization_user_type(self): + class G: + @property + def x(self): + return 9 + + self._assign_valid_version_or_skip(G) + + def load_x_1(instance): + instance.x + + self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) + del load_x_1 + + self._no_more_versions(G) + + def load_x_2(instance): + instance.x + + self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + + def test_store_attr_specialization_user_type(self): + class B: + __slots__ = ("bar",) + + self._assign_valid_version_or_skip(B) + + def store_bar_1(type_): + type_.bar = 10 + + self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) + del store_bar_1 + + self._no_more_versions(B) + + def store_bar_2(type_): + type_.bar = 10 + + self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + + def test_class_call_specialization_user_type(self): + class F: + def __init__(self): + pass + + self._assign_valid_version_or_skip(F) + + def call_class_1(type_): + type_() + + self._check_specialization(call_class_1, F, "CALL", should_specialize=True) + del call_class_1 + + self._no_more_versions(F) + + def call_class_2(type_): + type_() + + self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + + def test_to_bool_specialization_user_type(self): + class H: + pass + + self._assign_valid_version_or_skip(H) + + def to_bool_1(instance): + not instance + + self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) + del to_bool_1 + + self._no_more_versions(H) + + def to_bool_2(instance): + not instance + + self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 9a11fab237235e..ee8939f62d082c 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -66,6 +66,14 @@ def foo(): pass """ +parenthesized_withstmt = """\ +with (a as b): # type: int + pass + +with (a, b): # type: int + pass +""" + vardecl = """\ a = 0 # type: int """ @@ -300,6 +308,14 @@ def test_withstmt(self): tree = self.classic_parse(withstmt) self.assertEqual(tree.body[0].type_comment, None) + def test_parenthesized_withstmt(self): + for tree in self.parse_all(parenthesized_withstmt): + self.assertEqual(tree.body[0].type_comment, "int") + self.assertEqual(tree.body[1].type_comment, "int") + tree = self.classic_parse(parenthesized_withstmt) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + def test_vardecl(self): for tree in self.parse_all(vardecl): self.assertEqual(tree.body[0].type_comment, "int") diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 25ee188731f31f..bf1a34b9fc82b3 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -6,7 +6,7 @@ import weakref from test.support import requires_working_socket, check_syntax_error, run_code -from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -412,6 +412,14 @@ def test_comprehension_02(self): func, = T.__bound__ self.assertEqual(func(), 1) + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + def test_gen_exp_in_nested_class(self): code = """ from test.test_type_params import make_base @@ -436,9 +444,11 @@ class C[T]: class Inner[U](make_base(T for _ in (1,)), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_listcomp_in_nested_class(self): code = """ @@ -464,9 +474,11 @@ class C[T]: class Inner[U](make_base([T for _ in (1,)]), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_gen_exp_in_generic_method(self): code = """ @@ -475,29 +487,86 @@ class C[T]: def meth[U](x: (T for _ in (1,)), y: T): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") def test_nested_scope_in_generic_alias(self): code = """ - class C[T]: + T = "global" + class C: T = "class" {} """ - error_cases = [ - "type Alias1[T] = lambda: T", - "type Alias2 = lambda: T", - "type Alias3[T] = (T for _ in (1,))", - "type Alias4 = (T for _ in (1,))", - "type Alias5[T] = [T for _ in (1,)]", - "type Alias6 = [T for _ in (1,)]", + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", ] - for case in error_cases: + for case in cases: with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - r"Cannot use [a-z]+ in annotation scope within class scope"): - run_code(code.format(case)) + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") + + def test_lambda_in_alias_in_class(self): + code = """ + T = "global" + class C: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertEqual(C.Alias.__value__(), "global") + + def test_lambda_in_alias_in_generic_class(self): + code = """ + class C[T]: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertIs(C.Alias.__value__(), C.__type_params__[0]) + + def test_lambda_in_generic_alias_in_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + T = U = "global" + class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertEqual(U, "global") + + def test_lambda_in_generic_alias_in_generic_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + class C[T, U]: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertIs(U, C.__type_params__[1]) + + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) def make_base(arg): @@ -535,10 +604,12 @@ class Foo[T: Foo, U: (Foo, Foo)]: self.assertEqual(type_params[0].__name__, "T") self.assertIs(type_params[0].__bound__, Foo) self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) self.assertEqual(type_params[1].__name__, "U") self.assertIs(type_params[1].__bound__, None) self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) def test_evaluation_error(self): class Foo[T: Undefined, U: (Undefined,)]: @@ -549,6 +620,8 @@ class Foo[T: Undefined, U: (Undefined,)]: type_params[0].__bound__ self.assertEqual(type_params[0].__constraints__, ()) self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) with self.assertRaises(NameError): type_params[1].__constraints__ @@ -708,6 +781,31 @@ class D[U](T): self.assertIn(int, C.D.__bases__) self.assertIs(C.D.x, str) + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + class TypeParamsManglingTest(unittest.TestCase): def test_mangling(self): class Foo[__T]: @@ -730,6 +828,100 @@ def meth[__U](self, arg: __T, arg2: __U): self.assertEqual(Foo.Alias.__value__, (T, V)) + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + class TypeParamsComplexCallsTest(unittest.TestCase): def test_defaults(self): @@ -1102,3 +1294,103 @@ class Inner[U](T): """ with self.assertRaises(RuntimeError): run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index da32c4ea6477ce..38a98828085e2f 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,8 +1,8 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only +from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS import collections.abc -from collections import namedtuple +from collections import namedtuple, UserDict import copy import _datetime import gc @@ -10,6 +10,7 @@ import pickle import locale import sys +import textwrap import types import unittest.mock import weakref @@ -598,6 +599,8 @@ def test_slot_wrapper_types(self): self.assertIsInstance(object.__lt__, types.WrapperDescriptorType) self.assertIsInstance(int.__lt__, types.WrapperDescriptorType) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_dunder_get_signature(self): sig = inspect.signature(object.__init__.__get__) self.assertEqual(list(sig.parameters), ["instance", "owner"]) @@ -711,6 +714,26 @@ def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) self.assertEqual(hash(int | str), hash(typing.Union[int, str])) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual((A | B).__args__, (A, B)) + union1 = A | B + with self.assertRaises(TypeError): + hash(union1) + + union2 = int | B + with self.assertRaises(TypeError): + hash(union2) + + union3 = A | int + with self.assertRaises(TypeError): + hash(union3) + def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): with self.subTest(x=x): @@ -1733,21 +1756,50 @@ class Model(metaclass=ModelBase): class SimpleNamespaceTests(unittest.TestCase): def test_constructor(self): - ns1 = types.SimpleNamespace() - ns2 = types.SimpleNamespace(x=1, y=2) - ns3 = types.SimpleNamespace(**dict(x=1, y=2)) + def check(ns, expected): + self.assertEqual(len(ns.__dict__), len(expected)) + self.assertEqual(vars(ns), expected) + # check order + self.assertEqual(list(vars(ns).items()), list(expected.items())) + for name in expected: + self.assertEqual(getattr(ns, name), expected[name]) + + check(types.SimpleNamespace(), {}) + check(types.SimpleNamespace(x=1, y=2), {'x': 1, 'y': 2}) + check(types.SimpleNamespace(**dict(x=1, y=2)), {'x': 1, 'y': 2}) + check(types.SimpleNamespace({'x': 1, 'y': 2}, x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace([['x', 1], ['y', 2]], x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace(UserDict({'x': 1, 'y': 2}), x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace({'x': 1, 'y': 2}), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([['x', 1], ['y', 2]]), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([], x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace({}, x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace([]), {}) + check(types.SimpleNamespace({}), {}) with self.assertRaises(TypeError): - types.SimpleNamespace(1, 2, 3) + types.SimpleNamespace([], []) # too many positional arguments with self.assertRaises(TypeError): - types.SimpleNamespace(**{1: 2}) - - self.assertEqual(len(ns1.__dict__), 0) - self.assertEqual(vars(ns1), {}) - self.assertEqual(len(ns2.__dict__), 2) - self.assertEqual(vars(ns2), {'y': 2, 'x': 1}) - self.assertEqual(len(ns3.__dict__), 2) - self.assertEqual(vars(ns3), {'y': 2, 'x': 1}) + types.SimpleNamespace(1) # not a mapping or iterable + with self.assertRaises(TypeError): + types.SimpleNamespace([1]) # non-iterable + with self.assertRaises(ValueError): + types.SimpleNamespace([['x']]) # not a pair + with self.assertRaises(ValueError): + types.SimpleNamespace([['x', 'y', 'z']]) + with self.assertRaises(TypeError): + types.SimpleNamespace(**{1: 2}) # non-string key + with self.assertRaises(TypeError): + types.SimpleNamespace({1: 2}) + with self.assertRaises(TypeError): + types.SimpleNamespace([[1, 2]]) + with self.assertRaises(TypeError): + types.SimpleNamespace(UserDict({1: 2})) + with self.assertRaises(TypeError): + types.SimpleNamespace([[[], 2]]) # non-hashable key def test_unbound(self): ns1 = vars(types.SimpleNamespace()) @@ -2261,5 +2313,73 @@ def coro(): 'close', 'throw'})) +class FunctionTests(unittest.TestCase): + def test_function_type_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + func = types.FunctionType( + ex.__code__, {}, "func", (1, 2), None, {'c': 3}, + ) + + self.assertEqual(func(), 6) + self.assertEqual(func.__defaults__, (1, 2)) + self.assertEqual(func.__kwdefaults__, {'c': 3}) + + func = types.FunctionType( + ex.__code__, {}, "func", None, None, None, + ) + self.assertEqual(func.__defaults__, None) + self.assertEqual(func.__kwdefaults__, None) + + def test_function_type_wrong_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + with self.assertRaisesRegex(TypeError, 'arg 4'): + types.FunctionType( + ex.__code__, {}, "func", 1, None, {'c': 3}, + ) + with self.assertRaisesRegex(TypeError, 'arg 6'): + types.FunctionType( + ex.__code__, {}, "func", None, None, 3, + ) + + +class SubinterpreterTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + global interpreters + try: + from test.support import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + import test.support.interpreters.channels + + @cpython_only + def test_slot_wrappers(self): + rch, sch = interpreters.channels.create() + + # For now it's sufficient to check int.__str__. + # See https://github.com/python/cpython/issues/117482 + # and https://github.com/python/cpython/pull/117660. + script = textwrap.dedent(''' + text = repr(int.__str__) + sch.send_nowait(text) + ''') + + exec(script) + expected = rch.recv() + + interp = interpreters.create() + interp.exec('from test.support import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + results = rch.recv() + + self.assertEqual(results, expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1a1e0a259fd042..a931da55908236 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2,15 +2,15 @@ import collections import collections.abc from collections import defaultdict -from functools import lru_cache, wraps +from functools import lru_cache, wraps, reduce import gc import inspect import itertools +import operator import pickle import re import sys -import warnings -from unittest import TestCase, main, skipUnless, skip +from unittest import TestCase, main, skip from unittest.mock import patch from copy import copy, deepcopy @@ -31,23 +31,22 @@ from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, NotRequired, Required, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs, NoDefault import abc import textwrap import typing import weakref import types -from test.support import import_helper, captured_stderr, cpython_only -from test import mod_generics_cache -from test import _typed_dict_helper +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper +from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' @@ -141,6 +140,26 @@ class MockSomething(Something, Mock): pass self.assertIsInstance(ms, Something) self.assertIsInstance(ms, Mock) + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) + def test_cannot_instantiate(self): with self.assertRaises(TypeError): Any() @@ -561,6 +580,191 @@ def test_constructor(self): self.assertIs(T.__bound__, None) +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) + + class A(Generic[T]): ... + Alias = Optional[T] + + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) + + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) + + class A(Generic[P]): ... + Alias = typing.Callable[P, None] + + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) + + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) + + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) + + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] + + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) + + with self.assertRaises(TypeError): + class X(Generic[*Ts, T]): ... + + with self.assertRaises(TypeError): + class Y(Generic[*Ts_default, T]): ... + + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) + + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) + + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) + + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) + + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) + + def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: """Renders templates with possible combinations of replacements. @@ -959,6 +1163,38 @@ def foo(**kwargs: Unpack[Movie]): ... self.assertEqual(repr(foo.__annotations__['kwargs']), f"typing.Unpack[{__name__}.Movie]") + def test_builtin_tuple(self): + Ts = TypeVarTuple("Ts") + + class Old(Generic[*Ts]): ... + class New[*Ts]: ... + + PartOld = Old[int, *Ts] + self.assertEqual(PartOld[str].__args__, (int, str)) + self.assertEqual(PartOld[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[Tuple[str]]].__args__, (int, str)) + + PartNew = New[int, *Ts] + self.assertEqual(PartNew[str].__args__, (int, str)) + self.assertEqual(PartNew[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) + + def test_unpack_wrong_type(self): + Ts = TypeVarTuple("Ts") + class Gen[*Ts]: ... + PartGen = Gen[int, *Ts] + + bad_unpack_param = re.escape("Unpack[...] must be used with a tuple type") + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[list[int]]] + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[List[int]]] + + class TypeVarTupleTests(BaseTestCase): def assertEndsWith(self, string, tail): @@ -1771,6 +2007,26 @@ def test_union_union(self): v = Union[u, Employee] self.assertEqual(v, Union[int, float, Employee]) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual(Union[A, B].__args__, (A, B)) + union1 = Union[A, B] + with self.assertRaises(TypeError): + hash(union1) + + union2 = Union[int, B] + with self.assertRaises(TypeError): + hash(union2) + + union3 = Union[A, int] + with self.assertRaises(TypeError): + hash(union3) + def test_repr(self): self.assertEqual(repr(Union), 'typing.Union') u = Union[Employee, int] @@ -2011,13 +2267,13 @@ def test_callable_instance_type_error(self): def f(): pass with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], None]) + isinstance(f, Callable[[], None]) with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], Any]) + isinstance(f, Callable[[], Any]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], None]) + isinstance(None, Callable[[], None]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], Any]) + isinstance(None, Callable[[], Any]) def test_repr(self): Callable = self.Callable @@ -3450,8 +3706,8 @@ def meth(self): pass self.assertNotIn("__protocol_attrs__", vars(NonP)) self.assertNotIn("__protocol_attrs__", vars(NonPR)) - self.assertNotIn("__callable_proto_members_only__", vars(NonP)) - self.assertNotIn("__callable_proto_members_only__", vars(NonPR)) + self.assertNotIn("__non_callable_proto_members__", vars(NonP)) + self.assertNotIn("__non_callable_proto_members__", vars(NonPR)) self.assertEqual(get_protocol_members(P), {"x"}) self.assertEqual(get_protocol_members(PR), {"meth"}) @@ -3467,7 +3723,7 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( @@ -3535,13 +3791,26 @@ def __subclasshook__(cls, other): def test_issubclass_fails_correctly(self): @runtime_checkable - class P(Protocol): + class NonCallableMembers(Protocol): x = 1 + class NotRuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + + @runtime_checkable + class RuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + class C: pass - with self.assertRaisesRegex(TypeError, r"issubclass\(\) arg 1 must be a class"): - issubclass(C(), P) + # These three all exercise different code paths, + # but should result in the same error message: + for protocol in NonCallableMembers, NotRuntimeCheckable, RuntimeCheckable: + with self.subTest(proto_name=protocol.__name__): + with self.assertRaisesRegex( + TypeError, r"issubclass\(\) arg 1 must be a class" + ): + issubclass(C(), protocol) def test_defining_generic_protocols(self): T = TypeVar('T') @@ -4093,6 +4362,56 @@ def method(self) -> None: ... self.assertIsInstance(Foo(), ProtocolWithMixedMembers) self.assertNotIsInstance(42, ProtocolWithMixedMembers) + def test_protocol_issubclass_error_message(self): + @runtime_checkable + class Vec2D(Protocol): + x: float + y: float + + def square_norm(self) -> float: + return self.x ** 2 + self.y ** 2 + + self.assertEqual(Vec2D.__protocol_attrs__, {'x', 'y', 'square_norm'}) + expected_error_message = ( + "Protocols with non-method members don't support issubclass()." + " Non-method members: 'x', 'y'." + ) + with self.assertRaisesRegex(TypeError, re.escape(expected_error_message)): + issubclass(int, Vec2D) + + def test_nonruntime_protocol_interaction_with_evil_classproperty(self): + class classproperty: + def __get__(self, instance, type): + raise RuntimeError("NO") + + class Commentable(Protocol): + evil = classproperty() + + # recognised as a protocol attr, + # but not actually accessed by the protocol metaclass + # (which would raise RuntimeError) for non-runtime protocols. + # See gh-113320 + self.assertEqual(get_protocol_members(Commentable), {"evil"}) + + def test_runtime_protocol_interaction_with_evil_classproperty(self): + class CustomError(Exception): pass + + class classproperty: + def __get__(self, instance, type): + raise CustomError + + with self.assertRaises(TypeError) as cm: + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() + + exc = cm.exception + self.assertEqual( + exc.args[0], + "Failed to determine whether protocol member 'evil' is a method member" + ) + self.assertIs(type(exc.__cause__), CustomError) + class GenericTests(BaseTestCase): @@ -4262,6 +4581,16 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -4529,6 +4858,88 @@ def f(x: X): ... {'x': list[list[ForwardRef('X')]]} ) + def test_pep695_generic_class_with_future_annotations(self): + original_globals = dict(ann_module695.__dict__) + + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + # should not have changed as a result of the get_type_hints() calls! + self.assertEqual(ann_module695.__dict__, original_globals) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + hints_for_C = get_type_hints(ann_module695.C) + self.assertEqual( + set(hints_for_C.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.generic_function_2).values()), + set(ann_module695.generic_function_2.__type_params__) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + hints_for_generic_method = get_type_hints(ann_module695.D.generic_method) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + hints_for_generic_method, + {"x": params["Foo"], "y": params["Bar"], "return": types.NoneType} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.D.generic_method_2).values()), + set(ann_module695.D.generic_method_2.__type_params__) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = ann_module695.nested() + + self.assertEqual( + set(results.hints_for_E.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.hints_for_E_meth.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.hints_for_generic_func.values()), + set(results.generic_func.__type_params__) + ) + def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... @@ -4563,8 +4974,6 @@ def test_fail_with_bare_union(self): List[Union] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar[int]] with self.assertRaises(TypeError): List[ClassVar[int]] @@ -4859,6 +5268,75 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -5028,6 +5506,7 @@ def test_subclass_special_form(self): Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): with self.assertRaisesRegex( @@ -5366,10 +5845,8 @@ def some(self): self.assertFalse(hasattr(WithOverride.some, "__override__")) def test_multiple_decorators(self): - import functools - def with_wraps(f): # similar to `lru_cache` definition - @functools.wraps(f) + @wraps(f) def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper @@ -5623,10 +6100,11 @@ def fun(x: a): pass def cmp(o1, o2): return o1 == o2 - r1 = namespace1() - r2 = namespace2() - self.assertIsNot(r1, r2) - self.assertRaises(RecursionError, cmp, r1, r2) + with infinite_recursion(25): + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertRaises(RecursionError, cmp, r1, r2) def test_union_forward_recursion(self): ValueList = List['Value'] @@ -5707,6 +6185,12 @@ def foo(a: 'Node[T'): with self.assertRaises(SyntaxError): get_type_hints(foo) + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[''] + def test_name_error(self): def foo(a: 'Noode[T]'): @@ -5848,16 +6332,6 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): depr_msg = ( "'typing.no_type_check_decorator' is deprecated " @@ -5910,6 +6384,42 @@ def test_or(self): self.assertEqual("x" | X, Union["x", X]) +class InternalsTests(BaseTestCase): + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + + @lru_cache() def cached_func(x, y): return 3 * x + y @@ -6015,8 +6525,6 @@ def test_overload_registry_repeated(self): self.assertEqual(list(get_overloads(impl)), overloads) -# Definitions needed for features introduced in Python 3.6 - from test.typinganndata import ( ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ) @@ -6184,7 +6692,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6202,7 +6710,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) @@ -6576,6 +7084,7 @@ class C(Generic[T]): pass self.assertEqual(get_args(NotRequired[int]), (int,)) self.assertEqual(get_args(TypeAlias), ()) self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) Ts = TypeVarTuple('Ts') self.assertEqual(get_args(Ts), ()) self.assertEqual(get_args((*Ts,)[0]), (Ts,)) @@ -6609,24 +7118,16 @@ def test_iterator(self): self.assertNotIsInstance(42, typing.Iterator) def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -6688,16 +7189,6 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) - def test_bytestring(self): - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(b'', typing.ByteString) - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(bytearray(b''), typing.ByteString) - with self.assertWarns(DeprecationWarning): - class Foo(typing.ByteString): ... - with self.assertWarns(DeprecationWarning): - class Bar(typing.ByteString, typing.Awaitable): ... - def test_list(self): self.assertIsSubclass(list, typing.List) @@ -6891,6 +7382,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -6900,10 +7402,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -6990,9 +7491,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) @@ -7077,6 +7577,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -7088,11 +7597,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -7520,6 +8035,83 @@ class GenericNamedTuple(NamedTuple, Generic[T]): self.assertEqual(CallNamedTuple.__orig_bases__, (NamedTuple,)) + def test_setname_called_on_values_in_class_dictionary(self): + class Vanilla: + def __set_name__(self, owner, name): + self.name = name + + class Foo(NamedTuple): + attr = Vanilla() + + foo = Foo() + self.assertEqual(len(foo), 0) + self.assertNotIn('attr', Foo._fields) + self.assertIsInstance(foo.attr, Vanilla) + self.assertEqual(foo.attr.name, "attr") + + class Bar(NamedTuple): + attr: Vanilla = Vanilla() + + bar = Bar() + self.assertEqual(len(bar), 1) + self.assertIn('attr', Bar._fields) + self.assertIsInstance(bar.attr, Vanilla) + self.assertEqual(bar.attr.name, "attr") + + def test_setname_raises_the_same_as_on_other_classes(self): + class CustomException(BaseException): pass + + class Annoying: + def __set_name__(self, owner, name): + raise CustomException + + annoying = Annoying() + + with self.assertRaises(CustomException) as cm: + class NormalClass: + attr = annoying + normal_exception = cm.exception + + with self.assertRaises(CustomException) as cm: + class NamedTupleClass(NamedTuple): + attr = annoying + namedtuple_exception = cm.exception + + self.assertIs(type(namedtuple_exception), CustomException) + self.assertIs(type(namedtuple_exception), type(normal_exception)) + + self.assertEqual(len(namedtuple_exception.__notes__), 1) + self.assertEqual( + len(namedtuple_exception.__notes__), len(normal_exception.__notes__) + ) + + expected_note = ( + "Error calling __set_name__ on 'Annoying' instance " + "'attr' in 'NamedTupleClass'" + ) + self.assertEqual(namedtuple_exception.__notes__[0], expected_note) + self.assertEqual( + namedtuple_exception.__notes__[0], + normal_exception.__notes__[0].replace("NormalClass", "NamedTupleClass") + ) + + def test_strange_errors_when_accessing_set_name_itself(self): + class CustomException(Exception): pass + + class Meta(type): + def __getattribute__(self, attr): + if attr == "__set_name__": + raise CustomException + return object.__getattribute__(self, attr) + + class VeryAnnoying(metaclass=Meta): pass + + very_annoying = VeryAnnoying() + + with self.assertRaises(CustomException): + class Foo(NamedTuple): + attr = very_annoying + class TypedDictTests(BaseTestCase): def test_basics_functional_syntax(self): @@ -7677,6 +8269,46 @@ class Cat(Animal): 'voice': str, }) + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) @@ -8053,6 +8685,69 @@ class T4(TypedDict, Generic[S]): pass self.assertEqual(klass.__optional_keys__, set()) self.assertIsInstance(klass(), dict) + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: ReadOnly[int] + + class Child2(Base2): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + class RequiredTests(BaseTestCase): @@ -8268,6 +8963,76 @@ def test_flatten(self): self.assertEqual(A.__metadata__, (4, 5)) self.assertEqual(A.__origin__, int) + def test_deduplicate_from_union(self): + # Regular: + self.assertEqual(get_args(Annotated[int, 1] | int), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], int]), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[int, 2] | int), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[int, 2], int]), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[str, 1] | int), + (Annotated[int, 1], Annotated[str, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[str, 1], int]), + (Annotated[int, 1], Annotated[str, 1], int)) + + # Duplicates: + self.assertEqual(Annotated[int, 1] | Annotated[int, 1] | int, + Annotated[int, 1] | int) + self.assertEqual(Union[Annotated[int, 1], Annotated[int, 1], int], + Union[Annotated[int, 1], int]) + + # Unhashable metadata: + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[int, set()] | int), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[int, set()], int]), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[str, {}] | int), + (str, Annotated[int, {}], Annotated[str, {}], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[str, {}], int]), + (str, Annotated[int, {}], Annotated[str, {}], int)) + + self.assertEqual(get_args(Annotated[int, 1] | str | Annotated[str, {}] | int), + (Annotated[int, 1], str, Annotated[str, {}], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], str, Annotated[str, {}], int]), + (Annotated[int, 1], str, Annotated[str, {}], int)) + + import dataclasses + @dataclasses.dataclass + class ValueRange: + lo: int + hi: int + v = ValueRange(1, 2) + self.assertEqual(get_args(Annotated[int, v] | None), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Union[Annotated[int, v], None]), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Optional[Annotated[int, v]]), + (Annotated[int, v], types.NoneType)) + + # Unhashable metadata duplicated: + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + Annotated[int, {}] | int) + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + int | Annotated[int, {}]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[Annotated[int, {}], int]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[int, Annotated[int, {}]]) + + def test_order_in_union(self): + expr1 = Annotated[int, 1] | str | Annotated[str, {}] | int + for args in itertools.permutations(get_args(expr1)): + with self.subTest(args=args): + self.assertEqual(expr1, reduce(operator.or_, args)) + + expr2 = Union[Annotated[int, 1], str, Annotated[str, {}], int] + for args in itertools.permutations(get_args(expr2)): + with self.subTest(args=args): + self.assertEqual(expr2, Union[args]) + def test_specialize(self): L = Annotated[List[T], "my decoration"] LI = Annotated[List[int], "my decoration"] @@ -8288,6 +9053,16 @@ def test_hash_eq(self): {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, {Annotated[int, 4, 5], Annotated[T, 4, 5]} ) + # Unhashable `metadata` raises `TypeError`: + a1 = Annotated[int, []] + with self.assertRaises(TypeError): + hash(a1) + + class A: + __hash__ = None + a2 = Annotated[int, A()] + with self.assertRaises(TypeError): + hash(a2) def test_instantiate(self): class C: @@ -8313,6 +9088,17 @@ def test_instantiate_generic(self): self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): @@ -8344,6 +9130,34 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] + + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) + def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): @@ -8543,6 +9357,40 @@ class X(Annotated[int, (1, 10)]): ... self.assertEqual(X.__mro__, (X, int, object), "Annotated should be transparent.") + def test_annotated_cached_with_types(self): + class A(str): ... + class B(str): ... + + field_a1 = Annotated[str, A("X")] + field_a2 = Annotated[str, B("X")] + a1_metadata = field_a1.__metadata__[0] + a2_metadata = field_a2.__metadata__[0] + + self.assertIs(type(a1_metadata), A) + self.assertEqual(a1_metadata, A("X")) + self.assertIs(type(a2_metadata), B) + self.assertEqual(a2_metadata, B("X")) + self.assertIsNot(type(a1_metadata), type(a2_metadata)) + + field_b1 = Annotated[str, A("Y")] + field_b2 = Annotated[str, B("Y")] + b1_metadata = field_b1.__metadata__[0] + b2_metadata = field_b2.__metadata__[0] + + self.assertIs(type(b1_metadata), A) + self.assertEqual(b1_metadata, A("Y")) + self.assertIs(type(b2_metadata), B) + self.assertEqual(b2_metadata, B("Y")) + self.assertIsNot(type(b1_metadata), type(b2_metadata)) + + field_c1 = Annotated[int, 1] + field_c2 = Annotated[int, 1.0] + field_c3 = Annotated[int, True] + + self.assertIs(type(field_c1.__metadata__[0]), int) + self.assertIs(type(field_c2.__metadata__[0]), float) + self.assertIs(type(field_c3.__metadata__[0]), bool) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -9087,6 +9935,56 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) @@ -9102,7 +10000,6 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -9136,7 +10033,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -9146,7 +10043,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -9186,6 +10083,7 @@ def test_special_attrs(self): typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', typing.Union: 'Union', typing.Self: 'Self', @@ -9200,6 +10098,7 @@ def test_special_attrs(self): typing.Literal[True, 2]: 'Literal', typing.Optional[Any]: 'Optional', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) @@ -9386,6 +10285,45 @@ class CustomerModel(ModelBase, init=False): self.assertIsInstance(CustomerModel, Decorated) +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + type(NoDefault)(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo + + class AllTests(BaseTestCase): """Tests for __all__.""" @@ -9465,5 +10403,11 @@ def test_is_not_instance_of_iterable(self): self.assertNotIsInstance(type_to_test, collections.abc.Iterable) +def load_tests(loader, tests, pattern): + import doctest + tests.addTests(doctest.DocTestSuite(typing)) + return tests + + if __name__ == '__main__': main() diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 47619c8807bafe..25c16e3a0b7e43 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -119,11 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -142,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 63c6c055824b20..3680072d643792 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -19,7 +19,7 @@ def test_non_bmp_normalized(self): def test_invalid(self): try: - from test.tokenizedata import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 # noqa: F401 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 6adf03316ca0bb..d3bf4ea7c7d437 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -104,6 +104,26 @@ def test_name_inverse_lookup(self): if looked_name := self.db.name(char, None): self.assertEqual(self.db.lookup(looked_name), char) + def test_no_names_in_pua(self): + puas = [*range(0xe000, 0xf8ff), + *range(0xf0000, 0xfffff), + *range(0x100000, 0x10ffff)] + for i in puas: + char = chr(i) + self.assertRaises(ValueError, self.db.name, char) + + def test_lookup_nonexistant(self): + # just make sure that lookup can fail + for nonexistant in [ + "LATIN SMLL LETR A", + "OPEN HANDS SIGHS", + "DREGS", + "HANDBUG", + "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", + "???", + ]: + self.assertRaises(KeyError, self.db.lookup, nonexistant) + def test_digit(self): self.assertEqual(self.db.digit('A', None), None) self.assertEqual(self.db.digit('9'), 9) diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 5c1a28ecda5b49..1dec947ea76d23 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -386,6 +386,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index a465103b59b6ec..00ef55bdf9bc83 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -312,18 +312,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] @@ -484,5 +487,19 @@ async def test_demo1(self): result = test.run() self.assertTrue(result.wasSuccessful()) + def test_loop_factory(self): + asyncio.set_event_loop_policy(None) + + class TestCase1(unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + + async def test_demo1(self): + pass + + test = TestCase1('test_demo1') + result = test.run() + self.assertTrue(result.wasSuccessful()) + self.assertIsNone(support.maybe_get_event_loop_policy()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index ed5eb5609a5dd1..b4b2194a09cf9f 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -325,18 +325,37 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + Foo('test1').run() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): @@ -1132,6 +1151,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1148,6 +1169,8 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualwithEmptyString(self): '''Verify when there is an empty string involved, the diff output @@ -1165,6 +1188,8 @@ def testAssertEqualwithEmptyString(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMissingNewlineTerminator(self): '''Verifying format of diff output from assertEqual involving strings @@ -1185,6 +1210,8 @@ def testAssertEqualMultipleLinesMissingNewlineTerminator(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): '''Verifying format of diff output from assertEqual involving strings @@ -1208,6 +1235,8 @@ def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index dcb72d73efceab..a44b18406c08be 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -406,10 +406,34 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). @@ -571,7 +595,7 @@ def _get_module_from_name(name): result = unittest.TestResult() suite.run(result) self.assertEqual(len(result.skipped), 1) - self.assertEqual(result.testsRun, 0) + self.assertEqual(result.testsRun, 1) self.assertEqual(import_calls, ['my_package']) # Check picklability diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index f6d52f93e4a25f..7241cf59f73d4f 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -167,6 +167,18 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + def test_ExitSkippedSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar", "-k", "testSkipped"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 0) + out = stream.getvalue() + expected = '\n\nOK (skipped=1)\n' + self.assertTrue(out.endswith(expected)) + def test_ExitEmptySuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -447,8 +459,8 @@ def _join(name): def testParseArgsAbsolutePathsThatCannotBeConverted(self): program = self.program - # even on Windows '/...' is considered absolute by os.path.abspath - argv = ['progname', '/foo/bar/baz.py', '/green/red.py'] + drive = os.path.splitdrive(os.getcwd())[0] + argv = ['progname', f'{drive}/foo/bar/baz.py', f'{drive}/green/red.py'] self._patch_isfile(argv) program.createTests = lambda: None diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index 1a6af06d32b433..f146dcac18ecc0 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -103,16 +103,16 @@ def test_dont_skip(self): pass result = LoggingResult(events) self.assertIs(suite.run(result), result) self.assertEqual(len(result.skipped), 1) - expected = ['addSkip', 'stopTest', 'startTest', - 'addSuccess', 'stopTest'] + expected = ['startTest', 'addSkip', 'stopTest', + 'startTest', 'addSuccess', 'stopTest'] self.assertEqual(events, expected) - self.assertEqual(result.testsRun, 1) + self.assertEqual(result.testsRun, 2) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) self.assertTrue(result.wasSuccessful()) events = [] result = test_do_skip.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test_do_skip, "testing")]) @@ -135,13 +135,13 @@ def test_1(self): test = Foo("test_1") suite = unittest.TestSuite([test]) self.assertIs(suite.run(result), result) - self.assertEqual(events, ['addSkip', 'stopTest']) + self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) events = [] result = test.run() - self.assertEqual(events, ['startTestRun', 'addSkip', + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 00000000000000..d590a333930278 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,33 @@ +import unittest +from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py index 49986d65dc47af..6c535b7944f261 100644 --- a/Lib/test/test_unittest/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 74785a83757a92..c9c20f008ca5a2 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1127,6 +1127,14 @@ def test_propertymock_side_effect(self): p.assert_called_once_with() + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index d23eb87696f406..e1b108f81e513c 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -115,6 +115,24 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) + + def test_create_autospec_correctly_handles_name(self): + class X: ... + mock = create_autospec(X, spec_set=True, name="Y") + self.assertEqual(mock._mock_name, "Y") def test_repr(self): mock = Mock(name='foo') @@ -245,6 +263,65 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + def test_explicit_parent(self): parent = Mock() mock1 = Mock(parent=parent, return_value=None) @@ -622,6 +699,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() @@ -1058,7 +1143,7 @@ def test_assert_called_with_failure_message(self): actual = 'not called.' expected = "mock(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), mock.assert_called_with, 1, '2', 3, bar='foo' @@ -1073,7 +1158,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, '2', 3, bar='foo' @@ -1083,7 +1168,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, bar='foo' @@ -1093,7 +1178,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, 2, 3)" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, 2, 3 @@ -1103,7 +1188,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo()" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth ) @@ -1547,25 +1632,33 @@ def f(x=None): pass mock = Mock(spec=f) mock(1) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape('Calls not found.\n' - 'Expected: [call()]\n' - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: [call(1)]' + ) self.assertIsNone(cm.exception.__cause__) + uncalled_mock = Mock() + with self.assertRaises(AssertionError) as cm: + uncalled_mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: []' + ) + self.assertIsNone(cm.exception.__cause__) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape( - 'Error processing expected calls.\n' - "Errors: [None, TypeError('too many positional arguments')]\n" - "Expected: [call(), call(1, 2)]\n" - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call(), call(1, 2)]) + self.assertEqual(str(cm.exception), + 'Error processing expected calls.\n' + "Errors: [None, TypeError('too many positional arguments')]\n" + 'Expected: [call(), call(1, 2)]\n' + ' Actual: [call(1)]' + ) self.assertIsInstance(cm.exception.__cause__, TypeError) def test_assert_any_call(self): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index 833d7da1f31a20..f26e74ce0bc1ba 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -1728,6 +1730,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, @@ -1912,7 +1979,7 @@ def foo(x=0): with patch.object(foo, '__module__', "testpatch2"): self.assertEqual(foo.__module__, "testpatch2") - self.assertEqual(foo.__module__, 'test.test_unittest.testmock.testpatch') + self.assertEqual(foo.__module__, __name__) with patch.object(foo, '__annotations__', dict([('s', 1, )])): self.assertEqual(foo.__annotations__, dict([('s', 1, )])) @@ -1978,6 +2045,13 @@ def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index c201d08f61b8cd..9e2d54bd3a8c4e 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -26,6 +26,12 @@ >>> a == [7, 8, 9] True +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + Unpack string... fun! >>> a, *b = 'one' diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 6f698a8d891815..35394f29fbe49d 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -370,13 +370,13 @@ def test_slices(self): self.check_ast_roundtrip("a[i:j, k]") def test_invalid_raise(self): - self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ - ast.Name(id="test"), + ast.Name(id="test", ctx=ast.Load()), ast.Constant(value="test") ] ) @@ -649,6 +649,44 @@ def test_multiquote_joined_string(self): self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + def test_backslash_in_format_spec(self): + import re + msg = re.escape("invalid escape sequence '\\ '") + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -694,11 +732,25 @@ def test_function_with_type_params_and_bound(self): body=[ast.Pass()], decorator_list=[], returns=None, - type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], ) ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + def test_async_function(self): node = ast.AsyncFunctionDef( name="f", @@ -722,6 +774,20 @@ def test_async_function_with_type_params(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99c9e24994732f..eed0599642edfb 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import os_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib from unittest import mock @@ -14,10 +15,11 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) from urllib.parse import urlparse @@ -775,7 +777,7 @@ def connect_ftp(self, user, passwd, host, port, dirs, ["foo", "bar"], "", None), ("ftp://localhost/baz.gif;type=a", "localhost", ftplib.FTP_PORT, "", "", "A", - [], "baz.gif", None), # XXX really this should guess image/gif + [], "baz.gif", "image/gif"), ]: req = Request(url) req.timeout = None @@ -998,6 +1000,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the @@ -1399,6 +1402,15 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") + def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1483,6 +1495,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; " + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 625c6dc88796b6..d6c83a75c1c03a 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -19,6 +19,10 @@ ("=a", [('', 'a')]), ("a", [('a', '')]), ("a=", [('a', '')]), + ("a=b=c", [('a', 'b=c')]), + ("a%3Db=c", [('a=b', 'c')]), + ("a=b&c=d", [('a', 'b'), ('c', 'd')]), + ("a=b%26c=d", [('a', 'b&c=d')]), ("&a=b", [('a', 'b')]), ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), ("a=1&a=2", [('a', '1'), ('a', '2')]), @@ -29,6 +33,10 @@ (b"=a", [(b'', b'a')]), (b"a", [(b'a', b'')]), (b"a=", [(b'a', b'')]), + (b"a=b=c", [(b'a', b'b=c')]), + (b"a%3Db=c", [(b'a=b', b'c')]), + (b"a=b&c=d", [(b'a', b'b'), (b'c', b'd')]), + (b"a=b%26c=d", [(b'a', b'b&c=d')]), (b"&a=b", [(b'a', b'b')]), (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), @@ -36,6 +44,14 @@ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), (b";a=b", [(b';a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), + + ("\u0141=\xE9", [('\u0141', '\xE9')]), + ("%C5%81=%C3%A9", [('\u0141', '\xE9')]), + ("%81=%A9", [('\ufffd', '\ufffd')]), + (b"\xc5\x81=\xc3\xa9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"%C5%81=%C3%A9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"\x81=\xA9", [(b'\x81', b'\xa9')]), + (b"%81=%A9", [(b'\x81', b'\xa9')]), ] # Each parse_qs testcase is a two-tuple that contains @@ -49,6 +65,10 @@ ("=a", {'': ['a']}), ("a", {'a': ['']}), ("a=", {'a': ['']}), + ("a=b=c", {'a': ['b=c']}), + ("a%3Db=c", {'a=b': ['c']}), + ("a=b&c=d", {'a': ['b'], 'c': ['d']}), + ("a=b%26c=d", {'a': ['b&c=d']}), ("&a=b", {'a': ['b']}), ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), ("a=1&a=2", {'a': ['1', '2']}), @@ -59,6 +79,10 @@ (b"=a", {b'': [b'a']}), (b"a", {b'a': [b'']}), (b"a=", {b'a': [b'']}), + (b"a=b=c", {b'a': [b'b=c']}), + (b"a%3Db=c", {b'a=b': [b'c']}), + (b"a=b&c=d", {b'a': [b'b'], b'c': [b'd']}), + (b"a=b%26c=d", {b'a': [b'b&c=d']}), (b"&a=b", {b'a': [b'b']}), (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1&a=2", {b'a': [b'1', b'2']}), @@ -66,11 +90,22 @@ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), (b";a=b", {b';a': [b'b']}), (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), + (b"a=a%E2%80%99b", {b'a': [b'a\xe2\x80\x99b']}), + + ("\u0141=\xE9", {'\u0141': ['\xE9']}), + ("%C5%81=%C3%A9", {'\u0141': ['\xE9']}), + ("%81=%A9", {'\ufffd': ['\ufffd']}), + (b"\xc5\x81=\xc3\xa9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"%C5%81=%C3%A9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"\x81=\xA9", {b'\x81': [b'\xa9']}), + (b"%81=%A9", {b'\x81': [b'\xa9']}), ] class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, @@ -78,7 +113,7 @@ def checkRoundtrips(self, url, parsed, split): self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it @@ -104,7 +139,7 @@ def checkRoundtrips(self, url, parsed, split): result.query, result.fragment) self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() @@ -142,9 +177,45 @@ def test_qs(self): def test_roundtrips(self): str_cases = [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -171,15 +242,58 @@ def test_roundtrips(self): 'action=download-manifest&url=https://example.com/app', ''), ('itms-services', '', '', 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), ] def _encode(t): return (t[0].encode('ascii'), tuple(x.encode('ascii') for x in t[1]), tuple(x.encode('ascii') for x in t[2])) bytes_cases = [_encode(x) for x in str_cases] + str_cases += [ + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ] for url, parsed, split in str_cases + bytes_cases: self.checkRoundtrips(url, parsed, split) + def test_roundtrips_normalization(self): + str_cases = [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ] + def _encode(t): + return (t[0].encode('ascii'), + t[1].encode('ascii'), + tuple(x.encode('ascii') for x in t[2]), + tuple(x.encode('ascii') for x in t[3])) + bytes_cases = [_encode(x) for x in str_cases] + for url, url2, parsed, split in str_cases + bytes_cases: + self.checkRoundtrips(url, parsed, split, url2) + def test_http_roundtrips(self): # urllib.parse.urlsplit treats 'http:' as an optimized special case, # so we test both 'http:' and 'https:' in all the following. @@ -995,8 +1109,8 @@ def test_parse_qsl_encoding(self): def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): - urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) - urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) def test_parse_qs_separator(self): parse_qs_semicolon_cases = [ @@ -1039,6 +1153,30 @@ def test_parse_qsl_separator(self): result_bytes = urllib.parse.parse_qsl(orig, separator=b';') self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, 0, 0.0, [], {}, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: @@ -1369,13 +1507,6 @@ def test_unwrap(self): class DeprecationTest(unittest.TestCase): - - def test_Quoter_deprecation(self): - with self.assertWarns(DeprecationWarning) as cm: - old_class = urllib.parse.Quoter - self.assertIs(old_class, urllib.parse._Quoter) - self.assertIn('Quoter will be removed', str(cm.warning)) - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 483910aaa4620e..61e79f553e8ec9 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,6 +1,6 @@ # Check every path through every method of UserDict -from test import mapping_tests +from test import mapping_tests, support import unittest import collections @@ -213,6 +213,11 @@ class G(collections.UserDict): else: self.fail("g[42] didn't raise KeyError") + # Decorate existing test with recursion limit, because + # the test is for C structure, but `UserDict` is a Python structure. + test_repr_deep = support.infinite_recursion(25)( + mapping_tests.TestHashMappingProtocol.test_repr_deep, + ) if __name__ == "__main__": diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 1ed67dac805967..312702c8e398b9 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -3,6 +3,8 @@ from collections import UserList from test import list_tests import unittest +from test import support + class UserListTest(list_tests.CommonTest): type2test = UserList @@ -65,5 +67,11 @@ def test_userlist_copy(self): self.assertEqual(u, v) self.assertEqual(type(u), type(v)) + # Decorate existing test with recursion limit, because + # the test is for C structure, but `UserList` is a Python structure. + test_repr_deep = support.infinite_recursion(25)( + list_tests.CommonTest.test_repr_deep, + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_utf8source.py b/Lib/test/test_utf8source.py index c42b6aaaab579d..7336cf00a71183 100644 --- a/Lib/test/test_utf8source.py +++ b/Lib/test/test_utf8source.py @@ -14,7 +14,7 @@ def test_pep3120(self): def test_badsyntax(self): try: - import test.tokenizedata.badsyntax_pep3120 + import test.tokenizedata.badsyntax_pep3120 # noqa: F401 except SyntaxError as msg: msg = str(msg).lower() self.assertTrue('utf-8' in msg) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 9cec1e87fd3c2d..e177464c00f7a6 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -530,7 +530,14 @@ def test_uuid1(self): @support.requires_mac_ver(10, 5) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: + try: + import _uuid + except ImportError: + has_uuid_generate_time_safe = False + else: + has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe + + if not has_uuid_generate_time_safe or not self.uuid._generate_time_safe: self.skipTest('requires uuid_generate_time_safe(3)') u = self.uuid.uuid1() @@ -546,7 +553,6 @@ def mock_generate_time_safe(self, safe_value): """ if os.name != 'posix': self.skipTest('POSIX-only test') - self.uuid._load_system_functions() f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') @@ -581,8 +587,7 @@ def test_uuid1_bogus_return_value(self): self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ mock.patch('time.time_ns', return_value=1545052026752910643), \ @@ -590,8 +595,7 @@ def test_uuid1_time(self): u = self.uuid.uuid1() self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch('time.time_ns', return_value=1545052026752910643): u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 890672c5d27eec..2b7d297f011741 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,10 +19,12 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten, is_wasi, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) import unittest import venv from unittest.mock import patch, Mock @@ -39,21 +41,26 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_emscripten or is_wasi: - raise unittest.SkipTest("venv is not available on Emscripten/WASI.") +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: if verbose and err: - print(err.decode('utf-8', 'backslashreplace')) + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err class BaseTest(unittest.TestCase): @@ -68,7 +75,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -118,12 +125,12 @@ def test_defaults_with_str_path(self): self.run_with_capture(venv.create, self.env_dir) self._check_output_of_default_create() - def test_defaults_with_pathlib_path(self): + def test_defaults_with_pathlike(self): """ - Test the create function with default arguments and a pathlib.Path path. + Test the create function with default arguments and a path-like path. """ rmtree(self.env_dir) - self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self.run_with_capture(venv.create, FakePath(self.env_dir)) self._check_output_of_default_create() def _check_output_of_default_create(self): @@ -164,7 +171,7 @@ def test_config_file_command_key(self): ('--clear', 'clear', True), ('--upgrade', 'upgrade', True), ('--upgrade-deps', 'upgrade_deps', True), - ('--prompt', 'prompt', True), + ('--prompt="foobar"', 'prompt', 'foobar'), ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), ] for opt, attr, value in options: @@ -196,7 +203,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertEqual(context.prompt, env_name) self.assertNotIn("prompt = ", data) rmtree(self.env_dir) @@ -204,7 +211,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(My prompt) ') + self.assertEqual(context.prompt, 'My prompt') self.assertIn("prompt = 'My prompt'\n", data) rmtree(self.env_dir) @@ -213,13 +220,19 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertEqual(context.prompt, cwd) self.assertIn("prompt = '%s'\n" % cwd, data) def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -261,7 +274,8 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), prefix) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) @requireVenvCreate def test_sysconfig(self): @@ -278,11 +292,23 @@ def test_sysconfig(self): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -300,11 +326,24 @@ def test_sysconfig_symlinks(self): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + # HACK: Test fails on POSIX with unversioned binary (PR gh-113033) + #('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( @@ -494,7 +533,7 @@ def test_multiprocessing_recursion(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') - subprocess.check_call([self.envpy(real_env_dir=True), script]) + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): @@ -533,7 +572,7 @@ def test_pathsep_error(self): rmtree(self.env_dir) bad_itempath = self.env_dir + os.pathsep self.assertRaises(ValueError, venv.create, bad_itempath) - self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') @requireVenvCreate @@ -554,7 +593,8 @@ def test_zippath_from_non_installed_posix(self): libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") - stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" zip_landmark = os.path.join(non_installed_dir, platlibdir, stdlib_zip) @@ -706,6 +746,36 @@ def test_cli_without_scm_ignore_files(self): with self.assertRaises(FileNotFoundError): self.get_text_file_contents('.gitignore') + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 83237f5fe0d1b3..f9b2b07fbd6576 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -5,17 +5,21 @@ import re import sys import textwrap +import types +from typing import overload, get_overloads import unittest from test import support from test.support import import_helper from test.support import os_helper from test.support import warnings_helper +from test.support import force_not_colorized from test.support.script_helper import assert_python_ok, assert_python_failure from test.test_warnings.data import package_helper from test.test_warnings.data import stacklevel as warning_tests import warnings as original_warnings +from warnings import deprecated py_warnings = import_helper.import_fresh_module('warnings', @@ -90,7 +94,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(self.module, '__all__')) target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] self.assertSetEqual(set(self.module.__all__), set(target_api)) @@ -151,40 +155,42 @@ def f(): f() self.assertEqual(len(w), 1) - def test_always(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - self.module.filterwarnings("always", category=UserWarning) - message = "FilterTests.test_always" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings(mode, category=UserWarning) + message = "FilterTests.test_always_and_all" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) - def test_always_after_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - message = "FilterTests.test_always_after_ignore" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 1) - self.module.filterwarnings("always", category=UserWarning) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 3) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all_after_default(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_and_all_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings(mode, category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) def test_default(self): with original_warnings.catch_warnings(record=True, @@ -372,6 +378,28 @@ def test_append_duplicate(self): "appended duplicate changed order of filters" ) + def test_argument_validation(self): + with self.assertRaises(ValueError): + self.module.filterwarnings(action='foo') + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', message=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=int) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', module=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.filterwarnings('ignore', lineno=-1) + with self.assertRaises(ValueError): + self.module.simplefilter(action='foo') + with self.assertRaises(TypeError): + self.module.simplefilter('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.simplefilter('ignore', lineno=-1) + def test_catchwarnings_with_simplefilter_ignore(self): with original_warnings.catch_warnings(module=self.module): self.module.resetwarnings() @@ -464,7 +492,7 @@ def test_stacklevel(self): warning_tests.inner("spam7", stacklevel=9999) self.assertEqual(os.path.basename(w[-1].filename), - "sys") + "") def test_stacklevel_import(self): # Issue #24305: With stacklevel=2, module-level warnings should work. @@ -473,7 +501,7 @@ def test_stacklevel_import(self): with original_warnings.catch_warnings(record=True, module=self.module) as w: self.module.simplefilter('always') - import test.test_warnings.data.import_warning + import test.test_warnings.data.import_warning # noqa: F401 self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, __file__) @@ -1214,6 +1242,7 @@ def test_comma_separated_warnings(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_envvar_and_command_line(self): rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", "import sys; sys.stdout.write(str(sys.warnoptions))", @@ -1222,6 +1251,7 @@ def test_envvar_and_command_line(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_conflicting_envvar_and_command_line(self): rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " @@ -1233,6 +1263,10 @@ def test_conflicting_envvar_and_command_line(self): self.assertEqual(stderr.splitlines(), [b"Traceback (most recent call last):", b" File \"\", line 1, in ", + b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' + b"arn('Message', DeprecationWarning)", + b' ~~~~~~~~~~' + b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', b"DeprecationWarning: Message"]) def test_default_filter_configuration(self): @@ -1359,7 +1393,7 @@ def test_late_resource_warning(self): # Issue #21925: Emitting a ResourceWarning late during the Python # shutdown must be logged. - expected = b"sys:1: ResourceWarning: unclosed file " + expected = b":0: ResourceWarning: unclosed file " # don't import the warnings module # (_warnings will try to import it) @@ -1373,6 +1407,283 @@ def test_late_resource_warning(self): self.assertTrue(err.startswith(expected), ascii(err)) +class DeprecatedTests(unittest.TestCase): + def test_dunder_deprecated(self): + @deprecated("A will go away soon") + class A: + pass + + self.assertEqual(A.__deprecated__, "A will go away soon") + self.assertIsInstance(A, type) + + @deprecated("b will go away soon") + def b(): + pass + + self.assertEqual(b.__deprecated__, "b will go away soon") + self.assertIsInstance(b, types.FunctionType) + + @overload + @deprecated("no more ints") + def h(x: int) -> int: ... + @overload + def h(x: str) -> str: ... + def h(x): + return x + + overloads = get_overloads(h) + self.assertEqual(len(overloads), 2) + self.assertEqual(overloads[0].__deprecated__, "no more ints") + + def test_class(self): + @deprecated("A will go away soon") + class A: + pass + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + A() + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + with self.assertRaises(TypeError): + A(42) + + def test_class_with_init(self): + @deprecated("HasInit will go away soon") + class HasInit: + def __init__(self, x): + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasInit will go away soon"): + instance = HasInit(42) + self.assertEqual(instance.x, 42) + + def test_class_with_new(self): + has_new_called = False + + @deprecated("HasNew will go away soon") + class HasNew: + def __new__(cls, x): + nonlocal has_new_called + has_new_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasNew will go away soon"): + instance = HasNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(has_new_called) + + def test_class_with_inherited_new(self): + new_base_called = False + + class NewBase: + def __new__(cls, x): + nonlocal new_base_called + new_base_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + @deprecated("HasInheritedNew will go away soon") + class HasInheritedNew(NewBase): + pass + + with self.assertWarnsRegex(DeprecationWarning, "HasInheritedNew will go away soon"): + instance = HasInheritedNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_base_called) + + def test_class_with_new_but_no_init(self): + new_called = False + + @deprecated("HasNewNoInit will go away soon") + class HasNewNoInit: + def __new__(cls, x): + nonlocal new_called + new_called = True + obj = super().__new__(cls) + obj.x = x + return obj + + with self.assertWarnsRegex(DeprecationWarning, "HasNewNoInit will go away soon"): + instance = HasNewNoInit(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_called) + + def test_mixin_class(self): + @deprecated("Mixin will go away soon") + class Mixin: + pass + + class Base: + def __init__(self, a) -> None: + self.a = a + + with self.assertWarnsRegex(DeprecationWarning, "Mixin will go away soon"): + class Child(Base, Mixin): + pass + + instance = Child(42) + self.assertEqual(instance.a, 42) + + def test_existing_init_subclass(self): + @deprecated("C will go away soon") + class C: + def __init_subclass__(cls) -> None: + cls.inited = True + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C): + pass + + self.assertTrue(D.inited) + self.assertIsInstance(D(), D) # no deprecation + + def test_existing_init_subclass_in_base(self): + class Base: + def __init_subclass__(cls, x) -> None: + cls.inited = x + + @deprecated("C will go away soon") + class C(Base, x=42): + pass + + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C, x=3): + pass + + self.assertEqual(D.inited, 3) + + def test_init_subclass_has_correct_cls(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_init_subclass_with_explicit_classmethod(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + @classmethod + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_function(self): + @deprecated("b will go away soon") + def b(): + pass + + with self.assertWarnsRegex(DeprecationWarning, "b will go away soon"): + b() + + def test_method(self): + class Capybara: + @deprecated("x will go away soon") + def x(self): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x() + + def test_property(self): + class Capybara: + @property + @deprecated("x will go away soon") + def x(self): + pass + + @property + def no_more_setting(self): + return 42 + + @no_more_setting.setter + @deprecated("no more setting") + def no_more_setting(self, value): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + self.assertEqual(instance.no_more_setting, 42) + + with self.assertWarnsRegex(DeprecationWarning, "no more setting"): + instance.no_more_setting = 42 + + def test_category(self): + @deprecated("c will go away soon", category=RuntimeWarning) + def c(): + pass + + with self.assertWarnsRegex(RuntimeWarning, "c will go away soon"): + c() + + def test_turn_off_warnings(self): + @deprecated("d will go away soon", category=None) + def d(): + pass + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + d() + + def test_only_strings_allowed(self): + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'type'" + ): + @deprecated + class Foo: ... + + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'function'" + ): + @deprecated + def foo(): ... + + def test_no_retained_references_to_wrapper_instance(self): + @deprecated('depr') + def d(): pass + + self.assertFalse(any( + isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ + )) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4cdd66d3769e0c..2b9b2a04db8298 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -10,11 +10,14 @@ import threading import time import random +import textwrap from test import support -from test.support import script_helper, ALWAYS_EQ +from test.support import script_helper, ALWAYS_EQ, suppress_immortalization from test.support import gc_collect +from test.support import import_helper from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -79,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ @@ -116,6 +119,33 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + self.assertRegex(repr(ref), + rf"") + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + self.assertRegex(repr(ref2), + rf"") + def test_repr_failure_gh99184(self): class MyConfig(dict): def __getattr__(self, x): @@ -134,7 +164,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) @@ -195,6 +225,20 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + self.assertRegex(repr(ref), + rf"") + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) @@ -608,6 +652,7 @@ class C(object): # deallocation of c2. del c2 + @suppress_immortalization() def test_callback_in_cycle(self): import gc @@ -700,6 +745,7 @@ class D: del c1, c2, C, D gc.collect() + @suppress_immortalization() def test_callback_in_cycle_resurrection(self): import gc @@ -835,6 +881,7 @@ def test_init(self): # No exception should be raised here gc.collect() + @suppress_immortalization() def test_classes(self): # Check that classes are weakrefable. class A(object): @@ -914,6 +961,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing @@ -967,6 +1015,31 @@ def __del__(self): pass del x support.gc_collect() + @support.cpython_only + def test_no_memory_when_clearing(self): + # gh-118331: Make sure we do not raise an exception from the destructor + # when clearing weakrefs if allocating the intermediate tuple fails. + code = textwrap.dedent(""" + import _testcapi + import weakref + + class TestObj: + pass + + def callback(obj): + pass + + obj = TestObj() + # The choice of 50 is arbitrary, but must be large enough to ensure + # the allocation won't be serviced by the free list. + wrs = [weakref.ref(obj, callback) for _ in range(50)] + _testcapi.set_nomemory(0) + del obj + """).strip() + res, _ = script_helper.run_python_until_end("-c", code) + stderr = res.err.decode("ascii", "backslashreplace") + self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + class SubclassableWeakrefTestCase(TestBase): @@ -1213,6 +1286,12 @@ class MappingTestCase(TestBase): COUNT = 10 + if support.check_sanitizer(thread=True) and support.Py_GIL_DISABLED: + # Reduce iteration count to get acceptable latency + NUM_THREADED_ITERATIONS = 1000 + else: + NUM_THREADED_ITERATIONS = 100000 + def check_len_cycles(self, dict_type, cons): N = 20 items = [RefCycle() for i in range(N)] @@ -1838,7 +1917,7 @@ def test_make_weak_keyed_dict_repr(self): def test_threaded_weak_valued_setdefault(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): x = d.setdefault(10, RefCycle()) self.assertIsNot(x, None) # we never put None in there! del x @@ -1847,7 +1926,7 @@ def test_threaded_weak_valued_setdefault(self): def test_threaded_weak_valued_pop(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): d[10] = RefCycle() x = d.pop(10, 10) self.assertIsNot(x, None) # we never put None in there! @@ -1858,13 +1937,32 @@ def test_threaded_weak_valued_consistency(self): # WeakValueDictionary when collecting from another thread. d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(200000): + for i in range(2 * self.NUM_THREADED_ITERATIONS): o = RefCycle() d[10] = o # o is still alive, so the dict can't be empty self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. @@ -1927,6 +2025,7 @@ def pop_and_collect(lst): raise exc[0] @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -1940,6 +2039,7 @@ def test_threaded_weak_key_dict_deepcopy(self): self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 2d695bc883131f..ae8d776e8413ff 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,15 +1,20 @@ -import webbrowser -import unittest import os -import sys +import re +import shlex import subprocess -from unittest import mock +import sys +import unittest +import webbrowser from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper +from test.support import requires_subprocess +from test.support import threading_helper +from unittest import mock -if not support.has_subprocess_support: - raise unittest.SkipTest("test webserver requires subprocess") +# The webbrowser module uses threading locks +threading_helper.requires_working_threading(module=True) URL = 'https://www.example.com' CMD_NAME = 'test' @@ -24,6 +29,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -94,6 +100,15 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_open_bad_new_parameter(self): + with self.assertRaisesRegex(webbrowser.Error, + re.escape("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got 999")): + self._test('open', + options=[], + arguments=[URL], + kw=dict(new=999)) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): @@ -201,22 +216,89 @@ class ELinksCommandTest(CommandTestMixin, unittest.TestCase): def test_open(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_with_autoraise_false(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_new(self): self._test('open_new', options=['-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + arguments=[f'openURL({URL},new-window)']) def test_open_new_tab(self): self._test('open_new_tab', options=['-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + arguments=[f'openURL({URL},new-tab)']) + + +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'https://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') class BrowserRegistrationTest(unittest.TestCase): @@ -271,6 +353,16 @@ def test_register_default(self): def test_register_preferred(self): self._check_registration(preferred=True) + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") + def test_no_xdg_settings_on_macOS(self): + # On macOS webbrowser should not use xdg-settings to + # look for X11 based browsers (for those users with + # XQuartz installed) + with mock.patch("subprocess.check_output") as ck_o: + webbrowser.register_standard_browsers() + + ck_o.assert_not_called() + class ImportTest(unittest.TestCase): def test_register(self): @@ -296,12 +388,17 @@ def test_get(self): webbrowser.get('fakebrowser') self.assertIsNotNone(webbrowser._tryorder) + @unittest.skipIf(" " in sys.executable, "test assumes no space in path (GH-114452)") def test_synthesize(self): webbrowser = import_helper.import_fresh_module('webbrowser') name = os.path.basename(sys.executable).lower() webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -313,6 +410,10 @@ def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -332,5 +433,74 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) -if __name__=='__main__': +class CliTest(unittest.TestCase): + def test_parse_args(self): + for command, url, new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + args = webbrowser.parse_args(shlex.split(command)) + + self.assertEqual(args.url, url) + self.assertEqual(args.new_win, new_win) + + def test_parse_args_error(self): + for command in [ + # Arguments must not both be given + "https://example.com -n -t", + "https://example.com --new-window --new-tab", + "https://example.com -n --new-tab", + "https://example.com --new-window -t", + ]: + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split(command)) + self.assertIn( + 'error: argument -t/--new-tab: not allowed with argument -n/--new-window', + stderr.getvalue(), + ) + + # Ensure ambiguous shortening fails + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split("https://example.com --new")) + self.assertIn( + 'error: ambiguous option: --new could match --new-window, --new-tab', + stderr.getvalue() + ) + + def test_main(self): + for command, expected_url, expected_new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + with ( + mock.patch("webbrowser.open", return_value=None) as mock_open, + mock.patch("builtins.print", return_value=None), + ): + webbrowser.main(shlex.split(command)) + mock_open.assert_called_once_with(expected_url, expected_new_win) + + +if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py new file mode 100644 index 00000000000000..e64208330ad2f9 --- /dev/null +++ b/Lib/test/test_winapi.py @@ -0,0 +1,158 @@ +# Test the Windows-only _winapi module + +import os +import pathlib +import re +import unittest +from test.support import import_helper, os_helper + +_winapi = import_helper.import_module('_winapi', required_on=['win']) + +MAXIMUM_WAIT_OBJECTS = 64 +MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): + def _events_waitall_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + + # Ensure no errors raised when all are triggered + for e in evts: + _winapi.SetEvent(e) + try: + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + except TimeoutError: + self.fail("expected wait to complete immediately") + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Replace events with invalid handles to make sure we fail + for i in chosen: + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + evts[i] = old_evt + + + def _events_waitany_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Trigger one by one. They are auto-reset events, so will only trigger once + for i in chosen: + with self.subTest(f"trigger event {i} of {len(evts)}"): + _winapi.SetEvent(evts[i]) + triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000) + self.assertSetEqual(set(triggered), {i}) + + # Trigger all at once. This may require multiple calls + for i in chosen: + _winapi.SetEvent(evts[i]) + triggered = set() + while len(triggered) < len(chosen): + triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000)) + self.assertSetEqual(triggered, set(chosen)) + + # Replace events with invalid handles to make sure we fail + for i in chosen: + with self.subTest(f"corrupt event {i} of {len(evts)}"): + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + evts[i] = old_evt + + + def test_few_events_waitall(self): + self._events_waitall_test(16) + + def test_many_events_waitall(self): + self._events_waitall_test(256) + + def test_max_events_waitall(self): + self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + + def test_few_events_waitany(self): + self._events_waitany_test(16) + + def test_many_events_waitany(self): + self._events_waitany_test(256) + + def test_max_events_waitany(self): + self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + +class WinAPITests(unittest.TestCase): + def test_getlongpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" + if not os.path.isdir(testfn): + raise unittest.SkipTest("require x:\\PROGRA~1 to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetLongPathName(testfn) + + actual = _winapi.GetLongPathName(os.fsdecode(testfn)) + + # Can't assume that PROGRA~1 expands to any particular variation, so + # ensure it matches any one of them. + candidates = set(testfn.parent.glob("Progra*")) + self.assertIn(pathlib.Path(actual), candidates) + + def test_getshortpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")) + if not os.path.isdir(testfn): + raise unittest.SkipTest("require '%ProgramFiles%' to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetShortPathName(testfn) + + actual = _winapi.GetShortPathName(os.fsdecode(testfn)) + + # Should contain "PROGRA~" but we can't predict the number + self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 70a85552cc03b0..a10d63dfdc9753 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -6,7 +6,7 @@ import sys import tempfile import unittest -from test.support import os_helper +from test.support import os_helper, requires_resource if sys.platform != 'win32': raise unittest.SkipTest("test only relevant on win32") @@ -43,6 +43,9 @@ def test_open_fd(self): self.assertEqual(0, f.fileno()) f.close() # multiple close should not crash f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(1, 'w') @@ -55,6 +58,9 @@ def test_open_fd(self): self.assertEqual(1, f.fileno()) f.close() f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(2, 'w') @@ -98,6 +104,16 @@ def test_open_name(self): self.assertIsInstance(f, ConIO) f.close() + def test_subclass_repr(self): + class TestSubclass(ConIO): + pass + + f = TestSubclass("CON") + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + @unittest.skipIf(sys.getwindowsversion()[:2] <= (6, 1), "test does not work on Windows 7 and earlier") def test_conin_conout_names(self): @@ -140,6 +156,7 @@ def assertStdinRoundTrip(self, text): sys.stdin = old_stdin self.assertEqual(actual, text) + @requires_resource('console') def test_input(self): # ASCII self.assertStdinRoundTrip('abc123') @@ -154,6 +171,7 @@ def test_input_nonbmp(self): # Non-BMP self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + @requires_resource('console') def test_partial_reads(self): # Test that reading less than 1 full character works when stdin # contains multibyte UTF-8 sequences @@ -189,6 +207,7 @@ def test_partial_surrogate_reads(self): self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + @requires_resource('console') def test_ctrl_z(self): with open('CONIN$', 'rb', buffering=0) as stdin: source = '\xC4\x1A\r\n'.encode('utf-16-le') diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index a59d0d24f5db48..870ab7bd41d8ce 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -1,12 +1,13 @@ # Ridiculously simple test of the winsound module for Windows. import functools -import pathlib +import os import time import unittest from test import support from test.support import import_helper +from test.support import os_helper support.requires('audio') @@ -85,13 +86,6 @@ def test_keyword_args(self): safe_MessageBeep(type=winsound.MB_OK) -# A class for testing winsound when the given path resolves -# to bytes rather than str. -class BytesPath(pathlib.WindowsPath): - def __fspath__(self): - return bytes(super().__fspath__(), 'UTF-8') - - class PlaySoundTest(unittest.TestCase): def test_errors(self): @@ -126,7 +120,7 @@ def test_snd_filename(self): def test_snd_filepath(self): fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') - path = pathlib.Path(fn) + path = os_helper.FakePath(fn) safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) def test_snd_filepath_as_bytes(self): @@ -134,7 +128,7 @@ def test_snd_filepath_as_bytes(self): self.assertRaises( TypeError, winsound.PlaySound, - BytesPath(fn), + os_helper.FakePath(os.fsencode(fn)), winsound.SND_FILENAME | winsound.SND_NODEFAULT ) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index d81902327a7e0a..e8c4ddf979e2ee 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -5,6 +5,7 @@ __email__ = "mbland at acm dot org" import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext @@ -749,5 +750,48 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3445702846d8a0..f667926d1f8ddf 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,17 +1,31 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others +import time import unittest -from test.support import import_helper, requires_resource +from test.support import import_helper, requires_resource, LOOPBACK_TIMEOUT # Do this first so test will be skipped if module doesn't exist _wmi = import_helper.import_module('_wmi', required_on=['win']) +def wmi_exec_query(query): + # gh-112278: WMI maybe slow response when first call. + try: + return _wmi.exec_query(query) + except BrokenPipeError: + pass + except WindowsError as e: + if e.winerror != 258: + raise + time.sleep(LOOPBACK_TIMEOUT) + return _wmi.exec_query(query) + + class WmiTests(unittest.TestCase): def test_wmi_query_os_version(self): - r = _wmi.exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") + r = wmi_exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") self.assertEqual(1, len(r)) k, eq, v = r[0].partition("=") self.assertEqual("=", eq, r[0]) @@ -28,7 +42,7 @@ def test_wmi_query_repeated(self): def test_wmi_query_error(self): # Invalid queries fail with OSError try: - _wmi.exec_query("SELECT InvalidColumnName FROM InvalidTableName") + wmi_exec_query("SELECT InvalidColumnName FROM InvalidTableName") except OSError as ex: if ex.winerror & 0xFFFFFFFF == 0x80041010: # This is the expected error code. All others should fail the test @@ -42,7 +56,7 @@ def test_wmi_query_repeated_error(self): def test_wmi_query_not_select(self): # Queries other than SELECT are blocked to avoid potential exploits with self.assertRaises(ValueError): - _wmi.exec_query("not select, just in case someone tries something") + wmi_exec_query("not select, just in case someone tries something") @requires_resource('cpu') def test_wmi_query_overflow(self): @@ -50,11 +64,11 @@ def test_wmi_query_overflow(self): # Test multiple times to ensure consistency for _ in range(2): with self.assertRaises(OSError): - _wmi.exec_query("SELECT * FROM CIM_DataFile") + wmi_exec_query("SELECT * FROM CIM_DataFile") def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator - r = _wmi.exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") + r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") self.assertFalse(r.startswith("\0"), r) self.assertFalse(r.endswith("\0"), r) it = iter(r.split("\0")) @@ -69,6 +83,6 @@ def test_wmi_query_threads(self): from concurrent.futures import ThreadPoolExecutor query = "SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000" with ThreadPoolExecutor(4) as pool: - task = [pool.submit(_wmi.exec_query, query) for _ in range(32)] + task = [pool.submit(wmi_exec_query, query) for _ in range(32)] for t in task: self.assertRegex(t.result(), "ProcessId=") diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 9316d0ecbcf1ae..b047f7b06f85d3 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -137,7 +137,7 @@ def test_environ(self): def test_request_length(self): out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") self.assertEqual(out.splitlines()[0], - b"HTTP/1.0 414 Request-URI Too Long") + b"HTTP/1.0 414 URI Too Long") def test_validated_hello(self): out, err = run_amock(validator(hello_app)) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 6d413aa68a338d..930501633d1f38 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -13,6 +13,7 @@ import operator import os import pickle +import pyexpat import sys import textwrap import types @@ -137,9 +138,9 @@ class ModuleTest(unittest.TestCase): def test_sanity(self): # Import sanity. - from xml.etree import ElementTree - from xml.etree import ElementInclude - from xml.etree import ElementPath + from xml.etree import ElementTree # noqa: F401 + from xml.etree import ElementInclude # noqa: F401 + from xml.etree import ElementPath # noqa: F401 def test_all(self): names = ("xml.etree.ElementTree", "_elementtree") @@ -328,7 +329,7 @@ def test_simpleops(self): self.serialize_check(element, '') # 5 with self.assertRaises(ValueError) as cm: element.remove(subelement) - self.assertIn('not in list', str(cm.exception)) + self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '') @@ -536,7 +537,9 @@ def test_iterparse(self): iterparse = ET.iterparse context = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(context.root) action, elem = next(context) + self.assertIsNone(context.root) self.assertEqual((action, elem.tag), ('end', 'element')) self.assertEqual([(action, elem.tag) for action, elem in context], [ ('end', 'element'), @@ -553,6 +556,17 @@ def test_iterparse(self): ('end', '{namespace}root'), ]) + with open(SIMPLE_XMLFILE, 'rb') as source: + context = iterparse(source) + action, elem = next(context) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in context], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(context.root.tag, 'root') + events = () context = iterparse(SIMPLE_XMLFILE, events) self.assertEqual([(action, elem.tag) for action, elem in context], []) @@ -644,12 +658,81 @@ def test_iterparse(self): # Not exhausting the iterator still closes the resource (bpo-43292) with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) + it = iterparse(SIMPLE_XMLFILE) + del it + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + it.close() del it + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + with self.assertRaises(FileNotFoundError): iterparse("nonexistent") + def test_iterparse_close(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1375,12 +1458,14 @@ def test_attlist_default(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1398,28 +1483,35 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "\n text\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + def test_simple_xml(self, chunk_size=None, flush=False): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, + "\n text\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1, flush=True) + + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5, flush=True) + + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) def test_feed_while_iterating(self): parser = ET.XMLPullParser() @@ -1615,6 +1707,56 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started + + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) @@ -2535,6 +2677,7 @@ def __eq__(self, o): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) + @support.infinite_recursion(25) def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') @@ -3041,8 +3184,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) @@ -3946,7 +4088,7 @@ class BoolTest(unittest.TestCase): def test_warning(self): e = ET.fromstring('') msg = ( - r"Testing an element's truth value will raise an exception in " + r"Testing an element's truth value will always return True in " r"future versions. " r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") with self.assertWarnsRegex(DeprecationWarning, msg): diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 6c4b8384a3202e..2803c6d45c27bf 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -308,7 +308,7 @@ def test_get_host_info(self): def test_ssl_presence(self): try: - import ssl + import ssl # noqa: F401 except ImportError: has_ssl = False else: diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index f1c6b2d97621ee..00a5ed6626ddc5 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -265,14 +265,15 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(target), new_target, interpreter='python2.7') self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index 7050937738af18..b505dd7c376462 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -20,7 +20,7 @@ class TestComplexity(unittest.TestCase): @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), + compose(consume, zipfile._path.CompleteDirs._implied_dirs), lambda size: [ '/'.join(string.ascii_lowercase + str(n)) for n in range(size) ], @@ -43,13 +43,17 @@ def make_zip_path(self, depth=1, width=1) -> zipfile.Path: @classmethod def make_names(cls, width, letters=string.ascii_lowercase): """ + >>> list(TestComplexity.make_names(1)) + ['a'] >>> list(TestComplexity.make_names(2)) ['a', 'b'] >>> list(TestComplexity.make_names(30)) ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] """ # determine how many products are needed to produce width - n_products = math.ceil(math.log(width, len(letters))) + n_products = max(1, math.ceil(math.log(width, len(letters)))) inputs = (letters,) * n_products combinations = itertools.product(*inputs) names = map(''.join, combinations) @@ -80,7 +84,7 @@ def test_glob_depth(self): max_n=100, min_n=1, ) - assert best <= big_o.complexities.Quadratic + assert best <= big_o.complexities.Linear @pytest.mark.flaky def test_glob_width(self): diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index c66cb3cba69ebd..99842ffd63a64e 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -3,31 +3,27 @@ import contextlib import pathlib import pickle +import stat import sys import unittest import zipfile +import zipfile._path + +from test.support.os_helper import temp_dir, FakePath from ._functools import compose from ._itertools import Counter from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir - class jaraco: class itertools: Counter = Counter -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 def build_alpharep_fixture(): @@ -36,6 +32,7 @@ def build_alpharep_fixture(): . ├── a.txt + ├── n.txt (-> a.txt) ├── b │ ├── c.txt │ ├── d @@ -56,6 +53,7 @@ def build_alpharep_fixture(): - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) "alpha" because it uses alphabet "rep" because it's a representative example @@ -70,13 +68,16 @@ def build_alpharep_fixture(): zf.writestr("j/k.bin", b"content of k") zf.writestr("j/l.baz", b"content of l") zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + zf.filename = "alpharep.zip" return zf alpharep_generators = [ Invoked.wrap(build_alpharep_fixture), - Invoked.wrap(compose(add_dirs, build_alpharep_fixture)), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), ] pass_alpharep = parameterize(['alpharep'], alpharep_generators) @@ -100,7 +101,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -120,7 +121,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -135,7 +136,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -210,11 +211,12 @@ def test_open_write(self): with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: strm.write('text file') - def test_open_extant_directory(self): + @pass_alpharep + def test_open_extant_directory(self, alpharep): """ Attempting to open a directory raises IsADirectoryError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(IsADirectoryError): zf.joinpath('b').open() @@ -226,18 +228,19 @@ def test_open_binary_invalid_args(self, alpharep): with self.assertRaises(ValueError): root.joinpath('a.txt').open('rb', 'utf-8') - def test_open_missing_directory(self): + @pass_alpharep + def test_open_missing_directory(self, alpharep): """ Attempting to open a missing directory raises FileNotFoundError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(FileNotFoundError): zf.joinpath('z').open() @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -271,13 +274,13 @@ def test_pathlike_construction(self, alpharep): zipfile.Path should be constructable from a path-like object """ zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) + pathlike = FakePath(str(zipfile_ondisk)) zipfile.Path(pathlike) @pass_alpharep def test_traverse_pathlike(self, alpharep): root = zipfile.Path(alpharep) - root / pathlib.Path("a") + root / FakePath("a") @pass_alpharep def test_parent(self, alpharep): @@ -303,7 +306,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -520,12 +523,9 @@ def test_eq_hash(self, alpharep): @pass_alpharep def test_is_symlink(self, alpharep): - """ - See python/cpython#82102 for symlink support beyond this object. - """ - root = zipfile.Path(alpharep) - assert not root.is_symlink() + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() @pass_alpharep def test_relative_to(self, alpharep): @@ -546,12 +546,12 @@ def test_inheritance(self, alpharep): ['alpharep', 'path_type', 'subpath'], itertools.product( alpharep_generators, - [str, pathlib.Path], + [str, FakePath], ['', 'b/'], ), ) def test_pickle(self, alpharep, path_type, subpath): - zipfile_ondisk = path_type(self.zipfile_ondisk(alpharep)) + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) restored_1 = pickle.loads(saved_1) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 519d2ba1a231b5..423974aada4ac1 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4,7 +4,6 @@ import io import itertools import os -import pathlib import posixpath import struct import subprocess @@ -18,13 +17,14 @@ from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, captured_stdout, captured_stderr, requires_subprocess ) from test.support.os_helper import ( - TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath ) @@ -159,7 +159,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -315,7 +315,7 @@ def test_writestr_compresslevel(self): # Compression level follows the constructor. a_info = zipfp.getinfo('a.txt') self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compresslevel, 1) + self.assertEqual(a_info.compress_level, 1) # Compression level is overridden. b_info = zipfp.getinfo('b.txt') @@ -389,7 +389,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -408,7 +407,7 @@ def test_per_file_compresslevel(self): one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') self.assertEqual(one_info._compresslevel, 1) - self.assertEqual(nine_info._compresslevel, 9) + self.assertEqual(nine_info.compress_level, 9) def test_writing_errors(self): class BrokenFile(io.BytesIO): @@ -446,6 +445,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'rb') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'rb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -577,17 +597,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -1284,6 +1303,25 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1486,7 +1524,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1544,7 +1582,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1579,7 +1617,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1687,6 +1725,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -1754,7 +1819,7 @@ def test_unsupported_version(self): @requires_zlib() def test_read_unicode_filenames(self): # bug #10801 - fname = findfile('zip_cp437_header.zip') + fname = findfile('zip_cp437_header.zip', subdir='archivetestdata') with zipfile.ZipFile(fname) as zipfp: for name in zipfp.namelist(): zipfp.open(name).close() @@ -1865,7 +1930,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2246,6 +2311,22 @@ def test_seek_tell(self): fp.seek(0, os.SEEK_SET) self.assertEqual(fp.tell(), 0) + def test_read_after_seek(self): + # Issue 102956: Make sure seek(x, os.SEEK_CUR) doesn't break read() + txt = b"Charge men!" + bloc = txt.find(b"men") + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.read(6) + fp.seek(1, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' @@ -2256,6 +2337,66 @@ def test_decompress_without_3rd_party_library(self): with zipfile.ZipFile(zip_file) as zf: self.assertRaises(RuntimeError, zf.extract, 'a.txt') + @requires_zlib() + def test_full_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('a')), 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('b') + + @requires_zlib() + def test_quoted_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 52) + self.assertEqual(zi.file_size, 1064) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 36) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + self.assertEqual(len(zipf.read('b')), 1033) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) @@ -2788,7 +2929,7 @@ def setUp(self): os.mkdir(TESTFN2) def test_extract_dir(self): - with zipfile.ZipFile(findfile("zipdir.zip")) as zipf: + with zipfile.ZipFile(findfile("zipdir.zip", subdir="archivetestdata")) as zipf: zipf.extractall(TESTFN2) self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) @@ -2799,6 +2940,22 @@ def test_bug_6050(self): os.mkdir(os.path.join(TESTFN2, "a")) self.test_extract_dir() + def test_extract_dir_backslash(self): + zfname = findfile("zipdir_backslash.zip", subdir="archivetestdata") + with zipfile.ZipFile(zfname) as zipf: + zipf.extractall(TESTFN2) + if os.name == 'nt': + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a", "b", "c"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d", "e"))) + else: + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a\\b\\c"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "d\\e\\"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "a"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "d"))) + def test_write_dir(self): dirpath = os.path.join(TESTFN2, "x") os.mkdir(dirpath) @@ -2883,7 +3040,7 @@ def test_create_directory_with_write(self): directory = os.path.join(TESTFN2, "directory2") os.mkdir(directory) - mode = os.stat(directory).st_mode + mode = os.stat(directory).st_mode & 0xFFFF zf.write(directory, arcname="directory2/") zinfo = zf.filelist[1] self.assertEqual(zinfo.filename, "directory2/") @@ -2895,6 +3052,17 @@ def test_create_directory_with_write(self): self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): @@ -2909,7 +3077,7 @@ def test_from_file(self): self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) @@ -2935,6 +3103,17 @@ def test_from_dir(self): self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) self.assertEqual(zi.file_size, 0) + def test_compresslevel_property(self): + zinfo = zipfile.ZipInfo("xxx") + self.assertFalse(zinfo._compresslevel) + self.assertFalse(zinfo.compress_level) + zinfo._compresslevel = 99 # test the legacy @property.setter + self.assertEqual(zinfo.compress_level, 99) + self.assertEqual(zinfo._compresslevel, 99) + zinfo.compress_level = 8 + self.assertEqual(zinfo.compress_level, 8) + self.assertEqual(zinfo._compresslevel, 8) + class CommandLineTest(unittest.TestCase): @@ -2957,7 +3136,7 @@ def test_bad_use(self): self.assertNotEqual(err.strip(), b'') def test_test_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') for opt in '-t', '--test': out = self.zipfilecmd(opt, zip_name) self.assertEqual(out.rstrip(), b'Done testing') @@ -2966,7 +3145,7 @@ def test_test_command(self): self.assertEqual(out, b'') def test_list_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') t = io.StringIO() with zipfile.ZipFile(zip_name, 'r') as tf: tf.printdir(t) @@ -2999,7 +3178,7 @@ def test_create_command(self): unlink(TESTFN2) def test_extract_command(self): - zip_name = findfile('zipdir.zip') + zip_name = findfile('zipdir.zip', subdir='archivetestdata') for opt in '-e', '--extract': with temp_dir() as extdir: out = self.zipfilecmd(opt, zip_name, extdir) @@ -3020,8 +3199,8 @@ class TestExecutablePrependedZip(unittest.TestCase): """Test our ability to open zip files with an executable prepended.""" def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') + self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') + self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') def _test_zip_works(self, name): # bpo28494 sanity check: ensure is_zipfile works on these. diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index c12798d221e9b7..1861616d5ec3bf 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -50,10 +52,14 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -92,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -104,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -128,20 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + + def getZip64Files(self): + # This is the simplest way to make zipfile generate the zip64 EOCD block + return {f"f{n}.py": test_src for n in range(65537)} + def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -167,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -178,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -220,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -248,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -271,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -282,29 +313,104 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) + def testPackageExplicitDirectories(self): + # Test explicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.writestr('a/__init__.py', test_src) + z.mkdir('a/b') + z.writestr('a/b/__init__.py', test_src) + z.mkdir('a/b/c') + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testPackageImplicitDirectories(self): + # Test explicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/__init__.py', test_src) + z.writestr('a/b/__init__.py', test_src) + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testNamespacePackageExplicitDirectories(self): + # Test implicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def testNamespacePackageImplicitDirectories(self): + # Test implicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def _testPackage(self, initfile): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a')) + if initfile is None: + # XXX Should it work? + self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b') + else: + self.assertTrue(zi.is_package('b')) + self.assertEqual(zi.get_source('b'), test_src) + self.assertEqual(zi.get_code('b').co_filename, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + + sys.path.insert(0, TEMP_ZIP) + self.assertNotIn('a', sys.modules) + + mod = importlib.import_module(f'a.b') + self.assertIn('a', sys.modules) + self.assertIs(sys.modules['a.b'], mod) + if initfile is None: + self.assertIsNone(mod.__file__) + else: + self.assertEqual(mod.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + self.assertEqual(len(mod.__path__), 1, mod.__path__) + self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b')) + + mod2 = importlib.import_module(f'a.b.c.d') + self.assertIn('a.b.c', sys.modules) + self.assertIn('a.b.c.d', sys.modules) + self.assertIs(sys.modules['a.b.c.d'], mod2) + self.assertIs(mod.c.d, mod2) + self.assertEqual(mod2.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py')) + def testMixedNamespacePackage(self): # Test implicit namespace packages spread between a # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -337,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -355,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -369,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -413,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -429,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -507,35 +606,26 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Check that the file information remains accurate after reloading zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) spec = zi.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -549,36 +639,27 @@ def testInvalidateCaches(self): def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Zipimporter for the same path. zi2 = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive to make the cache wrong. - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Invalidate the cache of the first zipimporter. zi.invalidate_caches() # Check that the second zipimporter detects the new file and isn't using a stale cache. - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) spec = zi2.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -586,16 +667,9 @@ def testInvalidateCachesWithMultipleZipimports(self): def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -641,17 +715,33 @@ def testZipImporterMethodsInSubDirectory(self): self.assertIsNone(loader.get_source(mod_name)) self.assertEqual(loader.get_filename(mod_name), mod.__file__) - def testGetData(self): + def testGetDataExplicitDirectories(self): self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - z.compression = self.compression - name = "testdata.dat" - data = bytes(x for x in range(256)) - z.writestr(name, data) - - zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(data, zi.get_data(name)) - self.assertIn('zipimporter object', repr(zi)) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def testGetDataImplicitDirectories(self): + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def _testGetData(self): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored')) + pathname = os.path.join('a', 'b', 'c', 'testdata.dat') + data = bytes(range(256)) + self.assertEqual(zi.get_data(pathname), data) + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data) + self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'') + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'') + self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b')) + self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b')) def testImporterAttr(self): src = """if 1: # indent hack @@ -660,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -670,7 +760,7 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) @@ -683,7 +773,7 @@ def test(val): def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -691,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -754,29 +844,21 @@ def doTraceback(self, module): raise AssertionError("This ought to be impossible") def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) with self.assertRaises(TypeError): @@ -787,17 +869,143 @@ def testBytesPath(self): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + @support.requires_resource('cpu') + def testZip64(self): + files = self.getZip64Files() + self.doTest(".py", files, "f6") + + @support.requires_resource('cpu') + def testZip64CruftAndComment(self): + files = self.getZip64Files() + self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do alot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 7bf50a33728e53..ae8a8c99762313 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -29,8 +29,9 @@ # test_cmd_line_script (covers the zipimport support in runpy) # Retrieve some helpers from other test cases -from test import (test_doctest, sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings) +from test.test_doctest import (test_doctest, + sample_doctest, sample_doctest_no_doctests, + sample_doctest_no_docstrings, sample_doctest_skip) def _run_object_doctest(obj, module): @@ -100,18 +101,18 @@ def test_doctest_issue4197(self): # everything still works correctly test_src = inspect.getsource(test_doctest) test_src = test_src.replace( - "from test import test_doctest", + "from test.test_doctest import test_doctest", "import test_zipped_doctest as test_doctest") - test_src = test_src.replace("test.test_doctest", + test_src = test_src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") - test_src = test_src.replace("test.sample_doctest", + test_src = test_src.replace("test.test_doctest.sample_doctest", "sample_zipped_doctest") # The sample doctest files rewritten to include in the zipped version. sample_sources = {} for mod in [sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings]: + sample_doctest_no_docstrings, sample_doctest_skip]: src = inspect.getsource(mod) - src = src.replace("test.test_doctest", "test_zipped_doctest") + src = src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") # Rewrite the module name so that, for example, # "test.sample_doctest" becomes "sample_zipped_doctest". mod_name = mod.__name__.split(".")[-1] diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 1dc8b91a453f92..ef02c64f886f8a 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -18,6 +18,21 @@ hasattr(zlib.decompressobj(), "copy"), 'requires Decompress.copy()') + +def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): + # Register "1.2.3" as "1.2.3.0" + # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" + v = zlib_version.split('-', 1)[0].split('.') + if len(v) < 4: + v.append('0') + elif not v[-1].isnumeric(): + v[-1] = '0' + return tuple(map(int, v)) + + +ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() + + # bpo-46623: On s390x, when a hardware accelerator is used, using different # ways to compress data with zlib can produce different compressed data. # Simplified test_pair() code: @@ -473,9 +488,8 @@ def test_flushes(self): sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', 'Z_PARTIAL_FLUSH'] - ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) # Z_BLOCK has a known failure prior to 1.2.5.3 - if ver >= (1, 2, 5, 3): + if ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 5, 3): sync_opt.append('Z_BLOCK') sync_opt = [getattr(zlib, opt) for opt in sync_opt @@ -793,16 +807,7 @@ def test_large_unconsumed_tail(self, size): def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 - # Register "1.2.3" as "1.2.3.0" - # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" - v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.') - if len(v) < 4: - v.append('0') - elif not v[-1].isnumeric(): - v[-1] = '0' - - v = tuple(map(int, v)) - supports_wbits_0 = v >= (1, 2, 3, 5) + supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5) co = zlib.compressobj(level=1, wbits=15) zlib15 = co.compress(HAMLET_SCENE) + co.flush() diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index c3ea567103275d..4b16ecc31156a5 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1,2 +1,5 @@ -from .test_zoneinfo import * -from .test_zoneinfo_property import * +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index ae921f7432c466..8bcd6d2e9951b9 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -17,9 +17,10 @@ from datetime import date, datetime, time, timedelta, timezone from functools import cached_property +from test.support import MISSING_C_DOCSTRINGS, requires_gil_enabled from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase -from test.support.import_helper import import_module +from test.support.import_helper import import_module, CleanImport lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -35,6 +36,7 @@ TEMP_DIR = None DATA_DIR = pathlib.Path(__file__).parent / "data" ZONEINFO_JSON = DATA_DIR / "zoneinfo_data.json" +DRIVE = os.path.splitdrive('x:')[0] # Useful constants ZERO = timedelta(0) @@ -404,6 +406,8 @@ def test_time_fixed_offset(self): class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signatures(self): """Ensure that C module has valid method signatures.""" import inspect @@ -1001,6 +1005,80 @@ def test_tzstr_from_utc(self): self.assertEqual(dt_act, dt_utc) + def test_extreme_tzstr(self): + tzstrs = [ + # Extreme offset hour + "AAA24", + "AAA+24", + "AAA-24", + "AAA24BBB,J60/2,J300/2", + "AAA+24BBB,J60/2,J300/2", + "AAA-24BBB,J60/2,J300/2", + "AAA4BBB24,J60/2,J300/2", + "AAA4BBB+24,J60/2,J300/2", + "AAA4BBB-24,J60/2,J300/2", + # Extreme offset minutes + "AAA4:00BBB,J60/2,J300/2", + "AAA4:59BBB,J60/2,J300/2", + "AAA4BBB5:00,J60/2,J300/2", + "AAA4BBB5:59,J60/2,J300/2", + # Extreme offset seconds + "AAA4:00:00BBB,J60/2,J300/2", + "AAA4:00:59BBB,J60/2,J300/2", + "AAA4BBB5:00:00,J60/2,J300/2", + "AAA4BBB5:00:59,J60/2,J300/2", + # Extreme total offset + "AAA24:59:59BBB5,J60/2,J300/2", + "AAA-24:59:59BBB5,J60/2,J300/2", + "AAA4BBB24:59:59,J60/2,J300/2", + "AAA4BBB-24:59:59,J60/2,J300/2", + # Extreme months + "AAA4BBB,M12.1.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M12.1.1/2", + # Extreme weeks + "AAA4BBB,M1.5.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M1.5.1/2", + # Extreme weekday + "AAA4BBB,M1.1.6/2,M2.1.1/2", + "AAA4BBB,M1.1.1/2,M2.1.6/2", + # Extreme numeric offset + "AAA4BBB,0/2,20/2", + "AAA4BBB,0/2,0/14", + "AAA4BBB,20/2,365/2", + "AAA4BBB,365/2,365/14", + # Extreme julian offset + "AAA4BBB,J1/2,J20/2", + "AAA4BBB,J1/2,J1/14", + "AAA4BBB,J20/2,J365/2", + "AAA4BBB,J365/2,J365/14", + # Extreme transition hour + "AAA4BBB,J60/167,J300/2", + "AAA4BBB,J60/+167,J300/2", + "AAA4BBB,J60/-167,J300/2", + "AAA4BBB,J60/2,J300/167", + "AAA4BBB,J60/2,J300/+167", + "AAA4BBB,J60/2,J300/-167", + # Extreme transition minutes + "AAA4BBB,J60/2:00,J300/2", + "AAA4BBB,J60/2:59,J300/2", + "AAA4BBB,J60/2,J300/2:00", + "AAA4BBB,J60/2,J300/2:59", + # Extreme transition seconds + "AAA4BBB,J60/2:00:00,J300/2", + "AAA4BBB,J60/2:00:59,J300/2", + "AAA4BBB,J60/2,J300/2:00:00", + "AAA4BBB,J60/2,J300/2:00:59", + # Extreme total transition time + "AAA4BBB,J60/167:59:59,J300/2", + "AAA4BBB,J60/-167:59:59,J300/2", + "AAA4BBB,J60/2,J300/167:59:59", + "AAA4BBB,J60/2,J300/-167:59:59", + ] + + for tzstr in tzstrs: + with self.subTest(tzstr=tzstr): + self.zone_from_tzstr(tzstr) + def test_invalid_tzstr(self): invalid_tzstrs = [ "PST8PDT", # DST but no transition specified @@ -1008,16 +1086,33 @@ def test_invalid_tzstr(self): "GMT,M3.2.0/2,M11.1.0/3", # Transition rule but no DST "GMT0+11,M3.2.0/2,M11.1.0/3", # Unquoted alphanumeric in DST "PST8PDT,M3.2.0/2", # Only one transition rule - # Invalid offsets - "STD+25", - "STD-25", - "STD+374", - "STD+374DST,M3.2.0/2,M11.1.0/3", - "STD+23DST+25,M3.2.0/2,M11.1.0/3", - "STD-23DST-25,M3.2.0/2,M11.1.0/3", + # Invalid offset hours + "AAA168", + "AAA+168", + "AAA-168", + "AAA168BBB,J60/2,J300/2", + "AAA+168BBB,J60/2,J300/2", + "AAA-168BBB,J60/2,J300/2", + "AAA4BBB168,J60/2,J300/2", + "AAA4BBB+168,J60/2,J300/2", + "AAA4BBB-168,J60/2,J300/2", + # Invalid offset minutes + "AAA4:0BBB,J60/2,J300/2", + "AAA4:100BBB,J60/2,J300/2", + "AAA4BBB5:0,J60/2,J300/2", + "AAA4BBB5:100,J60/2,J300/2", + # Invalid offset seconds + "AAA4:00:0BBB,J60/2,J300/2", + "AAA4:00:100BBB,J60/2,J300/2", + "AAA4BBB5:00:0,J60/2,J300/2", + "AAA4BBB5:00:100,J60/2,J300/2", # Completely invalid dates "AAA4BBB,M1443339,M11.1.0/3", "AAA4BBB,M3.2.0/2,0349309483959c", + "AAA4BBB,,J300/2", + "AAA4BBB,z,J300/2", + "AAA4BBB,J60/2,", + "AAA4BBB,J60/2,z", # Invalid months "AAA4BBB,M13.1.1/2,M1.1.1/2", "AAA4BBB,M1.1.1/2,M13.1.1/2", @@ -1037,6 +1132,26 @@ def test_invalid_tzstr(self): # Invalid julian offset "AAA4BBB,J0/2,J20/2", "AAA4BBB,J20/2,J366/2", + # Invalid transition time + "AAA4BBB,J60/2/3,J300/2", + "AAA4BBB,J60/2,J300/2/3", + # Invalid transition hour + "AAA4BBB,J60/168,J300/2", + "AAA4BBB,J60/+168,J300/2", + "AAA4BBB,J60/-168,J300/2", + "AAA4BBB,J60/2,J300/168", + "AAA4BBB,J60/2,J300/+168", + "AAA4BBB,J60/2,J300/-168", + # Invalid transition minutes + "AAA4BBB,J60/2:0,J300/2", + "AAA4BBB,J60/2:100,J300/2", + "AAA4BBB,J60/2,J300/2:0", + "AAA4BBB,J60/2,J300/2:100", + # Invalid transition seconds + "AAA4BBB,J60/2:00:0,J300/2", + "AAA4BBB,J60/2:00:100,J300/2", + "AAA4BBB,J60/2,J300/2:00:0", + "AAA4BBB,J60/2,J300/2:00:100", ] for invalid_tzstr in invalid_tzstrs: @@ -1565,8 +1680,8 @@ def test_env_variable(self): """Tests that the environment variable works with reset_tzpath.""" new_paths = [ ("", []), - ("/etc/zoneinfo", ["/etc/zoneinfo"]), - (f"/a/b/c{os.pathsep}/d/e/f", ["/a/b/c", "/d/e/f"]), + (f"{DRIVE}/etc/zoneinfo", [f"{DRIVE}/etc/zoneinfo"]), + (f"{DRIVE}/a/b/c{os.pathsep}{DRIVE}/d/e/f", [f"{DRIVE}/a/b/c", f"{DRIVE}/d/e/f"]), ] for new_path_var, expected_result in new_paths: @@ -1580,22 +1695,22 @@ def test_env_variable_relative_paths(self): test_cases = [ [("path/to/somewhere",), ()], [ - ("/usr/share/zoneinfo", "path/to/somewhere",), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo", "path/to/somewhere",), + (f"{DRIVE}/usr/share/zoneinfo",), ], [("../relative/path",), ()], [ - ("/usr/share/zoneinfo", "../relative/path",), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo", "../relative/path",), + (f"{DRIVE}/usr/share/zoneinfo",), ], [("path/to/somewhere", "../relative/path",), ()], [ ( - "/usr/share/zoneinfo", + f"{DRIVE}/usr/share/zoneinfo", "path/to/somewhere", "../relative/path", ), - ("/usr/share/zoneinfo",), + (f"{DRIVE}/usr/share/zoneinfo",), ], ] @@ -1605,17 +1720,30 @@ def test_env_variable_relative_paths(self): with self.subTest("warning", path_var=path_var): # Note: Per PEP 615 the warning is implementation-defined # behavior, other implementations need not warn. - with self.assertWarns(self.module.InvalidTZPathWarning): + with self.assertWarns(self.module.InvalidTZPathWarning) as w: self.module.reset_tzpath() + self.assertEqual(w.warnings[0].filename, __file__) tzpath = self.module.TZPATH with self.subTest("filtered", path_var=path_var): self.assertSequenceEqual(tzpath, expected_paths) + def test_env_variable_relative_paths_warning_location(self): + path_var = "path/to/somewhere" + + with self.python_tzpath_context(path_var): + with CleanImport("zoneinfo", "zoneinfo._tzpath"): + with self.assertWarns(RuntimeWarning) as w: + import zoneinfo + InvalidTZPathWarning = zoneinfo.InvalidTZPathWarning + self.assertIsInstance(w.warnings[0].message, InvalidTZPathWarning) + # It should represent the current file: + self.assertEqual(w.warnings[0].filename, __file__) + def test_reset_tzpath_kwarg(self): - self.module.reset_tzpath(to=["/a/b/c"]) + self.module.reset_tzpath(to=[f"{DRIVE}/a/b/c"]) - self.assertSequenceEqual(self.module.TZPATH, ("/a/b/c",)) + self.assertSequenceEqual(self.module.TZPATH, (f"{DRIVE}/a/b/c",)) def test_reset_tzpath_relative_paths(self): bad_values = [ @@ -1644,8 +1772,8 @@ def test_tzpath_type_error(self): self.module.reset_tzpath(bad_value) def test_tzpath_attribute(self): - tzpath_0 = ["/one", "/two"] - tzpath_1 = ["/three"] + tzpath_0 = [f"{DRIVE}/one", f"{DRIVE}/two"] + tzpath_1 = [f"{DRIVE}/three"] with self.tzpath_context(tzpath_0): query_0 = self.module.TZPATH @@ -1803,6 +1931,7 @@ def test_cache_location(self): self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + @requires_gil_enabled("gh-117783: types may be immortalized") def test_gc_tracked(self): import gc diff --git a/Lib/test/time_hashlib.py b/Lib/test/time_hashlib.py deleted file mode 100644 index 55ebac62912fe1..00000000000000 --- a/Lib/test/time_hashlib.py +++ /dev/null @@ -1,88 +0,0 @@ -# It's intended that this script be run by hand. It runs speed tests on -# hashlib functions; it does not test for correctness. - -import sys -import time -import hashlib - - -def creatorFunc(): - raise RuntimeError("eek, creatorFunc not overridden") - -def test_scaled_msg(scale, name): - iterations = 106201//scale * 20 - longStr = b'Z'*scale - - localCF = creatorFunc - start = time.perf_counter() - for f in range(iterations): - x = localCF(longStr).digest() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", iterations, "x", len(longStr), "bytes", name) - -def test_create(): - start = time.perf_counter() - for f in range(20000): - d = creatorFunc() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", '[20000 creations]') - -def test_zero(): - start = time.perf_counter() - for f in range(20000): - x = creatorFunc().digest() - end = time.perf_counter() - - print(('%2.2f' % (end-start)), "seconds", '[20000 "" digests]') - - - -hName = sys.argv[1] - -# -# setup our creatorFunc to test the requested hash -# -if hName in ('_md5', '_sha'): - exec('import '+hName) - exec('creatorFunc = '+hName+'.new') - print("testing speed of old", hName, "legacy interface") -elif hName == '_hashlib' and len(sys.argv) > 3: - import _hashlib - exec('creatorFunc = _hashlib.%s' % sys.argv[2]) - print("testing speed of _hashlib.%s" % sys.argv[2], getattr(_hashlib, sys.argv[2])) -elif hName == '_hashlib' and len(sys.argv) == 3: - import _hashlib - exec('creatorFunc = lambda x=_hashlib.new : x(%r)' % sys.argv[2]) - print("testing speed of _hashlib.new(%r)" % sys.argv[2]) -elif hasattr(hashlib, hName) and hasattr(getattr(hashlib, hName), '__call__'): - creatorFunc = getattr(hashlib, hName) - print("testing speed of hashlib."+hName, getattr(hashlib, hName)) -else: - exec("creatorFunc = lambda x=hashlib.new : x(%r)" % hName) - print("testing speed of hashlib.new(%r)" % hName) - -try: - test_create() -except ValueError: - print() - print("pass argument(s) naming the hash to run a speed test on:") - print(" '_md5' and '_sha' test the legacy builtin md5 and sha") - print(" '_hashlib' 'openssl_hName' 'fast' tests the builtin _hashlib") - print(" '_hashlib' 'hName' tests builtin _hashlib.new(shaFOO)") - print(" 'hName' tests the hashlib.hName() implementation if it exists") - print(" otherwise it uses hashlib.new(hName).") - print() - raise - -test_zero() -test_scaled_msg(scale=106201, name='[huge data]') -test_scaled_msg(scale=10620, name='[large data]') -test_scaled_msg(scale=1062, name='[medium data]') -test_scaled_msg(scale=424, name='[4*small data]') -test_scaled_msg(scale=336, name='[3*small data]') -test_scaled_msg(scale=212, name='[2*small data]') -test_scaled_msg(scale=106, name='[small data]') -test_scaled_msg(scale=creatorFunc().digest_size, name='[digest_size data]') -test_scaled_msg(scale=10, name='[tiny data]') diff --git a/Lib/test/_typed_dict_helper.py b/Lib/test/typinganndata/_typed_dict_helper.py similarity index 100% rename from Lib/test/_typed_dict_helper.py rename to Lib/test/typinganndata/_typed_dict_helper.py diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 00000000000000..b6f3b06bd5065f --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from typing import get_type_hints + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + hints_for_E=get_type_hints(E), + hints_for_E_meth=get_type_hints(E.generic_method), + generic_func=generic_function, + hints_for_generic_func=get_type_hints(generic_function) + ) diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/typinganndata/mod_generics_cache.py similarity index 100% rename from Lib/test/mod_generics_cache.py rename to Lib/test/typinganndata/mod_generics_cache.py diff --git a/Lib/test/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheel-0.40.0-py3-none-any.whl deleted file mode 100644 index 410132385bba4d..00000000000000 Binary files a/Lib/test/wheel-0.40.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/setuptools-67.6.1-py3-none-any.whl b/Lib/test/wheeldata/setuptools-67.6.1-py3-none-any.whl similarity index 100% rename from Lib/test/setuptools-67.6.1-py3-none-any.whl rename to Lib/test/wheeldata/setuptools-67.6.1-py3-none-any.whl diff --git a/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl new file mode 100644 index 00000000000000..67e2308717d675 Binary files /dev/null and b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000000..c6beae8e2552d6 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000000..74ab03b4648948 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000000..9769a404f675d4 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/threading.py b/Lib/threading.py index 41c3a9ff93856f..31ab77c92b1c20 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,7 +3,6 @@ import os as _os import sys as _sys import _thread -import functools import warnings from time import monotonic as _time @@ -33,11 +32,15 @@ 'setprofile_all_threads','settrace_all_threads'] # Rename some stuff so "from threading import *" is safe -_start_new_thread = _thread.start_new_thread +_start_joinable_thread = _thread.start_joinable_thread _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock -_set_sentinel = _thread._set_sentinel +_LockType = _thread.LockType +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident _is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id @@ -53,6 +56,13 @@ TIMEOUT_MAX = _thread.TIMEOUT_MAX del _thread +# get thread-local implementation, either from the thread +# module, or from the python fallback + +try: + from _thread import _local as local +except ImportError: + from _threading_local import local # Support for profile and trace hooks @@ -107,7 +117,7 @@ def gettrace(): # Synchronization classes -Lock = _allocate_lock +Lock = _LockType def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. @@ -589,7 +599,7 @@ def __repr__(self): return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: {status}>" def _at_fork_reinit(self): - # Private method called by Thread._reset_internal_locks() + # Private method called by Thread._after_fork() self._cond._at_fork_reinit() def is_set(self): @@ -840,25 +850,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -923,9 +914,8 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None + self._handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -933,30 +923,24 @@ class is implemented. # For debugging and _after_fork() _dangling.add(self) - def _reset_internal_locks(self, is_alive): - # private! Called by _after_fork() to reset our internal locks as - # they may be in an invalid state leading to a deadlock or crash. + def _after_fork(self, new_ident=None): + # Private! Called by threading._after_fork(). self._started._at_fork_reinit() - if is_alive: - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() + if new_ident is not None: + # This thread is alive. + self._ident = new_ident + assert self._handle.ident == new_ident else: - # The thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -983,12 +967,14 @@ def start(self): with _active_limbo_lock: _limbo[self] = self try: - _start_new_thread(self._bootstrap, ()) + # Start joinable thread + _start_joinable_thread(self._bootstrap, handle=self._handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] raise - self._started.wait() + self._started.wait() # Will set ident and native_id def run(self): """Method representing the thread's activity. @@ -1034,23 +1020,9 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) - def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() self._started.set() @@ -1070,33 +1042,6 @@ def _bootstrap_inner(self): finally: self._delete() - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() - def _delete(self): "Remove current thread from the dict of currently running threads." with _active_limbo_lock: @@ -1137,39 +1082,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._handle.join(timeout) @property def name(self): @@ -1220,10 +1138,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - return not self._is_stopped + return self._started.is_set() and not self._handle.is_done() @property def daemon(self): @@ -1432,19 +1347,45 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self +# Helper thread-local instance to detect when a _DummyThread +# is collected. Not a part of the public API. +_thread_local_info = local() + + +class _DeleteDummyThreadOnDel: + ''' + Helper class to remove a dummy thread from threading._active on __del__. + ''' + + def __init__(self, dummy_thread): + self._dummy_thread = dummy_thread + self._tident = dummy_thread.ident + # Put the thread on a thread local variable so that when + # the related thread finishes this instance is collected. + # + # Note: no other references to this instance may be created. + # If any client code creates a reference to this instance, + # the related _DummyThread will be kept forever! + _thread_local_info._track_dummy_thread_ref = self + + def __del__(self): + with _active_limbo_lock: + if _active.get(self._tident) is self._dummy_thread: + _active.pop(self._tident, None) + + # Dummy thread class to represent threads not started here. -# These aren't garbage collected when they die, nor can they be waited for. -# If they invoke anything in threading.py that calls current_thread(), they -# leave an entry in the _active dict forever after. +# These should be added to `_active` and removed automatically +# when they die, although they can't be waited for. # Their purpose is to return *something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). @@ -1454,25 +1395,30 @@ class _DummyThread(Thread): def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=_daemon_threads_allowed()) - self._started.set() self._set_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self - - def _stop(self): - pass + _DeleteDummyThreadOnDel(self) def is_alive(self): - if not self._is_stopped and self._started.is_set(): + if not self._handle.is_done() and self._started.is_set(): return True raise RuntimeError("thread is not alive") def join(self, timeout=None): raise RuntimeError("cannot join a dummy thread") + def _after_fork(self, new_ident=None): + if new_ident is not None: + self.__class__ = _MainThread + self._name = 'MainThread' + self._daemonic = False + Thread._after_fork(self, new_ident=new_ident) + # Global API functions @@ -1554,8 +1500,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size @@ -1570,12 +1515,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped and _is_main_interpreter(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1587,42 +1531,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - if tlock.locked(): - # It should have been released already by - # _PyInterpreterState_SetNotRunningMain(), but there may be - # embedders that aren't calling that yet. - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1634,14 +1547,6 @@ def main_thread(): # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread -# get thread-local implementation, either from the thread -# module, or from the python fallback - -try: - from _thread import _local as local -except ImportError: - from _threading_local import local - def _after_fork(): """ @@ -1650,7 +1555,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1666,10 +1570,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1679,16 +1579,13 @@ def _after_fork(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: - # There is only one active thread. We reset the ident to - # its new value since it can have changed. - thread._reset_internal_locks(True) + # This is the one and only active thread. ident = get_ident() - thread._ident = ident + thread._after_fork(new_ident=ident) new_active[ident] = thread else: # All the others are already stopped. - thread._reset_internal_locks(False) - thread._stop() + thread._after_fork() _limbo.clear() _active.clear() diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index 02cfafaf36e5d1..c106e0f67356da --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 440e7f100c8c47..5352276e874bf5 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -41,6 +41,7 @@ import re wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) TclVersion = float(_tkinter.TCL_VERSION) @@ -69,7 +70,10 @@ def _stringify(value): else: value = '{%s}' % _join(value) else: - value = str(value) + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) if not value: value = '{}' elif _magic_re.search(value): @@ -411,7 +415,6 @@ def __del__(self): self._tk.globalunsetvar(self._name) if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self._tk.deletecommand(name) self._tclCommands = None @@ -683,7 +686,6 @@ def destroy(self): this widget in the Tcl interpreter.""" if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) self._tclCommands = None @@ -691,7 +693,6 @@ def deletecommand(self, name): """Internal function. Delete the Tcl command provided in NAME.""" - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) try: self._tclCommands.remove(name) @@ -897,6 +898,21 @@ def after_cancel(self, id): pass self.tk.call('after', 'cancel', id) + def after_info(self, id=None): + """Return information about existing event handlers. + + With no argument, return a tuple of the identifiers for all existing + event handlers created by the after and after_idle commands for this + interpreter. If id is supplied, it specifies an existing handler; id + must have been the return value from some previous call to after or + after_idle and it must not have triggered yet or been canceled. If the + id doesn't exist, a TclError is raised. Otherwise, the return value is + a tuple containing (script, type) where script is a reference to the + function to be called by the event handler and type is either 'idle' + or 'timer' to indicate what kind of event handler it is. + """ + return self.tk.splitlist(self.tk.call('after', 'info', id)) + def bell(self, displayof=0): """Ring a display's bell.""" self.tk.call(('bell',) + self._displayof(displayof)) @@ -1260,6 +1276,8 @@ def winfo_parent(self): def winfo_pathname(self, id, displayof=0): """Return the pathname of the widget given by ID.""" + if isinstance(id, int): + id = hex(id) args = ('winfo', 'pathname') \ + self._displayof(displayof) + (id,) return self.tk.call(args) @@ -1527,10 +1545,27 @@ def bind(self, sequence=None, func=None, add=None): return self._bind(('bind', self._w), sequence, func, add) def unbind(self, sequence, funcid=None): - """Unbind for this widget for event SEQUENCE the - function identified with FUNCID.""" - self.tk.call('bind', self._w, sequence, '') - if funcid: + """Unbind for this widget the event SEQUENCE. + + If FUNCID is given, only unbind the function identified with FUNCID + and also delete the corresponding Tcl command. + + Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE + unbound. + """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): + if funcid is None: + self.tk.call(*what, '') + else: + lines = self.tk.call(what).split('\n') + prefix = f'if {{"[{funcid} ' + keep = '\n'.join(line for line in lines + if not line.startswith(prefix)) + if not keep.strip(): + keep = '' + self.tk.call(*what, keep) self.deletecommand(funcid) def bind_all(self, sequence=None, func=None, add=None): @@ -1538,11 +1573,11 @@ def bind_all(self, sequence=None, func=None, add=None): An additional boolean parameter ADD specifies whether FUNC will be called additionally to the other bound function or whether it will replace the previous function. See bind for the return value.""" - return self._bind(('bind', 'all'), sequence, func, add, 0) + return self._root()._bind(('bind', 'all'), sequence, func, add, True) def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" - self.tk.call('bind', 'all' , sequence, '') + self._root()._unbind(('bind', 'all', sequence)) def bind_class(self, className, sequence=None, func=None, add=None): """Bind to widgets with bindtag CLASSNAME at event @@ -1552,12 +1587,12 @@ def bind_class(self, className, sequence=None, func=None, add=None): whether it will replace the previous function. See bind for the return value.""" - return self._bind(('bind', className), sequence, func, add, 0) + return self._root()._bind(('bind', className), sequence, func, add, True) def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE all functions.""" - self.tk.call('bind', className , sequence, '') + self._root()._unbind(('bind', className, sequence)) def mainloop(self, n=0): """Call the mainloop of Tk.""" @@ -1692,6 +1727,9 @@ def getint_event(s): except (ValueError, TclError): return s + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args # Missing: (a, c, d, m, o, v, B, R) e = Event() @@ -1727,7 +1765,10 @@ def getint_event(s): try: e.type = EventType(T) except ValueError: - e.type = T + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T try: e.widget = self._nametowidget(W) except KeyError: @@ -2089,26 +2130,39 @@ def wm_aspect(self, aspect = wm_aspect - def wm_attributes(self, *args): - """This subcommand returns or sets platform specific attributes - - The first form returns a list of the platform specific flags and - their values. The second form returns the value for the specific - option. The third form sets one or more of the values. The values - are as follows: - - On Windows, -disabled gets or sets whether the window is in a - disabled state. -toolwindow gets or sets the style of the window - to toolwindow (as defined in the MSDN). -topmost gets or sets - whether this is a topmost window (displays above all other - windows). + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. - On Macintosh, XXXXX + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. - On Unix, there are currently no special attribute values. + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. """ - args = ('wm', 'attributes', self._w) + args - return self.tk.call(args) + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) attributes = wm_attributes @@ -2403,6 +2457,8 @@ def __init__(self, screenName=None, baseName=None, className='Tk', baseName = baseName + ext interactive = False self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) if useTk: self._loadtk() if not sys.flags.ignore_environment: @@ -2489,6 +2545,14 @@ def __getattr__(self, attr): "Delegate attribute access to the interpreter object" return getattr(self.tk, attr) + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + # Ideally, the classes Pack, Place and Grid disappear, the # pack/place/grid methods are defined on the Widget class, and # everybody uses w.pack_whatever(...) instead of Pack.whatever(w, @@ -2869,9 +2933,7 @@ def bbox(self, *args): def tag_unbind(self, tagOrId, sequence, funcid=None): """Unbind for all items with TAGORID for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'bind', tagOrId, sequence, '') - if funcid: - self.deletecommand(funcid) + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) def tag_bind(self, tagOrId, sequence=None, func=None, add=None): """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. @@ -3136,11 +3198,16 @@ def __init__(self, master=None, cnf={}, **kw): Widget.__init__(self, master, 'checkbutton', cnf, kw) def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. if not cnf.get('name'): global _checkbutton_count name = self.__class__.__name__.lower() _checkbutton_count += 1 - cnf['name'] = f'!{name}{_checkbutton_count}' + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' super()._setup(master, cnf) def deselect(self): @@ -3715,25 +3782,35 @@ def compare(self, index1, op, index2): return self.tk.getboolean(self.tk.call( self._w, 'compare', index1, op, index2)) - def count(self, index1, index2, *args): # new in Tk 8.5 + def count(self, index1, index2, *options, return_ints=False): # new in Tk 8.5 """Counts the number of relevant things between the two indices. - If index1 is after index2, the result will be a negative number + + If INDEX1 is after INDEX2, the result will be a negative number (and this holds for each of the possible options). - The actual items which are counted depends on the options given by - args. The result is a list of integers, one for the result of each - counting option given. Valid counting options are "chars", - "displaychars", "displayindices", "displaylines", "indices", - "lines", "xpixels" and "ypixels". There is an additional possible - option "update", which if given then all subsequent options ensure - that any possible out of date information is recalculated.""" - args = ['-%s' % arg for arg in args] - args += [index1, index2] - res = self.tk.call(self._w, 'count', *args) or None - if res is not None and len(args) <= 3: - return (res, ) - else: - return res + The actual items which are counted depends on the options given. + The result is a tuple of integers, one for the result of each + counting option given, if more than one option is specified or + return_ints is false (default), otherwise it is an integer. + Valid counting options are "chars", "displaychars", + "displayindices", "displaylines", "indices", "lines", "xpixels" + and "ypixels". The default value, if no option is specified, is + "indices". There is an additional possible option "update", + which if given then all subsequent options ensure that any + possible out of date information is recalculated. + """ + options = ['-%s' % arg for arg in options] + res = self.tk.call(self._w, 'count', *options, index1, index2) + if not isinstance(res, int): + res = self._getints(res) + if len(res) == 1: + res, = res + if not return_ints: + if not res: + res = None + elif len(options) <= 1: + res = (res,) + return res def debug(self, boolean=None): """Turn on the internal consistency checks of the B-Tree inside the text @@ -3978,9 +4055,7 @@ def tag_add(self, tagName, index1, *args): def tag_unbind(self, tagName, sequence, funcid=None): """Unbind for all characters with TAGNAME for event SEQUENCE the function identified with FUNCID.""" - self.tk.call(self._w, 'tag', 'bind', tagName, sequence, '') - if funcid: - self.deletecommand(funcid) + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) def tag_bind(self, tagName, sequence, func, add=None): """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. @@ -3991,6 +4066,11 @@ def tag_bind(self, tagName, sequence, func, add=None): return self._bind((self._w, 'tag', 'bind', tagName), sequence, func, add) + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + def tag_cget(self, tagName, option): """Return the value of OPTION for tag TAGNAME.""" if option[:1] != '-': @@ -4215,33 +4295,112 @@ def cget(self, option): def __getitem__(self, key): return self.tk.call(self.name, 'cget', '-' + key) - # XXX copy -from, -to, ...? - def copy(self): - """Return a new PhotoImage with the same image as this widget.""" + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ destImage = PhotoImage(master=self.tk) - self.tk.call(destImage, 'copy', self.name) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) return destImage - def zoom(self, x, y=''): + def zoom(self, x, y='', *, from_coords=None): """Return a new PhotoImage with the same image as this widget - but zoom it with a factor of x in the X direction and y in the Y - direction. If y is not given, the default value is the same as x. + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) - return destImage + return self.copy(zoom=(x, y), from_coords=from_coords) - def subsample(self, x, y=''): + def subsample(self, x, y='', *, from_coords=None): """Return a new PhotoImage based on the same image as this widget - but use only every Xth or Yth pixel. If y is not given, the - default value is the same as x. + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) - return destImage + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) def get(self, x, y): """Return the color (red, green, blue) of the pixel at X,Y.""" @@ -4256,17 +4415,117 @@ def put(self, data, to=None): to = to[1:] args = args + ('-to',) + tuple(to) self.tk.call(args) - # XXX read - - def write(self, filename, format=None, from_coords=None): - """Write image to file FILENAME in FORMAT starting from - position FROM_COORDS.""" - args = (self.name, 'write', filename) - if format: - args = args + ('-format', format) - if from_coords: - args = args + ('-from',) + tuple(from_coords) - self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containings space separated (one element per pixel/column) colors + in “#RRGGBB†format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data def transparency_get(self, x, y): """Return True if the pixel at x,y is transparent.""" @@ -4696,7 +4955,7 @@ def panes(self): def _test(): root = Tk() - text = "This is Tcl/Tk version %s" % TclVersion + text = "This is Tcl/Tk %s" % root.globalgetvar('tk_patchLevel') text += "\nThis should be a cedilla: \xe7" label = Label(root, text=text) label.pack() diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 538bbfc318d704..0f0dc66460f798 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -262,7 +262,7 @@ def _setup_dialog(w): w.tk.call("::tk::unsupported::MacWindowStyle", "style", w, "moveableModal", "") elif w._windowingsystem == "x11": - w.wm_attributes("-type", "dialog") + w.wm_attributes(type="dialog") # -------------------------------------------------------------------- # convenience dialogues diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index efeabb7a92c627..073b3ae20797c3 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -95,40 +95,47 @@ def _format_mapdict(mapdict, script=False): def _format_elemcreate(etype, script=False, *args, **kw): """Formats args and kw according to the given element factory etype.""" - spec = None + specs = () opts = () - if etype in ("image", "vsapi"): - if etype == "image": # define an element based on an image - # first arg should be the default image name - iname = args[0] - # next args, if any, are statespec/value pairs which is almost - # a mapdict, but we just need the value - imagespec = _join(_mapdict_values(args[1:])) - spec = "%s %s" % (iname, imagespec) - + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = (iname, *_mapdict_values(args[1:])) + if script: + specs = (imagespec,) else: - # define an element whose visual appearance is drawn using the - # Microsoft Visual Styles API which is responsible for the - # themed styles on Windows XP and Vista. - # Availability: Tk 8.6, Windows XP and Vista. - class_name, part_id = args[:2] - statemap = _join(_mapdict_values(args[2:])) - spec = "%s %s %s" % (class_name, part_id, statemap) + specs = (_join(imagespec),) + opts = _format_optdict(kw, script) + if etype == "vsapi": + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + if len(args) < 3: + class_name, part_id = args + statemap = (((), 1),) + else: + class_name, part_id, statemap = args + specs = (class_name, part_id, tuple(_mapdict_values(statemap))) opts = _format_optdict(kw, script) elif etype == "from": # clone an element # it expects a themename and optionally an element to clone from, # otherwise it will clone {} (empty element) - spec = args[0] # theme name + specs = (args[0],) # theme name if len(args) > 1: # elementfrom specified opts = (_format_optvalue(args[1], script),) if script: - spec = '{%s}' % spec + specs = _join(specs) opts = ' '.join(opts) + return specs, opts + else: + return *specs, opts - return spec, opts def _format_layoutlist(layout, indent=0, indent_size=2): """Formats a layout list so we can pass the result to ttk::style @@ -214,10 +221,10 @@ def _script_from_settings(settings): elemargs = eopts[1:argc] elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} - spec, opts = _format_elemcreate(etype, True, *elemargs, **elemkw) + specs, eopts = _format_elemcreate(etype, True, *elemargs, **elemkw) script.append("ttk::style element create %s %s %s %s" % ( - name, etype, spec, opts)) + name, etype, specs, eopts)) return '\n'.join(script) @@ -434,9 +441,9 @@ def layout(self, style, layoutspec=None): def element_create(self, elementname, etype, *args, **kw): """Create a new element in the current theme of given etype.""" - spec, opts = _format_elemcreate(etype, False, *args, **kw) + *specs, opts = _format_elemcreate(etype, False, *args, **kw) self.tk.call(self._name, "element", "create", elementname, etype, - spec, *opts) + *specs, *opts) def element_names(self): @@ -683,7 +690,10 @@ def current(self, newindex=None): returns the index of the current value in the list of values or -1 if the current value does not appear in the list.""" if newindex is None: - return self.tk.getint(self.tk.call(self._w, "current")) + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) return self.tk.call(self._w, "current", newindex) @@ -1515,7 +1525,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): self.label.place(anchor='n' if label_side == 'top' else 's') # update the label as scale or variable changes - self.__tracecb = self._variable.trace_variable('w', self._adjust) + self.__tracecb = self._variable.trace_add('write', self._adjust) self.bind('', self._adjust) self.bind('', self._adjust) @@ -1523,7 +1533,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): def destroy(self): """Destroy this widget and possibly its associated variable.""" try: - self._variable.trace_vdelete('w', self.__tracecb) + self._variable.trace_remove('write', self.__tracecb) except AttributeError: pass else: diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 0ab1893d42f72f..7f418bb7a1b37f 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -168,6 +168,7 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None self.encoding = None def add_whitespace(self, start): @@ -183,6 +184,29 @@ def add_whitespace(self, start): if col_offset: self.tokens.append(" " * col_offset) + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0: + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -214,11 +238,13 @@ def untokenize(self, iterable): startline = False elif tok_type == FSTRING_MIDDLE: if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] end_line, end_col = end - end = (end_line, end_col + token.count('{') + token.count('}')) - token = re.sub('{', '{{', token) - token = re.sub('}', '}}', token) - + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + elif tok_type in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") self.add_whitespace(start) self.tokens.append(token) @@ -226,6 +252,7 @@ def untokenize(self, iterable): if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type return "".join(self.tokens) def compat(self, token, iterable): @@ -233,6 +260,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -251,6 +279,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum == FSTRING_START: + in_fstring += 1 + elif toknum == FSTRING_END: + in_fstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -263,11 +295,18 @@ def compat(self, token, iterable): toks_append(indents[-1]) startline = False elif toknum == FSTRING_MIDDLE: - if '{' in tokval or '}' in tokval: - tokval = re.sub('{', '{{', tokval) - tokval = re.sub('}', '}}', tokval) + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): diff --git a/Lib/trace.py b/Lib/trace.py old mode 100755 new mode 100644 index fb9a423ea09fce..bb3d34fd8d6550 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. @@ -202,7 +200,8 @@ def update(self, other): for key in other_callers: callers[key] = 1 - def write_results(self, show_missing=True, summary=False, coverdir=None): + def write_results(self, show_missing=True, summary=False, coverdir=None, *, + ignore_missing_files=False): """ Write the coverage results. @@ -211,6 +210,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): :param coverdir: If None, the results of each module are placed in its directory, otherwise it is included in the directory specified. + :param ignore_missing_files: If True, counts for files that no longer + exist are silently ignored. Otherwise, a missing file + will raise a FileNotFoundError. """ if self.calledfuncs: print() @@ -253,13 +255,15 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): if filename.endswith(".pyc"): filename = filename[:-1] + if ignore_missing_files and not os.path.isfile(filename): + continue + if coverdir is None: dir = os.path.dirname(os.path.abspath(filename)) modulename = _modname(filename) else: dir = coverdir - if not os.path.exists(dir): - os.makedirs(dir) + os.makedirs(dir, exist_ok=True) modulename = _fullmodname(filename) # If desired, get a list of the line numbers which represent @@ -278,7 +282,6 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): percent = int(100 * n_hits / n_lines) sums[modulename] = n_lines, percent, modulename, filename - if summary and sums: print("lines cov% module (path)") for m in sorted(sums): @@ -397,7 +400,7 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides - `count' and `trace' + 'count' and 'trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @@ -529,7 +532,7 @@ def globaltrace_countfuncs(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): """Handler for call events. - If the code block being entered is to be ignored, returns `None', + If the code block being entered is to be ignored, returns 'None', else returns self.localtrace. """ if why == 'call': @@ -560,8 +563,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -573,8 +580,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): diff --git a/Lib/traceback.py b/Lib/traceback.py index 7cc84b9c762aeb..6ee1a50ca6804a 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -5,7 +5,10 @@ import linecache import sys import textwrap +import warnings from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -18,6 +21,7 @@ # Formatting and printing lists of traceback lines. # + def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" @@ -109,7 +113,7 @@ def _parse_value_tb(exc, value, tb): def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - file=None, chain=True): + file=None, chain=True, **kwargs): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -120,9 +124,10 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ occurred with a caret on the next line indicating the approximate position of the error. """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - te.print(file=file, chain=chain) + te.print(file=file, chain=chain, colorize=colorize) BUILTIN_EXCEPTION_LIMIT = object() @@ -130,11 +135,12 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file) + colorize = _colorize.can_colorize() + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -143,41 +149,52 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only()) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. -def _format_final_exc_line(etype, value): +def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize=False): valuestr = _safe_string(value, 'exception') - if value is None or not valuestr: - line = "%s\n" % etype + end_char = "\n" if insert_final_newline else "" + if colorize: + if value is None or not valuestr: + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" + else: + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: - line = "%s: %s\n" % (etype, valuestr) + if value is None or not valuestr: + line = f"{etype}{end_char}" + else: + line = f"{etype}: {valuestr}{end_char}" return line + def _safe_string(value, what, func=str): try: return func(value) @@ -271,7 +288,7 @@ class FrameSummary: """ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', - 'name', '_line', 'locals') + 'name', '_lines', '_lines_dedented', 'locals') def __init__(self, filename, lineno, name, *, lookup_line=True, locals=None, line=None, @@ -287,15 +304,16 @@ def __init__(self, filename, lineno, name, *, lookup_line=True, """ self.filename = filename self.lineno = lineno + self.end_lineno = lineno if end_lineno is None else end_lineno + self.colno = colno + self.end_colno = end_colno self.name = name - self._line = line + self._lines = line + self._lines_dedented = None if lookup_line: self.line self.locals = {k: _safe_string(v, 'local', func=repr) for k, v in locals.items()} if locals else None - self.end_lineno = end_lineno - self.colno = colno - self.end_colno = end_colno def __eq__(self, other): if isinstance(other, FrameSummary): @@ -320,19 +338,39 @@ def __repr__(self): def __len__(self): return 4 + def _set_lines(self): + if ( + self._lines is None + and self.lineno is not None + and self.end_lineno is not None + ): + lines = [] + for lineno in range(self.lineno, self.end_lineno + 1): + # treat errors (empty string) and empty lines (newline) as the same + lines.append(linecache.getline(self.filename, lineno).rstrip()) + self._lines = "\n".join(lines) + "\n" + @property - def _original_line(self): + def _original_lines(self): # Returns the line as-is from the source, without modifying whitespace. - self.line - return self._line + self._set_lines() + return self._lines + + @property + def _dedented_lines(self): + # Returns _original_lines, but dedented + self._set_lines() + if self._lines_dedented is None and self._lines is not None: + self._lines_dedented = textwrap.dedent(self._lines) + return self._lines_dedented @property def line(self): - if self._line is None: - if self.lineno is None: - return None - self._line = linecache.getline(self.filename, self.lineno) - return self._line.strip() + self._set_lines() + if self._lines is None: + return None + # return only the first line, stripped + return self._lines.partition("\n")[0].strip() def walk_stack(f): @@ -382,6 +420,7 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. + class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -472,66 +511,221 @@ def from_list(klass, a_list): result.append(FrameSummary(filename, lineno, name, line=line)) return result - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, **kwargs): """Format the lines for a single FrameSummary. Returns a string representing one frame involved in the stack. This gets called for every frame to be printed in the stack summary. """ + colorize = kwargs.get("colorize", False) row = [] - if frame_summary.filename.startswith("", line {}, in {}\n'.format( - frame_summary.lineno, frame_summary.name)) + filename = frame_summary.filename + if frame_summary.filename.startswith("-"): + filename = "" + if colorize: + row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( + ANSIColors.MAGENTA, + filename, + ANSIColors.RESET, + ANSIColors.MAGENTA, + frame_summary.lineno, + ANSIColors.RESET, + ANSIColors.MAGENTA, + frame_summary.name, + ANSIColors.RESET, + ) + ) else: row.append(' File "{}", line {}, in {}\n'.format( - frame_summary.filename, frame_summary.lineno, frame_summary.name)) - if frame_summary.line: - stripped_line = frame_summary.line.strip() - row.append(' {}\n'.format(stripped_line)) - - orig_line_len = len(frame_summary._original_line) - frame_line_len = len(frame_summary.line.lstrip()) - stripped_characters = orig_line_len - frame_line_len + filename, frame_summary.lineno, frame_summary.name)) + if frame_summary._dedented_lines and frame_summary._dedented_lines.strip(): if ( - frame_summary.colno is not None - and frame_summary.end_colno is not None + frame_summary.colno is None or + frame_summary.end_colno is None ): - start_offset = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.colno) + 1 - end_offset = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.end_colno) + 1 - + # only output first line if column information is missing + row.append(textwrap.indent(frame_summary.line, ' ') + "\n") + else: + # get first and last line + all_lines_original = frame_summary._original_lines.splitlines() + first_line = all_lines_original[0] + # assume all_lines_original has enough lines (since we constructed it) + last_line = all_lines_original[frame_summary.end_lineno - frame_summary.lineno] + + # character index of the start/end of the instruction + start_offset = _byte_offset_to_character_offset(first_line, frame_summary.colno) + end_offset = _byte_offset_to_character_offset(last_line, frame_summary.end_colno) + + all_lines = frame_summary._dedented_lines.splitlines()[ + :frame_summary.end_lineno - frame_summary.lineno + 1 + ] + + # adjust start/end offset based on dedent + dedent_characters = len(first_line) - len(all_lines[0]) + start_offset = max(0, start_offset - dedent_characters) + end_offset = max(0, end_offset - dedent_characters) + + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(all_lines[0], offset=start_offset) + dp_end_offset = _display_width(all_lines[-1], offset=end_offset) + + # get exact code segment corresponding to the instruction + segment = "\n".join(all_lines) + segment = segment[start_offset:len(segment) - (len(all_lines[-1]) - end_offset)] + + # attempt to parse for anchors anchors = None - if frame_summary.lineno == frame_summary.end_lineno: - with suppress(Exception): - anchors = _extract_caret_anchors_from_line_segment( - frame_summary._original_line[start_offset - 1:end_offset - 1] - ) - else: - end_offset = stripped_characters + len(stripped_line) - - # show indicators if primary char doesn't span the frame line - if end_offset - start_offset < len(stripped_line) or ( - anchors and anchors.right_start_offset - anchors.left_end_offset > 0): - row.append(' ') - row.append(' ' * (start_offset - stripped_characters)) - - if anchors: - row.append(anchors.primary_char * (anchors.left_end_offset)) - row.append(anchors.secondary_char * (anchors.right_start_offset - anchors.left_end_offset)) - row.append(anchors.primary_char * (end_offset - start_offset - anchors.right_start_offset)) - else: - row.append('^' * (end_offset - start_offset)) + show_carets = False + with suppress(Exception): + anchors = _extract_caret_anchors_from_line_segment(segment) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) + + result = [] + + # only display first line, last line, and lines around anchor start/end + significant_lines = {0, len(all_lines) - 1} + + anchors_left_end_offset = 0 + anchors_right_start_offset = 0 + primary_char = "^" + secondary_char = "^" + if anchors: + anchors_left_end_offset = anchors.left_end_offset + anchors_right_start_offset = anchors.right_start_offset + # computed anchor positions do not take start_offset into account, + # so account for it here + if anchors.left_end_lineno == 0: + anchors_left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors_right_start_offset += start_offset + + # account for display width + anchors_left_end_offset = _display_width( + all_lines[anchors.left_end_lineno], offset=anchors_left_end_offset + ) + anchors_right_start_offset = _display_width( + all_lines[anchors.right_start_lineno], offset=anchors_right_start_offset + ) - row.append('\n') + primary_char = anchors.primary_char + secondary_char = anchors.secondary_char + significant_lines.update( + range(anchors.left_end_lineno - 1, anchors.left_end_lineno + 2) + ) + significant_lines.update( + range(anchors.right_start_lineno - 1, anchors.right_start_lineno + 2) + ) + # remove bad line numbers + significant_lines.discard(-1) + significant_lines.discard(len(all_lines)) + + def output_line(lineno): + """output all_lines[lineno] along with carets""" + result.append(all_lines[lineno] + "\n") + if not show_carets: + return + num_spaces = len(all_lines[lineno]) - len(all_lines[lineno].lstrip()) + carets = [] + num_carets = dp_end_offset if lineno == len(all_lines) - 1 else _display_width(all_lines[lineno]) + # compute caret character for each position + for col in range(num_carets): + if col < num_spaces or (lineno == 0 and col < dp_start_offset): + # before first non-ws char of the line, or before start of instruction + carets.append(' ') + elif anchors and ( + lineno > anchors.left_end_lineno or + (lineno == anchors.left_end_lineno and col >= anchors_left_end_offset) + ) and ( + lineno < anchors.right_start_lineno or + (lineno == anchors.right_start_lineno and col < anchors_right_start_offset) + ): + # within anchors + carets.append(secondary_char) + else: + carets.append(primary_char) + if colorize: + # Replace the previous line with a red version of it only in the parts covered + # by the carets. + line = result[-1] + colorized_line_parts = [] + colorized_carets_parts = [] + + for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + caret_group = list(group) + if color == "^": + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) + elif color == "~": + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) + else: + colorized_line_parts.append("".join(char for char, _ in caret_group)) + colorized_carets_parts.append("".join(caret for _, caret in caret_group)) + + colorized_line = "".join(colorized_line_parts) + colorized_carets = "".join(colorized_carets_parts) + result[-1] = colorized_line + result.append(colorized_carets + "\n") + else: + result.append("".join(carets) + "\n") + + # display significant lines + sig_lines_list = sorted(significant_lines) + for i, lineno in enumerate(sig_lines_list): + if i: + linediff = lineno - sig_lines_list[i - 1] + if linediff == 2: + # 1 line in between - just output it + output_line(lineno - 1) + elif linediff > 2: + # > 1 line in between - abbreviate + result.append(f"...<{linediff - 1} lines>...\n") + output_line(lineno) + + row.append( + textwrap.indent(textwrap.dedent("".join(result)), ' ', lambda line: True) + ) if frame_summary.locals: for name, value in sorted(frame_summary.locals.items()): row.append(' {name} = {value}\n'.format(name=name, value=value)) return ''.join(row) - def format(self): + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + + def format(self, **kwargs): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the @@ -543,13 +737,14 @@ def format(self): repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ + colorize = kwargs.get("colorize", False) result = [] last_file = None last_line = None last_name = None count = 0 for frame_summary in self: - formatted_frame = self.format_frame_summary(frame_summary) + formatted_frame = self.format_frame_summary(frame_summary, colorize=colorize) if formatted_frame is None: continue if (last_file is None or last_file != frame_summary.filename or @@ -587,7 +782,9 @@ def _byte_offset_to_character_offset(str, offset): _Anchors = collections.namedtuple( "_Anchors", [ + "left_end_lineno", "left_end_offset", + "right_start_lineno", "right_start_offset", "primary_char", "secondary_char", @@ -596,52 +793,173 @@ def _byte_offset_to_character_offset(str, offset): ) def _extract_caret_anchors_from_line_segment(segment): + """ + Given source code `segment` corresponding to a FrameSummary, determine: + - for binary ops, the location of the binary op + - for indexing and function calls, the location of the brackets. + `segment` is expected to be a valid Python expression. + """ import ast try: - tree = ast.parse(segment) + # Without parentheses, `segment` is parsed as a statement. + # Binary ops, subscripts, and calls are expressions, so + # we can wrap them with parentheses to parse them as + # (possibly multi-line) expressions. + # e.g. if we try to highlight the addition in + # x = ( + # a + + # b + # ) + # then we would ast.parse + # a + + # b + # which is not a valid statement because of the newline. + # Adding brackets makes it a valid expression. + # ( + # a + + # b + # ) + # Line locations will be different than the original, + # which is taken into account later on. + tree = ast.parse(f"(\n{segment}\n)") except SyntaxError: return None if len(tree.body) != 1: return None - normalize = lambda offset: _byte_offset_to_character_offset(segment, offset) + lines = segment.splitlines() + + def normalize(lineno, offset): + """Get character index given byte offset""" + return _byte_offset_to_character_offset(lines[lineno], offset) + + def next_valid_char(lineno, col): + """Gets the next valid character index in `lines`, if + the current location is not valid. Handles empty lines. + """ + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + def increment(lineno, col): + """Get the next valid character index in `lines`.""" + col += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def nextline(lineno, col): + """Get the next valid character at least on the next line""" + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def increment_until(lineno, col, stop): + """Get the next valid non-"\\#" character that satisfies the `stop` predicate""" + while True: + ch = lines[lineno][col] + if ch in "\\#": + lineno, col = nextline(lineno, col) + elif not stop(ch): + lineno, col = increment(lineno, col) + else: + break + return lineno, col + + def setup_positions(expr, force_valid=True): + """Get the lineno/col position of the end of `expr`. If `force_valid` is True, + forces the position to be a valid character (e.g. if the position is beyond the + end of the line, move to the next line) + """ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket + newline to `segment` when calling ast.parse + lineno = expr.end_lineno - 2 + col = normalize(lineno, expr.end_col_offset) + return next_valid_char(lineno, col) if force_valid else (lineno, col) + statement = tree.body[0] match statement: case ast.Expr(expr): match expr: case ast.BinOp(): - operator_start = normalize(expr.left.end_col_offset) - operator_end = normalize(expr.right.col_offset) - operator_str = segment[operator_start:operator_end] - operator_offset = len(operator_str) - len(operator_str.lstrip()) + # ast gives these locations for BinOp subexpressions + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + lineno, col = setup_positions(expr.left) - left_anchor = expr.left.end_col_offset + operator_offset - right_anchor = left_anchor + 1 + # First operator character is the first non-space/')' character + lineno, col = increment_until(lineno, col, lambda x: not x.isspace() and x != ')') + + # binary op is 1 or 2 characters long, on the same line, + # before the right subexpression + right_col = col + 1 if ( - operator_offset + 1 < len(operator_str) - and not operator_str[operator_offset + 1].isspace() + right_col < len(lines[lineno]) + and ( + # operator char should not be in the right subexpression + expr.right.lineno - 2 > lineno or + right_col < normalize(expr.right.lineno - 2, expr.right.col_offset) + ) + and not (ch := lines[lineno][right_col]).isspace() + and ch not in "\\#" ): - right_anchor += 1 + right_col += 1 - while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch in ")#"): - left_anchor += 1 - right_anchor += 1 - return _Anchors(normalize(left_anchor), normalize(right_anchor)) + # right_col can be invalid since it is exclusive + return _Anchors(lineno, col, lineno, right_col) case ast.Subscript(): - left_anchor = normalize(expr.value.end_col_offset) - right_anchor = normalize(expr.slice.end_col_offset + 1) - while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch != "["): - left_anchor += 1 - while right_anchor < len(segment) and ((ch := segment[right_anchor]).isspace() or ch != "]"): - right_anchor += 1 - if right_anchor < len(segment): - right_anchor += 1 - return _Anchors(left_anchor, right_anchor) + # ast gives these locations for value and slice subexpressions + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.value) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '[') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + case ast.Call(): + # ast gives these locations for function call expressions + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.func) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '(') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) return None +_WIDE_CHAR_SPECIFIERS = "WF" + +def _display_width(line, offset=None): + """Calculate the extra amount of width space the given source + code segment might take if it were to be displayed on a fixed + width output device. Supports wide unicode characters and emojis.""" + + if offset is None: + offset = len(line) + + # Fast track for ASCII-only strings + if line.isascii(): + return offset + + import unicodedata + + return sum( + 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 + for char in line[:offset] + ) + + class _ExceptionPrintContext: def __init__(self): @@ -689,7 +1007,8 @@ class TracebackException: - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -707,7 +1026,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - max_group_width=15, max_group_depth=10, _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -724,11 +1043,26 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _walk_tb_with_full_positions(exc_traceback), limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line self._str = _safe_string(exc_value, 'exception') - self.__notes__ = getattr(exc_value, '__notes__', None) + try: + self.__notes__ = getattr(exc_value, '__notes__', None) + except Exception as e: + self.__notes__ = [ + f'Ignored error getting __notes__: {_safe_string(e, '__notes__', repr)}'] + + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially @@ -741,6 +1075,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True elif exc_type and issubclass(exc_type, ImportError) and \ getattr(exc_value, "name_from", None) is not None: wrong_name = getattr(exc_value, "name_from", None) @@ -839,6 +1174,24 @@ def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -852,36 +1205,45 @@ def __eq__(self, other): def __str__(self): return self._str - def format_exception_only(self, *, show_group=False, _depth=0): + def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. + + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) indent = 3 * _depth * ' ' - if self.exc_type is None: - yield indent + _format_final_exc_line(None, self._str) + if not self._have_exc_type: + yield indent + _format_final_exc_line(None, self._str, colorize=colorize) return - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "" - stype = smod + '.' + stype - - if not issubclass(self.exc_type, SyntaxError): - yield indent + _format_final_exc_line(stype, self._str) + stype = self.exc_type_str + if not self._is_syntax_error: + if _depth > 0: + # Nested exceptions needs correct handling of multiline messages. + formatted = _format_final_exc_line( + stype, self._str, insert_final_newline=False, colorize=colorize + ).split('\n') + yield from [ + indent + l + '\n' + for l in formatted + ] + else: + yield _format_final_exc_line(stype, self._str, colorize=colorize) else: - yield from [indent + l for l in self._format_syntax_error(stype)] + yield from [indent + l for l in self._format_syntax_error(stype, colorize=colorize)] if ( isinstance(self.__notes__, collections.abc.Sequence) @@ -895,15 +1257,26 @@ def format_exception_only(self, *, show_group=False, _depth=0): if self.exceptions and show_group: for ex in self.exceptions: - yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1) + yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) - def _format_syntax_error(self, stype): + def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. + colorize = kwargs.get("colorize", False) filename_suffix = '' if self.lineno is not None: - yield ' File "{}", line {}\n'.format( - self.filename or "", self.lineno) + if colorize: + yield ' File {}"{}"{}, line {}{}{}\n'.format( + ANSIColors.MAGENTA, + self.filename or "", + ANSIColors.RESET, + ANSIColors.MAGENTA, + self.lineno, + ANSIColors.RESET, + ) + else: + yield ' File "{}", line {}\n'.format( + self.filename or "", self.lineno) elif self.filename is not None: filename_suffix = ' ({})'.format(self.filename) @@ -915,9 +1288,9 @@ def _format_syntax_error(self, stype): rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) - yield ' {}\n'.format(ltext) - - if self.offset is not None: + if self.offset is None: + yield ' {}\n'.format(ltext) + else: offset = self.offset end_offset = self.end_offset if self.end_offset not in {None, 0} else offset if self.text and offset > len(self.text): @@ -930,14 +1303,43 @@ def _format_syntax_error(self, stype): # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces + caretspace = ' ' if colno >= 0: # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) - yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) + start_color = end_color = "" + if colorize: + # colorize from colno to end_colno + ltext = ( + ltext[:colno] + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + + ltext[end_colno:] + ) + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET + yield ' {}\n'.format(ltext) + yield ' {}{}{}{}\n'.format( + "".join(caretspace), + start_color, + ('^' * (end_colno - colno)), + end_color, + ) + else: + yield ' {}\n'.format(ltext) msg = self.msg or "" - yield "{}: {}{}\n".format(stype, msg, filename_suffix) + if colorize: + yield "{}{}{}: {}{}{}{}\n".format( + ANSIColors.BOLD_MAGENTA, + stype, + ANSIColors.RESET, + ANSIColors.MAGENTA, + msg, + ANSIColors.RESET, + filename_suffix) + else: + yield "{}: {}{}\n".format(stype, msg, filename_suffix) - def format(self, *, chain=True, _ctx=None): + def format(self, *, chain=True, _ctx=None, **kwargs): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -949,7 +1351,7 @@ def format(self, *, chain=True, _ctx=None): The message indicating which exception occurred is always the last string in the output. """ - + colorize = kwargs.get("colorize", False) if _ctx is None: _ctx = _ExceptionPrintContext() @@ -979,8 +1381,8 @@ def format(self, *, chain=True, _ctx=None): if exc.exceptions is None: if exc.stack: yield from _ctx.emit('Traceback (most recent call last):\n') - yield from _ctx.emit(exc.stack.format()) - yield from _ctx.emit(exc.format_exception_only()) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) elif _ctx.exception_group_depth > self.max_group_depth: # exception group, but depth exceeds limit yield from _ctx.emit( @@ -995,9 +1397,9 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit( 'Exception Group Traceback (most recent call last):\n', margin_char = '+' if is_toplevel else None) - yield from _ctx.emit(exc.stack.format()) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) - yield from _ctx.emit(exc.format_exception_only()) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) num_excs = len(exc.exceptions) if num_excs <= self.max_group_width: n = num_excs @@ -1038,11 +1440,12 @@ def format(self, *, chain=True, _ctx=None): _ctx.exception_group_depth = 0 - def print(self, *, file=None, chain=True): + def print(self, *, file=None, chain=True, **kwargs): """Print the result of self.format(chain=chain) to 'file'.""" + colorize = kwargs.get("colorize", False) if file is None: file = sys.stderr - for line in self.format(chain=chain): + for line in self.format(chain=chain, colorize=colorize): print(line, file=file, end="") @@ -1067,12 +1470,23 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): obj = exc_value.obj try: d = dir(obj) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] except Exception: return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) d = dir(mod) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] except Exception: return None else: @@ -1096,6 +1510,13 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if hasattr(self, wrong_name): return f"self.{wrong_name}" + try: + import _suggestions + except ImportError: + pass + else: + return _suggestions._generate_suggestions(d, wrong_name) + # Compute closest match if len(d) > _MAX_CANDIDATE_ITEMS: diff --git a/Lib/tty.py b/Lib/tty.py index 283e5c334f5751..5a49e0400425f3 100644 --- a/Lib/tty.py +++ b/Lib/tty.py @@ -45,9 +45,6 @@ def cfmakeraw(mode): def cfmakecbreak(mode): """Make termios mode cbreak.""" - # Do not map CR to NL on input. - mode[IFLAG] &= ~(ICRNL) - # Do not echo characters; disable canonical input. mode[LFLAG] &= ~(ECHO | ICANON) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py old mode 100755 new mode 100644 index f6c9d6aa6f9a32..9c15916fb6672e --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ ---------------------------------------------- turtleDemo - Help @@ -92,13 +90,15 @@ from idlelib.colorizer import ColorDelegator, color_config from idlelib.percolator import Percolator from idlelib.textview import view_text +import turtle from turtledemo import __doc__ as about_turtledemo -import turtle +if sys.platform == 'win32': + from idlelib.util import fix_win_hidpi + fix_win_hidpi() demo_dir = os.path.dirname(os.path.abspath(__file__)) darwin = sys.platform == 'darwin' - STARTUP = 1 READY = 2 RUNNING = 3 @@ -161,7 +161,7 @@ def __init__(self, filename=None): label='Help', underline=0) root['menu'] = self.mBar - pane = PanedWindow(orient=HORIZONTAL, sashwidth=5, + pane = PanedWindow(root, orient=HORIZONTAL, sashwidth=5, sashrelief=SOLID, bg='#ddd') pane.add(self.makeTextFrame(pane)) pane.add(self.makeGraphFrame(pane)) @@ -216,7 +216,7 @@ def makeTextFrame(self, root): self.vbar = vbar = Scrollbar(text_frame, name='vbar') vbar['command'] = text.yview - vbar.pack(side=LEFT, fill=Y) + vbar.pack(side=RIGHT, fill=Y) self.hbar = hbar = Scrollbar(text_frame, name='hbar', orient=HORIZONTAL) hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) @@ -292,7 +292,7 @@ def configGUI(self, start, stop, clear, txt="", color="blue"): self.output_lbl.config(text=txt, fg=color) def makeLoadDemoMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=1) # TJR: leave this one. for entry in getExampleEntries(): def load(entry=entry): @@ -302,7 +302,7 @@ def load(entry=entry): return menu def makeFontMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=0) menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, font=menufont) menu.add_command(label="Increase (C-'+')", command=self.increase_size, @@ -317,7 +317,7 @@ def resize(size=size): return menu def makeHelpMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=0) for help_label, help_file in help_entries: def show(help_label=help_label, help_file=help_file): diff --git a/Lib/turtledemo/bytedesign.py b/Lib/turtledemo/bytedesign.py old mode 100755 new mode 100644 index 1b7452b512c6eb..476cdaabfceab1 --- a/Lib/turtledemo/bytedesign.py +++ b/Lib/turtledemo/bytedesign.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_bytedesign.py diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py old mode 100755 new mode 100644 index 9f8585bd11e053..fd3b3992d466bf --- a/Lib/turtledemo/clock.py +++ b/Lib/turtledemo/clock.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: cp1252 -*- """ turtle-example-suite: diff --git a/Lib/turtledemo/forest.py b/Lib/turtledemo/forest.py old mode 100755 new mode 100644 index 55b7da947d2476..cac553223828db --- a/Lib/turtledemo/forest.py +++ b/Lib/turtledemo/forest.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtlegraphics-example-suite: tdemo_forest.py diff --git a/Lib/turtledemo/fractalcurves.py b/Lib/turtledemo/fractalcurves.py old mode 100755 new mode 100644 index 54ade96a0ad05e..fda193e06fedee --- a/Lib/turtledemo/fractalcurves.py +++ b/Lib/turtledemo/fractalcurves.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_fractalCurves.py diff --git a/Lib/turtledemo/lindenmayer.py b/Lib/turtledemo/lindenmayer.py old mode 100755 new mode 100644 index 3925f25da61870..7c7a84796c3c28 --- a/Lib/turtledemo/lindenmayer.py +++ b/Lib/turtledemo/lindenmayer.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: xtx_lindenmayer_indian.py diff --git a/Lib/turtledemo/minimal_hanoi.py b/Lib/turtledemo/minimal_hanoi.py old mode 100755 new mode 100644 index 4a432f2b2908d5..08d8b630fec3b4 --- a/Lib/turtledemo/minimal_hanoi.py +++ b/Lib/turtledemo/minimal_hanoi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_minimal_hanoi.py diff --git a/Lib/turtledemo/paint.py b/Lib/turtledemo/paint.py old mode 100755 new mode 100644 index fc6852a20082f5..6e63d004454589 --- a/Lib/turtledemo/paint.py +++ b/Lib/turtledemo/paint.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_paint.py diff --git a/Lib/turtledemo/peace.py b/Lib/turtledemo/peace.py old mode 100755 new mode 100644 index e2ba9288d9e42e..fd6abe390ef198 --- a/Lib/turtledemo/peace.py +++ b/Lib/turtledemo/peace.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_peace.py diff --git a/Lib/turtledemo/penrose.py b/Lib/turtledemo/penrose.py old mode 100755 new mode 100644 index 045722a2286061..ac12c899d3844e --- a/Lib/turtledemo/penrose.py +++ b/Lib/turtledemo/penrose.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ xturtle-example-suite: xtx_kites_and_darts.py diff --git a/Lib/turtledemo/planet_and_moon.py b/Lib/turtledemo/planet_and_moon.py old mode 100755 new mode 100644 index 021ff99383aa65..c0e2c5b79e173e --- a/Lib/turtledemo/planet_and_moon.py +++ b/Lib/turtledemo/planet_and_moon.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_planets_and_moon.py diff --git a/Lib/turtledemo/sorting_animate.py b/Lib/turtledemo/sorting_animate.py old mode 100755 new mode 100644 index d25a0ab6cebdc0..55735cd7001278 --- a/Lib/turtledemo/sorting_animate.py +++ b/Lib/turtledemo/sorting_animate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ sorting_animation.py diff --git a/Lib/turtledemo/tree.py b/Lib/turtledemo/tree.py old mode 100755 new mode 100644 index 98a20da7f15c11..12729e23688a48 --- a/Lib/turtledemo/tree.py +++ b/Lib/turtledemo/tree.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_tree.py diff --git a/Lib/turtledemo/yinyang.py b/Lib/turtledemo/yinyang.py old mode 100755 new mode 100644 index 11d1f47cae2549..791060d17e6b6a --- a/Lib/turtledemo/yinyang.py +++ b/Lib/turtledemo/yinyang.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_yinyang.py diff --git a/Lib/typing.py b/Lib/typing.py index 14845b36028ca1..bc17d136082891 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -38,6 +38,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + NoDefault, ) # Please keep __all__ alphabetized within each category. @@ -63,7 +64,6 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. - 'ByteString', 'Container', 'ContextManager', 'Hashable', @@ -138,12 +138,14 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', 'NotRequired', 'overload', 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', 'Required', 'reveal_type', 'runtime_checkable', @@ -152,6 +154,7 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', 'TypeAliasType', 'Unpack', ] @@ -212,8 +215,12 @@ def _should_unflatten_callable_args(typ, args): For example:: - assert collections.abc.Callable[[int, int], str].__args__ == (int, int, str) - assert collections.abc.Callable[ParamSpec, str].__args__ == (ParamSpec, str) + >>> import collections.abc + >>> P = ParamSpec('P') + >>> collections.abc.Callable[[int, int], str].__args__ == (int, int, str) + True + >>> collections.abc.Callable[P, str].__args__ == (P, str) + True As a result, if we need to reconstruct the Callable from its __args__, we need to unflatten it. @@ -249,14 +256,21 @@ def _type_repr(obj): return repr(obj) -def _collect_parameters(args): - """Collect all type variables and parameter specifications in args +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args in order of first appearance (lexicographic order). For example:: - assert _collect_parameters((T, Callable[P, T])) == (T, P) + >>> P = ParamSpec('P') + >>> T = TypeVar('T') + >>> _collect_type_parameters((T, Callable[P, T])) + (~T, ~P) """ + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False parameters = [] for t in args: if isinstance(t, type): @@ -266,32 +280,64 @@ def _collect_parameters(args): # `t` might be a tuple, when `ParamSpec` is substituted with # `[T, int]`, or `[int, *Ts]`, etc. for x in t: - for collected in _collect_parameters([x]): + for collected in _collect_type_parameters([x]): if collected not in parameters: parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_type_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") -def _unpack_args(args): + +def _unpack_args(*args): newargs = [] for arg in args: subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) @@ -301,19 +347,33 @@ def _unpack_args(args): newargs.append(arg) return newargs -def _deduplicate(params): +def _deduplicate(params, *, unhashable_fallback=False): # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - return params - + try: + return dict.fromkeys(params) + except TypeError: + if not unhashable_fallback: + raise + # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` + return _deduplicate_unhashable(params) + +def _deduplicate_unhashable(unhashable_params): + new_unhashable = [] + for t in unhashable_params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable + +def _compare_args_orderless(first_args, second_args): + first_unhashable = _deduplicate_unhashable(first_args) + second_unhashable = _deduplicate_unhashable(second_args) + t = list(second_unhashable) + try: + for elem in first_unhashable: + t.remove(elem) + except ValueError: + return False + return not t def _remove_dups_flatten(parameters): """Internal helper for Union creation and substitution. @@ -328,7 +388,7 @@ def _remove_dups_flatten(parameters): else: params.append(p) - return tuple(_deduplicate(params)) + return tuple(_deduplicate(params, unhashable_fallback=True)) def _flatten_literal_params(parameters): @@ -376,15 +436,41 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) + return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): if isinstance(t, GenericAlias): args = tuple( @@ -398,7 +484,13 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): t = t.__origin__[args] if is_unpacked: t = Unpack[t] - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): @@ -483,7 +575,7 @@ def __getitem__(self, parameters): return self._getitem(self, parameters) -class _LiteralSpecialForm(_SpecialForm, _root=True): +class _TypedCacheSpecialForm(_SpecialForm, _root=True): def __getitem__(self, parameters): if not isinstance(parameters, tuple): parameters = (parameters,) @@ -517,7 +609,7 @@ class Any(metaclass=_AnyMeta): def __new__(cls, *args, **kwargs): if cls is Any: raise TypeError("Any cannot be instantiated") - return super().__new__(cls, *args, **kwargs) + return super().__new__(cls) @_SpecialForm @@ -632,7 +724,7 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -654,7 +746,7 @@ class FastConnector(Connection): There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -716,7 +808,7 @@ def Optional(self, parameters): arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] -@_LiteralSpecialForm +@_TypedCacheSpecialForm @_tp_cache(typed=True) def Literal(self, *parameters): """Special typing form to define literal types (a.k.a. value types). @@ -796,46 +888,52 @@ def Concatenate(self, parameters): @_SpecialForm def TypeGuard(self, parameters): - """Special typing construct for marking user-defined type guard functions. + """Special typing construct for marking user-defined type predicate functions. ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. - For example:: + For example:: - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) + + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -844,6 +942,75 @@ def is_str(val: Union[str, float]): return _GenericAlias(self, (item,)) +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. + + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. + + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". + + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) + + class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" @@ -859,7 +1026,7 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. # Unfortunately, this isn't a valid expression on its own, so we # do the unpacking manually. - if arg[0] == '*': + if arg.startswith('*'): arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] else: arg_to_compile = arg @@ -876,7 +1043,10 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, recursive_guard): + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") + type_params = () if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: @@ -890,6 +1060,22 @@ def _evaluate(self, globalns, localns, recursive_guard): globalns = getattr( sys.modules.get(self.__forward_module__, None), '__dict__', globalns ) + + # type parameters require some special handling, + # as they exist in their own scope + # but `eval()` does not have a dedicated parameter for that scope. + # For classes, names in type parameter scopes should override + # names in the global scope (which here are called `localns`!), + # but should in turn be overridden by names in the class scope + # (which here are called `globalns`!) + if type_params: + globalns, localns = dict(globalns), dict(localns) + for param in type_params: + param_name = param.__name__ + if not self.__forward_is_class__ or param_name not in globalns: + globalns[param_name] = param + localns.pop(param_name, None) + type_ = _type_check( eval(self.__forward_code__, globalns, localns), "Forward references must evaluate to types.", @@ -897,7 +1083,11 @@ def _evaluate(self, globalns, localns, recursive_guard): allow_special_forms=self.__forward_is_class__, ) self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} + type_, + globalns, + localns, + type_params, + recursive_guard=(recursive_guard | {self.__forward_arg__}), ) self.__forward_evaluated__ = True return self.__forward_value__ @@ -973,11 +1163,15 @@ def _typevartuple_prepare_subst(self, alias, args): elif left + right > alen: raise TypeError(f"Too few arguments for {alias};" f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] return ( *args[:left], *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), + replacement, *([fillarg]*(plen - right - left - typevartuple_index - 1)), *args[alen - right:], ) @@ -995,6 +1189,8 @@ def _paramspec_subst(self, arg): def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1008,33 +1204,33 @@ def _paramspec_prepare_subst(self, alias, args): @_tp_cache -def _generic_class_getitem(cls, params): +def _generic_class_getitem(cls, args): """Parameterizes a generic class. At least, parameterizing a generic class is the *main* thing this method does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. + do `Foo[int]` - there, with `cls=Foo` and `args=int`. However, note that this method is also called when defining generic classes in the first place with `class Foo(Generic[T]): ...`. """ - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) - params = tuple(_type_convert(p) for p in params) + args = tuple(_type_convert(p) for p in args) is_generic_or_protocol = cls in (Generic, Protocol) if is_generic_or_protocol: # Generic and Protocol can only be subscripted with unique type variables. - if not params: + if not args: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty" ) - if not all(_is_typevar_like(p) for p in params): + if not all(_is_typevar_like(p) for p in args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " f"or parameter specification variables.") - if len(set(params)) != len(params): + if len(set(args)) != len(args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: @@ -1042,18 +1238,18 @@ def _generic_class_getitem(cls, params): for param in cls.__parameters__: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) + args = prepare(cls, args) + _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, params): + for param, new_arg in zip(cls.__parameters__, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: new_args.append(new_arg) - params = tuple(new_args) + args = tuple(new_args) - return _GenericAlias(cls, params) + return _GenericAlias(cls, args) def _generic_init_subclass(cls, *args, **kwargs): @@ -1068,7 +1264,7 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -1120,7 +1316,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -1128,9 +1326,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) @@ -1147,7 +1365,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1194,11 +1412,12 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # A = Callable[[], None] # _CallableGenericAlias # B = Callable[[T], None] # _CallableGenericAlias # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar` and `TypeGuard`: + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: # # All _GenericAlias # Final[int] # ClassVar[float] - # TypeVar[bool] + # TypeGuard[bool] + # TypeIs[range] def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) @@ -1206,7 +1425,11 @@ def __init__(self, origin, args, *, inst=True, name=None): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) - self.__parameters__ = _collect_parameters(args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_type_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ @@ -1251,8 +1474,7 @@ def __getitem__(self, args): # Preprocess `args`. if not isinstance(args, tuple): args = (args,) - args = tuple(_type_convert(p) for p in args) - args = _unpack_args(args) + args = _unpack_args(*(_type_convert(p) for p in args)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1397,11 +1619,12 @@ def __iter__(self): # parameters are accepted (needs custom __getitem__). class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1413,7 +1636,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1440,21 +1678,6 @@ def __ror__(self, left): return Union[left, self] -class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): - def __init__( - self, origin, nparams, *, removal_version, inst=True, name=None - ): - super().__init__(origin, nparams, inst=inst, name=name) - self._removal_version = removal_version - - def __instancecheck__(self, inst): - import warnings - warnings._deprecated( - f"{self.__module__}.{self._name}", remove=self._removal_version - ) - return super().__instancecheck__(inst) - - class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1526,7 +1749,10 @@ def copy_with(self, params): def __eq__(self, other): if not isinstance(other, (_UnionGenericAlias, types.UnionType)): return NotImplemented - return set(self.__args__) == set(other.__args__) + try: # fast path + return set(self.__args__) == set(other.__args__) + except TypeError: # not hashable, slow path + return _compare_args_orderless(self.__args__, other.__args__) def __hash__(self): return hash(frozenset(self.__args__)) @@ -1644,8 +1870,9 @@ def __typing_unpacked_tuple_args__(self): assert self.__origin__ is Unpack assert len(self.__args__) == 1 arg, = self.__args__ - if isinstance(arg, _GenericAlias): - assert arg.__origin__ is tuple + if isinstance(arg, (_GenericAlias, types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") return arg.__args__ return None @@ -1663,14 +1890,15 @@ class _TypingEllipsis: _TYPING_INTERNALS = frozenset({ '__parameters__', '__orig_bases__', '__orig_class__', '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', - '__callable_proto_members_only__', '__type_params__', + '__non_callable_proto_members__', '__type_params__', }) _SPECIAL_NAMES = frozenset({ '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', + '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', }) # These special attributes will be not collected as protocol members. @@ -1775,6 +2003,31 @@ def _pickle_pskwargs(pskwargs): del _pickle_psargs, _pickle_pskwargs +# Preload these once, as globals, as a micro-optimisation. +# This makes a significant difference to the time it takes +# to do `isinstance()`/`issubclass()` checks +# against runtime-checkable protocols with only one callable member. +_abc_instancecheck = ABCMeta.__instancecheck__ +_abc_subclasscheck = ABCMeta.__subclasscheck__ + + +def _type_check_issubclass_arg_1(arg): + """Raise TypeError if `arg` is not an instance of `type` + in `issubclass(arg, )`. + + In most cases, this is verified by type.__subclasscheck__. + Checking it again unnecessarily would slow down issubclass() checks, + so, we don't perform this check unless we absolutely have to. + + For various error paths, however, + we want to ensure that *this* error message is shown to the user + where relevant, rather than a typing.py-specific error message. + """ + if not isinstance(arg, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') + + class _ProtocolMeta(ABCMeta): # This metaclass is somewhat unfortunate, # but is necessary for several reasons... @@ -1801,11 +2054,6 @@ def __init__(cls, *args, **kwargs): super().__init__(*args, **kwargs) if getattr(cls, "_is_protocol", False): cls.__protocol_attrs__ = _get_protocol_attrs(cls) - # PEP 544 prohibits using issubclass() - # with protocols that have non-method members. - cls.__callable_proto_members_only__ = all( - callable(getattr(cls, attr, None)) for attr in cls.__protocol_attrs__ - ) def __subclasscheck__(cls, other): if cls is Protocol: @@ -1814,22 +2062,24 @@ def __subclasscheck__(cls, other): getattr(cls, '_is_protocol', False) and not _allow_reckless_class_checks() ): - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') - if ( - not cls.__callable_proto_members_only__ - and cls.__dict__.get("__subclasshook__") is _proto_hook - ): - raise TypeError( - "Protocols with non-method members don't support issubclass()" - ) if not getattr(cls, '_is_runtime_protocol', False): + _type_check_issubclass_arg_1(other) raise TypeError( "Instance and class checks can only be used with " "@runtime_checkable protocols" ) - return super().__subclasscheck__(other) + if ( + # this attribute is set by @runtime_checkable: + cls.__non_callable_proto_members__ + and cls.__dict__.get("__subclasshook__") is _proto_hook + ): + _type_check_issubclass_arg_1(other) + non_method_attrs = sorted(cls.__non_callable_proto_members__) + raise TypeError( + "Protocols with non-method members don't support issubclass()." + f" Non-method members: {str(non_method_attrs)[1:-1]}." + ) + return _abc_subclasscheck(cls, other) def __instancecheck__(cls, instance): # We need this method for situations where attributes are @@ -1838,7 +2088,7 @@ def __instancecheck__(cls, instance): return type.__instancecheck__(cls, instance) if not getattr(cls, "_is_protocol", False): # i.e., it's a concrete subclass of a protocol - return super().__instancecheck__(instance) + return _abc_instancecheck(cls, instance) if ( not getattr(cls, '_is_runtime_protocol', False) and @@ -1847,7 +2097,7 @@ def __instancecheck__(cls, instance): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if super().__instancecheck__(instance): + if _abc_instancecheck(cls, instance): return True getattr_static = _lazy_load_getattr_static() @@ -1856,7 +2106,8 @@ def __instancecheck__(cls, instance): val = getattr_static(instance, attr) except AttributeError: break - if val is None and callable(getattr(cls, attr, None)): + # this attribute is set by @runtime_checkable: + if val is None and attr not in cls.__non_callable_proto_members__: break else: return True @@ -1993,8 +2244,9 @@ def __mro_entries__(self, bases): return (self.__origin__,) -@_SpecialForm -def Annotated(self, params): +@_TypedCacheSpecialForm +@_tp_cache(typed=True) +def Annotated(self, *params): """Add context-specific metadata to a type. Example: Annotated[int, runtime_check.Unsigned] indicates to the @@ -2041,7 +2293,7 @@ def Annotated(self, params): where T1, T2 etc. are TypeVars, which would be invalid, because only one type should be passed to Annotated. """ - if not isinstance(params, tuple) or len(params) < 2: + if len(params) < 2: raise TypeError("Annotated[...] should be used " "with at least two arguments (a type and an " "annotation).") @@ -2077,6 +2329,22 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls @@ -2153,7 +2421,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = getattr(base, '__annotations__', {}) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2170,7 +2438,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): value = type(None) if isinstance(value, str): value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) + value = _eval_type(value, base_globals, base_locals, base.__type_params__) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2196,6 +2464,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): raise TypeError('{!r} is not a module, class, method, ' 'or function.'.format(obj)) hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: value = type(None) @@ -2207,7 +2476,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns) + hints[name] = _eval_type(value, globalns, localns, type_params) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2215,7 +2484,7 @@ def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) - if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) @@ -2244,14 +2513,15 @@ def get_origin(tp): Examples:: - assert get_origin(Literal[42]) is Literal - assert get_origin(int) is None - assert get_origin(ClassVar[int]) is ClassVar - assert get_origin(Generic) is Generic - assert get_origin(Generic[T]) is Generic - assert get_origin(Union[T, int]) is Union - assert get_origin(List[Tuple[T, T]][int]) is list - assert get_origin(P.args) is P + >>> P = ParamSpec('P') + >>> assert get_origin(Literal[42]) is Literal + >>> assert get_origin(int) is None + >>> assert get_origin(ClassVar[int]) is ClassVar + >>> assert get_origin(Generic) is Generic + >>> assert get_origin(Generic[T]) is Generic + >>> assert get_origin(Union[T, int]) is Union + >>> assert get_origin(List[Tuple[T, T]][int]) is list + >>> assert get_origin(P.args) is P """ if isinstance(tp, _AnnotatedAlias): return Annotated @@ -2272,11 +2542,12 @@ def get_args(tp): Examples:: - assert get_args(Dict[str, int]) == (str, int) - assert get_args(int) == () - assert get_args(Union[int, Union[T, int], str][int]) == (int, str) - assert get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - assert get_args(Callable[[], T][int]) == ([], int) + >>> T = TypeVar('T') + >>> assert get_args(Dict[str, int]) == (str, int) + >>> assert get_args(int) == () + >>> assert get_args(Union[int, Union[T, int], str][int]) == (int, str) + >>> assert get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) + >>> assert get_args(Callable[[], T][int]) == ([], int) """ if isinstance(tp, _AnnotatedAlias): return (tp.__origin__,) + tp.__metadata__ @@ -2295,12 +2566,15 @@ def is_typeddict(tp): For example:: - class Film(TypedDict): - title: str - year: int - - is_typeddict(Film) # => True - is_typeddict(Union[list, str]) # => False + >>> from typing import TypedDict + >>> class Film(TypedDict): + ... title: str + ... year: int + ... + >>> is_typeddict(Film) + True + >>> is_typeddict(dict) + False """ return isinstance(tp, _TypedDictMeta) @@ -2548,9 +2822,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. -) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -2577,8 +2848,8 @@ class Other(Leaf): # Error reported by type checker OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ """Deprecated alias to builtins.type. @@ -2708,7 +2979,12 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif "__annotate__" in ns: + types = ns["__annotate__"](1) # VALUE + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -2726,25 +3002,31 @@ def __new__(cls, typename, bases, ns): class_getitem = _generic_class_getitem nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes - for key in ns: + for key, val in ns.items(): if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) + elif key not in _special: + if key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + try: + set_name = type(val).__set_name__ + except AttributeError: + pass + else: + try: + set_name(val, nm_tpl, key) + except BaseException as e: + e.add_note( + f"Error calling __set_name__ on {type(val).__name__!r} " + f"instance {key!r} in {typename!r}" + ) + raise + if Generic in bases: nm_tpl.__init_subclass__() return nm_tpl -class _Sentinel: - __slots__ = () - def __repr__(self): - return '' - - -_sentinel = _Sentinel() - - def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. @@ -2816,6 +3098,28 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): """Create a new typed dict class object. @@ -2841,7 +3145,12 @@ def __new__(cls, name, bases, ns, total=True): tp_dict.__orig_bases__ = bases annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotations = ns["__annotate__"](1) # VALUE + else: + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) @@ -2849,33 +3158,65 @@ def __new__(cls, name, bases, ns, total=True): } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + # TODO: Avoid eagerly evaluating annotations in VALUE format. + # Instead, evaluate in FORWARDREF format to figure out which + # keys have Required/NotRequired/ReadOnly qualifiers, and create + # a new __annotate__ function for the resulting TypedDict that + # combines the annotations from this class and its parents. + annotations.update(base.__annotations__) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional + + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): - annotation_origin = get_origin(annotation_type) - if annotation_origin is Annotated: - annotation_args = get_args(annotation_type) - if annotation_args: - annotation_type = annotation_args[0] - annotation_origin = get_origin(annotation_type) - - if annotation_origin is Required: - required_keys.add(annotation_key) - elif annotation_origin is NotRequired: - optional_keys.add(annotation_key) - elif total: + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: + is_required = True + elif NotRequired in qualifiers: + is_required = False + else: + is_required = total + + if is_required: required_keys.add(annotation_key) + optional_keys.discard(annotation_key) else: optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) tp_dict.__total__ = total return tp_dict @@ -2898,15 +3239,15 @@ def TypedDict(typename, fields=_sentinel, /, *, total=True): Usage:: - class Point2D(TypedDict): - x: int - y: int - label: str - - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check - - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + >>> class Point2D(TypedDict): + ... x: int + ... y: int + ... label: str + ... + >>> a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + >>> b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + True The type info can be accessed via the Point2D.__annotations__ dict, and the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. @@ -2934,6 +3275,14 @@ class Point2D(TypedDict): y: NotRequired[int] # the "y" key can be omitted See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ if fields is _sentinel or fields is None: import warnings @@ -3010,6 +3359,26 @@ class Movie(TypedDict): return _GenericAlias(self, (item,)) +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. @@ -3229,7 +3598,7 @@ def __enter__(self) -> 'TextIO': def reveal_type[T](obj: T, /) -> T: - """Reveal the inferred type of a variable. + """Ask a static type checker to reveal the inferred type of an expression. When a static type checker encounters a call to ``reveal_type()``, it will emit the inferred type of the argument:: @@ -3241,7 +3610,7 @@ def reveal_type[T](obj: T, /) -> T: will produce output similar to 'Revealed type is "builtins.int"'. At runtime, the function prints the runtime type of the - argument and returns it unchanged. + argument and returns the argument unchanged. """ print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr) return obj @@ -3345,7 +3714,7 @@ def override[F: _Func](method: F, /) -> F: Usage:: class Base: - def method(self) -> None: ... + def method(self) -> None: pass class Child(Base): @@ -3404,8 +3773,8 @@ def get_protocol_members(tp: type, /) -> frozenset[str]: >>> class P(Protocol): ... def a(self) -> str: ... ... b: int - >>> get_protocol_members(P) - frozenset({'a', 'b'}) + >>> get_protocol_members(P) == frozenset({'a', 'b'}) + True Raise a TypeError for arguments that are not Protocols. """ @@ -3425,7 +3794,17 @@ def __getattr__(attr): obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib - obj = _alias(getattr(contextlib, f"Abstract{attr}"), 1, name=attr) + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index f1f6c911ef17d9..324e5d038aef03 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -57,9 +57,9 @@ def testMultiply(self): from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure, doModuleCleanups, enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .suite import BaseTestSuite, TestSuite # noqa: F401 from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main +from .main import TestProgram, main # noqa: F401 from .runner import TextTestRunner, TextTestResult from .signals import installHandler, registerResult, removeResult, removeHandler # IsolatedAsyncioTestCase will be imported lazily. diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index bd2a471156065b..bd06eb3207697a 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -25,12 +25,15 @@ class IsolatedAsyncioTestCase(TestCase): # them inside the same task. # Note: the test case modifies event loop policy if the policy was not instantiated - # yet. + # yet, unless loop_factory=asyncio.EventLoop is set. # asyncio.get_event_loop_policy() creates a default policy on demand but never # returns None # I believe this is not an issue in user level tests but python itself for testing # should reset a policy in every test module # by calling asyncio.set_event_loop_policy(None) in tearDownModule() + # or set loop_factory=asyncio.EventLoop + + loop_factory = None def __init__(self, methodName='runTest'): super().__init__(methodName) @@ -87,9 +90,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) @@ -118,7 +125,7 @@ def _callMaybeAsync(self, func, /, *args, **kwargs): def _setupAsyncioRunner(self): assert self._asyncioRunner is None, 'asyncio runner is already initialized' - runner = asyncio.Runner(debug=True) + runner = asyncio.Runner(debug=True, loop_factory=self.loop_factory) self._asyncioRunner = runner def _tearDownAsyncioRunner(self): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 811557498bb30e..55c79d353539ca 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -332,6 +332,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -586,9 +603,18 @@ def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() @@ -606,6 +632,7 @@ def run(self, result=None): else: stopTestRun = None + result.startTest(self) try: testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or @@ -616,9 +643,6 @@ def run(self, result=None): _addSkip(result, self, skip_why) return result - # Increase the number of tests only if it hasn't been skipped - result.startTest(self) - expecting_failure = ( getattr(self, "__unittest_expecting_failure__", False) or getattr(testMethod, "__unittest_expecting_failure__", False) @@ -813,6 +837,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 9a3e5cc4bf30e5..22797b83a68bc8 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -254,6 +254,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -307,6 +308,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): raise ImportError('Start directory is not importable: %r' % start_dir) tests = list(self._find_tests(start_dir, pattern)) + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index d29a9f91fcca42..c3869de3f6f18e 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,7 +269,7 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0: + if self.result.testsRun == 0 and len(self.result.skipped) == 0: sys.exit(_NO_TESTS_EXITCODE) elif self.result.wasSuccessful(): sys.exit(0) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index e8c8360e0ae13e..2bbbcf40e21543 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -573,7 +573,7 @@ def __get_return_value(self): if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -830,6 +830,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) @@ -857,7 +860,7 @@ def _format_mock_call_signature(self, args, kwargs): def _format_mock_failure_message(self, args, kwargs, action='call'): - message = 'expected %s not found.\nExpected: %s\nActual: %s' + message = 'expected %s not found.\nExpected: %s\n Actual: %s' expected_string = self._format_mock_call_signature(args, kwargs) call_args = self.call_args actual_string = self._format_mock_call_signature(*call_args) @@ -960,7 +963,7 @@ def assert_called_with(self, /, *args, **kwargs): if self.call_args is None: expected = self._format_mock_call_signature(args, kwargs) actual = 'not called.' - error_message = ('expected call not found.\nExpected: %s\nActual: %s' + error_message = ('expected call not found.\nExpected: %s\n Actual: %s' % (expected, actual)) raise AssertionError(error_message) @@ -1010,8 +1013,8 @@ def assert_has_calls(self, calls, any_order=False): for e in expected]) raise AssertionError( f'{problem}\n' - f'Expected: {_CallList(calls)}' - f'{self._calls_repr(prefix="Actual").rstrip(".")}' + f'Expected: {_CallList(calls)}\n' + f' Actual: {safe_repr(self.mock_calls)}' ) from cause return @@ -1085,7 +1088,7 @@ def _get_child_mock(self, /, **kw): return klass(**kw) - def _calls_repr(self, prefix="Calls"): + def _calls_repr(self): """Renders self.mock_calls as a string. Example: "\nCalls: [call(1), call(2)]." @@ -1095,7 +1098,7 @@ def _calls_repr(self, prefix="Calls"): """ if not self.mock_calls: return "" - return f"\n{prefix}: {safe_repr(self.mock_calls)}." + return f"\nCalls: {safe_repr(self.mock_calls)}." # Denylist for forbidden attribute names in safe mode @@ -1234,6 +1237,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -1505,13 +1511,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1524,7 +1529,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: @@ -1745,7 +1755,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created @@ -1819,7 +1829,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -2229,8 +2240,11 @@ def __get__(self, obj, _type=None): return self.create_mock() -_CODE_ATTRS = dir(CodeType) -_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) +try: + _CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + _CODE_ATTRS = dir(CodeType) +except ValueError: + _CODE_SIG = None class AsyncMockMixin(Base): @@ -2250,9 +2264,12 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=_CODE_ATTRS) - code_mock.__dict__["_spec_class"] = CodeType - code_mock.__dict__["_spec_signature"] = _CODE_SIG + if _CODE_SIG: + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG + else: + code_mock = NonCallableMock(spec_set=CodeType) code_mock.co_flags = ( inspect.CO_COROUTINE + inspect.CO_VARARGS @@ -2739,6 +2756,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if not unsafe: _check_spec_arg_typos(kwargs) + _name = kwargs.pop('name', _name) + _new_name = _name + if _parent is None: + # for a top level object no _new_name should be set + _new_name = '' + _kwargs.update(kwargs) Klass = MagicMock @@ -2756,13 +2779,6 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, elif is_type and instance and not _instance_callable(spec): Klass = NonCallableMagicMock - _name = _kwargs.pop('name', _name) - - _new_name = _name - if _parent is None: - # for a top level object no _new_name should be set - _new_name = '' - mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) @@ -2779,9 +2795,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) for entry in dir(spec): if _is_magic(entry): @@ -2802,9 +2821,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, except AttributeError: continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2815,14 +2837,13 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) @@ -2833,6 +2854,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 9e56f658027f4d..3ace0a5b7bf2ef 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -97,12 +97,10 @@ def _restoreStdout(self): sys.stdout = self._original_stdout sys.stderr = self._original_stderr - if self._stdout_buffer is not None: - self._stdout_buffer.seek(0) - self._stdout_buffer.truncate() - if self._stderr_buffer is not None: - self._stderr_buffer.seek(0) - self._stderr_buffer.truncate() + self._stdout_buffer.seek(0) + self._stdout_buffer.truncate() + self._stderr_buffer.seek(0) + self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index e3c020e0ace96d..2bcadf0c998bd9 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -274,7 +274,7 @@ def run(self, test): infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) - elif run == 0: + elif run == 0 and not skipped: self.stream.write("NO TESTS RAN") else: self.stream.write("OK") diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index c129b0d7971d71..8f724f907d4217 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -525,7 +525,7 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if netloc or (scheme and scheme in uses_netloc) or url[:2] == '//': if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -763,42 +763,48 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - qs, _coerce_result = _coerce_args(qs) - separator, _ = _coerce_args(separator) - if not separator or (not isinstance(separator, (str, bytes))): + if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") + if isinstance(qs, str): + if not isinstance(separator, str): + separator = str(separator, 'ascii') + eq = '=' + def _unquote(s): + return unquote_plus(s, encoding=encoding, errors=errors) + else: + if not qs: + return [] + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + if isinstance(separator, str): + separator = bytes(separator, 'ascii') + eq = b'=' + def _unquote(s): + return unquote_to_bytes(s.replace(b'+', b' ')) + + if not qs: + return [] # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS # attack via post bodies with many fields. if max_num_fields is not None: - num_fields = 1 + qs.count(separator) if qs else 0 + num_fields = 1 + qs.count(separator) if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') r = [] - query_args = qs.split(separator) if qs else [] - for name_value in query_args: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: + for name_value in qs.split(separator): + if name_value or strict_parsing: + name, has_eq, value = name_value.partition(eq) + if not has_eq and strict_parsing: raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) + if value or keep_blank_values: + name = _unquote(name) + value = _unquote(value) + r.append((name, value)) return r def unquote_plus(string, encoding='utf-8', errors='replace'): @@ -816,14 +822,6 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -def __getattr__(name): - if name == 'Quoter': - warnings.warn('Deprecated in 3.11. ' - 'urllib.parse.Quoter will be removed in Python 3.14. ' - 'It was not intended to be a public API.', - DeprecationWarning, stacklevel=2) - return _Quoter - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') class _Quoter(dict): """A mapping from bytes numbers (in range(0,256)) to strings. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 1d03259b918c33..58b0cb574a764a 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -108,7 +108,7 @@ # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: @@ -650,6 +650,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) @@ -2490,7 +2491,7 @@ def getproxies_environment(): # select only environment variables which end in (after making lowercase) _proxy proxies = {} environment = [] - for name in os.environ.keys(): + for name in os.environ: # fast screen underscore position before more expensive case-folding if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy": value = os.environ[name] @@ -2563,6 +2564,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2579,20 +2581,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2615,6 +2614,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "" should bypass the proxy server for all intranet addresses + if test == '': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2713,7 +2737,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2723,40 +2747,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/uuid.py b/Lib/uuid.py index 470bc0d68597ab..c286eac38e1ef4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,13 +53,16 @@ __author__ = 'Ka-Ping Yee ' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin', 'emscripten', 'wasi'): +if sys.platform in {'win32', 'darwin', 'emscripten', 'wasi'}: _AIX = _LINUX = False +elif sys.platform == 'linux': + _LINUX = True + _AIX = False else: import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False @@ -564,32 +567,16 @@ def _netstat_getnode(): # This works on AIX and might work on Tru64 UNIX. return _find_mac_under_heading('netstat', '-ian', b'Address') -def _ipconfig_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - -def _netbios_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) _UuidCreate = getattr(_uuid, "UuidCreate", None) - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None _generate_time_safe = None _UuidCreate = None - _has_uuid_generate_time_safe = None - - -def _load_system_functions(): - """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index d960bf3bd82ac5..fa69d5846f2fa7 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -107,6 +107,33 @@ def _venv_path(self, env_dir, name): } return sysconfig.get_path(name, scheme='venv', vars=vars) + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -129,8 +156,7 @@ def create_if_needed(d): context = types.SimpleNamespace() context.env_dir = env_dir context.env_name = os.path.split(env_dir)[1] - prompt = self.prompt if self.prompt is not None else context.env_name - context.prompt = '(%s) ' % prompt + context.prompt = self.prompt if self.prompt is not None else context.env_name create_if_needed(env_dir) executable = sys._base_executable if not executable: # see gh-96861 @@ -139,6 +165,11 @@ def create_if_needed(d): 'check that your PATH environment variable is ' 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename @@ -167,7 +198,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -222,67 +253,26 @@ def create_configuration(self, context): args = ' '.join(args) f.write(f'command = {sys.executable} -m venv {args}\n') - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build() or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) def create_git_ignore_file(self, context): @@ -298,22 +288,23 @@ def create_git_ignore_file(self, context): 'see https://docs.python.org/3/library/venv.html\n') file.write('*\n') - def setup_python(self, context): - """ - Set up a Python executable in the environment. + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. - :param context: The information for the environment creation request - being processed. - """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -321,30 +312,105 @@ def setup_python(self, context): copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + else: + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dst) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) if sysconfig.is_python_build(): # copy init.tcl @@ -437,6 +503,14 @@ def install_scripts(self, context, path): """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: @@ -444,8 +518,7 @@ def install_scripts(self, context, path): dirs.remove(d) continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -456,20 +529,25 @@ def install_scripts(self, context, path): if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 458740a35b0d20..cbd4873f012246 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,12 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting + # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -39,14 +36,18 @@ deactivate () { deactivate nondestructive # on Windows, a path can contain colons and backslashes and has to be converted: -if [ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ] ; then - # transform D:\path\to\venv to /d/path/to/venv on MSYS - # and to /cygdrive/d/path/to/venv on Cygwin - export VIRTUAL_ENV=$(cygpath "__VENV_DIR__") -else - # use the path as-is - export VIRTUAL_ENV="__VENV_DIR__" -fi +case "$(uname)" in + CYGWIN*|MSYS*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath "__VENV_DIR__") + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV="__VENV_DIR__" + ;; +esac _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" @@ -65,13 +66,10 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1="(__VENV_PROMPT__) ${PS1:-}" export PS1 fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 96% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index 565df23d1e2a13..25c42756789bbc 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -57,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index c1c3c82ee37f10..dd5ea8eb67b90a 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -15,8 +15,8 @@ if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index 9caf138a919a86..c707f1988b0acc 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -19,7 +19,7 @@ setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" + set prompt = "(__VENV_PROMPT__) $prompt" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index 32e58072b9cc33..d344ceaebb0378 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -5,7 +5,7 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" @@ -58,15 +58,16 @@ def _formatwarnmsg_impl(msg): # catch Exception, not only ImportError and RecursionError. except Exception: # don't suggest to enable tracemalloc if it's not available - tracing = True + suggest_tracemalloc = False tb = None else: - tracing = tracemalloc.is_tracing() try: + suggest_tracemalloc = not tracemalloc.is_tracing() tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore + suggest_tracemalloc = False tb = None if tb is not None: @@ -85,7 +86,7 @@ def _formatwarnmsg_impl(msg): if line: line = line.strip() s += ' %s\n' % line - elif not tracing: + elif suggest_tracemalloc: s += (f'{category}: Enable tracemalloc to get the object ' f'allocation traceback\n') return s @@ -131,7 +132,7 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'message' -- a regex that the warning message must match 'category' -- a class that the warning must be a subclass of @@ -139,14 +140,18 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(message, str), "message must be a string" - assert isinstance(category, type), "category must be a class" - assert issubclass(category, Warning), "category must be a Warning subclass" - assert isinstance(module, str), "module must be a string" - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(message, str): + raise TypeError("message must be a string") + if not isinstance(category, type) or not issubclass(category, Warning): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") if message or module: import re @@ -166,16 +171,18 @@ def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). A simple filter matches all modules and messages. - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'category' -- a class that the warning must be a subclass of 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") _add_filter(action, None, category, None, lineno, append=append) def _add_filter(*item, append): @@ -241,8 +248,7 @@ def _setoption(arg): def _getaction(action): if not action: return "default" - if action == "all": return "always" # Alias - for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): + for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'): if a.startswith(action): return a raise _OptionError("invalid action: %r" % (action,)) @@ -325,8 +331,8 @@ def warn(message, category=None, stacklevel=1, source=None, raise ValueError except ValueError: globals = sys.__dict__ - filename = "sys" - lineno = 1 + filename = "" + lineno = 0 else: globals = frame.f_globals filename = frame.f_code.co_filename @@ -390,7 +396,7 @@ def warn_explicit(message, category, filename, lineno, if onceregistry.get(oncekey): return onceregistry[oncekey] = 1 - elif action == "always": + elif action in {"always", "all"}: pass elif action == "module": registry[key] = 1 @@ -508,6 +514,135 @@ def __exit__(self, *exc_info): self._module._showwarnmsg_impl = self._showwarnmsg_impl +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, *args, **kwargs): + if cls is arg: + warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python) + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + # Or otherwise, which likely means it's a builtin such as + # object's implementation of __init_subclass__. + else: + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = __init_subclass__ + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): @@ -554,7 +689,7 @@ def extract(): # filters contains a sequence of filter 5-tuples # The components of the 5-tuple are: -# - an action: error, ignore, always, default, module, or once +# - an action: error, ignore, always, all, default, module, or once # - a compiled regex that must match the warning message # - a class representing the warning category # - a compiled regex that must match the module that is being warned diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py old mode 100755 new mode 100644 index 8b0628745c57fc..6fca257c02664f --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 """Interfaces for launching and remotely controlling web browsers.""" # Maintained by Georg Brandl. @@ -11,14 +10,17 @@ __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + class Error(Exception): pass + _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser + def register(name, klass, instance=None, *, preferred=False): """Register a browser connector.""" with _lock: @@ -34,6 +36,7 @@ def register(name, klass, instance=None, *, preferred=False): else: _tryorder.append(name) + def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: @@ -64,6 +67,7 @@ def get(using=None): return command[0]() raise Error("could not locate runnable browser") + # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import *". @@ -87,6 +91,7 @@ def open(url, new=0, autoraise=True): return True return False + def open_new(url): """Open url in a new window of the default browser. @@ -94,6 +99,7 @@ def open_new(url): """ return open(url, 1) + def open_new_tab(url): """Open url in a new page ("tab") of the default browser. @@ -136,7 +142,7 @@ def _synthesize(browser, *, preferred=False): # General parent classes -class BaseBrowser(object): +class BaseBrowser: """Parent class for all browsers. Do not use directly.""" args = ['%s'] @@ -197,7 +203,7 @@ def open(self, url, new=0, autoraise=True): else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) - return (p.poll() is None) + return p.poll() is None except OSError: return False @@ -225,7 +231,8 @@ def _invoke(self, args, remote, autoraise, url=None): # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] - if opt: raise_opt = [opt] + if opt: + raise_opt = [opt] cmdline = [self.name] + raise_opt + args @@ -266,8 +273,8 @@ def open(self, url, new=0, autoraise=True): else: action = self.remote_action_newtab else: - raise Error("Bad 'new' parameter to open(); " + - "expected 0, 1, or 2, got %s" % new) + raise Error("Bad 'new' parameter to open(); " + f"expected 0, 1, or 2, got {new}") args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] @@ -302,7 +309,7 @@ class Epiphany(UnixBrowser): class Chrome(UnixBrowser): - "Launcher class for Google Chrome browser." + """Launcher class for Google Chrome browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -310,11 +317,12 @@ class Chrome(UnixBrowser): remote_action_newtab = "" background = True + Chromium = Chrome class Opera(UnixBrowser): - "Launcher class for Opera browser." + """Launcher class for Opera browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -324,7 +332,7 @@ class Opera(UnixBrowser): class Elinks(UnixBrowser): - "Launcher class for Elinks browsers." + """Launcher class for Elinks browsers.""" remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" @@ -387,11 +395,11 @@ def open(self, url, new=0, autoraise=True): except OSError: return False else: - return (p.poll() is None) + return p.poll() is None class Edge(UnixBrowser): - "Launcher class for Microsoft Edge browser." + """Launcher class for Microsoft Edge browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -418,12 +426,18 @@ def register_X_browsers(): if shutil.which("gio"): register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"])) - # Equivalent of gio open before 2015 - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + xdg_desktop = os.getenv("XDG_CURRENT_DESKTOP", "").split(":") + + # The default GNOME3 browser + if (("GNOME" in xdg_desktop or + "GNOME_DESKTOP_SESSION_ID" in os.environ) and + shutil.which("gvfs-open")): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) # The default KDE browser - if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + if (("KDE" in xdg_desktop or + "KDE_FULL_SESSION" in os.environ) and + shutil.which("kfmclient")): register("kfmclient", Konqueror, Konqueror("kfmclient")) # Common symbolic link for the default X11 browser @@ -455,7 +469,6 @@ def register_X_browsers(): if shutil.which("opera"): register("opera", None, Opera("opera")) - if shutil.which("microsoft-edge"): register("microsoft-edge", None, Edge("microsoft-edge")) @@ -472,6 +485,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -495,12 +511,18 @@ def register_standard_browsers(): register("microsoft-edge", None, Edge("MicrosoftEdge.exe")) else: # Prefer X browsers if present - if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + # + # NOTE: Do not check for X11 browser on macOS, + # XQuartz installation sets a DISPLAY environment variable and will + # autostart when someone tries to access the display. Mac users in + # general don't need an X11 browser. + if sys.platform != "darwin" and (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError, NotADirectoryError): pass else: global _os_preferred_browser @@ -569,15 +591,17 @@ def __init__(self, name='default'): super().__init__(name) def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + url = url.replace('"', '%22') if self.name == 'default': - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + script = f'open location "{url}"' # opens in default browser else: script = f''' - tell application "%s" + tell application "{self.name}" activate - open location "%s" + open location "{url}" end - '''%(self.name, url.replace('"', '%22')) + ''' osapipe = os.popen("osascript", "w") if osapipe is None: @@ -587,34 +611,96 @@ def open(self, url, new=0, autoraise=True): rc = osapipe.close() return not rc +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong -def main(): - import getopt - usage = """Usage: %s [-n | -t | -h] url - -n: open new window - -t: open new tab - -h, --help: show help""" % sys.argv[0] - try: - opts, args = getopt.getopt(sys.argv[1:], 'ntdh',['help']) - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - new_win = 0 - for o, a in opts: - if o == '-n': new_win = 1 - elif o == '-t': new_win = 2 - elif o == '-h' or o == '--help': - print(usage, file=sys.stderr) - sys.exit() - if len(args) != 1: - print(usage, file=sys.stderr) - sys.exit(1) - - url = args[0] - open(url, new_win) + class IOSBrowser(BaseBrowser): + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: + return False + + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + + return True + + +def parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser(description="Open URL in a web browser.") + parser.add_argument("url", help="URL to open") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-n", "--new-window", action="store_const", + const=1, default=0, dest="new_win", + help="open new window") + group.add_argument("-t", "--new-tab", action="store_const", + const=2, default=0, dest="new_win", + help="open new tab") + + args = parser.parse_args(arg_list) + + return args + + +def main(arg_list: list[str] | None = None): + args = parse_args(arg_list) + + open(args.url, args.new_win) print("\a") + if __name__ == "__main__": main() diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fab851c5a44430..05d2ba4c664e5e 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -5,7 +5,7 @@ written by Barry Warsaw. """ -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index 93d01a863adf59..a0f2397fcf0006 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -84,10 +84,6 @@ def get_environ(self): env['PATH_INFO'] = urllib.parse.unquote(path, 'iso-8859-1') env['QUERY_STRING'] = query - - host = self.address_string() - if host != self.client_address[0]: - env['REMOTE_HOST'] = host env['REMOTE_ADDR'] = self.client_address[0] if self.headers.get('content-type') is None: diff --git a/Lib/wsgiref/util.py b/Lib/wsgiref/util.py index cbbe094cba1624..63b923317373f5 100644 --- a/Lib/wsgiref/util.py +++ b/Lib/wsgiref/util.py @@ -4,7 +4,7 @@ __all__ = [ 'FileWrapper', 'guess_scheme', 'application_uri', 'request_uri', - 'shift_path_info', 'setup_testing_defaults', + 'shift_path_info', 'setup_testing_defaults', 'is_hop_by_hop', ] diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py index 97cf9a6429993d..dd7fb996afd616 100644 --- a/Lib/xml/dom/__init__.py +++ b/Lib/xml/dom/__init__.py @@ -137,4 +137,4 @@ class UserDataHandler: EMPTY_NAMESPACE = None EMPTY_PREFIX = None -from .domreg import getDOMImplementation, registerDOMImplementation +from .domreg import getDOMImplementation, registerDOMImplementation # noqa: F401 diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 40a9b22292479f..986e6c3bbe90f6 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -79,8 +79,8 @@ class LimitedRecursiveIncludeError(FatalIncludeError): # @param parse Parse mode. Either "xml" or "text". # @param encoding Optional text encoding (UTF-8 by default for "text"). # @return The expanded resource. If the parse mode is "xml", this -# is an ElementTree instance. If the parse mode is "text", this -# is a Unicode string. If the loader fails, it can return None +# is an Element instance. If the parse mode is "text", this +# is a string. If the loader fails, it can return None # or raise an OSError exception. # @throws OSError If the loader fails to load the resource. @@ -98,7 +98,7 @@ def default_loader(href, parse, encoding=None): ## # Expand XInclude directives. # -# @param elem Root element. +# @param elem Root Element or any ElementTree of a tree to be expanded # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. @@ -106,12 +106,13 @@ def default_loader(href, parse, encoding=None): # relative include file references. # @param max_depth The maximum number of recursive inclusions. # Limited to reduce the risk of malicious content explosion. -# Pass a negative value to disable the limitation. +# Pass None to disable the limitation. # @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws IOError If the function fails to load a given resource. -# @returns the node or its replacement if it was an XInclude node +# @throws OSError If the function fails to load a given resource. +# @throws ValueError If negative {@link max_depth} is passed. +# @returns None. Modifies tree pointed by {@link elem} def include(elem, loader=None, base_url=None, max_depth=DEFAULT_MAX_INCLUSION_DEPTH): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 42574eefd81beb..ce67d7d7d54748 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -99,6 +99,7 @@ import collections import collections.abc import contextlib +import weakref from . import ElementPath @@ -200,7 +201,7 @@ def __len__(self): def __bool__(self): warnings.warn( - "Testing an element's truth value will raise an exception in " + "Testing an element's truth value will always return True in " "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", DeprecationWarning, stacklevel=2 @@ -1223,13 +1224,14 @@ def iterparse(source, events=None, parser=None): # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(source): + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + else: close_source = False + + def iterator(source): try: - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - yield None while True: yield from pullparser.read_events() # load event buffer @@ -1239,18 +1241,30 @@ def iterator(source): pullparser.feed(data) root = pullparser._close_and_return_root() yield from pullparser.read_events() - it.root = root + it = wr() + if it is not None: + it.root = root finally: if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() + + def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) + if close_source: + source.close() + it = IterParseIterator() it.root = None - del iterator, IterParseIterator - - next(it) + wr = weakref.ref(it) return it @@ -1306,6 +1320,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1712,6 +1731,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index b657310207cfe5..fe4582c6f8b758 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,7 +55,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: @@ -92,3 +92,9 @@ def make_parser(parser_list=()): def _create_parser(parser_name): drv_module = __import__(parser_name,{},{},['create_parser']) return drv_module.create_parser() + + +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index b9ad52692db8dd..ba3c1e98517429 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -214,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 4dddb1d10e08bd..90a356fbb8eae4 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -578,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = True # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 995d2678e87536..e2aaf8bab4913d 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -371,7 +371,7 @@ def _sanitize_filename(filename): return filename -class ZipInfo (object): +class ZipInfo: """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( @@ -379,7 +379,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - '_compresslevel', + 'compress_level', 'comment', 'extra', 'create_system', @@ -395,6 +395,7 @@ class ZipInfo (object): 'compress_size', 'file_size', '_raw_time', + '_end_offset', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): @@ -412,7 +413,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self._compresslevel = None # Level for the compressor + self.compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -429,10 +430,20 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.external_attr = 0 # External file attributes self.compress_size = 0 # Size of the compressed file self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file + # Maintain backward compatibility with the old protected attribute name. + @property + def _compresslevel(self): + return self.compress_level + + @_compresslevel.setter + def _compresslevel(self, value): + self.compress_level = value + def __repr__(self): result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] if self.compress_type != ZIP_STORED: @@ -594,7 +605,15 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): def is_dir(self): """Return True if this archive member is a directory.""" - return self.filename.endswith('/') + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False # ZIP encryption uses the CRC32 one-byte primitive for scrambling some @@ -921,7 +940,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1136,8 +1155,12 @@ def seek(self, offset, whence=os.SEEK_SET): read_offset = new_pos - curr_pos buff_offset = read_offset + self._offset + if buff_offset >= 0 and buff_offset < len(self._readbuffer): + # Just move the _offset index if the new position is in the _readbuffer + self._offset = buff_offset + read_offset = 0 # Fast seek uncompressed unencrypted file - if self._compress_type == ZIP_STORED and self._decrypter is None and read_offset > 0: + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset > 0: # disable CRC checking after first seeking - it would be invalid self._expected_crc = None # seek actual file taking already buffered data into account @@ -1148,10 +1171,6 @@ def seek(self, offset, whence=os.SEEK_SET): # flush read buffer self._readbuffer = b'' self._offset = 0 - elif buff_offset >= 0 and buff_offset < len(self._readbuffer): - # Just move the _offset index if the new position is in the _readbuffer - self._offset = buff_offset - read_offset = 0 elif read_offset < 0: # Position is before the current position. Reset the ZipExtFile self._fileobj.seek(self._orig_compress_start) @@ -1189,7 +1208,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compresslevel) + zinfo.compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1198,6 +1217,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1488,6 +1515,12 @@ def _RealGetContents(self): if self.debug > 2: print("total", total) + end_offset = self.start_dir + for zinfo in sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset, + reverse=True): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset def namelist(self): """Return a list of file names in the archive.""" @@ -1560,7 +1593,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1595,7 +1629,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1644,6 +1678,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)") + # check for encrypted flag & handle password is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED if is_encrypted: @@ -1657,7 +1695,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise @@ -1707,8 +1745,9 @@ def _open_to_write(self, zinfo, force_zip64=False): def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + as possible. 'member' may be a filename or a ZipInfo object. You can + specify a different directory using 'path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1719,9 +1758,10 @@ def extract(self, member, path=None, pwd=None): def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). + directory. 'path' specifies a different directory to extract to. + 'members' is optional and must be a subset of the list returned + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() @@ -1772,7 +1812,7 @@ def _extract_member(self, member, targetpath, pwd): # filter illegal characters on Windows arcname = self._sanitize_windows_name(arcname, os.path.sep) - if not arcname: + if not arcname and not member.is_dir(): raise ValueError("Empty filename.") targetpath = os.path.join(targetpath, arcname) @@ -1781,11 +1821,15 @@ def _extract_member(self, member, targetpath, pwd): # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if member.is_dir(): if not os.path.isdir(targetpath): - os.mkdir(targetpath) + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise return targetpath with self.open(member, pwd=pwd) as source, \ @@ -1843,9 +1887,9 @@ def write(self, filename, arcname=None, zinfo.compress_type = self.compression if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel else: - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) @@ -1863,7 +1907,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel if zinfo.filename.endswith('/'): zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag @@ -1884,7 +1928,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo.compress_type = compress_type if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: @@ -2227,12 +2271,79 @@ def _compile(file, optimize=-1): return (fname, archivename) +def main(args=None): + import argparse + + description = 'A simple command-line interface for zipfile module.' + parser = argparse.ArgumentParser(description=description) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-l', '--list', metavar='', + help='Show listing of a zipfile') + group.add_argument('-e', '--extract', nargs=2, + metavar=('', ''), + help='Extract zipfile into target dir') + group.add_argument('-c', '--create', nargs='+', + metavar=('', ''), + help='Create zipfile from sources') + group.add_argument('-t', '--test', metavar='', + help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='', + help='Specify encoding of member names for -l, -e and -t') + args = parser.parse_args(args) + + encoding = args.metadata_encoding + + if args.test is not None: + src = args.test + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + badfile = zf.testzip() + if badfile: + print("The following enclosed file is corrupted: {!r}".format(badfile)) + print("Done testing") + + elif args.list is not None: + src = args.list + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.printdir() + + elif args.extract is not None: + src, curdir = args.extract + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.extractall(curdir) + + elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + + zip_name = args.create.pop(0) + files = args.create + + def addToZip(zf, path, zippath): + if os.path.isfile(path): + zf.write(path, zippath, ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in sorted(os.listdir(path)): + addToZip(zf, + os.path.join(path, nm), os.path.join(zippath, nm)) + # else: ignore + + with ZipFile(zip_name, 'w') as zf: + for path in files: + zippath = os.path.basename(path) + if not zippath: + zippath = os.path.basename(os.path.dirname(path)) + if zippath in ('', os.curdir, os.pardir): + zippath = '' + addToZip(zf, path, zippath) + + from ._path import ( # noqa: E402 Path, # used privately for tests CompleteDirs, # noqa: F401 ) - -# used privately for tests -from .__main__ import main # noqa: F401, E402 diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py index a9e5fb1b8d72c4..868d99efc3c4a3 100644 --- a/Lib/zipfile/__main__.py +++ b/Lib/zipfile/__main__.py @@ -1,77 +1,4 @@ -import sys -import os -from . import ZipFile, ZIP_DEFLATED - - -def main(args=None): - import argparse - - description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='', - help='Show listing of a zipfile') - group.add_argument('-e', '--extract', nargs=2, - metavar=('', ''), - help='Extract zipfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('', ''), - help='Create zipfile from sources') - group.add_argument('-t', '--test', metavar='', - help='Test if a zipfile is valid') - parser.add_argument('--metadata-encoding', metavar='', - help='Specify encoding of member names for -l, -e and -t') - args = parser.parse_args(args) - - encoding = args.metadata_encoding - - if args.test is not None: - src = args.test - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print("Done testing") - - elif args.list is not None: - src = args.list - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.printdir() - - elif args.extract is not None: - src, curdir = args.extract - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.extractall(curdir) - - elif args.create is not None: - if encoding: - print("Non-conforming encodings not supported with -c.", - file=sys.stderr) - sys.exit(1) - - zip_name = args.create.pop(0) - files = args.create - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in sorted(os.listdir(path)): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(zip_name, 'w') as zf: - for path in files: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - +from . import main if __name__ == "__main__": main() diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 78c413563bb2b1..f5ea18cee61930 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -5,8 +5,10 @@ import contextlib import pathlib import re +import stat +import sys -from .glob import translate +from .glob import Translator __all__ = ['Path'] @@ -147,6 +149,16 @@ def make(cls, source): source.__class__ = cls return source + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + class FastLookup(CompleteDirs): """ @@ -168,8 +180,10 @@ def _name_set(self): def _extract_text_encoding(encoding=None, *args, **kwargs): - # stacklevel=3 so that the caller of the caller see any warning. - return io.text_encoding(encoding, 3), args, kwargs + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + stack_level = 3 + is_pypy + return io.text_encoding(encoding, stack_level), args, kwargs class Path: @@ -194,13 +208,13 @@ class Path: Path accepts the zipfile object itself or a filename - >>> root = Path(zf) + >>> path = Path(zf) From there, several path operations are available. Directory iteration (including the zip file itself): - >>> a, b = root.iterdir() + >>> a, b = path.iterdir() >>> a Path('mem/abcde.zip', 'a.txt') >>> b @@ -238,16 +252,38 @@ class Path: 'mem/abcde.zip/b/c.txt' At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. + resolve to the zipfile. - >>> root.name + >>> str(path) + 'mem/abcde.zip/' + >>> path.name 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass """ __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" @@ -355,17 +391,21 @@ def match(self, path_pattern): def is_symlink(self): """ - Return whether this path is a symlink. Always false (python/cpython#82102). + Return whether this path is a symlink. """ - return False + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) def glob(self, pattern): if not pattern: raise ValueError(f"Unacceptable pattern: {pattern!r}") prefix = re.escape(self.at) - matches = re.compile(prefix + translate(pattern)).fullmatch - return map(self._next, filter(matches, self.root.namelist())) + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch + names = (data.filename for data in self.root.filelist) + return map(self._next, filter(matches, names)) def rglob(self, pattern): return self.glob(f'**/{pattern}') diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index 4a2e665e27078a..69c41d77c3f654 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,18 +1,97 @@ +import os import re -def translate(pattern): - r""" - Given a glob pattern, produce a regex that matches it. +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) - >>> translate('*.txt') - '[^/]*\\.txt' - >>> translate('a?txt') - 'a.txt' - >>> translate('**/*') - '.*/[^/]*' + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators """ - return ''.join(map(replace, separate(pattern))) + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.translate_core(pattern)) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) def separate(pattern): @@ -25,16 +104,3 @@ def separate(pattern): ['a', '[?]', 'txt'] """ return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) - - -def replace(match): - """ - Perform the replacements for a match from :func:`separate`. - """ - - return match.group('set') or ( - re.escape(match.group(0)) - .replace('\\*\\*', r'.*') - .replace('\\*', r'[^/]*') - .replace('\\?', r'.') - ) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 5b9f614f02f7af..68f031f89c9996 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths @@ -15,7 +13,7 @@ #from importlib import _bootstrap_external #from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external -from _frozen_importlib_external import _unpack_uint16, _unpack_uint32 +from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open @@ -40,8 +38,14 @@ class ZipImportError(ImportError): _module_type = type(sys) END_CENTRAL_DIR_SIZE = 22 -STRING_END_ARCHIVE = b'PK\x05\x06' +END_CENTRAL_DIR_SIZE_64 = 56 +END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 +STRING_END_ARCHIVE = b'PK\x05\x06' # standard EOCD signature +STRING_END_LOCATOR_64 = b'PK\x06\x07' # Zip64 EOCD Locator signature +STRING_END_ZIP_64 = b'PK\x06\x06' # Zip64 EOCD signature MAX_COMMENT_LEN = (1 << 16) - 1 +MAX_UINT32 = 0xffffffff +ZIP64_EXTRA_TAG = 0x1 class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object @@ -151,6 +155,8 @@ def get_data(self, pathname): toc_entry = self._get_files()[key] except KeyError: raise OSError(0, '', key) + if toc_entry is None: + return b'' return _get_data(self.archive, toc_entry) @@ -352,53 +358,76 @@ def _read_directory(archive): with fp: # GH-87235: On macOS all file descriptors for /dev/fd/N share the same - # file offset, reset the file offset after scanning the zipfile diretory + # file offset, reset the file offset after scanning the zipfile directory # to not cause problems when some runs 'python3 /dev/fd/9 9= 0 and pos64+END_CENTRAL_DIR_SIZE_64+END_CENTRAL_DIR_LOCATOR_SIZE_64==pos): + # Zip64 at "correct" offset from standard EOCD + buffer = data[pos64:pos64 + END_CENTRAL_DIR_SIZE_64] + if len(buffer) != END_CENTRAL_DIR_SIZE_64: + raise ZipImportError( + f"corrupt Zip64 file: Expected {END_CENTRAL_DIR_SIZE_64} byte " + f"zip64 central directory, but read {len(buffer)} bytes.", + path=archive) + header_position = file_size - len(data) + pos64 + + central_directory_size = _unpack_uint64(buffer[40:48]) + central_directory_position = _unpack_uint64(buffer[48:56]) + num_entries = _unpack_uint64(buffer[24:32]) + elif pos >= 0: buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] if len(buffer) != END_CENTRAL_DIR_SIZE: raise ZipImportError(f"corrupt Zip file: {archive!r}", path=archive) + header_position = file_size - len(data) + pos - header_size = _unpack_uint32(buffer[12:16]) - header_offset = _unpack_uint32(buffer[16:20]) - if header_position < header_size: + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + central_directory_size = _unpack_uint32(buffer[12:16]) + central_directory_position = _unpack_uint32(buffer[16:20]) + num_entries = _unpack_uint16(buffer[8:10]) + + # N.b. if someday you want to prefer the standard (non-zip64) EOCD, + # you need to adjust position by 76 for arc to be 0. + else: + raise ZipImportError(f'not a Zip file: {archive!r}', + path=archive) + + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + # XXX: These are cursory checks but are not as exact or strict as they + # could be. Checking the arc-adjusted value is probably good too. + if header_position < central_directory_size: raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) - if header_position < header_offset: + if header_position < central_directory_position: raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) - header_position -= header_size - arc_offset = header_position - header_offset + header_position -= central_directory_size + # On just-a-zipfile these values are the same and arc_offset is zero; if + # the file has some bytes prepended, `arc_offset` is the number of such + # bytes. This is used for pex as well as self-extracting .exe. + arc_offset = header_position - central_directory_position if arc_offset < 0: raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) @@ -415,6 +444,11 @@ def _read_directory(archive): raise EOFError('EOF read where not expected') # Start of file header if buffer[:4] != b'PK\x01\x02': + if count != num_entries: + raise ZipImportError( + f"mismatched num_entries: {count} should be {num_entries} in {archive!r}", + path=archive, + ) break # Bad: Central Dir File Header if len(buffer) != 46: raise EOFError('EOF read where not expected') @@ -430,9 +464,6 @@ def _read_directory(archive): comment_size = _unpack_uint16(buffer[32:34]) file_offset = _unpack_uint32(buffer[42:46]) header_size = name_size + extra_size + comment_size - if file_offset > header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset try: name = fp.read(name_size) @@ -444,7 +475,10 @@ def _read_directory(archive): # slower than reading the data because fseek flushes stdio's # internal buffers. See issue #8745. try: - if len(fp.read(header_size - name_size)) != header_size - name_size: + extra_data_len = header_size - name_size + extra_data = memoryview(fp.read(extra_data_len)) + + if len(extra_data) != extra_data_len: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) @@ -461,12 +495,83 @@ def _read_directory(archive): name = name.replace('/', path_sep) path = _bootstrap_external._path_join(archive, name) + + # Ordering matches unpacking below. + if ( + file_size == MAX_UINT32 or + data_size == MAX_UINT32 or + file_offset == MAX_UINT32 + ): + # need to decode extra_data looking for a zip64 extra (which might not + # be present) + while extra_data: + if len(extra_data) < 4: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + tag = _unpack_uint16(extra_data[:2]) + size = _unpack_uint16(extra_data[2:4]) + if len(extra_data) < 4 + size: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + if tag == ZIP64_EXTRA_TAG: + if (len(extra_data) - 4) % 8 != 0: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + num_extra_values = (len(extra_data) - 4) // 8 + if num_extra_values > 3: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) + + # N.b. Here be dragons: the ordering of these is different than + # the header fields, and it's really easy to get it wrong since + # naturally-occuring zips that use all 3 are >4GB + if file_size == MAX_UINT32: + file_size = values.pop(0) + if data_size == MAX_UINT32: + data_size = values.pop(0) + if file_offset == MAX_UINT32: + file_offset = values.pop(0) + + break + + # For a typical zip, this bytes-slicing only happens 2-3 times, on + # small data like timestamps and filesizes. + extra_data = extra_data[4+size:] + else: + _bootstrap._verbose_message( + "zipimport: suspected zip64 but no zip64 extra for {!r}", + path, + ) + # XXX These two statements seem swapped because `central_directory_position` + # is a position within the actual file, but `file_offset` (when compared) is + # as encoded in the entry, not adjusted for this file. + # N.b. this must be after we've potentially read the zip64 extra which can + # change `file_offset`. + if file_offset > central_directory_position: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset + t = (path, compress, data_size, file_size, file_offset, time, date, crc) files[name] = t count += 1 finally: fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) + + # Add implicit directories. + count = 0 + for name in list(files): + while True: + i = name.rstrip(path_sep).rfind(path_sep) + if i < 0: + break + name = name[:i + 1] + if name in files: + break + files[name] = None + count += 1 + if count: + _bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}', + count, archive) return files # During bootstrap, we may need to load the encodings diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 4985dce2dc36d0..5db17bea045d8c 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -2,7 +2,7 @@ import sysconfig -def reset_tzpath(to=None): +def _reset_tzpath(to=None, stacklevel=4): global TZPATH tzpaths = to @@ -18,17 +18,22 @@ def reset_tzpath(to=None): base_tzpath = tzpaths else: env_var = os.environ.get("PYTHONTZPATH", None) - if env_var is not None: - base_tzpath = _parse_python_tzpath(env_var) - else: - base_tzpath = _parse_python_tzpath( - sysconfig.get_config_var("TZPATH") - ) + if env_var is None: + env_var = sysconfig.get_config_var("TZPATH") + base_tzpath = _parse_python_tzpath(env_var, stacklevel) TZPATH = tuple(base_tzpath) -def _parse_python_tzpath(env_var): +def reset_tzpath(to=None): + """Reset global TZPATH.""" + # We need `_reset_tzpath` helper function because it produces a warning, + # it is used as both a module-level call and a public API. + # This is how we equalize the stacklevel for both calls. + _reset_tzpath(to) + + +def _parse_python_tzpath(env_var, stacklevel): if not env_var: return () @@ -45,6 +50,7 @@ def _parse_python_tzpath(env_var): "Invalid paths specified in PYTHONTZPATH environment variable. " + msg, InvalidTZPathWarning, + stacklevel=stacklevel, ) return new_tzpath @@ -172,4 +178,4 @@ class InvalidTZPathWarning(RuntimeWarning): TZPATH = () -reset_tzpath() +_reset_tzpath(stacklevel=5) diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index eede15b8271058..b77dc0ed391bb3 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -517,8 +517,8 @@ class _DayOffset: __slots__ = ["d", "julian", "hour", "minute", "second"] def __init__(self, d, julian, hour=2, minute=0, second=0): - if not (0 + julian) <= d <= 365: - min_day = 0 + julian + min_day = 0 + julian # convert bool to int + if not min_day <= d <= 365: raise ValueError(f"d must be in [{min_day}, 365], not: {d}") self.d = d @@ -560,11 +560,11 @@ class _CalendarOffset: ) def __init__(self, m, w, d, hour=2, minute=0, second=0): - if not 0 < m <= 12: - raise ValueError("m must be in (0, 12]") + if not 1 <= m <= 12: + raise ValueError("m must be in [1, 12]") - if not 0 < w <= 5: - raise ValueError("w must be in (0, 5]") + if not 1 <= w <= 5: + raise ValueError("w must be in [1, 5]") if not 0 <= d <= 6: raise ValueError("d must be in [0, 6]") @@ -634,18 +634,21 @@ def _parse_tz_str(tz_str): offset_str, *start_end_str = tz_str.split(",", 1) - # fmt: off parser_re = re.compile( - r"(?P[^<0-9:.+-]+|<[a-zA-Z0-9+\-]+>)" + - r"((?P[+-]?\d{1,2}(:\d{2}(:\d{2})?)?)" + - r"((?P[^0-9:.+-]+|<[a-zA-Z0-9+\-]+>)" + - r"((?P[+-]?\d{1,2}(:\d{2}(:\d{2})?)?))?" + - r")?" + # dst - r")?$" # stdoff + r""" + (?P[^<0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?: + (?P[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?) + (?: + (?P[^0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?P[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?)? + )? # dst + )? # stdoff + """, + re.ASCII|re.VERBOSE ) - # fmt: on - m = parser_re.match(offset_str) + m = parser_re.fullmatch(offset_str) if m is None: raise ValueError(f"{tz_str} is not a valid TZ string") @@ -696,16 +699,17 @@ def _parse_tz_str(tz_str): def _parse_dst_start_end(dststr): - date, *time = dststr.split("/") - if date[0] == "M": + date, *time = dststr.split("/", 1) + type = date[:1] + if type == "M": n_is_julian = False - m = re.match(r"M(\d{1,2})\.(\d).(\d)$", date) + m = re.fullmatch(r"M(\d{1,2})\.(\d).(\d)", date, re.ASCII) if m is None: raise ValueError(f"Invalid dst start/end date: {dststr}") date_offset = tuple(map(int, m.groups())) offset = _CalendarOffset(*date_offset) else: - if date[0] == "J": + if type == "J": n_is_julian = True date = date[1:] else: @@ -715,38 +719,54 @@ def _parse_dst_start_end(dststr): offset = _DayOffset(doy, n_is_julian) if time: - time_components = list(map(int, time[0].split(":"))) - n_components = len(time_components) - if n_components < 3: - time_components.extend([0] * (3 - n_components)) - offset.hour, offset.minute, offset.second = time_components + offset.hour, offset.minute, offset.second = _parse_transition_time(time[0]) return offset +def _parse_transition_time(time_str): + match = re.fullmatch( + r"(?P[+-])?(?P\d{1,3})(:(?P\d{2})(:(?P\d{2}))?)?", + time_str, + re.ASCII + ) + if match is None: + raise ValueError(f"Invalid time: {time_str}") + + h, m, s = (int(v or 0) for v in match.group("h", "m", "s")) + + if h > 167: + raise ValueError( + f"Hour must be in [0, 167]: {time_str}" + ) + + if match.group("sign") == "-": + h, m, s = -h, -m, -s + + return h, m, s + + def _parse_tz_delta(tz_delta): - match = re.match( - r"(?P[+-])?(?P\d{1,2})(:(?P\d{2})(:(?P\d{2}))?)?", + match = re.fullmatch( + r"(?P[+-])?(?P\d{1,3})(:(?P\d{2})(:(?P\d{2}))?)?", tz_delta, + re.ASCII ) # Anything passed to this function should already have hit an equivalent # regular expression to find the section to parse. assert match is not None, tz_delta - h, m, s = ( - int(v) if v is not None else 0 - for v in map(match.group, ("h", "m", "s")) - ) + h, m, s = (int(v or 0) for v in match.group("h", "m", "s")) total = h * 3600 + m * 60 + s - if not -86400 < total < 86400: + if h > 24: raise ValueError( - f"Offset must be strictly between -24h and +24h: {tz_delta}" + f"Offset hours must be in [0, 24]: {tz_delta}" ) # Yes, +5 maps to an offset of -5h if match.group("sign") != "-": - total *= -1 + total = -total return total diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 4f74e7dc00520a..e44e74f3a49234 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -17,7 +17,7 @@ Gatekeeper download quarantine, the final package must be signed with a valid Apple Developer ID certificate using productsign. Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated -notarization service using the altool command. To pass notarization, +notarization service using the ``notarytool`` command. To pass notarization, the binaries included in the package must be built with at least the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with @@ -27,7 +27,7 @@ available in the on-line Apple Developer Documentation and man pages. A goal of PSF-provided (python.org) Python binaries for macOS is to support a wide-range of operating system releases with one set of binaries. Currently, the oldest release supported by python.org -binaries is macOS 10.9; it is still possible to build Python and +binaries is macOS 10.9; it should still be possible to build Python and Python installers on older versions of macOS but we not regularly test on those systems nor provide binaries for them. @@ -49,20 +49,17 @@ Starting with 3.9.1, Python fully supports macOS "weaklinking", meaning it is now possible to build a Python on a current macOS version with a deployment target of an earlier macOS system. For 3.9.1 and later systems, we provide a "macOS 64-bit universal2 installer" -variant, currently build on macOS 11 Big Sur with fat binaries +variant, currently built on macOS 11 Big Sur with fat binaries natively supporting both Apple Silicon (arm64) and Intel-64 (x86_64) Macs running macOS 10.9 or later. -The legacy "macOS 64-bit Intel installer" variant is expected to -be retired prior to the end of 3.9.x support. - build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later in the 10.6 support cycle.) build-installer.py also must be run -with recent versions of Python 3.x or 2.7. On older systems, +with recent versions of Python 3.x. On older systems, due to changes in TLS practices, it may be easier to manually download and cache third-party source distributions used by build-installer.py rather than have it attempt to automatically @@ -76,12 +73,12 @@ download them. - builds the following third-party libraries - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 + * OpenSSL 3.0.x + * Tcl/Tk 8.6.x * NCurses * SQLite * XZ - * libffi + * mpdecimal - uses system-supplied versions of third-party libraries @@ -97,35 +94,6 @@ download them. * ``MACOSX_DEPLOYMENT_TARGET=10.9`` * Apple ``clang`` -2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - - /path/to/bootstrap/python3 build-installer.py \ - --universal-archs=intel-64 \ - --dep-target=10.9 - - - builds the following third-party libraries - - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 - * NCurses - * SQLite - * XZ - * libffi - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - * zlib - * bz2 - - - recommended build environment: - - * Mac OS X 10.9.5 - * Xcode Command Line Tools 6.2 - * ``MacOSX10.9`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.9`` - * Apple ``clang`` - General Prerequisites --------------------- diff --git a/Mac/BuildScript/backport_gh110950_fix.patch b/Mac/BuildScript/backport_gh110950_fix.patch new file mode 100644 index 00000000000000..793f5a78d8c6ab --- /dev/null +++ b/Mac/BuildScript/backport_gh110950_fix.patch @@ -0,0 +1,25 @@ +From https://core.tcl-lang.org/tk/info/ed7cfbac8db11aa0 + +Note: the diff here is hand-tweaked so that it applies cleanly to both Tk 8.6.8 and 8.6.13. + +diff --git a/macosx/tkMacOSXInit.c b/macosx/tkMacOSXInit.c +index 71d7c3385..e6a68356c 100644 +--- a/macosx/tkMacOSXInit.c ++++ b/macosx/tkMacOSXInit.c +@@ -128,6 +128,16 @@ static int TkMacOSXGetAppPathCmd(ClientData cd, Tcl_Interp *ip, + observe(NSApplicationDidChangeScreenParametersNotification, displayChanged:); + observe(NSTextInputContextKeyboardSelectionDidChangeNotification, keyboardChanged:); + #undef observe ++} ++ ++ ++/* ++ * Fix for 10b38a7a7c. ++ */ ++ ++- (BOOL)applicationSupportsSecureRestorableState:(NSApplication *)app ++{ ++ return YES; + } + + -(void)applicationWillFinishLaunching:(NSNotification *)aNotification diff --git a/Mac/BuildScript/backport_gh71383_fix.patch b/Mac/BuildScript/backport_gh71383_fix.patch new file mode 100644 index 00000000000000..d77b1045e8c165 --- /dev/null +++ b/Mac/BuildScript/backport_gh71383_fix.patch @@ -0,0 +1,89 @@ +Adapted from https://core.tcl-lang.org/tk/info/b1876b9ebc4b + +Index: generic/tkInt.h +================================================================== +--- a/generic/tkInt.h.orig ++++ b/generic/tkInt.h +@@ -1094,10 +1094,11 @@ + /* + * Themed widget set init function: + */ + + MODULE_SCOPE int Ttk_Init(Tcl_Interp *interp); ++MODULE_SCOPE void Ttk_TkDestroyedHandler(Tcl_Interp *interp); + + /* + * Internal functions shared among Tk modules but not exported to the outside + * world: + */ + +Index: generic/tkWindow.c +================================================================== +--- a/generic/tkWindow.c.orig ++++ b/generic/tkWindow.c +@@ -1619,10 +1619,11 @@ + TkBindFree(winPtr->mainPtr); + TkDeleteAllImages(winPtr->mainPtr); + TkFontPkgFree(winPtr->mainPtr); + TkFocusFree(winPtr->mainPtr); + TkStylePkgFree(winPtr->mainPtr); ++ Ttk_TkDestroyedHandler(winPtr->mainPtr->interp); + + /* + * When embedding Tk into other applications, make sure that all + * destroy events reach the server. Otherwise the embedding + * application may also attempt to destroy the windows, resulting + +Index: generic/ttk/ttkTheme.c +================================================================== +--- a/generic/ttk/ttkTheme.c.orig ++++ b/generic/ttk/ttkTheme.c +@@ -415,17 +415,10 @@ + StylePackageData *pkgPtr = (StylePackageData *)clientData; + Tcl_HashSearch search; + Tcl_HashEntry *entryPtr; + Cleanup *cleanup; + +- /* +- * Cancel any pending ThemeChanged calls: +- */ +- if (pkgPtr->themeChangePending) { +- Tcl_CancelIdleCall(ThemeChangedProc, pkgPtr); +- } +- + /* + * Free themes. + */ + entryPtr = Tcl_FirstHashEntry(&pkgPtr->themeTable, &search); + while (entryPtr != NULL) { +@@ -528,10 +521,29 @@ + if (!pkgPtr->themeChangePending) { + Tcl_DoWhenIdle(ThemeChangedProc, pkgPtr); + pkgPtr->themeChangePending = 1; + } + } ++ ++/* Ttk_TkDestroyedHandler -- ++ * See bug [310c74ecf440]: idle calls to ThemeChangedProc() ++ * need to be canceled when Tk is destroyed, since the interp ++ * may still be active afterward; canceling them from ++ * Ttk_StylePkgFree() would be too late. ++ */ ++void Ttk_TkDestroyedHandler( ++ Tcl_Interp* interp) ++{ ++ StylePackageData* pkgPtr = GetStylePackageData(interp); ++ ++ /* ++ * Cancel any pending ThemeChanged calls: ++ */ ++ if (pkgPtr->themeChangePending) { ++ Tcl_CancelIdleCall(ThemeChangedProc, pkgPtr); ++ } ++} + + /* + * Ttk_CreateTheme -- + * Create a new theme and register it in the global theme table. + * + diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index a07d38aedcb5d6..8386e407f49aa3 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.11", - url="https://www.openssl.org/source/openssl-3.0.11.tar.gz", - checksum='b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55', + name="OpenSSL 3.0.13", + url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", + checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -261,14 +261,14 @@ def library_recipes(): tcl_checksum='81656d3367af032e0ae6157eff134f89' tk_checksum='5e0faecba458ee1386078fb228d008ba' - tk_patches = ['tk868_on_10_8_10_9.patch'] + tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.13' - tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' + tcl_tk_ver='8.6.14' + tcl_checksum='5880225babf7954c58d4fb0f5cf6279104ce1cd6aa9b71e9a6322540e1c4de66' - tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' - tk_patches = [ ] + tk_checksum='8ffdb720f47a6ca6107eac2dd877e30b0ef7fac14f3a84ebbd0b3612cee41a94' + tk_patches = [] base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz" @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.43.1", - url="https://sqlite.org/2023/sqlite-autoconf-3430100.tar.gz", - checksum="77e61befe9c3298da0504f87772a24b0", + name="SQLite 3.45.3", + url="https://sqlite.org/2024/sqlite-autoconf-3450300.tar.gz", + checksum="b2809ca53124c19c60f42bf627736eae011afdcc205bb48270a5ee9a38191531", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -378,6 +378,15 @@ def library_recipes(): '--disable-dependency-tracking', ] ), + dict( + name="libmpdec 4.0.0", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", + checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + configure_pre=[ + "--disable-cxx", + "MACHINE=universal", + ] + ), ]) if not PYTHON_3: @@ -1147,7 +1156,10 @@ def buildPython(): # will find them during its extension import sanity checks. print("Running configure...") + print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " + "--with-mimalloc=no " + "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " @@ -1490,7 +1502,7 @@ def packageFromRecipe(targetDir, recipe): IFPkgFlagRelocatable=False, IFPkgFlagRestartAction="NoRestart", IFPkgFlagRootVolumeOnly=True, - IFPkgFlagUpdateInstalledLangauges=False, + IFPkgFlagUpdateInstalledLanguages=False, ) writePlist(pl, os.path.join(packageContents, 'Info.plist')) diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index efd76b9b1ae64b..ee5ba4707dfea4 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -69,7 +69,7 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma \f1\b \ul Apple Silicon Mac support\ \f0\b0 \ulnone \ -On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the +On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the \f4 python3-intel64 \f0 command instead of just \f4 python3 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 79851e1f4a69cc..49d6e22286be26 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 68b8e4bb044e10..9caf62211ddd16 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -77,16 +77,17 @@ bash) fi ;; fish) - CONFIG_DIR="${HOME}/.config/fish" - RC="${CONFIG_DIR}/config.fish" + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" mkdir -p "$CONFIG_DIR" if [ -f "${RC}" ]; then cp -fp "${RC}" "${RC}.pysave" fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - echo "set -x PATH \"${PYTHON_ROOT}/bin\" \"\$PATH\"" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" if [ `id -ur` = 0 ]; then chown "${USER}" "${RC}" fi diff --git a/Mac/IDLE/IDLE.app/Contents/Info.plist b/Mac/IDLE/IDLE.app/Contents/Info.plist index fea06d46324999..8549e405e2a65a 100644 --- a/Mac/IDLE/IDLE.app/Contents/Info.plist +++ b/Mac/IDLE/IDLE.app/Contents/Info.plist @@ -37,7 +37,7 @@ CFBundleExecutable IDLE CFBundleGetInfoString - %version%, © 2001-2023 Python Software Foundation + %version%, © 2001-2024 Python Software Foundation CFBundleIconFile IDLE.icns CFBundleIdentifier @@ -56,5 +56,7 @@ %version% NSHighResolutionCapable + CFBundleAllowMixedLocalizations + diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 78b4499cca986a..32e3a0f55c5d71 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -257,6 +257,8 @@ install_IDLE: rm "$(DESTDIR)$(LIBDEST)/idlelib/config-extensions.def~" ; \ fi touch "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" + chmod -R ugo+rX,go-w "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" + chmod ugo+x "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/IDLE" $(INSTALLED_PYTHONAPP): install_Python diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index b7cddac0729fc2..233694788ac2b7 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -40,9 +40,9 @@ CFBundleExecutable Python Launcher NSHumanReadableCopyright - Copyright © 2001-2023 Python Software Foundation + Copyright © 2001-2024 Python Software Foundation CFBundleGetInfoString - %VERSION%, © 2001-2023 Python Software Foundation + %VERSION%, © 2001-2024 Python Software Foundation CFBundleIconFile PythonLauncher.icns CFBundleIdentifier diff --git a/Mac/PythonLauncher/Makefile.in b/Mac/PythonLauncher/Makefile.in index 4c05f26e8358bc..9decadbdc60f00 100644 --- a/Mac/PythonLauncher/Makefile.in +++ b/Mac/PythonLauncher/Makefile.in @@ -27,6 +27,8 @@ install: Python\ Launcher.app -test -d "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" && rm -r "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" /bin/cp -r "Python Launcher.app" "$(DESTDIR)$(PYTHONAPPSDIR)" touch "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" + chmod -R ugo+rX,go-w "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" + chmod ugo+x "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app/Contents/MacOS/Python Launcher" clean: rm -f *.o "Python Launcher" diff --git a/Mac/Resources/app-store-compliance.patch b/Mac/Resources/app-store-compliance.patch new file mode 100644 index 00000000000000..f4b7decc01cf1f --- /dev/null +++ b/Mac/Resources/app-store-compliance.patch @@ -0,0 +1,29 @@ +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index d6c83a75c1c..19ed4e01091 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -237,11 +237,6 @@ def test_roundtrips(self): + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), +- ('itms-services://?action=download-manifest&url=https://example.com/app', +- ('itms-services', '', '', '', +- 'action=download-manifest&url=https://example.com/app', ''), +- ('itms-services', '', '', +- 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), +diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py +index 8f724f907d4..148caf742c9 100644 +--- a/Lib/urllib/parse.py ++++ b/Lib/urllib/parse.py +@@ -59,7 +59,7 @@ + 'imap', 'wais', 'file', 'mms', 'https', 'shttp', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', + 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', +- 'ws', 'wss', 'itms-services'] ++ 'ws', 'wss'] + + uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index 4ec828ff176e7b..a1fc1511c40e96 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -20,7 +20,7 @@ CFBundleExecutable Python CFBundleGetInfoString - %version%, (c) 2001-2023 Python Software Foundation. + %version%, (c) 2001-2024 Python Software Foundation. CFBundleHelpBookFolder Documentation @@ -37,7 +37,7 @@ CFBundleInfoDictionaryVersion 6.0 CFBundleLongVersionString - %version%, (c) 2001-2023 Python Software Foundation. + %version%, (c) 2001-2024 Python Software Foundation. CFBundleName Python CFBundlePackageType @@ -55,8 +55,10 @@ NSAppleScriptEnabled NSHumanReadableCopyright - (c) 2001-2023 Python Software Foundation. + (c) 2001-2024 Python Software Foundation. NSHighResolutionCapable + CFBundleAllowMixedLocalizations + diff --git a/Mac/Resources/framework/Info.plist.in b/Mac/Resources/framework/Info.plist.in index e131c205ef0b28..4c42971ed90ee4 100644 --- a/Mac/Resources/framework/Info.plist.in +++ b/Mac/Resources/framework/Info.plist.in @@ -17,12 +17,14 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION%, (c) 2001-2023 Python Software Foundation. + %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleLongVersionString - %VERSION%, (c) 2001-2023 Python Software Foundation. + %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature ???? CFBundleVersion %VERSION% + CFBundleAllowMixedLocalizations + diff --git a/Makefile.pre.in b/Makefile.pre.in index 4996c5c3094bc5..d380c422714a32 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -40,8 +40,10 @@ LINKCC= @LINKCC@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ +ABIFLAGS= @ABIFLAGS@ +ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ -LIBPYTHON= @LIBPYTHON@ +MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -100,7 +102,7 @@ CONFIGURE_LDFLAGS= @LDFLAGS@ # command line to append to these values without stomping the pre-set # values. PY_CFLAGS= $(BASECFLAGS) $(OPT) $(CONFIGURE_CFLAGS) $(CFLAGS) $(EXTRA_CFLAGS) -PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include/internal +PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include/internal -I$(srcdir)/Include/internal/mimalloc # Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to # be able to build extension modules using the directories specified in the # environment variables @@ -150,7 +152,6 @@ INCLUDEDIR= @includedir@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) -ABIFLAGS= @ABIFLAGS@ # executable name for shebangs EXENAME= $(BINDIR)/python$(LDVERSION)$(EXE) # Variable used by ensurepip @@ -158,7 +159,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ -LIBDEST= $(SCRIPTDIR)/python$(VERSION) +LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) @@ -178,18 +179,29 @@ EXPORTSFROM= @EXPORTSFROM@ EXE= @EXEEXT@ BUILDEXE= @BUILDEXEEXT@ +# Name of the patch file to apply for app store compliance +APP_STORE_COMPLIANCE_PATCH=@APP_STORE_COMPLIANCE_PATCH@ + # Short name and location for Mac OS X Python framework UNIVERSALSDK=@UNIVERSALSDK@ PYTHONFRAMEWORK= @PYTHONFRAMEWORK@ PYTHONFRAMEWORKDIR= @PYTHONFRAMEWORKDIR@ PYTHONFRAMEWORKPREFIX= @PYTHONFRAMEWORKPREFIX@ PYTHONFRAMEWORKINSTALLDIR= @PYTHONFRAMEWORKINSTALLDIR@ -# Deployment target selected during configure, to be checked +PYTHONFRAMEWORKINSTALLNAMEPREFIX= @PYTHONFRAMEWORKINSTALLNAMEPREFIX@ +RESSRCDIR= @RESSRCDIR@ +# macOS deployment target selected during configure, to be checked # by distutils. The export statement is needed to ensure that the # deployment target is active during build. MACOSX_DEPLOYMENT_TARGET=@CONFIGURE_MACOSX_DEPLOYMENT_TARGET@ @EXPORT_MACOSX_DEPLOYMENT_TARGET@export MACOSX_DEPLOYMENT_TARGET +# iOS Deployment target selected during configure. Unlike macOS, the iOS +# deployment target is controlled using `-mios-version-min` arguments added to +# CFLAGS and LDFLAGS by the configure script. This variable is not used during +# the build, and is only listed here so it will be included in sysconfigdata. +IPHONEOS_DEPLOYMENT_TARGET=@IPHONEOS_DEPLOYMENT_TARGET@ + # Option to install to strip binaries STRIPFLAG=-s @@ -225,6 +237,9 @@ LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var TZPATH=@TZPATH@ +# If to install mimalloc headers +INSTALL_MIMALLOC=@INSTALL_MIMALLOC@ + # Modes for directories, executables and data files created by the # install process. Default to user-only-writable for all file types. DIRMODE= 755 @@ -336,6 +351,20 @@ IO_OBJS= \ Modules/_io/bytesio.o \ Modules/_io/stringio.o + +########################################################################## +# mimalloc + +MIMALLOC_HEADERS= \ + $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc/atomic.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc/internal.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc/prim.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc/track.h \ + $(srcdir)/Include/internal/mimalloc/mimalloc/types.h + + ########################################################################## # Parser @@ -391,17 +420,22 @@ PYTHON_OBJS= \ Python/ast_opt.o \ Python/ast_unparse.o \ Python/bltinmodule.o \ + Python/brc.o \ Python/ceval.o \ Python/codecs.o \ Python/compile.o \ Python/context.o \ + Python/critical_section.o \ + Python/crossinterp.o \ Python/dynamic_annotations.o \ Python/errors.o \ - Python/executor.o \ Python/flowgraph.o \ Python/frame.o \ Python/frozenmain.o \ Python/future.o \ + Python/gc.o \ + Python/gc_free_threading.o \ + Python/gc_gil.o \ Python/getargs.o \ Python/getcompiler.o \ Python/getcopyright.o \ @@ -413,16 +447,21 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ + Python/instruction_sequence.o \ Python/intrinsics.o \ + Python/jit.o \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ Python/mystrtoul.o \ + Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ @@ -435,6 +474,7 @@ PYTHON_OBJS= \ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ + Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ Python/structmember.o \ @@ -452,6 +492,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -482,7 +523,6 @@ OBJECT_OBJS= \ Objects/floatobject.o \ Objects/frameobject.o \ Objects/funcobject.o \ - Objects/interpreteridobject.o \ Objects/iterobject.o \ Objects/listobject.o \ Objects/longobject.o \ @@ -531,7 +571,7 @@ LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@ # On some systems, object files that reference DTrace probes need to be modified # in-place by dtrace(1). DTRACE_DEPS = \ - Python/ceval.o Python/import.o Python/sysmodule.o Modules/gcmodule.o + Python/ceval.o Python/gc.o Python/import.o Python/sysmodule.o ########################################################################## # decimal's libmpdec @@ -595,7 +635,9 @@ LIBEXPAT_HEADERS= \ Modules/expat/utf8tab.h \ Modules/expat/xmlrole.h \ Modules/expat/xmltok.h \ - Modules/expat/xmltok_impl.h + Modules/expat/xmltok_impl.h \ + Modules/expat/xmltok_impl.c \ + Modules/expat/xmltok_ns.c ########################################################################## # hashlib's HACL* library @@ -628,8 +670,32 @@ all: @DEF_MAKE_ALL_RULE@ # all. .PHONY: all +# Provide quick help for common Makefile targets. +.PHONY: help +help: + @echo "Run 'make' to build the Python executable and extension modules" + @echo "" + @echo "or 'make ' where is one of:" + @echo " test run the test suite" + @echo " install install built files" + @echo " regen-all regenerate a number of generated source files" + @echo " clinic run Argument Clinic over source files" + @echo "" + @echo " clean to remove build files" + @echo " distclean 'clean' + remove other generated files (patch, exe, etc)" + @echo "" + @echo " recheck rerun configure with last cmdline options" + @echo " reindent reindent .py files in Lib directory" + @echo " tags build a tags file (useful for Emacs and other editors)" + @echo " list-targets list all targets in the Makefile" + +# Display a full list of Makefile targets +.PHONY: list-targets +list-targets: + @grep -E '^[A-Za-z][-A-Za-z0-9]+:' Makefile | awk -F : '{print $$1}' + .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src @APP_STORE_COMPLIANCE_PATCH_TARGET@ $(BUILDPYTHON) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -642,11 +708,13 @@ check-clean-src: @if test -n "$(VPATH)" -a \( \ -f "$(srcdir)/$(BUILDPYTHON)" \ -o -f "$(srcdir)/Programs/python.o" \ - -o -f "$(srcdir)\Python/frozen_modules/importlib._bootstrap.h" \ + -o -f "$(srcdir)/Python/frozen_modules/importlib._bootstrap.h" \ \); then \ echo "Error: The source directory ($(srcdir)) is not clean" ; \ echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ - echo "Try to run: make -C \"$(srcdir)\" clean" ; \ + echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ + echo "Try to run:" ; \ + echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -832,18 +900,16 @@ $(LIBRARY): $(LIBRARY_OBJS) $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ - else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) @@ -865,17 +931,28 @@ SRC_GDB_HOOKS=$(srcdir)/Tools/gdb/libpython.py $(BUILDPYTHON)-gdb.py: $(SRC_GDB_HOOKS) $(INSTALL_DATA) $(SRC_GDB_HOOKS) $(BUILDPYTHON)-gdb.py +# Compliance with app stores (such as iOS and macOS) sometimes requires making +# modifications to the Python standard library. If enabled, apply the patch of +# known modifications to the source tree before building. The patch will be +# applied in a dry-run mode (validating, but not applying the patch) on builds +# that *have* a compliance patch, but where compliance has not been enabled. +build/app-store-compliant: + patch @APP_STORE_COMPLIANCE_PATCH_FLAGS@ --forward --strip=1 --directory="$(srcdir)" --input "$(APP_STORE_COMPLIANCE_PATCH)" + @if test "@APP_STORE_COMPLIANCE_PATCH_FLAGS@" == ""; then \ + mkdir -p build ; \ + echo "$(APP_STORE_COMPLIANCE_PATCH)" > build/app-store-compliant ; \ + fi + # This rule is here for OPENSTEP/Rhapsody/MacOSX. It builds a temporary # minimal framework (not including the Lib directory and such) in the current # directory. -RESSRCDIR=Mac/Resources/framework $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LIBRARY) \ $(RESSRCDIR)/Info.plist $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ - -all_load $(LIBRARY) -Wl,-single_module \ - -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \ + -all_load $(LIBRARY) \ + -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ -current_version $(VERSION) \ -framework CoreFoundation $(LIBS); @@ -887,6 +964,21 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources +# This rule is for iOS, which requires an annoyingly just slighly different +# format for frameworks to macOS. It *doesn't* use a versioned framework, and +# the Info.plist must be in the root of the framework. +$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK): \ + $(LIBRARY) \ + $(RESSRCDIR)/Info.plist + $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR) + $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ + -all_load $(LIBRARY) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -compatibility_version $(VERSION) \ + -current_version $(VERSION) \ + -framework CoreFoundation $(LIBS); + $(INSTALL_DATA) $(RESSRCDIR)/Info.plist $(PYTHONFRAMEWORKDIR)/Info.plist + # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) @@ -914,6 +1006,273 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js python.worker.js: $(srcdir)/Tools/wasm/python.worker.js @cp $(srcdir)/Tools/wasm/python.worker.js $@ +############################################################################ +# Header files + +PYTHON_HEADERS= \ + $(srcdir)/Include/Python.h \ + $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/bltinmodule.h \ + $(srcdir)/Include/boolobject.h \ + $(srcdir)/Include/bytearrayobject.h \ + $(srcdir)/Include/bytesobject.h \ + $(srcdir)/Include/ceval.h \ + $(srcdir)/Include/codecs.h \ + $(srcdir)/Include/compile.h \ + $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/critical_section.h \ + $(srcdir)/Include/descrobject.h \ + $(srcdir)/Include/dictobject.h \ + $(srcdir)/Include/dynamic_annotations.h \ + $(srcdir)/Include/enumobject.h \ + $(srcdir)/Include/errcode.h \ + $(srcdir)/Include/exports.h \ + $(srcdir)/Include/fileobject.h \ + $(srcdir)/Include/fileutils.h \ + $(srcdir)/Include/floatobject.h \ + $(srcdir)/Include/frameobject.h \ + $(srcdir)/Include/genericaliasobject.h \ + $(srcdir)/Include/import.h \ + $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iterobject.h \ + $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/lock.h \ + $(srcdir)/Include/longobject.h \ + $(srcdir)/Include/marshal.h \ + $(srcdir)/Include/memoryobject.h \ + $(srcdir)/Include/methodobject.h \ + $(srcdir)/Include/modsupport.h \ + $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/monitoring.h \ + $(srcdir)/Include/object.h \ + $(srcdir)/Include/objimpl.h \ + $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ + $(srcdir)/Include/osdefs.h \ + $(srcdir)/Include/osmodule.h \ + $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyatomic.h \ + $(srcdir)/Include/pybuffer.h \ + $(srcdir)/Include/pycapsule.h \ + $(srcdir)/Include/pydtrace.h \ + $(srcdir)/Include/pyerrors.h \ + $(srcdir)/Include/pyexpat.h \ + $(srcdir)/Include/pyframe.h \ + $(srcdir)/Include/pyhash.h \ + $(srcdir)/Include/pylifecycle.h \ + $(srcdir)/Include/pymacconfig.h \ + $(srcdir)/Include/pymacro.h \ + $(srcdir)/Include/pymath.h \ + $(srcdir)/Include/pymem.h \ + $(srcdir)/Include/pyport.h \ + $(srcdir)/Include/pystate.h \ + $(srcdir)/Include/pystats.h \ + $(srcdir)/Include/pystrcmp.h \ + $(srcdir)/Include/pystrtod.h \ + $(srcdir)/Include/pythonrun.h \ + $(srcdir)/Include/pythread.h \ + $(srcdir)/Include/pytypedefs.h \ + $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/refcount.h \ + $(srcdir)/Include/setobject.h \ + $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/structmember.h \ + $(srcdir)/Include/structseq.h \ + $(srcdir)/Include/sysmodule.h \ + $(srcdir)/Include/traceback.h \ + $(srcdir)/Include/tupleobject.h \ + $(srcdir)/Include/typeslots.h \ + $(srcdir)/Include/unicodeobject.h \ + $(srcdir)/Include/warnings.h \ + $(srcdir)/Include/weakrefobject.h \ + \ + pyconfig.h \ + $(PARSER_HEADERS) \ + \ + $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/bytearrayobject.h \ + $(srcdir)/Include/cpython/bytesobject.h \ + $(srcdir)/Include/cpython/cellobject.h \ + $(srcdir)/Include/cpython/ceval.h \ + $(srcdir)/Include/cpython/classobject.h \ + $(srcdir)/Include/cpython/code.h \ + $(srcdir)/Include/cpython/compile.h \ + $(srcdir)/Include/cpython/complexobject.h \ + $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/critical_section.h \ + $(srcdir)/Include/cpython/descrobject.h \ + $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/fileobject.h \ + $(srcdir)/Include/cpython/fileutils.h \ + $(srcdir)/Include/cpython/floatobject.h \ + $(srcdir)/Include/cpython/frameobject.h \ + $(srcdir)/Include/cpython/funcobject.h \ + $(srcdir)/Include/cpython/genobject.h \ + $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/initconfig.h \ + $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/lock.h \ + $(srcdir)/Include/cpython/longintrepr.h \ + $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/memoryobject.h \ + $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/modsupport.h \ + $(srcdir)/Include/cpython/monitoring.h \ + $(srcdir)/Include/cpython/object.h \ + $(srcdir)/Include/cpython/objimpl.h \ + $(srcdir)/Include/cpython/odictobject.h \ + $(srcdir)/Include/cpython/picklebufobject.h \ + $(srcdir)/Include/cpython/pthread_stubs.h \ + $(srcdir)/Include/cpython/pyatomic.h \ + $(srcdir)/Include/cpython/pyatomic_gcc.h \ + $(srcdir)/Include/cpython/pyatomic_std.h \ + $(srcdir)/Include/cpython/pyctype.h \ + $(srcdir)/Include/cpython/pydebug.h \ + $(srcdir)/Include/cpython/pyerrors.h \ + $(srcdir)/Include/cpython/pyfpe.h \ + $(srcdir)/Include/cpython/pyframe.h \ + $(srcdir)/Include/cpython/pyhash.h \ + $(srcdir)/Include/cpython/pylifecycle.h \ + $(srcdir)/Include/cpython/pymem.h \ + $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/pystats.h \ + $(srcdir)/Include/cpython/pythonrun.h \ + $(srcdir)/Include/cpython/pythread.h \ + $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sysmodule.h \ + $(srcdir)/Include/cpython/traceback.h \ + $(srcdir)/Include/cpython/tracemalloc.h \ + $(srcdir)/Include/cpython/tupleobject.h \ + $(srcdir)/Include/cpython/unicodeobject.h \ + $(srcdir)/Include/cpython/warnings.h \ + $(srcdir)/Include/cpython/weakrefobject.h \ + \ + $(MIMALLOC_HEADERS) \ + \ + $(srcdir)/Include/internal/pycore_abstract.h \ + $(srcdir)/Include/internal/pycore_asdl.h \ + $(srcdir)/Include/internal/pycore_ast.h \ + $(srcdir)/Include/internal/pycore_ast_state.h \ + $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_backoff.h \ + $(srcdir)/Include/internal/pycore_bitutils.h \ + $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ + $(srcdir)/Include/internal/pycore_brc.h \ + $(srcdir)/Include/internal/pycore_bytes_methods.h \ + $(srcdir)/Include/internal/pycore_bytesobject.h \ + $(srcdir)/Include/internal/pycore_call.h \ + $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_cell.h \ + $(srcdir)/Include/internal/pycore_ceval.h \ + $(srcdir)/Include/internal/pycore_ceval_state.h \ + $(srcdir)/Include/internal/pycore_code.h \ + $(srcdir)/Include/internal/pycore_codecs.h \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_complexobject.h \ + $(srcdir)/Include/internal/pycore_condvar.h \ + $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_critical_section.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_descrobject.h \ + $(srcdir)/Include/internal/pycore_dict.h \ + $(srcdir)/Include/internal/pycore_dict_state.h \ + $(srcdir)/Include/internal/pycore_dtoa.h \ + $(srcdir)/Include/internal/pycore_exceptions.h \ + $(srcdir)/Include/internal/pycore_faulthandler.h \ + $(srcdir)/Include/internal/pycore_fileutils.h \ + $(srcdir)/Include/internal/pycore_floatobject.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_format.h \ + $(srcdir)/Include/internal/pycore_frame.h \ + $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_function.h \ + $(srcdir)/Include/internal/pycore_gc.h \ + $(srcdir)/Include/internal/pycore_genobject.h \ + $(srcdir)/Include/internal/pycore_getopt.h \ + $(srcdir)/Include/internal/pycore_gil.h \ + $(srcdir)/Include/internal/pycore_global_objects.h \ + $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ + $(srcdir)/Include/internal/pycore_global_strings.h \ + $(srcdir)/Include/internal/pycore_hamt.h \ + $(srcdir)/Include/internal/pycore_hashtable.h \ + $(srcdir)/Include/internal/pycore_import.h \ + $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_initconfig.h \ + $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ + $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_jit.h \ + $(srcdir)/Include/internal/pycore_list.h \ + $(srcdir)/Include/internal/pycore_llist.h \ + $(srcdir)/Include/internal/pycore_lock.h \ + $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_memoryobject.h \ + $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/pycore_modsupport.h \ + $(srcdir)/Include/internal/pycore_moduleobject.h \ + $(srcdir)/Include/internal/pycore_namespace.h \ + $(srcdir)/Include/internal/pycore_object.h \ + $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_deferred.h \ + $(srcdir)/Include/internal/pycore_object_stack.h \ + $(srcdir)/Include/internal/pycore_object_state.h \ + $(srcdir)/Include/internal/pycore_obmalloc.h \ + $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_parking_lot.h \ + $(srcdir)/Include/internal/pycore_parser.h \ + $(srcdir)/Include/internal/pycore_pathconfig.h \ + $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pyatomic_ft_wrappers.h \ + $(srcdir)/Include/internal/pycore_pybuffer.h \ + $(srcdir)/Include/internal/pycore_pyerrors.h \ + $(srcdir)/Include/internal/pycore_pyhash.h \ + $(srcdir)/Include/internal/pycore_pylifecycle.h \ + $(srcdir)/Include/internal/pycore_pymath.h \ + $(srcdir)/Include/internal/pycore_pymem.h \ + $(srcdir)/Include/internal/pycore_pymem_init.h \ + $(srcdir)/Include/internal/pycore_pystate.h \ + $(srcdir)/Include/internal/pycore_pystats.h \ + $(srcdir)/Include/internal/pycore_pythonrun.h \ + $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_qsbr.h \ + $(srcdir)/Include/internal/pycore_range.h \ + $(srcdir)/Include/internal/pycore_runtime.h \ + $(srcdir)/Include/internal/pycore_runtime_init.h \ + $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ + $(srcdir)/Include/internal/pycore_semaphore.h \ + $(srcdir)/Include/internal/pycore_setobject.h \ + $(srcdir)/Include/internal/pycore_signal.h \ + $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_strhex.h \ + $(srcdir)/Include/internal/pycore_structseq.h \ + $(srcdir)/Include/internal/pycore_symtable.h \ + $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ + $(srcdir)/Include/internal/pycore_time.h \ + $(srcdir)/Include/internal/pycore_token.h \ + $(srcdir)/Include/internal/pycore_traceback.h \ + $(srcdir)/Include/internal/pycore_tracemalloc.h \ + $(srcdir)/Include/internal/pycore_tstate.h \ + $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typeobject.h \ + $(srcdir)/Include/internal/pycore_typevarobject.h \ + $(srcdir)/Include/internal/pycore_ucnhash.h \ + $(srcdir)/Include/internal/pycore_unicodeobject.h \ + $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ + $(srcdir)/Include/internal/pycore_unionobject.h \ + $(srcdir)/Include/internal/pycore_uop_ids.h \ + $(srcdir)/Include/internal/pycore_uop_metadata.h \ + $(srcdir)/Include/internal/pycore_warnings.h \ + $(srcdir)/Include/internal/pycore_weakref.h \ + $(DTRACE_HEADERS) \ + @PLATFORM_HEADERS@ \ + \ + $(srcdir)/Python/stdlib_module_names.h + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) @@ -1079,7 +1438,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# "Bootstrap Python" used to run deepfreeze.py +# "Bootstrap Python" used to run Programs/_freeze_module.py BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -1098,7 +1457,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Freezing is a multi step process. It works differently for standard builds # and cross builds. Standard builds use Programs/_freeze_module and -# _bootstrap_python for freezing and deepfreezing, so users can build Python +# _bootstrap_python for freezing, so users can build Python # without an existing Python installation. Cross builds cannot execute # compiled binaries and therefore rely on an external build Python # interpreter. The build interpreter must have same version and same bytecode @@ -1112,12 +1471,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 5) create remaining frozen module headers with # ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python # script is used to test the cross compile code path. -# 6) deepfreeze modules with _bootstrap_python # # Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. -# 2) deepfreeze modules with external build Python. # # FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. @@ -1263,41 +1620,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules @echo "The Makefile was updated, you may need to re-run make." -############################################################################ -# Deepfreeze targets - -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) - -# BEGIN: deepfreeze modules -$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ - Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ - Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ - Python/frozen_modules/zipimport.h:zipimport \ - Python/frozen_modules/abc.h:abc \ - Python/frozen_modules/codecs.h:codecs \ - Python/frozen_modules/io.h:io \ - Python/frozen_modules/_collections_abc.h:_collections_abc \ - Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ - Python/frozen_modules/genericpath.h:genericpath \ - Python/frozen_modules/ntpath.h:ntpath \ - Python/frozen_modules/posixpath.h:posixpath \ - Python/frozen_modules/os.h:os \ - Python/frozen_modules/site.h:site \ - Python/frozen_modules/stat.h:stat \ - Python/frozen_modules/importlib.util.h:importlib.util \ - Python/frozen_modules/importlib.machinery.h:importlib.machinery \ - Python/frozen_modules/runpy.h:runpy \ - Python/frozen_modules/__hello__.h:__hello__ \ - Python/frozen_modules/__phello__.h:__phello__ \ - Python/frozen_modules/__phello__.ham.h:__phello__.ham \ - Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ - Python/frozen_modules/__phello__.spam.h:__phello__.spam \ - Python/frozen_modules/frozen_only.h:frozen_only \ - -o Python/deepfreeze/deepfreeze.c -# END: deepfreeze modules - # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen @@ -1328,6 +1650,14 @@ check-abidump: all regen-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml +############################################################################ +# Regenerate Unicode Data + +.PHONY: regen-unicodedata +regen-unicodedata: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/unicode/makeunicodedata.py + + ############################################################################ # Regenerate all generated files @@ -1336,10 +1666,10 @@ regen-limited-abi: all regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects + regen-test-levenshtein regen-global-objects regen-jit @echo - @echo "Note: make regen-stdlib-module-names, make regen-limited-abi" - @echo "and make regen-configure should be run manually" + @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " + @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" ############################################################################ # Special rules for object files @@ -1370,7 +1700,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M Programs/python.o: $(srcdir)/Programs/python.c $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c -Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h +Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c Modules/_sre/sre.o: $(srcdir)/Modules/_sre/sre.c $(srcdir)/Modules/_sre/sre.h $(srcdir)/Modules/_sre/sre_constants.h $(srcdir)/Modules/_sre/sre_lib.h @@ -1383,8 +1713,20 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h -Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile - $(CC) -c $(PY_CORE_CFLAGS) \ +Modules/_interpretersmodule.o: $(srcdir)/Modules/_interpretersmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_interpqueuesmodule.o: $(srcdir)/Modules/_interpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h + +Modules/_interpchannelsmodule.o: $(srcdir)/Modules/_interpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h + +Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h + +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + +Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ -o $@ $(srcdir)/Python/dynload_shlib.c @@ -1490,7 +1832,7 @@ regen-sre: $(srcdir)/Modules/_sre/sre_constants.h \ $(srcdir)/Modules/_sre/sre_targets.h -Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h +Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -1522,6 +1864,7 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ + $(srcdir)/Objects/stringlib/repr.h \ $(srcdir)/Objects/stringlib/split.h \ $(srcdir)/Objects/stringlib/ucs1lib.h \ $(srcdir)/Objects/stringlib/ucs2lib.h \ @@ -1538,45 +1881,107 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS) Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h +Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \ + $(srcdir)/Objects/mimalloc/alloc-aligned.c \ + $(srcdir)/Objects/mimalloc/alloc-posix.c \ + $(srcdir)/Objects/mimalloc/arena.c \ + $(srcdir)/Objects/mimalloc/bitmap.c \ + $(srcdir)/Objects/mimalloc/heap.c \ + $(srcdir)/Objects/mimalloc/init.c \ + $(srcdir)/Objects/mimalloc/options.c \ + $(srcdir)/Objects/mimalloc/os.c \ + $(srcdir)/Objects/mimalloc/page.c \ + $(srcdir)/Objects/mimalloc/random.c \ + $(srcdir)/Objects/mimalloc/segment.c \ + $(srcdir)/Objects/mimalloc/segment-map.c \ + $(srcdir)/Objects/mimalloc/stats.c \ + $(srcdir)/Objects/mimalloc/prim/prim.c \ + $(srcdir)/Objects/mimalloc/prim/osx/prim.c \ + $(srcdir)/Objects/mimalloc/prim/unix/prim.c \ + $(srcdir)/Objects/mimalloc/prim/wasi/prim.c + +Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c + + +# Regenerate various files from Python/bytecodes.c +# Pass CASESFLAG=-l to insert #line directives in the output .PHONY: regen-cases -regen-cases: - # Regenerate various files from Python/bytecodes.c - # Pass CASESFLAG=-l to insert #line directives in the output - PYTHONPATH=$(srcdir)/Tools/cases_generator \ - $(PYTHON_FOR_REGEN) \ - $(srcdir)/Tools/cases_generator/generate_cases.py \ - $(CASESFLAG) \ - -o $(srcdir)/Python/generated_cases.c.h.new \ - -n $(srcdir)/Include/opcode_ids.h.new \ - -t $(srcdir)/Python/opcode_targets.h.new \ - -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ - -e $(srcdir)/Python/executor_cases.c.h.new \ - -p $(srcdir)/Lib/_opcode_metadata.py.new \ - -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ - $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new +regen-cases: \ + regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ + regen-generated-cases regen-executor-cases regen-optimizer-cases \ + regen-opcode-metadata regen-uop-metadata + +.PHONY: regen-opcode-ids +regen-opcode-ids: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_id_generator.py \ + -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + +.PHONY: regen-opcode-targets +regen-opcode-targets: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \ + -o $(srcdir)/Python/opcode_targets.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new - $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new + +.PHONY: regen-uop-ids +regen-uop-ids: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_id_generator.py \ + -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new + +.PHONY: regen-opcode-metadata-py +regen-opcode-metadata-py: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/py_metadata_generator.py \ + -o $(srcdir)/Lib/_opcode_metadata.py.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new -Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h +.PHONY: regen-generated-cases +regen-generated-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \ + -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new + +.PHONY: regen-executor-cases +regen-executor-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ + -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + +.PHONY: regen-optimizer-cases +regen-optimizer-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/optimizer_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ + $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new + +.PHONY: regen-opcode-metadata +regen-opcode-metadata: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ + -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new + +.PHONY: regen-uop-metadata +regen-uop-metadata: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_metadata_generator.py -o \ + $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new + +Python/compile.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/condvar.h \ $(srcdir)/Python/generated_cases.c.h \ + $(srcdir)/Python/executor_cases.c.h \ $(srcdir)/Python/opcode_targets.h -Python/executor.o: \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/ceval_macros.h \ - $(srcdir)/Python/executor_cases.c.h - Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1587,7 +1992,8 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_optimizer.h + $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Python/optimizer_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -1602,8 +2008,8 @@ Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d mv $@.tmp $@ Python/ceval.o: $(srcdir)/Include/pydtrace.h +Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h -Modules/gcmodule.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) @@ -1619,237 +2025,6 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -############################################################################ -# Header files - -PYTHON_HEADERS= \ - $(srcdir)/Include/Python.h \ - $(srcdir)/Include/abstract.h \ - $(srcdir)/Include/bltinmodule.h \ - $(srcdir)/Include/boolobject.h \ - $(srcdir)/Include/bytearrayobject.h \ - $(srcdir)/Include/bytesobject.h \ - $(srcdir)/Include/ceval.h \ - $(srcdir)/Include/codecs.h \ - $(srcdir)/Include/compile.h \ - $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/descrobject.h \ - $(srcdir)/Include/dictobject.h \ - $(srcdir)/Include/dynamic_annotations.h \ - $(srcdir)/Include/enumobject.h \ - $(srcdir)/Include/errcode.h \ - $(srcdir)/Include/fileobject.h \ - $(srcdir)/Include/fileutils.h \ - $(srcdir)/Include/floatobject.h \ - $(srcdir)/Include/frameobject.h \ - $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/intrcheck.h \ - $(srcdir)/Include/iterobject.h \ - $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/longobject.h \ - $(srcdir)/Include/marshal.h \ - $(srcdir)/Include/memoryobject.h \ - $(srcdir)/Include/methodobject.h \ - $(srcdir)/Include/modsupport.h \ - $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/object.h \ - $(srcdir)/Include/objimpl.h \ - $(srcdir)/Include/opcode.h \ - $(srcdir)/Include/opcode_ids.h \ - $(srcdir)/Include/osdefs.h \ - $(srcdir)/Include/osmodule.h \ - $(srcdir)/Include/patchlevel.h \ - $(srcdir)/Include/pybuffer.h \ - $(srcdir)/Include/pycapsule.h \ - $(srcdir)/Include/pydtrace.h \ - $(srcdir)/Include/pyerrors.h \ - $(srcdir)/Include/pyframe.h \ - $(srcdir)/Include/pyhash.h \ - $(srcdir)/Include/pylifecycle.h \ - $(srcdir)/Include/pymacconfig.h \ - $(srcdir)/Include/pymacro.h \ - $(srcdir)/Include/pymath.h \ - $(srcdir)/Include/pymem.h \ - $(srcdir)/Include/pyport.h \ - $(srcdir)/Include/pystate.h \ - $(srcdir)/Include/pystats.h \ - $(srcdir)/Include/pystrcmp.h \ - $(srcdir)/Include/pystrtod.h \ - $(srcdir)/Include/pythonrun.h \ - $(srcdir)/Include/pythread.h \ - $(srcdir)/Include/pytypedefs.h \ - $(srcdir)/Include/rangeobject.h \ - $(srcdir)/Include/setobject.h \ - $(srcdir)/Include/sliceobject.h \ - $(srcdir)/Include/structmember.h \ - $(srcdir)/Include/structseq.h \ - $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/traceback.h \ - $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/unicodeobject.h \ - $(srcdir)/Include/warnings.h \ - $(srcdir)/Include/weakrefobject.h \ - \ - pyconfig.h \ - $(PARSER_HEADERS) \ - \ - $(srcdir)/Include/cpython/abstract.h \ - $(srcdir)/Include/cpython/bytearrayobject.h \ - $(srcdir)/Include/cpython/bytesobject.h \ - $(srcdir)/Include/cpython/cellobject.h \ - $(srcdir)/Include/cpython/ceval.h \ - $(srcdir)/Include/cpython/classobject.h \ - $(srcdir)/Include/cpython/code.h \ - $(srcdir)/Include/cpython/compile.h \ - $(srcdir)/Include/cpython/complexobject.h \ - $(srcdir)/Include/cpython/context.h \ - $(srcdir)/Include/cpython/descrobject.h \ - $(srcdir)/Include/cpython/dictobject.h \ - $(srcdir)/Include/cpython/fileobject.h \ - $(srcdir)/Include/cpython/fileutils.h \ - $(srcdir)/Include/cpython/floatobject.h \ - $(srcdir)/Include/cpython/frameobject.h \ - $(srcdir)/Include/cpython/funcobject.h \ - $(srcdir)/Include/cpython/genobject.h \ - $(srcdir)/Include/cpython/import.h \ - $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/longintrepr.h \ - $(srcdir)/Include/cpython/longobject.h \ - $(srcdir)/Include/cpython/memoryobject.h \ - $(srcdir)/Include/cpython/methodobject.h \ - $(srcdir)/Include/cpython/modsupport.h \ - $(srcdir)/Include/cpython/object.h \ - $(srcdir)/Include/cpython/objimpl.h \ - $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ - $(srcdir)/Include/cpython/picklebufobject.h \ - $(srcdir)/Include/cpython/pthread_stubs.h \ - $(srcdir)/Include/cpython/pyatomic.h \ - $(srcdir)/Include/cpython/pyatomic_gcc.h \ - $(srcdir)/Include/cpython/pyatomic_std.h \ - $(srcdir)/Include/cpython/pyctype.h \ - $(srcdir)/Include/cpython/pydebug.h \ - $(srcdir)/Include/cpython/pyerrors.h \ - $(srcdir)/Include/cpython/pyfpe.h \ - $(srcdir)/Include/cpython/pyframe.h \ - $(srcdir)/Include/cpython/pylifecycle.h \ - $(srcdir)/Include/cpython/pymem.h \ - $(srcdir)/Include/cpython/pystate.h \ - $(srcdir)/Include/cpython/pystats.h \ - $(srcdir)/Include/cpython/pythonrun.h \ - $(srcdir)/Include/cpython/pythread.h \ - $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ - $(srcdir)/Include/cpython/traceback.h \ - $(srcdir)/Include/cpython/tracemalloc.h \ - $(srcdir)/Include/cpython/tupleobject.h \ - $(srcdir)/Include/cpython/unicodeobject.h \ - $(srcdir)/Include/cpython/warnings.h \ - $(srcdir)/Include/cpython/weakrefobject.h \ - \ - $(srcdir)/Include/internal/pycore_abstract.h \ - $(srcdir)/Include/internal/pycore_asdl.h \ - $(srcdir)/Include/internal/pycore_ast.h \ - $(srcdir)/Include/internal/pycore_ast_state.h \ - $(srcdir)/Include/internal/pycore_atexit.h \ - $(srcdir)/Include/internal/pycore_atomic.h \ - $(srcdir)/Include/internal/pycore_bitutils.h \ - $(srcdir)/Include/internal/pycore_bytes_methods.h \ - $(srcdir)/Include/internal/pycore_bytesobject.h \ - $(srcdir)/Include/internal/pycore_call.h \ - $(srcdir)/Include/internal/pycore_capsule.h \ - $(srcdir)/Include/internal/pycore_ceval.h \ - $(srcdir)/Include/internal/pycore_ceval_state.h \ - $(srcdir)/Include/internal/pycore_code.h \ - $(srcdir)/Include/internal/pycore_codecs.h \ - $(srcdir)/Include/internal/pycore_compile.h \ - $(srcdir)/Include/internal/pycore_complexobject.h \ - $(srcdir)/Include/internal/pycore_condvar.h \ - $(srcdir)/Include/internal/pycore_context.h \ - $(srcdir)/Include/internal/pycore_dict.h \ - $(srcdir)/Include/internal/pycore_dict_state.h \ - $(srcdir)/Include/internal/pycore_descrobject.h \ - $(srcdir)/Include/internal/pycore_dtoa.h \ - $(srcdir)/Include/internal/pycore_exceptions.h \ - $(srcdir)/Include/internal/pycore_faulthandler.h \ - $(srcdir)/Include/internal/pycore_fileutils.h \ - $(srcdir)/Include/internal/pycore_floatobject.h \ - $(srcdir)/Include/internal/pycore_format.h \ - $(srcdir)/Include/internal/pycore_frame.h \ - $(srcdir)/Include/internal/pycore_function.h \ - $(srcdir)/Include/internal/pycore_genobject.h \ - $(srcdir)/Include/internal/pycore_getopt.h \ - $(srcdir)/Include/internal/pycore_gil.h \ - $(srcdir)/Include/internal/pycore_global_objects.h \ - $(srcdir)/Include/internal/pycore_global_objects_fini_generated.h \ - $(srcdir)/Include/internal/pycore_hamt.h \ - $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ - $(srcdir)/Include/internal/pycore_import.h \ - $(srcdir)/Include/internal/pycore_initconfig.h \ - $(srcdir)/Include/internal/pycore_interp.h \ - $(srcdir)/Include/internal/pycore_intrinsics.h \ - $(srcdir)/Include/internal/pycore_list.h \ - $(srcdir)/Include/internal/pycore_llist.h \ - $(srcdir)/Include/internal/pycore_lock.h \ - $(srcdir)/Include/internal/pycore_long.h \ - $(srcdir)/Include/internal/pycore_modsupport.h \ - $(srcdir)/Include/internal/pycore_moduleobject.h \ - $(srcdir)/Include/internal/pycore_namespace.h \ - $(srcdir)/Include/internal/pycore_object.h \ - $(srcdir)/Include/internal/pycore_object_state.h \ - $(srcdir)/Include/internal/pycore_obmalloc.h \ - $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_opcode_utils.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Include/internal/pycore_parking_lot.h \ - $(srcdir)/Include/internal/pycore_pathconfig.h \ - $(srcdir)/Include/internal/pycore_pyarena.h \ - $(srcdir)/Include/internal/pycore_pybuffer.h \ - $(srcdir)/Include/internal/pycore_pyerrors.h \ - $(srcdir)/Include/internal/pycore_pyhash.h \ - $(srcdir)/Include/internal/pycore_pylifecycle.h \ - $(srcdir)/Include/internal/pycore_pymem.h \ - $(srcdir)/Include/internal/pycore_pymem_init.h \ - $(srcdir)/Include/internal/pycore_pystate.h \ - $(srcdir)/Include/internal/pycore_pystats.h \ - $(srcdir)/Include/internal/pycore_pythonrun.h \ - $(srcdir)/Include/internal/pycore_pythread.h \ - $(srcdir)/Include/internal/pycore_range.h \ - $(srcdir)/Include/internal/pycore_runtime.h \ - $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ - $(srcdir)/Include/internal/pycore_runtime_init.h \ - $(srcdir)/Include/internal/pycore_semaphore.h \ - $(srcdir)/Include/internal/pycore_setobject.h \ - $(srcdir)/Include/internal/pycore_signal.h \ - $(srcdir)/Include/internal/pycore_sliceobject.h \ - $(srcdir)/Include/internal/pycore_strhex.h \ - $(srcdir)/Include/internal/pycore_structseq.h \ - $(srcdir)/Include/internal/pycore_symtable.h \ - $(srcdir)/Include/internal/pycore_sysmodule.h \ - $(srcdir)/Include/internal/pycore_time.h \ - $(srcdir)/Include/internal/pycore_token.h \ - $(srcdir)/Include/internal/pycore_traceback.h \ - $(srcdir)/Include/internal/pycore_tracemalloc.h \ - $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeobject.h \ - $(srcdir)/Include/internal/pycore_typevarobject.h \ - $(srcdir)/Include/internal/pycore_ucnhash.h \ - $(srcdir)/Include/internal/pycore_unionobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject.h \ - $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ - $(srcdir)/Include/internal/pycore_uops.h \ - $(srcdir)/Include/internal/pycore_warnings.h \ - $(srcdir)/Include/internal/pycore_weakref.h \ - $(DTRACE_HEADERS) \ - @PLATFORM_HEADERS@ \ - \ - $(srcdir)/Python/stdlib_module_names.h - $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -1885,6 +2060,54 @@ testuniversal: all $(RUNSHARED) /usr/libexec/oah/translate \ ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) +# Run the test suite on the iOS simulator. Must be run on a macOS machine with +# a full Xcode install that has an iPhone SE (3rd edition) simulator available. +# This must be run *after* a `make install` has completed the build. The +# `--with-framework-name` argument *cannot* be used when configuring the build. +XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) +XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult +.PHONY: testios +testios: + @if test "$(MACHDEP)" != "ios"; then \ + echo "Cannot run the iOS testbed for a non-iOS build."; \ + exit 1;\ + fi + @if test "$(findstring -iphonesimulator,$(MULTIARCH))" != "-iphonesimulator"; then \ + echo "Cannot run the iOS testbed for non-simulator builds."; \ + exit 1;\ + fi + @if test $(PYTHONFRAMEWORK) != "Python"; then \ + echo "Cannot run the iOS testbed with a non-default framework name."; \ + exit 1;\ + fi + @if ! test -d $(PYTHONFRAMEWORKPREFIX); then \ + echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ + exit 1;\ + fi + # Copy the testbed project into the build folder + cp -r $(srcdir)/iOS/testbed $(XCFOLDER) + # Copy the framework from the install location to the testbed project. + cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator + + # Run the test suite for the Xcode project, targeting the iOS simulator. + # If the suite fails, touch a file in the test folder as a marker + if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ + touch $(XCFOLDER)/failed; \ + fi + + # Regardless of success or failure, extract and print the test output + xcrun xcresulttool get --path $(XCRESULT) \ + --id $$( \ + xcrun xcresulttool get --path $(XCRESULT) --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ + ) \ + --format json | \ + $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + + @if test -e $(XCFOLDER)/failed ; then \ + exit 1; \ + fi + # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include # information about the build environment. @@ -1924,7 +2147,7 @@ multissltest: all # which can lead to two parallel `./python setup.py build` processes that # step on each others toes. .PHONY: install -install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ +install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ upgrade) ensurepip="--upgrade" ;; \ @@ -2060,10 +2283,10 @@ bininstall: commoninstall altbininstall -if test "$(VERSION)" != "$(LDVERSION)"; then \ rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python-$(LDVERSION).pc); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION)-embed.pc python-$(LDVERSION)-embed.pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION).pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION).pc python-$(VERSION).pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION)-embed.pc python-$(VERSION)-embed.pc); \ fi -rm -f $(DESTDIR)$(BINDIR)/python3-config (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config) @@ -2123,6 +2346,7 @@ LIBSUBDIRS= asyncio \ json \ logging \ multiprocessing multiprocessing/dummy \ + pathlib \ pydoc_data \ re \ site-packages \ @@ -2139,13 +2363,16 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ + test/archivetestdata \ test/audiodata \ test/certdata \ test/certdata/capath \ test/cjkencodings \ + test/configdata \ test/crashers \ test/data \ test/decimaltestdata \ @@ -2154,19 +2381,26 @@ TESTSUBDIRS= idlelib/idle_test \ test/leakers \ test/libregrtest \ test/mathdata \ + test/regrtestdata \ + test/regrtestdata/import_from_tests \ + test/regrtestdata/import_from_tests/test_regrtest_b \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ + test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_cext \ + test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ test/test_dataclasses \ + test/test_doctest \ test/test_email \ test/test_email/data \ + test/test_free_threading \ test/test_future_stmt \ test/test_gdb \ - test/test_inspect \ test/test_import \ test/test_import/data \ test/test_import/data/circular_imports \ @@ -2175,13 +2409,21 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ - test/test_importlib/data \ test/test_importlib/extension \ test/test_importlib/frozen \ test/test_importlib/import_ \ + test/test_importlib/metadata \ + test/test_importlib/metadata/data \ + test/test_importlib/metadata/data/sources \ + test/test_importlib/metadata/data/sources/example \ + test/test_importlib/metadata/data/sources/example/example \ + test/test_importlib/metadata/data/sources/example2 \ + test/test_importlib/metadata/data/sources/example2/example2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ @@ -2216,12 +2458,21 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/data03/namespace/portion1 \ test/test_importlib/resources/data03/namespace/portion2 \ test/test_importlib/resources/namespacedata01 \ + test/test_importlib/resources/namespacedata01/subdirectory \ test/test_importlib/resources/zipdata01 \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ + test/test_inspect \ + test/test_interpreters \ test/test_json \ test/test_module \ + test/test_multiprocessing_fork \ + test/test_multiprocessing_forkserver \ + test/test_multiprocessing_spawn \ + test/test_pathlib \ test/test_peg_generator \ + test/test_pydoc \ + test/test_pyrepl \ test/test_sqlite3 \ test/test_tkinter \ test/test_tomllib \ @@ -2256,9 +2507,10 @@ TESTSUBDIRS= idlelib/idle_test \ test/tokenizedata \ test/tracedmodules \ test/typinganndata \ + test/wheeldata \ test/xmltestdata \ test/xmltestdata/c14n-20 \ - test/ziptestdata + test/zipimport_data COMPILEALL_OPTS=-j0 @@ -2401,6 +2653,12 @@ inclinstall: $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal; \ else true; \ fi + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + if test ! -d $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; then \ + echo "Creating directory $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc"; \ + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + fi; \ + fi @for i in $(srcdir)/Include/*.h; \ do \ echo $(INSTALL_DATA) $$i $(INCLUDEPY); \ @@ -2416,6 +2674,16 @@ inclinstall: echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal; \ $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \ done + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + echo $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \ + do \ + echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc/mimalloc; \ + $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + done; \ + fi + echo $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h # Install the library and miscellaneous stuff needed for extending/embedding @@ -2456,8 +2724,8 @@ libainstall: all scripts $(INSTALL_DATA) Modules/Setup.bootstrap $(DESTDIR)$(LIBPL)/Setup.bootstrap $(INSTALL_DATA) Modules/Setup.stdlib $(DESTDIR)$(LIBPL)/Setup.stdlib $(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local - $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc - $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc + $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc + $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc $(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py @@ -2498,10 +2766,11 @@ frameworkinstall: install # only have to cater for the structural bits of the framework. .PHONY: frameworkinstallframework -frameworkinstallframework: frameworkinstallstructure install frameworkinstallmaclib +frameworkinstallframework: @FRAMEWORKINSTALLFIRST@ install frameworkinstallmaclib -.PHONY: frameworkinstallstructure -frameworkinstallstructure: $(LDLIBRARY) +# macOS uses a versioned frameworks structure that includes a full install +.PHONY: frameworkinstallversionedstructure +frameworkinstallversionedstructure: $(LDLIBRARY) @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ echo Not configured with --enable-framework; \ exit 1; \ @@ -2522,6 +2791,27 @@ frameworkinstallstructure: $(LDLIBRARY) $(LN) -fsn Versions/Current/Resources $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Resources $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) +# iOS/tvOS/watchOS uses a non-versioned framework with Info.plist in the +# framework root, no .lproj data, and only stub compilation assistance binaries +.PHONY: frameworkinstallunversionedstructure +frameworkinstallunversionedstructure: $(LDLIBRARY) + @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ + echo Not configured with --enable-framework; \ + exit 1; \ + else true; \ + fi + if test -d $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; then \ + echo "Clearing stale header symlink directory"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; \ + fi + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) + sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist + $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) + for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ + $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ + done + # This installs Mac/Lib into the framework # Install a number of symlinks to keep software that expects a normal unix # install (which includes python-config) happy. @@ -2562,6 +2852,19 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# On iOS, bin/lib can't live inside the framework; include needs to be called +# "Headers", but *must* be in the framework, and *not* include the `python3.X` +# subdirectory. The install has put these folders in the same folder as +# Python.framework; Move the headers to their final framework-compatible home. +.PHONY: frameworkinstallmobileheaders +frameworkinstallmobileheaders: frameworkinstallunversionedstructure inclinstall + if test -d $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; then \ + echo "Removing old framework headers"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ + fi + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2576,6 +2879,23 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +JIT_DEPS = \ + $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.py \ + $(srcdir)/Python/executor_cases.c.h \ + pyconfig.h + +jit_stencils.h: $(JIT_DEPS) + @REGEN_JIT_COMMAND@ + +Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +.PHONY: regen-jit +regen-jit: + @REGEN_JIT_COMMAND@ + # Some make's put the object file in the current directory .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< @@ -2604,15 +2924,13 @@ recheck: autoconf: (cd $(srcdir); autoreconf -ivf -Werror) -# See https://github.com/tiran/cpython_autoconf container .PHONY: regen-configure regen-configure: - @if command -v podman >/dev/null; then RUNTIME="podman"; else RUNTIME="docker"; fi; \ - if ! command -v $$RUNTIME; then echo "$@ needs either Podman or Docker container runtime." >&2; exit 1; fi; \ - if command -v selinuxenabled >/dev/null && selinuxenabled; then OPT=":Z"; fi; \ - CMD="$$RUNTIME run --rm --pull=always -v $(abs_srcdir):/src$$OPT quay.io/tiran/cpython_autoconf:271"; \ - echo $$CMD; \ - $$CMD || exit $? + $(srcdir)/Tools/build/regen-configure.sh + +.PHONY: regen-sbom +regen-sbom: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_sbom.py # Create a tags file for vi tags:: @@ -2667,12 +2985,17 @@ clean-retain-profile: pycremoval -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) -rm -f Programs/_testembed Programs/_freeze_module - -rm -f Python/deepfreeze/*.[co] + -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST + -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp + -rm -rf iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf iOS/testbed/Python.xcframework/ios-*/include + -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -2698,7 +3021,10 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) + -rm -rf iOS/Frameworks + -rm -rf iOSTestbed.* -rm -f python-config.py python-config + -rm -rf cross-build # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] @@ -2791,14 +3117,18 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c -MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ +MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h +MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h +MODULE__DATETIME_DEPS=$(srcdir)/Include/datetime.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h -MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h +MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h $(srcdir)/Modules/_complex.h +MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ @@ -2809,8 +3139,9 @@ MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/H MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c -MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h -MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h +MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h diff --git a/Misc/ACKS b/Misc/ACKS index 94cb1965676f48..53258dbfd9f478 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -31,6 +31,7 @@ Farhan Ahmad Matthew Ahrens Nir Aides Akira +Ege Akman Yaniv Aknin Jyrki Alakuijala Tatiana Al-Chueyr @@ -190,6 +191,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Wannes Boeykens Andra Bogildea Matt Bogosian Nikolay Bogoychev @@ -313,6 +315,7 @@ Greg Chapman Mitch Chapman Matt Chaput William Chargin +Ben Chatterton Yogesh Chaudhari Gautam Chaudhuri David Chaum @@ -469,6 +472,7 @@ Allen Downey Cesar Douady Dean Draayer Fred L. Drake, Jr. +Mehdi Drissi Derk Drukker John DuBois Paul Dubois @@ -493,6 +497,7 @@ David Edelsohn John Edmonds Benjamin Edwards Grant Edwards +Vlad Efanov Zvi Effron John Ehresman Tal Einat @@ -605,6 +610,7 @@ Nitin Ganatra Soumendra Ganguly (गङà¥à¤—ोपाधà¥à¤¯à¤¾à¤¯) Fred Gansevles Paul Ganssle +Tian Gao Lars Marius Garshol Jake Garver Dan Gass @@ -630,6 +636,7 @@ Dinu Gherman Subhendu Ghosh Jonathan Giddy Johannes Gijsbers +Stephen Gildea Michael Gilfix Julian Gindi Yannick Gingras @@ -637,6 +644,7 @@ Neil Girdhar Matt Giuca Andrea Giudiceandrea Franz Glasner +Jeff Glass Wim Glenn Michael Goderbauer Karan Goel @@ -664,6 +672,7 @@ Eddy De Greef Duane Griffin Grant Griffin Andrea Griffini +Semyon Grigoryev Duncan Grisby Olivier Grisel Fabian Groffen @@ -744,6 +753,7 @@ Kasun Herath Chris Herborth Ivan Herman Jürgen Hermann +Joshua Jay Herman Gary Herron Ernie Hershey Thomas Herve @@ -753,6 +763,7 @@ Raymond Hettinger Lisa Hewus Fresh Kevan Heydon Wouter van Heyst +Derek Higgins Kelsey Hightower Jason Hildebrand Ryan Hileman @@ -787,6 +798,7 @@ Thomas Holmes Craig Holmquist Philip Homburg Naofumi Honda +Constantin Hong Weipeng Hong Jeffrey Honig Rob Hooft @@ -921,6 +933,7 @@ Hiroaki Kawai Dmitry Kazakov Brian Kearns Sebastien Keim +Russell Keith-Magee Ryan Kelly Hugo van Kemenade Dan Kenigsberg @@ -960,6 +973,7 @@ Carsten Klein Bastian Kleineidam Joel Klimont Bob Kline +Alois Klink Matthias Klose Jeremy Kloth Thomas Kluyver @@ -1046,6 +1060,7 @@ Mark Lawrence Chris Laws Michael Layzell Michael Lazar +Peter Lazorchak Brian Leair Mathieu Leduc-Hamel Amandine Lee @@ -1084,6 +1099,7 @@ Ivan Levkivskyi Ben Lewis William Lewis Akira Li +Jiahao Li Robert Li Xuanji Li Zekun Li @@ -1302,6 +1318,7 @@ Hrvoje NikÅ¡ić Gregory Nofi Jesse Noller Bill Noon +Janek Nouvertné Stefan Norberg Tim Northover Joe Norton @@ -1345,6 +1362,7 @@ Michele Orrù Tomáš Orsava Oleg Oshmyan Denis Osipov +Savannah Ostrowski Denis S. Otkidach Peter Otten Michael Otteneder @@ -1457,6 +1475,7 @@ Paul Prescod Donovan Preston Eric Price Paul Price +Matt Prodani Iuliia Proskurnia Dorian Pula Jyrki Pulliainen @@ -1480,6 +1499,7 @@ Ajith Ramachandran Dhushyanth Ramasamy Ashwin Ramaswami Jeff Ramnani +Grant Ramsay Bayard Randel Varpu Rantala Brodie Rao @@ -1568,6 +1588,7 @@ Liam Routt Todd Rovito Craig Rowland Clinton Roy +Ujan RoyBandyopadhyay Paul Rubin Sam Ruby Demur Rumed @@ -2038,6 +2059,7 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Yanbo, Xie Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/HISTORY b/Misc/HISTORY index e66b695f21c977..a74d7e06acd071 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -607,7 +607,7 @@ Library MemoryError. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no + ambiguous reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. - Issue #23745: The new email header parser now handles duplicate MIME @@ -2030,7 +2030,7 @@ Library initialization of the unquote_to_bytes() table of the urllib.parse module, to not waste memory if these modules are not used. -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 +- Issue #19157: Include the broadcast address in the usable hosts for IPv6 in ipaddress. - Issue #11599: When an external command (e.g. compiler) fails, distutils now @@ -2620,7 +2620,7 @@ Library - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if - the timeout occcurs; tweaked the set of exported symbols; renamed + the timeout occurs; tweaked the set of exported symbols; renamed Empty/Full to QueueEmpty/QueueFull; "with (yield from lock)" now uses a separate context manager; readexactly() raises if not enough data was read; PTY support tweaks. @@ -3944,7 +3944,7 @@ Library - Issue #18996: TestCase.assertEqual() now more cleverly shorten differing strings in error report. -- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string reperesentation. +- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string representation. - Issue #18978: ``urllib.request.Request`` now allows the method to be indicated on the class and no longer sets it to None in ``__init__``. @@ -3952,7 +3952,7 @@ Library - Issue #18626: the inspect module now offers a basic command line introspection interface (Initial patch by Claudiu Popa) -- Issue #3015: Fixed tkinter with wantobject=False. Any Tcl command call +- Issue #3015: Fixed tkinter with ``wantobjects=False``. Any Tcl command call returned empty string. - Issue #19037: The mailbox module now makes all changes to maildir files @@ -4191,7 +4191,7 @@ Library - Issue #18532: Change the builtin hash algorithms' names to lower case names as promised by hashlib's documentation. -- Issue #8713: add new spwan and forkserver start methods, and new functions +- Issue #8713: add new spawn and forkserver start methods, and new functions get_all_start_methods, get_start_method, and set_start_method, to multiprocessing. @@ -4524,7 +4524,7 @@ Core and Builtins - Issue #16613: Add *m* argument to ``collections.Chainmap.new_child`` to allow the new child map to be specified explicitly. -- Issue #16730: importlib.machinery.FileFinder now no longers raises an +- Issue #16730: importlib.machinery.FileFinder now no longer raises an exception when trying to populate its cache and it finds out the directory is unreadable or has turned into a file. Reported and diagnosed by David Pritchard. @@ -4832,7 +4832,7 @@ Library on Windows and adds no value over and above python -m pydoc ... - Issue #18155: The csv module now correctly handles csv files that use - a delimter character that has a special meaning in regexes, instead of + a delimiter character that has a special meaning in regexes, instead of throwing an exception. - Issue #14360: encode_quopri can now be successfully used as an encoder @@ -6329,7 +6329,7 @@ Documentation - Issue #15940: Specify effect of locale on time functions. -- Issue #17538: Document XML vulnerabilties +- Issue #17538: Document XML vulnerabilities - Issue #16642: sched.scheduler timefunc initial default is time.monotonic. Patch by Ramchandra Apte @@ -6676,7 +6676,7 @@ Library - Issue #14669: Fix pickling of connections and sockets on Mac OS X by sending/receiving an acknowledgment after file descriptor transfer. - TestPicklingConnection has been reenabled for Mac OS X. + TestPicklingConnection has been re-enabled for Mac OS X. - Issue #11062: Fix adding a message from file to Babyl mailbox. @@ -7114,7 +7114,7 @@ Build - Issue #14330: For cross builds, don't use host python, use host search paths for host compiler. -- Issue #15235: Allow Berkley DB versions up to 5.3 to build the dbm module. +- Issue #15235: Allow Berkeley DB versions up to 5.3 to build the dbm module. - Issue #15268: Search curses.h in /usr/include/ncursesw. @@ -7264,7 +7264,7 @@ Library called with no arguments. - Issue #14653: email.utils.mktime_tz() no longer relies on system - mktime() when timezone offest is supplied. + mktime() when timezone offset is supplied. - Issue #14684: zlib.compressobj() and zlib.decompressobj() now support the use of predefined compression dictionaries. Original patch by Sam Rushing. @@ -7606,7 +7606,7 @@ Library - Issue #14773: Fix os.fwalk() failing on dangling symlinks. - Issue #12541: Be lenient with quotes around Realm field of HTTP Basic - Authentation in urllib2. + Authentication in urllib2. - Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add doc entry. Add tarfile.filemode alias with deprecation warning. @@ -7673,7 +7673,7 @@ Library IDLE ---- -- Issue #14958: Change IDLE systax highlighting to recognize all string and +- Issue #14958: Change IDLE syntax highlighting to recognize all string and byte literals supported in Python 3.3. - Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. @@ -10176,7 +10176,7 @@ IDLE - Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Issue #9871: Prevent IDLE 3 crash when given byte stings +- Issue #9871: Prevent IDLE 3 crash when given byte strings with invalid hex escape sequences, like b'\x0'. (Original patch by Claudiu Popa.) @@ -12098,7 +12098,7 @@ Library - Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING environment variable to set the filesystem encoding at Python startup. sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. + re-encode all filenames in all objects. - Issue #9410: Various optimizations to the pickle module, leading to speedups up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; @@ -12509,7 +12509,7 @@ Library - Issue #9605: posix.getlogin() decodes the username with file filesystem encoding and surrogateescape error handler. Patch written by David Watson. -- Issue #9604: posix.initgroups() encodes the username using the fileystem +- Issue #9604: posix.initgroups() encodes the username using the filesystem encoding and surrogateescape error handler. Patch written by David Watson. - Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name @@ -12667,7 +12667,7 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- -- Issue #8991: convertbuffer() rejects discontigious buffers. +- Issue #8991: convertbuffer() rejects discontiguous buffers. - Issue #7616: Fix copying of overlapping memoryview slices with the Intel compiler. @@ -13211,7 +13211,7 @@ Library - Issue #7989: Added pure python implementation of the `datetime` module. The C module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based + defined in datetime with fast C implementation. Python implementation is based on the original python prototype for the datetime module by Tim Peters with minor modifications by the PyPy project. The test suite now tests `datetime` module with and without `_datetime` acceleration using the same test cases. @@ -15049,7 +15049,7 @@ Extension Modules an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. The version number has been bumped to 0.3. -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built +- Issue #5359: Re-add the Berkeley DB detection code to allow _dbm be built using Berkeley DB. Tests @@ -16169,7 +16169,7 @@ Core and Builtins codec subsystem, for example UTF-8 is turned into utf-8. - Issue #4200: Changed the atexit module to store its state in its - PyModuleDef atexitmodule. This fixes a bug with multiple subinterpeters. + PyModuleDef atexitmodule. This fixes a bug with multiple subinterpreters. - Issue #4237: io.FileIO() was raising invalid warnings caused by insufficient initialization of PyFileIOObject struct members. @@ -17028,7 +17028,7 @@ Extension Modules and renamed to filter(), map(), and zip(). Also, renamed izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). -- Issue #1762972: Readded the reload() function as imp.reload(). +- Issue #1762972: Re-added the reload() function as imp.reload(). - Bug #2111: mmap segfaults when trying to write a block opened with PROT_READ. @@ -18448,7 +18448,7 @@ Core and builtins - Fixed bug #1459029 - unicode reprs were double-escaped. -- Patch #1396919: The system scope threads are reenabled on FreeBSD +- Patch #1396919: The system scope threads are re-enabled on FreeBSD 5.4 and later versions. - Bug #1115379: Compiling a Unicode string with an encoding declaration @@ -21803,7 +21803,7 @@ Library - New csv package makes it easy to read/write CSV files. - Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and + including a split() function for easy splitting of quoted strings and commands. An iterator interface was also implemented. Tools/Demos @@ -27751,7 +27751,7 @@ Fri Mar 12 22:15:43 1999 Guido van Rossum The filename to URL conversion didn't properly quote special characters. - The URL to filename didn't properly unquote special chatacters. + The URL to filename didn't properly unquote special characters. * Objects/floatobject.c: OK, try again. Vladimir gave me a fix for the alignment bus error, @@ -27807,7 +27807,7 @@ Wed Mar 10 22:55:47 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). @@ -28130,7 +28130,7 @@ webchecker and other ftp retrieves. - ConfigParser's get() method now accepts an optional keyword argument (vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your +__init__. You can now also have recursive references in your configuration file. - Some improvements to the Queue module, including a put_nowait() @@ -28209,7 +28209,7 @@ core. not. - The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) +w.scroll(). (It then calls wscrl(win, nlines) instead of scroll(win).) Changes to tools ---------------- @@ -28504,7 +28504,7 @@ PyEval_GetGlobals. - glmodule.c: check in the changed version after running the stubber again -- this solves the conflict with curses over the 'clear' entry point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) +eons ago, but I never regenerated glmodule.c :-( ) - frameobject.c: fix reference count bug in PyFrame_New. Vladimir Marangozov. @@ -28581,7 +28581,7 @@ idiom L1[len(L1):] = L2. - Better error messages when a sequence is indexed with a non-integer. -- Bettter error message when calling a non-callable object (include +- Better error message when calling a non-callable object (include the type in the message). Python services @@ -28656,7 +28656,7 @@ Internet Protocols and Support - imaplib.py: new version from Piers Lauder. - smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). +list or strings as the destination (common newbie mistake). - poplib.py: LIST with a msg argument fixed. @@ -31109,7 +31109,7 @@ encoding/decoding CGI form arguments. Catch all errors from the ftp module. HTTP requests now add the Host: header line. The proxy variable names are now mapped to lower case, for Windows. The spliturl() function no longer erroneously throws away all data past -the first newline. The basejoin() function now intereprets "../" +the first newline. The basejoin() function now interprets "../" correctly. I *believe* that the problems with "exception raised in __del__" under certain circumstances have been fixed (mostly by changes elsewher in the interpreter). @@ -31397,7 +31397,7 @@ changes and fixes. - Added a bunch of new winfo options to Tkinter.py; we should now be up to date with Tk 4.2. The new winfo options supported are: -mananger, pointerx, pointerxy, pointery, server, viewable, visualid, +manager, pointerx, pointerxy, pointery, server, viewable, visualid, visualsavailable. - The broken bind() method on Canvas objects defined in the Canvas.py @@ -32552,7 +32552,7 @@ The same applies to posixfile.open() and the socket method makefile(). is being maintained and distributed separately. - Improved support for the Apple Macintosh, in part by Jack Jansen, -e.g. interfaces to (a few) resource mananger functions, get/set file +e.g. interfaces to (a few) resource manager functions, get/set file type and creator, gestalt, sound manager, speech manager, MacTCP, comm toolbox, and the think C console library. This is being maintained and distributed separately. @@ -33229,7 +33229,7 @@ sys.argv[0]; it can simply do "if __name__ == '__main__': main()". * When an object is printed by the print statement, its implementation of str() is used. This means that classes can define __str__(self) to direct how their instances are printed. This is different from -__repr__(self), which should define an unambigous string +__repr__(self), which should define an unambiguous string representation of the instance. (If __str__() is not defined, it defaults to __repr__().) @@ -34366,7 +34366,7 @@ eval_code) and ceval.h (which doesn't need compile.hand declares the rest) ceval.h defines macros BGN_SAVE / END_SAVE for use with threads (to -improve the parallellism of multi-threaded programs by letting other +improve the parallelism of multi-threaded programs by letting other Python code run when a blocking system call or something similar is made) @@ -34514,7 +34514,7 @@ names listed in a 'global' statement must not be used in the function before the statement is reached. Remember that you don't need to use 'global' if you only want to *use* -a global variable in a function; nor do you need ot for assignments to +a global variable in a function; nor do you need to for assignments to parts of global variables (e.g., list or dictionary items or attributes of class instances). This has not changed; in fact assignment to part of a global variable was the standard workaround. diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index a9f25b482508ba..f30ed548e7e033 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -4,8 +4,8 @@ .. release date: 2020-10-05 .. section: Security -Fixes `python3x._pth` being ignored on Windows, caused by the fix for -:issue:`29778` (CVE-2020-15801). +Fixes ``python3x._pth`` being ignored on Windows, caused by the fix for +:issue:`29778` (:cve:`2020-15801`). .. @@ -25,7 +25,7 @@ events. .. section: Security Ensure :file:`python3.dll` is loaded from correct locations when Python is -embedded (CVE-2020-15523). +embedded (:cve:`2020-15523`). .. @@ -97,7 +97,7 @@ convention. Patch by Donghee Na. .. nonce: aJS9B3 .. section: Core and Builtins -Port the :mod:`_bisect` module to the multi-phase initialization API +Port the :mod:`!_bisect` module to the multi-phase initialization API (:pep:`489`). .. @@ -128,7 +128,7 @@ Taskaya. .. nonce: lh335O .. section: Core and Builtins -Port the :mod:`_lsprof` extension module to multi-phase initialization +Port the :mod:`!_lsprof` extension module to multi-phase initialization (:pep:`489`). .. @@ -148,7 +148,7 @@ Port the :mod:`cmath` extension module to multi-phase initialization .. nonce: jiXmyT .. section: Core and Builtins -Port the :mod:`_scproxy` extension module to multi-phase initialization +Port the :mod:`!_scproxy` extension module to multi-phase initialization (:pep:`489`). .. @@ -168,7 +168,7 @@ Port the :mod:`termios` extension module to multi-phase initialization .. nonce: QuDIut .. section: Core and Builtins -Convert the :mod:`_sha256` extension module types to heap types. +Convert the :mod:`!_sha256` extension module types to heap types. .. @@ -187,7 +187,7 @@ classes with a huge amount of arguments. Patch by Pablo Galindo. .. nonce: CnRME3 .. section: Core and Builtins -Port the :mod:`_overlapped` extension module to multi-phase initialization +Port the :mod:`!_overlapped` extension module to multi-phase initialization (:pep:`489`). .. @@ -197,7 +197,7 @@ Port the :mod:`_overlapped` extension module to multi-phase initialization .. nonce: X9CZgo .. section: Core and Builtins -Port the :mod:`_curses_panel` extension module to multi-phase initialization +Port the :mod:`!_curses_panel` extension module to multi-phase initialization (:pep:`489`). .. @@ -207,7 +207,7 @@ Port the :mod:`_curses_panel` extension module to multi-phase initialization .. nonce: 5jZymK .. section: Core and Builtins -Port the :mod:`_opcode` extension module to multi-phase initialization +Port the :mod:`!_opcode` extension module to multi-phase initialization (:pep:`489`). .. @@ -217,7 +217,7 @@ Port the :mod:`_opcode` extension module to multi-phase initialization .. nonce: 3-VJiH .. section: Core and Builtins -Fixes the wrong error description in the error raised by using 2 `,` in +Fixes the wrong error description in the error raised by using 2 ``,`` in format string in f-string and :meth:`str.format`. .. @@ -282,7 +282,7 @@ initialized ``_ast`` module. .. nonce: vcxSUa .. section: Core and Builtins -Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_operator` to use :c:func:`PyType_FromSpec`. .. @@ -291,7 +291,7 @@ Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. .. nonce: fubBkb .. section: Core and Builtins -Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. +Port :mod:`!_sha3` to multi-phase init. Convert static types to heap types. .. @@ -300,7 +300,7 @@ Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. .. nonce: FC13e7 .. section: Core and Builtins -Port the :mod:`_blake2` extension module to the multi-phase initialization +Port the :mod:`!_blake2` extension module to the multi-phase initialization API (:pep:`489`). .. @@ -339,7 +339,7 @@ The output of ``python --help`` contains now only ASCII characters. .. nonce: O0d3ym .. section: Core and Builtins -Port the :mod:`_sha1`, :mod:`_sha512`, and :mod:`_md5` extension modules to +Port the :mod:`!_sha1`, :mod:`!_sha512`, and :mod:`!_md5` extension modules to multi-phase initialization API (:pep:`489`). .. @@ -605,8 +605,8 @@ Opt out serialization/deserialization for _random.Random .. nonce: jxJ4yn .. section: Core and Builtins -Rename `PyPegen*` functions to `PyParser*`, so that we can remove the old -set of `PyParser*` functions that were using the old parser, but keep +Rename ``PyPegen*`` functions to ``PyParser*``, so that we can remove the old +set of ``PyParser*`` functions that were using the old parser, but keep everything backwards-compatible. .. @@ -636,7 +636,7 @@ Remove the remaining files from the old parser and the :mod:`symbol` module. .. nonce: _yI-ax .. section: Core and Builtins -Convert :mod:`_bz2` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_bz2` to use :c:func:`PyType_FromSpec`. .. @@ -666,7 +666,7 @@ by Brandt Bucher. .. nonce: 61iyYh .. section: Core and Builtins -Port :mod:`_gdbm` to multiphase initialization. +Port :mod:`!_gdbm` to multiphase initialization. .. @@ -696,7 +696,7 @@ for emitting syntax errors. Patch by Pablo Galindo. .. nonce: mmlp3Q .. section: Core and Builtins -Port :mod:`_dbm` to multiphase initialization. +Port :mod:`!_dbm` to multiphase initialization. .. @@ -747,7 +747,7 @@ Galindo. Fix a bug where a line with only a line continuation character is not considered a blank line at tokenizer level. In such cases, more than a -single `NEWLINE` token was emitted. The old parser was working around the +single ``NEWLINE`` token was emitted. The old parser was working around the issue, but the new parser threw a :exc:`SyntaxError` for valid input due to this. For example, an empty line following a line continuation character was interpreted as a :exc:`SyntaxError`. @@ -1010,7 +1010,7 @@ Port :mod:`mmap` to multiphase initialization. .. nonce: Kfe9fT .. section: Core and Builtins -Port :mod:`_lzma` to multiphase initialization. +Port :mod:`!_lzma` to multiphase initialization. .. @@ -1157,7 +1157,7 @@ The :class:`threading.Thread` constructor now uses the target name if the .. nonce: bnh-VG .. section: Library -fix `tkinter.EventType` Enum so all members are strings, and none are tuples +fix ``tkinter.EventType`` Enum so all members are strings, and none are tuples .. @@ -1220,7 +1220,7 @@ class Previously there was no way to check that without using private API. See the `relevant issue in python/typing -` +`_. .. @@ -1229,8 +1229,8 @@ Previously there was no way to check that without using private API. See the .. nonce: pI_uZQ .. section: Library -Honor `object` overrides in `Enum` class creation (specifically, `__str__`, -`__repr__`, `__format__`, and `__reduce_ex__`). +Honor ``object`` overrides in ``Enum`` class creation (specifically, ``__str__``, +``__repr__``, ``__format__``, and ``__reduce_ex__``). .. @@ -1239,7 +1239,7 @@ Honor `object` overrides in `Enum` class creation (specifically, `__str__`, .. nonce: IpYkEe .. section: Library -`enum.Flag` and `enum.IntFlag` members are now iterable +``enum.Flag`` and ``enum.IntFlag`` members are now iterable. .. @@ -1557,7 +1557,7 @@ activation. .. nonce: wqrj8C .. section: Library -Recursive evaluation of `typing.ForwardRef` in `get_type_hints`. +Recursive evaluation of ``typing.ForwardRef`` in ``get_type_hints``. .. @@ -1596,7 +1596,7 @@ UnpicklingError instead of crashing. .. section: Library Avoid infinite loop when reading specially crafted TAR files using the -tarfile module (CVE-2019-20907). +tarfile module (:cve:`2019-20907`). .. @@ -1851,8 +1851,8 @@ Merry. .. nonce: 1dk8Bu .. section: Library -:mod:`ensurepip` now disables the use of `pip` cache when installing the -bundled versions of `pip` and `setuptools`. Patch by Krzysztof Konopko. +:mod:`ensurepip` now disables the use of ``pip`` cache when installing the +bundled versions of ``pip`` and ``setuptools``. Patch by Krzysztof Konopko. .. @@ -1861,8 +1861,8 @@ bundled versions of `pip` and `setuptools`. Patch by Krzysztof Konopko. .. nonce: _dx3OO .. section: Library -Removed :meth:`asyncio.Task.current_task` and -:meth:`asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. +Removed :meth:`!asyncio.Task.current_task` and +:meth:`!asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. .. @@ -1933,7 +1933,7 @@ Smith and Tal Einat. .. nonce: n7fOwS .. section: Library -Added a `defaults` parameter to :class:`logging.Formatter`, to allow +Added a ``defaults`` parameter to :class:`logging.Formatter`, to allow specifying default values for custom fields. Patch by Asaf Alon and Bar Harel. @@ -2225,7 +2225,7 @@ policy. .. nonce: ps7Yf1 .. section: Library -func:`hashlib.new` passed ``usedforsecurity`` to OpenSSL EVP constructor +:func:`hashlib.new` passed ``usedforsecurity`` to OpenSSL EVP constructor ``_hashlib.new()``. test_hashlib and test_smtplib handle strict security policy better. @@ -2393,8 +2393,8 @@ closes connection during TLS negotiation .. nonce: kOOaHn .. section: Library -fix default `_missing_` so a duplicate `ValueError` is not set as the -`__context__` of the original `ValueError` +fix default ``_missing_`` so a duplicate ``ValueError`` is not set as the +``__context__`` of the original ``ValueError``. .. @@ -2527,7 +2527,7 @@ in Python 3.4 and removed in Python 3.5. .. nonce: BE7zbu .. section: Library -Fix `cgi.parse_multipart` without content_length. Patch by Roger Duran +Fix ``cgi.parse_multipart`` without content_length. Patch by Roger Duran .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 78f4377656b0cc..bdf9488c81bae1 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -88,10 +88,10 @@ Goldschmidt. .. nonce: d4a-8o .. section: Core and Builtins -Fix a bug in the parser, where a curly brace following a `primary` didn't -fail immediately. This led to invalid expressions like `a {b}` to throw a +Fix a bug in the parser, where a curly brace following a ``primary`` didn't +fail immediately. This led to invalid expressions like ``a {b}`` to throw a :exc:`SyntaxError` with a wrong offset, or invalid expressions ending with a -curly brace like `a {` to not fail immediately in the REPL. +curly brace like ``a {`` to not fail immediately in the REPL. .. @@ -214,7 +214,7 @@ Micro optimization for range.index if step is 1. Patch by Donghee Na. .. nonce: qPWjJA .. section: Core and Builtins -Add `sys._current_exceptions()` function to retrieve a dictionary mapping +Add ``sys._current_exceptions()`` function to retrieve a dictionary mapping each thread's identifier to the topmost exception currently active in that thread at the time the function is called. @@ -302,7 +302,7 @@ type to a heap type. Patch by Mohamed Koubaa and Victor Stinner. .. section: Library Fix memory leak in :func:`subprocess.Popen` in case an uid (gid) specified -in `user` (`group`, `extra_groups`) overflows `uid_t` (`gid_t`). +in ``user`` (``group``, ``extra_groups``) overflows ``uid_t`` (``gid_t``). .. @@ -362,7 +362,7 @@ plistlib: fix parsing XML plists with hexadecimal integer values .. nonce: 85BsRA .. section: Library -Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when +Fix an incorrectly formatted error from :meth:`!_codecs.charmap_decode` when called with a mapped value outside the range of valid Unicode code points. PR by Max Bernstein. @@ -649,7 +649,7 @@ Document __format__ functionality for IP addresses. .. nonce: CzBMit .. section: Documentation -Document the default implementation of `object.__eq__`. +Document the default implementation of ``object.__eq__``. .. @@ -878,7 +878,7 @@ targeting 3.10 or later. .. nonce: sh8IDY .. section: C API -Add `_Py_closerange` function to provide performant closing of a range of +Add ``_Py_closerange`` function to provide performant closing of a range of file descriptors. .. @@ -898,7 +898,7 @@ available again in limited API. .. nonce: ZZ5wJG .. section: C API -Add `PyIter_Send` function to allow sending value into +Add ``PyIter_Send`` function to allow sending value into generator/coroutine/iterator without raising StopIteration exception to signal return. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 7112819c1b4118..2aef87ab929aab 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -247,8 +247,8 @@ fixes union type expressions not de-duplicating ``GenericAlias`` objects. .. nonce: B-Veg7 .. section: Core and Builtins -The import system triggers a `ImportWarning` when it falls back to using -`load_module()`. +The import system triggers a ``ImportWarning`` when it falls back to using +``load_module()``. .. @@ -464,8 +464,8 @@ Support signal module on VxWorks. .. nonce: r9rNCj .. section: Library -We fixed an issue in `pickle.whichmodule` in which importing -`multiprocessing` could change the how pickle identifies which module an +We fixed an issue in ``pickle.whichmodule`` in which importing +``multiprocessing`` could change the how pickle identifies which module an object belongs to, potentially breaking the unpickling of those objects. .. @@ -602,7 +602,7 @@ function when ``os.open`` fails. .. nonce: F363jO .. section: Library -Fix `os.sendfile()` on illumos. +Fix ``os.sendfile()`` on illumos. .. @@ -718,8 +718,8 @@ Improve asyncio.wait function to create the futures set just one time. .. nonce: BzizYV .. section: Library -Update various modules in the stdlib to fall back on `__spec__.loader` when -`__loader__` isn't defined on a module. +Update various modules in the stdlib to fall back on ``__spec__.loader`` when +``__loader__`` isn't defined on a module. .. @@ -728,8 +728,8 @@ Update various modules in the stdlib to fall back on `__spec__.loader` when .. nonce: CAsI3O .. section: Library -The `load_module()` methods found in importlib now trigger a -DeprecationWarning. +The ``load_module()`` methods found in ``importlib`` now trigger a +``DeprecationWarning``. .. @@ -872,7 +872,7 @@ extension during TLS handshake when no custom context is supplied. .. nonce: 5mi7b0 .. section: Library -Add func:`os.eventfd` to provide a low level interface for Linux's event +Add :func:`os.eventfd` to provide a low level interface for Linux's event notification file descriptor. .. @@ -1386,7 +1386,7 @@ Python already implicitly installs signal handlers: see The ``Py_TRASHCAN_BEGIN`` macro no longer accesses PyTypeObject attributes, but now can get the condition by calling the new private -:c:func:`_PyTrash_cond()` function which hides implementation details. +:c:func:`!_PyTrash_cond()` function which hides implementation details. .. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 414823f162d85c..5cea16c259d5ee 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -193,7 +193,7 @@ subinterpreters. Patch by Victor Stinner. .. nonce: j7nl6A .. section: Core and Builtins -Make :c:func:`_PyUnicode_FromId` function compatible with subinterpreters. +Make :c:func:`!_PyUnicode_FromId` function compatible with subinterpreters. Each interpreter now has an array of identifier objects (interned strings decoded from UTF-8). Patch by Victor Stinner. @@ -367,7 +367,7 @@ uses "options" instead. .. nonce: Quy3zn .. section: Library -Port the :mod:`_thread` extension module to the multiphase initialization +Port the :mod:`!_thread` extension module to the multiphase initialization API (:pep:`489`) and convert its static types to heap types. .. @@ -412,7 +412,7 @@ be created automatically. ``logging.disable`` will now validate the types and value of its parameter. It also now accepts strings representing the levels (as does -``loging.setLevel``) instead of only the numerical values. +``logging.setLevel``) instead of only the numerical values. .. @@ -516,7 +516,7 @@ Define THREAD_STACK_SIZE for VxWorks. .. nonce: x8TASR .. section: Library -[Enum] `_EnumDict.update()` is now supported +[Enum] ``_EnumDict.update()`` is now supported. .. @@ -960,8 +960,8 @@ explicitly and so not exported. .. nonce: Je08Ny .. section: C API -Remove the private :c:func:`_Py_fopen` function which is no longer needed. -Use :c:func:`_Py_wfopen` or :c:func:`_Py_fopen_obj` instead. Patch by Victor +Remove the private :c:func:`!_Py_fopen` function which is no longer needed. +Use :c:func:`!_Py_wfopen` or :c:func:`!_Py_fopen_obj` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index dc95e8ce072fd9..a85ea1ff1c2817 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -108,7 +108,7 @@ a slice at the start of the ``bytearray`` to a shorter byte string). .. nonce: WfTdfg .. section: Core and Builtins -Fix the :c:func:`_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when +Fix the :c:func:`!_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when :c:func:`Py_Initialize` / :c:func:`Py_Finalize` is called multiple times: preserve ``_PyRuntime.unicode_ids.next_index`` value. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index c379b968c9885b..31b7df2c61158e 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -158,7 +158,7 @@ tests that are unrelated to :class:`ProcessPoolExecutor` on those platforms. .. nonce: hsCNgX .. section: Core and Builtins -If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator +If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. @@ -315,7 +315,7 @@ Adds :const:`resource.RLIMIT_KQUEUES` constant from FreeBSD to the .. section: Library Make the pure Python implementation of :mod:`xml.etree.ElementTree` behave -the same as the C implementation (:mod:`_elementree`) regarding default +the same as the C implementation (:mod:`!_elementree`) regarding default attribute values (by not setting ``specified_attributes=1``). .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 3a1694f444616a..32ee34d9a68910 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -4,7 +4,7 @@ .. release date: 2021-04-05 .. section: Security -CVE-2021-3426: Remove the ``getfile`` feature of the :mod:`pydoc` module +:cve:`2021-3426`: Remove the ``getfile`` feature of the :mod:`pydoc` module which could be abused to read arbitrary files on the disk (directory traversal vulnerability). Moreover, even source code of Python modules can contain sensitive data like passwords. Vulnerability reported by David @@ -83,7 +83,7 @@ instruction dispatch a bit. .. nonce: PhaT-B .. section: Core and Builtins -Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo +Fix reference leak in the :mod:`!_hashopenssl` extension. Patch by Pablo Galindo. .. @@ -261,7 +261,7 @@ the annotations _Py_NO_SANITIZE_ADDRESS, _Py_NO_SANITIZE_THREAD, and _Py_NO_SANITIZE_MEMORY. Those annotations are no longer needed. To disable the radix tree map, set a preprocessor flag as follows: -`-DWITH_PYMALLOC_RADIX_TREE=0`. +``-DWITH_PYMALLOC_RADIX_TREE=0``. Co-authored-by: Tim Peters @@ -274,7 +274,7 @@ Co-authored-by: Tim Peters Only handle asynchronous exceptions and requests to drop the GIL when returning from a call or on the back edges of loops. Makes sure that -:meth:`__exit__` is always called in with statements, even for interrupts. +:meth:`~object.__exit__` is always called in with statements, even for interrupts. .. @@ -664,11 +664,11 @@ on failure. .. nonce: f1dr_5 .. section: Library -Enum's `repr()` and `str()` have changed: `repr()` is now -*EnumClass.MemberName* and `str()` is *MemberName*. Additionally, stdlib +Enum's ``repr()`` and ``str()`` have changed: ``repr()`` is now +*EnumClass.MemberName* and ``str()`` is *MemberName*. Additionally, stdlib Enum's whose contents are available as module attributes, such as -`RegexFlag.IGNORECASE`, have their `repr()` as *module.name*, e.g. -`re.IGNORECASE`. +``RegexFlag.IGNORECASE``, have their ``repr()`` as *module.name*, e.g. +``re.IGNORECASE``. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index e7b6b93d0b6df3..306e987a41612e 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -182,7 +182,7 @@ normally be possible, but might occur in some unusual circumstances. .. nonce: u5Y6bS .. section: Core and Builtins -Importing the :mod:`_signal` module in a subinterpreter has no longer side +Importing the :mod:`!_signal` module in a subinterpreter has no longer side effects. .. @@ -776,11 +776,11 @@ builtins.open() is now io.open(). .. nonce: o1zEk_ .. section: Library -The Python :func:`_pyio.open` function becomes a static method to behave as +The Python :func:`!_pyio.open` function becomes a static method to behave as :func:`io.open` built-in function: don't become a bound method when stored as a class variable. It becomes possible since static methods are now -callable in Python 3.10. Moreover, :func:`_pyio.OpenWrapper` becomes a -simple alias to :func:`_pyio.open`. Patch by Victor Stinner. +callable in Python 3.10. Moreover, :func:`!_pyio.OpenWrapper` becomes a +simple alias to :func:`!_pyio.open`. Patch by Victor Stinner. .. @@ -1181,8 +1181,8 @@ Lewis Gaul. .. nonce: NzLVaR .. section: Library -Add `pathlib.Path.hardlink_to()` method that supersedes `link_to()`. The new -method has the same argument order as `symlink_to()`. +Add ``pathlib.Path.hardlink_to()`` method that supersedes ``link_to()``. The new +method has the same argument order as ``symlink_to()``. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 7c991e7667b01a..23b13c058f96bd 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -38,7 +38,7 @@ significant performance overhead when loading from ``.pyc`` files. .. section: Security Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix -for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +for the :cve:`2013-0340` "Billion Laughs" vulnerability. This copy is most used on Windows and macOS. .. @@ -613,7 +613,7 @@ Rename ``types.Union`` to ``types.UnionType``. .. section: Core and Builtins Expose specialization stats in python via -:func:`_opcode.get_specialization_stats`. +:func:`!_opcode.get_specialization_stats`. .. @@ -660,7 +660,7 @@ Karabas. .. section: Core and Builtins Parameter substitution of the union type with wrong types now raises -``TypeError`` instead of returning ``NotImplemented``. +``TypeError`` instead of returning :data:`NotImplemented`. .. @@ -819,7 +819,7 @@ always available when needed. Patch by Mark Shannon. .. nonce: qKnSqV .. section: Core and Builtins -The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a debug build of Python. Patch by Victor Stinner. @@ -972,7 +972,7 @@ manager` protocols correspondingly. .. section: Core and Builtins Make sure that the line number is set when entering a comprehension scope. -Ensures that backtraces inclusing generator expressions show the correct +This ensures that backtraces including generator expressions show the correct line number. .. @@ -1642,9 +1642,9 @@ interval specified with nanosecond precision. .. nonce: UptGAn .. section: Library -Remove from the :mod:`configparser` module: the :class:`SafeConfigParser` -class, the :attr:`filename` property of the :class:`ParsingError` class, the -:meth:`readfp` method of the :class:`ConfigParser` class, deprecated since +Remove from the :mod:`configparser` module: the :class:`!SafeConfigParser` +class, the :attr:`!filename` property of the :class:`~configparser.ParsingError` class, the +:meth:`!readfp` method of the :class:`~configparser.ConfigParser` class, deprecated since Python 3.2. Patch by Hugo van Kemenade. @@ -1701,7 +1701,7 @@ Remove many old deprecated :mod:`unittest` features: .. nonce: y1kEfP .. section: Library -Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +Remove the deprecated ``split()`` method of :class:`!_tkinter.TkappType`. Patch by Erlend E. Aasland. .. @@ -1720,7 +1720,7 @@ Improve the speed and accuracy of statistics.pvariance(). .. nonce: WI9zQY .. section: Library -Remove :meth:`__getitem__` methods of +Remove :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput`, deprecated since Python 3.9. @@ -2298,9 +2298,9 @@ Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. .. nonce: ThuDMI .. section: Library -Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` -when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` -for a big value of the ``len`` parameter. Patch by Pablo Galindo +Fixed a bug in the :mod:`!_ssl` module that was throwing :exc:`OverflowError` +when using :meth:`!_ssl._SSLSocket.write` and :meth:`!_ssl._SSLSocket.read` +for a big value of the ``len`` parameter. Patch by Pablo Galindo. .. @@ -2398,7 +2398,7 @@ class in the interactive session. Instead of :exc:`TypeError`, it should be .. nonce: R3IcM1 .. section: Library -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +Fix memory leak in :func:`!_tkinter._flatten` if it is called with a sequence or set, but not list or tuple. .. @@ -2722,7 +2722,7 @@ Importing typing.io or typing.re now prints a ``DeprecationWarning``. .. section: Library argparse actions store_const and append_const each receive a default value -of None when the ``const`` kwarg is not provided. Previously, this raised a +of ``None`` when the ``const`` kwarg is not provided. Previously, this raised a :exc:`TypeError`. .. @@ -2808,7 +2808,7 @@ behaves differently than the similar implementation in :mod:`sysconfig`. .. nonce: 3hmkWw .. section: Library -:class:`smtpd.MailmanProxy` is now removed as it is unusable without an +:class:`!smtpd.MailmanProxy` is now removed as it is unusable without an external module, ``mailman``. Patch by Donghee Na. .. @@ -3204,7 +3204,7 @@ deprecated, for removal in 3.14. .. section: Library When :class:`http.server.SimpleHTTPRequestHandler` sends a ``301 (Moved -Permanently)`` for a directory path not ending with `/`, add a +Permanently)`` for a directory path not ending with ``/``, add a ``Content-Length: 0`` header. This improves the behavior for certain clients. @@ -3240,7 +3240,7 @@ Patch by Erlend E. Aasland. .. nonce: m72tlH .. section: Library -AIX: `Lib/_aix_support.get_platform()` may fail in an AIX WPAR. The fileset +AIX: ``Lib/_aix_support.get_platform()`` may fail in an AIX WPAR. The fileset bos.rte appears to have a builddate in both LPAR and WPAR so this fileset is queried rather than bos.mp64. To prevent a similar situation (no builddate in ODM) a value (9988) sufficient for completing a build is provided. Patch @@ -3483,9 +3483,9 @@ Improved reprs of :mod:`threading` synchronization objects: Deprecated the following :mod:`unittest` functions, scheduled for removal in Python 3.13: -* :func:`~!unittest.findTestCases` -* :func:`~!unittest.makeSuite` -* :func:`~!unittest.getTestCaseNames` +* :func:`!findTestCases` +* :func:`!makeSuite` +* :func:`!getTestCaseNames` Use :class:`~unittest.TestLoader` methods instead: @@ -3685,9 +3685,9 @@ equivalents are valid. .. nonce: aJuvQF .. section: Documentation -Removed the othergui.rst file, any references to it, and the list of GUI +Removed the ``othergui.rst`` file, any references to it, and the list of GUI frameworks in the FAQ. In their place I've added links to the Python Wiki -`page on GUI frameworks `. +`page on GUI frameworks `_. .. @@ -3995,7 +3995,7 @@ operator expressions. .. section: Documentation Document that :class:`collections.defaultdict` parameter ``default_factory`` -defaults to None and is positional-only. +defaults to ``None`` and is positional-only. .. @@ -4187,7 +4187,7 @@ Add calls of :func:`gc.collect` in tests to support PyPy. .. nonce: mQZdXU .. section: Tests -Made tests relying on the :mod:`_asyncio` C extension module optional to +Made tests relying on the :mod:`!_asyncio` C extension module optional to allow running on alternative Python implementations. Patch by Serhiy Storchaka. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 503e489b658e4d..48cf2c1e428d87 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -15,7 +15,7 @@ Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or .. section: Core and Builtins :data:`sys.stdlib_module_names` now contains the macOS-specific module -:mod:`_scproxy`. +:mod:`!_scproxy`. .. @@ -34,7 +34,7 @@ module but frozen modules are disabled. .. nonce: veL4lJ .. section: Core and Builtins -Specialize simple calls to Python functions (no starargs, keyowrd dict, or +Specialize simple calls to Python functions (no starargs, keyword dict, or closure) .. @@ -331,8 +331,8 @@ underlying SQLite API signals memory error. Patch by Erlend E. Aasland. .. nonce: 4MQt4r .. section: Library -pprint.pprint() now handles underscore_numbers correctly. Previously it was -always setting it to False. +:func:`pprint.pprint` now handles *underscore_numbers* correctly. +Previously it was always setting it to ``False``. .. @@ -507,7 +507,7 @@ Waygood. .. section: Library Make :func:`inspect.getmodule` catch ``FileNotFoundError`` raised by -:'func:`inspect.getabsfile`, and return ``None`` to indicate that the module +:func:`inspect.getabsfile`, and return ``None`` to indicate that the module could not be determined. .. @@ -897,8 +897,8 @@ was often the case on macOS. .. nonce: F18qcE .. section: Tests -Add more test cases for `@functools.singledispatchmethod` when combined with -`@classmethod` or `@staticmethod`. +Add more test cases for ``@functools.singledispatchmethod`` when combined with +``@classmethod`` or ``@staticmethod``. .. @@ -1023,7 +1023,7 @@ compile shared modules. .. nonce: 61gM2A .. section: Build -:mod:`pyexpat` and :mod:`_elementtree` no longer define obsolete macros +:mod:`pyexpat` and :mod:`!_elementtree` no longer define obsolete macros ``HAVE_EXPAT_CONFIG_H`` and ``USE_PYEXPAT_CAPI``. ``XML_POOR_ENTROPY`` is now defined in ``expat_config.h``. @@ -1189,7 +1189,7 @@ context objects can now be disabled. .. section: C API Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never -worked since the :c:type:`PyWeakReference` structure is opaque in the +worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. .. diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index a96a59115797ee..6a0ae20d1fb5ed 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -27,7 +27,7 @@ invalid targets. Patch by Pablo Galindo .. nonce: 3TmTSw .. section: Core and Builtins -:c:func:`_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including +:c:func:`!_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including setting the traceback on the exception instance) because ``exc_info`` is always normalized. @@ -350,7 +350,7 @@ Galindo. .. section: Library Fix possible crash when getting an attribute of -class:`xml.etree.ElementTree.Element` simultaneously with replacing the +:class:`xml.etree.ElementTree.Element` simultaneously with replacing the ``attrib`` dict. .. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 3dd335929d655f..64e2f39ad9db18 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -7,7 +7,7 @@ :c:func:`Py_EndInterpreter` now explicitly untracks all objects currently tracked by the GC. Previously, if an object was used later by another interpreter, calling :c:func:`PyObject_GC_UnTrack` on the object crashed if -the previous or the next object of the :c:type:`PyGC_Head` structure became +the previous or the next object of the :c:type:`!PyGC_Head` structure became a dangling pointer. Patch by Victor Stinner. .. @@ -161,7 +161,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: 7oGp-I .. section: Core and Builtins -:opcode:`PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. +:opcode:`!PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack. .. @@ -170,7 +170,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney. .. nonce: IKx4v6 .. section: Core and Builtins -Remove :opcode:`POP_EXCEPT_AND_RERAISE` and replace it by an equivalent +Remove :opcode:`!POP_EXCEPT_AND_RERAISE` and replace it by an equivalent sequence of other opcodes. .. @@ -258,7 +258,7 @@ instruction which performs the same operation, but without the loop. .. nonce: ADVaPT .. section: Core and Builtins -The code called from :c:func:`_PyErr_Display` was refactored to improve +The code called from :c:func:`!_PyErr_Display` was refactored to improve error handling. It now exits immediately upon an unrecoverable error. .. @@ -510,7 +510,7 @@ Remove special-casing of ``__new__`` in :meth:`enum.Enum.__dir__`. Improve day constants in :mod:`calendar`. -Now all constants (`MONDAY` ... `SUNDAY`) are documented, tested, and added +Now all constants (``MONDAY`` ... ``SUNDAY``) are documented, tested, and added to ``__all__``. .. @@ -1171,7 +1171,7 @@ Replaced deprecated usage of :c:func:`PyImport_ImportModuleNoBlock` with .. nonce: sMgDLz .. section: C API -The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the +The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the limited C API. It was never usable there, because it used internal structures which are not available in the limited C API. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 08d94e82ed8ccf..954f5c18b48000 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -275,7 +275,7 @@ Do not use POSIX semaphores on NetBSD .. nonce: M9m8Qd .. section: Core and Builtins -Improve the exc:`TypeError` message for non-string second arguments passed +Improve the :exc:`TypeError` message for non-string second arguments passed to the built-in functions :func:`getattr` and :func:`hasattr`. Patch by Géry Ogam. @@ -673,7 +673,7 @@ file .. section: Tests Mocks can no longer be provided as the specs for other Mocks. As a result, -an already-mocked object cannot be passed to `mock.Mock()`. This can uncover +an already-mocked object cannot be passed to ``mock.Mock()``. This can uncover bugs in tests since these Mock-derived Mocks will always pass certain tests (e.g. isinstance) and builtin assert functions (e.g. assert_called_once_with) will unconditionally pass. @@ -748,7 +748,7 @@ tests to use ``support.infinite_recursion()``. Patch by Victor Stinner. Skip test_builtin PTY tests on non-ASCII characters if the readline module is loaded. The readline module changes input() behavior, but test_builtin is -not intented to test the readline module. Patch by Victor Stinner. +not intended to test the readline module. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 52055b3fafd485..66ffa4ffba52e5 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -248,7 +248,7 @@ Don't un-adapt :opcode:`COMPARE_OP` when collecting specialization stats. .. nonce: RX_AzJ .. section: Core and Builtins -Fix specialization stats gathering for :opcode:`PRECALL` instructions. +Fix specialization stats gathering for :opcode:`!PRECALL` instructions. .. @@ -941,7 +941,7 @@ uvloop library. Make the :class:`configparser.ConfigParser` constructor raise :exc:`TypeError` if the ``interpolation`` parameter is not of type -:class:`configparser.Interpolation` +:class:`!configparser.Interpolation` .. @@ -1034,7 +1034,7 @@ if zlib uses the s390x hardware accelerator. Patch by Victor Stinner. .. nonce: jfciLG .. section: Build -Respect `--with-suffix` when building on case-insensitive file systems. +Respect ``--with-suffix`` when building on case-insensitive file systems. .. @@ -1088,7 +1088,7 @@ Patch by Kumar Aditya. Fix wasm32-emscripten test failures and platform issues. - Disable syscalls that are not supported or don't work, e.g. wait, getrusage, prlimit, -mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to cound open fds. - +mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to count open fds. - Add more checks for subprocess and fork. - Add workarounds for missing _multiprocessing and failing socket.accept(). - Enable bzip2. - Disable large file support. - Disable signal.alarm. @@ -1151,7 +1151,7 @@ Make query dialogs on Windows start with a cursor in the entry box. .. nonce: FhiH5P .. section: IDLE -Apply IDLE syntax highlighting to `.pyi` files. Patch by Alex Waygood and +Apply IDLE syntax highlighting to ``.pyi`` files. Patch by Alex Waygood and Terry Jan Reedy. .. @@ -1162,7 +1162,7 @@ Terry Jan Reedy. .. section: C API Python's public headers no longer import ````, leaving code that -embedd/extends Python free to define ``bool``, ``true`` and ``false``. +embeds/extends Python free to define ``bool``, ``true`` and ``false``. .. @@ -1182,7 +1182,7 @@ internal C API ``pycore_frame.h`` header file. Patch by Victor Stinner. .. section: C API Rename ``Include/buffer.h`` header file to ``Include/pybuffer.h`` to avoid -conflits with projects having an existing ``buffer.h`` header file. Patch by +conflicts with projects having an existing ``buffer.h`` header file. Patch by Victor Stinner. .. @@ -1202,5 +1202,5 @@ API). Patch by Victor Stinner. .. nonce: __ZdpH .. section: C API -Added function :c:func:`PyType_GetModuleByDef`, which allows accesss to +Added function :c:func:`PyType_GetModuleByDef`, which allows access to module state when a method's defining class is not available. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index 6e41f9cbd933b5..a376c8becea9f4 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -138,7 +138,7 @@ Replaced :opcode:`JUMP_ABSOLUTE` by the relative jump .. nonce: SwrrFO .. section: Core and Builtins -:c:func:`PyFrame_FastToLocalsWithError` and :c:func:`PyFrame_LocalsToFast` +:c:func:`!PyFrame_FastToLocalsWithError` and :c:func:`!PyFrame_LocalsToFast` are no longer called during profiling nor tracing. C code can access the ``f_locals`` attribute of :c:type:`PyFrameObject` by calling :c:func:`PyFrame_GetLocals`. @@ -295,7 +295,7 @@ oparg) as an adaptive counter. .. nonce: O12Pba .. section: Core and Builtins -Use inline caching for :opcode:`PRECALL` and :opcode:`CALL`, and remove the +Use inline caching for :opcode:`!PRECALL` and :opcode:`CALL`, and remove the internal machinery for managing the (now unused) non-inline caches. .. @@ -717,7 +717,7 @@ Fix :class:`asyncio.Semaphore` re-aquiring FIFO order. .. nonce: uaEDcI .. section: Library -The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings and have now been updated to note they will removed in Python 3.12 (:pep:`594`). @@ -1038,8 +1038,8 @@ Add optional parameter *dir_fd* in :func:`shutil.rmtree`. .. nonce: AixHW7 .. section: Library -:meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed -in 3.13. +:meth:`!unittest.TestProgram.usageExit` is marked as deprecated, +to be removed in Python 3.13. .. @@ -1173,7 +1173,7 @@ implemented. .. section: Library Add an Barrier object in synchronization primitives of *asyncio* Lib in -order to be consistant with Barrier from *threading* and *multiprocessing* +order to be consistent with Barrier from *threading* and *multiprocessing* libs* .. @@ -1211,7 +1211,7 @@ Update PEP URLs to :pep:`676`'s new canonical form. .. nonce: 4Dn48U .. section: Documentation -Clarified the old Python versions compatiblity note of +Clarified the old Python versions compatibility note of :func:`binascii.crc32` / :func:`zlib.adler32` / :func:`zlib.crc32` functions. @@ -1324,7 +1324,7 @@ extensions. .. section: Tests A test case for :func:`os.sendfile` is converted from deprecated -:mod:`asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. +:mod:`!asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. .. @@ -1421,7 +1421,7 @@ Patch by Victor Stinner. .. nonce: IB0XL4 .. section: Windows -Update ``zlib`` to v1.2.12 to resolve CVE-2018-25032. +Update ``zlib`` to v1.2.12 to resolve :cve:`2018-25032`. .. @@ -1472,8 +1472,8 @@ Update Windows installer to use SQLite 3.38.1. .. nonce: SPrGS9 .. section: Windows -Update bzip2 to 1.0.8 in Windows builds to mitigate CVE-2016-3189 and -CVE-2019-12900 +Update bzip2 to 1.0.8 in Windows builds to mitigate :cve:`2016-3189` and +:cve:`2019-12900`. .. @@ -1482,7 +1482,7 @@ CVE-2019-12900 .. nonce: Ufd4tG .. section: Windows -Prevent CVE-2022-26488 by ensuring the Add to PATH option in the Windows +Prevent :cve:`2022-26488` by ensuring the Add to PATH option in the Windows installer uses the correct path when being repaired. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index a4cdda2cafdb43..a035d0f5addbf2 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -58,10 +58,10 @@ may have prevented Python-to-Python calls respecting PEP 523. .. nonce: -igcjS .. section: Core and Builtins -Add a closure keyword-only parameter to exec(). It can only be specified +Add a closure keyword-only parameter to :func:`exec()`. It can only be specified when exec-ing a code object that uses free variables. When specified, it must be a tuple, with exactly the number of cell variables referenced by the -code object. closure has a default value of None, and it must be None if the +code object. closure has a default value of ``None``, and it must be ``None`` if the code object doesn't refer to any free variables. .. @@ -285,7 +285,7 @@ macros. .. nonce: 11YXHQ .. section: Core and Builtins -Add a new :c:func:`_PyFrame_IsEntryFrame` API function, to check if a +Add a new :c:func:`!_PyFrame_IsEntryFrame` API function, to check if a :c:type:`PyFrameObject` is an entry frame. Patch by Pablo Galindo. .. @@ -403,8 +403,8 @@ so this led to crashes. The problem is now fixed. .. nonce: 6S_uoU .. section: Core and Builtins -Make opcodes :opcode:`JUMP_IF_TRUE_OR_POP` and -:opcode:`JUMP_IF_FALSE_OR_POP` relative rather than absolute. +Make opcodes :opcode:`!JUMP_IF_TRUE_OR_POP` and +:opcode:`!JUMP_IF_FALSE_OR_POP` relative rather than absolute. .. @@ -437,7 +437,7 @@ cache when applicable. .. section: Core and Builtins Classes and functions that unconditionally declared their docstrings -ignoring the `--without-doc-strings` compilation flag no longer do so. +ignoring the ``--without-doc-strings`` compilation flag no longer do so. The classes affected are :class:`ctypes.UnionType`, :class:`pickle.PickleBuffer`, :class:`testcapi.RecursingInfinitelyError`, @@ -576,7 +576,7 @@ planned). Patch by Alex Waygood. .. section: Library Deprecate nested classes in enum definitions becoming members -- in 3.13 -they will be normal classes; add `member` and `nonmember` functions to allow +they will be normal classes; add ``member`` and ``nonmember`` functions to allow control over results now. .. @@ -664,8 +664,9 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For @dataclass, add weakref_slot. Default is False. If True, and if -slots=True, add a slot named "__weakref__", which will allow instances to be +For :func:`@dataclass `, add *weakref_slot*. +The new parameter defaults to ``False``. If true, and if +``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith .. @@ -785,7 +786,7 @@ avoid :exc:`BrokenPipeError` at garbage collection and at .. nonce: V0YveU .. section: Library -Add `datetime.UTC` alias for `datetime.timezone.utc`. +Add ``datetime.UTC`` alias for ``datetime.timezone.utc``. Patch by Kabir Kwatra. @@ -1657,9 +1658,9 @@ Convert :mod:`csv` to use Argument Clinic for :func:`csv.field_size_limit`, .. nonce: z2WhDQ .. section: Library -Raise an ArgumentError when the same subparser name is added twice to an -`argparse.ArgumentParser`. This is consistent with the (default) behavior -when the same option string is added twice to an ArgumentParser. +Raise an ``ArgumentError`` when the same subparser name is added twice to an +``argparse.ArgumentParser``. This is consistent with the (default) behavior +when the same option string is added twice to an ``ArgumentParser``. .. @@ -1712,7 +1713,7 @@ Aviv Palivoda and Erlend E. Aasland. .. nonce: kTjJLx .. section: Documentation -Add a new `gh` role to the documentation to link to GitHub issues. +Add a new ``gh`` role to the documentation to link to GitHub issues. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 633738de92bef7..77a34124fb39e6 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -29,8 +29,7 @@ process. This was a potential privilege escalation. Filesystem based socket permissions restrict this to the *forkserver* process user as was the default in Python 3.8 and earlier. -This prevents Linux `CVE-2022-42919 -`_. +This prevents Linux :cve:`2022-42919`. .. @@ -103,7 +102,7 @@ well as generator expressions. .. section: Core and Builtins Added unicode check for ``name`` attribute of ``spec`` argument passed in -:func:`_imp.create_builtin` function. +:func:`!_imp.create_builtin` function. .. @@ -484,7 +483,7 @@ Fix case of undefined behavior in ceval.c .. nonce: AfCi36 .. section: Core and Builtins -Convert :mod:`_functools` to argument clinic. +Convert :mod:`!_functools` to argument clinic. .. @@ -493,7 +492,7 @@ Convert :mod:`_functools` to argument clinic. .. nonce: wky0Fc .. section: Core and Builtins -Do not expose ``KeyWrapper`` in :mod:`_functools`. +Do not expose ``KeyWrapper`` in :mod:`!_functools`. .. @@ -561,7 +560,7 @@ versions prior to 3.11 .. nonce: 9lmTCC .. section: Core and Builtins -Remove two cases of undefined behavoir, by adding NULL checks. +Remove two cases of undefined behavior, by adding NULL checks. .. @@ -1462,7 +1461,7 @@ expression, but there's no trailing brace. For example, f"{i=". .. nonce: Jf6gAj .. section: Core and Builtins -Cache the result of :c:func:`PyCode_GetCode` function to restore the O(1) +Cache the result of :c:func:`PyCode_GetCode` function to restore the *O*\ (1) lookup of the :attr:`~types.CodeType.co_code` attribute. .. @@ -1732,7 +1731,7 @@ tracing functions implemented in C. .. nonce: lenv9h .. section: Core and Builtins -:meth:`_warnings.warn_explicit` is ported to Argument Clinic. +:meth:`!_warnings.warn_explicit` is ported to Argument Clinic. .. @@ -1913,7 +1912,7 @@ Stinner. .. nonce: Uxc9al .. section: Library -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +Allow :mod:`venv` to pass along :envvar:`!PYTHON*` variables to ``ensurepip`` and ``pip`` when they do not impact path resolution .. @@ -2592,7 +2591,7 @@ Update bundled pip to 22.2.2. Fix :class:`asyncio.TaskGroup` to propagate exception when :exc:`asyncio.CancelledError` was replaced with another exception by a -context manger. Patch by Kumar Aditya and Guido van Rossum. +context manager. Patch by Kumar Aditya and Guido van Rossum. .. @@ -2722,7 +2721,7 @@ on future on an error - e.g. TimeoutError or KeyboardInterrupt. Fix a :mod:`sqlite3` regression where ``*args`` and ``**kwds`` were incorrectly relayed from :py:func:`~sqlite3.connect` to the :class:`~sqlite3.Connection` factory. The regression was introduced in -3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland.` +3.11a1 with PR 24421 (:gh:`85128`). Patch by Erlend E. Aasland. .. @@ -2988,7 +2987,7 @@ Kumar Aditya. .. section: Library Fix crash in :class:`struct.Struct` when it was not completely initialized -by initializing it in :meth:`~object.__new__``. Patch by Kumar Aditya. +by initializing it in :meth:`~object.__new__`. Patch by Kumar Aditya. .. @@ -3143,8 +3142,8 @@ test.test_codecs.EncodedFileTest`` instead. .. nonce: VhS1eS .. section: Library -Made :class:`_struct.Struct` GC-tracked in order to fix a reference leak in -the :mod:`_struct` module. +Made :class:`!_struct.Struct` GC-tracked in order to fix a reference leak in +the :mod:`!_struct` module. .. @@ -3200,9 +3199,8 @@ Remove the :func:`ssl.wrap_socket` function, deprecated in Python 3.7: instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`ssl.wrap_socket` is broken and insecure. The function neither sends a -SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 -`_: Improper Certificate -Validation. Patch by Victor Stinner. +SNI TLS extension nor validates server hostname. Code is subject to :cwe:`295` +Improper Certificate Validation. Patch by Victor Stinner. .. @@ -3260,7 +3258,7 @@ on the main thread Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) -function is a built-in function. Since Python 3.10, :func:`_pyio.open` is +function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is also a static method. Patch by Victor Stinner. .. @@ -3500,7 +3498,7 @@ Illia Volochii. .. nonce: tjfu9L .. section: Library -Deprecate :data:`sqlite3.version` and :data:`sqlite3.version_info`. +Deprecate :data:`!version` and :data:`!version_info`. .. @@ -3563,8 +3561,8 @@ with :func:`os.pidfd_open` in non-blocking mode. Patch by Kumar Aditya. .. nonce: mkYl5q .. section: Library -Implement Enum __contains__ that returns True or False to replace the -deprecated behaviour that would sometimes raise a TypeError. +Implement ``Enum.__contains__`` that returns ``True`` or ``False`` to replace the +deprecated behaviour that would sometimes raise a :exc:`TypeError`. .. @@ -3617,7 +3615,7 @@ allow access to handlers by name. .. nonce: uw6x5z .. section: Library -The :mod:`smtpd` module was removed per the schedule in :pep:`594`. +The :mod:`!smtpd` module was removed per the schedule in :pep:`594`. .. @@ -3731,7 +3729,7 @@ In a very special case, the email package tried to append the nonexistent .. nonce: e6uKxj .. section: Library -Fix :func:`ast.unparse` when ``ImportFrom.level`` is None +Fix :func:`ast.unparse` when ``ImportFrom.level`` is ``None`` .. @@ -3793,7 +3791,7 @@ the :c:type:`time_t` type in C. .. section: Library Fixed crash resulting from calling bisect.insort() or bisect.insort_left() -with the key argument not equal to None. +with the key argument not equal to ``None``. .. @@ -4082,7 +4080,7 @@ replacement strings containing group references by 2--3 times. .. section: Library Fix findtext in the xml module to only give an empty string when the text -attribute is set to None. +attribute is set to ``None``. .. @@ -4404,8 +4402,7 @@ Remove extra row .. section: Documentation Deprecated tools ``make suspicious`` and ``rstlint.py`` are now removed. -They have been replaced by `spinx-lint -`_. +They have been replaced by :pypi:`sphinx-lint`. .. @@ -4479,7 +4476,7 @@ they are deprecated. Contributed by C.A.M. Gerlach. .. nonce: we7AFm .. section: Documentation -Replaced incorrectly written true/false values in documentiation. Patch by +Replaced incorrectly written true/false values in documentation. Patch by Robert O'Shea .. @@ -5613,7 +5610,7 @@ Accept os.PathLike for the argument to winsound.PlaySound Support native Windows case-insensitive path comparisons by using ``LCMapStringEx`` instead of :func:`str.lower` in :func:`ntpath.normcase`. -Add ``LCMapStringEx`` to the :mod:`_winapi` module. +Add ``LCMapStringEx`` to the :mod:`!_winapi` module. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 1a04ed473f329d..3626f8b1e20809 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -527,7 +527,7 @@ Stinner. .. nonce: Ai2KDh .. section: Library -Now :mod:`_pyio` is consistent with :mod:`_io` in raising ``ValueError`` +Now :mod:`!_pyio` is consistent with :mod:`!_io` in raising ``ValueError`` when executing methods over closed buffers. .. @@ -537,7 +537,7 @@ when executing methods over closed buffers. .. nonce: 0v8iyw .. section: Library -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` +Clean up refleak on failed module initialisation in :mod:`!_zoneinfo` .. @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`!_pickle` .. @@ -555,7 +555,7 @@ Clean up refleaks on failed module initialisation in :mod:`_pickle` .. nonce: LBl79O .. section: Library -Clean up refleak on failed module initialisation in :mod:`_io`. +Clean up refleak on failed module initialisation in :mod:`!_io`. .. @@ -584,8 +584,8 @@ Use the frame bound builtins when offering a name suggestion in .. nonce: qtm-9T .. section: Library -In :mod:`importlib._bootstrap`, enhance namespace package repr to ``. +In :mod:`importlib._bootstrap`, enhance namespace package repr to ````. .. @@ -695,7 +695,7 @@ Make sure ``patch.dict()`` can be applied on async functions. .. nonce: jUpzF3 .. section: Library -Remove modules :mod:`asyncore` and :mod:`asynchat`, which were deprecated by +Remove modules :mod:`!asyncore` and :mod:`!asynchat`, which were deprecated by :pep:`594`. .. @@ -822,7 +822,7 @@ and to indicate when it became late-bound. .. nonce: 7KinCV .. section: Tests -The Python test suite now fails wit exit code 4 if no tests ran. It should +The Python test suite now fails with exit code 4 if no tests ran. It should help detecting typos in test names and test methods. .. @@ -968,7 +968,7 @@ if :option:`--with-system-expat` is passed to :program:`configure`. .. nonce: 0f6e_N .. section: Windows -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +Update Windows builds to zlib v1.2.13. v1.2.12 has :cve:`2022-37434`, but the vulnerable ``inflateGetHeader`` API is not used by Python. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index ce128fd5f80c77..f6a4dc75d456f4 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -70,7 +70,7 @@ Fix bug where compiler crashes on an if expression with an empty body block. .. nonce: DcKoBJ .. section: Core and Builtins -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +Fix a reference bug in :func:`!_imp.create_builtin` after the creation of the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor Stinner. @@ -82,7 +82,7 @@ Victor Stinner. .. section: Core and Builtins Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the +line missing the newline character from a file that is as long as the available tokenizer buffer. Patch by Pablo galindo .. @@ -496,7 +496,7 @@ Created packages from zipfile and test_zipfile modules, separating Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would -return True from this method; now they correctly return False. +return ``True`` from this method; now they correctly return ``False``. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 75246f3f13503e..53e1688b802bae 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -13,8 +13,8 @@ Fix misleading default value in :func:`input`'s ``__text_signature__``. .. nonce: cmGwxv .. section: Core and Builtins -Remove :opcode:`UNARY_POSITIVE`, :opcode:`ASYNC_GEN_WRAP` and -:opcode:`LIST_TO_TUPLE`, replacing them with intrinsics. +Remove :opcode:`!UNARY_POSITIVE`, :opcode:`!ASYNC_GEN_WRAP` and +:opcode:`!LIST_TO_TUPLE`, replacing them with intrinsics. .. @@ -65,8 +65,8 @@ redundant. .. nonce: M2n6Kg .. section: Core and Builtins -Fix :func:`int.__sizeof__` calculation to include the 1 element ob_digit -array for 0 and False. +Fix :func:`int.__sizeof__` calculation to include the 1-element ``ob_digit`` +array for ``0`` and ``False``. .. @@ -147,8 +147,8 @@ clinic. .. nonce: yRWQ1y .. section: Core and Builtins -Improve the output of ``co_lines`` by emitting only one entry for each line -range. +Improve the output of :meth:`codeobject.co_lines` by emitting only one entry +for each line range. .. @@ -241,7 +241,7 @@ are now always dumped, even if switched off. Improve ``BUILD_LIST`` opcode so that it works similarly to the ``BUILD_TUPLE`` opcode, by stealing references from the stack rather than repeatedly using stack operations to set list elements. Implementation -details are in a new private API :c:func:`_PyList_FromArraySteal`. +details are in a new private API :c:func:`!_PyList_FromArraySteal`. .. @@ -830,7 +830,7 @@ Reduced the memory usage of :func:`urllib.parse.unquote` and .. section: Library ``inspect.signature`` was raising ``TypeError`` on call with mock objects. -Now it correctly returns ``(*args, **kwargs)`` as infered signature. +Now it correctly returns ``(*args, **kwargs)`` as inferred signature. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 8cf90b0e9cde46..effda2be6fd26c 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -506,7 +506,7 @@ inheritance. .. nonce: 7sQz5l .. section: Build -Update BOLT configration not to use depreacted usage of ``--split +Update BOLT configuration not to use deprecated usage of ``--split functions``. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index 5bd600cd8b6fc0..382dae33fcaee1 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -15,7 +15,7 @@ from the HACL* project. .. section: Security Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +to 1.1.1t to address :cve:`2023-0286`, :cve:`2022-4303`, and :cve:`2022-4303` per `the OpenSSL 2023-02-07 security advisory `_. @@ -170,7 +170,7 @@ all as not all platform C libraries generate an error. .. section: Core and Builtins Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. +:opcode:`!PREP_RERAISE_STAR`. .. @@ -453,7 +453,7 @@ E. Aasland. .. section: Library Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. +formatting instead of pairs of keys and values. .. diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index 1ef81747558857..a859be8a047456 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -24,7 +24,7 @@ Reduce the number of inline :opcode:`CACHE` entries for .. nonce: PRkGca .. section: Core and Builtins -Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP` +Removed :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` instructions. .. @@ -429,7 +429,7 @@ an awaitable object. Patch by Kumar Aditya. Speed up setting or deleting mutable attributes on non-dataclass subclasses of frozen dataclasses. Due to the implementation of ``__setattr__`` and ``__delattr__`` for frozen dataclasses, this previously had a time -complexity of ``O(n)``. It now has a time complexity of ``O(1)``. +complexity of *O*\ (*n*). It now has a time complexity of *O*\ (1). .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 0944dfd0e90ab9..7126e08a20c7fd 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -37,7 +37,7 @@ or lacks SHA3. :func:`urllib.parse.urlsplit` now strips leading C0 control and space characters following the specification for URLs defined by WHATWG in -response to CVE-2023-24329. Patch by Illia Volochii. +response to :cve:`2023-24329`. Patch by Illia Volochii. .. @@ -395,7 +395,7 @@ Fix bug in line numbers of instructions emitted for :keyword:`except* .. section: Core and Builtins Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying -which literal is problematic, since comparisons using ``is`` with e.g. None +which literal is problematic, since comparisons using ``is`` with e.g. ``None`` and bool literals are idiomatic. .. @@ -563,10 +563,10 @@ Complex function calls are now faster and consume no C stack space. .. nonce: fvgsCl .. section: Core and Builtins -``len()`` for 0-dimensional :class:`memoryview`` objects (such as +``len()`` for 0-dimensional :class:`memoryview` objects (such as ``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` -raising an :exc:`IndexError``. +raising an :exc:`IndexError`. .. @@ -880,7 +880,7 @@ Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. .. section: Library Make :mod:`dis` display the names of the args for -:opcode:`CALL_INTRINSIC_*`. +:opcode:`!CALL_INTRINSIC_*`. .. @@ -1008,7 +1008,7 @@ Update the bundled copy of pip to version 23.1.2. .. nonce: pst8iT .. section: Library -Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. +Make :mod:`dis` display the value of oparg of :opcode:`!KW_NAMES`. .. @@ -1446,7 +1446,7 @@ Adapt the :mod:`winsound` extension module to :pep:`687`. .. nonce: jurMzv .. section: Library -Remove deprecation of enum ``memmber.member`` access. +Remove deprecation of enum ``member.member`` access. .. @@ -1828,7 +1828,7 @@ is relative. .. nonce: 511Tbh .. section: Library -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. +Convert private :meth:`!_posixsubprocess.fork_exec` to use Argument Clinic. .. @@ -2371,7 +2371,7 @@ Add a new C-API function to eagerly assign a version tag to a PyTypeObject: .. nonce: _paFIF .. section: C API -:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. +:c:macro:`PyObject_GC_Resize` should calculate preheader size if needed. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 102bddcee5c5c2..0ba61b43411792 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -8,9 +8,7 @@ Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. Security issue reported as -`CVE-2023-40217 -`_ by Aapo -Oksman. Patch by Gregory P. Smith. +:cve:`2023-40217` by Aapo Oksman. Patch by Gregory P. Smith. .. @@ -565,7 +563,7 @@ on deallocation. Fix :meth:`multiprocessing.synchronize.SemLock.__setstate__` to properly initialize :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx`. This -fixes a regression when passing a SemLock accross nested processes. +fixes a regression when passing a SemLock across nested processes. Rename :attr:`multiprocessing.synchronize.SemLock.is_fork_ctx` to :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx` to avoid exposing @@ -710,7 +708,7 @@ Fixes crash when tracing in recursive calls to Python classes. .. section: Core and Builtins Remove the ``_PyCFrame`` struct, moving the pointer to the current -intepreter frame back to the threadstate, as it was for 3.10 and earlier. +interpreter frame back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` existed as a performance optimization for tracing. Since PEP 669 has been implemented, this optimization no longer applies. @@ -928,7 +926,7 @@ Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. Add the exception as the third argument to ``PY_UNIND`` callbacks in ``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with -the other exception hanlding callbacks. +the other exception handling callbacks. .. @@ -937,7 +935,7 @@ the other exception hanlding callbacks. .. nonce: DdEwV8 .. section: Core and Builtins -Raise a ``ValueError`` when a monitoring callback funtion returns +Raise a ``ValueError`` when a monitoring callback function returns ``DISABLE`` for events that cannot be disabled locally. .. @@ -1008,7 +1006,7 @@ Add :meth:`dbm.gnu.gdbm.clear` to :mod:`dbm.gnu`. Patch By Donghee Na. .. section: Core and Builtins The ASYNC and AWAIT tokens are removed from the Grammar, which removes the -posibility of making ``async`` and ``await`` soft keywords when using +possibility of making ``async`` and ``await`` soft keywords when using ``feature_version<7`` in :func:`ast.parse`. .. @@ -1030,7 +1028,7 @@ the call is not a classmethod. .. nonce: DdqHFg .. section: Core and Builtins -Python no longer crashes due an infrequent race when initialzing +Python no longer crashes due an infrequent race when initializing per-interpreter interned strings. The crash would manifest when the interpreter was finalized. @@ -1924,7 +1922,7 @@ objects .. nonce: RDGe8- .. section: Library -Deprecation warning about non-integer number in :mod:`gettext` now alwais +Deprecation warning about non-integer number in :mod:`gettext` now always refers to the line in the user code where gettext function or method is used. Previously it could refer to a line in ``gettext`` code. @@ -1966,7 +1964,7 @@ debugging. .. nonce: LCxiau .. section: Library -Fix :func:`termios.tcsetattr` bug that was overwritting existing errors +Fix :func:`termios.tcsetattr` bug that was overwriting existing errors during parsing integers from ``term`` list. .. @@ -2049,7 +2047,7 @@ point. .. nonce: fECxTj .. section: Library -On Windows, multiprocessing ``Popen.terminate()`` now catchs +On Windows, multiprocessing ``Popen.terminate()`` now catches :exc:`PermissionError` and get the process exit code. If the process is still running, raise again the :exc:`PermissionError`. Otherwise, the process terminated as expected: store its exit code. Patch by Victor @@ -2276,7 +2274,7 @@ creation. .. nonce: m2H5Bk .. section: Library -Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure +Remove unnecessary extra ``__slots__`` in :class:`~datetime.datetime`\'s pure python implementation to reduce memory size, as they are defined in the superclass. Patch by James Hilton-Balfe @@ -2859,7 +2857,7 @@ Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` instead of ``SSL_ERROR_SSL`` when a certification verification has failed, but the error parameters will still contain ``ERR_LIB_SSL`` and ``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +raising the appropriate ``ssl.SSLCertVerificationError``. Patch by Pablo Galindo .. @@ -2890,9 +2888,9 @@ documented and were not intended to be used externally. .. nonce: vMbmj_ .. section: Library -:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never +:data:`!opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to -:data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. +:data:`!_opcode.ENABLE_SPECIALIZATION` where tests can access it. .. @@ -2981,7 +2979,7 @@ method. Patch by James Cave. .. section: Library Fix overflow on 32-bit systems with :mod:`asyncio` :func:`os.sendfile` -implemention. +implementation. .. @@ -3055,7 +3053,7 @@ Donghee Na. .. nonce: U9nD_B .. section: Library -Optimize :meth:`_PollLikeSelector.select` for many iteration case. +Optimize :meth:`!_PollLikeSelector.select` for many iteration case. .. @@ -3175,7 +3173,7 @@ Disable tab completion in multiline mode of :mod:`pdb` .. nonce: pYSwMj .. section: Library -Expose opcode metadata through :mod:`_opcode`. +Expose opcode metadata through :mod:`!_opcode`. .. @@ -3253,7 +3251,7 @@ Eliseev. .. nonce: NN35-U .. section: Library -Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. +Optimize ``(?!)`` (pattern which always fails) in regular expressions. .. @@ -3438,7 +3436,8 @@ added support for this decorator. Patch by Alex Waygood. .. nonce: C1ahtk .. section: Library -Make pydoc.doc catch bad module ImportError when output stream is not None. +Make :func:`pydoc.doc` catch bad module :exc:`ImportError` +when output stream is not ``None``. .. @@ -3496,7 +3495,7 @@ star imports. .. nonce: TJEUkd .. section: Library -Zipapp will now skip over apending an archive to itself. +Zipapp will now skip over appending an archive to itself. .. @@ -3736,7 +3735,7 @@ overwritten. .. nonce: _sZilh .. section: Library -Fix bugs in :mod:`_ctypes` where exceptions could end up being overwritten. +Fix bugs in :mod:`!_ctypes` where exceptions could end up being overwritten. .. @@ -4184,8 +4183,7 @@ Hugo van Kemenade. .. section: Library :pep:`594`: Remove the :mod:`!spwd` module, deprecated in Python 3.11: the -`python-pam project `_ can be used -instead. Patch by Victor Stinner. +:pypi:`python-pam` project can be used instead. Patch by Victor Stinner. .. @@ -4380,7 +4378,7 @@ Patch by Victor Stinner. .. nonce: I6MQhb .. section: Library -:pep:`594`: Remove the :mod:`!cgi`` and :mod:`!cgitb` modules, deprecated in +:pep:`594`: Remove the :mod:`!cgi` and :mod:`!cgitb` modules, deprecated in Python 3.11. Patch by Victor Stinner. .. @@ -4566,7 +4564,7 @@ Deprecate passing any arguments to :func:`threading.RLock`. .. nonce: o5Zb0t .. section: Library -Refactored ``zipfile._strip_extra`` to use higher level abstactions for +Refactored ``zipfile._strip_extra`` to use higher level abstractions for extras instead of a heavy-state loop. .. @@ -5016,7 +5014,7 @@ by Victor Stinner. Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future which may or may not complete depending if it takes longer than the timeout -ot not. Keep the second future which does not complete before wait() +or not. Keep the second future which does not complete before wait() timeout. Patch by Victor Stinner. .. @@ -5106,7 +5104,7 @@ Victor Stinner. regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). -``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +``--slow-ci`` uses a default timeout of 20 minutes and ``-u all`` (run all tests). Patch by Victor Stinner. .. @@ -5234,7 +5232,7 @@ and ``sysctl net.inet.udp.blackhole=1``). Patch by Victor Stinner. Skip ``test_gdb`` if gdb is unable to retrieve Python frame objects: if a frame is ````. When Python is built with "clang -Og", gdb can -fail to retrive the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In +fail to retrieve the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In this case, tests like ``py_bt()`` are likely to fail. Without getting access to Python frames, ``python-gdb.py`` is mostly clueless on retrieving the Python traceback. Moreover, ``test_gdb`` is no longer skipped on macOS if diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst new file mode 100644 index 00000000000000..f4a637bf624d03 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -0,0 +1,1622 @@ +.. date: 2023-11-20-14-13-02 +.. gh-issue: 112243 +.. nonce: FKdQnr +.. release date: 2023-11-22 +.. section: Core and Builtins + +Don't include comments in f-string debug expressions. Patch by Pablo Galindo + +.. + +.. date: 2023-11-20-10-40-40 +.. gh-issue: 112287 +.. nonce: 15gWAK +.. section: Core and Builtins + +Slightly optimize the Tier 2 (uop) interpreter by only loading ``oparg`` and +``operand`` when needed. Also double the trace size limit again, to 512 this +time. + +.. + +.. date: 2023-11-19-15-57-23 +.. gh-issue: 112266 +.. nonce: BSJMbR +.. section: Core and Builtins + +Change docstrings of :attr:`~object.__dict__` and +:attr:`~object.__weakref__`. + +.. + +.. date: 2023-11-17-16-49-32 +.. gh-issue: 111807 +.. nonce: QvjP9_ +.. section: Core and Builtins + +Lower the max parser stack depth to 1000 under WASI debug builds. + +.. + +.. date: 2023-11-15-20-20-51 +.. gh-issue: 111798 +.. nonce: cs-3t3 +.. section: Core and Builtins + +When Python is built in debug mode, set the C recursion limit to 500 instead +of 1500. A debug build is likely built with low optimization level which +implies higher stack memory usage than a release build. Patch by Victor +Stinner. + +.. + +.. date: 2023-11-15-16-14-10 +.. gh-issue: 106529 +.. nonce: Y48ax9 +.. section: Core and Builtins + +Enable translating unspecialized ``FOR_ITER`` to Tier 2. + +.. + +.. date: 2023-11-14-22-12-11 +.. gh-issue: 111916 +.. nonce: ZGCayL +.. section: Core and Builtins + +Make hashlib related modules thread-safe without the GIL + +.. + +.. date: 2023-11-07-12-59-02 +.. gh-issue: 81137 +.. nonce: qFpJCY +.. section: Core and Builtins + +Deprecate assignment to a function's ``__code__`` field when the new code +object is of a mismatched type (e.g., from a generator to a plain function). + +.. + +.. date: 2023-11-06-16-44-09 +.. gh-issue: 79932 +.. nonce: 2qv7uD +.. section: Core and Builtins + +Raise exception if :meth:`frame.clear` is called on a suspended frame. + +.. + +.. date: 2023-11-05-20-59-10 +.. gh-issue: 81925 +.. nonce: wKHLSS +.. section: Core and Builtins + +Implement native thread ids for GNU KFreeBSD. + +.. + +.. date: 2023-11-05-06-40-35 +.. gh-issue: 111843 +.. nonce: c045cB +.. section: Core and Builtins + +Use exponential backoff to reduce the number of failed tier 2 optimization +attempts by over 99%. + +.. + +.. date: 2023-11-04-13-36-51 +.. gh-issue: 110829 +.. nonce: Pa0CJI +.. section: Core and Builtins + +Joining a thread now ensures the underlying OS thread has exited. This is +required for safer fork() in multi-threaded processes. + +.. + +.. date: 2023-11-03-22-48-29 +.. gh-issue: 109369 +.. nonce: ELYaxJ +.. section: Core and Builtins + +Make sure that tier 2 traces are de-optimized if the code is instrumented + +.. + +.. date: 2023-11-03-19-25-38 +.. gh-issue: 111772 +.. nonce: aRQvOn +.. section: Core and Builtins + +Specialize slot loads and stores for _Py_T_OBJECT as well as Py_T_OBJECT_EX + +.. + +.. date: 2023-11-03-01-23-48 +.. gh-issue: 111666 +.. nonce: l8Q8G5 +.. section: Core and Builtins + +Speed up :meth:`BaseExceptionGroup.derive`, +:meth:`BaseExceptionGroup.subgroup`, and :meth:`BaseExceptionGroup.split` by +changing how they parse passed arguments. + +.. + +.. date: 2023-11-03-01-04-55 +.. gh-issue: 111654 +.. nonce: scUhDO +.. section: Core and Builtins + +Fix runtime crash when some error happens in opcode +``LOAD_FROM_DICT_OR_DEREF``. + +.. + +.. date: 2023-11-02-15-00-57 +.. gh-issue: 111623 +.. nonce: BZxYc8 +.. section: Core and Builtins + +Add support for sharing tuples between interpreters using the +cross-interpreter API. Patch by Anthony Shaw. + +.. + +.. date: 2023-11-02-14-49-19 +.. gh-issue: 111354 +.. nonce: gIS3f- +.. section: Core and Builtins + +The oparg of :opcode:`YIELD_VALUE` is now ``1`` if the instruction is part +of a yield-from or await, and ``0`` otherwise. + +The SUSPENDED frame state is now split into ``SUSPENDED`` and +``SUSPENDED_YIELD_FROM``. This simplifies the code in ``_PyGen_yf``. + +.. + +.. date: 2023-10-31-21-33-35 +.. gh-issue: 111520 +.. nonce: vw-rxJ +.. section: Core and Builtins + +Merge the Tier 1 (bytecode) and Tier 2 (micro-ops) interpreters together, +moving the Tier 2 interpreter loop and switch into +``_PyEval_EvalFrameDefault()`` in ``Python/ceval.c``. The +``Python/executor.c`` file is gone. Also the ``TIER_ONE`` and ``TIER_TWO`` +macros are now handled by the code generator. + +**Beware!** This changes the environment variables to enable micro-ops and +their debugging to ``PYTHON_UOPS`` and ``PYTHON_LLTRACE``. + +.. + +.. date: 2023-10-31-14-25-21 +.. gh-issue: 109181 +.. nonce: 11h6Mc +.. section: Core and Builtins + +Speed up :class:`Traceback` object creation by lazily compute the line number. +Patch by Pablo Galindo + +.. + +.. date: 2023-10-29-20-11-21 +.. gh-issue: 111420 +.. nonce: IUT-GK +.. section: Core and Builtins + +Allow type comments in parenthesized ``with`` statements + +.. + +.. date: 2023-10-29-12-33-33 +.. gh-issue: 111438 +.. nonce: bHTLLl +.. section: Core and Builtins + +Add support for sharing floats between interpreters using the +cross-interpreter API. Patch by Anthony Shaw. + +.. + +.. date: 2023-10-29-11-35-21 +.. gh-issue: 111435 +.. nonce: ageUWQ +.. section: Core and Builtins + +Add support for sharing of ``True`` and ``False`` between interpreters using the +cross-interpreter API. Patch by Anthony Shaw. + +.. + +.. date: 2023-10-27-19-38-33 +.. gh-issue: 102388 +.. nonce: vd5YUZ +.. section: Core and Builtins + +Fix a bug where ``iso2022_jp_3`` and ``iso2022_jp_2004`` codecs read out of +bounds + +.. + +.. date: 2023-10-27-12-17-49 +.. gh-issue: 111366 +.. nonce: _TSknV +.. section: Core and Builtins + +Fix an issue in the :mod:`codeop` that was causing :exc:`SyntaxError` +exceptions raised in the presence of invalid syntax to not contain precise +error messages. Patch by Pablo Galindo + +.. + +.. date: 2023-10-27-11-51-40 +.. gh-issue: 111380 +.. nonce: vgSbir +.. section: Core and Builtins + +Fix a bug that was causing :exc:`SyntaxWarning` to appear twice when parsing +if invalid syntax is encountered later. Patch by Pablo galindo + +.. + +.. date: 2023-10-27-11-22-09 +.. gh-issue: 111374 +.. nonce: e9lrPZ +.. section: Core and Builtins + +Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It +determines whether or not frozen modules are ignored by the import +machinery, equivalent of the :option:`-X frozen_modules <-X>` command-line +option. + +.. + +.. date: 2023-10-26-18-45-20 +.. gh-issue: 111354 +.. nonce: GrT-Wf +.. section: Core and Builtins + +Remove ``oparg`` from :opcode:`YIELD_VALUE`. Change ``oparg`` of +:opcode:`RESUME` to include information about the except-depth. These +changes make it possible to simplify the code in generator close. + +.. + +.. date: 2023-10-23-22-11-09 +.. gh-issue: 94438 +.. nonce: y2pITu +.. section: Core and Builtins + +Fix a regression that prevented jumping across ``is None`` and ``is not +None`` when debugging. Patch by Savannah Ostrowski. + +.. + +.. date: 2023-10-23-15-44-47 +.. gh-issue: 67224 +.. nonce: S4D6CR +.. section: Core and Builtins + +Show source lines in tracebacks when using the ``-c`` option when running +Python. Patch by Pablo Galindo + +.. + +.. date: 2023-10-20-23-14-06 +.. gh-issue: 111123 +.. nonce: jjVc3M +.. section: Core and Builtins + +Fix a bug where a :keyword:`global` declaration in an :keyword:`except` +block is rejected when the global is used in the :keyword:`else` block. + +.. + +.. date: 2023-10-17-11-03-45 +.. gh-issue: 110938 +.. nonce: X3sbMb +.. section: Core and Builtins + +Fix error messages for indented blocks with functions and classes with +generic type parameters. Patch by Pablo Galindo + +.. + +.. date: 2023-10-16-15-51-37 +.. gh-issue: 109214 +.. nonce: -RGTFH +.. section: Core and Builtins + +Remove unnecessary instruction pointer updates before returning from frames. + +.. + +.. date: 2023-10-16-12-12-48 +.. gh-issue: 110912 +.. nonce: uEJGi_ +.. section: Core and Builtins + +Correctly display the traceback for :exc:`MemoryError` exceptions using the +:mod:`traceback` module. Patch by Pablo Galindo + +.. + +.. date: 2023-10-15-22-18-45 +.. gh-issue: 109894 +.. nonce: UAmo06 +.. section: Core and Builtins + +Fixed crash due to improperly initialized static :exc:`MemoryError` in +subinterpreter. + +.. + +.. date: 2023-10-15-20-45-35 +.. gh-issue: 110892 +.. nonce: oA6eVY +.. section: Core and Builtins + +Return ``NULL`` for ``PyTrace_RETURN`` events caused by an exception + +.. + +.. date: 2023-10-14-12-19-34 +.. gh-issue: 110864 +.. nonce: -baPDE +.. section: Core and Builtins + +Fix argument parsing by ``_PyArg_UnpackKeywordsWithVararg`` for functions +defining pos-or-keyword, vararg, and kw-only parameters. + +.. + +.. date: 2023-10-13-16-55-55 +.. gh-issue: 109094 +.. nonce: ziL4cJ +.. section: Core and Builtins + +Replace ``prev_instr`` on the interpreter frame by ``instr_ptr`` which +points to the beginning of the instruction that is currently executing (or +will execute once the frame resumes). + +.. + +.. date: 2023-10-13-09-21-29 +.. gh-issue: 110805 +.. nonce: vhU7A7 +.. section: Core and Builtins + +Allow the repl to show source code and complete tracebacks. Patch by Pablo +Galindo + +.. + +.. date: 2023-10-12-17-15-23 +.. gh-issue: 110722 +.. nonce: sjMwQe +.. section: Core and Builtins + +Add :envvar:`PYTHON_PRESITE=package.module` to import a module early in the +interpreter lifecycle before ``site.py`` is executed. Python needs to be +:ref:`built in debug mode ` for this option to exist. + +.. + +.. date: 2023-10-12-12-09-01 +.. gh-issue: 110481 +.. nonce: 3Er3it +.. section: Core and Builtins + +Implement biased reference counting in ``--disable-gil`` builds. + +.. + +.. date: 2023-10-09-19-54-33 +.. gh-issue: 110543 +.. nonce: 1wrxO8 +.. section: Core and Builtins + +Fix regression in Python 3.12 where :meth:`types.CodeType.replace` would +produce a broken code object if called on a module or class code object that +contains a comprehension. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-09-30-17-30-11 +.. gh-issue: 89519 +.. nonce: hz2pZf +.. section: Core and Builtins + +Removed chained :class:`classmethod` descriptors (introduced in +:issue:`19072`). This can no longer be used to wrap other descriptors such +as :class:`property`. The core design of this feature was flawed and caused +a number of downstream problems. To "pass-through" a :class:`classmethod`, +consider using the :attr:`!__wrapped__` attribute that was added in Python +3.10. + +.. + +.. date: 2023-09-15-23-39-43 +.. gh-issue: 103615 +.. nonce: WZavly +.. section: Core and Builtins + +Use local events for opcode tracing + +.. + +.. bpo: 46657 +.. date: 2023-09-06-12-36-11 +.. nonce: xea1T_ +.. section: Core and Builtins + +Add mimalloc memory allocator support. + +.. + +.. date: 2023-08-31-11-42-16 +.. gh-issue: 106718 +.. nonce: _-57DA +.. section: Core and Builtins + +When PyConfig.stdlib_dir is explicitly set, it's now respected and won't be +overridden by PyConfig.home. + +.. + +.. date: 2023-07-20-11-41-16 +.. gh-issue: 106905 +.. nonce: AyZpuB +.. section: Core and Builtins + +Fix incorrect SystemError about AST constructor recursion depth mismatch. + +.. + +.. date: 2022-12-27-02-51-45 +.. gh-issue: 100445 +.. nonce: C8f6ph +.. section: Core and Builtins + +Improve error message for unterminated strings with escapes. + +.. + +.. bpo: 45759 +.. date: 2021-11-10-10-40-05 +.. nonce: WJoB3D +.. section: Core and Builtins + +Improved error messages for ``elif``/``else`` statements not matching any +valid statements. Patch by Jeremiah Vivian. + +.. + +.. date: 2023-11-14-18-43-55 +.. gh-issue: 111942 +.. nonce: x1pnrj +.. section: Library + +Fix SystemError in the TextIOWrapper constructor with non-encodable "errors" +argument in non-debug mode. + +.. + +.. date: 2023-11-14-16-31-59 +.. gh-issue: 111995 +.. nonce: OoX8JJ +.. section: Library + +Added the ``NI_IDN`` constant to the :mod:`socket` module when present in C +at build time for use with :func:`socket.getnameinfo`. + +.. + +.. date: 2023-11-11-16-42-48 +.. gh-issue: 109538 +.. nonce: cMG5ux +.. section: Library + +Issue warning message instead of having :class:`RuntimeError` be displayed +when event loop has already been closed at :meth:`StreamWriter.__del__`. + +.. + +.. date: 2023-11-10-22-08-28 +.. gh-issue: 111942 +.. nonce: MDFm6v +.. section: Library + +Fix crashes in :meth:`io.TextIOWrapper.reconfigure` when pass invalid +arguments, e.g. non-string encoding. + +.. + +.. date: 2023-11-09-12-57-43 +.. gh-issue: 111460 +.. nonce: TQaz9I +.. section: Library + +:mod:`curses`: restore wide character support (including +:func:`curses.unget_wch` and :meth:`~curses.window.get_wch`) on macOS, which +was unavailable due to a regression in Python 3.12. + +.. + +.. date: 2023-11-09-10-45-56 +.. gh-issue: 103791 +.. nonce: sdfkja +.. section: Library + +:class:`contextlib.suppress` now supports suppressing exceptions raised as +part of a :exc:`BaseExceptionGroup`, in addition to the recent support for +:exc:`ExceptionGroup`. + +.. + +.. date: 2023-11-08-23-32-03 +.. gh-issue: 111835 +.. nonce: ufFiuW +.. section: Library + +The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method +that can be used when a seekable file-like object is required. +The :meth:`~mmap.mmap.seek` method now returns the new absolute position. +Patch by Donghee Na. + +.. + +.. date: 2023-11-08-15-58-57 +.. gh-issue: 111804 +.. nonce: uAXTOL +.. section: Library + +Remove posix.fallocate() under WASI as the underlying posix_fallocate() is +not available in WASI preview2. + +.. + +.. date: 2023-11-08-11-50-49 +.. gh-issue: 111841 +.. nonce: iSqdQf +.. section: Library + +Fix truncating arguments on an embedded null character in :meth:`os.putenv` +and :meth:`os.unsetenv` on Windows. + +.. + +.. date: 2023-11-08-07-42-53 +.. gh-issue: 111768 +.. nonce: g-WpnV +.. section: Library + +:func:`wsgiref.util.is_hop_by_hop` is now exposed correctly in ``__all__``. + +.. + +.. date: 2023-11-04-21-12-27 +.. gh-issue: 80731 +.. nonce: Wq51xg +.. section: Library + +Avoid executing the default function in :class:`cmd.Cmd` in an except block + +.. + +.. date: 2023-11-04-10-24-25 +.. gh-issue: 111541 +.. nonce: x0RBI1 +.. section: Library + +Fix :mod:`doctest` for :exc:`SyntaxError` not-builtin subclasses. + +.. + +.. date: 2023-11-04-01-20-23 +.. gh-issue: 111719 +.. nonce: fUiKBD +.. section: Library + +Add extra argument validation for ``alias`` command in :mod:`pdb` + +.. + +.. date: 2023-11-02-12-15-46 +.. gh-issue: 111482 +.. nonce: FWqZIU +.. section: Library + +:mod:`time`: Make :func:`time.clock_gettime()` and +:func:`time.clock_gettime_ns()` functions up to 2x faster by faster calling +convention. Patch by Victor Stinner. + +.. + +.. date: 2023-11-01-14-03-24 +.. gh-issue: 110894 +.. nonce: 7-wZxC +.. section: Library + +Call loop exception handler for exceptions in ``client_connected_cb`` of +:func:`asyncio.start_server` so that applications can handle it. Patch by +Kumar Aditya. + +.. + +.. date: 2023-10-31-07-46-56 +.. gh-issue: 111531 +.. nonce: 6zUV_G +.. section: Library + +Fix reference leaks in ``bind_class()`` and ``bind_all()`` methods of +:mod:`tkinter` widgets. + +.. + +.. date: 2023-10-30-14-47-23 +.. gh-issue: 111246 +.. nonce: QJ_ehs +.. section: Library + +:meth:`asyncio.loop.create_unix_server` will now automatically remove the +Unix socket when the server is closed. + +.. + +.. date: 2023-10-30-08-50-46 +.. gh-issue: 111356 +.. nonce: Bc8LvA +.. section: Library + +Added :func:`io.text_encoding()`, :data:`io.DEFAULT_BUFFER_SIZE`, and +:class:`io.IncrementalNewlineDecoder` to ``io.__all__``. + +.. + +.. date: 2023-10-29-03-46-27 +.. gh-issue: 66425 +.. nonce: FWTdDo +.. section: Library + +Remove the code to set the REMOTE_HOST header from wsgiref module, as it is +unreachable. This header is used for performance reasons, which is not +necessary in the wsgiref module. + +.. + +.. date: 2023-10-28-22-11-11 +.. gh-issue: 111429 +.. nonce: mJGxuQ +.. section: Library + +Speed up :meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. + +.. + +.. date: 2023-10-28-04-21-17 +.. gh-issue: 111342 +.. nonce: m8Ln1k +.. section: Library + +Fixed typo in :func:`math.sumprod`. + +.. + +.. date: 2023-10-27-12-46-56 +.. gh-issue: 68166 +.. nonce: 0EbWW4 +.. section: Library + +Remove mention of not supported "vsapi" element type in +:meth:`tkinter.ttk.Style.element_create`. Add tests for ``element_create()`` +and other ``ttk.Style`` methods. Add examples for ``element_create()`` in +the documentation. + +.. + +.. date: 2023-10-27-09-56-20 +.. gh-issue: 111388 +.. nonce: SlmDbC +.. section: Library + +Add ``show_group`` parameter to :func:`traceback.format_exception_only`, +which allows to format :exc:`ExceptionGroup` instances. + +.. + +.. date: 2023-10-25-11-54-00 +.. gh-issue: 79033 +.. nonce: 5ePgFl +.. section: Library + +Another attempt at fixing :func:`asyncio.Server.wait_closed()`. It now +blocks until both conditions are true: the server is closed, *and* there are +no more active connections. (This means that in some cases where in 3.12.0 +this function would *incorrectly* have returned immediately, it will now +block; in particular, when there are no active connections but the server +hasn't been closed yet.) + +.. + +.. date: 2023-10-25-11-13-35 +.. gh-issue: 111259 +.. nonce: z7ndeA +.. section: Library + +Optimize recursive wildcards in :mod:`pathlib`. + +.. + +.. date: 2023-10-25-08-42-05 +.. gh-issue: 111295 +.. nonce: H2K4lf +.. section: Library + +Fix :mod:`time` not checking for errors when initializing. + +.. + +.. date: 2023-10-24-12-20-46 +.. gh-issue: 111253 +.. nonce: HFywSK +.. section: Library + +Add error checking during :mod:`!_socket` module init. + +.. + +.. date: 2023-10-24-12-09-46 +.. gh-issue: 111251 +.. nonce: urFYtn +.. section: Library + +Fix :mod:`!_blake2` not checking for errors when initializing. + +.. + +.. date: 2023-10-23-23-14-54 +.. gh-issue: 111233 +.. nonce: sCdCC0 +.. section: Library + +Fix :mod:`select` not checking for errors when initializing. + +.. + +.. date: 2023-10-23-22-40-47 +.. gh-issue: 111230 +.. nonce: k3Jm84 +.. section: Library + +Fix :mod:`ssl` not checking for errors when initializing. + +.. + +.. date: 2023-10-23-13-53-58 +.. gh-issue: 111174 +.. nonce: Oohmzd +.. section: Library + +Fix crash in :meth:`io.BytesIO.getbuffer` called repeatedly for empty +BytesIO. + +.. + +.. date: 2023-10-22-21-28-05 +.. gh-issue: 111187 +.. nonce: _W11Ab +.. section: Library + +Postpone removal version for locale.getdefaultlocale() to Python 3.15. + +.. + +.. date: 2023-10-21-13-57-06 +.. gh-issue: 111159 +.. nonce: GoHp7s +.. section: Library + +Fix :mod:`doctest` output comparison for exceptions with notes. + +.. + +.. date: 2023-10-20-15-29-10 +.. gh-issue: 110910 +.. nonce: u2oPwX +.. section: Library + +Fix invalid state handling in :class:`asyncio.TaskGroup` and +:class:`asyncio.Timeout`. They now raise proper RuntimeError if they are +improperly used and are left in consistent state after this. + +.. + +.. date: 2023-10-19-22-46-34 +.. gh-issue: 111092 +.. nonce: hgut12 +.. section: Library + +Make turtledemo run without default root enabled. + +.. + +.. date: 2023-10-16-18-41-51 +.. gh-issue: 110944 +.. nonce: CmUKXo +.. section: Library + +Support alias and convenience vars for :mod:`pdb` completion + +.. + +.. date: 2023-10-15-08-08-26 +.. gh-issue: 110745 +.. nonce: mxEkh0 +.. section: Library + +Added *newline* parameter to :meth:`pathlib.Path.read_text`. Patch by Junya +Okabe. + +.. + +.. date: 2023-10-14-21-33-57 +.. gh-issue: 84583 +.. nonce: -Cmn4_ +.. section: Library + +Make :mod:`pdb` enter post-mortem mode even for :exc:`SyntaxError` + +.. + +.. date: 2023-10-14-20-15-53 +.. gh-issue: 80675 +.. nonce: _M-cQC +.. section: Library + +Set ``f_trace_lines = True`` on all frames upon :func:`pdb.set_trace()` + +.. + +.. date: 2023-10-13-06-47-20 +.. gh-issue: 110771 +.. nonce: opwdlc +.. section: Library + +Expose the setup and cleanup portions of ``asyncio.run_forever()`` as the +standalone methods ``asyncio.run_forever_setup()`` and +``asyncio.run_forever_cleanup()``. This allows for tighter integration with +GUI event loops. + +.. + +.. date: 2023-10-12-15-16-44 +.. gh-issue: 110774 +.. nonce: AdCb5A +.. section: Library + +Support setting the :class:`asyncio.Runner` loop_factory kwarg in +:class:`unittest.IsolatedAsyncioTestCase` + +.. + +.. date: 2023-10-10-17-56-41 +.. gh-issue: 110392 +.. nonce: 6g6CnP +.. section: Library + +Fix :func:`tty.setraw` and :func:`tty.setcbreak`: previously they returned +partially modified list of the original tty attributes. +:func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` now make a copy of the +list of special characters before modifying it. + +.. + +.. date: 2023-10-09-23-59-04 +.. gh-issue: 59013 +.. nonce: qPbS-G +.. section: Library + +Make line number of function breakpoint more precise in :mod:`pdb` + +.. + +.. date: 2023-10-08-18-38-09 +.. gh-issue: 88434 +.. nonce: 2Q_IkG +.. section: Library + +Emit deprecation warning for non-integer numbers in :mod:`gettext` functions +and methods that consider plural forms even if the translation was not +found. + +.. + +.. date: 2023-10-08-14-17-06 +.. gh-issue: 110395 +.. nonce: _tdCsV +.. section: Library + +Ensure that :func:`select.kqueue` objects correctly appear as closed in +forked children, to prevent operations on an invalid file descriptor. + +.. + +.. date: 2023-10-02-05-23-27 +.. gh-issue: 110196 +.. nonce: djwt0z +.. section: Library + +Add ``__reduce__`` method to :class:`IPv6Address` in order to keep +``scope_id`` + +.. + +.. date: 2023-09-25-20-05-41 +.. gh-issue: 109747 +.. nonce: _cRJH8 +.. section: Library + +Improve errors for unsupported look-behind patterns. Now re.error is raised +instead of OverflowError or RuntimeError for too large width of look-behind +pattern. + +.. + +.. date: 2023-09-15-12-30-21 +.. gh-issue: 109466 +.. nonce: 6ah-aw +.. section: Library + +Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the +IPv4-mapped IPv6 address. + +.. + +.. date: 2023-09-08-12-10-10 +.. gh-issue: 85098 +.. nonce: DfQbeJ +.. section: Library + +Implement the CLI of the :mod:`symtable` module and improve the repr of +:class:`~symtable.Symbol`. + +.. + +.. date: 2023-09-02-16-07-23 +.. gh-issue: 108791 +.. nonce: fBcAqh +.. section: Library + +Improved error handling in :mod:`pdb` command line interface, making it +produce more concise error messages. + +.. + +.. date: 2023-08-30-19-10-35 +.. gh-issue: 105931 +.. nonce: Lpwve8 +.. section: Library + +Change :mod:`compileall` to only strip the stripdir prefix from the full +path recorded in the compiled ``.pyc`` file, when the prefix matches the +start of the full path in its entirety. When the prefix does not match, no +stripping is performed and a warning to this effect is displayed. + +Previously all path components of the stripdir prefix that matched the full +path were removed, while those that did not match were left alone (including +ones interspersed between matching components). + +.. + +.. date: 2023-07-29-19-00-39 +.. gh-issue: 107431 +.. nonce: 1GzJ2p +.. section: Library + +Make the ``DictProxy`` and ``ListProxy`` types in +:mod:`multiprocessing.managers` :ref:`Generic Alias +Types` for ``[]`` use in typing contexts. + +.. + +.. date: 2023-07-13-00-24-52 +.. gh-issue: 72904 +.. nonce: Yn5-j0 +.. section: Library + +Add :func:`glob.translate`. This function converts a pathname with +shell-style wildcards to a regular expression. + +.. + +.. date: 2023-05-30-02-01-14 +.. gh-issue: 90026 +.. nonce: FyCXw8 +.. section: Library + +Define ``USE_XATTRS`` on Cygwin so that XATTR-related functions in the +:mod:`os` module become available. + +.. + +.. date: 2023-04-26-16-37-00 +.. gh-issue: 90890 +.. nonce: fIag4w +.. section: Library + +New methods :meth:`mailbox.Maildir.get_info`, +:meth:`mailbox.Maildir.set_info`, :meth:`mailbox.Maildir.get_flags`, +:meth:`mailbox.Maildir.set_flags`, :meth:`mailbox.Maildir.add_flag`, +:meth:`mailbox.Maildir.remove_flag`. These methods speed up accessing a +message's info and/or flags and are useful when it is not necessary to +access the message's contents, as when iterating over a Maildir to find +messages with specific flags. + +.. + +.. date: 2023-04-15-14-45-21 +.. gh-issue: 102956 +.. nonce: Z6qeUy +.. section: Library + +Fix returning of empty byte strings after seek in zipfile module + +.. + +.. date: 2023-03-22-02-01-30 +.. gh-issue: 102895 +.. nonce: HiEqaZ +.. section: Library + +Added a parameter ``local_exit`` for :func:`code.interact` to prevent +``exit()`` and ``quit`` from closing ``sys.stdin`` and raise ``SystemExit``. + +.. + +.. date: 2022-10-14-21-11-10 +.. gh-issue: 97928 +.. nonce: Pdxh1G +.. section: Library + +Change the behavior of :meth:`tkinter.Text.count`. It now always returns an +integer if one or less counting options are specified. Previously it could +return a single count as a 1-tuple, an integer (only if option ``"update"`` +was specified) or ``None`` if no items found. The result is now the same if +``wantobjects`` is set to ``0``. + +.. + +.. date: 2022-10-05-15-01-36 +.. gh-issue: 96954 +.. nonce: ezwkrU +.. section: Library + +Switch the storage of the unicode codepoint names to use a different +data-structure, a `directed acyclic word graph +`_. +This makes the unicodedata shared library about 440 KiB smaller. Contributed +by Carl Friedrich Bolz-Tereick using code from the PyPy project. + +.. + +.. date: 2022-05-28-20-55-07 +.. gh-issue: 73561 +.. nonce: YRmAvy +.. section: Library + +Omit the interface scope from an IPv6 address when used as Host header by +:mod:`http.client`. + +.. + +.. date: 2022-05-06-15-49-57 +.. gh-issue: 86826 +.. nonce: rf006W +.. section: Library + +:mod:`zipinfo` now supports the full range of values in the TZ string +determined by RFC 8536 and detects all invalid formats. Both Python and C +implementations now raise exceptions of the same type on invalid data. + +.. + +.. date: 2023-11-17-15-20-41 +.. gh-issue: 111808 +.. nonce: jtIayt +.. section: Tests + +Make the default value of ``test.support.infinite_recursion()`` to be +conditional based on whether optimizations were used when compiling the +interpreter. This helps with platforms like WASI whose stack size is greatly +restricted in debug builds. + +.. + +.. date: 2023-11-03-18-59-13 +.. gh-issue: 110722 +.. nonce: jvT1pb +.. section: Tests + +Gathering line coverage of standard libraries within the regression test +suite is now precise, as well as much faster. Patch by Åukasz Langa. + +.. + +.. date: 2023-10-31-22-09-25 +.. gh-issue: 110367 +.. nonce: UhQi44 +.. section: Tests + +Make regrtest ``--verbose3`` option compatible with ``--huntrleaks -jN`` +options. The ``./python -m test -j1 -R 3:3 --verbose3`` command now works as +expected. Patch by Victor Stinner. + +.. + +.. date: 2023-10-21-19-27-36 +.. gh-issue: 111165 +.. nonce: FU6mUk +.. section: Tests + +Remove no longer used functions ``run_unittest()`` and ``run_doctest()`` +from the :mod:`test.support` module. + +.. + +.. date: 2023-10-21-00-10-36 +.. gh-issue: 110932 +.. nonce: jktjJU +.. section: Tests + +Fix regrtest if the ``SOURCE_DATE_EPOCH`` environment variable is defined: +use the variable value as the random seed. Patch by Victor Stinner. + +.. + +.. date: 2023-10-17-17-54-36 +.. gh-issue: 110995 +.. nonce: Fx8KRD +.. section: Tests + +test_gdb: Fix detection of gdb built without Python scripting support. Patch +by Victor Stinner. + +.. + +.. date: 2023-10-16-13-47-24 +.. gh-issue: 110918 +.. nonce: aFgZK3 +.. section: Tests + +Test case matching patterns specified by options ``--match``, ``--ignore``, +``--matchfile`` and ``--ignorefile`` are now tested in the order of +specification, and the last match determines whether the test case be run or +ignored. + +.. + +.. date: 2023-09-15-15-00-14 +.. gh-issue: 108747 +.. nonce: ql0owS +.. section: Tests + +Add unit test for ``usercustomize`` and ``sitecustomize`` hooks from +:class:`site`. + +.. + +.. date: 2023-11-15-16-56-20 +.. gh-issue: 96954 +.. nonce: 6FYvKn +.. section: Build + +Make ``make regen-unicodedata`` work for out-of-tree builds of CPython. + +.. + +.. date: 2023-11-15-13-40-29 +.. gh-issue: 112088 +.. nonce: UJQxxh +.. section: Build + +Add ``Tools/build/regen-configure.sh`` script to regenerate the +``configure`` with an Ubuntu container image. The +``quay.io/tiran/cpython_autoconf:271`` container image +(`tiran/cpython_autoconf `_) is +no longer used. Patch by Victor Stinner. + +.. + +.. date: 2023-10-20-15-29-31 +.. gh-issue: 111046 +.. nonce: 2DxQl8 +.. section: Build + +For wasi-threads, memory is now exported to fix compatibility issues with +some wasm runtimes. + +.. + +.. date: 2023-10-17-03-10-40 +.. gh-issue: 110828 +.. nonce: 31vQ9B +.. section: Build + +AIX 32bit needs ``-latomic`` to build the :mod:`!_testcapi` extension +module. + +.. + +.. date: 2023-10-17-01-56-11 +.. gh-issue: 85283 +.. nonce: V156T2 +.. section: Build + +The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, +``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, +``_testimportmultiple`` and ``_uuid`` C extensions are now built with the +:ref:`limited C API `. Patch by Victor Stinner. + +.. + +.. date: 2023-11-13-22-35-27 +.. gh-issue: 111856 +.. nonce: vEtA5z +.. section: Windows + +Fixes :func:`~os.fstat` on file systems that do not support file ID +requests. This includes FAT32 and exFAT. + +.. + +.. date: 2023-10-25-05-01-28 +.. gh-issue: 111293 +.. nonce: FSsLT6 +.. section: Windows + +Fix :data:`os.DirEntry.inode` dropping higher 64 bits of a file id on some +filesystems on Windows. + +.. + +.. date: 2023-10-19-21-46-18 +.. gh-issue: 110913 +.. nonce: CWlPfg +.. section: Windows + +WindowsConsoleIO now correctly chunks large buffers without splitting up +UTF-8 sequences. + +.. + +.. date: 2023-10-31-22-13-05 +.. gh-issue: 59703 +.. nonce: SML6Ag +.. section: macOS + +For macOS framework builds, in ``getpath.c`` use the system ``dladdr`` +function to find the path to the shared library rather than depending on +deprecated macOS APIs. + +.. + +.. date: 2023-10-18-17-26-36 +.. gh-issue: 110950 +.. nonce: sonoma +.. section: macOS + +Update macOS installer to include an upstream Tcl/Tk fix for the ``Secure +coding is not enabled for restorable state!`` warning encountered in Tkinter +on macOS 14 Sonoma. + +.. + +.. date: 2023-10-18-01-40-36 +.. gh-issue: 111015 +.. nonce: NaLI2L +.. section: macOS + +Ensure that IDLE.app and Python Launcher.app are installed with appropriate +permissions on macOS builds. + +.. + +.. date: 2023-09-02-08-49-57 +.. gh-issue: 71383 +.. nonce: Ttkchg +.. section: macOS + +Update macOS installer to include an upstream Tcl/Tk fix for the +``ttk::ThemeChanged`` error encountered in Tkinter. + +.. + +.. date: 2023-08-30-16-33-57 +.. gh-issue: 92603 +.. nonce: ATkKVO +.. section: macOS + +Update macOS installer to include a fix accepted by upstream Tcl/Tk for a +crash encountered after the first :meth:`tkinter.Tk` instance is destroyed. + +.. + +.. bpo: 35668 +.. date: 2019-01-07-06-18-25 +.. nonce: JimxP5 +.. section: IDLE + +Add docstrings to the IDLE debugger module. Fix two bugs: initialize +``Idb.botframe`` (should be in Bdb); in ``Idb.in_rpc_code``, check whether +``prev_frame`` is ``None`` before trying to use it. Greatly expand test_debugger. + +.. + +.. date: 2023-11-09-13-04-29 +.. gh-issue: 111903 +.. nonce: 7Prryr +.. section: Tools/Demos + +Argument Clinic now supports the ``@critical_section`` directive that +instructs Argument Clinic to generate a critical section around the function +call, which locks the ``self`` object in ``--disable-gil`` builds. Patch by +Sam Gross. + +.. + +.. date: 2023-11-15-18-36-21 +.. gh-issue: 112026 +.. nonce: _Yybr5 +.. section: C API + +Add again the private ``_PyThreadState_UncheckedGet()`` function as an alias +to the new public :c:func:`PyThreadState_GetUnchecked` function. Patch by +Victor Stinner. + +.. + +.. date: 2023-11-15-17-10-09 +.. gh-issue: 112026 +.. nonce: ts9yyn +.. section: C API + +Restore the removed ``_PyDict_GetItemStringWithError()`` function. It is +used by numpy. Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-16-07-57 +.. gh-issue: 112026 +.. nonce: bnr8dd +.. section: C API + +Restore removed private C API functions, macros and structures which have no +simple replacement for now: + +* _PyDict_GetItem_KnownHash() +* _PyDict_NewPresized() +* _PyHASH_BITS +* _PyHASH_IMAG +* _PyHASH_INF +* _PyHASH_MODULUS +* _PyHASH_MULTIPLIER +* _PyLong_Copy() +* _PyLong_FromDigits() +* _PyLong_New() +* _PyLong_Sign() +* _PyObject_CallMethodId() +* _PyObject_CallMethodNoArgs() +* _PyObject_CallMethodOneArg() +* _PyObject_CallOneArg() +* _PyObject_EXTRA_INIT +* _PyObject_FastCallDict() +* _PyObject_GetAttrId() +* _PyObject_Vectorcall() +* _PyObject_VectorcallMethod() +* _PyStack_AsDict() +* _PyThread_CurrentFrames() +* _PyUnicodeWriter structure +* _PyUnicodeWriter_Dealloc() +* _PyUnicodeWriter_Finish() +* _PyUnicodeWriter_Init() +* _PyUnicodeWriter_Prepare() +* _PyUnicodeWriter_PrepareKind() +* _PyUnicodeWriter_WriteASCIIString() +* _PyUnicodeWriter_WriteChar() +* _PyUnicodeWriter_WriteLatin1String() +* _PyUnicodeWriter_WriteStr() +* _PyUnicodeWriter_WriteSubstring() +* _PyUnicode_AsString() +* _PyUnicode_FromId() +* _PyVectorcall_Function() +* _Py_IDENTIFIER() +* _Py_c_abs() +* _Py_c_diff() +* _Py_c_neg() +* _Py_c_pow() +* _Py_c_prod() +* _Py_c_quot() +* _Py_c_sum() +* _Py_static_string() +* _Py_static_string_init() + +Patch by Victor Stinner. + +.. + +.. date: 2023-11-13-17-57-11 +.. gh-issue: 112026 +.. nonce: WJLJcI +.. section: C API + +Add again ```` and ```` includes in ``Python.h``, but +don't include them in the limited C API version 3.13 and newer. Patch by +Victor Stinner. + +.. + +.. date: 2023-11-10-10-24-28 +.. gh-issue: 111956 +.. nonce: ImE6Cx +.. section: C API + +Add internal-only one-time initialization API: ``_PyOnceFlag`` and +``_PyOnceFlag_CallOnce``. + +.. + +.. date: 2023-11-10-10-21-38 +.. gh-issue: 111262 +.. nonce: 2utB5m +.. section: C API + +Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a +key from a dictionary and optionally return the removed value. This is +similar to :meth:`dict.pop`, but without the default value and not raising +:exc:`KeyError` if the key missing. Patch by Stefan Behnel and Victor +Stinner. + +.. + +.. date: 2023-11-08-20-28-03 +.. gh-issue: 111863 +.. nonce: RPeFAX +.. section: C API + +Rename ``Py_NOGIL`` to ``Py_GIL_DISABLED``. Patch by Hugo van Kemenade. + +.. + +.. date: 2023-11-08-18-37-19 +.. gh-issue: 111138 +.. nonce: 3Ypq8h +.. section: C API + +Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to +Python ``list.extend()`` and ``list.clear()`` methods. Patch by Victor +Stinner. + +.. + +.. date: 2023-10-31-18-22-03 +.. gh-issue: 108765 +.. nonce: _beYv8 +.. section: C API + +On Windows, ``Python.h`` no longer includes the ```` standard +header file. If needed, it should now be included explicitly. Patch by +Victor Stinner. + +.. + +.. date: 2023-10-31-14-58-17 +.. gh-issue: 111569 +.. nonce: _V8iu4 +.. section: C API + +Implement "Python Critical Sections" from :pep:`703`. These are macros to +help replace the GIL with per-object locks in the ``--disable-gil`` build of +CPython. The macros are no-ops in the default build. + +.. + +.. date: 2023-10-30-18-13-01 +.. gh-issue: 111506 +.. nonce: EUdO22 +.. section: C API + +In the limited C API version 3.13, :c:func:`Py_SET_REFCNT` function is now +implemented as an opaque function call. Patch by Victor Stinner. + +.. + +.. date: 2023-10-19-22-39-24 +.. gh-issue: 108082 +.. nonce: uJytvc +.. section: C API + +Add :c:func:`PyErr_FormatUnraisable` function. + +.. + +.. date: 2023-10-17-10-21-59 +.. gh-issue: 110964 +.. nonce: OxqEjd +.. section: C API + +Move the undocumented private _PyArg functions and _PyArg_Parser structure +to internal C API (``pycore_modsupport.h``). Patch by Victor Stinner. + +.. + +.. date: 2023-10-13-14-18-06 +.. gh-issue: 110815 +.. nonce: tEFLVl +.. section: C API + +Support non-ASCII keyword names in :c:func:`PyArg_ParseTupleAndKeywords`. + +.. + +.. date: 2023-10-02-23-08-53 +.. gh-issue: 109587 +.. nonce: UqqnDY +.. section: C API + +Introduced :c:func:`PyUnstable_PerfTrampoline_CompileCode`, +:c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` and +:c:func:`PyUnstable_CopyPerfMapFile`. These functions allow extension +modules to initialize trampolines eagerly, after the application is "warmed +up". This makes it possible to have perf-trampolines running in an +always-enabled fashion. + +.. + +.. date: 2023-08-28-17-40-51 +.. gh-issue: 85283 +.. nonce: raFNiD +.. section: C API + +Add the :c:func:`PySys_Audit` function to the limited C API. Patch by Victor +Stinner. + +.. + +.. date: 2023-08-28-17-34-10 +.. gh-issue: 85283 +.. nonce: f1zXcc +.. section: C API + +Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, +:c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API. +Patch by Victor Stinner. + +.. + +.. date: 2023-07-12-12-14-52 +.. gh-issue: 106672 +.. nonce: fkRjmi +.. section: C API + +Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, +:c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, +:c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and +:c:func:`PySys_GetObject`, which clear all errors occurred during calling +the function, report now them using :func:`sys.unraisablehook`. + +.. + +.. date: 2023-06-08-21-12-44 +.. gh-issue: 67565 +.. nonce: UkK3x- +.. section: C API + +Remove redundant C-contiguity check in :file:`getargs.c`, :mod:`binascii`, +:mod:`ssl` and Argument Clinic. Patched by Stefan Krah and Furkan Onder diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst new file mode 100644 index 00000000000000..29fbe00efef76d --- /dev/null +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -0,0 +1,2414 @@ +.. date: 2024-01-02-19-52-23 +.. gh-issue: 113659 +.. nonce: DkmnQc +.. release date: 2024-01-17 +.. section: Security + +Skip ``.pth`` files with names starting with a dot or hidden file attribute. + +.. + +.. date: 2023-12-06-14-06-59 +.. gh-issue: 112302 +.. nonce: 3bl20f +.. section: Security + +Created a Software Bill-of-Materials document and tooling for tracking +dependencies. + +.. + +.. date: 2024-01-11-16-54-55 +.. gh-issue: 107901 +.. nonce: Td3JPI +.. section: Core and Builtins + +Compiler duplicates basic blocks that have an eval breaker check, no line +number, and multiple predecessors. + +.. + +.. date: 2024-01-11-14-03-31 +.. gh-issue: 107901 +.. nonce: U65IyC +.. section: Core and Builtins + +A jump leaving an exception handler back to normal code no longer checks the +eval breaker. + +.. + +.. date: 2024-01-11-01-28-25 +.. gh-issue: 113655 +.. nonce: Mfioxp +.. section: Core and Builtins + +Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This +seems to be near the sweet spot to maintain safety, but not compromise +backwards compatibility. + +.. + +.. date: 2024-01-09-23-01-00 +.. gh-issue: 113710 +.. nonce: pe3flY +.. section: Core and Builtins + +Add typed stack effects to the interpreter DSL, along with various +instruction annotations. + +.. + +.. date: 2024-01-08-14-34-02 +.. gh-issue: 77046 +.. nonce: sDUh2d +.. section: Core and Builtins + +On Windows, file descriptors wrapping Windows handles are now created non +inheritable by default (:pep:`446`). Patch by Zackery Spytz and Victor +Stinner. + +.. + +.. date: 2024-01-08-05-36-59 +.. gh-issue: 113853 +.. nonce: lm-6_a +.. section: Core and Builtins + +Guarantee that all executors make progress. This then guarantees that tier 2 +execution always makes progress. + +.. + +.. date: 2024-01-05-21-28-48 +.. gh-issue: 113753 +.. nonce: 2HNiuq +.. section: Core and Builtins + +Fix an issue where the finalizer of ``PyAsyncGenASend`` objects might not be +called if they were allocated from a free list. + +.. + +.. date: 2024-01-05-00-49-14 +.. gh-issue: 107901 +.. nonce: 6JRrb6 +.. section: Core and Builtins + +Compiler changed so that synthetic jumps which are not at loop end no longer +check the eval breaker. + +.. + +.. date: 2024-01-04-17-15-30 +.. gh-issue: 113703 +.. nonce: Zsk0pY +.. section: Core and Builtins + +Fix a regression in the :mod:`codeop` module that was causing it to +incorrectly identify incomplete f-strings. Patch by Pablo Galindo + +.. + +.. date: 2024-01-03-12-19-37 +.. gh-issue: 89811 +.. nonce: cZOj6d +.. section: Core and Builtins + +Check for a valid ``tp_version_tag`` before performing bytecode +specializations that rely on this value being usable. + +.. + +.. date: 2024-01-02-17-22-57 +.. gh-issue: 111488 +.. nonce: EJH3Oh +.. section: Core and Builtins + +Changed error message in case of no 'in' keyword after 'for' in list +comprehensions + +.. + +.. date: 2024-01-02-11-14-29 +.. gh-issue: 113657 +.. nonce: CQo9vF +.. section: Core and Builtins + +Fix an issue that caused important instruction pointer updates to be +optimized out of tier two traces. + +.. + +.. date: 2024-01-01-23-57-24 +.. gh-issue: 113603 +.. nonce: ySwovr +.. section: Core and Builtins + +Fixed bug where a redundant NOP is not removed, causing an assertion to fail +in the compiler in debug mode. + +.. + +.. date: 2024-01-01-00-07-02 +.. gh-issue: 113602 +.. nonce: cWuTzk +.. section: Core and Builtins + +Fix an error that was causing the parser to try to overwrite existing errors +and crashing in the process. Patch by Pablo Galindo + +.. + +.. date: 2023-12-31-07-46-01 +.. gh-issue: 113486 +.. nonce: uki19C +.. section: Core and Builtins + +No longer issue spurious ``PY_UNWIND`` events for optimized calls to +classes. + +.. + +.. date: 2023-12-20-18-27-11 +.. gh-issue: 113297 +.. nonce: BZyAI_ +.. section: Core and Builtins + +Fix segfault in the compiler on with statement with 19 context managers. + +.. + +.. date: 2023-12-20-08-54-54 +.. gh-issue: 113212 +.. nonce: 62AUlw +.. section: Core and Builtins + +Improve :py:class:`super` error messages. + +.. + +.. date: 2023-12-19-22-03-43 +.. gh-issue: 111375 +.. nonce: M9vuA6 +.. section: Core and Builtins + +Only use ``NULL`` in the exception stack to indicate an exception was +handled. Patch by Carey Metcalfe. + +.. + +.. date: 2023-12-15-16-26-01 +.. gh-issue: 112215 +.. nonce: xJS6_6 +.. section: Core and Builtins + +Increase the C recursion limit by a factor of 3 for non-debug builds, except +for webassembly and s390 platforms which are unchanged. This mitigates some +regressions in 3.12 with deep recursion mixing builtin (C) and Python code. + +.. + +.. date: 2023-12-14-20-08-35 +.. gh-issue: 113054 +.. nonce: e20CtM +.. section: Core and Builtins + +Fixed bug where a redundant NOP is not removed, causing an assertion to fail +in the compiler in debug mode. + +.. + +.. date: 2023-12-13-11-45-53 +.. gh-issue: 106905 +.. nonce: 5dslTN +.. section: Core and Builtins + +Use per AST-parser state rather than global state to track recursion depth +within the AST parser to prevent potential race condition due to +simultaneous parsing. + +The issue primarily showed up in 3.11 by multithreaded users of +:func:`ast.parse`. In 3.12 a change to when garbage collection can be +triggered prevented the race condition from occurring. + +.. + +.. date: 2023-12-12-04-53-19 +.. gh-issue: 108866 +.. nonce: xbJ-9a +.. section: Core and Builtins + +Change the API and contract of ``_PyExecutorObject`` to return the +next_instr pointer, instead of the frame, and to always execute at least one +instruction. + +.. + +.. date: 2023-12-11-19-53-32 +.. gh-issue: 90350 +.. nonce: -FQy3E +.. section: Core and Builtins + +Optimize builtin functions :func:`min` and :func:`max`. + +.. + +.. date: 2023-12-11-00-50-00 +.. gh-issue: 112943 +.. nonce: RHNZie +.. section: Core and Builtins + +Correctly compute end column offsets for multiline tokens in the +:mod:`tokenize` module. Patch by Pablo Galindo + +.. + +.. date: 2023-12-07-13-19-55 +.. gh-issue: 112125 +.. nonce: 4ADN7i +.. section: Core and Builtins + +Fix ``None.__ne__(None)`` returning ``NotImplemented`` instead of ``False``. + +.. + +.. date: 2023-12-07-12-00-04 +.. gh-issue: 74616 +.. nonce: kgTGVb +.. section: Core and Builtins + +:func:`input` now raises a ValueError when output on the terminal if the +prompt contains embedded null characters instead of silently truncating it. + +.. + +.. date: 2023-12-05-20-41-58 +.. gh-issue: 112716 +.. nonce: hOcx0Y +.. section: Core and Builtins + +Fix SystemError in the ``import`` statement and in ``__reduce__()`` methods +of builtin types when ``__builtins__`` is not a dict. + +.. + +.. date: 2023-12-04-23-09-07 +.. gh-issue: 112730 +.. nonce: BXHlFa +.. section: Core and Builtins + +Use color to highlight error locations in tracebacks. Patch by Pablo Galindo + +.. + +.. date: 2023-12-03-19-34-51 +.. gh-issue: 112625 +.. nonce: QWTlwS +.. section: Core and Builtins + +Fixes a bug where a bytearray object could be cleared while iterating over +an argument in the ``bytearray.join()`` method that could result in reading +memory after it was freed. + +.. + +.. date: 2023-12-03-15-29-53 +.. gh-issue: 112660 +.. nonce: gldBvh +.. section: Core and Builtins + +Do not clear unexpected errors during formatting error messages for +ImportError and AttributeError for modules. + +.. + +.. date: 2023-12-01-19-02-21 +.. gh-issue: 105967 +.. nonce: Puq5Cn +.. section: Core and Builtins + +Workaround a bug in Apple's macOS platform zlib library where +:func:`zlib.crc32` and :func:`binascii.crc32` could produce incorrect +results on multi-gigabyte inputs. Including when using :mod:`zipfile` on +zips containing large data. + +.. + +.. date: 2023-12-01-08-16-10 +.. gh-issue: 95754 +.. nonce: ae4gwy +.. section: Core and Builtins + +Provide a better error message when accessing invalid attributes on +partially initialized modules. The origin of the module being accessed is +now included in the message to help with the common issue of shadowing other +modules. + +.. + +.. date: 2023-11-27-18-55-30 +.. gh-issue: 112217 +.. nonce: SwFLMj +.. section: Core and Builtins + +Add check for the type of ``__cause__`` returned from calling the type ``T`` +in ``raise from T``. + +.. + +.. date: 2023-11-26-21-30-11 +.. gh-issue: 111058 +.. nonce: q4DqDY +.. section: Core and Builtins + +Change coro.cr_frame/gen.gi_frame to return ``None`` after the +coroutine/generator has been closed. This fixes a bug where +:func:`~inspect.getcoroutinestate` and :func:`~inspect.getgeneratorstate` +return the wrong state for a closed coroutine/generator. + +.. + +.. date: 2023-11-25-22-58-49 +.. gh-issue: 112388 +.. nonce: MU3cIM +.. section: Core and Builtins + +Fix an error that was causing the parser to try to overwrite tokenizer +errors. Patch by pablo Galindo + +.. + +.. date: 2023-11-25-22-39-44 +.. gh-issue: 112387 +.. nonce: AbBq5W +.. section: Core and Builtins + +Fix error positions for decoded strings with backwards tokenize errors. +Patch by Pablo Galindo + +.. + +.. date: 2023-11-25-20-36-38 +.. gh-issue: 99606 +.. nonce: fDY5hK +.. section: Core and Builtins + +Make code generated for an empty f-string identical to the code of an empty +normal string. + +.. + +.. date: 2023-11-24-14-10-57 +.. gh-issue: 112367 +.. nonce: 9z1IDp +.. section: Core and Builtins + +Avoid undefined behaviour when using the perf trampolines by not freeing the +code arenas until shutdown. Patch by Pablo Galindo + +.. + +.. date: 2023-11-22-13-17-54 +.. gh-issue: 112320 +.. nonce: EddM51 +.. section: Core and Builtins + +The Tier 2 translator now tracks the confidence level for staying "on trace" +(i.e. not exiting back to the Tier 1 interpreter) for branch instructions +based on the number of bits set in the branch "counter". Trace translation +ends when the confidence drops below 1/3rd. + +.. + +.. date: 2023-09-21-11-54-28 +.. gh-issue: 109598 +.. nonce: CRidSy +.. section: Core and Builtins + +:c:func:`PyComplex_RealAsDouble`/:c:func:`PyComplex_ImagAsDouble` now tries +to convert an object to a :class:`complex` instance using its +``__complex__()`` method before falling back to the ``__float__()`` method. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2022-07-07-05-37-53 +.. gh-issue: 94606 +.. nonce: hojJ54 +.. section: Core and Builtins + +Fix UnicodeEncodeError when :func:`email.message.get_payload` reads a +message with a Unicode surrogate character and the message content is not +well-formed for surrogateescape encoding. Patch by Sidney Markowitz. + +.. + +.. bpo: 21861 +.. date: 2022-01-23-18-00-10 +.. nonce: N8E1zw +.. section: Core and Builtins + +Use the object's actual class name in :meth:`!_io.FileIO.__repr__`, +:meth:`!_io._WindowsConsoleIO` and :meth:`!_io.TextIOWrapper.__repr__`, to +make these methods subclass friendly. + +.. + +.. bpo: 45369 +.. date: 2021-10-05-05-00-16 +.. nonce: tluk_X +.. section: Core and Builtins + +Remove LibreSSL workarounds as per :pep:`644`. + +.. + +.. bpo: 34392 +.. date: 2018-08-13-13-25-15 +.. nonce: 9kIlMF +.. section: Core and Builtins + +Added :func:`sys._is_interned`. + +.. + +.. date: 2024-01-15-12-12-54 +.. gh-issue: 114077 +.. nonce: KcVnfj +.. section: Library + +Fix possible :exc:`OverflowError` in :meth:`socket.socket.sendfile` when +pass *count* larger than 2 GiB on 32-bit platform. + +.. + +.. date: 2024-01-13-14-20-31 +.. gh-issue: 111803 +.. nonce: llpLAw +.. section: Library + +:mod:`plistlib` now supports loading more deeply nested lists in binary +format. + +.. + +.. date: 2024-01-13-11-34-29 +.. gh-issue: 114014 +.. nonce: WRHifN +.. section: Library + +Fixed a bug in :class:`fractions.Fraction` where an invalid string using +``d`` in the decimals part creates a different error compared to other +invalid letters/characters. Patch by Jeremiah Gabriel Pascual. + +.. + +.. date: 2024-01-11-22-22-51 +.. gh-issue: 108364 +.. nonce: QH7C-1 +.. section: Library + +:meth:`sqlite3.Connection.iterdump` now ensures that foreign key support is +disabled before dumping the database schema, if there is any foreign key +violation. Patch by Erlend E. Aasland and Mariusz Felisiak. + +.. + +.. date: 2024-01-11-16-58-10 +.. gh-issue: 113971 +.. nonce: skJZ4g +.. section: Library + +The :class:`zipfile.ZipInfo` previously protected ``._compresslevel`` +attribute has been made public as ``.compress_level`` with the old +``_compresslevel`` name remaining available as a property to retain +compatibility. + +.. + +.. date: 2024-01-10-12-03-38 +.. gh-issue: 113877 +.. nonce: RxKlrQ +.. section: Library + +Fix :mod:`tkinter` method ``winfo_pathname()`` on 64-bit Windows. + +.. + +.. date: 2024-01-09-18-07-08 +.. gh-issue: 113868 +.. nonce: DlZG2r +.. section: Library + +Added :data:`mmap.MAP_NORESERVE`, :data:`mmap.MAP_NOEXTEND`, +:data:`mmap.MAP_HASSEMAPHORE`, :data:`mmap.MAP_NOCACHE`, +:data:`mmap.MAP_JIT`, :data:`mmap.MAP_RESILIENT_CODESIGN`, +:data:`mmap.MAP_RESILIENT_MEDIA`, :data:`mmap.MAP_32BIT`, +:data:`mmap.MAP_TRANSLATED_ALLOW_EXECUTE`, :data:`mmap.MAP_UNIX03` and +:data:`mmap.MAP_TPRO`. All of them are ``mmap(2)`` flags on macOS. + +.. + +.. date: 2024-01-09-12-19-55 +.. gh-issue: 113848 +.. nonce: kXoCy0 +.. section: Library + +:func:`asyncio.TaskGroup()` and :func:`asyncio.timeout()` context managers +now handle :exc:`~asyncio.CancelledError` subclasses as well as exact +:exc:`!CancelledError`. + +.. + +.. date: 2024-01-09-08-59-43 +.. gh-issue: 113661 +.. nonce: asvXSx +.. section: Library + +unittest runner: Don't exit 5 if tests were skipped. The intention of +exiting 5 was to detect issues where the test suite wasn't discovered at +all. If we skipped tests, it was correctly discovered. + +.. + +.. date: 2024-01-08-19-38-42 +.. gh-issue: 96037 +.. nonce: Yr2Y1C +.. section: Library + +Insert :exc:`TimeoutError` in the context of the exception that was raised +during exiting an expired :func:`asyncio.timeout` block. + +.. + +.. date: 2024-01-08-14-57-09 +.. gh-issue: 113781 +.. nonce: IoTnwi +.. section: Library + +Silence unraisable AttributeError when warnings are emitted during Python +finalization. + +.. + +.. date: 2024-01-07-23-31-44 +.. gh-issue: 113238 +.. nonce: wFWBfW +.. section: Library + +Add ``Anchor`` to ``importlib.resources`` (in order for the code to comply +with the documentation) + +.. + +.. date: 2024-01-07-13-36-03 +.. gh-issue: 111693 +.. nonce: xN2LuL +.. section: Library + +:func:`asyncio.Condition.wait()` now re-raises the same +:exc:`CancelledError` instance that may have caused it to be interrupted. +Fixed race condition in :func:`asyncio.Semaphore.acquire` when interrupted +with a :exc:`CancelledError`. + +.. + +.. date: 2024-01-07-11-45-56 +.. gh-issue: 113791 +.. nonce: XF5xSW +.. section: Library + +Add ``CLOCK_MONOTONIC_RAW_APPROX`` and ``CLOCK_UPTIME_RAW_APPROX`` to +:mod:`time` on macOS. These are clocks available on macOS 10.12 or later. + +.. + +.. date: 2024-01-07-00-56-41 +.. gh-issue: 112932 +.. nonce: OfhUu7 +.. section: Library + +Restore the ability for :mod:`zipfile` to ``extractall`` from zip files with +a "/" directory entry in them as is commonly added to zips by some wiki or +bug tracker data exporters. + +.. + +.. date: 2024-01-05-21-52-59 +.. gh-issue: 113568 +.. nonce: _0FkpZ +.. section: Library + +Raise deprecation warnings from :class:`pathlib.PurePath` and not its +private base class ``PurePathBase``. + +.. + +.. date: 2024-01-05-12-42-07 +.. gh-issue: 113594 +.. nonce: 4t8HiR +.. section: Library + +Fix :exc:`UnicodeEncodeError` in :mod:`email` when re-fold lines that +contain unknown-8bit encoded part followed by non-unknown-8bit encoded part. + +.. + +.. date: 2024-01-03-14-19-26 +.. gh-issue: 113538 +.. nonce: ahuBCo +.. section: Library + +In :meth:`asyncio.StreamReaderProtocol.connection_made`, there is callback +that logs an error if the task wrapping the "connected callback" fails. This +callback would itself fail if the task was cancelled. Prevent this by +checking whether the task was cancelled first. If so, close the transport +but don't log an error. + +.. + +.. date: 2024-01-02-12-41-59 +.. gh-issue: 113626 +.. nonce: i1PPY_ +.. section: Library + +Add support for the *allow_code* argument in the :mod:`marshal` module. +Passing ``allow_code=False`` prevents serialization and de-serialization of +code objects which is incompatible between Python versions. + +.. + +.. date: 2024-01-01-13-26-02 +.. gh-issue: 85567 +.. nonce: K4U15m +.. section: Library + +Fix resource warnings for unclosed files in :mod:`pickle` and +:mod:`pickletools` command line interfaces. + +.. + +.. date: 2023-12-30-20-30-05 +.. gh-issue: 113537 +.. nonce: v1W5_X +.. section: Library + +Support loads ``str`` in :func:`plistlib.loads`. + +.. + +.. date: 2023-12-29-22-29-34 +.. gh-issue: 89850 +.. nonce: KnxiZA +.. section: Library + +Add default implementations of :meth:`pickle.Pickler.persistent_id` and +:meth:`pickle.Unpickler.persistent_load` methods in the C implementation. +Calling ``super().persistent_id()`` and ``super().persistent_load()`` in +subclasses of the C implementation of :class:`pickle.Pickler` and +:class:`pickle.Unpickler` classes no longer causes infinite recursion. + +.. + +.. date: 2023-12-29-17-57-45 +.. gh-issue: 113569 +.. nonce: qcRCEI +.. section: Library + +Indicate if there were no actual calls in unittest +:meth:`~unittest.mock.Mock.assert_has_calls` failure. + +.. + +.. date: 2023-12-29-17-46-06 +.. gh-issue: 101225 +.. nonce: QaEyxF +.. section: Library + +Increase the backlog for :class:`multiprocessing.connection.Listener` +objects created by :mod:`multiprocessing.manager` and +:mod:`multiprocessing.resource_sharer` to significantly reduce the risk of +getting a connection refused error when creating a +:class:`multiprocessing.connection.Connection` to them. + +.. + +.. date: 2023-12-29-17-30-49 +.. gh-issue: 113568 +.. nonce: UpWNAI +.. section: Library + +Raise audit events from :class:`pathlib.Path` and not its private base class +``PathBase``. + +.. + +.. date: 2023-12-28-14-36-20 +.. gh-issue: 113543 +.. nonce: 2iWkOR +.. section: Library + +Make sure that ``webbrowser.MacOSXOSAScript`` sends ``webbrowser.open`` +audit event. + +.. + +.. date: 2023-12-23-16-51-17 +.. gh-issue: 113028 +.. nonce: 3Jmdoj +.. section: Library + +When a second reference to a string appears in the input to :mod:`pickle`, +and the Python implementation is in use, we are guaranteed that a single +copy gets pickled and a single object is shared when reloaded. Previously, +in protocol 0, when a string contained certain characters (e.g. newline) it +resulted in duplicate objects. + +.. + +.. date: 2023-12-23-16-10-07 +.. gh-issue: 113421 +.. nonce: w7vs08 +.. section: Library + +Fix multiprocessing logger for ``%(filename)s``. + +.. + +.. date: 2023-12-23-13-10-42 +.. gh-issue: 111784 +.. nonce: Nb4L1j +.. section: Library + +Fix segfaults in the ``_elementtree`` module. Fix first segfault during +deallocation of ``_elementtree.XMLParser`` instances by keeping strong +reference to ``pyexpat`` module in module state for capsule lifetime. Fix +second segfault which happens in the same deallocation process by keeping +strong reference to ``_elementtree`` module in ``XMLParser`` structure for +``_elementtree`` module lifetime. + +.. + +.. date: 2023-12-22-20-49-52 +.. gh-issue: 113407 +.. nonce: C_O13_ +.. section: Library + +Fix import of :mod:`unittest.mock` when CPython is built without docstrings. + +.. + +.. date: 2023-12-22-11-30-57 +.. gh-issue: 113320 +.. nonce: Vp5suS +.. section: Library + +Fix regression in Python 3.12 where :class:`~typing.Protocol` classes that +were not marked as :func:`runtime-checkable ` +would be unnecessarily introspected, potentially causing exceptions to be +raised if the protocol had problematic members. Patch by Alex Waygood. + +.. + +.. date: 2023-12-21-23-47-42 +.. gh-issue: 53502 +.. nonce: dercJI +.. section: Library + +Add a new option ``aware_datetime`` in :mod:`plistlib` to loads or dumps +aware datetime. + +.. + +.. date: 2023-12-21-14-55-06 +.. gh-issue: 113358 +.. nonce: nRkiSL +.. section: Library + +Fix rendering tracebacks with exceptions with a broken __getattr__ + +.. + +.. date: 2023-12-20-21-18-51 +.. gh-issue: 113214 +.. nonce: JcV9Mn +.. section: Library + +Fix an ``AttributeError`` during asyncio SSL protocol aborts in SSL-over-SSL +scenarios. + +.. + +.. date: 2023-12-18-09-47-54 +.. gh-issue: 113246 +.. nonce: em930H +.. section: Library + +Update bundled pip to 23.3.2. + +.. + +.. date: 2023-12-17-13-56-30 +.. gh-issue: 87264 +.. nonce: RgfHCv +.. section: Library + +Fixed tarfile list() method to show file type. + +.. + +.. date: 2023-12-17-10-22-55 +.. gh-issue: 112182 +.. nonce: jLWGlr +.. section: Library + +:meth:`asyncio.futures.Future.set_exception()` now transforms +:exc:`StopIteration` into :exc:`RuntimeError` instead of hanging or other +misbehavior. Patch contributed by Jamie Phan. + +.. + +.. date: 2023-12-17-04-43-57 +.. gh-issue: 113225 +.. nonce: dhxhiZ +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by using :attr:`os.DirEntry.path` where +possible. + +.. + +.. date: 2023-12-16-23-56-42 +.. gh-issue: 113149 +.. nonce: 7LWgTS +.. section: Library + +Improve error message when a JSON array or object contains a trailing comma. +Patch by Carson Radtke. + +.. + +.. date: 2023-12-16-10-58-34 +.. gh-issue: 113117 +.. nonce: 0zF7bH +.. section: Library + +The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function +with ``close_fds=True`` on platforms where +``posix_spawn_file_actions_addclosefrom_np`` is available. Patch by Jakub +Kulik. + +.. + +.. date: 2023-12-16-01-10-47 +.. gh-issue: 113199 +.. nonce: oDjnjL +.. section: Library + +Make ``http.client.HTTPResponse.read1`` and +``http.client.HTTPResponse.readline`` close IO after reading all data when +content length is known. Patch by Illia Volochii. + +.. + +.. date: 2023-12-15-21-33-42 +.. gh-issue: 113191 +.. nonce: Il155b +.. section: Library + +Add support of :func:`os.fchmod` and a file descriptor in :func:`os.chmod` +on Windows. + +.. + +.. date: 2023-12-15-20-29-49 +.. gh-issue: 113188 +.. nonce: AvoraB +.. section: Library + +Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. +Previously they worked differently if *dst* is a symbolic link: they modified +the permission bits of *dst* itself rather than the file it points to if +*follow_symlinks* is true or *src* is not a symbolic link, and did not +modify the permission bits if *follow_symlinks* is false and *src* is a +symbolic link. + +.. + +.. date: 2023-12-15-18-13-59 +.. gh-issue: 113119 +.. nonce: al-569 +.. section: Library + +:func:`os.posix_spawn` now accepts ``env=None``, which makes the newly +spawned process use the current process environment. Patch by Jakub Kulik. + +.. + +.. date: 2023-12-15-18-10-26 +.. gh-issue: 113202 +.. nonce: xv_Ww8 +.. section: Library + +Add a ``strict`` option to ``batched()`` in the ``itertools`` module. + +.. + +.. date: 2023-12-15-12-35-28 +.. gh-issue: 61648 +.. nonce: G-4pz0 +.. section: Library + +Detect line numbers of properties in doctests. + +.. + +.. date: 2023-12-15-09-51-41 +.. gh-issue: 113175 +.. nonce: RHsNwE +.. section: Library + +Sync with importlib_metadata 7.0, including improved type annotations, fixed +issue with symlinked packages in ``package_distributions``, added +``EntryPoints.__repr__``, introduced the ``diagnose`` script, added +``Distribution.origin`` property, and removed deprecated ``EntryPoint`` +access by numeric index (tuple behavior). + +.. + +.. date: 2023-12-13-17-08-21 +.. gh-issue: 59616 +.. nonce: JNlWSs +.. section: Library + +Add support of :func:`os.lchmod` and the *follow_symlinks* argument in +:func:`os.chmod` on Windows. Note that the default value of +*follow_symlinks* in :func:`!os.lchmod` is ``False`` on Windows. + +.. + +.. date: 2023-12-12-20-15-57 +.. gh-issue: 112559 +.. nonce: IgXkje +.. section: Library + +:func:`signal.signal` and :func:`signal.getsignal` no longer call ``repr`` +on callable handlers. :func:`asyncio.run` and :meth:`asyncio.Runner.run` no +longer call ``repr`` on the task results. Patch by Yilei Yang. + +.. + +.. date: 2023-12-12-16-32-55 +.. gh-issue: 112962 +.. nonce: ZZWXZn +.. section: Library + +:mod:`dis` module functions add cache information to the +:class:`~dis.Instruction` instance rather than creating fake +:class:`~dis.Instruction` instances to represent the cache entries. + +.. + +.. date: 2023-12-12-05-48-17 +.. gh-issue: 112989 +.. nonce: ZAa_eq +.. section: Library + +Reduce overhead to connect sockets with :mod:`asyncio` SelectorEventLoop. + +.. + +.. date: 2023-12-11-16-13-15 +.. gh-issue: 112970 +.. nonce: 87jmKP +.. section: Library + +Use :c:func:`!closefrom` on Linux where available (e.g. glibc-2.34), rather +than only FreeBSD. + +.. + +.. date: 2023-12-11-14-12-46 +.. gh-issue: 110190 +.. nonce: e0iEUa +.. section: Library + +Fix ctypes structs with array on PPC64LE platform by setting +``MAX_STRUCT_SIZE`` to 64 in stgdict. Patch by Diego Russo. + +.. + +.. date: 2023-12-08-11-17-17 +.. gh-issue: 112540 +.. nonce: Pm5egX +.. section: Library + +The statistics.geometric_mean() function now returns zero for datasets +containing a zero. Formerly, it would raise an exception. + +.. + +.. date: 2023-12-07-16-55-41 +.. gh-issue: 87286 +.. nonce: MILC9_ +.. section: Library + +Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, +:const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD` tot the +:mod:`syslog` module, all of them constants on used on macOS. + +.. + +.. date: 2023-12-06-16-01-33 +.. gh-issue: 112800 +.. nonce: TNsGJ- +.. section: Library + +Fix :mod:`asyncio` ``SubprocessTransport.close()`` not to throw +``PermissionError`` when used with setuid executables. + +.. + +.. date: 2023-12-06-14-06-14 +.. gh-issue: 51944 +.. nonce: -5qq_L +.. section: Library + +Add the following constants to the :mod:`termios` module. These values are +present in macOS system headers: ``ALTWERASE``, ``B14400``, ``B28800``, +``B7200``, ``B76800``, ``CCAR_OFLOW``, ``CCTS_OFLOW``, ``CDSR_OFLOW``, +``CDTR_IFLOW``, ``CIGNORE``, ``CRTS_IFLOW``, ``EXTPROC``, ``IUTF8``, +``MDMBUF``, ``NL2``, ``NL3``, ``NOKERNINFO``, ``ONOEOT``, ``OXTABS``, +``VDSUSP``, ``VSTATUS``. + +.. + +.. date: 2023-12-05-18-57-53 +.. gh-issue: 79325 +.. nonce: P2vMVK +.. section: Library + +Fix an infinite recursion error in :func:`tempfile.TemporaryDirectory` +cleanup on Windows. + +.. + +.. date: 2023-12-05-16-20-40 +.. gh-issue: 94692 +.. nonce: -e5C3c +.. section: Library + +:func:`shutil.rmtree` now only catches OSError exceptions. Previously a +symlink attack resistant version of ``shutil.rmtree()`` could ignore or pass +to the error handler arbitrary exception when invalid arguments were +provided. + +.. + +.. date: 2023-12-05-01-19-28 +.. gh-issue: 112736 +.. nonce: rdHDrU +.. section: Library + +The use of del-safe symbols in ``subprocess`` was refactored to allow for +use in cross-platform build environments. + +.. + +.. date: 2023-12-04-21-30-34 +.. gh-issue: 112727 +.. nonce: jpgNRB +.. section: Library + +Speed up :meth:`pathlib.Path.absolute`. Patch by Barney Gale. + +.. + +.. date: 2023-12-04-16-45-11 +.. gh-issue: 74690 +.. nonce: pQYP5U +.. section: Library + +Speedup :func:`issubclass` checks against simple :func:`runtime-checkable +protocols ` by around 6%. Patch by Alex Waygood. + +.. + +.. date: 2023-12-04-14-05-24 +.. gh-issue: 74690 +.. nonce: eODKRm +.. section: Library + +Speedup :func:`isinstance` checks by roughly 20% for +:func:`runtime-checkable protocols ` that only +have one callable member. Speedup :func:`issubclass` checks for these +protocols by roughly 10%. Patch by Alex Waygood. + +.. + +.. date: 2023-12-03-12-41-48 +.. gh-issue: 112645 +.. nonce: blMsKf +.. section: Library + +Remove deprecation error on passing ``onerror`` to :func:`shutil.rmtree`. + +.. + +.. date: 2023-12-03-11-15-53 +.. gh-issue: 112640 +.. nonce: -FVwP7 +.. section: Library + +Add ``kwdefaults`` parameter to :data:`types.FunctionType` to set default +keyword argument values. + +.. + +.. date: 2023-12-03-01-01-52 +.. gh-issue: 112622 +.. nonce: 1Z8cpx +.. section: Library + +Ensure ``name`` parameter is passed to event loop in +:func:`asyncio.create_task`. + +.. + +.. date: 2023-12-02-12-55-17 +.. gh-issue: 112618 +.. nonce: 7_FT8- +.. section: Library + +Fix a caching bug relating to :data:`typing.Annotated`. ``Annotated[str, +True]`` is no longer identical to ``Annotated[str, 1]``. + +.. + +.. date: 2023-12-01-21-05-46 +.. gh-issue: 112334 +.. nonce: DmNXKh +.. section: Library + +Fixed a performance regression in 3.12's :mod:`subprocess` on Linux where it +would no longer use the fast-path ``vfork()`` system call when it could have +due to a logic bug, instead falling back to the safe but slower ``fork()``. + +Also fixed a second 3.12.0 potential security bug. If a value of +``extra_groups=[]`` was passed to :mod:`subprocess.Popen` or related APIs, +the underlying ``setgroups(0, NULL)`` system call to clear the groups list +would not be made in the child process prior to ``exec()``. + +This was identified via code inspection in the process of fixing the first +bug. + +.. + +.. date: 2023-12-01-18-05-09 +.. gh-issue: 110190 +.. nonce: 5bf-c9 +.. section: Library + +Fix ctypes structs with array on Arm platform by setting ``MAX_STRUCT_SIZE`` +to 32 in stgdict. Patch by Diego Russo. + +.. + +.. date: 2023-12-01-16-09-59 +.. gh-issue: 81194 +.. nonce: FFad1c +.. section: Library + +Fix a crash in :func:`socket.if_indextoname` with specific value (UINT_MAX). +Fix an integer overflow in :func:`socket.if_indextoname` on 64-bit +non-Windows platforms. + +.. + +.. date: 2023-12-01-08-28-09 +.. gh-issue: 112578 +.. nonce: bfNbfi +.. section: Library + +Fix a spurious :exc:`RuntimeWarning` when executing the :mod:`zipfile` +module. + +.. + +.. date: 2023-11-29-10-51-41 +.. gh-issue: 112516 +.. nonce: rFKUKN +.. section: Library + +Update the bundled copy of pip to version 23.3.1. + +.. + +.. date: 2023-11-29-02-26-32 +.. gh-issue: 112510 +.. nonce: j-zXGc +.. section: Library + +Add :data:`readline.backend` for the backend readline uses (``editline`` or +``readline``) + +.. + +.. date: 2023-11-28-20-47-39 +.. gh-issue: 112328 +.. nonce: Z2AxEY +.. section: Library + +[Enum] Make ``EnumDict``, ``EnumDict.member_names``, +``EnumType._add_alias_`` and ``EnumType._add_value_alias_`` public. + +.. + +.. date: 2023-11-28-20-01-33 +.. gh-issue: 112509 +.. nonce: QtoKed +.. section: Library + +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-11-28-02-39-30 +.. gh-issue: 101336 +.. nonce: ya433z +.. section: Library + +Add ``keep_alive`` keyword parameter for +:meth:`AbstractEventLoop.create_server` and +:meth:`BaseEventLoop.create_server`. + +.. + +.. date: 2023-11-27-12-41-23 +.. gh-issue: 63284 +.. nonce: q2Qi9q +.. section: Library + +Added support for TLS-PSK (pre-shared key) mode to the :mod:`ssl` module. + +.. + +.. date: 2023-11-26-13-44-19 +.. gh-issue: 112414 +.. nonce: kx2E7S +.. section: Library + +Fix regression in Python 3.12 where calling :func:`repr` on a module that +had been imported using a custom :term:`loader` could fail with +:exc:`AttributeError`. Patch by Alex Waygood. + +.. + +.. date: 2023-11-26-13-26-56 +.. gh-issue: 112358 +.. nonce: smhaeZ +.. section: Library + +Revert change to :class:`struct.Struct` initialization that broke some cases +of subclassing. + +.. + +.. date: 2023-11-25-20-29-28 +.. gh-issue: 112405 +.. nonce: cOtzxC +.. section: Library + +Optimize :meth:`pathlib.PurePath.relative_to`. Patch by Alex Waygood. + +.. + +.. date: 2023-11-24-21-00-24 +.. gh-issue: 94722 +.. nonce: GMIQIn +.. section: Library + +Fix bug where comparison between instances of :class:`~doctest.DocTest` +fails if one of them has ``None`` as its lineno. + +.. + +.. date: 2023-11-24-09-27-01 +.. gh-issue: 112361 +.. nonce: kYtnHW +.. section: Library + +Speed up a small handful of :mod:`pathlib` methods by removing some +temporary objects. + +.. + +.. date: 2023-11-23-17-25-27 +.. gh-issue: 112345 +.. nonce: FFApHx +.. section: Library + +Improve error message when trying to call :func:`issubclass` against a +:class:`typing.Protocol` that has non-method members. Patch by Randolf +Scholz. + +.. + +.. date: 2023-11-23-12-37-22 +.. gh-issue: 112137 +.. nonce: kM46Q6 +.. section: Library + +Change :mod:`dis` output to display no-lineno as "--" instead of "None". + +.. + +.. date: 2023-11-23-10-41-21 +.. gh-issue: 112332 +.. nonce: rhTBaa +.. section: Library + +Deprecate the ``exc_type`` field of :class:`traceback.TracebackException`. +Add ``exc_type_str`` to replace it. + +.. + +.. date: 2023-11-22-23-08-47 +.. gh-issue: 81620 +.. nonce: mfZ2Wf +.. section: Library + +Add extra tests for :func:`random.binomialvariate` + +.. + +.. date: 2023-11-22-19-43-54 +.. gh-issue: 112292 +.. nonce: 5nDU87 +.. section: Library + +Fix a crash in :mod:`readline` when imported from a sub interpreter. Patch +by Anthony Shaw + +.. + +.. date: 2023-11-21-02-58-14 +.. gh-issue: 77621 +.. nonce: MYv5XS +.. section: Library + +Slightly improve the import time of the :mod:`pathlib` module by deferring +some imports. Patch by Barney Gale. + +.. + +.. date: 2023-11-16-17-18-09 +.. gh-issue: 112137 +.. nonce: QvjGjN +.. section: Library + +Change :mod:`dis` output to display logical labels for jump targets instead +of offsets. + +.. + +.. date: 2023-11-16-10-42-15 +.. gh-issue: 112139 +.. nonce: WpHosf +.. section: Library + +Add :meth:`Signature.format` to format signatures to string with extra +options. And use it in :mod:`pydoc` to render more readable signatures that +have new lines between parameters. + +.. + +.. date: 2023-11-15-04-53-37 +.. gh-issue: 112105 +.. nonce: I3RcVN +.. section: Library + +Make :func:`readline.set_completer_delims` work with libedit + +.. + +.. date: 2023-11-15-01-36-04 +.. gh-issue: 106922 +.. nonce: qslOVH +.. section: Library + +Display multiple lines with ``traceback`` when errors span multiple lines. + +.. + +.. date: 2023-11-09-11-07-34 +.. gh-issue: 111874 +.. nonce: dzYc3j +.. section: Library + +When creating a :class:`typing.NamedTuple` class, ensure +:func:`~object.__set_name__` is called on all objects that define +``__set_name__`` and exist in the values of the ``NamedTuple`` class's class +dictionary. Patch by Alex Waygood. + +.. + +.. date: 2023-11-08-18-53-07 +.. gh-issue: 68166 +.. nonce: 1iTh4Y +.. section: Library + +Add support of the "vsapi" element type in +:meth:`tkinter.ttk.Style.element_create`. + +.. + +.. date: 2023-11-08-16-11-04 +.. gh-issue: 110275 +.. nonce: Bm6GwR +.. section: Library + +Named tuple's methods ``_replace()`` and ``__replace__()`` now raise +TypeError instead of ValueError for invalid keyword arguments. + +.. + +.. date: 2023-11-05-20-09-27 +.. gh-issue: 99367 +.. nonce: HLaWKo +.. section: Library + +Do not mangle ``sys.path[0]`` in :mod:`pdb` if safe_path is set + +.. + +.. date: 2023-11-02-10-13-31 +.. gh-issue: 111615 +.. nonce: 3SMixi +.. section: Library + +Fix a regression caused by a fix to gh-93162 whereby you couldn't configure +a :class:`QueueHandler` without specifying handlers. + +.. + +.. date: 2023-10-25-16-37-13 +.. gh-issue: 75666 +.. nonce: BpsWut +.. section: Library + +Fix the behavior of :mod:`tkinter` widget's ``unbind()`` method with two +arguments. Previously, ``widget.unbind(sequence, funcid)`` destroyed the +current binding for *sequence*, leaving *sequence* unbound, and deleted the +*funcid* command. Now it removes only *funcid* from the binding for +*sequence*, keeping other commands, and deletes the *funcid* command. It +leaves *sequence* unbound only if *funcid* was the last bound command. + +.. + +.. date: 2023-10-25-13-07-53 +.. gh-issue: 67790 +.. nonce: jMn9Ad +.. section: Library + +Implement basic formatting support (minimum width, alignment, fill) for +:class:`fractions.Fraction`. + +.. + +.. date: 2023-10-23-18-42-26 +.. gh-issue: 111049 +.. nonce: Ys7-o_ +.. section: Library + +Fix crash during garbage collection of the :class:`io.BytesIO` buffer +object. + +.. + +.. date: 2023-10-23-03-49-34 +.. gh-issue: 102980 +.. nonce: aXBd54 +.. section: Library + +Redirect the output of ``interact`` command of :mod:`pdb` to the same +channel as the debugger. Add tests and improve docs. + +.. + +.. date: 2023-10-20-15-28-08 +.. gh-issue: 102988 +.. nonce: dStNO7 +.. section: Library + +:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now +return ``('', '')`` 2-tuples in more situations where invalid email +addresses are encountered instead of potentially inaccurate values. Add +optional *strict* parameter to these two functions: use ``strict=False`` to +get the old behavior, accept malformed inputs. ``getattr(email.utils, +'supports_strict_parsing', False)`` can be use to check if the *strict* +parameter is available. Patch by Thomas Dwyer and Victor Stinner to improve +the :cve:`2023-27043` fix. + +.. + +.. date: 2023-10-17-16-11-03 +.. gh-issue: 52161 +.. nonce: WBYyCJ +.. section: Library + +:meth:`cmd.Cmd.do_help` now cleans docstrings with :func:`inspect.cleandoc` +before writing them. Patch by Filip Åapkiewicz. + +.. + +.. date: 2023-10-12-18-19-47 +.. gh-issue: 82300 +.. nonce: P8-O38 +.. section: Library + +Add ``track`` parameter to +:class:`multiprocessing.shared_memory.SharedMemory` that allows using shared +memory blocks without having to register with the POSIX resource tracker +that automatically releases them upon process exit. + +.. + +.. date: 2023-10-11-02-34-01 +.. gh-issue: 110109 +.. nonce: RFCmHs +.. section: Library + +Add private ``pathlib._PurePathBase`` class: a base class for +:class:`pathlib.PurePath` that omits certain magic methods. It may be made +public (along with ``_PathBase``) in future. + +.. + +.. date: 2023-09-28-13-15-51 +.. gh-issue: 109858 +.. nonce: 43e2dg +.. section: Library + +Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises +BadZipFile when try to read an entry that overlaps with other entry or +central directory. + +.. + +.. date: 2023-09-23-14-40-51 +.. gh-issue: 109786 +.. nonce: UX3pKv +.. section: Library + +Fix possible reference leaks and crash when re-enter the ``__next__()`` +method of :class:`itertools.pairwise`. + +.. + +.. date: 2023-09-01-15-33-18 +.. gh-issue: 91539 +.. nonce: xoNLEI +.. section: Library + +Small (10 - 20%) and trivial performance improvement of +:func:`urllib.request.getproxies_environment`, typically useful when there +are many environment variables to go over. + +.. + +.. date: 2023-08-14-21-10-52 +.. gh-issue: 103363 +.. nonce: u64_QI +.. section: Library + +Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.owner` +and :meth:`~pathlib.Path.group`, defaulting to ``True``. + +.. + +.. date: 2023-08-07-21-11-24 +.. gh-issue: 102130 +.. nonce: _UyI5i +.. section: Library + +Support tab completion in :mod:`cmd` for ``editline``. + +.. + +.. date: 2023-08-04-18-43-21 +.. gh-issue: 99437 +.. nonce: Et8hu8 +.. section: Library + +:func:`runpy.run_path` now decodes path-like objects, making sure __file__ +and sys.argv[0] of the module being run are always strings. + +.. + +.. date: 2023-04-29-20-49-13 +.. gh-issue: 104003 +.. nonce: -8Ruk2 +.. section: Library + +Add :func:`warnings.deprecated`, a decorator to mark deprecated functions to +static type checkers and to warn on usage of deprecated classes and +functions. See :pep:`702`. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-04-23-11-08-02 +.. gh-issue: 103708 +.. nonce: Y17C7p +.. section: Library + +Make hardcoded python name, a configurable parameter so that different +implementations of python can override it instead of making huge diffs in +sysconfig.py + +.. + +.. date: 2023-04-09-21-05-43 +.. gh-issue: 66515 +.. nonce: 0DS8Ya +.. section: Library + +:class:`mailbox.MH` now supports folders that do not contain a +``.mh_sequences`` file (e.g. Claws Mail IMAP-cache folders). Patch by Serhiy +Storchaka. + +.. + +.. date: 2023-02-08-00-43-29 +.. gh-issue: 83162 +.. nonce: ufdI9F +.. section: Library + +Renamed :exc:`!re.error` to :exc:`PatternError` for clarity, and kept +:exc:`!re.error` for backward compatibility. Patch by Matthias Bussonnier +and Adam Chhina. + +.. + +.. date: 2022-12-01-16-57-44 +.. gh-issue: 91133 +.. nonce: LKMVCV +.. section: Library + +Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no +longer dereferences symlinks when working around file system permission +errors. + +.. + +.. bpo: 43153 +.. date: 2021-12-06-22-10-53 +.. nonce: J7mjSy +.. section: Library + +On Windows, ``tempfile.TemporaryDirectory`` previously masked a +``PermissionError`` with ``NotADirectoryError`` during directory cleanup. It +now correctly raises ``PermissionError`` if errors are not ignored. Patch by +Andrei Kulakov and Ken Jin. + +.. + +.. bpo: 32731 +.. date: 2021-11-23-22-22-49 +.. nonce: kNOASr +.. section: Library + +:func:`getpass.getuser` now raises :exc:`OSError` for all failures rather +than :exc:`ImportError` on systems lacking the :mod:`pwd` module or +:exc:`KeyError` if the password database is empty. + +.. + +.. bpo: 34321 +.. date: 2021-04-15-10-41-51 +.. nonce: 36m6_l +.. section: Library + +:class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is +``False``, the file descriptor specified by *fileno* will not be duplicated. + +.. + +.. bpo: 35332 +.. date: 2020-12-14-09-31-13 +.. nonce: s22wAx +.. section: Library + +The :func:`shutil.rmtree` function now ignores errors when calling +:func:`os.close` when *ignore_errors* is ``True``, and :func:`os.close` no +longer retried after error. + +.. + +.. bpo: 35928 +.. date: 2020-10-03-23-47-28 +.. nonce: E0iPAa +.. section: Library + +:class:`io.TextIOWrapper` now correctly handles the decoding buffer after +``read()`` and ``write()``. + +.. + +.. bpo: 26791 +.. date: 2020-08-06-14-43-55 +.. nonce: KxoEfO +.. section: Library + +:func:`shutil.move` now moves a symlink into a directory when that directory +is the target of the symlink. This provides the same behavior as the mv +shell command. The previous behavior raised an exception. Patch by Jeffrey +Kintscher. + +.. + +.. bpo: 41422 +.. date: 2020-07-28-20-48-05 +.. nonce: iMwnMu +.. section: Library + +Fixed memory leaks of :class:`pickle.Pickler` and :class:`pickle.Unpickler` +involving cyclic references via the internal memo mapping. + +.. + +.. bpo: 19821 +.. date: 2020-06-15-23-44-53 +.. nonce: ihBk39 +.. section: Library + +The :func:`!pydoc.ispackage` function has been deprecated. + +.. + +.. bpo: 40262 +.. date: 2020-05-21-23-32-46 +.. nonce: z4fQv1 +.. section: Library + +The :meth:`ssl.SSLSocket.recv_into` method no longer requires the *buffer* +argument to implement ``__len__`` and supports buffers with arbitrary item +size. + +.. + +.. bpo: 39912 +.. date: 2020-03-09-15-08-29 +.. nonce: xPOBBY +.. section: Library + +:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now +raise appropriate exceptions instead of ``AssertionError``. Patch +contributed by Rémi Lapeyre. + +.. + +.. bpo: 37260 +.. date: 2019-06-14-22-37-32 +.. nonce: oecdIf +.. section: Library + +Fixed a race condition in :func:`shutil.rmtree` in which directory entries +removed by another process or thread while ``shutil.rmtree()`` is running +can cause it to raise FileNotFoundError. Patch by Jeffrey Kintscher. + +.. + +.. bpo: 36959 +.. date: 2019-05-18-15-50-14 +.. nonce: ew6WZ4 +.. section: Library + +Fix some error messages for invalid ISO format string combinations in +``strptime()`` that referred to directives not contained in the format +string. Patch by Gordon P. Hemsley. + +.. + +.. bpo: 18060 +.. date: 2019-05-17-07-22-33 +.. nonce: 5mqTQM +.. section: Library + +Fixed a class inheritance issue that can cause segfaults when deriving two +or more levels of subclasses from a base class of Structure or Union. + +.. + +.. bpo: 29779 +.. date: 2019-05-08-13-14-11 +.. nonce: jg33dp +.. section: Library + +Add a new :envvar:`PYTHON_HISTORY` environment variable to set the location +of a ``.python_history`` file. + +.. + +.. bpo: 21360 +.. date: 2019-02-12-16-12-54 +.. nonce: gkSSfx +.. section: Library + +:class:`mailbox.Maildir` now ignores files with a leading dot. + +.. + +.. date: 2023-11-30-02-33-59 +.. gh-issue: 111699 +.. nonce: _O5G_y +.. section: Documentation + +Relocate ``smtpd`` deprecation notice to its own section rather than under +``locale`` in What's New in Python 3.12 document + +.. + +.. date: 2023-10-23-23-43-43 +.. gh-issue: 110746 +.. nonce: yg77IE +.. section: Documentation + +Improved markup for valid options/values for methods ttk.treeview.column and +ttk.treeview.heading, and for Layouts. + +.. + +.. date: 2023-08-01-13-11-39 +.. gh-issue: 95649 +.. nonce: F4KhPS +.. section: Documentation + +Document that the :mod:`asyncio` module contains code taken from `v0.16.0 of +the uvloop project `_, as +well as the required MIT licensing information. + +.. + +.. date: 2024-01-12-14-34-24 +.. gh-issue: 111798 +.. nonce: hd9B_- +.. section: Tests + +Disable ``test_super_deep()`` from ``test_call`` under pydebug builds on +WASI; the stack depth is too small to make the test useful. + +.. + +.. date: 2024-01-12-13-19-12 +.. gh-issue: 111801 +.. nonce: 9hh9DY +.. section: Tests + +Lower the recursion limit in ``test_isinstance`` for +``test_infinitely_many_bases()``. This prevents a stack overflow on a +pydebug build of WASI. + +.. + +.. date: 2024-01-12-12-45-24 +.. gh-issue: 111802 +.. nonce: gN41vt +.. section: Tests + +Specify a low recursion depth for ``test_bad_getattr()`` in +``test.pickletester`` to avoid exhausting the stack under a pydebug build +for WASI. + +.. + +.. date: 2024-01-08-21-15-48 +.. gh-issue: 44626 +.. nonce: DRq-PR +.. section: Tests + +Fix :func:`os.path.isabs` incorrectly returning ``True`` when given a path +that starts with exactly one (back)slash on Windows. + +Fix :meth:`pathlib.PureWindowsPath.is_absolute` incorrectly returning +``False`` for some paths beginning with two (back)slashes. + +.. + +.. date: 2024-01-01-14-40-02 +.. gh-issue: 113633 +.. nonce: VOY5ai +.. section: Tests + +Use module state for the _testcapi extension module. + +.. + +.. date: 2023-12-09-21-27-46 +.. gh-issue: 109980 +.. nonce: y--500 +.. section: Tests + +Fix ``test_tarfile_vs_tar`` in ``test_shutil`` for macOS, where system tar +can include more information in the archive than :mod:`shutil.make_archive`. + +.. + +.. date: 2023-12-05-19-50-03 +.. gh-issue: 112769 +.. nonce: kdLJmS +.. section: Tests + +The tests now correctly compare zlib version when +:const:`zlib.ZLIB_RUNTIME_VERSION` contains non-integer suffixes. For +example zlib-ng defines the version as ``1.3.0.zlib-ng``. + +.. + +.. date: 2023-12-04-15-56-11 +.. gh-issue: 112334 +.. nonce: FFc9Ti +.. section: Tests + +Adds a regression test to verify that ``vfork()`` is used when expected by +:mod:`subprocess` on vfork enabled POSIX systems (Linux). + +.. + +.. date: 2023-09-05-20-46-35 +.. gh-issue: 108927 +.. nonce: TpwWav +.. section: Tests + +Fixed order dependence in running tests in the same process when a test that +has submodules (e.g. test_importlib) follows a test that imports its +submodule (e.g. test_importlib.util) and precedes a test (e.g. test_unittest +or test_compileall) that uses that submodule. + +.. + +.. bpo: 40648 +.. date: 2020-05-16-18-00-21 +.. nonce: p2uPqy +.. section: Tests + +Test modes that file can get with chmod() on Windows. + +.. + +.. date: 2024-01-15-16-58-43 +.. gh-issue: 114013 +.. nonce: FoSeQf +.. section: Build + +Fix ``Tools/wasm/wasi.py`` to not include the path to ``python.wasm`` as +part of ``HOSTRUNNER``. The environment variable is meant to specify how to +run the WASI host only, having ``python.wasm`` and relevant flags appended +to the ``HOSTRUNNER``. This fixes ``make test`` work. + +.. + +.. date: 2023-12-23-09-35-48 +.. gh-issue: 113258 +.. nonce: GlsAyH +.. section: Build + +Changed the Windows build to write out generated frozen modules into the +build tree instead of the source tree. + +.. + +.. date: 2023-12-21-05-35-06 +.. gh-issue: 112305 +.. nonce: VfqQPx +.. section: Build + +Fixed the ``check-clean-src`` step performed on out of tree builds to detect +errant ``$(srcdir)/Python/frozen_modules/*.h`` files and recommend +appropriate source tree cleanup steps to get a working build again. + +.. + +.. date: 2023-12-17-18-23-02 +.. gh-issue: 112536 +.. nonce: 8lr3Ep +.. section: Build + +Add support for thread sanitizer (TSAN) + +.. + +.. date: 2023-12-08-11-33-37 +.. gh-issue: 112867 +.. nonce: ZzDfXQ +.. section: Build + +Fix the build for the case that WITH_PYMALLOC_RADIX_TREE=0 set. + +.. + +.. date: 2023-11-27-13-55-47 +.. gh-issue: 103065 +.. nonce: o72OiA +.. section: Build + +Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. + +.. + +.. bpo: 11102 +.. date: 2020-05-01-23-44-31 +.. nonce: Fw9zeS +.. section: Build + +The :func:`os.major`, :func:`os.makedev`, and :func:`os.minor` functions are +now available on HP-UX v3. + +.. + +.. bpo: 36351 +.. date: 2020-01-11-23-49-17 +.. nonce: ce8BBh +.. section: Build + +Do not set ipv6type when cross-compiling. + +.. + +.. date: 2024-01-15-23-53-25 +.. gh-issue: 114096 +.. nonce: G-Myja +.. section: Windows + +Process privileges that are activated for creating directory junctions are +now restored afterwards, avoiding behaviour changes in other parts of the +program. + +.. + +.. date: 2024-01-04-21-16-31 +.. gh-issue: 111877 +.. nonce: fR-B4c +.. section: Windows + +:func:`os.stat` calls were returning incorrect time values for files that +could not be accessed directly. + +.. + +.. date: 2023-12-19-10-56-46 +.. gh-issue: 111973 +.. nonce: A9Wtsb +.. section: Windows + +Update Windows installer to use SQLite 3.44.2. + +.. + +.. date: 2023-12-14-19-00-29 +.. gh-issue: 113009 +.. nonce: 6LNdjz +.. section: Windows + +:mod:`multiprocessing`: On Windows, fix a race condition in +``Process.terminate()``: no longer set the ``returncode`` attribute to +always call ``WaitForSingleObject()`` in ``Process.wait()``. Previously, +sometimes the process was still running after ``TerminateProcess()`` even if +``GetExitCodeProcess()`` is not ``STILL_ACTIVE``. Patch by Victor Stinner. + +.. + +.. date: 2023-12-12-20-58-09 +.. gh-issue: 86179 +.. nonce: YYSk_6 +.. section: Windows + +Fixes path calculations when launching Python on Windows through a symlink. + +.. + +.. date: 2023-12-11-20-23-04 +.. gh-issue: 71383 +.. nonce: 9pZh6t +.. section: Windows + +Update Tcl/Tk in Windows installer to 8.6.13 with a patch to suppress +incorrect ThemeChanged warnings. + +.. + +.. date: 2023-12-05-22-56-30 +.. gh-issue: 111650 +.. nonce: xlWmvM +.. section: Windows + +Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in +:file:`pyconfig.h` so that extension modules written in C are able to use +it. + +.. + +.. date: 2023-12-03-19-22-37 +.. gh-issue: 112278 +.. nonce: FiloCE +.. section: Windows + +Reduce the time cost for some functions in :mod:`platform` on Windows if +current user has no permission to the WMI. + +.. + +.. date: 2023-08-08-01-42-14 +.. gh-issue: 73427 +.. nonce: WOpiNt +.. section: Windows + +Deprecate :func:`sys._enablelegacywindowsfsencoding`. Use +:envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. Patch by Inada Naoki. + +.. + +.. date: 2023-03-15-23-53-45 +.. gh-issue: 87868 +.. nonce: 4C36oQ +.. section: Windows + +Correctly sort and remove duplicate environment variables in +:py:func:`!_winapi.CreateProcess`. + +.. + +.. bpo: 37308 +.. date: 2019-06-16-11-27-05 +.. nonce: Iz_NU_ +.. section: Windows + +Fix mojibake in :class:`mmap.mmap` when using a non-ASCII *tagname* argument +on Windows. + +.. + +.. date: 2024-01-02-22-25-21 +.. gh-issue: 113666 +.. nonce: xKZoBm +.. section: macOS + +Add the following constants to module :mod:`stat`: ``UF_SETTABLE``, +``UF_TRACKED``, ``UF_DATAVAULT``, ``SF_SUPPORTED``, ``SF_SETTABLE``, +``SF_SYNTHETIC``, ``SF_RESTRICTED``, ``SF_FIRMLINK`` and ``SF_DATALESS``. +The values ``UF_SETTABLE``, ``SF_SUPPORTED``, ``SF_SETTABLE`` and +``SF_SYNTHETIC`` are only available on macOS. + +.. + +.. date: 2023-12-28-12-18-39 +.. gh-issue: 113536 +.. nonce: 0ythg7 +.. section: macOS + +:func:`os.waitid` is now available on macOS + +.. + +.. date: 2023-12-23-22-41-07 +.. gh-issue: 110459 +.. nonce: NaMBJy +.. section: macOS + +Running ``configure ... --with-openssl-rpath=X/Y/Z`` no longer fails to +detect OpenSSL on macOS. + +.. + +.. date: 2023-12-21-11-53-47 +.. gh-issue: 74573 +.. nonce: MA6Vys +.. section: macOS + +Document that :mod:`dbm.ndbm` can silently corrupt DBM files on updates when +exceeding undocumented platform limits, and can crash (segmentation fault) +when reading such a corrupted file. (FB8919203) + +.. + +.. date: 2023-12-21-10-20-41 +.. gh-issue: 65701 +.. nonce: Q2hNbN +.. section: macOS + +The :program:`freeze` tool doesn't work with framework builds of Python. +Document this and bail out early when running the tool with such a build. + +.. + +.. date: 2023-12-21-09-41-42 +.. gh-issue: 87277 +.. nonce: IF6EZZ +.. section: macOS + +webbrowser: Don't look for X11 browsers on macOS. Those are generally not +used and probing for them can result in starting XQuartz even if it isn't +used otherwise. + +.. + +.. date: 2023-12-19-10-50-08 +.. gh-issue: 111973 +.. nonce: HMHJfy +.. section: macOS + +Update macOS installer to use SQLite 3.44.2. + +.. + +.. date: 2023-12-16-11-45-32 +.. gh-issue: 108269 +.. nonce: wVgCHF +.. section: macOS + +Set ``CFBundleAllowMixedLocalizations`` to true in the Info.plist for the +framework, embedded Python.app and IDLE.app with framework installs on +macOS. This allows applications to pick up the user's preferred locale when +that's different from english. + +.. + +.. date: 2023-12-10-20-30-06 +.. gh-issue: 102362 +.. nonce: y8svbF +.. section: macOS + +Make sure the result of :func:`sysconfig.get_plaform` includes at least a +major and minor versions, even if ``MACOSX_DEPLOYMENT_TARGET`` is set to +only a major version during build to match the format expected by pip. + +.. + +.. date: 2023-12-07-15-53-16 +.. gh-issue: 110017 +.. nonce: UMYzMR +.. section: macOS + +Disable a signal handling stress test on macOS due to a bug in macOS +(FB13453490). + +.. + +.. date: 2023-12-07-14-19-46 +.. gh-issue: 110820 +.. nonce: DIxb_F +.. section: macOS + +Make sure the preprocessor definitions for ``ALIGNOF_MAX_ALIGN_T``, +``SIZEOF_LONG_DOUBLE`` and ``HAVE_GCC_ASM_FOR_X64`` are correct for +Universal 2 builds on macOS. + +.. + +.. date: 2023-12-06-12-11-13 +.. gh-issue: 109981 +.. nonce: mOHg10 +.. section: macOS + +Use ``/dev/fd`` on macOS to determine the number of open files in +``test.support.os_helper.fd_count`` to avoid a crash with "guarded" file +descriptors when probing for open files. + +.. + +.. date: 2024-01-17-02-15-33 +.. gh-issue: 72284 +.. nonce: cAQiYO +.. section: IDLE + +Improve the lists of features, editor key bindings, and shell key bingings +in the IDLE doc. + +.. + +.. date: 2024-01-11-21-26-58 +.. gh-issue: 113903 +.. nonce: __GLlQ +.. section: IDLE + +Fix rare failure of test.test_idle, in test_configdialog. + +.. + +.. date: 2024-01-05-12-24-01 +.. gh-issue: 113729 +.. nonce: qpluea +.. section: IDLE + +Fix the "Help -> IDLE Doc" menu bug in 3.11.7 and 3.12.1. + +.. + +.. date: 2023-12-19-00-03-12 +.. gh-issue: 113269 +.. nonce: lrU-IC +.. section: IDLE + +Fix test_editor hang on macOS Catalina. + +.. + +.. date: 2023-12-10-20-01-11 +.. gh-issue: 112898 +.. nonce: 98aWv2 +.. section: IDLE + +Fix processing unsaved files when quitting IDLE on macOS. + +.. + +.. bpo: 13586 +.. date: 2019-12-13-12-26-56 +.. nonce: 1grqsR +.. section: IDLE + +Enter the selected text when opening the "Replace" dialog. + +.. + +.. date: 2023-12-02-02-08-11 +.. gh-issue: 106560 +.. nonce: THvuji +.. section: C API + +Fix redundant declarations in the public C API. Declare PyBool_Type, +PyLong_Type and PySys_Audit() only once. Patch by Victor Stinner. + +.. + +.. date: 2023-11-27-09-44-16 +.. gh-issue: 112438 +.. nonce: GdNZiI +.. section: C API + +Fix support of format units "es", "et", "es#", and "et#" in nested tuples in +:c:func:`PyArg_ParseTuple`-like functions. + +.. + +.. date: 2023-11-15-01-26-59 +.. gh-issue: 111545 +.. nonce: iAoFtA +.. section: C API + +Add :c:func:`Py_HashPointer` function to hash a pointer. Patch by Victor +Stinner. + +.. + +.. date: 2023-06-21-11-53-09 +.. gh-issue: 65210 +.. nonce: PhFRBJ +.. section: C API + +Change the declaration of the *keywords* parameter of +:c:func:`PyArg_ParseTupleAndKeywords` and +:c:func:`PyArg_VaParseTupleAndKeywords` for better compatibility with C++. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst new file mode 100644 index 00000000000000..5efc244c6086cc --- /dev/null +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -0,0 +1,1433 @@ +.. date: 2024-02-13-15-14-39 +.. gh-issue: 115399 +.. nonce: xT-scP +.. release date: 2024-02-15 +.. section: Security + +Update bundled libexpat to 2.6.0 + +.. + +.. date: 2024-02-12-00-33-01 +.. gh-issue: 115243 +.. nonce: e1oGX8 +.. section: Security + +Fix possible crashes in :meth:`collections.deque.index` when the deque is +concurrently modified. + +.. + +.. date: 2024-02-14-23-50-55 +.. gh-issue: 112087 +.. nonce: H_4W_v +.. section: Core and Builtins + +For an empty reverse iterator for list will be reduced to :func:`reversed`. +Patch by Donghee Na + +.. + +.. date: 2024-02-12-17-18-26 +.. gh-issue: 114570 +.. nonce: BzwMlJ +.. section: Core and Builtins + +Add :exc:`PythonFinalizationError` exception. This exception derived from +:exc:`RuntimeError` is raised when an operation is blocked during the +:term:`Python finalization `. Patch by Victor Stinner. + +.. + +.. date: 2024-02-07-18-04-36 +.. gh-issue: 114695 +.. nonce: o9wP5P +.. section: Core and Builtins + +Add :func:`sys._clear_internal_caches`, which clears all internal +performance-related caches (and deprecate the less-general +:func:`sys._clear_type_cache` function). + +.. + +.. date: 2024-02-07-07-50-12 +.. gh-issue: 114828 +.. nonce: nSXwMi +.. section: Core and Builtins + +Fix compilation crashes in uncommon code examples using :func:`super` inside +a comprehension in a class body. + +.. + +.. date: 2024-02-07-00-18-42 +.. gh-issue: 112069 +.. nonce: jRDRR5 +.. section: Core and Builtins + +Adapt :class:`set` and :class:`frozenset` methods to Argument Clinic. + +.. + +.. date: 2024-02-05-12-40-26 +.. gh-issue: 115011 +.. nonce: L1AKF5 +.. section: Core and Builtins + +Setters for members with an unsigned integer type now support the same range +of valid values for objects that has a :meth:`~object.__index__` method as +for :class:`int`. + +.. + +.. date: 2024-02-03-04-07-18 +.. gh-issue: 114887 +.. nonce: uLSFmN +.. section: Core and Builtins + +Changed socket type validation in +:meth:`~asyncio.loop.create_datagram_endpoint` to accept all non-stream +sockets. This fixes a regression in compatibility with raw sockets. + +.. + +.. date: 2024-02-03-01-48-38 +.. gh-issue: 114944 +.. nonce: 4J5ELD +.. section: Core and Builtins + +Fixes a race between ``PyParkingLot_Park`` and ``_PyParkingLot_UnparkAll``. + +.. + +.. date: 2024-02-02-05-27-48 +.. gh-issue: 113462 +.. nonce: VMml8q +.. section: Core and Builtins + +Limit the number of versions that a single class can use. Prevents a few +wayward classes using up all the version numbers. + +.. + +.. date: 2024-02-01-23-43-49 +.. gh-issue: 76763 +.. nonce: o_2J6i +.. section: Core and Builtins + +The :func:`chr` builtin function now always raises :exc:`ValueError` for +values outside the valid range. Previously it raised :exc:`OverflowError` +for very large or small values. + +.. + +.. date: 2024-02-01-18-16-52 +.. gh-issue: 114806 +.. nonce: wrH2J6 +.. section: Core and Builtins + +No longer specialize calls to classes, if those classes have metaclasses. +Fixes bug where the ``__call__`` method of the metaclass was not being +called. + +.. + +.. date: 2024-01-31-09-10-10 +.. gh-issue: 107944 +.. nonce: XWm1B- +.. section: Core and Builtins + +Improve error message for function calls with bad keyword arguments via +getargs + +.. + +.. date: 2024-01-25-18-50-49 +.. gh-issue: 112529 +.. nonce: IbbApA +.. section: Core and Builtins + +The free-threaded build no longer allocates space for the ``PyGC_Head`` +structure in objects that support cyclic garbage collection. A number of +other fields and data structures are used as replacements, including +``ob_gc_bits``, ``ob_tid``, and mimalloc internal data structures. + +.. + +.. date: 2024-01-22-15-10-01 +.. gh-issue: 114456 +.. nonce: fBFEJF +.. section: Core and Builtins + +Lower the recursion limit under a debug build of WASI. + +.. + +.. date: 2024-01-22-09-49-02 +.. gh-issue: 114083 +.. nonce: hf1-ku +.. section: Core and Builtins + +Compiler applies folding of LOAD_CONST with following instruction in a +separate pass before other optimisations. This enables jump threading in +certain circumstances. + +.. + +.. date: 2024-01-21-17-29-32 +.. gh-issue: 114388 +.. nonce: UVGO4K +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` emitted when assign an integer-like value that +is not an instance of :class:`int` to an attribute that corresponds to a C +struct member of :ref:`type ` T_UINT and T_ULONG. Fix a +double :exc:`RuntimeWarning` emitted when assign a negative integer value to +an attribute that corresponds to a C struct member of type T_UINT. + +.. + +.. date: 2024-01-19-13-18-13 +.. gh-issue: 114265 +.. nonce: 7HAi-- +.. section: Core and Builtins + +Compiler propagates line numbers before optimization, leading to more +optimization opportunities and removing the need for the +``guarantee_lineno_for_exits`` hack. + +.. + +.. date: 2024-01-18-20-20-37 +.. gh-issue: 112529 +.. nonce: oVNvDG +.. section: Core and Builtins + +The free-threaded build now has its own thread-safe GC implementation that +uses mimalloc to find GC tracked objects. It is non-generational, unlike the +existing GC implementation. + +.. + +.. date: 2024-01-17-23-39-20 +.. gh-issue: 114050 +.. nonce: Lnv1oq +.. section: Core and Builtins + +Fix segmentation fault caused by an incorrect format string in ``TypeError`` +exception when more than two arguments are passed to ``int``. + +.. + +.. date: 2024-01-17-05-09-32 +.. gh-issue: 112354 +.. nonce: Run9ko +.. section: Core and Builtins + +The ``END_FOR`` instruction now pops only one value. This is to better +support side exits in loops. + +.. + +.. date: 2024-01-17-00-52-57 +.. gh-issue: 113884 +.. nonce: CvEjUE +.. section: Core and Builtins + +Make :class:`queue.SimpleQueue` thread safe when the GIL is disabled. + +.. + +.. date: 2024-01-16-14-41-54 +.. gh-issue: 114058 +.. nonce: Cb2b8h +.. section: Core and Builtins + +Implement the foundations of the Tier 2 redundancy eliminator. + +.. + +.. date: 2024-01-12-16-40-07 +.. gh-issue: 113939 +.. nonce: Yi3L-e +.. section: Core and Builtins + +frame.clear(): Clear frame.f_locals as well, and not only the fast locals. +This is relevant once frame.f_locals was accessed, which would contain also +references to all the locals. + +.. + +.. date: 2024-01-11-22-58-45 +.. gh-issue: 112050 +.. nonce: hDuvDW +.. section: Core and Builtins + +Convert :class:`collections.deque` to use Argument Clinic. + +.. + +.. date: 2024-01-08-21-57-41 +.. gh-issue: 112050 +.. nonce: qwgjx1 +.. section: Core and Builtins + +Make methods on :class:`collections.deque` thread-safe when the GIL is +disabled. + +.. + +.. date: 2023-12-24-03-25-28 +.. gh-issue: 113464 +.. nonce: dvjQmA +.. section: Core and Builtins + +Add an option (``--enable-experimental-jit`` for ``configure``-based builds +or ``--experimental-jit`` for ``PCbuild``-based ones) to build an +*experimental* just-in-time compiler, based on `copy-and-patch +`_ + +.. + +.. date: 2023-12-22-13-21-39 +.. gh-issue: 113055 +.. nonce: 47xBMF +.. section: Core and Builtins + +Make interp->obmalloc a pointer. For interpreters that share state with the +main interpreter, this points to the same static memory structure. For +interpreters with their own obmalloc state, it is heap allocated. Add +free_obmalloc_arenas() which will free the obmalloc arenas and radix tree +structures for interpreters with their own obmalloc state. + +.. + +.. date: 2023-06-06-19-09-00 +.. gh-issue: 55664 +.. nonce: vYYl0V +.. section: Core and Builtins + +Add warning when creating :class:`type` using a namespace dictionary with +non-string keys. Patched by Daniel Urban and Furkan Onder. + +.. + +.. date: 2023-05-16-06-52-34 +.. gh-issue: 104530 +.. nonce: mJnA0W +.. section: Core and Builtins + +Use native Win32 condition variables. + +.. + +.. date: 2024-02-13-18-27-03 +.. gh-issue: 115392 +.. nonce: gle5tp +.. section: Library + +Fix a bug in :mod:`doctest` where incorrect line numbers would be reported +for decorated functions. + +.. + +.. date: 2024-02-11-20-23-36 +.. gh-issue: 114563 +.. nonce: RzxNYT +.. section: Library + +Fix several :func:`format()` bugs when using the C implementation of +:class:`~decimal.Decimal`: * memory leak in some rare cases when using the +``z`` format option (coerce negative 0) * incorrect output when applying the +``z`` format option to type ``F`` (fixed-point with capital ``NAN`` / +``INF``) * incorrect output when applying the ``#`` format option (alternate +form) + +.. + +.. date: 2024-02-10-15-24-20 +.. gh-issue: 102840 +.. nonce: 4mnDq1 +.. section: Library + +Fix confused traceback when floordiv, mod, or divmod operations happens +between instances of :class:`fractions.Fraction` and :class:`complex`. + +.. + +.. date: 2024-02-09-07-20-16 +.. gh-issue: 115165 +.. nonce: yfJLXA +.. section: Library + +Most exceptions are now ignored when attempting to set the +``__orig_class__`` attribute on objects returned when calling :mod:`typing` +generic aliases (including generic aliases created using +:data:`typing.Annotated`). Previously only :exc:`AttributeError` was +ignored. Patch by Dave Shawley. + +.. + +.. date: 2024-02-08-17-04-58 +.. gh-issue: 112903 +.. nonce: SN_vUs +.. section: Library + +Fix "issubclass() arg 1 must be a class" errors in certain cases of multiple +inheritance with generic aliases (regression in early 3.13 alpha releases). + +.. + +.. date: 2024-02-08-14-21-28 +.. gh-issue: 115133 +.. nonce: ycl4ko +.. section: Library + +Fix tests for :class:`~xml.etree.ElementTree.XMLPullParser` with Expat +2.6.0. + +.. + +.. date: 2024-02-08-13-26-14 +.. gh-issue: 115059 +.. nonce: DqP9dr +.. section: Library + +:meth:`io.BufferedRandom.read1` now flushes the underlying write buffer. + +.. + +.. date: 2024-02-07-12-37-52 +.. gh-issue: 79382 +.. nonce: Yz_5WB +.. section: Library + +Trailing ``**`` no longer allows to match files and non-existing paths in +recursive :func:`~glob.glob`. + +.. + +.. date: 2024-02-06-15-16-28 +.. gh-issue: 67837 +.. nonce: _JKa73 +.. section: Library + +Avoid race conditions in the creation of directories during concurrent +extraction in :mod:`tarfile` and :mod:`zipfile`. + +.. + +.. date: 2024-02-06-03-55-46 +.. gh-issue: 115060 +.. nonce: EkWRpP +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by removing redundant regex matching. + +.. + +.. date: 2024-02-05-16-48-06 +.. gh-issue: 97928 +.. nonce: JZCies +.. section: Library + +Partially revert the behavior of :meth:`tkinter.Text.count`. By default it +preserves the behavior of older Python versions, except that setting +``wantobjects`` to 0 no longer has effect. Add a new parameter +*return_ints*: specifying ``return_ints=True`` makes ``Text.count()`` always +returning the single count as an integer instead of a 1-tuple or ``None``. + +.. + +.. date: 2024-02-04-13-17-33 +.. gh-issue: 114628 +.. nonce: WJpqqS +.. section: Library + +When csv.Error is raised when handling TypeError, do not print the TypeError +traceback. + +.. + +.. date: 2024-02-04-02-28-37 +.. gh-issue: 85984 +.. nonce: NHZVTQ +.. section: Library + +Added ``_POSIX_VDISABLE`` from C's ```` to :mod:`termios`. + +.. + +.. date: 2024-02-03-17-54-17 +.. gh-issue: 114965 +.. nonce: gHksCK +.. section: Library + +Update bundled pip to 24.0 + +.. + +.. date: 2024-02-03-16-59-25 +.. gh-issue: 114959 +.. nonce: dCfAG2 +.. section: Library + +:mod:`tarfile` no longer ignores errors when trying to extract a directory +on top of a file. + +.. + +.. date: 2024-02-02-15-50-13 +.. gh-issue: 114894 +.. nonce: DF-dSd +.. section: Library + +Add :meth:`array.array.clear`. + +.. + +.. date: 2024-02-01-10-19-11 +.. gh-issue: 114071 +.. nonce: vkm2G_ +.. section: Library + +Support tuple subclasses using auto() for enum member value. + +.. + +.. date: 2024-01-31-20-07-11 +.. gh-issue: 109475 +.. nonce: lmTb9S +.. section: Library + +Fix support of explicit option value "--" in :mod:`argparse` (e.g. +``--option=--``). + +.. + +.. date: 2024-01-30-22-10-50 +.. gh-issue: 49766 +.. nonce: yulJL_ +.. section: Library + +Fix :class:`~datetime.date`-:class:`~datetime.datetime` comparison. Now the +special comparison methods like ``__eq__`` and ``__lt__`` return +:data:`NotImplemented` if one of comparands is :class:`!date` and other is +:class:`!datetime` instead of ignoring the time part and the time zone or +forcefully return "not equal" or raise :exc:`TypeError`. It makes comparison +of :class:`!date` and :class:`!datetime` subclasses more symmetric and +allows to change the default behavior by overriding the special comparison +methods in subclasses. + +.. + +.. date: 2024-01-30-15-34-08 +.. gh-issue: 110190 +.. nonce: Z5PQQX +.. section: Library + +Fix ctypes structs with array on Windows ARM64 platform by setting +``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo + +.. + +.. date: 2024-01-28-19-40-40 +.. gh-issue: 114678 +.. nonce: kYKcJw +.. section: Library + +Ensure that deprecation warning for 'N' specifier in +:class:`~decimal.Decimal` format is not raised for cases where 'N' appears +in other places in the format specifier. Based on patch by Stefan Krah. + +.. + +.. date: 2024-01-28-18-38-18 +.. gh-issue: 70303 +.. nonce: _Lt_pj +.. section: Library + +Return both files and directories from :meth:`pathlib.Path.glob` if a +pattern ends with "``**``". Previously only directories were returned. + +.. + +.. date: 2024-01-28-00-48-12 +.. gh-issue: 109653 +.. nonce: vF4exe +.. section: Library + +Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`. + +.. + +.. date: 2024-01-27-20-11-24 +.. gh-issue: 113280 +.. nonce: CZPQMf +.. section: Library + +Fix a leak of open socket in rare cases when error occurred in +:class:`ssl.SSLSocket` creation. + +.. + +.. date: 2024-01-26-16-46-21 +.. gh-issue: 77749 +.. nonce: NY_7TS +.. section: Library + +:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII +characters in headers if :attr:`~email.policy.EmailPolicy.utf8` is false. + +.. + +.. date: 2024-01-25-19-22-17 +.. gh-issue: 83383 +.. nonce: 3GwO9v +.. section: Library + +Synchronization of the :mod:`dbm.dumb` database is now no-op if there was no +modification since opening or last synchronization. The directory file for a +newly created empty :mod:`dbm.dumb` database is now created immediately +after opening instead of deferring this until synchronizing or closing. + +.. + +.. date: 2024-01-24-20-51-49 +.. gh-issue: 91602 +.. nonce: 8fOH8l +.. section: Library + +Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` +for filtering database objects to dump. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-20-11-46 +.. gh-issue: 112451 +.. nonce: 7YrG4p +.. section: Library + +Prohibit subclassing pure-Python :class:`datetime.timezone`. This is +consistent with C-extension implementation. Patch by Mariusz Felisiak. + +.. + +.. date: 2024-01-24-17-25-18 +.. gh-issue: 69893 +.. nonce: PQq5fR +.. section: Library + +Add the :meth:`!close` method for the iterator returned by +:func:`xml.etree.ElementTree.iterparse`. + +.. + +.. date: 2024-01-23-23-13-47 +.. gh-issue: 109653 +.. nonce: KLBHmT +.. section: Library + +Reduce the import time of :mod:`threading` module by ~50%. Patch by Daniel +Hollas. + +.. + +.. date: 2024-01-23-21-20-40 +.. gh-issue: 114492 +.. nonce: vKxl5o +.. section: Library + +Make the result of :func:`termios.tcgetattr` reproducible on Alpine Linux. +Previously it could leave a random garbage in some fields. + +.. + +.. date: 2024-01-23-14-11-49 +.. gh-issue: 114315 +.. nonce: KeVdzl +.. section: Library + +Make :class:`threading.Lock` a real class, not a factory function. Add +``__new__`` to ``_thread.lock`` type. + +.. + +.. date: 2024-01-23-13-03-22 +.. gh-issue: 100414 +.. nonce: 5kTdU5 +.. section: Library + +Add :mod:`dbm.sqlite3` as a backend to :mod:`dbm`, and make it the new +default :mod:`!dbm` backend. Patch by Raymond Hettinger and Erlend E. +Aasland. + +.. + +.. date: 2024-01-23-11-04-21 +.. gh-issue: 113267 +.. nonce: xe_Pxe +.. section: Library + +Revert changes in :gh:`106584` which made calls of ``TestResult`` methods +``startTest()`` and ``stopTest()`` unbalanced. + +.. + +.. date: 2024-01-22-12-10-34 +.. gh-issue: 75128 +.. nonce: 4FGlRS +.. section: Library + +Ignore an :exc:`OSError` in :meth:`asyncio.BaseEventLoop.create_server` when +IPv6 is available but the interface cannot actually support it. + +.. + +.. date: 2024-01-22-11-43-38 +.. gh-issue: 114423 +.. nonce: 6mMoPH +.. section: Library + +``_DummyThread`` entries in ``threading._active`` are now automatically +removed when the related thread dies. + +.. + +.. date: 2024-01-21-16-32-55 +.. gh-issue: 114257 +.. nonce: bCFld5 +.. section: Library + +Dismiss the :exc:`FileNotFound` error in :func:`ctypes.util.find_library` +and just return ``None`` on Linux. + +.. + +.. date: 2024-01-19-18-41-02 +.. gh-issue: 114321 +.. nonce: yj_Xw3 +.. section: Library + +Expose more platform specific constants in the :mod:`fcntl` module on Linux, +macOS, FreeBSD and NetBSD. + +.. + +.. date: 2024-01-19-15-48-06 +.. gh-issue: 114328 +.. nonce: hixxW3 +.. section: Library + +The :func:`tty.setcbreak` and new :func:`tty.cfmakecbreak` no longer clears +the terminal input ICRLF flag. This fixes a regression introduced in 3.12 +that no longer matched how OSes define cbreak mode in their ``stty(1)`` +manual pages. + +.. + +.. date: 2024-01-19-12-05-22 +.. gh-issue: 114281 +.. nonce: H5JQe4 +.. section: Library + +Remove type hints from ``Lib/asyncio/staggered.py``. The annotations in the +`typeshed `__ project should be used +instead. + +.. + +.. date: 2024-01-18-22-29-28 +.. gh-issue: 101438 +.. nonce: 1-uUi_ +.. section: Library + +Avoid reference cycle in ElementTree.iterparse. The iterator returned by +``ElementTree.iterparse`` may hold on to a file descriptor. The reference +cycle prevented prompt clean-up of the file descriptor if the returned +iterator was not exhausted. + +.. + +.. date: 2024-01-18-10-07-52 +.. gh-issue: 114198 +.. nonce: lK4Iif +.. section: Library + +The signature for the ``__replace__`` method on :mod:`dataclasses` now has +the first argument named ``self``, rather than ``obj``. + +.. + +.. date: 2024-01-17-18-53-51 +.. gh-issue: 104522 +.. nonce: 3NyDf4 +.. section: Library + +:exc:`OSError` raised when run a subprocess now only has *filename* +attribute set to *cwd* if the error was caused by a failed attempt to change +the current directory. + +.. + +.. date: 2024-01-16-15-59-06 +.. gh-issue: 114149 +.. nonce: LJ8IPm +.. section: Library + +Enum: correctly handle tuple subclasses in custom ``__new__``. + +.. + +.. date: 2024-01-15-20-21-33 +.. gh-issue: 83648 +.. nonce: HzD_fY +.. section: Library + +Support deprecation of options, positional arguments and subcommands in +:mod:`argparse`. + +.. + +.. date: 2024-01-15-19-54-41 +.. gh-issue: 114087 +.. nonce: Xic5vY +.. section: Library + +Speed up ``dataclasses.asdict`` up to 1.35x. + +.. + +.. date: 2024-01-15-18-42-44 +.. gh-issue: 109534 +.. nonce: wYaLMZ +.. section: Library + +Fix a reference leak in +:class:`asyncio.selector_events.BaseSelectorEventLoop` when SSL handshakes +fail. Patch contributed by Jamie Phan. + +.. + +.. date: 2024-01-12-17-32-36 +.. gh-issue: 79634 +.. nonce: uTSTRI +.. section: Library + +Accept :term:`path-like objects ` as patterns in +:meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob`. + +.. + +.. date: 2024-01-12-09-35-07 +.. gh-issue: 112202 +.. nonce: t_0V1m +.. section: Library + +Ensure that a :func:`asyncio.Condition.notify` call does not get lost if the +awakened ``Task`` is simultaneously cancelled or encounters any other error. + +.. + +.. date: 2024-01-11-20-47-49 +.. gh-issue: 113951 +.. nonce: AzlqFK +.. section: Library + +Fix the behavior of ``tag_unbind()`` methods of :class:`tkinter.Text` and +:class:`tkinter.Canvas` classes with three arguments. Previously, +``widget.tag_unbind(tag, sequence, funcid)`` destroyed the current binding +for *sequence*, leaving *sequence* unbound, and deleted the *funcid* +command. Now it removes only *funcid* from the binding for *sequence*, +keeping other commands, and deletes the *funcid* command. It leaves +*sequence* unbound only if *funcid* was the last bound command. + +.. + +.. date: 2024-01-11-15-10-53 +.. gh-issue: 97959 +.. nonce: UOj6d4 +.. section: Library + +Fix rendering class methods, bound methods, method and function aliases in +:mod:`pydoc`. Class methods no longer have "method of builtins.type +instance" note. Corresponding notes are now added for class and unbound +methods. Method and function aliases now have references to the module or +the class where the origin was defined if it differs from the current. Bound +methods are now listed in the static methods section. Methods of builtin +classes are now supported as well as methods of Python classes. + +.. + +.. date: 2024-01-07-21-04-24 +.. gh-issue: 113796 +.. nonce: 6iNsCR +.. section: Library + +Add more validation checks in the :class:`csv.Dialect` constructor. +:exc:`ValueError` is now raised if the same character is used in different +roles. + +.. + +.. date: 2024-01-05-16-27-34 +.. gh-issue: 113732 +.. nonce: fgDRXA +.. section: Library + +Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in +:func:`csv.reader`. + +.. + +.. date: 2024-01-04-20-58-17 +.. gh-issue: 113225 +.. nonce: -nyJM4 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by using :attr:`os.DirEntry.path` where +possible. + +.. + +.. date: 2023-12-18-20-10-50 +.. gh-issue: 89039 +.. nonce: gqFdtU +.. section: Library + +When replace() method is called on a subclass of datetime, date or time, +properly call derived constructor. Previously, only the base class's +constructor was called. + +Also, make sure to pass non-zero fold values when creating subclasses in +various methods. Previously, fold was silently ignored. + +.. + +.. date: 2023-12-09-23-31-17 +.. gh-issue: 112919 +.. nonce: S5k9QN +.. section: Library + +Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` +and :func:`datetime.time.replace`. + +.. + +.. date: 2023-11-27-19-54-43 +.. gh-issue: 59013 +.. nonce: chpQ0e +.. section: Library + +Set breakpoint on the first executable line of the function, instead of the +line of function definition when the user do ``break func`` using :mod:`pdb` + +.. + +.. date: 2023-11-24-19-08-50 +.. gh-issue: 112343 +.. nonce: RarGFC +.. section: Library + +Improve handling of pdb convenience variables to avoid replacing string +contents. + +.. + +.. date: 2023-11-18-16-30-21 +.. gh-issue: 112240 +.. nonce: YXS0tj +.. section: Library + +Add option to calendar module CLI to specify the weekday to start each week. +Patch by Steven Ward. + +.. + +.. date: 2023-11-04-22-32-27 +.. gh-issue: 111741 +.. nonce: f1ufr8 +.. section: Library + +Recognise ``image/webp`` as a standard format in the :mod:`mimetypes` +module. + +.. + +.. date: 2023-10-27-19-24-58 +.. gh-issue: 43457 +.. nonce: 84lx9H +.. section: Library + +Fix the :mod:`tkinter` widget method :meth:`!wm_attributes`. It now accepts +the attribute name without the minus prefix to get window attributes and +allows to specify attributes and values to set as keyword arguments. Add new +optional keyword argument *return_python_dict*: calling +``w.wm_attributes(return_python_dict=True)`` returns the attributes as a +dict instead of a tuple. Calling ``w.wm_attributes()`` now returns a tuple +instead of string if *wantobjects* was set to 0. + +.. + +.. date: 2023-10-24-19-19-54 +.. gh-issue: 82626 +.. nonce: _hfLRf +.. section: Library + +Many functions now emit a warning if a boolean value is passed as a file +descriptor argument. + +.. + +.. date: 2023-10-19-02-08-12 +.. gh-issue: 111051 +.. nonce: 8h1Dpk +.. section: Library + +Added check for file modification during debugging with :mod:`pdb` + +.. + +.. date: 2023-10-04-11-09-30 +.. gh-issue: 110345 +.. nonce: fZU1ud +.. section: Library + +Show the Tcl/Tk patchlevel (rather than version) in :meth:`tkinter._test`. + +.. + +.. date: 2023-09-22-22-17-45 +.. gh-issue: 38807 +.. nonce: m9McRN +.. section: Library + +Fix race condition in :mod:`trace`. Instead of checking if a directory +exists and creating it, directly call :func:`os.makedirs` with the kwarg +``exist_ok=True``. + +.. + +.. date: 2023-07-23-12-28-26 +.. gh-issue: 75705 +.. nonce: aB2-Ww +.. section: Library + +Set unixfrom envelope in :class:`mailbox.mbox` and :class:`mailbox.MMDF`. + +.. + +.. date: 2023-06-29-14-26-56 +.. gh-issue: 106233 +.. nonce: Aqw2HI +.. section: Library + +Fix stacklevel in ``InvalidTZPathWarning`` during :mod:`zoneinfo` module +import. + +.. + +.. date: 2023-05-30-18-30-11 +.. gh-issue: 105102 +.. nonce: SnpK04 +.. section: Library + +Allow :class:`ctypes.Union` to be nested in :class:`ctypes.Structure` when +the system endianness is the opposite of the classes. + +.. + +.. date: 2023-05-08-09-30-00 +.. gh-issue: 104282 +.. nonce: h4c6Eb +.. section: Library + +Fix null pointer dereference in :func:`lzma._decode_filter_properties` due +to improper handling of BCJ filters with properties of zero length. Patch by +Radislav Chugunov. + +.. + +.. date: 2023-05-06-04-57-10 +.. gh-issue: 96471 +.. nonce: C9wAU7 +.. section: Library + +Add :py:class:`queue.Queue` termination with +:py:meth:`~queue.Queue.shutdown`. + +.. + +.. date: 2023-04-08-11-41-07 +.. gh-issue: 101599 +.. nonce: PaWNFh +.. section: Library + +Changed argparse flag options formatting to remove redundancy. + +.. + +.. date: 2023-03-15-03-21-18 +.. gh-issue: 85984 +.. nonce: Xaq6ZN +.. section: Library + +Add POSIX pseudo-terminal functions :func:`os.posix_openpt`, +:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`. + +.. + +.. date: 2023-03-08-00-02-30 +.. gh-issue: 102512 +.. nonce: LiugDr +.. section: Library + +When :func:`os.fork` is called from a foreign thread (aka ``_DummyThread``), +the type of the thread in a child process is changed to ``_MainThread``. +Also changed its name and daemonic status, it can be now joined. + +.. + +.. date: 2022-07-31-01-24-40 +.. gh-issue: 88569 +.. nonce: eU0--b +.. section: Library + +Add :func:`os.path.isreserved`, which identifies reserved pathnames such as +"NUL", "AUX" and "CON". This function is only available on Windows. + +Deprecate :meth:`pathlib.PurePath.is_reserved`. + +.. + +.. bpo: 38364 +.. date: 2019-10-05-22-56-50 +.. nonce: sYTCWF +.. section: Library + +The ``inspect`` functions ``isgeneratorfunction``, ``iscoroutinefunction``, +``isasyncgenfunction`` now support ``functools.partialmethod`` wrapped +functions the same way they support ``functools.partial``. + +.. + +.. date: 2024-02-12-12-26-17 +.. gh-issue: 115233 +.. nonce: aug6r9 +.. section: Documentation + +Fix an example for :class:`~logging.LoggerAdapter` in the Logging Cookbook. + +.. + +.. date: 2024-01-17-11-40-03 +.. gh-issue: 114123 +.. nonce: LuueXf +.. section: Documentation + +Move the :mod:`csv` module docstring to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove ``__doc__`` +from ``csv.__all__``. + +Move :attr:`!csv.__version__` to the :mod:`!csv` module instead of +reexporting it from the internal :mod:`!_csv` module, and remove +``__version__`` from ``csv.__all__``. + +.. + +.. date: 2024-02-02-13-18-55 +.. gh-issue: 114099 +.. nonce: C_ycWg +.. section: Tests + +Added test exclusions required to run the test suite on iOS. + +.. + +.. date: 2023-06-02-05-04-15 +.. gh-issue: 105089 +.. nonce: KaZFtU +.. section: Tests + +Fix +``test.test_zipfile.test_core.TestWithDirectory.test_create_directory_with_write`` +test in AIX by doing a bitwise AND of 0xFFFF on mode , so that it will be in +sync with ``zinfo.external_attr`` + +.. + +.. date: 2024-02-08-19-36-20 +.. gh-issue: 115167 +.. nonce: LB9nDK +.. section: Build + +Avoid vendoring ``vcruntime140_threads.dll`` when building with Visual +Studio 2022 version 17.8. + +.. + +.. date: 2024-02-08-17-38-56 +.. gh-issue: 113632 +.. nonce: y9KIGb +.. section: Build + +Promote WASI to a tier 2 platform and drop Emscripten from tier 3 in +configure.ac. + +.. + +.. date: 2024-02-07-08-23-48 +.. gh-issue: 114099 +.. nonce: XcEXEZ +.. section: Build + +configure and Makefile were refactored to accommodate framework builds on +Apple platforms other than macOS. + +.. + +.. date: 2024-02-01-20-08-11 +.. gh-issue: 114875 +.. nonce: x_2iZ9 +.. section: Build + +Add :c:func:`!getgrent` as a prerequisite for building the :mod:`grp` +module. + +.. + +.. date: 2024-02-08-21-37-22 +.. gh-issue: 115049 +.. nonce: X1ObpJ +.. section: Windows + +Fixes ``py.exe`` launcher failing when run as users without user profiles. + +.. + +.. date: 2024-02-06-09-05-13 +.. gh-issue: 115009 +.. nonce: ShMjZs +.. section: Windows + +Update Windows installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-16-53-12 +.. gh-issue: 109991 +.. nonce: YqjnDz +.. section: Windows + +Update Windows build to use OpenSSL 3.0.13. + +.. + +.. date: 2024-02-01-14-35-05 +.. gh-issue: 111239 +.. nonce: SO7SUF +.. section: Windows + +Update Windows builds to use zlib v1.3.1. + +.. + +.. date: 2024-01-23-00-05-05 +.. gh-issue: 100107 +.. nonce: lkbP_Q +.. section: Windows + +The ``py.exe`` launcher will no longer attempt to run the Microsoft Store +redirector when launching a script containing a ``/usr/bin/env`` shebang + +.. + +.. date: 2023-12-19-22-32-28 +.. gh-issue: 112984 +.. nonce: F7kFMl +.. section: Windows + +Adds free-threaded binaries to Windows installer as an optional component. + +.. + +.. date: 2023-08-11-18-21-38 +.. gh-issue: 89240 +.. nonce: dtSOLG +.. section: Windows + +Allows :mod:`multiprocessing` to create pools of greater than 62 processes. + +.. + +.. date: 2024-02-06-09-01-10 +.. gh-issue: 115009 +.. nonce: ysau7e +.. section: macOS + +Update macOS installer to use SQLite 3.45.1. + +.. + +.. date: 2024-02-05-18-30-27 +.. gh-issue: 109991 +.. nonce: tun6Yu +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.13. + +.. + +.. date: 2024-01-23-11-35-26 +.. gh-issue: 114490 +.. nonce: FrQOQ0 +.. section: macOS + +Add Mach-O linkage support for :func:`platform.architecture()`. + +.. + +.. date: 2022-11-18-10-05-35 +.. gh-issue: 87804 +.. nonce: rhlDmD +.. section: macOS + +On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly +report the size of very large disks, in previous versions the reported +number of blocks was wrong for disks with at least 2**32 blocks. + +.. + +.. date: 2024-01-17-23-18-15 +.. gh-issue: 96905 +.. nonce: UYaxoU +.. section: IDLE + +In idlelib code, stop redefining built-ins 'dict' and 'object'. + +.. + +.. date: 2023-04-25-03-01-23 +.. gh-issue: 103820 +.. nonce: LCSpza +.. section: IDLE + +Revise IDLE bindings so that events from mouse button 4/5 on non-X11 +windowing systems (i.e. Win32 and Aqua) are not mistaken for scrolling. + +.. + +.. date: 2024-02-14-15-58-13 +.. gh-issue: 113516 +.. nonce: TyIHWx +.. section: Tools/Demos + +Don't set ``LDSHARED`` when building for WASI. + +.. + +.. date: 2024-02-05-19-00-32 +.. gh-issue: 109991 +.. nonce: yJSEkw +.. section: Tools/Demos + +Update GitHub CI workflows to use OpenSSL 3.0.13 and multissltests to use +1.1.1w, 3.0.13, 3.1.5, and 3.2.1. + +.. + +.. date: 2024-02-05-02-45-51 +.. gh-issue: 115015 +.. nonce: rgtiDB +.. section: Tools/Demos + +Fix a bug in Argument Clinic that generated incorrect code for methods with +no parameters that use the :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS +` calling convention. Only the +positional parameter count was checked; any keyword argument passed would be +silently accepted. + +.. + +.. date: 2024-02-05-17-11-15 +.. gh-issue: 111140 +.. nonce: WMEjid +.. section: C API + +Adds :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and +:c:func:`PyLong_FromUnsignedNativeBytes` functions. + +.. + +.. date: 2024-01-31-15-43-35 +.. gh-issue: 114685 +.. nonce: n7aRmX +.. section: C API + +:c:func:`PyBuffer_FillInfo` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-29-12-13-24 +.. gh-issue: 114685 +.. nonce: B07RME +.. section: C API + +:c:func:`PyObject_GetBuffer` now raises a :exc:`SystemError` if called with +:c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE` as flags. These flags should +only be used with the ``PyMemoryView_*`` C API. + +.. + +.. date: 2024-01-26-21-54-42 +.. gh-issue: 114626 +.. nonce: SKhbh_ +.. section: C API + +Add ``PyCFunctionFast`` and ``PyCFunctionFastWithKeywords`` typedefs +(identical to the existing ``_PyCFunctionFast`` and +``_PyCFunctionFastWithKeywords`` typedefs, just without a leading ``_`` +prefix). + +.. + +.. date: 2024-01-23-21-45-02 +.. gh-issue: 114329 +.. nonce: YRaBoe +.. section: C API + +Add :c:func:`PyList_GetItemRef`, which is similar to +:c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of a +:term:`borrowed reference`. + +.. + +.. date: 2023-11-16-02-07-48 +.. gh-issue: 110850 +.. nonce: DQGNfF +.. section: C API + +Add PyTime C API: + +* :c:type:`PyTime_t` type. +* :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. +* :c:func:`PyTime_AsSecondsDouble`, + :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and + :c:func:`PyTime_Time` functions. + +Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-13-47-48 +.. gh-issue: 112066 +.. nonce: 22WsqR +.. section: C API + +Add :c:func:`PyDict_SetDefaultRef`: insert a key and value into a dictionary +if the key is not already present. This is similar to +:meth:`dict.setdefault`, but returns an integer value indicating if the key +was already present. It is also similar to :c:func:`PyDict_SetDefault`, but +returns a strong reference instead of a borrowed reference. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst new file mode 100644 index 00000000000000..d8cc88c8756a17 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -0,0 +1,1163 @@ +.. date: 2024-02-18-03-14-40 +.. gh-issue: 115398 +.. nonce: tzvxH8 +.. release date: 2024-03-12 +.. section: Security + +Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` + +.. + +.. date: 2024-01-26-22-14-09 +.. gh-issue: 114572 +.. nonce: t1QMQD +.. section: Security + +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. + +.. + +.. date: 2024-03-11-22-24-59 +.. gh-issue: 116604 +.. nonce: LCEzAT +.. section: Core and Builtins + +Respect the status of the garbage collector when indirect calls are made via +:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo +Galindo + +.. + +.. date: 2024-03-09-11-10-53 +.. gh-issue: 112087 +.. nonce: nbI0Pw +.. section: Core and Builtins + +:class:`list` is now compatible with the implementation of :pep:`703`. + +.. + +.. date: 2024-03-05-22-00-58 +.. gh-issue: 116381 +.. nonce: 0Nq9iO +.. section: Core and Builtins + +Add specialization for ``CONTAINS_OP``. + +.. + +.. date: 2024-03-04-10-19-51 +.. gh-issue: 116296 +.. nonce: gvtxyU +.. section: Core and Builtins + +Fix possible refleak in :meth:`!object.__reduce__` internal error handling. + +.. + +.. date: 2024-02-22-16-17-53 +.. gh-issue: 115823 +.. nonce: c1TreJ +.. section: Core and Builtins + +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo + +.. + +.. date: 2024-02-22-11-33-20 +.. gh-issue: 115778 +.. nonce: jksd1D +.. section: Core and Builtins + +Add ``tierN`` annotation for instruction definition in interpreter DSL. + +.. + +.. date: 2024-02-20-18-49-02 +.. gh-issue: 115733 +.. nonce: 51Zb85 +.. section: Core and Builtins + +Fix crash when calling ``next()`` on exhausted list iterators. + +.. + +.. date: 2024-02-20-12-46-20 +.. gh-issue: 115700 +.. nonce: KLJ5r4 +.. section: Core and Builtins + +The regen-cases build stage now works on Windows. + +.. + +.. date: 2024-02-14-23-50-43 +.. gh-issue: 115347 +.. nonce: VkHvQC +.. section: Core and Builtins + +Fix bug where docstring was replaced by a redundant NOP when Python is run +with ``-OO``. + +.. + +.. date: 2024-02-12-23-29-17 +.. gh-issue: 115323 +.. nonce: 3t6687 +.. section: Core and Builtins + +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. + +.. + +.. date: 2024-02-09-18-59-22 +.. gh-issue: 112175 +.. nonce: qglugr +.. section: Core and Builtins + +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific +threads to be interrupted. + +.. + +.. date: 2024-02-08-16-01-18 +.. gh-issue: 115154 +.. nonce: ji96FV +.. section: Core and Builtins + +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo + +.. + +.. date: 2024-01-28-02-46-12 +.. gh-issue: 112433 +.. nonce: FUX-nT +.. section: Core and Builtins + +Add ability to force alignment of :mod:`ctypes.Structure` by way of the new +``_align_`` attribute on the class. + +.. + +.. date: 2023-07-16-15-02-47 +.. gh-issue: 104090 +.. nonce: oMjNa9 +.. section: Core and Builtins + +The multiprocessing resource tracker now exits with non-zero status code if +a resource leak was detected. It still exits with status code 0 otherwise. + +.. + +.. date: 2023-06-16-21-29-06 +.. gh-issue: 105858 +.. nonce: Q7h0EV +.. section: Core and Builtins + +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to +``None``. AST nodes now have an ``__annotations__`` attribute with the +expected types of their attributes. Passing unrecognized extra arguments to +AST nodes is deprecated and will become an error in Python 3.15. Omitting a +required argument to an AST node is deprecated and will become an error in +Python 3.15. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-02-13-11-36-50 +.. gh-issue: 101860 +.. nonce: CKCMbC +.. section: Core and Builtins + +Expose ``__name__`` attribute on property. + +.. + +.. date: 2022-09-04-16-51-56 +.. gh-issue: 96497 +.. nonce: HTBuIL +.. section: Core and Builtins + +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. + +.. + +.. date: 2024-03-11-12-11-10 +.. gh-issue: 116600 +.. nonce: FcNBy_ +.. section: Library + +Fix :func:`repr` for global :class:`~enum.Flag` members. + +.. + +.. date: 2024-03-07-21-57-50 +.. gh-issue: 116349 +.. nonce: fD2pbP +.. section: Library + +:func:`platform.java_ver` is deprecated and will be removed in 3.15. It was +largely untested, had a confusing API, and was only useful for Jython +support. + +.. + +.. date: 2024-03-05-20-53-34 +.. gh-issue: 116143 +.. nonce: sww6Zl +.. section: Library + +Fix a race in pydoc ``_start_server``, eliminating a window in which +``_start_server`` can return a thread that is "serving" but without a +``docserver`` set. + +.. + +.. date: 2024-03-05-14-34-22 +.. gh-issue: 116127 +.. nonce: 5uktu3 +.. section: Library + +:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` +support to :class:`typing.TypedDict`. + +.. + +.. date: 2024-03-05-02-09-18 +.. gh-issue: 116325 +.. nonce: FmlBYv +.. section: Library + +:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` on +forward references as empty strings. + +.. + +.. date: 2024-03-02-11-31-49 +.. gh-issue: 115957 +.. nonce: C-3Z_U +.. section: Library + +When ``asyncio.TaskGroup.create_task`` is called on an inactive +``asyncio.TaskGroup``, the given coroutine will be closed (which prevents a +``RuntimeWarning``). + +.. + +.. date: 2024-03-01-14-22-08 +.. gh-issue: 115978 +.. nonce: r2ePTo +.. section: Library + +Disable preadv(), readv(), pwritev(), and writev() on WASI. + +Under wasmtime for WASI 0.2, these functions don't pass test_posix +(https://github.com/bytecodealliance/wasmtime/issues/7830). + +.. + +.. date: 2024-03-01-11-57-32 +.. gh-issue: 88352 +.. nonce: bZ68rw +.. section: Library + +Fix the computation of the next rollover time in the +:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` +now always returns a timestamp larger than the specified time and works +correctly during the DST change. :meth:`!doRollover` no longer overwrite the +already rolled over file, saving from data loss when run at midnight or +during repeated time at the DST change. + +.. + +.. date: 2024-02-29-20-06-06 +.. gh-issue: 87115 +.. nonce: FVMiOR +.. section: Library + +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` + +.. + +.. date: 2024-02-29-17-06-54 +.. gh-issue: 76511 +.. nonce: WqjRLP +.. section: Library + +Fix UnicodeEncodeError in :meth:`email.Message.as_string` that results when +a message that claims to be in the ascii character set actually has +non-ascii characters. Non-ascii characters are now replaced with the U+FFFD +replacement character, like in the ``replace`` error handler. + +.. + +.. date: 2024-02-28-17-50-42 +.. gh-issue: 89547 +.. nonce: GetF38 +.. section: Library + +Add support for nested typing special forms like Final[ClassVar[int]]. + +.. + +.. date: 2024-02-28-17-04-28 +.. gh-issue: 65824 +.. nonce: gG8KR1 +.. section: Library + +Improve the ``less`` prompt in :mod:`pydoc`. + +.. + +.. date: 2024-02-28-13-10-17 +.. gh-issue: 116040 +.. nonce: wDidHd +.. section: Library + +[Enum] fix by-value calls when second value is falsey; e.g. Cardinal(1, 0) + +.. + +.. date: 2024-02-28-12-14-31 +.. gh-issue: 115821 +.. nonce: YO2vKA +.. section: Library + +[Enum] Improve error message when calling super().__new__() in custom +__new__. + +.. + +.. date: 2024-02-27-20-11-29 +.. gh-issue: 85644 +.. nonce: 3rgcBm +.. section: Library + +Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to +check desktop. Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for +GNOME detection. + +.. + +.. date: 2024-02-27-13-05-51 +.. gh-issue: 75988 +.. nonce: In6LlB +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to pass the call through to the +wrapped object to return the real result. + +.. + +.. date: 2024-02-25-19-20-05 +.. gh-issue: 115881 +.. nonce: ro_Kuw +.. section: Library + +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the grammar +made as part of gh-94949. + +.. + +.. date: 2024-02-24-18-48-14 +.. gh-issue: 115886 +.. nonce: rgM6AF +.. section: Library + +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. + +.. + +.. date: 2024-02-23-11-08-31 +.. gh-issue: 115532 +.. nonce: zVd3gK +.. section: Library + +Add kernel density estimation to the statistics module. + +.. + +.. date: 2024-02-22-12-10-18 +.. gh-issue: 115714 +.. nonce: P2JsU1 +.. section: Library + +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. + +.. + +.. date: 2024-02-22-11-29-27 +.. gh-issue: 115809 +.. nonce: 9H1DhB +.. section: Library + +Improve algorithm for computing which rolled-over log files to delete in +:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers +without ``namer`` and with arbitrary deterministic ``namer`` that leaves the +datetime part in the file name unmodified. + +.. + +.. date: 2024-02-21-17-54-59 +.. gh-issue: 74668 +.. nonce: JT-Q8W +.. section: Library + +:mod:`urllib.parse` functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl` now support bytes arguments containing raw +and percent-encoded non-ASCII data. + +.. + +.. date: 2024-02-20-22-02-34 +.. gh-issue: 67044 +.. nonce: QF9_Ru +.. section: Library + +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. + +.. + +.. date: 2024-02-20-16-42-54 +.. gh-issue: 115712 +.. nonce: EXVMXw +.. section: Library + +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. + +.. + +.. date: 2024-02-20-07-38-15 +.. gh-issue: 112364 +.. nonce: EX7uGI +.. section: Library + +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. +Patched by Frank Hoffmann. + +.. + +.. date: 2024-02-19-16-53-48 +.. gh-issue: 112997 +.. nonce: sYBXRZ +.. section: Library + +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. + +.. + +.. date: 2024-02-19-15-52-30 +.. gh-issue: 114914 +.. nonce: M5-1d8 +.. section: Library + +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. + +.. + +.. date: 2024-02-18-12-18-12 +.. gh-issue: 111358 +.. nonce: 9yJUMD +.. section: Library + +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. + +.. + +.. date: 2024-02-17-18-47-12 +.. gh-issue: 115618 +.. nonce: napiNp +.. section: Library + +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. + +.. + +.. date: 2024-02-16-16-40-10 +.. gh-issue: 112720 +.. nonce: io6_Ac +.. section: Library + +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. + +.. + +.. date: 2024-02-15-23-42-54 +.. gh-issue: 112006 +.. nonce: 4wxcK- +.. section: Library + +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. + +.. + +.. date: 2024-02-15-19-11-49 +.. gh-issue: 101293 +.. nonce: 898b8l +.. section: Library + +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`. + +.. + +.. date: 2024-02-12-11-42-48 +.. gh-issue: 103092 +.. nonce: sGMKr0 +.. section: Library + +Isolate :mod:`!_lsprof` (apply :pep:`687`). + +.. + +.. date: 2024-02-11-20-12-39 +.. gh-issue: 113942 +.. nonce: i72sMJ +.. section: Library + +:mod:`pydoc` no longer skips global functions implemented as builtin +methods, such as :class:`~type.MethodDescriptorType` and +:class:`~type.WrapperDescriptorType`. + +.. + +.. date: 2024-02-10-17-18-49 +.. gh-issue: 115256 +.. nonce: 41Fy9P +.. section: Library + +Added DeprecationWarning when accessing the tarfile attribute of TarInfo +objects. The attribute is never used internally and is only attached to +TarInfos when the tarfile is opened in write-mode, not read-mode. The +attribute creates an unnecessary reference cycle which may cause corruption +when not closing the handle after writing a tarfile. + +.. + +.. date: 2024-02-09-19-41-48 +.. gh-issue: 115197 +.. nonce: 20wkWH +.. section: Library + +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. + +.. + +.. date: 2024-02-09-12-22-47 +.. gh-issue: 113812 +.. nonce: wOraaG +.. section: Library + +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. + +.. + +.. date: 2024-01-30-23-28-29 +.. gh-issue: 114763 +.. nonce: BRjKkg +.. section: Library + +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. + +.. + +.. date: 2024-01-29-13-46-41 +.. gh-issue: 114709 +.. nonce: SQ998l +.. section: Library + +:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. + +.. + +.. date: 2024-01-26-16-42-31 +.. gh-issue: 114610 +.. nonce: S18Vuz +.. section: Library + +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. + +.. + +.. date: 2023-11-24-23-40-00 +.. gh-issue: 107361 +.. nonce: v54gh46 +.. section: Library + +Add :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT` to +the default SSL context created with :func:`ssl.create_default_context`. + +.. + +.. date: 2023-11-20-16-15-44 +.. gh-issue: 112281 +.. nonce: gH4EVk +.. section: Library + +Allow creating :ref:`union of types` for +:class:`typing.Annotated` with unhashable metadata. + +.. + +.. date: 2023-11-07-10-22-06 +.. gh-issue: 111775 +.. nonce: IoVxfX +.. section: Library + +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. + +.. + +.. date: 2023-10-07-06-15-13 +.. gh-issue: 90095 +.. nonce: gWn1ka +.. section: Library + +Make .pdbrc and -c work with any valid pdb commands. + +.. + +.. date: 2023-08-05-08-41-58 +.. gh-issue: 107625 +.. nonce: cVSHCT +.. section: Library + +Raise :exc:`configparser.ParsingError` from +:meth:`~configparser.ConfigParser.read` and +:meth:`~configparser.ConfigParser.read_file` methods of +:class:`configparser.ConfigParser` if a key without a corresponding value is +continued (that is, followed by an indented line). + +.. + +.. date: 2023-08-02-01-17-32 +.. gh-issue: 107155 +.. nonce: Mj1K9L +.. section: Library + +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. + +.. + +.. date: 2023-07-12-14-52-04 +.. gh-issue: 57141 +.. nonce: L2k8Xb +.. section: Library + +Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like +:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by Tobias +Rautenkranz + +.. + +.. date: 2023-05-17-21-33-21 +.. gh-issue: 69990 +.. nonce: Blvz9G +.. section: Library + +:meth:`Profile.print_stats` has been improved to accept multiple sort +arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. + +.. + +.. date: 2023-05-01-22-28-57 +.. gh-issue: 104061 +.. nonce: vxfBXf +.. section: Library + +Add :data:`socket.SO_BINDTOIFINDEX` constant. + +.. + +.. date: 2023-04-02-21-20-35 +.. gh-issue: 60346 +.. nonce: 7mjgua +.. section: Library + +Fix ArgumentParser inconsistent with parse_known_args. + +.. + +.. date: 2023-03-03-09-05-42 +.. gh-issue: 102389 +.. nonce: ucmo0_ +.. section: Library + +Add ``windows_31j`` to aliases for ``cp932`` codec + +.. + +.. date: 2023-02-14-17-19-59 +.. gh-issue: 72249 +.. nonce: fv35wU +.. section: Library + +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. + +.. + +.. date: 2023-01-12-14-16-01 +.. gh-issue: 100985 +.. nonce: GT5Fvd +.. section: Library + +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. + +.. + +.. date: 2023-01-09-14-08-02 +.. gh-issue: 100884 +.. nonce: DcmdLl +.. section: Library + +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. + +.. + +.. date: 2022-11-22-23-17-43 +.. gh-issue: 95782 +.. nonce: an_and +.. section: Library + +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`!_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`!_pyio.BufferedRandom.tell` being +able to return negative offsets. + +.. + +.. date: 2022-08-26-15-50-53 +.. gh-issue: 96310 +.. nonce: 0NssDh +.. section: Library + +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. + +.. + +.. date: 2022-05-25-17-49-04 +.. gh-issue: 93205 +.. nonce: DjhFVR +.. section: Library + +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where +multiple rotating handler instances pointing to files with the same name but +different extensions would conflict and not delete the correct files. + +.. + +.. bpo: 31116 +.. date: 2022-01-14-10-50-17 +.. nonce: 0bduV9 +.. section: Library + +Add Z85 encoding to ``base64``. + +.. + +.. bpo: 44865 +.. date: 2021-08-24-20-47-37 +.. nonce: c3BhZS +.. section: Library + +Add missing call to localization function in :mod:`argparse`. + +.. + +.. bpo: 43952 +.. date: 2021-05-03-11-04-12 +.. nonce: Me7fJe +.. section: Library + +Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty +bytes as authkey. Not accepting empty bytes as key causes it to hang +indefinitely. + +.. + +.. bpo: 42125 +.. date: 2020-12-15-22-30-49 +.. nonce: UGyseY +.. section: Library + +linecache: get module name from ``__spec__`` if available. This allows +getting source code for the ``__main__`` module when a custom loader is +used. + +.. + +.. bpo: 41122 +.. date: 2020-07-13-23-59-42 +.. nonce: 8P_Brh +.. section: Library + +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds internally. + +.. + +.. bpo: 40818 +.. date: 2020-05-29-18-08-54 +.. nonce: Ij8ffq +.. section: Library + +The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The +default implementation of :data:`sys.__interactivehook__` provides +auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. + +.. + +.. bpo: 33775 +.. date: 2019-04-06-23-50-59 +.. nonce: 0yhMDc +.. section: Library + +Add 'default' and 'version' help text for localization in argparse. + +.. + +.. date: 2024-02-14-20-17-04 +.. gh-issue: 115399 +.. nonce: fb9a0R +.. section: Documentation + +Document :cve:`2023-52425` of Expat <2.6.0 under "XML vulnerabilities". + +.. + +.. date: 2024-02-08-08-51-37 +.. gh-issue: 109653 +.. nonce: QHLW4w +.. section: Documentation + +Improve import time of :mod:`uuid` on Linux. + +.. + +.. date: 2024-02-25-16-28-26 +.. gh-issue: 71052 +.. nonce: lSb9EC +.. section: Tests + +Add test exclusions to support running the test suite on Android. + +.. + +.. date: 2024-02-25-15-58-28 +.. gh-issue: 71052 +.. nonce: lxBjqY +.. section: Tests + +Enable ``test_concurrent_futures`` on platforms that support threading but +not multiprocessing. + +.. + +.. date: 2024-02-22-00-17-06 +.. gh-issue: 115796 +.. nonce: d4hpKy +.. section: Tests + +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. + +.. + +.. date: 2024-02-20-15-47-41 +.. gh-issue: 115720 +.. nonce: w8i8UG +.. section: Tests + +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. + +.. + +.. date: 2024-02-18-14-20-52 +.. gh-issue: 115122 +.. nonce: 3rGNo9 +.. section: Tests + +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. + +.. + +.. date: 2024-02-17-08-25-01 +.. gh-issue: 115596 +.. nonce: RGPCrR +.. section: Tests + +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. + +.. + +.. date: 2024-02-16-13-04-28 +.. gh-issue: 115556 +.. nonce: rjaQ9w +.. section: Tests + +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. + +.. + +.. date: 2024-02-13-18-24-04 +.. gh-issue: 115420 +.. nonce: -dlzfI +.. section: Tests + +Fix translation of exception handler targets by +``_testinternalcapi.optimize_cfg``. + +.. + +.. date: 2024-02-12-22-35-01 +.. gh-issue: 115376 +.. nonce: n9vubZ +.. section: Tests + +Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. + +.. + +.. date: 2024-03-04-12-43-42 +.. gh-issue: 116313 +.. nonce: cLLb8S +.. section: Build + +Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. + +.. + +.. date: 2024-03-01-16-44-19 +.. gh-issue: 71052 +.. nonce: Hs-9EP +.. section: Build + +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. + +.. + +.. date: 2024-02-29-15-12-31 +.. gh-issue: 116117 +.. nonce: eENkQK +.. section: Build + +Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows with +``clang-cl``. + +.. + +.. date: 2024-02-26-14-54-58 +.. gh-issue: 71052 +.. nonce: XvFay1 +.. section: Build + +Fix several Android build issues + +.. + +.. date: 2024-02-26-13-13-53 +.. gh-issue: 114099 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on iOS. + +.. + +.. date: 2024-02-24-12-50-43 +.. gh-issue: 115350 +.. nonce: naQA6y +.. section: Build + +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined + +.. + +.. date: 2024-02-21-18-22-49 +.. gh-issue: 111225 +.. nonce: Z8C3av +.. section: Build + +Link extension modules against libpython on Android. + +.. + +.. date: 2024-02-21-11-58-30 +.. gh-issue: 115737 +.. nonce: dpNl2T +.. section: Build + +The install name for libPython is now correctly set for non-framework macOS +builds. + +.. + +.. date: 2024-02-13-14-52-59 +.. gh-issue: 114099 +.. nonce: zjXsQr +.. section: Build + +Makefile targets were added to support compiling an iOS-compatible framework +build. + +.. + +.. date: 2024-02-27-23-21-55 +.. gh-issue: 116012 +.. nonce: B9_IwM +.. section: Windows + +Ensure the value of ``GetLastError()`` is preserved across GIL operations. + +.. + +.. date: 2024-02-23-11-43-43 +.. gh-issue: 115582 +.. nonce: sk1XPi +.. section: Windows + +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. +This is expected to change in future releases. + +.. + +.. date: 2024-02-21-23-48-59 +.. gh-issue: 115554 +.. nonce: 02mpQC +.. section: Windows + +The installer now has more strict rules about updating the :ref:`launcher`. +In general, most users only have a single launcher installed and will see no +difference. When multiple launchers have been installed, the option to +install the launcher is disabled until all but one have been removed. +Downgrading the launcher (which was never allowed) is now more obviously +blocked. + +.. + +.. date: 2024-02-15-23-16-31 +.. gh-issue: 115543 +.. nonce: otrWnw +.. section: Windows + +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. + +.. + +.. date: 2024-02-29-20-52-23 +.. gh-issue: 116145 +.. nonce: ygafim +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.14. + +.. + +.. date: 2023-12-09-11-04-26 +.. gh-issue: 88516 +.. nonce: SIIvfs +.. section: IDLE + +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. + +.. + +.. date: 2023-02-12-19-28-08 +.. gh-issue: 100176 +.. nonce: Kzs4Zw +.. section: Tools/Demos + +Remove outdated Tools/{io,cc,string}bench + +.. + +.. bpo: 45101 +.. date: 2021-09-05-02-47-48 +.. nonce: 60Zqmt +.. section: Tools/Demos + +Add consistency in usage message IO between 2 versions of python-config. + +.. + +.. date: 2024-02-16-15-56-53 +.. gh-issue: 114626 +.. nonce: ie2esA +.. section: C API + +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-09-24-51 +.. gh-issue: 111418 +.. nonce: FYYetY +.. section: C API + +Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` +and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst new file mode 100644 index 00000000000000..0cdbb8232250d7 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -0,0 +1,1270 @@ +.. date: 2024-04-04-13-42-59 +.. gh-issue: 117494 +.. nonce: GPQH64 +.. release date: 2024-04-09 +.. section: Core and Builtins + +Refactored the instruction sequence data structure out of compile.c into +instruction_sequence.c. + +.. + +.. date: 2024-04-03-13-44-04 +.. gh-issue: 116968 +.. nonce: zgcdG2 +.. section: Core and Builtins + +Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), +shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The +API used for adaptive specialization counters is changed but the behavior is +(supposed to be) identical. + +The behavior of the Tier 2 counters is changed: + +* There are no longer dynamic thresholds (we never varied these). +* All counters now use the same exponential backoff. +* The counter for ``JUMP_BACKWARD`` starts counting down from 16. +* The ``temperature`` in side exits starts counting down from 64. + +.. + +.. date: 2024-04-03-09-49-15 +.. gh-issue: 117431 +.. nonce: WAqRgc +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!endswith` +* :meth:`!startswith` + +.. + +.. date: 2024-04-02-17-37-35 +.. gh-issue: 117431 +.. nonce: vDKAOn +.. section: Core and Builtins + +Improve the performance of the following :class:`str` methods by adapting +them to the :c:macro:`METH_FASTCALL` calling convention: + +* :meth:`~str.count` +* :meth:`~str.endswith` +* :meth:`~str.find` +* :meth:`~str.index` +* :meth:`~str.rfind` +* :meth:`~str.rindex` +* :meth:`~str.startswith` + +.. + +.. date: 2024-04-02-10-04-57 +.. gh-issue: 117411 +.. nonce: YdyVmG +.. section: Core and Builtins + +Move ``PyFutureFeatures`` to an internal header and make it private. + +.. + +.. date: 2024-04-02-06-16-49 +.. gh-issue: 109120 +.. nonce: X485oN +.. section: Core and Builtins + +Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by +Grigoryev Semyon + +.. + +.. date: 2024-03-26-17-22-38 +.. gh-issue: 117266 +.. nonce: Kwh79O +.. section: Core and Builtins + +Fix crashes for certain user-created subclasses of :class:`ast.AST`. Such +classes are now expected to set the ``_field_types`` attribute. + +.. + +.. date: 2024-03-25-17-04-54 +.. gh-issue: 99108 +.. nonce: 8bjdO6 +.. section: Core and Builtins + +Updated the :mod:`hashlib` built-in `HACL\* project`_ C code from upstream +that we use for many implementations when they are not present via OpenSSL +in a given build. This also avoids the rare potential for a C symbol name +one definition rule linking issue. + +.. _HACL\* project: https://github.com/hacl-star/hacl-star + +.. + +.. date: 2024-03-25-12-51-12 +.. gh-issue: 117108 +.. nonce: tNqDEo +.. section: Core and Builtins + +Change the old space bit of objects in the young generation from 0 to +gcstate->visited, so that any objects created during GC will have the old +bit set correctly if they get moved into the old generation. + +.. + +.. date: 2024-03-21-12-10-11 +.. gh-issue: 117108 +.. nonce: _6jIrB +.. section: Core and Builtins + +The cycle GC now chooses the size of increments based on the total heap +size, instead of the rate of object creation. This ensures that it can keep +up with growing heaps. + +.. + +.. date: 2024-03-13-16-55-25 +.. gh-issue: 116735 +.. nonce: o3w6y8 +.. section: Core and Builtins + +For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to +``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` +event. + +.. + +.. date: 2024-03-12-20-31-57 +.. gh-issue: 113964 +.. nonce: bJppzg +.. section: Core and Builtins + +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. + +.. + +.. date: 2024-03-11-22-05-56 +.. gh-issue: 116626 +.. nonce: GsyczB +.. section: Core and Builtins + +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits +:monitoring-event:`CALL` + +.. + +.. date: 2024-03-11-00-45-39 +.. gh-issue: 116554 +.. nonce: gYumG5 +.. section: Core and Builtins + +``list.sort()`` now exploits more cases of partial ordering, particularly +those with long descending runs with sub-runs of equal values. Those are +recognized as single runs now (previously, each block of repeated values +caused a new run to be created). + +.. + +.. date: 2024-03-07-16-12-39 +.. gh-issue: 114099 +.. nonce: ujdjn2 +.. section: Core and Builtins + +Added a Loader that can discover extension modules in an iOS-style +Frameworks folder. + +.. + +.. date: 2024-02-25-14-17-25 +.. gh-issue: 115775 +.. nonce: CNbGbJ +.. section: Core and Builtins + +Compiler populates the new ``__static_attributes__`` field on a class with +the names of attributes of this class which are accessed through self.X from +any function in its body. + +.. + +.. date: 2024-02-24-03-39-09 +.. gh-issue: 115776 +.. nonce: THJXqg +.. section: Core and Builtins + +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. + +.. + +.. date: 2024-01-07-04-22-51 +.. gh-issue: 108362 +.. nonce: oB9Gcf +.. section: Core and Builtins + +Implement an incremental cyclic garbage collector. By collecting the old +generation in increments, there is no need for a full heap scan. This can +hugely reduce maximum pause time for programs with large heaps. + +Reduce the number of generations from three to two. The old generation is +split into two spaces, "visited" and "pending". + +Collection happens in two steps:: * An increment is formed from the young +generation and a small part of the pending space. * This increment is +scanned and the survivors moved to the end of the visited space. + +When the collecting space becomes empty, the two spaces are swapped. + +.. + +.. date: 2023-10-14-00-05-17 +.. gh-issue: 109870 +.. nonce: oKpJ3P +.. section: Core and Builtins + +Dataclasses now calls :func:`exec` once per dataclass, instead of once per +method being added. This can speed up dataclass creation by up to 20%. + +.. + +.. date: 2022-10-05-09-33-48 +.. gh-issue: 97901 +.. nonce: BOLluU +.. section: Core and Builtins + +Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. + +.. + +.. bpo: 24612 +.. date: 2021-09-04-22-33-01 +.. nonce: SsTuUX +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` that happens when 'not' appears after an +operator. Patch by Pablo Galindo + +.. + +.. date: 2024-04-08-20-26-15 +.. gh-issue: 117648 +.. nonce: NzVEa7 +.. section: Library + +Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. + +.. + +.. date: 2024-04-06-16-42-34 +.. gh-issue: 117584 +.. nonce: hqk9Hn +.. section: Library + +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath()`. + +.. + +.. date: 2024-04-03-18-36-53 +.. gh-issue: 117467 +.. nonce: l6rWlj +.. section: Library + +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Patch by Tony Mountifield. + +.. + +.. date: 2024-04-02-20-30-12 +.. gh-issue: 114848 +.. nonce: YX4pEc +.. section: Library + +Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', +which can happen on Linux >= 2.6.36 with glibc < 2.27. + +.. + +.. date: 2024-04-02-13-13-46 +.. gh-issue: 117459 +.. nonce: jiIZmH +.. section: Library + +:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of +:class:`CancelledError`, :class:`TimeoutError` and +:class:`InvalidStateError` which are raised in the coroutine. + +.. + +.. date: 2024-03-29-21-43-19 +.. gh-issue: 117381 +.. nonce: fT0JFM +.. section: Library + +Fix error message for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-29-15-58-01 +.. gh-issue: 117337 +.. nonce: 7w3Qwp +.. section: Library + +Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` +functions. Use :func:`glob.glob` and pass a directory to its *root_dir* +argument instead. + +.. + +.. date: 2024-03-29-15-04-13 +.. gh-issue: 117349 +.. nonce: OB9kQQ +.. section: Library + +Optimise several functions in :mod:`os.path`. + +.. + +.. date: 2024-03-29-12-07-26 +.. gh-issue: 117348 +.. nonce: WjCYvK +.. section: Library + +Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric +complexity and improve comprehensibility. + +.. + +.. date: 2024-03-28-19-13-20 +.. gh-issue: 117335 +.. nonce: d6uKJu +.. section: Library + +Raise TypeError for non-sequences for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-28-17-55-22 +.. gh-issue: 66449 +.. nonce: 4jhuEV +.. section: Library + +:class:`configparser.ConfigParser` now accepts unnamed sections before named +ones, if configured to do so. + +.. + +.. date: 2024-03-28-13-54-20 +.. gh-issue: 88014 +.. nonce: zJz31I +.. section: Library + +In documentation of :class:`gzip.GzipFile` in module gzip, explain data type +of optional constructor argument *mtime*, and recommend ``mtime = 0`` for +generating deterministic streams. + +.. + +.. date: 2024-03-27-21-05-52 +.. gh-issue: 117310 +.. nonce: Bt2wox +.. section: Library + +Fixed an unlikely early & extra ``Py_DECREF`` triggered crash in :mod:`ssl` +when creating a new ``_ssl._SSLContext`` if CPython was built implausibly +such that the default cipher list is empty **or** the SSL library it was +linked against reports a failure from its C ``SSL_CTX_set_cipher_list()`` +API. + +.. + +.. date: 2024-03-27-16-43-42 +.. gh-issue: 117294 +.. nonce: wbXNFv +.. section: Library + +A ``DocTestCase`` now reports as skipped if all examples in the doctest are +skipped. + +.. + +.. date: 2024-03-26-11-48-39 +.. gh-issue: 98966 +.. nonce: SayV9y +.. section: Library + +In :mod:`subprocess`, raise a more informative message when +``stdout=STDOUT``. + +.. + +.. date: 2024-03-25-21-15-56 +.. gh-issue: 117225 +.. nonce: oOaZXb +.. section: Library + +doctest: only print "and X failed" when non-zero, don't pluralise "1 items". +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-03-25-00-20-16 +.. gh-issue: 117205 +.. nonce: yV7xGb +.. section: Library + +Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by +increasing ``chunksize``. + +.. + +.. date: 2024-03-23-14-26-18 +.. gh-issue: 117178 +.. nonce: vTisTG +.. section: Library + +Fix regression in lazy loading of self-referential modules, introduced in +gh-114781. + +.. + +.. date: 2024-03-23-13-40-13 +.. gh-issue: 112383 +.. nonce: XuHf3G +.. section: Library + +Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. + +.. + +.. date: 2024-03-23-12-28-05 +.. gh-issue: 117182 +.. nonce: a0KANW +.. section: Library + +Lazy-loading of modules that modify their own ``__class__`` no longer +reverts the ``__class__`` to :class:`types.ModuleType`. + +.. + +.. date: 2024-03-21-17-07-38 +.. gh-issue: 117084 +.. nonce: w1mTpT +.. section: Library + +Fix :mod:`zipfile` extraction for directory entries with the name containing +backslashes on Windows. + +.. + +.. date: 2024-03-21-09-57-57 +.. gh-issue: 117114 +.. nonce: Qu-p55 +.. section: Library + +Make :func:`os.path.isdevdrive` available on all platforms. For those that +do not offer Dev Drives, it will always return ``False``. + +.. + +.. date: 2024-03-21-07-27-36 +.. gh-issue: 117110 +.. nonce: 9K1InX +.. section: Library + +Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated +with arguments. Patch by Chris Fu. + +.. + +.. date: 2024-03-20-23-07-58 +.. gh-issue: 109653 +.. nonce: uu3lrX +.. section: Library + +Deferred select imports in importlib.metadata and importlib.resources for a +14% speedup. + +.. + +.. date: 2024-03-20-16-10-29 +.. gh-issue: 70647 +.. nonce: FpD6Ar +.. section: Library + +Start the deprecation period for the current behavior of +:func:`datetime.datetime.strptime` and :func:`time.strptime` which always +fails to parse a date string with a :exc:`ValueError` involving a day of +month such as ``strptime("02-29", "%m-%d")`` when a year is **not** +specified and the date happen to be February 29th. This should help avoid +users finding new bugs every four years due to a natural mistaken assumption +about the API when parsing partial date values. + +.. + +.. date: 2024-03-19-19-42-25 +.. gh-issue: 116987 +.. nonce: ZVKUH1 +.. section: Library + +Fixed :func:`inspect.findsource` for class code objects. + +.. + +.. date: 2024-03-19-14-35-57 +.. gh-issue: 114099 +.. nonce: siNSpK +.. section: Library + +Modify standard library to allow for iOS platform differences. + +.. + +.. date: 2024-03-19-11-08-26 +.. gh-issue: 90872 +.. nonce: ghys95 +.. section: Library + +On Windows, :meth:`subprocess.Popen.wait` no longer calls +``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the +timeout is negative. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-14-36-50 +.. gh-issue: 116957 +.. nonce: dTCs4f +.. section: Library + +configparser: Don't leave ConfigParser values in an invalid state (stored as +a list instead of a str) after an earlier read raised DuplicateSectionError +or DuplicateOptionError. + +.. + +.. date: 2024-03-17-18-12-39 +.. gh-issue: 115538 +.. nonce: PBiRQB +.. section: Library + +:class:`!_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. + +.. + +.. date: 2024-03-14-20-59-28 +.. gh-issue: 90095 +.. nonce: 7UaJ1U +.. section: Library + +Ignore empty lines and comments in ``.pdbrc`` + +.. + +.. date: 2024-03-14-17-24-59 +.. gh-issue: 106531 +.. nonce: 9ehywi +.. section: Library + +Refreshed zipfile._path from `zipp 3.18 +`_, providing +better compatibility for PyPy, better glob performance for deeply nested +zipfiles, and providing internal access to ``CompleteDirs.inject`` for use +in other tests (like importlib.resources). + +.. + +.. date: 2024-03-14-17-21-25 +.. gh-issue: 63207 +.. nonce: LV16SL +.. section: Library + +On Windows, :func:`time.time()` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than +1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a +resolution of 15.6 ms. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-14-01-46 +.. gh-issue: 116764 +.. nonce: moB3Lc +.. section: Library + +Restore support of ``None`` and other false values in :mod:`urllib.parse` +functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for +non-zero integers and non-empty sequences. + +.. + +.. date: 2024-03-14-10-01-23 +.. gh-issue: 116811 +.. nonce: _h5iKP +.. section: Library + +In ``PathFinder.invalidate_caches``, delegate to +``MetadataPathFinder.invalidate_caches``. + +.. + +.. date: 2024-03-14-09-38-51 +.. gh-issue: 116647 +.. nonce: h0d_zj +.. section: Library + +Fix recursive child in dataclasses + +.. + +.. date: 2024-03-14-01-38-44 +.. gh-issue: 113171 +.. nonce: VFnObz +.. section: Library + +Fixed various false positives and false negatives in + +* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) +* :attr:`ipaddress.IPv4Address.is_global` +* :attr:`ipaddress.IPv6Address.is_private` +* :attr:`ipaddress.IPv6Address.is_global` + +Also in the corresponding :class:`ipaddress.IPv4Network` and +:class:`ipaddress.IPv6Network` attributes. + +.. + +.. date: 2024-03-13-15-45-54 +.. gh-issue: 63283 +.. nonce: OToJnG +.. section: Library + +In :mod:`encodings.idna`, any capitalization of the the ACE prefix +(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. + +.. + +.. date: 2024-03-12-19-32-17 +.. gh-issue: 71042 +.. nonce: oI0Ron +.. section: Library + +Add :func:`platform.android_ver`, which provides device and OS information +on Android. + +.. + +.. date: 2024-03-12-17-53-14 +.. gh-issue: 73468 +.. nonce: z4ZzvJ +.. section: Library + +Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: +fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. + +.. + +.. date: 2024-03-11-17-04-55 +.. gh-issue: 116608 +.. nonce: 30f58- +.. section: Library + +The :mod:`importlib.resources` functions +:func:`~importlib.resources.is_resource()`, +:func:`~importlib.resources.open_binary()`, +:func:`~importlib.resources.open_text()`, +:func:`~importlib.resources.path()`, +:func:`~importlib.resources.read_binary()`, and +:func:`~importlib.resources.read_text()` are un-deprecated, and support +subdirectories via multiple positional arguments. The +:func:`~importlib.resources.contents()` function also allows subdirectories, +but remains deprecated. + +.. + +.. date: 2024-03-08-11-31-49 +.. gh-issue: 116484 +.. nonce: VMAsU7 +.. section: Library + +Change automatically generated :class:`tkinter.Checkbutton` widget names to +avoid collisions with automatically generated +:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. + +.. + +.. date: 2024-03-07-11-10-27 +.. gh-issue: 114314 +.. nonce: iEhAMH +.. section: Library + +In :mod:`ctypes`, ctype data is now stored in type objects directly rather +than in a dict subclass. This is an internal change that should not affect +usage. + +.. + +.. date: 2024-03-06-18-30-37 +.. gh-issue: 116401 +.. nonce: 3Wcda2 +.. section: Library + +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. + +.. + +.. date: 2024-03-05-19-56-29 +.. gh-issue: 71052 +.. nonce: PMDK-- +.. section: Library + +Implement :func:`ctypes.util.find_library` on Android. + +.. + +.. date: 2024-03-01-20-23-57 +.. gh-issue: 90535 +.. nonce: wXm-jC +.. section: Library + +Fix support of *interval* values > 1 in +:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and +``when='Wx'``. + +.. + +.. date: 2024-02-26-10-06-50 +.. gh-issue: 113308 +.. nonce: MbvOFt +.. section: Library + +Remove some internal protected parts from :mod:`uuid`: +``_has_uuid_generate_time_safe``, ``_netbios_getnode``, +``_ipconfig_getnode``, and ``_load_system_functions``. They were unused. + +.. + +.. date: 2024-02-18-09-50-31 +.. gh-issue: 115627 +.. nonce: HGchj0 +.. section: Library + +Fix the :mod:`ssl` module error handling of connection terminate by peer. It +now throws an OSError with the appropriate error code instead of an +EOFError. + +.. + +.. date: 2024-02-01-08-09-20 +.. gh-issue: 114847 +.. nonce: -JrWrR +.. section: Library + +Speed up :func:`os.path.realpath` on non-Windows platforms. + +.. + +.. date: 2024-02-01-03-09-38 +.. gh-issue: 114271 +.. nonce: raCkt5 +.. section: Library + +Fix a race in ``threading.Thread.join()``. + +``threading._MainThread`` now always represents the main thread of the main +interpreter. + +``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been +removed. + +.. + +.. date: 2024-01-22-15-50-58 +.. gh-issue: 113538 +.. nonce: v2wrwg +.. section: Library + +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. + +.. + +.. date: 2024-01-02-22-47-12 +.. gh-issue: 85287 +.. nonce: ZC5DLj +.. section: Library + +Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, +rather than just UnicodeError. + +.. + +.. date: 2023-12-28-22-52-45 +.. gh-issue: 113548 +.. nonce: j6TJ7O +.. section: Library + +:mod:`pdb` now allows CLI arguments to ``pdb -m``. + +.. + +.. date: 2023-12-11-00-51-51 +.. gh-issue: 112948 +.. nonce: k-OKp5 +.. section: Library + +Make completion of :mod:`pdb` similar to Python REPL + +.. + +.. date: 2023-06-16-19-17-06 +.. gh-issue: 105866 +.. nonce: 0NBveV +.. section: Library + +Fixed ``_get_slots`` bug which caused error when defining dataclasses with +slots and a weakref_slot. + +.. + +.. date: 2023-05-06-05-00-42 +.. gh-issue: 96471 +.. nonce: S3X5I- +.. section: Library + +Add :py:class:`asyncio.Queue` termination with +:py:meth:`~asyncio.Queue.shutdown` method. + +.. + +.. date: 2022-06-22-14-45-32 +.. gh-issue: 89739 +.. nonce: CqZcRL +.. section: Library + +The :mod:`zipimport` module can now read ZIP64 files. + +.. + +.. bpo: 33533 +.. date: 2020-10-02-17-35-19 +.. nonce: GLIhM5 +.. section: Library + +:func:`asyncio.as_completed` now returns an object that is both an +asynchronous iterator and plain iterator. The new asynchronous iteration +pattern allows for easier correlation between prior tasks and their +completed results. This is a closer match to +:func:`concurrent.futures.as_completed`'s iteration pattern. Patch by Justin +Arthur. + +.. + +.. bpo: 27578 +.. date: 2020-06-11-16-20-33 +.. nonce: CIA-fu +.. section: Library + +:func:`inspect.getsource` (and related functions) work with empty module +files, returning ``'\n'`` (or reasonable equivalent) instead of raising +``OSError``. Patch by Kernc. + +.. + +.. bpo: 37141 +.. date: 2019-09-26-17-52-52 +.. nonce: onYY2- +.. section: Library + +Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, +stopping when one of them is encountered. + +.. + +.. date: 2019-08-27-01-03-26 +.. gh-issue: 66543 +.. nonce: _TRpYr +.. section: Library + +Make :func:`mimetypes.guess_type` properly parsing of URLs with only a host +name, URLs containing fragment or query, and filenames with only a UNC +sharepoint on Windows. Based on patch by Dong-hee Na. + +.. + +.. bpo: 15010 +.. date: 2019-08-12-19-08-06 +.. nonce: 3bY2CF +.. section: Library + +:meth:`unittest.TestLoader.discover` now saves the original value of +``unittest.TestLoader._top_level_dir`` and restores it at the end of the +call. + +.. + +.. date: 2024-03-20-15-12-37 +.. gh-issue: 115977 +.. nonce: IMLi6K +.. section: Documentation + +Remove compatibility references to Emscripten. + +.. + +.. date: 2024-03-20-12-41-47 +.. gh-issue: 114099 +.. nonce: ad_Ck9 +.. section: Documentation + +Add an iOS platform guide, and flag modules not available on iOS. + +.. + +.. date: 2022-04-15-13-15-23 +.. gh-issue: 91565 +.. nonce: OznXwC +.. section: Documentation + +Changes to documentation files and config outputs to reflect the new +location for reporting bugs - i.e. GitHub rather than bugs.python.org. + +.. + +.. date: 2024-03-25-21-31-49 +.. gh-issue: 83434 +.. nonce: U7Z8cY +.. section: Tests + +Disable JUnit XML output (``--junit-xml=FILE`` command line option) in +regrtest when hunting for reference leaks (``-R`` option). Patch by Victor +Stinner. + +.. + +.. date: 2024-03-24-23-49-25 +.. gh-issue: 117187 +.. nonce: eMLT5n +.. section: Tests + +Fix XML tests for vanilla Expat <2.6.0. + +.. + +.. date: 2024-03-21-11-32-29 +.. gh-issue: 116333 +.. nonce: F-9Ram +.. section: Tests + +Tests of TLS related things (error codes, etc) were updated to be more +lenient about specific error message strings and behaviors as seen in the +BoringSSL and AWS-LC forks of OpenSSL. + +.. + +.. date: 2024-03-20-14-19-32 +.. gh-issue: 117089 +.. nonce: WwR1Z1 +.. section: Tests + +Consolidated tests for importlib.metadata in their own ``metadata`` package. + +.. + +.. date: 2024-03-13-12-06-49 +.. gh-issue: 115979 +.. nonce: zsNpQD +.. section: Tests + +Update test_importlib so that it passes under WASI SDK 21. + +.. + +.. date: 2024-03-11-23-20-28 +.. gh-issue: 112536 +.. nonce: Qv1RrX +.. section: Tests + +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. + +.. + +.. date: 2024-03-06-11-00-36 +.. gh-issue: 116307 +.. nonce: Uij0t_ +.. section: Tests + +Added import helper ``isolated_modules`` as ``CleanImport`` does not remove +modules imported during the context. Use it in importlib.resources tests to +avoid leaving ``mod`` around to impede importlib.metadata tests. + +.. + +.. date: 2024-03-13-16-16-43 +.. gh-issue: 114736 +.. nonce: ZhmauG +.. section: Build + +Have WASI builds use WASI SDK 21. + +.. + +.. date: 2024-03-08-17-05-15 +.. gh-issue: 115983 +.. nonce: ZQqk0Q +.. section: Build + +Skip building test modules that must be built as shared under WASI. + +.. + +.. date: 2024-03-06-17-26-55 +.. gh-issue: 71052 +.. nonce: vLbu9u +.. section: Build + +Add Android build script and instructions. + +.. + +.. date: 2024-03-28-22-12-00 +.. gh-issue: 117267 +.. nonce: K_tki1 +.. section: Windows + +Ensure ``DirEntry.stat().st_ctime`` behaves consistently with +:func:`os.stat` during the deprecation period of ``st_ctime`` by containing +the same value as ``st_birthtime``. After the deprecation period, +``st_ctime`` will be the metadata change time (or unavailable through +``DirEntry``), and only ``st_birthtime`` will contain the creation time. + +.. + +.. date: 2024-03-14-20-46-23 +.. gh-issue: 116195 +.. nonce: Cu_rYs +.. section: Windows + +Improves performance of :func:`os.getppid` by using an alternate system API +when available. Contributed by vxiiduu. + +.. + +.. date: 2024-03-14-09-14-21 +.. gh-issue: 88494 +.. nonce: Bwfmp7 +.. section: Windows + +On Windows, :func:`time.monotonic()` now uses the +``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, +instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. +Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-01-58-22 +.. gh-issue: 116773 +.. nonce: H2UldY +.. section: Windows + +Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has +pending operation at deallocation, the process may crash``. + +.. + +.. date: 2024-02-24-23-03-43 +.. gh-issue: 91227 +.. nonce: sL4zWC +.. section: Windows + +Fix the asyncio ProactorEventLoop implementation so that sending a datagram +to an address that is not listening does not prevent receiving any more +datagrams. + +.. + +.. date: 2024-02-08-14-48-15 +.. gh-issue: 115119 +.. nonce: qMt32O +.. section: Windows + +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. + +.. + +.. date: 2024-04-08-18-53-33 +.. gh-issue: 117642 +.. nonce: _-tYH_ +.. section: C API + +Fix :pep:`737` implementation for ``%#T`` and ``%#N``. + +.. + +.. date: 2024-03-22-19-29-24 +.. gh-issue: 87193 +.. nonce: u7O-jY +.. section: C API + +:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference +count > 1, including 1-byte bytes objects. It creates a new bytes object and +destroys the old one if it has reference count > 1. + +.. + +.. date: 2024-03-20-13-13-22 +.. gh-issue: 117021 +.. nonce: 0Q5jBx +.. section: C API + +Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit +platforms. + +.. + +.. date: 2024-03-19-09-49-04 +.. gh-issue: 115756 +.. nonce: 4Ls_Tl +.. section: C API + +:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to +:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in +:gh:`115781`) + +.. + +.. date: 2024-03-18-10-58-47 +.. gh-issue: 116869 +.. nonce: lN0GBl +.. section: C API + +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-09-58-46 +.. gh-issue: 116869 +.. nonce: LFDVKM +.. section: C API + +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. + +.. + +.. date: 2024-03-17-22-42-21 +.. gh-issue: 116936 +.. nonce: tNrzfm +.. section: C API + +Add :c:func:`PyType_GetModuleByDef` to the limited C API. Patch by Victor +Stinner. + +.. + +.. date: 2024-03-16-12-21-00 +.. gh-issue: 116809 +.. nonce: JL786L +.. section: C API + +Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-57-33 +.. gh-issue: 115754 +.. nonce: zLdv82 +.. section: C API + +In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, +``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now +implemented as function calls at the stable ABI level to hide implementation +details. Getting these constants still return borrowed references. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-55-24 +.. gh-issue: 115754 +.. nonce: xnzc__ +.. section: C API + +Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions +to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns +a :term:`strong reference` to the constant zero. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-22-30-07 +.. gh-issue: 111696 +.. nonce: 76UMKi +.. section: C API + +Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object +type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for +more information. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-18-00-32 +.. gh-issue: 111696 +.. nonce: L6oIPq +.. section: C API + +Add :c:func:`PyType_GetModuleName` function to get the type's module name. +Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow +and Victor Stinner. + +.. + +.. date: 2024-03-14-15-17-11 +.. gh-issue: 111696 +.. nonce: YmnvAi +.. section: C API + +Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully +qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, +or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal +to ``"builtins"``. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-10-33-58 +.. gh-issue: 85283 +.. nonce: LOgmdU +.. section: C API + +The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and +``_testconsole`` C extensions are now built with the :ref:`limited C API +`. Patch by Victor Stinner. + +.. + +.. date: 2024-02-28-15-50-01 +.. gh-issue: 111140 +.. nonce: mpwcUg +.. section: C API + +Add additional flags to :c:func:`PyLong_AsNativeBytes` and +:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to +handle edge cases around values that fill the entire buffer. + +.. + +.. date: 2023-12-12-19-48-31 +.. gh-issue: 113024 +.. nonce: rXcQs7 +.. section: C API + +Add :c:func:`PyObject_GenericHash` function. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst new file mode 100644 index 00000000000000..ba3b3dbbc08c41 --- /dev/null +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -0,0 +1,1671 @@ +.. date: 2024-03-27-13-50-02 +.. gh-issue: 116741 +.. nonce: ZoGryG +.. release date: 2024-05-08 +.. section: Security + +Update bundled libexpat to 2.6.2 + +.. + +.. date: 2024-03-25-21-25-28 +.. gh-issue: 117233 +.. nonce: E4CyI_ +.. section: Security + +Detect BLAKE2, SHA3, Shake, & truncated SHA512 support in the OpenSSL-ish +libcrypto library at build time. This allows :mod:`hashlib` to be used with +libraries that do not to support every algorithm that upstream OpenSSL does. + +.. + +.. date: 2024-05-07-01-39-24 +.. gh-issue: 118414 +.. nonce: G5GG7l +.. section: Core and Builtins + +Add instrumented opcodes to YIELD_VALUE assertion for tracing cases. + +.. + +.. date: 2024-05-06-10-57-54 +.. gh-issue: 117953 +.. nonce: DqCzIs +.. section: Core and Builtins + +When a builtin or extension module is imported for the first time, while a +subinterpreter is active, the module's init function is now run by the main +interpreter first before import continues in the subinterpreter. +Consequently, single-phase init modules now fail in an isolated +subinterpreter without the init function running under that interpreter, +whereas before it would run under the subinterpreter *before* failing, +potentially leaving behind global state and callbacks and otherwise leaving +the module in an inconsistent state. + +.. + +.. date: 2024-05-05-12-04-02 +.. gh-issue: 117549 +.. nonce: kITawD +.. section: Core and Builtins + +Don't use designated initializer syntax in inline functions in internal +headers. They cause problems for C++ or MSVC users who aren't yet using the +latest C++ standard (C++20). While internal, pycore_backoff.h, is included +(indirectly, via pycore_code.h) by some key 3rd party software that does so +for speed. + +.. + +.. date: 2024-05-03-18-01-26 +.. gh-issue: 95382 +.. nonce: 73FSEv +.. section: Core and Builtins + +Improve performance of :func:`json.dumps` and :func:`json.dump` when using +the argument *indent*. Depending on the data the encoding using +:func:`json.dumps` with *indent* can be up to 2 to 3 times faster. + +.. + +.. date: 2024-05-03-17-49-37 +.. gh-issue: 116322 +.. nonce: Gy6M4j +.. section: Core and Builtins + +In ``--disable-gil`` builds, the GIL will be enabled while loading C +extension modules. If the module indicates that it supports running without +the GIL, the GIL will be disabled once loading is complete. Otherwise, the +GIL will remain enabled for the remainder of the interpreter's lifetime. +This behavior does not apply if the GIL has been explicitly enabled or +disabled with ``PYTHON_GIL`` or ``-Xgil``. + +.. + +.. date: 2024-05-02-21-19-35 +.. gh-issue: 118513 +.. nonce: qHODjb +.. section: Core and Builtins + +Fix incorrect :exc:`UnboundLocalError` when two comprehensions in the same +function both reference the same name, and in one comprehension the name is +bound while in the other it's an implicit global. + +.. + +.. date: 2024-05-02-20-32-42 +.. gh-issue: 118518 +.. nonce: m-JbTi +.. section: Core and Builtins + +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHON_PERF_JIT_SUPPORT`` environment variable or when running Python with +``-Xperf_jit``. Patch by Pablo Galindo. + +.. + +.. date: 2024-05-02-16-04-51 +.. gh-issue: 117514 +.. nonce: CJiuC0 +.. section: Core and Builtins + +Add ``sys._is_gil_enabled()`` function that returns whether the GIL is +currently enabled. In the default build it always returns ``True`` because +the GIL is always enabled. In the free-threaded build, it may return +``True`` or ``False``. + +.. + +.. date: 2024-05-02-15-57-07 +.. gh-issue: 118164 +.. nonce: AF6kwI +.. section: Core and Builtins + +Break a loop between the Python implementation of the :mod:`decimal` module +and the Python code for integer to string conversion. Also optimize integer +to string conversion for values in the range from 9_000 to 135_000 decimal +digits. + +.. + +.. date: 2024-05-01-22-43-54 +.. gh-issue: 118473 +.. nonce: QIvq9R +.. section: Core and Builtins + +Fix :func:`sys.set_asyncgen_hooks` not to be partially set when raising +:exc:`TypeError`. + +.. + +.. date: 2024-05-01-17-12-36 +.. gh-issue: 118465 +.. nonce: g3Q8iE +.. section: Core and Builtins + +Compiler populates the new ``__firstlineno__`` field on a class with the +line number of the first line of the class definition. + +.. + +.. date: 2024-05-01-14-20-28 +.. gh-issue: 118492 +.. nonce: VUsSfn +.. section: Core and Builtins + +Fix an issue where the type cache can expose a previously accessed attribute +when a finalizer is run. + +.. + +.. date: 2024-05-01-07-06-48 +.. gh-issue: 117714 +.. nonce: Ip_dm5 +.. section: Core and Builtins + +update ``async_generator.athrow().close()`` and +``async_generator.asend().close()`` to close their section of the underlying +async generator + +.. + +.. date: 2024-04-28-00-41-17 +.. gh-issue: 111201 +.. nonce: cQsh5U +.. section: Core and Builtins + +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo, Åukasz Langa and +Lysandros Nikolaou based on code from the PyPy project. + +.. + +.. date: 2024-04-27-21-44-40 +.. gh-issue: 74929 +.. nonce: C2nESp +.. section: Core and Builtins + +Implement PEP 667: converted :attr:`FrameType.f_locals ` and +:c:func:`PyFrame_GetLocals` to return a write-through proxy object when the +frame refers to a function or comprehension. + +.. + +.. date: 2024-04-27-16-23-29 +.. gh-issue: 116767 +.. nonce: z9UFpr +.. section: Core and Builtins + +Fix crash in compiler on 'async with' that has many context managers. + +.. + +.. date: 2024-04-26-14-06-18 +.. gh-issue: 118335 +.. nonce: SRFsxO +.. section: Core and Builtins + +Change how to use the tier 2 interpreter. Instead of running Python with +``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this +choice is now made at build time by configuring with +``--enable-experimental-jit=interpreter``. + +**Beware!** This changes the environment variable to enable or disable +micro-ops to ``PYTHON_JIT``. The old ``PYTHON_UOPS`` is no longer used. + +.. + +.. date: 2024-04-26-05-38-18 +.. gh-issue: 118306 +.. nonce: vRUEOU +.. section: Core and Builtins + +Update JIT compilation to use LLVM 18 + +.. + +.. date: 2024-04-25-21-18-19 +.. gh-issue: 118160 +.. nonce: GH5SMc +.. section: Core and Builtins + +:ref:`Annotation scopes ` within classes can now contain +comprehensions. However, such comprehensions are not inlined into their +parent scope at runtime. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-25-12-55-47 +.. gh-issue: 118272 +.. nonce: 5ptjk_ +.. section: Core and Builtins + +Fix bug where ``generator.close`` does not free the generator frame's +locals. + +.. + +.. date: 2024-04-25-11-48-28 +.. gh-issue: 118216 +.. nonce: SVg700 +.. section: Core and Builtins + +Don't consider :mod:`__future__` imports with dots before the module name. + +.. + +.. date: 2024-04-22-08-34-28 +.. gh-issue: 118074 +.. nonce: 5_JnIa +.. section: Core and Builtins + +Make sure that the Executor objects in the COLD_EXITS array aren't assumed +to be GC-able (which would access bytes outside the object). + +.. + +.. date: 2024-04-20-20-30-15 +.. gh-issue: 107674 +.. nonce: GZPOP7 +.. section: Core and Builtins + +Lazy load frame line number to improve performance of tracing + +.. + +.. date: 2024-04-19-11-59-57 +.. gh-issue: 118082 +.. nonce: _FLuOT +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for imports without names, like in ``from +x import`` and ``import`` cases. It now points out to users that +:keyword:`import` expects at least one name after it. + +.. + +.. date: 2024-04-19-11-57-46 +.. gh-issue: 118090 +.. nonce: eGAQ0B +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for empty type param brackets. + +.. + +.. date: 2024-04-18-03-49-41 +.. gh-issue: 117958 +.. nonce: -EsfUs +.. section: Core and Builtins + +Added a ``get_jit_code()`` method to access JIT compiled machine code from +the UOp Executor when the experimental JIT is enabled. Patch by Anthony +Shaw. + +.. + +.. date: 2024-04-17-22-53-52 +.. gh-issue: 117901 +.. nonce: SsEcVJ +.. section: Core and Builtins + +Add option for compiler's codegen to save nested instruction sequences for +introspection. + +.. + +.. date: 2024-04-17-22-49-15 +.. gh-issue: 116622 +.. nonce: tthNUF +.. section: Core and Builtins + +Redirect stdout and stderr to system log when embedded in an Android app. + +.. + +.. date: 2024-04-17-17-52-32 +.. gh-issue: 109118 +.. nonce: q9iPEI +.. section: Core and Builtins + +:ref:`annotation scope ` within class scopes can now +contain lambdas. + +.. + +.. date: 2024-04-15-13-53-59 +.. gh-issue: 117894 +.. nonce: 8LpZ6m +.. section: Core and Builtins + +Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. + +.. + +.. date: 2024-04-15-07-37-09 +.. gh-issue: 117881 +.. nonce: 07H0wI +.. section: Core and Builtins + +prevent concurrent access to an async generator via athrow().throw() or +asend().throw() + +.. + +.. date: 2024-04-13-16-55-53 +.. gh-issue: 117536 +.. nonce: xkVbfv +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. + +.. + +.. date: 2024-04-12-12-28-49 +.. gh-issue: 117755 +.. nonce: 6ct8kU +.. section: Core and Builtins + +Fix mimalloc allocator for huge memory allocation (around 8,589,934,592 GiB) +on s390x. Patch by Victor Stinner. + +.. + +.. date: 2024-04-12-11-19-18 +.. gh-issue: 117750 +.. nonce: YttK6h +.. section: Core and Builtins + +Fix issue where an object's dict would get out of sync with the object's +internal values when being cleared. ``obj.__dict__.clear()`` now clears the +internal values, but leaves the dict attached to the object. + +.. + +.. date: 2024-04-12-09-09-11 +.. gh-issue: 117431 +.. nonce: lxFEeJ +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!count` +* :meth:`!find` +* :meth:`!index` +* :meth:`!rfind` +* :meth:`!rindex` + +.. + +.. date: 2024-04-10-22-16-18 +.. gh-issue: 117709 +.. nonce: -_1YL0 +.. section: Core and Builtins + +Speed up calls to :func:`str` with positional-only argument, by using the +:pep:`590` ``vectorcall`` calling convention. Patch by Erlend Aasland. + +.. + +.. date: 2024-04-09-16-07-00 +.. gh-issue: 117680 +.. nonce: MRZ78K +.. section: Core and Builtins + +Give ``_PyInstructionSequence`` a Python interface and use it in tests. + +.. + +.. date: 2024-04-09-11-31-25 +.. gh-issue: 115776 +.. nonce: 5Nthd0 +.. section: Core and Builtins + +Statically allocated objects are, by definition, immortal so must be marked +as such regardless of whether they are in extension modules or not. + +.. + +.. date: 2024-03-30-00-37-53 +.. gh-issue: 117385 +.. nonce: h0OJti +.. section: Core and Builtins + +Remove unhandled ``PY_MONITORING_EVENT_BRANCH`` and +``PY_MONITORING_EVENT_EXCEPTION_HANDLED`` events from :func:`sys.settrace`. + +.. + +.. date: 2024-03-12-13-51-09 +.. gh-issue: 116322 +.. nonce: q8TcDQ +.. section: Core and Builtins + +Extension modules may indicate to the runtime that they can run without the +GIL. Multi-phase init modules do so by calling providing +``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init +modules call ``PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +their init function. + +.. + +.. date: 2024-02-29-18-55-45 +.. gh-issue: 116129 +.. nonce: wsFnIq +.. section: Core and Builtins + +Implement :pep:`696`, adding support for defaults on type parameters. Patch +by Jelle Zijlstra. + +.. + +.. date: 2024-02-26-13-14-52 +.. gh-issue: 93502 +.. nonce: JMWRvA +.. section: Core and Builtins + +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo + +.. + +.. date: 2024-02-04-07-45-29 +.. gh-issue: 107674 +.. nonce: q8mCmi +.. section: Core and Builtins + +Improved the performance of :func:`sys.settrace` significantly + +.. + +.. date: 2024-01-07-03-38-34 +.. gh-issue: 95754 +.. nonce: aPjEBG +.. section: Core and Builtins + +Improve the error message when a script shadowing a module from the standard +library causes :exc:`AttributeError` to be raised. Similarly, improve the +error message when a script shadowing a third party module attempts to +access an attribute from that third party module while still initialising. + +.. + +.. date: 2023-12-03-18-21-59 +.. gh-issue: 99180 +.. nonce: 5m0V0q +.. section: Core and Builtins + +Elide uninformative traceback indicators in ``return`` and simple +``assignment`` statements. Patch by Pablo Galindo. + +.. + +.. date: 2023-06-18-00-27-57 +.. gh-issue: 105879 +.. nonce: dPw78k +.. section: Core and Builtins + +Allow the *globals* and *locals* arguments to :func:`exec` and :func:`eval` +to be passed as keywords. + +.. + +.. date: 2024-05-07-11-23-11 +.. gh-issue: 118418 +.. nonce: QPMdJm +.. section: Library + +A :exc:`DeprecationWarning` is now emitted if you fail to pass a value to +the new *type_params* parameter of ``typing._eval_type()`` or +``typing.ForwardRef._evaluate()``. (Using either of these private and +undocumented functions is discouraged to begin with, but failing to pass a +value to the ``type_params`` parameter may lead to incorrect behaviour on +Python 3.12 or newer.) + +.. + +.. date: 2024-05-06-18-13-02 +.. gh-issue: 118660 +.. nonce: n01Vb7 +.. section: Library + +Add an optional second type parameter to :class:`typing.ContextManager` and +:class:`typing.AsyncContextManager`, representing the return types of +:meth:`~object.__exit__` and :meth:`~object.__aexit__` respectively. This +parameter defaults to ``bool | None``. + +.. + +.. date: 2024-05-06-16-52-40 +.. gh-issue: 118650 +.. nonce: qKz5lp +.. section: Library + +The ``enum`` module allows method named ``_repr_*`` to be defined on +``Enum`` types. + +.. + +.. date: 2024-05-06-08-23-01 +.. gh-issue: 118648 +.. nonce: OVA3jJ +.. section: Library + +Add type parameter defaults to :class:`typing.Generator` and +:class:`typing.AsyncGenerator`. + +.. + +.. date: 2024-05-05-16-08-03 +.. gh-issue: 101137 +.. nonce: 71ECXu +.. section: Library + +Mime type ``text/x-rst`` is now supported by :mod:`mimetypes`. + +.. + +.. date: 2024-05-04-20-22-59 +.. gh-issue: 118164 +.. nonce: 9D02MQ +.. section: Library + +The Python implementation of the ``decimal`` module could appear to hang in +relatively small power cases (like ``2**117``) if context precision was set +to a very high value. A different method to check for exactly representable +results is used now that doesn't rely on computing ``10**precision`` (which +could be effectively too large to compute). + +.. + +.. date: 2024-05-04-18-40-43 +.. gh-issue: 111744 +.. nonce: nuCtwN +.. section: Library + +``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately +after the call rather than before the next line is executed. + +.. + +.. date: 2024-05-02-04-27-12 +.. gh-issue: 118500 +.. nonce: pBGGtQ +.. section: Library + +Add :mod:`pdb` support for zipapps + +.. + +.. date: 2024-04-30-15-18-19 +.. gh-issue: 118406 +.. nonce: y-GnMo +.. section: Library + +Add signature for :class:`sqlite3.Connection` objects. + +.. + +.. date: 2024-04-30-12-59-04 +.. gh-issue: 101732 +.. nonce: 29zUDu +.. section: Library + +Use a Y2038 compatible openssl time function when available. + +.. + +.. date: 2024-04-29-22-11-54 +.. gh-issue: 118404 +.. nonce: GYfMaD +.. section: Library + +Fix :func:`inspect.signature` for non-comparable callables. + +.. + +.. date: 2024-04-29-21-51-28 +.. gh-issue: 118402 +.. nonce: Z_06Th +.. section: Library + +Fix :func:`inspect.signature` for the result of the +:func:`functools.cmp_to_key` call. + +.. + +.. date: 2024-04-27-20-34-56 +.. gh-issue: 116622 +.. nonce: YlQgXv +.. section: Library + +On Android, :any:`sysconfig.get_platform` now returns the format specified +by :pep:`738`. + +.. + +.. date: 2024-04-26-14-53-28 +.. gh-issue: 118285 +.. nonce: A0_pte +.. section: Library + +Allow to specify the signature of custom callable instances of extension +type by the ``__text_signature__`` attribute. Specify signatures of +:class:`operator.attrgetter`, :class:`operator.itemgetter`, and +:class:`operator.methodcaller` instances. + +.. + +.. date: 2024-04-26-12-42-29 +.. gh-issue: 118314 +.. nonce: Z7reGc +.. section: Library + +Fix an edge case in :func:`binascii.a2b_base64` strict mode, where excessive +padding is not detected when no padding is necessary. + +.. + +.. date: 2024-04-25-11-49-11 +.. gh-issue: 118271 +.. nonce: 5N2Xcy +.. section: Library + +Add the :class:`!PhotoImage` methods :meth:`!read` to +read an image from a file and :meth:`!data` to get the +image data. Add *background* and *grayscale* parameters to +:class:`!PhotoImage` method :meth:`!write`. + +.. + +.. date: 2024-04-24-16-07-26 +.. gh-issue: 118225 +.. nonce: KdrcgL +.. section: Library + +Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region +from one image to other image, possibly with pixel zooming and/or +subsampling. Add *from_coords* parameter to :class:`!PhotoImage` methods +:meth:`!copy()`, :meth:`!zoom()` and :meth:`!subsample()`. Add *zoom* and +*subsample* parameters to :class:`!PhotoImage` method :meth:`!copy()`. + +.. + +.. date: 2024-04-24-12-29-33 +.. gh-issue: 118221 +.. nonce: 2k_bac +.. section: Library + +Fix a bug where :meth:`sqlite3.Connection.iterdump` could fail if a custom +:attr:`row factory ` was used. Patch by +Erlend Aasland. + +.. + +.. date: 2024-04-24-12-20-48 +.. gh-issue: 118013 +.. nonce: TKn_kZ +.. section: Library + +Fix regression introduced in gh-103193 that meant that calling +:func:`inspect.getattr_static` on an instance would cause a strong reference +to that instance's class to persist in an internal cache in the +:mod:`inspect` module. This caused unexpected memory consumption if the +class was dynamically created, the class held strong references to other +objects which took up a significant amount of memory, and the cache +contained the sole strong reference to the class. The fix for the regression +leads to a slowdown in :func:`!getattr_static`, but the function should +still be significantly faster than it was in Python 3.11. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-24-07-45-08 +.. gh-issue: 118218 +.. nonce: m1OHbN +.. section: Library + +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. + +.. + +.. date: 2024-04-23-21-17-00 +.. gh-issue: 117486 +.. nonce: ea3KYD +.. section: Library + +Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such +classes will now require no changes in the usual case to conform with the +behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-22-21-54-12 +.. gh-issue: 90848 +.. nonce: 5jHEEc +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to configure parent mock with +keyword arguments. + +.. + +.. date: 2024-04-22-20-42-29 +.. gh-issue: 118168 +.. nonce: Igni7h +.. section: Library + +Fix incorrect argument substitution when :data:`typing.Unpack` is used with +the builtin :class:`tuple`. :data:`!typing.Unpack` now raises +:exc:`TypeError` when used with certain invalid types. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-21-18-55-42 +.. gh-issue: 118131 +.. nonce: eAT0is +.. section: Library + +Add command-line interface for the :mod:`random` module. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-04-19-09-28-43 +.. gh-issue: 118107 +.. nonce: Mdsr1J +.. section: Library + +Fix :mod:`zipimport` reading of ZIP64 files with file entries that are too +big or offset too far. + +.. + +.. date: 2024-04-19-08-50-48 +.. gh-issue: 102511 +.. nonce: qDEB66 +.. section: Library + +Fix :func:`os.path.normpath` for UNC paths on Windows. +Speed up :func:`os.path.splitroot` with a native implementation. + +.. + +.. date: 2024-04-18-00-35-11 +.. gh-issue: 117535 +.. nonce: 0m6SIM +.. section: Library + +Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to +clarify that it's not a real filename. + +.. + +.. date: 2024-04-17-22-00-15 +.. gh-issue: 114053 +.. nonce: _JBV4D +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on +a class that made use of :pep:`695` type parameters in a module that had +``from __future__ import annotations`` at the top of the file. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-17-21-28-24 +.. gh-issue: 116931 +.. nonce: _AS09h +.. section: Library + +Add parameter *fileobj* check for :func:`tarfile.TarFile.addfile` + +.. + +.. date: 2024-04-17-19-41-59 +.. gh-issue: 117995 +.. nonce: Vt76Rv +.. section: Library + +Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters +is used to bind indexed, nameless placeholders. See also :gh:`100668`. + +.. + +.. date: 2024-04-17-18-00-30 +.. gh-issue: 80361 +.. nonce: RstWg- +.. section: Library + +Fix TypeError in :func:`email.message.Message.get_payload` when the charset is +:rfc:`2231` encoded. + +.. + +.. date: 2024-04-16-18-34-11 +.. gh-issue: 86650 +.. nonce: Zeydyg +.. section: Library + +Fix IndexError when parse some emails with invalid Message-ID (including +one-off addresses generated by Microsoft Outlook). + +.. + +.. date: 2024-04-14-15-59-28 +.. gh-issue: 117691 +.. nonce: 1mtREE +.. section: Library + +Improve the error messages emitted by :mod:`tarfile` deprecation warnings +relating to PEP 706. If a ``filter`` argument is not provided to +``extract()`` or ``extractall``, the deprecation warning now points to the +line in the user's code where the relevant function was called. Patch by +Alex Waygood. + +.. + +.. date: 2024-04-13-18-59-25 +.. gh-issue: 115874 +.. nonce: c3xG-E +.. section: Library + +Fixed a possible segfault during garbage collection of +``_asyncio.FutureIter`` objects. Patch by Savannah Ostrowski. + +.. + +.. date: 2024-04-13-01-45-15 +.. gh-issue: 115060 +.. nonce: IxoM03 +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by omitting an initial +:meth:`~pathlib.Path.is_dir` call. As a result of this change, +:meth:`~pathlib.Path.glob` can no longer raise :exc:`OSError`. + +.. + +.. date: 2024-04-12-17-37-11 +.. gh-issue: 77102 +.. nonce: Mk6X_E +.. section: Library + +:mod:`site` module now parses ``.pth`` file with UTF-8 first, and +:term:`locale encoding` if ``UnicodeDecodeError`` happened. It supported +only locale encoding before. + +.. + +.. date: 2024-04-11-18-11-37 +.. gh-issue: 76785 +.. nonce: BWNkhC +.. section: Library + +We've exposed the low-level :mod:`!_interpreters` module for the sake of the +PyPI implementation of :pep:`734`. It was sometimes available as the +:mod:`!_xxsubinterpreters` module and was formerly used only for testing. +For the most part, it should be considered an internal module, like +:mod:`!_thread` and :mod:`!_imp`. See +https://discuss.python.org/t/pep-734-multiple-interpreters-in-the-stdlib/41147/26. + +.. + +.. date: 2024-04-10-22-35-24 +.. gh-issue: 115060 +.. nonce: XEVuOb +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by not scanning directories for +non-wildcard pattern segments. + +.. + +.. date: 2024-04-10-21-30-37 +.. gh-issue: 117727 +.. nonce: uAYNVS +.. section: Library + +Speed up :meth:`pathlib.Path.iterdir` by using :func:`os.scandir` +internally. + +.. + +.. date: 2024-04-10-21-08-32 +.. gh-issue: 117586 +.. nonce: UCL__1 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by working with strings internally. + +.. + +.. date: 2024-04-10-20-59-10 +.. gh-issue: 117722 +.. nonce: oxIUEI +.. section: Library + +Change the new multi-separator support in :meth:`asyncio.StreamReader.readuntil` +to only accept tuples of separators rather than arbitrary iterables. + +.. + +.. date: 2024-04-09-23-22-21 +.. gh-issue: 117692 +.. nonce: EciInD +.. section: Library + +Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped +``builtin_function_or_method``. + +.. + +.. date: 2024-04-09-20-14-44 +.. gh-issue: 117348 +.. nonce: A2NAAz +.. section: Library + +Largely restored import time performance of configparser by avoiding +dataclasses. + +.. + +.. date: 2024-04-08-19-30-38 +.. gh-issue: 117641 +.. nonce: oaBGSJ +.. section: Library + +Speedup :func:`os.path.commonpath` on Unix. + +.. + +.. date: 2024-04-08-19-12-26 +.. gh-issue: 117663 +.. nonce: CPfc_p +.. section: Library + +Fix ``_simple_enum`` to detect aliases when multiple arguments are present +but only one is the member value. + +.. + +.. date: 2024-04-08-14-33-38 +.. gh-issue: 117636 +.. nonce: exnRKd +.. section: Library + +Speedup :func:`os.path.join`. + +.. + +.. date: 2024-04-08-03-23-22 +.. gh-issue: 117618 +.. nonce: -4DCUw +.. section: Library + +Support ``package.module`` as ``filename`` for ``break`` command of +:mod:`pdb` + +.. + +.. date: 2024-04-07-19-39-20 +.. gh-issue: 102247 +.. nonce: h8rqiX +.. section: Library + +the status codes enum with constants in http.HTTPStatus are updated to +include the names from RFC9110. This RFC includes some HTTP statuses +previously only used for WEBDAV and assigns more generic names to them. + +The old constants are preserved for backwards compatibility. + +.. + +.. date: 2024-04-07-18-42-09 +.. gh-issue: 117607 +.. nonce: C978BD +.. section: Library + +Speedup :func:`os.path.relpath`. + +.. + +.. date: 2024-04-06-20-31-09 +.. gh-issue: 117586 +.. nonce: UgWdRK +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by working with strings internally. + +.. + +.. date: 2024-04-06-18-41-36 +.. gh-issue: 117225 +.. nonce: tJh1Hw +.. section: Library + +Add colour to doctest output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-04-05-15-51-01 +.. gh-issue: 117566 +.. nonce: 54nABf +.. section: Library + +:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for +IPv4-mapped loopback addresses, i.e. addresses in the +``::ffff:127.0.0.0/104`` address space. + +.. + +.. date: 2024-04-05-13-38-53 +.. gh-issue: 117546 +.. nonce: lWjhHE +.. section: Library + +Fix issue where :func:`os.path.realpath` stopped resolving symlinks after +encountering a symlink loop on POSIX. + +.. + +.. date: 2024-04-04-15-28-12 +.. gh-issue: 116720 +.. nonce: aGhXns +.. section: Library + +Improved behavior of :class:`asyncio.TaskGroup` when an external +cancellation collides with an internal cancellation. For example, when two +task groups are nested and both experience an exception in a child task +simultaneously, it was possible that the outer task group would misbehave, +because its internal cancellation was swallowed by the inner task group. + +In the case where a task group is cancelled externally and also must raise +an :exc:`ExceptionGroup`, it will now call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next :keyword:`await`, +so the cancellation is not lost. + +An added benefit of these changes is that task groups now preserve the +cancellation count (:meth:`asyncio.Task.cancelling`). + +In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now +reset the undocumented ``_must_cancel`` flag when the cancellation count +reaches zero. + +.. + +.. date: 2024-04-03-16-01-31 +.. gh-issue: 117516 +.. nonce: 7DlHje +.. section: Library + +Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-03-15-04-23 +.. gh-issue: 117503 +.. nonce: NMfwup +.. section: Library + +Fix support of non-ASCII user names in bytes paths in +:func:`os.path.expanduser` on Posix. + +.. + +.. date: 2024-04-02-11-17-44 +.. gh-issue: 117394 +.. nonce: 2aoSlb +.. section: Library + +:func:`os.path.ismount` is now 2-3 times faster if the user has permissions. + +.. + +.. date: 2024-03-29-15-14-51 +.. gh-issue: 117313 +.. nonce: ks_ONu +.. section: Library + +Only treat ``'\n'``, ``'\r'`` and ``'\r\n'`` as line separators in +re-folding the :mod:`email` messages. Preserve control characters ``'\v'``, +``'\f'``, ``'\x1c'``, ``'\x1d'`` and ``'\x1e'`` and Unicode line separators +``'\x85'``, ``'\u2028'`` and ``'\u2029'`` as is. + +.. + +.. date: 2024-03-29-12-21-40 +.. gh-issue: 117142 +.. nonce: U0agfh +.. section: Library + +Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). + +.. + +.. date: 2024-03-26-15-29-39 +.. gh-issue: 66543 +.. nonce: OZBhU5 +.. section: Library + +Add the :func:`mimetypes.guess_file_type` function which works with file +path. Passing file path instead of URL in :func:`~mimetypes.guess_type` is +:term:`soft deprecated`. + +.. + +.. date: 2024-03-20-00-11-39 +.. gh-issue: 68583 +.. nonce: mIlxxb +.. section: Library + +webbrowser CLI: replace getopt with argparse, add long options. Patch by +Hugo van Kemenade. + +.. + +.. date: 2024-03-17-18-24-23 +.. gh-issue: 116871 +.. nonce: 9uSl8M +.. section: Library + +Name suggestions for :exc:`AttributeError` and :exc:`ImportError` now only +include underscored names if the original name was underscored. + +.. + +.. date: 2024-02-28-11-51-51 +.. gh-issue: 116023 +.. nonce: CGYhFh +.. section: Library + +Don't show empty fields (value ``None`` or ``[]``) in :func:`ast.dump` by +default. Add ``show_empty=False`` parameter to optionally show them. + +.. + +.. date: 2024-02-28-10-41-24 +.. gh-issue: 115961 +.. nonce: P-_DU0 +.. section: Library + +Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived +file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and +:mod:`zipfile`. The value of the :attr:`!mode` attribute of +:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string +(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the +readable file-like object returned by :meth:`zipfile.ZipFile.open` was +changed from ``'r'`` to ``'rb'``. + +.. + +.. date: 2024-02-11-07-31-43 +.. gh-issue: 82062 +.. nonce: eeS6w7 +.. section: Library + +Fix :func:`inspect.signature()` to correctly handle parameter defaults on +methods in extension modules that use names defined in the module namespace. + +.. + +.. date: 2024-01-19-05-40-46 +.. gh-issue: 83856 +.. nonce: jN5M80 +.. section: Library + +Honor :mod:`atexit` for all :mod:`multiprocessing` start methods + +.. + +.. date: 2023-12-14-02-51-38 +.. gh-issue: 113081 +.. nonce: S-9Qyn +.. section: Library + +Print colorized exception just like built-in traceback in :mod:`pdb` + +.. + +.. date: 2023-12-07-20-05-54 +.. gh-issue: 112855 +.. nonce: ph4ehh +.. section: Library + +Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney +Gale. + +.. + +.. date: 2023-11-07-22-41-42 +.. gh-issue: 111744 +.. nonce: TbLxF0 +.. section: Library + +Support opcode events in :mod:`bdb` + +.. + +.. date: 2023-10-24-12-39-04 +.. gh-issue: 109617 +.. nonce: YoI8TV +.. section: Library + +:mod:`!ncurses`: fixed a crash that could occur on macOS 13 or earlier when +Python was built with Apple Xcode 15's SDK. + +.. + +.. date: 2023-10-20-03-50-17 +.. gh-issue: 83151 +.. nonce: bcsD40 +.. section: Library + +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access +the local variables of the current frame, which made it possible for +multi-scope code like generators or nested function to work. + +.. + +.. date: 2023-10-02-10-35-58 +.. gh-issue: 110209 +.. nonce: b5zfIz +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`types.GeneratorType` and +:class:`types.CoroutineType` for type hinting purposes. Patch by James +Hilton-Balfe. + +.. + +.. date: 2023-08-21-10-34-43 +.. gh-issue: 108191 +.. nonce: GZM3mv +.. section: Library + +The :class:`types.SimpleNamespace` now accepts an optional positional +argument which specifies initial values of attributes as a dict or an +iterable of key-value pairs. + +.. + +.. date: 2023-05-28-11-25-18 +.. gh-issue: 62090 +.. nonce: opAhDn +.. section: Library + +Fix assertion errors caused by whitespace in metavars or ``SUPPRESS``-ed +groups in :mod:`argparse` by simplifying usage formatting. Patch by Ali +Hamdan. + +.. + +.. date: 2023-03-03-21-13-08 +.. gh-issue: 102402 +.. nonce: fpkRO1 +.. section: Library + +Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug +related to floating point math. + +.. + +.. date: 2022-12-14-15-53-38 +.. gh-issue: 100242 +.. nonce: Ny7VUO +.. section: Library + +Bring pure Python implementation ``functools.partial.__new__`` more in line +with the C-implementation by not just always checking for the presence of +the attribute ``'func'`` on the first argument of ``partial``. Instead, both +the Python version and the C version perform an ``isinstance(func, +partial)`` check on the first argument of ``partial``. + +.. + +.. date: 2022-11-23-17-16-31 +.. gh-issue: 99730 +.. nonce: bDQdaX +.. section: Library + +HEAD requests are no longer upgraded to GET request during redirects in +urllib. + +.. + +.. date: 2022-10-24-12-05-19 +.. gh-issue: 66410 +.. nonce: du4UKW +.. section: Library + +Setting the :mod:`!tkinter` module global :data:`!wantobjects` to ``2`` +before creating the :class:`~tkinter.Tk` object or call the +:meth:`!wantobjects` method of the :class:`!Tk` object with argument +``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module +to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), +depending on their internal representation in Tcl, instead of always ``str``. +:data:`!tkinter.wantobjects` is now set to ``2`` by default. + +.. + +.. bpo: 40943 +.. date: 2020-06-10-19-24-17 +.. nonce: vjiiN_ +.. section: Library + +Fix several IndexError when parse emails with truncated Message-ID, address, +routes, etc, e.g. ``example@``. + +.. + +.. bpo: 39324 +.. date: 2020-01-14-09-46-51 +.. nonce: qUcDrM +.. section: Library + +Add mime type mapping for .md <-> text/markdown + +.. + +.. bpo: 18108 +.. date: 2019-09-09-18-18-34 +.. nonce: ajPLAO +.. section: Library + +:func:`shutil.chown` now supports *dir_fd* and *follow_symlinks* keyword +arguments. + +.. + +.. bpo: 30988 +.. date: 2019-08-29-20-26-08 +.. nonce: b-_h5O +.. section: Library + +Fix parsing of emails with invalid address headers having a leading or +trailing dot. Patch by tsufeki. + +.. + +.. bpo: 32839 +.. date: 2018-02-13-10-02-54 +.. nonce: McbVz3 +.. section: Library + +Add the :meth:`!after_info` method for Tkinter widgets. + +.. + +.. date: 2024-04-25-22-12-20 +.. gh-issue: 117928 +.. nonce: LKdTno +.. section: Documentation + +The minimum Sphinx version required for the documentation is now 6.2.1. + +.. + +.. date: 2024-05-07-21-15-47 +.. gh-issue: 118734 +.. nonce: --GHiS +.. section: Build + +Fixes Windows build when invoked directly (not through the :file:`build.bat` +script) without specifying a value for ``UseTIER2``. + +.. + +.. date: 2024-05-06-00-39-06 +.. gh-issue: 115119 +.. nonce: LT27pF +.. section: Build + +The :file:`configure` option :option:`--with-system-libmpdec` now defaults +to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python +3.15. + +.. + +.. date: 2024-04-15-08-35-06 +.. gh-issue: 117845 +.. nonce: IowzyW +.. section: Build + +Fix building against recent libedit versions by detecting readline hook +signatures in :program:`configure`. + +.. + +.. date: 2024-04-14-19-35-35 +.. gh-issue: 116622 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on Android. + +.. + +.. date: 2024-04-09-12-59-06 +.. gh-issue: 117645 +.. nonce: 0oEVAa +.. section: Build + +Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from +10 MiB to 20 MiB. Patch by Victor Stinner. + +.. + +.. date: 2024-02-13-15-31-28 +.. gh-issue: 115119 +.. nonce: FnQzAW +.. section: Build + +:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` +dependencies if the :option:`--with-system-libmpdec` option is given. + +.. + +.. date: 2024-05-02-09-28-04 +.. gh-issue: 115119 +.. nonce: cUKMXo +.. section: Windows + +Update Windows installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Windows + +:func:`os.mkdir` now accepts *mode* of ``0o700`` to restrict the new +directory to the current user. + +.. + +.. date: 2024-04-29-13-53-25 +.. gh-issue: 118347 +.. nonce: U5ZRm_ +.. section: Windows + +Fixes launcher updates not being installed. + +.. + +.. date: 2024-04-26-14-23-07 +.. gh-issue: 118293 +.. nonce: ohhPtW +.. section: Windows + +The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` +flag when spawning processes to tell Windows not to change the mouse cursor. + +.. + +.. date: 2024-04-15-21-23-34 +.. gh-issue: 115009 +.. nonce: uhisHP +.. section: Windows + +Update Windows installer to use SQLite 3.45.3. + +.. + +.. date: 2024-04-12-14-02-58 +.. gh-issue: 90329 +.. nonce: YpEeaO +.. section: Windows + +Suppress the warning displayed on virtual environment creation when the +requested and created paths differ only by a short (8.3 style) name. +Warnings will continue to be shown if a junction or symlink in the path +caused the venv to be created in a different location than originally +requested. + +.. + +.. date: 2024-04-12-13-18-42 +.. gh-issue: 117786 +.. nonce: LpI01s +.. section: Windows + +Fixes virtual environments not correctly launching when created from a Store +install. + +.. + +.. date: 2024-05-03-12-13-27 +.. gh-issue: 115119 +.. nonce: ltDtoR +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-04-19-08-40-00 +.. gh-issue: 114099 +.. nonce: _iDfrQ +.. section: macOS + +iOS preprocessor symbol usage was made compatible with older macOS SDKs. + +.. + +.. date: 2024-04-15-21-19-39 +.. gh-issue: 115009 +.. nonce: IdxH9N +.. section: macOS + +Update macOS installer to use SQLite 3.45.3. + +.. + +.. date: 2022-04-17-01-07-42 +.. gh-issue: 91629 +.. nonce: YBGAAt +.. section: macOS + +Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to +set :envvar:`PATH` when installing for the Fish shell. + +.. + +.. bpo: 34774 +.. date: 2018-09-23-01-36-39 +.. nonce: VeM-X- +.. section: IDLE + +Use user-selected color theme for Help => IDLE Doc. + +.. + +.. date: 2024-04-29-17-44-15 +.. gh-issue: 118124 +.. nonce: czQQ9G +.. section: C API + +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. Patch by +Victor Stinner. + +.. + +.. date: 2024-04-29-17-19-07 +.. gh-issue: 110850 +.. nonce: vcpLn1 +.. section: C API + +Add "Raw" variant of PyTime functions + +* :c:func:`PyTime_MonotonicRaw` +* :c:func:`PyTime_PerfCounterRaw` +* :c:func:`PyTime_TimeRaw` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-17-16-48-17 +.. gh-issue: 117987 +.. nonce: zsvNL1 +.. section: C API + +Restore functions removed in Python 3.13 alpha 1: + +* :c:func:`Py_SetPythonHome` +* :c:func:`Py_SetProgramName` +* :c:func:`PySys_SetArgvEx` +* :c:func:`PySys_SetArgv` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-16-13-34-01 +.. gh-issue: 117929 +.. nonce: HSr419 +.. section: C API + +Restore removed :c:func:`PyEval_InitThreads` function. Patch by Victor +Stinner. + +.. + +.. date: 2024-04-08-09-44-29 +.. gh-issue: 117534 +.. nonce: 54ZE_n +.. section: C API + +Improve validation logic in the C implementation of +:meth:`datetime.datetime.fromisoformat` to better handle invalid years. +Patch by Vlad Efanov. + +.. + +.. date: 2024-03-18-17-29-52 +.. gh-issue: 68114 +.. nonce: W7R_lI +.. section: C API + +Fixed skipitem()'s handling of the old 'w' and 'w#' formatters. These are +no longer supported and now raise an exception if used. + +.. + +.. date: 2024-03-13-17-48-24 +.. gh-issue: 111997 +.. nonce: 8ZbHlA +.. section: C API + +Add a C-API for firing monitoring events. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 96e59206cb1291..35f340f503df18 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -251,8 +251,8 @@ and "surrogatepass" error handlers. .. nonce: FM72m- .. section: Core and Builtins -speed up `PyObject_IsInstance` and `PyObject_IsSubclass` in the common case -that the second argument has metaclass `type`. +speed up ``PyObject_IsInstance`` and ``PyObject_IsSubclass`` in the common case +that the second argument has metaclass ``type``. .. @@ -261,8 +261,8 @@ that the second argument has metaclass `type`. .. nonce: ds5wQa .. section: Core and Builtins -Add a new `PyErr_FormatV` function, similar to `PyErr_Format` but accepting -a `va_list` argument. +Add a new ``PyErr_FormatV`` function, similar to ``PyErr_Format`` but accepting +a ``va_list`` argument. .. @@ -1284,7 +1284,7 @@ Add function :func:`sys.is_finalizing` to know about interpreter shutdown. .. section: Library Add a default limit for the amount of data xmlrpclib.gzip_decode will -return. This resolves CVE-2013-1753. +return. This resolves :cve:`2013-1753`. .. @@ -1345,7 +1345,7 @@ newer worked. .. section: Library The "ip" command is now used on Linux to determine MAC address in -uuid.getnode(). Pach by Bruno Cauet. +uuid.getnode(). Patch by Bruno Cauet. .. @@ -1522,7 +1522,7 @@ Fixed fcntl() with integer argument on 64-bit big-endian platforms. .. nonce: 62MLqr .. section: Library -Add an `--sort-keys` option to json.tool CLI. +Add an ``--sort-keys`` option to ``json.tool`` CLI. .. @@ -1745,7 +1745,7 @@ already failed. .. section: Library Make it possible to examine the errors from unittest discovery without -executing the test suite. The new `errors` attribute on TestLoader exposes +executing the test suite. The new ``errors`` attribute on ``TestLoader`` exposes these non-fatal errors encountered during discovery. .. @@ -2342,9 +2342,9 @@ as normal call attributes. .. nonce: Nghn-Y .. section: Library -load_tests() is now unconditionally run when it is present in a package's -__init__.py. TestLoader.loadTestsFromModule() still accepts use_load_tests, -but it is deprecated and ignored. A new keyword-only attribute `pattern` is +``load_tests()`` is now unconditionally run when it is present in a package's +``__init__.py``. ``TestLoader.loadTestsFromModule()`` still accepts use_load_tests, +but it is deprecated and ignored. A new keyword-only attribute ``pattern`` is added and documented. Patch given by Robert Collins, tweaked by Barry Warsaw. @@ -2648,7 +2648,7 @@ module. .. nonce: THJSYB .. section: Library -Changed FeedParser feed() to avoid O(N\ :sup:`2`) behavior when parsing long line. +Changed FeedParser feed() to avoid *O*\ (*n*\ :sup:`2`) behavior when parsing long line. Original patch by Raymond Hettinger. .. @@ -2736,8 +2736,8 @@ Convert posixmodule to use Argument Clinic. .. nonce: YccmZF .. section: Library -Add an *exists_ok* argument to `Pathlib.mkdir()` to mimic `mkdir -p` and -`os.makedirs()` functionality. When true, ignore FileExistsErrors. Patch +Add an *exists_ok* argument to ``Pathlib.mkdir()`` to mimic ``mkdir -p`` and +``os.makedirs()`` functionality. When true, ignore ``FileExistsErrors``. Patch by Berker Peksag. .. @@ -3447,7 +3447,8 @@ tkinter.ttk now works when default root window is not set. .. nonce: FE_PII .. section: Library -_tkinter.create() now creates tkapp object with wantobject=1 by default. +``_tkinter.create()`` now creates ``tkapp`` object with ``wantobjects=1`` by +default. .. @@ -3930,7 +3931,7 @@ has been called. .. nonce: 5CDoox .. section: Library -New keyword argument `unsafe` to Mock. It raises `AttributeError` incase of +New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` in case of an attribute startswith assert or assret. .. @@ -4030,7 +4031,7 @@ unittest.mock.MagicMock now supports division. Patch by Johannes Baiter. .. section: Library Fix arbitrary memory access in JSONDecoder.raw_decode with a negative second -parameter. Bug reported by Guido Vranken. (See also: CVE-2014-4616) +parameter. Bug reported by Guido Vranken. (See also: :cve:`2014-4616`) .. @@ -4173,7 +4174,7 @@ provide better security by default. .. nonce: FP5FY0 .. section: Library -`assertRaisesRegex` and `assertWarnsRegex` now raise a TypeError if the +``assertRaisesRegex`` and ``assertWarnsRegex`` now raise a ``TypeError`` if the second argument is not a string or compiled regex. .. @@ -4339,7 +4340,7 @@ these modules are not used. .. nonce: V1-XhC .. section: Library -Include the broadcast address in the usuable hosts for IPv6 in ipaddress. +Include the broadcast address in the usable hosts for IPv6 in ipaddress. .. diff --git a/Misc/NEWS.d/3.5.0b4.rst b/Misc/NEWS.d/3.5.0b4.rst index 2b1b98a4316fc1..e42d93689d01a0 100644 --- a/Misc/NEWS.d/3.5.0b4.rst +++ b/Misc/NEWS.d/3.5.0b4.rst @@ -129,7 +129,7 @@ Random.setstate() now validates the value of state last element. .. nonce: HvJf6T .. section: Library -Fixed an issue that caused `inspect.getsource` to return incorrect results +Fixed an issue that caused ``inspect.getsource`` to return incorrect results on nested functions. .. diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index 01fcd866a896ae..f9409b62e352ac 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,8 +15,8 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team -Oststrom +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team +Oststrom. .. @@ -710,9 +710,9 @@ Fixed the comparison of plistlib.Data with other types. .. nonce: RMRMtM .. section: Library -Fix an uninitialized variable in `ctypes.util`. +Fix an uninitialized variable in ``ctypes.util``. The bug only occurs on SunOS when the ctypes implementation searches for the -`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. +``crle`` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. .. diff --git a/Misc/NEWS.d/3.5.3.rst b/Misc/NEWS.d/3.5.3.rst index c3fcb67a4563f9..25db389ba5734f 100644 --- a/Misc/NEWS.d/3.5.3.rst +++ b/Misc/NEWS.d/3.5.3.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: zYPqUK .. release date: 2017-01-17 +.. section: Library There were no code changes between 3.5.3rc1 and 3.5.3 final. diff --git a/Misc/NEWS.d/3.5.3rc1.rst b/Misc/NEWS.d/3.5.3rc1.rst index bf4ef9302c9d1d..2424604249a65c 100644 --- a/Misc/NEWS.d/3.5.3rc1.rst +++ b/Misc/NEWS.d/3.5.3rc1.rst @@ -1048,7 +1048,7 @@ certs. .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. @@ -1251,7 +1251,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.5.4rc1.rst b/Misc/NEWS.d/3.5.4rc1.rst index d65d5d14ee78bb..d5a85b3a2d8666 100644 --- a/Misc/NEWS.d/3.5.4rc1.rst +++ b/Misc/NEWS.d/3.5.4rc1.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -46,8 +46,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.5.5rc1.rst b/Misc/NEWS.d/3.5.5rc1.rst index 9ccbf7b8060cd4..4a44840039e388 100644 --- a/Misc/NEWS.d/3.5.5rc1.rst +++ b/Misc/NEWS.d/3.5.5rc1.rst @@ -24,7 +24,7 @@ also be affected) .. nonce: Fd8kId .. section: Security -Fixed possible integer overflow in PyBytes_DecodeEscape, CVE-2017-1000158. +Fixed possible integer overflow in PyBytes_DecodeEscape, :cve:`2017-1000158`. Original patch by Jay Bosamiya; rebased to Python 3 by Miro HronÄok. .. diff --git a/Misc/NEWS.d/3.6.0.rst b/Misc/NEWS.d/3.6.0.rst index f9805cab28615e..d5c41f38838d93 100644 --- a/Misc/NEWS.d/3.6.0.rst +++ b/Misc/NEWS.d/3.6.0.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2016-12-23 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 98f1215fb91873..803c9fc5925fa6 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1113,9 +1113,9 @@ Fixed the comparison of plistlib.Data with other types. .. nonce: RMRMtM .. section: Library -Fix an uninitialized variable in `ctypes.util`. +Fix an uninitialized variable in ``ctypes.util``. The bug only occurs on SunOS when the ctypes implementation searches for the -`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. +``crle`` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. .. @@ -1401,7 +1401,7 @@ array is extended. .. section: Library doctest.DocFileTest and doctest.testfile() now support packages (module -splitted into multiple directories) for the package parameter. +split into multiple directories) for the package parameter. .. @@ -1484,9 +1484,9 @@ on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). .. nonce: RWN1jR .. section: Library -Add C functions :c:func:`_PyTraceMalloc_Track` and -:c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the -:mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get +Add C functions :c:func:`!_PyTraceMalloc_Track` and +:c:func:`!_PyTraceMalloc_Untrack` to track memory blocks using the +:mod:`tracemalloc` module. Add :c:func:`!_PyTraceMalloc_GetTraceback` to get the traceback of an object. .. @@ -3915,7 +3915,7 @@ Fix output of python-config --extension-suffix. .. nonce: yLO-r4 .. section: Tools/Demos -The pyvenv script has been deprecated in favour of `python3 -m venv`. +The pyvenv script has been deprecated in favour of ``python3 -m venv``. .. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst index 1b336d7bc5137a..89d68ab3f8078f 100644 --- a/Misc/NEWS.d/3.6.0a2.rst +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,7 +15,7 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team Oststrom. .. @@ -603,7 +603,7 @@ configuring text widget colors to a new function. .. nonce: RbyFuV .. section: IDLE -Rename many `idlelib/*.py` and `idle_test/test_*.py` files. Edit files to +Rename many ``idlelib/*.py`` and ``idle_test/test_*.py`` files. Edit files to replace old names with new names when the old name referred to the module rather than the class it contained. See the issue and IDLE section in What's New in 3.6 for more. diff --git a/Misc/NEWS.d/3.6.0a4.rst b/Misc/NEWS.d/3.6.0a4.rst index d613fd5d928b65..3abbdecb57038b 100644 --- a/Misc/NEWS.d/3.6.0a4.rst +++ b/Misc/NEWS.d/3.6.0a4.rst @@ -359,7 +359,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.6.0b1.rst b/Misc/NEWS.d/3.6.0b1.rst index 3fbae5c6a4b3a8..bd54cf601d053b 100644 --- a/Misc/NEWS.d/3.6.0b1.rst +++ b/Misc/NEWS.d/3.6.0b1.rst @@ -166,7 +166,7 @@ a DeprecationWarning. Patch by Emanuel Barry. .. nonce: aABzcL .. section: Core and Builtins -`dict` implementation is changed like PyPy. It is more compact and preserves +``dict`` implementation is changed like PyPy. It is more compact and preserves insertion order. (Concept developed by Raymond Hettinger and patch by Inada Naoki.) @@ -949,7 +949,7 @@ Add scrypt (password-based key derivation function) to hashlib module .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. diff --git a/Misc/NEWS.d/3.6.0b2.rst b/Misc/NEWS.d/3.6.0b2.rst index 9413c6e01917d5..23dd69efb23b88 100644 --- a/Misc/NEWS.d/3.6.0b2.rst +++ b/Misc/NEWS.d/3.6.0b2.rst @@ -215,7 +215,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. diff --git a/Misc/NEWS.d/3.6.2.rst b/Misc/NEWS.d/3.6.2.rst index dba43d146df954..ee50670bd9f442 100644 --- a/Misc/NEWS.d/3.6.2.rst +++ b/Misc/NEWS.d/3.6.2.rst @@ -3,5 +3,6 @@ .. no changes: True .. nonce: F9ENBV .. release date: 2017-07-17 +.. section: Library No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.2rc1.rst b/Misc/NEWS.d/3.6.2rc1.rst index 28eb88f79130c5..8e28bc9691921b 100644 --- a/Misc/NEWS.d/3.6.2rc1.rst +++ b/Misc/NEWS.d/3.6.2rc1.rst @@ -5,8 +5,8 @@ .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.6.2rc2.rst b/Misc/NEWS.d/3.6.2rc2.rst index 8c6545f6dbbeec..5ae7425828b692 100644 --- a/Misc/NEWS.d/3.6.2rc2.rst +++ b/Misc/NEWS.d/3.6.2rc2.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. diff --git a/Misc/NEWS.d/3.6.3.rst b/Misc/NEWS.d/3.6.3.rst index 4d591d77ffe545..58fd009aea1fed 100644 --- a/Misc/NEWS.d/3.6.3.rst +++ b/Misc/NEWS.d/3.6.3.rst @@ -4,7 +4,7 @@ .. release date: 2017-10-03 .. section: Library -Re-allow arbitrary iterables in `concurrent.futures.as_completed()`. Fixes +Re-allow arbitrary iterables in ``concurrent.futures.as_completed()``. Fixes regression in 3.6.3rc1. .. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst index 4b2aae9dc88441..6a20e07f05956c 100644 --- a/Misc/NEWS.d/3.6.3rc1.rst +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -24,8 +24,8 @@ fixes. .. nonce: 0yiA5Q .. section: Core and Builtins -Fix an assertion failure in `subprocess.Popen()` on Windows, in case the env -argument has a bad keys() method. Patch by Oren Milman. +Fix an assertion failure in ``subprocess.Popen()`` on Windows, in case the env +argument has a bad ``keys()`` method. Patch by Oren Milman. .. @@ -34,7 +34,7 @@ argument has a bad keys() method. Patch by Oren Milman. .. nonce: rS-FlC .. section: Core and Builtins -Fix an assertion failure in `PyErr_WriteUnraisable()` in case of an +Fix an assertion failure in ``PyErr_WriteUnraisable()`` in case of an exception with a bad ``__module__`` attribute. Patch by Oren Milman. .. @@ -85,7 +85,7 @@ wrong line (typically the first line of the file). .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. @@ -95,7 +95,7 @@ plans to remove the functions from sys/types.h. .. nonce: t8QggK .. section: Core and Builtins -Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +Fix an assertion failure in ``zipimport.zipimporter.get_data`` on Windows, when the return value of ``pathname.replace('/','\\')`` isn't a string. Patch by Oren Milman. @@ -106,7 +106,7 @@ Patch by Oren Milman. .. nonce: YMduKF .. section: Core and Builtins -Fix an assertion failure in the write() method of `io.TextIOWrapper`, when +Fix an assertion failure in the ``write()`` method of ``io.TextIOWrapper``, when the encoder doesn't return a bytes object. Patch by Oren Milman. .. @@ -116,7 +116,7 @@ the encoder doesn't return a bytes object. Patch by Oren Milman. .. nonce: dRJzqR .. section: Core and Builtins -Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +Fix a crash in some methods of ``io.TextIOWrapper``, when the decoder's state is invalid. Patch by Oren Milman. .. @@ -477,7 +477,7 @@ attributes to help the garbage collector to destroy all widgets. .. nonce: 1t2hn5 .. section: Library -Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +Fix ``copyreg._slotnames()`` mangled attribute calculation for classes whose name begins with an underscore. Patch by Shane Harvey. .. @@ -585,7 +585,7 @@ socket.close() now ignores ECONNRESET error. .. nonce: CLvEvV .. section: Library -Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. +Fix out of bounds write in ``asyncio.CFuture.remove_done_callback()``. .. @@ -933,7 +933,7 @@ Add tests for configdialog keys tab. Patch by Cheryl Sabella. .. nonce: sqE1FS .. section: IDLE -IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +IDLE: Calltips use ``inspect.signature`` instead of ``inspect.getfullargspec``. This improves calltips for builtins converted to use Argument Clinic. Patch by Louie Lu. diff --git a/Misc/NEWS.d/3.6.4rc1.rst b/Misc/NEWS.d/3.6.4rc1.rst index ae4534be62c5c4..afa5b8b0efb148 100644 --- a/Misc/NEWS.d/3.6.4rc1.rst +++ b/Misc/NEWS.d/3.6.4rc1.rst @@ -138,7 +138,7 @@ integer with binary base. .. section: Core and Builtins Fixed an assertion failure in Python parser in case of a bad -`unicodedata.normalize()`. Patch by Oren Milman. +``unicodedata.normalize()``. Patch by Oren Milman. .. @@ -147,7 +147,7 @@ Fixed an assertion failure in Python parser in case of a bad .. nonce: wT9Iy7 .. section: Core and Builtins -Raise a `TypeError` with a helpful error message when class creation fails +Raise a ``TypeError`` with a helpful error message when class creation fails due to a metaclass with a bad ``__prepare__()`` method. Patch by Oren Milman. @@ -158,7 +158,7 @@ Milman. .. nonce: OxwINs .. section: Core and Builtins -Fix an assertion failure in `_warnings.warn()` in case of a bad ``__name__`` +Fix an assertion failure in ``_warnings.warn()`` in case of a bad ``__name__`` global. Patch by Oren Milman. .. @@ -168,8 +168,8 @@ global. Patch by Oren Milman. .. nonce: VomaFa .. section: Core and Builtins -Fix an assertion failure in `json`, in case `_json.make_encoder()` received -a bad `encoder()` argument. Patch by Oren Milman. +Fix an assertion failure in ``json``, in case ``_json.make_encoder()`` received +a bad ``encoder()`` argument. Patch by Oren Milman. .. @@ -189,7 +189,7 @@ such a module. Patch by Oren Milman. .. nonce: r7m2sj .. section: Core and Builtins -Fix an assertion failure in `ctypes` class definition, in case the class has +Fix an assertion failure in ``ctypes`` class definition, in case the class has an attribute whose name is specified in ``_anonymous_`` but not in ``_fields_``. Patch by Oren Milman. @@ -200,7 +200,7 @@ an attribute whose name is specified in ``_anonymous_`` but not in .. nonce: o06iKD .. section: Core and Builtins -Fix an assertion failure in `_random.Random.seed()` in case the argument has +Fix an assertion failure in ``_random.Random.seed()`` in case the argument has a bad ``__abs__()`` method. Patch by Oren Milman. .. @@ -220,7 +220,7 @@ string. Patch by Oren Milman. .. nonce: bNE2l- .. section: Core and Builtins -Fix a crash in the ``__setstate__()`` method of `ctypes._CData`, in case of +Fix a crash in the ``__setstate__()`` method of ``ctypes._CData``, in case of a bad ``__dict__``. Patch by Oren Milman. .. @@ -240,8 +240,8 @@ float with a bad as_integer_ratio() method. Patch by Oren Milman. .. nonce: 7lzaKV .. section: Core and Builtins -Fix an assertion failure in `warnings.warn_explicit`, when the return value -of the received loader's get_source() has a bad splitlines() method. Patch +Fix an assertion failure in ``warnings.warn_explicit``, when the return value +of the received loader's ``get_source()`` has a bad ``splitlines()`` method. Patch by Oren Milman. .. @@ -251,8 +251,8 @@ by Oren Milman. .. nonce: j7ZvN_ .. section: Core and Builtins -`PyErr_PrintEx()` clears now the ignored exception that may be raised by -`_PySys_SetObjectId()`, for example when no memory. +``PyErr_PrintEx()`` clears now the ignored exception that may be raised by +``_PySys_SetObjectId()``, for example when no memory. .. @@ -599,8 +599,8 @@ On Windows, faulthandler.enable() now ignores MSC and COM exceptions. .. nonce: XrVMME .. section: Library -Prevent crashes in `_elementtree` due to unsafe cleanup of `Element.text` -and `Element.tail`. Patch by Oren Milman. +Prevent crashes in ``_elementtree`` due to unsafe cleanup of ``Element.text`` +and ``Element.tail``. Patch by Oren Milman. .. @@ -813,8 +813,8 @@ reference leaks. .. nonce: lo7FQX .. section: Tests -Add the `set_nomemory(start, stop)` and `remove_mem_hooks()` functions to -the _testcapi module. +Add the ``set_nomemory(start, stop)`` and ``remove_mem_hooks()`` functions to +the ``_testcapi`` module. .. @@ -897,7 +897,7 @@ Prevent double substitution of prefix in python-config.sh. .. nonce: KUDjno .. section: Build -Avoid wholesale rebuild after `make regen-all` if nothing changed. +Avoid wholesale rebuild after ``make regen-all`` if nothing changed. .. @@ -1123,7 +1123,7 @@ and Py_SetPath() .. nonce: Q3T_8n .. section: C API -The `PyExc_RecursionErrorInst` singleton is removed and -`PyErr_NormalizeException()` does not use it anymore. This singleton is +The ``PyExc_RecursionErrorInst`` singleton is removed and +``PyErr_NormalizeException()`` does not use it anymore. This singleton is persistent and its members being never cleared may cause a segfault during finalization of the interpreter. See also issue #22898. diff --git a/Misc/NEWS.d/3.6.5rc1.rst b/Misc/NEWS.d/3.6.5rc1.rst index 448baed5413ecb..3d14cc49049c8f 100644 --- a/Misc/NEWS.d/3.6.5rc1.rst +++ b/Misc/NEWS.d/3.6.5rc1.rst @@ -15,7 +15,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -283,7 +283,7 @@ Make sure sys.argv remains as a list when running trace. .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -599,7 +599,7 @@ deprecation. .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. diff --git a/Misc/NEWS.d/3.6.6rc1.rst b/Misc/NEWS.d/3.6.6rc1.rst index 9624195c79043b..f70649af055b4e 100644 --- a/Misc/NEWS.d/3.6.6rc1.rst +++ b/Misc/NEWS.d/3.6.6rc1.rst @@ -110,7 +110,7 @@ on Windows. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -248,7 +248,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -346,7 +346,7 @@ tree of tuples or lists with ``line_info=False`` and ``col_info=True``. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index bee424241fd712..fd6ba07b53a617 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -46,10 +46,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -75,8 +75,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. @@ -86,7 +86,7 @@ information. .. nonce: r7m2sj .. section: Core and Builtins -Fix an assertion failure in `ctypes` class definition, in case the class has +Fix an assertion failure in ``ctypes`` class definition, in case the class has an attribute whose name is specified in ``_anonymous_`` but not in ``_fields_``. Patch by Oren Milman. @@ -97,8 +97,8 @@ an attribute whose name is specified in ``_anonymous_`` but not in .. nonce: 0yiA5Q .. section: Core and Builtins -Fix an assertion failure in `subprocess.Popen()` on Windows, in case the env -argument has a bad keys() method. Patch by Oren Milman. +Fix an assertion failure in ``subprocess.Popen()`` on Windows, in case the env +argument has a bad ``keys()`` method. Patch by Oren Milman. .. @@ -107,7 +107,7 @@ argument has a bad keys() method. Patch by Oren Milman. .. nonce: rS-FlC .. section: Core and Builtins -Fix an assertion failure in `PyErr_WriteUnraisable()` in case of an +Fix an assertion failure in ``PyErr_WriteUnraisable()`` in case of an exception with a bad ``__module__`` attribute. Patch by Oren Milman. .. @@ -214,7 +214,7 @@ Fix possible undefined behavior in _PyObject_FastCall_Prepend. .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. @@ -224,7 +224,7 @@ plans to remove the functions from sys/types.h. .. nonce: t8QggK .. section: Core and Builtins -Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +Fix an assertion failure in ``zipimport.zipimporter.get_data`` on Windows, when the return value of ``pathname.replace('/','\\')`` isn't a string. Patch by Oren Milman. @@ -235,7 +235,7 @@ Patch by Oren Milman. .. nonce: YMduKF .. section: Core and Builtins -Fix an assertion failure in the write() method of `io.TextIOWrapper`, when +Fix an assertion failure in the ``write()`` method of ``io.TextIOWrapper``, when the encoder doesn't return a bytes object. Patch by Oren Milman. .. @@ -245,7 +245,7 @@ the encoder doesn't return a bytes object. Patch by Oren Milman. .. nonce: dRJzqR .. section: Core and Builtins -Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +Fix a crash in some methods of ``io.TextIOWrapper``, when the decoder's state is invalid. Patch by Oren Milman. .. @@ -1479,7 +1479,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. @@ -1855,7 +1855,7 @@ docserver attribute to None to break a reference cycle. .. nonce: gwnthq .. section: Library -Many asserts in `multiprocessing` are now more informative, and some error +Many asserts in ``multiprocessing`` are now more informative, and some error types have been changed to more specific ones. .. @@ -1896,7 +1896,7 @@ child exit. .. nonce: -2_YGj .. section: Library -`dis` now works with asynchronous generator and coroutine objects. Patch by +``dis`` now works with asynchronous generator and coroutine objects. Patch by George Collins based on diagnosis by Luciano Ramalho. .. @@ -1906,10 +1906,10 @@ George Collins based on diagnosis by Luciano Ramalho. .. nonce: huQi2Y .. section: Library -There are a number of uninformative asserts in the `multiprocessing` module, +There are a number of uninformative asserts in the ``multiprocessing`` module, as noted in issue 5001. This change fixes two of the most potentially problematic ones, since they are in error-reporting code, in the -`multiprocessing.managers.convert_to_error` function. (It also makes more +``multiprocessing.managers.convert_to_error`` function. (It also makes more informative a ValueError message.) The only potentially problematic change is that the AssertionError is now a TypeError; however, this should also help distinguish it from an AssertionError being *reported* by the @@ -1973,7 +1973,7 @@ attributes to help the garbage collector to destroy all widgets. .. nonce: 1t2hn5 .. section: Library -Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +Fix ``copyreg._slotnames()`` mangled attribute calculation for classes whose name begins with an underscore. Patch by Shane Harvey. .. @@ -1983,7 +1983,7 @@ name begins with an underscore. Patch by Shane Harvey. .. nonce: 2CFVCO .. section: Library -Allow `logging.config.fileConfig` to accept kwargs and/or args. +Allow ``logging.config.fileConfig`` to accept kwargs and/or args. .. @@ -2159,7 +2159,7 @@ socket.close() now ignores ECONNRESET error. .. nonce: CLvEvV .. section: Library -Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. +Fix out of bounds write in ``asyncio.CFuture.remove_done_callback()``. .. @@ -2406,7 +2406,7 @@ Don't log exceptions if Task/Future "cancel()" method was called. .. nonce: xihJ4Y .. section: Library -Fix path calculation in `imp.load_package()`, fixing it for cases when a +Fix path calculation in ``imp.load_package()``, fixing it for cases when a package is only shipped with bytecodes. Patch by Alexandru Ardelean. .. @@ -2516,7 +2516,7 @@ stdin.write() if the child process is still running but closed the pipe. .. nonce: CdOuSl .. section: Library -Addded empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ +Added empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ and __weakref__ creation. Patch by Aaron Hall. .. @@ -4947,8 +4947,8 @@ run test_zipfile64. .. nonce: lo7FQX .. section: Tests -Add the `set_nomemory(start, stop)` and `remove_mem_hooks()` functions to -the _testcapi module. +Add the ``set_nomemory(start, stop)`` and ``remove_mem_hooks()`` functions to +the ``_testcapi`` module. .. @@ -5076,7 +5076,7 @@ on the Android armv7 qemu emulator. .. nonce: 4f-VJK .. section: Build -Allow --with-lto to be used on all builds, not just `make profile-opt`. +Allow ``--with-lto`` to be used on all builds, not just ``make profile-opt``. .. @@ -5819,7 +5819,7 @@ Add tests for configdialog keys tab. Patch by Cheryl Sabella. .. nonce: sqE1FS .. section: IDLE -IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +IDLE: Calltips use ``inspect.signature`` instead of ``inspect.getfullargspec``. This improves calltips for builtins converted to use Argument Clinic. Patch by Louie Lu. @@ -6224,9 +6224,9 @@ Added the slice index converter in Argument Clinic. .. nonce: KPFC7o .. section: Tools/Demos -Argument Clinic now uses the converter `bool(accept={int})` rather than -`int` for semantical booleans. This avoids repeating the default value for -Python and C and will help in converting to `bool` in future. +Argument Clinic now uses the converter ``bool(accept={int})`` rather than +``int`` for semantical booleans. This avoids repeating the default value for +Python and C and will help in converting to ``bool`` in future. .. diff --git a/Misc/NEWS.d/3.7.0a2.rst b/Misc/NEWS.d/3.7.0a2.rst index 0f107d8c5f5a93..06ca32d37fa4fa 100644 --- a/Misc/NEWS.d/3.7.0a2.rst +++ b/Misc/NEWS.d/3.7.0a2.rst @@ -65,8 +65,8 @@ integer with binary base. .. nonce: MtgLCn .. section: Core and Builtins -Fix an assertion failure in `zipimporter.get_source()` in case of a bad -`zlib.decompress()`. Patch by Oren Milman. +Fix an assertion failure in ``zipimporter.get_source()`` in case of a bad +``zlib.decompress()``. Patch by Oren Milman. .. @@ -76,7 +76,7 @@ Fix an assertion failure in `zipimporter.get_source()` in case of a bad .. section: Core and Builtins Fixed an assertion failure in Python parser in case of a bad -`unicodedata.normalize()`. Patch by Oren Milman. +``unicodedata.normalize()``. Patch by Oren Milman. .. @@ -85,7 +85,7 @@ Fixed an assertion failure in Python parser in case of a bad .. nonce: wT9Iy7 .. section: Core and Builtins -Raise a `TypeError` with a helpful error message when class creation fails +Raise a ``TypeError`` with a helpful error message when class creation fails due to a metaclass with a bad ``__prepare__()`` method. Patch by Oren Milman. @@ -105,7 +105,7 @@ Importlib was instrumented with two dtrace probes to profile import timing. .. nonce: OxwINs .. section: Core and Builtins -Fix an assertion failure in `_warnings.warn()` in case of a bad ``__name__`` +Fix an assertion failure in ``_warnings.warn()`` in case of a bad ``__name__`` global. Patch by Oren Milman. .. @@ -115,7 +115,7 @@ global. Patch by Oren Milman. .. nonce: pRVTRB .. section: Core and Builtins -Improved the error message logic for object.__new__ and object.__init__. +Improved the error message logic for ``object.__new__`` and ``object.__init__``. .. @@ -124,8 +124,8 @@ Improved the error message logic for object.__new__ and object.__init__. .. nonce: VomaFa .. section: Core and Builtins -Fix an assertion failure in `json`, in case `_json.make_encoder()` received -a bad `encoder()` argument. Patch by Oren Milman. +Fix an assertion failure in ``json``, in case ``_json.make_encoder()`` received +a bad ``encoder()`` argument. Patch by Oren Milman. .. @@ -145,7 +145,7 @@ such a module. Patch by Oren Milman. .. nonce: o06iKD .. section: Core and Builtins -Fix an assertion failure in `_random.Random.seed()` in case the argument has +Fix an assertion failure in ``_random.Random.seed()`` in case the argument has a bad ``__abs__()`` method. Patch by Oren Milman. .. @@ -218,7 +218,7 @@ string. Patch by Oren Milman. .. nonce: bNE2l- .. section: Core and Builtins -Fix a crash in the ``__setstate__()`` method of `ctypes._CData`, in case of +Fix a crash in the ``__setstate__()`` method of ``ctypes._CData``, in case of a bad ``__dict__``. Patch by Oren Milman. .. @@ -238,8 +238,8 @@ float with a bad as_integer_ratio() method. Patch by Oren Milman. .. nonce: 7lzaKV .. section: Core and Builtins -Fix an assertion failure in `warnings.warn_explicit`, when the return value -of the received loader's get_source() has a bad splitlines() method. Patch +Fix an assertion failure in ``warnings.warn_explicit``, when the return value +of the received loader's ``get_source()`` has a bad ``splitlines()`` method. Patch by Oren Milman. .. @@ -286,8 +286,8 @@ On Windows, faulthandler.enable() now ignores MSC and COM exceptions. .. nonce: XrVMME .. section: Library -Prevent crashes in `_elementtree` due to unsafe cleanup of `Element.text` -and `Element.tail`. Patch by Oren Milman. +Prevent crashes in ``_elementtree`` due to unsafe cleanup of ``Element.text`` +and ``Element.tail``. Patch by Oren Milman. .. @@ -307,8 +307,8 @@ compiling. bm_regex_compile benchmark shows 14% performance improvements. .. section: Library The types of compiled regular objects and match objects are now exposed as -`re.Pattern` and `re.Match`. This adds information in pydoc output for the -re module. +``re.Pattern`` and ``re.Match``. This adds information in pydoc output for the +``re`` module. .. @@ -485,10 +485,10 @@ since Python 3.3.) .. nonce: cIMFJW .. section: Library -Reprs of subclasses of some collection and iterator classes (`bytearray`, -`array.array`, `collections.deque`, `collections.defaultdict`, -`itertools.count`, `itertools.repeat`) now contain actual type name insteads -of hardcoded name of the base class. +Reprs of subclasses of some collection and iterator classes (``bytearray``, +``array.array``, ``collections.deque``, ``collections.defaultdict``, +``itertools.count``, ``itertools.repeat``) now contain actual type name instead +of hardcoded names of the base class. .. @@ -584,7 +584,7 @@ Correct PCBuild/ case to PCbuild/ in build scripts and documentation. .. nonce: KUDjno .. section: Build -Avoid wholesale rebuild after `make regen-all` if nothing changed. +Avoid wholesale rebuild after ``make regen-all`` if nothing changed. .. @@ -657,8 +657,8 @@ for code and tests by Guilherme Polo and Cheryl Sabella, respectively. .. nonce: K_EjpO .. section: C API -Make `PyMapping_Keys()`, `PyMapping_Values()` and `PyMapping_Items()` always -return a `list` (rather than a `list` or a `tuple`). Patch by Oren Milman. +Make ``PyMapping_Keys()``, ``PyMapping_Values()`` and ``PyMapping_Items()`` always +return a ``list`` (rather than a ``list`` or a ``tuple``). Patch by Oren Milman. .. diff --git a/Misc/NEWS.d/3.7.0a3.rst b/Misc/NEWS.d/3.7.0a3.rst index a968616f55be68..08cabeda7e9a46 100644 --- a/Misc/NEWS.d/3.7.0a3.rst +++ b/Misc/NEWS.d/3.7.0a3.rst @@ -240,8 +240,8 @@ after shrinking a memory block. .. nonce: j7ZvN_ .. section: Core and Builtins -`PyErr_PrintEx()` clears now the ignored exception that may be raised by -`_PySys_SetObjectId()`, for example when no memory. +``PyErr_PrintEx()`` clears now the ignored exception that may be raised by +``_PySys_SetObjectId()``, for example when no memory. .. @@ -643,7 +643,7 @@ undocumented. .. nonce: RwietE .. section: Library -cProfile command line now accepts `-m module_name` as an alternative to +cProfile command line now accepts ``-m module_name`` as an alternative to script path. Patch by Sanyam Khurana. .. @@ -926,7 +926,7 @@ Fix multiprocessing.Process when stdout and/or stderr is closed or None. .. nonce: -1MBEy .. section: Library -Add `subnet_of` and `superset_of` containment tests to +Add ``subnet_of`` and ``superset_of`` containment tests to :class:`ipaddress.IPv6Network` and :class:`ipaddress.IPv4Network`. Patch by Michel Albert and Cheryl Sabella. @@ -948,7 +948,7 @@ for backward compatibility with Python 2.2, and was deprecated since Python .. nonce: IxCvGB .. section: Library -Add a ``subprocess.Popen(text=False)`` keyword argument to `subprocess` +Add a ``subprocess.Popen(text=False)`` keyword argument to ``subprocess`` functions to be more explicit about when the library should attempt to decode outputs into text. Patch by Andrew Clegg. @@ -1191,7 +1191,7 @@ interruptions. If it crashes, restart it when necessary. .. nonce: AniZuz .. section: Library -Added support for AF_UNIX socket in asyncio `create_datagram_endpoint`. +Added support for AF_UNIX socket in asyncio ``create_datagram_endpoint``. .. @@ -1609,7 +1609,7 @@ comparison functions. .. nonce: Q3T_8n .. section: C API -The `PyExc_RecursionErrorInst` singleton is removed and -`PyErr_NormalizeException()` does not use it anymore. This singleton is +The ``PyExc_RecursionErrorInst`` singleton is removed and +``PyErr_NormalizeException()`` does not use it anymore. This singleton is persistent and its members being never cleared may cause a segfault during finalization of the interpreter. See also issue #22898. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index ebae046a7a6ba0..679f72ee0a44d4 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -152,7 +152,7 @@ option or the ``PYTHONWARNINGS`` environment variable. .. nonce: PgGQaB .. section: Core and Builtins -`-X dev` now injects a ``'default'`` entry into sys.warnoptions, ensuring +``-X dev`` now injects a ``'default'`` entry into sys.warnoptions, ensuring that it behaves identically to actually passing ``-Wdefault`` at the command line. @@ -192,7 +192,7 @@ by Ivan Levkivskyi. .. nonce: mDeCLK .. section: Core and Builtins -The `atexit` module now has its callback stored per interpreter. +The ``atexit`` module now has its callback stored per interpreter. .. @@ -595,7 +595,7 @@ Add asyncio.get_running_loop() function. .. section: Library All class and static methods of builtin types now are correctly classified -by inspect.classify_class_attrs() and grouped in pydoc ouput. Added +by inspect.classify_class_attrs() and grouped in pydoc output. Added types.ClassMethodDescriptorType for unbound class methods of builtin types. .. @@ -792,7 +792,7 @@ to more readily adjust to platform dependent behaviour. .. nonce: ODpc9y .. section: Windows -Implement support for `subprocess.Popen(close_fds=True)` on Windows. Patch +Implement support for ``subprocess.Popen(close_fds=True)`` on Windows. Patch by Segev Finer. .. diff --git a/Misc/NEWS.d/3.7.0b1.rst b/Misc/NEWS.d/3.7.0b1.rst index d1beec9cdcc33a..b6477127818eb5 100644 --- a/Misc/NEWS.d/3.7.0b1.rst +++ b/Misc/NEWS.d/3.7.0b1.rst @@ -875,4 +875,4 @@ by Stéphane Wirtel .. section: C API Add C API access to the ``datetime.timezone`` constructor and -``datetime.timzone.UTC`` singleton. +``datetime.timezone.UTC`` singleton. diff --git a/Misc/NEWS.d/3.7.0b2.rst b/Misc/NEWS.d/3.7.0b2.rst index 9590914599bb86..702dbc960c018d 100644 --- a/Misc/NEWS.d/3.7.0b2.rst +++ b/Misc/NEWS.d/3.7.0b2.rst @@ -214,7 +214,7 @@ helper methods that can be used instead ``_dump_registry``, .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -497,7 +497,7 @@ deprecation. .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. diff --git a/Misc/NEWS.d/3.7.0b3.rst b/Misc/NEWS.d/3.7.0b3.rst index c86963b7e42daf..a0c4cb15dc2b40 100644 --- a/Misc/NEWS.d/3.7.0b3.rst +++ b/Misc/NEWS.d/3.7.0b3.rst @@ -4,7 +4,7 @@ .. release date: 2018-03-29 .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -26,7 +26,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index 1d4fc921406fd0..fd0ce25cd8fb41 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -46,8 +46,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -371,8 +371,8 @@ suffixes). .. nonce: o7G_UO .. section: Build -By default, modules configured in `Modules/Setup` are no longer built with -`-DPy_BUILD_CORE`. Instead, modules that specifically need that preprocessor +By default, modules configured in ``Modules/Setup`` are no longer built with +``-DPy_BUILD_CORE``. Instead, modules that specifically need that preprocessor definition include it in their individual entries. .. diff --git a/Misc/NEWS.d/3.7.0b5.rst b/Misc/NEWS.d/3.7.0b5.rst index fb29109869188b..459eaddbc91f59 100644 --- a/Misc/NEWS.d/3.7.0b5.rst +++ b/Misc/NEWS.d/3.7.0b5.rst @@ -26,7 +26,7 @@ module_globals is not a dict. .. nonce: kqBNzv .. section: Core and Builtins -The new `os.posix_spawn` added in 3.7.0b1 was removed as we are still +The new ``os.posix_spawn`` added in 3.7.0b1 was removed as we are still working on what the API should look like. Expect this in 3.8 instead. .. @@ -77,7 +77,7 @@ their body. Based on patch by Inada Naoki. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -254,8 +254,8 @@ default. .. nonce: C6Hnd1 .. section: Library -Do not simplify arguments to `typing.Union`. Now `Union[Manager, Employee]` -is not simplified to `Employee` at runtime. Such simplification previously +Do not simplify arguments to ``typing.Union``. Now ``Union[Manager, Employee]`` +is not simplified to ``Employee`` at runtime. Such simplification previously caused several bugs and limited possibilities for introspection. .. @@ -314,7 +314,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -324,7 +324,7 @@ event loop. .. nonce: pj2Mbb .. section: Library -Fix failure in `typing.get_type_hints()` when ClassVar was provided as a +Fix failure in ``typing.get_type_hints()`` when ClassVar was provided as a string forward reference. .. @@ -427,7 +427,7 @@ Donghee Na. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. @@ -466,7 +466,7 @@ Support arrays >=2GiB in :mod:`ctypes`. Patch by Segev Finer. .. nonce: E5gba1 .. section: Documentation -Document that `asyncio.wait()` does not cancel its futures on timeout. +Document that ``asyncio.wait()`` does not cancel its futures on timeout. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 0dc6e945719ec7..35b9e7fca27a7b 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -4,7 +4,7 @@ .. release date: 2019-02-03 .. section: Security -[CVE-2019-5010] Fix a NULL pointer deref in ssl module. The cert parser did +:cve:`2019-5010`: Fix a NULL pointer deref in ssl module. The cert parser did not handle CRL distribution points with empty DP or URI correctly. A malicious or buggy certificate can result into segfault. Vulnerability (TALOS-2018-0758) reported by Colin Read and Nicolas Edet of Cisco. @@ -50,7 +50,7 @@ files or create network connections. .. nonce: Ua9jMv .. section: Security -CVE-2018-14647: The C accelerated _elementtree module now initializes hash +:cve:`2018-14647`: The C accelerated _elementtree module now initializes hash randomization salt from _Py_HashSecret instead of libexpat's default CSPRNG. .. @@ -89,7 +89,7 @@ Fixed thread-safety of error handling in _ssl. .. nonce: TzSN4x .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -111,7 +111,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -882,7 +882,7 @@ Update valgrind suppression list to use .. nonce: GIOm_8 .. section: Core and Builtins -Added the "socket" option in the `stat.filemode()` Python implementation to +Added the "socket" option in the ``stat.filemode()`` Python implementation to match the C implementation. .. @@ -1271,8 +1271,8 @@ parentheses in the string representation. .. nonce: tDBciE .. section: Core and Builtins -Added support for the `setpgroup`, `resetids`, `setsigmask`, `setsigdef` and -`scheduler` parameters of `posix_spawn`. Patch by Pablo Galindo. +Added support for the ``setpgroup``, ``resetids``, ``setsigmask``, ``setsigdef`` and +``scheduler`` parameters of ``posix_spawn``. Patch by Pablo Galindo. .. @@ -1406,8 +1406,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -1735,7 +1735,7 @@ Patch by Zackery Spytz. .. nonce: UoC319 .. section: Core and Builtins -Fix a crash in `ctypes.cast()` in case the type argument is a ctypes +Fix a crash in ``ctypes.cast()`` in case the type argument is a ctypes structured data type. Patch by Eryk Sun and Oren Milman. .. @@ -1745,7 +1745,7 @@ structured data type. Patch by Eryk Sun and Oren Milman. .. nonce: jgYsSA .. section: Core and Builtins -Fix a crash in `os.utime()` in case of a bad ns argument. Patch by Oren +Fix a crash in ``os.utime()`` in case of a bad ns argument. Patch by Oren Milman. .. @@ -2006,8 +2006,8 @@ Improved support of custom data descriptors in :func:`help` and .. nonce: V4kNN3 .. section: Library -The `crypt` module now internally uses the `crypt_r()` library function -instead of `crypt()` when available. +The ``crypt`` module now internally uses the ``crypt_r()`` library function +instead of ``crypt()`` when available. .. @@ -2025,7 +2025,7 @@ Fixed help() on metaclasses. Patch by Sanyam Khurana. .. nonce: PutiOC .. section: Library -Expose ``raise(signum)`` as `raise_signal` +Expose ``raise(signum)`` as ``raise_signal`` .. @@ -2248,7 +2248,7 @@ the pool is still running. .. nonce: abB4BN .. section: Library -When a :class:`Mock` instance was used to wrap an object, if `side_effect` +When a :class:`Mock` instance was used to wrap an object, if ``side_effect`` is used in one of the mocks of it methods, don't call the original implementation and return the result of using the side effect the same way that it is done with return_value. @@ -2377,7 +2377,7 @@ compliant, and will not throw an exception on a trailing '%'. .. nonce: vepCSJ .. section: Library -The function `platform.popen` has been removed, it was deprecated since +The function ``platform.popen`` has been removed, it was deprecated since Python 3.3: use :func:`os.popen` instead. .. @@ -2519,7 +2519,7 @@ non-Windows systems. .. nonce: dQS1ng .. section: Library -Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s +Fix incorrect parsing of :class:`io.IncrementalNewlineDecoder`'s *translate* argument. .. @@ -2529,9 +2529,9 @@ Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s .. nonce: CulMN8 .. section: Library -Remove `StreamReaderProtocol._untrack_reader`. The call to `_untrack_reader` +Remove ``StreamReaderProtocol._untrack_reader``. The call to ``_untrack_reader`` is currently performed too soon, causing the protocol to forget about the -reader before `connection_lost` can run and feed the EOF to the reader. +reader before ``connection_lost`` can run and feed the EOF to the reader. .. @@ -2583,8 +2583,8 @@ polling for new events. .. nonce: ltSrtr .. section: Library -`importlib` no longer logs `wrote ` redundantly after -`(created|could not create) ` is already logged. Patch by +``importlib`` no longer logs ``wrote `` redundantly after +``(created|could not create) `` is already logged. Patch by Quentin Agren. .. @@ -2717,7 +2717,7 @@ Use :func:`socket.CMSG_SPACE` to calculate ancillary data size instead of .. nonce: rWBb43 .. section: Library -The `mailbox.mbox.get_string` function *from_* parameter can now +The ``mailbox.mbox.get_string`` function *from_* parameter can now successfully be set to a non-default value. .. @@ -2799,7 +2799,7 @@ children which are instances of ``Element`` subclasses. .. nonce: z2FbOp .. section: Library -:class:`smtplib.SMTP` objects now always have a `sock` attribute present +:class:`smtplib.SMTP` objects now always have a ``sock`` attribute present .. @@ -2914,7 +2914,7 @@ Fix inspect module polluted ``sys.modules`` when parsing .. nonce: Wo2PoJ .. section: Library -Add `mtime` argument to `gzip.compress` for reproducible output. Patch by +Add ``mtime`` argument to ``gzip.compress`` for reproducible output. Patch by Guo Ci Teo. .. @@ -3019,7 +3019,7 @@ argument. Patch by Andrés Delfino. .. nonce: rSPBW9 .. section: Library -In :class:`QueueHandler`, clear `exc_text` from :class:`LogRecord` to +In :class:`QueueHandler`, clear ``exc_text`` from :class:`LogRecord` to prevent traceback from being written twice. .. @@ -3060,8 +3060,8 @@ and will be removed in future Python versions. .. nonce: CUE8LU .. section: Library -Add deprecation warning when `loop` is used in methods: `asyncio.sleep`, -`asyncio.wait` and `asyncio.wait_for`. +Add deprecation warning when ``loop`` is used in methods: ``asyncio.sleep``, +``asyncio.wait`` and ``asyncio.wait_for``. .. @@ -3191,7 +3191,7 @@ the stream is deleted (garbage collected) without ``close()`` call. .. nonce: 3IPIH5 .. section: Library -`Enum._missing_`: raise `ValueError` if None returned and `TypeError` if +``Enum._missing_``: raise ``ValueError`` if None returned and ``TypeError`` if non-member is returned. .. @@ -3302,8 +3302,8 @@ issues on Windows. .. nonce: xL7-kG .. section: Library -Fix possible mojibake in the error message of `pwd.getpwnam` and -`grp.getgrnam` using string representation because of invisible characters +Fix possible mojibake in the error message of ``pwd.getpwnam`` and +``grp.getgrnam`` using string representation because of invisible characters or trailing whitespaces. Patch by William Grzybowski. .. @@ -3395,8 +3395,8 @@ Zackery Spytz. .. nonce: S0Irst .. section: Library -Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP -3131). +Fix parsing non-ASCII identifiers in :mod:`!lib2to3.pgen2.tokenize` +(:pep:`3131`). .. @@ -3592,7 +3592,7 @@ Python 3.5. .. nonce: V8Ou3K .. section: Library -Deprecate :meth:`__getitem__` methods of +Deprecate :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput`. @@ -4575,8 +4575,8 @@ OpenSSL 1.1.1 .. nonce: nzQgD8 .. section: Library -Release GIL on `grp.getgrnam`, `grp.getgrgid`, `pwd.getpwnam` and -`pwd.getpwuid` if reentrant variants of these functions are available. Patch +Release GIL on ``grp.getgrnam``, ``grp.getgrgid``, ``pwd.getpwnam`` and +``pwd.getpwuid`` if reentrant variants of these functions are available. Patch by William Grzybowski. .. @@ -4656,8 +4656,8 @@ default. .. nonce: C6Hnd1 .. section: Library -Do not simplify arguments to `typing.Union`. Now `Union[Manager, Employee]` -is not simplified to `Employee` at runtime. Such simplification previously +Do not simplify arguments to ``typing.Union``. Now ``Union[Manager, Employee]`` +is not simplified to ``Employee`` at runtime. Such simplification previously caused several bugs and limited possibilities for introspection. .. @@ -4736,7 +4736,7 @@ Patch by Zvi Effron .. nonce: taxbVT .. section: Library -Fix race condition with `ReadTransport.resume_reading` in Windows proactor +Fix race condition with ``ReadTransport.resume_reading`` in Windows proactor event loop. .. @@ -4746,7 +4746,7 @@ event loop. .. nonce: pj2Mbb .. section: Library -Fix failure in `typing.get_type_hints()` when ClassVar was provided as a +Fix failure in ``typing.get_type_hints()`` when ClassVar was provided as a string forward reference. .. @@ -5044,8 +5044,8 @@ functionality. .. nonce: C_K-J9 .. section: Library -`ConfigParser.items()` was fixed so that key-value pairs passed in via -`vars` are not included in the resulting output. +``ConfigParser.items()`` was fixed so that key-value pairs passed in via +:func:`vars` are not included in the resulting output. .. @@ -5150,7 +5150,7 @@ instead of a wrapper function for exit callbacks. .. nonce: B56Hc1 .. section: Library -Fix FD leak in `_SelectorSocketTransport` Patch by Vlad Starostin. +Fix FD leak in ``_SelectorSocketTransport`` Patch by Vlad Starostin. .. @@ -5777,7 +5777,7 @@ helper methods that can be used instead ``_dump_registry``, .. nonce: bvHDOc .. section: Library -Fixed `asyncio.Condition` issue which silently ignored cancellation after +Fixed ``asyncio.Condition`` issue which silently ignored cancellation after notifying and cancelling a conditional lock. Patch by Bar Harel. .. @@ -5992,7 +5992,7 @@ Add Ttk spinbox widget to :mod:`tkinter.ttk`. Patch by Alan D Moore. .. nonce: flC-dE .. section: Library -:mod:`profile` CLI accepts `-m module_name` as an alternative to script +:mod:`profile` CLI accepts ``-m module_name`` as an alternative to script path. .. @@ -6147,8 +6147,8 @@ Support arrays >=2GiB in :mod:`ctypes`. Patch by Segev Finer. .. nonce: pDsFJl .. section: Library -Removed support of arguments in `tkinter.ttk.Treeview.selection`. It was -deprecated in 3.6. Use specialized methods like `selection_set` for +Removed support of arguments in ``tkinter.ttk.Treeview.selection``. It was +deprecated in 3.6. Use specialized methods like ``selection_set`` for changing the selection. .. @@ -6224,7 +6224,7 @@ imported from ``typing`` directly. .. nonce: 2eVOYS .. section: Documentation -Fix the documentation about an unexisting `f_restricted` attribute in the +Fix the documentation about an unexisting ``f_restricted`` attribute in the frame object. Patch by Stéphane Wirtel .. @@ -6282,7 +6282,7 @@ Add documentation about the new command line interface of the gzip module. .. nonce: YO9CYm .. section: Documentation -chm document displays non-ASCII charaters properly on some MBCS Windows +chm document displays non-ASCII characters properly on some MBCS Windows systems. .. @@ -6311,7 +6311,7 @@ Document how passing coroutines to asyncio.wait() can be confusing. .. nonce: p9PoYv .. section: Documentation -Make clear that ``==`` operator sometimes is equivalent to `is`. The ``<``, +Make clear that ``==`` operator sometimes is equivalent to ``is``. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense. .. @@ -6409,7 +6409,7 @@ Improve the documentation of :func:`asyncio.open_connection`, .. nonce: E5gba1 .. section: Documentation -Document that `asyncio.wait()` does not cancel its futures on timeout. +Document that ``asyncio.wait()`` does not cancel its futures on timeout. .. @@ -6660,7 +6660,7 @@ Stéphane Wirtel .. nonce: w1m_8r .. section: Documentation -Improve docstrings for `pathlib.PurePath` subclasses. +Improve docstrings for ``pathlib.PurePath`` subclasses. .. @@ -6997,7 +6997,7 @@ Use 3072 RSA keys and SHA-256 signature for test certs and keys. .. nonce: H8fCGa .. section: Tests -Remove special condition for AIX in `test_subprocess.test_undecodable_env` +Remove special condition for AIX in ``test_subprocess.test_undecodable_env`` .. @@ -7006,7 +7006,7 @@ Remove special condition for AIX in `test_subprocess.test_undecodable_env` .. nonce: IsRDPB .. section: Tests -Fix `test_utf8_mode.test_cmd_line` for AIX +Fix ``test_utf8_mode.test_cmd_line`` for AIX .. @@ -7034,9 +7034,9 @@ Fix ftplib test for TLS 1.3 by reading from data socket. .. nonce: g7TwYm .. section: Tests -Fix `test_socket` on AIX 6.1 and later IPv6 zone id supports only -supported by inet_pton6_zone() Switch to runtime-based platform.system() to -establish current platform rather than build-time based sys.platform() +Fix ``test_socket`` on AIX 6.1 and later IPv6 zone id supports only +supported by ``inet_pton6_zone()``. Switch to runtime-based ``platform.system()`` to +establish current platform rather than build-time based ``sys.platform()`` .. @@ -7172,8 +7172,8 @@ Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of .. nonce: IKDsqu .. section: Tests -Making sure the `SMTPUTF8SimTests` class of tests gets run in -test_smtplib.py. +Making sure the ``SMTPUTF8SimTests`` class of tests gets run in +``test_smtplib.py``. .. @@ -7282,7 +7282,7 @@ CFLAGS to build third-party C extensions through distutils. .. nonce: XZTttb .. section: Build -Fix a compiler error when statically linking `pyexpat` in `Modules/Setup`. +Fix a compiler error when statically linking ``pyexpat`` in ``Modules/Setup``. .. @@ -7520,8 +7520,8 @@ suffixes). .. nonce: o7G_UO .. section: Build -By default, modules configured in `Modules/Setup` are no longer built with -`-DPy_BUILD_CORE`. Instead, modules that specifically need that preprocessor +By default, modules configured in ``Modules/Setup`` are no longer built with +``-DPy_BUILD_CORE``. Instead, modules that specifically need that preprocessor definition include it in their individual entries. .. @@ -8051,7 +8051,7 @@ Update macOS 10.9+ installer to Tcl/Tk 8.6.8. .. nonce: K6jCVG .. section: macOS -In :mod:`_scproxy`, drop the GIL when calling into ``SystemConfiguration`` +In :mod:`!_scproxy`, drop the GIL when calling into ``SystemConfiguration`` to avoid deadlocks. .. @@ -8090,7 +8090,7 @@ Fix imports in idlelib.window. .. nonce: QEaANl .. section: IDLE -Proper format `calltip` when the function has no docstring. +Proper format ``calltip`` when the function has no docstring. .. diff --git a/Misc/NEWS.d/3.8.0a2.rst b/Misc/NEWS.d/3.8.0a2.rst index 223126145c77f1..c8620aeea7f133 100644 --- a/Misc/NEWS.d/3.8.0a2.rst +++ b/Misc/NEWS.d/3.8.0a2.rst @@ -4,7 +4,7 @@ .. release date: 2019-02-25 .. section: Core and Builtins -Raise a :exc:`SyntaxError` when assigning a value to `__debug__` with the +Raise a :exc:`SyntaxError` when assigning a value to ``__debug__`` with the Assignment Operator. Contributed by Stéphane Wirtel and Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.8.0a3.rst b/Misc/NEWS.d/3.8.0a3.rst index 66308c6bca2e26..a06efb9abe795d 100644 --- a/Misc/NEWS.d/3.8.0a3.rst +++ b/Misc/NEWS.d/3.8.0a3.rst @@ -324,7 +324,7 @@ Raise ModuleNotFoundError in pyclbr when a module can't be found. Thanks to .. nonce: MDXLw6 .. section: Library -Switch the default format used for writing tars with mod:`tarfile` to the +Switch the default format used for writing tars with :mod:`tarfile` to the modern POSIX.1-2001 pax standard, from the vendor-specific GNU. Contributed by C.A.M. Gerlach. @@ -473,7 +473,7 @@ path string. .. nonce: NA_rXa .. section: Library -Ensure custom :func:`warnings.formatwarning` function can receive `line` as +Ensure custom :func:`warnings.formatwarning` function can receive ``line`` as positional argument. Based on patch by Tashrif Billah. .. @@ -533,10 +533,10 @@ Kumar Akshay. .. nonce: yffB3F .. section: Library -`pprint.pp` has been added to pretty-print objects with dictionary keys +``pprint.pp`` has been added to pretty-print objects with dictionary keys being sorted with their insertion order by default. Parameter *sort_dicts* -has been added to `pprint.pprint`, `pprint.pformat` and -`pprint.PrettyPrinter`. Contributed by Rémi Lapeyre. +has been added to ``pprint.pprint``, ``pprint.pformat`` and +``pprint.PrettyPrinter``. Contributed by Rémi Lapeyre. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 7e8bfa5c4364a9..edce71b2555a89 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -13,7 +13,7 @@ Fixes mishandling of pre-normalization characters in urlsplit(). .. nonce: 51E-DA .. section: Security -Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or +Address :cve:`2019-9740` by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised. @@ -168,7 +168,7 @@ decoder. Changing ``dict`` keys during iteration of the dict itself, ``keys()``, ``values()``, or ``items()`` will now be detected in certain corner cases where keys are deleted/added so that the number of keys isn't changed. A -`RuntimeError` will be raised after ``len(dict)`` iterations. Contributed by +``RuntimeError`` will be raised after ``len(dict)`` iterations. Contributed by Thomas Perl. .. @@ -255,7 +255,7 @@ all tags in a namespace. Patch by Stefan Behnel. .. nonce: Lpm-SI .. section: Library -`pathlib.path.link_to()` is now implemented. It creates a hard link pointing +``pathlib.path.link_to()`` is now implemented. It creates a hard link pointing to a path. .. @@ -580,7 +580,7 @@ is a dictionary. .. section: Library Calling ``stop()`` on an unstarted or stopped :func:`unittest.mock.patch` -object will now return `None` instead of raising :exc:`RuntimeError`, making +object will now return ``None`` instead of raising :exc:`RuntimeError`, making the method idempotent. Patch by Karthikeyan Singaravelan. .. @@ -600,7 +600,7 @@ exceptions. .. nonce: 9sjd38 .. section: Library -Add time module support and fix test_time faiures for VxWorks. +Add time module support and fix test_time failures for VxWorks. .. @@ -609,9 +609,9 @@ Add time module support and fix test_time faiures for VxWorks. .. nonce: i2Z1XR .. section: Library -Added support for keyword arguments `default_namespace` and -`xml_declaration` in functions ElementTree.tostring() and -ElementTree.tostringlist(). +Added support for keyword arguments ``default_namespace`` and +``xml_declaration`` in functions ``ElementTree.tostring()`` and +``ElementTree.tostringlist()``. .. @@ -744,7 +744,7 @@ Remove stale unix datagram socket before binding .. nonce: _4Q_bi .. section: Library -Implemented Happy Eyeballs in `asyncio.create_connection()`. Added two new +Implemented Happy Eyeballs in ``asyncio.create_connection()``. Added two new arguments, *happy_eyeballs_delay* and *interleave*, to specify Happy Eyeballs behavior. @@ -843,7 +843,7 @@ Using the code of the ``Tools/scripts/serve.py`` script as an example in the .. nonce: nF1pP1 .. section: Documentation -Added Documention for PyInterpreterState_Main(). +Added documentation for PyInterpreterState_Main(). .. @@ -945,7 +945,7 @@ P. Hemsley. .. nonce: __FTq9 .. section: Tests -Add a new :mod:`_testinternalcapi` module to test the internal C API. +Add a new :mod:`!_testinternalcapi` module to test the internal C API. .. @@ -1383,7 +1383,7 @@ Since Python 3.7.0, calling :c:func:`Py_DecodeLocale` before coerced and/or if the UTF-8 Mode is enabled by the user configuration. The LC_CTYPE coercion and UTF-8 Mode are now disabled by default to fix the mojibake issue. They must now be enabled explicitly (opt-in) using the new -:c:func:`_Py_PreInitialize` API with ``_PyPreConfig``. +:c:func:`!_Py_PreInitialize` API with ``_PyPreConfig``. .. @@ -1405,7 +1405,7 @@ only present in alpha releases of Python 3.8. Patch by Paul Ganssle. .. nonce: wpbWeb .. section: C API -Modify ``PyObject_Init`` to correctly increase the refcount of heap- -allocated Type objects. Also fix the refcounts of the heap-allocated types +Modify ``PyObject_Init`` to correctly increase the refcount of heap-allocated +Type objects. Also fix the refcounts of the heap-allocated types that were either doing this manually or not decreasing the type's refcount in tp_dealloc diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index c5e27b04ef8e8b..fc4e3a9bd887fb 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -4,7 +4,7 @@ .. release date: 2019-06-04 .. section: Security -CVE-2019-9948: Avoid file reading by disallowing ``local-file://`` and +:cve:`2019-9948`: Avoid file reading by disallowing ``local-file://`` and ``local_file://`` URL schemes in ``URLopener().open()`` and ``URLopener().retrieve()`` of :mod:`urllib.request`. @@ -146,8 +146,8 @@ constant expressions inside the f-string). .. nonce: VeVvhJ .. section: Core and Builtins -The `bytes.hex`, `bytearray.hex`, and `memoryview.hex` methods as well as -the `binascii.hexlify` and `b2a_hex` functions now have the ability to +The ``bytes.hex``, ``bytearray.hex``, and ``memoryview.hex`` methods as well as +the ``binascii.hexlify`` and ``b2a_hex`` functions now have the ability to include an optional separator between hex bytes. This functionality was inspired by MicroPython's hexlify implementation. @@ -198,8 +198,8 @@ any error. .. nonce: QwLa3P .. section: Core and Builtins -Only accept text after `# type: ignore` if the first character is ASCII. -This is to disallow things like `# type: ignoreé`. +Only accept text after ``# type: ignore`` if the first character is ASCII. +This is to disallow things like ``# type: ignoreé``. .. @@ -208,9 +208,9 @@ This is to disallow things like `# type: ignoreé`. .. nonce: EFRHZ3 .. section: Core and Builtins -Store text appearing after a `# type: ignore` comment in the AST. For -example a type ignore like `# type: ignore[E1000]` will have the string -`"[E1000]"` stored in its AST node. +Store text appearing after a ``# type: ignore`` comment in the AST. For +example a type ignore like ``# type: ignore[E1000]`` will have the string +``"[E1000]"`` stored in its AST node. .. @@ -414,7 +414,7 @@ Fix incorrect use of ``%p`` in format strings. Patch by Zackery Spytz. .. nonce: RO20OV .. section: Core and Builtins -builtins.help() now prefixes `async` for async functions +``builtins.help()`` now prefixes ``async`` for async functions. .. @@ -443,7 +443,7 @@ Added fix for broken symlinks in combination with pathlib .. section: Core and Builtins Added new trashcan macros to deal with a double deallocation that could -occur when the `tp_dealloc` of a subclass calls the `tp_dealloc` of a base +occur when the ``tp_dealloc`` of a subclass calls the ``tp_dealloc`` of a base class and that base class uses the trashcan mechanism. Patch by Jeroen Demeyer. @@ -600,7 +600,7 @@ default. .. nonce: sLULGQ .. section: Library -Fix destructor :class:`_pyio.BytesIO` and :class:`_pyio.TextIOWrapper`: +Fix destructor :class:`!_pyio.BytesIO` and :class:`!_pyio.TextIOWrapper`: initialize their ``_buffer`` attribute as soon as possible (in the class body), because it's used by ``__del__()`` which calls ``close()``. @@ -768,7 +768,7 @@ Taskaya .. nonce: u7cxu7 .. section: Library -PDB command `args` now display positional only arguments. Patch contributed +PDB command ``args`` now display positional only arguments. Patch contributed by Rémi Lapeyre. .. @@ -778,7 +778,7 @@ by Rémi Lapeyre. .. nonce: JkZORP .. section: Library -PDB command `args` now display keyword only arguments. Patch contributed by +PDB command ``args`` now display keyword only arguments. Patch contributed by Rémi Lapeyre. .. @@ -799,7 +799,7 @@ Add missing names to ``typing.__all__``: ``ChainMap``, ``ForwardRef``, .. section: Library Add SupportsIndex protocol to the typing module to allow type checking to -detect classes that can be passed to `hex()`, `oct()` and `bin()`. +detect classes that can be passed to ``hex()``, ``oct()`` and ``bin()``. .. @@ -1023,10 +1023,10 @@ reading metadata from third-party packages. .. nonce: iigeqk .. section: Library -When using `type_comments=True` in `ast.parse`, treat `# type: ignore` +When using ``type_comments=True`` in ``ast.parse``, treat ``# type: ignore`` followed by a non-alphanumeric character and then arbitrary text as a type ignore, instead of requiring nothing but whitespace or another comment. This -is to permit formations such as `# type: ignore[E1000]`. +is to permit formations such as ``# type: ignore[E1000]``. .. @@ -1066,7 +1066,7 @@ exposed to the user. Patch by Aviv Palivoda. .. nonce: WK8Y-k .. section: Library -In `shutil.copystat()`, first copy extended file attributes and then file +In ``shutil.copystat()``, first copy extended file attributes and then file permissions, since extended attributes can only be set on the destination while it is still writeable. @@ -1120,7 +1120,7 @@ add_done_callback correctly when the Future has already completed. .. nonce: 4payXb .. section: Library -Limit `max_workers` in `ProcessPoolExecutor` to 61 to work around a +Limit ``max_workers`` in ``ProcessPoolExecutor`` to 61 to work around a WaitForMultipleObjects limitation. .. @@ -1676,7 +1676,7 @@ documentation covers it independently. .. nonce: 6hg6J8 .. section: Documentation -Add detail to the documentation on the `pty.spawn` function. +Add detail to the documentation on the ``pty.spawn`` function. .. @@ -1704,7 +1704,7 @@ Added the context variable in glossary. .. nonce: Q7s2FB .. section: Documentation -Clarify that `copy()` is not part of the `MutableSequence` ABC. +Clarify that ``copy()`` is not part of the ``MutableSequence`` ABC. .. @@ -1713,7 +1713,7 @@ Clarify that `copy()` is not part of the `MutableSequence` ABC. .. nonce: jalAaQ .. section: Documentation -Make `codecs.StreamRecoder.writelines` take a list of bytes. +Make ``codecs.StreamRecoder.writelines`` take a list of bytes. .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 9818c17705074b..b0f63c3b9c3537 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -44,7 +44,7 @@ rendering the document page as HTML. (Contributed by Donghee Na in .. section: Security Update vendorized expat library version to 2.2.8, which resolves -CVE-2019-15903. +:cve:`2019-15903`. .. @@ -1304,7 +1304,7 @@ parameter. Patch by Pablo Galindo. .. nonce: J12cWT .. section: Library -Fixed `hmac.new` and `hmac.HMAC` to raise TypeError instead of ValueError +Fixed ``hmac.new`` and ``hmac.HMAC`` to raise TypeError instead of ValueError when the digestmod parameter, now required in 3.8, is omitted. Also clarified the hmac module documentation and docstrings. @@ -1384,7 +1384,7 @@ Nested subclasses of :class:`typing.NamedTuple` are now pickleable. .. nonce: hwrPN7 .. section: Library -Prevent :exc:`KeyError` thrown by :func:`_encoded_words.decode` when given +Prevent :exc:`KeyError` thrown by :func:`!_encoded_words.decode` when given an encoded-word with invalid content-type encoding from propagating all the way to :func:`email.message.get`. @@ -1395,8 +1395,8 @@ way to :func:`email.message.get`. .. nonce: S6Klvm .. section: Library -Deprecated the ``split()`` method in :class:`_tkinter.TkappType` in favour -of the ``splitlist()`` method which has more consistent and predicable +Deprecated the ``split()`` method in :class:`!_tkinter.TkappType` in favour +of the ``splitlist()`` method which has more consistent and predictable behavior. .. @@ -1478,7 +1478,7 @@ Removes _AwaitEvent from AsyncMock. .. section: Library Allow the rare code that wants to send invalid http requests from the -`http.client` library a way to do so. The fixes for bpo-30458 led to +``http.client`` library a way to do so. The fixes for bpo-30458 led to breakage for some projects that were relying on this ability to test their own behavior in the face of bad requests. @@ -1813,9 +1813,9 @@ Update importlib.metadata with changes from `importlib_metadata 0.21 .. nonce: 8zn2o3 .. section: Library -Remove `__code__` check in AsyncMock that incorrectly evaluated function -specs as async objects but failed to evaluate classes with `__await__` but -no `__code__` attribute defined as async objects. +Remove ``__code__`` check in AsyncMock that incorrectly evaluated function +specs as async objects but failed to evaluate classes with ``__await__`` but +no ``__code__`` attribute defined as async objects. .. @@ -1976,11 +1976,11 @@ thread at a time. .. nonce: kP-n4L .. section: Library -Subscripts to the `unittest.mock.call` objects now receive the same chaining +Subscripts to the ``unittest.mock.call`` objects now receive the same chaining mechanism as any other custom attributes, so that the following usage no -longer raises a `TypeError`: +longer raises a ``TypeError``: -call().foo().__getitem__('bar') +``call().foo().__getitem__('bar')`` Patch by blhsing @@ -2069,7 +2069,7 @@ Restores instantiation of Windows IOCP event loops from the non-main thread. .. section: Library Add default implementation of the :meth:`ast.NodeVisitor.visit_Constant` -method which emits a deprecation warning and calls corresponding methody +method which emits a deprecation warning and calls corresponding methods ``visit_Num()``, ``visit_Str()``, etc. .. @@ -2210,7 +2210,7 @@ Add C fastpath for statistics.NormalDist.inv_cdf() Patch by Donghee Na .. nonce: Ene6L- .. section: Library -Remove the deprecated method `threading.Thread.isAlive()`. Patch by Donghee +Remove the deprecated method ``threading.Thread.isAlive()``. Patch by Donghee Na. .. @@ -2260,8 +2260,8 @@ directories are added and that duplicates are excluded. .. nonce: xfvdb_ .. section: Library -Renamed and documented `test.bytecode_helper` as -`test.support.bytecode_helper`. Patch by Joannah Nanjekye. +Renamed and documented ``test.bytecode_helper`` as +``test.support.bytecode_helper``. Patch by Joannah Nanjekye. .. @@ -2289,8 +2289,8 @@ many small lines are passed. Patch by Sergey Fedoseev. .. nonce: ycbL2z .. section: Library -`ensurepip` now uses `importlib.resources.read_binary()` to read data -instead of `pkgutil.get_data()`. Patch by Joannah Nanjekye. +``ensurepip`` now uses ``importlib.resources.read_binary()`` to read data +instead of ``pkgutil.get_data()``. Patch by Joannah Nanjekye. .. @@ -2524,8 +2524,8 @@ Make internal attributes for statistics.NormalDist() private. .. nonce: S5am28 .. section: Library -Fix `NonCallableMock._call_matcher` returning tuple instead of `_Call` -object when `self._spec_signature` exists. Patch by Elizabeth Uselton +Fix ``NonCallableMock._call_matcher`` returning tuple instead of ``_Call`` +object when ``self._spec_signature`` exists. Patch by Elizabeth Uselton .. @@ -2534,7 +2534,7 @@ object when `self._spec_signature` exists. Patch by Elizabeth Uselton .. nonce: iXGuoi .. section: Library -Make `from tkinter import *` import only the expected objects. +Make ``from tkinter import *`` import only the expected objects. .. @@ -2673,7 +2673,7 @@ which had a too large value in some situations. .. nonce: 0i1MR- .. section: Library -Fixes a possible hang when using a timeout on `subprocess.run()` while +Fixes a possible hang when using a timeout on ``subprocess.run()`` while capturing output. If the child process spawned its own children or otherwise connected its stdout or stderr handles with another process, we could hang after the timeout was reached and our child was killed when @@ -2708,8 +2708,8 @@ The distutils ``bdist_wininst`` command is deprecated in Python 3.8, use .. nonce: O53a5S .. section: Library -When `Enum.__str__` is overridden in a derived class, the override will be -used by `Enum.__format__` regardless of whether mixin classes are present. +When ``Enum.__str__`` is overridden in a derived class, the override will be +used by ``Enum.__format__`` regardless of whether mixin classes are present. .. @@ -2857,8 +2857,8 @@ Patch by Justin Blanchard. Add formal support for UDPLITE sockets. Support was present before, but it is now easier to detect support with ``hasattr(socket, 'IPPROTO_UDPLITE')`` and there are constants defined for each of the values needed: -:py:obj:`socket.IPPROTO_UDPLITE`, :py:obj:`UDPLITE_SEND_CSCOV`, and -:py:obj:`UDPLITE_RECV_CSCOV`. Patch by Gabe Appleton. +``socket.IPPROTO_UDPLITE``, ``UDPLITE_SEND_CSCOV``, and +``UDPLITE_RECV_CSCOV``. Patch by Gabe Appleton. .. @@ -3013,7 +3013,7 @@ thread was still running. .. section: Library Allow pure Python implementation of :mod:`pickle` to work even when the C -:mod:`_pickle` module is unavailable. +:mod:`!_pickle` module is unavailable. .. @@ -3064,8 +3064,8 @@ internal tasks weak set is changed by another thread during iteration. .. nonce: ADqCkq .. section: Library -:class:`_pyio.IOBase` destructor now does nothing if getting the ``closed`` -attribute fails to better mimic :class:`_io.IOBase` finalizer. +:class:`!_pyio.IOBase` destructor now does nothing if getting the ``closed`` +attribute fails to better mimic :class:`!_io.IOBase` finalizer. .. @@ -3117,9 +3117,9 @@ Ensure cookies with ``expires`` attribute are handled in .. section: Library Fix an unintended ValueError from :func:`subprocess.run` when checking for -conflicting `input` and `stdin` or `capture_output` and `stdout` or `stderr` -args when they were explicitly provided but with `None` values within a -passed in `**kwargs` dict rather than as passed directly by name. Patch +conflicting *input* and *stdin* or *capture_output* and *stdout* or *stderr* +args when they were explicitly provided but with ``None`` values within a +passed in ``**kwargs`` dict rather than as passed directly by name. Patch contributed by Rémi Lapeyre. .. @@ -3195,9 +3195,9 @@ Remove ``Enum._convert`` method, deprecated in 3.8. .. nonce: TTzHxj .. section: Library -`argparse._ActionsContainer.add_argument` now throws error, if someone +``argparse._ActionsContainer.add_argument`` now throws error, if someone accidentally pass FileType class object instead of instance of FileType as -`type` argument +``type`` argument. .. @@ -3458,7 +3458,7 @@ Patch contributed by Rémi Lapeyre. .. section: Library Fixes a bug in :mod:`!cgi` module when a multipart/form-data request has no -`Content-Length` header. +``Content-Length`` header. .. @@ -3546,7 +3546,7 @@ Patch by Stein Karlsen. .. nonce: XaJDei .. section: Library -lib2to3 now recognizes expressions after ``*`` and `**` like in ``f(*[] or +lib2to3 now recognizes expressions after ``*`` and ``**`` like in ``f(*[] or [])``. .. @@ -3566,8 +3566,8 @@ source argument. Patch by Emily Morehouse and Maxwell "5.13b" McKinnon. .. nonce: 0stF0u .. section: Library -Added __format__ to IPv4 and IPv6 classes. Always outputs a fully zero- -padded string. Supports b/x/n modifiers (bin/hex/native format). Native +Added ``__format__`` to IPv4 and IPv6 classes. Always outputs a fully +zero-padded string. Supports b/x/n modifiers (bin/hex/native format). Native format for IPv4 is bin, native format for IPv6 is hex. Also supports '#' and '_' modifiers. @@ -3789,7 +3789,7 @@ verify the maildir folder layout correctness. Patch by Sviatoslav Sydorenko. .. nonce: 7tiFR- .. section: Documentation -Fix `importlib` examples to insert any newly created modules via +Fix ``importlib`` examples to insert any newly created modules via importlib.util.module_from_spec() immediately into sys.modules instead of after calling loader.exec_module(). @@ -3888,7 +3888,7 @@ Make C-API docs clear about what the "main" interpreter is. .. nonce: Iqiqtm .. section: Documentation -The documentation for decimal string formatting using the `:g` specifier has +The documentation for decimal string formatting using the ``:g`` specifier has been updated to reflect the correct exponential notation cutoff point. Original patch contributed by Tuomas Suutari. @@ -4450,8 +4450,8 @@ Fix _hashlib build when Blake2 is disabled, but OpenSSL supports it. .. nonce: buCO84 .. section: Build -Misc/python-config.in now uses `getvar()` for all still existing -`sysconfig.get_config_var()` calls. Patch by Joannah Nanjekye. +Misc/python-config.in now uses ``getvar()`` for all still existing +``sysconfig.get_config_var()`` calls. Patch by Joannah Nanjekye. .. @@ -4993,7 +4993,7 @@ Make :const:`winreg.REG_MULTI_SZ` support zero-length strings. .. section: Windows Replace use of :c:func:`strcasecmp` for the system function -:c:func:`_stricmp`. Patch by Minmin Gong. +:c:func:`!_stricmp`. Patch by Minmin Gong. .. @@ -5616,7 +5616,7 @@ heap type .. nonce: 4DcUaI .. section: C API -Add :c:func:`_PyObject_FunctionStr` to get a user-friendly string +Add :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. Patch by Jeroen Demeyer. .. @@ -5696,8 +5696,8 @@ Add :c:func:`PyConfig_SetWideStringList` function. .. section: C API Add fast functions for calling methods: -:c:func:`_PyObject_VectorcallMethod`, :c:func:`_PyObject_CallMethodNoArgs` -and :c:func:`_PyObject_CallMethodOneArg`. +:c:func:`!_PyObject_VectorcallMethod`, :c:func:`!_PyObject_CallMethodNoArgs` +and :c:func:`!_PyObject_CallMethodOneArg`. .. @@ -5715,8 +5715,8 @@ The :c:macro:`METH_FASTCALL` calling convention has been documented. .. nonce: 4tClQT .. section: C API -The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create code -objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount* +The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create code +objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount* parameter for indicating the number of positonal-only arguments. .. diff --git a/Misc/NEWS.d/3.9.0a2.rst b/Misc/NEWS.d/3.9.0a2.rst index a03eb10f1d523a..39b1c308312aa4 100644 --- a/Misc/NEWS.d/3.9.0a2.rst +++ b/Misc/NEWS.d/3.9.0a2.rst @@ -403,7 +403,7 @@ locale encoding is not UTF-8. Prevent UnboundLocalError to pop up in parse_message_id. parse_message_id() was improperly using a token defined inside an exception -handler, which was raising `UnboundLocalError` on parsing an invalid value. +handler, which was raising ``UnboundLocalError`` on parsing an invalid value. Patch by Claudiu Popa. .. @@ -444,12 +444,12 @@ random.choices() now raises a ValueError when all the weights are zero. Raise pickle.UnpicklingError when loading an item from memo for invalid input. -The previous code was raising a `KeyError` for both the Python and C +The previous code was raising a ``KeyError`` for both the Python and C implementation. This was caused by the specified index of an invalid input which did not exist in the memo structure, where the pickle stores what -objects it has seen. The malformed input would have caused either a `BINGET` -or `LONG_BINGET` load from the memo, leading to a `KeyError` as the -determined index was bogus. Patch by Claudiu Popa +objects it has seen. The malformed input would have caused either a ``BINGET`` +or ``LONG_BINGET`` load from the memo, leading to a ``KeyError`` as the +determined index was bogus. Patch by Claudiu Popa. .. @@ -458,7 +458,7 @@ determined index was bogus. Patch by Claudiu Popa .. nonce: iKx23z .. section: Library -Calling func:`shutil.copytree` to copy a directory tree from one directory +Calling func:``shutil.copytree`` to copy a directory tree from one directory to another subdirectory resulted in an endless loop and a RecursionError. A fix was added to consume an iterator and create the list of the entries to be copied, avoiding the recursion for newly created directories. Patch by @@ -685,7 +685,7 @@ added. .. section: Documentation Update documentation to state that to activate virtual environments under -fish one should use `source`, not `.` as documented at +fish one should use ``source``, not ``.`` as documented at https://fishshell.com/docs/current/cmds/source.html. .. @@ -844,7 +844,7 @@ test.regrtest now can receive a list of test patterns to ignore (using the .. nonce: cNsA7S .. section: Build -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. Patch by Ido Michael. .. @@ -884,7 +884,7 @@ Add support for building and releasing Windows ARM64 packages. Fixed a crash on OSX dynamic builds that occurred when re-initializing the posix module after a Py_Finalize if the environment had changed since the -previous `import posix`. Patch by Benoît Hudson. +previous ``import posix``. Patch by Benoît Hudson. .. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 8a94848427382b..78194791ab5277 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -266,8 +266,8 @@ The :func:`os.unsetenv` function is now also available on Windows. .. nonce: D2tSXk .. section: Library -Fixed a regression with the `ignore` callback of :func:`shutil.copytree`. -The argument types are now str and List[str] again. +Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`. +The argument types are now ``str`` and ``List[str]`` again. .. @@ -454,7 +454,7 @@ resilients to inaccessible sys.path entries (importlib_metadata v1.4.0). .. nonce: _S5VjC .. section: Library -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. Patch by Donghee Na. @@ -498,7 +498,7 @@ prevent the creation of a non-blocking socket. Patch by Donghee Na. .. section: Library Updated the Gmane domain from news.gmane.org to news.gmane.io which is used -for examples of :class:`~!nntplib.NNTP` news reader server and nntplib tests. +for examples of :class:`!NNTP` news reader server and nntplib tests. .. @@ -507,8 +507,8 @@ for examples of :class:`~!nntplib.NNTP` news reader server and nntplib tests. .. nonce: ihRT1z .. section: Library -Proxy the `SimpleHTTPRequestHandler.guess_type` to `mimetypes.guess_type` so -the `mimetypes.init` is called lazily to avoid unnecessary costs when +Proxy the ``SimpleHTTPRequestHandler.guess_type`` to ``mimetypes.guess_type`` so +the ``mimetypes.init`` is called lazily to avoid unnecessary costs when :mod:`http.server` module is imported. .. @@ -547,10 +547,10 @@ all options. Giovanni Lombardo contributed part of the patch. .. nonce: nzwGyG .. section: Library -If an exception were to be thrown in `Logger.isEnabledFor` (say, by asyncio -timeouts or stopit) , the `logging` global lock may not be released +If an exception were to be thrown in ``Logger.isEnabledFor`` (say, by asyncio +timeouts or stopit) , the ``logging`` global lock may not be released appropriately, resulting in deadlock. This change wraps that block of code -with `try...finally` to ensure the lock is released. +with ``try...finally`` to ensure the lock is released. .. @@ -571,7 +571,7 @@ new task spawning before exception raising. .. section: Library Correctly parenthesize filter-based statements that contain lambda -expressions in mod:`!lib2to3`. Patch by Donghee Na. +expressions in :mod:`!lib2to3`. Patch by Donghee Na. .. @@ -732,9 +732,9 @@ different process. .. nonce: beZ0Sk .. section: Library -Removes trailing space in formatted currency with `international=True` and a -locale with symbol following value. E.g. `locale.currency(12.34, -international=True)` returned `'12,34 EUR '` instead of `'12,34 EUR'`. +Removes trailing space in formatted currency with ``international=True`` and a +locale with symbol following value. E.g. ``locale.currency(12.34, +international=True)`` returned ``'12,34 EUR '`` instead of ``'12,34 EUR'``. .. @@ -835,7 +835,7 @@ functions are now required to build Python. .. nonce: aBmj13 .. section: Build -Updated the documentation in `./configure --help` to show default values, +Updated the documentation in ``./configure --help`` to show default values, reference documentation where required and add additional explanation where needed. diff --git a/Misc/NEWS.d/3.9.0a4.rst b/Misc/NEWS.d/3.9.0a4.rst index e59435b5509acf..ca0eb2abf1d654 100644 --- a/Misc/NEWS.d/3.9.0a4.rst +++ b/Misc/NEWS.d/3.9.0a4.rst @@ -4,8 +4,8 @@ .. release date: 2020-02-25 .. section: Security -Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, -`shutil`, `signal` and `syslog`. +Add audit events to functions in ``fcntl``, ``msvcrt``, ``os``, ``resource``, +``shutil``, ``signal`` and ``syslog``. .. @@ -81,9 +81,9 @@ Fix regression caused by fix for bpo-39386, that prevented calling .. nonce: itNmC0 .. section: Core and Builtins -Change the ending column offset of `Attribute` nodes constructed in -`ast_for_dotted_name` to point at the end of the current node and not at the -end of the last `NAME` node. +Change the ending column offset of ``Attribute`` nodes constructed in +``ast_for_dotted_name`` to point at the end of the current node and not at the +end of the last ``NAME`` node. .. @@ -269,7 +269,7 @@ codec. .. nonce: qiubSp .. section: Library -Remove obsolete check for `__args__` in bdb.Bdb.format_stack_entry. +Remove obsolete check for ``__args__`` in ``bdb.Bdb.format_stack_entry``. .. @@ -599,7 +599,7 @@ default return values. Patch by Karthikeyan Singaravelan. .. nonce: udRSWE .. section: Library -`inspect.Signature.parameters` and `inspect.BoundArguments.arguments` are +``inspect.Signature.parameters`` and ``inspect.BoundArguments.arguments`` are now dicts instead of OrderedDicts. Patch contributed by Rémi Lapeyre. .. @@ -619,9 +619,9 @@ multiprocessing.Process. .. nonce: e0C5dF .. section: Library -* Add `lazycache` function to `__all__`. -* Use `dict.clear` to clear the cache. -* Refactoring `getline` function and `checkcache` function. +* Add ``lazycache`` function to ``__all__``. +* Use ``dict.clear`` to clear the cache. +* Refactoring ``getline`` function and ``checkcache`` function. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 6ff05788214723..7f7480539f2f1b 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -5,7 +5,7 @@ .. section: Security Disallow control characters in hostnames in http.client, addressing -CVE-2019-18348. Such potentially malicious header injection URLs now cause a +:cve:`2019-18348`. Such potentially malicious header injection URLs now cause a InvalidURL to be raised. .. @@ -691,7 +691,7 @@ Fix AttributeError when calling get_stack on a PyAsyncGenObject Task .. section: Library The :func:`compileall.compile_dir` function's *ddir* parameter and the -compileall command line flag `-d` no longer write the wrong pathname to the +compileall command line flag ``-d`` no longer write the wrong pathname to the generated pyc file for submodules beneath the root of the directory tree being compiled. This fixes a regression introduced with Python 3.5. @@ -1122,7 +1122,7 @@ a different condition than the GIL. .. nonce: Nbl7lF .. section: Tools/Demos -Added support to fix ``getproxies`` in the :mod:`lib2to3.fixes.fix_urllib` +Added support to fix ``getproxies`` in the :mod:`!lib2to3.fixes.fix_urllib` module. Patch by José Roberto Meza Cabrera. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 519c7f833ebcb8..4ba4cfe818c2d0 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -23,7 +23,7 @@ header injection attacks. .. nonce: B299Yq .. section: Security -CVE-2020-8492: The :class:`~urllib.request.AbstractBasicAuthHandler` class +:cve:`2020-8492`: The :class:`~urllib.request.AbstractBasicAuthHandler` class of the :mod:`urllib.request` module uses an inefficient regular expression which can be exploited by an attacker to cause a denial of service. Fix the regex to prevent the catastrophic backtracking. Vulnerability reported by @@ -59,8 +59,8 @@ anything that depends on it. .. section: Core and Builtins Fix the tokenizer to display the correct error message, when there is a -SyntaxError on the last input character and no newline follows. It used to -be `unexpected EOF while parsing`, while it should be `invalid syntax`. +``SyntaxError`` on the last input character and no newline follows. It used to +be ``unexpected EOF while parsing``, while it should be ``invalid syntax``. .. @@ -79,7 +79,7 @@ evaluation for annotations activated. Patch by Batuhan Taskaya. .. nonce: vXPze5 .. section: Core and Builtins -Report a specialized error message, `invalid string prefix`, when the +Report a specialized error message, ``invalid string prefix``, when the tokenizer encounters a string with an invalid prefix. .. @@ -111,7 +111,7 @@ str.decode(). .. nonce: m15TTX .. section: Core and Builtins -Fix possible refleaks in :mod:`_json`, memo of PyScannerObject should be +Fix possible refleaks in :mod:`!_json`, memo of PyScannerObject should be traversed. .. @@ -434,7 +434,7 @@ for the correspondent concrete type (``list`` in this case). .. nonce: ux8FUr .. section: Library -func:`inspect.getdoc` no longer returns docstring inherited from the type of +:func:`inspect.getdoc` no longer returns docstring inherited from the type of the object or from parent class if it is a class if it is not defined in the object itself. In :mod:`pydoc` the documentation string is now shown not only for class, function, method etc, but for any object that has its own @@ -504,7 +504,7 @@ extension. This allows the use of functions such as ``json_object``. .. nonce: 4EcyIN .. section: Library -Wait in `KqueueSelector.select` when no fds are registered +Wait in ``KqueueSelector.select`` when no fds are registered .. @@ -564,7 +564,7 @@ Implement traverse and clear slots in _abc._abc_data type. .. nonce: 3rO_q7 .. section: Library -Remove deprecated :meth:`symtable.SymbolTable.has_exec`. +Remove deprecated :meth:`!symtable.SymbolTable.has_exec`. .. @@ -635,7 +635,7 @@ script is killed by signal 11, it now logs: "CGI script exit code -11." .. section: Library Improve the error message when triying to import a module using :mod:`runpy` -and incorrently use the ".py" extension at the end of the module name. Patch +and incorrectly using the ".py" extension at the end of the module name. Patch by Pablo Galindo. .. @@ -666,8 +666,8 @@ for _main_thread, instead of a _DummyThread instance. .. nonce: VTq_8s .. section: Library -Add a private ``_at_fork_reinit()`` method to :class:`_thread.Lock`, -:class:`_thread.RLock`, :class:`threading.RLock` and +Add a private ``_at_fork_reinit()`` method to :class:`!_thread.Lock`, +:class:`!_thread.RLock`, :class:`threading.RLock` and :class:`threading.Condition` classes: reinitialize the lock at fork in the child process, reset the lock to the unlocked state. Rename also the private ``_reset_internal_locks()`` method of :class:`threading.Event` to @@ -701,7 +701,7 @@ per header: use the realm of the first Basic challenge. .. section: Library Removed daemon threads from :mod:`concurrent.futures` by adding an internal -`threading._register_atexit()`, which calls registered functions prior to +``threading._register_atexit()``, which calls registered functions prior to joining all non-daemon threads. This allows for compatibility with subinterpreters, which don't support daemon threads. @@ -781,9 +781,9 @@ Added :pep:`584` operators to :class:`weakref.WeakKeyDictionary`. .. nonce: 56Yokh .. section: Library -Fix linear runtime behaviour of the `__getitem__` and `__setitem__` methods +Fix linear runtime behaviour of the ``__getitem__`` and ``__setitem__`` methods in :class:`multiprocessing.shared_memory.ShareableList`. This avoids -quadratic performance when iterating a `ShareableList`. Patch by Thomas +quadratic performance when iterating a ``ShareableList``. Patch by Thomas Krennwallner. .. @@ -793,9 +793,9 @@ Krennwallner. .. nonce: AxXZNz .. section: Library -Remove undocumented support for *closing* a `pathlib.Path` object via its +Remove undocumented support for *closing* a ``pathlib.Path`` object via its context manager. The context manager magic methods remain, but they are now -a no-op, making `Path` objects immutable. +a no-op, making ``Path`` objects immutable. .. @@ -1118,7 +1118,7 @@ into an exit code. .. nonce: _FOf7E .. section: C API -Move the :c:type:`PyGC_Head` structure to the internal C API. +Move the :c:type:`!PyGC_Head` structure to the internal C API. .. @@ -1149,8 +1149,8 @@ the garbage collector respectively. Patch by Pablo Galindo. .. nonce: Seuh3D .. section: C API -The :c:func:`PyObject_NEW` macro becomes an alias to the -:c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro +The :c:func:`!PyObject_NEW` macro becomes an alias to the +:c:func:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:func:`PyObject_NewVar` macro, to hide implementation details. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. @@ -1174,7 +1174,7 @@ used. .. nonce: 6nFYbY .. section: C API -Convert the :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro to a function to +Convert the :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro to a function to hide implementation details: the macro accessed directly to the :c:member:`PyTypeObject.tp_weaklistoffset` member. diff --git a/Misc/NEWS.d/3.9.0b1.rst b/Misc/NEWS.d/3.9.0b1.rst index ee87315ad334e9..40fb8474bf9364 100644 --- a/Misc/NEWS.d/3.9.0b1.rst +++ b/Misc/NEWS.d/3.9.0b1.rst @@ -112,8 +112,8 @@ Port :mod:`syslog` to multiphase initialization (:pep:`489`). Reporting a specialised error message for invalid string prefixes, which was introduced in :issue:`40246`, is being reverted due to backwards compatibility concerns for strings that immediately follow a reserved -keyword without whitespace between them. Constructs like `bg="#d00" if clear -else"#fca"` were failing to parse, which is not an acceptable breakage on +keyword without whitespace between them. Constructs like ``bg="#d00" if clear +else"#fca"`` were failing to parse, which is not an acceptable breakage on such short notice. .. @@ -684,7 +684,7 @@ The ``isocalendar()`` methods of :class:`datetime.date` and .. nonce: t6kW_1 .. section: Documentation -Add version of removal for explicit passing of coros to `asyncio.wait()`'s +Add version of removal for explicit passing of coros to ``asyncio.wait()``'s documentation .. diff --git a/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst new file mode 100644 index 00000000000000..4e886be034fb82 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst @@ -0,0 +1,3 @@ +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which compilation errors could be caused +by an incompletely-generated header file. diff --git a/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst b/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst new file mode 100644 index 00000000000000..5212af7b32b940 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst @@ -0,0 +1,2 @@ +Fix an ``AssertionError`` when building with ``--enable-experimental-jit`` +and the compiler emits a ``SHT_NOTE`` section. diff --git a/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst b/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst new file mode 100644 index 00000000000000..44fe2a1a1f6725 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst @@ -0,0 +1,2 @@ +Update :data:`sys.version` to identify whether the build is default build or +free-threading build. Patch By Donghee Na. diff --git a/Misc/NEWS.d/next/Build/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst b/Misc/NEWS.d/next/Build/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst new file mode 100644 index 00000000000000..acaca9e0ebbdfb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst @@ -0,0 +1 @@ +Fall back to the bundled libmpdec if a system version cannot be found. diff --git a/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst b/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst new file mode 100644 index 00000000000000..7ac300ecf40326 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst @@ -0,0 +1,5 @@ +On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI +flags, which may include debug ("d") and free-threaded ("t"). For example: +* ``python-3.14.pc`` (default, non-debug build) +* ``python-3.14d.pc`` (default, debug build) +* ``python-3.14t.pc`` (free-threaded build) diff --git a/Misc/NEWS.d/next/Build/2024-06-02-13-23-26.gh-issue-113565.8xBlId.rst b/Misc/NEWS.d/next/Build/2024-06-02-13-23-26.gh-issue-113565.8xBlId.rst new file mode 100644 index 00000000000000..e26509cd434110 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-02-13-23-26.gh-issue-113565.8xBlId.rst @@ -0,0 +1,2 @@ +Improve :mod:`curses` and :mod:`curses.panel` dependency checks in +:program:`configure`. diff --git a/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst new file mode 100644 index 00000000000000..d0bb297b51dc6e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst @@ -0,0 +1 @@ +Make the ``python-config`` shell script compatible with non-bash shells. diff --git a/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst new file mode 100644 index 00000000000000..25cbdf6ba50ab8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst @@ -0,0 +1,2 @@ +On Windows, fix build error when ``--disable-gil`` and ``--experimental-jit`` +options are combined. diff --git a/Misc/NEWS.d/next/Build/2024-06-18-15-32-36.gh-issue-120688.tjIPLD.rst b/Misc/NEWS.d/next/Build/2024-06-18-15-32-36.gh-issue-120688.tjIPLD.rst new file mode 100644 index 00000000000000..90f1f9138b6b58 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-18-15-32-36.gh-issue-120688.tjIPLD.rst @@ -0,0 +1,3 @@ +On WASI in debug mode, Python is now built with compiler flag ``-O3`` +instead of ``-Og``, to support more recursive calls. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/Build/2024-06-19-21-05-15.gh-issue-120602.UyDARz.rst b/Misc/NEWS.d/next/Build/2024-06-19-21-05-15.gh-issue-120602.UyDARz.rst new file mode 100644 index 00000000000000..f0d90ec3bb5089 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-19-21-05-15.gh-issue-120602.UyDARz.rst @@ -0,0 +1,2 @@ +Correctly handle LLVM installs with ``LLVM_VERSION_SUFFIX`` when building +with ``--enable-experimental-jit``. diff --git a/Misc/NEWS.d/next/Build/2024-06-21-09-24-03.gh-issue-120671.Z8sBQB.rst b/Misc/NEWS.d/next/Build/2024-06-21-09-24-03.gh-issue-120671.Z8sBQB.rst new file mode 100644 index 00000000000000..bbe4a3038bc0ff --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-21-09-24-03.gh-issue-120671.Z8sBQB.rst @@ -0,0 +1 @@ +Fix failing configure tests due to a missing space when appending to CFLAGS. diff --git a/Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst b/Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst new file mode 100644 index 00000000000000..4a397a445ccc3b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst @@ -0,0 +1,2 @@ +Added a :option:`--with-app-store-compliance` option to patch out known issues +with macOS/iOS App Store review processes. diff --git a/Misc/NEWS.d/next/Build/2024-06-27-18-03-20.gh-issue-121082.w3AfRx.rst b/Misc/NEWS.d/next/Build/2024-06-27-18-03-20.gh-issue-121082.w3AfRx.rst new file mode 100644 index 00000000000000..7657672ba880c8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-27-18-03-20.gh-issue-121082.w3AfRx.rst @@ -0,0 +1 @@ +Fix build failure when the developer use ``--enable-pystats`` arguments in configuration command after #118450. diff --git a/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst new file mode 100644 index 00000000000000..4bc8c6de0b7733 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst @@ -0,0 +1,3 @@ +On POSIX systems, excluding macOS framework installs, the lib directory +for the free-threaded build now includes a "t" suffix to avoid conflicts +with a co-located default build installation. diff --git a/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst b/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst new file mode 100644 index 00000000000000..a2238475546eaa --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst @@ -0,0 +1 @@ +Fix a Makefile bug that prevented mimalloc header files from being installed. diff --git a/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst b/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst new file mode 100644 index 00000000000000..e30d4dcdbfe779 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst @@ -0,0 +1 @@ +Fix deprecation warning for ATOMIC_VAR_INIT in mimalloc. diff --git a/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst b/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst new file mode 100644 index 00000000000000..36e0f86a0ae455 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst @@ -0,0 +1 @@ +Fix mimalloc compile error on GNU/Hurd diff --git a/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst new file mode 100644 index 00000000000000..0bb0f5bcd501ef --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst @@ -0,0 +1,2 @@ +Fix creating bitfields in :mod:`ctypes` structures and unions. Fields +no longer overlap. diff --git a/Misc/NEWS.d/next/C API/2023-10-13-14-18-06.gh-issue-110815.tEFLVl.rst b/Misc/NEWS.d/next/C API/2023-10-13-14-18-06.gh-issue-110815.tEFLVl.rst deleted file mode 100644 index 216d2d211644a8..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-10-13-14-18-06.gh-issue-110815.tEFLVl.rst +++ /dev/null @@ -1 +0,0 @@ -Support non-ASCII keyword names in :c:func:`PyArg_ParseTupleAndKeywords`. diff --git a/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst b/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst new file mode 100644 index 00000000000000..9bcadfd9247f78 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst @@ -0,0 +1 @@ +Add :c:func:`PyLong_GetSign` function. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C API/2024-04-10-16-48-04.gh-issue-117511.RZtBRK.rst b/Misc/NEWS.d/next/C API/2024-04-10-16-48-04.gh-issue-117511.RZtBRK.rst new file mode 100644 index 00000000000000..586685a3407a3d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-10-16-48-04.gh-issue-117511.RZtBRK.rst @@ -0,0 +1 @@ +Make the :c:type:`PyMutex` public in the non-limited C API. diff --git a/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst b/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst new file mode 100644 index 00000000000000..a2acc16b2c1d01 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst @@ -0,0 +1,2 @@ +Restore ``_PyWeakref_ClearRef`` that was previously removed in Python 3.13 +alpha 1. diff --git a/Misc/NEWS.d/next/C API/2024-05-08-21-57-50.gh-issue-118789.Ni4UQx.rst b/Misc/NEWS.d/next/C API/2024-05-08-21-57-50.gh-issue-118789.Ni4UQx.rst new file mode 100644 index 00000000000000..32a9ec6d0710f6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-21-57-50.gh-issue-118789.Ni4UQx.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_ClearWeakRefsNoCallbacks`, which clears +weakrefs without calling their callbacks. diff --git a/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst new file mode 100644 index 00000000000000..561417b80d444d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst @@ -0,0 +1,3 @@ +Make mimalloc includes relative to the current file to avoid embedders or +extensions needing to include ``Internal/mimalloc`` if they are already +including internal CPython headers. diff --git a/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst b/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst new file mode 100644 index 00000000000000..2ed8089dfe8444 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst @@ -0,0 +1,3 @@ +Several C declarations with names that didn't start with the ``Py`` or ``_Py`` +prefixes, which were added by mistake in 3.13 alpha and beta releases, were +moved to internal headers. diff --git a/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst b/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst new file mode 100644 index 00000000000000..f47662f2e0a778 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst @@ -0,0 +1,2 @@ +Add :c:macro:`PyHASH_MULTIPLIER` constant: prime multiplier used in string +and various other hashes. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst b/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst new file mode 100644 index 00000000000000..3b2cdc8cf2dc5c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst @@ -0,0 +1,4 @@ +Added ``Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST`` and +``Py_END_CRITICAL_SECTION_SEQUENCE_FAST`` macros to make it possible to use +PySequence_Fast APIs safely when free-threaded, and update str.join to work +without the GIL using them. diff --git a/Misc/NEWS.d/next/C API/2024-05-21-19-41-41.gh-issue-119344.QKvzQb.rst b/Misc/NEWS.d/next/C API/2024-05-21-19-41-41.gh-issue-119344.QKvzQb.rst new file mode 100644 index 00000000000000..5a2e4d980b59be --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-21-19-41-41.gh-issue-119344.QKvzQb.rst @@ -0,0 +1 @@ +The critical section API is now public as part of the non-limited C API. diff --git a/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst b/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst new file mode 100644 index 00000000000000..e530bb45d35e76 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst @@ -0,0 +1 @@ +Restore the removed ``_PyLong_NumBits()`` function. It is used by the pywin32 project. Patch by Ethan Smith diff --git a/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst b/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst new file mode 100644 index 00000000000000..196a4722a98c70 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst @@ -0,0 +1,2 @@ +Macros ``Py_IS_NAN``, ``Py_IS_INFINITY`` and ``Py_IS_FINITE`` +are :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst b/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst new file mode 100644 index 00000000000000..038dec2dbf90d1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst @@ -0,0 +1,5 @@ +Fix crash when a thread state that was created by :c:func:`PyGILState_Ensure` +calls a destructor that during :c:func:`PyThreadState_Clear` that +calls back into :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. +This might occur when in the free-threaded build or when using thread-local +variables whose destructors call :c:func:`PyGILState_Ensure`. diff --git a/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst b/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst new file mode 100644 index 00000000000000..c342a3814ed5db --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst @@ -0,0 +1,2 @@ +Creating :c:data:`immutable types ` with mutable +bases was deprecated since 3.12 and now raises a :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst b/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst new file mode 100644 index 00000000000000..11f075b79e6f67 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst @@ -0,0 +1,2 @@ +Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +instead. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-06-07-22-12-30.gh-issue-119182.yt8Ar7.rst b/Misc/NEWS.d/next/C API/2024-06-07-22-12-30.gh-issue-119182.yt8Ar7.rst new file mode 100644 index 00000000000000..243f290fbd47e2 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-07-22-12-30.gh-issue-119182.yt8Ar7.rst @@ -0,0 +1,16 @@ +Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` object: + +* :c:func:`PyUnicodeWriter_Create`. +* :c:func:`PyUnicodeWriter_Discard`. +* :c:func:`PyUnicodeWriter_Finish`. +* :c:func:`PyUnicodeWriter_WriteChar`. +* :c:func:`PyUnicodeWriter_WriteUTF8`. +* :c:func:`PyUnicodeWriter_WriteUCS4`. +* :c:func:`PyUnicodeWriter_WriteWideChar`. +* :c:func:`PyUnicodeWriter_WriteStr`. +* :c:func:`PyUnicodeWriter_WriteRepr`. +* :c:func:`PyUnicodeWriter_WriteSubstring`. +* :c:func:`PyUnicodeWriter_Format`. +* :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-06-11-21-38-32.gh-issue-70278.WDE4zM.rst b/Misc/NEWS.d/next/C API/2024-06-11-21-38-32.gh-issue-70278.WDE4zM.rst new file mode 100644 index 00000000000000..1eca36a86bc97e --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-11-21-38-32.gh-issue-70278.WDE4zM.rst @@ -0,0 +1,4 @@ +:c:func:`PyUnicode_FromFormat` no longer produces the ending ``\ufffd`` +character for truncated C string when use precision with ``%s`` and ``%V``. +It now truncates the string before the start of truncated multibyte +sequences. diff --git a/Misc/NEWS.d/next/C API/2024-06-16-22-58-47.gh-issue-120600.TJdf0w.rst b/Misc/NEWS.d/next/C API/2024-06-16-22-58-47.gh-issue-120600.TJdf0w.rst new file mode 100644 index 00000000000000..12ffd9b348d2b7 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-16-22-58-47.gh-issue-120600.TJdf0w.rst @@ -0,0 +1,2 @@ +In the limited C API 3.14 and newer, :c:func:`Py_TYPE` is now implemented as an +opaque function call to hide implementation details. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-06-19-21-27-42.gh-issue-120642.UlKClN.rst b/Misc/NEWS.d/next/C API/2024-06-19-21-27-42.gh-issue-120642.UlKClN.rst new file mode 100644 index 00000000000000..a61224ec8ef119 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-19-21-27-42.gh-issue-120642.UlKClN.rst @@ -0,0 +1,10 @@ +Remove the following unstable functions: + +* ``PyUnstable_Replace_Executor()`` +* ``PyUnstable_SetOptimizer()`` +* ``PyUnstable_GetOptimizer()`` +* ``PyUnstable_GetExecutor()`` +* ``PyUnstable_Optimizer_NewCounter()`` +* ``PyUnstable_Optimizer_NewUOpOptimizer()`` + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-06-21-16-41-21.gh-issue-120858.Z5_-Mn.rst b/Misc/NEWS.d/next/C API/2024-06-21-16-41-21.gh-issue-120858.Z5_-Mn.rst new file mode 100644 index 00000000000000..b5df2a567b9da8 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-21-16-41-21.gh-issue-120858.Z5_-Mn.rst @@ -0,0 +1,3 @@ +:c:func:`PyDict_Next` no longer locks the dictionary in the free-threaded +build. The locking needs to be done by the caller around the entire iteration +loop. diff --git a/Misc/NEWS.d/next/C API/2024-06-26-11-29-01.gh-issue-120642.H7P9qK.rst b/Misc/NEWS.d/next/C API/2024-06-26-11-29-01.gh-issue-120642.H7P9qK.rst new file mode 100644 index 00000000000000..24fb6ca569f4f3 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-26-11-29-01.gh-issue-120642.H7P9qK.rst @@ -0,0 +1,3 @@ +Remove the private ``_Py_CODEUNIT`` type from the public C API. The internal +``pycore_code.h`` header should now be used to get this internal type. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-07-02-11-03-40.gh-issue-112136.f3fiY8.rst b/Misc/NEWS.d/next/C API/2024-07-02-11-03-40.gh-issue-112136.f3fiY8.rst new file mode 100644 index 00000000000000..a240b4e852c4d1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-07-02-11-03-40.gh-issue-112136.f3fiY8.rst @@ -0,0 +1,3 @@ +Restore the private ``_PyArg_Parser`` structure and the private +``_PyArg_ParseTupleAndKeywordsFast()`` function, previously removed in Python +3.13 alpha 1. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst b/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst new file mode 100644 index 00000000000000..b82e78446e4e87 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst @@ -0,0 +1,3 @@ +Export the :c:func:`PySignal_SetWakeupFd` function. Previously, the function +was documented but it couldn't be used in 3rd party code. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst new file mode 100644 index 00000000000000..93a8562efe6d6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst @@ -0,0 +1 @@ +Fix handling of ``doc`` argument to subclasses of ``property``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst new file mode 100644 index 00000000000000..83b1126a8a455f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst @@ -0,0 +1,2 @@ +Improve the error message from ``a in b`` when ``b`` is not a container +to mention the term "container". diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst deleted file mode 100644 index be90bb3564fd54..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-13-09-21-29.gh-issue-110805.vhU7A7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow the repl to show source code and complete tracebacks. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-21-51-26.gh-issue-114091.VOtSJl.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-21-51-26.gh-issue-114091.VOtSJl.rst new file mode 100644 index 00000000000000..55b7d9104baed9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-21-51-26.gh-issue-114091.VOtSJl.rst @@ -0,0 +1 @@ +Changed the error message for awaiting something that can't be awaited from "object can't be used in an await expression" to "'' object can't be awaited". diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst new file mode 100644 index 00000000000000..ae1e5acc5c333b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst @@ -0,0 +1,2 @@ +Avoid assertion failure for debug builds when calling +``object.__sizeof__(1)`` diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst new file mode 100644 index 00000000000000..222c516d2ef1b9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst @@ -0,0 +1,2 @@ +Improve error messages when a string, bytes or bytearray object of length 1 +is expected. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst new file mode 100644 index 00000000000000..93b176d5767335 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst @@ -0,0 +1 @@ +Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` and :func:`difflib.context_diff`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst new file mode 100644 index 00000000000000..07d5dd53aac30a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst @@ -0,0 +1,5 @@ +Convert the Python evaluation stack to use internal stack references. The +purpose is to support tagged pointers. In :pep:`703`, this will +allow for its form of deferred reference counting. For both +the default and free-threaded builds, this sets up the infrastructure +for unboxed integers in the future. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst new file mode 100644 index 00000000000000..c506a8cefd00f1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst @@ -0,0 +1,2 @@ +Fix race condition in free-threaded build where :meth:`!list.extend` could +expose uninitialised memory to concurrent readers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst new file mode 100644 index 00000000000000..4828f8fbf50cea --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst @@ -0,0 +1,2 @@ +Using :data:`NotImplemented` in a boolean context now raises +:exc:`TypeError`. Contributed by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst new file mode 100644 index 00000000000000..727427d451d1e0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst @@ -0,0 +1 @@ +If the C version of the ``decimal`` module is available, ``int(str)`` now uses it to supply an asymptotically much faster conversion. However, this only applies if the string contains over about 2 million digits. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst new file mode 100644 index 00000000000000..db4c5813ca610c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst @@ -0,0 +1 @@ +Fix data races on the field that stores a pointer to the interpreter's main thread that occur in free-threaded builds. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst new file mode 100644 index 00000000000000..39ccf472067cfd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst @@ -0,0 +1 @@ +Add ``copy()`` method for ``FrameLocalsProxy`` which returns a snapshot ``dict`` for local variables. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst new file mode 100644 index 00000000000000..6e80b773889413 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst @@ -0,0 +1 @@ +Fix build failures when configuring with both ``--disable-gil`` and ``--enable-experimental-jit``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst new file mode 100644 index 00000000000000..d252888906c348 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst @@ -0,0 +1,4 @@ +Improve :exc:`ZeroDivisionError` error message. +Now, all error messages are harmonized: all ``/``, ``//``, and ``%`` +operations just use "division by zero" message. +And ``0 ** -1`` operation uses "zero to a negative power". diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst new file mode 100644 index 00000000000000..1d7aad8d1e5be6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst @@ -0,0 +1,2 @@ +Fix displaying the source line for warnings created by the C API if the +:mod:`warnings` module had not yet been imported. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst new file mode 100644 index 00000000000000..5a88ce097274fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst @@ -0,0 +1,2 @@ +Add an ``__annotate__`` attribute to functions, classes, and modules as part +of :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst new file mode 100644 index 00000000000000..11d177886df5b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst @@ -0,0 +1 @@ +Avoid creating unnecessary :exc:`StopIteration` instances for monitoring. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst new file mode 100644 index 00000000000000..0083c18da13278 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst @@ -0,0 +1,2 @@ +Fixes ``type.__type_params__`` to return an empty tuple instead of a +descriptor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst new file mode 100644 index 00000000000000..e9073b4ba08798 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst @@ -0,0 +1,3 @@ +Non-builtin modules built with argument clinic were crashing if used in a +subinterpreter before the main interpreter. The objects that were causing +the problem by leaking between interpreters carelessly have been fixed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst new file mode 100644 index 00000000000000..54c6639e0fa524 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst @@ -0,0 +1,2 @@ +Replace :opcode:`!LOAD_ASSERTION_ERROR` opcode with :opcode:`LOAD_COMMON_CONSTANT` +and add support for :exc:`NotImplementedError`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst new file mode 100644 index 00000000000000..265ffb32e6a1f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst @@ -0,0 +1 @@ +Evaluation of annotations is now deferred. See :pep:`649` for details. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-12-49-03.gh-issue-119372.PXig1R.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-12-49-03.gh-issue-119372.PXig1R.rst new file mode 100644 index 00000000000000..aa628299abbd95 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-12-49-03.gh-issue-119372.PXig1R.rst @@ -0,0 +1,2 @@ +Correct invalid corner cases in complex division (resulted in ``(nan+nanj)`` +output), e.g. ``1/complex('(inf+infj)')``. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst new file mode 100644 index 00000000000000..24cd90a8e5e5db --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst @@ -0,0 +1,2 @@ +Fix bug where names appearing after a generic class are mangled as if they +are in the generic class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst new file mode 100644 index 00000000000000..9e0db37340c49a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst @@ -0,0 +1,2 @@ +Fix bug where names are unexpectedly mangled in the bases of generic +classes. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst new file mode 100644 index 00000000000000..68f1ec1efa5751 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst @@ -0,0 +1,3 @@ +Eliminate type version guards in the tier two interpreter. + +Note that setting the ``tp_version_tag`` manually (which has never been supported) may result in crashes. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst new file mode 100644 index 00000000000000..83c29a16e572d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst @@ -0,0 +1,2 @@ +Fix deadlock involving ``_PyType_Lookup()`` cache in the free-threaded build +when the GIL is dynamically enabled at runtime. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst new file mode 100644 index 00000000000000..7abdd5cd85ccd6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst @@ -0,0 +1,2 @@ +Fix deadlock during thread deletion in free-threaded build, which could +occur when the GIL was enabled at runtime. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst new file mode 100644 index 00000000000000..3a28a94df0f7cf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst @@ -0,0 +1,3 @@ +An invalid assert in beta 1 has been removed. The assert would fail if +``PyState_FindModule()`` was used in an extension module's init function +before the module def had been initialized. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst new file mode 100644 index 00000000000000..4b1ca6ca5b0765 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst @@ -0,0 +1 @@ +Fix the signature of :meth:`str.format_map`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst new file mode 100644 index 00000000000000..0318790d46f0a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst @@ -0,0 +1 @@ +Add a ``clear`` command to the REPL. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst new file mode 100644 index 00000000000000..db762174a8c1e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst @@ -0,0 +1,3 @@ +:func:`complex` accepts now a string only as a positional argument. Passing +a complex number as the "real" or "imag" argument is deprecated; it should +only be passed as a single positional argument. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst new file mode 100644 index 00000000000000..56be31326216eb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst @@ -0,0 +1 @@ +Generate stack effect metadata for pseudo instructions from bytecodes.c. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst new file mode 100644 index 00000000000000..111e096d262ea0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst @@ -0,0 +1,2 @@ +Remove the previously-deprecated delegation of :func:`int` to +:meth:`~object.__trunc__`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst new file mode 100644 index 00000000000000..cc25eee6dd6ae4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst @@ -0,0 +1,2 @@ +Fix execution of :ref:`annotation scopes ` within classes +when ``globals`` is set to a non-dict. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst new file mode 100644 index 00000000000000..1e5ad7d08eed7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst @@ -0,0 +1,3 @@ +:func:`classmethod` and :func:`staticmethod` now wrap the +:attr:`__annotations__` and :attr:`!__annotate__` attributes of their +underlying callable lazily. See :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst new file mode 100644 index 00000000000000..2fcb170f6226e5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst @@ -0,0 +1 @@ +Honor :c:func:`PyOS_InputHook` in the new REPL. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst new file mode 100644 index 00000000000000..89de6b0299a35a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst @@ -0,0 +1 @@ +String search is now slightly faster for certain cases. It now utilizes last character gap (good suffix rule) for two-way periodic needles. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst new file mode 100644 index 00000000000000..78dc48da934cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst @@ -0,0 +1,3 @@ +Reverted improvements to error messages for ``elif``/``else`` statements not +matching any valid statements, which made in hard to locate the syntax +errors inside those ``elif``/``else`` blocks. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-48-44.gh-issue-119933.Kc0HG5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-48-44.gh-issue-119933.Kc0HG5.rst new file mode 100644 index 00000000000000..513a0200dcc48a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-48-44.gh-issue-119933.Kc0HG5.rst @@ -0,0 +1,4 @@ +Improve :exc:`SyntaxError` messages for invalid expressions in a type +parameters bound, a type parameter constraint tuple or a default type +parameter. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst new file mode 100644 index 00000000000000..8c5602fcdb4ad2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst @@ -0,0 +1,2 @@ +Direct call to the :meth:`!int.__round__` now accepts ``None`` +as a valid argument. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-10-32-44.gh-issue-120097.9S2klk.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-10-32-44.gh-issue-120097.9S2klk.rst new file mode 100644 index 00000000000000..39d829bb0ed310 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-10-32-44.gh-issue-120097.9S2klk.rst @@ -0,0 +1,2 @@ +``FrameLocalsProxy`` now subclasses ``collections.abc.Mapping`` and can be +matched as a mapping in ``match`` statements diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst new file mode 100644 index 00000000000000..c06d5a276c03eb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst @@ -0,0 +1 @@ +Fix source locations of instructions generated for with statements. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst new file mode 100644 index 00000000000000..d00b9aaa8192e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst @@ -0,0 +1 @@ +Fix crash in compiler on empty block at end of exception handler. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-07-22-54-15.gh-issue-119726.D9EE-o.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-22-54-15.gh-issue-119726.D9EE-o.rst new file mode 100644 index 00000000000000..595d8dda25fe1b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-22-54-15.gh-issue-119726.D9EE-o.rst @@ -0,0 +1 @@ +JIT: Re-use trampolines on AArch64 when creating stencils. Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst new file mode 100644 index 00000000000000..09c1f553c48702 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst @@ -0,0 +1 @@ +Fix a compiler crash in the case where two comprehensions in class scope both reference ``__class__``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst new file mode 100644 index 00000000000000..2872006ee34b8b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst @@ -0,0 +1,2 @@ +Fix use-after free in ``list_richcompare_impl`` which can be invoked via +some specifically tailored evil input. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst new file mode 100644 index 00000000000000..8dc8aec44d80c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst @@ -0,0 +1 @@ +Fix a crash when multiple threads read and write to the same ``__class__`` of an object concurrently. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst new file mode 100644 index 00000000000000..294f8d892b459b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst @@ -0,0 +1,2 @@ +Fix source locations of instructions generated for the iterator of a for +statement. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst new file mode 100644 index 00000000000000..757a21625cfb83 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst @@ -0,0 +1,2 @@ +Respect :envvar:`PYTHON_BASIC_REPL` when running in interactive inspect mode +(``python -i``). Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst new file mode 100644 index 00000000000000..3781576bc5a257 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst @@ -0,0 +1,2 @@ +Deliver real signals on Ctrl-C and Ctrl-Z in the new REPL. Patch by Pablo +Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst new file mode 100644 index 00000000000000..24f046d9d89d51 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst @@ -0,0 +1,2 @@ +Improve the throughput by up to two times for the :meth:`str.count`, :meth:`bytes.count` and :meth:`bytearray.count` +methods for counting single characters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-23-15.gh-issue-120380.edtqjq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-23-15.gh-issue-120380.edtqjq.rst new file mode 100644 index 00000000000000..c682a0b7666416 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-23-15.gh-issue-120380.edtqjq.rst @@ -0,0 +1,3 @@ +Fix Python implementation of :class:`pickle.Pickler` for :class:`bytes` and +:class:`bytearray` objects when using protocol version 5. Patch by Bénédikt +Tran. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-50-29.gh-issue-120367.LmXx2y.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-50-29.gh-issue-120367.LmXx2y.rst new file mode 100644 index 00000000000000..2d7212a66f7a84 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-18-50-29.gh-issue-120367.LmXx2y.rst @@ -0,0 +1,2 @@ +Fix crash in compiler on code with redundant NOPs and JUMPs which show up +after exception handlers are moved to the end of the code. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst new file mode 100644 index 00000000000000..4a4db821ce29b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst @@ -0,0 +1,3 @@ +Fix an array out of bounds crash in ``list_ass_subscript``, which could be +invoked via some specificly tailored input: including concurrent modification +of a list object, where one thread assigns a slice and another clears it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst new file mode 100644 index 00000000000000..f41c233908362f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst @@ -0,0 +1 @@ +Add an override of ``allow_reuse_port`` on classes subclassing ``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-14-22-02-25.gh-issue-113993.MiA0vX.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-22-02-25.gh-issue-113993.MiA0vX.rst new file mode 100644 index 00000000000000..9931787cb36d4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-22-02-25.gh-issue-113993.MiA0vX.rst @@ -0,0 +1,5 @@ +Strings interned with :func:`sys.intern` are again garbage-collected when no +longer used, as per the documentation. Strings interned with the C function +:c:func:`PyUnicode_InternInPlace` are still immortal. Internals of the +string interning mechanism have been changed. This may affect performance +and identities of :class:`str` objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-17-12-20-20.gh-issue-120507.94lz2J.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-17-12-20-20.gh-issue-120507.94lz2J.rst new file mode 100644 index 00000000000000..c12e104fa68e70 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-17-12-20-20.gh-issue-120507.94lz2J.rst @@ -0,0 +1,3 @@ +Remove the ``BEFORE_WITH`` and ``BEFORE_ASYNC_WITH`` +instructions. Add the new :opcode:`LOAD_SPECIAL` instruction. Generate code +for ``with`` and ``async with`` statements using the new instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-18-21-34-30.gh-issue-120367.zDwffP.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-21-34-30.gh-issue-120367.zDwffP.rst new file mode 100644 index 00000000000000..087640e5400b98 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-21-34-30.gh-issue-120367.zDwffP.rst @@ -0,0 +1 @@ +Fix bug where compiler creates a redundant jump during pseudo-op replacement. Can only happen with a synthetic AST that has a try on the same line as the instruction following the exception handler. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-18-22-41-05.gh-issue-120722.rS7tkE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-22-41-05.gh-issue-120722.rS7tkE.rst new file mode 100644 index 00000000000000..df83e69c601a32 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-22-41-05.gh-issue-120722.rS7tkE.rst @@ -0,0 +1,2 @@ +Correctly set the bytecode position on return instructions within lambdas. +Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-19-01-58-54.gh-issue-120437.nCkIoI.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-01-58-54.gh-issue-120437.nCkIoI.rst new file mode 100644 index 00000000000000..8923f3fcefe3c1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-01-58-54.gh-issue-120437.nCkIoI.rst @@ -0,0 +1 @@ +Fix ``_CHECK_STACK_SPACE`` optimization problems introduced in :gh:`118322`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-19-11-10-50.gh-issue-119462.DpcqSe.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-11-10-50.gh-issue-119462.DpcqSe.rst new file mode 100644 index 00000000000000..7a3b74b63b2e40 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-11-10-50.gh-issue-119462.DpcqSe.rst @@ -0,0 +1,4 @@ +Make sure that invariants of type versioning are maintained: +* Superclasses always have their version number assigned before subclasses +* The version tag is always zero if the tag is not valid. +* The version tag is always non-if the tag is valid. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-19-19-54-35.gh-issue-120754.uF29sj.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-19-54-35.gh-issue-120754.uF29sj.rst new file mode 100644 index 00000000000000..46481d8f31aaba --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-19-54-35.gh-issue-120754.uF29sj.rst @@ -0,0 +1 @@ +Reduce the number of system calls invoked when reading a whole file (ex. ``open('a.txt').read()``). For a sample program that reads the contents of the 400+ ``.rst`` files in the cpython repository ``Doc`` folder, there is an over 10% reduction in system call count. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-19-21-34-21.gh-issue-98442.cqhjkN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-21-34-21.gh-issue-98442.cqhjkN.rst new file mode 100644 index 00000000000000..fb0a93f41a583f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-19-21-34-21.gh-issue-98442.cqhjkN.rst @@ -0,0 +1,2 @@ +Fix too wide source locations of the cleanup instructions of a with +statement. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-24-08-39-23.gh-issue-116017.-Bw2UY.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-24-08-39-23.gh-issue-116017.-Bw2UY.rst new file mode 100644 index 00000000000000..3ca1b37f701e46 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-24-08-39-23.gh-issue-116017.-Bw2UY.rst @@ -0,0 +1,3 @@ +Simplify the warmup mechanism used for "side exits" in JIT code, resulting +in slightly better performance and slightly lower memory usage for most +platforms. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-25-16-26-44.gh-issue-119726.WqvHxB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-25-16-26-44.gh-issue-119726.WqvHxB.rst new file mode 100644 index 00000000000000..2e5132f61e504f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-25-16-26-44.gh-issue-119726.WqvHxB.rst @@ -0,0 +1,2 @@ +Improve the speed and memory use of C function calls from JIT code on AArch64. +Patch by Diego Russo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-26-13-42-36.gh-issue-113433.xKAtLB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-26-13-42-36.gh-issue-113433.xKAtLB.rst new file mode 100644 index 00000000000000..bf8377ac488bcc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-26-13-42-36.gh-issue-113433.xKAtLB.rst @@ -0,0 +1,2 @@ +Subinterpreters now get cleaned up automatically during runtime +finalization. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-26-14-09-31.gh-issue-120838.nFeTL9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-26-14-09-31.gh-issue-120838.nFeTL9.rst new file mode 100644 index 00000000000000..057d00aeeaba11 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-26-14-09-31.gh-issue-120838.nFeTL9.rst @@ -0,0 +1,2 @@ +:c:func:`Py_Finalize()` and :c:func:`Py_FinalizeEx()` now always run with +the main interpreter active. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst new file mode 100644 index 00000000000000..aaecc873551cc7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst @@ -0,0 +1,3 @@ +:c:func:`PyLong_AsNativeBytes` no longer uses :meth:`~object.__index__` +methods by default. The ``Py_ASNATIVEBYTES_ALLOW_INDEX`` flag has been added +to allow it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-28-18-34-49.gh-issue-119726.Fjv_Ab.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-18-34-49.gh-issue-119726.Fjv_Ab.rst new file mode 100644 index 00000000000000..cf5d61450aa3ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-18-34-49.gh-issue-119726.Fjv_Ab.rst @@ -0,0 +1,2 @@ +Optimize code layout for calls to C functions from the JIT on AArch64. +Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-30-03-48-10.gh-issue-121149.lLBMKe.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-30-03-48-10.gh-issue-121149.lLBMKe.rst new file mode 100644 index 00000000000000..38d618f06090fd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-30-03-48-10.gh-issue-121149.lLBMKe.rst @@ -0,0 +1,2 @@ +Added specialization for summation of complexes, this also improves accuracy +of builtin :func:`sum` for such inputs. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst new file mode 100644 index 00000000000000..3df5b216cbc0af --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst @@ -0,0 +1,3 @@ +Fix race condition in ``_PyType_Lookup`` in the free-threaded build due to +a missing memory fence. This could lead to ``_PyType_Lookup`` returning +incorrect results on arm64. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-05-11-29-27.gh-issue-121288.lYKYYP.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-05-11-29-27.gh-issue-121288.lYKYYP.rst new file mode 100644 index 00000000000000..bd3e20b5658562 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-05-11-29-27.gh-issue-121288.lYKYYP.rst @@ -0,0 +1,5 @@ +:exc:`ValueError` messages for :meth:`!list.index()`, :meth:`!range.index()`, +:meth:`!deque.index()`, :meth:`!deque.remove()` and +:meth:`!ShareableList.index()` no longer contain the repr of the searched +value (which can be arbitrary large) and are consistent with error messages +for other :meth:`!index()` and :meth:`!remove()` methods. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst new file mode 100644 index 00000000000000..361f9fc71186c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst @@ -0,0 +1 @@ +Allow tuples of length 20 in the freelist to be reused. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst new file mode 100644 index 00000000000000..7b04eb68b03752 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst @@ -0,0 +1,2 @@ +Tier 2 execution now ensures that list iterators remain exhausted, once they +become exhausted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst new file mode 100644 index 00000000000000..33de31abebe7c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst @@ -0,0 +1,2 @@ +Fix a bug that was preventing the REPL to correctly respect the history when +an input hook was set. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst new file mode 100644 index 00000000000000..aec8ab9d4662b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst @@ -0,0 +1,2 @@ +Fix a bug affecting how multi-line history was being rendered in the new +REPL after interacting with the new screen cache. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst new file mode 100644 index 00000000000000..940380971f407f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst @@ -0,0 +1,2 @@ +Optimized performance of hex_from_char by replacing switch-case with a +lookup table diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst new file mode 100644 index 00000000000000..ec1e7327b77f19 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst @@ -0,0 +1,2 @@ +Unexpected slot wrappers are no longer created for builtin static types in +subinterpreters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst new file mode 100644 index 00000000000000..cb18629d79f5ce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst @@ -0,0 +1,2 @@ +Improve the :exc:`SyntaxError` message if the user tries to use +:keyword:`yield from ` outside a function. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst new file mode 100644 index 00000000000000..72b5c071a5c67b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst @@ -0,0 +1 @@ +Fix pasting of characters containing unicode character joiners in the new REPL. Patch by Marta Gomez Macias diff --git a/Misc/NEWS.d/next/Documentation/2024-06-03-22-06-26.gh-issue-119574.Ik9kOO.rst b/Misc/NEWS.d/next/Documentation/2024-06-03-22-06-26.gh-issue-119574.Ik9kOO.rst new file mode 100644 index 00000000000000..902e7c17fc2e9d --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-06-03-22-06-26.gh-issue-119574.Ik9kOO.rst @@ -0,0 +1 @@ +Added some missing environment variables to the output of :option:`--help-env`. diff --git a/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst new file mode 100644 index 00000000000000..2bf0c977b90387 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst @@ -0,0 +1,3 @@ +Clarify the behaviours of :meth:`multiprocessing.Queue.empty` and +:meth:`multiprocessing.SimpleQueue.empty` on closed queues. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst b/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst new file mode 100644 index 00000000000000..17dc60c11468f4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst @@ -0,0 +1 @@ +Fix documentation for :c:func:`PyModule_AddObjectRef`. diff --git a/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst b/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst new file mode 100644 index 00000000000000..a727c1aa9a0571 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst @@ -0,0 +1 @@ +Improved documentation for :func:`unittest.mock.patch.dict` diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst new file mode 100644 index 00000000000000..22457df03e65c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst @@ -0,0 +1,2 @@ +Fix :func:`urllib.parse.urlunparse` and :func:`urllib.parse.urlunsplit` for URIs with path starting with multiple slashes and no authority. +Based on patch by Ashwin Ramaswami. diff --git a/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst b/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst new file mode 100644 index 00000000000000..b906393449656d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst @@ -0,0 +1,2 @@ +Implemented :func:`ast.compare` for comparing two ASTs. Patch by Batuhan +Taskaya with some help from Jeremy Hylton. diff --git a/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst b/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst new file mode 100644 index 00000000000000..0cdf1e84157777 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst @@ -0,0 +1,3 @@ +Fix integer conversion in :func:`os.major`, :func:`os.minor`, and +:func:`os.makedev`. Support device numbers larger than ``2**63-1``. Support +non-existent device number (``NODEV``). diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst new file mode 100644 index 00000000000000..11559dce0ae2b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -0,0 +1,6 @@ +Add additional methods to :ref:`proxy objects ` +in the :mod:`!multiprocessing` module: + +* :meth:`!clear` and :meth:`!copy` for proxies of :class:`list` +* :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict` diff --git a/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst new file mode 100644 index 00000000000000..2c736e72476313 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst @@ -0,0 +1,3 @@ +Fixed handling in :meth:`inspect.Signature.bind` of keyword arguments having +the same name as positional-only arguments when a variadic keyword argument +(e.g. ``**kwargs``) is present. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst new file mode 100644 index 00000000000000..bc9187309c6a53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst @@ -0,0 +1,4 @@ +Prepare Tkinter for C API changes in Tcl 8.7/9.0 to avoid +:class:`!_tkinter.Tcl_Obj` being unexpectedly returned +instead of :class:`bool`, :class:`str`, +:class:`bytearray`, or :class:`int`. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst b/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst new file mode 100644 index 00000000000000..0302e957b884cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst @@ -0,0 +1 @@ +Fix missing spaces in email headers when the spaces are mixed with encoded 8-bit characters. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst b/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst new file mode 100644 index 00000000000000..4ce1491ffa91e2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst @@ -0,0 +1 @@ +Fix lack of newline characters in :mod:`trace` module output when line tracing is enabled but source code line for current frame is not available. diff --git a/Misc/NEWS.d/next/Library/2023-06-17-09-07-06.gh-issue-105623.5G06od.rst b/Misc/NEWS.d/next/Library/2023-06-17-09-07-06.gh-issue-105623.5G06od.rst new file mode 100644 index 00000000000000..2890674aac4bbc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-17-09-07-06.gh-issue-105623.5G06od.rst @@ -0,0 +1,2 @@ +Fix performance degradation in +:class:`logging.handlers.RotatingFileHandler`. Patch by Craig Robson. diff --git a/Misc/NEWS.d/next/Library/2023-09-25-20-05-41.gh-issue-109747._cRJH8.rst b/Misc/NEWS.d/next/Library/2023-09-25-20-05-41.gh-issue-109747._cRJH8.rst deleted file mode 100644 index b64ba627897a1a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-09-25-20-05-41.gh-issue-109747._cRJH8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve errors for unsupported look-behind patterns. Now re.error is raised -instead of OverflowError or RuntimeError for too large width of look-behind -pattern. diff --git a/Misc/NEWS.d/next/Library/2023-10-08-18-38-09.gh-issue-88434.2Q_IkG.rst b/Misc/NEWS.d/next/Library/2023-10-08-18-38-09.gh-issue-88434.2Q_IkG.rst deleted file mode 100644 index 9722be251bfcad..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-08-18-38-09.gh-issue-88434.2Q_IkG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Emit deprecation warning for non-integer numbers in :mod:`gettext` functions -and methods that consider plural forms even if the translation was not -found. diff --git a/Misc/NEWS.d/next/Library/2023-10-10-17-56-41.gh-issue-110392.6g6CnP.rst b/Misc/NEWS.d/next/Library/2023-10-10-17-56-41.gh-issue-110392.6g6CnP.rst deleted file mode 100644 index 47e4e8ee1f058d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-10-17-56-41.gh-issue-110392.6g6CnP.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :func:`tty.setraw` and :func:`tty.setcbreak`: previously they returned -partially modified list of the original tty attributes. -:func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` now make a copy of the -list of special characters before modifying it. diff --git a/Misc/NEWS.d/next/Library/2023-10-13-06-47-20.gh-issue-110771.opwdlc.rst b/Misc/NEWS.d/next/Library/2023-10-13-06-47-20.gh-issue-110771.opwdlc.rst deleted file mode 100644 index a22f8a0d5e56c3..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-10-13-06-47-20.gh-issue-110771.opwdlc.rst +++ /dev/null @@ -1 +0,0 @@ -Expose the setup and cleanup portions of ``asyncio.run_forever()`` as the standalone methods ``asyncio.run_forever_setup()`` and ``asyncio.run_forever_cleanup()``. This allows for tighter integration with GUI event loops. diff --git a/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst b/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst new file mode 100644 index 00000000000000..b8f9f255e0a75d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst @@ -0,0 +1 @@ +Ignore warnings on text completion inside REPL. diff --git a/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst new file mode 100644 index 00000000000000..069ac68b4f3a95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst @@ -0,0 +1 @@ +Improve performance of :func:`copy.deepcopy` by adding a fast path for atomic types. diff --git a/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst b/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst new file mode 100644 index 00000000000000..6a5783c5ad9846 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst @@ -0,0 +1,6 @@ +In :mod:`importlib.resources`, sync with `importlib_resources 6.3.2 +`_, +including: ``MultiplexedPath`` now expects ``Traversable`` paths, +deprecating string arguments to ``MultiplexedPath``; Enabled support for +resources in namespace packages in zip files; Fixed ``NotADirectoryError`` +when calling files on a subdirectory of a namespace package. diff --git a/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst b/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst new file mode 100644 index 00000000000000..7ceb29330abf22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst @@ -0,0 +1,2 @@ +Fix :func:`dataclasses.dataclass` not creating a ``__weakref__`` slot when +subclassing :class:`typing.Generic`. diff --git a/Misc/NEWS.d/next/Library/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst b/Misc/NEWS.d/next/Library/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst new file mode 100644 index 00000000000000..165a1ba69a811b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst @@ -0,0 +1 @@ +Speed up :func:`os.path.splitroot` & :func:`os.path.normpath` with a direct C call. diff --git a/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst b/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst new file mode 100644 index 00000000000000..37e27a6e37c7d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst @@ -0,0 +1,2 @@ +:func:`os.path.realpath` now suppresses any :exc:`OSError` from +:func:`os.readlink` when *strict* mode is disabled (the default). diff --git a/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst b/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst new file mode 100644 index 00000000000000..f0a87d2a91df3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst @@ -0,0 +1 @@ +Removed executable bits and shebang from stdlib modules. diff --git a/Misc/NEWS.d/next/Library/2024-05-07-17-38-53.gh-issue-118714.XXKpVZ.rst b/Misc/NEWS.d/next/Library/2024-05-07-17-38-53.gh-issue-118714.XXKpVZ.rst new file mode 100644 index 00000000000000..f41baee303482a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-07-17-38-53.gh-issue-118714.XXKpVZ.rst @@ -0,0 +1,2 @@ +Allow ``restart`` in post-mortem debugging of :mod:`pdb`. Removed restart message +when the user quits pdb from post-mortem mode. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst b/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst new file mode 100644 index 00000000000000..474454b36da956 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst @@ -0,0 +1,2 @@ +Allow :class:`typing.TypeVar` instances without a default to follow +instances without a default in some cases. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst b/Misc/NEWS.d/next/Library/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst new file mode 100644 index 00000000000000..67b1fea4f83cb4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst @@ -0,0 +1,2 @@ +Fix :func:`os.path.isfile` on Windows for pipes. +Speedup :func:`os.path.isjunction` and :func:`os.path.lexists` on Windows with a native implementation. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst b/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst new file mode 100644 index 00000000000000..c73bab97b75838 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst @@ -0,0 +1,3 @@ +Drop support for passing additional positional arguments to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst new file mode 100644 index 00000000000000..c99a7e5f024823 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst @@ -0,0 +1,5 @@ +Suppress all :exc:`OSError` exceptions from :meth:`pathlib.Path.exists` and +``is_*()`` methods, rather than a selection of more common errors. The new +behaviour is consistent with :func:`os.path.exists`, :func:`os.path.isdir`, +etc. Use :meth:`pathlib.Path.stat` to retrieve the file status without +suppressing exceptions. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst b/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst new file mode 100644 index 00000000000000..e6ff47e1a3e57b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst @@ -0,0 +1 @@ +Drop support for passing keyword arguments to :class:`pathlib.Path`. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst b/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst new file mode 100644 index 00000000000000..89ef9334fbc65d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst @@ -0,0 +1 @@ +Fix errors in calling Tkinter bindings on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst b/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst new file mode 100644 index 00000000000000..0e2712c26b1c13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst @@ -0,0 +1 @@ +Restore the default value of ``tkiter.wantobjects`` to ``1``. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst b/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst new file mode 100644 index 00000000000000..28847e13207ffe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst @@ -0,0 +1,2 @@ +The *isdst* parameter has been removed from :func:`email.utils.localtime`. +Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst b/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst new file mode 100644 index 00000000000000..2d86dff57faf72 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst @@ -0,0 +1,3 @@ +:class:`!typing.ByteString` and :class:`!collections.abc.ByteString` are +removed. They had previously raised a :exc:`DeprecationWarning` since Python +3.12. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst b/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst new file mode 100644 index 00000000000000..4f1db04d8bd67f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst @@ -0,0 +1,3 @@ +Remove *type*, *choices*, and *metavar* parameters of +:class:`!argparse.BooleanOptionalAction`. +They were deprecated since Python 3.12. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst b/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst new file mode 100644 index 00000000000000..3e0f496047bc8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst @@ -0,0 +1,2 @@ +Remove copy, deepcopy, and pickle from itertools. These had previously +raised a DeprecationWarning since Python 3.12. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst new file mode 100644 index 00000000000000..d036d0cda617ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst @@ -0,0 +1,2 @@ +``ctx`` arguments to the constructors of :mod:`ast` node classes now default +to :class:`ast.Load() `. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst b/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst new file mode 100644 index 00000000000000..c9254f1b9dbea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst @@ -0,0 +1,3 @@ +Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. +Use :func:`pty.openpty` instead. +Patch by Nikita Sobolev. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst b/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst new file mode 100644 index 00000000000000..40612dd93bd6da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst @@ -0,0 +1,3 @@ +Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. It had +previously raised a :exc:`DeprecationWarning` since Python 3.11. +Patch by Nikita Sobolev. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst b/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst new file mode 100644 index 00000000000000..372a809d9594b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst @@ -0,0 +1,2 @@ +Fixed issue where kwargs were no longer passed to the logging handler +QueueHandler diff --git a/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst b/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst new file mode 100644 index 00000000000000..226c8d612a039c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst @@ -0,0 +1,2 @@ +Setting attributes on :data:`typing.NoDefault` now raises +:exc:`AttributeError` instead of :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst b/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst new file mode 100644 index 00000000000000..91c95e4a5395d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst @@ -0,0 +1,2 @@ +Disallow using a sequence of parameters with named placeholders in +:mod:`sqlite3` queries. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst b/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst new file mode 100644 index 00000000000000..36581dbb9bb11b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst @@ -0,0 +1,2 @@ +Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. +Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst new file mode 100644 index 00000000000000..8715deda7d9c41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst @@ -0,0 +1,5 @@ +Support single-dot file extensions in :attr:`pathlib.PurePath.suffix` and +related attributes and methods. For example, the +:attr:`~pathlib.PurePath.suffixes` of ``PurePath('foo.bar.')`` are now +``['.bar', '.']`` rather than ``[]``. This brings file extension splitting +in line with :func:`os.path.splitext`. diff --git a/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst b/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst new file mode 100644 index 00000000000000..fa70b954e1e9ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst @@ -0,0 +1 @@ +Make :mod:`pdb` return to caller frame correctly when ``f_trace`` of the caller frame is not set diff --git a/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst new file mode 100644 index 00000000000000..c2953c65b2720f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst @@ -0,0 +1,2 @@ +Add :meth:`pathlib.Path.copy`, which copies the content of one file to another, +like :func:`shutil.copyfile`. diff --git a/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst b/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst new file mode 100644 index 00000000000000..e86a49af74c9d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst @@ -0,0 +1,2 @@ +Fix an AttributeError in the :mod:`email` module when re-fold a long address +list. Also fix more cases of incorrect encoding of the address separator in the address list. diff --git a/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst b/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst new file mode 100644 index 00000000000000..195be067138b2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst @@ -0,0 +1,2 @@ +Fix issue where :meth:`pathlib.PurePath.with_suffix` didn't raise +:exc:`TypeError` when given ``None`` as a suffix. diff --git a/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst b/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst new file mode 100644 index 00000000000000..30b5f97b8059f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst @@ -0,0 +1 @@ +``difflib.Differ`` is much faster for some cases of diffs where many pairs of lines are equally similar. diff --git a/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst b/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst new file mode 100644 index 00000000000000..fd562ea4f73317 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst @@ -0,0 +1,2 @@ +Fix a NameError happening in ``asyncio.staggered.staggered_race``. This +function is now tested. diff --git a/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst b/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst new file mode 100644 index 00000000000000..7b467b9ebd0d80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst @@ -0,0 +1,3 @@ +Fix high DPI causes turtledemo(turtle-graphics examples) windows blurry +Patch by Wulian233 and Terry Jan Reedy + diff --git a/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst b/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst new file mode 100644 index 00000000000000..e5cfbcf95a0b81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst @@ -0,0 +1,3 @@ +When using the ``**`` operator or :func:`pow` with :class:`~fractions.Fraction` +as the base and an exponent that is not rational, a float, or a complex, the +fraction is no longer converted to a float. diff --git a/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst b/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst new file mode 100644 index 00000000000000..15cd79dec378ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst @@ -0,0 +1 @@ +Remove dependency to :mod:`readline` from the new Python REPL. diff --git a/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst b/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst new file mode 100644 index 00000000000000..2b65eaa6dd70ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst @@ -0,0 +1 @@ +Raise error on certain technically valid but pathological ISO 8601 strings passed to :meth:`datetime.time.fromisoformat` that were previously parsed incorrectly. diff --git a/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst b/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst new file mode 100644 index 00000000000000..4f15c1b67c9774 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst @@ -0,0 +1,5 @@ +In PyREPL, updated ``maybe-accept``'s logic so that if the user hits +:kbd:`Enter` twice, they are able to terminate the block even if there's +trailing whitespace. Also, now when the user hits arrow up, the cursor +is on the last functional line. This matches IPython's behavior. +Patch by Aya Elsayed. diff --git a/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst new file mode 100644 index 00000000000000..ffc4ae336dc54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst @@ -0,0 +1 @@ +:mod:`asyncio` REPL now has the same capabilities as PyREPL. diff --git a/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst new file mode 100644 index 00000000000000..ac595f1b7fc84c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst @@ -0,0 +1,3 @@ +Objects in the datetime C-API are now all statically allocated, which means +better memory safety, especially when the module is reloaded. This should be +transparent to users. diff --git a/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst new file mode 100644 index 00000000000000..48e18f42b5556a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst @@ -0,0 +1 @@ +Add ``socket.VMADDR_CID_LOCAL`` constant. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst b/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst new file mode 100644 index 00000000000000..4470c566a37d88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst @@ -0,0 +1,2 @@ +The interactive REPL no longer runs with ``from __future__ import +annotations`` enabled. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst new file mode 100644 index 00000000000000..3205061a68ce7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst @@ -0,0 +1 @@ +``difflib``'s ``DIffer.compare()`` (and so also ``ndiff``) can no longer be provoked into cubic-time behavior, or into unbounded recursion, and should generally be faster in ordinary cases too. Results may change in some cases, although that should be rare. Correctness of diffs is not affected. Some similar lines far apart may be reported as deleting one and adding the other, where before they were displayed on adjacent output lines with markup showing the intraline differences. diff --git a/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst new file mode 100644 index 00000000000000..8c3a36c9f56475 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst @@ -0,0 +1,3 @@ +Fix ``fcntl.ioctl()`` *request* parameter: use an ``unsigned long`` instead of +an ``unsigned int`` for the *request* parameter of :func:`fcntl.ioctl` to +support requests larger than ``UINT_MAX``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst b/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst new file mode 100644 index 00000000000000..f9b764ae0c49b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst @@ -0,0 +1 @@ +Fix :meth:`!io.TextIOWrapper.write` method breaks internal buffer when the method is called again during flushing internal buffer. diff --git a/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst b/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst new file mode 100644 index 00000000000000..639d5abe878344 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst @@ -0,0 +1,3 @@ +Now, the method ``sock_connect`` of :class:`asyncio.ProactorEventLoop` +raises a :exc:`ValueError` if given socket is not in +non-blocking mode, as well as in other loop implementations. diff --git a/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst b/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst new file mode 100644 index 00000000000000..48cd390d1bb128 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst @@ -0,0 +1,2 @@ +Improve the import time of the :mod:`ast` module by deferring the import of +:mod:`re`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst new file mode 100644 index 00000000000000..bf58d7277fcd51 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst @@ -0,0 +1,2 @@ +Limit exposed globals from internal imports and definitions on new REPL +startup. Patch by Eugene Triguba and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst b/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst new file mode 100644 index 00000000000000..e16cb28b471a7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst @@ -0,0 +1,2 @@ +Catch :exc:`SyntaxError` from :func:`compile` in the runsource() method of +the InteractiveColoredConsole. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst new file mode 100644 index 00000000000000..dd23466b9d2cef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst @@ -0,0 +1,3 @@ +Remove :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, +:class:`!ast.NameConstant` and :class:`!ast.Ellipsis`. They had all emitted +deprecation warnings since Python 3.12. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst b/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst new file mode 100644 index 00000000000000..01321d8bfe2ad5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst @@ -0,0 +1 @@ +``zipfile.Path.is_symlink`` now assesses if the given path is a symlink. diff --git a/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst b/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst new file mode 100644 index 00000000000000..d2de5273edf571 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst @@ -0,0 +1 @@ +If one calls pow(fractions.Fraction, x, module) with modulo not None, the error message now says that the types are incompatible rather than saying pow only takes 2 arguments. Patch by Wim Jeantine-Glenn and Mark Dickinson. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst new file mode 100644 index 00000000000000..92222bc673350f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` was raised on +deep directory trees by adjusting the implementation to be iterative instead +of recursive. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst b/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst new file mode 100644 index 00000000000000..3cf61662fe7767 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst @@ -0,0 +1,2 @@ +Fix performance regression in the :mod:`tokenize` module by caching the ``line`` +token attribute and calculating the column offset more efficiently. diff --git a/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst b/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst new file mode 100644 index 00000000000000..d093c8e35a5994 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst @@ -0,0 +1,2 @@ +Remove deprecated names from ``importlib.abc`` as found in +``importlib.resources.abc``. diff --git a/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst b/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst new file mode 100644 index 00000000000000..3b73d2789fd6f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst @@ -0,0 +1,3 @@ +Partially fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` +is raised on deep directory trees. A recursion error is no longer raised +when :data:`!rmtree.avoids_symlink_attacks` is false. diff --git a/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst new file mode 100644 index 00000000000000..bd2daf3fb5c16d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst @@ -0,0 +1,4 @@ +The :exc:`DeprecationWarning` emitted when testing the truth value of an +:class:`xml.etree.ElementTree.Element` now describes unconditionally +returning ``True`` in a future version rather than raising an exception in +Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst b/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst new file mode 100644 index 00000000000000..854c56609acb8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst @@ -0,0 +1,2 @@ +Fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` is raised +on deep directory trees. diff --git a/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst b/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst new file mode 100644 index 00000000000000..94265e442db584 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst @@ -0,0 +1 @@ +Make :mod:`termios` ``ioctl()`` constants positive. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst b/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst new file mode 100644 index 00000000000000..17a87327b5b1d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst @@ -0,0 +1,3 @@ +In mixed arithmetic operations with :class:`~fractions.Fraction` and +complex, the fraction is now converted to :class:`float` instead of +:class:`complex`. diff --git a/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst new file mode 100644 index 00000000000000..fd6d8d79a9d157 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst @@ -0,0 +1 @@ +Print stack entry in :mod:`pdb` when and only when user input is needed. diff --git a/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst b/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst new file mode 100644 index 00000000000000..b0fe06663248f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst @@ -0,0 +1,2 @@ +The ``_datetime`` module (C implementation for :mod:`datetime`) now supports +being imported in multiple interpreters. diff --git a/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst new file mode 100644 index 00000000000000..d53cc73e728d54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst @@ -0,0 +1,3 @@ +:meth:`!help` and :meth:`!showtopic` methods now respect a +configured *output* argument to :class:`!pydoc.Helper` and not use the +pager in such cases. Patch by Enrico Tröger. diff --git a/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst b/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst new file mode 100644 index 00000000000000..ec9ca20a487d76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst @@ -0,0 +1 @@ +Fix _pyrepl crash when using custom prompt with ANSI escape codes. diff --git a/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst b/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst new file mode 100644 index 00000000000000..80734ef3946300 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst @@ -0,0 +1,2 @@ +The :mod:`ctypes` module may now be imported in all subinterpreters, including +those that have their own GIL. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst new file mode 100644 index 00000000000000..0b232cf8ca1baf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst @@ -0,0 +1 @@ +:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a ``PropertyMock`` being attached. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst b/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst new file mode 100644 index 00000000000000..f9e49c00f671f2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst @@ -0,0 +1,2 @@ +Fix regression to allow logging configuration with multiprocessing queue +types. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst b/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst new file mode 100644 index 00000000000000..e8ea1077139f71 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst @@ -0,0 +1,2 @@ +Expose :meth:`symtable.Symbol.is_type_parameter` in the :mod:`symtable` +module. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst new file mode 100644 index 00000000000000..955be59821ee0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst @@ -0,0 +1,4 @@ +Added the :data:`os.environ.refresh() ` method to update +:data:`os.environ` with changes to the environment made by :func:`os.putenv`, +by :func:`os.unsetenv`, or made outside Python in the same process. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst b/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst new file mode 100644 index 00000000000000..46345bff117b19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst @@ -0,0 +1 @@ +Support building :mod:`tkinter` with Tcl 9.0. diff --git a/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst new file mode 100644 index 00000000000000..0adb70f51e8a0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst @@ -0,0 +1,3 @@ +Add :data:`!socket.IP_RECVERR` and :data:`!socket.IP_RECVTTL` constants +(both available since Linux 2.2). +And :data:`!socket.IP_RECVORIGDSTADDR` constant (available since Linux 2.6.29). diff --git a/Misc/NEWS.d/next/Library/2024-06-05-08-02-46.gh-issue-120108.4U9BL8.rst b/Misc/NEWS.d/next/Library/2024-06-05-08-02-46.gh-issue-120108.4U9BL8.rst new file mode 100644 index 00000000000000..e310695656255d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-08-02-46.gh-issue-120108.4U9BL8.rst @@ -0,0 +1,2 @@ +Fix calling :func:`copy.deepcopy` on :mod:`ast` trees that have been +modified to have references to parent nodes. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst new file mode 100644 index 00000000000000..d1b2c592a113ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst @@ -0,0 +1,4 @@ +Expose :class:`symtable.Symbol` methods :meth:`~symtable.Symbol.is_free_class`, +:meth:`~symtable.Symbol.is_comp_iter` and :meth:`~symtable.Symbol.is_comp_cell`. +Patch by Bénédikt Tran. + diff --git a/Misc/NEWS.d/next/Library/2024-06-05-11-39-21.gh-issue-119933.ooJXQV.rst b/Misc/NEWS.d/next/Library/2024-06-05-11-39-21.gh-issue-119933.ooJXQV.rst new file mode 100644 index 00000000000000..475da88914bde3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-11-39-21.gh-issue-119933.ooJXQV.rst @@ -0,0 +1,3 @@ +Add the :class:`symtable.SymbolTableType` enumeration to represent the +possible outputs of the :class:`symtable.SymbolTable.get_type` method. Patch +by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst new file mode 100644 index 00000000000000..4f3526477c8cce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst @@ -0,0 +1 @@ +Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``. diff --git a/Misc/NEWS.d/next/Library/2024-06-06-12-07-57.gh-issue-119698.rRrprk.rst b/Misc/NEWS.d/next/Library/2024-06-06-12-07-57.gh-issue-119698.rRrprk.rst new file mode 100644 index 00000000000000..d4cca1439816b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-12-07-57.gh-issue-119698.rRrprk.rst @@ -0,0 +1,2 @@ +Fix :meth:`symtable.Class.get_methods` and document its behaviour. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst new file mode 100644 index 00000000000000..c378cac44c97bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst @@ -0,0 +1,2 @@ +:mod:`datetime` no longer crashes in certain complex reference cycle +situations. diff --git a/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst new file mode 100644 index 00000000000000..3e905125797af7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst @@ -0,0 +1 @@ +Remove unused constant ``concurrent.futures._base._FUTURE_STATES`` in :mod:`concurrent.futures`. Patch by Clinton Christian (pygeek). diff --git a/Misc/NEWS.d/next/Library/2024-06-07-10-10-32.gh-issue-117983.NeMR9n.rst b/Misc/NEWS.d/next/Library/2024-06-07-10-10-32.gh-issue-117983.NeMR9n.rst new file mode 100644 index 00000000000000..cca97f50a20496 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-10-10-32.gh-issue-117983.NeMR9n.rst @@ -0,0 +1,2 @@ +Defer the ``threading`` import in ``importlib.util`` until lazy loading is +used. diff --git a/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst new file mode 100644 index 00000000000000..50a662977993f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst @@ -0,0 +1,2 @@ +Fix crash in C version of :meth:`datetime.datetime.strptime` when called again +on the restarted interpreter. diff --git a/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst new file mode 100644 index 00000000000000..0106f2d93318b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst @@ -0,0 +1 @@ +Fix :mod:`tkinter.ttk` with Tcl/Tk 9.0. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst new file mode 100644 index 00000000000000..d21532f22a1d38 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst @@ -0,0 +1 @@ +Fix memory leak in :func:`re.sub()` when the replacement string contains backreferences. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst new file mode 100644 index 00000000000000..d48d43cd047f7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst @@ -0,0 +1,2 @@ +Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp` +to achieve consistency with C-extension implementation. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst new file mode 100644 index 00000000000000..be49577a712867 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst @@ -0,0 +1,4 @@ +Fix erroneous :exc:`NameError` when calling :func:`inspect.get_annotations` +with ``eval_str=True``` on a class that made use of :pep:`695` type +parameters in a module that had ``from __future__ import annotations`` at +the top of the file. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-15-46-35.gh-issue-114053.Ub2XgJ.rst b/Misc/NEWS.d/next/Library/2024-06-08-15-46-35.gh-issue-114053.Ub2XgJ.rst new file mode 100644 index 00000000000000..8aea591da5274c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-15-46-35.gh-issue-114053.Ub2XgJ.rst @@ -0,0 +1,4 @@ +Fix edge-case bug where :func:`typing.get_type_hints` would produce +incorrect results if type parameters in a class scope were overridden by +assignments in a class scope and ``from __future__ import annotations`` +semantics were enabled. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst new file mode 100644 index 00000000000000..04c9ca9c3fd737 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst @@ -0,0 +1,2 @@ +Fix :func:`unittest.mock.patch` to not read attributes of the target when +``new_callable`` is set. Patch by Robert Collins. diff --git a/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst new file mode 100644 index 00000000000000..76714b0c394eef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst @@ -0,0 +1 @@ +Fix column offset reporting for tokens that come after multiline f-strings in the :mod:`tokenize` module. diff --git a/Misc/NEWS.d/next/Library/2024-06-12-10-00-31.gh-issue-90425.5CfkKG.rst b/Misc/NEWS.d/next/Library/2024-06-12-10-00-31.gh-issue-90425.5CfkKG.rst new file mode 100644 index 00000000000000..d152af49287a0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-12-10-00-31.gh-issue-90425.5CfkKG.rst @@ -0,0 +1,2 @@ +The OS byte in gzip headers is now always set to 255 when using +:func:`gzip.compress`. diff --git a/Misc/NEWS.d/next/Library/2024-06-12-11-54-05.gh-issue-120381.O-BNLs.rst b/Misc/NEWS.d/next/Library/2024-06-12-11-54-05.gh-issue-120381.O-BNLs.rst new file mode 100644 index 00000000000000..44f49bc19a4c99 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-12-11-54-05.gh-issue-120381.O-BNLs.rst @@ -0,0 +1,2 @@ +Correct :func:`inspect.ismethoddescriptor` to check also for the lack of +:meth:`~object.__delete__`. Patch by Jan Kaliszewski. diff --git a/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst new file mode 100644 index 00000000000000..d13df7d88b776c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst @@ -0,0 +1,3 @@ +Improve a warning message when a test method in :mod:`unittest` returns +something other than ``None``. Now we show the returned object type and +optional asyncio-related tip. diff --git a/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst new file mode 100644 index 00000000000000..d5114c3d3c904c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst @@ -0,0 +1 @@ +Fix incorrect exception handling in Tab Nanny. Patch by Wulian233. diff --git a/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst new file mode 100644 index 00000000000000..bf8830c6c50386 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst @@ -0,0 +1,2 @@ +Improve the prompt in the "less" pager when :func:`help` is called with +non-string argument. diff --git a/Misc/NEWS.d/next/Library/2024-06-16-21-33-56.gh-issue-120606.kugbwR.rst b/Misc/NEWS.d/next/Library/2024-06-16-21-33-56.gh-issue-120606.kugbwR.rst new file mode 100644 index 00000000000000..874823ea3486fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-16-21-33-56.gh-issue-120606.kugbwR.rst @@ -0,0 +1 @@ +Allow users to use EOF to exit ``commands`` definition in :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2024-06-17-20-04-13.gh-issue-120633.kZC5wt.rst b/Misc/NEWS.d/next/Library/2024-06-17-20-04-13.gh-issue-120633.kZC5wt.rst new file mode 100644 index 00000000000000..9b396988205589 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-17-20-04-13.gh-issue-120633.kZC5wt.rst @@ -0,0 +1 @@ +Move scrollbar and remove tear-off menus in turtledemo. diff --git a/Misc/NEWS.d/next/Library/2024-06-18-19-18-10.gh-issue-120683.xmRez7.rst b/Misc/NEWS.d/next/Library/2024-06-18-19-18-10.gh-issue-120683.xmRez7.rst new file mode 100644 index 00000000000000..50fc9279e4bad1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-18-19-18-10.gh-issue-120683.xmRez7.rst @@ -0,0 +1,4 @@ +Fix an error in :class:`logging.LogRecord`, when the integer part of the +timestamp is rounded up, while the millisecond calculation truncates, +causing the log timestamp to be wrong by up to 999 ms (affected roughly 1 in +8 million timestamps). diff --git a/Misc/NEWS.d/next/Library/2024-06-19-03-09-11.gh-issue-73991.lU_jK9.rst b/Misc/NEWS.d/next/Library/2024-06-19-03-09-11.gh-issue-73991.lU_jK9.rst new file mode 100644 index 00000000000000..60a1b68d5bb1a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-03-09-11.gh-issue-73991.lU_jK9.rst @@ -0,0 +1 @@ +Add :meth:`pathlib.Path.copytree`, which recursively copies a directory. diff --git a/Misc/NEWS.d/next/Library/2024-06-19-13-20-01.gh-issue-111259.Wki5PV.rst b/Misc/NEWS.d/next/Library/2024-06-19-13-20-01.gh-issue-111259.Wki5PV.rst new file mode 100644 index 00000000000000..91ed5f550e400f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-13-20-01.gh-issue-111259.Wki5PV.rst @@ -0,0 +1,3 @@ +:mod:`re` now handles patterns like ``"[\s\S]"`` or ``"\s|\S"`` which match +any character as effectively as a dot with the ``DOTALL`` modifier +(``"(?s:.)"``). diff --git a/Misc/NEWS.d/next/Library/2024-06-19-15-06-58.gh-issue-120732.OvYV9b.rst b/Misc/NEWS.d/next/Library/2024-06-19-15-06-58.gh-issue-120732.OvYV9b.rst new file mode 100644 index 00000000000000..e31c4dd3192d60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-15-06-58.gh-issue-120732.OvYV9b.rst @@ -0,0 +1,2 @@ +Fix ``name`` passing to :class:`unittest.mock.Mock` object when using +:func:`unittest.mock.create_autospec`. diff --git a/Misc/NEWS.d/next/Library/2024-06-19-15-43-04.gh-issue-120743.CMMl2P.rst b/Misc/NEWS.d/next/Library/2024-06-19-15-43-04.gh-issue-120743.CMMl2P.rst new file mode 100644 index 00000000000000..e06dcf8af26a60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-15-43-04.gh-issue-120743.CMMl2P.rst @@ -0,0 +1,3 @@ +:term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* +` functions. They should no longer be used to write new code. The +:mod:`subprocess` module is recommended instead. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-06-19-23-08-25.gh-issue-120780.0Omopb.rst b/Misc/NEWS.d/next/Library/2024-06-19-23-08-25.gh-issue-120780.0Omopb.rst new file mode 100644 index 00000000000000..df3cfbcdbd2e29 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-23-08-25.gh-issue-120780.0Omopb.rst @@ -0,0 +1 @@ +Show string value of LOAD_SPECIAL oparg in :mod:`dis` output. diff --git a/Misc/NEWS.d/next/Library/2024-06-20-01-31-24.gh-issue-120769.PfiMrc.rst b/Misc/NEWS.d/next/Library/2024-06-20-01-31-24.gh-issue-120769.PfiMrc.rst new file mode 100644 index 00000000000000..8ee6bf1a9c6480 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-20-01-31-24.gh-issue-120769.PfiMrc.rst @@ -0,0 +1 @@ +Make empty line in :mod:`pdb` repeats the last command even when the command is from ``cmdqueue``. diff --git a/Misc/NEWS.d/next/Library/2024-06-21-06-37-46.gh-issue-120713.WBbQx4.rst b/Misc/NEWS.d/next/Library/2024-06-21-06-37-46.gh-issue-120713.WBbQx4.rst new file mode 100644 index 00000000000000..18386a43eddc6f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-21-06-37-46.gh-issue-120713.WBbQx4.rst @@ -0,0 +1,2 @@ +:meth:`datetime.datetime.strftime` now 0-pads years with less than four digits for the format specifiers ``%Y`` and ``%G`` on Linux. +Patch by Ben Hsing diff --git a/Misc/NEWS.d/next/Library/2024-06-21-12-00-16.gh-issue-120782.LOE8tj.rst b/Misc/NEWS.d/next/Library/2024-06-21-12-00-16.gh-issue-120782.LOE8tj.rst new file mode 100644 index 00000000000000..02acbd2873009b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-21-12-00-16.gh-issue-120782.LOE8tj.rst @@ -0,0 +1 @@ +Fix wrong references of the :mod:`datetime` types after reloading the module. diff --git a/Misc/NEWS.d/next/Library/2024-06-21-14-32-56.gh-issue-120811.eBmVTV.rst b/Misc/NEWS.d/next/Library/2024-06-21-14-32-56.gh-issue-120811.eBmVTV.rst new file mode 100644 index 00000000000000..62cd7b5620474a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-21-14-32-56.gh-issue-120811.eBmVTV.rst @@ -0,0 +1 @@ +Fix possible memory leak in :meth:`contextvars.Context.run`. diff --git a/Misc/NEWS.d/next/Library/2024-06-22-22-23-56.gh-issue-101830.1BAoxH.rst b/Misc/NEWS.d/next/Library/2024-06-22-22-23-56.gh-issue-101830.1BAoxH.rst new file mode 100644 index 00000000000000..46c18b040f30d7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-22-22-23-56.gh-issue-101830.1BAoxH.rst @@ -0,0 +1,2 @@ +Accessing the :mod:`tkinter` object's string representation no longer converts +the underlying Tcl object to a string on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-06-22-22-52-24.gh-issue-120888.sd8I3N.rst b/Misc/NEWS.d/next/Library/2024-06-22-22-52-24.gh-issue-120888.sd8I3N.rst new file mode 100644 index 00000000000000..c733ff5159aa40 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-22-22-52-24.gh-issue-120888.sd8I3N.rst @@ -0,0 +1 @@ +Upgrade pip wheel bundled with ensurepip (pip 24.1.1) diff --git a/Misc/NEWS.d/next/Library/2024-06-23-07-23-08.gh-issue-61103.ca_U_l.rst b/Misc/NEWS.d/next/Library/2024-06-23-07-23-08.gh-issue-61103.ca_U_l.rst new file mode 100644 index 00000000000000..890eb62010eb33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-23-07-23-08.gh-issue-61103.ca_U_l.rst @@ -0,0 +1,5 @@ +Support :c:expr:`float complex`, :c:expr:`double complex` and +:c:expr:`long double complex` C types in :mod:`ctypes` as +:class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and +:class:`~ctypes.c_longdouble_complex` if the compiler has C11 complex arithmetic. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-06-23-11-21-27.gh-issue-120910.t0QXdB.rst b/Misc/NEWS.d/next/Library/2024-06-23-11-21-27.gh-issue-120910.t0QXdB.rst new file mode 100644 index 00000000000000..3773cdc6ee3bf3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-23-11-21-27.gh-issue-120910.t0QXdB.rst @@ -0,0 +1,2 @@ +When reading installed files from an egg, use ``relative_to(walk_up=True)`` +to honor files installed outside of the installation root. diff --git a/Misc/NEWS.d/next/Library/2024-06-23-17-50-40.gh-issue-119614.vwPGLB.rst b/Misc/NEWS.d/next/Library/2024-06-23-17-50-40.gh-issue-119614.vwPGLB.rst new file mode 100644 index 00000000000000..d518265a7fe55a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-23-17-50-40.gh-issue-119614.vwPGLB.rst @@ -0,0 +1,2 @@ +Fix truncation of strings with embedded null characters in some internal +operations in :mod:`tkinter`. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst new file mode 100644 index 00000000000000..346a89879cad41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst @@ -0,0 +1,3 @@ +Fixed issues where :meth:`!argparse.ArgumentParser.parse_args` did not honor +``exit_on_error=False``. +Based on patch by Ben Hsing. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-10-13-40.gh-issue-121025.M-XXlV.rst b/Misc/NEWS.d/next/Library/2024-06-26-10-13-40.gh-issue-121025.M-XXlV.rst new file mode 100644 index 00000000000000..38cad610396787 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-10-13-40.gh-issue-121025.M-XXlV.rst @@ -0,0 +1,2 @@ +Improve the :meth:`~object.__repr__` of :class:`functools.partialmethod`. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-17-00-39.gh-issue-117784.inCtAV.rst b/Misc/NEWS.d/next/Library/2024-06-26-17-00-39.gh-issue-117784.inCtAV.rst new file mode 100644 index 00000000000000..3f576eebc9a85d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-17-00-39.gh-issue-117784.inCtAV.rst @@ -0,0 +1 @@ +CPython now detects whether its linked TLS library supports TLSv1.3 post-handshake authentication and disables that feature if support is lacking. diff --git a/Misc/NEWS.d/next/Library/2024-06-27-12-27-52.gh-issue-121027.D4K1OX.rst b/Misc/NEWS.d/next/Library/2024-06-27-12-27-52.gh-issue-121027.D4K1OX.rst new file mode 100644 index 00000000000000..a450726d9afed9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-27-12-27-52.gh-issue-121027.D4K1OX.rst @@ -0,0 +1 @@ +Make the :class:`functools.partial` object a method descriptor. diff --git a/Misc/NEWS.d/next/Library/2024-06-27-13-47-14.gh-issue-121027.jh55EC.rst b/Misc/NEWS.d/next/Library/2024-06-27-13-47-14.gh-issue-121027.jh55EC.rst new file mode 100644 index 00000000000000..8470c8b37ac83d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-27-13-47-14.gh-issue-121027.jh55EC.rst @@ -0,0 +1,2 @@ +Add a future warning in :meth:`!functools.partial.__get__`. In future Python +versions :class:`functools.partial` will be a method descriptor. diff --git a/Misc/NEWS.d/next/Library/2024-06-29-05-08-59.gh-issue-87744.rpF6Jw.rst b/Misc/NEWS.d/next/Library/2024-06-29-05-08-59.gh-issue-87744.rpF6Jw.rst new file mode 100644 index 00000000000000..c0b4f349fb6dac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-29-05-08-59.gh-issue-87744.rpF6Jw.rst @@ -0,0 +1 @@ +Fix waitpid race while calling :meth:`~asyncio.subprocess.Process.send_signal` in asyncio. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2024-06-29-15-21-12.gh-issue-121141.4evD6q.rst b/Misc/NEWS.d/next/Library/2024-06-29-15-21-12.gh-issue-121141.4evD6q.rst new file mode 100644 index 00000000000000..f2dc621050ff4b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-29-15-21-12.gh-issue-121141.4evD6q.rst @@ -0,0 +1 @@ +Add support for :func:`copy.replace` to AST nodes. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst b/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst new file mode 100644 index 00000000000000..50f945ab9f1436 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst @@ -0,0 +1,2 @@ +Add support for ``all`` as an valid ``action`` for :func:`warnings.simplefilter` +and :func:`warnings.filterwarnings`. diff --git a/Misc/NEWS.d/next/Library/2024-07-01-11-23-18.gh-issue-121210.cD0zfn.rst b/Misc/NEWS.d/next/Library/2024-07-01-11-23-18.gh-issue-121210.cD0zfn.rst new file mode 100644 index 00000000000000..55d5b221bf0765 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-01-11-23-18.gh-issue-121210.cD0zfn.rst @@ -0,0 +1,2 @@ +Handle AST nodes with missing runtime fields or attributes in +:func:`ast.compare`. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-07-02-11-34-06.gh-issue-121245.sSkDAr.rst b/Misc/NEWS.d/next/Library/2024-07-02-11-34-06.gh-issue-121245.sSkDAr.rst new file mode 100644 index 00000000000000..6e9dec2545166f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-02-11-34-06.gh-issue-121245.sSkDAr.rst @@ -0,0 +1,2 @@ +Fix a bug in the handling of the command history of the new :term:`REPL` that caused +the history file to be wiped at REPL exit. diff --git a/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst b/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst new file mode 100644 index 00000000000000..480f27e05953a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst @@ -0,0 +1,4 @@ +Fix constructor of :mod:`ast` nodes with custom ``_attributes``. Previously, +passing custom attributes would raise a :py:exc:`DeprecationWarning`. Passing +arguments to the constructor that are not in ``_fields`` or ``_attributes`` +remains deprecated. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-07-04-17-36-03.gh-issue-59110.IlI9Fz.rst b/Misc/NEWS.d/next/Library/2024-07-04-17-36-03.gh-issue-59110.IlI9Fz.rst new file mode 100644 index 00000000000000..b8e3ee0720cfe6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-04-17-36-03.gh-issue-59110.IlI9Fz.rst @@ -0,0 +1,2 @@ +:mod:`zipimport` supports now namespace packages when no directory entry +exists. diff --git a/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst b/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst new file mode 100644 index 00000000000000..5d9b50d452a9cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst @@ -0,0 +1 @@ +Slightly speed up :func:`os.walk` by simplifying exception handling. diff --git a/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst b/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst new file mode 100644 index 00000000000000..4a65fb737f025b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst @@ -0,0 +1,4 @@ +Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace()`) now +reuse the most recent ``Pdb`` instance that calls ``Pdb.set_trace()``, +instead of creating a new one each time. As a result, all the instance specific +data like ``display`` and ``commands`` are preserved across Hard-coded breakpoints. diff --git a/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst b/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst new file mode 100644 index 00000000000000..33e9ab94852e35 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst @@ -0,0 +1,2 @@ +The *shallow* argument to :class:`filecmp.dircmp` (new in Python 3.13) is +now keyword-only. diff --git a/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst new file mode 100644 index 00000000000000..8ac48aac816a60 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst @@ -0,0 +1,4 @@ +:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict +the new directory to the current user. This fixes :cve:`2024-4030` +affecting :func:`tempfile.mkdtemp` in scenarios where the base temporary +directory is more permissive than the default. diff --git a/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst new file mode 100644 index 00000000000000..bfec178f6318a7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst @@ -0,0 +1,2 @@ +Fixes creation of ACLs in :func:`os.mkdir` on Windows to work correctly on +non-English machines. diff --git a/Misc/NEWS.d/next/Security/2024-06-25-04-42-43.gh-issue-112301.god4IC.rst b/Misc/NEWS.d/next/Security/2024-06-25-04-42-43.gh-issue-112301.god4IC.rst new file mode 100644 index 00000000000000..68058a06f0bf49 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-06-25-04-42-43.gh-issue-112301.god4IC.rst @@ -0,0 +1,2 @@ +Add default compiler options to improve security. Enable +-Wimplicit-fallthrough, -fstack-protector-strong, -Wtrampolines. diff --git a/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst b/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst new file mode 100644 index 00000000000000..cfc70c16b2b279 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst @@ -0,0 +1,2 @@ +regrtest test runner: Add XML support to the refleak checker (-R option). +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst b/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst new file mode 100644 index 00000000000000..905b4e3a1c9043 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst @@ -0,0 +1,3 @@ +Python test runner no longer runs tests using TTY (ex: test_ioctl) in a +process group (using ``setsid()``). Previously, tests using TTY were +skipped. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst new file mode 100644 index 00000000000000..bf28d8bb77b8a2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst @@ -0,0 +1,2 @@ +Add ``--single-process`` command line option to Python test runner (regrtest). +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-06-20-12-51-26.gh-issue-120801.lMVXC9.rst b/Misc/NEWS.d/next/Tests/2024-06-20-12-51-26.gh-issue-120801.lMVXC9.rst new file mode 100644 index 00000000000000..8559cb8b99c384 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-06-20-12-51-26.gh-issue-120801.lMVXC9.rst @@ -0,0 +1,2 @@ +Cleaned up fixtures for importlib.metadata tests and consolidated behavior +with 'test.support.os_helper'. diff --git a/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst b/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst new file mode 100644 index 00000000000000..c92002d8fe3cd2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst @@ -0,0 +1,3 @@ +When creating the JUnit XML file, regrtest now escapes characters which are +invalid in XML, such as the chr(27) control character used in ANSI escape +sequences. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-07-01-16-15-06.gh-issue-121200.4Pc-gc.rst b/Misc/NEWS.d/next/Tests/2024-07-01-16-15-06.gh-issue-121200.4Pc-gc.rst new file mode 100644 index 00000000000000..01e0d9b9f217d4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-01-16-15-06.gh-issue-121200.4Pc-gc.rst @@ -0,0 +1,3 @@ +Fix ``test_expanduser_pwd2()`` of ``test_posixpath``. Call ``getpwnam()`` +to get ``pw_dir``, since it can be different than ``getpwall()`` ``pw_dir``. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-07-04-15-10-29.gh-issue-121084.qxcd5d.rst b/Misc/NEWS.d/next/Tests/2024-07-04-15-10-29.gh-issue-121084.qxcd5d.rst new file mode 100644 index 00000000000000..b91ea8acfadbf1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-04-15-10-29.gh-issue-121084.qxcd5d.rst @@ -0,0 +1,3 @@ +Fix test_typing random leaks. Clear typing ABC caches when running tests for +refleaks (``-R`` option): call ``_abc_caches_clear()`` on typing abstract +classes and their subclasses. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst b/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst new file mode 100644 index 00000000000000..7d904f26a36ff2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst @@ -0,0 +1,3 @@ +:func:`os.getloadavg` may throw :exc:`OSError` when running regression tests +under certain conditions (e.g. chroot). This error is now caught and +ignored, since reporting load average is optional. diff --git a/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst b/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst new file mode 100644 index 00000000000000..7f840b0556048a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst @@ -0,0 +1 @@ +Updated bundled Tcl/Tk to 8.6.14. diff --git a/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst b/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst new file mode 100644 index 00000000000000..da70b2528919e1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst @@ -0,0 +1,2 @@ +Avoid crashing in :mod:`mmap` on Windows when the mapped memory is inaccessible +due to file system errors or access violations. diff --git a/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst b/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst new file mode 100644 index 00000000000000..0931687ecc521c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst @@ -0,0 +1 @@ +Fixes an issue with the Windows installer not running ensurepip in a fully isolated environment. This could cause unexpected interactions with the user site-packages. diff --git a/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst b/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst new file mode 100644 index 00000000000000..aab26f57209864 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst @@ -0,0 +1,3 @@ +Fixes ``py.exe`` handling of shebangs like ``/usr/bin/env python3.12``, +which were previously interpreted as ``python3.exe`` instead of +``python3.12.exe``. diff --git a/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst b/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst new file mode 100644 index 00000000000000..f3918ed633d78c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst @@ -0,0 +1 @@ +Add support for new pyrepl on Windows diff --git a/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst b/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst new file mode 100644 index 00000000000000..84dd2161aa1db8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst @@ -0,0 +1 @@ +Adds Unicode support and fixes audit events for ``_winapi.CreateNamedPipe``. diff --git a/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst b/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst new file mode 100644 index 00000000000000..db9e798d3ddcb8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst @@ -0,0 +1 @@ +Ensures correct import libraries are included in Windows installs. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json new file mode 100644 index 00000000000000..758d41910054ce --- /dev/null +++ b/Misc/externals.spdx.json @@ -0,0 +1,196 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-bzip2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "bzip2", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libffi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "libffi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-4.0.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:4.0.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "openssl", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.0.13" + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "730e4a3efd6a63828bee499940fb13acc2a32c182502ce8a1d970387895d0504" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.3.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.3.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "sqlite", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.45.3.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ad7623a44e1b6e42df47ba8f16b2b0435ac605650b5054077c4355a30473074c" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.14.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tcl-core", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.14.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tk", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e8d5cbe97952037962518b69aba85e324d80aa189054c163ab0ee764a448e802" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.14.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tk", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.14.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-xz", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "xz", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.2.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "zlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.3.1" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 3307260544e8a6..ec0857a4a998c0 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -12,8 +12,20 @@ #undef powerpc #undef sparc #undef unix + #if defined(__ANDROID__) - # Android is not a multiarch system. +# if defined(__x86_64__) +PLATFORM_TRIPLET=x86_64-linux-android +# elif defined(__i386__) +PLATFORM_TRIPLET=i686-linux-android +# elif defined(__aarch64__) +PLATFORM_TRIPLET=aarch64-linux-android +# elif defined(__arm__) +PLATFORM_TRIPLET=arm-linux-androideabi +# else +# error unknown Android platform +# endif + #elif defined(__linux__) /* * BEGIN of Linux block @@ -233,7 +245,24 @@ PLATFORM_TRIPLET=i386-gnu # error unknown platform triplet # endif #elif defined(__APPLE__) +# include "TargetConditionals.h" +// Older macOS SDKs do not define TARGET_OS_* +# if defined(TARGET_OS_IOS) && TARGET_OS_IOS +# if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR +# if __x86_64__ +PLATFORM_TRIPLET=x86_64-iphonesimulator +# else +PLATFORM_TRIPLET=arm64-iphonesimulator +# endif +# else +PLATFORM_TRIPLET=arm64-iphoneos +# endif +// Older macOS SDKs do not define TARGET_OS_OSX +# elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# else +# error unknown Apple platform +# endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks #elif defined(__wasm32__) diff --git a/Misc/python-config.in b/Misc/python-config.in index 81c3316e334a48..dd5d161ab2286f 100644 --- a/Misc/python-config.in +++ b/Misc/python-config.in @@ -13,7 +13,8 @@ valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', def exit_with_usage(code=1): print("Usage: {0} [{1}]".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), file=sys.stderr) + sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), + file=sys.stdout if code == 0 else sys.stderr) sys.exit(code) try: diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..9929f5b2653dca 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,13 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + local usage + usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [ "$1" -eq 0 ]; then + echo "$usage" + else + echo "$usage" >&2 + fi exit $1 } @@ -41,7 +47,7 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" -LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" +LIBS="@MODULE_LDFLAGS@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" LDLIBRARY="@LDLIBRARY@" diff --git a/Misc/python.man b/Misc/python.man index 9f89c94adf5028..4076b8d3d1ba30 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -251,6 +251,7 @@ emitted by a process (even those that are otherwise ignored by default): -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -601,6 +602,16 @@ show how long each import takes. This is exactly equivalent to setting .IP PYTHONBREAKPOINT If this environment variable is set to 0, it disables the default debugger. It can be set to the callable of your debugger of choice. +.IP PYTHON_COLORS +If this variable is set to 1, the interpreter will colorize various kinds of +output. Setting it to 0 deactivates this behavior. +.IP PYTHON_HISTORY +This environment variable can be used to set the location of a history file +(on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Misc/python.pc.in b/Misc/python.pc.in index 027dba38585a89..c2c740e82b1fde 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: -L${libdir} @LIBPYTHON@ +Libs: -L${libdir} @MODULE_LDFLAGS@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json new file mode 100644 index 00000000000000..b60adcfd362f68 --- /dev/null +++ b/Misc/sbom.spdx.json @@ -0,0 +1,2229 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "files": [ + { + "SPDXID": "SPDXRef-FILE-Modules-expat-COPYING", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "39e6f567a10e36b2e77727e98e60bbcb3eb3af0b" + }, + { + "algorithm": "SHA256", + "checksumValue": "122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534" + } + ], + "fileName": "Modules/expat/COPYING" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-ascii.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b0235fa3cf845a7d68e8e66dd344d5e32e8951b5" + }, + { + "algorithm": "SHA256", + "checksumValue": "42f8b392c70366743eacbc60ce021389ccaa333598dd49eef6ee5c93698ca205" + } + ], + "fileName": "Modules/expat/ascii.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-asciitab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cbb53d16ca1f35ee9c9e296116efd222ae611ed9" + }, + { + "algorithm": "SHA256", + "checksumValue": "1cc0ae749019fc0e488cd1cf245f6beaa6d4f7c55a1fc797e5aa40a408bc266b" + } + ], + "fileName": "Modules/expat/asciitab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-expat.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4076a884f0ca96873589b5c8159e2e5bfb8b829a" + }, + { + "algorithm": "SHA256", + "checksumValue": "1a434bf3d2f9fb8a0b5adb79201a942788d11824c3e5b46a0b9962c0c482016c" + } + ], + "fileName": "Modules/expat/expat.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-expat-external.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7" + }, + { + "algorithm": "SHA256", + "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d" + } + ], + "fileName": "Modules/expat/expat_external.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1b0e9014c0baa4c6254d2b5e6a67c70148309c34" + }, + { + "algorithm": "SHA256", + "checksumValue": "ad8b01e9f323cc4208bcd22241df383d7e8641fe3c8b3415aa513de82531f89f" + } + ], + "fileName": "Modules/expat/iasciitab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-internal.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "e23d160cc33cc2c25a4b48f7b242f906444418e0" + }, + { + "algorithm": "SHA256", + "checksumValue": "f7523357d8009749e7dba94b0bd7d0fa60e011cc254e55c4ebccd6313f031122" + } + ], + "fileName": "Modules/expat/internal.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d335ecca380e331a0ea7dc33838a4decd93ec1e4" + }, + { + "algorithm": "SHA256", + "checksumValue": "eab66226da100372e01e42e1cbcd8ac2bbbb5c1b5f95d735289cc85c7a8fc2ba" + } + ], + "fileName": "Modules/expat/latin1tab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-nametab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cf2bc9626c945826602ba9170786e9a2a44645e4" + }, + { + "algorithm": "SHA256", + "checksumValue": "67dcf415d37a4b692a6a8bb46f990c02d83f2ef3d01a65cd61c8594a084246f2" + } + ], + "fileName": "Modules/expat/nametab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" + }, + { + "algorithm": "SHA256", + "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" + } + ], + "fileName": "Modules/expat/pyexpatns.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c49b5df2bc702f663ba3b5a52d1940ec363226b" + }, + { + "algorithm": "SHA256", + "checksumValue": "b5ec29f6560acc183f1ee8ab92bb3aea17b87b4c2120cd2e3f78deba7a12491e" + } + ], + "fileName": "Modules/expat/siphash.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-utf8tab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b77c8fcfb551553c81d6fbd94c798c8aa04ad021" + }, + { + "algorithm": "SHA256", + "checksumValue": "8cd26bd461d334d5e1caedb3af4518d401749f2fc66d56208542b29085159c18" + } + ], + "fileName": "Modules/expat/utf8tab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-winconfig.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "a3a8c44efd55dbf2cfea8fcee009ec63120ec0a3" + }, + { + "algorithm": "SHA256", + "checksumValue": "e70948500d34dfcba4e9f0b305319dfe2a937c7cbfb687905128b56e1a6f8b33" + } + ], + "fileName": "Modules/expat/winconfig.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlparse.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fed1311be8577491b7f63085a27014eabf2caec8" + }, + { + "algorithm": "SHA256", + "checksumValue": "3dc233eca5fa1bb7387c503f8a12d840707e4374b229e05d5657db9645725040" + } + ], + "fileName": "Modules/expat/xmlparse.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlrole.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ef767128d2dda99436712dcf3465dde5dbaab876" + }, + { + "algorithm": "SHA256", + "checksumValue": "71fb52aa302cf6f56e41943009965804f49ff2210d9bd15b258f70aaf70db772" + } + ], + "fileName": "Modules/expat/xmlrole.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlrole.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c961fb1a80f7b0601a63e69fba793fe5f6dff157" + }, + { + "algorithm": "SHA256", + "checksumValue": "228470eb9181a9a7575b63137edcb61b817ee4e0923faffdbeba29e07c939713" + } + ], + "fileName": "Modules/expat/xmlrole.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "8394790c0199c8f88108542ad78f23095d28a3fe" + }, + { + "algorithm": "SHA256", + "checksumValue": "5b16c671ccc42496374762768e4bf48f614aecfd2025a07925b8d94244aec645" + } + ], + "fileName": "Modules/expat/xmltok.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7d2943a0128094455004b1a98007b98734221bae" + }, + { + "algorithm": "SHA256", + "checksumValue": "6b8919dc951606dc6f2b0175f8955a9ced901ce8bd08db47f291b6c04227ae7f" + } + ], + "fileName": "Modules/expat/xmltok.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7756f7c0d3625ae7dde6cf7d386685ffacb57c7e" + }, + { + "algorithm": "SHA256", + "checksumValue": "a3fe18ff32b21fbcb7c190895c68158404e1b9fb449db6431bc08b261dc03938" + } + ], + "fileName": "Modules/expat/xmltok_impl.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "788332fe8040bed71172cddedb69abd848cc62f7" + }, + { + "algorithm": "SHA256", + "checksumValue": "f05ad4fe5e98429a7349ff04f57192cac58c324601f2a2e5e697ab0bc05d36d5" + } + ], + "fileName": "Modules/expat/xmltok_impl.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2d82d0a1201f78d478b30d108ff8fc27ee3e2672" + }, + { + "algorithm": "SHA256", + "checksumValue": "6ce6d03193279078d55280150fe91e7370370b504a6c123a79182f28341f3e90" + } + ], + "fileName": "Modules/expat/xmltok_ns.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f8ba39b46ebdfa7d031d9c33130c6ded680a8120" + }, + { + "algorithm": "SHA256", + "checksumValue": "f71cf6a0e8f09354c2af2c785a1d36e0cba7613a589be01ca8a3d8478f4c8874" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "eaaab54cea2b0bb8ec0eedf0b373d42f1a0f8f6c" + }, + { + "algorithm": "SHA256", + "checksumValue": "9a02e2a6e163515ea0228a859d5e55c1f57b11fae5908c42f9f9814ce9bca230" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f4f42faf8da78a230199f649c0f2a1b865799a31" + }, + { + "algorithm": "SHA256", + "checksumValue": "5b29bd9951646861e0e19427be5d923a5bab7a4516824ccc068f696469195eec" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "722b57139737ceeb88e41d3839e6f7d70578741b" + }, + { + "algorithm": "SHA256", + "checksumValue": "5640295c790d56b1b4df147d6a6c58803b1845cd7d93365bf7cc7b75ba3cacd5" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f2aa3ed6acce621c162bc3a0592780ce5aa3bc4d" + }, + { + "algorithm": "SHA256", + "checksumValue": "30638efb75c8b185bb09c3df6977e3f3c5d21a1e696218cf7ade6bc4d5201b31" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4903e10291d07367be3bc283935bc52926e57ba1" + }, + { + "algorithm": "SHA256", + "checksumValue": "093d7693084af0999d2a13d207311d74b5bdfdc9c08447ed4a979e3f7505ae6b" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "66644fd3325c414fef7d985536bb477c849c8f9a" + }, + { + "algorithm": "SHA256", + "checksumValue": "17c0db96d40d1849f02546d5f55428fa89b61b07748d5b5df45cec25c5f29c0f" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "580e9a73813281e99a98871380b3726576295a96" + }, + { + "algorithm": "SHA256", + "checksumValue": "d8d4d14bbc3a561a4e590d9b18b326e6a8095efb12423edbd949cf3c00953621" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" + }, + { + "algorithm": "SHA256", + "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" + } + ], + "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "12c0c680c93b8112b97cc575faacbb3cbbd315b1" + }, + { + "algorithm": "SHA256", + "checksumValue": "455e94f24a0900deda7e6e36f4714e4253d32cea077f97e23f90c569a717bc48" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "62b44acbbdc77b749c36c242cda027bacf7679f8" + }, + { + "algorithm": "SHA256", + "checksumValue": "65decdb74c24049aa19430462a51219250cfc65d8c162778e42df88b3142fa42" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1987119a563a8fdc5966286e274f716dbcea77ee" + }, + { + "algorithm": "SHA256", + "checksumValue": "fe57e1bc5ce3224d106e36cb8829b5399c63a68a70b0ccd0c91d82a4565c8869" + } + ], + "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ba64394679643c6d4ceaf6bd2616d48d12f996a7" + }, + { + "algorithm": "SHA256", + "checksumValue": "d16a59f37a1d4982626870e370889eb9d332a9ad035661b8062f549fc734d061" + } + ], + "fileName": "Modules/_hacl/include/krml/internal/target.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "964e09bd99ff2366afd6193b59863fc925e7fb05" + }, + { + "algorithm": "SHA256", + "checksumValue": "3734c7942bec9a434e16df069fa45bdcb84b130f14417bc5f7bfe8546272d9f5" + } + ], + "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" + }, + { + "algorithm": "SHA256", + "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" + } + ], + "fileName": "Modules/_hacl/include/krml/types.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "60f02d21f045c8a4c2b6b84a8f7e023d9490c8e5" + }, + { + "algorithm": "SHA256", + "checksumValue": "370d8ef9c48cb55472ece11e12eaf94c58118de3f5515b6df1c130b696597828" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6346c30a140e7d3010c98fe19d14fa229a54eb16" + }, + { + "algorithm": "SHA256", + "checksumValue": "ab52c6092bdbbfc9884f841bf4824016792ffa96167577cbe0df00dd96f56a34" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0018e084339058dd454b4e49d10d236b4f896bf8" + }, + { + "algorithm": "SHA256", + "checksumValue": "10e959a92b3288a6165a404c8fae2bbcd7fb00a9abbae2b7809fa55d6fe9068d" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "eae8a5226bf993f07584cf4c0d269022328cf3d4" + }, + { + "algorithm": "SHA256", + "checksumValue": "6853125de10d0f605e9bc3a3dbbd7254713709e9893cc3f69929ea8d3f254934" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d8063060cc707a7ac70108a15934d33e7b448db6" + }, + { + "algorithm": "SHA256", + "checksumValue": "347dfdf856ed1e584d124d6709b51267598ea5b37c1a2e03beeb358c978beada" + } + ], + "fileName": "Modules/_hacl/python_hacl_namespaces.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ff5e3ae2360adf7279a9c54d12a1d32e16a1f223" + }, + { + "algorithm": "SHA256", + "checksumValue": "1eb919e885244e43cdf7b2104ad30dc9271513478c0026f6bfb4bad6e2f0ab42" + } + ], + "fileName": "Modules/_blake2/impl/blake2-config.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "28b947b43bdc680b9f4335712bb2a5f2d5d32623" + }, + { + "algorithm": "SHA256", + "checksumValue": "4277092643b289f1d36d32cf0fd2efc30ead8bdd99342e5da3b3609dd8ea7d86" + } + ], + "fileName": "Modules/_blake2/impl/blake2-impl.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "caa3da7953109d0d2961e3b686d2d285c484b901" + }, + { + "algorithm": "SHA256", + "checksumValue": "2f6c9d0ecf70be474f2853b52394993625a32960e0a64eae147ef97a3a5c1460" + } + ], + "fileName": "Modules/_blake2/impl/blake2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "029a98f87a178936d9e5211c7798b3e0fc622f94" + }, + { + "algorithm": "SHA256", + "checksumValue": "b392a6e7b43813a05609e994db5fc3552c5912bd482efc781daa0778eb56ab4e" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-load-sse2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fb466dd72344170d09e311e5ea12de99ce071357" + }, + { + "algorithm": "SHA256", + "checksumValue": "cc3072c92164142bf2f9dda4e6c08db61be68ec15a95442415e861090d08f6a2" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-load-sse41.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c0d79128cf891a95b1f668031d55c0c6d2e0270" + }, + { + "algorithm": "SHA256", + "checksumValue": "07b257d44e9cc2d95d4911629c92138feafd16d63fef0a5fa7b38914dfd82349" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-ref.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c7418e2026417c9c6736fcd305a31f23e05a661" + }, + { + "algorithm": "SHA256", + "checksumValue": "fa34a60c2d198a0585033f43fd4003f4ba279c9ebcabdf5d6650def0e6d1e914" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-round.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f935d64cc633c38e09fc2d89281c95edfbc1fb05" + }, + { + "algorithm": "SHA256", + "checksumValue": "b932aa273b2504606a48895a50ff08c883f7a68a7e4aced5daa909c43348605a" + } + ], + "fileName": "Modules/_blake2/impl/blake2b.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ad3f79b6cbe3fd812722114a0d5d08064e69e4d0" + }, + { + "algorithm": "SHA256", + "checksumValue": "57f1ac6c09f4a50d95811529062220eab4f29cec3805bc6081dec00426c6df62" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-sse2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "51c32d79f419f3d2eb9875cd9a7f5c0d7892f8a8" + }, + { + "algorithm": "SHA256", + "checksumValue": "ecc9e09adcbe098629eafd305596bed8d7004be1d83f326995def42bbde93b23" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-sse41.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2749a7ba0104b765d4f56f13faf70b6eb89cf203" + }, + { + "algorithm": "SHA256", + "checksumValue": "8bc95595cec4c50f5d70f2b330d3798de07cc784e8890791b3328890e602d5c5" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-xop.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "883fcfe85f9063819f21b1100296d1f9eb55bac1" + }, + { + "algorithm": "SHA256", + "checksumValue": "9715c00d0f11587a139b07fa26678e6d26e44d3d4910b96158d158da2b022bfb" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-ref.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5d9f69adda40ed163b287b9ed4cedb35b88f2daa" + }, + { + "algorithm": "SHA256", + "checksumValue": "65d90111c89c43bb18a9e1d1a4fdbd9f85bebd1ff00129335b85995d0f30ee8b" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-round.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "13ac5bb93578a7ee8f815b4e247e82c849992bbe" + }, + { + "algorithm": "SHA256", + "checksumValue": "25ec5dd5c79f916307358059fe9f633781f27df1c0e0962c4fcccdda1feb93a7" + } + ], + "fileName": "Modules/_blake2/impl/blake2s.c" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0fbc026a9771d9675e7094790b5b945334d3cb53" + }, + { + "algorithm": "SHA256", + "checksumValue": "1e77c01eec8f167ed10b754f153c0c743c8e5196ae9c81dffc08f129ab56dbfd" + } + ], + "fileName": "Lib/ctypes/macholib/__init__.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-dyld.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4a78ebd73ce4167c722689781a15fe0b4578e967" + }, + { + "algorithm": "SHA256", + "checksumValue": "eb8e7b17f1533bc3e86e23e8695f7a5e4b7a99ef1b1575d10af54f389161b655" + } + ], + "fileName": "Lib/ctypes/macholib/dyld.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-dylib.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f339420cc01bd01f8d0da19b6102f099075e8bcd" + }, + { + "algorithm": "SHA256", + "checksumValue": "f19ee056b18165cc6735efab0b4ca3508be9405b9646c38113316c15e8278a6f" + } + ], + "fileName": "Lib/ctypes/macholib/dylib.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-framework.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0b219f58467d7f193fa1de0c1b118485840d855b" + }, + { + "algorithm": "SHA256", + "checksumValue": "302439e40d9cbdd61b8b7cffd0b7e1278a6811b635044ee366a36e0d991f62da" + } + ], + "fileName": "Lib/ctypes/macholib/framework.py" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bda6e0bd6121f7069b420bdc0bc7c49414d948d1" + }, + { + "algorithm": "SHA256", + "checksumValue": "89926cd0fe6cfb33a2b5b7416c101e9b5d42b0d639d348e0871acf6ffc8258a3" + } + ], + "fileName": "Modules/_decimal/libmpdec/README.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "33757ce2ec0c93c1b5e03c45a495563a00e498ae" + }, + { + "algorithm": "SHA256", + "checksumValue": "ad498362c31a5b99ab19fce320ac540cf14c5c4ec09478f0ad3858da1428113d" + } + ], + "fileName": "Modules/_decimal/libmpdec/basearith.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bf03919412c068e6969e7ac48850f91bfcd3b2b1" + }, + { + "algorithm": "SHA256", + "checksumValue": "2eaac88a71b9bcf3144396c12dcfeced573e0e550a0050d75b9ed3903248596d" + } + ], + "fileName": "Modules/_decimal/libmpdec/basearith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c925b7f26754ae182aaa461d51802e8b6a2bb5e9" + }, + { + "algorithm": "SHA256", + "checksumValue": "007e38542ec8d9d8805fe243b5390d79211b9360e2797a20079e833e68ad9e45" + } + ], + "fileName": "Modules/_decimal/libmpdec/bench.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cb22686269685a53a17afdea9ed984714e399d9d" + }, + { + "algorithm": "SHA256", + "checksumValue": "1b9e892d4b268deea835ec8906f20a1e5d25e037b2e698edcd34315613f3608c" + } + ], + "fileName": "Modules/_decimal/libmpdec/bench_full.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fc91c2450cdf1e785d1347411662294c3945eb27" + }, + { + "algorithm": "SHA256", + "checksumValue": "ce7741e58ea761a24250c0bfa10058cec8c4fd220dca70a41de3927a2e4f5376" + } + ], + "fileName": "Modules/_decimal/libmpdec/bits.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7187c18916b0a546ec19b4fc4bec43d0d9fb5fc2" + }, + { + "algorithm": "SHA256", + "checksumValue": "cd430b8657cf8a616916e02f9bd5ca044d5fc19e69333f5d427e1fdb90b0864b" + } + ], + "fileName": "Modules/_decimal/libmpdec/constants.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "af9cbd016fb0ef0b30ced49c0aa4ce2ca3c20125" + }, + { + "algorithm": "SHA256", + "checksumValue": "19dc46df04abb7ee08e9a403f87c8aac8d4a077efcce314c597f8b73e22884f2" + } + ], + "fileName": "Modules/_decimal/libmpdec/constants.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "666162870230bebd3f2383020d908806fd03909e" + }, + { + "algorithm": "SHA256", + "checksumValue": "9a265d366f31894aad78bca7fcdc1457bc4a3aa3887ca231b7d78e41f79541c0" + } + ], + "fileName": "Modules/_decimal/libmpdec/context.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0545547a8b37b922fbe2574fbad8fc3bf16f1d33" + }, + { + "algorithm": "SHA256", + "checksumValue": "66fe27b9bb37039cad5be32b105ed509e5aefa15c1957a9058af8ee23cddc97a" + } + ], + "fileName": "Modules/_decimal/libmpdec/convolute.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "05ff0936c5bb08f40d460f5843004a1cc0751d9b" + }, + { + "algorithm": "SHA256", + "checksumValue": "c00d17450c2b8e1d7f1eb8a084f7e6a68f257a453f8701600e860bf357c531d7" + } + ], + "fileName": "Modules/_decimal/libmpdec/convolute.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fe8176849bc99a306332ba25caa4e91bfa3c6f7d" + }, + { + "algorithm": "SHA256", + "checksumValue": "1f4e65c44864c3e911a6e91f33adec76765293e90553459e3ebce35a58898dba" + } + ], + "fileName": "Modules/_decimal/libmpdec/crt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1930b9e0910014b3479aec4e940f02118d9e4a08" + }, + { + "algorithm": "SHA256", + "checksumValue": "7d31f1d0dd73b62964dab0f7a1724473bf87f1f95d8febf0b40c15430ae9a47c" + } + ], + "fileName": "Modules/_decimal/libmpdec/crt.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "415c51e7d7f517b6366bec2a809610d0d38ada14" + }, + { + "algorithm": "SHA256", + "checksumValue": "0a9fef8a374f55277e9f6000b7277bb037b9763c32b156c29950422b057498bd" + } + ], + "fileName": "Modules/_decimal/libmpdec/difradix2.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d8a998c3bee4c3d9059ba7bf9ae6a8b64649c2ba" + }, + { + "algorithm": "SHA256", + "checksumValue": "5c6766496224de657400995b58b64db3e7084004bf00daebdd7e08d0c5995243" + } + ], + "fileName": "Modules/_decimal/libmpdec/difradix2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "158f6ad18edf348efa4fdd7cf61114c77c1d22e9" + }, + { + "algorithm": "SHA256", + "checksumValue": "7b0da2758097a2688f06b3c7ca46b2ebc8329addbd28bb4f5fe95626cc81f8a9" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/README.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ef80ba26847287fb351ab0df0a78b5f08ba0b5b7" + }, + { + "algorithm": "SHA256", + "checksumValue": "452666ee4eb10a8cf0a926cb3bcf5e95b5c361fa129dbdfe27b654e6d640417e" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/compare.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6ca3a369b3d1e140fdc93c4fdbedb724f7daf969" + }, + { + "algorithm": "SHA256", + "checksumValue": "6d369f5a24d0bb1e7cb6a4f8b0e97a273260e7668c8a540a8fcc92e039f7af2e" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/div.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "3872a28b4f77e07e1760256067ea338a8dd183f8" + }, + { + "algorithm": "SHA256", + "checksumValue": "5db54bae75ac3d7fa12f1bb0f7ce1bf797df86a81030e8c3ce44d3b1f9b958b7" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/divmod.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "25dbc94fd4ee5dec21061d2d40dd5d0f88849cb1" + }, + { + "algorithm": "SHA256", + "checksumValue": "22ed39b18fa740a27aacfd29a7bb40066be24500ba49b9b1f24e2af1e039fcd9" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/multiply.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "13d3b7657dc2dc5000fea428f57963d520792ef7" + }, + { + "algorithm": "SHA256", + "checksumValue": "cd8c037649b3d4d6897c9acd2f92f3f9d5390433061d5e48623a5d526a3f4f9c" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/pow.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1f7e6c3d3e38df52bbcec0f5a180a8f328679618" + }, + { + "algorithm": "SHA256", + "checksumValue": "e29614b43abf1856b656a84d6b67c22cc5dc7af8cbae8ddc7acf17022220ee12" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/powmod.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0bd9ce89c7987d1109eb7b0c8f1f9a1298e1422e" + }, + { + "algorithm": "SHA256", + "checksumValue": "203f2dbf11d115580cb3c7c524ac6ccca2a7b31d89545db1b6263381b5de2b6a" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/shift.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b401ba0814e17c9164c0df26e01cc0a355382f46" + }, + { + "algorithm": "SHA256", + "checksumValue": "f3dc2ce321833bbd4b3d1d9ea6fa2e0bcc1bfe1e39abb8d55be53e46c33949db" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/sqrt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "060615ddef089a5a8f879a57e4968d920972a0e2" + }, + { + "algorithm": "SHA256", + "checksumValue": "a9f923524d53a9445769f27405375ec3d95fa804bb11db5ee249ae047f11cfce" + } + ], + "fileName": "Modules/_decimal/libmpdec/fnt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b205043ebeaf065b16505a299342a992654f19b0" + }, + { + "algorithm": "SHA256", + "checksumValue": "3b03e69adf78fde68c8f87d33595d557237581d33fc067e1039eed9e9f2cc44c" + } + ], + "fileName": "Modules/_decimal/libmpdec/fnt.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "702c27599b43280c94906235d7e1a74193ba701b" + }, + { + "algorithm": "SHA256", + "checksumValue": "cf2e69b946ec14b087e523c0ff606553070d13c23e851fb0ba1df51a728017e6" + } + ], + "fileName": "Modules/_decimal/libmpdec/fourstep.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ee5291c265ef1f5ae373bc243a4d96975eb3e7b5" + }, + { + "algorithm": "SHA256", + "checksumValue": "dbaced03b52d0f880c377b86c943bcb36f24d557c99a5e9732df3ad5debb5917" + } + ], + "fileName": "Modules/_decimal/libmpdec/fourstep.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9dcb50e3f9c3245972731be5da0b28e7583198d9" + }, + { + "algorithm": "SHA256", + "checksumValue": "7cac49fef5e9d952ec9390bf81c54d83f1b5da32fdf76091c2f0770ed943b7fe" + } + ], + "fileName": "Modules/_decimal/libmpdec/io.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "28c653cd40b1ce46575e41f5dbfda5f6dd0db4d1" + }, + { + "algorithm": "SHA256", + "checksumValue": "259eab89fe27914e0e39e61199094a357ac60d86b2aab613c909040ff64a4a0c" + } + ], + "fileName": "Modules/_decimal/libmpdec/io.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "218d1d7bedb335cd2c31eae89a15873c3139e13f" + }, + { + "algorithm": "SHA256", + "checksumValue": "a57e8bed93ded481ef264166aec2c49d1a7f3252f29a873ee41fff053cfd9c20" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/REFERENCES.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f67eab2431336cf6eeafb30cdafd7e54c251def3" + }, + { + "algorithm": "SHA256", + "checksumValue": "dc34aa122c208ce79e3fc6baee8628094ffaf6a662862dd5647836241f6ebd79" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/bignum.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "a58cfbcd8ea57d66ddfd11fb5a170138c8bbfb3a" + }, + { + "algorithm": "SHA256", + "checksumValue": "122de20eebf87274af2d02072251a94500e7df2d5ef29e81aeabeda991c079e3" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/fnt.py" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9a947f6b660150cbd457c4458da2956a36c5824d" + }, + { + "algorithm": "SHA256", + "checksumValue": "592659e7192e3a939b797f5bc7455455834a285f5d8b643ccd780b5114914f73" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/matrix-transform.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "69fe9afb8353b5a2b57917469c51c64ac518169d" + }, + { + "algorithm": "SHA256", + "checksumValue": "229a80ca940c594a32e3345412370cbc097043fe59c66a6153cbcf01e7837266" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/mulmod-64.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "720d468a1f51098036c7a0c869810fff22ed9b79" + }, + { + "algorithm": "SHA256", + "checksumValue": "f3549fc73f697a087267c7b042e30a409e191cbba69a2c0902685e507fbae9f7" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/mulmod-ppro.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6815ec3a39baebebe7b3f51d45d10c180a659f17" + }, + { + "algorithm": "SHA256", + "checksumValue": "bf15f73910a173c98fca9db56122b6cc71983668fa8b934c46ca21a57398ec54" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/six-step.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c91ac4438e661ce78f86e981257546e5adff39ae" + }, + { + "algorithm": "SHA256", + "checksumValue": "783a1b4b9b7143677b0c3d30ffaf28aa0cb01956409031fa38ed8011970bdee0" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/umodarith.lisp" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7e8dfb4b7a801b48c501969b001153203b14679e" + }, + { + "algorithm": "SHA256", + "checksumValue": "5ba2f4c80302e71fb216aa247c858e0bf6c8cfabffe7980ac17d4d023c0fef2b" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpalloc.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bccb6a6ae76fd7f6c8a9102a78958bcad7862950" + }, + { + "algorithm": "SHA256", + "checksumValue": "f7412521de38afb837fcabc2b1d48b971b86bfaa55f3f40d58ff9e46e92debd3" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpalloc.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f4539afb1ace58c52d18ffd0cc7704f53ca55182" + }, + { + "algorithm": "SHA256", + "checksumValue": "4f89b8095e408a18deff79cfb605299e615bae747898eb105d8936064f7fb626" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpdecimal.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4b80e25ac49b7e1ea0d1e84967ee32a3d111fc4c" + }, + { + "algorithm": "SHA256", + "checksumValue": "ea0b9c6b296c13aed6ecaa50b463e39a9c1bdc059b84f50507fd8247b2e660f9" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpdecimal.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5c7305a6db0fddf64c6d97e29d3b0c402e3d5d6e" + }, + { + "algorithm": "SHA256", + "checksumValue": "653171cf2549719478417db7e9800fa0f9d99c02dec6da6876329ccf2c07b93f" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpsignal.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d736b874c43777ca021dde5289ea718893f39219" + }, + { + "algorithm": "SHA256", + "checksumValue": "bdbf2e246f341a3ba3f6f9d8759e7cb222eb9b15f9ed1e7c9f6a59cbb9f8bc91" + } + ], + "fileName": "Modules/_decimal/libmpdec/numbertheory.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d341508d8c6dd4c4cbd8b99afc8029945f9bbe0d" + }, + { + "algorithm": "SHA256", + "checksumValue": "2f7d5b40af508fa6ac86f5d62101fa3bf683c63b24aa87c9548e3fdd13abc57b" + } + ], + "fileName": "Modules/_decimal/libmpdec/numbertheory.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cbd05d68bb3940d0d7d0818b14cc03b090a4dd74" + }, + { + "algorithm": "SHA256", + "checksumValue": "7602aaf98ec9525bc4b3cab9631615e1be2efd9af894002ef4e3f5ec63924fcf" + } + ], + "fileName": "Modules/_decimal/libmpdec/sixstep.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c059463ec4b4522562dab24760fc64c172c9eee" + }, + { + "algorithm": "SHA256", + "checksumValue": "a191366348b3d3dd49b9090ec5c77dbd77bb3a523c01ff32adafa137e5097ce7" + } + ], + "fileName": "Modules/_decimal/libmpdec/sixstep.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cc5593ac9fdb60480cc23fc9d6f27d85670bd35f" + }, + { + "algorithm": "SHA256", + "checksumValue": "2d12fcae512143a9376c8a0d4c1ba3008e420e024497a7e7ec64c6bec23fcddc" + } + ], + "fileName": "Modules/_decimal/libmpdec/transpose.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2f616425756b6cbdf7d189744870b98b613455bd" + }, + { + "algorithm": "SHA256", + "checksumValue": "fafeb2b901b2b41bf0df00be7d99b84df1a78e3cc1e582e09cbfc3b6d44d4abe" + } + ], + "fileName": "Modules/_decimal/libmpdec/transpose.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b1e9341e173cc8e219ad4aa45fad36d92cce10d3" + }, + { + "algorithm": "SHA256", + "checksumValue": "25e0a0703b51744277834e6b2398d7b7d2c17f92bf30f8b6f949e0486ae2b346" + } + ], + "fileName": "Modules/_decimal/libmpdec/typearith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "46f6483fce136cd3cc2f7516ee119a487d86333e" + }, + { + "algorithm": "SHA256", + "checksumValue": "bfe1ddb2ca92906456b80745adcbe02c83cadac3ef69caa21bc09b7292cc152b" + } + ], + "fileName": "Modules/_decimal/libmpdec/umodarith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d0cc1052fcba08b773d935b0ae2dc6b80d0f2f68" + }, + { + "algorithm": "SHA256", + "checksumValue": "aacc3e47ea8f41e8840c6c67f64ec96d54696a16889903098fa1aab56949a00f" + } + ], + "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" + } + ], + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-expat", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d4cf38d26e21a56654ffe4acd9cd5481164619626802328506a2869afab29ab3" + } + ], + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_2/expat-2.6.2.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.2:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "expat", + "originator": "Organization: Expat development team", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.6.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-hacl-star", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e31e4ca10da91c585793c0eaf1b98aee3cb43e3a58d3d8d478593e5a6bd82927" + } + ], + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/bb3d0dc8d9d15a5cd51094d5b69e70aa09005ff0.zip", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:bb3d0dc8d9d15a5cd51094d5b69e70aa09005ff0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "hacl-star", + "originator": "Organization: HACL* Developers", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "bb3d0dc8d9d15a5cd51094d5b69e70aa09005ff0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libb2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "53626fddce753c454a3fea581cbbc7fe9bbcf0bc70416d48fdbbf5d87ef6c72e" + } + ], + "downloadLocation": "https://github.com/BLAKE2/libb2/releases/download/v0.98.1/libb2-0.98.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:blake2:libb2:0.98.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "libb2", + "originator": "Organization: BLAKE2 - fast secure hashing", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.98.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-macholib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "c76f268f5054024e962f2515a0e522baf85313064f6740d80375afc850787a38" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/ec/57/f0a712efc3ed982cf4038a3cee172057303b9be914c32c93b2fbec27f785/macholib-1.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/macholib@1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "macholib", + "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9f9cd4c041f99b5c49ffb7b59d9f12d95b683d88585608aa56a6307667b2b21f" + } + ], + "downloadLocation": "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.5.1" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-ascii.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-asciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-external.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-internal.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-nametab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-siphash.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-utf8tab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-winconfig.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlparse.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-dyld.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-dylib.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-framework.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 9d66b92eb8edf0..73012193d61485 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -78,6 +78,10 @@ [feature_macro.Py_REF_DEBUG] doc = 'when Python is compiled in debug mode (with Py_REF_DEBUG)' windows = 'maybe' +[feature_macro.Py_TRACE_REFS] + # nb. This mode is not compatible with Stable ABI/Limited API. + doc = 'when Python is compiled with Py_TRACE_REFS' + windows = 'maybe' # Mentioned in PEP 384: @@ -698,7 +702,6 @@ added = '3.2' [function.PyEval_InitThreads] added = '3.2' - abi_only = true [function.PyEval_ReleaseLock] added = '3.2' abi_only = true @@ -1333,10 +1336,8 @@ added = '3.2' [function.PySys_SetArgv] added = '3.2' - abi_only = true [function.PySys_SetArgvEx] added = '3.2' - abi_only = true [function.PySys_SetObject] added = '3.2' [function.PySys_SetPath] @@ -1669,10 +1670,8 @@ added = '3.2' [function.Py_SetProgramName] added = '3.2' - abi_only = true [function.Py_SetPythonHome] added = '3.2' - abi_only = true [function.Py_SetRecursionLimit] added = '3.2' [function.Py_VaBuildValue] @@ -2466,3 +2465,46 @@ added = '3.13' [function.PyUnicode_EqualToUTF8AndSize] added = '3.13' +[function.PyMem_RawMalloc] + added = '3.13' +[function.PyMem_RawCalloc] + added = '3.13' +[function.PyMem_RawRealloc] + added = '3.13' +[function.PyMem_RawFree] + added = '3.13' +[function.PySys_Audit] + added = '3.13' +[function.PySys_AuditTuple] + added = '3.13' +[function._Py_SetRefcnt] + added = '3.13' + abi_only = true +[function.PyList_GetItemRef] + added = '3.13' +[typedef.PyCFunctionFast] + added = '3.13' + # "abi-only" since 3.10. (Callback type names aren't used in C code, + # but this function signature was expected with METH_FASTCALL.) +[typedef.PyCFunctionFastWithKeywords] + added = '3.13' + # "abi-only" since 3.10. (Same story as PyCFunctionFast.) +[function.PyType_GetFullyQualifiedName] + added = '3.13' +[function.PyType_GetModuleName] + added = '3.13' +[function.Py_GetConstant] + added = '3.13' +[function.Py_GetConstantBorrowed] + added = '3.13' +[function.PyType_GetModuleByDef] + added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13' +[function.PyEval_GetFrameGlobals] + added = '3.13' +[function.PyEval_GetFrameLocals] + added = '3.13' + +[function.Py_TYPE] + added = '3.14' diff --git a/Misc/valgrind-python.supp b/Misc/valgrind-python.supp index 29954c1bbad350..8b2027cd452767 100644 --- a/Misc/valgrind-python.supp +++ b/Misc/valgrind-python.supp @@ -95,6 +95,49 @@ fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING } +# +# Leaks: dlopen() called without dlclose() +# + +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_load_cache_lookup +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_map_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_check_map_versions +} + + # # Non-python specific leaks # diff --git a/Modules/Setup b/Modules/Setup index 1367f0ef4fa54a..e4acf6bc7de8ea 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -137,6 +137,9 @@ PYTHONPATH=$(COREPYTHONPATH) #_datetime _datetimemodule.c #_decimal _decimal/_decimal.c #_heapq _heapqmodule.c +#_interpchannels _interpchannelsmodule.c +#_interpqueues _interpqueuesmodule.c +#_interpreters _interpretersmodule.c #_json _json.c #_lsprof _lsprof.c rotatingtree.c #_multiprocessing -I$(srcdir)/Modules/_multiprocessing _multiprocessing/multiprocessing.c _multiprocessing/semaphore.c @@ -271,8 +274,6 @@ PYTHONPATH=$(COREPYTHONPATH) # Testing -#_xxsubinterpreters _xxsubinterpretersmodule.c -#_xxinterpchannels _xxinterpchannelsmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c #_testbuffer _testbuffer.c #_testinternalcapi _testinternalcapi.c @@ -284,6 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c +#_testexternalinspection _testexternalinspection.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index cd12c1bd0df8f9..4dcc0f55176d0e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -11,6 +11,7 @@ faulthandler faulthandler.c posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c +_suggestions _suggestions.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c @@ -29,6 +30,7 @@ _weakref _weakref.c _abc _abc.c _functools _functoolsmodule.c _locale _localemodule.c +_opcode _opcode.c _operator _operator.c _stat _stat.c _symtable symtablemodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 647f44280b9ea1..dfc75077650df8 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -36,13 +36,16 @@ @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c -@MODULE__OPCODE_TRUE@_opcode _opcode.c @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c @MODULE__STRUCT_TRUE@_struct _struct.c -@MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c -@MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c + +# build supports subinterpreters +@MODULE__INTERPRETERS_TRUE@_interpreters _interpretersmodule.c +@MODULE__INTERPCHANNELS_TRUE@_interpchannels _interpchannelsmodule.c +@MODULE__INTERPQUEUES_TRUE@_interpqueues _interpqueuesmodule.c + @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c # needs libm @@ -158,8 +161,9 @@ @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c -@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c +@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -167,7 +171,8 @@ *shared* @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c -@MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTSINGLEPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_abc.c b/Modules/_abc.c index 9473905243d438..4f4b24b035db4a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -8,7 +8,6 @@ #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_typeobject.h" // _PyType_GetMRO() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "clinic/_abc.c.h" @@ -22,7 +21,7 @@ PyDoc_STRVAR(_abc__doc__, typedef struct { PyTypeObject *_abc_data_type; - unsigned long long abc_invalidation_counter; + uint64_t abc_invalidation_counter; } _abcmodule_state; static inline _abcmodule_state* @@ -33,17 +32,61 @@ get_abc_state(PyObject *module) return (_abcmodule_state *)state; } +static inline uint64_t +get_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&state->abc_invalidation_counter); +#else + return state->abc_invalidation_counter; +#endif +} + +static inline void +increment_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_add_uint64(&state->abc_invalidation_counter, 1); +#else + state->abc_invalidation_counter++; +#endif +} + /* This object stores internal state for ABCs. Note that we can use normal sets for caches, since they are never iterated over. */ typedef struct { PyObject_HEAD + /* These sets of weak references are lazily created. Once created, they + will point to the same sets until the ABCMeta object is destroyed or + cleared, both of which will only happen while the object is visible to a + single thread. */ PyObject *_abc_registry; - PyObject *_abc_cache; /* Normal set of weak references. */ - PyObject *_abc_negative_cache; /* Normal set of weak references. */ - unsigned long long _abc_negative_cache_version; + PyObject *_abc_cache; + PyObject *_abc_negative_cache; + uint64_t _abc_negative_cache_version; } _abc_data; +static inline uint64_t +get_cache_version(_abc_data *impl) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&impl->_abc_negative_cache_version); +#else + return impl->_abc_negative_cache_version; +#endif +} + +static inline void +set_cache_version(_abc_data *impl, uint64_t version) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uint64(&impl->_abc_negative_cache_version, version); +#else + impl->_abc_negative_cache_version = version; +#endif +} + static int abc_data_traverse(_abc_data *self, visitproc visit, void *arg) { @@ -91,7 +134,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->_abc_registry = NULL; self->_abc_cache = NULL; self->_abc_negative_cache = NULL; - self->_abc_negative_cache_version = state->abc_invalidation_counter; + self->_abc_negative_cache_version = get_invalidation_counter(state); return (PyObject *) self; } @@ -131,8 +174,12 @@ _get_impl(PyObject *module, PyObject *self) } static int -_in_weak_set(PyObject *set, PyObject *obj) +_in_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + Py_END_CRITICAL_SECTION(); if (set == NULL || PySet_GET_SIZE(set) == 0) { return 0; } @@ -169,16 +216,19 @@ static PyMethodDef _destroy_def = { }; static int -_add_to_weak_set(PyObject **pset, PyObject *obj) +_add_to_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { - if (*pset == NULL) { - *pset = PySet_New(NULL); - if (*pset == NULL) { - return -1; - } + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + if (set == NULL) { + set = *pset = PySet_New(NULL); + } + Py_END_CRITICAL_SECTION(); + if (set == NULL) { + return -1; } - PyObject *set = *pset; PyObject *ref, *wr; PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); @@ -221,7 +271,11 @@ _abc__reset_registry(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_registry != NULL && PySet_Clear(impl->_abc_registry) < 0) { + PyObject *registry; + Py_BEGIN_CRITICAL_SECTION(impl); + registry = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry != NULL && PySet_Clear(registry) < 0) { Py_DECREF(impl); return NULL; } @@ -248,13 +302,17 @@ _abc__reset_caches(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_cache != NULL && PySet_Clear(impl->_abc_cache) < 0) { + PyObject *cache, *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + cache = impl->_abc_cache; + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (cache != NULL && PySet_Clear(cache) < 0) { Py_DECREF(impl); return NULL; } /* also the second cache */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) { + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { Py_DECREF(impl); return NULL; } @@ -283,11 +341,14 @@ _abc__get_dump(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - PyObject *res = Py_BuildValue("NNNK", - PySet_New(impl->_abc_registry), - PySet_New(impl->_abc_cache), - PySet_New(impl->_abc_negative_cache), - impl->_abc_negative_cache_version); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(impl); + res = Py_BuildValue("NNNK", + PySet_New(impl->_abc_registry), + PySet_New(impl->_abc_cache), + PySet_New(impl->_abc_negative_cache), + get_cache_version(impl)); + Py_END_CRITICAL_SECTION(); Py_DECREF(impl); return res; } @@ -454,56 +515,27 @@ _abc__abc_init(PyObject *module, PyObject *self) if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; PyObject *dict = _PyType_GetDict(cls); - PyObject *flags = PyDict_GetItemWithError(dict, - &_Py_ID(__abc_tpflags__)); - if (flags == NULL) { - if (PyErr_Occurred()) { - return NULL; - } + PyObject *flags = NULL; + if (PyDict_Pop(dict, &_Py_ID(__abc_tpflags__), &flags) < 0) { + return NULL; } - else { - if (PyLong_CheckExact(flags)) { - long val = PyLong_AsLong(flags); - if (val == -1 && PyErr_Occurred()) { - return NULL; - } - if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { - PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); - return NULL; - } - ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); - } - if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { - return NULL; - } + if (flags == NULL || !PyLong_CheckExact(flags)) { + Py_XDECREF(flags); + Py_RETURN_NONE; } - } - Py_RETURN_NONE; -} - -static void -set_collection_flag_recursive(PyTypeObject *child, unsigned long flag) -{ - assert(flag == Py_TPFLAGS_MAPPING || flag == Py_TPFLAGS_SEQUENCE); - if (PyType_HasFeature(child, Py_TPFLAGS_IMMUTABLETYPE) || - (child->tp_flags & COLLECTION_FLAGS) == flag) - { - return; - } - - child->tp_flags &= ~COLLECTION_FLAGS; - child->tp_flags |= flag; - PyObject *grandchildren = _PyType_GetSubclasses(child); - if (grandchildren == NULL) { - return; - } - - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(grandchildren); i++) { - PyObject *grandchild = PyList_GET_ITEM(grandchildren, i); - set_collection_flag_recursive((PyTypeObject *)grandchild, flag); + long val = PyLong_AsLong(flags); + Py_DECREF(flags); + if (val == -1 && PyErr_Occurred()) { + return NULL; + } + if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { + PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); + return NULL; + } + _PyType_SetFlags((PyTypeObject *)self, 0, val & COLLECTION_FLAGS); } - Py_DECREF(grandchildren); + Py_RETURN_NONE; } /*[clinic input] @@ -546,20 +578,23 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) if (impl == NULL) { return NULL; } - if (_add_to_weak_set(&impl->_abc_registry, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_registry, subclass) < 0) { Py_DECREF(impl); return NULL; } Py_DECREF(impl); /* Invalidate negative cache */ - get_abc_state(module)->abc_invalidation_counter++; + increment_invalidation_counter(get_abc_state(module)); - /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ + /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ if (PyType_Check(self)) { - unsigned long collection_flag = ((PyTypeObject *)self)->tp_flags & COLLECTION_FLAGS; + unsigned long collection_flag = + PyType_GetFlags((PyTypeObject *)self) & COLLECTION_FLAGS; if (collection_flag) { - set_collection_flag_recursive((PyTypeObject *)subclass, collection_flag); + _PyType_SetFlagsRecursive((PyTypeObject *)subclass, + COLLECTION_FLAGS, + collection_flag); } } return Py_NewRef(subclass); @@ -593,7 +628,7 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, return NULL; } /* Inline the cache checking. */ - int incache = _in_weak_set(impl->_abc_cache, subclass); + int incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -603,8 +638,8 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - if (impl->_abc_negative_cache_version == get_abc_state(module)->abc_invalidation_counter) { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + if (get_cache_version(impl) == get_invalidation_counter(get_abc_state(module))) { + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -682,7 +717,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* 1. Check cache. */ - incache = _in_weak_set(impl->_abc_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -693,17 +728,20 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, state = get_abc_state(module); /* 2. Check negative cache; may have to invalidate. */ - if (impl->_abc_negative_cache_version < state->abc_invalidation_counter) { + uint64_t invalidation_counter = get_invalidation_counter(state); + if (get_cache_version(impl) < invalidation_counter) { /* Invalidate the negative cache. */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) - { + PyObject *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { goto end; } - impl->_abc_negative_cache_version = state->abc_invalidation_counter; + set_cache_version(impl, invalidation_counter); } else { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -721,7 +759,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -729,7 +767,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -744,18 +782,12 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - assert(mro_item != NULL); - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; + if (PyType_IsSubtype((PyTypeObject *)subclass, (PyTypeObject *)self)) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } + result = Py_True; + goto end; } /* 5. Check if it's a subclass of a registered class (recursive). */ @@ -774,12 +806,14 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, goto end; } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - PyObject *scls = PyList_GET_ITEM(subclasses, pos); - Py_INCREF(scls); + PyObject *scls = PyList_GetItemRef(subclasses, pos); + if (scls == NULL) { + goto end; + } int r = PyObject_IsSubclass(subclass, scls); Py_DECREF(scls); if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -791,7 +825,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* No dice; update negative cache. */ - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -808,7 +842,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result) { // Fast path: check subclass is in weakref directly. - int ret = _in_weak_set(impl->_abc_registry, subclass); + int ret = _in_weak_set(impl, &impl->_abc_registry, subclass); if (ret < 0) { *result = NULL; return -1; @@ -818,33 +852,27 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 1; } - if (impl->_abc_registry == NULL) { + PyObject *registry_shared; + Py_BEGIN_CRITICAL_SECTION(impl); + registry_shared = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry_shared == NULL) { return 0; } - Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); - if (registry_size == 0) { - return 0; - } - // Weakref callback may remove entry from set. - // So we take snapshot of registry first. - PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); - if (copy == NULL) { - PyErr_NoMemory(); + + // Make a local copy of the registry to protect against concurrent + // modifications of _abc_registry. + PyObject *registry = PyFrozenSet_New(registry_shared); + if (registry == NULL) { return -1; } PyObject *key; Py_ssize_t pos = 0; Py_hash_t hash; - Py_ssize_t i = 0; - while (_PySet_NextEntry(impl->_abc_registry, &pos, &key, &hash)) { - copy[i++] = Py_NewRef(key); - } - assert(i == registry_size); - - for (i = 0; i < registry_size; i++) { + while (_PySet_NextEntry(registry, &pos, &key, &hash)) { PyObject *rkey; - if (PyWeakref_GetRef(copy[i], &rkey) < 0) { + if (PyWeakref_GetRef(key, &rkey) < 0) { // Someone inject non-weakref type in the registry. ret = -1; break; @@ -860,7 +888,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, break; } if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { ret = -1; break; } @@ -870,10 +898,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, } } - for (i = 0; i < registry_size; i++) { - Py_DECREF(copy[i]); - } - PyMem_Free(copy); + Py_DECREF(registry); return ret; } @@ -892,7 +917,7 @@ _abc_get_cache_token_impl(PyObject *module) /*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ { _abcmodule_state *state = get_abc_state(module); - return PyLong_FromUnsignedLongLong(state->abc_invalidation_counter); + return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); } static struct PyMethodDef _abcmodule_methods[] = { @@ -945,6 +970,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e911286660b56e..5824c470699d88 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -4,7 +4,9 @@ #include "Python.h" #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _Py_SetImmortalUntracked #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -18,13 +20,78 @@ module _asyncio [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +typedef enum { + STATE_PENDING, + STATE_CANCELLED, + STATE_FINISHED +} fut_state; + +#define FutureObj_HEAD(prefix) \ + PyObject_HEAD \ + PyObject *prefix##_loop; \ + PyObject *prefix##_callback0; \ + PyObject *prefix##_context0; \ + PyObject *prefix##_callbacks; \ + PyObject *prefix##_exception; \ + PyObject *prefix##_exception_tb; \ + PyObject *prefix##_result; \ + PyObject *prefix##_source_tb; \ + PyObject *prefix##_cancel_msg; \ + PyObject *prefix##_cancelled_exc; \ + fut_state prefix##_state; \ + /* These bitfields need to be at the end of the struct + so that these and bitfields from TaskObj are contiguous. + */ \ + unsigned prefix##_log_tb: 1; \ + unsigned prefix##_blocking: 1; + +typedef struct { + FutureObj_HEAD(fut) +} FutureObj; + +typedef struct TaskObj { + FutureObj_HEAD(task) + unsigned task_must_cancel: 1; + unsigned task_log_destroy_pending: 1; + int task_num_cancels_requested; + PyObject *task_fut_waiter; + PyObject *task_coro; + PyObject *task_name; + PyObject *task_context; + struct TaskObj *next; + struct TaskObj *prev; +} TaskObj; + +typedef struct { + PyObject_HEAD + TaskObj *sw_task; + PyObject *sw_arg; +} TaskStepMethWrapper; + + +#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) +#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) + +#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) +#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) #define FI_FREELIST_MAXLEN 255 +#ifdef Py_GIL_DISABLED +# define ASYNCIO_STATE_LOCK(state) PyMutex_Lock(&state->mutex) +# define ASYNCIO_STATE_UNLOCK(state) PyMutex_Unlock(&state->mutex) +#else +# define ASYNCIO_STATE_LOCK(state) ((void)state) +# define ASYNCIO_STATE_UNLOCK(state) ((void)state) +#endif + typedef struct futureiterobject futureiterobject; /* State of the _asyncio module */ typedef struct { +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif PyTypeObject *FutureIterType; PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; @@ -37,8 +104,9 @@ typedef struct { all running event loops. {EventLoop: Task} */ PyObject *current_tasks; - /* WeakSet containing all tasks scheduled to run on event loops. */ - PyObject *scheduled_tasks; + /* WeakSet containing scheduled 3rd party tasks which don't + inherit from native asyncio.Task */ + PyObject *non_asyncio_tasks; /* Set containing all eagerly executing tasks. */ PyObject *eager_tasks; @@ -75,6 +143,51 @@ typedef struct { futureiterobject *fi_freelist; Py_ssize_t fi_freelist_len; + + /* Linked-list of all tasks which are instances of asyncio.Task or subclasses + of it. Third party tasks implementations which don't inherit from + asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. + `tail` is used as a sentinel to mark the end of the linked-list. It avoids one + branch in checking for empty list when adding a new task, the list is + initialized with `head` pointing to `tail` to mark an empty list. + + Invariants: + * When the list is empty: + - asyncio_tasks.head == &asyncio_tasks.tail + - asyncio_tasks.head->prev == NULL + - asyncio_tasks.head->next == NULL + + * After adding the first task 'task1': + - asyncio_tasks.head == task1 + - task1->next == &asyncio_tasks.tail + - task1->prev == NULL + - asyncio_tasks.tail.prev == task1 + + * After adding a second task 'task2': + - asyncio_tasks.head == task2 + - task2->next == task1 + - task2->prev == NULL + - task1->prev == task2 + - asyncio_tasks.tail.prev == task1 + + * After removing task 'task1': + - asyncio_tasks.head == task2 + - task2->next == &asyncio_tasks.tail + - task2->prev == NULL + - asyncio_tasks.tail.prev == task2 + + * After removing task 'task2', the list is empty: + - asyncio_tasks.head == &asyncio_tasks.tail + - asyncio_tasks.head->prev == NULL + - asyncio_tasks.tail.prev == NULL + - asyncio_tasks.tail.next == NULL + */ + + struct { + TaskObj tail; + TaskObj *head; + } asyncio_tasks; + } asyncio_state; static inline asyncio_state * @@ -104,59 +217,6 @@ get_asyncio_state_by_def(PyObject *self) return get_asyncio_state(mod); } -typedef enum { - STATE_PENDING, - STATE_CANCELLED, - STATE_FINISHED -} fut_state; - -#define FutureObj_HEAD(prefix) \ - PyObject_HEAD \ - PyObject *prefix##_loop; \ - PyObject *prefix##_callback0; \ - PyObject *prefix##_context0; \ - PyObject *prefix##_callbacks; \ - PyObject *prefix##_exception; \ - PyObject *prefix##_exception_tb; \ - PyObject *prefix##_result; \ - PyObject *prefix##_source_tb; \ - PyObject *prefix##_cancel_msg; \ - PyObject *prefix##_cancelled_exc; \ - fut_state prefix##_state; \ - /* These bitfields need to be at the end of the struct - so that these and bitfields from TaskObj are contiguous. - */ \ - unsigned prefix##_log_tb: 1; \ - unsigned prefix##_blocking: 1; - -typedef struct { - FutureObj_HEAD(fut) -} FutureObj; - -typedef struct { - FutureObj_HEAD(task) - unsigned task_must_cancel: 1; - unsigned task_log_destroy_pending: 1; - int task_num_cancels_requested; - PyObject *task_fut_waiter; - PyObject *task_coro; - PyObject *task_name; - PyObject *task_context; -} TaskObj; - -typedef struct { - PyObject_HEAD - TaskObj *sw_task; - PyObject *sw_arg; -} TaskStepMethWrapper; - - -#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) -#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) - -#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) -#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) - #include "clinic/_asynciomodule.c.h" @@ -292,6 +352,8 @@ get_running_loop(asyncio_state *state, PyObject **loop) } } + // TODO GH-121621: This should be moved to PyThreadState + // for easier and quicker access. state->cached_running_loop = rl; state->cached_running_loop_tsid = ts_id; } @@ -335,6 +397,9 @@ set_running_loop(asyncio_state *state, PyObject *loop) return -1; } + + // TODO GH-121621: This should be moved to PyThreadState + // for easier and quicker access. state->cached_running_loop = loop; // borrowed, kept alive by ts_dict state->cached_running_loop_tsid = PyThreadState_GetID(tstate); @@ -596,12 +661,27 @@ future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) PyErr_SetString(PyExc_TypeError, "invalid exception object"); return NULL; } - if (Py_IS_TYPE(exc_val, (PyTypeObject *)PyExc_StopIteration)) { + if (PyErr_GivenExceptionMatches(exc_val, PyExc_StopIteration)) { + const char *msg = "StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future"; + PyObject *message = PyUnicode_FromString(msg); + if (message == NULL) { + Py_DECREF(exc_val); + return NULL; + } + PyObject *err = PyObject_CallOneArg(PyExc_RuntimeError, message); + Py_DECREF(message); + if (err == NULL) { + Py_DECREF(exc_val); + return NULL; + } + assert(PyExceptionInstance_Check(err)); + + PyException_SetCause(err, Py_NewRef(exc_val)); + PyException_SetContext(err, Py_NewRef(exc_val)); Py_DECREF(exc_val); - PyErr_SetString(PyExc_TypeError, - "StopIteration interacts badly with generators " - "and cannot be raised into a Future"); - return NULL; + exc_val = err; } assert(!fut->fut_exception); @@ -1585,11 +1665,23 @@ static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); - asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); + + assert(_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + + PyObject *module = ((PyHeapTypeObject*)tp)->ht_module; + asyncio_state *state = NULL; + PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { + // GH-115874: We can't use PyType_GetModuleByDef here as the type might have + // already been cleared, which is also why we must check if ht_module != NULL. + if (module && _PyModule_GetDef(module) == &_asynciomodule) { + state = get_asyncio_state(module); + } + + // TODO GH-121621: This should be moved to thread state as well. + if (state && state->fi_freelist_len < FI_FREELIST_MAXLEN) { state->fi_freelist_len++; it->future = (FutureObj*) state->fi_freelist; state->fi_freelist = it; @@ -1937,16 +2029,24 @@ static PyMethodDef TaskWakeupDef = { /* ----- Task introspection helpers */ -static int -register_task(asyncio_state *state, PyObject *task) -{ - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(add), task); - if (res == NULL) { - return -1; +static void +register_task(asyncio_state *state, TaskObj *task) +{ + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + assert(task != &state->asyncio_tasks.tail); + if (task->next != NULL) { + // already registered + ASYNCIO_STATE_UNLOCK(state); + return; } - Py_DECREF(res); - return 0; + assert(task->prev == NULL); + assert(state->asyncio_tasks.head != NULL); + + task->next = state->asyncio_tasks.head; + state->asyncio_tasks.head->prev = task; + state->asyncio_tasks.head = task; + ASYNCIO_STATE_UNLOCK(state); } static int @@ -1955,16 +2055,30 @@ register_eager_task(asyncio_state *state, PyObject *task) return PySet_Add(state->eager_tasks, task); } -static int -unregister_task(asyncio_state *state, PyObject *task) -{ - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(discard), task); - if (res == NULL) { - return -1; +static void +unregister_task(asyncio_state *state, TaskObj *task) +{ + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + assert(task != &state->asyncio_tasks.tail); + if (task->next == NULL) { + // not registered + assert(task->prev == NULL); + assert(state->asyncio_tasks.head != task); + ASYNCIO_STATE_UNLOCK(state); + return; } - Py_DECREF(res); - return 0; + task->next->prev = task->prev; + if (task->prev == NULL) { + assert(state->asyncio_tasks.head == task); + state->asyncio_tasks.head = task->next; + } else { + task->prev->next = task->next; + } + task->next = NULL; + task->prev = NULL; + assert(state->asyncio_tasks.head != task); + ASYNCIO_STATE_UNLOCK(state); } static int @@ -2029,12 +2143,22 @@ static PyObject * swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *prev_task; + + if (task == Py_None) { + if (PyDict_Pop(state->current_tasks, loop, &prev_task) < 0) { + return NULL; + } + if (prev_task == NULL) { + Py_RETURN_NONE; + } + return prev_task; + } + Py_hash_t hash; hash = PyObject_Hash(loop); if (hash == -1) { return NULL; } - prev_task = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (prev_task == NULL) { if (PyErr_Occurred()) { @@ -2043,22 +2167,12 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) prev_task = Py_None; } Py_INCREF(prev_task); - - if (task == Py_None) { - if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { - goto error; - } - } else { - if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { - goto error; - } + if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { + Py_DECREF(prev_task); + return NULL; } return prev_task; - -error: - Py_DECREF(prev_task); - return NULL; } /* ----- Task */ @@ -2119,7 +2233,12 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, // optimization: defer task name formatting // store the task counter as PyLong in the name // for deferred formatting in get_name - name = PyLong_FromUnsignedLongLong(++state->task_name_counter); +#ifdef Py_GIL_DISABLED + unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; +#else + unsigned long long counter = ++state->task_name_counter; +#endif + name = PyLong_FromUnsignedLongLong(counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2148,7 +2267,8 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (task_call_step_soon(state, self, NULL)) { return -1; } - return register_task(state, (PyObject*)self); + register_task(state, self); + return 0; } static int @@ -2377,6 +2497,9 @@ _asyncio_Task_uncancel_impl(TaskObj *self) { if (self->task_num_cancels_requested > 0) { self->task_num_cancels_requested -= 1; + if (self->task_num_cancels_requested == 0) { + self->task_must_cancel = 0; + } } return PyLong_FromLong(self->task_num_cancels_requested); } @@ -2553,6 +2676,15 @@ _asyncio_Task_set_name(TaskObj *self, PyObject *value) static void TaskObj_finalize(TaskObj *task) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + // Unregister the task from the linked list of tasks. + // Since task is a native task, we directly call the + // unregister_task function. Third party event loops + // should use the asyncio._unregister_task function. + // See https://docs.python.org/3/library/asyncio-extending.html#task-lifetime-support + + unregister_task(state, task); + PyObject *context; PyObject *message = NULL; PyObject *func; @@ -2753,7 +2885,6 @@ gen_status_from_result(PyObject **result) static PyObject * task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { - int res; int clear_exc = 0; PyObject *result = NULL; PyObject *coro; @@ -2761,7 +2892,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_state != STATE_PENDING) { PyErr_Format(state->asyncio_InvalidStateError, - "_step(): already done: %R %R", + "__step(): already done: %R %R", task, exc ? exc : Py_None); goto fail; @@ -2770,20 +2901,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_must_cancel) { assert(exc != Py_None); - if (exc) { - /* Check if exc is a CancelledError */ - res = PyObject_IsInstance(exc, state->asyncio_CancelledError); - if (res == -1) { - /* An error occurred, abort */ - goto fail; - } - if (res == 0) { - /* exc is not CancelledError; reset it to NULL */ - exc = NULL; - } - } - - if (!exc) { + if (!exc || !PyErr_GivenExceptionMatches(exc, state->asyncio_CancelledError)) { /* exc was not a CancelledError */ exc = create_cancelled_error(state, (FutureObj*)task); @@ -3178,9 +3296,7 @@ task_eager_start(asyncio_state *state, TaskObj *task) } if (task->task_state == STATE_PENDING) { - if (register_task(state, (PyObject *)task) == -1) { - retval = -1; - } + register_task(state, task); } else { // This seems to really help performance on pyperformance benchmarks Py_CLEAR(task->task_coro); @@ -3346,9 +3462,20 @@ _asyncio__register_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=8672dadd69a7d4e2 input=21075aaea14dfbad]*/ { asyncio_state *state = get_asyncio_state(module); - if (register_task(state, task) < 0) { + if (Task_Check(state, task)) { + // task is an asyncio.Task instance or subclass, use efficient + // linked-list implementation. + register_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + // As task does not inherit from asyncio.Task, fallback to less efficient + // weakset implementation. + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(add), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3389,9 +3516,16 @@ _asyncio__unregister_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=6e5585706d568a46 input=28fb98c3975f7bdc]*/ { asyncio_state *state = get_asyncio_state(module); - if (unregister_task(state, task) < 0) { + if (Task_Check(state, task)) { + unregister_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(discard), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3513,21 +3647,126 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) Py_INCREF(loop); } - ret = PyDict_GetItemWithError(state->current_tasks, loop); + int rc = PyDict_GetItemRef(state->current_tasks, loop, &ret); Py_DECREF(loop); - if (ret == NULL && PyErr_Occurred()) { - return NULL; - } - else if (ret == NULL) { + if (rc == 0) { Py_RETURN_NONE; } - Py_INCREF(ret); return ret; } +static inline int +add_one_task(asyncio_state *state, PyObject *tasks, PyObject *task, PyObject *loop) +{ + PyObject *done = PyObject_CallMethodNoArgs(task, &_Py_ID(done)); + if (done == NULL) { + return -1; + } + if (Py_IsTrue(done)) { + return 0; + } + Py_DECREF(done); + PyObject *task_loop = get_future_loop(state, task); + if (task_loop == NULL) { + return -1; + } + if (task_loop == loop) { + if (PySet_Add(tasks, task) < 0) { + Py_DECREF(task_loop); + return -1; + } + } + Py_DECREF(task_loop); + return 0; +} + /*********************** Module **************************/ +/*[clinic input] +_asyncio.all_tasks + + loop: object = None + +Return a set of all tasks for the loop. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop) +/*[clinic end generated code: output=0e107cbb7f72aa7b input=43a1b423c2d95bfa]*/ +{ + + asyncio_state *state = get_asyncio_state(module); + PyObject *tasks = PySet_New(NULL); + if (tasks == NULL) { + return NULL; + } + if (loop == Py_None) { + loop = _asyncio_get_running_loop_impl(module); + if (loop == NULL) { + Py_DECREF(tasks); + return NULL; + } + } else { + Py_INCREF(loop); + } + // First add eager tasks to the set so that we don't miss + // any tasks which graduates from eager to non-eager + PyObject *eager_iter = PyObject_GetIter(state->eager_tasks); + if (eager_iter == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(eager_iter)) != NULL) { + if (add_one_task(state, tasks, item, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(item); + Py_DECREF(eager_iter); + return NULL; + } + Py_DECREF(item); + } + Py_DECREF(eager_iter); + TaskObj *head = state->asyncio_tasks.head; + Py_INCREF(head); + assert(head != NULL); + assert(head->prev == NULL); + TaskObj *tail = &state->asyncio_tasks.tail; + while (head != tail) + { + if (add_one_task(state, tasks, (PyObject *)head, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(head); + return NULL; + } + Py_INCREF(head->next); + Py_SETREF(head, head->next); + } + PyObject *scheduled_iter = PyObject_GetIter(state->non_asyncio_tasks); + if (scheduled_iter == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + while ((item = PyIter_Next(scheduled_iter)) != NULL) { + if (add_one_task(state, tasks, item, loop) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(item); + Py_DECREF(scheduled_iter); + return NULL; + } + Py_DECREF(item); + } + Py_DECREF(scheduled_iter); + Py_DECREF(loop); + return tasks; +} static void module_free_freelists(asyncio_state *state) @@ -3569,7 +3808,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_InvalidStateError); Py_VISIT(state->asyncio_CancelledError); - Py_VISIT(state->scheduled_tasks); + Py_VISIT(state->non_asyncio_tasks); Py_VISIT(state->eager_tasks); Py_VISIT(state->current_tasks); Py_VISIT(state->iscoroutine_typecache); @@ -3607,7 +3846,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_InvalidStateError); Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(state->scheduled_tasks); + Py_CLEAR(state->non_asyncio_tasks); Py_CLEAR(state->eager_tasks); Py_CLEAR(state->current_tasks); Py_CLEAR(state->iscoroutine_typecache); @@ -3688,9 +3927,9 @@ module_init(asyncio_state *state) PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - state->scheduled_tasks = PyObject_CallNoArgs(weak_set); + state->non_asyncio_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (state->scheduled_tasks == NULL) { + if (state->non_asyncio_tasks == NULL) { goto fail; } @@ -3725,6 +3964,7 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__ENTER_TASK_METHODDEF _ASYNCIO__LEAVE_TASK_METHODDEF _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF + _ASYNCIO_ALL_TASKS_METHODDEF {NULL, NULL} }; @@ -3732,6 +3972,9 @@ static int module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); + Py_SET_TYPE(&state->asyncio_tasks.tail, state->TaskType); + _Py_SetImmortalUntracked((PyObject *)&state->asyncio_tasks.tail); + state->asyncio_tasks.head = &state->asyncio_tasks.tail; #define CREATE_TYPE(m, tp, spec, base) \ do { \ @@ -3761,7 +4004,7 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->scheduled_tasks) < 0) { + if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->non_asyncio_tasks) < 0) { return -1; } @@ -3780,6 +4023,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9e0fd336419b44..56322c48b7cd35 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -462,6 +462,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index c2cac98c7529eb..0c3ae5a2fac275 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -17,6 +17,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() @@ -42,7 +43,8 @@ typedef struct { PyObject_HEAD blake2b_param param; blake2b_state state; - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; } BLAKE2bObject; #include "clinic/blake2b_impl.c.h" @@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type) { BLAKE2bObject *self; self = (BLAKE2bObject *)type->tp_alloc(type, 0); - if (self != NULL) { - self->lock = NULL; + if (self == NULL) { + return NULL; } + HASHLIB_INIT_MUTEX(self); + return self; } @@ -278,18 +282,19 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) - self->lock = PyThread_allocate_lock(); - - if (self->lock != NULL) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - blake2b_update(&self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + blake2b_update(&self->state, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS } else { blake2b_update(&self->state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -389,10 +394,6 @@ py_blake2b_dealloc(PyObject *self) /* Try not to leave state in memory. */ secure_zero_memory(&obj->param, sizeof(obj->param)); secure_zero_memory(&obj->state, sizeof(obj->state)); - if (obj->lock) { - PyThread_free_lock(obj->lock); - obj->lock = NULL; - } PyTypeObject *type = Py_TYPE(self); PyObject_Free(self); diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 0d1d88c6603684..78242214764f2b 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -74,6 +74,12 @@ _blake2_free(void *module) Py_DECREF(x); \ } while(0) +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + static int blake2_exec(PyObject *m) { @@ -95,10 +101,10 @@ blake2_exec(PyObject *m) ADD_INT(d, "MAX_KEY_SIZE", BLAKE2B_KEYBYTES); ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); + ADD_INT_CONST("BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); + ADD_INT_CONST("BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); + ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); + ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); /* BLAKE2s */ st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( @@ -117,17 +123,21 @@ blake2_exec(PyObject *m) ADD_INT(d, "MAX_KEY_SIZE", BLAKE2S_KEYBYTES); ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); + ADD_INT_CONST("BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); + ADD_INT_CONST("BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); + ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); + ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); return 0; } +#undef ADD_INT +#undef ADD_INT_CONST + static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 1c47328ece13e8..3014773ab52331 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -17,6 +17,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() @@ -42,7 +43,8 @@ typedef struct { PyObject_HEAD blake2s_param param; blake2s_state state; - PyThread_type_lock lock; + bool use_mutex; + PyMutex mutex; } BLAKE2sObject; #include "clinic/blake2s_impl.c.h" @@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type) { BLAKE2sObject *self; self = (BLAKE2sObject *)type->tp_alloc(type, 0); - if (self != NULL) { - self->lock = NULL; + if (self == NULL) { + return NULL; } + HASHLIB_INIT_MUTEX(self); + return self; } @@ -278,18 +282,19 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) - self->lock = PyThread_allocate_lock(); - - if (self->lock != NULL) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - blake2s_update(&self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + blake2s_update(&self->state, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS } else { blake2s_update(&self->state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -389,10 +394,6 @@ py_blake2s_dealloc(PyObject *self) /* Try not to leave state in memory. */ secure_zero_memory(&obj->param, sizeof(obj->param)); secure_zero_memory(&obj->state, sizeof(obj->state)); - if (obj->lock) { - PyThread_free_lock(obj->lock); - obj->lock = NULL; - } PyTypeObject *type = Py_TYPE(self); PyObject_Free(self); diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 869cbea78dd267..47d62717eb76e7 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_blake2b_new__doc__, "blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" @@ -97,10 +98,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'key'", "contiguous buffer", fastargs[2]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -109,10 +106,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'salt'", "contiguous buffer", fastargs[3]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -121,10 +114,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&person, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'person'", "contiguous buffer", fastargs[4]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -276,4 +265,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=2ad807e0c83d8c25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e18eeaee40623bfc input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index affc203486b174..7a0f6eeff5b5b5 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_blake2s_new__doc__, "blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" @@ -97,10 +98,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'key'", "contiguous buffer", fastargs[2]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -109,10 +106,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'salt'", "contiguous buffer", fastargs[3]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -121,10 +114,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&person, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'person'", "contiguous buffer", fastargs[4]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -276,4 +265,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=90ca2b52b8c40785 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=24690e4e2586cafd input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/impl/blake2b.c b/Modules/_blake2/impl/blake2b.c index c1068e8640546a..cef22838917d9d 100644 --- a/Modules/_blake2/impl/blake2b.c +++ b/Modules/_blake2/impl/blake2b.c @@ -27,7 +27,7 @@ #if defined(HAVE_SSE2) #include // MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) +#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) { return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); diff --git a/Modules/_blake2/impl/blake2s.c b/Modules/_blake2/impl/blake2s.c index 47514685b8f30b..e7f63fd274f212 100644 --- a/Modules/_blake2/impl/blake2s.c +++ b/Modules/_blake2/impl/blake2s.c @@ -27,7 +27,7 @@ #if defined(HAVE_SSE2) #include // MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) +#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) { return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 3d0d4ee5e79c2b..661847ad26702e 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -802,6 +802,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index c31c1b6d6f2bbc..32373f0799bfeb 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1050,6 +1050,7 @@ static PyMethodDef _codecs_functions[] = { static PyModuleDef_Slot _codecs_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c8cd53de5e2262..fbfed59995c21e 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -3,6 +3,7 @@ #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() #include @@ -44,8 +45,11 @@ find_module_state_by_def(PyTypeObject *type) /*[clinic input] module _collections class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" +class _collections.deque "dequeobject *" "clinic_state()->deque_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7356042a89862e0e]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a033cc2a8476b3f1]*/ + +typedef struct dequeobject dequeobject; /* We can safely assume type to be the defining class, * since tuplegetter is not a base type */ @@ -53,6 +57,12 @@ class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type" #include "clinic/_collectionsmodule.c.h" #undef clinic_state +/*[python input] +class dequeobject_converter(self_converter): + type = "dequeobject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b6ae4a3ff852be2f]*/ + /* collections module implementation of a deque() datatype Written and maintained by Raymond D. Hettinger */ @@ -121,7 +131,7 @@ typedef struct BLOCK { struct BLOCK *rightlink; } block; -typedef struct { +struct dequeobject { PyObject_VAR_HEAD block *leftblock; block *rightblock; @@ -132,7 +142,7 @@ typedef struct { Py_ssize_t numfreeblocks; block *freeblocks[MAXFREEBLOCKS]; PyObject *weakreflist; -} dequeobject; +}; /* For debug builds, add error checking to track the endpoints * in the chain of links. The goal is to make sure that link @@ -219,8 +229,18 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)deque; } +/*[clinic input] +@critical_section +_collections.deque.pop as deque_pop + + deque: dequeobject + +Remove and return the rightmost element. +[clinic start generated code]*/ + static PyObject * -deque_pop(dequeobject *deque, PyObject *unused) +deque_pop_impl(dequeobject *deque) +/*[clinic end generated code: output=2e5f7890c4251f07 input=55c5b6a8ad51d72f]*/ { PyObject *item; block *prevblock; @@ -254,10 +274,18 @@ deque_pop(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); +/*[clinic input] +@critical_section +_collections.deque.popleft as deque_popleft + + deque: dequeobject + +Remove and return the leftmost element. +[clinic start generated code]*/ static PyObject * -deque_popleft(dequeobject *deque, PyObject *unused) +deque_popleft_impl(dequeobject *deque) +/*[clinic end generated code: output=62b154897097ff68 input=1571ce88fe3053de]*/ { PyObject *item; block *prevblock; @@ -292,8 +320,6 @@ deque_popleft(dequeobject *deque, PyObject *unused) return item; } -PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); - /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * @@ -309,7 +335,7 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) static inline int -deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_append_lock_held(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(deque); @@ -326,7 +352,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); + PyObject *olditem = deque_popleft_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -334,18 +360,29 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +@critical_section +_collections.deque.append as deque_append + + deque: dequeobject + item: object + / + +Add an element to the right side of the deque. +[clinic start generated code]*/ + static PyObject * -deque_append(dequeobject *deque, PyObject *item) +deque_append_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9c7bcb8b599c6362 input=b0eeeb09b9f5cf18]*/ { - if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_append_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); - static inline int -deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) +deque_appendleft_lock_held(dequeobject *deque, PyObject *item, + Py_ssize_t maxlen) { if (deque->leftindex == 0) { block *b = newblock(deque); @@ -361,8 +398,8 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) Py_SET_SIZE(deque, Py_SIZE(deque) + 1); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); + if (NEEDS_TRIM(deque, maxlen)) { + PyObject *olditem = deque_pop_impl(deque); Py_DECREF(olditem); } else { deque->state++; @@ -370,16 +407,26 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) return 0; } +/*[clinic input] +@critical_section +_collections.deque.appendleft as deque_appendleft + + deque: dequeobject + item: object + / + +Add an element to the left side of the deque. +[clinic start generated code]*/ + static PyObject * -deque_appendleft(dequeobject *deque, PyObject *item) +deque_appendleft_impl(dequeobject *deque, PyObject *item) +/*[clinic end generated code: output=9a192edbcd0f20db input=236c2fbceaf08e14]*/ { - if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) + if (deque_appendleft_lock_held(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); - static PyObject* finalize_iterator(PyObject *it) { @@ -410,8 +457,20 @@ consume_iterator(PyObject *it) return finalize_iterator(it); } +/*[clinic input] +@critical_section +_collections.deque.extend as deque_extend + + deque: dequeobject + iterable: object + / + +Extend the right side of the deque with elements from the iterable. +[clinic start generated code]*/ + static PyObject * -deque_extend(dequeobject *deque, PyObject *iterable) +deque_extend_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=8b5ffa57ce82d980 input=85861954127c81da]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -445,7 +504,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_append_internal(deque, item, maxlen) == -1) { + if (deque_append_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -454,11 +513,20 @@ deque_extend(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extend_doc, -"Extend the right side of the deque with elements from the iterable"); +/*[clinic input] +@critical_section +_collections.deque.extendleft as deque_extendleft + + deque: dequeobject + iterable: object + / + +Extend the left side of the deque with elements from the iterable. +[clinic start generated code]*/ static PyObject * -deque_extendleft(dequeobject *deque, PyObject *iterable) +deque_extendleft_impl(dequeobject *deque, PyObject *iterable) +/*[clinic end generated code: output=ba44191aa8e35a26 input=640dabd086115689]*/ { PyObject *it, *item; PyObject *(*iternext)(PyObject *); @@ -470,7 +538,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) PyObject *s = PySequence_List(iterable); if (s == NULL) return NULL; - result = deque_extendleft(deque, s); + result = deque_extendleft_impl(deque, s); Py_DECREF(s); return result; } @@ -492,7 +560,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque_appendleft_internal(deque, item, maxlen) == -1) { + if (deque_appendleft_lock_held(deque, item, maxlen) == -1) { Py_DECREF(item); Py_DECREF(it); return NULL; @@ -501,14 +569,12 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return finalize_iterator(it); } -PyDoc_STRVAR(extendleft_doc, -"Extend the left side of the deque with elements from the iterable"); - static PyObject * deque_inplace_concat(dequeobject *deque, PyObject *other) { PyObject *result; + // deque_extend is thread-safe result = deque_extend(deque, other); if (result == NULL) return result; @@ -517,8 +583,18 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } +/*[clinic input] +@critical_section +_collections.deque.copy as deque_copy + + deque: dequeobject + +Return a shallow copy of a deque. +[clinic start generated code]*/ + static PyObject * -deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) +deque_copy_impl(dequeobject *deque) +/*[clinic end generated code: output=6409b3d1ad2898b5 input=51d2ed1a23bab5e2]*/ { PyObject *result; dequeobject *old_deque = (dequeobject *)deque; @@ -532,12 +608,16 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) if (new_deque == NULL) return NULL; new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + /* Fast path for the deque_repeat() common case where len(deque) == 1 + * + * It's safe to not acquire the per-object lock for new_deque; it's + * invisible to other threads. + */ if (Py_SIZE(deque) == 1) { PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append(new_deque, item); + rv = deque_append_impl(new_deque, item); } else { - rv = deque_extend(new_deque, deque); + rv = deque_extend_impl(new_deque, (PyObject *)deque); } if (rv != NULL) { Py_DECREF(rv); @@ -547,7 +627,8 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } if (old_deque->maxlen < 0) - result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), deque); + result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), + (PyObject *)deque); else result = PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", deque, old_deque->maxlen, NULL); @@ -561,10 +642,22 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) return result; } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); +/*[clinic input] +@critical_section +_collections.deque.__copy__ as deque___copy__ = _collections.deque.copy + +Return a shallow copy of a deque. +[clinic start generated code]*/ static PyObject * -deque_concat(dequeobject *deque, PyObject *other) +deque___copy___impl(dequeobject *deque) +/*[clinic end generated code: output=7c5821504342bf23 input=f5464036f9686a55]*/ +{ + return deque_copy_impl(deque); +} + +static PyObject * +deque_concat_lock_held(dequeobject *deque, PyObject *other) { PyObject *new_deque, *result; int rv; @@ -580,10 +673,13 @@ deque_concat(dequeobject *deque, PyObject *other) return NULL; } - new_deque = deque_copy((PyObject *)deque, NULL); + new_deque = deque_copy_impl(deque); if (new_deque == NULL) return NULL; - result = deque_extend((dequeobject *)new_deque, other); + + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + result = deque_extend_impl((dequeobject *)new_deque, other); if (result == NULL) { Py_DECREF(new_deque); return NULL; @@ -592,6 +688,16 @@ deque_concat(dequeobject *deque, PyObject *other) return new_deque; } +static PyObject * +deque_concat(dequeobject *deque, PyObject *other) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_concat_lock_held(deque, other); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_clear(dequeobject *deque) { @@ -669,24 +775,32 @@ deque_clear(dequeobject *deque) alternate_method: while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); + item = deque_pop_impl(deque); assert (item != NULL); Py_DECREF(item); } return 0; } +/*[clinic input] +@critical_section +_collections.deque.clear as deque_clearmethod + + deque: dequeobject + +Remove all elements from the deque. +[clinic start generated code]*/ + static PyObject * -deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_clearmethod_impl(dequeobject *deque) +/*[clinic end generated code: output=79b2513e097615c1 input=3a22e9605d20c5e9]*/ { deque_clear(deque); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static PyObject * -deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +deque_inplace_repeat_lock_held(dequeobject *deque, Py_ssize_t n) { Py_ssize_t i, m, size; PyObject *seq; @@ -750,7 +864,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) n = (deque->maxlen + size - 1) / size; for (i = 0 ; i < n-1 ; i++) { - rv = deque_extend(deque, seq); + rv = deque_extend_impl(deque, seq); if (rv == NULL) { Py_DECREF(seq); return NULL; @@ -762,16 +876,30 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } +static PyObject * +deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_inplace_repeat_lock_held(deque, n); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * deque_repeat(dequeobject *deque, Py_ssize_t n) { dequeobject *new_deque; PyObject *rv; - new_deque = (dequeobject *)deque_copy((PyObject *) deque, NULL); + Py_BEGIN_CRITICAL_SECTION(deque); + new_deque = (dequeobject *)deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); if (new_deque == NULL) return NULL; - rv = deque_inplace_repeat(new_deque, n); + // It's safe to not acquire the per-object lock for new_deque; it's + // invisible to other threads. + rv = deque_inplace_repeat_lock_held(new_deque, n); Py_DECREF(new_deque); return rv; } @@ -925,36 +1053,38 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) return rv; } -static PyObject * -deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) -{ - Py_ssize_t n=1; +/*[clinic input] +@critical_section +_collections.deque.rotate as deque_rotate - if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) { - return NULL; - } - if (nargs) { - PyObject *index = _PyNumber_Index(args[0]); - if (index == NULL) { - return NULL; - } - n = PyLong_AsSsize_t(index); - Py_DECREF(index); - if (n == -1 && PyErr_Occurred()) { - return NULL; - } - } + deque: dequeobject + n: Py_ssize_t = 1 + / + +Rotate the deque n steps to the right. If n is negative, rotates left. +[clinic start generated code]*/ +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n) +/*[clinic end generated code: output=96c2402a371eb15d input=5bf834296246e002]*/ +{ if (!_deque_rotate(deque, n)) Py_RETURN_NONE; return NULL; } -PyDoc_STRVAR(rotate_doc, -"Rotate the deque n steps to the right (default n=1). If n is negative, rotates left."); +/*[clinic input] +@critical_section +_collections.deque.reverse as deque_reverse + + deque: dequeobject + +Reverse *IN PLACE*. +[clinic start generated code]*/ static PyObject * -deque_reverse(dequeobject *deque, PyObject *unused) +deque_reverse_impl(dequeobject *deque) +/*[clinic end generated code: output=bdeebc2cf8c1f064 input=26f4167fd623027f]*/ { block *leftblock = deque->leftblock; block *rightblock = deque->rightblock; @@ -991,11 +1121,20 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_RETURN_NONE; } -PyDoc_STRVAR(reverse_doc, -"D.reverse() -- reverse *IN PLACE*"); +/*[clinic input] +@critical_section +_collections.deque.count as deque_count + + deque: dequeobject + value as v: object + / + +Return number of occurrences of value. +[clinic start generated code]*/ static PyObject * -deque_count(dequeobject *deque, PyObject *v) +deque_count_impl(dequeobject *deque, PyObject *v) +/*[clinic end generated code: output=2ca26c49b6ab0400 input=4ef67ef2b34dc1fc]*/ { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1030,11 +1169,8 @@ deque_count(dequeobject *deque, PyObject *v) return PyLong_FromSsize_t(count); } -PyDoc_STRVAR(count_doc, -"D.count(value) -- return number of occurrences of value"); - static int -deque_contains(dequeobject *deque, PyObject *v) +deque_contains_lock_held(dequeobject *deque, PyObject *v) { block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; @@ -1065,28 +1201,50 @@ deque_contains(dequeobject *deque, PyObject *v) return 0; } +static int +deque_contains(dequeobject *deque, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_contains_lock_held(deque, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t deque_len(dequeobject *deque) { - return Py_SIZE(deque); + return FT_ATOMIC_LOAD_SSIZE(((PyVarObject *)deque)->ob_size); } +/*[clinic input] +@critical_section +@text_signature "($self, value, [start, [stop]])" +_collections.deque.index as deque_index + + deque: dequeobject + value as v: object + start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + / + +Return first index of value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + static PyObject * -deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop) +/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ { - Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); - PyObject *v, *item; + Py_ssize_t i, n; + PyObject *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; - if (!_PyArg_ParseStack(args, nargs, "O|O&O&:index", &v, - _PyEval_SliceIndexNotNone, &start, - _PyEval_SliceIndexNotNone, &stop)) { - return NULL; - } - if (start < 0) { start += Py_SIZE(deque); if (start < 0) @@ -1117,8 +1275,9 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) n = stop - i; while (--n >= 0) { CHECK_NOT_END(b); - item = b->data[index]; + item = Py_NewRef(b->data[index]); cmp = PyObject_RichCompareBool(item, v, Py_EQ); + Py_DECREF(item); if (cmp > 0) return PyLong_FromSsize_t(stop - n - 1); if (cmp < 0) @@ -1134,14 +1293,10 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) index = 0; } } - PyErr_Format(PyExc_ValueError, "%R is not in deque", v); + PyErr_SetString(PyExc_ValueError, "deque.index(x): x not in deque"); return NULL; } -PyDoc_STRVAR(index_doc, -"D.index(value, [start, [stop]]) -- return first index of value.\n" -"Raises ValueError if the value is not present."); - /* insert(), remove(), and delitem() are implemented in terms of rotate() for simplicity and reasonable performance near the end points. If for some reason these methods become popular, it is not @@ -1150,32 +1305,39 @@ PyDoc_STRVAR(index_doc, boost (by moving each pointer only once instead of twice). */ +/*[clinic input] +@critical_section +_collections.deque.insert as deque_insert + + deque: dequeobject + index: Py_ssize_t + value: object + / + +Insert value before index. +[clinic start generated code]*/ + static PyObject * -deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value) +/*[clinic end generated code: output=ef4d2c15d5532b80 input=dbee706586cc9cde]*/ { - Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); - PyObject *value; PyObject *rv; - if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) { - return NULL; - } - if (deque->maxlen == Py_SIZE(deque)) { PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); return NULL; } if (index >= n) - return deque_append(deque, value); + return deque_append_impl(deque, value); if (index <= -n || index == 0) - return deque_appendleft(deque, value); + return deque_appendleft_impl(deque, value); if (_deque_rotate(deque, -index)) return NULL; if (index < 0) - rv = deque_append(deque, value); + rv = deque_append_impl(deque, value); else - rv = deque_appendleft(deque, value); + rv = deque_appendleft_impl(deque, value); if (rv == NULL) return NULL; Py_DECREF(rv); @@ -1184,12 +1346,6 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) Py_RETURN_NONE; } -PyDoc_STRVAR(insert_doc, -"D.insert(index, object) -- insert object before index"); - -PyDoc_STRVAR(remove_doc, -"D.remove(value) -- remove first occurrence of value."); - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1199,7 +1355,7 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) } static PyObject * -deque_item(dequeobject *deque, Py_ssize_t i) +deque_item_lock_held(dequeobject *deque, Py_ssize_t i) { block *b; PyObject *item; @@ -1237,6 +1393,16 @@ deque_item(dequeobject *deque, Py_ssize_t i) return Py_NewRef(item); } +static PyObject * +deque_item(dequeobject *deque, Py_ssize_t i) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_item_lock_held(deque, i); + Py_END_CRITICAL_SECTION(); + return result; +} + static int deque_del_item(dequeobject *deque, Py_ssize_t i) { @@ -1246,15 +1412,27 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) assert (i >= 0 && i < Py_SIZE(deque)); if (_deque_rotate(deque, -i)) return -1; - item = deque_popleft(deque, NULL); + item = deque_popleft_impl(deque); rv = _deque_rotate(deque, i); assert (item != NULL); Py_DECREF(item); return rv; } +/*[clinic input] +@critical_section +_collections.deque.remove as deque_remove + + deque: dequeobject + value: object + / + +Remove first occurrence of value. +[clinic start generated code]*/ + static PyObject * -deque_remove(dequeobject *deque, PyObject *value) +deque_remove_impl(dequeobject *deque, PyObject *value) +/*[clinic end generated code: output=54cff28b8ef78c5b input=60eb3f8aa4de532a]*/ { PyObject *item; block *b = deque->leftblock; @@ -1284,7 +1462,7 @@ deque_remove(dequeobject *deque, PyObject *value) } } if (i == n) { - PyErr_Format(PyExc_ValueError, "%R is not in deque", value); + PyErr_SetString(PyExc_ValueError, "deque.remove(x): x not in deque"); return NULL; } rv = deque_del_item(deque, i); @@ -1295,7 +1473,7 @@ deque_remove(dequeobject *deque, PyObject *value) } static int -deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +deque_ass_item_lock_held(dequeobject *deque, Py_ssize_t i, PyObject *v) { block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1326,6 +1504,16 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) return 0; } +static int +deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(deque); + result = deque_ass_item_lock_held(deque, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + static void deque_dealloc(dequeobject *deque) { @@ -1375,8 +1563,17 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) return 0; } +/*[clinic input] +_collections.deque.__reduce__ as deque___reduce__ + + deque: dequeobject + +Return state information for pickling. +[clinic start generated code]*/ + static PyObject * -deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reduce___impl(dequeobject *deque) +/*[clinic end generated code: output=cb85d9e0b7d2c5ad input=991a933a5bc7a526]*/ { PyObject *state, *it; @@ -1391,6 +1588,8 @@ deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored)) return NULL; } + // It's safe to access deque->maxlen here without holding the per object + // lock for deque; deque->maxlen is only assigned during construction. if (deque->maxlen < 0) { return Py_BuildValue("O()NN", Py_TYPE(deque), state, it); } @@ -1510,26 +1709,23 @@ deque_richcompare(PyObject *v, PyObject *w, int op) return NULL; } +/*[clinic input] +@critical_section +@text_signature "([iterable[, maxlen]])" +_collections.deque.__init__ as deque_init + + deque: dequeobject + iterable: object = NULL + maxlen as maxlenobj: object = NULL + +A list-like sequence optimized for data accesses near its endpoints. +[clinic start generated code]*/ + static int -deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj) +/*[clinic end generated code: output=7084a39d71218dcd input=2b9e37af1fd73143]*/ { - PyObject *iterable = NULL; - PyObject *maxlenobj = NULL; Py_ssize_t maxlen = -1; - char *kwlist[] = {"iterable", "maxlen", 0}; - - if (kwdargs == NULL && PyTuple_GET_SIZE(args) <= 2) { - if (PyTuple_GET_SIZE(args) > 0) { - iterable = PyTuple_GET_ITEM(args, 0); - } - if (PyTuple_GET_SIZE(args) > 1) { - maxlenobj = PyTuple_GET_ITEM(args, 1); - } - } else { - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, - &iterable, &maxlenobj)) - return -1; - } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1543,7 +1739,7 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) if (Py_SIZE(deque) > 0) deque_clear(deque); if (iterable != NULL) { - PyObject *rv = deque_extend(deque, iterable); + PyObject *rv = deque_extend_impl(deque, iterable); if (rv == NULL) return -1; Py_DECREF(rv); @@ -1551,8 +1747,18 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) return 0; } +/*[clinic input] +@critical_section +_collections.deque.__sizeof__ as deque___sizeof__ + + deque: dequeobject + +Return the size of the deque in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -deque_sizeof(dequeobject *deque, void *unused) +deque___sizeof___impl(dequeobject *deque) +/*[clinic end generated code: output=4d36e9fb4f30bbaf input=762312f2d4813535]*/ { size_t res = _PyObject_SIZE(Py_TYPE(deque)); size_t blocks; @@ -1563,9 +1769,6 @@ deque_sizeof(dequeobject *deque, void *unused) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, -"D.__sizeof__() -- size of D in memory, in bytes"); - static PyObject * deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) { @@ -1574,6 +1777,22 @@ deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored)) return PyLong_FromSsize_t(deque->maxlen); } +static PyObject *deque_reviter(dequeobject *deque); + +/*[clinic input] +_collections.deque.__reversed__ as deque___reversed__ + + deque: dequeobject + +Return a reverse iterator over the deque. +[clinic start generated code]*/ + +static PyObject * +deque___reversed___impl(dequeobject *deque) +/*[clinic end generated code: output=3e7e7e715883cf2e input=3d494c25a6fe5c7e]*/ +{ + return deque_reviter(deque); +} /* deque object ********************************************************/ @@ -1584,47 +1803,26 @@ static PyGetSetDef deque_getset[] = { }; static PyObject *deque_iter(dequeobject *deque); -static PyObject *deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)); -PyDoc_STRVAR(reversed_doc, - "D.__reversed__() -- return a reverse iterator over the deque"); static PyMethodDef deque_methods[] = { - {"append", (PyCFunction)deque_append, - METH_O, append_doc}, - {"appendleft", (PyCFunction)deque_appendleft, - METH_O, appendleft_doc}, - {"clear", (PyCFunction)deque_clearmethod, - METH_NOARGS, clear_doc}, - {"__copy__", deque_copy, - METH_NOARGS, copy_doc}, - {"copy", deque_copy, - METH_NOARGS, copy_doc}, - {"count", (PyCFunction)deque_count, - METH_O, count_doc}, - {"extend", (PyCFunction)deque_extend, - METH_O, extend_doc}, - {"extendleft", (PyCFunction)deque_extendleft, - METH_O, extendleft_doc}, - {"index", _PyCFunction_CAST(deque_index), - METH_FASTCALL, index_doc}, - {"insert", _PyCFunction_CAST(deque_insert), - METH_FASTCALL, insert_doc}, - {"pop", (PyCFunction)deque_pop, - METH_NOARGS, pop_doc}, - {"popleft", (PyCFunction)deque_popleft, - METH_NOARGS, popleft_doc}, - {"__reduce__", (PyCFunction)deque_reduce, - METH_NOARGS, reduce_doc}, - {"remove", (PyCFunction)deque_remove, - METH_O, remove_doc}, - {"__reversed__", (PyCFunction)deque_reviter, - METH_NOARGS, reversed_doc}, - {"reverse", (PyCFunction)deque_reverse, - METH_NOARGS, reverse_doc}, - {"rotate", _PyCFunction_CAST(deque_rotate), - METH_FASTCALL, rotate_doc}, - {"__sizeof__", (PyCFunction)deque_sizeof, - METH_NOARGS, sizeof_doc}, + DEQUE_APPEND_METHODDEF + DEQUE_APPENDLEFT_METHODDEF + DEQUE_CLEARMETHOD_METHODDEF + DEQUE___COPY___METHODDEF + DEQUE_COPY_METHODDEF + DEQUE_COUNT_METHODDEF + DEQUE_EXTEND_METHODDEF + DEQUE_EXTENDLEFT_METHODDEF + DEQUE_INDEX_METHODDEF + DEQUE_INSERT_METHODDEF + DEQUE_POP_METHODDEF + DEQUE_POPLEFT_METHODDEF + DEQUE___REDUCE___METHODDEF + DEQUE_REMOVE_METHODDEF + DEQUE___REVERSED___METHODDEF + DEQUE_REVERSE_METHODDEF + DEQUE_ROTATE_METHODDEF + DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -1635,17 +1833,12 @@ static PyMemberDef deque_members[] = { {NULL}, }; -PyDoc_STRVAR(deque_doc, -"deque([iterable[, maxlen]]) --> deque object\n\ -\n\ -A list-like sequence optimized for data accesses near its endpoints."); - static PyType_Slot deque_slots[] = { {Py_tp_dealloc, deque_dealloc}, {Py_tp_repr, deque_repr}, {Py_tp_hash, PyObject_HashNotImplemented}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_doc, (void *)deque_doc}, + {Py_tp_doc, (void *)deque_init__doc__}, {Py_tp_traverse, deque_traverse}, {Py_tp_clear, deque_clear}, {Py_tp_richcompare, deque_richcompare}, @@ -1699,11 +1892,13 @@ deque_iter(dequeobject *deque) it = PyObject_GC_New(dequeiterobject, state->dequeiter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->leftblock; it->index = deque->leftindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } @@ -1735,7 +1930,7 @@ dequeiter_dealloc(dequeiterobject *dio) } static PyObject * -dequeiter_next(dequeiterobject *it) +dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; @@ -1761,6 +1956,20 @@ dequeiter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequeiter_next(dequeiterobject *it) +{ + PyObject *result; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + result = dequeiter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + + return result; +} + static PyObject * dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1781,6 +1990,11 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; @@ -1794,7 +2008,8 @@ dequeiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * dequeiter_len(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromSsize_t(it->counter); + Py_ssize_t len = FT_ATOMIC_LOAD_SSIZE(it->counter); + return PyLong_FromSsize_t(len); } PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); @@ -1802,7 +2017,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * dequeiter_reduce(dequeiterobject *it, PyObject *Py_UNUSED(ignored)) { - return Py_BuildValue("O(On)", Py_TYPE(it), it->deque, Py_SIZE(it->deque) - it->counter); + PyTypeObject *ty = Py_TYPE(it); + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_ssize_t size, counter; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + size = Py_SIZE(deque); + counter = it->counter; + Py_END_CRITICAL_SECTION2(); + return Py_BuildValue("O(On)", ty, deque, size - counter); } static PyMethodDef dequeiter_methods[] = { @@ -1834,7 +2058,7 @@ static PyType_Spec dequeiter_spec = { /*********************** Deque Reverse Iterator **************************/ static PyObject * -deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_reviter(dequeobject *deque) { dequeiterobject *it; collections_state *state = find_module_state_by_def(Py_TYPE(deque)); @@ -1842,17 +2066,19 @@ deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored)) it = PyObject_GC_New(dequeiterobject, state->dequereviter_type); if (it == NULL) return NULL; + Py_BEGIN_CRITICAL_SECTION(deque); it->b = deque->rightblock; it->index = deque->rightindex; it->deque = (dequeobject*)Py_NewRef(deque); it->state = deque->state; it->counter = Py_SIZE(deque); + Py_END_CRITICAL_SECTION(); PyObject_GC_Track(it); return (PyObject *)it; } static PyObject * -dequereviter_next(dequeiterobject *it) +dequereviter_next_lock_held(dequeiterobject *it, dequeobject *deque) { PyObject *item; if (it->counter == 0) @@ -1878,6 +2104,19 @@ dequereviter_next(dequeiterobject *it) return Py_NewRef(item); } +static PyObject * +dequereviter_next(dequeiterobject *it) +{ + PyObject *item; + // It's safe to access it->deque without holding the per-object lock for it + // here; it->deque is only assigned during construction of it. + dequeobject *deque = it->deque; + Py_BEGIN_CRITICAL_SECTION2(it, deque); + item = dequereviter_next_lock_held(it, deque); + Py_END_CRITICAL_SECTION2(); + return item; +} + static PyObject * dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1889,7 +2128,7 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; assert(type == state->dequereviter_type); - it = (dequeiterobject*)deque_reviter((dequeobject *)deque, NULL); + it = (dequeiterobject *)deque_reviter((dequeobject *)deque); if (!it) return NULL; /* consume items from the queue */ @@ -1898,6 +2137,11 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (item) { Py_DECREF(item); } else { + /* + * It's safe to read directly from it without acquiring the + * per-object lock; the iterator isn't visible to any other threads + * yet. + */ if (it->counter) { Py_DECREF(it); return NULL; @@ -2267,9 +2511,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /* Only take the fast path when get() and __setitem__() * have not been overridden. */ - mapping_get = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(get)); + mapping_get = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(get)); dict_get = _PyType_Lookup(&PyDict_Type, &_Py_ID(get)); - mapping_setitem = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(__setitem__)); + mapping_setitem = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(__setitem__)); dict_setitem = _PyType_Lookup(&PyDict_Type, &_Py_ID(__setitem__)); if (mapping_get != NULL && mapping_get == dict_get && @@ -2293,12 +2537,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, if (key == NULL) break; - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) - goto done; + hash = _PyObject_HashFast(key); + if (hash == -1) { + goto done; } oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); @@ -2331,7 +2572,11 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, oldval = PyObject_CallFunctionObjArgs(bound_get, key, zero, NULL); if (oldval == NULL) break; - newval = PyNumber_Add(oldval, one); + if (oldval == zero) { + newval = Py_NewRef(one); + } else { + newval = PyNumber_Add(oldval, one); + } Py_DECREF(oldval); if (newval == NULL) break; @@ -2343,6 +2588,8 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, } done: + Py_XDECREF(mapping_get); + Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); Py_XDECREF(newval); @@ -2573,6 +2820,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_complex.h b/Modules/_complex.h new file mode 100644 index 00000000000000..28d4a32794b97c --- /dev/null +++ b/Modules/_complex.h @@ -0,0 +1,54 @@ +/* Workarounds for buggy complex number arithmetic implementations. */ + +#ifndef Py_HAVE_C_COMPLEX +# error "this header file should only be included if Py_HAVE_C_COMPLEX is defined" +#endif + +#include + +/* Other compilers (than clang), that claims to + implement C11 *and* define __STDC_IEC_559_COMPLEX__ - don't have + issue with CMPLX(). This is specific to glibc & clang combination: + https://sourceware.org/bugzilla/show_bug.cgi?id=26287 + + Here we fallback to using __builtin_complex(), available in clang + v12+. Else CMPLX implemented following C11 6.2.5p13: "Each complex type + has the same representation and alignment requirements as an array + type containing exactly two elements of the corresponding real type; + the first element is equal to the real part, and the second element + to the imaginary part, of the complex number. + */ +#if !defined(CMPLX) +# if defined(__clang__) && __has_builtin(__builtin_complex) +# define CMPLX(x, y) __builtin_complex ((double) (x), (double) (y)) +# define CMPLXF(x, y) __builtin_complex ((float) (x), (float) (y)) +# define CMPLXL(x, y) __builtin_complex ((long double) (x), (long double) (y)) +# else +static inline double complex +CMPLX(double real, double imag) +{ + double complex z; + ((double *)(&z))[0] = real; + ((double *)(&z))[1] = imag; + return z; +} + +static inline float complex +CMPLXF(float real, float imag) +{ + float complex z; + ((float *)(&z))[0] = real; + ((float *)(&z))[1] = imag; + return z; +} + +static inline long double complex +CMPLXL(long double real, long double imag) +{ + long double complex z; + ((long double *)(&z))[0] = real; + ((long double *)(&z))[1] = imag; + return z; +} +# endif +#endif diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index f621c1de6d42d6..3f96f07909b69a 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 9568334b8069c8..9964c383b8ad09 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -8,7 +8,10 @@ module instead. */ -#define MODULE_VERSION "1.0" +// clinic/_csv.c.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif #include "Python.h" @@ -128,7 +131,7 @@ typedef struct { Py_UCS4 *field; /* temporary buffer */ Py_ssize_t field_size; /* size of allocated buffer */ Py_ssize_t field_len; /* length of current field */ - int numeric_field; /* treat field as numeric */ + bool unquoted_field; /* true if no quotes around the current field */ unsigned long line_num; /* Source-file line number */ } ReaderObj; @@ -155,15 +158,9 @@ static PyObject * get_dialect_from_registry(PyObject *name_obj, _csvstate *module_state) { PyObject *dialect_obj; - - dialect_obj = PyDict_GetItemWithError(module_state->dialects, name_obj); - if (dialect_obj == NULL) { - if (!PyErr_Occurred()) - PyErr_Format(module_state->error_obj, "unknown dialect"); + if (PyDict_GetItemRef(module_state->dialects, name_obj, &dialect_obj) == 0) { + PyErr_SetString(module_state->error_obj, "unknown dialect"); } - else - Py_INCREF(dialect_obj); - return dialect_obj; } @@ -334,6 +331,33 @@ dialect_check_quoting(int quoting) return -1; } +static int +dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect, bool allowspace) +{ + if (c == '\r' || c == '\n' || (c == ' ' && !allowspace)) { + PyErr_Format(PyExc_ValueError, "bad %s value", name); + return -1; + } + if (PyUnicode_FindChar( + dialect->lineterminator, c, 0, + PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) + { + PyErr_Format(PyExc_ValueError, "bad %s or lineterminator value", name); + return -1; + } + return 0; +} + + static int +dialect_check_chars(const char *name1, const char *name2, Py_UCS4 c1, Py_UCS4 c2) +{ + if (c1 == c2 && c1 != NOT_SET) { + PyErr_Format(PyExc_ValueError, "bad %s or %s value", name1, name2); + return -1; + } + return 0; +} + #define D_OFF(x) offsetof(DialectObj, x) static struct PyMemberDef Dialect_memberlist[] = { @@ -511,6 +535,20 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } + if (dialect_check_char("delimiter", self->delimiter, self, true) || + dialect_check_char("escapechar", self->escapechar, self, + !self->skipinitialspace) || + dialect_check_char("quotechar", self->quotechar, self, + !self->skipinitialspace) || + dialect_check_chars("delimiter", "escapechar", + self->delimiter, self->escapechar) || + dialect_check_chars("delimiter", "quotechar", + self->delimiter, self->quotechar) || + dialect_check_chars("escapechar", "quotechar", + self->escapechar, self->quotechar)) + { + goto err; + } ret = Py_NewRef(self); err: @@ -608,22 +646,33 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs) static int parse_save_field(ReaderObj *self) { + int quoting = self->dialect->quoting; PyObject *field; - field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - (void *) self->field, self->field_len); - if (field == NULL) - return -1; - self->field_len = 0; - if (self->numeric_field) { - PyObject *tmp; - - self->numeric_field = 0; - tmp = PyNumber_Float(field); - Py_DECREF(field); - if (tmp == NULL) + if (self->unquoted_field && + self->field_len == 0 && + (quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS)) + { + field = Py_NewRef(Py_None); + } + else { + field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->field, self->field_len); + if (field == NULL) { return -1; - field = tmp; + } + if (self->unquoted_field && + self->field_len != 0 && + (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS)) + { + PyObject *tmp = PyNumber_Float(field); + Py_DECREF(field); + if (tmp == NULL) { + return -1; + } + field = tmp; + } + self->field_len = 0; } if (PyList_Append(self->fields, field) < 0) { Py_DECREF(field); @@ -682,9 +731,10 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } /* normal character - handle as START_FIELD */ self->state = START_FIELD; - /* fallthru */ + _Py_FALLTHROUGH; case START_FIELD: /* expecting field */ + self->unquoted_field = true; if (c == '\n' || c == '\r' || c == EOL) { /* save empty field - return [fields] */ if (parse_save_field(self) < 0) @@ -694,10 +744,12 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) else if (c == dialect->quotechar && dialect->quoting != QUOTE_NONE) { /* start quoted field */ + self->unquoted_field = false; self->state = IN_QUOTED_FIELD; } else if (c == dialect->escapechar) { /* possible escaped character */ + self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -710,8 +762,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else { /* begin new unquoted field */ - if (dialect->quoting == QUOTE_NONNUMERIC) - self->numeric_field = 1; if (parse_add_char(self, module_state, c) < 0) return -1; self->state = IN_FIELD; @@ -735,7 +785,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) case AFTER_ESCAPED_CRNL: if (c == EOL) break; - /*fallthru*/ + _Py_FALLTHROUGH; case IN_FIELD: /* in unquoted field */ @@ -838,7 +888,8 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) self->state = START_RECORD; else { PyErr_Format(module_state->error_obj, - "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?"); + "new-line character seen in unquoted field - " + "do you need to open the file with newline=''?"); return -1; } break; @@ -855,7 +906,7 @@ parse_reset(ReaderObj *self) return -1; self->field_len = 0; self->state = START_RECORD; - self->numeric_field = 0; + self->unquoted_field = false; return 0; } @@ -1101,6 +1152,8 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, if (c == dialect->delimiter || c == dialect->escapechar || c == dialect->quotechar || + c == '\n' || + c == '\r' || PyUnicode_FindChar( dialect->lineterminator, c, 0, PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { @@ -1172,6 +1225,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) static int join_append(WriterObj *self, PyObject *field, int quoted) { + DialectObj *dialect = self->dialect; int field_kind = -1; const void *field_data = NULL; Py_ssize_t field_len = 0; @@ -1182,6 +1236,19 @@ join_append(WriterObj *self, PyObject *field, int quoted) field_data = PyUnicode_DATA(field); field_len = PyUnicode_GET_LENGTH(field); } + if (!field_len && dialect->delimiter == ' ' && dialect->skipinitialspace) { + if (dialect->quoting == QUOTE_NONE || + (field == NULL && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { + PyErr_Format(self->error_obj, + "empty field must be quoted if delimiter is a space " + "and skipinitialspace is true"); + return 0; + } + quoted = 1; + } rec_len = join_append_data(self, field_kind, field_data, field_len, "ed, 0); if (rec_len < 0) @@ -1233,6 +1300,7 @@ csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; + bool null_field = false; iter = PyObject_GetIter(seq); if (iter == NULL) { @@ -1269,11 +1337,12 @@ csv_writerow(WriterObj *self, PyObject *seq) break; } + null_field = (field == Py_None); if (PyUnicode_Check(field)) { append_ok = join_append(self, field, quoted); Py_DECREF(field); } - else if (field == Py_None) { + else if (null_field) { append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } @@ -1299,7 +1368,11 @@ csv_writerow(WriterObj *self, PyObject *seq) return NULL; if (self->num_fields > 0 && self->rec_len == 0) { - if (dialect->quoting == QUOTE_NONE) { + if (dialect->quoting == QUOTE_NONE || + (null_field && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { PyErr_Format(self->error_obj, "single empty field record must be quoted"); return NULL; @@ -1533,10 +1606,12 @@ _csv_unregister_dialect_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=0813ebca6c058df4 input=6b5c1557bf60c7e7]*/ { _csvstate *module_state = get_csv_state(module); - if (PyDict_DelItem(module_state->dialects, name) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Format(module_state->error_obj, "unknown dialect"); - } + int rc = PyDict_Pop(module_state->dialects, name, NULL); + if (rc < 0) { + return NULL; + } + if (rc == 0) { + PyErr_Format(module_state->error_obj, "unknown dialect"); return NULL; } Py_RETURN_NONE; @@ -1607,68 +1682,7 @@ PyType_Spec error_spec = { * MODULE */ -PyDoc_STRVAR(csv_module_doc, -"CSV parsing and writing.\n" -"\n" -"This module provides classes that assist in the reading and writing\n" -"of Comma Separated Value (CSV) files, and implements the interface\n" -"described by PEP 305. Although many CSV files are simple to parse,\n" -"the format is not formally defined by a stable specification and\n" -"is subtle enough that parsing lines of a CSV file with something\n" -"like line.split(\",\") is bound to fail. The module supports three\n" -"basic APIs: reading, writing, and registration of dialects.\n" -"\n" -"\n" -"DIALECT REGISTRATION:\n" -"\n" -"Readers and writers support a dialect argument, which is a convenient\n" -"handle on a group of settings. When the dialect argument is a string,\n" -"it identifies one of the dialects previously registered with the module.\n" -"If it is a class or instance, the attributes of the argument are used as\n" -"the settings for the reader or writer:\n" -"\n" -" class excel:\n" -" delimiter = ','\n" -" quotechar = '\"'\n" -" escapechar = None\n" -" doublequote = True\n" -" skipinitialspace = False\n" -" lineterminator = '\\r\\n'\n" -" quoting = QUOTE_MINIMAL\n" -"\n" -"SETTINGS:\n" -"\n" -" * quotechar - specifies a one-character string to use as the\n" -" quoting character. It defaults to '\"'.\n" -" * delimiter - specifies a one-character string to use as the\n" -" field separator. It defaults to ','.\n" -" * skipinitialspace - specifies how to interpret spaces which\n" -" immediately follow a delimiter. It defaults to False, which\n" -" means that spaces immediately following a delimiter is part\n" -" of the following field.\n" -" * lineterminator - specifies the character sequence which should\n" -" terminate rows.\n" -" * quoting - controls when quotes should be generated by the writer.\n" -" It can take on any of the following module constants:\n" -"\n" -" csv.QUOTE_MINIMAL means only when required, for example, when a\n" -" field contains either the quotechar or the delimiter\n" -" csv.QUOTE_ALL means that quotes are always placed around fields.\n" -" csv.QUOTE_NONNUMERIC means that quotes are always placed around\n" -" fields which do not parse as integers or floating point\n" -" numbers.\n" -" csv.QUOTE_STRINGS means that quotes are always placed around\n" -" fields which are strings. Note that the Python value None\n" -" is not a string.\n" -" csv.QUOTE_NOTNULL means that quotes are only placed around fields\n" -" that are not the Python value None.\n" -" csv.QUOTE_NONE means that quotes are never placed around fields.\n" -" * escapechar - specifies a one-character string used to escape\n" -" the delimiter when quoting is set to QUOTE_NONE.\n" -" * doublequote - controls the handling of quotes inside fields. When\n" -" True, two consecutive quotes are interpreted as one during read,\n" -" and when writing, each quote character embedded in the data is\n" -" written as two quotes\n"); +PyDoc_STRVAR(csv_module_doc, "CSV parsing and writing.\n"); PyDoc_STRVAR(csv_reader_doc, " csv_reader = reader(iterable [, dialect='excel']\n" @@ -1741,12 +1755,6 @@ csv_exec(PyObject *module) { return -1; } - /* Add version to the module. */ - if (PyModule_AddStringConstant(module, "__version__", - MODULE_VERSION) == -1) { - return -1; - } - /* Set the field limit */ module_state->field_limit = 128 * 1024; @@ -1788,6 +1796,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 184af2132c2707..db58f33511c166 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2,7 +2,7 @@ ToDo: Get rid of the checker (and also the converters) field in PyCFuncPtrObject and - StgDictObject, and replace them by slot functions in StgDictObject. + StgInfo, and replace them by slot functions in StgInfo. think about a buffer-like object (memory? bytes?) @@ -36,7 +36,6 @@ PyCData_Type Simple_Type __new__(), __init__(), _as_parameter_ PyCField_Type -PyCStgDict_Type ============================================================================== @@ -82,7 +81,6 @@ bytes(cdata) */ /* - * PyCStgDict_Type * PyCStructType_Type * UnionType_Type * PyCPointerType_Type @@ -110,7 +108,9 @@ bytes(cdata) #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() +#ifdef MS_WIN32 +# include "pycore_modsupport.h" // _PyArg_NoKeywords() +#endif #include @@ -126,19 +126,16 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -ctypes_state global_state; - -PyObject *PyExc_ArgError = NULL; - -/* This dict maps ctypes types to POINTER types */ -PyObject *_ctypes_ptrtype_cache = NULL; - -static PyTypeObject Simple_Type; - -/* a callable object used for unpickling: - strong reference to _ctypes._unpickle() function */ -static PyObject *_unpickle; +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ +#define clinic_state() (get_module_state_by_class(cls)) +#define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) +#include "clinic/_ctypes.c.h" +#undef clinic_state +#undef clinic_state_sub /****************************************************************/ @@ -182,7 +179,7 @@ _DictRemover_call(PyObject *myself, PyObject *args, PyObject *kw) DictRemoverObject *self = (DictRemoverObject *)myself; if (self->key && self->dict) { if (-1 == PyDict_DelItem(self->dict, self->key)) { - _PyErr_WriteUnraisableMsg("on calling _ctypes.DictRemover", NULL); + PyErr_FormatUnraisable("Exception ignored on calling _ctypes.DictRemover"); } Py_CLEAR(self->key); Py_CLEAR(self->dict); @@ -210,14 +207,13 @@ static PyType_Spec dictremover_spec = { }; int -PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) +PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item) { PyObject *obj; DictRemoverObject *remover; PyObject *proxy; int result; - ctypes_state *st = GLOBAL_STATE(); obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); if (obj == NULL) return -1; @@ -241,26 +237,13 @@ PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) static int _PyDict_GetItemProxy(PyObject *dict, PyObject *key, PyObject **presult) { - PyObject *item = PyDict_GetItemWithError(dict, key); - if (item == NULL) { - if (PyErr_Occurred()) { - return -1; - } - *presult = NULL; - return 0; - } - - if (!PyWeakref_CheckProxy(item)) { - *presult = Py_NewRef(item); - return 0; + int rc = PyDict_GetItemRef(dict, key, presult); + PyObject *item = *presult; + if (item && PyWeakref_CheckProxy(item)) { + rc = PyWeakref_GetRef(item, presult); + Py_DECREF(item); } - PyObject *ref; - if (PyWeakref_GetRef(item, &ref) < 0) { - return -1; - } - // ref is NULL if the referenced object was destroyed - *presult = ref; - return 0; + return rc; } /******************************************************************/ @@ -459,20 +442,153 @@ static PyType_Spec structparam_spec = { .slots = structparam_slots, }; +/* + CType_Type - a base metaclass. Its instances (classes) have a StgInfo. + */ + +/*[clinic input] +class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/ + +static int +CType_Type_traverse(PyObject *self, visitproc visit, void *arg) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); + } + Py_VISIT(Py_TYPE(self)); + return PyType_Type.tp_traverse(self, visit, arg); +} + +void +ctype_clear_stginfo(StgInfo *info) +{ + assert(info); + Py_CLEAR(info->proto); + Py_CLEAR(info->argtypes); + Py_CLEAR(info->converters); + Py_CLEAR(info->restype); + Py_CLEAR(info->checker); + Py_CLEAR(info->module); +} + +static int +CType_Type_clear(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); + } + return PyType_Type.tp_clear(self); +} + +static void +CType_Type_dealloc(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); + } + + PyTypeObject *tp = Py_TYPE(self); + PyType_Type.tp_dealloc(self); + Py_DECREF(tp); +} + +/*[clinic input] +_ctypes.CType_Type.__sizeof__ + + cls: defining_class + / +Return memory consumption of the type object. +[clinic start generated code]*/ + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=c68c235be84d03f3 input=d064433b6110d1ce]*/ +{ + Py_ssize_t size = Py_TYPE(self)->tp_basicsize; + size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return NULL; + } + if (info) { + if (info->format) { + size += strlen(info->format) + 1; + } + if (info->ffi_type_pointer.elements) { + size += (info->length + 1) * sizeof(ffi_type *); + } + size += info->ndim * sizeof(Py_ssize_t); + } + + return PyLong_FromSsize_t(size); +} + +static PyObject * +CType_Type_repeat(PyObject *self, Py_ssize_t length); + + +static PyMethodDef ctype_methods[] = { + _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF + {0}, +}; + +static PyType_Slot ctype_type_slots[] = { + {Py_tp_traverse, CType_Type_traverse}, + {Py_tp_clear, CType_Type_clear}, + {Py_tp_dealloc, CType_Type_dealloc}, + {Py_tp_methods, ctype_methods}, + // Sequence protocol. + {Py_sq_repeat, CType_Type_repeat}, + {0, NULL}, +}; + +static PyType_Spec pyctype_type_spec = { + .name = "_ctypes.CType_Type", + .basicsize = -(Py_ssize_t)sizeof(StgInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE ), + .slots = ctype_type_slots, +}; /* PyCStructType_Type - a meta type/class. Creating a new class using this one as - __metaclass__ will call the constructor StructUnionType_new. It replaces the - tp_dict member with a new instance of StgDict, and initializes the C - accessible fields somehow. + __metaclass__ will call the constructor StructUnionType_new. + It initializes the C accessible fields somehow. */ static PyCArgObject * -StructUnionType_paramfunc(CDataObject *self) +StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; PyObject *obj; - StgDictObject *stgdict; void *ptr; if ((size_t)self->b_size > sizeof(void*)) { @@ -485,7 +601,6 @@ StructUnionType_paramfunc(CDataObject *self) /* Create a Python object which calls PyMem_Free(ptr) in its deallocator. The object will be destroyed at _ctypes_callproc() cleanup. */ - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->StructParam_Type; obj = tp->tp_alloc(tp, 0); if (obj == NULL) { @@ -501,113 +616,134 @@ StructUnionType_paramfunc(CDataObject *self) obj = Py_NewRef(self); } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; } + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + Py_DECREF(obj); + return NULL; + } + assert(stginfo); /* Cannot be NULL for structure/union instances */ + parg->tag = 'V'; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for structure/union instances */ - parg->pffi_type = &stgdict->ffi_type_pointer; + parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; parg->obj = obj; return parg; } -static PyObject * -StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isStruct) +static int +StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruct) { - PyTypeObject *result; PyObject *fields; - StgDictObject *dict; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (!result) - return NULL; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } /* keep this for bw compatibility */ - int r = PyDict_Contains(result->tp_dict, &_Py_ID(_abstract_)); - if (r > 0) - return (PyObject *)result; + int r = PyDict_Contains(attrdict, &_Py_ID(_abstract_)); + if (r > 0) { + Py_DECREF(attrdict); + return 0; + } if (r < 0) { - Py_DECREF(result); - return NULL; + Py_DECREF(attrdict); + return -1; } - dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)&PyCStgDict_Type); - if (!dict) { - Py_DECREF(result); - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *info = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!info) { + Py_DECREF(attrdict); + return -1; } if (!isStruct) { - dict->flags |= TYPEFLAG_HASUNION; - } - /* replace the class dict by our updated stgdict, which holds info - about storage requirements of the instances */ - if (-1 == PyDict_Update((PyObject *)dict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)dict); - return NULL; + info->flags |= TYPEFLAG_HASUNION; } - Py_SETREF(result->tp_dict, (PyObject *)dict); - dict->format = _ctypes_alloc_format_string(NULL, "B"); - if (dict->format == NULL) { - Py_DECREF(result); - return NULL; + + info->format = _ctypes_alloc_format_string(NULL, "B"); + if (info->format == NULL) { + Py_DECREF(attrdict); + return -1; } - dict->paramfunc = StructUnionType_paramfunc; + info->paramfunc = StructUnionType_paramfunc; - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_fields_), &fields) < 0) { + Py_DECREF(attrdict); + return -1; + } + Py_CLEAR(attrdict); if (fields) { - if (PyObject_SetAttr((PyObject *)result, &_Py_ID(_fields_), fields) < 0) { - Py_DECREF(result); - return NULL; + if (PyObject_SetAttr(self, &_Py_ID(_fields_), fields) < 0) { + Py_DECREF(fields); + return -1; } - return (PyObject *)result; - } - else if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + Py_DECREF(fields); + return 0; } else { - StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)self)->tp_base, + &baseinfo) < 0) { + return -1; + } + if (baseinfo == NULL) { + return 0; + } - if (basedict == NULL) - return (PyObject *)result; - /* copy base dict */ - if (-1 == PyCStgDict_clone(dict, basedict)) { - Py_DECREF(result); - return NULL; + /* copy base info */ + if (PyCStgInfo_clone(info, baseinfo) < 0) { + return -1; } - dict->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass dict */ - basedict->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass dict */ - return (PyObject *)result; + info->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass info */ + baseinfo->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass info */ } + return 0; } -static PyObject * -PyCStructType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCStructType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 1); + return StructUnionType_init(self, args, kwds, 1); } -static PyObject * -UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +UnionType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 0); + return StructUnionType_init(self, args, kwds, 0); } -PyDoc_STRVAR(from_address_doc, -"C.from_address(integer) -> C instance\naccess a C instance at the specified address"); +/*[clinic input] +class _ctypes.CDataType "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=466a505a93d73156]*/ + + +/*[clinic input] +_ctypes.CDataType.from_address as CDataType_from_address + + type: self + cls: defining_class + value: object + / + +C.from_address(integer) -> C instance + +Access a C instance at the specified address. +[clinic start generated code]*/ static PyObject * -CDataType_from_address(PyObject *type, PyObject *value) +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=5be4a7c0d9aa6c74 input=827a22cefe380c01]*/ { void *buf; if (!PyLong_Check(value)) { @@ -618,32 +754,45 @@ CDataType_from_address(PyObject *type, PyObject *value) buf = (void *)PyLong_AsVoidPtr(value); if (PyErr_Occurred()) return NULL; - return PyCData_AtAddress(type, buf); + ctypes_state *st = get_module_state_by_class(cls); + return PyCData_AtAddress(st, type, buf); } -PyDoc_STRVAR(from_buffer_doc, -"C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"); - static int KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep); +/*[clinic input] +_ctypes.CDataType.from_buffer as CDataType_from_buffer + + type: self + cls: defining_class + obj: object + offset: Py_ssize_t = 0 + / + +C.from_buffer(object, offset=0) -> C instance + +Create a C instance from a writeable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer(PyObject *type, PyObject *args) +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset) +/*[clinic end generated code: output=57604e99635abd31 input=0f36cedd105ca28d]*/ { - PyObject *obj; PyObject *mv; PyObject *result; Py_buffer *buffer; - Py_ssize_t offset = 0; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } mv = PyMemoryView_FromObject(obj); if (mv == NULL) @@ -672,11 +821,11 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - if (dict->size > buffer->len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small " "(%zd instead of at least %zd bytes)", - buffer->len, dict->size + offset); + buffer->len, info->size + offset); Py_DECREF(mv); return NULL; } @@ -687,7 +836,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - result = PyCData_AtAddress(type, (char *)buffer->buf + offset); + result = PyCData_AtAddress(st, type, (char *)buffer->buf + offset); if (result == NULL) { Py_DECREF(mv); return NULL; @@ -701,72 +850,94 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -PyDoc_STRVAR(from_buffer_copy_doc, -"C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"); +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds); static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +/*[clinic input] +_ctypes.CDataType.from_buffer_copy as CDataType_from_buffer_copy + + type: self + cls: defining_class + buffer: Py_buffer + offset: Py_ssize_t = 0 + / + +C.from_buffer_copy(object, offset=0) -> C instance + +Create a C instance from a readable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer_copy(PyObject *type, PyObject *args) +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset) +/*[clinic end generated code: output=c8fc62b03e5cc6fa input=2a81e11b765a6253]*/ { - Py_buffer buffer; - Py_ssize_t offset = 0; PyObject *result; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "y*|n:from_buffer_copy", &buffer, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset cannot be negative"); - PyBuffer_Release(&buffer); return NULL; } - if (dict->size > buffer.len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", - buffer.len, dict->size + offset); - PyBuffer_Release(&buffer); + buffer->len, info->size + offset); return NULL; } if (PySys_Audit("ctypes.cdata/buffer", "nnn", - (Py_ssize_t)buffer.buf, buffer.len, offset) < 0) { - PyBuffer_Release(&buffer); + (Py_ssize_t)buffer->buf, buffer->len, offset) < 0) { return NULL; } - result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL); + result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer.buf + offset, dict->size); + (char *)buffer->buf + offset, info->size); } - PyBuffer_Release(&buffer); return result; } -PyDoc_STRVAR(in_dll_doc, -"C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"); +/*[clinic input] +_ctypes.CDataType.in_dll as CDataType_in_dll + + type: self + cls: defining_class + dll: object + name: str + / + +C.in_dll(dll, name) -> C instance + +Access a C instance in a dll. +[clinic start generated code]*/ static PyObject * -CDataType_in_dll(PyObject *type, PyObject *args) +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name) +/*[clinic end generated code: output=d0e5c43b66bfa21f input=f85bf281477042b4]*/ { - PyObject *dll; - char *name; PyObject *obj; void *handle; void *address; - if (!PyArg_ParseTuple(args, "Os:in_dll", &dll, &name)) - return NULL; - if (PySys_Audit("ctypes.dlsym", "O", args) < 0) { + if (PySys_Audit("ctypes.dlsym", "Os", dll, name) < 0) { return NULL; } @@ -811,14 +982,24 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #endif - return PyCData_AtAddress(type, address); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return PyCData_AtAddress(st, type, address); } -PyDoc_STRVAR(from_param_doc, -"Convert a Python object into a function call parameter."); +/*[clinic input] +_ctypes.CDataType.from_param as CDataType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -CDataType_from_param(PyObject *type, PyObject *value) +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=8da9e34263309f9e input=275a52c4899ddff0]*/ { PyObject *as_parameter; int res = PyObject_IsInstance(value, type); @@ -827,18 +1008,19 @@ CDataType_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); if (PyCArg_CheckExact(st, value)) { PyCArgObject *p = (PyCArgObject *)value; PyObject *ob = p->obj; const char *ob_name; - StgDictObject *dict; - dict = PyType_stgdict(type); - + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } /* If we got a PyCArgObject, we must check if the object packed in it - is an instance of the type's dict->proto */ - if(dict && ob) { - res = PyObject_IsInstance(ob, dict->proto); + is an instance of the type's info->proto */ + if(info && ob) { + res = PyObject_IsInstance(ob, info->proto); if (res == -1) return NULL; if (res) { @@ -856,7 +1038,7 @@ CDataType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = CDataType_from_param(type, as_parameter); + value = CDataType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -868,55 +1050,25 @@ CDataType_from_param(PyObject *type, PyObject *value) } static PyMethodDef CDataType_methods[] = { - { "from_param", CDataType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc }, + CDATATYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; static PyObject * -CDataType_repeat(PyObject *self, Py_ssize_t length) +CType_Type_repeat(PyObject *self, Py_ssize_t length) { if (length < 0) return PyErr_Format(PyExc_ValueError, "Array length must be >= 0, not %zd", length); - return PyCArrayType_from_ctype(self, length); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + return PyCArrayType_from_ctype(st, self, length); } -static PySequenceMethods CDataType_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - CDataType_repeat, /* intargfunc sq_repeat; */ - 0, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - 0, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - -static int -CDataType_clear(PyTypeObject *self) -{ - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) - Py_CLEAR(dict->proto); - return PyType_Type.tp_clear((PyObject *)self); -} - -static int -CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) -{ - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) - Py_VISIT(dict->proto); - return PyType_Type.tp_traverse((PyObject *)self, visit, arg); -} static int PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) @@ -927,7 +1079,7 @@ PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) if (value && PyUnicode_Check(key) && _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 1); + return PyCStructUnionType_update_stginfo(self, value, 1); return 0; } @@ -936,100 +1088,44 @@ static int UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) { /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyObject_GenericSetAttr(self, key, value)) + if (-1 == PyType_Type.tp_setattro(self, key, value)) return -1; if (PyUnicode_Check(key) && _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 0); + return PyCStructUnionType_update_stginfo(self, value, 0); return 0; } +static PyType_Slot pycstruct_type_slots[] = { + {Py_tp_setattro, PyCStructType_setattro}, + {Py_tp_doc, PyDoc_STR("metatype for the CData Objects")}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_init, PyCStructType_init}, + {0, NULL}, +}; -PyTypeObject PyCStructType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCStructType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - PyCStructType_setattro, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the CData Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCStructType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycstruct_type_spec = { + .name = "_ctypes.PyCStructType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_type_slots, }; -static PyTypeObject UnionType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.UnionType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - UnionType_setattro, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the CData Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - UnionType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot union_type_slots[] = { + {Py_tp_setattro, UnionType_setattro}, + {Py_tp_doc, PyDoc_STR("metatype for the Union Objects")}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_init, UnionType_init}, + {0, NULL}, }; +static PyType_Spec union_type_spec = { + .name = "_ctypes.UnionType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = union_type_slots, +}; /******************************************************************/ @@ -1047,30 +1143,40 @@ size property/method, and the sequence protocol. */ +/*[clinic input] +class _ctypes.PyCPointerType "PyObject *" "clinic_state()->PyCPointerType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c45e96c1f7645ab7]*/ + + static int -PyCPointerType_SetProto(StgDictObject *stgdict, PyObject *proto) +PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) { if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, "_type_ must be a type"); return -1; } - if (!PyType_stgdict(proto)) { + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); return -1; } Py_INCREF(proto); - Py_XSETREF(stgdict->proto, proto); + Py_XSETREF(stginfo->proto, proto); return 0; } static PyCArgObject * -PyCPointerType_paramfunc(CDataObject *self) +PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1081,123 +1187,145 @@ PyCPointerType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; PyObject *typedict; - typedict = PyTuple_GetItem(args, 2); - if (!typedict) - return NULL; + if (!typedict) { + return -1; + } + /* - stgdict items size, align, length contain info about pointers itself, - stgdict->proto has info about the pointed to type! + stginfo items size, align, length contain info about pointers itself, + stginfo->proto has info about the pointed to type! */ - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) - return NULL; - stgdict->size = sizeof(void *); - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->ffi_type_pointer = ffi_type_pointer; - stgdict->paramfunc = PyCPointerType_paramfunc; - stgdict->flags |= TYPEFLAG_ISPOINTER; - - proto = PyDict_GetItemWithError(typedict, &_Py_ID(_type_)); /* Borrowed ref */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + return -1; + } + stginfo->size = sizeof(void *); + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->ffi_type_pointer = ffi_type_pointer; + stginfo->paramfunc = PyCPointerType_paramfunc; + stginfo->flags |= TYPEFLAG_ISPOINTER; + + if (PyDict_GetItemRef(typedict, &_Py_ID(_type_), &proto) < 0) { + return -1; + } if (proto) { - StgDictObject *itemdict; const char *current_format; - if (-1 == PyCPointerType_SetProto(stgdict, proto)) { - Py_DECREF((PyObject *)stgdict); - return NULL; + if (PyCPointerType_SetProto(st, stginfo, proto) < 0) { + Py_DECREF(proto); + return -1; + } + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + Py_DECREF(proto); + return -1; } - itemdict = PyType_stgdict(proto); - /* PyCPointerType_SetProto has verified proto has a stgdict. */ - assert(itemdict); - /* If itemdict->format is NULL, then this is a pointer to an + /* PyCPointerType_SetProto has verified proto has a stginfo. */ + assert(iteminfo); + /* If iteminfo->format is NULL, then this is a pointer to an incomplete type. We create a generic format string 'pointer to bytes' in this case. XXX Better would be to fix the format string later... */ - current_format = itemdict->format ? itemdict->format : "B"; - if (itemdict->shape != NULL) { + current_format = iteminfo->format ? iteminfo->format : "B"; + if (iteminfo->shape != NULL) { /* pointer to an array: the shape needs to be prefixed */ - stgdict->format = _ctypes_alloc_format_string_with_shape( - itemdict->ndim, itemdict->shape, "&", current_format); + stginfo->format = _ctypes_alloc_format_string_with_shape( + iteminfo->ndim, iteminfo->shape, "&", current_format); } else { - stgdict->format = _ctypes_alloc_format_string("&", current_format); + stginfo->format = _ctypes_alloc_format_string("&", current_format); } - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; + Py_DECREF(proto); + if (stginfo->format == NULL) { + return -1; } } - else if (PyErr_Occurred()) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); - return (PyObject *)result; + return 0; } +/*[clinic input] +_ctypes.PyCPointerType.set_type as PyCPointerType_set_type + + self: self(type="PyTypeObject *") + cls: defining_class + type: object + / +[clinic start generated code]*/ static PyObject * -PyCPointerType_set_type(PyTypeObject *self, PyObject *type) +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type) +/*[clinic end generated code: output=51459d8f429a70ac input=67e1e8df921f123e]*/ { - StgDictObject *dict; - - - dict = PyType_stgdict((PyObject *)self); - if (!dict) { + PyObject *attrdict = PyType_GetDict(self); + if (!attrdict) { + return NULL; + } + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + Py_DECREF(attrdict); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); + Py_DECREF(attrdict); return NULL; } - if (-1 == PyCPointerType_SetProto(dict, type)) + if (PyCPointerType_SetProto(st, info, type) < 0) { + Py_DECREF(attrdict); return NULL; + } - if (-1 == PyDict_SetItem((PyObject *)dict, &_Py_ID(_type_), type)) + if (-1 == PyDict_SetItem(attrdict, &_Py_ID(_type_), type)) { + Py_DECREF(attrdict); return NULL; + } + Py_DECREF(attrdict); Py_RETURN_NONE; } -static PyObject *_byref(PyObject *); +static PyObject *_byref(ctypes_state *, PyObject *); + +/*[clinic input] +_ctypes.PyCPointerType.from_param as PyCPointerType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -PyCPointerType_from_param(PyObject *type, PyObject *value) +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=a4b32d929aabaf64 input=6c231276e3997884]*/ { - StgDictObject *typedict; - if (value == Py_None) { /* ConvParam will convert to a NULL pointer later */ return Py_NewRef(value); } - typedict = PyType_stgdict(type); - if (!typedict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *typeinfo; + if (PyStgInfo_FromType(st, type, &typeinfo) < 0) { + return NULL; + } + if (!typeinfo) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; @@ -1206,23 +1334,27 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) /* If we expect POINTER(), but receive a instance, accept it by calling byref(). */ - switch (PyObject_IsInstance(value, typedict->proto)) { + assert(typeinfo->proto); + switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ - return _byref(value); + return _byref(st, value); case -1: return NULL; default: break; } - if (PointerObject_Check(value) || ArrayObject_Check(value)) { + if (PointerObject_Check(st, value) || ArrayObject_Check(st, value)) { /* Array instances are also pointers when the item types are the same. */ - StgDictObject *v = PyObject_stgdict(value); + StgInfo *v; + if (PyStgInfo_FromObject(st, value, &v) < 0) { + return NULL; + } assert(v); /* Cannot be NULL for pointer or array objects */ - int ret = PyObject_IsSubclass(v->proto, typedict->proto); + int ret = PyObject_IsSubclass(v->proto, typeinfo->proto); if (ret < 0) { return NULL; } @@ -1230,59 +1362,31 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - return CDataType_from_param(type, value); + return CDataType_from_param_impl(type, cls, value); } static PyMethodDef PyCPointerType_methods[] = { - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, - { "from_param", (PyCFunction)PyCPointerType_from_param, METH_O, from_param_doc}, - { "set_type", (PyCFunction)PyCPointerType_set_type, METH_O }, + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF + PYCPOINTERTYPE_FROM_PARAM_METHODDEF + PYCPOINTERTYPE_SET_TYPE_METHODDEF { NULL, NULL }, }; -PyTypeObject PyCPointerType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCPointerType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for the Pointer Objects"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCPointerType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCPointerType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycpointer_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the Pointer Objects")}, + {Py_tp_methods, PyCPointerType_methods}, + {Py_tp_init, PyCPointerType_init}, + {0, NULL}, +}; + +static PyType_Spec pycpointer_type_spec = { + .name = "_ctypes.PyCPointerType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_type_slots, }; @@ -1291,7 +1395,7 @@ PyTypeObject PyCPointerType_Type = { PyCArrayType_Type */ /* - PyCArrayType_new ensures that the new Array subclass created has a _length_ + PyCArrayType_init ensures that the new Array subclass created has a _length_ attribute, and a _type_ attribute. */ @@ -1463,9 +1567,9 @@ add_getset(PyTypeObject *type, PyGetSetDef *gsp) } static PyCArgObject * -PyCArrayType_paramfunc(CDataObject *self) +PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) { - PyCArgObject *p = PyCArgObject_new(); + PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; p->tag = 'P'; @@ -1475,28 +1579,18 @@ PyCArrayType_paramfunc(CDataObject *self) return p; } -static PyObject * -PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; - StgDictObject *itemdict; PyObject *length_attr, *type_attr; Py_ssize_t length; Py_ssize_t itemsize, itemalign; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - /* Initialize these variables to NULL so that we can simplify error handling by using Py_XDECREF. */ - stgdict = NULL; type_attr = NULL; - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_length_), &length_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_length_), &length_attr) < 0) { goto error; } if (!length_attr) { @@ -1512,7 +1606,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (_PyLong_Sign(length_attr) == -1) { + if (_PyLong_IsNegative((PyLongObject *)length_attr)) { Py_DECREF(length_attr); PyErr_SetString(PyExc_ValueError, "The '_length_' attribute must not be negative"); @@ -1529,7 +1623,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &type_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &type_attr) < 0) { goto error; } if (!type_attr) { @@ -1538,125 +1632,92 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject*)self); + if (!stginfo) { goto error; + } - itemdict = PyType_stgdict(type_attr); - if (!itemdict) { + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, type_attr, &iteminfo) < 0) { + goto error; + } + if (!iteminfo) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); goto error; } - assert(itemdict->format); - stgdict->format = _ctypes_alloc_format_string(NULL, itemdict->format); - if (stgdict->format == NULL) + assert(iteminfo->format); + stginfo->format = _ctypes_alloc_format_string(NULL, iteminfo->format); + if (stginfo->format == NULL) goto error; - stgdict->ndim = itemdict->ndim + 1; - stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim); - if (stgdict->shape == NULL) { + stginfo->ndim = iteminfo->ndim + 1; + stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim); + if (stginfo->shape == NULL) { PyErr_NoMemory(); goto error; } - stgdict->shape[0] = length; - if (stgdict->ndim > 1) { - memmove(&stgdict->shape[1], itemdict->shape, - sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + stginfo->shape[0] = length; + if (stginfo->ndim > 1) { + memmove(&stginfo->shape[1], iteminfo->shape, + sizeof(Py_ssize_t) * (stginfo->ndim - 1)); } - itemsize = itemdict->size; + itemsize = iteminfo->size; if (itemsize != 0 && length > PY_SSIZE_T_MAX / itemsize) { PyErr_SetString(PyExc_OverflowError, "array too large"); goto error; } - itemalign = itemdict->align; + itemalign = iteminfo->align; - if (itemdict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; + if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; - stgdict->size = itemsize * length; - stgdict->align = itemalign; - stgdict->length = length; - stgdict->proto = type_attr; + stginfo->size = itemsize * length; + stginfo->align = itemalign; + stginfo->length = length; + stginfo->proto = type_attr; type_attr = NULL; - stgdict->paramfunc = &PyCArrayType_paramfunc; + stginfo->paramfunc = &PyCArrayType_paramfunc; /* Arrays are passed as pointers to function calls. */ - stgdict->ffi_type_pointer = ffi_type_pointer; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) - goto error; - Py_SETREF(result->tp_dict, (PyObject *)stgdict); /* steal the reference */ - stgdict = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; /* Special case for character arrays. A permanent annoyance: char arrays are also strings! */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - if (-1 == add_getset(result, CharArray_getsets)) + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, CharArray_getsets)) goto error; } - else if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - if (-1 == add_getset(result, WCharArray_getsets)) + else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, WCharArray_getsets)) goto error; } - return (PyObject *)result; + return 0; error: - Py_XDECREF((PyObject*)stgdict); Py_XDECREF(type_attr); - Py_DECREF(result); - return NULL; + return -1; } -PyTypeObject PyCArrayType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCArrayType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("metatype for the Array Objects"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCArrayType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_init, PyCArrayType_init}, + {0, NULL}, }; +static PyType_Spec pycarray_type_spec = { + .name = "_ctypes.PyCArrayType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_type_slots, +}; /******************************************************************/ /* @@ -1664,26 +1725,61 @@ PyTypeObject PyCArrayType_Type = { */ /* -PyCSimpleType_new ensures that the new Simple_Type subclass created has a valid +PyCSimpleType_init ensures that the new Simple_Type subclass created has a valid _type_ attribute. */ +/*[clinic input] +class _ctypes.PyCSimpleType "PyObject *" "clinic_state()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d5a45772668e7f49]*/ + +/*[clinic input] +class _ctypes.c_wchar_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=468de7283d622d47]*/ + +/*[clinic input] +class _ctypes.c_char_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e750865616e7dcea]*/ + +/*[clinic input] +class _ctypes.c_void_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd4d9646c56f43a9]*/ + +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdCEFfuzZqQPXOv?g"; +#else static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; +#endif + +/*[clinic input] +_ctypes.c_wchar_p.from_param as c_wchar_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ static PyObject * -c_wchar_p_from_param(PyObject *type, PyObject *value) +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=e453949a2f725a4c input=d322c7237a319607]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyUnicode_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1701,22 +1797,31 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } @@ -1725,7 +1830,7 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_wchar_p_from_param(type, as_parameter); + value = c_wchar_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1735,19 +1840,30 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_char_p.from_param as c_char_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_char_p_from_param(PyObject *type, PyObject *value) +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=219652ab7c174aa1 input=6cf0d1b6bb4ede11]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyBytes_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1765,22 +1881,31 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } @@ -1789,7 +1914,7 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_char_p_from_param(type, as_parameter); + value = c_char_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1799,10 +1924,19 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_void_p.from_param as c_void_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_void_p_from_param(PyObject *type, PyObject *value) +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=984d0075b6038cc7 input=0e8b343fc19c77d4]*/ { - StgDictObject *stgd; PyObject *as_parameter; int res; @@ -1810,13 +1944,15 @@ c_void_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); + /* Should probably allow buffer interface as well */ /* int, long */ if (PyLong_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("P"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1834,7 +1970,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1851,7 +1987,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1872,12 +2008,11 @@ c_void_p_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } /* ctypes array or pointer instance */ - if (ArrayObject_Check(value) || PointerObject_Check(value)) { + if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* Any array or pointer is accepted */ return Py_NewRef(value); } /* byref(...) */ - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; @@ -1886,11 +2021,11 @@ c_void_p_from_param(PyObject *type, PyObject *value) } } /* function pointer */ - if (PyCFuncPtrObject_Check(value)) { + if (PyCFuncPtrObject_Check(st, value)) { PyCArgObject *parg; PyCFuncPtrObject *func; func = (PyCFuncPtrObject *)value; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1901,14 +2036,21 @@ c_void_p_from_param(PyObject *type, PyObject *value) return (PyObject *)parg; } /* c_char_p, c_wchar_p */ - stgd = PyObject_stgdict(value); - if (stgd && CDataObject_Check(value) && stgd->proto && PyUnicode_Check(stgd->proto)) { + StgInfo *stgi; + if (PyStgInfo_FromObject(st, value, &stgi) < 0) { + return NULL; + } + if (stgi + && CDataObject_Check(st, value) + && stgi->proto + && PyUnicode_Check(stgi->proto)) + { PyCArgObject *parg; - switch (PyUnicode_AsUTF8(stgd->proto)[0]) { + switch (PyUnicode_AsUTF8(stgi->proto)[0]) { case 'z': /* c_char_p */ case 'Z': /* c_wchar_p */ - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1924,7 +2066,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_void_p_from_param(type, as_parameter); + value = c_void_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1934,37 +2076,37 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } -static PyMethodDef c_void_p_method = { "from_param", c_void_p_from_param, METH_O }; -static PyMethodDef c_char_p_method = { "from_param", c_char_p_from_param, METH_O }; -static PyMethodDef c_wchar_p_method = { "from_param", c_wchar_p_from_param, METH_O }; +static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; -static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject *kwds, +static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, + PyObject *args, PyObject *kwds, PyObject *proto, struct fielddesc *fmt) { PyTypeObject *result; - StgDictObject *stgdict; PyObject *name = PyTuple_GET_ITEM(args, 0); PyObject *newname; PyObject *swapped_args; - static PyObject *suffix; Py_ssize_t i; swapped_args = PyTuple_New(PyTuple_GET_SIZE(args)); if (!swapped_args) return NULL; - if (suffix == NULL) + if (st->swapped_suffix == NULL) { #ifdef WORDS_BIGENDIAN - suffix = PyUnicode_InternFromString("_le"); + st->swapped_suffix = PyUnicode_InternFromString("_le"); #else - suffix = PyUnicode_InternFromString("_be"); + st->swapped_suffix = PyUnicode_InternFromString("_be"); #endif - if (suffix == NULL) { + } + if (st->swapped_suffix == NULL) { Py_DECREF(swapped_args); return NULL; } - newname = PyUnicode_Concat(name, suffix); + newname = PyUnicode_Concat(name, st->swapped_suffix); if (newname == NULL) { Py_DECREF(swapped_args); return NULL; @@ -1984,50 +2126,43 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) { + StgInfo *stginfo = PyStgInfo_Init(st, result); + if (!stginfo) { Py_DECREF(result); return NULL; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc_swapped; - stgdict->getfunc = fmt->getfunc_swapped; + stginfo->ffi_type_pointer = *fmt->pffi_type; + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc_swapped; + stginfo->getfunc = fmt->getfunc_swapped; - stgdict->proto = Py_NewRef(proto); - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = Py_NewRef(proto); return (PyObject *)result; } static PyCArgObject * -PyCSimpleType_paramfunc(CDataObject *self) +PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - fmt = PyUnicode_AsUTF8(dict->proto); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2038,33 +2173,27 @@ PyCSimpleType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; const char *proto_str; Py_ssize_t proto_len; PyMethodDef *ml; struct fielddesc *fmt; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &proto) < 0) { - return NULL; + if (PyType_Type.tp_init(self, args, kwds) < 0) { + return -1; + } + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &proto) < 0) { + return -1; } if (!proto) { PyErr_SetString(PyExc_AttributeError, "class must define a '_type_' attribute"); error: Py_XDECREF(proto); - Py_DECREF(result); - return NULL; + return -1; } if (PyUnicode_Check(proto)) { proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); @@ -2095,70 +2224,72 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { goto error; + } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc; - stgdict->getfunc = fmt->getfunc; + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + const size_t els_size = sizeof(fmt->pffi_type->elements); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc; + stginfo->getfunc = fmt->getfunc; #ifdef WORDS_BIGENDIAN - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); #else - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); #endif - if (stgdict->format == NULL) { - Py_DECREF(result); + if (stginfo->format == NULL) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - stgdict->paramfunc = PyCSimpleType_paramfunc; + stginfo->paramfunc = PyCSimpleType_paramfunc; /* - if (result->tp_base != &Simple_Type) { - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; + if (self->tp_base != st->Simple_Type) { + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; } */ /* This consumes the refcount on proto which we have */ - stgdict->proto = proto; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = proto; /* Install from_param class methods in ctypes base classes. Overrides the PyCSimpleType_from_param generic method. */ - if (result->tp_base == &Simple_Type) { + if (((PyTypeObject *)self)->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ - ml = &c_char_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_char_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'Z': /* c_wchar_p */ - ml = &c_wchar_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_wchar_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'P': /* c_void_p */ - ml = &c_void_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_void_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 's': case 'X': case 'O': ml = NULL; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; default: ml = NULL; @@ -2168,64 +2299,80 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ml) { PyObject *meth; int x; - meth = PyDescr_NewClassMethod(result, ml); + meth = PyDescr_NewClassMethod((PyTypeObject*)self, ml); if (!meth) { - Py_DECREF(result); - return NULL; + return -1; } - x = PyDict_SetItemString(result->tp_dict, + x = PyDict_SetItemString(((PyTypeObject*)self)->tp_dict, ml->ml_name, meth); Py_DECREF(meth); if (x == -1) { - Py_DECREF(result); - return NULL; + return -1; } } } - if (type == &PyCSimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { - PyObject *swapped = CreateSwappedType(type, args, kwds, + PyTypeObject *type = Py_TYPE(self); + if (type == st->PyCSimpleType_Type + && fmt->setfunc_swapped + && fmt->getfunc_swapped) + { + PyObject *swapped = CreateSwappedType(st, type, args, kwds, proto, fmt); - StgDictObject *sw_dict; if (swapped == NULL) { - Py_DECREF(result); - return NULL; + return -1; + } + StgInfo *sw_info; + if (PyStgInfo_FromType(st, swapped, &sw_info) < 0) { + return -1; } - sw_dict = PyType_stgdict(swapped); + assert(sw_info); #ifdef WORDS_BIGENDIAN - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_le__", swapped); + PyObject_SetAttrString(self, "__ctype_be__", self); + PyObject_SetAttrString(swapped, "__ctype_be__", self); PyObject_SetAttrString(swapped, "__ctype_le__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string("<", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1); #else - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_be__", swapped); + PyObject_SetAttrString(self, "__ctype_le__", self); + PyObject_SetAttrString(swapped, "__ctype_le__", self); PyObject_SetAttrString(swapped, "__ctype_be__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string(">", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1); #endif Py_DECREF(swapped); if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + return -1; } }; - return (PyObject *)result; + return 0; } /* * This is a *class method*. * Convert a parameter into something that ConvParam can handle. */ + +/*[clinic input] +_ctypes.PyCSimpleType.from_param as PyCSimpleType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ + static PyObject * -PyCSimpleType_from_param(PyObject *type, PyObject *value) +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=8a8453d9663e3a2e input=61cc48ce3a87a570]*/ { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; @@ -2241,21 +2388,25 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } - dict = PyType_stgdict(type); - if (!dict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } /* I think we can rely on this being a one-character string */ - fmt = PyUnicode_AsUTF8(dict->proto); + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2277,7 +2428,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) Py_XDECREF(exc); return NULL; } - value = PyCSimpleType_from_param(type, as_parameter); + value = PyCSimpleType_from_param_impl(type, cls, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); @@ -2293,54 +2444,26 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) } static PyMethodDef PyCSimpleType_methods[] = { - { "from_param", PyCSimpleType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, + PYCSIMPLETYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; -PyTypeObject PyCSimpleType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCSimpleType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("metatype for the PyCSimpleType Objects"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCSimpleType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCSimpleType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, + {Py_tp_methods, PyCSimpleType_methods}, + {Py_tp_init, PyCSimpleType_init}, + {0, NULL}, +}; + +static PyType_Spec pycsimple_type_spec = { + .name = "_ctypes.PyCSimpleType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_type_slots, }; /******************************************************************/ @@ -2349,7 +2472,7 @@ PyTypeObject PyCSimpleType_Type = { */ static PyObject * -converters_from_argtypes(PyObject *ob) +converters_from_argtypes(ctypes_state *st, PyObject *ob) { PyObject *converters; Py_ssize_t i; @@ -2364,7 +2487,7 @@ converters_from_argtypes(PyObject *ob) Py_ssize_t nArgs = PyTuple_GET_SIZE(ob); if (nArgs > CTYPES_MAX_ARGCOUNT) { Py_DECREF(ob); - PyErr_Format(PyExc_ArgError, + PyErr_Format(st->PyExc_ArgError, "_argtypes_ has too many arguments (%zi), maximum is %i", nArgs, CTYPES_MAX_ARGCOUNT); return NULL; @@ -2401,10 +2524,13 @@ converters_from_argtypes(PyObject *ob) * not bitfields, the bitfields check is also being disabled as a * precaution. - StgDictObject *stgdict = PyType_stgdict(tp); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, tp, &stginfo) < 0) { + return -1; + } - if (stgdict != NULL) { - if (stgdict->flags & TYPEFLAG_HASUNION) { + if (stginfo != NULL) { + if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2415,7 +2541,7 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - if (stgdict->flags & TYPEFLAG_HASBITFIELD) { + if (stginfo->flags & TYPEFLAG_HASBITFIELD) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2447,81 +2573,88 @@ converters_from_argtypes(PyObject *ob) } static int -make_funcptrtype_dict(StgDictObject *stgdict) +make_funcptrtype_dict(ctypes_state *st, PyObject *attrdict, StgInfo *stginfo) { PyObject *ob; PyObject *converters = NULL; - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->size = sizeof(void *); - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; - stgdict->ffi_type_pointer = ffi_type_pointer; + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->size = sizeof(void *); + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_flags_)); + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_flags_), &ob) < 0) { + return -1; + } if (!ob || !PyLong_Check(ob)) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, + PyErr_SetString(PyExc_TypeError, "class must define _flags_ which must be an integer"); - } + Py_XDECREF(ob); return -1; } - stgdict->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + stginfo->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + Py_DECREF(ob); /* _argtypes_ is optional... */ - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_argtypes_)); + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_argtypes_), &ob) < 0) { + return -1; + } if (ob) { - converters = converters_from_argtypes(ob); - if (!converters) + converters = converters_from_argtypes(st, ob); + if (!converters) { + Py_DECREF(ob); return -1; - stgdict->argtypes = Py_NewRef(ob); - stgdict->converters = converters; + } + stginfo->argtypes = ob; + stginfo->converters = converters; } - else if (PyErr_Occurred()) { + + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_restype_), &ob) < 0) { return -1; } - - ob = PyDict_GetItemWithError((PyObject *)stgdict, &_Py_ID(_restype_)); if (ob) { - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_restype_ must be a type, a callable, or None"); + Py_DECREF(ob); return -1; } - stgdict->restype = Py_NewRef(ob); + stginfo->restype = ob; if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), - &stgdict->checker) < 0) + &stginfo->checker) < 0) { return -1; } } - else if (PyErr_Occurred()) { +/* XXX later, maybe. + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py _ID(_errcheck_), &ob) < 0) { return -1; } -/* XXX later, maybe. - ob = _PyDict_GetItemIdWithError((PyObject *)stgdict, &PyId__errcheck_); if (ob) { if (!PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_errcheck_ must be callable"); + Py_DECREF(ob); return -1; } - stgdict->errcheck = Py_NewRef(ob); - } - else if (PyErr_Occurred()) { - return -1; + stginfo->errcheck = ob; } */ return 0; } static PyCArgObject * -PyCFuncPtrType_paramfunc(CDataObject *self) +PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2532,95 +2665,57 @@ PyCFuncPtrType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)&PyCStgDict_Type); - if (!stgdict) - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + Py_DECREF(attrdict); + return -1; + } + + stginfo->paramfunc = PyCFuncPtrType_paramfunc; - stgdict->paramfunc = PyCFuncPtrType_paramfunc; /* We do NOT expose the function signature in the format string. It is impossible, generally, because the only requirement for the argtypes items is that they have a .from_param method - we do not know the types of the arguments (although, in practice, most argtypes would be a ctypes type). - */ - stgdict->format = _ctypes_alloc_format_string(NULL, "X{}"); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - stgdict->flags |= TYPEFLAG_ISPOINTER; - - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated storage dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; + */ + stginfo->format = _ctypes_alloc_format_string(NULL, "X{}"); + if (stginfo->format == NULL) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->flags |= TYPEFLAG_ISPOINTER; - if (-1 == make_funcptrtype_dict(stgdict)) { - Py_DECREF(result); - return NULL; + if (make_funcptrtype_dict(st, attrdict, stginfo) < 0) { + Py_DECREF(attrdict); + return -1; } - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } -PyTypeObject PyCFuncPtrType_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.PyCFuncPtrType", /* tp_name */ - 0, /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &CDataType_as_sequence, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PyDoc_STR("metatype for C function pointers"), /* tp_doc */ - (traverseproc)CDataType_traverse, /* tp_traverse */ - (inquiry)CDataType_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - CDataType_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtrType_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycfuncptr_type_slots[] = { + {Py_tp_doc, PyDoc_STR("metatype for C function pointers")}, + {Py_tp_methods, CDataType_methods}, + {Py_tp_init, PyCFuncPtrType_init}, + {0, NULL}, +}; + +static PyType_Spec pycfuncptr_type_spec = { + .name = "_ctypes.PyCFuncPtrType", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_type_slots, }; @@ -2732,11 +2827,20 @@ KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep) /* PyCData_Type */ + +/*[clinic input] +class _ctypes.PyCData "PyObject *" "clinic_state()->PyCData_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac13df38dee3c22c]*/ + + static int PyCData_traverse(CDataObject *self, visitproc visit, void *arg) { Py_VISIT(self->b_objects); Py_VISIT((PyObject *)self->b_base); + PyTypeObject *type = Py_TYPE(self); + Py_VISIT(type); return 0; } @@ -2755,8 +2859,11 @@ PyCData_clear(CDataObject *self) static void PyCData_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); PyCData_clear((CDataObject *)self); - Py_TYPE(self)->tp_free(self); + type->tp_free(self); + Py_DECREF(type); } static PyMemberDef PyCData_members[] = { @@ -2774,18 +2881,20 @@ static PyMemberDef PyCData_members[] = { /* Find the innermost type of an array type, returning a borrowed reference */ static PyObject * -PyCData_item_type(PyObject *type) +PyCData_item_type(ctypes_state *st, PyObject *type) { - if (PyCArrayTypeObject_Check(type)) { - StgDictObject *stg_dict; + if (PyCArrayTypeObject_Check(st, type)) { PyObject *elem_type; /* asserts used here as these are all guaranteed by construction */ - stg_dict = PyType_stgdict(type); - assert(stg_dict); - elem_type = stg_dict->proto; + StgInfo *stg_info; + if (PyStgInfo_FromType(st, type, &stg_info) < 0) { + return NULL; + } + assert(stg_info); + elem_type = stg_info->proto; assert(elem_type); - return PyCData_item_type(elem_type); + return PyCData_item_type(st, elem_type); } else { return type; @@ -2796,32 +2905,42 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; - StgDictObject *dict = PyObject_stgdict(myself); - PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); - StgDictObject *item_dict = PyType_stgdict(item_type); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return -1; + } + assert(info); + + PyObject *item_type = PyCData_item_type(st, (PyObject*)Py_TYPE(myself)); + if (item_type == NULL) { + return 0; + } if (view == NULL) return 0; + StgInfo *item_info; + if (PyStgInfo_FromType(st, item_type, &item_info) < 0) { + return -1; + } + assert(item_info); + view->buf = self->b_ptr; view->obj = Py_NewRef(myself); view->len = self->b_size; view->readonly = 0; /* use default format character if not set */ - view->format = dict->format ? dict->format : "B"; - view->ndim = dict->ndim; - view->shape = dict->shape; - view->itemsize = item_dict->size; + view->format = info->format ? info->format : "B"; + view->ndim = info->ndim; + view->shape = info->shape; + view->itemsize = item_info->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; return 0; } -static PyBufferProcs PyCData_as_buffer = { - PyCData_NewGetBuffer, - NULL, -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2832,12 +2951,28 @@ PyCData_nohash(PyObject *self) return -1; } +/*[clinic input] +_ctypes.PyCData.__reduce__ as PyCData_reduce + + myself: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -PyCData_reduce(PyObject *myself, PyObject *args) +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls) +/*[clinic end generated code: output=1a025ccfdd8c935d input=34097a5226ea63c1]*/ { CDataObject *self = (CDataObject *)myself; - if (PyObject_stgdict(myself)->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return NULL; + } + assert(info); + + if (info->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { PyErr_SetString(PyExc_ValueError, "ctypes objects containing pointers cannot be pickled"); return NULL; @@ -2846,7 +2981,7 @@ PyCData_reduce(PyObject *myself, PyObject *args) if (dict == NULL) { return NULL; } - return Py_BuildValue("O(O(NN))", _unpickle, Py_TYPE(myself), dict, + return Py_BuildValue("O(O(NN))", st->_unpickle, Py_TYPE(myself), dict, PyBytes_FromStringAndSize(self->b_ptr, self->b_size)); } @@ -2895,56 +3030,35 @@ PyCData_from_outparam(PyObject *self, PyObject *args) static PyMethodDef PyCData_methods[] = { { "__ctypes_from_outparam__", PyCData_from_outparam, METH_NOARGS, }, - { "__reduce__", PyCData_reduce, METH_NOARGS, }, + PYCDATA_REDUCE_METHODDEF { "__setstate__", PyCData_setstate, METH_VARARGS, }, { NULL, NULL }, }; -PyTypeObject PyCData_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._CData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - PyCData_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyCData_nohash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCData_methods, /* tp_methods */ - PyCData_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycdata_slots[] = { + {Py_tp_dealloc, PyCData_dealloc}, + {Py_tp_hash, PyCData_nohash}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_traverse, PyCData_traverse}, + {Py_tp_clear, PyCData_clear}, + {Py_tp_methods, PyCData_methods}, + {Py_tp_members, PyCData_members}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycdata_spec = { + .name = "_ctypes._CData", + .basicsize = sizeof(CDataObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = pycdata_slots, }; -static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) +static int +PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) { - if ((size_t)dict->size <= sizeof(obj->b_value)) { + if ((size_t)info->size <= sizeof(obj->b_value)) { /* No need to call malloc, can use the default buffer */ obj->b_ptr = (char *)&obj->b_value; /* The b_needsfree flag does not mean that we actually did @@ -2958,51 +3072,56 @@ static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) /* In python 2.4, and ctypes 0.9.6, the malloc call took about 33% of the creation time for c_int(). */ - obj->b_ptr = (char *)PyMem_Malloc(dict->size); + obj->b_ptr = (char *)PyMem_Malloc(info->size); if (obj->b_ptr == NULL) { PyErr_NoMemory(); return -1; } obj->b_needsfree = 1; - memset(obj->b_ptr, 0, dict->size); + memset(obj->b_ptr, 0, info->size); } - obj->b_size = dict->size; + obj->b_size = info->size; return 0; } PyObject * -PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) +PyCData_FromBaseObj(ctypes_state *st, + PyObject *type, PyObject *base, Py_ssize_t index, char *adr) { CDataObject *cmem; - StgDictObject *dict; assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; cmem = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); - if (cmem == NULL) + if (cmem == NULL) { return NULL; - assert(CDataObject_Check(cmem)); - - cmem->b_length = dict->length; - cmem->b_size = dict->size; + } + assert(CDataObject_Check(st, cmem)); + cmem->b_length = info->length; + cmem->b_size = info->size; if (base) { /* use base's buffer */ - assert(CDataObject_Check(base)); + assert(CDataObject_Check(st, base)); cmem->b_ptr = adr; cmem->b_needsfree = 0; cmem->b_base = (CDataObject *)Py_NewRef(base); cmem->b_index = index; } else { /* copy contents of adr */ - if (-1 == PyCData_MallocBuffer(cmem, dict)) { + if (-1 == PyCData_MallocBuffer(cmem, info)) { Py_DECREF(cmem); return NULL; } - memcpy(cmem->b_ptr, adr, dict->size); + memcpy(cmem->b_ptr, adr, info->size); cmem->b_index = index; } return (PyObject *)cmem; @@ -3012,31 +3131,36 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) Box a memory block into a CData instance. */ PyObject * -PyCData_AtAddress(PyObject *type, void *buf) +PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf) { CDataObject *pd; - StgDictObject *dict; if (PySys_Audit("ctypes.cdata", "n", (Py_ssize_t)buf) < 0) { return NULL; } assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; pd = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); - if (!pd) + if (!pd) { return NULL; - assert(CDataObject_Check(pd)); + } + assert(CDataObject_Check(st, pd)); pd->b_ptr = (char *)buf; - pd->b_length = dict->length; - pd->b_size = dict->size; + pd->b_length = info->length; + pd->b_size = info->size; return (PyObject *)pd; } @@ -3045,46 +3169,54 @@ PyCData_AtAddress(PyObject *type, void *buf) classes. FALSE otherwise FALSE also for subclasses of c_int and such. */ -int _ctypes_simple_instance(PyObject *obj) +int _ctypes_simple_instance(ctypes_state *st, PyObject *obj) { PyTypeObject *type = (PyTypeObject *)obj; - if (PyCSimpleTypeObject_Check(type)) - return type->tp_base != &Simple_Type; + if (PyCSimpleTypeObject_Check(st, type)) { + return type->tp_base != st->Simple_Type; + } return 0; } PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { - StgDictObject *dict; if (getfunc) return getfunc(adr, size); assert(type); - dict = PyType_stgdict(type); - if (dict && dict->getfunc && !_ctypes_simple_instance(type)) - return dict->getfunc(adr, size); - return PyCData_FromBaseObj(type, src, index, adr); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->getfunc && !_ctypes_simple_instance(st, type)) { + return info->getfunc(adr, size); + } + return PyCData_FromBaseObj(st, type, src, index, adr); } /* Helper function for PyCData_set below. */ static PyObject * -_PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +_PyCData_set(ctypes_state *st, + CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t size, char *ptr) { CDataObject *src; int err; - if (setfunc) + if (setfunc) { return setfunc(ptr, value, size); - - if (!CDataObject_Check(value)) { - StgDictObject *dict = PyType_stgdict(type); - if (dict && dict->setfunc) - return dict->setfunc(ptr, value, size); + } + if (!CDataObject_Check(st, value)) { + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->setfunc) + return info->setfunc(ptr, value, size); /* If value is a tuple, we try to call the type with the tuple and use the result! @@ -3099,11 +3231,11 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, ((PyTypeObject *)type)->tp_name); return NULL; } - result = _PyCData_set(dst, type, setfunc, ob, + result = _PyCData_set(st, dst, type, setfunc, ob, size, ptr); Py_DECREF(ob); return result; - } else if (value == Py_None && PyCPointerTypeObject_Check(type)) { + } else if (value == Py_None && PyCPointerTypeObject_Check(st, type)) { *(void **)ptr = NULL; Py_RETURN_NONE; } else { @@ -3124,7 +3256,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, src->b_ptr, size); - if (PyCPointerTypeObject_Check(type)) { + if (PyCPointerTypeObject_Check(st, type)) { /* XXX */ } @@ -3135,13 +3267,18 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return Py_NewRef(value); } - if (PyCPointerTypeObject_Check(type) - && ArrayObject_Check(value)) { - StgDictObject *p1, *p2; + if (PyCPointerTypeObject_Check(st, type) + && ArrayObject_Check(st, value)) { PyObject *keep; - p1 = PyObject_stgdict(value); + + StgInfo *p1, *p2; + if (PyStgInfo_FromObject(st, value, &p1) < 0) { + return NULL; + } assert(p1); /* Cannot be NULL for array instances */ - p2 = PyType_stgdict(type); + if (PyStgInfo_FromType(st, type, &p2) < 0) { + return NULL; + } assert(p2); /* Cannot be NULL for pointer types */ if (p1->proto != p2->proto) { @@ -3179,19 +3316,20 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, * to the value 'value'. */ int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr) { CDataObject *mem = (CDataObject *)dst; PyObject *result; - if (!CDataObject_Check(dst)) { + if (!CDataObject_Check(st, dst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return -1; } - result = _PyCData_set(mem, type, setfunc, value, + result = _PyCData_set(st, mem, type, setfunc, value, size, ptr); if (result == NULL) return -1; @@ -3206,17 +3344,28 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, /******************************************************************/ static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return generic_pycdata_new(st, type, args, kwds); +} + +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds) { CDataObject *obj; - StgDictObject *dict; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; obj = (CDataObject *)type->tp_alloc(type, 0); if (!obj) @@ -3225,9 +3374,9 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) obj->b_base = NULL; obj->b_index = 0; obj->b_objects = NULL; - obj->b_length = dict->length; + obj->b_length = info->length; - if (-1 == PyCData_MallocBuffer(obj, dict)) { + if (-1 == PyCData_MallocBuffer(obj, info)) { Py_DECREF(obj); return NULL; } @@ -3271,7 +3420,12 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign Py_XDECREF(oldchecker); return 0; } - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; @@ -3290,14 +3444,17 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign static PyObject * PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->restype) { return Py_NewRef(self->restype); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->restype) { - return Py_NewRef(dict->restype); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->restype) { + return Py_NewRef(info->restype); } else { Py_RETURN_NONE; } @@ -3312,7 +3469,8 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { - converters = converters_from_argtypes(ob); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + converters = converters_from_argtypes(st, ob); if (!converters) return -1; Py_XSETREF(self->converters, converters); @@ -3325,14 +3483,17 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig static PyObject * PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->argtypes) { return Py_NewRef(self->argtypes); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->argtypes) { - return Py_NewRef(dict->argtypes); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->argtypes) { + return Py_NewRef(info->argtypes); } else { Py_RETURN_NONE; } @@ -3364,7 +3525,6 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) #else char *mangled_name; int i; - StgDictObject *dict; Py_BEGIN_ALLOW_THREADS address = (PPROC)GetProcAddress(handle, name); @@ -3375,9 +3535,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) return NULL; } - dict = PyType_stgdict((PyObject *)type); - /* It should not happen that dict is NULL, but better be safe */ - if (dict==NULL || dict->flags & FUNCFLAG_CDECL) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + /* It should not happen that info is NULL, but better be safe */ + if (info==NULL || info->flags & FUNCFLAG_CDECL) return address; /* for stdcall, try mangled names: @@ -3402,22 +3566,23 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) /* Return 1 if usable, 0 else and exception set. */ static int -_check_outarg_type(PyObject *arg, Py_ssize_t index) +_check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) { - StgDictObject *dict; - - if (PyCPointerTypeObject_Check(arg)) + if (PyCPointerTypeObject_Check(st, arg)) { return 1; - - if (PyCArrayTypeObject_Check(arg)) + } + if (PyCArrayTypeObject_Check(st, arg)) { return 1; - - dict = PyType_stgdict(arg); - if (dict + } + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return -1; + } + if (info /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ - && PyUnicode_Check(dict->proto) + && PyUnicode_Check(info->proto) /* We only allow c_void_p, c_char_p and c_wchar_p as a simple output parameter type */ - && (strchr("PzZ", PyUnicode_AsUTF8(dict->proto)[0]))) { + && (strchr("PzZ", PyUnicode_AsUTF8(info->proto)[0]))) { return 1; } @@ -3432,21 +3597,23 @@ _check_outarg_type(PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) { Py_ssize_t i, len; - StgDictObject *dict; PyObject *argtypes; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return 0; } - argtypes = dict->argtypes; + argtypes = info->argtypes; - if (paramflags == NULL || dict->argtypes == NULL) + if (paramflags == NULL || info->argtypes == NULL) return 1; if (!PyTuple_Check(paramflags)) { @@ -3456,7 +3623,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(dict->argtypes)) { + if (len != PyTuple_GET_SIZE(info->argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3483,7 +3650,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) case PARAMFLAG_FIN | PARAMFLAG_FOUT: break; case PARAMFLAG_FOUT: - if (!_check_outarg_type(typ, i+1)) + if (!_check_outarg_type(st, typ, i+1)) return 0; break; default: @@ -3615,12 +3782,13 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif - if (!_validate_paramflags(type, paramflags)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); return NULL; } - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (!self) { Py_DECREF(ftuple); return NULL; @@ -3656,10 +3824,11 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) if (paramflags == Py_None) paramflags = NULL; - if (!_validate_paramflags(type, paramflags)) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { return NULL; - - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + } + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); self->index = index + 0x1000; self->paramflags = Py_XNewRef(paramflags); if (iid_len == sizeof(GUID)) @@ -3686,7 +3855,6 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyCFuncPtrObject *self; PyObject *callable; - StgDictObject *dict; CThunkObject *thunk; if (PyTuple_GET_SIZE(args) == 0) @@ -3737,23 +3905,28 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } */ - dict = PyType_stgdict((PyObject *)type); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } /* XXXX Fails if we do: 'PyCFuncPtr(lambda x: x)' */ - if (!dict || !dict->argtypes) { + if (!info || !info->argtypes) { PyErr_SetString(PyExc_TypeError, "cannot construct instance of this class:" " no argtypes"); return NULL; } - thunk = _ctypes_alloc_callback(callable, - dict->argtypes, - dict->restype, - dict->flags); + thunk = _ctypes_alloc_callback(st, + callable, + info->argtypes, + info->restype, + info->flags); if (!thunk) return NULL; - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (self == NULL) { Py_DECREF(thunk); return NULL; @@ -3777,16 +3950,17 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _byref consumes a refcount to its argument */ static PyObject * -_byref(PyObject *obj) +_byref(ctypes_state *st, PyObject *obj) { PyCArgObject *parg; - if (!CDataObject_Check(obj)) { + + if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "expected CData instance"); return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; @@ -3810,13 +3984,12 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje return Py_NewRef(v); } if (kwds && name) { - v = PyDict_GetItemWithError(kwds, name); + if (PyDict_GetItemRef(kwds, name, &v) < 0) { + return NULL; + } if (v) { ++*pindex; - return Py_NewRef(v); - } - else if (PyErr_Occurred()) { - return NULL; + return v; } } if (defval) { @@ -3851,13 +4024,12 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje function. */ static PyObject * -_build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, +_build_callargs(ctypes_state *st, PyCFuncPtrObject *self, PyObject *argtypes, PyObject *inargs, PyObject *kwds, int *poutmask, int *pinoutmask, unsigned int *pnumretvals) { PyObject *paramflags = self->paramflags; PyObject *callargs; - StgDictObject *dict; Py_ssize_t i, len; int inargs_index = 0; /* It's a little bit difficult to determine how many arguments the @@ -3917,7 +4089,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, case (PARAMFLAG_FIN | PARAMFLAG_FOUT): *pinoutmask |= (1 << i); /* mark as inout arg */ (*pnumretvals)++; - /* fall through */ + _Py_FALLTHROUGH; case 0: case PARAMFLAG_FIN: /* 'in' parameter. Copy it from inargs. */ @@ -3945,26 +4117,31 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, break; } ob = PyTuple_GET_ITEM(argtypes, i); - dict = PyType_stgdict(ob); - if (dict == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + goto error; + } + if (info == NULL) { /* Cannot happen: _validate_paramflags() would not accept such an object */ PyErr_Format(PyExc_RuntimeError, - "NULL stgdict unexpected"); + "NULL stginfo unexpected"); goto error; } - if (PyUnicode_Check(dict->proto)) { + if (PyUnicode_Check(info->proto)) { PyErr_Format( PyExc_TypeError, "%s 'out' parameter must be passed as default value", ((PyTypeObject *)ob)->tp_name); goto error; } - if (PyCArrayTypeObject_Check(ob)) + if (PyCArrayTypeObject_Check(st, ob)) { ob = _PyObject_CallNoArgs(ob); - else + } + else { /* Create an instance of the pointed-to type */ - ob = _PyObject_CallNoArgs(dict->proto); + ob = _PyObject_CallNoArgs(info->proto); + } /* XXX Is the following correct any longer? We must not pass a byref() to the array then but @@ -4083,7 +4260,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) PyObject *converters; PyObject *checker; PyObject *argtypes; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); PyObject *result; PyObject *callargs; PyObject *errcheck; @@ -4096,13 +4272,19 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int outmask; unsigned int numretvals; - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : dict->restype; - converters = self->converters ? self->converters : dict->converters; - checker = self->checker ? self->checker : dict->checker; - argtypes = self->argtypes ? self->argtypes : dict->argtypes; -/* later, we probably want to have an errcheck field in stgdict */ - errcheck = self->errcheck /* ? self->errcheck : dict->errcheck */; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + + restype = self->restype ? self->restype : info->restype; + converters = self->converters ? self->converters : info->converters; + checker = self->checker ? self->checker : info->checker; + argtypes = self->argtypes ? self->argtypes : info->argtypes; +/* later, we probably want to have an errcheck field in stginfo */ + errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; pProc = *(void **)self->b_ptr; @@ -4116,7 +4298,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) "native com method call without 'this' parameter"); return NULL; } - if (!CDataObject_Check(this)) { + if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); return NULL; @@ -4137,7 +4319,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } #endif - callargs = _build_callargs(self, argtypes, + callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); if (callargs == NULL) @@ -4149,7 +4331,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int actual = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(callargs), Py_ssize_t, int); - if ((dict->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { + if ((info->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { /* For cdecl functions, we allow more actual arguments than the length of the argtypes tuple. */ @@ -4173,13 +4355,14 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) } } - result = _ctypes_callproc(pProc, + result = _ctypes_callproc(st, + pProc, callargs, #ifdef MS_WIN32 piunk, self->iid, #endif - dict->flags, + info->flags, converters, restype, checker); @@ -4239,8 +4422,11 @@ PyCFuncPtr_clear(PyCFuncPtrObject *self) static void PyCFuncPtr_dealloc(PyCFuncPtrObject *self) { + PyObject_GC_UnTrack(self); PyCFuncPtr_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + PyTypeObject *type = Py_TYPE(self); + type->tp_free((PyObject *)self); + Py_DECREF(type); } static PyObject * @@ -4268,59 +4454,26 @@ PyCFuncPtr_bool(PyCFuncPtrObject *self) ); } -static PyNumberMethods PyCFuncPtr_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)PyCFuncPtr_bool, /* nb_bool */ +static PyType_Slot pycfuncptr_slots[] = { + {Py_tp_dealloc, PyCFuncPtr_dealloc}, + {Py_tp_repr, PyCFuncPtr_repr}, + {Py_tp_call, PyCFuncPtr_call}, + {Py_tp_doc, PyDoc_STR("Function Pointer")}, + {Py_tp_traverse, PyCFuncPtr_traverse}, + {Py_tp_clear, PyCFuncPtr_clear}, + {Py_tp_getset, PyCFuncPtr_getsets}, + {Py_tp_new, PyCFuncPtr_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, PyCFuncPtr_bool}, + {0, NULL}, }; -PyTypeObject PyCFuncPtr_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.CFuncPtr", - sizeof(PyCFuncPtrObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PyCFuncPtr_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)PyCFuncPtr_repr, /* tp_repr */ - &PyCFuncPtr_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)PyCFuncPtr_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Function Pointer"), /* tp_doc */ - (traverseproc)PyCFuncPtr_traverse, /* tp_traverse */ - (inquiry)PyCFuncPtr_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCFuncPtr_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtr_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycfuncptr_spec = { + .name = "_ctypes.CFuncPtr", + .basicsize = sizeof(PyCFuncPtrObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_slots, }; /*****************************************************************/ @@ -4340,11 +4493,15 @@ _init_pos_args(PyObject *self, PyTypeObject *type, PyObject *args, PyObject *kwds, Py_ssize_t index) { - StgDictObject *dict; PyObject *fields; Py_ssize_t i; - if (PyType_stgdict((PyObject *)type->tp_base)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)type->tp_base, &baseinfo) < 0) { + return -1; + } + if (baseinfo) { index = _init_pos_args(self, type->tp_base, args, kwds, index); @@ -4352,8 +4509,17 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return -1; } - dict = PyType_stgdict((PyObject *)type); - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + assert(info); + + PyObject *attrdict = PyType_GetDict(type); + assert(attrdict); + + fields = PyDict_GetItemWithError((PyObject *)attrdict, &_Py_ID(_fields_)); + Py_CLEAR(attrdict); if (fields == NULL) { if (PyErr_Occurred()) { return -1; @@ -4361,10 +4527,10 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return index; } - for (i = 0; - i < dict->length && (i+index) < PyTuple_GET_SIZE(args); + for (i = index; + i < info->length && i < PyTuple_GET_SIZE(args); ++i) { - PyObject *pair = PySequence_GetItem(fields, i); + PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; int res; if (!pair) @@ -4374,7 +4540,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, Py_DECREF(pair); return -1; } - val = PyTuple_GET_ITEM(args, i + index); + val = PyTuple_GET_ITEM(args, i); if (kwds) { res = PyDict_Contains(kwds, name); if (res != 0) { @@ -4395,7 +4561,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return index + dict->length; + return info->length; } static int @@ -4432,88 +4598,34 @@ Struct_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject Struct_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Structure", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Structure base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycstruct_slots[] = { + {Py_tp_doc, PyDoc_STR("Structure base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycstruct_spec = { + .name = "_ctypes.Structure", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_slots, +}; + +static PyType_Slot pycunion_slots[] = { + {Py_tp_doc, PyDoc_STR("Union base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, }; -static PyTypeObject Union_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Union", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Union base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycunion_spec = { + .name = "_ctypes.Union", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycunion_slots, }; @@ -4547,8 +4659,6 @@ Array_item(PyObject *myself, Py_ssize_t index) { CDataObject *self = (CDataObject *)myself; Py_ssize_t offset, size; - StgDictObject *stgdict; - if (index < 0 || index >= self->b_length) { PyErr_SetString(PyExc_IndexError, @@ -4556,15 +4666,19 @@ Array_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + /* Would it be clearer if we got the item size from - stgdict->proto's stgdict? + stginfo->proto's stginfo? */ - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; - return PyCData_get(stgdict->proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(st, stginfo->proto, stginfo->getfunc, (PyObject *)self, index, size, self->b_ptr + offset); } @@ -4583,7 +4697,6 @@ Array_subscript(PyObject *myself, PyObject *item) return Array_item(myself, i); } else if (PySlice_Check(item)) { - StgDictObject *stgdict, *itemdict; PyObject *proto; PyObject *np; Py_ssize_t start, stop, step, slicelen, i; @@ -4594,14 +4707,21 @@ Array_subscript(PyObject *myself, PyObject *item) } slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - proto = stgdict->proto; - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the array, a + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + proto = stginfo->proto; + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the array, a ctypes type, so this cannot be NULL */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = (char *)self->b_ptr; char *dest; @@ -4625,7 +4745,7 @@ Array_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = (wchar_t *)self->b_ptr; wchar_t *dest; @@ -4680,7 +4800,6 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) { CDataObject *self = (CDataObject *)myself; Py_ssize_t size, offset; - StgDictObject *stgdict; char *ptr; if (value == NULL) { @@ -4689,18 +4808,23 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - if (index < 0 || index >= stgdict->length) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + + if (index < 0 || index >= stginfo->length) { PyErr_SetString(PyExc_IndexError, "invalid index"); return -1; } - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; ptr = self->b_ptr + offset; - return PyCData_set((PyObject *)self, stgdict->proto, stgdict->setfunc, value, + return PyCData_set(st, (PyObject *)self, stginfo->proto, stginfo->setfunc, value, index, size, ptr); } @@ -4774,26 +4898,6 @@ static PyMethodDef Array_methods[] = { { NULL, NULL } }; -static PySequenceMethods Array_as_sequence = { - Array_length, /* sq_length; */ - 0, /* sq_concat; */ - 0, /* sq_repeat; */ - Array_item, /* sq_item; */ - 0, /* sq_slice; */ - Array_ass_item, /* sq_ass_item; */ - 0, /* sq_ass_slice; */ - 0, /* sq_contains; */ - - 0, /* sq_inplace_concat; */ - 0, /* sq_inplace_repeat; */ -}; - -static PyMappingMethods Array_as_mapping = { - Array_length, - Array_subscript, - Array_ass_subscript, -}; - PyDoc_STRVAR(array_doc, "Abstract base class for arrays.\n" "\n" @@ -4804,60 +4908,40 @@ PyDoc_STRVAR(array_doc, "reads, the resulting object is not itself an Array." ); -PyTypeObject PyCArray_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Array", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &Array_as_sequence, /* tp_as_sequence */ - &Array_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - array_doc, /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Array_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Array_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_slots[] = { + {Py_tp_doc, (char*)array_doc}, + {Py_tp_methods, Array_methods}, + {Py_tp_init, Array_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_sq_length, Array_length}, + {Py_sq_item, Array_item}, + {Py_sq_ass_item, Array_ass_item}, + {Py_mp_length, Array_length}, + {Py_mp_subscript, Array_subscript}, + {Py_mp_ass_subscript, Array_ass_subscript}, + {0, NULL}, +}; + +static PyType_Spec pycarray_spec = { + .name = "_ctypes.Array", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_slots, }; PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) { - static PyObject *cache; PyObject *key; char name[256]; PyObject *len; - if (cache == NULL) { - cache = PyDict_New(); - if (cache == NULL) + if (st->array_cache == NULL) { + st->array_cache = PyDict_New(); + if (st->array_cache == NULL) { return NULL; + } } len = PyLong_FromSsize_t(length); if (len == NULL) @@ -4868,15 +4952,12 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) return NULL; PyObject *result; - if (_PyDict_GetItemProxy(cache, key, &result) < 0) { - Py_DECREF(key); - return NULL; - } - if (result) { + if (_PyDict_GetItemProxy(st->array_cache, key, &result) != 0) { + // found or error Py_DECREF(key); return result; } - + // not found if (!PyType_Check(itemtype)) { PyErr_SetString(PyExc_TypeError, "Expected a type object"); @@ -4890,11 +4971,10 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) sprintf(name, "%.200s_Array_%ld", ((PyTypeObject *)itemtype)->tp_name, (long)length); #endif - - result = PyObject_CallFunction((PyObject *)&PyCArrayType_Type, + result = PyObject_CallFunction((PyObject *)st->PyCArrayType_Type, "s(O){s:n,s:O}", name, - &PyCArray_Type, + st->PyCArray_Type, "_length_", length, "_type_", @@ -4904,7 +4984,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Py_DECREF(key); return NULL; } - if (-1 == PyDict_SetItemProxy(cache, key, result)) { + if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) { Py_DECREF(key); Py_DECREF(result); return NULL; @@ -4919,20 +4999,32 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Simple_Type */ +/*[clinic input] +class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ + + static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { PyObject *result; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->setfunc); - result = dict->setfunc(self->b_ptr, value, dict->size); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return -1; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->setfunc); + + result = info->setfunc(self->b_ptr, value, info->size); if (!result) return -1; @@ -4954,11 +5046,14 @@ Simple_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->getfunc); - return dict->getfunc(self->b_ptr, self->b_size); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->getfunc); + return info->getfunc(self->b_ptr, self->b_size); } static PyGetSetDef Simple_getsets[] = { @@ -4967,18 +5062,28 @@ static PyGetSetDef Simple_getsets[] = { { NULL, NULL } }; +/*[clinic input] +_ctypes.Simple.__ctypes_from_outparam__ as Simple_from_outparm + + self: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -Simple_from_outparm(PyObject *self, PyObject *args) +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=6c61d90da8aa9b4f input=0f362803fb4629d5]*/ { - if (_ctypes_simple_instance((PyObject *)Py_TYPE(self))) { + ctypes_state *st = get_module_state_by_class(cls); + if (_ctypes_simple_instance(st, (PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } - /* call stgdict->getfunc */ + /* call stginfo->getfunc */ return Simple_get_value((CDataObject *)self, NULL); } static PyMethodDef Simple_methods[] = { - { "__ctypes_from_outparam__", Simple_from_outparm, METH_NOARGS, }, + SIMPLE_FROM_OUTPARM_METHODDEF { NULL, NULL }, }; @@ -4987,26 +5092,14 @@ static int Simple_bool(CDataObject *self) return memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); } -static PyNumberMethods Simple_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Simple_bool, /* nb_bool */ -}; - /* "%s(%s)" % (self.__class__.__name__, self.value) */ static PyObject * Simple_repr(CDataObject *self) { PyObject *val, *result; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); - if (Py_TYPE(self)->tp_base != &Simple_Type) { + if (Py_TYPE(self)->tp_base != st->Simple_Type) { return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(self)->tp_name, self); } @@ -5021,48 +5114,26 @@ Simple_repr(CDataObject *self) return result; } -static PyTypeObject Simple_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._SimpleCData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)&Simple_repr, /* tp_repr */ - &Simple_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Simple_methods, /* tp_methods */ - 0, /* tp_members */ - Simple_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Simple_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_slots[] = { + {Py_tp_repr, &Simple_repr}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_methods, Simple_methods}, + {Py_tp_getset, Simple_getsets}, + {Py_tp_init, Simple_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Simple_bool}, + {0, NULL}, +}; + +static PyType_Spec pycsimple_spec = { + .name = "_ctypes._SimpleCData", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_slots, }; + /******************************************************************/ /* PyCPointer_Type @@ -5073,7 +5144,6 @@ Pointer_item(PyObject *myself, Py_ssize_t index) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (*(void **)self->b_ptr == NULL) { @@ -5082,19 +5152,27 @@ Pointer_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer object instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer object instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the pointer, a ctypes + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the pointer, a ctypes type, so this cannot be NULL */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_get(proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(st, proto, stginfo->getfunc, (PyObject *)self, index, size, (*(char **)self->b_ptr) + offset); } @@ -5104,7 +5182,6 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (value == NULL) { @@ -5119,37 +5196,47 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* Cannot be NULL because the itemtype of a pointer + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return -1; + } + assert(iteminfo); /* Cannot be NULL because the itemtype of a pointer is always a ctypes type */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_set((PyObject *)self, proto, stgdict->setfunc, value, + return PyCData_set(st, (PyObject *)self, proto, stginfo->setfunc, value, index, size, (*(char **)self->b_ptr) + offset); } static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { - StgDictObject *stgdict; - if (*(void **)self->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - return PyCData_FromBaseObj(stgdict->proto, + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + + return PyCData_FromBaseObj(st, stginfo->proto, (PyObject *)self, 0, *(void **)self->b_ptr); } @@ -5157,7 +5244,6 @@ Pointer_get_contents(CDataObject *self, void *closure) static int Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) { - StgDictObject *stgdict; CDataObject *dst; PyObject *keep; @@ -5166,17 +5252,21 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) "Pointer does not support item deletion"); return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - assert(stgdict->proto); - if (!CDataObject_Check(value)) { - int res = PyObject_IsInstance(value, stgdict->proto); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + assert(stginfo->proto); + if (!CDataObject_Check(st, value)) { + int res = PyObject_IsInstance(value, stginfo->proto); if (res == -1) return -1; if (!res) { PyErr_Format(PyExc_TypeError, "expected %s instead of %s", - ((PyTypeObject *)(stgdict->proto))->tp_name, + ((PyTypeObject *)(stginfo->proto))->tp_name, Py_TYPE(value)->tp_name); return -1; } @@ -5224,13 +5314,17 @@ Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - StgDictObject *dict = PyType_stgdict((PyObject *)type); - if (!dict || !dict->proto) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info || !info->proto) { PyErr_SetString(PyExc_TypeError, "Cannot create instance: has no _type_"); return NULL; } - return GenericPyCData_new(type, args, kw); + return generic_pycdata_new(st, type, args, kw); } static PyObject * @@ -5247,7 +5341,6 @@ Pointer_subscript(PyObject *myself, PyObject *item) PySliceObject *slice = (PySliceObject *)item; Py_ssize_t start, stop, step; PyObject *np; - StgDictObject *stgdict, *itemdict; PyObject *proto; Py_ssize_t i, len; size_t cur; @@ -5301,13 +5394,20 @@ Pointer_subscript(PyObject *myself, PyObject *item) else len = (stop - start + 1) / step + 1; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = *(char **)self->b_ptr; char *dest; @@ -5327,7 +5427,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = *(wchar_t **)self->b_ptr; wchar_t *dest; @@ -5365,87 +5465,32 @@ Pointer_subscript(PyObject *myself, PyObject *item) } } -static PySequenceMethods Pointer_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - 0, /* intargfunc sq_repeat; */ - Pointer_item, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - Pointer_ass_item, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - /* Added in release 2.0 */ - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - -static PyMappingMethods Pointer_as_mapping = { - 0, - Pointer_subscript, -}; - static int Pointer_bool(CDataObject *self) { return (*(void **)self->b_ptr != NULL); } -static PyNumberMethods Pointer_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Pointer_bool, /* nb_bool */ +static PyType_Slot pycpointer_slots[] = { + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_getset, Pointer_getsets}, + {Py_tp_init, Pointer_init}, + {Py_tp_new, Pointer_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Pointer_bool}, + {Py_mp_subscript, Pointer_subscript}, + {Py_sq_item, Pointer_item}, + {Py_sq_ass_item, Pointer_ass_item}, + {0, NULL}, }; -PyTypeObject PyCPointer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._Pointer", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - &Pointer_as_number, /* tp_as_number */ - &Pointer_as_sequence, /* tp_as_sequence */ - &Pointer_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - Pointer_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Pointer_init, /* tp_init */ - 0, /* tp_alloc */ - Pointer_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycpointer_spec = { + .name = "_ctypes._Pointer", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_slots, }; - /******************************************************************/ /* * Module initialization. @@ -5548,18 +5593,21 @@ string_at(const char *ptr, int size) } static int -cast_check_pointertype(PyObject *arg) +cast_check_pointertype(ctypes_state *st, PyObject *arg) { - StgDictObject *dict; - - if (PyCPointerTypeObject_Check(arg)) + if (PyCPointerTypeObject_Check(st, arg)) { return 1; - if (PyCFuncPtrTypeObject_Check(arg)) + } + if (PyCFuncPtrTypeObject_Check(st, arg)) { return 1; - dict = PyType_stgdict(arg); - if (dict != NULL && dict->proto != NULL) { - if (PyUnicode_Check(dict->proto) - && (strchr("sPzUZXO", PyUnicode_AsUTF8(dict->proto)[0]))) { + } + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return 0; + } + if (info != NULL && info->proto != NULL) { + if (PyUnicode_Check(info->proto) + && (strchr("sPzUZXO", PyUnicode_AsUTF8(info->proto)[0]))) { /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ return 1; } @@ -5575,9 +5623,18 @@ cast_check_pointertype(PyObject *arg) static PyObject * cast(void *ptr, PyObject *src, PyObject *ctype) { + PyObject *mod = PyType_GetModuleByDef(Py_TYPE(ctype), &_ctypesmodule); + if (!mod) { + PyErr_SetString(PyExc_TypeError, + "cast() argument 2 must be a pointer type"); + return NULL; + } + ctypes_state *st = get_module_state(mod); + CDataObject *result; - if (0 == cast_check_pointertype(ctype)) + if (cast_check_pointertype(st, ctype) == 0) { return NULL; + } result = (CDataObject *)_PyObject_CallNoArgs(ctype); if (result == NULL) return NULL; @@ -5588,7 +5645,7 @@ cast(void *ptr, PyObject *src, PyObject *ctype) It must certainly contain the source objects one. It must contain the source object itself. */ - if (CDataObject_Check(src)) { + if (CDataObject_Check(st, src)) { CDataObject *obj = (CDataObject *)src; CDataObject *container; @@ -5641,15 +5698,6 @@ wstring_at(const wchar_t *ptr, int size) } -static struct PyModuleDef _ctypesmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ctypes", - .m_doc = _ctypes__doc__, - .m_size = -1, - .m_methods = _ctypes_module_methods, -}; - - static int _ctypes_add_types(PyObject *mod) { @@ -5657,26 +5705,8 @@ _ctypes_add_types(PyObject *mod) if (PyType_Ready(TYPE) < 0) { \ return -1; \ } - -#define TYPE_READY_BASE(TYPE_EXPR, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - type->tp_base = (TP_BASE); \ - TYPE_READY(type); \ - } while (0) - -#define MOD_ADD_TYPE(TYPE_EXPR, TP_TYPE, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - Py_SET_TYPE(type, TP_TYPE); \ - type->tp_base = TP_BASE; \ - if (PyModule_AddType(mod, type) < 0) { \ - return -1; \ - } \ - } while (0) - -#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ +#define CREATE_TYPE(TP, SPEC, META, BASE) do { \ + PyObject *type = PyType_FromMetaclass(META, mod, SPEC, \ (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ @@ -5684,62 +5714,82 @@ _ctypes_add_types(PyObject *mod) TP = (PyTypeObject *)type; \ } while (0) - ctypes_state *st = GLOBAL_STATE(); +#define MOD_ADD_TYPE(TP, SPEC, META, BASE) do { \ + CREATE_TYPE(TP, SPEC, META, BASE); \ + if (PyModule_AddType(mod, (PyTypeObject *)(TP)) < 0) { \ + return -1; \ + } \ +} while (0) + + ctypes_state *st = get_module_state(mod); /* Note: ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); - TYPE_READY(&PyCData_Type); - /* StgDict is derived from PyDict_Type */ - TYPE_READY_BASE(&PyCStgDict_Type, &PyDict_Type); + CREATE_TYPE(st->PyCArg_Type, &carg_spec, NULL, NULL); + CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, NULL, NULL); + CREATE_TYPE(st->PyCData_Type, &pycdata_spec, NULL, NULL); + + // Common Metaclass + CREATE_TYPE(st->PyCType_Type, &pyctype_type_spec, + NULL, &PyType_Type); /************************************************* * * Metaclasses */ - TYPE_READY_BASE(&PyCStructType_Type, &PyType_Type); - TYPE_READY_BASE(&UnionType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCPointerType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCArrayType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCSimpleType_Type, &PyType_Type); - TYPE_READY_BASE(&PyCFuncPtrType_Type, &PyType_Type); + CREATE_TYPE(st->PyCStructType_Type, &pycstruct_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->UnionType_Type, &union_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCPointerType_Type, &pycpointer_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCArrayType_Type, &pycarray_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCSimpleType_Type, &pycsimple_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, + NULL, st->PyCType_Type); /************************************************* * * Classes using a custom metaclass */ - MOD_ADD_TYPE(&Struct_Type, &PyCStructType_Type, &PyCData_Type); - MOD_ADD_TYPE(&Union_Type, &UnionType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCPointer_Type, &PyCPointerType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCArray_Type, &PyCArrayType_Type, &PyCData_Type); - MOD_ADD_TYPE(&Simple_Type, &PyCSimpleType_Type, &PyCData_Type); - MOD_ADD_TYPE(&PyCFuncPtr_Type, &PyCFuncPtrType_Type, &PyCData_Type); + MOD_ADD_TYPE(st->Struct_Type, &pycstruct_spec, + st->PyCStructType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Union_Type, &pycunion_spec, + st->UnionType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCPointer_Type, &pycpointer_spec, + st->PyCPointerType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCArray_Type, &pycarray_spec, + st->PyCArrayType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Simple_Type, &pycsimple_spec, + st->PyCSimpleType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCFuncPtr_Type, &pycfuncptr_spec, + st->PyCFuncPtrType_Type, st->PyCData_Type); /************************************************* * * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); + CREATE_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); + CREATE_TYPE(st->DictRemover_Type, &dictremover_spec, NULL, NULL); + CREATE_TYPE(st->StructParam_Type, &structparam_spec, NULL, NULL); #ifdef MS_WIN32 - CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); + CREATE_TYPE(st->PyComError_Type, &comerror_spec, NULL, PyExc_Exception); #endif #undef TYPE_READY -#undef TYPE_READY_BASE #undef MOD_ADD_TYPE #undef CREATE_TYPE return 0; @@ -5756,10 +5806,10 @@ _ctypes_add_objects(PyObject *mod) } \ } while (0) - MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); + ctypes_state *st = get_module_state(mod); + MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - ctypes_state *st = GLOBAL_STATE(); MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); @@ -5789,7 +5839,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("RTLD_LOCAL", PyLong_FromLong(RTLD_LOCAL)); MOD_ADD("RTLD_GLOBAL", PyLong_FromLong(RTLD_GLOBAL)); MOD_ADD("CTYPES_MAX_ARGCOUNT", PyLong_FromLong(CTYPES_MAX_ARGCOUNT)); - MOD_ADD("ArgumentError", Py_NewRef(PyExc_ArgError)); + MOD_ADD("ArgumentError", Py_NewRef(st->PyExc_ArgError)); MOD_ADD("SIZEOF_TIME_T", PyLong_FromSsize_t(SIZEOF_TIME_T)); return 0; #undef MOD_ADD @@ -5799,18 +5849,19 @@ _ctypes_add_objects(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - _unpickle = PyObject_GetAttrString(mod, "_unpickle"); - if (_unpickle == NULL) { + ctypes_state *st = get_module_state(mod); + st->_unpickle = PyObject_GetAttrString(mod, "_unpickle"); + if (st->_unpickle == NULL) { return -1; } - _ctypes_ptrtype_cache = PyDict_New(); - if (_ctypes_ptrtype_cache == NULL) { + st->_ctypes_ptrtype_cache = PyDict_New(); + if (st->_ctypes_ptrtype_cache == NULL) { return -1; } - PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); - if (!PyExc_ArgError) { + st->PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); + if (!st->PyExc_ArgError) { return -1; } @@ -5825,19 +5876,105 @@ _ctypes_mod_exec(PyObject *mod) } +static int +module_traverse(PyObject *module, visitproc visit, void *arg) { + ctypes_state *st = get_module_state(module); + Py_VISIT(st->_ctypes_ptrtype_cache); + Py_VISIT(st->_unpickle); + Py_VISIT(st->array_cache); + Py_VISIT(st->error_object_name); + Py_VISIT(st->PyExc_ArgError); + Py_VISIT(st->swapped_suffix); + + Py_VISIT(st->DictRemover_Type); + Py_VISIT(st->PyCArg_Type); + Py_VISIT(st->PyCField_Type); + Py_VISIT(st->PyCThunk_Type); + Py_VISIT(st->StructParam_Type); + Py_VISIT(st->PyCStructType_Type); + Py_VISIT(st->UnionType_Type); + Py_VISIT(st->PyCPointerType_Type); + Py_VISIT(st->PyCArrayType_Type); + Py_VISIT(st->PyCSimpleType_Type); + Py_VISIT(st->PyCFuncPtrType_Type); + Py_VISIT(st->PyCData_Type); + Py_VISIT(st->Struct_Type); + Py_VISIT(st->Union_Type); + Py_VISIT(st->PyCArray_Type); + Py_VISIT(st->Simple_Type); + Py_VISIT(st->PyCPointer_Type); + Py_VISIT(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_VISIT(st->PyComError_Type); +#endif + Py_VISIT(st->PyCType_Type); + return 0; +} + +static int +module_clear(PyObject *module) { + ctypes_state *st = get_module_state(module); + Py_CLEAR(st->_ctypes_ptrtype_cache); + Py_CLEAR(st->_unpickle); + Py_CLEAR(st->array_cache); + Py_CLEAR(st->error_object_name); + Py_CLEAR(st->PyExc_ArgError); + Py_CLEAR(st->swapped_suffix); + + Py_CLEAR(st->DictRemover_Type); + Py_CLEAR(st->PyCArg_Type); + Py_CLEAR(st->PyCField_Type); + Py_CLEAR(st->PyCThunk_Type); + Py_CLEAR(st->StructParam_Type); + Py_CLEAR(st->PyCStructType_Type); + Py_CLEAR(st->UnionType_Type); + Py_CLEAR(st->PyCPointerType_Type); + Py_CLEAR(st->PyCArrayType_Type); + Py_CLEAR(st->PyCSimpleType_Type); + Py_CLEAR(st->PyCFuncPtrType_Type); + Py_CLEAR(st->PyCData_Type); + Py_CLEAR(st->Struct_Type); + Py_CLEAR(st->Union_Type); + Py_CLEAR(st->PyCArray_Type); + Py_CLEAR(st->Simple_Type); + Py_CLEAR(st->PyCPointer_Type); + Py_CLEAR(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_CLEAR(st->PyComError_Type); +#endif + Py_CLEAR(st->PyCType_Type); + return 0; +} + +static void +module_free(void *module) +{ + (void)module_clear((PyObject *)module); +} + +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _ctypes_mod_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +struct PyModuleDef _ctypesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ctypes", + .m_doc = _ctypes__doc__, + .m_size = sizeof(ctypes_state), + .m_methods = _ctypes_module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; + PyMODINIT_FUNC PyInit__ctypes(void) { - PyObject *mod = PyModule_Create(&_ctypesmodule); - if (!mod) { - return NULL; - } - - if (_ctypes_mod_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&_ctypesmodule); } /* diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ddfb2c8a332a9e..b8e613fd669d1b 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,13 +1,35 @@ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +// gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt +// linking the extension to python3.lib (which fails). Py_BUILD_CORE_MODULE is +// needed to import Python symbols. Then Python.h undefines Py_BUILD_CORE and +// Py_BUILD_CORE_MODULE if Py_LIMITED_API is defined. +#define Py_BUILD_CORE +#define Py_BUILD_CORE_MODULE + #include -#ifdef MS_WIN32 -#include -#endif +#include // FFI_TARGET_HAS_COMPLEX_TYPE +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +# include "../_complex.h" // csqrt() +# undef I // for _ctypes_test_generated.c.h +#endif +#include // printf() #include // qsort() +#include // memset() +#ifdef MS_WIN32 +# include +#endif #define EXPORT(x) Py_EXPORTED_SYMBOL x +#include "_ctypes_test_generated.c.h" + /* some functions handy for testing */ EXPORT(int) @@ -81,11 +103,10 @@ typedef struct { } Test2; EXPORT(int) -_testfunc_array_in_struct1(Test2 in) +_testfunc_array_in_struct2(Test2 in) { int result = 0; - - for (unsigned i = 0; i < 16; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. @@ -94,22 +115,25 @@ _testfunc_array_in_struct1(Test2 in) return result; } -typedef struct { - double data[2]; -} Test3; - +/* + * Test3A struct test the MAX_STRUCT_SIZE 16 with single precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ typedef struct { float data[2]; float more_data[2]; -} Test3B; +} Test3A; EXPORT(double) -_testfunc_array_in_struct2(Test3 in) +_testfunc_array_in_struct3A(Test3A in) { double result = 0; - for (unsigned i = 0; i < 2; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; + for (unsigned i = 0; i < sizeof(in.more_data)/sizeof(in.more_data[0]); i++) + result += in.more_data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. */ @@ -117,20 +141,116 @@ _testfunc_array_in_struct2(Test3 in) return result; } +/* The structs Test3B..Test3E use the same functions hence using the MACRO + * to define their implementation. + */ + +#define _TESTFUNC_ARRAY_IN_STRUCT_IMPL \ + double result = 0; \ + \ + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) \ + result += in.data[i]; \ + /* As the structure is passed by value, changes to it shouldn't be \ + * reflected in the caller. \ + */ \ + memset(in.data, 0, sizeof(in.data)); \ + return result; \ + +#define _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL \ + for (unsigned i = 0; i < sizeof(s.data)/sizeof(s.data[0]); i++) \ + s.data[i] = (double)i+1; \ + return s; \ + + +/* + * Test3B struct test the MAX_STRUCT_SIZE 16 with double precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ +typedef struct { + double data[2]; +} Test3B; + EXPORT(double) -_testfunc_array_in_struct2a(Test3B in) +_testfunc_array_in_struct3B(Test3B in) { - double result = 0; + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} - for (unsigned i = 0; i < 2; i++) - result += in.data[i]; - for (unsigned i = 0; i < 2; i++) - result += in.more_data[i]; - /* As the structure is passed by value, changes to it shouldn't be - * reflected in the caller. - */ - memset(in.data, 0, sizeof(in.data)); - return result; +EXPORT(Test3B) +_testfunc_array_in_struct3B_set_defaults(void) +{ + Test3B s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up + * to four floating point types are passed in registers on Arm platforms. + * This struct is used for within bounds test on Arm platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * See gh-110190. + */ +typedef struct { + double data[4]; +} Test3C; + +EXPORT(double) +_testfunc_array_in_struct3C(Test3C in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3C) +_testfunc_array_in_struct3C_set_defaults(void) +{ + Test3C s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up + * to eight floating point types are passed in registers on PPC64LE platforms. + * This struct is used for within bounds test on PPC64LE platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. + * See gh-110190. + */ +typedef struct { + double data[8]; +} Test3D; + +EXPORT(double) +_testfunc_array_in_struct3D(Test3D in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3D) +_testfunc_array_in_struct3D_set_defaults(void) +{ + Test3D s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3E struct tests the out-of-bounds for all architectures. + * See gh-110190. + */ +typedef struct { + double data[9]; +} Test3E; + +EXPORT(double) +_testfunc_array_in_struct3E(Test3E in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3E) +_testfunc_array_in_struct3E_set_defaults(void) +{ + Test3E s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL } typedef union { @@ -231,6 +351,31 @@ _testfunc_bitfield_by_reference2(Test7 *in) { return result; } +typedef struct{ + uint16_t A ; + uint16_t B : 9; + uint16_t C : 1; + uint16_t D : 1; + uint16_t E : 1; + uint16_t F : 1; + uint16_t G : 3; + uint32_t H : 10; + uint32_t I : 20; + uint32_t J : 2; +} Test9; + +EXPORT(long) +_testfunc_bitfield_by_reference3(Test9 *in, long pos) { + long data[] = {in->A , in->B , in->C , in->D , in->E , in->F , in->G , in->H , in->I , in->J}; + long data_length = (long) (sizeof(data)/sizeof(data[0])); + if(pos < 0) + return -1; + if(pos >= data_length) + return -1; + + return data[pos]; +} + typedef union { signed int A: 1, B:2, C:3, D:2; } Test8; @@ -304,6 +449,23 @@ EXPORT(double) my_sqrt(double a) return sqrt(a); } +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +EXPORT(double complex) my_csqrt(double complex a) +{ + return csqrt(a); +} + +EXPORT(float complex) my_csqrtf(float complex a) +{ + return csqrtf(a); +} + +EXPORT(long double complex) my_csqrtl(long double complex a) +{ + return csqrtl(a); +} +#endif + EXPORT(void) my_qsort(void *base, size_t num, size_t width, int(*compare)(const void*, const void*)) { qsort(base, num, width, compare); @@ -592,7 +754,7 @@ struct BITS { */ #ifndef __xlc__ #define SIGNED_SHORT_BITFIELDS - short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; #endif }; @@ -622,12 +784,58 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +#if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) +struct +#ifndef MS_WIN32 +__attribute__ ((ms_struct)) +#endif +BITS_msvc +{ + signed int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; +/* + * The test case needs/uses "signed short" bitfields, but the + * IBM XLC compiler does not support this + */ +#ifndef __xlc__ +#define SIGNED_SHORT_BITFIELDS + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +#endif +}; + +EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + +#ifdef SIGNED_SHORT_BITFIELDS + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; +#endif + } + return 999; +} +#endif + static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS}, */ {"func_si", py_func_si, METH_VARARGS}, {"func", py_func, METH_NOARGS}, + {"get_generated_test_data", get_generated_test_data, METH_O}, { NULL, NULL, 0, NULL}, }; @@ -1055,6 +1263,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) static struct PyModuleDef_Slot _ctypes_test_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h new file mode 100644 index 00000000000000..46a3e4b01e2259 --- /dev/null +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -0,0 +1,1885 @@ + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \ + PyObject *item = ITEM; \ + if (!item) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + int rv = PyList_Append(result, item); \ + Py_DECREF(item); \ + if (rv < 0) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \ + TYPE v = VAL; \ + TARGET = v; \ + APPEND(PyBytes_FromStringAndSize( \ + (char*)&value, sizeof(value))); \ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt") == 0) { + + struct SingleInt { + int a; + }; + struct SingleInt value = {0}; + APPEND(PyUnicode_FromString("SingleInt")); + APPEND(PyLong_FromLong(sizeof(struct SingleInt))); + APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt_Union") == 0) { + + union SingleInt_Union { + int a; + }; + union SingleInt_Union value = {0}; + APPEND(PyUnicode_FromString("SingleInt_Union")); + APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); + APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleU32") == 0) { + + struct SingleU32 { + uint32_t a; + }; + struct SingleU32 value = {0}; + APPEND(PyUnicode_FromString("SingleU32")); + APPEND(PyLong_FromLong(sizeof(struct SingleU32))); + APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); + TEST_FIELD(uint32_t, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleStruct") == 0) { + + struct SimpleStruct { + int32_t x; + int8_t y; + uint16_t z; + }; + struct SimpleStruct value = {0}; + APPEND(PyUnicode_FromString("SimpleStruct")); + APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); + APPEND(PyLong_FromLong(_Alignof(struct SimpleStruct))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleUnion") == 0) { + + union SimpleUnion { + int32_t x; + int8_t y; + uint16_t z; + }; + union SimpleUnion value = {0}; + APPEND(PyUnicode_FromString("SimpleUnion")); + APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); + APPEND(PyLong_FromLong(_Alignof(union SimpleUnion))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypes") == 0) { + + struct ManyTypes { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + struct ManyTypes value = {0}; + APPEND(PyUnicode_FromString("ManyTypes")); + APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); + APPEND(PyLong_FromLong(_Alignof(struct ManyTypes))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypesU") == 0) { + + union ManyTypesU { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + union ManyTypesU value = {0}; + APPEND(PyUnicode_FromString("ManyTypesU")); + APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); + APPEND(PyLong_FromLong(_Alignof(union ManyTypesU))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Nested") == 0) { + + struct Nested { + struct { + int32_t x; + int8_t y; + uint16_t z; + } a; + union { + int32_t x; + int8_t y; + uint16_t z; + } b; + struct { + int32_t x; + int8_t y; + uint16_t z; + }; + }; + struct Nested value = {0}; + APPEND(PyUnicode_FromString("Nested")); + APPEND(PyLong_FromLong(sizeof(struct Nested))); + APPEND(PyLong_FromLong(_Alignof(struct Nested))); + TEST_FIELD(int32_t, value.a.x); + TEST_FIELD(int8_t, value.a.y); + TEST_FIELD(uint16_t, value.a.z); + TEST_FIELD(int32_t, value.b.x); + TEST_FIELD(int8_t, value.b.y); + TEST_FIELD(uint16_t, value.b.z); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Packed1 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed1 value = {0}; + APPEND(PyUnicode_FromString("Packed1")); + APPEND(PyLong_FromLong(sizeof(struct Packed1))); + APPEND(PyLong_FromLong(_Alignof(struct Packed1))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 2) + struct GCC_ATTR(ms_struct) Packed2 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed2 value = {0}; + APPEND(PyUnicode_FromString("Packed2")); + APPEND(PyLong_FromLong(sizeof(struct Packed2))); + APPEND(PyLong_FromLong(_Alignof(struct Packed2))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 4) + struct GCC_ATTR(ms_struct) Packed3 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed3 value = {0}; + APPEND(PyUnicode_FromString("Packed3")); + APPEND(PyLong_FromLong(sizeof(struct Packed3))); + APPEND(PyLong_FromLong(_Alignof(struct Packed3))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 8) + struct GCC_ATTR(ms_struct) Packed4 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed4 value = {0}; + APPEND(PyUnicode_FromString("Packed4")); + APPEND(PyLong_FromLong(sizeof(struct Packed4))); + APPEND(PyLong_FromLong(_Alignof(struct Packed4))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "X86_32EdgeCase") == 0) { + + struct X86_32EdgeCase { + int32_t a; + int64_t b; + int32_t c; + }; + struct X86_32EdgeCase value = {0}; + APPEND(PyUnicode_FromString("X86_32EdgeCase")); + APPEND(PyLong_FromLong(sizeof(struct X86_32EdgeCase))); + APPEND(PyLong_FromLong(_Alignof(struct X86_32EdgeCase))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int32_t, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSBitFieldExample") == 0) { + + struct MSBitFieldExample { + unsigned int a :4; + unsigned int b :5; + unsigned int c :7; + }; + struct MSBitFieldExample value = {0}; + APPEND(PyUnicode_FromString("MSBitFieldExample")); + APPEND(PyLong_FromLong(sizeof(struct MSBitFieldExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSBitFieldExample))); + TEST_FIELD(unsigned int, value.a); + TEST_FIELD(unsigned int, value.b); + TEST_FIELD(unsigned int, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSStraddlingExample") == 0) { + + struct MSStraddlingExample { + unsigned int first :9; + unsigned int second :7; + unsigned int may_straddle :30; + unsigned int last :18; + }; + struct MSStraddlingExample value = {0}; + APPEND(PyUnicode_FromString("MSStraddlingExample")); + APPEND(PyLong_FromLong(sizeof(struct MSStraddlingExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSStraddlingExample))); + TEST_FIELD(unsigned int, value.first); + TEST_FIELD(unsigned int, value.second); + TEST_FIELD(unsigned int, value.may_straddle); + TEST_FIELD(unsigned int, value.last); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { + + struct IntBits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits value = {0}; + APPEND(PyUnicode_FromString("IntBits")); + APPEND(PyLong_FromLong(sizeof(struct IntBits))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits") == 0) { + + #if (!defined(__xlc__)) + + struct Bits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits value = {0}; + APPEND(PyUnicode_FromString("Bits")); + APPEND(PyLong_FromLong(sizeof(struct Bits))); + APPEND(PyLong_FromLong(_Alignof(struct Bits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) IntBits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits_MSVC value = {0}; + APPEND(PyUnicode_FromString("IntBits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) Bits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits_MSVC value = {0}; + APPEND(PyUnicode_FromString("Bits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct Bits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "I64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct I64Bits { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct I64Bits value = {0}; + APPEND(PyUnicode_FromString("I64Bits")); + APPEND(PyLong_FromLong(sizeof(struct I64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct I64Bits))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "U64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct U64Bits { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct U64Bits value = {0}; + APPEND(PyUnicode_FromString("U64Bits")); + APPEND(PyLong_FromLong(sizeof(struct U64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct U64Bits))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_8 { + int8_t a :3; + int8_t b :3; + int8_t c :1; + }; + struct Struct331_8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_8 { + int8_t a :1; + int8_t b :6; + int8_t c :1; + }; + struct Struct1x1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_8 { + int8_t a :1; + int8_t full; + int8_t b :6; + int8_t c :1; + }; + struct Struct1nx1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.full); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_8 { + int8_t a :3; + int8_t b :6; + int8_t c :6; + }; + struct Struct3xx_8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u8 { + uint8_t a :3; + uint8_t b :3; + uint8_t c :1; + }; + struct Struct331_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u8 { + uint8_t a :1; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1x1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u8 { + uint8_t a :1; + uint8_t full; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1nx1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.full); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u8 { + uint8_t a :3; + uint8_t b :6; + uint8_t c :6; + }; + struct Struct3xx_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_16 { + int16_t a :3; + int16_t b :3; + int16_t c :1; + }; + struct Struct331_16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_16 { + int16_t a :1; + int16_t b :14; + int16_t c :1; + }; + struct Struct1x1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_16 { + int16_t a :1; + int16_t full; + int16_t b :14; + int16_t c :1; + }; + struct Struct1nx1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.full); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_16 { + int16_t a :3; + int16_t b :14; + int16_t c :14; + }; + struct Struct3xx_16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u16 { + uint16_t a :3; + uint16_t b :3; + uint16_t c :1; + }; + struct Struct331_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u16 { + uint16_t a :1; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1x1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u16 { + uint16_t a :1; + uint16_t full; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1nx1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.full); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u16 { + uint16_t a :3; + uint16_t b :14; + uint16_t c :14; + }; + struct Struct3xx_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_32 { + int32_t a :3; + int32_t b :3; + int32_t c :1; + }; + struct Struct331_32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_32 { + int32_t a :1; + int32_t b :30; + int32_t c :1; + }; + struct Struct1x1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_32 { + int32_t a :1; + int32_t full; + int32_t b :30; + int32_t c :1; + }; + struct Struct1nx1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.full); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_32 { + int32_t a :3; + int32_t b :30; + int32_t c :30; + }; + struct Struct3xx_32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u32 { + uint32_t a :3; + uint32_t b :3; + uint32_t c :1; + }; + struct Struct331_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u32 { + uint32_t a :1; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1x1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u32 { + uint32_t a :1; + uint32_t full; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1nx1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.full); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u32 { + uint32_t a :3; + uint32_t b :30; + uint32_t c :30; + }; + struct Struct3xx_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_64 { + int64_t a :3; + int64_t b :3; + int64_t c :1; + }; + struct Struct331_64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_64 { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct Struct1x1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_64 { + int64_t a :1; + int64_t full; + int64_t b :62; + int64_t c :1; + }; + struct Struct1nx1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.full); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_64 { + int64_t a :3; + int64_t b :62; + int64_t c :62; + }; + struct Struct3xx_64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u64 { + uint64_t a :3; + uint64_t b :3; + uint64_t c :1; + }; + struct Struct331_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u64 { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1x1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u64 { + uint64_t a :1; + uint64_t full; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1nx1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.full); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u64 { + uint64_t a :3; + uint64_t b :62; + uint64_t c :62; + }; + struct Struct3xx_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed1") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed1 { + signed char a :4; + int b :4; + }; + struct Mixed1 value = {0}; + APPEND(PyUnicode_FromString("Mixed1")); + APPEND(PyLong_FromLong(sizeof(struct Mixed1))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed2") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed2 { + signed char a :4; + int32_t b :32; + }; + struct Mixed2 value = {0}; + APPEND(PyUnicode_FromString("Mixed2")); + APPEND(PyLong_FromLong(sizeof(struct Mixed2))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int32_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed3") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed3 { + signed char a :4; + unsigned char b :4; + }; + struct Mixed3 value = {0}; + APPEND(PyUnicode_FromString("Mixed3")); + APPEND(PyLong_FromLong(sizeof(struct Mixed3))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed4") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed4 { + short a :4; + short b :4; + int c :24; + short d :4; + short e :4; + int f :24; + }; + struct Mixed4 value = {0}; + APPEND(PyUnicode_FromString("Mixed4")); + APPEND(PyLong_FromLong(sizeof(struct Mixed4))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed4))); + TEST_FIELD(short, value.a); + TEST_FIELD(short, value.b); + TEST_FIELD(int, value.c); + TEST_FIELD(short, value.d); + TEST_FIELD(short, value.e); + TEST_FIELD(int, value.f); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed5") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed5 { + unsigned int A :1; + unsigned short B :16; + }; + struct Mixed5 value = {0}; + APPEND(PyUnicode_FromString("Mixed5")); + APPEND(PyLong_FromLong(sizeof(struct Mixed5))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); + TEST_FIELD(unsigned int, value.A); + TEST_FIELD(unsigned short, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed6") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed6 { + unsigned long long A :1; + unsigned int B :32; + }; + struct Mixed6 value = {0}; + APPEND(PyUnicode_FromString("Mixed6")); + APPEND(PyLong_FromLong(sizeof(struct Mixed6))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); + TEST_FIELD(unsigned long long, value.A); + TEST_FIELD(unsigned int, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed7") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed7 { + uint32_t A; + uint32_t B :20; + uint64_t C :24; + }; + struct Mixed7 value = {0}; + APPEND(PyUnicode_FromString("Mixed7")); + APPEND(PyLong_FromLong(sizeof(struct Mixed7))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed7))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(uint64_t, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_a") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_a { + uint32_t A; + uint32_t B :32; + unsigned long long C :1; + }; + struct Mixed8_a value = {0}; + APPEND(PyUnicode_FromString("Mixed8_a")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_a))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_b") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_b { + uint32_t A; + uint32_t B; + unsigned long long C :1; + }; + struct Mixed8_b value = {0}; + APPEND(PyUnicode_FromString("Mixed8_b")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_b))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed9") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed9 { + uint8_t A; + uint32_t B :1; + }; + struct Mixed9 value = {0}; + APPEND(PyUnicode_FromString("Mixed9")); + APPEND(PyLong_FromLong(sizeof(struct Mixed9))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); + TEST_FIELD(uint8_t, value.A); + TEST_FIELD(uint32_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed10") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed10 { + uint32_t A :1; + uint64_t B :1; + }; + struct Mixed10 value = {0}; + APPEND(PyUnicode_FromString("Mixed10")); + APPEND(PyLong_FromLong(sizeof(struct Mixed10))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_95496") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_95496 { + uint32_t A :1; + uint64_t B :1; + }; + struct Example_gh_95496 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_95496")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_bad value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_bad")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_bad))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + }; + #pragma pack(pop) + struct Example_gh_84039_good_a value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good_a))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good { + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + } a; + #pragma pack(pop) + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_good value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good))); + TEST_FIELD(uint8_t, value.a.a0); + TEST_FIELD(uint8_t, value.a.a1); + TEST_FIELD(uint8_t, value.a.a2); + TEST_FIELD(uint8_t, value.a.a3); + TEST_FIELD(uint8_t, value.a.a4); + TEST_FIELD(uint8_t, value.a.a5); + TEST_FIELD(uint8_t, value.a.a6); + TEST_FIELD(uint8_t, value.a.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_73939 { + uint16_t P; + uint16_t L :9; + uint16_t Pro :1; + uint16_t G :1; + uint16_t IB :1; + uint16_t IR :1; + uint16_t R :3; + uint32_t T :10; + uint32_t C :20; + uint32_t R2 :2; + }; + #pragma pack(pop) + struct Example_gh_73939 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_73939")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_73939))); + TEST_FIELD(uint16_t, value.P); + TEST_FIELD(uint16_t, value.L); + TEST_FIELD(uint16_t, value.Pro); + TEST_FIELD(uint16_t, value.G); + TEST_FIELD(uint16_t, value.IB); + TEST_FIELD(uint16_t, value.IR); + TEST_FIELD(uint16_t, value.R); + TEST_FIELD(uint32_t, value.T); + TEST_FIELD(uint32_t, value.C); + TEST_FIELD(uint32_t, value.R2); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_86098 { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + struct Example_gh_86098 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + #pragma pack(pop) + struct Example_gh_86098_pack value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098_pack")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098_pack))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "AnonBitfields") == 0) { + + #if (!defined(__xlc__)) + + struct AnonBitfields { + struct { + signed char a :4; + unsigned char b :4; + }; + signed char y; + }; + struct AnonBitfields value = {0}; + APPEND(PyUnicode_FromString("AnonBitfields")); + APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); + APPEND(PyLong_FromLong(_Alignof(struct AnonBitfields))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + TEST_FIELD(signed char, value.y); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 1bd8fec97179e9..7b9f6437c7d55f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -9,7 +9,6 @@ #endif #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_runtime.h" // _Py_ID() #include @@ -110,10 +109,14 @@ PrintError(const char *msg, ...) * slower. */ static void -TryAddRef(StgDictObject *dict, CDataObject *obj) +TryAddRef(PyObject *cnv, CDataObject *obj) { IUnknown *punk; - int r = PyDict_Contains((PyObject *)dict, &_Py_ID(_needs_com_addref_)); + PyObject *attrdict = _PyType_GetDict((PyTypeObject *)cnv); + if (!attrdict) { + return; + } + int r = PyDict_Contains(attrdict, &_Py_ID(_needs_com_addref_)); if (r <= 0) { if (r < 0) { PrintError("getting _needs_com_addref_"); @@ -133,7 +136,10 @@ TryAddRef(StgDictObject *dict, CDataObject *obj) * Call the python object with all arguments * */ -static void _CallPythonObject(void *mem, + +// BEWARE: The GIL needs to be held throughout the function +static void _CallPythonObject(ctypes_state *st, + void *mem, ffi_type *restype, SETFUNC setfunc, PyObject *callable, @@ -145,7 +151,6 @@ static void _CallPythonObject(void *mem, Py_ssize_t i = 0, j = 0, nargs = 0; PyObject *error_object = NULL; int *space; - PyGILState_STATE state = PyGILState_Ensure(); assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); @@ -154,37 +159,41 @@ static void _CallPythonObject(void *mem, PyObject **cnvs = PySequence_Fast_ITEMS(converters); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref - StgDictObject *dict; - dict = PyType_stgdict(cnv); - if (dict && dict->getfunc && !_ctypes_simple_instance(cnv)) { - PyObject *v = dict->getfunc(*pArgs, dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, cnv, &info) < 0) { + goto Done; + } + + if (info && info->getfunc && !_ctypes_simple_instance(st, cnv)) { + PyObject *v = info->getfunc(*pArgs, info->size); if (!v) { PrintError("create argument %zd:\n", i); goto Done; } args[i] = v; /* XXX XXX XX - We have the problem that c_byte or c_short have dict->size of + We have the problem that c_byte or c_short have info->size of 1 resp. 4, but these parameters are pushed as sizeof(int) bytes. BTW, the same problem occurs when they are pushed as parameters */ - } else if (dict) { + } + else if (info) { /* Hm, shouldn't we use PyCData_AtAddress() or something like that instead? */ CDataObject *obj = (CDataObject *)_PyObject_CallNoArgs(cnv); if (!obj) { PrintError("create argument %zd:\n", i); goto Done; } - if (!CDataObject_Check(obj)) { + if (!CDataObject_Check(st, obj)) { Py_DECREF(obj); PrintError("unexpected result of create argument %zd:\n", i); goto Done; } - memcpy(obj->b_ptr, *pArgs, dict->size); + memcpy(obj->b_ptr, *pArgs, info->size); args[i] = (PyObject *)obj; #ifdef MS_WIN32 - TryAddRef(dict, obj); + TryAddRef(cnv, obj); #endif } else { PyErr_SetString(PyExc_TypeError, @@ -197,7 +206,7 @@ static void _CallPythonObject(void *mem, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) goto Done; if (flags & FUNCFLAG_USE_ERRNO) { @@ -216,8 +225,9 @@ static void _CallPythonObject(void *mem, result = PyObject_Vectorcall(callable, args, nargs, NULL); if (result == NULL) { - _PyErr_WriteUnraisableMsg("on calling ctypes callback function", - callable); + PyErr_FormatUnraisable( + "Exception ignored on calling ctypes callback function %R", + callable); } #ifdef MS_WIN32 @@ -258,9 +268,10 @@ static void _CallPythonObject(void *mem, if (keep == NULL) { /* Could not convert callback result. */ - _PyErr_WriteUnraisableMsg("on converting result " - "of ctypes callback function", - callable); + PyErr_FormatUnraisable( + "Exception ignored on converting result " + "of ctypes callback function %R", + callable); } else if (setfunc != _ctypes_get_fielddesc("O")->setfunc) { if (keep == Py_None) { @@ -270,9 +281,10 @@ static void _CallPythonObject(void *mem, else if (PyErr_WarnEx(PyExc_RuntimeWarning, "memory leak in callback function.", 1) == -1) { - _PyErr_WriteUnraisableMsg("on converting result " - "of ctypes callback function", - callable); + PyErr_FormatUnraisable( + "Exception ignored on converting result " + "of ctypes callback function %R", + callable); } } } @@ -283,7 +295,6 @@ static void _CallPythonObject(void *mem, for (j = 0; j < i; j++) { Py_DECREF(args[j]); } - PyGILState_Release(state); } static void closure_fcn(ffi_cif *cif, @@ -291,23 +302,28 @@ static void closure_fcn(ffi_cif *cif, void **args, void *userdata) { + PyGILState_STATE state = PyGILState_Ensure(); + CThunkObject *p = (CThunkObject *)userdata; + ctypes_state *st = get_module_state_by_class(Py_TYPE(p)); - _CallPythonObject(resp, + _CallPythonObject(st, + resp, p->ffi_restype, p->setfunc, p->callable, p->converters, p->flags, args); + + PyGILState_Release(state); } -static CThunkObject* CThunkObject_new(Py_ssize_t nargs) +static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) { CThunkObject *p; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_NewVar(CThunkObject, st->PyCThunk_Type, nargs); if (p == NULL) { return NULL; @@ -329,7 +345,8 @@ static CThunkObject* CThunkObject_new(Py_ssize_t nargs) return p; } -CThunkObject *_ctypes_alloc_callback(PyObject *callable, +CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags) @@ -341,14 +358,11 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); - p = CThunkObject_new(nargs); + p = CThunkObject_new(st, nargs); if (p == NULL) return NULL; -#ifdef Py_DEBUG - ctypes_state *st = GLOBAL_STATE(); assert(CThunk_CheckExact(st, (PyObject *)p)); -#endif p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { @@ -360,7 +374,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyObject **cnvs = PySequence_Fast_ITEMS(converters); for (i = 0; i < nargs; ++i) { PyObject *cnv = cnvs[i]; // borrowed ref - p->atypes[i] = _ctypes_get_ffi_type(cnv); + p->atypes[i] = _ctypes_get_ffi_type(st, cnv); } p->atypes[i] = NULL; @@ -369,14 +383,18 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, p->setfunc = NULL; p->ffi_restype = &ffi_type_void; } else { - StgDictObject *dict = PyType_stgdict(restype); - if (dict == NULL || dict->setfunc == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + goto error; + } + + if (info == NULL || info->setfunc == NULL) { PyErr_SetString(PyExc_TypeError, "invalid result type for callback function"); goto error; } - p->setfunc = dict->setfunc; - p->ffi_restype = &dict->ffi_type_pointer; + p->setfunc = info->setfunc; + p->ffi_restype = &info->ffi_type_pointer; } cc = FFI_DEFAULT_ABI; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index fc08c42bd3574a..fd89d9c67b3fc0 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -105,6 +105,10 @@ module _ctypes #include "pycore_global_objects.h"// _Py_ID() #include "pycore_traceback.h" // _PyTraceback_Add() +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +#include "../_complex.h" // complex +#endif + #include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -153,31 +157,33 @@ static void pymem_destructor(PyObject *ptr) kept alive in the thread state dictionary as long as the thread itself. */ PyObject * -_ctypes_get_errobj(int **pspace) +_ctypes_get_errobj(ctypes_state *st, int **pspace) { PyObject *dict = PyThreadState_GetDict(); PyObject *errobj; - static PyObject *error_object_name; if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - if (error_object_name == NULL) { - error_object_name = PyUnicode_InternFromString("ctypes.error_object"); - if (error_object_name == NULL) + if (st->error_object_name == NULL) { + st->error_object_name = PyUnicode_InternFromString("ctypes.error_object"); + if (st->error_object_name == NULL) { return NULL; + } + } + if (PyDict_GetItemRef(dict, st->error_object_name, &errobj) < 0) { + return NULL; } - errobj = PyDict_GetItemWithError(dict, error_object_name); if (errobj) { if (!PyCapsule_IsValid(errobj, CTYPES_CAPSULE_NAME_PYMEM)) { PyErr_SetString(PyExc_RuntimeError, "ctypes.error_object is an invalid capsule"); + Py_DECREF(errobj); return NULL; } - Py_INCREF(errobj); } - else if (!PyErr_Occurred()) { + else { void *space = PyMem_Calloc(2, sizeof(int)); if (space == NULL) return NULL; @@ -186,15 +192,11 @@ _ctypes_get_errobj(int **pspace) PyMem_Free(space); return NULL; } - if (-1 == PyDict_SetItem(dict, error_object_name, - errobj)) { + if (PyDict_SetItem(dict, st->error_object_name, errobj) < 0) { Py_DECREF(errobj); return NULL; } } - else { - return NULL; - } *pspace = (int *)PyCapsule_GetPointer(errobj, CTYPES_CAPSULE_NAME_PYMEM); return errobj; } @@ -203,7 +205,8 @@ static PyObject * get_error_internal(PyObject *self, PyObject *args, int index) { int *space; - PyObject *errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; if (errobj == NULL) @@ -223,7 +226,8 @@ set_error_internal(PyObject *self, PyObject *args, int index) if (!PyArg_ParseTuple(args, "i", &new_errno)) { return NULL; } - errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; old_errno = space[index]; @@ -474,10 +478,9 @@ check_hresult(PyObject *self, PyObject *args) /**************************************************************/ PyCArgObject * -PyCArgObject_new(void) +PyCArgObject_new(ctypes_state *st) { PyCArgObject *p; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_New(PyCArgObject, st->PyCArg_Type); if (p == NULL) return NULL; @@ -652,6 +655,11 @@ union result { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) + double complex C; + float complex E; + long double complex F; +#endif }; struct argument { @@ -663,17 +671,22 @@ struct argument { /* * Convert a single Python object into a PyCArgObject and return it. */ -static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) +static int ConvParam(ctypes_state *st, + PyObject *obj, Py_ssize_t index, struct argument *pa) { - StgDictObject *dict; pa->keep = NULL; /* so we cannot forget it later */ - dict = PyObject_stgdict(obj); - if (dict) { + StgInfo *info; + int result = PyStgInfo_FromObject(st, obj, &info); + if (result < 0) { + return -1; + } + if (info) { + assert(info); PyCArgObject *carg; - assert(dict->paramfunc); - /* If it has an stgdict, it is a CDataObject */ - carg = dict->paramfunc((CDataObject *)obj); + assert(info->paramfunc); + /* If it has an stginfo, it is a CDataObject */ + carg = info->paramfunc(st, (CDataObject *)obj); if (carg == NULL) return -1; pa->ffi_type = carg->pffi_type; @@ -682,7 +695,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) return 0; } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, obj)) { PyCArgObject *carg = (PyCArgObject *)obj; pa->ffi_type = carg->pffi_type; @@ -745,7 +757,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) */ if (arg) { int result; - result = ConvParam(arg, index, pa); + result = ConvParam(st, arg, index, pa); Py_DECREF(arg); return result; } @@ -779,26 +791,33 @@ int can_return_struct_as_sint64(size_t s) #endif -ffi_type *_ctypes_get_ffi_type(PyObject *obj) +// returns NULL with exception set on error +ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj) { - StgDictObject *dict; - if (obj == NULL) + if (obj == NULL) { return &ffi_type_sint; - dict = PyType_stgdict(obj); - if (dict == NULL) + } + + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + + if (info == NULL) { return &ffi_type_sint; + } #if defined(MS_WIN32) && !defined(_WIN32_WCE) /* This little trick works correctly with MSVC. It returns small structures in registers */ - if (dict->ffi_type_pointer.type == FFI_TYPE_STRUCT) { - if (can_return_struct_as_int(dict->ffi_type_pointer.size)) + if (info->ffi_type_pointer.type == FFI_TYPE_STRUCT) { + if (can_return_struct_as_int(info->ffi_type_pointer.size)) return &ffi_type_sint32; - else if (can_return_struct_as_sint64 (dict->ffi_type_pointer.size)) + else if (can_return_struct_as_sint64 (info->ffi_type_pointer.size)) return &ffi_type_sint64; } #endif - return &dict->ffi_type_pointer; + return &info->ffi_type_pointer; } @@ -814,7 +833,8 @@ ffi_type *_ctypes_get_ffi_type(PyObject *obj) * * void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); */ -static int _call_function_pointer(int flags, +static int _call_function_pointer(ctypes_state *st, + int flags, PPROC pProc, void **avalues, ffi_type **atypes, @@ -915,7 +935,7 @@ static int _call_function_pointer(int flags, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) return -1; } @@ -982,9 +1002,9 @@ static int _call_function_pointer(int flags, * - If restype is another ctypes type, return an instance of that. * - Otherwise, call restype and return the result. */ -static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) +static PyObject *GetResult(ctypes_state *st, + PyObject *restype, void *result, PyObject *checker) { - StgDictObject *dict; PyObject *retval, *v; if (restype == NULL) @@ -994,22 +1014,27 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) Py_RETURN_NONE; } - dict = PyType_stgdict(restype); - if (dict == NULL) + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + return NULL; + } + if (info == NULL) { return PyObject_CallFunction(restype, "i", *(int *)result); + } - if (dict->getfunc && !_ctypes_simple_instance(restype)) { - retval = dict->getfunc(result, dict->size); + if (info->getfunc && !_ctypes_simple_instance(st, restype)) { + retval = info->getfunc(result, info->size); /* If restype is py_object (detected by comparing getfunc with O_get), we have to call Py_DECREF because O_get has already called Py_INCREF. */ - if (dict->getfunc == _ctypes_get_fielddesc("O")->getfunc) { + if (info->getfunc == _ctypes_get_fielddesc("O")->getfunc) { Py_DECREF(retval); } - } else - retval = PyCData_FromBaseObj(restype, NULL, 0, result); - + } + else { + retval = PyCData_FromBaseObj(st, restype, NULL, 0, result); + } if (!checker || !retval) return retval; @@ -1071,7 +1096,7 @@ void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) #ifdef MS_WIN32 static PyObject * -GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) +GetComError(ctypes_state *st, HRESULT errcode, GUID *riid, IUnknown *pIunk) { HRESULT hr; ISupportErrorInfo *psei = NULL; @@ -1123,7 +1148,6 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - ctypes_state *st = GLOBAL_STATE(); PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } @@ -1154,7 +1178,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) * * - XXX various requirements for restype, not yet collected */ -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *argtuple, #ifdef MS_WIN32 IUnknown *pIunk, @@ -1184,7 +1209,7 @@ PyObject *_ctypes_callproc(PPROC pProc, if (argcount > CTYPES_MAX_ARGCOUNT) { - PyErr_Format(PyExc_ArgError, "too many arguments (%zi), maximum is %i", + PyErr_Format(st->PyExc_ArgError, "too many arguments (%zi), maximum is %i", argcount, CTYPES_MAX_ARGCOUNT); return NULL; } @@ -1217,20 +1242,20 @@ PyObject *_ctypes_callproc(PPROC pProc, converter = PyTuple_GET_ITEM(argtypes, i); v = PyObject_CallOneArg(converter, arg); if (v == NULL) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } - err = ConvParam(v, i+1, pa); + err = ConvParam(st, v, i+1, pa); Py_DECREF(v); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } } else { - err = ConvParam(arg, i+1, pa); + err = ConvParam(st, arg, i+1, pa); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; /* leaking ? */ } } @@ -1239,7 +1264,10 @@ PyObject *_ctypes_callproc(PPROC pProc, if (restype == Py_None) { rtype = &ffi_type_void; } else { - rtype = _ctypes_get_ffi_type(restype); + rtype = _ctypes_get_ffi_type(st, restype); + } + if (!rtype) { + goto cleanup; } resbuf = alloca(max(rtype->size, sizeof(ffi_arg))); @@ -1278,7 +1306,7 @@ PyObject *_ctypes_callproc(PPROC pProc, avalues[i] = (void *)&args[i].value; } - if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, + if (-1 == _call_function_pointer(st, flags, pProc, avalues, atypes, rtype, resbuf, Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) @@ -1306,7 +1334,7 @@ PyObject *_ctypes_callproc(PPROC pProc, #ifdef MS_WIN32 if (iid && pIunk) { if (*(int *)resbuf & 0x80000000) - retval = GetComError(*(HRESULT *)resbuf, iid, pIunk); + retval = GetComError(st, *(HRESULT *)resbuf, iid, pIunk); else retval = PyLong_FromLong(*(int *)resbuf); } else if (flags & FUNCFLAG_HRESULT) { @@ -1316,7 +1344,7 @@ PyObject *_ctypes_callproc(PPROC pProc, retval = PyLong_FromLong(*(int *)resbuf); } else #endif - retval = GetResult(restype, resbuf, checker); + retval = GetResult(st, restype, resbuf, checker); cleanup: for (i = 0; i < argcount; ++i) Py_XDECREF(args[i].keep); @@ -1405,7 +1433,7 @@ static PyObject *load_library(PyObject *self, PyObject *args) #ifdef _WIN64 return PyLong_FromVoidPtr(hMod); #else - return Py_BuildValue("i", hMod); + return PyLong_FromLong((int)hMod); #endif } @@ -1445,8 +1473,10 @@ copy_com_pointer(PyObject *self, PyObject *args) return NULL; a.keep = b.keep = NULL; - if (-1 == ConvParam(p1, 0, &a) || -1 == ConvParam(p2, 1, &b)) + ctypes_state *st = get_module_state(self); + if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; + } src = (IUnknown *)a.value.p; pdst = (IUnknown **)b.value.p; @@ -1625,7 +1655,9 @@ call_function(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1660,7 +1692,9 @@ call_cdeclfunction(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1684,14 +1718,19 @@ PyDoc_STRVAR(sizeof_doc, static PyObject * sizeof_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; + ctypes_state *st = get_module_state(self); - dict = PyType_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->size); + } - if (CDataObject_Check(obj)) + if (CDataObject_Check(st, obj)) { return PyLong_FromSsize_t(((CDataObject *)obj)->b_size); + } PyErr_SetString(PyExc_TypeError, "this type has no size"); return NULL; @@ -1705,16 +1744,14 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; - - dict = PyType_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - - dict = PyObject_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->align); + } PyErr_SetString(PyExc_TypeError, "no alignment info"); return NULL; @@ -1745,14 +1782,15 @@ byref(PyObject *self, PyObject *args) if (offset == -1 && PyErr_Occurred()) return NULL; } - if (!CDataObject_Check(obj)) { + ctypes_state *st = get_module_state(self); + if (!CDataObject_Check(st, obj)) { PyErr_Format(PyExc_TypeError, "byref() argument must be a ctypes instance, not '%s'", Py_TYPE(obj)->tp_name); return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1770,7 +1808,8 @@ PyDoc_STRVAR(addressof_doc, static PyObject * addressof(PyObject *self, PyObject *obj) { - if (!CDataObject_Check(obj)) { + ctypes_state *st = get_module_state(self); + if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "invalid type"); return NULL; @@ -1821,7 +1860,6 @@ static PyObject * resize(PyObject *self, PyObject *args) { CDataObject *obj; - StgDictObject *dict; Py_ssize_t size; if (!PyArg_ParseTuple(args, @@ -1829,16 +1867,21 @@ resize(PyObject *self, PyObject *args) &obj, &size)) return NULL; - dict = PyObject_stgdict((PyObject *)obj); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + int result = PyStgInfo_FromObject(st, (PyObject *)obj, &info); + if (result < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "expected ctypes instance"); return NULL; } - if (size < dict->size) { + if (size < info->size) { PyErr_Format(PyExc_ValueError, "minimum size is %zd", - dict->size); + info->size); return NULL; } if (obj->b_needsfree == 0) { @@ -1922,19 +1965,19 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - result = PyDict_GetItemWithError(_ctypes_ptrtype_cache, cls); - if (result) { - return Py_NewRef(result); - } - else if (PyErr_Occurred()) { - return NULL; + assert(module); + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { + // found or error + return result; } + // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); - result = PyObject_CallFunction((PyObject *)Py_TYPE(&PyCPointer_Type), + result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), "N(O){}", name, - &PyCPointer_Type); + st->PyCPointer_Type); if (result == NULL) return result; key = PyLong_FromVoidPtr(result); @@ -1945,10 +1988,10 @@ create_pointer_type(PyObject *module, PyObject *cls) } else if (PyType_Check(cls)) { typ = (PyTypeObject *)cls; PyObject *name = PyUnicode_FromFormat("LP_%s", typ->tp_name); - result = PyObject_CallFunction((PyObject *)Py_TYPE(&PyCPointer_Type), + result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), "N(O){sO}", name, - &PyCPointer_Type, + st->PyCPointer_Type, "_type_", cls); if (result == NULL) return result; @@ -1957,7 +2000,7 @@ create_pointer_type(PyObject *module, PyObject *cls) PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); return NULL; } - if (-1 == PyDict_SetItem(_ctypes_ptrtype_cache, key, result)) { + if (PyDict_SetItem(st->_ctypes_ptrtype_cache, key, result) < 0) { Py_DECREF(result); Py_DECREF(key); return NULL; @@ -1986,16 +2029,15 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - typ = PyDict_GetItemWithError(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg)); - if (typ) { - return PyObject_CallOneArg(typ, arg); - } - else if (PyErr_Occurred()) { + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } - typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); - if (typ == NULL) - return NULL; + if (typ == NULL) { + typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); + if (typ == NULL) + return NULL; + } result = PyObject_CallOneArg(typ, arg); Py_DECREF(typ); return result; @@ -2004,28 +2046,30 @@ create_pointer_inst(PyObject *module, PyObject *arg) static PyObject * buffer_info(PyObject *self, PyObject *arg) { - StgDictObject *dict = PyType_stgdict(arg); PyObject *shape; Py_ssize_t i; - if (dict == NULL) - dict = PyObject_stgdict(arg); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, arg, &info) < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "not a ctypes type or object"); return NULL; } - shape = PyTuple_New(dict->ndim); + shape = PyTuple_New(info->ndim); if (shape == NULL) return NULL; - for (i = 0; i < (int)dict->ndim; ++i) - PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i])); + for (i = 0; i < (int)info->ndim; ++i) + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(info->shape[i])); if (PyErr_Occurred()) { Py_DECREF(shape); return NULL; } - return Py_BuildValue("siN", dict->format, dict->ndim, shape); + return Py_BuildValue("siN", info->format, info->ndim, shape); } diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index bfb40e5c5393fc..2c1fb9b862e12d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -14,6 +14,9 @@ #include #include "ctypes.h" +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +# include "../_complex.h" // complex +#endif #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" @@ -31,103 +34,212 @@ static void pymem_destructor(PyObject *ptr) PyCField_Type */ -/* - * Expects the size, index and offset for the current field in *psize and - * *poffset, stores the total size so far in *psize, the offset for the next - * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field descriptor for this field. - */ -/* - * bitfields extension: - * bitsize != 0: this is a bit field. - * pbitofs points to the current bit offset, this will be updated. - * prev_desc points to the type of the previous bitfield, if any. - */ -PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, +static inline +Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + assert(multiple >= 0); + if (multiple == 0) + return numToRound; + return (numToRound / multiple) * multiple; +} + +static inline +Py_ssize_t round_up(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + assert(multiple >= 0); + if (multiple == 0) + return numToRound; + return ((numToRound + multiple - 1) / multiple) * multiple; +} + +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize); +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset); +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); + +/* PyCField_FromDesc creates and returns a struct/union field descriptor. + +The function expects to be called repeatedly for all fields in a struct or +union. It uses helper functions PyCField_FromDesc_gcc and +PyCField_FromDesc_msvc to simulate the corresponding compilers. + +GCC mode places fields one after another, bit by bit. But "each bit field must +fit within a single object of its specified type" (GCC manual, section 15.8 +"Bit Field Packing"). When it doesn't, we insert a few bits of padding to +avoid that. + +MSVC mode works similar except for bitfield packing. Adjacent bit-fields are +packed into the same 1-, 2-, or 4-byte allocation unit if the integral types +are the same size and if the next bit-field fits into the current allocation +unit without crossing the boundary imposed by the common alignment requirements +of the bit-fields. + +See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. + +We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere +to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. + +PyCField_FromDesc manages: +- *psize: the size of the structure / union so far. +- *poffset, *pbitofs: 8* (*poffset) + *pbitofs points to where the next field + would start. +- *palign: the alignment requirements of the last field we placed. +*/ + +static int +PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + CFieldObject* self, StgInfo* info, + int is_bitfield + ) +{ + // We don't use poffset here, so clear it, if it has been set. + *pbitofs += *poffset * 8; + *poffset = 0; + + *palign = info->align; + + if (bitsize > 0) { + // Determine whether the bit field, if placed at the next free bit, + // fits within a single object of its specified type. + // That is: determine a "slot", sized & aligned for the specified type, + // which contains the bitfield's beginning: + Py_ssize_t slot_start_bit = round_down(*pbitofs, 8 * info->align); + Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; + // And see if it also contains the bitfield's last bit: + Py_ssize_t field_end_bit = *pbitofs + bitsize; + if (field_end_bit > slot_end_bit) { + // It doesn't: add padding (bump up to the next alignment boundary) + *pbitofs = round_up(*pbitofs, 8*info->align); + } + } + assert(*poffset == 0); + + self->offset = round_down(*pbitofs, 8*info->align) / 8; + if(is_bitfield) { + Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; + self->size = BUILD_SIZE(bitsize, effective_bitsof); + assert(effective_bitsof <= info->size * 8); + } else { + self->size = info->size; + } + + *pbitofs += bitsize; + *psize = round_up(*pbitofs, 8) / 8; + + return 0; +} + +static int +PyCField_FromDesc_msvc( + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, int pack, + CFieldObject* self, StgInfo* info, + int is_bitfield + ) +{ + if (pack) { + *palign = Py_MIN(pack, info->align); + } else { + *palign = info->align; + } + + // *poffset points to end of current bitfield. + // *pbitofs is generally non-positive, + // and 8 * (*poffset) + *pbitofs points just behind + // the end of the last field we placed. + if (0 < *pbitofs + bitsize || 8 * info->size != *pfield_size) { + // Close the previous bitfield (if any). + // and start a new bitfield: + *poffset = round_up(*poffset, *palign); + + *poffset += info->size; + + *pfield_size = info->size * 8; + // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a + // new field. Ie just behind where we placed the last field plus an + // allowance for alignment. + *pbitofs = - *pfield_size; + } + + assert(8 * info->size == *pfield_size); + + self->offset = *poffset - (*pfield_size) / 8; + if(is_bitfield) { + assert(0 <= (*pfield_size + *pbitofs)); + assert((*pfield_size + *pbitofs) < info->size * 8); + self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); + } else { + self->size = info->size; + } + assert(*pfield_size + *pbitofs <= info->size * 8); + + *pbitofs += bitsize; + *psize = *poffset; + + return 0; +} + +PyObject * +PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + int pack, int big_endian, LayoutMode layout_mode) { - CFieldObject *self; - PyObject *proto; - Py_ssize_t size, align; - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - StgDictObject *dict; - int fieldtype; -#define NO_BITFIELD 0 -#define NEW_BITFIELD 1 -#define CONT_BITFIELD 2 -#define EXPAND_BITFIELD 3 - - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->PyCField_Type; - self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) + CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + if (self == NULL) { return NULL; - dict = PyType_stgdict(desc); - if (!dict) { + } + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(self); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); Py_DECREF(self); return NULL; } - if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ -#ifdef MS_WIN32 - /* MSVC, GCC with -mms-bitfields */ - && dict->size * 8 == *pfield_size -#else - /* GCC */ - && dict->size * 8 <= *pfield_size -#endif - && (*pbitofs + bitsize) <= *pfield_size) { - /* continue bit field */ - fieldtype = CONT_BITFIELD; -#ifndef MS_WIN32 - } else if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ - && dict->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= dict->size * 8) { - /* expand bit field */ - fieldtype = EXPAND_BITFIELD; -#endif - } else if (bitsize) { - /* start new bitfield */ - fieldtype = NEW_BITFIELD; - *pbitofs = 0; - *pfield_size = dict->size * 8; - } else { - /* not a bit field */ - fieldtype = NO_BITFIELD; - *pbitofs = 0; - *pfield_size = 0; - } - size = dict->size; - proto = desc; + PyObject* proto = desc; /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... */ - if (PyCArrayTypeObject_Check(proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { + SETFUNC setfunc = NULL; + GETFUNC getfunc = NULL; + if (PyCArrayTypeObject_Check(st, proto)) { + StgInfo *ainfo; + if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { + Py_DECREF(self); + return NULL; + } + + if (ainfo && ainfo->proto) { + StgInfo *iinfo; + if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { + Py_DECREF(self); + return NULL; + } + if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); Py_DECREF(self); return NULL; } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); getfunc = fd->getfunc; setfunc = fd->setfunc; } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); getfunc = fd->getfunc; setfunc = fd->setfunc; @@ -141,61 +253,43 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, self->proto = Py_NewRef(proto); - switch (fieldtype) { - case NEW_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - *pbitofs = bitsize; - /* fall through */ - case NO_BITFIELD: - if (pack) - align = min(pack, dict->align); - else - align = dict->align; - if (align && *poffset % align) { - Py_ssize_t delta = align - (*poffset % align); - *psize += delta; - *poffset += delta; - } - - if (bitsize == 0) - self->size = size; - *psize += size; - - self->offset = *poffset; - *poffset += size; - - *palign = align; - break; - - case EXPAND_BITFIELD: - *poffset += dict->size - *pfield_size/8; - *psize += dict->size - *pfield_size/8; - - *pfield_size = dict->size * 8; - - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - - case CONT_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; + int is_bitfield = !!bitsize; + if(!is_bitfield) { + assert(info->size >= 0); + // assert: no overflow; + assert((unsigned long long int) info->size + < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); + bitsize = 8 * info->size; + // Caution: bitsize might still be 0 now. + } + assert(bitsize <= info->size * 8); + + int result; + if (layout_mode == LAYOUT_MODE_MS) { + result = PyCField_FromDesc_msvc( + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, + self, info, + is_bitfield + ); + } else { + assert(pack == 0); + result = PyCField_FromDesc_gcc( + bitsize, pbitofs, + psize, poffset, palign, + self, info, + is_bitfield + ); + } + if (result < 0) { + Py_DECREF(self); + return NULL; + } + assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); + if(big_endian && is_bitfield) { + self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } - return (PyObject *)self; } @@ -204,7 +298,8 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - if (!CDataObject_Check(inst)) { + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); + if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return -1; @@ -216,7 +311,7 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) "can't delete attribute"); return -1; } - return PyCData_set(inst, self->proto, self->setfunc, value, + return PyCData_set(st, inst, self->proto, self->setfunc, value, self->index, self->size, ptr); } @@ -227,13 +322,14 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) if (inst == NULL) { return Py_NewRef(self); } - if (!CDataObject_Check(inst)) { + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); + if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return NULL; } src = (CDataObject *)inst; - return PyCData_get(self->proto, self->getfunc, inst, + return PyCData_get(st, self->proto, self->getfunc, inst, self->index, self->size, src->b_ptr + self->offset); } @@ -284,8 +380,8 @@ static PyObject * PyCField_repr(CFieldObject *self) { PyObject *result; - Py_ssize_t bits = self->size >> 16; - Py_ssize_t size = self->size & 0xFFFF; + Py_ssize_t bits = NUM_BITS(self->size); + Py_ssize_t size = LOW_BIT(self->size); const char *name; name = ((PyTypeObject *)self->proto)->tp_name; @@ -382,8 +478,28 @@ get_ulonglong(PyObject *v, unsigned long long *p) */ /* how to decode the size field, for integer get/set functions */ -#define LOW_BIT(x) ((x) & 0xFFFF) -#define NUM_BITS(x) ((x) >> 16) +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset) { + return offset & 0xFFFF; +} +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { + return bitsize >> 16; +} + +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset) { + assert(0 <= offset); + assert(offset <= 0xFFFF); + // We don't support zero length bitfields. + // And GET_BITFIELD uses NUM_BITS(size)==0, + // to figure out whether we are handling a bitfield. + assert(0 < bitsize); + Py_ssize_t result = (bitsize << 16) + offset; + assert(bitsize == NUM_BITS(result)); + assert(offset == LOW_BIT(result)); + return result; +} /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ #define BIT_MASK(type, size) (((((type)1 << (NUM_BITS(size) - 1)) - 1) << 1) + 1) @@ -974,6 +1090,74 @@ d_get(void *ptr, Py_ssize_t size) return PyFloat_FromDouble(val); } +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +static PyObject * +C_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + double complex x = CMPLX(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +C_get(void *ptr, Py_ssize_t size) +{ + double complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} + +static PyObject * +E_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + float complex x = CMPLXF((float)c.real, (float)c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +E_get(void *ptr, Py_ssize_t size) +{ + float complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles(crealf(x), cimagf(x)); +} + +static PyObject * +F_set(void *ptr, PyObject *value, Py_ssize_t size) +{ + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } + long double complex x = CMPLXL(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); + _RET(value); +} + +static PyObject * +F_get(void *ptr, Py_ssize_t size) +{ + long double complex x; + + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles((double)creall(x), (double)cimagl(x)); +} +#endif + static PyObject * d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1086,25 +1270,45 @@ O_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { - if (PyBytes_Check(value) && PyBytes_GET_SIZE(value) == 1) { + if (PyBytes_Check(value)) { + if (PyBytes_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length %zd", + PyBytes_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyBytes_AS_STRING(value)[0]; _RET(value); } - if (PyByteArray_Check(value) && PyByteArray_GET_SIZE(value) == 1) { + if (PyByteArray_Check(value)) { + if (PyByteArray_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytearray of length %zd", + PyByteArray_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyByteArray_AS_STRING(value)[0]; _RET(value); } - if (PyLong_Check(value)) - { - long longval = PyLong_AsLong(value); - if (longval < 0 || longval >= 256) - goto error; + if (PyLong_Check(value)) { + int overflow; + long longval = PyLong_AsLongAndOverflow(value, &overflow); + if (longval == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow || longval < 0 || longval >= 256) { + PyErr_SetString(PyExc_TypeError, "integer not in range(256)"); + return NULL; + } *(char *)ptr = (char)longval; _RET(value); } - error: PyErr_Format(PyExc_TypeError, - "one character bytes, bytearray or integer expected"); + "one character bytes, bytearray, or an integer " + "in range(256) expected, not %T", + value); return NULL; } @@ -1123,22 +1327,27 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) wchar_t chars[2]; if (!PyUnicode_Check(value)) { PyErr_Format(PyExc_TypeError, - "unicode string expected instead of %s instance", - Py_TYPE(value)->tp_name); + "a unicode character expected, not instance of %T", + value); return NULL; - } else - Py_INCREF(value); + } len = PyUnicode_AsWideChar(value, chars, 2); if (len != 1) { - Py_DECREF(value); - PyErr_SetString(PyExc_TypeError, - "one character unicode string expected"); + if (PyUnicode_GET_LENGTH(value) != 1) { + PyErr_Format(PyExc_TypeError, + "a unicode character expected, not a string of length %zd", + PyUnicode_GET_LENGTH(value)); + } + else { + PyErr_Format(PyExc_TypeError, + "the string %A cannot be converted to a single wchar_t character", + value); + } return NULL; } *(wchar_t *)ptr = chars[0]; - Py_DECREF(value); _RET(value); } @@ -1454,6 +1663,11 @@ static struct fielddesc formattable[] = { { 'B', B_set, B_get, NULL}, { 'c', c_set, c_get, NULL}, { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) + { 'C', C_set, C_get, NULL}, + { 'E', E_set, E_get, NULL}, + { 'F', F_set, F_get, NULL}, +#endif { 'g', g_set, g_get, NULL}, { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, @@ -1504,6 +1718,11 @@ _ctypes_init_fielddesc(void) case 'B': fd->pffi_type = &ffi_type_uchar; break; case 'c': fd->pffi_type = &ffi_type_schar; break; case 'd': fd->pffi_type = &ffi_type_double; break; +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) + case 'C': fd->pffi_type = &ffi_type_complex_double; break; + case 'E': fd->pffi_type = &ffi_type_complex_float; break; + case 'F': fd->pffi_type = &ffi_type_complex_longdouble; break; +#endif case 'g': fd->pffi_type = &ffi_type_longdouble; break; case 'f': fd->pffi_type = &ffi_type_float; break; case 'h': fd->pffi_type = &ffi_type_sshort; break; diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h new file mode 100644 index 00000000000000..e1d5a17cbe7d68 --- /dev/null +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -0,0 +1,613 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return memory consumption of the type object."); + +#define _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF \ + {"__sizeof__", _PyCFunction_CAST(_ctypes_CType_Type___sizeof__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _ctypes_CType_Type___sizeof____doc__}, + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__sizeof__() takes no arguments"); + return NULL; + } + return _ctypes_CType_Type___sizeof___impl(self, cls); +} + +PyDoc_STRVAR(CDataType_from_address__doc__, +"from_address($self, value, /)\n" +"--\n" +"\n" +"C.from_address(integer) -> C instance\n" +"\n" +"Access a C instance at the specified address."); + +#define CDATATYPE_FROM_ADDRESS_METHODDEF \ + {"from_address", _PyCFunction_CAST(CDataType_from_address), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_address__doc__}, + +static PyObject * +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +CDataType_from_address(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_address", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_address_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer__doc__, +"from_buffer($self, obj, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a writeable buffer."); + +#define CDATATYPE_FROM_BUFFER_METHODDEF \ + {"from_buffer", _PyCFunction_CAST(CDataType_from_buffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer__doc__}, + +static PyObject * +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *obj; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_impl(type, cls, obj, offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer_copy__doc__, +"from_buffer_copy($self, buffer, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer_copy(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a readable buffer."); + +#define CDATATYPE_FROM_BUFFER_COPY_METHODDEF \ + {"from_buffer_copy", _PyCFunction_CAST(CDataType_from_buffer_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer_copy__doc__}, + +static PyObject * +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer_copy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_copy_impl(type, cls, &buffer, offset); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +PyDoc_STRVAR(CDataType_in_dll__doc__, +"in_dll($self, dll, name, /)\n" +"--\n" +"\n" +"C.in_dll(dll, name) -> C instance\n" +"\n" +"Access a C instance in a dll."); + +#define CDATATYPE_IN_DLL_METHODDEF \ + {"in_dll", _PyCFunction_CAST(CDataType_in_dll), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_in_dll__doc__}, + +static PyObject * +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name); + +static PyObject * +CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "in_dll", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *dll; + const char *name; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + dll = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("in_dll", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = CDataType_in_dll_impl(type, cls, dll, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define CDATATYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(CDataType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_param__doc__}, + +static PyObject * +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +CDataType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_set_type__doc__, +"set_type($self, type, /)\n" +"--\n" +"\n"); + +#define PYCPOINTERTYPE_SET_TYPE_METHODDEF \ + {"set_type", _PyCFunction_CAST(PyCPointerType_set_type), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_set_type__doc__}, + +static PyObject * +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type); + +static PyObject * +PyCPointerType_set_type(PyTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_type", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *type; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + type = args[0]; + return_value = PyCPointerType_set_type_impl(self, cls, type); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCPOINTERTYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCPointerType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_from_param__doc__}, + +static PyObject * +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCPointerType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCPointerType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_wchar_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_WCHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_wchar_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_wchar_p_from_param__doc__}, + +static PyObject * +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_wchar_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_wchar_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_char_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_CHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_char_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_char_p_from_param__doc__}, + +static PyObject * +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_char_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_char_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_void_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_VOID_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_void_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_void_p_from_param__doc__}, + +static PyObject * +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_void_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_void_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCSimpleType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCSIMPLETYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCSimpleType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCSimpleType_from_param__doc__}, + +static PyObject * +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCSimpleType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCSimpleType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCData_reduce__doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define PYCDATA_REDUCE_METHODDEF \ + {"__reduce__", _PyCFunction_CAST(PyCData_reduce), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCData_reduce__doc__}, + +static PyObject * +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls); + +static PyObject * +PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); + return NULL; + } + return PyCData_reduce_impl(myself, cls); +} + +PyDoc_STRVAR(Simple_from_outparm__doc__, +"__ctypes_from_outparam__($self, /)\n" +"--\n" +"\n"); + +#define SIMPLE_FROM_OUTPARM_METHODDEF \ + {"__ctypes_from_outparam__", _PyCFunction_CAST(Simple_from_outparm), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Simple_from_outparm__doc__}, + +static PyObject * +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__ctypes_from_outparam__() takes no arguments"); + return NULL; + } + return Simple_from_outparm_impl(self, cls); +} +/*[clinic end generated code: output=a90886be2a294ee6 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 8891a0a741de7b..a794cfe86b5f42 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -2,6 +2,15 @@ # include #endif +#include // FFI_TARGET_HAS_COMPLEX_TYPE + +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) +# include "../_complex.h" // complex +#endif + #ifndef MS_WIN32 #define max(a, b) ((a) > (b) ? (a) : (b)) #define min(a, b) ((a) < (b) ? (a) : (b)) @@ -32,20 +41,72 @@ #endif #endif +#ifdef MS_WIN32 +#include // for IUnknown interface +#endif + typedef struct { PyTypeObject *DictRemover_Type; PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; + PyTypeObject *StructParam_Type; + PyTypeObject *PyCType_Type; + PyTypeObject *PyCStructType_Type; + PyTypeObject *UnionType_Type; + PyTypeObject *PyCPointerType_Type; + PyTypeObject *PyCArrayType_Type; + PyTypeObject *PyCSimpleType_Type; + PyTypeObject *PyCFuncPtrType_Type; + PyTypeObject *PyCData_Type; + PyTypeObject *Struct_Type; + PyTypeObject *Union_Type; + PyTypeObject *PyCArray_Type; + PyTypeObject *Simple_Type; + PyTypeObject *PyCPointer_Type; + PyTypeObject *PyCFuncPtr_Type; #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif - PyTypeObject *StructParam_Type; + /* This dict maps ctypes types to POINTER types */ + PyObject *_ctypes_ptrtype_cache; + /* a callable object used for unpickling: + strong reference to _ctypes._unpickle() function */ + PyObject *_unpickle; + PyObject *array_cache; + PyObject *error_object_name; // callproc.c + PyObject *PyExc_ArgError; + PyObject *swapped_suffix; } ctypes_state; -extern ctypes_state global_state; -#define GLOBAL_STATE() (&global_state) +extern struct PyModuleDef _ctypesmodule; + + +static inline ctypes_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (ctypes_state *)state; +} + +static inline ctypes_state * +get_module_state_by_class(PyTypeObject *cls) +{ + ctypes_state *state = (ctypes_state *)_PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static inline ctypes_state * +get_module_state_by_def(PyTypeObject *cls) +{ + PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); + assert(mod != NULL); + return get_module_state(mod); +} + extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; @@ -55,7 +116,7 @@ typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); -typedef PyCArgObject *(* PARAMFUNC)(CDataObject *obj); +typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If this buffer is too small, PyMem_Malloc will be called to create a larger one, @@ -118,7 +179,7 @@ typedef struct { Py_ssize_t b_size; /* size of memory block in bytes */ Py_ssize_t b_length; /* number of references we need */ Py_ssize_t b_index; /* index of this object into base's - b_object list */ + b_object list */ PyObject *b_objects; /* list of references we need to keep */ union value b_value; /* end of tagCDataObject, additional fields follow */ @@ -126,7 +187,7 @@ typedef struct { CThunkObject *thunk; PyObject *callable; - /* These two fields will override the ones in the type's stgdict if + /* These two fields will override the ones in the type's stginfo if they are set */ PyObject *converters; PyObject *argtypes; @@ -140,59 +201,51 @@ typedef struct { PyObject *paramflags; } PyCFuncPtrObject; -extern PyTypeObject PyCStgDict_Type; -#define PyCStgDict_CheckExact(v) Py_IS_TYPE(v, &PyCStgDict_Type) -#define PyCStgDict_Check(v) PyObject_TypeCheck(v, &PyCStgDict_Type) - -extern int PyCStructUnionType_update_stgdict(PyObject *fields, PyObject *type, int isStruct); +extern int PyCStructUnionType_update_stginfo(PyObject *fields, PyObject *type, int isStruct); extern int PyType_stginfo(PyTypeObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); -extern PyTypeObject PyCData_Type; -#define CDataObject_CheckExact(v) Py_IS_TYPE(v, &PyCData_Type) -#define CDataObject_Check(v) PyObject_TypeCheck(v, &PyCData_Type) +#define CDataObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCData_Type) +#define CDataObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCData_Type) #define _CDataObject_HasExternalBuffer(v) ((v)->b_ptr != (char *)&(v)->b_value) -extern PyTypeObject PyCSimpleType_Type; -#define PyCSimpleTypeObject_CheckExact(v) Py_IS_TYPE(v, &PyCSimpleType_Type) -#define PyCSimpleTypeObject_Check(v) PyObject_TypeCheck(v, &PyCSimpleType_Type) +#define PyCSimpleTypeObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCSimpleType_Type) +#define PyCSimpleTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCSimpleType_Type) extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); +typedef enum { + LAYOUT_MODE_MS, + LAYOUT_MODE_GCC_SYSV, +} LayoutMode; extern PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian); - -extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); -extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); - -extern PyTypeObject PyCArrayType_Type; -extern PyTypeObject PyCArray_Type; -extern PyTypeObject PyCPointerType_Type; -extern PyTypeObject PyCPointer_Type; -extern PyTypeObject PyCFuncPtr_Type; -extern PyTypeObject PyCFuncPtrType_Type; -extern PyTypeObject PyCStructType_Type; - -#define PyCArrayTypeObject_Check(v) PyObject_TypeCheck(v, &PyCArrayType_Type) -#define ArrayObject_Check(v) PyObject_TypeCheck(v, &PyCArray_Type) -#define PointerObject_Check(v) PyObject_TypeCheck(v, &PyCPointer_Type) -#define PyCPointerTypeObject_Check(v) PyObject_TypeCheck(v, &PyCPointerType_Type) -#define PyCFuncPtrObject_Check(v) PyObject_TypeCheck(v, &PyCFuncPtr_Type) -#define PyCFuncPtrTypeObject_Check(v) PyObject_TypeCheck(v, &PyCFuncPtrType_Type) -#define PyCStructTypeObject_Check(v) PyObject_TypeCheck(v, &PyCStructType_Type) +PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, + int pack, int is_big_endian, LayoutMode layout_mode); + +extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); +extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); + +#define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) +#define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) +#define PointerObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCPointer_Type) +#define PyCPointerTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCPointerType_Type) +#define PyCFuncPtrObject_Check(st,v) PyObject_TypeCheck((v), (st)->PyCFuncPtr_Type) +#define PyCFuncPtrTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCFuncPtrType_Type) +#define PyCStructTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStructType_Type) extern PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length); +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length); extern PyMethodDef _ctypes_module_methods[]; -extern CThunkObject *_ctypes_alloc_callback(PyObject *callable, +extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags); @@ -218,45 +271,22 @@ typedef struct { int anonymous; } CFieldObject; -/* A subclass of PyDictObject, used as the instance dictionary of ctypes - metatypes */ -typedef struct { - PyDictObject dict; /* first part identical to PyDictObject */ -/* The size and align fields are unneeded, they are in ffi_type as well. As - an experiment shows, it's trivial to get rid of them, the only thing to - remember is that in PyCArrayType_new the ffi_type fields must be filled in - - so far it was unneeded because libffi doesn't support arrays at all - (because they are passed as pointers to function calls anyway). But it's - too much risk to change that now, and there are other fields which doesn't - belong into this structure anyway. Maybe in ctypes 2.0... (ctypes 2000?) -*/ - Py_ssize_t size; /* number of bytes */ - Py_ssize_t align; /* alignment requirements */ - Py_ssize_t length; /* number of fields */ - ffi_type ffi_type_pointer; - PyObject *proto; /* Only for Pointer/ArrayObject */ - SETFUNC setfunc; /* Only for simple objects */ - GETFUNC getfunc; /* Only for simple objects */ - PARAMFUNC paramfunc; +/**************************************************************** + StgInfo - /* Following fields only used by PyCFuncPtrType_Type instances */ - PyObject *argtypes; /* tuple of CDataObjects */ - PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ - PyObject *restype; /* CDataObject or NULL */ - PyObject *checker; - int flags; /* calling convention and such */ + Since Python 3.13, ctypes-specific type information is stored in the + corresponding type object, in a `StgInfo` struct accessed by the helpers + below. + Before that, each type's `tp_dict` was set to a dict *subclass* that included + the fields that are now in StgInfo. The mechanism was called "StgDict"; a few + references to that name might remain. - /* pep3118 fields, pointers need PyMem_Free */ - char *format; - int ndim; - Py_ssize_t *shape; -/* Py_ssize_t *strides; */ /* unused in ctypes */ -/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + Functions for accessing StgInfo are `static inline` for performance; + see later in this file. -} StgDictObject; + **************************************************************** -/**************************************************************** - StgDictObject fields + StgInfo fields setfunc and getfunc is only set for simple data types, it is copied from the corresponding fielddesc entry. These are functions to set and get the value @@ -267,11 +297,11 @@ typedef struct { object. Probably all the magic ctypes methods (like from_param) should have C - callable wrappers in the StgDictObject. For simple data type, for example, + callable wrappers in the StgInfo. For simple data type, for example, the fielddesc table could have entries for C codec from_param functions or other methods as well, if a subtype overrides this method in Python at construction time, or assigns to it later, tp_setattro should update the - StgDictObject function to a generic one. + StgInfo function to a generic one. Currently, PyCFuncPtr types have 'converters' and 'checker' entries in their type dict. They are only used to cache attributes from other entries, which @@ -295,17 +325,40 @@ typedef struct { *****************************************************************/ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyType_stgdict(PyObject *obj); +typedef struct { + int initialized; + Py_ssize_t size; /* number of bytes */ + Py_ssize_t align; /* alignment requirements */ + Py_ssize_t length; /* number of fields */ + ffi_type ffi_type_pointer; + PyObject *proto; /* Only for Pointer/ArrayObject */ + SETFUNC setfunc; /* Only for simple objects */ + GETFUNC getfunc; /* Only for simple objects */ + PARAMFUNC paramfunc; -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyObject_stgdict(PyObject *self); + /* Following fields only used by PyCFuncPtrType_Type instances */ + PyObject *argtypes; /* tuple of CDataObjects */ + PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ + PyObject *restype; /* CDataObject or NULL */ + PyObject *checker; + PyObject *module; + int flags; /* calling convention and such */ -extern int PyCStgDict_clone(StgDictObject *src, StgDictObject *dst); + /* pep3118 fields, pointers need PyMem_Free */ + char *format; + int ndim; + Py_ssize_t *shape; +/* Py_ssize_t *strides; */ /* unused in ctypes */ +/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ +} StgInfo; + +extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); +extern void ctype_clear_stginfo(StgInfo *info); typedef int(* PPROC)(void); -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *arguments, #ifdef MS_WIN32 IUnknown *pIUnk, @@ -346,20 +399,26 @@ struct tagPyCArgObject { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(FFI_TARGET_HAS_COMPLEX_TYPE) + double complex C; + float complex E; + long double complex F; +#endif } value; PyObject *obj; Py_ssize_t size; /* for the 'V' tag */ }; #define PyCArg_CheckExact(st, v) Py_IS_TYPE(v, st->PyCArg_Type) -extern PyCArgObject *PyCArgObject_new(void); +extern PyCArgObject *PyCArgObject_new(ctypes_state *st); extern PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *ptr); extern int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); @@ -370,30 +429,21 @@ struct basespec { char *adr; }; -extern char basespec_string[]; - -extern ffi_type *_ctypes_get_ffi_type(PyObject *obj); - -/* exception classes */ -extern PyObject *PyExc_ArgError; - -extern char *_ctypes_conversion_encoding; -extern char *_ctypes_conversion_errors; - +extern ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj); extern void _ctypes_free_closure(void *); extern void *_ctypes_alloc_closure(void); -extern PyObject *PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr); +extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, + PyObject *base, Py_ssize_t index, char *adr); extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); extern char *_ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix); -extern int _ctypes_simple_instance(PyObject *obj); +extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); -extern PyObject *_ctypes_ptrtype_cache; -PyObject *_ctypes_get_errobj(int **pspace); +PyObject *_ctypes_get_errobj(ctypes_state *st, int **pspace); #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); @@ -403,8 +453,94 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc); #define Py_ffi_closure_alloc ffi_closure_alloc #endif -/* - Local Variables: - compile-command: "python setup.py -q build install --home ~" - End: -*/ + +/**************************************************************** + * Accessing StgInfo -- these are inlined for performance reasons. + */ + +// `PyStgInfo_From**` functions get a PyCTypeDataObject. +// These return -1 on error, 0 if "not found", 1 on OK. +// (Currently, these do not return -1 in practice. This might change +// in the future.) + +// +// Common helper: +static inline int +_stginfo_from_type(ctypes_state *state, PyTypeObject *type, StgInfo **result) +{ + *result = NULL; + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + // not a ctypes class. + return 0; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + assert(info != NULL); + if (!info->initialized) { + // StgInfo is not initialized. This happens in abstract classes. + return 0; + } + *result = info; + return 1; +} +// from a type: +static inline int +PyStgInfo_FromType(ctypes_state *state, PyObject *type, StgInfo **result) +{ + return _stginfo_from_type(state, (PyTypeObject *)type, result); +} +// from an instance: +static inline int +PyStgInfo_FromObject(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + return _stginfo_from_type(state, Py_TYPE(obj), result); +} +// from either a type or an instance: +static inline int +PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + if (PyType_Check(obj)) { + return _stginfo_from_type(state, (PyTypeObject *)obj, result); + } + return _stginfo_from_type(state, Py_TYPE(obj), result); +} + +/* A variant of PyStgInfo_FromType that doesn't need the state, + * so it can be called from finalization functions when the module + * state is torn down. Does no checks; cannot fail. + * This inlines the current implementation PyObject_GetTypeData, + * so it might break in the future. + */ +static inline StgInfo * +_PyStgInfo_FromType_NoState(PyObject *type) +{ + size_t type_basicsize =_Py_SIZE_ROUND_UP(PyType_Type.tp_basicsize, + ALIGNOF_MAX_ALIGN_T); + return (StgInfo *)((char *)type + type_basicsize); +} + +// Initialize StgInfo on a newly created type +static inline StgInfo * +PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) +{ + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + PyErr_Format(PyExc_SystemError, + "'%s' is not a ctypes class.", + type->tp_name); + return NULL; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + if (info->initialized) { + PyErr_Format(PyExc_SystemError, + "StgInfo of '%s' is already initialized.", + type->tp_name); + return NULL; + } + PyObject *module = PyType_GetModule(state->PyCType_Type); + if (!module) { + return NULL; + } + info->module = Py_NewRef(module); + + info->initialized = 1; + return info; +} diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6fbcf77a115371..970f0a033fbb0b 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -16,196 +16,64 @@ #endif #include "ctypes.h" -/******************************************************************/ -/* - StdDict - a dictionary subclass, containing additional C accessible fields - - XXX blabla more -*/ - -/* Seems we need this, otherwise we get problems when calling - * PyDict_SetItem() (ma_lookup is NULL) +/* This file relates to StgInfo -- type-specific information for ctypes. + * See ctypes.h for details. */ -static int -PyCStgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds) -{ - if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->format = NULL; - self->ndim = 0; - self->shape = NULL; - return 0; -} - -static int -PyCStgDict_clear(StgDictObject *self) -{ - Py_CLEAR(self->proto); - Py_CLEAR(self->argtypes); - Py_CLEAR(self->converters); - Py_CLEAR(self->restype); - Py_CLEAR(self->checker); - return 0; -} - -static void -PyCStgDict_dealloc(StgDictObject *self) -{ - PyCStgDict_clear(self); - PyMem_Free(self->format); - PyMem_Free(self->shape); - PyMem_Free(self->ffi_type_pointer.elements); - PyDict_Type.tp_dealloc((PyObject *)self); -} - -static PyObject * -PyCStgDict_sizeof(StgDictObject *self, void *unused) -{ - Py_ssize_t res; - - res = _PyDict_SizeOf((PyDictObject *)self); - res += sizeof(StgDictObject) - sizeof(PyDictObject); - if (self->format) - res += strlen(self->format) + 1; - res += self->ndim * sizeof(Py_ssize_t); - if (self->ffi_type_pointer.elements) - res += (self->length + 1) * sizeof(ffi_type *); - return PyLong_FromSsize_t(res); -} int -PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) +PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { - char *d, *s; Py_ssize_t size; - PyCStgDict_clear(dst); - PyMem_Free(dst->ffi_type_pointer.elements); - PyMem_Free(dst->format); - dst->format = NULL; - PyMem_Free(dst->shape); - dst->shape = NULL; - dst->ffi_type_pointer.elements = NULL; - - d = (char *)dst; - s = (char *)src; - memcpy(d + sizeof(PyDictObject), - s + sizeof(PyDictObject), - sizeof(StgDictObject) - sizeof(PyDictObject)); - - Py_XINCREF(dst->proto); - Py_XINCREF(dst->argtypes); - Py_XINCREF(dst->converters); - Py_XINCREF(dst->restype); - Py_XINCREF(dst->checker); - - if (src->format) { - dst->format = PyMem_Malloc(strlen(src->format) + 1); - if (dst->format == NULL) { + ctype_clear_stginfo(dst_info); + PyMem_Free(dst_info->ffi_type_pointer.elements); + PyMem_Free(dst_info->format); + dst_info->format = NULL; + PyMem_Free(dst_info->shape); + dst_info->shape = NULL; + dst_info->ffi_type_pointer.elements = NULL; + + memcpy(dst_info, src_info, sizeof(StgInfo)); + + Py_XINCREF(dst_info->proto); + Py_XINCREF(dst_info->argtypes); + Py_XINCREF(dst_info->converters); + Py_XINCREF(dst_info->restype); + Py_XINCREF(dst_info->checker); + Py_XINCREF(dst_info->module); + + if (src_info->format) { + dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); + if (dst_info->format == NULL) { PyErr_NoMemory(); return -1; } - strcpy(dst->format, src->format); + strcpy(dst_info->format, src_info->format); } - if (src->shape) { - dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); - if (dst->shape == NULL) { + if (src_info->shape) { + dst_info->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src_info->ndim); + if (dst_info->shape == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->shape, src->shape, - sizeof(Py_ssize_t) * src->ndim); + memcpy(dst_info->shape, src_info->shape, + sizeof(Py_ssize_t) * src_info->ndim); } - if (src->ffi_type_pointer.elements == NULL) + if (src_info->ffi_type_pointer.elements == NULL) return 0; - size = sizeof(ffi_type *) * (src->length + 1); - dst->ffi_type_pointer.elements = PyMem_Malloc(size); - if (dst->ffi_type_pointer.elements == NULL) { + size = sizeof(ffi_type *) * (src_info->length + 1); + dst_info->ffi_type_pointer.elements = PyMem_Malloc(size); + if (dst_info->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->ffi_type_pointer.elements, - src->ffi_type_pointer.elements, + memcpy(dst_info->ffi_type_pointer.elements, + src_info->ffi_type_pointer.elements, size); return 0; } -static struct PyMethodDef PyCStgDict_methods[] = { - {"__sizeof__", (PyCFunction)PyCStgDict_sizeof, METH_NOARGS}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyCStgDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "StgDict", - sizeof(StgDictObject), - 0, - (destructor)PyCStgDict_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCStgDict_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyCStgDict_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ -}; - -/* May return NULL, but does not set an exception! */ -StgDictObject * -PyType_stgdict(PyObject *obj) -{ - PyTypeObject *type; - - if (!PyType_Check(obj)) - return NULL; - type = (PyTypeObject *)obj; - if (!type->tp_dict || !PyCStgDict_CheckExact(type->tp_dict)) - return NULL; - return (StgDictObject *)type->tp_dict; -} - -/* May return NULL, but does not set an exception! */ -/* - This function should be as fast as possible, so we don't call PyType_stgdict - above but inline the code, and avoid the PyType_Check(). -*/ -StgDictObject * -PyObject_stgdict(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - if (!type->tp_dict || !PyCStgDict_CheckExact(type->tp_dict)) - return NULL; - return (StgDictObject *)type->tp_dict; -} - /* descr is the descriptor for a field marked as anonymous. Get all the _fields_ descriptors from descr->proto, create new descriptors with offset and index adjusted, and stuff them into type. @@ -226,7 +94,7 @@ MakeFields(PyObject *type, CFieldObject *descr, if (fieldlist == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(descr)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(fieldlist); ++i) { PyObject *pair = PySequence_Fast_GET_ITEM(fieldlist, i); /* borrowed */ @@ -307,7 +175,7 @@ MakeAnonFields(PyObject *type) if (anon_names == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(anon_names); ++i) { PyObject *fname = PySequence_Fast_GET_ITEM(anon_names, i); /* borrowed */ @@ -367,18 +235,18 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, - and create an StgDictObject. Used for Structure and Union subclasses. + and initialize StgInfo. Used for Structure and Union subclasses. */ int -PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) +PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - StgDictObject *stgdict, *basedict; Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align, aligned_size; Py_ssize_t field_size = 0; - int bitofs; + Py_ssize_t bitofs = 0; PyObject *tmp; int pack; + int forced_alignment = 1; Py_ssize_t ffi_ofs; int big_endian; int arrays_seen = 0; @@ -419,6 +287,60 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct pack = 0; } + #ifdef MS_WIN32 + LayoutMode layout_mode = LAYOUT_MODE_MS; + #else + LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; + #endif + + if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { + return -1; + } + if (tmp) { + if (!PyUnicode_Check(tmp)) { + PyErr_SetString(PyExc_TypeError, + "_layout_ must be a string"); + return -1; + } + if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { + layout_mode = LAYOUT_MODE_MS; + } + else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { + layout_mode = LAYOUT_MODE_GCC_SYSV; + if (pack > 0) { + PyErr_SetString(PyExc_ValueError, + "_pack_ is not compatible with _layout_=\"gcc-sysv\""); + return -1; + } + } + else { + PyErr_Format(PyExc_ValueError, + "unknown _layout_ %R", tmp); + return -1; + } + } + if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { + return -1; + } + if (tmp) { + forced_alignment = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (forced_alignment < 0) { + if (!PyErr_Occurred() || + PyErr_ExceptionMatches(PyExc_TypeError) || + PyErr_ExceptionMatches(PyExc_OverflowError)) + { + PyErr_SetString(PyExc_ValueError, + "_align_ must be a non-negative integer"); + } + return -1; + } + } + else { + /* Setting `_align_ = 0` amounts to using the default alignment */ + forced_alignment = 1; + } + len = PySequence_Size(fields); if (len == -1) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -428,114 +350,130 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } - stgdict = PyType_stgdict(type); - if (!stgdict) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, type, &stginfo) < 0) { + return -1; + } + if (!stginfo) { PyErr_SetString(PyExc_TypeError, "ctypes state is not initialized"); return -1; } + /* If this structure/union is already marked final we cannot assign _fields_ anymore. */ - if (stgdict->flags & DICTFLAG_FINAL) {/* is final ? */ + if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ PyErr_SetString(PyExc_AttributeError, "_fields_ is final"); return -1; } - if (stgdict->format) { - PyMem_Free(stgdict->format); - stgdict->format = NULL; + if (stginfo->format) { + PyMem_Free(stginfo->format); + stginfo->format = NULL; } - if (stgdict->ffi_type_pointer.elements) - PyMem_Free(stgdict->ffi_type_pointer.elements); + if (stginfo->ffi_type_pointer.elements) + PyMem_Free(stginfo->ffi_type_pointer.elements); - basedict = PyType_stgdict((PyObject *)((PyTypeObject *)type)->tp_base); - if (basedict) { - stgdict->flags |= (basedict->flags & + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)type)->tp_base, + &baseinfo) < 0) { + return -1; + } + if (baseinfo) { + stginfo->flags |= (baseinfo->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); } if (!isStruct) { - stgdict->flags |= TYPEFLAG_HASUNION; + stginfo->flags |= TYPEFLAG_HASUNION; } - if (basedict) { - size = offset = basedict->size; - align = basedict->align; + if (baseinfo) { + size = offset = baseinfo->size; + align = baseinfo->align; union_size = 0; total_align = align ? align : 1; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + total_align = max(total_align, forced_alignment); + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memset(stgdict->ffi_type_pointer.elements, 0, - sizeof(ffi_type *) * (basedict->length + len + 1)); - if (basedict->length > 0) { - memcpy(stgdict->ffi_type_pointer.elements, - basedict->ffi_type_pointer.elements, - sizeof(ffi_type *) * (basedict->length)); + memset(stginfo->ffi_type_pointer.elements, 0, + sizeof(ffi_type *) * (baseinfo->length + len + 1)); + if (baseinfo->length > 0) { + memcpy(stginfo->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, + sizeof(ffi_type *) * (baseinfo->length)); } - ffi_ofs = basedict->length; + ffi_ofs = baseinfo->length; } else { offset = 0; size = 0; align = 0; union_size = 0; - total_align = 1; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + total_align = forced_alignment; + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memset(stgdict->ffi_type_pointer.elements, 0, + memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - assert(stgdict->format == NULL); + assert(stginfo->format == NULL); if (isStruct) { - stgdict->format = _ctypes_alloc_format_string(NULL, "T{"); + stginfo->format = _ctypes_alloc_format_string(NULL, "T{"); } else { /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stgdict->format = _ctypes_alloc_format_string(NULL, "B"); + stginfo->format = _ctypes_alloc_format_string(NULL, "B"); } - if (stgdict->format == NULL) + if (stginfo->format == NULL) return -1; for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); PyObject *prop; - StgDictObject *dict; - int bitsize = 0; + Py_ssize_t bitsize = 0; - if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); return -1; } - if (PyCArrayTypeObject_Check(desc)) + if (PyCArrayTypeObject_Check(st, desc)) { arrays_seen = 1; - dict = PyType_stgdict(desc); - if (dict == NULL) { + } + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + return -1; + } + if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); return -1; } - stgdict->ffi_type_pointer.elements[ffi_ofs + i] = &dict->ffi_type_pointer; - if (dict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; - stgdict->flags |= dict->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); - dict->flags |= DICTFLAG_FINAL; /* mark field type final */ + + stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; + if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; + stginfo->flags |= info->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); + info->flags |= DICTFLAG_FINAL; /* mark field type final */ if (PyTuple_Size(pair) == 3) { /* bits specified */ - stgdict->flags |= TYPEFLAG_HASBITFIELD; - switch(dict->ffi_type_pointer.type) { + stginfo->flags |= TYPEFLAG_HASBITFIELD; + switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: case FFI_TYPE_UINT32: @@ -546,11 +484,12 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: case FFI_TYPE_SINT32: - if (dict->getfunc != _ctypes_get_fielddesc("c")->getfunc - && dict->getfunc != _ctypes_get_fielddesc("u")->getfunc - ) + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) + { break; - /* else fall through */ + } + _Py_FALLTHROUGH; /* else fall through */ default: PyErr_Format(PyExc_TypeError, "bit fields not allowed for type %s", @@ -558,9 +497,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_DECREF(pair); return -1; } - if (bitsize <= 0 || bitsize > dict->size * 8) { - PyErr_SetString(PyExc_ValueError, - "number of bits invalid for bit field"); + if (bitsize <= 0 || bitsize > info->size * 8) { + PyErr_Format(PyExc_ValueError, + "number of bits invalid for bit field %R", + name); Py_DECREF(pair); return -1; } @@ -568,7 +508,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct bitsize = 0; if (isStruct) { - const char *fieldfmt = dict->format ? dict->format : "B"; + const char *fieldfmt = info->format ? info->format : "B"; const char *fieldname = PyUnicode_AsUTF8(name); char *ptr; Py_ssize_t len; @@ -584,10 +524,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - prop = PyCField_FromDesc(desc, i, + prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; @@ -598,10 +538,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct padding = ((CFieldObject *)prop)->offset - last_size; if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); return -1; @@ -619,29 +559,31 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct } sprintf(buf, "%s:%s:", fieldfmt, fieldname); - ptr = stgdict->format; - if (dict->shape != NULL) { - stgdict->format = _ctypes_alloc_format_string_with_shape( - dict->ndim, dict->shape, stgdict->format, buf); + ptr = stginfo->format; + if (info->shape != NULL) { + stginfo->format = _ctypes_alloc_format_string_with_shape( + info->ndim, info->shape, stginfo->format, buf); } else { - stgdict->format = _ctypes_alloc_format_string(stgdict->format, buf); + stginfo->format = _ctypes_alloc_format_string(stginfo->format, buf); } PyMem_Free(ptr); PyMem_Free(buf); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); return -1; } } else /* union */ { + field_size = 0; size = 0; + bitofs = 0; offset = 0; align = 0; - prop = PyCField_FromDesc(desc, i, + prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; @@ -673,53 +615,71 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct /* Pad up to the full size of the struct */ padding = aligned_size - size; if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { return -1; } } - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_string(stgdict->format, "}"); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_string(stginfo->format, "}"); PyMem_Free(ptr); - if (stgdict->format == NULL) + if (stginfo->format == NULL) return -1; } - stgdict->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, + stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stgdict->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = aligned_size; - stgdict->size = aligned_size; - stgdict->align = total_align; - stgdict->length = len; /* ADD ffi_ofs? */ + stginfo->size = aligned_size; + stginfo->align = total_align; + stginfo->length = ffi_ofs + len; -#define MAX_STRUCT_SIZE 16 +/* + * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. + */ +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) +# define MAX_STRUCT_SIZE 32 +#elif defined(__powerpc64__) +# define MAX_STRUCT_SIZE 64 +#else +# define MAX_STRUCT_SIZE 16 +#endif if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { /* - * See bpo-22273. Arrays are normally treated as pointers, which is - * fine when an array name is being passed as parameter, but not when - * passing structures by value that contain arrays. On 64-bit Linux, - * small structures passed by value are passed in registers, and in + * See bpo-22273 and gh-110190. Arrays are normally treated as + * pointers, which is fine when an array name is being passed as + * parameter, but not when passing structures by value that contain + * arrays. + * Small structures passed by value are passed in registers, and in * order to do this, libffi needs to know the true type of the array * members of structs. Treating them as pointers breaks things. * - * By small structures, we mean ones that are 16 bytes or less. In that - * case, there can't be more than 16 elements after unrolling arrays, - * as we (will) disallow bitfields. So we can collect the true ffi_type - * values in a fixed-size local array on the stack and, if any arrays - * were seen, replace the ffi_type_pointer.elements with a more - * accurate set, to allow libffi to marshal them into registers - * correctly. It means one more loop over the fields, but if we got - * here, the structure is small, so there aren't too many of those. + * Small structures have different sizes depending on the platform + * where Python is running on: + * + * * x86-64: 16 bytes or less + * * Arm platforms (both 32 and 64 bit): 32 bytes or less + * * PowerPC 64 Little Endian: 64 bytes or less + * + * In that case, there can't be more than 16, 32 or 64 elements after + * unrolling arrays, as we (will) disallow bitfields. + * So we can collect the true ffi_type values in a fixed-size local + * array on the stack and, if any arrays were seen, replace the + * ffi_type_pointer.elements with a more accurate set, to allow + * libffi to marshal them into registers correctly. + * It means one more loop over the fields, but if we got here, + * the structure is small, so there aren't too many of those. * - * Although the passing in registers is specific to 64-bit Linux, the - * array-in-struct vs. pointer problem is general. But we restrict the - * type transformation to small structs nonetheless. + * Although the passing in registers is specific to the above + * platforms, the array-in-struct vs. pointer problem is general. + * But we restrict the type transformation to small structs + * nonetheless. * * Note that although a union may be small in terms of memory usage, it * could contain many overlapping declarations of arrays, e.g. @@ -745,6 +705,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * struct { uint_32 e1; uint_32 e2; ... uint_32 e_4; } f6; * } * + * The same principle applies for a struct 32 or 64 bytes in size. + * * So the struct/union needs setting up as follows: all non-array * elements copied across as is, and all array elements replaced with * an equivalent struct which has as many fields as the array has @@ -766,7 +728,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct for (i = 0; i < len; ++i) { PyObject *name, *desc; PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; int bitsize = 0; if (pair == NULL) { @@ -778,25 +739,34 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_DECREF(pair); return -1; } - dict = PyType_stgdict(desc); - if (dict == NULL) { + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + return -1; + } + if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); return -1; } - if (!PyCArrayTypeObject_Check(desc)) { + + if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } else { /* It's an array. */ - Py_ssize_t length = dict->length; - StgDictObject *edict; + Py_ssize_t length = info->length; - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + Py_DECREF(pair); + return -1; + } + if (einfo == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", @@ -844,9 +814,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (num_ffi_types > 0) { memset(structs, 0, num_ffi_types * sizeof(ffi_type)); } - if (ffi_ofs && (basedict != NULL)) { + if (ffi_ofs && (baseinfo != NULL)) { memcpy(element_types, - basedict->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, ffi_ofs * sizeof(ffi_type *)); } element_index = ffi_ofs; @@ -855,7 +825,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct for (i = 0; i < len; ++i) { PyObject *name, *desc; PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; int bitsize = 0; if (pair == NULL) { @@ -875,9 +844,16 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct PyMem_Free(type_block); return -1; } - dict = PyType_stgdict(desc); + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + PyMem_Free(type_block); + return -1; + } + /* Possibly this check could be avoided, but see above comment. */ - if (dict == NULL) { + if (info == NULL) { Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, @@ -885,17 +861,21 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct i); return -1; } + assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(desc)) { + if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just copy over the element ffi_type. */ - element_types[element_index++] = &dict->ffi_type_pointer; + element_types[element_index++] = &info->ffi_type_pointer; } else { - Py_ssize_t length = dict->length; - StgDictObject *edict; - - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { + Py_ssize_t length = info->length; + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + Py_DECREF(pair); + PyMem_Free(type_block); + return -1; + } + if (einfo == NULL) { Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, @@ -904,15 +884,15 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } element_types[element_index++] = &structs[struct_index]; - structs[struct_index].size = length * edict->ffi_type_pointer.size; - structs[struct_index].alignment = edict->ffi_type_pointer.alignment; + structs[struct_index].size = length * einfo->ffi_type_pointer.size; + structs[struct_index].alignment = einfo->ffi_type_pointer.alignment; structs[struct_index].type = FFI_TYPE_STRUCT; structs[struct_index].elements = &dummy_types[dummy_index]; ++struct_index; /* Copy over the element's type, length times. */ while (length > 0) { assert(dummy_index < (num_ffi_type_pointers)); - dummy_types[dummy_index++] = &edict->ffi_type_pointer; + dummy_types[dummy_index++] = &einfo->ffi_type_pointer; length--; } assert(dummy_index < (num_ffi_type_pointers)); @@ -926,19 +906,19 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * Replace the old elements with the new, taking into account * base class elements where necessary. */ - assert(stgdict->ffi_type_pointer.elements); - PyMem_Free(stgdict->ffi_type_pointer.elements); - stgdict->ffi_type_pointer.elements = element_types; + assert(stginfo->ffi_type_pointer.elements); + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = element_types; } /* We did check that this flag was NOT set above, it must not have been set until now. */ - if (stgdict->flags & DICTFLAG_FINAL) { + if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); return -1; } - stgdict->flags |= DICTFLAG_FINAL; + stginfo->flags |= DICTFLAG_FINAL; return MakeAnonFields(type); } diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 292b57c083d0c8..bbbb62c9066df0 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -10,11 +10,22 @@ static const char PyCursesVersion[] = "2.1"; /* Includes */ +// clinic/_curses_panel.c.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "py_curses.h" -#include +#if defined(HAVE_NCURSESW_PANEL_H) +# include +#elif defined(HAVE_NCURSES_PANEL_H) +# include +#elif defined(HAVE_PANEL_H) +# include +#endif typedef struct { PyObject *PyCursesError; @@ -692,6 +703,7 @@ static PyModuleDef_Slot _curses_slots[] = { // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d339a8aa798361..b5854e8c33f28a 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -128,7 +128,7 @@ static const char PyCursesVersion[] = "2.2"; #include #endif -#if !defined(HAVE_NCURSES_H) && (defined(sgi) || defined(__sun) || defined(SCO5)) +#if !defined(NCURSES_VERSION) && (defined(sgi) || defined(__sun) || defined(SCO5)) #define STRICT_SYSV_CURSES /* Don't use ncurses extensions */ typedef chtype attr_t; /* No attr_t type is available */ #endif @@ -233,13 +233,20 @@ static int PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) { long value; - if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyUnicode_Check(obj)) { - if (PyUnicode_GetLength(obj) != 1) { + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -272,7 +279,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -315,7 +322,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, #ifdef HAVE_NCURSESW if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -326,7 +333,14 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, return PyCurses_ConvertToChtype(win, obj, ch); #endif } - else if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + else if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyLong_CheckExact(obj)) { @@ -340,7 +354,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -1156,8 +1170,10 @@ int py_mvwdelch(WINDOW *w, int y, int x) #endif #if defined(HAVE_CURSES_IS_PAD) +// is_pad() is defined, either as a macro or as a function #define py_is_pad(win) is_pad(win) #elif defined(WINDOW_HAS_FLAGS) +// is_pad() is not defined, but we can inspect WINDOW structure members #define py_is_pad(win) ((win) ? ((win)->_flags & _ISPAD) != 0 : FALSE) #endif @@ -3673,7 +3689,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) #endif /*[clinic input] -_curses.napms +_curses.napms -> int ms: int Duration in milliseconds. @@ -3682,13 +3698,13 @@ _curses.napms Sleep for specified time. [clinic start generated code]*/ -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms) -/*[clinic end generated code: output=a40a1da2e39ea438 input=20cd3af2b6900f56]*/ +/*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { PyCursesInitialised; - return Py_BuildValue("i", napms(ms)); + return napms(ms); } @@ -4441,7 +4457,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, wchar_t buffer[2]; if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, " + "expect int or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -4468,7 +4484,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, got %s", + "expect int or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -4586,7 +4602,14 @@ make_ncurses_version(PyTypeObject *type) if (ncurses_version == NULL) { return NULL; } - + const char *str = curses_version(); + unsigned long major = 0, minor = 0, patch = 0; + if (!str || sscanf(str, "%*[^0-9]%lu.%lu.%lu", &major, &minor, &patch) < 3) { + // Fallback to header version, which cannot be that wrong + major = NCURSES_VERSION_MAJOR; + minor = NCURSES_VERSION_MINOR; + patch = NCURSES_VERSION_PATCH; + } #define SetIntItem(flag) \ PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ if (PyErr_Occurred()) { \ @@ -4594,9 +4617,9 @@ make_ncurses_version(PyTypeObject *type) return NULL; \ } - SetIntItem(NCURSES_VERSION_MAJOR) - SetIntItem(NCURSES_VERSION_MINOR) - SetIntItem(NCURSES_VERSION_PATCH) + SetIntItem(major) + SetIntItem(minor) + SetIntItem(patch) #undef SetIntItem return ncurses_version; @@ -4743,6 +4766,9 @@ PyInit__curses(void) m = PyModule_Create(&_cursesmodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 684a628806fbb5..f20efd3d36d275 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,8 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_time.h" // _PyTime_ObjectToTime_t() + #include "datetime.h" @@ -23,24 +25,20 @@ # include /* struct timeval */ #endif -#define PyDate_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateType) -#define PyDate_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateType) - -#define PyDateTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateTimeType) -#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateTimeType) - -#define PyTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeType) -#define PyTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TimeType) - -#define PyDelta_Check(op) PyObject_TypeCheck(op, &PyDateTime_DeltaType) -#define PyDelta_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DeltaType) -#define PyTZInfo_Check(op) PyObject_TypeCheck(op, &PyDateTime_TZInfoType) -#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TZInfoType) +/* forward declarations */ +static PyTypeObject PyDateTime_DateType; +static PyTypeObject PyDateTime_DateTimeType; +static PyTypeObject PyDateTime_TimeType; +static PyTypeObject PyDateTime_DeltaType; +static PyTypeObject PyDateTime_TZInfoType; +static PyTypeObject PyDateTime_TimeZoneType; -#define PyTimezone_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeZoneType) typedef struct { + /* Module heap types. */ + PyTypeObject *isocalendar_date_type; + /* Conversion factors. */ PyObject *us_per_ms; // 1_000 PyObject *us_per_second; // 1_000_000 @@ -50,26 +48,192 @@ typedef struct { PyObject *us_per_week; // 1e6 * 3600 * 24 * 7 as Python int PyObject *seconds_per_day; // 3600 * 24 as Python int - /* The interned UTC timezone instance */ - PyObject *utc; - /* The interned Unix epoch datetime instance */ PyObject *epoch; } datetime_state; -static datetime_state _datetime_global_state; +/* The module has a fixed number of static objects, due to being exposed + * through the datetime C-API. There are five types exposed directly, + * one type exposed indirectly, and one singleton constant (UTC). + * + * Each of these objects is hidden behind a macro in the same way as + * the per-module objects stored in module state. The macros for the + * static objects don't need to be passed a state, but the consistency + * of doing so is more clear. We use a dedicated noop macro, NO_STATE, + * to make the special case obvious. */ + +#define NO_STATE NULL -#define STATIC_STATE() (&_datetime_global_state) +#define DATE_TYPE(st) &PyDateTime_DateType +#define DATETIME_TYPE(st) &PyDateTime_DateTimeType +#define TIME_TYPE(st) &PyDateTime_TimeType +#define DELTA_TYPE(st) &PyDateTime_DeltaType +#define TZINFO_TYPE(st) &PyDateTime_TZInfoType +#define TIMEZONE_TYPE(st) &PyDateTime_TimeZoneType +#define ISOCALENDAR_DATE_TYPE(st) st->isocalendar_date_type -/*[clinic input] -module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81bec0fa19837f63]*/ +#define PyDate_Check(op) PyObject_TypeCheck(op, DATE_TYPE(NO_STATE)) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, DATE_TYPE(NO_STATE)) + +#define PyDateTime_Check(op) PyObject_TypeCheck(op, DATETIME_TYPE(NO_STATE)) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, DATETIME_TYPE(NO_STATE)) + +#define PyTime_Check(op) PyObject_TypeCheck(op, TIME_TYPE(NO_STATE)) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, TIME_TYPE(NO_STATE)) + +#define PyDelta_Check(op) PyObject_TypeCheck(op, DELTA_TYPE(NO_STATE)) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, DELTA_TYPE(NO_STATE)) + +#define PyTZInfo_Check(op) PyObject_TypeCheck(op, TZINFO_TYPE(NO_STATE)) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, TZINFO_TYPE(NO_STATE)) + +#define PyTimezone_Check(op) PyObject_TypeCheck(op, TIMEZONE_TYPE(NO_STATE)) + +#define CONST_US_PER_MS(st) st->us_per_ms +#define CONST_US_PER_SECOND(st) st->us_per_second +#define CONST_US_PER_MINUTE(st) st->us_per_minute +#define CONST_US_PER_HOUR(st) st->us_per_hour +#define CONST_US_PER_DAY(st) st->us_per_day +#define CONST_US_PER_WEEK(st) st->us_per_week +#define CONST_SEC_PER_DAY(st) st->seconds_per_day +#define CONST_EPOCH(st) st->epoch +#define CONST_UTC(st) ((PyObject *)&utc_timezone) + +static datetime_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (datetime_state *)state; +} + + +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) + +static PyObject * +get_current_module(PyInterpreterState *interp, int *p_reloading) +{ + PyObject *mod = NULL; + int reloading = 0; + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + reloading = 1; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); + } + } + if (p_reloading != NULL) { + *p_reloading = reloading; + } + return mod; + +error: + assert(PyErr_Occurred()); + return NULL; +} + +static PyModuleDef datetimemodule; + +static datetime_state * +_get_current_state(PyObject **p_mod) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *mod = get_current_module(interp, NULL); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + datetime_state *st = get_module_state(mod); + *p_mod = mod; + return st; +} + +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) + +static int +set_current_module(PyInterpreterState *interp, PyObject *mod) +{ + assert(mod != NULL); + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { + return -1; + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; +} + +static void +clear_current_module(PyInterpreterState *interp, PyObject *expected) +{ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + + if (expected != NULL) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + PyObject *current = NULL; + int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } + } + } + + /* We use None to identify that the module was previously loaded. */ + if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { + goto error; + } + + goto finally; + +error: + PyErr_WriteUnraisable(NULL); + +finally: + PyErr_SetRaisedException(exc); +} -#include "clinic/_datetimemodule.c.h" /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python @@ -150,17 +314,19 @@ class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCa */ #define MONTH_IS_SANE(M) ((unsigned int)(M) - 1 < 12) -/* Forward declarations. */ -static PyTypeObject PyDateTime_DateType; -static PyTypeObject PyDateTime_DateTimeType; -static PyTypeObject PyDateTime_DeltaType; -static PyTypeObject PyDateTime_IsoCalendarDateType; -static PyTypeObject PyDateTime_TimeType; -static PyTypeObject PyDateTime_TZInfoType; -static PyTypeObject PyDateTime_TimeZoneType; - static int check_tzinfo_subclass(PyObject *p); +/*[clinic input] +module datetime +class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_type" +class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" +class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ + +#include "clinic/_datetimemodule.c.h" + /* --------------------------------------------------------------------------- * Math utilities. @@ -413,6 +579,10 @@ iso_week1_monday(int year) static int iso_to_ymd(const int iso_year, const int iso_week, const int iso_day, int *year, int *month, int *day) { + // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) + if (iso_year < MINYEAR || iso_year > MAXYEAR) { + return -4; + } if (iso_week <= 0 || iso_week >= 53) { int out_of_range = 1; if (iso_week == 53) { @@ -759,7 +929,7 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, * -2: Inconsistent date separator usage * -3: Failed to parse ISO week. * -4: Failed to parse ISO day. - * -5, -6: Failure in iso_to_ymd + * -5, -6, -7: Failure in iso_to_ymd */ const char *p = dtstr; p = parse_digits(p, year, 4); @@ -850,6 +1020,9 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, continue; } else if (c == '.' || c == ',') { + if (i < 2) { + return -3; // Decimal mark on hour or minute + } break; } else if (!has_separator) { --p; @@ -972,7 +1145,7 @@ new_date_ex(int year, int month, int day, PyTypeObject *type) } #define new_date(year, month, day) \ - new_date_ex(year, month, day, &PyDateTime_DateType) + new_date_ex(year, month, day, DATE_TYPE(NO_STATE)) // Forward declaration static PyObject * @@ -984,10 +1157,10 @@ new_date_subclass_ex(int year, int month, int day, PyObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == &PyDateTime_DateType) { + if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { result = new_date_ex(year, month, day, (PyTypeObject *)cls); } - else if ((PyTypeObject *)cls == &PyDateTime_DateTimeType) { + else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, (PyTypeObject *)cls); } @@ -1041,29 +1214,57 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, } #define new_datetime(y, m, d, hh, mm, ss, us, tzinfo, fold) \ - new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ - &PyDateTime_DateTimeType) + new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) + +static PyObject * +call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +{ + PyObject *kwargs = NULL, *res = NULL; + va_list va; + + va_start(va, format); + PyObject *args = Py_VaBuildValue(format, va); + va_end(va); + if (args == NULL) { + return NULL; + } + if (fold) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto Done; + } + PyObject *obj = PyLong_FromLong(fold); + if (obj == NULL) { + goto Done; + } + int err = PyDict_SetItemString(kwargs, "fold", obj); + Py_DECREF(obj); + if (err < 0) { + goto Done; + } + } + res = PyObject_Call(cls, args, kwargs); +Done: + Py_DECREF(args); + Py_XDECREF(kwargs); + return res; +} static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) { + int fold, PyObject *cls) +{ PyObject* dt; - if ((PyTypeObject*)cls == &PyDateTime_DateTimeType) { + if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); - } else { + } + else { // Subclass - dt = PyObject_CallFunction(cls, "iiiiiiiO", - year, - month, - day, - hour, - minute, - second, - usecond, - tzinfo); + dt = call_subclass_fold(cls, fold, "iiiiiiiO", year, month, day, + hour, minute, second, usecond, tzinfo); } return dt; @@ -1116,8 +1317,28 @@ new_time_ex(int hour, int minute, int second, int usecond, return new_time_ex2(hour, minute, second, usecond, tzinfo, 0, type); } -#define new_time(hh, mm, ss, us, tzinfo, fold) \ - new_time_ex2(hh, mm, ss, us, tzinfo, fold, &PyDateTime_TimeType) +#define new_time(hh, mm, ss, us, tzinfo, fold) \ + new_time_ex2(hh, mm, ss, us, tzinfo, fold, TIME_TYPE(NO_STATE)) + +static PyObject * +new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, + PyObject *tzinfo, int fold, PyObject *cls) +{ + PyObject *t; + if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { + // Use the fast path constructor + t = new_time(hour, minute, second, usecond, tzinfo, fold); + } + else { + // Subclass + t = call_subclass_fold(cls, fold, "iiiiO", hour, minute, second, + usecond, tzinfo); + } + + return t; +} + +static PyDateTime_Delta * look_up_delta(int, int, int, PyTypeObject *); /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure @@ -1139,6 +1360,12 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, if (check_delta_day_range(days) < 0) return NULL; + self = look_up_delta(days, seconds, microseconds, type); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_Delta *) (type->tp_alloc(type, 0)); if (self != NULL) { self->hashcode = -1; @@ -1150,7 +1377,7 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, } #define new_delta(d, s, us, normalize) \ - new_delta_ex(d, s, us, normalize, &PyDateTime_DeltaType) + new_delta_ex(d, s, us, normalize, DELTA_TYPE(NO_STATE)) typedef struct @@ -1160,6 +1387,8 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; +static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name); + /* Create new timezone instance checking offset range. This function does not check the name argument. Caller must assure that offset is a timedelta instance and name is either NULL @@ -1168,12 +1397,18 @@ static PyObject * create_timezone(PyObject *offset, PyObject *name) { PyDateTime_TimeZone *self; - PyTypeObject *type = &PyDateTime_TimeZoneType; + PyTypeObject *type = TIMEZONE_TYPE(NO_STATE); assert(offset != NULL); assert(PyDelta_Check(offset)); assert(name == NULL || PyUnicode_Check(name)); + self = look_up_timezone(offset, name); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_TimeZone *)(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; @@ -1184,6 +1419,7 @@ create_timezone(PyObject *offset, PyObject *name) } static int delta_bool(PyDateTime_Delta *self); +static PyDateTime_TimeZone utc_timezone; static PyObject * new_timezone(PyObject *offset, PyObject *name) @@ -1193,8 +1429,7 @@ new_timezone(PyObject *offset, PyObject *name) assert(name == NULL || PyUnicode_Check(name)); if (name == NULL && delta_bool((PyDateTime_Delta *)offset) == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } if ((GET_TD_DAYS(offset) == -1 && GET_TD_SECONDS(offset) == 0 && @@ -1407,8 +1642,7 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) if (rv == 1) { // Create a timezone from offset in seconds (0 returns UTC) if (tzoffset == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } PyObject *delta = new_delta(0, tzoffset, tz_useconds, 1); @@ -1617,6 +1851,11 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, const char *ptoappend; /* ptr to string to append to output buffer */ Py_ssize_t ntoappend; /* # of bytes to append to output buffer */ +#ifdef Py_NORMALIZE_CENTURY + /* Buffer of maximum size of formatted year permitted by long. */ + char buf[SIZEOF_LONG*5/2+2]; +#endif + assert(object && format && timetuple); assert(PyUnicode_Check(format)); /* Convert the input format to a C string and size */ @@ -1624,6 +1863,11 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, if (!pin) return NULL; + PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); + if (strftime == NULL) { + goto Done; + } + /* Scan the input format, looking for %z/%Z/%f escapes, building * a new format. Since computing the replacements for those codes * is expensive, don't unless they're actually used. @@ -1705,8 +1949,47 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, ptoappend = PyBytes_AS_STRING(freplacement); ntoappend = PyBytes_GET_SIZE(freplacement); } +#ifdef Py_NORMALIZE_CENTURY + else if (ch == 'Y' || ch == 'G') { + /* 0-pad year with century as necessary */ + PyObject *item = PyTuple_GET_ITEM(timetuple, 0); + long year_long = PyLong_AsLong(item); + + if (year_long == -1 && PyErr_Occurred()) { + goto Done; + } + /* Note that datetime(1000, 1, 1).strftime('%G') == '1000' so year + 1000 for %G can go on the fast path. */ + if (year_long >= 1000) { + goto PassThrough; + } + if (ch == 'G') { + PyObject *year_str = PyObject_CallFunction(strftime, "sO", + "%G", timetuple); + if (year_str == NULL) { + goto Done; + } + PyObject *year = PyNumber_Long(year_str); + Py_DECREF(year_str); + if (year == NULL) { + goto Done; + } + year_long = PyLong_AsLong(year); + Py_DECREF(year); + if (year_long == -1 && PyErr_Occurred()) { + goto Done; + } + } + + ntoappend = PyOS_snprintf(buf, sizeof(buf), "%04ld", year_long); + ptoappend = buf; + } +#endif else { /* percent followed by something else */ +#ifdef Py_NORMALIZE_CENTURY + PassThrough: +#endif ptoappend = pin - 2; ntoappend = 2; } @@ -1738,17 +2021,13 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, goto Done; { PyObject *format; - PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); - if (strftime == NULL) - goto Done; format = PyUnicode_FromString(PyBytes_AS_STRING(newfmt)); if (format != NULL) { result = PyObject_CallFunctionObjArgs(strftime, format, timetuple, NULL); Py_DECREF(format); } - Py_DECREF(strftime); } Done: Py_XDECREF(freplacement); @@ -1756,6 +2035,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, Py_XDECREF(colonzreplacement); Py_XDECREF(Zreplacement); Py_XDECREF(newfmt); + Py_XDECREF(strftime); return result; } @@ -1815,16 +2095,6 @@ diff_to_bool(int diff, int op) Py_RETURN_RICHCOMPARE(diff, 0, op); } -/* Raises a "can't compare" TypeError and returns NULL. */ -static PyObject * -cmperror(PyObject *a, PyObject *b) -{ - PyErr_Format(PyExc_TypeError, - "can't compare %s to %s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); - return NULL; -} - /* --------------------------------------------------------------------------- * Class implementations. */ @@ -1847,11 +2117,13 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) goto Done; - datetime_state *st = STATIC_STATE(); - x2 = PyNumber_Multiply(x1, st->seconds_per_day); /* days in seconds */ + x2 = PyNumber_Multiply(x1, CONST_SEC_PER_DAY(st)); /* days in seconds */ if (x2 == NULL) goto Done; Py_SETREF(x1, NULL); @@ -1868,7 +2140,7 @@ delta_to_microseconds(PyDateTime_Delta *self) /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ - x1 = PyNumber_Multiply(x3, st->us_per_second); /* us */ + x1 = PyNumber_Multiply(x3, CONST_US_PER_SECOND(st)); /* us */ if (x1 == NULL) goto Done; Py_SETREF(x3, NULL); @@ -1884,6 +2156,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -1923,8 +2196,10 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = STATIC_STATE(); - tuple = checked_divmod(pyus, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { goto Done; } @@ -1942,7 +2217,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) num = Py_NewRef(PyTuple_GET_ITEM(tuple, 0)); /* leftover seconds */ Py_DECREF(tuple); - tuple = checked_divmod(num, st->seconds_per_day); + tuple = checked_divmod(num, CONST_SEC_PER_DAY(st)); if (tuple == NULL) goto Done; Py_DECREF(num); @@ -1967,6 +2242,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -1976,7 +2252,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } #define microseconds_to_delta(pymicros) \ - microseconds_to_delta_ex(pymicros, &PyDateTime_DeltaType) + microseconds_to_delta_ex(pymicros, DELTA_TYPE(NO_STATE)) static PyObject * multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) @@ -2504,6 +2780,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + /* Argument objects. */ PyObject *day = NULL; PyObject *second = NULL; @@ -2542,29 +2821,28 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); CLEANUP; } - datetime_state *st = STATIC_STATE(); if (ms) { - y = accum("milliseconds", x, ms, st->us_per_ms, &leftover_us); + y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } if (second) { - y = accum("seconds", x, second, st->us_per_second, &leftover_us); + y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } if (minute) { - y = accum("minutes", x, minute, st->us_per_minute, &leftover_us); + y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } if (hour) { - y = accum("hours", x, hour, st->us_per_hour, &leftover_us); + y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } if (day) { - y = accum("days", x, day, st->us_per_day, &leftover_us); + y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } if (week) { - y = accum("weeks", x, week, st->us_per_week, &leftover_us); + y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -2606,7 +2884,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) self = microseconds_to_delta_ex(x, type); Py_DECREF(x); + Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2719,9 +2999,12 @@ delta_total_seconds(PyObject *self, PyObject *Py_UNUSED(ignored)) if (total_microseconds == NULL) return NULL; - datetime_state *st = STATIC_STATE(); - total_seconds = PyNumber_TrueDivide(total_microseconds, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -2842,6 +3125,25 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_free */ }; +// XXX Can we make this const? +static PyDateTime_Delta zero_delta = { + PyObject_HEAD_INIT(&PyDateTime_DeltaType) + /* Letting this be set lazily is a benign race. */ + .hashcode = -1, +}; + +static PyDateTime_Delta * +look_up_delta(int days, int seconds, int microseconds, PyTypeObject *type) +{ + if (days == 0 && seconds == 0 && microseconds == 0 + && type == Py_TYPE(&zero_delta)) + { + return &zero_delta; + } + return NULL; +} + + /* * PyDateTime_Date implementation. */ @@ -3103,15 +3405,13 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) - if (year < MINYEAR || year > MAXYEAR) { - PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); - return NULL; - } - int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); + if (rv == -4) { + PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); + return NULL; + } if (rv == -2) { PyErr_Format(PyExc_ValueError, "Invalid week: %d", week); @@ -3374,17 +3674,40 @@ static PyMethodDef iso_calendar_date_methods[] = { {NULL, NULL}, }; -static PyTypeObject PyDateTime_IsoCalendarDateType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "datetime.IsoCalendarDate", - .tp_basicsize = sizeof(PyDateTime_IsoCalendarDate), - .tp_repr = (reprfunc) iso_calendar_date_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = iso_calendar_date__doc__, - .tp_methods = iso_calendar_date_methods, - .tp_getset = iso_calendar_date_getset, - // .tp_base = &PyTuple_Type, // filled in PyInit__datetime - .tp_new = iso_calendar_date_new, +static int +iso_calendar_date_traverse(PyDateTime_IsoCalendarDate *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return PyTuple_Type.tp_traverse((PyObject *)self, visit, arg); +} + +static void +iso_calendar_date_dealloc(PyDateTime_IsoCalendarDate *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyTuple_Type.tp_dealloc((PyObject *)self); // delegate GC-untrack as well + Py_DECREF(tp); +} + +static PyType_Slot isocal_slots[] = { + {Py_tp_repr, iso_calendar_date_repr}, + {Py_tp_doc, (void *)iso_calendar_date__doc__}, + {Py_tp_methods, iso_calendar_date_methods}, + {Py_tp_getset, iso_calendar_date_getset}, + {Py_tp_new, iso_calendar_date_new}, + {Py_tp_dealloc, iso_calendar_date_dealloc}, + {Py_tp_traverse, iso_calendar_date_traverse}, + {0, NULL}, +}; + +static PyType_Spec isocal_spec = { + .name = "datetime.IsoCalendarDate", + .basicsize = sizeof(PyDateTime_IsoCalendarDate), + .flags = (Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = isocal_slots, }; /*[clinic input] @@ -3434,8 +3757,12 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) week = 0; } - PyObject* v = iso_calendar_date_new_impl(&PyDateTime_IsoCalendarDateType, - year, week + 1, day + 1); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), + year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -3447,7 +3774,15 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) static PyObject * date_richcompare(PyObject *self, PyObject *other, int op) { - if (PyDate_Check(other)) { + /* Since DateTime is a subclass of Date, if the other object is + * a DateTime, it would compute an equality testing or an ordering + * based on the date part alone, and we don't want that. + * So return NotImplemented here in that case. + * If a subclass wants to change this, it's up to the subclass to do so. + * The behavior is the same as if Date and DateTime were independent + * classes. + */ + if (PyDate_Check(other) && !PyDateTime_Check(other)) { int diff = memcmp(((PyDateTime_Date *)self)->data, ((PyDateTime_Date *)other)->data, _PyDateTime_DATE_DATASIZE); @@ -3466,24 +3801,22 @@ date_timetuple(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) 0, 0, 0, -1); } +/*[clinic input] +datetime.date.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + +Return date with new specified fields. +[clinic start generated code]*/ + static PyObject * -date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw) +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day) +/*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - PyObject *clone; - PyObject *tuple; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iii:replace", date_kws, - &year, &month, &day)) - return NULL; - tuple = Py_BuildValue("iii", year, month, day); - if (tuple == NULL) - return NULL; - clone = date_new(Py_TYPE(self), tuple, NULL); - Py_DECREF(tuple); - return clone; + return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); } static Py_hash_t @@ -3596,10 +3929,10 @@ static PyMethodDef date_methods[] = { PyDoc_STR("Return the day of the week represented by the date.\n" "Monday == 0 ... Sunday == 6")}, - {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return date with new specified fields.")}, + DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -3899,7 +4232,7 @@ timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *offset; PyObject *name = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - &PyDateTime_DeltaType, &offset, &name)) + DELTA_TYPE(NO_STATE), &offset, &name)) return new_timezone(offset, name); return NULL; @@ -3952,8 +4285,7 @@ timezone_repr(PyDateTime_TimeZone *self) to use Py_TYPE(self)->tp_name here. */ const char *type_name = Py_TYPE(self)->tp_name; - datetime_state *st = STATIC_STATE(); - if (((PyObject *)self) == st->utc) { + if ((PyObject *)self == CONST_UTC(NO_STATE)) { return PyUnicode_FromFormat("%s.utc", type_name); } @@ -3975,8 +4307,7 @@ timezone_str(PyDateTime_TimeZone *self) if (self->name != NULL) { return Py_NewRef(self->name); } - datetime_state *st = STATIC_STATE(); - if ((PyObject *)self == st->utc || + if ((PyObject *)self == CONST_UTC(NO_STATE) || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) @@ -4060,8 +4391,8 @@ static PyObject * timezone_getinitargs(PyDateTime_TimeZone *self, PyObject *Py_UNUSED(ignored)) { if (self->name == NULL) - return Py_BuildValue("(O)", self->offset); - return Py_BuildValue("(OO)", self->offset, self->name); + return PyTuple_Pack(1, self->offset); + return PyTuple_Pack(2, self->offset, self->name); } static PyMethodDef timezone_methods[] = { @@ -4128,6 +4459,23 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_new, /* tp_new */ }; +// XXX Can we make this const? +static PyDateTime_TimeZone utc_timezone = { + PyObject_HEAD_INIT(&PyDateTime_TimeZoneType) + .offset = (PyObject *)&zero_delta, + .name = NULL, +}; + +static PyDateTime_TimeZone * +look_up_timezone(PyObject *offset, PyObject *name) +{ + if (offset == utc_timezone.offset && name == NULL) { + return (PyDateTime_TimeZone *)CONST_UTC(NO_STATE); + } + return NULL; +} + + /* * PyDateTime_Time implementation. */ @@ -4573,36 +4921,28 @@ time_hash(PyDateTime_Time *self) return self->hashcode; } +/*[clinic input] +datetime.time.replace + + hour: int(c_default="TIME_GET_HOUR(self)") = unchanged + minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged + second: int(c_default="TIME_GET_SECOND(self)") = unchanged + microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="TIME_GET_FOLD(self)") = unchanged + +Return time with new specified fields. +[clinic start generated code]*/ + static PyObject * -time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ { - PyObject *clone; - PyObject *tuple; - int hh = TIME_GET_HOUR(self); - int mm = TIME_GET_MINUTE(self); - int ss = TIME_GET_SECOND(self); - int us = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i:replace", - time_kws, - &hh, &mm, &ss, &us, &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = time_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - TIME_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, + fold, (PyObject *)Py_TYPE(self)); } static PyObject * @@ -4630,7 +4970,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } int hour = 0, minute = 0, second = 0, microsecond = 0; - int tzoffset, tzimicrosecond = 0; + int tzoffset = 0, tzimicrosecond = 0; int rv = parse_isoformat_time(p, len, &hour, &minute, &second, µsecond, &tzoffset, &tzimicrosecond); @@ -4647,7 +4987,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == &PyDateTime_TimeType ) { + if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { t = PyObject_CallFunction(cls, "iiiiO", @@ -4732,10 +5072,10 @@ static PyMethodDef time_methods[] = { {"dst", (PyCFunction)time_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return time with new specified fields.")}, + DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -5098,7 +5438,11 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_t ts = _PyTime_GetSystemClock(); + PyTime_t ts; + if (PyTime_Time(&ts) < 0) { + return NULL; + } + time_t secs; int us; @@ -5216,19 +5560,19 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { - static PyObject *module = NULL; - PyObject *string, *format; + PyObject *string, *format, *result; if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { - module = PyImport_ImportModule("_strptime"); - if (module == NULL) - return NULL; + return NULL; } - return PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), - cls, string, format, NULL); + result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), + cls, string, format, NULL); + Py_DECREF(module); + return result; } /* Return new datetime from date/datetime and time arguments. */ @@ -5242,8 +5586,8 @@ datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) PyObject *result = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - &PyDateTime_DateType, &date, - &PyDateTime_TimeType, &time, &tzinfo)) { + DATE_TYPE(NO_STATE), &date, + TIME_TYPE(NO_STATE), &time, &tzinfo)) { if (tzinfo == NULL) { if (HASTZINFO(time)) tzinfo = ((PyDateTime_Time *)time)->tzinfo; @@ -5891,21 +6235,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyObject *offset1, *offset2; int diff; - if (! PyDateTime_Check(other)) { - if (PyDate_Check(other)) { - /* Prevent invocation of date_richcompare. We want to - return NotImplemented here to give the other object - a chance. But since DateTime is a subclass of - Date, if the other object is a Date, it would - compute an ordering based on the date part alone, - and we don't want that. So force unequal or - uncomparable here in that case. */ - if (op == Py_EQ) - Py_RETURN_FALSE; - if (op == Py_NE) - Py_RETURN_TRUE; - return cmperror(self, other); - } + if (!PyDateTime_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } @@ -6042,40 +6372,33 @@ datetime_hash(PyDateTime_DateTime *self) return self->hashcode; } +/*[clinic input] +datetime.datetime.replace + + year: int(c_default="GET_YEAR(self)") = unchanged + month: int(c_default="GET_MONTH(self)") = unchanged + day: int(c_default="GET_DAY(self)") = unchanged + hour: int(c_default="DATE_GET_HOUR(self)") = unchanged + minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged + second: int(c_default="DATE_GET_SECOND(self)") = unchanged + microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + * + fold: int(c_default="DATE_GET_FOLD(self)") = unchanged + +Return datetime with new specified fields. +[clinic start generated code]*/ + static PyObject * -datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold) +/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ { - PyObject *clone; - PyObject *tuple; - int y = GET_YEAR(self); - int m = GET_MONTH(self); - int d = GET_DAY(self); - int hh = DATE_GET_HOUR(self); - int mm = DATE_GET_MINUTE(self); - int ss = DATE_GET_SECOND(self); - int us = DATE_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; - int fold = DATE_GET_FOLD(self); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiiiiO$i:replace", - datetime_kws, - &y, &m, &d, &hh, &mm, &ss, &us, - &tzinfo, &fold)) - return NULL; - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } - tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); - if (tuple == NULL) - return NULL; - clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - DATE_SET_FOLD(clone, fold); - } - Py_DECREF(tuple); - return clone; + return new_datetime_subclass_fold_ex(year, month, day, hour, minute, + second, microsecond, tzinfo, fold, + (PyObject *)Py_TYPE(self)); } static PyObject * @@ -6148,10 +6471,14 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = STATIC_STATE(); - delta = datetime_subtract((PyObject *)utc_time, st->epoch); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; + one_second = new_delta(0, 1, 0, 0); if (one_second == NULL) { Py_DECREF(delta); @@ -6261,7 +6588,6 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) if (result == NULL) return NULL; - datetime_state *st = STATIC_STATE(); /* Make sure result is aware and UTC. */ if (!HASTZINFO(result)) { temp = (PyObject *)result; @@ -6273,7 +6599,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) DATE_GET_MINUTE(result), DATE_GET_SECOND(result), DATE_GET_MICROSECOND(result), - st->utc, + CONST_UTC(NO_STATE), DATE_GET_FOLD(result), Py_TYPE(result)); Py_DECREF(temp); @@ -6282,7 +6608,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) } else { /* Result is already aware - just replace tzinfo. */ - Py_SETREF(result->tzinfo, Py_NewRef(st->utc)); + Py_SETREF(result->tzinfo, Py_NewRef(CONST_UTC(NO_STATE))); } /* Attach new tzinfo and let fromutc() do the rest. */ @@ -6386,9 +6712,12 @@ datetime_timestamp(PyDateTime_DateTime *self, PyObject *Py_UNUSED(ignored)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = STATIC_STATE(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + PyObject *delta; - delta = datetime_subtract((PyObject *)self, st->epoch); + delta = datetime_subtract((PyObject *)self, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -6597,10 +6926,10 @@ static PyMethodDef datetime_methods[] = { {"dst", (PyCFunction)datetime_dst, METH_NOARGS, PyDoc_STR("Return self.tzinfo.dst(self).")}, - {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return datetime with new specified fields.")}, + DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, @@ -6677,11 +7006,45 @@ static PyTypeObject PyDateTime_DateTimeType = { }; /* --------------------------------------------------------------------------- - * Module methods and initialization. + * datetime C-API. */ -static PyMethodDef module_methods[] = { - {NULL, NULL} +static PyTypeObject * const capi_types[] = { + &PyDateTime_DateType, + &PyDateTime_DateTimeType, + &PyDateTime_TimeType, + &PyDateTime_DeltaType, + &PyDateTime_TZInfoType, + /* Indirectly, via the utc object. */ + &PyDateTime_TimeZoneType, +}; + +/* The C-API is process-global. This violates interpreter isolation + * due to the objects stored here. Thus each of those objects must + * be managed carefully. */ +// XXX Can we make this const? +static PyDateTime_CAPI capi = { + /* The classes must be readied before used here. + * That will happen the first time the module is loaded. + * They aren't safe to be shared between interpreters, + * but that's okay as long as the module is single-phase init. */ + .DateType = &PyDateTime_DateType, + .DateTimeType = &PyDateTime_DateTimeType, + .TimeType = &PyDateTime_TimeType, + .DeltaType = &PyDateTime_DeltaType, + .TZInfoType = &PyDateTime_TZInfoType, + + .TimeZone_UTC = (PyObject *)&utc_timezone, + + .Date_FromDate = new_date_ex, + .DateTime_FromDateAndTime = new_datetime_ex, + .Time_FromTime = new_time_ex, + .Delta_FromDelta = new_delta_ex, + .TimeZone_FromTimeZone = new_timezone, + .DateTime_FromTimestamp = datetime_fromtimestamp, + .Date_FromTimestamp = datetime_date_fromtimestamp_capi, + .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, + .Time_FromTimeAndFold = new_time_ex2, }; /* Get a new C API by calling this function. @@ -6690,55 +7053,7 @@ static PyMethodDef module_methods[] = { static inline PyDateTime_CAPI * get_datetime_capi(void) { - PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->DateType = &PyDateTime_DateType; - capi->DateTimeType = &PyDateTime_DateTimeType; - capi->TimeType = &PyDateTime_TimeType; - capi->DeltaType = &PyDateTime_DeltaType; - capi->TZInfoType = &PyDateTime_TZInfoType; - capi->Date_FromDate = new_date_ex; - capi->DateTime_FromDateAndTime = new_datetime_ex; - capi->Time_FromTime = new_time_ex; - capi->Delta_FromDelta = new_delta_ex; - capi->TimeZone_FromTimeZone = new_timezone; - capi->DateTime_FromTimestamp = datetime_fromtimestamp; - capi->Date_FromTimestamp = datetime_date_fromtimestamp_capi; - capi->DateTime_FromDateAndTimeAndFold = new_datetime_ex2; - capi->Time_FromTimeAndFold = new_time_ex2; - // Make sure this function is called after utc has - // been initialized. - datetime_state *st = STATIC_STATE(); - assert(st->utc != NULL); - capi->TimeZone_UTC = st->utc; // borrowed ref - return capi; -} - -static void -datetime_destructor(PyObject *op) -{ - void *ptr = PyCapsule_GetPointer(op, PyDateTime_CAPSULE_NAME); - PyMem_Free(ptr); -} - -static int -datetime_clear(PyObject *module) -{ - datetime_state *st = STATIC_STATE(); - - Py_CLEAR(st->us_per_ms); - Py_CLEAR(st->us_per_second); - Py_CLEAR(st->us_per_minute); - Py_CLEAR(st->us_per_hour); - Py_CLEAR(st->us_per_day); - Py_CLEAR(st->us_per_week); - Py_CLEAR(st->seconds_per_day); - Py_CLEAR(st->utc); - Py_CLEAR(st->epoch); - return 0; + return &capi; } static PyObject * @@ -6753,9 +7068,45 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) return tz; } + +/* --------------------------------------------------------------------------- + * Module state lifecycle. + */ + static int -init_state(datetime_state *st) -{ +init_state(datetime_state *st, PyObject *module, PyObject *old_module) +{ + /* Each module gets its own heap types. */ +#define ADD_TYPE(FIELD, SPEC, BASE) \ + do { \ + PyObject *cls = PyType_FromModuleAndSpec( \ + module, SPEC, (PyObject *)BASE); \ + if (cls == NULL) { \ + return -1; \ + } \ + st->FIELD = (PyTypeObject *)cls; \ + } while (0) + + ADD_TYPE(isocalendar_date_type, &isocal_spec, &PyTuple_Type); +#undef ADD_TYPE + + if (old_module != NULL) { + assert(old_module != module); + datetime_state *st_old = get_module_state(old_module); + *st = (datetime_state){ + .isocalendar_date_type = st->isocalendar_date_type, + .us_per_ms = Py_NewRef(st_old->us_per_ms), + .us_per_second = Py_NewRef(st_old->us_per_second), + .us_per_minute = Py_NewRef(st_old->us_per_minute), + .us_per_hour = Py_NewRef(st_old->us_per_hour), + .us_per_day = Py_NewRef(st_old->us_per_day), + .us_per_week = Py_NewRef(st_old->us_per_week), + .seconds_per_day = Py_NewRef(st_old->seconds_per_day), + .epoch = Py_NewRef(st_old->epoch), + }; + return 0; + } + st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6789,51 +7140,111 @@ init_state(datetime_state *st) return -1; } - /* Init UTC timezone */ - st->utc = create_timezone_from_delta(0, 0, 0, 0); - if (st->utc == NULL) { - return -1; - } - /* Init Unix epoch */ - st->epoch = new_datetime(1970, 1, 1, 0, 0, 0, 0, st->utc, 0); + st->epoch = new_datetime( + 1970, 1, 1, 0, 0, 0, 0, (PyObject *)&utc_timezone, 0); if (st->epoch == NULL) { return -1; } + return 0; } static int -_datetime_exec(PyObject *module) +traverse_state(datetime_state *st, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(st->isocalendar_date_type); + + return 0; +} + +static int +clear_state(datetime_state *st) +{ + Py_CLEAR(st->isocalendar_date_type); + Py_CLEAR(st->us_per_ms); + Py_CLEAR(st->us_per_second); + Py_CLEAR(st->us_per_minute); + Py_CLEAR(st->us_per_hour); + Py_CLEAR(st->us_per_day); + Py_CLEAR(st->us_per_week); + Py_CLEAR(st->seconds_per_day); + Py_CLEAR(st->epoch); + return 0; +} + + +static int +init_static_types(PyInterpreterState *interp, int reloading) { + if (reloading) { + return 0; + } + // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. // See https://bugs.python.org/issue40777 - PyDateTime_IsoCalendarDateType.tp_base = &PyTuple_Type; PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - PyTypeObject *types[] = { - &PyDateTime_DateType, - &PyDateTime_DateTimeType, - &PyDateTime_TimeType, - &PyDateTime_DeltaType, - &PyDateTime_TZInfoType, - &PyDateTime_TimeZoneType, - }; + /* Bases classes must be initialized before subclasses, + * so capi_types must have the types in the appropriate order. */ + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + if (_PyStaticType_InitForExtension(interp, type) < 0) { + return -1; + } + } + + return 0; +} + + +/* --------------------------------------------------------------------------- + * Module methods and initialization. + */ + +static PyMethodDef module_methods[] = { + {NULL, NULL} +}; + + +static int +_datetime_exec(PyObject *module) +{ + int rc = -1; + datetime_state *st = get_module_state(module); + int reloading = 0; - for (size_t i = 0; i < Py_ARRAY_LENGTH(types); i++) { - if (PyModule_AddType(module, types[i]) < 0) { + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *old_module = get_current_module(interp, &reloading); + if (PyErr_Occurred()) { + assert(old_module == NULL); + goto error; + } + /* We actually set the "current" module right before a successful return. */ + + if (init_static_types(interp, reloading) < 0) { + goto error; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + const char *name = _PyType_Name(type); + assert(name != NULL); + if (PyModule_AddObjectRef(module, name, (PyObject *)type) < 0) { goto error; } } - if (PyType_Ready(&PyDateTime_IsoCalendarDateType) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ + assert(!PyErr_Occurred()); \ PyObject *value = (value_expr); \ if (value == NULL) { \ goto error; \ @@ -6846,40 +7257,35 @@ _datetime_exec(PyObject *module) } while(0) /* timedelta values */ - PyObject *d = PyDateTime_DeltaType.tp_dict; + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); DATETIME_ADD_MACRO(d, "max", new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); /* date values */ - d = PyDateTime_DateType.tp_dict; + d = _PyType_GetDict(&PyDateTime_DateType); DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); /* time values */ - d = PyDateTime_TimeType.tp_dict; + d = _PyType_GetDict(&PyDateTime_TimeType); DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* datetime values */ - d = PyDateTime_DateTimeType.tp_dict; + d = _PyType_GetDict(&PyDateTime_DateTimeType); DATETIME_ADD_MACRO(d, "min", new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - datetime_state *st = STATIC_STATE(); - if (init_state(st) < 0) { - goto error; - } - /* timezone values */ - d = PyDateTime_TimeZoneType.tp_dict; - if (PyDict_SetItemString(d, "utc", st->utc) < 0) { + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -6888,12 +7294,13 @@ _datetime_exec(PyObject *module) * values. This may change in the future.*/ /* -23:59 */ - PyObject *min = create_timezone_from_delta(-1, 60, 0, 1); - DATETIME_ADD_MACRO(d, "min", min); + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); /* +23:59 */ - PyObject *max = create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0); - DATETIME_ADD_MACRO(d, "max", max); + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { @@ -6902,7 +7309,7 @@ _datetime_exec(PyObject *module) if (PyModule_AddIntMacro(module, MAXYEAR) < 0) { goto error; } - if (PyModule_AddObjectRef(module, "UTC", st->utc) < 0) { + if (PyModule_AddObjectRef(module, "UTC", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -6911,14 +7318,9 @@ _datetime_exec(PyObject *module) if (capi == NULL) { goto error; } - PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, - datetime_destructor); - if (capsule == NULL) { - PyMem_Free(capi); - goto error; - } + PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); + // (capsule == NULL) is handled by PyModule_Add if (PyModule_Add(module, "datetime_CAPI", capsule) < 0) { - PyMem_Free(capi); goto error; } @@ -6940,35 +7342,79 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - return 0; + if (reloading) { + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyType_Modified(capi_types[i]); + } + } + + if (set_current_module(interp, module) < 0) { + goto error; + } + + rc = 0; + goto finally; error: - datetime_clear(module); - return -1; + clear_state(st); + +finally: + Py_XDECREF(old_module); + return rc; } -#undef DATETIME_ADD_MACRO -static struct PyModuleDef datetimemodule = { +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + datetime_state *st = get_module_state(mod); + traverse_state(st, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + datetime_state *st = get_module_state(mod); + clear_state(st); + + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + + // The runtime takes care of the static types for us. + // See _PyTypes_FiniExtTypes().. + + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); +} + +static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", .m_doc = "Fast implementation of the datetime type.", - .m_size = -1, + .m_size = sizeof(datetime_state), .m_methods = module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, }; PyMODINIT_FUNC PyInit__datetime(void) { - PyObject *mod = PyModule_Create(&datetimemodule); - if (mod == NULL) - return NULL; - - if (_datetime_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - - return mod; + return PyModuleDef_Init(&datetimemodule); } /* --------------------------------------------------------------------------- diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index bd807698927e86..1be4234aad3291 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -2,6 +2,11 @@ /* DBM module using dictionary interface */ +// clinic/_dbmmodule.c.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include @@ -552,11 +557,6 @@ dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, } const char *name = PyBytes_AS_STRING(filenamebytes); - if (strlen(name) != (size_t)PyBytes_GET_SIZE(filenamebytes)) { - Py_DECREF(filenamebytes); - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return NULL; - } PyObject *self = newdbmobject(state, name, iflags, mode); Py_DECREF(filenamebytes); return self; @@ -616,6 +616,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 8b93f8e2cbcf0b..94a2cc2c8e5f8a 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -32,8 +32,21 @@ #include #include "pycore_long.h" // _PyLong_IsZero() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_typeobject.h" #include "complexobject.h" -#include "mpdecimal.h" + +#include + +// Reuse config from mpdecimal.h if present. +#if defined(MPD_CONFIG_64) + #ifndef CONFIG_64 + #define CONFIG_64 MPD_CONFIG_64 + #endif +#elif defined(MPD_CONFIG_32) + #ifndef CONFIG_32 + #define CONFIG_32 MPD_CONFIG_32 + #endif +#endif #include // isascii() #include @@ -82,6 +95,9 @@ typedef struct { /* Convert rationals for comparison */ PyObject *Rational; + /* Invariant: NULL or pointer to _pydecimal.Decimal */ + PyObject *PyDecimal; + PyObject *SignalTuple; struct DecCondMap *signal_map; @@ -117,11 +133,8 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = PyType_GetModuleByDef(Py_TYPE(left), &_decimal_module); - if (mod == NULL) { - PyErr_Clear(); - mod = PyType_GetModuleByDef(Py_TYPE(right), &_decimal_module); - } + PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), + &_decimal_module); assert(mod != NULL); return get_module_state(mod); } @@ -2412,12 +2425,12 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, } sign = (copysign(1.0, x) == 1.0) ? 0 : 1; - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) { + if (isnan(x) || isinf(x)) { dec = PyDecType_New(type); if (dec == NULL) { return NULL; } - if (Py_IS_NAN(x)) { + if (isnan(x)) { /* decimal.py calls repr(float(+-nan)), * which always gives a positive result. */ mpd_setspecial(MPD(dec), MPD_POS, MPD_NAN); @@ -3336,56 +3349,6 @@ dotsep_as_utf8(const char *s) return utf8; } -/* copy of libmpdec _mpd_round() */ -static void -_mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t exp = a->exp + a->digits - prec; - - if (prec <= 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isspecial(a) || mpd_iszero(a)) { - mpd_qcopy(result, a, status); - return; - } - - mpd_qrescale_fmt(result, a, exp, ctx, status); - if (result->digits > prec) { - mpd_qrescale_fmt(result, result, exp+1, ctx, status); - } -} - -/* Locate negative zero "z" option within a UTF-8 format spec string. - * Returns pointer to "z", else NULL. - * The portion of the spec we're working with is [[fill]align][sign][z] */ -static const char * -format_spec_z_search(char const *fmt, Py_ssize_t size) { - char const *pos = fmt; - char const *fmt_end = fmt + size; - /* skip over [[fill]align] (fill may be multi-byte character) */ - pos += 1; - while (pos < fmt_end && *pos & 0x80) { - pos += 1; - } - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } else { - /* fill not present-- skip over [align] */ - pos = fmt; - if (pos < fmt_end && strchr("<>=^", *pos) != NULL) { - pos += 1; - } - } - /* skip over [sign] */ - if (pos < fmt_end && strchr("+- ", *pos) != NULL) { - pos += 1; - } - return pos < fmt_end && *pos == 'z' ? pos : NULL; -} - static int dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const char **valuestr) { @@ -3411,6 +3374,48 @@ dict_get_item_string(PyObject *dict, const char *key, PyObject **valueobj, const return 0; } +/* + * Fallback _pydecimal formatting for new format specifiers that mpdecimal does + * not yet support. As documented, libmpdec follows the PEP-3101 format language: + * https://www.bytereef.org/mpdecimal/doc/libmpdec/assign-convert.html#to-string + */ +static PyObject * +pydec_format(PyObject *dec, PyObject *context, PyObject *fmt, decimal_state *state) +{ + PyObject *result; + PyObject *pydec; + PyObject *u; + + if (state->PyDecimal == NULL) { + state->PyDecimal = _PyImport_GetModuleAttrString("_pydecimal", "Decimal"); + if (state->PyDecimal == NULL) { + return NULL; + } + } + + u = dec_str(dec); + if (u == NULL) { + return NULL; + } + + pydec = PyObject_CallOneArg(state->PyDecimal, u); + Py_DECREF(u); + if (pydec == NULL) { + return NULL; + } + + result = PyObject_CallMethod(pydec, "__format__", "(OO)", fmt, context); + Py_DECREF(pydec); + + if (result == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) { + /* Do not confuse users with the _pydecimal exception */ + PyErr_Clear(); + PyErr_SetString(PyExc_ValueError, "invalid format string"); + } + + return result; +} + /* Formatted representation of a PyDecObject. */ static PyObject * dec_format(PyObject *dec, PyObject *args) @@ -3423,16 +3428,11 @@ dec_format(PyObject *dec, PyObject *args) PyObject *fmtarg; PyObject *context; mpd_spec_t spec; - char const *fmt; - char *fmt_copy = NULL; + char *fmt; char *decstring = NULL; uint32_t status = 0; int replace_fillchar = 0; - int no_neg_0 = 0; Py_ssize_t size; - mpd_t *mpd = MPD(dec); - mpd_uint_t dt[MPD_MINALLOC_MAX]; - mpd_t tmp = {MPD_STATIC|MPD_STATIC_DATA,0,0,0,MPD_MINALLOC_MAX,dt}; decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); @@ -3442,39 +3442,27 @@ dec_format(PyObject *dec, PyObject *args) } if (PyUnicode_Check(fmtarg)) { - fmt = PyUnicode_AsUTF8AndSize(fmtarg, &size); + fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size); if (fmt == NULL) { return NULL; } - /* NOTE: If https://github.com/python/cpython/pull/29438 lands, the - * format string manipulation below can be eliminated by enhancing - * the forked mpd_parse_fmt_str(). */ + + if (size > 0 && fmt[size-1] == 'N') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Format specifier 'N' is deprecated", 1) < 0) { + return NULL; + } + } + if (size > 0 && fmt[0] == '\0') { /* NUL fill character: must be replaced with a valid UTF-8 char before calling mpd_parse_fmt_str(). */ replace_fillchar = 1; - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { + fmt = dec_strdup(fmt, size); + if (fmt == NULL) { return NULL; } - fmt_copy[0] = '_'; - } - /* Strip 'z' option, which isn't understood by mpd_parse_fmt_str(). - * NOTE: fmt is always null terminated by PyUnicode_AsUTF8AndSize() */ - char const *z_position = format_spec_z_search(fmt, size); - if (z_position != NULL) { - no_neg_0 = 1; - size_t z_index = z_position - fmt; - if (fmt_copy == NULL) { - fmt = fmt_copy = dec_strdup(fmt, size); - if (fmt_copy == NULL) { - return NULL; - } - } - /* Shift characters (including null terminator) left, - overwriting the 'z' option. */ - memmove(fmt_copy + z_index, fmt_copy + z_index + 1, size - z_index); - size -= 1; + fmt[0] = '_'; } } else { @@ -3484,10 +3472,13 @@ dec_format(PyObject *dec, PyObject *args) } if (!mpd_parse_fmt_str(&spec, fmt, CtxCaps(context))) { - PyErr_SetString(PyExc_ValueError, - "invalid format string"); - goto finish; + if (replace_fillchar) { + PyMem_Free(fmt); + } + + return pydec_format(dec, context, fmtarg, state); } + if (replace_fillchar) { /* In order to avoid clobbering parts of UTF-8 thousands separators or decimal points when the substitution is reversed later, the actual @@ -3540,45 +3531,8 @@ dec_format(PyObject *dec, PyObject *args) } } - if (no_neg_0 && mpd_isnegative(mpd) && !mpd_isspecial(mpd)) { - /* Round into a temporary (carefully mirroring the rounding - of mpd_qformat_spec()), and check if the result is negative zero. - If so, clear the sign and format the resulting positive zero. */ - mpd_ssize_t prec; - mpd_qcopy(&tmp, mpd, &status); - if (spec.prec >= 0) { - switch (spec.type) { - case 'f': - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case '%': - tmp.exp += 2; - mpd_qrescale(&tmp, &tmp, -spec.prec, CTX(context), &status); - break; - case 'g': - prec = (spec.prec == 0) ? 1 : spec.prec; - if (tmp.digits > prec) { - _mpd_round(&tmp, &tmp, prec, CTX(context), &status); - } - break; - case 'e': - if (!mpd_iszero(&tmp)) { - _mpd_round(&tmp, &tmp, spec.prec+1, CTX(context), &status); - } - break; - } - } - if (status & MPD_Errors) { - PyErr_SetString(PyExc_ValueError, "unexpected error when rounding"); - goto finish; - } - if (mpd_iszero(&tmp)) { - mpd_set_positive(&tmp); - mpd = &tmp; - } - } - decstring = mpd_qformat_spec(mpd, &spec, CTX(context), &status); + decstring = mpd_qformat_spec(MPD(dec), &spec, CTX(context), &status); if (decstring == NULL) { if (status & MPD_Malloc_error) { PyErr_NoMemory(); @@ -3593,12 +3547,6 @@ dec_format(PyObject *dec, PyObject *args) if (replace_fillchar) { dec_replace_fillchar(decstring); } - if (strchr(fmt, 'N') != NULL) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { - goto finish; - } - } result = PyUnicode_DecodeUTF8(decstring, size, NULL); @@ -3607,7 +3555,7 @@ dec_format(PyObject *dec, PyObject *args) Py_XDECREF(grouping); Py_XDECREF(sep); Py_XDECREF(dot); - if (fmt_copy) PyMem_Free(fmt_copy); + if (replace_fillchar) PyMem_Free(fmt); if (decstring) mpd_free(decstring); return result; } @@ -4339,7 +4287,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -4842,7 +4790,7 @@ _dec_hash(PyDecObject *v) return -1; } else if (mpd_isnan(MPD(v))) { - return _Py_HashPointer(v); + return PyObject_GenericHash((PyObject *)v); } else { return py_hash_inf * mpd_arith_sign(MPD(v)); @@ -5364,7 +5312,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -5985,6 +5933,9 @@ _decimal_exec(PyObject *m) Py_CLEAR(collections_abc); Py_CLEAR(MutableMapping); + /* For format specifiers not yet supported by libmpdec */ + state->PyDecimal = NULL; + /* Add types to the module */ CHECK_INT(PyModule_AddType(m, state->PyDec_Type)); CHECK_INT(PyModule_AddType(m, state->PyDecContext_Type)); @@ -6190,6 +6141,7 @@ decimal_clear(PyObject *module) Py_CLEAR(state->extended_context_template); Py_CLEAR(state->Rational); Py_CLEAR(state->SignalTuple); + Py_CLEAR(state->PyDecimal); PyMem_Free(state->signal_map); PyMem_Free(state->cond_map); @@ -6205,6 +6157,7 @@ decimal_free(void *module) static struct PyModuleDef_Slot _decimal_slots[] = { {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index e7bd6aee170056..4e95b8964c8e5d 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -48,6 +48,7 @@ #if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #pragma GCC diagnostic ignored "-Wmisleading-indentation" + #pragma GCC diagnostic ignored "-Warray-bounds" #endif diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h new file mode 100644 index 00000000000000..77bc6229fbc119 --- /dev/null +++ b/Modules/_decimal/windows/mpdecimal.h @@ -0,0 +1,17 @@ +/* Windows mpdecimal.h shim + * + * Generally, the mpdecimal library build will copy the correct header into + * place named "mpdecimal.h", but since we're building it ourselves directly + * into _decimal.pyd, we need to pick the right one. + * + * */ + +#if defined(_MSC_VER) + #if defined(CONFIG_64) + #include + #elif defined(CONFIG_32) + #include + #else + #error "Unknown configuration!" + #endif +#endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f9d5793f9b6497..3818e20b4f0f28 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -98,6 +98,7 @@ typedef struct { PyTypeObject *TreeBuilder_Type; PyTypeObject *XMLParser_Type; + PyObject *expat_capsule; struct PyExpat_CAPI *expat_capi; } elementtreestate; @@ -155,6 +156,7 @@ elementtree_clear(PyObject *m) Py_CLEAR(st->ElementIter_Type); Py_CLEAR(st->TreeBuilder_Type); Py_CLEAR(st->XMLParser_Type); + Py_CLEAR(st->expat_capsule); st->expat_capi = NULL; return 0; @@ -175,6 +177,7 @@ elementtree_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->ElementIter_Type); Py_VISIT(st->TreeBuilder_Type); Py_VISIT(st->XMLParser_Type); + Py_VISIT(st->expat_capsule); return 0; } @@ -264,7 +267,7 @@ typedef struct { LOCAL(int) create_extra(ElementObject* self, PyObject* attrib) { - self->extra = PyObject_Malloc(sizeof(ElementObjectExtra)); + self->extra = PyMem_Malloc(sizeof(ElementObjectExtra)); if (!self->extra) { PyErr_NoMemory(); return -1; @@ -292,10 +295,11 @@ dealloc_extra(ElementObjectExtra *extra) for (i = 0; i < extra->length; i++) Py_DECREF(extra->children[i]); - if (extra->children != extra->_children) - PyObject_Free(extra->children); + if (extra->children != extra->_children) { + PyMem_Free(extra->children); + } - PyObject_Free(extra); + PyMem_Free(extra); } LOCAL(void) @@ -368,33 +372,27 @@ element_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject* get_attrib_from_keywords(PyObject *kwds) { - PyObject *attrib_str = PyUnicode_FromString("attrib"); - if (attrib_str == NULL) { + PyObject *attrib; + if (PyDict_PopString(kwds, "attrib", &attrib) < 0) { return NULL; } - PyObject *attrib = PyDict_GetItemWithError(kwds, attrib_str); if (attrib) { /* If attrib was found in kwds, copy its value and remove it from * kwds */ if (!PyDict_Check(attrib)) { - Py_DECREF(attrib_str); PyErr_Format(PyExc_TypeError, "attrib must be dict, not %.100s", Py_TYPE(attrib)->tp_name); + Py_DECREF(attrib); return NULL; } - attrib = PyDict_Copy(attrib); - if (attrib && PyDict_DelItem(kwds, attrib_str) < 0) { - Py_SETREF(attrib, NULL); - } + Py_SETREF(attrib, PyDict_Copy(attrib)); } - else if (!PyErr_Occurred()) { + else { attrib = PyDict_New(); } - Py_DECREF(attrib_str); - if (attrib != NULL && PyDict_Update(attrib, kwds) < 0) { Py_DECREF(attrib); return NULL; @@ -492,14 +490,16 @@ element_resize(ElementObject* self, Py_ssize_t extra) * "children", which needs at least 4 bytes. Although it's a * false alarm always assume at least one child to be safe. */ - children = PyObject_Realloc(self->extra->children, - size * sizeof(PyObject*)); - if (!children) + children = PyMem_Realloc(self->extra->children, + size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } } else { - children = PyObject_Malloc(size * sizeof(PyObject*)); - if (!children) + children = PyMem_Malloc(size * sizeof(PyObject*)); + if (!children) { goto nomemory; + } /* copy existing children from static area to malloc buffer */ memcpy(children, self->extra->children, self->extra->length * sizeof(PyObject*)); @@ -1502,7 +1502,7 @@ element_bool(PyObject* self_) { ElementObject* self = (ElementObject*) self_; if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Testing an element's truth value will raise an exception " + "Testing an element's truth value will always return True " "in future versions. Use specific 'len(elem)' or " "'elem is not None' test instead.", 1) < 0) { @@ -3041,7 +3041,7 @@ _elementtree_TreeBuilder_start_impl(TreeBuilderObject *self, PyObject *tag, #define EXPAT(st, func) ((st)->expat_capi->func) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; typedef struct { PyObject_HEAD @@ -3066,6 +3066,7 @@ typedef struct { PyObject *handle_close; elementtreestate *state; + PyObject *elementtree_module; } XMLParserObject; /* helpers */ @@ -3607,7 +3608,11 @@ xmlparser_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->handle_start = self->handle_data = self->handle_end = NULL; self->handle_comment = self->handle_pi = self->handle_close = NULL; self->handle_doctype = NULL; - self->state = get_elementtree_state_by_type(type); + self->elementtree_module = PyType_GetModuleByDef(type, &elementtreemodule); + assert(self->elementtree_module != NULL); + Py_INCREF(self->elementtree_module); + // See gh-111784 for explanation why is reference to module needed here. + self->state = get_elementtree_state(self->elementtree_module); } return (PyObject *)self; } @@ -3784,6 +3789,7 @@ xmlparser_gc_clear(XMLParserObject *self) EXPAT(st, ParserFree)(parser); } + Py_CLEAR(self->elementtree_module); Py_CLEAR(self->handle_close); Py_CLEAR(self->handle_pi); Py_CLEAR(self->handle_comment); @@ -3882,6 +3888,40 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self) } } +/*[clinic input] +_elementtree.XMLParser.flush + +[clinic start generated code]*/ + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self) +/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/ +{ + if (!_check_xmlparser(self)) { + return NULL; + } + + elementtreestate *st = self->state; + + if (EXPAT(st, SetReparseDeferralEnabled) == NULL) { + Py_RETURN_NONE; + } + + // NOTE: The Expat parser in the C implementation of ElementTree is not + // exposed to the outside; as a result we known that reparse deferral + // is currently enabled, or we would not even have access to function + // XML_SetReparseDeferralEnabled in the first place (which we checked + // for, a few lines up). + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_FALSE); + + PyObject *res = expat_parse(st, self, "", 0, XML_FALSE); + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_TRUE); + + return res; +} + /*[clinic input] _elementtree.XMLParser.feed @@ -4276,6 +4316,7 @@ static PyType_Spec treebuilder_spec = { static PyMethodDef xmlparser_methods[] = { _ELEMENTTREE_XMLPARSER_FEED_METHODDEF _ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF + _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF _ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF _ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF {NULL, NULL} @@ -4343,7 +4384,10 @@ module_exec(PyObject *m) goto error; /* link against pyexpat */ - st->expat_capi = PyCapsule_Import(PyExpat_CAPSULE_NAME, 0); + if (!(st->expat_capsule = _PyImport_GetModuleAttrString("pyexpat", "expat_CAPI"))) + goto error; + if (!(st->expat_capi = PyCapsule_GetPointer(st->expat_capsule, PyExpat_CAPSULE_NAME))) + goto error; if (st->expat_capi) { /* check that it's usable */ if (strcmp(st->expat_capi->magic, PyExpat_CAPI_MAGIC) != 0 || @@ -4418,9 +4462,8 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 8ea493ad9ab278..64766b474514bf 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -79,12 +79,19 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + _functools_state *state = get_functools_state_by_type(type); + if (state == NULL) { + return NULL; + } + pargs = pkw = NULL; func = PyTuple_GET_ITEM(args, 0); - if (Py_TYPE(func)->tp_call == (ternaryfunc)partial_call) { - // The type of "func" might not be exactly the same type object - // as "type", but if it is called using partial_call, it must have the - // same memory layout (fn, args and kw members). + + int res = PyObject_TypeCheck(func, state->partial_type); + if (res == -1) { + return NULL; + } + if (res == 1) { // We can use its underlying function directly and merge the arguments. partialobject *part = (partialobject *)func; if (part->dict == NULL) { @@ -190,6 +197,14 @@ partial_dealloc(partialobject *pto) Py_DECREF(tp); } +static PyObject * +partial_descr_get(PyObject *self, PyObject *obj, PyObject *type) +{ + if (obj == Py_None || obj == NULL) { + return Py_NewRef(self); + } + return PyMethod_New(self, obj); +} /* Merging keyword arguments using the vectorcall convention is messy, so * if we would need to do that, we stop using vectorcall and fall back @@ -335,8 +350,9 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(partial_doc, -"partial(func, *args, **keywords) - new function with partial application\n\ - of the given arguments and keywords.\n"); +"partial(func, /, *args, **keywords)\n--\n\n\ +Create a new function with partial application of the given arguments\n\ +and keywords."); #define OFF(x) offsetof(partialobject, x) static PyMemberDef partial_memberlist[] = { @@ -365,6 +381,8 @@ partial_repr(partialobject *pto) { PyObject *result = NULL; PyObject *arglist; + PyObject *mod; + PyObject *name; Py_ssize_t i, n; PyObject *key, *value; int status; @@ -399,13 +417,28 @@ partial_repr(partialobject *pto) if (arglist == NULL) goto done; } - result = PyUnicode_FromFormat("%s(%R%U)", Py_TYPE(pto)->tp_name, - pto->fn, arglist); + + mod = PyType_GetModuleName(Py_TYPE(pto)); + if (mod == NULL) { + goto error; + } + name = PyType_GetQualName(Py_TYPE(pto)); + if (name == NULL) { + Py_DECREF(mod); + goto error; + } + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); + Py_DECREF(mod); + Py_DECREF(name); Py_DECREF(arglist); done: Py_ReprLeave((PyObject *)pto); return result; + error: + Py_DECREF(arglist); + Py_ReprLeave((PyObject *)pto); + return NULL; } /* Pickle strategy: @@ -489,6 +522,7 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, + {Py_tp_descr_get, (descrgetfunc)partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} @@ -546,6 +580,17 @@ static PyMemberDef keyobject_members[] = { {NULL} }; +static PyObject * +keyobject_text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj)"); +} + +static PyGetSetDef keyobject_getset[] = { + {"__text_signature__", keyobject_text_signature, (setter)NULL}, + {NULL} +}; + static PyObject * keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); @@ -560,6 +605,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_getset, keyobject_getset}, {0, 0} }; @@ -1087,19 +1133,9 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds The cache dict holds one reference to the link. We created one other reference when the link was created. The linked list only has borrowed references. */ - popresult = _PyDict_Pop_KnownHash(self->cache, link->key, - link->hash, Py_None); - if (popresult == Py_None) { - /* Getting here means that the user function call or another - thread has already removed the old key from the dictionary. - This link is now an orphan. Since we don't want to leave the - cache in an inconsistent state, we don't restore the link. */ - Py_DECREF(popresult); - Py_DECREF(link); - Py_DECREF(key); - return result; - } - if (popresult == NULL) { + int res = _PyDict_Pop_KnownHash((PyDictObject*)self->cache, link->key, + link->hash, &popresult); + if (res < 0) { /* An error arose while trying to remove the oldest key (the one being evicted) from the cache. We restore the link to its original position as the oldest link. Then we allow the @@ -1110,10 +1146,22 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds Py_DECREF(result); return NULL; } + if (res == 0) { + /* Getting here means that the user function call or another + thread has already removed the old key from the dictionary. + This link is now an orphan. Since we don't want to leave the + cache in an inconsistent state, we don't restore the link. */ + assert(popresult == NULL); + Py_DECREF(link); + Py_DECREF(key); + return result; + } + /* Keep a reference to the old key and old result to prevent their ref counts from going to zero during the update. That will prevent potentially arbitrary object clean-up code (i.e. __del__) from running while we're still adjusting the links. */ + assert(popresult != NULL); oldkey = link->key; oldresult = link->result; @@ -1272,7 +1320,11 @@ lru_cache_dealloc(lru_cache_object *obj) static PyObject * lru_cache_call(lru_cache_object *self, PyObject *args, PyObject *kwds) { - return self->wrapper(self, args, kwds); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = self->wrapper(self, args, kwds); + Py_END_CRITICAL_SECTION(); + return result; } static PyObject * @@ -1285,6 +1337,7 @@ lru_cache_descr_get(PyObject *self, PyObject *obj, PyObject *type) } /*[clinic input] +@critical_section _functools._lru_cache_wrapper.cache_info Report cache statistics @@ -1292,7 +1345,7 @@ Report cache statistics static PyObject * _functools__lru_cache_wrapper_cache_info_impl(PyObject *self) -/*[clinic end generated code: output=cc796a0b06dbd717 input=f05e5b6ebfe38645]*/ +/*[clinic end generated code: output=cc796a0b06dbd717 input=00e1acb31aa21ecc]*/ { lru_cache_object *_self = (lru_cache_object *) self; if (_self->maxsize == -1) { @@ -1306,6 +1359,7 @@ _functools__lru_cache_wrapper_cache_info_impl(PyObject *self) } /*[clinic input] +@critical_section _functools._lru_cache_wrapper.cache_clear Clear the cache and cache statistics @@ -1313,7 +1367,7 @@ Clear the cache and cache statistics static PyObject * _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self) -/*[clinic end generated code: output=58423b35efc3e381 input=6ca59dba09b12584]*/ +/*[clinic end generated code: output=58423b35efc3e381 input=dfa33acbecf8b4b2]*/ { lru_cache_object *_self = (lru_cache_object *) self; lru_list_elem *list = lru_cache_unlink_list(_self); @@ -1514,6 +1568,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index eff36fd7fb669b..df7fba67810ed0 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -1,8 +1,12 @@ - /* GDBM module using dictionary interface */ /* Author: Anthony Baxter, after dbmmodule.c */ /* Doc strings: Mitch Chapman */ +// clinic/_gdbmmodule.c.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "gdbm.h" @@ -821,6 +825,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 222ac824f01961..ed294839ed8dc0 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -25,37 +25,29 @@ #include "internal/Hacl_Hash_MD5.h" -static uint32_t -_h0[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; +static uint32_t _h0[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; static uint32_t _t[64U] = { - (uint32_t)0xd76aa478U, (uint32_t)0xe8c7b756U, (uint32_t)0x242070dbU, (uint32_t)0xc1bdceeeU, - (uint32_t)0xf57c0fafU, (uint32_t)0x4787c62aU, (uint32_t)0xa8304613U, (uint32_t)0xfd469501U, - (uint32_t)0x698098d8U, (uint32_t)0x8b44f7afU, (uint32_t)0xffff5bb1U, (uint32_t)0x895cd7beU, - (uint32_t)0x6b901122U, (uint32_t)0xfd987193U, (uint32_t)0xa679438eU, (uint32_t)0x49b40821U, - (uint32_t)0xf61e2562U, (uint32_t)0xc040b340U, (uint32_t)0x265e5a51U, (uint32_t)0xe9b6c7aaU, - (uint32_t)0xd62f105dU, (uint32_t)0x02441453U, (uint32_t)0xd8a1e681U, (uint32_t)0xe7d3fbc8U, - (uint32_t)0x21e1cde6U, (uint32_t)0xc33707d6U, (uint32_t)0xf4d50d87U, (uint32_t)0x455a14edU, - (uint32_t)0xa9e3e905U, (uint32_t)0xfcefa3f8U, (uint32_t)0x676f02d9U, (uint32_t)0x8d2a4c8aU, - (uint32_t)0xfffa3942U, (uint32_t)0x8771f681U, (uint32_t)0x6d9d6122U, (uint32_t)0xfde5380cU, - (uint32_t)0xa4beea44U, (uint32_t)0x4bdecfa9U, (uint32_t)0xf6bb4b60U, (uint32_t)0xbebfbc70U, - (uint32_t)0x289b7ec6U, (uint32_t)0xeaa127faU, (uint32_t)0xd4ef3085U, (uint32_t)0x4881d05U, - (uint32_t)0xd9d4d039U, (uint32_t)0xe6db99e5U, (uint32_t)0x1fa27cf8U, (uint32_t)0xc4ac5665U, - (uint32_t)0xf4292244U, (uint32_t)0x432aff97U, (uint32_t)0xab9423a7U, (uint32_t)0xfc93a039U, - (uint32_t)0x655b59c3U, (uint32_t)0x8f0ccc92U, (uint32_t)0xffeff47dU, (uint32_t)0x85845dd1U, - (uint32_t)0x6fa87e4fU, (uint32_t)0xfe2ce6e0U, (uint32_t)0xa3014314U, (uint32_t)0x4e0811a1U, - (uint32_t)0xf7537e82U, (uint32_t)0xbd3af235U, (uint32_t)0x2ad7d2bbU, (uint32_t)0xeb86d391U + 0xd76aa478U, 0xe8c7b756U, 0x242070dbU, 0xc1bdceeeU, 0xf57c0fafU, 0x4787c62aU, 0xa8304613U, + 0xfd469501U, 0x698098d8U, 0x8b44f7afU, 0xffff5bb1U, 0x895cd7beU, 0x6b901122U, 0xfd987193U, + 0xa679438eU, 0x49b40821U, 0xf61e2562U, 0xc040b340U, 0x265e5a51U, 0xe9b6c7aaU, 0xd62f105dU, + 0x02441453U, 0xd8a1e681U, 0xe7d3fbc8U, 0x21e1cde6U, 0xc33707d6U, 0xf4d50d87U, 0x455a14edU, + 0xa9e3e905U, 0xfcefa3f8U, 0x676f02d9U, 0x8d2a4c8aU, 0xfffa3942U, 0x8771f681U, 0x6d9d6122U, + 0xfde5380cU, 0xa4beea44U, 0x4bdecfa9U, 0xf6bb4b60U, 0xbebfbc70U, 0x289b7ec6U, 0xeaa127faU, + 0xd4ef3085U, 0x4881d05U, 0xd9d4d039U, 0xe6db99e5U, 0x1fa27cf8U, 0xc4ac5665U, 0xf4292244U, + 0x432aff97U, 0xab9423a7U, 0xfc93a039U, 0x655b59c3U, 0x8f0ccc92U, 0xffeff47dU, 0x85845dd1U, + 0x6fa87e4fU, 0xfe2ce6e0U, 0xa3014314U, 0x4e0811a1U, 0xf7537e82U, 0xbd3af235U, 0x2ad7d2bbU, + 0xeb86d391U }; -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s) +void Hacl_Hash_MD5_init(uint32_t *s) { - KRML_MAYBE_FOR4(i, (uint32_t)0U, (uint32_t)4U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *abcd, uint8_t *x) +static void update(uint32_t *abcd, uint8_t *x) { uint32_t aa = abcd[0U]; uint32_t bb = abcd[1U]; @@ -74,14 +66,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb0 + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << (uint32_t)7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> (uint32_t)25U); + << 7U + | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; uint32_t vc1 = abcd[1U]; uint32_t vd1 = abcd[2U]; - uint8_t *b1 = x + (uint32_t)4U; + uint8_t *b1 = x + 4U; uint32_t u0 = load32_le(b1); uint32_t xk0 = u0; uint32_t ti1 = _t[1U]; @@ -90,14 +82,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb1 + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << (uint32_t)12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> (uint32_t)20U); + << 12U + | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; uint32_t vc2 = abcd[0U]; uint32_t vd2 = abcd[1U]; - uint8_t *b2 = x + (uint32_t)8U; + uint8_t *b2 = x + 8U; uint32_t u1 = load32_le(b2); uint32_t xk1 = u1; uint32_t ti2 = _t[2U]; @@ -106,14 +98,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb2 + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << (uint32_t)17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> (uint32_t)15U); + << 17U + | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; uint32_t vc3 = abcd[3U]; uint32_t vd3 = abcd[0U]; - uint8_t *b3 = x + (uint32_t)12U; + uint8_t *b3 = x + 12U; uint32_t u2 = load32_le(b3); uint32_t xk2 = u2; uint32_t ti3 = _t[3U]; @@ -122,14 +114,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb3 + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << (uint32_t)22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> (uint32_t)10U); + << 22U + | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; uint32_t vc4 = abcd[2U]; uint32_t vd4 = abcd[3U]; - uint8_t *b4 = x + (uint32_t)16U; + uint8_t *b4 = x + 16U; uint32_t u3 = load32_le(b4); uint32_t xk3 = u3; uint32_t ti4 = _t[4U]; @@ -138,14 +130,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb4 + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << (uint32_t)7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> (uint32_t)25U); + << 7U + | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; uint32_t vc5 = abcd[1U]; uint32_t vd5 = abcd[2U]; - uint8_t *b5 = x + (uint32_t)20U; + uint8_t *b5 = x + 20U; uint32_t u4 = load32_le(b5); uint32_t xk4 = u4; uint32_t ti5 = _t[5U]; @@ -154,14 +146,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb5 + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << (uint32_t)12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> (uint32_t)20U); + << 12U + | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; uint32_t vc6 = abcd[0U]; uint32_t vd6 = abcd[1U]; - uint8_t *b6 = x + (uint32_t)24U; + uint8_t *b6 = x + 24U; uint32_t u5 = load32_le(b6); uint32_t xk5 = u5; uint32_t ti6 = _t[6U]; @@ -170,14 +162,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb6 + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << (uint32_t)17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> (uint32_t)15U); + << 17U + | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; uint32_t vc7 = abcd[3U]; uint32_t vd7 = abcd[0U]; - uint8_t *b7 = x + (uint32_t)28U; + uint8_t *b7 = x + 28U; uint32_t u6 = load32_le(b7); uint32_t xk6 = u6; uint32_t ti7 = _t[7U]; @@ -186,14 +178,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb7 + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << (uint32_t)22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> (uint32_t)10U); + << 22U + | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; uint32_t vc8 = abcd[2U]; uint32_t vd8 = abcd[3U]; - uint8_t *b8 = x + (uint32_t)32U; + uint8_t *b8 = x + 32U; uint32_t u7 = load32_le(b8); uint32_t xk7 = u7; uint32_t ti8 = _t[8U]; @@ -202,14 +194,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb8 + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << (uint32_t)7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> (uint32_t)25U); + << 7U + | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; uint32_t vc9 = abcd[1U]; uint32_t vd9 = abcd[2U]; - uint8_t *b9 = x + (uint32_t)36U; + uint8_t *b9 = x + 36U; uint32_t u8 = load32_le(b9); uint32_t xk8 = u8; uint32_t ti9 = _t[9U]; @@ -218,14 +210,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb9 + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << (uint32_t)12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> (uint32_t)20U); + << 12U + | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; uint32_t vc10 = abcd[0U]; uint32_t vd10 = abcd[1U]; - uint8_t *b10 = x + (uint32_t)40U; + uint8_t *b10 = x + 40U; uint32_t u9 = load32_le(b10); uint32_t xk9 = u9; uint32_t ti10 = _t[10U]; @@ -234,14 +226,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb10 + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << (uint32_t)17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> (uint32_t)15U); + << 17U + | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; uint32_t vc11 = abcd[3U]; uint32_t vd11 = abcd[0U]; - uint8_t *b11 = x + (uint32_t)44U; + uint8_t *b11 = x + 44U; uint32_t u10 = load32_le(b11); uint32_t xk10 = u10; uint32_t ti11 = _t[11U]; @@ -250,14 +242,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb11 + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << (uint32_t)22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> (uint32_t)10U); + << 22U + | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; uint32_t vc12 = abcd[2U]; uint32_t vd12 = abcd[3U]; - uint8_t *b12 = x + (uint32_t)48U; + uint8_t *b12 = x + 48U; uint32_t u11 = load32_le(b12); uint32_t xk11 = u11; uint32_t ti12 = _t[12U]; @@ -266,14 +258,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb12 + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << (uint32_t)7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> (uint32_t)25U); + << 7U + | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; uint32_t vc13 = abcd[1U]; uint32_t vd13 = abcd[2U]; - uint8_t *b13 = x + (uint32_t)52U; + uint8_t *b13 = x + 52U; uint32_t u12 = load32_le(b13); uint32_t xk12 = u12; uint32_t ti13 = _t[13U]; @@ -282,14 +274,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb13 + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << (uint32_t)12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> (uint32_t)20U); + << 12U + | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; uint32_t vc14 = abcd[0U]; uint32_t vd14 = abcd[1U]; - uint8_t *b14 = x + (uint32_t)56U; + uint8_t *b14 = x + 56U; uint32_t u13 = load32_le(b14); uint32_t xk13 = u13; uint32_t ti14 = _t[14U]; @@ -298,14 +290,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb14 + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << (uint32_t)17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> (uint32_t)15U); + << 17U + | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; uint32_t vc15 = abcd[3U]; uint32_t vd15 = abcd[0U]; - uint8_t *b15 = x + (uint32_t)60U; + uint8_t *b15 = x + 60U; uint32_t u14 = load32_le(b15); uint32_t xk14 = u14; uint32_t ti15 = _t[15U]; @@ -314,14 +306,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb15 + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << (uint32_t)22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> (uint32_t)10U); + << 22U + | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; uint32_t vc16 = abcd[2U]; uint32_t vd16 = abcd[3U]; - uint8_t *b16 = x + (uint32_t)4U; + uint8_t *b16 = x + 4U; uint32_t u15 = load32_le(b16); uint32_t xk15 = u15; uint32_t ti16 = _t[16U]; @@ -330,14 +322,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb16 + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << (uint32_t)5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> (uint32_t)27U); + << 5U + | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; uint32_t vc17 = abcd[1U]; uint32_t vd17 = abcd[2U]; - uint8_t *b17 = x + (uint32_t)24U; + uint8_t *b17 = x + 24U; uint32_t u16 = load32_le(b17); uint32_t xk16 = u16; uint32_t ti17 = _t[17U]; @@ -346,14 +338,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb17 + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << (uint32_t)9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> (uint32_t)23U); + << 9U + | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; uint32_t vc18 = abcd[0U]; uint32_t vd18 = abcd[1U]; - uint8_t *b18 = x + (uint32_t)44U; + uint8_t *b18 = x + 44U; uint32_t u17 = load32_le(b18); uint32_t xk17 = u17; uint32_t ti18 = _t[18U]; @@ -362,8 +354,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb18 + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << (uint32_t)14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> (uint32_t)18U); + << 14U + | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -378,14 +370,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb19 + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << (uint32_t)20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> (uint32_t)12U); + << 20U + | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; uint32_t vc20 = abcd[2U]; uint32_t vd20 = abcd[3U]; - uint8_t *b20 = x + (uint32_t)20U; + uint8_t *b20 = x + 20U; uint32_t u19 = load32_le(b20); uint32_t xk19 = u19; uint32_t ti20 = _t[20U]; @@ -394,14 +386,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb20 + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << (uint32_t)5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> (uint32_t)27U); + << 5U + | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; uint32_t vc21 = abcd[1U]; uint32_t vd21 = abcd[2U]; - uint8_t *b21 = x + (uint32_t)40U; + uint8_t *b21 = x + 40U; uint32_t u20 = load32_le(b21); uint32_t xk20 = u20; uint32_t ti21 = _t[21U]; @@ -410,14 +402,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb21 + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << (uint32_t)9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> (uint32_t)23U); + << 9U + | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; uint32_t vc22 = abcd[0U]; uint32_t vd22 = abcd[1U]; - uint8_t *b22 = x + (uint32_t)60U; + uint8_t *b22 = x + 60U; uint32_t u21 = load32_le(b22); uint32_t xk21 = u21; uint32_t ti22 = _t[22U]; @@ -426,14 +418,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb22 + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << (uint32_t)14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> (uint32_t)18U); + << 14U + | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; uint32_t vc23 = abcd[3U]; uint32_t vd23 = abcd[0U]; - uint8_t *b23 = x + (uint32_t)16U; + uint8_t *b23 = x + 16U; uint32_t u22 = load32_le(b23); uint32_t xk22 = u22; uint32_t ti23 = _t[23U]; @@ -442,14 +434,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb23 + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << (uint32_t)20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> (uint32_t)12U); + << 20U + | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; uint32_t vc24 = abcd[2U]; uint32_t vd24 = abcd[3U]; - uint8_t *b24 = x + (uint32_t)36U; + uint8_t *b24 = x + 36U; uint32_t u23 = load32_le(b24); uint32_t xk23 = u23; uint32_t ti24 = _t[24U]; @@ -458,14 +450,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb24 + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << (uint32_t)5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> (uint32_t)27U); + << 5U + | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; uint32_t vc25 = abcd[1U]; uint32_t vd25 = abcd[2U]; - uint8_t *b25 = x + (uint32_t)56U; + uint8_t *b25 = x + 56U; uint32_t u24 = load32_le(b25); uint32_t xk24 = u24; uint32_t ti25 = _t[25U]; @@ -474,14 +466,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb25 + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << (uint32_t)9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> (uint32_t)23U); + << 9U + | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; uint32_t vc26 = abcd[0U]; uint32_t vd26 = abcd[1U]; - uint8_t *b26 = x + (uint32_t)12U; + uint8_t *b26 = x + 12U; uint32_t u25 = load32_le(b26); uint32_t xk25 = u25; uint32_t ti26 = _t[26U]; @@ -490,14 +482,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb26 + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << (uint32_t)14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> (uint32_t)18U); + << 14U + | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; uint32_t vc27 = abcd[3U]; uint32_t vd27 = abcd[0U]; - uint8_t *b27 = x + (uint32_t)32U; + uint8_t *b27 = x + 32U; uint32_t u26 = load32_le(b27); uint32_t xk26 = u26; uint32_t ti27 = _t[27U]; @@ -506,14 +498,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb27 + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << (uint32_t)20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> (uint32_t)12U); + << 20U + | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; uint32_t vc28 = abcd[2U]; uint32_t vd28 = abcd[3U]; - uint8_t *b28 = x + (uint32_t)52U; + uint8_t *b28 = x + 52U; uint32_t u27 = load32_le(b28); uint32_t xk27 = u27; uint32_t ti28 = _t[28U]; @@ -522,14 +514,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb28 + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << (uint32_t)5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> (uint32_t)27U); + << 5U + | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; uint32_t vc29 = abcd[1U]; uint32_t vd29 = abcd[2U]; - uint8_t *b29 = x + (uint32_t)8U; + uint8_t *b29 = x + 8U; uint32_t u28 = load32_le(b29); uint32_t xk28 = u28; uint32_t ti29 = _t[29U]; @@ -538,14 +530,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb29 + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << (uint32_t)9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> (uint32_t)23U); + << 9U + | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; uint32_t vc30 = abcd[0U]; uint32_t vd30 = abcd[1U]; - uint8_t *b30 = x + (uint32_t)28U; + uint8_t *b30 = x + 28U; uint32_t u29 = load32_le(b30); uint32_t xk29 = u29; uint32_t ti30 = _t[30U]; @@ -554,14 +546,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb30 + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << (uint32_t)14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> (uint32_t)18U); + << 14U + | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; uint32_t vc31 = abcd[3U]; uint32_t vd31 = abcd[0U]; - uint8_t *b31 = x + (uint32_t)48U; + uint8_t *b31 = x + 48U; uint32_t u30 = load32_le(b31); uint32_t xk30 = u30; uint32_t ti31 = _t[31U]; @@ -570,14 +562,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb31 + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << (uint32_t)20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> (uint32_t)12U); + << 20U + | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; uint32_t vc32 = abcd[2U]; uint32_t vd32 = abcd[3U]; - uint8_t *b32 = x + (uint32_t)20U; + uint8_t *b32 = x + 20U; uint32_t u31 = load32_le(b32); uint32_t xk31 = u31; uint32_t ti32 = _t[32U]; @@ -586,14 +578,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb32 + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << (uint32_t)4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> (uint32_t)28U); + << 4U + | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; uint32_t vc33 = abcd[1U]; uint32_t vd33 = abcd[2U]; - uint8_t *b33 = x + (uint32_t)32U; + uint8_t *b33 = x + 32U; uint32_t u32 = load32_le(b33); uint32_t xk32 = u32; uint32_t ti33 = _t[33U]; @@ -602,14 +594,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb33 + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << (uint32_t)11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> (uint32_t)21U); + << 11U + | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; uint32_t vc34 = abcd[0U]; uint32_t vd34 = abcd[1U]; - uint8_t *b34 = x + (uint32_t)44U; + uint8_t *b34 = x + 44U; uint32_t u33 = load32_le(b34); uint32_t xk33 = u33; uint32_t ti34 = _t[34U]; @@ -618,14 +610,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb34 + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << (uint32_t)16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> (uint32_t)16U); + << 16U + | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; uint32_t vc35 = abcd[3U]; uint32_t vd35 = abcd[0U]; - uint8_t *b35 = x + (uint32_t)56U; + uint8_t *b35 = x + 56U; uint32_t u34 = load32_le(b35); uint32_t xk34 = u34; uint32_t ti35 = _t[35U]; @@ -634,14 +626,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb35 + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << (uint32_t)23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> (uint32_t)9U); + << 23U + | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; uint32_t vc36 = abcd[2U]; uint32_t vd36 = abcd[3U]; - uint8_t *b36 = x + (uint32_t)4U; + uint8_t *b36 = x + 4U; uint32_t u35 = load32_le(b36); uint32_t xk35 = u35; uint32_t ti36 = _t[36U]; @@ -650,14 +642,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb36 + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << (uint32_t)4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> (uint32_t)28U); + << 4U + | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; uint32_t vc37 = abcd[1U]; uint32_t vd37 = abcd[2U]; - uint8_t *b37 = x + (uint32_t)16U; + uint8_t *b37 = x + 16U; uint32_t u36 = load32_le(b37); uint32_t xk36 = u36; uint32_t ti37 = _t[37U]; @@ -666,14 +658,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb37 + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << (uint32_t)11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> (uint32_t)21U); + << 11U + | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; uint32_t vc38 = abcd[0U]; uint32_t vd38 = abcd[1U]; - uint8_t *b38 = x + (uint32_t)28U; + uint8_t *b38 = x + 28U; uint32_t u37 = load32_le(b38); uint32_t xk37 = u37; uint32_t ti38 = _t[38U]; @@ -682,14 +674,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb38 + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << (uint32_t)16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> (uint32_t)16U); + << 16U + | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; uint32_t vc39 = abcd[3U]; uint32_t vd39 = abcd[0U]; - uint8_t *b39 = x + (uint32_t)40U; + uint8_t *b39 = x + 40U; uint32_t u38 = load32_le(b39); uint32_t xk38 = u38; uint32_t ti39 = _t[39U]; @@ -698,14 +690,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb39 + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << (uint32_t)23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> (uint32_t)9U); + << 23U + | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; uint32_t vc40 = abcd[2U]; uint32_t vd40 = abcd[3U]; - uint8_t *b40 = x + (uint32_t)52U; + uint8_t *b40 = x + 52U; uint32_t u39 = load32_le(b40); uint32_t xk39 = u39; uint32_t ti40 = _t[40U]; @@ -714,8 +706,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb40 + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << (uint32_t)4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> (uint32_t)28U); + << 4U + | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -730,14 +722,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb41 + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << (uint32_t)11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> (uint32_t)21U); + << 11U + | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; uint32_t vc42 = abcd[0U]; uint32_t vd42 = abcd[1U]; - uint8_t *b42 = x + (uint32_t)12U; + uint8_t *b42 = x + 12U; uint32_t u41 = load32_le(b42); uint32_t xk41 = u41; uint32_t ti42 = _t[42U]; @@ -746,14 +738,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb42 + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << (uint32_t)16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> (uint32_t)16U); + << 16U + | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; uint32_t vc43 = abcd[3U]; uint32_t vd43 = abcd[0U]; - uint8_t *b43 = x + (uint32_t)24U; + uint8_t *b43 = x + 24U; uint32_t u42 = load32_le(b43); uint32_t xk42 = u42; uint32_t ti43 = _t[43U]; @@ -762,14 +754,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb43 + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << (uint32_t)23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> (uint32_t)9U); + << 23U + | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; uint32_t vc44 = abcd[2U]; uint32_t vd44 = abcd[3U]; - uint8_t *b44 = x + (uint32_t)36U; + uint8_t *b44 = x + 36U; uint32_t u43 = load32_le(b44); uint32_t xk43 = u43; uint32_t ti44 = _t[44U]; @@ -778,14 +770,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb44 + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << (uint32_t)4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> (uint32_t)28U); + << 4U + | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; uint32_t vc45 = abcd[1U]; uint32_t vd45 = abcd[2U]; - uint8_t *b45 = x + (uint32_t)48U; + uint8_t *b45 = x + 48U; uint32_t u44 = load32_le(b45); uint32_t xk44 = u44; uint32_t ti45 = _t[45U]; @@ -794,14 +786,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb45 + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << (uint32_t)11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> (uint32_t)21U); + << 11U + | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; uint32_t vc46 = abcd[0U]; uint32_t vd46 = abcd[1U]; - uint8_t *b46 = x + (uint32_t)60U; + uint8_t *b46 = x + 60U; uint32_t u45 = load32_le(b46); uint32_t xk45 = u45; uint32_t ti46 = _t[46U]; @@ -810,14 +802,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb46 + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << (uint32_t)16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> (uint32_t)16U); + << 16U + | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; uint32_t vc47 = abcd[3U]; uint32_t vd47 = abcd[0U]; - uint8_t *b47 = x + (uint32_t)8U; + uint8_t *b47 = x + 8U; uint32_t u46 = load32_le(b47); uint32_t xk46 = u46; uint32_t ti47 = _t[47U]; @@ -826,8 +818,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb47 + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << (uint32_t)23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> (uint32_t)9U); + << 23U + | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -842,14 +834,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb48 + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << (uint32_t)6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> (uint32_t)26U); + << 6U + | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; uint32_t vc49 = abcd[1U]; uint32_t vd49 = abcd[2U]; - uint8_t *b49 = x + (uint32_t)28U; + uint8_t *b49 = x + 28U; uint32_t u48 = load32_le(b49); uint32_t xk48 = u48; uint32_t ti49 = _t[49U]; @@ -858,14 +850,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb49 + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << (uint32_t)10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> (uint32_t)22U); + << 10U + | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; uint32_t vc50 = abcd[0U]; uint32_t vd50 = abcd[1U]; - uint8_t *b50 = x + (uint32_t)56U; + uint8_t *b50 = x + 56U; uint32_t u49 = load32_le(b50); uint32_t xk49 = u49; uint32_t ti50 = _t[50U]; @@ -874,14 +866,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb50 + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << (uint32_t)15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> (uint32_t)17U); + << 15U + | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; uint32_t vc51 = abcd[3U]; uint32_t vd51 = abcd[0U]; - uint8_t *b51 = x + (uint32_t)20U; + uint8_t *b51 = x + 20U; uint32_t u50 = load32_le(b51); uint32_t xk50 = u50; uint32_t ti51 = _t[51U]; @@ -890,14 +882,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb51 + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << (uint32_t)21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> (uint32_t)11U); + << 21U + | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; uint32_t vc52 = abcd[2U]; uint32_t vd52 = abcd[3U]; - uint8_t *b52 = x + (uint32_t)48U; + uint8_t *b52 = x + 48U; uint32_t u51 = load32_le(b52); uint32_t xk51 = u51; uint32_t ti52 = _t[52U]; @@ -906,14 +898,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb52 + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << (uint32_t)6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> (uint32_t)26U); + << 6U + | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; uint32_t vc53 = abcd[1U]; uint32_t vd53 = abcd[2U]; - uint8_t *b53 = x + (uint32_t)12U; + uint8_t *b53 = x + 12U; uint32_t u52 = load32_le(b53); uint32_t xk52 = u52; uint32_t ti53 = _t[53U]; @@ -922,14 +914,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb53 + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << (uint32_t)10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> (uint32_t)22U); + << 10U + | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; uint32_t vc54 = abcd[0U]; uint32_t vd54 = abcd[1U]; - uint8_t *b54 = x + (uint32_t)40U; + uint8_t *b54 = x + 40U; uint32_t u53 = load32_le(b54); uint32_t xk53 = u53; uint32_t ti54 = _t[54U]; @@ -938,14 +930,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb54 + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << (uint32_t)15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> (uint32_t)17U); + << 15U + | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; uint32_t vc55 = abcd[3U]; uint32_t vd55 = abcd[0U]; - uint8_t *b55 = x + (uint32_t)4U; + uint8_t *b55 = x + 4U; uint32_t u54 = load32_le(b55); uint32_t xk54 = u54; uint32_t ti55 = _t[55U]; @@ -954,14 +946,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb55 + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << (uint32_t)21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> (uint32_t)11U); + << 21U + | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; uint32_t vc56 = abcd[2U]; uint32_t vd56 = abcd[3U]; - uint8_t *b56 = x + (uint32_t)32U; + uint8_t *b56 = x + 32U; uint32_t u55 = load32_le(b56); uint32_t xk55 = u55; uint32_t ti56 = _t[56U]; @@ -970,14 +962,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb56 + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << (uint32_t)6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> (uint32_t)26U); + << 6U + | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; uint32_t vc57 = abcd[1U]; uint32_t vd57 = abcd[2U]; - uint8_t *b57 = x + (uint32_t)60U; + uint8_t *b57 = x + 60U; uint32_t u56 = load32_le(b57); uint32_t xk56 = u56; uint32_t ti57 = _t[57U]; @@ -986,14 +978,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb57 + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << (uint32_t)10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> (uint32_t)22U); + << 10U + | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; uint32_t vc58 = abcd[0U]; uint32_t vd58 = abcd[1U]; - uint8_t *b58 = x + (uint32_t)24U; + uint8_t *b58 = x + 24U; uint32_t u57 = load32_le(b58); uint32_t xk57 = u57; uint32_t ti58 = _t[58U]; @@ -1002,14 +994,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb58 + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << (uint32_t)15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> (uint32_t)17U); + << 15U + | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; uint32_t vc59 = abcd[3U]; uint32_t vd59 = abcd[0U]; - uint8_t *b59 = x + (uint32_t)52U; + uint8_t *b59 = x + 52U; uint32_t u58 = load32_le(b59); uint32_t xk58 = u58; uint32_t ti59 = _t[59U]; @@ -1018,14 +1010,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb59 + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << (uint32_t)21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> (uint32_t)11U); + << 21U + | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; uint32_t vc60 = abcd[2U]; uint32_t vd60 = abcd[3U]; - uint8_t *b60 = x + (uint32_t)16U; + uint8_t *b60 = x + 16U; uint32_t u59 = load32_le(b60); uint32_t xk59 = u59; uint32_t ti60 = _t[60U]; @@ -1034,14 +1026,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb60 + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << (uint32_t)6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> (uint32_t)26U); + << 6U + | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; uint32_t vc61 = abcd[1U]; uint32_t vd61 = abcd[2U]; - uint8_t *b61 = x + (uint32_t)44U; + uint8_t *b61 = x + 44U; uint32_t u60 = load32_le(b61); uint32_t xk60 = u60; uint32_t ti61 = _t[61U]; @@ -1050,14 +1042,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb61 + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << (uint32_t)10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> (uint32_t)22U); + << 10U + | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; uint32_t vc62 = abcd[0U]; uint32_t vd62 = abcd[1U]; - uint8_t *b62 = x + (uint32_t)8U; + uint8_t *b62 = x + 8U; uint32_t u61 = load32_le(b62); uint32_t xk61 = u61; uint32_t ti62 = _t[62U]; @@ -1066,14 +1058,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb62 + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << (uint32_t)15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> (uint32_t)17U); + << 15U + | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; uint32_t vc = abcd[3U]; uint32_t vd = abcd[0U]; - uint8_t *b63 = x + (uint32_t)36U; + uint8_t *b63 = x + 36U; uint32_t u62 = load32_le(b63); uint32_t xk62 = u62; uint32_t ti = _t[63U]; @@ -1082,8 +1074,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << (uint32_t)21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> (uint32_t)11U); + << 21U + | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1095,98 +1087,69 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) abcd[3U] = d + dd; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_le(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_le(dst3, len << 3U); } -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR4(i, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, - store32_le(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(dst + i * 4U, s[i]);); } -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_MD5_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_MD5_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -1195,75 +1158,75 @@ void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_MD5_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_MD5_legacy_finish(s, dst); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_MD5_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); return p; } -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1273,74 +1236,74 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1349,114 +1312,109 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[4U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)4U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 4U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_MD5_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_MD5_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_MD5_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_MD5_legacy_finish(tmp_block_state, dst); + Hacl_Hash_MD5_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_MD5_finish(tmp_block_state, output); } -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)4U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 4U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -1465,8 +1423,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_sta return p; } -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_MD5_legacy_hash(input, input_len, dst); + Hacl_Hash_MD5_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h index 13c19fd40f4d12..f69d6e5a81d63a 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.h +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_MD5_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_MD5_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void); -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 5ecb3c0b3a56e0..1a8b09b1711894 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -25,19 +25,14 @@ #include "internal/Hacl_Hash_SHA1.h" -static uint32_t -_h0[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; +static uint32_t _h0[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s) +void Hacl_Hash_SHA1_init(uint32_t *s) { - KRML_MAYBE_FOR5(i, (uint32_t)0U, (uint32_t)5U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *h, uint8_t *l) +static void update(uint32_t *h, uint8_t *l) { uint32_t ha = h[0U]; uint32_t hb = h[1U]; @@ -45,29 +40,26 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t hd = h[3U]; uint32_t he = h[4U]; uint32_t _w[80U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t v; - if (i < (uint32_t)16U) + if (i < 16U) { - uint8_t *b = l + i * (uint32_t)4U; + uint8_t *b = l + i * 4U; uint32_t u = load32_be(b); v = u; } else { - uint32_t wmit3 = _w[i - (uint32_t)3U]; - uint32_t wmit8 = _w[i - (uint32_t)8U]; - uint32_t wmit14 = _w[i - (uint32_t)14U]; - uint32_t wmit16 = _w[i - (uint32_t)16U]; - v = - (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) - << (uint32_t)1U - | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> (uint32_t)31U; + uint32_t wmit3 = _w[i - 3U]; + uint32_t wmit8 = _w[i - 8U]; + uint32_t wmit14 = _w[i - 14U]; + uint32_t wmit16 = _w[i - 16U]; + v = (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) << 1U | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> 31U; } _w[i] = v; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t _a = h[0U]; uint32_t _b = h[1U]; @@ -76,11 +68,11 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t _e = h[4U]; uint32_t wmit = _w[i]; uint32_t ite0; - if (i < (uint32_t)20U) + if (i < 20U) { ite0 = (_b & _c) ^ (~_b & _d); } - else if ((uint32_t)39U < i && i < (uint32_t)60U) + else if (39U < i && i < 60U) { ite0 = (_b & _c) ^ ((_b & _d) ^ (_c & _d)); } @@ -89,32 +81,32 @@ static void legacy_update(uint32_t *h, uint8_t *l) ite0 = _b ^ (_c ^ _d); } uint32_t ite; - if (i < (uint32_t)20U) + if (i < 20U) { - ite = (uint32_t)0x5a827999U; + ite = 0x5a827999U; } - else if (i < (uint32_t)40U) + else if (i < 40U) { - ite = (uint32_t)0x6ed9eba1U; + ite = 0x6ed9eba1U; } - else if (i < (uint32_t)60U) + else if (i < 60U) { - ite = (uint32_t)0x8f1bbcdcU; + ite = 0x8f1bbcdcU; } else { - ite = (uint32_t)0xca62c1d6U; + ite = 0xca62c1d6U; } - uint32_t _T = (_a << (uint32_t)5U | _a >> (uint32_t)27U) + ite0 + _e + ite + wmit; + uint32_t _T = (_a << 5U | _a >> 27U) + ite0 + _e + ite + wmit; h[0U] = _T; h[1U] = _a; - h[2U] = _b << (uint32_t)30U | _b >> (uint32_t)2U; + h[2U] = _b << 30U | _b >> 2U; h[3U] = _c; h[4U] = _d; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { - _w[i] = (uint32_t)0U; + _w[i] = 0U; } uint32_t sta = h[0U]; uint32_t stb = h[1U]; @@ -128,101 +120,69 @@ static void legacy_update(uint32_t *h, uint8_t *l) h[4U] = ste + he; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_be(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_be(dst3, len << 3U); } -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - store32_be(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, store32_be(dst + i * 4U, s[i]);); } -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_SHA1_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_SHA1_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -231,75 +191,75 @@ void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_SHA1_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_SHA1_legacy_finish(s, dst); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_SHA1_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); return p; } -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -309,74 +269,74 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -385,114 +345,109 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[5U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)5U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 5U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_SHA1_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_SHA1_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_SHA1_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_SHA1_legacy_finish(tmp_block_state, dst); + Hacl_Hash_SHA1_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_SHA1_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)5U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 5U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -501,8 +456,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_st return p; } -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_SHA1_legacy_hash(input, input_len, dst); + Hacl_Hash_SHA1_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h index dc50aa6f6d3902..ad1e8e72a739ec 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA1_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA1_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void); -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index 08e3f7edbf4ede..4b6af5fc78c680 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -27,14 +27,14 @@ -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h256[i]; + uint32_t x = Hacl_Hash_SHA2_h256[i]; os[i] = x;); } @@ -42,49 +42,49 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) { uint32_t hash_old[8U] = { 0U }; uint32_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint32_t)); + memcpy(hash_old, hash, 8U * sizeof (uint32_t)); uint8_t *b10 = b; uint32_t u = load32_be(b10); ws[0U] = u; - uint32_t u0 = load32_be(b10 + (uint32_t)4U); + uint32_t u0 = load32_be(b10 + 4U); ws[1U] = u0; - uint32_t u1 = load32_be(b10 + (uint32_t)8U); + uint32_t u1 = load32_be(b10 + 8U); ws[2U] = u1; - uint32_t u2 = load32_be(b10 + (uint32_t)12U); + uint32_t u2 = load32_be(b10 + 12U); ws[3U] = u2; - uint32_t u3 = load32_be(b10 + (uint32_t)16U); + uint32_t u3 = load32_be(b10 + 16U); ws[4U] = u3; - uint32_t u4 = load32_be(b10 + (uint32_t)20U); + uint32_t u4 = load32_be(b10 + 20U); ws[5U] = u4; - uint32_t u5 = load32_be(b10 + (uint32_t)24U); + uint32_t u5 = load32_be(b10 + 24U); ws[6U] = u5; - uint32_t u6 = load32_be(b10 + (uint32_t)28U); + uint32_t u6 = load32_be(b10 + 28U); ws[7U] = u6; - uint32_t u7 = load32_be(b10 + (uint32_t)32U); + uint32_t u7 = load32_be(b10 + 32U); ws[8U] = u7; - uint32_t u8 = load32_be(b10 + (uint32_t)36U); + uint32_t u8 = load32_be(b10 + 36U); ws[9U] = u8; - uint32_t u9 = load32_be(b10 + (uint32_t)40U); + uint32_t u9 = load32_be(b10 + 40U); ws[10U] = u9; - uint32_t u10 = load32_be(b10 + (uint32_t)44U); + uint32_t u10 = load32_be(b10 + 44U); ws[11U] = u10; - uint32_t u11 = load32_be(b10 + (uint32_t)48U); + uint32_t u11 = load32_be(b10 + 48U); ws[12U] = u11; - uint32_t u12 = load32_be(b10 + (uint32_t)52U); + uint32_t u12 = load32_be(b10 + 52U); ws[13U] = u12; - uint32_t u13 = load32_be(b10 + (uint32_t)56U); + uint32_t u13 = load32_be(b10 + 56U); ws[14U] = u13; - uint32_t u14 = load32_be(b10 + (uint32_t)60U); + uint32_t u14 = load32_be(b10 + 60U); ws[15U] = u14; KRML_MAYBE_FOR4(i0, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, + 0U, + 4U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint32_t k_t = Hacl_Impl_SHA2_Generic_k224_256[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint32_t k_t = Hacl_Hash_SHA2_k224_256[16U * i0 + i]; uint32_t ws_t = ws[i]; uint32_t a0 = hash[0U]; uint32_t b0 = hash[1U]; @@ -98,20 +98,13 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t t1 = h02 - + - ((e0 << (uint32_t)26U | e0 >> (uint32_t)6U) - ^ - ((e0 << (uint32_t)21U | e0 >> (uint32_t)11U) - ^ (e0 << (uint32_t)7U | e0 >> (uint32_t)25U))) + + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << (uint32_t)30U | a0 >> (uint32_t)2U) - ^ - ((a0 << (uint32_t)19U | a0 >> (uint32_t)13U) - ^ (a0 << (uint32_t)10U | a0 >> (uint32_t)22U))) + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; @@ -129,74 +122,63 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)3U) + if (i0 < 3U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint32_t t16 = ws[i]; - uint32_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint32_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint32_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint32_t - s1 = - (t2 << (uint32_t)15U | t2 >> (uint32_t)17U) - ^ ((t2 << (uint32_t)13U | t2 >> (uint32_t)19U) ^ t2 >> (uint32_t)10U); - uint32_t - s0 = - (t15 << (uint32_t)25U | t15 >> (uint32_t)7U) - ^ ((t15 << (uint32_t)14U | t15 >> (uint32_t)18U) ^ t15 >> (uint32_t)3U); + uint32_t t15 = ws[(i + 1U) % 16U]; + uint32_t t7 = ws[(i + 9U) % 16U]; + uint32_t t2 = ws[(i + 14U) % 16U]; + uint32_t s1 = (t2 << 15U | t2 >> 17U) ^ ((t2 << 13U | t2 >> 19U) ^ t2 >> 10U); + uint32_t s0 = (t15 << 25U | t15 >> 7U) ^ ((t15 << 14U | t15 >> 18U) ^ t15 >> 3U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; uint32_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - uint32_t blocks = len / (uint32_t)64U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 64U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)64U; + uint8_t *mb = b0 + i * 64U; sha256_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -) +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash) { uint32_t blocks; - if (len + (uint32_t)8U + (uint32_t)1U <= (uint32_t)64U) + if (len + 8U + 1U <= 64U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)64U; + uint32_t fin = blocks * 64U; uint8_t last[128U] = { 0U }; uint8_t totlen_buf[8U] = { 0U }; - uint64_t total_len_bits = totlen << (uint32_t)3U; + uint64_t total_len_bits = totlen << 3U; store64_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)8U, totlen_buf, (uint32_t)8U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 8U, totlen_buf, 8U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)64U; + uint8_t *last10 = last + 64U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -204,65 +186,56 @@ Hacl_SHA2_Scalar32_sha256_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha256_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha256_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)32U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 32U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h224[i]; + uint32_t x = Hacl_Hash_SHA2_h224[i]; os[i] = x;); } static inline void sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha256_update_nblocks(len, b, st); } -void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha256_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)28U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 28U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h512[i]; + uint64_t x = Hacl_Hash_SHA2_h512[i]; os[i] = x;); } @@ -270,49 +243,49 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) { uint64_t hash_old[8U] = { 0U }; uint64_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint64_t)); + memcpy(hash_old, hash, 8U * sizeof (uint64_t)); uint8_t *b10 = b; uint64_t u = load64_be(b10); ws[0U] = u; - uint64_t u0 = load64_be(b10 + (uint32_t)8U); + uint64_t u0 = load64_be(b10 + 8U); ws[1U] = u0; - uint64_t u1 = load64_be(b10 + (uint32_t)16U); + uint64_t u1 = load64_be(b10 + 16U); ws[2U] = u1; - uint64_t u2 = load64_be(b10 + (uint32_t)24U); + uint64_t u2 = load64_be(b10 + 24U); ws[3U] = u2; - uint64_t u3 = load64_be(b10 + (uint32_t)32U); + uint64_t u3 = load64_be(b10 + 32U); ws[4U] = u3; - uint64_t u4 = load64_be(b10 + (uint32_t)40U); + uint64_t u4 = load64_be(b10 + 40U); ws[5U] = u4; - uint64_t u5 = load64_be(b10 + (uint32_t)48U); + uint64_t u5 = load64_be(b10 + 48U); ws[6U] = u5; - uint64_t u6 = load64_be(b10 + (uint32_t)56U); + uint64_t u6 = load64_be(b10 + 56U); ws[7U] = u6; - uint64_t u7 = load64_be(b10 + (uint32_t)64U); + uint64_t u7 = load64_be(b10 + 64U); ws[8U] = u7; - uint64_t u8 = load64_be(b10 + (uint32_t)72U); + uint64_t u8 = load64_be(b10 + 72U); ws[9U] = u8; - uint64_t u9 = load64_be(b10 + (uint32_t)80U); + uint64_t u9 = load64_be(b10 + 80U); ws[10U] = u9; - uint64_t u10 = load64_be(b10 + (uint32_t)88U); + uint64_t u10 = load64_be(b10 + 88U); ws[11U] = u10; - uint64_t u11 = load64_be(b10 + (uint32_t)96U); + uint64_t u11 = load64_be(b10 + 96U); ws[12U] = u11; - uint64_t u12 = load64_be(b10 + (uint32_t)104U); + uint64_t u12 = load64_be(b10 + 104U); ws[13U] = u12; - uint64_t u13 = load64_be(b10 + (uint32_t)112U); + uint64_t u13 = load64_be(b10 + 112U); ws[14U] = u13; - uint64_t u14 = load64_be(b10 + (uint32_t)120U); + uint64_t u14 = load64_be(b10 + 120U); ws[15U] = u14; KRML_MAYBE_FOR5(i0, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, + 0U, + 5U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint64_t k_t = Hacl_Impl_SHA2_Generic_k384_512[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint64_t k_t = Hacl_Hash_SHA2_k384_512[16U * i0 + i]; uint64_t ws_t = ws[i]; uint64_t a0 = hash[0U]; uint64_t b0 = hash[1U]; @@ -326,20 +299,13 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t t1 = h02 - + - ((e0 << (uint32_t)50U | e0 >> (uint32_t)14U) - ^ - ((e0 << (uint32_t)46U | e0 >> (uint32_t)18U) - ^ (e0 << (uint32_t)23U | e0 >> (uint32_t)41U))) + + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << (uint32_t)36U | a0 >> (uint32_t)28U) - ^ - ((a0 << (uint32_t)30U | a0 >> (uint32_t)34U) - ^ (a0 << (uint32_t)25U | a0 >> (uint32_t)39U))) + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; @@ -357,48 +323,42 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)4U) + if (i0 < 4U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint64_t t16 = ws[i]; - uint64_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint64_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint64_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint64_t - s1 = - (t2 << (uint32_t)45U | t2 >> (uint32_t)19U) - ^ ((t2 << (uint32_t)3U | t2 >> (uint32_t)61U) ^ t2 >> (uint32_t)6U); - uint64_t - s0 = - (t15 << (uint32_t)63U | t15 >> (uint32_t)1U) - ^ ((t15 << (uint32_t)56U | t15 >> (uint32_t)8U) ^ t15 >> (uint32_t)7U); + uint64_t t15 = ws[(i + 1U) % 16U]; + uint64_t t7 = ws[(i + 9U) % 16U]; + uint64_t t2 = ws[(i + 14U) % 16U]; + uint64_t s1 = (t2 << 45U | t2 >> 19U) ^ ((t2 << 3U | t2 >> 61U) ^ t2 >> 6U); + uint64_t s0 = (t15 << 63U | t15 >> 1U) ^ ((t15 << 56U | t15 >> 8U) ^ t15 >> 7U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; uint64_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - uint32_t blocks = len / (uint32_t)128U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 128U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)128U; + uint8_t *mb = b0 + i * 128U; sha512_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, @@ -406,25 +366,25 @@ Hacl_SHA2_Scalar32_sha512_update_last( ) { uint32_t blocks; - if (len + (uint32_t)16U + (uint32_t)1U <= (uint32_t)128U) + if (len + 16U + 1U <= 128U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)128U; + uint32_t fin = blocks * 128U; uint8_t last[256U] = { 0U }; uint8_t totlen_buf[16U] = { 0U }; - FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, (uint32_t)3U); + FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, 3U); store128_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)16U, totlen_buf, (uint32_t)16U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 16U, totlen_buf, 16U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)128U; + uint8_t *last10 = last + 128U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -432,76 +392,68 @@ Hacl_SHA2_Scalar32_sha512_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha512_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha512_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)64U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 64U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h384[i]; + uint64_t x = Hacl_Hash_SHA2_h384[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - Hacl_SHA2_Scalar32_sha512_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha512_update_nblocks(len, b, st); } void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ) { - Hacl_SHA2_Scalar32_sha512_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha512_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)48U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 48U * sizeof (uint8_t)); } /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); return p; } @@ -511,16 +463,16 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -532,54 +484,54 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -589,76 +541,74 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -667,55 +617,48 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -725,209 +668,203 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha256_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha256_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha256_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha256_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha256_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha256_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha256_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha256_finish(st, rb); + Hacl_Hash_SHA2_sha256_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha256_finish(st, rb); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - sha224_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + sha224_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha224_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha224_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha224_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha224_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p) +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_SHA2_free_256(p); + Hacl_Hash_SHA2_free_256(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha224_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha224_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; sha224_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha224_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha224_finish(st, rb); + Hacl_Hash_SHA2_sha224_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha224_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); return p; } @@ -937,16 +874,16 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s0; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint64_t)); Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_64 @@ -955,54 +892,54 @@ Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state return p; } -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) +update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_64 s = *p; + Hacl_Streaming_MD_state_64 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) + if ((uint64_t)chunk_len > 18446744073709551615ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)128U; + sz = 128U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + sz = (uint32_t)(total_len % (uint64_t)128U); } - if (len <= (uint32_t)128U - sz) + if (chunk_len <= 128U - sz) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1012,76 +949,74 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)128U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); } - uint32_t n_blocks = (len - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)128U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_64 s1 = *p; + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)128U; + sz10 = 128U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)128U); + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1090,55 +1025,48 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_64 s10 = *p; + Hacl_Streaming_MD_state_64 s10 = *state; uint64_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)128U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -1148,198 +1076,198 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha512_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha512_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha512_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha512_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha512_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha512_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha512_finish(st, rb); + Hacl_Hash_SHA2_sha512_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha512_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha384_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha384_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha384_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha384_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p) +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_SHA2_free_512(p); + Hacl_Hash_SHA2_free_512(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha384_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha384_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha384_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha384_finish(st, rb); + Hacl_Hash_SHA2_sha384_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha384_finish(st, rb); } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.h b/Modules/_hacl/Hacl_Hash_SHA2.h index a0e731094dfaa5..d8204b504baf82 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/Hacl_Hash_SHA2.h @@ -39,19 +39,19 @@ extern "C" { #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_224; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_224; -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_256; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_256; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_384; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_384; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_512; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_512; /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -59,73 +59,73 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state); /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output); /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void); -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p); +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -133,68 +133,68 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state); -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output); /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void); -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p); +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index b3febdfeb2b221..4f502866fe06bb 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -31,27 +31,27 @@ static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)144U; + return 144U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)136U; + return 136U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)104U; + return 104U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)72U; + return 72U; } case Spec_Hash_Definitions_Shake128: { - return (uint32_t)168U; + return 168U; } case Spec_Hash_Definitions_Shake256: { - return (uint32_t)136U; + return 136U; } default: { @@ -67,19 +67,19 @@ static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)28U; + return 28U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)32U; + return 32U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)48U; + return 48U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)64U; + return 64U; } default: { @@ -97,10 +97,10 @@ Hacl_Hash_SHA3_update_multi_sha3( uint32_t n_blocks ) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { uint8_t *block = blocks + i * block_len(a); - Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + Hacl_Hash_SHA3_absorb_inner(block_len(a), block, s); } } @@ -115,139 +115,139 @@ Hacl_Hash_SHA3_update_last_sha3( uint8_t suffix; if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) { - suffix = (uint8_t)0x1fU; + suffix = 0x1fU; } else { - suffix = (uint8_t)0x06U; + suffix = 0x06U; } uint32_t len = block_len(a); if (input_len == len) { - Hacl_Impl_SHA3_absorb_inner(len, input, s); - uint8_t *uu____0 = input + input_len; + Hacl_Hash_SHA3_absorb_inner(len, input, s); uint8_t lastBlock_[200U] = { 0U }; uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); + memcpy(lastBlock, input + input_len, 0U * sizeof (uint8_t)); lastBlock[0U] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) + Hacl_Hash_SHA3_loadState(len, lastBlock, s); + if (!(((uint32_t)suffix & 0x80U) == 0U) && 0U == len - 1U) { - Hacl_Impl_SHA3_state_permute(s); + Hacl_Hash_SHA3_state_permute(s); } uint8_t nextBlock_[200U] = { 0U }; uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + nextBlock[len - 1U] = 0x80U; + Hacl_Hash_SHA3_loadState(len, nextBlock, s); + Hacl_Hash_SHA3_state_permute(s); return; } uint8_t lastBlock_[200U] = { 0U }; uint8_t *lastBlock = lastBlock_; memcpy(lastBlock, input, input_len * sizeof (uint8_t)); lastBlock[input_len] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + Hacl_Hash_SHA3_loadState(len, lastBlock, s); + if (!(((uint32_t)suffix & 0x80U) == 0U) && input_len == len - 1U) { - Hacl_Impl_SHA3_state_permute(s); + Hacl_Hash_SHA3_state_permute(s); } uint8_t nextBlock_[200U] = { 0U }; uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + nextBlock[len - 1U] = 0x80U; + Hacl_Hash_SHA3_loadState(len, nextBlock, s); + Hacl_Hash_SHA3_state_permute(s); } typedef struct hash_buf2_s { - Hacl_Streaming_Keccak_hash_buf fst; - Hacl_Streaming_Keccak_hash_buf snd; + Hacl_Hash_SHA3_hash_buf fst; + Hacl_Hash_SHA3_hash_buf snd; } hash_buf2; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s) { - Hacl_Streaming_Keccak_state scrut = *s; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + Hacl_Hash_SHA3_hash_buf block_state = (*s).block_state; return block_state.fst; } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a) { KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; - Hacl_Streaming_Keccak_state - s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_state_t + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)0U }; + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); + memset(s1, 0U, 25U * sizeof (uint64_t)); return p; } -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - KRML_HOST_FREE(s1); - KRML_HOST_FREE(buf); + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + uint64_t *s = block_state.snd; KRML_HOST_FREE(s); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut0 = *s0; - Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state0 = scrut0.block_state; uint8_t *buf0 = scrut0.buf; uint64_t total_len0 = scrut0.total_len; Spec_Hash_Definitions_hash_alg i = block_state0.fst; KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = i, .snd = buf }; hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; return p; } -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + KRML_MAYBE_UNUSED_VAR(i); + uint64_t *s = block_state.snd; + memset(s, 0U, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_Keccak_state s = *p; - Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + Hacl_Hash_SHA3_state_t s = *state; + Hacl_Hash_SHA3_hash_buf block_state = s.block_state; uint64_t total_len = s.total_len; Spec_Hash_Definitions_hash_alg i = block_state.fst; - if ((uint64_t)len > (uint64_t)0xFFFFFFFFFFFFFFFFU - total_len) + if ((uint64_t)chunk_len > 0xFFFFFFFFFFFFFFFFULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(i) == 0ULL && total_len > 0ULL) { sz = block_len(i); } @@ -255,14 +255,14 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz = (uint32_t)(total_len % (uint64_t)block_len(i)); } - if (len <= block_len(i) - sz) + if (chunk_len <= block_len(i) - sz) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -271,26 +271,20 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = - ( - (Hacl_Streaming_Keccak_state){ - .block_state = block_state1, - .buf = buf, - .total_len = total_len2 - } - ); + ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -298,52 +292,52 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); } uint32_t ite; - if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)block_len(i) == 0ULL && (uint64_t)chunk_len > 0ULL) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { uint32_t diff = block_len(i) - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)block_len(i) == 0ULL && total_len10 > 0ULL) { sz10 = block_len(i); } @@ -352,23 +346,23 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_Keccak_state s10 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + Hacl_Hash_SHA3_state_t s10 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -376,7 +370,7 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; @@ -385,35 +379,35 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint uint32_t ite; if ( - (uint64_t)(len - diff) + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U + == 0ULL + && (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - diff - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -421,19 +415,19 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint } static void -finish_( +digest_( Spec_Hash_Definitions_hash_alg a, - Hacl_Streaming_Keccak_state *p, - uint8_t *dst, + Hacl_Hash_SHA3_state_t *state, + uint8_t *output, uint32_t l ) { - Hacl_Streaming_Keccak_state scrut0 = *p; - Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state = scrut0.block_state; uint8_t *buf_ = scrut0.buf; uint64_t total_len = scrut0.total_len; uint32_t r; - if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(a) == 0ULL && total_len > 0ULL) { r = block_len(a); } @@ -443,25 +437,25 @@ finish_( } uint8_t *buf_1 = buf_; uint64_t buf[25U] = { 0U }; - Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_hash_buf tmp_block_state = { .fst = a, .snd = buf }; hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - uint32_t ite0; - if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + uint32_t ite; + if (r % block_len(a) == 0U && r > 0U) { - ite0 = block_len(a); + ite = block_len(a); } else { - ite0 = r % block_len(a); + ite = r % block_len(a); } - uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; uint64_t *s0 = tmp_block_state.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, 0U / block_len(a1)); Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; uint64_t *s1 = tmp_block_state.snd; Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); @@ -469,267 +463,182 @@ finish_( uint64_t *s = tmp_block_state.snd; if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) { - uint32_t ite; - if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) - { - ite = l; - } - else - { - ite = hash_len(a11); - } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + Hacl_Hash_SHA3_squeeze0(s, block_len(a11), l, output); return; } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); + Hacl_Hash_SHA3_squeeze0(s, block_len(a11), hash_len(a11), output); } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(state); if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) { return Hacl_Streaming_Types_InvalidAlgorithm; } - finish_(a1, s, dst, hash_len(a1)); + digest_(a1, state, output, hash_len(a1)); return Hacl_Streaming_Types_Success; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) { return Hacl_Streaming_Types_InvalidAlgorithm; } - if (l == (uint32_t)0U) + if (l == 0U) { return Hacl_Streaming_Types_InvalidLength; } - finish_(a1, s, dst, l); + digest_(a1, s, dst, l); return Hacl_Streaming_Types_Success; } -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return block_len(a1); } -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return hash_len(a1); } -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Hash_SHA3_get_alg(s); return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; } void -Hacl_SHA3_shake128_hacl( +Hacl_Hash_SHA3_shake128_hacl( uint32_t inputByteLen, uint8_t *input, uint32_t outputByteLen, uint8_t *output ) { - Hacl_Impl_SHA3_keccak((uint32_t)1344U, - (uint32_t)256U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + Hacl_Hash_SHA3_keccak(1344U, 256U, inputByteLen, input, 0x1FU, outputByteLen, output); } void -Hacl_SHA3_shake256_hacl( +Hacl_Hash_SHA3_shake256_hacl( uint32_t inputByteLen, uint8_t *input, uint32_t outputByteLen, uint8_t *output ) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + Hacl_Hash_SHA3_keccak(1088U, 512U, inputByteLen, input, 0x1FU, outputByteLen, output); } -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Impl_SHA3_keccak((uint32_t)1152U, - (uint32_t)448U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)28U, - output); + Hacl_Hash_SHA3_keccak(1152U, 448U, input_len, input, 0x06U, 28U, output); } -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)32U, - output); + Hacl_Hash_SHA3_keccak(1088U, 512U, input_len, input, 0x06U, 32U, output); } -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Impl_SHA3_keccak((uint32_t)832U, - (uint32_t)768U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)48U, - output); + Hacl_Hash_SHA3_keccak(832U, 768U, input_len, input, 0x06U, 48U, output); } -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Impl_SHA3_keccak((uint32_t)576U, - (uint32_t)1024U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)64U, - output); + Hacl_Hash_SHA3_keccak(576U, 1024U, input_len, input, 0x06U, 64U, output); } static const uint32_t keccak_rotc[24U] = { - (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, - (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, - (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, - (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U + 1U, 3U, 6U, 10U, 15U, 21U, 28U, 36U, 45U, 55U, 2U, 14U, 27U, 41U, 56U, 8U, 25U, 43U, 62U, 18U, + 39U, 61U, 20U, 44U }; static const uint32_t keccak_piln[24U] = { - (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, - (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, - (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, - (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U + 10U, 7U, 11U, 17U, 18U, 3U, 5U, 16U, 8U, 21U, 24U, 4U, 15U, 23U, 19U, 13U, 12U, 2U, 20U, 14U, + 22U, 9U, 6U, 1U }; static const uint64_t keccak_rndc[24U] = { - (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, - (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, - (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, - (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, - (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, - (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, - (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, - (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL }; -void Hacl_Impl_SHA3_state_permute(uint64_t *s) +void Hacl_Hash_SHA3_state_permute(uint64_t *s) { - for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + for (uint32_t i0 = 0U; i0 < 24U; i0++) { uint64_t _C[5U] = { 0U }; KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - _C[i] = - s[i - + (uint32_t)0U] - ^ - (s[i - + (uint32_t)5U] - ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); KRML_MAYBE_FOR5(i1, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; - uint64_t - _D = - _C[(i1 + (uint32_t)4U) - % (uint32_t)5U] - ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); uint64_t x = s[1U]; uint64_t current = x; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + for (uint32_t i = 0U; i < 24U; i++) { uint32_t _Y = keccak_piln[i]; uint32_t r = keccak_rotc[i]; uint64_t temp = s[_Y]; uint64_t uu____1 = current; - s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); + s[_Y] = uu____1 << r | uu____1 >> (64U - r); current = temp; } KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t - v0 = - s[(uint32_t)0U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); - uint64_t - v1 = - s[(uint32_t)1U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); - uint64_t - v2 = - s[(uint32_t)2U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); - uint64_t - v3 = - s[(uint32_t)3U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); - uint64_t - v4 = - s[(uint32_t)4U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); - s[(uint32_t)0U + (uint32_t)5U * i] = v0; - s[(uint32_t)1U + (uint32_t)5U * i] = v1; - s[(uint32_t)2U + (uint32_t)5U * i] = v2; - s[(uint32_t)3U + (uint32_t)5U * i] = v3; - s[(uint32_t)4U + (uint32_t)5U * i] = v4;); + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); uint64_t c = keccak_rndc[i0]; s[0U] = s[0U] ^ c; } } -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +void Hacl_Hash_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) { uint8_t block[200U] = { 0U }; memcpy(block, input, rateInBytes * sizeof (uint8_t)); - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + for (uint32_t i = 0U; i < 25U; i++) { - uint64_t u = load64_le(block + i * (uint32_t)8U); + uint64_t u = load64_le(block + i * 8U); uint64_t x = u; s[i] = s[i] ^ x; } @@ -738,18 +647,18 @@ void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) { uint8_t block[200U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + for (uint32_t i = 0U; i < 25U; i++) { uint64_t sj = s[i]; - store64_le(block + i * (uint32_t)8U, sj); + store64_le(block + i * 8U, sj); } memcpy(res, block, rateInBytes * sizeof (uint8_t)); } -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +void Hacl_Hash_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) { - Hacl_Impl_SHA3_loadState(rateInBytes, block, s); - Hacl_Impl_SHA3_state_permute(s); + Hacl_Hash_SHA3_loadState(rateInBytes, block, s); + Hacl_Hash_SHA3_state_permute(s); } static void @@ -763,30 +672,30 @@ absorb( { uint32_t n_blocks = inputByteLen / rateInBytes; uint32_t rem = inputByteLen % rateInBytes; - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { uint8_t *block = input + i * rateInBytes; - Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); + Hacl_Hash_SHA3_absorb_inner(rateInBytes, block, s); } uint8_t *last = input + n_blocks * rateInBytes; uint8_t lastBlock_[200U] = { 0U }; uint8_t *lastBlock = lastBlock_; memcpy(lastBlock, last, rem * sizeof (uint8_t)); lastBlock[rem] = delimitedSuffix; - Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); - if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) + Hacl_Hash_SHA3_loadState(rateInBytes, lastBlock, s); + if (!(((uint32_t)delimitedSuffix & 0x80U) == 0U) && rem == rateInBytes - 1U) { - Hacl_Impl_SHA3_state_permute(s); + Hacl_Hash_SHA3_state_permute(s); } uint8_t nextBlock_[200U] = { 0U }; uint8_t *nextBlock = nextBlock_; - nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + nextBlock[rateInBytes - 1U] = 0x80U; + Hacl_Hash_SHA3_loadState(rateInBytes, nextBlock, s); + Hacl_Hash_SHA3_state_permute(s); } void -Hacl_Impl_SHA3_squeeze( +Hacl_Hash_SHA3_squeeze0( uint64_t *s, uint32_t rateInBytes, uint32_t outputByteLen, @@ -797,16 +706,16 @@ Hacl_Impl_SHA3_squeeze( uint32_t remOut = outputByteLen % rateInBytes; uint8_t *last = output + outputByteLen - remOut; uint8_t *blocks = output; - for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + for (uint32_t i = 0U; i < outBlocks; i++) { storeState(rateInBytes, s, blocks + i * rateInBytes); - Hacl_Impl_SHA3_state_permute(s); + Hacl_Hash_SHA3_state_permute(s); } storeState(remOut, s, last); } void -Hacl_Impl_SHA3_keccak( +Hacl_Hash_SHA3_keccak( uint32_t rate, uint32_t capacity, uint32_t inputByteLen, @@ -816,9 +725,10 @@ Hacl_Impl_SHA3_keccak( uint8_t *output ) { - uint32_t rateInBytes = rate / (uint32_t)8U; + KRML_MAYBE_UNUSED_VAR(capacity); + uint32_t rateInBytes = rate / 8U; uint64_t s[25U] = { 0U }; absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); - Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); + Hacl_Hash_SHA3_squeeze0(s, rateInBytes, outputByteLen, output); } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h index 681b6af4a80e77..678e9f2fbe15e8 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -31,54 +31,55 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef struct Hacl_Streaming_Keccak_hash_buf_s +typedef struct Hacl_Hash_SHA3_hash_buf_s { Spec_Hash_Definitions_hash_alg fst; uint64_t *snd; } -Hacl_Streaming_Keccak_hash_buf; +Hacl_Hash_SHA3_hash_buf; -typedef struct Hacl_Streaming_Keccak_state_s +typedef struct Hacl_Hash_SHA3_state_t_s { - Hacl_Streaming_Keccak_hash_buf block_state; + Hacl_Hash_SHA3_hash_buf block_state; uint8_t *buf; uint64_t total_len; } -Hacl_Streaming_Keccak_state; +Hacl_Hash_SHA3_state_t; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a); -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state); -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l); -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s); -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s); -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s); void -Hacl_SHA3_shake128_hacl( +Hacl_Hash_SHA3_shake128_hacl( uint32_t inputByteLen, uint8_t *input, uint32_t outputByteLen, @@ -86,25 +87,25 @@ Hacl_SHA3_shake128_hacl( ); void -Hacl_SHA3_shake256_hacl( +Hacl_Hash_SHA3_shake256_hacl( uint32_t inputByteLen, uint8_t *input, uint32_t outputByteLen, uint8_t *output ); -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t input_len); -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t input_len); -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t input_len); -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t input_len); -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); +void Hacl_Hash_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); void -Hacl_Impl_SHA3_squeeze( +Hacl_Hash_SHA3_squeeze0( uint64_t *s, uint32_t rateInBytes, uint32_t outputByteLen, @@ -112,7 +113,7 @@ Hacl_Impl_SHA3_squeeze( ); void -Hacl_Impl_SHA3_keccak( +Hacl_Hash_SHA3_keccak( uint32_t rate, uint32_t capacity, uint32_t inputByteLen, diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index 3d36d440735530..bdf25898f2bc25 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -15,7 +15,7 @@ static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) { - return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; + return (a ^ ((a ^ b) | ((a - b) ^ b))) >> 63U; } static inline uint64_t FStar_UInt128_carry(uint64_t a, uint64_t b) @@ -118,7 +118,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_lognot(FStar_UInt128_uint128 a return lit; } -static uint32_t FStar_UInt128_u32_64 = (uint32_t)64U; +static uint32_t FStar_UInt128_u32_64 = 64U; static inline uint64_t FStar_UInt128_add_u64_shift_left(uint64_t hi, uint64_t lo, uint32_t s) { @@ -134,7 +134,7 @@ FStar_UInt128_add_u64_shift_left_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -151,7 +151,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; - lit.low = (uint64_t)0U; + lit.low = 0ULL; lit.high = a.low << (s - FStar_UInt128_u32_64); return lit; } @@ -183,7 +183,7 @@ FStar_UInt128_add_u64_shift_right_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_right_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -201,7 +201,7 @@ FStar_UInt128_shift_right_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; lit.low = a.high >> (s - FStar_UInt128_u32_64); - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -269,7 +269,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_uint64_to_uint128(uint64_t a) { FStar_UInt128_uint128 lit; lit.low = a; - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -280,10 +280,10 @@ static inline uint64_t FStar_UInt128_uint128_to_uint64(FStar_UInt128_uint128 a) static inline uint64_t FStar_UInt128_u64_mod_32(uint64_t a) { - return a & (uint64_t)0xffffffffU; + return a & 0xffffffffULL; } -static uint32_t FStar_UInt128_u32_32 = (uint32_t)32U; +static uint32_t FStar_UInt128_u32_32 = 32U; static inline uint64_t FStar_UInt128_u32_combine(uint64_t hi, uint64_t lo) { diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index a56c7d613498b7..1bdec972a2f249 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -14,16 +14,16 @@ #include "krml/types.h" #include "krml/internal/target.h" -static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { uint64_t x = a ^ b; - uint64_t minus_x = ~x + (uint64_t)1U; + uint64_t minus_x = ~x + 1ULL; uint64_t x_or_minus_x = x | minus_x; - uint64_t xnx = x_or_minus_x >> (uint32_t)63U; - return xnx - (uint64_t)1U; + uint64_t xnx = x_or_minus_x >> 63U; + return xnx - 1ULL; } -static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) { uint64_t x = a; uint64_t y = b; @@ -32,20 +32,20 @@ static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) uint64_t x_sub_y_xor_y = x_sub_y ^ y; uint64_t q = x_xor_y | x_sub_y_xor_y; uint64_t x_xor_q = x ^ q; - uint64_t x_xor_q_ = x_xor_q >> (uint32_t)63U; - return x_xor_q_ - (uint64_t)1U; + uint64_t x_xor_q_ = x_xor_q >> 63U; + return x_xor_q_ - 1ULL; } -static inline uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) { uint32_t x = a ^ b; - uint32_t minus_x = ~x + (uint32_t)1U; + uint32_t minus_x = ~x + 1U; uint32_t x_or_minus_x = x | minus_x; - uint32_t xnx = x_or_minus_x >> (uint32_t)31U; - return xnx - (uint32_t)1U; + uint32_t xnx = x_or_minus_x >> 31U; + return xnx - 1U; } -static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) { uint32_t x = a; uint32_t y = b; @@ -54,52 +54,52 @@ static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) uint32_t x_sub_y_xor_y = x_sub_y ^ y; uint32_t q = x_xor_y | x_sub_y_xor_y; uint32_t x_xor_q = x ^ q; - uint32_t x_xor_q_ = x_xor_q >> (uint32_t)31U; - return x_xor_q_ - (uint32_t)1U; + uint32_t x_xor_q_ = x_xor_q >> 31U; + return x_xor_q_ - 1U; } -static inline uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) { - uint16_t x = a ^ b; - uint16_t minus_x = ~x + (uint16_t)1U; - uint16_t x_or_minus_x = x | minus_x; - uint16_t xnx = x_or_minus_x >> (uint32_t)15U; - return xnx - (uint16_t)1U; + uint16_t x = (uint32_t)a ^ (uint32_t)b; + uint16_t minus_x = (uint32_t)~x + 1U; + uint16_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint16_t xnx = (uint32_t)x_or_minus_x >> 15U; + return (uint32_t)xnx - 1U; } -static inline uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) { uint16_t x = a; uint16_t y = b; - uint16_t x_xor_y = x ^ y; - uint16_t x_sub_y = x - y; - uint16_t x_sub_y_xor_y = x_sub_y ^ y; - uint16_t q = x_xor_y | x_sub_y_xor_y; - uint16_t x_xor_q = x ^ q; - uint16_t x_xor_q_ = x_xor_q >> (uint32_t)15U; - return x_xor_q_ - (uint16_t)1U; + uint16_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint16_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint16_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint16_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint16_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint16_t x_xor_q_ = (uint32_t)x_xor_q >> 15U; + return (uint32_t)x_xor_q_ - 1U; } -static inline uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) { - uint8_t x = a ^ b; - uint8_t minus_x = ~x + (uint8_t)1U; - uint8_t x_or_minus_x = x | minus_x; - uint8_t xnx = x_or_minus_x >> (uint32_t)7U; - return xnx - (uint8_t)1U; + uint8_t x = (uint32_t)a ^ (uint32_t)b; + uint8_t minus_x = (uint32_t)~x + 1U; + uint8_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint8_t xnx = (uint32_t)x_or_minus_x >> 7U; + return (uint32_t)xnx - 1U; } -static inline uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) { uint8_t x = a; uint8_t y = b; - uint8_t x_xor_y = x ^ y; - uint8_t x_sub_y = x - y; - uint8_t x_sub_y_xor_y = x_sub_y ^ y; - uint8_t q = x_xor_y | x_sub_y_xor_y; - uint8_t x_xor_q = x ^ q; - uint8_t x_xor_q_ = x_xor_q >> (uint32_t)7U; - return x_xor_q_ - (uint8_t)1U; + uint8_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint8_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint8_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint8_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint8_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint8_t x_xor_q_ = (uint32_t)x_xor_q >> 7U; + return (uint32_t)x_xor_q_ - 1U; } diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index 5a2f94eb2ec8da..c7fcc0151e6f10 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -4,13 +4,13 @@ #ifndef __KRML_TARGET_H #define __KRML_TARGET_H -#include -#include -#include -#include +#include #include #include -#include +#include +#include +#include +#include /* Since KaRaMeL emits the inline keyword unconditionally, we follow the * guidelines at https://gcc.gnu.org/onlinedocs/gcc/Inline.html and make this @@ -57,6 +57,31 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +#ifndef KRML_MAYBE_UNUSED_VAR +# define KRML_MAYBE_UNUSED_VAR(x) KRML_HOST_IGNORE(x) +#endif + +#ifndef KRML_MAYBE_UNUSED +# if defined(__GNUC__) +# define KRML_MAYBE_UNUSED __attribute__((unused)) +# else +# define KRML_MAYBE_UNUSED +# endif +#endif + +#ifndef KRML_NOINLINE +# if defined(_MSC_VER) +# define KRML_NOINLINE __declspec(noinline) +# elif defined (__GNUC__) +# define KRML_NOINLINE __attribute__((noinline,unused)) +# else +# define KRML_NOINLINE +# warning "The KRML_NOINLINE macro is not defined for this toolchain!" +# warning "The compiler may defeat side-channel resistance with optimizations." +# warning "Please locate target.h and try to fill it out with a suitable definition for this compiler." +# endif +#endif + /* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). */ @@ -83,184 +108,186 @@ #define KRML_LOOP1(i, n, x) { \ x \ i += n; \ + (void) i; \ } -#define KRML_LOOP2(i, n, x) \ - KRML_LOOP1(i, n, x) \ +#define KRML_LOOP2(i, n, x) \ + KRML_LOOP1(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP3(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP3(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP4(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP4(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP5(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP5(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP6(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP6(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP7(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP7(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP8(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP8(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP9(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP9(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP10(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP10(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP11(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP11(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP12(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP12(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP13(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP13(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP5(i, n, x) -#define KRML_LOOP14(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP14(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP6(i, n, x) -#define KRML_LOOP15(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP15(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP7(i, n, x) -#define KRML_LOOP16(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP16(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP8(i, n, x) -#define KRML_UNROLL_FOR(i, z, n, k, x) do { \ - uint32_t i = z; \ - KRML_LOOP##n(i, k, x) \ -} while (0) +#define KRML_UNROLL_FOR(i, z, n, k, x) \ + do { \ + uint32_t i = z; \ + KRML_LOOP##n(i, k, x) \ + } while (0) -#define KRML_ACTUAL_FOR(i, z, n, k, x) \ - do { \ - for (uint32_t i = z; i < n; i += k) { \ - x \ - } \ +#define KRML_ACTUAL_FOR(i, z, n, k, x) \ + do { \ + for (uint32_t i = z; i < n; i += k) { \ + x \ + } \ } while (0) #ifndef KRML_UNROLL_MAX -#define KRML_UNROLL_MAX 16 +# define KRML_UNROLL_MAX 16 #endif /* 1 is the number of loop iterations, i.e. (n - z)/k as evaluated by krml */ #if 0 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR0(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) #else -#define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 1 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) #else -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 2 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) #else -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 3 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) #else -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 4 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) #else -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 5 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) #else -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 6 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) #else -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 7 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) #else -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 8 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) #else -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 9 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) #else -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 10 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) #else -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 11 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) #else -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 12 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) #else -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 13 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) #else -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 14 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) #else -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 15 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) #else -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 16 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) #else -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #endif diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h index 87ad4cf228d91b..a50ec407f53e39 100644 --- a/Modules/_hacl/internal/Hacl_Hash_MD5.h +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_MD5.h" -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s); +void Hacl_Hash_MD5_init(uint32_t *s); -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h index d2d9df44c6c14c..b39bad3f3b93e8 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_SHA1.h" -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s); +void Hacl_Hash_SHA1_init(uint32_t *s); -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 851f7dc60c94c2..0127f4373fb1a1 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -40,141 +40,121 @@ extern "C" { static const uint32_t -Hacl_Impl_SHA2_Generic_h224[8U] = +Hacl_Hash_SHA2_h224[8U] = { - (uint32_t)0xc1059ed8U, (uint32_t)0x367cd507U, (uint32_t)0x3070dd17U, (uint32_t)0xf70e5939U, - (uint32_t)0xffc00b31U, (uint32_t)0x68581511U, (uint32_t)0x64f98fa7U, (uint32_t)0xbefa4fa4U + 0xc1059ed8U, 0x367cd507U, 0x3070dd17U, 0xf70e5939U, 0xffc00b31U, 0x68581511U, 0x64f98fa7U, + 0xbefa4fa4U }; static const uint32_t -Hacl_Impl_SHA2_Generic_h256[8U] = +Hacl_Hash_SHA2_h256[8U] = { - (uint32_t)0x6a09e667U, (uint32_t)0xbb67ae85U, (uint32_t)0x3c6ef372U, (uint32_t)0xa54ff53aU, - (uint32_t)0x510e527fU, (uint32_t)0x9b05688cU, (uint32_t)0x1f83d9abU, (uint32_t)0x5be0cd19U + 0x6a09e667U, 0xbb67ae85U, 0x3c6ef372U, 0xa54ff53aU, 0x510e527fU, 0x9b05688cU, 0x1f83d9abU, + 0x5be0cd19U }; static const uint64_t -Hacl_Impl_SHA2_Generic_h384[8U] = +Hacl_Hash_SHA2_h384[8U] = { - (uint64_t)0xcbbb9d5dc1059ed8U, (uint64_t)0x629a292a367cd507U, (uint64_t)0x9159015a3070dd17U, - (uint64_t)0x152fecd8f70e5939U, (uint64_t)0x67332667ffc00b31U, (uint64_t)0x8eb44a8768581511U, - (uint64_t)0xdb0c2e0d64f98fa7U, (uint64_t)0x47b5481dbefa4fa4U + 0xcbbb9d5dc1059ed8ULL, 0x629a292a367cd507ULL, 0x9159015a3070dd17ULL, 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, 0x8eb44a8768581511ULL, 0xdb0c2e0d64f98fa7ULL, 0x47b5481dbefa4fa4ULL }; static const uint64_t -Hacl_Impl_SHA2_Generic_h512[8U] = +Hacl_Hash_SHA2_h512[8U] = { - (uint64_t)0x6a09e667f3bcc908U, (uint64_t)0xbb67ae8584caa73bU, (uint64_t)0x3c6ef372fe94f82bU, - (uint64_t)0xa54ff53a5f1d36f1U, (uint64_t)0x510e527fade682d1U, (uint64_t)0x9b05688c2b3e6c1fU, - (uint64_t)0x1f83d9abfb41bd6bU, (uint64_t)0x5be0cd19137e2179U + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL }; static const uint32_t -Hacl_Impl_SHA2_Generic_k224_256[64U] = +Hacl_Hash_SHA2_k224_256[64U] = { - (uint32_t)0x428a2f98U, (uint32_t)0x71374491U, (uint32_t)0xb5c0fbcfU, (uint32_t)0xe9b5dba5U, - (uint32_t)0x3956c25bU, (uint32_t)0x59f111f1U, (uint32_t)0x923f82a4U, (uint32_t)0xab1c5ed5U, - (uint32_t)0xd807aa98U, (uint32_t)0x12835b01U, (uint32_t)0x243185beU, (uint32_t)0x550c7dc3U, - (uint32_t)0x72be5d74U, (uint32_t)0x80deb1feU, (uint32_t)0x9bdc06a7U, (uint32_t)0xc19bf174U, - (uint32_t)0xe49b69c1U, (uint32_t)0xefbe4786U, (uint32_t)0x0fc19dc6U, (uint32_t)0x240ca1ccU, - (uint32_t)0x2de92c6fU, (uint32_t)0x4a7484aaU, (uint32_t)0x5cb0a9dcU, (uint32_t)0x76f988daU, - (uint32_t)0x983e5152U, (uint32_t)0xa831c66dU, (uint32_t)0xb00327c8U, (uint32_t)0xbf597fc7U, - (uint32_t)0xc6e00bf3U, (uint32_t)0xd5a79147U, (uint32_t)0x06ca6351U, (uint32_t)0x14292967U, - (uint32_t)0x27b70a85U, (uint32_t)0x2e1b2138U, (uint32_t)0x4d2c6dfcU, (uint32_t)0x53380d13U, - (uint32_t)0x650a7354U, (uint32_t)0x766a0abbU, (uint32_t)0x81c2c92eU, (uint32_t)0x92722c85U, - (uint32_t)0xa2bfe8a1U, (uint32_t)0xa81a664bU, (uint32_t)0xc24b8b70U, (uint32_t)0xc76c51a3U, - (uint32_t)0xd192e819U, (uint32_t)0xd6990624U, (uint32_t)0xf40e3585U, (uint32_t)0x106aa070U, - (uint32_t)0x19a4c116U, (uint32_t)0x1e376c08U, (uint32_t)0x2748774cU, (uint32_t)0x34b0bcb5U, - (uint32_t)0x391c0cb3U, (uint32_t)0x4ed8aa4aU, (uint32_t)0x5b9cca4fU, (uint32_t)0x682e6ff3U, - (uint32_t)0x748f82eeU, (uint32_t)0x78a5636fU, (uint32_t)0x84c87814U, (uint32_t)0x8cc70208U, - (uint32_t)0x90befffaU, (uint32_t)0xa4506cebU, (uint32_t)0xbef9a3f7U, (uint32_t)0xc67178f2U + 0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U, 0x3956c25bU, 0x59f111f1U, 0x923f82a4U, + 0xab1c5ed5U, 0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U, 0x72be5d74U, 0x80deb1feU, + 0x9bdc06a7U, 0xc19bf174U, 0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU, 0x2de92c6fU, + 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU, 0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U, + 0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U, 0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, + 0x53380d13U, 0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U, 0xa2bfe8a1U, 0xa81a664bU, + 0xc24b8b70U, 0xc76c51a3U, 0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U, 0x19a4c116U, + 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U, 0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U, + 0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U, 0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, + 0xc67178f2U }; static const uint64_t -Hacl_Impl_SHA2_Generic_k384_512[80U] = +Hacl_Hash_SHA2_k384_512[80U] = { - (uint64_t)0x428a2f98d728ae22U, (uint64_t)0x7137449123ef65cdU, (uint64_t)0xb5c0fbcfec4d3b2fU, - (uint64_t)0xe9b5dba58189dbbcU, (uint64_t)0x3956c25bf348b538U, (uint64_t)0x59f111f1b605d019U, - (uint64_t)0x923f82a4af194f9bU, (uint64_t)0xab1c5ed5da6d8118U, (uint64_t)0xd807aa98a3030242U, - (uint64_t)0x12835b0145706fbeU, (uint64_t)0x243185be4ee4b28cU, (uint64_t)0x550c7dc3d5ffb4e2U, - (uint64_t)0x72be5d74f27b896fU, (uint64_t)0x80deb1fe3b1696b1U, (uint64_t)0x9bdc06a725c71235U, - (uint64_t)0xc19bf174cf692694U, (uint64_t)0xe49b69c19ef14ad2U, (uint64_t)0xefbe4786384f25e3U, - (uint64_t)0x0fc19dc68b8cd5b5U, (uint64_t)0x240ca1cc77ac9c65U, (uint64_t)0x2de92c6f592b0275U, - (uint64_t)0x4a7484aa6ea6e483U, (uint64_t)0x5cb0a9dcbd41fbd4U, (uint64_t)0x76f988da831153b5U, - (uint64_t)0x983e5152ee66dfabU, (uint64_t)0xa831c66d2db43210U, (uint64_t)0xb00327c898fb213fU, - (uint64_t)0xbf597fc7beef0ee4U, (uint64_t)0xc6e00bf33da88fc2U, (uint64_t)0xd5a79147930aa725U, - (uint64_t)0x06ca6351e003826fU, (uint64_t)0x142929670a0e6e70U, (uint64_t)0x27b70a8546d22ffcU, - (uint64_t)0x2e1b21385c26c926U, (uint64_t)0x4d2c6dfc5ac42aedU, (uint64_t)0x53380d139d95b3dfU, - (uint64_t)0x650a73548baf63deU, (uint64_t)0x766a0abb3c77b2a8U, (uint64_t)0x81c2c92e47edaee6U, - (uint64_t)0x92722c851482353bU, (uint64_t)0xa2bfe8a14cf10364U, (uint64_t)0xa81a664bbc423001U, - (uint64_t)0xc24b8b70d0f89791U, (uint64_t)0xc76c51a30654be30U, (uint64_t)0xd192e819d6ef5218U, - (uint64_t)0xd69906245565a910U, (uint64_t)0xf40e35855771202aU, (uint64_t)0x106aa07032bbd1b8U, - (uint64_t)0x19a4c116b8d2d0c8U, (uint64_t)0x1e376c085141ab53U, (uint64_t)0x2748774cdf8eeb99U, - (uint64_t)0x34b0bcb5e19b48a8U, (uint64_t)0x391c0cb3c5c95a63U, (uint64_t)0x4ed8aa4ae3418acbU, - (uint64_t)0x5b9cca4f7763e373U, (uint64_t)0x682e6ff3d6b2b8a3U, (uint64_t)0x748f82ee5defb2fcU, - (uint64_t)0x78a5636f43172f60U, (uint64_t)0x84c87814a1f0ab72U, (uint64_t)0x8cc702081a6439ecU, - (uint64_t)0x90befffa23631e28U, (uint64_t)0xa4506cebde82bde9U, (uint64_t)0xbef9a3f7b2c67915U, - (uint64_t)0xc67178f2e372532bU, (uint64_t)0xca273eceea26619cU, (uint64_t)0xd186b8c721c0c207U, - (uint64_t)0xeada7dd6cde0eb1eU, (uint64_t)0xf57d4f7fee6ed178U, (uint64_t)0x06f067aa72176fbaU, - (uint64_t)0x0a637dc5a2c898a6U, (uint64_t)0x113f9804bef90daeU, (uint64_t)0x1b710b35131c471bU, - (uint64_t)0x28db77f523047d84U, (uint64_t)0x32caab7b40c72493U, (uint64_t)0x3c9ebe0a15c9bebcU, - (uint64_t)0x431d67c49c100d4cU, (uint64_t)0x4cc5d4becb3e42b6U, (uint64_t)0x597f299cfc657e2aU, - (uint64_t)0x5fcb6fab3ad6faecU, (uint64_t)0x6c44198c4a475817U + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL }; -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash); -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -); +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash); -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash); void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); +Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *hash ); -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ); -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h index 1c9808b8dd497c..b80e81fafb9780 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -53,9 +53,9 @@ Hacl_Hash_SHA3_update_last_sha3( uint32_t input_len ); -void Hacl_Impl_SHA3_state_permute(uint64_t *s); +void Hacl_Hash_SHA3_state_permute(uint64_t *s); -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); +void Hacl_Hash_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); #if defined(__cplusplus) } diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index 0df236282ac509..684e7fd2fbefbc 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -5,59 +5,61 @@ * C's excuse for namespaces: Use globally unique names to avoid linkage * conflicts with builds linking or dynamically loading other code potentially * using HACL* libraries. + * + * To make sure this is effective: cd Modules && nm -a *.o | grep Hacl */ -#define Hacl_Streaming_SHA2_state_sha2_224_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_224_s -#define Hacl_Streaming_SHA2_state_sha2_224 python_hashlib_Hacl_Streaming_SHA2_state_sha2_224 -#define Hacl_Streaming_SHA2_state_sha2_256 python_hashlib_Hacl_Streaming_SHA2_state_sha2_256 -#define Hacl_Streaming_SHA2_state_sha2_384_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_384_s -#define Hacl_Streaming_SHA2_state_sha2_384 python_hashlib_Hacl_Streaming_SHA2_state_sha2_384 -#define Hacl_Streaming_SHA2_state_sha2_512 python_hashlib_Hacl_Streaming_SHA2_state_sha2_512 -#define Hacl_Streaming_SHA2_create_in_256 python_hashlib_Hacl_Streaming_SHA2_create_in_256 -#define Hacl_Streaming_SHA2_create_in_224 python_hashlib_Hacl_Streaming_SHA2_create_in_224 -#define Hacl_Streaming_SHA2_create_in_512 python_hashlib_Hacl_Streaming_SHA2_create_in_512 -#define Hacl_Streaming_SHA2_create_in_384 python_hashlib_Hacl_Streaming_SHA2_create_in_384 -#define Hacl_Streaming_SHA2_copy_256 python_hashlib_Hacl_Streaming_SHA2_copy_256 -#define Hacl_Streaming_SHA2_copy_224 python_hashlib_Hacl_Streaming_SHA2_copy_224 -#define Hacl_Streaming_SHA2_copy_512 python_hashlib_Hacl_Streaming_SHA2_copy_512 -#define Hacl_Streaming_SHA2_copy_384 python_hashlib_Hacl_Streaming_SHA2_copy_384 -#define Hacl_Streaming_SHA2_init_256 python_hashlib_Hacl_Streaming_SHA2_init_256 -#define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 -#define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 -#define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 +#define Hacl_Hash_SHA2_state_sha2_224_s python_hashlib_Hacl_Hash_SHA2_state_sha2_224_s +#define Hacl_Hash_SHA2_state_sha2_224 python_hashlib_Hacl_Hash_SHA2_state_sha2_224 +#define Hacl_Hash_SHA2_state_sha2_256 python_hashlib_Hacl_Hash_SHA2_state_sha2_256 +#define Hacl_Hash_SHA2_state_sha2_384_s python_hashlib_Hacl_Hash_SHA2_state_sha2_384_s +#define Hacl_Hash_SHA2_state_sha2_384 python_hashlib_Hacl_Hash_SHA2_state_sha2_384 +#define Hacl_Hash_SHA2_state_sha2_512 python_hashlib_Hacl_Hash_SHA2_state_sha2_512 +#define Hacl_Hash_SHA2_malloc_256 python_hashlib_Hacl_Hash_SHA2_malloc_256 +#define Hacl_Hash_SHA2_malloc_224 python_hashlib_Hacl_Hash_SHA2_malloc_224 +#define Hacl_Hash_SHA2_malloc_512 python_hashlib_Hacl_Hash_SHA2_malloc_512 +#define Hacl_Hash_SHA2_malloc_384 python_hashlib_Hacl_Hash_SHA2_malloc_384 +#define Hacl_Hash_SHA2_copy_256 python_hashlib_Hacl_Hash_SHA2_copy_256 +#define Hacl_Hash_SHA2_copy_224 python_hashlib_Hacl_Hash_SHA2_copy_224 +#define Hacl_Hash_SHA2_copy_512 python_hashlib_Hacl_Hash_SHA2_copy_512 +#define Hacl_Hash_SHA2_copy_384 python_hashlib_Hacl_Hash_SHA2_copy_384 +#define Hacl_Hash_SHA2_init_256 python_hashlib_Hacl_Hash_SHA2_init_256 +#define Hacl_Hash_SHA2_init_224 python_hashlib_Hacl_Hash_SHA2_init_224 +#define Hacl_Hash_SHA2_init_512 python_hashlib_Hacl_Hash_SHA2_init_512 +#define Hacl_Hash_SHA2_init_384 python_hashlib_Hacl_Hash_SHA2_init_384 #define Hacl_SHA2_Scalar32_sha512_init python_hashlib_Hacl_SHA2_Scalar32_sha512_init -#define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 -#define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 -#define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 -#define Hacl_Streaming_SHA2_update_384 python_hashlib_Hacl_Streaming_SHA2_update_384 -#define Hacl_Streaming_SHA2_finish_256 python_hashlib_Hacl_Streaming_SHA2_finish_256 -#define Hacl_Streaming_SHA2_finish_224 python_hashlib_Hacl_Streaming_SHA2_finish_224 -#define Hacl_Streaming_SHA2_finish_512 python_hashlib_Hacl_Streaming_SHA2_finish_512 -#define Hacl_Streaming_SHA2_finish_384 python_hashlib_Hacl_Streaming_SHA2_finish_384 -#define Hacl_Streaming_SHA2_free_256 python_hashlib_Hacl_Streaming_SHA2_free_256 -#define Hacl_Streaming_SHA2_free_224 python_hashlib_Hacl_Streaming_SHA2_free_224 -#define Hacl_Streaming_SHA2_free_512 python_hashlib_Hacl_Streaming_SHA2_free_512 -#define Hacl_Streaming_SHA2_free_384 python_hashlib_Hacl_Streaming_SHA2_free_384 -#define Hacl_Streaming_SHA2_sha256 python_hashlib_Hacl_Streaming_SHA2_sha256 -#define Hacl_Streaming_SHA2_sha224 python_hashlib_Hacl_Streaming_SHA2_sha224 -#define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 -#define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 +#define Hacl_Hash_SHA2_update_256 python_hashlib_Hacl_Hash_SHA2_update_256 +#define Hacl_Hash_SHA2_update_224 python_hashlib_Hacl_Hash_SHA2_update_224 +#define Hacl_Hash_SHA2_update_512 python_hashlib_Hacl_Hash_SHA2_update_512 +#define Hacl_Hash_SHA2_update_384 python_hashlib_Hacl_Hash_SHA2_update_384 +#define Hacl_Hash_SHA2_digest_256 python_hashlib_Hacl_Hash_SHA2_digest_256 +#define Hacl_Hash_SHA2_digest_224 python_hashlib_Hacl_Hash_SHA2_digest_224 +#define Hacl_Hash_SHA2_digest_512 python_hashlib_Hacl_Hash_SHA2_digest_512 +#define Hacl_Hash_SHA2_digest_384 python_hashlib_Hacl_Hash_SHA2_digest_384 +#define Hacl_Hash_SHA2_free_256 python_hashlib_Hacl_Hash_SHA2_free_256 +#define Hacl_Hash_SHA2_free_224 python_hashlib_Hacl_Hash_SHA2_free_224 +#define Hacl_Hash_SHA2_free_512 python_hashlib_Hacl_Hash_SHA2_free_512 +#define Hacl_Hash_SHA2_free_384 python_hashlib_Hacl_Hash_SHA2_free_384 +#define Hacl_Hash_SHA2_sha256 python_hashlib_Hacl_Hash_SHA2_sha256 +#define Hacl_Hash_SHA2_sha224 python_hashlib_Hacl_Hash_SHA2_sha224 +#define Hacl_Hash_SHA2_sha512 python_hashlib_Hacl_Hash_SHA2_sha512 +#define Hacl_Hash_SHA2_sha384 python_hashlib_Hacl_Hash_SHA2_sha384 -#define Hacl_Streaming_MD5_legacy_create_in python_hashlib_Hacl_Streaming_MD5_legacy_create_in -#define Hacl_Streaming_MD5_legacy_init python_hashlib_Hacl_Streaming_MD5_legacy_init -#define Hacl_Streaming_MD5_legacy_update python_hashlib_Hacl_Streaming_MD5_legacy_update -#define Hacl_Streaming_MD5_legacy_finish python_hashlib_Hacl_Streaming_MD5_legacy_finish -#define Hacl_Streaming_MD5_legacy_free python_hashlib_Hacl_Streaming_MD5_legacy_free -#define Hacl_Streaming_MD5_legacy_copy python_hashlib_Hacl_Streaming_MD5_legacy_copy -#define Hacl_Streaming_MD5_legacy_hash python_hashlib_Hacl_Streaming_MD5_legacy_hash +#define Hacl_Hash_MD5_malloc python_hashlib_Hacl_Hash_MD5_malloc +#define Hacl_Hash_MD5_init python_hashlib_Hacl_Hash_MD5_init +#define Hacl_Hash_MD5_update python_hashlib_Hacl_Hash_MD5_update +#define Hacl_Hash_MD5_digest python_hashlib_Hacl_Hash_MD5_digest +#define Hacl_Hash_MD5_free python_hashlib_Hacl_Hash_MD5_free +#define Hacl_Hash_MD5_copy python_hashlib_Hacl_Hash_MD5_copy +#define Hacl_Hash_MD5_hash python_hashlib_Hacl_Hash_MD5_hash -#define Hacl_Streaming_SHA1_legacy_create_in python_hashlib_Hacl_Streaming_SHA1_legacy_create_in -#define Hacl_Streaming_SHA1_legacy_init python_hashlib_Hacl_Streaming_SHA1_legacy_init -#define Hacl_Streaming_SHA1_legacy_update python_hashlib_Hacl_Streaming_SHA1_legacy_update -#define Hacl_Streaming_SHA1_legacy_finish python_hashlib_Hacl_Streaming_SHA1_legacy_finish -#define Hacl_Streaming_SHA1_legacy_free python_hashlib_Hacl_Streaming_SHA1_legacy_free -#define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy -#define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA1_malloc python_hashlib_Hacl_Hash_SHA1_malloc +#define Hacl_Hash_SHA1_init python_hashlib_Hacl_Hash_SHA1_init +#define Hacl_Hash_SHA1_update python_hashlib_Hacl_Hash_SHA1_update +#define Hacl_Hash_SHA1_digest python_hashlib_Hacl_Hash_SHA1_digest +#define Hacl_Hash_SHA1_free python_hashlib_Hacl_Hash_SHA1_free +#define Hacl_Hash_SHA1_copy python_hashlib_Hacl_Hash_SHA1_copy +#define Hacl_Hash_SHA1_hash python_hashlib_Hacl_Hash_SHA1_hash #define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 #define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 @@ -72,15 +74,16 @@ #define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 #define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl #define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl -#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len -#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy -#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish -#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free -#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg -#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len -#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake -#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc -#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset -#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update +#define Hacl_Hash_SHA3_block_len python_hashlib_Hacl_Hash_SHA3_block_len +#define Hacl_Hash_SHA3_copy python_hashlib_Hacl_Hash_SHA3_copy +#define Hacl_Hash_SHA3_digest python_hashlib_Hacl_Hash_SHA3_digest +#define Hacl_Hash_SHA3_free python_hashlib_Hacl_Hash_SHA3_free +#define Hacl_Hash_SHA3_get_alg python_hashlib_Hacl_Hash_SHA3_get_alg +#define Hacl_Hash_SHA3_hash_len python_hashlib_Hacl_Hash_SHA3_hash_len +#define Hacl_Hash_SHA3_is_shake python_hashlib_Hacl_Hash_SHA3_is_shake +#define Hacl_Hash_SHA3_malloc python_hashlib_Hacl_Hash_SHA3_malloc +#define Hacl_Hash_SHA3_reset python_hashlib_Hacl_Hash_SHA3_reset +#define Hacl_Hash_SHA3_update python_hashlib_Hacl_Hash_SHA3_update +#define Hacl_Hash_SHA3_squeeze python_hashlib_Hacl_Hash_SHA3_squeeze #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index c1b3e37f3afb9d..3878e02af31a21 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=521af282fdf6d60227335120f18ae9309a4b8e8c +expected_hacl_star_rev=bb3d0dc8d9d15a5cd51094d5b69e70aa09005ff0 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -127,7 +127,7 @@ $sed -i -z 's!\(extern\|typedef\)[^;]*;\n\n!!g' include/krml/FStar_UInt_8_16_32_ $sed -i 's!#include.*Hacl_Krmllib.h"!!g' "${all_files[@]}" # Use globally unique names for the Hacl_ C APIs to avoid linkage conflicts. -$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_SHA2.h +$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_*.h # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ee6fb8b4b03643..14d9c186151232 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -22,6 +22,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif +#include #include "Python.h" #include "pycore_hashtable.h" #include "pycore_pyhash.h" // _Py_HashBytes() @@ -44,9 +45,15 @@ #define MUNCH_SIZE INT_MAX #define PY_OPENSSL_HAS_SCRYPT 1 +#if defined(NID_sha3_224) && defined(NID_sha3_256) && defined(NID_sha3_384) && defined(NID_sha3_512) #define PY_OPENSSL_HAS_SHA3 1 +#endif +#if defined(NID_shake128) || defined(NID_shake256) #define PY_OPENSSL_HAS_SHAKE 1 +#endif +#if defined(NID_blake2s256) || defined(NID_blake2b512) #define PY_OPENSSL_HAS_BLAKE2 1 +#endif #if OPENSSL_VERSION_NUMBER >= 0x30000000L #define PY_EVP_MD EVP_MD @@ -87,22 +94,45 @@ typedef struct { PY_EVP_MD *evp_nosecurity; } py_hashentry_t; +// Fundamental to TLS, assumed always present in any libcrypto: #define Py_hash_md5 "md5" #define Py_hash_sha1 "sha1" #define Py_hash_sha224 "sha224" #define Py_hash_sha256 "sha256" #define Py_hash_sha384 "sha384" #define Py_hash_sha512 "sha512" -#define Py_hash_sha512_224 "sha512_224" -#define Py_hash_sha512_256 "sha512_256" -#define Py_hash_sha3_224 "sha3_224" -#define Py_hash_sha3_256 "sha3_256" -#define Py_hash_sha3_384 "sha3_384" -#define Py_hash_sha3_512 "sha3_512" -#define Py_hash_shake_128 "shake_128" -#define Py_hash_shake_256 "shake_256" -#define Py_hash_blake2s "blake2s" -#define Py_hash_blake2b "blake2b" + +// Not all OpenSSL-like libcrypto libraries provide these: +#if defined(NID_sha512_224) +# define Py_hash_sha512_224 "sha512_224" +#endif +#if defined(NID_sha512_256) +# define Py_hash_sha512_256 "sha512_256" +#endif +#if defined(NID_sha3_224) +# define Py_hash_sha3_224 "sha3_224" +#endif +#if defined(NID_sha3_256) +# define Py_hash_sha3_256 "sha3_256" +#endif +#if defined(NID_sha3_384) +# define Py_hash_sha3_384 "sha3_384" +#endif +#if defined(NID_sha3_512) +# define Py_hash_sha3_512 "sha3_512" +#endif +#if defined(NID_shake128) +# define Py_hash_shake_128 "shake_128" +#endif +#if defined(NID_shake256) +# define Py_hash_shake_256 "shake_256" +#endif +#if defined(NID_blake2s256) +# define Py_hash_blake2s "blake2s" +#endif +#if defined(NID_blake2b512) +# define Py_hash_blake2b "blake2b" +#endif #define PY_HASH_ENTRY(py_name, py_alias, ossl_name, ossl_nid) \ {py_name, py_alias, ossl_name, ossl_nid, 0, NULL, NULL} @@ -118,19 +148,39 @@ static const py_hashentry_t py_hashes[] = { PY_HASH_ENTRY(Py_hash_sha384, "SHA384", SN_sha384, NID_sha384), PY_HASH_ENTRY(Py_hash_sha512, "SHA512", SN_sha512, NID_sha512), /* truncated sha2 */ +#ifdef Py_hash_sha512_224 PY_HASH_ENTRY(Py_hash_sha512_224, "SHA512_224", SN_sha512_224, NID_sha512_224), +#endif +#ifdef Py_hash_sha512_256 PY_HASH_ENTRY(Py_hash_sha512_256, "SHA512_256", SN_sha512_256, NID_sha512_256), +#endif /* sha3 */ +#ifdef Py_hash_sha3_224 PY_HASH_ENTRY(Py_hash_sha3_224, NULL, SN_sha3_224, NID_sha3_224), +#endif +#ifdef Py_hash_sha3_256 PY_HASH_ENTRY(Py_hash_sha3_256, NULL, SN_sha3_256, NID_sha3_256), +#endif +#ifdef Py_hash_sha3_384 PY_HASH_ENTRY(Py_hash_sha3_384, NULL, SN_sha3_384, NID_sha3_384), +#endif +#ifdef Py_hash_sha3_512 PY_HASH_ENTRY(Py_hash_sha3_512, NULL, SN_sha3_512, NID_sha3_512), +#endif /* sha3 shake */ +#ifdef Py_hash_shake_128 PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), +#endif +#ifdef Py_hash_shake_256 PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), +#endif /* blake2 digest */ +#ifdef Py_hash_blake2s PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), +#endif +#ifdef Py_hash_blake2b PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), +#endif PY_HASH_ENTRY(NULL, NULL, NULL, 0), }; @@ -227,16 +277,16 @@ typedef struct { PyObject_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; /* OpenSSL context lock */ + bool use_mutex; + PyMutex mutex; /* OpenSSL context lock */ } EVPobject; typedef struct { PyObject_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; /* HMAC context lock */ + bool use_mutex; + PyMutex mutex; /* HMAC context lock */ } HMACobject; #include "clinic/_hashopenssl.c.h" @@ -414,8 +464,7 @@ newEVPobject(PyTypeObject *type) if (retval == NULL) { return NULL; } - - retval->lock = NULL; + HASHLIB_INIT_MUTEX(retval); retval->ctx = EVP_MD_CTX_new(); if (retval->ctx == NULL) { @@ -453,8 +502,6 @@ static void EVP_dealloc(EVPobject *self) { PyTypeObject *tp = Py_TYPE(self); - if (self->lock != NULL) - PyThread_free_lock(self->lock); EVP_MD_CTX_free(self->ctx); PyObject_Free(self); Py_DECREF(tp); @@ -582,16 +629,14 @@ EVP_update(EVPobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &view); - if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - /* fail? lock = NULL and we fail over to non-threaded code. */ + if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); result = EVP_hash(self, view.buf, view.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { result = EVP_hash(self, view.buf, view.len); @@ -1540,7 +1585,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, } self->ctx = ctx; - self->lock = NULL; + HASHLIB_INIT_MUTEX(self); if ((msg_obj != NULL) && (msg_obj != Py_None)) { if (!_hmac_update(self, msg_obj)) @@ -1582,16 +1627,14 @@ _hmac_update(HMACobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0); - if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - /* fail? lock = NULL and we fail over to non-threaded code. */ + if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len); @@ -1633,7 +1676,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self) return NULL; } retval->ctx = ctx; - retval->lock = NULL; + HASHLIB_INIT_MUTEX(retval); return (PyObject *)retval; } @@ -1642,9 +1685,6 @@ static void _hmac_dealloc(HMACobject *self) { PyTypeObject *tp = Py_TYPE(self); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } HMAC_CTX_free(self->ctx); PyObject_Free(self); Py_DECREF(tp); @@ -2249,6 +2289,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 9d4ec256ee9e3e..80fe9cff98509d 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -585,7 +585,7 @@ non-existing elements are considered to be infinite. The interesting\n\ property of a heap is that a[0] is always its smallest element.\n" "\n\ The strange invariant above is meant to be an efficient memory\n\ -representation for a tournament. The numbers below are `k', not a[k]:\n\ +representation for a tournament. The numbers below are 'k', not a[k]:\n\ \n\ 0\n\ \n\ @@ -598,7 +598,7 @@ representation for a tournament. The numbers below are `k', not a[k]:\n\ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30\n\ \n\ \n\ -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In\n\ +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In\n\ a usual binary tournament we see in sports, each cell is the winner\n\ over the two cells it tops, and we can trace the winner down the tree\n\ to see all opponents s/he had. However, in many computer applications\n\ @@ -653,7 +653,7 @@ vanishes, you switch heaps and start a new run. Clever and quite\n\ effective!\n\ \n\ In a word, heaps are useful memory structures to know. I use them in\n\ -a few applications, and I think it is good to keep a `heap' module\n\ +a few applications, and I think it is good to keep a 'heap' module\n\ around. :-)\n" "\n\ --------------------\n\ @@ -681,6 +681,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_interpchannelsmodule.c similarity index 60% rename from Modules/_xxinterpchannelsmodule.c rename to Modules/_interpchannelsmodule.c index bc8cd0e2cff4c1..f0447475c49116 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -6,9 +6,20 @@ #endif #include "Python.h" -#include "interpreteridobject.h" -#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() +#include "pycore_crossinterp.h" // struct _xid #include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject() + +#ifdef MS_WINDOWS +#define WIN32_LEAN_AND_MEAN +#include // SwitchToThread() +#elif defined(HAVE_SCHED_H) +#include // sched_yield() +#endif + +#define REGISTERS_HEAP_TYPES +#include "_interpreters_common.h" +#undef REGISTERS_HEAP_TYPES /* @@ -21,7 +32,7 @@ _globals (static struct globals): next_id; (int64_t) mutex (PyThread_type_lock) head (linked list of struct _channelref *): - id (int64_t) + cid (int64_t) objcount (Py_ssize_t) next (struct _channelref *): ... @@ -35,7 +46,7 @@ _globals (static struct globals): numsendopen (int64_t) numrecvopen (int64_t) send (struct _channelend *): - interp (int64_t) + interpid (int64_t) open (int) next (struct _channelend *) recv (struct _channelend *): @@ -48,7 +59,7 @@ _globals (static struct globals): data (_PyCrossInterpreterData *): data (void *) obj (PyObject *) - interp (int64_t) + interpid (int64_t) new_object (xid_newobjectfunc) free (xid_freefunc) last (struct _channelitem *): @@ -73,7 +84,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*() API.. The module does not create any objects that are shared globally. */ -#define MODULE_NAME "_xxinterpchannels" +#define MODULE_NAME _interpchannels +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) #define GLOBAL_MALLOC(TYPE) \ @@ -82,38 +95,6 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) -struct xid_class_registry { - size_t count; -#define MAX_XID_CLASSES 5 - struct { - PyTypeObject *cls; - } added[MAX_XID_CLASSES]; -}; - -static int -register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - int res = _PyCrossInterpreterData_RegisterClass(cls, shared); - if (res == 0) { - assert(classes->count < MAX_XID_CLASSES); - // The class has refs elsewhere, so we need to incref here. - classes->added[classes->count].cls = cls; - classes->count += 1; - } - return res; -} - -static void -clear_xid_class_registry(struct xid_class_registry *classes) -{ - while (classes->count > 0) { - classes->count -= 1; - PyTypeObject *cls = classes->added[classes->count].cls; - _PyCrossInterpreterData_UnregisterClass(cls); - } -} - #define XID_IGNORE_EXC 1 #define XID_FREE 2 @@ -160,7 +141,7 @@ _get_current_interp(void) static PyObject * _get_current_module(void) { - PyObject *name = PyUnicode_FromString(MODULE_NAME); + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); if (name == NULL) { return NULL; } @@ -210,168 +191,25 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) } #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ - add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) - -static PyTypeObject * -add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass( - NULL, mod, spec, NULL); - if (cls == NULL) { - return NULL; - } - if (PyModule_AddType(mod, cls) < 0) { - Py_DECREF(cls); - return NULL; - } - if (shared != NULL) { - if (register_xid_class(cls, shared, classes)) { - Py_DECREF(cls); - return NULL; - } - } - return cls; -} - -static void -wait_for_lock(PyThread_type_lock mutex) -{ - Py_BEGIN_ALLOW_THREADS - // XXX Handle eintr, etc. - PyThread_acquire_lock(mutex, WAIT_LOCK); - Py_END_ALLOW_THREADS - - PyThread_release_lock(mutex); -} - - -/* Cross-interpreter Buffer Views *******************************************/ - -// XXX Release when the original interpreter is destroyed. - -typedef struct { - PyObject_HEAD - Py_buffer *view; - int64_t interp; -} XIBufferViewObject; - -static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) -{ - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interp >= 0); - XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); - if (self == NULL) { - return NULL; - } - PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interp = data->interp; - return (PyObject *)self; -} - -static void -xibufferview_dealloc(XIBufferViewObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interp); - /* If the interpreter is no longer alive then we have problems, - since other objects may be using the buffer still. */ - assert(interp != NULL); - - if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { - // XXX Emit a warning? - PyErr_Clear(); - } - - PyTypeObject *tp = Py_TYPE(self); - tp->tp_free(self); - /* "Instances of heap-allocated types hold a reference to their type." - * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol - * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse - */ - // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, - // like we do for _abc._abc_data? - Py_DECREF(tp); -} + add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) static int -xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) -{ - /* Only PyMemoryView_FromObject() should ever call this, - via _memoryview_from_xid() below. */ - *view = *self->view; - view->obj = (PyObject *)self; - // XXX Should we leave it alone? - view->internal = NULL; - return 0; -} - -static PyType_Slot XIBufferViewType_slots[] = { - {Py_tp_dealloc, (destructor)xibufferview_dealloc}, - {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, - // We don't bother with Py_bf_releasebuffer since we don't need it. - {0, NULL}, -}; - -static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", - .basicsize = sizeof(XIBufferViewObject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .slots = XIBufferViewType_slots, -}; - - -/* extra XID types **********************************************************/ - -static PyTypeObject * _get_current_xibufferview_type(void); - -static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) -{ - PyTypeObject *cls = _get_current_xibufferview_type(); - if (cls == NULL) { - return NULL; - } - PyObject *obj = xibufferview_from_xid(cls, data); - if (obj == NULL) { - return NULL; - } - return PyMemoryView_FromObject(obj); -} - -static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) { - Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); - if (view == NULL) { - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { - PyMem_RawFree(view); + PyLockStatus res = PyThread_acquire_lock_timed_with_retries(mutex, timeout); + if (res == PY_LOCK_INTR) { + /* KeyboardInterrupt, etc. */ + assert(PyErr_Occurred()); return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); - return 0; -} - -static int -register_builtin_xid_types(struct xid_class_registry *classes) -{ - PyTypeObject *cls; - crossinterpdatafunc func; - - // builtin memoryview - cls = &PyMemoryView_Type; - func = _memoryview_shared; - if (register_xid_class(cls, func, classes)) { + else if (res == PY_LOCK_FAILURE) { + assert(!PyErr_Occurred()); + assert(timeout > 0); + PyErr_SetString(PyExc_TimeoutError, "timed out"); return -1; } - + assert(res == PY_LOCK_ACQUIRED); + PyThread_release_lock(mutex); return 0; } @@ -379,15 +217,13 @@ register_builtin_xid_types(struct xid_class_registry *classes) /* module state *************************************************************/ typedef struct { - struct xid_class_registry xid_classes; - /* Added at runtime by interpreters module. */ PyTypeObject *send_channel_type; PyTypeObject *recv_channel_type; /* heap types */ + PyTypeObject *ChannelInfoType; PyTypeObject *ChannelIDType; - PyTypeObject *XIBufferViewType; /* exceptions */ PyObject *ChannelError; @@ -413,7 +249,7 @@ _get_current_module_state(void) if (mod == NULL) { // XXX import it? PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); + MODULE_NAME_STR " module not imported yet"); return NULL; } module_state *state = get_module_state(mod); @@ -424,9 +260,13 @@ _get_current_module_state(void) static int traverse_module_state(module_state *state, visitproc visit, void *arg) { + /* external types */ + Py_VISIT(state->send_channel_type); + Py_VISIT(state->recv_channel_type); + /* heap types */ + Py_VISIT(state->ChannelInfoType); Py_VISIT(state->ChannelIDType); - Py_VISIT(state->XIBufferViewType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -438,18 +278,33 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) return 0; } -static int -clear_module_state(module_state *state) +static void +clear_xid_types(module_state *state) { - Py_CLEAR(state->send_channel_type); - Py_CLEAR(state->recv_channel_type); + /* external types */ + if (state->send_channel_type != NULL) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + } + if (state->recv_channel_type != NULL) { + (void)clear_xid_class(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); + } /* heap types */ if (state->ChannelIDType != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + (void)clear_xid_class(state->ChannelIDType); + Py_CLEAR(state->ChannelIDType); } - Py_CLEAR(state->ChannelIDType); - Py_CLEAR(state->XIBufferViewType); +} + +static int +clear_module_state(module_state *state) +{ + clear_xid_types(state); + + /* heap types */ + Py_CLEAR(state->ChannelInfoType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -462,23 +317,13 @@ clear_module_state(module_state *state) } -static PyTypeObject * -_get_current_xibufferview_type(void) -{ - module_state *state = _get_current_module_state(); - if (state == NULL) { - return NULL; - } - return state->XIBufferViewType; -} - - /* channel-specific code ****************************************************/ #define CHANNEL_SEND 1 #define CHANNEL_BOTH 0 #define CHANNEL_RECV -1 + /* channel errors */ #define ERR_CHANNEL_NOT_FOUND -2 @@ -489,6 +334,7 @@ _get_current_xibufferview_type(void) #define ERR_CHANNEL_MUTEX_INIT -7 #define ERR_CHANNELS_MUTEX_INIT -8 #define ERR_NO_NEXT_CHANNEL_ID -9 +#define ERR_CHANNEL_CLOSED_WAITING -10 static int exceptions_init(PyObject *mod) @@ -540,6 +386,10 @@ handle_channel_error(int err, PyObject *mod, int64_t cid) PyErr_Format(state->ChannelClosedError, "channel %" PRId64 " is closed", cid); } + else if (err == ERR_CHANNEL_CLOSED_WAITING) { + PyErr_Format(state->ChannelClosedError, + "channel %" PRId64 " has closed", cid); + } else if (err == ERR_CHANNEL_INTERP_CLOSED) { PyErr_Format(state->ChannelClosedError, "channel %" PRId64 " is already closed", cid); @@ -572,38 +422,148 @@ handle_channel_error(int err, PyObject *mod, int64_t cid) return 1; } + /* the channel queue */ +typedef uintptr_t _channelitem_id_t; + +typedef struct wait_info { + PyThread_type_lock mutex; + enum { + WAITING_NO_STATUS = 0, + WAITING_ACQUIRED = 1, + WAITING_RELEASING = 2, + WAITING_RELEASED = 3, + } status; + int received; + _channelitem_id_t itemid; +} _waiting_t; + +static int +_waiting_init(_waiting_t *waiting) +{ + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + PyErr_NoMemory(); + return -1; + } + + *waiting = (_waiting_t){ + .mutex = mutex, + .status = WAITING_NO_STATUS, + }; + return 0; +} + +static void +_waiting_clear(_waiting_t *waiting) +{ + assert(waiting->status != WAITING_ACQUIRED + && waiting->status != WAITING_RELEASING); + if (waiting->mutex != NULL) { + PyThread_free_lock(waiting->mutex); + waiting->mutex = NULL; + } +} + +static _channelitem_id_t +_waiting_get_itemid(_waiting_t *waiting) +{ + return waiting->itemid; +} + +static void +_waiting_acquire(_waiting_t *waiting) +{ + assert(waiting->status == WAITING_NO_STATUS); + PyThread_acquire_lock(waiting->mutex, NOWAIT_LOCK); + waiting->status = WAITING_ACQUIRED; +} + +static void +_waiting_release(_waiting_t *waiting, int received) +{ + assert(waiting->mutex != NULL); + assert(waiting->status == WAITING_ACQUIRED); + assert(!waiting->received); + + waiting->status = WAITING_RELEASING; + PyThread_release_lock(waiting->mutex); + if (waiting->received != received) { + assert(received == 1); + waiting->received = received; + } + waiting->status = WAITING_RELEASED; +} + +static void +_waiting_finish_releasing(_waiting_t *waiting) +{ + while (waiting->status == WAITING_RELEASING) { +#ifdef MS_WINDOWS + SwitchToThread(); +#elif defined(HAVE_SCHED_H) + sched_yield(); +#endif + } +} + struct _channelitem; typedef struct _channelitem { _PyCrossInterpreterData *data; - PyThread_type_lock recv_mutex; + _waiting_t *waiting; struct _channelitem *next; } _channelitem; -static _channelitem * -_channelitem_new(void) +static inline _channelitem_id_t +_channelitem_ID(_channelitem *item) { - _channelitem *item = GLOBAL_MALLOC(_channelitem); - if (item == NULL) { - PyErr_NoMemory(); - return NULL; + return (_channelitem_id_t)item; +} + +static void +_channelitem_init(_channelitem *item, + _PyCrossInterpreterData *data, _waiting_t *waiting) +{ + *item = (_channelitem){ + .data = data, + .waiting = waiting, + }; + if (waiting != NULL) { + waiting->itemid = _channelitem_ID(item); } - item->data = NULL; - item->next = NULL; - return item; } static void _channelitem_clear(_channelitem *item) { + item->next = NULL; + if (item->data != NULL) { - // It was allocated in _channel_send(). + // It was allocated in channel_send(). (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); item->data = NULL; } - item->next = NULL; + + if (item->waiting != NULL) { + if (item->waiting->status == WAITING_ACQUIRED) { + _waiting_release(item->waiting, 0); + } + item->waiting = NULL; + } +} + +static _channelitem * +_channelitem_new(_PyCrossInterpreterData *data, _waiting_t *waiting) +{ + _channelitem *item = GLOBAL_MALLOC(_channelitem); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + _channelitem_init(item, data, waiting); + return item; } static void @@ -623,14 +583,17 @@ _channelitem_free_all(_channelitem *item) } } -static _PyCrossInterpreterData * -_channelitem_popped(_channelitem *item, PyThread_type_lock *recv_mutex) +static void +_channelitem_popped(_channelitem *item, + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) { - _PyCrossInterpreterData *data = item->data; + assert(item->waiting == NULL || item->waiting->status == WAITING_ACQUIRED); + *p_data = item->data; + *p_waiting = item->waiting; + // We clear them here, so they won't be released in _channelitem_clear(). item->data = NULL; - *recv_mutex = item->recv_mutex; + item->waiting = NULL; _channelitem_free(item); - return data; } typedef struct _channelqueue { @@ -670,15 +633,13 @@ _channelqueue_free(_channelqueue *queue) } static int -_channelqueue_put(_channelqueue *queue, _PyCrossInterpreterData *data, - PyThread_type_lock recv_mutex) +_channelqueue_put(_channelqueue *queue, + _PyCrossInterpreterData *data, _waiting_t *waiting) { - _channelitem *item = _channelitem_new(); + _channelitem *item = _channelitem_new(data, waiting); if (item == NULL) { return -1; } - item->data = data; - item->recv_mutex = recv_mutex; queue->count += 1; if (queue->first == NULL) { @@ -688,15 +649,21 @@ _channelqueue_put(_channelqueue *queue, _PyCrossInterpreterData *data, queue->last->next = item; } queue->last = item; + + if (waiting != NULL) { + _waiting_acquire(waiting); + } + return 0; } -static _PyCrossInterpreterData * -_channelqueue_get(_channelqueue *queue, PyThread_type_lock *recv_mutex) +static int +_channelqueue_get(_channelqueue *queue, + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) { _channelitem *item = queue->first; if (item == NULL) { - return NULL; + return ERR_CHANNEL_EMPTY; } queue->first = item->next; if (queue->last == item) { @@ -704,18 +671,84 @@ _channelqueue_get(_channelqueue *queue, PyThread_type_lock *recv_mutex) } queue->count -= 1; - return _channelitem_popped(item, recv_mutex); + _channelitem_popped(item, p_data, p_waiting); + return 0; +} + +static int +_channelqueue_find(_channelqueue *queue, _channelitem_id_t itemid, + _channelitem **p_item, _channelitem **p_prev) +{ + _channelitem *prev = NULL; + _channelitem *item = NULL; + if (queue->first != NULL) { + if (_channelitem_ID(queue->first) == itemid) { + item = queue->first; + } + else { + prev = queue->first; + while (prev->next != NULL) { + if (_channelitem_ID(prev->next) == itemid) { + item = prev->next; + break; + } + prev = prev->next; + } + if (item == NULL) { + prev = NULL; + } + } + } + if (p_item != NULL) { + *p_item = item; + } + if (p_prev != NULL) { + *p_prev = prev; + } + return (item != NULL); } static void -_channelqueue_drop_interpreter(_channelqueue *queue, int64_t interp) +_channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) +{ + _channelitem *prev = NULL; + _channelitem *item = NULL; + int found = _channelqueue_find(queue, itemid, &item, &prev); + if (!found) { + return; + } + + assert(item->waiting != NULL); + assert(!item->waiting->received); + if (prev == NULL) { + assert(queue->first == item); + queue->first = item->next; + } + else { + assert(queue->first != item); + assert(prev->next == item); + prev->next = item->next; + } + item->next = NULL; + + if (queue->last == item) { + queue->last = prev; + } + queue->count -= 1; + + _channelitem_popped(item, p_data, p_waiting); +} + +static void +_channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid) { _channelitem *prev = NULL; _channelitem *next = queue->first; while (next != NULL) { _channelitem *item = next; next = item->next; - if (item->data->interp == interp) { + if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { if (prev == NULL) { queue->first = item->next; } @@ -731,18 +764,19 @@ _channelqueue_drop_interpreter(_channelqueue *queue, int64_t interp) } } + /* channel-interpreter associations */ struct _channelend; typedef struct _channelend { struct _channelend *next; - int64_t interp; + int64_t interpid; int open; } _channelend; static _channelend * -_channelend_new(int64_t interp) +_channelend_new(int64_t interpid) { _channelend *end = GLOBAL_MALLOC(_channelend); if (end == NULL) { @@ -750,7 +784,7 @@ _channelend_new(int64_t interp) return NULL; } end->next = NULL; - end->interp = interp; + end->interpid = interpid; end->open = 1; return end; } @@ -772,12 +806,12 @@ _channelend_free_all(_channelend *end) } static _channelend * -_channelend_find(_channelend *first, int64_t interp, _channelend **pprev) +_channelend_find(_channelend *first, int64_t interpid, _channelend **pprev) { _channelend *prev = NULL; _channelend *end = first; while (end != NULL) { - if (end->interp == interp) { + if (end->interpid == interpid) { break; } prev = end; @@ -834,10 +868,10 @@ _channelends_free(_channelends *ends) } static _channelend * -_channelends_add(_channelends *ends, _channelend *prev, int64_t interp, +_channelends_add(_channelends *ends, _channelend *prev, int64_t interpid, int send) { - _channelend *end = _channelend_new(interp); + _channelend *end = _channelend_new(interpid); if (end == NULL) { return NULL; } @@ -863,11 +897,11 @@ _channelends_add(_channelends *ends, _channelend *prev, int64_t interp, } static int -_channelends_associate(_channelends *ends, int64_t interp, int send) +_channelends_associate(_channelends *ends, int64_t interpid, int send) { _channelend *prev; _channelend *end = _channelend_find(send ? ends->send : ends->recv, - interp, &prev); + interpid, &prev); if (end != NULL) { if (!end->open) { return ERR_CHANNEL_CLOSED; @@ -875,7 +909,7 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) // already associated return 0; } - if (_channelends_add(ends, prev, interp, send) == NULL) { + if (_channelends_add(ends, prev, interpid, send) == NULL) { return -1; } return 0; @@ -885,16 +919,20 @@ static int _channelends_is_open(_channelends *ends) { if (ends->numsendopen != 0 || ends->numrecvopen != 0) { + // At least one interpreter is still associated with the channel + // (and hasn't been released). return 1; } + // XXX This is wrong if an end can ever be removed. if (ends->send == NULL && ends->recv == NULL) { + // The channel has never had any interpreters associated with it. return 1; } return 0; } static void -_channelends_close_end(_channelends *ends, _channelend *end, int send) +_channelends_release_end(_channelends *ends, _channelend *end, int send) { end->open = 0; if (send) { @@ -906,51 +944,37 @@ _channelends_close_end(_channelends *ends, _channelend *end, int send) } static int -_channelends_close_interpreter(_channelends *ends, int64_t interp, int which) +_channelends_release_interpreter(_channelends *ends, int64_t interpid, int which) { _channelend *prev; _channelend *end; if (which >= 0) { // send/both - end = _channelend_find(ends->send, interp, &prev); + end = _channelend_find(ends->send, interpid, &prev); if (end == NULL) { // never associated so add it - end = _channelends_add(ends, prev, interp, 1); + end = _channelends_add(ends, prev, interpid, 1); if (end == NULL) { return -1; } } - _channelends_close_end(ends, end, 1); + _channelends_release_end(ends, end, 1); } if (which <= 0) { // recv/both - end = _channelend_find(ends->recv, interp, &prev); + end = _channelend_find(ends->recv, interpid, &prev); if (end == NULL) { // never associated so add it - end = _channelends_add(ends, prev, interp, 0); + end = _channelends_add(ends, prev, interpid, 0); if (end == NULL) { return -1; } } - _channelends_close_end(ends, end, 0); + _channelends_release_end(ends, end, 0); } return 0; } static void -_channelends_drop_interpreter(_channelends *ends, int64_t interp) -{ - _channelend *end; - end = _channelend_find(ends->send, interp, NULL); - if (end != NULL) { - _channelends_close_end(ends, end, 1); - } - end = _channelend_find(ends->recv, interp, NULL); - if (end != NULL) { - _channelends_close_end(ends, end, 0); - } -} - -static void -_channelends_close_all(_channelends *ends, int which, int force) +_channelends_release_all(_channelends *ends, int which, int force) { // XXX Handle the ends. // XXX Handle force is True. @@ -958,16 +982,32 @@ _channelends_close_all(_channelends *ends, int which, int force) // Ensure all the "send"-associated interpreters are closed. _channelend *end; for (end = ends->send; end != NULL; end = end->next) { - _channelends_close_end(ends, end, 1); + _channelends_release_end(ends, end, 1); } // Ensure all the "recv"-associated interpreters are closed. for (end = ends->recv; end != NULL; end = end->next) { - _channelends_close_end(ends, end, 0); + _channelends_release_end(ends, end, 0); + } +} + +static void +_channelends_clear_interpreter(_channelends *ends, int64_t interpid) +{ + // XXX Actually remove the entries? + _channelend *end; + end = _channelend_find(ends->send, interpid, NULL); + if (end != NULL) { + _channelends_release_end(ends, end, 1); + } + end = _channelend_find(ends->recv, interpid, NULL); + if (end != NULL) { + _channelends_release_end(ends, end, 0); } } -/* channels */ + +/* each channel's state */ struct _channel; struct _channel_closing; @@ -980,12 +1020,12 @@ typedef struct _channel { _channelends *ends; int open; struct _channel_closing *closing; -} _PyChannelState; +} _channel_state; -static _PyChannelState * +static _channel_state * _channel_new(PyThread_type_lock mutex) { - _PyChannelState *chan = GLOBAL_MALLOC(_PyChannelState); + _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { return NULL; } @@ -1007,7 +1047,7 @@ _channel_new(PyThread_type_lock mutex) } static void -_channel_free(_PyChannelState *chan) +_channel_free(_channel_state *chan) { _channel_clear_closing(chan); PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1020,8 +1060,8 @@ _channel_free(_PyChannelState *chan) } static int -_channel_add(_PyChannelState *chan, int64_t interp, - _PyCrossInterpreterData *data, PyThread_type_lock recv_mutex) +_channel_add(_channel_state *chan, int64_t interpid, + _PyCrossInterpreterData *data, _waiting_t *waiting) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1030,14 +1070,15 @@ _channel_add(_PyChannelState *chan, int64_t interp, res = ERR_CHANNEL_CLOSED; goto done; } - if (_channelends_associate(chan->ends, interp, 1) != 0) { + if (_channelends_associate(chan->ends, interpid, 1) != 0) { res = ERR_CHANNEL_INTERP_CLOSED; goto done; } - if (_channelqueue_put(chan->queue, data, recv_mutex) != 0) { + if (_channelqueue_put(chan->queue, data, waiting) != 0) { goto done; } + // Any errors past this point must cause a _waiting_release() call. res = 0; done: @@ -1046,8 +1087,8 @@ _channel_add(_PyChannelState *chan, int64_t interp, } static int -_channel_next(_PyChannelState *chan, int64_t interp, - _PyCrossInterpreterData **res) +_channel_next(_channel_state *chan, int64_t interpid, + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) { int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1056,32 +1097,48 @@ _channel_next(_PyChannelState *chan, int64_t interp, err = ERR_CHANNEL_CLOSED; goto done; } - if (_channelends_associate(chan->ends, interp, 0) != 0) { + if (_channelends_associate(chan->ends, interpid, 0) != 0) { err = ERR_CHANNEL_INTERP_CLOSED; goto done; } - PyThread_type_lock recv_mutex = NULL; - _PyCrossInterpreterData *data = _channelqueue_get(chan->queue, &recv_mutex); - if (data == NULL && !PyErr_Occurred() && chan->closing != NULL) { - chan->open = 0; - } - *res = data; + int empty = _channelqueue_get(chan->queue, p_data, p_waiting); + assert(empty == 0 || empty == ERR_CHANNEL_EMPTY); + assert(!PyErr_Occurred()); + if (empty && chan->closing != NULL) { + chan->open = 0; + } + +done: + PyThread_release_lock(chan->mutex); + if (chan->queue->count == 0) { + _channel_finish_closing(chan); + } + return err; +} + +static void +_channel_remove(_channel_state *chan, _channelitem_id_t itemid) +{ + _PyCrossInterpreterData *data = NULL; + _waiting_t *waiting = NULL; + + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + _channelqueue_remove(chan->queue, itemid, &data, &waiting); + PyThread_release_lock(chan->mutex); - if (recv_mutex != NULL) { - PyThread_release_lock(recv_mutex); + (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); + if (waiting != NULL) { + _waiting_release(waiting, 0); } -done: - PyThread_release_lock(chan->mutex); if (chan->queue->count == 0) { _channel_finish_closing(chan); } - return err; } static int -_channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) +_channel_release_interpreter(_channel_state *chan, int64_t interpid, int end) { PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1091,10 +1148,12 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) goto done; } - if (_channelends_close_interpreter(chan->ends, interp, end) != 0) { + if (_channelends_release_interpreter(chan->ends, interpid, end) != 0) { goto done; } chan->open = _channelends_is_open(chan->ends); + // XXX Clear the queue if not empty? + // XXX Activate the "closing" mechanism? res = 0; done: @@ -1102,20 +1161,8 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) return res; } -static void -_channel_drop_interpreter(_PyChannelState *chan, int64_t interp) -{ - PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - - _channelqueue_drop_interpreter(chan->queue, interp); - _channelends_drop_interpreter(chan->ends, interp); - chan->open = _channelends_is_open(chan->ends); - - PyThread_release_lock(chan->mutex); -} - static int -_channel_close_all(_PyChannelState *chan, int end, int force) +_channel_release_all(_channel_state *chan, int end, int force) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1129,12 +1176,13 @@ _channel_close_all(_PyChannelState *chan, int end, int force) res = ERR_CHANNEL_NOT_EMPTY; goto done; } + // XXX Clear the queue? chan->open = 0; // We *could* also just leave these in place, since we've marked // the channel as closed already. - _channelends_close_all(chan->ends, end, force); + _channelends_release_all(chan->ends, end, force); res = 0; done: @@ -1142,25 +1190,39 @@ _channel_close_all(_PyChannelState *chan, int end, int force) return res; } +static void +_channel_clear_interpreter(_channel_state *chan, int64_t interpid) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + + _channelqueue_clear_interpreter(chan->queue, interpid); + _channelends_clear_interpreter(chan->ends, interpid); + chan->open = _channelends_is_open(chan->ends); + + PyThread_release_lock(chan->mutex); +} + + /* the set of channels */ struct _channelref; typedef struct _channelref { - int64_t id; - _PyChannelState *chan; + int64_t cid; + _channel_state *chan; struct _channelref *next; + // The number of ChannelID objects referring to this channel. Py_ssize_t objcount; } _channelref; static _channelref * -_channelref_new(int64_t id, _PyChannelState *chan) +_channelref_new(int64_t cid, _channel_state *chan) { _channelref *ref = GLOBAL_MALLOC(_channelref); if (ref == NULL) { return NULL; } - ref->id = id; + ref->cid = cid; ref->chan = chan; ref->next = NULL; ref->objcount = 0; @@ -1170,7 +1232,7 @@ _channelref_new(int64_t id, _PyChannelState *chan) //static void //_channelref_clear(_channelref *ref) //{ -// ref->id = -1; +// ref->cid = -1; // ref->chan = NULL; // ref->next = NULL; // ref->objcount = 0; @@ -1187,12 +1249,12 @@ _channelref_free(_channelref *ref) } static _channelref * -_channelref_find(_channelref *first, int64_t id, _channelref **pprev) +_channelref_find(_channelref *first, int64_t cid, _channelref **pprev) { _channelref *prev = NULL; _channelref *ref = first; while (ref != NULL) { - if (ref->id == id) { + if (ref->cid == cid) { break; } prev = ref; @@ -1204,6 +1266,7 @@ _channelref_find(_channelref *first, int64_t id, _channelref **pprev) return ref; } + typedef struct _channels { PyThread_type_lock mutex; _channelref *head; @@ -1234,27 +1297,27 @@ _channels_fini(_channels *channels) static int64_t _channels_next_id(_channels *channels) // needs lock { - int64_t id = channels->next_id; - if (id < 0) { + int64_t cid = channels->next_id; + if (cid < 0) { /* overflow */ return -1; } channels->next_id += 1; - return id; + return cid; } static int -_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex, - _PyChannelState **res) +_channels_lookup(_channels *channels, int64_t cid, PyThread_type_lock *pmutex, + _channel_state **res) { int err = -1; - _PyChannelState *chan = NULL; + _channel_state *chan = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); if (pmutex != NULL) { *pmutex = NULL; } - _channelref *ref = _channelref_find(channels->head, id, NULL); + _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { err = ERR_CHANNEL_NOT_FOUND; goto done; @@ -1281,18 +1344,18 @@ _channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex, } static int64_t -_channels_add(_channels *channels, _PyChannelState *chan) +_channels_add(_channels *channels, _channel_state *chan) { int64_t cid = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); // Create a new ref. - int64_t id = _channels_next_id(channels); - if (id < 0) { + int64_t _cid = _channels_next_id(channels); + if (_cid < 0) { cid = ERR_NO_NEXT_CHANNEL_ID; goto done; } - _channelref *ref = _channelref_new(id, chan); + _channelref *ref = _channelref_new(_cid, chan); if (ref == NULL) { goto done; } @@ -1303,17 +1366,17 @@ _channels_add(_channels *channels, _PyChannelState *chan) channels->head = ref; channels->numopen += 1; - cid = id; + cid = _cid; done: PyThread_release_lock(channels->mutex); return cid; } /* forward */ -static int _channel_set_closing(struct _channelref *, PyThread_type_lock); +static int _channel_set_closing(_channelref *, PyThread_type_lock); static int -_channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, +_channels_close(_channels *channels, int64_t cid, _channel_state **pchan, int end, int force) { int res = -1; @@ -1337,7 +1400,7 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, goto done; } else { - int err = _channel_close_all(ref->chan, end, force); + int err = _channel_release_all(ref->chan, end, force); if (err != 0) { if (end == CHANNEL_SEND && err == ERR_CHANNEL_NOT_EMPTY) { if (ref->chan->closing != NULL) { @@ -1379,7 +1442,7 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, static void _channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, - _PyChannelState **pchan) + _channel_state **pchan) { if (ref == channels->head) { channels->head = ref->next; @@ -1396,7 +1459,7 @@ _channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, } static int -_channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) +_channels_remove(_channels *channels, int64_t cid, _channel_state **pchan) { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -1406,7 +1469,7 @@ _channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) } _channelref *prev = NULL; - _channelref *ref = _channelref_find(channels->head, id, &prev); + _channelref *ref = _channelref_find(channels->head, cid, &prev); if (ref == NULL) { res = ERR_CHANNEL_NOT_FOUND; goto done; @@ -1421,12 +1484,12 @@ _channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) } static int -_channels_add_id_object(_channels *channels, int64_t id) +_channels_add_id_object(_channels *channels, int64_t cid) { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - _channelref *ref = _channelref_find(channels->head, id, NULL); + _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { res = ERR_CHANNEL_NOT_FOUND; goto done; @@ -1440,12 +1503,12 @@ _channels_add_id_object(_channels *channels, int64_t id) } static void -_channels_drop_id_object(_channels *channels, int64_t id) +_channels_release_cid_object(_channels *channels, int64_t cid) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); _channelref *prev = NULL; - _channelref *ref = _channelref_find(channels->head, id, &prev); + _channelref *ref = _channelref_find(channels->head, cid, &prev); if (ref == NULL) { // Already destroyed. goto done; @@ -1454,7 +1517,7 @@ _channels_drop_id_object(_channels *channels, int64_t id) // Destroy if no longer used. if (ref->objcount == 0) { - _PyChannelState *chan = NULL; + _channel_state *chan = NULL; _channels_remove_ref(channels, ref, prev, &chan); if (chan != NULL) { _channel_free(chan); @@ -1476,7 +1539,7 @@ _channels_list_all(_channels *channels, int64_t *count) } _channelref *ref = channels->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->id; + ids[i] = ref->cid; } *count = channels->numopen; @@ -1487,29 +1550,30 @@ _channels_list_all(_channels *channels, int64_t *count) } static void -_channels_drop_interpreter(_channels *channels, int64_t interp) +_channels_clear_interpreter(_channels *channels, int64_t interpid) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); _channelref *ref = channels->head; for (; ref != NULL; ref = ref->next) { if (ref->chan != NULL) { - _channel_drop_interpreter(ref->chan, interp); + _channel_clear_interpreter(ref->chan, interpid); } } PyThread_release_lock(channels->mutex); } + /* support for closing non-empty channels */ struct _channel_closing { - struct _channelref *ref; + _channelref *ref; }; static int -_channel_set_closing(struct _channelref *ref, PyThread_type_lock mutex) { - struct _channel *chan = ref->chan; +_channel_set_closing(_channelref *ref, PyThread_type_lock mutex) { + _channel_state *chan = ref->chan; if (chan == NULL) { // already closed return 0; @@ -1533,7 +1597,7 @@ _channel_set_closing(struct _channelref *ref, PyThread_type_lock mutex) { } static void -_channel_clear_closing(struct _channel *chan) { +_channel_clear_closing(_channel_state *chan) { PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (chan->closing != NULL) { GLOBAL_FREE(chan->closing); @@ -1543,7 +1607,7 @@ _channel_clear_closing(struct _channel *chan) { } static void -_channel_finish_closing(struct _channel *chan) { +_channel_finish_closing(_channel_state *chan) { struct _channel_closing *closing = chan->closing; if (closing == NULL) { return; @@ -1555,32 +1619,35 @@ _channel_finish_closing(struct _channel *chan) { _channel_free(chan); } + /* "high"-level channel-related functions */ +// Create a new channel. static int64_t -_channel_create(_channels *channels) +channel_create(_channels *channels) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _PyChannelState *chan = _channel_new(mutex); + _channel_state *chan = _channel_new(mutex); if (chan == NULL) { PyThread_free_lock(mutex); return -1; } - int64_t id = _channels_add(channels, chan); - if (id < 0) { + int64_t cid = _channels_add(channels, chan); + if (cid < 0) { _channel_free(chan); } - return id; + return cid; } +// Completely destroy the channel. static int -_channel_destroy(_channels *channels, int64_t id) +channel_destroy(_channels *channels, int64_t cid) { - _PyChannelState *chan = NULL; - int err = _channels_remove(channels, id, &chan); + _channel_state *chan = NULL; + int err = _channels_remove(channels, cid, &chan); if (err != 0) { return err; } @@ -1590,19 +1657,23 @@ _channel_destroy(_channels *channels, int64_t id) return 0; } +// Push an object onto the channel. +// The current interpreter gets associated with the send end of the channel. +// Optionally request to be notified when it is received. static int -_channel_send(_channels *channels, int64_t id, PyObject *obj, - PyThread_type_lock recv_mutex) +channel_send(_channels *channels, int64_t cid, PyObject *obj, + _waiting_t *waiting) { PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } + int64_t interpid = PyInterpreterState_GetID(interp); // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = NULL; - int err = _channels_lookup(channels, id, &mutex, &chan); + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); if (err != 0) { return err; } @@ -1627,8 +1698,7 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj, } // Add the data to the channel. - int res = _channel_add(chan, PyInterpreterState_GetID(interp), data, - recv_mutex); + int res = _channel_add(chan, interpid, data, waiting); PyThread_release_lock(mutex); if (res != 0) { // We may chain an exception here: @@ -1640,8 +1710,85 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj, return 0; } +// Basically, un-send an object. +static void +channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) +{ + // Look up the channel. + PyThread_type_lock mutex = NULL; + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); + if (err != 0) { + // The channel was already closed, etc. + assert(waiting->status == WAITING_RELEASED); + return; // Ignore the error. + } + assert(chan != NULL); + // Past this point we are responsible for releasing the mutex. + + _channelitem_id_t itemid = _waiting_get_itemid(waiting); + _channel_remove(chan, itemid); + + PyThread_release_lock(mutex); +} + +// Like channel_send(), but strictly wait for the object to be received. +static int +channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, + PY_TIMEOUT_T timeout) +{ + // We use a stack variable here, so we must ensure that &waiting + // is not held by any channel item at the point this function exits. + _waiting_t waiting; + if (_waiting_init(&waiting) < 0) { + assert(PyErr_Occurred()); + return -1; + } + + /* Queue up the object. */ + int res = channel_send(channels, cid, obj, &waiting); + if (res < 0) { + assert(waiting.status == WAITING_NO_STATUS); + goto finally; + } + + /* Wait until the object is received. */ + if (wait_for_lock(waiting.mutex, timeout) < 0) { + assert(PyErr_Occurred()); + _waiting_finish_releasing(&waiting); + /* The send() call is failing now, so make sure the item + won't be received. */ + channel_clear_sent(channels, cid, &waiting); + assert(waiting.status == WAITING_RELEASED); + if (!waiting.received) { + res = -1; + goto finally; + } + // XXX Emit a warning if not a TimeoutError? + PyErr_Clear(); + } + else { + _waiting_finish_releasing(&waiting); + assert(waiting.status == WAITING_RELEASED); + if (!waiting.received) { + res = ERR_CHANNEL_CLOSED_WAITING; + goto finally; + } + } + + /* success! */ + res = 0; + +finally: + _waiting_clear(&waiting); + return res; +} + +// Pop the next object off the channel. Fail if empty. +// The current interpreter gets associated with the recv end of the channel. +// XXX Support a "wait" mutex? static int -_channel_recv(_channels *channels, int64_t id, PyObject **res) +channel_recv(_channels *channels, int64_t cid, PyObject **res) { int err; *res = NULL; @@ -1654,11 +1801,12 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res) } return 0; } + int64_t interpid = PyInterpreterState_GetID(interp); // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = NULL; - err = _channels_lookup(channels, id, &mutex, &chan); + _channel_state *chan = NULL; + err = _channels_lookup(channels, cid, &mutex, &chan); if (err != 0) { return err; } @@ -1667,7 +1815,8 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res) // Pop off the next item from the channel. _PyCrossInterpreterData *data = NULL; - err = _channel_next(chan, PyInterpreterState_GetID(interp), &data); + _waiting_t *waiting = NULL; + err = _channel_next(chan, interpid, &data, &waiting); PyThread_release_lock(mutex); if (err != 0) { return err; @@ -1681,57 +1830,77 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res) PyObject *obj = _PyCrossInterpreterData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); - // It was allocated in _channel_send(), so we free it. + // It was allocated in channel_send(), so we free it. (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); + if (waiting != NULL) { + _waiting_release(waiting, 0); + } return -1; } - // It was allocated in _channel_send(), so we free it. + // It was allocated in channel_send(), so we free it. int release_res = _release_xid_data(data, XID_FREE); if (release_res < 0) { // The source interpreter has been destroyed already. assert(PyErr_Occurred()); Py_DECREF(obj); + if (waiting != NULL) { + _waiting_release(waiting, 0); + } return -1; } + // Notify the sender. + if (waiting != NULL) { + _waiting_release(waiting, 1); + } + *res = obj; return 0; } +// Disallow send/recv for the current interpreter. +// The channel is marked as closed if no other interpreters +// are currently associated. static int -_channel_drop(_channels *channels, int64_t id, int send, int recv) +channel_release(_channels *channels, int64_t cid, int send, int recv) { PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } + int64_t interpid = PyInterpreterState_GetID(interp); // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = NULL; - int err = _channels_lookup(channels, id, &mutex, &chan); + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); if (err != 0) { return err; } // Past this point we are responsible for releasing the mutex. // Close one or both of the two ends. - int res = _channel_close_interpreter(chan, PyInterpreterState_GetID(interp), send-recv); + int res = _channel_release_interpreter(chan, interpid, send-recv); PyThread_release_lock(mutex); return res; } +// Close the channel (for all interpreters). Fail if it's already closed. +// Close immediately if it's empty. Otherwise, disallow sending and +// finally close once empty. Optionally, immediately clear and close it. static int -_channel_close(_channels *channels, int64_t id, int end, int force) +channel_close(_channels *channels, int64_t cid, int end, int force) { - return _channels_close(channels, id, NULL, end, force); + return _channels_close(channels, cid, NULL, end, force); } +// Return true if the identified interpreter is associated +// with the given end of the channel. static int -_channel_is_associated(_channels *channels, int64_t cid, int64_t interp, +channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, int send) { - _PyChannelState *chan = NULL; + _channel_state *chan = NULL; int err = _channels_lookup(channels, cid, NULL, &chan); if (err != 0) { return err; @@ -1741,16 +1910,247 @@ _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, } _channelend *end = _channelend_find(send ? chan->ends->send : chan->ends->recv, - interp, NULL); + interpid, NULL); return (end != NULL && end->open); } + +/* channel info */ + +struct channel_info { + struct { + // 1: closed; -1: closing + int closed; + struct { + Py_ssize_t nsend_only; // not released + Py_ssize_t nsend_only_released; + Py_ssize_t nrecv_only; // not released + Py_ssize_t nrecv_only_released; + Py_ssize_t nboth; // not released + Py_ssize_t nboth_released; + Py_ssize_t nboth_send_released; + Py_ssize_t nboth_recv_released; + } all; + struct { + // 1: associated; -1: released + int send; + int recv; + } cur; + } status; + Py_ssize_t count; +}; + +static int +_channel_get_info(_channels *channels, int64_t cid, struct channel_info *info) +{ + int err = 0; + *info = (struct channel_info){0}; + + // Get the current interpreter. + PyInterpreterState *interp = _get_current_interp(); + if (interp == NULL) { + return -1; + } + Py_ssize_t interpid = PyInterpreterState_GetID(interp); + + // Hold the global lock until we're done. + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + // Find the channel. + _channelref *ref = _channelref_find(channels->head, cid, NULL); + if (ref == NULL) { + err = ERR_CHANNEL_NOT_FOUND; + goto finally; + } + _channel_state *chan = ref->chan; + + // Check if open. + if (chan == NULL) { + info->status.closed = 1; + goto finally; + } + if (!chan->open) { + assert(chan->queue->count == 0); + info->status.closed = 1; + goto finally; + } + if (chan->closing != NULL) { + assert(chan->queue->count > 0); + info->status.closed = -1; + } + else { + info->status.closed = 0; + } + + // Get the number of queued objects. + info->count = chan->queue->count; + + // Get the ends statuses. + assert(info->status.cur.send == 0); + assert(info->status.cur.recv == 0); + _channelend *send = chan->ends->send; + while (send != NULL) { + if (send->interpid == interpid) { + info->status.cur.send = send->open ? 1 : -1; + } + + if (send->open) { + info->status.all.nsend_only += 1; + } + else { + info->status.all.nsend_only_released += 1; + } + send = send->next; + } + _channelend *recv = chan->ends->recv; + while (recv != NULL) { + if (recv->interpid == interpid) { + info->status.cur.recv = recv->open ? 1 : -1; + } + + // XXX This is O(n*n). Why do we have 2 linked lists? + _channelend *send = chan->ends->send; + while (send != NULL) { + if (send->interpid == recv->interpid) { + break; + } + send = send->next; + } + if (send == NULL) { + if (recv->open) { + info->status.all.nrecv_only += 1; + } + else { + info->status.all.nrecv_only_released += 1; + } + } + else { + if (recv->open) { + if (send->open) { + info->status.all.nboth += 1; + info->status.all.nsend_only -= 1; + } + else { + info->status.all.nboth_recv_released += 1; + info->status.all.nsend_only_released -= 1; + } + } + else { + if (send->open) { + info->status.all.nboth_send_released += 1; + info->status.all.nsend_only -= 1; + } + else { + info->status.all.nboth_released += 1; + info->status.all.nsend_only_released -= 1; + } + } + } + recv = recv->next; + } + +finally: + PyThread_release_lock(channels->mutex); + return err; +} + +PyDoc_STRVAR(channel_info_doc, +"ChannelInfo\n\ +\n\ +A named tuple of a channel's state."); + +static PyStructSequence_Field channel_info_fields[] = { + {"open", "both ends are open"}, + {"closing", "send is closed, recv is non-empty"}, + {"closed", "both ends are closed"}, + {"count", "queued objects"}, + + {"num_interp_send", "interpreters bound to the send end"}, + {"num_interp_send_released", + "interpreters bound to the send end and released"}, + + {"num_interp_recv", "interpreters bound to the send end"}, + {"num_interp_recv_released", + "interpreters bound to the send end and released"}, + + {"num_interp_both", "interpreters bound to both ends"}, + {"num_interp_both_released", + "interpreters bound to both ends and released_from_both"}, + {"num_interp_both_send_released", + "interpreters bound to both ends and released_from_the send end"}, + {"num_interp_both_recv_released", + "interpreters bound to both ends and released_from_the recv end"}, + + {"send_associated", "current interpreter is bound to the send end"}, + {"send_released", "current interpreter *was* bound to the send end"}, + {"recv_associated", "current interpreter is bound to the recv end"}, + {"recv_released", "current interpreter *was* bound to the recv end"}, + {0} +}; + +static PyStructSequence_Desc channel_info_desc = { + .name = MODULE_NAME_STR ".ChannelInfo", + .doc = channel_info_doc, + .fields = channel_info_fields, + .n_in_sequence = 8, +}; + +static PyObject * +new_channel_info(PyObject *mod, struct channel_info *info) +{ + module_state *state = get_module_state(mod); + if (state == NULL) { + return NULL; + } + + assert(state->ChannelInfoType != NULL); + PyObject *self = PyStructSequence_New(state->ChannelInfoType); + if (self == NULL) { + return NULL; + } + + int pos = 0; +#define SET_BOOL(val) \ + PyStructSequence_SET_ITEM(self, pos++, \ + Py_NewRef(val ? Py_True : Py_False)) +#define SET_COUNT(val) \ + do { \ + PyObject *obj = PyLong_FromLongLong(val); \ + if (obj == NULL) { \ + Py_CLEAR(self); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(self, pos++, obj); \ + } while(0) + SET_BOOL(info->status.closed == 0); + SET_BOOL(info->status.closed == -1); + SET_BOOL(info->status.closed == 1); + SET_COUNT(info->count); + SET_COUNT(info->status.all.nsend_only); + SET_COUNT(info->status.all.nsend_only_released); + SET_COUNT(info->status.all.nrecv_only); + SET_COUNT(info->status.all.nrecv_only_released); + SET_COUNT(info->status.all.nboth); + SET_COUNT(info->status.all.nboth_released); + SET_COUNT(info->status.all.nboth_send_released); + SET_COUNT(info->status.all.nboth_recv_released); + SET_BOOL(info->status.cur.send == 1); + SET_BOOL(info->status.cur.send == -1); + SET_BOOL(info->status.cur.recv == 1); + SET_BOOL(info->status.cur.recv == -1); +#undef SET_COUNT +#undef SET_BOOL + assert(!PyErr_Occurred()); + return self; +} + + /* ChannelID class */ typedef struct channelid { PyObject_HEAD - int64_t id; + int64_t cid; int end; int resolve; _channels *channels; @@ -1771,7 +2171,7 @@ channel_id_converter(PyObject *arg, void *ptr) module_state *state = get_module_state(data->module); assert(state != NULL); if (PyObject_TypeCheck(arg, state->ChannelIDType)) { - cid = ((channelid *)arg)->id; + cid = ((channelid *)arg)->cid; end = ((channelid *)arg)->end; } else if (PyIndex_Check(arg)) { @@ -1806,7 +2206,7 @@ newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, if (self == NULL) { return -1; } - self->id = cid; + self->cid = cid; self->end = end; self->resolve = resolve; self->channels = channels; @@ -1871,22 +2271,22 @@ _channelid_new(PyObject *mod, PyTypeObject *cls, end = CHANNEL_RECV; } - PyObject *id = NULL; + PyObject *cidobj = NULL; int err = newchannelid(cls, cid, end, _global_channels(), force, resolve, - (channelid **)&id); + (channelid **)&cidobj); if (handle_channel_error(err, mod, cid)) { - assert(id == NULL); + assert(cidobj == NULL); return NULL; } - assert(id != NULL); - return id; + assert(cidobj != NULL); + return cidobj; } static void channelid_dealloc(PyObject *self) { - int64_t cid = ((channelid *)self)->id; + int64_t cid = ((channelid *)self)->cid; _channels *channels = ((channelid *)self)->channels; PyTypeObject *tp = Py_TYPE(self); @@ -1899,7 +2299,7 @@ channelid_dealloc(PyObject *self) // like we do for _abc._abc_data? Py_DECREF(tp); - _channels_drop_id_object(channels, cid); + _channels_release_cid_object(channels, cid); } static PyObject * @@ -1908,44 +2308,44 @@ channelid_repr(PyObject *self) PyTypeObject *type = Py_TYPE(self); const char *name = _PyType_Name(type); - channelid *cid = (channelid *)self; + channelid *cidobj = (channelid *)self; const char *fmt; - if (cid->end == CHANNEL_SEND) { + if (cidobj->end == CHANNEL_SEND) { fmt = "%s(%" PRId64 ", send=True)"; } - else if (cid->end == CHANNEL_RECV) { + else if (cidobj->end == CHANNEL_RECV) { fmt = "%s(%" PRId64 ", recv=True)"; } else { fmt = "%s(%" PRId64 ")"; } - return PyUnicode_FromFormat(fmt, name, cid->id); + return PyUnicode_FromFormat(fmt, name, cidobj->cid); } static PyObject * channelid_str(PyObject *self) { - channelid *cid = (channelid *)self; - return PyUnicode_FromFormat("%" PRId64 "", cid->id); + channelid *cidobj = (channelid *)self; + return PyUnicode_FromFormat("%" PRId64 "", cidobj->cid); } static PyObject * channelid_int(PyObject *self) { - channelid *cid = (channelid *)self; - return PyLong_FromLongLong(cid->id); + channelid *cidobj = (channelid *)self; + return PyLong_FromLongLong(cidobj->cid); } static Py_hash_t channelid_hash(PyObject *self) { - channelid *cid = (channelid *)self; - PyObject *id = PyLong_FromLongLong(cid->id); - if (id == NULL) { + channelid *cidobj = (channelid *)self; + PyObject *pyid = PyLong_FromLongLong(cidobj->cid); + if (pyid == NULL) { return -1; } - Py_hash_t hash = PyObject_Hash(id); - Py_DECREF(id); + Py_hash_t hash = PyObject_Hash(pyid); + Py_DECREF(pyid); return hash; } @@ -1971,11 +2371,11 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) goto done; } - channelid *cid = (channelid *)self; + channelid *cidobj = (channelid *)self; int equal; if (PyObject_TypeCheck(other, state->ChannelIDType)) { - channelid *othercid = (channelid *)other; - equal = (cid->end == othercid->end) && (cid->id == othercid->id); + channelid *othercidobj = (channelid *)other; + equal = (cidobj->end == othercidobj->end) && (cidobj->cid == othercidobj->cid); } else if (PyLong_Check(other)) { /* Fast path */ @@ -1984,10 +2384,10 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) if (othercid == -1 && PyErr_Occurred()) { goto done; } - equal = !overflow && (othercid >= 0) && (cid->id == othercid); + equal = !overflow && (othercid >= 0) && (cidobj->cid == othercid); } else if (PyNumber_Check(other)) { - PyObject *pyid = PyLong_FromLongLong(cid->id); + PyObject *pyid = PyLong_FromLongLong(cidobj->cid); if (pyid == NULL) { goto done; } @@ -2012,16 +2412,16 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) return res; } -static PyTypeObject * _get_current_channel_end_type(int end); +static PyTypeObject * _get_current_channelend_type(int end); static PyObject * -_channel_from_cid(PyObject *cid, int end) +_channelobj_from_cidobj(PyObject *cidobj, int end) { - PyObject *cls = (PyObject *)_get_current_channel_end_type(end); + PyObject *cls = (PyObject *)_get_current_channelend_type(end); if (cls == NULL) { return NULL; } - PyObject *chan = PyObject_CallFunctionObjArgs(cls, cid, NULL); + PyObject *chan = PyObject_CallFunctionObjArgs(cls, cidobj, NULL); Py_DECREF(cls); if (chan == NULL) { return NULL; @@ -2030,7 +2430,7 @@ _channel_from_cid(PyObject *cid, int end) } struct _channelid_xid { - int64_t id; + int64_t cid; int end; int resolve; }; @@ -2038,10 +2438,11 @@ struct _channelid_xid { static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); // It might not be imported yet, so we can't use _get_current_module(). - PyObject *mod = PyImport_ImportModule(MODULE_NAME); + PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { return NULL; } @@ -2052,16 +2453,16 @@ _channelid_from_xid(_PyCrossInterpreterData *data) } // Note that we do not preserve the "resolve" flag. - PyObject *cid = NULL; - int err = newchannelid(state->ChannelIDType, xid->id, xid->end, + PyObject *cidobj = NULL; + int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, - (channelid **)&cid); + (channelid **)&cidobj); if (err != 0) { - assert(cid == NULL); - (void)handle_channel_error(err, mod, xid->id); + assert(cidobj == NULL); + (void)handle_channel_error(err, mod, xid->cid); goto done; } - assert(cid != NULL); + assert(cidobj != NULL); if (xid->end == 0) { goto done; } @@ -2070,17 +2471,17 @@ _channelid_from_xid(_PyCrossInterpreterData *data) } /* Try returning a high-level channel end but fall back to the ID. */ - PyObject *chan = _channel_from_cid(cid, xid->end); + PyObject *chan = _channelobj_from_cidobj(cidobj, xid->end); if (chan == NULL) { PyErr_Clear(); goto done; } - Py_DECREF(cid); - cid = chan; + Py_DECREF(cidobj); + cidobj = chan; done: Py_DECREF(mod); - return cid; + return cidobj; } static int @@ -2094,8 +2495,9 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj, { return -1; } - struct _channelid_xid *xid = (struct _channelid_xid *)data->data; - xid->id = ((channelid *)obj)->id; + struct _channelid_xid *xid = \ + (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + xid->cid = ((channelid *)obj)->cid; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; return 0; @@ -2105,30 +2507,30 @@ static PyObject * channelid_end(PyObject *self, void *end) { int force = 1; - channelid *cid = (channelid *)self; + channelid *cidobj = (channelid *)self; if (end != NULL) { - PyObject *id = NULL; - int err = newchannelid(Py_TYPE(self), cid->id, *(int *)end, - cid->channels, force, cid->resolve, - (channelid **)&id); + PyObject *obj = NULL; + int err = newchannelid(Py_TYPE(self), cidobj->cid, *(int *)end, + cidobj->channels, force, cidobj->resolve, + (channelid **)&obj); if (err != 0) { - assert(id == NULL); + assert(obj == NULL); PyObject *mod = get_module_from_type(Py_TYPE(self)); if (mod == NULL) { return NULL; } - (void)handle_channel_error(err, mod, cid->id); + (void)handle_channel_error(err, mod, cidobj->cid); Py_DECREF(mod); return NULL; } - assert(id != NULL); - return id; + assert(obj != NULL); + return obj; } - if (cid->end == CHANNEL_SEND) { + if (cidobj->end == CHANNEL_SEND) { return PyUnicode_InternFromString("send"); } - if (cid->end == CHANNEL_RECV) { + if (cidobj->end == CHANNEL_RECV) { return PyUnicode_InternFromString("recv"); } return PyUnicode_InternFromString("both"); @@ -2150,7 +2552,7 @@ static PyGetSetDef channelid_getsets[] = { PyDoc_STRVAR(channelid_doc, "A channel ID identifies a channel and may be used as an int."); -static PyType_Slot ChannelIDType_slots[] = { +static PyType_Slot channelid_typeslots[] = { {Py_tp_dealloc, (destructor)channelid_dealloc}, {Py_tp_doc, (void *)channelid_doc}, {Py_tp_repr, (reprfunc)channelid_repr}, @@ -2164,21 +2566,40 @@ static PyType_Slot ChannelIDType_slots[] = { {0, NULL}, }; -static PyType_Spec ChannelIDType_spec = { - .name = MODULE_NAME ".ChannelID", +static PyType_Spec channelid_typespec = { + .name = MODULE_NAME_STR ".ChannelID", .basicsize = sizeof(channelid), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .slots = ChannelIDType_slots, + .slots = channelid_typeslots, }; +static PyTypeObject * +add_channelid_type(PyObject *mod) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &channelid_typespec, NULL); + if (cls == NULL) { + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return NULL; + } + if (ensure_xid_class(cls, _channelid_shared) < 0) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + /* SendChannel and RecvChannel classes */ // XXX Use a new __xid__ protocol instead? static PyTypeObject * -_get_current_channel_end_type(int end) +_get_current_channelend_type(int end) { module_state *state = _get_current_module_state(); if (state == NULL) { @@ -2193,10 +2614,11 @@ _get_current_channel_end_type(int end) cls = state->recv_channel_type; } if (cls == NULL) { - PyObject *highlevel = PyImport_ImportModule("interpreters"); + // Force the module to be loaded, to register the type. + PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters"); + highlevel = PyImport_ImportModule("test.support.interpreters.channels"); if (highlevel == NULL) { return NULL; } @@ -2214,24 +2636,24 @@ _get_current_channel_end_type(int end) } static PyObject * -_channel_end_from_xid(_PyCrossInterpreterData *data) +_channelend_from_xid(_PyCrossInterpreterData *data) { - channelid *cid = (channelid *)_channelid_from_xid(data); - if (cid == NULL) { + channelid *cidobj = (channelid *)_channelid_from_xid(data); + if (cidobj == NULL) { return NULL; } - PyTypeObject *cls = _get_current_channel_end_type(cid->end); + PyTypeObject *cls = _get_current_channelend_type(cidobj->end); if (cls == NULL) { - Py_DECREF(cid); + Py_DECREF(cidobj); return NULL; } - PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)cid); - Py_DECREF(cid); + PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)cidobj); + Py_DECREF(cidobj); return obj; } static int -_channel_end_shared(PyThreadState *tstate, PyObject *obj, +_channelend_shared(PyThreadState *tstate, PyObject *obj, _PyCrossInterpreterData *data) { PyObject *cidobj = PyObject_GetAttrString(obj, "_id"); @@ -2243,38 +2665,47 @@ _channel_end_shared(PyThreadState *tstate, PyObject *obj, if (res < 0) { return -1; } - data->new_object = _channel_end_from_xid; + _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid); return 0; } static int -set_channel_end_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) +set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) { module_state *state = get_module_state(mod); if (state == NULL) { return -1; } - struct xid_class_registry *xid_classes = &state->xid_classes; - if (state->send_channel_type != NULL - || state->recv_channel_type != NULL) - { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + // Clear the old values if the .py module was reloaded. + if (state->send_channel_type != NULL) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + } + if (state->recv_channel_type != NULL) { + (void)clear_xid_class(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); } + + // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - - if (register_xid_class(send, _channel_end_shared, xid_classes)) { + if (ensure_xid_class(send, _channelend_shared) < 0) { + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } - if (register_xid_class(recv, _channel_end_shared, xid_classes)) { + if (ensure_xid_class(recv, _channelend_shared) < 0) { + (void)clear_xid_class(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } return 0; } + /* module level code ********************************************************/ /* globals is the process-global state for the module. It holds all @@ -2330,15 +2761,15 @@ clear_interpreter(void *data) } PyInterpreterState *interp = (PyInterpreterState *)data; assert(interp == _get_current_interp()); - int64_t id = PyInterpreterState_GetID(interp); - _channels_drop_interpreter(&_globals.channels, id); + int64_t interpid = PyInterpreterState_GetID(interp); + _channels_clear_interpreter(&_globals.channels, interpid); } static PyObject * -channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) +channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) { - int64_t cid = _channel_create(&_globals.channels); + int64_t cid = channel_create(&_globals.channels); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2347,30 +2778,30 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) if (state == NULL) { return NULL; } - PyObject *id = NULL; + PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, cid, 0, &_globals.channels, 0, 0, - (channelid **)&id); + (channelid **)&cidobj); if (handle_channel_error(err, self, cid)) { - assert(id == NULL); - err = _channel_destroy(&_globals.channels, cid); + assert(cidobj == NULL); + err = channel_destroy(&_globals.channels, cid); if (handle_channel_error(err, self, cid)) { // XXX issue a warning? } return NULL; } - assert(id != NULL); - assert(((channelid *)id)->channels != NULL); - return id; + assert(cidobj != NULL); + assert(((channelid *)cidobj)->channels != NULL); + return cidobj; } -PyDoc_STRVAR(channel_create_doc, +PyDoc_STRVAR(channelsmod_create_doc, "channel_create() -> cid\n\ \n\ Create a new cross-interpreter channel and return a unique generated ID."); static PyObject * -channel_destroy(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", NULL}; int64_t cid; @@ -2383,21 +2814,21 @@ channel_destroy(PyObject *self, PyObject *args, PyObject *kwds) } cid = cid_data.cid; - int err = _channel_destroy(&_globals.channels, cid); + int err = channel_destroy(&_globals.channels, cid); if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(channel_destroy_doc, +PyDoc_STRVAR(channelsmod_destroy_doc, "channel_destroy(cid)\n\ \n\ Close and finalize the channel. Afterward attempts to use the channel\n\ will behave as though it never existed."); static PyObject * -channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; int64_t *cids = _channels_list_all(&_globals.channels, &count); @@ -2419,17 +2850,17 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *id = NULL; + PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, *cur, 0, &_globals.channels, 0, 0, - (channelid **)&id); + (channelid **)&cidobj); if (handle_channel_error(err, self, *cur)) { - assert(id == NULL); + assert(cidobj == NULL); Py_SETREF(ids, NULL); break; } - assert(id != NULL); - PyList_SET_ITEM(ids, (Py_ssize_t)i, id); + assert(cidobj != NULL); + PyList_SET_ITEM(ids, (Py_ssize_t)i, cidobj); } finally: @@ -2437,13 +2868,13 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) return ids; } -PyDoc_STRVAR(channel_list_all_doc, +PyDoc_STRVAR(channelsmod_list_all_doc, "channel_list_all() -> [cid]\n\ \n\ Return the list of all IDs for active channels."); static PyObject * -channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", NULL}; int64_t cid; /* Channel ID */ @@ -2451,8 +2882,8 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) .module = self, }; int send = 0; /* Send or receive end? */ - int64_t id; - PyObject *ids, *id_obj; + int64_t interpid; + PyObject *ids, *interpid_obj; PyInterpreterState *interp; if (!PyArg_ParseTupleAndKeywords( @@ -2469,20 +2900,20 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = PyInterpreterState_GetID(interp); - assert(id >= 0); - int res = _channel_is_associated(&_globals.channels, cid, id, send); + interpid = PyInterpreterState_GetID(interp); + assert(interpid >= 0); + int res = channel_is_associated(&_globals.channels, cid, interpid, send); if (res < 0) { (void)handle_channel_error(res, self, cid); goto except; } if (res) { - id_obj = PyInterpreterState_GetIDObject(interp); - if (id_obj == NULL) { + interpid_obj = _PyInterpreterState_GetIDObject(interp); + if (interpid_obj == NULL) { goto except; } - res = PyList_Insert(ids, 0, id_obj); - Py_DECREF(id_obj); + res = PyList_Insert(ids, 0, interpid_obj); + Py_DECREF(interpid_obj); if (res < 0) { goto except; } @@ -2499,7 +2930,7 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) return ids; } -PyDoc_STRVAR(channel_list_interpreters_doc, +PyDoc_STRVAR(channelsmod_list_interpreters_doc, "channel_list_interpreters(cid, *, send) -> [id]\n\ \n\ Return the list of all interpreter IDs associated with an end of the channel.\n\ @@ -2509,121 +2940,100 @@ receive end."); static PyObject * -channel_send(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - // XXX Add a timeout arg. - static char *kwlist[] = {"cid", "obj", "blocking", NULL}; - int64_t cid; + static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; int blocking = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|$p:channel_send", kwlist, + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &blocking)) { + &blocking, &timeout_obj)) { return NULL; } - cid = cid_data.cid; - - if (blocking) { - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - PyErr_NoMemory(); - return NULL; - } - PyThread_acquire_lock(mutex, WAIT_LOCK); - /* Queue up the object. */ - int err = _channel_send(&_globals.channels, cid, obj, mutex); - if (handle_channel_error(err, self, cid)) { - PyThread_release_lock(mutex); - return NULL; - } + int64_t cid = cid_data.cid; + PY_TIMEOUT_T timeout; + if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { + return NULL; + } - /* Wait until the object is received. */ - wait_for_lock(mutex); + /* Queue up the object. */ + int err = 0; + if (blocking) { + err = channel_send_wait(&_globals.channels, cid, obj, timeout); } else { - /* Queue up the object. */ - int err = _channel_send(&_globals.channels, cid, obj, NULL); - if (handle_channel_error(err, self, cid)) { - return NULL; - } + err = channel_send(&_globals.channels, cid, obj, NULL); + } + if (handle_channel_error(err, self, cid)) { + return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(channel_send_doc, -"channel_send(cid, obj, blocking=True)\n\ +PyDoc_STRVAR(channelsmod_send_doc, +"channel_send(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's data to the channel's queue.\n\ By default this waits for the object to be received."); static PyObject * -channel_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", NULL}; - int64_t cid; + static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; int blocking = 1; + PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|$p:channel_send_buffer", kwlist, + "O&O|$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &blocking)) { + &blocking, &timeout_obj)) { + return NULL; + } + + int64_t cid = cid_data.cid; + PY_TIMEOUT_T timeout; + if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } - cid = cid_data.cid; PyObject *tempobj = PyMemoryView_FromObject(obj); if (tempobj == NULL) { return NULL; } + /* Queue up the object. */ + int err = 0; if (blocking) { - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - Py_DECREF(tempobj); - PyErr_NoMemory(); - return NULL; - } - PyThread_acquire_lock(mutex, WAIT_LOCK); - - /* Queue up the buffer. */ - int err = _channel_send(&_globals.channels, cid, tempobj, mutex); - Py_DECREF(tempobj); - if (handle_channel_error(err, self, cid)) { - PyThread_acquire_lock(mutex, WAIT_LOCK); - return NULL; - } - - /* Wait until the buffer is received. */ - wait_for_lock(mutex); + err = channel_send_wait(&_globals.channels, cid, tempobj, timeout); } else { - /* Queue up the buffer. */ - int err = _channel_send(&_globals.channels, cid, tempobj, NULL); - Py_DECREF(tempobj); - if (handle_channel_error(err, self, cid)) { - return NULL; - } + err = channel_send(&_globals.channels, cid, tempobj, NULL); + } + Py_DECREF(tempobj); + if (handle_channel_error(err, self, cid)) { + return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(channel_send_buffer_doc, -"channel_send_buffer(cid, obj, blocking=True)\n\ +PyDoc_STRVAR(channelsmod_send_buffer_doc, +"channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's buffer to the channel's queue.\n\ By default this waits for the object to be received."); static PyObject * -channel_recv(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "default", NULL}; int64_t cid; @@ -2638,7 +3048,7 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int err = _channel_recv(&_globals.channels, cid, &obj); + int err = channel_recv(&_globals.channels, cid, &obj); if (handle_channel_error(err, self, cid)) { return NULL; } @@ -2655,7 +3065,7 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds) return obj; } -PyDoc_STRVAR(channel_recv_doc, +PyDoc_STRVAR(channelsmod_recv_doc, "channel_recv(cid, [default]) -> obj\n\ \n\ Return a new object from the data at the front of the channel's queue.\n\ @@ -2664,7 +3074,7 @@ If there is nothing to receive then raise ChannelEmptyError, unless\n\ a default value is provided. In that case return it."); static PyObject * -channel_close(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_close(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; int64_t cid; @@ -2682,14 +3092,14 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds) } cid = cid_data.cid; - int err = _channel_close(&_globals.channels, cid, send-recv, force); + int err = channel_close(&_globals.channels, cid, send-recv, force); if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(channel_close_doc, +PyDoc_STRVAR(channelsmod_close_doc, "channel_close(cid, *, send=None, recv=None, force=False)\n\ \n\ Close the channel for all interpreters.\n\ @@ -2717,7 +3127,7 @@ Once the channel's ID has no more ref counts in any interpreter\n\ the channel will be destroyed."); static PyObject * -channel_release(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_release(PyObject *self, PyObject *args, PyObject *kwds) { // Note that only the current interpreter is affected. static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; @@ -2743,14 +3153,14 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Handle force is True. // XXX Fix implicit release. - int err = _channel_drop(&_globals.channels, cid, send, recv); + int err = channel_release(&_globals.channels, cid, send, recv); if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(channel_release_doc, +PyDoc_STRVAR(channelsmod_release_doc, "channel_release(cid, *, send=None, recv=None, force=True)\n\ \n\ Close the channel for the current interpreter. 'send' and 'recv'\n\ @@ -2758,7 +3168,34 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ ends are closed. Closing an already closed end is a noop."); static PyObject * -channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod_get_info(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:_get_info", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + struct channel_info info; + int err = _channel_get_info(&_globals.channels, cid, &info); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + return new_channel_info(self, &info); +} + +PyDoc_STRVAR(channelsmod_get_info_doc, +"get_info(cid)\n\ +\n\ +Return details about the channel."); + +static PyObject * +channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { module_state *state = get_module_state(self); if (state == NULL) { @@ -2774,7 +3211,7 @@ channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) } static PyObject * -channel__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) +channelsmod__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"send", "recv", NULL}; PyObject *send; @@ -2795,7 +3232,7 @@ channel__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) PyTypeObject *cls_send = (PyTypeObject *)send; PyTypeObject *cls_recv = (PyTypeObject *)recv; - if (set_channel_end_types(self, cls_send, cls_recv) < 0) { + if (set_channelend_types(self, cls_send, cls_recv) < 0) { return NULL; } @@ -2803,27 +3240,29 @@ channel__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) } static PyMethodDef module_functions[] = { - {"create", channel_create, - METH_NOARGS, channel_create_doc}, - {"destroy", _PyCFunction_CAST(channel_destroy), - METH_VARARGS | METH_KEYWORDS, channel_destroy_doc}, - {"list_all", channel_list_all, - METH_NOARGS, channel_list_all_doc}, - {"list_interpreters", _PyCFunction_CAST(channel_list_interpreters), - METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, - {"send", _PyCFunction_CAST(channel_send), - METH_VARARGS | METH_KEYWORDS, channel_send_doc}, - {"send_buffer", _PyCFunction_CAST(channel_send_buffer), - METH_VARARGS | METH_KEYWORDS, channel_send_buffer_doc}, - {"recv", _PyCFunction_CAST(channel_recv), - METH_VARARGS | METH_KEYWORDS, channel_recv_doc}, - {"close", _PyCFunction_CAST(channel_close), - METH_VARARGS | METH_KEYWORDS, channel_close_doc}, - {"release", _PyCFunction_CAST(channel_release), - METH_VARARGS | METH_KEYWORDS, channel_release_doc}, - {"_channel_id", _PyCFunction_CAST(channel__channel_id), + {"create", channelsmod_create, + METH_NOARGS, channelsmod_create_doc}, + {"destroy", _PyCFunction_CAST(channelsmod_destroy), + METH_VARARGS | METH_KEYWORDS, channelsmod_destroy_doc}, + {"list_all", channelsmod_list_all, + METH_NOARGS, channelsmod_list_all_doc}, + {"list_interpreters", _PyCFunction_CAST(channelsmod_list_interpreters), + METH_VARARGS | METH_KEYWORDS, channelsmod_list_interpreters_doc}, + {"send", _PyCFunction_CAST(channelsmod_send), + METH_VARARGS | METH_KEYWORDS, channelsmod_send_doc}, + {"send_buffer", _PyCFunction_CAST(channelsmod_send_buffer), + METH_VARARGS | METH_KEYWORDS, channelsmod_send_buffer_doc}, + {"recv", _PyCFunction_CAST(channelsmod_recv), + METH_VARARGS | METH_KEYWORDS, channelsmod_recv_doc}, + {"close", _PyCFunction_CAST(channelsmod_close), + METH_VARARGS | METH_KEYWORDS, channelsmod_close_doc}, + {"release", _PyCFunction_CAST(channelsmod_release), + METH_VARARGS | METH_KEYWORDS, channelsmod_release_doc}, + {"get_info", _PyCFunction_CAST(channelsmod_get_info), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_info_doc}, + {"_channel_id", _PyCFunction_CAST(channelsmod__channel_id), METH_VARARGS | METH_KEYWORDS, NULL}, - {"_register_end_types", _PyCFunction_CAST(channel__register_end_types), + {"_register_end_types", _PyCFunction_CAST(channelsmod__register_end_types), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */ @@ -2842,13 +3281,11 @@ module_exec(PyObject *mod) if (_globals_init() != 0) { return -1; } - struct xid_class_registry *xid_classes = NULL; module_state *state = get_module_state(mod); if (state == NULL) { goto error; } - xid_classes = &state->xid_classes; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -2857,20 +3294,18 @@ module_exec(PyObject *mod) /* Add other types */ - // ChannelID - state->ChannelIDType = add_new_type( - mod, &ChannelIDType_spec, _channelid_shared, xid_classes); - if (state->ChannelIDType == NULL) { + // ChannelInfo + state->ChannelInfoType = PyStructSequence_NewType(&channel_info_desc); + if (state->ChannelInfoType == NULL) { goto error; } - - state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, - xid_classes); - if (state->XIBufferViewType == NULL) { + if (PyModule_AddType(mod, state->ChannelInfoType) < 0) { goto error; } - if (register_builtin_xid_types(xid_classes) < 0) { + // ChannelID + state->ChannelIDType = add_channelid_type(mod); + if (state->ChannelIDType == NULL) { goto error; } @@ -2881,8 +3316,8 @@ module_exec(PyObject *mod) return 0; error: - if (xid_classes != NULL) { - clear_xid_class_registry(xid_classes); + if (state != NULL) { + clear_xid_types(state); } _globals_fini(); return -1; @@ -2891,6 +3326,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -2909,9 +3345,6 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); return 0; @@ -2923,9 +3356,6 @@ module_free(void *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); @@ -2934,7 +3364,7 @@ module_free(void *mod) static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, + .m_name = MODULE_NAME_STR, .m_doc = module_doc, .m_size = sizeof(module_state), .m_methods = module_functions, @@ -2945,7 +3375,7 @@ static struct PyModuleDef moduledef = { }; PyMODINIT_FUNC -PyInit__xxinterpchannels(void) +MODINIT_FUNC_NAME(void) { return PyModuleDef_Init(&moduledef); } diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c new file mode 100644 index 00000000000000..c99111b861cc32 --- /dev/null +++ b/Modules/_interpqueuesmodule.c @@ -0,0 +1,1882 @@ +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_crossinterp.h" // struct _xid + +#define REGISTERS_HEAP_TYPES +#include "_interpreters_common.h" +#undef REGISTERS_HEAP_TYPES + + +#define MODULE_NAME _interpqueues +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) + + +#define GLOBAL_MALLOC(TYPE) \ + PyMem_RawMalloc(sizeof(TYPE)) +#define GLOBAL_FREE(VAR) \ + PyMem_RawFree(VAR) + + +#define XID_IGNORE_EXC 1 +#define XID_FREE 2 + +static int +_release_xid_data(_PyCrossInterpreterData *data, int flags) +{ + int ignoreexc = flags & XID_IGNORE_EXC; + PyObject *exc; + if (ignoreexc) { + exc = PyErr_GetRaisedException(); + } + int res; + if (flags & XID_FREE) { + res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + } + else { + res = _PyCrossInterpreterData_Release(data); + } + if (res < 0) { + /* The owning interpreter is already destroyed. */ + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (flags & XID_FREE) { + /* Either way, we free the data. */ + } + if (ignoreexc) { + PyErr_SetRaisedException(exc); + } + return res; +} + + +static PyInterpreterState * +_get_current_interp(void) +{ + // PyInterpreterState_Get() aborts if lookup fails, so don't need + // to check the result for NULL. + return PyInterpreterState_Get(); +} + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +struct idarg_int64_converter_data { + // input: + const char *label; + // output: + int64_t id; +}; + +static int +idarg_int64_converter(PyObject *arg, void *ptr) +{ + int64_t id; + struct idarg_int64_converter_data *data = ptr; + + const char *label = data->label; + if (label == NULL) { + label = "ID"; + } + + if (PyIndex_Check(arg)) { + int overflow = 0; + id = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (id == -1 && PyErr_Occurred()) { + return 0; + } + else if (id == -1 && overflow == 1) { + PyErr_Format(PyExc_OverflowError, + "max %s is %lld, got %R", label, INT64_MAX, arg); + return 0; + } + else if (id < 0) { + PyErr_Format(PyExc_ValueError, + "%s must be a non-negative int, got %R", label, arg); + return 0; + } + } + else { + PyErr_Format(PyExc_TypeError, + "%s must be an int, got %.100s", + label, Py_TYPE(arg)->tp_name); + return 0; + } + data->id = id; + return 1; +} + + +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queues"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + + +/* module state *************************************************************/ + +typedef struct { + /* external types (added at runtime by interpreters module) */ + PyTypeObject *queue_type; + + /* QueueError (and its subclasses) */ + PyObject *QueueError; + PyObject *QueueNotFoundError; + PyObject *QueueEmpty; + PyObject *QueueFull; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* external types */ + Py_VISIT(state->queue_type); + + /* QueueError */ + Py_VISIT(state->QueueError); + Py_VISIT(state->QueueNotFoundError); + Py_VISIT(state->QueueEmpty); + Py_VISIT(state->QueueFull); + + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* external types */ + if (state->queue_type != NULL) { + (void)clear_xid_class(state->queue_type); + } + Py_CLEAR(state->queue_type); + + /* QueueError */ + Py_CLEAR(state->QueueError); + Py_CLEAR(state->QueueNotFoundError); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + + return 0; +} + + +/* error codes **************************************************************/ + +#define ERR_EXCEPTION_RAISED (-1) +// multi-queue errors +#define ERR_QUEUES_ALLOC (-11) +#define ERR_QUEUE_ALLOC (-12) +#define ERR_NO_NEXT_QUEUE_ID (-13) +#define ERR_QUEUE_NOT_FOUND (-14) +// single-queue errors +#define ERR_QUEUE_EMPTY (-21) +#define ERR_QUEUE_FULL (-22) +#define ERR_QUEUE_NEVER_BOUND (-23) + +static int ensure_external_exc_types(module_state *); + +static int +resolve_module_errcode(module_state *state, int errcode, int64_t qid, + PyObject **p_exctype, PyObject **p_msgobj) +{ + PyObject *exctype = NULL; + PyObject *msg = NULL; + switch (errcode) { + case ERR_NO_NEXT_QUEUE_ID: + exctype = state->QueueError; + msg = PyUnicode_FromString("ran out of queue IDs"); + break; + case ERR_QUEUE_NOT_FOUND: + exctype = state->QueueNotFoundError; + msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); + break; + case ERR_QUEUE_EMPTY: + if (ensure_external_exc_types(state) < 0) { + return -1; + } + exctype = state->QueueEmpty; + msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); + break; + case ERR_QUEUE_FULL: + if (ensure_external_exc_types(state) < 0) { + return -1; + } + exctype = state->QueueFull; + msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); + break; + case ERR_QUEUE_NEVER_BOUND: + exctype = state->QueueError; + msg = PyUnicode_FromFormat("queue %" PRId64 " never bound", qid); + break; + default: + PyErr_Format(PyExc_ValueError, + "unsupported error code %d", errcode); + return -1; + } + + if (msg == NULL) { + assert(PyErr_Occurred()); + return -1; + } + *p_exctype = exctype; + *p_msgobj = msg; + return 0; +} + + +/* QueueError ***************************************************************/ + +static int +add_exctype(PyObject *mod, PyObject **p_state_field, + const char *qualname, const char *doc, PyObject *base) +{ +#ifndef NDEBUG + const char *dot = strrchr(qualname, '.'); + assert(dot != NULL); + const char *name = dot+1; + assert(*p_state_field == NULL); + assert(!PyObject_HasAttrStringWithError(mod, name)); +#endif + PyObject *exctype = PyErr_NewExceptionWithDoc(qualname, doc, base, NULL); + if (exctype == NULL) { + return -1; + } + if (PyModule_AddType(mod, (PyTypeObject *)exctype) < 0) { + Py_DECREF(exctype); + return -1; + } + *p_state_field = exctype; + return 0; +} + +static int +add_QueueError(PyObject *mod) +{ + module_state *state = get_module_state(mod); + +#define PREFIX "test.support.interpreters." +#define ADD_EXCTYPE(NAME, BASE, DOC) \ + assert(state->NAME == NULL); \ + if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ + return -1; \ + } + ADD_EXCTYPE(QueueError, PyExc_RuntimeError, + "Indicates that a queue-related error happened.") + ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) + // QueueEmpty and QueueFull are set by set_external_exc_types(). + state->QueueEmpty = NULL; + state->QueueFull = NULL; +#undef ADD_EXCTYPE +#undef PREFIX + + return 0; +} + +static int +set_external_exc_types(module_state *state, + PyObject *emptyerror, PyObject *fullerror) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + } + else { + assert(state->QueueFull == NULL); + } + assert(PyObject_IsSubclass(emptyerror, state->QueueError)); + assert(PyObject_IsSubclass(fullerror, state->QueueError)); + state->QueueEmpty = Py_NewRef(emptyerror); + state->QueueFull = Py_NewRef(fullerror); + return 0; +} + +static int +ensure_external_exc_types(module_state *state) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + return 0; + } + assert(state->QueueFull == NULL); + + // Force the module to be loaded, to register the type. + if (ensure_highlevel_module_loaded() < 0) { + return -1; + } + assert(state->QueueEmpty != NULL); + assert(state->QueueFull != NULL); + return 0; +} + +static int +handle_queue_error(int err, PyObject *mod, int64_t qid) +{ + if (err == 0) { + assert(!PyErr_Occurred()); + return 0; + } + assert(err < 0); + assert((err == -1) == (PyErr_Occurred() != NULL)); + + module_state *state; + switch (err) { + case ERR_QUEUE_ALLOC: _Py_FALLTHROUGH; + case ERR_QUEUES_ALLOC: + PyErr_NoMemory(); + break; + case -1: + return -1; + default: + state = get_module_state(mod); + assert(state->QueueError != NULL); + PyObject *exctype = NULL; + PyObject *msg = NULL; + if (resolve_module_errcode(state, err, qid, &exctype, &msg) < 0) { + return -1; + } + PyObject *exc = PyObject_CallOneArg(exctype, msg); + Py_DECREF(msg); + if (exc == NULL) { + return -1; + } + PyErr_SetObject(exctype, exc); + Py_DECREF(exc); + } + return 1; +} + + +/* the basic queue **********************************************************/ + +struct _queueitem; + +typedef struct _queueitem { + _PyCrossInterpreterData *data; + int fmt; + struct _queueitem *next; +} _queueitem; + +static void +_queueitem_init(_queueitem *item, + _PyCrossInterpreterData *data, int fmt) +{ + *item = (_queueitem){ + .data = data, + .fmt = fmt, + }; +} + +static void +_queueitem_clear(_queueitem *item) +{ + item->next = NULL; + + if (item->data != NULL) { + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; + } +} + +static _queueitem * +_queueitem_new(_PyCrossInterpreterData *data, int fmt) +{ + _queueitem *item = GLOBAL_MALLOC(_queueitem); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + _queueitem_init(item, data, fmt); + return item; +} + +static void +_queueitem_free(_queueitem *item) +{ + _queueitem_clear(item); + GLOBAL_FREE(item); +} + +static void +_queueitem_free_all(_queueitem *item) +{ + while (item != NULL) { + _queueitem *last = item; + item = item->next; + _queueitem_free(last); + } +} + +static void +_queueitem_popped(_queueitem *item, + _PyCrossInterpreterData **p_data, int *p_fmt) +{ + *p_data = item->data; + *p_fmt = item->fmt; + // We clear them here, so they won't be released in _queueitem_clear(). + item->data = NULL; + _queueitem_free(item); +} + + +/* the queue */ + +typedef struct _queue { + Py_ssize_t num_waiters; // protected by global lock + PyThread_type_lock mutex; + int alive; + struct _queueitems { + Py_ssize_t maxsize; + Py_ssize_t count; + _queueitem *first; + _queueitem *last; + } items; + int fmt; +} _queue; + +static int +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) +{ + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUE_ALLOC; + } + *queue = (_queue){ + .mutex = mutex, + .alive = 1, + .items = { + .maxsize = maxsize, + }, + .fmt = fmt, + }; + return 0; +} + +static void +_queue_clear(_queue *queue) +{ + assert(!queue->alive); + assert(queue->num_waiters == 0); + _queueitem_free_all(queue->items.first); + assert(queue->mutex != NULL); + PyThread_free_lock(queue->mutex); + *queue = (_queue){0}; +} + +static void _queue_free(_queue *); + +static void +_queue_kill_and_wait(_queue *queue) +{ + // Mark it as dead. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + assert(queue->alive); + queue->alive = 0; + PyThread_release_lock(queue->mutex); + + // Wait for all waiters to fail. + while (queue->num_waiters > 0) { + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + PyThread_release_lock(queue->mutex); + }; +} + +static void +_queue_mark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters += 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters += 1; + } +} + +static void +_queue_unmark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters -= 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters -= 1; + } +} + +static int +_queue_lock(_queue *queue) +{ + // The queue must be marked as a waiter already. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + if (!queue->alive) { + PyThread_release_lock(queue->mutex); + return ERR_QUEUE_NOT_FOUND; + } + return 0; +} + +static void +_queue_unlock(_queue *queue) +{ + PyThread_release_lock(queue->mutex); +} + +static int +_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + Py_ssize_t maxsize = queue->items.maxsize; + if (maxsize <= 0) { + maxsize = PY_SSIZE_T_MAX; + } + if (queue->items.count >= maxsize) { + _queue_unlock(queue); + return ERR_QUEUE_FULL; + } + + _queueitem *item = _queueitem_new(data, fmt); + if (item == NULL) { + _queue_unlock(queue); + return -1; + } + + queue->items.count += 1; + if (queue->items.first == NULL) { + queue->items.first = item; + } + else { + queue->items.last->next = item; + } + queue->items.last = item; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_next(_queue *queue, + _PyCrossInterpreterData **p_data, int *p_fmt) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count >= 0); + _queueitem *item = queue->items.first; + if (item == NULL) { + _queue_unlock(queue); + return ERR_QUEUE_EMPTY; + } + queue->items.first = item->next; + if (queue->items.last == item) { + queue->items.last = NULL; + } + queue->items.count -= 1; + + _queueitem_popped(item, p_data, p_fmt); + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_maxsize(_queue *queue, Py_ssize_t *p_maxsize) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_maxsize = queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_is_full(_queue *queue, int *p_is_full) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count <= queue->items.maxsize); + *p_is_full = queue->items.count == queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_count(_queue *queue, Py_ssize_t *p_count) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_count = queue->items.count; + + _queue_unlock(queue); + return 0; +} + +static void +_queue_clear_interpreter(_queue *queue, int64_t interpid) +{ + int err = _queue_lock(queue); + if (err == ERR_QUEUE_NOT_FOUND) { + // The queue is already destroyed, so there's nothing to clear. + assert(!PyErr_Occurred()); + return; + } + assert(err == 0); // There should be no other errors. + + _queueitem *prev = NULL; + _queueitem *next = queue->items.first; + while (next != NULL) { + _queueitem *item = next; + next = item->next; + if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { + if (prev == NULL) { + queue->items.first = item->next; + } + else { + prev->next = item->next; + } + _queueitem_free(item); + queue->items.count -= 1; + } + else { + prev = item; + } + } + + _queue_unlock(queue); +} + + +/* external queue references ************************************************/ + +struct _queueref; + +typedef struct _queueref { + struct _queueref *next; + int64_t qid; + Py_ssize_t refcount; + _queue *queue; +} _queueref; + +static _queueref * +_queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) +{ + _queueref *prev = NULL; + _queueref *ref = first; + while (ref != NULL) { + if (ref->qid == qid) { + break; + } + prev = ref; + ref = ref->next; + } + if (pprev != NULL) { + *pprev = prev; + } + return ref; +} + +static void +_queuerefs_clear(_queueref *head) +{ + _queueref *next = head; + while (next != NULL) { + _queueref *ref = next; + next = ref->next; + +#ifdef Py_DEBUG + int64_t qid = ref->qid; + fprintf(stderr, "queue %" PRId64 " still exists\n", qid); +#endif + _queue *queue = ref->queue; + GLOBAL_FREE(ref); + + _queue_kill_and_wait(queue); +#ifdef Py_DEBUG + if (queue->items.count > 0) { + fprintf(stderr, "queue %" PRId64 " still holds %zd items\n", + qid, queue->items.count); + } +#endif + _queue_free(queue); + } +} + + +/* a collection of queues ***************************************************/ + +typedef struct _queues { + PyThread_type_lock mutex; + _queueref *head; + int64_t count; + int64_t next_id; +} _queues; + +static void +_queues_init(_queues *queues, PyThread_type_lock mutex) +{ + queues->mutex = mutex; + queues->head = NULL; + queues->count = 0; + queues->next_id = 1; +} + +static void +_queues_fini(_queues *queues) +{ + if (queues->count > 0) { + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + assert((queues->count == 0) != (queues->head != NULL)); + _queueref *head = queues->head; + queues->head = NULL; + queues->count = 0; + PyThread_release_lock(queues->mutex); + _queuerefs_clear(head); + } + if (queues->mutex != NULL) { + PyThread_free_lock(queues->mutex); + queues->mutex = NULL; + } +} + +static int64_t +_queues_next_id(_queues *queues) // needs lock +{ + int64_t qid = queues->next_id; + if (qid < 0) { + /* overflow */ + return ERR_NO_NEXT_QUEUE_ID; + } + queues->next_id += 1; + return qid; +} + +static int +_queues_lookup(_queues *queues, int64_t qid, _queue **res) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + assert(ref->queue != NULL); + _queue *queue = ref->queue; + _queue_mark_waiter(queue, NULL); + // The caller must unmark it. + + PyThread_release_lock(queues->mutex); + + *res = queue; + return 0; +} + +static int64_t +_queues_add(_queues *queues, _queue *queue) +{ + int64_t qid = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + // Create a new ref. + int64_t _qid = _queues_next_id(queues); + if (_qid < 0) { + goto done; + } + _queueref *ref = GLOBAL_MALLOC(_queueref); + if (ref == NULL) { + qid = ERR_QUEUE_ALLOC; + goto done; + } + *ref = (_queueref){ + .qid = _qid, + .queue = queue, + }; + + // Add it to the list. + // We assume that the queue is a new one (not already in the list). + ref->next = queues->head; + queues->head = ref; + queues->count += 1; + + qid = _qid; +done: + PyThread_release_lock(queues->mutex); + return qid; +} + +static void +_queues_remove_ref(_queues *queues, _queueref *ref, _queueref *prev, + _queue **p_queue) +{ + assert(ref->queue != NULL); + + if (ref == queues->head) { + queues->head = ref->next; + } + else { + prev->next = ref->next; + } + ref->next = NULL; + queues->count -= 1; + + *p_queue = ref->queue; + ref->queue = NULL; + GLOBAL_FREE(ref); +} + +static int +_queues_remove(_queues *queues, int64_t qid, _queue **p_queue) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + + _queues_remove_ref(queues, ref, prev, p_queue); + PyThread_release_lock(queues->mutex); + + return 0; +} + +static int +_queues_incref(_queues *queues, int64_t qid) +{ + // XXX Track interpreter IDs? + int res = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + assert(!PyErr_Occurred()); + res = ERR_QUEUE_NOT_FOUND; + goto done; + } + ref->refcount += 1; + + res = 0; +done: + PyThread_release_lock(queues->mutex); + return res; +} + +static int +_queues_decref(_queues *queues, int64_t qid) +{ + int res = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + assert(!PyErr_Occurred()); + res = ERR_QUEUE_NOT_FOUND; + goto finally; + } + if (ref->refcount == 0) { + res = ERR_QUEUE_NEVER_BOUND; + goto finally; + } + assert(ref->refcount > 0); + ref->refcount -= 1; + + // Destroy if no longer used. + assert(ref->queue != NULL); + if (ref->refcount == 0) { + _queue *queue = NULL; + _queues_remove_ref(queues, ref, prev, &queue); + PyThread_release_lock(queues->mutex); + + _queue_kill_and_wait(queue); + _queue_free(queue); + return 0; + } + + res = 0; +finally: + PyThread_release_lock(queues->mutex); + return res; +} + +struct queue_id_and_fmt { + int64_t id; + int fmt; +}; + +static struct queue_id_and_fmt * +_queues_list_all(_queues *queues, int64_t *count) +{ + struct queue_id_and_fmt *qids = NULL; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, + (Py_ssize_t)(queues->count)); + if (ids == NULL) { + goto done; + } + _queueref *ref = queues->head; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i].id = ref->qid; + assert(ref->queue != NULL); + ids[i].fmt = ref->queue->fmt; + } + *count = queues->count; + + qids = ids; +done: + PyThread_release_lock(queues->mutex); + return qids; +} + +static void +_queues_clear_interpreter(_queues *queues, int64_t interpid) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = queues->head; + for (; ref != NULL; ref = ref->next) { + assert(ref->queue != NULL); + _queue_clear_interpreter(ref->queue, interpid); + } + + PyThread_release_lock(queues->mutex); +} + + +/* "high"-level queue-related functions *************************************/ + +static void +_queue_free(_queue *queue) +{ + _queue_clear(queue); + GLOBAL_FREE(queue); +} + +// Create a new queue. +static int64_t +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) +{ + _queue *queue = GLOBAL_MALLOC(_queue); + if (queue == NULL) { + return ERR_QUEUE_ALLOC; + } + int err = _queue_init(queue, maxsize, fmt); + if (err < 0) { + GLOBAL_FREE(queue); + return (int64_t)err; + } + int64_t qid = _queues_add(queues, queue); + if (qid < 0) { + _queue_clear(queue); + GLOBAL_FREE(queue); + } + return qid; +} + +// Completely destroy the queue. +static int +queue_destroy(_queues *queues, int64_t qid) +{ + _queue *queue = NULL; + int err = _queues_remove(queues, qid, &queue); + if (err < 0) { + return err; + } + _queue_kill_and_wait(queue); + _queue_free(queue); + return 0; +} + +// Push an object onto the queue. +static int +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) +{ + // Look up the queue. + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + assert(queue != NULL); + + // Convert the object to cross-interpreter data. + _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + if (data == NULL) { + _queue_unmark_waiter(queue, queues->mutex); + return -1; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _queue_unmark_waiter(queue, queues->mutex); + GLOBAL_FREE(data); + return -1; + } + + // Add the data to the queue. + int res = _queue_add(queue, data, fmt); + _queue_unmark_waiter(queue, queues->mutex); + if (res != 0) { + // We may chain an exception here: + (void)_release_xid_data(data, 0); + GLOBAL_FREE(data); + return res; + } + + return 0; +} + +// Pop the next object off the queue. Fail if empty. +// XXX Support a "wait" mutex? +static int +queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) +{ + int err; + *res = NULL; + + // Look up the queue. + _queue *queue = NULL; + err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + // Past this point we are responsible for releasing the mutex. + assert(queue != NULL); + + // Pop off the next item from the queue. + _PyCrossInterpreterData *data = NULL; + err = _queue_next(queue, &data, p_fmt); + _queue_unmark_waiter(queue, queues->mutex); + if (err != 0) { + return err; + } + else if (data == NULL) { + assert(!PyErr_Occurred()); + return 0; + } + + // Convert the data back to an object. + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + if (obj == NULL) { + assert(PyErr_Occurred()); + // It was allocated in queue_put(), so we free it. + (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); + return -1; + } + // It was allocated in queue_put(), so we free it. + int release_res = _release_xid_data(data, XID_FREE); + if (release_res < 0) { + // The source interpreter has been destroyed already. + assert(PyErr_Occurred()); + Py_DECREF(obj); + return -1; + } + + *res = obj; + return 0; +} + +static int +queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_maxsize(queue, p_maxsize); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_is_full(_queues *queues, int64_t qid, int *p_is_full) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_is_full(queue, p_is_full); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_get_count(_queues *queues, int64_t qid, Py_ssize_t *p_count) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_count(queue, p_count); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + + +/* external Queue objects ***************************************************/ + +static int _queueobj_shared(PyThreadState *, + PyObject *, _PyCrossInterpreterData *); + +static int +set_external_queue_type(module_state *state, PyTypeObject *queue_type) +{ + // Clear the old value if the .py module was reloaded. + if (state->queue_type != NULL) { + (void)clear_xid_class(state->queue_type); + Py_CLEAR(state->queue_type); + } + + // Add and register the new type. + if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { + return -1; + } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); + + return 0; +} + +static PyTypeObject * +get_external_queue_type(PyObject *module) +{ + module_state *state = get_module_state(module); + + PyTypeObject *cls = state->queue_type; + if (cls == NULL) { + // Force the module to be loaded, to register the type. + if (ensure_highlevel_module_loaded() < 0) { + return NULL; + } + cls = state->queue_type; + assert(cls != NULL); + } + return cls; +} + + +// XXX Use a new __xid__ protocol instead? + +struct _queueid_xid { + int64_t qid; +}; + +static _queues * _get_global_queues(void); + +static void * +_queueid_xid_new(int64_t qid) +{ + _queues *queues = _get_global_queues(); + if (_queues_incref(queues, qid) < 0) { + return NULL; + } + + struct _queueid_xid *data = PyMem_RawMalloc(sizeof(struct _queueid_xid)); + if (data == NULL) { + _queues_incref(queues, qid); + return NULL; + } + data->qid = qid; + return (void *)data; +} + +static void +_queueid_xid_free(void *data) +{ + int64_t qid = ((struct _queueid_xid *)data)->qid; + PyMem_RawFree(data); + _queues *queues = _get_global_queues(); + int res = _queues_decref(queues, qid); + if (res == ERR_QUEUE_NOT_FOUND) { + // Already destroyed. + // XXX Warn? + } + else { + assert(res == 0); + } +} + +static PyObject * +_queueobj_from_xid(_PyCrossInterpreterData *data) +{ + int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data); + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + return NULL; + } + + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME_STR " module not imported yet"); + return NULL; + } + + PyTypeObject *cls = get_external_queue_type(mod); + Py_DECREF(mod); + if (cls == NULL) { + Py_DECREF(qidobj); + return NULL; + } + PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)qidobj); + Py_DECREF(qidobj); + return obj; +} + +static int +_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, + _PyCrossInterpreterData *data) +{ + PyObject *qidobj = PyObject_GetAttrString(queueobj, "_id"); + if (qidobj == NULL) { + return -1; + } + struct idarg_int64_converter_data converted = { + .label = "queue ID", + }; + int res = idarg_int64_converter(qidobj, &converted); + Py_CLEAR(qidobj); + if (!res) { + assert(PyErr_Occurred()); + return -1; + } + + void *raw = _queueid_xid_new(converted.id); + if (raw == NULL) { + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, + _queueobj_from_xid); + _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); + return 0; +} + + +/* module level code ********************************************************/ + +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + int module_count; + _queues queues; +} _globals = {0}; + +static int +_globals_init(void) +{ + // XXX This isn't thread-safe. + _globals.module_count++; + if (_globals.module_count > 1) { + // Already initialized. + return 0; + } + + assert(_globals.queues.mutex == NULL); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUES_ALLOC; + } + _queues_init(&_globals.queues, mutex); + return 0; +} + +static void +_globals_fini(void) +{ + // XXX This isn't thread-safe. + _globals.module_count--; + if (_globals.module_count > 0) { + return; + } + + _queues_fini(&_globals.queues); +} + +static _queues * +_get_global_queues(void) +{ + return &_globals.queues; +} + + +static void +clear_interpreter(void *data) +{ + if (_globals.module_count == 0) { + return; + } + PyInterpreterState *interp = (PyInterpreterState *)data; + assert(interp == _get_current_interp()); + int64_t interpid = PyInterpreterState_GetID(interp); + _queues_clear_interpreter(&_globals.queues, interpid); +} + + +typedef struct idarg_int64_converter_data qidarg_converter_data; + +static int +qidarg_converter(PyObject *arg, void *ptr) +{ + qidarg_converter_data *data = ptr; + if (data->label == NULL) { + data->label = "queue ID"; + } + return idarg_int64_converter(arg, ptr); +} + + +static PyObject * +queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"maxsize", "fmt", NULL}; + Py_ssize_t maxsize; + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, + &maxsize, &fmt)) { + return NULL; + } + + int64_t qid = queue_create(&_globals.queues, maxsize, fmt); + if (qid < 0) { + (void)handle_queue_error((int)qid, self, qid); + return NULL; + } + + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + // XXX issue a warning? + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + return NULL; + } + + return qidobj; +} + +PyDoc_STRVAR(queuesmod_create_doc, +"create(maxsize, fmt) -> qid\n\ +\n\ +Create a new cross-interpreter queue and return its unique generated ID.\n\ +It is a new reference as though bind() had been called on the queue.\n\ +\n\ +The caller is responsible for calling destroy() for the new queue\n\ +before the runtime is finalized."); + +static PyObject * +queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_destroy_doc, +"destroy(qid)\n\ +\n\ +Clear and destroy the queue. Afterward attempts to use the queue\n\ +will behave as though it never existed."); + +static PyObject * +queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t count = 0; + struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); + if (qids == NULL) { + if (count == 0) { + return PyList_New(0); + } + return NULL; + } + PyObject *ids = PyList_New((Py_ssize_t)count); + if (ids == NULL) { + goto finally; + } + struct queue_id_and_fmt *cur = qids; + for (int64_t i=0; i < count; cur++, i++) { + PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + if (item == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); + } + +finally: + PyMem_Free(qids); + return ids; +} + +PyDoc_STRVAR(queuesmod_list_all_doc, +"list_all() -> [(qid, fmt)]\n\ +\n\ +Return the list of IDs for all queues.\n\ +Each corresponding default format is also included."); + +static PyObject * +queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", "obj", "fmt", NULL}; + qidarg_converter_data qidarg; + PyObject *obj; + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt)) { + return NULL; + } + int64_t qid = qidarg.id; + + /* Queue up the object. */ + int err = queue_put(&_globals.queues, qid, obj, fmt); + // This is the only place that raises QueueFull. + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_put_doc, +"put(qid, obj, fmt)\n\ +\n\ +Add the object's data to the queue."); + +static PyObject * +queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + PyObject *obj = NULL; + int fmt = 0; + int err = queue_get(&_globals.queues, qid, &obj, &fmt); + // This is the only place that raises QueueEmpty. + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + PyObject *res = Py_BuildValue("Oi", obj, fmt); + Py_DECREF(obj); + return res; +} + +PyDoc_STRVAR(queuesmod_get_doc, +"get(qid) -> (obj, fmt)\n\ +\n\ +Return a new object from the data at the front of the queue.\n\ +The object's format is also returned.\n\ +\n\ +If there is nothing to receive then raise QueueEmpty."); + +static PyObject * +queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + + int err = _queues_incref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + // XXX Update module state. + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_bind_doc, +"bind(qid)\n\ +\n\ +Take a reference to the identified queue.\n\ +The queue is not destroyed until there are no references left."); + +static PyObject * +queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) +{ + // Note that only the current interpreter is affected. + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:release", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + // XXX Update module state. + + int err = _queues_decref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_release_doc, +"release(qid)\n\ +\n\ +Release a reference to the queue.\n\ +The queue is destroyed once there are no references left."); + +static PyObject * +queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_maxsize", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t maxsize = -1; + int err = queue_get_maxsize(&_globals.queues, qid, &maxsize); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + return PyLong_FromLongLong(maxsize); +} + +PyDoc_STRVAR(queuesmod_get_maxsize_doc, +"get_maxsize(qid)\n\ +\n\ +Return the maximum number of items in the queue."); + +static PyObject * +queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_queue_defaults", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + _queue *queue = NULL; + int err = _queues_lookup(&_globals.queues, qid, &queue); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + int fmt = queue->fmt; + _queue_unmark_waiter(queue, _globals.queues.mutex); + + PyObject *fmt_obj = PyLong_FromLong(fmt); + if (fmt_obj == NULL) { + return NULL; + } + // For now queues only have one default. + PyObject *res = PyTuple_Pack(1, fmt_obj); + Py_DECREF(fmt_obj); + return res; +} + +PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, +"get_queue_defaults(qid)\n\ +\n\ +Return the queue's default values, set when it was created."); + +static PyObject * +queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:is_full", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int is_full = 0; + int err = queue_is_full(&_globals.queues, qid, &is_full); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + if (is_full) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(queuesmod_is_full_doc, +"is_full(qid)\n\ +\n\ +Return true if the queue has a maxsize and has reached it."); + +static PyObject * +queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t count = -1; + int err = queue_get_count(&_globals.queues, qid, &count); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(queuesmod_get_count_doc, +"get_count(qid)\n\ +\n\ +Return the number of items in the queue."); + +static PyObject * +queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"queuetype", "emptyerror", "fullerror", NULL}; + PyObject *queuetype; + PyObject *emptyerror; + PyObject *fullerror; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OOO:_register_heap_types", kwlist, + &queuetype, &emptyerror, &fullerror)) { + return NULL; + } + if (!PyType_Check(queuetype)) { + PyErr_SetString(PyExc_TypeError, + "expected a type for 'queuetype'"); + return NULL; + } + if (!PyExceptionClass_Check(emptyerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'emptyerror'"); + return NULL; + } + if (!PyExceptionClass_Check(fullerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'fullerror'"); + return NULL; + } + + module_state *state = get_module_state(self); + + if (set_external_queue_type(state, (PyTypeObject *)queuetype) < 0) { + return NULL; + } + if (set_external_exc_types(state, emptyerror, fullerror) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyMethodDef module_functions[] = { + {"create", _PyCFunction_CAST(queuesmod_create), + METH_VARARGS | METH_KEYWORDS, queuesmod_create_doc}, + {"destroy", _PyCFunction_CAST(queuesmod_destroy), + METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, + {"list_all", queuesmod_list_all, + METH_NOARGS, queuesmod_list_all_doc}, + {"put", _PyCFunction_CAST(queuesmod_put), + METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, + {"get", _PyCFunction_CAST(queuesmod_get), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, + {"bind", _PyCFunction_CAST(queuesmod_bind), + METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, + {"release", _PyCFunction_CAST(queuesmod_release), + METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, + {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"get_queue_defaults", _PyCFunction_CAST(queuesmod_get_queue_defaults), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_queue_defaults_doc}, + {"is_full", _PyCFunction_CAST(queuesmod_is_full), + METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, + {"get_count", _PyCFunction_CAST(queuesmod_get_count), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, + {"_register_heap_types", _PyCFunction_CAST(queuesmod__register_heap_types), + METH_VARARGS | METH_KEYWORDS, NULL}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static int +module_exec(PyObject *mod) +{ + if (_globals_init() != 0) { + return -1; + } + + /* Add exception types */ + if (add_QueueError(mod) < 0) { + goto error; + } + + /* Make sure queues drop objects owned by this interpreter. */ + PyInterpreterState *interp = _get_current_interp(); + PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); + + return 0; + +error: + _globals_fini(); + return -1; +} + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + + // Now we clear the module state. + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + + // Now we clear the module state. + clear_module_state(state); + + _globals_fini(); +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME_STR, + .m_doc = module_doc, + .m_size = sizeof(module_state), + .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +MODINIT_FUNC_NAME(void) +{ + return PyModuleDef_Init(&moduledef); +} diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h new file mode 100644 index 00000000000000..07120f6ccc7207 --- /dev/null +++ b/Modules/_interpreters_common.h @@ -0,0 +1,21 @@ + +#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \ + PyInit_ ## NAME +#define RESOLVE_MODINIT_FUNC_NAME(NAME) \ + _RESOLVE_MODINIT_FUNC_NAME(NAME) + + +static int +ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) +{ + //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + return _PyCrossInterpreterData_RegisterClass(cls, getdata); +} + +#ifdef REGISTERS_HEAP_TYPES +static int +clear_xid_class(PyTypeObject *cls) +{ + return _PyCrossInterpreterData_UnregisterClass(cls); +} +#endif diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c new file mode 100644 index 00000000000000..6f3392fe6ea28d --- /dev/null +++ b/Modules/_interpretersmodule.c @@ -0,0 +1,1567 @@ +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_crossinterp.h" // struct _xid +#include "pycore_interp.h" // _PyInterpreterState_IDIncref() +#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() +#include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() + +#include "marshal.h" // PyMarshal_ReadObjectFromString() + +#include "_interpreters_common.h" + + +#define MODULE_NAME _interpreters +#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) +#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) + + +static PyInterpreterState * +_get_current_interp(void) +{ + // PyInterpreterState_Get() aborts if lookup fails, so don't need + // to check the result for NULL. + return PyInterpreterState_Get(); +} + +#define look_up_interp _PyInterpreterState_LookUpIDObject + + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME_STR); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +static int +is_running_main(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IsRunningMain(interp)) { + return 1; + } + // Unlike with the general C-API, we can be confident that someone + // using this module for the main interpreter is doing so through + // the main program. Thus we can make this extra check. This benefits + // applications that embed Python but haven't been updated yet + // to call_PyInterpreterState_SetRunningMain(). + if (_Py_IsMainInterpreter(interp)) { + return 1; + } + return 0; +} + + +/* Cross-interpreter Buffer Views *******************************************/ + +// XXX Release when the original interpreter is destroyed. + +typedef struct { + PyObject_HEAD + Py_buffer *view; + int64_t interpid; +} XIBufferViewObject; + +static PyObject * +xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +{ + assert(_PyCrossInterpreterData_DATA(data) != NULL); + assert(_PyCrossInterpreterData_OBJ(data) == NULL); + assert(_PyCrossInterpreterData_INTERPID(data) >= 0); + XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); + if (self == NULL) { + return NULL; + } + PyObject_Init((PyObject *)self, cls); + self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data); + self->interpid = _PyCrossInterpreterData_INTERPID(data); + return (PyObject *)self; +} + +static void +xibufferview_dealloc(XIBufferViewObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); + /* If the interpreter is no longer alive then we have problems, + since other objects may be using the buffer still. */ + assert(interp != NULL); + + if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + // XXX Emit a warning? + PyErr_Clear(); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); +} + +static int +xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) +{ + /* Only PyMemoryView_FromObject() should ever call this, + via _memoryview_from_xid() below. */ + *view = *self->view; + view->obj = (PyObject *)self; + // XXX Should we leave it alone? + view->internal = NULL; + return 0; +} + +static PyType_Slot XIBufferViewType_slots[] = { + {Py_tp_dealloc, (destructor)xibufferview_dealloc}, + {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, + // We don't bother with Py_bf_releasebuffer since we don't need it. + {0, NULL}, +}; + +static PyType_Spec XIBufferViewType_spec = { + .name = MODULE_NAME_STR ".CrossInterpreterBufferView", + .basicsize = sizeof(XIBufferViewObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = XIBufferViewType_slots, +}; + + +static PyTypeObject * _get_current_xibufferview_type(void); + +static PyObject * +_memoryview_from_xid(_PyCrossInterpreterData *data) +{ + PyTypeObject *cls = _get_current_xibufferview_type(); + if (cls == NULL) { + return NULL; + } + PyObject *obj = xibufferview_from_xid(cls, data); + if (obj == NULL) { + return NULL; + } + return PyMemoryView_FromObject(obj); +} + +static int +_memoryview_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); + if (view == NULL) { + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { + PyMem_RawFree(view); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, + _memoryview_from_xid); + return 0; +} + +static int +register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) +{ + // XIBufferView + assert(*p_state == NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &XIBufferViewType_spec, NULL); + if (cls == NULL) { + return -1; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return -1; + } + *p_state = cls; + + // Register XID for the builtin memoryview type. + if (ensure_xid_class(&PyMemoryView_Type, _memoryview_shared) < 0) { + return -1; + } + // We don't ever bother un-registering memoryview. + + return 0; +} + + + +/* module state *************************************************************/ + +typedef struct { + int _notused; + + /* heap types */ + PyTypeObject *XIBufferViewType; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static module_state * +_get_current_module_state(void) +{ + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME_STR " module not imported yet"); + return NULL; + } + module_state *state = get_module_state(mod); + Py_DECREF(mod); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(state->XIBufferViewType); + + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* heap types */ + Py_CLEAR(state->XIBufferViewType); + + return 0; +} + + +static PyTypeObject * +_get_current_xibufferview_type(void) +{ + module_state *state = _get_current_module_state(); + if (state == NULL) { + return NULL; + } + return state->XIBufferViewType; +} + + +/* Python code **************************************************************/ + +static const char * +check_code_str(PyUnicodeObject *text) +{ + assert(text != NULL); + if (PyUnicode_GET_LENGTH(text) == 0) { + return "too short"; + } + + // XXX Verify that it parses? + + return NULL; +} + +static const char * +check_code_object(PyCodeObject *code) +{ + assert(code != NULL); + if (code->co_argcount > 0 + || code->co_posonlyargcount > 0 + || code->co_kwonlyargcount > 0 + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) + { + return "arguments not supported"; + } + if (code->co_ncellvars > 0) { + return "closures not supported"; + } + // We trust that no code objects under co_consts have unbound cell vars. + + if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { + return "only basic functions are supported"; + } + if (code->_co_monitoring != NULL) { + return "only basic functions are supported"; + } + if (code->co_extra != NULL) { + return "only basic functions are supported"; + } + + return NULL; +} + +#define RUN_TEXT 1 +#define RUN_CODE 2 + +static const char * +get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) +{ + const char *codestr = NULL; + Py_ssize_t len = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + + if (PyUnicode_Check(arg)) { + assert(PyUnicode_CheckExact(arg) + && (check_code_str((PyUnicodeObject *)arg) == NULL)); + codestr = PyUnicode_AsUTF8AndSize(arg, &len); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + flags = RUN_TEXT; + } + else { + assert(PyCode_Check(arg) + && (check_code_object((PyCodeObject *)arg) == NULL)); + flags = RUN_CODE; + + // Serialize the code object. + bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); + if (bytes_obj == NULL) { + return NULL; + } + codestr = PyBytes_AS_STRING(bytes_obj); + len = PyBytes_GET_SIZE(bytes_obj); + } + + *flags_p = flags; + *bytes_p = bytes_obj; + *len_p = len; + return codestr; +} + + +/* interpreter-specific code ************************************************/ + +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + +static int +_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) +{ + PyObject *result = NULL; + if (flags & RUN_TEXT) { + result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + } + else if (flags & RUN_CODE) { + PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); + if (code != NULL) { + result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + } + } + else { + Py_UNREACHABLE(); + } + if (result == NULL) { + return -1; + } + Py_DECREF(result); // We throw away the result. + return 0; +} + +static int +_run_in_interpreter(PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, + PyObject *shareables, int flags, + PyObject **p_excinfo) +{ + assert(!PyErr_Occurred()); + _PyXI_session session = {0}; + + // Prep and switch interpreters. + if (_PyXI_Enter(&session, interp, shareables) < 0) { + assert(!PyErr_Occurred()); + PyObject *excinfo = _PyXI_ApplyError(session.error); + if (excinfo != NULL) { + *p_excinfo = excinfo; + } + assert(PyErr_Occurred()); + return -1; + } + + // Run the script. + int res = _run_script(session.main_ns, codestr, codestrlen, flags); + + // Clean up and switch back. + _PyXI_Exit(&session); + + // Propagate any exception out to the caller. + assert(!PyErr_Occurred()); + if (res < 0) { + PyObject *excinfo = _PyXI_ApplyCapturedException(&session); + if (excinfo != NULL) { + *p_excinfo = excinfo; + } + } + else { + assert(!_PyXI_HasCapturedException(&session)); + } + + return res; +} + + +/* module level code ********************************************************/ + +static long +get_whence(PyInterpreterState *interp) +{ + return _PyInterpreterState_GetWhence(interp); +} + + +static PyInterpreterState * +resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op) +{ + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = look_up_interp(idobj); + if (interp == NULL) { + return NULL; + } + } + + if (reqready && !_PyInterpreterState_IsReady(interp)) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s current interpreter (not ready)", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s interpreter %R (not ready)", op, idobj); + } + return NULL; + } + + if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized current interpreter", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized interpreter %R", op, idobj); + } + return NULL; + } + + return interp; +} + + +static PyObject * +get_summary(PyInterpreterState *interp) +{ + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + return NULL; + } + PyObject *whenceobj = PyLong_FromLong( + get_whence(interp)); + if (whenceobj == NULL) { + Py_DECREF(idobj); + return NULL; + } + PyObject *res = PyTuple_Pack(2, idobj, whenceobj); + Py_DECREF(idobj); + Py_DECREF(whenceobj); + return res; +} + + +static PyObject * +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; + + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + +static PyObject * +interp_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"config", "reqrefs", NULL}; + PyObject *configobj = NULL; + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { + return NULL; + } + + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { + return NULL; + } + + long whence = _PyInterpreterState_WHENCE_STDLIB; + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, &whence, NULL, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + PyErr_SetString(PyExc_InterpreterError, "interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + assert(_PyInterpreterState_IsReady(interp)); + + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); + return NULL; + } + + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } + + return idobj; +} + + +PyDoc_STRVAR(create_doc, +"create([config], *, reqrefs=False) -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID.\n\ +\n\ +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref() appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *id; + int restricted = 0; + // XXX Use "L" for id? + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:destroy", kwlist, &id, &restricted)) + { + return NULL; + } + + // Look up the interpreter. + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "destroy"); + if (interp == NULL) { + return NULL; + } + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current_interp(); + if (current == NULL) { + return NULL; + } + if (interp == current) { + PyErr_SetString(PyExc_InterpreterError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (is_running_main(interp)) { + PyErr_Format(PyExc_InterpreterError, "interpreter running"); + return NULL; + } + + // Destroy the interpreter. + _PyXI_EndInterpreter(interp, NULL, NULL); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(id, *, restrict=False)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter raises InterpreterError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"require_ready", NULL}; + int reqready = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|$p:" MODULE_NAME_STR ".list_all", + kwlist, &reqready)) + { + return NULL; + } + + PyObject *ids = PyList_New(0); + if (ids == NULL) { + return NULL; + } + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + if (!reqready || _PyInterpreterState_IsReady(interp)) { + PyObject *item = get_summary(interp); + if (item == NULL) { + Py_DECREF(ids); + return NULL; + } + + // insert at front of list + int res = PyList_Insert(ids, 0, item); + Py_DECREF(item); + if (res < 0) { + Py_DECREF(ids); + return NULL; + } + } + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(list_all_doc, +"list_all() -> [(ID, whence)]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyInterpreterState *interp =_get_current_interp(); + if (interp == NULL) { + return NULL; + } + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> (ID, whence)\n\ +\n\ +Return the ID of current interpreter."); + + +static PyObject * +interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyInterpreterState *interp = _PyInterpreterState_Main(); + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> (ID, whence)\n\ +\n\ +Return the ID of main interpreter."); + + +static PyObject * +interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", "updates", "restrict", NULL}; + PyObject *id, *updates; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OO|$p:" MODULE_NAME_STR ".set___main___attrs", + kwlist, &id, &updates, &restricted)) + { + return NULL; + } + + // Look up the interpreter. + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "update __main__ for"); + if (interp == NULL) { + return NULL; + } + + // Check the updates. + if (updates != Py_None) { + Py_ssize_t size = PyObject_Size(updates); + if (size < 0) { + return NULL; + } + if (size == 0) { + PyErr_SetString(PyExc_ValueError, + "arg 2 must be a non-empty mapping"); + return NULL; + } + } + + _PyXI_session session = {0}; + + // Prep and switch interpreters, including apply the updates. + if (_PyXI_Enter(&session, interp, updates) < 0) { + if (!PyErr_Occurred()) { + _PyXI_ApplyCapturedException(&session); + assert(PyErr_Occurred()); + } + else { + assert(!_PyXI_HasCapturedException(&session)); + } + return NULL; + } + + // Clean up and switch back. + _PyXI_Exit(&session); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(set___main___attrs_doc, +"set___main___attrs(id, ns, *, restrict=False)\n\ +\n\ +Bind the given attributes in the interpreter's __main__ module."); + + +static PyUnicodeObject * +convert_script_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + PyUnicodeObject *str = NULL; + if (PyUnicode_CheckExact(arg)) { + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else if (PyUnicode_Check(arg)) { + // XXX str = PyUnicode_FromObject(arg); + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_str(str); + if (err != NULL) { + Py_DECREF(str); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); + return NULL; + } + + return str; +} + +static PyCodeObject * +convert_code_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + const char *kind = NULL; + PyCodeObject *code = NULL; + if (PyFunction_Check(arg)) { + if (PyFunction_GetClosure(arg) != NULL) { + PyErr_Format(PyExc_ValueError, + "%.200s(): closures not supported", fname); + return NULL; + } + code = (PyCodeObject *)PyFunction_GetCode(arg); + if (code == NULL) { + if (PyErr_Occurred()) { + // This chains. + PyErr_Format(PyExc_ValueError, + "%.200s(): bad func", fname); + } + else { + PyErr_Format(PyExc_ValueError, + "%.200s(): func.__code__ missing", fname); + } + return NULL; + } + Py_INCREF(code); + kind = "func"; + } + else if (PyCode_Check(arg)) { + code = (PyCodeObject *)Py_NewRef(arg); + kind = "code object"; + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_object(code); + if (err != NULL) { + Py_DECREF(code); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad %s (%s)", fname, kind, err); + return NULL; + } + + return code; +} + +static int +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) +{ + // Extract code. + Py_ssize_t codestrlen = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + const char *codestr = get_code_str(code_arg, + &codestrlen, &bytes_obj, &flags); + if (codestr == NULL) { + return -1; + } + + // Run the code in the interpreter. + int res = _run_in_interpreter(interp, codestr, codestrlen, + shared_arg, flags, p_excinfo); + Py_XDECREF(bytes_obj); + if (res < 0) { + return -1; + } + + return 0; +} + +static PyObject * +interp_exec(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; + PyObject *id, *code; + PyObject *shared = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, + &id, &code, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "exec code for"); + if (interp == NULL) { + return NULL; + } + + const char *expected = "a string, a function, or a code object"; + if (PyUnicode_Check(code)) { + code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", + "argument 2", expected); + } + else { + code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", + "argument 2", expected); + } + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, interp, code, shared, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(exec_doc, +"exec(id, code, shared=None, *, restrict=False)\n\ +\n\ +Execute the provided code in the identified interpreter.\n\ +This is equivalent to running the builtin exec() under the target\n\ +interpreter, using the __dict__ of its __main__ module as both\n\ +globals and locals.\n\ +\n\ +\"code\" may be a string containing the text of a Python script.\n\ +\n\ +Functions (and code objects) are also supported, with some restrictions.\n\ +The code/function must not take any arguments or be a closure\n\ +(i.e. have cell vars). Methods and other callables are not supported.\n\ +\n\ +If a function is provided, its code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + +static PyObject * +interp_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "callable", "args", "kwargs", + "restrict", NULL}; + PyObject *id, *callable; + PyObject *args_obj = NULL; + PyObject *kwargs_obj = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj, + &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "make a call in"); + if (interp == NULL) { + return NULL; + } + + if (args_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected args"); + return NULL; + } + if (kwargs_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + return NULL; + } + + PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + "argument 2", "a function"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, interp, code, NULL, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(call_doc, +"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ +\n\ +Call the provided object in the identified interpreter.\n\ +Pass the given args and kwargs, if possible.\n\ +\n\ +\"callable\" may be a plain function with no free vars that takes\n\ +no arguments.\n\ +\n\ +The function's code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + +static PyObject * +interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; + PyObject *id, *script; + PyObject *shared = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OU|O$p:" MODULE_NAME_STR ".run_string", + kwlist, &id, &script, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a string in"); + if (interp == NULL) { + return NULL; + } + + script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".exec", + "argument 2", "a string"); + if (script == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, interp, script, shared, &excinfo); + Py_DECREF(script); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_string_doc, +"run_string(id, script, shared=None, *, restrict=False)\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +\n\ +(See " MODULE_NAME_STR ".exec()."); + +static PyObject * +interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; + PyObject *id, *func; + PyObject *shared = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O$p:" MODULE_NAME_STR ".run_func", + kwlist, &id, &func, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a function in"); + if (interp == NULL) { + return NULL; + } + + PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", + "argument 2", + "a function or a code object"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_func_doc, +"run_func(id, func, shared=None, *, restrict=False)\n\ +\n\ +Execute the body of the provided function in the identified interpreter.\n\ +Code objects are also supported. In both cases, closures and args\n\ +are not supported. Methods and other callables are not supported either.\n\ +\n\ +(See " MODULE_NAME_STR ".exec()."); + + +static PyObject * +object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"obj", NULL}; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:is_shareable", kwlist, &obj)) { + return NULL; + } + + if (_PyObject_CheckCrossInterpreterData(obj) == 0) { + Py_RETURN_TRUE; + } + PyErr_Clear(); + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_shareable_doc, +"is_shareable(obj) -> bool\n\ +\n\ +Return True if the object's data may be shared between interpreters and\n\ +False otherwise."); + + +static PyObject * +interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *id; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:is_running", kwlist, + &id, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "check if running for"); + if (interp == NULL) { + return NULL; + } + + if (is_running_main(interp)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id, *, restrict=False) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + + +static PyObject * +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *idobj = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:get_config", kwlist, + &idobj, &restricted)) + { + return NULL; + } + if (idobj == Py_None) { + idobj = NULL; + } + + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(idobj, restricted, reqready, "get the config of"); + if (interp == NULL) { + return NULL; + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + +static PyObject * +interp_whence(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:whence", kwlist, &id)) + { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + + long whence = get_whence(interp); + return PyLong_FromLong(whence); +} + +PyDoc_STRVAR(whence_doc, +"whence(id) -> int\n\ +\n\ +Return an identifier for where the interpreter was created."); + + +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; + PyObject *id; + int implieslink = 0; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$pp:incref", kwlist, + &id, &implieslink, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "incref"); + if (interp == NULL) { + return NULL; + } + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } + _PyInterpreterState_IDIncref(interp); + + Py_RETURN_NONE; +} + + +static PyObject * +interp_decref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *id; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:decref", kwlist, &id, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "decref"); + if (interp == NULL) { + return NULL; + } + + _PyInterpreterState_IDDecref(interp); + + Py_RETURN_NONE; +} + + +static PyObject * +capture_exception(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"exc", NULL}; + PyObject *exc_arg = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|O:capture_exception", kwlist, + &exc_arg)) + { + return NULL; + } + + PyObject *exc = exc_arg; + if (exc == NULL || exc == Py_None) { + exc = PyErr_GetRaisedException(); + if (exc == NULL) { + Py_RETURN_NONE; + } + } + else if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc); + return NULL; + } + PyObject *captured = NULL; + + _PyXI_excinfo info = {0}; + if (_PyXI_InitExcInfo(&info, exc) < 0) { + goto finally; + } + captured = _PyXI_ExcInfoAsObject(&info); + if (captured == NULL) { + goto finally; + } + + PyObject *formatted = _PyXI_FormatExcInfo(&info); + if (formatted == NULL) { + Py_CLEAR(captured); + goto finally; + } + int res = PyObject_SetAttrString(captured, "formatted", formatted); + Py_DECREF(formatted); + if (res < 0) { + Py_CLEAR(captured); + goto finally; + } + +finally: + _PyXI_ClearExcInfo(&info); + if (exc != exc_arg) { + if (PyErr_Occurred()) { + PyErr_SetRaisedException(exc); + } + else { + _PyErr_ChainExceptions1(exc); + } + } + return captured; +} + +PyDoc_STRVAR(capture_exception_doc, +"capture_exception(exc=None) -> types.SimpleNamespace\n\ +\n\ +Return a snapshot of an exception. If \"exc\" is None\n\ +then the current exception, if any, is used (but not cleared).\n\ +\n\ +The returned snapshot is the same as what _interpreters.exec() returns."); + + +static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, + + {"create", _PyCFunction_CAST(interp_create), + METH_VARARGS | METH_KEYWORDS, create_doc}, + {"destroy", _PyCFunction_CAST(interp_destroy), + METH_VARARGS | METH_KEYWORDS, destroy_doc}, + {"list_all", _PyCFunction_CAST(interp_list_all), + METH_VARARGS | METH_KEYWORDS, list_all_doc}, + {"get_current", interp_get_current, + METH_NOARGS, get_current_doc}, + {"get_main", interp_get_main, + METH_NOARGS, get_main_doc}, + + {"is_running", _PyCFunction_CAST(interp_is_running), + METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, + {"whence", _PyCFunction_CAST(interp_whence), + METH_VARARGS | METH_KEYWORDS, whence_doc}, + {"exec", _PyCFunction_CAST(interp_exec), + METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"call", _PyCFunction_CAST(interp_call), + METH_VARARGS | METH_KEYWORDS, call_doc}, + {"run_string", _PyCFunction_CAST(interp_run_string), + METH_VARARGS | METH_KEYWORDS, run_string_doc}, + {"run_func", _PyCFunction_CAST(interp_run_func), + METH_VARARGS | METH_KEYWORDS, run_func_doc}, + + {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), + METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, + + {"incref", _PyCFunction_CAST(interp_incref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {"decref", _PyCFunction_CAST(interp_decref), + METH_VARARGS | METH_KEYWORDS, NULL}, + + {"is_shareable", _PyCFunction_CAST(object_is_shareable), + METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + + {"capture_exception", _PyCFunction_CAST(capture_exception), + METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static int +module_exec(PyObject *mod) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + module_state *state = get_module_state(mod); + +#define ADD_WHENCE(NAME) \ + if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \ + _PyInterpreterState_WHENCE_##NAME) < 0) \ + { \ + goto error; \ + } + ADD_WHENCE(UNKNOWN) + ADD_WHENCE(RUNTIME) + ADD_WHENCE(LEGACY_CAPI) + ADD_WHENCE(CAPI) + ADD_WHENCE(XI) + ADD_WHENCE(STDLIB) +#undef ADD_WHENCE + + // exceptions + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { + goto error; + } + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { + goto error; + } + PyObject *PyExc_NotShareableError = \ + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + goto error; + } + + if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { + goto error; + } + + return 0; + +error: + return -1; +} + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME_STR, + .m_doc = module_doc, + .m_size = sizeof(module_state), + .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +MODINIT_FUNC_NAME(void) +{ + return PyModuleDef_Init(&moduledef); +} diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 173f5b55e5f732..1238e6074246d0 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,7 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_long.h" // _PyLong_Sign() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -202,7 +202,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, const char *newline, int closefd, PyObject *opener) /*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ { - unsigned i; + size_t i; int creating = 0, reading = 0, writing = 0, appending = 0, updating = 0; int text = 0, binary = 0; @@ -544,10 +544,7 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err) */ if (!err) { assert(PyLong_Check(value)); - /* Whether or not it is less than or equal to - zero is determined by the sign of ob_size - */ - if (_PyLong_Sign(value) < 0) + if (_PyLong_IsNegative((PyLongObject *)value)) result = PY_OFF_T_MIN; else result = PY_OFF_T_MAX; @@ -720,6 +717,7 @@ iomodule_exec(PyObject *m) static struct PyModuleDef_Slot iomodule_slots[] = { {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index e8caf9f0df6dbf..aa52711941d374 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -8,11 +8,11 @@ */ #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytes_Join() -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _Py_FatalErrorFormat() -#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() +#include "pycore_bytesobject.h" // _PyBytes_Join() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat() +#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "_iomodule.h" @@ -82,6 +82,7 @@ _bufferediobase_readinto_generic(PyObject *self, Py_buffer *buffer, char readint } /*[clinic input] +@critical_section _io._BufferedIOBase.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -89,12 +90,13 @@ _io._BufferedIOBase.readinto static PyObject * _io__BufferedIOBase_readinto_impl(PyObject *self, Py_buffer *buffer) -/*[clinic end generated code: output=8c8cda6684af8038 input=00a6b9a38f29830a]*/ +/*[clinic end generated code: output=8c8cda6684af8038 input=5273d20db7f56e1a]*/ { return _bufferediobase_readinto_generic(self, buffer, 0); } /*[clinic input] +@critical_section _io._BufferedIOBase.readinto1 buffer: Py_buffer(accept={rwbuffer}) / @@ -102,7 +104,7 @@ _io._BufferedIOBase.readinto1 static PyObject * _io__BufferedIOBase_readinto1_impl(PyObject *self, Py_buffer *buffer) -/*[clinic end generated code: output=358623e4fd2b69d3 input=ebad75b4aadfb9be]*/ +/*[clinic end generated code: output=358623e4fd2b69d3 input=d6eb723dedcee654]*/ { return _bufferediobase_readinto_generic(self, buffer, 1); } @@ -431,12 +433,13 @@ buffered_dealloc(buffered *self) } /*[clinic input] +@critical_section _io._Buffered.__sizeof__ [clinic start generated code]*/ static PyObject * _io__Buffered___sizeof___impl(buffered *self) -/*[clinic end generated code: output=0231ef7f5053134e input=753c782d808d34df]*/ +/*[clinic end generated code: output=0231ef7f5053134e input=07a32d578073ea64]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buffer) { @@ -488,12 +491,13 @@ _io__Buffered__dealloc_warn(buffered *self, PyObject *source) /* Flush and close */ /*[clinic input] +@critical_section _io._Buffered.flush as _io__Buffered_simple_flush [clinic start generated code]*/ static PyObject * _io__Buffered_simple_flush_impl(buffered *self) -/*[clinic end generated code: output=29ebb3820db1bdfd input=f33ef045e7250767]*/ +/*[clinic end generated code: output=29ebb3820db1bdfd input=5248cb84a65f80bd]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(flush)); @@ -513,20 +517,28 @@ buffered_closed(buffered *self) return closed; } +/*[clinic input] +@critical_section +@getter +_io._Buffered.closed +[clinic start generated code]*/ + static PyObject * -buffered_closed_get(buffered *self, void *context) +_io__Buffered_closed_get_impl(buffered *self) +/*[clinic end generated code: output=f08ce57290703a1a input=18eddefdfe4a3d2f]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(closed)); } /*[clinic input] +@critical_section _io._Buffered.close [clinic start generated code]*/ static PyObject * _io__Buffered_close_impl(buffered *self) -/*[clinic end generated code: output=7280b7b42033be0c input=d20b83d1ddd7d805]*/ +/*[clinic end generated code: output=7280b7b42033be0c input=56d95935b03fd326]*/ { PyObject *res = NULL; int r; @@ -583,12 +595,13 @@ _io__Buffered_close_impl(buffered *self) } /*[clinic input] +@critical_section _io._Buffered.detach [clinic start generated code]*/ static PyObject * _io__Buffered_detach_impl(buffered *self) -/*[clinic end generated code: output=dd0fc057b8b779f7 input=482762a345cc9f44]*/ +/*[clinic end generated code: output=dd0fc057b8b779f7 input=d4ef1828a678be37]*/ { PyObject *raw; CHECK_INITIALIZED(self) @@ -605,50 +618,68 @@ _io__Buffered_detach_impl(buffered *self) /* Inquiries */ /*[clinic input] +@critical_section _io._Buffered.seekable [clinic start generated code]*/ static PyObject * _io__Buffered_seekable_impl(buffered *self) -/*[clinic end generated code: output=90172abb5ceb6e8f input=7d35764f5fb5262b]*/ +/*[clinic end generated code: output=90172abb5ceb6e8f input=e3a4fc1d297b2fd3]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(seekable)); } /*[clinic input] +@critical_section _io._Buffered.readable [clinic start generated code]*/ static PyObject * _io__Buffered_readable_impl(buffered *self) -/*[clinic end generated code: output=92afa07661ecb698 input=640619addb513b8b]*/ +/*[clinic end generated code: output=92afa07661ecb698 input=abe54107d59bca9a]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(readable)); } /*[clinic input] +@critical_section _io._Buffered.writable [clinic start generated code]*/ static PyObject * _io__Buffered_writable_impl(buffered *self) -/*[clinic end generated code: output=4e3eee8d6f9d8552 input=b35ea396b2201554]*/ +/*[clinic end generated code: output=4e3eee8d6f9d8552 input=45eb76bf6a10e6f7]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(writable)); } + +/*[clinic input] +@critical_section +@getter +_io._Buffered.name +[clinic start generated code]*/ + static PyObject * -buffered_name_get(buffered *self, void *context) +_io__Buffered_name_get_impl(buffered *self) +/*[clinic end generated code: output=d2adf384051d3d10 input=6b84a0e6126f545e]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(name)); } +/*[clinic input] +@critical_section +@getter +_io._Buffered.mode +[clinic start generated code]*/ + static PyObject * -buffered_mode_get(buffered *self, void *context) +_io__Buffered_mode_get_impl(buffered *self) +/*[clinic end generated code: output=0feb205748892fa4 input=0762d5e28542fd8c]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(mode)); @@ -657,24 +688,26 @@ buffered_mode_get(buffered *self, void *context) /* Lower-level APIs */ /*[clinic input] +@critical_section _io._Buffered.fileno [clinic start generated code]*/ static PyObject * _io__Buffered_fileno_impl(buffered *self) -/*[clinic end generated code: output=b717648d58a95ee3 input=768ea30b3f6314a7]*/ +/*[clinic end generated code: output=b717648d58a95ee3 input=1c4fead777bae20a]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(fileno)); } /*[clinic input] +@critical_section _io._Buffered.isatty [clinic start generated code]*/ static PyObject * _io__Buffered_isatty_impl(buffered *self) -/*[clinic end generated code: output=c20e55caae67baea input=9ea007b11559bee4]*/ +/*[clinic end generated code: output=c20e55caae67baea input=e53d182d7e490e3a]*/ { CHECK_INITIALIZED(self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(isatty)); @@ -880,12 +913,13 @@ buffered_flush_and_rewind_unlocked(buffered *self) } /*[clinic input] +@critical_section _io._Buffered.flush [clinic start generated code]*/ static PyObject * _io__Buffered_flush_impl(buffered *self) -/*[clinic end generated code: output=da2674ef1ce71f3a input=fda63444697c6bf4]*/ +/*[clinic end generated code: output=da2674ef1ce71f3a input=6b30de9f083419c2]*/ { PyObject *res; @@ -901,6 +935,7 @@ _io__Buffered_flush_impl(buffered *self) } /*[clinic input] +@critical_section _io._Buffered.peek size: Py_ssize_t = 0 / @@ -909,7 +944,7 @@ _io._Buffered.peek static PyObject * _io__Buffered_peek_impl(buffered *self, Py_ssize_t size) -/*[clinic end generated code: output=ba7a097ca230102b input=37ffb97d06ff4adb]*/ +/*[clinic end generated code: output=ba7a097ca230102b input=56733376f926d982]*/ { PyObject *res = NULL; @@ -933,6 +968,7 @@ _io__Buffered_peek_impl(buffered *self, Py_ssize_t size) } /*[clinic input] +@critical_section _io._Buffered.read size as n: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -940,7 +976,7 @@ _io._Buffered.read static PyObject * _io__Buffered_read_impl(buffered *self, Py_ssize_t n) -/*[clinic end generated code: output=f41c78bb15b9bbe9 input=7df81e82e08a68a2]*/ +/*[clinic end generated code: output=f41c78bb15b9bbe9 input=bdb4b0425b295472]*/ { PyObject *res; @@ -974,6 +1010,7 @@ _io__Buffered_read_impl(buffered *self, Py_ssize_t n) } /*[clinic input] +@critical_section _io._Buffered.read1 size as n: Py_ssize_t = -1 / @@ -981,7 +1018,7 @@ _io._Buffered.read1 static PyObject * _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) -/*[clinic end generated code: output=bcc4fb4e54d103a3 input=7d22de9630b61774]*/ +/*[clinic end generated code: output=bcc4fb4e54d103a3 input=3d0ad241aa52b36c]*/ { Py_ssize_t have, r; PyObject *res = NULL; @@ -1013,6 +1050,16 @@ _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) Py_DECREF(res); return NULL; } + /* Flush the write buffer if necessary */ + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) { + LEAVE_BUFFERED(self) + Py_DECREF(res); + return NULL; + } + Py_DECREF(r); + } _bufferedreader_reset_buf(self); r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); LEAVE_BUFFERED(self) @@ -1110,6 +1157,7 @@ _buffered_readinto_generic(buffered *self, Py_buffer *buffer, char readinto1) } /*[clinic input] +@critical_section _io._Buffered.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -1117,12 +1165,13 @@ _io._Buffered.readinto static PyObject * _io__Buffered_readinto_impl(buffered *self, Py_buffer *buffer) -/*[clinic end generated code: output=bcb376580b1d8170 input=ed6b98b7a20a3008]*/ +/*[clinic end generated code: output=bcb376580b1d8170 input=777c33e7adaa2bcd]*/ { return _buffered_readinto_generic(self, buffer, 0); } /*[clinic input] +@critical_section _io._Buffered.readinto1 buffer: Py_buffer(accept={rwbuffer}) / @@ -1130,7 +1179,7 @@ _io._Buffered.readinto1 static PyObject * _io__Buffered_readinto1_impl(buffered *self, Py_buffer *buffer) -/*[clinic end generated code: output=6e5c6ac5868205d6 input=4455c5d55fdf1687]*/ +/*[clinic end generated code: output=6e5c6ac5868205d6 input=ef03cc5fc92a6895]*/ { return _buffered_readinto_generic(self, buffer, 1); } @@ -1245,6 +1294,7 @@ _buffered_readline(buffered *self, Py_ssize_t limit) } /*[clinic input] +@critical_section _io._Buffered.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -1252,7 +1302,7 @@ _io._Buffered.readline static PyObject * _io__Buffered_readline_impl(buffered *self, Py_ssize_t size) -/*[clinic end generated code: output=24dd2aa6e33be83c input=673b6240e315ef8a]*/ +/*[clinic end generated code: output=24dd2aa6e33be83c input=e81ca5abd4280776]*/ { CHECK_INITIALIZED(self) return _buffered_readline(self, size); @@ -1260,12 +1310,13 @@ _io__Buffered_readline_impl(buffered *self, Py_ssize_t size) /*[clinic input] +@critical_section _io._Buffered.tell [clinic start generated code]*/ static PyObject * _io__Buffered_tell_impl(buffered *self) -/*[clinic end generated code: output=386972ae84716c1e input=ad61e04a6b349573]*/ +/*[clinic end generated code: output=386972ae84716c1e input=ab12e67d8abcb42f]*/ { Py_off_t pos; @@ -1274,11 +1325,16 @@ _io__Buffered_tell_impl(buffered *self) if (pos == -1) return NULL; pos -= RAW_OFFSET(self); - /* TODO: sanity check (pos >= 0) */ + + // GH-95782 + if (pos < 0) + pos = 0; + return PyLong_FromOff_t(pos); } /*[clinic input] +@critical_section _io._Buffered.seek target as targetobj: object whence: int = 0 @@ -1287,7 +1343,7 @@ _io._Buffered.seek static PyObject * _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) -/*[clinic end generated code: output=7ae0e8dc46efdefb input=a9c4920bfcba6163]*/ +/*[clinic end generated code: output=7ae0e8dc46efdefb input=b5a12be70e0ad07b]*/ { Py_off_t target, n; PyObject *res = NULL; @@ -1343,6 +1399,11 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) offset = target; if (offset >= -self->pos && offset <= avail) { self->pos += offset; + + // GH-95782 + if (current - avail + offset < 0) + return PyLong_FromOff_t(0); + return PyLong_FromOff_t(current - avail + offset); } } @@ -1376,6 +1437,7 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) } /*[clinic input] +@critical_section _io._Buffered.truncate cls: defining_class pos: object = None @@ -1384,7 +1446,7 @@ _io._Buffered.truncate static PyObject * _io__Buffered_truncate_impl(buffered *self, PyTypeObject *cls, PyObject *pos) -/*[clinic end generated code: output=fe3882fbffe79f1a input=f5b737d97d76303f]*/ +/*[clinic end generated code: output=fe3882fbffe79f1a input=e3cbf794575bd794]*/ { PyObject *res = NULL; @@ -1998,6 +2060,7 @@ _bufferedwriter_flush_unlocked(buffered *self) } /*[clinic input] +@critical_section _io.BufferedWriter.write buffer: Py_buffer / @@ -2005,7 +2068,7 @@ _io.BufferedWriter.write static PyObject * _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) -/*[clinic end generated code: output=7f8d1365759bfc6b input=dd87dd85fc7f8850]*/ +/*[clinic end generated code: output=7f8d1365759bfc6b input=6a9c041de0c337be]*/ { PyObject *res = NULL; Py_ssize_t written, avail, remaining; @@ -2029,7 +2092,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) self->raw_pos = 0; } avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t); - if (buffer->len <= avail) { + if (buffer->len <= avail && buffer->len < self->buffer_size) { memcpy(self->buffer + self->pos, buffer->buf, buffer->len); if (!VALID_WRITE_BUFFER(self) || self->write_pos > self->pos) { self->write_pos = self->pos; @@ -2098,7 +2161,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Then write buf itself. At this point the buffer has been emptied. */ remaining = buffer->len; written = 0; - while (remaining > self->buffer_size) { + while (remaining >= self->buffer_size) { Py_ssize_t n = _bufferedwriter_raw_write( self, (char *) buffer->buf + written, buffer->len - written); if (n == -1) { @@ -2468,8 +2531,8 @@ static PyMethodDef bufferedreader_methods[] = { _IO__BUFFERED_TRUNCATE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2482,9 +2545,9 @@ static PyMemberDef bufferedreader_members[] = { }; static PyGetSetDef bufferedreader_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2528,8 +2591,8 @@ static PyMethodDef bufferedwriter_methods[] = { _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2542,9 +2605,9 @@ static PyMemberDef bufferedwriter_members[] = { }; static PyGetSetDef bufferedwriter_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2646,8 +2709,8 @@ static PyMethodDef bufferedrandom_methods[] = { _IO_BUFFEREDWRITER_WRITE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2660,9 +2723,9 @@ static PyMemberDef bufferedrandom_members[] = { }; static PyGetSetDef bufferedrandom_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index f3074203f54ea2..fb66d3db0f7a1f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -126,12 +126,13 @@ unshare_buffer(bytesio *self, size_t size) static int resize_buffer(bytesio *self, size_t size) { + assert(self->buf != NULL); + assert(self->exports == 0); + /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ size_t alloc = PyBytes_GET_SIZE(self->buf); - assert(self->buf != NULL); - /* For simplicity, stay in the range of the signed type. Anyway, Python doesn't allow strings to be longer than this. */ if (size > PY_SSIZE_T_MAX) @@ -154,9 +155,6 @@ resize_buffer(bytesio *self, size_t size) alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(char)) - goto overflow; - if (SHARED_BUF(self)) { if (unshare_buffer(self, alloc) < 0) return -1; @@ -989,7 +987,9 @@ static int bytesio_clear(bytesio *self) { Py_CLEAR(self->dict); - Py_CLEAR(self->buf); + if (self->exports == 0) { + Py_CLEAR(self->buf); + } return 0; } @@ -1074,7 +1074,7 @@ bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) "bytesiobuf_getbuffer: view==NULL argument is obsolete"); return -1; } - if (SHARED_BUF(b)) { + if (b->exports == 0 && SHARED_BUF(b)) { if (unshare_buffer(b, b->string_size) < 0) return -1; } @@ -1094,13 +1094,6 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } -static int -bytesiobuf_clear(bytesiobuf *self) -{ - Py_CLEAR(self->source); - return 0; -} - static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { @@ -1115,7 +1108,7 @@ bytesiobuf_dealloc(bytesiobuf *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - (void)bytesiobuf_clear(self); + Py_CLEAR(self->source); tp->tp_free(self); Py_DECREF(tp); } @@ -1123,7 +1116,6 @@ bytesiobuf_dealloc(bytesiobuf *self) static PyType_Slot bytesiobuf_slots[] = { {Py_tp_dealloc, bytesiobuf_dealloc}, {Py_tp_traverse, bytesiobuf_traverse}, - {Py_tp_clear, bytesiobuf_clear}, // Buffer protocol {Py_bf_getbuffer, bytesiobuf_getbuffer}, diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 49a3573ab97473..112408a95df036 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_io_open__doc__, "open($module, /, file, mode=\'r\', buffering=-1, encoding=None,\n" @@ -403,4 +404,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=aaf96c8d9bd20abc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5d60f4e778a600a4 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 7577bdec5c3b20..708bef638887e2 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -4,9 +4,11 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_BadArgument() PyDoc_STRVAR(_io__BufferedIOBase_readinto__doc__, "readinto($self, buffer, /)\n" @@ -29,11 +31,9 @@ _io__BufferedIOBase_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__BufferedIOBase_readinto_impl(self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -65,11 +65,9 @@ _io__BufferedIOBase_readinto1(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto1", "argument", "contiguous buffer", arg); - goto exit; - } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__BufferedIOBase_readinto1_impl(self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -98,7 +96,7 @@ _io__BufferedIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__BufferedIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -289,7 +287,13 @@ _io__Buffered___sizeof___impl(buffered *self); static PyObject * _io__Buffered___sizeof__(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered___sizeof___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered___sizeof___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered__dealloc_warn__doc__, @@ -314,7 +318,40 @@ _io__Buffered_simple_flush_impl(buffered *self); static PyObject * _io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_simple_flush_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_simple_flush_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io__Buffered_closed_HAS_DOCSTR) +# define _io__Buffered_closed_DOCSTR _io__Buffered_closed__doc__ +#else +# define _io__Buffered_closed_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_CLOSED_GETSETDEF) +# undef _IO__BUFFERED_CLOSED_GETSETDEF +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, (setter)_io__Buffered_closed_set, _io__Buffered_closed_DOCSTR}, +#else +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, NULL, _io__Buffered_closed_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_closed_get_impl(buffered *self); + +static PyObject * +_io__Buffered_closed_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_close__doc__, @@ -331,7 +368,13 @@ _io__Buffered_close_impl(buffered *self); static PyObject * _io__Buffered_close(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_detach__doc__, @@ -348,7 +391,13 @@ _io__Buffered_detach_impl(buffered *self); static PyObject * _io__Buffered_detach(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_detach_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_detach_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_seekable__doc__, @@ -365,7 +414,13 @@ _io__Buffered_seekable_impl(buffered *self); static PyObject * _io__Buffered_seekable(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_seekable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_seekable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_readable__doc__, @@ -382,7 +437,13 @@ _io__Buffered_readable_impl(buffered *self); static PyObject * _io__Buffered_readable(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_readable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_readable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_writable__doc__, @@ -399,7 +460,67 @@ _io__Buffered_writable_impl(buffered *self); static PyObject * _io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_writable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_writable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io__Buffered_name_HAS_DOCSTR) +# define _io__Buffered_name_DOCSTR _io__Buffered_name__doc__ +#else +# define _io__Buffered_name_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_NAME_GETSETDEF) +# undef _IO__BUFFERED_NAME_GETSETDEF +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, (setter)_io__Buffered_name_set, _io__Buffered_name_DOCSTR}, +#else +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, NULL, _io__Buffered_name_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_name_get_impl(buffered *self); + +static PyObject * +_io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_name_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io__Buffered_mode_HAS_DOCSTR) +# define _io__Buffered_mode_DOCSTR _io__Buffered_mode__doc__ +#else +# define _io__Buffered_mode_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_MODE_GETSETDEF) +# undef _IO__BUFFERED_MODE_GETSETDEF +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, (setter)_io__Buffered_mode_set, _io__Buffered_mode_DOCSTR}, +#else +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, NULL, _io__Buffered_mode_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_mode_get_impl(buffered *self); + +static PyObject * +_io__Buffered_mode_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_mode_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_fileno__doc__, @@ -416,7 +537,13 @@ _io__Buffered_fileno_impl(buffered *self); static PyObject * _io__Buffered_fileno(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_fileno_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_isatty__doc__, @@ -433,7 +560,13 @@ _io__Buffered_isatty_impl(buffered *self); static PyObject * _io__Buffered_isatty(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_isatty_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_isatty_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_flush__doc__, @@ -450,7 +583,13 @@ _io__Buffered_flush_impl(buffered *self); static PyObject * _io__Buffered_flush(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_flush_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_flush_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_peek__doc__, @@ -489,7 +628,9 @@ _io__Buffered_peek(buffered *self, PyObject *const *args, Py_ssize_t nargs) size = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_peek_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -522,7 +663,9 @@ _io__Buffered_read(buffered *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_read_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -564,7 +707,9 @@ _io__Buffered_read1(buffered *self, PyObject *const *args, Py_ssize_t nargs) n = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_read1_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -591,11 +736,9 @@ _io__Buffered_readinto(buffered *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_readinto_impl(self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -627,11 +770,9 @@ _io__Buffered_readinto1(buffered *self, PyObject *arg) _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto1", "argument", "contiguous buffer", arg); - goto exit; - } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_readinto1_impl(self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -669,7 +810,9 @@ _io__Buffered_readline(buffered *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_readline_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -689,7 +832,13 @@ _io__Buffered_tell_impl(buffered *self); static PyObject * _io__Buffered_tell(buffered *self, PyObject *Py_UNUSED(ignored)) { - return _io__Buffered_tell_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_tell_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io__Buffered_seek__doc__, @@ -722,7 +871,9 @@ _io__Buffered_seek(buffered *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_seek_impl(self, targetobj, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -768,7 +919,9 @@ _io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, } pos = args[0]; skip_optional_posonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io__Buffered_truncate_impl(self, cls, pos); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -944,11 +1097,9 @@ _io_BufferedWriter_write(buffered *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BufferedWriter_write_impl(self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -1094,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=f940cea085f0bf91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8eead000083dc5fa input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index d42ab48cef2859..620e9e3b84ea19 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_BytesIO_readable__doc__, "readable($self, /)\n" @@ -95,7 +96,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); return NULL; } @@ -331,10 +332,6 @@ _io_BytesIO_readinto(bytesio *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io_BytesIO_readinto_impl(self, &buffer); exit: @@ -537,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b753fdf1ba36c461 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index deb99fa9d99bd0..5b5487d63eba90 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_io_FileIO_close__doc__, "close($self, /)\n" @@ -26,7 +27,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * _io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -248,10 +249,6 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io_FileIO_readinto_impl(self, cls, &buffer); exit: @@ -380,10 +377,6 @@ _io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssiz if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io_FileIO_write_impl(self, cls, &b); exit: @@ -535,4 +528,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=2ce6ce923ccef86e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 9ac80b687925a4..a35cac7dc0b8d7 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -2,7 +2,11 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_io__IOBase_seek__doc__, "seek($self, offset, whence=os.SEEK_SET, /)\n" @@ -261,7 +265,7 @@ _io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); return NULL; } @@ -437,4 +441,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=95e1633805d10294 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dab5e9323d191e32 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index ccdae18d2af24a..6bdb2181985f7d 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -7,6 +7,8 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_StringIO_getvalue__doc__, "getvalue($self, /)\n" @@ -23,7 +25,13 @@ _io_StringIO_getvalue_impl(stringio *self); static PyObject * _io_StringIO_getvalue(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_getvalue_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_getvalue_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_StringIO_tell__doc__, @@ -41,7 +49,13 @@ _io_StringIO_tell_impl(stringio *self); static PyObject * _io_StringIO_tell(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_tell_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_tell_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_StringIO_read__doc__, @@ -75,7 +89,9 @@ _io_StringIO_read(stringio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_StringIO_read_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -111,7 +127,9 @@ _io_StringIO_readline(stringio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_StringIO_readline_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -149,7 +167,9 @@ _io_StringIO_truncate(stringio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_StringIO_truncate_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -203,7 +223,9 @@ _io_StringIO_seek(stringio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_StringIO_seek_impl(self, pos, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -221,6 +243,21 @@ PyDoc_STRVAR(_io_StringIO_write__doc__, #define _IO_STRINGIO_WRITE_METHODDEF \ {"write", (PyCFunction)_io_StringIO_write, METH_O, _io_StringIO_write__doc__}, +static PyObject * +_io_StringIO_write_impl(stringio *self, PyObject *obj); + +static PyObject * +_io_StringIO_write(stringio *self, PyObject *obj) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_write_impl(self, obj); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_io_StringIO_close__doc__, "close($self, /)\n" "--\n" @@ -241,7 +278,13 @@ _io_StringIO_close_impl(stringio *self); static PyObject * _io_StringIO_close(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_StringIO___init____doc__, @@ -329,7 +372,13 @@ _io_StringIO_readable_impl(stringio *self); static PyObject * _io_StringIO_readable(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_readable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_readable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_StringIO_writable__doc__, @@ -347,7 +396,13 @@ _io_StringIO_writable_impl(stringio *self); static PyObject * _io_StringIO_writable(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_writable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_writable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_StringIO_seekable__doc__, @@ -365,6 +420,139 @@ _io_StringIO_seekable_impl(stringio *self); static PyObject * _io_StringIO_seekable(stringio *self, PyObject *Py_UNUSED(ignored)) { - return _io_StringIO_seekable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_seekable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_io_StringIO___getstate____doc__, +"__getstate__($self, /)\n" +"--\n" +"\n"); + +#define _IO_STRINGIO___GETSTATE___METHODDEF \ + {"__getstate__", (PyCFunction)_io_StringIO___getstate__, METH_NOARGS, _io_StringIO___getstate____doc__}, + +static PyObject * +_io_StringIO___getstate___impl(stringio *self); + +static PyObject * +_io_StringIO___getstate__(stringio *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO___getstate___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_io_StringIO___setstate____doc__, +"__setstate__($self, state, /)\n" +"--\n" +"\n"); + +#define _IO_STRINGIO___SETSTATE___METHODDEF \ + {"__setstate__", (PyCFunction)_io_StringIO___setstate__, METH_O, _io_StringIO___setstate____doc__}, + +static PyObject * +_io_StringIO___setstate___impl(stringio *self, PyObject *state); + +static PyObject * +_io_StringIO___setstate__(stringio *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO___setstate___impl(self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_StringIO_closed_HAS_DOCSTR) +# define _io_StringIO_closed_DOCSTR _io_StringIO_closed__doc__ +#else +# define _io_StringIO_closed_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_CLOSED_GETSETDEF) +# undef _IO_STRINGIO_CLOSED_GETSETDEF +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, (setter)_io_StringIO_closed_set, _io_StringIO_closed_DOCSTR}, +#else +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, NULL, _io_StringIO_closed_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_closed_get_impl(stringio *self); + +static PyObject * +_io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_StringIO_line_buffering_HAS_DOCSTR) +# define _io_StringIO_line_buffering_DOCSTR _io_StringIO_line_buffering__doc__ +#else +# define _io_StringIO_line_buffering_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) +# undef _IO_STRINGIO_LINE_BUFFERING_GETSETDEF +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, (setter)_io_StringIO_line_buffering_set, _io_StringIO_line_buffering_DOCSTR}, +#else +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, NULL, _io_StringIO_line_buffering_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_line_buffering_get_impl(stringio *self); + +static PyObject * +_io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_line_buffering_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_StringIO_newlines_HAS_DOCSTR) +# define _io_StringIO_newlines_DOCSTR _io_StringIO_newlines__doc__ +#else +# define _io_StringIO_newlines_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) +# undef _IO_STRINGIO_NEWLINES_GETSETDEF +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, (setter)_io_StringIO_newlines_set, _io_StringIO_newlines_DOCSTR}, +#else +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, NULL, _io_StringIO_newlines_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_newlines_get_impl(stringio *self); + +static PyObject * +_io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_newlines_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=57e86cd679344ee7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ffea20cd32d4cd8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index bc14327388fb2f..669e2aa637ebbf 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -4,9 +4,11 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "detach($self, /)\n" @@ -25,7 +27,7 @@ _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls); static PyObject * _io__TextIOBase_detach(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "detach() takes no arguments"); return NULL; } @@ -199,6 +201,89 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, return return_value; } +PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, +"Encoding of the text stream.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_encoding_HAS_DOCSTR + +#if defined(_io__TextIOBase_encoding_HAS_DOCSTR) +# define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ +#else +# define _io__TextIOBase_encoding_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_ENCODING_GETSETDEF) +# undef _IO__TEXTIOBASE_ENCODING_GETSETDEF +# define _IO__TEXTIOBASE_ENCODING_GETSETDEF {"encoding", (getter)_io__TextIOBase_encoding_get, (setter)_io__TextIOBase_encoding_set, _io__TextIOBase_encoding_DOCSTR}, +#else +# define _IO__TEXTIOBASE_ENCODING_GETSETDEF {"encoding", (getter)_io__TextIOBase_encoding_get, NULL, _io__TextIOBase_encoding_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_encoding_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_encoding_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_encoding_get_impl(self); +} + +PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, +"Line endings translated so far.\n" +"\n" +"Only line endings translated during reading are considered.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_newlines_HAS_DOCSTR + +#if defined(_io__TextIOBase_newlines_HAS_DOCSTR) +# define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ +#else +# define _io__TextIOBase_newlines_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_NEWLINES_GETSETDEF) +# undef _IO__TEXTIOBASE_NEWLINES_GETSETDEF +# define _IO__TEXTIOBASE_NEWLINES_GETSETDEF {"newlines", (getter)_io__TextIOBase_newlines_get, (setter)_io__TextIOBase_newlines_set, _io__TextIOBase_newlines_DOCSTR}, +#else +# define _IO__TEXTIOBASE_NEWLINES_GETSETDEF {"newlines", (getter)_io__TextIOBase_newlines_get, NULL, _io__TextIOBase_newlines_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_newlines_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_newlines_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_newlines_get_impl(self); +} + +PyDoc_STRVAR(_io__TextIOBase_errors__doc__, +"The error setting of the decoder or encoder.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_errors_HAS_DOCSTR + +#if defined(_io__TextIOBase_errors_HAS_DOCSTR) +# define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ +#else +# define _io__TextIOBase_errors_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_ERRORS_GETSETDEF) +# undef _IO__TEXTIOBASE_ERRORS_GETSETDEF +# define _IO__TEXTIOBASE_ERRORS_GETSETDEF {"errors", (getter)_io__TextIOBase_errors_get, (setter)_io__TextIOBase_errors_set, _io__TextIOBase_errors_DOCSTR}, +#else +# define _IO__TEXTIOBASE_ERRORS_GETSETDEF {"errors", (getter)_io__TextIOBase_errors_get, NULL, _io__TextIOBase_errors_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_errors_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_errors_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_errors_get_impl(self); +} + PyDoc_STRVAR(_io_IncrementalNewlineDecoder___init____doc__, "IncrementalNewlineDecoder(decoder, translate, errors=\'strict\')\n" "--\n" @@ -629,7 +714,9 @@ _io_TextIOWrapper_reconfigure(textio *self, PyObject *const *args, Py_ssize_t na } write_through_obj = args[4]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_reconfigure_impl(self, encoding, errors, newline_obj, line_buffering_obj, write_through_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -649,7 +736,13 @@ _io_TextIOWrapper_detach_impl(textio *self); static PyObject * _io_TextIOWrapper_detach(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_detach_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_detach_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_write__doc__, @@ -674,7 +767,9 @@ _io_TextIOWrapper_write(textio *self, PyObject *arg) goto exit; } text = arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_write_impl(self, text); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -707,7 +802,9 @@ _io_TextIOWrapper_read(textio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_read_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -749,7 +846,9 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs size = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_readline_impl(self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -803,7 +902,9 @@ _io_TextIOWrapper_seek(textio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_seek_impl(self, cookieObj, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -827,7 +928,13 @@ _io_TextIOWrapper_tell_impl(textio *self); static PyObject * _io_TextIOWrapper_tell(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_tell_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_tell_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_truncate__doc__, @@ -855,7 +962,9 @@ _io_TextIOWrapper_truncate(textio *self, PyObject *const *args, Py_ssize_t nargs } pos = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper_truncate_impl(self, pos); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -875,7 +984,13 @@ _io_TextIOWrapper_fileno_impl(textio *self); static PyObject * _io_TextIOWrapper_fileno(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_fileno_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_seekable__doc__, @@ -892,7 +1007,13 @@ _io_TextIOWrapper_seekable_impl(textio *self); static PyObject * _io_TextIOWrapper_seekable(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_seekable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_seekable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_readable__doc__, @@ -909,7 +1030,13 @@ _io_TextIOWrapper_readable_impl(textio *self); static PyObject * _io_TextIOWrapper_readable(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_readable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_readable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_writable__doc__, @@ -926,7 +1053,13 @@ _io_TextIOWrapper_writable_impl(textio *self); static PyObject * _io_TextIOWrapper_writable(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_writable_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_writable_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_isatty__doc__, @@ -943,7 +1076,13 @@ _io_TextIOWrapper_isatty_impl(textio *self); static PyObject * _io_TextIOWrapper_isatty(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_isatty_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_isatty_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_flush__doc__, @@ -960,7 +1099,13 @@ _io_TextIOWrapper_flush_impl(textio *self); static PyObject * _io_TextIOWrapper_flush(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_flush_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_flush_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper_close__doc__, @@ -977,6 +1122,174 @@ _io_TextIOWrapper_close_impl(textio *self); static PyObject * _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { - return _io_TextIOWrapper_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_name_HAS_DOCSTR) +# define _io_TextIOWrapper_name_DOCSTR _io_TextIOWrapper_name__doc__ +#else +# define _io_TextIOWrapper_name_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NAME_GETSETDEF +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, (setter)_io_TextIOWrapper_name_set, _io_TextIOWrapper_name_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, NULL, _io_TextIOWrapper_name_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_name_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_name_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_closed_HAS_DOCSTR) +# define _io_TextIOWrapper_closed_DOCSTR _io_TextIOWrapper_closed__doc__ +#else +# define _io_TextIOWrapper_closed_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, (setter)_io_TextIOWrapper_closed_set, _io_TextIOWrapper_closed_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, NULL, _io_TextIOWrapper_closed_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_closed_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_newlines_HAS_DOCSTR) +# define _io_TextIOWrapper_newlines_DOCSTR _io_TextIOWrapper_newlines__doc__ +#else +# define _io_TextIOWrapper_newlines_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, (setter)_io_TextIOWrapper_newlines_set, _io_TextIOWrapper_newlines_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, NULL, _io_TextIOWrapper_newlines_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_newlines_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_newlines_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_errors_HAS_DOCSTR) +# define _io_TextIOWrapper_errors_DOCSTR _io_TextIOWrapper_errors__doc__ +#else +# define _io_TextIOWrapper_errors_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, (setter)_io_TextIOWrapper_errors_set, _io_TextIOWrapper_errors_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, NULL, _io_TextIOWrapper_errors_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_errors_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_errors_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper__CHUNK_SIZE_HAS_DOCSTR) +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ +#else +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, NULL, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_HAS_DOCSTR) +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ +#else +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", NULL, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, +#endif + +static int +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value); + +static int +_io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=175e1723a462a722 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=04cb7c67791a9ec1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index ecc71e552c23f4..4696ecc5c843e6 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #if defined(HAVE_WINDOWS_CONSOLE_IO) @@ -28,7 +29,7 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * _io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } @@ -246,10 +247,6 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io__WindowsConsoleIO_readinto_impl(self, cls, &buffer); exit: @@ -390,10 +387,6 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *con if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io__WindowsConsoleIO_write_impl(self, cls, &b); exit: @@ -464,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=37febc4c96732b3b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8a73ea0365b7a3..5d9d87d6118a75 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -54,6 +54,9 @@ # define SMALLCHUNK BUFSIZ #endif +/* Size at which a buffer is considered "large" and behavior should change to + avoid excessive memory allocation */ +#define LARGE_BUFFER_CUTOFF_SIZE 65536 /*[clinic input] module _io @@ -72,6 +75,7 @@ typedef struct { unsigned int closefd : 1; char finalizing; unsigned int blksize; + Py_off_t estimated_size; PyObject *weakreflist; PyObject *dict; } fileio; @@ -157,7 +161,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) return res; } - PyObject *exc; + PyObject *exc = NULL; if (res == NULL) { exc = PyErr_GetRaisedException(); } @@ -196,6 +200,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->appending = 0; self->seekable = -1; self->blksize = 0; + self->estimated_size = -1; self->closefd = 1; self->weakreflist = NULL; } @@ -269,6 +274,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { @@ -475,6 +487,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (fdfstat.st_blksize > 1) self->blksize = fdfstat.st_blksize; #endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ + if (fdfstat.st_size < PY_SSIZE_T_MAX) { + self->estimated_size = (Py_off_t)fdfstat.st_size; + } } #if defined(MS_WINDOWS) || defined(__CYGWIN__) @@ -677,7 +692,7 @@ new_buffersize(fileio *self, size_t currentsize) giving us amortized linear-time behavior. For bigger sizes, use a less-than-double growth factor to avoid excessive allocation. */ assert(currentsize <= PY_SSIZE_T_MAX); - if (currentsize > 65536) + if (currentsize > LARGE_BUFFER_CUTOFF_SIZE) addend = currentsize >> 3; else addend = 256 + currentsize; @@ -700,43 +715,56 @@ static PyObject * _io_FileIO_readall_impl(fileio *self) /*[clinic end generated code: output=faa0292b213b4022 input=dbdc137f55602834]*/ { - struct _Py_stat_struct status; Py_off_t pos, end; PyObject *result; Py_ssize_t bytes_read = 0; Py_ssize_t n; size_t bufsize; - int fstat_result; - if (self->fd < 0) + if (self->fd < 0) { return err_closed(); + } - Py_BEGIN_ALLOW_THREADS - _Py_BEGIN_SUPPRESS_IPH -#ifdef MS_WINDOWS - pos = _lseeki64(self->fd, 0L, SEEK_CUR); -#else - pos = lseek(self->fd, 0L, SEEK_CUR); -#endif - _Py_END_SUPPRESS_IPH - fstat_result = _Py_fstat_noraise(self->fd, &status); - Py_END_ALLOW_THREADS - - if (fstat_result == 0) - end = status.st_size; - else - end = (Py_off_t)-1; - - if (end > 0 && end >= pos && pos >= 0 && end - pos < PY_SSIZE_T_MAX) { + end = self->estimated_size; + if (end <= 0) { + /* Use a default size and resize as needed. */ + bufsize = SMALLCHUNK; + } + else { /* This is probably a real file, so we try to allocate a buffer one byte larger than the rest of the file. If the calculation is right then we should get EOF without having to enlarge the buffer. */ - bufsize = (size_t)(end - pos + 1); - } else { - bufsize = SMALLCHUNK; + if (end > _PY_READ_MAX - 1) { + bufsize = _PY_READ_MAX; + } + else { + bufsize = (size_t)end + 1; + } + + /* While a lot of code does open().read() to get the whole contents + of a file it is possible a caller seeks/reads a ways into the file + then calls readall() to get the rest, which would result in allocating + more than required. Guard against that for larger files where we expect + the I/O time to dominate anyways while keeping small files fast. */ + if (bufsize > LARGE_BUFFER_CUTOFF_SIZE) { + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH +#ifdef MS_WINDOWS + pos = _lseeki64(self->fd, 0L, SEEK_CUR); +#else + pos = lseek(self->fd, 0L, SEEK_CUR); +#endif + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + + if (end >= pos && pos >= 0 && (end - pos) < (_PY_READ_MAX - 1)) { + bufsize = (size_t)(end - pos) + 1; + } + } } + result = PyBytes_FromStringAndSize(NULL, bufsize); if (result == NULL) return NULL; @@ -776,7 +804,6 @@ _io_FileIO_readall_impl(fileio *self) return NULL; } bytes_read += n; - pos += n; } if (PyBytes_GET_SIZE(result) > bytes_read) { @@ -1067,6 +1094,12 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) return NULL; } + /* Sometimes a large file is truncated. While estimated_size is used as a + estimate, that it is much larger than the actual size can result in a + significant over allocation and sometimes a MemoryError / running out of + memory. */ + self->estimated_size = pos; + return posobj; } #endif /* HAVE_FTRUNCATE */ @@ -1100,31 +1133,32 @@ static PyObject * fileio_repr(fileio *self) { PyObject *nameobj, *res; + const char *type_name = Py_TYPE((PyObject *) self)->tp_name; - if (self->fd < 0) - return PyUnicode_FromFormat("<_io.FileIO [closed]>"); + if (self->fd < 0) { + return PyUnicode_FromFormat("<%.100s [closed]>", type_name); + } if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { return NULL; } if (nameobj == NULL) { res = PyUnicode_FromFormat( - "<_io.FileIO fd=%d mode='%s' closefd=%s>", - self->fd, mode_string(self), self->closefd ? "True" : "False"); + "<%.100s fd=%d mode='%s' closefd=%s>", + type_name, self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { int status = Py_ReprEnter((PyObject *)self); res = NULL; if (status == 0) { res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + "<%.100s name=%R mode='%s' closefd=%s>", + type_name, nameobj, mode_string(self), self->closefd ? "True" : "False"); Py_ReprLeave((PyObject *)self); } else if (status > 0) { PyErr_Format(PyExc_RuntimeError, - "reentrant call inside %s.__repr__", - Py_TYPE(self)->tp_name); + "reentrant call inside %.100s.__repr__", type_name); } Py_DECREF(nameobj); } @@ -1170,8 +1204,8 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 4da8e5bd572d74..184e0b7d1aa7f1 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -66,12 +66,19 @@ PyDoc_STRVAR(iobase_doc, "with open('spam.txt', 'r') as fp:\n" " fp.write('Spam and eggs!')\n"); -/* Use this macro whenever you want to check the internal `closed` status + +/* Internal methods */ + +/* Use this function whenever you want to check the internal `closed` status of the IOBase object rather than the virtual `closed` attribute as returned by whatever subclass. */ +static int +iobase_is_closed(PyObject *self) +{ + return PyObject_HasAttrWithError(self, &_Py_ID(__IOBase_closed)); +} -/* Internal methods */ static PyObject * iobase_unsupported(_PyIO_State *state, const char *message) { @@ -145,14 +152,6 @@ _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, return iobase_unsupported(state, "truncate"); } -static int -iobase_is_closed(PyObject *self) -{ - /* This gets the derived attribute, which is *not* __IOBase_closed - in most cases! */ - return PyObject_HasAttrWithError(self, &_Py_ID(__IOBase_closed)); -} - /* Flush and close methods */ /*[clinic input] diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 1856b07108bab6..6d48bcb552b4bf 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -45,6 +45,10 @@ typedef struct { _PyIO_State *module_state; } stringio; +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) +#include "clinic/stringio.c.h" +#undef clinic_state + static int _io_StringIO___init__(PyObject *self, PyObject *args, PyObject *kwargs); #define CHECK_INITIALIZED(self) \ @@ -192,7 +196,7 @@ write_str(stringio *self, PyObject *obj) } if (self->writenl) { PyObject *translated = PyUnicode_Replace( - decoded, &_Py_STR(newline), self->writenl, -1); + decoded, _Py_LATIN1_CHR('\n'), self->writenl, -1); Py_SETREF(decoded, translated); } if (decoded == NULL) @@ -263,6 +267,7 @@ write_str(stringio *self, PyObject *obj) } /*[clinic input] +@critical_section _io.StringIO.getvalue Retrieve the entire contents of the object. @@ -270,7 +275,7 @@ Retrieve the entire contents of the object. static PyObject * _io_StringIO_getvalue_impl(stringio *self) -/*[clinic end generated code: output=27b6a7bfeaebce01 input=d23cb81d6791cf88]*/ +/*[clinic end generated code: output=27b6a7bfeaebce01 input=fb5dee06b8d467f3]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -281,6 +286,7 @@ _io_StringIO_getvalue_impl(stringio *self) } /*[clinic input] +@critical_section _io.StringIO.tell Tell the current file position. @@ -288,7 +294,7 @@ Tell the current file position. static PyObject * _io_StringIO_tell_impl(stringio *self) -/*[clinic end generated code: output=2e87ac67b116c77b input=ec866ebaff02f405]*/ +/*[clinic end generated code: output=2e87ac67b116c77b input=98a08f3e2dae3550]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -296,6 +302,7 @@ _io_StringIO_tell_impl(stringio *self) } /*[clinic input] +@critical_section _io.StringIO.read size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -308,7 +315,7 @@ is reached. Return an empty string at EOF. static PyObject * _io_StringIO_read_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=ae8cf6002f71626c input=0921093383dfb92d]*/ +/*[clinic end generated code: output=ae8cf6002f71626c input=9fbef45d8aece8e7]*/ { Py_ssize_t n; Py_UCS4 *output; @@ -368,6 +375,7 @@ _stringio_readline(stringio *self, Py_ssize_t limit) } /*[clinic input] +@critical_section _io.StringIO.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -379,7 +387,7 @@ Returns an empty string if EOF is hit immediately. static PyObject * _io_StringIO_readline_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=cabd6452f1b7e85d input=a5bd70bf682aa276]*/ +/*[clinic end generated code: output=cabd6452f1b7e85d input=4d14b8495dea1d98]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -427,6 +435,7 @@ stringio_iternext(stringio *self) } /*[clinic input] +@critical_section _io.StringIO.truncate pos as size: Py_ssize_t(accept={int, NoneType}, c_default="self->pos") = None / @@ -440,7 +449,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_truncate_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=eb3aef8e06701365 input=5505cff90ca48b96]*/ +/*[clinic end generated code: output=eb3aef8e06701365 input=461b872dce238452]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -462,6 +471,7 @@ _io_StringIO_truncate_impl(stringio *self, Py_ssize_t size) } /*[clinic input] +@critical_section _io.StringIO.seek pos: Py_ssize_t whence: int = 0 @@ -478,7 +488,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=e3855b24e7cae06a]*/ +/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=c75ced09343a00d7]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -515,6 +525,7 @@ _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) } /*[clinic input] +@critical_section _io.StringIO.write s as obj: object / @@ -526,8 +537,8 @@ the length of the string. [clinic start generated code]*/ static PyObject * -_io_StringIO_write(stringio *self, PyObject *obj) -/*[clinic end generated code: output=0deaba91a15b94da input=cf96f3b16586e669]*/ +_io_StringIO_write_impl(stringio *self, PyObject *obj) +/*[clinic end generated code: output=d53b1d841d7db288 input=1561272c0da4651f]*/ { Py_ssize_t size; @@ -547,6 +558,7 @@ _io_StringIO_write(stringio *self, PyObject *obj) } /*[clinic input] +@critical_section _io.StringIO.close Close the IO object. @@ -559,7 +571,7 @@ This method has no effect if the file is already closed. static PyObject * _io_StringIO_close_impl(stringio *self) -/*[clinic end generated code: output=04399355cbe518f1 input=cbc10b45f35d6d46]*/ +/*[clinic end generated code: output=04399355cbe518f1 input=305d19aa29cc40b9]*/ { self->closed = 1; /* Free up some memory */ @@ -756,6 +768,7 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, /* Properties and pseudo-properties */ /*[clinic input] +@critical_section _io.StringIO.readable Returns True if the IO object can be read. @@ -763,7 +776,7 @@ Returns True if the IO object can be read. static PyObject * _io_StringIO_readable_impl(stringio *self) -/*[clinic end generated code: output=b19d44dd8b1ceb99 input=39ce068b224c21ad]*/ +/*[clinic end generated code: output=b19d44dd8b1ceb99 input=6cd2ffd65a8e8763]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -771,6 +784,7 @@ _io_StringIO_readable_impl(stringio *self) } /*[clinic input] +@critical_section _io.StringIO.writable Returns True if the IO object can be written. @@ -778,7 +792,7 @@ Returns True if the IO object can be written. static PyObject * _io_StringIO_writable_impl(stringio *self) -/*[clinic end generated code: output=13e4dd77187074ca input=7a691353aac38835]*/ +/*[clinic end generated code: output=13e4dd77187074ca input=1b3c63dbaa761c69]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -786,6 +800,7 @@ _io_StringIO_writable_impl(stringio *self) } /*[clinic input] +@critical_section _io.StringIO.seekable Returns True if the IO object can be seeked. @@ -793,7 +808,7 @@ Returns True if the IO object can be seeked. static PyObject * _io_StringIO_seekable_impl(stringio *self) -/*[clinic end generated code: output=4d20b4641c756879 input=4c606d05b32952e6]*/ +/*[clinic end generated code: output=4d20b4641c756879 input=a820fad2cf085fc3]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -812,8 +827,15 @@ _io_StringIO_seekable_impl(stringio *self) supported. */ +/*[clinic input] +@critical_section +_io.StringIO.__getstate__ + +[clinic start generated code]*/ + static PyObject * -stringio_getstate(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO___getstate___impl(stringio *self) +/*[clinic end generated code: output=780be4a996410199 input=76f27255ef83bb92]*/ { PyObject *initvalue = _io_StringIO_getvalue_impl(self); PyObject *dict; @@ -839,8 +861,17 @@ stringio_getstate(stringio *self, PyObject *Py_UNUSED(ignored)) return state; } +/*[clinic input] +@critical_section +_io.StringIO.__setstate__ + + state: object + / +[clinic start generated code]*/ + static PyObject * -stringio_setstate(stringio *self, PyObject *state) +_io_StringIO___setstate___impl(stringio *self, PyObject *state) +/*[clinic end generated code: output=cb3962bc6d5c5609 input=8a27784b11b82e47]*/ { PyObject *initarg; PyObject *position_obj; @@ -939,36 +970,53 @@ stringio_setstate(stringio *self, PyObject *state) Py_RETURN_NONE; } +/*[clinic input] +@critical_section +@getter +_io.StringIO.closed +[clinic start generated code]*/ static PyObject * -stringio_closed(stringio *self, void *context) +_io_StringIO_closed_get_impl(stringio *self) +/*[clinic end generated code: output=531ddca7954331d6 input=178d2ef24395fd49]*/ { CHECK_INITIALIZED(self); return PyBool_FromLong(self->closed); } +/*[clinic input] +@critical_section +@getter +_io.StringIO.line_buffering +[clinic start generated code]*/ + static PyObject * -stringio_line_buffering(stringio *self, void *context) +_io_StringIO_line_buffering_get_impl(stringio *self) +/*[clinic end generated code: output=360710e0112966ae input=6a7634e7f890745e]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); Py_RETURN_FALSE; } +/*[clinic input] +@critical_section +@getter +_io.StringIO.newlines +[clinic start generated code]*/ + static PyObject * -stringio_newlines(stringio *self, void *context) +_io_StringIO_newlines_get_impl(stringio *self) +/*[clinic end generated code: output=35d7c0b66d7e0160 input=092a14586718244b]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); - if (self->decoder == NULL) + if (self->decoder == NULL) { Py_RETURN_NONE; + } return PyObject_GetAttr(self->decoder, &_Py_ID(newlines)); } -#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) -#include "clinic/stringio.c.h" -#undef clinic_state - static struct PyMethodDef stringio_methods[] = { _IO_STRINGIO_CLOSE_METHODDEF _IO_STRINGIO_GETVALUE_METHODDEF @@ -983,21 +1031,21 @@ static struct PyMethodDef stringio_methods[] = { _IO_STRINGIO_READABLE_METHODDEF _IO_STRINGIO_WRITABLE_METHODDEF - {"__getstate__", (PyCFunction)stringio_getstate, METH_NOARGS}, - {"__setstate__", (PyCFunction)stringio_setstate, METH_O}, + _IO_STRINGIO___GETSTATE___METHODDEF + _IO_STRINGIO___SETSTATE___METHODDEF {NULL, NULL} /* sentinel */ }; static PyGetSetDef stringio_getset[] = { - {"closed", (getter)stringio_closed, NULL, NULL}, - {"newlines", (getter)stringio_newlines, NULL, NULL}, + _IO_STRINGIO_CLOSED_GETSETDEF + _IO_STRINGIO_NEWLINES_GETSETDEF /* (following comments straight off of the original Python wrapper:) XXX Cruft to support the TextIOWrapper API. This would only be meaningful if StringIO supported the buffer attribute. Hopefully, a better solution, than adding these pseudo-attributes, will be found. */ - {"line_buffering", (getter)stringio_line_buffering, NULL, NULL}, + _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {NULL} }; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 10ef8a803c50fd..c162d8106ec1fd 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -7,14 +7,14 @@ */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallMethod() -#include "pycore_codecs.h" // _PyCodecInfo_GetIncrementalDecoder() -#include "pycore_fileutils.h" // _Py_GetLocaleEncoding() -#include "pycore_interp.h" // PyInterpreterState.fs_codec -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_call.h" // _PyObject_CallMethod() +#include "pycore_codecs.h" // _PyCodecInfo_GetIncrementalDecoder() +#include "pycore_fileutils.h" // _Py_GetLocaleEncoding() +#include "pycore_interp.h" // PyInterpreterState.fs_codec +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "_iomodule.h" @@ -131,40 +131,52 @@ _io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, return _unsupported(state, "write"); } -PyDoc_STRVAR(textiobase_encoding_doc, - "Encoding of the text stream.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.encoding + +Encoding of the text stream. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_encoding_get(PyObject *self, void *context) +_io__TextIOBase_encoding_get_impl(PyObject *self) +/*[clinic end generated code: output=e0f5d8f548b92432 input=4736d7621dd38f43]*/ { Py_RETURN_NONE; } -PyDoc_STRVAR(textiobase_newlines_doc, - "Line endings translated so far.\n" - "\n" - "Only line endings translated during reading are considered.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.newlines + +Line endings translated so far. + +Only line endings translated during reading are considered. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_newlines_get(PyObject *self, void *context) +_io__TextIOBase_newlines_get_impl(PyObject *self) +/*[clinic end generated code: output=46ec147fb9f00c2a input=a5b196d076af1164]*/ { Py_RETURN_NONE; } -PyDoc_STRVAR(textiobase_errors_doc, - "The error setting of the decoder or encoder.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.errors + +The error setting of the decoder or encoder. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_errors_get(PyObject *self, void *context) +_io__TextIOBase_errors_get_impl(PyObject *self) +/*[clinic end generated code: output=c6623d6addcd087d input=974aa52d1db93a82]*/ { Py_RETURN_NONE; } @@ -179,9 +191,9 @@ static PyMethodDef textiobase_methods[] = { }; static PyGetSetDef textiobase_getset[] = { - {"encoding", (getter)textiobase_encoding_get, NULL, textiobase_encoding_doc}, - {"newlines", (getter)textiobase_newlines_get, NULL, textiobase_newlines_doc}, - {"errors", (getter)textiobase_errors_get, NULL, textiobase_errors_doc}, + _IO__TEXTIOBASE_ENCODING_GETSETDEF + _IO__TEXTIOBASE_NEWLINES_GETSETDEF + _IO__TEXTIOBASE_ERRORS_GETSETDEF {NULL} }; @@ -1020,15 +1032,10 @@ io_check_errors(PyObject *errors) return 0; } - Py_ssize_t name_length; - const char *name = PyUnicode_AsUTF8AndSize(errors, &name_length); + const char *name = _PyUnicode_AsUTF8NoNUL(errors); if (name == NULL) { return -1; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character in errors"); - return -1; - } PyObject *handler = PyCodec_LookupError(name); if (handler != NULL) { Py_DECREF(handler); @@ -1117,6 +1124,10 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, else if (io_check_errors(errors)) { return -1; } + const char *errors_str = _PyUnicode_AsUTF8NoNUL(errors); + if (errors_str == NULL) { + return -1; + } if (validate_newline(newline) < 0) { return -1; @@ -1189,11 +1200,11 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Build the decoder object */ _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->state = state; - if (_textiowrapper_set_decoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) + if (_textiowrapper_set_decoder(self, codec_info, errors_str) != 0) goto error; /* Build the encoder object */ - if (_textiowrapper_set_encoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) + if (_textiowrapper_set_encoder(self, codec_info, errors_str) != 0) goto error; /* Finished sorting out the codec details */ @@ -1289,35 +1300,46 @@ textiowrapper_change_encoding(textio *self, PyObject *encoding, errors = &_Py_ID(strict); } } + Py_INCREF(errors); + const char *c_encoding = PyUnicode_AsUTF8(encoding); + if (c_encoding == NULL) { + Py_DECREF(encoding); + Py_DECREF(errors); + return -1; + } const char *c_errors = PyUnicode_AsUTF8(errors); if (c_errors == NULL) { Py_DECREF(encoding); + Py_DECREF(errors); return -1; } // Create new encoder & decoder PyObject *codec_info = _PyCodec_LookupTextEncoding( - PyUnicode_AsUTF8(encoding), "codecs.open()"); + c_encoding, "codecs.open()"); if (codec_info == NULL) { Py_DECREF(encoding); + Py_DECREF(errors); return -1; } if (_textiowrapper_set_decoder(self, codec_info, c_errors) != 0 || _textiowrapper_set_encoder(self, codec_info, c_errors) != 0) { Py_DECREF(codec_info); Py_DECREF(encoding); + Py_DECREF(errors); return -1; } Py_DECREF(codec_info); Py_SETREF(self->encoding, encoding); - Py_SETREF(self->errors, Py_NewRef(errors)); + Py_SETREF(self->errors, errors); return _textiowrapper_fix_encoder_state(self); } /*[clinic input] +@critical_section _io.TextIOWrapper.reconfigure * encoding: object = None @@ -1337,12 +1359,32 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, PyObject *errors, PyObject *newline_obj, PyObject *line_buffering_obj, PyObject *write_through_obj) -/*[clinic end generated code: output=52b812ff4b3d4b0f input=671e82136e0f5822]*/ +/*[clinic end generated code: output=52b812ff4b3d4b0f input=dc3bd35ebda702a7]*/ { int line_buffering; int write_through; const char *newline = NULL; + if (encoding != Py_None && !PyUnicode_Check(encoding)) { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'encoding' must be str or None, not %s", + Py_TYPE(encoding)->tp_name); + return NULL; + } + if (errors != Py_None && !PyUnicode_Check(errors)) { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'errors' must be str or None, not %s", + Py_TYPE(errors)->tp_name); + return NULL; + } + if (newline_obj != NULL && newline_obj != Py_None && + !PyUnicode_Check(newline_obj)) + { + PyErr_Format(PyExc_TypeError, + "reconfigure() argument 'newline' must be str or None, not %s", + Py_TYPE(newline_obj)->tp_name); + return NULL; + } /* Check if something is in the read buffer */ if (self->decoded_chars != NULL) { if (encoding != Py_None || errors != Py_None || newline_obj != NULL) { @@ -1362,9 +1404,12 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, line_buffering = convert_optional_bool(line_buffering_obj, self->line_buffering); + if (line_buffering < 0) { + return NULL; + } write_through = convert_optional_bool(write_through_obj, self->write_through); - if (line_buffering < 0 || write_through < 0) { + if (write_through < 0) { return NULL; } @@ -1442,7 +1487,7 @@ textiowrapper_traverse(textio *self, visitproc visit, void *arg) } static PyObject * -textiowrapper_closed_get(textio *self, void *context); +_io_TextIOWrapper_closed_get_impl(textio *self); /* This macro takes some shortcuts to make the common case faster. */ #define CHECK_CLOSED(self) \ @@ -1453,7 +1498,7 @@ textiowrapper_closed_get(textio *self, void *context); if (self->raw != NULL) \ r = _PyFileIO_closed(self->raw); \ else { \ - _res = textiowrapper_closed_get(self, NULL); \ + _res = _io_TextIOWrapper_closed_get_impl(self); \ if (_res == NULL) \ return NULL; \ r = PyObject_IsTrue(_res); \ @@ -1499,12 +1544,13 @@ textiowrapper_closed_get(textio *self, void *context); /*[clinic input] +@critical_section _io.TextIOWrapper.detach [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_detach_impl(textio *self) -/*[clinic end generated code: output=7ba3715cd032d5f2 input=e5a71fbda9e1d9f9]*/ +/*[clinic end generated code: output=7ba3715cd032d5f2 input=c908a3b4ef203b0f]*/ { PyObject *buffer; CHECK_ATTACHED(self); @@ -1590,6 +1636,7 @@ _textiowrapper_writeflush(textio *self) } /*[clinic input] +@critical_section _io.TextIOWrapper.write text: unicode / @@ -1597,7 +1644,7 @@ _io.TextIOWrapper.write static PyObject * _io_TextIOWrapper_write_impl(textio *self, PyObject *text) -/*[clinic end generated code: output=d2deb0d50771fcec input=fdf19153584a0e44]*/ +/*[clinic end generated code: output=d2deb0d50771fcec input=73ec95c5c4a3489c]*/ { PyObject *ret; PyObject *b; @@ -1672,16 +1719,26 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) bytes_len = PyBytes_GET_SIZE(b); } - if (self->pending_bytes == NULL) { - self->pending_bytes_count = 0; - self->pending_bytes = b; - } - else if (self->pending_bytes_count + bytes_len > self->chunk_size) { - // Prevent to concatenate more than chunk_size data. - if (_textiowrapper_writeflush(self) < 0) { - Py_DECREF(b); - return NULL; + // We should avoid concatinating huge data. + // Flush the buffer before adding b to the buffer if b is not small. + // https://github.com/python/cpython/issues/87426 + if (bytes_len >= self->chunk_size) { + // _textiowrapper_writeflush() calls buffer.write(). + // self->pending_bytes can be appended during buffer->write() + // or other thread. + // We need to loop until buffer becomes empty. + // https://github.com/python/cpython/issues/118138 + // https://github.com/python/cpython/issues/119506 + while (self->pending_bytes != NULL) { + if (_textiowrapper_writeflush(self) < 0) { + Py_DECREF(b); + return NULL; + } } + } + + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); self->pending_bytes = b; } else if (!PyList_CheckExact(self->pending_bytes)) { @@ -1690,6 +1747,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) Py_DECREF(b); return NULL; } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); PyList_SET_ITEM(list, 0, self->pending_bytes); PyList_SET_ITEM(list, 1, b); self->pending_bytes = list; @@ -1715,8 +1775,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } } - textiowrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); + if (self->snapshot != NULL) { + textiowrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + } if (self->decoder) { ret = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(reset)); @@ -1901,6 +1963,7 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) } /*[clinic input] +@critical_section _io.TextIOWrapper.read size as n: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -1908,7 +1971,7 @@ _io.TextIOWrapper.read static PyObject * _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) -/*[clinic end generated code: output=7e651ce6cc6a25a6 input=123eecbfe214aeb8]*/ +/*[clinic end generated code: output=7e651ce6cc6a25a6 input=67d14c5661121377]*/ { PyObject *result = NULL, *chunks = NULL; @@ -1951,8 +2014,10 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (result == NULL) goto fail; - textiowrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); + if (self->snapshot != NULL) { + textiowrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + } return result; } else { @@ -2276,6 +2341,7 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) } /*[clinic input] +@critical_section _io.TextIOWrapper.readline size: Py_ssize_t = -1 / @@ -2283,7 +2349,7 @@ _io.TextIOWrapper.readline static PyObject * _io_TextIOWrapper_readline_impl(textio *self, Py_ssize_t size) -/*[clinic end generated code: output=344afa98804e8b25 input=56c7172483b36db6]*/ +/*[clinic end generated code: output=344afa98804e8b25 input=b65bab871dc3ddba]*/ { CHECK_ATTACHED(self); return _textiowrapper_readline(self, size); @@ -2340,7 +2406,7 @@ textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj) return -1; if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), - PY_LITTLE_ENDIAN, 0) < 0) { + PY_LITTLE_ENDIAN, 0, 1) < 0) { Py_DECREF(cookieLong); return -1; } @@ -2422,6 +2488,7 @@ _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie) } /*[clinic input] +@critical_section _io.TextIOWrapper.seek cookie as cookieObj: object Zero or an opaque number returned by tell(). @@ -2446,7 +2513,7 @@ and may raise exceptions. static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) -/*[clinic end generated code: output=0a15679764e2d04d input=0f68adcb02cf2823]*/ +/*[clinic end generated code: output=0a15679764e2d04d input=4bea78698be23d7e]*/ { PyObject *posobj; cookie_type cookie; @@ -2633,6 +2700,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) } /*[clinic input] +@critical_section _io.TextIOWrapper.tell Return the stream position as an opaque number. @@ -2643,7 +2711,7 @@ previous stream position. static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=0852d627d76fb520]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=415d6b4e4f8e6e8c]*/ { PyObject *res; PyObject *posobj = NULL; @@ -2869,6 +2937,7 @@ _io_TextIOWrapper_tell_impl(textio *self) } /*[clinic input] +@critical_section _io.TextIOWrapper.truncate pos: object = None / @@ -2876,7 +2945,7 @@ _io.TextIOWrapper.truncate static PyObject * _io_TextIOWrapper_truncate_impl(textio *self, PyObject *pos) -/*[clinic end generated code: output=90ec2afb9bb7745f input=56ec8baa65aea377]*/ +/*[clinic end generated code: output=90ec2afb9bb7745f input=8bddb320834c93ee]*/ { CHECK_ATTACHED(self) @@ -2892,10 +2961,11 @@ textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; int status; + const char *type_name = Py_TYPE(self)->tp_name; CHECK_INITIALIZED(self); - res = PyUnicode_FromString("<_io.TextIOWrapper"); + res = PyUnicode_FromFormat("<%.100s", type_name); if (res == NULL) return NULL; @@ -2903,8 +2973,8 @@ textiowrapper_repr(textio *self) if (status != 0) { if (status > 0) { PyErr_Format(PyExc_RuntimeError, - "reentrant call inside %s.__repr__", - Py_TYPE(self)->tp_name); + "reentrant call inside %.100s.__repr__", + type_name); } goto error; } @@ -2956,72 +3026,78 @@ textiowrapper_repr(textio *self) /* Inquiries */ /*[clinic input] +@critical_section _io.TextIOWrapper.fileno [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_fileno_impl(textio *self) -/*[clinic end generated code: output=21490a4c3da13e6c input=c488ca83d0069f9b]*/ +/*[clinic end generated code: output=21490a4c3da13e6c input=515e1196aceb97ab]*/ { CHECK_ATTACHED(self); return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(fileno)); } /*[clinic input] +@critical_section _io.TextIOWrapper.seekable [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_seekable_impl(textio *self) -/*[clinic end generated code: output=ab223dbbcffc0f00 input=8b005ca06e1fca13]*/ +/*[clinic end generated code: output=ab223dbbcffc0f00 input=71c4c092736c549b]*/ { CHECK_ATTACHED(self); return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(seekable)); } /*[clinic input] +@critical_section _io.TextIOWrapper.readable [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_readable_impl(textio *self) -/*[clinic end generated code: output=72ff7ba289a8a91b input=0704ea7e01b0d3eb]*/ +/*[clinic end generated code: output=72ff7ba289a8a91b input=80438d1f01b0a89b]*/ { CHECK_ATTACHED(self); return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(readable)); } /*[clinic input] +@critical_section _io.TextIOWrapper.writable [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_writable_impl(textio *self) -/*[clinic end generated code: output=a728c71790d03200 input=c41740bc9d8636e8]*/ +/*[clinic end generated code: output=a728c71790d03200 input=9d6c22befb0c340a]*/ { CHECK_ATTACHED(self); return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(writable)); } /*[clinic input] +@critical_section _io.TextIOWrapper.isatty [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_isatty_impl(textio *self) -/*[clinic end generated code: output=12be1a35bace882e input=fb68d9f2c99bbfff]*/ +/*[clinic end generated code: output=12be1a35bace882e input=7f83ff04d4d1733d]*/ { CHECK_ATTACHED(self); return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(isatty)); } /*[clinic input] +@critical_section _io.TextIOWrapper.flush [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_flush_impl(textio *self) -/*[clinic end generated code: output=59de9165f9c2e4d2 input=928c60590694ab85]*/ +/*[clinic end generated code: output=59de9165f9c2e4d2 input=3ac3bf521bfed59d]*/ { CHECK_ATTACHED(self); CHECK_CLOSED(self); @@ -3032,18 +3108,19 @@ _io_TextIOWrapper_flush_impl(textio *self) } /*[clinic input] +@critical_section _io.TextIOWrapper.close [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_close_impl(textio *self) -/*[clinic end generated code: output=056ccf8b4876e4f4 input=9c2114315eae1948]*/ +/*[clinic end generated code: output=056ccf8b4876e4f4 input=8e12d7079d5ac5c1]*/ { PyObject *res; int r; CHECK_ATTACHED(self); - res = textiowrapper_closed_get(self, NULL); + res = _io_TextIOWrapper_closed_get_impl(self); if (res == NULL) return NULL; r = PyObject_IsTrue(res); @@ -3117,22 +3194,43 @@ textiowrapper_iternext(textio *self) return line; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.name +[clinic start generated code]*/ + static PyObject * -textiowrapper_name_get(textio *self, void *context) +_io_TextIOWrapper_name_get_impl(textio *self) +/*[clinic end generated code: output=8c2f1d6d8756af40 input=26ecec9b39e30e07]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(name)); } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.closed +[clinic start generated code]*/ + static PyObject * -textiowrapper_closed_get(textio *self, void *context) +_io_TextIOWrapper_closed_get_impl(textio *self) +/*[clinic end generated code: output=b49b68f443a85e3c input=7dfcf43f63c7003d]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(closed)); } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.newlines +[clinic start generated code]*/ + static PyObject * -textiowrapper_newlines_get(textio *self, void *context) +_io_TextIOWrapper_newlines_get_impl(textio *self) +/*[clinic end generated code: output=53aa03ac35573180 input=610df647e514b3e8]*/ { PyObject *res; CHECK_ATTACHED(self); @@ -3144,30 +3242,51 @@ textiowrapper_newlines_get(textio *self, void *context) return res; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.errors +[clinic start generated code]*/ + static PyObject * -textiowrapper_errors_get(textio *self, void *context) +_io_TextIOWrapper_errors_get_impl(textio *self) +/*[clinic end generated code: output=dca3a3ef21b09484 input=b45f983e6d43c4d8]*/ { CHECK_INITIALIZED(self); return Py_NewRef(self->errors); } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ + static PyObject * -textiowrapper_chunk_size_get(textio *self, void *context) +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self) +/*[clinic end generated code: output=039925cd2df375bc input=e9715b0e06ff0fa6]*/ { CHECK_ATTACHED(self); return PyLong_FromSsize_t(self->chunk_size); } +/*[clinic input] +@critical_section +@setter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ + static int -textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value) +/*[clinic end generated code: output=edb86d2db660a5ab input=32fc99861db02a0a]*/ { Py_ssize_t n; CHECK_ATTACHED_INT(self); - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - n = PyNumber_AsSsize_t(arg, PyExc_ValueError); + n = PyNumber_AsSsize_t(value, PyExc_ValueError); if (n == -1 && PyErr_Occurred()) return -1; if (n <= 0) { @@ -3231,8 +3350,8 @@ static PyMethodDef textiowrapper_methods[] = { _IO_TEXTIOWRAPPER_TELL_METHODDEF _IO_TEXTIOWRAPPER_TRUNCATE_METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -3248,14 +3367,13 @@ static PyMemberDef textiowrapper_members[] = { }; static PyGetSetDef textiowrapper_getset[] = { - {"name", (getter)textiowrapper_name_get, NULL, NULL}, - {"closed", (getter)textiowrapper_closed_get, NULL, NULL}, + _IO_TEXTIOWRAPPER_NAME_GETSETDEF + _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF /* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL}, */ - {"newlines", (getter)textiowrapper_newlines_get, NULL, NULL}, - {"errors", (getter)textiowrapper_errors_get, NULL, NULL}, - {"_CHUNK_SIZE", (getter)textiowrapper_chunk_size_get, - (setter)textiowrapper_chunk_size_set, NULL}, + _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF + _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF + _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 50b8818aad410b..ec5c298066a587 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -134,6 +134,23 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { return m; } +static DWORD +_find_last_utf8_boundary(const char *buf, DWORD len) +{ + /* This function never returns 0, returns the original len instead */ + DWORD count = 1; + if (len == 0 || (buf[len - 1] & 0x80) == 0) { + return len; + } + for (;; count++) { + if (count > 3 || count >= len) { + return len; + } + if ((buf[len - count] & 0xc0) != 0x80) { + return len - count; + } + } +} /*[clinic input] module _io @@ -281,6 +298,13 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { @@ -374,9 +398,9 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, } if (self->writable) - self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY | _O_NOINHERIT); else - self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY | _O_NOINHERIT); if (self->fd < 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); CloseHandle(handle); @@ -975,7 +999,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, { BOOL res = TRUE; wchar_t *wbuf; - DWORD len, wlen, orig_len, n = 0; + DWORD len, wlen, n = 0; HANDLE handle; if (self->fd == -1) @@ -1007,21 +1031,8 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, have to reduce and recalculate. */ while (wlen > 32766 / sizeof(wchar_t)) { len /= 2; - orig_len = len; - /* Reduce the length until we hit the final byte of a UTF-8 sequence - * (top bit is unset). Fix for github issue 82052. - */ - while (len > 0 && (((char *)b->buf)[len-1] & 0x80) != 0) - --len; - /* If we hit a length of 0, something has gone wrong. This shouldn't - * be possible, as valid UTF-8 can have at most 3 non-final bytes - * before a final one, and our buffer is way longer than that. - * But to be on the safe side, if we hit this issue we just restore - * the original length and let the console API sort it out. - */ - if (len == 0) { - len = orig_len; - } + /* Fix for github issues gh-110913 and gh-82052. */ + len = _find_last_utf8_boundary(b->buf, len); wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); } Py_END_ALLOW_THREADS @@ -1066,15 +1077,22 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, static PyObject * winconsoleio_repr(winconsoleio *self) { - if (self->fd == -1) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO [closed]>"); - - if (self->readable) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='rb' closefd=%s>", - self->closefd ? "True" : "False"); - if (self->writable) - return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='wb' closefd=%s>", - self->closefd ? "True" : "False"); + const char *type_name = (Py_TYPE((PyObject *)self)->tp_name); + + if (self->fd == -1) { + return PyUnicode_FromFormat("<%.100s [closed]>", type_name); + } + + if (self->readable) { + return PyUnicode_FromFormat("<%.100s mode='rb' closefd=%s>", + type_name, + self->closefd ? "True" : "False"); + } + if (self->writable) { + return PyUnicode_FromFormat("<%.100s mode='wb' closefd=%s>", + type_name, + self->closefd ? "True" : "False"); + } PyErr_SetString(PyExc_SystemError, "_WindowsConsoleIO has invalid mode"); return NULL; diff --git a/Modules/_json.c b/Modules/_json.c index 41495e2012f152..c7fe1561bb1018 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -24,7 +24,6 @@ typedef struct _PyScannerObject { PyObject *parse_float; PyObject *parse_int; PyObject *parse_constant; - PyObject *memo; } PyScannerObject; static PyMemberDef scanner_members[] = { @@ -70,7 +69,7 @@ ascii_escape_unicode(PyObject *pystr); static PyObject * py_encode_basestring_ascii(PyObject* Py_UNUSED(self), PyObject *pystr); static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); +scan_once_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); static PyObject * @@ -86,11 +85,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); static PyObject * _encoded_const(PyObject *obj); static void @@ -631,7 +630,6 @@ scanner_traverse(PyScannerObject *self, visitproc visit, void *arg) Py_VISIT(self->parse_float); Py_VISIT(self->parse_int); Py_VISIT(self->parse_constant); - Py_VISIT(self->memo); return 0; } @@ -643,12 +641,11 @@ scanner_clear(PyScannerObject *self) Py_CLEAR(self->parse_float); Py_CLEAR(self->parse_int); Py_CLEAR(self->parse_constant); - Py_CLEAR(self->memo); return 0; } static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +_parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { /* Read a JSON object from PyUnicode pystr. idx is the index of the first character after the opening curly brace. @@ -665,6 +662,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss PyObject *key = NULL; int has_pairs_hook = (s->object_pairs_hook != Py_None); Py_ssize_t next_idx; + Py_ssize_t comma_idx; str = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); @@ -693,11 +691,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx); if (key == NULL) goto bail; - memokey = PyDict_SetDefault(s->memo, key, key); - if (memokey == NULL) { + if (PyDict_SetDefaultRef(memo, key, key, &memokey) < 0) { goto bail; } - Py_SETREF(key, Py_NewRef(memokey)); + Py_SETREF(key, memokey); idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ @@ -710,7 +707,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); + val = scan_once_unicode(s, memo, pystr, idx, &next_idx); if (val == NULL) goto bail; @@ -744,10 +741,16 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss raise_errmsg("Expecting ',' delimiter", pystr, idx); goto bail; } + comma_idx = idx; idx++; /* skip whitespace after , delimiter */ while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) == '}') { + raise_errmsg("Illegal trailing comma before end of object", pystr, comma_idx); + goto bail; + } } } @@ -774,7 +777,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss } static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { +_parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { /* Read a JSON array from PyUnicode pystr. idx is the index of the first character after the opening brace. *next_idx_ptr is a return-by-reference index to the first character after @@ -788,6 +791,7 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi PyObject *val = NULL; PyObject *rval; Py_ssize_t next_idx; + Py_ssize_t comma_idx; rval = PyList_New(0); if (rval == NULL) @@ -805,7 +809,7 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi while (1) { /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); + val = scan_once_unicode(s, memo, pystr, idx, &next_idx); if (val == NULL) goto bail; @@ -825,10 +829,16 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi raise_errmsg("Expecting ',' delimiter", pystr, idx); goto bail; } + comma_idx = idx; idx++; /* skip whitespace after , */ while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) == ']') { + raise_errmsg("Illegal trailing comma before end of array", pystr, comma_idx); + goto bail; + } } } @@ -986,7 +996,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +scan_once_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { /* Read one JSON term (of any kind) from PyUnicode pystr. idx is the index of the first character of the term @@ -1022,7 +1032,7 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ if (_Py_EnterRecursiveCall(" while decoding a JSON object " "from a unicode string")) return NULL; - res = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); + res = _parse_object_unicode(s, memo, pystr, idx + 1, next_idx_ptr); _Py_LeaveRecursiveCall(); return res; case '[': @@ -1030,7 +1040,7 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ if (_Py_EnterRecursiveCall(" while decoding a JSON array " "from a unicode string")) return NULL; - res = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); + res = _parse_array_unicode(s, memo, pystr, idx + 1, next_idx_ptr); _Py_LeaveRecursiveCall(); return res; case 'n': @@ -1106,16 +1116,19 @@ scanner_call(PyScannerObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:scan_once", kwlist, &pystr, &idx)) return NULL; - if (PyUnicode_Check(pystr)) { - rval = scan_once_unicode(self, pystr, idx, &next_idx); - } - else { + if (!PyUnicode_Check(pystr)) { PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); return NULL; } - PyDict_Clear(self->memo); + + PyObject *memo = PyDict_New(); + if (memo == NULL) { + return NULL; + } + rval = scan_once_unicode(self, memo, pystr, idx, &next_idx); + Py_DECREF(memo); if (rval == NULL) return NULL; return _build_rval_index_tuple(rval, next_idx); @@ -1137,10 +1150,6 @@ scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - /* All of these will fail "gracefully" so we don't need to verify them */ strict = PyObject_GetAttrString(ctx, "strict"); if (strict == NULL) @@ -1242,6 +1251,17 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } +static PyObject * +_create_newline_indent(PyObject *indent, Py_ssize_t indent_level) +{ + PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); + if (newline_indent != NULL && indent_level) { + PyUnicode_AppendAndDel(&newline_indent, + PySequence_Repeat(indent, indent_level)); + } + return newline_indent; +} + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { @@ -1258,10 +1278,20 @@ encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) _PyUnicodeWriter_Init(&writer); writer.overallocate = 1; - if (encoder_listencode_obj(self, &writer, obj, indent_level)) { + PyObject *newline_indent = NULL; + if (self->indent != Py_None) { + newline_indent = _create_newline_indent(self->indent, indent_level); + if (newline_indent == NULL) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + } + if (encoder_listencode_obj(self, &writer, obj, newline_indent)) { _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(newline_indent); return NULL; } + Py_XDECREF(newline_indent); result = PyTuple_New(1); if (result == NULL || @@ -1296,7 +1326,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) { /* Return the JSON representation of a PyFloat. */ double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { + if (!isfinite(i)) { if (!s->allow_nan) { PyErr_Format( PyExc_ValueError, @@ -1349,7 +1379,7 @@ _steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) static int encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *obj, Py_ssize_t indent_level) + PyObject *obj, PyObject *newline_indent) { /* Encode Python object obj to a JSON term */ PyObject *newobj; @@ -1385,14 +1415,14 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, indent_level); + rv = encoder_listencode_list(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, indent_level); + rv = encoder_listencode_dict(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } @@ -1426,7 +1456,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, indent_level); + rv = encoder_listencode_obj(s, writer, newobj, newline_indent); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1447,7 +1477,9 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, static int encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, Py_ssize_t indent_level) + PyObject *key, PyObject *value, + PyObject *newline_indent, + PyObject *item_separator) { PyObject *keystr = NULL; PyObject *encoded; @@ -1484,7 +1516,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator) < 0) { + if (_PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1502,7 +1534,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, indent_level) < 0) { + if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { return -1; } return 0; @@ -1510,13 +1542,15 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir static int encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *dct, Py_ssize_t indent_level) + PyObject *dct, PyObject *newline_indent) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; PyObject *items = NULL; PyObject *key, *value; bool first = true; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); @@ -1540,14 +1574,21 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '{')) goto bail; + PyObject *current_item_separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + separator_indent = PyUnicode_Concat(current_item_separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + // update item separator with a borrowed reference + current_item_separator = separator_indent; + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1565,7 +1606,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } Py_CLEAR(items); @@ -1573,7 +1616,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } } @@ -1583,12 +1628,15 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, '}')) goto bail; return 0; @@ -1596,16 +1644,20 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(items); Py_XDECREF(ident); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } static int encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *seq, Py_ssize_t indent_level) + PyObject *seq, PyObject *newline_indent) { PyObject *ident = NULL; PyObject *s_fast = NULL; Py_ssize_t i; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); @@ -1634,22 +1686,31 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '[')) goto bail; + + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } + + separator_indent = PyUnicode_Concat(separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + separator = separator_indent; // assign separator with borrowed reference } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator)) + if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, indent_level)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) goto bail; } if (ident != NULL) { @@ -1658,12 +1719,14 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(ident); } - /* TODO: DOES NOT RUN if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, ']')) goto bail; Py_DECREF(s_fast); @@ -1672,6 +1735,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(ident); Py_DECREF(s_fast); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } @@ -1712,7 +1777,7 @@ encoder_clear(PyEncoderObject *self) return 0; } -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); +PyDoc_STRVAR(encoder_doc, "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)"); static PyType_Slot PyEncoderType_slots[] = { {Py_tp_doc, (void *)encoder_doc}, @@ -1771,6 +1836,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index fe8e4c5e30035b..d4923442478b3e 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -860,6 +860,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index d23a756ace887d..5cf9eba243bd20 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -5,8 +5,8 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_FromLong() #include "rotatingtree.h" @@ -18,8 +18,8 @@ struct _ProfilerEntry; /* represents a function called from another function */ typedef struct _ProfilerSubEntry { rotating_node_t header; - _PyTime_t tt; - _PyTime_t it; + PyTime_t tt; + PyTime_t it; long callcount; long recursivecallcount; long recursionLevel; @@ -29,8 +29,8 @@ typedef struct _ProfilerSubEntry { typedef struct _ProfilerEntry { rotating_node_t header; PyObject *userObj; /* PyCodeObject, or a descriptive str for builtins */ - _PyTime_t tt; /* total time in this entry */ - _PyTime_t it; /* inline time in this entry (not in subcalls) */ + PyTime_t tt; /* total time in this entry */ + PyTime_t it; /* inline time in this entry (not in subcalls) */ long callcount; /* how many times this was called */ long recursivecallcount; /* how many times called recursively */ long recursionLevel; @@ -38,8 +38,8 @@ typedef struct _ProfilerEntry { } ProfilerEntry; typedef struct _ProfilerContext { - _PyTime_t t0; - _PyTime_t subt; + PyTime_t t0; + PyTime_t subt; struct _ProfilerContext *previous; ProfilerEntry *ctxEntry; } ProfilerContext; @@ -85,7 +85,7 @@ _lsprof_get_state(PyObject *module) /*** External Timers ***/ -static _PyTime_t CallExternalTimer(ProfilerObject *pObj) +static PyTime_t CallExternalTimer(ProfilerObject *pObj) { PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); if (o == NULL) { @@ -93,16 +93,16 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return 0; } - _PyTime_t result; + PyTime_t result; int err; if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromNanosecondsObject(&result, o); + err = _PyTime_FromLong(&result, o); } else { /* interpret the result as a double measured in seconds. - As the profiler works with _PyTime_t internally + As the profiler works with PyTime_t internally we convert it to a large integer */ err = _PyTime_FromSecondsObject(&result, o, _PyTime_ROUND_FLOOR); } @@ -114,14 +114,16 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return result; } -static inline _PyTime_t +static inline PyTime_t call_timer(ProfilerObject *pObj) { if (pObj->externalTimer != NULL) { return CallExternalTimer(pObj); } else { - return _PyTime_GetPerfCounter(); + PyTime_t t; + (void)PyTime_PerfCounterRaw(&t); + return t; } } @@ -175,8 +177,7 @@ normalizeUserObj(PyObject *obj) PyObject *modname = fn->m_module; if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); + PyObject *mo = _PyType_LookupRef(Py_TYPE(self), name); Py_DECREF(name); if (mo != NULL) { PyObject *res = PyObject_Repr(mo); @@ -312,8 +313,8 @@ initContext(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) static void Stop(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) { - _PyTime_t tt = call_timer(pObj) - self->t0; - _PyTime_t it = tt - self->subt; + PyTime_t tt = call_timer(pObj) - self->t0; + PyTime_t it = tt - self->subt; if (self->previous) self->previous->subt += tt; pObj->currentProfilerContext = self->previous; @@ -558,7 +559,7 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) return NULL; } if (!self->externalTimer || self->externalTimerUnit == 0.0) { - _PyTime_t onesec = _PyTime_FromSeconds(1); + PyTime_t onesec = _PyTime_FromSeconds(1); collect.factor = (double)1 / onesec; } else { @@ -847,7 +848,7 @@ profiler_dealloc(ProfilerObject *op) if (op->flags & POF_ENABLED) { PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { - _PyErr_WriteUnraisableMsg("When destroying _lsprof profiler", NULL); + PyErr_FormatUnraisable("Exception ignored when destroying _lsprof profiler"); } } @@ -1005,9 +1006,8 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index eb90c308d16d19..97f3a8f03da9a8 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -492,7 +492,9 @@ build_filter_spec(const lzma_filter *f) case LZMA_FILTER_ARMTHUMB: case LZMA_FILTER_SPARC: { lzma_options_bcj *options = f->options; - ADD_FIELD(options, start_offset); + if (options) { + ADD_FIELD(options, start_offset); + } break; } default: @@ -1602,6 +1604,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/clinic/multiprocessing.c.h b/Modules/_multiprocessing/clinic/multiprocessing.c.h index 0ae0039a666061..6d4f5c2afcfd39 100644 --- a/Modules/_multiprocessing/clinic/multiprocessing.c.h +++ b/Modules/_multiprocessing/clinic/multiprocessing.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + #if defined(MS_WINDOWS) PyDoc_STRVAR(_multiprocessing_closesocket__doc__, @@ -102,10 +104,6 @@ _multiprocessing_send(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buf, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buf, 'C')) { - _PyArg_BadArgument("send", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _multiprocessing_send_impl(module, handle, &buf); exit: @@ -166,4 +164,4 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) #ifndef _MULTIPROCESSING_SEND_METHODDEF #define _MULTIPROCESSING_SEND_METHODDEF #endif /* !defined(_MULTIPROCESSING_SEND_METHODDEF) */ -/*[clinic end generated code: output=8b91c020d4353cc5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b4cb8428d816da input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 09dfa11da23b86..a545ff4d80f067 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -2,11 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, @@ -16,69 +11,25 @@ PyDoc_STRVAR(_posixshmem_shm_open__doc__, "Open a shared memory object. Returns a file descriptor (integer)."); #define _POSIXSHMEM_SHM_OPEN_METHODDEF \ - {"shm_open", _PyCFunction_CAST(_posixshmem_shm_open), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_open__doc__}, + {"shm_open", (PyCFunction)(void(*)(void))_posixshmem_shm_open, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_open__doc__}, static int _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, int mode); static PyObject * -_posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), &_Py_ID(flags), &_Py_ID(mode), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", "flags", "mode", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_open", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + static char *_keywords[] = {"path", "flags", "mode", NULL}; PyObject *path; int flags; int mode = 511; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_open", "argument 'path'", "str", args[0]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ui|i:shm_open", _keywords, + &path, &flags, &mode)) goto exit; - } - path = args[0]; - flags = PyLong_AsInt(args[1]); - if (flags == -1 && PyErr_Occurred()) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - mode = PyLong_AsInt(args[2]); - if (mode == -1 && PyErr_Occurred()) { - goto exit; - } -skip_optional_pos: _return_value = _posixshmem_shm_open_impl(module, path, flags, mode); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -94,7 +45,7 @@ _posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, #if defined(HAVE_SHM_UNLINK) PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, -"shm_unlink($module, /, path)\n" +"shm_unlink($module, path, /)\n" "--\n" "\n" "Remove a shared memory object (similar to unlink()).\n" @@ -104,52 +55,22 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", _PyCFunction_CAST(_posixshmem_shm_unlink), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_unlink(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_unlink", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_unlink", "argument 'path'", "str", args[0]); + if (!PyUnicode_Check(arg)) { + PyErr_Format(PyExc_TypeError, "shm_unlink() argument must be str, not %T", arg); goto exit; } - path = args[0]; + path = arg; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -165,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=2f356903a281d857 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index a4357376ee1292..512e5a016192fb 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -6,6 +6,8 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) @@ -74,7 +76,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -99,7 +103,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) */ @@ -171,7 +181,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -196,7 +208,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && !defined(MS_WINDOWS) */ @@ -339,7 +357,13 @@ _multiprocessing_SemLock__count_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock__count(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__count_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__count_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -449,7 +473,13 @@ _multiprocessing_SemLock___enter___impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock___enter__(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock___enter___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___enter___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -494,7 +524,9 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py } exc_tb = args[2]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock___exit___impl(self, exc_type, exc_value, exc_tb); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -541,4 +573,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=e8ea65f8cba8e173 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dea36482d23a355f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2e6d8eb68c0243..cee8cf7b9a83c0 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -181,7 +181,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} @@ -277,6 +277,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index debef3267f77d1..ab45e4136c7d46 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,13 +2,21 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include -// for shm_open() and shm_unlink() +#include // strlen() +#include // EINTR #ifdef HAVE_SYS_MMAN_H -#include +# include // shm_open(), shm_unlink() #endif + /*[clinic input] module _posixshmem [clinic start generated code]*/ @@ -40,10 +48,15 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, { int fd; int async_err = 0; - const char *name = PyUnicode_AsUTF8(path); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return -1; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } do { Py_BEGIN_ALLOW_THREADS fd = shm_open(name, flags, mode); @@ -64,6 +77,7 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, /*[clinic input] _posixshmem.shm_unlink path: unicode + / Remove a shared memory object (similar to unlink()). @@ -75,14 +89,19 @@ region. static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ { int rv; int async_err = 0; - const char *name = PyUnicode_AsUTF8(path); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return NULL; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } do { Py_BEGIN_ALLOW_THREADS rv = shm_unlink(name); @@ -110,6 +129,7 @@ static PyMethodDef module_methods[ ] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index f8f2afda28d06d..4de4ee6c78fbd1 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -81,6 +81,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) } /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -92,7 +93,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -172,6 +173,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -179,7 +181,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -297,6 +299,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) #endif /* !HAVE_SEM_TIMEDWAIT */ /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -308,7 +311,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -382,6 +385,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -389,7 +393,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -583,6 +587,7 @@ semlock_dealloc(SemLockObject* self) } /*[clinic input] +@critical_section _multiprocessing.SemLock._count Num of `acquire()`s minus num of `release()`s for this process. @@ -590,7 +595,7 @@ Num of `acquire()`s minus num of `release()`s for this process. static PyObject * _multiprocessing_SemLock__count_impl(SemLockObject *self) -/*[clinic end generated code: output=5ba8213900e517bb input=36fc59b1cd1025ab]*/ +/*[clinic end generated code: output=5ba8213900e517bb input=9fa6e0b321b16935]*/ { return PyLong_FromLong((long)self->count); } @@ -677,6 +682,7 @@ _multiprocessing_SemLock__after_fork_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock.__enter__ Enter the semaphore/lock. @@ -684,12 +690,13 @@ Enter the semaphore/lock. static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self) -/*[clinic end generated code: output=beeb2f07c858511f input=c5e27d594284690b]*/ +/*[clinic end generated code: output=beeb2f07c858511f input=d35c9860992ee790]*/ { return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); } /*[clinic input] +@critical_section _multiprocessing.SemLock.__exit__ exc_type: object = None @@ -704,7 +711,7 @@ static PyObject * _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=3b37c1a9f8b91a03 input=7d644b64a89903f8]*/ +/*[clinic end generated code: output=3b37c1a9f8b91a03 input=1610c8cc3e0e337e]*/ { return _multiprocessing_SemLock_release_impl(self); } diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dac9c019b249e9..dc93063aee7e54 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -5,8 +5,13 @@ #include "Python.h" #include "compile.h" #include "opcode.h" -#include "internal/pycore_code.h" -#include "internal/pycore_intrinsics.h" +#include "pycore_ceval.h" +#include "pycore_code.h" +#include "pycore_compile.h" +#include "pycore_intrinsics.h" +#include "pycore_optimizer.h" // _Py_GetExecutor() +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, OPCODE_HAS_*, etc +#include "pycore_opcode_utils.h" /*[clinic input] module _opcode @@ -78,7 +83,7 @@ static int _opcode_is_valid_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ { - return PyUnstable_OpcodeIsValid(opcode); + return IS_VALID_OPCODE(opcode); } /*[clinic input] @@ -94,8 +99,7 @@ static int _opcode_has_arg_impl(PyObject *module, int opcode) /*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasArg(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_ARG(opcode); } /*[clinic input] @@ -111,8 +115,7 @@ static int _opcode_has_const_impl(PyObject *module, int opcode) /*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasConst(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_CONST(opcode); } /*[clinic input] @@ -128,8 +131,7 @@ static int _opcode_has_name_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasName(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } /*[clinic input] @@ -145,9 +147,7 @@ static int _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasJump(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_JUMP(opcode); } /*[clinic input] @@ -168,9 +168,7 @@ static int _opcode_has_free_impl(PyObject *module, int opcode) /*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasFree(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_FREE(opcode); } /*[clinic input] @@ -186,8 +184,7 @@ static int _opcode_has_local_impl(PyObject *module, int opcode) /*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasLocal(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_LOCAL(opcode); } /*[clinic input] @@ -203,8 +200,7 @@ static int _opcode_has_exc_impl(PyObject *module, int opcode) /*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { - return PyUnstable_OpcodeIsValid(opcode) && - PyUnstable_OpcodeHasExc(opcode); + return IS_VALID_OPCODE(opcode) && IS_BLOCK_PUSH_OPCODE(opcode); } /*[clinic input] @@ -244,8 +240,7 @@ _opcode_get_nb_ops_impl(PyObject *module) } #define ADD_NB_OP(NUM, STR) \ do { \ - PyObject *pair = Py_BuildValue( \ - "NN", PyUnicode_FromString(#NUM), PyUnicode_FromString(STR)); \ + PyObject *pair = Py_BuildValue("ss", #NUM, STR); \ if (pair == NULL) { \ Py_DECREF(list); \ return NULL; \ @@ -310,7 +305,7 @@ _opcode_get_intrinsic1_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_1; i++) { - PyObject *name = PyUnstable_GetUnaryIntrinsicName(i); + PyObject *name = _PyCompile_GetUnaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; @@ -337,7 +332,7 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return NULL; } for (int i=0; i <= MAX_INTRINSIC_2; i++) { - PyObject *name = PyUnstable_GetBinaryIntrinsicName(i); + PyObject *name = _PyCompile_GetBinaryIntrinsicName(i); if (name == NULL) { Py_DECREF(list); return NULL; @@ -347,6 +342,60 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return list; } +/*[clinic input] + +_opcode.get_special_method_names + +Return a list of special method names. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module) +/*[clinic end generated code: output=fce72614cd988d17 input=25f2115560bdf163]*/ +{ + PyObject *list = PyList_New(SPECIAL_MAX + 1); + if (list == NULL) { + return NULL; + } + for (int i=0; i <= SPECIAL_MAX; i++) { + PyObject *name = _Py_SpecialMethods[i].name; + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + PyList_SET_ITEM(list, i, name); + } + return list; +} + +/*[clinic input] + +_opcode.get_executor + + code: object + offset: int + +Return the executor object at offset in code if exists, None otherwise. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) +/*[clinic end generated code: output=c035c7a47b16648f input=85eff93ea7aac282]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format(PyExc_TypeError, + "expected a code object, not '%.100s'", + Py_TYPE(code)->tp_name); + return NULL; + } +#ifdef _Py_TIER2 + return (PyObject *)_Py_GetExecutor((PyCodeObject *)code, offset); +#else + PyErr_Format(PyExc_RuntimeError, + "Executors are not available in this build"); + return NULL; +#endif +} static PyMethodDef opcode_functions[] = { @@ -363,10 +412,12 @@ opcode_functions[] = { _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF + _OPCODE_GET_EXECUTOR_METHODDEF + _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF {NULL, NULL, 0, NULL} }; -int +static int _opcode_exec(PyObject *m) { if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { return -1; @@ -377,6 +428,7 @@ _opcode_exec(PyObject *m) { static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index 1f6496d381adac..793a2d0dfc6443 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -2,6 +2,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "clinic/_operator.c.h" @@ -966,6 +967,18 @@ static struct PyMethodDef operator_methods[] = { }; + +static PyObject * +text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj, /)"); +} + +static PyGetSetDef common_getset[] = { + {"__text_signature__", text_signature, (setter)NULL}, + {NULL} +}; + /* itemgetter object **********************************************************/ typedef struct { @@ -1171,6 +1184,7 @@ static PyType_Slot itemgetter_type_slots[] = { {Py_tp_clear, itemgetter_clear}, {Py_tp_methods, itemgetter_methods}, {Py_tp_members, itemgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, itemgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, itemgetter_repr}, @@ -1223,6 +1237,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; /* prepare attr while checking args */ + PyInterpreterState *interp = _PyInterpreterState_GET(); for (idx = 0; idx < nattrs; ++idx) { PyObject *item = PyTuple_GET_ITEM(args, idx); int dot_count; @@ -1246,7 +1261,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (dot_count == 0) { Py_INCREF(item); - PyUnicode_InternInPlace(&item); + _PyUnicode_InternMortal(interp, &item); PyTuple_SET_ITEM(attr, idx, item); } else { /* make it a tuple of non-dotted attrnames */ PyObject *attr_chain = PyTuple_New(dot_count + 1); @@ -1272,7 +1287,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); ++attr_chain_idx; unibuff_till = unibuff_from = unibuff_till + 1; @@ -1286,7 +1301,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); PyTuple_SET_ITEM(attr, idx, attr_chain); @@ -1528,6 +1543,7 @@ static PyType_Slot attrgetter_type_slots[] = { {Py_tp_clear, attrgetter_clear}, {Py_tp_methods, attrgetter_methods}, {Py_tp_members, attrgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, attrgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, attrgetter_repr}, @@ -1648,7 +1664,8 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); mc->name = name; mc->xargs = Py_XNewRef(args); // allows us to use borrowed references @@ -1863,6 +1880,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, {Py_tp_members, methodcaller_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr}, @@ -1913,6 +1931,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a3cf34699ba509..4a2191db0cf984 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -9,15 +9,16 @@ #endif #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytesWriter -#include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_long.h" // _PyLong_AsByteArray() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _PyNone_Type -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_runtime.h" // _Py_ID() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_AsByteArray() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyNone_Type +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // _Py_ID() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include // strtol() @@ -399,64 +400,6 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ -/* Retrieve and deconstruct a method for avoiding a reference cycle - (pickler -> bound method of pickler -> pickler) */ -static int -init_method_ref(PyObject *self, PyObject *name, - PyObject **method_func, PyObject **method_self) -{ - PyObject *func, *func2; - int ret; - - /* *method_func and *method_self should be consistent. All refcount decrements - should be occurred after setting *method_self and *method_func. */ - ret = PyObject_GetOptionalAttr(self, name, &func); - if (func == NULL) { - *method_self = NULL; - Py_CLEAR(*method_func); - return ret; - } - - if (PyMethod_Check(func) && PyMethod_GET_SELF(func) == self) { - /* Deconstruct a bound Python method */ - *method_self = self; /* borrowed */ - func2 = PyMethod_GET_FUNCTION(func); - Py_XSETREF(*method_func, Py_NewRef(func2)); - Py_DECREF(func); - return 0; - } - else { - *method_self = NULL; - Py_XSETREF(*method_func, func); - return 0; - } -} - -/* Bind a method if it was deconstructed */ -static PyObject * -reconstruct_method(PyObject *func, PyObject *self) -{ - if (self) { - return PyMethod_New(func, self); - } - else { - return Py_NewRef(func); - } -} - -static PyObject * -call_method(PyObject *func, PyObject *self, PyObject *obj) -{ - if (self) { - return PyObject_CallFunctionObjArgs(func, self, obj, NULL); - } - else { - return PyObject_CallOneArg(func, obj); - } -} - -/*************************************************************************/ - /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD @@ -668,9 +611,7 @@ typedef struct PicklerObject { PyMemoTable *memo; /* Memo table, keep track of the seen objects to support self-referential objects pickling. */ - PyObject *pers_func; /* persistent_id() method, can be NULL */ - PyObject *pers_func_self; /* borrowed reference to self if pers_func - is an unbound method, NULL otherwise */ + PyObject *persistent_id; /* persistent_id() method, can be NULL */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling @@ -712,9 +653,7 @@ typedef struct UnpicklerObject { size_t memo_size; /* Capacity of the memo array */ size_t memo_len; /* Number of objects in the memo */ - PyObject *pers_func; /* persistent_load() method, can be NULL. */ - PyObject *pers_func_self; /* borrowed reference to self if pers_func - is an unbound method, NULL otherwise */ + PyObject *persistent_load; /* persistent_load() method, can be NULL. */ Py_buffer buffer; char *input_buffer; @@ -1167,8 +1106,7 @@ _Pickler_New(PickleState *st) } self->memo = memo; - self->pers_func = NULL; - self->pers_func_self = NULL; + self->persistent_id = NULL; self->dispatch_table = NULL; self->reducer_override = NULL; self->write = NULL; @@ -1662,8 +1600,7 @@ _Unpickler_New(PyObject *module) self->memo = memo; self->memo_size = MEMO_SIZE; self->memo_len = 0; - self->pers_func = NULL; - self->pers_func_self = NULL; + self->persistent_load = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); self->input_buffer = NULL; self->input_line = NULL; @@ -1870,8 +1807,7 @@ get_dotted_path(PyObject *obj, PyObject *name) { PyObject *dotted_path; Py_ssize_t i, n; - _Py_DECLARE_STR(dot, "."); - dotted_path = PyUnicode_Split(name, &_Py_STR(dot), -1); + dotted_path = PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1); if (dotted_path == NULL) return NULL; n = PyList_GET_SIZE(dotted_path); @@ -2226,7 +2162,8 @@ save_long(PicklerObject *self, PyObject *obj) pdata = (unsigned char *)PyBytes_AS_STRING(repr); i = _PyLong_AsByteArray((PyLongObject *)obj, pdata, nbytes, - 1 /* little endian */ , 1 /* signed */ ); + 1 /* little endian */ , 1 /* signed */ , + 1 /* with exceptions */); if (i < 0) goto error; /* If the int is negative, this may be a byte more than @@ -3476,15 +3413,21 @@ save_set(PickleState *state, PicklerObject *self, PyObject *obj) i = 0; if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { - Py_INCREF(item); - int err = save(state, self, item, 0); + + int err = 0; + Py_BEGIN_CRITICAL_SECTION(obj); + while (_PySet_NextEntryRef(obj, &ppos, &item, &hash)) { + err = save(state, self, item, 0); Py_CLEAR(item); if (err < 0) - return -1; + break; if (++i == BATCHSIZE) break; } + Py_END_CRITICAL_SECTION(); + if (err < 0) { + return -1; + } if (_Pickler_Write(self, &additems_op, 1) < 0) return -1; if (PySet_GET_SIZE(obj) != set_size) { @@ -3929,7 +3872,7 @@ save_pers(PickleState *state, PicklerObject *self, PyObject *obj) const char persid_op = PERSID; const char binpersid_op = BINPERSID; - pid = call_method(self->pers_func, self->pers_func_self, obj); + pid = PyObject_CallOneArg(self->persistent_id, obj); if (pid == NULL) return -1; @@ -4317,7 +4260,7 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) /* The extra pers_save argument is necessary to avoid calling save_pers() on its returned object. */ - if (!pers_save && self->pers_func) { + if (!pers_save && self->persistent_id) { /* save_pers() returns: -1 to signal an error; 0 if it did nothing successfully; @@ -4522,6 +4465,12 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) return status; } +static PyObject * +persistent_id(PyObject *self, PyObject *obj) +{ + Py_RETURN_NONE; +} + static int dump(PickleState *state, PicklerObject *self, PyObject *obj) { @@ -4529,17 +4478,25 @@ dump(PickleState *state, PicklerObject *self, PyObject *obj) int status = -1; PyObject *tmp; - if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(reducer_override), - &tmp) < 0) { - goto error; + /* Cache the persistent_id method. */ + tmp = PyObject_GetAttr((PyObject *)self, &_Py_ID(persistent_id)); + if (tmp == NULL) { + goto error; } - /* Cache the reducer_override method, if it exists. */ - if (tmp != NULL) { - Py_XSETREF(self->reducer_override, tmp); + if (PyCFunction_Check(tmp) && + PyCFunction_GET_SELF(tmp) == (PyObject *)self && + PyCFunction_GET_FUNCTION(tmp) == persistent_id) + { + Py_CLEAR(tmp); } - else { - Py_CLEAR(self->reducer_override); + Py_XSETREF(self->persistent_id, tmp); + + /* Cache the reducer_override method, if it exists. */ + if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(reducer_override), + &tmp) < 0) { + goto error; } + Py_XSETREF(self->reducer_override, tmp); if (self->proto >= 2) { char header[2]; @@ -4565,11 +4522,12 @@ dump(PickleState *state, PicklerObject *self, PyObject *obj) self->framing = 0; /* Break the reference cycle we generated at the beginning this function - * call when setting the reducer_override attribute of the Pickler instance - * to a bound method of the same instance. This is important as the Pickler - * instance holds a reference to each object it has pickled (through its - * memo): thus, these objects won't be garbage-collected as long as the - * Pickler itself is not collected. */ + * call when setting the persistent_id and the reducer_override attributes + * of the Pickler instance to a bound method of the same instance. + * This is important as the Pickler instance holds a reference to each + * object it has pickled (through its memo): thus, these objects won't + * be garbage-collected as long as the Pickler itself is not collected. */ + Py_CLEAR(self->persistent_id); Py_CLEAR(self->reducer_override); return status; } @@ -4662,6 +4620,8 @@ _pickle_Pickler___sizeof___impl(PicklerObject *self) } static struct PyMethodDef Pickler_methods[] = { + {"persistent_id", persistent_id, METH_O, + PyDoc_STR("persistent_id($self, obj, /)\n--\n\n")}, _PICKLE_PICKLER_DUMP_METHODDEF _PICKLE_PICKLER_CLEAR_MEMO_METHODDEF _PICKLE_PICKLER___SIZEOF___METHODDEF @@ -4673,7 +4633,7 @@ Pickler_clear(PicklerObject *self) { Py_CLEAR(self->output_buffer); Py_CLEAR(self->write); - Py_CLEAR(self->pers_func); + Py_CLEAR(self->persistent_id); Py_CLEAR(self->dispatch_table); Py_CLEAR(self->fast_memo); Py_CLEAR(self->reducer_override); @@ -4702,11 +4662,19 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->write); - Py_VISIT(self->pers_func); + Py_VISIT(self->persistent_id); Py_VISIT(self->dispatch_table); Py_VISIT(self->fast_memo); Py_VISIT(self->reducer_override); Py_VISIT(self->buffer_callback); + PyMemoTable *memo = self->memo; + if (memo && memo->mt_table) { + Py_ssize_t i = memo->mt_allocated; + while (--i >= 0) { + Py_VISIT(memo->mt_table[i].me_key); + } + } + return 0; } @@ -4791,11 +4759,6 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, self->fast_nesting = 0; self->fast_memo = NULL; - if (init_method_ref((PyObject *)self, &_Py_ID(persistent_id), - &self->pers_func, &self->pers_func_self) < 0) - { - return -1; - } if (self->dispatch_table != NULL) { return 0; } @@ -5044,36 +5007,6 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } -static PyObject * -Pickler_get_persid(PicklerObject *self, void *Py_UNUSED(ignored)) -{ - if (self->pers_func == NULL) { - PyErr_SetString(PyExc_AttributeError, "persistent_id"); - return NULL; - } - return reconstruct_method(self->pers_func, self->pers_func_self); -} - -static int -Pickler_set_persid(PicklerObject *self, PyObject *value, void *Py_UNUSED(ignored)) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "attribute deletion is not supported"); - return -1; - } - if (!PyCallable_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "persistent_id must be a callable taking one argument"); - return -1; - } - - self->pers_func_self = NULL; - Py_XSETREF(self->pers_func, Py_NewRef(value)); - - return 0; -} - static PyMemberDef Pickler_members[] = { {"bin", Py_T_INT, offsetof(PicklerObject, bin)}, {"fast", Py_T_INT, offsetof(PicklerObject, fast)}, @@ -5084,8 +5017,6 @@ static PyMemberDef Pickler_members[] = { static PyGetSetDef Pickler_getsets[] = { {"memo", (getter)Pickler_get_memo, (setter)Pickler_set_memo}, - {"persistent_id", (getter)Pickler_get_persid, - (setter)Pickler_set_persid}, {NULL} }; @@ -6048,36 +5979,28 @@ load_persid(PickleState *st, UnpicklerObject *self) Py_ssize_t len; char *s; - if (self->pers_func) { - if ((len = _Unpickler_Readline(st, self, &s)) < 0) - return -1; - if (len < 1) - return bad_readline(st); + if ((len = _Unpickler_Readline(st, self, &s)) < 0) + return -1; + if (len < 1) + return bad_readline(st); - pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); - if (pid == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_SetString(st->UnpicklingError, - "persistent IDs in protocol 0 must be " - "ASCII strings"); - } - return -1; + pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); + if (pid == NULL) { + if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + PyErr_SetString(st->UnpicklingError, + "persistent IDs in protocol 0 must be " + "ASCII strings"); } - - obj = call_method(self->pers_func, self->pers_func_self, pid); - Py_DECREF(pid); - if (obj == NULL) - return -1; - - PDATA_PUSH(self->stack, obj, -1); - return 0; - } - else { - PyErr_SetString(st->UnpicklingError, - "A load persistent id instruction was encountered, " - "but no persistent_load function was specified."); return -1; } + + obj = PyObject_CallOneArg(self->persistent_load, pid); + Py_DECREF(pid); + if (obj == NULL) + return -1; + + PDATA_PUSH(self->stack, obj, -1); + return 0; } static int @@ -6085,25 +6008,17 @@ load_binpersid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; - if (self->pers_func) { - PDATA_POP(st, self->stack, pid); - if (pid == NULL) - return -1; - - obj = call_method(self->pers_func, self->pers_func_self, pid); - Py_DECREF(pid); - if (obj == NULL) - return -1; + PDATA_POP(st, self->stack, pid); + if (pid == NULL) + return -1; - PDATA_PUSH(self->stack, obj, -1); - return 0; - } - else { - PyErr_SetString(st->UnpicklingError, - "A load persistent id instruction was encountered, " - "but no persistent_load function was specified."); + obj = PyObject_CallOneArg(self->persistent_load, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - } + + PDATA_PUSH(self->stack, obj, -1); + return 0; } static int @@ -6609,11 +6524,13 @@ load_additems(PickleState *state, UnpicklerObject *self) if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); return -1; } Py_DECREF(result); } Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); } return 0; @@ -6689,8 +6606,10 @@ load_build(PickleState *st, UnpicklerObject *self) /* normally the keys for instance attributes are interned. we should try to do that here. */ Py_INCREF(d_key); - if (PyUnicode_CheckExact(d_key)) - PyUnicode_InternInPlace(&d_key); + if (PyUnicode_CheckExact(d_key)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &d_key); + } if (PyObject_SetItem(dict, d_key, d_value) < 0) { Py_DECREF(d_key); goto error; @@ -6829,6 +6748,7 @@ static PyObject * load(PickleState *st, UnpicklerObject *self) { PyObject *value = NULL; + PyObject *tmp; char *s = NULL; self->num_marks = 0; @@ -6838,6 +6758,13 @@ load(PickleState *st, UnpicklerObject *self) if (Py_SIZE(self->stack)) Pdata_clear(self->stack, 0); + /* Cache the persistent_load method. */ + tmp = PyObject_GetAttr((PyObject *)self, &_Py_ID(persistent_load)); + if (tmp == NULL) { + goto error; + } + Py_XSETREF(self->persistent_load, tmp); + /* Convenient macros for the dispatch while-switch loop just below. */ #define OP(opcode, load_func) \ case opcode: if (load_func(st, self) < 0) break; continue; @@ -6850,7 +6777,7 @@ load(PickleState *st, UnpicklerObject *self) if (PyErr_ExceptionMatches(st->UnpicklingError)) { PyErr_Format(PyExc_EOFError, "Ran out of input"); } - return NULL; + goto error; } switch ((enum opcode)s[0]) { @@ -6936,7 +6863,7 @@ load(PickleState *st, UnpicklerObject *self) PyErr_Format(st->UnpicklingError, "invalid load key, '\\x%02x'.", c); } - return NULL; + goto error; } } @@ -6944,14 +6871,41 @@ load(PickleState *st, UnpicklerObject *self) } if (PyErr_Occurred()) { - return NULL; + goto error; } if (_Unpickler_SkipConsumed(self) < 0) - return NULL; + goto error; + Py_CLEAR(self->persistent_load); PDATA_POP(st, self->stack, value); return value; + +error: + Py_CLEAR(self->persistent_load); + return NULL; +} + +/*[clinic input] + +_pickle.Unpickler.persistent_load + + cls: defining_class + pid: object + / + +[clinic start generated code]*/ + +static PyObject * +_pickle_Unpickler_persistent_load_impl(UnpicklerObject *self, + PyTypeObject *cls, PyObject *pid) +/*[clinic end generated code: output=9f4706f1330cb14d input=2f9554fae051276e]*/ +{ + PickleState *st = _Pickle_GetStateByClass(cls); + PyErr_SetString(st->UnpicklingError, + "A load persistent id instruction was encountered, " + "but no persistent_load function was specified."); + return NULL; } /*[clinic input] @@ -7120,6 +7074,7 @@ _pickle_Unpickler___sizeof___impl(UnpicklerObject *self) } static struct PyMethodDef Unpickler_methods[] = { + _PICKLE_UNPICKLER_PERSISTENT_LOAD_METHODDEF _PICKLE_UNPICKLER_LOAD_METHODDEF _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF _PICKLE_UNPICKLER___SIZEOF___METHODDEF @@ -7134,7 +7089,7 @@ Unpickler_clear(UnpicklerObject *self) Py_CLEAR(self->read); Py_CLEAR(self->peek); Py_CLEAR(self->stack); - Py_CLEAR(self->pers_func); + Py_CLEAR(self->persistent_load); Py_CLEAR(self->buffers); if (self->buffer.buf != NULL) { PyBuffer_Release(&self->buffer); @@ -7173,8 +7128,15 @@ Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->read); Py_VISIT(self->peek); Py_VISIT(self->stack); - Py_VISIT(self->pers_func); + Py_VISIT(self->persistent_load); Py_VISIT(self->buffers); + PyObject **memo = self->memo; + if (memo) { + Py_ssize_t i = self->memo_size; + while (--i >= 0) { + Py_VISIT(memo[i]); + } + } return 0; } @@ -7232,12 +7194,6 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, self->fix_imports = fix_imports; - if (init_method_ref((PyObject *)self, &_Py_ID(persistent_load), - &self->pers_func, &self->pers_func_self) < 0) - { - return -1; - } - PyTypeObject *tp = Py_TYPE(self); PickleState *state = _Pickle_FindStateByType(tp); self->stack = (Pdata *)Pdata_New(state); @@ -7506,41 +7462,8 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } -static PyObject * -Unpickler_get_persload(UnpicklerObject *self, void *Py_UNUSED(ignored)) -{ - if (self->pers_func == NULL) { - PyErr_SetString(PyExc_AttributeError, "persistent_load"); - return NULL; - } - return reconstruct_method(self->pers_func, self->pers_func_self); -} - -static int -Unpickler_set_persload(UnpicklerObject *self, PyObject *value, void *Py_UNUSED(ignored)) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "attribute deletion is not supported"); - return -1; - } - if (!PyCallable_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "persistent_load must be a callable taking " - "one argument"); - return -1; - } - - self->pers_func_self = NULL; - Py_XSETREF(self->pers_func, Py_NewRef(value)); - - return 0; -} - static PyGetSetDef Unpickler_getsets[] = { {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo}, - {"persistent_load", (getter)Unpickler_get_persload, - (setter)Unpickler_set_persload}, {NULL} }; @@ -7943,6 +7866,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 2898eedc3e3a8f..daec4ad708dea4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -673,9 +673,10 @@ child_exec(char *const exec_array[], PyObject *preexec_fn, PyObject *preexec_fn_args_tuple) { - int i, saved_errno, reached_preexec = 0; + int i, saved_errno; PyObject *result; - const char* err_msg = ""; + /* Indicate to the parent that the error happened before exec(). */ + const char *err_msg = "noexec"; /* Buffer large enough to hold a hex integer. We can't malloc. */ char hex_errno[sizeof(saved_errno)*2+1]; @@ -735,8 +736,12 @@ child_exec(char *const exec_array[], /* We no longer manually close p2cread, c2pwrite, and errwrite here as * _close_open_fds takes care when it is not already non-inheritable. */ - if (cwd) - POSIX_CALL(chdir(cwd)); + if (cwd) { + if (chdir(cwd) == -1) { + err_msg = "noexec:chdir"; + goto error; + } + } if (child_umask >= 0) umask(child_umask); /* umask() always succeeds. */ @@ -767,8 +772,10 @@ child_exec(char *const exec_array[], #endif #ifdef HAVE_SETGROUPS - if (extra_group_size > 0) + if (extra_group_size >= 0) { + assert((extra_group_size == 0) == (extra_groups == NULL)); POSIX_CALL(setgroups(extra_group_size, extra_groups)); + } #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID @@ -782,7 +789,7 @@ child_exec(char *const exec_array[], #endif /* HAVE_SETREUID */ - reached_preexec = 1; + err_msg = ""; if (preexec_fn != Py_None && preexec_fn_args_tuple) { /* This is where the user has asked us to deadlock their program. */ result = PyObject_Call(preexec_fn, preexec_fn_args_tuple, NULL); @@ -840,16 +847,12 @@ child_exec(char *const exec_array[], } _Py_write_noraise(errpipe_write, cur, hex_errno + sizeof(hex_errno) - cur); _Py_write_noraise(errpipe_write, ":", 1); - if (!reached_preexec) { - /* Indicate to the parent that the error happened before exec(). */ - _Py_write_noraise(errpipe_write, "noexec", 6); - } /* We can't call strerror(saved_errno). It is not async signal safe. * The parent process will look the error message up. */ } else { _Py_write_noraise(errpipe_write, "SubprocessError:0:", 18); - _Py_write_noraise(errpipe_write, err_msg, strlen(err_msg)); } + _Py_write_noraise(errpipe_write, err_msg, strlen(err_msg)); } @@ -1022,15 +1025,16 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pid = -1; int need_to_reenable_gc = 0; char *const *argv = NULL, *const *envp = NULL; - Py_ssize_t extra_group_size = 0; int need_after_fork = 0; int saved_errno = 0; int *c_fds_to_keep = NULL; Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = _PyInterpreterState_GET(); - if ((preexec_fn != Py_None) && interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { + PyErr_SetString(PyExc_PythonFinalizationError, "preexec_fn not supported at interpreter shutdown"); return NULL; } @@ -1103,6 +1107,13 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, cwd = PyBytes_AsString(cwd_obj2); } + // Special initial value meaning that subprocess API was called with + // extra_groups=None leading to _posixsubprocess.fork_exec(gids=None). + // We use this to differentiate between code desiring a setgroups(0, NULL) + // call vs no call at all. The fast vfork() code path could be used when + // there is no setgroups call. + Py_ssize_t extra_group_size = -2; + if (extra_groups_packed != Py_None) { #ifdef HAVE_SETGROUPS if (!PyList_Check(extra_groups_packed)) { @@ -1306,6 +1317,7 @@ static PyMethodDef module_methods[] = { static PyModuleDef_Slot _posixsubprocess_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index b4bafb375c999d..aee8db802d8c3f 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -3,10 +3,12 @@ #endif #include "Python.h" -#include "pycore_ceval.h" // _PyEval_MakePendingCalls() +#include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_time.h" // _PyTime_t +#include "pycore_parking_lot.h" +#include "pycore_time.h" // _PyTime_FromSecondsObject() +#include #include // offsetof() typedef struct { @@ -25,12 +27,173 @@ static struct PyModuleDef queuemodule; #define simplequeue_get_state_by_type(type) \ (simplequeue_get_state(PyType_GetModuleByDef(type, &queuemodule))) +static const Py_ssize_t INITIAL_RING_BUF_CAPACITY = 8; + +typedef struct { + // Where to place the next item + Py_ssize_t put_idx; + + // Where to get the next item + Py_ssize_t get_idx; + + PyObject **items; + + // Total number of items that may be stored + Py_ssize_t items_cap; + + // Number of items stored + Py_ssize_t num_items; +} RingBuf; + +static int +RingBuf_Init(RingBuf *buf) +{ + buf->put_idx = 0; + buf->get_idx = 0; + buf->items_cap = INITIAL_RING_BUF_CAPACITY; + buf->num_items = 0; + buf->items = PyMem_Calloc(buf->items_cap, sizeof(PyObject *)); + if (buf->items == NULL) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + +static PyObject * +RingBuf_At(RingBuf *buf, Py_ssize_t idx) +{ + assert(idx >= 0 && idx < buf->num_items); + return buf->items[(buf->get_idx + idx) % buf->items_cap]; +} + +static void +RingBuf_Fini(RingBuf *buf) +{ + PyObject **items = buf->items; + Py_ssize_t num_items = buf->num_items; + Py_ssize_t cap = buf->items_cap; + Py_ssize_t idx = buf->get_idx; + buf->items = NULL; + buf->put_idx = 0; + buf->get_idx = 0; + buf->num_items = 0; + buf->items_cap = 0; + for (Py_ssize_t n = num_items; n > 0; idx = (idx + 1) % cap, n--) { + Py_DECREF(items[idx]); + } + PyMem_Free(items); +} + +// Resize the underlying items array of buf to the new capacity and arrange +// the items contiguously in the new items array. +// +// Returns -1 on allocation failure or 0 on success. +static int +resize_ringbuf(RingBuf *buf, Py_ssize_t capacity) +{ + Py_ssize_t new_capacity = Py_MAX(INITIAL_RING_BUF_CAPACITY, capacity); + if (new_capacity == buf->items_cap) { + return 0; + } + assert(buf->num_items <= new_capacity); + + PyObject **new_items = PyMem_Calloc(new_capacity, sizeof(PyObject *)); + if (new_items == NULL) { + return -1; + } + + // Copy the "tail" of the old items array. This corresponds to "head" of + // the abstract ring buffer. + Py_ssize_t tail_size = + Py_MIN(buf->num_items, buf->items_cap - buf->get_idx); + if (tail_size > 0) { + memcpy(new_items, buf->items + buf->get_idx, + tail_size * sizeof(PyObject *)); + } + + // Copy the "head" of the old items array, if any. This corresponds to the + // "tail" of the abstract ring buffer. + Py_ssize_t head_size = buf->num_items - tail_size; + if (head_size > 0) { + memcpy(new_items + tail_size, buf->items, + head_size * sizeof(PyObject *)); + } + + PyMem_Free(buf->items); + buf->items = new_items; + buf->items_cap = new_capacity; + buf->get_idx = 0; + buf->put_idx = buf->num_items; + + return 0; +} + +// Returns a strong reference from the head of the buffer. +static PyObject * +RingBuf_Get(RingBuf *buf) +{ + assert(buf->num_items > 0); + + if (buf->num_items < (buf->items_cap / 4)) { + // Items is less than 25% occupied, shrink it by 50%. This allows for + // growth without immediately needing to resize the underlying items + // array. + // + // It's safe it ignore allocation failures here; shrinking is an + // optimization that isn't required for correctness. + (void)resize_ringbuf(buf, buf->items_cap / 2); + } + + PyObject *item = buf->items[buf->get_idx]; + buf->items[buf->get_idx] = NULL; + buf->get_idx = (buf->get_idx + 1) % buf->items_cap; + buf->num_items--; + return item; +} + +// Returns 0 on success or -1 if the buffer failed to grow. +// +// Steals a reference to item. +static int +RingBuf_Put(RingBuf *buf, PyObject *item) +{ + assert(buf->num_items <= buf->items_cap); + + if (buf->num_items == buf->items_cap) { + // Buffer is full, grow it. + if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { + PyErr_NoMemory(); + return -1; + } + } + buf->items[buf->put_idx] = item; + buf->put_idx = (buf->put_idx + 1) % buf->items_cap; + buf->num_items++; + return 0; +} + +static Py_ssize_t +RingBuf_Len(RingBuf *buf) +{ + return buf->num_items; +} + +static bool +RingBuf_IsEmpty(RingBuf *buf) +{ + return buf->num_items == 0; +} + typedef struct { PyObject_HEAD - PyThread_type_lock lock; - int locked; - PyObject *lst; - Py_ssize_t lst_pos; + + // Are there threads waiting for items + bool has_threads_waiting; + + // Items in the queue + RingBuf buf; + PyObject *weakreflist; } simplequeueobject; @@ -43,7 +206,7 @@ class _queue.SimpleQueue "simplequeueobject *" "simplequeue_get_state_by_type(ty static int simplequeue_clear(simplequeueobject *self) { - Py_CLEAR(self->lst); + RingBuf_Fini(&self->buf); return 0; } @@ -53,12 +216,6 @@ simplequeue_dealloc(simplequeueobject *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked > 0) - PyThread_release_lock(self->lock); - PyThread_free_lock(self->lock); - } (void)simplequeue_clear(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -69,7 +226,10 @@ simplequeue_dealloc(simplequeueobject *self) static int simplequeue_traverse(simplequeueobject *self, visitproc visit, void *arg) { - Py_VISIT(self->lst); + RingBuf *buf = &self->buf; + for (Py_ssize_t i = 0, num_items = buf->num_items; i < num_items; i++) { + Py_VISIT(RingBuf_At(buf, i)); + } Py_VISIT(Py_TYPE(self)); return 0; } @@ -90,15 +250,7 @@ simplequeue_new_impl(PyTypeObject *type) self = (simplequeueobject *) type->tp_alloc(type, 0); if (self != NULL) { self->weakreflist = NULL; - self->lst = PyList_New(0); - self->lock = PyThread_allocate_lock(); - self->lst_pos = 0; - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "can't allocate lock"); - return NULL; - } - if (self->lst == NULL) { + if (RingBuf_Init(&self->buf) < 0) { Py_DECREF(self); return NULL; } @@ -107,7 +259,29 @@ simplequeue_new_impl(PyTypeObject *type) return (PyObject *) self; } +typedef struct { + bool handed_off; + simplequeueobject *queue; + PyObject *item; +} HandoffData; + +static void +maybe_handoff_item(HandoffData *data, PyObject **item, int has_more_waiters) +{ + if (item == NULL) { + // No threads were waiting + data->handed_off = false; + } + else { + // There was at least one waiting thread, hand off the item + *item = data->item; + data->handed_off = true; + } + data->queue->has_threads_waiting = has_more_waiters; +} + /*[clinic input] +@critical_section _queue.SimpleQueue.put item: object block: bool = True @@ -123,21 +297,28 @@ never blocks. They are provided for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=6e601fa707a782d5]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ { - /* BEGIN GIL-protected critical section */ - if (PyList_Append(self->lst, item) < 0) - return NULL; - if (self->locked) { - /* A get() may be waiting, wake it up */ - self->locked = 0; - PyThread_release_lock(self->lock); + HandoffData data = { + .handed_off = 0, + .item = Py_NewRef(item), + .queue = self, + }; + if (self->has_threads_waiting) { + // Try to hand the item off directly if there are threads waiting + _PyParkingLot_Unpark(&self->has_threads_waiting, + (_Py_unpark_fn_t *)maybe_handoff_item, &data); + } + if (!data.handed_off) { + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } } - /* END GIL-protected critical section */ Py_RETURN_NONE; } /*[clinic input] +@critical_section _queue.SimpleQueue.put_nowait item: object @@ -150,39 +331,23 @@ for compatibility with the Queue class. static PyObject * _queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item) -/*[clinic end generated code: output=0990536715efb1f1 input=36b1ea96756b2ece]*/ +/*[clinic end generated code: output=0990536715efb1f1 input=ce949cc2cd8a4119]*/ { return _queue_SimpleQueue_put_impl(self, item, 0, Py_None); } static PyObject * -simplequeue_pop_item(simplequeueobject *self) +empty_error(PyTypeObject *cls) { - Py_ssize_t count, n; - PyObject *item; - - n = PyList_GET_SIZE(self->lst); - assert(self->lst_pos < n); - - item = PyList_GET_ITEM(self->lst, self->lst_pos); - Py_INCREF(Py_None); - PyList_SET_ITEM(self->lst, self->lst_pos, Py_None); - self->lst_pos += 1; - count = n - self->lst_pos; - if (self->lst_pos > count) { - /* The list is more than 50% empty, reclaim space at the beginning */ - if (PyList_SetSlice(self->lst, 0, self->lst_pos, NULL)) { - /* Undo pop */ - self->lst_pos -= 1; - PyList_SET_ITEM(self->lst, self->lst_pos, item); - return NULL; - } - self->lst_pos = 0; - } - return item; + PyObject *module = PyType_GetModule(cls); + assert(module != NULL); + simplequeue_state *state = simplequeue_get_state(module); + PyErr_SetNone(state->EmptyError); + return NULL; } /*[clinic input] +@critical_section _queue.SimpleQueue.get cls: defining_class @@ -205,21 +370,15 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=5b4047bfbc645ec1]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { - _PyTime_t endtime = 0; - _PyTime_t timeout; - PyObject *item; - PyLockStatus r; - PY_TIMEOUT_T microseconds; - PyThreadState *tstate = PyThreadState_Get(); + PyTime_t endtime = 0; - if (block == 0) { - /* Non-blocking */ - microseconds = 0; - } - else if (timeout_obj != Py_None) { + // XXX Use PyThread_ParseTimeoutArg(). + + if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; @@ -229,66 +388,64 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, "'timeout' must be a non-negative number"); return NULL; } - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); - if (microseconds > PY_TIMEOUT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "timeout value is too large"); - return NULL; - } endtime = _PyDeadline_Init(timeout); } - else { - /* Infinitely blocking */ - microseconds = -1; - } - /* put() signals the queue to be non-empty by releasing the lock. - * So we simply try to acquire the lock in a loop, until the condition - * (queue non-empty) becomes true. - */ - while (self->lst_pos == PyList_GET_SIZE(self->lst)) { - /* First a simple non-blocking try without releasing the GIL */ - r = PyThread_acquire_lock_timed(self->lock, 0, 0); - if (r == PY_LOCK_FAILURE && microseconds != 0) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); - Py_END_ALLOW_THREADS + for (;;) { + if (!RingBuf_IsEmpty(&self->buf)) { + return RingBuf_Get(&self->buf); } - if (r == PY_LOCK_INTR && _PyEval_MakePendingCalls(tstate) < 0) { - return NULL; + if (!block) { + return empty_error(cls); } - if (r == PY_LOCK_FAILURE) { - PyObject *module = PyType_GetModule(cls); - simplequeue_state *state = simplequeue_get_state(module); - /* Timed out */ - PyErr_SetNone(state->EmptyError); - return NULL; - } - self->locked = 1; - /* Adjust timeout for next iteration (if any) */ - if (microseconds > 0) { - timeout = _PyDeadline_Get(endtime); - microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_CEILING); + int64_t timeout_ns = -1; + if (endtime != 0) { + timeout_ns = _PyDeadline_Get(endtime); + if (timeout_ns < 0) { + return empty_error(cls); + } } - } - /* BEGIN GIL-protected critical section */ - assert(self->lst_pos < PyList_GET_SIZE(self->lst)); - item = simplequeue_pop_item(self); - if (self->locked) { - PyThread_release_lock(self->lock); - self->locked = 0; + bool waiting = 1; + self->has_threads_waiting = waiting; + + PyObject *item = NULL; + int st = _PyParkingLot_Park(&self->has_threads_waiting, &waiting, + sizeof(bool), timeout_ns, &item, + /* detach */ 1); + switch (st) { + case Py_PARK_OK: { + assert(item != NULL); + return item; + } + case Py_PARK_TIMEOUT: { + return empty_error(cls); + } + case Py_PARK_INTR: { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return NULL; + } + break; + } + case Py_PARK_AGAIN: { + // This should be impossible with the current implementation of + // PyParkingLot, but would be possible if critical sections / + // the GIL were released before the thread was added to the + // internal thread queue in the parking lot. + break; + } + default: { + Py_UNREACHABLE(); + } + } } - /* END GIL-protected critical section */ - - return item; } /*[clinic input] +@critical_section _queue.SimpleQueue.get_nowait cls: defining_class @@ -303,12 +460,13 @@ raise the Empty exception. static PyObject * _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=620c58e2750f8b8a input=842f732bf04216d3]*/ +/*[clinic end generated code: output=620c58e2750f8b8a input=d48be63633fefae9]*/ { return _queue_SimpleQueue_get_impl(self, cls, 0, Py_None); } /*[clinic input] +@critical_section _queue.SimpleQueue.empty -> bool Return True if the queue is empty, False otherwise (not reliable!). @@ -316,12 +474,13 @@ Return True if the queue is empty, False otherwise (not reliable!). static int _queue_SimpleQueue_empty_impl(simplequeueobject *self) -/*[clinic end generated code: output=1a02a1b87c0ef838 input=1a98431c45fd66f9]*/ +/*[clinic end generated code: output=1a02a1b87c0ef838 input=96cb22df5a67d831]*/ { - return self->lst_pos == PyList_GET_SIZE(self->lst); + return RingBuf_IsEmpty(&self->buf); } /*[clinic input] +@critical_section _queue.SimpleQueue.qsize -> Py_ssize_t Return the approximate size of the queue (not reliable!). @@ -329,9 +488,9 @@ Return the approximate size of the queue (not reliable!). static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self) -/*[clinic end generated code: output=f9dcd9d0a90e121e input=7a74852b407868a1]*/ +/*[clinic end generated code: output=f9dcd9d0a90e121e input=e218623cb8c16a79]*/ { - return PyList_GET_SIZE(self->lst) - self->lst_pos; + return RingBuf_Len(&self->buf); } static int @@ -435,6 +594,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index d41093c8806476..140640ae8fbf3a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -71,7 +71,8 @@ #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_AsByteArray() +#include "pycore_long.h" // _PyLong_NumBits() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() @@ -174,6 +175,7 @@ genrand_uint32(RandomObject *self) */ /*[clinic input] +@critical_section _random.Random.random self: self(type="RandomObject *") @@ -183,7 +185,7 @@ random() -> x in the interval [0, 1). static PyObject * _random_Random_random_impl(RandomObject *self) -/*[clinic end generated code: output=117ff99ee53d755c input=afb2a59cbbb00349]*/ +/*[clinic end generated code: output=117ff99ee53d755c input=26492e52d26e8b7b]*/ { uint32_t a=genrand_uint32(self)>>5, b=genrand_uint32(self)>>6; return PyFloat_FromDouble((a*67108864.0+b)*(1.0/9007199254740992.0)); @@ -257,13 +259,15 @@ random_seed_urandom(RandomObject *self) return 0; } -static void +static int random_seed_time_pid(RandomObject *self) { - _PyTime_t now; - uint32_t key[5]; + PyTime_t now; + if (PyTime_Time(&now) < 0) { + return -1; + } - now = _PyTime_GetSystemClock(); + uint32_t key[5]; key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); @@ -275,11 +279,14 @@ random_seed_time_pid(RandomObject *self) key[2] = 0; #endif - now = _PyTime_GetMonotonicClock(); + if (PyTime_Monotonic(&now) < 0) { + return -1; + } key[3] = (uint32_t)(now & 0xffffffffU); key[4] = (uint32_t)(now >> 32); init_by_array(self, key, Py_ARRAY_LENGTH(key)); + return 0; } static int @@ -297,7 +304,9 @@ random_seed(RandomObject *self, PyObject *arg) /* Reading system entropy failed, fall back on the worst entropy: use the current time and process identifier. */ - random_seed_time_pid(self); + if (random_seed_time_pid(self) < 0) { + return -1; + } } return 0; } @@ -340,7 +349,8 @@ random_seed(RandomObject *self, PyObject *arg) res = _PyLong_AsByteArray((PyLongObject *)n, (unsigned char *)key, keyused * 4, PY_LITTLE_ENDIAN, - 0); /* unsigned */ + 0, /* unsigned */ + 1); /* with exceptions */ if (res == -1) { goto Done; } @@ -367,6 +377,7 @@ random_seed(RandomObject *self, PyObject *arg) } /*[clinic input] +@critical_section _random.Random.seed self: self(type="RandomObject *") @@ -381,7 +392,7 @@ of the current time and the process identifier. static PyObject * _random_Random_seed_impl(RandomObject *self, PyObject *n) -/*[clinic end generated code: output=0fad1e16ba883681 input=78d6ef0d52532a54]*/ +/*[clinic end generated code: output=0fad1e16ba883681 input=46d01d2ba938c7b1]*/ { if (random_seed(self, n) < 0) { return NULL; @@ -390,6 +401,7 @@ _random_Random_seed_impl(RandomObject *self, PyObject *n) } /*[clinic input] +@critical_section _random.Random.getstate self: self(type="RandomObject *") @@ -399,7 +411,7 @@ getstate() -> tuple containing the current state. static PyObject * _random_Random_getstate_impl(RandomObject *self) -/*[clinic end generated code: output=bf6cef0c092c7180 input=b937a487928c0e89]*/ +/*[clinic end generated code: output=bf6cef0c092c7180 input=b6621f31eb639694]*/ { PyObject *state; PyObject *element; @@ -427,6 +439,7 @@ _random_Random_getstate_impl(RandomObject *self) /*[clinic input] +@critical_section _random.Random.setstate self: self(type="RandomObject *") @@ -437,8 +450,8 @@ setstate(state) -> None. Restores generator state. [clinic start generated code]*/ static PyObject * -_random_Random_setstate(RandomObject *self, PyObject *state) -/*[clinic end generated code: output=fd1c3cd0037b6681 input=b3b4efbb1bc66af8]*/ +_random_Random_setstate_impl(RandomObject *self, PyObject *state) +/*[clinic end generated code: output=babfc2c2eac6b027 input=358e898ec07469b7]*/ { int i; unsigned long element; @@ -478,7 +491,7 @@ _random_Random_setstate(RandomObject *self, PyObject *state) } /*[clinic input] - +@critical_section _random.Random.getrandbits self: self(type="RandomObject *") @@ -490,7 +503,7 @@ getrandbits(k) -> x. Generates an int with k random bits. static PyObject * _random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=8c0e6396dd176fc0]*/ +/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ { int i, words; uint32_t r; @@ -629,6 +642,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 6cc09088bdc869..e9170f2ce1ae87 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -2,6 +2,13 @@ * Helper method for urllib to fetch the proxy configuration settings * using the SystemConfiguration framework. */ + +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include #include @@ -21,8 +28,7 @@ cfstring_to_pystring(CFStringRef ref) s = CFStringGetCStringPtr(ref, kCFStringEncodingUTF8); if (s) { - return PyUnicode_DecodeUTF8( - s, strlen(s), NULL); + return PyUnicode_DecodeUTF8(s, strlen(s), NULL); } else { CFIndex len = CFStringGetLength(ref); @@ -43,8 +49,7 @@ cfstring_to_pystring(CFStringRef ref) PyMem_Free(buf); return NULL; } else { - result = PyUnicode_DecodeUTF8( - buf, strlen(buf), NULL); + result = PyUnicode_DecodeUTF8(buf, strlen(buf), NULL); PyMem_Free(buf); } return result; @@ -84,7 +89,7 @@ get_proxy_settings(PyObject* Py_UNUSED(mod), PyObject *Py_UNUSED(ignored)) if (v == NULL) goto error; r = PyDict_SetItemString(result, "exclude_simple", v); - Py_SETREF(v, NULL); + Py_CLEAR(v); if (r == -1) goto error; anArray = CFDictionaryGetValue(proxyDict, @@ -104,13 +109,11 @@ get_proxy_settings(PyObject* Py_UNUSED(mod), PyObject *Py_UNUSED(ignored)) aString = CFArrayGetValueAtIndex(anArray, i); if (aString == NULL) { - PyTuple_SetItem(v, i, Py_None); - Py_INCREF(Py_None); + PyTuple_SetItem(v, i, Py_NewRef(Py_None)); } else { PyObject* t = cfstring_to_pystring(aString); if (!t) { - PyTuple_SetItem(v, i, Py_None); - Py_INCREF(Py_None); + PyTuple_SetItem(v, i, Py_NewRef(Py_None)); } else { PyTuple_SetItem(v, i, t); } @@ -148,15 +151,13 @@ set_proxy(PyObject* proxies, const char* proto, CFDictionaryRef proxyDict, if (h) { if (aNum) { int32_t port = cfnum_to_int32(aNum); - v = PyUnicode_FromFormat("http://%U:%ld", - h, (long)port); + v = PyUnicode_FromFormat("http://%U:%ld", h, (long)port); } else { v = PyUnicode_FromFormat("http://%U", h); } Py_DECREF(h); if (!v) return -1; - r = PyDict_SetItemString(proxies, proto, - v); + r = PyDict_SetItemString(proxies, proto, v); Py_DECREF(v); return r; } @@ -238,6 +239,7 @@ static PyMethodDef mod_methods[] = { static PyModuleDef_Slot _scproxy_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f099020c5f4e6f..d1a549a971c24a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,7 +4,6 @@ #include "blob.h" #include "util.h" -#include "pycore_weakref.h" // _PyWeakref_GET_REF() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -100,10 +99,10 @@ blob_close_impl(pysqlite_Blob *self) void pysqlite_close_all_blobs(pysqlite_Connection *self) { - for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); - PyObject *blob = _PyWeakref_GET_REF(weakref); - if (blob == NULL) { + PyObject *blob; + if (!PyWeakref_GetRef(weakref, &blob)) { continue; } close_blob((pysqlite_Blob *)blob); diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index eceb5a7bf23673..1bcda7702c2568 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pysqlite_connect__doc__, "connect($module, /, database, timeout=5.0, detect_types=0,\n" @@ -27,4 +28,4 @@ PyDoc_STRVAR(pysqlite_connect__doc__, #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=03bd99542e3aec9d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=69b9b00da71c3c0a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index e7c60264722f7b..b95ba948aaf97f 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(blob_close__doc__, "close($self, /)\n" "--\n" @@ -86,10 +88,6 @@ blob_write(pysqlite_Blob *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = blob_write_impl(self, &data); exit: @@ -213,4 +211,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=8bfd79ab12ac5385 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31abd55660e0c5af input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index fcc657799dfaf0..6cb610a12b59c6 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static int pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, @@ -743,7 +744,7 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -836,14 +837,14 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(progress_handler), &_Py_ID(n), }, + .ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -924,7 +925,7 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1203,21 +1204,67 @@ pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ign } PyDoc_STRVAR(pysqlite_connection_iterdump__doc__, -"iterdump($self, /)\n" +"iterdump($self, /, *, filter=None)\n" "--\n" "\n" -"Returns iterator to the dump of the database in an SQL text format."); +"Returns iterator to the dump of the database in an SQL text format.\n" +"\n" +" filter\n" +" An optional LIKE pattern for database objects to dump"); #define PYSQLITE_CONNECTION_ITERDUMP_METHODDEF \ - {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, pysqlite_connection_iterdump__doc__}, + {"iterdump", _PyCFunction_CAST(pysqlite_connection_iterdump), METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_iterdump__doc__}, static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self); +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter); static PyObject * -pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return pysqlite_connection_iterdump_impl(self); + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iterdump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *filter = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + filter = args[0]; +skip_optional_kwonly: + return_value = pysqlite_connection_iterdump_impl(self, filter); + +exit: + return return_value; } PyDoc_STRVAR(pysqlite_connection_backup__doc__, @@ -1550,16 +1597,14 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("deserialize", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (!noptargs) { goto skip_optional_kwonly; @@ -1821,4 +1866,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=166bf41ad5ca1655 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5d4d7e4abe17b164 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index 3cb637bde4ff6c..a13e0d0745b58d 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() static int pysqlite_cursor_init_impl(pysqlite_Cursor *self, @@ -312,4 +313,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl(self); } -/*[clinic end generated code: output=0c52a9cf54d00543 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a8ce095c3c80cf65 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 49ba7a341ed587..529dc4e281e0eb 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pysqlite_complete_statement__doc__, "complete_statement($module, /, statement)\n" @@ -207,4 +208,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=a14893a7c2eead5e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=457ab0fdbb9e1880 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/row.c.h b/Modules/_sqlite/clinic/row.c.h index cdf850d01e01a3..e8d1dbf2ba8bc9 100644 --- a/Modules/_sqlite/clinic/row.c.h +++ b/Modules/_sqlite/clinic/row.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + static PyObject * pysqlite_row_new_impl(PyTypeObject *type, pysqlite_Cursor *cursor, PyObject *data); @@ -54,4 +56,4 @@ pysqlite_row_keys(pysqlite_Row *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_row_keys_impl(self); } -/*[clinic end generated code: output=972487d535d2e7d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=788bf817acc02b8e input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 21bdbc12814698..fc03e4a085c179 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -35,9 +35,10 @@ #include "util.h" #include "pycore_import.h" // _PyImport_GetModuleAttrString() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() -#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() +#include "pycore_weakref.h" #include @@ -75,16 +76,10 @@ isolation_level_converter(PyObject *str_or_none, const char **result) *result = NULL; } else if (PyUnicode_Check(str_or_none)) { - Py_ssize_t sz; - const char *str = PyUnicode_AsUTF8AndSize(str_or_none, &sz); + const char *str = _PyUnicode_AsUTF8NoNUL(str_or_none); if (str == NULL) { return 0; } - if (strlen(str) != (size_t)sz) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return 0; - } - const char *level = get_isolation_level(str); if (level == NULL) { return 0; @@ -1070,7 +1065,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { PyObject* weakref = PyList_GetItem(self->cursors, i); - if (_PyWeakref_IS_DEAD(weakref)) { + if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { @@ -1984,12 +1979,17 @@ pysqlite_connection_interrupt_impl(pysqlite_Connection *self) /*[clinic input] _sqlite3.Connection.iterdump as pysqlite_connection_iterdump + * + filter: object = None + An optional LIKE pattern for database objects to dump + Returns iterator to the dump of the database in an SQL text format. [clinic start generated code]*/ static PyObject * -pysqlite_connection_iterdump_impl(pysqlite_Connection *self) -/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/ +pysqlite_connection_iterdump_impl(pysqlite_Connection *self, + PyObject *filter) +/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/ { if (!pysqlite_check_connection(self)) { return NULL; @@ -2003,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self) } return NULL; } - - PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self); + PyObject *args[3] = {NULL, (PyObject *)self, filter}; + PyObject *kwnames = Py_BuildValue("(s)", "filter"); + if (!kwnames) { + Py_DECREF(iterdump); + return NULL; + } + Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames); Py_DECREF(iterdump); + Py_DECREF(kwnames); return retval; } @@ -2554,6 +2561,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2563,6 +2576,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} }; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 618ce532b2518d..0fbd408f18cf6a 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -669,16 +669,13 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } for (i = 0; i < num_params; i++) { const char *name = sqlite3_bind_parameter_name(self->st, i+1); - if (name != NULL) { - int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + if (name != NULL && name[0] != '?') { + PyErr_Format(state->ProgrammingError, "Binding %d ('%s') is a named parameter, but you " "supplied a sequence which requires nameless (qmark) " - "placeholders. Starting with Python 3.14 an " - "sqlite3.ProgrammingError will be raised.", + "placeholders.", i+1, name); - if (ret < 0) { - return; - } + return; } if (PyTuple_CheckExact(parameters)) { @@ -721,7 +718,6 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } else if (PyDict_Check(parameters)) { /* parameters passed as dictionary */ for (i = 1; i <= num_params_needed; i++) { - PyObject *binding_name_obj; Py_BEGIN_ALLOW_THREADS binding_name = sqlite3_bind_parameter_name(self->st, i); Py_END_ALLOW_THREADS @@ -733,17 +729,8 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } binding_name++; /* skip first char (the colon) */ - binding_name_obj = PyUnicode_FromString(binding_name); - if (!binding_name_obj) { - return; - } - if (PyDict_CheckExact(parameters)) { - PyObject *item = PyDict_GetItemWithError(parameters, binding_name_obj); - current_param = Py_XNewRef(item); - } else { - current_param = PyObject_GetItem(parameters, binding_name_obj); - } - Py_DECREF(binding_name_obj); + PyObject *current_param; + (void)PyMapping_GetOptionalItemString(parameters, binding_name, ¤t_param); if (!current_param) { if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_LookupError)) { PyErr_Format(state->ProgrammingError, diff --git a/Modules/_sqlite/microprotocols.c b/Modules/_sqlite/microprotocols.c index 92f0148bd4c217..f77458d94a8573 100644 --- a/Modules/_sqlite/microprotocols.c +++ b/Modules/_sqlite/microprotocols.c @@ -85,17 +85,16 @@ pysqlite_microprotocols_adapt(pysqlite_state *state, PyObject *obj, if (!key) { return NULL; } - adapter = PyDict_GetItemWithError(state->psyco_adapters, key); + if (PyDict_GetItemRef(state->psyco_adapters, key, &adapter) < 0) { + Py_DECREF(key); + return NULL; + } Py_DECREF(key); if (adapter) { - Py_INCREF(adapter); adapted = PyObject_CallOneArg(adapter, obj); Py_DECREF(adapter); return adapted; } - if (PyErr_Occurred()) { - return NULL; - } /* try to have the protocol adapt this object */ if (PyObject_GetOptionalAttr(proto, state->str___adapt__, &adapter) < 0) { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 46fed9f13281f3..698e81d9b897d0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -714,10 +714,6 @@ module_exec(PyObject *module) goto error; } - if (PyModule_AddStringConstant(module, "_deprecated_version", PYSQLITE_VERSION) < 0) { - goto error; - } - if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } @@ -758,6 +754,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c index 1a1943285ce008..14555076a7e79a 100644 --- a/Modules/_sqlite/row.c +++ b/Modules/_sqlite/row.c @@ -21,6 +21,10 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "row.h" #include "cursor.h" diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 833a666301d8ff..9e8613ef67916e 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -162,7 +162,7 @@ _pysqlite_long_as_int64(PyObject * py_val) sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), - IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { + IS_LITTLE_ENDIAN, 1 /* signed */, 0) >= 0) { return int64val; } } diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index 1b5e1a710af9db..48336c7a2fca26 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_sre_getcodesize__doc__, "getcodesize($module, /)\n" @@ -1433,7 +1434,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_match(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "match() takes no arguments"); return NULL; } @@ -1454,10 +1455,10 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * _sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "search() takes no arguments"); return NULL; } return _sre_SRE_Scanner_search_impl(self, cls); } -/*[clinic end generated code: output=46d83927cbafa93a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 0f134b194de6f6..0a888af31b0497 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,13 +39,14 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" -#include "pycore_dict.h" // _PyDict_Next() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Next() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "sre.h" // SRE_CODE +#include "sre.h" // SRE_CODE -#include // tolower(), toupper(), isalnum() +#include // tolower(), toupper(), isalnum() #define SRE_CODE_BITS (8 * sizeof(SRE_CODE)) @@ -107,9 +108,11 @@ static unsigned int sre_toupper(unsigned int ch) { #if VERBOSE == 0 # define INIT_TRACE(state) +# define DO_TRACE 0 # define TRACE(v) #elif VERBOSE == 1 # define INIT_TRACE(state) int _debug = (state)->debug +# define DO_TRACE (_debug) # define TRACE(v) do { \ if (_debug) { \ printf v; \ @@ -117,6 +120,7 @@ static unsigned int sre_toupper(unsigned int ch) { } while (0) #elif VERBOSE == 2 # define INIT_TRACE(state) +# define DO_TRACE 1 # define TRACE(v) printf v #else # error VERBOSE must be 0, 1 or 2 @@ -1618,6 +1622,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } self->items[i].literal = Py_XNewRef(literal); } + PyObject_GC_Track(self); return (PyObject*) self; bad_template: @@ -2212,6 +2217,8 @@ match_getindex(MatchObject* self, PyObject* index) return -1; } + // Check that i*2 cannot overflow to make static analyzers happy + assert(i <= SRE_MAXGROUPS); return i; } @@ -2346,26 +2353,28 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) if (!result || !self->pattern->groupindex) return result; + Py_BEGIN_CRITICAL_SECTION(self->pattern->groupindex); while (_PyDict_Next(self->pattern->groupindex, &pos, &key, &value, &hash)) { int status; Py_INCREF(key); value = match_getslice(self, key, default_value); if (!value) { Py_DECREF(key); - goto failed; + Py_CLEAR(result); + goto exit; } status = _PyDict_SetItem_KnownHash(result, key, value, hash); Py_DECREF(value); Py_DECREF(key); - if (status < 0) - goto failed; + if (status < 0) { + Py_CLEAR(result); + goto exit; + } } +exit:; + Py_END_CRITICAL_SECTION(); return result; - -failed: - Py_DECREF(result); - return NULL; } /*[clinic input] @@ -3266,6 +3275,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 92dd725c70fd38..97fbb0a75e54b6 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -373,6 +373,19 @@ SRE(count)(SRE_STATE* state, const SRE_CODE* pattern, Py_ssize_t maxcount) state->lastindex = ctx->lastindex; \ } while (0) +#define LAST_PTR_PUSH() \ + do { \ + TRACE(("push last_ptr: %zd", \ + PTR_TO_INDEX(ctx->u.rep->last_ptr))); \ + DATA_PUSH(&ctx->u.rep->last_ptr); \ + } while (0) +#define LAST_PTR_POP() \ + do { \ + DATA_POP(&ctx->u.rep->last_ptr); \ + TRACE(("pop last_ptr: %zd", \ + PTR_TO_INDEX(ctx->u.rep->last_ptr))); \ + } while (0) + #define RETURN_ERROR(i) do { return i; } while(0) #define RETURN_FAILURE do { ret = 0; goto exit; } while(0) #define RETURN_SUCCESS do { ret = 1; goto exit; } while(0) @@ -449,8 +462,27 @@ do { \ #define DATA_LOOKUP_AT(t,p,pos) \ DATA_STACK_LOOKUP_AT(state,t,p,pos) +#define PTR_TO_INDEX(ptr) \ + ((ptr) ? ((char*)(ptr) - (char*)state->beginning) / state->charsize : -1) + +#if VERBOSE +# define MARK_TRACE(label, lastmark) \ + do if (DO_TRACE) { \ + TRACE(("%s %d marks:", (label), (lastmark)+1)); \ + for (int j = 0; j <= (lastmark); j++) { \ + if (j && (j & 1) == 0) { \ + TRACE((" ")); \ + } \ + TRACE((" %zd", PTR_TO_INDEX(state->mark[j]))); \ + } \ + TRACE(("\n")); \ + } while (0) +#else +# define MARK_TRACE(label, lastmark) +#endif #define MARK_PUSH(lastmark) \ do if (lastmark >= 0) { \ + MARK_TRACE("push", (lastmark)); \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_PUSH(state, state->mark, _marks_size); \ } while (0) @@ -458,16 +490,19 @@ do { \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP(state, state->mark, _marks_size, 1); \ + MARK_TRACE("pop", (lastmark)); \ } while (0) #define MARK_POP_KEEP(lastmark) \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP(state, state->mark, _marks_size, 0); \ + MARK_TRACE("pop keep", (lastmark)); \ } while (0) #define MARK_POP_DISCARD(lastmark) \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP_DISCARD(state, _marks_size); \ + MARK_TRACE("pop discard", (lastmark)); \ } while (0) #define JUMP_NONE 0 @@ -1087,7 +1122,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* install new repeat context */ /* TODO(https://github.com/python/cpython/issues/67877): Fix this * potential memory leak. */ - ctx->u.rep = (SRE_REPEAT*) PyObject_Malloc(sizeof(*ctx->u.rep)); + ctx->u.rep = (SRE_REPEAT*) PyMem_Malloc(sizeof(*ctx->u.rep)); if (!ctx->u.rep) { PyErr_NoMemory(); RETURN_FAILURE; @@ -1101,7 +1136,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; - PyObject_Free(ctx->u.rep); + PyMem_Free(ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); @@ -1150,11 +1185,11 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) LASTMARK_SAVE(); MARK_PUSH(ctx->lastmark); /* zero-width match protection */ - DATA_PUSH(&ctx->u.rep->last_ptr); + LAST_PTR_PUSH(); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, ctx->u.rep->pattern+3); - DATA_POP(&ctx->u.rep->last_ptr); + LAST_PTR_POP(); if (ret) { MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); @@ -1235,11 +1270,11 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ctx->u.rep->count = ctx->count; /* zero-width match protection */ - DATA_PUSH(&ctx->u.rep->last_ptr); + LAST_PTR_PUSH(); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, ctx->u.rep->pattern+3); - DATA_POP(&ctx->u.rep->last_ptr); + LAST_PTR_POP(); if (ret) { RETURN_ON_ERROR(ret); RETURN_SUCCESS; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index cecc3785c7661c..1f5f0215980971 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,7 +28,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_time.h" // _PyDeadline_Init() /* Include symbols from _socket module */ #include "socketmodule.h" @@ -125,10 +125,10 @@ static void _PySSLFixErrno(void) { #include "_ssl_data_31.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) #include "_ssl_data_300.h" -#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) && !defined(LIBRESSL_VERSION_NUMBER) +#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) #include "_ssl_data_111.h" #else -#include "_ssl_data.h" +#error Unsupported OpenSSL version #endif /* OpenSSL API 1.1.0+ does not include version methods */ @@ -187,6 +187,11 @@ extern const SSL_METHOD *TLSv1_2_method(void); #endif +#if defined(SSL_VERIFY_POST_HANDSHAKE) && defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + #define PySSL_HAVE_POST_HS_AUTH +#endif + + enum py_ssl_error { /* these mirror ssl.h */ PY_SSL_ERROR_NONE, @@ -231,7 +236,7 @@ enum py_proto_version { PY_PROTO_TLSv1 = TLS1_VERSION, PY_PROTO_TLSv1_1 = TLS1_1_VERSION, PY_PROTO_TLSv1_2 = TLS1_2_VERSION, -#ifdef TLS1_3_VERSION +#if defined(TLS1_3_VERSION) PY_PROTO_TLSv1_3 = TLS1_3_VERSION, #else PY_PROTO_TLSv1_3 = 0x304, @@ -293,7 +298,7 @@ typedef struct { */ unsigned int hostflags; int protocol; -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int post_handshake_auth; #endif PyObject *msg_cb; @@ -301,6 +306,10 @@ typedef struct { BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; +#ifndef OPENSSL_NO_PSK + PyObject *psk_client_callback; + PyObject *psk_server_callback; +#endif } PySSLContext; typedef struct { @@ -365,7 +374,7 @@ class _ssl.SSLSession "PySSLSession *" "get_state_type(type)->PySSLSession_Type" #include "clinic/_ssl.c.h" -static int PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout); +static int PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout); static int PySSL_set_owner(PySSLSocket *, PyObject *, void *); static int PySSL_set_session(PySSLSocket *, PyObject *, void *); @@ -387,8 +396,8 @@ typedef enum { // Return a borrowed reference. static inline PySocketSockObject* GET_SOCKET(PySSLSocket *obj) { if (obj->Socket) { - PyObject *sock = _PyWeakref_GET_REF(obj->Socket); - if (sock != NULL) { + PyObject *sock; + if (PyWeakref_GetRef(obj->Socket, &sock)) { // GET_SOCKET() returns a borrowed reference Py_DECREF(sock); } @@ -594,7 +603,7 @@ PySSL_ChainExceptions(PySSLSocket *sslsock) { } static PyObject * -PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) +PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) { PyObject *type; char *errstr = NULL; @@ -607,7 +616,6 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) _sslmodulestate *state = get_state_sock(sslsock); type = state->PySSLErrorObject; - assert(ret <= 0); e = ERR_peek_last_error(); if (sslsock->ssl != NULL) { @@ -640,32 +648,21 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) case SSL_ERROR_SYSCALL: { if (e == 0) { - PySocketSockObject *s = GET_SOCKET(sslsock); - if (ret == 0 || (((PyObject *)s) == Py_None)) { + /* underlying BIO reported an I/O error */ + ERR_clear_error(); +#ifdef MS_WINDOWS + if (err.ws) { + return PyErr_SetFromWindowsErr(err.ws); + } +#endif + if (err.c) { + errno = err.c; + return PyErr_SetFromErrno(PyExc_OSError); + } + else { p = PY_SSL_ERROR_EOF; type = state->PySSLEOFErrorObject; errstr = "EOF occurred in violation of protocol"; - } else if (s && ret == -1) { - /* underlying BIO reported an I/O error */ - ERR_clear_error(); -#ifdef MS_WINDOWS - if (err.ws) { - return PyErr_SetFromWindowsErr(err.ws); - } -#endif - if (err.c) { - errno = err.c; - return PyErr_SetFromErrno(PyExc_OSError); - } - else { - p = PY_SSL_ERROR_EOF; - type = state->PySSLEOFErrorObject; - errstr = "EOF occurred in violation of protocol"; - } - } else { /* possible? */ - p = PY_SSL_ERROR_SYSCALL; - type = state->PySSLSyscallErrorObject; - errstr = "Some I/O error occurred"; } } else { if (ERR_GET_LIB(e) == ERR_LIB_SSL && @@ -863,7 +860,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ -#if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION < 0x101010cf +#if OPENSSL_VERSION < 0x101010cf X509_VERIFY_PARAM *ssl_params = SSL_get0_param(self->ssl); X509_VERIFY_PARAM_set_hostflags(ssl_params, sslctx->hostflags); #endif @@ -881,7 +878,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, SSL_set_mode(self->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) if (sslctx->post_handshake_auth == 1) { if (socket_type == PY_SSL_SERVER) { /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. @@ -889,10 +886,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, * only in combination with SSL_VERIFY_PEER flag. */ int mode = SSL_get_verify_mode(self->ssl); if (mode & SSL_VERIFY_PEER) { - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; - verify_cb = SSL_get_verify_callback(self->ssl); mode |= SSL_VERIFY_POST_HANDSHAKE; - SSL_set_verify(self->ssl, mode, verify_cb); + SSL_set_verify(self->ssl, mode, NULL); } } else { /* client socket */ @@ -961,7 +956,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) _PySSLError err; int sockstate, nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock) { @@ -1026,8 +1021,9 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) } while (err.ssl == SSL_ERROR_WANT_READ || err.ssl == SSL_ERROR_WANT_WRITE); Py_XDECREF(sock); + if (ret < 1) - return PySSL_SetError(self, ret, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; Py_RETURN_NONE; @@ -2214,8 +2210,8 @@ PySSL_get_owner(PySSLSocket *self, void *c) if (self->owner == NULL) { Py_RETURN_NONE; } - PyObject *owner = _PyWeakref_GET_REF(self->owner); - if (owner == NULL) { + PyObject *owner; + if (!PyWeakref_GetRef(self->owner, &owner)) { Py_RETURN_NONE; } return owner; @@ -2271,12 +2267,12 @@ PySSL_dealloc(PySSLSocket *self) */ static int -PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) +PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) { int rc; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else int nfds; fd_set fds; @@ -2355,7 +2351,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2434,7 +2430,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_XDECREF(sock); if (retval == 0) - return PySSL_SetError(self, retval, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; return PyLong_FromSize_t(count); @@ -2464,7 +2460,7 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) self->err = err; if (count < 0) - return PySSL_SetError(self, count, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); else return PyLong_FromLong(count); } @@ -2493,7 +2489,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (!group_right_1 && len < 0) { @@ -2587,7 +2583,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err.ssl == SSL_ERROR_WANT_WRITE); if (retval == 0) { - PySSL_SetError(self, retval, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); goto error; } if (self->exc != NULL) @@ -2625,7 +2621,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) int sockstate, nonblocking, ret; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2713,7 +2709,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } if (ret < 0) { Py_XDECREF(sock); - PySSL_SetError(self, ret, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); return NULL; } if (self->exc != NULL) @@ -2785,7 +2781,7 @@ static PyObject * _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ { -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); @@ -2993,7 +2989,6 @@ static int _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) { int mode; - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; switch(n) { case PY_SSL_CERT_NONE: @@ -3014,9 +3009,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for * server sockets and SSL_set_post_handshake_auth() for client. */ - /* keep current verify cb */ - verify_cb = SSL_CTX_get_verify_callback(self->ctx); - SSL_CTX_set_verify(self->ctx, mode, verify_cb); + SSL_CTX_set_verify(self->ctx, mode, NULL); return 0; } @@ -3123,6 +3116,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->alpn_protocols = NULL; self->set_sni_cb = NULL; self->state = get_ssl_state(module); +#ifndef OPENSSL_NO_PSK + self->psk_client_callback = NULL; + self->psk_server_callback = NULL; +#endif /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3174,7 +3171,6 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) result = SSL_CTX_set_cipher_list(ctx, "HIGH:!aNULL:!eNULL"); } if (result == 0) { - Py_DECREF(self); ERR_clear_error(); PyErr_SetString(get_state_ctx(self)->PySSLErrorObject, "No cipher can be selected."); @@ -3208,7 +3204,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) self->post_handshake_auth = 0; SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); #endif @@ -3235,6 +3231,10 @@ context_clear(PySSLContext *self) Py_CLEAR(self->set_sni_cb); Py_CLEAR(self->msg_cb); Py_CLEAR(self->keylog_filename); +#ifndef OPENSSL_NO_PSK + Py_CLEAR(self->psk_client_callback); + Py_CLEAR(self->psk_server_callback); +#endif if (self->keylog_bio != NULL) { PySSL_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); @@ -3478,8 +3478,8 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) } switch(self->protocol) { - case PY_SSL_VERSION_TLS_CLIENT: /* fall through */ - case PY_SSL_VERSION_TLS_SERVER: /* fall through */ + case PY_SSL_VERSION_TLS_CLIENT: _Py_FALLTHROUGH; + case PY_SSL_VERSION_TLS_SERVER: _Py_FALLTHROUGH; case PY_SSL_VERSION_TLS: break; default: @@ -3582,7 +3582,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) return set_min_max_proto_version(self, arg, 1); } -#ifdef TLS1_3_VERSION +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) static PyObject * get_num_tickets(PySSLContext *self, void *c) { @@ -3613,7 +3613,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) PyDoc_STRVAR(PySSLContext_num_tickets_doc, "Control the number of TLSv1.3 session tickets"); -#endif /* TLS1_3_VERSION */ +#endif /* defined(TLS1_3_VERSION) */ static PyObject * get_security_level(PySSLContext *self, void *c) @@ -3716,14 +3716,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) static PyObject * get_post_handshake_auth(PySSLContext *self, void *c) { -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) return PyBool_FromLong(self->post_handshake_auth); #else Py_RETURN_NONE; #endif } -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) static int set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { if (arg == NULL) { @@ -4133,7 +4133,8 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, if (PyObject_GetBuffer(cadata, &buf, PyBUF_SIMPLE)) { goto error; } - if (!PyBuffer_IsContiguous(&buf, 'C') || buf.ndim > 1) { + assert(PyBuffer_IsContiguous(&buf, 'C')); + if (buf.ndim > 1) { PyBuffer_Release(&buf); PyErr_SetString(PyExc_TypeError, "cadata should be a contiguous buffer with " @@ -4437,9 +4438,9 @@ _servername_callback(SSL *s, int *al, void *args) * will be passed. If both do not exist only then the C-level object is * passed. */ if (ssl->owner) - ssl_socket = _PyWeakref_GET_REF(ssl->owner); + PyWeakref_GetRef(ssl->owner, &ssl_socket); else if (ssl->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl->Socket); + PyWeakref_GetRef(ssl->Socket, &ssl_socket); else ssl_socket = Py_NewRef(ssl); @@ -4545,6 +4546,50 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) return 0; } +#if OPENSSL_VERSION_NUMBER < 0x30300000L +static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) +{ + int ok; + X509_OBJECT *ret = X509_OBJECT_new(); + if (ret == NULL) { + return NULL; + } + switch (X509_OBJECT_get_type(obj)) { + case X509_LU_X509: + ok = X509_OBJECT_set1_X509(ret, X509_OBJECT_get0_X509(obj)); + break; + case X509_LU_CRL: + /* X509_OBJECT_get0_X509_CRL was not const-correct prior to 3.0.*/ + ok = X509_OBJECT_set1_X509_CRL( + ret, X509_OBJECT_get0_X509_CRL((X509_OBJECT *)obj)); + break; + default: + /* We cannot duplicate unrecognized types in a polyfill, but it is + * safe to leave an empty object. The caller will ignore it. */ + ok = 1; + break; + } + if (!ok) { + X509_OBJECT_free(ret); + return NULL; + } + return ret; +} + +static STACK_OF(X509_OBJECT) * +X509_STORE_get1_objects(X509_STORE *store) +{ + STACK_OF(X509_OBJECT) *ret; + if (!X509_STORE_lock(store)) { + return NULL; + } + ret = sk_X509_OBJECT_deep_copy(X509_STORE_get0_objects(store), + x509_object_dup, X509_OBJECT_free); + X509_STORE_unlock(store); + return ret; +} +#endif + PyDoc_STRVAR(PySSLContext_sni_callback_doc, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ \n\ @@ -4574,7 +4619,12 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) int x509 = 0, crl = 0, ca = 0, i; store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { obj = sk_X509_OBJECT_value(objs, i); switch (X509_OBJECT_get_type(obj)) { @@ -4588,12 +4638,11 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) crl++; break; default: - /* Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY. - * As far as I can tell they are internal states and never - * stored in a cert store */ + /* Ignore unrecognized types. */ break; } } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return Py_BuildValue("{sisisi}", "x509", x509, "crl", crl, "x509_ca", ca); } @@ -4625,7 +4674,12 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + goto error; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509_OBJECT *obj; X509 *cert; @@ -4653,14 +4707,248 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } Py_CLEAR(ci); } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return rlist; error: + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); Py_XDECREF(ci); Py_XDECREF(rlist); return NULL; } +#ifndef OPENSSL_NO_PSK +static unsigned int psk_client_callback(SSL *s, + const char *hint, + char *identity, + unsigned int max_identity_len, + unsigned char *psk, + unsigned int max_psk_len) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *callback = NULL; + + PySSLSocket *ssl = SSL_get_app_data(s); + if (ssl == NULL || ssl->ctx == NULL) { + goto error; + } + callback = ssl->ctx->psk_client_callback; + if (callback == NULL) { + goto error; + } + + PyObject *hint_str = (hint != NULL && hint[0] != '\0') ? + PyUnicode_DecodeUTF8(hint, strlen(hint), "strict") : + Py_NewRef(Py_None); + if (hint_str == NULL) { + /* The remote side has sent an invalid UTF-8 string + * (breaking the standard), drop the connection without + * raising a decode exception. */ + PyErr_Clear(); + goto error; + } + PyObject *result = PyObject_CallFunctionObjArgs(callback, hint_str, NULL); + Py_DECREF(hint_str); + + if (result == NULL) { + goto error; + } + + const char *psk_; + const char *identity_; + Py_ssize_t psk_len_; + Py_ssize_t identity_len_ = 0; + if (!PyArg_ParseTuple(result, "z#y#", &identity_, &identity_len_, &psk_, &psk_len_)) { + Py_DECREF(result); + goto error; + } + + if (identity_len_ + 1 > max_identity_len || psk_len_ > max_psk_len) { + Py_DECREF(result); + goto error; + } + memcpy(psk, psk_, psk_len_); + if (identity_ != NULL) { + memcpy(identity, identity_, identity_len_); + } + identity[identity_len_] = 0; + + Py_DECREF(result); + + PyGILState_Release(gstate); + return (unsigned int)psk_len_; + +error: + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(callback); + } + PyGILState_Release(gstate); + return 0; +} +#endif + +/*[clinic input] +_ssl._SSLContext.set_psk_client_callback + callback: object + +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, + PyObject *callback) +/*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ +{ +#ifndef OPENSSL_NO_PSK + if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { + _setSSLError(get_state_ctx(self), + "Cannot add PSK client callback to a " + "PROTOCOL_TLS_SERVER context", 0, __FILE__, __LINE__); + return NULL; + } + + SSL_psk_client_cb_func ssl_callback; + if (callback == Py_None) { + callback = NULL; + // Delete the existing callback + ssl_callback = NULL; + } else { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback must be callable"); + return NULL; + } + ssl_callback = psk_client_callback; + } + + Py_XDECREF(self->psk_client_callback); + Py_XINCREF(callback); + + self->psk_client_callback = callback; + SSL_CTX_set_psk_client_callback(self->ctx, ssl_callback); + + Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif +} + +#ifndef OPENSSL_NO_PSK +static unsigned int psk_server_callback(SSL *s, + const char *identity, + unsigned char *psk, + unsigned int max_psk_len) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *callback = NULL; + + PySSLSocket *ssl = SSL_get_app_data(s); + if (ssl == NULL || ssl->ctx == NULL) { + goto error; + } + callback = ssl->ctx->psk_server_callback; + if (callback == NULL) { + goto error; + } + + PyObject *identity_str = (identity != NULL && identity[0] != '\0') ? + PyUnicode_DecodeUTF8(identity, strlen(identity), "strict") : + Py_NewRef(Py_None); + if (identity_str == NULL) { + /* The remote side has sent an invalid UTF-8 string + * (breaking the standard), drop the connection without + * raising a decode exception. */ + PyErr_Clear(); + goto error; + } + PyObject *result = PyObject_CallFunctionObjArgs(callback, identity_str, NULL); + Py_DECREF(identity_str); + + if (result == NULL) { + goto error; + } + + char *psk_; + Py_ssize_t psk_len_; + if (PyBytes_AsStringAndSize(result, &psk_, &psk_len_) < 0) { + Py_DECREF(result); + goto error; + } + + if (psk_len_ > max_psk_len) { + Py_DECREF(result); + goto error; + } + memcpy(psk, psk_, psk_len_); + + Py_DECREF(result); + + PyGILState_Release(gstate); + return (unsigned int)psk_len_; + +error: + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(callback); + } + PyGILState_Release(gstate); + return 0; +} +#endif + +/*[clinic input] +_ssl._SSLContext.set_psk_server_callback + callback: object + identity_hint: str(accept={str, NoneType}) = None + +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, + PyObject *callback, + const char *identity_hint) +/*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ +{ +#ifndef OPENSSL_NO_PSK + if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { + _setSSLError(get_state_ctx(self), + "Cannot add PSK server callback to a " + "PROTOCOL_TLS_CLIENT context", 0, __FILE__, __LINE__); + return NULL; + } + + SSL_psk_server_cb_func ssl_callback; + if (callback == Py_None) { + callback = NULL; + // Delete the existing callback and hint + ssl_callback = NULL; + identity_hint = NULL; + } else { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback must be callable"); + return NULL; + } + ssl_callback = psk_server_callback; + } + + if (SSL_CTX_use_psk_identity_hint(self->ctx, identity_hint) != 1) { + PyErr_SetString(PyExc_ValueError, "failed to set identity hint"); + return NULL; + } + + Py_XDECREF(self->psk_server_callback); + Py_XINCREF(callback); + + self->psk_server_callback = callback; + SSL_CTX_set_psk_server_callback(self->ctx, ssl_callback); + + Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif +} + static PyGetSetDef context_getsetlist[] = { {"check_hostname", (getter) get_check_hostname, @@ -4677,14 +4965,14 @@ static PyGetSetDef context_getsetlist[] = { (setter) _PySSLContext_set_msg_callback, NULL}, {"sni_callback", (getter) get_sni_callback, (setter) set_sni_callback, PySSLContext_sni_callback_doc}, -#ifdef TLS1_3_VERSION +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) {"num_tickets", (getter) get_num_tickets, (setter) set_num_tickets, PySSLContext_num_tickets_doc}, #endif {"options", (getter) get_options, (setter) set_options, NULL}, {"post_handshake_auth", (getter) get_post_handshake_auth, -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) (setter) set_post_handshake_auth, #else NULL, @@ -4715,6 +5003,8 @@ static struct PyMethodDef context_methods[] = { _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF + _SSL__SSLCONTEXT_SET_PSK_CLIENT_CALLBACK_METHODDEF + _SSL__SSLCONTEXT_SET_PSK_SERVER_CALLBACK_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -5045,7 +5335,11 @@ PySSLSession_clear(PySSLSession *self) static PyObject * PySSLSession_get_time(PySSLSession *self, void *closure) { +#if OPENSSL_VERSION_NUMBER >= 0x30300000L + return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); +#else return PyLong_FromLong(SSL_SESSION_get_time(self->session)); +#endif } PyDoc_STRVAR(PySSLSession_get_time_doc, @@ -5786,52 +6080,44 @@ sslmodule_add_option(PyObject *m, const char *name, uint64_t value) static int sslmodule_init_constants(PyObject *m) { - PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", - PY_SSL_DEFAULT_CIPHER_STRING); - - PyModule_AddIntConstant(m, "SSL_ERROR_ZERO_RETURN", - PY_SSL_ERROR_ZERO_RETURN); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_READ", - PY_SSL_ERROR_WANT_READ); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_WRITE", - PY_SSL_ERROR_WANT_WRITE); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_X509_LOOKUP", - PY_SSL_ERROR_WANT_X509_LOOKUP); - PyModule_AddIntConstant(m, "SSL_ERROR_SYSCALL", - PY_SSL_ERROR_SYSCALL); - PyModule_AddIntConstant(m, "SSL_ERROR_SSL", - PY_SSL_ERROR_SSL); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_CONNECT", - PY_SSL_ERROR_WANT_CONNECT); + if (PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", + PY_SSL_DEFAULT_CIPHER_STRING) < 0) + { + return -1; + } + +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + + ADD_INT_CONST("SSL_ERROR_ZERO_RETURN", PY_SSL_ERROR_ZERO_RETURN); + ADD_INT_CONST("SSL_ERROR_WANT_READ", PY_SSL_ERROR_WANT_READ); + ADD_INT_CONST("SSL_ERROR_WANT_WRITE", PY_SSL_ERROR_WANT_WRITE); + ADD_INT_CONST("SSL_ERROR_WANT_X509_LOOKUP", PY_SSL_ERROR_WANT_X509_LOOKUP); + ADD_INT_CONST("SSL_ERROR_SYSCALL", PY_SSL_ERROR_SYSCALL); + ADD_INT_CONST("SSL_ERROR_SSL", PY_SSL_ERROR_SSL); + ADD_INT_CONST("SSL_ERROR_WANT_CONNECT", PY_SSL_ERROR_WANT_CONNECT); /* non ssl.h errorcodes */ - PyModule_AddIntConstant(m, "SSL_ERROR_EOF", - PY_SSL_ERROR_EOF); - PyModule_AddIntConstant(m, "SSL_ERROR_INVALID_ERROR_CODE", - PY_SSL_ERROR_INVALID_ERROR_CODE); + ADD_INT_CONST("SSL_ERROR_EOF", PY_SSL_ERROR_EOF); + ADD_INT_CONST("SSL_ERROR_INVALID_ERROR_CODE", + PY_SSL_ERROR_INVALID_ERROR_CODE); /* cert requirements */ - PyModule_AddIntConstant(m, "CERT_NONE", - PY_SSL_CERT_NONE); - PyModule_AddIntConstant(m, "CERT_OPTIONAL", - PY_SSL_CERT_OPTIONAL); - PyModule_AddIntConstant(m, "CERT_REQUIRED", - PY_SSL_CERT_REQUIRED); + ADD_INT_CONST("CERT_NONE", PY_SSL_CERT_NONE); + ADD_INT_CONST("CERT_OPTIONAL", PY_SSL_CERT_OPTIONAL); + ADD_INT_CONST("CERT_REQUIRED", PY_SSL_CERT_REQUIRED); /* CRL verification for verification_flags */ - PyModule_AddIntConstant(m, "VERIFY_DEFAULT", - 0); - PyModule_AddIntConstant(m, "VERIFY_CRL_CHECK_LEAF", - X509_V_FLAG_CRL_CHECK); - PyModule_AddIntConstant(m, "VERIFY_CRL_CHECK_CHAIN", - X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); - PyModule_AddIntConstant(m, "VERIFY_X509_STRICT", - X509_V_FLAG_X509_STRICT); - PyModule_AddIntConstant(m, "VERIFY_ALLOW_PROXY_CERTS", - X509_V_FLAG_ALLOW_PROXY_CERTS); - PyModule_AddIntConstant(m, "VERIFY_X509_TRUSTED_FIRST", - X509_V_FLAG_TRUSTED_FIRST); + ADD_INT_CONST("VERIFY_DEFAULT", 0); + ADD_INT_CONST("VERIFY_CRL_CHECK_LEAF", X509_V_FLAG_CRL_CHECK); + ADD_INT_CONST("VERIFY_CRL_CHECK_CHAIN", + X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); + ADD_INT_CONST("VERIFY_X509_STRICT", X509_V_FLAG_X509_STRICT); + ADD_INT_CONST("VERIFY_ALLOW_PROXY_CERTS", X509_V_FLAG_ALLOW_PROXY_CERTS); + ADD_INT_CONST("VERIFY_X509_TRUSTED_FIRST", X509_V_FLAG_TRUSTED_FIRST); #ifdef X509_V_FLAG_PARTIAL_CHAIN - PyModule_AddIntConstant(m, "VERIFY_X509_PARTIAL_CHAIN", - X509_V_FLAG_PARTIAL_CHAIN); + ADD_INT_CONST("VERIFY_X509_PARTIAL_CHAIN", X509_V_FLAG_PARTIAL_CHAIN); #endif /* Alert Descriptions from ssl.h */ @@ -5839,7 +6125,7 @@ sslmodule_init_constants(PyObject *m) /* http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6 */ #define ADD_AD_CONSTANT(s) \ - PyModule_AddIntConstant(m, "ALERT_DESCRIPTION_"#s, \ + ADD_INT_CONST("ALERT_DESCRIPTION_"#s, \ SSL_AD_##s) ADD_AD_CONSTANT(CLOSE_NOTIFY); @@ -5887,23 +6173,15 @@ sslmodule_init_constants(PyObject *m) /* protocol versions */ #ifndef OPENSSL_NO_SSL3 - PyModule_AddIntConstant(m, "PROTOCOL_SSLv3", - PY_SSL_VERSION_SSL3); + ADD_INT_CONST("PROTOCOL_SSLv3", PY_SSL_VERSION_SSL3); #endif - PyModule_AddIntConstant(m, "PROTOCOL_SSLv23", - PY_SSL_VERSION_TLS); - PyModule_AddIntConstant(m, "PROTOCOL_TLS", - PY_SSL_VERSION_TLS); - PyModule_AddIntConstant(m, "PROTOCOL_TLS_CLIENT", - PY_SSL_VERSION_TLS_CLIENT); - PyModule_AddIntConstant(m, "PROTOCOL_TLS_SERVER", - PY_SSL_VERSION_TLS_SERVER); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1", - PY_SSL_VERSION_TLS1); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_1", - PY_SSL_VERSION_TLS1_1); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_2", - PY_SSL_VERSION_TLS1_2); + ADD_INT_CONST("PROTOCOL_SSLv23", PY_SSL_VERSION_TLS); + ADD_INT_CONST("PROTOCOL_TLS", PY_SSL_VERSION_TLS); + ADD_INT_CONST("PROTOCOL_TLS_CLIENT", PY_SSL_VERSION_TLS_CLIENT); + ADD_INT_CONST("PROTOCOL_TLS_SERVER", PY_SSL_VERSION_TLS_SERVER); + ADD_INT_CONST("PROTOCOL_TLSv1", PY_SSL_VERSION_TLS1); + ADD_INT_CONST("PROTOCOL_TLSv1_1", PY_SSL_VERSION_TLS1_1); + ADD_INT_CONST("PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); #define ADD_OPTION(NAME, VALUE) if (sslmodule_add_option(m, NAME, (VALUE)) < 0) return -1 @@ -5948,50 +6226,52 @@ sslmodule_init_constants(PyObject *m) ADD_OPTION("OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); #endif +#undef ADD_OPTION + #ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT - PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT", - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); + ADD_INT_CONST("HOSTFLAG_ALWAYS_CHECK_SUBJECT", + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); #endif #ifdef X509_CHECK_FLAG_NEVER_CHECK_SUBJECT - PyModule_AddIntConstant(m, "HOSTFLAG_NEVER_CHECK_SUBJECT", - X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); + ADD_INT_CONST("HOSTFLAG_NEVER_CHECK_SUBJECT", + X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); #endif #ifdef X509_CHECK_FLAG_NO_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_NO_WILDCARDS", - X509_CHECK_FLAG_NO_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_NO_WILDCARDS", + X509_CHECK_FLAG_NO_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_NO_PARTIAL_WILDCARDS", - X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_NO_PARTIAL_WILDCARDS", + X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_MULTI_LABEL_WILDCARDS", - X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_MULTI_LABEL_WILDCARDS", + X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS - PyModule_AddIntConstant(m, "HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", - X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); + ADD_INT_CONST("HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", + X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); #endif /* file types */ - PyModule_AddIntConstant(m, "ENCODING_PEM", PY_SSL_ENCODING_PEM); - PyModule_AddIntConstant(m, "ENCODING_DER", PY_SSL_ENCODING_DER); + ADD_INT_CONST("ENCODING_PEM", PY_SSL_ENCODING_PEM); + ADD_INT_CONST("ENCODING_DER", PY_SSL_ENCODING_DER); /* protocol versions */ - PyModule_AddIntConstant(m, "PROTO_MINIMUM_SUPPORTED", - PY_PROTO_MINIMUM_SUPPORTED); - PyModule_AddIntConstant(m, "PROTO_MAXIMUM_SUPPORTED", - PY_PROTO_MAXIMUM_SUPPORTED); - PyModule_AddIntConstant(m, "PROTO_SSLv3", PY_PROTO_SSLv3); - PyModule_AddIntConstant(m, "PROTO_TLSv1", PY_PROTO_TLSv1); - PyModule_AddIntConstant(m, "PROTO_TLSv1_1", PY_PROTO_TLSv1_1); - PyModule_AddIntConstant(m, "PROTO_TLSv1_2", PY_PROTO_TLSv1_2); - PyModule_AddIntConstant(m, "PROTO_TLSv1_3", PY_PROTO_TLSv1_3); + ADD_INT_CONST("PROTO_MINIMUM_SUPPORTED", PY_PROTO_MINIMUM_SUPPORTED); + ADD_INT_CONST("PROTO_MAXIMUM_SUPPORTED", PY_PROTO_MAXIMUM_SUPPORTED); + ADD_INT_CONST("PROTO_SSLv3", PY_PROTO_SSLv3); + ADD_INT_CONST("PROTO_TLSv1", PY_PROTO_TLSv1); + ADD_INT_CONST("PROTO_TLSv1_1", PY_PROTO_TLSv1_1); + ADD_INT_CONST("PROTO_TLSv1_2", PY_PROTO_TLSv1_2); + ADD_INT_CONST("PROTO_TLSv1_3", PY_PROTO_TLSv1_3); #define addbool(m, key, value) \ do { \ PyObject *bool_obj = (value) ? Py_True : Py_False; \ - PyModule_AddObjectRef((m), (key), bool_obj); \ + if (PyModule_AddObjectRef((m), (key), bool_obj) < 0) { \ + return -1; \ + } \ } while (0) addbool(m, "HAS_SNI", 1); @@ -6032,6 +6312,15 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_TLSv1_3", 0); #endif +#ifdef OPENSSL_NO_PSK + addbool(m, "HAS_PSK", 0); +#else + addbool(m, "HAS_PSK", 1); +#endif + +#undef addbool +#undef ADD_INT_CONST + return 0; } @@ -6232,6 +6521,7 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_strings}, {Py_mod_exec, sslmodule_init_lock}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ssl/clinic/cert.c.h b/Modules/_ssl/clinic/cert.c.h index db43c88e411139..0748d32291e3c5 100644 --- a/Modules/_ssl/clinic/cert.c.h +++ b/Modules/_ssl/clinic/cert.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_ssl_Certificate_public_bytes__doc__, "public_bytes($self, /, format=Encoding.PEM)\n" @@ -85,4 +86,4 @@ _ssl_Certificate_get_info(PySSLCertificate *self, PyObject *Py_UNUSED(ignored)) { return _ssl_Certificate_get_info_impl(self); } -/*[clinic end generated code: output=8e438b54fbebd53e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5b668226f933a677 input=a9049054013a1b77]*/ diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 07e9ce7a6fce2d..9c87f8b4d21e68 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -28,12 +28,12 @@ _PySSL_msg_callback(int write_p, int version, int content_type, PyObject *ssl_socket; /* ssl.SSLSocket or ssl.SSLObject */ if (ssl_obj->owner) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->owner); + PyWeakref_GetRef(ssl_obj->owner, &ssl_socket); else if (ssl_obj->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->Socket); + PyWeakref_GetRef(ssl_obj->Socket, &ssl_socket); else ssl_socket = (PyObject *)Py_NewRef(ssl_obj); - assert(ssl_socket != NULL); // _PyWeakref_GET_REF() can return NULL + assert(ssl_socket != NULL); // PyWeakref_GetRef() can return NULL /* assume that OpenSSL verifies all payload and buf len is of sufficient length */ diff --git a/Modules/_ssl_data.h b/Modules/_ssl_data.h deleted file mode 100644 index 8f2994f52dfef8..00000000000000 --- a/Modules/_ssl_data.h +++ /dev/null @@ -1,6323 +0,0 @@ -/* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2020-04-13T21:45:54.559159 */ - -static struct py_ssl_library_code library_codes[] = { -#ifdef ERR_LIB_ASN1 - {"ASN1", ERR_LIB_ASN1}, -#endif -#ifdef ERR_LIB_ASYNC - {"ASYNC", ERR_LIB_ASYNC}, -#endif -#ifdef ERR_LIB_BIO - {"BIO", ERR_LIB_BIO}, -#endif -#ifdef ERR_LIB_BN - {"BN", ERR_LIB_BN}, -#endif -#ifdef ERR_LIB_CMS - {"CMS", ERR_LIB_CMS}, -#endif -#ifdef ERR_LIB_COMP - {"COMP", ERR_LIB_COMP}, -#endif -#ifdef ERR_LIB_CONF - {"CONF", ERR_LIB_CONF}, -#endif -#ifdef ERR_LIB_CRYPTO - {"CRYPTO", ERR_LIB_CRYPTO}, -#endif -#ifdef ERR_LIB_CT - {"CT", ERR_LIB_CT}, -#endif -#ifdef ERR_LIB_DH - {"DH", ERR_LIB_DH}, -#endif -#ifdef ERR_LIB_DSA - {"DSA", ERR_LIB_DSA}, -#endif -#ifdef ERR_LIB_EC - {"EC", ERR_LIB_EC}, -#endif -#ifdef ERR_LIB_ENGINE - {"ENGINE", ERR_LIB_ENGINE}, -#endif -#ifdef ERR_LIB_EVP - {"EVP", ERR_LIB_EVP}, -#endif -#ifdef ERR_LIB_KDF - {"KDF", ERR_LIB_KDF}, -#endif -#ifdef ERR_LIB_OCSP - {"OCSP", ERR_LIB_OCSP}, -#endif -#ifdef ERR_LIB_PEM - {"PEM", ERR_LIB_PEM}, -#endif -#ifdef ERR_LIB_PKCS12 - {"PKCS12", ERR_LIB_PKCS12}, -#endif -#ifdef ERR_LIB_PKCS7 - {"PKCS7", ERR_LIB_PKCS7}, -#endif -#ifdef ERR_LIB_RAND - {"RAND", ERR_LIB_RAND}, -#endif -#ifdef ERR_LIB_RSA - {"RSA", ERR_LIB_RSA}, -#endif -#ifdef ERR_LIB_SSL - {"SSL", ERR_LIB_SSL}, -#endif -#ifdef ERR_LIB_TS - {"TS", ERR_LIB_TS}, -#endif -#ifdef ERR_LIB_UI - {"UI", ERR_LIB_UI}, -#endif -#ifdef ERR_LIB_X509 - {"X509", ERR_LIB_X509}, -#endif -#ifdef ERR_LIB_X509V3 - {"X509V3", ERR_LIB_X509V3}, -#endif - { NULL } -}; - -static struct py_ssl_error_code error_codes[] = { - #ifdef ASN1_R_ADDING_OBJECT - {"ADDING_OBJECT", ERR_LIB_ASN1, ASN1_R_ADDING_OBJECT}, - #else - {"ADDING_OBJECT", 13, 171}, - #endif - #ifdef ASN1_R_ASN1_PARSE_ERROR - {"ASN1_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_ASN1_PARSE_ERROR}, - #else - {"ASN1_PARSE_ERROR", 13, 203}, - #endif - #ifdef ASN1_R_ASN1_SIG_PARSE_ERROR - {"ASN1_SIG_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_ASN1_SIG_PARSE_ERROR}, - #else - {"ASN1_SIG_PARSE_ERROR", 13, 204}, - #endif - #ifdef ASN1_R_AUX_ERROR - {"AUX_ERROR", ERR_LIB_ASN1, ASN1_R_AUX_ERROR}, - #else - {"AUX_ERROR", 13, 100}, - #endif - #ifdef ASN1_R_BAD_OBJECT_HEADER - {"BAD_OBJECT_HEADER", ERR_LIB_ASN1, ASN1_R_BAD_OBJECT_HEADER}, - #else - {"BAD_OBJECT_HEADER", 13, 102}, - #endif - #ifdef ASN1_R_BMPSTRING_IS_WRONG_LENGTH - {"BMPSTRING_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_BMPSTRING_IS_WRONG_LENGTH}, - #else - {"BMPSTRING_IS_WRONG_LENGTH", 13, 214}, - #endif - #ifdef ASN1_R_BN_LIB - {"BN_LIB", ERR_LIB_ASN1, ASN1_R_BN_LIB}, - #else - {"BN_LIB", 13, 105}, - #endif - #ifdef ASN1_R_BOOLEAN_IS_WRONG_LENGTH - {"BOOLEAN_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_BOOLEAN_IS_WRONG_LENGTH}, - #else - {"BOOLEAN_IS_WRONG_LENGTH", 13, 106}, - #endif - #ifdef ASN1_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_ASN1, ASN1_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 13, 107}, - #endif - #ifdef ASN1_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", ERR_LIB_ASN1, ASN1_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER}, - #else - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", 13, 108}, - #endif - #ifdef ASN1_R_CONTEXT_NOT_INITIALISED - {"CONTEXT_NOT_INITIALISED", ERR_LIB_ASN1, ASN1_R_CONTEXT_NOT_INITIALISED}, - #else - {"CONTEXT_NOT_INITIALISED", 13, 217}, - #endif - #ifdef ASN1_R_DATA_IS_WRONG - {"DATA_IS_WRONG", ERR_LIB_ASN1, ASN1_R_DATA_IS_WRONG}, - #else - {"DATA_IS_WRONG", 13, 109}, - #endif - #ifdef ASN1_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_ASN1, ASN1_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 13, 110}, - #endif - #ifdef ASN1_R_DEPTH_EXCEEDED - {"DEPTH_EXCEEDED", ERR_LIB_ASN1, ASN1_R_DEPTH_EXCEEDED}, - #else - {"DEPTH_EXCEEDED", 13, 174}, - #endif - #ifdef ASN1_R_DIGEST_AND_KEY_TYPE_NOT_SUPPORTED - {"DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", ERR_LIB_ASN1, ASN1_R_DIGEST_AND_KEY_TYPE_NOT_SUPPORTED}, - #else - {"DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", 13, 198}, - #endif - #ifdef ASN1_R_ENCODE_ERROR - {"ENCODE_ERROR", ERR_LIB_ASN1, ASN1_R_ENCODE_ERROR}, - #else - {"ENCODE_ERROR", 13, 112}, - #endif - #ifdef ASN1_R_ERROR_GETTING_TIME - {"ERROR_GETTING_TIME", ERR_LIB_ASN1, ASN1_R_ERROR_GETTING_TIME}, - #else - {"ERROR_GETTING_TIME", 13, 173}, - #endif - #ifdef ASN1_R_ERROR_LOADING_SECTION - {"ERROR_LOADING_SECTION", ERR_LIB_ASN1, ASN1_R_ERROR_LOADING_SECTION}, - #else - {"ERROR_LOADING_SECTION", 13, 172}, - #endif - #ifdef ASN1_R_ERROR_SETTING_CIPHER_PARAMS - {"ERROR_SETTING_CIPHER_PARAMS", ERR_LIB_ASN1, ASN1_R_ERROR_SETTING_CIPHER_PARAMS}, - #else - {"ERROR_SETTING_CIPHER_PARAMS", 13, 114}, - #endif - #ifdef ASN1_R_EXPECTING_AN_INTEGER - {"EXPECTING_AN_INTEGER", ERR_LIB_ASN1, ASN1_R_EXPECTING_AN_INTEGER}, - #else - {"EXPECTING_AN_INTEGER", 13, 115}, - #endif - #ifdef ASN1_R_EXPECTING_AN_OBJECT - {"EXPECTING_AN_OBJECT", ERR_LIB_ASN1, ASN1_R_EXPECTING_AN_OBJECT}, - #else - {"EXPECTING_AN_OBJECT", 13, 116}, - #endif - #ifdef ASN1_R_EXPLICIT_LENGTH_MISMATCH - {"EXPLICIT_LENGTH_MISMATCH", ERR_LIB_ASN1, ASN1_R_EXPLICIT_LENGTH_MISMATCH}, - #else - {"EXPLICIT_LENGTH_MISMATCH", 13, 119}, - #endif - #ifdef ASN1_R_EXPLICIT_TAG_NOT_CONSTRUCTED - {"EXPLICIT_TAG_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_EXPLICIT_TAG_NOT_CONSTRUCTED}, - #else - {"EXPLICIT_TAG_NOT_CONSTRUCTED", 13, 120}, - #endif - #ifdef ASN1_R_FIELD_MISSING - {"FIELD_MISSING", ERR_LIB_ASN1, ASN1_R_FIELD_MISSING}, - #else - {"FIELD_MISSING", 13, 121}, - #endif - #ifdef ASN1_R_FIRST_NUM_TOO_LARGE - {"FIRST_NUM_TOO_LARGE", ERR_LIB_ASN1, ASN1_R_FIRST_NUM_TOO_LARGE}, - #else - {"FIRST_NUM_TOO_LARGE", 13, 122}, - #endif - #ifdef ASN1_R_HEADER_TOO_LONG - {"HEADER_TOO_LONG", ERR_LIB_ASN1, ASN1_R_HEADER_TOO_LONG}, - #else - {"HEADER_TOO_LONG", 13, 123}, - #endif - #ifdef ASN1_R_ILLEGAL_BITSTRING_FORMAT - {"ILLEGAL_BITSTRING_FORMAT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_BITSTRING_FORMAT}, - #else - {"ILLEGAL_BITSTRING_FORMAT", 13, 175}, - #endif - #ifdef ASN1_R_ILLEGAL_BOOLEAN - {"ILLEGAL_BOOLEAN", ERR_LIB_ASN1, ASN1_R_ILLEGAL_BOOLEAN}, - #else - {"ILLEGAL_BOOLEAN", 13, 176}, - #endif - #ifdef ASN1_R_ILLEGAL_CHARACTERS - {"ILLEGAL_CHARACTERS", ERR_LIB_ASN1, ASN1_R_ILLEGAL_CHARACTERS}, - #else - {"ILLEGAL_CHARACTERS", 13, 124}, - #endif - #ifdef ASN1_R_ILLEGAL_FORMAT - {"ILLEGAL_FORMAT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_FORMAT}, - #else - {"ILLEGAL_FORMAT", 13, 177}, - #endif - #ifdef ASN1_R_ILLEGAL_HEX - {"ILLEGAL_HEX", ERR_LIB_ASN1, ASN1_R_ILLEGAL_HEX}, - #else - {"ILLEGAL_HEX", 13, 178}, - #endif - #ifdef ASN1_R_ILLEGAL_IMPLICIT_TAG - {"ILLEGAL_IMPLICIT_TAG", ERR_LIB_ASN1, ASN1_R_ILLEGAL_IMPLICIT_TAG}, - #else - {"ILLEGAL_IMPLICIT_TAG", 13, 179}, - #endif - #ifdef ASN1_R_ILLEGAL_INTEGER - {"ILLEGAL_INTEGER", ERR_LIB_ASN1, ASN1_R_ILLEGAL_INTEGER}, - #else - {"ILLEGAL_INTEGER", 13, 180}, - #endif - #ifdef ASN1_R_ILLEGAL_NEGATIVE_VALUE - {"ILLEGAL_NEGATIVE_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NEGATIVE_VALUE}, - #else - {"ILLEGAL_NEGATIVE_VALUE", 13, 226}, - #endif - #ifdef ASN1_R_ILLEGAL_NESTED_TAGGING - {"ILLEGAL_NESTED_TAGGING", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NESTED_TAGGING}, - #else - {"ILLEGAL_NESTED_TAGGING", 13, 181}, - #endif - #ifdef ASN1_R_ILLEGAL_NULL - {"ILLEGAL_NULL", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NULL}, - #else - {"ILLEGAL_NULL", 13, 125}, - #endif - #ifdef ASN1_R_ILLEGAL_NULL_VALUE - {"ILLEGAL_NULL_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_NULL_VALUE}, - #else - {"ILLEGAL_NULL_VALUE", 13, 182}, - #endif - #ifdef ASN1_R_ILLEGAL_OBJECT - {"ILLEGAL_OBJECT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OBJECT}, - #else - {"ILLEGAL_OBJECT", 13, 183}, - #endif - #ifdef ASN1_R_ILLEGAL_OPTIONAL_ANY - {"ILLEGAL_OPTIONAL_ANY", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OPTIONAL_ANY}, - #else - {"ILLEGAL_OPTIONAL_ANY", 13, 126}, - #endif - #ifdef ASN1_R_ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE - {"ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE}, - #else - {"ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", 13, 170}, - #endif - #ifdef ASN1_R_ILLEGAL_PADDING - {"ILLEGAL_PADDING", ERR_LIB_ASN1, ASN1_R_ILLEGAL_PADDING}, - #else - {"ILLEGAL_PADDING", 13, 221}, - #endif - #ifdef ASN1_R_ILLEGAL_TAGGED_ANY - {"ILLEGAL_TAGGED_ANY", ERR_LIB_ASN1, ASN1_R_ILLEGAL_TAGGED_ANY}, - #else - {"ILLEGAL_TAGGED_ANY", 13, 127}, - #endif - #ifdef ASN1_R_ILLEGAL_TIME_VALUE - {"ILLEGAL_TIME_VALUE", ERR_LIB_ASN1, ASN1_R_ILLEGAL_TIME_VALUE}, - #else - {"ILLEGAL_TIME_VALUE", 13, 184}, - #endif - #ifdef ASN1_R_ILLEGAL_ZERO_CONTENT - {"ILLEGAL_ZERO_CONTENT", ERR_LIB_ASN1, ASN1_R_ILLEGAL_ZERO_CONTENT}, - #else - {"ILLEGAL_ZERO_CONTENT", 13, 222}, - #endif - #ifdef ASN1_R_INTEGER_NOT_ASCII_FORMAT - {"INTEGER_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_INTEGER_NOT_ASCII_FORMAT}, - #else - {"INTEGER_NOT_ASCII_FORMAT", 13, 185}, - #endif - #ifdef ASN1_R_INTEGER_TOO_LARGE_FOR_LONG - {"INTEGER_TOO_LARGE_FOR_LONG", ERR_LIB_ASN1, ASN1_R_INTEGER_TOO_LARGE_FOR_LONG}, - #else - {"INTEGER_TOO_LARGE_FOR_LONG", 13, 128}, - #endif - #ifdef ASN1_R_INVALID_BIT_STRING_BITS_LEFT - {"INVALID_BIT_STRING_BITS_LEFT", ERR_LIB_ASN1, ASN1_R_INVALID_BIT_STRING_BITS_LEFT}, - #else - {"INVALID_BIT_STRING_BITS_LEFT", 13, 220}, - #endif - #ifdef ASN1_R_INVALID_BMPSTRING_LENGTH - {"INVALID_BMPSTRING_LENGTH", ERR_LIB_ASN1, ASN1_R_INVALID_BMPSTRING_LENGTH}, - #else - {"INVALID_BMPSTRING_LENGTH", 13, 129}, - #endif - #ifdef ASN1_R_INVALID_DIGIT - {"INVALID_DIGIT", ERR_LIB_ASN1, ASN1_R_INVALID_DIGIT}, - #else - {"INVALID_DIGIT", 13, 130}, - #endif - #ifdef ASN1_R_INVALID_MIME_TYPE - {"INVALID_MIME_TYPE", ERR_LIB_ASN1, ASN1_R_INVALID_MIME_TYPE}, - #else - {"INVALID_MIME_TYPE", 13, 205}, - #endif - #ifdef ASN1_R_INVALID_MODIFIER - {"INVALID_MODIFIER", ERR_LIB_ASN1, ASN1_R_INVALID_MODIFIER}, - #else - {"INVALID_MODIFIER", 13, 186}, - #endif - #ifdef ASN1_R_INVALID_NUMBER - {"INVALID_NUMBER", ERR_LIB_ASN1, ASN1_R_INVALID_NUMBER}, - #else - {"INVALID_NUMBER", 13, 187}, - #endif - #ifdef ASN1_R_INVALID_OBJECT_ENCODING - {"INVALID_OBJECT_ENCODING", ERR_LIB_ASN1, ASN1_R_INVALID_OBJECT_ENCODING}, - #else - {"INVALID_OBJECT_ENCODING", 13, 216}, - #endif - #ifdef ASN1_R_INVALID_SCRYPT_PARAMETERS - {"INVALID_SCRYPT_PARAMETERS", ERR_LIB_ASN1, ASN1_R_INVALID_SCRYPT_PARAMETERS}, - #else - {"INVALID_SCRYPT_PARAMETERS", 13, 227}, - #endif - #ifdef ASN1_R_INVALID_SEPARATOR - {"INVALID_SEPARATOR", ERR_LIB_ASN1, ASN1_R_INVALID_SEPARATOR}, - #else - {"INVALID_SEPARATOR", 13, 131}, - #endif - #ifdef ASN1_R_INVALID_STRING_TABLE_VALUE - {"INVALID_STRING_TABLE_VALUE", ERR_LIB_ASN1, ASN1_R_INVALID_STRING_TABLE_VALUE}, - #else - {"INVALID_STRING_TABLE_VALUE", 13, 218}, - #endif - #ifdef ASN1_R_INVALID_UNIVERSALSTRING_LENGTH - {"INVALID_UNIVERSALSTRING_LENGTH", ERR_LIB_ASN1, ASN1_R_INVALID_UNIVERSALSTRING_LENGTH}, - #else - {"INVALID_UNIVERSALSTRING_LENGTH", 13, 133}, - #endif - #ifdef ASN1_R_INVALID_UTF8STRING - {"INVALID_UTF8STRING", ERR_LIB_ASN1, ASN1_R_INVALID_UTF8STRING}, - #else - {"INVALID_UTF8STRING", 13, 134}, - #endif - #ifdef ASN1_R_INVALID_VALUE - {"INVALID_VALUE", ERR_LIB_ASN1, ASN1_R_INVALID_VALUE}, - #else - {"INVALID_VALUE", 13, 219}, - #endif - #ifdef ASN1_R_LIST_ERROR - {"LIST_ERROR", ERR_LIB_ASN1, ASN1_R_LIST_ERROR}, - #else - {"LIST_ERROR", 13, 188}, - #endif - #ifdef ASN1_R_MIME_NO_CONTENT_TYPE - {"MIME_NO_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_MIME_NO_CONTENT_TYPE}, - #else - {"MIME_NO_CONTENT_TYPE", 13, 206}, - #endif - #ifdef ASN1_R_MIME_PARSE_ERROR - {"MIME_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_MIME_PARSE_ERROR}, - #else - {"MIME_PARSE_ERROR", 13, 207}, - #endif - #ifdef ASN1_R_MIME_SIG_PARSE_ERROR - {"MIME_SIG_PARSE_ERROR", ERR_LIB_ASN1, ASN1_R_MIME_SIG_PARSE_ERROR}, - #else - {"MIME_SIG_PARSE_ERROR", 13, 208}, - #endif - #ifdef ASN1_R_MISSING_EOC - {"MISSING_EOC", ERR_LIB_ASN1, ASN1_R_MISSING_EOC}, - #else - {"MISSING_EOC", 13, 137}, - #endif - #ifdef ASN1_R_MISSING_SECOND_NUMBER - {"MISSING_SECOND_NUMBER", ERR_LIB_ASN1, ASN1_R_MISSING_SECOND_NUMBER}, - #else - {"MISSING_SECOND_NUMBER", 13, 138}, - #endif - #ifdef ASN1_R_MISSING_VALUE - {"MISSING_VALUE", ERR_LIB_ASN1, ASN1_R_MISSING_VALUE}, - #else - {"MISSING_VALUE", 13, 189}, - #endif - #ifdef ASN1_R_MSTRING_NOT_UNIVERSAL - {"MSTRING_NOT_UNIVERSAL", ERR_LIB_ASN1, ASN1_R_MSTRING_NOT_UNIVERSAL}, - #else - {"MSTRING_NOT_UNIVERSAL", 13, 139}, - #endif - #ifdef ASN1_R_MSTRING_WRONG_TAG - {"MSTRING_WRONG_TAG", ERR_LIB_ASN1, ASN1_R_MSTRING_WRONG_TAG}, - #else - {"MSTRING_WRONG_TAG", 13, 140}, - #endif - #ifdef ASN1_R_NESTED_ASN1_STRING - {"NESTED_ASN1_STRING", ERR_LIB_ASN1, ASN1_R_NESTED_ASN1_STRING}, - #else - {"NESTED_ASN1_STRING", 13, 197}, - #endif - #ifdef ASN1_R_NESTED_TOO_DEEP - {"NESTED_TOO_DEEP", ERR_LIB_ASN1, ASN1_R_NESTED_TOO_DEEP}, - #else - {"NESTED_TOO_DEEP", 13, 201}, - #endif - #ifdef ASN1_R_NON_HEX_CHARACTERS - {"NON_HEX_CHARACTERS", ERR_LIB_ASN1, ASN1_R_NON_HEX_CHARACTERS}, - #else - {"NON_HEX_CHARACTERS", 13, 141}, - #endif - #ifdef ASN1_R_NOT_ASCII_FORMAT - {"NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_NOT_ASCII_FORMAT}, - #else - {"NOT_ASCII_FORMAT", 13, 190}, - #endif - #ifdef ASN1_R_NOT_ENOUGH_DATA - {"NOT_ENOUGH_DATA", ERR_LIB_ASN1, ASN1_R_NOT_ENOUGH_DATA}, - #else - {"NOT_ENOUGH_DATA", 13, 142}, - #endif - #ifdef ASN1_R_NO_CONTENT_TYPE - {"NO_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_NO_CONTENT_TYPE}, - #else - {"NO_CONTENT_TYPE", 13, 209}, - #endif - #ifdef ASN1_R_NO_MATCHING_CHOICE_TYPE - {"NO_MATCHING_CHOICE_TYPE", ERR_LIB_ASN1, ASN1_R_NO_MATCHING_CHOICE_TYPE}, - #else - {"NO_MATCHING_CHOICE_TYPE", 13, 143}, - #endif - #ifdef ASN1_R_NO_MULTIPART_BODY_FAILURE - {"NO_MULTIPART_BODY_FAILURE", ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BODY_FAILURE}, - #else - {"NO_MULTIPART_BODY_FAILURE", 13, 210}, - #endif - #ifdef ASN1_R_NO_MULTIPART_BOUNDARY - {"NO_MULTIPART_BOUNDARY", ERR_LIB_ASN1, ASN1_R_NO_MULTIPART_BOUNDARY}, - #else - {"NO_MULTIPART_BOUNDARY", 13, 211}, - #endif - #ifdef ASN1_R_NO_SIG_CONTENT_TYPE - {"NO_SIG_CONTENT_TYPE", ERR_LIB_ASN1, ASN1_R_NO_SIG_CONTENT_TYPE}, - #else - {"NO_SIG_CONTENT_TYPE", 13, 212}, - #endif - #ifdef ASN1_R_NULL_IS_WRONG_LENGTH - {"NULL_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_NULL_IS_WRONG_LENGTH}, - #else - {"NULL_IS_WRONG_LENGTH", 13, 144}, - #endif - #ifdef ASN1_R_OBJECT_NOT_ASCII_FORMAT - {"OBJECT_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_OBJECT_NOT_ASCII_FORMAT}, - #else - {"OBJECT_NOT_ASCII_FORMAT", 13, 191}, - #endif - #ifdef ASN1_R_ODD_NUMBER_OF_CHARS - {"ODD_NUMBER_OF_CHARS", ERR_LIB_ASN1, ASN1_R_ODD_NUMBER_OF_CHARS}, - #else - {"ODD_NUMBER_OF_CHARS", 13, 145}, - #endif - #ifdef ASN1_R_SECOND_NUMBER_TOO_LARGE - {"SECOND_NUMBER_TOO_LARGE", ERR_LIB_ASN1, ASN1_R_SECOND_NUMBER_TOO_LARGE}, - #else - {"SECOND_NUMBER_TOO_LARGE", 13, 147}, - #endif - #ifdef ASN1_R_SEQUENCE_LENGTH_MISMATCH - {"SEQUENCE_LENGTH_MISMATCH", ERR_LIB_ASN1, ASN1_R_SEQUENCE_LENGTH_MISMATCH}, - #else - {"SEQUENCE_LENGTH_MISMATCH", 13, 148}, - #endif - #ifdef ASN1_R_SEQUENCE_NOT_CONSTRUCTED - {"SEQUENCE_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_SEQUENCE_NOT_CONSTRUCTED}, - #else - {"SEQUENCE_NOT_CONSTRUCTED", 13, 149}, - #endif - #ifdef ASN1_R_SEQUENCE_OR_SET_NEEDS_CONFIG - {"SEQUENCE_OR_SET_NEEDS_CONFIG", ERR_LIB_ASN1, ASN1_R_SEQUENCE_OR_SET_NEEDS_CONFIG}, - #else - {"SEQUENCE_OR_SET_NEEDS_CONFIG", 13, 192}, - #endif - #ifdef ASN1_R_SHORT_LINE - {"SHORT_LINE", ERR_LIB_ASN1, ASN1_R_SHORT_LINE}, - #else - {"SHORT_LINE", 13, 150}, - #endif - #ifdef ASN1_R_SIG_INVALID_MIME_TYPE - {"SIG_INVALID_MIME_TYPE", ERR_LIB_ASN1, ASN1_R_SIG_INVALID_MIME_TYPE}, - #else - {"SIG_INVALID_MIME_TYPE", 13, 213}, - #endif - #ifdef ASN1_R_STREAMING_NOT_SUPPORTED - {"STREAMING_NOT_SUPPORTED", ERR_LIB_ASN1, ASN1_R_STREAMING_NOT_SUPPORTED}, - #else - {"STREAMING_NOT_SUPPORTED", 13, 202}, - #endif - #ifdef ASN1_R_STRING_TOO_LONG - {"STRING_TOO_LONG", ERR_LIB_ASN1, ASN1_R_STRING_TOO_LONG}, - #else - {"STRING_TOO_LONG", 13, 151}, - #endif - #ifdef ASN1_R_STRING_TOO_SHORT - {"STRING_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_STRING_TOO_SHORT}, - #else - {"STRING_TOO_SHORT", 13, 152}, - #endif - #ifdef ASN1_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", ERR_LIB_ASN1, ASN1_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD}, - #else - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", 13, 154}, - #endif - #ifdef ASN1_R_TIME_NOT_ASCII_FORMAT - {"TIME_NOT_ASCII_FORMAT", ERR_LIB_ASN1, ASN1_R_TIME_NOT_ASCII_FORMAT}, - #else - {"TIME_NOT_ASCII_FORMAT", 13, 193}, - #endif - #ifdef ASN1_R_TOO_LARGE - {"TOO_LARGE", ERR_LIB_ASN1, ASN1_R_TOO_LARGE}, - #else - {"TOO_LARGE", 13, 223}, - #endif - #ifdef ASN1_R_TOO_LONG - {"TOO_LONG", ERR_LIB_ASN1, ASN1_R_TOO_LONG}, - #else - {"TOO_LONG", 13, 155}, - #endif - #ifdef ASN1_R_TOO_SMALL - {"TOO_SMALL", ERR_LIB_ASN1, ASN1_R_TOO_SMALL}, - #else - {"TOO_SMALL", 13, 224}, - #endif - #ifdef ASN1_R_TYPE_NOT_CONSTRUCTED - {"TYPE_NOT_CONSTRUCTED", ERR_LIB_ASN1, ASN1_R_TYPE_NOT_CONSTRUCTED}, - #else - {"TYPE_NOT_CONSTRUCTED", 13, 156}, - #endif - #ifdef ASN1_R_TYPE_NOT_PRIMITIVE - {"TYPE_NOT_PRIMITIVE", ERR_LIB_ASN1, ASN1_R_TYPE_NOT_PRIMITIVE}, - #else - {"TYPE_NOT_PRIMITIVE", 13, 195}, - #endif - #ifdef ASN1_R_UNEXPECTED_EOC - {"UNEXPECTED_EOC", ERR_LIB_ASN1, ASN1_R_UNEXPECTED_EOC}, - #else - {"UNEXPECTED_EOC", 13, 159}, - #endif - #ifdef ASN1_R_UNIVERSALSTRING_IS_WRONG_LENGTH - {"UNIVERSALSTRING_IS_WRONG_LENGTH", ERR_LIB_ASN1, ASN1_R_UNIVERSALSTRING_IS_WRONG_LENGTH}, - #else - {"UNIVERSALSTRING_IS_WRONG_LENGTH", 13, 215}, - #endif - #ifdef ASN1_R_UNKNOWN_FORMAT - {"UNKNOWN_FORMAT", ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT}, - #else - {"UNKNOWN_FORMAT", 13, 160}, - #endif - #ifdef ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM - {"UNKNOWN_MESSAGE_DIGEST_ALGORITHM", ERR_LIB_ASN1, ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_MESSAGE_DIGEST_ALGORITHM", 13, 161}, - #endif - #ifdef ASN1_R_UNKNOWN_OBJECT_TYPE - {"UNKNOWN_OBJECT_TYPE", ERR_LIB_ASN1, ASN1_R_UNKNOWN_OBJECT_TYPE}, - #else - {"UNKNOWN_OBJECT_TYPE", 13, 162}, - #endif - #ifdef ASN1_R_UNKNOWN_PUBLIC_KEY_TYPE - {"UNKNOWN_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_UNKNOWN_PUBLIC_KEY_TYPE}, - #else - {"UNKNOWN_PUBLIC_KEY_TYPE", 13, 163}, - #endif - #ifdef ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM - {"UNKNOWN_SIGNATURE_ALGORITHM", ERR_LIB_ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM}, - #else - {"UNKNOWN_SIGNATURE_ALGORITHM", 13, 199}, - #endif - #ifdef ASN1_R_UNKNOWN_TAG - {"UNKNOWN_TAG", ERR_LIB_ASN1, ASN1_R_UNKNOWN_TAG}, - #else - {"UNKNOWN_TAG", 13, 194}, - #endif - #ifdef ASN1_R_UNSUPPORTED_ANY_DEFINED_BY_TYPE - {"UNSUPPORTED_ANY_DEFINED_BY_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_ANY_DEFINED_BY_TYPE}, - #else - {"UNSUPPORTED_ANY_DEFINED_BY_TYPE", 13, 164}, - #endif - #ifdef ASN1_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 13, 228}, - #endif - #ifdef ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE - {"UNSUPPORTED_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE}, - #else - {"UNSUPPORTED_PUBLIC_KEY_TYPE", 13, 167}, - #endif - #ifdef ASN1_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 13, 196}, - #endif - #ifdef ASN1_R_WRONG_INTEGER_TYPE - {"WRONG_INTEGER_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_INTEGER_TYPE}, - #else - {"WRONG_INTEGER_TYPE", 13, 225}, - #endif - #ifdef ASN1_R_WRONG_PUBLIC_KEY_TYPE - {"WRONG_PUBLIC_KEY_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE}, - #else - {"WRONG_PUBLIC_KEY_TYPE", 13, 200}, - #endif - #ifdef ASN1_R_WRONG_TAG - {"WRONG_TAG", ERR_LIB_ASN1, ASN1_R_WRONG_TAG}, - #else - {"WRONG_TAG", 13, 168}, - #endif - #ifdef ASYNC_R_FAILED_TO_SET_POOL - {"FAILED_TO_SET_POOL", ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SET_POOL}, - #else - {"FAILED_TO_SET_POOL", 51, 101}, - #endif - #ifdef ASYNC_R_FAILED_TO_SWAP_CONTEXT - {"FAILED_TO_SWAP_CONTEXT", ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SWAP_CONTEXT}, - #else - {"FAILED_TO_SWAP_CONTEXT", 51, 102}, - #endif - #ifdef ASYNC_R_INIT_FAILED - {"INIT_FAILED", ERR_LIB_ASYNC, ASYNC_R_INIT_FAILED}, - #else - {"INIT_FAILED", 51, 105}, - #endif - #ifdef ASYNC_R_INVALID_POOL_SIZE - {"INVALID_POOL_SIZE", ERR_LIB_ASYNC, ASYNC_R_INVALID_POOL_SIZE}, - #else - {"INVALID_POOL_SIZE", 51, 103}, - #endif - #ifdef BIO_R_ACCEPT_ERROR - {"ACCEPT_ERROR", ERR_LIB_BIO, BIO_R_ACCEPT_ERROR}, - #else - {"ACCEPT_ERROR", 32, 100}, - #endif - #ifdef BIO_R_ADDRINFO_ADDR_IS_NOT_AF_INET - {"ADDRINFO_ADDR_IS_NOT_AF_INET", ERR_LIB_BIO, BIO_R_ADDRINFO_ADDR_IS_NOT_AF_INET}, - #else - {"ADDRINFO_ADDR_IS_NOT_AF_INET", 32, 141}, - #endif - #ifdef BIO_R_AMBIGUOUS_HOST_OR_SERVICE - {"AMBIGUOUS_HOST_OR_SERVICE", ERR_LIB_BIO, BIO_R_AMBIGUOUS_HOST_OR_SERVICE}, - #else - {"AMBIGUOUS_HOST_OR_SERVICE", 32, 129}, - #endif - #ifdef BIO_R_BAD_FOPEN_MODE - {"BAD_FOPEN_MODE", ERR_LIB_BIO, BIO_R_BAD_FOPEN_MODE}, - #else - {"BAD_FOPEN_MODE", 32, 101}, - #endif - #ifdef BIO_R_BROKEN_PIPE - {"BROKEN_PIPE", ERR_LIB_BIO, BIO_R_BROKEN_PIPE}, - #else - {"BROKEN_PIPE", 32, 124}, - #endif - #ifdef BIO_R_CONNECT_ERROR - {"CONNECT_ERROR", ERR_LIB_BIO, BIO_R_CONNECT_ERROR}, - #else - {"CONNECT_ERROR", 32, 103}, - #endif - #ifdef BIO_R_GETHOSTBYNAME_ADDR_IS_NOT_AF_INET - {"GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", ERR_LIB_BIO, BIO_R_GETHOSTBYNAME_ADDR_IS_NOT_AF_INET}, - #else - {"GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", 32, 107}, - #endif - #ifdef BIO_R_GETSOCKNAME_ERROR - {"GETSOCKNAME_ERROR", ERR_LIB_BIO, BIO_R_GETSOCKNAME_ERROR}, - #else - {"GETSOCKNAME_ERROR", 32, 132}, - #endif - #ifdef BIO_R_GETSOCKNAME_TRUNCATED_ADDRESS - {"GETSOCKNAME_TRUNCATED_ADDRESS", ERR_LIB_BIO, BIO_R_GETSOCKNAME_TRUNCATED_ADDRESS}, - #else - {"GETSOCKNAME_TRUNCATED_ADDRESS", 32, 133}, - #endif - #ifdef BIO_R_GETTING_SOCKTYPE - {"GETTING_SOCKTYPE", ERR_LIB_BIO, BIO_R_GETTING_SOCKTYPE}, - #else - {"GETTING_SOCKTYPE", 32, 134}, - #endif - #ifdef BIO_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 32, 125}, - #endif - #ifdef BIO_R_INVALID_SOCKET - {"INVALID_SOCKET", ERR_LIB_BIO, BIO_R_INVALID_SOCKET}, - #else - {"INVALID_SOCKET", 32, 135}, - #endif - #ifdef BIO_R_IN_USE - {"IN_USE", ERR_LIB_BIO, BIO_R_IN_USE}, - #else - {"IN_USE", 32, 123}, - #endif - #ifdef BIO_R_LENGTH_TOO_LONG - {"LENGTH_TOO_LONG", ERR_LIB_BIO, BIO_R_LENGTH_TOO_LONG}, - #else - {"LENGTH_TOO_LONG", 32, 102}, - #endif - #ifdef BIO_R_LISTEN_V6_ONLY - {"LISTEN_V6_ONLY", ERR_LIB_BIO, BIO_R_LISTEN_V6_ONLY}, - #else - {"LISTEN_V6_ONLY", 32, 136}, - #endif - #ifdef BIO_R_LOOKUP_RETURNED_NOTHING - {"LOOKUP_RETURNED_NOTHING", ERR_LIB_BIO, BIO_R_LOOKUP_RETURNED_NOTHING}, - #else - {"LOOKUP_RETURNED_NOTHING", 32, 142}, - #endif - #ifdef BIO_R_MALFORMED_HOST_OR_SERVICE - {"MALFORMED_HOST_OR_SERVICE", ERR_LIB_BIO, BIO_R_MALFORMED_HOST_OR_SERVICE}, - #else - {"MALFORMED_HOST_OR_SERVICE", 32, 130}, - #endif - #ifdef BIO_R_NBIO_CONNECT_ERROR - {"NBIO_CONNECT_ERROR", ERR_LIB_BIO, BIO_R_NBIO_CONNECT_ERROR}, - #else - {"NBIO_CONNECT_ERROR", 32, 110}, - #endif - #ifdef BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED - {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED}, - #else - {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", 32, 143}, - #endif - #ifdef BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED - {"NO_HOSTNAME_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED}, - #else - {"NO_HOSTNAME_OR_SERVICE_SPECIFIED", 32, 144}, - #endif - #ifdef BIO_R_NO_PORT_DEFINED - {"NO_PORT_DEFINED", ERR_LIB_BIO, BIO_R_NO_PORT_DEFINED}, - #else - {"NO_PORT_DEFINED", 32, 113}, - #endif - #ifdef BIO_R_NO_SUCH_FILE - {"NO_SUCH_FILE", ERR_LIB_BIO, BIO_R_NO_SUCH_FILE}, - #else - {"NO_SUCH_FILE", 32, 128}, - #endif - #ifdef BIO_R_NULL_PARAMETER - {"NULL_PARAMETER", ERR_LIB_BIO, BIO_R_NULL_PARAMETER}, - #else - {"NULL_PARAMETER", 32, 115}, - #endif - #ifdef BIO_R_UNABLE_TO_BIND_SOCKET - {"UNABLE_TO_BIND_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_BIND_SOCKET}, - #else - {"UNABLE_TO_BIND_SOCKET", 32, 117}, - #endif - #ifdef BIO_R_UNABLE_TO_CREATE_SOCKET - {"UNABLE_TO_CREATE_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_CREATE_SOCKET}, - #else - {"UNABLE_TO_CREATE_SOCKET", 32, 118}, - #endif - #ifdef BIO_R_UNABLE_TO_KEEPALIVE - {"UNABLE_TO_KEEPALIVE", ERR_LIB_BIO, BIO_R_UNABLE_TO_KEEPALIVE}, - #else - {"UNABLE_TO_KEEPALIVE", 32, 137}, - #endif - #ifdef BIO_R_UNABLE_TO_LISTEN_SOCKET - {"UNABLE_TO_LISTEN_SOCKET", ERR_LIB_BIO, BIO_R_UNABLE_TO_LISTEN_SOCKET}, - #else - {"UNABLE_TO_LISTEN_SOCKET", 32, 119}, - #endif - #ifdef BIO_R_UNABLE_TO_NODELAY - {"UNABLE_TO_NODELAY", ERR_LIB_BIO, BIO_R_UNABLE_TO_NODELAY}, - #else - {"UNABLE_TO_NODELAY", 32, 138}, - #endif - #ifdef BIO_R_UNABLE_TO_REUSEADDR - {"UNABLE_TO_REUSEADDR", ERR_LIB_BIO, BIO_R_UNABLE_TO_REUSEADDR}, - #else - {"UNABLE_TO_REUSEADDR", 32, 139}, - #endif - #ifdef BIO_R_UNAVAILABLE_IP_FAMILY - {"UNAVAILABLE_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNAVAILABLE_IP_FAMILY}, - #else - {"UNAVAILABLE_IP_FAMILY", 32, 145}, - #endif - #ifdef BIO_R_UNINITIALIZED - {"UNINITIALIZED", ERR_LIB_BIO, BIO_R_UNINITIALIZED}, - #else - {"UNINITIALIZED", 32, 120}, - #endif - #ifdef BIO_R_UNKNOWN_INFO_TYPE - {"UNKNOWN_INFO_TYPE", ERR_LIB_BIO, BIO_R_UNKNOWN_INFO_TYPE}, - #else - {"UNKNOWN_INFO_TYPE", 32, 140}, - #endif - #ifdef BIO_R_UNSUPPORTED_IP_FAMILY - {"UNSUPPORTED_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNSUPPORTED_IP_FAMILY}, - #else - {"UNSUPPORTED_IP_FAMILY", 32, 146}, - #endif - #ifdef BIO_R_UNSUPPORTED_METHOD - {"UNSUPPORTED_METHOD", ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD}, - #else - {"UNSUPPORTED_METHOD", 32, 121}, - #endif - #ifdef BIO_R_UNSUPPORTED_PROTOCOL_FAMILY - {"UNSUPPORTED_PROTOCOL_FAMILY", ERR_LIB_BIO, BIO_R_UNSUPPORTED_PROTOCOL_FAMILY}, - #else - {"UNSUPPORTED_PROTOCOL_FAMILY", 32, 131}, - #endif - #ifdef BIO_R_WRITE_TO_READ_ONLY_BIO - {"WRITE_TO_READ_ONLY_BIO", ERR_LIB_BIO, BIO_R_WRITE_TO_READ_ONLY_BIO}, - #else - {"WRITE_TO_READ_ONLY_BIO", 32, 126}, - #endif - #ifdef BIO_R_WSASTARTUP - {"WSASTARTUP", ERR_LIB_BIO, BIO_R_WSASTARTUP}, - #else - {"WSASTARTUP", 32, 122}, - #endif - #ifdef BN_R_ARG2_LT_ARG3 - {"ARG2_LT_ARG3", ERR_LIB_BN, BN_R_ARG2_LT_ARG3}, - #else - {"ARG2_LT_ARG3", 3, 100}, - #endif - #ifdef BN_R_BAD_RECIPROCAL - {"BAD_RECIPROCAL", ERR_LIB_BN, BN_R_BAD_RECIPROCAL}, - #else - {"BAD_RECIPROCAL", 3, 101}, - #endif - #ifdef BN_R_BIGNUM_TOO_LONG - {"BIGNUM_TOO_LONG", ERR_LIB_BN, BN_R_BIGNUM_TOO_LONG}, - #else - {"BIGNUM_TOO_LONG", 3, 114}, - #endif - #ifdef BN_R_BITS_TOO_SMALL - {"BITS_TOO_SMALL", ERR_LIB_BN, BN_R_BITS_TOO_SMALL}, - #else - {"BITS_TOO_SMALL", 3, 118}, - #endif - #ifdef BN_R_CALLED_WITH_EVEN_MODULUS - {"CALLED_WITH_EVEN_MODULUS", ERR_LIB_BN, BN_R_CALLED_WITH_EVEN_MODULUS}, - #else - {"CALLED_WITH_EVEN_MODULUS", 3, 102}, - #endif - #ifdef BN_R_DIV_BY_ZERO - {"DIV_BY_ZERO", ERR_LIB_BN, BN_R_DIV_BY_ZERO}, - #else - {"DIV_BY_ZERO", 3, 103}, - #endif - #ifdef BN_R_ENCODING_ERROR - {"ENCODING_ERROR", ERR_LIB_BN, BN_R_ENCODING_ERROR}, - #else - {"ENCODING_ERROR", 3, 104}, - #endif - #ifdef BN_R_EXPAND_ON_STATIC_BIGNUM_DATA - {"EXPAND_ON_STATIC_BIGNUM_DATA", ERR_LIB_BN, BN_R_EXPAND_ON_STATIC_BIGNUM_DATA}, - #else - {"EXPAND_ON_STATIC_BIGNUM_DATA", 3, 105}, - #endif - #ifdef BN_R_INPUT_NOT_REDUCED - {"INPUT_NOT_REDUCED", ERR_LIB_BN, BN_R_INPUT_NOT_REDUCED}, - #else - {"INPUT_NOT_REDUCED", 3, 110}, - #endif - #ifdef BN_R_INVALID_LENGTH - {"INVALID_LENGTH", ERR_LIB_BN, BN_R_INVALID_LENGTH}, - #else - {"INVALID_LENGTH", 3, 106}, - #endif - #ifdef BN_R_INVALID_RANGE - {"INVALID_RANGE", ERR_LIB_BN, BN_R_INVALID_RANGE}, - #else - {"INVALID_RANGE", 3, 115}, - #endif - #ifdef BN_R_INVALID_SHIFT - {"INVALID_SHIFT", ERR_LIB_BN, BN_R_INVALID_SHIFT}, - #else - {"INVALID_SHIFT", 3, 119}, - #endif - #ifdef BN_R_NOT_A_SQUARE - {"NOT_A_SQUARE", ERR_LIB_BN, BN_R_NOT_A_SQUARE}, - #else - {"NOT_A_SQUARE", 3, 111}, - #endif - #ifdef BN_R_NOT_INITIALIZED - {"NOT_INITIALIZED", ERR_LIB_BN, BN_R_NOT_INITIALIZED}, - #else - {"NOT_INITIALIZED", 3, 107}, - #endif - #ifdef BN_R_NO_INVERSE - {"NO_INVERSE", ERR_LIB_BN, BN_R_NO_INVERSE}, - #else - {"NO_INVERSE", 3, 108}, - #endif - #ifdef BN_R_NO_SOLUTION - {"NO_SOLUTION", ERR_LIB_BN, BN_R_NO_SOLUTION}, - #else - {"NO_SOLUTION", 3, 116}, - #endif - #ifdef BN_R_PRIVATE_KEY_TOO_LARGE - {"PRIVATE_KEY_TOO_LARGE", ERR_LIB_BN, BN_R_PRIVATE_KEY_TOO_LARGE}, - #else - {"PRIVATE_KEY_TOO_LARGE", 3, 117}, - #endif - #ifdef BN_R_P_IS_NOT_PRIME - {"P_IS_NOT_PRIME", ERR_LIB_BN, BN_R_P_IS_NOT_PRIME}, - #else - {"P_IS_NOT_PRIME", 3, 112}, - #endif - #ifdef BN_R_TOO_MANY_ITERATIONS - {"TOO_MANY_ITERATIONS", ERR_LIB_BN, BN_R_TOO_MANY_ITERATIONS}, - #else - {"TOO_MANY_ITERATIONS", 3, 113}, - #endif - #ifdef BN_R_TOO_MANY_TEMPORARY_VARIABLES - {"TOO_MANY_TEMPORARY_VARIABLES", ERR_LIB_BN, BN_R_TOO_MANY_TEMPORARY_VARIABLES}, - #else - {"TOO_MANY_TEMPORARY_VARIABLES", 3, 109}, - #endif - #ifdef CMS_R_ADD_SIGNER_ERROR - {"ADD_SIGNER_ERROR", ERR_LIB_CMS, CMS_R_ADD_SIGNER_ERROR}, - #else - {"ADD_SIGNER_ERROR", 46, 99}, - #endif - #ifdef CMS_R_ATTRIBUTE_ERROR - {"ATTRIBUTE_ERROR", ERR_LIB_CMS, CMS_R_ATTRIBUTE_ERROR}, - #else - {"ATTRIBUTE_ERROR", 46, 161}, - #endif - #ifdef CMS_R_CERTIFICATE_ALREADY_PRESENT - {"CERTIFICATE_ALREADY_PRESENT", ERR_LIB_CMS, CMS_R_CERTIFICATE_ALREADY_PRESENT}, - #else - {"CERTIFICATE_ALREADY_PRESENT", 46, 175}, - #endif - #ifdef CMS_R_CERTIFICATE_HAS_NO_KEYID - {"CERTIFICATE_HAS_NO_KEYID", ERR_LIB_CMS, CMS_R_CERTIFICATE_HAS_NO_KEYID}, - #else - {"CERTIFICATE_HAS_NO_KEYID", 46, 160}, - #endif - #ifdef CMS_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_CMS, CMS_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 46, 100}, - #endif - #ifdef CMS_R_CIPHER_INITIALISATION_ERROR - {"CIPHER_INITIALISATION_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_INITIALISATION_ERROR}, - #else - {"CIPHER_INITIALISATION_ERROR", 46, 101}, - #endif - #ifdef CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR - {"CIPHER_PARAMETER_INITIALISATION_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR}, - #else - {"CIPHER_PARAMETER_INITIALISATION_ERROR", 46, 102}, - #endif - #ifdef CMS_R_CMS_DATAFINAL_ERROR - {"CMS_DATAFINAL_ERROR", ERR_LIB_CMS, CMS_R_CMS_DATAFINAL_ERROR}, - #else - {"CMS_DATAFINAL_ERROR", 46, 103}, - #endif - #ifdef CMS_R_CMS_LIB - {"CMS_LIB", ERR_LIB_CMS, CMS_R_CMS_LIB}, - #else - {"CMS_LIB", 46, 104}, - #endif - #ifdef CMS_R_CONTENTIDENTIFIER_MISMATCH - {"CONTENTIDENTIFIER_MISMATCH", ERR_LIB_CMS, CMS_R_CONTENTIDENTIFIER_MISMATCH}, - #else - {"CONTENTIDENTIFIER_MISMATCH", 46, 170}, - #endif - #ifdef CMS_R_CONTENT_NOT_FOUND - {"CONTENT_NOT_FOUND", ERR_LIB_CMS, CMS_R_CONTENT_NOT_FOUND}, - #else - {"CONTENT_NOT_FOUND", 46, 105}, - #endif - #ifdef CMS_R_CONTENT_TYPE_MISMATCH - {"CONTENT_TYPE_MISMATCH", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_MISMATCH}, - #else - {"CONTENT_TYPE_MISMATCH", 46, 171}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_COMPRESSED_DATA - {"CONTENT_TYPE_NOT_COMPRESSED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_COMPRESSED_DATA}, - #else - {"CONTENT_TYPE_NOT_COMPRESSED_DATA", 46, 106}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA - {"CONTENT_TYPE_NOT_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA}, - #else - {"CONTENT_TYPE_NOT_ENVELOPED_DATA", 46, 107}, - #endif - #ifdef CMS_R_CONTENT_TYPE_NOT_SIGNED_DATA - {"CONTENT_TYPE_NOT_SIGNED_DATA", ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_SIGNED_DATA}, - #else - {"CONTENT_TYPE_NOT_SIGNED_DATA", 46, 108}, - #endif - #ifdef CMS_R_CONTENT_VERIFY_ERROR - {"CONTENT_VERIFY_ERROR", ERR_LIB_CMS, CMS_R_CONTENT_VERIFY_ERROR}, - #else - {"CONTENT_VERIFY_ERROR", 46, 109}, - #endif - #ifdef CMS_R_CTRL_ERROR - {"CTRL_ERROR", ERR_LIB_CMS, CMS_R_CTRL_ERROR}, - #else - {"CTRL_ERROR", 46, 110}, - #endif - #ifdef CMS_R_CTRL_FAILURE - {"CTRL_FAILURE", ERR_LIB_CMS, CMS_R_CTRL_FAILURE}, - #else - {"CTRL_FAILURE", 46, 111}, - #endif - #ifdef CMS_R_DECRYPT_ERROR - {"DECRYPT_ERROR", ERR_LIB_CMS, CMS_R_DECRYPT_ERROR}, - #else - {"DECRYPT_ERROR", 46, 112}, - #endif - #ifdef CMS_R_ERROR_GETTING_PUBLIC_KEY - {"ERROR_GETTING_PUBLIC_KEY", ERR_LIB_CMS, CMS_R_ERROR_GETTING_PUBLIC_KEY}, - #else - {"ERROR_GETTING_PUBLIC_KEY", 46, 113}, - #endif - #ifdef CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE - {"ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", ERR_LIB_CMS, CMS_R_ERROR_READING_MESSAGEDIGEST_ATTRIBUTE}, - #else - {"ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", 46, 114}, - #endif - #ifdef CMS_R_ERROR_SETTING_KEY - {"ERROR_SETTING_KEY", ERR_LIB_CMS, CMS_R_ERROR_SETTING_KEY}, - #else - {"ERROR_SETTING_KEY", 46, 115}, - #endif - #ifdef CMS_R_ERROR_SETTING_RECIPIENTINFO - {"ERROR_SETTING_RECIPIENTINFO", ERR_LIB_CMS, CMS_R_ERROR_SETTING_RECIPIENTINFO}, - #else - {"ERROR_SETTING_RECIPIENTINFO", 46, 116}, - #endif - #ifdef CMS_R_INVALID_ENCRYPTED_KEY_LENGTH - {"INVALID_ENCRYPTED_KEY_LENGTH", ERR_LIB_CMS, CMS_R_INVALID_ENCRYPTED_KEY_LENGTH}, - #else - {"INVALID_ENCRYPTED_KEY_LENGTH", 46, 117}, - #endif - #ifdef CMS_R_INVALID_KEY_ENCRYPTION_PARAMETER - {"INVALID_KEY_ENCRYPTION_PARAMETER", ERR_LIB_CMS, CMS_R_INVALID_KEY_ENCRYPTION_PARAMETER}, - #else - {"INVALID_KEY_ENCRYPTION_PARAMETER", 46, 176}, - #endif - #ifdef CMS_R_INVALID_KEY_LENGTH - {"INVALID_KEY_LENGTH", ERR_LIB_CMS, CMS_R_INVALID_KEY_LENGTH}, - #else - {"INVALID_KEY_LENGTH", 46, 118}, - #endif - #ifdef CMS_R_MD_BIO_INIT_ERROR - {"MD_BIO_INIT_ERROR", ERR_LIB_CMS, CMS_R_MD_BIO_INIT_ERROR}, - #else - {"MD_BIO_INIT_ERROR", 46, 119}, - #endif - #ifdef CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH - {"MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH}, - #else - {"MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", 46, 120}, - #endif - #ifdef CMS_R_MESSAGEDIGEST_WRONG_LENGTH - {"MESSAGEDIGEST_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_WRONG_LENGTH}, - #else - {"MESSAGEDIGEST_WRONG_LENGTH", 46, 121}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_ERROR - {"MSGSIGDIGEST_ERROR", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_ERROR}, - #else - {"MSGSIGDIGEST_ERROR", 46, 172}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_VERIFICATION_FAILURE - {"MSGSIGDIGEST_VERIFICATION_FAILURE", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_VERIFICATION_FAILURE}, - #else - {"MSGSIGDIGEST_VERIFICATION_FAILURE", 46, 162}, - #endif - #ifdef CMS_R_MSGSIGDIGEST_WRONG_LENGTH - {"MSGSIGDIGEST_WRONG_LENGTH", ERR_LIB_CMS, CMS_R_MSGSIGDIGEST_WRONG_LENGTH}, - #else - {"MSGSIGDIGEST_WRONG_LENGTH", 46, 163}, - #endif - #ifdef CMS_R_NEED_ONE_SIGNER - {"NEED_ONE_SIGNER", ERR_LIB_CMS, CMS_R_NEED_ONE_SIGNER}, - #else - {"NEED_ONE_SIGNER", 46, 164}, - #endif - #ifdef CMS_R_NOT_A_SIGNED_RECEIPT - {"NOT_A_SIGNED_RECEIPT", ERR_LIB_CMS, CMS_R_NOT_A_SIGNED_RECEIPT}, - #else - {"NOT_A_SIGNED_RECEIPT", 46, 165}, - #endif - #ifdef CMS_R_NOT_ENCRYPTED_DATA - {"NOT_ENCRYPTED_DATA", ERR_LIB_CMS, CMS_R_NOT_ENCRYPTED_DATA}, - #else - {"NOT_ENCRYPTED_DATA", 46, 122}, - #endif - #ifdef CMS_R_NOT_KEK - {"NOT_KEK", ERR_LIB_CMS, CMS_R_NOT_KEK}, - #else - {"NOT_KEK", 46, 123}, - #endif - #ifdef CMS_R_NOT_KEY_AGREEMENT - {"NOT_KEY_AGREEMENT", ERR_LIB_CMS, CMS_R_NOT_KEY_AGREEMENT}, - #else - {"NOT_KEY_AGREEMENT", 46, 181}, - #endif - #ifdef CMS_R_NOT_KEY_TRANSPORT - {"NOT_KEY_TRANSPORT", ERR_LIB_CMS, CMS_R_NOT_KEY_TRANSPORT}, - #else - {"NOT_KEY_TRANSPORT", 46, 124}, - #endif - #ifdef CMS_R_NOT_PWRI - {"NOT_PWRI", ERR_LIB_CMS, CMS_R_NOT_PWRI}, - #else - {"NOT_PWRI", 46, 177}, - #endif - #ifdef CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 46, 125}, - #endif - #ifdef CMS_R_NO_CIPHER - {"NO_CIPHER", ERR_LIB_CMS, CMS_R_NO_CIPHER}, - #else - {"NO_CIPHER", 46, 126}, - #endif - #ifdef CMS_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_CMS, CMS_R_NO_CONTENT}, - #else - {"NO_CONTENT", 46, 127}, - #endif - #ifdef CMS_R_NO_CONTENT_TYPE - {"NO_CONTENT_TYPE", ERR_LIB_CMS, CMS_R_NO_CONTENT_TYPE}, - #else - {"NO_CONTENT_TYPE", 46, 173}, - #endif - #ifdef CMS_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 46, 128}, - #endif - #ifdef CMS_R_NO_DIGEST_SET - {"NO_DIGEST_SET", ERR_LIB_CMS, CMS_R_NO_DIGEST_SET}, - #else - {"NO_DIGEST_SET", 46, 129}, - #endif - #ifdef CMS_R_NO_KEY - {"NO_KEY", ERR_LIB_CMS, CMS_R_NO_KEY}, - #else - {"NO_KEY", 46, 130}, - #endif - #ifdef CMS_R_NO_KEY_OR_CERT - {"NO_KEY_OR_CERT", ERR_LIB_CMS, CMS_R_NO_KEY_OR_CERT}, - #else - {"NO_KEY_OR_CERT", 46, 174}, - #endif - #ifdef CMS_R_NO_MATCHING_DIGEST - {"NO_MATCHING_DIGEST", ERR_LIB_CMS, CMS_R_NO_MATCHING_DIGEST}, - #else - {"NO_MATCHING_DIGEST", 46, 131}, - #endif - #ifdef CMS_R_NO_MATCHING_RECIPIENT - {"NO_MATCHING_RECIPIENT", ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT}, - #else - {"NO_MATCHING_RECIPIENT", 46, 132}, - #endif - #ifdef CMS_R_NO_MATCHING_SIGNATURE - {"NO_MATCHING_SIGNATURE", ERR_LIB_CMS, CMS_R_NO_MATCHING_SIGNATURE}, - #else - {"NO_MATCHING_SIGNATURE", 46, 166}, - #endif - #ifdef CMS_R_NO_MSGSIGDIGEST - {"NO_MSGSIGDIGEST", ERR_LIB_CMS, CMS_R_NO_MSGSIGDIGEST}, - #else - {"NO_MSGSIGDIGEST", 46, 167}, - #endif - #ifdef CMS_R_NO_PASSWORD - {"NO_PASSWORD", ERR_LIB_CMS, CMS_R_NO_PASSWORD}, - #else - {"NO_PASSWORD", 46, 178}, - #endif - #ifdef CMS_R_NO_PRIVATE_KEY - {"NO_PRIVATE_KEY", ERR_LIB_CMS, CMS_R_NO_PRIVATE_KEY}, - #else - {"NO_PRIVATE_KEY", 46, 133}, - #endif - #ifdef CMS_R_NO_PUBLIC_KEY - {"NO_PUBLIC_KEY", ERR_LIB_CMS, CMS_R_NO_PUBLIC_KEY}, - #else - {"NO_PUBLIC_KEY", 46, 134}, - #endif - #ifdef CMS_R_NO_RECEIPT_REQUEST - {"NO_RECEIPT_REQUEST", ERR_LIB_CMS, CMS_R_NO_RECEIPT_REQUEST}, - #else - {"NO_RECEIPT_REQUEST", 46, 168}, - #endif - #ifdef CMS_R_NO_SIGNERS - {"NO_SIGNERS", ERR_LIB_CMS, CMS_R_NO_SIGNERS}, - #else - {"NO_SIGNERS", 46, 135}, - #endif - #ifdef CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_CMS, CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 46, 136}, - #endif - #ifdef CMS_R_RECEIPT_DECODE_ERROR - {"RECEIPT_DECODE_ERROR", ERR_LIB_CMS, CMS_R_RECEIPT_DECODE_ERROR}, - #else - {"RECEIPT_DECODE_ERROR", 46, 169}, - #endif - #ifdef CMS_R_RECIPIENT_ERROR - {"RECIPIENT_ERROR", ERR_LIB_CMS, CMS_R_RECIPIENT_ERROR}, - #else - {"RECIPIENT_ERROR", 46, 137}, - #endif - #ifdef CMS_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_CMS, CMS_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 46, 138}, - #endif - #ifdef CMS_R_SIGNFINAL_ERROR - {"SIGNFINAL_ERROR", ERR_LIB_CMS, CMS_R_SIGNFINAL_ERROR}, - #else - {"SIGNFINAL_ERROR", 46, 139}, - #endif - #ifdef CMS_R_SMIME_TEXT_ERROR - {"SMIME_TEXT_ERROR", ERR_LIB_CMS, CMS_R_SMIME_TEXT_ERROR}, - #else - {"SMIME_TEXT_ERROR", 46, 140}, - #endif - #ifdef CMS_R_STORE_INIT_ERROR - {"STORE_INIT_ERROR", ERR_LIB_CMS, CMS_R_STORE_INIT_ERROR}, - #else - {"STORE_INIT_ERROR", 46, 141}, - #endif - #ifdef CMS_R_TYPE_NOT_COMPRESSED_DATA - {"TYPE_NOT_COMPRESSED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_COMPRESSED_DATA}, - #else - {"TYPE_NOT_COMPRESSED_DATA", 46, 142}, - #endif - #ifdef CMS_R_TYPE_NOT_DATA - {"TYPE_NOT_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_DATA}, - #else - {"TYPE_NOT_DATA", 46, 143}, - #endif - #ifdef CMS_R_TYPE_NOT_DIGESTED_DATA - {"TYPE_NOT_DIGESTED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_DIGESTED_DATA}, - #else - {"TYPE_NOT_DIGESTED_DATA", 46, 144}, - #endif - #ifdef CMS_R_TYPE_NOT_ENCRYPTED_DATA - {"TYPE_NOT_ENCRYPTED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_ENCRYPTED_DATA}, - #else - {"TYPE_NOT_ENCRYPTED_DATA", 46, 145}, - #endif - #ifdef CMS_R_TYPE_NOT_ENVELOPED_DATA - {"TYPE_NOT_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_TYPE_NOT_ENVELOPED_DATA}, - #else - {"TYPE_NOT_ENVELOPED_DATA", 46, 146}, - #endif - #ifdef CMS_R_UNABLE_TO_FINALIZE_CONTEXT - {"UNABLE_TO_FINALIZE_CONTEXT", ERR_LIB_CMS, CMS_R_UNABLE_TO_FINALIZE_CONTEXT}, - #else - {"UNABLE_TO_FINALIZE_CONTEXT", 46, 147}, - #endif - #ifdef CMS_R_UNKNOWN_CIPHER - {"UNKNOWN_CIPHER", ERR_LIB_CMS, CMS_R_UNKNOWN_CIPHER}, - #else - {"UNKNOWN_CIPHER", 46, 148}, - #endif - #ifdef CMS_R_UNKNOWN_DIGEST_ALGORITHM - {"UNKNOWN_DIGEST_ALGORITHM", ERR_LIB_CMS, CMS_R_UNKNOWN_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_DIGEST_ALGORITHM", 46, 149}, - #endif - #ifdef CMS_R_UNKNOWN_ID - {"UNKNOWN_ID", ERR_LIB_CMS, CMS_R_UNKNOWN_ID}, - #else - {"UNKNOWN_ID", 46, 150}, - #endif - #ifdef CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM - {"UNSUPPORTED_COMPRESSION_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM}, - #else - {"UNSUPPORTED_COMPRESSION_ALGORITHM", 46, 151}, - #endif - #ifdef CMS_R_UNSUPPORTED_CONTENT_TYPE - {"UNSUPPORTED_CONTENT_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_CONTENT_TYPE}, - #else - {"UNSUPPORTED_CONTENT_TYPE", 46, 152}, - #endif - #ifdef CMS_R_UNSUPPORTED_KEK_ALGORITHM - {"UNSUPPORTED_KEK_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_KEK_ALGORITHM}, - #else - {"UNSUPPORTED_KEK_ALGORITHM", 46, 153}, - #endif - #ifdef CMS_R_UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM - {"UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM}, - #else - {"UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", 46, 179}, - #endif - #ifdef CMS_R_UNSUPPORTED_RECIPIENTINFO_TYPE - {"UNSUPPORTED_RECIPIENTINFO_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_RECIPIENTINFO_TYPE}, - #else - {"UNSUPPORTED_RECIPIENTINFO_TYPE", 46, 155}, - #endif - #ifdef CMS_R_UNSUPPORTED_RECIPIENT_TYPE - {"UNSUPPORTED_RECIPIENT_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_RECIPIENT_TYPE}, - #else - {"UNSUPPORTED_RECIPIENT_TYPE", 46, 154}, - #endif - #ifdef CMS_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 46, 156}, - #endif - #ifdef CMS_R_UNWRAP_ERROR - {"UNWRAP_ERROR", ERR_LIB_CMS, CMS_R_UNWRAP_ERROR}, - #else - {"UNWRAP_ERROR", 46, 157}, - #endif - #ifdef CMS_R_UNWRAP_FAILURE - {"UNWRAP_FAILURE", ERR_LIB_CMS, CMS_R_UNWRAP_FAILURE}, - #else - {"UNWRAP_FAILURE", 46, 180}, - #endif - #ifdef CMS_R_VERIFICATION_FAILURE - {"VERIFICATION_FAILURE", ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE}, - #else - {"VERIFICATION_FAILURE", 46, 158}, - #endif - #ifdef CMS_R_WRAP_ERROR - {"WRAP_ERROR", ERR_LIB_CMS, CMS_R_WRAP_ERROR}, - #else - {"WRAP_ERROR", 46, 159}, - #endif - #ifdef COMP_R_ZLIB_DEFLATE_ERROR - {"ZLIB_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_DEFLATE_ERROR}, - #else - {"ZLIB_DEFLATE_ERROR", 41, 99}, - #endif - #ifdef COMP_R_ZLIB_INFLATE_ERROR - {"ZLIB_INFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_INFLATE_ERROR}, - #else - {"ZLIB_INFLATE_ERROR", 41, 100}, - #endif - #ifdef COMP_R_ZLIB_NOT_SUPPORTED - {"ZLIB_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_ZLIB_NOT_SUPPORTED}, - #else - {"ZLIB_NOT_SUPPORTED", 41, 101}, - #endif - #ifdef CONF_R_ERROR_LOADING_DSO - {"ERROR_LOADING_DSO", ERR_LIB_CONF, CONF_R_ERROR_LOADING_DSO}, - #else - {"ERROR_LOADING_DSO", 14, 110}, - #endif - #ifdef CONF_R_LIST_CANNOT_BE_NULL - {"LIST_CANNOT_BE_NULL", ERR_LIB_CONF, CONF_R_LIST_CANNOT_BE_NULL}, - #else - {"LIST_CANNOT_BE_NULL", 14, 115}, - #endif - #ifdef CONF_R_MISSING_CLOSE_SQUARE_BRACKET - {"MISSING_CLOSE_SQUARE_BRACKET", ERR_LIB_CONF, CONF_R_MISSING_CLOSE_SQUARE_BRACKET}, - #else - {"MISSING_CLOSE_SQUARE_BRACKET", 14, 100}, - #endif - #ifdef CONF_R_MISSING_EQUAL_SIGN - {"MISSING_EQUAL_SIGN", ERR_LIB_CONF, CONF_R_MISSING_EQUAL_SIGN}, - #else - {"MISSING_EQUAL_SIGN", 14, 101}, - #endif - #ifdef CONF_R_MISSING_INIT_FUNCTION - {"MISSING_INIT_FUNCTION", ERR_LIB_CONF, CONF_R_MISSING_INIT_FUNCTION}, - #else - {"MISSING_INIT_FUNCTION", 14, 112}, - #endif - #ifdef CONF_R_MODULE_INITIALIZATION_ERROR - {"MODULE_INITIALIZATION_ERROR", ERR_LIB_CONF, CONF_R_MODULE_INITIALIZATION_ERROR}, - #else - {"MODULE_INITIALIZATION_ERROR", 14, 109}, - #endif - #ifdef CONF_R_NO_CLOSE_BRACE - {"NO_CLOSE_BRACE", ERR_LIB_CONF, CONF_R_NO_CLOSE_BRACE}, - #else - {"NO_CLOSE_BRACE", 14, 102}, - #endif - #ifdef CONF_R_NO_CONF - {"NO_CONF", ERR_LIB_CONF, CONF_R_NO_CONF}, - #else - {"NO_CONF", 14, 105}, - #endif - #ifdef CONF_R_NO_CONF_OR_ENVIRONMENT_VARIABLE - {"NO_CONF_OR_ENVIRONMENT_VARIABLE", ERR_LIB_CONF, CONF_R_NO_CONF_OR_ENVIRONMENT_VARIABLE}, - #else - {"NO_CONF_OR_ENVIRONMENT_VARIABLE", 14, 106}, - #endif - #ifdef CONF_R_NO_SECTION - {"NO_SECTION", ERR_LIB_CONF, CONF_R_NO_SECTION}, - #else - {"NO_SECTION", 14, 107}, - #endif - #ifdef CONF_R_NO_SUCH_FILE - {"NO_SUCH_FILE", ERR_LIB_CONF, CONF_R_NO_SUCH_FILE}, - #else - {"NO_SUCH_FILE", 14, 114}, - #endif - #ifdef CONF_R_NO_VALUE - {"NO_VALUE", ERR_LIB_CONF, CONF_R_NO_VALUE}, - #else - {"NO_VALUE", 14, 108}, - #endif - #ifdef CONF_R_NUMBER_TOO_LARGE - {"NUMBER_TOO_LARGE", ERR_LIB_CONF, CONF_R_NUMBER_TOO_LARGE}, - #else - {"NUMBER_TOO_LARGE", 14, 121}, - #endif - #ifdef CONF_R_RECURSIVE_DIRECTORY_INCLUDE - {"RECURSIVE_DIRECTORY_INCLUDE", ERR_LIB_CONF, CONF_R_RECURSIVE_DIRECTORY_INCLUDE}, - #else - {"RECURSIVE_DIRECTORY_INCLUDE", 14, 111}, - #endif - #ifdef CONF_R_SSL_COMMAND_SECTION_EMPTY - {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_CONF, CONF_R_SSL_COMMAND_SECTION_EMPTY}, - #else - {"SSL_COMMAND_SECTION_EMPTY", 14, 117}, - #endif - #ifdef CONF_R_SSL_COMMAND_SECTION_NOT_FOUND - {"SSL_COMMAND_SECTION_NOT_FOUND", ERR_LIB_CONF, CONF_R_SSL_COMMAND_SECTION_NOT_FOUND}, - #else - {"SSL_COMMAND_SECTION_NOT_FOUND", 14, 118}, - #endif - #ifdef CONF_R_SSL_SECTION_EMPTY - {"SSL_SECTION_EMPTY", ERR_LIB_CONF, CONF_R_SSL_SECTION_EMPTY}, - #else - {"SSL_SECTION_EMPTY", 14, 119}, - #endif - #ifdef CONF_R_SSL_SECTION_NOT_FOUND - {"SSL_SECTION_NOT_FOUND", ERR_LIB_CONF, CONF_R_SSL_SECTION_NOT_FOUND}, - #else - {"SSL_SECTION_NOT_FOUND", 14, 120}, - #endif - #ifdef CONF_R_UNABLE_TO_CREATE_NEW_SECTION - {"UNABLE_TO_CREATE_NEW_SECTION", ERR_LIB_CONF, CONF_R_UNABLE_TO_CREATE_NEW_SECTION}, - #else - {"UNABLE_TO_CREATE_NEW_SECTION", 14, 103}, - #endif - #ifdef CONF_R_UNKNOWN_MODULE_NAME - {"UNKNOWN_MODULE_NAME", ERR_LIB_CONF, CONF_R_UNKNOWN_MODULE_NAME}, - #else - {"UNKNOWN_MODULE_NAME", 14, 113}, - #endif - #ifdef CONF_R_VARIABLE_EXPANSION_TOO_LONG - {"VARIABLE_EXPANSION_TOO_LONG", ERR_LIB_CONF, CONF_R_VARIABLE_EXPANSION_TOO_LONG}, - #else - {"VARIABLE_EXPANSION_TOO_LONG", 14, 116}, - #endif - #ifdef CONF_R_VARIABLE_HAS_NO_VALUE - {"VARIABLE_HAS_NO_VALUE", ERR_LIB_CONF, CONF_R_VARIABLE_HAS_NO_VALUE}, - #else - {"VARIABLE_HAS_NO_VALUE", 14, 104}, - #endif - #ifdef CRYPTO_R_FIPS_MODE_NOT_SUPPORTED - {"FIPS_MODE_NOT_SUPPORTED", ERR_LIB_CRYPTO, CRYPTO_R_FIPS_MODE_NOT_SUPPORTED}, - #else - {"FIPS_MODE_NOT_SUPPORTED", 15, 101}, - #endif - #ifdef CRYPTO_R_ILLEGAL_HEX_DIGIT - {"ILLEGAL_HEX_DIGIT", ERR_LIB_CRYPTO, CRYPTO_R_ILLEGAL_HEX_DIGIT}, - #else - {"ILLEGAL_HEX_DIGIT", 15, 102}, - #endif - #ifdef CRYPTO_R_ODD_NUMBER_OF_DIGITS - {"ODD_NUMBER_OF_DIGITS", ERR_LIB_CRYPTO, CRYPTO_R_ODD_NUMBER_OF_DIGITS}, - #else - {"ODD_NUMBER_OF_DIGITS", 15, 103}, - #endif - #ifdef CT_R_BASE64_DECODE_ERROR - {"BASE64_DECODE_ERROR", ERR_LIB_CT, CT_R_BASE64_DECODE_ERROR}, - #else - {"BASE64_DECODE_ERROR", 50, 108}, - #endif - #ifdef CT_R_INVALID_LOG_ID_LENGTH - {"INVALID_LOG_ID_LENGTH", ERR_LIB_CT, CT_R_INVALID_LOG_ID_LENGTH}, - #else - {"INVALID_LOG_ID_LENGTH", 50, 100}, - #endif - #ifdef CT_R_LOG_CONF_INVALID - {"LOG_CONF_INVALID", ERR_LIB_CT, CT_R_LOG_CONF_INVALID}, - #else - {"LOG_CONF_INVALID", 50, 109}, - #endif - #ifdef CT_R_LOG_CONF_INVALID_KEY - {"LOG_CONF_INVALID_KEY", ERR_LIB_CT, CT_R_LOG_CONF_INVALID_KEY}, - #else - {"LOG_CONF_INVALID_KEY", 50, 110}, - #endif - #ifdef CT_R_LOG_CONF_MISSING_DESCRIPTION - {"LOG_CONF_MISSING_DESCRIPTION", ERR_LIB_CT, CT_R_LOG_CONF_MISSING_DESCRIPTION}, - #else - {"LOG_CONF_MISSING_DESCRIPTION", 50, 111}, - #endif - #ifdef CT_R_LOG_CONF_MISSING_KEY - {"LOG_CONF_MISSING_KEY", ERR_LIB_CT, CT_R_LOG_CONF_MISSING_KEY}, - #else - {"LOG_CONF_MISSING_KEY", 50, 112}, - #endif - #ifdef CT_R_LOG_KEY_INVALID - {"LOG_KEY_INVALID", ERR_LIB_CT, CT_R_LOG_KEY_INVALID}, - #else - {"LOG_KEY_INVALID", 50, 113}, - #endif - #ifdef CT_R_SCT_FUTURE_TIMESTAMP - {"SCT_FUTURE_TIMESTAMP", ERR_LIB_CT, CT_R_SCT_FUTURE_TIMESTAMP}, - #else - {"SCT_FUTURE_TIMESTAMP", 50, 116}, - #endif - #ifdef CT_R_SCT_INVALID - {"SCT_INVALID", ERR_LIB_CT, CT_R_SCT_INVALID}, - #else - {"SCT_INVALID", 50, 104}, - #endif - #ifdef CT_R_SCT_INVALID_SIGNATURE - {"SCT_INVALID_SIGNATURE", ERR_LIB_CT, CT_R_SCT_INVALID_SIGNATURE}, - #else - {"SCT_INVALID_SIGNATURE", 50, 107}, - #endif - #ifdef CT_R_SCT_LIST_INVALID - {"SCT_LIST_INVALID", ERR_LIB_CT, CT_R_SCT_LIST_INVALID}, - #else - {"SCT_LIST_INVALID", 50, 105}, - #endif - #ifdef CT_R_SCT_LOG_ID_MISMATCH - {"SCT_LOG_ID_MISMATCH", ERR_LIB_CT, CT_R_SCT_LOG_ID_MISMATCH}, - #else - {"SCT_LOG_ID_MISMATCH", 50, 114}, - #endif - #ifdef CT_R_SCT_NOT_SET - {"SCT_NOT_SET", ERR_LIB_CT, CT_R_SCT_NOT_SET}, - #else - {"SCT_NOT_SET", 50, 106}, - #endif - #ifdef CT_R_SCT_UNSUPPORTED_VERSION - {"SCT_UNSUPPORTED_VERSION", ERR_LIB_CT, CT_R_SCT_UNSUPPORTED_VERSION}, - #else - {"SCT_UNSUPPORTED_VERSION", 50, 115}, - #endif - #ifdef CT_R_UNRECOGNIZED_SIGNATURE_NID - {"UNRECOGNIZED_SIGNATURE_NID", ERR_LIB_CT, CT_R_UNRECOGNIZED_SIGNATURE_NID}, - #else - {"UNRECOGNIZED_SIGNATURE_NID", 50, 101}, - #endif - #ifdef CT_R_UNSUPPORTED_ENTRY_TYPE - {"UNSUPPORTED_ENTRY_TYPE", ERR_LIB_CT, CT_R_UNSUPPORTED_ENTRY_TYPE}, - #else - {"UNSUPPORTED_ENTRY_TYPE", 50, 102}, - #endif - #ifdef CT_R_UNSUPPORTED_VERSION - {"UNSUPPORTED_VERSION", ERR_LIB_CT, CT_R_UNSUPPORTED_VERSION}, - #else - {"UNSUPPORTED_VERSION", 50, 103}, - #endif - #ifdef DH_R_BAD_GENERATOR - {"BAD_GENERATOR", ERR_LIB_DH, DH_R_BAD_GENERATOR}, - #else - {"BAD_GENERATOR", 5, 101}, - #endif - #ifdef DH_R_BN_DECODE_ERROR - {"BN_DECODE_ERROR", ERR_LIB_DH, DH_R_BN_DECODE_ERROR}, - #else - {"BN_DECODE_ERROR", 5, 109}, - #endif - #ifdef DH_R_BN_ERROR - {"BN_ERROR", ERR_LIB_DH, DH_R_BN_ERROR}, - #else - {"BN_ERROR", 5, 106}, - #endif - #ifdef DH_R_CHECK_INVALID_J_VALUE - {"CHECK_INVALID_J_VALUE", ERR_LIB_DH, DH_R_CHECK_INVALID_J_VALUE}, - #else - {"CHECK_INVALID_J_VALUE", 5, 115}, - #endif - #ifdef DH_R_CHECK_INVALID_Q_VALUE - {"CHECK_INVALID_Q_VALUE", ERR_LIB_DH, DH_R_CHECK_INVALID_Q_VALUE}, - #else - {"CHECK_INVALID_Q_VALUE", 5, 116}, - #endif - #ifdef DH_R_CHECK_PUBKEY_INVALID - {"CHECK_PUBKEY_INVALID", ERR_LIB_DH, DH_R_CHECK_PUBKEY_INVALID}, - #else - {"CHECK_PUBKEY_INVALID", 5, 122}, - #endif - #ifdef DH_R_CHECK_PUBKEY_TOO_LARGE - {"CHECK_PUBKEY_TOO_LARGE", ERR_LIB_DH, DH_R_CHECK_PUBKEY_TOO_LARGE}, - #else - {"CHECK_PUBKEY_TOO_LARGE", 5, 123}, - #endif - #ifdef DH_R_CHECK_PUBKEY_TOO_SMALL - {"CHECK_PUBKEY_TOO_SMALL", ERR_LIB_DH, DH_R_CHECK_PUBKEY_TOO_SMALL}, - #else - {"CHECK_PUBKEY_TOO_SMALL", 5, 124}, - #endif - #ifdef DH_R_CHECK_P_NOT_PRIME - {"CHECK_P_NOT_PRIME", ERR_LIB_DH, DH_R_CHECK_P_NOT_PRIME}, - #else - {"CHECK_P_NOT_PRIME", 5, 117}, - #endif - #ifdef DH_R_CHECK_P_NOT_SAFE_PRIME - {"CHECK_P_NOT_SAFE_PRIME", ERR_LIB_DH, DH_R_CHECK_P_NOT_SAFE_PRIME}, - #else - {"CHECK_P_NOT_SAFE_PRIME", 5, 118}, - #endif - #ifdef DH_R_CHECK_Q_NOT_PRIME - {"CHECK_Q_NOT_PRIME", ERR_LIB_DH, DH_R_CHECK_Q_NOT_PRIME}, - #else - {"CHECK_Q_NOT_PRIME", 5, 119}, - #endif - #ifdef DH_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_DH, DH_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 5, 104}, - #endif - #ifdef DH_R_INVALID_PARAMETER_NAME - {"INVALID_PARAMETER_NAME", ERR_LIB_DH, DH_R_INVALID_PARAMETER_NAME}, - #else - {"INVALID_PARAMETER_NAME", 5, 110}, - #endif - #ifdef DH_R_INVALID_PARAMETER_NID - {"INVALID_PARAMETER_NID", ERR_LIB_DH, DH_R_INVALID_PARAMETER_NID}, - #else - {"INVALID_PARAMETER_NID", 5, 114}, - #endif - #ifdef DH_R_INVALID_PUBKEY - {"INVALID_PUBKEY", ERR_LIB_DH, DH_R_INVALID_PUBKEY}, - #else - {"INVALID_PUBKEY", 5, 102}, - #endif - #ifdef DH_R_KDF_PARAMETER_ERROR - {"KDF_PARAMETER_ERROR", ERR_LIB_DH, DH_R_KDF_PARAMETER_ERROR}, - #else - {"KDF_PARAMETER_ERROR", 5, 112}, - #endif - #ifdef DH_R_KEYS_NOT_SET - {"KEYS_NOT_SET", ERR_LIB_DH, DH_R_KEYS_NOT_SET}, - #else - {"KEYS_NOT_SET", 5, 108}, - #endif - #ifdef DH_R_MISSING_PUBKEY - {"MISSING_PUBKEY", ERR_LIB_DH, DH_R_MISSING_PUBKEY}, - #else - {"MISSING_PUBKEY", 5, 125}, - #endif - #ifdef DH_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_DH, DH_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 5, 103}, - #endif - #ifdef DH_R_NOT_SUITABLE_GENERATOR - {"NOT_SUITABLE_GENERATOR", ERR_LIB_DH, DH_R_NOT_SUITABLE_GENERATOR}, - #else - {"NOT_SUITABLE_GENERATOR", 5, 120}, - #endif - #ifdef DH_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_DH, DH_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 5, 107}, - #endif - #ifdef DH_R_NO_PRIVATE_VALUE - {"NO_PRIVATE_VALUE", ERR_LIB_DH, DH_R_NO_PRIVATE_VALUE}, - #else - {"NO_PRIVATE_VALUE", 5, 100}, - #endif - #ifdef DH_R_PARAMETER_ENCODING_ERROR - {"PARAMETER_ENCODING_ERROR", ERR_LIB_DH, DH_R_PARAMETER_ENCODING_ERROR}, - #else - {"PARAMETER_ENCODING_ERROR", 5, 105}, - #endif - #ifdef DH_R_PEER_KEY_ERROR - {"PEER_KEY_ERROR", ERR_LIB_DH, DH_R_PEER_KEY_ERROR}, - #else - {"PEER_KEY_ERROR", 5, 111}, - #endif - #ifdef DH_R_SHARED_INFO_ERROR - {"SHARED_INFO_ERROR", ERR_LIB_DH, DH_R_SHARED_INFO_ERROR}, - #else - {"SHARED_INFO_ERROR", 5, 113}, - #endif - #ifdef DH_R_UNABLE_TO_CHECK_GENERATOR - {"UNABLE_TO_CHECK_GENERATOR", ERR_LIB_DH, DH_R_UNABLE_TO_CHECK_GENERATOR}, - #else - {"UNABLE_TO_CHECK_GENERATOR", 5, 121}, - #endif - #ifdef DSA_R_BAD_Q_VALUE - {"BAD_Q_VALUE", ERR_LIB_DSA, DSA_R_BAD_Q_VALUE}, - #else - {"BAD_Q_VALUE", 10, 102}, - #endif - #ifdef DSA_R_BN_DECODE_ERROR - {"BN_DECODE_ERROR", ERR_LIB_DSA, DSA_R_BN_DECODE_ERROR}, - #else - {"BN_DECODE_ERROR", 10, 108}, - #endif - #ifdef DSA_R_BN_ERROR - {"BN_ERROR", ERR_LIB_DSA, DSA_R_BN_ERROR}, - #else - {"BN_ERROR", 10, 109}, - #endif - #ifdef DSA_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_DSA, DSA_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 10, 104}, - #endif - #ifdef DSA_R_INVALID_DIGEST_TYPE - {"INVALID_DIGEST_TYPE", ERR_LIB_DSA, DSA_R_INVALID_DIGEST_TYPE}, - #else - {"INVALID_DIGEST_TYPE", 10, 106}, - #endif - #ifdef DSA_R_INVALID_PARAMETERS - {"INVALID_PARAMETERS", ERR_LIB_DSA, DSA_R_INVALID_PARAMETERS}, - #else - {"INVALID_PARAMETERS", 10, 112}, - #endif - #ifdef DSA_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_DSA, DSA_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 10, 101}, - #endif - #ifdef DSA_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_DSA, DSA_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 10, 111}, - #endif - #ifdef DSA_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_DSA, DSA_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 10, 103}, - #endif - #ifdef DSA_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_DSA, DSA_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 10, 107}, - #endif - #ifdef DSA_R_PARAMETER_ENCODING_ERROR - {"PARAMETER_ENCODING_ERROR", ERR_LIB_DSA, DSA_R_PARAMETER_ENCODING_ERROR}, - #else - {"PARAMETER_ENCODING_ERROR", 10, 105}, - #endif - #ifdef DSA_R_Q_NOT_PRIME - {"Q_NOT_PRIME", ERR_LIB_DSA, DSA_R_Q_NOT_PRIME}, - #else - {"Q_NOT_PRIME", 10, 113}, - #endif - #ifdef DSA_R_SEED_LEN_SMALL - {"SEED_LEN_SMALL", ERR_LIB_DSA, DSA_R_SEED_LEN_SMALL}, - #else - {"SEED_LEN_SMALL", 10, 110}, - #endif - #ifdef EC_R_ASN1_ERROR - {"ASN1_ERROR", ERR_LIB_EC, EC_R_ASN1_ERROR}, - #else - {"ASN1_ERROR", 16, 115}, - #endif - #ifdef EC_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_EC, EC_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 16, 156}, - #endif - #ifdef EC_R_BIGNUM_OUT_OF_RANGE - {"BIGNUM_OUT_OF_RANGE", ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE}, - #else - {"BIGNUM_OUT_OF_RANGE", 16, 144}, - #endif - #ifdef EC_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_EC, EC_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 16, 100}, - #endif - #ifdef EC_R_CANNOT_INVERT - {"CANNOT_INVERT", ERR_LIB_EC, EC_R_CANNOT_INVERT}, - #else - {"CANNOT_INVERT", 16, 165}, - #endif - #ifdef EC_R_COORDINATES_OUT_OF_RANGE - {"COORDINATES_OUT_OF_RANGE", ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE}, - #else - {"COORDINATES_OUT_OF_RANGE", 16, 146}, - #endif - #ifdef EC_R_CURVE_DOES_NOT_SUPPORT_ECDH - {"CURVE_DOES_NOT_SUPPORT_ECDH", ERR_LIB_EC, EC_R_CURVE_DOES_NOT_SUPPORT_ECDH}, - #else - {"CURVE_DOES_NOT_SUPPORT_ECDH", 16, 160}, - #endif - #ifdef EC_R_CURVE_DOES_NOT_SUPPORT_SIGNING - {"CURVE_DOES_NOT_SUPPORT_SIGNING", ERR_LIB_EC, EC_R_CURVE_DOES_NOT_SUPPORT_SIGNING}, - #else - {"CURVE_DOES_NOT_SUPPORT_SIGNING", 16, 159}, - #endif - #ifdef EC_R_D2I_ECPKPARAMETERS_FAILURE - {"D2I_ECPKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_D2I_ECPKPARAMETERS_FAILURE}, - #else - {"D2I_ECPKPARAMETERS_FAILURE", 16, 117}, - #endif - #ifdef EC_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_EC, EC_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 16, 142}, - #endif - #ifdef EC_R_DISCRIMINANT_IS_ZERO - {"DISCRIMINANT_IS_ZERO", ERR_LIB_EC, EC_R_DISCRIMINANT_IS_ZERO}, - #else - {"DISCRIMINANT_IS_ZERO", 16, 118}, - #endif - #ifdef EC_R_EC_GROUP_NEW_BY_NAME_FAILURE - {"EC_GROUP_NEW_BY_NAME_FAILURE", ERR_LIB_EC, EC_R_EC_GROUP_NEW_BY_NAME_FAILURE}, - #else - {"EC_GROUP_NEW_BY_NAME_FAILURE", 16, 119}, - #endif - #ifdef EC_R_FIELD_TOO_LARGE - {"FIELD_TOO_LARGE", ERR_LIB_EC, EC_R_FIELD_TOO_LARGE}, - #else - {"FIELD_TOO_LARGE", 16, 143}, - #endif - #ifdef EC_R_GF2M_NOT_SUPPORTED - {"GF2M_NOT_SUPPORTED", ERR_LIB_EC, EC_R_GF2M_NOT_SUPPORTED}, - #else - {"GF2M_NOT_SUPPORTED", 16, 147}, - #endif - #ifdef EC_R_GROUP2PKPARAMETERS_FAILURE - {"GROUP2PKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_GROUP2PKPARAMETERS_FAILURE}, - #else - {"GROUP2PKPARAMETERS_FAILURE", 16, 120}, - #endif - #ifdef EC_R_I2D_ECPKPARAMETERS_FAILURE - {"I2D_ECPKPARAMETERS_FAILURE", ERR_LIB_EC, EC_R_I2D_ECPKPARAMETERS_FAILURE}, - #else - {"I2D_ECPKPARAMETERS_FAILURE", 16, 121}, - #endif - #ifdef EC_R_INCOMPATIBLE_OBJECTS - {"INCOMPATIBLE_OBJECTS", ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS}, - #else - {"INCOMPATIBLE_OBJECTS", 16, 101}, - #endif - #ifdef EC_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_EC, EC_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 16, 112}, - #endif - #ifdef EC_R_INVALID_COMPRESSED_POINT - {"INVALID_COMPRESSED_POINT", ERR_LIB_EC, EC_R_INVALID_COMPRESSED_POINT}, - #else - {"INVALID_COMPRESSED_POINT", 16, 110}, - #endif - #ifdef EC_R_INVALID_COMPRESSION_BIT - {"INVALID_COMPRESSION_BIT", ERR_LIB_EC, EC_R_INVALID_COMPRESSION_BIT}, - #else - {"INVALID_COMPRESSION_BIT", 16, 109}, - #endif - #ifdef EC_R_INVALID_CURVE - {"INVALID_CURVE", ERR_LIB_EC, EC_R_INVALID_CURVE}, - #else - {"INVALID_CURVE", 16, 141}, - #endif - #ifdef EC_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_EC, EC_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 16, 151}, - #endif - #ifdef EC_R_INVALID_DIGEST_TYPE - {"INVALID_DIGEST_TYPE", ERR_LIB_EC, EC_R_INVALID_DIGEST_TYPE}, - #else - {"INVALID_DIGEST_TYPE", 16, 138}, - #endif - #ifdef EC_R_INVALID_ENCODING - {"INVALID_ENCODING", ERR_LIB_EC, EC_R_INVALID_ENCODING}, - #else - {"INVALID_ENCODING", 16, 102}, - #endif - #ifdef EC_R_INVALID_FIELD - {"INVALID_FIELD", ERR_LIB_EC, EC_R_INVALID_FIELD}, - #else - {"INVALID_FIELD", 16, 103}, - #endif - #ifdef EC_R_INVALID_FORM - {"INVALID_FORM", ERR_LIB_EC, EC_R_INVALID_FORM}, - #else - {"INVALID_FORM", 16, 104}, - #endif - #ifdef EC_R_INVALID_GROUP_ORDER - {"INVALID_GROUP_ORDER", ERR_LIB_EC, EC_R_INVALID_GROUP_ORDER}, - #else - {"INVALID_GROUP_ORDER", 16, 122}, - #endif - #ifdef EC_R_INVALID_KEY - {"INVALID_KEY", ERR_LIB_EC, EC_R_INVALID_KEY}, - #else - {"INVALID_KEY", 16, 116}, - #endif - #ifdef EC_R_INVALID_OUTPUT_LENGTH - {"INVALID_OUTPUT_LENGTH", ERR_LIB_EC, EC_R_INVALID_OUTPUT_LENGTH}, - #else - {"INVALID_OUTPUT_LENGTH", 16, 161}, - #endif - #ifdef EC_R_INVALID_PEER_KEY - {"INVALID_PEER_KEY", ERR_LIB_EC, EC_R_INVALID_PEER_KEY}, - #else - {"INVALID_PEER_KEY", 16, 133}, - #endif - #ifdef EC_R_INVALID_PENTANOMIAL_BASIS - {"INVALID_PENTANOMIAL_BASIS", ERR_LIB_EC, EC_R_INVALID_PENTANOMIAL_BASIS}, - #else - {"INVALID_PENTANOMIAL_BASIS", 16, 132}, - #endif - #ifdef EC_R_INVALID_PRIVATE_KEY - {"INVALID_PRIVATE_KEY", ERR_LIB_EC, EC_R_INVALID_PRIVATE_KEY}, - #else - {"INVALID_PRIVATE_KEY", 16, 123}, - #endif - #ifdef EC_R_INVALID_TRINOMIAL_BASIS - {"INVALID_TRINOMIAL_BASIS", ERR_LIB_EC, EC_R_INVALID_TRINOMIAL_BASIS}, - #else - {"INVALID_TRINOMIAL_BASIS", 16, 137}, - #endif - #ifdef EC_R_KDF_PARAMETER_ERROR - {"KDF_PARAMETER_ERROR", ERR_LIB_EC, EC_R_KDF_PARAMETER_ERROR}, - #else - {"KDF_PARAMETER_ERROR", 16, 148}, - #endif - #ifdef EC_R_KEYS_NOT_SET - {"KEYS_NOT_SET", ERR_LIB_EC, EC_R_KEYS_NOT_SET}, - #else - {"KEYS_NOT_SET", 16, 140}, - #endif - #ifdef EC_R_LADDER_POST_FAILURE - {"LADDER_POST_FAILURE", ERR_LIB_EC, EC_R_LADDER_POST_FAILURE}, - #else - {"LADDER_POST_FAILURE", 16, 136}, - #endif - #ifdef EC_R_LADDER_PRE_FAILURE - {"LADDER_PRE_FAILURE", ERR_LIB_EC, EC_R_LADDER_PRE_FAILURE}, - #else - {"LADDER_PRE_FAILURE", 16, 153}, - #endif - #ifdef EC_R_LADDER_STEP_FAILURE - {"LADDER_STEP_FAILURE", ERR_LIB_EC, EC_R_LADDER_STEP_FAILURE}, - #else - {"LADDER_STEP_FAILURE", 16, 162}, - #endif - #ifdef EC_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_EC, EC_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 16, 124}, - #endif - #ifdef EC_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_EC, EC_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 16, 125}, - #endif - #ifdef EC_R_NEED_NEW_SETUP_VALUES - {"NEED_NEW_SETUP_VALUES", ERR_LIB_EC, EC_R_NEED_NEW_SETUP_VALUES}, - #else - {"NEED_NEW_SETUP_VALUES", 16, 157}, - #endif - #ifdef EC_R_NOT_A_NIST_PRIME - {"NOT_A_NIST_PRIME", ERR_LIB_EC, EC_R_NOT_A_NIST_PRIME}, - #else - {"NOT_A_NIST_PRIME", 16, 135}, - #endif - #ifdef EC_R_NOT_IMPLEMENTED - {"NOT_IMPLEMENTED", ERR_LIB_EC, EC_R_NOT_IMPLEMENTED}, - #else - {"NOT_IMPLEMENTED", 16, 126}, - #endif - #ifdef EC_R_NOT_INITIALIZED - {"NOT_INITIALIZED", ERR_LIB_EC, EC_R_NOT_INITIALIZED}, - #else - {"NOT_INITIALIZED", 16, 111}, - #endif - #ifdef EC_R_NO_PARAMETERS_SET - {"NO_PARAMETERS_SET", ERR_LIB_EC, EC_R_NO_PARAMETERS_SET}, - #else - {"NO_PARAMETERS_SET", 16, 139}, - #endif - #ifdef EC_R_NO_PRIVATE_VALUE - {"NO_PRIVATE_VALUE", ERR_LIB_EC, EC_R_NO_PRIVATE_VALUE}, - #else - {"NO_PRIVATE_VALUE", 16, 154}, - #endif - #ifdef EC_R_OPERATION_NOT_SUPPORTED - {"OPERATION_NOT_SUPPORTED", ERR_LIB_EC, EC_R_OPERATION_NOT_SUPPORTED}, - #else - {"OPERATION_NOT_SUPPORTED", 16, 152}, - #endif - #ifdef EC_R_PASSED_NULL_PARAMETER - {"PASSED_NULL_PARAMETER", ERR_LIB_EC, EC_R_PASSED_NULL_PARAMETER}, - #else - {"PASSED_NULL_PARAMETER", 16, 134}, - #endif - #ifdef EC_R_PEER_KEY_ERROR - {"PEER_KEY_ERROR", ERR_LIB_EC, EC_R_PEER_KEY_ERROR}, - #else - {"PEER_KEY_ERROR", 16, 149}, - #endif - #ifdef EC_R_PKPARAMETERS2GROUP_FAILURE - {"PKPARAMETERS2GROUP_FAILURE", ERR_LIB_EC, EC_R_PKPARAMETERS2GROUP_FAILURE}, - #else - {"PKPARAMETERS2GROUP_FAILURE", 16, 127}, - #endif - #ifdef EC_R_POINT_ARITHMETIC_FAILURE - {"POINT_ARITHMETIC_FAILURE", ERR_LIB_EC, EC_R_POINT_ARITHMETIC_FAILURE}, - #else - {"POINT_ARITHMETIC_FAILURE", 16, 155}, - #endif - #ifdef EC_R_POINT_AT_INFINITY - {"POINT_AT_INFINITY", ERR_LIB_EC, EC_R_POINT_AT_INFINITY}, - #else - {"POINT_AT_INFINITY", 16, 106}, - #endif - #ifdef EC_R_POINT_COORDINATES_BLIND_FAILURE - {"POINT_COORDINATES_BLIND_FAILURE", ERR_LIB_EC, EC_R_POINT_COORDINATES_BLIND_FAILURE}, - #else - {"POINT_COORDINATES_BLIND_FAILURE", 16, 163}, - #endif - #ifdef EC_R_POINT_IS_NOT_ON_CURVE - {"POINT_IS_NOT_ON_CURVE", ERR_LIB_EC, EC_R_POINT_IS_NOT_ON_CURVE}, - #else - {"POINT_IS_NOT_ON_CURVE", 16, 107}, - #endif - #ifdef EC_R_RANDOM_NUMBER_GENERATION_FAILED - {"RANDOM_NUMBER_GENERATION_FAILED", ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED}, - #else - {"RANDOM_NUMBER_GENERATION_FAILED", 16, 158}, - #endif - #ifdef EC_R_SHARED_INFO_ERROR - {"SHARED_INFO_ERROR", ERR_LIB_EC, EC_R_SHARED_INFO_ERROR}, - #else - {"SHARED_INFO_ERROR", 16, 150}, - #endif - #ifdef EC_R_SLOT_FULL - {"SLOT_FULL", ERR_LIB_EC, EC_R_SLOT_FULL}, - #else - {"SLOT_FULL", 16, 108}, - #endif - #ifdef EC_R_UNDEFINED_GENERATOR - {"UNDEFINED_GENERATOR", ERR_LIB_EC, EC_R_UNDEFINED_GENERATOR}, - #else - {"UNDEFINED_GENERATOR", 16, 113}, - #endif - #ifdef EC_R_UNDEFINED_ORDER - {"UNDEFINED_ORDER", ERR_LIB_EC, EC_R_UNDEFINED_ORDER}, - #else - {"UNDEFINED_ORDER", 16, 128}, - #endif - #ifdef EC_R_UNKNOWN_COFACTOR - {"UNKNOWN_COFACTOR", ERR_LIB_EC, EC_R_UNKNOWN_COFACTOR}, - #else - {"UNKNOWN_COFACTOR", 16, 164}, - #endif - #ifdef EC_R_UNKNOWN_GROUP - {"UNKNOWN_GROUP", ERR_LIB_EC, EC_R_UNKNOWN_GROUP}, - #else - {"UNKNOWN_GROUP", 16, 129}, - #endif - #ifdef EC_R_UNKNOWN_ORDER - {"UNKNOWN_ORDER", ERR_LIB_EC, EC_R_UNKNOWN_ORDER}, - #else - {"UNKNOWN_ORDER", 16, 114}, - #endif - #ifdef EC_R_UNSUPPORTED_FIELD - {"UNSUPPORTED_FIELD", ERR_LIB_EC, EC_R_UNSUPPORTED_FIELD}, - #else - {"UNSUPPORTED_FIELD", 16, 131}, - #endif - #ifdef EC_R_WRONG_CURVE_PARAMETERS - {"WRONG_CURVE_PARAMETERS", ERR_LIB_EC, EC_R_WRONG_CURVE_PARAMETERS}, - #else - {"WRONG_CURVE_PARAMETERS", 16, 145}, - #endif - #ifdef EC_R_WRONG_ORDER - {"WRONG_ORDER", ERR_LIB_EC, EC_R_WRONG_ORDER}, - #else - {"WRONG_ORDER", 16, 130}, - #endif - #ifdef ENGINE_R_ALREADY_LOADED - {"ALREADY_LOADED", ERR_LIB_ENGINE, ENGINE_R_ALREADY_LOADED}, - #else - {"ALREADY_LOADED", 38, 100}, - #endif - #ifdef ENGINE_R_ARGUMENT_IS_NOT_A_NUMBER - {"ARGUMENT_IS_NOT_A_NUMBER", ERR_LIB_ENGINE, ENGINE_R_ARGUMENT_IS_NOT_A_NUMBER}, - #else - {"ARGUMENT_IS_NOT_A_NUMBER", 38, 133}, - #endif - #ifdef ENGINE_R_CMD_NOT_EXECUTABLE - {"CMD_NOT_EXECUTABLE", ERR_LIB_ENGINE, ENGINE_R_CMD_NOT_EXECUTABLE}, - #else - {"CMD_NOT_EXECUTABLE", 38, 134}, - #endif - #ifdef ENGINE_R_COMMAND_TAKES_INPUT - {"COMMAND_TAKES_INPUT", ERR_LIB_ENGINE, ENGINE_R_COMMAND_TAKES_INPUT}, - #else - {"COMMAND_TAKES_INPUT", 38, 135}, - #endif - #ifdef ENGINE_R_COMMAND_TAKES_NO_INPUT - {"COMMAND_TAKES_NO_INPUT", ERR_LIB_ENGINE, ENGINE_R_COMMAND_TAKES_NO_INPUT}, - #else - {"COMMAND_TAKES_NO_INPUT", 38, 136}, - #endif - #ifdef ENGINE_R_CONFLICTING_ENGINE_ID - {"CONFLICTING_ENGINE_ID", ERR_LIB_ENGINE, ENGINE_R_CONFLICTING_ENGINE_ID}, - #else - {"CONFLICTING_ENGINE_ID", 38, 103}, - #endif - #ifdef ENGINE_R_CTRL_COMMAND_NOT_IMPLEMENTED - {"CTRL_COMMAND_NOT_IMPLEMENTED", ERR_LIB_ENGINE, ENGINE_R_CTRL_COMMAND_NOT_IMPLEMENTED}, - #else - {"CTRL_COMMAND_NOT_IMPLEMENTED", 38, 119}, - #endif - #ifdef ENGINE_R_DSO_FAILURE - {"DSO_FAILURE", ERR_LIB_ENGINE, ENGINE_R_DSO_FAILURE}, - #else - {"DSO_FAILURE", 38, 104}, - #endif - #ifdef ENGINE_R_DSO_NOT_FOUND - {"DSO_NOT_FOUND", ERR_LIB_ENGINE, ENGINE_R_DSO_NOT_FOUND}, - #else - {"DSO_NOT_FOUND", 38, 132}, - #endif - #ifdef ENGINE_R_ENGINES_SECTION_ERROR - {"ENGINES_SECTION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINES_SECTION_ERROR}, - #else - {"ENGINES_SECTION_ERROR", 38, 148}, - #endif - #ifdef ENGINE_R_ENGINE_CONFIGURATION_ERROR - {"ENGINE_CONFIGURATION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINE_CONFIGURATION_ERROR}, - #else - {"ENGINE_CONFIGURATION_ERROR", 38, 102}, - #endif - #ifdef ENGINE_R_ENGINE_IS_NOT_IN_LIST - {"ENGINE_IS_NOT_IN_LIST", ERR_LIB_ENGINE, ENGINE_R_ENGINE_IS_NOT_IN_LIST}, - #else - {"ENGINE_IS_NOT_IN_LIST", 38, 105}, - #endif - #ifdef ENGINE_R_ENGINE_SECTION_ERROR - {"ENGINE_SECTION_ERROR", ERR_LIB_ENGINE, ENGINE_R_ENGINE_SECTION_ERROR}, - #else - {"ENGINE_SECTION_ERROR", 38, 149}, - #endif - #ifdef ENGINE_R_FAILED_LOADING_PRIVATE_KEY - {"FAILED_LOADING_PRIVATE_KEY", ERR_LIB_ENGINE, ENGINE_R_FAILED_LOADING_PRIVATE_KEY}, - #else - {"FAILED_LOADING_PRIVATE_KEY", 38, 128}, - #endif - #ifdef ENGINE_R_FAILED_LOADING_PUBLIC_KEY - {"FAILED_LOADING_PUBLIC_KEY", ERR_LIB_ENGINE, ENGINE_R_FAILED_LOADING_PUBLIC_KEY}, - #else - {"FAILED_LOADING_PUBLIC_KEY", 38, 129}, - #endif - #ifdef ENGINE_R_FINISH_FAILED - {"FINISH_FAILED", ERR_LIB_ENGINE, ENGINE_R_FINISH_FAILED}, - #else - {"FINISH_FAILED", 38, 106}, - #endif - #ifdef ENGINE_R_ID_OR_NAME_MISSING - {"ID_OR_NAME_MISSING", ERR_LIB_ENGINE, ENGINE_R_ID_OR_NAME_MISSING}, - #else - {"ID_OR_NAME_MISSING", 38, 108}, - #endif - #ifdef ENGINE_R_INIT_FAILED - {"INIT_FAILED", ERR_LIB_ENGINE, ENGINE_R_INIT_FAILED}, - #else - {"INIT_FAILED", 38, 109}, - #endif - #ifdef ENGINE_R_INTERNAL_LIST_ERROR - {"INTERNAL_LIST_ERROR", ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR}, - #else - {"INTERNAL_LIST_ERROR", 38, 110}, - #endif - #ifdef ENGINE_R_INVALID_ARGUMENT - {"INVALID_ARGUMENT", ERR_LIB_ENGINE, ENGINE_R_INVALID_ARGUMENT}, - #else - {"INVALID_ARGUMENT", 38, 143}, - #endif - #ifdef ENGINE_R_INVALID_CMD_NAME - {"INVALID_CMD_NAME", ERR_LIB_ENGINE, ENGINE_R_INVALID_CMD_NAME}, - #else - {"INVALID_CMD_NAME", 38, 137}, - #endif - #ifdef ENGINE_R_INVALID_CMD_NUMBER - {"INVALID_CMD_NUMBER", ERR_LIB_ENGINE, ENGINE_R_INVALID_CMD_NUMBER}, - #else - {"INVALID_CMD_NUMBER", 38, 138}, - #endif - #ifdef ENGINE_R_INVALID_INIT_VALUE - {"INVALID_INIT_VALUE", ERR_LIB_ENGINE, ENGINE_R_INVALID_INIT_VALUE}, - #else - {"INVALID_INIT_VALUE", 38, 151}, - #endif - #ifdef ENGINE_R_INVALID_STRING - {"INVALID_STRING", ERR_LIB_ENGINE, ENGINE_R_INVALID_STRING}, - #else - {"INVALID_STRING", 38, 150}, - #endif - #ifdef ENGINE_R_NOT_INITIALISED - {"NOT_INITIALISED", ERR_LIB_ENGINE, ENGINE_R_NOT_INITIALISED}, - #else - {"NOT_INITIALISED", 38, 117}, - #endif - #ifdef ENGINE_R_NOT_LOADED - {"NOT_LOADED", ERR_LIB_ENGINE, ENGINE_R_NOT_LOADED}, - #else - {"NOT_LOADED", 38, 112}, - #endif - #ifdef ENGINE_R_NO_CONTROL_FUNCTION - {"NO_CONTROL_FUNCTION", ERR_LIB_ENGINE, ENGINE_R_NO_CONTROL_FUNCTION}, - #else - {"NO_CONTROL_FUNCTION", 38, 120}, - #endif - #ifdef ENGINE_R_NO_INDEX - {"NO_INDEX", ERR_LIB_ENGINE, ENGINE_R_NO_INDEX}, - #else - {"NO_INDEX", 38, 144}, - #endif - #ifdef ENGINE_R_NO_LOAD_FUNCTION - {"NO_LOAD_FUNCTION", ERR_LIB_ENGINE, ENGINE_R_NO_LOAD_FUNCTION}, - #else - {"NO_LOAD_FUNCTION", 38, 125}, - #endif - #ifdef ENGINE_R_NO_REFERENCE - {"NO_REFERENCE", ERR_LIB_ENGINE, ENGINE_R_NO_REFERENCE}, - #else - {"NO_REFERENCE", 38, 130}, - #endif - #ifdef ENGINE_R_NO_SUCH_ENGINE - {"NO_SUCH_ENGINE", ERR_LIB_ENGINE, ENGINE_R_NO_SUCH_ENGINE}, - #else - {"NO_SUCH_ENGINE", 38, 116}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_CIPHER - {"UNIMPLEMENTED_CIPHER", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_CIPHER}, - #else - {"UNIMPLEMENTED_CIPHER", 38, 146}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_DIGEST - {"UNIMPLEMENTED_DIGEST", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_DIGEST}, - #else - {"UNIMPLEMENTED_DIGEST", 38, 147}, - #endif - #ifdef ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD - {"UNIMPLEMENTED_PUBLIC_KEY_METHOD", ERR_LIB_ENGINE, ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD}, - #else - {"UNIMPLEMENTED_PUBLIC_KEY_METHOD", 38, 101}, - #endif - #ifdef ENGINE_R_VERSION_INCOMPATIBILITY - {"VERSION_INCOMPATIBILITY", ERR_LIB_ENGINE, ENGINE_R_VERSION_INCOMPATIBILITY}, - #else - {"VERSION_INCOMPATIBILITY", 38, 145}, - #endif - #ifdef EVP_R_AES_KEY_SETUP_FAILED - {"AES_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_AES_KEY_SETUP_FAILED}, - #else - {"AES_KEY_SETUP_FAILED", 6, 143}, - #endif - #ifdef EVP_R_ARIA_KEY_SETUP_FAILED - {"ARIA_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_ARIA_KEY_SETUP_FAILED}, - #else - {"ARIA_KEY_SETUP_FAILED", 6, 176}, - #endif - #ifdef EVP_R_BAD_DECRYPT - {"BAD_DECRYPT", ERR_LIB_EVP, EVP_R_BAD_DECRYPT}, - #else - {"BAD_DECRYPT", 6, 100}, - #endif - #ifdef EVP_R_BAD_KEY_LENGTH - {"BAD_KEY_LENGTH", ERR_LIB_EVP, EVP_R_BAD_KEY_LENGTH}, - #else - {"BAD_KEY_LENGTH", 6, 195}, - #endif - #ifdef EVP_R_BUFFER_TOO_SMALL - {"BUFFER_TOO_SMALL", ERR_LIB_EVP, EVP_R_BUFFER_TOO_SMALL}, - #else - {"BUFFER_TOO_SMALL", 6, 155}, - #endif - #ifdef EVP_R_CAMELLIA_KEY_SETUP_FAILED - {"CAMELLIA_KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_CAMELLIA_KEY_SETUP_FAILED}, - #else - {"CAMELLIA_KEY_SETUP_FAILED", 6, 157}, - #endif - #ifdef EVP_R_CIPHER_PARAMETER_ERROR - {"CIPHER_PARAMETER_ERROR", ERR_LIB_EVP, EVP_R_CIPHER_PARAMETER_ERROR}, - #else - {"CIPHER_PARAMETER_ERROR", 6, 122}, - #endif - #ifdef EVP_R_COMMAND_NOT_SUPPORTED - {"COMMAND_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED}, - #else - {"COMMAND_NOT_SUPPORTED", 6, 147}, - #endif - #ifdef EVP_R_COPY_ERROR - {"COPY_ERROR", ERR_LIB_EVP, EVP_R_COPY_ERROR}, - #else - {"COPY_ERROR", 6, 173}, - #endif - #ifdef EVP_R_CTRL_NOT_IMPLEMENTED - {"CTRL_NOT_IMPLEMENTED", ERR_LIB_EVP, EVP_R_CTRL_NOT_IMPLEMENTED}, - #else - {"CTRL_NOT_IMPLEMENTED", 6, 132}, - #endif - #ifdef EVP_R_CTRL_OPERATION_NOT_IMPLEMENTED - {"CTRL_OPERATION_NOT_IMPLEMENTED", ERR_LIB_EVP, EVP_R_CTRL_OPERATION_NOT_IMPLEMENTED}, - #else - {"CTRL_OPERATION_NOT_IMPLEMENTED", 6, 133}, - #endif - #ifdef EVP_R_DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH - {"DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", ERR_LIB_EVP, EVP_R_DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH}, - #else - {"DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", 6, 138}, - #endif - #ifdef EVP_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_EVP, EVP_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 6, 114}, - #endif - #ifdef EVP_R_DIFFERENT_KEY_TYPES - {"DIFFERENT_KEY_TYPES", ERR_LIB_EVP, EVP_R_DIFFERENT_KEY_TYPES}, - #else - {"DIFFERENT_KEY_TYPES", 6, 101}, - #endif - #ifdef EVP_R_DIFFERENT_PARAMETERS - {"DIFFERENT_PARAMETERS", ERR_LIB_EVP, EVP_R_DIFFERENT_PARAMETERS}, - #else - {"DIFFERENT_PARAMETERS", 6, 153}, - #endif - #ifdef EVP_R_ERROR_LOADING_SECTION - {"ERROR_LOADING_SECTION", ERR_LIB_EVP, EVP_R_ERROR_LOADING_SECTION}, - #else - {"ERROR_LOADING_SECTION", 6, 165}, - #endif - #ifdef EVP_R_ERROR_SETTING_FIPS_MODE - {"ERROR_SETTING_FIPS_MODE", ERR_LIB_EVP, EVP_R_ERROR_SETTING_FIPS_MODE}, - #else - {"ERROR_SETTING_FIPS_MODE", 6, 166}, - #endif - #ifdef EVP_R_EXPECTING_AN_HMAC_KEY - {"EXPECTING_AN_HMAC_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_AN_HMAC_KEY}, - #else - {"EXPECTING_AN_HMAC_KEY", 6, 174}, - #endif - #ifdef EVP_R_EXPECTING_AN_RSA_KEY - {"EXPECTING_AN_RSA_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_AN_RSA_KEY}, - #else - {"EXPECTING_AN_RSA_KEY", 6, 127}, - #endif - #ifdef EVP_R_EXPECTING_A_DH_KEY - {"EXPECTING_A_DH_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_DH_KEY}, - #else - {"EXPECTING_A_DH_KEY", 6, 128}, - #endif - #ifdef EVP_R_EXPECTING_A_DSA_KEY - {"EXPECTING_A_DSA_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_DSA_KEY}, - #else - {"EXPECTING_A_DSA_KEY", 6, 129}, - #endif - #ifdef EVP_R_EXPECTING_A_EC_KEY - {"EXPECTING_A_EC_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_EC_KEY}, - #else - {"EXPECTING_A_EC_KEY", 6, 142}, - #endif - #ifdef EVP_R_EXPECTING_A_POLY1305_KEY - {"EXPECTING_A_POLY1305_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_POLY1305_KEY}, - #else - {"EXPECTING_A_POLY1305_KEY", 6, 164}, - #endif - #ifdef EVP_R_EXPECTING_A_SIPHASH_KEY - {"EXPECTING_A_SIPHASH_KEY", ERR_LIB_EVP, EVP_R_EXPECTING_A_SIPHASH_KEY}, - #else - {"EXPECTING_A_SIPHASH_KEY", 6, 175}, - #endif - #ifdef EVP_R_FIPS_MODE_NOT_SUPPORTED - {"FIPS_MODE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_FIPS_MODE_NOT_SUPPORTED}, - #else - {"FIPS_MODE_NOT_SUPPORTED", 6, 167}, - #endif - #ifdef EVP_R_GET_RAW_KEY_FAILED - {"GET_RAW_KEY_FAILED", ERR_LIB_EVP, EVP_R_GET_RAW_KEY_FAILED}, - #else - {"GET_RAW_KEY_FAILED", 6, 182}, - #endif - #ifdef EVP_R_ILLEGAL_SCRYPT_PARAMETERS - {"ILLEGAL_SCRYPT_PARAMETERS", ERR_LIB_EVP, EVP_R_ILLEGAL_SCRYPT_PARAMETERS}, - #else - {"ILLEGAL_SCRYPT_PARAMETERS", 6, 171}, - #endif - #ifdef EVP_R_INITIALIZATION_ERROR - {"INITIALIZATION_ERROR", ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR}, - #else - {"INITIALIZATION_ERROR", 6, 134}, - #endif - #ifdef EVP_R_INPUT_NOT_INITIALIZED - {"INPUT_NOT_INITIALIZED", ERR_LIB_EVP, EVP_R_INPUT_NOT_INITIALIZED}, - #else - {"INPUT_NOT_INITIALIZED", 6, 111}, - #endif - #ifdef EVP_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_EVP, EVP_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 6, 152}, - #endif - #ifdef EVP_R_INVALID_FIPS_MODE - {"INVALID_FIPS_MODE", ERR_LIB_EVP, EVP_R_INVALID_FIPS_MODE}, - #else - {"INVALID_FIPS_MODE", 6, 168}, - #endif - #ifdef EVP_R_INVALID_IV_LENGTH - {"INVALID_IV_LENGTH", ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH}, - #else - {"INVALID_IV_LENGTH", 6, 194}, - #endif - #ifdef EVP_R_INVALID_KEY - {"INVALID_KEY", ERR_LIB_EVP, EVP_R_INVALID_KEY}, - #else - {"INVALID_KEY", 6, 163}, - #endif - #ifdef EVP_R_INVALID_KEY_LENGTH - {"INVALID_KEY_LENGTH", ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH}, - #else - {"INVALID_KEY_LENGTH", 6, 130}, - #endif - #ifdef EVP_R_INVALID_OPERATION - {"INVALID_OPERATION", ERR_LIB_EVP, EVP_R_INVALID_OPERATION}, - #else - {"INVALID_OPERATION", 6, 148}, - #endif - #ifdef EVP_R_KEYGEN_FAILURE - {"KEYGEN_FAILURE", ERR_LIB_EVP, EVP_R_KEYGEN_FAILURE}, - #else - {"KEYGEN_FAILURE", 6, 120}, - #endif - #ifdef EVP_R_KEY_SETUP_FAILED - {"KEY_SETUP_FAILED", ERR_LIB_EVP, EVP_R_KEY_SETUP_FAILED}, - #else - {"KEY_SETUP_FAILED", 6, 180}, - #endif - #ifdef EVP_R_MEMORY_LIMIT_EXCEEDED - {"MEMORY_LIMIT_EXCEEDED", ERR_LIB_EVP, EVP_R_MEMORY_LIMIT_EXCEEDED}, - #else - {"MEMORY_LIMIT_EXCEEDED", 6, 172}, - #endif - #ifdef EVP_R_MESSAGE_DIGEST_IS_NULL - {"MESSAGE_DIGEST_IS_NULL", ERR_LIB_EVP, EVP_R_MESSAGE_DIGEST_IS_NULL}, - #else - {"MESSAGE_DIGEST_IS_NULL", 6, 159}, - #endif - #ifdef EVP_R_METHOD_NOT_SUPPORTED - {"METHOD_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_METHOD_NOT_SUPPORTED}, - #else - {"METHOD_NOT_SUPPORTED", 6, 144}, - #endif - #ifdef EVP_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_EVP, EVP_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 6, 103}, - #endif - #ifdef EVP_R_NOT_XOF_OR_INVALID_LENGTH - {"NOT_XOF_OR_INVALID_LENGTH", ERR_LIB_EVP, EVP_R_NOT_XOF_OR_INVALID_LENGTH}, - #else - {"NOT_XOF_OR_INVALID_LENGTH", 6, 178}, - #endif - #ifdef EVP_R_NO_CIPHER_SET - {"NO_CIPHER_SET", ERR_LIB_EVP, EVP_R_NO_CIPHER_SET}, - #else - {"NO_CIPHER_SET", 6, 131}, - #endif - #ifdef EVP_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_EVP, EVP_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 6, 158}, - #endif - #ifdef EVP_R_NO_DIGEST_SET - {"NO_DIGEST_SET", ERR_LIB_EVP, EVP_R_NO_DIGEST_SET}, - #else - {"NO_DIGEST_SET", 6, 139}, - #endif - #ifdef EVP_R_NO_KEY_SET - {"NO_KEY_SET", ERR_LIB_EVP, EVP_R_NO_KEY_SET}, - #else - {"NO_KEY_SET", 6, 154}, - #endif - #ifdef EVP_R_NO_OPERATION_SET - {"NO_OPERATION_SET", ERR_LIB_EVP, EVP_R_NO_OPERATION_SET}, - #else - {"NO_OPERATION_SET", 6, 149}, - #endif - #ifdef EVP_R_ONLY_ONESHOT_SUPPORTED - {"ONLY_ONESHOT_SUPPORTED", ERR_LIB_EVP, EVP_R_ONLY_ONESHOT_SUPPORTED}, - #else - {"ONLY_ONESHOT_SUPPORTED", 6, 177}, - #endif - #ifdef EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, - #else - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 6, 150}, - #endif - #ifdef EVP_R_OPERATON_NOT_INITIALIZED - {"OPERATON_NOT_INITIALIZED", ERR_LIB_EVP, EVP_R_OPERATON_NOT_INITIALIZED}, - #else - {"OPERATON_NOT_INITIALIZED", 6, 151}, - #endif - #ifdef EVP_R_PARTIALLY_OVERLAPPING - {"PARTIALLY_OVERLAPPING", ERR_LIB_EVP, EVP_R_PARTIALLY_OVERLAPPING}, - #else - {"PARTIALLY_OVERLAPPING", 6, 162}, - #endif - #ifdef EVP_R_PBKDF2_ERROR - {"PBKDF2_ERROR", ERR_LIB_EVP, EVP_R_PBKDF2_ERROR}, - #else - {"PBKDF2_ERROR", 6, 181}, - #endif - #ifdef EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED - {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", ERR_LIB_EVP, EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED}, - #else - {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", 6, 179}, - #endif - #ifdef EVP_R_PRIVATE_KEY_DECODE_ERROR - {"PRIVATE_KEY_DECODE_ERROR", ERR_LIB_EVP, EVP_R_PRIVATE_KEY_DECODE_ERROR}, - #else - {"PRIVATE_KEY_DECODE_ERROR", 6, 145}, - #endif - #ifdef EVP_R_PRIVATE_KEY_ENCODE_ERROR - {"PRIVATE_KEY_ENCODE_ERROR", ERR_LIB_EVP, EVP_R_PRIVATE_KEY_ENCODE_ERROR}, - #else - {"PRIVATE_KEY_ENCODE_ERROR", 6, 146}, - #endif - #ifdef EVP_R_PUBLIC_KEY_NOT_RSA - {"PUBLIC_KEY_NOT_RSA", ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA}, - #else - {"PUBLIC_KEY_NOT_RSA", 6, 106}, - #endif - #ifdef EVP_R_UNKNOWN_CIPHER - {"UNKNOWN_CIPHER", ERR_LIB_EVP, EVP_R_UNKNOWN_CIPHER}, - #else - {"UNKNOWN_CIPHER", 6, 160}, - #endif - #ifdef EVP_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_EVP, EVP_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 6, 161}, - #endif - #ifdef EVP_R_UNKNOWN_OPTION - {"UNKNOWN_OPTION", ERR_LIB_EVP, EVP_R_UNKNOWN_OPTION}, - #else - {"UNKNOWN_OPTION", 6, 169}, - #endif - #ifdef EVP_R_UNKNOWN_PBE_ALGORITHM - {"UNKNOWN_PBE_ALGORITHM", ERR_LIB_EVP, EVP_R_UNKNOWN_PBE_ALGORITHM}, - #else - {"UNKNOWN_PBE_ALGORITHM", 6, 121}, - #endif - #ifdef EVP_R_UNSUPPORTED_ALGORITHM - {"UNSUPPORTED_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM}, - #else - {"UNSUPPORTED_ALGORITHM", 6, 156}, - #endif - #ifdef EVP_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_EVP, EVP_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 6, 107}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEYLENGTH - {"UNSUPPORTED_KEYLENGTH", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEYLENGTH}, - #else - {"UNSUPPORTED_KEYLENGTH", 6, 123}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEY_DERIVATION_FUNCTION - {"UNSUPPORTED_KEY_DERIVATION_FUNCTION", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_DERIVATION_FUNCTION}, - #else - {"UNSUPPORTED_KEY_DERIVATION_FUNCTION", 6, 124}, - #endif - #ifdef EVP_R_UNSUPPORTED_KEY_SIZE - {"UNSUPPORTED_KEY_SIZE", ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_SIZE}, - #else - {"UNSUPPORTED_KEY_SIZE", 6, 108}, - #endif - #ifdef EVP_R_UNSUPPORTED_NUMBER_OF_ROUNDS - {"UNSUPPORTED_NUMBER_OF_ROUNDS", ERR_LIB_EVP, EVP_R_UNSUPPORTED_NUMBER_OF_ROUNDS}, - #else - {"UNSUPPORTED_NUMBER_OF_ROUNDS", 6, 135}, - #endif - #ifdef EVP_R_UNSUPPORTED_PRF - {"UNSUPPORTED_PRF", ERR_LIB_EVP, EVP_R_UNSUPPORTED_PRF}, - #else - {"UNSUPPORTED_PRF", 6, 125}, - #endif - #ifdef EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM - {"UNSUPPORTED_PRIVATE_KEY_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM}, - #else - {"UNSUPPORTED_PRIVATE_KEY_ALGORITHM", 6, 118}, - #endif - #ifdef EVP_R_UNSUPPORTED_SALT_TYPE - {"UNSUPPORTED_SALT_TYPE", ERR_LIB_EVP, EVP_R_UNSUPPORTED_SALT_TYPE}, - #else - {"UNSUPPORTED_SALT_TYPE", 6, 126}, - #endif - #ifdef EVP_R_WRAP_MODE_NOT_ALLOWED - {"WRAP_MODE_NOT_ALLOWED", ERR_LIB_EVP, EVP_R_WRAP_MODE_NOT_ALLOWED}, - #else - {"WRAP_MODE_NOT_ALLOWED", 6, 170}, - #endif - #ifdef EVP_R_WRONG_FINAL_BLOCK_LENGTH - {"WRONG_FINAL_BLOCK_LENGTH", ERR_LIB_EVP, EVP_R_WRONG_FINAL_BLOCK_LENGTH}, - #else - {"WRONG_FINAL_BLOCK_LENGTH", 6, 109}, - #endif - #ifdef EVP_R_XTS_DUPLICATED_KEYS - {"XTS_DUPLICATED_KEYS", ERR_LIB_EVP, EVP_R_XTS_DUPLICATED_KEYS}, - #else - {"XTS_DUPLICATED_KEYS", 6, 183}, - #endif - #ifdef KDF_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_KDF, KDF_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 52, 100}, - #endif - #ifdef KDF_R_MISSING_ITERATION_COUNT - {"MISSING_ITERATION_COUNT", ERR_LIB_KDF, KDF_R_MISSING_ITERATION_COUNT}, - #else - {"MISSING_ITERATION_COUNT", 52, 109}, - #endif - #ifdef KDF_R_MISSING_KEY - {"MISSING_KEY", ERR_LIB_KDF, KDF_R_MISSING_KEY}, - #else - {"MISSING_KEY", 52, 104}, - #endif - #ifdef KDF_R_MISSING_MESSAGE_DIGEST - {"MISSING_MESSAGE_DIGEST", ERR_LIB_KDF, KDF_R_MISSING_MESSAGE_DIGEST}, - #else - {"MISSING_MESSAGE_DIGEST", 52, 105}, - #endif - #ifdef KDF_R_MISSING_PARAMETER - {"MISSING_PARAMETER", ERR_LIB_KDF, KDF_R_MISSING_PARAMETER}, - #else - {"MISSING_PARAMETER", 52, 101}, - #endif - #ifdef KDF_R_MISSING_PASS - {"MISSING_PASS", ERR_LIB_KDF, KDF_R_MISSING_PASS}, - #else - {"MISSING_PASS", 52, 110}, - #endif - #ifdef KDF_R_MISSING_SALT - {"MISSING_SALT", ERR_LIB_KDF, KDF_R_MISSING_SALT}, - #else - {"MISSING_SALT", 52, 111}, - #endif - #ifdef KDF_R_MISSING_SECRET - {"MISSING_SECRET", ERR_LIB_KDF, KDF_R_MISSING_SECRET}, - #else - {"MISSING_SECRET", 52, 107}, - #endif - #ifdef KDF_R_MISSING_SEED - {"MISSING_SEED", ERR_LIB_KDF, KDF_R_MISSING_SEED}, - #else - {"MISSING_SEED", 52, 106}, - #endif - #ifdef KDF_R_UNKNOWN_PARAMETER_TYPE - {"UNKNOWN_PARAMETER_TYPE", ERR_LIB_KDF, KDF_R_UNKNOWN_PARAMETER_TYPE}, - #else - {"UNKNOWN_PARAMETER_TYPE", 52, 103}, - #endif - #ifdef KDF_R_VALUE_ERROR - {"VALUE_ERROR", ERR_LIB_KDF, KDF_R_VALUE_ERROR}, - #else - {"VALUE_ERROR", 52, 108}, - #endif - #ifdef KDF_R_VALUE_MISSING - {"VALUE_MISSING", ERR_LIB_KDF, KDF_R_VALUE_MISSING}, - #else - {"VALUE_MISSING", 52, 102}, - #endif - #ifdef OCSP_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_OCSP, OCSP_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 39, 101}, - #endif - #ifdef OCSP_R_DIGEST_ERR - {"DIGEST_ERR", ERR_LIB_OCSP, OCSP_R_DIGEST_ERR}, - #else - {"DIGEST_ERR", 39, 102}, - #endif - #ifdef OCSP_R_ERROR_IN_NEXTUPDATE_FIELD - {"ERROR_IN_NEXTUPDATE_FIELD", ERR_LIB_OCSP, OCSP_R_ERROR_IN_NEXTUPDATE_FIELD}, - #else - {"ERROR_IN_NEXTUPDATE_FIELD", 39, 122}, - #endif - #ifdef OCSP_R_ERROR_IN_THISUPDATE_FIELD - {"ERROR_IN_THISUPDATE_FIELD", ERR_LIB_OCSP, OCSP_R_ERROR_IN_THISUPDATE_FIELD}, - #else - {"ERROR_IN_THISUPDATE_FIELD", 39, 123}, - #endif - #ifdef OCSP_R_ERROR_PARSING_URL - {"ERROR_PARSING_URL", ERR_LIB_OCSP, OCSP_R_ERROR_PARSING_URL}, - #else - {"ERROR_PARSING_URL", 39, 121}, - #endif - #ifdef OCSP_R_MISSING_OCSPSIGNING_USAGE - {"MISSING_OCSPSIGNING_USAGE", ERR_LIB_OCSP, OCSP_R_MISSING_OCSPSIGNING_USAGE}, - #else - {"MISSING_OCSPSIGNING_USAGE", 39, 103}, - #endif - #ifdef OCSP_R_NEXTUPDATE_BEFORE_THISUPDATE - {"NEXTUPDATE_BEFORE_THISUPDATE", ERR_LIB_OCSP, OCSP_R_NEXTUPDATE_BEFORE_THISUPDATE}, - #else - {"NEXTUPDATE_BEFORE_THISUPDATE", 39, 124}, - #endif - #ifdef OCSP_R_NOT_BASIC_RESPONSE - {"NOT_BASIC_RESPONSE", ERR_LIB_OCSP, OCSP_R_NOT_BASIC_RESPONSE}, - #else - {"NOT_BASIC_RESPONSE", 39, 104}, - #endif - #ifdef OCSP_R_NO_CERTIFICATES_IN_CHAIN - {"NO_CERTIFICATES_IN_CHAIN", ERR_LIB_OCSP, OCSP_R_NO_CERTIFICATES_IN_CHAIN}, - #else - {"NO_CERTIFICATES_IN_CHAIN", 39, 105}, - #endif - #ifdef OCSP_R_NO_RESPONSE_DATA - {"NO_RESPONSE_DATA", ERR_LIB_OCSP, OCSP_R_NO_RESPONSE_DATA}, - #else - {"NO_RESPONSE_DATA", 39, 108}, - #endif - #ifdef OCSP_R_NO_REVOKED_TIME - {"NO_REVOKED_TIME", ERR_LIB_OCSP, OCSP_R_NO_REVOKED_TIME}, - #else - {"NO_REVOKED_TIME", 39, 109}, - #endif - #ifdef OCSP_R_NO_SIGNER_KEY - {"NO_SIGNER_KEY", ERR_LIB_OCSP, OCSP_R_NO_SIGNER_KEY}, - #else - {"NO_SIGNER_KEY", 39, 130}, - #endif - #ifdef OCSP_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_OCSP, OCSP_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 39, 110}, - #endif - #ifdef OCSP_R_REQUEST_NOT_SIGNED - {"REQUEST_NOT_SIGNED", ERR_LIB_OCSP, OCSP_R_REQUEST_NOT_SIGNED}, - #else - {"REQUEST_NOT_SIGNED", 39, 128}, - #endif - #ifdef OCSP_R_RESPONSE_CONTAINS_NO_REVOCATION_DATA - {"RESPONSE_CONTAINS_NO_REVOCATION_DATA", ERR_LIB_OCSP, OCSP_R_RESPONSE_CONTAINS_NO_REVOCATION_DATA}, - #else - {"RESPONSE_CONTAINS_NO_REVOCATION_DATA", 39, 111}, - #endif - #ifdef OCSP_R_ROOT_CA_NOT_TRUSTED - {"ROOT_CA_NOT_TRUSTED", ERR_LIB_OCSP, OCSP_R_ROOT_CA_NOT_TRUSTED}, - #else - {"ROOT_CA_NOT_TRUSTED", 39, 112}, - #endif - #ifdef OCSP_R_SERVER_RESPONSE_ERROR - {"SERVER_RESPONSE_ERROR", ERR_LIB_OCSP, OCSP_R_SERVER_RESPONSE_ERROR}, - #else - {"SERVER_RESPONSE_ERROR", 39, 114}, - #endif - #ifdef OCSP_R_SERVER_RESPONSE_PARSE_ERROR - {"SERVER_RESPONSE_PARSE_ERROR", ERR_LIB_OCSP, OCSP_R_SERVER_RESPONSE_PARSE_ERROR}, - #else - {"SERVER_RESPONSE_PARSE_ERROR", 39, 115}, - #endif - #ifdef OCSP_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_OCSP, OCSP_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 39, 117}, - #endif - #ifdef OCSP_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_OCSP, OCSP_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 39, 118}, - #endif - #ifdef OCSP_R_STATUS_EXPIRED - {"STATUS_EXPIRED", ERR_LIB_OCSP, OCSP_R_STATUS_EXPIRED}, - #else - {"STATUS_EXPIRED", 39, 125}, - #endif - #ifdef OCSP_R_STATUS_NOT_YET_VALID - {"STATUS_NOT_YET_VALID", ERR_LIB_OCSP, OCSP_R_STATUS_NOT_YET_VALID}, - #else - {"STATUS_NOT_YET_VALID", 39, 126}, - #endif - #ifdef OCSP_R_STATUS_TOO_OLD - {"STATUS_TOO_OLD", ERR_LIB_OCSP, OCSP_R_STATUS_TOO_OLD}, - #else - {"STATUS_TOO_OLD", 39, 127}, - #endif - #ifdef OCSP_R_UNKNOWN_MESSAGE_DIGEST - {"UNKNOWN_MESSAGE_DIGEST", ERR_LIB_OCSP, OCSP_R_UNKNOWN_MESSAGE_DIGEST}, - #else - {"UNKNOWN_MESSAGE_DIGEST", 39, 119}, - #endif - #ifdef OCSP_R_UNKNOWN_NID - {"UNKNOWN_NID", ERR_LIB_OCSP, OCSP_R_UNKNOWN_NID}, - #else - {"UNKNOWN_NID", 39, 120}, - #endif - #ifdef OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE - {"UNSUPPORTED_REQUESTORNAME_TYPE", ERR_LIB_OCSP, OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE}, - #else - {"UNSUPPORTED_REQUESTORNAME_TYPE", 39, 129}, - #endif - #ifdef PEM_R_BAD_BASE64_DECODE - {"BAD_BASE64_DECODE", ERR_LIB_PEM, PEM_R_BAD_BASE64_DECODE}, - #else - {"BAD_BASE64_DECODE", 9, 100}, - #endif - #ifdef PEM_R_BAD_DECRYPT - {"BAD_DECRYPT", ERR_LIB_PEM, PEM_R_BAD_DECRYPT}, - #else - {"BAD_DECRYPT", 9, 101}, - #endif - #ifdef PEM_R_BAD_END_LINE - {"BAD_END_LINE", ERR_LIB_PEM, PEM_R_BAD_END_LINE}, - #else - {"BAD_END_LINE", 9, 102}, - #endif - #ifdef PEM_R_BAD_IV_CHARS - {"BAD_IV_CHARS", ERR_LIB_PEM, PEM_R_BAD_IV_CHARS}, - #else - {"BAD_IV_CHARS", 9, 103}, - #endif - #ifdef PEM_R_BAD_MAGIC_NUMBER - {"BAD_MAGIC_NUMBER", ERR_LIB_PEM, PEM_R_BAD_MAGIC_NUMBER}, - #else - {"BAD_MAGIC_NUMBER", 9, 116}, - #endif - #ifdef PEM_R_BAD_PASSWORD_READ - {"BAD_PASSWORD_READ", ERR_LIB_PEM, PEM_R_BAD_PASSWORD_READ}, - #else - {"BAD_PASSWORD_READ", 9, 104}, - #endif - #ifdef PEM_R_BAD_VERSION_NUMBER - {"BAD_VERSION_NUMBER", ERR_LIB_PEM, PEM_R_BAD_VERSION_NUMBER}, - #else - {"BAD_VERSION_NUMBER", 9, 117}, - #endif - #ifdef PEM_R_BIO_WRITE_FAILURE - {"BIO_WRITE_FAILURE", ERR_LIB_PEM, PEM_R_BIO_WRITE_FAILURE}, - #else - {"BIO_WRITE_FAILURE", 9, 118}, - #endif - #ifdef PEM_R_CIPHER_IS_NULL - {"CIPHER_IS_NULL", ERR_LIB_PEM, PEM_R_CIPHER_IS_NULL}, - #else - {"CIPHER_IS_NULL", 9, 127}, - #endif - #ifdef PEM_R_ERROR_CONVERTING_PRIVATE_KEY - {"ERROR_CONVERTING_PRIVATE_KEY", ERR_LIB_PEM, PEM_R_ERROR_CONVERTING_PRIVATE_KEY}, - #else - {"ERROR_CONVERTING_PRIVATE_KEY", 9, 115}, - #endif - #ifdef PEM_R_EXPECTING_PRIVATE_KEY_BLOB - {"EXPECTING_PRIVATE_KEY_BLOB", ERR_LIB_PEM, PEM_R_EXPECTING_PRIVATE_KEY_BLOB}, - #else - {"EXPECTING_PRIVATE_KEY_BLOB", 9, 119}, - #endif - #ifdef PEM_R_EXPECTING_PUBLIC_KEY_BLOB - {"EXPECTING_PUBLIC_KEY_BLOB", ERR_LIB_PEM, PEM_R_EXPECTING_PUBLIC_KEY_BLOB}, - #else - {"EXPECTING_PUBLIC_KEY_BLOB", 9, 120}, - #endif - #ifdef PEM_R_HEADER_TOO_LONG - {"HEADER_TOO_LONG", ERR_LIB_PEM, PEM_R_HEADER_TOO_LONG}, - #else - {"HEADER_TOO_LONG", 9, 128}, - #endif - #ifdef PEM_R_INCONSISTENT_HEADER - {"INCONSISTENT_HEADER", ERR_LIB_PEM, PEM_R_INCONSISTENT_HEADER}, - #else - {"INCONSISTENT_HEADER", 9, 121}, - #endif - #ifdef PEM_R_KEYBLOB_HEADER_PARSE_ERROR - {"KEYBLOB_HEADER_PARSE_ERROR", ERR_LIB_PEM, PEM_R_KEYBLOB_HEADER_PARSE_ERROR}, - #else - {"KEYBLOB_HEADER_PARSE_ERROR", 9, 122}, - #endif - #ifdef PEM_R_KEYBLOB_TOO_SHORT - {"KEYBLOB_TOO_SHORT", ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT}, - #else - {"KEYBLOB_TOO_SHORT", 9, 123}, - #endif - #ifdef PEM_R_MISSING_DEK_IV - {"MISSING_DEK_IV", ERR_LIB_PEM, PEM_R_MISSING_DEK_IV}, - #else - {"MISSING_DEK_IV", 9, 129}, - #endif - #ifdef PEM_R_NOT_DEK_INFO - {"NOT_DEK_INFO", ERR_LIB_PEM, PEM_R_NOT_DEK_INFO}, - #else - {"NOT_DEK_INFO", 9, 105}, - #endif - #ifdef PEM_R_NOT_ENCRYPTED - {"NOT_ENCRYPTED", ERR_LIB_PEM, PEM_R_NOT_ENCRYPTED}, - #else - {"NOT_ENCRYPTED", 9, 106}, - #endif - #ifdef PEM_R_NOT_PROC_TYPE - {"NOT_PROC_TYPE", ERR_LIB_PEM, PEM_R_NOT_PROC_TYPE}, - #else - {"NOT_PROC_TYPE", 9, 107}, - #endif - #ifdef PEM_R_NO_START_LINE - {"NO_START_LINE", ERR_LIB_PEM, PEM_R_NO_START_LINE}, - #else - {"NO_START_LINE", 9, 108}, - #endif - #ifdef PEM_R_PROBLEMS_GETTING_PASSWORD - {"PROBLEMS_GETTING_PASSWORD", ERR_LIB_PEM, PEM_R_PROBLEMS_GETTING_PASSWORD}, - #else - {"PROBLEMS_GETTING_PASSWORD", 9, 109}, - #endif - #ifdef PEM_R_PUBLIC_KEY_NO_RSA - {"PUBLIC_KEY_NO_RSA", ERR_LIB_PEM, PEM_R_PUBLIC_KEY_NO_RSA}, - #else - {"PUBLIC_KEY_NO_RSA", 9, 110}, - #endif - #ifdef PEM_R_PVK_DATA_TOO_SHORT - {"PVK_DATA_TOO_SHORT", ERR_LIB_PEM, PEM_R_PVK_DATA_TOO_SHORT}, - #else - {"PVK_DATA_TOO_SHORT", 9, 124}, - #endif - #ifdef PEM_R_PVK_TOO_SHORT - {"PVK_TOO_SHORT", ERR_LIB_PEM, PEM_R_PVK_TOO_SHORT}, - #else - {"PVK_TOO_SHORT", 9, 125}, - #endif - #ifdef PEM_R_READ_KEY - {"READ_KEY", ERR_LIB_PEM, PEM_R_READ_KEY}, - #else - {"READ_KEY", 9, 111}, - #endif - #ifdef PEM_R_SHORT_HEADER - {"SHORT_HEADER", ERR_LIB_PEM, PEM_R_SHORT_HEADER}, - #else - {"SHORT_HEADER", 9, 112}, - #endif - #ifdef PEM_R_UNEXPECTED_DEK_IV - {"UNEXPECTED_DEK_IV", ERR_LIB_PEM, PEM_R_UNEXPECTED_DEK_IV}, - #else - {"UNEXPECTED_DEK_IV", 9, 130}, - #endif - #ifdef PEM_R_UNSUPPORTED_CIPHER - {"UNSUPPORTED_CIPHER", ERR_LIB_PEM, PEM_R_UNSUPPORTED_CIPHER}, - #else - {"UNSUPPORTED_CIPHER", 9, 113}, - #endif - #ifdef PEM_R_UNSUPPORTED_ENCRYPTION - {"UNSUPPORTED_ENCRYPTION", ERR_LIB_PEM, PEM_R_UNSUPPORTED_ENCRYPTION}, - #else - {"UNSUPPORTED_ENCRYPTION", 9, 114}, - #endif - #ifdef PEM_R_UNSUPPORTED_KEY_COMPONENTS - {"UNSUPPORTED_KEY_COMPONENTS", ERR_LIB_PEM, PEM_R_UNSUPPORTED_KEY_COMPONENTS}, - #else - {"UNSUPPORTED_KEY_COMPONENTS", 9, 126}, - #endif - #ifdef PKCS12_R_CANT_PACK_STRUCTURE - {"CANT_PACK_STRUCTURE", ERR_LIB_PKCS12, PKCS12_R_CANT_PACK_STRUCTURE}, - #else - {"CANT_PACK_STRUCTURE", 35, 100}, - #endif - #ifdef PKCS12_R_CONTENT_TYPE_NOT_DATA - {"CONTENT_TYPE_NOT_DATA", ERR_LIB_PKCS12, PKCS12_R_CONTENT_TYPE_NOT_DATA}, - #else - {"CONTENT_TYPE_NOT_DATA", 35, 121}, - #endif - #ifdef PKCS12_R_DECODE_ERROR - {"DECODE_ERROR", ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR}, - #else - {"DECODE_ERROR", 35, 101}, - #endif - #ifdef PKCS12_R_ENCODE_ERROR - {"ENCODE_ERROR", ERR_LIB_PKCS12, PKCS12_R_ENCODE_ERROR}, - #else - {"ENCODE_ERROR", 35, 102}, - #endif - #ifdef PKCS12_R_ENCRYPT_ERROR - {"ENCRYPT_ERROR", ERR_LIB_PKCS12, PKCS12_R_ENCRYPT_ERROR}, - #else - {"ENCRYPT_ERROR", 35, 103}, - #endif - #ifdef PKCS12_R_ERROR_SETTING_ENCRYPTED_DATA_TYPE - {"ERROR_SETTING_ENCRYPTED_DATA_TYPE", ERR_LIB_PKCS12, PKCS12_R_ERROR_SETTING_ENCRYPTED_DATA_TYPE}, - #else - {"ERROR_SETTING_ENCRYPTED_DATA_TYPE", 35, 120}, - #endif - #ifdef PKCS12_R_INVALID_NULL_ARGUMENT - {"INVALID_NULL_ARGUMENT", ERR_LIB_PKCS12, PKCS12_R_INVALID_NULL_ARGUMENT}, - #else - {"INVALID_NULL_ARGUMENT", 35, 104}, - #endif - #ifdef PKCS12_R_INVALID_NULL_PKCS12_POINTER - {"INVALID_NULL_PKCS12_POINTER", ERR_LIB_PKCS12, PKCS12_R_INVALID_NULL_PKCS12_POINTER}, - #else - {"INVALID_NULL_PKCS12_POINTER", 35, 105}, - #endif - #ifdef PKCS12_R_IV_GEN_ERROR - {"IV_GEN_ERROR", ERR_LIB_PKCS12, PKCS12_R_IV_GEN_ERROR}, - #else - {"IV_GEN_ERROR", 35, 106}, - #endif - #ifdef PKCS12_R_KEY_GEN_ERROR - {"KEY_GEN_ERROR", ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR}, - #else - {"KEY_GEN_ERROR", 35, 107}, - #endif - #ifdef PKCS12_R_MAC_ABSENT - {"MAC_ABSENT", ERR_LIB_PKCS12, PKCS12_R_MAC_ABSENT}, - #else - {"MAC_ABSENT", 35, 108}, - #endif - #ifdef PKCS12_R_MAC_GENERATION_ERROR - {"MAC_GENERATION_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_GENERATION_ERROR}, - #else - {"MAC_GENERATION_ERROR", 35, 109}, - #endif - #ifdef PKCS12_R_MAC_SETUP_ERROR - {"MAC_SETUP_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_SETUP_ERROR}, - #else - {"MAC_SETUP_ERROR", 35, 110}, - #endif - #ifdef PKCS12_R_MAC_STRING_SET_ERROR - {"MAC_STRING_SET_ERROR", ERR_LIB_PKCS12, PKCS12_R_MAC_STRING_SET_ERROR}, - #else - {"MAC_STRING_SET_ERROR", 35, 111}, - #endif - #ifdef PKCS12_R_MAC_VERIFY_FAILURE - {"MAC_VERIFY_FAILURE", ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE}, - #else - {"MAC_VERIFY_FAILURE", 35, 113}, - #endif - #ifdef PKCS12_R_PARSE_ERROR - {"PARSE_ERROR", ERR_LIB_PKCS12, PKCS12_R_PARSE_ERROR}, - #else - {"PARSE_ERROR", 35, 114}, - #endif - #ifdef PKCS12_R_PKCS12_ALGOR_CIPHERINIT_ERROR - {"PKCS12_ALGOR_CIPHERINIT_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_ALGOR_CIPHERINIT_ERROR}, - #else - {"PKCS12_ALGOR_CIPHERINIT_ERROR", 35, 115}, - #endif - #ifdef PKCS12_R_PKCS12_CIPHERFINAL_ERROR - {"PKCS12_CIPHERFINAL_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_CIPHERFINAL_ERROR}, - #else - {"PKCS12_CIPHERFINAL_ERROR", 35, 116}, - #endif - #ifdef PKCS12_R_PKCS12_PBE_CRYPT_ERROR - {"PKCS12_PBE_CRYPT_ERROR", ERR_LIB_PKCS12, PKCS12_R_PKCS12_PBE_CRYPT_ERROR}, - #else - {"PKCS12_PBE_CRYPT_ERROR", 35, 117}, - #endif - #ifdef PKCS12_R_UNKNOWN_DIGEST_ALGORITHM - {"UNKNOWN_DIGEST_ALGORITHM", ERR_LIB_PKCS12, PKCS12_R_UNKNOWN_DIGEST_ALGORITHM}, - #else - {"UNKNOWN_DIGEST_ALGORITHM", 35, 118}, - #endif - #ifdef PKCS12_R_UNSUPPORTED_PKCS12_MODE - {"UNSUPPORTED_PKCS12_MODE", ERR_LIB_PKCS12, PKCS12_R_UNSUPPORTED_PKCS12_MODE}, - #else - {"UNSUPPORTED_PKCS12_MODE", 35, 119}, - #endif - #ifdef PKCS7_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_PKCS7, PKCS7_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 33, 117}, - #endif - #ifdef PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", ERR_LIB_PKCS7, PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER}, - #else - {"CIPHER_HAS_NO_OBJECT_IDENTIFIER", 33, 144}, - #endif - #ifdef PKCS7_R_CIPHER_NOT_INITIALIZED - {"CIPHER_NOT_INITIALIZED", ERR_LIB_PKCS7, PKCS7_R_CIPHER_NOT_INITIALIZED}, - #else - {"CIPHER_NOT_INITIALIZED", 33, 116}, - #endif - #ifdef PKCS7_R_CONTENT_AND_DATA_PRESENT - {"CONTENT_AND_DATA_PRESENT", ERR_LIB_PKCS7, PKCS7_R_CONTENT_AND_DATA_PRESENT}, - #else - {"CONTENT_AND_DATA_PRESENT", 33, 118}, - #endif - #ifdef PKCS7_R_CTRL_ERROR - {"CTRL_ERROR", ERR_LIB_PKCS7, PKCS7_R_CTRL_ERROR}, - #else - {"CTRL_ERROR", 33, 152}, - #endif - #ifdef PKCS7_R_DECRYPT_ERROR - {"DECRYPT_ERROR", ERR_LIB_PKCS7, PKCS7_R_DECRYPT_ERROR}, - #else - {"DECRYPT_ERROR", 33, 119}, - #endif - #ifdef PKCS7_R_DIGEST_FAILURE - {"DIGEST_FAILURE", ERR_LIB_PKCS7, PKCS7_R_DIGEST_FAILURE}, - #else - {"DIGEST_FAILURE", 33, 101}, - #endif - #ifdef PKCS7_R_ENCRYPTION_CTRL_FAILURE - {"ENCRYPTION_CTRL_FAILURE", ERR_LIB_PKCS7, PKCS7_R_ENCRYPTION_CTRL_FAILURE}, - #else - {"ENCRYPTION_CTRL_FAILURE", 33, 149}, - #endif - #ifdef PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_PKCS7, PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 33, 150}, - #endif - #ifdef PKCS7_R_ERROR_ADDING_RECIPIENT - {"ERROR_ADDING_RECIPIENT", ERR_LIB_PKCS7, PKCS7_R_ERROR_ADDING_RECIPIENT}, - #else - {"ERROR_ADDING_RECIPIENT", 33, 120}, - #endif - #ifdef PKCS7_R_ERROR_SETTING_CIPHER - {"ERROR_SETTING_CIPHER", ERR_LIB_PKCS7, PKCS7_R_ERROR_SETTING_CIPHER}, - #else - {"ERROR_SETTING_CIPHER", 33, 121}, - #endif - #ifdef PKCS7_R_INVALID_NULL_POINTER - {"INVALID_NULL_POINTER", ERR_LIB_PKCS7, PKCS7_R_INVALID_NULL_POINTER}, - #else - {"INVALID_NULL_POINTER", 33, 143}, - #endif - #ifdef PKCS7_R_INVALID_SIGNED_DATA_TYPE - {"INVALID_SIGNED_DATA_TYPE", ERR_LIB_PKCS7, PKCS7_R_INVALID_SIGNED_DATA_TYPE}, - #else - {"INVALID_SIGNED_DATA_TYPE", 33, 155}, - #endif - #ifdef PKCS7_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_PKCS7, PKCS7_R_NO_CONTENT}, - #else - {"NO_CONTENT", 33, 122}, - #endif - #ifdef PKCS7_R_NO_DEFAULT_DIGEST - {"NO_DEFAULT_DIGEST", ERR_LIB_PKCS7, PKCS7_R_NO_DEFAULT_DIGEST}, - #else - {"NO_DEFAULT_DIGEST", 33, 151}, - #endif - #ifdef PKCS7_R_NO_MATCHING_DIGEST_TYPE_FOUND - {"NO_MATCHING_DIGEST_TYPE_FOUND", ERR_LIB_PKCS7, PKCS7_R_NO_MATCHING_DIGEST_TYPE_FOUND}, - #else - {"NO_MATCHING_DIGEST_TYPE_FOUND", 33, 154}, - #endif - #ifdef PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE - {"NO_RECIPIENT_MATCHES_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE}, - #else - {"NO_RECIPIENT_MATCHES_CERTIFICATE", 33, 115}, - #endif - #ifdef PKCS7_R_NO_SIGNATURES_ON_DATA - {"NO_SIGNATURES_ON_DATA", ERR_LIB_PKCS7, PKCS7_R_NO_SIGNATURES_ON_DATA}, - #else - {"NO_SIGNATURES_ON_DATA", 33, 123}, - #endif - #ifdef PKCS7_R_NO_SIGNERS - {"NO_SIGNERS", ERR_LIB_PKCS7, PKCS7_R_NO_SIGNERS}, - #else - {"NO_SIGNERS", 33, 142}, - #endif - #ifdef PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE - {"OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", ERR_LIB_PKCS7, PKCS7_R_OPERATION_NOT_SUPPORTED_ON_THIS_TYPE}, - #else - {"OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", 33, 104}, - #endif - #ifdef PKCS7_R_PKCS7_ADD_SIGNATURE_ERROR - {"PKCS7_ADD_SIGNATURE_ERROR", ERR_LIB_PKCS7, PKCS7_R_PKCS7_ADD_SIGNATURE_ERROR}, - #else - {"PKCS7_ADD_SIGNATURE_ERROR", 33, 124}, - #endif - #ifdef PKCS7_R_PKCS7_ADD_SIGNER_ERROR - {"PKCS7_ADD_SIGNER_ERROR", ERR_LIB_PKCS7, PKCS7_R_PKCS7_ADD_SIGNER_ERROR}, - #else - {"PKCS7_ADD_SIGNER_ERROR", 33, 153}, - #endif - #ifdef PKCS7_R_PKCS7_DATASIGN - {"PKCS7_DATASIGN", ERR_LIB_PKCS7, PKCS7_R_PKCS7_DATASIGN}, - #else - {"PKCS7_DATASIGN", 33, 145}, - #endif - #ifdef PKCS7_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 33, 127}, - #endif - #ifdef PKCS7_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_PKCS7, PKCS7_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 33, 105}, - #endif - #ifdef PKCS7_R_SIGNER_CERTIFICATE_NOT_FOUND - {"SIGNER_CERTIFICATE_NOT_FOUND", ERR_LIB_PKCS7, PKCS7_R_SIGNER_CERTIFICATE_NOT_FOUND}, - #else - {"SIGNER_CERTIFICATE_NOT_FOUND", 33, 128}, - #endif - #ifdef PKCS7_R_SIGNING_CTRL_FAILURE - {"SIGNING_CTRL_FAILURE", ERR_LIB_PKCS7, PKCS7_R_SIGNING_CTRL_FAILURE}, - #else - {"SIGNING_CTRL_FAILURE", 33, 147}, - #endif - #ifdef PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE - {"SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", ERR_LIB_PKCS7, PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE}, - #else - {"SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", 33, 148}, - #endif - #ifdef PKCS7_R_SMIME_TEXT_ERROR - {"SMIME_TEXT_ERROR", ERR_LIB_PKCS7, PKCS7_R_SMIME_TEXT_ERROR}, - #else - {"SMIME_TEXT_ERROR", 33, 129}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_CERTIFICATE - {"UNABLE_TO_FIND_CERTIFICATE", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_CERTIFICATE}, - #else - {"UNABLE_TO_FIND_CERTIFICATE", 33, 106}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_MEM_BIO - {"UNABLE_TO_FIND_MEM_BIO", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_MEM_BIO}, - #else - {"UNABLE_TO_FIND_MEM_BIO", 33, 107}, - #endif - #ifdef PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST - {"UNABLE_TO_FIND_MESSAGE_DIGEST", ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST}, - #else - {"UNABLE_TO_FIND_MESSAGE_DIGEST", 33, 108}, - #endif - #ifdef PKCS7_R_UNKNOWN_DIGEST_TYPE - {"UNKNOWN_DIGEST_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNKNOWN_DIGEST_TYPE}, - #else - {"UNKNOWN_DIGEST_TYPE", 33, 109}, - #endif - #ifdef PKCS7_R_UNKNOWN_OPERATION - {"UNKNOWN_OPERATION", ERR_LIB_PKCS7, PKCS7_R_UNKNOWN_OPERATION}, - #else - {"UNKNOWN_OPERATION", 33, 110}, - #endif - #ifdef PKCS7_R_UNSUPPORTED_CIPHER_TYPE - {"UNSUPPORTED_CIPHER_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNSUPPORTED_CIPHER_TYPE}, - #else - {"UNSUPPORTED_CIPHER_TYPE", 33, 111}, - #endif - #ifdef PKCS7_R_UNSUPPORTED_CONTENT_TYPE - {"UNSUPPORTED_CONTENT_TYPE", ERR_LIB_PKCS7, PKCS7_R_UNSUPPORTED_CONTENT_TYPE}, - #else - {"UNSUPPORTED_CONTENT_TYPE", 33, 112}, - #endif - #ifdef PKCS7_R_WRONG_CONTENT_TYPE - {"WRONG_CONTENT_TYPE", ERR_LIB_PKCS7, PKCS7_R_WRONG_CONTENT_TYPE}, - #else - {"WRONG_CONTENT_TYPE", 33, 113}, - #endif - #ifdef PKCS7_R_WRONG_PKCS7_TYPE - {"WRONG_PKCS7_TYPE", ERR_LIB_PKCS7, PKCS7_R_WRONG_PKCS7_TYPE}, - #else - {"WRONG_PKCS7_TYPE", 33, 114}, - #endif - #ifdef RAND_R_ADDITIONAL_INPUT_TOO_LONG - {"ADDITIONAL_INPUT_TOO_LONG", ERR_LIB_RAND, RAND_R_ADDITIONAL_INPUT_TOO_LONG}, - #else - {"ADDITIONAL_INPUT_TOO_LONG", 36, 102}, - #endif - #ifdef RAND_R_ALREADY_INSTANTIATED - {"ALREADY_INSTANTIATED", ERR_LIB_RAND, RAND_R_ALREADY_INSTANTIATED}, - #else - {"ALREADY_INSTANTIATED", 36, 103}, - #endif - #ifdef RAND_R_ARGUMENT_OUT_OF_RANGE - {"ARGUMENT_OUT_OF_RANGE", ERR_LIB_RAND, RAND_R_ARGUMENT_OUT_OF_RANGE}, - #else - {"ARGUMENT_OUT_OF_RANGE", 36, 105}, - #endif - #ifdef RAND_R_CANNOT_OPEN_FILE - {"CANNOT_OPEN_FILE", ERR_LIB_RAND, RAND_R_CANNOT_OPEN_FILE}, - #else - {"CANNOT_OPEN_FILE", 36, 121}, - #endif - #ifdef RAND_R_DRBG_ALREADY_INITIALIZED - {"DRBG_ALREADY_INITIALIZED", ERR_LIB_RAND, RAND_R_DRBG_ALREADY_INITIALIZED}, - #else - {"DRBG_ALREADY_INITIALIZED", 36, 129}, - #endif - #ifdef RAND_R_DRBG_NOT_INITIALISED - {"DRBG_NOT_INITIALISED", ERR_LIB_RAND, RAND_R_DRBG_NOT_INITIALISED}, - #else - {"DRBG_NOT_INITIALISED", 36, 104}, - #endif - #ifdef RAND_R_ENTROPY_INPUT_TOO_LONG - {"ENTROPY_INPUT_TOO_LONG", ERR_LIB_RAND, RAND_R_ENTROPY_INPUT_TOO_LONG}, - #else - {"ENTROPY_INPUT_TOO_LONG", 36, 106}, - #endif - #ifdef RAND_R_ENTROPY_OUT_OF_RANGE - {"ENTROPY_OUT_OF_RANGE", ERR_LIB_RAND, RAND_R_ENTROPY_OUT_OF_RANGE}, - #else - {"ENTROPY_OUT_OF_RANGE", 36, 124}, - #endif - #ifdef RAND_R_ERROR_ENTROPY_POOL_WAS_IGNORED - {"ERROR_ENTROPY_POOL_WAS_IGNORED", ERR_LIB_RAND, RAND_R_ERROR_ENTROPY_POOL_WAS_IGNORED}, - #else - {"ERROR_ENTROPY_POOL_WAS_IGNORED", 36, 127}, - #endif - #ifdef RAND_R_ERROR_INITIALISING_DRBG - {"ERROR_INITIALISING_DRBG", ERR_LIB_RAND, RAND_R_ERROR_INITIALISING_DRBG}, - #else - {"ERROR_INITIALISING_DRBG", 36, 107}, - #endif - #ifdef RAND_R_ERROR_INSTANTIATING_DRBG - {"ERROR_INSTANTIATING_DRBG", ERR_LIB_RAND, RAND_R_ERROR_INSTANTIATING_DRBG}, - #else - {"ERROR_INSTANTIATING_DRBG", 36, 108}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_ADDITIONAL_INPUT - {"ERROR_RETRIEVING_ADDITIONAL_INPUT", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_ADDITIONAL_INPUT}, - #else - {"ERROR_RETRIEVING_ADDITIONAL_INPUT", 36, 109}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_ENTROPY - {"ERROR_RETRIEVING_ENTROPY", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_ENTROPY}, - #else - {"ERROR_RETRIEVING_ENTROPY", 36, 110}, - #endif - #ifdef RAND_R_ERROR_RETRIEVING_NONCE - {"ERROR_RETRIEVING_NONCE", ERR_LIB_RAND, RAND_R_ERROR_RETRIEVING_NONCE}, - #else - {"ERROR_RETRIEVING_NONCE", 36, 111}, - #endif - #ifdef RAND_R_FAILED_TO_CREATE_LOCK - {"FAILED_TO_CREATE_LOCK", ERR_LIB_RAND, RAND_R_FAILED_TO_CREATE_LOCK}, - #else - {"FAILED_TO_CREATE_LOCK", 36, 126}, - #endif - #ifdef RAND_R_FUNC_NOT_IMPLEMENTED - {"FUNC_NOT_IMPLEMENTED", ERR_LIB_RAND, RAND_R_FUNC_NOT_IMPLEMENTED}, - #else - {"FUNC_NOT_IMPLEMENTED", 36, 101}, - #endif - #ifdef RAND_R_FWRITE_ERROR - {"FWRITE_ERROR", ERR_LIB_RAND, RAND_R_FWRITE_ERROR}, - #else - {"FWRITE_ERROR", 36, 123}, - #endif - #ifdef RAND_R_GENERATE_ERROR - {"GENERATE_ERROR", ERR_LIB_RAND, RAND_R_GENERATE_ERROR}, - #else - {"GENERATE_ERROR", 36, 112}, - #endif - #ifdef RAND_R_INTERNAL_ERROR - {"INTERNAL_ERROR", ERR_LIB_RAND, RAND_R_INTERNAL_ERROR}, - #else - {"INTERNAL_ERROR", 36, 113}, - #endif - #ifdef RAND_R_IN_ERROR_STATE - {"IN_ERROR_STATE", ERR_LIB_RAND, RAND_R_IN_ERROR_STATE}, - #else - {"IN_ERROR_STATE", 36, 114}, - #endif - #ifdef RAND_R_NOT_A_REGULAR_FILE - {"NOT_A_REGULAR_FILE", ERR_LIB_RAND, RAND_R_NOT_A_REGULAR_FILE}, - #else - {"NOT_A_REGULAR_FILE", 36, 122}, - #endif - #ifdef RAND_R_NOT_INSTANTIATED - {"NOT_INSTANTIATED", ERR_LIB_RAND, RAND_R_NOT_INSTANTIATED}, - #else - {"NOT_INSTANTIATED", 36, 115}, - #endif - #ifdef RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED - {"NO_DRBG_IMPLEMENTATION_SELECTED", ERR_LIB_RAND, RAND_R_NO_DRBG_IMPLEMENTATION_SELECTED}, - #else - {"NO_DRBG_IMPLEMENTATION_SELECTED", 36, 128}, - #endif - #ifdef RAND_R_PARENT_LOCKING_NOT_ENABLED - {"PARENT_LOCKING_NOT_ENABLED", ERR_LIB_RAND, RAND_R_PARENT_LOCKING_NOT_ENABLED}, - #else - {"PARENT_LOCKING_NOT_ENABLED", 36, 130}, - #endif - #ifdef RAND_R_PARENT_STRENGTH_TOO_WEAK - {"PARENT_STRENGTH_TOO_WEAK", ERR_LIB_RAND, RAND_R_PARENT_STRENGTH_TOO_WEAK}, - #else - {"PARENT_STRENGTH_TOO_WEAK", 36, 131}, - #endif - #ifdef RAND_R_PERSONALISATION_STRING_TOO_LONG - {"PERSONALISATION_STRING_TOO_LONG", ERR_LIB_RAND, RAND_R_PERSONALISATION_STRING_TOO_LONG}, - #else - {"PERSONALISATION_STRING_TOO_LONG", 36, 116}, - #endif - #ifdef RAND_R_PREDICTION_RESISTANCE_NOT_SUPPORTED - {"PREDICTION_RESISTANCE_NOT_SUPPORTED", ERR_LIB_RAND, RAND_R_PREDICTION_RESISTANCE_NOT_SUPPORTED}, - #else - {"PREDICTION_RESISTANCE_NOT_SUPPORTED", 36, 133}, - #endif - #ifdef RAND_R_PRNG_NOT_SEEDED - {"PRNG_NOT_SEEDED", ERR_LIB_RAND, RAND_R_PRNG_NOT_SEEDED}, - #else - {"PRNG_NOT_SEEDED", 36, 100}, - #endif - #ifdef RAND_R_RANDOM_POOL_OVERFLOW - {"RANDOM_POOL_OVERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW}, - #else - {"RANDOM_POOL_OVERFLOW", 36, 125}, - #endif - #ifdef RAND_R_RANDOM_POOL_UNDERFLOW - {"RANDOM_POOL_UNDERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_UNDERFLOW}, - #else - {"RANDOM_POOL_UNDERFLOW", 36, 134}, - #endif - #ifdef RAND_R_REQUEST_TOO_LARGE_FOR_DRBG - {"REQUEST_TOO_LARGE_FOR_DRBG", ERR_LIB_RAND, RAND_R_REQUEST_TOO_LARGE_FOR_DRBG}, - #else - {"REQUEST_TOO_LARGE_FOR_DRBG", 36, 117}, - #endif - #ifdef RAND_R_RESEED_ERROR - {"RESEED_ERROR", ERR_LIB_RAND, RAND_R_RESEED_ERROR}, - #else - {"RESEED_ERROR", 36, 118}, - #endif - #ifdef RAND_R_SELFTEST_FAILURE - {"SELFTEST_FAILURE", ERR_LIB_RAND, RAND_R_SELFTEST_FAILURE}, - #else - {"SELFTEST_FAILURE", 36, 119}, - #endif - #ifdef RAND_R_TOO_LITTLE_NONCE_REQUESTED - {"TOO_LITTLE_NONCE_REQUESTED", ERR_LIB_RAND, RAND_R_TOO_LITTLE_NONCE_REQUESTED}, - #else - {"TOO_LITTLE_NONCE_REQUESTED", 36, 135}, - #endif - #ifdef RAND_R_TOO_MUCH_NONCE_REQUESTED - {"TOO_MUCH_NONCE_REQUESTED", ERR_LIB_RAND, RAND_R_TOO_MUCH_NONCE_REQUESTED}, - #else - {"TOO_MUCH_NONCE_REQUESTED", 36, 136}, - #endif - #ifdef RAND_R_UNSUPPORTED_DRBG_FLAGS - {"UNSUPPORTED_DRBG_FLAGS", ERR_LIB_RAND, RAND_R_UNSUPPORTED_DRBG_FLAGS}, - #else - {"UNSUPPORTED_DRBG_FLAGS", 36, 132}, - #endif - #ifdef RAND_R_UNSUPPORTED_DRBG_TYPE - {"UNSUPPORTED_DRBG_TYPE", ERR_LIB_RAND, RAND_R_UNSUPPORTED_DRBG_TYPE}, - #else - {"UNSUPPORTED_DRBG_TYPE", 36, 120}, - #endif - #ifdef RSA_R_ALGORITHM_MISMATCH - {"ALGORITHM_MISMATCH", ERR_LIB_RSA, RSA_R_ALGORITHM_MISMATCH}, - #else - {"ALGORITHM_MISMATCH", 4, 100}, - #endif - #ifdef RSA_R_BAD_E_VALUE - {"BAD_E_VALUE", ERR_LIB_RSA, RSA_R_BAD_E_VALUE}, - #else - {"BAD_E_VALUE", 4, 101}, - #endif - #ifdef RSA_R_BAD_FIXED_HEADER_DECRYPT - {"BAD_FIXED_HEADER_DECRYPT", ERR_LIB_RSA, RSA_R_BAD_FIXED_HEADER_DECRYPT}, - #else - {"BAD_FIXED_HEADER_DECRYPT", 4, 102}, - #endif - #ifdef RSA_R_BAD_PAD_BYTE_COUNT - {"BAD_PAD_BYTE_COUNT", ERR_LIB_RSA, RSA_R_BAD_PAD_BYTE_COUNT}, - #else - {"BAD_PAD_BYTE_COUNT", 4, 103}, - #endif - #ifdef RSA_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_RSA, RSA_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 4, 104}, - #endif - #ifdef RSA_R_BLOCK_TYPE_IS_NOT_01 - {"BLOCK_TYPE_IS_NOT_01", ERR_LIB_RSA, RSA_R_BLOCK_TYPE_IS_NOT_01}, - #else - {"BLOCK_TYPE_IS_NOT_01", 4, 106}, - #endif - #ifdef RSA_R_BLOCK_TYPE_IS_NOT_02 - {"BLOCK_TYPE_IS_NOT_02", ERR_LIB_RSA, RSA_R_BLOCK_TYPE_IS_NOT_02}, - #else - {"BLOCK_TYPE_IS_NOT_02", 4, 107}, - #endif - #ifdef RSA_R_DATA_GREATER_THAN_MOD_LEN - {"DATA_GREATER_THAN_MOD_LEN", ERR_LIB_RSA, RSA_R_DATA_GREATER_THAN_MOD_LEN}, - #else - {"DATA_GREATER_THAN_MOD_LEN", 4, 108}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE - {"DATA_TOO_LARGE", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE}, - #else - {"DATA_TOO_LARGE", 4, 109}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE - {"DATA_TOO_LARGE_FOR_KEY_SIZE", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE}, - #else - {"DATA_TOO_LARGE_FOR_KEY_SIZE", 4, 110}, - #endif - #ifdef RSA_R_DATA_TOO_LARGE_FOR_MODULUS - {"DATA_TOO_LARGE_FOR_MODULUS", ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS}, - #else - {"DATA_TOO_LARGE_FOR_MODULUS", 4, 132}, - #endif - #ifdef RSA_R_DATA_TOO_SMALL - {"DATA_TOO_SMALL", ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL}, - #else - {"DATA_TOO_SMALL", 4, 111}, - #endif - #ifdef RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE - {"DATA_TOO_SMALL_FOR_KEY_SIZE", ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE}, - #else - {"DATA_TOO_SMALL_FOR_KEY_SIZE", 4, 122}, - #endif - #ifdef RSA_R_DIGEST_DOES_NOT_MATCH - {"DIGEST_DOES_NOT_MATCH", ERR_LIB_RSA, RSA_R_DIGEST_DOES_NOT_MATCH}, - #else - {"DIGEST_DOES_NOT_MATCH", 4, 158}, - #endif - #ifdef RSA_R_DIGEST_NOT_ALLOWED - {"DIGEST_NOT_ALLOWED", ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED}, - #else - {"DIGEST_NOT_ALLOWED", 4, 145}, - #endif - #ifdef RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY - {"DIGEST_TOO_BIG_FOR_RSA_KEY", ERR_LIB_RSA, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY}, - #else - {"DIGEST_TOO_BIG_FOR_RSA_KEY", 4, 112}, - #endif - #ifdef RSA_R_DMP1_NOT_CONGRUENT_TO_D - {"DMP1_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_DMP1_NOT_CONGRUENT_TO_D}, - #else - {"DMP1_NOT_CONGRUENT_TO_D", 4, 124}, - #endif - #ifdef RSA_R_DMQ1_NOT_CONGRUENT_TO_D - {"DMQ1_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_DMQ1_NOT_CONGRUENT_TO_D}, - #else - {"DMQ1_NOT_CONGRUENT_TO_D", 4, 125}, - #endif - #ifdef RSA_R_D_E_NOT_CONGRUENT_TO_1 - {"D_E_NOT_CONGRUENT_TO_1", ERR_LIB_RSA, RSA_R_D_E_NOT_CONGRUENT_TO_1}, - #else - {"D_E_NOT_CONGRUENT_TO_1", 4, 123}, - #endif - #ifdef RSA_R_FIRST_OCTET_INVALID - {"FIRST_OCTET_INVALID", ERR_LIB_RSA, RSA_R_FIRST_OCTET_INVALID}, - #else - {"FIRST_OCTET_INVALID", 4, 133}, - #endif - #ifdef RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE - {"ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", ERR_LIB_RSA, RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE}, - #else - {"ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", 4, 144}, - #endif - #ifdef RSA_R_INVALID_DIGEST - {"INVALID_DIGEST", ERR_LIB_RSA, RSA_R_INVALID_DIGEST}, - #else - {"INVALID_DIGEST", 4, 157}, - #endif - #ifdef RSA_R_INVALID_DIGEST_LENGTH - {"INVALID_DIGEST_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_DIGEST_LENGTH}, - #else - {"INVALID_DIGEST_LENGTH", 4, 143}, - #endif - #ifdef RSA_R_INVALID_HEADER - {"INVALID_HEADER", ERR_LIB_RSA, RSA_R_INVALID_HEADER}, - #else - {"INVALID_HEADER", 4, 137}, - #endif - #ifdef RSA_R_INVALID_LABEL - {"INVALID_LABEL", ERR_LIB_RSA, RSA_R_INVALID_LABEL}, - #else - {"INVALID_LABEL", 4, 160}, - #endif - #ifdef RSA_R_INVALID_MESSAGE_LENGTH - {"INVALID_MESSAGE_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_MESSAGE_LENGTH}, - #else - {"INVALID_MESSAGE_LENGTH", 4, 131}, - #endif - #ifdef RSA_R_INVALID_MGF1_MD - {"INVALID_MGF1_MD", ERR_LIB_RSA, RSA_R_INVALID_MGF1_MD}, - #else - {"INVALID_MGF1_MD", 4, 156}, - #endif - #ifdef RSA_R_INVALID_MULTI_PRIME_KEY - {"INVALID_MULTI_PRIME_KEY", ERR_LIB_RSA, RSA_R_INVALID_MULTI_PRIME_KEY}, - #else - {"INVALID_MULTI_PRIME_KEY", 4, 167}, - #endif - #ifdef RSA_R_INVALID_OAEP_PARAMETERS - {"INVALID_OAEP_PARAMETERS", ERR_LIB_RSA, RSA_R_INVALID_OAEP_PARAMETERS}, - #else - {"INVALID_OAEP_PARAMETERS", 4, 161}, - #endif - #ifdef RSA_R_INVALID_PADDING - {"INVALID_PADDING", ERR_LIB_RSA, RSA_R_INVALID_PADDING}, - #else - {"INVALID_PADDING", 4, 138}, - #endif - #ifdef RSA_R_INVALID_PADDING_MODE - {"INVALID_PADDING_MODE", ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE}, - #else - {"INVALID_PADDING_MODE", 4, 141}, - #endif - #ifdef RSA_R_INVALID_PSS_PARAMETERS - {"INVALID_PSS_PARAMETERS", ERR_LIB_RSA, RSA_R_INVALID_PSS_PARAMETERS}, - #else - {"INVALID_PSS_PARAMETERS", 4, 149}, - #endif - #ifdef RSA_R_INVALID_PSS_SALTLEN - {"INVALID_PSS_SALTLEN", ERR_LIB_RSA, RSA_R_INVALID_PSS_SALTLEN}, - #else - {"INVALID_PSS_SALTLEN", 4, 146}, - #endif - #ifdef RSA_R_INVALID_SALT_LENGTH - {"INVALID_SALT_LENGTH", ERR_LIB_RSA, RSA_R_INVALID_SALT_LENGTH}, - #else - {"INVALID_SALT_LENGTH", 4, 150}, - #endif - #ifdef RSA_R_INVALID_TRAILER - {"INVALID_TRAILER", ERR_LIB_RSA, RSA_R_INVALID_TRAILER}, - #else - {"INVALID_TRAILER", 4, 139}, - #endif - #ifdef RSA_R_INVALID_X931_DIGEST - {"INVALID_X931_DIGEST", ERR_LIB_RSA, RSA_R_INVALID_X931_DIGEST}, - #else - {"INVALID_X931_DIGEST", 4, 142}, - #endif - #ifdef RSA_R_IQMP_NOT_INVERSE_OF_Q - {"IQMP_NOT_INVERSE_OF_Q", ERR_LIB_RSA, RSA_R_IQMP_NOT_INVERSE_OF_Q}, - #else - {"IQMP_NOT_INVERSE_OF_Q", 4, 126}, - #endif - #ifdef RSA_R_KEY_PRIME_NUM_INVALID - {"KEY_PRIME_NUM_INVALID", ERR_LIB_RSA, RSA_R_KEY_PRIME_NUM_INVALID}, - #else - {"KEY_PRIME_NUM_INVALID", 4, 165}, - #endif - #ifdef RSA_R_KEY_SIZE_TOO_SMALL - {"KEY_SIZE_TOO_SMALL", ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL}, - #else - {"KEY_SIZE_TOO_SMALL", 4, 120}, - #endif - #ifdef RSA_R_LAST_OCTET_INVALID - {"LAST_OCTET_INVALID", ERR_LIB_RSA, RSA_R_LAST_OCTET_INVALID}, - #else - {"LAST_OCTET_INVALID", 4, 134}, - #endif - #ifdef RSA_R_MGF1_DIGEST_NOT_ALLOWED - {"MGF1_DIGEST_NOT_ALLOWED", ERR_LIB_RSA, RSA_R_MGF1_DIGEST_NOT_ALLOWED}, - #else - {"MGF1_DIGEST_NOT_ALLOWED", 4, 152}, - #endif - #ifdef RSA_R_MISSING_PRIVATE_KEY - {"MISSING_PRIVATE_KEY", ERR_LIB_RSA, RSA_R_MISSING_PRIVATE_KEY}, - #else - {"MISSING_PRIVATE_KEY", 4, 179}, - #endif - #ifdef RSA_R_MODULUS_TOO_LARGE - {"MODULUS_TOO_LARGE", ERR_LIB_RSA, RSA_R_MODULUS_TOO_LARGE}, - #else - {"MODULUS_TOO_LARGE", 4, 105}, - #endif - #ifdef RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R - {"MP_COEFFICIENT_NOT_INVERSE_OF_R", ERR_LIB_RSA, RSA_R_MP_COEFFICIENT_NOT_INVERSE_OF_R}, - #else - {"MP_COEFFICIENT_NOT_INVERSE_OF_R", 4, 168}, - #endif - #ifdef RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D - {"MP_EXPONENT_NOT_CONGRUENT_TO_D", ERR_LIB_RSA, RSA_R_MP_EXPONENT_NOT_CONGRUENT_TO_D}, - #else - {"MP_EXPONENT_NOT_CONGRUENT_TO_D", 4, 169}, - #endif - #ifdef RSA_R_MP_R_NOT_PRIME - {"MP_R_NOT_PRIME", ERR_LIB_RSA, RSA_R_MP_R_NOT_PRIME}, - #else - {"MP_R_NOT_PRIME", 4, 170}, - #endif - #ifdef RSA_R_NO_PUBLIC_EXPONENT - {"NO_PUBLIC_EXPONENT", ERR_LIB_RSA, RSA_R_NO_PUBLIC_EXPONENT}, - #else - {"NO_PUBLIC_EXPONENT", 4, 140}, - #endif - #ifdef RSA_R_NULL_BEFORE_BLOCK_MISSING - {"NULL_BEFORE_BLOCK_MISSING", ERR_LIB_RSA, RSA_R_NULL_BEFORE_BLOCK_MISSING}, - #else - {"NULL_BEFORE_BLOCK_MISSING", 4, 113}, - #endif - #ifdef RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES - {"N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", ERR_LIB_RSA, RSA_R_N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES}, - #else - {"N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", 4, 172}, - #endif - #ifdef RSA_R_N_DOES_NOT_EQUAL_P_Q - {"N_DOES_NOT_EQUAL_P_Q", ERR_LIB_RSA, RSA_R_N_DOES_NOT_EQUAL_P_Q}, - #else - {"N_DOES_NOT_EQUAL_P_Q", 4, 127}, - #endif - #ifdef RSA_R_OAEP_DECODING_ERROR - {"OAEP_DECODING_ERROR", ERR_LIB_RSA, RSA_R_OAEP_DECODING_ERROR}, - #else - {"OAEP_DECODING_ERROR", 4, 121}, - #endif - #ifdef RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_RSA, RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, - #else - {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 4, 148}, - #endif - #ifdef RSA_R_PADDING_CHECK_FAILED - {"PADDING_CHECK_FAILED", ERR_LIB_RSA, RSA_R_PADDING_CHECK_FAILED}, - #else - {"PADDING_CHECK_FAILED", 4, 114}, - #endif - #ifdef RSA_R_PKCS_DECODING_ERROR - {"PKCS_DECODING_ERROR", ERR_LIB_RSA, RSA_R_PKCS_DECODING_ERROR}, - #else - {"PKCS_DECODING_ERROR", 4, 159}, - #endif - #ifdef RSA_R_PSS_SALTLEN_TOO_SMALL - {"PSS_SALTLEN_TOO_SMALL", ERR_LIB_RSA, RSA_R_PSS_SALTLEN_TOO_SMALL}, - #else - {"PSS_SALTLEN_TOO_SMALL", 4, 164}, - #endif - #ifdef RSA_R_P_NOT_PRIME - {"P_NOT_PRIME", ERR_LIB_RSA, RSA_R_P_NOT_PRIME}, - #else - {"P_NOT_PRIME", 4, 128}, - #endif - #ifdef RSA_R_Q_NOT_PRIME - {"Q_NOT_PRIME", ERR_LIB_RSA, RSA_R_Q_NOT_PRIME}, - #else - {"Q_NOT_PRIME", 4, 129}, - #endif - #ifdef RSA_R_RSA_OPERATIONS_NOT_SUPPORTED - {"RSA_OPERATIONS_NOT_SUPPORTED", ERR_LIB_RSA, RSA_R_RSA_OPERATIONS_NOT_SUPPORTED}, - #else - {"RSA_OPERATIONS_NOT_SUPPORTED", 4, 130}, - #endif - #ifdef RSA_R_SLEN_CHECK_FAILED - {"SLEN_CHECK_FAILED", ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED}, - #else - {"SLEN_CHECK_FAILED", 4, 136}, - #endif - #ifdef RSA_R_SLEN_RECOVERY_FAILED - {"SLEN_RECOVERY_FAILED", ERR_LIB_RSA, RSA_R_SLEN_RECOVERY_FAILED}, - #else - {"SLEN_RECOVERY_FAILED", 4, 135}, - #endif - #ifdef RSA_R_SSLV3_ROLLBACK_ATTACK - {"SSLV3_ROLLBACK_ATTACK", ERR_LIB_RSA, RSA_R_SSLV3_ROLLBACK_ATTACK}, - #else - {"SSLV3_ROLLBACK_ATTACK", 4, 115}, - #endif - #ifdef RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", ERR_LIB_RSA, RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD}, - #else - {"THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", 4, 116}, - #endif - #ifdef RSA_R_UNKNOWN_ALGORITHM_TYPE - {"UNKNOWN_ALGORITHM_TYPE", ERR_LIB_RSA, RSA_R_UNKNOWN_ALGORITHM_TYPE}, - #else - {"UNKNOWN_ALGORITHM_TYPE", 4, 117}, - #endif - #ifdef RSA_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_RSA, RSA_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 4, 166}, - #endif - #ifdef RSA_R_UNKNOWN_MASK_DIGEST - {"UNKNOWN_MASK_DIGEST", ERR_LIB_RSA, RSA_R_UNKNOWN_MASK_DIGEST}, - #else - {"UNKNOWN_MASK_DIGEST", 4, 151}, - #endif - #ifdef RSA_R_UNKNOWN_PADDING_TYPE - {"UNKNOWN_PADDING_TYPE", ERR_LIB_RSA, RSA_R_UNKNOWN_PADDING_TYPE}, - #else - {"UNKNOWN_PADDING_TYPE", 4, 118}, - #endif - #ifdef RSA_R_UNSUPPORTED_ENCRYPTION_TYPE - {"UNSUPPORTED_ENCRYPTION_TYPE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE}, - #else - {"UNSUPPORTED_ENCRYPTION_TYPE", 4, 162}, - #endif - #ifdef RSA_R_UNSUPPORTED_LABEL_SOURCE - {"UNSUPPORTED_LABEL_SOURCE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_LABEL_SOURCE}, - #else - {"UNSUPPORTED_LABEL_SOURCE", 4, 163}, - #endif - #ifdef RSA_R_UNSUPPORTED_MASK_ALGORITHM - {"UNSUPPORTED_MASK_ALGORITHM", ERR_LIB_RSA, RSA_R_UNSUPPORTED_MASK_ALGORITHM}, - #else - {"UNSUPPORTED_MASK_ALGORITHM", 4, 153}, - #endif - #ifdef RSA_R_UNSUPPORTED_MASK_PARAMETER - {"UNSUPPORTED_MASK_PARAMETER", ERR_LIB_RSA, RSA_R_UNSUPPORTED_MASK_PARAMETER}, - #else - {"UNSUPPORTED_MASK_PARAMETER", 4, 154}, - #endif - #ifdef RSA_R_UNSUPPORTED_SIGNATURE_TYPE - {"UNSUPPORTED_SIGNATURE_TYPE", ERR_LIB_RSA, RSA_R_UNSUPPORTED_SIGNATURE_TYPE}, - #else - {"UNSUPPORTED_SIGNATURE_TYPE", 4, 155}, - #endif - #ifdef RSA_R_VALUE_MISSING - {"VALUE_MISSING", ERR_LIB_RSA, RSA_R_VALUE_MISSING}, - #else - {"VALUE_MISSING", 4, 147}, - #endif - #ifdef RSA_R_WRONG_SIGNATURE_LENGTH - {"WRONG_SIGNATURE_LENGTH", ERR_LIB_RSA, RSA_R_WRONG_SIGNATURE_LENGTH}, - #else - {"WRONG_SIGNATURE_LENGTH", 4, 119}, - #endif - #ifdef SSL_R_APPLICATION_DATA_AFTER_CLOSE_NOTIFY - {"APPLICATION_DATA_AFTER_CLOSE_NOTIFY", ERR_LIB_SSL, SSL_R_APPLICATION_DATA_AFTER_CLOSE_NOTIFY}, - #else - {"APPLICATION_DATA_AFTER_CLOSE_NOTIFY", 20, 291}, - #endif - #ifdef SSL_R_APP_DATA_IN_HANDSHAKE - {"APP_DATA_IN_HANDSHAKE", ERR_LIB_SSL, SSL_R_APP_DATA_IN_HANDSHAKE}, - #else - {"APP_DATA_IN_HANDSHAKE", 20, 100}, - #endif - #ifdef SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT - {"ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", ERR_LIB_SSL, SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT}, - #else - {"ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", 20, 272}, - #endif - #ifdef SSL_R_AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE - {"AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE", ERR_LIB_SSL, SSL_R_AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE}, - #else - {"AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE", 20, 143}, - #endif - #ifdef SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE - {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE}, - #else - {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", 20, 158}, - #endif - #ifdef SSL_R_BAD_CHANGE_CIPHER_SPEC - {"BAD_CHANGE_CIPHER_SPEC", ERR_LIB_SSL, SSL_R_BAD_CHANGE_CIPHER_SPEC}, - #else - {"BAD_CHANGE_CIPHER_SPEC", 20, 103}, - #endif - #ifdef SSL_R_BAD_CIPHER - {"BAD_CIPHER", ERR_LIB_SSL, SSL_R_BAD_CIPHER}, - #else - {"BAD_CIPHER", 20, 186}, - #endif - #ifdef SSL_R_BAD_DATA - {"BAD_DATA", ERR_LIB_SSL, SSL_R_BAD_DATA}, - #else - {"BAD_DATA", 20, 390}, - #endif - #ifdef SSL_R_BAD_DATA_RETURNED_BY_CALLBACK - {"BAD_DATA_RETURNED_BY_CALLBACK", ERR_LIB_SSL, SSL_R_BAD_DATA_RETURNED_BY_CALLBACK}, - #else - {"BAD_DATA_RETURNED_BY_CALLBACK", 20, 106}, - #endif - #ifdef SSL_R_BAD_DECOMPRESSION - {"BAD_DECOMPRESSION", ERR_LIB_SSL, SSL_R_BAD_DECOMPRESSION}, - #else - {"BAD_DECOMPRESSION", 20, 107}, - #endif - #ifdef SSL_R_BAD_DH_VALUE - {"BAD_DH_VALUE", ERR_LIB_SSL, SSL_R_BAD_DH_VALUE}, - #else - {"BAD_DH_VALUE", 20, 102}, - #endif - #ifdef SSL_R_BAD_DIGEST_LENGTH - {"BAD_DIGEST_LENGTH", ERR_LIB_SSL, SSL_R_BAD_DIGEST_LENGTH}, - #else - {"BAD_DIGEST_LENGTH", 20, 111}, - #endif - #ifdef SSL_R_BAD_EARLY_DATA - {"BAD_EARLY_DATA", ERR_LIB_SSL, SSL_R_BAD_EARLY_DATA}, - #else - {"BAD_EARLY_DATA", 20, 233}, - #endif - #ifdef SSL_R_BAD_ECC_CERT - {"BAD_ECC_CERT", ERR_LIB_SSL, SSL_R_BAD_ECC_CERT}, - #else - {"BAD_ECC_CERT", 20, 304}, - #endif - #ifdef SSL_R_BAD_ECDSA_SIGNATURE - {"BAD_ECDSA_SIGNATURE", ERR_LIB_SSL, SSL_R_BAD_ECDSA_SIGNATURE}, - #else - {"BAD_ECDSA_SIGNATURE", 20, 305}, - #endif - #ifdef SSL_R_BAD_ECPOINT - {"BAD_ECPOINT", ERR_LIB_SSL, SSL_R_BAD_ECPOINT}, - #else - {"BAD_ECPOINT", 20, 306}, - #endif - #ifdef SSL_R_BAD_EXTENSION - {"BAD_EXTENSION", ERR_LIB_SSL, SSL_R_BAD_EXTENSION}, - #else - {"BAD_EXTENSION", 20, 110}, - #endif - #ifdef SSL_R_BAD_HANDSHAKE_LENGTH - {"BAD_HANDSHAKE_LENGTH", ERR_LIB_SSL, SSL_R_BAD_HANDSHAKE_LENGTH}, - #else - {"BAD_HANDSHAKE_LENGTH", 20, 332}, - #endif - #ifdef SSL_R_BAD_HANDSHAKE_STATE - {"BAD_HANDSHAKE_STATE", ERR_LIB_SSL, SSL_R_BAD_HANDSHAKE_STATE}, - #else - {"BAD_HANDSHAKE_STATE", 20, 236}, - #endif - #ifdef SSL_R_BAD_HELLO_REQUEST - {"BAD_HELLO_REQUEST", ERR_LIB_SSL, SSL_R_BAD_HELLO_REQUEST}, - #else - {"BAD_HELLO_REQUEST", 20, 105}, - #endif - #ifdef SSL_R_BAD_HRR_VERSION - {"BAD_HRR_VERSION", ERR_LIB_SSL, SSL_R_BAD_HRR_VERSION}, - #else - {"BAD_HRR_VERSION", 20, 263}, - #endif - #ifdef SSL_R_BAD_KEY_SHARE - {"BAD_KEY_SHARE", ERR_LIB_SSL, SSL_R_BAD_KEY_SHARE}, - #else - {"BAD_KEY_SHARE", 20, 108}, - #endif - #ifdef SSL_R_BAD_KEY_UPDATE - {"BAD_KEY_UPDATE", ERR_LIB_SSL, SSL_R_BAD_KEY_UPDATE}, - #else - {"BAD_KEY_UPDATE", 20, 122}, - #endif - #ifdef SSL_R_BAD_LEGACY_VERSION - {"BAD_LEGACY_VERSION", ERR_LIB_SSL, SSL_R_BAD_LEGACY_VERSION}, - #else - {"BAD_LEGACY_VERSION", 20, 292}, - #endif - #ifdef SSL_R_BAD_LENGTH - {"BAD_LENGTH", ERR_LIB_SSL, SSL_R_BAD_LENGTH}, - #else - {"BAD_LENGTH", 20, 271}, - #endif - #ifdef SSL_R_BAD_MAC_LENGTH - {"BAD_MAC_LENGTH", ERR_LIB_SSL, SSL_R_BAD_MAC_LENGTH}, - #else - {"BAD_MAC_LENGTH", 20, 333}, - #endif - #ifdef SSL_R_BAD_PACKET - {"BAD_PACKET", ERR_LIB_SSL, SSL_R_BAD_PACKET}, - #else - {"BAD_PACKET", 20, 240}, - #endif - #ifdef SSL_R_BAD_PACKET_LENGTH - {"BAD_PACKET_LENGTH", ERR_LIB_SSL, SSL_R_BAD_PACKET_LENGTH}, - #else - {"BAD_PACKET_LENGTH", 20, 115}, - #endif - #ifdef SSL_R_BAD_PROTOCOL_VERSION_NUMBER - {"BAD_PROTOCOL_VERSION_NUMBER", ERR_LIB_SSL, SSL_R_BAD_PROTOCOL_VERSION_NUMBER}, - #else - {"BAD_PROTOCOL_VERSION_NUMBER", 20, 116}, - #endif - #ifdef SSL_R_BAD_PSK - {"BAD_PSK", ERR_LIB_SSL, SSL_R_BAD_PSK}, - #else - {"BAD_PSK", 20, 219}, - #endif - #ifdef SSL_R_BAD_PSK_IDENTITY - {"BAD_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_BAD_PSK_IDENTITY}, - #else - {"BAD_PSK_IDENTITY", 20, 114}, - #endif - #ifdef SSL_R_BAD_PSK_IDENTITY_HINT_LENGTH - {"BAD_PSK_IDENTITY_HINT_LENGTH", ERR_LIB_SSL, SSL_R_BAD_PSK_IDENTITY_HINT_LENGTH}, - #else - {"BAD_PSK_IDENTITY_HINT_LENGTH", 20, 316}, - #endif - #ifdef SSL_R_BAD_RECORD_TYPE - {"BAD_RECORD_TYPE", ERR_LIB_SSL, SSL_R_BAD_RECORD_TYPE}, - #else - {"BAD_RECORD_TYPE", 20, 443}, - #endif - #ifdef SSL_R_BAD_RSA_ENCRYPT - {"BAD_RSA_ENCRYPT", ERR_LIB_SSL, SSL_R_BAD_RSA_ENCRYPT}, - #else - {"BAD_RSA_ENCRYPT", 20, 119}, - #endif - #ifdef SSL_R_BAD_SIGNATURE - {"BAD_SIGNATURE", ERR_LIB_SSL, SSL_R_BAD_SIGNATURE}, - #else - {"BAD_SIGNATURE", 20, 123}, - #endif - #ifdef SSL_R_BAD_SRP_A_LENGTH - {"BAD_SRP_A_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_A_LENGTH}, - #else - {"BAD_SRP_A_LENGTH", 20, 347}, - #endif - #ifdef SSL_R_BAD_SRP_B_LENGTH - {"BAD_SRP_B_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_B_LENGTH}, - #else - {"BAD_SRP_B_LENGTH", 20, 348}, - #endif - #ifdef SSL_R_BAD_SRP_G_LENGTH - {"BAD_SRP_G_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_G_LENGTH}, - #else - {"BAD_SRP_G_LENGTH", 20, 349}, - #endif - #ifdef SSL_R_BAD_SRP_N_LENGTH - {"BAD_SRP_N_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_N_LENGTH}, - #else - {"BAD_SRP_N_LENGTH", 20, 350}, - #endif - #ifdef SSL_R_BAD_SRP_PARAMETERS - {"BAD_SRP_PARAMETERS", ERR_LIB_SSL, SSL_R_BAD_SRP_PARAMETERS}, - #else - {"BAD_SRP_PARAMETERS", 20, 371}, - #endif - #ifdef SSL_R_BAD_SRP_S_LENGTH - {"BAD_SRP_S_LENGTH", ERR_LIB_SSL, SSL_R_BAD_SRP_S_LENGTH}, - #else - {"BAD_SRP_S_LENGTH", 20, 351}, - #endif - #ifdef SSL_R_BAD_SRTP_MKI_VALUE - {"BAD_SRTP_MKI_VALUE", ERR_LIB_SSL, SSL_R_BAD_SRTP_MKI_VALUE}, - #else - {"BAD_SRTP_MKI_VALUE", 20, 352}, - #endif - #ifdef SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST - {"BAD_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST}, - #else - {"BAD_SRTP_PROTECTION_PROFILE_LIST", 20, 353}, - #endif - #ifdef SSL_R_BAD_SSL_FILETYPE - {"BAD_SSL_FILETYPE", ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE}, - #else - {"BAD_SSL_FILETYPE", 20, 124}, - #endif - #ifdef SSL_R_BAD_VALUE - {"BAD_VALUE", ERR_LIB_SSL, SSL_R_BAD_VALUE}, - #else - {"BAD_VALUE", 20, 384}, - #endif - #ifdef SSL_R_BAD_WRITE_RETRY - {"BAD_WRITE_RETRY", ERR_LIB_SSL, SSL_R_BAD_WRITE_RETRY}, - #else - {"BAD_WRITE_RETRY", 20, 127}, - #endif - #ifdef SSL_R_BINDER_DOES_NOT_VERIFY - {"BINDER_DOES_NOT_VERIFY", ERR_LIB_SSL, SSL_R_BINDER_DOES_NOT_VERIFY}, - #else - {"BINDER_DOES_NOT_VERIFY", 20, 253}, - #endif - #ifdef SSL_R_BIO_NOT_SET - {"BIO_NOT_SET", ERR_LIB_SSL, SSL_R_BIO_NOT_SET}, - #else - {"BIO_NOT_SET", 20, 128}, - #endif - #ifdef SSL_R_BLOCK_CIPHER_PAD_IS_WRONG - {"BLOCK_CIPHER_PAD_IS_WRONG", ERR_LIB_SSL, SSL_R_BLOCK_CIPHER_PAD_IS_WRONG}, - #else - {"BLOCK_CIPHER_PAD_IS_WRONG", 20, 129}, - #endif - #ifdef SSL_R_BN_LIB - {"BN_LIB", ERR_LIB_SSL, SSL_R_BN_LIB}, - #else - {"BN_LIB", 20, 130}, - #endif - #ifdef SSL_R_CALLBACK_FAILED - {"CALLBACK_FAILED", ERR_LIB_SSL, SSL_R_CALLBACK_FAILED}, - #else - {"CALLBACK_FAILED", 20, 234}, - #endif - #ifdef SSL_R_CANNOT_CHANGE_CIPHER - {"CANNOT_CHANGE_CIPHER", ERR_LIB_SSL, SSL_R_CANNOT_CHANGE_CIPHER}, - #else - {"CANNOT_CHANGE_CIPHER", 20, 109}, - #endif - #ifdef SSL_R_CA_DN_LENGTH_MISMATCH - {"CA_DN_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_CA_DN_LENGTH_MISMATCH}, - #else - {"CA_DN_LENGTH_MISMATCH", 20, 131}, - #endif - #ifdef SSL_R_CA_KEY_TOO_SMALL - {"CA_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_CA_KEY_TOO_SMALL}, - #else - {"CA_KEY_TOO_SMALL", 20, 397}, - #endif - #ifdef SSL_R_CA_MD_TOO_WEAK - {"CA_MD_TOO_WEAK", ERR_LIB_SSL, SSL_R_CA_MD_TOO_WEAK}, - #else - {"CA_MD_TOO_WEAK", 20, 398}, - #endif - #ifdef SSL_R_CCS_RECEIVED_EARLY - {"CCS_RECEIVED_EARLY", ERR_LIB_SSL, SSL_R_CCS_RECEIVED_EARLY}, - #else - {"CCS_RECEIVED_EARLY", 20, 133}, - #endif - #ifdef SSL_R_CERTIFICATE_VERIFY_FAILED - {"CERTIFICATE_VERIFY_FAILED", ERR_LIB_SSL, SSL_R_CERTIFICATE_VERIFY_FAILED}, - #else - {"CERTIFICATE_VERIFY_FAILED", 20, 134}, - #endif - #ifdef SSL_R_CERT_CB_ERROR - {"CERT_CB_ERROR", ERR_LIB_SSL, SSL_R_CERT_CB_ERROR}, - #else - {"CERT_CB_ERROR", 20, 377}, - #endif - #ifdef SSL_R_CERT_LENGTH_MISMATCH - {"CERT_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_CERT_LENGTH_MISMATCH}, - #else - {"CERT_LENGTH_MISMATCH", 20, 135}, - #endif - #ifdef SSL_R_CIPHERSUITE_DIGEST_HAS_CHANGED - {"CIPHERSUITE_DIGEST_HAS_CHANGED", ERR_LIB_SSL, SSL_R_CIPHERSUITE_DIGEST_HAS_CHANGED}, - #else - {"CIPHERSUITE_DIGEST_HAS_CHANGED", 20, 218}, - #endif - #ifdef SSL_R_CIPHER_CODE_WRONG_LENGTH - {"CIPHER_CODE_WRONG_LENGTH", ERR_LIB_SSL, SSL_R_CIPHER_CODE_WRONG_LENGTH}, - #else - {"CIPHER_CODE_WRONG_LENGTH", 20, 137}, - #endif - #ifdef SSL_R_CIPHER_OR_HASH_UNAVAILABLE - {"CIPHER_OR_HASH_UNAVAILABLE", ERR_LIB_SSL, SSL_R_CIPHER_OR_HASH_UNAVAILABLE}, - #else - {"CIPHER_OR_HASH_UNAVAILABLE", 20, 138}, - #endif - #ifdef SSL_R_CLIENTHELLO_TLSEXT - {"CLIENTHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_CLIENTHELLO_TLSEXT}, - #else - {"CLIENTHELLO_TLSEXT", 20, 226}, - #endif - #ifdef SSL_R_COMPRESSED_LENGTH_TOO_LONG - {"COMPRESSED_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_COMPRESSED_LENGTH_TOO_LONG}, - #else - {"COMPRESSED_LENGTH_TOO_LONG", 20, 140}, - #endif - #ifdef SSL_R_COMPRESSION_DISABLED - {"COMPRESSION_DISABLED", ERR_LIB_SSL, SSL_R_COMPRESSION_DISABLED}, - #else - {"COMPRESSION_DISABLED", 20, 343}, - #endif - #ifdef SSL_R_COMPRESSION_FAILURE - {"COMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_COMPRESSION_FAILURE}, - #else - {"COMPRESSION_FAILURE", 20, 141}, - #endif - #ifdef SSL_R_COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE - {"COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", ERR_LIB_SSL, SSL_R_COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE}, - #else - {"COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", 20, 307}, - #endif - #ifdef SSL_R_COMPRESSION_LIBRARY_ERROR - {"COMPRESSION_LIBRARY_ERROR", ERR_LIB_SSL, SSL_R_COMPRESSION_LIBRARY_ERROR}, - #else - {"COMPRESSION_LIBRARY_ERROR", 20, 142}, - #endif - #ifdef SSL_R_CONNECTION_TYPE_NOT_SET - {"CONNECTION_TYPE_NOT_SET", ERR_LIB_SSL, SSL_R_CONNECTION_TYPE_NOT_SET}, - #else - {"CONNECTION_TYPE_NOT_SET", 20, 144}, - #endif - #ifdef SSL_R_CONTEXT_NOT_DANE_ENABLED - {"CONTEXT_NOT_DANE_ENABLED", ERR_LIB_SSL, SSL_R_CONTEXT_NOT_DANE_ENABLED}, - #else - {"CONTEXT_NOT_DANE_ENABLED", 20, 167}, - #endif - #ifdef SSL_R_COOKIE_GEN_CALLBACK_FAILURE - {"COOKIE_GEN_CALLBACK_FAILURE", ERR_LIB_SSL, SSL_R_COOKIE_GEN_CALLBACK_FAILURE}, - #else - {"COOKIE_GEN_CALLBACK_FAILURE", 20, 400}, - #endif - #ifdef SSL_R_COOKIE_MISMATCH - {"COOKIE_MISMATCH", ERR_LIB_SSL, SSL_R_COOKIE_MISMATCH}, - #else - {"COOKIE_MISMATCH", 20, 308}, - #endif - #ifdef SSL_R_CUSTOM_EXT_HANDLER_ALREADY_INSTALLED - {"CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", ERR_LIB_SSL, SSL_R_CUSTOM_EXT_HANDLER_ALREADY_INSTALLED}, - #else - {"CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", 20, 206}, - #endif - #ifdef SSL_R_DANE_ALREADY_ENABLED - {"DANE_ALREADY_ENABLED", ERR_LIB_SSL, SSL_R_DANE_ALREADY_ENABLED}, - #else - {"DANE_ALREADY_ENABLED", 20, 172}, - #endif - #ifdef SSL_R_DANE_CANNOT_OVERRIDE_MTYPE_FULL - {"DANE_CANNOT_OVERRIDE_MTYPE_FULL", ERR_LIB_SSL, SSL_R_DANE_CANNOT_OVERRIDE_MTYPE_FULL}, - #else - {"DANE_CANNOT_OVERRIDE_MTYPE_FULL", 20, 173}, - #endif - #ifdef SSL_R_DANE_NOT_ENABLED - {"DANE_NOT_ENABLED", ERR_LIB_SSL, SSL_R_DANE_NOT_ENABLED}, - #else - {"DANE_NOT_ENABLED", 20, 175}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_CERTIFICATE - {"DANE_TLSA_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE}, - #else - {"DANE_TLSA_BAD_CERTIFICATE", 20, 180}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_CERTIFICATE_USAGE - {"DANE_TLSA_BAD_CERTIFICATE_USAGE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE_USAGE}, - #else - {"DANE_TLSA_BAD_CERTIFICATE_USAGE", 20, 184}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_DATA_LENGTH - {"DANE_TLSA_BAD_DATA_LENGTH", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DATA_LENGTH}, - #else - {"DANE_TLSA_BAD_DATA_LENGTH", 20, 189}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH - {"DANE_TLSA_BAD_DIGEST_LENGTH", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH}, - #else - {"DANE_TLSA_BAD_DIGEST_LENGTH", 20, 192}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_MATCHING_TYPE - {"DANE_TLSA_BAD_MATCHING_TYPE", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_MATCHING_TYPE}, - #else - {"DANE_TLSA_BAD_MATCHING_TYPE", 20, 200}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_PUBLIC_KEY - {"DANE_TLSA_BAD_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_PUBLIC_KEY}, - #else - {"DANE_TLSA_BAD_PUBLIC_KEY", 20, 201}, - #endif - #ifdef SSL_R_DANE_TLSA_BAD_SELECTOR - {"DANE_TLSA_BAD_SELECTOR", ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_SELECTOR}, - #else - {"DANE_TLSA_BAD_SELECTOR", 20, 202}, - #endif - #ifdef SSL_R_DANE_TLSA_NULL_DATA - {"DANE_TLSA_NULL_DATA", ERR_LIB_SSL, SSL_R_DANE_TLSA_NULL_DATA}, - #else - {"DANE_TLSA_NULL_DATA", 20, 203}, - #endif - #ifdef SSL_R_DATA_BETWEEN_CCS_AND_FINISHED - {"DATA_BETWEEN_CCS_AND_FINISHED", ERR_LIB_SSL, SSL_R_DATA_BETWEEN_CCS_AND_FINISHED}, - #else - {"DATA_BETWEEN_CCS_AND_FINISHED", 20, 145}, - #endif - #ifdef SSL_R_DATA_LENGTH_TOO_LONG - {"DATA_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_DATA_LENGTH_TOO_LONG}, - #else - {"DATA_LENGTH_TOO_LONG", 20, 146}, - #endif - #ifdef SSL_R_DECRYPTION_FAILED - {"DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_DECRYPTION_FAILED}, - #else - {"DECRYPTION_FAILED", 20, 147}, - #endif - #ifdef SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC - {"DECRYPTION_FAILED_OR_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC}, - #else - {"DECRYPTION_FAILED_OR_BAD_RECORD_MAC", 20, 281}, - #endif - #ifdef SSL_R_DH_KEY_TOO_SMALL - {"DH_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_DH_KEY_TOO_SMALL}, - #else - {"DH_KEY_TOO_SMALL", 20, 394}, - #endif - #ifdef SSL_R_DH_PUBLIC_VALUE_LENGTH_IS_WRONG - {"DH_PUBLIC_VALUE_LENGTH_IS_WRONG", ERR_LIB_SSL, SSL_R_DH_PUBLIC_VALUE_LENGTH_IS_WRONG}, - #else - {"DH_PUBLIC_VALUE_LENGTH_IS_WRONG", 20, 148}, - #endif - #ifdef SSL_R_DIGEST_CHECK_FAILED - {"DIGEST_CHECK_FAILED", ERR_LIB_SSL, SSL_R_DIGEST_CHECK_FAILED}, - #else - {"DIGEST_CHECK_FAILED", 20, 149}, - #endif - #ifdef SSL_R_DTLS_MESSAGE_TOO_BIG - {"DTLS_MESSAGE_TOO_BIG", ERR_LIB_SSL, SSL_R_DTLS_MESSAGE_TOO_BIG}, - #else - {"DTLS_MESSAGE_TOO_BIG", 20, 334}, - #endif - #ifdef SSL_R_DUPLICATE_COMPRESSION_ID - {"DUPLICATE_COMPRESSION_ID", ERR_LIB_SSL, SSL_R_DUPLICATE_COMPRESSION_ID}, - #else - {"DUPLICATE_COMPRESSION_ID", 20, 309}, - #endif - #ifdef SSL_R_ECC_CERT_NOT_FOR_KEY_AGREEMENT - {"ECC_CERT_NOT_FOR_KEY_AGREEMENT", ERR_LIB_SSL, SSL_R_ECC_CERT_NOT_FOR_KEY_AGREEMENT}, - #else - {"ECC_CERT_NOT_FOR_KEY_AGREEMENT", 20, 317}, - #endif - #ifdef SSL_R_ECC_CERT_NOT_FOR_SIGNING - {"ECC_CERT_NOT_FOR_SIGNING", ERR_LIB_SSL, SSL_R_ECC_CERT_NOT_FOR_SIGNING}, - #else - {"ECC_CERT_NOT_FOR_SIGNING", 20, 318}, - #endif - #ifdef SSL_R_ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE - {"ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE", ERR_LIB_SSL, SSL_R_ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE}, - #else - {"ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE", 20, 322}, - #endif - #ifdef SSL_R_ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE - {"ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE", ERR_LIB_SSL, SSL_R_ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE}, - #else - {"ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE", 20, 323}, - #endif - #ifdef SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE - {"ECDH_REQUIRED_FOR_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE}, - #else - {"ECDH_REQUIRED_FOR_SUITEB_MODE", 20, 374}, - #endif - #ifdef SSL_R_ECGROUP_TOO_LARGE_FOR_CIPHER - {"ECGROUP_TOO_LARGE_FOR_CIPHER", ERR_LIB_SSL, SSL_R_ECGROUP_TOO_LARGE_FOR_CIPHER}, - #else - {"ECGROUP_TOO_LARGE_FOR_CIPHER", 20, 310}, - #endif - #ifdef SSL_R_EE_KEY_TOO_SMALL - {"EE_KEY_TOO_SMALL", ERR_LIB_SSL, SSL_R_EE_KEY_TOO_SMALL}, - #else - {"EE_KEY_TOO_SMALL", 20, 399}, - #endif - #ifdef SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST - {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST}, - #else - {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", 20, 354}, - #endif - #ifdef SSL_R_ENCRYPTED_LENGTH_TOO_LONG - {"ENCRYPTED_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_ENCRYPTED_LENGTH_TOO_LONG}, - #else - {"ENCRYPTED_LENGTH_TOO_LONG", 20, 150}, - #endif - #ifdef SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST - {"ERROR_IN_RECEIVED_CIPHER_LIST", ERR_LIB_SSL, SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST}, - #else - {"ERROR_IN_RECEIVED_CIPHER_LIST", 20, 151}, - #endif - #ifdef SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN - {"ERROR_SETTING_TLSA_BASE_DOMAIN", ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN}, - #else - {"ERROR_SETTING_TLSA_BASE_DOMAIN", 20, 204}, - #endif - #ifdef SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE - {"EXCEEDS_MAX_FRAGMENT_SIZE", ERR_LIB_SSL, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE}, - #else - {"EXCEEDS_MAX_FRAGMENT_SIZE", 20, 194}, - #endif - #ifdef SSL_R_EXCESSIVE_MESSAGE_SIZE - {"EXCESSIVE_MESSAGE_SIZE", ERR_LIB_SSL, SSL_R_EXCESSIVE_MESSAGE_SIZE}, - #else - {"EXCESSIVE_MESSAGE_SIZE", 20, 152}, - #endif - #ifdef SSL_R_EXTENSION_NOT_RECEIVED - {"EXTENSION_NOT_RECEIVED", ERR_LIB_SSL, SSL_R_EXTENSION_NOT_RECEIVED}, - #else - {"EXTENSION_NOT_RECEIVED", 20, 279}, - #endif - #ifdef SSL_R_EXTRA_DATA_IN_MESSAGE - {"EXTRA_DATA_IN_MESSAGE", ERR_LIB_SSL, SSL_R_EXTRA_DATA_IN_MESSAGE}, - #else - {"EXTRA_DATA_IN_MESSAGE", 20, 153}, - #endif - #ifdef SSL_R_EXT_LENGTH_MISMATCH - {"EXT_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_EXT_LENGTH_MISMATCH}, - #else - {"EXT_LENGTH_MISMATCH", 20, 163}, - #endif - #ifdef SSL_R_FAILED_TO_INIT_ASYNC - {"FAILED_TO_INIT_ASYNC", ERR_LIB_SSL, SSL_R_FAILED_TO_INIT_ASYNC}, - #else - {"FAILED_TO_INIT_ASYNC", 20, 405}, - #endif - #ifdef SSL_R_FRAGMENTED_CLIENT_HELLO - {"FRAGMENTED_CLIENT_HELLO", ERR_LIB_SSL, SSL_R_FRAGMENTED_CLIENT_HELLO}, - #else - {"FRAGMENTED_CLIENT_HELLO", 20, 401}, - #endif - #ifdef SSL_R_GOT_A_FIN_BEFORE_A_CCS - {"GOT_A_FIN_BEFORE_A_CCS", ERR_LIB_SSL, SSL_R_GOT_A_FIN_BEFORE_A_CCS}, - #else - {"GOT_A_FIN_BEFORE_A_CCS", 20, 154}, - #endif - #ifdef SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS - {"GOT_NEXT_PROTO_BEFORE_A_CCS", ERR_LIB_SSL, SSL_R_GOT_NEXT_PROTO_BEFORE_A_CCS}, - #else - {"GOT_NEXT_PROTO_BEFORE_A_CCS", 20, 355}, - #endif - #ifdef SSL_R_GOT_NEXT_PROTO_WITHOUT_EXTENSION - {"GOT_NEXT_PROTO_WITHOUT_EXTENSION", ERR_LIB_SSL, SSL_R_GOT_NEXT_PROTO_WITHOUT_EXTENSION}, - #else - {"GOT_NEXT_PROTO_WITHOUT_EXTENSION", 20, 356}, - #endif - #ifdef SSL_R_HTTPS_PROXY_REQUEST - {"HTTPS_PROXY_REQUEST", ERR_LIB_SSL, SSL_R_HTTPS_PROXY_REQUEST}, - #else - {"HTTPS_PROXY_REQUEST", 20, 155}, - #endif - #ifdef SSL_R_HTTP_REQUEST - {"HTTP_REQUEST", ERR_LIB_SSL, SSL_R_HTTP_REQUEST}, - #else - {"HTTP_REQUEST", 20, 156}, - #endif - #ifdef SSL_R_ILLEGAL_POINT_COMPRESSION - {"ILLEGAL_POINT_COMPRESSION", ERR_LIB_SSL, SSL_R_ILLEGAL_POINT_COMPRESSION}, - #else - {"ILLEGAL_POINT_COMPRESSION", 20, 162}, - #endif - #ifdef SSL_R_ILLEGAL_SUITEB_DIGEST - {"ILLEGAL_SUITEB_DIGEST", ERR_LIB_SSL, SSL_R_ILLEGAL_SUITEB_DIGEST}, - #else - {"ILLEGAL_SUITEB_DIGEST", 20, 380}, - #endif - #ifdef SSL_R_INAPPROPRIATE_FALLBACK - {"INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_INAPPROPRIATE_FALLBACK}, - #else - {"INAPPROPRIATE_FALLBACK", 20, 373}, - #endif - #ifdef SSL_R_INCONSISTENT_COMPRESSION - {"INCONSISTENT_COMPRESSION", ERR_LIB_SSL, SSL_R_INCONSISTENT_COMPRESSION}, - #else - {"INCONSISTENT_COMPRESSION", 20, 340}, - #endif - #ifdef SSL_R_INCONSISTENT_EARLY_DATA_ALPN - {"INCONSISTENT_EARLY_DATA_ALPN", ERR_LIB_SSL, SSL_R_INCONSISTENT_EARLY_DATA_ALPN}, - #else - {"INCONSISTENT_EARLY_DATA_ALPN", 20, 222}, - #endif - #ifdef SSL_R_INCONSISTENT_EARLY_DATA_SNI - {"INCONSISTENT_EARLY_DATA_SNI", ERR_LIB_SSL, SSL_R_INCONSISTENT_EARLY_DATA_SNI}, - #else - {"INCONSISTENT_EARLY_DATA_SNI", 20, 231}, - #endif - #ifdef SSL_R_INCONSISTENT_EXTMS - {"INCONSISTENT_EXTMS", ERR_LIB_SSL, SSL_R_INCONSISTENT_EXTMS}, - #else - {"INCONSISTENT_EXTMS", 20, 104}, - #endif - #ifdef SSL_R_INSUFFICIENT_SECURITY - {"INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_INSUFFICIENT_SECURITY}, - #else - {"INSUFFICIENT_SECURITY", 20, 241}, - #endif - #ifdef SSL_R_INVALID_ALERT - {"INVALID_ALERT", ERR_LIB_SSL, SSL_R_INVALID_ALERT}, - #else - {"INVALID_ALERT", 20, 205}, - #endif - #ifdef SSL_R_INVALID_CCS_MESSAGE - {"INVALID_CCS_MESSAGE", ERR_LIB_SSL, SSL_R_INVALID_CCS_MESSAGE}, - #else - {"INVALID_CCS_MESSAGE", 20, 260}, - #endif - #ifdef SSL_R_INVALID_CERTIFICATE_OR_ALG - {"INVALID_CERTIFICATE_OR_ALG", ERR_LIB_SSL, SSL_R_INVALID_CERTIFICATE_OR_ALG}, - #else - {"INVALID_CERTIFICATE_OR_ALG", 20, 238}, - #endif - #ifdef SSL_R_INVALID_COMMAND - {"INVALID_COMMAND", ERR_LIB_SSL, SSL_R_INVALID_COMMAND}, - #else - {"INVALID_COMMAND", 20, 280}, - #endif - #ifdef SSL_R_INVALID_COMPRESSION_ALGORITHM - {"INVALID_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_INVALID_COMPRESSION_ALGORITHM}, - #else - {"INVALID_COMPRESSION_ALGORITHM", 20, 341}, - #endif - #ifdef SSL_R_INVALID_CONFIG - {"INVALID_CONFIG", ERR_LIB_SSL, SSL_R_INVALID_CONFIG}, - #else - {"INVALID_CONFIG", 20, 283}, - #endif - #ifdef SSL_R_INVALID_CONFIGURATION_NAME - {"INVALID_CONFIGURATION_NAME", ERR_LIB_SSL, SSL_R_INVALID_CONFIGURATION_NAME}, - #else - {"INVALID_CONFIGURATION_NAME", 20, 113}, - #endif - #ifdef SSL_R_INVALID_CONTEXT - {"INVALID_CONTEXT", ERR_LIB_SSL, SSL_R_INVALID_CONTEXT}, - #else - {"INVALID_CONTEXT", 20, 282}, - #endif - #ifdef SSL_R_INVALID_CT_VALIDATION_TYPE - {"INVALID_CT_VALIDATION_TYPE", ERR_LIB_SSL, SSL_R_INVALID_CT_VALIDATION_TYPE}, - #else - {"INVALID_CT_VALIDATION_TYPE", 20, 212}, - #endif - #ifdef SSL_R_INVALID_KEY_UPDATE_TYPE - {"INVALID_KEY_UPDATE_TYPE", ERR_LIB_SSL, SSL_R_INVALID_KEY_UPDATE_TYPE}, - #else - {"INVALID_KEY_UPDATE_TYPE", 20, 120}, - #endif - #ifdef SSL_R_INVALID_MAX_EARLY_DATA - {"INVALID_MAX_EARLY_DATA", ERR_LIB_SSL, SSL_R_INVALID_MAX_EARLY_DATA}, - #else - {"INVALID_MAX_EARLY_DATA", 20, 174}, - #endif - #ifdef SSL_R_INVALID_NULL_CMD_NAME - {"INVALID_NULL_CMD_NAME", ERR_LIB_SSL, SSL_R_INVALID_NULL_CMD_NAME}, - #else - {"INVALID_NULL_CMD_NAME", 20, 385}, - #endif - #ifdef SSL_R_INVALID_SEQUENCE_NUMBER - {"INVALID_SEQUENCE_NUMBER", ERR_LIB_SSL, SSL_R_INVALID_SEQUENCE_NUMBER}, - #else - {"INVALID_SEQUENCE_NUMBER", 20, 402}, - #endif - #ifdef SSL_R_INVALID_SERVERINFO_DATA - {"INVALID_SERVERINFO_DATA", ERR_LIB_SSL, SSL_R_INVALID_SERVERINFO_DATA}, - #else - {"INVALID_SERVERINFO_DATA", 20, 388}, - #endif - #ifdef SSL_R_INVALID_SESSION_ID - {"INVALID_SESSION_ID", ERR_LIB_SSL, SSL_R_INVALID_SESSION_ID}, - #else - {"INVALID_SESSION_ID", 20, 999}, - #endif - #ifdef SSL_R_INVALID_SRP_USERNAME - {"INVALID_SRP_USERNAME", ERR_LIB_SSL, SSL_R_INVALID_SRP_USERNAME}, - #else - {"INVALID_SRP_USERNAME", 20, 357}, - #endif - #ifdef SSL_R_INVALID_STATUS_RESPONSE - {"INVALID_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_INVALID_STATUS_RESPONSE}, - #else - {"INVALID_STATUS_RESPONSE", 20, 328}, - #endif - #ifdef SSL_R_INVALID_TICKET_KEYS_LENGTH - {"INVALID_TICKET_KEYS_LENGTH", ERR_LIB_SSL, SSL_R_INVALID_TICKET_KEYS_LENGTH}, - #else - {"INVALID_TICKET_KEYS_LENGTH", 20, 325}, - #endif - #ifdef SSL_R_KRB5_S_TKT_NYV - {"KRB5_S_TKT_NYV", ERR_LIB_SSL, SSL_R_KRB5_S_TKT_NYV}, - #else - {"KRB5_S_TKT_NYV", 20, 294}, - #endif - #ifdef SSL_R_KRB5_S_TKT_SKEW - {"KRB5_S_TKT_SKEW", ERR_LIB_SSL, SSL_R_KRB5_S_TKT_SKEW}, - #else - {"KRB5_S_TKT_SKEW", 20, 295}, - #endif - #ifdef SSL_R_LENGTH_MISMATCH - {"LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_LENGTH_MISMATCH}, - #else - {"LENGTH_MISMATCH", 20, 159}, - #endif - #ifdef SSL_R_LENGTH_TOO_LONG - {"LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_LENGTH_TOO_LONG}, - #else - {"LENGTH_TOO_LONG", 20, 404}, - #endif - #ifdef SSL_R_LENGTH_TOO_SHORT - {"LENGTH_TOO_SHORT", ERR_LIB_SSL, SSL_R_LENGTH_TOO_SHORT}, - #else - {"LENGTH_TOO_SHORT", 20, 160}, - #endif - #ifdef SSL_R_LIBRARY_BUG - {"LIBRARY_BUG", ERR_LIB_SSL, SSL_R_LIBRARY_BUG}, - #else - {"LIBRARY_BUG", 20, 274}, - #endif - #ifdef SSL_R_LIBRARY_HAS_NO_CIPHERS - {"LIBRARY_HAS_NO_CIPHERS", ERR_LIB_SSL, SSL_R_LIBRARY_HAS_NO_CIPHERS}, - #else - {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, - #endif - #ifdef SSL_R_MESSAGE_TOO_LONG - {"MESSAGE_TOO_LONG", ERR_LIB_SSL, SSL_R_MESSAGE_TOO_LONG}, - #else - {"MESSAGE_TOO_LONG", 20, 296}, - #endif - #ifdef SSL_R_MISSING_DSA_SIGNING_CERT - {"MISSING_DSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_DSA_SIGNING_CERT}, - #else - {"MISSING_DSA_SIGNING_CERT", 20, 165}, - #endif - #ifdef SSL_R_MISSING_ECDH_CERT - {"MISSING_ECDH_CERT", ERR_LIB_SSL, SSL_R_MISSING_ECDH_CERT}, - #else - {"MISSING_ECDH_CERT", 20, 382}, - #endif - #ifdef SSL_R_MISSING_ECDSA_SIGNING_CERT - {"MISSING_ECDSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_ECDSA_SIGNING_CERT}, - #else - {"MISSING_ECDSA_SIGNING_CERT", 20, 381}, - #endif - #ifdef SSL_R_MISSING_FATAL - {"MISSING_FATAL", ERR_LIB_SSL, SSL_R_MISSING_FATAL}, - #else - {"MISSING_FATAL", 20, 256}, - #endif - #ifdef SSL_R_MISSING_PARAMETERS - {"MISSING_PARAMETERS", ERR_LIB_SSL, SSL_R_MISSING_PARAMETERS}, - #else - {"MISSING_PARAMETERS", 20, 290}, - #endif - #ifdef SSL_R_MISSING_RSA_CERTIFICATE - {"MISSING_RSA_CERTIFICATE", ERR_LIB_SSL, SSL_R_MISSING_RSA_CERTIFICATE}, - #else - {"MISSING_RSA_CERTIFICATE", 20, 168}, - #endif - #ifdef SSL_R_MISSING_RSA_ENCRYPTING_CERT - {"MISSING_RSA_ENCRYPTING_CERT", ERR_LIB_SSL, SSL_R_MISSING_RSA_ENCRYPTING_CERT}, - #else - {"MISSING_RSA_ENCRYPTING_CERT", 20, 169}, - #endif - #ifdef SSL_R_MISSING_RSA_SIGNING_CERT - {"MISSING_RSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_RSA_SIGNING_CERT}, - #else - {"MISSING_RSA_SIGNING_CERT", 20, 170}, - #endif - #ifdef SSL_R_MISSING_SIGALGS_EXTENSION - {"MISSING_SIGALGS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SIGALGS_EXTENSION}, - #else - {"MISSING_SIGALGS_EXTENSION", 20, 112}, - #endif - #ifdef SSL_R_MISSING_SIGNING_CERT - {"MISSING_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_SIGNING_CERT}, - #else - {"MISSING_SIGNING_CERT", 20, 221}, - #endif - #ifdef SSL_R_MISSING_SRP_PARAM - {"MISSING_SRP_PARAM", ERR_LIB_SSL, SSL_R_MISSING_SRP_PARAM}, - #else - {"MISSING_SRP_PARAM", 20, 358}, - #endif - #ifdef SSL_R_MISSING_SUPPORTED_GROUPS_EXTENSION - {"MISSING_SUPPORTED_GROUPS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SUPPORTED_GROUPS_EXTENSION}, - #else - {"MISSING_SUPPORTED_GROUPS_EXTENSION", 20, 209}, - #endif - #ifdef SSL_R_MISSING_TMP_DH_KEY - {"MISSING_TMP_DH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_DH_KEY}, - #else - {"MISSING_TMP_DH_KEY", 20, 171}, - #endif - #ifdef SSL_R_MISSING_TMP_ECDH_KEY - {"MISSING_TMP_ECDH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_ECDH_KEY}, - #else - {"MISSING_TMP_ECDH_KEY", 20, 311}, - #endif - #ifdef SSL_R_MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA - {"MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", ERR_LIB_SSL, SSL_R_MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA}, - #else - {"MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", 20, 293}, - #endif - #ifdef SSL_R_MULTIPLE_SGC_RESTARTS - {"MULTIPLE_SGC_RESTARTS", ERR_LIB_SSL, SSL_R_MULTIPLE_SGC_RESTARTS}, - #else - {"MULTIPLE_SGC_RESTARTS", 20, 346}, - #endif - #ifdef SSL_R_NOT_ON_RECORD_BOUNDARY - {"NOT_ON_RECORD_BOUNDARY", ERR_LIB_SSL, SSL_R_NOT_ON_RECORD_BOUNDARY}, - #else - {"NOT_ON_RECORD_BOUNDARY", 20, 182}, - #endif - #ifdef SSL_R_NOT_REPLACING_CERTIFICATE - {"NOT_REPLACING_CERTIFICATE", ERR_LIB_SSL, SSL_R_NOT_REPLACING_CERTIFICATE}, - #else - {"NOT_REPLACING_CERTIFICATE", 20, 289}, - #endif - #ifdef SSL_R_NOT_SERVER - {"NOT_SERVER", ERR_LIB_SSL, SSL_R_NOT_SERVER}, - #else - {"NOT_SERVER", 20, 284}, - #endif - #ifdef SSL_R_NO_APPLICATION_PROTOCOL - {"NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_NO_APPLICATION_PROTOCOL}, - #else - {"NO_APPLICATION_PROTOCOL", 20, 235}, - #endif - #ifdef SSL_R_NO_CERTIFICATES_RETURNED - {"NO_CERTIFICATES_RETURNED", ERR_LIB_SSL, SSL_R_NO_CERTIFICATES_RETURNED}, - #else - {"NO_CERTIFICATES_RETURNED", 20, 176}, - #endif - #ifdef SSL_R_NO_CERTIFICATE_ASSIGNED - {"NO_CERTIFICATE_ASSIGNED", ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_ASSIGNED}, - #else - {"NO_CERTIFICATE_ASSIGNED", 20, 177}, - #endif - #ifdef SSL_R_NO_CERTIFICATE_SET - {"NO_CERTIFICATE_SET", ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_SET}, - #else - {"NO_CERTIFICATE_SET", 20, 179}, - #endif - #ifdef SSL_R_NO_CHANGE_FOLLOWING_HRR - {"NO_CHANGE_FOLLOWING_HRR", ERR_LIB_SSL, SSL_R_NO_CHANGE_FOLLOWING_HRR}, - #else - {"NO_CHANGE_FOLLOWING_HRR", 20, 214}, - #endif - #ifdef SSL_R_NO_CIPHERS_AVAILABLE - {"NO_CIPHERS_AVAILABLE", ERR_LIB_SSL, SSL_R_NO_CIPHERS_AVAILABLE}, - #else - {"NO_CIPHERS_AVAILABLE", 20, 181}, - #endif - #ifdef SSL_R_NO_CIPHERS_SPECIFIED - {"NO_CIPHERS_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_CIPHERS_SPECIFIED}, - #else - {"NO_CIPHERS_SPECIFIED", 20, 183}, - #endif - #ifdef SSL_R_NO_CIPHER_MATCH - {"NO_CIPHER_MATCH", ERR_LIB_SSL, SSL_R_NO_CIPHER_MATCH}, - #else - {"NO_CIPHER_MATCH", 20, 185}, - #endif - #ifdef SSL_R_NO_CLIENT_CERT_METHOD - {"NO_CLIENT_CERT_METHOD", ERR_LIB_SSL, SSL_R_NO_CLIENT_CERT_METHOD}, - #else - {"NO_CLIENT_CERT_METHOD", 20, 331}, - #endif - #ifdef SSL_R_NO_COMPRESSION_SPECIFIED - {"NO_COMPRESSION_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_COMPRESSION_SPECIFIED}, - #else - {"NO_COMPRESSION_SPECIFIED", 20, 187}, - #endif - #ifdef SSL_R_NO_COOKIE_CALLBACK_SET - {"NO_COOKIE_CALLBACK_SET", ERR_LIB_SSL, SSL_R_NO_COOKIE_CALLBACK_SET}, - #else - {"NO_COOKIE_CALLBACK_SET", 20, 287}, - #endif - #ifdef SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER - {"NO_GOST_CERTIFICATE_SENT_BY_PEER", ERR_LIB_SSL, SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER}, - #else - {"NO_GOST_CERTIFICATE_SENT_BY_PEER", 20, 330}, - #endif - #ifdef SSL_R_NO_METHOD_SPECIFIED - {"NO_METHOD_SPECIFIED", ERR_LIB_SSL, SSL_R_NO_METHOD_SPECIFIED}, - #else - {"NO_METHOD_SPECIFIED", 20, 188}, - #endif - #ifdef SSL_R_NO_PEM_EXTENSIONS - {"NO_PEM_EXTENSIONS", ERR_LIB_SSL, SSL_R_NO_PEM_EXTENSIONS}, - #else - {"NO_PEM_EXTENSIONS", 20, 389}, - #endif - #ifdef SSL_R_NO_PRIVATE_KEY_ASSIGNED - {"NO_PRIVATE_KEY_ASSIGNED", ERR_LIB_SSL, SSL_R_NO_PRIVATE_KEY_ASSIGNED}, - #else - {"NO_PRIVATE_KEY_ASSIGNED", 20, 190}, - #endif - #ifdef SSL_R_NO_PROTOCOLS_AVAILABLE - {"NO_PROTOCOLS_AVAILABLE", ERR_LIB_SSL, SSL_R_NO_PROTOCOLS_AVAILABLE}, - #else - {"NO_PROTOCOLS_AVAILABLE", 20, 191}, - #endif - #ifdef SSL_R_NO_RENEGOTIATION - {"NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_NO_RENEGOTIATION}, - #else - {"NO_RENEGOTIATION", 20, 339}, - #endif - #ifdef SSL_R_NO_REQUIRED_DIGEST - {"NO_REQUIRED_DIGEST", ERR_LIB_SSL, SSL_R_NO_REQUIRED_DIGEST}, - #else - {"NO_REQUIRED_DIGEST", 20, 324}, - #endif - #ifdef SSL_R_NO_SHARED_CIPHER - {"NO_SHARED_CIPHER", ERR_LIB_SSL, SSL_R_NO_SHARED_CIPHER}, - #else - {"NO_SHARED_CIPHER", 20, 193}, - #endif - #ifdef SSL_R_NO_SHARED_GROUPS - {"NO_SHARED_GROUPS", ERR_LIB_SSL, SSL_R_NO_SHARED_GROUPS}, - #else - {"NO_SHARED_GROUPS", 20, 410}, - #endif - #ifdef SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS - {"NO_SHARED_SIGNATURE_ALGORITHMS", ERR_LIB_SSL, SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS}, - #else - {"NO_SHARED_SIGNATURE_ALGORITHMS", 20, 376}, - #endif - #ifdef SSL_R_NO_SRTP_PROFILES - {"NO_SRTP_PROFILES", ERR_LIB_SSL, SSL_R_NO_SRTP_PROFILES}, - #else - {"NO_SRTP_PROFILES", 20, 359}, - #endif - #ifdef SSL_R_NO_SUITABLE_KEY_SHARE - {"NO_SUITABLE_KEY_SHARE", ERR_LIB_SSL, SSL_R_NO_SUITABLE_KEY_SHARE}, - #else - {"NO_SUITABLE_KEY_SHARE", 20, 101}, - #endif - #ifdef SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM - {"NO_SUITABLE_SIGNATURE_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM}, - #else - {"NO_SUITABLE_SIGNATURE_ALGORITHM", 20, 118}, - #endif - #ifdef SSL_R_NO_VALID_SCTS - {"NO_VALID_SCTS", ERR_LIB_SSL, SSL_R_NO_VALID_SCTS}, - #else - {"NO_VALID_SCTS", 20, 216}, - #endif - #ifdef SSL_R_NO_VERIFY_COOKIE_CALLBACK - {"NO_VERIFY_COOKIE_CALLBACK", ERR_LIB_SSL, SSL_R_NO_VERIFY_COOKIE_CALLBACK}, - #else - {"NO_VERIFY_COOKIE_CALLBACK", 20, 403}, - #endif - #ifdef SSL_R_NULL_SSL_CTX - {"NULL_SSL_CTX", ERR_LIB_SSL, SSL_R_NULL_SSL_CTX}, - #else - {"NULL_SSL_CTX", 20, 195}, - #endif - #ifdef SSL_R_NULL_SSL_METHOD_PASSED - {"NULL_SSL_METHOD_PASSED", ERR_LIB_SSL, SSL_R_NULL_SSL_METHOD_PASSED}, - #else - {"NULL_SSL_METHOD_PASSED", 20, 196}, - #endif - #ifdef SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED - {"OLD_SESSION_CIPHER_NOT_RETURNED", ERR_LIB_SSL, SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED}, - #else - {"OLD_SESSION_CIPHER_NOT_RETURNED", 20, 197}, - #endif - #ifdef SSL_R_OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED - {"OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", ERR_LIB_SSL, SSL_R_OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED}, - #else - {"OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", 20, 344}, - #endif - #ifdef SSL_R_ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE - {"ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE}, - #else - {"ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE", 20, 387}, - #endif - #ifdef SSL_R_ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE - {"ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE", ERR_LIB_SSL, SSL_R_ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE}, - #else - {"ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE", 20, 379}, - #endif - #ifdef SSL_R_ONLY_TLS_ALLOWED_IN_FIPS_MODE - {"ONLY_TLS_ALLOWED_IN_FIPS_MODE", ERR_LIB_SSL, SSL_R_ONLY_TLS_ALLOWED_IN_FIPS_MODE}, - #else - {"ONLY_TLS_ALLOWED_IN_FIPS_MODE", 20, 297}, - #endif - #ifdef SSL_R_OPAQUE_PRF_INPUT_TOO_LONG - {"OPAQUE_PRF_INPUT_TOO_LONG", ERR_LIB_SSL, SSL_R_OPAQUE_PRF_INPUT_TOO_LONG}, - #else - {"OPAQUE_PRF_INPUT_TOO_LONG", 20, 327}, - #endif - #ifdef SSL_R_OVERFLOW_ERROR - {"OVERFLOW_ERROR", ERR_LIB_SSL, SSL_R_OVERFLOW_ERROR}, - #else - {"OVERFLOW_ERROR", 20, 237}, - #endif - #ifdef SSL_R_PACKET_LENGTH_TOO_LONG - {"PACKET_LENGTH_TOO_LONG", ERR_LIB_SSL, SSL_R_PACKET_LENGTH_TOO_LONG}, - #else - {"PACKET_LENGTH_TOO_LONG", 20, 198}, - #endif - #ifdef SSL_R_PARSE_TLSEXT - {"PARSE_TLSEXT", ERR_LIB_SSL, SSL_R_PARSE_TLSEXT}, - #else - {"PARSE_TLSEXT", 20, 227}, - #endif - #ifdef SSL_R_PATH_TOO_LONG - {"PATH_TOO_LONG", ERR_LIB_SSL, SSL_R_PATH_TOO_LONG}, - #else - {"PATH_TOO_LONG", 20, 270}, - #endif - #ifdef SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE - {"PEER_DID_NOT_RETURN_A_CERTIFICATE", ERR_LIB_SSL, SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE}, - #else - {"PEER_DID_NOT_RETURN_A_CERTIFICATE", 20, 199}, - #endif - #ifdef SSL_R_PEM_NAME_BAD_PREFIX - {"PEM_NAME_BAD_PREFIX", ERR_LIB_SSL, SSL_R_PEM_NAME_BAD_PREFIX}, - #else - {"PEM_NAME_BAD_PREFIX", 20, 391}, - #endif - #ifdef SSL_R_PEM_NAME_TOO_SHORT - {"PEM_NAME_TOO_SHORT", ERR_LIB_SSL, SSL_R_PEM_NAME_TOO_SHORT}, - #else - {"PEM_NAME_TOO_SHORT", 20, 392}, - #endif - #ifdef SSL_R_PIPELINE_FAILURE - {"PIPELINE_FAILURE", ERR_LIB_SSL, SSL_R_PIPELINE_FAILURE}, - #else - {"PIPELINE_FAILURE", 20, 406}, - #endif - #ifdef SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR - {"POST_HANDSHAKE_AUTH_ENCODING_ERR", ERR_LIB_SSL, SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR}, - #else - {"POST_HANDSHAKE_AUTH_ENCODING_ERR", 20, 278}, - #endif - #ifdef SSL_R_PRIVATE_KEY_MISMATCH - {"PRIVATE_KEY_MISMATCH", ERR_LIB_SSL, SSL_R_PRIVATE_KEY_MISMATCH}, - #else - {"PRIVATE_KEY_MISMATCH", 20, 288}, - #endif - #ifdef SSL_R_PROTOCOL_IS_SHUTDOWN - {"PROTOCOL_IS_SHUTDOWN", ERR_LIB_SSL, SSL_R_PROTOCOL_IS_SHUTDOWN}, - #else - {"PROTOCOL_IS_SHUTDOWN", 20, 207}, - #endif - #ifdef SSL_R_PSK_IDENTITY_NOT_FOUND - {"PSK_IDENTITY_NOT_FOUND", ERR_LIB_SSL, SSL_R_PSK_IDENTITY_NOT_FOUND}, - #else - {"PSK_IDENTITY_NOT_FOUND", 20, 223}, - #endif - #ifdef SSL_R_PSK_NO_CLIENT_CB - {"PSK_NO_CLIENT_CB", ERR_LIB_SSL, SSL_R_PSK_NO_CLIENT_CB}, - #else - {"PSK_NO_CLIENT_CB", 20, 224}, - #endif - #ifdef SSL_R_PSK_NO_SERVER_CB - {"PSK_NO_SERVER_CB", ERR_LIB_SSL, SSL_R_PSK_NO_SERVER_CB}, - #else - {"PSK_NO_SERVER_CB", 20, 225}, - #endif - #ifdef SSL_R_READ_BIO_NOT_SET - {"READ_BIO_NOT_SET", ERR_LIB_SSL, SSL_R_READ_BIO_NOT_SET}, - #else - {"READ_BIO_NOT_SET", 20, 211}, - #endif - #ifdef SSL_R_READ_TIMEOUT_EXPIRED - {"READ_TIMEOUT_EXPIRED", ERR_LIB_SSL, SSL_R_READ_TIMEOUT_EXPIRED}, - #else - {"READ_TIMEOUT_EXPIRED", 20, 312}, - #endif - #ifdef SSL_R_RECORD_LENGTH_MISMATCH - {"RECORD_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_RECORD_LENGTH_MISMATCH}, - #else - {"RECORD_LENGTH_MISMATCH", 20, 213}, - #endif - #ifdef SSL_R_RECORD_TOO_SMALL - {"RECORD_TOO_SMALL", ERR_LIB_SSL, SSL_R_RECORD_TOO_SMALL}, - #else - {"RECORD_TOO_SMALL", 20, 298}, - #endif - #ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG - {"RENEGOTIATE_EXT_TOO_LONG", ERR_LIB_SSL, SSL_R_RENEGOTIATE_EXT_TOO_LONG}, - #else - {"RENEGOTIATE_EXT_TOO_LONG", 20, 335}, - #endif - #ifdef SSL_R_RENEGOTIATION_ENCODING_ERR - {"RENEGOTIATION_ENCODING_ERR", ERR_LIB_SSL, SSL_R_RENEGOTIATION_ENCODING_ERR}, - #else - {"RENEGOTIATION_ENCODING_ERR", 20, 336}, - #endif - #ifdef SSL_R_RENEGOTIATION_MISMATCH - {"RENEGOTIATION_MISMATCH", ERR_LIB_SSL, SSL_R_RENEGOTIATION_MISMATCH}, - #else - {"RENEGOTIATION_MISMATCH", 20, 337}, - #endif - #ifdef SSL_R_REQUEST_PENDING - {"REQUEST_PENDING", ERR_LIB_SSL, SSL_R_REQUEST_PENDING}, - #else - {"REQUEST_PENDING", 20, 285}, - #endif - #ifdef SSL_R_REQUEST_SENT - {"REQUEST_SENT", ERR_LIB_SSL, SSL_R_REQUEST_SENT}, - #else - {"REQUEST_SENT", 20, 286}, - #endif - #ifdef SSL_R_REQUIRED_CIPHER_MISSING - {"REQUIRED_CIPHER_MISSING", ERR_LIB_SSL, SSL_R_REQUIRED_CIPHER_MISSING}, - #else - {"REQUIRED_CIPHER_MISSING", 20, 215}, - #endif - #ifdef SSL_R_REQUIRED_COMPRESSION_ALGORITHM_MISSING - {"REQUIRED_COMPRESSION_ALGORITHM_MISSING", ERR_LIB_SSL, SSL_R_REQUIRED_COMPRESSION_ALGORITHM_MISSING}, - #else - {"REQUIRED_COMPRESSION_ALGORITHM_MISSING", 20, 342}, - #endif - #ifdef SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING - {"SCSV_RECEIVED_WHEN_RENEGOTIATING", ERR_LIB_SSL, SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING}, - #else - {"SCSV_RECEIVED_WHEN_RENEGOTIATING", 20, 345}, - #endif - #ifdef SSL_R_SCT_VERIFICATION_FAILED - {"SCT_VERIFICATION_FAILED", ERR_LIB_SSL, SSL_R_SCT_VERIFICATION_FAILED}, - #else - {"SCT_VERIFICATION_FAILED", 20, 208}, - #endif - #ifdef SSL_R_SERVERHELLO_TLSEXT - {"SERVERHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_SERVERHELLO_TLSEXT}, - #else - {"SERVERHELLO_TLSEXT", 20, 275}, - #endif - #ifdef SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED - {"SESSION_ID_CONTEXT_UNINITIALIZED", ERR_LIB_SSL, SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED}, - #else - {"SESSION_ID_CONTEXT_UNINITIALIZED", 20, 277}, - #endif - #ifdef SSL_R_SHUTDOWN_WHILE_IN_INIT - {"SHUTDOWN_WHILE_IN_INIT", ERR_LIB_SSL, SSL_R_SHUTDOWN_WHILE_IN_INIT}, - #else - {"SHUTDOWN_WHILE_IN_INIT", 20, 407}, - #endif - #ifdef SSL_R_SIGNATURE_ALGORITHMS_ERROR - {"SIGNATURE_ALGORITHMS_ERROR", ERR_LIB_SSL, SSL_R_SIGNATURE_ALGORITHMS_ERROR}, - #else - {"SIGNATURE_ALGORITHMS_ERROR", 20, 360}, - #endif - #ifdef SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE - {"SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", ERR_LIB_SSL, SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE}, - #else - {"SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", 20, 220}, - #endif - #ifdef SSL_R_SRP_A_CALC - {"SRP_A_CALC", ERR_LIB_SSL, SSL_R_SRP_A_CALC}, - #else - {"SRP_A_CALC", 20, 361}, - #endif - #ifdef SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES - {"SRTP_COULD_NOT_ALLOCATE_PROFILES", ERR_LIB_SSL, SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES}, - #else - {"SRTP_COULD_NOT_ALLOCATE_PROFILES", 20, 362}, - #endif - #ifdef SSL_R_SRTP_PROTECTION_PROFILE_LIST_TOO_LONG - {"SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", ERR_LIB_SSL, SSL_R_SRTP_PROTECTION_PROFILE_LIST_TOO_LONG}, - #else - {"SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", 20, 363}, - #endif - #ifdef SSL_R_SRTP_UNKNOWN_PROTECTION_PROFILE - {"SRTP_UNKNOWN_PROTECTION_PROFILE", ERR_LIB_SSL, SSL_R_SRTP_UNKNOWN_PROTECTION_PROFILE}, - #else - {"SRTP_UNKNOWN_PROTECTION_PROFILE", 20, 364}, - #endif - #ifdef SSL_R_SSL2_CONNECTION_ID_TOO_LONG - {"SSL2_CONNECTION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL2_CONNECTION_ID_TOO_LONG}, - #else - {"SSL2_CONNECTION_ID_TOO_LONG", 20, 299}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_ECPOINTFORMAT - {"SSL3_EXT_INVALID_ECPOINTFORMAT", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_ECPOINTFORMAT}, - #else - {"SSL3_EXT_INVALID_ECPOINTFORMAT", 20, 321}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH - {"SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH}, - #else - {"SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", 20, 232}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_SERVERNAME - {"SSL3_EXT_INVALID_SERVERNAME", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_SERVERNAME}, - #else - {"SSL3_EXT_INVALID_SERVERNAME", 20, 319}, - #endif - #ifdef SSL_R_SSL3_EXT_INVALID_SERVERNAME_TYPE - {"SSL3_EXT_INVALID_SERVERNAME_TYPE", ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_SERVERNAME_TYPE}, - #else - {"SSL3_EXT_INVALID_SERVERNAME_TYPE", 20, 320}, - #endif - #ifdef SSL_R_SSL3_SESSION_ID_TOO_LONG - {"SSL3_SESSION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL3_SESSION_ID_TOO_LONG}, - #else - {"SSL3_SESSION_ID_TOO_LONG", 20, 300}, - #endif - #ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE - {"SSLV3_ALERT_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE}, - #else - {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, - #endif - #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC - {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, - #else - {"SSLV3_ALERT_BAD_RECORD_MAC", 20, 1020}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED - {"SSLV3_ALERT_CERTIFICATE_EXPIRED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED}, - #else - {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED - {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, - #else - {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, - #endif - #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN - {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, - #else - {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", 20, 1046}, - #endif - #ifdef SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE - {"SSLV3_ALERT_DECOMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE}, - #else - {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, - #endif - #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE - {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, - #else - {"SSLV3_ALERT_HANDSHAKE_FAILURE", 20, 1040}, - #endif - #ifdef SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER - {"SSLV3_ALERT_ILLEGAL_PARAMETER", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER}, - #else - {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, - #endif - #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE - {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, - #else - {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, - #endif - #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE - {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, - #else - {"SSLV3_ALERT_UNEXPECTED_MESSAGE", 20, 1010}, - #endif - #ifdef SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE - {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE}, - #else - {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, - #endif - #ifdef SSL_R_SSL_COMMAND_SECTION_EMPTY - {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_EMPTY}, - #else - {"SSL_COMMAND_SECTION_EMPTY", 20, 117}, - #endif - #ifdef SSL_R_SSL_COMMAND_SECTION_NOT_FOUND - {"SSL_COMMAND_SECTION_NOT_FOUND", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_NOT_FOUND}, - #else - {"SSL_COMMAND_SECTION_NOT_FOUND", 20, 125}, - #endif - #ifdef SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION - {"SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", ERR_LIB_SSL, SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION}, - #else - {"SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", 20, 228}, - #endif - #ifdef SSL_R_SSL_HANDSHAKE_FAILURE - {"SSL_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSL_HANDSHAKE_FAILURE}, - #else - {"SSL_HANDSHAKE_FAILURE", 20, 229}, - #endif - #ifdef SSL_R_SSL_LIBRARY_HAS_NO_CIPHERS - {"SSL_LIBRARY_HAS_NO_CIPHERS", ERR_LIB_SSL, SSL_R_SSL_LIBRARY_HAS_NO_CIPHERS}, - #else - {"SSL_LIBRARY_HAS_NO_CIPHERS", 20, 230}, - #endif - #ifdef SSL_R_SSL_NEGATIVE_LENGTH - {"SSL_NEGATIVE_LENGTH", ERR_LIB_SSL, SSL_R_SSL_NEGATIVE_LENGTH}, - #else - {"SSL_NEGATIVE_LENGTH", 20, 372}, - #endif - #ifdef SSL_R_SSL_SECTION_EMPTY - {"SSL_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_SECTION_EMPTY}, - #else - {"SSL_SECTION_EMPTY", 20, 126}, - #endif - #ifdef SSL_R_SSL_SECTION_NOT_FOUND - {"SSL_SECTION_NOT_FOUND", ERR_LIB_SSL, SSL_R_SSL_SECTION_NOT_FOUND}, - #else - {"SSL_SECTION_NOT_FOUND", 20, 136}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CALLBACK_FAILED - {"SSL_SESSION_ID_CALLBACK_FAILED", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CALLBACK_FAILED}, - #else - {"SSL_SESSION_ID_CALLBACK_FAILED", 20, 301}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CONFLICT - {"SSL_SESSION_ID_CONFLICT", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CONFLICT}, - #else - {"SSL_SESSION_ID_CONFLICT", 20, 302}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG - {"SSL_SESSION_ID_CONTEXT_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG}, - #else - {"SSL_SESSION_ID_CONTEXT_TOO_LONG", 20, 273}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_HAS_BAD_LENGTH - {"SSL_SESSION_ID_HAS_BAD_LENGTH", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_HAS_BAD_LENGTH}, - #else - {"SSL_SESSION_ID_HAS_BAD_LENGTH", 20, 303}, - #endif - #ifdef SSL_R_SSL_SESSION_ID_TOO_LONG - {"SSL_SESSION_ID_TOO_LONG", ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_TOO_LONG}, - #else - {"SSL_SESSION_ID_TOO_LONG", 20, 408}, - #endif - #ifdef SSL_R_SSL_SESSION_VERSION_MISMATCH - {"SSL_SESSION_VERSION_MISMATCH", ERR_LIB_SSL, SSL_R_SSL_SESSION_VERSION_MISMATCH}, - #else - {"SSL_SESSION_VERSION_MISMATCH", 20, 210}, - #endif - #ifdef SSL_R_STILL_IN_INIT - {"STILL_IN_INIT", ERR_LIB_SSL, SSL_R_STILL_IN_INIT}, - #else - {"STILL_IN_INIT", 20, 121}, - #endif - #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED - {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, - #else - {"TLSV13_ALERT_CERTIFICATE_REQUIRED", 20, 1116}, - #endif - #ifdef SSL_R_TLSV13_ALERT_MISSING_EXTENSION - {"TLSV13_ALERT_MISSING_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_MISSING_EXTENSION}, - #else - {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, - #endif - #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED - {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, - #else - {"TLSV1_ALERT_ACCESS_DENIED", 20, 1049}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECODE_ERROR - {"TLSV1_ALERT_DECODE_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECODE_ERROR}, - #else - {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED - {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, - #else - {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, - #endif - #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR - {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, - #else - {"TLSV1_ALERT_DECRYPT_ERROR", 20, 1051}, - #endif - #ifdef SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION - {"TLSV1_ALERT_EXPORT_RESTRICTION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION}, - #else - {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK - {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, - #else - {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", 20, 1086}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY - {"TLSV1_ALERT_INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY}, - #else - {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, - #endif - #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR - {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, - #else - {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, - #endif - #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION - {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, - #else - {"TLSV1_ALERT_NO_RENEGOTIATION", 20, 1100}, - #endif - #ifdef SSL_R_TLSV1_ALERT_PROTOCOL_VERSION - {"TLSV1_ALERT_PROTOCOL_VERSION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_PROTOCOL_VERSION}, - #else - {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, - #endif - #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW - {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, - #else - {"TLSV1_ALERT_RECORD_OVERFLOW", 20, 1022}, - #endif - #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA - {"TLSV1_ALERT_UNKNOWN_CA", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_CA}, - #else - {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, - #endif - #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED - {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, - #else - {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, - #endif - #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE - {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, - #else - {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", 20, 1114}, - #endif - #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE - {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE}, - #else - {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, - #endif - #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE - {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, - #else - {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, - #endif - #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME - {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, - #else - {"TLSV1_UNRECOGNIZED_NAME", 20, 1112}, - #endif - #ifdef SSL_R_TLSV1_UNSUPPORTED_EXTENSION - {"TLSV1_UNSUPPORTED_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV1_UNSUPPORTED_EXTENSION}, - #else - {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, - #endif - #ifdef SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT - {"TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", ERR_LIB_SSL, SSL_R_TLS_HEARTBEAT_PEER_DOESNT_ACCEPT}, - #else - {"TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", 20, 365}, - #endif - #ifdef SSL_R_TLS_HEARTBEAT_PENDING - {"TLS_HEARTBEAT_PENDING", ERR_LIB_SSL, SSL_R_TLS_HEARTBEAT_PENDING}, - #else - {"TLS_HEARTBEAT_PENDING", 20, 366}, - #endif - #ifdef SSL_R_TLS_ILLEGAL_EXPORTER_LABEL - {"TLS_ILLEGAL_EXPORTER_LABEL", ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL}, - #else - {"TLS_ILLEGAL_EXPORTER_LABEL", 20, 367}, - #endif - #ifdef SSL_R_TLS_INVALID_ECPOINTFORMAT_LIST - {"TLS_INVALID_ECPOINTFORMAT_LIST", ERR_LIB_SSL, SSL_R_TLS_INVALID_ECPOINTFORMAT_LIST}, - #else - {"TLS_INVALID_ECPOINTFORMAT_LIST", 20, 157}, - #endif - #ifdef SSL_R_TOO_MANY_KEY_UPDATES - {"TOO_MANY_KEY_UPDATES", ERR_LIB_SSL, SSL_R_TOO_MANY_KEY_UPDATES}, - #else - {"TOO_MANY_KEY_UPDATES", 20, 132}, - #endif - #ifdef SSL_R_TOO_MANY_WARN_ALERTS - {"TOO_MANY_WARN_ALERTS", ERR_LIB_SSL, SSL_R_TOO_MANY_WARN_ALERTS}, - #else - {"TOO_MANY_WARN_ALERTS", 20, 409}, - #endif - #ifdef SSL_R_TOO_MUCH_EARLY_DATA - {"TOO_MUCH_EARLY_DATA", ERR_LIB_SSL, SSL_R_TOO_MUCH_EARLY_DATA}, - #else - {"TOO_MUCH_EARLY_DATA", 20, 164}, - #endif - #ifdef SSL_R_UNABLE_TO_DECODE_ECDH_CERTS - {"UNABLE_TO_DECODE_ECDH_CERTS", ERR_LIB_SSL, SSL_R_UNABLE_TO_DECODE_ECDH_CERTS}, - #else - {"UNABLE_TO_DECODE_ECDH_CERTS", 20, 313}, - #endif - #ifdef SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS - {"UNABLE_TO_FIND_ECDH_PARAMETERS", ERR_LIB_SSL, SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS}, - #else - {"UNABLE_TO_FIND_ECDH_PARAMETERS", 20, 314}, - #endif - #ifdef SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS - {"UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", ERR_LIB_SSL, SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS}, - #else - {"UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", 20, 239}, - #endif - #ifdef SSL_R_UNABLE_TO_LOAD_SSL3_MD5_ROUTINES - {"UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", ERR_LIB_SSL, SSL_R_UNABLE_TO_LOAD_SSL3_MD5_ROUTINES}, - #else - {"UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", 20, 242}, - #endif - #ifdef SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES - {"UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", ERR_LIB_SSL, SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES}, - #else - {"UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", 20, 243}, - #endif - #ifdef SSL_R_UNEXPECTED_CCS_MESSAGE - {"UNEXPECTED_CCS_MESSAGE", ERR_LIB_SSL, SSL_R_UNEXPECTED_CCS_MESSAGE}, - #else - {"UNEXPECTED_CCS_MESSAGE", 20, 262}, - #endif - #ifdef SSL_R_UNEXPECTED_END_OF_EARLY_DATA - {"UNEXPECTED_END_OF_EARLY_DATA", ERR_LIB_SSL, SSL_R_UNEXPECTED_END_OF_EARLY_DATA}, - #else - {"UNEXPECTED_END_OF_EARLY_DATA", 20, 178}, - #endif - #ifdef SSL_R_UNEXPECTED_MESSAGE - {"UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_UNEXPECTED_MESSAGE}, - #else - {"UNEXPECTED_MESSAGE", 20, 244}, - #endif - #ifdef SSL_R_UNEXPECTED_RECORD - {"UNEXPECTED_RECORD", ERR_LIB_SSL, SSL_R_UNEXPECTED_RECORD}, - #else - {"UNEXPECTED_RECORD", 20, 245}, - #endif - #ifdef SSL_R_UNINITIALIZED - {"UNINITIALIZED", ERR_LIB_SSL, SSL_R_UNINITIALIZED}, - #else - {"UNINITIALIZED", 20, 276}, - #endif - #ifdef SSL_R_UNKNOWN_ALERT_TYPE - {"UNKNOWN_ALERT_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_ALERT_TYPE}, - #else - {"UNKNOWN_ALERT_TYPE", 20, 246}, - #endif - #ifdef SSL_R_UNKNOWN_CERTIFICATE_TYPE - {"UNKNOWN_CERTIFICATE_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE}, - #else - {"UNKNOWN_CERTIFICATE_TYPE", 20, 247}, - #endif - #ifdef SSL_R_UNKNOWN_CIPHER_RETURNED - {"UNKNOWN_CIPHER_RETURNED", ERR_LIB_SSL, SSL_R_UNKNOWN_CIPHER_RETURNED}, - #else - {"UNKNOWN_CIPHER_RETURNED", 20, 248}, - #endif - #ifdef SSL_R_UNKNOWN_CIPHER_TYPE - {"UNKNOWN_CIPHER_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_CIPHER_TYPE}, - #else - {"UNKNOWN_CIPHER_TYPE", 20, 249}, - #endif - #ifdef SSL_R_UNKNOWN_CMD_NAME - {"UNKNOWN_CMD_NAME", ERR_LIB_SSL, SSL_R_UNKNOWN_CMD_NAME}, - #else - {"UNKNOWN_CMD_NAME", 20, 386}, - #endif - #ifdef SSL_R_UNKNOWN_COMMAND - {"UNKNOWN_COMMAND", ERR_LIB_SSL, SSL_R_UNKNOWN_COMMAND}, - #else - {"UNKNOWN_COMMAND", 20, 139}, - #endif - #ifdef SSL_R_UNKNOWN_DIGEST - {"UNKNOWN_DIGEST", ERR_LIB_SSL, SSL_R_UNKNOWN_DIGEST}, - #else - {"UNKNOWN_DIGEST", 20, 368}, - #endif - #ifdef SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE - {"UNKNOWN_KEY_EXCHANGE_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE}, - #else - {"UNKNOWN_KEY_EXCHANGE_TYPE", 20, 250}, - #endif - #ifdef SSL_R_UNKNOWN_PKEY_TYPE - {"UNKNOWN_PKEY_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_PKEY_TYPE}, - #else - {"UNKNOWN_PKEY_TYPE", 20, 251}, - #endif - #ifdef SSL_R_UNKNOWN_PROTOCOL - {"UNKNOWN_PROTOCOL", ERR_LIB_SSL, SSL_R_UNKNOWN_PROTOCOL}, - #else - {"UNKNOWN_PROTOCOL", 20, 252}, - #endif - #ifdef SSL_R_UNKNOWN_SSL_VERSION - {"UNKNOWN_SSL_VERSION", ERR_LIB_SSL, SSL_R_UNKNOWN_SSL_VERSION}, - #else - {"UNKNOWN_SSL_VERSION", 20, 254}, - #endif - #ifdef SSL_R_UNKNOWN_STATE - {"UNKNOWN_STATE", ERR_LIB_SSL, SSL_R_UNKNOWN_STATE}, - #else - {"UNKNOWN_STATE", 20, 255}, - #endif - #ifdef SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED - {"UNSAFE_LEGACY_RENEGOTIATION_DISABLED", ERR_LIB_SSL, SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED}, - #else - {"UNSAFE_LEGACY_RENEGOTIATION_DISABLED", 20, 338}, - #endif - #ifdef SSL_R_UNSOLICITED_EXTENSION - {"UNSOLICITED_EXTENSION", ERR_LIB_SSL, SSL_R_UNSOLICITED_EXTENSION}, - #else - {"UNSOLICITED_EXTENSION", 20, 217}, - #endif - #ifdef SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM - {"UNSUPPORTED_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM}, - #else - {"UNSUPPORTED_COMPRESSION_ALGORITHM", 20, 257}, - #endif - #ifdef SSL_R_UNSUPPORTED_DIGEST_TYPE - {"UNSUPPORTED_DIGEST_TYPE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_DIGEST_TYPE}, - #else - {"UNSUPPORTED_DIGEST_TYPE", 20, 326}, - #endif - #ifdef SSL_R_UNSUPPORTED_ELLIPTIC_CURVE - {"UNSUPPORTED_ELLIPTIC_CURVE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_ELLIPTIC_CURVE}, - #else - {"UNSUPPORTED_ELLIPTIC_CURVE", 20, 315}, - #endif - #ifdef SSL_R_UNSUPPORTED_PROTOCOL - {"UNSUPPORTED_PROTOCOL", ERR_LIB_SSL, SSL_R_UNSUPPORTED_PROTOCOL}, - #else - {"UNSUPPORTED_PROTOCOL", 20, 258}, - #endif - #ifdef SSL_R_UNSUPPORTED_SSL_VERSION - {"UNSUPPORTED_SSL_VERSION", ERR_LIB_SSL, SSL_R_UNSUPPORTED_SSL_VERSION}, - #else - {"UNSUPPORTED_SSL_VERSION", 20, 259}, - #endif - #ifdef SSL_R_UNSUPPORTED_STATUS_TYPE - {"UNSUPPORTED_STATUS_TYPE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_STATUS_TYPE}, - #else - {"UNSUPPORTED_STATUS_TYPE", 20, 329}, - #endif - #ifdef SSL_R_USE_SRTP_NOT_NEGOTIATED - {"USE_SRTP_NOT_NEGOTIATED", ERR_LIB_SSL, SSL_R_USE_SRTP_NOT_NEGOTIATED}, - #else - {"USE_SRTP_NOT_NEGOTIATED", 20, 369}, - #endif - #ifdef SSL_R_VERSION_TOO_HIGH - {"VERSION_TOO_HIGH", ERR_LIB_SSL, SSL_R_VERSION_TOO_HIGH}, - #else - {"VERSION_TOO_HIGH", 20, 166}, - #endif - #ifdef SSL_R_VERSION_TOO_LOW - {"VERSION_TOO_LOW", ERR_LIB_SSL, SSL_R_VERSION_TOO_LOW}, - #else - {"VERSION_TOO_LOW", 20, 396}, - #endif - #ifdef SSL_R_WRONG_CERTIFICATE_TYPE - {"WRONG_CERTIFICATE_TYPE", ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE}, - #else - {"WRONG_CERTIFICATE_TYPE", 20, 383}, - #endif - #ifdef SSL_R_WRONG_CIPHER_RETURNED - {"WRONG_CIPHER_RETURNED", ERR_LIB_SSL, SSL_R_WRONG_CIPHER_RETURNED}, - #else - {"WRONG_CIPHER_RETURNED", 20, 261}, - #endif - #ifdef SSL_R_WRONG_CURVE - {"WRONG_CURVE", ERR_LIB_SSL, SSL_R_WRONG_CURVE}, - #else - {"WRONG_CURVE", 20, 378}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_LENGTH - {"WRONG_SIGNATURE_LENGTH", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_LENGTH}, - #else - {"WRONG_SIGNATURE_LENGTH", 20, 264}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_SIZE - {"WRONG_SIGNATURE_SIZE", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_SIZE}, - #else - {"WRONG_SIGNATURE_SIZE", 20, 265}, - #endif - #ifdef SSL_R_WRONG_SIGNATURE_TYPE - {"WRONG_SIGNATURE_TYPE", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_TYPE}, - #else - {"WRONG_SIGNATURE_TYPE", 20, 370}, - #endif - #ifdef SSL_R_WRONG_SSL_VERSION - {"WRONG_SSL_VERSION", ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION}, - #else - {"WRONG_SSL_VERSION", 20, 266}, - #endif - #ifdef SSL_R_WRONG_VERSION_NUMBER - {"WRONG_VERSION_NUMBER", ERR_LIB_SSL, SSL_R_WRONG_VERSION_NUMBER}, - #else - {"WRONG_VERSION_NUMBER", 20, 267}, - #endif - #ifdef SSL_R_X509_LIB - {"X509_LIB", ERR_LIB_SSL, SSL_R_X509_LIB}, - #else - {"X509_LIB", 20, 268}, - #endif - #ifdef SSL_R_X509_VERIFICATION_SETUP_PROBLEMS - {"X509_VERIFICATION_SETUP_PROBLEMS", ERR_LIB_SSL, SSL_R_X509_VERIFICATION_SETUP_PROBLEMS}, - #else - {"X509_VERIFICATION_SETUP_PROBLEMS", 20, 269}, - #endif - #ifdef TS_R_BAD_PKCS7_TYPE - {"BAD_PKCS7_TYPE", ERR_LIB_TS, TS_R_BAD_PKCS7_TYPE}, - #else - {"BAD_PKCS7_TYPE", 47, 132}, - #endif - #ifdef TS_R_BAD_TYPE - {"BAD_TYPE", ERR_LIB_TS, TS_R_BAD_TYPE}, - #else - {"BAD_TYPE", 47, 133}, - #endif - #ifdef TS_R_CANNOT_LOAD_CERT - {"CANNOT_LOAD_CERT", ERR_LIB_TS, TS_R_CANNOT_LOAD_CERT}, - #else - {"CANNOT_LOAD_CERT", 47, 137}, - #endif - #ifdef TS_R_CANNOT_LOAD_KEY - {"CANNOT_LOAD_KEY", ERR_LIB_TS, TS_R_CANNOT_LOAD_KEY}, - #else - {"CANNOT_LOAD_KEY", 47, 138}, - #endif - #ifdef TS_R_CERTIFICATE_VERIFY_ERROR - {"CERTIFICATE_VERIFY_ERROR", ERR_LIB_TS, TS_R_CERTIFICATE_VERIFY_ERROR}, - #else - {"CERTIFICATE_VERIFY_ERROR", 47, 100}, - #endif - #ifdef TS_R_COULD_NOT_SET_ENGINE - {"COULD_NOT_SET_ENGINE", ERR_LIB_TS, TS_R_COULD_NOT_SET_ENGINE}, - #else - {"COULD_NOT_SET_ENGINE", 47, 127}, - #endif - #ifdef TS_R_COULD_NOT_SET_TIME - {"COULD_NOT_SET_TIME", ERR_LIB_TS, TS_R_COULD_NOT_SET_TIME}, - #else - {"COULD_NOT_SET_TIME", 47, 115}, - #endif - #ifdef TS_R_DETACHED_CONTENT - {"DETACHED_CONTENT", ERR_LIB_TS, TS_R_DETACHED_CONTENT}, - #else - {"DETACHED_CONTENT", 47, 134}, - #endif - #ifdef TS_R_ESS_ADD_SIGNING_CERT_ERROR - {"ESS_ADD_SIGNING_CERT_ERROR", ERR_LIB_TS, TS_R_ESS_ADD_SIGNING_CERT_ERROR}, - #else - {"ESS_ADD_SIGNING_CERT_ERROR", 47, 116}, - #endif - #ifdef TS_R_ESS_ADD_SIGNING_CERT_V2_ERROR - {"ESS_ADD_SIGNING_CERT_V2_ERROR", ERR_LIB_TS, TS_R_ESS_ADD_SIGNING_CERT_V2_ERROR}, - #else - {"ESS_ADD_SIGNING_CERT_V2_ERROR", 47, 139}, - #endif - #ifdef TS_R_ESS_SIGNING_CERTIFICATE_ERROR - {"ESS_SIGNING_CERTIFICATE_ERROR", ERR_LIB_TS, TS_R_ESS_SIGNING_CERTIFICATE_ERROR}, - #else - {"ESS_SIGNING_CERTIFICATE_ERROR", 47, 101}, - #endif - #ifdef TS_R_INVALID_NULL_POINTER - {"INVALID_NULL_POINTER", ERR_LIB_TS, TS_R_INVALID_NULL_POINTER}, - #else - {"INVALID_NULL_POINTER", 47, 102}, - #endif - #ifdef TS_R_INVALID_SIGNER_CERTIFICATE_PURPOSE - {"INVALID_SIGNER_CERTIFICATE_PURPOSE", ERR_LIB_TS, TS_R_INVALID_SIGNER_CERTIFICATE_PURPOSE}, - #else - {"INVALID_SIGNER_CERTIFICATE_PURPOSE", 47, 117}, - #endif - #ifdef TS_R_MESSAGE_IMPRINT_MISMATCH - {"MESSAGE_IMPRINT_MISMATCH", ERR_LIB_TS, TS_R_MESSAGE_IMPRINT_MISMATCH}, - #else - {"MESSAGE_IMPRINT_MISMATCH", 47, 103}, - #endif - #ifdef TS_R_NONCE_MISMATCH - {"NONCE_MISMATCH", ERR_LIB_TS, TS_R_NONCE_MISMATCH}, - #else - {"NONCE_MISMATCH", 47, 104}, - #endif - #ifdef TS_R_NONCE_NOT_RETURNED - {"NONCE_NOT_RETURNED", ERR_LIB_TS, TS_R_NONCE_NOT_RETURNED}, - #else - {"NONCE_NOT_RETURNED", 47, 105}, - #endif - #ifdef TS_R_NO_CONTENT - {"NO_CONTENT", ERR_LIB_TS, TS_R_NO_CONTENT}, - #else - {"NO_CONTENT", 47, 106}, - #endif - #ifdef TS_R_NO_TIME_STAMP_TOKEN - {"NO_TIME_STAMP_TOKEN", ERR_LIB_TS, TS_R_NO_TIME_STAMP_TOKEN}, - #else - {"NO_TIME_STAMP_TOKEN", 47, 107}, - #endif - #ifdef TS_R_PKCS7_ADD_SIGNATURE_ERROR - {"PKCS7_ADD_SIGNATURE_ERROR", ERR_LIB_TS, TS_R_PKCS7_ADD_SIGNATURE_ERROR}, - #else - {"PKCS7_ADD_SIGNATURE_ERROR", 47, 118}, - #endif - #ifdef TS_R_PKCS7_ADD_SIGNED_ATTR_ERROR - {"PKCS7_ADD_SIGNED_ATTR_ERROR", ERR_LIB_TS, TS_R_PKCS7_ADD_SIGNED_ATTR_ERROR}, - #else - {"PKCS7_ADD_SIGNED_ATTR_ERROR", 47, 119}, - #endif - #ifdef TS_R_PKCS7_TO_TS_TST_INFO_FAILED - {"PKCS7_TO_TS_TST_INFO_FAILED", ERR_LIB_TS, TS_R_PKCS7_TO_TS_TST_INFO_FAILED}, - #else - {"PKCS7_TO_TS_TST_INFO_FAILED", 47, 129}, - #endif - #ifdef TS_R_POLICY_MISMATCH - {"POLICY_MISMATCH", ERR_LIB_TS, TS_R_POLICY_MISMATCH}, - #else - {"POLICY_MISMATCH", 47, 108}, - #endif - #ifdef TS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", ERR_LIB_TS, TS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE}, - #else - {"PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", 47, 120}, - #endif - #ifdef TS_R_RESPONSE_SETUP_ERROR - {"RESPONSE_SETUP_ERROR", ERR_LIB_TS, TS_R_RESPONSE_SETUP_ERROR}, - #else - {"RESPONSE_SETUP_ERROR", 47, 121}, - #endif - #ifdef TS_R_SIGNATURE_FAILURE - {"SIGNATURE_FAILURE", ERR_LIB_TS, TS_R_SIGNATURE_FAILURE}, - #else - {"SIGNATURE_FAILURE", 47, 109}, - #endif - #ifdef TS_R_THERE_MUST_BE_ONE_SIGNER - {"THERE_MUST_BE_ONE_SIGNER", ERR_LIB_TS, TS_R_THERE_MUST_BE_ONE_SIGNER}, - #else - {"THERE_MUST_BE_ONE_SIGNER", 47, 110}, - #endif - #ifdef TS_R_TIME_SYSCALL_ERROR - {"TIME_SYSCALL_ERROR", ERR_LIB_TS, TS_R_TIME_SYSCALL_ERROR}, - #else - {"TIME_SYSCALL_ERROR", 47, 122}, - #endif - #ifdef TS_R_TOKEN_NOT_PRESENT - {"TOKEN_NOT_PRESENT", ERR_LIB_TS, TS_R_TOKEN_NOT_PRESENT}, - #else - {"TOKEN_NOT_PRESENT", 47, 130}, - #endif - #ifdef TS_R_TOKEN_PRESENT - {"TOKEN_PRESENT", ERR_LIB_TS, TS_R_TOKEN_PRESENT}, - #else - {"TOKEN_PRESENT", 47, 131}, - #endif - #ifdef TS_R_TSA_NAME_MISMATCH - {"TSA_NAME_MISMATCH", ERR_LIB_TS, TS_R_TSA_NAME_MISMATCH}, - #else - {"TSA_NAME_MISMATCH", 47, 111}, - #endif - #ifdef TS_R_TSA_UNTRUSTED - {"TSA_UNTRUSTED", ERR_LIB_TS, TS_R_TSA_UNTRUSTED}, - #else - {"TSA_UNTRUSTED", 47, 112}, - #endif - #ifdef TS_R_TST_INFO_SETUP_ERROR - {"TST_INFO_SETUP_ERROR", ERR_LIB_TS, TS_R_TST_INFO_SETUP_ERROR}, - #else - {"TST_INFO_SETUP_ERROR", 47, 123}, - #endif - #ifdef TS_R_TS_DATASIGN - {"TS_DATASIGN", ERR_LIB_TS, TS_R_TS_DATASIGN}, - #else - {"TS_DATASIGN", 47, 124}, - #endif - #ifdef TS_R_UNACCEPTABLE_POLICY - {"UNACCEPTABLE_POLICY", ERR_LIB_TS, TS_R_UNACCEPTABLE_POLICY}, - #else - {"UNACCEPTABLE_POLICY", 47, 125}, - #endif - #ifdef TS_R_UNSUPPORTED_MD_ALGORITHM - {"UNSUPPORTED_MD_ALGORITHM", ERR_LIB_TS, TS_R_UNSUPPORTED_MD_ALGORITHM}, - #else - {"UNSUPPORTED_MD_ALGORITHM", 47, 126}, - #endif - #ifdef TS_R_UNSUPPORTED_VERSION - {"UNSUPPORTED_VERSION", ERR_LIB_TS, TS_R_UNSUPPORTED_VERSION}, - #else - {"UNSUPPORTED_VERSION", 47, 113}, - #endif - #ifdef TS_R_VAR_BAD_VALUE - {"VAR_BAD_VALUE", ERR_LIB_TS, TS_R_VAR_BAD_VALUE}, - #else - {"VAR_BAD_VALUE", 47, 135}, - #endif - #ifdef TS_R_VAR_LOOKUP_FAILURE - {"VAR_LOOKUP_FAILURE", ERR_LIB_TS, TS_R_VAR_LOOKUP_FAILURE}, - #else - {"VAR_LOOKUP_FAILURE", 47, 136}, - #endif - #ifdef TS_R_WRONG_CONTENT_TYPE - {"WRONG_CONTENT_TYPE", ERR_LIB_TS, TS_R_WRONG_CONTENT_TYPE}, - #else - {"WRONG_CONTENT_TYPE", 47, 114}, - #endif - #ifdef UI_R_COMMON_OK_AND_CANCEL_CHARACTERS - {"COMMON_OK_AND_CANCEL_CHARACTERS", ERR_LIB_UI, UI_R_COMMON_OK_AND_CANCEL_CHARACTERS}, - #else - {"COMMON_OK_AND_CANCEL_CHARACTERS", 40, 104}, - #endif - #ifdef UI_R_INDEX_TOO_LARGE - {"INDEX_TOO_LARGE", ERR_LIB_UI, UI_R_INDEX_TOO_LARGE}, - #else - {"INDEX_TOO_LARGE", 40, 102}, - #endif - #ifdef UI_R_INDEX_TOO_SMALL - {"INDEX_TOO_SMALL", ERR_LIB_UI, UI_R_INDEX_TOO_SMALL}, - #else - {"INDEX_TOO_SMALL", 40, 103}, - #endif - #ifdef UI_R_NO_RESULT_BUFFER - {"NO_RESULT_BUFFER", ERR_LIB_UI, UI_R_NO_RESULT_BUFFER}, - #else - {"NO_RESULT_BUFFER", 40, 105}, - #endif - #ifdef UI_R_PROCESSING_ERROR - {"PROCESSING_ERROR", ERR_LIB_UI, UI_R_PROCESSING_ERROR}, - #else - {"PROCESSING_ERROR", 40, 107}, - #endif - #ifdef UI_R_RESULT_TOO_LARGE - {"RESULT_TOO_LARGE", ERR_LIB_UI, UI_R_RESULT_TOO_LARGE}, - #else - {"RESULT_TOO_LARGE", 40, 100}, - #endif - #ifdef UI_R_RESULT_TOO_SMALL - {"RESULT_TOO_SMALL", ERR_LIB_UI, UI_R_RESULT_TOO_SMALL}, - #else - {"RESULT_TOO_SMALL", 40, 101}, - #endif - #ifdef UI_R_SYSASSIGN_ERROR - {"SYSASSIGN_ERROR", ERR_LIB_UI, UI_R_SYSASSIGN_ERROR}, - #else - {"SYSASSIGN_ERROR", 40, 109}, - #endif - #ifdef UI_R_SYSDASSGN_ERROR - {"SYSDASSGN_ERROR", ERR_LIB_UI, UI_R_SYSDASSGN_ERROR}, - #else - {"SYSDASSGN_ERROR", 40, 110}, - #endif - #ifdef UI_R_SYSQIOW_ERROR - {"SYSQIOW_ERROR", ERR_LIB_UI, UI_R_SYSQIOW_ERROR}, - #else - {"SYSQIOW_ERROR", 40, 111}, - #endif - #ifdef UI_R_UNKNOWN_CONTROL_COMMAND - {"UNKNOWN_CONTROL_COMMAND", ERR_LIB_UI, UI_R_UNKNOWN_CONTROL_COMMAND}, - #else - {"UNKNOWN_CONTROL_COMMAND", 40, 106}, - #endif - #ifdef UI_R_UNKNOWN_TTYGET_ERRNO_VALUE - {"UNKNOWN_TTYGET_ERRNO_VALUE", ERR_LIB_UI, UI_R_UNKNOWN_TTYGET_ERRNO_VALUE}, - #else - {"UNKNOWN_TTYGET_ERRNO_VALUE", 40, 108}, - #endif - #ifdef UI_R_USER_DATA_DUPLICATION_UNSUPPORTED - {"USER_DATA_DUPLICATION_UNSUPPORTED", ERR_LIB_UI, UI_R_USER_DATA_DUPLICATION_UNSUPPORTED}, - #else - {"USER_DATA_DUPLICATION_UNSUPPORTED", 40, 112}, - #endif - #ifdef X509V3_R_BAD_IP_ADDRESS - {"BAD_IP_ADDRESS", ERR_LIB_X509V3, X509V3_R_BAD_IP_ADDRESS}, - #else - {"BAD_IP_ADDRESS", 34, 118}, - #endif - #ifdef X509V3_R_BAD_OBJECT - {"BAD_OBJECT", ERR_LIB_X509V3, X509V3_R_BAD_OBJECT}, - #else - {"BAD_OBJECT", 34, 119}, - #endif - #ifdef X509V3_R_BN_DEC2BN_ERROR - {"BN_DEC2BN_ERROR", ERR_LIB_X509V3, X509V3_R_BN_DEC2BN_ERROR}, - #else - {"BN_DEC2BN_ERROR", 34, 100}, - #endif - #ifdef X509V3_R_BN_TO_ASN1_INTEGER_ERROR - {"BN_TO_ASN1_INTEGER_ERROR", ERR_LIB_X509V3, X509V3_R_BN_TO_ASN1_INTEGER_ERROR}, - #else - {"BN_TO_ASN1_INTEGER_ERROR", 34, 101}, - #endif - #ifdef X509V3_R_DIRNAME_ERROR - {"DIRNAME_ERROR", ERR_LIB_X509V3, X509V3_R_DIRNAME_ERROR}, - #else - {"DIRNAME_ERROR", 34, 149}, - #endif - #ifdef X509V3_R_DISTPOINT_ALREADY_SET - {"DISTPOINT_ALREADY_SET", ERR_LIB_X509V3, X509V3_R_DISTPOINT_ALREADY_SET}, - #else - {"DISTPOINT_ALREADY_SET", 34, 160}, - #endif - #ifdef X509V3_R_DUPLICATE_ZONE_ID - {"DUPLICATE_ZONE_ID", ERR_LIB_X509V3, X509V3_R_DUPLICATE_ZONE_ID}, - #else - {"DUPLICATE_ZONE_ID", 34, 133}, - #endif - #ifdef X509V3_R_ERROR_CONVERTING_ZONE - {"ERROR_CONVERTING_ZONE", ERR_LIB_X509V3, X509V3_R_ERROR_CONVERTING_ZONE}, - #else - {"ERROR_CONVERTING_ZONE", 34, 131}, - #endif - #ifdef X509V3_R_ERROR_CREATING_EXTENSION - {"ERROR_CREATING_EXTENSION", ERR_LIB_X509V3, X509V3_R_ERROR_CREATING_EXTENSION}, - #else - {"ERROR_CREATING_EXTENSION", 34, 144}, - #endif - #ifdef X509V3_R_ERROR_IN_EXTENSION - {"ERROR_IN_EXTENSION", ERR_LIB_X509V3, X509V3_R_ERROR_IN_EXTENSION}, - #else - {"ERROR_IN_EXTENSION", 34, 128}, - #endif - #ifdef X509V3_R_EXPECTED_A_SECTION_NAME - {"EXPECTED_A_SECTION_NAME", ERR_LIB_X509V3, X509V3_R_EXPECTED_A_SECTION_NAME}, - #else - {"EXPECTED_A_SECTION_NAME", 34, 137}, - #endif - #ifdef X509V3_R_EXTENSION_EXISTS - {"EXTENSION_EXISTS", ERR_LIB_X509V3, X509V3_R_EXTENSION_EXISTS}, - #else - {"EXTENSION_EXISTS", 34, 145}, - #endif - #ifdef X509V3_R_EXTENSION_NAME_ERROR - {"EXTENSION_NAME_ERROR", ERR_LIB_X509V3, X509V3_R_EXTENSION_NAME_ERROR}, - #else - {"EXTENSION_NAME_ERROR", 34, 115}, - #endif - #ifdef X509V3_R_EXTENSION_NOT_FOUND - {"EXTENSION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_EXTENSION_NOT_FOUND}, - #else - {"EXTENSION_NOT_FOUND", 34, 102}, - #endif - #ifdef X509V3_R_EXTENSION_SETTING_NOT_SUPPORTED - {"EXTENSION_SETTING_NOT_SUPPORTED", ERR_LIB_X509V3, X509V3_R_EXTENSION_SETTING_NOT_SUPPORTED}, - #else - {"EXTENSION_SETTING_NOT_SUPPORTED", 34, 103}, - #endif - #ifdef X509V3_R_EXTENSION_VALUE_ERROR - {"EXTENSION_VALUE_ERROR", ERR_LIB_X509V3, X509V3_R_EXTENSION_VALUE_ERROR}, - #else - {"EXTENSION_VALUE_ERROR", 34, 116}, - #endif - #ifdef X509V3_R_ILLEGAL_EMPTY_EXTENSION - {"ILLEGAL_EMPTY_EXTENSION", ERR_LIB_X509V3, X509V3_R_ILLEGAL_EMPTY_EXTENSION}, - #else - {"ILLEGAL_EMPTY_EXTENSION", 34, 151}, - #endif - #ifdef X509V3_R_INCORRECT_POLICY_SYNTAX_TAG - {"INCORRECT_POLICY_SYNTAX_TAG", ERR_LIB_X509V3, X509V3_R_INCORRECT_POLICY_SYNTAX_TAG}, - #else - {"INCORRECT_POLICY_SYNTAX_TAG", 34, 152}, - #endif - #ifdef X509V3_R_INVALID_ASNUMBER - {"INVALID_ASNUMBER", ERR_LIB_X509V3, X509V3_R_INVALID_ASNUMBER}, - #else - {"INVALID_ASNUMBER", 34, 162}, - #endif - #ifdef X509V3_R_INVALID_ASRANGE - {"INVALID_ASRANGE", ERR_LIB_X509V3, X509V3_R_INVALID_ASRANGE}, - #else - {"INVALID_ASRANGE", 34, 163}, - #endif - #ifdef X509V3_R_INVALID_BOOLEAN_STRING - {"INVALID_BOOLEAN_STRING", ERR_LIB_X509V3, X509V3_R_INVALID_BOOLEAN_STRING}, - #else - {"INVALID_BOOLEAN_STRING", 34, 104}, - #endif - #ifdef X509V3_R_INVALID_EXTENSION_STRING - {"INVALID_EXTENSION_STRING", ERR_LIB_X509V3, X509V3_R_INVALID_EXTENSION_STRING}, - #else - {"INVALID_EXTENSION_STRING", 34, 105}, - #endif - #ifdef X509V3_R_INVALID_INHERITANCE - {"INVALID_INHERITANCE", ERR_LIB_X509V3, X509V3_R_INVALID_INHERITANCE}, - #else - {"INVALID_INHERITANCE", 34, 165}, - #endif - #ifdef X509V3_R_INVALID_IPADDRESS - {"INVALID_IPADDRESS", ERR_LIB_X509V3, X509V3_R_INVALID_IPADDRESS}, - #else - {"INVALID_IPADDRESS", 34, 166}, - #endif - #ifdef X509V3_R_INVALID_MULTIPLE_RDNS - {"INVALID_MULTIPLE_RDNS", ERR_LIB_X509V3, X509V3_R_INVALID_MULTIPLE_RDNS}, - #else - {"INVALID_MULTIPLE_RDNS", 34, 161}, - #endif - #ifdef X509V3_R_INVALID_NAME - {"INVALID_NAME", ERR_LIB_X509V3, X509V3_R_INVALID_NAME}, - #else - {"INVALID_NAME", 34, 106}, - #endif - #ifdef X509V3_R_INVALID_NULL_ARGUMENT - {"INVALID_NULL_ARGUMENT", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_ARGUMENT}, - #else - {"INVALID_NULL_ARGUMENT", 34, 107}, - #endif - #ifdef X509V3_R_INVALID_NULL_NAME - {"INVALID_NULL_NAME", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_NAME}, - #else - {"INVALID_NULL_NAME", 34, 108}, - #endif - #ifdef X509V3_R_INVALID_NULL_VALUE - {"INVALID_NULL_VALUE", ERR_LIB_X509V3, X509V3_R_INVALID_NULL_VALUE}, - #else - {"INVALID_NULL_VALUE", 34, 109}, - #endif - #ifdef X509V3_R_INVALID_NUMBER - {"INVALID_NUMBER", ERR_LIB_X509V3, X509V3_R_INVALID_NUMBER}, - #else - {"INVALID_NUMBER", 34, 140}, - #endif - #ifdef X509V3_R_INVALID_NUMBERS - {"INVALID_NUMBERS", ERR_LIB_X509V3, X509V3_R_INVALID_NUMBERS}, - #else - {"INVALID_NUMBERS", 34, 141}, - #endif - #ifdef X509V3_R_INVALID_OBJECT_IDENTIFIER - {"INVALID_OBJECT_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_INVALID_OBJECT_IDENTIFIER}, - #else - {"INVALID_OBJECT_IDENTIFIER", 34, 110}, - #endif - #ifdef X509V3_R_INVALID_OPTION - {"INVALID_OPTION", ERR_LIB_X509V3, X509V3_R_INVALID_OPTION}, - #else - {"INVALID_OPTION", 34, 138}, - #endif - #ifdef X509V3_R_INVALID_POLICY_IDENTIFIER - {"INVALID_POLICY_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_INVALID_POLICY_IDENTIFIER}, - #else - {"INVALID_POLICY_IDENTIFIER", 34, 134}, - #endif - #ifdef X509V3_R_INVALID_PROXY_POLICY_SETTING - {"INVALID_PROXY_POLICY_SETTING", ERR_LIB_X509V3, X509V3_R_INVALID_PROXY_POLICY_SETTING}, - #else - {"INVALID_PROXY_POLICY_SETTING", 34, 153}, - #endif - #ifdef X509V3_R_INVALID_PURPOSE - {"INVALID_PURPOSE", ERR_LIB_X509V3, X509V3_R_INVALID_PURPOSE}, - #else - {"INVALID_PURPOSE", 34, 146}, - #endif - #ifdef X509V3_R_INVALID_SAFI - {"INVALID_SAFI", ERR_LIB_X509V3, X509V3_R_INVALID_SAFI}, - #else - {"INVALID_SAFI", 34, 164}, - #endif - #ifdef X509V3_R_INVALID_SECTION - {"INVALID_SECTION", ERR_LIB_X509V3, X509V3_R_INVALID_SECTION}, - #else - {"INVALID_SECTION", 34, 135}, - #endif - #ifdef X509V3_R_INVALID_SYNTAX - {"INVALID_SYNTAX", ERR_LIB_X509V3, X509V3_R_INVALID_SYNTAX}, - #else - {"INVALID_SYNTAX", 34, 143}, - #endif - #ifdef X509V3_R_ISSUER_DECODE_ERROR - {"ISSUER_DECODE_ERROR", ERR_LIB_X509V3, X509V3_R_ISSUER_DECODE_ERROR}, - #else - {"ISSUER_DECODE_ERROR", 34, 126}, - #endif - #ifdef X509V3_R_MISSING_VALUE - {"MISSING_VALUE", ERR_LIB_X509V3, X509V3_R_MISSING_VALUE}, - #else - {"MISSING_VALUE", 34, 124}, - #endif - #ifdef X509V3_R_NEED_ORGANIZATION_AND_NUMBERS - {"NEED_ORGANIZATION_AND_NUMBERS", ERR_LIB_X509V3, X509V3_R_NEED_ORGANIZATION_AND_NUMBERS}, - #else - {"NEED_ORGANIZATION_AND_NUMBERS", 34, 142}, - #endif - #ifdef X509V3_R_NO_CONFIG_DATABASE - {"NO_CONFIG_DATABASE", ERR_LIB_X509V3, X509V3_R_NO_CONFIG_DATABASE}, - #else - {"NO_CONFIG_DATABASE", 34, 136}, - #endif - #ifdef X509V3_R_NO_ISSUER_CERTIFICATE - {"NO_ISSUER_CERTIFICATE", ERR_LIB_X509V3, X509V3_R_NO_ISSUER_CERTIFICATE}, - #else - {"NO_ISSUER_CERTIFICATE", 34, 121}, - #endif - #ifdef X509V3_R_NO_ISSUER_DETAILS - {"NO_ISSUER_DETAILS", ERR_LIB_X509V3, X509V3_R_NO_ISSUER_DETAILS}, - #else - {"NO_ISSUER_DETAILS", 34, 127}, - #endif - #ifdef X509V3_R_NO_POLICY_IDENTIFIER - {"NO_POLICY_IDENTIFIER", ERR_LIB_X509V3, X509V3_R_NO_POLICY_IDENTIFIER}, - #else - {"NO_POLICY_IDENTIFIER", 34, 139}, - #endif - #ifdef X509V3_R_NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED - {"NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", ERR_LIB_X509V3, X509V3_R_NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED}, - #else - {"NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", 34, 154}, - #endif - #ifdef X509V3_R_NO_PUBLIC_KEY - {"NO_PUBLIC_KEY", ERR_LIB_X509V3, X509V3_R_NO_PUBLIC_KEY}, - #else - {"NO_PUBLIC_KEY", 34, 114}, - #endif - #ifdef X509V3_R_NO_SUBJECT_DETAILS - {"NO_SUBJECT_DETAILS", ERR_LIB_X509V3, X509V3_R_NO_SUBJECT_DETAILS}, - #else - {"NO_SUBJECT_DETAILS", 34, 125}, - #endif - #ifdef X509V3_R_OPERATION_NOT_DEFINED - {"OPERATION_NOT_DEFINED", ERR_LIB_X509V3, X509V3_R_OPERATION_NOT_DEFINED}, - #else - {"OPERATION_NOT_DEFINED", 34, 148}, - #endif - #ifdef X509V3_R_OTHERNAME_ERROR - {"OTHERNAME_ERROR", ERR_LIB_X509V3, X509V3_R_OTHERNAME_ERROR}, - #else - {"OTHERNAME_ERROR", 34, 147}, - #endif - #ifdef X509V3_R_POLICY_LANGUAGE_ALREADY_DEFINED - {"POLICY_LANGUAGE_ALREADY_DEFINED", ERR_LIB_X509V3, X509V3_R_POLICY_LANGUAGE_ALREADY_DEFINED}, - #else - {"POLICY_LANGUAGE_ALREADY_DEFINED", 34, 155}, - #endif - #ifdef X509V3_R_POLICY_PATH_LENGTH - {"POLICY_PATH_LENGTH", ERR_LIB_X509V3, X509V3_R_POLICY_PATH_LENGTH}, - #else - {"POLICY_PATH_LENGTH", 34, 156}, - #endif - #ifdef X509V3_R_POLICY_PATH_LENGTH_ALREADY_DEFINED - {"POLICY_PATH_LENGTH_ALREADY_DEFINED", ERR_LIB_X509V3, X509V3_R_POLICY_PATH_LENGTH_ALREADY_DEFINED}, - #else - {"POLICY_PATH_LENGTH_ALREADY_DEFINED", 34, 157}, - #endif - #ifdef X509V3_R_POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY - {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", ERR_LIB_X509V3, X509V3_R_POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY}, - #else - {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", 34, 159}, - #endif - #ifdef X509V3_R_SECTION_NOT_FOUND - {"SECTION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_SECTION_NOT_FOUND}, - #else - {"SECTION_NOT_FOUND", 34, 150}, - #endif - #ifdef X509V3_R_UNABLE_TO_GET_ISSUER_DETAILS - {"UNABLE_TO_GET_ISSUER_DETAILS", ERR_LIB_X509V3, X509V3_R_UNABLE_TO_GET_ISSUER_DETAILS}, - #else - {"UNABLE_TO_GET_ISSUER_DETAILS", 34, 122}, - #endif - #ifdef X509V3_R_UNABLE_TO_GET_ISSUER_KEYID - {"UNABLE_TO_GET_ISSUER_KEYID", ERR_LIB_X509V3, X509V3_R_UNABLE_TO_GET_ISSUER_KEYID}, - #else - {"UNABLE_TO_GET_ISSUER_KEYID", 34, 123}, - #endif - #ifdef X509V3_R_UNKNOWN_BIT_STRING_ARGUMENT - {"UNKNOWN_BIT_STRING_ARGUMENT", ERR_LIB_X509V3, X509V3_R_UNKNOWN_BIT_STRING_ARGUMENT}, - #else - {"UNKNOWN_BIT_STRING_ARGUMENT", 34, 111}, - #endif - #ifdef X509V3_R_UNKNOWN_EXTENSION - {"UNKNOWN_EXTENSION", ERR_LIB_X509V3, X509V3_R_UNKNOWN_EXTENSION}, - #else - {"UNKNOWN_EXTENSION", 34, 129}, - #endif - #ifdef X509V3_R_UNKNOWN_EXTENSION_NAME - {"UNKNOWN_EXTENSION_NAME", ERR_LIB_X509V3, X509V3_R_UNKNOWN_EXTENSION_NAME}, - #else - {"UNKNOWN_EXTENSION_NAME", 34, 130}, - #endif - #ifdef X509V3_R_UNKNOWN_OPTION - {"UNKNOWN_OPTION", ERR_LIB_X509V3, X509V3_R_UNKNOWN_OPTION}, - #else - {"UNKNOWN_OPTION", 34, 120}, - #endif - #ifdef X509V3_R_UNSUPPORTED_OPTION - {"UNSUPPORTED_OPTION", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_OPTION}, - #else - {"UNSUPPORTED_OPTION", 34, 117}, - #endif - #ifdef X509V3_R_UNSUPPORTED_TYPE - {"UNSUPPORTED_TYPE", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_TYPE}, - #else - {"UNSUPPORTED_TYPE", 34, 167}, - #endif - #ifdef X509V3_R_USER_TOO_LONG - {"USER_TOO_LONG", ERR_LIB_X509V3, X509V3_R_USER_TOO_LONG}, - #else - {"USER_TOO_LONG", 34, 132}, - #endif - #ifdef X509_R_AKID_MISMATCH - {"AKID_MISMATCH", ERR_LIB_X509, X509_R_AKID_MISMATCH}, - #else - {"AKID_MISMATCH", 11, 110}, - #endif - #ifdef X509_R_BAD_SELECTOR - {"BAD_SELECTOR", ERR_LIB_X509, X509_R_BAD_SELECTOR}, - #else - {"BAD_SELECTOR", 11, 133}, - #endif - #ifdef X509_R_BAD_X509_FILETYPE - {"BAD_X509_FILETYPE", ERR_LIB_X509, X509_R_BAD_X509_FILETYPE}, - #else - {"BAD_X509_FILETYPE", 11, 100}, - #endif - #ifdef X509_R_BASE64_DECODE_ERROR - {"BASE64_DECODE_ERROR", ERR_LIB_X509, X509_R_BASE64_DECODE_ERROR}, - #else - {"BASE64_DECODE_ERROR", 11, 118}, - #endif - #ifdef X509_R_CANT_CHECK_DH_KEY - {"CANT_CHECK_DH_KEY", ERR_LIB_X509, X509_R_CANT_CHECK_DH_KEY}, - #else - {"CANT_CHECK_DH_KEY", 11, 114}, - #endif - #ifdef X509_R_CERT_ALREADY_IN_HASH_TABLE - {"CERT_ALREADY_IN_HASH_TABLE", ERR_LIB_X509, X509_R_CERT_ALREADY_IN_HASH_TABLE}, - #else - {"CERT_ALREADY_IN_HASH_TABLE", 11, 101}, - #endif - #ifdef X509_R_CRL_ALREADY_DELTA - {"CRL_ALREADY_DELTA", ERR_LIB_X509, X509_R_CRL_ALREADY_DELTA}, - #else - {"CRL_ALREADY_DELTA", 11, 127}, - #endif - #ifdef X509_R_CRL_VERIFY_FAILURE - {"CRL_VERIFY_FAILURE", ERR_LIB_X509, X509_R_CRL_VERIFY_FAILURE}, - #else - {"CRL_VERIFY_FAILURE", 11, 131}, - #endif - #ifdef X509_R_ERR_ASN1_LIB - {"ERR_ASN1_LIB", ERR_LIB_X509, X509_R_ERR_ASN1_LIB}, - #else - {"ERR_ASN1_LIB", 11, 102}, - #endif - #ifdef X509_R_IDP_MISMATCH - {"IDP_MISMATCH", ERR_LIB_X509, X509_R_IDP_MISMATCH}, - #else - {"IDP_MISMATCH", 11, 128}, - #endif - #ifdef X509_R_INVALID_ATTRIBUTES - {"INVALID_ATTRIBUTES", ERR_LIB_X509, X509_R_INVALID_ATTRIBUTES}, - #else - {"INVALID_ATTRIBUTES", 11, 138}, - #endif - #ifdef X509_R_INVALID_DIRECTORY - {"INVALID_DIRECTORY", ERR_LIB_X509, X509_R_INVALID_DIRECTORY}, - #else - {"INVALID_DIRECTORY", 11, 113}, - #endif - #ifdef X509_R_INVALID_FIELD_NAME - {"INVALID_FIELD_NAME", ERR_LIB_X509, X509_R_INVALID_FIELD_NAME}, - #else - {"INVALID_FIELD_NAME", 11, 119}, - #endif - #ifdef X509_R_INVALID_TRUST - {"INVALID_TRUST", ERR_LIB_X509, X509_R_INVALID_TRUST}, - #else - {"INVALID_TRUST", 11, 123}, - #endif - #ifdef X509_R_ISSUER_MISMATCH - {"ISSUER_MISMATCH", ERR_LIB_X509, X509_R_ISSUER_MISMATCH}, - #else - {"ISSUER_MISMATCH", 11, 129}, - #endif - #ifdef X509_R_KEY_TYPE_MISMATCH - {"KEY_TYPE_MISMATCH", ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH}, - #else - {"KEY_TYPE_MISMATCH", 11, 115}, - #endif - #ifdef X509_R_KEY_VALUES_MISMATCH - {"KEY_VALUES_MISMATCH", ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH}, - #else - {"KEY_VALUES_MISMATCH", 11, 116}, - #endif - #ifdef X509_R_LOADING_CERT_DIR - {"LOADING_CERT_DIR", ERR_LIB_X509, X509_R_LOADING_CERT_DIR}, - #else - {"LOADING_CERT_DIR", 11, 103}, - #endif - #ifdef X509_R_LOADING_DEFAULTS - {"LOADING_DEFAULTS", ERR_LIB_X509, X509_R_LOADING_DEFAULTS}, - #else - {"LOADING_DEFAULTS", 11, 104}, - #endif - #ifdef X509_R_METHOD_NOT_SUPPORTED - {"METHOD_NOT_SUPPORTED", ERR_LIB_X509, X509_R_METHOD_NOT_SUPPORTED}, - #else - {"METHOD_NOT_SUPPORTED", 11, 124}, - #endif - #ifdef X509_R_NAME_TOO_LONG - {"NAME_TOO_LONG", ERR_LIB_X509, X509_R_NAME_TOO_LONG}, - #else - {"NAME_TOO_LONG", 11, 134}, - #endif - #ifdef X509_R_NEWER_CRL_NOT_NEWER - {"NEWER_CRL_NOT_NEWER", ERR_LIB_X509, X509_R_NEWER_CRL_NOT_NEWER}, - #else - {"NEWER_CRL_NOT_NEWER", 11, 132}, - #endif - #ifdef X509_R_NO_CERTIFICATE_FOUND - {"NO_CERTIFICATE_FOUND", ERR_LIB_X509, X509_R_NO_CERTIFICATE_FOUND}, - #else - {"NO_CERTIFICATE_FOUND", 11, 135}, - #endif - #ifdef X509_R_NO_CERTIFICATE_OR_CRL_FOUND - {"NO_CERTIFICATE_OR_CRL_FOUND", ERR_LIB_X509, X509_R_NO_CERTIFICATE_OR_CRL_FOUND}, - #else - {"NO_CERTIFICATE_OR_CRL_FOUND", 11, 136}, - #endif - #ifdef X509_R_NO_CERT_SET_FOR_US_TO_VERIFY - {"NO_CERT_SET_FOR_US_TO_VERIFY", ERR_LIB_X509, X509_R_NO_CERT_SET_FOR_US_TO_VERIFY}, - #else - {"NO_CERT_SET_FOR_US_TO_VERIFY", 11, 105}, - #endif - #ifdef X509_R_NO_CRL_FOUND - {"NO_CRL_FOUND", ERR_LIB_X509, X509_R_NO_CRL_FOUND}, - #else - {"NO_CRL_FOUND", 11, 137}, - #endif - #ifdef X509_R_NO_CRL_NUMBER - {"NO_CRL_NUMBER", ERR_LIB_X509, X509_R_NO_CRL_NUMBER}, - #else - {"NO_CRL_NUMBER", 11, 130}, - #endif - #ifdef X509_R_PUBLIC_KEY_DECODE_ERROR - {"PUBLIC_KEY_DECODE_ERROR", ERR_LIB_X509, X509_R_PUBLIC_KEY_DECODE_ERROR}, - #else - {"PUBLIC_KEY_DECODE_ERROR", 11, 125}, - #endif - #ifdef X509_R_PUBLIC_KEY_ENCODE_ERROR - {"PUBLIC_KEY_ENCODE_ERROR", ERR_LIB_X509, X509_R_PUBLIC_KEY_ENCODE_ERROR}, - #else - {"PUBLIC_KEY_ENCODE_ERROR", 11, 126}, - #endif - #ifdef X509_R_SHOULD_RETRY - {"SHOULD_RETRY", ERR_LIB_X509, X509_R_SHOULD_RETRY}, - #else - {"SHOULD_RETRY", 11, 106}, - #endif - #ifdef X509_R_UNABLE_TO_FIND_PARAMETERS_IN_CHAIN - {"UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", ERR_LIB_X509, X509_R_UNABLE_TO_FIND_PARAMETERS_IN_CHAIN}, - #else - {"UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", 11, 107}, - #endif - #ifdef X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY - {"UNABLE_TO_GET_CERTS_PUBLIC_KEY", ERR_LIB_X509, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY}, - #else - {"UNABLE_TO_GET_CERTS_PUBLIC_KEY", 11, 108}, - #endif - #ifdef X509_R_UNKNOWN_KEY_TYPE - {"UNKNOWN_KEY_TYPE", ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE}, - #else - {"UNKNOWN_KEY_TYPE", 11, 117}, - #endif - #ifdef X509_R_UNKNOWN_NID - {"UNKNOWN_NID", ERR_LIB_X509, X509_R_UNKNOWN_NID}, - #else - {"UNKNOWN_NID", 11, 109}, - #endif - #ifdef X509_R_UNKNOWN_PURPOSE_ID - {"UNKNOWN_PURPOSE_ID", ERR_LIB_X509, X509_R_UNKNOWN_PURPOSE_ID}, - #else - {"UNKNOWN_PURPOSE_ID", 11, 121}, - #endif - #ifdef X509_R_UNKNOWN_TRUST_ID - {"UNKNOWN_TRUST_ID", ERR_LIB_X509, X509_R_UNKNOWN_TRUST_ID}, - #else - {"UNKNOWN_TRUST_ID", 11, 120}, - #endif - #ifdef X509_R_UNSUPPORTED_ALGORITHM - {"UNSUPPORTED_ALGORITHM", ERR_LIB_X509, X509_R_UNSUPPORTED_ALGORITHM}, - #else - {"UNSUPPORTED_ALGORITHM", 11, 111}, - #endif - #ifdef X509_R_WRONG_LOOKUP_TYPE - {"WRONG_LOOKUP_TYPE", ERR_LIB_X509, X509_R_WRONG_LOOKUP_TYPE}, - #else - {"WRONG_LOOKUP_TYPE", 11, 112}, - #endif - #ifdef X509_R_WRONG_TYPE - {"WRONG_TYPE", ERR_LIB_X509, X509_R_WRONG_TYPE}, - #else - {"WRONG_TYPE", 11, 122}, - #endif - { NULL } -}; diff --git a/Modules/_stat.c b/Modules/_stat.c index 402fbbaecf8dd8..a4f15e8e65e894 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -12,7 +12,10 @@ */ // Need limited C API version 3.13 for PyModule_Add() on Windows -#define Py_LIMITED_API 0x030d0000 +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif #include "Python.h" @@ -198,6 +201,10 @@ typedef unsigned short mode_t; /* Names for file flags */ +#ifndef UF_SETTABLE +# define UF_SETTABLE 0x0000ffff +#endif + #ifndef UF_NODUMP # define UF_NODUMP 0x00000001 #endif @@ -222,10 +229,22 @@ typedef unsigned short mode_t; # define UF_COMPRESSED 0x00000020 #endif +#ifndef UF_TRACKED +# define UF_TRACKED 0x00000040 +#endif + +#ifndef UF_DATAVAULT +# define UF_DATAVAULT 0x00000080 +#endif + #ifndef UF_HIDDEN # define UF_HIDDEN 0x00008000 #endif +#ifndef SF_SETTABLE +# define SF_SETTABLE 0xffff0000 +#endif + #ifndef SF_ARCHIVED # define SF_ARCHIVED 0x00010000 #endif @@ -246,6 +265,30 @@ typedef unsigned short mode_t; # define SF_SNAPSHOT 0x00200000 #endif +#ifndef SF_FIRMLINK +# define SF_FIRMLINK 0x00800000 +#endif + +#ifndef SF_DATALESS +# define SF_DATALESS 0x40000000 +#endif + +#if defined(__APPLE__) && !defined(SF_SUPPORTED) + /* On older macOS versions the definition of SF_SUPPORTED is different + * from that on newer versions. + * + * Provide a consistent experience by redefining. + * + * None of bit bits set in the actual SF_SUPPORTED but not in this + * definition are defined on these versions of macOS. + */ +# undef SF_SETTABLE +# define SF_SUPPORTED 0x009f0000 +# define SF_SETTABLE 0x3fff0000 +# define SF_SYNTHETIC 0xc0000000 +#endif + + static mode_t _PyLong_AsMode_t(PyObject *op) { @@ -463,18 +506,29 @@ S_IWOTH: write by others\n\ S_IXOTH: execute by others\n\ \n" -"UF_NODUMP: do not dump file\n\ +"UF_SETTABLE: mask of owner changable flags\n\ +UF_NODUMP: do not dump file\n\ UF_IMMUTABLE: file may not be changed\n\ UF_APPEND: file may only be appended to\n\ UF_OPAQUE: directory is opaque when viewed through a union stack\n\ UF_NOUNLINK: file may not be renamed or deleted\n\ -UF_COMPRESSED: OS X: file is hfs-compressed\n\ -UF_HIDDEN: OS X: file should not be displayed\n\ +UF_COMPRESSED: macOS: file is hfs-compressed\n\ +UF_TRACKED: used for dealing with document IDs\n\ +UF_DATAVAULT: entitlement required for reading and writing\n\ +UF_HIDDEN: macOS: file should not be displayed\n\ +SF_SETTABLE: mask of super user changeable flags\n\ SF_ARCHIVED: file may be archived\n\ SF_IMMUTABLE: file may not be changed\n\ SF_APPEND: file may only be appended to\n\ +SF_RESTRICTED: entitlement required for writing\n\ SF_NOUNLINK: file may not be renamed or deleted\n\ SF_SNAPSHOT: file is a snapshot file\n\ +SF_FIRMLINK: file is a firmlink\n\ +SF_DATALESS: file is a dataless object\n\ +\n\ +On macOS:\n\ +SF_SUPPORTED: mask of super user supported flags\n\ +SF_SYNTHETIC: mask of read-only synthetic flags\n\ \n" "ST_MODE\n\ @@ -539,18 +593,32 @@ stat_exec(PyObject *module) ADD_INT_MACRO(module, S_IWOTH); ADD_INT_MACRO(module, S_IXOTH); + ADD_INT_MACRO(module, UF_SETTABLE); ADD_INT_MACRO(module, UF_NODUMP); ADD_INT_MACRO(module, UF_IMMUTABLE); ADD_INT_MACRO(module, UF_APPEND); ADD_INT_MACRO(module, UF_OPAQUE); ADD_INT_MACRO(module, UF_NOUNLINK); ADD_INT_MACRO(module, UF_COMPRESSED); + ADD_INT_MACRO(module, UF_TRACKED); + ADD_INT_MACRO(module, UF_DATAVAULT); ADD_INT_MACRO(module, UF_HIDDEN); + ADD_INT_MACRO(module, SF_SETTABLE); ADD_INT_MACRO(module, SF_ARCHIVED); ADD_INT_MACRO(module, SF_IMMUTABLE); ADD_INT_MACRO(module, SF_APPEND); ADD_INT_MACRO(module, SF_NOUNLINK); ADD_INT_MACRO(module, SF_SNAPSHOT); + ADD_INT_MACRO(module, SF_FIRMLINK); + ADD_INT_MACRO(module, SF_DATALESS); + +#ifdef SF_SUPPORTED + ADD_INT_MACRO(module, SF_SUPPORTED); +#endif +#ifdef SF_SYNTHETIC + ADD_INT_MACRO(module, SF_SYNTHETIC); +#endif + const char* st_constants[] = { "ST_MODE", @@ -611,6 +679,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index 1d5465fbe6d04e..b84f731ad6a1da 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,5 +1,11 @@ /* statistics accelerator C extension: _statistics module. */ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "clinic/_statisticsmodule.c.h" @@ -130,6 +136,7 @@ PyDoc_STRVAR(statistics_doc, static struct PyModuleDef_Slot _statisticsmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index ff1bf4e96c5f21..6a68478dd45d36 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1000,9 +1000,10 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 0, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1024,9 +1025,10 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 0, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, @@ -1260,9 +1262,10 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) (unsigned char *)p, 8, 1, /* little_endian */ - 1 /* signed */); + 1, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires %lld <= number <= %lld", f->format, @@ -1284,9 +1287,10 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f (unsigned char *)p, 8, 1, /* little_endian */ - 0 /* signed */); + 0, /* signed */ + 0 /* !with_exceptions */); Py_DECREF(v); - if (res == -1 && PyErr_Occurred()) { + if (res < 0) { PyErr_Format(state->StructError, "'%c' format requires 0 <= number <= %llu", f->format, @@ -1369,7 +1373,7 @@ whichtable(const char **pfmt) } default: --*pfmt; /* Back out of pointer increment */ - /* Fall through */ + _Py_FALLTHROUGH; case '@': return native_table; } @@ -1471,7 +1475,7 @@ prepare_s(PyStructObject *self) return -1; switch (c) { - case 's': /* fall through */ + case 's': _Py_FALLTHROUGH; case 'p': len++; ncodes++; break; case 'x': break; default: len += num; if (num) ncodes++; break; @@ -1553,9 +1557,28 @@ prepare_s(PyStructObject *self) return -1; } +static PyObject * +s_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *self; + + assert(type != NULL); + allocfunc alloc_func = PyType_GetSlot(type, Py_tp_alloc); + assert(alloc_func != NULL); + + self = alloc_func(type, 0); + if (self != NULL) { + PyStructObject *s = (PyStructObject*)self; + s->s_format = Py_NewRef(Py_None); + s->s_codes = NULL; + s->s_size = -1; + s->s_len = -1; + } + return self; +} + /*[clinic input] -@classmethod -Struct.__new__ +Struct.__init__ format: object @@ -1567,24 +1590,16 @@ the format string. See help(struct) for more on format strings. [clinic start generated code]*/ -static PyObject * -Struct_impl(PyTypeObject *type, PyObject *format) -/*[clinic end generated code: output=49468b044e334308 input=8b91868eb1df0e28]*/ +static int +Struct___init___impl(PyStructObject *self, PyObject *format) +/*[clinic end generated code: output=b8e80862444e92d0 input=192a4575a3dde802]*/ { - allocfunc alloc = PyType_GetSlot(type, Py_tp_alloc); - assert(alloc != NULL); - PyStructObject *self = (PyStructObject *)alloc(type, 0); - - if (self == NULL) { - return NULL; - } + int ret = 0; if (PyUnicode_Check(format)) { format = PyUnicode_AsASCIIString(format); - if (format == NULL) { - Py_DECREF(self); - return NULL; - } + if (format == NULL) + return -1; } else { Py_INCREF(format); @@ -1592,24 +1607,19 @@ Struct_impl(PyTypeObject *type, PyObject *format) if (!PyBytes_Check(format)) { Py_DECREF(format); - Py_DECREF(self); PyErr_Format(PyExc_TypeError, "Struct() argument 1 must be a str or bytes object, " "not %.200s", _PyType_Name(Py_TYPE(format))); - return NULL; + return -1; } - self->s_format = format; + Py_SETREF(self->s_format, format); - if (prepare_s(self) < 0) { - Py_DECREF(self); - return NULL; - } - return (PyObject *)self; + ret = prepare_s(self); + return ret; } - static int s_clear(PyStructObject *s) { @@ -2219,8 +2229,9 @@ static PyType_Slot PyStructType_slots[] = { {Py_tp_methods, s_methods}, {Py_tp_members, s_members}, {Py_tp_getset, s_getsetlist}, - {Py_tp_new, Struct}, + {Py_tp_init, Struct___init__}, {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, s_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0}, }; @@ -2250,20 +2261,13 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - if (state->cache == NULL) { - state->cache = PyDict_New(); - if (state->cache == NULL) - return 0; + if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + return 0; } - - s_object = PyDict_GetItemWithError(state->cache, fmt); if (s_object != NULL) { - *ptr = (PyStructObject *)Py_NewRef(s_object); + *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } - else if (PyErr_Occurred()) { - return 0; - } s_object = PyObject_CallOneArg(state->PyStructType, fmt); if (s_object != NULL) { @@ -2288,7 +2292,7 @@ static PyObject * _clearcache_impl(PyObject *module) /*[clinic end generated code: output=ce4fb8a7bf7cb523 input=463eaae04bab3211]*/ { - Py_CLEAR(get_struct_state(module)->cache); + PyDict_Clear(get_struct_state(module)->cache); Py_RETURN_NONE; } @@ -2512,6 +2516,11 @@ _structmodule_exec(PyObject *m) { _structmodulestate *state = get_struct_state(m); + state->cache = PyDict_New(); + if (state->cache == NULL) { + return -1; + } + state->PyStructType = PyType_FromModuleAndSpec( m, &PyStructType_spec, NULL); if (state->PyStructType == NULL) { @@ -2584,6 +2593,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c new file mode 100644 index 00000000000000..80c7179c4c251c --- /dev/null +++ b/Modules/_suggestions.c @@ -0,0 +1,69 @@ +#include "Python.h" +#include "pycore_pyerrors.h" +#include "clinic/_suggestions.c.h" + +/*[clinic input] +module _suggestions +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e58d81fafad5637b]*/ + +/*[clinic input] +_suggestions._generate_suggestions + candidates: object + item: unicode + / +Returns the candidate in candidates that's closest to item +[clinic start generated code]*/ + +static PyObject * +_suggestions__generate_suggestions_impl(PyObject *module, + PyObject *candidates, PyObject *item) +/*[clinic end generated code: output=79be7b653ae5e7ca input=ba2a8dddc654e33a]*/ +{ + // Check if dir is a list + if (!PyList_Check(candidates)) { + PyErr_SetString(PyExc_TypeError, "candidates must be a list"); + return NULL; + } + + // Check if all elements in the list are Unicode + Py_ssize_t size = PyList_Size(candidates); + for (Py_ssize_t i = 0; i < size; ++i) { + PyObject *elem = PyList_GetItem(candidates, i); + if (!PyUnicode_Check(elem)) { + PyErr_SetString(PyExc_TypeError, "all elements in 'candidates' must be strings"); + return NULL; + } + } + + PyObject* result = _Py_CalculateSuggestions(candidates, item); + if (!result && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return result; +} + + +static PyMethodDef module_methods[] = { + _SUGGESTIONS__GENERATE_SUGGESTIONS_METHODDEF + {NULL, NULL, 0, NULL} // Sentinel +}; + +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef suggestions_module = { + PyModuleDef_HEAD_INIT, + "_suggestions", + NULL, + 0, + module_methods, + module_slots, +}; + +PyMODINIT_FUNC PyInit__suggestions(void) { + return PyModuleDef_Init(&suggestions_module); +} diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index 6f1cc16b58467d..c50c5cfabc2f1f 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -57,12 +57,12 @@ _sysconfig_config_vars_impl(PyObject *module) } #endif -#ifdef Py_NOGIL - PyObject *py_nogil = _PyLong_GetOne(); +#ifdef Py_GIL_DISABLED + PyObject *py_gil_disabled = _PyLong_GetOne(); #else - PyObject *py_nogil = _PyLong_GetZero(); + PyObject *py_gil_disabled = _PyLong_GetZero(); #endif - if (PyDict_SetItemString(config, "Py_NOGIL", py_nogil) < 0) { + if (PyDict_SetItemString(config, "Py_GIL_DISABLED", py_gil_disabled) < 0) { Py_DECREF(config); return NULL; } @@ -80,6 +80,7 @@ static struct PyMethodDef sysconfig_methods[] = { static PyModuleDef_Slot sysconfig_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5101834cfe1387..b47bc8a830cb5e 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -24,11 +24,13 @@ static PyTypeObject NDArray_Type; #define NDArray_Check(v) Py_IS_TYPE(v, &NDArray_Type) #define CHECK_LIST_OR_TUPLE(v) \ - if (!PyList_Check(v) && !PyTuple_Check(v)) { \ - PyErr_SetString(PyExc_TypeError, \ - #v " must be a list or a tuple"); \ - return NULL; \ - } \ + do { \ + if (!PyList_Check(v) && !PyTuple_Check(v)) { \ + PyErr_SetString(PyExc_TypeError, \ + #v " must be a list or a tuple"); \ + return NULL; \ + } \ + } while (0) #define PyMem_XFree(v) \ do { if (v) PyMem_Free(v); } while (0) @@ -1180,7 +1182,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, Py_ssize_t itemsize; /* ndim = len(shape) */ - CHECK_LIST_OR_TUPLE(shape) + CHECK_LIST_OR_TUPLE(shape); ndim = PySequence_Fast_GET_SIZE(shape); if (ndim > ND_MAX_NDIM) { PyErr_Format(PyExc_ValueError, @@ -1190,7 +1192,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* len(strides) = len(shape) */ if (strides) { - CHECK_LIST_OR_TUPLE(strides) + CHECK_LIST_OR_TUPLE(strides); if (PySequence_Fast_GET_SIZE(strides) == 0) strides = NULL; else if (flags & ND_FORTRAN) { @@ -1217,12 +1219,12 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* convert scalar to list */ if (ndim == 0) { - items = Py_BuildValue("(O)", items); + items = PyTuple_Pack(1, items); if (items == NULL) return NULL; } else { - CHECK_LIST_OR_TUPLE(items) + CHECK_LIST_OR_TUPLE(items); Py_INCREF(items); } @@ -2816,70 +2818,97 @@ static struct PyModuleDef _testbuffermodule = { NULL }; - -PyMODINIT_FUNC -PyInit__testbuffer(void) +static int +_testbuffer_exec(PyObject *mod) { - PyObject *m; - - m = PyModule_Create(&_testbuffermodule); - if (m == NULL) - return NULL; - Py_SET_TYPE(&NDArray_Type, &PyType_Type); - Py_INCREF(&NDArray_Type); - PyModule_AddObject(m, "ndarray", (PyObject *)&NDArray_Type); + if (PyType_Ready(&NDArray_Type)) { + return -1; + } + if (PyModule_AddType(mod, &NDArray_Type) < 0) { + return -1; + } Py_SET_TYPE(&StaticArray_Type, &PyType_Type); - Py_INCREF(&StaticArray_Type); - PyModule_AddObject(m, "staticarray", (PyObject *)&StaticArray_Type); + if (PyModule_AddType(mod, &StaticArray_Type) < 0) { + return -1; + } structmodule = PyImport_ImportModule("struct"); - if (structmodule == NULL) - return NULL; + if (structmodule == NULL) { + return -1; + } Struct = PyObject_GetAttrString(structmodule, "Struct"); + if (Struct == NULL) { + return -1; + } calcsize = PyObject_GetAttrString(structmodule, "calcsize"); - if (Struct == NULL || calcsize == NULL) - return NULL; + if (calcsize == NULL) { + return -1; + } simple_format = PyUnicode_FromString(simple_fmt); - if (simple_format == NULL) - return NULL; - - PyModule_AddIntMacro(m, ND_MAX_NDIM); - PyModule_AddIntMacro(m, ND_VAREXPORT); - PyModule_AddIntMacro(m, ND_WRITABLE); - PyModule_AddIntMacro(m, ND_FORTRAN); - PyModule_AddIntMacro(m, ND_SCALAR); - PyModule_AddIntMacro(m, ND_PIL); - PyModule_AddIntMacro(m, ND_GETBUF_FAIL); - PyModule_AddIntMacro(m, ND_GETBUF_UNDEFINED); - PyModule_AddIntMacro(m, ND_REDIRECT); - - PyModule_AddIntMacro(m, PyBUF_SIMPLE); - PyModule_AddIntMacro(m, PyBUF_WRITABLE); - PyModule_AddIntMacro(m, PyBUF_FORMAT); - PyModule_AddIntMacro(m, PyBUF_ND); - PyModule_AddIntMacro(m, PyBUF_STRIDES); - PyModule_AddIntMacro(m, PyBUF_INDIRECT); - PyModule_AddIntMacro(m, PyBUF_C_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_F_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_ANY_CONTIGUOUS); - PyModule_AddIntMacro(m, PyBUF_FULL); - PyModule_AddIntMacro(m, PyBUF_FULL_RO); - PyModule_AddIntMacro(m, PyBUF_RECORDS); - PyModule_AddIntMacro(m, PyBUF_RECORDS_RO); - PyModule_AddIntMacro(m, PyBUF_STRIDED); - PyModule_AddIntMacro(m, PyBUF_STRIDED_RO); - PyModule_AddIntMacro(m, PyBUF_CONTIG); - PyModule_AddIntMacro(m, PyBUF_CONTIG_RO); - - PyModule_AddIntMacro(m, PyBUF_READ); - PyModule_AddIntMacro(m, PyBUF_WRITE); - - return m; -} + if (simple_format == NULL) { + return -1; + } +#define ADD_INT_MACRO(mod, macro) \ + do { \ + if (PyModule_AddIntConstant(mod, #macro, macro) < 0) { \ + return -1; \ + } \ + } while (0) + + ADD_INT_MACRO(mod, ND_MAX_NDIM); + ADD_INT_MACRO(mod, ND_VAREXPORT); + ADD_INT_MACRO(mod, ND_WRITABLE); + ADD_INT_MACRO(mod, ND_FORTRAN); + ADD_INT_MACRO(mod, ND_SCALAR); + ADD_INT_MACRO(mod, ND_PIL); + ADD_INT_MACRO(mod, ND_GETBUF_FAIL); + ADD_INT_MACRO(mod, ND_GETBUF_UNDEFINED); + ADD_INT_MACRO(mod, ND_REDIRECT); + + ADD_INT_MACRO(mod, PyBUF_SIMPLE); + ADD_INT_MACRO(mod, PyBUF_WRITABLE); + ADD_INT_MACRO(mod, PyBUF_FORMAT); + ADD_INT_MACRO(mod, PyBUF_ND); + ADD_INT_MACRO(mod, PyBUF_STRIDES); + ADD_INT_MACRO(mod, PyBUF_INDIRECT); + ADD_INT_MACRO(mod, PyBUF_C_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_F_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_ANY_CONTIGUOUS); + ADD_INT_MACRO(mod, PyBUF_FULL); + ADD_INT_MACRO(mod, PyBUF_FULL_RO); + ADD_INT_MACRO(mod, PyBUF_RECORDS); + ADD_INT_MACRO(mod, PyBUF_RECORDS_RO); + ADD_INT_MACRO(mod, PyBUF_STRIDED); + ADD_INT_MACRO(mod, PyBUF_STRIDED_RO); + ADD_INT_MACRO(mod, PyBUF_CONTIG); + ADD_INT_MACRO(mod, PyBUF_CONTIG_RO); + + ADD_INT_MACRO(mod, PyBUF_READ); + ADD_INT_MACRO(mod, PyBUF_WRITE); + +#undef ADD_INT_MACRO + return 0; +} +PyMODINIT_FUNC +PyInit__testbuffer(void) +{ + PyObject *mod = PyModule_Create(&_testbuffermodule); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + if (_testbuffer_exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index a93477a7090422..b126aee5b9777b 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,31 +2,6 @@ #include "util.h" -static PyObject * -object_getattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyObject_GetAttr(obj, attr_name); -} - -static PyObject * -object_getattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyObject_GetAttrString(obj, attr_name); -} - static PyObject * object_getoptionalattr(PyObject *self, PyObject *args) { @@ -78,31 +53,6 @@ object_getoptionalattrstring(PyObject *self, PyObject *args) } } -static PyObject * -object_hasattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); -} - -static PyObject * -object_hasattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); -} - static PyObject * object_hasattrwitherror(PyObject *self, PyObject *args) { @@ -129,115 +79,17 @@ object_hasattrstringwitherror(PyObject *self, PyObject *args) } static PyObject * -object_setattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value; - if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - NULLABLE(value); - RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); -} - -static PyObject * -object_setattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj, *value; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); -} - -static PyObject * -object_delattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; -if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - RETURN_INT(PyObject_DelAttr(obj, attr_name)); -} - -static PyObject * -object_delattrstring(PyObject *self, PyObject *args) +mapping_getoptionalitemstring(PyObject *self, PyObject *args) { - PyObject *obj; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { return NULL; } NULLABLE(obj); - RETURN_INT(PyObject_DelAttrString(obj, attr_name)); -} - - -static PyObject * -mapping_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyMapping_Check(obj)); -} - -static PyObject * -mapping_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Size(obj)); -} - -static PyObject * -mapping_length(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Length(obj)); -} - -static PyObject * -object_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyObject_GetItem(mapping, key); -} - -static PyObject * -mapping_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyMapping_GetItemString(mapping, key); -} - -static PyObject * -mapping_getoptionalitem(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -247,23 +99,22 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_getoptionalitemstring(PyObject *self, PyObject *args) +mapping_getoptionalitem(PyObject *self, PyObject *args) { - PyObject *obj, *value = UNINITIALIZED_PTR; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } NULLABLE(obj); + NULLABLE(attr_name); - switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -273,393 +124,19 @@ mapping_getoptionalitemstring(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); Py_UNREACHABLE(); } } -static PyObject * -mapping_haskey(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyLong_FromLong(PyMapping_HasKey(mapping, key)); -} - -static PyObject * -mapping_haskeystring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); -} - -static PyObject * -mapping_haskeywitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); -} - -static PyObject * -mapping_haskeystringwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); -} - -static PyObject * -object_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyObject_SetItem(mapping, key, value)); -} - -static PyObject * -mapping_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyMapping_SetItemString(mapping, key, value)); -} - -static PyObject * -object_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyObject_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_DelItemString(mapping, key)); -} - -static PyObject * -mapping_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Keys(obj); -} - -static PyObject * -mapping_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Values(obj); -} - -static PyObject * -mapping_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Items(obj); -} - - -static PyObject * -sequence_check(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PySequence_Check(obj)); -} - -static PyObject * -sequence_size(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Size(obj)); -} - -static PyObject * -sequence_length(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Length(obj)); -} - -static PyObject * -sequence_concat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_Concat(seq1, seq2); -} - -static PyObject * -sequence_repeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_Repeat(seq, count); -} - -static PyObject * -sequence_inplaceconcat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_InPlaceConcat(seq1, seq2); -} - -static PyObject * -sequence_inplacerepeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_InPlaceRepeat(seq, count); -} - -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_GetItem(seq, i); -} - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(val); - - RETURN_INT(PySequence_SetItem(seq, i, val)); -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - RETURN_INT(PySequence_DelItem(seq, i)); -} - -static PyObject * -sequence_setslice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - NULLABLE(sequence); - NULLABLE(obj); - - RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); -} - -static PyObject * -sequence_delslice(PyObject *self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - NULLABLE(sequence); - - RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); -} - -static PyObject * -sequence_count(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Count(seq, value)); -} - -static PyObject * -sequence_contains(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_INT(PySequence_Contains(seq, value)); -} - -static PyObject * -sequence_index(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Index(seq, value)); -} - -static PyObject * -sequence_list(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_List(obj); -} - -static PyObject * -sequence_tuple(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_Tuple(obj); -} - static PyMethodDef test_methods[] = { - {"object_getattr", object_getattr, METH_VARARGS}, - {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, - {"object_hasattr", object_hasattr, METH_VARARGS}, - {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, {"object_hasattrwitherror", object_hasattrwitherror, METH_VARARGS}, {"object_hasattrstringwitherror", object_hasattrstringwitherror, METH_VARARGS}, - {"object_setattr", object_setattr, METH_VARARGS}, - {"object_setattrstring", object_setattrstring, METH_VARARGS}, - {"object_delattr", object_delattr, METH_VARARGS}, - {"object_delattrstring", object_delattrstring, METH_VARARGS}, - - {"mapping_check", mapping_check, METH_O}, - {"mapping_size", mapping_size, METH_O}, - {"mapping_length", mapping_length, METH_O}, - {"object_getitem", object_getitem, METH_VARARGS}, - {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, - {"mapping_haskey", mapping_haskey, METH_VARARGS}, - {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, - {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, - {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, - {"object_setitem", object_setitem, METH_VARARGS}, - {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, - {"object_delitem", object_delitem, METH_VARARGS}, - {"mapping_delitem", mapping_delitem, METH_VARARGS}, - {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, - {"mapping_keys", mapping_keys, METH_O}, - {"mapping_values", mapping_values, METH_O}, - {"mapping_items", mapping_items, METH_O}, - - {"sequence_check", sequence_check, METH_O}, - {"sequence_size", sequence_size, METH_O}, - {"sequence_length", sequence_length, METH_O}, - {"sequence_concat", sequence_concat, METH_VARARGS}, - {"sequence_repeat", sequence_repeat, METH_VARARGS}, - {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, - {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"sequence_setslice", sequence_setslice, METH_VARARGS}, - {"sequence_delslice", sequence_delslice, METH_VARARGS}, - {"sequence_count", sequence_count, METH_VARARGS}, - {"sequence_contains", sequence_contains, METH_VARARGS}, - {"sequence_index", sequence_index, METH_VARARGS}, - {"sequence_list", sequence_list, METH_O}, - {"sequence_tuple", sequence_tuple, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 942774156c6c47..7e2f6e5e29482c 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,8 +54,10 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef(self)); - self->references++; + if (buf == 0) { + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + } return buf; } diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c new file mode 100644 index 00000000000000..02294d8887abb7 --- /dev/null +++ b/Modules/_testcapi/bytes.c @@ -0,0 +1,53 @@ +#include "parts.h" +#include "util.h" + + +/* Test _PyBytes_Resize() */ +static PyObject * +bytes_resize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t newsize; + int new; + + if (!PyArg_ParseTuple(args, "Onp", &obj, &newsize, &new)) + return NULL; + + NULLABLE(obj); + if (new) { + assert(obj != NULL); + assert(PyBytes_CheckExact(obj)); + PyObject *newobj = PyBytes_FromStringAndSize(NULL, PyBytes_Size(obj)); + if (newobj == NULL) { + return NULL; + } + memcpy(PyBytes_AsString(newobj), PyBytes_AsString(obj), PyBytes_Size(obj)); + obj = newobj; + } + else { + Py_XINCREF(obj); + } + if (_PyBytes_Resize(&obj, newsize) < 0) { + assert(obj == NULL); + } + else { + assert(obj != NULL); + } + return obj; +} + + +static PyMethodDef test_methods[] = { + {"bytes_resize", bytes_resize, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Bytes(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 39b5f8b91a00db..a797444c1a72b9 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_testcapi_err_set_raised__doc__, "err_set_raised($module, exception, /)\n" @@ -455,4 +456,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=ff19512450b3bbdb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0b11ef105030a48e input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/float.c.h b/Modules/_testcapi/clinic/float.c.h index fb0931a0703e11..d5a00c8072da1e 100644 --- a/Modules/_testcapi/clinic/float.c.h +++ b/Modules/_testcapi/clinic/float.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_testcapi_float_pack__doc__, "float_pack($module, size, d, le, /)\n" "--\n" @@ -79,4 +81,4 @@ _testcapi_float_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=50146051f1341cce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b43dfd3a77fe04ba input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index e2f7042be12c48..767c671abb8eae 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -2,137 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_testcapi_test_long_api__doc__, -"test_long_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_API_METHODDEF \ - {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, -"test_longlong_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ - {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_longlong_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, -"test_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongAndOverflow API.\n" -"\n" -"General conversion to PY_LONG is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, -"test_long_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongLongAndOverflow API.\n" -"\n" -"General conversion to long long is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, -"test_long_as_size_t($module, /)\n" -"--\n" -"\n" -"Test the PyLong_As{Size,Ssize}_t API.\n" -"\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); - -#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ - {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_size_t_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, -"test_long_as_unsigned_long_long_mask($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ - {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, -"test_long_as_double($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ - {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_double_impl(module); -} - PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, "call_long_compact_api($module, arg, /)\n" "--\n" @@ -140,12 +9,4 @@ PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, #define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, - -PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, -"PyLong_AsInt($module, arg, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_PYLONG_ASINT_METHODDEF \ - {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=de762870526e241d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ddcbc6ea06e5e21 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/vectorcall.c.h b/Modules/_testcapi/clinic/vectorcall.c.h index 48688e9ade050a..c6049a438dc8fc 100644 --- a/Modules/_testcapi/clinic/vectorcall.c.h +++ b/Modules/_testcapi/clinic/vectorcall.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_testcapi_pyobject_fastcalldict__doc__, "pyobject_fastcalldict($module, func, func_args, kwargs, /)\n" "--\n" @@ -204,4 +206,4 @@ _testcapi_has_vectorcall_flag(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=0667266b825ec9ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=210ae67caab177ba input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h deleted file mode 100644 index a233aefec79ecd..00000000000000 --- a/Modules/_testcapi/clinic/vectorcall_limited.c.h +++ /dev/null @@ -1,20 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, -"call_vectorcall($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ - {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, - -PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, -"call_vectorcall_method($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ - {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, -/*[clinic end generated code: output=e980906a39602528 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/watchers.c.h b/Modules/_testcapi/clinic/watchers.c.h index fd2ef608334049..ebd71d119fa347 100644 --- a/Modules/_testcapi/clinic/watchers.c.h +++ b/Modules/_testcapi/clinic/watchers.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_testcapi_watch_dict__doc__, "watch_dict($module, watcher_id, dict, /)\n" "--\n" @@ -189,4 +191,4 @@ _testcapi_set_func_kwdefaults_via_capi(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=5ad5771d6b29dfb9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0e07ce7f295917a5 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/codec.c b/Modules/_testcapi/codec.c new file mode 100644 index 00000000000000..d13f51e20331a1 --- /dev/null +++ b/Modules/_testcapi/codec.c @@ -0,0 +1,17 @@ +#include "parts.h" +#include "util.h" + + +static PyMethodDef test_methods[] = { + {NULL}, +}; + +int +_PyTestCapi_Init_Codec(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0){ + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c new file mode 100644 index 00000000000000..eceb1310bfe874 --- /dev/null +++ b/Modules/_testcapi/complex.c @@ -0,0 +1,106 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) +{ + Py_complex complex; + + if (!PyArg_Parse(obj, "D", &complex)) { + return NULL; + } + + return PyComplex_FromCComplex(complex); +} + +static PyObject * +complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) +{ + Py_complex complex; + + NULLABLE(obj); + complex = PyComplex_AsCComplex(obj); + + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyComplex_FromCComplex(complex); +} + +static PyObject* +_py_c_neg(PyObject *Py_UNUSED(module), PyObject *num) +{ + Py_complex complex; + + complex = PyComplex_AsCComplex(num); + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyComplex_FromCComplex(_Py_c_neg(complex)); +} + +#define _PY_C_FUNC2(suffix) \ + static PyObject * \ + _py_c_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex num, exp, res; \ + \ + if (!PyArg_ParseTuple(args, "DD", &num, &exp)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_c_##suffix(num, exp); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +_PY_C_FUNC2(sum) +_PY_C_FUNC2(diff) +_PY_C_FUNC2(prod) +_PY_C_FUNC2(quot) +_PY_C_FUNC2(pow) + +static PyObject* +_py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) +{ + Py_complex complex; + double res; + + NULLABLE(obj); + complex = PyComplex_AsCComplex(obj); + + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + errno = 0; + res = _Py_c_abs(complex); + return Py_BuildValue("di", res, errno); +} + + +static PyMethodDef test_methods[] = { + {"complex_fromccomplex", complex_fromccomplex, METH_O}, + {"complex_asccomplex", complex_asccomplex, METH_O}, + {"_py_c_sum", _py_c_sum, METH_VARARGS}, + {"_py_c_diff", _py_c_diff, METH_VARARGS}, + {"_py_c_neg", _py_c_neg, METH_O}, + {"_py_c_prod", _py_c_prod, METH_VARARGS}, + {"_py_c_quot", _py_c_quot, METH_VARARGS}, + {"_py_c_pow", _py_c_pow, METH_VARARGS}, + {"_py_c_abs", _py_c_abs, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Complex(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 88f992915fa8c1..f3d54215e04232 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -22,10 +22,17 @@ test_datetime_capi(PyObject *self, PyObject *args) test_run_counter++; PyDateTime_IMPORT; - if (PyDateTimeAPI) { - Py_RETURN_NONE; + if (PyDateTimeAPI == NULL) { + return NULL; } - return NULL; + // The following C API types need to outlive interpreters, since the + // borrowed references to them can be held by users without being updated. + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; } /* Functions exposing the C API type checking for testing */ @@ -85,17 +92,25 @@ make_timezones_capi(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, -18000, 0); PyObject *name = PyUnicode_FromString("EST"); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } PyObject *est_zone_capi = PyDateTimeAPI->TimeZone_FromTimeZone(offset, name); PyObject *est_zone_macro = PyTimeZone_FromOffsetAndName(offset, name); PyObject *est_zone_macro_noname = PyTimeZone_FromOffset(offset); - - Py_DecRef(offset); - Py_DecRef(name); - + Py_DECREF(offset); + Py_DECREF(name); + if (est_zone_capi == NULL || est_zone_macro == NULL || + est_zone_macro_noname == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); if (rv == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(rv, 0, est_zone_capi); @@ -103,6 +118,11 @@ make_timezones_capi(PyObject *self, PyObject *args) PyTuple_SET_ITEM(rv, 2, est_zone_macro_noname); return rv; +error: + Py_XDECREF(est_zone_capi); + Py_XDECREF(est_zone_macro); + Py_XDECREF(est_zone_macro_noname); + return NULL; } static PyObject * @@ -110,6 +130,11 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); PyObject *name = PyUnicode_FromString(""); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } // These two should return the UTC singleton PyObject *utc_singleton_0 = PyTimeZone_FromOffset(offset); @@ -117,16 +142,28 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) // This one will return +00:00 zone, but not the UTC singleton PyObject *non_utc_zone = PyTimeZone_FromOffsetAndName(offset, name); - - Py_DecRef(offset); - Py_DecRef(name); + Py_DECREF(offset); + Py_DECREF(name); + if (utc_singleton_0 == NULL || utc_singleton_1 == NULL || + non_utc_zone == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); + if (rv == NULL) { + goto error; + } PyTuple_SET_ITEM(rv, 0, utc_singleton_0); PyTuple_SET_ITEM(rv, 1, utc_singleton_1); PyTuple_SET_ITEM(rv, 2, non_utc_zone); return rv; +error: + Py_XDECREF(utc_singleton_0); + Py_XDECREF(utc_singleton_1); + Py_XDECREF(non_utc_zone); + return NULL; } static PyObject * @@ -449,3 +486,38 @@ _PyTestCapi_Init_DateTime(PyObject *mod) } return 0; } + + +/* --------------------------------------------------------------------------- + * Test module for subinterpreters. + */ + +static int +_testcapi_datetime_exec(PyObject *mod) +{ + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcapi_datetime_slots[] = { + {Py_mod_exec, _testcapi_datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef _testcapi_datetime_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testcapi_datetime", + .m_size = 0, + .m_methods = test_methods, + .m_slots = _testcapi_datetime_slots, +}; + +PyMODINIT_FUNC +PyInit__testcapi_datetime(void) +{ + return PyModuleDef_Init(&_testcapi_datetime_module); +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 5f6a1a037dcba2..e80d898118daa5 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,60 +1,6 @@ #include "parts.h" #include "util.h" - -static PyObject * -dict_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_Check(obj)); -} - -static PyObject * -dict_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_CheckExact(obj)); -} - -static PyObject * -dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyDict_New(); -} - -static PyObject * -dictproxy_new(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDictProxy_New(obj); -} - -static PyObject * -dict_clear(PyObject *self, PyObject *obj) -{ - PyDict_Clear(obj); - Py_RETURN_NONE; -} - -static PyObject * -dict_copy(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Copy(obj); -} - -static PyObject * -dict_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *key; - if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(key); - RETURN_INT(PyDict_Contains(obj, key)); -} - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -68,72 +14,6 @@ dict_containsstring(PyObject *self, PyObject *args) RETURN_INT(PyDict_ContainsString(obj, key)); } -static PyObject * -dict_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyDict_Size(obj)); -} - -static PyObject * -dict_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItem(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - PyObject *value = PyDict_GetItemString(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItemWithError(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - - static PyObject * dict_getitemref(PyObject *self, PyObject *args) { @@ -185,33 +65,6 @@ dict_getitemstringref(PyObject *self, PyObject *args) } } -static PyObject * -dict_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyDict_SetItem(mapping, key, value)); -} - -static PyObject * -dict_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyDict_SetItemString(mapping, key, value)); -} - static PyObject * dict_setdefault(PyObject *self, PyObject *args) { @@ -226,139 +79,132 @@ dict_setdefault(PyObject *self, PyObject *args) } static PyObject * -dict_delitem(PyObject *self, PyObject *args) +dict_setdefaultref(PyObject *self, PyObject *args) { - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + PyObject *obj, *key, *default_value, *result = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OOO", &obj, &key, &default_value)) { return NULL; } - NULLABLE(mapping); + NULLABLE(obj); NULLABLE(key); - RETURN_INT(PyDict_DelItem(mapping, key)); + NULLABLE(default_value); + switch (PyDict_SetDefaultRef(obj, key, default_value, &result)) { + case -1: + assert(result == NULL); + return NULL; + case 0: + assert(result == default_value); + return result; + case 1: + return result; + default: + Py_FatalError("PyDict_SetDefaultRef() returned invalid code"); + Py_UNREACHABLE(); + } } static PyObject * -dict_delitemstring(PyObject *self, PyObject *args) +dict_pop(PyObject *self, PyObject *args) { - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + // Test PyDict_Pop(dict, key, &value) + PyObject *dict, *key; + if (!PyArg_ParseTuple(args, "OO", &dict, &key)) { return NULL; } - NULLABLE(mapping); - RETURN_INT(PyDict_DelItemString(mapping, key)); -} - -static PyObject * -dict_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Keys(obj); -} - -static PyObject * -dict_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Values(obj); + NULLABLE(dict); + NULLABLE(key); + PyObject *result = UNINITIALIZED_PTR; + int res = PyDict_Pop(dict, key, &result); + if (res < 0) { + assert(result == NULL); + return NULL; + } + if (res == 0) { + assert(result == NULL); + result = Py_NewRef(Py_None); + } + else { + assert(result != NULL); + } + return Py_BuildValue("iN", res, result); } static PyObject * -dict_items(PyObject *self, PyObject *obj) +dict_pop_null(PyObject *self, PyObject *args) { - NULLABLE(obj); - return PyDict_Items(obj); + // Test PyDict_Pop(dict, key, NULL) + PyObject *dict, *key; + if (!PyArg_ParseTuple(args, "OO", &dict, &key)) { + return NULL; + } + NULLABLE(dict); + NULLABLE(key); + RETURN_INT(PyDict_Pop(dict, key, NULL)); } static PyObject * -dict_next(PyObject *self, PyObject *args) +dict_popstring(PyObject *self, PyObject *args) { - PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; - Py_ssize_t pos; - if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + PyObject *dict; + const char *key; + Py_ssize_t key_size; + if (!PyArg_ParseTuple(args, "Oz#", &dict, &key, &key_size)) { return NULL; } - NULLABLE(mapping); - int rc = PyDict_Next(mapping, &pos, &key, &value); - if (rc != 0) { - return Py_BuildValue("inOO", rc, pos, key, value); - } - assert(key == UNINITIALIZED_PTR); - assert(value == UNINITIALIZED_PTR); - if (PyErr_Occurred()) { + NULLABLE(dict); + PyObject *result = UNINITIALIZED_PTR; + int res = PyDict_PopString(dict, key, &result); + if (res < 0) { + assert(result == NULL); return NULL; } - Py_RETURN_NONE; -} - -static PyObject * -dict_merge(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { - return NULL; + if (res == 0) { + assert(result == NULL); + result = Py_NewRef(Py_None); } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Merge(mapping, mapping2, override)); + else { + assert(result != NULL); + } + return Py_BuildValue("iN", res, result); } static PyObject * -dict_update(PyObject *self, PyObject *args) +dict_popstring_null(PyObject *self, PyObject *args) { - PyObject *mapping, *mapping2; - if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + PyObject *dict; + const char *key; + Py_ssize_t key_size; + if (!PyArg_ParseTuple(args, "Oz#", &dict, &key, &key_size)) { return NULL; } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Update(mapping, mapping2)); + NULLABLE(dict); + RETURN_INT(PyDict_PopString(dict, key, NULL)); } static PyObject * -dict_mergefromseq2(PyObject *self, PyObject *args) +dict_version(PyObject *self, PyObject *dict) { - PyObject *mapping, *seq; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "expected dict"); return NULL; } - NULLABLE(mapping); - NULLABLE(seq); - RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag); +_Py_COMP_DIAG_POP } - static PyMethodDef test_methods[] = { - {"dict_check", dict_check, METH_O}, - {"dict_checkexact", dict_checkexact, METH_O}, - {"dict_new", dict_new, METH_NOARGS}, - {"dictproxy_new", dictproxy_new, METH_O}, - {"dict_clear", dict_clear, METH_O}, - {"dict_copy", dict_copy, METH_O}, - {"dict_size", dict_size, METH_O}, - {"dict_getitem", dict_getitem, METH_VARARGS}, - {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, - {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, - {"dict_contains", dict_contains, METH_VARARGS}, - {"dict_containsstring", dict_containsstring, METH_VARARGS}, - {"dict_setitem", dict_setitem, METH_VARARGS}, - {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, - {"dict_delitem", dict_delitem, METH_VARARGS}, - {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, - {"dict_keys", dict_keys, METH_O}, - {"dict_values", dict_values, METH_O}, - {"dict_items", dict_items, METH_O}, - {"dict_next", dict_next, METH_VARARGS}, - {"dict_merge", dict_merge, METH_VARARGS}, - {"dict_update", dict_update, METH_VARARGS}, - {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, - + {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, + {"dict_pop", dict_pop, METH_VARARGS}, + {"dict_pop_null", dict_pop_null, METH_VARARGS}, + {"dict_popstring", dict_popstring, METH_VARARGS}, + {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, + {"dict_version", dict_version, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index d99fbdd904b594..3f7acbae1b181b 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -169,6 +169,13 @@ static PyMethodDef DocStringUnrepresentableSignatureTest_methods[] = { "--\n\n" "This docstring has a signature with unrepresentable default." )}, + {"with_default", + (PyCFunction)test_with_docstring, METH_VARARGS, + PyDoc_STR( + "with_default($self, /, x=ONE)\n" + "--\n\n" + "This instance method has a default parameter value from the module scope." + )}, {NULL}, }; @@ -193,5 +200,8 @@ _PyTestCapi_Init_Docstring(PyObject *mod) if (PyModule_AddType(mod, &DocStringUnrepresentableSignatureTest) < 0) { return -1; } + if (PyModule_AddObject(mod, "ONE", PyLong_FromLong(1)) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index b54ce0cbb0dd20..316ef0e7ad7e55 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -1,3 +1,6 @@ +// clinic/exceptions.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "util.h" #include "clinic/exceptions.c.h" @@ -31,11 +34,11 @@ err_restore(PyObject *self, PyObject *args) { case 3: traceback = PyTuple_GetItem(args, 2); Py_INCREF(traceback); - /* fall through */ + _Py_FALLTHROUGH; case 2: value = PyTuple_GetItem(args, 1); Py_INCREF(value); - /* fall through */ + _Py_FALLTHROUGH; case 1: type = PyTuple_GetItem(args, 0); Py_INCREF(type); @@ -300,6 +303,46 @@ _testcapi_traceback_print_impl(PyObject *module, PyObject *traceback, Py_RETURN_NONE; } +static PyObject * +err_writeunraisable(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc, *obj; + if (!PyArg_ParseTuple(args, "OO", &exc, &obj)) { + return NULL; + } + NULLABLE(exc); + NULLABLE(obj); + if (exc) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + PyErr_WriteUnraisable(obj); + Py_RETURN_NONE; +} + +static PyObject * +err_formatunraisable(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + const char *fmt; + Py_ssize_t fmtlen; + PyObject *objs[10] = {NULL}; + + if (!PyArg_ParseTuple(args, "Oz#|OOOOOOOOOO", &exc, &fmt, &fmtlen, + &objs[0], &objs[1], &objs[2], &objs[3], &objs[4], + &objs[5], &objs[6], &objs[7], &objs[8], &objs[9])) + { + return NULL; + } + NULLABLE(exc); + if (exc) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + PyErr_FormatUnraisable(fmt, + objs[0], objs[1], objs[2], objs[3], objs[4], + objs[5], objs[6], objs[7], objs[8], objs[9]); + Py_RETURN_NONE; +} + /*[clinic input] _testcapi.unstable_exc_prep_reraise_star orig: object @@ -344,6 +387,8 @@ static PyTypeObject PyRecursingInfinitelyError_Type = { static PyMethodDef test_methods[] = { {"err_restore", err_restore, METH_VARARGS}, + {"err_writeunraisable", err_writeunraisable, METH_VARARGS}, + {"err_formatunraisable", err_formatunraisable, METH_VARARGS}, _TESTCAPI_ERR_SET_RAISED_METHODDEF _TESTCAPI_EXCEPTION_PRINT_METHODDEF _TESTCAPI_FATAL_ERROR_METHODDEF diff --git a/Modules/_testcapi/file.c b/Modules/_testcapi/file.c new file mode 100644 index 00000000000000..634563f6ea12cb --- /dev/null +++ b/Modules/_testcapi/file.c @@ -0,0 +1,17 @@ +#include "parts.h" +#include "util.h" + + +static PyMethodDef test_methods[] = { + {NULL}, +}; + +int +_PyTestCapi_Init_File(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0){ + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index cff53fb950fcc6..15ea97ec4520b7 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -1,4 +1,8 @@ +// clinic/float.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" +#include "util.h" #include "clinic/float.c.h" diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index 829200ad12cd3c..b472a4185a98af 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -99,10 +99,11 @@ slot_tp_del(PyObject *self) return; } /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); + del = _PyType_LookupRef(Py_TYPE(self), tp_del); Py_DECREF(tp_del); if (del != NULL) { res = PyObject_CallOneArg(del, self); + Py_DECREF(del); if (res == NULL) PyErr_WriteUnraisable(del); else @@ -126,9 +127,7 @@ slot_tp_del(PyObject *self) * never happened. */ { - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); + _Py_ResurrectReference(self); } assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); } diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 86d8f5984dd3e5..ee04c760d27213 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -56,9 +56,9 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) keywords[i] = PyBytes_AS_STRING(o); } else { - PyErr_Format(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "parse_tuple_and_keywords: " - "keywords must be str or bytes", i); + "keywords must be str or bytes"); goto exit; } } @@ -71,18 +71,22 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) if (result) { int objects_only = 1; + int count = 0; for (const char *f = sub_format; *f; f++) { - if (Py_ISALNUM(*f) && strchr("OSUY", *f) == NULL) { - objects_only = 0; - break; + if (Py_ISALNUM(*f)) { + if (strchr("OSUY", *f) == NULL) { + objects_only = 0; + break; + } + count++; } } if (objects_only) { - return_value = PyTuple_New(size); + return_value = PyTuple_New(count); if (return_value == NULL) { goto exit; } - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < count; i++) { PyObject *arg = *(PyObject **)(buffers + i); if (arg == NULL) { arg = Py_None; @@ -138,32 +142,141 @@ getargs_w_star(PyObject *self, PyObject *args) } static PyObject * -test_empty_argparse(PyObject *self, PyObject *Py_UNUSED(ignored)) +getargs_w_star_opt(PyObject *self, PyObject *args) { - /* Test that formats can begin with '|'. See issue #4720. */ - PyObject *dict = NULL; - static char *kwlist[] = {NULL}; - PyObject *tuple = PyTuple_New(0); - if (!tuple) { + Py_buffer buffer; + Py_buffer buf2; + int number = 1; + + if (!PyArg_ParseTuple(args, "w*|w*i:getargs_w_star", + &buffer, &buf2, &number)) { + return NULL; + } + + if (2 <= buffer.len) { + char *str = buffer.buf; + str[0] = '['; + str[buffer.len-1] = ']'; + } + + PyObject *result = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + PyBuffer_Release(&buffer); + return result; +} + +/* Test the old w and w# codes that no longer work */ +static PyObject * +test_w_code_invalid(PyObject *self, PyObject *arg) +{ + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; + char *formats_3[] = {"O|w#$O", + "O|w$O", + "O|w#O", + "O|wO", + NULL}; + char *formats_4[] = {"O|w#O$O", + "O|wO$O", + "O|Ow#O", + "O|OwO", + "O|Ow#$O", + "O|Ow$O", + NULL}; + size_t n; + PyObject *args; + PyObject *kwargs; + PyObject *tmp; + + if (!(args = PyTuple_Pack(1, Py_None))) { + return NULL; + } + + kwargs = PyDict_New(); + if (!kwargs) { + Py_DECREF(args); + return NULL; + } + + if (PyDict_SetItemString(kwargs, "c", Py_None)) { + Py_DECREF(args); + Py_XDECREF(kwargs); + return NULL; + } + + for (n = 0; formats_3[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_3[n], + (char**) keywords, + &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_3[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + if (PyDict_DelItemString(kwargs, "c") || + PyDict_SetItemString(kwargs, "d", Py_None)) { + + Py_DECREF(kwargs); + Py_DECREF(args); return NULL; } + + for (n = 0; formats_4[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_4[n], + (char**) keywords, + &tmp, &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_4[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + Py_DECREF(args); + Py_DECREF(kwargs); + Py_RETURN_NONE; +} + +static PyObject * +getargs_empty(PyObject *self, PyObject *args, PyObject *kwargs) +{ + /* Test that formats can begin with '|'. See issue #4720. */ + assert(PyTuple_CheckExact(args)); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + int result; - if (!(result = PyArg_ParseTuple(tuple, "|:test_empty_argparse"))) { - goto done; - } - dict = PyDict_New(); - if (!dict) { - goto done; - } - result = PyArg_ParseTupleAndKeywords(tuple, dict, "|:test_empty_argparse", - kwlist); - done: - Py_DECREF(tuple); - Py_XDECREF(dict); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { + static char *kwlist[] = {NULL}; + result = PyArg_ParseTupleAndKeywords(args, kwargs, "|:getargs_empty", + kwlist); + } + else { + result = PyArg_ParseTuple(args, "|:getargs_empty"); + } if (!result) { return NULL; } - Py_RETURN_NONE; + return PyLong_FromLong(result); } /* Test tuple argument processing */ @@ -354,75 +467,6 @@ getargs_K(PyObject *self, PyObject *args) return PyLong_FromUnsignedLongLong(value); } -/* This function not only tests the 'k' getargs code, but also the - PyLong_AsUnsignedLongMask() function. */ -static PyObject * -test_k_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *tuple, *num; - unsigned long value; - - tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - /* a number larger than ULONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) { - return NULL; - } - - value = PyLong_AsUnsignedLongMask(num); - if (value != ULONG_MAX) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: " - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = 0; - if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; - } - if (value != ULONG_MAX) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: k code returned wrong value for long 0xFFF...FFF"); - return NULL; - } - - Py_DECREF(num); - num = PyLong_FromString("-FFFFFFFF000000000000000042", NULL, 16); - if (num == NULL) { - return NULL; - } - - value = PyLong_AsUnsignedLongMask(num); - if (value != (unsigned long)-0x42) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: " - "PyLong_AsUnsignedLongMask() returned wrong value for long -0xFFF..000042"); - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = 0; - if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; - } - if (value != (unsigned long)-0x42) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: k code returned wrong value for long -0xFFF..000042"); - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - static PyObject * getargs_f(PyObject *self, PyObject *args) { @@ -703,95 +747,6 @@ getargs_et_hash(PyObject *self, PyObject *args) return result; } -/* Test the L code for PyArg_ParseTuple. This should deliver a long long - for both long and int arguments. The test may leak a little memory if - it fails. -*/ -static PyObject * -test_L_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *tuple, *num; - long long value; - - tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - num = PyLong_FromLong(42); - if (num == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = -1; - if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; - } - if (value != 42) { - PyErr_SetString(PyExc_AssertionError, - "test_L_code: L code returned wrong value for long 42"); - return NULL; - } - - Py_DECREF(num); - num = PyLong_FromLong(42); - if (num == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = -1; - if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; - } - if (value != 42) { - PyErr_SetString(PyExc_AssertionError, - "test_L_code: L code returned wrong value for int 42"); - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - -/* Test the s and z codes for PyArg_ParseTuple. -*/ -static PyObject * -test_s_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - /* Unicode strings should be accepted */ - PyObject *tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - PyObject *obj = PyUnicode_Decode("t\xeate", strlen("t\xeate"), - "latin-1", NULL); - if (obj == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, obj); - - /* These two blocks used to raise a TypeError: - * "argument must be string without null bytes, not str" - */ - char *value; - if (!PyArg_ParseTuple(tuple, "s:test_s_code1", &value)) { - return NULL; - } - - if (!PyArg_ParseTuple(tuple, "z:test_s_code2", &value)) { - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - static PyObject * gh_99240_clear_args(PyObject *self, PyObject *args) { @@ -845,6 +800,8 @@ static PyMethodDef test_methods[] = { {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, + {"getargs_w_star_opt", getargs_w_star_opt, METH_VARARGS}, + {"getargs_empty", _PyCFunction_CAST(getargs_empty), METH_VARARGS|METH_KEYWORDS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, {"getargs_y_star", getargs_y_star, METH_VARARGS}, @@ -852,11 +809,8 @@ static PyMethodDef test_methods[] = { {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, - {"test_L_code", test_L_code, METH_NOARGS}, - {"test_empty_argparse", test_empty_argparse, METH_NOARGS}, - {"test_k_code", test_k_code, METH_NOARGS}, - {"test_s_code", test_s_code, METH_NOARGS}, {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, + {"test_w_code_invalid", test_w_code_invalid, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c new file mode 100644 index 00000000000000..809d537bfef0d3 --- /dev/null +++ b/Modules/_testcapi/hash.c @@ -0,0 +1,83 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +hash_getfuncdef(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // bind PyHash_GetFuncDef() + PyHash_FuncDef *def = PyHash_GetFuncDef(); + + PyObject *types = PyImport_ImportModule("types"); + if (types == NULL) { + return NULL; + } + + PyObject *result = PyObject_CallMethod(types, "SimpleNamespace", NULL); + Py_DECREF(types); + if (result == NULL) { + return NULL; + } + + // ignore PyHash_FuncDef.hash + + PyObject *value = PyUnicode_FromString(def->name); + int res = PyObject_SetAttrString(result, "name", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + value = PyLong_FromLong(def->hash_bits); + res = PyObject_SetAttrString(result, "hash_bits", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + value = PyLong_FromLong(def->seed_bits); + res = PyObject_SetAttrString(result, "seed_bits", value); + Py_DECREF(value); + if (res < 0) { + return NULL; + } + + return result; +} + + +static PyObject * +hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) +{ + void *ptr = PyLong_AsVoidPtr(arg); + if (ptr == NULL && PyErr_Occurred()) { + return NULL; + } + + Py_hash_t hash = Py_HashPointer(ptr); + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + +static PyObject * +object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + Py_hash_t hash = PyObject_GenericHash(arg); + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + +static PyMethodDef test_methods[] = { + {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, + {"hash_pointer", hash_pointer, METH_O}, + {"object_generichash", object_generichash, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Hash(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c new file mode 100644 index 00000000000000..09cec4c30c8c36 --- /dev/null +++ b/Modules/_testcapi/list.c @@ -0,0 +1,81 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_GET_SIZE(obj)); +} + + +static PyObject * +list_get_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GET_ITEM(obj, i)); +} + + +static PyObject * +list_set_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + PyList_SET_ITEM(obj, i, Py_XNewRef(value)); + Py_RETURN_NONE; + +} + + +static PyObject * +list_clear(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Clear(obj)); +} + + +static PyObject * +list_extend(PyObject* Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *arg; + if (!PyArg_ParseTuple(args, "OO", &obj, &arg)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(arg); + RETURN_INT(PyList_Extend(obj, arg)); +} + + +static PyMethodDef test_methods[] = { + {"list_get_size", list_get_size, METH_O}, + {"list_get_item", list_get_item, METH_VARARGS}, + {"list_set_item", list_set_item, METH_VARARGS}, + {"list_clear", list_clear, METH_O}, + {"list_extend", list_extend, METH_VARARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 32ad8d32ab8523..2b5e85d5707522 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -12,529 +12,6 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ -static PyObject * -raiseTestError(const char* test_name, const char* msg) -{ - PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); - return NULL; -} - -/* Tests of PyLong_{As, From}{Unsigned,}Long(), and - PyLong_{As, From}{Unsigned,}LongLong(). - - Note that the meat of the test is contained in testcapi_long.h. - This is revolting, but delicate code duplication is worse: "almost - exactly the same" code is needed to test long long, but the ubiquitous - dependence on type names makes it impossible to use a parameterized - function. A giant macro would be even worse than this. A C++ template - would be perfect. - - The "report an error" functions are deliberately not part of the #include - file: if the test fails, you can set a breakpoint in the appropriate - error function directly, and crawl back from there in the debugger. -*/ - -#define UNBIND(X) Py_DECREF(X); (X) = NULL - -static PyObject * -raise_test_long_error(const char* msg) -{ - return raiseTestError("test_long_api", msg); -} - -// Test PyLong_FromLong()/PyLong_AsLong() -// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). - -#define TESTNAME test_long_api_inner -#define TYPENAME long -#define F_S_TO_PY PyLong_FromLong -#define F_PY_TO_S PyLong_AsLong -#define F_U_TO_PY PyLong_FromUnsignedLong -#define F_PY_TO_U PyLong_AsUnsignedLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_long_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module) -/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ -{ - return TESTNAME(raise_test_long_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - -// Test PyLong_FromLongLong()/PyLong_AsLongLong() -// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). - -static PyObject * -raise_test_longlong_error(const char* msg) -{ - return raiseTestError("test_longlong_api", msg); -} - -#define TESTNAME test_longlong_api_inner -#define TYPENAME long long -#define F_S_TO_PY PyLong_FromLongLong -#define F_PY_TO_S PyLong_AsLongLong -#define F_U_TO_PY PyLong_FromUnsignedLongLong -#define F_PY_TO_U PyLong_AsUnsignedLongLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_longlong_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module) -/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ -{ - return TESTNAME(raise_test_longlong_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - - -/*[clinic input] -_testcapi.test_long_and_overflow - -Test the PyLong_AsLongAndOverflow API. - -General conversion to PY_LONG is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ -{ - PyObject *num, *one, *temp; - long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LONG_MAX + 1 */ - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LONG_MIN even on 64-bit platforms */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LONG_MIN - 1 */ - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MAX) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MIN) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_long_and_overflow - -Test the PyLong_AsLongLongAndOverflow API. - -General conversion to long long is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ -{ - PyObject *num, *one, *temp; - long long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LLONG_MAX on a typical machine. */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LLONG_MAX + 1 */ - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LLONG_MIN on a typical platform */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LLONG_MIN - 1 */ - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MAX) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MIN) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_size_t - -Test the PyLong_As{Size,Ssize}_t API. - -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ -{ - size_t out_u; - Py_ssize_t out_s; - - Py_INCREF(Py_None); - - out_u = PyLong_AsSize_t(Py_None); - if (out_u != (size_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - out_s = PyLong_AsSsize_t(Py_None); - if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - -/*[clinic input] -_testcapi.test_long_as_unsigned_long_long_mask -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) -/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ -{ - unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); - - if (res != (unsigned long long)-1 || !PyErr_Occurred()) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) didn't " - "complain"); - } - if (!PyErr_ExceptionMatches(PyExc_SystemError)) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) raised " - "something other than SystemError"); - } - PyErr_Clear(); - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_double -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module) -/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ -{ - double out; - - Py_INCREF(Py_None); - - out = PyLong_AsDouble(Py_None); - if (out != -1.0 || !PyErr_Occurred()) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - /*[clinic input] _testcapi.call_long_compact_api arg: object @@ -555,48 +32,6 @@ _testcapi_call_long_compact_api(PyObject *module, PyObject *arg) return Py_BuildValue("in", is_compact, value); } -static PyObject * -pylong_check(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_Check(obj)); -} - -static PyObject * -pylong_checkexact(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_CheckExact(obj)); -} - -static PyObject * -pylong_fromdouble(PyObject *module, PyObject *arg) -{ - double value; - if (!PyArg_Parse(arg, "d", &value)) { - return NULL; - } - return PyLong_FromDouble(value); -} - -static PyObject * -pylong_fromstring(PyObject *module, PyObject *args) -{ - const char *str; - Py_ssize_t len; - int base; - char *end = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { - return NULL; - } - - PyObject *result = PyLong_FromString(str, &end, base); - if (result == NULL) { - // XXX 'end' is not always set. - return NULL; - } - return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); -} static PyObject * pylong_fromunicodeobject(PyObject *module, PyObject *args) @@ -611,199 +46,84 @@ pylong_fromunicodeobject(PyObject *module, PyObject *args) return PyLong_FromUnicodeObject(unicode, base); } -static PyObject * -pylong_fromvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - return PyLong_FromVoidPtr((void *)arg); -} - -/*[clinic input] -_testcapi.PyLong_AsInt - arg: object - / -[clinic start generated code]*/ static PyObject * -_testcapi_PyLong_AsInt(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=0df9f19de5fa575b input=9561b97105493a67]*/ +pylong_asnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - assert(!PyErr_Occurred()); - int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) { + PyObject *v; + Py_buffer buffer; + Py_ssize_t n, flags; + if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &flags)) { return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long value = PyLong_AsLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (buffer.readonly) { + PyErr_SetString(PyExc_TypeError, "buffer must be writable"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long value = PyLong_AsLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return Py_BuildValue("li", value, overflow); + Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res >= 0 ? PyLong_FromSsize_t(res) : NULL; } -static PyObject * -pylong_asunsignedlong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLong(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLong(value); -} static PyObject * -pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +pylong_fromnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLongMask(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { + Py_buffer buffer; + Py_ssize_t n, flags, signed_; + if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &flags, &signed_)) { return NULL; } - return PyLong_FromUnsignedLong(value); -} - -static PyObject * -pylong_aslonglong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long long value = PyLong_AsLongLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromLongLong(value); + PyObject *res = signed_ + ? PyLong_FromNativeBytes(buffer.buf, n, (int)flags) + : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res; } -static PyObject * -pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); - return NULL; - } - return Py_BuildValue("Li", value, overflow); -} static PyObject * -pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +pylong_getsign(PyObject *module, PyObject *arg) { + int sign; NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLong(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { + if (PyLong_GetSign(arg, &sign) == -1) { return NULL; } - return PyLong_FromUnsignedLongLong(value); + return PyLong_FromLong(sign); } -static PyObject * -pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); -} static PyObject * -pylong_as_ssize_t(PyObject *module, PyObject *arg) +pylong_aspid(PyObject *module, PyObject *arg) { NULLABLE(arg); - Py_ssize_t value = PyLong_AsSsize_t(arg); + pid_t value = PyLong_AsPid(arg); if (value == -1 && PyErr_Occurred()) { return NULL; } - return PyLong_FromSsize_t(value); -} - -static PyObject * -pylong_as_size_t(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - size_t value = PyLong_AsSize_t(arg); - if (value == (size_t)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromSize_t(value); + return PyLong_FromPid(value); } -static PyObject * -pylong_asdouble(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - double value = PyLong_AsDouble(arg); - if (value == -1.0 && PyErr_Occurred()) { - return NULL; - } - return PyFloat_FromDouble(value); -} - -static PyObject * -pylong_asvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - void *value = PyLong_AsVoidPtr(arg); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; - } - return Py_NewRef((PyObject *)value); -} static PyMethodDef test_methods[] = { - _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONG_API_METHODDEF - _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF - _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF - _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF - _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF - {"pylong_check", pylong_check, METH_O}, - {"pylong_checkexact", pylong_checkexact, METH_O}, - {"pylong_fromdouble", pylong_fromdouble, METH_O}, - {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, - {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, - _TESTCAPI_PYLONG_ASINT_METHODDEF - {"pylong_aslong", pylong_aslong, METH_O}, - {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, - {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, - {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, - {"pylong_aslonglong", pylong_aslonglong, METH_O}, - {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, - {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, - {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, - {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, - {"pylong_as_size_t", pylong_as_size_t, METH_O}, - {"pylong_asdouble", pylong_asdouble, METH_O}, - {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, + {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, + {"pylong_getsign", pylong_getsign, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, {NULL}, }; @@ -813,6 +133,5 @@ _PyTestCapi_Init_Long(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } - return 0; } diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ef9234d7f8955f..ab4ad934644c38 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -613,5 +613,14 @@ _PyTestCapi_Init_Mem(PyObject *mod) return -1; } +#ifdef WITH_MIMALLOC + v = Py_True; +#else + v = Py_False; +#endif + if (PyModule_AddObjectRef(mod, "WITH_MIMALLOC", v) < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c new file mode 100644 index 00000000000000..6fd4a405688f48 --- /dev/null +++ b/Modules/_testcapi/monitoring.c @@ -0,0 +1,508 @@ +#include "parts.h" +#include "util.h" + +#include "monitoring.h" + +#define Py_BUILD_CORE +#include "internal/pycore_instruments.h" + +typedef struct { + PyObject_HEAD + PyMonitoringState *monitoring_states; + uint64_t version; + int num_events; + /* Other fields */ +} PyCodeLikeObject; + + +static PyObject * +CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int num_events; + if (!PyArg_ParseTuple(args, "i", &num_events)) { + return NULL; + } + PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc( + num_events, sizeof(PyMonitoringState)); + if (states == NULL) { + return NULL; + } + PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->version = 0; + self->monitoring_states = states; + self->num_events = num_events; + } + else { + PyMem_Free(states); + } + return (PyObject *) self; +} + +static void +CodeLike_dealloc(PyCodeLikeObject *self) +{ + if (self->monitoring_states) { + PyMem_Free(self->monitoring_states); + } + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CodeLike_str(PyCodeLikeObject *self) +{ + PyObject *res = NULL; + PyObject *sep = NULL; + PyObject *parts = NULL; + if (self->monitoring_states) { + parts = PyList_New(0); + if (parts == NULL) { + goto end; + } + + PyObject *heading = PyUnicode_FromString("PyCodeLikeObject"); + if (heading == NULL) { + goto end; + } + int err = PyList_Append(parts, heading); + Py_DECREF(heading); + if (err < 0) { + goto end; + } + + for (int i = 0; i < self->num_events; i++) { + PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active); + if (part == NULL) { + goto end; + } + int err = PyList_Append(parts, part); + Py_XDECREF(part); + if (err < 0) { + goto end; + } + } + sep = PyUnicode_FromString(": "); + if (sep == NULL) { + goto end; + } + res = PyUnicode_Join(sep, parts); + } +end: + Py_XDECREF(sep); + Py_XDECREF(parts); + return res; +} + +static PyTypeObject PyCodeLike_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "monitoring.CodeLike", + .tp_doc = PyDoc_STR("CodeLike objects"), + .tp_basicsize = sizeof(PyCodeLikeObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = CodeLike_new, + .tp_dealloc = (destructor) CodeLike_dealloc, + .tp_str = (reprfunc) CodeLike_str, +}; + +#define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + return NULL; \ + } + +/*******************************************************************/ + +static PyMonitoringState * +setup_fire(PyObject *codelike, int offset, PyObject *exc) +{ + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike); + assert(offset >= 0 && offset < cl->num_events); + PyMonitoringState *state = &cl->monitoring_states[offset]; + + if (exc != NULL) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + return state; +} + +static int +teardown_fire(int res, PyMonitoringState *state, PyObject *exception) +{ + if (res == -1) { + return -1; + } + if (exception) { + assert(PyErr_Occurred()); + assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred()); + } + + else { + assert(!PyErr_Occurred()); + } + PyErr_Clear(); + return state->active; +} + +static PyObject * +fire_event_py_start(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyStartEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_resume(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_yield(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_call(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *callable, *arg0; + if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_line(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset, lineno; + if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_jump(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_throw(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_reraise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireReraiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_exception_handled(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_unwind(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_stop_iteration(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *value; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &value)) { + return NULL; + } + NULLABLE(value); + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset, value); + RETURN_INT(teardown_fire(res, state, exception)); +} + +/*******************************************************************/ + +static PyObject * +enter_scope(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int event1, event2=0; + Py_ssize_t num_events = PyTuple_Size(args) - 1; + if (num_events == 1) { + if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) { + return NULL; + } + } + else { + assert(num_events == 2); + if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) { + return NULL; + } + } + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike; + + uint8_t events[] = { event1, event2 }; + + PyMonitoring_EnterScope(cl->monitoring_states, + &cl->version, + events, + num_events); + + Py_RETURN_NONE; +} + +static PyObject * +exit_scope(PyObject *self, PyObject *args) +{ + PyMonitoring_ExitScope(); + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"fire_event_py_start", fire_event_py_start, METH_VARARGS}, + {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS}, + {"fire_event_py_return", fire_event_py_return, METH_VARARGS}, + {"fire_event_c_return", fire_event_c_return, METH_VARARGS}, + {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS}, + {"fire_event_call", fire_event_call, METH_VARARGS}, + {"fire_event_line", fire_event_line, METH_VARARGS}, + {"fire_event_jump", fire_event_jump, METH_VARARGS}, + {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, + {"fire_event_raise", fire_event_raise, METH_VARARGS}, + {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, + {"fire_event_reraise", fire_event_reraise, METH_VARARGS}, + {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS}, + {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS}, + {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS}, + {"monitoring_enter_scope", enter_scope, METH_VARARGS}, + {"monitoring_exit_scope", exit_scope, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Monitoring(PyObject *m) +{ + if (PyType_Ready(&PyCodeLike_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) { + Py_DECREF(m); + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/numbers.c b/Modules/_testcapi/numbers.c new file mode 100644 index 00000000000000..6f7fa3fa7a4186 --- /dev/null +++ b/Modules/_testcapi/numbers.c @@ -0,0 +1,16 @@ +#include "parts.h" +#include "util.h" + +static PyMethodDef test_methods[] = { + {NULL}, +}; + +int +_PyTestCapi_Init_Numbers(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c new file mode 100644 index 00000000000000..1c76e766a790f0 --- /dev/null +++ b/Modules/_testcapi/object.c @@ -0,0 +1,145 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +call_pyobject_print(PyObject *self, PyObject * args) +{ + PyObject *object; + PyObject *filename; + PyObject *print_raw; + FILE *fp; + int flags = 0; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 3, 3, + &object, &filename, &print_raw)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (Py_IsTrue(print_raw)) { + flags = Py_PRINT_RAW; + } + + if (PyObject_Print(object, fp, flags) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_null(PyObject *self, PyObject *args) +{ + PyObject *filename; + FILE *fp; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(NULL, fp, 0) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_noref_object(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + char correct_string[100]; + + test_string = PyUnicode_FromString("Spam spam spam"); + + Py_SET_REFCNT(test_string, 0); + + PyOS_snprintf(correct_string, 100, "", + Py_REFCNT(test_string), (void *)test_string); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(test_string, fp, 0) < 0){ + fclose(fp); + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + + return PyUnicode_FromString(correct_string); +} + +static PyObject * +pyobject_print_os_error(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + + test_string = PyUnicode_FromString("Spam spam spam"); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + // open file in read mode to induce OSError + fp = _Py_fopen_obj(filename, "r"); + + if (PyObject_Print(test_string, fp, 0) < 0) { + fclose(fp); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + Py_DECREF(test_string); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj) +{ + PyUnstable_Object_ClearWeakRefsNoCallbacks(obj); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, + {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, + {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS}, + {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, + {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index acdba86504f58e..41d190961c69ee 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -4,27 +4,34 @@ // Always enable assertions #undef NDEBUG -// The _testcapi extension tests the public C API: header files in Include/ and -// Include/cpython/ directories. The internal C API must not be tested by -// _testcapi: use _testinternalcapi for that. -// -// _testcapi C files can built with the Py_BUILD_CORE_BUILTIN macro defined if -// one of the Modules/Setup files asks to build _testcapi as "static" -// (gh-109723). -// -// The Visual Studio projects builds _testcapi with Py_BUILD_CORE_MODULE. -#undef Py_BUILD_CORE_MODULE -#undef Py_BUILD_CORE_BUILTIN +#ifdef PYTESTCAPI_NEED_INTERNAL_API +# ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +# endif +#else + // The _testcapi extension tests the public C API: header files in Include/ + // and Include/cpython/ directories. The internal C API must not be tested + // by _testcapi: use _testinternalcapi for that. + // + // _testcapi C files can built with the Py_BUILD_CORE_BUILTIN macro defined + // if one of the Modules/Setup files asks to build _testcapi as "static" + // (gh-109723). + // + // The Visual Studio projects builds _testcapi with Py_BUILD_CORE_MODULE. +# undef Py_BUILD_CORE_MODULE +# undef Py_BUILD_CORE_BUILTIN +#endif #include "Python.h" -#ifdef Py_BUILD_CORE +#if defined(Py_BUILD_CORE) && !defined(PYTESTCAPI_NEED_INTERNAL_API) # error "_testcapi must test the public Python C API, not the internal C API" #endif int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); +int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); @@ -33,18 +40,25 @@ int _PyTestCapi_Init_Mem(PyObject *module); int _PyTestCapi_Init_Watchers(PyObject *module); int _PyTestCapi_Init_Long(PyObject *module); int _PyTestCapi_Init_Float(PyObject *module); +int _PyTestCapi_Init_Complex(PyObject *module); +int _PyTestCapi_Init_Numbers(PyObject *module); int _PyTestCapi_Init_Dict(PyObject *module); int _PyTestCapi_Init_Set(PyObject *module); +int _PyTestCapi_Init_List(PyObject *module); +int _PyTestCapi_Init_Tuple(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyAtomic(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Run(PyObject *module); +int _PyTestCapi_Init_File(PyObject *module); +int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); -int _PyTestCapi_Init_GC(PyObject *mod); - -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); -int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_GC(PyObject *module); +int _PyTestCapi_Init_Hash(PyObject *module); +int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Monitoring(PyObject *module); +int _PyTestCapi_Init_Object(PyObject *module); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 5aedf687705707..850de6f9c3366b 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -125,6 +125,7 @@ test_atomic_fences(PyObject *self, PyObject *obj) { // Just make sure that the fences compile. We are not // testing any synchronizing ordering. _Py_atomic_fence_seq_cst(); + _Py_atomic_fence_acquire(); _Py_atomic_fence_release(); Py_RETURN_NONE; } @@ -140,6 +141,21 @@ test_atomic_release_acquire(PyObject *self, PyObject *obj) { Py_RETURN_NONE; } +static PyObject * +test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ + int x = 0; + int y = 1; + int z = 2; + assert(_Py_atomic_load_int_acquire(&x) == 0); + _Py_atomic_store_int_release(&x, y); + assert(x == y); + assert(_Py_atomic_load_int_acquire(&x) == y); + _Py_atomic_store_int_release(&x, z); + assert(x == z); + assert(_Py_atomic_load_int_acquire(&x) == z); + Py_RETURN_NONE; +} + // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py @@ -162,6 +178,7 @@ static PyMethodDef test_methods[] = { FOR_BITWISE_TYPES(BIND_TEST_AND_OR) {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, + {"test_atomic_load_store_int_release_acquire", test_atomic_load_store_int_release_acquire, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c new file mode 100644 index 00000000000000..21244d02967ebf --- /dev/null +++ b/Modules/_testcapi/run.c @@ -0,0 +1,113 @@ +#define PYTESTCAPI_NEED_INTERNAL_API +#include "parts.h" +#include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() + +#include +#include + + +static PyObject * +run_stringflags(PyObject *mod, PyObject *pos_args) +{ + const char *str; + Py_ssize_t size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oii", + &str, &size, &start, &globals, &locals, + &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return PyRun_StringFlags(str, start, globals, locals, pflags); +} + +static PyObject * +run_fileexflags(PyObject *mod, PyObject *pos_args) +{ + PyObject *result = NULL; + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + FILE *fp = NULL; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oiii", + &filename, &filename_size, &start, &globals, &locals, + &closeit, &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); + + result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); + + if (closeit && result && _Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "File was not closed after excution"); + Py_DECREF(result); + fclose(fp); + return NULL; + } + + if (!closeit && !_Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after excution"); + Py_XDECREF(result); + return NULL; + } + + if (!closeit) { + fclose(fp); /* don't need open file any more*/ + } + + return result; +} + +static PyMethodDef test_methods[] = { + {"run_stringflags", run_stringflags, METH_VARARGS}, + {"run_fileexflags", run_fileexflags, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Run(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/set.c b/Modules/_testcapi/set.c index 2fbd0aeffcd9f9..31b52cee5e9623 100644 --- a/Modules/_testcapi/set.c +++ b/Modules/_testcapi/set.c @@ -1,75 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -set_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Check(obj)); -} - -static PyObject * -set_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_CheckExact(obj)); -} - -static PyObject * -frozenset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_Check(obj)); -} - -static PyObject * -frozenset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_CheckExact(obj)); -} - -static PyObject * -anyset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_Check(obj)); -} - -static PyObject * -anyset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_CheckExact(obj)); -} - -static PyObject * -set_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PySet_New(iterable); -} - -static PyObject * -frozenset_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PyFrozenSet_New(iterable); -} - -static PyObject * -set_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySet_Size(obj)); -} - static PyObject * set_get_size(PyObject *self, PyObject *obj) { @@ -77,111 +8,8 @@ set_get_size(PyObject *self, PyObject *obj) RETURN_SIZE(PySet_GET_SIZE(obj)); } -static PyObject * -set_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Contains(obj, item)); -} - -static PyObject * -set_add(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Add(obj, item)); -} - -static PyObject * -set_discard(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Discard(obj, item)); -} - -static PyObject * -set_pop(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySet_Pop(obj); -} - -static PyObject * -set_clear(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Clear(obj)); -} - -static PyObject * -test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) -{ - // Test that `frozenset` can be used with `PySet_Add`, - // when frozenset is just created in CAPI. - PyObject *fs = PyFrozenSet_New(NULL); - if (fs == NULL) { - return NULL; - } - PyObject *num = PyLong_FromLong(1); - if (num == NULL) { - goto error; - } - if (PySet_Add(fs, num) < 0) { - goto error; - } - int contains = PySet_Contains(fs, num); - if (contains < 0) { - goto error; - } - else if (contains == 0) { - goto unexpected; - } - Py_DECREF(fs); - Py_DECREF(num); - Py_RETURN_NONE; - -unexpected: - PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); -error: - Py_DECREF(fs); - Py_XDECREF(num); - return NULL; -} - static PyMethodDef test_methods[] = { - {"set_check", set_check, METH_O}, - {"set_checkexact", set_checkexact, METH_O}, - {"frozenset_check", frozenset_check, METH_O}, - {"frozenset_checkexact", frozenset_checkexact, METH_O}, - {"anyset_check", anyset_check, METH_O}, - {"anyset_checkexact", anyset_checkexact, METH_O}, - - {"set_new", set_new, METH_VARARGS}, - {"frozenset_new", frozenset_new, METH_VARARGS}, - - {"set_size", set_size, METH_O}, {"set_get_size", set_get_size, METH_O}, - {"set_contains", set_contains, METH_VARARGS}, - {"set_add", set_add, METH_VARARGS}, - {"set_discard", set_discard, METH_VARARGS}, - {"set_pop", set_pop, METH_O}, - {"set_clear", set_clear, METH_O}, - - {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c new file mode 100644 index 00000000000000..464cf5c3125012 --- /dev/null +++ b/Modules/_testcapi/time.c @@ -0,0 +1,169 @@ +#include "parts.h" + + +static int +pytime_from_nanoseconds(PyTime_t *tp, PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", + Py_TYPE(obj)->tp_name); + return -1; + } + + long long nsec = PyLong_AsLongLong(obj); + if (nsec == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + *tp = (PyTime_t)nsec; + return 0; +} + + +static PyObject * +test_pytime_assecondsdouble(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + PyTime_t ts; + if (pytime_from_nanoseconds(&ts, obj) < 0) { + return NULL; + } + double d = PyTime_AsSecondsDouble(ts); + return PyFloat_FromDouble(d); +} + + +static PyObject* +pytime_as_float(PyTime_t t) +{ + return PyFloat_FromDouble(PyTime_AsSecondsDouble(t)); +} + + + +static PyObject* +test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Monotonic(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_monotonic_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_MonotonicRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_MonotonicRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_PerfCounter(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_PerfCounterRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_PerfCounterRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res = PyTime_Time(&t); + if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_TimeRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_TimeRaw() failed"); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyMethodDef test_methods[] = { + {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, + {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_MonotonicRaw", test_pytime_monotonic_raw, METH_NOARGS}, + {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_PerfCounterRaw", test_pytime_perf_counter_raw, METH_NOARGS}, + {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {"PyTime_TimeRaw", test_pytime_time_raw, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Time(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + Py_BUILD_ASSERT(sizeof(long long) == sizeof(PyTime_t)); + if (PyModule_AddObject(m, "PyTime_MIN", PyLong_FromLongLong(PyTime_MIN)) < 0) { + return 1; + } + if (PyModule_AddObject(m, "PyTime_MAX", PyLong_FromLongLong(PyTime_MAX)) < 0) { + return 1; + } + return 0; +} diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c new file mode 100644 index 00000000000000..95dde8c0edadbe --- /dev/null +++ b/Modules/_testcapi/tuple.c @@ -0,0 +1,17 @@ +#include "parts.h" +#include "util.h" + + +static PyMethodDef test_methods[] = { + {NULL}, +}; + +int +_PyTestCapi_Init_Tuple(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0){ + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d52d88a65d86fc..b8ecf53f4f8b9c 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,103 +3,29 @@ #include "parts.h" #include "util.h" -static struct PyModuleDef *_testcapimodule = NULL; // set at initialization - -static PyObject * -codec_incrementalencoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalEncoder(encoding, errors); -} - -static PyObject * -codec_incrementaldecoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalDecoder(encoding, errors); -} - -static PyObject * -test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); - int result; - if (py_s == NULL) - return NULL; - result = PyUnicode_CompareWithASCIIString(py_s, "str"); - Py_DECREF(py_s); - if (!result) { - PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " - "should not compare equal to c string."); - return NULL; - } - Py_RETURN_NONE; -} - +/* Test PyUnicode_New() */ static PyObject * -test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +unicode_new(PyObject *self, PyObject *args) { -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; - size_t wtextlen = 1; - const wchar_t invalid[1] = {(wchar_t)0x110000u}; -#else - const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; - size_t wtextlen = 2; -#endif - PyObject *wide, *utf8; - - wide = PyUnicode_FromWideChar(wtext, wtextlen); - if (wide == NULL) - return NULL; + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; - utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); - if (utf8 == NULL) { - Py_DECREF(wide); + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { return NULL; } - if (PyUnicode_GET_LENGTH(wide) != PyUnicode_GET_LENGTH(utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "have different length"); - return NULL; - } - if (PyUnicode_Compare(wide, utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - if (PyErr_Occurred()) - return NULL; - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "are different"); + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { return NULL; } - - Py_DECREF(wide); - Py_DECREF(utf8); - -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - wide = PyUnicode_FromWideChar(invalid, 1); - if (wide == NULL) - PyErr_Clear(); - else { - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); return NULL; } -#endif - Py_RETURN_NONE; + return result; } @@ -130,30 +56,6 @@ unicode_copy(PyObject *unicode) return copy; } -/* Test PyUnicode_New() */ -static PyObject * -unicode_new(PyObject *self, PyObject *args) -{ - Py_ssize_t size; - unsigned int maxchar; - PyObject *result; - - if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { - return NULL; - } - - result = PyUnicode_New(size, (Py_UCS4)maxchar); - if (!result) { - return NULL; - } - if (size > 0 && maxchar <= 0x10ffff && - PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) - { - Py_DECREF(result); - return NULL; - } - return result; -} /* Test PyUnicode_Fill() */ static PyObject * @@ -180,1929 +82,490 @@ unicode_fill(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, filled); } -/* Test PyUnicode_WriteChar() */ + +/* Test PyUnicode_FromKindAndData() */ static PyObject * -unicode_writechar(PyObject *self, PyObject *args) +unicode_fromkindanddata(PyObject *self, PyObject *args) { - PyObject *to, *to_copy; - Py_ssize_t index; - unsigned int character; - int result; + int kind; + void *buffer; + Py_ssize_t bsize; + Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { return NULL; } - NULLABLE(to); - if (!(to_copy = unicode_copy(to)) && to) { - return NULL; + if (size == -100) { + size = bsize; } - - result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); - if (result == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + if (kind && size % kind) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromkindanddata()"); return NULL; } - return Py_BuildValue("(Ni)", to_copy, result); + return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); } -/* Test PyUnicode_Resize() */ + +// Test PyUnicode_AsUCS4(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_resize(PyObject *self, PyObject *args) +unicode_asucs4(PyObject *self, PyObject *args) { - PyObject *obj, *copy; - Py_ssize_t length; - int result; + PyObject *unicode, *result; + Py_UCS4 *buffer; + int copy_null; + Py_ssize_t str_len, buf_len; - if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { return NULL; } - NULLABLE(obj); - if (!(copy = unicode_copy(obj)) && obj) { - return NULL; + NULLABLE(unicode); + buf_len = str_len + 1; + buffer = PyMem_NEW(Py_UCS4, buf_len); + if (buffer == NULL) { + return PyErr_NoMemory(); } - result = PyUnicode_Resize(©, length); - if (result == -1 && PyErr_Occurred()) { - Py_XDECREF(copy); + memset(buffer, 0, sizeof(Py_UCS4)*buf_len); + buffer[str_len] = 0xffffU; + + if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { + PyMem_Free(buffer); return NULL; } - if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { - if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { - Py_DECREF(copy); - return NULL; - } - } - return Py_BuildValue("(Ni)", copy, result); + + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); + PyMem_Free(buffer); + return result; } -/* Test PyUnicode_Append() */ + +// Test PyUnicode_AsUCS4Copy(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_append(PyObject *self, PyObject *args) +unicode_asucs4copy(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_UCS4 *buffer; + PyObject *result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "O", &unicode)) { return NULL; + } - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + buffer = PyUnicode_AsUCS4Copy(unicode); + if (buffer == NULL) { return NULL; } - PyUnicode_Append(&left_copy, right); - return left_copy; + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buffer, + PyUnicode_GET_LENGTH(unicode) + 1); + PyMem_FREE(buffer); + return result; } -/* Test PyUnicode_AppendAndDel() */ + +/* Test PyUnicode_AsUTF8() */ static PyObject * -unicode_appendanddel(PyObject *self, PyObject *args) +unicode_asutf8(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_ssize_t buflen; + const char *s; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + s = PyUnicode_AsUTF8(unicode); + if (s == NULL) return NULL; - } - Py_XINCREF(right); - PyUnicode_AppendAndDel(&left_copy, right); - return left_copy; + + return PyBytes_FromStringAndSize(s, buflen); } -/* Test PyUnicode_FromStringAndSize() */ + +/* Test PyUnicode_CopyCharacters() */ static PyObject * -unicode_fromstringandsize(PyObject *self, PyObject *args) +unicode_copycharacters(PyObject *self, PyObject *args) { - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; + PyObject *from, *to, *to_copy; + Py_ssize_t from_start, to_start, how_many, copied; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, + &from, &from_start, &how_many)) { return NULL; } - if (size == -100) { - size = bsize; + NULLABLE(from); + if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), + PyUnicode_MAX_CHAR_VALUE(to)))) { + return NULL; + } + if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { + Py_DECREF(to_copy); + return NULL; + } + + copied = PyUnicode_CopyCharacters(to_copy, to_start, from, + from_start, how_many); + if (copied == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; } - return PyUnicode_FromStringAndSize(s, size); + + return Py_BuildValue("(Nn)", to_copy, copied); } -/* Test PyUnicode_FromString() */ + +// --- PyUnicodeWriter type ------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyUnicodeWriter *writer; +} WriterObject; + + static PyObject * -unicode_fromstring(PyObject *self, PyObject *arg) +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { return NULL; } - return PyUnicode_FromString(s); + self->writer = NULL; + return (PyObject*)self; } -/* Test PyUnicode_FromKindAndData() */ -static PyObject * -unicode_fromkindanddata(PyObject *self, PyObject *args) + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) { - int kind; - void *buffer; - Py_ssize_t bsize; - Py_ssize_t size = -100; + WriterObject *self = (WriterObject *)self_raw; - if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { - return NULL; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return -1; } - if (size == -100) { - size = bsize; + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - if (kind && size % kind) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromkindanddata()"); - return NULL; + + self->writer = PyUnicodeWriter_Create(size); + if (self->writer == NULL) { + return -1; } - return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); + return 0; } -/* Test PyUnicode_Substring() */ -static PyObject * -unicode_substring(PyObject *self, PyObject *args) -{ - PyObject *str; - Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { - return NULL; +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - - NULLABLE(str); - return PyUnicode_Substring(str, start, end); + tp->tp_free(self); + Py_DECREF(tp); } -/* Test PyUnicode_GetLength() */ -static PyObject * -unicode_getlength(PyObject *self, PyObject *arg) + +static inline int +writer_check(WriterObject *self) { - NULLABLE(arg); - RETURN_SIZE(PyUnicode_GetLength(arg)); + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; } -/* Test PyUnicode_ReadChar() */ -static PyObject * -unicode_readchar(PyObject *self, PyObject *args) + +static PyObject* +writer_write_char(PyObject *self_raw, PyObject *args) { - PyObject *unicode; - Py_ssize_t index; - Py_UCS4 result; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { return NULL; } + if (PyUnicode_GET_LENGTH(str) != 1) { + PyErr_SetString(PyExc_ValueError, "expect a single character"); + } + Py_UCS4 ch = PyUnicode_READ_CHAR(str, 0); - NULLABLE(unicode); - result = PyUnicode_ReadChar(unicode, index); - if (result == (Py_UCS4)-1) + if (PyUnicodeWriter_WriteChar(self->writer, ch) < 0) { return NULL; - return PyLong_FromUnsignedLong(result); + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromEncodedObject() */ -static PyObject * -unicode_fromencodedobject(PyObject *self, PyObject *args) + +static PyObject* +writer_write_utf8(PyObject *self_raw, PyObject *args) { - PyObject *obj; - const char *encoding; - const char *errors = NULL; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + char *str; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &str, &size)) { return NULL; } - NULLABLE(obj); - return PyUnicode_FromEncodedObject(obj, encoding, errors); + if (PyUnicodeWriter_WriteUTF8(self->writer, str, size) < 0) { + return NULL; + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromObject() */ -static PyObject * -unicode_fromobject(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_FromObject(arg); -} -/* Test PyUnicode_InternInPlace() */ -static PyObject * -unicode_interninplace(PyObject *self, PyObject *arg) +static PyObject* +writer_write_widechar(PyObject *self_raw, PyObject *args) { - NULLABLE(arg); - Py_XINCREF(arg); - PyUnicode_InternInPlace(&arg); - return arg; -} + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { + return NULL; + } -/* Test PyUnicode_InternFromString() */ -static PyObject * -unicode_internfromstring(PyObject *self, PyObject *arg) -{ - const char *s; Py_ssize_t size; + wchar_t *wstr = PyUnicode_AsWideCharString(str, &size); + if (wstr == NULL) { + return NULL; + } - if (!PyArg_Parse(arg, "z#", &s, &size)) { + int res = PyUnicodeWriter_WriteWideChar(self->writer, wstr, size); + PyMem_Free(wstr); + if (res < 0) { return NULL; } - return PyUnicode_InternFromString(s); + Py_RETURN_NONE; } -/* Test PyUnicode_FromWideChar() */ -static PyObject * -unicode_fromwidechar(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { +static PyObject* +writer_write_ucs4(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - if (size == -100) { - if (bsize % SIZEOF_WCHAR_T) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromwidechar()"); - return NULL; - } - size = bsize / SIZEOF_WCHAR_T; - } - return PyUnicode_FromWideChar((const wchar_t *)s, size); -} -/* Test PyUnicode_AsWideChar() */ -static PyObject * -unicode_aswidechar(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t buflen, size; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - buffer = PyMem_New(wchar_t, buflen); - if (buffer == NULL) - return PyErr_NoMemory(); - - size = PyUnicode_AsWideChar(unicode, buffer, buflen); - if (size == -1) { - PyMem_Free(buffer); - return NULL; - } - - if (size < buflen) - buflen = size + 1; - else - buflen = size; - result = PyUnicode_FromWideChar(buffer, buflen); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as buffer */ -static PyObject * -unicode_aswidechar_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); -} - -/* Test PyUnicode_AsWideCharString() */ -static PyObject * -unicode_aswidecharstring(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t size = UNINITIALIZED_SIZE; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, &size); - if (buffer == NULL) { - assert(size == UNINITIALIZED_SIZE); - return NULL; - } - - result = PyUnicode_FromWideChar(buffer, size + 1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as the size address */ -static PyObject * -unicode_aswidecharstring_null(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, NULL); - if (buffer == NULL) - return NULL; - - result = PyUnicode_FromWideChar(buffer, -1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return result; -} - -/* Test PyUnicode_AsUCS4() */ -static PyObject * -unicode_asucs4(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_UCS4 *buffer; - int copy_null; - Py_ssize_t str_len, buf_len; - - if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { - return NULL; - } - - NULLABLE(unicode); - buf_len = str_len + 1; - buffer = PyMem_NEW(Py_UCS4, buf_len); - if (buffer == NULL) { - return PyErr_NoMemory(); - } - memset(buffer, 0, sizeof(Py_UCS4)*buf_len); - buffer[str_len] = 0xffffU; - - if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { - PyMem_Free(buffer); - return NULL; - } - - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); - PyMem_Free(buffer); - return result; -} - -/* Test PyUnicode_AsUCS4Copy() */ -static PyObject * -unicode_asucs4copy(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_UCS4 *buffer; - PyObject *result; - - if (!PyArg_ParseTuple(args, "O", &unicode)) { - return NULL; - } - - NULLABLE(unicode); - buffer = PyUnicode_AsUCS4Copy(unicode); - if (buffer == NULL) { - return NULL; - } - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buffer, - PyUnicode_GET_LENGTH(unicode) + 1); - PyMem_FREE(buffer); - return result; -} - -/* Test PyUnicode_FromOrdinal() */ -static PyObject * -unicode_fromordinal(PyObject *self, PyObject *args) -{ - int ordinal; - - if (!PyArg_ParseTuple(args, "i", &ordinal)) - return NULL; - - return PyUnicode_FromOrdinal(ordinal); -} - -/* Test PyUnicode_AsUTF8() */ -static PyObject * -unicode_asutf8(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8(unicode); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_AsUTF8AndSize() */ -static PyObject * -unicode_asutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, &size); - if (s == NULL) { - assert(size == UNINITIALIZED_SIZE); - return NULL; - } - - return Py_BuildValue("(y#n)", s, buflen, size); -} - -/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ -static PyObject * -unicode_asutf8andsize_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, NULL); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_GetDefaultEncoding() */ -static PyObject * -unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - const char *s = PyUnicode_GetDefaultEncoding(); - if (s == NULL) - return NULL; - - return PyBytes_FromString(s); -} - -/* Test PyUnicode_Decode() */ -static PyObject * -unicode_decode(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t size; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) - return NULL; - - return PyUnicode_Decode(s, size, encoding, errors); -} - -/* Test PyUnicode_AsEncodedString() */ -static PyObject * -unicode_asencodedstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_AsEncodedString(unicode, encoding, errors); -} - -/* Test PyUnicode_BuildEncodingMap() */ -static PyObject * -unicode_buildencodingmap(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_BuildEncodingMap(arg); -} - -/* Test PyUnicode_DecodeUTF7() */ -static PyObject * -unicode_decodeutf7(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF7(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF7Stateful() */ -static PyObject * -unicode_decodeutf7stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeUTF8() */ -static PyObject * -unicode_decodeutf8(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF8(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF8Stateful() */ -static PyObject * -unicode_decodeutf8stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsUTF8String() */ -static PyObject * -unicode_asutf8string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF8String(arg); -} - -/* Test PyUnicode_DecodeUTF32() */ -static PyObject * -unicode_decodeutf32(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF32Stateful() */ -static PyObject * -unicode_decodeutf32stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF32String() */ -static PyObject * -unicode_asutf32string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF32String(arg); -} - -/* Test PyUnicode_DecodeUTF16() */ -static PyObject * -unicode_decodeutf16(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF16Stateful() */ -static PyObject * -unicode_decodeutf16stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF16String() */ -static PyObject * -unicode_asutf16string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF16String(arg); -} - -/* Test PyUnicode_DecodeUnicodeEscape() */ -static PyObject * -unicode_decodeunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsUnicodeEscapeString() */ -static PyObject * -unicode_asunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decoderawunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsRawUnicodeEscapeString() */ -static PyObject * -unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsRawUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decodelatin1(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLatin1(data, size, errors); -} - -/* Test PyUnicode_AsLatin1String() */ -static PyObject * -unicode_aslatin1string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsLatin1String(arg); -} - -/* Test PyUnicode_DecodeASCII() */ -static PyObject * -unicode_decodeascii(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeASCII(data, size, errors); -} - -/* Test PyUnicode_AsASCIIString() */ -static PyObject * -unicode_asasciistring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsASCIIString(arg); -} - -/* Test PyUnicode_DecodeCharmap() */ -static PyObject * -unicode_decodecharmap(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - PyObject *mapping; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) - return NULL; - - NULLABLE(mapping); - return PyUnicode_DecodeCharmap(data, size, mapping, errors); -} - -/* Test PyUnicode_AsCharmapString() */ -static PyObject * -unicode_ascharmapstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - PyObject *mapping; - - if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) - return NULL; - - NULLABLE(unicode); - NULLABLE(mapping); - return PyUnicode_AsCharmapString(unicode, mapping); -} - -#ifdef MS_WINDOWS - -/* Test PyUnicode_DecodeMBCS() */ -static PyObject * -unicode_decodembcs(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeMBCS(data, size, errors); -} - -/* Test PyUnicode_DecodeMBCSStateful() */ -static PyObject * -unicode_decodembcsstateful(PyObject *self, PyObject *args) -{ - const char *data; + PyObject *str; Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + if (!PyArg_ParseTuple(args, "Un", &str, &size)) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeCodePageStateful() */ -static PyObject * -unicode_decodecodepagestateful(PyObject *self, PyObject *args) -{ - int code_page; - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) - return NULL; + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + size = Py_MIN(size, len); - result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(str); + if (ucs4 == NULL) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsMBCSString() */ -static PyObject * -unicode_asmbcsstring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsMBCSString(arg); -} - -/* Test PyUnicode_EncodeCodePage() */ -static PyObject * -unicode_encodecodepage(PyObject *self, PyObject *args) -{ - int code_page; - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeCodePage(code_page, unicode, errors); -} - -#endif /* MS_WINDOWS */ - -/* Test PyUnicode_DecodeLocaleAndSize() */ -static PyObject * -unicode_decodelocaleandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocaleAndSize(data, size, errors); -} - -/* Test PyUnicode_DecodeLocale() */ -static PyObject * -unicode_decodelocale(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocale(data, errors); -} - -/* Test PyUnicode_EncodeLocale() */ -static PyObject * -unicode_encodelocale(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeLocale(unicode, errors); -} -/* Test PyUnicode_DecodeFSDefault() */ -static PyObject * -unicode_decodefsdefault(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#", &data, &size)) - return NULL; - - return PyUnicode_DecodeFSDefault(data); -} - -/* Test PyUnicode_DecodeFSDefaultAndSize() */ -static PyObject * -unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + int res = PyUnicodeWriter_WriteUCS4(self->writer, ucs4, size); + PyMem_Free(ucs4); + if (res < 0) { return NULL; - - return PyUnicode_DecodeFSDefaultAndSize(data, size); -} - -/* Test PyUnicode_EncodeFSDefault() */ -static PyObject * -unicode_encodefsdefault(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_EncodeFSDefault(arg); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Concat() */ -static PyObject * -unicode_concat(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_Concat(left, right); -} -/* Test PyUnicode_Split() */ -static PyObject * -unicode_split(PyObject *self, PyObject *args) +static PyObject* +writer_write_str(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Split(s, sep, maxsplit); -} - -/* Test PyUnicode_RSplit() */ -static PyObject * -unicode_rsplit(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RSplit(s, sep, maxsplit); -} - -/* Test PyUnicode_Splitlines() */ -static PyObject * -unicode_splitlines(PyObject *self, PyObject *args) -{ - PyObject *s; - int keepends = 0; - - if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) { return NULL; - - NULLABLE(s); - return PyUnicode_Splitlines(s, keepends); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Partition() */ -static PyObject * -unicode_partition(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Partition(s, sep); -} -/* Test PyUnicode_RPartition() */ -static PyObject * -unicode_rpartition(PyObject *self, PyObject *args) +static PyObject* +writer_write_repr(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RPartition(s, sep); -} - -/* Test PyUnicode_Translate() */ -static PyObject * -unicode_translate(PyObject *self, PyObject *args) -{ PyObject *obj; - PyObject *table; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(obj); - NULLABLE(table); - return PyUnicode_Translate(obj, table, errors); -} - -/* Test PyUnicode_Join() */ -static PyObject * -unicode_join(PyObject *self, PyObject *args) -{ - PyObject *sep; - PyObject *seq; - - if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + if (PyUnicodeWriter_WriteRepr(self->writer, obj) < 0) { return NULL; - - NULLABLE(sep); - NULLABLE(seq); - return PyUnicode_Join(sep, seq); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Count() */ -static PyObject * -unicode_count(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - - if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); -} -/* Test PyUnicode_Find() */ -static PyObject * -unicode_find(PyObject *self, PyObject *args) +static PyObject* +writer_write_substring(PyObject *self_raw, PyObject *args) { - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - Py_ssize_t result; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - result = PyUnicode_Find(str, substr, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Tailmatch() */ -static PyObject * -unicode_tailmatch(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); -} -/* Test PyUnicode_FindChar() */ -static PyObject * -unicode_findchar(PyObject *self, PyObject *args) -{ PyObject *str; - int direction; - unsigned int ch; - Py_ssize_t result; Py_ssize_t start, end; - - if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, - &start, &end, &direction)) { - return NULL; - } - NULLABLE(str); - result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + if (!PyArg_ParseTuple(args, "Unn", &str, &start, &end)) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Replace() */ -static PyObject * -unicode_replace(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - PyObject *replstr; - Py_ssize_t maxcount = -1; - - if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - NULLABLE(replstr); - return PyUnicode_Replace(str, substr, replstr, maxcount); -} - -/* Test PyUnicode_Compare() */ -static PyObject * -unicode_compare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - result = PyUnicode_Compare(left, right); - if (result == -1 && PyErr_Occurred()) { + if (PyUnicodeWriter_WriteSubstring(self->writer, str, start, end) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); + Py_RETURN_NONE; } -/* Test PyUnicode_CompareWithASCIIString() */ -static PyObject * -unicode_comparewithasciistring(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - - if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) - return NULL; - NULLABLE(left); - result = PyUnicode_CompareWithASCIIString(left, right); - if (result == -1 && PyErr_Occurred()) { +static PyObject* +writer_decodeutf8stateful(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8() */ -static PyObject * -unicode_equaltoutf8(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + const char *str; + Py_ssize_t len; + const char *errors; + int use_consumed = 0; + if (!PyArg_ParseTuple(args, "yny|i", &str, &len, &errors, &use_consumed)) { return NULL; } - NULLABLE(left); - result = PyUnicode_EqualToUTF8(left, right); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8AndSize() */ -static PyObject * -unicode_equaltoutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - Py_ssize_t size = -100; - int result; - - if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + Py_ssize_t consumed = 12345; + Py_ssize_t *pconsumed = use_consumed ? &consumed : NULL; + if (PyUnicodeWriter_DecodeUTF8Stateful(self->writer, str, len, + errors, pconsumed) < 0) { + if (use_consumed) { + assert(consumed == 0); + } return NULL; } - NULLABLE(left); - if (size == -100) { - size = right_len; + if (use_consumed) { + return PyLong_FromSsize_t(consumed); } - result = PyUnicode_EqualToUTF8AndSize(left, right, size); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_RichCompare() */ -static PyObject * -unicode_richcompare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int op; - - if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_RichCompare(left, right, op); + Py_RETURN_NONE; } -/* Test PyUnicode_Format() */ -static PyObject * -unicode_format(PyObject *self, PyObject *args) -{ - PyObject *format; - PyObject *fargs; - - if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) - return NULL; - - NULLABLE(format); - NULLABLE(fargs); - return PyUnicode_Format(format, fargs); -} -/* Test PyUnicode_Contains() */ -static PyObject * -unicode_contains(PyObject *self, PyObject *args) +static PyObject* +writer_get_pointer(PyObject *self_raw, PyObject *args) { - PyObject *container; - PyObject *element; - - if (!PyArg_ParseTuple(args, "OO", &container, &element)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(container); - NULLABLE(element); - RETURN_INT(PyUnicode_Contains(container, element)); + return PyLong_FromVoidPtr(self->writer); } -/* Test PyUnicode_IsIdentifier() */ -static PyObject * -unicode_isidentifier(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - RETURN_INT(PyUnicode_IsIdentifier(arg)); -} -/* Test PyUnicode_CopyCharacters() */ -static PyObject * -unicode_copycharacters(PyObject *self, PyObject *args) +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { - PyObject *from, *to, *to_copy; - Py_ssize_t from_start, to_start, how_many, copied; - - if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, - &from, &from_start, &how_many)) { - return NULL; - } - - NULLABLE(from); - if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), - PyUnicode_MAX_CHAR_VALUE(to)))) { - return NULL; - } - if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { - Py_DECREF(to_copy); - return NULL; - } - - copied = PyUnicode_CopyCharacters(to_copy, to_start, from, - from_start, how_many); - if (copied == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return Py_BuildValue("(Nn)", to_copy, copied); + PyObject *str = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; + return str; } -static int -check_raised_systemerror(PyObject *result, char* msg) -{ - if (result) { - // no exception - PyErr_Format(PyExc_AssertionError, - "SystemError not raised: %s", - msg); - return 0; - } - if (PyErr_ExceptionMatches(PyExc_SystemError)) { - // expected exception - PyErr_Clear(); - return 1; - } - // unexpected exception - return 0; -} -static PyObject * -test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *result; - PyObject *unicode = PyUnicode_FromString("None"); - -#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ - result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ - if (EXPECTED == NULL) { \ - if (!check_raised_systemerror(result, FORMAT)) { \ - goto Fail; \ - } \ - } \ - else if (result == NULL) \ - return NULL; \ - else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ - PyErr_Format(PyExc_AssertionError, \ - "test_string_from_format: failed at \"%s\" " \ - "expected \"%s\" got \"%s\"", \ - FORMAT, EXPECTED, PyUnicode_AsUTF8(result)); \ - goto Fail; \ - } \ - Py_XDECREF(result) - -#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) - -#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) - - // Unrecognized - CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); - - // "%%" (options are rejected) - CHECK_FORMAT_0( "%%", "%"); - CHECK_FORMAT_0( "%0%", NULL); - CHECK_FORMAT_0("%00%", NULL); - CHECK_FORMAT_0( "%2%", NULL); - CHECK_FORMAT_0("%02%", NULL); - CHECK_FORMAT_0("%.0%", NULL); - CHECK_FORMAT_0("%.2%", NULL); - - // "%c" - CHECK_FORMAT_1( "%c", "c", 'c'); - CHECK_FORMAT_1( "%0c", "c", 'c'); - CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", NULL, 'c'); - CHECK_FORMAT_1("%02c", NULL, 'c'); - CHECK_FORMAT_1("%.0c", NULL, 'c'); - CHECK_FORMAT_1("%.2c", NULL, 'c'); - - // Integers - CHECK_FORMAT_1("%d", "123", (int)123); - CHECK_FORMAT_1("%i", "123", (int)123); - CHECK_FORMAT_1("%u", "123", (unsigned int)123); - CHECK_FORMAT_1("%x", "7b", (unsigned int)123); - CHECK_FORMAT_1("%X", "7B", (unsigned int)123); - CHECK_FORMAT_1("%o", "173", (unsigned int)123); - CHECK_FORMAT_1("%ld", "123", (long)123); - CHECK_FORMAT_1("%li", "123", (long)123); - CHECK_FORMAT_1("%lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); - CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); - CHECK_FORMAT_1("%lo", "173", (unsigned long)123); - CHECK_FORMAT_1("%lld", "123", (long long)123); - CHECK_FORMAT_1("%lli", "123", (long long)123); - CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); - CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); - CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); - CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%zx", "7b", (size_t)123); - CHECK_FORMAT_1("%zX", "7B", (size_t)123); - CHECK_FORMAT_1("%zo", "173", (size_t)123); - CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); - CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); - CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); - CHECK_FORMAT_1("%jd", "123", (intmax_t)123); - CHECK_FORMAT_1("%ji", "123", (intmax_t)123); - CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); - CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); - CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); - CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); - - CHECK_FORMAT_1("%d", "-123", (int)-123); - CHECK_FORMAT_1("%i", "-123", (int)-123); - CHECK_FORMAT_1("%ld", "-123", (long)-123); - CHECK_FORMAT_1("%li", "-123", (long)-123); - CHECK_FORMAT_1("%lld", "-123", (long long)-123); - CHECK_FORMAT_1("%lli", "-123", (long long)-123); - CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); - CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); - - // Integers: width < length - CHECK_FORMAT_1("%1d", "123", (int)123); - CHECK_FORMAT_1("%1i", "123", (int)123); - CHECK_FORMAT_1("%1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%1ld", "123", (long)123); - CHECK_FORMAT_1("%1li", "123", (long)123); - CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%1lld", "123", (long long)123); - CHECK_FORMAT_1("%1lli", "123", (long long)123); - CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zu", "123", (size_t)123); - CHECK_FORMAT_1("%1x", "7b", (int)123); - - CHECK_FORMAT_1("%1d", "-123", (int)-123); - CHECK_FORMAT_1("%1i", "-123", (int)-123); - CHECK_FORMAT_1("%1ld", "-123", (long)-123); - CHECK_FORMAT_1("%1li", "-123", (long)-123); - CHECK_FORMAT_1("%1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - - // Integers: width > length - CHECK_FORMAT_1("%5d", " 123", (int)123); - CHECK_FORMAT_1("%5i", " 123", (int)123); - CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); - CHECK_FORMAT_1("%5ld", " 123", (long)123); - CHECK_FORMAT_1("%5li", " 123", (long)123); - CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); - CHECK_FORMAT_1("%5lld", " 123", (long long)123); - CHECK_FORMAT_1("%5lli", " 123", (long long)123); - CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); - CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zu", " 123", (size_t)123); - CHECK_FORMAT_1("%5x", " 7b", (int)123); - - CHECK_FORMAT_1("%5d", " -123", (int)-123); - CHECK_FORMAT_1("%5i", " -123", (int)-123); - CHECK_FORMAT_1("%5ld", " -123", (long)-123); - CHECK_FORMAT_1("%5li", " -123", (long)-123); - CHECK_FORMAT_1("%5lld", " -123", (long long)-123); - CHECK_FORMAT_1("%5lli", " -123", (long long)-123); - CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - - // Integers: width > length, 0-flag - CHECK_FORMAT_1("%05d", "00123", (int)123); - CHECK_FORMAT_1("%05i", "00123", (int)123); - CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%05ld", "00123", (long)123); - CHECK_FORMAT_1("%05li", "00123", (long)123); - CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%05lld", "00123", (long long)123); - CHECK_FORMAT_1("%05lli", "00123", (long long)123); - CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zu", "00123", (size_t)123); - CHECK_FORMAT_1("%05x", "0007b", (int)123); - - CHECK_FORMAT_1("%05d", "-0123", (int)-123); - CHECK_FORMAT_1("%05i", "-0123", (int)-123); - CHECK_FORMAT_1("%05ld", "-0123", (long)-123); - CHECK_FORMAT_1("%05li", "-0123", (long)-123); - CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); - CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); - CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - - // Integers: precision < length - CHECK_FORMAT_1("%.1d", "123", (int)123); - CHECK_FORMAT_1("%.1i", "123", (int)123); - CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%.1ld", "123", (long)123); - CHECK_FORMAT_1("%.1li", "123", (long)123); - CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%.1lld", "123", (long long)123); - CHECK_FORMAT_1("%.1lli", "123", (long long)123); - CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zu", "123", (size_t)123); - CHECK_FORMAT_1("%.1x", "7b", (int)123); - - CHECK_FORMAT_1("%.1d", "-123", (int)-123); - CHECK_FORMAT_1("%.1i", "-123", (int)-123); - CHECK_FORMAT_1("%.1ld", "-123", (long)-123); - CHECK_FORMAT_1("%.1li", "-123", (long)-123); - CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - - // Integers: precision > length - CHECK_FORMAT_1("%.5d", "00123", (int)123); - CHECK_FORMAT_1("%.5i", "00123", (int)123); - CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%.5ld", "00123", (long)123); - CHECK_FORMAT_1("%.5li", "00123", (long)123); - CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%.5lld", "00123", (long long)123); - CHECK_FORMAT_1("%.5lli", "00123", (long long)123); - CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); - CHECK_FORMAT_1("%.5x", "0007b", (int)123); - - CHECK_FORMAT_1("%.5d", "-00123", (int)-123); - CHECK_FORMAT_1("%.5i", "-00123", (int)-123); - CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); - CHECK_FORMAT_1("%.5li", "-00123", (long)-123); - CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - - // Integers: width > precision > length - CHECK_FORMAT_1("%7.5d", " 00123", (int)123); - CHECK_FORMAT_1("%7.5i", " 00123", (int)123); - CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); - CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); - CHECK_FORMAT_1("%7.5li", " 00123", (long)123); - CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); - CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); - CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); - CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); - - CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - - // Integers: width > precision > length, 0-flag - CHECK_FORMAT_1("%07.5d", "0000123", (int)123); - CHECK_FORMAT_1("%07.5i", "0000123", (int)123); - CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); - CHECK_FORMAT_1("%07.5li", "0000123", (long)123); - CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%07.5x", "000007b", (int)123); - - CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - - // Integers: precision > width > length - CHECK_FORMAT_1("%5.7d", "0000123", (int)123); - CHECK_FORMAT_1("%5.7i", "0000123", (int)123); - CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%5.7li", "0000123", (long)123); - CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%5.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision > width > length, 0-flag - CHECK_FORMAT_1("%05.7d", "0000123", (int)123); - CHECK_FORMAT_1("%05.7i", "0000123", (int)123); - CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%05.7li", "0000123", (long)123); - CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%05.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision = 0, arg = 0 (empty string in C) - CHECK_FORMAT_1("%.0d", "0", (int)0); - CHECK_FORMAT_1("%.0i", "0", (int)0); - CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); - CHECK_FORMAT_1("%.0ld", "0", (long)0); - CHECK_FORMAT_1("%.0li", "0", (long)0); - CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); - CHECK_FORMAT_1("%.0lld", "0", (long long)0); - CHECK_FORMAT_1("%.0lli", "0", (long long)0); - CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); - CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zu", "0", (size_t)0); - CHECK_FORMAT_1("%.0x", "0", (int)0); - - // Strings - CHECK_FORMAT_1("%s", "None", "None"); - CHECK_FORMAT_1("%ls", "None", L"None"); - CHECK_FORMAT_1("%U", "None", unicode); - CHECK_FORMAT_1("%A", "None", Py_None); - CHECK_FORMAT_1("%S", "None", Py_None); - CHECK_FORMAT_1("%R", "None", Py_None); - CHECK_FORMAT_2("%V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%V", "None", NULL, "None"); - CHECK_FORMAT_2("%lV", "None", NULL, L"None"); - - // Strings: width < length - CHECK_FORMAT_1("%1s", "None", "None"); - CHECK_FORMAT_1("%1ls", "None", L"None"); - CHECK_FORMAT_1("%1U", "None", unicode); - CHECK_FORMAT_1("%1A", "None", Py_None); - CHECK_FORMAT_1("%1S", "None", Py_None); - CHECK_FORMAT_1("%1R", "None", Py_None); - CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1V", "None", NULL, "None"); - CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); - - // Strings: width > length - CHECK_FORMAT_1("%5s", " None", "None"); - CHECK_FORMAT_1("%5ls", " None", L"None"); - CHECK_FORMAT_1("%5U", " None", unicode); - CHECK_FORMAT_1("%5A", " None", Py_None); - CHECK_FORMAT_1("%5S", " None", Py_None); - CHECK_FORMAT_1("%5R", " None", Py_None); - CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); - CHECK_FORMAT_2("%5V", " None", NULL, "None"); - CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); - - // Strings: precision < length - CHECK_FORMAT_1("%.1s", "N", "None"); - CHECK_FORMAT_1("%.1ls", "N", L"None"); - CHECK_FORMAT_1("%.1U", "N", unicode); - CHECK_FORMAT_1("%.1A", "N", Py_None); - CHECK_FORMAT_1("%.1S", "N", Py_None); - CHECK_FORMAT_1("%.1R", "N", Py_None); - CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); - CHECK_FORMAT_2("%.1V", "N", NULL, "None"); - CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); - - // Strings: precision > length - CHECK_FORMAT_1("%.5s", "None", "None"); - CHECK_FORMAT_1("%.5ls", "None", L"None"); - CHECK_FORMAT_1("%.5U", "None", unicode); - CHECK_FORMAT_1("%.5A", "None", Py_None); - CHECK_FORMAT_1("%.5S", "None", Py_None); - CHECK_FORMAT_1("%.5R", "None", Py_None); - CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); - - // Strings: precision < length, width > length - CHECK_FORMAT_1("%5.1s", " N", "None"); - CHECK_FORMAT_1("%5.1ls"," N", L"None"); - CHECK_FORMAT_1("%5.1U", " N", unicode); - CHECK_FORMAT_1("%5.1A", " N", Py_None); - CHECK_FORMAT_1("%5.1S", " N", Py_None); - CHECK_FORMAT_1("%5.1R", " N", Py_None); - CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); - CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); - CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); - - // Strings: width < length, precision > length - CHECK_FORMAT_1("%1.5s", "None", "None"); - CHECK_FORMAT_1("%1.5ls", "None", L"None"); - CHECK_FORMAT_1("%1.5U", "None", unicode); - CHECK_FORMAT_1("%1.5A", "None", Py_None); - CHECK_FORMAT_1("%1.5S", "None", Py_None); - CHECK_FORMAT_1("%1.5R", "None", Py_None); - CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); - - Py_XDECREF(unicode); - Py_RETURN_NONE; +static PyMethodDef writer_methods[] = { + {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, + {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, + {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, + {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, + {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_VARARGS}, + {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, + {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, + {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; - Fail: - Py_XDECREF(result); - Py_XDECREF(unicode); - return NULL; +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyUnicodeWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; -#undef CHECK_FORMAT_2 -#undef CHECK_FORMAT_1 -#undef CHECK_FORMAT_0 -} static PyMethodDef TestMethods[] = { - {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, - {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, - {"test_unicode_compare_with_ascii", - test_unicode_compare_with_ascii, METH_NOARGS}, - {"test_string_from_format", test_string_from_format, METH_NOARGS}, - {"test_widechar", test_widechar, METH_NOARGS}, {"unicode_new", unicode_new, METH_VARARGS}, {"unicode_fill", unicode_fill, METH_VARARGS}, - {"unicode_writechar", unicode_writechar, METH_VARARGS}, - {"unicode_resize", unicode_resize, METH_VARARGS}, - {"unicode_append", unicode_append, METH_VARARGS}, - {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, - {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, - {"unicode_fromstring", unicode_fromstring, METH_O}, {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, - {"unicode_substring", unicode_substring, METH_VARARGS}, - {"unicode_getlength", unicode_getlength, METH_O}, - {"unicode_readchar", unicode_readchar, METH_VARARGS}, - {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, - {"unicode_fromobject", unicode_fromobject, METH_O}, - {"unicode_interninplace", unicode_interninplace, METH_O}, - {"unicode_internfromstring", unicode_internfromstring, METH_O}, - {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, - {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, - {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, - {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, - {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, - {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, - {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, - {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, - {"unicode_decode", unicode_decode, METH_VARARGS}, - {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, - {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, - {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, - {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, - {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, - {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, - {"unicode_asutf8string", unicode_asutf8string, METH_O}, - {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, - {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, - {"unicode_asutf16string", unicode_asutf16string, METH_O}, - {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, - {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, - {"unicode_asutf32string", unicode_asutf32string, METH_O}, - {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, - {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, - {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, - {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, - {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, - {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, - {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, - {"unicode_asasciistring", unicode_asasciistring, METH_O}, - {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, - {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, -#ifdef MS_WINDOWS - {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, - {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, - {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, - {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, - {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, -#endif /* MS_WINDOWS */ - {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, - {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, - {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, - {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, - {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, - {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, - {"unicode_concat", unicode_concat, METH_VARARGS}, - {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, - {"unicode_split", unicode_split, METH_VARARGS}, - {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, - {"unicode_partition", unicode_partition, METH_VARARGS}, - {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, - {"unicode_translate", unicode_translate, METH_VARARGS}, - {"unicode_join", unicode_join, METH_VARARGS}, - {"unicode_count", unicode_count, METH_VARARGS}, - {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, - {"unicode_find", unicode_find, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, - {"unicode_replace", unicode_replace, METH_VARARGS}, - {"unicode_compare", unicode_compare, METH_VARARGS}, - {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, - {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, - {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, - {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, - {"unicode_format", unicode_format, METH_VARARGS}, - {"unicode_contains", unicode_contains, METH_VARARGS}, - {"unicode_isidentifier", unicode_isidentifier, METH_O}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {NULL}, }; int _PyTestCapi_Init_Unicode(PyObject *m) { - _testcapimodule = PyModule_GetDef(m); - if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 2b5110fcba2c91..b30c5e8704c8af 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -1,3 +1,6 @@ +// clinic/vectorcall.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "clinic/vectorcall.c.h" diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 8a264bba4ed6ed..1eb0db2c2e6576 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -1,3 +1,6 @@ +// clinic/watchers.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "clinic/watchers.c.h" @@ -12,8 +15,8 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ // Test dict watching -static PyObject *g_dict_watch_events; -static int g_dict_watchers_installed; +static PyObject *g_dict_watch_events = NULL; +static int g_dict_watchers_installed = 0; static int dict_watch_callback(PyDict_WatchEvent event, diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc index a076e714980074..f5f3524f2c0177 100644 --- a/Modules/_testcapi_feature_macros.inc +++ b/Modules/_testcapi_feature_macros.inc @@ -38,6 +38,15 @@ if (res) { Py_DECREF(result); return NULL; } +#ifdef Py_TRACE_REFS + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_True); +#else + res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_False); +#endif +if (res) { + Py_DECREF(result); return NULL; +} + #ifdef USE_STACKCHECK res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True); #else diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c6a8dda58f6c22..5ebcfef6143e02 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -32,15 +32,32 @@ // Forward declarations static struct PyModuleDef _testcapimodule; -static PyObject *TestError; /* set to exception object in init */ +// Module state +typedef struct { + PyObject *error; // _testcapi.error object +} testcapistate_t; -/* Raise TestError with test_name + ": " + msg, and return NULL. */ +static testcapistate_t* +get_testcapi_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (testcapistate_t *)state; +} static PyObject * -raiseTestError(const char* test_name, const char* msg) +get_testerror(PyObject *self) { + testcapistate_t *state = get_testcapi_state(self); + return state->error; +} + +/* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ + +static PyObject * +raiseTestError(PyObject *self, const char* test_name, const char* msg) { - PyErr_Format(TestError, "%s: %s", test_name, msg); + PyErr_Format(get_testerror(self), "%s: %s", test_name, msg); return NULL; } @@ -51,10 +68,10 @@ raiseTestError(const char* test_name, const char* msg) platforms have these hardcoded. Better safe than sorry. */ static PyObject* -sizeof_error(const char* fatname, const char* typname, +sizeof_error(PyObject *self, const char* fatname, const char* typname, int expected, int got) { - PyErr_Format(TestError, + PyErr_Format(get_testerror(self), "%s #define == %d but sizeof(%s) == %d", fatname, expected, typname, got); return (PyObject*)NULL; @@ -64,8 +81,11 @@ static PyObject* test_config(PyObject *self, PyObject *Py_UNUSED(ignored)) { #define CHECK_SIZEOF(FATNAME, TYPE) \ - if (FATNAME != sizeof(TYPE)) \ - return sizeof_error(#FATNAME, #TYPE, FATNAME, sizeof(TYPE)) + do { \ + if (FATNAME != sizeof(TYPE)) { \ + return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)); \ + } \ + } while (0) CHECK_SIZEOF(SIZEOF_SHORT, short); CHECK_SIZEOF(SIZEOF_INT, int); @@ -86,21 +106,25 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wtype-limits" #endif -#define CHECK_SIZEOF(TYPE, EXPECTED) \ - if (EXPECTED != sizeof(TYPE)) { \ - PyErr_Format(TestError, \ - "sizeof(%s) = %u instead of %u", \ - #TYPE, sizeof(TYPE), EXPECTED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIZEOF(TYPE, EXPECTED) \ + do { \ + if (EXPECTED != sizeof(TYPE)) { \ + PyErr_Format(get_testerror(self), \ + "sizeof(%s) = %u instead of %u", \ + #TYPE, sizeof(TYPE), EXPECTED); \ + return (PyObject*)NULL; \ + } \ + } while (0) #define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0) -#define CHECK_SIGNNESS(TYPE, SIGNED) \ - if (IS_SIGNED(TYPE) != SIGNED) { \ - PyErr_Format(TestError, \ - "%s signness is, instead of %i", \ - #TYPE, IS_SIGNED(TYPE), SIGNED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIGNNESS(TYPE, SIGNED) \ + do { \ + if (IS_SIGNED(TYPE) != SIGNED) { \ + PyErr_Format(get_testerror(self), \ + "%s signness is %i, instead of %i", \ + #TYPE, IS_SIGNED(TYPE), SIGNED); \ + return (PyObject*)NULL; \ + } \ + } while (0) /* integer types */ CHECK_SIZEOF(Py_UCS1, 1); @@ -169,7 +193,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) for (i = 0; i < NLIST; ++i) { PyObject* anint = PyList_GET_ITEM(list, i); if (PyLong_AS_LONG(anint) != NLIST-1-i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_list_api: reverse screwed up"); Py_DECREF(list); return (PyObject*)NULL; @@ -182,7 +206,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) } static int -test_dict_inner(int count) +test_dict_inner(PyObject *self, int count) { Py_ssize_t pos = 0, iterations = 0; int i; @@ -195,11 +219,11 @@ test_dict_inner(int count) for (i = 0; i < count; i++) { v = PyLong_FromLong(i); if (v == NULL) { - return -1; + goto error; } if (PyDict_SetItem(dict, v, v) < 0) { Py_DECREF(v); - return -1; + goto error; } Py_DECREF(v); } @@ -213,11 +237,12 @@ test_dict_inner(int count) assert(v != UNINITIALIZED_PTR); i = PyLong_AS_LONG(v) + 1; o = PyLong_FromLong(i); - if (o == NULL) - return -1; + if (o == NULL) { + goto error; + } if (PyDict_SetItem(dict, k, o) < 0) { Py_DECREF(o); - return -1; + goto error; } Py_DECREF(o); k = v = UNINITIALIZED_PTR; @@ -229,12 +254,15 @@ test_dict_inner(int count) if (iterations != count) { PyErr_SetString( - TestError, + get_testerror(self), "test_dict_iteration: dict iteration went wrong "); return -1; } else { return 0; } +error: + Py_DECREF(dict); + return -1; } @@ -245,7 +273,7 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) int i; for (i = 0; i < 200; i++) { - if (test_dict_inner(i) < 0) { + if (test_dict_inner(self, i) < 0) { return NULL; } } @@ -329,14 +357,14 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (obj == NULL) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: failed to create object"); return NULL; } if (type->tp_dict != NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type initialised too soon"); Py_DECREF(obj); return NULL; @@ -346,7 +374,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if ((hash == -1) && PyErr_Occurred()) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: could not hash object"); Py_DECREF(obj); return NULL; @@ -354,7 +382,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_dict == NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type not initialised by hash()"); Py_DECREF(obj); return NULL; @@ -362,7 +390,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_hash != PyType_Type.tp_hash) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: unexpected hash function"); Py_DECREF(obj); return NULL; @@ -422,7 +450,7 @@ py_buildvalue_ints(PyObject *self, PyObject *args) } static int -test_buildvalue_N_error(const char *fmt) +test_buildvalue_N_error(PyObject *self, const char *fmt) { PyObject *arg, *res; @@ -438,7 +466,7 @@ test_buildvalue_N_error(const char *fmt) } Py_DECREF(res); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in successful " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -447,13 +475,13 @@ test_buildvalue_N_error(const char *fmt) Py_INCREF(arg); res = Py_BuildValue(fmt, raise_error, NULL, arg); if (res != NULL || !PyErr_Occurred()) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "Py_BuildValue(\"%s\") didn't complain", fmt); return -1; } PyErr_Clear(); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in failed " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -477,25 +505,25 @@ test_buildvalue_N(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } if (res != arg) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "Py_BuildValue(\"N\") returned wrong result"); } if (Py_REFCNT(arg) != 2) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "arg was not decrefed in Py_BuildValue(\"N\")"); } Py_DECREF(res); Py_DECREF(arg); - if (test_buildvalue_N_error("O&N") < 0) + if (test_buildvalue_N_error(self, "O&N") < 0) return NULL; - if (test_buildvalue_N_error("(O&N)") < 0) + if (test_buildvalue_N_error(self, "(O&N)") < 0) return NULL; - if (test_buildvalue_N_error("[O&N]") < 0) + if (test_buildvalue_N_error(self, "[O&N]") < 0) return NULL; - if (test_buildvalue_N_error("{O&N}") < 0) + if (test_buildvalue_N_error(self, "{O&N}") < 0) return NULL; - if (test_buildvalue_N_error("{()O&(())N}") < 0) + if (test_buildvalue_N_error(self, "{()O&(())N}") < 0) return NULL; Py_RETURN_NONE; @@ -570,82 +598,44 @@ static PyType_Spec HeapTypeNameType_Spec = { }; static PyObject * -test_get_type_name(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_heaptype_for_name(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *tp_name = PyType_GetName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "int") == 0); - Py_DECREF(tp_name); - - tp_name = PyType_GetName(&PyModule_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "module") == 0); - Py_DECREF(tp_name); - - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "HeapTypeNameType") == 0); - Py_DECREF(tp_name); + return PyType_FromSpec(&HeapTypeNameType_Spec); +} - PyObject *name = PyUnicode_FromString("test_name"); - if (name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, "__name__", name) < 0) { - Py_DECREF(name); - goto done; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "test_name") == 0); - Py_DECREF(name); - Py_DECREF(tp_name); - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; +static PyObject * +get_type_name(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetName((PyTypeObject *)type); } static PyObject * -test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_qualname(PyObject *self, PyObject *type) { - PyObject *tp_qualname = PyType_GetQualName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "int") == 0); - Py_DECREF(tp_qualname); + assert(PyType_Check(type)); + return PyType_GetQualName((PyTypeObject *)type); +} - tp_qualname = PyType_GetQualName(&PyODict_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "OrderedDict") == 0); - Py_DECREF(tp_qualname); - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "HeapTypeNameType") == 0); - Py_DECREF(tp_qualname); +static PyObject * +get_type_fullyqualname(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetFullyQualifiedName((PyTypeObject *)type); +} - PyObject *spec_name = PyUnicode_FromString(HeapTypeNameType_Spec.name); - if (spec_name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, - "__qualname__", spec_name) < 0) { - Py_DECREF(spec_name); - goto done; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), - "_testcapi.HeapTypeNameType") == 0); - Py_DECREF(spec_name); - Py_DECREF(tp_qualname); - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; +static PyObject * +get_type_module_name(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetModuleName((PyTypeObject *)type); } + static PyObject * test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -781,6 +771,14 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +gilstate_ensure_release(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyGILState_Release(state); + Py_RETURN_NONE; +} + #ifndef MS_WINDOWS static PyThread_type_lock wait_done = NULL; @@ -836,25 +834,55 @@ static int _pending_callback(void *arg) * run from any python thread. */ static PyObject * -pending_threadfunc(PyObject *self, PyObject *arg) +pending_threadfunc(PyObject *self, PyObject *arg, PyObject *kwargs) { + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; PyObject *callable; - int r; - if (PyArg_ParseTuple(arg, "O", &callable) == 0) + unsigned int num = 1; + int blocking = 0; + int ensure_added = 0; + if (!PyArg_ParseTupleAndKeywords(arg, kwargs, + "O|I$pp:_pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) + { return NULL; + } /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - Py_BEGIN_ALLOW_THREADS - r = Py_AddPendingCall(&_pending_callback, callable); - Py_END_ALLOW_THREADS + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + int r; + do { + r = Py_AddPendingCall(&_pending_callback, callable); + } while (r < 0); + } + else { + if (Py_AddPendingCall(&_pending_callback, callable) < 0) { + break; + } + } + } + + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } - if (r<0) { + for (unsigned int i = num_added; i < num; i++) { Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ - Py_RETURN_FALSE; } - Py_RETURN_TRUE; + /* The callable is decref'ed above in each added _pending_callback(). */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } /* Test PyOS_string_to_double. */ @@ -863,27 +891,34 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { double result; const char *msg; -#define CHECK_STRING(STR, expected) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) \ - return NULL; \ - if (result != (double)expected) { \ - msg = "conversion of " STR " to float failed"; \ - goto fail; \ - } +#define CHECK_STRING(STR, expected) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + return NULL; \ + } \ + if (result != (double)expected) { \ + msg = "conversion of " STR " to float failed"; \ + goto fail; \ + } \ + } while (0) -#define CHECK_INVALID(STR) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) { \ - if (PyErr_ExceptionMatches(PyExc_ValueError)) \ - PyErr_Clear(); \ - else \ - return NULL; \ - } \ - else { \ - msg = "conversion of " STR " didn't raise ValueError"; \ - goto fail; \ - } +#define CHECK_INVALID(STR) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + if (PyErr_ExceptionMatches(PyExc_ValueError)) { \ + PyErr_Clear(); \ + } \ + else { \ + return NULL; \ + } \ + } \ + else { \ + msg = "conversion of " STR " didn't raise ValueError"; \ + goto fail; \ + } \ + } while (0) CHECK_STRING("0.1", 0.1); CHECK_STRING("1.234", 1.234); @@ -899,7 +934,7 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { Py_RETURN_NONE; fail: - return raiseTestError("test_string_to_double", msg); + return raiseTestError(self, "test_string_to_double", msg); #undef CHECK_STRING #undef CHECK_INVALID } @@ -950,16 +985,22 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) }; known_capsule *known = &known_capsules[0]; -#define FAIL(x) { error = (x); goto exit; } +#define FAIL(x) \ + do { \ + error = (x); \ + goto exit; \ + } while (0) #define CHECK_DESTRUCTOR \ - if (capsule_error) { \ - FAIL(capsule_error); \ - } \ - else if (!capsule_destructor_call_count) { \ - FAIL("destructor not called!"); \ - } \ - capsule_destructor_call_count = 0; \ + do { \ + if (capsule_error) { \ + FAIL(capsule_error); \ + } \ + else if (!capsule_destructor_call_count) { \ + FAIL("destructor not called!"); \ + } \ + capsule_destructor_call_count = 0; \ + } while (0) object = PyCapsule_New(capsule_pointer, capsule_name, capsule_destructor); PyCapsule_SetContext(object, capsule_context); @@ -1003,12 +1044,12 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) static char buffer[256]; #undef FAIL #define FAIL(x) \ - { \ - sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ - x, known->module, known->attribute); \ - error = buffer; \ - goto exit; \ - } \ + do { \ + sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ + x, known->module, known->attribute); \ + error = buffer; \ + goto exit; \ + } while (0) PyObject *module = PyImport_ImportModule(known->module); if (module) { @@ -1050,7 +1091,7 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: if (error) { - return raiseTestError("test_capsule", error); + return raiseTestError(self, "test_capsule", error); } Py_RETURN_NONE; #undef FAIL @@ -1233,6 +1274,26 @@ make_memoryview_from_NULL_pointer(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyMemoryView_FromBuffer(&info); } +static PyObject * +buffer_fill_info(PyObject *self, PyObject *args) +{ + Py_buffer info; + const char *data; + Py_ssize_t size; + int readonly; + int flags; + + if (!PyArg_ParseTuple(args, "s#ii:buffer_fill_info", + &data, &size, &readonly, &flags)) { + return NULL; + } + + if (PyBuffer_FillInfo(&info, NULL, (void *)data, size, readonly, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&info); +} + static PyObject * test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) { @@ -1261,7 +1322,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (ptr[2*i] != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1274,7 +1335,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (*(ptr-2*i) != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1327,7 +1388,7 @@ test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; error: - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_pep3118_obsolete_write_locks: failure"); return NULL; } @@ -1445,58 +1506,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static void -_xid_capsule_destructor(PyObject *capsule) -{ - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data != NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); - } -} - -static PyObject * -get_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *obj = NULL; - if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { - return NULL; - } - - _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); - if (data == NULL) { - PyErr_NoMemory(); - return NULL; - } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - PyMem_Free(data); - return NULL; - } - PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); - if (capsule == NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); - } - return capsule; -} - -static PyObject * -restore_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *capsule = NULL; - if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { - return NULL; - } - - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data == NULL) { - return NULL; - } - return _PyCrossInterpreterData_NewObject(data); -} - static PyMethodDef ml; static PyObject * @@ -1571,7 +1580,9 @@ test_structseq_newtype_doesnt_leak(PyObject *Py_UNUSED(self), descr.n_in_sequence = 1; PyTypeObject* structseq_type = PyStructSequence_NewType(&descr); - assert(structseq_type != NULL); + if (structseq_type == NULL) { + return NULL; + } assert(PyType_Check(structseq_type)); assert(PyType_FastSubclass(structseq_type, Py_TPFLAGS_TUPLE_SUBCLASS)); Py_DECREF(structseq_type); @@ -1968,7 +1979,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) { Py_tss_t tss_key = Py_tss_NEEDS_INIT; if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "creation time"); } @@ -1977,27 +1988,31 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (!PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create succeeded, " "but with TSS key in an uninitialized state"); } if (PyThread_tss_create(&tss_key) != 0) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create unsuccessful with " "an already initialized key"); } #define CHECK_TSS_API(expr) \ + do { \ (void)(expr); \ if (!PyThread_tss_is_created(&tss_key)) { \ - return raiseTestError("test_pythread_tss_key_state", \ + return raiseTestError(self, "test_pythread_tss_key_state", \ "TSS key initialization state was not " \ - "preserved after calling " #expr); } + "preserved after calling " #expr); \ + } \ + } while (0) + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); CHECK_TSS_API(PyThread_tss_get(&tss_key)); #undef CHECK_TSS_API PyThread_tss_delete(&tss_key); if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_delete called, but did not " "set the key state to uninitialized"); } @@ -2008,7 +2023,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (PyThread_tss_is_created(ptr_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "allocation time"); } @@ -2023,10 +2038,10 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return repr(self) */ static PyObject* -bad_get(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +bad_get(PyObject *module, PyObject *args) { PyObject *self, *obj, *cls; - if (!_PyArg_UnpackStack(args, nargs, "bad_get", 3, 3, &self, &obj, &cls)) { + if (!PyArg_ParseTuple(args, "OOO", &self, &obj, &cls)) { return NULL; } @@ -2313,7 +2328,7 @@ test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) \ Py_DECREF(obj); \ Py_RETURN_NONE; \ - } while (0) \ + } while (0) // Test Py_NewRef() and Py_XNewRef() macros @@ -2401,6 +2416,17 @@ type_get_version(PyObject *self, PyObject *type) return res; } +static PyObject * +type_modified(PyObject *self, PyObject *type) +{ + if (!PyType_Check(type)) { + PyErr_SetString(PyExc_TypeError, "argument must be a type"); + return NULL; + } + PyType_Modified((PyTypeObject *)type); + Py_RETURN_NONE; +} + static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -2466,7 +2492,7 @@ get_basic_static_type(PyObject *self, PyObject *args) PyTypeObject *cls = &BasicStaticTypes[num_basic_static_types_used++]; if (base != NULL) { - cls->tp_bases = Py_BuildValue("(O)", base); + cls->tp_bases = PyTuple_Pack(1, base); if (cls->tp_bases == NULL) { return NULL; } @@ -2666,107 +2692,60 @@ eval_eval_code_ex(PyObject *mod, PyObject *pos_args) PyObject **c_kwargs = NULL; - if (!PyArg_UnpackTuple(pos_args, - "eval_code_ex", - 2, - 8, - &code, - &globals, - &locals, - &args, - &kwargs, - &defaults, - &kw_defaults, - &closure)) + if (!PyArg_ParseTuple(pos_args, + "OO|OO!O!O!OO:eval_code_ex", + &code, + &globals, + &locals, + &PyTuple_Type, &args, + &PyDict_Type, &kwargs, + &PyTuple_Type, &defaults, + &kw_defaults, + &closure)) { goto exit; } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, - "code must be a Python code object"); - goto exit; - } - - if (!PyDict_Check(globals)) { - PyErr_SetString(PyExc_TypeError, "globals must be a dict"); - goto exit; - } - - if (locals && !PyMapping_Check(locals)) { - PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); - goto exit; - } - if (locals == Py_None) { - locals = NULL; - } + NULLABLE(code); + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(kw_defaults); + NULLABLE(closure); PyObject **c_args = NULL; Py_ssize_t c_args_len = 0; - - if (args) - { - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_TypeError, "args must be a tuple"); - goto exit; - } else { - c_args = &PyTuple_GET_ITEM(args, 0); - c_args_len = PyTuple_Size(args); - } + if (args) { + c_args = &PyTuple_GET_ITEM(args, 0); + c_args_len = PyTuple_Size(args); } Py_ssize_t c_kwargs_len = 0; + if (kwargs) { + c_kwargs_len = PyDict_Size(kwargs); + if (c_kwargs_len > 0) { + c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); + if (!c_kwargs) { + PyErr_NoMemory(); + goto exit; + } - if (kwargs) - { - if (!PyDict_Check(kwargs)) { - PyErr_SetString(PyExc_TypeError, "keywords must be a dict"); - goto exit; - } else { - c_kwargs_len = PyDict_Size(kwargs); - if (c_kwargs_len > 0) { - c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); - if (!c_kwargs) { - PyErr_NoMemory(); - goto exit; - } - - Py_ssize_t i = 0; - Py_ssize_t pos = 0; - - while (PyDict_Next(kwargs, - &pos, - &c_kwargs[i], - &c_kwargs[i + 1])) - { - i += 2; - } - c_kwargs_len = i / 2; - /* XXX This is broken if the caller deletes dict items! */ + Py_ssize_t i = 0; + Py_ssize_t pos = 0; + while (PyDict_Next(kwargs, &pos, &c_kwargs[i], &c_kwargs[i + 1])) { + i += 2; } + c_kwargs_len = i / 2; + /* XXX This is broken if the caller deletes dict items! */ } } - PyObject **c_defaults = NULL; Py_ssize_t c_defaults_len = 0; - - if (defaults && PyTuple_Check(defaults)) { + if (defaults) { c_defaults = &PyTuple_GET_ITEM(defaults, 0); c_defaults_len = PyTuple_Size(defaults); } - if (kw_defaults && !PyDict_Check(kw_defaults)) { - PyErr_SetString(PyExc_TypeError, "kw_defaults must be a dict"); - goto exit; - } - - if (closure && !PyTuple_Check(closure)) { - PyErr_SetString(PyExc_TypeError, "closure must be a tuple of cells"); - goto exit; - } - - result = PyEval_EvalCodeEx( code, globals, @@ -3084,6 +3063,33 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +function_get_closure(PyObject *self, PyObject *func) +{ + PyObject *closure = PyFunction_GetClosure(func); + if (closure != NULL) { + return Py_NewRef(closure); + } else if (PyErr_Occurred()) { + return NULL; + } else { + Py_RETURN_NONE; // This can happen when `closure` is set to `None` + } +} + +static PyObject * +function_set_closure(PyObject *self, PyObject *args) +{ + PyObject *func = NULL, *closure = NULL; + if (!PyArg_ParseTuple(args, "OO", &func, &closure)) { + return NULL; + } + int result = PyFunction_SetClosure(func, closure); + if (result == -1) { + return NULL; + } + Py_RETURN_NONE; +} + static PyObject * check_pyimport_addmodule(PyObject *self, PyObject *args) { @@ -3230,35 +3236,110 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} static PyObject * -sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) { - const char *name; - Py_ssize_t size; - if (!PyArg_Parse(arg, "z#", &name, &size)) { - return NULL; + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + + struct simpletracer_data tracer_data = {0}; + void* the_data = &tracer_data; + // Install a simple tracer function + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + goto failed; } - PyObject *result = PySys_GetObject(name); - if (result == NULL) { - result = PyExc_AttributeError; + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); + (void)PyRefTracer_SetTracer(NULL, NULL); + goto failed; } - return Py_NewRef(result); + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + goto failed; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + goto failed; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + (void)PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + goto failed; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + goto failed; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + goto failed; + } + PyRefTracer_SetTracer(current_tracer, current_data); + Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; } static PyObject * -sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +function_set_warning(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { - const char *name; - Py_ssize_t size; - PyObject *value; - if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, "Testing PyErr_WarnEx", 2)) { return NULL; } - NULLABLE(value); - RETURN_INT(PySys_SetObject(name, value)); + Py_RETURN_NONE; } +static PyObject * +test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) +{ + Py_BEGIN_CRITICAL_SECTION(module); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2(module, module); + Py_END_CRITICAL_SECTION2(); + + Py_RETURN_NONE; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3291,15 +3372,21 @@ static PyMethodDef TestMethods[] = { {"py_buildvalue_ints", py_buildvalue_ints, METH_VARARGS}, {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, - {"test_get_type_name", test_get_type_name, METH_NOARGS}, - {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"get_heaptype_for_name", get_heaptype_for_name, METH_NOARGS}, + {"get_type_name", get_type_name, METH_O}, + {"get_type_qualname", get_type_qualname, METH_O}, + {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, + {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, + {"gilstate_ensure_release", gilstate_ensure_release, METH_NOARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, #endif - {"_pending_threadfunc", pending_threadfunc, METH_VARARGS}, + {"_pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), + METH_VARARGS|METH_KEYWORDS}, #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, #endif @@ -3308,11 +3395,10 @@ static PyMethodDef TestMethods[] = { {"eval_code_ex", eval_eval_code_ex, METH_VARARGS}, {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer, METH_NOARGS}, + {"buffer_fill_info", buffer_fill_info, METH_VARARGS}, {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, - {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3340,7 +3426,7 @@ static PyMethodDef TestMethods[] = { {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, - {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, + {"bad_get", bad_get, METH_VARARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, {"decref_freed_object", decref_freed_object, METH_NOARGS}, @@ -3361,6 +3447,7 @@ static PyMethodDef TestMethods[] = { {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, + {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, @@ -3390,10 +3477,12 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, + {"function_get_closure", function_get_closure, METH_O, NULL}, + {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, - {"sys_getobject", sys_getobject, METH_O}, - {"sys_setobject", sys_setobject, METH_VARARGS}, + {"function_set_warning", function_set_warning, METH_NOARGS}, + {"test_critical_sections", test_critical_sections, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -3508,7 +3597,7 @@ typedef struct { static PyObject * ipowType_ipow(PyObject *self, PyObject *other, PyObject *mod) { - return Py_BuildValue("OO", other, mod); + return PyTuple_Pack(2, other, mod); } static PyNumberMethods ipowType_as_number = { @@ -3869,14 +3958,9 @@ static PyTypeObject ContainerNoGC_type = { static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, - "_testcapi", - NULL, - -1, - TestMethods, - NULL, - NULL, - NULL, - NULL + .m_name = "_testcapi", + .m_size = sizeof(testcapistate_t), + .m_methods = TestMethods, }; /* Per PEP 489, this module will not be converted to multi-phase initialization @@ -3890,9 +3974,14 @@ PyInit__testcapi(void) m = PyModule_Create(&_testcapimodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); - + if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { + return NULL; + } if (PyType_Ready(&matmulType) < 0) return NULL; Py_INCREF(&matmulType); @@ -3964,6 +4053,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_VOID_P", PyLong_FromSsize_t(sizeof(void*))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); + PyModule_AddObject(m, "SIZEOF_PID_T", PyLong_FromSsize_t(sizeof(pid_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); @@ -3971,9 +4061,19 @@ PyInit__testcapi(void) PyModule_AddIntConstant(m, "the_number_three", 3); PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); - TestError = PyErr_NewException("_testcapi.error", NULL, NULL); - Py_INCREF(TestError); - PyModule_AddObject(m, "error", TestError); + if (PyModule_AddIntMacro(m, Py_single_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_file_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_eval_input)) { + return NULL; + } + + testcapistate_t *state = get_testcapi_state(m); + state->error = PyErr_NewException("_testcapi.error", NULL, NULL); + PyModule_AddObject(m, "error", state->error); if (PyType_Ready(&ContainerNoGC_type) < 0) { return NULL; @@ -3993,6 +4093,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Abstract(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Bytes(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4017,12 +4120,24 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Float(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Complex(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Numbers(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Dict(m) < 0) { return NULL; } if (_PyTestCapi_Init_Set(m) < 0) { return NULL; } + if (_PyTestCapi_Init_List(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Tuple(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Structmember(m) < 0) { return NULL; } @@ -4035,7 +4150,10 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Buffer(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(m) < 0) { + if (_PyTestCapi_Init_File(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Codec(m) < 0) { return NULL; } if (_PyTestCapi_Init_Immortal(m) < 0) { @@ -4047,10 +4165,19 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyAtomic(m) < 0) { return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { + if (_PyTestCapi_Init_Run(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Hash(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Time(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Monitoring(m) < 0) { return NULL; } - if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + if (_PyTestCapi_Init_Object(m) < 0) { return NULL; } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 2e0535d52641e0..4187e13231dc69 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1128,6 +1128,26 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) Py_RETURN_NONE; } +/*[clinic input] +null_or_tuple_for_varargs + + name: object + *constraints: object + covariant: bool = False + +See https://github.com/python/cpython/issues/110864 +[clinic start generated code]*/ + +static PyObject * +null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, + PyObject *constraints, int covariant) +/*[clinic end generated code: output=a785b35421358983 input=c9bce186637956b3]*/ +{ + assert(name != NULL); + assert(constraints != NULL); + PyObject *c = covariant ? Py_True : Py_False; + return pack_arguments_newref(3, name, constraints, c); +} /*[clinic input] _testclinic.clone_f1 as clone_f1 @@ -1193,6 +1213,55 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +class _testclinic.TestClass "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ + +/*[clinic input] +_testclinic.TestClass.get_defining_class + cls: defining_class +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=94f9b0b5f7add930 input=537c59417471dee3]*/ +{ + return Py_NewRef(cls); +} + +/*[clinic input] +_testclinic.TestClass.get_defining_class_arg + cls: defining_class + arg: object +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg) +/*[clinic end generated code: output=fe7e49d96cbb7718 input=d1b83d3b853af6d9]*/ +{ + return PyTuple_Pack(2, cls, arg); +} + +static struct PyMethodDef test_class_methods[] = { + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF + {NULL, NULL} +}; + +static PyTypeObject TestClass = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.TestClass", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_methods = test_class_methods, +}; + + /*[clinic input] output push destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' @@ -1843,6 +1912,7 @@ static PyMethodDef tester_methods[] = { GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF + NULL_OR_TUPLE_FOR_VARARGS_METHODDEF CLONE_F1_METHODDEF CLONE_F2_METHODDEF CLONE_WITH_CONV_F1_METHODDEF @@ -1885,6 +1955,12 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + if (PyModule_AddType(m, &TestClass) < 0) { + goto error; + } if (PyModule_AddType(m, &DeprStarNew) < 0) { goto error; } diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 4273383816a0dd..370433b3e2a0d9 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,8 +4,11 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN -// For now, only limited C API 3.13 is supported -#define Py_LIMITED_API 0x030d0000 +// For now, AC only supports the limited C API version 3.13 +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif /* Always enable assertions */ #undef NDEBUG @@ -68,10 +71,64 @@ my_int_sum_impl(PyObject *module, int x, int y) } +/*[clinic input] +my_float_sum -> float + + x: float + y: float + / + +[clinic start generated code]*/ + +static float +my_float_sum_impl(PyObject *module, float x, float y) +/*[clinic end generated code: output=634f59a5a419cad7 input=d4b5313bdf4dc377]*/ +{ + return x + y; +} + + +/*[clinic input] +my_double_sum -> double + + x: double + y: double + / + +[clinic start generated code]*/ + +static double +my_double_sum_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=a75576d9e4d8557f input=16b11c8aba172801]*/ +{ + return x + y; +} + + +/*[clinic input] +get_file_descriptor -> int + + file as fd: fildes + / + +Get a file descriptor. +[clinic start generated code]*/ + +static int +get_file_descriptor_impl(PyObject *module, int fd) +/*[clinic end generated code: output=80051ebad54db8a8 input=82e2a1418848cd5b]*/ +{ + return fd; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF + MY_FLOAT_SUM_METHODDEF + MY_DOUBLE_SUM_METHODDEF + GET_FILE_DESCRIPTOR_METHODDEF {NULL, NULL} }; @@ -89,5 +146,8 @@ PyInit__testclinic_limited(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c new file mode 100644 index 00000000000000..2a665affb5e7f8 --- /dev/null +++ b/Modules/_testexternalinspection.c @@ -0,0 +1,642 @@ +#define _GNU_SOURCE + +#ifdef __linux__ +# include +# include +# if INTPTR_MAX == INT64_MAX +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Phdr Elf64_Phdr +# else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Phdr Elf32_Phdr +# endif +# include +#endif + +#if defined(__APPLE__) +# include +// Older macOS SDKs do not define TARGET_OS_OSX +# if !defined(TARGET_OS_OSX) +# define TARGET_OS_OSX 1 +# endif +# if TARGET_OS_OSX +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" +#include + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#if defined(__APPLE__) && TARGET_OS_OSX +static void* +analyze_macho64(mach_port_t proc_ref, void* base, void* map) +{ + struct mach_header_64* hdr = (struct mach_header_64*)map; + int ncmds = hdr->ncmds; + + int cmd_cnt = 0; + struct segment_command_64* cmd = map + sizeof(struct mach_header_64); + + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + mach_vm_address_t address = (mach_vm_address_t)base; + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { + while (cmd->filesize != size) { + address += size; + if (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] + &count, + &object_name) + != KERN_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return NULL; + } + } + base = (void*)address - cmd->vmaddr; + + int nsects = cmd->nsects; + struct section_64* sec = + (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + for (int j = 0; j < nsects; j++) { + if (strcmp(sec[j].sectname, "PyRuntime") == 0) { + return base + sec[j].addr; + } + } + cmd_cnt++; + } + + cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); + } + return NULL; +} + +static void* +analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +{ + int fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + return NULL; + } + + struct stat fs; + if (fstat(fd, &fs) == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + close(fd); + return NULL; + } + + void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + close(fd); + return NULL; + } + + void* result = NULL; + + struct mach_header_64* hdr = (struct mach_header_64*)map; + switch (hdr->magic) { + case MH_MAGIC: + case MH_CIGAM: + case FAT_MAGIC: + case FAT_CIGAM: + PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + break; + case MH_MAGIC_64: + case MH_CIGAM_64: + result = analyze_macho64(proc_ref, base, map); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + break; + } + + munmap(map, fs.st_size); + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + return result; +} + +static mach_port_t +pid_to_task(pid_t pid) +{ + mach_port_t task; + kern_return_t result; + + result = task_for_pid(mach_task_self(), pid, &task); + if (result != KERN_SUCCESS) { + PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + return 0; + } + return task; +} + +static void* +get_py_runtime_macos(pid_t pid) +{ + mach_vm_address_t address = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + mach_port_t proc_ref = pid_to_task(pid); + if (proc_ref == 0) { + PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + return NULL; + } + + int match_found = 0; + char map_filename[MAXPATHLEN + 1]; + void* result_address = NULL; + while (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, + &count, + &object_name) + == KERN_SUCCESS) + { + int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + if (path_len == 0) { + address += size; + continue; + } + + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + break; + } + + address += size; + } + return result_address; +} +#endif + +#ifdef __linux__ +void* +find_python_map_start_address(pid_t pid, char* result_filename) +{ + char maps_file_path[64]; + sprintf(maps_file_path, "/proc/%d/maps", pid); + + FILE* maps_file = fopen(maps_file_path, "r"); + if (maps_file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + int match_found = 0; + + char line[256]; + char map_filename[PATH_MAX]; + void* result_address = 0; + while (fgets(line, sizeof(line), maps_file) != NULL) { + unsigned long start_address = 0; + sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + break; + } + } + + fclose(maps_file); + + if (!match_found) { + map_filename[0] = '\0'; + } + + return result_address; +} + +void* +get_py_runtime_linux(pid_t pid) +{ + char elf_file[256]; + void* start_address = (void*)find_python_map_start_address(pid, elf_file); + + if (start_address == 0) { + PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); + return NULL; + } + + void* result = NULL; + void* file_memory = NULL; + + int fd = open(elf_file, O_RDONLY); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + struct stat file_stats; + if (fstat(fd, &file_stats) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (file_memory == MAP_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + + Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; + char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); + + Elf_Shdr* py_runtime_section = NULL; + for (int i = 0; i < elf_header->e_shnum; i++) { + if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { + py_runtime_section = §ion_header_table[i]; + break; + } + } + + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment + Elf_Phdr* first_load_segment = NULL; + for (int i = 0; i < elf_header->e_phnum; i++) { + if (program_header_table[i].p_type == PT_LOAD) { + first_load_segment = &program_header_table[i]; + break; + } + } + + if (py_runtime_section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + py_runtime_section->sh_addr - elf_load_addr; + } + +exit: + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + if (file_memory != NULL) { + munmap(file_memory, file_stats.st_size); + } + return result; +} +#endif + +ssize_t +read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +{ + ssize_t total_bytes_read = 0; +#if defined(__linux__) && HAVE_PROCESS_VM_READV + struct iovec local[1]; + struct iovec remote[1]; + ssize_t result = 0; + ssize_t read = 0; + + do { + local[0].iov_base = dst + result; + local[0].iov_len = len - result; + remote[0].iov_base = (void*)(remote_address + result); + remote[0].iov_len = len - result; + + read = process_vm_readv(pid, local, 1, remote, 1, 0); + if (read < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += read; + } while ((size_t)read != local[0].iov_len); + total_bytes_read = result; +#elif defined(__APPLE__) && TARGET_OS_OSX + ssize_t result = -1; + kern_return_t kr = mach_vm_read_overwrite( + pid_to_task(pid), + (mach_vm_address_t)remote_address, + len, + (mach_vm_address_t)dst, + (mach_vm_size_t*)&result); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + } + return -1; + } + total_bytes_read = len; +#else + return -1; +#endif + return total_bytes_read; +} + +int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +{ + Py_ssize_t len; + ssize_t bytes_read = + read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + if (bytes_read == -1) { + return -1; + } + if (len >= size) { + PyErr_SetString(PyExc_RuntimeError, "Buffer too small"); + return -1; + } + size_t offset = debug_offsets->unicode_object.asciiobject_size; + bytes_read = read_memory(pid, address + offset, len, buffer); + if (bytes_read == -1) { + return -1; + } + buffer[len] = '\0'; + return 0; +} + +void* +get_py_runtime(pid_t pid) +{ +#if defined(__linux__) + return get_py_runtime_linux(pid); +#elif defined(__APPLE__) && TARGET_OS_OSX + return get_py_runtime_macos(pid); +#else + return NULL; +#endif +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + void* address_of_function_name; + read_memory( + pid, + (void*)(address + offsets->code_object.name), + sizeof(void*), + &address_of_function_name); + + if (address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + char function_name[256]; + if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + return -1; + } + + PyObject* py_function_name = PyUnicode_FromString(function_name); + if (py_function_name == NULL) { + return -1; + } + + if (PyList_Append(result, py_function_name) == -1) { + Py_DECREF(py_function_name); + return -1; + } + Py_DECREF(py_function_name); + + return 0; +} + +static int +parse_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + ssize_t bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.previous), + sizeof(void*), + previous_frame); + if (bytes_read == -1) { + return -1; + } + + char owner; + bytes_read = + read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + void* address_of_code_object; + bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.executable), + sizeof(void*), + &address_of_code_object); + if (bytes_read == -1) { + return -1; + } + + if (address_of_code_object == NULL) { + return 0; + } + return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); +} + +static PyObject* +get_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + + void* runtime_start_address = get_py_runtime(pid); + if (runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return NULL; + } + size_t size = sizeof(struct _Py_DebugOffsets); + struct _Py_DebugOffsets local_debug_offsets; + + ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); + if (bytes_read == -1) { + return NULL; + } + off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + + void* address_of_interpreter_state; + bytes_read = read_memory( + pid, + (void*)(runtime_start_address + interpreter_state_list_head), + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return NULL; + } + + if (address_of_interpreter_state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return NULL; + } + + void* address_of_thread; + bytes_read = read_memory( + pid, + (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return NULL; + } + + PyObject* result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // No Python frames are available for us (can happen at tear-down). + if (address_of_thread != NULL) { + void* address_of_current_frame; + (void)read_memory( + pid, + (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), + sizeof(void*), + &address_of_current_frame); + while (address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + } + + return result; +} + +static PyMethodDef methods[] = { + {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC +PyInit__testexternalinspection(void) +{ + PyObject* mod = PyModule_Create(&module); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 1caeb66eb8c858..c147596f88a3a8 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -3,15 +3,27 @@ * file (issue16421). This file defines 3 modules (_testimportmodule, * foo, bar), only the first one is called the same as the compiled file. */ -#include + +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +#include + +static PyModuleDef_Slot shared_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; static struct PyModuleDef _testimportmultiple = { PyModuleDef_HEAD_INIT, "_testimportmultiple", "_testimportmultiple doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -19,16 +31,16 @@ static struct PyModuleDef _testimportmultiple = { PyMODINIT_FUNC PyInit__testimportmultiple(void) { - return PyModule_Create(&_testimportmultiple); + return PyModuleDef_Init(&_testimportmultiple); } static struct PyModuleDef _foomodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_foo", "_testimportmultiple_foo doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -36,22 +48,21 @@ static struct PyModuleDef _foomodule = { PyMODINIT_FUNC PyInit__testimportmultiple_foo(void) { - return PyModule_Create(&_foomodule); + return PyModuleDef_Init(&_foomodule); } static struct PyModuleDef _barmodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_bar", "_testimportmultiple_bar doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__testimportmultiple_bar(void){ - return PyModule_Create(&_barmodule); + return PyModuleDef_Init(&_barmodule); } - diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ddeb38938a331f..6e6386bc044dc3 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,26 +10,29 @@ #undef NDEBUG #include "Python.h" +#include "pycore_backoff.h" // JUMP_BACKWARD_INITIAL_VALUE #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() -#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() +#include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "interpreteridobject.h" // PyInterpreterID_LookUp() - #include "clinic/_testinternalcapi.c.h" // Include test definitions from _testinternalcapi/ @@ -109,6 +112,14 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject* +get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return PyLong_FromLong(tstate->c_recursion_remaining); +} + + static PyObject* test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -714,6 +725,19 @@ _testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) return _PyCompile_CleanDoc(doc); } +/*[clinic input] + +_testinternalcapi.new_instruction_sequence -> object + +Return a new, empty InstructionSequence. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module) +/*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ +{ + return _PyInstructionSequence_New(); +} /*[clinic input] @@ -822,6 +846,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -951,15 +976,28 @@ iframe_getlasti(PyObject *self, PyObject *frame) } static PyObject * -get_counter_optimizer(PyObject *self, PyObject *arg) +get_co_framesize(PyObject *self, PyObject *arg) { - return PyUnstable_Optimizer_NewCounter(); + if (!PyCode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)arg; + return PyLong_FromLong(code->co_framesize); } +#ifdef _Py_TIER2 + static PyObject * -get_uop_optimizer(PyObject *self, PyObject *arg) +new_counter_optimizer(PyObject *self, PyObject *arg) { - return PyUnstable_Optimizer_NewUOpOptimizer(); + return _PyOptimizer_NewCounter(); +} + +static PyObject * +new_uop_optimizer(PyObject *self, PyObject *arg) +{ + return _PyOptimizer_NewUOpOptimizer(); } static PyObject * @@ -968,14 +1006,19 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - PyUnstable_SetOptimizer((_PyOptimizerObject*)opt); + if (_Py_SetTier2Optimizer((_PyOptimizerObject*)opt) < 0) { + return NULL; + } Py_RETURN_NONE; } static PyObject * get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *opt = (PyObject *)PyUnstable_GetOptimizer(); + PyObject *opt = NULL; +#ifdef _Py_TIER2 + opt = (PyObject *)_Py_GetOptimizer(); +#endif if (opt == NULL) { Py_RETURN_NONE; } @@ -983,25 +1026,33 @@ get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -get_executor(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +add_executor_dependency(PyObject *self, PyObject *args) { - - if (!_PyArg_CheckPositional("get_executor", nargs, 2, 2)) { + PyObject *exec; + PyObject *obj; + if (!PyArg_ParseTuple(args, "OO", &exec, &obj)) { return NULL; } - PyObject *code = args[0]; - PyObject *offset = args[1]; - long ioffset = PyLong_AsLong(offset); - if (ioffset == -1 && PyErr_Occurred()) { + /* No way to tell in general if exec is an executor, so we only accept + * counting_executor */ + if (strcmp(Py_TYPE(exec)->tp_name, "counting_executor")) { + PyErr_SetString(PyExc_TypeError, "argument must be a counting_executor"); return NULL; } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); - return NULL; - } - return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, ioffset); + _Py_Executor_DependsOn((_PyExecutorObject *)exec, obj); + Py_RETURN_NONE; +} + +static PyObject * +invalidate_executors(PyObject *self, PyObject *obj) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _Py_Executors_InvalidateDependency(interp, obj, 1); + Py_RETURN_NONE; } +#endif + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -1019,37 +1070,56 @@ static PyObject * pending_threadfunc(PyObject *self, PyObject *args, PyObject *kwargs) { PyObject *callable; + unsigned int num = 1; + int blocking = 0; int ensure_added = 0; - static char *kwlist[] = {"", "ensure_added", NULL}; + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O|$p:pending_threadfunc", kwlist, - &callable, &ensure_added)) + "O|I$pp:pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - int r; - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - if (r < 0) { - /* unsuccessful add */ - if (!ensure_added) { - Py_DECREF(callable); - Py_RETURN_FALSE; + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + _Py_add_pending_call_result r; + do { + r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); + } + else { + if (_PyEval_AddPendingCall(interp, &_pending_callback, callable, 0) < 0) { + break; + } } - do { - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - } while (r < 0); } - Py_RETURN_TRUE; + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } + + for (unsigned int i = num_added; i < num; i++) { + Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ + } + + /* The callable is decref'ed in _pending_callback() above. */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } @@ -1075,7 +1145,7 @@ pending_identify(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) { return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(interpid); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(interpid); if (interp == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, "interpreter not found"); @@ -1092,14 +1162,16 @@ pending_identify(PyObject *self, PyObject *args) PyThread_acquire_lock(mutex, WAIT_LOCK); /* It gets released in _pending_identify_callback(). */ - int r; + _Py_add_pending_call_result r; do { Py_BEGIN_ALLOW_THREADS r = _PyEval_AddPendingCall(interp, &_pending_identify_callback, (void *)mutex, 0); Py_END_ALLOW_THREADS - } while (r < 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); /* Wait for the pending call to complete. */ PyThread_acquire_lock(mutex, WAIT_LOCK); @@ -1250,8 +1322,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER + /* ASan or TSan would report an use-after-free error */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); @@ -1282,14 +1354,13 @@ static PyObject * get_object_dict_values(PyObject *self, PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { Py_RETURN_NONE; } - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (!_PyDictOrValues_IsValues(dorv)) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (!values->valid) { Py_RETURN_NONE; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(keys != NULL); int size = (int)keys->dk_nentries; @@ -1341,135 +1412,428 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -/* To run some code in a sub-interpreter. */ -static PyObject * -run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +static int +_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj) { - const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; + if (obj == NULL) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + return 0; + } - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { + PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", obj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + return 0; +} + +static PyInterpreterState * +_new_interpreter(PyInterpreterConfig *config, long whence) +{ + if (whence == _PyInterpreterState_WHENCE_XI) { + return _PyXI_NewInterpreter(config, &whence, NULL, NULL); + } + PyObject *exc = NULL; + PyInterpreterState *interp = NULL; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { + assert(config == NULL); + interp = PyInterpreterState_New(); + } + else if (whence == _PyInterpreterState_WHENCE_CAPI + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + PyThreadState *tstate = NULL; + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) { + assert(config == NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + } + else { + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + assert(tstate == NULL); + _PyErr_SetFromPyStatus(status); + exc = PyErr_GetRaisedException(); + } + } + if (tstate != NULL) { + interp = PyThreadState_GetInterpreter(tstate); + // Throw away the initial tstate. + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported whence %ld", whence); return NULL; } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); + + if (interp == NULL) { + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } + } + return interp; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.create() +static PyObject * +create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"config", "whence", NULL}; + PyObject *configobj = NULL; + long whence = _PyInterpreterState_WHENCE_XI; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|O$l:create_interpreter", kwlist, + &configobj, &whence)) + { return NULL; } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); + if (configobj == Py_None) { + configobj = NULL; + } + + // Resolve the config. + PyInterpreterConfig *config = NULL; + PyInterpreterConfig _config; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + if (configobj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected config"); + return NULL; + } + } + else { + config = &_config; + if (_init_interp_config_from_object(config, configobj) < 0) { + return NULL; + } + } + + // Create the interpreter. + PyInterpreterState *interp = _new_interpreter(config, whence); + if (interp == NULL) { return NULL; } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); + + // Return the ID. + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); return NULL; } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); + + return idobj; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O:destroy_interpreter", kwlist, + &idobj)) + { return NULL; } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + _PyXI_EndInterpreter(interp, NULL, NULL); + Py_RETURN_NONE; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", "code", "main", NULL}; + PyObject *idobj; + const char *code; + int runningmain = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "Os|$p:exec_interpreter", kwlist, + &idobj, &code, &runningmain)) + { return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - mainstate = PyThreadState_Get(); + PyObject *res = NULL; + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); - PyThreadState_Swap(NULL); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; - PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - PyThreadState_Swap(mainstate); - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); - _PyErr_ChainExceptions1(exc); - return NULL; + if (runningmain) { + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + goto finally; + } } - assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); - Py_EndInterpreter(substate); - PyThreadState_Swap(mainstate); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } - return PyLong_FromLong(r); + if (runningmain) { + _PyInterpreterState_SetNotRunningMain(interp); + } + + res = PyLong_FromLong(r); + +finally: + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + return res; } -/*[clinic input] -_testinternalcapi.write_unraisable_exc - exception as exc: object - err_msg: object - obj: object - / -[clinic start generated code]*/ +/* To run some code in a sub-interpreter. +Generally you can use test.support.interpreters, +but we keep this helper as a distinct implementation. +That's especially important for testing test.support.interpreters. +*/ static PyObject * -_testinternalcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, - PyObject *err_msg, PyObject *obj) -/*[clinic end generated code: output=a0f063cdd04aad83 input=274381b1a3fa5cd6]*/ +run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { + const char *code; + PyObject *configobj; + int xi = 0; + static char *kwlist[] = {"code", "config", "xi", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "sO|$p:run_in_subinterp_with_config", kwlist, + &code, &configobj, &xi)) + { + return NULL; + } + + PyInterpreterConfig config; + if (_init_interp_config_from_object(&config, configobj) < 0) { + return NULL; + } - const char *err_msg_utf8; - if (err_msg != Py_None) { - err_msg_utf8 = PyUnicode_AsUTF8(err_msg); - if (err_msg_utf8 == NULL) { + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + + int r; + if (xi) { + PyThreadState *save_tstate; + PyThreadState *tstate; + + /* Create an interpreter, staying switched to it. */ + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, NULL, &tstate, &save_tstate); + if (interp == NULL) { return NULL; } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + _PyXI_EndInterpreter(interp, tstate, &save_tstate); } else { - err_msg_utf8 = NULL; + PyThreadState *substate; + PyThreadState *mainstate = PyThreadState_Swap(NULL); + + /* Create an interpreter, staying switched to it. */ + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyThreadState_Swap(mainstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + Py_EndInterpreter(substate); + PyThreadState_Swap(mainstate); } - PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); - _PyErr_WriteUnraisableMsg(err_msg_utf8, obj); + return PyLong_FromLong(r); +} + + +static PyObject * +normalize_interp_id(PyObject *self, PyObject *idobj) +{ + int64_t interpid = _PyInterpreterState_ObjectToID(idobj); + if (interpid < 0) { + return NULL; + } + return PyLong_FromLongLong(interpid); +} + +static PyObject * +next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = _PyRuntime.interpreters.next_id; + return PyLong_FromLongLong(interpid); +} + +static PyObject * +unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = INT64_MAX; + assert(interpid > _PyRuntime.interpreters.next_id); + return PyLong_FromLongLong(interpid); +} + +static PyObject * +interpreter_exists(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + if (PyErr_ExceptionMatches(PyExc_InterpreterNotFoundError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + assert(PyErr_Occurred()); + return NULL; + } + Py_RETURN_TRUE; +} + +static PyObject * +get_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + return PyLong_FromLongLong(interp->id_refcount); +} + +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); Py_RETURN_NONE; } +static PyObject * +interpreter_refcount_linked(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_RequiresIDRef(interp)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + + +static void +_xid_capsule_destructor(PyObject *capsule) +{ + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data != NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } +} + +static PyObject * +get_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *obj = NULL; + if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { + return NULL; + } + + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + return NULL; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _PyCrossInterpreterData_Free(data); + return NULL; + } + PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); + if (capsule == NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } + return capsule; +} + +static PyObject * +restore_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *capsule = NULL; + if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { + return NULL; + } + + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data == NULL) { + return NULL; + } + return _PyCrossInterpreterData_NewObject(data); +} + static PyObject * raiseTestError(const char* test_name, const char* msg) @@ -1530,10 +1894,151 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) Py_RETURN_NONE; } +static PyObject * +compile_perf_trampoline_entry(PyObject *self, PyObject *args) +{ + PyObject *co; + if (!PyArg_ParseTuple(args, "O!", &PyCode_Type, &co)) { + return NULL; + } + int ret = PyUnstable_PerfTrampoline_CompileCode((PyCodeObject *)co); + if (ret != 0) { + PyErr_SetString(PyExc_AssertionError, "Failed to compile trampoline"); + return NULL; + } + return PyLong_FromLong(ret); +} + +static PyObject * +perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args) +{ + int enable; + if (!PyArg_ParseTuple(args, "i", &enable)) { + return NULL; + } + int ret = PyUnstable_PerfTrampoline_SetPersistAfterFork(enable); + if (ret == 0) { + PyErr_SetString(PyExc_AssertionError, "Failed to set persist_after_fork"); + return NULL; + } + return PyLong_FromLong(ret); +} + + +static PyObject * +get_rare_event_counters(PyObject *self, PyObject *type) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + return Py_BuildValue( + "{sksksksksk}", + "set_class", (unsigned long)interp->rare_events.set_class, + "set_bases", (unsigned long)interp->rare_events.set_bases, + "set_eval_frame_func", (unsigned long)interp->rare_events.set_eval_frame_func, + "builtin_dict", (unsigned long)interp->rare_events.builtin_dict, + "func_modification", (unsigned long)interp->rare_events.func_modification + ); +} + +static PyObject * +reset_rare_event_counters(PyObject *self, PyObject *Py_UNUSED(type)) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + interp->rare_events.set_class = 0; + interp->rare_events.set_bases = 0; + interp->rare_events.set_eval_frame_func = 0; + interp->rare_events.builtin_dict = 0; + interp->rare_events.func_modification = 0; + + return Py_None; +} + + +#ifdef Py_GIL_DISABLED +static PyObject * +get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + uintptr_t tid = _Py_ThreadId(); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(tid)); + return PyLong_FromUnsignedLongLong(tid); +} +#endif + +static PyObject * +suppress_immortalization(PyObject *self, PyObject *value) +{ +#ifdef Py_GIL_DISABLED + int suppress = PyObject_IsTrue(value); + if (suppress < 0) { + return NULL; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + // Subtract two to suppress immortalization (so that 1 -> -1) + _Py_atomic_add_int(&interp->gc.immortalize, suppress ? -2 : 2); +#endif + Py_RETURN_NONE; +} + +static PyObject * +get_immortalize_deferred(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = PyInterpreterState_Get(); + return PyBool_FromLong(_Py_atomic_load_int(&interp->gc.immortalize) >= 0); +#else + Py_RETURN_FALSE; +#endif +} + +static PyObject * +has_inline_values(PyObject *self, PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + + +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + _PyType_SetVersion(type, version); + Py_RETURN_NONE; +} + +/*[clinic input] +gh_119213_getargs + + spam: object = None + +Test _PyArg_Parser.kwtuple +[clinic start generated code]*/ + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam) +/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/ +{ + // It must never have been called in the main interprer + assert(!_Py_IsMainInterpreter(PyInterpreterState_Get())); + return Py_NewRef(spam); +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, + {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS}, {"test_bswap", test_bswap, METH_NOARGS}, {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, @@ -1550,6 +2055,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF + _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF @@ -1560,11 +2066,15 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"get_co_framesize", get_co_framesize, METH_O, NULL}, +#ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, - {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, - {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, + {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, + {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, + {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, + {"invalidate_executors", invalidate_executors, METH_O, NULL}, +#endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, @@ -1582,11 +2092,43 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"create_interpreter", _PyCFunction_CAST(create_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"exec_interpreter", _PyCFunction_CAST(exec_interpreter), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, - _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF + {"normalize_interp_id", normalize_interp_id, METH_O}, + {"next_interpreter_id", next_interpreter_id, METH_NOARGS}, + {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"interpreter_exists", interpreter_exists, METH_O}, + {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, + {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, + {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, + {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, + {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, + {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF + {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, + {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, + {"has_inline_values", has_inline_values, METH_O}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, + +#ifdef Py_GIL_DISABLED + {"py_thread_id", get_py_thread_id, METH_NOARGS}, +#endif + {"suppress_immortalization", suppress_immortalization, METH_O}, + {"get_immortalize_deferred", get_immortalize_deferred, METH_NOARGS}, +#ifdef _Py_TIER2 + {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, +#endif + GH_119213_GETARGS_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1605,9 +2147,22 @@ module_exec(PyObject *module) if (_PyTestInternalCapi_Init_Set(module) < 0) { return 1; } + if (_PyTestInternalCapi_Init_CriticalSection(module) < 0) { + return 1; + } + + Py_ssize_t sizeof_gc_head = 0; +#ifndef Py_GIL_DISABLED + sizeof_gc_head = sizeof(PyGC_Head); +#endif if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", - PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { + PyLong_FromSsize_t(sizeof_gc_head)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SIZEOF_MANAGED_PRE_HEADER", + PyLong_FromSsize_t(2 * sizeof(PyObject*))) < 0) { return 1; } @@ -1621,12 +2176,18 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "TIER2_THRESHOLD", + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE)) < 0) { + return 1; + } + return 0; } static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testinternalcapi/clinic/test_lock.c.h b/Modules/_testinternalcapi/clinic/test_lock.c.h index 3cbe5ef12c5fa6..86875767343cd2 100644 --- a/Modules/_testinternalcapi/clinic/test_lock.c.h +++ b/Modules/_testinternalcapi/clinic/test_lock.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_testinternalcapi_benchmark_locks__doc__, "benchmark_locks($module, num_threads, use_pymutex=True,\n" @@ -71,4 +72,4 @@ _testinternalcapi_benchmark_locks(PyObject *module, PyObject *const *args, Py_ss exit: return return_value; } -/*[clinic end generated code: output=97c85dff601fed4b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=105105d759c0c271 input=a9049054013a1b77]*/ diff --git a/Modules/_testinternalcapi/parts.h b/Modules/_testinternalcapi/parts.h index 3d2774e3f1b404..49a1395f499fc3 100644 --- a/Modules/_testinternalcapi/parts.h +++ b/Modules/_testinternalcapi/parts.h @@ -13,5 +13,6 @@ int _PyTestInternalCapi_Init_Lock(PyObject *module); int _PyTestInternalCapi_Init_PyTime(PyObject *module); int _PyTestInternalCapi_Init_Set(PyObject *module); +int _PyTestInternalCapi_Init_CriticalSection(PyObject *module); #endif // Py_TESTINTERNALCAPI_PARTS_H diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b5f9eb0ef2851..2b0a205d158a96 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -2,10 +2,10 @@ #include "parts.h" -#include "pycore_time.h" +#include "pycore_time.h" // _PyTime_FromSeconds() #ifdef MS_WINDOWS -# include // struct timeval +# include // struct timeval #endif @@ -16,8 +16,8 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "i", &seconds)) { return NULL; } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); + PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsLong(ts); } static int @@ -45,26 +45,11 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ts; + PyTime_t ts; if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsNanosecondsObject(ts); -} - -static PyObject * -test_pytime_assecondsdouble(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { - return NULL; - } - double d = _PyTime_AsSecondsDouble(ts); - return PyFloat_FromDouble(d); + return _PyTime_AsLong(ts); } static PyObject * @@ -78,8 +63,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -105,8 +90,8 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -127,8 +112,8 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -145,8 +130,8 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -163,16 +148,15 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t ms = _PyTime_AsMilliseconds(t, round); + return _PyTime_AsLong(ms); } static PyObject * @@ -183,16 +167,15 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t us = _PyTime_AsMicroseconds(t, round); + return _PyTime_AsLong(us); } static PyObject * @@ -254,7 +237,6 @@ test_pytime_object_to_timespec(PyObject *self, PyObject *args) static PyMethodDef TestMethods[] = { {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, - {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, #ifdef HAVE_CLOCK_GETTIME {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, diff --git a/Modules/_testinternalcapi/set.c b/Modules/_testinternalcapi/set.c index 0305a7885d217c..01aab03cc109ed 100644 --- a/Modules/_testinternalcapi/set.c +++ b/Modules/_testinternalcapi/set.c @@ -1,6 +1,7 @@ #include "parts.h" #include "../_testcapi/util.h" // NULLABLE, RETURN_INT +#include "pycore_critical_section.h" #include "pycore_setobject.h" @@ -27,10 +28,13 @@ set_next_entry(PyObject *self, PyObject *args) return NULL; } NULLABLE(set); - - rc = _PySet_NextEntry(set, &pos, &item, &hash); + Py_BEGIN_CRITICAL_SECTION(set); + rc = _PySet_NextEntryRef(set, &pos, &item, &hash); + Py_END_CRITICAL_SECTION(); if (rc == 1) { - return Py_BuildValue("innO", rc, pos, hash, item); + PyObject *ret = Py_BuildValue("innO", rc, pos, hash, item); + Py_DECREF(item); + return ret; } assert(item == UNINITIALIZED_PTR); assert(hash == (Py_hash_t)UNINITIALIZED_SIZE); diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c new file mode 100644 index 00000000000000..1df960f9881f70 --- /dev/null +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -0,0 +1,302 @@ +/* + * C Extension module to test pycore_critical_section.h API. + */ + +#include "parts.h" + +#include "pycore_critical_section.h" + +#ifdef Py_GIL_DISABLED +#define assert_nogil assert +#define assert_gil(x) +#else +#define assert_gil assert +#define assert_nogil(x) +#endif + + +static PyObject * +test_critical_sections(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *d1 = PyDict_New(); + assert(d1 != NULL); + + PyObject *d2 = PyDict_New(); + assert(d2 != NULL); + + // Beginning a critical section should lock the associated object and + // push the critical section onto the thread's stack (in Py_GIL_DISABLED builds). + Py_BEGIN_CRITICAL_SECTION(d1); + assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); + assert_nogil(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert_gil(PyThreadState_GET()->critical_section == 0); + Py_END_CRITICAL_SECTION(); + assert_nogil(!PyMutex_IsLocked(&d1->ob_mutex)); + + assert_nogil(!PyMutex_IsLocked(&d1->ob_mutex)); + assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); + Py_BEGIN_CRITICAL_SECTION2(d1, d2); + assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); + assert_nogil(PyMutex_IsLocked(&d2->ob_mutex)); + Py_END_CRITICAL_SECTION2(); + assert_nogil(!PyMutex_IsLocked(&d1->ob_mutex)); + assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); + + // Passing the same object twice should work (and not deadlock). + assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); + Py_BEGIN_CRITICAL_SECTION2(d2, d2); + assert_nogil(PyMutex_IsLocked(&d2->ob_mutex)); + Py_END_CRITICAL_SECTION2(); + assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); + + Py_DECREF(d2); + Py_DECREF(d1); + Py_RETURN_NONE; +} + +static void +lock_unlock_object(PyObject *obj, int recurse_depth) +{ + Py_BEGIN_CRITICAL_SECTION(obj); + if (recurse_depth > 0) { + lock_unlock_object(obj, recurse_depth - 1); + } + Py_END_CRITICAL_SECTION(); +} + +static void +lock_unlock_two_objects(PyObject *a, PyObject *b, int recurse_depth) +{ + Py_BEGIN_CRITICAL_SECTION2(a, b); + if (recurse_depth > 0) { + lock_unlock_two_objects(a, b, recurse_depth - 1); + } + Py_END_CRITICAL_SECTION2(); +} + + +// Test that nested critical sections do not deadlock if they attempt to lock +// the same object. +static PyObject * +test_critical_sections_nest(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + // Locking an object recursively with this API should not deadlock. + assert_nogil(!PyMutex_IsLocked(&a->ob_mutex)); + Py_BEGIN_CRITICAL_SECTION(a); + assert_nogil(PyMutex_IsLocked(&a->ob_mutex)); + lock_unlock_object(a, 10); + assert_nogil(PyMutex_IsLocked(&a->ob_mutex)); + Py_END_CRITICAL_SECTION(); + assert_nogil(!PyMutex_IsLocked(&a->ob_mutex)); + + // Same test but with two objects. + Py_BEGIN_CRITICAL_SECTION2(b, a); + lock_unlock_two_objects(a, b, 10); + assert_nogil(PyMutex_IsLocked(&a->ob_mutex)); + assert_nogil(PyMutex_IsLocked(&b->ob_mutex)); + Py_END_CRITICAL_SECTION2(); + + Py_DECREF(b); + Py_DECREF(a); + Py_RETURN_NONE; +} + +// Test that a critical section is suspended by a Py_BEGIN_ALLOW_THREADS and +// resumed by a Py_END_ALLOW_THREADS. +static PyObject * +test_critical_sections_suspend(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + Py_BEGIN_CRITICAL_SECTION(a); + assert_nogil(PyMutex_IsLocked(&a->ob_mutex)); + + // Py_BEGIN_ALLOW_THREADS should suspend the active critical section + Py_BEGIN_ALLOW_THREADS + assert_nogil(!PyMutex_IsLocked(&a->ob_mutex)); + Py_END_ALLOW_THREADS; + + // After Py_END_ALLOW_THREADS the critical section should be resumed. + assert_nogil(PyMutex_IsLocked(&a->ob_mutex)); + Py_END_CRITICAL_SECTION(); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +#ifdef Py_CAN_START_THREADS +struct test_data { + PyObject *obj1; + PyObject *obj2; + PyObject *obj3; + Py_ssize_t countdown; + PyEvent done_event; +}; + +static void +thread_critical_sections(void *arg) +{ + const Py_ssize_t NUM_ITERS = 200; + struct test_data *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + for (Py_ssize_t i = 0; i < NUM_ITERS; i++) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj1); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION(test_data->obj2); + lock_unlock_object(test_data->obj1, 1); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2(test_data->obj3, test_data->obj1); + lock_unlock_object(test_data->obj2, 2); + Py_END_CRITICAL_SECTION2(); + + Py_BEGIN_CRITICAL_SECTION(test_data->obj3); + Py_BEGIN_ALLOW_THREADS + Py_END_ALLOW_THREADS + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) +{ + const Py_ssize_t NUM_THREADS = 4; + struct test_data test_data = { + .obj1 = PyDict_New(), + .obj2 = PyDict_New(), + .obj3 = PyDict_New(), + .countdown = NUM_THREADS, + }; + assert(test_data.obj1 != NULL); + assert(test_data.obj2 != NULL); + assert(test_data.obj3 != NULL); + + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_critical_sections, &test_data); + } + PyEvent_Wait(&test_data.done_event); + + Py_DECREF(test_data.obj3); + Py_DECREF(test_data.obj2); + Py_DECREF(test_data.obj1); + Py_RETURN_NONE; +} + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + +#endif + +static PyMethodDef test_methods[] = { + {"test_critical_sections", test_critical_sections, METH_NOARGS}, + {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, + {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_CAN_START_THREADS + {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, +#endif + {NULL, NULL} /* sentinel */ +}; + +int +_PyTestInternalCapi_Init_CriticalSection(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 33b49dacaa946e..8d678412fe7179 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -1,8 +1,9 @@ // C Extension module to test pycore_lock.h API #include "parts.h" - #include "pycore_lock.h" +#include "pycore_pythread.h" // PyThread_get_thread_ident_ex() + #include "clinic/test_lock.c.h" #ifdef MS_WINDOWS @@ -35,9 +36,9 @@ test_lock_basic(PyObject *self, PyObject *obj) // uncontended lock and unlock PyMutex_Lock(&m); - assert(m.v == 1); + assert(m._bits == 1); PyMutex_Unlock(&m); - assert(m.v == 0); + assert(m._bits == 0); Py_RETURN_NONE; } @@ -56,10 +57,10 @@ lock_thread(void *arg) _Py_atomic_store_int(&test_data->started, 1); PyMutex_Lock(m); - assert(m->v == 1); + assert(m->_bits == 1); PyMutex_Unlock(m); - assert(m->v == 0); + assert(m->_bits == 0); _PyEvent_Notify(&test_data->done); } @@ -72,18 +73,26 @@ test_lock_two_threads(PyObject *self, PyObject *obj) memset(&test_data, 0, sizeof(test_data)); PyMutex_Lock(&test_data.m); - assert(test_data.m.v == 1); + assert(test_data.m._bits == 1); PyThread_start_new_thread(lock_thread, &test_data); - while (!_Py_atomic_load_int(&test_data.started)) { - pysleep(10); - } - pysleep(10); // allow some time for the other thread to try to lock - assert(test_data.m.v == 3); + + // wait up to two seconds for the lock_thread to attempt to lock "m" + int iters = 0; + uint8_t v; + do { + pysleep(10); // allow some time for the other thread to try to lock + v = _Py_atomic_load_uint8_relaxed(&test_data.m._bits); + assert(v == 1 || v == 3); + iters++; + } while (v != 3 && iters < 200); + + // both the "locked" and the "has parked" bits should be set + assert(test_data.m._bits == 3); PyMutex_Unlock(&test_data.m); PyEvent_Wait(&test_data.done); - assert(test_data.m.v == 0); + assert(test_data.m._bits == 0); Py_RETURN_NONE; } @@ -281,7 +290,10 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - _PyTime_t start = _PyTime_GetMonotonicClock(); + PyTime_t start, end; + if (PyTime_PerfCounter(&start) < 0) { + goto exit; + } for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -298,7 +310,9 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - _PyTime_t end = _PyTime_GetMonotonicClock(); + if (PyTime_PerfCounter(&end) < 0) { + goto exit; + } // Return the total number of acquisitions and the number of acquisitions // for each thread. @@ -310,7 +324,8 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, PyList_SET_ITEM(thread_iters, i, iter); } - double rate = total_iters * 1000000000.0 / (end - start); + assert(end != start); + double rate = total_iters * 1e9 / (end - start); res = Py_BuildValue("(dO)", rate, thread_iters); exit: @@ -333,6 +348,158 @@ test_lock_benchmark(PyObject *module, PyObject *obj) Py_RETURN_NONE; } +static int +init_maybe_fail(void *arg) +{ + int *counter = (int *)arg; + (*counter)++; + if (*counter < 5) { + // failure + return -1; + } + assert(*counter == 5); + return 0; +} + +static PyObject * +test_lock_once(PyObject *self, PyObject *obj) +{ + _PyOnceFlag once = {0}; + int counter = 0; + for (int i = 0; i < 10; i++) { + int res = _PyOnceFlag_CallOnce(&once, init_maybe_fail, &counter); + if (i < 4) { + assert(res == -1); + } + else { + assert(res == 0); + assert(counter == 5); + } + } + Py_RETURN_NONE; +} + +struct test_rwlock_data { + Py_ssize_t nthreads; + _PyRWMutex rw; + PyEvent step1; + PyEvent step2; + PyEvent step3; + PyEvent done; +}; + +static void +rdlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // Acquire the lock in read mode + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step1); + _PyRWMutex_RUnlock(&test_data->rw); + + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step3); + _PyRWMutex_RUnlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} +static void +wrlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // First acquire the lock in write mode + _PyRWMutex_Lock(&test_data->rw); + PyEvent_Wait(&test_data->step2); + _PyRWMutex_Unlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} + +static void +wait_until(uintptr_t *ptr, uintptr_t value) +{ + // wait up to two seconds for *ptr == value + int iters = 0; + uintptr_t bits; + do { + pysleep(10); + bits = _Py_atomic_load_uintptr(ptr); + iters++; + } while (bits != value && iters < 200); +} + +static PyObject * +test_lock_rwlock(PyObject *self, PyObject *obj) +{ + struct test_rwlock_data test_data = {.nthreads = 3}; + + _PyRWMutex_Lock(&test_data.rw); + assert(test_data.rw.bits == 1); + + _PyRWMutex_Unlock(&test_data.rw); + assert(test_data.rw.bits == 0); + + // Start two readers + PyThread_start_new_thread(rdlock_thread, &test_data); + PyThread_start_new_thread(rdlock_thread, &test_data); + + // wait up to two seconds for the threads to attempt to read-lock "rw" + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // start writer (while readers hold lock) + PyThread_start_new_thread(wrlock_thread, &test_data); + wait_until(&test_data.rw.bits, 10); + assert(test_data.rw.bits == 10); + + // readers release lock, writer should acquire it + _PyEvent_Notify(&test_data.step1); + wait_until(&test_data.rw.bits, 3); + assert(test_data.rw.bits == 3); + + // writer releases lock, readers acquire it + _PyEvent_Notify(&test_data.step2); + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // readers release lock again + _PyEvent_Notify(&test_data.step3); + wait_until(&test_data.rw.bits, 0); + assert(test_data.rw.bits == 0); + + PyEvent_Wait(&test_data.done); + Py_RETURN_NONE; +} + +static PyObject * +test_lock_recursive(PyObject *self, PyObject *obj) +{ + _PyRecursiveMutex m = (_PyRecursiveMutex){0}; + assert(!_PyRecursiveMutex_IsLockedByCurrentThread(&m)); + + _PyRecursiveMutex_Lock(&m); + assert(m.thread == PyThread_get_thread_ident_ex()); + assert(PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + _PyRecursiveMutex_Lock(&m); + assert(m.level == 1); + _PyRecursiveMutex_Unlock(&m); + + _PyRecursiveMutex_Unlock(&m); + assert(m.thread == 0); + assert(!PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -340,6 +507,9 @@ static PyMethodDef test_methods[] = { {"test_lock_counter_slow", test_lock_counter_slow, METH_NOARGS}, _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, + {"test_lock_once", test_lock_once, METH_NOARGS}, + {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, + {"test_lock_recursive", test_lock_recursive, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c new file mode 100644 index 00000000000000..fb5cdb6ca9e1d3 --- /dev/null +++ b/Modules/_testlimitedcapi.c @@ -0,0 +1,78 @@ +/* + * Test the limited C API. + * + * The 'test_*' functions exported by this module are run as part of the + * standard Python regression test, via Lib/test/test_capi.py. + */ + +#include "_testlimitedcapi/parts.h" + +static PyMethodDef TestMethods[] = { + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testlimitedcapimodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_testlimitedcapi", + .m_size = 0, + .m_methods = TestMethods, +}; + +PyMODINIT_FUNC +PyInit__testlimitedcapi(void) +{ + PyObject *mod = PyModule_Create(&_testlimitedcapimodule); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + + if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_ByteArray(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_HeaptypeRelative(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_List(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Long(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Object(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Set(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Unicode(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { + return NULL; + } + return mod; +} diff --git a/Modules/_testlimitedcapi/abstract.c b/Modules/_testlimitedcapi/abstract.c new file mode 100644 index 00000000000000..6056dd100d6069 --- /dev/null +++ b/Modules/_testlimitedcapi/abstract.c @@ -0,0 +1,582 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + +static PyObject * +number_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyBool_FromLong(PyNumber_Check(obj)); +} + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +mapping_haskeywitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); +} + +static PyObject * +mapping_haskeystringwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"number_check", number_check, METH_O}, + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, + {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c new file mode 100644 index 00000000000000..68b029e25af677 --- /dev/null +++ b/Modules/_testlimitedcapi/bytearray.c @@ -0,0 +1,123 @@ +#include "parts.h" +#include "util.h" + + +/* Test PyByteArray_Check() */ +static PyObject * +bytearray_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyByteArray_Check(obj)); +} + +/* Test PyByteArray_CheckExact() */ +static PyObject * +bytearray_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyByteArray_CheckExact(obj)); +} + +/* Test PyByteArray_FromStringAndSize() */ +static PyObject * +bytearray_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyByteArray_FromStringAndSize(s, size); +} + +/* Test PyByteArray_FromObject() */ +static PyObject * +bytearray_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + return PyByteArray_FromObject(arg); +} + +/* Test PyByteArray_Size() */ +static PyObject * +bytearray_size(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyByteArray_Size(arg)); +} + +/* Test PyUnicode_AsString() */ +static PyObject * +bytearray_asstring(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + s = PyByteArray_AsString(obj); + if (s == NULL) + return NULL; + + return PyByteArray_FromStringAndSize(s, buflen); +} + +/* Test PyByteArray_Concat() */ +static PyObject * +bytearray_concat(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyByteArray_Concat(left, right); +} + +/* Test PyByteArray_Resize() */ +static PyObject * +bytearray_resize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "On", &obj, &size)) + return NULL; + + NULLABLE(obj); + RETURN_INT(PyByteArray_Resize(obj, size)); +} + + +static PyMethodDef test_methods[] = { + {"bytearray_check", bytearray_check, METH_O}, + {"bytearray_checkexact", bytearray_checkexact, METH_O}, + {"bytearray_fromstringandsize", bytearray_fromstringandsize, METH_VARARGS}, + {"bytearray_fromobject", bytearray_fromobject, METH_O}, + {"bytearray_size", bytearray_size, METH_O}, + {"bytearray_asstring", bytearray_asstring, METH_VARARGS}, + {"bytearray_concat", bytearray_concat, METH_VARARGS}, + {"bytearray_resize", bytearray_resize, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_ByteArray(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c new file mode 100644 index 00000000000000..157d4089954af5 --- /dev/null +++ b/Modules/_testlimitedcapi/bytes.c @@ -0,0 +1,255 @@ +#include "parts.h" +#include "util.h" + + +/* Test PyBytes_Check() */ +static PyObject * +bytes_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_Check(obj)); +} + +/* Test PyBytes_CheckExact() */ +static PyObject * +bytes_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_CheckExact(obj)); +} + +/* Test PyBytes_FromStringAndSize() */ +static PyObject * +bytes_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyBytes_FromStringAndSize(s, size); +} + +/* Test PyBytes_FromString() */ +static PyObject * +bytes_fromstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyBytes_FromString(s); +} + +/* Test PyBytes_FromObject() */ +static PyObject * +bytes_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + return PyBytes_FromObject(arg); +} + +/* Test PyBytes_Size() */ +static PyObject * +bytes_size(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyBytes_Size(arg)); +} + +/* Test PyUnicode_AsString() */ +static PyObject * +bytes_asstring(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + s = PyBytes_AsString(obj); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyBytes_AsStringAndSize() */ +static PyObject * +bytes_asstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, &size) < 0) { + return NULL; + } + + if (s == NULL) { + return Py_BuildValue("(On)", Py_None, size); + } + else { + return Py_BuildValue("(y#n)", s, buflen, size); + } +} + +static PyObject * +bytes_asstringandsize_null(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, NULL) < 0) { + return NULL; + } + + if (s == NULL) { + Py_RETURN_NONE; + } + else { + return PyBytes_FromStringAndSize(s, buflen); + } +} + +/* Test PyBytes_Repr() */ +static PyObject * +bytes_repr(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + int smartquotes; + if (!PyArg_ParseTuple(args, "Oi", &obj, &smartquotes)) + return NULL; + + NULLABLE(obj); + return PyBytes_Repr(obj, smartquotes); +} + +/* Test PyBytes_Concat() */ +static PyObject * +bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + PyBytes_Concat(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_ConcatAndDel() */ +static PyObject * +bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + Py_XINCREF(right); + PyBytes_ConcatAndDel(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_DecodeEscape() */ +static PyObject * +bytes_decodeescape(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "z#|zn", &s, &bsize, &errors, &size)) + return NULL; + + if (size == -100) { + size = bsize; + } + return PyBytes_DecodeEscape(s, size, errors, 0, NULL); +} + + +static PyMethodDef test_methods[] = { + {"bytes_check", bytes_check, METH_O}, + {"bytes_checkexact", bytes_checkexact, METH_O}, + {"bytes_fromstringandsize", bytes_fromstringandsize, METH_VARARGS}, + {"bytes_fromstring", bytes_fromstring, METH_O}, + {"bytes_fromobject", bytes_fromobject, METH_O}, + {"bytes_size", bytes_size, METH_O}, + {"bytes_asstring", bytes_asstring, METH_VARARGS}, + {"bytes_asstringandsize", bytes_asstringandsize, METH_VARARGS}, + {"bytes_asstringandsize_null", bytes_asstringandsize_null, METH_VARARGS}, + {"bytes_repr", bytes_repr, METH_VARARGS}, + {"bytes_concat", bytes_concat, METH_VARARGS}, + {"bytes_concatanddel", bytes_concatanddel, METH_VARARGS}, + {"bytes_decodeescape", bytes_decodeescape, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Bytes(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h new file mode 100644 index 00000000000000..ebaeb53921a82f --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -0,0 +1,143 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testlimitedcapi_test_long_api, METH_NOARGS, _testlimitedcapi_test_long_api__doc__}, + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testlimitedcapi_test_longlong_api, METH_NOARGS, _testlimitedcapi_test_longlong_api__doc__}, + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testlimitedcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testlimitedcapi_test_long_as_double, METH_NOARGS, _testlimitedcapi_test_long_as_double__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, +"PyLong_AsInt($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ + {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, +/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h new file mode 100644 index 00000000000000..6631a9c42deab0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h @@ -0,0 +1,20 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall__doc__, +"call_vectorcall($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF \ + {"call_vectorcall", (PyCFunction)_testlimitedcapi_call_vectorcall, METH_O, _testlimitedcapi_call_vectorcall__doc__}, + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall_method__doc__, +"call_vectorcall_method($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ + {"call_vectorcall_method", (PyCFunction)_testlimitedcapi_call_vectorcall_method, METH_O, _testlimitedcapi_call_vectorcall_method__doc__}, +/*[clinic end generated code: output=5976b9b360e1ff30 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/complex.c b/Modules/_testlimitedcapi/complex.c new file mode 100644 index 00000000000000..e4c244e5c88d06 --- /dev/null +++ b/Modules/_testlimitedcapi/complex.c @@ -0,0 +1,79 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +complex_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_Check(obj)); +} + +static PyObject * +complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_CheckExact(obj)); +} + +static PyObject * +complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) +{ + double real, imag; + + if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { + return NULL; + } + + return PyComplex_FromDoubles(real, imag); +} + +static PyObject * +complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double real; + + NULLABLE(obj); + real = PyComplex_RealAsDouble(obj); + + if (real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(real); +} + +static PyObject * +complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double imag; + + NULLABLE(obj); + imag = PyComplex_ImagAsDouble(obj); + + if (imag == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(imag); +} + + +static PyMethodDef test_methods[] = { + {"complex_check", complex_check, METH_O}, + {"complex_checkexact", complex_checkexact, METH_O}, + {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, + {"complex_realasdouble", complex_realasdouble, METH_O}, + {"complex_imagasdouble", complex_imagasdouble, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Complex(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/dict.c b/Modules/_testlimitedcapi/dict.c new file mode 100644 index 00000000000000..ec32712eef6434 --- /dev/null +++ b/Modules/_testlimitedcapi/dict.c @@ -0,0 +1,291 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + assert(key == UNINITIALIZED_PTR); + assert(value == UNINITIALIZED_PTR); + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/float.c b/Modules/_testlimitedcapi/float.c new file mode 100644 index 00000000000000..88dc91f682ff39 --- /dev/null +++ b/Modules/_testlimitedcapi/float.c @@ -0,0 +1,91 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +float_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_Check(obj)); +} + +static PyObject * +float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_CheckExact(obj)); +} + +static PyObject * +float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyFloat_FromString(obj); +} + +static PyObject * +float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + if (!PyArg_Parse(obj, "d", &d)) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + NULLABLE(obj); + d = PyFloat_AsDouble(obj); + if (d == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_GetInfo(); +} + +static PyObject * +float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMax()); +} + +static PyObject * +float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMin()); +} + + +static PyMethodDef test_methods[] = { + {"float_check", float_check, METH_O}, + {"float_checkexact", float_checkexact, METH_O}, + {"float_fromstring", float_fromstring, METH_O}, + {"float_fromdouble", float_fromdouble, METH_O}, + {"float_asdouble", float_asdouble, METH_O}, + {"float_getinfo", float_getinfo, METH_NOARGS}, + {"float_getmax", float_getmax, METH_NOARGS}, + {"float_getmin", float_getmin, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Float(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c similarity index 97% rename from Modules/_testcapi/heaptype_relative.c rename to Modules/_testlimitedcapi/heaptype_relative.c index 53dd01d1ed4f80..c2531518d86a51 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,4 +1,9 @@ -#define Py_LIMITED_API 0x030c0000 // 3.12 +// Need limited C API version 3.12 for PyType_FromMetaclass() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include // max_align_t #include // memset @@ -326,7 +331,8 @@ static PyMethodDef TestMethods[] = { }; int -_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { +_PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c new file mode 100644 index 00000000000000..ed492c3e719727 --- /dev/null +++ b/Modules/_testlimitedcapi/list.c @@ -0,0 +1,175 @@ +// Need limited C API version 3.13 for PyList_GetItemRef() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + +static PyObject * +list_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_Check(obj)); +} + +static PyObject * +list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_CheckExact(obj)); +} + +static PyObject * +list_new(PyObject* Py_UNUSED(module), PyObject *obj) +{ + return PyList_New(PyLong_AsSsize_t(obj)); +} + +static PyObject * +list_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_Size(obj)); +} + +static PyObject * +list_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GetItem(obj, i)); +} + +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + +static PyObject * +list_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); + +} + +static PyObject * +list_insert(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t where; + if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); + +} + +static PyObject * +list_append(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Append(obj, value)); +} + +static PyObject * +list_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetSlice(obj, ilow, ihigh); + +} + +static PyObject * +list_setslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); +} + +static PyObject * +list_sort(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Sort(obj)); +} + +static PyObject * +list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Reverse(obj)); +} + +static PyObject * +list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyList_AsTuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"list_check", list_check, METH_O}, + {"list_check_exact", list_check_exact, METH_O}, + {"list_new", list_new, METH_O}, + {"list_size", list_size, METH_O}, + {"list_getitem", list_getitem, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, + {"list_setitem", list_setitem, METH_VARARGS}, + {"list_insert", list_insert, METH_VARARGS}, + {"list_append", list_append, METH_VARARGS}, + {"list_getslice", list_getslice, METH_VARARGS}, + {"list_setslice", list_setslice, METH_VARARGS}, + {"list_sort", list_sort, METH_O}, + {"list_reverse", list_reverse, METH_O}, + {"list_astuple", list_astuple, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c new file mode 100644 index 00000000000000..5953009b6ef9b7 --- /dev/null +++ b/Modules/_testlimitedcapi/long.c @@ -0,0 +1,798 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.13 to test PyLong_AsInt() +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + + +static PyObject * +raiseTestError(const char* test_name, const char* msg) +{ + PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); + return NULL; +} + +/* Tests of PyLong_{As, From}{Unsigned,}Long(), and + PyLong_{As, From}{Unsigned,}LongLong(). + + Note that the meat of the test is contained in testcapi_long.h. + This is revolting, but delicate code duplication is worse: "almost + exactly the same" code is needed to test long long, but the ubiquitous + dependence on type names makes it impossible to use a parameterized + function. A giant macro would be even worse than this. A C++ template + would be perfect. + + The "report an error" functions are deliberately not part of the #include + file: if the test fails, you can set a breakpoint in the appropriate + error function directly, and crawl back from there in the debugger. +*/ + +#define UNBIND(X) Py_DECREF(X); (X) = NULL + +static PyObject * +raise_test_long_error(const char* msg) +{ + return raiseTestError("test_long_api", msg); +} + +// Test PyLong_FromLong()/PyLong_AsLong() +// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). + +#define TESTNAME test_long_api_inner +#define TYPENAME long +#define F_S_TO_PY PyLong_FromLong +#define F_PY_TO_S PyLong_AsLong +#define F_U_TO_PY PyLong_FromUnsignedLong +#define F_PY_TO_U PyLong_AsUnsignedLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_long_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=06a2c02366d1853a input=9012b3d6a483df63]*/ +{ + return TESTNAME(raise_test_long_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + +// Test PyLong_FromLongLong()/PyLong_AsLongLong() +// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). + +static PyObject * +raise_test_longlong_error(const char* msg) +{ + return raiseTestError("test_longlong_api", msg); +} + +#define TESTNAME test_longlong_api_inner +#define TYPENAME long long +#define F_S_TO_PY PyLong_FromLongLong +#define F_PY_TO_S PyLong_AsLongLong +#define F_U_TO_PY PyLong_FromUnsignedLongLong +#define F_PY_TO_U PyLong_AsUnsignedLongLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_longlong_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=8faa10e1c35214bf input=2b582a9d25bd68e7]*/ +{ + return TESTNAME(raise_test_longlong_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + + +/*[clinic input] +_testlimitedcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=fdfd3c1eeabb6d14 input=e3a18791de6519fe]*/ +{ + PyObject *num, *one, *temp; + long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LONG_MAX even on 64-bit platforms */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LONG_MAX + 1 */ + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); + num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LONG_MIN even on 64-bit platforms */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LONG_MIN - 1 */ + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MAX) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MIN) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=3d2721a49c09a307 input=741c593b606cc6b3]*/ +{ + PyObject *num, *one, *temp; + long long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LLONG_MAX on a typical machine. */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LLONG_MAX + 1 */ + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LLONG_MIN on a typical platform */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LLONG_MIN - 1 */ + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MAX) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MIN) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +{ + size_t out_u; + Py_ssize_t out_s; + + Py_INCREF(Py_None); + + out_u = PyLong_AsSize_t(Py_None); + if (out_u != (size_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + out_s = PyLong_AsSsize_t(Py_None); + if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=90be09ffeec8ecab input=17c660bd58becad5]*/ +{ + unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); + + if (res != (unsigned long long)-1 || !PyErr_Occurred()) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) didn't " + "complain"); + } + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) raised " + "something other than SystemError"); + } + PyErr_Clear(); + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_double +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=0e688c2acf224f88 input=e7b5712385064a48]*/ +{ + double out; + + Py_INCREF(Py_None); + + out = PyLong_AsDouble(Py_None); + if (out != -1.0 || !PyErr_Occurred()) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +static PyObject * +pylong_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_Check(obj)); +} + +static PyObject * +pylong_checkexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_CheckExact(obj)); +} + +static PyObject * +pylong_fromdouble(PyObject *module, PyObject *arg) +{ + double value; + if (!PyArg_Parse(arg, "d", &value)) { + return NULL; + } + return PyLong_FromDouble(value); +} + +static PyObject * +pylong_fromstring(PyObject *module, PyObject *args) +{ + const char *str; + Py_ssize_t len; + int base; + char *end = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { + return NULL; + } + + PyObject *result = PyLong_FromString(str, &end, base); + if (result == NULL) { + // XXX 'end' is not always set. + return NULL; + } + return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); +} + +static PyObject * +pylong_fromvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + return PyLong_FromVoidPtr((void *)arg); +} + +/*[clinic input] +_testlimitedcapi.PyLong_AsInt + arg: object + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_PyLong_AsInt(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=d91db4c1287f85fa input=32c66be86f3265a1]*/ +{ + NULLABLE(arg); + assert(!PyErr_Occurred()); + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long value = PyLong_AsLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long value = PyLong_AsLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("li", value, overflow); +} + +static PyObject * +pylong_asunsignedlong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLong(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLongMask(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_aslonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long long value = PyLong_AsLongLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLongLong(value); +} + +static PyObject * +pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("Li", value, overflow); +} + +static PyObject * +pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLong(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_as_ssize_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + Py_ssize_t value = PyLong_AsSsize_t(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSsize_t(value); +} + +static PyObject * +pylong_as_size_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + size_t value = PyLong_AsSize_t(arg); + if (value == (size_t)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSize_t(value); +} + +static PyObject * +pylong_asdouble(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + double value = PyLong_AsDouble(arg); + if (value == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(value); +} + +static PyObject * +pylong_asvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + void *value = PyLong_AsVoidPtr(arg); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef((PyObject *)value); +} + +static PyObject * +pylong_aspid(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + pid_t value = PyLong_AsPid(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromPid(value); +} + + +static PyMethodDef test_methods[] = { + _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF + {"pylong_check", pylong_check, METH_O}, + {"pylong_checkexact", pylong_checkexact, METH_O}, + {"pylong_fromdouble", pylong_fromdouble, METH_O}, + {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, + {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, + _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF + {"pylong_aslong", pylong_aslong, METH_O}, + {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, + {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, + {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, + {"pylong_aslonglong", pylong_aslonglong, METH_O}, + {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, + {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, + {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, + {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, + {"pylong_as_size_t", pylong_as_size_t, METH_O}, + {"pylong_asdouble", pylong_asdouble, METH_O}, + {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Long(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c new file mode 100644 index 00000000000000..da6fe3e4efa34c --- /dev/null +++ b/Modules/_testlimitedcapi/object.c @@ -0,0 +1,80 @@ +// Need limited C API version 3.13 for Py_GetConstant() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + + +/* Test Py_GetConstant() */ +static PyObject * +get_constant(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstant(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return obj; +} + + +/* Test Py_GetConstantBorrowed() */ +static PyObject * +get_constant_borrowed(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstantBorrowed(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_NewRef(obj); +} + + +/* Test constants */ +static PyObject * +test_constants(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Test that implementation of constants in the limited C API: + // check that the C code compiles. + // + // Test also that constants and Py_GetConstant() return the same + // objects. + assert(Py_None == Py_GetConstant(Py_CONSTANT_NONE)); + assert(Py_False == Py_GetConstant(Py_CONSTANT_FALSE)); + assert(Py_True == Py_GetConstant(Py_CONSTANT_TRUE)); + assert(Py_Ellipsis == Py_GetConstant(Py_CONSTANT_ELLIPSIS)); + assert(Py_NotImplemented == Py_GetConstant(Py_CONSTANT_NOT_IMPLEMENTED)); + // Other constants are tested in test_capi.test_object + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"get_constant", get_constant, METH_VARARGS}, + {"get_constant_borrowed", get_constant_borrowed, METH_VARARGS}, + {"test_constants", test_constants, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h new file mode 100644 index 00000000000000..d5e590a8dcd679 --- /dev/null +++ b/Modules/_testlimitedcapi/parts.h @@ -0,0 +1,41 @@ +#ifndef Py_TESTLIMITEDCAPI_PARTS_H +#define Py_TESTLIMITEDCAPI_PARTS_H + +// Always enable assertions +#undef NDEBUG + +#include "pyconfig.h" // Py_GIL_DISABLED + +// Use the limited C API +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) + // need limited C API version 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +// Make sure that the internal C API cannot be used. +#undef Py_BUILD_CORE_MODULE +#undef Py_BUILD_CORE_BUILTIN + +#include "Python.h" + +#ifdef Py_BUILD_CORE +# error "Py_BUILD_CORE macro must not be defined" +#endif + +int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); +int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); +int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Complex(PyObject *module); +int _PyTestLimitedCAPI_Init_Dict(PyObject *module); +int _PyTestLimitedCAPI_Init_Float(PyObject *module); +int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); +int _PyTestLimitedCAPI_Init_Object(PyObject *module); +int _PyTestLimitedCAPI_Init_List(PyObject *module); +int _PyTestLimitedCAPI_Init_Long(PyObject *module); +int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); +int _PyTestLimitedCAPI_Init_Set(PyObject *module); +int _PyTestLimitedCAPI_Init_Sys(PyObject *module); +int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); +int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); + +#endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c similarity index 97% rename from Modules/_testcapi/pyos.c rename to Modules/_testlimitedcapi/pyos.c index 63140e914875db..0f61801eae322a 100644 --- a/Modules/_testcapi/pyos.c +++ b/Modules/_testlimitedcapi/pyos.c @@ -50,7 +50,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_PyOS(PyObject *mod) +_PyTestLimitedCAPI_Init_PyOS(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c new file mode 100644 index 00000000000000..35da5fa5f008e1 --- /dev/null +++ b/Modules/_testlimitedcapi/set.c @@ -0,0 +1,189 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +set_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Check(obj)); +} + +static PyObject * +set_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_CheckExact(obj)); +} + +static PyObject * +frozenset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_Check(obj)); +} + +static PyObject * +frozenset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_CheckExact(obj)); +} + +static PyObject * +anyset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_Check(obj)); +} + +static PyObject * +anyset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_CheckExact(obj)); +} + +static PyObject * +set_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PySet_New(iterable); +} + +static PyObject * +frozenset_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PyFrozenSet_New(iterable); +} + +static PyObject * +set_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySet_Size(obj)); +} + +static PyObject * +set_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Contains(obj, item)); +} + +static PyObject * +set_add(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Add(obj, item)); +} + +static PyObject * +set_discard(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Discard(obj, item)); +} + +static PyObject * +set_pop(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySet_Pop(obj); +} + +static PyObject * +set_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Clear(obj)); +} + +static PyObject * +test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // Test that `frozenset` can be used with `PySet_Add`, + // when frozenset is just created in CAPI. + PyObject *fs = PyFrozenSet_New(NULL); + if (fs == NULL) { + return NULL; + } + PyObject *num = PyLong_FromLong(1); + if (num == NULL) { + goto error; + } + if (PySet_Add(fs, num) < 0) { + goto error; + } + int contains = PySet_Contains(fs, num); + if (contains < 0) { + goto error; + } + else if (contains == 0) { + goto unexpected; + } + Py_DECREF(fs); + Py_DECREF(num); + Py_RETURN_NONE; + +unexpected: + PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); +error: + Py_DECREF(fs); + Py_XDECREF(num); + return NULL; +} + +static PyMethodDef test_methods[] = { + {"set_check", set_check, METH_O}, + {"set_checkexact", set_checkexact, METH_O}, + {"frozenset_check", frozenset_check, METH_O}, + {"frozenset_checkexact", frozenset_checkexact, METH_O}, + {"anyset_check", anyset_check, METH_O}, + {"anyset_checkexact", anyset_checkexact, METH_O}, + + {"set_new", set_new, METH_VARARGS}, + {"frozenset_new", frozenset_new, METH_VARARGS}, + + {"set_size", set_size, METH_O}, + {"set_contains", set_contains, METH_VARARGS}, + {"set_add", set_add, METH_VARARGS}, + {"set_discard", set_discard, METH_VARARGS}, + {"set_pop", set_pop, METH_O}, + {"set_clear", set_clear, METH_O}, + + {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Set(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/sys.c b/Modules/_testlimitedcapi/sys.c new file mode 100644 index 00000000000000..7d8b7a8569e515 --- /dev/null +++ b/Modules/_testlimitedcapi/sys.c @@ -0,0 +1,56 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + +static PyObject * +sys_getxoptions(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(ignored)) +{ + PyObject *result = PySys_GetXOptions(); + return Py_XNewRef(result); +} + + +static PyMethodDef test_methods[] = { + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, + {"sys_getxoptions", sys_getxoptions, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Sys(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/testcapi_long.h b/Modules/_testlimitedcapi/testcapi_long.h similarity index 100% rename from Modules/_testcapi/testcapi_long.h rename to Modules/_testlimitedcapi/testcapi_long.h diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c new file mode 100644 index 00000000000000..2b70d09108a333 --- /dev/null +++ b/Modules/_testlimitedcapi/unicode.c @@ -0,0 +1,1938 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.13 to test PyUnicode_EqualToUTF8() +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + +#include // ptrdiff_t +#include // memset() + + +static PyObject * +codec_incrementalencoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incrementaldecoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); + int result; + if (py_s == NULL) + return NULL; + result = PyUnicode_CompareWithASCIIString(py_s, "str"); + Py_DECREF(py_s); + if (!result) { + PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " + "should not compare equal to c string."); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; + size_t wtextlen = 1; + const wchar_t invalid[1] = {(wchar_t)0x110000u}; +#else + const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; + size_t wtextlen = 2; +#endif + PyObject *wide, *utf8; + + wide = PyUnicode_FromWideChar(wtext, wtextlen); + if (wide == NULL) + return NULL; + + utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); + if (utf8 == NULL) { + Py_DECREF(wide); + return NULL; + } + + if (PyUnicode_GetLength(wide) != PyUnicode_GetLength(utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "have different length"); + return NULL; + } + if (PyUnicode_Compare(wide, utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + if (PyErr_Occurred()) + return NULL; + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "are different"); + return NULL; + } + + Py_DECREF(wide); + Py_DECREF(utf8); + +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + wide = PyUnicode_FromWideChar(invalid, 1); + if (wide == NULL) + PyErr_Clear(); + else { + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + return NULL; + } +#endif + Py_RETURN_NONE; +} + + +static PyObject * +unicode_copy(PyObject *unicode) +{ + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + // Create a new string by encoding to UTF-8 and then decode from UTF-8 + PyObject *utf8 = PyUnicode_AsUTF8String(unicode); + if (!utf8) { + return NULL; + } + + PyObject *copy = PyUnicode_DecodeUTF8( + PyBytes_AsString(utf8), + PyBytes_Size(utf8), + NULL); + Py_DECREF(utf8); + + return copy; +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +static void +unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch) +{ + assert(0 <= start); + assert(end <= PyUnicode_GetLength(str)); + for (Py_ssize_t i = start; i < end; i++) { + int res = PyUnicode_WriteChar(str, i, ch); + assert(res == 0); + } +} + + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GetLength(obj)) { + unicode_fill(copy, PyUnicode_GetLength(obj), length, 0U); + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyUnicode_GetLength(arg)); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + +/* Test PyUnicode_FromEncodedObject() */ +static PyObject * +unicode_fromencodedobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + return NULL; + } + + NULLABLE(obj); + return PyUnicode_FromEncodedObject(obj, encoding, errors); +} + +/* Test PyUnicode_FromObject() */ +static PyObject * +unicode_fromobject(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_FromObject(arg); +} + +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + +/* Test PyUnicode_AsWideChar() */ +static PyObject * +unicode_aswidechar(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t buflen, size; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + buffer = PyMem_New(wchar_t, buflen); + if (buffer == NULL) + return PyErr_NoMemory(); + + size = PyUnicode_AsWideChar(unicode, buffer, buflen); + if (size == -1) { + PyMem_Free(buffer); + return NULL; + } + + if (size < buflen) + buflen = size + 1; + else + buflen = size; + result = PyUnicode_FromWideChar(buffer, buflen); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); +} + +/* Test PyUnicode_AsWideCharString() */ +static PyObject * +unicode_aswidecharstring(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t size = UNINITIALIZED_SIZE; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, &size); + if (buffer == NULL) { + assert(size == UNINITIALIZED_SIZE); + return NULL; + } + + result = PyUnicode_FromWideChar(buffer, size + 1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; + + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8AndSize() */ +static PyObject * +unicode_asutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) { + assert(size == -1); + return NULL; + } + + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test PyUnicode_Decode() */ +static PyObject * +unicode_decode(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t size; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) + return NULL; + + return PyUnicode_Decode(s, size, encoding, errors); +} + +/* Test PyUnicode_AsEncodedString() */ +static PyObject * +unicode_asencodedstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_AsEncodedString(unicode, encoding, errors); +} + +/* Test PyUnicode_BuildEncodingMap() */ +static PyObject * +unicode_buildencodingmap(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_BuildEncodingMap(arg); +} + +/* Test PyUnicode_DecodeUTF7() */ +static PyObject * +unicode_decodeutf7(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF7(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF7Stateful() */ +static PyObject * +unicode_decodeutf7stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsUTF8String() */ +static PyObject * +unicode_asutf8string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF8String(arg); +} + +/* Test PyUnicode_DecodeUTF32() */ +static PyObject * +unicode_decodeutf32(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF32Stateful() */ +static PyObject * +unicode_decodeutf32stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF32String() */ +static PyObject * +unicode_asutf32string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF32String(arg); +} + +/* Test PyUnicode_DecodeUTF16() */ +static PyObject * +unicode_decodeutf16(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF16Stateful() */ +static PyObject * +unicode_decodeutf16stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF16String() */ +static PyObject * +unicode_asutf16string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF16String(arg); +} + +/* Test PyUnicode_DecodeUnicodeEscape() */ +static PyObject * +unicode_decodeunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsUnicodeEscapeString() */ +static PyObject * +unicode_asunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decoderawunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsRawUnicodeEscapeString() */ +static PyObject * +unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsRawUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decodelatin1(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLatin1(data, size, errors); +} + +/* Test PyUnicode_AsLatin1String() */ +static PyObject * +unicode_aslatin1string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsLatin1String(arg); +} + +/* Test PyUnicode_DecodeASCII() */ +static PyObject * +unicode_decodeascii(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeASCII(data, size, errors); +} + +/* Test PyUnicode_AsASCIIString() */ +static PyObject * +unicode_asasciistring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsASCIIString(arg); +} + +/* Test PyUnicode_DecodeCharmap() */ +static PyObject * +unicode_decodecharmap(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + PyObject *mapping; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) + return NULL; + + NULLABLE(mapping); + return PyUnicode_DecodeCharmap(data, size, mapping, errors); +} + +/* Test PyUnicode_AsCharmapString() */ +static PyObject * +unicode_ascharmapstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + PyObject *mapping; + + if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) + return NULL; + + NULLABLE(unicode); + NULLABLE(mapping); + return PyUnicode_AsCharmapString(unicode, mapping); +} + +#ifdef MS_WINDOWS + +/* Test PyUnicode_DecodeMBCS() */ +static PyObject * +unicode_decodembcs(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeMBCS(data, size, errors); +} + +/* Test PyUnicode_DecodeMBCSStateful() */ +static PyObject * +unicode_decodembcsstateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeCodePageStateful() */ +static PyObject * +unicode_decodecodepagestateful(PyObject *self, PyObject *args) +{ + int code_page; + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsMBCSString() */ +static PyObject * +unicode_asmbcsstring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsMBCSString(arg); +} + +/* Test PyUnicode_EncodeCodePage() */ +static PyObject * +unicode_encodecodepage(PyObject *self, PyObject *args) +{ + int code_page; + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeCodePage(code_page, unicode, errors); +} + +#endif /* MS_WINDOWS */ + +/* Test PyUnicode_DecodeLocaleAndSize() */ +static PyObject * +unicode_decodelocaleandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocaleAndSize(data, size, errors); +} + +/* Test PyUnicode_DecodeLocale() */ +static PyObject * +unicode_decodelocale(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocale(data, errors); +} + +/* Test PyUnicode_EncodeLocale() */ +static PyObject * +unicode_encodelocale(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeLocale(unicode, errors); +} + +/* Test PyUnicode_DecodeFSDefault() */ +static PyObject * +unicode_decodefsdefault(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#", &data, &size)) + return NULL; + + return PyUnicode_DecodeFSDefault(data); +} + +/* Test PyUnicode_DecodeFSDefaultAndSize() */ +static PyObject * +unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(data, size); +} + +/* Test PyUnicode_EncodeFSDefault() */ +static PyObject * +unicode_encodefsdefault(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_EncodeFSDefault(arg); +} + +/* Test PyUnicode_Concat() */ +static PyObject * +unicode_concat(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_Concat(left, right); +} + +/* Test PyUnicode_Split() */ +static PyObject * +unicode_split(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Split(s, sep, maxsplit); +} + +/* Test PyUnicode_RSplit() */ +static PyObject * +unicode_rsplit(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RSplit(s, sep, maxsplit); +} + +/* Test PyUnicode_Splitlines() */ +static PyObject * +unicode_splitlines(PyObject *self, PyObject *args) +{ + PyObject *s; + int keepends = 0; + + if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + return NULL; + + NULLABLE(s); + return PyUnicode_Splitlines(s, keepends); +} + +/* Test PyUnicode_Partition() */ +static PyObject * +unicode_partition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Partition(s, sep); +} + +/* Test PyUnicode_RPartition() */ +static PyObject * +unicode_rpartition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RPartition(s, sep); +} + +/* Test PyUnicode_Translate() */ +static PyObject * +unicode_translate(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyObject *table; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + return NULL; + + NULLABLE(obj); + NULLABLE(table); + return PyUnicode_Translate(obj, table, errors); +} + +/* Test PyUnicode_Join() */ +static PyObject * +unicode_join(PyObject *self, PyObject *args) +{ + PyObject *sep; + PyObject *seq; + + if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + return NULL; + + NULLABLE(sep); + NULLABLE(seq); + return PyUnicode_Join(sep, seq); +} + +/* Test PyUnicode_Count() */ +static PyObject * +unicode_count(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + + if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); +} + +/* Test PyUnicode_Find() */ +static PyObject * +unicode_find(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Find(str, substr, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Tailmatch() */ +static PyObject * +unicode_tailmatch(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); +} + +/* Test PyUnicode_FindChar() */ +static PyObject * +unicode_findchar(PyObject *self, PyObject *args) +{ + PyObject *str; + int direction; + unsigned int ch; + Py_ssize_t result; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, + &start, &end, &direction)) { + return NULL; + } + NULLABLE(str); + result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Replace() */ +static PyObject * +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + PyObject *replstr; + Py_ssize_t maxcount = -1; + + if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + NULLABLE(replstr); + return PyUnicode_Replace(str, substr, replstr, maxcount); +} + +/* Test PyUnicode_Compare() */ +static PyObject * +unicode_compare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int result; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CompareWithASCIIString() */ +static PyObject * +unicode_comparewithasciistring(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) + return NULL; + + NULLABLE(left); + result = PyUnicode_CompareWithASCIIString(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8() */ +static PyObject * +unicode_equaltoutf8(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + return NULL; + } + + NULLABLE(left); + result = PyUnicode_EqualToUTF8(left, right); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8AndSize() */ +static PyObject * +unicode_equaltoutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + Py_ssize_t size = -100; + int result; + + if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + return NULL; + } + + NULLABLE(left); + if (size == -100) { + size = right_len; + } + result = PyUnicode_EqualToUTF8AndSize(left, right, size); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_RichCompare() */ +static PyObject * +unicode_richcompare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int op; + + if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_RichCompare(left, right, op); +} + +/* Test PyUnicode_Format() */ +static PyObject * +unicode_format(PyObject *self, PyObject *args) +{ + PyObject *format; + PyObject *fargs; + + if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) + return NULL; + + NULLABLE(format); + NULLABLE(fargs); + return PyUnicode_Format(format, fargs); +} + +/* Test PyUnicode_Contains() */ +static PyObject * +unicode_contains(PyObject *self, PyObject *args) +{ + PyObject *container; + PyObject *element; + + if (!PyArg_ParseTuple(args, "OO", &container, &element)) + return NULL; + + NULLABLE(container); + NULLABLE(element); + RETURN_INT(PyUnicode_Contains(container, element)); +} + +/* Test PyUnicode_IsIdentifier() */ +static PyObject * +unicode_isidentifier(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyUnicode_IsIdentifier(arg)); +} + + +static int +check_raised_systemerror(PyObject *result, char* msg) +{ + if (result) { + // no exception + PyErr_Format(PyExc_AssertionError, + "SystemError not raised: %s", + msg); + return 0; + } + if (PyErr_ExceptionMatches(PyExc_SystemError)) { + // expected exception + PyErr_Clear(); + return 1; + } + // unexpected exception + return 0; +} + +static PyObject * +test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromString("None"); + +#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ + result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ + if (EXPECTED == NULL) { \ + if (!check_raised_systemerror(result, FORMAT)) { \ + goto Fail; \ + } \ + } \ + else if (result == NULL) \ + return NULL; \ + else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ + PyObject *utf8 = PyUnicode_AsUTF8String(result); \ + PyErr_Format(PyExc_AssertionError, \ + "test_string_from_format: failed at \"%s\" " \ + "expected \"%s\" got \"%s\"", \ + FORMAT, EXPECTED, utf8); \ + Py_XDECREF(utf8); \ + goto Fail; \ + } \ + Py_XDECREF(result) + +#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) + +#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) + + // Unrecognized + CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); + + // "%%" (options are rejected) + CHECK_FORMAT_0( "%%", "%"); + CHECK_FORMAT_0( "%0%", NULL); + CHECK_FORMAT_0("%00%", NULL); + CHECK_FORMAT_0( "%2%", NULL); + CHECK_FORMAT_0("%02%", NULL); + CHECK_FORMAT_0("%.0%", NULL); + CHECK_FORMAT_0("%.2%", NULL); + + // "%c" + CHECK_FORMAT_1( "%c", "c", 'c'); + CHECK_FORMAT_1( "%0c", "c", 'c'); + CHECK_FORMAT_1("%00c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); + + // Integers + CHECK_FORMAT_1("%d", "123", (int)123); + CHECK_FORMAT_1("%i", "123", (int)123); + CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); + CHECK_FORMAT_1("%ld", "123", (long)123); + CHECK_FORMAT_1("%li", "123", (long)123); + CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); + CHECK_FORMAT_1("%lld", "123", (long long)123); + CHECK_FORMAT_1("%lli", "123", (long long)123); + CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); + CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zu", "123", (size_t)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); + + CHECK_FORMAT_1("%d", "-123", (int)-123); + CHECK_FORMAT_1("%i", "-123", (int)-123); + CHECK_FORMAT_1("%ld", "-123", (long)-123); + CHECK_FORMAT_1("%li", "-123", (long)-123); + CHECK_FORMAT_1("%lld", "-123", (long long)-123); + CHECK_FORMAT_1("%lli", "-123", (long long)-123); + CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); + + // Integers: width < length + CHECK_FORMAT_1("%1d", "123", (int)123); + CHECK_FORMAT_1("%1i", "123", (int)123); + CHECK_FORMAT_1("%1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%1ld", "123", (long)123); + CHECK_FORMAT_1("%1li", "123", (long)123); + CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%1lld", "123", (long long)123); + CHECK_FORMAT_1("%1lli", "123", (long long)123); + CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zu", "123", (size_t)123); + CHECK_FORMAT_1("%1x", "7b", (int)123); + + CHECK_FORMAT_1("%1d", "-123", (int)-123); + CHECK_FORMAT_1("%1i", "-123", (int)-123); + CHECK_FORMAT_1("%1ld", "-123", (long)-123); + CHECK_FORMAT_1("%1li", "-123", (long)-123); + CHECK_FORMAT_1("%1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); + + // Integers: width > length + CHECK_FORMAT_1("%5d", " 123", (int)123); + CHECK_FORMAT_1("%5i", " 123", (int)123); + CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); + CHECK_FORMAT_1("%5ld", " 123", (long)123); + CHECK_FORMAT_1("%5li", " 123", (long)123); + CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); + CHECK_FORMAT_1("%5lld", " 123", (long long)123); + CHECK_FORMAT_1("%5lli", " 123", (long long)123); + CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); + CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zu", " 123", (size_t)123); + CHECK_FORMAT_1("%5x", " 7b", (int)123); + + CHECK_FORMAT_1("%5d", " -123", (int)-123); + CHECK_FORMAT_1("%5i", " -123", (int)-123); + CHECK_FORMAT_1("%5ld", " -123", (long)-123); + CHECK_FORMAT_1("%5li", " -123", (long)-123); + CHECK_FORMAT_1("%5lld", " -123", (long long)-123); + CHECK_FORMAT_1("%5lli", " -123", (long long)-123); + CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); + + // Integers: width > length, 0-flag + CHECK_FORMAT_1("%05d", "00123", (int)123); + CHECK_FORMAT_1("%05i", "00123", (int)123); + CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%05ld", "00123", (long)123); + CHECK_FORMAT_1("%05li", "00123", (long)123); + CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%05lld", "00123", (long long)123); + CHECK_FORMAT_1("%05lli", "00123", (long long)123); + CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zu", "00123", (size_t)123); + CHECK_FORMAT_1("%05x", "0007b", (int)123); + + CHECK_FORMAT_1("%05d", "-0123", (int)-123); + CHECK_FORMAT_1("%05i", "-0123", (int)-123); + CHECK_FORMAT_1("%05ld", "-0123", (long)-123); + CHECK_FORMAT_1("%05li", "-0123", (long)-123); + CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); + CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); + CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); + + // Integers: precision < length + CHECK_FORMAT_1("%.1d", "123", (int)123); + CHECK_FORMAT_1("%.1i", "123", (int)123); + CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%.1ld", "123", (long)123); + CHECK_FORMAT_1("%.1li", "123", (long)123); + CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%.1lld", "123", (long long)123); + CHECK_FORMAT_1("%.1lli", "123", (long long)123); + CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zu", "123", (size_t)123); + CHECK_FORMAT_1("%.1x", "7b", (int)123); + + CHECK_FORMAT_1("%.1d", "-123", (int)-123); + CHECK_FORMAT_1("%.1i", "-123", (int)-123); + CHECK_FORMAT_1("%.1ld", "-123", (long)-123); + CHECK_FORMAT_1("%.1li", "-123", (long)-123); + CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); + + // Integers: precision > length + CHECK_FORMAT_1("%.5d", "00123", (int)123); + CHECK_FORMAT_1("%.5i", "00123", (int)123); + CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%.5ld", "00123", (long)123); + CHECK_FORMAT_1("%.5li", "00123", (long)123); + CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%.5lld", "00123", (long long)123); + CHECK_FORMAT_1("%.5lli", "00123", (long long)123); + CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); + CHECK_FORMAT_1("%.5x", "0007b", (int)123); + + CHECK_FORMAT_1("%.5d", "-00123", (int)-123); + CHECK_FORMAT_1("%.5i", "-00123", (int)-123); + CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); + CHECK_FORMAT_1("%.5li", "-00123", (long)-123); + CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); + + // Integers: width > precision > length + CHECK_FORMAT_1("%7.5d", " 00123", (int)123); + CHECK_FORMAT_1("%7.5i", " 00123", (int)123); + CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); + CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); + CHECK_FORMAT_1("%7.5li", " 00123", (long)123); + CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); + CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); + CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); + CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); + + CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); + + // Integers: width > precision > length, 0-flag + CHECK_FORMAT_1("%07.5d", "0000123", (int)123); + CHECK_FORMAT_1("%07.5i", "0000123", (int)123); + CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); + CHECK_FORMAT_1("%07.5li", "0000123", (long)123); + CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%07.5x", "000007b", (int)123); + + CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); + + // Integers: precision > width > length + CHECK_FORMAT_1("%5.7d", "0000123", (int)123); + CHECK_FORMAT_1("%5.7i", "0000123", (int)123); + CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%5.7li", "0000123", (long)123); + CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%5.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision > width > length, 0-flag + CHECK_FORMAT_1("%05.7d", "0000123", (int)123); + CHECK_FORMAT_1("%05.7i", "0000123", (int)123); + CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%05.7li", "0000123", (long)123); + CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%05.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision = 0, arg = 0 (empty string in C) + CHECK_FORMAT_1("%.0d", "0", (int)0); + CHECK_FORMAT_1("%.0i", "0", (int)0); + CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); + CHECK_FORMAT_1("%.0ld", "0", (long)0); + CHECK_FORMAT_1("%.0li", "0", (long)0); + CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); + CHECK_FORMAT_1("%.0lld", "0", (long long)0); + CHECK_FORMAT_1("%.0lli", "0", (long long)0); + CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); + CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zu", "0", (size_t)0); + CHECK_FORMAT_1("%.0x", "0", (int)0); + + // Strings + CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); + CHECK_FORMAT_1("%U", "None", unicode); + CHECK_FORMAT_1("%A", "None", Py_None); + CHECK_FORMAT_1("%S", "None", Py_None); + CHECK_FORMAT_1("%R", "None", Py_None); + CHECK_FORMAT_2("%V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); + + // Strings: width < length + CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); + CHECK_FORMAT_1("%1U", "None", unicode); + CHECK_FORMAT_1("%1A", "None", Py_None); + CHECK_FORMAT_1("%1S", "None", Py_None); + CHECK_FORMAT_1("%1R", "None", Py_None); + CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); + + // Strings: width > length + CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); + CHECK_FORMAT_1("%5U", " None", unicode); + CHECK_FORMAT_1("%5A", " None", Py_None); + CHECK_FORMAT_1("%5S", " None", Py_None); + CHECK_FORMAT_1("%5R", " None", Py_None); + CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); + CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); + + // Strings: precision < length + CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); + CHECK_FORMAT_1("%.1U", "N", unicode); + CHECK_FORMAT_1("%.1A", "N", Py_None); + CHECK_FORMAT_1("%.1S", "N", Py_None); + CHECK_FORMAT_1("%.1R", "N", Py_None); + CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); + CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); + + // Strings: precision > length + CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); + CHECK_FORMAT_1("%.5U", "None", unicode); + CHECK_FORMAT_1("%.5A", "None", Py_None); + CHECK_FORMAT_1("%.5S", "None", Py_None); + CHECK_FORMAT_1("%.5R", "None", Py_None); + CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); + + // Strings: precision < length, width > length + CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); + CHECK_FORMAT_1("%5.1U", " N", unicode); + CHECK_FORMAT_1("%5.1A", " N", Py_None); + CHECK_FORMAT_1("%5.1S", " N", Py_None); + CHECK_FORMAT_1("%5.1R", " N", Py_None); + CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); + CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); + + // Strings: width < length, precision > length + CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); + CHECK_FORMAT_1("%1.5U", "None", unicode); + CHECK_FORMAT_1("%1.5A", "None", Py_None); + CHECK_FORMAT_1("%1.5S", "None", Py_None); + CHECK_FORMAT_1("%1.5R", "None", Py_None); + CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); + + Py_XDECREF(unicode); + Py_RETURN_NONE; + + Fail: + Py_XDECREF(result); + Py_XDECREF(unicode); + return NULL; + +#undef CHECK_FORMAT_2 +#undef CHECK_FORMAT_1 +#undef CHECK_FORMAT_0 +} + +static PyMethodDef TestMethods[] = { + {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, + {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, + {"test_unicode_compare_with_ascii", + test_unicode_compare_with_ascii, METH_NOARGS}, + {"test_string_from_format", test_string_from_format, METH_NOARGS}, + {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, + {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, + {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, + {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, + {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, + {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_decode", unicode_decode, METH_VARARGS}, + {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, + {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, + {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, + {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_asutf8string", unicode_asutf8string, METH_O}, + {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, + {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, + {"unicode_asutf16string", unicode_asutf16string, METH_O}, + {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, + {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, + {"unicode_asutf32string", unicode_asutf32string, METH_O}, + {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, + {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, + {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, + {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, + {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, + {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, + {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, + {"unicode_asasciistring", unicode_asasciistring, METH_O}, + {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, + {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, +#ifdef MS_WINDOWS + {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, + {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, + {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, + {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, + {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, +#endif /* MS_WINDOWS */ + {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, + {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, + {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, + {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, + {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, + {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, + {"unicode_concat", unicode_concat, METH_VARARGS}, + {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, + {"unicode_split", unicode_split, METH_VARARGS}, + {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, + {"unicode_partition", unicode_partition, METH_VARARGS}, + {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, + {"unicode_translate", unicode_translate, METH_VARARGS}, + {"unicode_join", unicode_join, METH_VARARGS}, + {"unicode_count", unicode_count, METH_VARARGS}, + {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, + {"unicode_find", unicode_find, METH_VARARGS}, + {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_replace", unicode_replace, METH_VARARGS}, + {"unicode_compare", unicode_compare, METH_VARARGS}, + {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, + {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, + {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, + {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, + {"unicode_format", unicode_format, METH_VARARGS}, + {"unicode_contains", unicode_contains, METH_VARARGS}, + {"unicode_isidentifier", unicode_isidentifier, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Unicode(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/util.h b/Modules/_testlimitedcapi/util.h new file mode 100644 index 00000000000000..f26d7656a10138 --- /dev/null +++ b/Modules/_testlimitedcapi/util.h @@ -0,0 +1,33 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + +/* Marker to check that pointer value was set. */ +static const char uninitialized[] = "uninitialized"; +#define UNINITIALIZED_PTR ((void *)uninitialized) +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c similarity index 80% rename from Modules/_testcapi/vectorcall_limited.c rename to Modules/_testlimitedcapi/vectorcall_limited.c index 3e81903098f954..5ef97ca8a063e1 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -1,13 +1,18 @@ /* Test Vectorcall in the limited API */ -#define Py_LIMITED_API 0x030c0000 // 3.12 +// Need limited C API version 3.12 for PyObject_Vectorcall() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include "clinic/vectorcall_limited.c.h" /*[clinic input] -module _testcapi +module _testlimitedcapi [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -35,15 +40,15 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) } /*[clinic input] -_testcapi.call_vectorcall +_testlimitedcapi.call_vectorcall callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ +_testlimitedcapi_call_vectorcall(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=9cbb7832263a8eef input=0743636c12dccb28]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -88,15 +93,15 @@ _testcapi_call_vectorcall(PyObject *module, PyObject *callable) } /*[clinic input] -_testcapi.call_vectorcall_method +_testlimitedcapi.call_vectorcall_method callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ +_testlimitedcapi_call_vectorcall_method(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=4558323a46cc09eb input=a736f7dbf15f1be5]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -162,7 +167,7 @@ static PyType_Slot LimitedVectorallClass_slots[] = { }; static PyType_Spec LimitedVectorCallClass_spec = { - .name = "_testcapi.LimitedVectorCallClass", + .name = "_testlimitedcapi.LimitedVectorCallClass", .basicsize = (int)(sizeof(PyObject) + sizeof(vectorcallfunc)), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_VECTORCALL @@ -171,13 +176,14 @@ static PyType_Spec LimitedVectorCallClass_spec = { }; static PyMethodDef TestMethods[] = { - _TESTCAPI_CALL_VECTORCALL_METHODDEF - _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF {NULL}, }; int -_PyTestCapi_Init_VectorcallLimited(PyObject *m) { +_PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } @@ -190,6 +196,6 @@ _PyTestCapi_Init_VectorcallLimited(PyObject *m) { if (PyModule_AddType(m, (PyTypeObject *)LimitedVectorCallClass) < 0) { return -1; } - + Py_DECREF(LimitedVectorCallClass); return 0; } diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index fdef06168bfc86..886b260aceb20d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -6,6 +6,7 @@ #endif #include "Python.h" +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_namespace.h" // _PyNamespace_New() /* State for testing module state access from methods */ @@ -430,6 +431,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -518,13 +520,18 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ +static PyModuleDef_Slot nonascii_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef def_nonascii_latin = { \ PyModuleDef_HEAD_INIT, /* m_base */ "_testmultiphase_nonascii_latin", /* m_name */ PyDoc_STR("Module named in Czech"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -542,7 +549,7 @@ static PyModuleDef def_nonascii_kana = { \ PyDoc_STR("Module named in Japanese"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -756,6 +763,7 @@ static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -777,6 +785,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -799,6 +808,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -821,6 +831,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -856,6 +867,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -888,6 +900,9 @@ PyInit__test_module_state_shared(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { Py_DECREF(module); @@ -902,6 +917,7 @@ PyInit__test_module_state_shared(void) static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -919,6 +935,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -939,6 +956,7 @@ static PyModuleDef_Slot shared_gil_only_slots[] = { We put it here explicitly to draw attention to the contrast with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index c42a15a0eff494..066e0dbfb63fbf 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,6 +1,198 @@ /* Testing module for single-phase initialization of extension modules - */ + +This file contains 8 distinct modules, meaning each as its own name +and its own init function (PyInit_...). The default import system will +only find the one matching the filename: _testsinglephase. To load the +others you must do so manually. For example: + +```python +name = '_testsinglephase_base_wrapper' +filename = _testsinglephase.__file__ +loader = importlib.machinery.ExtensionFileLoader(name, filename) +spec = importlib.util.spec_from_file_location(name, filename, loader=loader) +mod = importlib._bootstrap._load(spec) +``` + +Here are the 8 modules: + +* _testsinglephase + * def: _testsinglephase_basic, + * m_name: "_testsinglephase" + * m_size: -1 + * state + * process-global + * initialized_count (default to -1; will never be 0) + * module (see module state below) + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. clear .module + 3. initialize .module: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions + * (3 common, see below) + * initialized_count() - return .module.initialized_count + * import system + * caches + * global extensions cache: yes + * def.m_base.m_copy: yes + * def.m_base.m_init: no + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + 1. get init function from shared object file + 2. run init function + 3. copy __dict__ into def.m_base.m_copy + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. get module from sys.modules + 3. update module with contents of def.m_base.m_copy + * already loaded in other interpreter (already in global cache) + * same as reload, but create new module and update *it* + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * same as main interpreter + * unload: never (all single-phase init modules) +* _testsinglephase_basic_wrapper + * identical to _testsinglephase except module name +* _testsinglephase_basic_copy + * def: static local variable in init function + * m_name: "_testsinglephase_basic_copy" + * m_size: -1 + * state: same as _testsinglephase + * init function: same as _testsinglephase + * functions: same as _testsinglephase + * import system: same as _testsinglephase +* _testsinglephase_with_reinit + * def: _testsinglephase_with_reinit, + * m_name: "_testsinglephase_with_reinit" + * m_size: 0 + * state + * process-global state: no + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. initialize temporary module state (local var): see module state below + 3. initialize module: set initial __dict__ + * functions: see common functions below + * import system + * caches + * global extensions cache: only if loaded in main interpreter + * def.m_base.m_copy: no + * def.m_base.m_init: only if loaded in the main interpreter + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + * (same as _testsinglephase except step 3) + 1. get init function from shared object file + 2. run init function + 3. set def.m_base.m_init to the init function + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. call def->m_base.m_init to get a new module object + 3. replace the existing module in sys.modules + * already loaded in other interpreter (already in global cache) + * same as reload (since will only be in cache for main interp) + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * initial (not already in global cache) + * (same as main interpreter except skip steps 3 & 4 there) + 1. get init function from shared object file + 2. run init function + ... + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + * same as initial (load from scratch) + * already loaded in other interpreter (already in global cache) + * same as initial (load from scratch) + * not in any sys.modules, still in global cache + * same as initial (load from scratch) + * unload: never (all single-phase init modules) +* _testsinglephase_with_state + * def: _testsinglephase_with_state, + * m_name: "_testsinglephase_with_state" + * m_size: sizeof(module_state) + * state + * process-global: no + * module state: see module state below + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 3. initialize module state: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions: see common functions below + * import system: same as _testsinglephase_basic_copy +* _testsinglephase_check_cache_first + * def: _testsinglepahse_check_cache_first + * m_name: "_testsinglephase_check_cache_first" + * m_size: -1 + * state: none + * init function: + * tries PyState_FindModule() first + * otherwise creates empty module + * functions: none + * import system: same as _testsinglephase +* _testsinglephase_with_reinit_check_cache_first + * def: _testsinglepahse_with_reinit_check_cache_first + * m_name: "_testsinglephase_with_reinit_check_cache_first" + * m_size: 0 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_reinit +* _testsinglephase_with_state_check_cache_first + * def: _testsinglepahse_with_state_check_cache_first + * m_name: "_testsinglephase_with_state_check_cache_first" + * m_size: 42 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_state + +Module state: + +* fields + * initialized - when the module was first initialized + * *error + * *int_const + * *str_const +* initialization + 1. set state.initialized to the current time + 2. set state.error to a new exception class + 3. set state->int_const to int(1969) + 4. set state->str_const to "something different" + +Common initial __dict__: + +* error: state.error +* int_const: state.int_const +* str_const: state.str_const +* _module_initialized: state.initialized + +Common functions: + +* look_up_self() - return the module from the per-interpreter "by-index" cache +* sum() - return a + b +* state_initialized() - return state->initialized (or None if m_size == 0) + +See Python/import.c, especially the long comments, for more about +single-phase init modules. +*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -8,11 +200,10 @@ //#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_time.h" // _PyTime_t typedef struct { - _PyTime_t initialized; + PyTime_t initialized; PyObject *error; PyObject *int_const; PyObject *str_const; @@ -67,17 +258,17 @@ clear_state(module_state *state) } static int -_set_initialized(_PyTime_t *initialized) +_set_initialized(PyTime_t *initialized) { /* We go strictly monotonic to ensure each time is unique. */ - _PyTime_t prev; - if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + PyTime_t prev; + if (PyTime_Monotonic(&prev) != 0) { return -1; } /* We do a busy sleep since the interval should be super short. */ - _PyTime_t t; + PyTime_t t; do { - if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + if (PyTime_Monotonic(&t) != 0) { return -1; } } while (t == prev); @@ -136,7 +327,7 @@ init_module(PyObject *module, module_state *state) return -1; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); if (PyModule_Add(module, "_module_initialized", PyFloat_FromDouble(d)) < 0) { return -1; } @@ -157,7 +348,7 @@ common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) if (state == NULL) { Py_RETURN_NONE; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); return PyFloat_FromDouble(d); } @@ -306,6 +497,9 @@ init__testsinglephase_basic(PyModuleDef *def) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = &global_state.module; // It may have been set by a previous run or under a different name. @@ -397,6 +591,9 @@ PyInit__testsinglephase_with_reinit(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif assert(get_module_state(module) == NULL); @@ -459,6 +656,9 @@ PyInit__testsinglephase_with_state(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = get_module_state(module); assert(state != NULL); @@ -476,3 +676,67 @@ PyInit__testsinglephase_with_state(void) finally: return module; } + + +/****************************************************/ +/* the _testsinglephase_*_check_cache_first modules */ +/****************************************************/ + +/* Each of these modules should only be freshly loaded. That means + clearing the caches and each module def's m_base after each load. */ + +static struct PyModuleDef _testsinglephase_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_check_cache_first"), + .m_size = -1, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_check_cache_first(void) +{ + assert(_testsinglephase_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_reinit_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit_check_cache_first"), + .m_size = 0, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit_check_cache_first(void) +{ + assert(_testsinglephase_with_reinit_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_reinit_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_reinit_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_state_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state_check_cache_first"), + .m_size = 42, // not used +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state_check_cache_first(void) +{ + assert(_testsinglephase_with_state_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_state_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_state_check_cache_first); +} diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 86bd560b92ba6b..39d309729d88b8 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1,16 +1,15 @@ - /* Thread module */ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_MakePendingCalls() -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_interp.h" // _PyInterpreterState.threads.count +#include "pycore_lock.h" #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include // offsetof() @@ -21,16 +20,20 @@ // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError - // Forward declarations static struct PyModuleDef thread_module; - +// Module state typedef struct { PyTypeObject *excepthook_type; PyTypeObject *lock_type; PyTypeObject *local_type; PyTypeObject *local_dummy_type; + PyTypeObject *thread_handle_type; + + // Linked list of handles to all non-daemon threads created by the + // threading module. We wait for these to finish at shutdown. + struct llist_node shutdown_handles; } thread_module_state; static inline thread_module_state* @@ -41,6 +44,658 @@ get_thread_state(PyObject *module) return (thread_module_state *)state; } +// _ThreadHandle type + +// Handles state transitions according to the following diagram: +// +// NOT_STARTED -> STARTING -> RUNNING -> DONE +// | ^ +// | | +// +----- error --------+ +typedef enum { + THREAD_HANDLE_NOT_STARTED = 1, + THREAD_HANDLE_STARTING = 2, + THREAD_HANDLE_RUNNING = 3, + THREAD_HANDLE_DONE = 4, +} ThreadHandleState; + +// A handle to wait for thread completion. +// +// This may be used to wait for threads that were spawned by the threading +// module as well as for the "main" thread of the threading module. In the +// former case an OS thread, identified by the `os_handle` field, will be +// associated with the handle. The handle "owns" this thread and ensures that +// the thread is either joined or detached after the handle is destroyed. +// +// Joining the handle is idempotent; the underlying OS thread, if any, is +// joined or detached only once. Concurrent join operations are serialized +// until it is their turn to execute or an earlier operation completes +// successfully. Once a join has completed successfully all future joins +// complete immediately. +// +// This must be separately reference counted because it may be destroyed +// in `thread_run()` after the PyThreadState has been destroyed. +typedef struct { + struct llist_node node; // linked list node (see _pythread_runtime_state) + + // linked list node (see thread_module_state) + struct llist_node shutdown_node; + + // The `ident`, `os_handle`, `has_os_handle`, and `state` fields are + // protected by `mutex`. + PyThread_ident_t ident; + PyThread_handle_t os_handle; + int has_os_handle; + + // Holds a value from the `ThreadHandleState` enum. + int state; + + PyMutex mutex; + + // Set immediately before `thread_run` returns to indicate that the OS + // thread is about to exit. This is used to avoid false positives when + // detecting self-join attempts. See the comment in `ThreadHandle_join()` + // for a more detailed explanation. + PyEvent thread_is_exiting; + + // Serializes calls to `join` and `set_done`. + _PyOnceFlag once; + + Py_ssize_t refcount; +} ThreadHandle; + +static inline int +get_thread_handle_state(ThreadHandle *handle) +{ + PyMutex_Lock(&handle->mutex); + int state = handle->state; + PyMutex_Unlock(&handle->mutex); + return state; +} + +static inline void +set_thread_handle_state(ThreadHandle *handle, ThreadHandleState state) +{ + PyMutex_Lock(&handle->mutex); + handle->state = state; + PyMutex_Unlock(&handle->mutex); +} + +static PyThread_ident_t +ThreadHandle_ident(ThreadHandle *handle) +{ + PyMutex_Lock(&handle->mutex); + PyThread_ident_t ident = handle->ident; + PyMutex_Unlock(&handle->mutex); + return ident; +} + +static int +ThreadHandle_get_os_handle(ThreadHandle *handle, PyThread_handle_t *os_handle) +{ + PyMutex_Lock(&handle->mutex); + int has_os_handle = handle->has_os_handle; + if (has_os_handle) { + *os_handle = handle->os_handle; + } + PyMutex_Unlock(&handle->mutex); + return has_os_handle; +} + +static void +add_to_shutdown_handles(thread_module_state *state, ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&state->shutdown_handles, &handle->shutdown_node); + HEAD_UNLOCK(&_PyRuntime); +} + +static void +clear_shutdown_handles(thread_module_state *state) +{ + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + llist_remove(node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static void +remove_from_shutdown_handles(ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + if (handle->shutdown_node.next != NULL) { + llist_remove(&handle->shutdown_node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static ThreadHandle * +ThreadHandle_new(void) +{ + ThreadHandle *self = + (ThreadHandle *)PyMem_RawCalloc(1, sizeof(ThreadHandle)); + if (self == NULL) { + PyErr_NoMemory(); + return NULL; + } + self->ident = 0; + self->os_handle = 0; + self->has_os_handle = 0; + self->thread_is_exiting = (PyEvent){0}; + self->mutex = (PyMutex){_Py_UNLOCKED}; + self->once = (_PyOnceFlag){0}; + self->state = THREAD_HANDLE_NOT_STARTED; + self->refcount = 1; + + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&_PyRuntime.threads.handles, &self->node); + HEAD_UNLOCK(&_PyRuntime); + + return self; +} + +static void +ThreadHandle_incref(ThreadHandle *self) +{ + _Py_atomic_add_ssize(&self->refcount, 1); +} + +static int +detach_thread(ThreadHandle *self) +{ + if (!self->has_os_handle) { + return 0; + } + // This is typically short so no need to release the GIL + if (PyThread_detach_thread(self->os_handle)) { + fprintf(stderr, "detach_thread: failed detaching thread\n"); + return -1; + } + return 0; +} + +// NB: This may be called after the PyThreadState in `thread_run` has been +// deleted; it cannot call anything that relies on a valid PyThreadState +// existing. +static void +ThreadHandle_decref(ThreadHandle *self) +{ + if (_Py_atomic_add_ssize(&self->refcount, -1) > 1) { + return; + } + + // Remove ourself from the global list of handles + HEAD_LOCK(&_PyRuntime); + if (self->node.next != NULL) { + llist_remove(&self->node); + } + HEAD_UNLOCK(&_PyRuntime); + + assert(self->shutdown_node.next == NULL); + + // It's safe to access state non-atomically: + // 1. This is the destructor; nothing else holds a reference. + // 2. The refcount going to zero is a "synchronizes-with" event; all + // changes from other threads are visible. + if (self->state == THREAD_HANDLE_RUNNING && !detach_thread(self)) { + self->state = THREAD_HANDLE_DONE; + } + + PyMem_RawFree(self); +} + +void +_PyThread_AfterFork(struct _pythread_runtime_state *state) +{ + // gh-115035: We mark ThreadHandles as not joinable early in the child's + // after-fork handler. We do this before calling any Python code to ensure + // that it happens before any ThreadHandles are deallocated, such as by a + // GC cycle. + PyThread_ident_t current = PyThread_get_thread_ident_ex(); + + struct llist_node *node; + llist_for_each_safe(node, &state->handles) { + ThreadHandle *handle = llist_data(node, ThreadHandle, node); + if (handle->ident == current) { + continue; + } + + // Mark all threads as done. Any attempts to join or detach the + // underlying OS thread (if any) could crash. We are the only thread; + // it's safe to set this non-atomically. + handle->state = THREAD_HANDLE_DONE; + handle->once = (_PyOnceFlag){_Py_ONCE_INITIALIZED}; + handle->mutex = (PyMutex){_Py_UNLOCKED}; + _PyEvent_Notify(&handle->thread_is_exiting); + llist_remove(node); + remove_from_shutdown_handles(handle); + } +} + +// bootstate is used to "bootstrap" new threads. Any arguments needed by +// `thread_run()`, which can only take a single argument due to platform +// limitations, are contained in bootstate. +struct bootstate { + PyThreadState *tstate; + PyObject *func; + PyObject *args; + PyObject *kwargs; + ThreadHandle *handle; + PyEvent handle_ready; +}; + +static void +thread_bootstate_free(struct bootstate *boot, int decref) +{ + if (decref) { + Py_DECREF(boot->func); + Py_DECREF(boot->args); + Py_XDECREF(boot->kwargs); + } + ThreadHandle_decref(boot->handle); + PyMem_RawFree(boot); +} + +static void +thread_run(void *boot_raw) +{ + struct bootstate *boot = (struct bootstate *) boot_raw; + PyThreadState *tstate = boot->tstate; + + // Wait until the handle is marked as running + PyEvent_Wait(&boot->handle_ready); + + // `handle` needs to be manipulated after bootstate has been freed + ThreadHandle *handle = boot->handle; + ThreadHandle_incref(handle); + + // gh-108987: If _thread.start_new_thread() is called before or while + // Python is being finalized, thread_run() can called *after*. + // _PyRuntimeState_SetFinalizing() is called. At this point, all Python + // threads must exit, except of the thread calling Py_Finalize() whch holds + // the GIL and must not exit. + // + // At this stage, tstate can be a dangling pointer (point to freed memory), + // it's ok to call _PyThreadState_MustExit() with a dangling pointer. + if (_PyThreadState_MustExit(tstate)) { + // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). + // These functions are called on tstate indirectly by Py_Finalize() + // which calls _PyInterpreterState_Clear(). + // + // Py_DECREF() cannot be called because the GIL is not held: leak + // references on purpose. Python is being finalized anyway. + thread_bootstate_free(boot, 0); + goto exit; + } + + _PyThreadState_Bind(tstate); + PyEval_AcquireThread(tstate); + _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); + + PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_SystemExit)) + /* SystemExit is ignored silently */ + PyErr_Clear(); + else { + PyErr_FormatUnraisable( + "Exception ignored in thread started by %R", boot->func); + } + } + else { + Py_DECREF(res); + } + + thread_bootstate_free(boot, 1); + + _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); + PyThreadState_Clear(tstate); + _PyThreadState_DeleteCurrent(tstate); + +exit: + // Don't need to wait for this thread anymore + remove_from_shutdown_handles(handle); + + _PyEvent_Notify(&handle->thread_is_exiting); + ThreadHandle_decref(handle); + + // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with + // the glibc, pthread_exit() can abort the whole process if dlopen() fails + // to open the libgcc_s.so library (ex: EMFILE error). + return; +} + +static int +force_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_STARTING); + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, + PyObject *kwargs) +{ + // Mark the handle as starting to prevent any other threads from doing so + PyMutex_Lock(&self->mutex); + if (self->state != THREAD_HANDLE_NOT_STARTED) { + PyMutex_Unlock(&self->mutex); + PyErr_SetString(ThreadError, "thread already started"); + return -1; + } + self->state = THREAD_HANDLE_STARTING; + PyMutex_Unlock(&self->mutex); + + // Do all the heavy lifting outside of the mutex. All other operations on + // the handle should fail since the handle is in the starting state. + + // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), + // because it should be possible to call thread_bootstate_free() + // without holding the GIL. + struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); + if (boot == NULL) { + PyErr_NoMemory(); + goto start_failed; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); + if (boot->tstate == NULL) { + PyMem_RawFree(boot); + if (!PyErr_Occurred()) { + PyErr_NoMemory(); + } + goto start_failed; + } + boot->func = Py_NewRef(func); + boot->args = Py_NewRef(args); + boot->kwargs = Py_XNewRef(kwargs); + boot->handle = self; + ThreadHandle_incref(self); + boot->handle_ready = (PyEvent){0}; + + PyThread_ident_t ident; + PyThread_handle_t os_handle; + if (PyThread_start_joinable_thread(thread_run, boot, &ident, &os_handle)) { + PyThreadState_Clear(boot->tstate); + thread_bootstate_free(boot, 1); + PyErr_SetString(ThreadError, "can't start new thread"); + goto start_failed; + } + + // Mark the handle running + PyMutex_Lock(&self->mutex); + assert(self->state == THREAD_HANDLE_STARTING); + self->ident = ident; + self->has_os_handle = 1; + self->os_handle = os_handle; + self->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&self->mutex); + + // Unblock the thread + _PyEvent_Notify(&boot->handle_ready); + + return 0; + +start_failed: + _PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)force_done, self); + return -1; +} + +static int +join_thread(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + PyThread_handle_t os_handle; + if (ThreadHandle_get_os_handle(handle, &os_handle)) { + int err = 0; + Py_BEGIN_ALLOW_THREADS + err = PyThread_join_thread(os_handle); + Py_END_ALLOW_THREADS + if (err) { + PyErr_SetString(ThreadError, "Failed joining thread"); + return -1; + } + } + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +check_started(ThreadHandle *self) +{ + ThreadHandleState state = get_thread_handle_state(self); + if (state < THREAD_HANDLE_RUNNING) { + PyErr_SetString(ThreadError, "thread not started"); + return -1; + } + return 0; +} + +static int +ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) +{ + if (check_started(self) < 0) { + return -1; + } + + // We want to perform this check outside of the `_PyOnceFlag` to prevent + // deadlock in the scenario where another thread joins us and we then + // attempt to join ourselves. However, it's not safe to check thread + // identity once the handle's os thread has finished. We may end up reusing + // the identity stored in the handle and erroneously think we are + // attempting to join ourselves. + // + // To work around this, we set `thread_is_exiting` immediately before + // `thread_run` returns. We can be sure that we are not attempting to join + // ourselves if the handle's thread is about to exit. + if (!_PyEvent_IsSet(&self->thread_is_exiting) && + ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) { + // PyThread_join_thread() would deadlock or error out. + PyErr_SetString(ThreadError, "Cannot join current thread"); + return -1; + } + + // Wait until the deadline for the thread to exit. + PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { + if (deadline) { + // _PyDeadline_Get will return a negative value if the deadline has + // been exceeded. + timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0); + } + + if (timeout_ns) { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + else { + // Timed out + return 0; + } + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)join_thread, + self) == -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +static int +set_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + if (detach_thread(handle) < 0) { + PyErr_SetString(ThreadError, "failed detaching handle"); + return -1; + } + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_set_done(ThreadHandle *self) +{ + if (check_started(self) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)set_done, self) == + -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +// A wrapper around a ThreadHandle. +typedef struct { + PyObject_HEAD + + ThreadHandle *handle; +} PyThreadHandleObject; + +static PyThreadHandleObject * +PyThreadHandleObject_new(PyTypeObject *type) +{ + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + PyThreadHandleObject *self = + (PyThreadHandleObject *)type->tp_alloc(type, 0); + if (self == NULL) { + ThreadHandle_decref(handle); + return NULL; + } + + self->handle = handle; + + return self; +} + +static PyObject * +PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + return (PyObject *)PyThreadHandleObject_new(type); +} + +static int +PyThreadHandleObject_traverse(PyThreadHandleObject *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +PyThreadHandleObject_dealloc(PyThreadHandleObject *self) +{ + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + ThreadHandle_decref(self->handle); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyObject * +PyThreadHandleObject_repr(PyThreadHandleObject *self) +{ + PyThread_ident_t ident = ThreadHandle_ident(self->handle); + return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", + Py_TYPE(self)->tp_name, ident); +} + +static PyObject * +PyThreadHandleObject_get_ident(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(ThreadHandle_ident(self->handle)); +} + +static PyObject * +PyThreadHandleObject_join(PyThreadHandleObject *self, PyObject *args) +{ + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { + return NULL; + } + + PyTime_t timeout_ns = -1; + if (timeout_obj != NULL && timeout_obj != Py_None) { + if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, + _PyTime_ROUND_TIMEOUT) < 0) { + return NULL; + } + } + + if (ThreadHandle_join(self->handle, timeout_ns) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +PyThreadHandleObject_is_done(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } +} + +static PyObject * +PyThreadHandleObject_set_done(PyThreadHandleObject *self, + PyObject *Py_UNUSED(ignored)) +{ + if (ThreadHandle_set_done(self->handle) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyGetSetDef ThreadHandle_getsetlist[] = { + {"ident", (getter)PyThreadHandleObject_get_ident, NULL, NULL}, + {0}, +}; + +static PyMethodDef ThreadHandle_methods[] = { + {"join", (PyCFunction)PyThreadHandleObject_join, METH_VARARGS, NULL}, + {"_set_done", (PyCFunction)PyThreadHandleObject_set_done, METH_NOARGS, NULL}, + {"is_done", (PyCFunction)PyThreadHandleObject_is_done, METH_NOARGS, NULL}, + {0, 0} +}; + +static PyType_Slot ThreadHandle_Type_slots[] = { + {Py_tp_dealloc, (destructor)PyThreadHandleObject_dealloc}, + {Py_tp_repr, (reprfunc)PyThreadHandleObject_repr}, + {Py_tp_getset, ThreadHandle_getsetlist}, + {Py_tp_traverse, PyThreadHandleObject_traverse}, + {Py_tp_methods, ThreadHandle_methods}, + {Py_tp_new, PyThreadHandleObject_tp_new}, + {0, 0} +}; + +static PyType_Spec ThreadHandle_Type_spec = { + "_thread._ThreadHandle", + sizeof(PyThreadHandleObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, + ThreadHandle_Type_slots, +}; /* Lock objects */ @@ -76,74 +731,28 @@ lock_dealloc(lockobject *self) Py_DECREF(tp); } -/* Helper to acquire an interruptible lock with a timeout. If the lock acquire - * is interrupted, signal handlers are run, and if they raise an exception, - * PY_LOCK_INTR is returned. Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE - * are returned, depending on whether the lock can be acquired within the - * timeout. - */ -static PyLockStatus -acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) +static inline PyLockStatus +acquire_timed(PyThread_type_lock lock, PyTime_t timeout) { - PyThreadState *tstate = _PyThreadState_GET(); - _PyTime_t endtime = 0; - if (timeout > 0) { - endtime = _PyDeadline_Init(timeout); - } - - PyLockStatus r; - do { - _PyTime_t microseconds; - microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); - - /* first a simple non-blocking try without releasing the GIL */ - r = PyThread_acquire_lock_timed(lock, 0, 0); - if (r == PY_LOCK_FAILURE && microseconds != 0) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock_timed(lock, microseconds, 1); - Py_END_ALLOW_THREADS - } - - if (r == PY_LOCK_INTR) { - /* Run signal handlers if we were interrupted. Propagate - * exceptions from signal handlers, such as KeyboardInterrupt, by - * passing up PY_LOCK_INTR. */ - if (_PyEval_MakePendingCalls(tstate) < 0) { - return PY_LOCK_INTR; - } - - /* If we're using a timeout, recompute the timeout after processing - * signals, since those can take time. */ - if (timeout > 0) { - timeout = _PyDeadline_Get(endtime); - - /* Check for negative values, since those mean block forever. - */ - if (timeout < 0) { - r = PY_LOCK_FAILURE; - } - } - } - } while (r == PY_LOCK_INTR); /* Retry if we were interrupted. */ - - return r; + return PyThread_acquire_lock_timed_with_retries(lock, timeout); } static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, - _PyTime_t *timeout) + PyTime_t *timeout) { char *kwlist[] = {"blocking", "timeout", NULL}; int blocking = 1; PyObject *timeout_obj = NULL; - const _PyTime_t unset_timeout = _PyTime_FromSeconds(-1); - - *timeout = unset_timeout ; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, &blocking, &timeout_obj)) return -1; + // XXX Use PyThread_ParseTimeoutArg(). + + const PyTime_t unset_timeout = _PyTime_FromSeconds(-1); + *timeout = unset_timeout; + if (timeout_obj && _PyTime_FromSecondsObject(timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) @@ -156,13 +765,13 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, } if (*timeout < 0 && *timeout != unset_timeout) { PyErr_SetString(PyExc_ValueError, - "timeout value must be positive"); + "timeout value must be a non-negative number"); return -1; } if (!blocking) *timeout = 0; else if (*timeout != unset_timeout) { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); if (microseconds > PY_TIMEOUT_MAX) { @@ -177,7 +786,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, static PyObject * lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; + PyTime_t timeout; if (lock_acquire_parse_args(args, kwds, &timeout) < 0) return NULL; @@ -192,8 +801,8 @@ lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(acquire_doc, -"acquire(blocking=True, timeout=-1) -> bool\n\ -(acquire_lock() is an obsolete synonym)\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. Without argument, this blocks if the lock is already\n\ locked (even by the same thread), waiting for another thread to release\n\ @@ -202,6 +811,18 @@ With an argument, this will only block if the argument is true,\n\ and the return value reflects whether the lock is acquired.\n\ The blocking operation is interruptible."); +PyDoc_STRVAR(acquire_lock_doc, +"acquire_lock($self, /, blocking=True, timeout=-1)\n\ +--\n\ +\n\ +An obsolete synonym of acquire()."); + +PyDoc_STRVAR(enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * lock_PyThread_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) { @@ -211,19 +832,31 @@ lock_PyThread_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) return NULL; } - PyThread_release_lock(self->lock_lock); self->locked = 0; + PyThread_release_lock(self->lock_lock); Py_RETURN_NONE; } PyDoc_STRVAR(release_doc, -"release()\n\ -(release_lock() is an obsolete synonym)\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ but it needn't be locked by the same thread that unlocks it."); +PyDoc_STRVAR(release_lock_doc, +"release_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of release()."); + +PyDoc_STRVAR(lock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * lock_locked_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) { @@ -231,11 +864,17 @@ lock_locked_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(locked_doc, -"locked() -> bool\n\ -(locked_lock() is an obsolete synonym)\n\ +"locked($self, /)\n\ +--\n\ \n\ Return whether the lock is in the locked state."); +PyDoc_STRVAR(locked_lock_doc, +"locked_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of locked()."); + static PyObject * lock_repr(lockobject *self) { @@ -258,24 +897,45 @@ lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) } #endif /* HAVE_FORK */ +static lockobject *newlockobject(PyObject *module); + +static PyObject * +lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // convert to AC? + if (!_PyArg_NoKeywords("lock", kwargs)) { + goto error; + } + if (!_PyArg_CheckPositional("lock", PyTuple_GET_SIZE(args), 0, 0)) { + goto error; + } + + PyObject *module = PyType_GetModuleByDef(type, &thread_module); + assert(module != NULL); + return (PyObject *)newlockobject(module); + +error: + return NULL; +} + static PyMethodDef lock_methods[] = { {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, + METH_VARARGS | METH_KEYWORDS, acquire_lock_doc}, {"acquire", _PyCFunction_CAST(lock_PyThread_acquire_lock), METH_VARARGS | METH_KEYWORDS, acquire_doc}, {"release_lock", (PyCFunction)lock_PyThread_release_lock, - METH_NOARGS, release_doc}, + METH_NOARGS, release_lock_doc}, {"release", (PyCFunction)lock_PyThread_release_lock, METH_NOARGS, release_doc}, {"locked_lock", (PyCFunction)lock_locked_lock, - METH_NOARGS, locked_doc}, + METH_NOARGS, locked_lock_doc}, {"locked", (PyCFunction)lock_locked_lock, METH_NOARGS, locked_doc}, {"__enter__", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, + METH_VARARGS | METH_KEYWORDS, enter_doc}, {"__exit__", (PyCFunction)lock_PyThread_release_lock, - METH_VARARGS, release_doc}, + METH_VARARGS, lock_exit_doc}, #ifdef HAVE_FORK {"_at_fork_reinit", (PyCFunction)lock__at_fork_reinit, METH_NOARGS, NULL}, @@ -284,7 +944,10 @@ static PyMethodDef lock_methods[] = { }; PyDoc_STRVAR(lock_doc, -"A lock object is a synchronization primitive. To create a lock,\n\ +"lock()\n\ +--\n\ +\n\ +A lock object is a synchronization primitive. To create a lock,\n\ call threading.Lock(). Methods are:\n\ \n\ acquire() -- lock the lock, possibly blocking until it can be obtained\n\ @@ -307,6 +970,7 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, {Py_tp_members, lock_type_members}, + {Py_tp_new, lock_new}, {0, 0} }; @@ -314,7 +978,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -323,7 +987,7 @@ static PyType_Spec lock_type_spec = { typedef struct { PyObject_HEAD PyThread_type_lock rlock_lock; - unsigned long rlock_owner; + PyThread_ident_t rlock_owner; unsigned long rlock_count; PyObject *in_weakreflist; } rlockobject; @@ -356,18 +1020,26 @@ rlock_dealloc(rlockobject *self) Py_DECREF(tp); } +static bool +rlock_is_owned_by(rlockobject *self, PyThread_ident_t tid) +{ + PyThread_ident_t owner_tid = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return owner_tid == tid && self->rlock_count > 0; +} + static PyObject * rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; - unsigned long tid; + PyTime_t timeout; + PyThread_ident_t tid; PyLockStatus r = PY_LOCK_ACQUIRED; if (lock_acquire_parse_args(args, kwds, &timeout) < 0) return NULL; - tid = PyThread_get_thread_ident(); - if (self->rlock_count > 0 && tid == self->rlock_owner) { + tid = PyThread_get_thread_ident_ex(); + if (rlock_is_owned_by(self, tid)) { unsigned long count = self->rlock_count + 1; if (count <= self->rlock_count) { PyErr_SetString(PyExc_OverflowError, @@ -380,7 +1052,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) r = acquire_timed(self->rlock_lock, timeout); if (r == PY_LOCK_ACQUIRED) { assert(self->rlock_count == 0); - self->rlock_owner = tid; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, tid); self->rlock_count = 1; } else if (r == PY_LOCK_INTR) { @@ -391,7 +1063,8 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(rlock_acquire_doc, -"acquire(blocking=True) -> bool\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. `blocking` indicates whether we should wait\n\ for the lock to be available or not. If `blocking` is False\n\ @@ -406,25 +1079,32 @@ Precisely, if the current thread already holds the lock, its\n\ internal counter is simply incremented. If nobody holds the lock,\n\ the lock is taken and its internal counter initialized to 1."); +PyDoc_STRVAR(rlock_enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) { - unsigned long tid = PyThread_get_thread_ident(); + PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count == 0 || self->rlock_owner != tid) { + if (!rlock_is_owned_by(self, tid)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } if (--self->rlock_count == 0) { - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); } Py_RETURN_NONE; } PyDoc_STRVAR(rlock_release_doc, -"release()\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ @@ -435,14 +1115,21 @@ Do note that if the lock was acquire()d several times in a row by the\n\ current thread, release() needs to be called as many times for the lock\n\ to be available for other threads."); +PyDoc_STRVAR(rlock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * rlock_acquire_restore(rlockobject *self, PyObject *args) { - unsigned long owner; + PyThread_ident_t owner; unsigned long count; int r = 1; - if (!PyArg_ParseTuple(args, "(kk):_acquire_restore", &count, &owner)) + if (!PyArg_ParseTuple(args, "(k" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", + &count, &owner)) return NULL; if (!PyThread_acquire_lock(self->rlock_lock, 0)) { @@ -455,20 +1142,21 @@ rlock_acquire_restore(rlockobject *self, PyObject *args) return NULL; } assert(self->rlock_count == 0); - self->rlock_owner = owner; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, owner); self->rlock_count = count; Py_RETURN_NONE; } PyDoc_STRVAR(rlock_acquire_restore_doc, -"_acquire_restore(state) -> None\n\ +"_acquire_restore($self, state, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) { - unsigned long owner; + PyThread_ident_t owner; unsigned long count; if (self->rlock_count == 0) { @@ -480,42 +1168,46 @@ rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) owner = self->rlock_owner; count = self->rlock_count; self->rlock_count = 0; - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); - return Py_BuildValue("kk", count, owner); + return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); } PyDoc_STRVAR(rlock_release_save_doc, -"_release_save() -> tuple\n\ +"_release_save($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) { - unsigned long tid = PyThread_get_thread_ident(); - return PyLong_FromUnsignedLong( - self->rlock_owner == tid ? self->rlock_count : 0UL); + PyThread_ident_t tid = PyThread_get_thread_ident_ex(); + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return PyLong_FromUnsignedLong(owner == tid ? self->rlock_count : 0UL); } PyDoc_STRVAR(rlock_recursion_count_doc, -"_recursion_count() -> int\n\ +"_recursion_count($self, /)\n\ +--\n\ \n\ For internal use by reentrancy checks."); static PyObject * rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) { - unsigned long tid = PyThread_get_thread_ident(); + PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && self->rlock_owner == tid) { + if (rlock_is_owned_by(self, tid)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; } PyDoc_STRVAR(rlock_is_owned_doc, -"_is_owned() -> bool\n\ +"_is_owned($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); @@ -542,9 +1234,12 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * rlock_repr(rlockobject *self) { - return PyUnicode_FromFormat("<%s %s object owner=%ld count=%lu at %p>", + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return PyUnicode_FromFormat( + "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", self->rlock_count ? "locked" : "unlocked", - Py_TYPE(self)->tp_name, self->rlock_owner, + Py_TYPE(self)->tp_name, owner, self->rlock_count, self); } @@ -580,9 +1275,9 @@ static PyMethodDef rlock_methods[] = { {"_recursion_count", (PyCFunction)rlock_recursion_count, METH_NOARGS, rlock_recursion_count_doc}, {"__enter__", _PyCFunction_CAST(rlock_acquire), - METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, + METH_VARARGS | METH_KEYWORDS, rlock_enter_doc}, {"__exit__", (PyCFunction)rlock_release, - METH_VARARGS, rlock_release_doc}, + METH_VARARGS, rlock_exit_doc}, #ifdef HAVE_FORK {"_at_fork_reinit", (PyCFunction)rlock__at_fork_reinit, METH_NOARGS, NULL}, @@ -808,6 +1503,7 @@ local_new(PyTypeObject *type, PyObject *args, PyObject *kw) } PyObject *module = PyType_GetModuleByDef(type, &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); localobject *self = (localobject *)type->tp_alloc(type, 0); @@ -874,11 +1570,8 @@ local_clear(localobject *self) HEAD_UNLOCK(runtime); while (tstate) { if (tstate->dict) { - PyObject *v = _PyDict_Pop(tstate->dict, self->key, Py_None); - if (v != NULL) { - Py_DECREF(v); - } - else { + if (PyDict_Pop(tstate->dict, self->key, NULL) < 0) { + // Silently ignore error PyErr_Clear(); } } @@ -952,6 +1645,7 @@ static int local_setattro(localobject *self, PyObject *name, PyObject *v) { PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); PyObject *ldict = _ldict(self, state); @@ -984,7 +1678,7 @@ static PyType_Slot local_type_slots[] = { {Py_tp_dealloc, (destructor)local_dealloc}, {Py_tp_getattro, (getattrofunc)local_getattro}, {Py_tp_setattro, (setattrofunc)local_setattro}, - {Py_tp_doc, "Thread-local data"}, + {Py_tp_doc, "_local()\n--\n\nThread-local data"}, {Py_tp_traverse, (traverseproc)local_traverse}, {Py_tp_clear, (inquiry)local_clear}, {Py_tp_new, local_new}, @@ -1004,6 +1698,7 @@ static PyObject * local_getattro(localobject *self, PyObject *name) { PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + assert(module != NULL); thread_module_state *state = get_thread_state(module); PyObject *ldict = _ldict(self, state); @@ -1025,12 +1720,10 @@ local_getattro(localobject *self, PyObject *name) } /* Optimization: just look in dict ourselves */ - PyObject *value = PyDict_GetItemWithError(ldict, name); - if (value != NULL) { - return Py_NewRef(value); - } - if (PyErr_Occurred()) { - return NULL; + PyObject *value; + if (PyDict_GetItemRef(ldict, name, &value) != 0) { + // found or error + return value; } /* Fall back on generic to get __class__ and __dict__ */ @@ -1050,13 +1743,9 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* If the thread-local object is still alive and not being cleared, remove the corresponding local dict */ if (self->dummies != NULL) { - PyObject *ldict; - ldict = PyDict_GetItemWithError(self->dummies, dummyweakref); - if (ldict != NULL) { - PyDict_DelItem(self->dummies, dummyweakref); - } - if (PyErr_Occurred()) + if (PyDict_Pop(self->dummies, dummyweakref, NULL) < 0) { PyErr_WriteUnraisable((PyObject*)self); + } } Py_DECREF(self); Py_RETURN_NONE; @@ -1064,81 +1753,6 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* Module functions */ -struct bootstate { - PyThreadState *tstate; - PyObject *func; - PyObject *args; - PyObject *kwargs; -}; - - -static void -thread_bootstate_free(struct bootstate *boot, int decref) -{ - if (decref) { - Py_DECREF(boot->func); - Py_DECREF(boot->args); - Py_XDECREF(boot->kwargs); - } - PyMem_RawFree(boot); -} - - -static void -thread_run(void *boot_raw) -{ - struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate = boot->tstate; - - // gh-108987: If _thread.start_new_thread() is called before or while - // Python is being finalized, thread_run() can called *after*. - // _PyRuntimeState_SetFinalizing() is called. At this point, all Python - // threads must exit, except of the thread calling Py_Finalize() whch holds - // the GIL and must not exit. - // - // At this stage, tstate can be a dangling pointer (point to freed memory), - // it's ok to call _PyThreadState_MustExit() with a dangling pointer. - if (_PyThreadState_MustExit(tstate)) { - // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). - // These functions are called on tstate indirectly by Py_Finalize() - // which calls _PyInterpreterState_Clear(). - // - // Py_DECREF() cannot be called because the GIL is not held: leak - // references on purpose. Python is being finalized anyway. - thread_bootstate_free(boot, 0); - goto exit; - } - - _PyThreadState_Bind(tstate); - PyEval_AcquireThread(tstate); - tstate->interp->threads.count++; - - PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); - if (res == NULL) { - if (PyErr_ExceptionMatches(PyExc_SystemExit)) - /* SystemExit is ignored silently */ - PyErr_Clear(); - else { - _PyErr_WriteUnraisableMsg("in thread started by", boot->func); - } - } - else { - Py_DECREF(res); - } - - thread_bootstate_free(boot, 1); - - tstate->interp->threads.count--; - PyThreadState_Clear(tstate); - _PyThreadState_DeleteCurrent(tstate); - -exit: - // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with - // the glibc, pthread_exit() can abort the whole process if dlopen() fails - // to open the libgcc_s.so library (ex: EMFILE error). - return; -} - static PyObject * thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1152,15 +1766,50 @@ thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(daemon_threads_allowed_doc, -"daemon_threads_allowed()\n\ +"daemon_threads_allowed($module, /)\n\ +--\n\ \n\ Return True if daemon threads are allowed in the current interpreter,\n\ and False otherwise.\n"); +static int +do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, + PyObject *kwargs, ThreadHandle *handle, int daemon) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { + PyErr_SetString(PyExc_RuntimeError, + "thread is not supported for isolated subinterpreters"); + return -1; + } + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, + "can't create new thread at interpreter shutdown"); + return -1; + } + + if (!daemon) { + // Add the handle before starting the thread to avoid adding a handle + // to a thread that has already finished (i.e. if the thread finishes + // before the call to `ThreadHandle_start()` below returns). + add_to_shutdown_handles(state, handle); + } + + if (ThreadHandle_start(handle, func, args, kwargs) < 0) { + if (!daemon) { + remove_from_shutdown_handles(handle); + } + return -1; + } + + return 0; +} + static PyObject * -thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) +thread_PyThread_start_new_thread(PyObject *module, PyObject *fargs) { PyObject *func, *args, *kwargs = NULL; + thread_module_state *state = get_thread_state(module); if (!PyArg_UnpackTuple(fargs, "start_new_thread", 2, 3, &func, &args, &kwargs)) @@ -1186,57 +1835,112 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) return NULL; } - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { - PyErr_SetString(PyExc_RuntimeError, - "thread is not supported for isolated subinterpreters"); + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { return NULL; } - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, - "can't create new thread at interpreter shutdown"); + + int st = + do_start_new_thread(state, func, args, kwargs, handle, /*daemon=*/1); + if (st < 0) { + ThreadHandle_decref(handle); return NULL; } + PyThread_ident_t ident = ThreadHandle_ident(handle); + ThreadHandle_decref(handle); + return PyLong_FromUnsignedLongLong(ident); +} - // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), - // because it should be possible to call thread_bootstate_free() - // without holding the GIL. - struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); - if (boot == NULL) { - return PyErr_NoMemory(); +PyDoc_STRVAR(start_new_thread_doc, +"start_new_thread($module, function, args, kwargs={}, /)\n\ +--\n\ +\n\ +Start a new thread and return its identifier.\n\ +\n\ +The thread will call the function with positional arguments from the\n\ +tuple args and keyword arguments taken from the optional dictionary\n\ +kwargs. The thread exits when the function returns; the return value\n\ +is ignored. The thread will also exit when the function raises an\n\ +unhandled exception; a stack trace will be printed unless the exception\n\ +is SystemExit."); + +PyDoc_STRVAR(start_new_doc, +"start_new($module, function, args, kwargs={}, /)\n\ +--\n\ +\n\ +An obsolete synonym of start_new_thread()."); + +static PyObject * +thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, + PyObject *fkwargs) +{ + static char *keywords[] = {"function", "handle", "daemon", NULL}; + PyObject *func = NULL; + int daemon = 1; + thread_module_state *state = get_thread_state(module); + PyObject *hobj = NULL; + if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { + return NULL; } - boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); - if (boot->tstate == NULL) { - PyMem_RawFree(boot); - if (!PyErr_Occurred()) { - return PyErr_NoMemory(); - } + + if (!PyCallable_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "thread function must be callable"); return NULL; } - boot->func = Py_NewRef(func); - boot->args = Py_NewRef(args); - boot->kwargs = Py_XNewRef(kwargs); - unsigned long ident = PyThread_start_new_thread(thread_run, (void*) boot); - if (ident == PYTHREAD_INVALID_THREAD_ID) { - PyErr_SetString(ThreadError, "can't start new thread"); - PyThreadState_Clear(boot->tstate); - thread_bootstate_free(boot, 1); + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, + hobj) < 0) { + return NULL; + } + + if (hobj == Py_None) { + hobj = (PyObject *)PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + } + else { + Py_INCREF(hobj); + } + + PyObject* args = PyTuple_New(0); + if (args == NULL) { + return NULL; + } + int st = do_start_new_thread(state, func, args, + /*kwargs=*/ NULL, ((PyThreadHandleObject*)hobj)->handle, daemon); + Py_DECREF(args); + if (st < 0) { + Py_DECREF(hobj); return NULL; } - return PyLong_FromUnsignedLong(ident); + return (PyObject *) hobj; } -PyDoc_STRVAR(start_new_doc, -"start_new_thread(function, args[, kwargs])\n\ -(start_new() is an obsolete synonym)\n\ +PyDoc_STRVAR(start_joinable_doc, +"start_joinable_thread($module, /, function, handle=None, daemon=True)\n\ +--\n\ +\n\ +*For internal use only*: start a new thread.\n\ \n\ -Start a new thread and return its identifier. The thread will call the\n\ -function with positional arguments from the tuple args and keyword arguments\n\ -taken from the optional dictionary kwargs. The thread exits when the\n\ -function returns; the return value is ignored. The thread will also exit\n\ -when the function raises an unhandled exception; a stack trace will be\n\ -printed unless the exception is SystemExit.\n"); +Like start_new_thread(), this starts a new thread calling the given function.\n\ +Unlike start_new_thread(), this returns a handle object with methods to join\n\ +or detach the given thread.\n\ +This function is not for third-party code, please use the\n\ +`threading` module instead. During finalization the runtime will not wait for\n\ +the thread to exit if daemon is True. If handle is provided it must be a\n\ +newly created thread._ThreadHandle instance."); static PyObject * thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) @@ -1246,12 +1950,18 @@ thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(exit_doc, -"exit()\n\ -(exit_thread() is an obsolete synonym)\n\ +"exit($module, /)\n\ +--\n\ \n\ This is synonymous to ``raise SystemExit''. It will cause the current\n\ thread to exit silently unless the exception is caught."); +PyDoc_STRVAR(exit_thread_doc, +"exit_thread($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of exit()."); + static PyObject * thread_PyThread_interrupt_main(PyObject *self, PyObject *args) { @@ -1268,7 +1978,8 @@ thread_PyThread_interrupt_main(PyObject *self, PyObject *args) } PyDoc_STRVAR(interrupt_doc, -"interrupt_main(signum=signal.SIGINT, /)\n\ +"interrupt_main($module, signum=signal.SIGINT, /)\n\ +--\n\ \n\ Simulate the arrival of the given signal in the main thread,\n\ where the corresponding signal handler will be executed.\n\ @@ -1278,34 +1989,39 @@ A subthread can use this function to interrupt the main thread.\n\ Note: the default signal handler for SIGINT raises ``KeyboardInterrupt``." ); -static lockobject *newlockobject(PyObject *module); - static PyObject * thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) { return (PyObject *) newlockobject(module); } -PyDoc_STRVAR(allocate_doc, -"allocate_lock() -> lock object\n\ -(allocate() is an obsolete synonym)\n\ +PyDoc_STRVAR(allocate_lock_doc, +"allocate_lock($module, /)\n\ +--\n\ \n\ Create a new lock object. See help(type(threading.Lock())) for\n\ information about locks."); +PyDoc_STRVAR(allocate_doc, +"allocate($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of allocate_lock()."); + static PyObject * thread_get_ident(PyObject *self, PyObject *Py_UNUSED(ignored)) { - unsigned long ident = PyThread_get_thread_ident(); + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); if (ident == PYTHREAD_INVALID_THREAD_ID) { PyErr_SetString(ThreadError, "no current thread ident"); return NULL; } - return PyLong_FromUnsignedLong(ident); + return PyLong_FromUnsignedLongLong(ident); } PyDoc_STRVAR(get_ident_doc, -"get_ident() -> integer\n\ +"get_ident($module, /)\n\ +--\n\ \n\ Return a non-zero integer that uniquely identifies the current thread\n\ amongst other threads that exist simultaneously.\n\ @@ -1324,7 +2040,8 @@ thread_get_native_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(get_native_id_doc, -"get_native_id() -> integer\n\ +"get_native_id($module, /)\n\ +--\n\ \n\ Return a non-negative integer identifying the thread as reported\n\ by the OS (kernel). This may be used to uniquely identify a\n\ @@ -1335,13 +2052,13 @@ static PyObject * thread__count(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->threads.count); + return PyLong_FromSsize_t(_Py_atomic_load_ssize(&interp->threads.count)); } PyDoc_STRVAR(_count_doc, -"_count() -> integer\n\ +"_count($module, /)\n\ +--\n\ \n\ -\ Return the number of currently running Python threads, excluding\n\ the main thread. The returned number comprises all threads created\n\ through `start_new_thread()` as well as `threading.Thread`, and not\n\ @@ -1350,67 +2067,6 @@ yet finished.\n\ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); -static void -release_sentinel(void *weakref_raw) -{ - PyObject *weakref = _PyObject_CAST(weakref_raw); - - /* Tricky: this function is called when the current thread state - is being deleted. Therefore, only simple C code can safely - execute here. */ - lockobject *lock = (lockobject *)_PyWeakref_GET_REF(weakref); - if (lock != NULL) { - if (lock->locked) { - PyThread_release_lock(lock->lock_lock); - lock->locked = 0; - } - Py_DECREF(lock); - } - - /* Deallocating a weakref with a NULL callback only calls - PyObject_GC_Del(), which can't call any Python code. */ - Py_DECREF(weakref); -} - -static PyObject * -thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - PyObject *wr; - PyThreadState *tstate = _PyThreadState_GET(); - lockobject *lock; - - if (tstate->on_delete_data != NULL) { - /* We must support the re-creation of the lock from a - fork()ed child. */ - assert(tstate->on_delete == &release_sentinel); - wr = (PyObject *) tstate->on_delete_data; - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - Py_DECREF(wr); - } - lock = newlockobject(module); - if (lock == NULL) - return NULL; - /* The lock is owned by whoever called _set_sentinel(), but the weakref - hangs to the thread state. */ - wr = PyWeakref_NewRef((PyObject *) lock, NULL); - if (wr == NULL) { - Py_DECREF(lock); - return NULL; - } - tstate->on_delete_data = (void *) wr; - tstate->on_delete = &release_sentinel; - return (PyObject *) lock; -} - -PyDoc_STRVAR(_set_sentinel_doc, -"_set_sentinel() -> lock\n\ -\n\ -Set a sentinel lock that will be released when the current thread\n\ -state is finalized (after it is untied from the interpreter).\n\ -\n\ -This is a private API for the threading module."); - static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1446,7 +2102,8 @@ thread_stack_size(PyObject *self, PyObject *args) } PyDoc_STRVAR(stack_size_doc, -"stack_size([size]) -> size\n\ +"stack_size($module, size=0, /)\n\ +--\n\ \n\ Return the thread stack size used when creating new threads. The\n\ optional size argument specifies the stack size (in bytes) to be used\n\ @@ -1488,8 +2145,8 @@ thread_excepthook_file(PyObject *file, PyObject *exc_type, PyObject *exc_value, Py_DECREF(name); } else { - unsigned long ident = PyThread_get_thread_ident(); - PyObject *str = PyUnicode_FromFormat("%lu", ident); + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); + PyObject *str = PyUnicode_FromFormat("%" PY_FORMAT_THREAD_IDENT_T, ident); if (str != NULL) { if (PyFile_WriteObject(str, file, Py_PRINT_RAW) < 0) { Py_DECREF(str); @@ -1601,7 +2258,8 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"excepthook(exc_type, exc_value, exc_traceback, thread)\n\ +"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +--\n\ \n\ Handle uncaught Thread.run() exception."); @@ -1613,23 +2271,117 @@ thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(thread__is_main_interpreter_doc, -"_is_main_interpreter()\n\ +"_is_main_interpreter($module, /)\n\ +--\n\ \n\ Return True if the current interpreter is the main Python interpreter."); +static PyObject * +thread_shutdown(PyObject *self, PyObject *args) +{ + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); + thread_module_state *state = get_thread_state(self); + + for (;;) { + ThreadHandle *handle = NULL; + + // Find a thread that's not yet finished. + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + ThreadHandle *cur = llist_data(node, ThreadHandle, shutdown_node); + if (cur->ident != ident) { + ThreadHandle_incref(cur); + handle = cur; + break; + } + } + HEAD_UNLOCK(&_PyRuntime); + + if (!handle) { + // No more threads to wait on! + break; + } + + // Wait for the thread to finish. If we're interrupted, such + // as by a ctrl-c we print the error and exit early. + if (ThreadHandle_join(handle, -1) < 0) { + PyErr_WriteUnraisable(NULL); + ThreadHandle_decref(handle); + Py_RETURN_NONE; + } + + ThreadHandle_decref(handle); + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(shutdown_doc, +"_shutdown($module, /)\n\ +--\n\ +\n\ +Wait for all non-daemon threads (other than the calling thread) to stop."); + +static PyObject * +thread__make_thread_handle(PyObject *module, PyObject *identobj) +{ + thread_module_state *state = get_thread_state(module); + if (!PyLong_Check(identobj)) { + PyErr_SetString(PyExc_TypeError, "ident must be an integer"); + return NULL; + } + PyThread_ident_t ident = PyLong_AsUnsignedLongLong(identobj); + if (PyErr_Occurred()) { + return NULL; + } + PyThreadHandleObject *hobj = + PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + PyMutex_Lock(&hobj->handle->mutex); + hobj->handle->ident = ident; + hobj->handle->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&hobj->handle->mutex); + return (PyObject*) hobj; +} + +PyDoc_STRVAR(thread__make_thread_handle_doc, +"_make_thread_handle($module, ident, /)\n\ +--\n\ +\n\ +Internal only. Make a thread handle for threads not spawned\n\ +by the _thread or threading module."); + +static PyObject * +thread__get_main_thread_ident(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(_PyRuntime.main_thread); +} + +PyDoc_STRVAR(thread__get_main_thread_ident_doc, +"_get_main_thread_ident($module, /)\n\ +--\n\ +\n\ +Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ +of the main interpreter."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, start_new_doc}, + METH_VARARGS, start_new_thread_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, + {"start_joinable_thread", _PyCFunction_CAST(thread_PyThread_start_joinable_thread), + METH_VARARGS | METH_KEYWORDS, start_joinable_doc}, {"daemon_threads_allowed", (PyCFunction)thread_daemon_threads_allowed, METH_NOARGS, daemon_threads_allowed_doc}, {"allocate_lock", thread_PyThread_allocate_lock, - METH_NOARGS, allocate_doc}, + METH_NOARGS, allocate_lock_doc}, {"allocate", thread_PyThread_allocate_lock, METH_NOARGS, allocate_doc}, {"exit_thread", thread_PyThread_exit_thread, - METH_NOARGS, exit_doc}, + METH_NOARGS, exit_thread_doc}, {"exit", thread_PyThread_exit_thread, METH_NOARGS, exit_doc}, {"interrupt_main", (PyCFunction)thread_PyThread_interrupt_main, @@ -1644,12 +2396,16 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, - {"_set_sentinel", thread__set_sentinel, - METH_NOARGS, _set_sentinel_doc}, {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, {"_is_main_interpreter", thread__is_main_interpreter, METH_NOARGS, thread__is_main_interpreter_doc}, + {"_shutdown", thread_shutdown, + METH_NOARGS, shutdown_doc}, + {"_make_thread_handle", thread__make_thread_handle, + METH_O, thread__make_thread_handle_doc}, + {"_get_main_thread_ident", thread__get_main_thread_ident, + METH_NOARGS, thread__get_main_thread_ident_doc}, {NULL, NULL} /* sentinel */ }; @@ -1665,11 +2421,24 @@ thread_module_exec(PyObject *module) // Initialize the C thread library PyThread_init_thread(); + // _ThreadHandle + state->thread_handle_type = (PyTypeObject *)PyType_FromSpec(&ThreadHandle_Type_spec); + if (state->thread_handle_type == NULL) { + return -1; + } + if (PyDict_SetItemString(d, "_ThreadHandle", (PyObject *)state->thread_handle_type) < 0) { + return -1; + } + // Lock - state->lock_type = (PyTypeObject *)PyType_FromSpec(&lock_type_spec); + state->lock_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &lock_type_spec, NULL); if (state->lock_type == NULL) { return -1; } + if (PyModule_AddType(module, state->lock_type) < 0) { + return -1; + } + // Old alias: lock -> LockType if (PyDict_SetItemString(d, "LockType", (PyObject *)state->lock_type) < 0) { return -1; } @@ -1716,7 +2485,7 @@ thread_module_exec(PyObject *module) // TIMEOUT_MAX double timeout_max = (double)PY_TIMEOUT_MAX * 1e-6; - double time_max = _PyTime_AsSecondsDouble(_PyTime_MAX); + double time_max = PyTime_AsSecondsDouble(PyTime_MAX); timeout_max = Py_MIN(timeout_max, time_max); // Round towards minus infinity timeout_max = floor(timeout_max); @@ -1726,6 +2495,8 @@ thread_module_exec(PyObject *module) return -1; } + llist_init(&state->shutdown_handles); + return 0; } @@ -1738,6 +2509,7 @@ thread_module_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->lock_type); Py_VISIT(state->local_type); Py_VISIT(state->local_dummy_type); + Py_VISIT(state->thread_handle_type); return 0; } @@ -1749,6 +2521,11 @@ thread_module_clear(PyObject *module) Py_CLEAR(state->lock_type); Py_CLEAR(state->local_type); Py_CLEAR(state->local_dummy_type); + Py_CLEAR(state->thread_handle_type); + // Remove any remaining handles (e.g. if shutdown exited early due to + // interrupt) so that attempts to unlink the handle after our module state + // is destroyed do not crash. + clear_shutdown_handles(state); return 0; } @@ -1767,6 +2544,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f9a18644945c65..1542d3af42f755 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -69,6 +69,12 @@ Copyright (C) 1994 Steen Lumholt. #define USE_DEPRECATED_TOMMATH_API 1 #endif +// As suggested by https://core.tcl-lang.org/tcl/wiki?name=Migrating+C+extensions+to+Tcl+9 +#ifndef TCL_SIZE_MAX +typedef int Tcl_Size; +#define TCL_SIZE_MAX INT_MAX +#endif + #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -306,6 +312,7 @@ typedef struct { int threaded; /* True if tcl_platform[threaded] */ Tcl_ThreadId thread_id; int dispatching; + PyObject *trace; /* We cannot include tclInt.h, as this is internal. So we cache interesting types here. */ const Tcl_ObjType *OldBooleanType; @@ -317,6 +324,7 @@ typedef struct { const Tcl_ObjType *BignumType; const Tcl_ObjType *ListType; const Tcl_ObjType *StringType; + const Tcl_ObjType *UTF32StringType; } TkappObject; #define Tkapp_Interp(v) (((TkappObject *) (v))->interp) @@ -485,24 +493,28 @@ unicodeFromTclString(const char *s) } static PyObject * -unicodeFromTclObj(Tcl_Obj *value) +unicodeFromTclObj(TkappObject *tkapp, Tcl_Obj *value) { - int len; + Tcl_Size len; #if USE_TCL_UNICODE - int byteorder = NATIVE_BYTEORDER; - const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); - if (sizeof(Tcl_UniChar) == 2) - return PyUnicode_DecodeUTF16((const char *)u, len * 2, - "surrogatepass", &byteorder); - else if (sizeof(Tcl_UniChar) == 4) - return PyUnicode_DecodeUTF32((const char *)u, len * 4, - "surrogatepass", &byteorder); - else - Py_UNREACHABLE(); -#else + if (value->typePtr != NULL && tkapp != NULL && + (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType)) + { + int byteorder = NATIVE_BYTEORDER; + const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); + if (sizeof(Tcl_UniChar) == 2) + return PyUnicode_DecodeUTF16((const char *)u, len * 2, + "surrogatepass", &byteorder); + else if (sizeof(Tcl_UniChar) == 4) + return PyUnicode_DecodeUTF32((const char *)u, len * 4, + "surrogatepass", &byteorder); + else + Py_UNREACHABLE(); + } +#endif /* USE_TCL_UNICODE */ const char *s = Tcl_GetStringFromObj(value, &len); return unicodeFromTclStringAndSize(s, len); -#endif } /*[clinic input] @@ -515,6 +527,10 @@ class _tkinter.tktimertoken "TkttObject *" "&Tktt_Type_spec" /**** Tkapp Object ****/ +#if TK_MAJOR_VERSION >= 9 +int Tcl_AppInit(Tcl_Interp *); +#endif + #ifndef WITH_APPINIT int Tcl_AppInit(Tcl_Interp *interp) @@ -570,6 +586,7 @@ Tkapp_New(const char *screenName, const char *className, TCL_GLOBAL_ONLY) != NULL; v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; + v->trace = NULL; #ifndef TCL_THREADS if (v->threaded) { @@ -586,14 +603,40 @@ Tkapp_New(const char *screenName, const char *className, } v->OldBooleanType = Tcl_GetObjType("boolean"); - v->BooleanType = Tcl_GetObjType("booleanString"); - v->ByteArrayType = Tcl_GetObjType("bytearray"); + { + Tcl_Obj *value; + int boolValue; + + /* Tcl 8.5 "booleanString" type is not registered + and is renamed to "boolean" in Tcl 9.0. + Based on approach suggested at + https://core.tcl-lang.org/tcl/info/3bb3bcf2da5b */ + value = Tcl_NewStringObj("true", -1); + Tcl_GetBooleanFromObj(NULL, value, &boolValue); + v->BooleanType = value->typePtr; + Tcl_DecrRefCount(value); + + // "bytearray" type is not registered in Tcl 9.0 + value = Tcl_NewByteArrayObj(NULL, 0); + v->ByteArrayType = value->typePtr; + Tcl_DecrRefCount(value); + } v->DoubleType = Tcl_GetObjType("double"); + /* TIP 484 suggests retrieving the "int" type without Tcl_GetObjType("int") + since it is no longer registered in Tcl 9.0. But even though Tcl 8.7 + only uses the "wideInt" type on platforms with 32-bit long, it still has + a registered "int" type, which FromObj() should recognize just in case. */ v->IntType = Tcl_GetObjType("int"); + if (v->IntType == NULL) { + Tcl_Obj *value = Tcl_NewIntObj(0); + v->IntType = value->typePtr; + Tcl_DecrRefCount(value); + } v->WideIntType = Tcl_GetObjType("wideInt"); v->BignumType = Tcl_GetObjType("bignum"); v->ListType = Tcl_GetObjType("list"); v->StringType = Tcl_GetObjType("string"); + v->UTF32StringType = Tcl_GetObjType("utf32string"); /* Delete the 'exit' command, which can screw things up */ Tcl_DeleteCommand(v->interp, "exit"); @@ -735,8 +778,9 @@ newPyTclObject(Tcl_Obj *arg) } static void -PyTclObject_dealloc(PyTclObject *self) +PyTclObject_dealloc(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; PyObject *tp = (PyObject *) Py_TYPE(self); Tcl_DecrRefCount(self->value); Py_XDECREF(self->string); @@ -749,10 +793,11 @@ PyDoc_STRVAR(PyTclObject_string__doc__, "the string representation of this object, either as str or bytes"); static PyObject * -PyTclObject_string(PyTclObject *self, void *ignored) +PyTclObject_string(PyObject *_self, void *ignored) { + PyTclObject *self = (PyTclObject *)_self; if (!self->string) { - self->string = unicodeFromTclObj(self->value); + self->string = unicodeFromTclObj(NULL, self->value); if (!self->string) return NULL; } @@ -760,19 +805,21 @@ PyTclObject_string(PyTclObject *self, void *ignored) } static PyObject * -PyTclObject_str(PyTclObject *self) +PyTclObject_str(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; if (self->string) { return Py_NewRef(self->string); } /* XXX Could cache result if it is non-ASCII. */ - return unicodeFromTclObj(self->value); + return unicodeFromTclObj(NULL, self->value); } static PyObject * -PyTclObject_repr(PyTclObject *self) +PyTclObject_repr(PyObject *_self) { - PyObject *repr, *str = PyTclObject_str(self); + PyTclObject *self = (PyTclObject *)_self; + PyObject *repr, *str = PyTclObject_str(_self); if (str == NULL) return NULL; repr = PyUnicode_FromFormat("<%s object: %R>", @@ -809,23 +856,24 @@ PyTclObject_richcompare(PyObject *self, PyObject *other, int op) PyDoc_STRVAR(get_typename__doc__, "name of the Tcl type"); static PyObject* -get_typename(PyTclObject* obj, void* ignored) +get_typename(PyObject *self, void* ignored) { + PyTclObject *obj = (PyTclObject *)self; return unicodeFromTclString(obj->value->typePtr->name); } static PyGetSetDef PyTclObject_getsetlist[] = { - {"typename", (getter)get_typename, NULL, get_typename__doc__}, - {"string", (getter)PyTclObject_string, NULL, + {"typename", get_typename, NULL, get_typename__doc__}, + {"string", PyTclObject_string, NULL, PyTclObject_string__doc__}, {0}, }; static PyType_Slot PyTclObject_Type_slots[] = { - {Py_tp_dealloc, (destructor)PyTclObject_dealloc}, - {Py_tp_repr, (reprfunc)PyTclObject_repr}, - {Py_tp_str, (reprfunc)PyTclObject_str}, + {Py_tp_dealloc, PyTclObject_dealloc}, + {Py_tp_repr, PyTclObject_repr}, + {Py_tp_str, PyTclObject_str}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_richcompare, PyTclObject_richcompare}, {Py_tp_getset, PyTclObject_getsetlist}, @@ -921,7 +969,8 @@ AsObj(PyObject *value) (unsigned char *)(void *)&wideValue, sizeof(wideValue), PY_LITTLE_ENDIAN, - /* signed */ 1) == 0) { + /* signed */ 1, + /* with_exceptions */ 1) == 0) { return Tcl_NewWideIntObj(wideValue); } PyErr_Clear(); @@ -969,7 +1018,9 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } - if (PyUnicode_IS_ASCII(value)) { + if (PyUnicode_IS_ASCII(value) && + strlen(PyUnicode_DATA(value)) == (size_t)PyUnicode_GET_LENGTH(value)) + { return Tcl_NewStringObj((const char *)PyUnicode_DATA(value), (int)size); } @@ -984,9 +1035,6 @@ AsObj(PyObject *value) "surrogatepass", NATIVE_BYTEORDER); else Py_UNREACHABLE(); -#else - encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); -#endif if (!encoded) { return NULL; } @@ -996,12 +1044,39 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } -#if USE_TCL_UNICODE result = Tcl_NewUnicodeObj((const Tcl_UniChar *)PyBytes_AS_STRING(encoded), (int)(size / sizeof(Tcl_UniChar))); #else + encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + if (strlen(PyBytes_AS_STRING(encoded)) != (size_t)size) { + /* The string contains embedded null characters. + * Tcl needs a null character to be represented as \xc0\x80 in + * the Modified UTF-8 encoding. Otherwise the string can be + * truncated in some internal operations. + * + * NOTE: stringlib_replace() could be used here, but optimizing + * this obscure case isn't worth it unless stringlib_replace() + * was already exposed in the C API for other reasons. */ + Py_SETREF(encoded, + PyObject_CallMethod(encoded, "replace", "y#y#", + "\0", (Py_ssize_t)1, + "\xc0\x80", (Py_ssize_t)2)); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + } + if (size > INT_MAX) { + Py_DECREF(encoded); + PyErr_SetString(PyExc_OverflowError, "string is too long"); + return NULL; + } result = Tcl_NewStringObj(PyBytes_AS_STRING(encoded), (int)size); -#endif +#endif /* USE_TCL_UNICODE */ Py_DECREF(encoded); return result; } @@ -1098,7 +1173,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) Tcl_Interp *interp = Tkapp_Interp(tkapp); if (value->typePtr == NULL) { - return unicodeFromTclObj(value); + return unicodeFromTclObj(tkapp, value); } if (value->typePtr == tkapp->BooleanType || @@ -1107,7 +1182,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ByteArrayType) { - int size; + Tcl_Size size; char *data = (char*)Tcl_GetByteArrayFromObj(value, &size); return PyBytes_FromStringAndSize(data, size); } @@ -1116,14 +1191,6 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return PyFloat_FromDouble(value->internalRep.doubleValue); } - if (value->typePtr == tkapp->IntType) { - long longValue; - if (Tcl_GetLongFromObj(interp, value, &longValue) == TCL_OK) - return PyLong_FromLong(longValue); - /* If there is an error in the long conversion, - fall through to wideInt handling. */ - } - if (value->typePtr == tkapp->IntType || value->typePtr == tkapp->WideIntType) { result = fromWideIntObj(tkapp, value); @@ -1141,8 +1208,8 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ListType) { - int size; - int i, status; + Tcl_Size i, size; + int status; PyObject *elem; Tcl_Obj *tcl_elem; @@ -1168,15 +1235,10 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return result; } - if (value->typePtr == tkapp->StringType) { - return unicodeFromTclObj(value); - } - - if (tkapp->BooleanType == NULL && - strcmp(value->typePtr->name, "booleanString") == 0) { - /* booleanString type is not registered in Tcl */ - tkapp->BooleanType = value->typePtr; - return fromBoolean(tkapp, value); + if (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType) + { + return unicodeFromTclObj(tkapp, value); } if (tkapp->BignumType == NULL && @@ -1202,10 +1264,10 @@ typedef struct Tkapp_CallEvent { Tcl_Condition *done; } Tkapp_CallEvent; -void -Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) +static void +Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, Tcl_Size objc) { - int i; + Tcl_Size i; for (i = 0; i < objc; i++) Tcl_DecrRefCount(objv[i]); if (objv != objStore) @@ -1216,7 +1278,7 @@ Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) interpreter thread, which may or may not be the calling thread. */ static Tcl_Obj** -Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) +Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, Tcl_Size *pobjc) { Tcl_Obj **objv = objStore; Py_ssize_t objc = 0, i; @@ -1264,10 +1326,10 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) Tcl_IncrRefCount(objv[i]); } } - *pobjc = (int)objc; + *pobjc = (Tcl_Size)objc; return objv; finally: - Tkapp_CallDeallocArgs(objv, objStore, (int)objc); + Tkapp_CallDeallocArgs(objv, objStore, (Tcl_Size)objc); return NULL; } @@ -1276,7 +1338,7 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) static PyObject * Tkapp_UnicodeResult(TkappObject *self) { - return unicodeFromTclObj(Tcl_GetObjResult(self->interp)); + return unicodeFromTclObj(self, Tcl_GetObjResult(self->interp)); } @@ -1295,25 +1357,54 @@ Tkapp_ObjectResult(TkappObject *self) res = FromObj(self, value); Tcl_DecrRefCount(value); } else { - res = unicodeFromTclObj(value); + res = unicodeFromTclObj(self, value); } return res; } +static int +Tkapp_Trace(TkappObject *self, PyObject *args) +{ + if (args == NULL) { + return 0; + } + if (self->trace) { + PyObject *res = PyObject_CallObject(self->trace, args); + if (res == NULL) { + Py_DECREF(args); + return 0; + } + Py_DECREF(res); + } + Py_DECREF(args); + return 1; +} + +#define TRACE(_self, ARGS) do { \ + if ((_self)->trace && !Tkapp_Trace((_self), Py_BuildValue ARGS)) { \ + return NULL; \ + } \ + } while (0) /* Tkapp_CallProc is the event procedure that is executed in the context of the Tcl interpreter thread. Initially, it holds the Tcl lock, and doesn't hold the Python lock. */ static int -Tkapp_CallProc(Tkapp_CallEvent *e, int flags) +Tkapp_CallProc(Tcl_Event *evPtr, int flags) { + Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; - int objc; + Tcl_Size objc; int i; ENTER_PYTHON - objv = Tkapp_CallArgs(e->args, objStore, &objc); + if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { + objv = NULL; + } + else { + objv = Tkapp_CallArgs(e->args, objStore, &objc); + } if (!objv) { *(e->exc) = PyErr_GetRaisedException(); *(e->res) = NULL; @@ -1361,7 +1452,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) { Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv = NULL; - int objc, i; + Tcl_Size objc; PyObject *res = NULL; TkappObject *self = (TkappObject*)selfptr; int flags = TCL_EVAL_DIRECT | TCL_EVAL_GLOBAL; @@ -1377,7 +1468,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) marshal the parameters to the interpreter thread. */ Tkapp_CallEvent *ev; Tcl_Condition cond = NULL; - PyObject *exc; + PyObject *exc = NULL; // init to make static analyzers happy if (!WaitForMainloop(self)) return NULL; ev = (Tkapp_CallEvent*)attemptckalloc(sizeof(Tkapp_CallEvent)); @@ -1385,7 +1476,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CallProc; + ev->ev.proc = Tkapp_CallProc; ev->self = self; ev->args = args; ev->res = &res; @@ -1406,7 +1497,9 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) } else { + TRACE(self, ("(O)", args)); + int i; objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) return NULL; @@ -1448,6 +1541,8 @@ _tkinter_tkapp_eval_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "eval", script)); + ENTER_TCL err = Tcl_Eval(Tkapp_Interp(self), script); ENTER_OVERLAP @@ -1477,6 +1572,8 @@ _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName) CHECK_STRING_LENGTH(fileName); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "source", fileName)); + ENTER_TCL err = Tcl_EvalFile(Tkapp_Interp(self), fileName); ENTER_OVERLAP @@ -1506,6 +1603,8 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ssss))", "history", "add", script, "exec")); + ENTER_TCL err = Tcl_RecordAndEval(Tkapp_Interp(self), script, TCL_NO_EVAL); ENTER_OVERLAP @@ -1624,8 +1723,9 @@ var_perform(VarEvent *ev) } static int -var_proc(VarEvent* ev, int flags) +var_proc(Tcl_Event *evPtr, int flags) { + VarEvent *ev = (VarEvent *)evPtr; ENTER_PYTHON var_perform(ev); Tcl_MutexLock(&var_mutex); @@ -1642,7 +1742,8 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) TkappObject *self = (TkappObject*)selfptr; if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { VarEvent *ev; - PyObject *res, *exc; + // init 'res' and 'exc' to make static analyzers happy + PyObject *res = NULL, *exc = NULL; Tcl_Condition cond = NULL; /* The current thread is not the interpreter thread. Marshal @@ -1663,7 +1764,7 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) ev->res = &res; ev->exc = &exc; ev->cond = &cond; - ev->ev.proc = (Tcl_EventProc*)var_proc; + ev->ev.proc = var_proc; Tkapp_ThreadSend(self, (Tcl_Event*)ev, &cond, &var_mutex); Tcl_ConditionFinalize(&cond); if (!res) { @@ -1694,6 +1795,15 @@ SetVar(TkappObject *self, PyObject *args, int flags) newval = AsObj(newValue); if (newval == NULL) return NULL; + + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((ssssO))", "uplevel", "#0", "set", + name1, newValue)); + } + else { + TRACE((TkappObject *)self, ("((ssO))", "set", name1, newValue)); + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, newval, flags); @@ -1711,8 +1821,22 @@ SetVar(TkappObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((sssNO))", "uplevel", "#0", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + else { + TRACE((TkappObject *)self, ("((sNO))", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP @@ -1766,7 +1890,7 @@ GetVar(TkappObject *self, PyObject *args, int flags) res = FromObj(self, tres); } else { - res = unicodeFromTclObj(tres); + res = unicodeFromTclObj(self, tres); } } LEAVE_OVERLAP_TCL @@ -1799,6 +1923,28 @@ UnsetVar(TkappObject *self, PyObject *args, int flags) CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + if (name2) { + TRACE((TkappObject *)self, ("((sssN))", "uplevel", "#0", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ssss))", "uplevel", "#0", "unset", name1)); + } + } + else { + if (name2) { + TRACE((TkappObject *)self, ("((sN))", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ss))", "unset", name1)); + } + } + } + ENTER_TCL code = Tcl_UnsetVar2(Tkapp_Interp(self), name1, name2, flags); ENTER_OVERLAP @@ -1965,6 +2111,8 @@ _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprString(Tkapp_Interp(self), s); ENTER_OVERLAP @@ -1995,6 +2143,8 @@ _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprLong(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2024,6 +2174,9 @@ _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprDouble(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2053,6 +2206,9 @@ _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprBoolean(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2079,13 +2235,12 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /*[clinic end generated code: output=13b51d34386d36fb input=2b2e13351e3c0b53]*/ { char *list; - int argc; + Tcl_Size argc, i; const char **argv; PyObject *v; - int i; if (PyTclObject_Check(arg)) { - int objc; + Tcl_Size objc; Tcl_Obj **objv; if (Tcl_ListObjGetElements(Tkapp_Interp(self), ((PyTclObject*)arg)->value, @@ -2148,7 +2303,7 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /* Client data struct */ typedef struct { - PyObject *self; + TkappObject *self; PyObject *func; } PythonCmd_ClientData; @@ -2172,6 +2327,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, PyObject *args, *res; int i; Tcl_Obj *obj_res; + int objargs = data->self->wantobjects >= 2; ENTER_PYTHON @@ -2180,7 +2336,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, return PythonCmd_Error(interp); for (i = 0; i < (objc - 1); i++) { - PyObject *s = unicodeFromTclObj(objv[i + 1]); + PyObject *s = objargs ? FromObj(data->self, objv[i + 1]) + : unicodeFromTclObj(data->self, objv[i + 1]); if (!s) { Py_DECREF(args); return PythonCmd_Error(interp); @@ -2236,8 +2393,9 @@ typedef struct CommandEvent{ } CommandEvent; static int -Tkapp_CommandProc(CommandEvent *ev, int flags) +Tkapp_CommandProc(Tcl_Event *evPtr, int flags) { + CommandEvent *ev = (CommandEvent *)evPtr; if (ev->create) *ev->status = Tcl_CreateObjCommand( ev->interp, ev->name, PythonCmd, @@ -2277,12 +2435,17 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, !WaitForMainloop(self)) return NULL; + TRACE(self, ("((ss()O))", "proc", name, func)); + data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - data->self = Py_NewRef(self); + Py_INCREF(self); + data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); if (ev == NULL) { @@ -2290,7 +2453,7 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, PyMem_Free(data); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 1; ev->name = name; @@ -2335,7 +2498,11 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) CHECK_STRING_LENGTH(name); + TRACE(self, ("((sss))", "rename", name, "")); + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev; ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); @@ -2343,7 +2510,7 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 0; ev->name = name; @@ -2460,6 +2627,8 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, return NULL; } + TRACE(self, ("((ssiiO))", "#", "createfilehandler", tfile, mask, func)); + data = NewFHCD(func, file, tfile); if (data == NULL) return NULL; @@ -2491,6 +2660,8 @@ _tkinter_tkapp_deletefilehandler(TkappObject *self, PyObject *file) if (tfile < 0) return NULL; + TRACE(self, ("((ssi))", "#", "deletefilehandler", tfile)); + DeleteFHCD(tfile); /* Ought to check for null Tcl_File object... */ @@ -2525,6 +2696,7 @@ _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) PyObject *func = v->func; if (v->token != NULL) { + /* TRACE(...) */ Tcl_DeleteTimerHandler(v->token); v->token = NULL; } @@ -2627,6 +2799,8 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, CHECK_TCL_APPARTMENT; + TRACE(self, ("((siO))", "after", milliseconds, func)); + v = Tktt_New(func); if (v) { v->token = Tcl_CreateTimerHandler(milliseconds, TimerHandler, @@ -2785,15 +2959,56 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) - return PyBool_FromLong(((TkappObject*)self)->wantobjects); + return PyLong_FromLong(((TkappObject*)self)->wantobjects); ((TkappObject*)self)->wantobjects = wantobjects; Py_RETURN_NONE; } +/*[clinic input] +_tkinter.tkapp.settrace + + func: object + / + +Set the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_settrace(TkappObject *self, PyObject *func) +/*[clinic end generated code: output=847f6ebdf46e84fa input=31b260d46d3d018a]*/ +{ + if (func == Py_None) { + func = NULL; + } + else { + Py_INCREF(func); + } + Py_XSETREF(self->trace, func); + Py_RETURN_NONE; +} + +/*[clinic input] +_tkinter.tkapp.gettrace + +Get the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self) +/*[clinic end generated code: output=d4e2ba7d63e77bb5 input=ac2aea5be74e8c4c]*/ +{ + PyObject *func = self->trace; + if (!func) { + func = Py_None; + } + Py_INCREF(func); + return func; +} + /*[clinic input] _tkinter.tkapp.willdispatch @@ -2819,6 +3034,7 @@ Tkapp_Dealloc(PyObject *self) ENTER_TCL Tcl_DeleteInterp(Tkapp_Interp(self)); LEAVE_TCL + Py_XDECREF(((TkappObject *)self)->trace); PyObject_Free(self); Py_DECREF(tp); DisableEventHook(); @@ -2932,7 +3148,7 @@ _tkinter.create baseName: str = "" className: str = "Tk" interactive: bool = False - wantobjects: bool = False + wantobjects: int = 0 wantTk: bool = True if false, then Tk_Init() doesn't get called sync: bool = False @@ -2948,7 +3164,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=7e382ba431bed537]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ @@ -3029,6 +3245,8 @@ static PyMethodDef Tkapp_methods[] = { _TKINTER_TKAPP_WILLDISPATCH_METHODDEF {"wantobjects", Tkapp_WantObjects, METH_VARARGS}, + _TKINTER_TKAPP_SETTRACE_METHODDEF + _TKINTER_TKAPP_GETTRACE_METHODDEF {"call", Tkapp_Call, METH_VARARGS}, _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF @@ -3196,6 +3414,9 @@ PyInit__tkinter(void) m = PyModule_Create(&_tkintermodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); if (PyModule_AddObjectRef(m, "TclError", Tkinter_TclError)) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6dba3cac01c1c8..887a1e820e250e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -219,6 +219,9 @@ PyInit__tracemalloc(void) m = PyModule_Create(&module_def); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (_PyTraceMalloc_Init() < 0) { Py_DECREF(m); diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 9ea72bf89ce0b2..37af00f3071e1d 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,12 +63,19 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyType_Ready(&_PyNoDefault_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { + return -1; + } return 0; } static struct PyModuleDef_Slot _typingmodule_slots[] = { {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 2f5be1c5144d83..c5e78b1510b5e3 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,6 +3,12 @@ * DCE compatible Universally Unique Identifier library. */ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #if defined(HAVE_UUID_H) // AIX, FreeBSD, libuuid with pkgconf @@ -105,6 +111,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 4e2862e7467c3d..a5c15c0f10b930 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -1,8 +1,7 @@ #include "Python.h" -#include "pycore_dict.h" // _PyDict_DelItemIf() -#include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR -#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() - +#include "pycore_dict.h" // _PyDict_DelItemIf() +#include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() +#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() #define GET_WEAKREFS_LISTPTR(o) \ ((PyWeakReference **) _PyObject_GET_WEAKREFS_LISTPTR(o)) @@ -15,7 +14,6 @@ module _weakref #include "clinic/_weakref.c.h" /*[clinic input] - _weakref.getweakrefcount -> Py_ssize_t object: object @@ -26,15 +24,9 @@ Return the number of weak references to 'object'. static Py_ssize_t _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=301806d59558ff3e input=cedb69711b6a2507]*/ +/*[clinic end generated code: output=301806d59558ff3e input=7d4d04fcaccf64d5]*/ { - PyWeakReference **list; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) - return 0; - - list = GET_WEAKREFS_LISTPTR(object); - return _PyWeakref_GetWeakrefCount(*list); + return _PyWeakref_GetWeakrefCount(object); } @@ -94,19 +86,32 @@ _weakref_getweakrefs(PyObject *module, PyObject *object) return PyList_New(0); } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - - PyObject *result = PyList_New(count); + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; } - PyWeakReference *current = *list; - for (Py_ssize_t i = 0; i < count; ++i) { - PyList_SET_ITEM(result, i, Py_NewRef(current)); + LOCK_WEAKREFS(object); + PyWeakReference *current = *GET_WEAKREFS_LISTPTR(object); + while (current != NULL) { + PyObject *curobj = (PyObject *) current; + if (_Py_TryIncref(curobj)) { + if (PyList_Append(result, curobj)) { + UNLOCK_WEAKREFS(object); + Py_DECREF(curobj); + Py_DECREF(result); + return NULL; + } + else { + // Undo our _Py_TryIncref. This is safe to do with the lock + // held in free-threaded builds; the list holds a reference to + // curobj so we're guaranteed not to invoke the destructor. + Py_DECREF(curobj); + } + } current = current->wr_next; } + UNLOCK_WEAKREFS(object); return result; } @@ -166,6 +171,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eec33499b983fe..c90d6c5a9ef3ef 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -72,9 +72,45 @@ #ifndef STARTF_USESHOWWINDOW #define STARTF_USESHOWWINDOW 0x00000001 #endif +#ifndef STARTF_USESIZE +#define STARTF_USESIZE 0x00000002 +#endif +#ifndef STARTF_USEPOSITION +#define STARTF_USEPOSITION 0x00000004 +#endif +#ifndef STARTF_USECOUNTCHARS +#define STARTF_USECOUNTCHARS 0x00000008 +#endif +#ifndef STARTF_USEFILLATTRIBUTE +#define STARTF_USEFILLATTRIBUTE 0x00000010 +#endif +#ifndef STARTF_RUNFULLSCREEN +#define STARTF_RUNFULLSCREEN 0x00000020 +#endif +#ifndef STARTF_FORCEONFEEDBACK +#define STARTF_FORCEONFEEDBACK 0x00000040 +#endif +#ifndef STARTF_FORCEOFFFEEDBACK +#define STARTF_FORCEOFFFEEDBACK 0x00000080 +#endif #ifndef STARTF_USESTDHANDLES #define STARTF_USESTDHANDLES 0x00000100 #endif +#ifndef STARTF_USEHOTKEY +#define STARTF_USEHOTKEY 0x00000200 +#endif +#ifndef STARTF_TITLEISLINKNAME +#define STARTF_TITLEISLINKNAME 0x00000800 +#endif +#ifndef STARTF_TITLEISAPPID +#define STARTF_TITLEISAPPID 0x00001000 +#endif +#ifndef STARTF_PREVENTPINNING +#define STARTF_PREVENTPINNING 0x00002000 +#endif +#ifndef STARTF_UNTRUSTEDSOURCE +#define STARTF_UNTRUSTEDSOURCE 0x00008000 +#endif typedef struct { PyTypeObject *overlapped_type; @@ -139,7 +175,7 @@ overlapped_dealloc(OverlappedObject *self) { /* The operation is still pending -- give a warning. This will probably only happen on Windows XP. */ - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_PythonFinalizationError, "I/O operations still in flight while destroying " "Overlapped object, the process may crash"); PyErr_WriteUnraisable(NULL); @@ -188,7 +224,6 @@ create_converter('LPCVOID', '" F_POINTER "') create_converter('BOOL', 'i') # F_BOOL used previously (always 'i') create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter) -create_converter('LPCTSTR', 's') create_converter('UINT', 'I') # F_UINT used previously (always 'I') class LPCWSTR_converter(Py_UNICODE_converter): @@ -212,7 +247,7 @@ class DWORD_return_converter(CReturnConverter): self.declare(data) self.err_occurred_if("_return_value == PY_DWORD_MAX", data) data.return_conversion.append( - 'return_value = Py_BuildValue("k", _return_value);\n') + 'return_value = PyLong_FromUnsignedLong(_return_value);\n') class LPVOID_return_converter(CReturnConverter): type = 'LPVOID' @@ -223,7 +258,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=011ee0c3a2244bfe]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=da0a4db751936ee7]*/ #include "clinic/_winapi.c.h" @@ -438,10 +473,43 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateEventW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + manual_reset: BOOL + initial_state: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name) +/*[clinic end generated code: output=2d4c7d5852ecb298 input=4187cee28ac763f8]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateEventW", "bbu", manual_reset, initial_state, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateEventW(security_attributes, manual_reset, initial_state, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateFile -> HANDLE - file_name: LPCTSTR + file_name: LPCWSTR desired_access: DWORD share_mode: DWORD security_attributes: LPSECURITY_ATTRIBUTES @@ -452,30 +520,31 @@ _winapi.CreateFile -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, DWORD flags_and_attributes, HANDLE template_file) -/*[clinic end generated code: output=417ddcebfc5a3d53 input=6423c3e40372dbd5]*/ +/*[clinic end generated code: output=818c811e5e04d550 input=1fa870ed1c2e3d69]*/ { HANDLE handle; - if (PySys_Audit("_winapi.CreateFile", "uIIII", + if (PySys_Audit("_winapi.CreateFile", "ukkkk", file_name, desired_access, share_mode, creation_disposition, flags_and_attributes) < 0) { return INVALID_HANDLE_VALUE; } Py_BEGIN_ALLOW_THREADS - handle = CreateFile(file_name, desired_access, - share_mode, security_attributes, - creation_disposition, - flags_and_attributes, template_file); + handle = CreateFileW(file_name, desired_access, + share_mode, security_attributes, + creation_disposition, + flags_and_attributes, template_file); Py_END_ALLOW_THREADS - if (handle == INVALID_HANDLE_VALUE) + if (handle == INVALID_HANDLE_VALUE) { PyErr_SetFromWindowsErr(0); + } return handle; } @@ -532,7 +601,12 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, { /* Privilege adjustment */ HANDLE token = NULL; - TOKEN_PRIVILEGES tp; + struct { + TOKEN_PRIVILEGES base; + /* overallocate by a few array elements */ + LUID_AND_ATTRIBUTES privs[4]; + } tp, previousTp; + int previousTpSize = 0; /* Reparse data buffer */ const USHORT prefix_len = 4; @@ -556,17 +630,21 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, /* Adjust privileges to allow rewriting directory entry as a junction point. */ - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token)) + if (!OpenProcessToken(GetCurrentProcess(), + TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token)) { goto cleanup; + } - if (!LookupPrivilegeValue(NULL, SE_RESTORE_NAME, &tp.Privileges[0].Luid)) + if (!LookupPrivilegeValue(NULL, SE_RESTORE_NAME, &tp.base.Privileges[0].Luid)) { goto cleanup; + } - tp.PrivilegeCount = 1; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - if (!AdjustTokenPrivileges(token, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), - NULL, NULL)) + tp.base.PrivilegeCount = 1; + tp.base.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + if (!AdjustTokenPrivileges(token, FALSE, &tp.base, sizeof(previousTp), + &previousTp.base, &previousTpSize)) { goto cleanup; + } if (GetFileAttributesW(src_path) == INVALID_FILE_ATTRIBUTES) goto cleanup; @@ -647,6 +725,11 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, cleanup: ret = GetLastError(); + if (previousTpSize) { + AdjustTokenPrivileges(token, FALSE, &previousTp.base, previousTpSize, + NULL, NULL); + } + if (token != NULL) CloseHandle(token); if (junction != NULL) @@ -659,10 +742,41 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, Py_RETURN_NONE; } +/*[clinic input] +_winapi.CreateMutexW -> HANDLE + + security_attributes: LPSECURITY_ATTRIBUTES + initial_owner: BOOL + name: LPCWSTR(accept={str, NoneType}) +[clinic start generated code]*/ + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name) +/*[clinic end generated code: output=31b9ee8fc37e49a5 input=7d54b921e723254a]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.CreateMutexW", "bu", initial_owner, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = CreateMutexW(security_attributes, initial_owner, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.CreateNamedPipe -> HANDLE - name: LPCTSTR + name: LPCWSTR open_mode: DWORD pipe_mode: DWORD max_instances: DWORD @@ -674,25 +788,25 @@ _winapi.CreateNamedPipe -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, LPSECURITY_ATTRIBUTES security_attributes) -/*[clinic end generated code: output=80f8c07346a94fbc input=5a73530b84d8bc37]*/ +/*[clinic end generated code: output=7d6fde93227680ba input=5bd4e4a55639ee02]*/ { HANDLE handle; - if (PySys_Audit("_winapi.CreateNamedPipe", "uII", + if (PySys_Audit("_winapi.CreateNamedPipe", "ukk", name, open_mode, pipe_mode) < 0) { return INVALID_HANDLE_VALUE; } Py_BEGIN_ALLOW_THREADS - handle = CreateNamedPipe(name, open_mode, pipe_mode, - max_instances, out_buffer_size, - in_buffer_size, default_timeout, - security_attributes); + handle = CreateNamedPipeW(name, open_mode, pipe_mode, + max_instances, out_buffer_size, + in_buffer_size, default_timeout, + security_attributes); Py_END_ALLOW_THREADS if (handle == INVALID_HANDLE_VALUE) @@ -774,12 +888,157 @@ gethandle(PyObject* obj, const char* name) return ret; } +static PyObject * +sortenvironmentkey(PyObject *module, PyObject *item) +{ + return _winapi_LCMapStringEx_impl(NULL, LOCALE_NAME_INVARIANT, + LCMAP_UPPERCASE, item); +} + +static PyMethodDef sortenvironmentkey_def = { + "sortenvironmentkey", _PyCFunction_CAST(sortenvironmentkey), METH_O, "", +}; + +static int +sort_environment_keys(PyObject *keys) +{ + PyObject *keyfunc = PyCFunction_New(&sortenvironmentkey_def, NULL); + if (keyfunc == NULL) { + return -1; + } + PyObject *kwnames = Py_BuildValue("(s)", "key"); + if (kwnames == NULL) { + Py_DECREF(keyfunc); + return -1; + } + PyObject *args[] = { keys, keyfunc }; + PyObject *ret = PyObject_VectorcallMethod(&_Py_ID(sort), args, 1, kwnames); + Py_DECREF(keyfunc); + Py_DECREF(kwnames); + if (ret == NULL) { + return -1; + } + Py_DECREF(ret); + + return 0; +} + +static int +compare_string_ordinal(PyObject *str1, PyObject *str2, int *result) +{ + wchar_t *s1 = PyUnicode_AsWideCharString(str1, NULL); + if (s1 == NULL) { + return -1; + } + wchar_t *s2 = PyUnicode_AsWideCharString(str2, NULL); + if (s2 == NULL) { + PyMem_Free(s1); + return -1; + } + *result = CompareStringOrdinal(s1, -1, s2, -1, TRUE); + PyMem_Free(s1); + PyMem_Free(s2); + return 0; +} + +static PyObject * +dedup_environment_keys(PyObject *keys) +{ + PyObject *result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // Iterate over the pre-ordered keys, check whether the current key is equal + // to the next key (ignoring case), if different, insert the current value + // into the result list. If they are equal, do nothing because we always + // want to keep the last inserted one. + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); i++) { + PyObject *key = PyList_GET_ITEM(keys, i); + + // The last key will always be kept. + if (i + 1 == PyList_GET_SIZE(keys)) { + if (PyList_Append(result, key) < 0) { + Py_DECREF(result); + return NULL; + } + continue; + } + + PyObject *next_key = PyList_GET_ITEM(keys, i + 1); + int compare_result; + if (compare_string_ordinal(key, next_key, &compare_result) < 0) { + Py_DECREF(result); + return NULL; + } + if (compare_result == CSTR_EQUAL) { + continue; + } + if (PyList_Append(result, key) < 0) { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +static PyObject * +normalize_environment(PyObject *environment) +{ + PyObject *keys = PyMapping_Keys(environment); + if (keys == NULL) { + return NULL; + } + + if (sort_environment_keys(keys) < 0) { + Py_DECREF(keys); + return NULL; + } + + PyObject *normalized_keys = dedup_environment_keys(keys); + Py_DECREF(keys); + if (normalized_keys == NULL) { + return NULL; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + Py_DECREF(normalized_keys); + return NULL; + } + + for (int i = 0; i < PyList_GET_SIZE(normalized_keys); i++) { + PyObject *key = PyList_GET_ITEM(normalized_keys, i); + PyObject *value = PyObject_GetItem(environment, key); + if (value == NULL) { + Py_DECREF(normalized_keys); + Py_DECREF(result); + return NULL; + } + + int ret = PyObject_SetItem(result, key, value); + Py_DECREF(value); + if (ret < 0) { + Py_DECREF(normalized_keys); + Py_DECREF(result); + return NULL; + } + } + + Py_DECREF(normalized_keys); + + return result; +} + static wchar_t * getenvironment(PyObject* environment) { Py_ssize_t i, envsize, totalsize; wchar_t *buffer = NULL, *p, *end; - PyObject *keys, *values; + PyObject *normalized_environment = NULL; + PyObject *keys = NULL; + PyObject *values = NULL; /* convert environment dictionary to windows environment string */ if (! PyMapping_Check(environment)) { @@ -788,11 +1047,16 @@ getenvironment(PyObject* environment) return NULL; } - keys = PyMapping_Keys(environment); - if (!keys) { + normalized_environment = normalize_environment(environment); + if (normalize_environment == NULL) { return NULL; } - values = PyMapping_Values(environment); + + keys = PyMapping_Keys(normalized_environment); + if (!keys) { + goto error; + } + values = PyMapping_Values(normalized_environment); if (!values) { goto error; } @@ -884,6 +1148,7 @@ getenvironment(PyObject* environment) cleanup: error: + Py_XDECREF(normalized_environment); Py_XDECREF(keys); Py_XDECREF(values); return buffer; @@ -1287,6 +1552,49 @@ _winapi_GetLastError_impl(PyObject *module) return GetLastError(); } + +/*[clinic input] +_winapi.GetLongPathName + + path: LPCWSTR + +Return the long version of the provided path. + +If the path is already in its long form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=c4774b080275a2d0 input=9872e211e3a4a88f]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetModuleFileName @@ -1321,6 +1629,48 @@ _winapi_GetModuleFileName_impl(PyObject *module, HMODULE module_handle) return PyUnicode_FromWideChar(filename, wcslen(filename)); } +/*[clinic input] +_winapi.GetShortPathName + + path: LPCWSTR + +Return the short version of the provided path. + +If the path is already in its short form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=dab6ae494c621e81 input=43fa349aaf2ac718]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetStdHandle -> HANDLE @@ -1424,6 +1774,67 @@ _winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) Py_RETURN_NONE; } +/*[clinic input] +_winapi.OpenEventW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=c4a45e95545a4bd2 input=dec26598748d35aa]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenEventW", "ku", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenEventW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + + +/*[clinic input] +_winapi.OpenMutexW -> HANDLE + + desired_access: DWORD + inherit_handle: BOOL + name: LPCWSTR +[clinic start generated code]*/ + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name) +/*[clinic end generated code: output=dda39d7844397bf0 input=f3a7b466c5307712]*/ +{ + HANDLE handle; + + if (PySys_Audit("_winapi.OpenMutexW", "ku", desired_access, name) < 0) { + return INVALID_HANDLE_VALUE; + } + + Py_BEGIN_ALLOW_THREADS + handle = OpenMutexW(desired_access, inherit_handle, name); + Py_END_ALLOW_THREADS + + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + } + + return handle; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -1470,7 +1881,7 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenProcess", "II", + if (PySys_Audit("_winapi.OpenProcess", "kk", process_id, desired_access) < 0) { return INVALID_HANDLE_VALUE; } @@ -1654,6 +2065,75 @@ _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size, return Py_BuildValue("NI", buf, err); } +/*[clinic input] +_winapi.ReleaseMutex + + mutex: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex) +/*[clinic end generated code: output=5b9001a72dd8af37 input=49e9d20de3559d84]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ReleaseMutex(mutex)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.ResetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=81c8501d57c0530d input=e2d42d990322e87a]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!ResetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.SetEvent + + event: HANDLE +[clinic start generated code]*/ + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event) +/*[clinic end generated code: output=c18ba09eb9aa774d input=e660e830a37c09f8]*/ +{ + int err = 0; + + Py_BEGIN_ALLOW_THREADS + if (!SetEvent(event)) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + if (err) { + return PyErr_SetFromWindowsErr(err); + } + Py_RETURN_NONE; +} + /*[clinic input] _winapi.SetNamedPipeHandleState @@ -1755,19 +2235,19 @@ _winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address) /*[clinic input] _winapi.WaitNamedPipe - name: LPCTSTR + name: LPCWSTR timeout: DWORD / [clinic start generated code]*/ static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) -/*[clinic end generated code: output=c2866f4439b1fe38 input=36fc781291b1862c]*/ +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout) +/*[clinic end generated code: output=e161e2e630b3e9c2 input=099a4746544488fa]*/ { BOOL success; Py_BEGIN_ALLOW_THREADS - success = WaitNamedPipe(name, timeout); + success = WaitNamedPipeW(name, timeout); Py_END_ALLOW_THREADS if (!success) @@ -1776,6 +2256,310 @@ _winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) Py_RETURN_NONE; } + +typedef struct { + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE cancel_event; + DWORD handle_base; + DWORD handle_count; + HANDLE thread; + volatile DWORD result; +} BatchedWaitData; + +static DWORD WINAPI +_batched_WaitForMultipleObjects_thread(LPVOID param) +{ + BatchedWaitData *data = (BatchedWaitData *)param; + data->result = WaitForMultipleObjects( + data->handle_count, + data->handles, + FALSE, + INFINITE + ); + if (data->result == WAIT_FAILED) { + DWORD err = GetLastError(); + SetEvent(data->cancel_event); + return err; + } else if (data->result >= WAIT_ABANDONED_0 && data->result < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { + data->result = WAIT_FAILED; + SetEvent(data->cancel_event); + return ERROR_ABANDONED_WAIT_0; + } + return 0; +} + +/*[clinic input] +_winapi.BatchedWaitForMultipleObjects + + handle_seq: object + wait_all: BOOL + milliseconds: DWORD(c_default='INFINITE') = _winapi.INFINITE + +Supports a larger number of handles than WaitForMultipleObjects + +Note that the handles may be waited on other threads, which could cause +issues for objects like mutexes that become associated with the thread +that was waiting for them. Objects may also be left signalled, even if +the wait fails. + +It is recommended to use WaitForMultipleObjects whenever possible, and +only switch to BatchedWaitForMultipleObjects for scenarios where you +control all the handles involved, such as your own thread pool or +files, and all wait objects are left unmodified by a wait (for example, +manual reset events, threads, and files/pipes). + +Overlapped handles returned from this module use manual reset events. +[clinic start generated code]*/ + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds) +/*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/ +{ + Py_ssize_t thread_count = 0, handle_count = 0, i, j; + Py_ssize_t nhandles; + BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS]; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE sigint_event = NULL; + HANDLE cancel_event = NULL; + DWORD result; + + const Py_ssize_t _MAXIMUM_TOTAL_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1); + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_name); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) { + return NULL; + } + if (nhandles == 0) { + return wait_all ? Py_NewRef(Py_None) : PyList_New(0); + } + + /* If this is the main thread then make the wait interruptible + by Ctrl-C. When waiting for *all* handles, it is only checked + in between batches. */ + if (_PyOS_IsMainThread()) { + sigint_event = _PyOS_SigintEvent(); + assert(sigint_event != NULL); + } + + if (nhandles < 0 || nhandles > _MAXIMUM_TOTAL_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + _MAXIMUM_TOTAL_OBJECTS, nhandles); + return NULL; + } + + if (!wait_all) { + cancel_event = CreateEventW(NULL, TRUE, FALSE, NULL); + if (!cancel_event) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + return NULL; + } + } + + i = 0; + while (i < nhandles) { + BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData)); + if (!data) { + goto error; + } + thread_data[thread_count++] = data; + data->thread = NULL; + data->cancel_event = cancel_event; + data->handle_base = Py_SAFE_DOWNCAST(i, Py_ssize_t, DWORD); + data->handle_count = Py_SAFE_DOWNCAST(nhandles - i, Py_ssize_t, DWORD); + if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + data->handle_count = MAXIMUM_WAIT_OBJECTS - 1; + } + for (j = 0; j < data->handle_count; ++i, ++j) { + PyObject *v = PySequence_GetItem(handle_seq, i); + if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) { + Py_XDECREF(v); + goto error; + } + Py_DECREF(v); + } + if (!wait_all) { + data->handles[data->handle_count++] = cancel_event; + } + } + + DWORD err = 0; + + /* We need to use different strategies when waiting for ALL handles + as opposed to ANY handle. This is because there is no way to + (safely) interrupt a thread that is waiting for all handles in a + group. So for ALL handles, we loop over each set and wait. For + ANY handle, we use threads and wait on them. */ + if (wait_all) { + Py_BEGIN_ALLOW_THREADS + long long deadline = 0; + if (milliseconds != INFINITE) { + deadline = (long long)GetTickCount64() + milliseconds; + } + + for (i = 0; !err && i < thread_count; ++i) { + DWORD timeout = milliseconds; + if (deadline) { + long long time_to_deadline = deadline - GetTickCount64(); + if (time_to_deadline <= 0) { + err = WAIT_TIMEOUT; + break; + } else if (time_to_deadline < UINT_MAX) { + timeout = (DWORD)time_to_deadline; + } + } + result = WaitForMultipleObjects(thread_data[i]->handle_count, + thread_data[i]->handles, TRUE, timeout); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } + + if (!err && sigint_event) { + result = WaitForSingleObject(sigint_event, 0); + if (result == WAIT_OBJECT_0) { + err = ERROR_CONTROL_C_EXIT; + } else if (result == WAIT_FAILED) { + err = GetLastError(); + } + } + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + } else { + Py_BEGIN_ALLOW_THREADS + + for (i = 0; i < thread_count; ++i) { + BatchedWaitData *data = thread_data[i]; + data->thread = CreateThread( + NULL, + 1, // smallest possible initial stack + _batched_WaitForMultipleObjects_thread, + (LPVOID)data, + CREATE_SUSPENDED, + NULL + ); + if (!data->thread) { + err = GetLastError(); + break; + } + handles[handle_count++] = data->thread; + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + goto error; + } + if (handle_count > MAXIMUM_WAIT_OBJECTS - 1) { + // basically an assert, but stronger + PyErr_SetString(PyExc_SystemError, "allocated too many wait objects"); + goto error; + } + + Py_BEGIN_ALLOW_THREADS + + // Once we start resuming threads, can no longer "goto error" + for (i = 0; i < thread_count; ++i) { + ResumeThread(thread_data[i]->thread); + } + if (sigint_event) { + handles[handle_count++] = sigint_event; + } + result = WaitForMultipleObjects((DWORD)handle_count, handles, wait_all, milliseconds); + // ABANDONED is not possible here because we own all the handles + if (result == WAIT_FAILED) { + err = GetLastError(); + } else if (result == WAIT_TIMEOUT) { + err = WAIT_TIMEOUT; + } else if (sigint_event && result == WAIT_OBJECT_0 + handle_count) { + err = ERROR_CONTROL_C_EXIT; + } + + SetEvent(cancel_event); + + // Wait for all threads to finish before we start freeing their memory + if (sigint_event) { + handle_count -= 1; + } + WaitForMultipleObjects((DWORD)handle_count, handles, TRUE, INFINITE); + + for (i = 0; i < thread_count; ++i) { + if (!err && thread_data[i]->result == WAIT_FAILED) { + if (!GetExitCodeThread(thread_data[i]->thread, &err)) { + err = GetLastError(); + } + } + CloseHandle(thread_data[i]->thread); + } + + CloseHandle(cancel_event); + + Py_END_ALLOW_THREADS + + } + + PyObject *triggered_indices; + if (sigint_event != NULL && err == ERROR_CONTROL_C_EXIT) { + errno = EINTR; + PyErr_SetFromErrno(PyExc_OSError); + triggered_indices = NULL; + } else if (err) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, err); + triggered_indices = NULL; + } else if (wait_all) { + triggered_indices = Py_NewRef(Py_None); + } else { + triggered_indices = PyList_New(0); + if (triggered_indices) { + for (i = 0; i < thread_count; ++i) { + Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0; + if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) { + PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered); + if (!v || PyList_Append(triggered_indices, v) < 0) { + Py_XDECREF(v); + Py_CLEAR(triggered_indices); + break; + } + Py_DECREF(v); + } + } + } + } + + for (i = 0; i < thread_count; ++i) { + PyMem_Free((void *)thread_data[i]); + } + + return triggered_indices; + +error: + // We should only enter here before any threads start running. + // Once we start resuming threads, different cleanup is required + CloseHandle(cancel_event); + while (--thread_count >= 0) { + HANDLE t = thread_data[thread_count]->thread; + if (t) { + TerminateThread(t, WAIT_ABANDONED_0); + CloseHandle(t); + } + PyMem_Free((void *)thread_data[thread_count]); + } + return NULL; +} + /*[clinic input] _winapi.WaitForMultipleObjects @@ -2132,7 +2916,7 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, HRESULT hr; COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; - if (PySys_Audit("_winapi.CopyFile2", "uuI", + if (PySys_Audit("_winapi.CopyFile2", "uuk", existing_file_name, new_file_name, flags) < 0) { return NULL; } @@ -2169,8 +2953,10 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF + _WINAPI_CREATEEVENTW_METHODDEF _WINAPI_CREATEFILE_METHODDEF _WINAPI_CREATEFILEMAPPING_METHODDEF + _WINAPI_CREATEMUTEXW_METHODDEF _WINAPI_CREATENAMEDPIPE_METHODDEF _WINAPI_CREATEPIPE_METHODDEF _WINAPI_CREATEPROCESS_METHODDEF @@ -2180,21 +2966,29 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETCURRENTPROCESS_METHODDEF _WINAPI_GETEXITCODEPROCESS_METHODDEF _WINAPI_GETLASTERROR_METHODDEF + _WINAPI_GETLONGPATHNAME_METHODDEF _WINAPI_GETMODULEFILENAME_METHODDEF + _WINAPI_GETSHORTPATHNAME_METHODDEF _WINAPI_GETSTDHANDLE_METHODDEF _WINAPI_GETVERSION_METHODDEF _WINAPI_MAPVIEWOFFILE_METHODDEF + _WINAPI_OPENEVENTW_METHODDEF _WINAPI_OPENFILEMAPPING_METHODDEF + _WINAPI_OPENMUTEXW_METHODDEF _WINAPI_OPENPROCESS_METHODDEF _WINAPI_PEEKNAMEDPIPE_METHODDEF _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF + _WINAPI_RELEASEMUTEX_METHODDEF + _WINAPI_RESETEVENT_METHODDEF + _WINAPI_SETEVENT_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF + _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF _WINAPI_WAITFORSINGLEOBJECT_METHODDEF _WINAPI_WRITEFILE_METHODDEF _WINAPI_GETACP_METHODDEF @@ -2302,7 +3096,19 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, SEC_RESERVE); WINAPI_CONSTANT(F_DWORD, SEC_WRITECOMBINE); WINAPI_CONSTANT(F_DWORD, STARTF_USESHOWWINDOW); + WINAPI_CONSTANT(F_DWORD, STARTF_USESIZE); + WINAPI_CONSTANT(F_DWORD, STARTF_USEPOSITION); + WINAPI_CONSTANT(F_DWORD, STARTF_USECOUNTCHARS); + WINAPI_CONSTANT(F_DWORD, STARTF_USEFILLATTRIBUTE); + WINAPI_CONSTANT(F_DWORD, STARTF_RUNFULLSCREEN); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEONFEEDBACK); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEOFFFEEDBACK); WINAPI_CONSTANT(F_DWORD, STARTF_USESTDHANDLES); + WINAPI_CONSTANT(F_DWORD, STARTF_USEHOTKEY); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISLINKNAME); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISAPPID); + WINAPI_CONSTANT(F_DWORD, STARTF_PREVENTPINNING); + WINAPI_CONSTANT(F_DWORD, STARTF_UNTRUSTEDSOURCE); WINAPI_CONSTANT(F_DWORD, STD_INPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_OUTPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_ERROR_HANDLE); @@ -2360,6 +3166,11 @@ static int winapi_exec(PyObject *m) #define COPY_FILE_REQUEST_COMPRESSED_TRAFFIC 0x10000000 #endif WINAPI_CONSTANT(F_DWORD, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC); +#ifndef COPY_FILE_DIRECTORY + // Only defined in newer WinSDKs + #define COPY_FILE_DIRECTORY 0x00000080 +#endif + WINAPI_CONSTANT(F_DWORD, COPY_FILE_DIRECTORY); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_STARTED); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_FINISHED); @@ -2382,6 +3193,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c deleted file mode 100644 index 12c98ea61fb7ac..00000000000000 --- a/Modules/_xxsubinterpretersmodule.c +++ /dev/null @@ -1,1184 +0,0 @@ -/* interpreters module */ -/* low-level access to interpreter primitives */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() -#include "interpreteridobject.h" -#include "marshal.h" // PyMarshal_ReadObjectFromString() - - -#define MODULE_NAME "_xxsubinterpreters" - - -static const char * -_copy_raw_string(PyObject *strobj) -{ - const char *str = PyUnicode_AsUTF8(strobj); - if (str == NULL) { - return NULL; - } - char *copied = PyMem_RawMalloc(strlen(str)+1); - if (copied == NULL) { - PyErr_NoMemory(); - return NULL; - } - strcpy(copied, str); - return copied; -} - -static PyInterpreterState * -_get_current_interp(void) -{ - // PyInterpreterState_Get() aborts if lookup fails, so don't need - // to check the result for NULL. - return PyInterpreterState_Get(); -} - -static PyObject * -add_new_exception(PyObject *mod, const char *name, PyObject *base) -{ - assert(!PyObject_HasAttrStringWithError(mod, name)); - PyObject *exctype = PyErr_NewException(name, base, NULL); - if (exctype == NULL) { - return NULL; - } - int res = PyModule_AddType(mod, (PyTypeObject *)exctype); - if (res < 0) { - Py_DECREF(exctype); - return NULL; - } - return exctype; -} - -#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ - add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) - -static int -_release_xid_data(_PyCrossInterpreterData *data) -{ - PyObject *exc = PyErr_GetRaisedException(); - int res = _PyCrossInterpreterData_Release(data); - if (res < 0) { - /* The owning interpreter is already destroyed. */ - _PyCrossInterpreterData_Clear(NULL, data); - // XXX Emit a warning? - PyErr_Clear(); - } - PyErr_SetRaisedException(exc); - return res; -} - - -/* module state *************************************************************/ - -typedef struct { - /* exceptions */ - PyObject *RunFailedError; -} module_state; - -static inline module_state * -get_module_state(PyObject *mod) -{ - assert(mod != NULL); - module_state *state = PyModule_GetState(mod); - assert(state != NULL); - return state; -} - -static int -traverse_module_state(module_state *state, visitproc visit, void *arg) -{ - /* exceptions */ - Py_VISIT(state->RunFailedError); - - return 0; -} - -static int -clear_module_state(module_state *state) -{ - /* exceptions */ - Py_CLEAR(state->RunFailedError); - - return 0; -} - - -/* data-sharing-specific code ***********************************************/ - -struct _sharednsitem { - const char *name; - _PyCrossInterpreterData data; -}; - -static void _sharednsitem_clear(struct _sharednsitem *); // forward - -static int -_sharednsitem_init(struct _sharednsitem *item, PyObject *key, PyObject *value) -{ - item->name = _copy_raw_string(key); - if (item->name == NULL) { - return -1; - } - if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) { - _sharednsitem_clear(item); - return -1; - } - return 0; -} - -static void -_sharednsitem_clear(struct _sharednsitem *item) -{ - if (item->name != NULL) { - PyMem_RawFree((void *)item->name); - item->name = NULL; - } - (void)_release_xid_data(&item->data); -} - -static int -_sharednsitem_apply(struct _sharednsitem *item, PyObject *ns) -{ - PyObject *name = PyUnicode_FromString(item->name); - if (name == NULL) { - return -1; - } - PyObject *value = _PyCrossInterpreterData_NewObject(&item->data); - if (value == NULL) { - Py_DECREF(name); - return -1; - } - int res = PyDict_SetItem(ns, name, value); - Py_DECREF(name); - Py_DECREF(value); - return res; -} - -typedef struct _sharedns { - Py_ssize_t len; - struct _sharednsitem* items; -} _sharedns; - -static _sharedns * -_sharedns_new(Py_ssize_t len) -{ - _sharedns *shared = PyMem_RawCalloc(sizeof(_sharedns), 1); - if (shared == NULL) { - PyErr_NoMemory(); - return NULL; - } - shared->len = len; - shared->items = PyMem_RawCalloc(sizeof(struct _sharednsitem), len); - if (shared->items == NULL) { - PyErr_NoMemory(); - PyMem_RawFree(shared); - return NULL; - } - return shared; -} - -static void -_sharedns_free(_sharedns *shared) -{ - for (Py_ssize_t i=0; i < shared->len; i++) { - _sharednsitem_clear(&shared->items[i]); - } - PyMem_RawFree(shared->items); - PyMem_RawFree(shared); -} - -static _sharedns * -_get_shared_ns(PyObject *shareable) -{ - if (shareable == NULL || shareable == Py_None) { - return NULL; - } - Py_ssize_t len = PyDict_Size(shareable); - if (len == 0) { - return NULL; - } - - _sharedns *shared = _sharedns_new(len); - if (shared == NULL) { - return NULL; - } - Py_ssize_t pos = 0; - for (Py_ssize_t i=0; i < len; i++) { - PyObject *key, *value; - if (PyDict_Next(shareable, &pos, &key, &value) == 0) { - break; - } - if (_sharednsitem_init(&shared->items[i], key, value) != 0) { - break; - } - } - if (PyErr_Occurred()) { - _sharedns_free(shared); - return NULL; - } - return shared; -} - -static int -_sharedns_apply(_sharedns *shared, PyObject *ns) -{ - for (Py_ssize_t i=0; i < shared->len; i++) { - if (_sharednsitem_apply(&shared->items[i], ns) != 0) { - return -1; - } - } - return 0; -} - -// Ultimately we'd like to preserve enough information about the -// exception and traceback that we could re-constitute (or at least -// simulate, a la traceback.TracebackException), and even chain, a copy -// of the exception in the calling interpreter. - -typedef struct _sharedexception { - PyInterpreterState *interp; -#define ERR_NOT_SET 0 -#define ERR_NO_MEMORY 1 -#define ERR_ALREADY_RUNNING 2 - int code; - const char *name; - const char *msg; -} _sharedexception; - -static const struct _sharedexception no_exception = { - .name = NULL, - .msg = NULL, -}; - -static void -_sharedexception_clear(_sharedexception *exc) -{ - if (exc->name != NULL) { - PyMem_RawFree((void *)exc->name); - } - if (exc->msg != NULL) { - PyMem_RawFree((void *)exc->msg); - } -} - -static const char * -_sharedexception_bind(PyObject *exc, int code, _sharedexception *sharedexc) -{ - if (sharedexc->interp == NULL) { - sharedexc->interp = PyInterpreterState_Get(); - } - - if (code != ERR_NOT_SET) { - assert(exc == NULL); - assert(code > 0); - sharedexc->code = code; - return NULL; - } - - assert(exc != NULL); - const char *failure = NULL; - - PyObject *nameobj = PyUnicode_FromString(Py_TYPE(exc)->tp_name); - if (nameobj == NULL) { - failure = "unable to format exception type name"; - code = ERR_NO_MEMORY; - goto error; - } - sharedexc->name = _copy_raw_string(nameobj); - Py_DECREF(nameobj); - if (sharedexc->name == NULL) { - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - failure = "out of memory copying exception type name"; - } else { - failure = "unable to encode and copy exception type name"; - } - code = ERR_NO_MEMORY; - goto error; - } - - if (exc != NULL) { - PyObject *msgobj = PyUnicode_FromFormat("%S", exc); - if (msgobj == NULL) { - failure = "unable to format exception message"; - code = ERR_NO_MEMORY; - goto error; - } - sharedexc->msg = _copy_raw_string(msgobj); - Py_DECREF(msgobj); - if (sharedexc->msg == NULL) { - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - failure = "out of memory copying exception message"; - } else { - failure = "unable to encode and copy exception message"; - } - code = ERR_NO_MEMORY; - goto error; - } - } - - return NULL; - -error: - assert(failure != NULL); - PyErr_Clear(); - _sharedexception_clear(sharedexc); - *sharedexc = (_sharedexception){ - .interp = sharedexc->interp, - .code = code, - }; - return failure; -} - -static void -_sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) -{ - if (exc->name != NULL) { - assert(exc->code == ERR_NOT_SET); - if (exc->msg != NULL) { - PyErr_Format(wrapperclass, "%s: %s", exc->name, exc->msg); - } - else { - PyErr_SetString(wrapperclass, exc->name); - } - } - else if (exc->msg != NULL) { - assert(exc->code == ERR_NOT_SET); - PyErr_SetString(wrapperclass, exc->msg); - } - else if (exc->code == ERR_NO_MEMORY) { - PyErr_NoMemory(); - } - else if (exc->code == ERR_ALREADY_RUNNING) { - assert(exc->interp != NULL); - assert(_PyInterpreterState_IsRunningMain(exc->interp)); - _PyInterpreterState_FailIfRunningMain(exc->interp); - } - else { - assert(exc->code == ERR_NOT_SET); - PyErr_SetNone(wrapperclass); - } -} - - -/* Python code **************************************************************/ - -static const char * -check_code_str(PyUnicodeObject *text) -{ - assert(text != NULL); - if (PyUnicode_GET_LENGTH(text) == 0) { - return "too short"; - } - - // XXX Verify that it parses? - - return NULL; -} - -static const char * -check_code_object(PyCodeObject *code) -{ - assert(code != NULL); - if (code->co_argcount > 0 - || code->co_posonlyargcount > 0 - || code->co_kwonlyargcount > 0 - || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) - { - return "arguments not supported"; - } - if (code->co_ncellvars > 0) { - return "closures not supported"; - } - // We trust that no code objects under co_consts have unbound cell vars. - - if (code->co_executors != NULL - || code->_co_instrumentation_version > 0) - { - return "only basic functions are supported"; - } - if (code->_co_monitoring != NULL) { - return "only basic functions are supported"; - } - if (code->co_extra != NULL) { - return "only basic functions are supported"; - } - - return NULL; -} - -#define RUN_TEXT 1 -#define RUN_CODE 2 - -static const char * -get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) -{ - const char *codestr = NULL; - Py_ssize_t len = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - - if (PyUnicode_Check(arg)) { - assert(PyUnicode_CheckExact(arg) - && (check_code_str((PyUnicodeObject *)arg) == NULL)); - codestr = PyUnicode_AsUTF8AndSize(arg, &len); - if (codestr == NULL) { - return NULL; - } - if (strlen(codestr) != (size_t)len) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - flags = RUN_TEXT; - } - else { - assert(PyCode_Check(arg) - && (check_code_object((PyCodeObject *)arg) == NULL)); - flags = RUN_CODE; - - // Serialize the code object. - bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); - if (bytes_obj == NULL) { - return NULL; - } - codestr = PyBytes_AS_STRING(bytes_obj); - len = PyBytes_GET_SIZE(bytes_obj); - } - - *flags_p = flags; - *bytes_p = bytes_obj; - *len_p = len; - return codestr; -} - - -/* interpreter-specific code ************************************************/ - -static int -exceptions_init(PyObject *mod) -{ - module_state *state = get_module_state(mod); - if (state == NULL) { - return -1; - } - -#define ADD(NAME, BASE) \ - do { \ - assert(state->NAME == NULL); \ - state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ - if (state->NAME == NULL) { \ - return -1; \ - } \ - } while (0) - - // An uncaught exception came out of interp_run_string(). - ADD(RunFailedError, PyExc_RuntimeError); -#undef ADD - - return 0; -} - -static int -_run_script(PyInterpreterState *interp, - const char *codestr, Py_ssize_t codestrlen, - _sharedns *shared, _sharedexception *sharedexc, int flags) -{ - int errcode = ERR_NOT_SET; - - if (_PyInterpreterState_SetRunningMain(interp) < 0) { - assert(PyErr_Occurred()); - // In the case where we didn't switch interpreters, it would - // be more efficient to leave the exception in place and return - // immediately. However, life is simpler if we don't. - PyErr_Clear(); - errcode = ERR_ALREADY_RUNNING; - goto error; - } - - PyObject *excval = NULL; - PyObject *main_mod = PyUnstable_InterpreterState_GetMainModule(interp); - if (main_mod == NULL) { - goto error; - } - PyObject *ns = PyModule_GetDict(main_mod); // borrowed - Py_DECREF(main_mod); - if (ns == NULL) { - goto error; - } - Py_INCREF(ns); - - // Apply the cross-interpreter data. - if (shared != NULL) { - if (_sharedns_apply(shared, ns) != 0) { - Py_DECREF(ns); - goto error; - } - } - - // Run the script/code/etc. - PyObject *result = NULL; - if (flags & RUN_TEXT) { - result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); - } - else if (flags & RUN_CODE) { - PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); - if (code != NULL) { - result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); - } - } - else { - Py_UNREACHABLE(); - } - Py_DECREF(ns); - if (result == NULL) { - goto error; - } - else { - Py_DECREF(result); // We throw away the result. - } - _PyInterpreterState_SetNotRunningMain(interp); - - *sharedexc = no_exception; - return 0; - -error: - excval = PyErr_GetRaisedException(); - const char *failure = _sharedexception_bind(excval, errcode, sharedexc); - if (failure != NULL) { - fprintf(stderr, - "RunFailedError: script raised an uncaught exception (%s)", - failure); - } - if (excval != NULL) { - // XXX Instead, store the rendered traceback on sharedexc, - // attach it to the exception when applied, - // and teach PyErr_Display() to print it. - PyErr_Display(NULL, excval, NULL); - Py_DECREF(excval); - } - if (errcode != ERR_ALREADY_RUNNING) { - _PyInterpreterState_SetNotRunningMain(interp); - } - assert(!PyErr_Occurred()); - return -1; -} - -static int -_run_in_interpreter(PyObject *mod, PyInterpreterState *interp, - const char *codestr, Py_ssize_t codestrlen, - PyObject *shareables, int flags) -{ - module_state *state = get_module_state(mod); - assert(state != NULL); - - _sharedns *shared = _get_shared_ns(shareables); - if (shared == NULL && PyErr_Occurred()) { - return -1; - } - - // Switch to interpreter. - PyThreadState *save_tstate = NULL; - PyThreadState *tstate = NULL; - if (interp != PyInterpreterState_Get()) { - tstate = PyThreadState_New(interp); - tstate->_whence = _PyThreadState_WHENCE_EXEC; - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - } - - // Run the script. - _sharedexception exc = (_sharedexception){ .interp = interp }; - int result = _run_script(interp, codestr, codestrlen, shared, &exc, flags); - - // Switch back. - if (save_tstate != NULL) { - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - } - - // Propagate any exception out to the caller. - if (result < 0) { - assert(!PyErr_Occurred()); - _sharedexception_apply(&exc, state->RunFailedError); - assert(PyErr_Occurred()); - } - - if (shared != NULL) { - _sharedns_free(shared); - } - - return result; -} - - -/* module level code ********************************************************/ - -static PyObject * -interp_create(PyObject *self, PyObject *args, PyObject *kwds) -{ - - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { - return NULL; - } - - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); - _PyErr_ChainExceptions1(exc); - return NULL; - } - assert(tstate != NULL); - - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = PyInterpreterState_GetIDObject(interp); - if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; - } - - PyThreadState_Clear(tstate); - PyThreadState_Delete(tstate); - - _PyInterpreterState_RequireIDRef(interp, 1); - return idobj; -} - -PyDoc_STRVAR(create_doc, -"create() -> ID\n\ -\n\ -Create a new interpreter and return a unique generated ID."); - - -static PyObject * -interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *id; - // XXX Use "L" for id? - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:destroy", kwlist, &id)) { - return NULL; - } - - // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); - if (interp == NULL) { - return NULL; - } - - // Ensure we don't try to destroy the current interpreter. - PyInterpreterState *current = _get_current_interp(); - if (current == NULL) { - return NULL; - } - if (interp == current) { - PyErr_SetString(PyExc_RuntimeError, - "cannot destroy the current interpreter"); - return NULL; - } - - // Ensure the interpreter isn't running. - /* XXX We *could* support destroying a running interpreter but - aren't going to worry about it for now. */ - if (_PyInterpreterState_IsRunningMain(interp)) { - PyErr_Format(PyExc_RuntimeError, "interpreter running"); - return NULL; - } - - // Destroy the interpreter. - PyThreadState *tstate = PyThreadState_New(interp); - tstate->_whence = _PyThreadState_WHENCE_INTERP; - // XXX Possible GILState issues? - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(destroy_doc, -"destroy(id)\n\ -\n\ -Destroy the identified interpreter.\n\ -\n\ -Attempting to destroy the current interpreter results in a RuntimeError.\n\ -So does an unrecognized ID."); - - -static PyObject * -interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *ids, *id; - PyInterpreterState *interp; - - ids = PyList_New(0); - if (ids == NULL) { - return NULL; - } - - interp = PyInterpreterState_Head(); - while (interp != NULL) { - id = PyInterpreterState_GetIDObject(interp); - if (id == NULL) { - Py_DECREF(ids); - return NULL; - } - // insert at front of list - int res = PyList_Insert(ids, 0, id); - Py_DECREF(id); - if (res < 0) { - Py_DECREF(ids); - return NULL; - } - - interp = PyInterpreterState_Next(interp); - } - - return ids; -} - -PyDoc_STRVAR(list_all_doc, -"list_all() -> [ID]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); - - -static PyObject * -interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyInterpreterState *interp =_get_current_interp(); - if (interp == NULL) { - return NULL; - } - return PyInterpreterState_GetIDObject(interp); -} - -PyDoc_STRVAR(get_current_doc, -"get_current() -> ID\n\ -\n\ -Return the ID of current interpreter."); - - -static PyObject * -interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - // Currently, 0 is always the main interpreter. - int64_t id = 0; - return PyInterpreterID_New(id); -} - -PyDoc_STRVAR(get_main_doc, -"get_main() -> ID\n\ -\n\ -Return the ID of main interpreter."); - - -static PyUnicodeObject * -convert_script_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - PyUnicodeObject *str = NULL; - if (PyUnicode_CheckExact(arg)) { - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else if (PyUnicode_Check(arg)) { - // XXX str = PyUnicode_FromObject(arg); - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_str(str); - if (err != NULL) { - Py_DECREF(str); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad script text (%s)", fname, err); - return NULL; - } - - return str; -} - -static PyCodeObject * -convert_code_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - const char *kind = NULL; - PyCodeObject *code = NULL; - if (PyFunction_Check(arg)) { - if (PyFunction_GetClosure(arg) != NULL) { - PyErr_Format(PyExc_ValueError, - "%.200s(): closures not supported", fname); - return NULL; - } - code = (PyCodeObject *)PyFunction_GetCode(arg); - if (code == NULL) { - if (PyErr_Occurred()) { - // This chains. - PyErr_Format(PyExc_ValueError, - "%.200s(): bad func", fname); - } - else { - PyErr_Format(PyExc_ValueError, - "%.200s(): func.__code__ missing", fname); - } - return NULL; - } - Py_INCREF(code); - kind = "func"; - } - else if (PyCode_Check(arg)) { - code = (PyCodeObject *)Py_NewRef(arg); - kind = "code object"; - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_object(code); - if (err != NULL) { - Py_DECREF(code); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad %s (%s)", fname, kind, err); - return NULL; - } - - return code; -} - -static int -_interp_exec(PyObject *self, - PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg) -{ - // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); - if (interp == NULL) { - return -1; - } - - // Extract code. - Py_ssize_t codestrlen = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - const char *codestr = get_code_str(code_arg, - &codestrlen, &bytes_obj, &flags); - if (codestr == NULL) { - return -1; - } - - // Run the code in the interpreter. - int res = _run_in_interpreter(self, interp, codestr, codestrlen, - shared_arg, flags); - Py_XDECREF(bytes_obj); - if (res != 0) { - return -1; - } - - return 0; -} - -static PyObject * -interp_exec(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "code", "shared", NULL}; - PyObject *id, *code; - PyObject *shared = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".exec", kwlist, - &id, &code, &shared)) { - return NULL; - } - - const char *expected = "a string, a function, or a code object"; - if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec", - "argument 2", expected); - } - else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec", - "argument 2", expected); - } - if (code == NULL) { - return NULL; - } - - int res = _interp_exec(self, id, code, shared); - Py_DECREF(code); - if (res < 0) { - return NULL; - } - Py_RETURN_NONE; -} - -PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None)\n\ -\n\ -Execute the provided code in the identified interpreter.\n\ -This is equivalent to running the builtin exec() under the target\n\ -interpreter, using the __dict__ of its __main__ module as both\n\ -globals and locals.\n\ -\n\ -\"code\" may be a string containing the text of a Python script.\n\ -\n\ -Functions (and code objects) are also supported, with some restrictions.\n\ -The code/function must not take any arguments or be a closure\n\ -(i.e. have cell vars). Methods and other callables are not supported.\n\ -\n\ -If a function is provided, its code object is used and all its state\n\ -is ignored, including its __globals__ dict."); - -static PyObject * -interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "script", "shared", NULL}; - PyObject *id, *script; - PyObject *shared = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME ".run_string", kwlist, - &id, &script, &shared)) { - return NULL; - } - - script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec", - "argument 2", "a string"); - if (script == NULL) { - return NULL; - } - - int res = _interp_exec(self, id, (PyObject *)script, shared); - Py_DECREF(script); - if (res < 0) { - return NULL; - } - Py_RETURN_NONE; -} - -PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None)\n\ -\n\ -Execute the provided string in the identified interpreter.\n\ -\n\ -(See " MODULE_NAME ".exec()."); - -static PyObject * -interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "func", "shared", NULL}; - PyObject *id, *func; - PyObject *shared = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".run_func", kwlist, - &id, &func, &shared)) { - return NULL; - } - - PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec", - "argument 2", - "a function or a code object"); - if (code == NULL) { - return NULL; - } - - int res = _interp_exec(self, id, (PyObject *)code, shared); - Py_DECREF(code); - if (res < 0) { - return NULL; - } - Py_RETURN_NONE; -} - -PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None)\n\ -\n\ -Execute the body of the provided function in the identified interpreter.\n\ -Code objects are also supported. In both cases, closures and args\n\ -are not supported. Methods and other callables are not supported either.\n\ -\n\ -(See " MODULE_NAME ".exec()."); - - -static PyObject * -object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"obj", NULL}; - PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_shareable", kwlist, &obj)) { - return NULL; - } - - if (_PyObject_CheckCrossInterpreterData(obj) == 0) { - Py_RETURN_TRUE; - } - PyErr_Clear(); - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(is_shareable_doc, -"is_shareable(obj) -> bool\n\ -\n\ -Return True if the object's data may be shared between interpreters and\n\ -False otherwise."); - - -static PyObject * -interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *id; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_running", kwlist, &id)) { - return NULL; - } - - PyInterpreterState *interp = PyInterpreterID_LookUp(id); - if (interp == NULL) { - return NULL; - } - if (_PyInterpreterState_IsRunningMain(interp)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(is_running_doc, -"is_running(id) -> bool\n\ -\n\ -Return whether or not the identified interpreter is running."); - - -static PyMethodDef module_functions[] = { - {"create", _PyCFunction_CAST(interp_create), - METH_VARARGS | METH_KEYWORDS, create_doc}, - {"destroy", _PyCFunction_CAST(interp_destroy), - METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", interp_list_all, - METH_NOARGS, list_all_doc}, - {"get_current", interp_get_current, - METH_NOARGS, get_current_doc}, - {"get_main", interp_get_main, - METH_NOARGS, get_main_doc}, - - {"is_running", _PyCFunction_CAST(interp_is_running), - METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"exec", _PyCFunction_CAST(interp_exec), - METH_VARARGS | METH_KEYWORDS, exec_doc}, - {"run_string", _PyCFunction_CAST(interp_run_string), - METH_VARARGS | METH_KEYWORDS, run_string_doc}, - {"run_func", _PyCFunction_CAST(interp_run_func), - METH_VARARGS | METH_KEYWORDS, run_func_doc}, - - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, - - {NULL, NULL} /* sentinel */ -}; - - -/* initialization function */ - -PyDoc_STRVAR(module_doc, -"This module provides primitive operations to manage Python interpreters.\n\ -The 'interpreters' module provides a more convenient interface."); - -static int -module_exec(PyObject *mod) -{ - /* Add exception types */ - if (exceptions_init(mod) != 0) { - goto error; - } - - // PyInterpreterID - if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) { - goto error; - } - - return 0; - -error: - return -1; -} - -static struct PyModuleDef_Slot module_slots[] = { - {Py_mod_exec, module_exec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {0, NULL}, -}; - -static int -module_traverse(PyObject *mod, visitproc visit, void *arg) -{ - module_state *state = get_module_state(mod); - assert(state != NULL); - traverse_module_state(state, visit, arg); - return 0; -} - -static int -module_clear(PyObject *mod) -{ - module_state *state = get_module_state(mod); - assert(state != NULL); - clear_module_state(state); - return 0; -} - -static void -module_free(void *mod) -{ - module_state *state = get_module_state(mod); - assert(state != NULL); - clear_module_state(state); -} - -static struct PyModuleDef moduledef = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = MODULE_NAME, - .m_doc = module_doc, - .m_size = sizeof(module_state), - .m_methods = module_functions, - .m_slots = module_slots, - .m_traverse = module_traverse, - .m_clear = module_clear, - .m_free = (freefunc)module_free, -}; - -PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) -{ - return PyModuleDef_Init(&moduledef); -} diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index a2dbabce71ed67..2952d7043e01fe 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,13 +28,18 @@ static PyMethodDef module_methods[] = { {NULL}, }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _fuzzmodule = { PyModuleDef_HEAD_INIT, "_fuzz", NULL, 0, module_methods, - NULL, + module_slots, NULL, NULL, NULL @@ -43,5 +48,5 @@ static struct PyModuleDef _fuzzmodule = { PyMODINIT_FUNC PyInit__xxtestfuzz(void) { - return PyModule_Create(&_fuzzmodule); + return PyModuleDef_Init(&_fuzzmodule); } diff --git a/Modules/_xxtestfuzz/dictionaries/fuzz_elementtree_parsewhole.dict b/Modules/_xxtestfuzz/dictionaries/fuzz_elementtree_parsewhole.dict new file mode 100644 index 00000000000000..e1b58cdb248238 --- /dev/null +++ b/Modules/_xxtestfuzz/dictionaries/fuzz_elementtree_parsewhole.dict @@ -0,0 +1,134 @@ +tok_1="<" +tok_2=">" +tok_3="/" +tok_4="" +tok_6="" +tok_7="version" +tok_8="encoding" +tok_9="UTF-8" +tok_9a="UTF-16" +tok_9b="ASCII" +tok_9c="LATIN-1" +tok_9d="UTF-32" +tok_9e="UTF-7" +tok_10="\"" +tok_11="&" +tok_11a="&#" +tok_11b=";" +tok_12="'" +tok_13="" +tok_14="" +tag_doctype="" +tag_open_close="" +tag_open_exclamation="" +tag_xml_q="" + +encoding_utf="UTF-" +encoding_iso1="ISO-8859" +encoding_iso3="ISO-10646-UCS" +encoding_iso5="ISO-LATIN-1" +encoding_jis="SHIFT_JIS" +encoding_utf7="UTF-7" +encoding_utf16le="UTF-16BE" +encoding_utf16le="UTF-16LE" +encoding_ascii="US-ASCII" +encoding_latin1="latin1" diff --git a/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict new file mode 100644 index 00000000000000..c6a44d946284ef --- /dev/null +++ b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict @@ -0,0 +1,165 @@ +# bits of syntax +"( " +") " +"[ " +"] " +": " +", " +"; " +"{ " +"} " + +# operators +"+ " +"- " +"* " +"** " +"/ " +"// " +"| " +"& " +"< " +"> " +"= " +". " +"% " +"` " +"^ " +"~ " +"@ " +"== " +"!= " +"<> " +"<< " +"<= " +">= " +">> " +"+= " +"-= " +"*= " +"** " +"/= " +"//= " +"|= " +"%= " +"&= " +"^= " +"<<= " +">>= " +"**= " +":= " +"@= " + +# whitespace +" " +":\\n " + +# type signatures and functions +"-> " +": List[int]" +": Dict[int, str]" + +"# type:" +"# type: List[int]" +"# type: Dict[int, str]" + +", *" +", /" +", *args" +", **kwargs" +", x=42" + + +# literals +"0x0a" +"0b0000" +"42" +"0o70" +"42j" +"42.01" +"-5" +"+42e-3" +"0_0_0" +"1e1_0" +".1_4" + +"{}" + +# variable names +"x" +"y" + +# strings +"r'x'" + +"b'x'" + +"rb\"x\"" + +"br\"x\"" + +"f'{x + 5}'" +"f\"{x + 5}\"" + +"'''" +"\"\"\"" + +"\\u" +"\\x" + +# keywords +"def " +"del " +"pass " +"break " +"continue " +"return " +"raise " +"from " +"import " +".. " +"... " +"__future__ " +"as " +"global " +"nonlocal " +"assert " +"print " +"if " +"elif " +"else: " +"while " +"try: " +"except " +"finally: " +"with " +"lambda " +"or " +"and " +"not " +"None " +"__peg_parser__" +"True " +"False " +"yield " +"async " +"await " +"for " +"in " +"is " +"class " + +# shebangs and encodings +"#!" +"# coding:" +"# coding=" +"# coding: latin-1" +"# coding=latin-1" +"# coding: utf-8" +"# coding=utf-8" +"# coding: ascii" +"# coding=ascii" +"# coding: cp860" +"# coding=cp860" +"# coding: gbk" +"# coding=gbk" diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nComment.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nComment.xml new file mode 100644 index 00000000000000..e95aa302d04fdb --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nComment.xml @@ -0,0 +1,4 @@ + + true + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nDefault.xml new file mode 100644 index 00000000000000..c1364142cc59bf --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nDefault.xml @@ -0,0 +1,3 @@ + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefix.xml new file mode 100644 index 00000000000000..fb233b42b1334f --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefix.xml @@ -0,0 +1,4 @@ + + sequential + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQname.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQname.xml new file mode 100644 index 00000000000000..23188eedbc2451 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQname.xml @@ -0,0 +1,7 @@ + + sequential + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQnameXpathElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQnameXpathElem.xml new file mode 100644 index 00000000000000..626fc48f410fa0 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nPrefixQnameXpathElem.xml @@ -0,0 +1,8 @@ + + sequential + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQname.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQname.xml new file mode 100644 index 00000000000000..919e5903f5ce6e --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQname.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameElem.xml new file mode 100644 index 00000000000000..0321f8061952e6 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameElem.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameXpathElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameXpathElem.xml new file mode 100644 index 00000000000000..c4890bc8b01d5e --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nQnameXpathElem.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nTrim.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nTrim.xml new file mode 100644 index 00000000000000..ccb9cf65db7235 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/c14nTrim.xml @@ -0,0 +1,4 @@ + + true + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/expat224_utf8_bug.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/expat224_utf8_bug.xml new file mode 100644 index 00000000000000..d66a8e6b50f93b --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/expat224_utf8_bug.xml @@ -0,0 +1,2 @@ + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N1.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N1.xml new file mode 100644 index 00000000000000..ed450c7341d382 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N1.xml @@ -0,0 +1,14 @@ + + + + + + +Hello, world! + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N2.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N2.xml new file mode 100644 index 00000000000000..74eeea147c3791 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N2.xml @@ -0,0 +1,11 @@ + + + A B + + A + + B + A B + C + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N3.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N3.xml new file mode 100644 index 00000000000000..fea78213f1ae69 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N3.xml @@ -0,0 +1,18 @@ +]> + + + + + + + + + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N4.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N4.xml new file mode 100644 index 00000000000000..909a847435b86c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N4.xml @@ -0,0 +1,13 @@ + + +]> + + First line Second line + 2 + "0" && value<"10" ?"valid":"error"]]> + valid + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N5.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N5.xml new file mode 100644 index 00000000000000..501161bad5187f --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N5.xml @@ -0,0 +1,12 @@ + + + + + +]> + + &ent1;, &ent2;! + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N6.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N6.xml new file mode 100644 index 00000000000000..31e2071867257c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inC14N6.xml @@ -0,0 +1,2 @@ + +© diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsContent.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsContent.xml new file mode 100644 index 00000000000000..b9924660ba6da3 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsContent.xml @@ -0,0 +1,4 @@ + + xsd:string + /soap-env:body/child::b:foo[@att1 != "c:val" and @att2 != 'xsd:string'] + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsDefault.xml new file mode 100644 index 00000000000000..3e0d323bad27c2 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsDefault.xml @@ -0,0 +1,3 @@ + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsPushdown.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsPushdown.xml new file mode 100644 index 00000000000000..daa67d83f15914 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsPushdown.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsRedecl.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsRedecl.xml new file mode 100644 index 00000000000000..10bd97beda3baa --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsRedecl.xml @@ -0,0 +1,3 @@ + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSort.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSort.xml new file mode 100644 index 00000000000000..8e9fc01c647b24 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSort.xml @@ -0,0 +1,4 @@ + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSuperfluous.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSuperfluous.xml new file mode 100644 index 00000000000000..f77720f7b0b09d --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsSuperfluous.xml @@ -0,0 +1,4 @@ + + + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsXml.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsXml.xml new file mode 100644 index 00000000000000..7520cf3fb9eb28 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/inNsXml.xml @@ -0,0 +1,3 @@ + + data + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nComment.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nComment.xml new file mode 100644 index 00000000000000..d98d16840c6bcc --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nComment.xml @@ -0,0 +1,6 @@ + +Hello, world! + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nDefault.xml new file mode 100644 index 00000000000000..af9a9770578e9d --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N1_c14nDefault.xml @@ -0,0 +1,4 @@ + +Hello, world! + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nDefault.xml new file mode 100644 index 00000000000000..2afa15ccb36382 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nDefault.xml @@ -0,0 +1,11 @@ + + + A B + + A + + B + A B + C + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nTrim.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nTrim.xml new file mode 100644 index 00000000000000..7a1dc32946bce3 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N2_c14nTrim.xml @@ -0,0 +1 @@ +A BABA BC \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nDefault.xml new file mode 100644 index 00000000000000..662e108aa8a1e4 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nDefault.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nPrefix.xml new file mode 100644 index 00000000000000..041e1ec8ebe59a --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nPrefix.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nTrim.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nTrim.xml new file mode 100644 index 00000000000000..4f35ad9662df3b --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N3_c14nTrim.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nDefault.xml new file mode 100644 index 00000000000000..243d0e61f2e94f --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nDefault.xml @@ -0,0 +1,10 @@ + + First line +Second line + 2 + value>"0" && value<"10" ?"valid":"error" + valid + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nTrim.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nTrim.xml new file mode 100644 index 00000000000000..24d83ba8ab0012 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N4_c14nTrim.xml @@ -0,0 +1,2 @@ +First line +Second line2value>"0" && value<"10" ?"valid":"error"valid \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nDefault.xml new file mode 100644 index 00000000000000..c232e740aee4a7 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nDefault.xml @@ -0,0 +1,3 @@ + + Hello, world! + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nTrim.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nTrim.xml new file mode 100644 index 00000000000000..3fa84b1e986014 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N5_c14nTrim.xml @@ -0,0 +1 @@ +Hello, world! \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N6_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N6_c14nDefault.xml new file mode 100644 index 00000000000000..0be38f98cb1398 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inC14N6_c14nDefault.xml @@ -0,0 +1 @@ +© \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nDefault.xml new file mode 100644 index 00000000000000..62d7e004a44034 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nDefault.xml @@ -0,0 +1,4 @@ + + xsd:string + /soap-env:body/child::b:foo[@att1 != "c:val" and @att2 != 'xsd:string'] + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nPrefixQnameXpathElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nPrefixQnameXpathElem.xml new file mode 100644 index 00000000000000..20e1c2e9d6dfb4 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nPrefixQnameXpathElem.xml @@ -0,0 +1,4 @@ + + n1:string + /n3:body/child::n2:foo[@att1 != "c:val" and @att2 != 'xsd:string'] + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameElem.xml new file mode 100644 index 00000000000000..db8680daa033d7 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameElem.xml @@ -0,0 +1,4 @@ + + xsd:string + /soap-env:body/child::b:foo[@att1 != "c:val" and @att2 != 'xsd:string'] + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameXpathElem.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameXpathElem.xml new file mode 100644 index 00000000000000..df3b21579fac5e --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsContent_c14nQnameXpathElem.xml @@ -0,0 +1,4 @@ + + xsd:string + /soap-env:body/child::b:foo[@att1 != "c:val" and @att2 != 'xsd:string'] + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nDefault.xml new file mode 100644 index 00000000000000..674b076dd6d9a6 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nDefault.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nPrefix.xml new file mode 100644 index 00000000000000..83edaae91e7423 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsDefault_c14nPrefix.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nDefault.xml new file mode 100644 index 00000000000000..fa4f21b5d0af55 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nDefault.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nPrefix.xml new file mode 100644 index 00000000000000..6d579200c9dc8c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsPushdown_c14nPrefix.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nDefault.xml new file mode 100644 index 00000000000000..ba37f925103c70 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nDefault.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nPrefix.xml new file mode 100644 index 00000000000000..af3bb2d6f062cd --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsRedecl_c14nPrefix.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nDefault.xml new file mode 100644 index 00000000000000..8a92c5c61c2c2c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nDefault.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nPrefix.xml new file mode 100644 index 00000000000000..8d44c84fe5d307 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSort_c14nPrefix.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nDefault.xml new file mode 100644 index 00000000000000..6bb862d763d737 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nDefault.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nPrefix.xml new file mode 100644 index 00000000000000..700a16d42a7746 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsSuperfluous_c14nPrefix.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nDefault.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nDefault.xml new file mode 100644 index 00000000000000..1689f3bf423dc5 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nDefault.xml @@ -0,0 +1,3 @@ + + data + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefix.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefix.xml new file mode 100644 index 00000000000000..38508a47f6b904 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefix.xml @@ -0,0 +1,3 @@ + + data + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefixQname.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefixQname.xml new file mode 100644 index 00000000000000..867980f82bfa59 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nPrefixQname.xml @@ -0,0 +1,3 @@ + + data + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nQname.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nQname.xml new file mode 100644 index 00000000000000..0300f9d562db30 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/out_inNsXml_c14nQname.xml @@ -0,0 +1,3 @@ + + data + \ No newline at end of file diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple-ns.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple-ns.xml new file mode 100644 index 00000000000000..f1f34b2e29c73e --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple-ns.xml @@ -0,0 +1,7 @@ + + + + text + texttail + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple.xml new file mode 100644 index 00000000000000..b88c2c7e69a088 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/simple.xml @@ -0,0 +1,6 @@ + + + text + texttail + + diff --git a/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/test.xml b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/test.xml new file mode 100644 index 00000000000000..92136da76d3581 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_elementtree_parsewhole_corpus/test.xml @@ -0,0 +1,115 @@ + + +Introduction to XSL +

    Introduction to XSL

    + + + +
    +

    Overview +

    +
      + +
    • 1.Intro
    • + +
    • 2.History
    • + +
    • 3.XSL Basics
    • + +
    • Lunch
    • + +
    • 4.An XML Data Model
    • + +
    • 5.XSL Patterns
    • + +
    • 6.XSL Templates
    • + +
    • 7.XSL Formatting Model +
    • + +
    + + + + + + +
    +

    Intro

    +
      + +
    • Who am I?
    • + +
    • Who are you?
    • + +
    • Why are we here? +
    • + +
    + + + + + + +
    +

    History: XML and SGML

    +
      + +
    • XML is a subset of SGML.
    • + +
    • SGML allows the separation of abstract content from formatting.
    • + +
    • Also one of XML's primary virtues (in the doc publishing domain). +
    • + +
    + + + + + + +
    +

    History: What are stylesheets?

    +
      + +
    • Stylesheets specify the formatting of SGML/XML documents.
    • + +
    • Stylesheets put the "style" back into documents.
    • + +
    • New York Times content+NYT Stylesheet = NYT paper +
    • + +
    + + + + + + +
    +

    History: FOSI

    +
      + +
    • FOSI: "Formatted Output Specification Instance" +
        +
      • MIL-STD-28001 +
      • + +
      • FOSI's are SGML documents +
      • + +
      • A stylesheet for another document +
      • +
    • + +
    • Obsolete but implemented... +
    • + +
    + + +µ + + diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py new file mode 100644 index 00000000000000..c43994dda29eed --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +def test() -> None: + x: list[int] = [] + x: dict[int, str] = {} + x: set[bytes] = {} + print(5 + 42 * 3, x) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py new file mode 100644 index 00000000000000..7be326e95be0eb --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py @@ -0,0 +1,5 @@ +class Foo(metaclass=42): + __slots__ = ['x'] + pass + +foo = Foo() diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py new file mode 100644 index 00000000000000..9bc3a45ebe75da --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py @@ -0,0 +1,6 @@ +def evens(): + i = 0 + while True: + i += 1 + if i % 2 == 0: + yield i diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py new file mode 100644 index 00000000000000..490de90fb97b39 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py @@ -0,0 +1,3 @@ +async def hello(name: str): + await name + print(name) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py new file mode 100644 index 00000000000000..4cfcfe590ebc95 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py @@ -0,0 +1,7 @@ +try: + eval('importer exporter... really long matches') +except SyntaxError: + print("nothing to see here") +finally: + print("all done here") + raise diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py new file mode 100644 index 00000000000000..d8e59ade503a8c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py @@ -0,0 +1,8 @@ +"""Some module docstring""" +import sys + +def main(): + print("Hello world!", file=sys.stderr) + +if __name__ == '__main__': + main() diff --git a/Modules/_xxtestfuzz/fuzz_tests.txt b/Modules/_xxtestfuzz/fuzz_tests.txt index 4e046ecf6d898c..ea6f982eefc9da 100644 --- a/Modules/_xxtestfuzz/fuzz_tests.txt +++ b/Modules/_xxtestfuzz/fuzz_tests.txt @@ -7,3 +7,5 @@ fuzz_sre_match fuzz_csv_reader fuzz_struct_unpack fuzz_ast_literal_eval +fuzz_elementtree_parsewhole +fuzz_pycompile diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 816ba09c8fd7de..6ea9f64d628530 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -439,6 +439,132 @@ static int fuzz_ast_literal_eval(const char* data, size_t size) { return 0; } +#define MAX_ELEMENTTREE_PARSEWHOLE_TEST_SIZE 0x100000 +PyObject* xmlparser_type = NULL; +PyObject* bytesio_type = NULL; +/* Called by LLVMFuzzerTestOneInput for initialization */ +static int init_elementtree_parsewhole(void) { + PyObject* elementtree_module = PyImport_ImportModule("_elementtree"); + if (elementtree_module == NULL) { + return 0; + } + xmlparser_type = PyObject_GetAttrString(elementtree_module, "XMLParser"); + Py_DECREF(elementtree_module); + if (xmlparser_type == NULL) { + return 0; + } + + + PyObject* io_module = PyImport_ImportModule("io"); + if (io_module == NULL) { + return 0; + } + bytesio_type = PyObject_GetAttrString(io_module, "BytesIO"); + Py_DECREF(io_module); + if (bytesio_type == NULL) { + return 0; + } + + return 1; +} +/* Fuzz _elementtree.XMLParser._parse_whole(x) */ +static int fuzz_elementtree_parsewhole(const char* data, size_t size) { + if (size > MAX_ELEMENTTREE_PARSEWHOLE_TEST_SIZE) { + return 0; + } + + PyObject *input = PyObject_CallFunction(bytesio_type, "y#", data, (Py_ssize_t)size); + if (input == NULL) { + assert(PyErr_Occurred()); + PyErr_Print(); + abort(); + } + + PyObject *xmlparser_instance = PyObject_CallObject(xmlparser_type, NULL); + if (xmlparser_instance == NULL) { + assert(PyErr_Occurred()); + PyErr_Print(); + abort(); + } + + PyObject *result = PyObject_CallMethod(xmlparser_instance, "_parse_whole", "O", input); + if (result == NULL) { + /* Ignore exception here, which can be caused by invalid XML input */ + PyErr_Clear(); + } else { + Py_DECREF(result); + } + + Py_DECREF(xmlparser_instance); + Py_DECREF(input); + + return 0; +} + +#define MAX_PYCOMPILE_TEST_SIZE 16384 + +static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; +const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); + +static const int optimize_vals[] = {-1, 0, 1, 2}; +const size_t NUM_OPTIMIZE_VALS = sizeof(optimize_vals) / sizeof(optimize_vals[0]); + +/* Fuzz `PyCompileStringExFlags` using a variety of input parameters. + * That function is essentially behind the `compile` builtin */ +static int fuzz_pycompile(const char* data, size_t size) { + // Ignore overly-large inputs, and account for a NUL terminator + if (size > MAX_PYCOMPILE_TEST_SIZE - 1) { + return 0; + } + + // Need 2 bytes for parameter selection + if (size < 2) { + return 0; + } + + // Use first byte to determine element of `start_vals` to use + unsigned char start_idx = (unsigned char) data[0]; + int start = start_vals[start_idx % NUM_START_VALS]; + + // Use second byte to determine element of `optimize_vals` to use + unsigned char optimize_idx = (unsigned char) data[1]; + int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + + char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + + // Create a NUL-terminated C string from the remaining input + memcpy(pycompile_scratch, data + 2, size - 2); + // Put a NUL terminator just after the copied data. (Space was reserved already.) + pycompile_scratch[size - 2] = '\0'; + + // XXX: instead of always using NULL for the `flags` value to + // `Py_CompileStringExFlags`, there are many flags that conditionally + // change parser behavior: + // + // #define PyCF_TYPE_COMMENTS 0x1000 + // #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 + // #define PyCF_ONLY_AST 0x0400 + // + // It would be good to test various combinations of these, too. + PyCompilerFlags *flags = NULL; + + PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, flags, optimize); + if (result == NULL) { + /* Compilation failed, most likely from a syntax error. If it was a + SystemError we abort. There's no non-bug reason to raise a + SystemError. */ + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_SystemError)) { + PyErr_Print(); + abort(); + } + PyErr_Clear(); + } else { + Py_DECREF(result); + } + + return 0; +} + /* Run fuzzer and abort on failure. */ static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) { int rv = fuzzer((const char*) data, size); @@ -569,6 +695,20 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } rv |= _run_fuzz(data, size, fuzz_ast_literal_eval); +#endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_elementtree_parsewhole) + static int ELEMENTTREE_PARSEWHOLE_INITIALIZED = 0; + if (!ELEMENTTREE_PARSEWHOLE_INITIALIZED && !init_elementtree_parsewhole()) { + PyErr_Print(); + abort(); + } else { + ELEMENTTREE_PARSEWHOLE_INITIALIZED = 1; + } + + rv |= _run_fuzz(data, size, fuzz_elementtree_parsewhole); +#endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_pycompile) + rv |= _run_fuzz(data, size, fuzz_pycompile); #endif return rv; } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index eb4e522465181f..902ece795b575b 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -61,21 +61,21 @@ struct TransitionRuleType { typedef struct { TransitionRuleType base; - uint8_t month; - uint8_t week; - uint8_t day; - int8_t hour; - int8_t minute; - int8_t second; + uint8_t month; /* 1 - 12 */ + uint8_t week; /* 1 - 5 */ + uint8_t day; /* 0 - 6 */ + int16_t hour; /* -167 - 167, RFC 8536 §3.3.1 */ + int8_t minute; /* signed 2 digits */ + int8_t second; /* signed 2 digits */ } CalendarRule; typedef struct { TransitionRuleType base; - uint8_t julian; - unsigned int day; - int8_t hour; - int8_t minute; - int8_t second; + uint8_t julian; /* 0, 1 */ + uint16_t day; /* 0 - 365 */ + int16_t hour; /* -167 - 167, RFC 8536 §3.3.1 */ + int8_t minute; /* signed 2 digits */ + int8_t second; /* signed 2 digits */ } DayRule; struct StrongCacheNode { @@ -133,15 +133,14 @@ ts_to_local(size_t *trans_idx, int64_t *trans_utc, long *utcoff, static int parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out); -static Py_ssize_t -parse_abbr(const char *const p, PyObject **abbr); -static Py_ssize_t -parse_tz_delta(const char *const p, long *total_seconds); -static Py_ssize_t -parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, - int8_t *second); -static Py_ssize_t -parse_transition_rule(const char *const p, TransitionRuleType **out); +static int +parse_abbr(const char **p, PyObject **abbr); +static int +parse_tz_delta(const char **p, long *total_seconds); +static int +parse_transition_time(const char **p, int *hour, int *minute, int *second); +static int +parse_transition_rule(const char **p, TransitionRuleType **out); static _ttinfo * find_tzrule_ttinfo(_tzrule *rule, int64_t ts, unsigned char fold, int year); @@ -421,7 +420,7 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, return NULL; } - file_repr = PyUnicode_FromFormat("%R", file_obj); + file_repr = PyObject_Repr(file_obj); if (file_repr == NULL) { goto error; } @@ -817,7 +816,7 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = Py_BuildValue("(O)", key); + PyObject *val_args = PyTuple_Pack(1, key); if (val_args == NULL) { return NULL; } @@ -854,28 +853,19 @@ load_timedelta(zoneinfo_state *state, long seconds) if (pyoffset == NULL) { return NULL; } - rv = PyDict_GetItemWithError(state->TIMEDELTA_CACHE, pyoffset); - if (rv == NULL) { - if (PyErr_Occurred()) { - goto error; - } + if (PyDict_GetItemRef(state->TIMEDELTA_CACHE, pyoffset, &rv) == 0) { PyObject *tmp = PyDateTimeAPI->Delta_FromDelta( 0, seconds, 0, 1, PyDateTimeAPI->DeltaType); - if (tmp == NULL) { - goto error; + if (tmp != NULL) { + rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); + Py_XINCREF(rv); + Py_DECREF(tmp); } - - rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); - Py_DECREF(tmp); } - Py_XINCREF(rv); Py_DECREF(pyoffset); return rv; -error: - Py_DECREF(pyoffset); - return NULL; } /* Constructor for _ttinfo object - this starts by initializing the _ttinfo @@ -954,6 +944,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) static int load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { + int rv = 0; PyObject *data_tuple = NULL; long *utcoff = NULL; @@ -1230,7 +1221,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } } - int rv = 0; goto cleanup; error: // These resources only need to be freed if we have failed, if we succeed @@ -1327,14 +1317,14 @@ calendarrule_year_to_timestamp(TransitionRuleType *base_self, int year) } int64_t ordinal = ymd_to_ord(year, self->month, month_day) - EPOCHORDINAL; - return ((ordinal * 86400) + (int64_t)(self->hour * 3600) + - (int64_t)(self->minute * 60) + (int64_t)(self->second)); + return ordinal * 86400 + (int64_t)self->hour * 3600 + + (int64_t)self->minute * 60 + self->second; } /* Constructor for CalendarRule. */ int -calendarrule_new(uint8_t month, uint8_t week, uint8_t day, int8_t hour, - int8_t minute, int8_t second, CalendarRule *out) +calendarrule_new(int month, int week, int day, int hour, + int minute, int second, CalendarRule *out) { // These bounds come from the POSIX standard, which describes an Mm.n.d // rule as: @@ -1343,33 +1333,36 @@ calendarrule_new(uint8_t month, uint8_t week, uint8_t day, int8_t hour, // 5, 1 <= m <= 12, where week 5 means "the last d day in month m" which // may occur in either the fourth or the fifth week). Week 1 is the first // week in which the d'th day occurs. Day zero is Sunday. - if (month <= 0 || month > 12) { - PyErr_Format(PyExc_ValueError, "Month must be in (0, 12]"); + if (month < 1 || month > 12) { + PyErr_Format(PyExc_ValueError, "Month must be in [1, 12]"); return -1; } - if (week <= 0 || week > 5) { - PyErr_Format(PyExc_ValueError, "Week must be in (0, 5]"); + if (week < 1 || week > 5) { + PyErr_Format(PyExc_ValueError, "Week must be in [1, 5]"); return -1; } - // If the 'day' parameter type is changed to a signed type, - // "day < 0" check must be added. - if (/* day < 0 || */ day > 6) { + if (day < 0 || day > 6) { PyErr_Format(PyExc_ValueError, "Day must be in [0, 6]"); return -1; } + if (hour < -167 || hour > 167) { + PyErr_Format(PyExc_ValueError, "Hour must be in [0, 167]"); + return -1; + } + TransitionRuleType base = {&calendarrule_year_to_timestamp}; CalendarRule new_offset = { .base = base, - .month = month, - .week = week, - .day = day, - .hour = hour, - .minute = minute, - .second = second, + .month = (uint8_t)month, + .week = (uint8_t)week, + .day = (uint8_t)day, + .hour = (int16_t)hour, + .minute = (int8_t)minute, + .second = (int8_t)second, }; *out = new_offset; @@ -1409,40 +1402,45 @@ dayrule_year_to_timestamp(TransitionRuleType *base_self, int year) // always transitions on a given calendar day (other than February 29th), // you would use a Julian day, e.g. J91 always refers to April 1st and J365 // always refers to December 31st. - unsigned int day = self->day; + uint16_t day = self->day; if (self->julian && day >= 59 && is_leap_year(year)) { day += 1; } - return ((days_before_year + day) * 86400) + (self->hour * 3600) + - (self->minute * 60) + self->second; + return (days_before_year + day) * 86400 + (int64_t)self->hour * 3600 + + (int64_t)self->minute * 60 + self->second; } /* Constructor for DayRule. */ static int -dayrule_new(uint8_t julian, unsigned int day, int8_t hour, int8_t minute, - int8_t second, DayRule *out) +dayrule_new(int julian, int day, int hour, int minute, + int second, DayRule *out) { // The POSIX standard specifies that Julian days must be in the range (1 <= // n <= 365) and that non-Julian (they call it "0-based Julian") days must // be in the range (0 <= n <= 365). if (day < julian || day > 365) { - PyErr_Format(PyExc_ValueError, "day must be in [%u, 365], not: %u", + PyErr_Format(PyExc_ValueError, "day must be in [%d, 365], not: %d", julian, day); return -1; } + if (hour < -167 || hour > 167) { + PyErr_Format(PyExc_ValueError, "Hour must be in [0, 167]"); + return -1; + } + TransitionRuleType base = { &dayrule_year_to_timestamp, }; DayRule tmp = { .base = base, - .julian = julian, - .day = day, - .hour = hour, - .minute = minute, - .second = second, + .julian = (uint8_t)julian, + .day = (int16_t)day, + .hour = (int16_t)hour, + .minute = (int8_t)minute, + .second = (int8_t)second, }; *out = tmp; @@ -1599,21 +1597,18 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) const char *p = tz_str; // Read the `std` abbreviation, which must be at least 3 characters long. - Py_ssize_t num_chars = parse_abbr(p, &std_abbr); - if (num_chars < 1) { - PyErr_Format(PyExc_ValueError, "Invalid STD format in %R", tz_str_obj); + if (parse_abbr(&p, &std_abbr)) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "Invalid STD format in %R", tz_str_obj); + } goto error; } - p += num_chars; - // Now read the STD offset, which is required - num_chars = parse_tz_delta(p, &std_offset); - if (num_chars < 0) { + if (parse_tz_delta(&p, &std_offset)) { PyErr_Format(PyExc_ValueError, "Invalid STD offset in %R", tz_str_obj); goto error; } - p += num_chars; // If the string ends here, there is no DST, otherwise we must parse the // DST abbreviation and start and end dates and times. @@ -1621,12 +1616,12 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) goto complete; } - num_chars = parse_abbr(p, &dst_abbr); - if (num_chars < 1) { - PyErr_Format(PyExc_ValueError, "Invalid DST format in %R", tz_str_obj); + if (parse_abbr(&p, &dst_abbr)) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "Invalid DST format in %R", tz_str_obj); + } goto error; } - p += num_chars; if (*p == ',') { // From the POSIX standard: @@ -1636,14 +1631,11 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) dst_offset = std_offset + 3600; } else { - num_chars = parse_tz_delta(p, &dst_offset); - if (num_chars < 0) { + if (parse_tz_delta(&p, &dst_offset)) { PyErr_Format(PyExc_ValueError, "Invalid DST offset in %R", tz_str_obj); goto error; } - - p += num_chars; } TransitionRuleType **transitions[2] = {&start, &end}; @@ -1656,14 +1648,12 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) } p++; - num_chars = parse_transition_rule(p, transitions[i]); - if (num_chars < 0) { + if (parse_transition_rule(&p, transitions[i])) { PyErr_Format(PyExc_ValueError, "Malformed transition rule in TZ string: %R", tz_str_obj); goto error; } - p += num_chars; } if (*p != '\0') { @@ -1698,21 +1688,25 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) } static int -parse_uint(const char *const p, uint8_t *value) +parse_digits(const char **p, int min, int max, int *value) { - if (!Py_ISDIGIT(*p)) { - return -1; + assert(max <= 3); + *value = 0; + for (int i = 0; i < max; i++, (*p)++) { + if (!Py_ISDIGIT(**p)) { + return (i < min) ? -1 : 0; + } + *value *= 10; + *value += (**p) - '0'; } - - *value = (*p) - '0'; return 0; } /* Parse the STD and DST abbreviations from a TZ string. */ -static Py_ssize_t -parse_abbr(const char *const p, PyObject **abbr) +static int +parse_abbr(const char **p, PyObject **abbr) { - const char *ptr = p; + const char *ptr = *p; const char *str_start; const char *str_end; @@ -1741,7 +1735,7 @@ parse_abbr(const char *const p, PyObject **abbr) ptr++; } else { - str_start = p; + str_start = ptr; // From the POSIX standard: // // In the unquoted form, all characters in these fields shall be @@ -1751,6 +1745,9 @@ parse_abbr(const char *const p, PyObject **abbr) ptr++; } str_end = ptr; + if (str_end == str_start) { + return -1; + } } *abbr = PyUnicode_FromStringAndSize(str_start, str_end - str_start); @@ -1758,12 +1755,13 @@ parse_abbr(const char *const p, PyObject **abbr) return -1; } - return ptr - p; + *p = ptr; + return 0; } /* Parse a UTC offset from a TZ str. */ -static Py_ssize_t -parse_tz_delta(const char *const p, long *total_seconds) +static int +parse_tz_delta(const char **p, long *total_seconds) { // From the POSIX spec: // @@ -1778,75 +1776,30 @@ parse_tz_delta(const char *const p, long *total_seconds) // The POSIX spec says that the values for `hour` must be between 0 and 24 // hours, but RFC 8536 §3.3.1 specifies that the hours part of the // transition times may be signed and range from -167 to 167. - long sign = -1; - long hours = 0; - long minutes = 0; - long seconds = 0; - - const char *ptr = p; - char buff = *ptr; - if (buff == '-' || buff == '+') { - // Negative numbers correspond to *positive* offsets, from the spec: - // - // If preceded by a '-', the timezone shall be east of the Prime - // Meridian; otherwise, it shall be west (which may be indicated by - // an optional preceding '+' ). - if (buff == '-') { - sign = 1; - } - - ptr++; - } + int hours = 0; + int minutes = 0; + int seconds = 0; - // The hour can be 1 or 2 numeric characters - for (size_t i = 0; i < 2; ++i) { - buff = *ptr; - if (!Py_ISDIGIT(buff)) { - if (i == 0) { - return -1; - } - else { - break; - } - } - - hours *= 10; - hours += buff - '0'; - ptr++; - } - - if (hours > 24 || hours < 0) { + if (parse_transition_time(p, &hours, &minutes, &seconds)) { return -1; } - // Minutes and seconds always of the format ":dd" - long *outputs[2] = {&minutes, &seconds}; - for (size_t i = 0; i < 2; ++i) { - if (*ptr != ':') { - goto complete; - } - ptr++; - - for (size_t j = 0; j < 2; ++j) { - buff = *ptr; - if (!Py_ISDIGIT(buff)) { - return -1; - } - *(outputs[i]) *= 10; - *(outputs[i]) += buff - '0'; - ptr++; - } + if (hours > 24 || hours < -24) { + return -1; } -complete: - *total_seconds = sign * ((hours * 3600) + (minutes * 60) + seconds); - - return ptr - p; + // Negative numbers correspond to *positive* offsets, from the spec: + // + // If preceded by a '-', the timezone shall be east of the Prime + // Meridian; otherwise, it shall be west (which may be indicated by + // an optional preceding '+' ). + *total_seconds = -((hours * 3600L) + (minutes * 60) + seconds); + return 0; } /* Parse the date portion of a transition rule. */ -static Py_ssize_t -parse_transition_rule(const char *const p, TransitionRuleType **out) +static int +parse_transition_rule(const char **p, TransitionRuleType **out) { // The full transition rule indicates when to change back and forth between // STD and DST, and has the form: @@ -1858,10 +1811,10 @@ parse_transition_rule(const char *const p, TransitionRuleType **out) // does not include the ',' at the end of the first rule. // // The POSIX spec states that if *time* is not given, the default is 02:00. - const char *ptr = p; - int8_t hour = 2; - int8_t minute = 0; - int8_t second = 0; + const char *ptr = *p; + int hour = 2; + int minute = 0; + int second = 0; // Rules come in one of three flavors: // @@ -1870,44 +1823,30 @@ parse_transition_rule(const char *const p, TransitionRuleType **out) // 3. Mm.n.d: Specifying by month, week and day-of-week. if (*ptr == 'M') { - uint8_t month, week, day; + int month, week, day; ptr++; - if (parse_uint(ptr, &month)) { + + if (parse_digits(&ptr, 1, 2, &month)) { return -1; } - ptr++; - if (*ptr != '.') { - uint8_t tmp; - if (parse_uint(ptr, &tmp)) { - return -1; - } - - month *= 10; - month += tmp; - ptr++; + if (*ptr++ != '.') { + return -1; } - - uint8_t *values[2] = {&week, &day}; - for (size_t i = 0; i < 2; ++i) { - if (*ptr != '.') { - return -1; - } - ptr++; - - if (parse_uint(ptr, values[i])) { - return -1; - } - ptr++; + if (parse_digits(&ptr, 1, 1, &week)) { + return -1; + } + if (*ptr++ != '.') { + return -1; + } + if (parse_digits(&ptr, 1, 1, &day)) { + return -1; } if (*ptr == '/') { ptr++; - Py_ssize_t num_chars = - parse_transition_time(ptr, &hour, &minute, &second); - if (num_chars < 0) { + if (parse_transition_time(&ptr, &hour, &minute, &second)) { return -1; } - ptr += num_chars; } CalendarRule *rv = PyMem_Calloc(1, sizeof(CalendarRule)); @@ -1923,33 +1862,22 @@ parse_transition_rule(const char *const p, TransitionRuleType **out) *out = (TransitionRuleType *)rv; } else { - uint8_t julian = 0; - unsigned int day = 0; + int julian = 0; + int day = 0; if (*ptr == 'J') { julian = 1; ptr++; } - for (size_t i = 0; i < 3; ++i) { - if (!Py_ISDIGIT(*ptr)) { - if (i == 0) { - return -1; - } - break; - } - day *= 10; - day += (*ptr) - '0'; - ptr++; + if (parse_digits(&ptr, 1, 3, &day)) { + return -1; } if (*ptr == '/') { ptr++; - Py_ssize_t num_chars = - parse_transition_time(ptr, &hour, &minute, &second); - if (num_chars < 0) { + if (parse_transition_time(&ptr, &hour, &minute, &second)) { return -1; } - ptr += num_chars; } DayRule *rv = PyMem_Calloc(1, sizeof(DayRule)); @@ -1964,13 +1892,13 @@ parse_transition_rule(const char *const p, TransitionRuleType **out) *out = (TransitionRuleType *)rv; } - return ptr - p; + *p = ptr; + return 0; } /* Parse the time portion of a transition rule (e.g. following an /) */ -static Py_ssize_t -parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, - int8_t *second) +static int +parse_transition_time(const char **p, int *hour, int *minute, int *second) { // From the spec: // @@ -1982,12 +1910,9 @@ parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, // h[h][:mm[:ss]] // // RFC 8536 also allows transition times to be signed and to range from - // -167 to +167, but the current version only supports [0, 99]. - // - // TODO: Support the full range of transition hours. - int8_t *components[3] = {hour, minute, second}; - const char *ptr = p; - int8_t sign = 1; + // -167 to +167. + const char *ptr = *p; + int sign = 1; if (*ptr == '-' || *ptr == '+') { if (*ptr == '-') { @@ -1996,32 +1921,31 @@ parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, ptr++; } - for (size_t i = 0; i < 3; ++i) { - if (i > 0) { - if (*ptr != ':') { - break; - } - ptr++; + // The hour can be 1 to 3 numeric characters + if (parse_digits(&ptr, 1, 3, hour)) { + return -1; + } + *hour *= sign; + + // Minutes and seconds always of the format ":dd" + if (*ptr == ':') { + ptr++; + if (parse_digits(&ptr, 2, 2, minute)) { + return -1; } + *minute *= sign; - uint8_t buff = 0; - for (size_t j = 0; j < 2; j++) { - if (!Py_ISDIGIT(*ptr)) { - if (i == 0 && j > 0) { - break; - } + if (*ptr == ':') { + ptr++; + if (parse_digits(&ptr, 2, 2, second)) { return -1; } - - buff *= 10; - buff += (*ptr) - '0'; - ptr++; + *second *= sign; } - - *(components[i]) = sign * buff; } - return ptr - p; + *p = ptr; + return 0; } /* Constructor for a _tzrule. @@ -2376,8 +2300,8 @@ get_local_timestamp(PyObject *dt, int64_t *local_ts) } } - *local_ts = (int64_t)(ord - EPOCHORDINAL) * 86400 + - (int64_t)(hour * 3600 + minute * 60 + second); + *local_ts = (int64_t)(ord - EPOCHORDINAL) * 86400L + + (int64_t)(hour * 3600L + minute * 60 + second); return 0; } @@ -2836,6 +2760,7 @@ zoneinfomodule_exec(PyObject *m) static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 309a36919f3465..e6c84d588be98b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,7 +11,7 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_FromByteArray() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include // offsetof() @@ -247,7 +247,7 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((char *)ap->ob_item)[i] = x; + ((unsigned char *)ap->ob_item)[i] = x; return 0; } @@ -260,20 +260,32 @@ u_getitem(arrayobject *ap, Py_ssize_t i) static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - Py_ssize_t len = PyUnicode_AsWideChar(u, NULL, 0); + Py_ssize_t len = PyUnicode_AsWideChar(v, NULL, 0); if (len != 2) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + } + else { + PyErr_Format(PyExc_TypeError, + "string %A cannot be converted to " + "a single wchar_t character", + v); + } return -1; } wchar_t w; - len = PyUnicode_AsWideChar(u, &w, 1); + len = PyUnicode_AsWideChar(v, &w, 1); assert(len == 1); if (i >= 0) { @@ -291,19 +303,23 @@ w_getitem(arrayobject *ap, Py_ssize_t i) static int w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - if (PyUnicode_GetLength(u) != 1) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); return -1; } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(u, 0); + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } @@ -868,6 +884,21 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return (PyObject *)np; } +/*[clinic input] +array.array.clear + +Remove all items from the array. +[clinic start generated code]*/ + +static PyObject * +array_array_clear_impl(arrayobject *self) +/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +{ + if (array_resize(self, 0) == -1) { + return NULL; + } + Py_RETURN_NONE; +} /*[clinic input] array.array.__copy__ @@ -2342,6 +2373,7 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_APPEND_METHODDEF ARRAY_ARRAY_BUFFER_INFO_METHODDEF ARRAY_ARRAY_BYTESWAP_METHODDEF + ARRAY_ARRAY_CLEAR_METHODDEF ARRAY_ARRAY___COPY___METHODDEF ARRAY_ARRAY_COUNT_METHODDEF ARRAY_ARRAY___DEEPCOPY___METHODDEF @@ -3204,6 +3236,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 57e2ea67450453..297a8d74ba3bf4 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -10,7 +10,6 @@ #include "pycore_atexit.h" // export _Py_AtExit() #include "pycore_initconfig.h" // _PyStatus_NO_MEMORY #include "pycore_interp.h" // PyInterpreterState.atexit -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _PyInterpreterState_GET /* ===================================================================== */ @@ -137,7 +136,8 @@ atexit_callfuncs(struct atexit_state *state) PyObject* the_func = Py_NewRef(cb->func); PyObject *res = PyObject_Call(cb->func, cb->args, cb->kwargs); if (res == NULL) { - _PyErr_WriteUnraisableMsg("in atexit callback", the_func); + PyErr_FormatUnraisable( + "Exception ignored in atexit callback %R", the_func); } else { Py_DECREF(res); @@ -164,7 +164,8 @@ _PyAtExit_Call(PyInterpreterState *interp) PyDoc_STRVAR(atexit_register__doc__, -"register(func, *args, **kwargs) -> func\n\ +"register($module, func, /, *args, **kwargs)\n\ +--\n\ \n\ Register a function to be executed upon normal program termination\n\ \n\ @@ -221,7 +222,8 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(atexit_run_exitfuncs__doc__, -"_run_exitfuncs() -> None\n\ +"_run_exitfuncs($module, /)\n\ +--\n\ \n\ Run all registered exit functions.\n\ \n\ @@ -236,7 +238,8 @@ atexit_run_exitfuncs(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_clear__doc__, -"_clear() -> None\n\ +"_clear($module, /)\n\ +--\n\ \n\ Clear the list of previously registered exit functions."); @@ -248,7 +251,8 @@ atexit_clear(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_ncallbacks__doc__, -"_ncallbacks() -> int\n\ +"_ncallbacks($module, /)\n\ +--\n\ \n\ Return the number of registered exit functions."); @@ -260,7 +264,8 @@ atexit_ncallbacks(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_unregister__doc__, -"unregister(func) -> None\n\ +"unregister($module, func, /)\n\ +--\n\ \n\ Unregister an exit function which was previously registered using\n\ atexit.register\n\ @@ -317,6 +322,7 @@ Two public functions, register and unregister, are defined.\n\ static PyModuleDef_Slot atexitmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index a87a2ef2e89927..6bb01d148b6faa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -185,13 +185,7 @@ ascii_buffer_converter(PyObject *arg, Py_buffer *buf) "not '%.100s'", Py_TYPE(arg)->tp_name); return 0; } - if (!PyBuffer_IsContiguous(buf, 'C')) { - PyErr_Format(PyExc_TypeError, - "argument should be a contiguous buffer, " - "not '%.100s'", Py_TYPE(arg)->tp_name); - PyBuffer_Release(buf); - return 0; - } + assert(PyBuffer_IsContiguous(buf, 'C')); return Py_CLEANUP_SUPPORTED; } @@ -420,6 +414,13 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) if (this_ch == BASE64_PAD) { padding_started = 1; + if (strict_mode && quad_pos == 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Excess padding not allowed"); + } + goto error_end; + } if (quad_pos >= 2 && quad_pos + ++pads >= 4) { /* A pad sequence means we should not parse more input. ** We've already interpreted the data from the quad at this point. @@ -776,12 +777,20 @@ binascii_crc32_impl(PyObject *module, Py_buffer *data, unsigned int crc) Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. crc32() takes - length as an unsigned int, which may be narrower than Py_ssize_t. */ - while ((size_t)len > UINT_MAX) { - crc = crc32(crc, buf, UINT_MAX); - buf += (size_t) UINT_MAX; - len -= (size_t) UINT_MAX; + length as an unsigned int, which may be narrower than Py_ssize_t. + We further limit size due to bugs in Apple's macOS zlib. + See https://github.com/python/cpython/issues/105967 + */ +#define ZLIB_CRC_CHUNK_SIZE 0x40000000 +#if ZLIB_CRC_CHUNK_SIZE > INT_MAX +# error "unsupported less than 32-bit platform?" +#endif + while ((size_t)len > ZLIB_CRC_CHUNK_SIZE) { + crc = crc32(crc, buf, ZLIB_CRC_CHUNK_SIZE); + buf += (size_t) ZLIB_CRC_CHUNK_SIZE; + len -= (size_t) ZLIB_CRC_CHUNK_SIZE; } +#undef ZLIB_CRC_CHUNK_SIZE crc = crc32(crc, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { @@ -1276,6 +1285,7 @@ binascii_exec(PyObject *module) static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 86bb73b982a551..ef6faeb71274e1 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -207,8 +207,9 @@ ENCODER(iso2022) encoded = MAP_UNMAPPABLE; for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) { + Py_UCS4 buf[2] = {c, 0}; Py_ssize_t length = 1; - encoded = dsg->encoder(codec, &c, &length); + encoded = dsg->encoder(codec, buf, &length); if (encoded == MAP_MULTIPLE_AVAIL) { /* this implementation won't work for pair * of non-bmp characters. */ @@ -217,9 +218,11 @@ ENCODER(iso2022) return MBERR_TOOFEW; length = -1; } - else + else { + buf[1] = INCHAR2; length = 2; - encoded = dsg->encoder(codec, &c, &length); + } + encoded = dsg->encoder(codec, buf, &length); if (encoded != MAP_UNMAPPABLE) { insize = length; break; @@ -803,7 +806,7 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (coded != DBCINV) return coded; - /* fall through */ + _Py_FALLTHROUGH; case -1: /* flush unterminated */ *length = 1; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 766f82983025e4..2b446ba5226ac0 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -503,6 +503,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 0b73a7059eb584..73edd5c3b25553 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -6,12 +6,13 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" "--\n" "\n" -"Return an encoded string version of `input\'.\n" +"Return an encoded string version of \'input\'.\n" "\n" "\'errors\' may be given to set a different error handling scheme. Default is\n" "\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" @@ -153,10 +154,6 @@ _multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *cons if (PyObject_GetBuffer(args[0], &input, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&input, 'C')) { - _PyArg_BadArgument("decode", "argument 'input'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -373,10 +370,6 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb if (PyObject_GetBuffer(args[0], &input, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&input, 'C')) { - _PyArg_BadArgument("decode", "argument 'input'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -675,7 +668,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se static PyObject * _multibytecodec_MultibyteStreamWriter_reset(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); return NULL; } @@ -689,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=1ee928e7a85e9d34 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f09052c5a28cc6e6 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7c1da9004549a6..373518673dd352 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -9,7 +9,6 @@ #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_FromByteArray() #include "multibytecodec.h" #include "clinic/multibytecodec.c.h" @@ -575,7 +574,7 @@ _multibytecodec.MultibyteCodec.encode input: object errors: str(accept={str, NoneType}) = None -Return an encoded string version of `input'. +Return an encoded string version of 'input'. 'errors' may be given to set a different error handling scheme. Default is 'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible @@ -587,7 +586,7 @@ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -826,8 +825,15 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, if (inpos < datalen) { if (datalen - inpos > MAXENCPENDING) { /* normal codecs can't reach here */ - PyErr_SetString(PyExc_UnicodeError, - "pending buffer overflow"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + ctx->codec->encoding, + inbuf, + inpos, datalen, + "pending buffer overflow"); + if (excobj == NULL) goto errorexit; + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); goto errorexit; } ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen); @@ -858,7 +864,16 @@ decoder_append_pending(MultibyteStatefulDecoderContext *ctx, npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); if (npendings + ctx->pendingsize > MAXDECPENDING || npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); + Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); + PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding, + (const char *)buf->inbuf_top, + bufsize, + 0, + bufsize, + "pending buffer overflow"); + if (excobj == NULL) return -1; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return -1; } memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); @@ -939,7 +954,17 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn return NULL; } if (pendingsize > MAXENCPENDING*4) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + self->codec->encoding, + self->pending, + 0, PyUnicode_GET_LENGTH(self->pending), + "pending buffer too large"); + if (excobj == NULL) { + return NULL; + } + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); return NULL; } statebytes[0] = (unsigned char)pendingsize; @@ -974,7 +999,8 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { goto errorexit; } @@ -1256,7 +1282,8 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), 1 /* little-endian */ , - 0 /* unsigned */ ) < 0) { + 0 /* unsigned */ , + 1 /* with_exceptions */) < 0) { return NULL; } @@ -1266,7 +1293,13 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe } if (buffersize > MAXDECPENDING) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding, + PyBytes_AS_STRING(buffer), buffersize, + 0, buffersize, + "pending buffer too large"); + if (excobj == NULL) return NULL; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return NULL; } @@ -2061,6 +2094,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 8d3832e1b83d2d..04681fa2206a2a 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_abc__reset_registry__doc__, "_reset_registry($module, self, /)\n" "--\n" @@ -159,4 +161,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=babb3ce445fa9b21 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1989b6716c950e17 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 74dd06461e6e27..d619a124ccead5 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_asyncio_Future___init____doc__, "Future(*, loop=None)\n" @@ -119,7 +120,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); return NULL; } @@ -452,7 +453,7 @@ _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * _asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); return NULL; } @@ -1486,4 +1487,64 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=1b7658bfab7024f3 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_all_tasks__doc__, +"all_tasks($module, /, loop=None)\n" +"--\n" +"\n" +"Return a set of all tasks for the loop."); + +#define _ASYNCIO_ALL_TASKS_METHODDEF \ + {"all_tasks", _PyCFunction_CAST(_asyncio_all_tasks), METH_FASTCALL|METH_KEYWORDS, _asyncio_all_tasks__doc__}, + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop); + +static PyObject * +_asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "all_tasks", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *loop = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + loop = args[0]; +skip_optional_pos: + return_value = _asyncio_all_tasks_impl(module, loop); + +exit: + return return_value; +} +/*[clinic end generated code: output=ffe9b71bc65888b3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 5553b05acbf521..528602c84a9b23 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_bisect_bisect_right__doc__, "bisect_right($module, /, a, x, lo=0, hi=None, *, key=None)\n" @@ -43,7 +44,7 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -151,7 +152,7 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -256,7 +257,7 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -364,7 +365,7 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -433,4 +434,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=43ece163c3e972df input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a8d5a32dd0a3f04 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 29f5d3a68e06ae..de7b3993596446 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_BadArgument() PyDoc_STRVAR(_bz2_BZ2Compressor_compress__doc__, "compress($self, data, /)\n" @@ -34,10 +35,6 @@ _bz2_BZ2Compressor_compress(BZ2Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _bz2_BZ2Compressor_compress_impl(self, &data); exit: @@ -180,10 +177,6 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -241,4 +234,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=3dfc8436fa8eaefb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8daa62f47cc4853d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index bf9bd4a61f5ce4..1c0f37442ab350 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_BadArgument() PyDoc_STRVAR(_codecs_register__doc__, "register($module, search_function, /)\n" @@ -296,16 +297,14 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -421,10 +420,6 @@ _codecs_utf_7_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_7_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -491,10 +486,6 @@ _codecs_utf_8_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_8_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -561,10 +552,6 @@ _codecs_utf_16_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -631,10 +618,6 @@ _codecs_utf_16_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_le_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -701,10 +684,6 @@ _codecs_utf_16_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_be_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -773,10 +752,6 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_ex_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -850,10 +825,6 @@ _codecs_utf_32_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -920,10 +891,6 @@ _codecs_utf_32_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_le_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -990,10 +957,6 @@ _codecs_utf_32_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_be_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1062,10 +1025,6 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_ex_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1142,16 +1101,14 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("unicode_escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -1222,16 +1179,14 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("raw_unicode_escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -1298,10 +1253,6 @@ _codecs_latin_1_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("latin_1_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1360,10 +1311,6 @@ _codecs_ascii_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("ascii_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1423,10 +1370,6 @@ _codecs_charmap_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("charmap_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1492,10 +1435,6 @@ _codecs_mbcs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("mbcs_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1566,10 +1505,6 @@ _codecs_oem_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("oem_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1645,10 +1580,6 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[1], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("code_page_decode", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -1719,16 +1650,14 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("readbuffer_encode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -2817,4 +2746,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=3473564544f10403 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index a375b87261ce2d..fa0ff2133a05d2 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -2,7 +2,565 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(deque_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return the rightmost element."); + +#define DEQUE_POP_METHODDEF \ + {"pop", (PyCFunction)deque_pop, METH_NOARGS, deque_pop__doc__}, + +static PyObject * +deque_pop_impl(dequeobject *deque); + +static PyObject * +deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_pop_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_popleft__doc__, +"popleft($self, /)\n" +"--\n" +"\n" +"Remove and return the leftmost element."); + +#define DEQUE_POPLEFT_METHODDEF \ + {"popleft", (PyCFunction)deque_popleft, METH_NOARGS, deque_popleft__doc__}, + +static PyObject * +deque_popleft_impl(dequeobject *deque); + +static PyObject * +deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_popleft_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_append__doc__, +"append($self, item, /)\n" +"--\n" +"\n" +"Add an element to the right side of the deque."); + +#define DEQUE_APPEND_METHODDEF \ + {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__}, + +static PyObject * +deque_append_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_append(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_append_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_appendleft__doc__, +"appendleft($self, item, /)\n" +"--\n" +"\n" +"Add an element to the left side of the deque."); + +#define DEQUE_APPENDLEFT_METHODDEF \ + {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__}, + +static PyObject * +deque_appendleft_impl(dequeobject *deque, PyObject *item); + +static PyObject * +deque_appendleft(dequeobject *deque, PyObject *item) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_appendleft_impl(deque, item); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_extend__doc__, +"extend($self, iterable, /)\n" +"--\n" +"\n" +"Extend the right side of the deque with elements from the iterable."); + +#define DEQUE_EXTEND_METHODDEF \ + {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__}, + +static PyObject * +deque_extend_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extend(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extend_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_extendleft__doc__, +"extendleft($self, iterable, /)\n" +"--\n" +"\n" +"Extend the left side of the deque with elements from the iterable."); + +#define DEQUE_EXTENDLEFT_METHODDEF \ + {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__}, + +static PyObject * +deque_extendleft_impl(dequeobject *deque, PyObject *iterable); + +static PyObject * +deque_extendleft(dequeobject *deque, PyObject *iterable) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_extendleft_impl(deque, iterable); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE_COPY_METHODDEF \ + {"copy", (PyCFunction)deque_copy, METH_NOARGS, deque_copy__doc__}, + +static PyObject * +deque_copy_impl(dequeobject *deque); + +static PyObject * +deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_copy_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a deque."); + +#define DEQUE___COPY___METHODDEF \ + {"__copy__", (PyCFunction)deque___copy__, METH_NOARGS, deque___copy____doc__}, + +static PyObject * +deque___copy___impl(dequeobject *deque); + +static PyObject * +deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___copy___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_clearmethod__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from the deque."); + +#define DEQUE_CLEARMETHOD_METHODDEF \ + {"clear", (PyCFunction)deque_clearmethod, METH_NOARGS, deque_clearmethod__doc__}, + +static PyObject * +deque_clearmethod_impl(dequeobject *deque); + +static PyObject * +deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_clearmethod_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_rotate__doc__, +"rotate($self, n=1, /)\n" +"--\n" +"\n" +"Rotate the deque n steps to the right. If n is negative, rotates left."); + +#define DEQUE_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(deque_rotate), METH_FASTCALL, deque_rotate__doc__}, + +static PyObject * +deque_rotate_impl(dequeobject *deque, Py_ssize_t n); + +static PyObject * +deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t n = 1; + + if (!_PyArg_CheckPositional("rotate", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + n = ival; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_rotate_impl(deque, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_reverse__doc__, +"reverse($self, /)\n" +"--\n" +"\n" +"Reverse *IN PLACE*."); + +#define DEQUE_REVERSE_METHODDEF \ + {"reverse", (PyCFunction)deque_reverse, METH_NOARGS, deque_reverse__doc__}, + +static PyObject * +deque_reverse_impl(dequeobject *deque); + +static PyObject * +deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_reverse_impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Return number of occurrences of value."); + +#define DEQUE_COUNT_METHODDEF \ + {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__}, + +static PyObject * +deque_count_impl(dequeobject *deque, PyObject *v); + +static PyObject * +deque_count(dequeobject *deque, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_count_impl(deque, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque_index__doc__, +"index($self, value, [start, [stop]])\n" +"--\n" +"\n" +"Return first index of value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define DEQUE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(deque_index), METH_FASTCALL, deque_index__doc__}, + +static PyObject * +deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, + Py_ssize_t stop); + +static PyObject * +deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *v; + Py_ssize_t start = 0; + Py_ssize_t stop = Py_SIZE(deque); + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + v = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_index_impl(deque, v, start, stop); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_insert__doc__, +"insert($self, index, value, /)\n" +"--\n" +"\n" +"Insert value before index."); + +#define DEQUE_INSERT_METHODDEF \ + {"insert", _PyCFunction_CAST(deque_insert), METH_FASTCALL, deque_insert__doc__}, + +static PyObject * +deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value); + +static PyObject * +deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t index; + PyObject *value; + + if (!_PyArg_CheckPositional("insert", nargs, 2, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + value = args[1]; + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_insert_impl(deque, index, value); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque_remove__doc__, +"remove($self, value, /)\n" +"--\n" +"\n" +"Remove first occurrence of value."); + +#define DEQUE_REMOVE_METHODDEF \ + {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__}, + +static PyObject * +deque_remove_impl(dequeobject *deque, PyObject *value); + +static PyObject * +deque_remove(dequeobject *deque, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_remove_impl(deque, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define DEQUE___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)deque___reduce__, METH_NOARGS, deque___reduce____doc__}, + +static PyObject * +deque___reduce___impl(dequeobject *deque); + +static PyObject * +deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reduce___impl(deque); +} + +PyDoc_STRVAR(deque_init__doc__, +"deque([iterable[, maxlen]])\n" +"--\n" +"\n" +"A list-like sequence optimized for data accesses near its endpoints."); + +static int +deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj); + +static int +deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(iterable), &_Py_ID(maxlen), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"iterable", "maxlen", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "deque", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *iterable = NULL; + PyObject *maxlenobj = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + iterable = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + maxlenobj = fastargs[1]; +skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(deque___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the deque in memory, in bytes."); + +#define DEQUE___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)deque___sizeof__, METH_NOARGS, deque___sizeof____doc__}, + +static PyObject * +deque___sizeof___impl(dequeobject *deque); + +static PyObject * +deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(deque); + return_value = deque___sizeof___impl(deque); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(deque___reversed____doc__, +"__reversed__($self, /)\n" +"--\n" +"\n" +"Return a reverse iterator over the deque."); + +#define DEQUE___REVERSED___METHODDEF \ + {"__reversed__", (PyCFunction)deque___reversed__, METH_NOARGS, deque___reversed____doc__}, + +static PyObject * +deque___reversed___impl(dequeobject *deque); + +static PyObject * +deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +{ + return deque___reversed___impl(deque); +} PyDoc_STRVAR(_collections__count_elements__doc__, "_count_elements($module, mapping, iterable, /)\n" @@ -71,4 +629,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b01ddb9fdecc4a2d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=64c32b16df7be07a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 86ed0efd79028e..2442bdcde5679f 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_csv_list_dialects__doc__, "list_dialects($module, /)\n" @@ -205,4 +206,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=4704d708f5745e91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ec59717f5414d8b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index a8c32c6aa3fc10..c8788c461f745c 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,6 +2,11 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "bottom($self, /)\n" "--\n" @@ -17,7 +22,7 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_bottom(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); return NULL; } @@ -41,7 +46,7 @@ _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_hide(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); return NULL; } @@ -63,7 +68,7 @@ _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_show(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); return NULL; } @@ -85,7 +90,7 @@ _curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * _curses_panel_panel_top(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); return NULL; } @@ -325,7 +330,7 @@ _curses_panel_panel_userptr_impl(PyCursesPanelObject *self, static PyObject * _curses_panel_panel_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); return NULL; } @@ -416,4 +421,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=dd302cb9afc42f40 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18dc5571174c7189 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index ecc3c059d03c90..f7e0aaf7b23649 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_curses_window_addch__doc__, "addch([y, x,] ch, [attr=_curses.A_NORMAL])\n" @@ -3162,7 +3163,7 @@ PyDoc_STRVAR(_curses_napms__doc__, #define _CURSES_NAPMS_METHODDEF \ {"napms", (PyCFunction)_curses_napms, METH_O, _curses_napms__doc__}, -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms); static PyObject * @@ -3170,12 +3171,17 @@ _curses_napms(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int ms; + int _return_value; ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_napms_impl(module, ms); + _return_value = _curses_napms_impl(module, ms); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -4312,4 +4318,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=839faafb638935ea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 177a37e486e24d..48499e0aaf7783 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "fromtimestamp($type, timestamp, /)\n" @@ -81,6 +82,207 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl(self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_now__doc__, "now($type, /, tz=None)\n" "--\n" @@ -145,4 +347,153 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=e422c25b1c28f38b input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n" +" hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return datetime with new specified fields."); + +#define DATETIME_DATETIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_replace__doc__}, + +static PyObject * +datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, + int month, int day, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + int hour = DATE_GET_HOUR(self); + int minute = DATE_GET_MINUTE(self); + int second = DATE_GET_SECOND(self); + int microsecond = DATE_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + int fold = DATE_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + hour = PyLong_AsInt(args[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + minute = PyLong_AsInt(args[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[5]) { + second = PyLong_AsInt(args[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[6]) { + microsecond = PyLong_AsInt(args[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[7]) { + tzinfo = args[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} +/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index a76b8b30eded39..4379b433db3738 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -2,6 +2,11 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + PyDoc_STRVAR(_dbm_dbm_close__doc__, "close($self, /)\n" "--\n" @@ -35,7 +40,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -147,7 +152,7 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -216,4 +221,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=972d221f9da819d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f7d9a87d80a64278 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 3477257f9d2cf2..1a5a820d1f00b5 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -4,9 +4,10 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_elementtree_Element_append__doc__, "append($self, subelement, /)\n" @@ -86,7 +87,7 @@ _elementtree_Element___copy___impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element___copy__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -643,7 +644,7 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls); static PyObject * _elementtree_Element_itertext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "itertext() takes no arguments"); return NULL; } @@ -1168,6 +1169,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored) return _elementtree_XMLParser_close_impl(self); } +PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__}, + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self); + +static PyObject * +_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_XMLParser_flush_impl(self); +} + PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, "feed($self, data, /)\n" "--\n" @@ -1218,4 +1236,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=20d1869da79a43d7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd28eba33d9c1f25 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 25a3e5b0da60a8..e98984dc4d3a09 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -6,6 +6,8 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_functools_cmp_to_key__doc__, "cmp_to_key($module, /, mycmp)\n" @@ -80,7 +82,13 @@ _functools__lru_cache_wrapper_cache_info_impl(PyObject *self); static PyObject * _functools__lru_cache_wrapper_cache_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _functools__lru_cache_wrapper_cache_info_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _functools__lru_cache_wrapper_cache_info_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_clear__doc__, @@ -98,6 +106,12 @@ _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self); static PyObject * _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _functools__lru_cache_wrapper_cache_clear_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _functools__lru_cache_wrapper_cache_clear_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=de9cf85e85167f43 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=755265bb6d5ea751 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index e9138bbf0430c5..bbf4365114c0aa 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -2,6 +2,11 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_gdbm_gdbm_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -104,7 +109,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } @@ -130,7 +135,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); return NULL; } @@ -209,7 +214,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); return NULL; } @@ -234,7 +239,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); return NULL; } @@ -256,7 +261,7 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } @@ -338,4 +343,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=84f30c7fff0eadac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07bdeb4a8ecb328e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index e41f608bde5011..1d85ab1c524082 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(EVP_copy__doc__, "copy($self, /)\n" @@ -1289,17 +1290,9 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[1], &password, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&password, 'C')) { - _PyArg_BadArgument("pbkdf2_hmac", "argument 'password'", "contiguous buffer", args[1]); - goto exit; - } if (PyObject_GetBuffer(args[2], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("pbkdf2_hmac", "argument 'salt'", "contiguous buffer", args[2]); - goto exit; - } iterations = PyLong_AsLong(args[3]); if (iterations == -1 && PyErr_Occurred()) { goto exit; @@ -1354,7 +1347,7 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(password), &_Py_ID(salt), &_Py_ID(n), &_Py_ID(r), &_Py_ID(p), &_Py_ID(maxmem), &_Py_ID(dklen), }, + .ob_item = { &_Py_ID(password), &_Py_ID(salt), _Py_LATIN1_CHR('n'), _Py_LATIN1_CHR('r'), _Py_LATIN1_CHR('p'), &_Py_ID(maxmem), &_Py_ID(dklen), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1387,10 +1380,6 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &password, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&password, 'C')) { - _PyArg_BadArgument("scrypt", "argument 'password'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -1398,10 +1387,6 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[1], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("scrypt", "argument 'salt'", "contiguous buffer", args[1]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1521,17 +1506,9 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("hmac_digest", "argument 'key'", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &msg, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&msg, 'C')) { - _PyArg_BadArgument("hmac_digest", "argument 'msg'", "contiguous buffer", args[1]); - goto exit; - } digest = args[2]; return_value = _hashlib_hmac_singleshot_impl(module, &key, &msg, digest); @@ -1603,10 +1580,6 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (PyObject_GetBuffer(args[0], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("hmac_new", "argument 'key'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1851,4 +1824,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=75413752099f2dec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fef43fd9f4dbea49 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 8d73b5b48d6a0e..9046307990773b 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_heapq_heappush__doc__, "heappush($module, heap, item, /)\n" "--\n" @@ -265,4 +267,4 @@ _heapq__heapify_max(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=9a22715a8bf0c91d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05f2afdf3bc54c9d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_localemodule.c.h b/Modules/clinic/_localemodule.c.h index efbb9f709958cd..5e0880b0d0bb4c 100644 --- a/Modules/clinic/_localemodule.c.h +++ b/Modules/clinic/_localemodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_locale_setlocale__doc__, "setlocale($module, category, locale=, /)\n" "--\n" @@ -593,4 +595,4 @@ _locale_getencoding(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #define _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #endif /* !defined(_LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF) */ -/*[clinic end generated code: output=3abe7fade999eff6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=034a3c219466d207 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index dfc003eb54774c..b3b7fda5660bfd 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -39,10 +39,10 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); return NULL; } return _lsprof_Profiler_getstats_impl(self, cls); } -/*[clinic end generated code: output=0615a53cce828f06 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 528b48384f0bcb..51fab5eab3f7dc 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_BadArgument() PyDoc_STRVAR(_lzma_LZMACompressor_compress__doc__, "compress($self, data, /)\n" @@ -34,10 +35,6 @@ _lzma_LZMACompressor_compress(Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _lzma_LZMACompressor_compress_impl(self, &data); exit: @@ -138,10 +135,6 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -324,10 +317,6 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz if (PyObject_GetBuffer(args[1], &encoded_props, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&encoded_props, 'C')) { - _PyArg_BadArgument("_decode_filter_properties", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _lzma__decode_filter_properties_impl(module, filter_id, &encoded_props); exit: @@ -338,4 +327,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=eadc9ee7a11a06f5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e79c05ace76dc96 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 4b9a70d2ba9852..32ac9521a2b5cf 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_opcode_stack_effect__doc__, "stack_effect($module, opcode, oparg=None, /, *, jump=None)\n" @@ -667,4 +668,82 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=d608239a4c7a05a1 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_opcode_get_special_method_names__doc__, +"get_special_method_names($module, /)\n" +"--\n" +"\n" +"Return a list of special method names."); + +#define _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF \ + {"get_special_method_names", (PyCFunction)_opcode_get_special_method_names, METH_NOARGS, _opcode_get_special_method_names__doc__}, + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module); + +static PyObject * +_opcode_get_special_method_names(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _opcode_get_special_method_names_impl(module); +} + +PyDoc_STRVAR(_opcode_get_executor__doc__, +"get_executor($module, /, code, offset)\n" +"--\n" +"\n" +"Return the executor object at offset in code if exists, None otherwise."); + +#define _OPCODE_GET_EXECUTOR_METHODDEF \ + {"get_executor", _PyCFunction_CAST(_opcode_get_executor), METH_FASTCALL|METH_KEYWORDS, _opcode_get_executor__doc__}, + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset); + +static PyObject * +_opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(code), &_Py_ID(offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"code", "offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_executor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *code; + int offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + code = args[0]; + offset = PyLong_AsInt(args[1]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _opcode_get_executor_impl(module, code, offset); + +exit: + return return_value; +} +/*[clinic end generated code: output=3b4d4f32eedd636e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index 06bde243b4e85b..08615d690922a1 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_operator_truth__doc__, "truth($module, a, /)\n" @@ -1488,4 +1489,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=9658aca50a9ad991 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ddbba2cd943571eb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 5df40d62827754..693c7d59e9d7a6 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -4,8 +4,9 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_pickle_Pickler_clear_memo__doc__, "clear_memo($self, /)\n" @@ -265,6 +266,49 @@ _pickle_PicklerMemoProxy___reduce__(PicklerMemoProxyObject *self, PyObject *Py_U return _pickle_PicklerMemoProxy___reduce___impl(self); } +PyDoc_STRVAR(_pickle_Unpickler_persistent_load__doc__, +"persistent_load($self, pid, /)\n" +"--\n" +"\n"); + +#define _PICKLE_UNPICKLER_PERSISTENT_LOAD_METHODDEF \ + {"persistent_load", _PyCFunction_CAST(_pickle_Unpickler_persistent_load), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_persistent_load__doc__}, + +static PyObject * +_pickle_Unpickler_persistent_load_impl(UnpicklerObject *self, + PyTypeObject *cls, PyObject *pid); + +static PyObject * +_pickle_Unpickler_persistent_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "persistent_load", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *pid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + pid = args[0]; + return_value = _pickle_Unpickler_persistent_load_impl(self, cls, pid); + +exit: + return return_value; +} + PyDoc_STRVAR(_pickle_Unpickler_load__doc__, "load($self, /)\n" "--\n" @@ -284,7 +328,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * _pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); return NULL; } @@ -1033,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=57c209a12264146d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c7dd60d20ee4895f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index 83048a385d319c..dd7644de6b7534 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(subprocess_fork_exec__doc__, "fork_exec($module, args, executable_list, close_fds, pass_fds, cwd,\n" " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" @@ -153,4 +155,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=a83b11467169b97b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=48555f5965a871be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 8d4df14ae3b009..6f4c715c722965 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -6,6 +6,8 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() PyDoc_STRVAR(simplequeue_new__doc__, "SimpleQueue()\n" @@ -106,7 +108,9 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ } timeout = args[2]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_impl(self, item, block, timeout); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -164,7 +168,9 @@ _queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py goto exit; } item = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_put_nowait_impl(self, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -243,7 +249,9 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -268,11 +276,18 @@ _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, static PyObject * _queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + PyObject *return_value = NULL; + + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_nowait() takes no arguments"); - return NULL; + goto exit; } - return _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _queue_SimpleQueue_get_nowait_impl(self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, @@ -293,7 +308,9 @@ _queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; int _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_empty_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -321,7 +338,9 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) PyObject *return_value = NULL; Py_ssize_t _return_value; + Py_BEGIN_CRITICAL_SECTION(self); _return_value = _queue_SimpleQueue_qsize_impl(self); + Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -330,4 +349,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=5c326e4c1f2a1ad7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 008b531d832c5a..6193acac67e7ac 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_random_Random_random__doc__, "random($self, /)\n" "--\n" @@ -17,7 +20,13 @@ _random_Random_random_impl(RandomObject *self); static PyObject * _random_Random_random(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_random_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_random_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_seed__doc__, @@ -49,7 +58,9 @@ _random_Random_seed(RandomObject *self, PyObject *const *args, Py_ssize_t nargs) } n = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_seed_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -70,7 +81,13 @@ _random_Random_getstate_impl(RandomObject *self); static PyObject * _random_Random_getstate(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_getstate_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_getstate_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_setstate__doc__, @@ -82,6 +99,21 @@ PyDoc_STRVAR(_random_Random_setstate__doc__, #define _RANDOM_RANDOM_SETSTATE_METHODDEF \ {"setstate", (PyCFunction)_random_Random_setstate, METH_O, _random_Random_setstate__doc__}, +static PyObject * +_random_Random_setstate_impl(RandomObject *self, PyObject *state); + +static PyObject * +_random_Random_setstate(RandomObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_setstate_impl(self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_random_Random_getrandbits__doc__, "getrandbits($self, k, /)\n" "--\n" @@ -104,9 +136,11 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) if (k == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_getrandbits_impl(self, k); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=5e7e05d756a7e1c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf49ece1d341b1b6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 5f8ae29180a4fa..e8d1342ed35e66 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, "do_handshake($self, /)\n" @@ -236,10 +236,6 @@ _ssl__SSLSocket_write(PySSLSocket *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl__SSLSocket_write_impl(self, &b); exit: @@ -529,10 +525,6 @@ _ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&protos, 'C')) { - _PyArg_BadArgument("_set_alpn_protocols", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl__SSLContext__set_alpn_protocols_impl(self, &protos); exit: @@ -1021,6 +1013,141 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_set_psk_client_callback__doc__, +"set_psk_client_callback($self, /, callback)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_PSK_CLIENT_CALLBACK_METHODDEF \ + {"set_psk_client_callback", _PyCFunction_CAST(_ssl__SSLContext_set_psk_client_callback), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_set_psk_client_callback__doc__}, + +static PyObject * +_ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, + PyObject *callback); + +static PyObject * +_ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(callback), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"callback", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_psk_client_callback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *callback; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + callback = args[0]; + return_value = _ssl__SSLContext_set_psk_client_callback_impl(self, callback); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_psk_server_callback__doc__, +"set_psk_server_callback($self, /, callback, identity_hint=None)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_PSK_SERVER_CALLBACK_METHODDEF \ + {"set_psk_server_callback", _PyCFunction_CAST(_ssl__SSLContext_set_psk_server_callback), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_set_psk_server_callback__doc__}, + +static PyObject * +_ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, + PyObject *callback, + const char *identity_hint); + +static PyObject * +_ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(callback), &_Py_ID(identity_hint), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"callback", "identity_hint", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_psk_server_callback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *callback; + const char *identity_hint = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + callback = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1] == Py_None) { + identity_hint = NULL; + } + else if (PyUnicode_Check(args[1])) { + Py_ssize_t identity_hint_length; + identity_hint = PyUnicode_AsUTF8AndSize(args[1], &identity_hint_length); + if (identity_hint == NULL) { + goto exit; + } + if (strlen(identity_hint) != (size_t)identity_hint_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + } + else { + _PyArg_BadArgument("set_psk_server_callback", "argument 'identity_hint'", "str or None", args[1]); + goto exit; + } +skip_optional_pos: + return_value = _ssl__SSLContext_set_psk_server_callback_impl(self, callback, identity_hint); + +exit: + return return_value; +} + static PyObject * _ssl_MemoryBIO_impl(PyTypeObject *type); @@ -1107,10 +1234,6 @@ _ssl_MemoryBIO_write(PySSLMemoryBIO *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl_MemoryBIO_write_impl(self, &b); exit: @@ -1173,16 +1296,14 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (ptr == NULL) { goto exit; } - PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, 0); + if (PyBuffer_FillInfo(&view, args[0], (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) { + goto exit; + } } else { /* any bytes-like object */ if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&view, 'C')) { - _PyArg_BadArgument("RAND_add", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (PyFloat_CheckExact(args[1])) { entropy = PyFloat_AS_DOUBLE(args[1]); @@ -1542,4 +1663,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=a47d575abe0aceb6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28a22f2b09d631cb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_statisticsmodule.c.h b/Modules/clinic/_statisticsmodule.c.h index 03543e41af7f5a..cd5b9bea8db2f2 100644 --- a/Modules/clinic/_statisticsmodule.c.h +++ b/Modules/clinic/_statisticsmodule.c.h @@ -8,7 +8,7 @@ PyDoc_STRVAR(_statistics__normal_dist_inv_cdf__doc__, "\n"); #define _STATISTICS__NORMAL_DIST_INV_CDF_METHODDEF \ - {"_normal_dist_inv_cdf", _PyCFunction_CAST(_statistics__normal_dist_inv_cdf), METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, + {"_normal_dist_inv_cdf", (PyCFunction)(void(*)(void))_statistics__normal_dist_inv_cdf, METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, static double _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu, @@ -23,38 +23,21 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi double sigma; double _return_value; - if (!_PyArg_CheckPositional("_normal_dist_inv_cdf", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "_normal_dist_inv_cdf expected 3 arguments, got %zd", nargs); goto exit; } - if (PyFloat_CheckExact(args[0])) { - p = PyFloat_AS_DOUBLE(args[0]); - } - else - { - p = PyFloat_AsDouble(args[0]); - if (p == -1.0 && PyErr_Occurred()) { - goto exit; - } - } - if (PyFloat_CheckExact(args[1])) { - mu = PyFloat_AS_DOUBLE(args[1]); - } - else - { - mu = PyFloat_AsDouble(args[1]); - if (mu == -1.0 && PyErr_Occurred()) { - goto exit; - } + p = PyFloat_AsDouble(args[0]); + if (p == -1.0 && PyErr_Occurred()) { + goto exit; } - if (PyFloat_CheckExact(args[2])) { - sigma = PyFloat_AS_DOUBLE(args[2]); + mu = PyFloat_AsDouble(args[1]); + if (mu == -1.0 && PyErr_Occurred()) { + goto exit; } - else - { - sigma = PyFloat_AsDouble(args[2]); - if (sigma == -1.0 && PyErr_Occurred()) { - goto exit; - } + sigma = PyFloat_AsDouble(args[2]); + if (sigma == -1.0 && PyErr_Occurred()) { + goto exit; } _return_value = _statistics__normal_dist_inv_cdf_impl(module, p, mu, sigma); if ((_return_value == -1.0) && PyErr_Occurred()) { @@ -65,4 +48,4 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi exit: return return_value; } -/*[clinic end generated code: output=b807a8243e7801e6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f0c849d51f16f1b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 8c468e6e7205e6..1a07532bdd75ad 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -7,8 +7,9 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(Struct__doc__, +PyDoc_STRVAR(Struct___init____doc__, "Struct(format)\n" "--\n" "\n" @@ -19,13 +20,13 @@ PyDoc_STRVAR(Struct__doc__, "\n" "See help(struct) for more on format strings."); -static PyObject * -Struct_impl(PyTypeObject *type, PyObject *format); +static int +Struct___init___impl(PyStructObject *self, PyObject *format); -static PyObject * -Struct(PyTypeObject *type, PyObject *args, PyObject *kwargs) +static int +Struct___init__(PyObject *self, PyObject *args, PyObject *kwargs) { - PyObject *return_value = NULL; + int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 1 @@ -61,7 +62,7 @@ Struct(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } format = fastargs[0]; - return_value = Struct_impl(type, format); + return_value = Struct___init___impl((PyStructObject *)self, format); exit: return return_value; @@ -93,10 +94,6 @@ Struct_unpack(PyStructObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack", "argument", "contiguous buffer", arg); - goto exit; - } return_value = Struct_unpack_impl(self, &buffer); exit: @@ -169,10 +166,6 @@ Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack_from", "argument 'buffer'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -299,10 +292,6 @@ unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = unpack_impl(module, s_object, &buffer); exit: @@ -378,10 +367,6 @@ unpack_from(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack_from", "argument 'buffer'", "contiguous buffer", args[1]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -451,4 +436,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=5c1bc384ff87df1f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=67bd299e5d72fee0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_suggestions.c.h b/Modules/clinic/_suggestions.c.h new file mode 100644 index 00000000000000..51484b13d5af89 --- /dev/null +++ b/Modules/clinic/_suggestions.c.h @@ -0,0 +1,41 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(_suggestions__generate_suggestions__doc__, +"_generate_suggestions($module, candidates, item, /)\n" +"--\n" +"\n" +"Returns the candidate in candidates that\'s closest to item"); + +#define _SUGGESTIONS__GENERATE_SUGGESTIONS_METHODDEF \ + {"_generate_suggestions", _PyCFunction_CAST(_suggestions__generate_suggestions), METH_FASTCALL, _suggestions__generate_suggestions__doc__}, + +static PyObject * +_suggestions__generate_suggestions_impl(PyObject *module, + PyObject *candidates, PyObject *item); + +static PyObject * +_suggestions__generate_suggestions(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *candidates; + PyObject *item; + + if (!_PyArg_CheckPositional("_generate_suggestions", nargs, 2, 2)) { + goto exit; + } + candidates = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("_generate_suggestions", "argument 2", "str", args[1]); + goto exit; + } + item = args[1]; + return_value = _suggestions__generate_suggestions_impl(module, candidates, item); + +exit: + return return_value; +} +/*[clinic end generated code: output=1d8e963cdae30b13 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index ba3aeca49e26eb..e02f39d15cce0f 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -7,6 +7,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() PyDoc_STRVAR(test_empty_function__doc__, @@ -233,10 +234,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -246,10 +261,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -259,10 +288,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -272,10 +315,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -285,10 +342,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -298,10 +369,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -311,10 +396,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -324,10 +423,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -337,10 +450,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -350,10 +477,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -363,10 +504,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -376,10 +531,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -389,10 +558,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -402,10 +585,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -647,7 +844,10 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1258,7 +1458,7 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1314,7 +1514,7 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1370,7 +1570,7 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1439,7 +1639,7 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1519,7 +1719,7 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1587,7 +1787,7 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1643,7 +1843,7 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1700,7 +1900,7 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1759,7 +1959,7 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1830,7 +2030,7 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1906,7 +2106,7 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1977,7 +2177,7 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2053,7 +2253,7 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2128,7 +2328,7 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2212,7 +2412,7 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2291,7 +2491,7 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2346,7 +2546,7 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2441,7 +2641,7 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2499,7 +2699,7 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2566,7 +2766,7 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2813,6 +3013,76 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(null_or_tuple_for_varargs__doc__, +"null_or_tuple_for_varargs($module, /, name, *constraints,\n" +" covariant=False)\n" +"--\n" +"\n" +"See https://github.com/python/cpython/issues/110864"); + +#define NULL_OR_TUPLE_FOR_VARARGS_METHODDEF \ + {"null_or_tuple_for_varargs", _PyCFunction_CAST(null_or_tuple_for_varargs), METH_FASTCALL|METH_KEYWORDS, null_or_tuple_for_varargs__doc__}, + +static PyObject * +null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, + PyObject *constraints, int covariant); + +static PyObject * +null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(covariant), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "covariant", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "null_or_tuple_for_varargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name; + PyObject *constraints = NULL; + int covariant = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + name = args[0]; + constraints = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + covariant = PyObject_IsTrue(args[2]); + if (covariant < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = null_or_tuple_for_varargs_impl(module, name, constraints, covariant); + +exit: + Py_XDECREF(constraints); + return return_value; +} + PyDoc_STRVAR(clone_f1__doc__, "clone_f1($module, /, path)\n" "--\n" @@ -3070,4 +3340,82 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=c2a69a08ffdfc466 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class__doc__, +"get_defining_class($self, /)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF \ + {"get_defining_class", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class__doc__}, + +static PyObject * +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls); + +static PyObject * +_testclinic_TestClass_get_defining_class(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "get_defining_class() takes no arguments"); + return NULL; + } + return _testclinic_TestClass_get_defining_class_impl(self, cls); +} + +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class_arg__doc__, +"get_defining_class_arg($self, /, arg)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF \ + {"get_defining_class_arg", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class_arg), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class_arg__doc__}, + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg); + +static PyObject * +_testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_defining_class_arg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + arg = args[0]; + return_value = _testclinic_TestClass_get_defining_class_arg_impl(self, cls, arg); + +exit: + return return_value; +} +/*[clinic end generated code: output=0d0ceed6c46547bb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 36ff55bc002939..95a2cc4cb5ed6d 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -7,6 +7,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() PyDoc_STRVAR(depr_star_new__doc__, @@ -47,7 +48,7 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -132,7 +133,7 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -214,7 +215,7 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -299,7 +300,7 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -371,7 +372,7 @@ depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -450,7 +451,7 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -533,7 +534,7 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -607,7 +608,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -634,7 +635,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) { + if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PyDict_Contains() above can fail goto exit; } @@ -691,7 +692,7 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -769,7 +770,7 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -850,7 +851,7 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -934,7 +935,7 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1019,7 +1020,7 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1100,7 +1101,7 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1185,7 +1186,7 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1268,7 +1269,7 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1353,7 +1354,7 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1440,7 +1441,7 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1524,7 +1525,7 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1617,7 +1618,7 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1698,7 +1699,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1779,7 +1780,7 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1865,7 +1866,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1958,7 +1959,7 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2056,7 +2057,7 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2144,7 +2145,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2170,7 +2171,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) { + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PySequence_Contains() above can fail goto exit; } @@ -2231,7 +2232,7 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2332,7 +2333,7 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2392,4 +2393,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=689b1e2d0872e413 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ca6da2c7137554be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index eff72cee24975c..94897f4c6dc427 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -91,4 +91,119 @@ my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_float_sum__doc__, +"my_float_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_FLOAT_SUM_METHODDEF \ + {"my_float_sum", (PyCFunction)(void(*)(void))my_float_sum, METH_FASTCALL, my_float_sum__doc__}, + +static float +my_float_sum_impl(PyObject *module, float x, float y); + +static PyObject * +my_float_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float x; + float y; + float _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_float_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = (float) PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = (float) PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_float_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble((double)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(my_double_sum__doc__, +"my_double_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_DOUBLE_SUM_METHODDEF \ + {"my_double_sum", (PyCFunction)(void(*)(void))my_double_sum, METH_FASTCALL, my_double_sum__doc__}, + +static double +my_double_sum_impl(PyObject *module, double x, double y); + +static PyObject * +my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_double_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_double_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(get_file_descriptor__doc__, +"get_file_descriptor($module, file, /)\n" +"--\n" +"\n" +"Get a file descriptor."); + +#define GET_FILE_DESCRIPTOR_METHODDEF \ + {"get_file_descriptor", (PyCFunction)get_file_descriptor, METH_O, get_file_descriptor__doc__}, + +static int +get_file_descriptor_impl(PyObject *module, int fd); + +static PyObject * +get_file_descriptor(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + int _return_value; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + _return_value = get_file_descriptor_impl(module, fd); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=03fd7811c056dc74 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index c1b42672e13d53..dcca9ecf735723 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_testinternalcapi_compiler_cleandoc__doc__, "compiler_cleandoc($module, /, doc)\n" @@ -66,6 +67,24 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(_testinternalcapi_new_instruction_sequence__doc__, +"new_instruction_sequence($module, /)\n" +"--\n" +"\n" +"Return a new, empty InstructionSequence."); + +#define _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF \ + {"new_instruction_sequence", (PyCFunction)_testinternalcapi_new_instruction_sequence, METH_NOARGS, _testinternalcapi_new_instruction_sequence__doc__}, + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module); + +static PyObject * +_testinternalcapi_new_instruction_sequence(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_new_instruction_sequence_impl(module); +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -265,52 +284,80 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, return return_value; } -PyDoc_STRVAR(_testinternalcapi_write_unraisable_exc__doc__, -"write_unraisable_exc($module, exception, err_msg, obj, /)\n" +PyDoc_STRVAR(_testinternalcapi_test_long_numbits__doc__, +"test_long_numbits($module, /)\n" "--\n" "\n"); -#define _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF \ - {"write_unraisable_exc", _PyCFunction_CAST(_testinternalcapi_write_unraisable_exc), METH_FASTCALL, _testinternalcapi_write_unraisable_exc__doc__}, +#define _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF \ + {"test_long_numbits", (PyCFunction)_testinternalcapi_test_long_numbits, METH_NOARGS, _testinternalcapi_test_long_numbits__doc__}, static PyObject * -_testinternalcapi_write_unraisable_exc_impl(PyObject *module, PyObject *exc, - PyObject *err_msg, PyObject *obj); +_testinternalcapi_test_long_numbits_impl(PyObject *module); static PyObject * -_testinternalcapi_write_unraisable_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) { - PyObject *return_value = NULL; - PyObject *exc; - PyObject *err_msg; - PyObject *obj; - - if (!_PyArg_CheckPositional("write_unraisable_exc", nargs, 3, 3)) { - goto exit; - } - exc = args[0]; - err_msg = args[1]; - obj = args[2]; - return_value = _testinternalcapi_write_unraisable_exc_impl(module, exc, err_msg, obj); - -exit: - return return_value; + return _testinternalcapi_test_long_numbits_impl(module); } -PyDoc_STRVAR(_testinternalcapi_test_long_numbits__doc__, -"test_long_numbits($module, /)\n" +PyDoc_STRVAR(gh_119213_getargs__doc__, +"gh_119213_getargs($module, /, spam=None)\n" "--\n" -"\n"); +"\n" +"Test _PyArg_Parser.kwtuple"); -#define _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF \ - {"test_long_numbits", (PyCFunction)_testinternalcapi_test_long_numbits, METH_NOARGS, _testinternalcapi_test_long_numbits__doc__}, +#define GH_119213_GETARGS_METHODDEF \ + {"gh_119213_getargs", _PyCFunction_CAST(gh_119213_getargs), METH_FASTCALL|METH_KEYWORDS, gh_119213_getargs__doc__}, static PyObject * -_testinternalcapi_test_long_numbits_impl(PyObject *module); +gh_119213_getargs_impl(PyObject *module, PyObject *spam); static PyObject * -_testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) +gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _testinternalcapi_test_long_numbits_impl(module); + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(spam), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"spam", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_119213_getargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *spam = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + spam = args[0]; +skip_optional_pos: + return_value = gh_119213_getargs_impl(module, spam); + +exit: + return return_value; } -/*[clinic end generated code: output=59144f59957627bd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4d0770a1c20fbf40 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index a96d20bf33ea03..452897b3fae99e 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "get_defining_module($self, /)\n" @@ -26,7 +27,7 @@ _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject * static PyObject * _testmultiphase_StateAccessType_get_defining_module(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_defining_module() takes no arguments"); return NULL; } @@ -49,7 +50,7 @@ _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(StateAccessTypeObjec static PyObject * _testmultiphase_StateAccessType_getmodulebydef_bad_def(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getmodulebydef_bad_def() takes no arguments"); return NULL; } @@ -87,7 +88,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(n), &_Py_ID(twice), }, + .ob_item = { _Py_LATIN1_CHR('n'), &_Py_ID(twice), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -155,10 +156,10 @@ _testmultiphase_StateAccessType_get_count_impl(StateAccessTypeObject *self, static PyObject * _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_count() takes no arguments"); return NULL; } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=db1fdd15244ee59c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=59cb50dae2d11dc1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index fcfc406238808b..2b1ac954b4d570 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_BadArgument() + PyDoc_STRVAR(_tkinter_tkapp_eval__doc__, "eval($self, script, /)\n" "--\n" @@ -620,6 +622,33 @@ _tkinter_tkapp_loadtk(TkappObject *self, PyObject *Py_UNUSED(ignored)) return _tkinter_tkapp_loadtk_impl(self); } +PyDoc_STRVAR(_tkinter_tkapp_settrace__doc__, +"settrace($self, func, /)\n" +"--\n" +"\n" +"Set the tracing function."); + +#define _TKINTER_TKAPP_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)_tkinter_tkapp_settrace, METH_O, _tkinter_tkapp_settrace__doc__}, + +PyDoc_STRVAR(_tkinter_tkapp_gettrace__doc__, +"gettrace($self, /)\n" +"--\n" +"\n" +"Get the tracing function."); + +#define _TKINTER_TKAPP_GETTRACE_METHODDEF \ + {"gettrace", (PyCFunction)_tkinter_tkapp_gettrace, METH_NOARGS, _tkinter_tkapp_gettrace__doc__}, + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self); + +static PyObject * +_tkinter_tkapp_gettrace(TkappObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter_tkapp_gettrace_impl(self); +} + PyDoc_STRVAR(_tkinter_tkapp_willdispatch__doc__, "willdispatch($self, /)\n" "--\n" @@ -647,7 +676,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__, PyDoc_STRVAR(_tkinter_create__doc__, "create($module, screenName=None, baseName=\'\', className=\'Tk\',\n" -" interactive=False, wantobjects=False, wantTk=True, sync=False,\n" +" interactive=False, wantobjects=0, wantTk=True, sync=False,\n" " use=None, /)\n" "--\n" "\n" @@ -748,8 +777,8 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - wantobjects = PyObject_IsTrue(args[4]); - if (wantobjects < 0) { + wantobjects = PyLong_AsInt(args[4]); + if (wantobjects == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 6) { @@ -859,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=bcd9cdc8f6bdcfae input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d90c1a9850c63249 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tracemalloc.c.h b/Modules/clinic/_tracemalloc.c.h index c07ad797d6295b..1d100247423991 100644 --- a/Modules/clinic/_tracemalloc.c.h +++ b/Modules/clinic/_tracemalloc.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_tracemalloc_is_tracing__doc__, "is_tracing($module, /)\n" "--\n" @@ -212,4 +214,4 @@ _tracemalloc_reset_peak(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _tracemalloc_reset_peak_impl(module); } -/*[clinic end generated code: output=ad7d1fae89f2bdaa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d4d884b156c2ddb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_weakref.c.h b/Modules/clinic/_weakref.c.h index 541cba75e6813c..8d7bc5dc936610 100644 --- a/Modules/clinic/_weakref.c.h +++ b/Modules/clinic/_weakref.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_weakref_getweakrefcount__doc__, "getweakrefcount($module, object, /)\n" "--\n" @@ -110,4 +112,4 @@ _weakref_proxy(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f4be6b8177fbceb8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60f59adc1dc9eab8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 7fef127bef8606..b0c54fc809f4c1 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_Size_t_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_winapi_Overlapped_GetOverlappedResult__doc__, "GetOverlappedResult($self, wait, /)\n" @@ -150,6 +151,76 @@ _winapi_ConnectNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } +PyDoc_STRVAR(_winapi_CreateEventW__doc__, +"CreateEventW($module, /, security_attributes, manual_reset,\n" +" initial_state, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEEVENTW_METHODDEF \ + {"CreateEventW", _PyCFunction_CAST(_winapi_CreateEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateEventW__doc__}, + +static HANDLE +_winapi_CreateEventW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL manual_reset, BOOL initial_state, + LPCWSTR name); + +static PyObject * +_winapi_CreateEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(manual_reset), &_Py_ID(initial_state), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "manual_reset", "initial_state", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iiO&:CreateEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL manual_reset; + BOOL initial_state; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &manual_reset, &initial_state, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateEventW_impl(module, security_attributes, manual_reset, initial_state, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateFile__doc__, "CreateFile($module, file_name, desired_access, share_mode,\n" " security_attributes, creation_disposition,\n" @@ -161,7 +232,7 @@ PyDoc_STRVAR(_winapi_CreateFile__doc__, {"CreateFile", _PyCFunction_CAST(_winapi_CreateFile), METH_FASTCALL, _winapi_CreateFile__doc__}, static HANDLE -_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name, +_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, DWORD desired_access, DWORD share_mode, LPSECURITY_ATTRIBUTES security_attributes, DWORD creation_disposition, @@ -171,7 +242,7 @@ static PyObject * _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR file_name; + LPCWSTR file_name = NULL; DWORD desired_access; DWORD share_mode; LPSECURITY_ATTRIBUTES security_attributes; @@ -180,8 +251,8 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) HANDLE template_file; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skk" F_POINTER "kk" F_HANDLE ":CreateFile", - &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { + if (!_PyArg_ParseStack(args, nargs, "O&kk" F_POINTER "kk" F_HANDLE ":CreateFile", + _PyUnicode_WideCharString_Converter, &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) { goto exit; } _return_value = _winapi_CreateFile_impl(module, file_name, desired_access, share_mode, security_attributes, creation_disposition, flags_and_attributes, template_file); @@ -194,6 +265,9 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for file_name */ + PyMem_Free((void *)file_name); + return return_value; } @@ -293,6 +367,73 @@ _winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_winapi_CreateMutexW__doc__, +"CreateMutexW($module, /, security_attributes, initial_owner, name)\n" +"--\n" +"\n"); + +#define _WINAPI_CREATEMUTEXW_METHODDEF \ + {"CreateMutexW", _PyCFunction_CAST(_winapi_CreateMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateMutexW__doc__}, + +static HANDLE +_winapi_CreateMutexW_impl(PyObject *module, + LPSECURITY_ATTRIBUTES security_attributes, + BOOL initial_owner, LPCWSTR name); + +static PyObject * +_winapi_CreateMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(security_attributes), &_Py_ID(initial_owner), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"security_attributes", "initial_owner", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_POINTER "iO&:CreateMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPSECURITY_ATTRIBUTES security_attributes; + BOOL initial_owner; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &security_attributes, &initial_owner, _PyUnicode_WideCharString_Opt_Converter, &name)) { + goto exit; + } + _return_value = _winapi_CreateMutexW_impl(module, security_attributes, initial_owner, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, "CreateNamedPipe($module, name, open_mode, pipe_mode, max_instances,\n" " out_buffer_size, in_buffer_size, default_timeout,\n" @@ -304,7 +445,7 @@ PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, {"CreateNamedPipe", _PyCFunction_CAST(_winapi_CreateNamedPipe), METH_FASTCALL, _winapi_CreateNamedPipe__doc__}, static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, @@ -314,7 +455,7 @@ static PyObject * _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD open_mode; DWORD pipe_mode; DWORD max_instances; @@ -324,8 +465,8 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg LPSECURITY_ATTRIBUTES security_attributes; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skkkkkk" F_POINTER ":CreateNamedPipe", - &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { + if (!_PyArg_ParseStack(args, nargs, "O&kkkkkk" F_POINTER ":CreateNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { goto exit; } _return_value = _winapi_CreateNamedPipe_impl(module, name, open_mode, pipe_mode, max_instances, out_buffer_size, in_buffer_size, default_timeout, security_attributes); @@ -338,6 +479,9 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -567,7 +711,7 @@ _winapi_GetExitCodeProcess(PyObject *module, PyObject *arg) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -594,12 +738,82 @@ _winapi_GetLastError(PyObject *module, PyObject *Py_UNUSED(ignored)) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; } +PyDoc_STRVAR(_winapi_GetLongPathName__doc__, +"GetLongPathName($module, /, path)\n" +"--\n" +"\n" +"Return the long version of the provided path.\n" +"\n" +"If the path is already in its long form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETLONGPATHNAME_METHODDEF \ + {"GetLongPathName", _PyCFunction_CAST(_winapi_GetLongPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetLongPathName__doc__}, + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetLongPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetLongPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetLongPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetLongPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetModuleFileName__doc__, "GetModuleFileName($module, module_handle, /)\n" "--\n" @@ -634,6 +848,76 @@ _winapi_GetModuleFileName(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_GetShortPathName__doc__, +"GetShortPathName($module, /, path)\n" +"--\n" +"\n" +"Return the short version of the provided path.\n" +"\n" +"If the path is already in its short form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETSHORTPATHNAME_METHODDEF \ + {"GetShortPathName", _PyCFunction_CAST(_winapi_GetShortPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetShortPathName__doc__}, + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetShortPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetShortPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetShortPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetShortPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetStdHandle__doc__, "GetStdHandle($module, std_handle, /)\n" "--\n" @@ -767,6 +1051,138 @@ _winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_OpenEventW__doc__, +"OpenEventW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENEVENTW_METHODDEF \ + {"OpenEventW", _PyCFunction_CAST(_winapi_OpenEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenEventW__doc__}, + +static HANDLE +_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenEventW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenEventW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_OpenMutexW__doc__, +"OpenMutexW($module, /, desired_access, inherit_handle, name)\n" +"--\n" +"\n"); + +#define _WINAPI_OPENMUTEXW_METHODDEF \ + {"OpenMutexW", _PyCFunction_CAST(_winapi_OpenMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenMutexW__doc__}, + +static HANDLE +_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, + BOOL inherit_handle, LPCWSTR name); + +static PyObject * +_winapi_OpenMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "kiO&:OpenMutexW", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + DWORD desired_access; + BOOL inherit_handle; + LPCWSTR name = NULL; + HANDLE _return_value; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) { + goto exit; + } + _return_value = _winapi_OpenMutexW_impl(module, desired_access, inherit_handle, name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -987,6 +1403,162 @@ _winapi_ReadFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_winapi_ReleaseMutex__doc__, +"ReleaseMutex($module, /, mutex)\n" +"--\n" +"\n"); + +#define _WINAPI_RELEASEMUTEX_METHODDEF \ + {"ReleaseMutex", _PyCFunction_CAST(_winapi_ReleaseMutex), METH_FASTCALL|METH_KEYWORDS, _winapi_ReleaseMutex__doc__}, + +static PyObject * +_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex); + +static PyObject * +_winapi_ReleaseMutex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(mutex), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"mutex", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ReleaseMutex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE mutex; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &mutex)) { + goto exit; + } + return_value = _winapi_ReleaseMutex_impl(module, mutex); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_ResetEvent__doc__, +"ResetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_RESETEVENT_METHODDEF \ + {"ResetEvent", _PyCFunction_CAST(_winapi_ResetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_ResetEvent__doc__}, + +static PyObject * +_winapi_ResetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_ResetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":ResetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_ResetEvent_impl(module, event); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_SetEvent__doc__, +"SetEvent($module, /, event)\n" +"--\n" +"\n"); + +#define _WINAPI_SETEVENT_METHODDEF \ + {"SetEvent", _PyCFunction_CAST(_winapi_SetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_SetEvent__doc__}, + +static PyObject * +_winapi_SetEvent_impl(PyObject *module, HANDLE event); + +static PyObject * +_winapi_SetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(event), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"event", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "" F_HANDLE ":SetEvent", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + HANDLE event; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &event)) { + goto exit; + } + return_value = _winapi_SetEvent_impl(module, event); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_SetNamedPipeHandleState__doc__, "SetNamedPipeHandleState($module, named_pipe, mode,\n" " max_collection_count, collect_data_timeout, /)\n" @@ -1091,21 +1663,95 @@ PyDoc_STRVAR(_winapi_WaitNamedPipe__doc__, {"WaitNamedPipe", _PyCFunction_CAST(_winapi_WaitNamedPipe), METH_FASTCALL, _winapi_WaitNamedPipe__doc__}, static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout); +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout); static PyObject * _winapi_WaitNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD timeout; - if (!_PyArg_ParseStack(args, nargs, "sk:WaitNamedPipe", - &name, &timeout)) { + if (!_PyArg_ParseStack(args, nargs, "O&k:WaitNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &timeout)) { goto exit; } return_value = _winapi_WaitNamedPipe_impl(module, name, timeout); +exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_BatchedWaitForMultipleObjects__doc__, +"BatchedWaitForMultipleObjects($module, /, handle_seq, wait_all,\n" +" milliseconds=_winapi.INFINITE)\n" +"--\n" +"\n" +"Supports a larger number of handles than WaitForMultipleObjects\n" +"\n" +"Note that the handles may be waited on other threads, which could cause\n" +"issues for objects like mutexes that become associated with the thread\n" +"that was waiting for them. Objects may also be left signalled, even if\n" +"the wait fails.\n" +"\n" +"It is recommended to use WaitForMultipleObjects whenever possible, and\n" +"only switch to BatchedWaitForMultipleObjects for scenarios where you\n" +"control all the handles involved, such as your own thread pool or\n" +"files, and all wait objects are left unmodified by a wait (for example,\n" +"manual reset events, threads, and files/pipes).\n" +"\n" +"Overlapped handles returned from this module use manual reset events."); + +#define _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF \ + {"BatchedWaitForMultipleObjects", _PyCFunction_CAST(_winapi_BatchedWaitForMultipleObjects), METH_FASTCALL|METH_KEYWORDS, _winapi_BatchedWaitForMultipleObjects__doc__}, + +static PyObject * +_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, + PyObject *handle_seq, + BOOL wait_all, DWORD milliseconds); + +static PyObject * +_winapi_BatchedWaitForMultipleObjects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(handle_seq), &_Py_ID(wait_all), &_Py_ID(milliseconds), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"handle_seq", "wait_all", "milliseconds", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "Oi|k:BatchedWaitForMultipleObjects", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *handle_seq; + BOOL wait_all; + DWORD milliseconds = INFINITE; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &handle_seq, &wait_all, &milliseconds)) { + goto exit; + } + return_value = _winapi_BatchedWaitForMultipleObjects_impl(module, handle_seq, wait_all, milliseconds); + exit: return return_value; } @@ -1304,7 +1950,7 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -1478,4 +2124,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=6df38b5eb93f2e5a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2304c62187a90140 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 6691d39deb7c24..9905b6425e2f79 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "from_file($type, file_obj, /, key=None)\n" @@ -371,4 +372,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=a5384f79d49a593b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2a15f32fdd2ab6cd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 2a874b8c830609..2ed7eaa6abf7af 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -2,7 +2,29 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(array_array_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the array."); + +#define ARRAY_ARRAY_CLEAR_METHODDEF \ + {"clear", (PyCFunction)array_array_clear, METH_NOARGS, array_array_clear__doc__}, + +static PyObject * +array_array_clear_impl(arrayobject *self); + +static PyObject * +array_array_clear(arrayobject *self, PyObject *Py_UNUSED(ignored)) +{ + return array_array_clear_impl(self); +} PyDoc_STRVAR(array_array___copy____doc__, "__copy__($self, /)\n" @@ -441,10 +463,6 @@ array_array_frombytes(arrayobject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("frombytes", "argument", "contiguous buffer", arg); - goto exit; - } return_value = array_array_frombytes_impl(self, &buffer); exit: @@ -581,7 +599,10 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (PyUnicode_GET_LENGTH(args[1]) != 1) { - _PyArg_BadArgument("_array_reconstructor", "argument 2", "a unicode character", args[1]); + PyErr_Format(PyExc_TypeError, + "_array_reconstructor(): argument 2 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[1])); goto exit; } typecode = PyUnicode_READ_CHAR(args[1], 0); @@ -655,7 +676,7 @@ array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls); static PyObject * array_arrayiterator___reduce__(arrayiterobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); return NULL; } @@ -670,4 +691,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=8595b1906b5a6552 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ecd63acd7924c223 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index d80decf145bfdc..1adca415dfee12 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(binascii_a2b_uu__doc__, "a2b_uu($module, data, /)\n" @@ -91,10 +92,6 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_uu", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -242,10 +239,6 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_base64", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -290,10 +283,6 @@ binascii_crc_hqx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc_hqx", "argument 1", "contiguous buffer", args[0]); - goto exit; - } crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); if (crc == (unsigned int)-1 && PyErr_Occurred()) { goto exit; @@ -335,10 +324,6 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -434,10 +419,6 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_hex", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -527,10 +508,6 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("hexlify", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -754,10 +731,6 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_qp", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -794,4 +767,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=acc9419209dfd568 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=968767b663ed889d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index a665ac3c1c4994..50745fd4f407a3 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(cmath_acos__doc__, "acos($module, z, /)\n" @@ -907,7 +908,7 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -981,4 +982,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=96a5c4ae198dd5bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=454309b21cfa9bf6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 83e882bbbaf184..53b139e09afdf1 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() - PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" @@ -21,7 +19,7 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "corresponding to the return value of the fcntl call in the C code."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -34,10 +32,16 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -89,10 +93,10 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "code."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg); static PyObject * @@ -100,20 +104,27 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; - unsigned int code; + unsigned long code; PyObject *ob_arg = NULL; int mutate_arg = 1; - if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + if (nargs > 4) { + PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); goto exit; } - code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (code == (unsigned int)-1 && PyErr_Occurred()) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } + if (!PyLong_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); + goto exit; + } + code = PyLong_AsUnsignedLongMask(args[1]); if (nargs < 3) { goto skip_optional; } @@ -142,7 +153,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, + {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -154,10 +165,12 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -198,7 +211,7 @@ PyDoc_STRVAR(fcntl_lockf__doc__, " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -214,10 +227,16 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 5) { + PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -245,4 +264,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=4d4fac195494faec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45a56f53fd17ff3c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 40448404877e3e..9fff4da616ba00 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -213,6 +214,58 @@ gc_get_debug(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(gc_set_threshold__doc__, +"set_threshold(threshold0, [threshold1, [threshold2]])\n" +"Set the collection thresholds (the collection frequency).\n" +"\n" +"Setting \'threshold0\' to zero disables collection."); + +#define GC_SET_THRESHOLD_METHODDEF \ + {"set_threshold", (PyCFunction)gc_set_threshold, METH_VARARGS, gc_set_threshold__doc__}, + +static PyObject * +gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, + int threshold1, int group_right_2, int threshold2); + +static PyObject * +gc_set_threshold(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int threshold0; + int group_right_1 = 0; + int threshold1 = 0; + int group_right_2 = 0; + int threshold2 = 0; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "i:set_threshold", &threshold0)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "ii:set_threshold", &threshold0, &threshold1)) { + goto exit; + } + group_right_1 = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "iii:set_threshold", &threshold0, &threshold1, &threshold2)) { + goto exit; + } + group_right_1 = 1; + group_right_2 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "gc.set_threshold requires 1 to 3 arguments"); + goto exit; + } + return_value = gc_set_threshold_impl(module, threshold0, group_right_1, threshold1, group_right_2, threshold2); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_get_threshold__doc__, "get_threshold($module, /)\n" "--\n" @@ -249,6 +302,76 @@ gc_get_count(PyObject *module, PyObject *Py_UNUSED(ignored)) return gc_get_count_impl(module); } +PyDoc_STRVAR(gc_get_referrers__doc__, +"get_referrers($module, /, *objs)\n" +"--\n" +"\n" +"Return the list of objects that directly refer to any of \'objs\'."); + +#define GC_GET_REFERRERS_METHODDEF \ + {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, + +static PyObject * +gc_get_referrers_impl(PyObject *module, PyObject *args); + +static PyObject * +gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gc_get_referrers_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gc_get_referents__doc__, +"get_referents($module, /, *objs)\n" +"--\n" +"\n" +"Return the list of objects that are directly referred to by \'objs\'."); + +#define GC_GET_REFERENTS_METHODDEF \ + {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, + +static PyObject * +gc_get_referents_impl(PyObject *module, PyObject *args); + +static PyObject * +gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = gc_get_referents_impl(module, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gc_get_objects__doc__, "get_objects($module, /, generation=None)\n" "--\n" @@ -346,6 +469,25 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_tracked(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_tracked_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_is_finalized__doc__, "is_finalized($module, obj, /)\n" "--\n" @@ -355,6 +497,25 @@ PyDoc_STRVAR(gc_is_finalized__doc__, #define GC_IS_FINALIZED_METHODDEF \ {"is_finalized", (PyCFunction)gc_is_finalized, METH_O, gc_is_finalized__doc__}, +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_finalized(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_finalized_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_freeze__doc__, "freeze($module, /)\n" "--\n" @@ -424,4 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=63093e7724b94a37 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/grpmodule.c.h b/Modules/clinic/grpmodule.c.h index e6c9cccfce4a76..cc0ad210f42743 100644 --- a/Modules/clinic/grpmodule.c.h +++ b/Modules/clinic/grpmodule.c.h @@ -2,11 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - PyDoc_STRVAR(grp_getgrgid__doc__, "getgrgid($module, /, id)\n" "--\n" @@ -16,48 +11,21 @@ PyDoc_STRVAR(grp_getgrgid__doc__, "If id is not valid, raise KeyError."); #define GRP_GETGRGID_METHODDEF \ - {"getgrgid", _PyCFunction_CAST(grp_getgrgid), METH_FASTCALL|METH_KEYWORDS, grp_getgrgid__doc__}, + {"getgrgid", (PyCFunction)(void(*)(void))grp_getgrgid, METH_VARARGS|METH_KEYWORDS, grp_getgrgid__doc__}, static PyObject * grp_getgrgid_impl(PyObject *module, PyObject *id); static PyObject * -grp_getgrgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrgid(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(id), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"id", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrgid", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"id", NULL}; PyObject *id; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:getgrgid", _keywords, + &id)) goto exit; - } - id = args[0]; return_value = grp_getgrgid_impl(module, id); exit: @@ -73,52 +41,21 @@ PyDoc_STRVAR(grp_getgrnam__doc__, "If name is not valid, raise KeyError."); #define GRP_GETGRNAM_METHODDEF \ - {"getgrnam", _PyCFunction_CAST(grp_getgrnam), METH_FASTCALL|METH_KEYWORDS, grp_getgrnam__doc__}, + {"getgrnam", (PyCFunction)(void(*)(void))grp_getgrnam, METH_VARARGS|METH_KEYWORDS, grp_getgrnam__doc__}, static PyObject * grp_getgrnam_impl(PyObject *module, PyObject *name); static PyObject * -grp_getgrnam(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +grp_getgrnam(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"name", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "getgrnam", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"name", NULL}; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("getgrnam", "argument 'name'", "str", args[0]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:getgrnam", _keywords, + &name)) goto exit; - } - name = args[0]; return_value = grp_getgrnam_impl(module, name); exit: @@ -145,4 +82,4 @@ grp_getgrall(PyObject *module, PyObject *Py_UNUSED(ignored)) { return grp_getgrall_impl(module); } -/*[clinic end generated code: output=d4bdad9b26fb8558 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=81f180beb67fc585 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 74cd3ddc3c9a86..050c21460d79d7 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -7,9 +7,10 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(batched_new__doc__, -"batched(iterable, n)\n" +"batched(iterable, n, *, strict=False)\n" "--\n" "\n" "Batch data into tuples of length n. The last batch may be shorter than n.\n" @@ -24,10 +25,14 @@ PyDoc_STRVAR(batched_new__doc__, " ...\n" " (\'A\', \'B\', \'C\')\n" " (\'D\', \'E\', \'F\')\n" -" (\'G\',)"); +" (\'G\',)\n" +"\n" +"If \"strict\" is True, raises a ValueError if the final batch is shorter\n" +"than n."); static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n); +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict); static PyObject * batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -35,14 +40,14 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(n), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('n'), &_Py_ID(strict), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -51,18 +56,20 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"iterable", "n", NULL}; + static const char * const _keywords[] = {"iterable", "n", "strict", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "batched", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; PyObject *iterable; Py_ssize_t n; + int strict = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); if (!fastargs) { @@ -81,7 +88,15 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } n = ival; } - return_value = batched_new_impl(type, iterable, n); + if (!noptargs) { + goto skip_optional_kwonly; + } + strict = PyObject_IsTrue(fastargs[2]); + if (strict < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = batched_new_impl(type, iterable, n, strict); exit: return return_value; @@ -494,7 +509,7 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -565,7 +580,7 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -635,7 +650,7 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -913,4 +928,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=95bb863274717bd9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7b13be3075f2d6d3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index bfce397be77c8b..d16db722a74bc4 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" @@ -203,6 +204,67 @@ PyDoc_STRVAR(math_log10__doc__, #define MATH_LOG10_METHODDEF \ {"log10", (PyCFunction)math_log10, METH_O, math_log10__doc__}, +PyDoc_STRVAR(math_fma__doc__, +"fma($module, x, y, z, /)\n" +"--\n" +"\n" +"Fused multiply-add operation.\n" +"\n" +"Compute (x * y) + z with a single round."); + +#define MATH_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(math_fma), METH_FASTCALL, math_fma__doc__}, + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z); + +static PyObject * +math_fma(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double z; + + if (!_PyArg_CheckPositional("fma", nargs, 3, 3)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[2])) { + z = PyFloat_AS_DOUBLE(args[2]); + } + else + { + z = PyFloat_AsDouble(args[2]); + if (z == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_fma_impl(module, x, y, z); + +exit: + return return_value; +} + PyDoc_STRVAR(math_fmod__doc__, "fmod($module, x, y, /)\n" "--\n" @@ -586,7 +648,7 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -949,4 +1011,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=da62cea7eb3dc8bc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d03f84f77342496 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 6087a0f9c25c1a..ee7fb3d7c613f2 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(MD5Type_copy__doc__, "copy($self, /)\n" @@ -22,7 +23,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); static PyObject * MD5Type_copy(MD5object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -147,4 +148,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=943d42b9d17d9a5b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 9a71e25aa557a4..8b285e4a8f0a72 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_overlapped_CreateIoCompletionPort__doc__, "CreateIoCompletionPort($module, handle, port, key, concurrency, /)\n" @@ -623,10 +624,6 @@ _overlapped_Overlapped_ReadFileInto(OverlappedObject *self, PyObject *const *arg if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("ReadFileInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _overlapped_Overlapped_ReadFileInto_impl(self, handle, &bufobj); exit: @@ -714,10 +711,6 @@ _overlapped_Overlapped_WSARecvInto(OverlappedObject *self, PyObject *const *args if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSARecvInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -762,10 +755,6 @@ _overlapped_Overlapped_WriteFile(OverlappedObject *self, PyObject *const *args, if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WriteFile", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _overlapped_Overlapped_WriteFile_impl(self, handle, &bufobj); exit: @@ -808,10 +797,6 @@ _overlapped_Overlapped_WSASend(OverlappedObject *self, PyObject *const *args, Py if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSASend", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -1137,10 +1122,6 @@ _overlapped_Overlapped_WSASendTo(OverlappedObject *self, PyObject *const *args, if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSASendTo", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -1238,10 +1219,6 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSARecvFromInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &size)) { goto exit; } @@ -1262,4 +1239,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=31bcc780209593a2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 179132754f9aeb..ca40bec345bda9 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,8 +7,8 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(os_stat__doc__, "stat($module, /, path, *, dir_fd=None, follow_symlinks=True)\n" @@ -72,7 +72,7 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("stat", "path", 0, 1); + path_t path = PATH_T_INITIALIZE_P("stat", "path", 0, 0, 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -154,7 +154,7 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("lstat", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -250,7 +250,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("access", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("access", "path", 0, 0, 0, 0); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; @@ -409,7 +409,7 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chdir", "path", 0, PATH_HAVE_FCHDIR); + path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -480,7 +480,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fchdir_impl(module, fd); @@ -492,7 +493,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #endif /* defined(HAVE_FCHDIR) */ PyDoc_STRVAR(os_chmod__doc__, -"chmod($module, /, path, mode, *, dir_fd=None, follow_symlinks=True)\n" +"chmod($module, /, path, mode, *, dir_fd=None,\n" +" follow_symlinks=(os.name != \'nt\'))\n" "--\n" "\n" "Change the access permissions of a file.\n" @@ -558,10 +560,10 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); + path_t path = PATH_T_INITIALIZE_P("chmod", "path", 0, 0, 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; - int follow_symlinks = 1; + int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); if (!args) { @@ -599,7 +601,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw return return_value; } -#if defined(HAVE_FCHMOD) +#if (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_fchmod__doc__, "fchmod($module, /, fd, mode)\n" @@ -674,9 +676,9 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_FCHMOD) */ +#endif /* (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) */ -#if defined(HAVE_LCHMOD) +#if (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_lchmod__doc__, "lchmod($module, /, path, mode)\n" @@ -723,7 +725,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchmod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -746,7 +748,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_LCHMOD) */ +#endif /* (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) */ #if defined(HAVE_CHFLAGS) @@ -800,7 +802,7 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chflags", "path", 0, 0, 0, 0); unsigned long flags; int follow_symlinks = 1; @@ -882,7 +884,7 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -952,7 +954,7 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1022,7 +1024,8 @@ os_fsync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fsync_impl(module, fd); @@ -1105,7 +1108,8 @@ os_fdatasync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fdatasync_impl(module, fd); @@ -1186,7 +1190,7 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("chown", "path", 0, PATH_HAVE_FCHOWN); + path_t path = PATH_T_INITIALIZE_P("chown", "path", 0, 0, 0, PATH_HAVE_FCHOWN); uid_t uid; gid_t gid; int dir_fd = DEFAULT_DIR_FD; @@ -1351,7 +1355,7 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("lchown", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchown", "path", 0, 0, 0, 0); uid_t uid; gid_t gid; @@ -1472,8 +1476,8 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("link", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("link", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("link", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("link", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -1579,7 +1583,7 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listdir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -1695,7 +1699,7 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1759,7 +1763,7 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1796,7 +1800,7 @@ static PyObject * os__getfullpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfullpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1830,7 +1834,7 @@ static PyObject * os__getfinalpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfinalpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfinalpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1864,7 +1868,7 @@ static PyObject * os__findfirstfile(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_findfirstfile", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1924,7 +1928,7 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getvolumepathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1988,7 +1992,7 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_splitroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -2010,6 +2014,84 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) +PyDoc_STRVAR(os__path_exists__doc__, +"_path_exists($module, path, /)\n" +"--\n" +"\n" +"Test whether a path exists. Returns False for broken symbolic links."); + +#define OS__PATH_EXISTS_METHODDEF \ + {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__}, + +static int +os__path_exists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_exists(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); + int _return_value; + + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_exists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__path_lexists__doc__, +"_path_lexists($module, path, /)\n" +"--\n" +"\n" +"Test whether a path exists. Returns True for broken symbolic links."); + +#define OS__PATH_LEXISTS_METHODDEF \ + {"_path_lexists", (PyCFunction)os__path_lexists, METH_O, os__path_lexists__doc__}, + +static int +os__path_lexists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_lexists(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); + int _return_value; + + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_lexists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__path_isdir__doc__, "_path_isdir($module, /, s)\n" "--\n" @@ -2019,8 +2101,8 @@ PyDoc_STRVAR(os__path_isdir__doc__, #define OS__PATH_ISDIR_METHODDEF \ {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s); +static int +os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2035,7 +2117,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(s), }, + .ob_item = { _Py_LATIN1_CHR('s'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2052,16 +2134,26 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *s; + path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - s = args[0]; - return_value = os__path_isdir_impl(module, s); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isdir_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2078,8 +2170,8 @@ PyDoc_STRVAR(os__path_isfile__doc__, #define OS__PATH_ISFILE_METHODDEF \ {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path); +static int +os__path_isfile_impl(PyObject *module, path_t *path); static PyObject * os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2111,16 +2203,26 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isfile_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isfile_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2128,20 +2230,20 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_exists__doc__, -"_path_exists($module, /, path)\n" +PyDoc_STRVAR(os__path_islink__doc__, +"_path_islink($module, /, path)\n" "--\n" "\n" -"Test whether a path exists. Returns False for broken symbolic links"); +"Test whether a path is a symbolic link"); -#define OS__PATH_EXISTS_METHODDEF \ - {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, +#define OS__PATH_ISLINK_METHODDEF \ + {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path); +static int +os__path_islink_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2165,21 +2267,31 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_exists", + .fname = "_path_islink", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_exists_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_islink_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2187,20 +2299,20 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_islink__doc__, -"_path_islink($module, /, path)\n" +PyDoc_STRVAR(os__path_isjunction__doc__, +"_path_isjunction($module, /, path)\n" "--\n" "\n" -"Test whether a path is a symbolic link"); +"Test whether a path is a junction"); -#define OS__PATH_ISLINK_METHODDEF \ - {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, +#define OS__PATH_ISJUNCTION_METHODDEF \ + {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, -static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path); +static int +os__path_isjunction_impl(PyObject *module, path_t *path); static PyObject * -os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2224,37 +2336,109 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_islink", + .fname = "_path_isjunction", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_islink_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isjunction_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_splitroot_ex__doc__, +"_path_splitroot_ex($module, /, path)\n" +"--\n" +"\n" +"Split a pathname into drive, root and tail.\n" +"\n" +"The tail contains anything after the root."); + +#define OS__PATH_SPLITROOT_EX_METHODDEF \ + {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + +static PyObject * +os__path_splitroot_ex_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_splitroot_ex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_splitroot_ex_impl(module, &path); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" "\n" -"Basic path normalization."); +"Normalize path, eliminating double slashes, etc."); #define OS__PATH_NORMPATH_METHODDEF \ {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path); +os__path_normpath_impl(PyObject *module, path_t *path); static PyObject * os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2286,16 +2470,21 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_normpath_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_normpath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2350,7 +2539,7 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkdir", "path", 0, 0, 0, 0); int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -2611,8 +2800,8 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("rename", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("rename", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("rename", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("rename", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2702,8 +2891,8 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("replace", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("replace", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("replace", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("replace", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2791,7 +2980,7 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("rmdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3040,7 +3229,7 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("unlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3114,7 +3303,7 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("remove", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3232,7 +3421,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD); + path_t path = PATH_T_INITIALIZE_P("utime", "path", 0, 0, 0, PATH_UTIME_HAVE_FD); PyObject *times = Py_None; PyObject *ns = NULL; int dir_fd = DEFAULT_DIR_FD; @@ -3367,7 +3556,7 @@ static PyObject * os_execv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("execv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("execv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("execv", nargs, 2, 2)) { @@ -3439,7 +3628,7 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("execve", "path", 0, PATH_HAVE_FEXECVE); + path_t path = PATH_T_INITIALIZE_P("execve", "path", 0, 0, 0, PATH_HAVE_FEXECVE); PyObject *argv; PyObject *env; @@ -3535,7 +3724,7 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawn", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3685,7 +3874,7 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawnp", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3789,7 +3978,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { @@ -3843,7 +4032,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnve", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnve", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; @@ -4463,6 +4652,159 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) #endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETAFFINITY) */ +#if defined(HAVE_POSIX_OPENPT) + +PyDoc_STRVAR(os_posix_openpt__doc__, +"posix_openpt($module, oflag, /)\n" +"--\n" +"\n" +"Open and return a file descriptor for a master pseudo-terminal device.\n" +"\n" +"Performs a posix_openpt() C function call. The oflag argument is used to\n" +"set file status flags and file access modes as specified in the manual page\n" +"of posix_openpt() of your system."); + +#define OS_POSIX_OPENPT_METHODDEF \ + {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, + +static int +os_posix_openpt_impl(PyObject *module, int oflag); + +static PyObject * +os_posix_openpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int oflag; + int _return_value; + + oflag = PyLong_AsInt(arg); + if (oflag == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = os_posix_openpt_impl(module, oflag); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_POSIX_OPENPT) */ + +#if defined(HAVE_GRANTPT) + +PyDoc_STRVAR(os_grantpt__doc__, +"grantpt($module, fd, /)\n" +"--\n" +"\n" +"Grant access to the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs a grantpt() C function call."); + +#define OS_GRANTPT_METHODDEF \ + {"grantpt", (PyCFunction)os_grantpt, METH_O, os_grantpt__doc__}, + +static PyObject * +os_grantpt_impl(PyObject *module, int fd); + +static PyObject * +os_grantpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + return_value = os_grantpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_GRANTPT) */ + +#if defined(HAVE_UNLOCKPT) + +PyDoc_STRVAR(os_unlockpt__doc__, +"unlockpt($module, fd, /)\n" +"--\n" +"\n" +"Unlock a pseudo-terminal master/slave pair.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"Performs an unlockpt() C function call."); + +#define OS_UNLOCKPT_METHODDEF \ + {"unlockpt", (PyCFunction)os_unlockpt, METH_O, os_unlockpt__doc__}, + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd); + +static PyObject * +os_unlockpt(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + return_value = os_unlockpt_impl(module, fd); + +exit: + return return_value; +} + +#endif /* defined(HAVE_UNLOCKPT) */ + +#if (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) + +PyDoc_STRVAR(os_ptsname__doc__, +"ptsname($module, fd, /)\n" +"--\n" +"\n" +"Return the name of the slave pseudo-terminal device.\n" +"\n" +" fd\n" +" File descriptor of a master pseudo-terminal device.\n" +"\n" +"If the ptsname_r() C function is available, it is called;\n" +"otherwise, performs a ptsname() C function call."); + +#define OS_PTSNAME_METHODDEF \ + {"ptsname", (PyCFunction)os_ptsname, METH_O, os_ptsname__doc__}, + +static PyObject * +os_ptsname_impl(PyObject *module, int fd); + +static PyObject * +os_ptsname(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + return_value = os_ptsname_impl(module, fd); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) */ + #if (defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX)) PyDoc_STRVAR(os_openpty__doc__, @@ -4512,7 +4854,8 @@ os_login_tty(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_login_tty_impl(module, fd); @@ -5465,7 +5808,7 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #endif /* defined(HAVE_WAIT4) */ -#if (defined(HAVE_WAITID) && !defined(__APPLE__)) +#if defined(HAVE_WAITID) PyDoc_STRVAR(os_waitid__doc__, "waitid($module, idtype, id, options, /)\n" @@ -5508,7 +5851,7 @@ os_waitid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* (defined(HAVE_WAITID) && !defined(__APPLE__)) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -5729,7 +6072,8 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -5864,7 +6208,7 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("readlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -5948,8 +6292,8 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("symlink", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("symlink", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("symlink", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("symlink", "dst", 0, 0, 0, 0); int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; @@ -5996,8 +6340,6 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #endif /* defined(HAVE_SYMLINK) */ -#if defined(HAVE_TIMES) - PyDoc_STRVAR(os_times__doc__, "times($module, /)\n" "--\n" @@ -6020,8 +6362,6 @@ os_times(PyObject *module, PyObject *Py_UNUSED(ignored)) return os_times_impl(module); } -#endif /* defined(HAVE_TIMES) */ - #if defined(HAVE_TIMERFD_CREATE) PyDoc_STRVAR(os_timerfd_create__doc__, @@ -6131,8 +6471,8 @@ PyDoc_STRVAR(os_timerfd_settime__doc__, {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval); +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double); static PyObject * os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -6167,14 +6507,15 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int fd; int flags = 0; - double initial = 0.0; - double interval = 0.0; + double initial_double = 0.0; + double interval_double = 0.0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6191,12 +6532,12 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } if (args[2]) { if (PyFloat_CheckExact(args[2])) { - initial = PyFloat_AS_DOUBLE(args[2]); + initial_double = PyFloat_AS_DOUBLE(args[2]); } else { - initial = PyFloat_AsDouble(args[2]); - if (initial == -1.0 && PyErr_Occurred()) { + initial_double = PyFloat_AsDouble(args[2]); + if (initial_double == -1.0 && PyErr_Occurred()) { goto exit; } } @@ -6205,17 +6546,17 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } } if (PyFloat_CheckExact(args[3])) { - interval = PyFloat_AS_DOUBLE(args[3]); + interval_double = PyFloat_AS_DOUBLE(args[3]); } else { - interval = PyFloat_AsDouble(args[3]); - if (interval == -1.0 && PyErr_Occurred()) { + interval_double = PyFloat_AsDouble(args[3]); + if (interval_double == -1.0 && PyErr_Occurred()) { goto exit; } } skip_optional_kwonly: - return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + return_value = os_timerfd_settime_impl(module, fd, flags, initial_double, interval_double); exit: return return_value; @@ -6287,7 +6628,8 @@ os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6347,7 +6689,8 @@ os_timerfd_gettime(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_impl(module, fd); @@ -6381,7 +6724,8 @@ os_timerfd_gettime_ns(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_ns_impl(module, fd); @@ -6591,7 +6935,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("open", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("open", "path", 0, 0, 0, 0); int flags; int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -7207,10 +7551,6 @@ os_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("write", "argument 2", "contiguous buffer", args[1]); - goto exit; - } _return_value = os_write_impl(module, fd, &data); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -7809,10 +8149,6 @@ os_pwrite(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("pwrite", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!Py_off_t_converter(args[2], &offset)) { goto exit; } @@ -8187,7 +8523,7 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkfifo", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkfifo", "path", 0, 0, 0, 0); int mode = 438; int dir_fd = DEFAULT_DIR_FD; @@ -8287,7 +8623,7 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mknod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mknod", "path", 0, 0, 0, 0); int mode = 384; dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; @@ -8349,7 +8685,7 @@ PyDoc_STRVAR(os_major__doc__, #define OS_MAJOR_METHODDEF \ {"major", (PyCFunction)os_major, METH_O, os_major__doc__}, -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device); static PyObject * @@ -8357,16 +8693,11 @@ os_major(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_major_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_major_impl(module, device); exit: return return_value; @@ -8385,7 +8716,7 @@ PyDoc_STRVAR(os_minor__doc__, #define OS_MINOR_METHODDEF \ {"minor", (PyCFunction)os_minor, METH_O, os_minor__doc__}, -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device); static PyObject * @@ -8393,16 +8724,11 @@ os_minor(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_minor_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_minor_impl(module, device); exit: return return_value; @@ -8422,25 +8748,23 @@ PyDoc_STRVAR(os_makedev__doc__, {"makedev", _PyCFunction_CAST(os_makedev), METH_FASTCALL, os_makedev__doc__}, static dev_t -os_makedev_impl(PyObject *module, int major, int minor); +os_makedev_impl(PyObject *module, dev_t major, dev_t minor); static PyObject * os_makedev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int major; - int minor; + dev_t major; + dev_t minor; dev_t _return_value; if (!_PyArg_CheckPositional("makedev", nargs, 2, 2)) { goto exit; } - major = PyLong_AsInt(args[0]); - if (major == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[0], &major)) { goto exit; } - minor = PyLong_AsInt(args[1]); - if (minor == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[1], &minor)) { goto exit; } _return_value = os_makedev_impl(module, major, minor); @@ -8541,7 +8865,7 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("truncate", "path", 0, PATH_HAVE_FTRUNCATE); + path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -8565,7 +8889,7 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #endif /* (defined HAVE_TRUNCATE || defined MS_WINDOWS) */ -#if (defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG)) +#if (defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG) && !defined(__wasi__)) PyDoc_STRVAR(os_posix_fallocate__doc__, "posix_fallocate($module, fd, offset, length, /)\n" @@ -8610,7 +8934,7 @@ os_posix_fallocate(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* (defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG)) */ +#endif /* (defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG) && !defined(__wasi__)) */ #if (defined(HAVE_POSIX_FADVISE) && !defined(POSIX_FADVISE_AIX_BUG)) @@ -9440,7 +9764,7 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("statvfs", "path", 0, PATH_HAVE_FSTATVFS); + path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9504,7 +9828,7 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getdiskusage", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9551,7 +9875,8 @@ os_fpathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("fpathconf", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!conv_path_confname(args[1], &name)) { @@ -9617,7 +9942,7 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("pathconf", "path", 0, PATH_HAVE_FPATHCONF); + path_t path = PATH_T_INITIALIZE_P("pathconf", "path", 0, 0, 0, PATH_HAVE_FPATHCONF); int name; long _return_value; @@ -9807,10 +10132,10 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); + path_t filepath = PATH_T_INITIALIZE_P("startfile", "filepath", 0, 0, 0, 0); const wchar_t *operation = NULL; const wchar_t *arguments = NULL; - path_t cwd = PATH_T_INITIALIZE("startfile", "cwd", 1, 0); + path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); @@ -10145,8 +10470,8 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("getxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("getxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("getxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10232,8 +10557,8 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("setxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("setxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("setxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("setxattr", "attribute", 0, 0, 0, 0); Py_buffer value = {NULL, NULL}; int flags = 0; int follow_symlinks = 1; @@ -10251,10 +10576,6 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[2], &value, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&value, 'C')) { - _PyArg_BadArgument("setxattr", "argument 'value'", "contiguous buffer", args[2]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -10344,8 +10665,8 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("removexattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("removexattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("removexattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10430,7 +10751,7 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listxattr", "path", 1, 1); + path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); @@ -10698,7 +11019,8 @@ os_eventfd_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_eventfd_read_impl(module, fd); @@ -10760,7 +11082,8 @@ os_eventfd_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedLongLong_Converter(args[1], &value)) { @@ -11076,7 +11399,7 @@ os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *c PyObject *return_value = NULL; int _return_value; - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "is_symlink() takes no arguments"); goto exit; } @@ -11405,7 +11728,7 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("scandir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -11617,7 +11940,7 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_add_dll_directory", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -11771,6 +12094,82 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #endif /* (defined(WIFEXITED) || defined(MS_WINDOWS)) */ +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__supports_virtual_terminal__doc__, +"_supports_virtual_terminal($module, /)\n" +"--\n" +"\n" +"Checks if virtual terminal is supported in windows"); + +#define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF \ + {"_supports_virtual_terminal", (PyCFunction)os__supports_virtual_terminal, METH_NOARGS, os__supports_virtual_terminal__doc__}, + +static PyObject * +os__supports_virtual_terminal_impl(PyObject *module); + +static PyObject * +os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__supports_virtual_terminal_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +PyDoc_STRVAR(os__inputhook__doc__, +"_inputhook($module, /)\n" +"--\n" +"\n" +"Calls PyOS_CallInputHook droppong the GIL first"); + +#define OS__INPUTHOOK_METHODDEF \ + {"_inputhook", (PyCFunction)os__inputhook, METH_NOARGS, os__inputhook__doc__}, + +static PyObject * +os__inputhook_impl(PyObject *module); + +static PyObject * +os__inputhook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__inputhook_impl(module); +} + +PyDoc_STRVAR(os__is_inputhook_installed__doc__, +"_is_inputhook_installed($module, /)\n" +"--\n" +"\n" +"Checks if PyOS_CallInputHook is set"); + +#define OS__IS_INPUTHOOK_INSTALLED_METHODDEF \ + {"_is_inputhook_installed", (PyCFunction)os__is_inputhook_installed, METH_NOARGS, os__is_inputhook_installed__doc__}, + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module); + +static PyObject * +os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__is_inputhook_installed_impl(module); +} + +PyDoc_STRVAR(os__create_environ__doc__, +"_create_environ($module, /)\n" +"--\n" +"\n" +"Create the environment dictionary."); + +#define OS__CREATE_ENVIRON_METHODDEF \ + {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__}, + +static PyObject * +os__create_environ_impl(PyObject *module); + +static PyObject * +os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__create_environ_impl(module); +} + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -11867,6 +12266,14 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS__PATH_SPLITROOT_METHODDEF #endif /* !defined(OS__PATH_SPLITROOT_METHODDEF) */ +#ifndef OS__PATH_EXISTS_METHODDEF + #define OS__PATH_EXISTS_METHODDEF +#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ + +#ifndef OS__PATH_LEXISTS_METHODDEF + #define OS__PATH_LEXISTS_METHODDEF +#endif /* !defined(OS__PATH_LEXISTS_METHODDEF) */ + #ifndef OS__PATH_ISDIR_METHODDEF #define OS__PATH_ISDIR_METHODDEF #endif /* !defined(OS__PATH_ISDIR_METHODDEF) */ @@ -11875,14 +12282,14 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS__PATH_ISFILE_METHODDEF #endif /* !defined(OS__PATH_ISFILE_METHODDEF) */ -#ifndef OS__PATH_EXISTS_METHODDEF - #define OS__PATH_EXISTS_METHODDEF -#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ - #ifndef OS__PATH_ISLINK_METHODDEF #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ +#ifndef OS__PATH_ISJUNCTION_METHODDEF + #define OS__PATH_ISJUNCTION_METHODDEF +#endif /* !defined(OS__PATH_ISJUNCTION_METHODDEF) */ + #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -11983,6 +12390,22 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SCHED_GETAFFINITY_METHODDEF #endif /* !defined(OS_SCHED_GETAFFINITY_METHODDEF) */ +#ifndef OS_POSIX_OPENPT_METHODDEF + #define OS_POSIX_OPENPT_METHODDEF +#endif /* !defined(OS_POSIX_OPENPT_METHODDEF) */ + +#ifndef OS_GRANTPT_METHODDEF + #define OS_GRANTPT_METHODDEF +#endif /* !defined(OS_GRANTPT_METHODDEF) */ + +#ifndef OS_UNLOCKPT_METHODDEF + #define OS_UNLOCKPT_METHODDEF +#endif /* !defined(OS_UNLOCKPT_METHODDEF) */ + +#ifndef OS_PTSNAME_METHODDEF + #define OS_PTSNAME_METHODDEF +#endif /* !defined(OS_PTSNAME_METHODDEF) */ + #ifndef OS_OPENPTY_METHODDEF #define OS_OPENPTY_METHODDEF #endif /* !defined(OS_OPENPTY_METHODDEF) */ @@ -12127,10 +12550,6 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SYMLINK_METHODDEF #endif /* !defined(OS_SYMLINK_METHODDEF) */ -#ifndef OS_TIMES_METHODDEF - #define OS_TIMES_METHODDEF -#endif /* !defined(OS_TIMES_METHODDEF) */ - #ifndef OS_TIMERFD_CREATE_METHODDEF #define OS_TIMERFD_CREATE_METHODDEF #endif /* !defined(OS_TIMERFD_CREATE_METHODDEF) */ @@ -12414,4 +12833,8 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=7c3058135ed49d20 input=a9049054013a1b77]*/ + +#ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF +#endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ +/*[clinic end generated code: output=31dc3bb3cba924d0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 0e8aa5e6a9f470..365d99aab1dd22 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -34,7 +34,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("getpwnam", "argument", "str", arg); + PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); goto exit; } name = arg; @@ -71,4 +71,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=211c7a2516899b91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 21b09110d00853..682d8481a2a2f4 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -4,8 +4,56 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, +"SetReparseDeferralEnabled($self, enabled, /)\n" +"--\n" +"\n" +"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0."); + +#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \ + {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled); + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int enabled; + + enabled = PyObject_IsTrue(arg); + if (enabled < 0) { + goto exit; + } + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + +exit: + return return_value; +} + +PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__, +"GetReparseDeferralEnabled($self, /)\n" +"--\n" +"\n" +"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0."); + +#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \ + {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); +} PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" @@ -13,7 +61,7 @@ PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "\n" "Parse XML data.\n" "\n" -"`isfinal\' should be true at end of input."); +"\'isfinal\' should be true at end of input."); #define PYEXPAT_XMLPARSER_PARSE_METHODDEF \ {"Parse", _PyCFunction_CAST(pyexpat_xmlparser_Parse), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_Parse__doc__}, @@ -497,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=ac398d94833e802d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9f1e9a7192d29976 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index e5a784a5f79e5a..1c616d60f097cc 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(readline_parse_and_bind__doc__, "parse_and_bind($module, string, /)\n" "--\n" @@ -682,4 +684,4 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=790ed2ba01babb60 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=358ab465285c7949 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index 0ac272af25d154..9eda7de27532a1 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -66,7 +66,7 @@ PyDoc_STRVAR(resource_setrlimit__doc__, "\n"); #define RESOURCE_SETRLIMIT_METHODDEF \ - {"setrlimit", _PyCFunction_CAST(resource_setrlimit), METH_FASTCALL, resource_setrlimit__doc__}, + {"setrlimit", (PyCFunction)(void(*)(void))resource_setrlimit, METH_FASTCALL, resource_setrlimit__doc__}, static PyObject * resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits); @@ -78,7 +78,8 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits; - if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "setrlimit expected 2 arguments, got %zd", nargs); goto exit; } resource = PyLong_AsInt(args[0]); @@ -100,7 +101,7 @@ PyDoc_STRVAR(resource_prlimit__doc__, "\n"); #define RESOURCE_PRLIMIT_METHODDEF \ - {"prlimit", _PyCFunction_CAST(resource_prlimit), METH_FASTCALL, resource_prlimit__doc__}, + {"prlimit", (PyCFunction)(void(*)(void))resource_prlimit, METH_FASTCALL, resource_prlimit__doc__}, static PyObject * resource_prlimit_impl(PyObject *module, pid_t pid, int resource, @@ -114,7 +115,12 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits = Py_None; - if (!_PyArg_CheckPositional("prlimit", nargs, 2, 3)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "prlimit expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "prlimit expected at most 3 arguments, got %zd", nargs); goto exit; } pid = PyLong_AsPid(args[0]); @@ -172,4 +178,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=2376b9a3f03777aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e45883ace510414a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 282196b8f88973..0ccbf63b688f1b 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -6,8 +6,9 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(select_select__doc__, "select($module, rlist, wlist, xlist, timeout=None, /)\n" @@ -99,7 +100,8 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -109,7 +111,9 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -147,13 +151,16 @@ select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("modify", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedShort_Converter(args[1], &eventmask)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -181,10 +188,13 @@ select_poll_unregister(pollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -227,7 +237,9 @@ select_poll_poll(pollObject *self, PyObject *const *args, Py_ssize_t nargs) } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -267,7 +279,8 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -277,7 +290,9 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -317,7 +332,8 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("modify", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -327,7 +343,9 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -355,10 +373,13 @@ select_devpoll_unregister(devpollObject *self, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -401,7 +422,9 @@ select_devpoll_poll(devpollObject *self, PyObject *const *args, Py_ssize_t nargs } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -428,7 +451,13 @@ select_devpoll_close_impl(devpollObject *self); static PyObject * select_devpoll_close(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -450,7 +479,13 @@ select_devpoll_fileno_impl(devpollObject *self); static PyObject * select_devpoll_fileno(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_fileno_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -609,7 +644,13 @@ select_epoll_close_impl(pyEpoll_Object *self); static PyObject * select_epoll_close(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_epoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_EPOLL) */ @@ -729,7 +770,8 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -805,7 +847,8 @@ select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t narg if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); @@ -873,7 +916,8 @@ select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = select_epoll_unregister_impl(self, fd); @@ -1099,7 +1143,13 @@ select_kqueue_close_impl(kqueue_queue_Object *self); static PyObject * select_kqueue_close(kqueue_queue_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_kqueue_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_kqueue_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_KQUEUE) */ @@ -1310,4 +1360,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=35cd4aa7b6802838 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f31e724f492225b1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index fa4ec8bb51e787..b89c7e505c788e 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(SHA1Type_copy__doc__, "copy($self, /)\n" @@ -22,7 +23,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls); static PyObject * SHA1Type_copy(SHA1object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -147,4 +148,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=7efbe42154a7a7f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index 9fb8bf7de876fa..cf4b88d52856b8 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(SHA256Type_copy__doc__, "copy($self, /)\n" @@ -22,7 +23,7 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); static PyObject * SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -44,7 +45,7 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); static PyObject * SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -436,4 +437,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=1242ccc50bcefe98 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index fd65462d62c195..738e9589503900 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" @@ -193,4 +194,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=0888aed37ef39984 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a8e76a880d1421ec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 5e8957a4be780d..d074cc30d1e746 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -2,6 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(signal_default_int_handler__doc__, "default_int_handler($module, signalnum, frame, /)\n" "--\n" @@ -274,6 +280,77 @@ signal_siginterrupt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_SIGINTERRUPT) */ +PyDoc_STRVAR(signal_set_wakeup_fd__doc__, +"set_wakeup_fd($module, fd, /, *, warn_on_full_buffer=True)\n" +"--\n" +"\n" +"Sets the fd to be written to (with the signal number) when a signal comes in.\n" +"\n" +"A library can use this to wakeup select or poll.\n" +"The previous fd or -1 is returned.\n" +"\n" +"The fd must be non-blocking."); + +#define SIGNAL_SET_WAKEUP_FD_METHODDEF \ + {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_FASTCALL|METH_KEYWORDS, signal_set_wakeup_fd__doc__}, + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer); + +static PyObject * +signal_set_wakeup_fd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(warn_on_full_buffer), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "warn_on_full_buffer", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_wakeup_fd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *fdobj; + int warn_on_full_buffer = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + fdobj = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + warn_on_full_buffer = PyObject_IsTrue(args[1]); + if (warn_on_full_buffer < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = signal_set_wakeup_fd_impl(module, fdobj, warn_on_full_buffer); + +exit: + return return_value; +} + #if defined(HAVE_SETITIMER) PyDoc_STRVAR(signal_setitimer__doc__, @@ -699,4 +776,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=ef4c2ad1a2443063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1c11c1b6f12f26be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index c62050160f1117..7b0a3f8d4b1cc6 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,6 +6,34 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_socket_socket_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"close()\n" +"\n" +"Close the socket. It cannot be used after this call."); + +#define _SOCKET_SOCKET_CLOSE_METHODDEF \ + {"close", (PyCFunction)_socket_socket_close, METH_NOARGS, _socket_socket_close__doc__}, + +static PyObject * +_socket_socket_close_impl(PySocketSockObject *s); + +static PyObject * +_socket_socket_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(s); + return_value = _socket_socket_close_impl(s); + Py_END_CRITICAL_SECTION(); + + return return_value; +} static int sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, @@ -90,4 +118,172 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=1b68ae94d6cbeeb1 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_socket_socket_ntohs__doc__, +"ntohs($self, x, /)\n" +"--\n" +"\n" +"Convert a 16-bit unsigned integer from network to host byte order."); + +#define _SOCKET_SOCKET_NTOHS_METHODDEF \ + {"ntohs", (PyCFunction)_socket_socket_ntohs, METH_O, _socket_socket_ntohs__doc__}, + +static PyObject * +_socket_socket_ntohs_impl(PySocketSockObject *self, int x); + +static PyObject * +_socket_socket_ntohs(PySocketSockObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int x; + + x = PyLong_AsInt(arg); + if (x == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _socket_socket_ntohs_impl(self, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_socket_socket_htons__doc__, +"htons($self, x, /)\n" +"--\n" +"\n" +"Convert a 16-bit unsigned integer from host to network byte order."); + +#define _SOCKET_SOCKET_HTONS_METHODDEF \ + {"htons", (PyCFunction)_socket_socket_htons, METH_O, _socket_socket_htons__doc__}, + +static PyObject * +_socket_socket_htons_impl(PySocketSockObject *self, int x); + +static PyObject * +_socket_socket_htons(PySocketSockObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int x; + + x = PyLong_AsInt(arg); + if (x == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _socket_socket_htons_impl(self, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_socket_socket_inet_aton__doc__, +"inet_aton($self, ip_addr, /)\n" +"--\n" +"\n" +"Convert an IP address in string format (123.45.67.89) to the 32-bit packed binary format used in low-level network functions."); + +#define _SOCKET_SOCKET_INET_ATON_METHODDEF \ + {"inet_aton", (PyCFunction)_socket_socket_inet_aton, METH_O, _socket_socket_inet_aton__doc__}, + +static PyObject * +_socket_socket_inet_aton_impl(PySocketSockObject *self, const char *ip_addr); + +static PyObject * +_socket_socket_inet_aton(PySocketSockObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *ip_addr; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("inet_aton", "argument", "str", arg); + goto exit; + } + Py_ssize_t ip_addr_length; + ip_addr = PyUnicode_AsUTF8AndSize(arg, &ip_addr_length); + if (ip_addr == NULL) { + goto exit; + } + if (strlen(ip_addr) != (size_t)ip_addr_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = _socket_socket_inet_aton_impl(self, ip_addr); + +exit: + return return_value; +} + +#if defined(HAVE_INET_NTOA) + +PyDoc_STRVAR(_socket_socket_inet_ntoa__doc__, +"inet_ntoa($self, packed_ip, /)\n" +"--\n" +"\n" +"Convert an IP address from 32-bit packed binary format to string format."); + +#define _SOCKET_SOCKET_INET_NTOA_METHODDEF \ + {"inet_ntoa", (PyCFunction)_socket_socket_inet_ntoa, METH_O, _socket_socket_inet_ntoa__doc__}, + +static PyObject * +_socket_socket_inet_ntoa_impl(PySocketSockObject *self, Py_buffer *packed_ip); + +static PyObject * +_socket_socket_inet_ntoa(PySocketSockObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer packed_ip = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &packed_ip, PyBUF_SIMPLE) != 0) { + goto exit; + } + return_value = _socket_socket_inet_ntoa_impl(self, &packed_ip); + +exit: + /* Cleanup for packed_ip */ + if (packed_ip.obj) { + PyBuffer_Release(&packed_ip); + } + + return return_value; +} + +#endif /* defined(HAVE_INET_NTOA) */ + +#if (defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS)) + +PyDoc_STRVAR(_socket_socket_if_nametoindex__doc__, +"if_nametoindex($self, oname, /)\n" +"--\n" +"\n" +"Returns the interface index corresponding to the interface name if_name."); + +#define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF \ + {"if_nametoindex", (PyCFunction)_socket_socket_if_nametoindex, METH_O, _socket_socket_if_nametoindex__doc__}, + +static PyObject * +_socket_socket_if_nametoindex_impl(PySocketSockObject *self, PyObject *oname); + +static PyObject * +_socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *oname; + + if (!PyUnicode_FSConverter(arg, &oname)) { + goto exit; + } + return_value = _socket_socket_if_nametoindex_impl(self, oname); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS)) */ + +#ifndef _SOCKET_SOCKET_INET_NTOA_METHODDEF + #define _SOCKET_SOCKET_INET_NTOA_METHODDEF +#endif /* !defined(_SOCKET_SOCKET_INET_NTOA_METHODDEF) */ + +#ifndef _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF + #define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF +#endif /* !defined(_SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF) */ +/*[clinic end generated code: output=6037e47b012911c5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 2cd08f81782007..2ecd3afc00d2be 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_symtable_symtable__doc__, "symtable($module, source, filename, startstr, /)\n" "--\n" @@ -48,4 +50,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3f7ccf535d750238 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=931964a76a72f850 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index f0c6f2a871d042..77cf24ea5ba9ca 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -6,6 +6,8 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(syslog_openlog__doc__, "openlog($module, /, ident=, logoption=0,\n" @@ -87,7 +89,9 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_openlog_impl(module, ident, logopt, facility); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -128,7 +132,9 @@ syslog_syslog(PyObject *module, PyObject *args) PyErr_SetString(PyExc_TypeError, "syslog.syslog requires 1 to 2 arguments"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_syslog_impl(module, group_left_1, priority, message); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -149,7 +155,13 @@ syslog_closelog_impl(PyObject *module); static PyObject * syslog_closelog(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return syslog_closelog_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = syslog_closelog_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(syslog_setlogmask__doc__, @@ -250,4 +262,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=68942dad7fb3872e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d25899bd31969d3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 78828e57d73077..83f5a4f6e9f882 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() - PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" "--\n" @@ -29,7 +27,8 @@ termios_tcgetattr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetattr_impl(module, fd); @@ -52,7 +51,7 @@ PyDoc_STRVAR(termios_tcsetattr__doc__, "queued output and discarding all queued input."); #define TERMIOS_TCSETATTR_METHODDEF \ - {"tcsetattr", _PyCFunction_CAST(termios_tcsetattr), METH_FASTCALL, termios_tcsetattr__doc__}, + {"tcsetattr", (PyCFunction)(void(*)(void))termios_tcsetattr, METH_FASTCALL, termios_tcsetattr__doc__}, static PyObject * termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term); @@ -65,10 +64,12 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int when; PyObject *term; - if (!_PyArg_CheckPositional("tcsetattr", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "tcsetattr expected 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } when = PyLong_AsInt(args[1]); @@ -92,7 +93,7 @@ PyDoc_STRVAR(termios_tcsendbreak__doc__, "has a system dependent meaning."); #define TERMIOS_TCSENDBREAK_METHODDEF \ - {"tcsendbreak", _PyCFunction_CAST(termios_tcsendbreak), METH_FASTCALL, termios_tcsendbreak__doc__}, + {"tcsendbreak", (PyCFunction)(void(*)(void))termios_tcsendbreak, METH_FASTCALL, termios_tcsendbreak__doc__}, static PyObject * termios_tcsendbreak_impl(PyObject *module, int fd, int duration); @@ -104,10 +105,12 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int duration; - if (!_PyArg_CheckPositional("tcsendbreak", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsendbreak expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } duration = PyLong_AsInt(args[1]); @@ -138,7 +141,8 @@ termios_tcdrain(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcdrain_impl(module, fd); @@ -158,7 +162,7 @@ PyDoc_STRVAR(termios_tcflush__doc__, "both queues."); #define TERMIOS_TCFLUSH_METHODDEF \ - {"tcflush", _PyCFunction_CAST(termios_tcflush), METH_FASTCALL, termios_tcflush__doc__}, + {"tcflush", (PyCFunction)(void(*)(void))termios_tcflush, METH_FASTCALL, termios_tcflush__doc__}, static PyObject * termios_tcflush_impl(PyObject *module, int fd, int queue); @@ -170,10 +174,12 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int queue; - if (!_PyArg_CheckPositional("tcflush", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflush expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } queue = PyLong_AsInt(args[1]); @@ -197,7 +203,7 @@ PyDoc_STRVAR(termios_tcflow__doc__, "or termios.TCION to restart input."); #define TERMIOS_TCFLOW_METHODDEF \ - {"tcflow", _PyCFunction_CAST(termios_tcflow), METH_FASTCALL, termios_tcflow__doc__}, + {"tcflow", (PyCFunction)(void(*)(void))termios_tcflow, METH_FASTCALL, termios_tcflow__doc__}, static PyObject * termios_tcflow_impl(PyObject *module, int fd, int action); @@ -209,10 +215,12 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int action; - if (!_PyArg_CheckPositional("tcflow", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflow expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } action = PyLong_AsInt(args[1]); @@ -245,7 +253,8 @@ termios_tcgetwinsize(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetwinsize_impl(module, fd); @@ -264,7 +273,7 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ - {"tcsetwinsize", _PyCFunction_CAST(termios_tcsetwinsize), METH_FASTCALL, termios_tcsetwinsize__doc__}, + {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz); @@ -276,10 +285,12 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; PyObject *winsz; - if (!_PyArg_CheckPositional("tcsetwinsize", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsetwinsize expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } winsz = args[1]; @@ -288,4 +299,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=daecc8ebc8fe29f5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/timemodule.c.h b/Modules/clinic/timemodule.c.h new file mode 100644 index 00000000000000..bbc0748f9a9c0d --- /dev/null +++ b/Modules/clinic/timemodule.c.h @@ -0,0 +1,74 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(HAVE_CLOCK_GETTIME) + +PyDoc_STRVAR(time_clock_gettime__doc__, +"clock_gettime($module, clk_id, /)\n" +"--\n" +"\n" +"Return the time of the specified clock clk_id as a float."); + +#define TIME_CLOCK_GETTIME_METHODDEF \ + {"clock_gettime", (PyCFunction)time_clock_gettime, METH_O, time_clock_gettime__doc__}, + +static PyObject * +time_clock_gettime_impl(PyObject *module, clockid_t clk_id); + +static PyObject * +time_clock_gettime(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + clockid_t clk_id; + + if (!time_clockid_converter(arg, &clk_id)) { + goto exit; + } + return_value = time_clock_gettime_impl(module, clk_id); + +exit: + return return_value; +} + +#endif /* defined(HAVE_CLOCK_GETTIME) */ + +#if defined(HAVE_CLOCK_GETTIME) + +PyDoc_STRVAR(time_clock_gettime_ns__doc__, +"clock_gettime_ns($module, clk_id, /)\n" +"--\n" +"\n" +"Return the time of the specified clock clk_id as nanoseconds (int)."); + +#define TIME_CLOCK_GETTIME_NS_METHODDEF \ + {"clock_gettime_ns", (PyCFunction)time_clock_gettime_ns, METH_O, time_clock_gettime_ns__doc__}, + +static PyObject * +time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id); + +static PyObject * +time_clock_gettime_ns(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + clockid_t clk_id; + + if (!time_clockid_converter(arg, &clk_id)) { + goto exit; + } + return_value = time_clock_gettime_ns_impl(module, clk_id); + +exit: + return return_value; +} + +#endif /* defined(HAVE_CLOCK_GETTIME) */ + +#ifndef TIME_CLOCK_GETTIME_METHODDEF + #define TIME_CLOCK_GETTIME_METHODDEF +#endif /* !defined(TIME_CLOCK_GETTIME_METHODDEF) */ + +#ifndef TIME_CLOCK_GETTIME_NS_METHODDEF + #define TIME_CLOCK_GETTIME_NS_METHODDEF +#endif /* !defined(TIME_CLOCK_GETTIME_NS_METHODDEF) */ +/*[clinic end generated code: output=b589a2132aa9df47 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index 74362c6a6857b7..345440eeee89a6 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(unicodedata_UCD_decimal__doc__, "decimal($self, chr, default=, /)\n" "--\n" @@ -34,7 +36,10 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("decimal", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "decimal(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -80,7 +85,10 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("digit", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "digit(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -127,7 +135,10 @@ unicodedata_UCD_numeric(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("numeric", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "numeric(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -165,7 +176,10 @@ unicodedata_UCD_category(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("category", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "category(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -200,7 +214,10 @@ unicodedata_UCD_bidirectional(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("bidirectional", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "bidirectional(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -236,7 +253,10 @@ unicodedata_UCD_combining(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("combining", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "combining(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -277,7 +297,10 @@ unicodedata_UCD_mirrored(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("mirrored", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "mirrored(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -314,7 +337,10 @@ unicodedata_UCD_east_asian_width(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("east_asian_width", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "east_asian_width(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -349,7 +375,10 @@ unicodedata_UCD_decomposition(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("decomposition", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "decomposition(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -471,7 +500,10 @@ unicodedata_UCD_name(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("name", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "name(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -517,4 +549,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=ca10ded25747d182 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a59d430cee41058 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 895215307974f7..7ff3edf5a557f8 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(zlib_compress__doc__, "compress($module, data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS)\n" @@ -69,10 +70,6 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -163,10 +160,6 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -333,10 +326,6 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[5], &zdict, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&zdict, 'C')) { - _PyArg_BadArgument("compressobj", "argument 'zdict'", "contiguous buffer", args[5]); - goto exit; - } skip_optional_pos: return_value = zlib_compressobj_impl(module, level, method, wbits, memLevel, strategy, &zdict); @@ -472,10 +461,6 @@ zlib_Compress_compress(compobject *self, PyTypeObject *cls, PyObject *const *arg if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = zlib_Compress_compress_impl(self, cls, &data); exit: @@ -552,10 +537,6 @@ zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -656,7 +637,7 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -681,7 +662,7 @@ zlib_Compress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Compress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -754,7 +735,7 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } @@ -779,7 +760,7 @@ zlib_Decompress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * zlib_Decompress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs) { + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } @@ -964,10 +945,6 @@ zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1025,10 +1002,6 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("adler32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1079,10 +1052,6 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1129,4 +1098,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=6d90c72ba2dd04c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 57bc55632be485..bf86a211bcb188 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -117,7 +117,7 @@ enum special_types { static enum special_types special_type(double d) { - if (Py_IS_FINITE(d)) { + if (isfinite(d)) { if (d != 0) { if (copysign(1., d) == 1.) return ST_POS; @@ -131,7 +131,7 @@ special_type(double d) return ST_NZERO; } } - if (Py_IS_NAN(d)) + if (isnan(d)) return ST_NAN; if (copysign(1., d) == 1.) return ST_PINF; @@ -139,11 +139,11 @@ special_type(double d) return ST_NINF; } -#define SPECIAL_VALUE(z, table) \ - if (!Py_IS_FINITE((z).real) || !Py_IS_FINITE((z).imag)) { \ - errno = 0; \ - return table[special_type((z).real)] \ - [special_type((z).imag)]; \ +#define SPECIAL_VALUE(z, table) \ + if (!isfinite((z).real) || !isfinite((z).imag)) { \ + errno = 0; \ + return table[special_type((z).real)] \ + [special_type((z).imag)]; \ } #define P Py_MATH_PI @@ -329,10 +329,10 @@ cmath_atan_impl(PyObject *module, Py_complex z) static double c_atan2(Py_complex z) { - if (Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)) + if (isnan(z.real) || isnan(z.imag)) return Py_NAN; - if (Py_IS_INFINITY(z.imag)) { - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.imag)) { + if (isinf(z.real)) { if (copysign(1., z.real) == 1.) /* atan2(+-inf, +inf) == +-pi/4 */ return copysign(0.25*Py_MATH_PI, z.imag); @@ -343,7 +343,7 @@ c_atan2(Py_complex z) /* atan2(+-inf, x) == +-pi/2 for finite x */ return copysign(0.5*Py_MATH_PI, z.imag); } - if (Py_IS_INFINITY(z.real) || z.imag == 0.) { + if (isinf(z.real) || z.imag == 0.) { if (copysign(1., z.real) == 1.) /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ return copysign(0., z.imag); @@ -448,8 +448,8 @@ cmath_cosh_impl(PyObject *module, Py_complex z) double x_minus_one; /* special treatment for cosh(+/-inf + iy) if y is not a NaN */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) && + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -466,7 +466,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -484,7 +484,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * sinh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -509,8 +509,8 @@ cmath_exp_impl(PyObject *module, Py_complex z) Py_complex r; double l; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -527,9 +527,9 @@ cmath_exp_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN and not -infinity */ - if (Py_IS_INFINITY(z.imag) && - (Py_IS_FINITE(z.real) || - (Py_IS_INFINITY(z.real) && z.real > 0))) + if (isinf(z.imag) && + (isfinite(z.real) || + (isinf(z.real) && z.real > 0))) errno = EDOM; else errno = 0; @@ -546,7 +546,7 @@ cmath_exp_impl(PyObject *module, Py_complex z) r.imag = l*sin(z.imag); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -686,8 +686,8 @@ cmath_sinh_impl(PyObject *module, Py_complex z) /* special treatment for sinh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -704,7 +704,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -720,7 +720,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * cosh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -856,8 +856,8 @@ cmath_tanh_impl(PyObject *module, Py_complex z) /* special treatment for tanh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = 1.0; @@ -876,7 +876,7 @@ cmath_tanh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if z.imag is +/-infinity and z.real is finite */ - if (Py_IS_INFINITY(z.imag) && Py_IS_FINITE(z.real)) + if (isinf(z.imag) && isfinite(z.real)) errno = EDOM; else errno = 0; @@ -1030,11 +1030,11 @@ cmath_rect_impl(PyObject *module, double r, double phi) errno = 0; /* deal with special values */ - if (!Py_IS_FINITE(r) || !Py_IS_FINITE(phi)) { + if (!isfinite(r) || !isfinite(phi)) { /* if r is +/-infinity and phi is finite but nonzero then result is (+-INF +-INF i), but we need to compute cos(phi) and sin(phi) to figure out the signs. */ - if (Py_IS_INFINITY(r) && (Py_IS_FINITE(phi) + if (isinf(r) && (isfinite(phi) && (phi != 0.))) { if (r > 0) { z.real = copysign(INF, cos(phi)); @@ -1051,7 +1051,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) } /* need to set errno = EDOM if r is a nonzero number and phi is infinite */ - if (r != 0. && !Py_IS_NAN(r) && Py_IS_INFINITY(phi)) + if (r != 0. && !isnan(r) && isinf(phi)) errno = EDOM; else errno = 0; @@ -1085,7 +1085,7 @@ static PyObject * cmath_isfinite_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=ac76611e2c774a36 input=848e7ee701895815]*/ { - return PyBool_FromLong(Py_IS_FINITE(z.real) && Py_IS_FINITE(z.imag)); + return PyBool_FromLong(isfinite(z.real) && isfinite(z.imag)); } /*[clinic input] @@ -1098,7 +1098,7 @@ static PyObject * cmath_isnan_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=e7abf6e0b28beab7 input=71799f5d284c9baf]*/ { - return PyBool_FromLong(Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)); + return PyBool_FromLong(isnan(z.real) || isnan(z.imag)); } /*[clinic input] @@ -1111,8 +1111,7 @@ static PyObject * cmath_isinf_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=502a75a79c773469 input=363df155c7181329]*/ { - return PyBool_FromLong(Py_IS_INFINITY(z.real) || - Py_IS_INFINITY(z.imag)); + return PyBool_FromLong(isinf(z.real) || isinf(z.imag)); } /*[clinic input] @@ -1167,8 +1166,7 @@ cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, above. */ - if (Py_IS_INFINITY(a.real) || Py_IS_INFINITY(a.imag) || - Py_IS_INFINITY(b.real) || Py_IS_INFINITY(b.imag)) { + if (isinf(a.real) || isinf(a.imag) || isinf(b.real) || isinf(b.imag)) { return 0; } @@ -1363,6 +1361,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 301ad8313bc512..3f96f2f846d612 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,41 +1,48 @@ - /* Errno module */ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" +#include // EPIPE /* Windows socket errors (WSA*) */ #ifdef MS_WINDOWS -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include -/* The following constants were added to errno.h in VS2010 but have - preferred WSA equivalents. */ -#undef EADDRINUSE -#undef EADDRNOTAVAIL -#undef EAFNOSUPPORT -#undef EALREADY -#undef ECONNABORTED -#undef ECONNREFUSED -#undef ECONNRESET -#undef EDESTADDRREQ -#undef EHOSTUNREACH -#undef EINPROGRESS -#undef EISCONN -#undef ELOOP -#undef EMSGSIZE -#undef ENETDOWN -#undef ENETRESET -#undef ENETUNREACH -#undef ENOBUFS -#undef ENOPROTOOPT -#undef ENOTCONN -#undef ENOTSOCK -#undef EOPNOTSUPP -#undef EPROTONOSUPPORT -#undef EPROTOTYPE -#undef ETIMEDOUT -#undef EWOULDBLOCK +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include + + // The following constants were added to errno.h in VS2010 but have + // preferred WSA equivalents. +# undef EADDRINUSE +# undef EADDRNOTAVAIL +# undef EAFNOSUPPORT +# undef EALREADY +# undef ECONNABORTED +# undef ECONNREFUSED +# undef ECONNRESET +# undef EDESTADDRREQ +# undef EHOSTUNREACH +# undef EINPROGRESS +# undef EISCONN +# undef ELOOP +# undef EMSGSIZE +# undef ENETDOWN +# undef ENETRESET +# undef ENETUNREACH +# undef ENOBUFS +# undef ENOPROTOOPT +# undef ENOTCONN +# undef ENOTSOCK +# undef EOPNOTSUPP +# undef EPROTONOSUPPORT +# undef EPROTOTYPE +# undef ETIMEDOUT +# undef EWOULDBLOCK #endif /* @@ -944,6 +951,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 1c83563cbf68e7..c2770be3897e58 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,11 +11,14 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Cristian Rodríguez Copyright (c) 2016 Thomas Beutlich Copyright (c) 2017 Rhodri James Copyright (c) 2022 Thijs Schreijer + Copyright (c) 2023 Hanno Böck + Copyright (c) 2023 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -269,7 +272,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, const XML_Memory_Handling_Suite *memsuite, const XML_Char *namespaceSeparator); -/* Prepare a parser object to be re-used. This is particularly +/* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, such as when a large number of small documnents need to be parsed. All handlers are cleared from the parser, except for the @@ -951,7 +954,7 @@ XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); -/* If XML_CONTEXT_BYTES is defined, returns the input buffer, sets +/* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer of the current parse position, and sets the integer pointed to by size to the size of this buffer (the number of input bytes). Otherwise @@ -1025,7 +1028,9 @@ enum XML_FeatureEnum { XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, - XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + /* Added in Expat 2.6.0. */ + XML_FEATURE_GE /* Additional features must be added to the end of this enum. */ }; @@ -1038,24 +1043,30 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#ifdef XML_DTD -/* Added in Expat 2.4.0. */ +#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor); -/* Added in Expat 2.4.0. */ +/* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); #endif +/* Added in Expat 2.6.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); + /* Expat follows the semantic versioning convention. - See http://semver.org. + See https://semver.org */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 5 -#define XML_MICRO_VERSION 0 +#define XML_MINOR_VERSION 6 +#define XML_MICRO_VERSION 2 #ifdef __cplusplus } diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index 6671f7b689ba97..e7d9499d9078d9 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -16,6 +16,7 @@ #define XML_NS 1 #define XML_DTD 1 +#define XML_GE 1 #define XML_CONTEXT_BYTES 1024 // bpo-30947: Python uses best available entropy sources to diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index e09f533b23c9df..167ec36804a43b 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,9 +28,11 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -154,12 +156,21 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_DTD) +#if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); #endif +extern +#if ! defined(XML_TESTING) + const +#endif + XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c +#if defined(XML_TESTING) +extern unsigned int g_bytesScanned; // used for testing only +#endif + #ifdef __cplusplus } #endif diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index d45d9b6c457159..8ee03ef0792815 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -108,6 +108,7 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 303283ad2de98d..a1ed99e687bd6e 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -106,7 +106,7 @@ * if this code is included and compiled as C++; related GCC warning is: * warning: use of C++11 long long integer constant [-Wlong-long] */ -#define _SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) +#define SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) @@ -190,10 +190,10 @@ sip_round(struct siphash *H, const int rounds) { static struct siphash * sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; - H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; - H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; - H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; + H->v0 = SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 2ecd61b5b94820..05805514ec7fa2 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -9,7 +9,8 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Greg Stein Copyright (c) 2005 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2023 Sebastian Pipping + Copyright (c) 2023 Orgad Shaneh Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,7 +36,9 @@ #ifndef WINCONFIG_H #define WINCONFIG_H -#define WIN32_LEAN_AND_MEAN +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif #include #undef WIN32_LEAN_AND_MEAN diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b6c2eca97567ba..2951fec70c56cb 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 5ab094ffadd6edfc94c3eee53af44a86951f9f1f0933ada3114bbce2bfb02c99 (2.5.0+) +/* 2a14271ad4d35e82bde8ba210b4edb7998794bcbae54deab114046a300f9639a (2.6.2+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie Copyright (c) 2016 Eric Rahm - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Gaurav Copyright (c) 2016 Thomas Beutlich Copyright (c) 2016 Gustavo Grieco @@ -32,10 +32,13 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2019-2020 Ben Wagner Copyright (c) 2019 Vadim Zeitlin - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Copyright (c) 2022 Samanta Navarro Copyright (c) 2022 Jeffrey Walton Copyright (c) 2022 Jann Horn + Copyright (c) 2022 Sean McBride + Copyright (c) 2023 Owain Davies + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -60,10 +63,25 @@ #define XML_BUILDING_EXPAT 1 -#include +#include "expat_config.h" -#if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ +#if ! defined(XML_GE) || (1 - XML_GE - 1 == 2) || (XML_GE < 0) || (XML_GE > 1) +# error XML_GE (for general entities) must be defined, non-empty, either 1 or 0 (0 to disable, 1 to enable; 1 is a common default) +#endif + +#if defined(XML_DTD) && XML_GE == 0 +# error Either undefine XML_DTD or define XML_GE to 1. +#endif + +#if ! defined(XML_CONTEXT_BYTES) || (1 - XML_CONTEXT_BYTES - 1 == 2) \ + || (XML_CONTEXT_BYTES + 0 < 0) +# error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) +#endif + +#if defined(HAVE_SYSCALL_GETRANDOM) +# if ! defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +# endif #endif #ifdef _WIN32 @@ -73,6 +91,7 @@ # endif #endif +#include #include #include /* memset(), memcpy() */ #include @@ -131,8 +150,8 @@ Your options include: \ * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ - * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ @@ -191,11 +210,13 @@ typedef char ICHAR; #endif /* Round up n to be a multiple of sz, where sz is a power of 2. */ -#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1)) +#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) +#define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) + #include "internal.h" #include "xmltok.h" #include "xmlrole.h" @@ -227,7 +248,7 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); it odd, since odd numbers are always relative prime to a power of 2. */ #define SECOND_HASH(hash, mask, power) \ - ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2)) + ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2)) #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) @@ -279,7 +300,7 @@ typedef struct { XML_Parse()/XML_ParseBuffer(), the buffer is re-allocated to contain the 'raw' name as well. - A parser re-uses these structures, maintaining a list of allocated + A parser reuses these structures, maintaining a list of allocated TAG objects in a free list. */ typedef struct tag { @@ -408,12 +429,12 @@ enum XML_Account { XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ }; -#ifdef XML_DTD +#if XML_GE == 1 typedef unsigned long long XmlBigCount; typedef struct accounting { XmlBigCount countBytesDirect; XmlBigCount countBytesIndirect; - int debugLevel; + unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 unsigned long long activationThresholdBytes; } ACCOUNTING; @@ -422,9 +443,9 @@ typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; unsigned int maximumDepthSeen; - int debugLevel; + unsigned long debugLevel; } ENTITY_STATS; -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -464,41 +485,47 @@ static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, XML_Bool haveMore, enum XML_Account account); -static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore, enum XML_Account account); #ifdef XML_DTD -static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, +static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ static void freeBindings(XML_Parser parser, BINDING *bindings); -static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, - const char *s, TAG_NAME *tagNamePtr, +static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, + const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); -static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, - XML_Bool isId, const XML_Char *dfltValue, - XML_Parser parser); -static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, + XML_Bool isCdata, XML_Bool isId, + const XML_Char *value, XML_Parser parser); +static enum XML_Error storeAttributeValue(XML_Parser parser, + const ENCODING *enc, XML_Bool isCdata, + const char *ptr, const char *end, + STRING_POOL *pool, enum XML_Account account); -static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, - XML_Bool isCdata, const char *, - const char *, STRING_POOL *, +static enum XML_Error appendAttributeValue(XML_Parser parser, + const ENCODING *enc, + XML_Bool isCdata, const char *ptr, + const char *end, STRING_POOL *pool, enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); -static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); +static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType); +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, enum XML_Account account); +#else +static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); +#endif static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -518,21 +545,22 @@ static void dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_Handling_Suite *ms); -static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *, STRING_POOL *, - const HASH_TABLE *); +static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, + STRING_POOL *newPool, const HASH_TABLE *oldTable); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *, +static void FASTCALL hashTableInit(HASH_TABLE *table, const XML_Memory_Handling_Suite *ms); -static void FASTCALL hashTableClear(HASH_TABLE *); -static void FASTCALL hashTableDestroy(HASH_TABLE *); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *, const HASH_TABLE *); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *); +static void FASTCALL hashTableClear(HASH_TABLE *table); +static void FASTCALL hashTableDestroy(HASH_TABLE *table); +static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); -static void FASTCALL poolInit(STRING_POOL *, +static void FASTCALL poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms); -static void FASTCALL poolClear(STRING_POOL *); -static void FASTCALL poolDestroy(STRING_POOL *); +static void FASTCALL poolClear(STRING_POOL *pool); +static void FASTCALL poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, @@ -562,7 +590,7 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser); static void accountingReportStats(XML_Parser originParser, const char *epilog); static void accountingOnAbort(XML_Parser originParser); @@ -585,13 +613,12 @@ static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); #define poolStart(pool) ((pool)->start) -#define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) @@ -602,21 +629,41 @@ static unsigned long getDebugLevel(const char *variableName, ? 0 \ : ((*((pool)->ptr)++ = c), 1)) +#if ! defined(XML_TESTING) +const +#endif + XML_Bool g_reparseDeferralEnabledDefault + = XML_TRUE; // write ONLY in runtests.c +#if defined(XML_TESTING) +unsigned int g_bytesScanned = 0; // used for testing only +#endif + struct XML_ParserStruct { /* The first member must be m_userData so that the XML_GetUserData macro works. */ void *m_userData; void *m_handlerArg; - char *m_buffer; + + // How the four parse buffer pointers below relate in time and space: + // + // m_buffer <= m_bufferPtr <= m_bufferEnd <= m_bufferLim + // | | | | + // <--parsed-->| | | + // <---parsing--->| | + // <--unoccupied-->| + // <---------total-malloced/realloced-------->| + + char *m_buffer; // malloc/realloc base pointer of parse buffer const XML_Memory_Handling_Suite m_mem; - /* first character to be parsed */ - const char *m_bufferPtr; - /* past last character to be parsed */ - char *m_bufferEnd; - /* allocated end of m_buffer */ - const char *m_bufferLim; + const char *m_bufferPtr; // first character to be parsed + char *m_bufferEnd; // past last character to be parsed + const char *m_bufferLim; // allocated end of m_buffer + XML_Index m_parseEndByteIndex; const char *m_parseEndPtr; + size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ + XML_Bool m_reparseDeferralEnabled; + int m_lastBufferRequestSize; XML_Char *m_dataBuf; XML_Char *m_dataBufEnd; XML_StartElementHandler m_startElementHandler; @@ -703,7 +750,7 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; -#ifdef XML_DTD +#if XML_GE == 1 ACCOUNTING m_accounting; ENTITY_STATS m_entity_stats; #endif @@ -948,6 +995,49 @@ get_hash_secret_salt(XML_Parser parser) { return parser->m_hash_secret_salt; } +static enum XML_Error +callProcessor(XML_Parser parser, const char *start, const char *end, + const char **endPtr) { + const size_t have_now = EXPAT_SAFE_PTR_DIFF(end, start); + + if (parser->m_reparseDeferralEnabled + && ! parser->m_parsingStatus.finalBuffer) { + // Heuristic: don't try to parse a partial token again until the amount of + // available data has increased significantly. + const size_t had_before = parser->m_partialTokenBytesBefore; + // ...but *do* try anyway if we're close to causing a reallocation. + size_t available_buffer + = EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); +#if XML_CONTEXT_BYTES > 0 + available_buffer -= EXPAT_MIN(available_buffer, XML_CONTEXT_BYTES); +#endif + available_buffer + += EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd); + // m_lastBufferRequestSize is never assigned a value < 0, so the cast is ok + const bool enough + = (have_now >= 2 * had_before) + || ((size_t)parser->m_lastBufferRequestSize > available_buffer); + + if (! enough) { + *endPtr = start; // callers may expect this to be set + return XML_ERROR_NONE; + } + } +#if defined(XML_TESTING) + g_bytesScanned += (unsigned)have_now; +#endif + const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); + if (ret == XML_ERROR_NONE) { + // if we consumed nothing, remember what we had on this parse attempt. + if (*endPtr == start) { + parser->m_partialTokenBytesBefore = have_now; + } else { + parser->m_partialTokenBytesBefore = 0; + } + } + return ret; +} + static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ @@ -1129,6 +1219,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_bufferEnd = parser->m_buffer; parser->m_parseEndByteIndex = 0; parser->m_parseEndPtr = NULL; + parser->m_partialTokenBytesBefore = 0; + parser->m_reparseDeferralEnabled = g_reparseDeferralEnabledDefault; + parser->m_lastBufferRequestSize = 0; parser->m_declElementType = NULL; parser->m_declAttributeId = NULL; parser->m_declEntity = NULL; @@ -1163,7 +1256,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { #endif parser->m_hash_secret_salt = 0; -#ifdef XML_DTD +#if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); parser->m_accounting.maximumAmplificationFactor @@ -1298,6 +1391,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ unsigned long oldhash_secret_salt; + XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ if (oldParser == NULL) @@ -1342,6 +1436,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, to worry which hash secrets each table has. */ oldhash_secret_salt = parser->m_hash_secret_salt; + oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD if (! context) @@ -1394,6 +1489,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD parser->m_paramEntityParsing = oldParamEntityParsing; @@ -1848,55 +1944,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } - if (len == 0) { - parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - if (! isFinal) - return XML_STATUS_OK; - parser->m_positionPtr = parser->m_bufferPtr; - parser->m_parseEndPtr = parser->m_bufferEnd; - - /* If data are left over from last buffer, and we now know that these - data are the final chunk of input, then we have to check them again - to detect errors based on that fact. - */ - parser->m_errorCode - = parser->m_processor(parser, parser->m_bufferPtr, - parser->m_parseEndPtr, &parser->m_bufferPtr); - - if (parser->m_errorCode == XML_ERROR_NONE) { - switch (parser->m_parsingStatus.parsing) { - case XML_SUSPENDED: - /* It is hard to be certain, but it seems that this case - * cannot occur. This code is cleaning up a previous parse - * with no new data (since len == 0). Changing the parsing - * state requires getting to execute a handler function, and - * there doesn't seem to be an opportunity for that while in - * this circumstance. - * - * Given the uncertainty, we retain the code but exclude it - * from coverage tests. - * - * LCOV_EXCL_START - */ - XmlUpdatePosition(parser->m_encoding, parser->m_positionPtr, - parser->m_bufferPtr, &parser->m_position); - parser->m_positionPtr = parser->m_bufferPtr; - return XML_STATUS_SUSPENDED; - /* LCOV_EXCL_STOP */ - case XML_INITIALIZED: - case XML_PARSING: - parser->m_parsingStatus.parsing = XML_FINISHED; - /* fall through */ - default: - return XML_STATUS_OK; - } - } - parser->m_eventEndPtr = parser->m_eventPtr; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } -#ifndef XML_CONTEXT_BYTES - else if (parser->m_bufferPtr == parser->m_bufferEnd) { +#if XML_CONTEXT_BYTES == 0 + if (parser->m_bufferPtr == parser->m_bufferEnd) { const char *end; int nLeftOver; enum XML_Status result; @@ -1907,12 +1956,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_processor = errorProcessor; return XML_STATUS_ERROR; } + // though this isn't a buffer request, we assume that `len` is the app's + // preferred buffer fill size, and therefore save it here. + parser->m_lastBufferRequestSize = len; parser->m_parseEndByteIndex += len; parser->m_positionPtr = s; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; parser->m_errorCode - = parser->m_processor(parser, s, parser->m_parseEndPtr = s + len, &end); + = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -1939,23 +1991,25 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { &parser->m_position); nLeftOver = s + len - end; if (nLeftOver) { - if (parser->m_buffer == NULL - || nLeftOver > parser->m_bufferLim - parser->m_buffer) { - /* avoid _signed_ integer overflow */ - char *temp = NULL; - const int bytesToAllocate = (int)((unsigned)len * 2U); - if (bytesToAllocate > 0) { - temp = (char *)REALLOC(parser, parser->m_buffer, bytesToAllocate); - } - if (temp == NULL) { - parser->m_errorCode = XML_ERROR_NO_MEMORY; - parser->m_eventPtr = parser->m_eventEndPtr = NULL; - parser->m_processor = errorProcessor; - return XML_STATUS_ERROR; - } - parser->m_buffer = temp; - parser->m_bufferLim = parser->m_buffer + bytesToAllocate; + // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED + // (and XML_ERROR_FINISHED) from XML_GetBuffer. + const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; + parser->m_parsingStatus.parsing = XML_PARSING; + void *const temp = XML_GetBuffer(parser, nLeftOver); + parser->m_parsingStatus.parsing = originalStatus; + // GetBuffer may have overwritten this, but we want to remember what the + // app requested, not how many bytes were left over after parsing. + parser->m_lastBufferRequestSize = len; + if (temp == NULL) { + // NOTE: parser->m_errorCode has already been set by XML_GetBuffer(). + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; } + // Since we know that the buffer was empty and XML_CONTEXT_BYTES is 0, we + // don't have any data to preserve, and can copy straight into the start + // of the buffer rather than the GetBuffer return pointer (which may be + // pointing further into the allocated buffer). memcpy(parser->m_buffer, end, nLeftOver); } parser->m_bufferPtr = parser->m_buffer; @@ -1966,16 +2020,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_eventEndPtr = parser->m_bufferPtr; return result; } -#endif /* not defined XML_CONTEXT_BYTES */ - else { - void *buff = XML_GetBuffer(parser, len); - if (buff == NULL) - return XML_STATUS_ERROR; - else { - memcpy(buff, s, len); - return XML_ParseBuffer(parser, len, isFinal); - } +#endif /* XML_CONTEXT_BYTES == 0 */ + void *buff = XML_GetBuffer(parser, len); + if (buff == NULL) + return XML_STATUS_ERROR; + if (len > 0) { + assert(s != NULL); // make sure s==NULL && len!=0 was rejected above + memcpy(buff, s, len); } + return XML_ParseBuffer(parser, len, isFinal); } enum XML_Status XMLCALL @@ -2015,8 +2068,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parseEndByteIndex += len; parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; - parser->m_errorCode = parser->m_processor( - parser, start, parser->m_parseEndPtr, &parser->m_bufferPtr); + parser->m_errorCode = callProcessor(parser, start, parser->m_parseEndPtr, + &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -2061,10 +2114,14 @@ XML_GetBuffer(XML_Parser parser, int len) { default:; } - if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { -#ifdef XML_CONTEXT_BYTES + // whether or not the request succeeds, `len` seems to be the app's preferred + // buffer fill size; remember it. + parser->m_lastBufferRequestSize = len; + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) + || parser->m_buffer == NULL) { +#if XML_CONTEXT_BYTES > 0 int keep; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2073,7 +2130,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; @@ -2083,10 +2140,11 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } neededSize += keep; -#endif /* defined XML_CONTEXT_BYTES */ - if (neededSize - <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { -#ifdef XML_CONTEXT_BYTES +#endif /* XML_CONTEXT_BYTES > 0 */ + if (parser->m_buffer && parser->m_bufferPtr + && neededSize + <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { +#if XML_CONTEXT_BYTES > 0 if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { int offset = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) @@ -2099,19 +2157,17 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr -= offset; } #else - if (parser->m_buffer && parser->m_bufferPtr) { - memmove(parser->m_buffer, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - parser->m_bufferPtr = parser->m_buffer; - } -#endif /* not defined XML_CONTEXT_BYTES */ + memmove(parser->m_buffer, parser->m_bufferPtr, + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); + parser->m_bufferEnd + = parser->m_buffer + + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + parser->m_bufferPtr = parser->m_buffer; +#endif /* XML_CONTEXT_BYTES > 0 */ } else { char *newBuf; int bufferSize - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferPtr); + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { @@ -2128,7 +2184,7 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } parser->m_bufferLim = newBuf + bufferSize; -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser->m_bufferPtr) { memcpy(newBuf, &parser->m_bufferPtr[-keep], EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) @@ -2158,7 +2214,7 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferEnd = newBuf; } parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* not defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2208,7 +2264,7 @@ XML_ResumeParser(XML_Parser parser) { } parser->m_parsingStatus.parsing = XML_PARSING; - parser->m_errorCode = parser->m_processor( + parser->m_errorCode = callProcessor( parser, parser->m_bufferPtr, parser->m_parseEndPtr, &parser->m_bufferPtr); if (parser->m_errorCode != XML_ERROR_NONE) { @@ -2272,7 +2328,7 @@ XML_GetCurrentByteCount(XML_Parser parser) { const char *XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { @@ -2286,7 +2342,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)parser; (void)offset; (void)size; -#endif /* defined XML_CONTEXT_BYTES */ +#endif /* XML_CONTEXT_BYTES > 0 */ return (const char *)0; } @@ -2506,7 +2562,7 @@ XML_GetFeatureList(void) { #ifdef XML_DTD {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif -#ifdef XML_CONTEXT_BYTES +#if XML_CONTEXT_BYTES > 0 {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif @@ -2522,8 +2578,9 @@ XML_GetFeatureList(void) { #ifdef XML_ATTR_INFO {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, #endif -#ifdef XML_DTD - /* Added in Expat 2.4.0. */ +#if XML_GE == 1 + /* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_L("XML_BLAP_MAX_AMP"), (long int) @@ -2531,13 +2588,15 @@ XML_GetFeatureList(void) { {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, XML_L("XML_BLAP_ACT_THRES"), EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, + /* Added in Expat 2.6.0. */ + {XML_FEATURE_GE, XML_L("XML_GE"), 0}, #endif {XML_FEATURE_END, NULL, 0}}; return features; } -#ifdef XML_DTD +#if XML_GE == 1 XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { @@ -2559,7 +2618,16 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( parser->m_accounting.activationThresholdBytes = activationThresholdBytes; return XML_TRUE; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ + +XML_Bool XMLCALL +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { + parser->m_reparseDeferralEnabled = enabled; + return XML_TRUE; + } + return XML_FALSE; +} /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was @@ -2581,7 +2649,7 @@ storeRawNames(XML_Parser parser) { */ if (tag->rawName == rawNameBuf) break; - /* For re-use purposes we need to ensure that the + /* For reuse purposes we need to ensure that the size of tag->buf is a multiple of sizeof(XML_Char). */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); @@ -2645,13 +2713,13 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to @@ -2765,7 +2833,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 const char *accountAfter = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) ? (haveMore ? s /* i.e. 0 bytes */ : end) @@ -2831,14 +2899,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -3039,13 +3107,13 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (parser->m_ns && localPart) { /* localPart and prefix may have been overwritten in tag->name.str, since this points to the binding->uri - buffer which gets re-used; so we have to add them again + buffer which gets reused; so we have to add them again */ uri = (XML_Char *)tag->name.str + tag->name.uriLen; /* don't need to check for space - already done in storeAtts() */ while (*localPart) *uri++ = *localPart++; - prefix = (XML_Char *)tag->name.prefix; + prefix = tag->name.prefix; if (parser->m_ns_triplets && prefix) { *uri++ = parser->m_namespaceSeparator; while (*prefix) @@ -3112,7 +3180,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, However, now we have a start/endCdataSectionHandler, so it seems easier to let the user deal with this. */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -3141,8 +3209,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); } else parser->m_characterDataHandler( - parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3175,8 +3243,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4040,7 +4108,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, for (;;) { const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -4055,7 +4123,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, parser->m_endCdataSectionHandler(parser->m_handlerArg); /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if (0 && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); /* END disabled code */ @@ -4091,8 +4159,8 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; } } else - charDataHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)next - (XML_Char *)s)); + charDataHandler(parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)next - (const XML_Char *)s)); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4192,7 +4260,7 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); -# ifdef XML_DTD +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4284,7 +4352,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const XML_Char *storedversion = NULL; int standalone = -1; -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4482,16 +4550,16 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); } - /* If we are at the end of the buffer, this would cause XmlPrologTok to - return XML_TOK_NONE on the next call, which would then cause the - function to exit with *nextPtr set to s - that is what we want for other - tokens, but not for the BOM - we would rather like to skip it; - then, when this routine is entered the next time, XmlPrologTok will - return XML_TOK_INVALID, since the BOM is still in the buffer + /* XmlPrologTok has now set the encoding based on the BOM it found, and we + must move s and nextPtr forward to consume the BOM. + + If we didn't, and got XML_TOK_NONE from the next XmlPrologTok call, we + would leave the BOM in the buffer and return. On the next call to this + function, our XmlPrologTok call would return XML_TOK_INVALID, since it + is not valid to have multiple BOMs. */ - else if (tok == XML_TOK_BOM && next == end - && ! parser->m_parsingStatus.finalBuffer) { -# ifdef XML_DTD + else if (tok == XML_TOK_BOM) { +# if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -4500,7 +4568,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, # endif *nextPtr = next; - return XML_ERROR_NONE; + s = next; } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with @@ -4707,11 +4775,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); -#ifdef XML_DTD +#if XML_GE == 1 switch (role) { case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl - case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# ifdef XML_DTD + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl +# endif break; default: if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { @@ -5029,6 +5099,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { +#if XML_GE == 1 + // This will store the given replacement text in + // parser->m_declEntity->textPtr. enum XML_Error result = storeEntityValue(parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); @@ -5049,6 +5122,25 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolDiscard(&dtd->entityValuePool); if (result != XML_ERROR_NONE) return result; +#else + // This will store "&entity123;" in parser->m_declEntity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + + if (parser->m_entityDeclHandler) { + *eventEndPP = s; + parser->m_entityDeclHandler( + parser->m_handlerArg, parser->m_declEntity->name, + parser->m_declEntity->is_param, parser->m_declEntity->textPtr, + parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + handleDefault = XML_FALSE; + } + } +#endif } break; case XML_ROLE_DOCTYPE_SYSTEM_ID: @@ -5107,6 +5199,16 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } break; case XML_ROLE_ENTITY_COMPLETE: +#if XML_GE == 0 + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + if (parser->m_declEntity != NULL) { + const enum XML_Error result + = storeSelfEntityValue(parser, parser->m_declEntity); + if (result != XML_ERROR_NONE) + return result; + } +#endif if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; @@ -5648,7 +5750,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { accountingOnAbort(parser); @@ -5728,7 +5830,7 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif entity->processed = 0; @@ -5761,10 +5863,10 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; - } else { -#ifdef XML_DTD + } else if (parser->m_openInternalEntities->entity == entity) { +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5813,7 +5915,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return result; } -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -5892,7 +5994,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *next = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); -#ifdef XML_DTD +#if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { accountingOnAbort(parser); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; @@ -5957,14 +6059,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { -#ifdef XML_DTD +#if XML_GE == 1 /* NOTE: We are replacing 4-6 characters original input for 1 character * so there is no amplification and hence recording without * protection. */ accountingDiffTolerated(parser, tok, (char *)&ch, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -6042,14 +6144,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif result = appendAttributeValue(parser, parser->m_internalEncoding, isCdata, (const char *)entity->textPtr, (const char *)textEnd, pool, XML_ACCOUNT_ENTITY_EXPANSION); -#ifdef XML_DTD +#if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif entity->open = XML_FALSE; @@ -6079,6 +6181,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, /* not reached */ } +#if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *entityTextPtr, const char *entityTextEnd, @@ -6086,12 +6189,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; -#ifdef XML_DTD +# ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; -#else +# else UNUSED_P(account); -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we have to make sure that entityValuePool.start is not null */ @@ -6105,18 +6208,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); -#ifdef XML_DTD if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, account)) { accountingOnAbort(parser); result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; goto endEntityValue; } -#endif switch (tok) { case XML_TOK_PARAM_ENTITY_REF: -#ifdef XML_DTD +# ifdef XML_DTD if (parser->m_isParamEntity || enc != parser->m_encoding) { const XML_Char *name; ENTITY *entity; @@ -6139,7 +6240,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; goto endEntityValue; } - if (entity->open) { + if (entity->open || (entity == parser->m_declEntity)) { if (enc == parser->m_encoding) parser->m_eventPtr = entityTextPtr; result = XML_ERROR_RECURSIVE_ENTITY_REF; @@ -6178,7 +6279,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } break; } -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* In the internal subset, PE references are not legal within markup declarations, e.g entity values in this case. */ parser->m_eventPtr = entityTextPtr; @@ -6259,12 +6360,38 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, entityTextPtr = next; } endEntityValue: -#ifdef XML_DTD +# ifdef XML_DTD parser->m_prologState.inEntityValue = oldInEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ return result; } +#else /* XML_GE == 0 */ + +static enum XML_Error +storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { + // This will store "&entity123;" in entity->textPtr + // to end up as "&entity123;" in the handler. + const char *const entity_start = "&"; + const char *const entity_end = ";"; + + STRING_POOL *const pool = &(parser->m_dtd->entityValuePool); + if (! poolAppendString(pool, entity_start) + || ! poolAppendString(pool, entity->name) + || ! poolAppendString(pool, entity_end)) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } + + entity->textPtr = poolStart(pool); + entity->textLen = (int)(poolLength(pool)); + poolFinish(pool); + + return XML_ERROR_NONE; +} + +#endif /* XML_GE == 0 */ + static void FASTCALL normalizeLines(XML_Char *s) { XML_Char *p; @@ -6375,8 +6502,9 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else - parser->m_defaultHandler(parser->m_handlerArg, (XML_Char *)s, - (int)((XML_Char *)end - (XML_Char *)s)); + parser->m_defaultHandler( + parser->m_handlerArg, (const XML_Char *)s, + (int)((const XML_Char *)end - (const XML_Char *)s)); } static int @@ -6480,7 +6608,7 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, name = poolStoreString(&dtd->pool, enc, start, end); if (! name) return NULL; - /* skip quotation mark - its storage will be re-used (like in name[-1]) */ + /* skip quotation mark - its storage will be reused (like in name[-1]) */ ++name; id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, name, sizeof(ATTRIBUTE_ID)); @@ -6630,6 +6758,10 @@ getContext(XML_Parser parser) { static XML_Bool setContext(XML_Parser parser, const XML_Char *context) { + if (context == NULL) { + return XML_FALSE; + } + DTD *const dtd = parser->m_dtd; /* save one level of indirection */ const XML_Char *s = context; @@ -7220,7 +7352,7 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return NULL; for (;;) { const enum XML_Convert_Result convert_res = XmlConvert( - enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + enc, &ptr, end, (ICHAR **)&(pool->ptr), (const ICHAR *)pool->end); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; @@ -7651,10 +7783,12 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { return result; } -#ifdef XML_DTD +#if XML_GE == 1 static float accountingGetCurrentAmplification(XML_Parser rootParser) { + // 1.........1.........12 => 22 + const size_t lenOfShortestInclude = sizeof("") - 1; const XmlBigCount countBytesOutput = rootParser->m_accounting.countBytesDirect + rootParser->m_accounting.countBytesIndirect; @@ -7662,7 +7796,9 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { = rootParser->m_accounting.countBytesDirect ? (countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : 1.0f; + : ((lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) + / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; } @@ -7672,7 +7808,7 @@ accountingReportStats(XML_Parser originParser, const char *epilog) { const XML_Parser rootParser = getRootParserOf(originParser, NULL); assert(! rootParser->m_parentParser); - if (rootParser->m_accounting.debugLevel < 1) { + if (rootParser->m_accounting.debugLevel == 0u) { return; } @@ -7709,7 +7845,7 @@ accountingReportDiff(XML_Parser rootParser, /* Note: Performance is of no concern here */ const char *walker = before; - if ((rootParser->m_accounting.debugLevel >= 3) + if ((rootParser->m_accounting.debugLevel >= 3u) || (after - before) <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { for (; walker < after; walker++) { @@ -7774,7 +7910,7 @@ accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, || (amplificationFactor <= rootParser->m_accounting.maximumAmplificationFactor); - if (rootParser->m_accounting.debugLevel >= 2) { + if (rootParser->m_accounting.debugLevel >= 2u) { accountingReportStats(rootParser, ""); accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, bytesMore, source_line, account); @@ -7801,7 +7937,7 @@ static void entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *action, int sourceLine) { assert(! rootParser->m_parentParser); - if (rootParser->m_entity_stats.debugLevel < 1) + if (rootParser->m_entity_stats.debugLevel == 0u) return; # if defined(XML_UNICODE) @@ -8382,7 +8518,7 @@ unsignedCharToPrintable(unsigned char c) { assert(0); /* never gets here */ } -#endif /* XML_DTD */ +#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { @@ -8393,9 +8529,9 @@ getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { const char *const value = valueOrNull; errno = 0; - char *afterValue = (char *)value; + char *afterValue = NULL; unsigned long debugLevel = strtoul(value, &afterValue, 10); - if ((errno != 0) || (afterValue[0] != '\0')) { + if ((errno != 0) || (afterValue == value) || (afterValue[0] != '\0')) { errno = 0; return defaultDebugLevel; } diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 3f0f5c150c6278..2c48bf40867953 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,10 +12,10 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016-2023 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,7 +38,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include "expat_config.h" #include diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index d6e1fa150a108a..a7904274c91d4e 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017 Sebastian Pipping + Copyright (c) 2017-2024 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -127,9 +127,9 @@ typedef struct prolog_state { #endif /* XML_DTD */ } PROLOG_STATE; -void XmlPrologStateInit(PROLOG_STATE *); +void XmlPrologStateInit(PROLOG_STATE *state); #ifdef XML_DTD -void XmlPrologStateInitExternalEntity(PROLOG_STATE *); +void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); #endif /* XML_DTD */ #define XmlTokenRole(state, tok, ptr, end, enc) \ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 2b7012a58be419..29a66d72ceea5e 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -20,8 +20,10 @@ Copyright (c) 2017 Benbuck Nason Copyright (c) 2017 José Gutiérrez de la Concha Copyright (c) 2019 David Loffredo - Copyright (c) 2021 Dong-hee Na + Copyright (c) 2021 Donghee Na Copyright (c) 2022 Martin Ettl + Copyright (c) 2022 Sean McBride + Copyright (c) 2023 Hanno Böck Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -44,7 +46,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include "expat_config.h" #include #include /* memcpy */ @@ -76,7 +78,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo)&0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -100,7 +102,7 @@ & (1u << (((byte)[2]) & 0x1F))) /* Detection of invalid UTF-8 sequences is based on Table 3.1B - of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/ + of Unicode 3.2: https://www.unicode.org/unicode/reports/tr28/ with the additional restriction of not allowing the Unicode code points 0xFFFF and 0xFFFE (sequences EF,BF,BF and EF,BF,BE). Implementation details: @@ -225,7 +227,7 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int); +static int FASTCALL checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" @@ -243,7 +245,7 @@ static int FASTCALL checkCharRefNumber(int); #endif #define SB_BYTE_TYPE(enc, p) \ - (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) + (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) #ifdef XML_MIN_SIZE static int PTRFASTCALL @@ -407,7 +409,7 @@ utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short *to = *toP; const char *from = *fromP; while (from < fromLim && to < toLim) { - switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { + switch (SB_BYTE_TYPE(enc, from)) { case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; @@ -715,31 +717,26 @@ unicode_byte_type(char hi, char lo) { return res; \ } -#define SET2(ptr, ch) (((ptr)[0] = ((ch)&0xff)), ((ptr)[1] = ((ch) >> 8))) #define GET_LO(ptr) ((unsigned char)(ptr)[0]) #define GET_HI(ptr) ((unsigned char)(ptr)[1]) DEFINE_UTF16_TO_UTF8(little2_) DEFINE_UTF16_TO_UTF16(little2_) -#undef SET2 #undef GET_LO #undef GET_HI -#define SET2(ptr, ch) (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch)&0xFF))) #define GET_LO(ptr) ((unsigned char)(ptr)[1]) #define GET_HI(ptr) ((unsigned char)(ptr)[0]) DEFINE_UTF16_TO_UTF8(big2_) DEFINE_UTF16_TO_UTF16(big2_) -#undef SET2 #undef GET_LO #undef GET_HI #define LITTLE2_BYTE_TYPE(enc, p) \ - ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \ - : unicode_byte_type((p)[1], (p)[0])) + ((p)[1] == 0 ? SB_BYTE_TYPE(enc, p) : unicode_byte_type((p)[1], (p)[0])) #define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1) #define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == (c)) #define LITTLE2_IS_NAME_CHAR_MINBPC(p) \ @@ -872,9 +869,7 @@ static const struct normal_encoding internal_little2_encoding #endif #define BIG2_BYTE_TYPE(enc, p) \ - ((p)[0] == 0 \ - ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \ - : unicode_byte_type((p)[0], (p)[1])) + ((p)[0] == 0 ? SB_BYTE_TYPE(enc, p + 1) : unicode_byte_type((p)[0], (p)[1])) #define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1) #define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == (c)) #define BIG2_IS_NAME_CHAR_MINBPC(p) \ diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 6f630c2f9ba96d..c51fce1ec1518b 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2017 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -289,7 +289,8 @@ int XmlParseXmlDecl(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncoding(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); int FASTCALL XmlUtf8Encode(int charNumber, char *buf); @@ -307,7 +308,8 @@ int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, const char **encodingNamePtr, const ENCODING **namedEncodingPtr, int *standalonePtr); -int XmlInitEncodingNS(INIT_ENCODING *, const ENCODING **, const char *name); +int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, + const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 1971d74bf8c91f..239a2d06c4512c 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -126,7 +126,7 @@ # endif # define HAS_CHARS(enc, ptr, end, count) \ - ((end) - (ptr) >= ((count)*MINBPC(enc))) + ((end) - (ptr) >= ((count) * MINBPC(enc))) # define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index a2e3c2300b3ce8..cfa3cbdc34bc86 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -4,6 +4,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -119,6 +120,13 @@ faulthandler_get_fileno(PyObject **file_ptr) } } else if (PyLong_Check(file)) { + if (PyBool_Check(file)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; @@ -616,7 +624,7 @@ cancel_dump_traceback_later(void) #define SEC_TO_US (1000 * 1000) static char* -format_timeout(_PyTime_t us) +format_timeout(PyTime_t us) { unsigned long sec, min, hour; char buffer[100]; @@ -649,7 +657,7 @@ faulthandler_dump_traceback_later(PyObject *self, { static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; PyObject *timeout_obj; - _PyTime_t timeout, timeout_us; + PyTime_t timeout, timeout_us; int repeat = 0; PyObject *file = NULL; int fd; @@ -1184,58 +1192,67 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"enable", _PyCFunction_CAST(faulthandler_py_enable), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("enable(file=sys.stderr, all_threads=True): " - "enable the fault handler")}, + PyDoc_STR("enable($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Enable the fault handler.")}, {"disable", faulthandler_disable_py, METH_NOARGS, - PyDoc_STR("disable(): disable the fault handler")}, + PyDoc_STR("disable($module, /)\n--\n\n" + "Disable the fault handler.")}, {"is_enabled", faulthandler_is_enabled, METH_NOARGS, - PyDoc_STR("is_enabled()->bool: check if the handler is enabled")}, + PyDoc_STR("is_enabled($module, /)\n--\n\n" + "Check if the handler is enabled.")}, {"dump_traceback", _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): " - "dump the traceback of the current thread, or of all threads " - "if all_threads is True, into file")}, + PyDoc_STR("dump_traceback($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Dump the traceback of the current thread, or of all threads " + "if all_threads is True, into file.")}, {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False):\n" - "dump the traceback of all threads in timeout seconds,\n" + PyDoc_STR("dump_traceback_later($module, /, timeout, repeat=False, file=sys.stderr, exit=False)\n--\n\n" + "Dump the traceback of all threads in timeout seconds,\n" "or each timeout seconds if repeat is True. If exit is True, " "call _exit(1) which is not safe.")}, {"cancel_dump_traceback_later", faulthandler_cancel_dump_traceback_later_py, METH_NOARGS, - PyDoc_STR("cancel_dump_traceback_later():\ncancel the previous call " - "to dump_traceback_later().")}, + PyDoc_STR("cancel_dump_traceback_later($module, /)\n--\n\n" + "Cancel the previous call to dump_traceback_later().")}, #ifdef FAULTHANDLER_USER {"register", _PyCFunction_CAST(faulthandler_register_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("register(signum, file=sys.stderr, all_threads=True, chain=False): " - "register a handler for the signal 'signum': dump the " + PyDoc_STR("register($module, /, signum, file=sys.stderr, all_threads=True, chain=False)\n--\n\n" + "Register a handler for the signal 'signum': dump the " "traceback of the current thread, or of all threads if " - "all_threads is True, into file")}, + "all_threads is True, into file.")}, {"unregister", - _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("unregister(signum): unregister the handler of the signal " - "'signum' registered by register()")}, + _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS, + PyDoc_STR("unregister($module, signum, /)\n--\n\n" + "Unregister the handler of the signal " + "'signum' registered by register().")}, #endif {"_read_null", faulthandler_read_null, METH_NOARGS, - PyDoc_STR("_read_null(): read from NULL, raise " - "a SIGSEGV or SIGBUS signal depending on the platform")}, + PyDoc_STR("_read_null($module, /)\n--\n\n" + "Read from NULL, raise " + "a SIGSEGV or SIGBUS signal depending on the platform.")}, {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, - PyDoc_STR("_sigsegv(release_gil=False): raise a SIGSEGV signal")}, + PyDoc_STR("_sigsegv($module, release_gil=False, /)\n--\n\n" + "Raise a SIGSEGV signal.")}, {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, - PyDoc_STR("fatal_error_c_thread(): " - "call Py_FatalError() in a new C thread.")}, + PyDoc_STR("_fatal_error_c_thread($module, /)\n--\n\n" + "Call Py_FatalError() in a new C thread.")}, {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, - PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, + PyDoc_STR("_sigabrt($module, /)\n--\n\n" + "Raise a SIGABRT signal.")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, - PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, + PyDoc_STR("_sigfpe($module, /)\n--\n\n" + "Raise a SIGFPE signal.")}, #ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", faulthandler_stack_overflow, METH_NOARGS, - PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, + PyDoc_STR("_stack_overflow($module, /)\n--\n\n" + "Recursive call to raise a stack overflow.")}, #endif #ifdef MS_WINDOWS {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, - PyDoc_STR("raise_exception(code, flags=0): Call RaiseException(code, flags).")}, + PyDoc_STR("_raise_exception($module, code, flags=0, /)\n--\n\n" + "Call RaiseException(code, flags).")}, #endif {NULL, NULL} /* sentinel */ }; @@ -1275,6 +1292,7 @@ static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index fd03abf0561da6..0c06c03a6c403e 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,22 +1,25 @@ /* fcntl module */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // EINTR +#include // fcntl() +#include // memcpy() +#include // ioctl() #ifdef HAVE_SYS_FILE_H -#include +# include // flock() #endif #ifdef HAVE_LINUX_FS_H -#include +# include #endif - -#include -#include #ifdef HAVE_STROPTS_H -#include +# include // I_FLUSHBAND #endif /*[clinic input] @@ -109,7 +112,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) fcntl.ioctl fd: fildes - request as code: unsigned_int(bitwise=True) + request as code: unsigned_long(bitwise=True) arg as ob_arg: object(c_default='NULL') = 0 mutate_flag as mutate_arg: bool = True / @@ -145,9 +148,9 @@ code. [clinic start generated code]*/ static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg) -/*[clinic end generated code: output=7f7f5840c65991be input=967b4a4cbeceb0a8]*/ +/*[clinic end generated code: output=3d8eb6828666cea1 input=cee70f6a27311e58]*/ { #define IOCTL_BUFSZ 1024 /* We use the unsigned non-checked 'I' format for the 'code' parameter @@ -167,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ - if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + if (PySys_Audit("fcntl.ioctl", "ikO", fd, code, ob_arg ? ob_arg : Py_None) < 0) { return NULL; } @@ -583,6 +586,30 @@ all_ins(PyObject* m) #ifdef FICLONERANGE if (PyModule_AddIntMacro(m, FICLONERANGE)) return -1; #endif +#ifdef F_GETOWN_EX + // since Linux 2.6.32 + if (PyModule_AddIntMacro(m, F_GETOWN_EX)) return -1; + if (PyModule_AddIntMacro(m, F_SETOWN_EX)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_TID)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_PID)) return -1; + if (PyModule_AddIntMacro(m, F_OWNER_PGRP)) return -1; +#endif +#ifdef F_GET_RW_HINT + // since Linux 4.13 + if (PyModule_AddIntMacro(m, F_GET_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_SET_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_GET_FILE_RW_HINT)) return -1; + if (PyModule_AddIntMacro(m, F_SET_FILE_RW_HINT)) return -1; +#ifndef RWH_WRITE_LIFE_NOT_SET // typo in Linux < 5.5 +# define RWH_WRITE_LIFE_NOT_SET RWF_WRITE_LIFE_NOT_SET +#endif + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_NOT_SET)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_NONE)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_SHORT)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_MEDIUM)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_LONG)) return -1; + if (PyModule_AddIntMacro(m, RWH_WRITE_LIFE_EXTREME)) return -1; +#endif /* OS X specifics */ #ifdef F_FULLFSYNC @@ -599,6 +626,32 @@ all_ins(PyObject* m) #ifdef F_DUP2FD_CLOEXEC if (PyModule_AddIntMacro(m, F_DUP2FD_CLOEXEC)) return -1; #endif +#ifdef F_READAHEAD + if (PyModule_AddIntMacro(m, F_READAHEAD)) return -1; +#endif +#ifdef F_RDAHEAD + if (PyModule_AddIntMacro(m, F_RDAHEAD)) return -1; +#endif +#ifdef F_ISUNIONSTACK + if (PyModule_AddIntMacro(m, F_ISUNIONSTACK)) return -1; +#endif +#ifdef F_KINFO + if (PyModule_AddIntMacro(m, F_KINFO)) return -1; +#endif + +/* NetBSD specifics */ +#ifdef F_CLOSEM + if (PyModule_AddIntMacro(m, F_CLOSEM)) return -1; +#endif +#ifdef F_MAXFD + if (PyModule_AddIntMacro(m, F_MAXFD)) return -1; +#endif +#ifdef F_GETNOSIGPIPE + if (PyModule_AddIntMacro(m, F_GETNOSIGPIPE)) return -1; +#endif +#ifdef F_SETNOSIGPIPE + if (PyModule_AddIntMacro(m, F_SETNOSIGPIPE)) return -1; +#endif /* For F_{GET|SET}FL */ #ifdef FD_CLOEXEC @@ -673,6 +726,9 @@ all_ins(PyObject* m) if (PyModule_AddIntMacro(m, F_SEAL_SHRINK)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_GROW)) return -1; if (PyModule_AddIntMacro(m, F_SEAL_WRITE)) return -1; +#ifdef F_SEAL_FUTURE_WRITE + if (PyModule_AddIntMacro(m, F_SEAL_FUTURE_WRITE)) return -1; +#endif #endif return 0; } @@ -689,6 +745,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 592d527f0bd6a2..57e4aae9ed557e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1,1509 +1,27 @@ /* - - Reference Cycle Garbage Collection - ================================== - - Neil Schemenauer - - Based on a post on the python-dev list. Ideas from Guido van Rossum, - Eric Tiedemann, and various others. - - http://www.arctrix.com/nas/python/gc/ - - The following mailing list threads provide a historical perspective on - the design of this module. Note that a fair amount of refinement has - occurred since those discussions. - - http://mail.python.org/pipermail/python-dev/2000-March/002385.html - http://mail.python.org/pipermail/python-dev/2000-March/002434.html - http://mail.python.org/pipermail/python-dev/2000-March/002497.html - - For a highlevel view of the collection process, read the collect - function. - -*/ - -#include "Python.h" -#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() -#include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() -#include "pycore_initconfig.h" -#include "pycore_interp.h" // PyInterpreterState.gc -#include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_weakref.h" // _PyWeakref_ClearRef() -#include "pydtrace.h" - -typedef struct _gc_runtime_state GCState; - -/*[clinic input] -module gc -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5c9690ecc842d79]*/ - - -#ifdef Py_DEBUG -# define GC_DEBUG -#endif - -#define GC_NEXT _PyGCHead_NEXT -#define GC_PREV _PyGCHead_PREV - -// update_refs() set this bit for all objects in current generation. -// subtract_refs() and move_unreachable() uses this to distinguish -// visited object is in GCing or not. -// -// move_unreachable() removes this flag from reachable objects. -// Only unreachable objects have this flag. -// -// No objects in interpreter have this flag after GC ends. -#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING - -// Lowest bit of _gc_next is used for UNREACHABLE flag. -// -// This flag represents the object is in unreachable list in move_unreachable() -// -// Although this flag is used only in move_unreachable(), move_unreachable() -// doesn't clear this flag to skip unnecessary iteration. -// move_legacy_finalizers() removes this flag instead. -// Between them, unreachable list is not normal list and we can not use -// most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE (1) - -#define AS_GC(op) _Py_AS_GC(op) -#define FROM_GC(gc) _Py_FROM_GC(gc) - - -static inline int -gc_is_collecting(PyGC_Head *g) -{ - return (g->_gc_prev & PREV_MASK_COLLECTING) != 0; -} - -static inline void -gc_clear_collecting(PyGC_Head *g) -{ - g->_gc_prev &= ~PREV_MASK_COLLECTING; -} - -static inline Py_ssize_t -gc_get_refs(PyGC_Head *g) -{ - return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); -} - -static inline void -gc_set_refs(PyGC_Head *g, Py_ssize_t refs) -{ - g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) - | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); -} - -static inline void -gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) -{ - g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) - | PREV_MASK_COLLECTING - | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); -} - -static inline void -gc_decref(PyGC_Head *g) -{ - _PyObject_ASSERT_WITH_MSG(FROM_GC(g), - gc_get_refs(g) > 0, - "refcount is too small"); - g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; -} - -/* set for debugging information */ -#define DEBUG_STATS (1<<0) /* print collection statistics */ -#define DEBUG_COLLECTABLE (1<<1) /* print collectable objects */ -#define DEBUG_UNCOLLECTABLE (1<<2) /* print uncollectable objects */ -#define DEBUG_SAVEALL (1<<5) /* save all garbage in gc.garbage */ -#define DEBUG_LEAK DEBUG_COLLECTABLE | \ - DEBUG_UNCOLLECTABLE | \ - DEBUG_SAVEALL - -#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) - - -static GCState * -get_gc_state(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->gc; -} - - -void -_PyGC_InitState(GCState *gcstate) -{ -#define INIT_HEAD(GEN) \ - do { \ - GEN.head._gc_next = (uintptr_t)&GEN.head; \ - GEN.head._gc_prev = (uintptr_t)&GEN.head; \ - } while (0) - - for (int i = 0; i < NUM_GENERATIONS; i++) { - assert(gcstate->generations[i].count == 0); - INIT_HEAD(gcstate->generations[i]); - }; - gcstate->generation0 = GEN_HEAD(gcstate, 0); - INIT_HEAD(gcstate->permanent_generation); - -#undef INIT_HEAD -} - - -PyStatus -_PyGC_Init(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - - gcstate->garbage = PyList_New(0); - if (gcstate->garbage == NULL) { - return _PyStatus_NO_MEMORY(); - } - - gcstate->callbacks = PyList_New(0); - if (gcstate->callbacks == NULL) { - return _PyStatus_NO_MEMORY(); - } - - return _PyStatus_OK(); -} - - -/* -_gc_prev values ---------------- - -Between collections, _gc_prev is used for doubly linked list. - -Lowest two bits of _gc_prev are used for flags. -PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends -or _PyObject_GC_UNTRACK() is called. - -During a collection, _gc_prev is temporary used for gc_refs, and the gc list -is singly linked until _gc_prev is restored. - -gc_refs - At the start of a collection, update_refs() copies the true refcount - to gc_refs, for each object in the generation being collected. - subtract_refs() then adjusts gc_refs so that it equals the number of - times an object is referenced directly from outside the generation - being collected. - -PREV_MASK_COLLECTING - Objects in generation being collected are marked PREV_MASK_COLLECTING in - update_refs(). - - -_gc_next values ---------------- - -_gc_next takes these values: - -0 - The object is not tracked - -!= 0 - Pointer to the next object in the GC list. - Additionally, lowest bit is used temporary for - NEXT_MASK_UNREACHABLE flag described below. - -NEXT_MASK_UNREACHABLE - move_unreachable() then moves objects not reachable (whether directly or - indirectly) from outside the generation into an "unreachable" set and - set this flag. - - Objects that are found to be reachable have gc_refs set to 1. - When this flag is set for the reachable object, the object must be in - "unreachable" set. - The flag is unset and the object is moved back to "reachable" set. - - move_legacy_finalizers() will remove this flag from "unreachable" set. -*/ - -/*** list functions ***/ - -static inline void -gc_list_init(PyGC_Head *list) -{ - // List header must not have flags. - // We can assign pointer by simple cast. - list->_gc_prev = (uintptr_t)list; - list->_gc_next = (uintptr_t)list; -} - -static inline int -gc_list_is_empty(PyGC_Head *list) -{ - return (list->_gc_next == (uintptr_t)list); -} - -/* Append `node` to `list`. */ -static inline void -gc_list_append(PyGC_Head *node, PyGC_Head *list) -{ - PyGC_Head *last = (PyGC_Head *)list->_gc_prev; - - // last <-> node - _PyGCHead_SET_PREV(node, last); - _PyGCHead_SET_NEXT(last, node); - - // node <-> list - _PyGCHead_SET_NEXT(node, list); - list->_gc_prev = (uintptr_t)node; -} - -/* Remove `node` from the gc list it's currently in. */ -static inline void -gc_list_remove(PyGC_Head *node) -{ - PyGC_Head *prev = GC_PREV(node); - PyGC_Head *next = GC_NEXT(node); - - _PyGCHead_SET_NEXT(prev, next); - _PyGCHead_SET_PREV(next, prev); - - node->_gc_next = 0; /* object is not currently tracked */ -} - -/* Move `node` from the gc list it's currently in (which is not explicitly - * named here) to the end of `list`. This is semantically the same as - * gc_list_remove(node) followed by gc_list_append(node, list). - */ -static void -gc_list_move(PyGC_Head *node, PyGC_Head *list) -{ - /* Unlink from current list. */ - PyGC_Head *from_prev = GC_PREV(node); - PyGC_Head *from_next = GC_NEXT(node); - _PyGCHead_SET_NEXT(from_prev, from_next); - _PyGCHead_SET_PREV(from_next, from_prev); - - /* Relink at end of new list. */ - // list must not have flags. So we can skip macros. - PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; - _PyGCHead_SET_PREV(node, to_prev); - _PyGCHead_SET_NEXT(to_prev, node); - list->_gc_prev = (uintptr_t)node; - _PyGCHead_SET_NEXT(node, list); -} - -/* append list `from` onto list `to`; `from` becomes an empty list */ -static void -gc_list_merge(PyGC_Head *from, PyGC_Head *to) -{ - assert(from != to); - if (!gc_list_is_empty(from)) { - PyGC_Head *to_tail = GC_PREV(to); - PyGC_Head *from_head = GC_NEXT(from); - PyGC_Head *from_tail = GC_PREV(from); - assert(from_head != from); - assert(from_tail != from); - - _PyGCHead_SET_NEXT(to_tail, from_head); - _PyGCHead_SET_PREV(from_head, to_tail); - - _PyGCHead_SET_NEXT(from_tail, to); - _PyGCHead_SET_PREV(to, from_tail); - } - gc_list_init(from); -} - -static Py_ssize_t -gc_list_size(PyGC_Head *list) -{ - PyGC_Head *gc; - Py_ssize_t n = 0; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - n++; - } - return n; -} - -/* Walk the list and mark all objects as non-collecting */ -static inline void -gc_list_clear_collecting(PyGC_Head *collectable) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { - gc_clear_collecting(gc); - } -} - -/* Append objects in a GC list to a Python list. - * Return 0 if all OK, < 0 if error (out of memory for list) - */ -static int -append_objects(PyObject *py_list, PyGC_Head *gc_list) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - if (op != py_list) { - if (PyList_Append(py_list, op)) { - return -1; /* exception */ - } - } - } - return 0; -} - -// Constants for validate_list's flags argument. -enum flagstates {collecting_clear_unreachable_clear, - collecting_clear_unreachable_set, - collecting_set_unreachable_clear, - collecting_set_unreachable_set}; - -#ifdef GC_DEBUG -// validate_list checks list consistency. And it works as document -// describing when flags are expected to be set / unset. -// `head` must be a doubly-linked gc list, although it's fine (expected!) if -// the prev and next pointers are "polluted" with flags. -// What's checked: -// - The `head` pointers are not polluted. -// - The objects' PREV_MASK_COLLECTING and NEXT_MASK_UNREACHABLE flags are all -// `set or clear, as specified by the 'flags' argument. -// - The prev and next pointers are mutually consistent. -static void -validate_list(PyGC_Head *head, enum flagstates flags) -{ - assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); - assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - uintptr_t prev_value = 0, next_value = 0; - switch (flags) { - case collecting_clear_unreachable_clear: - break; - case collecting_set_unreachable_clear: - prev_value = PREV_MASK_COLLECTING; - break; - case collecting_clear_unreachable_set: - next_value = NEXT_MASK_UNREACHABLE; - break; - case collecting_set_unreachable_set: - prev_value = PREV_MASK_COLLECTING; - next_value = NEXT_MASK_UNREACHABLE; - break; - default: - assert(! "bad internal flags argument"); - } - PyGC_Head *prev = head; - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); - assert(truenext != NULL); - assert(trueprev == prev); - assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); - assert((gc->_gc_next & NEXT_MASK_UNREACHABLE) == next_value); - prev = gc; - gc = truenext; - } - assert(prev == GC_PREV(head)); -} -#else -#define validate_list(x, y) do{}while(0) -#endif - -/*** end of list stuff ***/ - - -/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and - * PREV_MASK_COLLECTING bit is set for all objects in containers. - */ -static void -update_refs(PyGC_Head *containers) -{ - PyGC_Head *next; - PyGC_Head *gc = GC_NEXT(containers); - - while (gc != containers) { - next = GC_NEXT(gc); - /* Move any object that might have become immortal to the - * permanent generation as the reference count is not accurately - * reflecting the actual number of live references to this object - */ - if (_Py_IsImmortal(FROM_GC(gc))) { - gc_list_move(gc, &get_gc_state()->permanent_generation.head); - gc = next; - continue; - } - gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); - /* Python's cyclic gc should never see an incoming refcount - * of 0: if something decref'ed to 0, it should have been - * deallocated immediately at that time. - * Possible cause (if the assert triggers): a tp_dealloc - * routine left a gc-aware object tracked during its teardown - * phase, and did something-- or allowed something to happen -- - * that called back into Python. gc can trigger then, and may - * see the still-tracked dying object. Before this assert - * was added, such mistakes went on to allow gc to try to - * delete the object again. In a debug build, that caused - * a mysterious segfault, when _Py_ForgetReference tried - * to remove the object from the doubly-linked list of all - * objects a second time. In a release build, an actual - * double deallocation occurred, which leads to corruption - * of the allocator's internal bookkeeping pointers. That's - * so serious that maybe this should be a release-build - * check instead of an assert? - */ - _PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0); - gc = next; - } -} - -/* A traversal callback for subtract_refs. */ -static int -visit_decref(PyObject *op, void *parent) -{ - OBJECT_STAT_INC(object_visits); - _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); - - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - /* We're only interested in gc_refs for objects in the - * generation being collected, which can be recognized - * because only they have positive gc_refs. - */ - if (gc_is_collecting(gc)) { - gc_decref(gc); - } - } - return 0; -} - -/* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly - * reachable from outside containers, and so can't be collected. - */ -static void -subtract_refs(PyGC_Head *containers) -{ - traverseproc traverse; - PyGC_Head *gc = GC_NEXT(containers); - for (; gc != containers; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - (visitproc)visit_decref, - op); - } -} - -/* A traversal callback for move_unreachable. */ -static int -visit_reachable(PyObject *op, PyGC_Head *reachable) -{ - OBJECT_STAT_INC(object_visits); - if (!_PyObject_IS_GC(op)) { - return 0; - } - - PyGC_Head *gc = AS_GC(op); - const Py_ssize_t gc_refs = gc_get_refs(gc); - - // Ignore objects in other generation. - // This also skips objects "to the left" of the current position in - // move_unreachable's scan of the 'young' list - they've already been - // traversed, and no longer have the PREV_MASK_COLLECTING flag. - if (! gc_is_collecting(gc)) { - return 0; - } - // It would be a logic error elsewhere if the collecting flag were set on - // an untracked object. - assert(gc->_gc_next != 0); - - if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { - /* This had gc_refs = 0 when move_unreachable got - * to it, but turns out it's reachable after all. - * Move it back to move_unreachable's 'young' list, - * and move_unreachable will eventually get to it - * again. - */ - // Manually unlink gc from unreachable list because the list functions - // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. - PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); - _PyObject_ASSERT(FROM_GC(prev), - prev->_gc_next & NEXT_MASK_UNREACHABLE); - _PyObject_ASSERT(FROM_GC(next), - next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE - _PyGCHead_SET_PREV(next, prev); - - gc_list_append(gc, reachable); - gc_set_refs(gc, 1); - } - else if (gc_refs == 0) { - /* This is in move_unreachable's 'young' list, but - * the traversal hasn't yet gotten to it. All - * we need to do is tell move_unreachable that it's - * reachable. - */ - gc_set_refs(gc, 1); - } - /* Else there's nothing to do. - * If gc_refs > 0, it must be in move_unreachable's 'young' - * list, and move_unreachable will eventually get to it. - */ - else { - _PyObject_ASSERT_WITH_MSG(op, gc_refs > 0, "refcount is too small"); - } - return 0; -} - -/* Move the unreachable objects from young to unreachable. After this, - * all objects in young don't have PREV_MASK_COLLECTING flag and - * unreachable have the flag. - * All objects in young after this are directly or indirectly reachable - * from outside the original young; and all objects in unreachable are - * not. - * - * This function restores _gc_prev pointer. young and unreachable are - * doubly linked list after this function. - * But _gc_next in unreachable list has NEXT_MASK_UNREACHABLE flag. - * So we can not gc_list_* functions for unreachable until we remove the flag. - */ -static void -move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) -{ - // previous elem in the young list, used for restore gc_prev. - PyGC_Head *prev = young; - PyGC_Head *gc = GC_NEXT(young); - - /* Invariants: all objects "to the left" of us in young are reachable - * (directly or indirectly) from outside the young list as it was at entry. - * - * All other objects from the original young "to the left" of us are in - * unreachable now, and have NEXT_MASK_UNREACHABLE. All objects to the - * left of us in 'young' now have been scanned, and no objects here - * or to the right have been scanned yet. - */ - - while (gc != young) { - if (gc_get_refs(gc)) { - /* gc is definitely reachable from outside the - * original 'young'. Mark it as such, and traverse - * its pointers to find any other objects that may - * be directly reachable from it. Note that the - * call to tp_traverse may append objects to young, - * so we have to wait until it returns to determine - * the next object to visit. - */ - PyObject *op = FROM_GC(gc); - traverseproc traverse = Py_TYPE(op)->tp_traverse; - _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) > 0, - "refcount is too small"); - // NOTE: visit_reachable may change gc->_gc_next when - // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! - (void) traverse(op, - (visitproc)visit_reachable, - (void *)young); - // relink gc_prev to prev element. - _PyGCHead_SET_PREV(gc, prev); - // gc is not COLLECTING state after here. - gc_clear_collecting(gc); - prev = gc; - } - else { - /* This *may* be unreachable. To make progress, - * assume it is. gc isn't directly reachable from - * any object we've already traversed, but may be - * reachable from an object we haven't gotten to yet. - * visit_reachable will eventually move gc back into - * young if that's so, and we'll see it again. - */ - // Move gc to unreachable. - // No need to gc->next->prev = prev because it is single linked. - prev->_gc_next = gc->_gc_next; - - // We can't use gc_list_append() here because we use - // NEXT_MASK_UNREACHABLE here. - PyGC_Head *last = GC_PREV(unreachable); - // NOTE: Since all objects in unreachable set has - // NEXT_MASK_UNREACHABLE flag, we set it unconditionally. - // But this may pollute the unreachable list head's 'next' pointer - // too. That's semantically senseless but expedient here - the - // damage is repaired when this function ends. - last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); - _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); - unreachable->_gc_prev = (uintptr_t)gc; - } - gc = (PyGC_Head*)prev->_gc_next; - } - // young->_gc_prev must be last element remained in the list. - young->_gc_prev = (uintptr_t)prev; - // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; -} - -static void -untrack_tuples(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - } - gc = next; - } -} - -/* Try to untrack all currently tracked dictionaries */ -static void -untrack_dicts(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - } - gc = next; - } -} - -/* Return true if object has a pre-PEP 442 finalization method. */ -static int -has_legacy_finalizer(PyObject *op) -{ - return Py_TYPE(op)->tp_del != NULL; -} - -/* Move the objects in unreachable with tp_del slots into `finalizers`. + * Python interface to the garbage collector. * - * This function also removes NEXT_MASK_UNREACHABLE flag - * from _gc_next in unreachable. - */ -static void -move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) -{ - PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - - /* March over unreachable. Move objects with finalizers into - * `finalizers`. - */ - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - PyObject *op = FROM_GC(gc); - - _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; - - if (has_legacy_finalizer(op)) { - gc_clear_collecting(gc); - gc_list_move(gc, finalizers); - } - } -} - -static inline void -clear_unreachable_mask(PyGC_Head *unreachable) -{ - /* Check that the list head does not have the unreachable bit set */ - assert(((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); - - PyGC_Head *gc, *next; - assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; - } - validate_list(unreachable, collecting_set_unreachable_clear); -} - -/* A traversal callback for move_legacy_finalizer_reachable. */ -static int -visit_move(PyObject *op, PyGC_Head *tolist) -{ - OBJECT_STAT_INC(object_visits); - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (gc_is_collecting(gc)) { - gc_list_move(gc, tolist); - gc_clear_collecting(gc); - } - } - return 0; -} - -/* Move objects that are reachable from finalizers, from the unreachable set - * into finalizers set. - */ -static void -move_legacy_finalizer_reachable(PyGC_Head *finalizers) -{ - traverseproc traverse; - PyGC_Head *gc = GC_NEXT(finalizers); - for (; gc != finalizers; gc = GC_NEXT(gc)) { - /* Note that the finalizers list may grow during this. */ - traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; - (void) traverse(FROM_GC(gc), - (visitproc)visit_move, - (void *)finalizers); - } -} - -/* Clear all weakrefs to unreachable objects, and if such a weakref has a - * callback, invoke it if necessary. Note that it's possible for such - * weakrefs to be outside the unreachable set -- indeed, those are precisely - * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for - * overview & some details. Some weakrefs with callbacks may be reclaimed - * directly by this routine; the number reclaimed is the return value. Other - * weakrefs with callbacks may be moved into the `old` generation. Objects - * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in - * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, - * no object in `unreachable` is weakly referenced anymore. - */ -static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) -{ - PyGC_Head *gc; - PyObject *op; /* generally FROM_GC(gc) */ - PyWeakReference *wr; /* generally a cast of op */ - PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ - PyGC_Head *next; - int num_freed = 0; - - gc_list_init(&wrcb_to_call); - - /* Clear all weakrefs to the objects in unreachable. If such a weakref - * also has a callback, move it into `wrcb_to_call` if the callback - * needs to be invoked. Note that we cannot invoke any callbacks until - * all weakrefs to unreachable objects are cleared, lest the callback - * resurrect an unreachable object via a still-active weakref. We - * make another pass over wrcb_to_call, invoking callbacks, after this - * pass completes. - */ - for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - PyWeakReference **wrlist; - - op = FROM_GC(gc); - next = GC_NEXT(gc); - - if (PyWeakref_Check(op)) { - /* A weakref inside the unreachable set must be cleared. If we - * allow its callback to execute inside delete_garbage(), it - * could expose objects that have tp_clear already called on - * them. Or, it could resurrect unreachable objects. One way - * this can happen is if some container objects do not implement - * tp_traverse. Then, wr_object can be outside the unreachable - * set but can be deallocated as a result of breaking the - * reference cycle. If we don't clear the weakref, the callback - * will run and potentially cause a crash. See bpo-38006 for - * one example. - */ - _PyWeakref_ClearRef((PyWeakReference *)op); - } - - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) - continue; - - /* It supports weakrefs. Does it have any? - * - * This is never triggered for static types so we can avoid the - * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). - */ - wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); - - /* `op` may have some weakrefs. March over the list, clear - * all the weakrefs, and move the weakrefs with callbacks - * that must be called into wrcb_to_call. - */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { - PyGC_Head *wrasgc; /* AS_GC(wr) */ - - /* _PyWeakref_ClearRef clears the weakref but leaves - * the callback pointer intact. Obscure: it also - * changes *wrlist. - */ - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - if (wr->wr_callback == NULL) { - /* no callback */ - continue; - } - - /* Headache time. `op` is going away, and is weakly referenced by - * `wr`, which has a callback. Should the callback be invoked? If wr - * is also trash, no: - * - * 1. There's no need to call it. The object and the weakref are - * both going away, so it's legitimate to pretend the weakref is - * going away first. The user has to ensure a weakref outlives its - * referent if they want a guarantee that the wr callback will get - * invoked. - * - * 2. It may be catastrophic to call it. If the callback is also in - * cyclic trash (CT), then although the CT is unreachable from - * outside the current generation, CT may be reachable from the - * callback. Then the callback could resurrect insane objects. - * - * Since the callback is never needed and may be unsafe in this case, - * wr is simply left in the unreachable set. Note that because we - * already called _PyWeakref_ClearRef(wr), its callback will never - * trigger. - * - * OTOH, if wr isn't part of CT, we should invoke the callback: the - * weakref outlived the trash. Note that since wr isn't CT in this - * case, its callback can't be CT either -- wr acted as an external - * root to this generation, and therefore its callback did too. So - * nothing in CT is reachable from the callback either, so it's hard - * to imagine how calling it later could create a problem for us. wr - * is moved to wrcb_to_call in this case. - */ - if (gc_is_collecting(AS_GC((PyObject *)wr))) { - /* it should already have been cleared above */ - assert(wr->wr_object == Py_None); - continue; - } - - /* Create a new reference so that wr can't go away - * before we can process it again. - */ - Py_INCREF(wr); - - /* Move wr to wrcb_to_call, for the next pass. */ - wrasgc = AS_GC((PyObject *)wr); - assert(wrasgc != next); /* wrasgc is reachable, but - next isn't, so they can't - be the same */ - gc_list_move(wrasgc, &wrcb_to_call); - } - } - - /* Invoke the callbacks we decided to honor. It's safe to invoke them - * because they can't reference unreachable objects. - */ - while (! gc_list_is_empty(&wrcb_to_call)) { - PyObject *temp; - PyObject *callback; - - gc = (PyGC_Head*)wrcb_to_call._gc_next; - op = FROM_GC(gc); - _PyObject_ASSERT(op, PyWeakref_Check(op)); - wr = (PyWeakReference *)op; - callback = wr->wr_callback; - _PyObject_ASSERT(op, callback != NULL); - - /* copy-paste of weakrefobject.c's handle_callback() */ - temp = PyObject_CallOneArg(callback, (PyObject *)wr); - if (temp == NULL) - PyErr_WriteUnraisable(callback); - else - Py_DECREF(temp); - - /* Give up the reference we created in the first pass. When - * op's refcount hits 0 (which it may or may not do right now), - * op's tp_dealloc will decref op->wr_callback too. Note - * that the refcount probably will hit 0 now, and because this - * weakref was reachable to begin with, gc didn't already - * add it to its count of freed objects. Example: a reachable - * weak value dict maps some key to this reachable weakref. - * The callback removes this key->weakref mapping from the - * dict, leaving no other references to the weakref (excepting - * ours). - */ - Py_DECREF(op); - if (wrcb_to_call._gc_next == (uintptr_t)gc) { - /* object is still alive -- move it */ - gc_list_move(gc, old); - } - else { - ++num_freed; - } - } - - return num_freed; -} - -static void -debug_cycle(const char *msg, PyObject *op) -{ - PySys_FormatStderr("gc: %s <%s %p>\n", - msg, Py_TYPE(op)->tp_name, op); -} - -/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable - * only from such cycles). - * If DEBUG_SAVEALL, all objects in finalizers are appended to the module - * garbage list (a Python list), else only the objects in finalizers with - * __del__ methods are appended to garbage. All objects in finalizers are - * merged into the old list regardless. - */ -static void -handle_legacy_finalizers(PyThreadState *tstate, - GCState *gcstate, - PyGC_Head *finalizers, PyGC_Head *old) -{ - assert(!_PyErr_Occurred(tstate)); - assert(gcstate->garbage != NULL); - - PyGC_Head *gc = GC_NEXT(finalizers); - for (; gc != finalizers; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - - if ((gcstate->debug & DEBUG_SAVEALL) || has_legacy_finalizer(op)) { - if (PyList_Append(gcstate->garbage, op) < 0) { - _PyErr_Clear(tstate); - break; - } - } - } - - gc_list_merge(finalizers, old); -} - -/* Run first-time finalizers (if any) on all the objects in collectable. - * Note that this may remove some (or even all) of the objects from the - * list, due to refcounts falling to 0. - */ -static void -finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) -{ - destructor finalize; - PyGC_Head seen; - - /* While we're going through the loop, `finalize(op)` may cause op, or - * other objects, to be reclaimed via refcounts falling to zero. So - * there's little we can rely on about the structure of the input - * `collectable` list across iterations. For safety, we always take the - * first object in that list and move it to a temporary `seen` list. - * If objects vanish from the `collectable` and `seen` lists we don't - * care. - */ - gc_list_init(&seen); - - while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = GC_NEXT(collectable); - PyObject *op = FROM_GC(gc); - gc_list_move(gc, &seen); - if (!_PyGCHead_FINALIZED(gc) && - (finalize = Py_TYPE(op)->tp_finalize) != NULL) { - _PyGCHead_SET_FINALIZED(gc); - Py_INCREF(op); - finalize(op); - assert(!_PyErr_Occurred(tstate)); - Py_DECREF(op); - } - } - gc_list_merge(&seen, collectable); -} - -/* Break reference cycles by clearing the containers involved. This is - * tricky business as the lists can be changing and we don't know which - * objects may be freed. It is possible I screwed something up here. - */ -static void -delete_garbage(PyThreadState *tstate, GCState *gcstate, - PyGC_Head *collectable, PyGC_Head *old) -{ - assert(!_PyErr_Occurred(tstate)); - - while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = GC_NEXT(collectable); - PyObject *op = FROM_GC(gc); - - _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, - "refcount is too small"); - - if (gcstate->debug & DEBUG_SAVEALL) { - assert(gcstate->garbage != NULL); - if (PyList_Append(gcstate->garbage, op) < 0) { - _PyErr_Clear(tstate); - } - } - else { - inquiry clear; - if ((clear = Py_TYPE(op)->tp_clear) != NULL) { - Py_INCREF(op); - (void) clear(op); - if (_PyErr_Occurred(tstate)) { - _PyErr_WriteUnraisableMsg("in tp_clear of", - (PyObject*)Py_TYPE(op)); - } - Py_DECREF(op); - } - } - if (GC_NEXT(collectable) == gc) { - /* object is still alive, move it, it may die later */ - gc_clear_collecting(gc); - gc_list_move(gc, old); - } - } -} - -/* Clear all free lists - * All free lists are cleared during the collection of the highest generation. - * Allocated items in the free list may keep a pymalloc arena occupied. - * Clearing the free lists may give back memory to the OS earlier. - */ -static void -clear_freelists(PyInterpreterState *interp) -{ - _PyTuple_ClearFreeList(interp); - _PyFloat_ClearFreeList(interp); - _PyList_ClearFreeList(interp); - _PyDict_ClearFreeList(interp); - _PyAsyncGen_ClearFreeLists(interp); - _PyContext_ClearFreeList(interp); -} - -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - -/* Deduce which objects among "base" are unreachable from outside the list - and move them to 'unreachable'. The process consist in the following steps: - -1. Copy all reference counts to a different field (gc_prev is used to hold - this copy to save memory). -2. Traverse all objects in "base" and visit all referred objects using - "tp_traverse" and for every visited object, subtract 1 to the reference - count (the one that we copied in the previous step). After this step, all - objects that can be reached directly from outside must have strictly positive - reference count, while all unreachable objects must have a count of exactly 0. -3. Identify all unreachable objects (the ones with 0 reference count) and move - them to the "unreachable" list. This step also needs to move back to "base" all - objects that were initially marked as unreachable but are referred transitively - by the reachable objects (the ones with strictly positive reference count). - -Contracts: - - * The "base" has to be a valid list with no mask set. - - * The "unreachable" list must be uninitialized (this function calls - gc_list_init over 'unreachable'). - -IMPORTANT: This function leaves 'unreachable' with the NEXT_MASK_UNREACHABLE -flag set but it does not clear it to skip unnecessary iteration. Before the -flag is cleared (for example, by using 'clear_unreachable_mask' function or -by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal -list and we can not use most gc_list_* functions for it. */ -static inline void -deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { - validate_list(base, collecting_clear_unreachable_clear); - /* Using ob_refcnt and gc_refs, calculate which objects in the - * container set are reachable from outside the set (i.e., have a - * refcount greater than 0 when all the references within the - * set are taken into account). - */ - update_refs(base); // gc_prev is used for gc_refs - subtract_refs(base); - - /* Leave everything reachable from outside base in base, and move - * everything else (in base) to unreachable. - * - * NOTE: This used to move the reachable objects into a reachable - * set instead. But most things usually turn out to be reachable, - * so it's more efficient to move the unreachable things. It "sounds slick" - * to move the unreachable objects, until you think about it - the reason it - * pays isn't actually obvious. - * - * Suppose we create objects A, B, C in that order. They appear in the young - * generation in the same order. If B points to A, and C to B, and C is - * reachable from outside, then the adjusted refcounts will be 0, 0, and 1 - * respectively. - * - * When move_unreachable finds A, A is moved to the unreachable list. The - * same for B when it's first encountered. Then C is traversed, B is moved - * _back_ to the reachable list. B is eventually traversed, and then A is - * moved back to the reachable list. - * - * So instead of not moving at all, the reachable objects B and A are moved - * twice each. Why is this a win? A straightforward algorithm to move the - * reachable objects instead would move A, B, and C once each. - * - * The key is that this dance leaves the objects in order C, B, A - it's - * reversed from the original order. On all _subsequent_ scans, none of - * them will move. Since most objects aren't in cycles, this can save an - * unbounded number of moves across an unbounded number of later collections. - * It can cost more only the first time the chain is scanned. - * - * Drawback: move_unreachable is also used to find out what's still trash - * after finalizers may resurrect objects. In _that_ case most unreachable - * objects will remain unreachable, so it would be more efficient to move - * the reachable objects instead. But this is a one-time cost, probably not - * worth complicating the code to speed just a little. - */ - gc_list_init(unreachable); - move_unreachable(base, unreachable); // gc_prev is pointer again - validate_list(base, collecting_clear_unreachable_clear); - validate_list(unreachable, collecting_set_unreachable_set); -} - -/* Handle objects that may have resurrected after a call to 'finalize_garbage', moving - them to 'old_generation' and placing the rest on 'still_unreachable'. - - Contracts: - * After this function 'unreachable' must not be used anymore and 'still_unreachable' - will contain the objects that did not resurrect. - - * The "still_unreachable" list must be uninitialized (this function calls - gc_list_init over 'still_unreachable'). - -IMPORTANT: After a call to this function, the 'still_unreachable' set will have the -PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so -we can skip the expense of clearing the flag to avoid extra iteration. */ -static inline void -handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, - PyGC_Head *old_generation) -{ - // Remove the PREV_MASK_COLLECTING from unreachable - // to prepare it for a new call to 'deduce_unreachable' - gc_list_clear_collecting(unreachable); - - // After the call to deduce_unreachable, the 'still_unreachable' set will - // have the PREV_MARK_COLLECTING set, but the objects are going to be - // removed so we can skip the expense of clearing the flag. - PyGC_Head* resurrected = unreachable; - deduce_unreachable(resurrected, still_unreachable); - clear_unreachable_mask(still_unreachable); - - // Move the resurrected objects to the old generation for future collection. - gc_list_merge(resurrected, old_generation); -} - -/* This is the main function. Read this to understand how the - * collection process works. */ -static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, - Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, - int nofail) -{ - GC_STAT_ADD(generation, collections, 1); -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; - } -#endif - int i; - Py_ssize_t m = 0; /* # objects collected */ - Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - PyGC_Head *young; /* the generation we are examining */ - PyGC_Head *old; /* next older generation */ - PyGC_Head unreachable; /* non-problematic unreachable trash */ - PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ - GCState *gcstate = &tstate->interp->gc; - - // gc_collect_main() must not be called before _PyGC_Init - // or after _PyGC_Fini() - assert(gcstate->garbage != NULL); - assert(!_PyErr_Occurred(tstate)); - - if (gcstate->debug & DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); - } - - if (PyDTrace_GC_START_ENABLED()) - PyDTrace_GC_START(generation); - - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) - gcstate->generations[generation+1].count += 1; - for (i = 0; i <= generation; i++) - gcstate->generations[i].count = 0; - - /* merge younger generations with one we are currently collecting */ - for (i = 0; i < generation; i++) { - gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); - } - - /* handy references */ - young = GEN_HEAD(gcstate, generation); - if (generation < NUM_GENERATIONS-1) - old = GEN_HEAD(gcstate, generation+1); - else - old = young; - validate_list(old, collecting_clear_unreachable_clear); - - deduce_unreachable(young, &unreachable); - - untrack_tuples(young); - /* Move reachable objects to next generation. */ - if (young != old) { - if (generation == NUM_GENERATIONS - 2) { - gcstate->long_lived_pending += gc_list_size(young); - } - gc_list_merge(young, old); - } - else { - /* We only un-track dicts in full collections, to avoid quadratic - dict build-up. See issue #14775. */ - untrack_dicts(young); - gcstate->long_lived_pending = 0; - gcstate->long_lived_total = gc_list_size(young); - } - - /* All objects in unreachable are trash, but objects reachable from - * legacy finalizers (e.g. tp_del) can't safely be deleted. - */ - gc_list_init(&finalizers); - // NEXT_MASK_UNREACHABLE is cleared here. - // After move_legacy_finalizers(), unreachable is normal list. - move_legacy_finalizers(&unreachable, &finalizers); - /* finalizers contains the unreachable objects with a legacy finalizer; - * unreachable objects reachable *from* those are also uncollectable, - * and we move those into the finalizers list too. - */ - move_legacy_finalizer_reachable(&finalizers); - - validate_list(&finalizers, collecting_clear_unreachable_clear); - validate_list(&unreachable, collecting_set_unreachable_clear); - - /* Print debugging information. */ - if (gcstate->debug & DEBUG_COLLECTABLE) { - for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { - debug_cycle("collectable", FROM_GC(gc)); - } - } - - /* Clear weakrefs and invoke callbacks as necessary. */ - m += handle_weakrefs(&unreachable, old); - - validate_list(old, collecting_clear_unreachable_clear); - validate_list(&unreachable, collecting_set_unreachable_clear); - - /* Call tp_finalize on objects which have one. */ - finalize_garbage(tstate, &unreachable); - - /* Handle any objects that may have resurrected after the call - * to 'finalize_garbage' and continue the collection with the - * objects that are still unreachable */ - PyGC_Head final_unreachable; - handle_resurrected_objects(&unreachable, &final_unreachable, old); - - /* Call tp_clear on objects in the final_unreachable set. This will cause - * the reference cycles to be broken. It may also cause some objects - * in finalizers to be freed. - */ - m += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, old); - - /* Collect statistics on uncollectable objects found and print - * debugging information. */ - for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { - n++; - if (gcstate->debug & DEBUG_UNCOLLECTABLE) - debug_cycle("uncollectable", FROM_GC(gc)); - } - if (gcstate->debug & DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); - PySys_WriteStderr( - "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - n+m, n, d); - } - - /* Append instances in the uncollectable set to a Python - * reachable list of garbage. The programmer has to deal with - * this if they insist on creating this type of structure. - */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, old); - validate_list(old, collecting_clear_unreachable_clear); - - /* Clear free list only during the collection of the highest - * generation */ - if (generation == NUM_GENERATIONS-1) { - clear_freelists(tstate->interp); - } - - if (_PyErr_Occurred(tstate)) { - if (nofail) { - _PyErr_Clear(tstate); - } - else { - _PyErr_WriteUnraisableMsg("in garbage collection", NULL); - } - } - - /* Update stats */ - if (n_collected) { - *n_collected = m; - } - if (n_uncollectable) { - *n_uncollectable = n; - } - - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; - stats->collections++; - stats->collected += m; - stats->uncollectable += n; - - GC_STAT_ADD(generation, objects_collected, m); -#ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; - } -#endif - - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(n + m); - } - - assert(!_PyErr_Occurred(tstate)); - return n + m; -} - -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping + * See Python/gc.c for the implementation of the garbage collector. */ -static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) -{ - assert(!_PyErr_Occurred(tstate)); - - /* we may get called very early */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_WriteUnraisable(NULL); - return; - } - } - - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_WriteUnraisable(NULL); - return; - } - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); - } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); -} +#include "Python.h" +#include "pycore_gc.h" +#include "pycore_object.h" // _PyObject_IS_GC() +#include "pycore_pystate.h" // _PyInterpreterState_GET() -/* Perform garbage collection of a generation and invoke - * progress callbacks. - */ -static Py_ssize_t -gc_collect_with_callback(PyThreadState *tstate, int generation) -{ - assert(!_PyErr_Occurred(tstate)); - Py_ssize_t result, collected, uncollectable; - invoke_gc_callback(tstate, "start", generation, 0, 0); - result = gc_collect_main(tstate, generation, &collected, &uncollectable, 0); - invoke_gc_callback(tstate, "stop", generation, collected, uncollectable); - assert(!_PyErr_Occurred(tstate)); - return result; -} +typedef struct _gc_runtime_state GCState; -static Py_ssize_t -gc_collect_generations(PyThreadState *tstate) +static GCState * +get_gc_state(void) { - GCState *gcstate = &tstate->interp->gc; - /* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ - Py_ssize_t n = 0; - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - continue; - n = gc_collect_with_callback(tstate, i); - break; - } - } - return n; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; } +/*[clinic input] +module gc +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5c9690ecc842d79]*/ #include "clinic/gcmodule.c.h" /*[clinic input] @@ -1572,18 +90,7 @@ gc_collect_impl(PyObject *module, int generation) return -1; } - GCState *gcstate = &tstate->interp->gc; - Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - n = gc_collect_with_callback(tstate, generation); - gcstate->collecting = 0; - } - return n; + return _PyGC_Collect(tstate, generation, _Py_GC_REASON_MANUAL); } /*[clinic input] @@ -1627,24 +134,36 @@ gc_get_debug_impl(PyObject *module) return gcstate->debug; } -PyDoc_STRVAR(gc_set_thresh__doc__, -"set_threshold(threshold0, [threshold1, threshold2]) -> None\n" -"\n" -"Sets the collection thresholds. Setting threshold0 to zero disables\n" -"collection.\n"); +/*[clinic input] +gc.set_threshold + + threshold0: int + [ + threshold1: int + [ + threshold2: int + ] + ] + / + +Set the collection thresholds (the collection frequency). + +Setting 'threshold0' to zero disables collection. +[clinic start generated code]*/ static PyObject * -gc_set_threshold(PyObject *self, PyObject *args) +gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, + int threshold1, int group_right_2, int threshold2) +/*[clinic end generated code: output=2e3c7c7dd59060f3 input=0d9612db50984eec]*/ { GCState *gcstate = get_gc_state(); - if (!PyArg_ParseTuple(args, "i|ii:set_threshold", - &gcstate->generations[0].threshold, - &gcstate->generations[1].threshold, - &gcstate->generations[2].threshold)) - return NULL; - for (int i = 3; i < NUM_GENERATIONS; i++) { - /* generations higher than 2 get the same threshold */ - gcstate->generations[i].threshold = gcstate->generations[2].threshold; + + gcstate->young.threshold = threshold0; + if (group_right_1) { + gcstate->old[0].threshold = threshold1; + } + if (group_right_2) { + gcstate->old[1].threshold = threshold2; } Py_RETURN_NONE; } @@ -1661,9 +180,9 @@ gc_get_threshold_impl(PyObject *module) { GCState *gcstate = get_gc_state(); return Py_BuildValue("(iii)", - gcstate->generations[0].threshold, - gcstate->generations[1].threshold, - gcstate->generations[2].threshold); + gcstate->young.threshold, + gcstate->old[0].threshold, + 0); } /*[clinic input] @@ -1677,104 +196,100 @@ gc_get_count_impl(PyObject *module) /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/ { GCState *gcstate = get_gc_state(); + +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _gc_thread_state *gc = &tstate->gc; + + // Flush the local allocation count to the global count + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + gc->alloc_count = 0; +#endif + return Py_BuildValue("(iii)", - gcstate->generations[0].count, - gcstate->generations[1].count, - gcstate->generations[2].count); + gcstate->young.count, + gcstate->old[gcstate->visited_space].count, + gcstate->old[gcstate->visited_space^1].count); } -static int -referrersvisit(PyObject* obj, PyObject *objs) -{ - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(objs); i++) - if (PyTuple_GET_ITEM(objs, i) == obj) - return 1; - return 0; -} +/*[clinic input] +gc.get_referrers -static int -gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) -{ - PyGC_Head *gc; - PyObject *obj; - traverseproc traverse; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - obj = FROM_GC(gc); - traverse = Py_TYPE(obj)->tp_traverse; - if (obj == objs || obj == resultlist) - continue; - if (traverse(obj, (visitproc)referrersvisit, objs)) { - if (PyList_Append(resultlist, obj) < 0) - return 0; /* error */ - } - } - return 1; /* no error */ -} + *objs as args: object -PyDoc_STRVAR(gc_get_referrers__doc__, -"get_referrers(*objs) -> list\n\ -Return the list of objects that directly refer to any of objs."); +Return the list of objects that directly refer to any of 'objs'. +[clinic start generated code]*/ static PyObject * -gc_get_referrers(PyObject *self, PyObject *args) +gc_get_referrers_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=296a09587f6a86b5 input=bae96961b14a0922]*/ { if (PySys_Audit("gc.get_referrers", "(O)", args) < 0) { return NULL; } - PyObject *result = PyList_New(0); - if (!result) { - return NULL; - } - - GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; i++) { - if (!(gc_referrers_for(args, GEN_HEAD(gcstate, i), result))) { - Py_DECREF(result); - return NULL; - } - } - return result; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetReferrers(interp, args); } /* Append obj to list; return true if error (out of memory), false if OK. */ static int -referentsvisit(PyObject *obj, PyObject *list) +referentsvisit(PyObject *obj, void *arg) { + PyObject *list = arg; return PyList_Append(list, obj) < 0; } -PyDoc_STRVAR(gc_get_referents__doc__, -"get_referents(*objs) -> list\n\ -Return the list of objects that are directly referred to by objs."); +static int +append_referrents(PyObject *result, PyObject *args) +{ + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + if (!_PyObject_IS_GC(obj)) { + continue; + } + + traverseproc traverse = Py_TYPE(obj)->tp_traverse; + if (!traverse) { + continue; + } + if (traverse(obj, referentsvisit, result)) { + return -1; + } + } + return 0; +} + +/*[clinic input] +gc.get_referents + + *objs as args: object + +Return the list of objects that are directly referred to by 'objs'. +[clinic start generated code]*/ static PyObject * -gc_get_referents(PyObject *self, PyObject *args) +gc_get_referents_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ { - Py_ssize_t i; if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { return NULL; } + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); if (result == NULL) return NULL; - for (i = 0; i < PyTuple_GET_SIZE(args); i++) { - traverseproc traverse; - PyObject *obj = PyTuple_GET_ITEM(args, i); + // NOTE: stop the world is a no-op in default build + _PyEval_StopTheWorld(interp); + int err = append_referrents(result, args); + _PyEval_StartTheWorld(interp); - if (!_PyObject_IS_GC(obj)) - continue; - traverse = Py_TYPE(obj)->tp_traverse; - if (! traverse) - continue; - if (traverse(obj, (visitproc)referentsvisit, result)) { - Py_DECREF(result); - return NULL; - } + if (err < 0) { + Py_CLEAR(result); } + return result; } @@ -1793,54 +308,25 @@ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) /*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - int i; - PyObject* result; - GCState *gcstate = &tstate->interp->gc; - if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; } - result = PyList_New(0); - if (result == NULL) { - return NULL; - } - - /* If generation is passed, we extract only that generation */ - if (generation != -1) { - if (generation >= NUM_GENERATIONS) { - _PyErr_Format(tstate, PyExc_ValueError, - "generation parameter must be less than the number of " - "available generations (%i)", - NUM_GENERATIONS); - goto error; - } - - if (generation < 0) { - _PyErr_SetString(tstate, PyExc_ValueError, - "generation parameter cannot be negative"); - goto error; - } - - if (append_objects(result, GEN_HEAD(gcstate, generation))) { - goto error; - } - - return result; + if (generation >= NUM_GENERATIONS) { + return PyErr_Format(PyExc_ValueError, + "generation parameter must be less than the number of " + "available generations (%i)", + NUM_GENERATIONS); } - /* If generation is not passed or None, get all objects from all generations */ - for (i = 0; i < NUM_GENERATIONS; i++) { - if (append_objects(result, GEN_HEAD(gcstate, i))) { - goto error; - } + if (generation < -1) { + PyErr_SetString(PyExc_ValueError, + "generation parameter cannot be negative"); + return NULL; } - return result; -error: - Py_DECREF(result); - return NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetObjects(interp, (int)generation); } /*[clinic input] @@ -1892,7 +378,7 @@ gc_get_stats_impl(PyObject *module) /*[clinic input] -gc.is_tracked +gc.is_tracked -> bool obj: object / @@ -1902,21 +388,15 @@ Returns true if the object is tracked by the garbage collector. Simple atomic objects will return false. [clinic start generated code]*/ -static PyObject * -gc_is_tracked(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=14f0103423b28e31 input=d83057f170ea2723]*/ +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=91c8d086b7f47a33 input=423b98ec680c3126]*/ { - PyObject *result; - - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) - result = Py_True; - else - result = Py_False; - return Py_NewRef(result); + return PyObject_GC_IsTracked(obj); } /*[clinic input] -gc.is_finalized +gc.is_finalized -> bool obj: object / @@ -1924,14 +404,11 @@ gc.is_finalized Returns true if the object has been already finalized by the GC. [clinic start generated code]*/ -static PyObject * -gc_is_finalized(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=e1516ac119a918ed input=201d0c58f69ae390]*/ +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=401ff5d6fc660429 input=ca4d111c8f8c4e3a]*/ { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return PyObject_GC_IsFinalized(obj); } /*[clinic input] @@ -1948,11 +425,8 @@ static PyObject * gc_freeze_impl(PyObject *module) /*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ { - GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; ++i) { - gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); - gcstate->generations[i].count = 0; - } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyGC_Freeze(interp); Py_RETURN_NONE; } @@ -1968,9 +442,8 @@ static PyObject * gc_unfreeze_impl(PyObject *module) /*[clinic end generated code: output=1c15f2043b25e169 input=2dd52b170f4cef6c]*/ { - GCState *gcstate = get_gc_state(); - gc_list_merge(&gcstate->permanent_generation.head, - GEN_HEAD(gcstate, NUM_GENERATIONS-1)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyGC_Unfreeze(interp); Py_RETURN_NONE; } @@ -1984,8 +457,8 @@ static Py_ssize_t gc_get_freeze_count_impl(PyObject *module) /*[clinic end generated code: output=61cbd9f43aa032e1 input=45ffbc65cfe2a6ed]*/ { - GCState *gcstate = get_gc_state(); - return gc_list_size(&gcstate->permanent_generation.head); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return _PyGC_GetFreezeCount(interp); } @@ -2018,17 +491,15 @@ static PyMethodDef GcMethods[] = { GC_SET_DEBUG_METHODDEF GC_GET_DEBUG_METHODDEF GC_GET_COUNT_METHODDEF - {"set_threshold", gc_set_threshold, METH_VARARGS, gc_set_thresh__doc__}, + GC_SET_THRESHOLD_METHODDEF GC_GET_THRESHOLD_METHODDEF GC_COLLECT_METHODDEF GC_GET_OBJECTS_METHODDEF GC_GET_STATS_METHODDEF GC_IS_TRACKED_METHODDEF GC_IS_FINALIZED_METHODDEF - {"get_referrers", gc_get_referrers, METH_VARARGS, - gc_get_referrers__doc__}, - {"get_referents", gc_get_referents, METH_VARARGS, - gc_get_referents__doc__}, + GC_GET_REFERRERS_METHODDEF + GC_GET_REFERENTS_METHODDEF GC_FREEZE_METHODDEF GC_UNFREEZE_METHODDEF GC_GET_FREEZE_COUNT_METHODDEF @@ -2051,7 +522,7 @@ gcmodule_exec(PyObject *module) return -1; } -#define ADD_INT(NAME) if (PyModule_AddIntConstant(module, #NAME, NAME) < 0) { return -1; } +#define ADD_INT(NAME) if (PyModule_AddIntConstant(module, #NAME, _PyGC_ ## NAME) < 0) { return -1; } ADD_INT(DEBUG_STATS); ADD_INT(DEBUG_COLLECTABLE); ADD_INT(DEBUG_UNCOLLECTABLE); @@ -2064,6 +535,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -2081,375 +553,3 @@ PyInit_gc(void) { return PyModuleDef_Init(&gcmodule); } - -/* C API for controlling the state of the garbage collector */ -int -PyGC_Enable(void) -{ - GCState *gcstate = get_gc_state(); - int old_state = gcstate->enabled; - gcstate->enabled = 1; - return old_state; -} - -int -PyGC_Disable(void) -{ - GCState *gcstate = get_gc_state(); - int old_state = gcstate->enabled; - gcstate->enabled = 0; - return old_state; -} - -int -PyGC_IsEnabled(void) -{ - GCState *gcstate = get_gc_state(); - return gcstate->enabled; -} - -/* Public API to invoke gc.collect() from C */ -Py_ssize_t -PyGC_Collect(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - - if (!gcstate->enabled) { - return 0; - } - - Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_with_callback(tstate, NUM_GENERATIONS - 1); - _PyErr_SetRaisedException(tstate, exc); - gcstate->collecting = 0; - } - - return n; -} - -Py_ssize_t -_PyGC_CollectNoFail(PyThreadState *tstate) -{ - /* Ideally, this function is only called on interpreter shutdown, - and therefore not recursively. Unfortunately, when there are daemon - threads, a daemon thread can start a cyclic garbage collection - during interpreter shutdown (and then never finish it). - See http://bugs.python.org/issue8713#msg195178 for an example. - */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->collecting) { - return 0; - } - - Py_ssize_t n; - gcstate->collecting = 1; - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, NULL, NULL, 1); - gcstate->collecting = 0; - return n; -} - -void -_PyGC_DumpShutdownStats(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - if (!(gcstate->debug & DEBUG_SAVEALL) - && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { - const char *message; - if (gcstate->debug & DEBUG_UNCOLLECTABLE) - message = "gc: %zd uncollectable objects at " \ - "shutdown"; - else - message = "gc: %zd uncollectable objects at " \ - "shutdown; use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; - /* PyErr_WarnFormat does too many things and we are at shutdown, - the warnings module's dependencies (e.g. linecache) may be gone - already. */ - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, message, - PyList_GET_SIZE(gcstate->garbage))) - PyErr_WriteUnraisable(NULL); - if (gcstate->debug & DEBUG_UNCOLLECTABLE) { - PyObject *repr = NULL, *bytes = NULL; - repr = PyObject_Repr(gcstate->garbage); - if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) - PyErr_WriteUnraisable(gcstate->garbage); - else { - PySys_WriteStderr( - " %s\n", - PyBytes_AS_STRING(bytes) - ); - } - Py_XDECREF(repr); - Py_XDECREF(bytes); - } - } -} - - -void -_PyGC_Fini(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - Py_CLEAR(gcstate->garbage); - Py_CLEAR(gcstate->callbacks); - - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ -} - -/* for debugging */ -void -_PyGC_Dump(PyGC_Head *g) -{ - _PyObject_Dump(FROM_GC(g)); -} - - -#ifdef Py_DEBUG -static int -visit_validate(PyObject *op, void *parent_raw) -{ - PyObject *parent = _PyObject_CAST(parent_raw); - if (_PyObject_IsFreed(op)) { - _PyObject_ASSERT_FAILED_MSG(parent, - "PyObject_GC_Track() object is not valid"); - } - return 0; -} -#endif - - -/* extension modules might be compiled with GC support so these - functions must always be available */ - -void -PyObject_GC_Track(void *op_raw) -{ - PyObject *op = _PyObject_CAST(op_raw); - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_ASSERT_FAILED_MSG(op, - "object already tracked " - "by the garbage collector"); - } - _PyObject_GC_TRACK(op); - -#ifdef Py_DEBUG - /* Check that the object is valid: validate objects traversed - by tp_traverse() */ - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void)traverse(op, visit_validate, op); -#endif -} - -void -PyObject_GC_UnTrack(void *op_raw) -{ - PyObject *op = _PyObject_CAST(op_raw); - /* Obscure: the Py_TRASHCAN mechanism requires that we be able to - * call PyObject_GC_UnTrack twice on an object. - */ - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_GC_UNTRACK(op); - } -} - -int -PyObject_IS_GC(PyObject *obj) -{ - return _PyObject_IS_GC(obj); -} - -void -_Py_ScheduleGC(PyInterpreterState *interp) -{ - GCState *gcstate = &interp->gc; - if (gcstate->collecting == 1) { - return; - } - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); -} - -void -_PyObject_GC_Link(PyObject *op) -{ - PyGC_Head *g = AS_GC(op); - assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0); // g must be correctly aligned - - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - g->_gc_next = 0; - g->_gc_prev = 0; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && - gcstate->enabled && - gcstate->generations[0].threshold && - !gcstate->collecting && - !_PyErr_Occurred(tstate)) - { - _Py_ScheduleGC(tstate->interp); - } -} - -void -_Py_RunGC(PyThreadState *tstate) -{ - GCState *gcstate = &tstate->interp->gc; - gcstate->collecting = 1; - gc_collect_generations(tstate); - gcstate->collecting = 0; -} - -static PyObject * -gc_alloc(size_t basicsize, size_t presize) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (basicsize > PY_SSIZE_T_MAX - presize) { - return _PyErr_NoMemory(tstate); - } - size_t size = presize + basicsize; - char *mem = PyObject_Malloc(size); - if (mem == NULL) { - return _PyErr_NoMemory(tstate); - } - ((PyObject **)mem)[0] = NULL; - ((PyObject **)mem)[1] = NULL; - PyObject *op = (PyObject *)(mem + presize); - _PyObject_GC_Link(op); - return op; -} - -PyObject * -_PyObject_GC_New(PyTypeObject *tp) -{ - size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(_PyObject_SIZE(tp), presize); - if (op == NULL) { - return NULL; - } - _PyObject_Init(op, tp); - return op; -} - -PyVarObject * -_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) -{ - PyVarObject *op; - - if (nitems < 0) { - PyErr_BadInternalCall(); - return NULL; - } - size_t presize = _PyType_PreHeaderSize(tp); - size_t size = _PyObject_VAR_SIZE(tp, nitems); - op = (PyVarObject *)gc_alloc(size, presize); - if (op == NULL) { - return NULL; - } - _PyObject_InitVar(op, tp, nitems); - return op; -} - -PyObject * -PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) -{ - size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize); - if (op == NULL) { - return NULL; - } - memset(op, 0, _PyObject_SIZE(tp) + extra_size); - _PyObject_Init(op, tp); - return op; -} - -PyVarObject * -_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) -{ - const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); - const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); - _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); - if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { - return (PyVarObject *)PyErr_NoMemory(); - } - char *mem = (char *)op - presize; - mem = (char *)PyObject_Realloc(mem, presize + basicsize); - if (mem == NULL) { - return (PyVarObject *)PyErr_NoMemory(); - } - op = (PyVarObject *) (mem + presize); - Py_SET_SIZE(op, nitems); - return op; -} - -void -PyObject_GC_Del(void *op) -{ - size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); - PyGC_Head *g = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op)) { -#ifdef Py_DEBUG - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, "Object of type %s is not untracked before destruction", - ((PyObject*)op)->ob_type->tp_name)) { - PyErr_WriteUnraisable(NULL); - } -#endif - gc_list_remove(g); - } - GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; - } - PyObject_Free(((char *)op)-presize); -} - -int -PyObject_GC_IsTracked(PyObject* obj) -{ - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { - return 1; - } - return 0; -} - -int -PyObject_GC_IsFinalized(PyObject *obj) -{ - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - return 1; - } - return 0; -} - -void -PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) -{ - size_t i; - GCState *gcstate = get_gc_state(); - int origenstate = gcstate->enabled; - gcstate->enabled = 0; - for (i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gc_list, *gc; - gc_list = GEN_HEAD(gcstate, i); - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - Py_INCREF(op); - int res = callback(op, arg); - Py_DECREF(op); - if (!res) { - goto done; - } - } - } -done: - gcstate->enabled = origenstate; -} diff --git a/Modules/getpath.c b/Modules/getpath.c index 6f76a84e78bf62..d0128b20faeeae 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -4,7 +4,6 @@ #include "pycore_fileutils.h" // _Py_abspath() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_pathconfig.h" // _PyPathConfig_ReadGlobal() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pymem.h" // _PyMem_RawWcsdup() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -18,11 +17,12 @@ #endif #ifdef __APPLE__ +# include # include #endif /* Reference the precompiled getpath.py */ -#include "../Python/frozen_modules/getpath.h" +#include "Python/frozen_modules/getpath.h" #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \ || !defined(VERSION) || !defined(VPATH) \ @@ -262,6 +262,10 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + if (parts == NULL) { + PyErr_NoMemory(); + return NULL; + } memset(parts, 0, n * sizeof(wchar_t *)); Py_ssize_t cchFinal = 0; Py_ssize_t first = 0; @@ -502,6 +506,54 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) PyMem_Free((void *)path); PyMem_Free((void *)narrow); return r; +#elif defined(MS_WINDOWS) + HANDLE hFile; + wchar_t resolved[MAXPATHLEN+1]; + int len = 0, err; + Py_ssize_t pathlen; + PyObject *result; + + wchar_t *path = PyUnicode_AsWideCharString(pathobj, &pathlen); + if (!path) { + return NULL; + } + if (wcslen(path) != pathlen) { + PyErr_SetString(PyExc_ValueError, "path contains embedded nulls"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + hFile = CreateFileW(path, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hFile != INVALID_HANDLE_VALUE) { + len = GetFinalPathNameByHandleW(hFile, resolved, MAXPATHLEN, VOLUME_NAME_DOS); + err = len ? 0 : GetLastError(); + CloseHandle(hFile); + } else { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetFromWindowsErr(err); + result = NULL; + } else if (len <= MAXPATHLEN) { + const wchar_t *p = resolved; + if (0 == wcsncmp(p, L"\\\\?\\", 4)) { + if (GetFileAttributesW(&p[4]) != INVALID_FILE_ATTRIBUTES) { + p += 4; + len -= 4; + } + } + if (CompareStringOrdinal(path, (int)pathlen, p, len, TRUE) == CSTR_EQUAL) { + result = Py_NewRef(pathobj); + } else { + result = PyUnicode_FromWideChar(p, len); + } + } else { + result = Py_NewRef(pathobj); + } + PyMem_Free(path); + return result; #endif return Py_NewRef(pathobj); @@ -769,16 +821,11 @@ library_to_dict(PyObject *dict, const char *key) which is in the framework, not relative to the executable, which may be outside of the framework. Except when we're in the build directory... */ - NSSymbol symbol = NSLookupAndBindSymbol("_Py_Initialize"); - if (symbol != NULL) { - NSModule pythonModule = NSModuleForSymbol(symbol); - if (pythonModule != NULL) { - /* Use dylib functions to find out where the framework was loaded from */ - const char *path = NSLibraryNameForModule(pythonModule); - if (path) { - strncpy(modPath, path, MAXPATHLEN); - modPathInitialized = 1; - } + Dl_info pythonInfo; + if (dladdr(&Py_Initialize, &pythonInfo)) { + if (pythonInfo.dli_fname) { + strncpy(modPath, pythonInfo.dli_fname, MAXPATHLEN); + modPathInitialized = 1; } } } @@ -904,6 +951,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifdef Py_GIL_DISABLED + !decode_to_dict(dict, "ABI_THREAD", "t") || +#else + !decode_to_dict(dict, "ABI_THREAD", "") || +#endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif @@ -911,7 +963,7 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) ) { Py_DECREF(co); Py_DECREF(dict); - _PyErr_WriteUnraisableMsg("error evaluating initial values", NULL); + PyErr_FormatUnraisable("Exception ignored in preparing getpath"); return PyStatus_Error("error evaluating initial values"); } @@ -920,13 +972,13 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) if (!r) { Py_DECREF(dict); - _PyErr_WriteUnraisableMsg("error evaluating path", NULL); + PyErr_FormatUnraisable("Exception ignored in running getpath"); return PyStatus_Error("error evaluating path"); } Py_DECREF(r); if (_PyConfig_FromDict(config, configDict) < 0) { - _PyErr_WriteUnraisableMsg("reading getpath results", NULL); + PyErr_FormatUnraisable("Exception ignored in reading getpath results"); Py_DECREF(dict); return PyStatus_Error("error getting getpath results"); } diff --git a/Modules/getpath.py b/Modules/getpath.py index 9913fcba497d30..1f1bfcb4f64dd4 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -40,6 +40,7 @@ # EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar # VERSION_MAJOR -- [in] sys.version_info.major # VERSION_MINOR -- [in] sys.version_info.minor +# ABI_THREAD -- [in] either 't' for free-threaded builds or '' # PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) # ** Values read from the environment ** @@ -172,17 +173,18 @@ # ****************************************************************************** platlibdir = config.get('platlibdir') or PLATLIBDIR +ABI_THREAD = ABI_THREAD or '' if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' - STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] - PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload' BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] VENV_LANDMARK = 'pyvenv.cfg' - ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' @@ -229,17 +231,16 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath = config.get('module_search_paths') pythonpath_was_set = config.get('module_search_paths_set') +stdlib_dir = config.get('stdlib_dir') +stdlib_dir_was_set_in_config = bool(stdlib_dir) real_executable_dir = None -stdlib_dir = None platstdlib_dir = None # ****************************************************************************** # CALCULATE program_name # ****************************************************************************** -program_name_was_set = bool(program_name) - if not program_name: try: program_name = config.get('orig_argv', [])[0] @@ -311,7 +312,10 @@ def search_up(prefix, *landmarks, test=isfile): # and should not affect base_executable. base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" else: - base_executable = executable + # Use the real executable as our base, or argv[0] otherwise + # (on Windows, argv[0] is likely to be ENV___PYVENV_LAUNCHER__; on + # other platforms, real_executable is likely to be empty) + base_executable = real_executable or executable if not real_executable: real_executable = base_executable @@ -409,13 +413,14 @@ def search_up(prefix, *landmarks, test=isfile): if not real_executable: real_executable = base_executable -try: - real_executable = realpath(real_executable) -except OSError as ex: - # Only warn if the file actually exists and was unresolvable - # Otherwise users who specify a fake executable may get spurious warnings. - if isfile(real_executable): - warn(f'Failed to find real location of {base_executable}') +if real_executable: + try: + real_executable = realpath(real_executable) + except OSError as ex: + # Only warn if the file actually exists and was unresolvable + # Otherwise users who specify a fake executable may get spurious warnings. + if isfile(real_executable): + warn(f'Failed to find real location of {base_executable}') if not executable_dir and os_name == 'darwin' and library: # QUIRK: macOS checks adjacent to its library early @@ -428,12 +433,12 @@ def search_up(prefix, *landmarks, test=isfile): # If we do not have the executable's directory, we can calculate it. # This is the directory used to find prefix/exec_prefix if necessary. -if not executable_dir: +if not executable_dir and real_executable: executable_dir = real_executable_dir = dirname(real_executable) # If we do not have the real executable's directory, we calculate it. # This is the directory used to detect build layouts. -if not real_executable_dir: +if not real_executable_dir and real_executable: real_executable_dir = dirname(real_executable) # ****************************************************************************** @@ -509,11 +514,12 @@ def search_up(prefix, *landmarks, test=isfile): build_stdlib_prefix = build_prefix else: build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) - # Always use the build prefix for stdlib - if build_stdlib_prefix: - stdlib_dir = joinpath(build_stdlib_prefix, 'Lib') - else: - stdlib_dir = joinpath(build_prefix, 'Lib') + # Use the build prefix for stdlib when not explicitly set + if not stdlib_dir_was_set_in_config: + if build_stdlib_prefix: + stdlib_dir = joinpath(build_stdlib_prefix, 'Lib') + else: + stdlib_dir = joinpath(build_prefix, 'Lib') # Only use the build prefix for prefix if it hasn't already been set if not prefix: prefix = build_stdlib_prefix @@ -545,8 +551,9 @@ def search_up(prefix, *landmarks, test=isfile): prefix, had_delim, exec_prefix = home.partition(DELIM) if not had_delim: exec_prefix = prefix - # Reset the standard library directory if it was already set - stdlib_dir = None + # Reset the standard library directory if it was not explicitly set + if not stdlib_dir_was_set_in_config: + stdlib_dir = None # First try to detect prefix by looking alongside our runtime library, if known @@ -562,7 +569,8 @@ def search_up(prefix, *landmarks, test=isfile): if STDLIB_SUBDIR and STDLIB_LANDMARKS and not prefix: if any(isfile(joinpath(library_dir, f)) for f in STDLIB_LANDMARKS): prefix = library_dir - stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + if not stdlib_dir_was_set_in_config: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) # Detect prefix by looking for zip file @@ -573,7 +581,7 @@ def search_up(prefix, *landmarks, test=isfile): prefix = executable_dir else: prefix = search_up(executable_dir, ZIP_LANDMARK) - if prefix: + if prefix and not stdlib_dir_was_set_in_config: stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) if not isdir(stdlib_dir): stdlib_dir = None diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 20e83de84e8340..f7d3e12f347ec2 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -1,10 +1,17 @@ - /* UNIX group file access module */ +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +#define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getgrgid_r() +#include // memcpy() #include // sysconf() #include "clinic/grpmodule.c.h" @@ -83,7 +90,7 @@ mkgrent(PyObject *module, struct group *p) Py_DECREF(x); } -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, val) +#define SET(i,val) PyStructSequence_SetItem(v, i, val) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_name)); if (p->gr_passwd) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_passwd)); @@ -335,6 +342,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/hashlib.h b/Modules/hashlib.h index a8bad9dd87a939..7105e68af7b806 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -1,5 +1,7 @@ /* Common code for use by all hashlib related modules. */ +#include "pycore_lock.h" // PyMutex + /* * Given a PyObject* obj, fill in the Py_buffer* viewp with the result * of PyObject_GetBuffer. Sets an exception and issues the erraction @@ -48,18 +50,28 @@ #include "pythread.h" #define ENTER_HASHLIB(obj) \ - if ((obj)->lock) { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } \ + if ((obj)->use_mutex) { \ + PyMutex_Lock(&(obj)->mutex); \ } #define LEAVE_HASHLIB(obj) \ - if ((obj)->lock) { \ - PyThread_release_lock((obj)->lock); \ + if ((obj)->use_mutex) { \ + PyMutex_Unlock(&(obj)->mutex); \ } +#ifdef Py_GIL_DISABLED +#define HASHLIB_INIT_MUTEX(obj) \ + do { \ + (obj)->mutex = (PyMutex){0}; \ + (obj)->use_mutex = true; \ + } while (0) +#else +#define HASHLIB_INIT_MUTEX(obj) \ + do { \ + (obj)->mutex = (PyMutex){0}; \ + (obj)->use_mutex = false; \ + } while (0) +#endif + /* TODO(gpshead): We should make this a module or class attribute * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4ed34aa5bde827..e740ec4d7625c3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,13 +1,14 @@ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_tuple.h" // _PyTuple_ITEMS() -#include // offsetof() +#include // offsetof() /* Itertools module written and maintained by Raymond D. Hettinger @@ -93,32 +94,14 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" #undef clinic_state_by_cls #undef clinic_state -/* Deprecation of pickle support: GH-101588 *********************************/ - -#define ITERTOOL_PICKLE_DEPRECATION \ - if (PyErr_WarnEx( \ - PyExc_DeprecationWarning, \ - "Pickle, copy, and deepcopy support will be " \ - "removed from itertools in Python 3.14.", 1) < 0) { \ - return NULL; \ - } /* batched object ************************************************************/ -/* Note: The built-in zip() function includes a "strict" argument - that was needed because that function would silently truncate data, - and there was no easy way for a user to detect the data loss. - The same reasoning does not apply to batched() which never drops data. - Instead, batched() produces a shorter tuple which can be handled - as the user sees fit. If requested, it would be reasonable to add - "fillvalue" support which had demonstrated value in zip_longest(). - For now, the API is kept simple and clean. - */ - typedef struct { PyObject_HEAD PyObject *it; Py_ssize_t batch_size; + bool strict; } batchedobject; /*[clinic input] @@ -126,6 +109,9 @@ typedef struct { itertools.batched.__new__ as batched_new iterable: object n: Py_ssize_t + * + strict: bool = False + Batch data into tuples of length n. The last batch may be shorter than n. Loops over the input iterable and accumulates data into tuples @@ -140,11 +126,15 @@ or when the input iterable is exhausted. ('D', 'E', 'F') ('G',) +If "strict" is True, raises a ValueError if the final batch is shorter +than n. + [clinic start generated code]*/ static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=7ebc954d655371b6 input=ffd70726927c5129]*/ +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict) +/*[clinic end generated code: output=c6de11b061529d3e input=7814b47e222f5467]*/ { PyObject *it; batchedobject *bo; @@ -170,6 +160,7 @@ batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) } bo->batch_size = n; bo->it = it; + bo->strict = (bool) strict; return (PyObject *)bo; } @@ -233,6 +224,12 @@ batched_next(batchedobject *bo) Py_DECREF(result); return NULL; } + if (bo->strict) { + Py_CLEAR(bo->it); + Py_DECREF(result); + PyErr_SetString(PyExc_ValueError, "batched(): incomplete batch"); + return NULL; + } _PyTuple_Resize(&result, i); return result; } @@ -265,6 +262,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -296,6 +294,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -306,6 +309,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -316,6 +320,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -330,21 +335,52 @@ pairwise_next(pairwiseobject *po) return NULL; } if (old == NULL) { - po->old = old = (*Py_TYPE(it)->tp_iternext)(it); + old = (*Py_TYPE(it)->tp_iternext)(it); + Py_XSETREF(po->old, old); if (old == NULL) { Py_CLEAR(po->it); return NULL; } + it = po->it; + if (it == NULL) { + Py_CLEAR(po->old); + return NULL; + } } + Py_INCREF(old); new = (*Py_TYPE(it)->tp_iternext)(it); if (new == NULL) { Py_CLEAR(po->it); Py_CLEAR(po->old); + Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); - Py_SETREF(po->old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + + Py_XSETREF(po->old, new); + Py_DECREF(old); return result; } @@ -510,57 +546,6 @@ groupby_next(groupbyobject *gbo) return r; } -static PyObject * -groupby_reduce(groupbyobject *lz, PyObject *Py_UNUSED(ignored)) -{ - /* reduce as a 'new' call with an optional 'setstate' if groupby - * has started - */ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *value; - if (lz->tgtkey && lz->currkey && lz->currvalue) - value = Py_BuildValue("O(OO)(OOO)", Py_TYPE(lz), - lz->it, lz->keyfunc, lz->currkey, lz->currvalue, lz->tgtkey); - else - value = Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->it, lz->keyfunc); - - return value; -} - -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -groupby_setstate(groupbyobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *currkey, *currvalue, *tgtkey; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "OOO", &currkey, &currvalue, &tgtkey)) { - return NULL; - } - Py_INCREF(currkey); - Py_XSETREF(lz->currkey, currkey); - Py_INCREF(currvalue); - Py_XSETREF(lz->currvalue, currvalue); - Py_INCREF(tgtkey); - Py_XSETREF(lz->tgtkey, tgtkey); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - -static PyMethodDef groupby_methods[] = { - {"__reduce__", (PyCFunction)groupby_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)groupby_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot groupby_slots[] = { {Py_tp_dealloc, groupby_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -568,7 +553,6 @@ static PyType_Slot groupby_slots[] = { {Py_tp_traverse, groupby_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, groupby_next}, - {Py_tp_methods, groupby_methods}, {Py_tp_new, itertools_groupby}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -669,29 +653,12 @@ _grouper_next(_grouperobject *igo) return r; } -static PyObject * -_grouper_reduce(_grouperobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (((groupbyobject *)lz->parent)->currgrouper != lz) { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); - } - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); -} - -static PyMethodDef _grouper_methods[] = { - {"__reduce__", (PyCFunction)_grouper_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, _grouper_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, _grouper_next}, - {Py_tp_methods, _grouper_methods}, {Py_tp_new, itertools__grouper}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -801,10 +768,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) +teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_IS_TYPE(obj, tdo_type) && - Py_REFCNT(obj) == 1) { + while (obj && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; Py_SETREF(obj, nextlink); @@ -823,8 +789,7 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); - teedataobject_safe_decref(tmp, state->teedataobject_type); + teedataobject_safe_decref(tmp); return 0; } @@ -838,25 +803,6 @@ teedataobject_dealloc(teedataobject *tdo) Py_DECREF(tp); } -static PyObject * -teedataobject_reduce(teedataobject *tdo, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - int i; - /* create a temporary list of already iterated values */ - PyObject *values = PyList_New(tdo->numread); - - if (!values) - return NULL; - for (i=0 ; inumread ; i++) { - Py_INCREF(tdo->values[i]); - PyList_SET_ITEM(values, i, tdo->values[i]); - } - return Py_BuildValue("O(ONO)", Py_TYPE(tdo), tdo->it, - values, - tdo->nextlink ? tdo->nextlink : Py_None); -} - /*[clinic input] @classmethod itertools.teedataobject.__new__ @@ -911,19 +857,12 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, return NULL; } -static PyMethodDef teedataobject_methods[] = { - {"__reduce__", (PyCFunction)teedataobject_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot teedataobject_slots[] = { {Py_tp_dealloc, teedataobject_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_doc, (void *)itertools_teedataobject__doc__}, {Py_tp_traverse, teedataobject_traverse}, {Py_tp_clear, teedataobject_clear}, - {Py_tp_methods, teedataobject_methods}, {Py_tp_new, itertools_teedataobject}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1052,41 +991,8 @@ tee_dealloc(teeobject *to) Py_DECREF(tp); } -static PyObject * -tee_reduce(teeobject *to, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(())(Oi)", Py_TYPE(to), to->dataobj, to->index); -} - -static PyObject * -tee_setstate(teeobject *to, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - teedataobject *tdo; - int index; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - PyTypeObject *tdo_type = to->state->teedataobject_type; - if (!PyArg_ParseTuple(state, "O!i", tdo_type, &tdo, &index)) { - return NULL; - } - if (index < 0 || index > LINKCELLS) { - PyErr_SetString(PyExc_ValueError, "Index out of range"); - return NULL; - } - Py_INCREF(tdo); - Py_XSETREF(to->dataobj, tdo); - to->index = index; - Py_RETURN_NONE; -} - static PyMethodDef tee_methods[] = { {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1288,59 +1194,6 @@ cycle_next(cycleobject *lz) return Py_NewRef(item); } -static PyObject * -cycle_reduce(cycleobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new cycle with the iterator tuple, then set the saved state */ - if (lz->it == NULL) { - PyObject *it = PyObject_GetIter(lz->saved); - if (it == NULL) - return NULL; - if (lz->index != 0) { - PyObject *res = _PyObject_CallMethod(it, &_Py_ID(__setstate__), - "n", lz->index); - if (res == NULL) { - Py_DECREF(it); - return NULL; - } - Py_DECREF(res); - } - return Py_BuildValue("O(N)(OO)", Py_TYPE(lz), it, lz->saved, Py_True); - } - return Py_BuildValue("O(O)(OO)", Py_TYPE(lz), lz->it, lz->saved, - lz->firstpass ? Py_True : Py_False); -} - -static PyObject * -cycle_setstate(cycleobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *saved=NULL; - int firstpass; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - // The second item can be 1/0 in old pickles and True/False in new pickles - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { - return NULL; - } - Py_INCREF(saved); - Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass != 0; - lz->index = 0; - Py_RETURN_NONE; -} - -static PyMethodDef cycle_methods[] = { - {"__reduce__", (PyCFunction)cycle_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cycle_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot cycle_slots[] = { {Py_tp_dealloc, cycle_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1348,7 +1201,6 @@ static PyType_Slot cycle_slots[] = { {Py_tp_traverse, cycle_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, cycle_next}, - {Py_tp_methods, cycle_methods}, {Py_tp_new, itertools_cycle}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1461,32 +1313,6 @@ dropwhile_next(dropwhileobject *lz) } } -static PyObject * -dropwhile_reduce(dropwhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); -} - -static PyObject * -dropwhile_setstate(dropwhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int start = PyObject_IsTrue(state); - if (start < 0) - return NULL; - lz->start = start; - Py_RETURN_NONE; -} - -static PyMethodDef dropwhile_methods[] = { - {"__reduce__", (PyCFunction)dropwhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)dropwhile_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot dropwhile_slots[] = { {Py_tp_dealloc, dropwhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1494,7 +1320,6 @@ static PyType_Slot dropwhile_slots[] = { {Py_tp_traverse, dropwhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, dropwhile_next}, - {Py_tp_methods, dropwhile_methods}, {Py_tp_new, itertools_dropwhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1601,33 +1426,6 @@ takewhile_next(takewhileobject *lz) return NULL; } -static PyObject * -takewhile_reduce(takewhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); -} - -static PyObject * -takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int stop = PyObject_IsTrue(state); - - if (stop < 0) - return NULL; - lz->stop = stop; - Py_RETURN_NONE; -} - -static PyMethodDef takewhile_reduce_methods[] = { - {"__reduce__", (PyCFunction)takewhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)takewhile_reduce_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot takewhile_slots[] = { {Py_tp_dealloc, takewhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1635,7 +1433,6 @@ static PyType_Slot takewhile_slots[] = { {Py_tp_traverse, takewhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, takewhile_next}, - {Py_tp_methods, takewhile_reduce_methods}, {Py_tp_new, itertools_takewhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1805,59 +1602,6 @@ islice_next(isliceobject *lz) return NULL; } -static PyObject * -islice_reduce(isliceobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* When unpickled, generate a new object with the same bounds, - * then 'setstate' with the next and count - */ - PyObject *stop; - - if (lz->it == NULL) { - PyObject *empty_list; - PyObject *empty_it; - empty_list = PyList_New(0); - if (empty_list == NULL) - return NULL; - empty_it = PyObject_GetIter(empty_list); - Py_DECREF(empty_list); - if (empty_it == NULL) - return NULL; - return Py_BuildValue("O(Nn)n", Py_TYPE(lz), empty_it, 0, 0); - } - if (lz->stop == -1) { - stop = Py_NewRef(Py_None); - } else { - stop = PyLong_FromSsize_t(lz->stop); - if (stop == NULL) - return NULL; - } - return Py_BuildValue("O(OnNn)n", Py_TYPE(lz), - lz->it, lz->next, stop, lz->step, - lz->cnt); -} - -static PyObject * -islice_setstate(isliceobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_ssize_t cnt = PyLong_AsSsize_t(state); - - if (cnt == -1 && PyErr_Occurred()) - return NULL; - lz->cnt = cnt; - Py_RETURN_NONE; -} - -static PyMethodDef islice_methods[] = { - {"__reduce__", (PyCFunction)islice_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)islice_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(islice_doc, "islice(iterable, stop) --> islice object\n\ islice(iterable, start, stop[, step]) --> islice object\n\ @@ -1876,7 +1620,6 @@ static PyType_Slot islice_slots[] = { {Py_tp_traverse, islice_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, islice_next}, - {Py_tp_methods, islice_methods}, {Py_tp_new, islice_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1974,20 +1717,6 @@ starmap_next(starmapobject *lz) return result; } -static PyObject * -starmap_reduce(starmapobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Just pickle the iterator */ - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef starmap_methods[] = { - {"__reduce__", (PyCFunction)starmap_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot starmap_slots[] = { {Py_tp_dealloc, starmap_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1995,7 +1724,6 @@ static PyType_Slot starmap_slots[] = { {Py_tp_traverse, starmap_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, starmap_next}, - {Py_tp_methods, starmap_methods}, {Py_tp_new, itertools_starmap}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -2131,53 +1859,9 @@ chain_next(chainobject *lz) return NULL; } -static PyObject * -chain_reduce(chainobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->source) { - /* we can't pickle function objects (itertools.from_iterable) so - * we must use setstate to replace the iterable. One day we - * will fix pickling of functions - */ - if (lz->active) { - return Py_BuildValue("O()(OO)", Py_TYPE(lz), lz->source, lz->active); - } else { - return Py_BuildValue("O()(O)", Py_TYPE(lz), lz->source); - } - } else { - return Py_BuildValue("O()", Py_TYPE(lz)); /* exhausted */ - } - return NULL; -} - -static PyObject * -chain_setstate(chainobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *source, *active=NULL; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O|O", &source, &active)) { - return NULL; - } - if (!PyIter_Check(source) || (active != NULL && !PyIter_Check(active))) { - PyErr_SetString(PyExc_TypeError, "Arguments must be iterators."); - return NULL; - } - - Py_INCREF(source); - Py_XSETREF(lz->source, source); - Py_XINCREF(active); - Py_XSETREF(lz->active, active); - Py_RETURN_NONE; -} - PyDoc_STRVAR(chain_doc, -"chain(*iterables) --> chain object\n\ +"chain(*iterables)\n\ +--\n\ \n\ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ @@ -2185,10 +1869,6 @@ iterable, until all of the iterables are exhausted."); static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF - {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)chain_setstate, METH_O, - setstate_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -2427,96 +2107,15 @@ product_next(productobject *lz) return NULL; } -static PyObject * -product_reduce(productobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->stopped) { - return Py_BuildValue("O(())", Py_TYPE(lz)); - } else if (lz->result == NULL) { - return Py_BuildValue("OO", Py_TYPE(lz), lz->pools); - } else { - PyObject *indices; - Py_ssize_t n, i; - - /* we must pickle the indices use them for setstate, and - * additionally indicate that the iterator has started - */ - n = PyTuple_GET_SIZE(lz->pools); - indices = PyTuple_New(n); - if (indices == NULL) - return NULL; - for (i=0; iindices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - return Py_BuildValue("OON", Py_TYPE(lz), lz->pools, indices); - } -} - -static PyObject * -product_setstate(productobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - n = PyTuple_GET_SIZE(lz->pools); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != n) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - for (i=0; ipools, i); - poolsize = PyTuple_GET_SIZE(pool); - if (poolsize == 0) { - lz->stopped = 1; - Py_RETURN_NONE; - } - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > poolsize-1) - index = poolsize-1; - lz->indices[i] = index; - } - - result = PyTuple_New(n); - if (!result) - return NULL; - for (i=0; ipools, i); - PyObject *element = PyTuple_GET_ITEM(pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef product_methods[] = { - {"__reduce__", (PyCFunction)product_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)product_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(product_doc, -"product(*iterables, repeat=1) --> product object\n\ +"product(*iterables, repeat=1)\n\ +--\n\ \n\ Cartesian product of input iterables. Equivalent to nested for-loops.\n\n\ For example, product(A, B) returns the same as: ((x,y) for x in A for y in B).\n\ @@ -2737,83 +2336,7 @@ combinations_next(combinationsobject *co) return NULL; } -static PyObject * -combinations_reduce(combinationsobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) - { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -combinations_setstate(combinationsobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t i; - Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; ir; i++) { - Py_ssize_t max; - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index == -1 && PyErr_Occurred()) - return NULL; /* not an integer */ - max = i + n - lz->r; - /* clamp the index (beware of negative max) */ - if (index > max) - index = max; - if (index < 0) - index = 0; - lz->indices[i] = index; - } - - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef combinations_methods[] = { - {"__reduce__", (PyCFunction)combinations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)combinations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3047,79 +2570,7 @@ cwr_next(cwrobject *co) return NULL; } -static PyObject * -cwr_reduce(cwrobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -cwr_setstate(cwrobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > n-1) - index = n-1; - lz->indices[i] = index; - } - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef cwr_methods[] = { - {"__reduce__", (PyCFunction)cwr_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cwr_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)cwr_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3384,113 +2835,7 @@ permutations_next(permutationsobject *po) return NULL; } -static PyObject * -permutations_reduce(permutationsobject *po, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (po->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(po), po->pool, po->r); - } else if (po->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(po), po->r); - } else { - PyObject *indices=NULL, *cycles=NULL; - Py_ssize_t n, i; - - /* we must pickle the indices and cycles and use them for setstate */ - n = PyTuple_GET_SIZE(po->pool); - indices = PyTuple_New(n); - if (indices == NULL) - goto err; - for (i=0; iindices[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(indices, i, index); - } - - cycles = PyTuple_New(po->r); - if (cycles == NULL) - goto err; - for (i=0 ; ir ; i++) { - PyObject* index = PyLong_FromSsize_t(po->cycles[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(cycles, i, index); - } - return Py_BuildValue("O(On)(NN)", Py_TYPE(po), - po->pool, po->r, - indices, cycles); - err: - Py_XDECREF(indices); - Py_XDECREF(cycles); - return NULL; - } -} - -static PyObject * -permutations_setstate(permutationsobject *po, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *indices, *cycles, *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O!O!", - &PyTuple_Type, &indices, - &PyTuple_Type, &cycles)) { - return NULL; - } - - n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; i n-1) - index = n-1; - po->indices[i] = index; - } - - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - if (index < 1) - index = 1; - else if (index > n-i) - index = n-i; - po->cycles[i] = index; - } - result = PyTuple_New(po->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(po->pool, po->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(po->result, result); - Py_RETURN_NONE; -} - static PyMethodDef permuations_methods[] = { - {"__reduce__", (PyCFunction)permutations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)permutations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)permutations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3625,59 +2970,6 @@ accumulate_next(accumulateobject *lz) return newtotal; } -static PyObject * -accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - itertools_state *state = lz->state; - - if (lz->initial != Py_None) { - PyObject *it; - - assert(lz->total == NULL); - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->initial, lz->it); - if (it == NULL) - return NULL; - return Py_BuildValue("O(NO)O", Py_TYPE(lz), - it, lz->binop?lz->binop:Py_None, Py_None); - } - if (lz->total == Py_None) { - PyObject *it; - - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->total, lz->it); - if (it == NULL) - return NULL; - it = PyObject_CallFunction((PyObject *)Py_TYPE(lz), "NO", - it, lz->binop ? lz->binop : Py_None); - if (it == NULL) - return NULL; - - return Py_BuildValue("O(NiO)", state->islice_type, it, 1, Py_None); - } - return Py_BuildValue("O(OO)O", Py_TYPE(lz), - lz->it, lz->binop?lz->binop:Py_None, - lz->total?lz->total:Py_None); -} - -static PyObject * -accumulate_setstate(accumulateobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->total, state); - Py_RETURN_NONE; -} - -static PyMethodDef accumulate_methods[] = { - {"__reduce__", (PyCFunction)accumulate_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)accumulate_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot accumulate_slots[] = { {Py_tp_dealloc, accumulate_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3685,7 +2977,6 @@ static PyType_Slot accumulate_slots[] = { {Py_tp_traverse, accumulate_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, accumulate_next}, - {Py_tp_methods, accumulate_methods}, {Py_tp_new, itertools_accumulate}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3810,20 +3101,6 @@ compress_next(compressobject *lz) } } -static PyObject * -compress_reduce(compressobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->data, lz->selectors); -} - -static PyMethodDef compress_methods[] = { - {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot compress_slots[] = { {Py_tp_dealloc, compress_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3831,7 +3108,6 @@ static PyType_Slot compress_slots[] = { {Py_tp_traverse, compress_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, compress_next}, - {Py_tp_methods, compress_methods}, {Py_tp_new, itertools_compress}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3943,19 +3219,6 @@ filterfalse_next(filterfalseobject *lz) } } -static PyObject * -filterfalse_reduce(filterfalseobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef filterfalse_methods[] = { - {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot filterfalse_slots[] = { {Py_tp_dealloc, filterfalse_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3963,7 +3226,6 @@ static PyType_Slot filterfalse_slots[] = { {Py_tp_traverse, filterfalse_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, filterfalse_next}, - {Py_tp_methods, filterfalse_methods}, {Py_tp_new, itertools_filterfalse}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3993,7 +3255,7 @@ fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 - When count hits Y_SSIZE_T_MAX, switch to slow_mode. + When count hits PY_SSIZE_T_MAX, switch to slow_mode. slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. @@ -4142,9 +3404,30 @@ count_nextlong(countobject *lz) static PyObject * count_next(countobject *lz) { +#ifndef Py_GIL_DISABLED if (lz->cnt == PY_SSIZE_T_MAX) return count_nextlong(lz); return PyLong_FromSsize_t(lz->cnt++); +#else + // free-threading version + // fast mode uses compare-exchange loop + // slow mode uses a critical section + PyObject *returned; + Py_ssize_t cnt; + + cnt = _Py_atomic_load_ssize_relaxed(&lz->cnt); + for (;;) { + if (cnt == PY_SSIZE_T_MAX) { + Py_BEGIN_CRITICAL_SECTION(lz); + returned = count_nextlong(lz); + Py_END_CRITICAL_SECTION(); + return returned; + } + if (_Py_atomic_compare_exchange_ssize(&lz->cnt, &cnt, cnt + 1)) { + return PyLong_FromSsize_t(cnt); + } + } +#endif } static PyObject * @@ -4171,21 +3454,6 @@ count_repr(countobject *lz) lz->long_cnt, lz->long_step); } -static PyObject * -count_reduce(countobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->cnt == PY_SSIZE_T_MAX) - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - return Py_BuildValue("O(n)", Py_TYPE(lz), lz->cnt); -} - -static PyMethodDef count_methods[] = { - {"__reduce__", (PyCFunction)count_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot count_slots[] = { {Py_tp_dealloc, count_dealloc}, {Py_tp_repr, count_repr}, @@ -4194,7 +3462,6 @@ static PyType_Slot count_slots[] = { {Py_tp_traverse, count_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, count_next}, - {Py_tp_methods, count_methods}, {Py_tp_new, itertools_count}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4295,22 +3562,8 @@ repeat_len(repeatobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject * -repeat_reduce(repeatobject *ro, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* unpickle this so that a new repeat iterator is constructed with an - * object, then call __setstate__ on it to set cnt - */ - if (ro->cnt >= 0) - return Py_BuildValue("O(On)", Py_TYPE(ro), ro->element, ro->cnt); - else - return Py_BuildValue("O(O)", Py_TYPE(ro), ro->element); -} - static PyMethodDef repeat_methods[] = { {"__length_hint__", (PyCFunction)repeat_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)repeat_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; @@ -4516,52 +3769,9 @@ zip_longest_next(ziplongestobject *lz) return result; } -static PyObject * -zip_longest_reduce(ziplongestobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new tuple with empty sequences where appropriate to pickle. - * Then use setstate to set the fillvalue - */ - int i; - PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); - - if (args == NULL) - return NULL; - for (i=0; iittuple); i++) { - PyObject *elem = PyTuple_GET_ITEM(lz->ittuple, i); - if (elem == NULL) { - elem = PyTuple_New(0); - if (elem == NULL) { - Py_DECREF(args); - return NULL; - } - } else - Py_INCREF(elem); - PyTuple_SET_ITEM(args, i, elem); - } - return Py_BuildValue("ONO", Py_TYPE(lz), args, lz->fillvalue); -} - -static PyObject * -zip_longest_setstate(ziplongestobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->fillvalue, state); - Py_RETURN_NONE; -} - -static PyMethodDef zip_longest_methods[] = { - {"__reduce__", (PyCFunction)zip_longest_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)zip_longest_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(zip_longest_doc, -"zip_longest(iter1 [,iter2 [...]], [fillvalue=None]) --> zip_longest object\n\ +"zip_longest(*iterables, fillvalue=None)\n\ +--\n\ \n\ Return a zip_longest object whose .__next__() method returns a tuple where\n\ the i-th element comes from the i-th iterable argument. The .__next__()\n\ @@ -4578,7 +3788,6 @@ static PyType_Slot ziplongest_slots[] = { {Py_tp_traverse, zip_longest_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, zip_longest_next}, - {Py_tp_methods, zip_longest_methods}, {Py_tp_new, zip_longest_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4609,15 +3818,15 @@ batched(p, n) --> [p0, p1, ..., p_n-1], [p_n, p_n+1, ..., p_2n-1], ...\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ...\n\ chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ...\n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ -dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ +dropwhile(predicate, seq) --> seq[n], seq[n+1], starting when predicate fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ -filterfalse(pred, seq) --> elements of seq where pred(elem) is False\n\ +filterfalse(predicate, seq) --> elements of seq where predicate(elem) is False\n\ islice(seq, [start,] stop [, step]) --> elements from\n\ seq[start:stop:step]\n\ pairwise(s) --> (s[0],s[1]), (s[1],s[2]), (s[2], s[3]), ...\n\ starmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ...\n\ tee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n\n\ -takewhile(pred, seq) --> seq[0], seq[1], until pred fails\n\ +takewhile(predicate, seq) --> seq[0], seq[1], until predicate fails\n\ zip_longest(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ...\n\ \n\ Combinatoric generators:\n\ @@ -4736,6 +3945,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/main.c b/Modules/main.c index b5ee34d0141daf..1a70b300b6ad17 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -249,7 +249,7 @@ pymain_run_command(wchar_t *command) PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags |= PyCF_IGNORE_COOKIE; - ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), &cf); + ret = _PyRun_SimpleStringFlagsWithName(PyBytes_AsString(bytes), "", &cf); Py_DECREF(bytes); return (ret != 0); @@ -513,8 +513,13 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +542,16 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_run_module(L"_pyrepl", 0); + *exitcode = (run != 0); + return; } diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e..d41b6640bb5186 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -87,18 +87,6 @@ esac NL='\ ' -# Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. -case `uname -s` in -CYGWIN*) if test $libdir = . - then - ExtraLibDir=. - else - ExtraLibDir='$(LIBPL)' - fi - ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; -esac - # Main loop for i in ${*-Setup} do @@ -286,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS) -o $file" echo "$rule" >>$rulesf done done diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a4d94665592351..6defa973da0952 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -237,7 +237,7 @@ m_sinpi(double x) double y, r; int n; /* this function should only ever be called for finite arguments */ - assert(Py_IS_FINITE(x)); + assert(isfinite(x)); y = fmod(fabs(x), 2.0); n = (int)round(2.0*y); assert(0 <= n && n <= 4); @@ -396,8 +396,8 @@ m_tgamma(double x) double absx, r, y, z, sqrtpow; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x) || x > 0.0) + if (!isfinite(x)) { + if (isnan(x) || x > 0.0) return x; /* tgamma(nan) = nan, tgamma(inf) = inf */ else { errno = EDOM; @@ -424,7 +424,7 @@ m_tgamma(double x) /* tiny arguments: tgamma(x) ~ 1/x for x near 0 */ if (absx < 1e-20) { r = 1.0/x; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -481,7 +481,7 @@ m_tgamma(double x) r *= sqrtpow; } } - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -498,8 +498,8 @@ m_lgamma(double x) double absx; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* lgamma(nan) = nan */ else return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ @@ -530,7 +530,7 @@ m_lgamma(double x) if (x < 0.0) /* Use reflection formula to get value for negative x. */ r = logpi - log(fabs(m_sinpi(absx))) - log(absx) - r; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -546,10 +546,10 @@ m_lgamma(double x) static double m_atan2(double y, double x) { - if (Py_IS_NAN(x) || Py_IS_NAN(y)) + if (isnan(x) || isnan(y)) return Py_NAN; - if (Py_IS_INFINITY(y)) { - if (Py_IS_INFINITY(x)) { + if (isinf(y)) { + if (isinf(x)) { if (copysign(1., x) == 1.) /* atan2(+-inf, +inf) == +-pi/4 */ return copysign(0.25*Py_MATH_PI, y); @@ -560,7 +560,7 @@ m_atan2(double y, double x) /* atan2(+-inf, x) == +-pi/2 for finite x */ return copysign(0.5*Py_MATH_PI, y); } - if (Py_IS_INFINITY(x) || y == 0.) { + if (isinf(x) || y == 0.) { if (copysign(1., x) == 1.) /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ return copysign(0., y); @@ -580,7 +580,7 @@ static double m_remainder(double x, double y) { /* Deal with most common case first. */ - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) { + if (isfinite(x) && isfinite(y)) { double absx, absy, c, m, r; if (y == 0.0) { @@ -653,16 +653,16 @@ m_remainder(double x, double y) } /* Special values. */ - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return y; } - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return Py_NAN; } - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); return x; } @@ -677,7 +677,7 @@ m_remainder(double x, double y) static double m_log(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log(x); errno = EDOM; @@ -686,7 +686,7 @@ m_log(double x) else return Py_NAN; /* log(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log(nan) = nan */ else if (x > 0.0) return x; /* log(inf) = inf */ @@ -709,8 +709,8 @@ m_log(double x) static double m_log2(double x) { - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* log2(nan) = nan */ else if (x > 0.0) return x; /* log2(+inf) = +inf */ @@ -736,7 +736,7 @@ m_log2(double x) static double m_log10(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log10(x); errno = EDOM; @@ -745,7 +745,7 @@ m_log10(double x) else return Py_NAN; /* log10(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log10(nan) = nan */ else if (x > 0.0) return x; /* log10(inf) = inf */ @@ -759,13 +759,17 @@ m_log10(double x) static PyObject * math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) { - PyObject *res, *x; - Py_ssize_t i; + // Fast-path for the common case: gcd(int, int) + if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) + { + return _PyLong_GCD(args[0], args[1]); + } if (nargs == 0) { return PyLong_FromLong(0); } - res = PyNumber_Index(args[0]); + + PyObject *res = PyNumber_Index(args[0]); if (res == NULL) { return NULL; } @@ -775,8 +779,8 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } PyObject *one = _PyLong_GetOne(); // borrowed ref - for (i = 1; i < nargs; i++) { - x = _PyNumber_Index(args[i]); + for (Py_ssize_t i = 1; i < nargs; i++) { + PyObject *x = _PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); return NULL; @@ -962,12 +966,12 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) return NULL; errno = 0; r = (*func)(x); - if (Py_IS_NAN(r) && !Py_IS_NAN(x)) { + if (isnan(r) && !isnan(x)) { PyErr_SetString(PyExc_ValueError, "math domain error"); /* invalid arg */ return NULL; } - if (Py_IS_INFINITY(r) && Py_IS_FINITE(x)) { + if (isinf(r) && isfinite(x)) { if (can_overflow) PyErr_SetString(PyExc_OverflowError, "math range error"); /* overflow */ @@ -976,7 +980,7 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) "math domain error"); /* singularity */ return NULL; } - if (Py_IS_FINITE(r) && errno && is_error(r)) + if (isfinite(r) && errno && is_error(r)) /* this branch unnecessary on most platforms */ return NULL; @@ -1045,14 +1049,14 @@ math_2(PyObject *const *args, Py_ssize_t nargs, } errno = 0; r = (*func)(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; } - else if (Py_IS_INFINITY(r)) { - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) + else if (isinf(r)) { + if (isfinite(x) && isfinite(y)) errno = ERANGE; else errno = 0; @@ -1249,7 +1253,7 @@ FUNC1(tanh, tanh, 0, "Return the hyperbolic tangent of x.") /* Precision summation function as msum() by Raymond Hettinger in - , + , enhanced with the exact partials sum and roundoff from Mark Dickinson's post at . See those links for more details, proofs and other references. @@ -1399,17 +1403,17 @@ math_fsum(PyObject *module, PyObject *seq) n = i; /* ps[i:] = [x] */ if (x != 0.0) { - if (! Py_IS_FINITE(x)) { + if (! isfinite(x)) { /* a nonfinite x could arise either as a result of intermediate overflow, or as a result of a nan or inf in the summands */ - if (Py_IS_FINITE(xsave)) { + if (isfinite(xsave)) { PyErr_SetString(PyExc_OverflowError, "intermediate overflow in fsum"); goto _fsum_error; } - if (Py_IS_INFINITY(xsave)) + if (isinf(xsave)) inf_sum += xsave; special_sum += xsave; /* reset partials */ @@ -1423,7 +1427,7 @@ math_fsum(PyObject *module, PyObject *seq) } if (special_sum != 0.0) { - if (Py_IS_NAN(inf_sum)) + if (isnan(inf_sum)) PyErr_SetString(PyExc_ValueError, "-inf + inf in fsum"); else @@ -2070,11 +2074,6 @@ math_trunc(PyObject *module, PyObject *x) return PyFloat_Type.tp_as_number->nb_int(x); } - if (!_PyType_IsReady(Py_TYPE(x))) { - if (PyType_Ready(Py_TYPE(x)) < 0) - return NULL; - } - math_module_state *state = get_math_module_state(module); trunc = _PyObject_LookupSpecial(x, state->str___trunc__); if (trunc == NULL) { @@ -2109,7 +2108,7 @@ math_frexp_impl(PyObject *module, double x) int i; /* deal with special cases directly, to sidestep platform differences */ - if (Py_IS_NAN(x) || Py_IS_INFINITY(x) || !x) { + if (isnan(x) || isinf(x) || !x) { i = 0; } else { @@ -2154,7 +2153,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) return NULL; } - if (x == 0. || !Py_IS_FINITE(x)) { + if (x == 0. || !isfinite(x)) { /* NaNs, zeros and infinities are returned unchanged */ r = x; errno = 0; @@ -2169,7 +2168,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; } @@ -2197,9 +2196,9 @@ math_modf_impl(PyObject *module, double x) double y; /* some platforms don't do the right thing for NaNs and infinities, so we take care of special cases directly. */ - if (Py_IS_INFINITY(x)) + if (isinf(x)) return Py_BuildValue("(dd)", copysign(0., x), x); - else if (Py_IS_NAN(x)) + else if (isnan(x)) return Py_BuildValue("(dd)", x, x); errno = 0; @@ -2322,6 +2321,48 @@ math_log10(PyObject *module, PyObject *x) } +/*[clinic input] +math.fma + + x: double + y: double + z: double + / + +Fused multiply-add operation. + +Compute (x * y) + z with a single round. +[clinic start generated code]*/ + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z) +/*[clinic end generated code: output=4fc8626dbc278d17 input=e3ad1f4a4c89626e]*/ +{ + double r = fma(x, y, z); + + /* Fast path: if we got a finite result, we're done. */ + if (isfinite(r)) { + return PyFloat_FromDouble(r); + } + + /* Non-finite result. Raise an exception if appropriate, else return r. */ + if (isnan(r)) { + if (!isnan(x) && !isnan(y) && !isnan(z)) { + /* NaN result from non-NaN inputs. */ + PyErr_SetString(PyExc_ValueError, "invalid operation in fma"); + return NULL; + } + } + else if (isfinite(x) && isfinite(y) && isfinite(z)) { + /* Infinite result from finite inputs. */ + PyErr_SetString(PyExc_OverflowError, "overflow in fma"); + return NULL; + } + + return PyFloat_FromDouble(r); +} + + /*[clinic input] math.fmod @@ -2340,12 +2381,12 @@ math_fmod_impl(PyObject *module, double x, double y) { double r; /* fmod(x, +/-Inf) returns x for finite x. */ - if (Py_IS_INFINITY(y) && Py_IS_FINITE(x)) + if (isinf(y) && isfinite(x)) return PyFloat_FromDouble(x); errno = 0; r = fmod(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; @@ -2467,7 +2508,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) int max_e; Py_ssize_t i; - if (Py_IS_INFINITY(max)) { + if (isinf(max)) { return max; } if (found_nan) { @@ -2489,7 +2530,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) assert(max * scale < 1.0); for (i=0 ; i < n ; i++) { x = vec[i]; - assert(Py_IS_FINITE(x) && fabs(x) <= max); + assert(isfinite(x) && fabs(x) <= max); x *= scale; // lossless scaling assert(fabs(x) < 1.0); pr = dl_mul(x, x); // lossless squaring @@ -2566,7 +2607,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) goto error_exit; } if (n > NUM_STACK_ELEMS) { - diffs = (double *) PyObject_Malloc(n * sizeof(double)); + diffs = (double *) PyMem_Malloc(n * sizeof(double)); if (diffs == NULL) { PyErr_NoMemory(); goto error_exit; @@ -2579,14 +2620,14 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) ASSIGN_DOUBLE(qx, item, error_exit); x = fabs(px - qx); diffs[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } } result = vector_norm(n, diffs, max, found_nan); if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2598,7 +2639,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) error_exit: if (diffs != diffs_on_stack) { - PyObject_Free(diffs); + PyMem_Free(diffs); } if (p_allocated) { Py_DECREF(p); @@ -2622,7 +2663,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) double *coordinates = coord_on_stack; if (nargs > NUM_STACK_ELEMS) { - coordinates = (double *) PyObject_Malloc(nargs * sizeof(double)); + coordinates = (double *) PyMem_Malloc(nargs * sizeof(double)); if (coordinates == NULL) { return PyErr_NoMemory(); } @@ -2632,20 +2673,20 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) ASSIGN_DOUBLE(x, item, error_exit); x = fabs(x); coordinates[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } } result = vector_norm(nargs, coordinates, max, found_nan); if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return PyFloat_FromDouble(result); error_exit: if (coordinates != coord_on_stack) { - PyObject_Free(coordinates); + PyMem_Free(coordinates); } return NULL; } @@ -2832,7 +2873,7 @@ math_sumprod_impl(PyObject *module, PyObject *p, PyObject *q) PyErr_Clear(); goto finalize_flt_path; } - } else if (q_type_float && (PyLong_CheckExact(p_i) || PyBool_Check(q_i))) { + } else if (q_type_float && (PyLong_CheckExact(p_i) || PyBool_Check(p_i))) { flt_q = PyFloat_AS_DOUBLE(q_i); flt_p = PyLong_AsDouble(p_i); if (flt_p == -1.0 && PyErr_Occurred()) { @@ -2935,14 +2976,14 @@ math_pow_impl(PyObject *module, double x, double y) /* deal directly with IEEE specials, to cope with problems on various platforms whose semantics don't exactly match C99 */ r = 0.; /* silence compiler warning */ - if (!Py_IS_FINITE(x) || !Py_IS_FINITE(y)) { + if (!isfinite(x) || !isfinite(y)) { errno = 0; - if (Py_IS_NAN(x)) + if (isnan(x)) r = y == 0. ? 1. : x; /* NaN**0 = 1 */ - else if (Py_IS_NAN(y)) + else if (isnan(y)) r = x == 1. ? 1. : y; /* 1**NaN = 1 */ - else if (Py_IS_INFINITY(x)) { - odd_y = Py_IS_FINITE(y) && fmod(fabs(y), 2.0) == 1.0; + else if (isinf(x)) { + odd_y = isfinite(y) && fmod(fabs(y), 2.0) == 1.0; if (y > 0.) r = odd_y ? x : fabs(x); else if (y == 0.) @@ -2951,7 +2992,7 @@ math_pow_impl(PyObject *module, double x, double y) r = odd_y ? copysign(0., x) : 0.; } else { - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); if (fabs(x) == 1.0) r = 1.; else if (y > 0. && fabs(x) > 1.0) @@ -2969,8 +3010,8 @@ math_pow_impl(PyObject *module, double x, double y) r = pow(x, y); /* a NaN result should arise only from (-ve)**(finite non-integer); in this case we want to raise ValueError. */ - if (!Py_IS_FINITE(r)) { - if (Py_IS_NAN(r)) { + if (!isfinite(r)) { + if (isnan(r)) { errno = EDOM; } /* @@ -2978,7 +3019,7 @@ math_pow_impl(PyObject *module, double x, double y) (A) (+/-0.)**negative (-> divide-by-zero) (B) overflow of x**y with x and y finite */ - else if (Py_IS_INFINITY(r)) { + else if (isinf(r)) { if (x == 0.) errno = EDOM; else @@ -3044,7 +3085,7 @@ static PyObject * math_isfinite_impl(PyObject *module, double x) /*[clinic end generated code: output=8ba1f396440c9901 input=46967d254812e54a]*/ { - return PyBool_FromLong((long)Py_IS_FINITE(x)); + return PyBool_FromLong((long)isfinite(x)); } @@ -3061,7 +3102,7 @@ static PyObject * math_isnan_impl(PyObject *module, double x) /*[clinic end generated code: output=f537b4d6df878c3e input=935891e66083f46a]*/ { - return PyBool_FromLong((long)Py_IS_NAN(x)); + return PyBool_FromLong((long)isnan(x)); } @@ -3078,7 +3119,7 @@ static PyObject * math_isinf_impl(PyObject *module, double x) /*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ { - return PyBool_FromLong((long)Py_IS_INFINITY(x)); + return PyBool_FromLong((long)isinf(x)); } @@ -3135,7 +3176,7 @@ math_isclose_impl(PyObject *module, double a, double b, double rel_tol, above. */ - if (Py_IS_INFINITY(a) || Py_IS_INFINITY(b)) { + if (isinf(a) || isinf(b)) { return 0; } @@ -3885,10 +3926,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) Bug fixed in bos.adt.libm 7.2.2.0 by APAR IV95512. */ return PyFloat_FromDouble(y); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } #endif @@ -3934,10 +3975,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) if (usteps == 0) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } @@ -4003,16 +4044,16 @@ static double math_ulp_impl(PyObject *module, double x) /*[clinic end generated code: output=f5207867a9384dd4 input=31f9bfbbe373fcaa]*/ { - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } x = fabs(x); - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return x; } double inf = Py_INFINITY; double x2 = nextafter(x, inf); - if (Py_IS_INFINITY(x2)) { + if (isinf(x2)) { /* special case: x is the largest positive representable float */ x2 = nextafter(x, -inf); return x - x2; @@ -4095,6 +4136,7 @@ static PyMethodDef math_methods[] = { {"fabs", math_fabs, METH_O, math_fabs_doc}, MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF + MATH_FMA_METHODDEF MATH_FMOD_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF @@ -4135,6 +4177,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 5463effb507de6..ef9163e8be5b6c 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -15,14 +15,13 @@ */ /* MD5 objects */ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" #include "hashlib.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() /*[clinic input] module _md5 @@ -50,9 +49,9 @@ typedef long long MD5_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; - Hacl_Streaming_MD5_state *hash_state; + bool use_mutex; + PyMutex mutex; + Hacl_Hash_MD5_state_t *hash_state; } MD5object; #include "clinic/md5module.c.h" @@ -74,7 +73,11 @@ static MD5object * newMD5object(MD5State * st) { MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type); - md5->lock = NULL; + if (!md5) { + return NULL; + } + HASHLIB_INIT_MUTEX(md5); + PyObject_GC_Track(md5); return md5; } @@ -90,11 +93,8 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg) static void MD5_dealloc(MD5object *ptr) { - Hacl_Streaming_MD5_legacy_free(ptr->hash_state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } - PyTypeObject *tp = Py_TYPE(ptr); + Hacl_Hash_MD5_free(ptr->hash_state); + PyTypeObject *tp = Py_TYPE((PyObject*)ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); Py_DECREF(tp); @@ -115,14 +115,14 @@ static PyObject * MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) /*[clinic end generated code: output=bf055e08244bf5ee input=d89087dcfb2a8620]*/ { - MD5State *st = _PyType_GetModuleState(cls); + MD5State *st = PyType_GetModuleState(cls); MD5object *newobj; if ((newobj = newMD5object(st))==NULL) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_MD5_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -139,7 +139,7 @@ MD5Type_digest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -156,20 +156,29 @@ MD5Type_hexdigest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); - return _Py_strhex((const char*)digest, MD5_DIGESTSIZE); + + const char *hexdigits = "0123456789abcdef"; + char digest_hex[MD5_DIGESTSIZE * 2]; + char *str = digest_hex; + for (size_t i=0; i < MD5_DIGESTSIZE; i++) { + unsigned char byte = digest[i]; + *str++ = hexdigits[byte >> 4]; + *str++ = hexdigits[byte & 0x0f]; + } + return PyUnicode_FromStringAndSize(digest_hex, sizeof(digest_hex)); } -static void update(Hacl_Streaming_MD5_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_MD5_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_MD5_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_MD5_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_MD5_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_MD5_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -189,14 +198,14 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update(self->hash_state, buf.buf, buf.len); @@ -293,7 +302,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_MD5_legacy_create_in(); + new->hash_state = Hacl_Hash_MD5_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -366,6 +375,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index d11200a4042551..99a85e9e49ad47 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -32,10 +32,6 @@ # include // close() #endif -// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA -// need to be replaced with OpenFileMappingW / CreateFileMappingW -#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) - #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -45,6 +41,7 @@ #ifdef MS_WINDOWS #include +#include // LsaNtStatusToWinError static int my_getpagesize(void) { @@ -116,11 +113,12 @@ typedef struct { #ifdef MS_WINDOWS HANDLE map_handle; HANDLE file_handle; - char * tagname; + wchar_t * tagname; #endif #ifdef UNIX int fd; + _Bool trackfd; #endif PyObject *weakreflist; @@ -171,7 +169,7 @@ mmap_object_dealloc(mmap_object *m_obj) } static PyObject * -mmap_close_method(mmap_object *self, PyObject *unused) +mmap_close_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { if (self->exports > 0) { PyErr_SetString(PyExc_BufferError, "cannot close "\ @@ -258,25 +256,231 @@ do { \ } while (0) #endif /* UNIX */ +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +static DWORD +filter_page_exception(EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static DWORD +filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, + EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + + ULONG_PTR address = record->ExceptionInformation[1]; + if (address >= (ULONG_PTR) self->data && + address < (ULONG_PTR) self->data + (ULONG_PTR) self->size) + { + return EXCEPTION_EXECUTE_HANDLER; + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception(GetExceptionInformation(), &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception_method(self, GetExceptionInformation(), \ + &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +int +safe_memcpy(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memcpy(dest, src, count); + ); + return 0; +} + +int +safe_byte_copy(char *dest, const char *src) +{ + HANDLE_INVALID_MEM( + *dest = *src; + ); + return 0; +} + +int +safe_memchr(char **out, const void *ptr, int ch, size_t count) +{ + HANDLE_INVALID_MEM( + *out = (char *) memchr(ptr, ch, count); + ); + return 0; +} + +int +safe_memmove(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memmove(dest, src, count); + ); + return 0; +} + +int +safe_copy_from_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[cur] = src[i]; + } + ); + return 0; +} + +int +safe_copy_to_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[i] = src[cur]; + } + ); + return 0; +} + + +int +_safe_PyBytes_Find(Py_ssize_t *out, mmap_object *self, const char *haystack, + Py_ssize_t len_haystack, const char *needle, + Py_ssize_t len_needle, Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_Find(haystack, len_haystack, needle, len_needle, offset); + ); + return 0; +} + +int +_safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, + const char *haystack, Py_ssize_t len_haystack, + const char *needle, Py_ssize_t len_needle, + Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_ReverseFind(haystack, len_haystack, needle, len_needle, + offset); + ); + return 0; +} + +PyObject * +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { + if (num_bytes == 1) { + char dest; + if (safe_byte_copy(&dest, start) < 0) { + return NULL; + } + else { + return PyBytes_FromStringAndSize(&dest, 1); + } + } + else { + PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); + if (result == NULL) { + return NULL; + } + if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { + Py_CLEAR(result); + } + return result; + } +} + static PyObject * mmap_read_byte_method(mmap_object *self, - PyObject *unused) + PyObject *Py_UNUSED(ignored)) { CHECK_VALID(NULL); if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "read byte out of range"); return NULL; } - return PyLong_FromLong((unsigned char)self->data[self->pos++]); + char dest; + if (safe_byte_copy(&dest, self->data + self->pos) < 0) { + return NULL; + } + self->pos++; + return PyLong_FromLong((unsigned char) dest); } static PyObject * mmap_read_line_method(mmap_object *self, - PyObject *unused) + PyObject *Py_UNUSED(ignored)) { Py_ssize_t remaining; char *start, *eol; - PyObject *result; CHECK_VALID(NULL); @@ -284,13 +488,20 @@ mmap_read_line_method(mmap_object *self, if (!remaining) return PyBytes_FromString(""); start = self->data + self->pos; - eol = memchr(start, '\n', remaining); + + if (safe_memchr(&eol, start, '\n', remaining) < 0) { + return NULL; + } + if (!eol) eol = self->data + self->size; else ++eol; /* advance past newline */ - result = PyBytes_FromStringAndSize(start, (eol - start)); - self->pos += (eol - start); + + PyObject *result = _safe_PyBytes_FromStringAndSize(start, eol - start); + if (result != NULL) { + self->pos += (eol - start); + } return result; } @@ -299,7 +510,6 @@ mmap_read_method(mmap_object *self, PyObject *args) { Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) @@ -310,8 +520,12 @@ mmap_read_method(mmap_object *self, remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (num_bytes < 0 || num_bytes > remaining) num_bytes = remaining; - result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes); - self->pos += num_bytes; + + PyObject *result = _safe_PyBytes_FromStringAndSize(self->data + self->pos, + num_bytes); + if (result != NULL) { + self->pos += num_bytes; + } return result; } @@ -344,25 +558,38 @@ mmap_gfind(mmap_object *self, else if (end > self->size) end = self->size; - Py_ssize_t res; + Py_ssize_t index; + PyObject *result; CHECK_VALID_OR_RELEASE(NULL, view); if (end < start) { - res = -1; + result = PyLong_FromSsize_t(-1); } else if (reverse) { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_ReverseFind( + if (_safe_PyBytes_ReverseFind(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } else { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_Find( + if (_safe_PyBytes_Find(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } PyBuffer_Release(&view); - return PyLong_FromSsize_t(res); + return result; } } @@ -397,6 +624,13 @@ is_resizeable(mmap_object *self) "mmap can't resize with extant buffers exported."); return 0; } +#ifdef UNIX + if (!self->trackfd) { + PyErr_SetString(PyExc_ValueError, + "mmap can't resize with trackfd=False."); + return 0; + } +#endif if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT)) return 1; PyErr_Format(PyExc_TypeError, @@ -428,10 +662,16 @@ mmap_write_method(mmap_object *self, } CHECK_VALID_OR_RELEASE(NULL, data); - memcpy(&self->data[self->pos], data.buf, data.len); - self->pos += data.len; + PyObject *result; + if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + result = NULL; + } + else { + self->pos += data.len; + result = PyLong_FromSsize_t(data.len); + } PyBuffer_Release(&data); - return PyLong_FromSsize_t(data.len); + return result; } static PyObject * @@ -448,19 +688,21 @@ mmap_write_byte_method(mmap_object *self, return NULL; CHECK_VALID(NULL); - if (self->pos < self->size) { - self->data[self->pos++] = value; - Py_RETURN_NONE; - } - else { + if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "write byte out of range"); return NULL; } + + if (safe_byte_copy(self->data + self->pos, &value) < 0) { + return NULL; + } + self->pos++; + Py_RETURN_NONE; } static PyObject * mmap_size_method(mmap_object *self, - PyObject *unused) + PyObject *Py_UNUSED(ignored)) { CHECK_VALID(NULL); @@ -534,7 +776,7 @@ mmap_resize_method(mmap_object *self, CloseHandle(self->map_handle); /* if the file mapping still exists, it cannot be resized. */ if (self->tagname) { - self->map_handle = OpenFileMappingA(FILE_MAP_WRITE, FALSE, + self->map_handle = OpenFileMappingW(FILE_MAP_WRITE, FALSE, self->tagname); if (self->map_handle) { PyErr_SetFromWindowsErr(ERROR_USER_MAPPED_FILE); @@ -563,7 +805,7 @@ mmap_resize_method(mmap_object *self, /* create a new file mapping and map a new view */ /* FIXME: call CreateFileMappingW with wchar_t tagname */ - self->map_handle = CreateFileMappingA( + self->map_handle = CreateFileMappingW( self->file_handle, NULL, PAGE_READWRITE, @@ -657,7 +899,7 @@ mmap_resize_method(mmap_object *self, } static PyObject * -mmap_tell_method(mmap_object *self, PyObject *unused) +mmap_tell_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { CHECK_VALID(NULL); return PyLong_FromSize_t(self->pos); @@ -729,7 +971,7 @@ mmap_seek_method(mmap_object *self, PyObject *args) if (where > self->size || where < 0) goto onoutofrange; self->pos = where; - Py_RETURN_NONE; + return PyLong_FromSsize_t(self->pos); } onoutofrange: @@ -737,6 +979,12 @@ mmap_seek_method(mmap_object *self, PyObject *args) return NULL; } +static PyObject * +mmap_seekable_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) +{ + Py_RETURN_TRUE; +} + static PyObject * mmap_move_method(mmap_object *self, PyObject *args) { @@ -753,8 +1001,9 @@ mmap_move_method(mmap_object *self, PyObject *args) goto bounds; CHECK_VALID(NULL); - memmove(&self->data[dest], &self->data[src], cnt); - + if (safe_memmove(self->data + dest, self->data + src, cnt) < 0) { + return NULL; + }; Py_RETURN_NONE; bounds: @@ -835,16 +1084,39 @@ mmap__repr__method(PyObject *self) #ifdef MS_WINDOWS static PyObject * -mmap__sizeof__method(mmap_object *self, void *unused) +mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) { size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { - res += strlen(self->tagname) + 1; + res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); } return PyLong_FromSize_t(res); } #endif +#if defined(MS_WINDOWS) && defined(Py_DEBUG) +static PyObject * +mmap_protect_method(mmap_object *self, PyObject *args) { + DWORD flNewProtect, flOldProtect; + Py_ssize_t start, length; + + CHECK_VALID(NULL); + + if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { + return NULL; + } + + if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, + &flOldProtect)) + { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_MADVISE static PyObject * mmap_madvise_method(mmap_object *self, PyObject *args) @@ -905,6 +1177,7 @@ static struct PyMethodDef mmap_object_methods[] = { {"readline", (PyCFunction) mmap_read_line_method, METH_NOARGS}, {"resize", (PyCFunction) mmap_resize_method, METH_VARARGS}, {"seek", (PyCFunction) mmap_seek_method, METH_VARARGS}, + {"seekable", (PyCFunction) mmap_seekable_method, METH_NOARGS}, {"size", (PyCFunction) mmap_size_method, METH_NOARGS}, {"tell", (PyCFunction) mmap_tell_method, METH_NOARGS}, {"write", (PyCFunction) mmap_write_method, METH_VARARGS}, @@ -913,7 +1186,10 @@ static struct PyMethodDef mmap_object_methods[] = { {"__exit__", (PyCFunction) mmap__exit__method, METH_VARARGS}, #ifdef MS_WINDOWS {"__sizeof__", (PyCFunction) mmap__sizeof__method, METH_NOARGS}, -#endif +#ifdef Py_DEBUG + {"_protect", (PyCFunction) mmap_protect_method, METH_VARARGS}, +#endif // Py_DEBUG +#endif // MS_WINDOWS {NULL, NULL} /* sentinel */ }; @@ -957,7 +1233,12 @@ mmap_item(mmap_object *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "mmap index out of range"); return NULL; } - return PyBytes_FromStringAndSize(self->data + i, 1); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(&dest, 1); } static PyObject * @@ -976,7 +1257,12 @@ mmap_subscript(mmap_object *self, PyObject *item) return NULL; } CHECK_VALID(NULL); - return PyLong_FromLong(Py_CHARMASK(self->data[i])); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyLong_FromLong(Py_CHARMASK(dest)); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; @@ -990,23 +1276,22 @@ mmap_subscript(mmap_object *self, PyObject *item) if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); else if (step == 1) - return PyBytes_FromStringAndSize(self->data + start, - slicelen); + return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { char *result_buf = (char *)PyMem_Malloc(slicelen); - size_t cur; - Py_ssize_t i; PyObject *result; if (result_buf == NULL) return PyErr_NoMemory(); - for (cur = start, i = 0; i < slicelen; - cur += step, i++) { - result_buf[i] = self->data[cur]; + if (safe_copy_to_slice(result_buf, self->data, start, step, + slicelen) < 0) + { + result = NULL; + } + else { + result = PyBytes_FromStringAndSize(result_buf, slicelen); } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); PyMem_Free(result_buf); return result; } @@ -1041,7 +1326,10 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v) if (!is_writable(self)) return -1; buf = PyBytes_AsString(v); - self->data[i] = buf[0]; + + if (safe_byte_copy(self->data + i, buf) < 0) { + return -1; + } return 0; } @@ -1086,7 +1374,11 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } CHECK_VALID(-1); - self->data[i] = (char) v; + + char v_char = (char) v; + if (safe_byte_copy(self->data + i, &v_char) < 0) { + return -1; + } return 0; } else if (PySlice_Check(item)) { @@ -1112,24 +1404,23 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) } CHECK_VALID_OR_RELEASE(-1, vbuf); + int result = 0; if (slicelen == 0) { } else if (step == 1) { - memcpy(self->data + start, vbuf.buf, slicelen); + if (safe_memcpy(self->data + start, vbuf.buf, slicelen) < 0) { + result = -1; + } } else { - size_t cur; - Py_ssize_t i; - - for (cur = start, i = 0; - i < slicelen; - cur += step, i++) + if (safe_copy_from_slice(self->data, (char *)vbuf.buf, start, step, + slicelen) < 0) { - self->data[cur] = ((char *)vbuf.buf)[i]; + result = -1; } } PyBuffer_Release(&vbuf); - return 0; + return result; } else { PyErr_SetString(PyExc_TypeError, @@ -1151,7 +1442,7 @@ is 0, the maximum length of the map is the current size of the file,\n\ except that if the file is empty Windows raises an exception (you cannot\n\ create an empty mapping on Windows).\n\ \n\ -Unix: mmap(fileno, length[, flags[, prot[, access[, offset]]]])\n\ +Unix: mmap(fileno, length[, flags[, prot[, access[, offset[, trackfd]]]]])\n\ \n\ Maps length bytes from the file specified by the file descriptor fileno,\n\ and returns a mmap object. If length is 0, the maximum length of the map\n\ @@ -1218,15 +1509,17 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) off_t offset = 0; int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ; int devzero = -1; - int access = (int)ACCESS_DEFAULT; + int access = (int)ACCESS_DEFAULT, trackfd = 1; static char *keywords[] = {"fileno", "length", "flags", "prot", - "access", "offset", NULL}; + "access", "offset", "trackfd", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|iii" _Py_PARSE_OFF_T, keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, + "in|iii" _Py_PARSE_OFF_T "$p", keywords, &fd, &map_size, &flags, &prot, - &access, &offset)) + &access, &offset, &trackfd)) { return NULL; + } if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, "memory mapped length must be positive"); @@ -1322,6 +1615,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; m_obj->offset = offset; + m_obj->trackfd = trackfd; if (fd == -1) { m_obj->fd = -1; /* Assume the caller wants to map anonymous memory. @@ -1347,13 +1641,16 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) } #endif } - else { + else if (trackfd) { m_obj->fd = _Py_dup(fd); if (m_obj->fd == -1) { Py_DECREF(m_obj); return NULL; } } + else { + m_obj->fd = -1; + } Py_BEGIN_ALLOW_THREADS m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); @@ -1393,7 +1690,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - const char *tagname = ""; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1403,7 +1700,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1536,17 +1833,19 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL && *tagname != '\0') { - m_obj->tagname = PyMem_Malloc(strlen(tagname)+1); + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } + m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { - PyErr_NoMemory(); Py_DECREF(m_obj); return NULL; } - strcpy(m_obj->tagname, tagname); } - else - m_obj->tagname = NULL; m_obj->access = (access_mode)access; size_hi = (DWORD)(size >> 32); @@ -1555,7 +1854,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) off_lo = (DWORD)(offset & 0xFFFFFFFF); /* For files, it would be sufficient to pass 0 as size. For anonymous maps, we have to pass the size explicitly. */ - m_obj->map_handle = CreateFileMappingA(m_obj->file_handle, + m_obj->map_handle = CreateFileMappingW(m_obj->file_handle, NULL, flProtect, size_hi, @@ -1648,6 +1947,39 @@ mmap_exec(PyObject *module) #endif #ifdef MAP_CONCEAL ADD_INT_MACRO(module, MAP_CONCEAL); +#endif +#ifdef MAP_NORESERVE + ADD_INT_MACRO(module, MAP_NORESERVE); +#endif +#ifdef MAP_NOEXTEND + ADD_INT_MACRO(module, MAP_NOEXTEND); +#endif +#ifdef MAP_HASSEMAPHORE + ADD_INT_MACRO(module, MAP_HASSEMAPHORE); +#endif +#ifdef MAP_NOCACHE + ADD_INT_MACRO(module, MAP_NOCACHE); +#endif +#ifdef MAP_JIT + ADD_INT_MACRO(module, MAP_JIT); +#endif +#ifdef MAP_RESILIENT_CODESIGN + ADD_INT_MACRO(module, MAP_RESILIENT_CODESIGN); +#endif +#ifdef MAP_RESILIENT_MEDIA + ADD_INT_MACRO(module, MAP_RESILIENT_MEDIA); +#endif +#ifdef MAP_32BIT + ADD_INT_MACRO(module, MAP_32BIT); +#endif +#ifdef MAP_TRANSLATED_ALLOW_EXECUTE + ADD_INT_MACRO(module, MAP_TRANSLATED_ALLOW_EXECUTE); +#endif +#ifdef MAP_UNIX03 + ADD_INT_MACRO(module, MAP_UNIX03); +#endif +#ifdef MAP_TPRO + ADD_INT_MACRO(module, MAP_TPRO); #endif if (PyModule_AddIntConstant(module, "PAGESIZE", (long)my_getpagesize()) < 0 ) { return -1; @@ -1749,6 +2081,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -1764,5 +2097,3 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } - -#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index e23db22dadb18b..308a0dab7fab1a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -599,8 +599,7 @@ _overlapped_FormatMessage_impl(PyObject *module, DWORD code) if (n) { while (iswspace(lpMsgBuf[n-1])) --n; - lpMsgBuf[n] = L'\0'; - res = Py_BuildValue("u", lpMsgBuf); + res = PyUnicode_FromWideChar(lpMsgBuf, n); } else { res = PyUnicode_FromFormat("unknown error code %u", code); } @@ -724,6 +723,24 @@ Overlapped_dealloc(OverlappedObject *self) if (!HasOverlappedIoCompleted(&self->overlapped) && self->type != TYPE_NOT_STARTED) { + // NOTE: We should not get here, if we do then something is wrong in + // the IocpProactor or ProactorEventLoop. Since everything uses IOCP if + // the overlapped IO hasn't completed yet then we should not be + // deallocating! + // + // The problem is likely that this OverlappedObject was removed from + // the IocpProactor._cache before it was complete. The _cache holds a + // reference while IO is pending so that it does not get deallocated + // while the kernel has retained the OVERLAPPED structure. + // + // CancelIoEx (likely called from self.cancel()) may have successfully + // completed, but the OVERLAPPED is still in use until either + // HasOverlappedIoCompleted() is true or GetQueuedCompletionStatus has + // returned this OVERLAPPED object. + // + // NOTE: Waiting when IOCP is in use can hang indefinitely, but this + // CancelIoEx is superfluous in that self.cancel() was already called, + // so I've only ever seen this return FALSE with GLE=ERROR_NOT_FOUND Py_BEGIN_ALLOW_THREADS if (CancelIoEx(self->handle, &self->overlapped)) wait = TRUE; @@ -906,7 +923,7 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) { break; } - /* fall through */ + _Py_FALLTHROUGH; default: return SetFromWindowsErr(err); } @@ -2039,6 +2056,7 @@ overlapped_exec(PyObject *module) WINAPI_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); + WINAPI_CONSTANT(F_DWORD, ERROR_PORT_UNREACHABLE); WINAPI_CONSTANT(F_DWORD, INFINITE); WINAPI_CONSTANT(F_HANDLE, INVALID_HANDLE_VALUE); WINAPI_CONSTANT(F_HANDLE, NULL); @@ -2052,6 +2070,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 650ae4bbd68656..fc218383d5ff95 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16,13 +16,14 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() -#include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() #include "pycore_pylifecycle.h" // _PyOS_URandom() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#include "pycore_time.h" // _PyLong_FromTime_t() #ifdef HAVE_UNISTD_H # include // symlink() @@ -36,6 +37,8 @@ # include # include // UNLEN # include "osdefs.h" // SEP +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -52,6 +55,12 @@ # define EX_OK EXIT_SUCCESS #endif +#ifdef __APPLE__ + /* Needed for the implementation of os.statvfs */ +# include +# include +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ @@ -236,15 +245,16 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_SYS_TYPES_H +/* Should be included before on HP-UX v3 */ +# include +#endif /* HAVE_SYS_TYPES_H */ + #ifdef HAVE_SYS_SYSMACROS_H /* GNU C Library: major(), minor(), makedev() */ # include #endif -#ifdef HAVE_SYS_TYPES_H -# include -#endif /* HAVE_SYS_TYPES_H */ - #ifdef HAVE_SYS_STAT_H # include #endif /* HAVE_SYS_STAT_H */ @@ -292,9 +302,15 @@ corresponding Unix manual entries for more information on calls."); # undef HAVE_SCHED_SETAFFINITY #endif -#if defined(HAVE_SYS_XATTR_H) && defined(HAVE_LINUX_LIMITS_H) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) -# define USE_XATTRS -# include // Needed for XATTR_SIZE_MAX on musl libc. +#if defined(HAVE_SYS_XATTR_H) +# if defined(HAVE_LINUX_LIMITS_H) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) +# define USE_XATTRS +# include // Needed for XATTR_SIZE_MAX on musl libc. +# endif +# if defined(__CYGWIN__) +# define USE_XATTRS +# include // Needed for XATTR_SIZE_MAX and XATTR_LIST_MAX. +# endif #endif #ifdef USE_XATTRS @@ -599,16 +615,18 @@ PyOS_BeforeFork(void) run_at_forkers(interp->before_forkers, 1); _PyImport_AcquireLock(interp); + _PyEval_StopTheWorldAll(&_PyRuntime); + HEAD_LOCK(&_PyRuntime); } void PyOS_AfterFork_Parent(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) <= 0) { - Py_FatalError("failed releasing import lock after fork"); - } + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorldAll(&_PyRuntime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_ReleaseLock(interp); run_at_forkers(interp->after_forkers_parent, 0); } @@ -618,6 +636,7 @@ PyOS_AfterFork_Child(void) PyStatus status; _PyRuntimeState *runtime = &_PyRuntime; + // re-creates runtime->interpreters.mutex (HEAD_UNLOCK) status = _PyRuntimeState_ReInitThreads(runtime); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -626,19 +645,33 @@ PyOS_AfterFork_Child(void) PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); + assert(tstate->thread_id == PyThread_get_thread_ident()); #ifdef PY_HAVE_THREAD_NATIVE_ID tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + _Py_brc_after_fork(tstate->interp); + _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); +#endif + + // Ideally we could guarantee tstate is running main. + _PyInterpreterState_ReinitRunningMain(tstate); + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } - status = _PyImport_ReInitLock(tstate->interp); - if (_PyStatus_EXCEPTION(status)) { - goto fatal_error; - } + // Remove the dead thread states. We "start the world" once we are the only + // thread state left to undo the stop the world call in `PyOS_BeforeFork`. + // That needs to happen before `_PyThreadState_DeleteList`, because that + // may call destructors. + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_DeleteList(list); + + _PyImport_ReleaseLock(tstate->interp); _PySignal_AfterFork(); @@ -928,16 +961,46 @@ _Py_Gid_Converter(PyObject *obj, gid_t *p) #endif /* MS_WINDOWS */ -#define _PyLong_FromDev PyLong_FromLongLong +static PyObject * +_PyLong_FromDev(dev_t dev) +{ +#ifdef NODEV + if (dev == NODEV) { + return PyLong_FromLongLong((long long)dev); + } +#endif + return PyLong_FromUnsignedLongLong((unsigned long long)dev); +} #if (defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV)) || defined(HAVE_DEVICE_MACROS) static int _Py_Dev_Converter(PyObject *obj, void *p) { - *((dev_t *)p) = PyLong_AsUnsignedLongLong(obj); - if (PyErr_Occurred()) +#ifdef NODEV + if (PyLong_Check(obj) && _PyLong_IsNegative((PyLongObject *)obj)) { + int overflow; + long long result = PyLong_AsLongLongAndOverflow(obj, &overflow); + if (result == -1 && PyErr_Occurred()) { + return 0; + } + if (!overflow && result == (long long)NODEV) { + *((dev_t *)p) = NODEV; + return 1; + } + } +#endif + + unsigned long long result = PyLong_AsUnsignedLongLong(obj); + if (result == (unsigned long long)-1 && PyErr_Occurred()) { return 0; + } + if ((unsigned long long)(dev_t)result != result) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C dev_t"); + return 0; + } + *((dev_t *)p) = (dev_t)result; return 1; } #endif /* (HAVE_MKNOD && HAVE_MAKEDEV) || HAVE_DEVICE_MACROS */ @@ -962,6 +1025,13 @@ _fd_converter(PyObject *o, int *p) int overflow; long long_value; + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return 0; + } + } PyObject *index = _PyNumber_Index(o); if (index == NULL) { return 0; @@ -1017,13 +1087,17 @@ typedef struct { PyObject *TerminalSizeType; PyObject *TimesResultType; PyObject *UnameResultType; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyObject *WaitidResultType; #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) PyObject *struct_rusage; #endif PyObject *st_mode; +#ifndef MS_WINDOWS + // times() clock frequency in hertz; used by os.times() + long ticks_per_second; +#endif } _posixstate; @@ -1042,16 +1116,15 @@ get_posix_state(PyObject *module) * * path_converter accepts (Unicode) strings and their * subclasses, and bytes and their subclasses. What - * it does with the argument depends on the platform: + * it does with the argument depends on path.make_wide: * - * * On Windows, if we get a (Unicode) string we - * extract the wchar_t * and return it; if we get - * bytes we decode to wchar_t * and return that. + * * If path.make_wide is nonzero, if we get a (Unicode) + * string we extract the wchar_t * and return it; if we + * get bytes we decode to wchar_t * and return that. * - * * On all other platforms, strings are encoded - * to bytes using PyUnicode_FSConverter, then we - * extract the char * from the bytes object and - * return that. + * * If path.make_wide is zero, if we get bytes we extract + * the char_t * and return it; if we get a (Unicode) + * string we encode to char_t * and return that. * * path_converter also optionally accepts signed * integers (representing open file descriptors) instead @@ -1060,6 +1133,15 @@ get_posix_state(PyObject *module) * Input fields: * path.nullable * If nonzero, the path is permitted to be None. + * path.nonstrict + * If nonzero, the path is permitted to contain + * embedded null characters and have any length. + * path.make_wide + * If nonzero, the converter always uses wide, decoding if necessary, else + * it always uses narrow, encoding if necessary. The default value is + * nonzero on Windows, else zero. + * path.suppress_value_error + * If nonzero, raising ValueError is suppressed. * path.allow_fd * If nonzero, the path is permitted to be a file handle * (a signed int) instead of a string. @@ -1075,12 +1157,10 @@ get_posix_state(PyObject *module) * Output fields: * path.wide * Points to the path if it was expressed as Unicode - * and was not encoded. (Only used on Windows.) + * or if it was bytes and decoded to Unicode. * path.narrow * Points to the path if it was expressed as bytes, - * or it was Unicode and was encoded to bytes. (On Windows, - * is a non-zero integer if the path was expressed as bytes. - * The type is deliberately incompatible to prevent misuse.) + * or if it was Unicode and encoded to bytes. * path.fd * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any @@ -1090,6 +1170,9 @@ get_posix_state(PyObject *module) * unspecified, path_converter will never get called. * So if you set allow_fd, you *MUST* initialize path.fd = -1 * yourself! + * path.value_error + * If nonzero, then suppress_value_error was specified and a ValueError + * occurred. * path.length * The length of the path in characters, if specified as * a string. @@ -1122,28 +1205,38 @@ get_posix_state(PyObject *module) * path_cleanup(). However it is safe to do so.) */ typedef struct { + // Input fields const char *function_name; const char *argument_name; int nullable; + int nonstrict; + int make_wide; + int suppress_value_error; int allow_fd; + // Output fields const wchar_t *wide; -#ifdef MS_WINDOWS - BOOL narrow; -#else const char *narrow; -#endif int fd; + int value_error; Py_ssize_t length; PyObject *object; PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ + make_wide, suppress_value_error, allow_fd) \ + {function_name, argument_name, nullable, nonstrict, make_wide, \ + suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} #ifdef MS_WINDOWS -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, FALSE, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 1, \ + suppress_value_error, allow_fd) #else -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, NULL, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 0, \ + suppress_value_error, allow_fd) #endif static void @@ -1164,10 +1257,8 @@ path_converter(PyObject *o, void *p) Py_ssize_t length = 0; int is_index, is_bytes, is_unicode; const char *narrow; -#ifdef MS_WINDOWS PyObject *wo = NULL; wchar_t *wide = NULL; -#endif #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -1188,11 +1279,7 @@ path_converter(PyObject *o, void *p) if ((o == Py_None) && path->nullable) { path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif path->fd = -1; goto success_exit; } @@ -1236,30 +1323,33 @@ path_converter(PyObject *o, void *p) } if (is_unicode) { + if (path->make_wide) { + wide = PyUnicode_AsWideCharString(o, &length); + if (!wide) { + goto error_exit; + } #ifdef MS_WINDOWS - wide = PyUnicode_AsWideCharString(o, &length); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } - path->wide = wide; - path->narrow = FALSE; - path->fd = -1; - wide = NULL; - goto success_exit; -#else - if (!PyUnicode_FSConverter(o, &bytes)) { + path->wide = wide; + path->narrow = NULL; + path->fd = -1; + wide = NULL; + goto success_exit; + } + bytes = PyUnicode_EncodeFSDefault(o); + if (!bytes) { goto error_exit; } -#endif } else if (is_bytes) { bytes = Py_NewRef(o); @@ -1269,11 +1359,7 @@ path_converter(PyObject *o, void *p) goto error_exit; } path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif goto success_exit; } else { @@ -1293,52 +1379,54 @@ path_converter(PyObject *o, void *p) length = PyBytes_GET_SIZE(bytes); narrow = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(narrow)) { + if (!path->nonstrict && strlen(narrow) != (size_t)length) { FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); goto error_exit; } -#ifdef MS_WINDOWS - wo = PyUnicode_DecodeFSDefaultAndSize( - narrow, - length - ); - if (!wo) { - goto error_exit; - } + if (path->make_wide) { + wo = PyUnicode_DecodeFSDefaultAndSize(narrow, length); + if (!wo) { + goto error_exit; + } - wide = PyUnicode_AsWideCharString(wo, &length); - Py_DECREF(wo); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } - path->wide = wide; - path->narrow = TRUE; - Py_DECREF(bytes); - wide = NULL; -#else - path->wide = NULL; - path->narrow = narrow; - if (bytes == o) { - /* Still a reference owned by path->object, don't have to - worry about path->narrow is used after free. */ + wide = PyUnicode_AsWideCharString(wo, &length); + Py_DECREF(wo); + if (!wide) { + goto error_exit; + } +#ifdef MS_WINDOWS + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } + path->wide = wide; + path->narrow = NULL; Py_DECREF(bytes); + wide = NULL; } else { - path->cleanup = bytes; + path->wide = NULL; + path->narrow = narrow; + if (bytes == o) { + /* Still a reference owned by path->object, don't have to + worry about path->narrow is used after free. */ + Py_DECREF(bytes); + } + else { + path->cleanup = bytes; + } } -#endif path->fd = -1; success_exit: + path->value_error = 0; path->length = length; path->object = o; return Py_CLEANUP_SUPPORTED; @@ -1346,10 +1434,20 @@ path_converter(PyObject *o, void *p) error_exit: Py_XDECREF(o); Py_XDECREF(bytes); -#ifdef MS_WINDOWS PyMem_Free(wide); -#endif - return 0; + if (!path->suppress_value_error || + !PyErr_ExceptionMatches(PyExc_ValueError)) + { + return 0; + } + PyErr_Clear(); + path->wide = NULL; + path->narrow = NULL; + path->fd = -1; + path->value_error = 1; + path->length = 0; + path->object = NULL; + return Py_CLEANUP_SUPPORTED; } static void @@ -1399,11 +1497,7 @@ follow_symlinks_specified(const char *function_name, int follow_symlinks) static int path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) { - if (!path->wide && (dir_fd != DEFAULT_DIR_FD) -#ifndef MS_WINDOWS - && !path->narrow -#endif - ) { + if (!path->wide && (dir_fd != DEFAULT_DIR_FD) && !path->narrow) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", function_name); @@ -1547,6 +1641,7 @@ _Py_Sigset_Converter(PyObject *obj, void *addr) ** man environ(7). */ #include +#define USE_DARWIN_NS_GET_ENVIRON 1 #elif !defined(_MSC_VER) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) extern char **environ; #endif /* !_MSC_VER */ @@ -1569,7 +1664,7 @@ convertenviron(void) through main() instead of wmain(). */ (void)_wgetenv(L""); e = _wenviron; -#elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) +#elif defined(USE_DARWIN_NS_GET_ENVIRON) /* environ is not accessible as an extern in a shared object on OSX; use _NSGetEnviron to resolve it. The value changes if you add environment variables between calls to Py_Initialize, so don't cache the value. */ @@ -1608,7 +1703,7 @@ convertenviron(void) Py_DECREF(d); return NULL; } - if (PyDict_SetDefault(d, k, v) == NULL) { + if (PyDict_SetDefaultRef(d, k, v, NULL) < 0) { Py_DECREF(v); Py_DECREF(k); Py_DECREF(d); @@ -1874,8 +1969,9 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, HANDLE hFile; BY_HANDLE_FILE_INFORMATION fileInfo; FILE_BASIC_INFO basicInfo; + FILE_BASIC_INFO *pBasicInfo = NULL; FILE_ID_INFO idInfo; - FILE_ID_INFO *pIdInfo = &idInfo; + FILE_ID_INFO *pIdInfo = NULL; FILE_ATTRIBUTE_TAG_INFO tagInfo = { 0 }; DWORD fileType, error; BOOL isUnhandledTag = FALSE; @@ -2026,14 +2122,17 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, retval = -1; goto cleanup; } - } - if (!GetFileInformationByHandleEx(hFile, FileIdInfo, &idInfo, sizeof(idInfo))) { - /* Failed to get FileIdInfo, so do not pass it along */ - pIdInfo = NULL; + /* Successfully got FileBasicInfo, so we'll pass it along */ + pBasicInfo = &basicInfo; + + if (GetFileInformationByHandleEx(hFile, FileIdInfo, &idInfo, sizeof(idInfo))) { + /* Successfully got FileIdInfo, so pass it along */ + pIdInfo = &idInfo; + } } - _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, &basicInfo, pIdInfo, result); + _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, pBasicInfo, pIdInfo, result); update_st_mode_from_path(path, fileInfo.dwFileAttributes, result); cleanup: @@ -2280,7 +2379,7 @@ static PyStructSequence_Desc statvfs_result_desc = { 10 }; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyDoc_STRVAR(waitid_result__doc__, "waitid_result: Result from waitid.\n\n\ This object may be accessed either as a tuple of\n\ @@ -2355,7 +2454,7 @@ _posix_clear(PyObject *module) Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); Py_CLEAR(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_CLEAR(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2380,7 +2479,7 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); Py_VISIT(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_VISIT(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2844,6 +2943,8 @@ FTRUNCATE #ifdef MS_WINDOWS #undef PATH_HAVE_FTRUNCATE #define PATH_HAVE_FTRUNCATE 1 + #undef PATH_HAVE_FCHMOD + #define PATH_HAVE_FCHMOD 1 #endif /*[python input] @@ -2856,7 +2957,9 @@ class path_t_converter(CConverter): converter = 'path_converter' - def converter_init(self, *, allow_fd=False, nullable=False): + def converter_init(self, *, allow_fd=False, make_wide=None, + nonstrict=False, nullable=False, + suppress_value_error=False): # right now path_t doesn't support default values. # to support a default value, you'll need to override initialize(). if self.default not in (unspecified, None): @@ -2866,6 +2969,9 @@ class path_t_converter(CConverter): raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable + self.nonstrict = nonstrict + self.make_wide = make_wide + self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd def pre_render(self): @@ -2875,11 +2981,24 @@ class path_t_converter(CConverter): return str(int(bool(value))) # add self.py_name here when merging with posixmodule conversion - self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {})'.format( - self.function.name, - self.name, - strify(self.nullable), - strify(self.allow_fd), + if self.make_wide is None: + self.c_default = 'PATH_T_INITIALIZE_P("{}", "{}", {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.suppress_value_error), + strify(self.allow_fd), + ) + else: + self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.make_wide), + strify(self.suppress_value_error), + strify(self.allow_fd), ) def cleanup(self): @@ -2959,7 +3078,7 @@ class sysconf_confname_converter(path_confname_converter): converter="conv_sysconf_confname" [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3338733161aa7879]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=577cb476e5d64960]*/ /*[clinic input] @@ -3299,6 +3418,60 @@ os_fchdir_impl(PyObject *module, int fd) } #endif /* HAVE_FCHDIR */ +#ifdef MS_WINDOWS +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 0 +#else +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 1 +#endif + +#ifdef MS_WINDOWS +static int +win32_lchmod(LPCWSTR path, int mode) +{ + DWORD attr = GetFileAttributesW(path); + if (attr == INVALID_FILE_ATTRIBUTES) { + return 0; + } + if (mode & _S_IWRITE) { + attr &= ~FILE_ATTRIBUTE_READONLY; + } + else { + attr |= FILE_ATTRIBUTE_READONLY; + } + return SetFileAttributesW(path, attr); +} + +static int +win32_hchmod(HANDLE hfile, int mode) +{ + FILE_BASIC_INFO info; + if (!GetFileInformationByHandleEx(hfile, FileBasicInfo, + &info, sizeof(info))) + { + return 0; + } + if (mode & _S_IWRITE) { + info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; + } + else { + info.FileAttributes |= FILE_ATTRIBUTE_READONLY; + } + return SetFileInformationByHandle(hfile, FileBasicInfo, + &info, sizeof(info)); +} + +static int +win32_fchmod(int fd, int mode) +{ + HANDLE hfile = _Py_get_osfhandle_noraise(fd); + if (hfile == INVALID_HANDLE_VALUE) { + SetLastError(ERROR_INVALID_HANDLE); + return 0; + } + return win32_hchmod(hfile, mode); +} + +#endif /* MS_WINDOWS */ /*[clinic input] os.chmod @@ -3321,7 +3494,8 @@ os.chmod and path should be relative; path will then be relative to that directory. - follow_symlinks: bool = True + follow_symlinks: bool(c_default="CHMOD_DEFAULT_FOLLOW_SYMLINKS", \ + py_default="(os.name != 'nt')") = CHMOD_DEFAULT_FOLLOW_SYMLINKS If False, and the last element of the path is a symbolic link, chmod will modify the symbolic link itself instead of the file the link points to. @@ -3338,20 +3512,16 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=674a14bc998de09d]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ { int result; -#ifdef MS_WINDOWS - DWORD attr; -#endif - #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; int fchmodat_unsupported = 0; #endif -#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) +#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) if (follow_symlinks_specified("chmod", follow_symlinks)) return NULL; #endif @@ -3362,19 +3532,25 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } #ifdef MS_WINDOWS + result = 0; Py_BEGIN_ALLOW_THREADS - attr = GetFileAttributesW(path->wide); - if (attr == INVALID_FILE_ATTRIBUTES) - result = 0; + if (path->fd != -1) { + result = win32_fchmod(path->fd, mode); + } + else if (follow_symlinks) { + HANDLE hfile = CreateFileW(path->wide, + FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES, + 0, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + result = win32_hchmod(hfile, mode); + (void)CloseHandle(hfile); + } + } else { - if (mode & _S_IWRITE) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - result = SetFileAttributesW(path->wide, attr); + result = win32_lchmod(path->wide, mode); } Py_END_ALLOW_THREADS - if (!result) { return path_error(path); } @@ -3463,7 +3639,7 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } -#ifdef HAVE_FCHMOD +#if defined(HAVE_FCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.fchmod @@ -3485,12 +3661,21 @@ os_fchmod_impl(PyObject *module, int fd, int mode) /*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ { int res; - int async_err = 0; if (PySys_Audit("os.chmod", "iii", fd, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + res = 0; + Py_BEGIN_ALLOW_THREADS + res = win32_fchmod(fd, mode); + Py_END_ALLOW_THREADS + if (!res) { + return PyErr_SetFromWindowsErr(0); + } +#else /* MS_WINDOWS */ + int async_err = 0; do { Py_BEGIN_ALLOW_THREADS res = fchmod(fd, mode); @@ -3498,13 +3683,14 @@ os_fchmod_impl(PyObject *module, int fd, int mode) } while (res != 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (res != 0) return (!async_err) ? posix_error() : NULL; +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_FCHMOD */ +#endif /* HAVE_FCHMOD || MS_WINDOWS */ -#ifdef HAVE_LCHMOD +#if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.lchmod @@ -3525,6 +3711,15 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + Py_BEGIN_ALLOW_THREADS + res = win32_lchmod(path->wide, mode); + Py_END_ALLOW_THREADS + if (!res) { + path_error(path); + return NULL; + } +#else /* MS_WINDOWS */ Py_BEGIN_ALLOW_THREADS res = lchmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3532,9 +3727,10 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) path_error(path); return NULL; } +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_LCHMOD */ +#endif /* HAVE_LCHMOD || MS_WINDOWS */ #ifdef HAVE_CHFLAGS @@ -3974,6 +4170,20 @@ posix_getcwd(int use_bytes) else { obj = PyUnicode_DecodeFSDefault(buf); } +#ifdef __linux__ + if (buf[0] != '/') { + /* + * On Linux >= 2.6.36 with glibc < 2.27, getcwd() can return a + * relative pathname starting with '(unreachable)'. We detect this + * and fail with ENOENT, matching newer glibc behaviour. + */ + errno = ENOENT; + path_object_error(obj); + PyMem_RawFree(buf); + return NULL; + } +#endif + assert(buf[0] == '/'); PyMem_RawFree(buf); return obj; @@ -4137,7 +4347,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; - BOOL result; + BOOL result, return_bytes; wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; @@ -4149,9 +4359,11 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; len = 1; + return_bytes = 0; } else { po_wchars = path->wide; len = wcslen(path->wide); + return_bytes = PyBytes_Check(path->object); } /* The +5 is so we can append "\\*.*\0" */ wnamebuf = PyMem_New(wchar_t, len + 5); @@ -4186,7 +4398,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) wcscmp(wFileData.cFileName, L"..") != 0) { v = PyUnicode_FromWideChar(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (path->narrow && v) { + if (return_bytes && v) { Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { @@ -4729,7 +4941,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) if (str == NULL) { return NULL; } - if (path->narrow) { + if (PyBytes_Check(path->object)) { Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); } return str; @@ -4802,7 +5014,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } result = PyUnicode_FromWideChar(target_path, result_length); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -4885,7 +5097,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) goto exit; } result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); - if (path->narrow) + if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); exit: @@ -4940,413 +5152,428 @@ os__path_splitroot_impl(PyObject *module, path_t *path) } -/*[clinic input] -os._path_isdir - - s: 'O' +#define PY_IFREG 1 // Regular file +#define PY_IFDIR 2 // Directory +#define PY_IFLNK 4 // Symlink +#define PY_IFMNT 8 // Mount Point (junction) +#define PY_IFLRP 16 // Link Reparse Point (name-surrogate, symlink, junction) +#define PY_IFRRP 32 // Regular Reparse Point -Return true if the pathname refers to an existing directory. +static inline BOOL +_testInfo(DWORD attributes, DWORD reparseTag, BOOL diskDevice, int testedType) +{ + switch (testedType) { + case PY_IFREG: + return diskDevice && attributes && + !(attributes & FILE_ATTRIBUTE_DIRECTORY); + case PY_IFDIR: + return attributes & FILE_ATTRIBUTE_DIRECTORY; + case PY_IFLNK: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_SYMLINK; + case PY_IFMNT: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_MOUNT_POINT; + case PY_IFLRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + IsReparseTagNameSurrogate(reparseTag); + case PY_IFRRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag && !IsReparseTagNameSurrogate(reparseTag); + } -[clinic start generated code]*/ + return FALSE; +} -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s) -/*[clinic end generated code: output=9d87ab3c8b8a4e61 input=c17f7ef21d22d64e]*/ +static BOOL +_testFileTypeByHandle(HANDLE hfile, int testedType, BOOL diskOnly) { - HANDLE hfile; - BOOL close_file = TRUE; + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + BOOL diskDevice = GetFileType(hfile) == FILE_TYPE_DISK; + if (diskOnly && !diskDevice) { + return FALSE; + } + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + FILE_ATTRIBUTE_TAG_INFO info; + return GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, info.ReparseTag, diskDevice, + testedType); + } FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isdir", "s", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + return GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, 0, diskDevice, testedType); +} - if (!path_converter(s, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; +static BOOL +_testFileTypeByName(LPCWSTR path, int testedType) +{ + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + BOOL diskDevice = info.DeviceType == FILE_DEVICE_DISK || + info.DeviceType == FILE_DEVICE_VIRTUAL_DISK || + info.DeviceType == FILE_DEVICE_CD_ROM; + BOOL result = _testInfo(info.FileAttributes, info.ReparseTag, + diskDevice, testedType); + if (!result || (testedType != PY_IFREG && testedType != PY_IFDIR) || + !(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) + { + return result; } - return NULL; + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } else if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - slow_path = FALSE; - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + BOOL result = _testFileTypeByHandle(hfile, testedType, FALSE); + CloseHandle(hfile); + return result; + } + + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + int rc; + STRUCT_STAT st; + if (testedType == PY_IFREG || testedType == PY_IFDIR) { + rc = STAT(path, &st); } else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + // PY_IFRRP is not generally supported in this case, except for + // unhandled reparse points such as IO_REPARSE_TAG_APPEXECLINK. + rc = LSTAT(path, &st); } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISDIR(st.st_mode); - } - break; - default: - result = 0; - } + if (!rc) { + return _testInfo(st.st_file_attributes, st.st_reparse_tag, + st.st_mode & S_IFREG, testedType); } } - Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return FALSE; } -/*[clinic input] -os._path_isfile - - path: 'O' - -Test whether a path is a regular file - -[clinic start generated code]*/ - -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2394ed7c4b5cfd85 input=de22d74960ade365]*/ +static BOOL +_testFileExistsByName(LPCWSTR path, BOOL followLinks) { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isfile", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; - - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + if (!(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) || + !followLinks && IsReparseTagNameSurrogate(info.ReparseTag)) + { + return TRUE; } - return NULL; + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = !(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } else if (statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - slow_path = FALSE; - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followLinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + if (followLinks) { + CloseHandle(hfile); + return TRUE; } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + // Regular Reparse Points (PY_IFRRP) have to be traversed. + BOOL result = _testFileTypeByHandle(hfile, PY_IFRRP, FALSE); + CloseHandle(hfile); + if (!result) { + return TRUE; } + hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = !(info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISREG(st.st_mode); - } - break; - default: - result = 0; - } + CloseHandle(hfile); + return TRUE; } } - Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + STRUCT_STAT _st; + return followLinks ? !STAT(path, &_st): !LSTAT(path, &_st); } - Py_RETURN_FALSE; + + return FALSE; } -/*[clinic input] -os._path_exists +static BOOL +_testFileExists(path_t *path, BOOL followLinks) +{ + BOOL result = FALSE; + if (path->value_error) { + return FALSE; + } - path: 'O' + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); + if (hfile != INVALID_HANDLE_VALUE) { + if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { + result = TRUE; + } + } + } + else if (path->wide) { + result = _testFileExistsByName(path->wide, followLinks); + } + Py_END_ALLOW_THREADS -Test whether a path exists. Returns False for broken symbolic links + return result; +} -[clinic start generated code]*/ -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/ +static BOOL +_testFileType(path_t *path, int testedType) { - HANDLE hfile; - BOOL close_file = TRUE; - path_t _path = PATH_T_INITIALIZE("exists", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; - - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; + BOOL result = FALSE; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = 1; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } - } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { - result = 1; - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = 1; - } - break; - default: - result = 0; - } + result = _testFileTypeByHandle(hfile, testedType, TRUE); } } + else if (path->wide) { + result = _testFileTypeByName(path->wide, testedType); + } Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return result; } /*[clinic input] -os._path_islink +os._path_exists -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + / - path: 'O' +Test whether a path exists. Returns False for broken symbolic links. + +[clinic start generated code]*/ + +static int +os__path_exists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=8da13acf666e16ba input=29198507a6082a57]*/ +{ + return _testFileExists(path, TRUE); +} + + +/*[clinic input] +os._path_lexists -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + / + +Test whether a path exists. Returns True for broken symbolic links. + +[clinic start generated code]*/ + +static int +os__path_lexists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e7240ed5fc45bff3 input=03d9fed8bc6ce96f]*/ +{ + return _testFileExists(path, FALSE); +} + + +/*[clinic input] +os._path_isdir -> bool + + s as path: path_t(allow_fd=True, suppress_value_error=True) + +Return true if the pathname refers to an existing directory. + +[clinic start generated code]*/ + +static int +os__path_isdir_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ +{ + return _testFileType(path, PY_IFDIR); +} + + +/*[clinic input] +os._path_isfile -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a regular file + +[clinic start generated code]*/ + +static int +os__path_isfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=5c3073bc212b9863 input=4ac1fd350b30a39e]*/ +{ + return _testFileType(path, PY_IFREG); +} + + +/*[clinic input] +os._path_islink -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a symbolic link [clinic start generated code]*/ +static int +os__path_islink_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=30da7bda8296adcc input=7510ce05b547debb]*/ +{ + return _testFileType(path, PY_IFLNK); +} + + +/*[clinic input] +os._path_isjunction -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a junction + +[clinic start generated code]*/ + +static int +os__path_isjunction_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e1d17a9dd18a9945 input=7dcb8bc4e972fcaf]*/ +{ + return _testFileType(path, PY_IFMNT); +} + +#undef PY_IFREG +#undef PY_IFDIR +#undef PY_IFLNK +#undef PY_IFMNT +#undef PY_IFLRP +#undef PY_IFRRP + +#endif /* MS_WINDOWS */ + + +/*[clinic input] +os._path_splitroot_ex + + path: path_t(make_wide=True, nonstrict=True) + +Split a pathname into drive, root and tail. + +The tail contains anything after the root. +[clinic start generated code]*/ + static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6d8640b1a390c054 input=38a3cb937ccf59bf]*/ +os__path_splitroot_ex_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=4b0072b6cdf4b611 input=6eb76e9173412c92]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_ATTRIBUTE_TAG_INFO info; - path_t _path = PATH_T_INITIALIZE("islink", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + Py_ssize_t drvsize, rootsize; + PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; + const wchar_t *buffer = path->wide; + _Py_skiproot(buffer, path->length, &drvsize, &rootsize); + drv = PyUnicode_FromWideChar(buffer, drvsize); + if (drv == NULL) { + goto exit; } - - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - slow_path = FALSE; - if (statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - result = (statInfo.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + root = PyUnicode_FromWideChar(&buffer[drvsize], rootsize); + if (root == NULL) { + goto exit; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, - NULL); + tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], + path->length - drvsize - rootsize); + if (tail == NULL) { + goto exit; + } + if (PyBytes_Check(path->object)) { + Py_SETREF(drv, PyUnicode_EncodeFSDefault(drv)); + if (drv == NULL) { + goto exit; } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, - sizeof(info))) - { - result = (info.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } + Py_SETREF(root, PyUnicode_EncodeFSDefault(root)); + if (root == NULL) { + goto exit; } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (LSTAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISLNK(st.st_mode); - } - break; - default: - result = 0; - } + Py_SETREF(tail, PyUnicode_EncodeFSDefault(tail)); + if (tail == NULL) { + goto exit; } } - Py_END_ALLOW_THREADS - - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + result = PyTuple_Pack(3, drv, root, tail); +exit: + Py_XDECREF(drv); + Py_XDECREF(root); + Py_XDECREF(tail); + return result; } -#endif /* MS_WINDOWS */ - /*[clinic input] os._path_normpath - path: object + path: path_t(make_wide=True, nonstrict=True) -Basic path normalization. +Normalize path, eliminating double slashes, etc. [clinic start generated code]*/ static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +os__path_normpath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d353e7ed9410c044 input=3d4ac23b06332dcb]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; + PyObject *result; + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size((wchar_t *)path->wide, + path->length, &norm_len); + if (!norm_len) { + result = PyUnicode_FromOrdinal('.'); } - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - return NULL; + else { + result = PyUnicode_FromWideChar(norm_path, norm_len); + } + if (PyBytes_Check(path->object)) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } - Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); - PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); - PyMem_Free(buffer); return result; } @@ -5379,6 +5606,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + int pathError = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; + SECURITY_ATTRIBUTES *pSecAttr = NULL; +#endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; #endif @@ -5390,11 +5623,38 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discretionary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + return PyErr_SetFromWindowsErr(error); + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT @@ -6777,6 +7037,9 @@ enum posix_spawn_file_actions_identifier { POSIX_SPAWN_OPEN, POSIX_SPAWN_CLOSE, POSIX_SPAWN_DUP2 +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + ,POSIX_SPAWN_CLOSEFROM +#endif }; #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) @@ -7017,6 +7280,24 @@ parse_file_actions(PyObject *file_actions, } break; } +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + case POSIX_SPAWN_CLOSEFROM: { + int fd; + if (!PyArg_ParseTuple(file_action, "Oi" + ";A closefrom file_action tuple must have 2 elements", + &tag_obj, &fd)) + { + goto fail; + } + errno = posix_spawn_file_actions_addclosefrom_np(file_actionsp, + fd); + if (errno) { + posix_error(); + goto fail; + } + break; + } +#endif default: { PyErr_SetString(PyExc_TypeError, "Unknown file_actions identifier"); @@ -7072,9 +7353,9 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a return NULL; } - if (!PyMapping_Check(env)) { + if (!PyMapping_Check(env) && env != Py_None) { PyErr_Format(PyExc_TypeError, - "%s: environment must be a mapping object", func_name); + "%s: environment must be a mapping object or None", func_name); goto exit; } @@ -7088,9 +7369,21 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a goto exit; } - envlist = parse_envlist(env, &envc); - if (envlist == NULL) { - goto exit; +#ifdef USE_DARWIN_NS_GET_ENVIRON + // There is no environ global in this situation. + char **environ = NULL; +#endif + + if (env == Py_None) { +#ifdef USE_DARWIN_NS_GET_ENVIRON + environ = *_NSGetEnviron(); +#endif + envlist = environ; + } else { + envlist = parse_envlist(env, &envc); + if (envlist == NULL) { + goto exit; + } } if (file_actions != NULL && file_actions != Py_None) { @@ -7153,7 +7446,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a if (attrp) { (void)posix_spawnattr_destroy(attrp); } - if (envlist) { + if (envlist && envlist != environ) { free_string_array(envlist, envc); } if (argvlist) { @@ -7580,14 +7873,20 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, } #endif /* HAVE_FORK */ +#if defined(HAVE_FORK1) || defined(HAVE_FORKPTY) || defined(HAVE_FORK) // Common code to raise a warning if we detect there is more than one thread // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This code might do an import, thus acquiring the import lock, which -// PyOS_BeforeFork() also does. As this should only be called from -// the parent process, it is in the same thread so that works. -static void warn_about_fork_with_threads(const char* name) { +// This should only be called from the parent process after +// PyOS_AfterFork_Parent(). +static void +warn_about_fork_with_threads(const char* name) +{ + // It's not safe to issue the warning while the world is stopped, because + // other threads might be holding locks that we need, which would deadlock. + assert(!_PyRuntime.stoptheworld.world_stopped); + // TODO: Consider making an `os` module API to return the current number // of threads in the process. That'd presumably use this platform code but // raise an error rather than using the inaccurate fallback. @@ -7678,6 +7977,7 @@ static void warn_about_fork_with_threads(const char* name) { PyErr_Clear(); } } +#endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 /*[clinic input] @@ -7695,8 +7995,8 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -7711,9 +8011,10 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork1"); } if (pid == -1) { errno = saved_errno; @@ -7739,8 +8040,8 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -7759,9 +8060,10 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork"); } if (pid == -1) { errno = saved_errno; @@ -8194,41 +8496,184 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid) ncpus *= 2; } - PyObject *res = PySet_New(NULL); - if (res == NULL) { - goto error; - } + PyObject *res = PySet_New(NULL); + if (res == NULL) { + goto error; + } + + int cpu = 0; + int count = CPU_COUNT_S(setsize, mask); + for (; count; cpu++) { + if (CPU_ISSET_S(cpu, setsize, mask)) { + PyObject *cpu_num = PyLong_FromLong(cpu); + --count; + if (cpu_num == NULL) { + goto error; + } + if (PySet_Add(res, cpu_num)) { + Py_DECREF(cpu_num); + goto error; + } + Py_DECREF(cpu_num); + } + } + CPU_FREE(mask); + return res; + +error: + if (mask) { + CPU_FREE(mask); + } + Py_XDECREF(res); + return NULL; +} +#endif /* HAVE_SCHED_SETAFFINITY */ + +#endif /* HAVE_SCHED_H */ + + +#ifdef HAVE_POSIX_OPENPT +/*[clinic input] +os.posix_openpt -> int + + oflag: int + / + +Open and return a file descriptor for a master pseudo-terminal device. + +Performs a posix_openpt() C function call. The oflag argument is used to +set file status flags and file access modes as specified in the manual page +of posix_openpt() of your system. +[clinic start generated code]*/ + +static int +os_posix_openpt_impl(PyObject *module, int oflag) +/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +{ + int fd; + +#if defined(O_CLOEXEC) + oflag |= O_CLOEXEC; +#endif + + fd = posix_openpt(oflag); + if (fd == -1) { + posix_error(); + return -1; + } + + // Just in case, likely a no-op given O_CLOEXEC above. + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } + + return fd; +} +#endif /* HAVE_POSIX_OPENPT */ + +#ifdef HAVE_GRANTPT +/*[clinic input] +os.grantpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Grant access to the slave pseudo-terminal device. + +Performs a grantpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_grantpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=dfd580015cf548ab input=0668e3b96760e849]*/ +{ + int ret; + int saved_errno; + PyOS_sighandler_t sig_saved; + + sig_saved = PyOS_setsig(SIGCHLD, SIG_DFL); + + ret = grantpt(fd); + if (ret == -1) + saved_errno = errno; + + PyOS_setsig(SIGCHLD, sig_saved); + + if (ret == -1) { + errno = saved_errno; + return posix_error(); + } + + Py_RETURN_NONE; +} +#endif /* HAVE_GRANTPT */ + +#ifdef HAVE_UNLOCKPT +/*[clinic input] +os.unlockpt + + fd: fildes + File descriptor of a master pseudo-terminal device. + / + +Unlock a pseudo-terminal master/slave pair. + +Performs an unlockpt() C function call. +[clinic start generated code]*/ + +static PyObject * +os_unlockpt_impl(PyObject *module, int fd) +/*[clinic end generated code: output=e08d354dec12d30c input=de7ab1f59f69a2b4]*/ +{ + if (unlockpt(fd) == -1) + return posix_error(); + + Py_RETURN_NONE; +} +#endif /* HAVE_UNLOCKPT */ + +#if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +/*[clinic input] +os.ptsname - int cpu = 0; - int count = CPU_COUNT_S(setsize, mask); - for (; count; cpu++) { - if (CPU_ISSET_S(cpu, setsize, mask)) { - PyObject *cpu_num = PyLong_FromLong(cpu); - --count; - if (cpu_num == NULL) { - goto error; - } - if (PySet_Add(res, cpu_num)) { - Py_DECREF(cpu_num); - goto error; - } - Py_DECREF(cpu_num); - } - } - CPU_FREE(mask); - return res; + fd: fildes + File descriptor of a master pseudo-terminal device. + / -error: - if (mask) { - CPU_FREE(mask); +Return the name of the slave pseudo-terminal device. + +If the ptsname_r() C function is available, it is called; +otherwise, performs a ptsname() C function call. +[clinic start generated code]*/ + +static PyObject * +os_ptsname_impl(PyObject *module, int fd) +/*[clinic end generated code: output=ef300fadc5675872 input=1369ccc0546f3130]*/ +{ +#ifdef HAVE_PTSNAME_R + int ret; + char name[MAXPATHLEN+1]; + + ret = ptsname_r(fd, name, sizeof(name)); + if (ret != 0) { + errno = ret; + return posix_error(); } - Py_XDECREF(res); - return NULL; -} -#endif /* HAVE_SCHED_SETAFFINITY */ +#else + char *name; -#endif /* HAVE_SCHED_H */ + name = ptsname(fd); + /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. + *MAY* set errno? Hmm... */ + if (name == NULL) + return posix_error(); +#endif /* HAVE_PTSNAME_R */ + return PyUnicode_DecodeFSDefault(name); +} +#endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) @@ -8429,8 +8874,8 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { - PyErr_SetString(PyExc_RuntimeError, + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; } @@ -8447,9 +8892,10 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("forkpty"); } if (pid == -1) { return posix_error(); @@ -8825,7 +9271,81 @@ os_setpgrp_impl(PyObject *module) #ifdef HAVE_GETPPID #ifdef MS_WINDOWS -#include +#include +#include + +// The structure definition in winternl.h may be incomplete. +// This structure is the full version from the MSDN documentation. +typedef struct _PROCESS_BASIC_INFORMATION_FULL { + NTSTATUS ExitStatus; + PVOID PebBaseAddress; + ULONG_PTR AffinityMask; + LONG BasePriority; + ULONG_PTR UniqueProcessId; + ULONG_PTR InheritedFromUniqueProcessId; +} PROCESS_BASIC_INFORMATION_FULL; + +typedef NTSTATUS (NTAPI *PNT_QUERY_INFORMATION_PROCESS) ( + IN HANDLE ProcessHandle, + IN PROCESSINFOCLASS ProcessInformationClass, + OUT PVOID ProcessInformation, + IN ULONG ProcessInformationLength, + OUT PULONG ReturnLength OPTIONAL); + +// This function returns the process ID of the parent process. +// Returns 0 on failure. +static ULONG +win32_getppid_fast(void) +{ + NTSTATUS status; + HMODULE ntdll; + PNT_QUERY_INFORMATION_PROCESS pNtQueryInformationProcess; + PROCESS_BASIC_INFORMATION_FULL basic_information; + static ULONG cached_ppid = 0; + + if (cached_ppid) { + // No need to query the kernel again. + return cached_ppid; + } + + ntdll = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) { + return 0; + } + + pNtQueryInformationProcess = (PNT_QUERY_INFORMATION_PROCESS) GetProcAddress(ntdll, "NtQueryInformationProcess"); + if (!pNtQueryInformationProcess) { + return 0; + } + + status = pNtQueryInformationProcess(GetCurrentProcess(), + ProcessBasicInformation, + &basic_information, + sizeof(basic_information), + NULL); + + if (!NT_SUCCESS(status)) { + return 0; + } + + // Perform sanity check on the parent process ID we received from NtQueryInformationProcess. + // The check covers values which exceed the 32-bit range (if running on x64) as well as + // zero and (ULONG) -1. + + if (basic_information.InheritedFromUniqueProcessId == 0 || + basic_information.InheritedFromUniqueProcessId >= ULONG_MAX) + { + return 0; + } + + // Now that we have reached this point, the BasicInformation.InheritedFromUniqueProcessId + // structure member contains a ULONG_PTR which represents the process ID of our parent + // process. This process ID will be correctly returned even if the parent process has + // exited or been terminated. + + cached_ppid = (ULONG) basic_information.InheritedFromUniqueProcessId; + return cached_ppid; +} static PyObject* win32_getppid(void) @@ -8833,8 +9353,16 @@ win32_getppid(void) DWORD error; PyObject* result = NULL; HANDLE process = GetCurrentProcess(); - HPSS snapshot = NULL; + ULONG pid; + + pid = win32_getppid_fast(); + if (pid != 0) { + return PyLong_FromUnsignedLong(pid); + } + + // If failure occurs in win32_getppid_fast(), fall back to using the PSS API. + error = PssCaptureSnapshot(process, PSS_CAPTURE_NONE, 0, &snapshot); if (error != ERROR_SUCCESS) { return PyErr_SetFromWindowsErr(error); @@ -9287,36 +9815,39 @@ wait_helper(PyObject *module, pid_t pid, int status, struct rusage *ru) if (!result) return NULL; + int pos = 0; + #ifndef doubletime #define doubletime(TV) ((double)(TV).tv_sec + (TV).tv_usec * 0.000001) #endif - PyStructSequence_SET_ITEM(result, 0, - PyFloat_FromDouble(doubletime(ru->ru_utime))); - PyStructSequence_SET_ITEM(result, 1, - PyFloat_FromDouble(doubletime(ru->ru_stime))); -#define SET_INT(result, index, value)\ - PyStructSequence_SET_ITEM(result, index, PyLong_FromLong(value)) - SET_INT(result, 2, ru->ru_maxrss); - SET_INT(result, 3, ru->ru_ixrss); - SET_INT(result, 4, ru->ru_idrss); - SET_INT(result, 5, ru->ru_isrss); - SET_INT(result, 6, ru->ru_minflt); - SET_INT(result, 7, ru->ru_majflt); - SET_INT(result, 8, ru->ru_nswap); - SET_INT(result, 9, ru->ru_inblock); - SET_INT(result, 10, ru->ru_oublock); - SET_INT(result, 11, ru->ru_msgsnd); - SET_INT(result, 12, ru->ru_msgrcv); - SET_INT(result, 13, ru->ru_nsignals); - SET_INT(result, 14, ru->ru_nvcsw); - SET_INT(result, 15, ru->ru_nivcsw); -#undef SET_INT - - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_utime))); + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_stime))); + SET_RESULT(PyLong_FromLong(ru->ru_maxrss)); + SET_RESULT(PyLong_FromLong(ru->ru_ixrss)); + SET_RESULT(PyLong_FromLong(ru->ru_idrss)); + SET_RESULT(PyLong_FromLong(ru->ru_isrss)); + SET_RESULT(PyLong_FromLong(ru->ru_minflt)); + SET_RESULT(PyLong_FromLong(ru->ru_majflt)); + SET_RESULT(PyLong_FromLong(ru->ru_nswap)); + SET_RESULT(PyLong_FromLong(ru->ru_inblock)); + SET_RESULT(PyLong_FromLong(ru->ru_oublock)); + SET_RESULT(PyLong_FromLong(ru->ru_msgsnd)); + SET_RESULT(PyLong_FromLong(ru->ru_msgrcv)); + SET_RESULT(PyLong_FromLong(ru->ru_nsignals)); + SET_RESULT(PyLong_FromLong(ru->ru_nvcsw)); + SET_RESULT(PyLong_FromLong(ru->ru_nivcsw)); +#undef SET_RESULT return Py_BuildValue("NiN", PyLong_FromPid(pid), status, result); } @@ -9394,7 +9925,7 @@ os_wait4_impl(PyObject *module, pid_t pid, int options) #endif /* HAVE_WAIT4 */ -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) /*[clinic input] os.waitid @@ -9439,19 +9970,29 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) if (!result) return NULL; - PyStructSequence_SET_ITEM(result, 0, PyLong_FromPid(si.si_pid)); - PyStructSequence_SET_ITEM(result, 1, _PyLong_FromUid(si.si_uid)); - PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si.si_signo))); - PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong((long)(si.si_status))); - PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si.si_code))); - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyLong_FromPid(si.si_pid)); + SET_RESULT(_PyLong_FromUid(si.si_uid)); + SET_RESULT(PyLong_FromLong((long)(si.si_signo))); + SET_RESULT(PyLong_FromLong((long)(si.si_status))); + SET_RESULT(PyLong_FromLong((long)(si.si_code))); + +#undef SET_RESULT return result; } -#endif /* defined(HAVE_WAITID) && !defined(__APPLE__) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -9761,7 +10302,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) name[1] = L'\\'; } result = PyUnicode_FromWideChar(name, nameLen); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } } @@ -9980,8 +10521,6 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #endif /* HAVE_SYMLINK */ - - static PyStructSequence_Field times_result_fields[] = { {"user", "user time"}, {"system", "system time"}, @@ -10007,12 +10546,6 @@ static PyStructSequence_Desc times_result_desc = { 5 }; -#ifdef MS_WINDOWS -#define HAVE_TIMES /* mandatory, for the method table */ -#endif - -#ifdef HAVE_TIMES - static PyObject * build_times_result(PyObject *module, double user, double system, double children_user, double children_system, @@ -10058,8 +10591,8 @@ All fields are floating point numbers. static PyObject * os_times_impl(PyObject *module) /*[clinic end generated code: output=35f640503557d32a input=2bf9df3d6ab2e48b]*/ -#ifdef MS_WINDOWS { +#ifdef MS_WINDOWS FILETIME create, exit, kernel, user; HANDLE hProc; hProc = GetCurrentProcess(); @@ -10077,28 +10610,26 @@ os_times_impl(PyObject *module) (double)0, (double)0, (double)0); -} #else /* MS_WINDOWS */ -{ - struct tms t; - clock_t c; + _posixstate *state = get_posix_state(module); + long ticks_per_second = state->ticks_per_second; + + struct tms process; + clock_t elapsed; errno = 0; - c = times(&t); - if (c == (clock_t) -1) { + elapsed = times(&process); + if (elapsed == (clock_t) -1) { return posix_error(); } - assert(_PyRuntime.time.ticks_per_second_initialized); -#define ticks_per_second _PyRuntime.time.ticks_per_second + return build_times_result(module, - (double)t.tms_utime / ticks_per_second, - (double)t.tms_stime / ticks_per_second, - (double)t.tms_cutime / ticks_per_second, - (double)t.tms_cstime / ticks_per_second, - (double)c / ticks_per_second); -#undef ticks_per_second -} + (double)process.tms_utime / ticks_per_second, + (double)process.tms_stime / ticks_per_second, + (double)process.tms_cutime / ticks_per_second, + (double)process.tms_cstime / ticks_per_second, + (double)elapsed / ticks_per_second); #endif /* MS_WINDOWS */ -#endif /* HAVE_TIMES */ +} #if defined(HAVE_TIMERFD_CREATE) @@ -10131,7 +10662,7 @@ build_itimerspec(const struct itimerspec* curr_value) static PyObject * build_itimerspec_ns(const struct itimerspec* curr_value) { - _PyTime_t value, interval; + PyTime_t value, interval; if (_PyTime_FromTimespec(&value, &curr_value->it_value) < 0) { return NULL; } @@ -10190,30 +10721,40 @@ os.timerfd_settime * flags: int = 0 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. - initial: double = 0.0 + initial as initial_double: double = 0.0 The initial expiration time, in seconds. - interval: double = 0.0 + interval as interval_double: double = 0.0 The timer's interval, in seconds. Alter a timer file descriptor's internal timer in seconds. [clinic start generated code]*/ static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval) -/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double) +/*[clinic end generated code: output=df4c1bce6859224e input=81d2c0d7e936e8a7]*/ { - struct itimerspec new_value; - struct itimerspec old_value; - int result; - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyTime_t initial, interval; + if (_PyTime_FromSecondsDouble(initial_double, _PyTime_ROUND_FLOOR, + &initial) < 0) { + return NULL; + } + if (_PyTime_FromSecondsDouble(interval_double, _PyTime_ROUND_FLOOR, + &interval) < 0) { + return NULL; + } + + struct itimerspec new_value, old_value; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { PyErr_SetString(PyExc_ValueError, "invalid initial value"); return NULL; } - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { PyErr_SetString(PyExc_ValueError, "invalid interval value"); return NULL; } + + int result; Py_BEGIN_ALLOW_THREADS result = timerfd_settime(fd, flags, &new_value, &old_value); Py_END_ALLOW_THREADS @@ -11276,9 +11817,9 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, done: #if !defined(HAVE_LARGEFILE_SUPPORT) - return Py_BuildValue("l", sbytes); + return PyLong_FromLong(sbytes); #else - return Py_BuildValue("L", sbytes); + return PyLong_FromLongLong(sbytes); #endif #else @@ -11291,7 +11832,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); } #endif off_t offset; @@ -11312,7 +11853,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, return (!async_err) ? posix_error() : NULL; if (offset >= st.st_size) { - return Py_BuildValue("i", 0); + return PyLong_FromLong(0); } // On illumos specifically sendfile() may perform a partial write but @@ -11338,7 +11879,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); #endif } #endif /* HAVE_SENDFILE */ @@ -11464,8 +12005,8 @@ os_pipe_impl(PyObject *module) Py_BEGIN_ALLOW_THREADS ok = CreatePipe(&read, &write, &attr, 0); if (ok) { - fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY); - fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY); + fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY | _O_NOINHERIT); + fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY | _O_NOINHERIT); if (fds[0] == -1 || fds[1] == -1) { CloseHandle(read); CloseHandle(write); @@ -12003,8 +12544,30 @@ os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, #ifdef HAVE_DEVICE_MACROS +static PyObject * +major_minor_conv(unsigned int value) +{ +#ifdef NODEV + if (value == (unsigned int)NODEV) { + return PyLong_FromLong((int)NODEV); + } +#endif + return PyLong_FromUnsignedLong(value); +} + +static int +major_minor_check(dev_t value) +{ +#ifdef NODEV + if (value == NODEV) { + return 1; + } +#endif + return (dev_t)(unsigned int)value == value; +} + /*[clinic input] -os.major -> unsigned_int +os.major device: dev_t / @@ -12012,16 +12575,16 @@ os.major -> unsigned_int Extracts a device major number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5b3b2589bafb498e input=1e16a4d30c4d4462]*/ +/*[clinic end generated code: output=4071ffee17647891 input=b1a0a14ec9448229]*/ { - return major(device); + return major_minor_conv(major(device)); } /*[clinic input] -os.minor -> unsigned_int +os.minor device: dev_t / @@ -12029,28 +12592,33 @@ os.minor -> unsigned_int Extracts a device minor number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5e1a25e630b0157d input=0842c6d23f24c65e]*/ +/*[clinic end generated code: output=306cb78e3bc5004f input=2f686e463682a9da]*/ { - return minor(device); + return major_minor_conv(minor(device)); } /*[clinic input] os.makedev -> dev_t - major: int - minor: int + major: dev_t + minor: dev_t / Composes a raw device number from the major and minor device numbers. [clinic start generated code]*/ static dev_t -os_makedev_impl(PyObject *module, int major, int minor) -/*[clinic end generated code: output=881aaa4aba6f6a52 input=4b9fd8fc73cbe48f]*/ +os_makedev_impl(PyObject *module, dev_t major, dev_t minor) +/*[clinic end generated code: output=cad6125c51f5af80 input=2146126ec02e55c1]*/ { + if (!major_minor_check(major) || !major_minor_check(minor)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C unsigned int"); + return (dev_t)-1; + } return makedev(major, minor); } #endif /* HAVE_DEVICE_MACROS */ @@ -12159,7 +12727,10 @@ os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) #endif -#if defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG) +/* GH-111804: Due to posix_fallocate() not having consistent semantics across + OSs, support was dropped in WASI preview2. */ +#if defined(HAVE_POSIX_FALLOCATE) && !defined(POSIX_FADVISE_AIX_BUG) && \ + !defined(__wasi__) /*[clinic input] os.posix_fallocate @@ -12197,7 +12768,7 @@ os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, errno = result; return posix_error(); } -#endif /* HAVE_POSIX_FALLOCATE) && !POSIX_FADVISE_AIX_BUG */ +#endif /* HAVE_POSIX_FALLOCATE) && !POSIX_FADVISE_AIX_BUG && !defined(__wasi__) */ #if defined(HAVE_POSIX_FADVISE) && !defined(POSIX_FADVISE_AIX_BUG) @@ -12271,7 +12842,6 @@ win32_putenv(PyObject *name, PyObject *value) } Py_ssize_t size; - /* PyUnicode_AsWideCharString() rejects embedded null characters */ wchar_t *env = PyUnicode_AsWideCharString(unicode, &size); Py_DECREF(unicode); @@ -12285,6 +12855,12 @@ win32_putenv(PyObject *name, PyObject *value) PyMem_Free(env); return NULL; } + if (wcslen(env) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "embedded null character"); + PyMem_Free(env); + return NULL; + } /* _wputenv() and SetEnvironmentVariableW() update the environment in the Process Environment Block (PEB). _wputenv() also updates CRT 'environ' @@ -12606,6 +13182,62 @@ os_WSTOPSIG_impl(PyObject *module, int status) #endif #include +#ifdef __APPLE__ +/* On macOS struct statvfs uses 32-bit integers for block counts, + * resulting in overflow when filesystems are larger than 4TB. Therefore + * os.statvfs is implemented in terms of statfs(2). + */ + +static PyObject* +_pystatvfs_fromstructstatfs(PyObject *module, struct statfs st) { + PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); + if (v == NULL) { + return NULL; + } + + long flags = 0; + if (st.f_flags & MNT_RDONLY) { + flags |= ST_RDONLY; + } + if (st.f_flags & MNT_NOSUID) { + flags |= ST_NOSUID; + } + + _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); + +#define SET_ITEM(SEQ, INDEX, EXPR) \ + do { \ + PyObject *obj = (EXPR); \ + if (obj == NULL) { \ + Py_DECREF((SEQ)); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM((SEQ), (INDEX), obj); \ + } while (0) + + SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); + SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); + SET_ITEM(v, 2, PyLong_FromLongLong((long long) st.f_blocks)); + SET_ITEM(v, 3, PyLong_FromLongLong((long long) st.f_bfree)); + SET_ITEM(v, 4, PyLong_FromLongLong((long long) st.f_bavail)); + SET_ITEM(v, 5, PyLong_FromLongLong((long long) st.f_files)); + SET_ITEM(v, 6, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 7, PyLong_FromLongLong((long long) st.f_ffree)); + SET_ITEM(v, 8, PyLong_FromLong((long) flags)); + + SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); + SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + +#undef SET_ITEM + + return v; +} + +#else + + + static PyObject* _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; @@ -12613,50 +13245,56 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { if (v == NULL) return NULL; + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, pos++, item); \ + } while(0) + #if !defined(HAVE_LARGEFILE_SUPPORT) - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, PyLong_FromLong((long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, PyLong_FromLong((long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, PyLong_FromLong((long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong((long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, PyLong_FromLong((long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLong((long) st.f_blocks)); + SET_RESULT(PyLong_FromLong((long) st.f_bfree)); + SET_RESULT(PyLong_FromLong((long) st.f_bavail)); + SET_RESULT(PyLong_FromLong((long) st.f_files)); + SET_RESULT(PyLong_FromLong((long) st.f_ffree)); + SET_RESULT(PyLong_FromLong((long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #else - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, - PyLong_FromLongLong((long long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, - PyLong_FromLongLong((long long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, - PyLong_FromLongLong((long long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, - PyLong_FromLongLong((long long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, - PyLong_FromLongLong((long long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_blocks)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bfree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bavail)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_files)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_ffree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #endif /* The _ALL_SOURCE feature test macro defines f_fsid as a structure * (issue #32390). */ #if defined(_AIX) && defined(_ALL_SOURCE) - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid.val[0])); #else - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid)); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid)); #endif - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } + +#undef SET_RESULT return v; } +#endif + /*[clinic input] os.fstatvfs @@ -12674,6 +13312,22 @@ os_fstatvfs_impl(PyObject *module, int fd) { int result; int async_err = 0; +#ifdef __APPLE__ + struct statfs st; + /* On macOS os.fstatvfs is implemented using fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + do { + Py_BEGIN_ALLOW_THREADS + result = fstatfs(fd, &st); + Py_END_ALLOW_THREADS + } while (result != 0 && errno == EINTR && + !(async_err = PyErr_CheckSignals())); + if (result != 0) + return (!async_err) ? posix_error() : NULL; + + return _pystatvfs_fromstructstatfs(module, st); +#else struct statvfs st; do { @@ -12686,6 +13340,7 @@ os_fstatvfs_impl(PyObject *module, int fd) return (!async_err) ? posix_error() : NULL; return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_FSTATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -12709,6 +13364,28 @@ os_statvfs_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ { int result; + +#ifdef __APPLE__ + /* On macOS os.statvfs is implemented using statfs(2)/fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + struct statfs st; + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = fstatfs(path->fd, &st); + } + else + result = statfs(path->narrow, &st); + Py_END_ALLOW_THREADS + + if (result) { + return path_error(path); + } + + return _pystatvfs_fromstructstatfs(module, st); + +#else struct statvfs st; Py_BEGIN_ALLOW_THREADS @@ -12726,6 +13403,7 @@ os_statvfs_impl(PyObject *module, path_t *path) } return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_STATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -14582,12 +15260,23 @@ os_get_terminal_size_impl(PyObject *module, int fd) termsize = PyStructSequence_New((PyTypeObject *)TerminalSizeType); if (termsize == NULL) return NULL; - PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns)); - PyStructSequence_SET_ITEM(termsize, 1, PyLong_FromLong(lines)); - if (PyErr_Occurred()) { - Py_DECREF(termsize); - return NULL; - } + + int pos = 0; + +#define SET_TERMSIZE(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(termsize); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(termsize, pos++, item); \ + } while(0) + + SET_TERMSIZE(PyLong_FromLong(columns)); + SET_TERMSIZE(PyLong_FromLong(lines)); +#undef SET_TERMSIZE + return termsize; } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ @@ -14805,6 +15494,7 @@ typedef struct { #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; uint64_t win32_file_index; + uint64_t win32_file_index_high; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -15134,11 +15824,10 @@ os_DirEntry_inode_impl(DirEntry *self) } self->win32_file_index = stat.st_ino; + self->win32_file_index_high = stat.st_ino_high; self->got_file_index = 1; } - static_assert(sizeof(unsigned long long) >= sizeof(self->win32_file_index), - "DirEntry.win32_file_index is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong(self->win32_file_index); + return _pystat_l128_from_l64_l64(self->win32_file_index, self->win32_file_index_high); #else /* POSIX */ static_assert(sizeof(unsigned long long) >= sizeof(self->d_ino), "DirEntry.d_ino is larger than unsigned long long"); @@ -15261,7 +15950,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) entry->name = PyUnicode_FromWideChar(dataW->cFileName, -1); if (!entry->name) goto error; - if (path->narrow) { + int return_bytes = path->wide && PyBytes_Check(path->object); + if (return_bytes) { Py_SETREF(entry->name, PyUnicode_EncodeFSDefault(entry->name)); if (!entry->name) goto error; @@ -15275,7 +15965,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) PyMem_Free(joined_path); if (!entry->path) goto error; - if (path->narrow) { + if (return_bytes) { Py_SETREF(entry->path, PyUnicode_EncodeFSDefault(entry->path)); if (!entry->path) goto error; @@ -15284,6 +15974,10 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) find_data_to_file_info(dataW, &file_info, &reparse_tag); _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + entry->win32_lstat.st_ctime = entry->win32_lstat.st_birthtime; + entry->win32_lstat.st_ctime_nsec = entry->win32_lstat.st_birthtime_nsec; + return (PyObject *)entry; error: @@ -16065,6 +16759,71 @@ os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) } #endif +#if defined(MS_WINDOWS) +/*[clinic input] +os._supports_virtual_terminal + +Checks if virtual terminal is supported in windows +[clinic start generated code]*/ + +static PyObject * +os__supports_virtual_terminal_impl(PyObject *module) +/*[clinic end generated code: output=bd0556a6d9d99fe6 input=0752c98e5d321542]*/ +{ + DWORD mode = 0; + HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); + if (!GetConsoleMode(handle, &mode)) { + Py_RETURN_FALSE; + } + return PyBool_FromLong(mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING); +} +#endif + +/*[clinic input] +os._inputhook + +Calls PyOS_CallInputHook droppong the GIL first +[clinic start generated code]*/ + +static PyObject * +os__inputhook_impl(PyObject *module) +/*[clinic end generated code: output=525aca4ef3c6149f input=fc531701930d064f]*/ +{ + int result = 0; + if (PyOS_InputHook) { + Py_BEGIN_ALLOW_THREADS; + result = PyOS_InputHook(); + Py_END_ALLOW_THREADS; + } + return PyLong_FromLong(result); +} + +/*[clinic input] +os._is_inputhook_installed + +Checks if PyOS_CallInputHook is set +[clinic start generated code]*/ + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module) +/*[clinic end generated code: output=3b3eab4f672c689a input=ff177c9938dd76d8]*/ +{ + return PyBool_FromLong(PyOS_InputHook != NULL); +} + +/*[clinic input] +os._create_environ + +Create the environment dictionary. +[clinic start generated code]*/ + +static PyObject * +os__create_environ_impl(PyObject *module) +/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/ +{ + return convertenviron(); +} + static PyMethodDef posix_methods[] = { @@ -16129,6 +16888,10 @@ static PyMethodDef posix_methods[] = { OS_SCHED_YIELD_METHODDEF OS_SCHED_SETAFFINITY_METHODDEF OS_SCHED_GETAFFINITY_METHODDEF + OS_POSIX_OPENPT_METHODDEF + OS_GRANTPT_METHODDEF + OS_UNLOCKPT_METHODDEF + OS_PTSNAME_METHODDEF OS_OPENPTY_METHODDEF OS_LOGIN_TTY_METHODDEF OS_FORKPTY_METHODDEF @@ -16225,6 +16988,7 @@ static PyMethodDef posix_methods[] = { OS__FINDFIRSTFILE_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_SPLITROOT_EX_METHODDEF OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF @@ -16268,7 +17032,14 @@ static PyMethodDef posix_methods[] = { OS__PATH_ISDIR_METHODDEF OS__PATH_ISFILE_METHODDEF OS__PATH_ISLINK_METHODDEF + OS__PATH_ISJUNCTION_METHODDEF OS__PATH_EXISTS_METHODDEF + OS__PATH_LEXISTS_METHODDEF + + OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + OS__INPUTHOOK_METHODDEF + OS__IS_INPUTHOOK_INSTALLED_METHODDEF + OS__CREATE_ENVIRON_METHODDEF {NULL, NULL} /* Sentinel */ }; @@ -16693,6 +17464,9 @@ all_ins(PyObject *m) if (PyModule_AddIntConstant(m, "POSIX_SPAWN_OPEN", POSIX_SPAWN_OPEN)) return -1; if (PyModule_AddIntConstant(m, "POSIX_SPAWN_CLOSE", POSIX_SPAWN_CLOSE)) return -1; if (PyModule_AddIntConstant(m, "POSIX_SPAWN_DUP2", POSIX_SPAWN_DUP2)) return -1; +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + if (PyModule_AddIntMacro(m, POSIX_SPAWN_CLOSEFROM)) return -1; +#endif #endif #if defined(HAVE_SPAWNV) || defined (HAVE_RTPSPAWN) @@ -17138,11 +17912,11 @@ posixmodule_exec(PyObject *m) return -1; } - if (PyDict_DelItemString(dct, "pwritev") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "pwritev", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "preadv") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "preadv", NULL) < 0) { + return -1; } } #endif @@ -17162,7 +17936,7 @@ posixmodule_exec(PyObject *m) return -1; } -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) waitid_result_desc.name = MODNAME ".waitid_result"; state->WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); if (PyModule_AddObjectRef(m, "waitid_result", state->WaitidResultType) < 0) { @@ -17265,6 +18039,15 @@ posixmodule_exec(PyObject *m) Py_DECREF(unicode); } +#ifndef MS_WINDOWS + if (_Py_GetTicksPerSecond(&state->ticks_per_second) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot read ticks_per_second"); + return -1; + } + assert(state->ticks_per_second >= 1); +#endif + return PyModule_Add(m, "_have_functions", list); } @@ -17272,6 +18055,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h index 8827ce153fed8c..be732db04faf94 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h @@ -2,7 +2,6 @@ #ifndef Py_POSIXMODULE_H #define Py_POSIXMODULE_H -#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif @@ -34,5 +33,4 @@ extern int _Py_Sigset_Converter(PyObject *, void *); #ifdef __cplusplus } #endif -#endif // !Py_LIMITED_API #endif // !Py_POSIXMODULE_H diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index b7034369c4731e..2240e2078b2d98 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,9 +1,16 @@ /* UNIX password file access module */ +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getpwuid() #include // sysconf() @@ -64,53 +71,52 @@ static struct PyModuleDef pwdmodule; #define DEFAULT_BUFFER_SIZE 1024 -static void -sets(PyObject *v, int i, const char* val) -{ - if (val) { - PyObject *o = PyUnicode_DecodeFSDefault(val); - PyStructSequence_SET_ITEM(v, i, o); - } - else { - PyStructSequence_SET_ITEM(v, i, Py_None); - Py_INCREF(Py_None); - } -} - static PyObject * mkpwent(PyObject *module, struct passwd *p) { - int setIndex = 0; PyObject *v = PyStructSequence_New(get_pwd_state(module)->StructPwdType); - if (v == NULL) + if (v == NULL) { return NULL; + } -#define SETS(i,val) sets(v, i, val) + int setIndex = 0; + +#define SET_STRING(VAL) \ + SET_RESULT((VAL) ? PyUnicode_DecodeFSDefault((VAL)) : Py_NewRef(Py_None)) - SETS(setIndex++, p->pw_name); +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SetItem(v, setIndex++, item); \ + } while(0) + + SET_STRING(p->pw_name); #if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) - SETS(setIndex++, p->pw_passwd); + SET_STRING(p->pw_passwd); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); + SET_RESULT(_PyLong_FromUid(p->pw_uid)); + SET_RESULT(_PyLong_FromGid(p->pw_gid)); #if defined(HAVE_STRUCT_PASSWD_PW_GECOS) - SETS(setIndex++, p->pw_gecos); + SET_STRING(p->pw_gecos); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - SETS(setIndex++, p->pw_dir); - SETS(setIndex++, p->pw_shell); + SET_STRING(p->pw_dir); + SET_STRING(p->pw_shell); -#undef SETS - - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_STRING +#undef SET_RESULT return v; + +error: + Py_DECREF(v); + return NULL; } /*[clinic input] @@ -338,6 +344,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index bd24523eac830b..9733bc34f7c80a 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -7,6 +7,7 @@ #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() +#include #include // offsetof() #include "expat.h" #include "pyexpat.h" @@ -21,7 +22,7 @@ module pyexpat #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) static XML_Memory_Handling_Suite ExpatMemoryHandler = { - PyObject_Malloc, PyObject_Realloc, PyObject_Free}; + PyMem_Malloc, PyMem_Realloc, PyMem_Free}; enum HandlerTypes { StartElement, @@ -81,6 +82,12 @@ typedef struct { /* NULL if not enabled */ int buffer_size; /* Size of buffer, in XML_Char units */ int buffer_used; /* Buffer units in use */ + bool reparse_deferral_enabled; /* Whether to defer reparsing of + unfinished XML tokens; a de-facto cache of + what Expat has the authority on, for lack + of a getter API function + "XML_GetReparseDeferralEnabled" in Expat + 2.6.0 */ PyObject *intern; /* Dictionary to intern strings */ PyObject **handlers; } xmlparseobject; @@ -240,19 +247,12 @@ string_intern(xmlparseobject *self, const char* str) return result; if (!self->intern) return result; - value = PyDict_GetItemWithError(self->intern, result); - if (!value) { - if (!PyErr_Occurred() && - PyDict_SetItem(self->intern, result, result) == 0) - { - return result; - } - else { - Py_DECREF(result); - return NULL; - } + if (PyDict_GetItemRef(self->intern, result, &value) == 0 && + PyDict_SetItem(self->intern, result, result) == 0) + { + return result; } - Py_INCREF(value); + assert((value != NULL) == !PyErr_Occurred()); Py_DECREF(result); return value; } @@ -710,6 +710,40 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) +/*[clinic input] +pyexpat.xmlparser.SetReparseDeferralEnabled + + enabled: bool + / + +Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled) +/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +{ +#if XML_COMBINED_VERSION >= 20600 + XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); + self->reparse_deferral_enabled = (bool)enabled; +#endif + Py_RETURN_NONE; +} + +/*[clinic input] +pyexpat.xmlparser.GetReparseDeferralEnabled + +Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) +/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +{ + return PyBool_FromLong(self->reparse_deferral_enabled); +} + /*[clinic input] pyexpat.xmlparser.Parse @@ -720,13 +754,13 @@ pyexpat.xmlparser.Parse Parse XML data. -`isfinal' should be true at end of input. +'isfinal' should be true at end of input. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=053e0f047e55c05a]*/ { const char *s; Py_ssize_t slen; @@ -895,7 +929,7 @@ static PyObject * pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) /*[clinic end generated code: output=2886cb21f9a8739a input=918d71c38009620e]*/ { - return Py_BuildValue("z", XML_GetBase(self->itself)); + return conv_string_to_unicode(XML_GetBase(self->itself)); } /*[clinic input] @@ -1070,6 +1104,8 @@ static struct PyMethodDef xmlparse_methods[] = { #if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif + PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF + PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1165,6 +1201,11 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, self->ns_prefixes = 0; self->handlers = NULL; self->intern = Py_XNewRef(intern); +#if XML_COMBINED_VERSION >= 20600 + self->reparse_deferral_enabled = true; +#else + self->reparse_deferral_enabled = false; +#endif /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, @@ -1585,7 +1626,7 @@ static PyObject * pyexpat_ErrorString_impl(PyObject *module, long code) /*[clinic end generated code: output=2feae50d166f2174 input=cc67de010d9e62b3]*/ { - return Py_BuildValue("z", XML_ErrorString((int)code)); + return conv_string_to_unicode(XML_ErrorString((int)code)); } /* List of methods defined in the module */ @@ -1610,7 +1651,7 @@ PyDoc_STRVAR(pyexpat_module_documentation, static int init_handler_descrs(pyexpat_state *state) { int i; - assert(!PyType_HasFeature(state->xml_parse_type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(state->xml_parse_type->tp_version_tag == 0); for (i = 0; handler_info[i].name != NULL; i++) { struct HandlerInfo *hi = &handler_info[i]; hi->getset.name = hi->name; @@ -1622,7 +1663,8 @@ static int init_handler_descrs(pyexpat_state *state) if (descr == NULL) return -1; - if (PyDict_SetDefault(state->xml_parse_type->tp_dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(state->xml_parse_type->tp_dict, + PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -2025,6 +2067,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20600 + capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; +#else + capi->SetReparseDeferralEnabled = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, @@ -2069,9 +2116,8 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index fde552d124bc77..35655c70a4618f 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -147,8 +147,19 @@ readline_free(void *m) static PyModuleDef readlinemodule; -#define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule))) - +static inline readlinestate* +get_hook_module_state(void) +{ + PyObject *mod = PyState_FindModule(&readlinemodule); + if (mod == NULL){ + PyErr_Clear(); + return NULL; + } + Py_INCREF(mod); + readlinestate *state = get_readline_state(mod); + Py_DECREF(mod); + return state; +} /* Convert to/from multibyte C strings */ @@ -438,14 +449,15 @@ readline_set_completion_display_matches_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=516e5cb8db75a328 input=4f0bfd5ab0179a26]*/ { + readlinestate *state = get_readline_state(module); PyObject *result = set_hook("completion_display_matches_hook", - &readlinestate_global->completion_display_matches_hook, + &state->completion_display_matches_hook, function); #ifdef HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK /* We cannot set this hook globally, since it replaces the default completion display. */ rl_completion_display_matches_hook = - readlinestate_global->completion_display_matches_hook ? + state->completion_display_matches_hook ? #if defined(HAVE_RL_COMPDISP_FUNC_T) (rl_compdisp_func_t *)on_completion_display_matches_hook : 0; #else @@ -472,7 +484,8 @@ static PyObject * readline_set_startup_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=02cd0e0c4fa082ad input=7783b4334b26d16d]*/ { - return set_hook("startup_hook", &readlinestate_global->startup_hook, + readlinestate *state = get_readline_state(module); + return set_hook("startup_hook", &state->startup_hook, function); } @@ -497,7 +510,8 @@ static PyObject * readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=fe1a96505096f464 input=4f3eaeaf7ce1fdbe]*/ { - return set_hook("pre_input_hook", &readlinestate_global->pre_input_hook, + readlinestate *state = get_readline_state(module); + return set_hook("pre_input_hook", &state->pre_input_hook, function); } #endif @@ -530,7 +544,8 @@ static PyObject * readline_get_begidx_impl(PyObject *module) /*[clinic end generated code: output=362616ee8ed1b2b1 input=e083b81c8eb4bac3]*/ { - return Py_NewRef(readlinestate_global->begidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->begidx); } /* Get the ending index for the scope of the tab-completion */ @@ -545,7 +560,8 @@ static PyObject * readline_get_endidx_impl(PyObject *module) /*[clinic end generated code: output=7f763350b12d7517 input=d4c7e34a625fd770]*/ { - return Py_NewRef(readlinestate_global->endidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->endidx); } /* Set the tab-completion word-delimiters that readline uses */ @@ -576,6 +592,13 @@ readline_set_completer_delims(PyObject *module, PyObject *string) if (break_chars) { free(completer_word_break_characters); completer_word_break_characters = break_chars; +#ifdef WITH_EDITLINE + rl_basic_word_break_characters = break_chars; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = break_chars; + } +#endif rl_completer_word_break_characters = break_chars; Py_RETURN_NONE; } @@ -772,7 +795,8 @@ static PyObject * readline_set_completer_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=171a2a60f81d3204 input=51e81e13118eb877]*/ { - return set_hook("completer", &readlinestate_global->completer, function); + readlinestate *state = get_readline_state(module); + return set_hook("completer", &state->completer, function); } /*[clinic input] @@ -785,10 +809,11 @@ static PyObject * readline_get_completer_impl(PyObject *module) /*[clinic end generated code: output=6e6bbd8226d14475 input=6457522e56d70d13]*/ { - if (readlinestate_global->completer == NULL) { + readlinestate *state = get_readline_state(module); + if (state->completer == NULL) { Py_RETURN_NONE; } - return Py_NewRef(readlinestate_global->completer); + return Py_NewRef(state->completer); } /* Private function to get current length of history. XXX It may be @@ -1016,34 +1041,40 @@ on_hook(PyObject *func) } static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_startup_hook(void) -#elif defined(WITH_APPLE_EDITLINE) -on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else -on_startup_hook(void) +on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #endif { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->startup_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->startup_hook); PyGILState_Release(gilstate); return r; } #ifdef HAVE_RL_PRE_INPUT_HOOK static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_pre_input_hook(void) -#elif defined(WITH_APPLE_EDITLINE) -on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else -on_pre_input_hook(void) +on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #endif { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->pre_input_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->pre_input_hook); PyGILState_Release(gilstate); return r; } @@ -1060,6 +1091,11 @@ on_completion_display_matches_hook(char **matches, int i; PyObject *sub, *m=NULL, *s=NULL, *r=NULL; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return; + } m = PyList_New(num_matches); if (m == NULL) goto error; @@ -1070,7 +1106,7 @@ on_completion_display_matches_hook(char **matches, PyList_SET_ITEM(m, i, s); } sub = decode(matches[0]); - r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook, + r = PyObject_CallFunction(state->completion_display_matches_hook, "NNi", sub, m, max_length); m=NULL; @@ -1118,12 +1154,17 @@ static char * on_completion(const char *text, int state) { char *result = NULL; - if (readlinestate_global->completer != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *module_state = get_hook_module_state(); + if (module_state == NULL) { + PyGILState_Release(gilstate); + return NULL; + } + if (module_state->completer != NULL) { PyObject *r = NULL, *t; - PyGILState_STATE gilstate = PyGILState_Ensure(); rl_attempted_completion_over = 1; t = decode(text); - r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state); + r = PyObject_CallFunction(module_state->completer, "Ni", t, state); if (r == NULL) goto error; if (r == Py_None) { @@ -1145,6 +1186,7 @@ on_completion(const char *text, int state) PyGILState_Release(gilstate); return result; } + PyGILState_Release(gilstate); return result; } @@ -1160,6 +1202,7 @@ flex_complete(const char *text, int start, int end) size_t start_size, end_size; wchar_t *s; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER rl_completion_append_character ='\0'; #endif @@ -1187,10 +1230,12 @@ flex_complete(const char *text, int start, int end) end = start + (int)end_size; done: - Py_XDECREF(readlinestate_global->begidx); - Py_XDECREF(readlinestate_global->endidx); - readlinestate_global->begidx = PyLong_FromLong((long) start); - readlinestate_global->endidx = PyLong_FromLong((long) end); + if (state) { + Py_XDECREF(state->begidx); + Py_XDECREF(state->endidx); + state->begidx = PyLong_FromLong((long) start); + state->endidx = PyLong_FromLong((long) end); + } result = completion_matches((char *)text, *on_completion); PyGILState_Release(gilstate); return result; @@ -1267,6 +1312,15 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ +#ifdef WITH_EDITLINE + // libedit uses rl_basic_word_break_characters instead of + // rl_completer_word_break_characters as complete delimiter + rl_basic_word_break_characters = completer_word_break_characters; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = completer_word_break_characters; + } +#endif rl_completer_word_break_characters = completer_word_break_characters; mod_state->begidx = PyLong_FromLong(0L); @@ -1480,6 +1534,7 @@ static struct PyModuleDef readlinemodule = { PyMODINIT_FUNC PyInit_readline(void) { + const char *backend = "readline"; PyObject *m; readlinestate *mod_state; @@ -1487,14 +1542,19 @@ PyInit_readline(void) using_libedit_emulation = 1; } - if (using_libedit_emulation) + if (using_libedit_emulation) { readlinemodule.m_doc = doc_module_le; + backend = "editline"; + } m = PyModule_Create(&readlinemodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION) < 0) { @@ -1510,13 +1570,22 @@ PyInit_readline(void) goto error; } + if (PyModule_AddStringConstant(m, "backend", backend) < 0) { + goto error; + } + mod_state = (readlinestate *) PyModule_GetState(m); + if (mod_state == NULL){ + goto error; + } PyOS_ReadlineFunctionPointer = call_readline; if (setup_readline(mod_state) < 0) { PyErr_NoMemory(); goto error; } - + if (PyErr_Occurred()){ + goto error; + } return m; error: diff --git a/Modules/resource.c b/Modules/resource.c index d65509ec3436a2..3fe18e7c98e3d8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,3 +1,9 @@ +// Need limited C API version 3.13 for PySys_Audit() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include // errno #include @@ -116,24 +122,24 @@ resource_getrusage_impl(PyObject *module, int who) if (!result) return NULL; - PyStructSequence_SET_ITEM(result, 0, + PyStructSequence_SetItem(result, 0, PyFloat_FromDouble(doubletime(ru.ru_utime))); - PyStructSequence_SET_ITEM(result, 1, + PyStructSequence_SetItem(result, 1, PyFloat_FromDouble(doubletime(ru.ru_stime))); - PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong(ru.ru_maxrss)); - PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong(ru.ru_ixrss)); - PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong(ru.ru_idrss)); - PyStructSequence_SET_ITEM(result, 5, PyLong_FromLong(ru.ru_isrss)); - PyStructSequence_SET_ITEM(result, 6, PyLong_FromLong(ru.ru_minflt)); - PyStructSequence_SET_ITEM(result, 7, PyLong_FromLong(ru.ru_majflt)); - PyStructSequence_SET_ITEM(result, 8, PyLong_FromLong(ru.ru_nswap)); - PyStructSequence_SET_ITEM(result, 9, PyLong_FromLong(ru.ru_inblock)); - PyStructSequence_SET_ITEM(result, 10, PyLong_FromLong(ru.ru_oublock)); - PyStructSequence_SET_ITEM(result, 11, PyLong_FromLong(ru.ru_msgsnd)); - PyStructSequence_SET_ITEM(result, 12, PyLong_FromLong(ru.ru_msgrcv)); - PyStructSequence_SET_ITEM(result, 13, PyLong_FromLong(ru.ru_nsignals)); - PyStructSequence_SET_ITEM(result, 14, PyLong_FromLong(ru.ru_nvcsw)); - PyStructSequence_SET_ITEM(result, 15, PyLong_FromLong(ru.ru_nivcsw)); + PyStructSequence_SetItem(result, 2, PyLong_FromLong(ru.ru_maxrss)); + PyStructSequence_SetItem(result, 3, PyLong_FromLong(ru.ru_ixrss)); + PyStructSequence_SetItem(result, 4, PyLong_FromLong(ru.ru_idrss)); + PyStructSequence_SetItem(result, 5, PyLong_FromLong(ru.ru_isrss)); + PyStructSequence_SetItem(result, 6, PyLong_FromLong(ru.ru_minflt)); + PyStructSequence_SetItem(result, 7, PyLong_FromLong(ru.ru_majflt)); + PyStructSequence_SetItem(result, 8, PyLong_FromLong(ru.ru_nswap)); + PyStructSequence_SetItem(result, 9, PyLong_FromLong(ru.ru_inblock)); + PyStructSequence_SetItem(result, 10, PyLong_FromLong(ru.ru_oublock)); + PyStructSequence_SetItem(result, 11, PyLong_FromLong(ru.ru_msgsnd)); + PyStructSequence_SetItem(result, 12, PyLong_FromLong(ru.ru_msgrcv)); + PyStructSequence_SetItem(result, 13, PyLong_FromLong(ru.ru_nsignals)); + PyStructSequence_SetItem(result, 14, PyLong_FromLong(ru.ru_nvcsw)); + PyStructSequence_SetItem(result, 15, PyLong_FromLong(ru.ru_nivcsw)); if (PyErr_Occurred()) { Py_DECREF(result); @@ -153,13 +159,13 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) /* Here limits is a borrowed reference */ return -1; - if (PyTuple_GET_SIZE(limits) != 2) { + if (PyTuple_Size(limits) != 2) { PyErr_SetString(PyExc_ValueError, "expected a tuple of 2 integers"); goto error; } - curobj = PyTuple_GET_ITEM(limits, 0); - maxobj = PyTuple_GET_ITEM(limits, 1); + curobj = PyTuple_GetItem(limits, 0); // borrowed + maxobj = PyTuple_GetItem(limits, 1); // borrowed #if !defined(HAVE_LARGEFILE_SUPPORT) rl_out->rlim_cur = PyLong_AsLong(curobj); if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) @@ -507,6 +513,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/rotatingtree.c b/Modules/rotatingtree.c index 07e08bc3167c0a..5910e25bed6389 100644 --- a/Modules/rotatingtree.c +++ b/Modules/rotatingtree.c @@ -1,3 +1,4 @@ +#include "Python.h" #include "rotatingtree.h" #define KEY_LOWER_THAN(key1, key2) ((char*)(key1) < (char*)(key2)) @@ -10,17 +11,20 @@ static unsigned int random_value = 1; static unsigned int random_stream = 0; +static PyMutex random_mutex = {0}; static int randombits(int bits) { int result; + PyMutex_Lock(&random_mutex); if (random_stream < (1U << bits)) { random_value *= 1082527; random_stream = random_value; } result = random_stream & ((1<>= bits; + PyMutex_Unlock(&random_mutex); return result; } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index c56e682b21e2a1..0a5b5a703a5aa1 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -14,8 +14,10 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() -#include "pycore_time.h" // _PyTime_t +#include "pycore_import.h" // _PyImport_GetModuleAttrString() +#include "pycore_time.h" // _PyTime_FromSecondsObject() +#include #include // offsetof() #ifndef MS_WINDOWS # include // close() @@ -75,13 +77,26 @@ extern void bzero(void *, int); # define POLLPRI 0 #endif +#ifdef HAVE_KQUEUE +// Linked list to track kqueue objects with an open fd, so +// that we can invalidate them at fork; +typedef struct _kqueue_list_item { + struct kqueue_queue_Object *obj; + struct _kqueue_list_item *next; +} _kqueue_list_item, *_kqueue_list; +#endif + typedef struct { PyObject *close; PyTypeObject *poll_Type; PyTypeObject *devpoll_Type; PyTypeObject *pyEpoll_Type; +#ifdef HAVE_KQUEUE PyTypeObject *kqueue_event_Type; PyTypeObject *kqueue_queue_Type; + _kqueue_list kqueue_open_list; + bool kqueue_tracking_initialized; +#endif } _selectstate; static struct PyModuleDef selectmodule; @@ -282,7 +297,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, struct timeval tv, *tvp; int imax, omax, emax, max; int n; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; if (timeout_obj == Py_None) tvp = (struct timeval *)NULL; @@ -458,6 +473,7 @@ update_ufd_array(pollObject *self) } /*[clinic input] +@critical_section select.poll.register fd: fildes @@ -471,7 +487,7 @@ Register a file descriptor with the polling object. static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=0dc7173c800a4a65 input=34e16cfb28d3c900]*/ +/*[clinic end generated code: output=0dc7173c800a4a65 input=c475e029ce6c2830]*/ { PyObject *key, *value; int err; @@ -499,6 +515,7 @@ select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.modify fd: fildes @@ -513,7 +530,7 @@ Modify an already registered file descriptor. static PyObject * select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=1a7b88bf079eff17 input=a8e383df075c32cf]*/ +/*[clinic end generated code: output=1a7b88bf079eff17 input=38c9db5346711872]*/ { PyObject *key, *value; int err; @@ -551,6 +568,7 @@ select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.unregister fd: fildes @@ -561,7 +579,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_poll_unregister_impl(pollObject *self, int fd) -/*[clinic end generated code: output=8c9f42e75e7d291b input=4b4fccc1040e79cb]*/ +/*[clinic end generated code: output=8c9f42e75e7d291b input=ae6315d7f5243704]*/ { PyObject *key; @@ -584,6 +602,7 @@ select_poll_unregister_impl(pollObject *self, int fd) } /*[clinic input] +@critical_section select.poll.poll timeout as timeout_obj: object = None @@ -599,12 +618,12 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=c2f6953ec45e5622]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ { PyObject *result_list = NULL; int poll_result, i, j; PyObject *value = NULL, *num = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; int async_err = 0; if (timeout_obj != Py_None) { @@ -802,10 +821,10 @@ static int devpoll_flush(devpollObject *self) ** clear what to do if a partial write occurred. For now, raise ** an exception and see if we actually found this problem in ** the wild. - ** See http://bugs.python.org/issue6397. + ** See https://github.com/python/cpython/issues/50646. */ PyErr_Format(PyExc_OSError, "failed to write all pollfds. " - "Please, report at http://bugs.python.org/. " + "Please, report at https://github.com/python/cpython/issues/. " "Data to report: Size tried: %d, actual size written: %d.", size, n); return -1; @@ -842,6 +861,7 @@ internal_devpoll_register(devpollObject *self, int fd, } /*[clinic input] +@critical_section select.devpoll.register fd: fildes @@ -857,12 +877,13 @@ Register a file descriptor with the polling object. static PyObject * select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=6e07fe8b74abba0c input=22006fabe9567522]*/ +/*[clinic end generated code: output=6e07fe8b74abba0c input=8d48bd2653a61c42]*/ { return internal_devpoll_register(self, fd, eventmask, 0); } /*[clinic input] +@critical_section select.devpoll.modify fd: fildes @@ -878,12 +899,13 @@ Modify a possible already registered file descriptor. static PyObject * select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=bc2e6d23aaff98b4 input=09fa335db7cdc09e]*/ +/*[clinic end generated code: output=bc2e6d23aaff98b4 input=773b37e9abca2460]*/ { return internal_devpoll_register(self, fd, eventmask, 1); } /*[clinic input] +@critical_section select.devpoll.unregister fd: fildes @@ -894,7 +916,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_devpoll_unregister_impl(devpollObject *self, int fd) -/*[clinic end generated code: output=95519ffa0c7d43fe input=b4ea42a4442fd467]*/ +/*[clinic end generated code: output=95519ffa0c7d43fe input=6052d368368d4d05]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -911,6 +933,7 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) } /*[clinic input] +@critical_section select.devpoll.poll timeout as timeout_obj: object = None The maximum time to wait in milliseconds, or else None (or a negative @@ -925,13 +948,13 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=3c3f0a355ec2bedb]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ { struct dvpoll dvp; PyObject *result_list = NULL; int poll_result, i; PyObject *value, *num1, *num2; - _PyTime_t timeout, ms, deadline = 0; + PyTime_t timeout, ms, deadline = 0; if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1044,6 +1067,7 @@ devpoll_internal_close(devpollObject *self) } /*[clinic input] +@critical_section select.devpoll.close Close the devpoll file descriptor. @@ -1053,7 +1077,7 @@ Further operations on the devpoll object will raise an exception. static PyObject * select_devpoll_close_impl(devpollObject *self) -/*[clinic end generated code: output=26b355bd6429f21b input=6273c30f5560a99b]*/ +/*[clinic end generated code: output=26b355bd6429f21b input=408fde21a377ccfb]*/ { errno = devpoll_internal_close(self); if (errno < 0) { @@ -1073,6 +1097,7 @@ devpoll_get_closed(devpollObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@critical_section select.devpoll.fileno Return the file descriptor. @@ -1080,7 +1105,7 @@ Return the file descriptor. static PyObject * select_devpoll_fileno_impl(devpollObject *self) -/*[clinic end generated code: output=26920929f8d292f4 input=ef15331ebde6c368]*/ +/*[clinic end generated code: output=26920929f8d292f4 input=8c9db2efa1ade538]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1363,6 +1388,7 @@ pyepoll_dealloc(pyEpoll_Object *self) } /*[clinic input] +@critical_section select.epoll.close Close the epoll control file descriptor. @@ -1372,7 +1398,7 @@ Further operations on the epoll object will raise an exception. static PyObject * select_epoll_close_impl(pyEpoll_Object *self) -/*[clinic end generated code: output=ee2144c446a1a435 input=ca6c66ba5a736bfd]*/ +/*[clinic end generated code: output=ee2144c446a1a435 input=f626a769192e1dbe]*/ { errno = pyepoll_internal_close(self); if (errno < 0) { @@ -1544,7 +1570,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int nfds, i; PyObject *elist = NULL, *etuple = NULL; struct epoll_event *evs = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; if (self->epfd < 0) return pyepoll_err_closed(); @@ -1754,7 +1780,7 @@ typedef struct { #define kqueue_event_Check(op, state) (PyObject_TypeCheck((op), state->kqueue_event_Type)) -typedef struct { +typedef struct kqueue_queue_Object { PyObject_HEAD SOCKET kqfd; /* kqueue control fd */ } kqueue_queue_Object; @@ -1940,6 +1966,128 @@ kqueue_queue_err_closed(void) return NULL; } +static PyObject* +kqueue_tracking_after_fork(PyObject *module) { + _selectstate *state = get_select_state(module); + _kqueue_list_item *item = state->kqueue_open_list; + state->kqueue_open_list = NULL; + while (item) { + // Safety: we hold the GIL, and references are removed from this list + // before the object is deallocated. + kqueue_queue_Object *obj = item->obj; + assert(obj->kqfd != -1); + obj->kqfd = -1; + _kqueue_list_item *next = item->next; + PyMem_Free(item); + item = next; + } + Py_RETURN_NONE; +} + +static PyMethodDef kqueue_tracking_after_fork_def = { + "kqueue_tracking_after_fork", (PyCFunction)kqueue_tracking_after_fork, + METH_NOARGS, "Invalidate open select.kqueue objects after fork." +}; + +static void +kqueue_tracking_init(PyObject *module) { + _selectstate *state = get_select_state(module); + assert(state->kqueue_open_list == NULL); + // Register a callback to invalidate kqueues with open fds after fork. + PyObject *register_at_fork = NULL, *cb = NULL, *args = NULL, + *kwargs = NULL, *result = NULL; + register_at_fork = _PyImport_GetModuleAttrString("posix", + "register_at_fork"); + if (register_at_fork == NULL) { + goto finally; + } + cb = PyCFunction_New(&kqueue_tracking_after_fork_def, module); + if (cb == NULL) { + goto finally; + } + args = PyTuple_New(0); + assert(args != NULL); + kwargs = Py_BuildValue("{sO}", "after_in_child", cb); + if (kwargs == NULL) { + goto finally; + } + result = PyObject_Call(register_at_fork, args, kwargs); + +finally: + if (PyErr_Occurred()) { + // There are a few reasons registration can fail, especially if someone + // touched posix.register_at_fork. But everything else still works so + // instead of raising we issue a warning and move along. + PyObject *exc = PyErr_GetRaisedException(); + PyObject *exctype = (PyObject*)Py_TYPE(exc); + PyErr_WarnFormat(PyExc_RuntimeWarning, 1, + "An exception of type %S was raised while registering an " + "after-fork handler for select.kqueue objects: %S", exctype, exc); + Py_DECREF(exc); + } + Py_XDECREF(register_at_fork); + Py_XDECREF(cb); + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(result); + state->kqueue_tracking_initialized = true; +} + +static int +kqueue_tracking_add_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ + assert(self->kqfd >= 0); + _kqueue_list_item *item = PyMem_New(_kqueue_list_item, 1); + if (item == NULL) { + PyErr_NoMemory(); + return -1; + } + item->obj = self; + item->next = state->kqueue_open_list; + state->kqueue_open_list = item; + return 0; +} + +static int +kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) +{ + int ret; + PyObject *module = PyType_GetModule(Py_TYPE(self)); + Py_BEGIN_CRITICAL_SECTION(module); + if (!state->kqueue_tracking_initialized) { + kqueue_tracking_init(module); + } + ret = kqueue_tracking_add_lock_held(state, self); + Py_END_CRITICAL_SECTION(); + return ret; +} + +static void +kqueue_tracking_remove_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ + _kqueue_list *listptr = &state->kqueue_open_list; + while (*listptr != NULL) { + _kqueue_list_item *item = *listptr; + if (item->obj == self) { + *listptr = item->next; + PyMem_Free(item); + return; + } + listptr = &item->next; + } + // The item should be in the list when we remove it, + // and it should only be removed once at close time. + assert(0); +} + +static void +kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) +{ + Py_BEGIN_CRITICAL_SECTION(PyType_GetModule(Py_TYPE(self))); + kqueue_tracking_remove_lock_held(state, self); + Py_END_CRITICAL_SECTION(); +} + static int kqueue_queue_internal_close(kqueue_queue_Object *self) { @@ -1947,6 +2095,8 @@ kqueue_queue_internal_close(kqueue_queue_Object *self) if (self->kqfd >= 0) { int kqfd = self->kqfd; self->kqfd = -1; + _selectstate *state = _selectstate_by_type(Py_TYPE(self)); + kqueue_tracking_remove(state, self); Py_BEGIN_ALLOW_THREADS if (close(kqfd) < 0) save_errno = errno; @@ -1987,6 +2137,13 @@ newKqueue_Object(PyTypeObject *type, SOCKET fd) return NULL; } } + + _selectstate *state = _selectstate_by_type(type); + if (kqueue_tracking_add(state, self) < 0) { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; } @@ -2017,16 +2174,15 @@ select_kqueue_impl(PyTypeObject *type) } static void -kqueue_queue_dealloc(kqueue_queue_Object *self) +kqueue_queue_finalize(kqueue_queue_Object *self) { - PyTypeObject* type = Py_TYPE(self); + PyObject* error = PyErr_GetRaisedException(); kqueue_queue_internal_close(self); - freefunc kqueue_free = PyType_GetSlot(type, Py_tp_free); - kqueue_free((PyObject *)self); - Py_DECREF((PyObject *)type); + PyErr_SetRaisedException(error); } /*[clinic input] +@critical_section select.kqueue.close Close the kqueue control file descriptor. @@ -2036,7 +2192,7 @@ Further operations on the kqueue object will raise an exception. static PyObject * select_kqueue_close_impl(kqueue_queue_Object *self) -/*[clinic end generated code: output=d1c7df0b407a4bc1 input=0b12d95430e0634c]*/ +/*[clinic end generated code: output=d1c7df0b407a4bc1 input=6d763c858b17b690]*/ { errno = kqueue_queue_internal_close(self); if (errno < 0) { @@ -2119,7 +2275,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, struct kevent *chl = NULL; struct timespec timeoutspec; struct timespec *ptimeoutspec; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; _selectstate *state = _selectstate_by_type(Py_TYPE(self)); if (self->kqfd < 0) @@ -2357,11 +2513,11 @@ static PyMethodDef kqueue_queue_methods[] = { }; static PyType_Slot kqueue_queue_Type_slots[] = { - {Py_tp_dealloc, kqueue_queue_dealloc}, {Py_tp_doc, (void*)select_kqueue__doc__}, {Py_tp_getset, kqueue_queue_getsetlist}, {Py_tp_methods, kqueue_queue_methods}, {Py_tp_new, select_kqueue}, + {Py_tp_finalize, kqueue_queue_finalize}, {0, 0}, }; @@ -2406,8 +2562,11 @@ _select_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->poll_Type); Py_VISIT(state->devpoll_Type); Py_VISIT(state->pyEpoll_Type); +#ifdef HAVE_KQUEUE Py_VISIT(state->kqueue_event_Type); Py_VISIT(state->kqueue_queue_Type); + // state->kqueue_open_list only holds borrowed refs +#endif return 0; } @@ -2420,8 +2579,10 @@ _select_clear(PyObject *module) Py_CLEAR(state->poll_Type); Py_CLEAR(state->devpoll_Type); Py_CLEAR(state->pyEpoll_Type); +#ifdef HAVE_KQUEUE Py_CLEAR(state->kqueue_event_Type); Py_CLEAR(state->kqueue_queue_Type); +#endif return 0; } @@ -2444,12 +2605,18 @@ _select_exec(PyObject *m) return -1; } +#define ADD_INT(VAL) do { \ + if (PyModule_AddIntConstant((m), #VAL, (VAL)) < 0) { \ + return -1; \ + } \ +} while (0) + #ifdef PIPE_BUF #ifdef HAVE_BROKEN_PIPE_BUF #undef PIPE_BUF #define PIPE_BUF 512 #endif - PyModule_AddIntMacro(m, PIPE_BUF); + ADD_INT(PIPE_BUF); #endif #if defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL) @@ -2468,31 +2635,31 @@ _select_exec(PyObject *m) return -1; } - PyModule_AddIntMacro(m, POLLIN); - PyModule_AddIntMacro(m, POLLPRI); - PyModule_AddIntMacro(m, POLLOUT); - PyModule_AddIntMacro(m, POLLERR); - PyModule_AddIntMacro(m, POLLHUP); - PyModule_AddIntMacro(m, POLLNVAL); + ADD_INT(POLLIN); + ADD_INT(POLLPRI); + ADD_INT(POLLOUT); + ADD_INT(POLLERR); + ADD_INT(POLLHUP); + ADD_INT(POLLNVAL); #ifdef POLLRDNORM - PyModule_AddIntMacro(m, POLLRDNORM); + ADD_INT(POLLRDNORM); #endif #ifdef POLLRDBAND - PyModule_AddIntMacro(m, POLLRDBAND); + ADD_INT(POLLRDBAND); #endif #ifdef POLLWRNORM - PyModule_AddIntMacro(m, POLLWRNORM); + ADD_INT(POLLWRNORM); #endif #ifdef POLLWRBAND - PyModule_AddIntMacro(m, POLLWRBAND); + ADD_INT(POLLWRBAND); #endif #ifdef POLLMSG - PyModule_AddIntMacro(m, POLLMSG); + ADD_INT(POLLMSG); #endif #ifdef POLLRDHUP /* Kernel 2.6.17+ */ - PyModule_AddIntMacro(m, POLLRDHUP); + ADD_INT(POLLRDHUP); #endif } #endif /* HAVE_POLL */ @@ -2515,46 +2682,57 @@ _select_exec(PyObject *m) return -1; } - PyModule_AddIntMacro(m, EPOLLIN); - PyModule_AddIntMacro(m, EPOLLOUT); - PyModule_AddIntMacro(m, EPOLLPRI); - PyModule_AddIntMacro(m, EPOLLERR); - PyModule_AddIntMacro(m, EPOLLHUP); + ADD_INT(EPOLLIN); + ADD_INT(EPOLLOUT); + ADD_INT(EPOLLPRI); + ADD_INT(EPOLLERR); + ADD_INT(EPOLLHUP); #ifdef EPOLLRDHUP /* Kernel 2.6.17 */ - PyModule_AddIntMacro(m, EPOLLRDHUP); + ADD_INT(EPOLLRDHUP); #endif - PyModule_AddIntMacro(m, EPOLLET); + ADD_INT(EPOLLET); #ifdef EPOLLONESHOT /* Kernel 2.6.2+ */ - PyModule_AddIntMacro(m, EPOLLONESHOT); + ADD_INT(EPOLLONESHOT); #endif #ifdef EPOLLEXCLUSIVE - PyModule_AddIntMacro(m, EPOLLEXCLUSIVE); + ADD_INT(EPOLLEXCLUSIVE); #endif #ifdef EPOLLRDNORM - PyModule_AddIntMacro(m, EPOLLRDNORM); + ADD_INT(EPOLLRDNORM); #endif #ifdef EPOLLRDBAND - PyModule_AddIntMacro(m, EPOLLRDBAND); + ADD_INT(EPOLLRDBAND); #endif #ifdef EPOLLWRNORM - PyModule_AddIntMacro(m, EPOLLWRNORM); + ADD_INT(EPOLLWRNORM); #endif #ifdef EPOLLWRBAND - PyModule_AddIntMacro(m, EPOLLWRBAND); + ADD_INT(EPOLLWRBAND); #endif #ifdef EPOLLMSG - PyModule_AddIntMacro(m, EPOLLMSG); + ADD_INT(EPOLLMSG); #endif #ifdef EPOLL_CLOEXEC - PyModule_AddIntMacro(m, EPOLL_CLOEXEC); + ADD_INT(EPOLL_CLOEXEC); #endif #endif /* HAVE_EPOLL */ +#undef ADD_INT + +#define ADD_INT_CONST(NAME, VAL) \ + do { \ + if (PyModule_AddIntConstant(m, NAME, VAL) < 0) { \ + return -1; \ + } \ + } while (0) + #ifdef HAVE_KQUEUE + state->kqueue_open_list = NULL; + state->kqueue_event_Type = (PyTypeObject *)PyType_FromModuleAndSpec( m, &kqueue_event_Type_spec, NULL); if (state->kqueue_event_Type == NULL) { @@ -2574,86 +2752,90 @@ _select_exec(PyObject *m) } /* event filters */ - PyModule_AddIntConstant(m, "KQ_FILTER_READ", EVFILT_READ); - PyModule_AddIntConstant(m, "KQ_FILTER_WRITE", EVFILT_WRITE); + ADD_INT_CONST("KQ_FILTER_READ", EVFILT_READ); + ADD_INT_CONST("KQ_FILTER_WRITE", EVFILT_WRITE); #ifdef EVFILT_AIO - PyModule_AddIntConstant(m, "KQ_FILTER_AIO", EVFILT_AIO); + ADD_INT_CONST("KQ_FILTER_AIO", EVFILT_AIO); #endif #ifdef EVFILT_VNODE - PyModule_AddIntConstant(m, "KQ_FILTER_VNODE", EVFILT_VNODE); + ADD_INT_CONST("KQ_FILTER_VNODE", EVFILT_VNODE); #endif #ifdef EVFILT_PROC - PyModule_AddIntConstant(m, "KQ_FILTER_PROC", EVFILT_PROC); + ADD_INT_CONST("KQ_FILTER_PROC", EVFILT_PROC); #endif #ifdef EVFILT_NETDEV - PyModule_AddIntConstant(m, "KQ_FILTER_NETDEV", EVFILT_NETDEV); + ADD_INT_CONST("KQ_FILTER_NETDEV", EVFILT_NETDEV); #endif #ifdef EVFILT_SIGNAL - PyModule_AddIntConstant(m, "KQ_FILTER_SIGNAL", EVFILT_SIGNAL); + ADD_INT_CONST("KQ_FILTER_SIGNAL", EVFILT_SIGNAL); #endif - PyModule_AddIntConstant(m, "KQ_FILTER_TIMER", EVFILT_TIMER); + ADD_INT_CONST("KQ_FILTER_TIMER", EVFILT_TIMER); /* event flags */ - PyModule_AddIntConstant(m, "KQ_EV_ADD", EV_ADD); - PyModule_AddIntConstant(m, "KQ_EV_DELETE", EV_DELETE); - PyModule_AddIntConstant(m, "KQ_EV_ENABLE", EV_ENABLE); - PyModule_AddIntConstant(m, "KQ_EV_DISABLE", EV_DISABLE); - PyModule_AddIntConstant(m, "KQ_EV_ONESHOT", EV_ONESHOT); - PyModule_AddIntConstant(m, "KQ_EV_CLEAR", EV_CLEAR); + ADD_INT_CONST("KQ_EV_ADD", EV_ADD); + ADD_INT_CONST("KQ_EV_DELETE", EV_DELETE); + ADD_INT_CONST("KQ_EV_ENABLE", EV_ENABLE); + ADD_INT_CONST("KQ_EV_DISABLE", EV_DISABLE); + ADD_INT_CONST("KQ_EV_ONESHOT", EV_ONESHOT); + ADD_INT_CONST("KQ_EV_CLEAR", EV_CLEAR); #ifdef EV_SYSFLAGS - PyModule_AddIntConstant(m, "KQ_EV_SYSFLAGS", EV_SYSFLAGS); + ADD_INT_CONST("KQ_EV_SYSFLAGS", EV_SYSFLAGS); #endif #ifdef EV_FLAG1 - PyModule_AddIntConstant(m, "KQ_EV_FLAG1", EV_FLAG1); + ADD_INT_CONST("KQ_EV_FLAG1", EV_FLAG1); #endif - PyModule_AddIntConstant(m, "KQ_EV_EOF", EV_EOF); - PyModule_AddIntConstant(m, "KQ_EV_ERROR", EV_ERROR); + ADD_INT_CONST("KQ_EV_EOF", EV_EOF); + ADD_INT_CONST("KQ_EV_ERROR", EV_ERROR); /* READ WRITE filter flag */ #ifdef NOTE_LOWAT - PyModule_AddIntConstant(m, "KQ_NOTE_LOWAT", NOTE_LOWAT); + ADD_INT_CONST("KQ_NOTE_LOWAT", NOTE_LOWAT); #endif /* VNODE filter flags */ #ifdef EVFILT_VNODE - PyModule_AddIntConstant(m, "KQ_NOTE_DELETE", NOTE_DELETE); - PyModule_AddIntConstant(m, "KQ_NOTE_WRITE", NOTE_WRITE); - PyModule_AddIntConstant(m, "KQ_NOTE_EXTEND", NOTE_EXTEND); - PyModule_AddIntConstant(m, "KQ_NOTE_ATTRIB", NOTE_ATTRIB); - PyModule_AddIntConstant(m, "KQ_NOTE_LINK", NOTE_LINK); - PyModule_AddIntConstant(m, "KQ_NOTE_RENAME", NOTE_RENAME); - PyModule_AddIntConstant(m, "KQ_NOTE_REVOKE", NOTE_REVOKE); + ADD_INT_CONST("KQ_NOTE_DELETE", NOTE_DELETE); + ADD_INT_CONST("KQ_NOTE_WRITE", NOTE_WRITE); + ADD_INT_CONST("KQ_NOTE_EXTEND", NOTE_EXTEND); + ADD_INT_CONST("KQ_NOTE_ATTRIB", NOTE_ATTRIB); + ADD_INT_CONST("KQ_NOTE_LINK", NOTE_LINK); + ADD_INT_CONST("KQ_NOTE_RENAME", NOTE_RENAME); + ADD_INT_CONST("KQ_NOTE_REVOKE", NOTE_REVOKE); #endif /* PROC filter flags */ #ifdef EVFILT_PROC - PyModule_AddIntConstant(m, "KQ_NOTE_EXIT", NOTE_EXIT); - PyModule_AddIntConstant(m, "KQ_NOTE_FORK", NOTE_FORK); - PyModule_AddIntConstant(m, "KQ_NOTE_EXEC", NOTE_EXEC); - PyModule_AddIntConstant(m, "KQ_NOTE_PCTRLMASK", NOTE_PCTRLMASK); - PyModule_AddIntConstant(m, "KQ_NOTE_PDATAMASK", NOTE_PDATAMASK); - - PyModule_AddIntConstant(m, "KQ_NOTE_TRACK", NOTE_TRACK); - PyModule_AddIntConstant(m, "KQ_NOTE_CHILD", NOTE_CHILD); - PyModule_AddIntConstant(m, "KQ_NOTE_TRACKERR", NOTE_TRACKERR); + ADD_INT_CONST("KQ_NOTE_EXIT", NOTE_EXIT); + ADD_INT_CONST("KQ_NOTE_FORK", NOTE_FORK); + ADD_INT_CONST("KQ_NOTE_EXEC", NOTE_EXEC); + ADD_INT_CONST("KQ_NOTE_PCTRLMASK", NOTE_PCTRLMASK); + ADD_INT_CONST("KQ_NOTE_PDATAMASK", NOTE_PDATAMASK); + + ADD_INT_CONST("KQ_NOTE_TRACK", NOTE_TRACK); + ADD_INT_CONST("KQ_NOTE_CHILD", NOTE_CHILD); + ADD_INT_CONST("KQ_NOTE_TRACKERR", NOTE_TRACKERR); #endif /* NETDEV filter flags */ #ifdef EVFILT_NETDEV - PyModule_AddIntConstant(m, "KQ_NOTE_LINKUP", NOTE_LINKUP); - PyModule_AddIntConstant(m, "KQ_NOTE_LINKDOWN", NOTE_LINKDOWN); - PyModule_AddIntConstant(m, "KQ_NOTE_LINKINV", NOTE_LINKINV); + ADD_INT_CONST("KQ_NOTE_LINKUP", NOTE_LINKUP); + ADD_INT_CONST("KQ_NOTE_LINKDOWN", NOTE_LINKDOWN); + ADD_INT_CONST("KQ_NOTE_LINKINV", NOTE_LINKINV); #endif #endif /* HAVE_KQUEUE */ + +#undef ADD_INT_CONST + return 0; } static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 3fd53123229ac4..34a427a39b5cf8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -49,9 +49,10 @@ typedef long long SHA1_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. + bool use_mutex; + PyMutex mutex; PyThread_type_lock lock; - Hacl_Streaming_SHA1_state *hash_state; + Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; #include "clinic/sha1module.c.h" @@ -76,7 +77,8 @@ newSHA1object(SHA1State *st) if (sha == NULL) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -93,10 +95,7 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) static void SHA1_dealloc(SHA1object *ptr) { - Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } + Hacl_Hash_SHA1_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -125,7 +124,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA1_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -142,7 +141,7 @@ SHA1Type_digest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -159,20 +158,20 @@ SHA1Type_hexdigest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } -static void update(Hacl_Streaming_SHA1_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_SHA1_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA1_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA1_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_SHA1_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA1_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -192,14 +191,14 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update(self->hash_state, buf.buf, buf.len); @@ -296,7 +295,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_SHA1_legacy_create_in(); + new->hash_state = Hacl_Hash_SHA1_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -372,6 +371,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 6ad1ff2e05bfd8..7d6a1e40243f9d 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -53,18 +53,18 @@ typedef struct { PyObject_HEAD int digestsize; // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; - Hacl_Streaming_SHA2_state_sha2_256 *state; + bool use_mutex; + PyMutex mutex; + Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; typedef struct { PyObject_HEAD int digestsize; // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; - Hacl_Streaming_SHA2_state_sha2_512 *state; + bool use_mutex; + PyMutex mutex; + Hacl_Hash_SHA2_state_t_512 *state; } SHA512object; #include "clinic/sha2module.c.h" @@ -89,13 +89,13 @@ sha2_get_state(PyObject *module) static void SHA256copy(SHA256object *src, SHA256object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_256(src->state); + dest->state = Hacl_Hash_SHA2_copy_256(src->state); } static void SHA512copy(SHA512object *src, SHA512object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_512(src->state); + dest->state = Hacl_Hash_SHA2_copy_512(src->state); } static SHA256object * @@ -106,7 +106,8 @@ newSHA224object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -119,7 +120,8 @@ newSHA256object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -132,7 +134,8 @@ newSHA384object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -145,7 +148,8 @@ newSHA512object(sha2_state *state) if (!sha) { return NULL; } - sha->lock = NULL; + HASHLIB_INIT_MUTEX(sha); + PyObject_GC_Track(sha); return sha; } @@ -162,10 +166,7 @@ SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) static void SHA256_dealloc(SHA256object *ptr) { - Hacl_Streaming_SHA2_free_256(ptr->state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } + Hacl_Hash_SHA2_free_256(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -175,10 +176,7 @@ SHA256_dealloc(SHA256object *ptr) static void SHA512_dealloc(SHA512object *ptr) { - Hacl_Streaming_SHA2_free_512(ptr->state); - if (ptr->lock != NULL) { - PyThread_free_lock(ptr->lock); - } + Hacl_Hash_SHA2_free_512(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -188,34 +186,34 @@ SHA512_dealloc(SHA512object *ptr) /* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be * 64 bits so we loop in <4gig chunks when needed. */ -static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { +static void update_256(Hacl_Hash_SHA2_state_t_256 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for SHA2-256 * (namely, 2^61-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_256(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_256(state, buf, (uint32_t) len); } -static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { +static void update_512(Hacl_Hash_SHA2_state_t_512 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for this API * (namely, 2^64-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_512(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_512(state, buf, (uint32_t) len); } @@ -298,7 +296,7 @@ SHA256Type_digest_impl(SHA256object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -318,7 +316,7 @@ SHA512Type_digest_impl(SHA512object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -336,7 +334,7 @@ SHA256Type_hexdigest_impl(SHA256object *self) uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -354,7 +352,7 @@ SHA512Type_hexdigest_impl(SHA512object *self) uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -376,14 +374,14 @@ SHA256Type_update(SHA256object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update_256(self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update_256(self->state, buf.buf, buf.len); @@ -410,14 +408,14 @@ SHA512Type_update(SHA512object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); update_512(self->state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { update_512(self->state, buf.buf, buf.len); @@ -599,7 +597,7 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_256(); + new->state = Hacl_Hash_SHA2_malloc_256(); new->digestsize = 32; if (PyErr_Occurred()) { @@ -653,7 +651,7 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_224(); + new->state = Hacl_Hash_SHA2_malloc_224(); new->digestsize = 28; if (PyErr_Occurred()) { @@ -707,7 +705,7 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_512(); + new->state = Hacl_Hash_SHA2_malloc_512(); new->digestsize = 64; if (PyErr_Occurred()) { @@ -760,7 +758,7 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_384(); + new->state = Hacl_Hash_SHA2_malloc_384(); new->digestsize = 48; if (PyErr_Occurred()) { @@ -868,6 +866,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 558d2005cff617..084332c1efa0e0 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -61,9 +61,9 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD // Prevents undefined behavior via multiple threads entering the C API. - // The lock will be NULL before threaded access has been enabled. - PyThread_type_lock lock; - Hacl_Streaming_Keccak_state *hash_state; + bool use_mutex; + PyMutex mutex; + Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -76,22 +76,23 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; + HASHLIB_INIT_MUTEX(newobj); + return newobj; } -static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { +static void sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to hash more than 2^64 bytes. */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA3_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA3_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -119,17 +120,17 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; @@ -168,10 +169,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - Hacl_Streaming_Keccak_free(self->hash_state); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } + Hacl_Hash_SHA3_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -197,7 +195,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) return NULL; } ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA3_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -217,10 +215,10 @@ _sha3_sha3_224_digest_impl(SHA3object *self) // This function errors out if the algorithm is Shake. Here, we know this // not to be the case, and therefore do not perform error checking. ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -236,10 +234,10 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) { unsigned char digest[SHA3_MAX_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -257,19 +255,22 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); + + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; } - if (self->lock != NULL) { + if (self->use_mutex) { Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); + PyMutex_Lock(&self->mutex); sha3_update(self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); + PyMutex_Unlock(&self->mutex); Py_END_ALLOW_THREADS } else { sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -287,7 +288,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -323,17 +324,17 @@ static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { // Preserving previous behavior: variable-length algorithms return 0 - if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + if (Hacl_Hash_SHA3_is_shake(self->hash_state)) return PyLong_FromLong(0); else - return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); + return PyLong_FromLong(Hacl_Hash_SHA3_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -342,7 +343,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -435,7 +436,7 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) * - the output length is zero -- we follow the existing behavior and return * an empty digest, without raising an error */ if (digestlen > 0) { - Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + Hacl_Hash_SHA3_squeeze(self->hash_state, digest, digestlen); } if (hex) { result = _Py_strhex((const char *)digest, digestlen); @@ -601,6 +602,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index bc5cdf0ab6b5f2..7de5ebe0899b35 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // _Py_RestoreSignals() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #ifndef MS_WINDOWS # include "posixmodule.h" // _PyLong_FromUid() @@ -21,6 +22,9 @@ # include "socketmodule.h" // SOCKET_T #endif +#ifdef HAVE_UNISTD_H +# include // alarm() +#endif #ifdef MS_WINDOWS # ifdef HAVE_PROCESS_H # include @@ -170,7 +174,7 @@ timeval_from_double(PyObject *obj, struct timeval *tv) return 0; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromSecondsObject(&t, obj, _PyTime_ROUND_CEILING) < 0) { return -1; } @@ -241,8 +245,7 @@ report_wakeup_write_error(void *data) errno = (int) (intptr_t) data; PyObject *exc = PyErr_GetRaisedException(); PyErr_SetFromErrno(PyExc_OSError); - _PyErr_WriteUnraisableMsg("when trying to write to the signal wakeup fd", - NULL); + PyErr_FormatUnraisable("Exception ignored when trying to write to the signal wakeup fd"); PyErr_SetRaisedException(exc); errno = save_errno; return 0; @@ -259,7 +262,7 @@ report_wakeup_send_error(void* data) recognizes the error codes used by both GetLastError() and WSAGetLastError */ PyErr_SetExcFromWindowsErr(PyExc_OSError, send_errno); - _PyErr_WriteUnraisableMsg("when trying to send to the signal wakeup fd", NULL); + PyErr_FormatUnraisable("Exception ignored when trying to send to the signal wakeup fd"); PyErr_SetRaisedException(exc); return 0; } @@ -274,11 +277,7 @@ trip_signal(int sig_num) cleared in PyErr_CheckSignals() before .tripped. */ _Py_atomic_store_int(&is_tripped, 1); - /* Signals are always handled by the main interpreter */ - PyInterpreterState *interp = _PyInterpreterState_Main(); - - /* Notify ceval.c */ - _PyEval_SignalReceived(interp); + _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and doing the _PyEval_SignalReceived. We used to write to the wakeup fd @@ -301,6 +300,7 @@ trip_signal(int sig_num) int fd = wakeup.fd; if (fd != INVALID_FD) { + PyInterpreterState *interp = _PyInterpreterState_Main(); unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS if (wakeup.use_send) { @@ -657,7 +657,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) Py_RETURN_NONE; #endif - return Py_BuildValue("s", res); + return PyUnicode_FromString(res); } #ifdef HAVE_SIGINTERRUPT @@ -706,35 +706,43 @@ signal_siginterrupt_impl(PyObject *module, int signalnum, int flag) #endif -static PyObject* -signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) +/*[clinic input] +signal.set_wakeup_fd + + fd as fdobj: object + / + * + warn_on_full_buffer: bool = True + +Sets the fd to be written to (with the signal number) when a signal comes in. + +A library can use this to wakeup select or poll. +The previous fd or -1 is returned. + +The fd must be non-blocking. +[clinic start generated code]*/ + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer) +/*[clinic end generated code: output=2280d72dd2a54c4f input=5b545946a28b8339]*/ { struct _Py_stat_struct status; - static char *kwlist[] = { - "", "warn_on_full_buffer", NULL, - }; - int warn_on_full_buffer = 1; #ifdef MS_WINDOWS - PyObject *fdobj; SOCKET_T sockfd, old_sockfd; int res; int res_size = sizeof res; PyObject *mod; int is_socket; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$p:set_wakeup_fd", kwlist, - &fdobj, &warn_on_full_buffer)) - return NULL; - sockfd = PyLong_AsSocket_t(fdobj); if (sockfd == (SOCKET_T)(-1) && PyErr_Occurred()) return NULL; #else - int fd; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i|$p:set_wakeup_fd", kwlist, - &fd, &warn_on_full_buffer)) + int fd = PyLong_AsInt(fdobj); + if (fd == -1 && PyErr_Occurred()) { return NULL; + } #endif PyThreadState *tstate = _PyThreadState_GET(); @@ -820,15 +828,6 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) #endif } -PyDoc_STRVAR(set_wakeup_fd_doc, -"set_wakeup_fd(fd, *, warn_on_full_buffer=True) -> fd\n\ -\n\ -Sets the fd to be written to (with the signal number) when a signal\n\ -comes in. A library can use this to wakeup select or poll.\n\ -The previous fd or -1 is returned.\n\ -\n\ -The fd must be non-blocking."); - /* C API for the same, without all the error checking */ int PySignal_SetWakeupFd(int fd) @@ -1208,7 +1207,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) /*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ { - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; @@ -1218,7 +1217,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, return NULL; } - _PyTime_t deadline = _PyDeadline_Init(timeout); + PyTime_t deadline = _PyDeadline_Init(timeout); siginfo_t si; do { @@ -1344,7 +1343,7 @@ static PyMethodDef signal_methods[] = { SIGNAL_RAISE_SIGNAL_METHODDEF SIGNAL_STRSIGNAL_METHODDEF SIGNAL_GETSIGNAL_METHODDEF - {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_VARARGS | METH_KEYWORDS, set_wakeup_fd_doc}, + SIGNAL_SET_WAKEUP_FD_METHODDEF SIGNAL_SIGINTERRUPT_METHODDEF SIGNAL_PAUSE_METHODDEF SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF @@ -1699,6 +1698,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -1768,8 +1768,8 @@ PyErr_CheckSignals(void) Python code to ensure signals are handled. Checking for the GC here allows long running native code to clean cycles created using the C-API even if it doesn't run the evaluation loop */ - if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0); + if (_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 3d099d41d1e761..3ffdaa45f16ac7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -107,9 +107,9 @@ Local naming conventions: #include "Python.h" #include "pycore_capsule.h" // _PyCapsule_SetTraverse() -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_time.h" // _PyTime_AsMilliseconds() #ifdef _Py_MEMORY_SANITIZER # include @@ -393,16 +393,10 @@ remove_unusable_flags(PyObject *m) break; } else { - PyObject *flag_name = PyUnicode_FromString(win_runtime_flags[i].flag_name); - if (flag_name == NULL) { + if (PyDict_PopString(dict, win_runtime_flags[i].flag_name, + NULL) < 0) { return -1; } - PyObject *v = _PyDict_Pop(dict, flag_name, Py_None); - Py_DECREF(flag_name); - if (v == NULL) { - return -1; - } - Py_DECREF(v); } } return 0; @@ -554,7 +548,7 @@ typedef struct _socket_state { PyObject *socket_gaierror; /* Default timeout for new sockets */ - _PyTime_t defaulttimeout; + PyTime_t defaulttimeout; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -779,13 +773,13 @@ internal_setblocking(PySocketSockObject *s, int block) } static int -internal_select(PySocketSockObject *s, int writing, _PyTime_t interval, +internal_select(PySocketSockObject *s, int writing, PyTime_t interval, int connect) { int n; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else fd_set fds, efds; struct timeval tv, *tvp; @@ -895,10 +889,10 @@ sock_call_ex(PySocketSockObject *s, void *data, int connect, int *err, - _PyTime_t timeout) + PyTime_t timeout) { int has_timeout = (timeout > 0); - _PyTime_t deadline = 0; + PyTime_t deadline = 0; int deadline_initialized = 0; int res; @@ -912,7 +906,7 @@ sock_call_ex(PySocketSockObject *s, runs asynchronously. */ if (has_timeout || connect) { if (has_timeout) { - _PyTime_t interval; + PyTime_t interval; if (deadline_initialized) { /* recompute the timeout */ @@ -1060,8 +1054,8 @@ init_sockobject(socket_state *state, PySocketSockObject *s, else #endif { - s->sock_timeout = state->defaulttimeout; - if (state->defaulttimeout >= 0) { + s->sock_timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (s->sock_timeout >= 0) { if (internal_setblocking(s, 0) == -1) { return -1; } @@ -1893,12 +1887,14 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ #ifdef AF_DIVERT case AF_DIVERT: - /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ + /* FreeBSD divert(4) sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_DIVERT */ case AF_INET: @@ -2220,7 +2216,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, switch (s->sock_proto) { #ifdef CAN_RAW case CAN_RAW: - /* fall-through */ + _Py_FALLTHROUGH; #endif #ifdef CAN_BCM case CAN_BCM: @@ -2596,7 +2592,8 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ case AF_INET: @@ -3018,13 +3015,13 @@ Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) +socket_parse_timeout(PyTime_t *timeout, PyObject *timeout_obj) { #ifdef MS_WINDOWS struct timeval tv; #endif #ifndef HAVE_POLL - _PyTime_t ms; + PyTime_t ms; #endif int overflow = 0; @@ -3067,7 +3064,7 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) static PyObject * sock_settimeout(PySocketSockObject *s, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -3119,7 +3116,7 @@ sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(s->sock_timeout); + double seconds = PyTime_AsSecondsDouble(s->sock_timeout); return PyFloat_FromDouble(seconds); } } @@ -3334,8 +3331,19 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])"); Set the file descriptor to -1 so operations tried subsequently will surely fail. */ +/*[clinic input] +@critical_section +_socket.socket.close + self as s: self(type="PySocketSockObject *") + +close() + +Close the socket. It cannot be used after this call. +[clinic start generated code]*/ + static PyObject * -sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +_socket_socket_close_impl(PySocketSockObject *s) +/*[clinic end generated code: output=038b2418e07f6f6c input=9839a261e05bcb97]*/ { SOCKET_T fd; int res; @@ -3360,11 +3368,6 @@ sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -PyDoc_STRVAR(sock_close_doc, -"close()\n\ -\n\ -Close the socket. It cannot be used after this call."); - static PyObject * sock_detach(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) { @@ -4389,8 +4392,8 @@ sock_sendall(PySocketSockObject *s, PyObject *args) Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); - _PyTime_t timeout = s->sock_timeout; - _PyTime_t deadline = 0; + PyTime_t timeout = s->sock_timeout; + PyTime_t deadline = 0; int deadline_initialized = 0; PyObject *res = NULL; @@ -5121,8 +5124,7 @@ static PyMethodDef sock_methods[] = { {"bind", (PyCFunction)sock_bind, METH_O, bind_doc}, #endif - {"close", (PyCFunction)sock_close, METH_NOARGS, - sock_close_doc}, + _SOCKET_SOCKET_CLOSE_METHODDEF #ifdef HAVE_CONNECT {"connect", (PyCFunction)sock_connect, METH_O, connect_doc}, @@ -6333,14 +6335,18 @@ AF_UNIX if defined on the platform; otherwise, the default is AF_INET."); #endif /* HAVE_SOCKETPAIR */ +/*[clinic input] +_socket.socket.ntohs + x: int + / + +Convert a 16-bit unsigned integer from network to host byte order. +[clinic start generated code]*/ + static PyObject * -socket_ntohs(PyObject *self, PyObject *args) +_socket_socket_ntohs_impl(PySocketSockObject *self, int x) +/*[clinic end generated code: output=a828a61a9fb205b2 input=9a79cb3a71652147]*/ { - int x; - - if (!PyArg_ParseTuple(args, "i:ntohs", &x)) { - return NULL; - } if (x < 0) { PyErr_SetString(PyExc_OverflowError, "ntohs: can't convert negative Python int to C " @@ -6356,11 +6362,6 @@ socket_ntohs(PyObject *self, PyObject *args) return PyLong_FromUnsignedLong(ntohs((unsigned short)x)); } -PyDoc_STRVAR(ntohs_doc, -"ntohs(integer) -> integer\n\ -\n\ -Convert a 16-bit unsigned integer from network to host byte order."); - static PyObject * socket_ntohl(PyObject *self, PyObject *arg) @@ -6396,14 +6397,18 @@ PyDoc_STRVAR(ntohl_doc, Convert a 32-bit integer from network to host byte order."); +/*[clinic input] +_socket.socket.htons + x: int + / + +Convert a 16-bit unsigned integer from host to network byte order. +[clinic start generated code]*/ + static PyObject * -socket_htons(PyObject *self, PyObject *args) +_socket_socket_htons_impl(PySocketSockObject *self, int x) +/*[clinic end generated code: output=d785ee692312da47 input=053252d8416f4337]*/ { - int x; - - if (!PyArg_ParseTuple(args, "i:htons", &x)) { - return NULL; - } if (x < 0) { PyErr_SetString(PyExc_OverflowError, "htons: can't convert negative Python int to C " @@ -6419,11 +6424,6 @@ socket_htons(PyObject *self, PyObject *args) return PyLong_FromUnsignedLong(htons((unsigned short)x)); } -PyDoc_STRVAR(htons_doc, -"htons(integer) -> integer\n\ -\n\ -Convert a 16-bit unsigned integer from host to network byte order."); - static PyObject * socket_htonl(PyObject *self, PyObject *arg) @@ -6460,14 +6460,17 @@ Convert a 32-bit integer from host to network byte order."); /* socket.inet_aton() and socket.inet_ntoa() functions. */ -PyDoc_STRVAR(inet_aton_doc, -"inet_aton(string) -> bytes giving packed 32-bit IP representation\n\ -\n\ -Convert an IP address in string format (123.45.67.89) to the 32-bit packed\n\ -binary format used in low-level network functions."); +/*[clinic input] +_socket.socket.inet_aton + ip_addr: str + / -static PyObject* -socket_inet_aton(PyObject *self, PyObject *args) +Convert an IP address in string format (123.45.67.89) to the 32-bit packed binary format used in low-level network functions. +[clinic start generated code]*/ + +static PyObject * +_socket_socket_inet_aton_impl(PySocketSockObject *self, const char *ip_addr) +/*[clinic end generated code: output=5bfe11a255423d8c input=a120e20cb52b9488]*/ { #ifdef HAVE_INET_ATON struct in_addr buf; @@ -6480,11 +6483,6 @@ socket_inet_aton(PyObject *self, PyObject *args) /* Have to use inet_addr() instead */ unsigned int packed_addr; #endif - const char *ip_addr; - - if (!PyArg_ParseTuple(args, "s:inet_aton", &ip_addr)) - return NULL; - #ifdef HAVE_INET_ATON @@ -6533,30 +6531,29 @@ socket_inet_aton(PyObject *self, PyObject *args) } #ifdef HAVE_INET_NTOA -PyDoc_STRVAR(inet_ntoa_doc, -"inet_ntoa(packed_ip) -> ip_address_string\n\ -\n\ -Convert an IP address from 32-bit packed binary format to string format"); +/*[clinic input] +_socket.socket.inet_ntoa + packed_ip: Py_buffer + / -static PyObject* -socket_inet_ntoa(PyObject *self, PyObject *args) +Convert an IP address from 32-bit packed binary format to string format. +[clinic start generated code]*/ + +static PyObject * +_socket_socket_inet_ntoa_impl(PySocketSockObject *self, Py_buffer *packed_ip) +/*[clinic end generated code: output=b671880a3f62461b input=95c2c4a1b2ee957c]*/ { - Py_buffer packed_ip; struct in_addr packed_addr; - if (!PyArg_ParseTuple(args, "y*:inet_ntoa", &packed_ip)) { - return NULL; - } - - if (packed_ip.len != sizeof(packed_addr)) { + if (packed_ip->len != sizeof(packed_addr)) { PyErr_SetString(PyExc_OSError, "packed IP wrong length for inet_ntoa"); - PyBuffer_Release(&packed_ip); + PyBuffer_Release(packed_ip); return NULL; } - memcpy(&packed_addr, packed_ip.buf, packed_ip.len); - PyBuffer_Release(&packed_ip); + memcpy(&packed_addr, packed_ip->buf, packed_ip->len); + PyBuffer_Release(packed_ip); SUPPRESS_DEPRECATED_CALL return PyUnicode_FromString(inet_ntoa(packed_addr)); @@ -6924,11 +6921,12 @@ static PyObject * socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) { socket_state *state = get_module_state(self); - if (state->defaulttimeout < 0) { + PyTime_t timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (timeout < 0) { Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(timeout); return PyFloat_FromDouble(seconds); } } @@ -6943,13 +6941,13 @@ When the socket module is first imported, the default is None."); static PyObject * socket_setdefaulttimeout(PyObject *self, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; socket_state *state = get_module_state(self); - state->defaulttimeout = timeout; + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, timeout); Py_RETURN_NONE; } @@ -7050,18 +7048,23 @@ PyDoc_STRVAR(if_nameindex_doc, \n\ Returns a list of network interface information (index, name) tuples."); +/*[clinic input] +_socket.socket.if_nametoindex + oname: object(converter="PyUnicode_FSConverter") + / + +Returns the interface index corresponding to the interface name if_name. +[clinic start generated code]*/ + static PyObject * -socket_if_nametoindex(PyObject *self, PyObject *args) +_socket_socket_if_nametoindex_impl(PySocketSockObject *self, PyObject *oname) +/*[clinic end generated code: output=f7fc00511a309a8e input=662688054482cd46]*/ { - PyObject *oname; #ifdef MS_WINDOWS NET_IFINDEX index; #else unsigned long index; #endif - if (!PyArg_ParseTuple(args, "O&:if_nametoindex", - PyUnicode_FSConverter, &oname)) - return NULL; index = if_nametoindex(PyBytes_AS_STRING(oname)); Py_DECREF(oname); @@ -7074,25 +7077,27 @@ socket_if_nametoindex(PyObject *self, PyObject *args) return PyLong_FromUnsignedLong(index); } -PyDoc_STRVAR(if_nametoindex_doc, -"if_nametoindex(if_name)\n\ -\n\ -Returns the interface index corresponding to the interface name if_name."); static PyObject * socket_if_indextoname(PyObject *self, PyObject *arg) { + unsigned long index_long = PyLong_AsUnsignedLong(arg); + if (index_long == (unsigned long) -1 && PyErr_Occurred()) { + return NULL; + } + #ifdef MS_WINDOWS - NET_IFINDEX index; + NET_IFINDEX index = (NET_IFINDEX)index_long; #else - unsigned long index; + unsigned int index = (unsigned int)index_long; #endif - char name[IF_NAMESIZE + 1]; - index = PyLong_AsUnsignedLong(arg); - if (index == (unsigned long) -1) + if ((unsigned long)index != index_long) { + PyErr_SetString(PyExc_OverflowError, "index is too large"); return NULL; + } + char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -7216,19 +7221,15 @@ static PyMethodDef socket_methods[] = { {"socketpair", socket_socketpair, METH_VARARGS, socketpair_doc}, #endif - {"ntohs", socket_ntohs, - METH_VARARGS, ntohs_doc}, + _SOCKET_SOCKET_NTOHS_METHODDEF {"ntohl", socket_ntohl, METH_O, ntohl_doc}, - {"htons", socket_htons, - METH_VARARGS, htons_doc}, + _SOCKET_SOCKET_HTONS_METHODDEF {"htonl", socket_htonl, METH_O, htonl_doc}, - {"inet_aton", socket_inet_aton, - METH_VARARGS, inet_aton_doc}, + _SOCKET_SOCKET_INET_ATON_METHODDEF #ifdef HAVE_INET_NTOA - {"inet_ntoa", socket_inet_ntoa, - METH_VARARGS, inet_ntoa_doc}, + _SOCKET_SOCKET_INET_NTOA_METHODDEF #endif #ifdef HAVE_INET_PTON {"inet_pton", socket_inet_pton, @@ -7251,8 +7252,7 @@ static PyMethodDef socket_methods[] = { #if defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) {"if_nameindex", socket_if_nameindex, METH_NOARGS, if_nameindex_doc}, - {"if_nametoindex", socket_if_nametoindex, - METH_VARARGS, if_nametoindex_doc}, + _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF {"if_indextoname", socket_if_indextoname, METH_O, if_indextoname_doc}, #endif @@ -7604,6 +7604,7 @@ socket_exec(PyObject *m) ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_LOCAL", 1); ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); @@ -7723,10 +7724,10 @@ socket_exec(PyObject *m) /* FreeBSD divert(4) */ #ifdef PF_DIVERT - PyModule_AddIntMacro(m, PF_DIVERT); + ADD_INT_MACRO(m, PF_DIVERT); #endif #ifdef AF_DIVERT - PyModule_AddIntMacro(m, AF_DIVERT); + ADD_INT_MACRO(m, AF_DIVERT); #endif #ifdef AF_PACKET @@ -7936,6 +7937,9 @@ socket_exec(PyObject *m) #ifdef SO_BINDTODEVICE ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif +#ifdef SO_BINDTOIFINDEX + ADD_INT_MACRO(m, SO_BINDTOIFINDEX); +#endif #ifdef SO_PRIORITY ADD_INT_MACRO(m, SO_PRIORITY); #endif @@ -8416,15 +8420,24 @@ socket_exec(PyObject *m) #ifdef IP_TTL ADD_INT_MACRO(m, IP_TTL); #endif +#ifdef IP_RECVERR + ADD_INT_MACRO(m, IP_RECVERR); +#endif #ifdef IP_RECVOPTS ADD_INT_MACRO(m, IP_RECVOPTS); #endif +#ifdef IP_RECVORIGDSTADDR + ADD_INT_MACRO(m, IP_RECVORIGDSTADDR); +#endif #ifdef IP_RECVRETOPTS ADD_INT_MACRO(m, IP_RECVRETOPTS); #endif #ifdef IP_RECVTOS ADD_INT_MACRO(m, IP_RECVTOS); #endif +#ifdef IP_RECVTTL + ADD_INT_MACRO(m, IP_RECVTTL); +#endif #ifdef IP_RECVDSTADDR ADD_INT_MACRO(m, IP_RECVDSTADDR); #endif @@ -8819,6 +8832,9 @@ socket_exec(PyObject *m) #ifdef NI_DGRAM ADD_INT_MACRO(m, NI_DGRAM); #endif +#ifdef NI_IDN + ADD_INT_MACRO(m, NI_IDN); +#endif /* shutdown() parameters */ #ifdef SHUT_RD @@ -8898,6 +8914,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 47146a28e02c8f..09fd70f351f1d8 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -1,7 +1,5 @@ /* Socket module header file */ -#include "pycore_time.h" // _PyTime_t - /* Includes needed for the sockaddr_* symbols below */ #ifndef MS_WINDOWS #ifdef __VMS @@ -324,7 +322,7 @@ typedef struct { PyObject *(*errorhandler)(void); /* Error handler; checks errno, returns NULL and sets a Python exception */ - _PyTime_t sock_timeout; /* Operation timeout in seconds; + PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; } PySocketSockObject; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index ddc9ac3324356d..d0d5223e5acea8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -75,24 +75,27 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_NONLOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) + return -1; if (PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_ANNOTATION", AnnotationBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_VAR_BOUND", TypeVarBoundBlock) < 0) - return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_ALIAS", TypeAliasBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAM", TypeParamBlock) < 0) + if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAMETERS", TypeParametersBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; @@ -110,6 +113,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 6db8de9c491dd9..14e7ca591a076b 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -49,6 +49,11 @@ Revision history: /* syslog module */ +// clinic/syslogmodule.c.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "osdefs.h" // SEP @@ -127,6 +132,7 @@ syslog_get_argv(void) /*[clinic input] +@critical_section syslog.openlog ident: unicode = NULL @@ -139,7 +145,7 @@ Set logging options of subsequent syslog() calls. static PyObject * syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) -/*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ +/*[clinic end generated code: output=5476c12829b6eb75 input=ee700b8786f81c23]*/ { // Since the sys.openlog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -184,6 +190,7 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, /*[clinic input] +@critical_section syslog.syslog [ @@ -200,7 +207,7 @@ Send the string message to the system logger. static PyObject * syslog_syslog_impl(PyObject *module, int group_left_1, int priority, const char *message) -/*[clinic end generated code: output=c3dbc73445a0e078 input=ac83d92b12ea3d4e]*/ +/*[clinic end generated code: output=c3dbc73445a0e078 input=6588ddb0b113af8e]*/ { if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { return NULL; @@ -238,6 +245,7 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /*[clinic input] +@critical_section syslog.closelog Reset the syslog module values and call the system library closelog(). @@ -245,7 +253,7 @@ Reset the syslog module values and call the system library closelog(). static PyObject * syslog_closelog_impl(PyObject *module) -/*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ +/*[clinic end generated code: output=97890a80a24b1b84 input=167f489868bd5a72]*/ { // Since the sys.closelog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -401,12 +409,37 @@ syslog_exec(PyObject *module) ADD_INT_MACRO(module, LOG_AUTHPRIV); #endif +#ifdef LOG_FTP + ADD_INT_MACRO(module, LOG_FTP); +#endif + +#ifdef LOG_NETINFO + ADD_INT_MACRO(module, LOG_NETINFO); +#endif + +#ifdef LOG_REMOTEAUTH + ADD_INT_MACRO(module, LOG_REMOTEAUTH); +#endif + +#ifdef LOG_INSTALL + ADD_INT_MACRO(module, LOG_INSTALL); +#endif + +#ifdef LOG_RAS + ADD_INT_MACRO(module, LOG_RAS); +#endif + +#ifdef LOG_LAUNCHD + ADD_INT_MACRO(module, LOG_LAUNCHD); +#endif + return 0; } static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index 9fc2673ce0e788..efb5fcc17fa5ef 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1,11 +1,19 @@ /* termios.c -- POSIX terminal I/O module implementation. */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +// in code generated by Argument Clinic. +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // memset() +#include +#include +#include // _POSIX_VDISABLE + // On QNX 6, struct termio must be declared by including sys/termio.h // if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must // be included before termios.h or it will generate an error. @@ -19,16 +27,10 @@ # define CTRL(c) ((c)&037) #endif +// We could do better. Check bpo-32660 #if defined(__sun) -/* We could do better. Check issue-32660 */ -#include -#include -#endif - -#include -#include -#if defined(__sun) && defined(__SVR4) -# include // ioctl() +# include +# include #endif /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, @@ -36,13 +38,15 @@ * defined as macros; these are not used here directly). */ #ifdef HAVE_SYS_MODEM_H -#include +# include #endif + /* HP-UX requires that this be included to pick up TIOCGPGRP and friends */ #ifdef HAVE_SYS_BSDTTY_H -#include +# include #endif + /*[clinic input] module termios [clinic start generated code]*/ @@ -98,6 +102,8 @@ termios_tcgetattr_impl(PyObject *module, int fd) struct termios mode; int r; + /* Alpine Linux can leave some fields uninitialized. */ + memset(&mode, 0, sizeof(mode)); Py_BEGIN_ALLOW_THREADS r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS @@ -120,7 +126,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) v = PyBytes_FromStringAndSize(&ch, 1); if (v == NULL) goto err; - PyList_SET_ITEM(cc, i, v); + PyList_SetItem(cc, i, v); } /* Convert the MIN and TIME slots to integer. On some systems, the @@ -154,7 +160,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) Py_DECREF(v); \ goto err; \ } \ - PyList_SET_ITEM(v, index, l); \ + PyList_SetItem(v, index, l); \ } while (0) ADD_LONG_ITEM(0, mode.c_iflag); @@ -165,7 +171,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) ADD_LONG_ITEM(5, ospeed); #undef ADD_LONG_ITEM - PyList_SET_ITEM(v, 6, cc); + PyList_SetItem(v, 6, cc); return v; err: Py_DECREF(cc); @@ -214,7 +220,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) speed_t ispeed, ospeed; #define SET_FROM_LIST(TYPE, VAR, LIST, N) do { \ - PyObject *item = PyList_GET_ITEM(LIST, N); \ + PyObject *item = PyList_GetItem(LIST, N); \ long num = PyLong_AsLong(item); \ if (num == -1 && PyErr_Occurred()) { \ return NULL; \ @@ -230,7 +236,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) SET_FROM_LIST(speed_t, ospeed, term, 5); #undef SET_FROM_LIST - PyObject *cc = PyList_GET_ITEM(term, 6); + PyObject *cc = PyList_GetItem(term, 6); if (!PyList_Check(cc) || PyList_Size(cc) != NCCS) { PyErr_Format(PyExc_TypeError, "tcsetattr: attributes[6] must be %d element list", @@ -254,7 +260,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) } else { PyErr_SetString(PyExc_TypeError, - "tcsetattr: elements of attributes must be characters or integers"); + "tcsetattr: elements of attributes must be bytes objects of length 1 or integers"); return NULL; } } @@ -702,6 +708,9 @@ static struct constant { #ifdef IMAXBEL {"IMAXBEL", IMAXBEL}, #endif +#ifdef IUTF8 + {"IUTF8", IUTF8}, +#endif /* struct termios.c_oflag constants */ {"OPOST", OPOST}, @@ -726,6 +735,12 @@ static struct constant { #ifdef OFDEL {"OFDEL", OFDEL}, #endif +#ifdef OXTABS + {"OXTABS", OXTABS}, +#endif +#ifdef ONOEOT + {"ONOEOT", ONOEOT}, +#endif #ifdef NLDLY {"NLDLY", NLDLY}, #endif @@ -752,6 +767,12 @@ static struct constant { #ifdef NL1 {"NL1", NL1}, #endif +#ifdef NL2 + {"NL2", NL2}, +#endif +#ifdef NL3 + {"NL3", NL3}, +#endif #ifdef CR0 {"CR0", CR0}, #endif @@ -799,6 +820,9 @@ static struct constant { #endif /* struct termios.c_cflag constants */ +#ifdef CIGNORE + {"CIGNORE", CIGNORE}, +#endif {"CSIZE", CSIZE}, {"CSTOPB", CSTOPB}, {"CREAD", CREAD}, @@ -813,6 +837,25 @@ static struct constant { {"CRTSCTS", (long)CRTSCTS}, #endif +#ifdef CRTS_IFLOW + {"CRTS_IFLOW", CRTS_IFLOW}, +#endif +#ifdef CDTR_IFLOW + {"CDTR_IFLOW", CDTR_IFLOW}, +#endif +#ifdef CDSR_OFLOW + {"CDSR_OFLOW", CDSR_OFLOW}, +#endif +#ifdef CCTS_OFLOW + {"CCTS_OFLOW", CCTS_OFLOW}, +#endif +#ifdef CCAR_OFLOW + {"CCAR_OFLOW", CCAR_OFLOW}, +#endif +#ifdef MDMBUF + {"MDMBUF", MDMBUF}, +#endif + /* struct termios.c_cflag-related values (character size) */ {"CS5", CS5}, {"CS6", CS6}, @@ -820,6 +863,9 @@ static struct constant { {"CS8", CS8}, /* struct termios.c_lflag constants */ +#ifdef ALTWERASE + {"ALTWERASE", ALTWERASE}, +#endif {"ISIG", ISIG}, {"ICANON", ICANON}, #ifdef XCASE @@ -840,6 +886,9 @@ static struct constant { #endif #ifdef FLUSHO {"FLUSHO", FLUSHO}, +#endif +#ifdef NOKERNINFO + {"NOKERNINFO", NOKERNINFO}, #endif {"NOFLSH", NOFLSH}, {"TOSTOP", TOSTOP}, @@ -847,6 +896,9 @@ static struct constant { {"PENDIN", PENDIN}, #endif {"IEXTEN", IEXTEN}, +#ifdef EXTPROC + {"EXTPROC", EXTPROC}, +#endif /* indexes into the control chars array returned by tcgetattr() */ {"VINTR", VINTR}, @@ -855,6 +907,9 @@ static struct constant { {"VKILL", VKILL}, {"VEOF", VEOF}, {"VTIME", VTIME}, +#ifdef VSTATUS + {"VSTATUS", VSTATUS}, +#endif {"VMIN", VMIN}, #ifdef VSWTC /* The #defines above ensure that if either is defined, both are, @@ -865,6 +920,9 @@ static struct constant { {"VSTART", VSTART}, {"VSTOP", VSTOP}, {"VSUSP", VSUSP}, +#ifdef VDSUSP + {"VDSUSP", VDSUSP}, +#endif {"VEOL", VEOL}, #ifdef VREPRINT {"VREPRINT", VREPRINT}, @@ -883,6 +941,18 @@ static struct constant { #endif +#ifdef B7200 + {"B7200", B7200}, +#endif +#ifdef B14400 + {"B14400", B14400}, +#endif +#ifdef B28800 + {"B28800", B28800}, +#endif +#ifdef B76800 + {"B76800", B76800}, +#endif #ifdef B460800 {"B460800", B460800}, #endif @@ -1249,6 +1319,9 @@ static struct constant { #ifdef TIOCTTYGSTRUCT {"TIOCTTYGSTRUCT", TIOCTTYGSTRUCT}, #endif +#ifdef _POSIX_VDISABLE + {"_POSIX_VDISABLE", _POSIX_VDISABLE}, +#endif /* sentinel */ {NULL, 0} @@ -1279,9 +1352,21 @@ termios_exec(PyObject *mod) } while (constant->name != NULL) { - if (PyModule_AddIntConstant( - mod, constant->name, constant->value) < 0) { - return -1; + if (strncmp(constant->name, "TIO", 3) == 0) { + // gh-119770: Convert value to unsigned int for ioctl() constants, + // constants can be negative on macOS whereas ioctl() expects an + // unsigned long 'request'. + unsigned int value = constant->value & UINT_MAX; + if (PyModule_Add(mod, constant->name, + PyLong_FromUnsignedLong(value)) < 0) { + return -1; + } + } + else { + if (PyModule_AddIntConstant( + mod, constant->name, constant->value) < 0) { + return -1; + } } ++constant; } @@ -1291,6 +1376,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 6a872a285d289b..4b10730436c1aa 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_time.h" // _PyTimeFraction #include // clock() #ifdef HAVE_SYS_TIMES_H @@ -63,62 +64,27 @@ #define SEC_TO_NS (1000 * 1000 * 1000) -#if defined(HAVE_TIMES) || defined(HAVE_CLOCK) -static int -check_ticks_per_second(long tps, const char *context) -{ - /* Effectively, check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) - cannot overflow. */ - if (tps >= 0 && (_PyTime_t)tps > _PyTime_MAX / SEC_TO_NS) { - PyErr_Format(PyExc_OverflowError, "%s is too large", context); - return -1; - } - return 0; -} -#endif /* HAVE_TIMES || HAVE_CLOCK */ - -#ifdef HAVE_TIMES - -# define ticks_per_second _PyRuntime.time.ticks_per_second - -static void -ensure_ticks_per_second(void) -{ - if (_PyRuntime.time.ticks_per_second_initialized) { - return; - } - _PyRuntime.time.ticks_per_second_initialized = 1; -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); - if (ticks_per_second < 1) { - ticks_per_second = -1; - } -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif -} - -#endif /* HAVE_TIMES */ - - -PyStatus -_PyTime_Init(void) -{ -#ifdef HAVE_TIMES - ensure_ticks_per_second(); -#endif - return PyStatus_Ok(); -} +/*[clinic input] +module time +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a668a08771581f36]*/ /* Forward declarations */ -static int pysleep(_PyTime_t timeout); +static int pysleep(PyTime_t timeout); typedef struct { PyTypeObject *struct_time_type; +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) + // times() clock frequency in hertz + _PyTimeFraction times_base; +#endif +#ifdef HAVE_CLOCK + // clock() frequency in hertz + _PyTimeFraction clock_base; +#endif } time_module_state; static inline time_module_state* @@ -131,26 +97,18 @@ get_time_state(PyObject *module) static PyObject* -_PyFloat_FromPyTime(_PyTime_t t) +_PyFloat_FromPyTime(PyTime_t t) { - double d = _PyTime_AsSecondsDouble(t); + double d = PyTime_AsSecondsDouble(t); return PyFloat_FromDouble(d); } -static int -get_system_time(_PyTime_t *t) -{ - // Avoid _PyTime_GetSystemClock() which silently ignores errors. - return _PyTime_GetSystemClockWithInfo(t, NULL); -} - - static PyObject * time_time(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -166,11 +124,11 @@ Fractions of a second may be present if the system clock provides them."); static PyObject * time_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(time_ns_doc, @@ -178,7 +136,7 @@ PyDoc_STRVAR(time_ns_doc, \n\ Return the current time in nanoseconds since the Epoch."); -#if defined(HAVE_CLOCK) +#ifdef HAVE_CLOCK #ifndef CLOCKS_PER_SEC # ifdef CLK_TCK @@ -189,15 +147,13 @@ Return the current time in nanoseconds since the Epoch."); #endif static int -_PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +py_clock(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { - if (check_ticks_per_second(CLOCKS_PER_SEC, "CLOCKS_PER_SEC") < 0) { - return -1; - } + _PyTimeFraction *base = &state->clock_base; if (info) { info->implementation = "clock()"; - info->resolution = 1.0 / (double)CLOCKS_PER_SEC; + info->resolution = _PyTimeFraction_Resolution(base); info->monotonic = 1; info->adjustable = 0; } @@ -209,8 +165,7 @@ _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) "or its value cannot be represented"); return -1; } - _PyTime_t ns = _PyTime_MulDiv(ticks, SEC_TO_NS, (_PyTime_t)CLOCKS_PER_SEC); - *tp = _PyTime_FromNanoseconds(ns); + *tp = _PyTimeFraction_Mul(ticks, base); return 0; } #endif /* HAVE_CLOCK */ @@ -227,23 +182,52 @@ _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #pragma clang diagnostic ignored "-Wunguarded-availability" #endif -static PyObject * -time_clock_gettime(PyObject *self, PyObject *args) +static int +time_clockid_converter(PyObject *obj, clockid_t *p) { - int ret; - struct timespec tp; - -#if defined(_AIX) && (SIZEOF_LONG == 8) - long clk_id; - if (!PyArg_ParseTuple(args, "l:clock_gettime", &clk_id)) { +#ifdef _AIX + long long clk_id = PyLong_AsLongLong(obj); #else - int clk_id; - if (!PyArg_ParseTuple(args, "i:clock_gettime", &clk_id)) { + int clk_id = PyLong_AsInt(obj); #endif - return NULL; + if (clk_id == -1 && PyErr_Occurred()) { + PyErr_Format(PyExc_TypeError, + "clk_id should be integer, not %s", + _PyType_Name(Py_TYPE(obj))); + return 0; } - ret = clock_gettime((clockid_t)clk_id, &tp); + // Make sure that we picked the right type (check sizes type) + Py_BUILD_ASSERT(sizeof(clk_id) == sizeof(*p)); + *p = (clockid_t)clk_id; + return 1; +} + +/*[python input] + +class clockid_t_converter(CConverter): + type = "clockid_t" + converter = 'time_clockid_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=53867111501f46c8]*/ + + +/*[clinic input] +time.clock_gettime + + clk_id: clockid_t + / + +Return the time of the specified clock clk_id as a float. +[clinic start generated code]*/ + +static PyObject * +time_clock_gettime_impl(PyObject *module, clockid_t clk_id) +/*[clinic end generated code: output=832b9ebc03328020 input=7e89fcc42ca15e5d]*/ +{ + struct timespec tp; + int ret = clock_gettime(clk_id, &tp); if (ret != 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -251,38 +235,32 @@ time_clock_gettime(PyObject *self, PyObject *args) return PyFloat_FromDouble(tp.tv_sec + tp.tv_nsec * 1e-9); } -PyDoc_STRVAR(clock_gettime_doc, -"clock_gettime(clk_id) -> float\n\ -\n\ -Return the time of the specified clock clk_id."); +/*[clinic input] +time.clock_gettime_ns + + clk_id: clockid_t + / + +Return the time of the specified clock clk_id as nanoseconds (int). +[clinic start generated code]*/ static PyObject * -time_clock_gettime_ns(PyObject *self, PyObject *args) +time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) +/*[clinic end generated code: output=4a045c3a36e60044 input=aabc248db8c8e3e5]*/ { - int ret; - int clk_id; struct timespec ts; - _PyTime_t t; - - if (!PyArg_ParseTuple(args, "i:clock_gettime", &clk_id)) { - return NULL; - } - - ret = clock_gettime((clockid_t)clk_id, &ts); + int ret = clock_gettime(clk_id, &ts); if (ret != 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } + + PyTime_t t; if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } - -PyDoc_STRVAR(clock_gettime_ns_doc, -"clock_gettime_ns(clk_id) -> int\n\ -\n\ -Return the time of the specified clock clk_id as nanoseconds."); #endif /* HAVE_CLOCK_GETTIME */ #ifdef HAVE_CLOCK_SETTIME @@ -291,7 +269,7 @@ time_clock_settime(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec tp; int ret; @@ -322,7 +300,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec ts; int ret; @@ -330,7 +308,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -417,7 +395,7 @@ time_sleep(PyObject *self, PyObject *timeout_obj) return NULL; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; if (timeout < 0) { @@ -484,7 +462,18 @@ tmtotuple(time_module_state *state, struct tm *p if (v == NULL) return NULL; -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, PyLong_FromLong((long) val)) +#define SET_ITEM(INDEX, CALL) \ + do { \ + PyObject *obj = (CALL); \ + if (obj == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, (INDEX), obj); \ + } while (0) + +#define SET(INDEX, VAL) \ + SET_ITEM((INDEX), PyLong_FromLong((long) (VAL))) SET(0, p->tm_year + 1900); SET(1, p->tm_mon + 1); /* Want January == 1 */ @@ -496,19 +485,15 @@ tmtotuple(time_module_state *state, struct tm *p SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); #ifdef HAVE_STRUCT_TM_TM_ZONE - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET_ITEM(9, PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); SET(10, p->tm_gmtoff); #else - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(zone, "surrogateescape")); - PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); + SET_ITEM(9, PyUnicode_DecodeLocale(zone, "surrogateescape")); + SET_ITEM(10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ + #undef SET - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_ITEM return v; } @@ -1170,19 +1155,11 @@ should not be relied on."); #endif /* HAVE_WORKING_TZSET */ -static int -get_monotonic(_PyTime_t *t) -{ - // Avoid _PyTime_GetMonotonicClock() which silently ignores errors. - return _PyTime_GetMonotonicClockWithInfo(t, NULL); -} - - static PyObject * time_monotonic(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1196,11 +1173,11 @@ Monotonic clock, cannot go backward."); static PyObject * time_monotonic_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1209,19 +1186,11 @@ PyDoc_STRVAR(monotonic_ns_doc, Monotonic clock, cannot go backward, as nanoseconds."); -static int -get_perf_counter(_PyTime_t *t) -{ - // Avoid _PyTime_GetPerfCounter() which silently ignores errors. - return _PyTime_GetPerfCounterWithInfo(t, NULL); -} - - static PyObject * time_perf_counter(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1236,11 +1205,11 @@ Performance counter for benchmarking."); static PyObject * time_perf_counter_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1248,14 +1217,45 @@ PyDoc_STRVAR(perf_counter_ns_doc, \n\ Performance counter for benchmarking as nanoseconds."); + +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) static int -_PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +process_time_times(time_module_state *state, PyTime_t *tp, + _Py_clock_info_t *info) +{ + _PyTimeFraction *base = &state->times_base; + + struct tms process; + if (times(&process) == (clock_t)-1) { + return 0; + } + + if (info) { + info->implementation = "times()"; + info->resolution = _PyTimeFraction_Resolution(base); + info->monotonic = 1; + info->adjustable = 0; + } + + PyTime_t ns; + ns = _PyTimeFraction_Mul(process.tms_utime, base); + ns += _PyTimeFraction_Mul(process.tms_stime, base); + *tp = ns; + return 1; +} +#endif + + +static int +py_process_time(time_module_state *state, PyTime_t *tp, + _Py_clock_info_t *info) { #if defined(MS_WINDOWS) HANDLE process; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; process = GetCurrentProcess(); @@ -1282,14 +1282,15 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; #else /* clock_gettime */ +// gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. #if defined(HAVE_CLOCK_GETTIME) \ - && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) + && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ + && !defined(__wasi__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1328,7 +1329,7 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) struct rusage ru; if (getrusage(RUSAGE_SELF, &ru) == 0) { - _PyTime_t utime, stime; + PyTime_t utime, stime; if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; @@ -1344,49 +1345,36 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) return -1; } - _PyTime_t total = utime + stime; + PyTime_t total = utime + stime; *tp = total; return 0; } #endif /* times() */ -#ifdef HAVE_TIMES - struct tms t; - - if (times(&t) != (clock_t)-1) { - assert(_PyRuntime.time.ticks_per_second_initialized); - if (check_ticks_per_second(ticks_per_second, "_SC_CLK_TCK") < 0) { - return -1; - } - if (ticks_per_second != -1) { - if (info) { - info->implementation = "times()"; - info->monotonic = 1; - info->adjustable = 0; - info->resolution = 1.0 / (double)ticks_per_second; - } - - _PyTime_t ns; - ns = _PyTime_MulDiv(t.tms_utime, SEC_TO_NS, ticks_per_second); - ns += _PyTime_MulDiv(t.tms_stime, SEC_TO_NS, ticks_per_second); - *tp = _PyTime_FromNanoseconds(ns); - return 0; - } +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) + int res = process_time_times(state, tp, info); + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; } + // times() failed, ignore failure #endif - /* clock */ - /* Currently, Python 3 requires clock() to build: see issue #22624 */ - return _PyTime_GetClockWithInfo(tp, info); + /* clock(). Python 3 requires clock() to build (see gh-66814) */ + return py_clock(state, tp, info); #endif } static PyObject * -time_process_time(PyObject *self, PyObject *unused) +time_process_time(PyObject *module, PyObject *unused) { - _PyTime_t t; - if (_PyTime_GetProcessTimeWithInfo(&t, NULL) < 0) { + time_module_state *state = get_time_state(module); + PyTime_t t; + if (py_process_time(state, &t, NULL) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1398,13 +1386,14 @@ PyDoc_STRVAR(process_time_doc, Process time for profiling: sum of the kernel and user-space CPU time."); static PyObject * -time_process_time_ns(PyObject *self, PyObject *unused) +time_process_time_ns(PyObject *module, PyObject *unused) { - _PyTime_t t; - if (_PyTime_GetProcessTimeWithInfo(&t, NULL) < 0) { + time_module_state *state = get_time_state(module); + PyTime_t t; + if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1417,12 +1406,12 @@ sum of the kernel and user-space CPU time."); #if defined(MS_WINDOWS) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { HANDLE thread; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; thread = GetCurrentThread(); @@ -1449,15 +1438,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; } #elif defined(_AIX) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-40192: On AIX, thread_cputime() is preferred: it has nanosecond resolution, whereas clock_gettime(CLOCK_THREAD_CPUTIME_ID) @@ -1474,14 +1462,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->adjustable = 0; info->resolution = 1e-9; } - *tp = _PyTime_FromNanoseconds(tc.stime + tc.utime); + *tp = (tc.stime + tc.utime); return 0; } #elif defined(__sun) && defined(__SVR4) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-35455: On Solaris, CLOCK_THREAD_CPUTIME_ID clock is not always available; use gethrvtime() to substitute this functionality. */ @@ -1491,7 +1479,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->monotonic = 1; info->adjustable = 0; } - *tp = _PyTime_FromNanoseconds(gethrvtime()); + *tp = gethrvtime(); return 0; } @@ -1500,9 +1488,9 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) !defined(__EMSCRIPTEN__) && !defined(__wasi__) #define HAVE_THREAD_TIME -#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +#if defined(__APPLE__) && _Py__has_attribute(availability) static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) __attribute__((availability(ios, introduced=10.0))) __attribute__((availability(tvos, introduced=10.0))) @@ -1510,7 +1498,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { struct timespec ts; const clockid_t clk_id = CLOCK_THREAD_CPUTIME_ID; @@ -1552,7 +1540,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) static PyObject * time_thread_time(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } @@ -1567,11 +1555,11 @@ Thread time for profiling: sum of the kernel and user-space CPU time."); static PyObject * time_thread_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(thread_time_ns_doc, @@ -1588,12 +1576,12 @@ sum of the kernel and user-space CPU time."); static PyObject * -time_get_clock_info(PyObject *self, PyObject *args) +time_get_clock_info(PyObject *module, PyObject *args) { char *name; _Py_clock_info_t info; PyObject *obj = NULL, *dict, *ns; - _PyTime_t t; + PyTime_t t; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) { return NULL; @@ -1612,22 +1600,23 @@ time_get_clock_info(PyObject *self, PyObject *args) #endif if (strcmp(name, "time") == 0) { - if (_PyTime_GetSystemClockWithInfo(&t, &info) < 0) { + if (_PyTime_TimeWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "monotonic") == 0) { - if (_PyTime_GetMonotonicClockWithInfo(&t, &info) < 0) { + if (_PyTime_MonotonicWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "perf_counter") == 0) { - if (_PyTime_GetPerfCounterWithInfo(&t, &info) < 0) { + if (_PyTime_PerfCounterWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "process_time") == 0) { - if (_PyTime_GetProcessTimeWithInfo(&t, &info) < 0) { + time_module_state *state = get_time_state(module); + if (py_process_time(state, &t, &info) < 0) { return NULL; } } @@ -1740,6 +1729,12 @@ get_gmtoff(time_t t, struct tm *p) static int init_timezone(PyObject *m) { +#define ADD_INT(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + assert(!PyErr_Occurred()); /* This code moved from PyInit_time wholesale to allow calling it from @@ -1763,13 +1758,13 @@ init_timezone(PyObject *m) #if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); #endif - PyModule_AddIntConstant(m, "timezone", _Py_timezone); + ADD_INT("timezone", _Py_timezone); #ifdef HAVE_ALTZONE - PyModule_AddIntConstant(m, "altzone", altzone); + ADD_INT("altzone", altzone); #else - PyModule_AddIntConstant(m, "altzone", _Py_timezone-3600); + ADD_INT("altzone", _Py_timezone-3600); #endif - PyModule_AddIntConstant(m, "daylight", _Py_daylight); + ADD_INT("daylight", _Py_daylight); #ifdef MS_WINDOWS TIME_ZONE_INFORMATION tzinfo = {0}; GetTimeZoneInformation(&tzinfo); @@ -1828,20 +1823,21 @@ init_timezone(PyObject *m) PyObject *tzname_obj; if (janzone < julyzone) { /* DST is reversed in the southern hemisphere */ - PyModule_AddIntConstant(m, "timezone", julyzone); - PyModule_AddIntConstant(m, "altzone", janzone); - PyModule_AddIntConstant(m, "daylight", janzone != julyzone); + ADD_INT("timezone", julyzone); + ADD_INT("altzone", janzone); + ADD_INT("daylight", janzone != julyzone); tzname_obj = Py_BuildValue("(zz)", julyname, janname); } else { - PyModule_AddIntConstant(m, "timezone", janzone); - PyModule_AddIntConstant(m, "altzone", julyzone); - PyModule_AddIntConstant(m, "daylight", janzone != julyzone); + ADD_INT("timezone", janzone); + ADD_INT("altzone", julyzone); + ADD_INT("daylight", janzone != julyzone); tzname_obj = Py_BuildValue("(zz)", janname, julyname); } if (PyModule_Add(m, "tzname", tzname_obj) < 0) { return -1; } #endif // !HAVE_DECL_TZNAME +#undef ADD_INT if (PyErr_Occurred()) { return -1; @@ -1850,12 +1846,16 @@ init_timezone(PyObject *m) } +// Include Argument Clinic code after defining converters such as +// time_clockid_converter(). +#include "clinic/timemodule.c.h" + static PyMethodDef time_methods[] = { {"time", time_time, METH_NOARGS, time_doc}, {"time_ns", time_time_ns, METH_NOARGS, time_ns_doc}, #ifdef HAVE_CLOCK_GETTIME - {"clock_gettime", time_clock_gettime, METH_VARARGS, clock_gettime_doc}, - {"clock_gettime_ns",time_clock_gettime_ns, METH_VARARGS, clock_gettime_ns_doc}, + TIME_CLOCK_GETTIME_METHODDEF + TIME_CLOCK_GETTIME_NS_METHODDEF #endif #ifdef HAVE_CLOCK_SETTIME {"clock_settime", time_clock_settime, METH_VARARGS, clock_settime_doc}, @@ -1903,8 +1903,8 @@ PyDoc_STRVAR(module_doc, There are two standard representations of time. One is the number\n\ of seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\n\ or a floating point number (to represent fractions of seconds).\n\ -The Epoch is system-defined; on Unix, it is generally January 1st, 1970.\n\ -The actual value can be retrieved by calling gmtime(0).\n\ +The epoch is the point where the time starts, the return value of time.gmtime(0).\n\ +It is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ @@ -1935,20 +1935,20 @@ time_exec(PyObject *module) return -1; } - if (PyDict_DelItemString(dct, "clock_gettime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_getres") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_getres", NULL) < 0) { + return -1; } } #endif @@ -1958,11 +1958,11 @@ time_exec(PyObject *module) } else { PyObject* dct = PyModule_GetDict(module); - if (PyDict_DelItemString(dct, "thread_time") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time_ns", NULL) < 0) { + return -1; } } #endif @@ -1979,20 +1979,16 @@ time_exec(PyObject *module) return -1; } #endif - #ifdef CLOCK_MONOTONIC - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { return -1; } - #endif #ifdef CLOCK_MONOTONIC_RAW if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { return -1; } #endif - #ifdef CLOCK_HIGHRES if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { return -1; @@ -2003,7 +1999,6 @@ time_exec(PyObject *module) return -1; } #endif - #ifdef CLOCK_THREAD_CPUTIME_ID if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { return -1; @@ -2030,10 +2025,19 @@ time_exec(PyObject *module) } #endif #ifdef CLOCK_UPTIME_RAW - if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { return -1; } +#endif +#ifdef CLOCK_MONOTONIC_RAW_APPROX + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW_APPROX) < 0) { + return -1; + } +#endif +#ifdef CLOCK_UPTIME_RAW_APPROX + if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW_APPROX) < 0) { + return -1; + } #endif } @@ -2076,6 +2080,29 @@ time_exec(PyObject *module) } #endif +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) + long ticks_per_second; + if (_Py_GetTicksPerSecond(&ticks_per_second) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot read ticks_per_second"); + return -1; + } + if (_PyTimeFraction_Set(&state->times_base, SEC_TO_NS, + ticks_per_second) < 0) { + PyErr_Format(PyExc_OverflowError, "ticks_per_second is too large"); + return -1; + } +#endif + +#ifdef HAVE_CLOCK + if (_PyTimeFraction_Set(&state->clock_base, SEC_TO_NS, + CLOCKS_PER_SEC) < 0) { + PyErr_Format(PyExc_OverflowError, "CLOCKS_PER_SEC is too large"); + return -1; + } +#endif + return 0; } @@ -2108,6 +2135,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -2134,7 +2162,7 @@ PyInit_time(void) // On error, raise an exception and return -1. // On success, return 0. static int -pysleep(_PyTime_t timeout) +pysleep(PyTime_t timeout) { assert(timeout >= 0); @@ -2146,10 +2174,10 @@ pysleep(_PyTime_t timeout) #else struct timeval timeout_tv; #endif - _PyTime_t deadline, monotonic; + PyTime_t deadline, monotonic; int err = 0; - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } deadline = monotonic + timeout; @@ -2202,7 +2230,7 @@ pysleep(_PyTime_t timeout) } #ifndef HAVE_CLOCK_NANOSLEEP - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } timeout = deadline - monotonic; @@ -2215,7 +2243,7 @@ pysleep(_PyTime_t timeout) return 0; #else // MS_WINDOWS - _PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, + PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, _PyTime_ROUND_CEILING); // Maintain Windows Sleep() semantics for time.sleep(0) diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 875a9c18c4e340..333ffe68a454e4 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -977,21 +977,6 @@ unicodedata_UCD_normalize_impl(PyObject *self, PyObject *form, /* -------------------------------------------------------------------- */ /* database code (cut and pasted from the unidb package) */ -static unsigned long -_gethash(const char *s, int len, int scale) -{ - int i; - unsigned long h = 0; - unsigned long ix; - for (i = 0; i < len; i++) { - h = (h * scale) + (unsigned char) Py_TOUPPER(s[i]); - ix = h & 0xff000000; - if (ix) - h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff; - } - return h; -} - static const char * const hangul_syllables[][3] = { { "G", "A", "" }, { "GG", "AE", "G" }, @@ -1046,6 +1031,247 @@ is_unified_ideograph(Py_UCS4 code) #define IS_NAMED_SEQ(cp) ((cp >= named_sequences_start) && \ (cp < named_sequences_end)) + +// DAWG decoding functions + +static unsigned int +_dawg_decode_varint_unsigned(unsigned int index, unsigned int* result) +{ + unsigned int res = 0; + unsigned int shift = 0; + for (;;) { + unsigned char byte = packed_name_dawg[index]; + res |= (byte & 0x7f) << shift; + index++; + shift += 7; + if (!(byte & 0x80)) { + *result = res; + return index; + } + } +} + +static int +_dawg_match_edge(const char* name, unsigned int namelen, unsigned int size, + unsigned int label_offset, unsigned int namepos) +{ + // This returns 1 if the edge matched, 0 if it didn't (but further edges + // could match) and -1 if the name cannot match at all. + if (size > 1 && namepos + size > namelen) { + return 0; + } + for (unsigned int i = 0; i < size; i++) { + if (packed_name_dawg[label_offset + i] != Py_TOUPPER(name[namepos + i])) { + if (i > 0) { + return -1; // cannot match at all + } + return 0; + } + } + return 1; +} + +// reading DAWG node information: +// a node is encoded by a varint. The lowest bit of that int is set if the node +// is a final, accepting state. The higher bits of that int represent the +// number of names that are encoded by the sub-DAWG started by this node. It's +// used to compute the position of a name. +// +// the starting node of the DAWG is at position 0. +// +// the varint representing a node is followed by the node's edges, the encoding +// is described below + + +static unsigned int +_dawg_decode_node(unsigned int node_offset, bool* final) +{ + unsigned int num; + node_offset = _dawg_decode_varint_unsigned(node_offset, &num); + *final = num & 1; + return node_offset; +} + +static bool +_dawg_node_is_final(unsigned int node_offset) +{ + unsigned int num; + _dawg_decode_varint_unsigned(node_offset, &num); + return num & 1; +} + +static unsigned int +_dawg_node_descendant_count(unsigned int node_offset) +{ + unsigned int num; + _dawg_decode_varint_unsigned(node_offset, &num); + return num >> 1; +} + + +// reading DAWG edge information: +// a DAWG edge is comprised of the following information: +// (1) the size of the label of the string attached to the edge +// (2) the characters of that edge +// (3) the target node +// (4) whether the edge is the last edge in the list of edges following a node +// +// this information is encoded in a compact form as follows: +// +// +---------+-----------------+--------------+-------------------- +// | varint | size (if != 1) | label chars | ... next edge ... +// +---------+-----------------+--------------+-------------------- +// +// - first comes a varint +// - the lowest bit of that varint is whether the edge is final (4) +// - the second lowest bit of that varint is true if the size of +// the length of the label is 1 (1) +// - the rest of the varint is an offset that can be used to compute +// the offset of the target node of that edge (3) +// - if the size is not 1, the first varint is followed by a +// character encoding the number of characters of the label (1) +// (unicode character names aren't larger than 256 bytes, therefore each +// edge label can be at most 256 chars, but is usually smaller) +// - the next size bytes are the characters of the label (2) +// +// the offset of the target node is computed as follows: the number in the +// upper bits of the varint needs to be added to the offset of the target node +// of the previous edge. For the first edge, where there is no previous target +// node, the offset of the first edge is used. +// The intuition here is that edges going out from a node often lead to nodes +// that are close by, leading to small offsets from the current node and thus +// fewer bytes. +// +// There is a special case: if a final node has no outgoing edges, it has to be +// followed by a 0 byte to indicate that there are no edges (because the end of +// the edge list is normally indicated in a bit in the edge encoding). This is +// indicated by _dawg_decode_edge returning -1 + + +static int +_dawg_decode_edge(bool is_first_edge, unsigned int prev_target_node_offset, + unsigned int edge_offset, unsigned int* size, + unsigned int* label_offset, unsigned int* target_node_offset) +{ + unsigned int num; + edge_offset = _dawg_decode_varint_unsigned(edge_offset, &num); + if (num == 0 && is_first_edge) { + return -1; // trying to decode past a final node without outgoing edges + } + bool last_edge = num & 1; + num >>= 1; + bool len_is_one = num & 1; + num >>= 1; + *target_node_offset = prev_target_node_offset + num; + if (len_is_one) { + *size = 1; + } else { + *size = packed_name_dawg[edge_offset++]; + } + *label_offset = edge_offset; + return last_edge; +} + +static int +_lookup_dawg_packed(const char* name, unsigned int namelen) +{ + unsigned int stringpos = 0; + unsigned int node_offset = 0; + unsigned int result = 0; // this is the number of final nodes that we skipped to match name + while (stringpos < namelen) { + bool final; + unsigned int edge_offset = _dawg_decode_node(node_offset, &final); + unsigned int prev_target_node_offset = edge_offset; + bool is_first_edge = true; + for (;;) { + unsigned int size; + unsigned int label_offset, target_node_offset; + int last_edge = _dawg_decode_edge( + is_first_edge, prev_target_node_offset, edge_offset, + &size, &label_offset, &target_node_offset); + if (last_edge == -1) { + return -1; + } + is_first_edge = false; + prev_target_node_offset = target_node_offset; + int matched = _dawg_match_edge(name, namelen, size, label_offset, stringpos); + if (matched == -1) { + return -1; + } + if (matched) { + if (final) + result += 1; + stringpos += size; + node_offset = target_node_offset; + break; + } + if (last_edge) { + return -1; + } + result += _dawg_node_descendant_count(target_node_offset); + edge_offset = label_offset + size; + } + } + if (_dawg_node_is_final(node_offset)) { + return result; + } + return -1; +} + +static int +_inverse_dawg_lookup(char* buffer, unsigned int buflen, unsigned int pos) +{ + unsigned int node_offset = 0; + unsigned int bufpos = 0; + for (;;) { + bool final; + unsigned int edge_offset = _dawg_decode_node(node_offset, &final); + + if (final) { + if (pos == 0) { + if (bufpos + 1 == buflen) { + return 0; + } + buffer[bufpos] = '\0'; + return 1; + } + pos--; + } + unsigned int prev_target_node_offset = edge_offset; + bool is_first_edge = true; + for (;;) { + unsigned int size; + unsigned int label_offset, target_node_offset; + int last_edge = _dawg_decode_edge( + is_first_edge, prev_target_node_offset, edge_offset, + &size, &label_offset, &target_node_offset); + if (last_edge == -1) { + return 0; + } + is_first_edge = false; + prev_target_node_offset = target_node_offset; + + unsigned int descendant_count = _dawg_node_descendant_count(target_node_offset); + if (pos < descendant_count) { + if (bufpos + size >= buflen) { + return 0; // buffer overflow + } + for (unsigned int i = 0; i < size; i++) { + buffer[bufpos++] = packed_name_dawg[label_offset++]; + } + node_offset = target_node_offset; + break; + } else if (!last_edge) { + pos -= descendant_count; + edge_offset = label_offset + size; + } else { + return 0; + } + } + } +} + + static int _getucname(PyObject *self, Py_UCS4 code, char* buffer, int buflen, int with_alias_and_seq) @@ -1054,9 +1280,6 @@ _getucname(PyObject *self, * If with_alias_and_seq is 1, check for names in the Private Use Area 15 * that we are using for aliases and named sequences. */ int offset; - int i; - int word; - const unsigned char* w; if (code >= 0x110000) return 0; @@ -1107,45 +1330,15 @@ _getucname(PyObject *self, return 1; } - /* get offset into phrasebook */ - offset = phrasebook_offset1[(code>>phrasebook_shift)]; - offset = phrasebook_offset2[(offset<>DAWG_CODEPOINT_TO_POS_SHIFT)]; + offset = dawg_codepoint_to_pos_index2[(offset<= 0) { - word = (word << 8) + phrasebook[offset+1]; - offset += 2; - } else - word = phrasebook[offset++]; - if (i) { - if (i > buflen) - return 0; /* buffer overflow */ - buffer[i++] = ' '; - } - /* copy word string from lexicon. the last character in the - word has bit 7 set. the last word in a string ends with - 0x80 */ - w = lexicon + lexicon_offset[word]; - while (*w < 128) { - if (i >= buflen) - return 0; /* buffer overflow */ - buffer[i++] = *w++; - } - if (i >= buflen) - return 0; /* buffer overflow */ - buffer[i++] = *w & 127; - if (*w == 128) - break; /* end of word */ - } - - return 1; + assert(buflen >= 0); + return _inverse_dawg_lookup(buffer, Py_SAFE_DOWNCAST(buflen, int, unsigned int), offset); } static int @@ -1157,21 +1350,6 @@ capi_getucname(Py_UCS4 code, } -static int -_cmpname(PyObject *self, int code, const char* name, int namelen) -{ - /* check if code corresponds to the given name */ - int i; - char buffer[NAME_MAXLEN+1]; - if (!_getucname(self, code, buffer, NAME_MAXLEN, 1)) - return 0; - for (i = 0; i < namelen; i++) { - if (Py_TOUPPER(name[i]) != buffer[i]) - return 0; - } - return buffer[namelen] == '\0'; -} - static void find_syllable(const char *str, int *len, int *pos, int count, int column) { @@ -1193,31 +1371,25 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) } static int -_check_alias_and_seq(unsigned int cp, Py_UCS4* code, int with_named_seq) +_check_alias_and_seq(Py_UCS4* code, int with_named_seq) { /* check if named sequences are allowed */ - if (!with_named_seq && IS_NAMED_SEQ(cp)) + if (!with_named_seq && IS_NAMED_SEQ(*code)) return 0; /* if the code point is in the PUA range that we use for aliases, * convert it to obtain the right code point */ - if (IS_ALIAS(cp)) - *code = name_aliases[cp-aliases_start]; - else - *code = cp; + if (IS_ALIAS(*code)) + *code = name_aliases[*code-aliases_start]; return 1; } + static int -_getcode(PyObject* self, - const char* name, int namelen, Py_UCS4* code, int with_named_seq) +_getcode(const char* name, int namelen, Py_UCS4* code) { /* Return the code point associated with the given name. - * Named aliases are resolved too (unless self != NULL (i.e. we are using - * 3.2.0)). If with_named_seq is 1, returns the PUA code point that we are - * using for the named sequence, and the caller must then convert it. */ - unsigned int h, v; - unsigned int mask = code_size-1; - unsigned int i, incr; + * Named aliases are not resolved, they are returned as a code point in the + * PUA */ /* Check for hangul syllables. */ if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) { @@ -1240,6 +1412,7 @@ _getcode(PyObject* self, /* Check for unified ideographs. */ if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) { /* Four or five hexdigits must follow. */ + unsigned int v; v = 0; name += 22; namelen -= 22; @@ -1261,41 +1434,24 @@ _getcode(PyObject* self, return 1; } - /* the following is the same as python's dictionary lookup, with - only minor changes. see the makeunicodedata script for more - details */ - - h = (unsigned int) _gethash(name, namelen, code_magic); - i = (~h) & mask; - v = code_hash[i]; - if (!v) + assert(namelen >= 0); + int position = _lookup_dawg_packed(name, Py_SAFE_DOWNCAST(namelen, int, unsigned int)); + if (position < 0) { return 0; - if (_cmpname(self, v, name, namelen)) { - return _check_alias_and_seq(v, code, with_named_seq); - } - incr = (h ^ (h >> 3)) & mask; - if (!incr) - incr = mask; - for (;;) { - i = (i + incr) & mask; - v = code_hash[i]; - if (!v) - return 0; - if (_cmpname(self, v, name, namelen)) { - return _check_alias_and_seq(v, code, with_named_seq); - } - incr = incr << 1; - if (incr > mask) - incr = incr ^ code_poly; } + *code = dawg_pos_to_codepoint[position]; + return 1; } + static int capi_getcode(const char* name, int namelen, Py_UCS4* code, int with_named_seq) { - return _getcode(NULL, name, namelen, code, with_named_seq); - + if (!_getcode(name, namelen, code)) { + return 0; + } + return _check_alias_and_seq(code, with_named_seq); } static void @@ -1388,10 +1544,17 @@ unicodedata_UCD_lookup_impl(PyObject *self, const char *name, return NULL; } - if (!_getcode(self, name, (int)name_length, &code, 1)) { + if (!_getcode(name, (int)name_length, &code)) { PyErr_Format(PyExc_KeyError, "undefined character name '%s'", name); return NULL; } + if (UCD_Check(self)) { + /* in 3.2.0 there are no aliases and named sequences */ + if (IS_ALIAS(code) || IS_NAMED_SEQ(code)) { + PyErr_Format(PyExc_KeyError, "undefined character name '%s'", name); + return 0; + } + } /* check if code is in the PUA range that we use for named sequences and convert it */ if (IS_NAMED_SEQ(code)) { @@ -1400,6 +1563,9 @@ unicodedata_UCD_lookup_impl(PyObject *self, const char *name, named_sequences[index].seq, named_sequences[index].seqlen); } + if (IS_ALIAS(code)) { + code = name_aliases[code-aliases_start]; + } return PyUnicode_FromOrdinal(code); } @@ -1502,6 +1668,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index ed4b0eea9a6c59..3e210863448b78 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -1,4 +1,4 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ #define UNIDATA_VERSION "15.1.0" /* a list of unique database records */ diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 1116905308177d..4fe60ad5ed4452 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -1,30130 +1,17171 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ #define NAME_MAXLEN 256 -/* lexicon */ -static const unsigned char lexicon[] = { - 76, 69, 84, 84, 69, 210, 83, 77, 65, 76, 204, 83, 73, 71, 206, 87, 73, - 84, 200, 83, 89, 76, 76, 65, 66, 76, 197, 67, 65, 80, 73, 84, 65, 204, - 72, 73, 69, 82, 79, 71, 76, 89, 80, 200, 76, 65, 84, 73, 206, 65, 82, 65, - 66, 73, 195, 67, 85, 78, 69, 73, 70, 79, 82, 205, 89, 201, 67, 74, 203, - 77, 65, 84, 72, 69, 77, 65, 84, 73, 67, 65, 204, 69, 71, 89, 80, 84, 73, - 65, 206, 67, 79, 77, 80, 65, 84, 73, 66, 73, 76, 73, 84, 217, 83, 89, 77, - 66, 79, 204, 68, 73, 71, 73, 212, 86, 79, 87, 69, 204, 84, 65, 78, 71, - 85, 212, 70, 79, 82, 77, 128, 67, 65, 78, 65, 68, 73, 65, 206, 83, 89, - 76, 76, 65, 66, 73, 67, 211, 65, 78, 196, 83, 73, 71, 78, 87, 82, 73, 84, - 73, 78, 199, 84, 73, 77, 69, 211, 66, 65, 77, 85, 205, 83, 67, 82, 73, - 80, 212, 66, 79, 76, 196, 65, 78, 65, 84, 79, 76, 73, 65, 206, 72, 65, - 78, 71, 85, 204, 78, 85, 77, 66, 69, 210, 76, 73, 78, 69, 65, 210, 67, - 79, 77, 66, 73, 78, 73, 78, 199, 76, 73, 71, 65, 84, 85, 82, 197, 71, 82, - 69, 69, 203, 69, 84, 72, 73, 79, 80, 73, 195, 77, 85, 83, 73, 67, 65, - 204, 67, 89, 82, 73, 76, 76, 73, 195, 70, 79, 210, 75, 72, 73, 84, 65, - 206, 193, 73, 84, 65, 76, 73, 195, 84, 65, 77, 73, 204, 76, 69, 70, 212, - 78, 85, 83, 72, 213, 67, 73, 82, 67, 76, 69, 196, 82, 65, 68, 73, 67, 65, - 204, 83, 65, 78, 83, 45, 83, 69, 82, 73, 198, 82, 73, 71, 72, 212, 83, - 81, 85, 65, 82, 197, 77, 79, 68, 73, 70, 73, 69, 210, 70, 73, 78, 65, - 204, 84, 65, 201, 68, 79, 85, 66, 76, 197, 83, 73, 71, 78, 128, 65, 82, - 82, 79, 87, 128, 65, 66, 79, 86, 69, 128, 86, 65, 201, 66, 69, 76, 79, - 87, 128, 72, 69, 78, 84, 65, 73, 71, 65, 78, 193, 66, 76, 65, 67, 203, - 87, 72, 73, 84, 197, 65, 82, 82, 79, 215, 65, 128, 85, 128, 86, 65, 82, - 73, 65, 84, 73, 79, 206, 73, 128, 66, 82, 65, 73, 76, 76, 197, 80, 65, - 84, 84, 69, 82, 206, 75, 65, 84, 65, 75, 65, 78, 193, 66, 89, 90, 65, 78, - 84, 73, 78, 197, 79, 128, 68, 79, 212, 73, 83, 79, 76, 65, 84, 69, 196, - 77, 65, 82, 75, 128, 194, 79, 198, 77, 89, 65, 78, 77, 65, 210, 86, 69, - 82, 84, 73, 67, 65, 204, 77, 73, 68, 68, 76, 197, 75, 65, 78, 71, 88, - 201, 75, 73, 75, 65, 75, 85, 201, 77, 69, 78, 68, 197, 84, 73, 66, 69, - 84, 65, 206, 77, 65, 82, 203, 72, 69, 65, 86, 217, 73, 78, 73, 84, 73, - 65, 204, 72, 77, 79, 78, 199, 79, 78, 197, 77, 69, 69, 205, 65, 66, 79, - 86, 197, 67, 79, 80, 84, 73, 195, 75, 72, 77, 69, 210, 82, 73, 71, 72, - 84, 87, 65, 82, 68, 211, 90, 78, 65, 77, 69, 78, 78, 217, 67, 65, 82, 82, - 73, 69, 210, 89, 69, 200, 68, 69, 86, 65, 78, 65, 71, 65, 82, 201, 71, - 69, 79, 82, 71, 73, 65, 206, 72, 79, 79, 75, 128, 67, 72, 69, 82, 79, 75, - 69, 197, 77, 79, 78, 71, 79, 76, 73, 65, 206, 84, 87, 207, 83, 84, 82, - 79, 75, 69, 128, 79, 78, 69, 128, 80, 76, 85, 211, 84, 87, 79, 128, 76, - 79, 87, 69, 210, 66, 79, 216, 83, 81, 85, 65, 82, 69, 196, 83, 89, 77, - 66, 79, 76, 128, 80, 72, 65, 83, 69, 45, 197, 84, 72, 82, 69, 197, 85, - 80, 80, 69, 210, 86, 79, 67, 65, 76, 73, 195, 76, 69, 70, 84, 87, 65, 82, - 68, 211, 84, 207, 67, 79, 78, 83, 79, 78, 65, 78, 212, 77, 73, 65, 207, - 68, 82, 65, 87, 73, 78, 71, 211, 84, 73, 76, 197, 68, 85, 80, 76, 79, 89, - 65, 206, 74, 79, 78, 71, 83, 69, 79, 78, 199, 80, 65, 82, 69, 78, 84, 72, - 69, 83, 73, 90, 69, 196, 84, 72, 65, 205, 71, 79, 78, 68, 201, 76, 79, - 215, 65, 76, 69, 198, 72, 65, 76, 198, 85, 208, 70, 79, 85, 82, 128, 71, - 76, 65, 71, 79, 76, 73, 84, 73, 195, 72, 69, 66, 82, 69, 215, 72, 73, 71, - 200, 84, 72, 82, 69, 69, 128, 79, 86, 69, 210, 72, 65, 128, 73, 78, 68, - 69, 216, 77, 65, 76, 65, 89, 65, 76, 65, 205, 83, 73, 89, 65, 209, 68, - 79, 87, 206, 80, 65, 72, 65, 87, 200, 67, 72, 79, 83, 69, 79, 78, 199, - 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 195, 66, 65, 76, 73, 78, 69, 83, - 197, 70, 73, 86, 69, 128, 72, 65, 76, 70, 87, 73, 68, 84, 200, 72, 65, - 78, 68, 45, 70, 73, 83, 212, 77, 69, 82, 79, 73, 84, 73, 195, 84, 85, 82, - 78, 69, 196, 75, 65, 128, 76, 73, 71, 72, 212, 73, 68, 69, 79, 71, 82, - 65, 205, 80, 72, 65, 83, 69, 45, 196, 84, 79, 128, 65, 76, 67, 72, 69, - 77, 73, 67, 65, 204, 78, 69, 85, 77, 197, 66, 82, 65, 72, 77, 201, 84, - 79, 78, 197, 66, 65, 82, 128, 82, 65, 128, 83, 73, 78, 72, 65, 76, 193, - 78, 85, 77, 69, 82, 73, 195, 80, 65, 128, 83, 73, 88, 128, 89, 65, 128, - 69, 73, 71, 72, 84, 128, 76, 65, 128, 77, 65, 128, 83, 69, 86, 69, 78, - 128, 84, 72, 85, 77, 194, 72, 85, 78, 71, 65, 82, 73, 65, 206, 78, 73, - 78, 69, 128, 82, 73, 71, 72, 84, 128, 76, 79, 78, 199, 78, 65, 128, 66, - 65, 82, 194, 72, 65, 200, 66, 76, 79, 67, 203, 68, 79, 84, 211, 78, 79, - 82, 84, 200, 83, 65, 128, 84, 72, 79, 85, 83, 65, 78, 68, 128, 84, 65, - 128, 90, 90, 89, 88, 128, 90, 90, 89, 84, 128, 90, 90, 89, 82, 88, 128, - 90, 90, 89, 82, 128, 90, 90, 89, 80, 128, 90, 90, 89, 65, 128, 90, 90, - 89, 128, 90, 90, 85, 88, 128, 90, 90, 85, 82, 88, 128, 90, 90, 85, 82, - 128, 90, 90, 85, 80, 128, 90, 90, 85, 128, 90, 90, 83, 89, 65, 128, 90, - 90, 83, 65, 128, 90, 90, 79, 88, 128, 90, 90, 79, 80, 128, 90, 90, 79, - 128, 90, 90, 73, 88, 128, 90, 90, 73, 84, 128, 90, 90, 73, 80, 128, 90, - 90, 73, 69, 88, 128, 90, 90, 73, 69, 84, 128, 90, 90, 73, 69, 80, 128, - 90, 90, 73, 69, 128, 90, 90, 73, 128, 90, 90, 69, 88, 128, 90, 90, 69, - 80, 128, 90, 90, 69, 69, 128, 90, 90, 69, 128, 90, 90, 65, 88, 128, 90, - 90, 65, 84, 128, 90, 90, 65, 80, 128, 90, 90, 65, 65, 128, 90, 90, 65, - 128, 90, 89, 71, 79, 83, 128, 90, 87, 83, 80, 128, 90, 87, 78, 74, 128, - 90, 87, 78, 66, 83, 80, 128, 90, 87, 74, 128, 90, 87, 202, 90, 87, 65, - 82, 65, 75, 65, 89, 128, 90, 87, 65, 128, 90, 85, 84, 128, 90, 85, 79, - 88, 128, 90, 85, 79, 80, 128, 90, 85, 79, 128, 90, 85, 77, 128, 90, 85, - 66, 85, 82, 128, 90, 85, 53, 128, 90, 85, 181, 90, 213, 90, 83, 72, 65, - 128, 90, 82, 65, 128, 90, 81, 65, 80, 72, 193, 90, 79, 84, 128, 90, 79, - 79, 128, 90, 79, 77, 66, 73, 69, 128, 90, 79, 65, 128, 90, 77, 69, 89, - 84, 83, 65, 128, 90, 76, 65, 77, 193, 90, 76, 65, 128, 90, 76, 193, 90, - 74, 69, 128, 90, 73, 90, 50, 128, 90, 73, 81, 65, 65, 128, 90, 73, 80, - 80, 69, 82, 45, 77, 79, 85, 84, 200, 90, 73, 78, 79, 82, 128, 90, 73, 76, - 68, 69, 128, 90, 73, 71, 90, 65, 199, 90, 73, 71, 128, 90, 73, 68, 193, - 90, 73, 66, 128, 90, 73, 194, 90, 73, 51, 128, 90, 201, 90, 72, 89, 88, - 128, 90, 72, 89, 84, 128, 90, 72, 89, 82, 88, 128, 90, 72, 89, 82, 128, - 90, 72, 89, 80, 128, 90, 72, 89, 128, 90, 72, 87, 69, 128, 90, 72, 87, - 65, 128, 90, 72, 85, 88, 128, 90, 72, 85, 84, 128, 90, 72, 85, 82, 88, - 128, 90, 72, 85, 82, 128, 90, 72, 85, 80, 128, 90, 72, 85, 79, 88, 128, - 90, 72, 85, 79, 80, 128, 90, 72, 85, 79, 128, 90, 72, 85, 128, 90, 72, - 79, 88, 128, 90, 72, 79, 84, 128, 90, 72, 79, 80, 128, 90, 72, 79, 79, - 128, 90, 72, 79, 73, 128, 90, 72, 79, 128, 90, 72, 73, 86, 69, 84, 69, - 128, 90, 72, 73, 76, 128, 90, 72, 73, 128, 90, 72, 69, 88, 128, 90, 72, - 69, 84, 128, 90, 72, 69, 80, 128, 90, 72, 69, 69, 128, 90, 72, 69, 128, - 90, 72, 197, 90, 72, 65, 89, 73, 78, 128, 90, 72, 65, 88, 128, 90, 72, - 65, 84, 128, 90, 72, 65, 82, 128, 90, 72, 65, 80, 128, 90, 72, 65, 73, - 78, 128, 90, 72, 65, 65, 128, 90, 72, 65, 128, 90, 72, 128, 90, 69, 86, - 79, 75, 128, 90, 69, 85, 83, 128, 90, 69, 84, 65, 128, 90, 69, 82, 79, - 128, 90, 69, 82, 207, 90, 69, 78, 128, 90, 69, 77, 76, 89, 65, 128, 90, - 69, 77, 76, 74, 65, 128, 90, 69, 76, 79, 128, 90, 69, 66, 82, 193, 90, - 69, 50, 128, 90, 197, 90, 65, 89, 78, 128, 90, 65, 89, 73, 78, 45, 89, - 79, 68, 72, 128, 90, 65, 89, 73, 78, 128, 90, 65, 89, 73, 206, 90, 65, - 86, 73, 89, 65, 78, 73, 128, 90, 65, 84, 65, 128, 90, 65, 82, 81, 65, - 128, 90, 65, 82, 76, 128, 90, 65, 81, 69, 198, 90, 65, 80, 89, 65, 84, - 89, 77, 73, 128, 90, 65, 80, 89, 65, 84, 79, 89, 128, 90, 65, 80, 89, 65, - 84, 79, 217, 90, 65, 80, 89, 65, 84, 65, 89, 65, 128, 90, 65, 78, 79, 90, - 72, 69, 75, 128, 90, 65, 78, 65, 66, 65, 90, 65, 210, 90, 65, 77, 88, - 128, 90, 65, 76, 128, 90, 65, 204, 90, 65, 75, 82, 89, 84, 79, 69, 128, - 90, 65, 75, 82, 89, 84, 65, 89, 65, 128, 90, 65, 75, 82, 89, 84, 65, 89, - 193, 90, 65, 73, 78, 128, 90, 65, 73, 206, 90, 65, 73, 128, 90, 65, 72, - 128, 90, 65, 200, 90, 65, 71, 128, 90, 65, 69, 70, 128, 90, 65, 68, 69, - 82, 90, 72, 75, 65, 128, 90, 65, 55, 128, 90, 193, 90, 48, 49, 54, 72, - 128, 90, 48, 49, 54, 71, 128, 90, 48, 49, 54, 70, 128, 90, 48, 49, 54, - 69, 128, 90, 48, 49, 54, 68, 128, 90, 48, 49, 54, 67, 128, 90, 48, 49, - 54, 66, 128, 90, 48, 49, 54, 65, 128, 90, 48, 49, 54, 128, 90, 48, 49, - 53, 73, 128, 90, 48, 49, 53, 72, 128, 90, 48, 49, 53, 71, 128, 90, 48, - 49, 53, 70, 128, 90, 48, 49, 53, 69, 128, 90, 48, 49, 53, 68, 128, 90, - 48, 49, 53, 67, 128, 90, 48, 49, 53, 66, 128, 90, 48, 49, 53, 65, 128, - 90, 48, 49, 53, 128, 90, 48, 49, 52, 128, 90, 48, 49, 51, 128, 90, 48, - 49, 50, 128, 90, 48, 49, 49, 128, 90, 48, 49, 48, 128, 90, 48, 48, 57, - 128, 90, 48, 48, 56, 128, 90, 48, 48, 55, 128, 90, 48, 48, 54, 128, 90, - 48, 48, 53, 65, 128, 90, 48, 48, 53, 128, 90, 48, 48, 52, 65, 128, 90, - 48, 48, 52, 128, 90, 48, 48, 51, 66, 128, 90, 48, 48, 51, 65, 128, 90, - 48, 48, 51, 128, 90, 48, 48, 50, 68, 128, 90, 48, 48, 50, 67, 128, 90, - 48, 48, 50, 66, 128, 90, 48, 48, 50, 65, 128, 90, 48, 48, 50, 128, 90, - 48, 48, 49, 128, 90, 128, 218, 89, 89, 88, 128, 89, 89, 84, 128, 89, 89, - 82, 88, 128, 89, 89, 82, 128, 89, 89, 80, 128, 89, 89, 69, 128, 89, 89, - 65, 65, 128, 89, 89, 65, 128, 89, 89, 128, 89, 87, 79, 79, 128, 89, 87, - 79, 128, 89, 87, 73, 73, 128, 89, 87, 73, 128, 89, 87, 69, 128, 89, 87, - 65, 65, 128, 89, 87, 65, 128, 89, 86, 128, 89, 85, 88, 128, 89, 85, 87, - 79, 81, 128, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 78, 84, 85, 128, 89, - 85, 85, 128, 89, 85, 84, 128, 89, 85, 83, 128, 89, 85, 211, 89, 85, 82, - 88, 128, 89, 85, 82, 128, 89, 85, 81, 128, 89, 85, 209, 89, 85, 80, 128, - 89, 85, 79, 88, 128, 89, 85, 79, 84, 128, 89, 85, 79, 80, 128, 89, 85, - 79, 77, 128, 89, 85, 79, 128, 89, 85, 78, 128, 89, 85, 77, 128, 89, 85, - 74, 128, 89, 85, 73, 128, 89, 85, 69, 81, 128, 89, 85, 69, 128, 89, 85, - 68, 72, 128, 89, 85, 68, 200, 89, 85, 65, 78, 128, 89, 85, 65, 69, 78, - 128, 89, 85, 45, 89, 69, 79, 128, 89, 85, 45, 89, 69, 128, 89, 85, 45, - 85, 128, 89, 85, 45, 79, 128, 89, 85, 45, 73, 128, 89, 85, 45, 69, 79, - 128, 89, 85, 45, 69, 128, 89, 85, 45, 65, 69, 128, 89, 85, 45, 65, 128, - 89, 85, 45, 52, 128, 89, 85, 45, 51, 128, 89, 85, 45, 50, 128, 89, 85, - 45, 49, 128, 89, 85, 128, 89, 213, 89, 82, 89, 128, 89, 80, 83, 73, 76, - 73, 128, 89, 80, 79, 82, 82, 79, 73, 128, 89, 80, 79, 75, 82, 73, 83, 73, - 83, 128, 89, 80, 79, 75, 82, 73, 83, 73, 211, 89, 80, 79, 71, 69, 71, 82, - 65, 77, 77, 69, 78, 73, 128, 89, 79, 89, 128, 89, 79, 88, 128, 89, 79, - 87, 68, 128, 89, 79, 85, 84, 72, 70, 85, 76, 78, 69, 83, 83, 128, 89, 79, - 85, 84, 72, 70, 85, 204, 89, 79, 213, 89, 79, 84, 128, 89, 79, 212, 89, - 79, 82, 73, 128, 89, 79, 81, 128, 89, 79, 209, 89, 79, 80, 128, 89, 79, - 79, 128, 89, 79, 77, 79, 128, 89, 79, 71, 72, 128, 89, 79, 68, 128, 89, - 79, 196, 89, 79, 65, 128, 89, 79, 45, 89, 79, 128, 89, 79, 45, 89, 69, - 79, 128, 89, 79, 45, 89, 65, 69, 128, 89, 79, 45, 89, 65, 128, 89, 79, - 45, 79, 128, 89, 79, 45, 73, 128, 89, 79, 45, 69, 79, 128, 89, 79, 45, - 65, 69, 128, 89, 79, 45, 65, 128, 89, 79, 45, 54, 128, 89, 79, 45, 53, - 128, 89, 79, 45, 52, 128, 89, 79, 45, 51, 128, 89, 79, 45, 50, 128, 89, - 79, 45, 49, 128, 89, 207, 89, 73, 90, 69, 84, 128, 89, 73, 88, 128, 89, - 73, 87, 78, 128, 89, 73, 84, 128, 89, 73, 80, 128, 89, 73, 78, 71, 128, - 89, 73, 73, 128, 89, 73, 72, 128, 89, 73, 199, 89, 73, 69, 88, 128, 89, - 73, 69, 84, 128, 89, 73, 69, 80, 128, 89, 73, 69, 69, 128, 89, 73, 69, - 128, 89, 73, 68, 68, 73, 83, 200, 89, 73, 45, 85, 128, 89, 73, 128, 89, - 72, 69, 128, 89, 72, 65, 128, 89, 70, 69, 83, 73, 83, 128, 89, 70, 69, - 83, 73, 211, 89, 70, 69, 206, 89, 69, 90, 73, 68, 201, 89, 69, 89, 128, - 89, 69, 87, 128, 89, 69, 85, 88, 128, 89, 69, 85, 82, 65, 69, 128, 89, - 69, 85, 81, 128, 89, 69, 85, 77, 128, 89, 69, 85, 65, 69, 84, 128, 89, - 69, 85, 65, 69, 128, 89, 69, 84, 73, 86, 128, 89, 69, 83, 84, 85, 128, - 89, 69, 83, 73, 69, 85, 78, 71, 45, 83, 83, 65, 78, 71, 75, 73, 89, 69, - 79, 75, 128, 89, 69, 83, 73, 69, 85, 78, 71, 45, 83, 73, 79, 83, 128, 89, - 69, 83, 73, 69, 85, 78, 71, 45, 80, 65, 78, 83, 73, 79, 83, 128, 89, 69, - 83, 73, 69, 85, 78, 71, 45, 77, 73, 69, 85, 77, 128, 89, 69, 83, 73, 69, - 85, 78, 71, 45, 75, 73, 89, 69, 79, 75, 128, 89, 69, 83, 73, 69, 85, 78, - 71, 45, 75, 72, 73, 69, 85, 75, 72, 128, 89, 69, 83, 73, 69, 85, 78, 71, - 45, 72, 73, 69, 85, 72, 128, 89, 69, 83, 73, 69, 85, 78, 71, 128, 89, 69, - 82, 85, 128, 89, 69, 82, 213, 89, 69, 82, 73, 128, 89, 69, 82, 65, 200, - 89, 69, 82, 128, 89, 69, 79, 82, 73, 78, 72, 73, 69, 85, 72, 128, 89, 69, - 79, 45, 89, 65, 128, 89, 69, 79, 45, 85, 128, 89, 69, 79, 45, 79, 128, - 89, 69, 78, 73, 83, 69, 201, 89, 69, 78, 65, 80, 128, 89, 69, 78, 128, - 89, 69, 206, 89, 69, 76, 76, 79, 87, 128, 89, 69, 76, 76, 79, 215, 89, - 69, 73, 78, 128, 89, 69, 72, 128, 89, 69, 69, 71, 128, 89, 69, 69, 128, - 89, 69, 65, 210, 89, 69, 65, 128, 89, 65, 90, 90, 128, 89, 65, 90, 72, - 128, 89, 65, 90, 128, 89, 65, 89, 68, 128, 89, 65, 89, 65, 78, 78, 65, - 128, 89, 65, 89, 128, 89, 65, 87, 78, 73, 78, 199, 89, 65, 87, 78, 128, - 89, 65, 87, 128, 89, 65, 86, 128, 89, 65, 85, 128, 89, 65, 84, 84, 128, - 89, 65, 84, 73, 128, 89, 65, 84, 72, 128, 89, 65, 84, 128, 89, 65, 83, - 83, 128, 89, 65, 83, 72, 128, 89, 65, 83, 128, 89, 65, 82, 82, 128, 89, - 65, 82, 78, 128, 89, 65, 82, 128, 89, 65, 210, 89, 65, 81, 128, 89, 65, - 80, 128, 89, 65, 78, 83, 65, 89, 65, 128, 89, 65, 78, 71, 128, 89, 65, - 78, 199, 89, 65, 78, 128, 89, 65, 77, 79, 75, 128, 89, 65, 77, 65, 75, - 75, 65, 78, 128, 89, 65, 77, 128, 89, 65, 76, 128, 89, 65, 75, 72, 72, - 128, 89, 65, 75, 72, 128, 89, 65, 75, 65, 83, 72, 128, 89, 65, 75, 128, - 89, 65, 74, 85, 82, 86, 69, 68, 73, 195, 89, 65, 74, 128, 89, 65, 73, - 128, 89, 65, 72, 72, 128, 89, 65, 72, 128, 89, 65, 71, 78, 128, 89, 65, - 71, 72, 72, 128, 89, 65, 71, 72, 128, 89, 65, 71, 128, 89, 65, 70, 213, - 89, 65, 70, 128, 89, 65, 69, 77, 77, 65, 69, 128, 89, 65, 68, 72, 128, - 89, 65, 68, 68, 72, 128, 89, 65, 68, 68, 128, 89, 65, 68, 128, 89, 65, - 67, 72, 128, 89, 65, 66, 72, 128, 89, 65, 66, 128, 89, 65, 65, 82, 85, - 128, 89, 65, 65, 73, 128, 89, 65, 65, 68, 79, 128, 89, 65, 45, 89, 79, - 128, 89, 65, 45, 85, 128, 89, 65, 45, 79, 128, 89, 65, 45, 53, 128, 89, - 65, 45, 52, 128, 89, 65, 45, 51, 128, 89, 65, 45, 50, 128, 89, 65, 45, - 49, 128, 89, 48, 48, 56, 128, 89, 48, 48, 55, 128, 89, 48, 48, 54, 128, - 89, 48, 48, 53, 128, 89, 48, 48, 52, 128, 89, 48, 48, 51, 128, 89, 48, - 48, 50, 128, 89, 48, 48, 49, 65, 128, 89, 48, 48, 49, 128, 89, 45, 67, - 82, 69, 197, 88, 89, 88, 128, 88, 89, 85, 128, 88, 89, 84, 128, 88, 89, - 82, 88, 128, 88, 89, 82, 128, 88, 89, 80, 128, 88, 89, 79, 79, 74, 128, - 88, 89, 79, 79, 128, 88, 89, 79, 128, 88, 89, 73, 128, 88, 89, 69, 69, - 205, 88, 89, 69, 69, 128, 88, 89, 69, 128, 88, 89, 65, 65, 128, 88, 89, - 65, 128, 88, 89, 128, 88, 87, 73, 128, 88, 87, 69, 69, 128, 88, 87, 69, - 128, 88, 87, 65, 65, 128, 88, 87, 65, 128, 88, 87, 128, 88, 215, 88, 86, - 69, 128, 88, 86, 65, 128, 88, 85, 79, 88, 128, 88, 85, 79, 128, 88, 85, - 128, 88, 83, 72, 65, 65, 89, 65, 84, 72, 73, 89, 65, 128, 88, 79, 88, - 128, 88, 79, 84, 128, 88, 79, 82, 128, 88, 79, 80, 72, 128, 88, 79, 80, - 128, 88, 79, 65, 128, 88, 79, 128, 88, 73, 88, 128, 88, 73, 84, 128, 88, - 73, 82, 79, 206, 88, 73, 80, 128, 88, 73, 69, 88, 128, 88, 73, 69, 84, - 128, 88, 73, 69, 80, 128, 88, 73, 69, 128, 88, 73, 65, 78, 71, 81, 201, - 88, 73, 65, 66, 128, 88, 73, 128, 88, 72, 69, 89, 78, 128, 88, 71, 128, - 88, 69, 89, 78, 128, 88, 69, 83, 84, 69, 211, 88, 69, 72, 128, 88, 69, - 69, 128, 88, 69, 128, 88, 65, 85, 83, 128, 88, 65, 85, 128, 88, 65, 80, - 72, 128, 88, 65, 78, 128, 88, 65, 65, 128, 88, 65, 128, 88, 48, 48, 56, - 65, 128, 88, 48, 48, 56, 128, 88, 48, 48, 55, 128, 88, 48, 48, 54, 65, - 128, 88, 48, 48, 54, 128, 88, 48, 48, 53, 128, 88, 48, 48, 52, 66, 128, - 88, 48, 48, 52, 65, 128, 88, 48, 48, 52, 128, 88, 48, 48, 51, 128, 88, - 48, 48, 50, 128, 88, 48, 48, 49, 128, 88, 45, 216, 88, 45, 82, 65, 89, - 128, 87, 90, 128, 87, 89, 78, 78, 128, 87, 89, 78, 206, 87, 86, 73, 128, - 87, 86, 69, 128, 87, 86, 65, 128, 87, 86, 128, 87, 85, 80, 128, 87, 85, - 79, 88, 128, 87, 85, 79, 80, 128, 87, 85, 79, 128, 87, 85, 78, 74, 207, - 87, 85, 78, 128, 87, 85, 76, 85, 128, 87, 85, 76, 213, 87, 85, 73, 128, - 87, 85, 69, 128, 87, 85, 65, 69, 84, 128, 87, 85, 65, 69, 78, 128, 87, - 85, 128, 87, 82, 217, 87, 82, 79, 78, 71, 128, 87, 82, 73, 83, 212, 87, - 82, 73, 78, 75, 76, 69, 83, 128, 87, 82, 73, 78, 75, 76, 69, 211, 87, 82, - 73, 78, 75, 76, 69, 68, 128, 87, 82, 69, 83, 84, 76, 69, 82, 83, 128, 87, - 82, 69, 78, 67, 72, 128, 87, 82, 69, 65, 84, 200, 87, 82, 65, 80, 80, 69, - 196, 87, 82, 65, 80, 128, 87, 79, 88, 128, 87, 79, 87, 128, 87, 79, 82, - 83, 72, 73, 80, 128, 87, 79, 82, 82, 73, 69, 196, 87, 79, 82, 77, 128, - 87, 79, 82, 76, 196, 87, 79, 82, 75, 69, 82, 128, 87, 79, 82, 75, 128, - 87, 79, 82, 203, 87, 79, 82, 68, 83, 80, 65, 67, 69, 128, 87, 79, 82, - 196, 87, 79, 80, 128, 87, 79, 79, 78, 128, 87, 79, 79, 76, 128, 87, 79, - 79, 68, 83, 45, 67, 82, 69, 197, 87, 79, 79, 68, 128, 87, 79, 78, 128, - 87, 79, 206, 87, 79, 77, 69, 78, 211, 87, 79, 77, 69, 206, 87, 79, 77, - 65, 78, 211, 87, 79, 77, 65, 78, 128, 87, 79, 77, 65, 206, 87, 79, 76, - 79, 83, 79, 128, 87, 79, 76, 198, 87, 79, 69, 128, 87, 79, 65, 128, 87, - 79, 45, 55, 128, 87, 79, 45, 54, 128, 87, 79, 45, 53, 128, 87, 79, 45, - 52, 128, 87, 79, 45, 51, 128, 87, 79, 45, 50, 128, 87, 79, 45, 49, 128, - 87, 73, 84, 72, 79, 85, 212, 87, 73, 84, 72, 73, 78, 128, 87, 73, 84, 72, - 73, 206, 87, 73, 82, 69, 76, 69, 83, 83, 128, 87, 73, 82, 69, 196, 87, - 73, 78, 84, 69, 82, 128, 87, 73, 78, 75, 73, 78, 199, 87, 73, 78, 75, - 128, 87, 73, 78, 74, 65, 128, 87, 73, 78, 71, 83, 128, 87, 73, 78, 71, - 128, 87, 73, 78, 69, 128, 87, 73, 78, 197, 87, 73, 78, 68, 85, 128, 87, - 73, 78, 68, 79, 87, 128, 87, 73, 78, 68, 128, 87, 73, 78, 196, 87, 73, - 78, 128, 87, 73, 76, 84, 69, 196, 87, 73, 71, 78, 89, 65, 78, 128, 87, - 73, 71, 71, 76, 217, 87, 73, 71, 71, 76, 69, 83, 128, 87, 73, 68, 84, 72, - 128, 87, 73, 68, 69, 78, 73, 78, 199, 87, 73, 68, 69, 45, 72, 69, 65, 68, - 69, 196, 87, 73, 68, 197, 87, 73, 65, 78, 71, 87, 65, 65, 75, 128, 87, - 73, 65, 78, 71, 128, 87, 73, 45, 53, 128, 87, 73, 45, 52, 128, 87, 73, - 45, 51, 128, 87, 73, 45, 50, 128, 87, 73, 45, 49, 128, 87, 72, 79, 76, - 197, 87, 72, 73, 84, 69, 45, 70, 69, 65, 84, 72, 69, 82, 69, 196, 87, 72, - 73, 84, 69, 128, 87, 72, 69, 69, 76, 69, 196, 87, 72, 69, 69, 76, 67, 72, - 65, 73, 82, 128, 87, 72, 69, 69, 76, 67, 72, 65, 73, 210, 87, 72, 69, 69, - 76, 128, 87, 72, 69, 69, 204, 87, 72, 69, 65, 84, 128, 87, 72, 65, 76, - 69, 128, 87, 72, 128, 87, 71, 128, 87, 69, 88, 128, 87, 69, 85, 88, 128, - 87, 69, 212, 87, 69, 83, 84, 69, 82, 206, 87, 69, 83, 84, 45, 67, 82, 69, - 197, 87, 69, 83, 84, 128, 87, 69, 83, 212, 87, 69, 80, 128, 87, 69, 79, - 128, 87, 69, 78, 128, 87, 69, 76, 76, 128, 87, 69, 73, 71, 72, 212, 87, - 69, 73, 69, 82, 83, 84, 82, 65, 83, 211, 87, 69, 73, 128, 87, 69, 69, 78, - 128, 87, 69, 68, 71, 69, 45, 84, 65, 73, 76, 69, 196, 87, 69, 68, 71, 69, - 128, 87, 69, 68, 68, 73, 78, 71, 128, 87, 69, 66, 128, 87, 69, 65, 82, - 217, 87, 69, 65, 80, 79, 78, 128, 87, 69, 45, 52, 128, 87, 69, 45, 51, - 128, 87, 69, 45, 50, 128, 87, 69, 45, 49, 128, 87, 67, 128, 87, 66, 128, - 87, 65, 89, 128, 87, 65, 217, 87, 65, 88, 73, 78, 199, 87, 65, 88, 128, - 87, 65, 87, 45, 65, 89, 73, 78, 45, 82, 69, 83, 72, 128, 87, 65, 87, 128, - 87, 65, 215, 87, 65, 86, 217, 87, 65, 86, 73, 78, 199, 87, 65, 86, 69, - 83, 128, 87, 65, 86, 69, 128, 87, 65, 86, 197, 87, 65, 85, 128, 87, 65, - 84, 84, 79, 128, 87, 65, 84, 69, 82, 77, 69, 76, 79, 78, 128, 87, 65, 84, - 69, 82, 128, 87, 65, 84, 69, 210, 87, 65, 84, 67, 72, 128, 87, 65, 84, - 128, 87, 65, 83, 84, 73, 78, 71, 128, 87, 65, 83, 84, 69, 66, 65, 83, 75, - 69, 84, 128, 87, 65, 83, 83, 65, 76, 76, 65, 77, 128, 87, 65, 83, 76, 65, - 128, 87, 65, 83, 76, 193, 87, 65, 83, 65, 76, 76, 65, 77, 128, 87, 65, - 83, 65, 76, 76, 65, 205, 87, 65, 83, 45, 83, 65, 76, 65, 65, 77, 128, 87, - 65, 82, 78, 73, 78, 199, 87, 65, 82, 65, 78, 199, 87, 65, 81, 70, 65, - 128, 87, 65, 80, 128, 87, 65, 78, 73, 78, 199, 87, 65, 78, 71, 75, 85, - 79, 81, 128, 87, 65, 78, 68, 69, 82, 69, 82, 128, 87, 65, 78, 68, 128, - 87, 65, 78, 67, 72, 207, 87, 65, 78, 128, 87, 65, 76, 76, 80, 76, 65, 78, - 197, 87, 65, 76, 76, 69, 196, 87, 65, 76, 76, 128, 87, 65, 76, 204, 87, - 65, 76, 75, 128, 87, 65, 76, 203, 87, 65, 73, 84, 73, 78, 71, 128, 87, - 65, 73, 83, 84, 128, 87, 65, 73, 128, 87, 65, 70, 70, 76, 69, 128, 87, - 65, 69, 78, 128, 87, 65, 69, 128, 87, 65, 68, 68, 65, 128, 87, 65, 65, - 86, 85, 128, 87, 65, 65, 74, 73, 66, 128, 87, 65, 65, 65, 76, 73, 72, 69, - 197, 87, 65, 45, 84, 65, 65, 65, 76, 65, 65, 128, 87, 65, 45, 83, 65, 76, - 76, 65, 77, 128, 87, 65, 45, 65, 65, 76, 73, 72, 128, 87, 65, 45, 53, - 128, 87, 65, 45, 52, 128, 87, 65, 45, 51, 128, 87, 65, 45, 50, 128, 87, - 65, 45, 49, 128, 87, 193, 87, 48, 50, 53, 128, 87, 48, 50, 52, 65, 128, - 87, 48, 50, 52, 128, 87, 48, 50, 51, 128, 87, 48, 50, 50, 128, 87, 48, - 50, 49, 128, 87, 48, 50, 48, 128, 87, 48, 49, 57, 128, 87, 48, 49, 56, - 65, 128, 87, 48, 49, 56, 128, 87, 48, 49, 55, 65, 128, 87, 48, 49, 55, - 128, 87, 48, 49, 54, 128, 87, 48, 49, 53, 128, 87, 48, 49, 52, 65, 128, - 87, 48, 49, 52, 128, 87, 48, 49, 51, 128, 87, 48, 49, 50, 128, 87, 48, - 49, 49, 128, 87, 48, 49, 48, 65, 128, 87, 48, 49, 48, 128, 87, 48, 48, - 57, 65, 128, 87, 48, 48, 57, 128, 87, 48, 48, 56, 128, 87, 48, 48, 55, - 128, 87, 48, 48, 54, 128, 87, 48, 48, 53, 128, 87, 48, 48, 52, 128, 87, - 48, 48, 51, 65, 128, 87, 48, 48, 51, 128, 87, 48, 48, 50, 128, 87, 48, - 48, 49, 128, 86, 90, 77, 69, 84, 128, 86, 90, 128, 86, 89, 88, 128, 86, - 89, 84, 128, 86, 89, 83, 79, 75, 79, 128, 86, 89, 83, 79, 75, 207, 86, - 89, 82, 88, 128, 86, 89, 82, 128, 86, 89, 80, 128, 86, 89, 128, 86, 88, - 128, 86, 87, 74, 128, 86, 87, 65, 128, 86, 87, 128, 86, 85, 88, 128, 86, - 85, 85, 128, 86, 85, 84, 128, 86, 85, 82, 88, 128, 86, 85, 82, 128, 86, - 85, 80, 128, 86, 85, 76, 71, 65, 210, 86, 85, 76, 67, 65, 78, 85, 83, - 128, 86, 85, 69, 81, 128, 86, 84, 83, 128, 86, 84, 128, 86, 83, 57, 57, - 128, 86, 83, 57, 56, 128, 86, 83, 57, 55, 128, 86, 83, 57, 54, 128, 86, - 83, 57, 53, 128, 86, 83, 57, 52, 128, 86, 83, 57, 51, 128, 86, 83, 57, - 50, 128, 86, 83, 57, 49, 128, 86, 83, 57, 48, 128, 86, 83, 57, 128, 86, - 83, 56, 57, 128, 86, 83, 56, 56, 128, 86, 83, 56, 55, 128, 86, 83, 56, - 54, 128, 86, 83, 56, 53, 128, 86, 83, 56, 52, 128, 86, 83, 56, 51, 128, - 86, 83, 56, 50, 128, 86, 83, 56, 49, 128, 86, 83, 56, 48, 128, 86, 83, - 56, 128, 86, 83, 55, 57, 128, 86, 83, 55, 56, 128, 86, 83, 55, 55, 128, - 86, 83, 55, 54, 128, 86, 83, 55, 53, 128, 86, 83, 55, 52, 128, 86, 83, - 55, 51, 128, 86, 83, 55, 50, 128, 86, 83, 55, 49, 128, 86, 83, 55, 48, - 128, 86, 83, 55, 128, 86, 83, 54, 57, 128, 86, 83, 54, 56, 128, 86, 83, - 54, 55, 128, 86, 83, 54, 54, 128, 86, 83, 54, 53, 128, 86, 83, 54, 52, - 128, 86, 83, 54, 51, 128, 86, 83, 54, 50, 128, 86, 83, 54, 49, 128, 86, - 83, 54, 48, 128, 86, 83, 54, 128, 86, 83, 53, 57, 128, 86, 83, 53, 56, - 128, 86, 83, 53, 55, 128, 86, 83, 53, 54, 128, 86, 83, 53, 53, 128, 86, - 83, 53, 52, 128, 86, 83, 53, 51, 128, 86, 83, 53, 50, 128, 86, 83, 53, - 49, 128, 86, 83, 53, 48, 128, 86, 83, 53, 128, 86, 83, 52, 57, 128, 86, - 83, 52, 56, 128, 86, 83, 52, 55, 128, 86, 83, 52, 54, 128, 86, 83, 52, - 53, 128, 86, 83, 52, 52, 128, 86, 83, 52, 51, 128, 86, 83, 52, 50, 128, - 86, 83, 52, 49, 128, 86, 83, 52, 48, 128, 86, 83, 52, 128, 86, 83, 51, - 57, 128, 86, 83, 51, 56, 128, 86, 83, 51, 55, 128, 86, 83, 51, 54, 128, - 86, 83, 51, 53, 128, 86, 83, 51, 52, 128, 86, 83, 51, 51, 128, 86, 83, - 51, 50, 128, 86, 83, 51, 49, 128, 86, 83, 51, 48, 128, 86, 83, 51, 128, - 86, 83, 50, 57, 128, 86, 83, 50, 56, 128, 86, 83, 50, 55, 128, 86, 83, - 50, 54, 128, 86, 83, 50, 53, 54, 128, 86, 83, 50, 53, 53, 128, 86, 83, - 50, 53, 52, 128, 86, 83, 50, 53, 51, 128, 86, 83, 50, 53, 50, 128, 86, - 83, 50, 53, 49, 128, 86, 83, 50, 53, 48, 128, 86, 83, 50, 53, 128, 86, - 83, 50, 52, 57, 128, 86, 83, 50, 52, 56, 128, 86, 83, 50, 52, 55, 128, - 86, 83, 50, 52, 54, 128, 86, 83, 50, 52, 53, 128, 86, 83, 50, 52, 52, - 128, 86, 83, 50, 52, 51, 128, 86, 83, 50, 52, 50, 128, 86, 83, 50, 52, - 49, 128, 86, 83, 50, 52, 48, 128, 86, 83, 50, 52, 128, 86, 83, 50, 51, - 57, 128, 86, 83, 50, 51, 56, 128, 86, 83, 50, 51, 55, 128, 86, 83, 50, - 51, 54, 128, 86, 83, 50, 51, 53, 128, 86, 83, 50, 51, 52, 128, 86, 83, - 50, 51, 51, 128, 86, 83, 50, 51, 50, 128, 86, 83, 50, 51, 49, 128, 86, - 83, 50, 51, 48, 128, 86, 83, 50, 51, 128, 86, 83, 50, 50, 57, 128, 86, - 83, 50, 50, 56, 128, 86, 83, 50, 50, 55, 128, 86, 83, 50, 50, 54, 128, - 86, 83, 50, 50, 53, 128, 86, 83, 50, 50, 52, 128, 86, 83, 50, 50, 51, - 128, 86, 83, 50, 50, 50, 128, 86, 83, 50, 50, 49, 128, 86, 83, 50, 50, - 48, 128, 86, 83, 50, 50, 128, 86, 83, 50, 49, 57, 128, 86, 83, 50, 49, - 56, 128, 86, 83, 50, 49, 55, 128, 86, 83, 50, 49, 54, 128, 86, 83, 50, - 49, 53, 128, 86, 83, 50, 49, 52, 128, 86, 83, 50, 49, 51, 128, 86, 83, - 50, 49, 50, 128, 86, 83, 50, 49, 49, 128, 86, 83, 50, 49, 48, 128, 86, - 83, 50, 49, 128, 86, 83, 50, 48, 57, 128, 86, 83, 50, 48, 56, 128, 86, - 83, 50, 48, 55, 128, 86, 83, 50, 48, 54, 128, 86, 83, 50, 48, 53, 128, - 86, 83, 50, 48, 52, 128, 86, 83, 50, 48, 51, 128, 86, 83, 50, 48, 50, - 128, 86, 83, 50, 48, 49, 128, 86, 83, 50, 48, 48, 128, 86, 83, 50, 48, - 128, 86, 83, 50, 128, 86, 83, 49, 57, 57, 128, 86, 83, 49, 57, 56, 128, - 86, 83, 49, 57, 55, 128, 86, 83, 49, 57, 54, 128, 86, 83, 49, 57, 53, - 128, 86, 83, 49, 57, 52, 128, 86, 83, 49, 57, 51, 128, 86, 83, 49, 57, - 50, 128, 86, 83, 49, 57, 49, 128, 86, 83, 49, 57, 48, 128, 86, 83, 49, - 57, 128, 86, 83, 49, 56, 57, 128, 86, 83, 49, 56, 56, 128, 86, 83, 49, - 56, 55, 128, 86, 83, 49, 56, 54, 128, 86, 83, 49, 56, 53, 128, 86, 83, - 49, 56, 52, 128, 86, 83, 49, 56, 51, 128, 86, 83, 49, 56, 50, 128, 86, - 83, 49, 56, 49, 128, 86, 83, 49, 56, 48, 128, 86, 83, 49, 56, 128, 86, - 83, 49, 55, 57, 128, 86, 83, 49, 55, 56, 128, 86, 83, 49, 55, 55, 128, - 86, 83, 49, 55, 54, 128, 86, 83, 49, 55, 53, 128, 86, 83, 49, 55, 52, - 128, 86, 83, 49, 55, 51, 128, 86, 83, 49, 55, 50, 128, 86, 83, 49, 55, - 49, 128, 86, 83, 49, 55, 48, 128, 86, 83, 49, 55, 128, 86, 83, 49, 54, - 57, 128, 86, 83, 49, 54, 56, 128, 86, 83, 49, 54, 55, 128, 86, 83, 49, - 54, 54, 128, 86, 83, 49, 54, 53, 128, 86, 83, 49, 54, 52, 128, 86, 83, - 49, 54, 51, 128, 86, 83, 49, 54, 50, 128, 86, 83, 49, 54, 49, 128, 86, - 83, 49, 54, 48, 128, 86, 83, 49, 54, 128, 86, 83, 49, 53, 57, 128, 86, - 83, 49, 53, 56, 128, 86, 83, 49, 53, 55, 128, 86, 83, 49, 53, 54, 128, - 86, 83, 49, 53, 53, 128, 86, 83, 49, 53, 52, 128, 86, 83, 49, 53, 51, - 128, 86, 83, 49, 53, 50, 128, 86, 83, 49, 53, 49, 128, 86, 83, 49, 53, - 48, 128, 86, 83, 49, 53, 128, 86, 83, 49, 52, 57, 128, 86, 83, 49, 52, - 56, 128, 86, 83, 49, 52, 55, 128, 86, 83, 49, 52, 54, 128, 86, 83, 49, - 52, 53, 128, 86, 83, 49, 52, 52, 128, 86, 83, 49, 52, 51, 128, 86, 83, - 49, 52, 50, 128, 86, 83, 49, 52, 49, 128, 86, 83, 49, 52, 48, 128, 86, - 83, 49, 52, 128, 86, 83, 49, 51, 57, 128, 86, 83, 49, 51, 56, 128, 86, - 83, 49, 51, 55, 128, 86, 83, 49, 51, 54, 128, 86, 83, 49, 51, 53, 128, - 86, 83, 49, 51, 52, 128, 86, 83, 49, 51, 51, 128, 86, 83, 49, 51, 50, - 128, 86, 83, 49, 51, 49, 128, 86, 83, 49, 51, 48, 128, 86, 83, 49, 51, - 128, 86, 83, 49, 50, 57, 128, 86, 83, 49, 50, 56, 128, 86, 83, 49, 50, - 55, 128, 86, 83, 49, 50, 54, 128, 86, 83, 49, 50, 53, 128, 86, 83, 49, - 50, 52, 128, 86, 83, 49, 50, 51, 128, 86, 83, 49, 50, 50, 128, 86, 83, - 49, 50, 49, 128, 86, 83, 49, 50, 48, 128, 86, 83, 49, 50, 128, 86, 83, - 49, 49, 57, 128, 86, 83, 49, 49, 56, 128, 86, 83, 49, 49, 55, 128, 86, - 83, 49, 49, 54, 128, 86, 83, 49, 49, 53, 128, 86, 83, 49, 49, 52, 128, - 86, 83, 49, 49, 51, 128, 86, 83, 49, 49, 50, 128, 86, 83, 49, 49, 49, - 128, 86, 83, 49, 49, 48, 128, 86, 83, 49, 49, 128, 86, 83, 49, 48, 57, - 128, 86, 83, 49, 48, 56, 128, 86, 83, 49, 48, 55, 128, 86, 83, 49, 48, - 54, 128, 86, 83, 49, 48, 53, 128, 86, 83, 49, 48, 52, 128, 86, 83, 49, - 48, 51, 128, 86, 83, 49, 48, 50, 128, 86, 83, 49, 48, 49, 128, 86, 83, - 49, 48, 48, 128, 86, 83, 49, 48, 128, 86, 83, 49, 128, 86, 83, 128, 86, - 82, 65, 75, 72, 73, 89, 193, 86, 82, 65, 67, 72, 89, 128, 86, 81, 128, - 86, 79, 88, 128, 86, 79, 87, 69, 76, 45, 67, 65, 82, 82, 73, 69, 210, 86, - 79, 87, 128, 86, 79, 85, 128, 86, 79, 84, 128, 86, 79, 211, 86, 79, 80, - 128, 86, 79, 79, 73, 128, 86, 79, 79, 128, 86, 79, 77, 73, 84, 73, 78, - 71, 128, 86, 79, 77, 128, 86, 79, 76, 85, 77, 197, 86, 79, 76, 84, 65, - 71, 197, 86, 79, 76, 76, 69, 89, 66, 65, 76, 76, 128, 86, 79, 76, 67, 65, - 78, 79, 128, 86, 79, 76, 65, 80, 85, 203, 86, 79, 73, 68, 69, 196, 86, - 79, 73, 196, 86, 79, 73, 67, 73, 78, 71, 128, 86, 79, 73, 67, 69, 76, 69, - 83, 211, 86, 79, 73, 67, 69, 196, 86, 79, 68, 128, 86, 79, 67, 65, 76, - 73, 90, 65, 84, 73, 79, 206, 86, 79, 67, 65, 204, 86, 79, 128, 86, 73, - 89, 79, 128, 86, 73, 88, 128, 86, 73, 84, 82, 73, 79, 76, 45, 50, 128, - 86, 73, 84, 82, 73, 79, 76, 128, 86, 73, 84, 72, 75, 85, 81, 201, 86, 73, - 84, 65, 69, 45, 50, 128, 86, 73, 84, 65, 69, 128, 86, 73, 84, 128, 86, - 73, 83, 73, 71, 79, 84, 72, 73, 195, 86, 73, 83, 65, 82, 71, 65, 89, 65, - 128, 86, 73, 83, 65, 82, 71, 65, 128, 86, 73, 83, 65, 82, 71, 193, 86, - 73, 82, 73, 65, 77, 128, 86, 73, 82, 71, 79, 128, 86, 73, 82, 71, 65, - 128, 86, 73, 82, 65, 77, 65, 128, 86, 73, 80, 128, 86, 73, 79, 76, 73, - 78, 128, 86, 73, 78, 69, 71, 65, 82, 45, 51, 128, 86, 73, 78, 69, 71, 65, - 82, 45, 50, 128, 86, 73, 78, 69, 71, 65, 82, 128, 86, 73, 78, 69, 71, 65, - 210, 86, 73, 78, 69, 128, 86, 73, 78, 197, 86, 73, 78, 128, 86, 73, 76, - 76, 65, 71, 69, 128, 86, 73, 73, 128, 86, 73, 71, 73, 78, 84, 73, 76, 69, - 128, 86, 73, 69, 88, 128, 86, 73, 69, 87, 73, 78, 199, 86, 73, 69, 87, - 69, 82, 128, 86, 73, 69, 87, 68, 65, 84, 193, 86, 73, 69, 84, 78, 65, 77, - 69, 83, 197, 86, 73, 69, 84, 128, 86, 73, 69, 212, 86, 73, 69, 80, 128, - 86, 73, 69, 128, 86, 73, 68, 74, 45, 50, 128, 86, 73, 68, 74, 128, 86, - 73, 68, 69, 79, 67, 65, 83, 83, 69, 84, 84, 69, 128, 86, 73, 68, 69, 207, - 86, 73, 68, 65, 128, 86, 73, 67, 84, 79, 82, 217, 86, 73, 66, 82, 65, 84, - 73, 79, 206, 86, 72, 65, 128, 86, 70, 65, 128, 86, 69, 89, 90, 128, 86, - 69, 88, 128, 86, 69, 87, 128, 86, 69, 215, 86, 69, 85, 88, 128, 86, 69, - 85, 77, 128, 86, 69, 85, 65, 69, 80, 69, 78, 128, 86, 69, 85, 65, 69, - 128, 86, 69, 83, 84, 65, 128, 86, 69, 83, 84, 128, 86, 69, 83, 83, 69, - 204, 86, 69, 82, 217, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, 128, 86, - 69, 82, 84, 73, 67, 65, 76, 76, 217, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 54, 45, 48, 54, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 54, 45, - 48, 53, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 54, 45, 48, 52, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 54, 45, 48, 51, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 54, 45, 48, 50, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 54, 45, 48, 49, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 54, 45, 48, 48, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 53, 45, - 48, 54, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 53, 45, 48, 53, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 53, 45, 48, 52, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 53, 45, 48, 51, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 53, 45, 48, 50, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 53, 45, 48, 49, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 53, 45, - 48, 48, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 52, 45, 48, 54, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 52, 45, 48, 53, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 52, 45, 48, 52, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 52, 45, 48, 51, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 52, 45, 48, 50, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 52, 45, - 48, 49, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 52, 45, 48, 48, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 51, 45, 48, 54, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 51, 45, 48, 53, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 51, 45, 48, 52, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 51, 45, 48, 51, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 51, 45, - 48, 50, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 51, 45, 48, 49, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 51, 45, 48, 48, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 50, 45, 48, 54, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 50, 45, 48, 53, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 50, 45, 48, 52, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 50, 45, - 48, 51, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 50, 45, 48, 50, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 50, 45, 48, 49, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 50, 45, 48, 48, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 49, 45, 48, 54, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 49, 45, 48, 53, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 49, 45, - 48, 52, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 49, 45, 48, 51, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 49, 45, 48, 50, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 49, 45, 48, 49, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 49, 45, 48, 48, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 48, 45, 48, 54, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 48, 45, - 48, 53, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 48, 45, 48, 52, 128, - 86, 69, 82, 84, 73, 67, 65, 76, 45, 48, 48, 45, 48, 51, 128, 86, 69, 82, - 84, 73, 67, 65, 76, 45, 48, 48, 45, 48, 50, 128, 86, 69, 82, 84, 73, 67, - 65, 76, 45, 48, 48, 45, 48, 49, 128, 86, 69, 82, 84, 73, 67, 65, 76, 45, - 48, 48, 45, 48, 48, 128, 86, 69, 82, 84, 73, 67, 65, 76, 128, 86, 69, 82, - 83, 73, 67, 76, 69, 128, 86, 69, 82, 83, 197, 86, 69, 82, 71, 69, 128, - 86, 69, 82, 68, 73, 71, 82, 73, 83, 128, 86, 69, 82, 128, 86, 69, 80, - 128, 86, 69, 78, 68, 128, 86, 69, 76, 73, 128, 86, 69, 73, 76, 128, 86, - 69, 72, 73, 67, 76, 69, 128, 86, 69, 72, 128, 86, 69, 200, 86, 69, 69, - 128, 86, 69, 197, 86, 69, 68, 69, 128, 86, 69, 67, 84, 79, 210, 86, 67, - 128, 86, 65, 89, 65, 78, 78, 65, 128, 86, 65, 88, 128, 86, 65, 86, 128, - 86, 65, 214, 86, 65, 85, 128, 86, 65, 84, 72, 89, 128, 86, 65, 84, 128, - 86, 65, 83, 84, 78, 69, 83, 211, 86, 65, 83, 73, 83, 128, 86, 65, 82, 89, - 211, 86, 65, 82, 73, 75, 65, 128, 86, 65, 82, 73, 65, 78, 84, 128, 86, - 65, 82, 73, 65, 78, 212, 86, 65, 82, 73, 65, 128, 86, 65, 82, 73, 193, - 86, 65, 82, 69, 73, 65, 201, 86, 65, 82, 69, 73, 193, 86, 65, 82, 65, 65, - 75, 65, 78, 128, 86, 65, 80, 79, 85, 82, 83, 128, 86, 65, 80, 128, 86, - 65, 78, 69, 128, 86, 65, 77, 80, 73, 82, 69, 128, 86, 65, 77, 65, 71, 79, - 77, 85, 75, 72, 65, 128, 86, 65, 77, 65, 71, 79, 77, 85, 75, 72, 193, 86, - 65, 76, 76, 69, 89, 128, 86, 65, 75, 65, 73, 89, 65, 82, 65, 65, 128, 86, - 65, 74, 128, 86, 65, 73, 128, 86, 65, 72, 128, 86, 65, 200, 86, 65, 65, - 86, 85, 128, 86, 65, 65, 128, 86, 193, 86, 48, 52, 48, 65, 128, 86, 48, - 52, 48, 128, 86, 48, 51, 57, 128, 86, 48, 51, 56, 128, 86, 48, 51, 55, - 65, 128, 86, 48, 51, 55, 128, 86, 48, 51, 54, 128, 86, 48, 51, 53, 128, - 86, 48, 51, 52, 128, 86, 48, 51, 51, 65, 128, 86, 48, 51, 51, 128, 86, - 48, 51, 50, 128, 86, 48, 51, 49, 65, 128, 86, 48, 51, 49, 128, 86, 48, - 51, 48, 65, 128, 86, 48, 51, 48, 128, 86, 48, 50, 57, 65, 128, 86, 48, - 50, 57, 128, 86, 48, 50, 56, 65, 128, 86, 48, 50, 56, 128, 86, 48, 50, - 55, 128, 86, 48, 50, 54, 128, 86, 48, 50, 53, 128, 86, 48, 50, 52, 128, - 86, 48, 50, 51, 65, 128, 86, 48, 50, 51, 128, 86, 48, 50, 50, 128, 86, - 48, 50, 49, 128, 86, 48, 50, 48, 76, 128, 86, 48, 50, 48, 75, 128, 86, - 48, 50, 48, 74, 128, 86, 48, 50, 48, 73, 128, 86, 48, 50, 48, 72, 128, - 86, 48, 50, 48, 71, 128, 86, 48, 50, 48, 70, 128, 86, 48, 50, 48, 69, - 128, 86, 48, 50, 48, 68, 128, 86, 48, 50, 48, 67, 128, 86, 48, 50, 48, - 66, 128, 86, 48, 50, 48, 65, 128, 86, 48, 50, 48, 128, 86, 48, 49, 57, - 128, 86, 48, 49, 56, 128, 86, 48, 49, 55, 128, 86, 48, 49, 54, 128, 86, - 48, 49, 53, 128, 86, 48, 49, 52, 128, 86, 48, 49, 51, 128, 86, 48, 49, - 50, 66, 128, 86, 48, 49, 50, 65, 128, 86, 48, 49, 50, 128, 86, 48, 49, - 49, 68, 128, 86, 48, 49, 49, 67, 128, 86, 48, 49, 49, 66, 128, 86, 48, - 49, 49, 65, 128, 86, 48, 49, 49, 128, 86, 48, 49, 48, 128, 86, 48, 48, - 57, 128, 86, 48, 48, 56, 128, 86, 48, 48, 55, 66, 128, 86, 48, 48, 55, - 65, 128, 86, 48, 48, 55, 128, 86, 48, 48, 54, 128, 86, 48, 48, 53, 128, - 86, 48, 48, 52, 128, 86, 48, 48, 51, 128, 86, 48, 48, 50, 65, 128, 86, - 48, 48, 50, 128, 86, 48, 48, 49, 73, 128, 86, 48, 48, 49, 72, 128, 86, - 48, 48, 49, 71, 128, 86, 48, 48, 49, 70, 128, 86, 48, 48, 49, 69, 128, - 86, 48, 48, 49, 68, 128, 86, 48, 48, 49, 67, 128, 86, 48, 48, 49, 66, - 128, 86, 48, 48, 49, 65, 128, 86, 48, 48, 49, 128, 85, 90, 85, 128, 85, - 90, 72, 65, 75, 75, 85, 128, 85, 90, 51, 128, 85, 90, 179, 85, 90, 128, - 85, 89, 71, 72, 85, 210, 85, 89, 65, 78, 78, 65, 128, 85, 89, 128, 85, - 87, 85, 128, 85, 85, 89, 65, 78, 78, 65, 128, 85, 85, 85, 85, 128, 85, - 85, 85, 51, 128, 85, 85, 85, 50, 128, 85, 85, 69, 128, 85, 84, 85, 75, - 73, 128, 85, 83, 83, 85, 51, 128, 85, 83, 83, 85, 128, 85, 83, 72, 88, - 128, 85, 83, 72, 85, 77, 88, 128, 85, 83, 72, 69, 78, 78, 65, 128, 85, - 83, 72, 50, 128, 85, 83, 72, 128, 85, 83, 200, 85, 83, 69, 196, 85, 83, - 69, 45, 50, 128, 85, 83, 69, 45, 49, 128, 85, 83, 69, 128, 85, 83, 197, - 85, 82, 85, 218, 85, 82, 85, 83, 128, 85, 82, 85, 68, 65, 128, 85, 82, - 85, 68, 193, 85, 82, 85, 128, 85, 82, 213, 85, 82, 78, 128, 85, 82, 73, - 78, 69, 128, 85, 82, 73, 51, 128, 85, 82, 73, 128, 85, 82, 65, 78, 85, - 83, 128, 85, 82, 65, 128, 85, 82, 52, 128, 85, 82, 50, 128, 85, 82, 178, - 85, 80, 87, 65, 82, 68, 83, 128, 85, 80, 87, 65, 82, 68, 211, 85, 80, 87, - 65, 82, 68, 128, 85, 80, 87, 65, 82, 196, 85, 80, 84, 85, 82, 78, 128, - 85, 80, 83, 73, 76, 79, 78, 128, 85, 80, 83, 73, 76, 79, 206, 85, 80, 83, - 73, 68, 69, 45, 68, 79, 87, 206, 85, 80, 82, 73, 71, 72, 212, 85, 80, 80, - 69, 82, 128, 85, 80, 65, 68, 72, 77, 65, 78, 73, 89, 65, 128, 85, 80, 45, - 80, 79, 73, 78, 84, 73, 78, 199, 85, 79, 78, 128, 85, 79, 71, 128, 85, - 78, 78, 128, 85, 78, 77, 65, 82, 82, 73, 69, 196, 85, 78, 75, 78, 79, 87, - 78, 128, 85, 78, 75, 128, 85, 78, 73, 86, 69, 82, 83, 65, 204, 85, 78, - 73, 84, 89, 128, 85, 78, 73, 84, 69, 196, 85, 78, 73, 84, 128, 85, 78, - 73, 212, 85, 78, 73, 79, 78, 128, 85, 78, 73, 79, 206, 85, 78, 73, 70, - 79, 82, 77, 128, 85, 78, 73, 70, 73, 69, 196, 85, 78, 73, 67, 79, 82, - 206, 85, 78, 69, 86, 69, 206, 85, 78, 68, 207, 85, 78, 68, 69, 82, 84, - 73, 69, 128, 85, 78, 68, 69, 82, 76, 73, 78, 197, 85, 78, 68, 69, 82, 68, - 79, 84, 128, 85, 78, 68, 69, 82, 66, 65, 82, 128, 85, 78, 68, 69, 82, - 128, 85, 78, 68, 69, 210, 85, 78, 67, 73, 193, 85, 78, 67, 69, 82, 84, - 65, 73, 78, 84, 217, 85, 78, 66, 76, 69, 78, 68, 69, 196, 85, 78, 65, 83, - 80, 73, 82, 65, 84, 69, 68, 128, 85, 78, 65, 80, 128, 85, 78, 65, 77, 85, - 83, 69, 196, 85, 78, 65, 128, 85, 206, 85, 77, 85, 77, 128, 85, 77, 85, - 205, 85, 77, 66, 82, 69, 76, 76, 65, 128, 85, 77, 66, 82, 69, 76, 76, - 193, 85, 77, 66, 73, 78, 128, 85, 75, 85, 128, 85, 75, 82, 65, 73, 78, - 73, 65, 206, 85, 75, 65, 82, 65, 128, 85, 75, 65, 82, 193, 85, 75, 128, - 85, 73, 90, 128, 85, 73, 88, 128, 85, 73, 85, 90, 128, 85, 73, 85, 88, - 128, 85, 73, 85, 81, 128, 85, 73, 85, 67, 128, 85, 73, 81, 128, 85, 73, - 76, 76, 69, 65, 78, 78, 128, 85, 73, 71, 72, 85, 210, 85, 73, 67, 128, - 85, 72, 68, 128, 85, 71, 65, 82, 73, 84, 73, 195, 85, 69, 90, 128, 85, - 69, 89, 128, 85, 69, 88, 128, 85, 69, 78, 128, 85, 69, 73, 128, 85, 69, - 69, 128, 85, 69, 67, 128, 85, 69, 65, 128, 85, 68, 85, 71, 128, 85, 68, - 65, 84, 84, 65, 128, 85, 68, 65, 84, 84, 193, 85, 68, 65, 82, 75, 65, - 128, 85, 68, 65, 65, 84, 128, 85, 68, 128, 85, 196, 85, 66, 85, 70, 73, - 76, 73, 128, 85, 66, 72, 65, 89, 65, 84, 207, 85, 66, 65, 68, 65, 77, 65, - 128, 85, 66, 128, 85, 65, 84, 72, 128, 85, 65, 78, 71, 128, 85, 65, 128, - 85, 178, 85, 48, 52, 50, 128, 85, 48, 52, 49, 128, 85, 48, 52, 48, 128, - 85, 48, 51, 57, 128, 85, 48, 51, 56, 128, 85, 48, 51, 55, 128, 85, 48, - 51, 54, 128, 85, 48, 51, 53, 128, 85, 48, 51, 52, 128, 85, 48, 51, 51, - 128, 85, 48, 51, 50, 65, 128, 85, 48, 51, 50, 128, 85, 48, 51, 49, 128, - 85, 48, 51, 48, 128, 85, 48, 50, 57, 65, 128, 85, 48, 50, 57, 128, 85, - 48, 50, 56, 128, 85, 48, 50, 55, 128, 85, 48, 50, 54, 128, 85, 48, 50, - 53, 128, 85, 48, 50, 52, 128, 85, 48, 50, 51, 65, 128, 85, 48, 50, 51, - 128, 85, 48, 50, 50, 128, 85, 48, 50, 49, 128, 85, 48, 50, 48, 128, 85, - 48, 49, 57, 128, 85, 48, 49, 56, 128, 85, 48, 49, 55, 128, 85, 48, 49, - 54, 128, 85, 48, 49, 53, 128, 85, 48, 49, 52, 128, 85, 48, 49, 51, 128, - 85, 48, 49, 50, 128, 85, 48, 49, 49, 128, 85, 48, 49, 48, 128, 85, 48, - 48, 57, 128, 85, 48, 48, 56, 128, 85, 48, 48, 55, 128, 85, 48, 48, 54, - 66, 128, 85, 48, 48, 54, 65, 128, 85, 48, 48, 54, 128, 85, 48, 48, 53, - 128, 85, 48, 48, 52, 128, 85, 48, 48, 51, 128, 85, 48, 48, 50, 128, 85, - 48, 48, 49, 128, 85, 45, 83, 72, 65, 80, 69, 196, 85, 45, 73, 45, 73, - 128, 85, 45, 69, 79, 45, 69, 85, 128, 85, 45, 66, 82, 74, 71, 85, 128, - 85, 45, 53, 128, 84, 90, 85, 128, 84, 90, 79, 65, 128, 84, 90, 79, 128, - 84, 90, 73, 210, 84, 90, 73, 128, 84, 90, 69, 69, 128, 84, 90, 69, 128, - 84, 90, 65, 65, 128, 84, 90, 65, 128, 84, 90, 128, 84, 89, 210, 84, 89, - 80, 69, 45, 183, 84, 89, 80, 69, 45, 54, 128, 84, 89, 80, 69, 45, 182, - 84, 89, 80, 69, 45, 53, 128, 84, 89, 80, 69, 45, 181, 84, 89, 80, 69, 45, - 52, 128, 84, 89, 80, 69, 45, 180, 84, 89, 80, 69, 45, 51, 128, 84, 89, - 80, 69, 45, 179, 84, 89, 80, 69, 45, 178, 84, 89, 80, 69, 45, 49, 45, 50, - 128, 84, 89, 80, 69, 45, 177, 84, 89, 80, 197, 84, 89, 79, 128, 84, 89, - 73, 128, 84, 89, 69, 128, 84, 89, 65, 89, 128, 84, 89, 65, 128, 84, 88, - 87, 86, 128, 84, 88, 87, 214, 84, 88, 72, 69, 69, 202, 84, 88, 65, 128, - 84, 87, 79, 79, 128, 84, 87, 79, 45, 87, 65, 217, 84, 87, 79, 45, 84, 72, - 73, 82, 84, 89, 128, 84, 87, 79, 45, 76, 73, 78, 197, 84, 87, 79, 45, 72, - 69, 65, 68, 69, 196, 84, 87, 79, 45, 69, 205, 84, 87, 79, 45, 67, 73, 82, - 67, 76, 197, 84, 87, 73, 83, 84, 73, 78, 71, 128, 84, 87, 73, 83, 84, 69, - 196, 84, 87, 73, 73, 128, 84, 87, 73, 128, 84, 87, 69, 78, 84, 89, 45, - 84, 87, 79, 128, 84, 87, 69, 78, 84, 89, 45, 84, 87, 207, 84, 87, 69, 78, - 84, 89, 45, 84, 72, 82, 69, 69, 128, 84, 87, 69, 78, 84, 89, 45, 83, 73, - 88, 128, 84, 87, 69, 78, 84, 89, 45, 83, 69, 86, 69, 78, 128, 84, 87, 69, - 78, 84, 89, 45, 79, 78, 69, 128, 84, 87, 69, 78, 84, 89, 45, 78, 73, 78, - 69, 128, 84, 87, 69, 78, 84, 89, 45, 70, 79, 85, 82, 128, 84, 87, 69, 78, - 84, 89, 45, 70, 73, 86, 69, 128, 84, 87, 69, 78, 84, 89, 45, 70, 73, 86, - 197, 84, 87, 69, 78, 84, 89, 45, 69, 73, 71, 72, 84, 200, 84, 87, 69, 78, - 84, 89, 45, 69, 73, 71, 72, 84, 128, 84, 87, 69, 78, 84, 89, 128, 84, 87, - 69, 78, 84, 217, 84, 87, 69, 78, 84, 73, 69, 84, 72, 83, 128, 84, 87, 69, - 78, 84, 73, 69, 84, 72, 128, 84, 87, 69, 76, 86, 69, 45, 84, 72, 73, 82, - 84, 89, 128, 84, 87, 69, 76, 86, 69, 128, 84, 87, 69, 76, 86, 197, 84, - 87, 69, 76, 70, 84, 72, 83, 128, 84, 87, 69, 76, 70, 84, 72, 128, 84, 87, - 69, 128, 84, 87, 65, 65, 128, 84, 87, 65, 128, 84, 86, 82, 73, 68, 79, - 128, 84, 86, 73, 77, 65, 68, 85, 210, 84, 85, 88, 69, 68, 79, 128, 84, - 85, 88, 128, 84, 85, 85, 77, 85, 128, 84, 85, 85, 128, 84, 85, 84, 84, - 89, 128, 84, 85, 84, 69, 89, 65, 83, 65, 84, 128, 84, 85, 84, 128, 84, - 85, 82, 88, 128, 84, 85, 82, 85, 128, 84, 85, 82, 84, 76, 69, 128, 84, - 85, 82, 79, 50, 128, 84, 85, 82, 78, 83, 84, 73, 76, 69, 128, 84, 85, 82, - 206, 84, 85, 82, 75, 73, 83, 200, 84, 85, 82, 75, 73, 195, 84, 85, 82, - 75, 69, 89, 128, 84, 85, 82, 66, 65, 78, 128, 84, 85, 82, 128, 84, 85, - 210, 84, 85, 80, 78, 73, 128, 84, 85, 80, 128, 84, 85, 79, 88, 128, 84, - 85, 79, 84, 128, 84, 85, 79, 80, 128, 84, 85, 79, 128, 84, 85, 78, 78, - 89, 128, 84, 85, 77, 69, 84, 69, 83, 128, 84, 85, 77, 66, 76, 69, 210, - 84, 85, 77, 65, 69, 128, 84, 85, 77, 128, 84, 85, 205, 84, 85, 76, 73, - 80, 128, 84, 85, 75, 87, 69, 78, 84, 73, 83, 128, 84, 85, 75, 128, 84, - 85, 71, 82, 73, 203, 84, 85, 71, 50, 128, 84, 85, 71, 178, 84, 85, 66, - 69, 128, 84, 85, 66, 128, 84, 85, 65, 82, 69, 199, 84, 85, 65, 69, 80, - 128, 84, 85, 65, 69, 128, 84, 85, 45, 84, 79, 128, 84, 85, 45, 52, 128, - 84, 85, 45, 51, 128, 84, 85, 45, 50, 128, 84, 85, 45, 49, 128, 84, 213, - 84, 84, 85, 85, 128, 84, 84, 85, 68, 68, 65, 71, 128, 84, 84, 85, 68, 68, - 65, 65, 71, 128, 84, 84, 85, 128, 84, 84, 84, 72, 65, 128, 84, 84, 84, - 65, 128, 84, 84, 83, 85, 128, 84, 84, 83, 79, 128, 84, 84, 83, 73, 128, - 84, 84, 83, 69, 69, 128, 84, 84, 83, 69, 128, 84, 84, 83, 65, 128, 84, - 84, 79, 79, 128, 84, 84, 73, 73, 128, 84, 84, 73, 128, 84, 84, 72, 87, - 69, 128, 84, 84, 72, 85, 128, 84, 84, 72, 79, 79, 128, 84, 84, 72, 79, - 128, 84, 84, 72, 73, 128, 84, 84, 72, 69, 69, 128, 84, 84, 72, 69, 128, - 84, 84, 72, 65, 65, 128, 84, 84, 72, 128, 84, 84, 69, 72, 69, 72, 128, - 84, 84, 69, 72, 69, 200, 84, 84, 69, 72, 128, 84, 84, 69, 200, 84, 84, - 69, 69, 128, 84, 84, 65, 89, 65, 78, 78, 65, 128, 84, 84, 65, 85, 128, - 84, 84, 65, 73, 128, 84, 84, 65, 65, 128, 84, 84, 50, 128, 84, 83, 87, - 69, 128, 84, 83, 87, 66, 128, 84, 83, 87, 65, 128, 84, 83, 86, 128, 84, - 83, 83, 69, 128, 84, 83, 83, 65, 128, 84, 83, 79, 214, 84, 83, 73, 85, - 128, 84, 83, 72, 85, 71, 83, 128, 84, 83, 72, 79, 79, 75, 128, 84, 83, - 72, 79, 79, 203, 84, 83, 72, 79, 79, 74, 128, 84, 83, 72, 69, 83, 128, - 84, 83, 72, 69, 71, 128, 84, 83, 72, 69, 199, 84, 83, 72, 69, 69, 74, - 128, 84, 83, 72, 69, 128, 84, 83, 72, 65, 194, 84, 83, 72, 65, 128, 84, - 83, 69, 82, 69, 128, 84, 83, 69, 69, 66, 128, 84, 83, 65, 84, 193, 84, - 83, 65, 68, 73, 128, 84, 83, 65, 68, 201, 84, 83, 65, 66, 128, 84, 83, - 65, 65, 68, 73, 89, 128, 84, 83, 65, 65, 128, 84, 83, 193, 84, 82, 89, - 66, 76, 73, 79, 206, 84, 82, 89, 65, 83, 79, 83, 84, 82, 69, 76, 78, 65, - 89, 65, 128, 84, 82, 89, 65, 83, 79, 80, 79, 86, 79, 68, 78, 65, 89, 65, - 128, 84, 82, 89, 65, 83, 79, 71, 76, 65, 83, 78, 65, 89, 65, 128, 84, 82, - 89, 65, 83, 75, 65, 128, 84, 82, 85, 84, 72, 128, 84, 82, 85, 78, 75, - 128, 84, 82, 85, 78, 67, 65, 84, 69, 196, 84, 82, 85, 77, 80, 69, 84, - 128, 84, 82, 85, 77, 80, 45, 57, 128, 84, 82, 85, 77, 80, 45, 56, 128, - 84, 82, 85, 77, 80, 45, 55, 128, 84, 82, 85, 77, 80, 45, 54, 128, 84, 82, - 85, 77, 80, 45, 53, 128, 84, 82, 85, 77, 80, 45, 52, 128, 84, 82, 85, 77, - 80, 45, 51, 128, 84, 82, 85, 77, 80, 45, 50, 49, 128, 84, 82, 85, 77, 80, - 45, 50, 48, 128, 84, 82, 85, 77, 80, 45, 50, 128, 84, 82, 85, 77, 80, 45, - 49, 57, 128, 84, 82, 85, 77, 80, 45, 49, 56, 128, 84, 82, 85, 77, 80, 45, - 49, 55, 128, 84, 82, 85, 77, 80, 45, 49, 54, 128, 84, 82, 85, 77, 80, 45, - 49, 53, 128, 84, 82, 85, 77, 80, 45, 49, 52, 128, 84, 82, 85, 77, 80, 45, - 49, 51, 128, 84, 82, 85, 77, 80, 45, 49, 50, 128, 84, 82, 85, 77, 80, 45, - 49, 49, 128, 84, 82, 85, 77, 80, 45, 49, 48, 128, 84, 82, 85, 77, 80, 45, - 49, 128, 84, 82, 85, 69, 128, 84, 82, 85, 197, 84, 82, 85, 67, 75, 128, - 84, 82, 79, 80, 73, 67, 65, 204, 84, 82, 79, 80, 72, 89, 128, 84, 82, 79, - 77, 73, 75, 79, 83, 89, 78, 65, 71, 77, 65, 128, 84, 82, 79, 77, 73, 75, - 79, 80, 83, 73, 70, 73, 83, 84, 79, 78, 128, 84, 82, 79, 77, 73, 75, 79, - 80, 65, 82, 65, 75, 65, 76, 69, 83, 77, 65, 128, 84, 82, 79, 77, 73, 75, - 79, 78, 128, 84, 82, 79, 77, 73, 75, 79, 206, 84, 82, 79, 77, 73, 75, 79, - 76, 89, 71, 73, 83, 77, 65, 128, 84, 82, 79, 76, 76, 69, 89, 66, 85, 83, - 128, 84, 82, 79, 76, 76, 69, 89, 128, 84, 82, 79, 76, 76, 128, 84, 82, - 79, 75, 85, 84, 65, 83, 84, 201, 84, 82, 79, 69, 90, 69, 78, 73, 65, 206, - 84, 82, 73, 85, 77, 80, 72, 128, 84, 82, 73, 84, 79, 211, 84, 82, 73, 84, - 73, 77, 79, 82, 73, 79, 78, 128, 84, 82, 73, 83, 73, 77, 79, 85, 128, 84, - 82, 73, 83, 69, 77, 69, 128, 84, 82, 73, 80, 79, 68, 128, 84, 82, 73, 80, - 76, 73, 128, 84, 82, 73, 80, 76, 69, 128, 84, 82, 73, 80, 76, 197, 84, - 82, 73, 79, 206, 84, 82, 73, 76, 76, 73, 79, 78, 83, 128, 84, 82, 73, 76, - 76, 128, 84, 82, 73, 73, 83, 65, 80, 128, 84, 82, 73, 71, 82, 65, 77, 77, - 79, 211, 84, 82, 73, 71, 82, 65, 205, 84, 82, 73, 71, 79, 82, 71, 79, 78, - 128, 84, 82, 73, 70, 79, 78, 73, 65, 83, 128, 84, 82, 73, 70, 79, 76, 73, - 65, 84, 197, 84, 82, 73, 68, 69, 78, 84, 128, 84, 82, 73, 68, 69, 78, - 212, 84, 82, 73, 67, 79, 76, 79, 78, 128, 84, 82, 73, 65, 78, 71, 85, 76, - 65, 210, 84, 82, 73, 65, 78, 71, 76, 69, 45, 82, 79, 85, 78, 196, 84, 82, - 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 196, 84, 82, 73, 65, 78, - 71, 76, 69, 128, 84, 82, 73, 65, 78, 71, 76, 197, 84, 82, 73, 65, 128, - 84, 82, 73, 128, 84, 82, 69, 83, 86, 69, 84, 76, 89, 128, 84, 82, 69, 83, - 86, 69, 84, 76, 79, 128, 84, 82, 69, 83, 86, 69, 84, 76, 65, 89, 65, 128, - 84, 82, 69, 83, 73, 76, 76, 79, 128, 84, 82, 69, 78, 68, 128, 84, 82, 69, - 78, 196, 84, 82, 69, 77, 79, 76, 79, 45, 51, 128, 84, 82, 69, 77, 79, 76, - 79, 45, 50, 128, 84, 82, 69, 77, 79, 76, 79, 45, 49, 128, 84, 82, 69, 69, - 128, 84, 82, 69, 197, 84, 82, 69, 68, 69, 67, 73, 76, 69, 128, 84, 82, - 69, 65, 68, 73, 78, 71, 128, 84, 82, 65, 89, 128, 84, 82, 65, 86, 69, 76, - 45, 87, 65, 76, 76, 80, 76, 65, 78, 197, 84, 82, 65, 86, 69, 76, 45, 70, - 76, 79, 79, 82, 80, 76, 65, 78, 197, 84, 82, 65, 80, 69, 90, 73, 85, 77, - 128, 84, 82, 65, 80, 128, 84, 82, 65, 78, 83, 86, 69, 82, 83, 65, 204, - 84, 82, 65, 78, 83, 80, 79, 83, 73, 84, 73, 79, 206, 84, 82, 65, 78, 83, - 80, 76, 85, 84, 79, 128, 84, 82, 65, 78, 83, 77, 73, 212, 84, 82, 65, 78, - 83, 77, 73, 83, 83, 73, 79, 78, 128, 84, 82, 65, 78, 83, 77, 73, 83, 83, - 73, 79, 206, 84, 82, 65, 77, 87, 65, 89, 128, 84, 82, 65, 77, 128, 84, - 82, 65, 205, 84, 82, 65, 73, 78, 128, 84, 82, 65, 73, 206, 84, 82, 65, - 73, 76, 73, 78, 199, 84, 82, 65, 70, 70, 73, 67, 128, 84, 82, 65, 70, 70, - 73, 195, 84, 82, 65, 68, 73, 84, 73, 79, 78, 65, 204, 84, 82, 65, 68, - 197, 84, 82, 65, 67, 84, 79, 82, 128, 84, 82, 65, 67, 75, 66, 65, 76, 76, - 128, 84, 82, 65, 67, 75, 128, 84, 82, 65, 128, 84, 82, 128, 84, 79, 89, - 79, 82, 128, 84, 79, 88, 128, 84, 79, 87, 69, 82, 128, 84, 79, 87, 65, - 82, 68, 211, 84, 79, 86, 128, 84, 79, 85, 82, 78, 79, 73, 211, 84, 79, - 85, 67, 72, 84, 79, 78, 197, 84, 79, 85, 67, 72, 73, 78, 199, 84, 79, 85, - 67, 72, 69, 211, 84, 79, 85, 67, 200, 84, 79, 84, 207, 84, 79, 84, 65, - 204, 84, 79, 84, 128, 84, 79, 83, 128, 84, 79, 82, 84, 79, 73, 83, 197, - 84, 79, 82, 83, 79, 45, 87, 65, 76, 76, 80, 76, 65, 78, 197, 84, 79, 82, - 83, 79, 45, 70, 76, 79, 79, 82, 80, 76, 65, 78, 197, 84, 79, 82, 83, 79, - 128, 84, 79, 82, 78, 65, 68, 79, 128, 84, 79, 82, 67, 85, 76, 85, 83, - 128, 84, 79, 82, 67, 85, 76, 85, 211, 84, 79, 82, 67, 72, 128, 84, 79, - 81, 128, 84, 79, 80, 66, 65, 82, 128, 84, 79, 80, 45, 76, 73, 71, 72, 84, - 69, 196, 84, 79, 80, 128, 84, 79, 208, 84, 79, 79, 84, 72, 66, 82, 85, - 83, 72, 128, 84, 79, 79, 84, 72, 128, 84, 79, 79, 78, 128, 84, 79, 79, - 76, 66, 79, 88, 128, 84, 79, 78, 79, 83, 128, 84, 79, 78, 71, 85, 69, - 128, 84, 79, 78, 71, 85, 197, 84, 79, 78, 71, 128, 84, 79, 78, 69, 45, - 86, 128, 84, 79, 78, 69, 45, 83, 128, 84, 79, 78, 69, 45, 77, 128, 84, - 79, 78, 69, 45, 74, 128, 84, 79, 78, 69, 45, 71, 128, 84, 79, 78, 69, 45, - 68, 128, 84, 79, 78, 69, 45, 66, 128, 84, 79, 78, 69, 45, 56, 128, 84, - 79, 78, 69, 45, 55, 128, 84, 79, 78, 69, 45, 54, 128, 84, 79, 78, 69, 45, - 53, 128, 84, 79, 78, 69, 45, 52, 128, 84, 79, 78, 69, 45, 51, 128, 84, - 79, 78, 69, 45, 50, 128, 84, 79, 78, 69, 45, 49, 128, 84, 79, 78, 69, - 128, 84, 79, 78, 65, 204, 84, 79, 77, 80, 73, 128, 84, 79, 77, 65, 84, - 79, 128, 84, 79, 76, 79, 78, 71, 128, 84, 79, 75, 89, 207, 84, 79, 73, - 76, 69, 84, 128, 84, 79, 71, 69, 84, 72, 69, 82, 128, 84, 79, 68, 207, - 84, 79, 67, 72, 75, 65, 128, 84, 79, 65, 78, 68, 65, 75, 72, 73, 65, 84, - 128, 84, 79, 65, 128, 84, 79, 45, 82, 65, 128, 84, 79, 45, 54, 128, 84, - 79, 45, 53, 128, 84, 79, 45, 52, 128, 84, 79, 45, 51, 128, 84, 79, 45, - 50, 128, 84, 79, 45, 49, 128, 84, 78, 128, 84, 76, 86, 128, 84, 76, 85, - 128, 84, 76, 73, 128, 84, 76, 72, 89, 65, 128, 84, 76, 72, 87, 69, 128, - 84, 76, 72, 85, 128, 84, 76, 72, 79, 79, 128, 84, 76, 72, 79, 128, 84, - 76, 72, 73, 128, 84, 76, 72, 69, 69, 128, 84, 76, 72, 69, 128, 84, 76, - 72, 65, 128, 84, 76, 69, 69, 128, 84, 76, 65, 128, 84, 74, 69, 128, 84, - 73, 88, 128, 84, 73, 87, 82, 128, 84, 73, 87, 78, 128, 84, 73, 87, 65, - 218, 84, 73, 84, 85, 65, 69, 80, 128, 84, 73, 84, 76, 79, 128, 84, 73, - 84, 76, 207, 84, 73, 84, 193, 84, 73, 84, 128, 84, 73, 82, 89, 65, 75, - 128, 84, 73, 82, 84, 193, 84, 73, 82, 79, 78, 73, 65, 206, 84, 73, 82, - 72, 85, 84, 193, 84, 73, 82, 69, 196, 84, 73, 82, 128, 84, 73, 210, 84, - 73, 80, 80, 73, 128, 84, 73, 80, 69, 72, 65, 128, 84, 73, 80, 128, 84, - 73, 208, 84, 73, 78, 89, 128, 84, 73, 78, 217, 84, 73, 78, 78, 69, 128, - 84, 73, 78, 67, 84, 85, 82, 69, 128, 84, 73, 78, 65, 71, 77, 65, 128, 84, - 73, 77, 69, 83, 128, 84, 73, 77, 69, 210, 84, 73, 77, 69, 128, 84, 73, - 76, 84, 73, 78, 71, 128, 84, 73, 76, 84, 73, 78, 199, 84, 73, 76, 84, - 128, 84, 73, 76, 69, 83, 128, 84, 73, 76, 68, 69, 128, 84, 73, 76, 68, - 197, 84, 73, 76, 128, 84, 73, 204, 84, 73, 75, 72, 89, 128, 84, 73, 75, - 72, 65, 89, 65, 128, 84, 73, 75, 72, 65, 89, 193, 84, 73, 75, 69, 85, 84, - 45, 84, 72, 73, 69, 85, 84, 72, 128, 84, 73, 75, 69, 85, 84, 45, 83, 73, - 79, 83, 45, 75, 73, 89, 69, 79, 75, 128, 84, 73, 75, 69, 85, 84, 45, 83, - 73, 79, 83, 128, 84, 73, 75, 69, 85, 84, 45, 82, 73, 69, 85, 76, 128, 84, - 73, 75, 69, 85, 84, 45, 80, 73, 69, 85, 80, 128, 84, 73, 75, 69, 85, 84, - 45, 77, 73, 69, 85, 77, 128, 84, 73, 75, 69, 85, 84, 45, 75, 73, 89, 69, - 79, 75, 128, 84, 73, 75, 69, 85, 84, 45, 67, 73, 69, 85, 67, 128, 84, 73, - 75, 69, 85, 84, 45, 67, 72, 73, 69, 85, 67, 72, 128, 84, 73, 75, 69, 85, - 84, 128, 84, 73, 75, 69, 85, 212, 84, 73, 71, 72, 84, 76, 89, 45, 67, 76, - 79, 83, 69, 196, 84, 73, 71, 72, 212, 84, 73, 71, 69, 82, 128, 84, 73, - 71, 69, 210, 84, 73, 70, 73, 78, 65, 71, 200, 84, 73, 69, 88, 128, 84, - 73, 69, 80, 128, 84, 73, 197, 84, 73, 67, 75, 69, 84, 83, 128, 84, 73, - 67, 75, 69, 84, 128, 84, 73, 67, 75, 128, 84, 73, 67, 203, 84, 73, 65, - 82, 65, 128, 84, 73, 50, 128, 84, 73, 45, 55, 128, 84, 73, 45, 54, 128, - 84, 73, 45, 53, 128, 84, 73, 45, 52, 128, 84, 73, 45, 51, 128, 84, 73, - 45, 50, 128, 84, 73, 45, 49, 128, 84, 72, 90, 128, 84, 72, 89, 79, 79, - 205, 84, 72, 87, 79, 79, 128, 84, 72, 87, 79, 128, 84, 72, 87, 73, 73, - 128, 84, 72, 87, 73, 128, 84, 72, 87, 69, 69, 128, 84, 72, 87, 65, 65, - 128, 84, 72, 87, 65, 128, 84, 72, 85, 82, 211, 84, 72, 85, 82, 73, 83, - 65, 218, 84, 72, 85, 78, 71, 128, 84, 72, 85, 78, 68, 69, 82, 83, 84, 79, - 82, 77, 128, 84, 72, 85, 78, 68, 69, 82, 128, 84, 72, 85, 78, 68, 69, - 210, 84, 72, 85, 77, 66, 211, 84, 72, 85, 77, 66, 128, 84, 72, 82, 79, - 87, 73, 78, 199, 84, 72, 82, 79, 85, 71, 72, 128, 84, 72, 82, 79, 85, 71, - 200, 84, 72, 82, 69, 69, 45, 84, 72, 73, 82, 84, 89, 128, 84, 72, 82, 69, - 69, 45, 81, 85, 65, 82, 84, 69, 210, 84, 72, 82, 69, 69, 45, 80, 69, 82, - 45, 69, 205, 84, 72, 82, 69, 69, 45, 76, 73, 78, 197, 84, 72, 82, 69, 69, - 45, 76, 69, 71, 71, 69, 196, 84, 72, 82, 69, 69, 45, 72, 85, 78, 68, 82, - 69, 68, 45, 65, 78, 68, 45, 84, 87, 69, 78, 84, 73, 69, 84, 72, 128, 84, - 72, 82, 69, 69, 45, 69, 205, 84, 72, 82, 69, 69, 45, 68, 79, 212, 84, 72, - 82, 69, 69, 45, 196, 84, 72, 82, 69, 69, 45, 67, 73, 82, 67, 76, 197, 84, - 72, 82, 69, 65, 68, 128, 84, 72, 79, 85, 83, 65, 78, 68, 83, 128, 84, 72, - 79, 85, 83, 65, 78, 68, 211, 84, 72, 79, 85, 83, 65, 78, 196, 84, 72, 79, - 85, 71, 72, 212, 84, 72, 79, 85, 128, 84, 72, 79, 82, 78, 128, 84, 72, - 79, 82, 206, 84, 72, 79, 78, 71, 128, 84, 72, 79, 78, 199, 84, 72, 79, - 77, 128, 84, 72, 79, 74, 128, 84, 72, 79, 65, 128, 84, 72, 207, 84, 72, - 73, 85, 84, 72, 128, 84, 72, 73, 84, 65, 128, 84, 72, 73, 82, 84, 89, 45, - 83, 69, 67, 79, 78, 68, 128, 84, 72, 73, 82, 84, 89, 45, 83, 69, 67, 79, - 78, 196, 84, 72, 73, 82, 84, 89, 45, 79, 78, 69, 128, 84, 72, 73, 82, 84, - 89, 45, 70, 73, 86, 197, 84, 72, 73, 82, 84, 217, 84, 72, 73, 82, 84, 69, - 69, 78, 128, 84, 72, 73, 82, 84, 69, 69, 206, 84, 72, 73, 82, 68, 83, - 128, 84, 72, 73, 82, 68, 211, 84, 72, 73, 82, 68, 45, 83, 84, 65, 71, - 197, 84, 72, 73, 82, 68, 128, 84, 72, 73, 82, 196, 84, 72, 73, 78, 75, - 73, 78, 199, 84, 72, 73, 78, 71, 128, 84, 72, 73, 73, 128, 84, 72, 73, - 71, 72, 128, 84, 72, 73, 69, 85, 84, 200, 84, 72, 73, 67, 203, 84, 72, - 73, 65, 66, 128, 84, 72, 69, 89, 128, 84, 72, 69, 84, 72, 69, 128, 84, - 72, 69, 84, 72, 128, 84, 72, 69, 84, 65, 128, 84, 72, 69, 84, 193, 84, - 72, 69, 83, 80, 73, 65, 206, 84, 72, 69, 83, 69, 79, 83, 128, 84, 72, 69, - 83, 69, 79, 211, 84, 72, 69, 211, 84, 72, 69, 82, 77, 79, 77, 69, 84, 69, - 82, 128, 84, 72, 69, 82, 77, 79, 68, 89, 78, 65, 77, 73, 67, 128, 84, 72, - 69, 82, 69, 70, 79, 82, 69, 128, 84, 72, 69, 82, 197, 84, 72, 69, 206, - 84, 72, 69, 77, 65, 84, 73, 83, 77, 79, 211, 84, 72, 69, 77, 65, 128, 84, - 72, 69, 77, 193, 84, 72, 69, 72, 128, 84, 72, 69, 200, 84, 72, 69, 65, - 128, 84, 72, 197, 84, 72, 65, 87, 128, 84, 72, 65, 78, 84, 72, 65, 75, - 72, 65, 84, 128, 84, 72, 65, 78, 78, 65, 128, 84, 72, 65, 78, 128, 84, - 72, 65, 206, 84, 72, 65, 77, 69, 68, 72, 128, 84, 72, 65, 76, 128, 84, - 72, 65, 204, 84, 72, 65, 74, 128, 84, 72, 65, 201, 84, 72, 65, 72, 65, - 78, 128, 84, 72, 65, 65, 78, 193, 84, 72, 65, 65, 76, 85, 128, 84, 72, - 45, 67, 82, 69, 197, 84, 69, 88, 84, 128, 84, 69, 88, 212, 84, 69, 88, - 128, 84, 69, 86, 73, 82, 128, 84, 69, 85, 84, 69, 85, 88, 128, 84, 69, - 85, 84, 69, 85, 87, 69, 78, 128, 84, 69, 85, 84, 128, 84, 69, 85, 78, - 128, 84, 69, 85, 65, 69, 81, 128, 84, 69, 85, 65, 69, 78, 128, 84, 69, - 85, 128, 84, 69, 84, 82, 65, 83, 73, 77, 79, 85, 128, 84, 69, 84, 82, 65, - 83, 69, 77, 69, 128, 84, 69, 84, 82, 65, 80, 76, 73, 128, 84, 69, 84, 82, - 65, 71, 82, 65, 205, 84, 69, 84, 82, 65, 70, 79, 78, 73, 65, 83, 128, 84, - 69, 84, 72, 128, 84, 69, 84, 200, 84, 69, 84, 65, 82, 84, 79, 211, 84, - 69, 84, 65, 82, 84, 73, 77, 79, 82, 73, 79, 78, 128, 84, 69, 84, 128, 84, - 69, 212, 84, 69, 83, 212, 84, 69, 83, 83, 69, 82, 65, 128, 84, 69, 83, - 83, 69, 82, 193, 84, 69, 83, 83, 65, 82, 79, 206, 84, 69, 83, 200, 84, - 69, 82, 77, 73, 78, 65, 84, 79, 82, 128, 84, 69, 82, 77, 73, 78, 65, 204, - 84, 69, 80, 128, 84, 69, 78, 85, 84, 79, 128, 84, 69, 78, 85, 128, 84, - 69, 78, 213, 84, 69, 78, 84, 72, 128, 84, 69, 78, 84, 128, 84, 69, 78, - 83, 69, 128, 84, 69, 78, 83, 197, 84, 69, 78, 83, 128, 84, 69, 78, 211, - 84, 69, 78, 78, 73, 211, 84, 69, 78, 71, 197, 84, 69, 78, 45, 84, 72, 73, - 82, 84, 89, 128, 84, 69, 78, 128, 84, 69, 206, 84, 69, 77, 80, 85, 211, - 84, 69, 77, 80, 76, 69, 128, 84, 69, 76, 85, 71, 213, 84, 69, 76, 85, - 128, 84, 69, 76, 79, 85, 211, 84, 69, 76, 76, 69, 210, 84, 69, 76, 73, - 83, 72, 193, 84, 69, 76, 69, 86, 73, 83, 73, 79, 78, 128, 84, 69, 76, 69, - 83, 67, 79, 80, 69, 128, 84, 69, 76, 69, 80, 72, 79, 78, 69, 128, 84, 69, - 76, 69, 80, 72, 79, 78, 197, 84, 69, 76, 69, 73, 65, 128, 84, 69, 76, 69, - 71, 82, 65, 80, 200, 84, 69, 75, 128, 84, 69, 73, 87, 83, 128, 84, 69, - 71, 69, 72, 128, 84, 69, 69, 84, 72, 128, 84, 69, 69, 84, 200, 84, 69, - 69, 78, 83, 128, 84, 69, 69, 69, 69, 128, 84, 69, 197, 84, 69, 68, 85, - 78, 71, 128, 84, 69, 68, 68, 217, 84, 69, 65, 82, 83, 128, 84, 69, 65, - 82, 211, 84, 69, 65, 82, 68, 82, 79, 80, 45, 83, 80, 79, 75, 69, 196, 84, - 69, 65, 82, 68, 82, 79, 80, 45, 83, 72, 65, 78, 75, 69, 196, 84, 69, 65, - 82, 68, 82, 79, 80, 45, 66, 65, 82, 66, 69, 196, 84, 69, 65, 82, 45, 79, - 70, 198, 84, 69, 65, 82, 128, 84, 69, 65, 80, 79, 84, 128, 84, 69, 65, - 67, 85, 208, 84, 69, 65, 128, 84, 69, 45, 85, 128, 84, 69, 45, 57, 128, - 84, 69, 45, 56, 128, 84, 69, 45, 55, 128, 84, 69, 45, 54, 128, 84, 69, - 45, 53, 128, 84, 69, 45, 52, 128, 84, 69, 45, 51, 128, 84, 69, 45, 50, - 128, 84, 69, 45, 49, 128, 84, 67, 72, 69, 72, 69, 72, 128, 84, 67, 72, - 69, 72, 69, 200, 84, 67, 72, 69, 72, 128, 84, 67, 72, 69, 200, 84, 67, - 72, 69, 128, 84, 195, 84, 65, 89, 128, 84, 65, 88, 73, 128, 84, 65, 88, - 128, 84, 65, 87, 69, 76, 76, 69, 77, 69, 212, 84, 65, 87, 65, 128, 84, - 65, 87, 128, 84, 65, 215, 84, 65, 86, 73, 89, 65, 78, 73, 128, 84, 65, - 86, 128, 84, 65, 214, 84, 65, 85, 82, 85, 83, 128, 84, 65, 85, 77, 128, - 84, 65, 213, 84, 65, 84, 87, 69, 69, 76, 128, 84, 65, 84, 87, 69, 69, - 204, 84, 65, 84, 84, 79, 79, 69, 196, 84, 65, 84, 128, 84, 65, 83, 83, - 73, 128, 84, 65, 83, 72, 69, 69, 76, 128, 84, 65, 83, 128, 84, 65, 82, - 85, 78, 71, 128, 84, 65, 82, 84, 65, 82, 45, 50, 128, 84, 65, 82, 84, 65, - 82, 128, 84, 65, 82, 71, 69, 84, 128, 84, 65, 81, 128, 84, 65, 80, 69, - 82, 128, 84, 65, 80, 197, 84, 65, 80, 128, 84, 65, 79, 128, 84, 65, 78, - 78, 69, 196, 84, 65, 78, 71, 83, 193, 84, 65, 78, 71, 69, 82, 73, 78, 69, - 128, 84, 65, 78, 71, 69, 78, 84, 128, 84, 65, 78, 71, 69, 78, 212, 84, - 65, 78, 199, 84, 65, 78, 65, 66, 65, 84, 193, 84, 65, 78, 65, 128, 84, - 65, 78, 128, 84, 65, 77, 73, 78, 71, 128, 84, 65, 77, 65, 206, 84, 65, - 77, 65, 76, 69, 128, 84, 65, 77, 128, 84, 65, 76, 76, 217, 84, 65, 76, - 76, 128, 84, 65, 76, 204, 84, 65, 76, 73, 78, 71, 128, 84, 65, 76, 73, - 78, 199, 84, 65, 76, 69, 78, 84, 83, 128, 84, 65, 76, 69, 78, 212, 84, - 65, 75, 82, 201, 84, 65, 75, 72, 65, 76, 76, 85, 83, 128, 84, 65, 75, 69, - 79, 85, 212, 84, 65, 75, 69, 128, 84, 65, 75, 52, 128, 84, 65, 75, 180, - 84, 65, 75, 128, 84, 65, 73, 83, 89, 79, 85, 128, 84, 65, 73, 76, 76, 69, - 83, 211, 84, 65, 73, 76, 128, 84, 65, 73, 204, 84, 65, 72, 65, 76, 65, - 128, 84, 65, 72, 128, 84, 65, 200, 84, 65, 71, 66, 65, 78, 87, 193, 84, - 65, 71, 65, 76, 79, 199, 84, 65, 71, 128, 84, 65, 199, 84, 65, 69, 206, - 84, 65, 67, 79, 128, 84, 65, 67, 75, 128, 84, 65, 67, 203, 84, 65, 66, - 85, 76, 65, 84, 73, 79, 78, 128, 84, 65, 66, 85, 76, 65, 84, 73, 79, 206, - 84, 65, 66, 83, 128, 84, 65, 66, 76, 69, 128, 84, 65, 66, 76, 197, 84, - 65, 66, 65, 65, 82, 65, 75, 193, 84, 65, 66, 128, 84, 65, 194, 84, 65, - 65, 83, 72, 65, 69, 128, 84, 65, 65, 81, 128, 84, 65, 65, 77, 128, 84, - 65, 65, 76, 85, 74, 193, 84, 65, 65, 73, 128, 84, 65, 65, 70, 128, 84, - 65, 50, 128, 84, 65, 45, 82, 79, 76, 128, 84, 65, 45, 52, 128, 84, 65, - 45, 51, 128, 84, 65, 45, 50, 128, 84, 65, 45, 49, 128, 84, 48, 51, 54, - 128, 84, 48, 51, 53, 128, 84, 48, 51, 52, 128, 84, 48, 51, 51, 65, 128, - 84, 48, 51, 51, 128, 84, 48, 51, 50, 65, 128, 84, 48, 51, 50, 128, 84, - 48, 51, 49, 128, 84, 48, 51, 48, 128, 84, 48, 50, 57, 128, 84, 48, 50, - 56, 128, 84, 48, 50, 55, 128, 84, 48, 50, 54, 128, 84, 48, 50, 53, 128, - 84, 48, 50, 52, 128, 84, 48, 50, 51, 128, 84, 48, 50, 50, 128, 84, 48, - 50, 49, 128, 84, 48, 50, 48, 128, 84, 48, 49, 57, 128, 84, 48, 49, 56, - 128, 84, 48, 49, 55, 128, 84, 48, 49, 54, 65, 128, 84, 48, 49, 54, 128, - 84, 48, 49, 53, 128, 84, 48, 49, 52, 128, 84, 48, 49, 51, 128, 84, 48, - 49, 50, 128, 84, 48, 49, 49, 65, 128, 84, 48, 49, 49, 128, 84, 48, 49, - 48, 128, 84, 48, 48, 57, 65, 128, 84, 48, 48, 57, 128, 84, 48, 48, 56, - 65, 128, 84, 48, 48, 56, 128, 84, 48, 48, 55, 65, 128, 84, 48, 48, 55, - 128, 84, 48, 48, 54, 128, 84, 48, 48, 53, 128, 84, 48, 48, 52, 128, 84, - 48, 48, 51, 65, 128, 84, 48, 48, 51, 128, 84, 48, 48, 50, 128, 84, 48, - 48, 49, 128, 84, 45, 83, 72, 73, 82, 84, 128, 84, 45, 82, 69, 88, 128, - 83, 90, 90, 128, 83, 90, 87, 71, 128, 83, 90, 87, 65, 128, 83, 90, 85, - 128, 83, 90, 79, 128, 83, 90, 73, 128, 83, 90, 69, 69, 128, 83, 90, 69, - 128, 83, 90, 65, 65, 128, 83, 90, 65, 128, 83, 90, 128, 83, 89, 88, 128, - 83, 89, 84, 128, 83, 89, 83, 84, 69, 205, 83, 89, 82, 88, 128, 83, 89, - 82, 77, 65, 84, 73, 75, 73, 128, 83, 89, 82, 77, 65, 128, 83, 89, 82, 73, - 78, 71, 69, 128, 83, 89, 82, 73, 65, 195, 83, 89, 82, 128, 83, 89, 80, - 128, 83, 89, 79, 85, 87, 65, 128, 83, 89, 78, 69, 86, 77, 65, 128, 83, - 89, 78, 68, 69, 83, 77, 79, 211, 83, 89, 78, 67, 72, 82, 79, 78, 79, 85, - 211, 83, 89, 78, 65, 71, 79, 71, 85, 69, 128, 83, 89, 78, 65, 71, 77, - 193, 83, 89, 78, 65, 70, 73, 128, 83, 89, 78, 128, 83, 89, 77, 77, 69, - 84, 82, 89, 128, 83, 89, 77, 77, 69, 84, 82, 73, 195, 83, 89, 77, 66, 79, - 76, 83, 128, 83, 89, 77, 66, 79, 76, 211, 83, 89, 77, 66, 79, 76, 45, 57, - 128, 83, 89, 77, 66, 79, 76, 45, 56, 128, 83, 89, 77, 66, 79, 76, 45, 55, - 128, 83, 89, 77, 66, 79, 76, 45, 54, 128, 83, 89, 77, 66, 79, 76, 45, 53, - 52, 128, 83, 89, 77, 66, 79, 76, 45, 53, 51, 128, 83, 89, 77, 66, 79, 76, - 45, 53, 50, 128, 83, 89, 77, 66, 79, 76, 45, 53, 49, 128, 83, 89, 77, 66, - 79, 76, 45, 53, 48, 128, 83, 89, 77, 66, 79, 76, 45, 53, 128, 83, 89, 77, - 66, 79, 76, 45, 52, 57, 128, 83, 89, 77, 66, 79, 76, 45, 52, 56, 128, 83, - 89, 77, 66, 79, 76, 45, 52, 55, 128, 83, 89, 77, 66, 79, 76, 45, 52, 53, - 128, 83, 89, 77, 66, 79, 76, 45, 52, 51, 128, 83, 89, 77, 66, 79, 76, 45, - 52, 50, 128, 83, 89, 77, 66, 79, 76, 45, 52, 48, 128, 83, 89, 77, 66, 79, - 76, 45, 52, 128, 83, 89, 77, 66, 79, 76, 45, 51, 57, 128, 83, 89, 77, 66, - 79, 76, 45, 51, 56, 128, 83, 89, 77, 66, 79, 76, 45, 51, 55, 128, 83, 89, - 77, 66, 79, 76, 45, 51, 54, 128, 83, 89, 77, 66, 79, 76, 45, 51, 50, 128, - 83, 89, 77, 66, 79, 76, 45, 51, 48, 128, 83, 89, 77, 66, 79, 76, 45, 51, - 128, 83, 89, 77, 66, 79, 76, 45, 50, 57, 128, 83, 89, 77, 66, 79, 76, 45, - 50, 55, 128, 83, 89, 77, 66, 79, 76, 45, 50, 54, 128, 83, 89, 77, 66, 79, - 76, 45, 50, 53, 128, 83, 89, 77, 66, 79, 76, 45, 50, 52, 128, 83, 89, 77, - 66, 79, 76, 45, 50, 51, 128, 83, 89, 77, 66, 79, 76, 45, 50, 50, 128, 83, - 89, 77, 66, 79, 76, 45, 50, 49, 128, 83, 89, 77, 66, 79, 76, 45, 50, 48, - 128, 83, 89, 77, 66, 79, 76, 45, 50, 128, 83, 89, 77, 66, 79, 76, 45, 49, - 57, 128, 83, 89, 77, 66, 79, 76, 45, 49, 56, 128, 83, 89, 77, 66, 79, 76, - 45, 49, 55, 128, 83, 89, 77, 66, 79, 76, 45, 49, 54, 128, 83, 89, 77, 66, - 79, 76, 45, 49, 53, 128, 83, 89, 77, 66, 79, 76, 45, 49, 52, 128, 83, 89, - 77, 66, 79, 76, 45, 49, 51, 128, 83, 89, 77, 66, 79, 76, 45, 49, 50, 128, - 83, 89, 77, 66, 79, 76, 45, 49, 49, 128, 83, 89, 77, 66, 79, 76, 45, 49, - 48, 128, 83, 89, 77, 66, 79, 76, 45, 49, 128, 83, 89, 76, 79, 84, 201, - 83, 89, 73, 128, 83, 89, 128, 83, 87, 90, 128, 83, 87, 85, 78, 199, 83, - 87, 79, 82, 68, 83, 128, 83, 87, 79, 82, 68, 128, 83, 87, 79, 79, 128, - 83, 87, 79, 128, 83, 87, 73, 82, 204, 83, 87, 73, 77, 83, 85, 73, 84, - 128, 83, 87, 73, 77, 77, 73, 78, 71, 128, 83, 87, 73, 77, 77, 69, 82, - 128, 83, 87, 73, 73, 128, 83, 87, 73, 128, 83, 87, 71, 128, 83, 87, 69, - 69, 84, 128, 83, 87, 69, 69, 212, 83, 87, 69, 65, 84, 128, 83, 87, 69, - 65, 212, 83, 87, 65, 83, 200, 83, 87, 65, 80, 80, 73, 78, 71, 128, 83, - 87, 65, 78, 128, 83, 87, 65, 65, 128, 83, 87, 128, 83, 86, 65, 83, 84, - 201, 83, 86, 65, 82, 73, 84, 65, 128, 83, 86, 65, 82, 73, 84, 193, 83, - 85, 88, 128, 83, 85, 85, 128, 83, 85, 84, 85, 72, 128, 83, 85, 84, 82, - 193, 83, 85, 84, 128, 83, 85, 83, 80, 69, 78, 83, 73, 79, 206, 83, 85, - 83, 72, 73, 128, 83, 85, 82, 89, 65, 128, 83, 85, 82, 88, 128, 83, 85, - 82, 82, 79, 85, 78, 68, 128, 83, 85, 82, 82, 79, 85, 78, 196, 83, 85, 82, - 70, 69, 82, 128, 83, 85, 82, 70, 65, 67, 197, 83, 85, 82, 69, 128, 83, - 85, 82, 65, 78, 71, 128, 83, 85, 82, 57, 128, 83, 85, 82, 128, 83, 85, - 210, 83, 85, 80, 82, 65, 76, 73, 78, 69, 65, 210, 83, 85, 80, 69, 82, 86, - 73, 83, 69, 128, 83, 85, 80, 69, 82, 86, 73, 76, 76, 65, 73, 78, 128, 83, - 85, 80, 69, 82, 83, 69, 84, 128, 83, 85, 80, 69, 82, 83, 69, 212, 83, 85, - 80, 69, 82, 83, 67, 82, 73, 80, 212, 83, 85, 80, 69, 82, 73, 77, 80, 79, - 83, 69, 196, 83, 85, 80, 69, 82, 72, 69, 82, 79, 128, 83, 85, 80, 69, 82, - 70, 73, 88, 69, 196, 83, 85, 80, 69, 210, 83, 85, 80, 128, 83, 85, 79, - 88, 128, 83, 85, 79, 80, 128, 83, 85, 79, 128, 83, 85, 78, 83, 69, 212, - 83, 85, 78, 82, 73, 83, 69, 128, 83, 85, 78, 82, 73, 83, 197, 83, 85, 78, - 71, 76, 65, 83, 83, 69, 83, 128, 83, 85, 78, 71, 128, 83, 85, 78, 70, 76, - 79, 87, 69, 82, 128, 83, 85, 78, 68, 65, 78, 69, 83, 197, 83, 85, 78, - 128, 83, 85, 206, 83, 85, 77, 77, 69, 82, 128, 83, 85, 77, 77, 65, 84, - 73, 79, 78, 128, 83, 85, 77, 77, 65, 84, 73, 79, 206, 83, 85, 77, 65, 83, - 72, 128, 83, 85, 77, 128, 83, 85, 76, 70, 85, 82, 128, 83, 85, 75, 85, - 78, 128, 83, 85, 75, 85, 206, 83, 85, 75, 85, 128, 83, 85, 75, 213, 83, - 85, 73, 84, 65, 66, 76, 69, 128, 83, 85, 73, 212, 83, 85, 72, 85, 82, - 128, 83, 85, 69, 128, 83, 85, 68, 50, 128, 83, 85, 68, 128, 83, 85, 67, - 75, 73, 78, 199, 83, 85, 67, 75, 69, 68, 128, 83, 85, 67, 203, 83, 85, - 67, 67, 69, 69, 68, 83, 128, 83, 85, 67, 67, 69, 69, 68, 211, 83, 85, 67, - 67, 69, 69, 68, 128, 83, 85, 67, 67, 69, 69, 196, 83, 85, 66, 85, 78, 73, - 84, 128, 83, 85, 66, 84, 82, 65, 67, 84, 73, 79, 78, 128, 83, 85, 66, 83, - 84, 73, 84, 85, 84, 73, 79, 206, 83, 85, 66, 83, 84, 73, 84, 85, 84, 69, - 128, 83, 85, 66, 83, 84, 73, 84, 85, 84, 197, 83, 85, 66, 83, 69, 84, - 128, 83, 85, 66, 83, 69, 212, 83, 85, 66, 83, 67, 82, 73, 80, 212, 83, - 85, 66, 80, 85, 78, 67, 84, 73, 83, 128, 83, 85, 66, 76, 73, 78, 69, 65, - 210, 83, 85, 66, 76, 73, 77, 65, 84, 73, 79, 78, 128, 83, 85, 66, 76, 73, - 77, 65, 84, 69, 45, 51, 128, 83, 85, 66, 76, 73, 77, 65, 84, 69, 45, 50, - 128, 83, 85, 66, 76, 73, 77, 65, 84, 69, 128, 83, 85, 66, 76, 73, 77, 65, - 84, 197, 83, 85, 66, 74, 79, 73, 78, 69, 82, 128, 83, 85, 66, 74, 79, 73, - 78, 69, 196, 83, 85, 66, 74, 69, 67, 84, 128, 83, 85, 66, 73, 84, 79, - 128, 83, 85, 66, 72, 65, 65, 78, 65, 72, 213, 83, 85, 66, 71, 82, 79, 85, - 80, 128, 83, 85, 66, 71, 82, 79, 85, 208, 83, 85, 66, 128, 83, 85, 65, - 77, 128, 83, 85, 65, 69, 84, 128, 83, 85, 65, 69, 78, 128, 83, 85, 65, - 69, 128, 83, 85, 65, 66, 128, 83, 85, 65, 128, 83, 85, 45, 56, 128, 83, - 85, 45, 55, 128, 83, 85, 45, 54, 128, 83, 85, 45, 53, 128, 83, 85, 45, - 52, 128, 83, 85, 45, 51, 128, 83, 85, 45, 50, 128, 83, 85, 45, 49, 128, - 83, 213, 83, 84, 88, 128, 83, 84, 87, 65, 128, 83, 84, 85, 80, 65, 128, - 83, 84, 85, 70, 70, 69, 196, 83, 84, 85, 68, 89, 128, 83, 84, 85, 68, 73, - 207, 83, 84, 85, 67, 75, 45, 79, 85, 212, 83, 84, 83, 128, 83, 84, 82, - 79, 78, 199, 83, 84, 82, 79, 75, 69, 83, 128, 83, 84, 82, 79, 75, 69, - 211, 83, 84, 82, 79, 75, 69, 45, 57, 128, 83, 84, 82, 79, 75, 69, 45, 56, - 128, 83, 84, 82, 79, 75, 69, 45, 55, 128, 83, 84, 82, 79, 75, 69, 45, 54, - 128, 83, 84, 82, 79, 75, 69, 45, 53, 128, 83, 84, 82, 79, 75, 69, 45, 52, - 128, 83, 84, 82, 79, 75, 69, 45, 51, 128, 83, 84, 82, 79, 75, 69, 45, 50, - 128, 83, 84, 82, 79, 75, 69, 45, 49, 49, 128, 83, 84, 82, 79, 75, 69, 45, - 49, 48, 128, 83, 84, 82, 79, 75, 69, 45, 49, 128, 83, 84, 82, 79, 75, - 197, 83, 84, 82, 73, 80, 69, 128, 83, 84, 82, 73, 78, 71, 128, 83, 84, - 82, 73, 78, 199, 83, 84, 82, 73, 75, 69, 84, 72, 82, 79, 85, 71, 72, 128, - 83, 84, 82, 73, 75, 197, 83, 84, 82, 73, 68, 69, 128, 83, 84, 82, 73, 67, - 84, 76, 217, 83, 84, 82, 69, 84, 67, 72, 69, 196, 83, 84, 82, 69, 84, 67, - 72, 128, 83, 84, 82, 69, 83, 211, 83, 84, 82, 69, 78, 71, 84, 72, 128, - 83, 84, 82, 69, 76, 193, 83, 84, 82, 69, 65, 77, 69, 82, 128, 83, 84, 82, - 65, 87, 66, 69, 82, 82, 89, 128, 83, 84, 82, 65, 87, 128, 83, 84, 82, 65, - 84, 85, 77, 45, 50, 128, 83, 84, 82, 65, 84, 85, 77, 128, 83, 84, 82, 65, - 84, 85, 205, 83, 84, 82, 65, 84, 73, 65, 206, 83, 84, 82, 65, 78, 78, 79, - 128, 83, 84, 82, 65, 78, 78, 207, 83, 84, 82, 65, 73, 78, 69, 82, 128, - 83, 84, 82, 65, 73, 71, 72, 84, 78, 69, 83, 83, 128, 83, 84, 82, 65, 73, - 71, 72, 84, 128, 83, 84, 82, 65, 73, 71, 72, 212, 83, 84, 82, 65, 73, 70, - 128, 83, 84, 82, 65, 71, 71, 73, 83, 77, 65, 84, 65, 128, 83, 84, 79, 86, - 69, 128, 83, 84, 79, 82, 69, 128, 83, 84, 79, 80, 87, 65, 84, 67, 72, - 128, 83, 84, 79, 80, 80, 73, 78, 71, 128, 83, 84, 79, 80, 80, 65, 71, 69, - 128, 83, 84, 79, 80, 73, 84, 83, 65, 128, 83, 84, 79, 80, 73, 84, 83, - 193, 83, 84, 79, 80, 128, 83, 84, 79, 208, 83, 84, 79, 78, 69, 128, 83, - 84, 79, 67, 75, 128, 83, 84, 79, 67, 203, 83, 84, 73, 82, 82, 85, 208, - 83, 84, 73, 77, 77, 69, 128, 83, 84, 73, 76, 204, 83, 84, 73, 76, 197, - 83, 84, 73, 71, 77, 65, 128, 83, 84, 73, 67, 75, 73, 78, 199, 83, 84, 73, - 67, 203, 83, 84, 69, 84, 72, 79, 83, 67, 79, 80, 69, 128, 83, 84, 69, 82, - 69, 79, 128, 83, 84, 69, 80, 128, 83, 84, 69, 78, 79, 71, 82, 65, 80, 72, - 73, 195, 83, 84, 69, 77, 128, 83, 84, 69, 65, 77, 217, 83, 84, 69, 65, - 77, 73, 78, 199, 83, 84, 69, 65, 77, 128, 83, 84, 69, 65, 205, 83, 84, - 65, 86, 82, 79, 85, 128, 83, 84, 65, 86, 82, 79, 83, 128, 83, 84, 65, 86, - 82, 79, 211, 83, 84, 65, 85, 82, 79, 83, 128, 83, 84, 65, 84, 89, 65, - 128, 83, 84, 65, 84, 89, 193, 83, 84, 65, 84, 85, 197, 83, 84, 65, 84, - 73, 79, 78, 128, 83, 84, 65, 84, 69, 82, 83, 128, 83, 84, 65, 84, 69, - 128, 83, 84, 65, 82, 84, 73, 78, 199, 83, 84, 65, 82, 84, 128, 83, 84, - 65, 82, 212, 83, 84, 65, 82, 83, 128, 83, 84, 65, 82, 82, 69, 196, 83, - 84, 65, 82, 75, 128, 83, 84, 65, 82, 128, 83, 84, 65, 210, 83, 84, 65, - 78, 68, 83, 84, 73, 76, 76, 128, 83, 84, 65, 78, 68, 73, 78, 199, 83, 84, - 65, 78, 68, 65, 82, 196, 83, 84, 65, 78, 68, 128, 83, 84, 65, 78, 128, - 83, 84, 65, 77, 80, 69, 196, 83, 84, 65, 76, 76, 73, 79, 78, 128, 83, 84, - 65, 70, 70, 128, 83, 84, 65, 70, 198, 83, 84, 65, 68, 73, 85, 77, 128, - 83, 84, 65, 67, 75, 69, 196, 83, 84, 65, 67, 67, 65, 84, 79, 128, 83, 84, - 65, 67, 67, 65, 84, 73, 83, 83, 73, 77, 79, 128, 83, 84, 50, 128, 83, 83, - 89, 88, 128, 83, 83, 89, 84, 128, 83, 83, 89, 82, 88, 128, 83, 83, 89, - 82, 128, 83, 83, 89, 80, 128, 83, 83, 89, 128, 83, 83, 85, 88, 128, 83, - 83, 85, 85, 128, 83, 83, 85, 84, 128, 83, 83, 85, 80, 128, 83, 83, 79, - 88, 128, 83, 83, 79, 84, 128, 83, 83, 79, 80, 128, 83, 83, 79, 79, 128, - 83, 83, 79, 128, 83, 83, 73, 88, 128, 83, 83, 73, 84, 128, 83, 83, 73, - 80, 128, 83, 83, 73, 73, 128, 83, 83, 73, 69, 88, 128, 83, 83, 73, 69, - 80, 128, 83, 83, 73, 69, 128, 83, 83, 72, 73, 78, 128, 83, 83, 72, 69, - 128, 83, 83, 69, 88, 128, 83, 83, 69, 80, 128, 83, 83, 69, 69, 128, 83, - 83, 65, 88, 128, 83, 83, 65, 85, 128, 83, 83, 65, 84, 128, 83, 83, 65, - 80, 128, 83, 83, 65, 78, 71, 89, 69, 83, 73, 69, 85, 78, 71, 128, 83, 83, - 65, 78, 71, 89, 69, 79, 82, 73, 78, 72, 73, 69, 85, 72, 128, 83, 83, 65, - 78, 71, 84, 73, 75, 69, 85, 84, 45, 80, 73, 69, 85, 80, 128, 83, 83, 65, - 78, 71, 84, 73, 75, 69, 85, 84, 128, 83, 83, 65, 78, 71, 84, 72, 73, 69, - 85, 84, 72, 128, 83, 83, 65, 78, 71, 83, 73, 79, 83, 45, 84, 73, 75, 69, - 85, 84, 128, 83, 83, 65, 78, 71, 83, 73, 79, 83, 45, 80, 73, 69, 85, 80, - 128, 83, 83, 65, 78, 71, 83, 73, 79, 83, 45, 75, 73, 89, 69, 79, 75, 128, - 83, 83, 65, 78, 71, 83, 73, 79, 83, 128, 83, 83, 65, 78, 71, 82, 73, 69, - 85, 76, 45, 75, 72, 73, 69, 85, 75, 72, 128, 83, 83, 65, 78, 71, 82, 73, - 69, 85, 76, 128, 83, 83, 65, 78, 71, 80, 73, 69, 85, 80, 128, 83, 83, 65, - 78, 71, 78, 73, 69, 85, 78, 128, 83, 83, 65, 78, 71, 77, 73, 69, 85, 77, - 128, 83, 83, 65, 78, 71, 73, 69, 85, 78, 71, 128, 83, 83, 65, 78, 71, 72, - 73, 69, 85, 72, 128, 83, 83, 65, 78, 71, 67, 73, 69, 85, 67, 45, 72, 73, - 69, 85, 72, 128, 83, 83, 65, 78, 71, 67, 73, 69, 85, 67, 128, 83, 83, 65, - 78, 71, 65, 82, 65, 69, 65, 128, 83, 83, 65, 73, 128, 83, 83, 65, 65, - 128, 83, 83, 51, 128, 83, 83, 50, 128, 83, 82, 69, 68, 78, 197, 83, 82, - 128, 83, 81, 85, 73, 83, 200, 83, 81, 85, 73, 82, 82, 69, 204, 83, 81, - 85, 73, 71, 71, 76, 197, 83, 81, 85, 73, 68, 128, 83, 81, 85, 69, 69, 90, - 69, 68, 128, 83, 81, 85, 69, 69, 90, 197, 83, 81, 85, 65, 212, 83, 81, - 85, 65, 82, 69, 83, 128, 83, 81, 85, 65, 82, 69, 68, 128, 83, 81, 85, 65, - 82, 69, 128, 83, 80, 89, 128, 83, 80, 87, 65, 128, 83, 80, 85, 78, 71, - 211, 83, 80, 82, 79, 85, 84, 128, 83, 80, 82, 73, 78, 71, 83, 128, 83, - 80, 82, 73, 78, 71, 128, 83, 80, 82, 69, 67, 72, 71, 69, 83, 65, 78, 199, - 83, 80, 82, 69, 65, 68, 128, 83, 80, 82, 69, 65, 196, 83, 80, 79, 85, 84, - 73, 78, 199, 83, 80, 79, 84, 128, 83, 80, 79, 82, 84, 211, 83, 80, 79, - 79, 78, 128, 83, 80, 79, 79, 204, 83, 80, 79, 78, 71, 69, 128, 83, 80, - 79, 128, 83, 80, 76, 73, 84, 84, 73, 78, 199, 83, 80, 76, 73, 84, 128, - 83, 80, 76, 73, 212, 83, 80, 76, 65, 89, 69, 68, 128, 83, 80, 76, 65, 83, - 72, 73, 78, 199, 83, 80, 73, 82, 73, 84, 85, 211, 83, 80, 73, 82, 73, 84, - 128, 83, 80, 73, 82, 73, 212, 83, 80, 73, 82, 65, 78, 84, 128, 83, 80, - 73, 82, 65, 76, 128, 83, 80, 73, 82, 65, 204, 83, 80, 73, 78, 69, 128, - 83, 80, 73, 68, 69, 82, 217, 83, 80, 73, 68, 69, 82, 128, 83, 80, 73, 68, - 69, 210, 83, 80, 73, 67, 69, 128, 83, 80, 73, 128, 83, 80, 72, 69, 82, - 73, 67, 65, 204, 83, 80, 69, 83, 77, 73, 76, 207, 83, 80, 69, 78, 212, - 83, 80, 69, 69, 68, 66, 79, 65, 84, 128, 83, 80, 69, 69, 67, 72, 128, 83, - 80, 69, 69, 67, 200, 83, 80, 69, 67, 73, 65, 76, 128, 83, 80, 69, 65, 82, - 128, 83, 80, 69, 65, 75, 73, 78, 199, 83, 80, 69, 65, 75, 69, 82, 128, - 83, 80, 69, 65, 75, 69, 210, 83, 80, 69, 65, 75, 45, 78, 79, 45, 69, 86, - 73, 204, 83, 80, 69, 128, 83, 80, 65, 84, 72, 73, 128, 83, 80, 65, 82, - 75, 76, 73, 78, 199, 83, 80, 65, 82, 75, 76, 69, 83, 128, 83, 80, 65, 82, - 75, 76, 69, 82, 128, 83, 80, 65, 82, 75, 76, 69, 128, 83, 80, 65, 71, 72, - 69, 84, 84, 73, 128, 83, 80, 65, 68, 69, 83, 128, 83, 80, 65, 68, 197, - 83, 80, 65, 67, 73, 78, 199, 83, 80, 65, 67, 197, 83, 80, 65, 128, 83, - 79, 89, 79, 77, 66, 207, 83, 79, 89, 128, 83, 79, 87, 73, 76, 207, 83, - 79, 87, 128, 83, 79, 85, 84, 72, 69, 82, 206, 83, 79, 85, 84, 72, 45, 83, - 76, 65, 86, 69, 217, 83, 79, 85, 84, 200, 83, 79, 85, 82, 67, 69, 128, - 83, 79, 85, 78, 68, 128, 83, 79, 85, 78, 196, 83, 79, 85, 78, 65, 80, - 128, 83, 79, 85, 128, 83, 79, 83, 128, 83, 79, 82, 79, 67, 72, 89, 193, - 83, 79, 82, 73, 128, 83, 79, 82, 193, 83, 79, 81, 128, 83, 79, 79, 206, - 83, 79, 78, 74, 65, 77, 128, 83, 79, 78, 71, 128, 83, 79, 78, 128, 83, - 79, 77, 80, 69, 78, 199, 83, 79, 77, 128, 83, 79, 205, 83, 79, 76, 73, - 68, 85, 83, 128, 83, 79, 76, 73, 68, 85, 211, 83, 79, 76, 73, 196, 83, - 79, 76, 68, 73, 69, 82, 128, 83, 79, 72, 128, 83, 79, 71, 68, 73, 65, - 206, 83, 79, 70, 84, 87, 65, 82, 69, 45, 70, 85, 78, 67, 84, 73, 79, 206, - 83, 79, 70, 84, 78, 69, 83, 83, 128, 83, 79, 70, 84, 66, 65, 76, 76, 128, - 83, 79, 70, 212, 83, 79, 198, 83, 79, 67, 75, 83, 128, 83, 79, 67, 73, - 69, 84, 89, 128, 83, 79, 67, 67, 69, 210, 83, 79, 65, 80, 128, 83, 79, - 65, 128, 83, 79, 45, 55, 128, 83, 79, 45, 54, 128, 83, 79, 45, 53, 128, - 83, 79, 45, 52, 128, 83, 79, 45, 51, 128, 83, 79, 45, 50, 128, 83, 79, - 45, 49, 128, 83, 207, 83, 78, 79, 87, 77, 65, 78, 128, 83, 78, 79, 87, - 77, 65, 206, 83, 78, 79, 87, 70, 76, 65, 75, 69, 128, 83, 78, 79, 87, 66, - 79, 65, 82, 68, 69, 82, 128, 83, 78, 79, 87, 128, 83, 78, 79, 215, 83, - 78, 79, 85, 84, 128, 83, 78, 79, 85, 212, 83, 78, 69, 69, 90, 73, 78, - 199, 83, 78, 65, 208, 83, 78, 65, 75, 69, 128, 83, 78, 65, 75, 197, 83, - 78, 65, 73, 76, 128, 83, 78, 193, 83, 77, 79, 75, 73, 78, 199, 83, 77, - 73, 82, 75, 73, 78, 199, 83, 77, 73, 76, 73, 78, 199, 83, 77, 73, 76, 69, - 128, 83, 77, 73, 76, 197, 83, 77, 69, 65, 82, 128, 83, 77, 65, 83, 200, - 83, 77, 65, 76, 76, 69, 210, 83, 77, 65, 76, 76, 128, 83, 76, 85, 82, - 128, 83, 76, 79, 90, 72, 73, 84, 73, 69, 128, 83, 76, 79, 90, 72, 73, 84, - 73, 197, 83, 76, 79, 87, 76, 89, 128, 83, 76, 79, 87, 128, 83, 76, 79, - 215, 83, 76, 79, 86, 79, 128, 83, 76, 79, 84, 72, 128, 83, 76, 79, 212, - 83, 76, 79, 80, 73, 78, 199, 83, 76, 79, 80, 69, 128, 83, 76, 79, 65, - 206, 83, 76, 73, 78, 71, 128, 83, 76, 73, 71, 72, 84, 76, 217, 83, 76, - 73, 68, 73, 78, 71, 128, 83, 76, 73, 68, 69, 82, 128, 83, 76, 73, 68, 69, - 128, 83, 76, 73, 67, 69, 128, 83, 76, 73, 67, 197, 83, 76, 69, 85, 84, - 200, 83, 76, 69, 69, 80, 217, 83, 76, 69, 69, 80, 73, 78, 199, 83, 76, - 69, 69, 208, 83, 76, 69, 68, 128, 83, 76, 65, 86, 79, 78, 73, 195, 83, - 76, 65, 86, 69, 128, 83, 76, 65, 83, 72, 128, 83, 76, 65, 83, 200, 83, - 76, 65, 78, 84, 69, 196, 83, 75, 87, 65, 128, 83, 75, 87, 128, 83, 75, - 85, 78, 75, 128, 83, 75, 85, 76, 76, 128, 83, 75, 85, 76, 204, 83, 75, - 79, 66, 65, 128, 83, 75, 76, 73, 82, 79, 206, 83, 75, 73, 78, 128, 83, - 75, 73, 69, 82, 128, 83, 75, 201, 83, 75, 69, 87, 69, 196, 83, 75, 65, - 84, 69, 66, 79, 65, 82, 68, 128, 83, 75, 65, 84, 69, 128, 83, 75, 65, 77, - 69, 89, 84, 83, 193, 83, 75, 128, 83, 74, 69, 128, 83, 73, 90, 197, 83, - 73, 88, 84, 89, 45, 70, 79, 85, 82, 84, 72, 83, 128, 83, 73, 88, 84, 89, - 45, 70, 79, 85, 82, 84, 72, 128, 83, 73, 88, 84, 89, 45, 70, 79, 85, 82, - 84, 200, 83, 73, 88, 84, 89, 128, 83, 73, 88, 84, 217, 83, 73, 88, 84, - 72, 83, 128, 83, 73, 88, 84, 72, 211, 83, 73, 88, 84, 72, 128, 83, 73, - 88, 84, 69, 69, 78, 84, 72, 83, 128, 83, 73, 88, 84, 69, 69, 78, 84, 72, - 45, 50, 128, 83, 73, 88, 84, 69, 69, 78, 84, 72, 45, 49, 128, 83, 73, 88, - 84, 69, 69, 78, 84, 72, 128, 83, 73, 88, 84, 69, 69, 78, 84, 200, 83, 73, - 88, 84, 69, 69, 78, 128, 83, 73, 88, 84, 69, 69, 206, 83, 73, 88, 45, 84, - 72, 73, 82, 84, 89, 128, 83, 73, 88, 45, 83, 84, 82, 73, 78, 199, 83, 73, - 88, 45, 80, 69, 82, 45, 69, 205, 83, 73, 88, 45, 76, 73, 78, 197, 83, 73, - 216, 83, 73, 84, 69, 128, 83, 73, 83, 65, 128, 83, 73, 82, 82, 65, 72, - 128, 83, 73, 82, 73, 78, 71, 85, 128, 83, 73, 79, 83, 45, 84, 72, 73, 69, - 85, 84, 72, 128, 83, 73, 79, 83, 45, 83, 83, 65, 78, 71, 83, 73, 79, 83, - 128, 83, 73, 79, 83, 45, 82, 73, 69, 85, 76, 128, 83, 73, 79, 83, 45, 80, - 73, 69, 85, 80, 45, 75, 73, 89, 69, 79, 75, 128, 83, 73, 79, 83, 45, 80, - 72, 73, 69, 85, 80, 72, 128, 83, 73, 79, 83, 45, 80, 65, 78, 83, 73, 79, - 83, 128, 83, 73, 79, 83, 45, 78, 73, 69, 85, 78, 128, 83, 73, 79, 83, 45, - 77, 73, 69, 85, 77, 128, 83, 73, 79, 83, 45, 75, 72, 73, 69, 85, 75, 72, - 128, 83, 73, 79, 83, 45, 75, 65, 80, 89, 69, 79, 85, 78, 80, 73, 69, 85, - 80, 128, 83, 73, 79, 83, 45, 73, 69, 85, 78, 71, 128, 83, 73, 79, 83, 45, - 72, 73, 69, 85, 72, 128, 83, 73, 79, 83, 45, 67, 73, 69, 85, 67, 128, 83, - 73, 79, 83, 45, 67, 72, 73, 69, 85, 67, 72, 128, 83, 73, 79, 211, 83, 73, - 78, 85, 83, 79, 73, 196, 83, 73, 78, 79, 76, 79, 71, 73, 67, 65, 204, 83, - 73, 78, 78, 89, 73, 73, 89, 72, 69, 128, 83, 73, 78, 75, 73, 78, 71, 128, - 83, 73, 78, 71, 76, 69, 45, 83, 72, 73, 70, 84, 45, 51, 128, 83, 73, 78, - 71, 76, 69, 45, 83, 72, 73, 70, 84, 45, 50, 128, 83, 73, 78, 71, 76, 69, - 45, 76, 73, 78, 197, 83, 73, 78, 71, 76, 69, 128, 83, 73, 78, 71, 76, - 197, 83, 73, 78, 71, 65, 65, 84, 128, 83, 73, 78, 197, 83, 73, 78, 68, - 72, 201, 83, 73, 78, 128, 83, 73, 206, 83, 73, 77, 85, 76, 84, 65, 78, - 69, 79, 85, 83, 128, 83, 73, 77, 85, 76, 84, 65, 78, 69, 79, 85, 211, 83, - 73, 77, 80, 76, 73, 70, 73, 69, 196, 83, 73, 77, 73, 76, 65, 82, 128, 83, - 73, 77, 73, 76, 65, 210, 83, 73, 77, 65, 78, 83, 73, 211, 83, 73, 77, 65, - 76, 85, 78, 71, 85, 206, 83, 73, 77, 65, 128, 83, 73, 76, 86, 69, 82, - 128, 83, 73, 76, 75, 128, 83, 73, 76, 73, 81, 85, 193, 83, 73, 76, 72, - 79, 85, 69, 84, 84, 69, 128, 83, 73, 76, 72, 79, 85, 69, 84, 84, 197, 83, - 73, 76, 65, 51, 128, 83, 73, 75, 73, 128, 83, 73, 75, 50, 128, 83, 73, - 75, 178, 83, 73, 71, 78, 83, 128, 83, 73, 71, 77, 79, 73, 196, 83, 73, - 71, 77, 65, 128, 83, 73, 71, 77, 193, 83, 73, 71, 69, 204, 83, 73, 71, - 52, 128, 83, 73, 71, 180, 83, 73, 71, 128, 83, 73, 69, 69, 128, 83, 73, - 68, 69, 87, 65, 89, 211, 83, 73, 68, 69, 128, 83, 73, 68, 197, 83, 73, - 68, 68, 72, 73, 128, 83, 73, 68, 68, 72, 65, 77, 128, 83, 73, 68, 68, 72, - 65, 205, 83, 73, 67, 75, 78, 69, 83, 83, 128, 83, 73, 67, 75, 76, 69, - 128, 83, 73, 66, 197, 83, 73, 65, 128, 83, 73, 45, 54, 128, 83, 73, 45, - 53, 128, 83, 73, 45, 52, 128, 83, 73, 45, 51, 128, 83, 73, 45, 50, 128, - 83, 73, 45, 49, 128, 83, 201, 83, 72, 89, 88, 128, 83, 72, 89, 84, 128, - 83, 72, 89, 82, 88, 128, 83, 72, 89, 82, 128, 83, 72, 89, 80, 128, 83, - 72, 89, 69, 128, 83, 72, 89, 65, 128, 83, 72, 89, 128, 83, 72, 87, 79, - 89, 128, 83, 72, 87, 79, 79, 128, 83, 72, 87, 79, 128, 83, 72, 87, 73, - 73, 128, 83, 72, 87, 73, 128, 83, 72, 87, 69, 128, 83, 72, 87, 197, 83, - 72, 87, 65, 65, 128, 83, 72, 87, 65, 128, 83, 72, 86, 128, 83, 72, 85, - 88, 128, 83, 72, 85, 85, 128, 83, 72, 85, 84, 84, 76, 69, 67, 79, 67, 75, - 128, 83, 72, 85, 84, 128, 83, 72, 85, 82, 88, 128, 83, 72, 85, 82, 128, - 83, 72, 85, 80, 128, 83, 72, 85, 79, 88, 128, 83, 72, 85, 79, 80, 128, - 83, 72, 85, 79, 128, 83, 72, 85, 77, 128, 83, 72, 85, 76, 128, 83, 72, - 85, 70, 70, 76, 197, 83, 72, 85, 69, 81, 128, 83, 72, 85, 69, 78, 83, 72, - 85, 69, 84, 128, 83, 72, 85, 66, 85, 82, 128, 83, 72, 85, 65, 78, 71, 88, - 73, 128, 83, 72, 85, 50, 128, 83, 72, 85, 178, 83, 72, 85, 128, 83, 72, - 84, 65, 80, 73, 67, 128, 83, 72, 84, 65, 128, 83, 72, 82, 85, 71, 128, - 83, 72, 82, 79, 79, 128, 83, 72, 82, 79, 128, 83, 72, 82, 73, 78, 69, - 128, 83, 72, 82, 73, 77, 80, 128, 83, 72, 82, 73, 73, 128, 83, 72, 82, - 73, 128, 83, 72, 82, 65, 65, 128, 83, 72, 82, 65, 128, 83, 72, 79, 89, - 128, 83, 72, 79, 88, 128, 83, 72, 79, 87, 69, 82, 128, 83, 72, 79, 85, - 76, 68, 69, 82, 69, 196, 83, 72, 79, 85, 76, 68, 69, 210, 83, 72, 79, 85, - 128, 83, 72, 79, 84, 128, 83, 72, 79, 82, 84, 83, 128, 83, 72, 79, 82, - 84, 211, 83, 72, 79, 82, 84, 72, 65, 78, 196, 83, 72, 79, 82, 84, 69, 78, - 69, 82, 128, 83, 72, 79, 82, 84, 67, 65, 75, 69, 128, 83, 72, 79, 82, 84, - 45, 84, 87, 73, 71, 45, 89, 82, 128, 83, 72, 79, 82, 84, 45, 84, 87, 73, - 71, 45, 84, 89, 210, 83, 72, 79, 82, 84, 45, 84, 87, 73, 71, 45, 83, 79, - 204, 83, 72, 79, 82, 84, 45, 84, 87, 73, 71, 45, 79, 83, 211, 83, 72, 79, - 82, 84, 45, 84, 87, 73, 71, 45, 78, 65, 85, 196, 83, 72, 79, 82, 84, 45, - 84, 87, 73, 71, 45, 77, 65, 68, 210, 83, 72, 79, 82, 84, 45, 84, 87, 73, - 71, 45, 72, 65, 71, 65, 76, 204, 83, 72, 79, 82, 84, 45, 84, 87, 73, 71, - 45, 66, 74, 65, 82, 75, 65, 206, 83, 72, 79, 82, 84, 45, 84, 87, 73, 71, - 45, 65, 210, 83, 72, 79, 82, 84, 128, 83, 72, 79, 82, 212, 83, 72, 79, - 81, 128, 83, 72, 79, 209, 83, 72, 79, 80, 80, 73, 78, 199, 83, 72, 79, - 80, 128, 83, 72, 79, 79, 84, 73, 78, 199, 83, 72, 79, 79, 84, 128, 83, - 72, 79, 79, 73, 128, 83, 72, 79, 79, 128, 83, 72, 79, 71, 201, 83, 72, - 79, 199, 83, 72, 79, 69, 83, 128, 83, 72, 79, 69, 128, 83, 72, 79, 197, - 83, 72, 79, 67, 75, 69, 196, 83, 72, 79, 65, 128, 83, 72, 79, 128, 83, - 72, 73, 89, 89, 65, 65, 76, 65, 65, 128, 83, 72, 73, 84, 65, 128, 83, 72, - 73, 84, 193, 83, 72, 73, 82, 212, 83, 72, 73, 82, 65, 69, 128, 83, 72, - 73, 82, 128, 83, 72, 73, 210, 83, 72, 73, 81, 128, 83, 72, 73, 78, 84, - 207, 83, 72, 73, 78, 73, 71, 128, 83, 72, 73, 78, 68, 193, 83, 72, 73, - 206, 83, 72, 73, 77, 65, 128, 83, 72, 73, 77, 193, 83, 72, 73, 77, 128, - 83, 72, 73, 205, 83, 72, 73, 73, 78, 128, 83, 72, 73, 73, 128, 83, 72, - 73, 70, 212, 83, 72, 73, 69, 76, 68, 128, 83, 72, 73, 68, 128, 83, 72, - 73, 196, 83, 72, 72, 65, 128, 83, 72, 72, 193, 83, 72, 69, 88, 128, 83, - 72, 69, 86, 65, 128, 83, 72, 69, 85, 88, 128, 83, 72, 69, 85, 79, 81, - 128, 83, 72, 69, 85, 65, 69, 81, 84, 85, 128, 83, 72, 69, 85, 65, 69, 81, - 128, 83, 72, 69, 85, 65, 69, 128, 83, 72, 69, 84, 128, 83, 72, 69, 212, - 83, 72, 69, 83, 72, 76, 65, 77, 128, 83, 72, 69, 83, 72, 73, 71, 128, 83, - 72, 69, 83, 72, 73, 199, 83, 72, 69, 83, 72, 50, 128, 83, 72, 69, 83, 72, - 128, 83, 72, 69, 83, 200, 83, 72, 69, 81, 69, 204, 83, 72, 69, 80, 128, - 83, 72, 69, 78, 128, 83, 72, 69, 76, 76, 128, 83, 72, 69, 76, 204, 83, - 72, 69, 76, 70, 128, 83, 72, 69, 73, 128, 83, 72, 69, 71, 57, 128, 83, - 72, 69, 69, 80, 128, 83, 72, 69, 69, 78, 85, 128, 83, 72, 69, 69, 78, - 128, 83, 72, 69, 69, 206, 83, 72, 69, 69, 128, 83, 72, 69, 45, 71, 79, - 65, 84, 128, 83, 72, 197, 83, 72, 67, 72, 79, 79, 73, 128, 83, 72, 67, - 72, 65, 128, 83, 72, 65, 89, 128, 83, 72, 65, 88, 128, 83, 72, 65, 86, - 73, 89, 65, 78, 73, 128, 83, 72, 65, 86, 73, 65, 206, 83, 72, 65, 86, 69, - 196, 83, 72, 65, 85, 128, 83, 72, 65, 84, 128, 83, 72, 65, 82, 85, 128, - 83, 72, 65, 82, 213, 83, 72, 65, 82, 80, 128, 83, 72, 65, 82, 208, 83, - 72, 65, 82, 75, 128, 83, 72, 65, 82, 65, 68, 193, 83, 72, 65, 82, 65, - 128, 83, 72, 65, 82, 50, 128, 83, 72, 65, 82, 178, 83, 72, 65, 80, 73, - 78, 71, 128, 83, 72, 65, 80, 69, 83, 128, 83, 72, 65, 80, 197, 83, 72, - 65, 80, 128, 83, 72, 65, 78, 71, 128, 83, 72, 65, 78, 128, 83, 72, 65, - 206, 83, 72, 65, 77, 82, 79, 67, 75, 128, 83, 72, 65, 76, 83, 72, 69, 76, - 69, 84, 128, 83, 72, 65, 76, 76, 79, 215, 83, 72, 65, 75, 84, 73, 128, - 83, 72, 65, 75, 73, 78, 71, 128, 83, 72, 65, 75, 73, 78, 199, 83, 72, 65, - 75, 69, 82, 128, 83, 72, 65, 75, 128, 83, 72, 65, 73, 128, 83, 72, 65, - 70, 84, 128, 83, 72, 65, 70, 212, 83, 72, 65, 68, 79, 87, 69, 196, 83, - 72, 65, 68, 69, 196, 83, 72, 65, 68, 69, 128, 83, 72, 65, 68, 197, 83, - 72, 65, 68, 68, 65, 128, 83, 72, 65, 68, 68, 193, 83, 72, 65, 68, 128, - 83, 72, 65, 196, 83, 72, 65, 66, 54, 128, 83, 72, 65, 65, 128, 83, 72, - 65, 54, 128, 83, 72, 65, 182, 83, 72, 65, 51, 128, 83, 72, 65, 179, 83, - 71, 82, 193, 83, 71, 79, 210, 83, 71, 67, 128, 83, 71, 65, 215, 83, 71, - 65, 194, 83, 71, 128, 83, 69, 89, 75, 128, 83, 69, 88, 84, 85, 76, 193, - 83, 69, 88, 84, 73, 76, 69, 128, 83, 69, 88, 84, 65, 78, 84, 45, 54, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, 52, 54, 128, 83, 69, 88, 84, - 65, 78, 84, 45, 52, 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 52, 53, - 128, 83, 69, 88, 84, 65, 78, 84, 45, 52, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 51, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 51, 53, 54, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 51, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, 51, - 52, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 51, 52, 53, 54, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 51, 52, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, - 51, 52, 128, 83, 69, 88, 84, 65, 78, 84, 45, 51, 128, 83, 69, 88, 84, 65, - 78, 84, 45, 50, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 53, 54, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 50, 53, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 50, 52, 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 52, 53, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 50, 52, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 50, 51, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 51, 53, 54, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 50, 51, 53, 128, 83, 69, 88, 84, 65, 78, - 84, 45, 50, 51, 52, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 51, 52, - 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 51, 52, 53, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 50, 51, 52, 128, 83, 69, 88, 84, 65, 78, 84, 45, - 50, 51, 128, 83, 69, 88, 84, 65, 78, 84, 45, 50, 128, 83, 69, 88, 84, 65, - 78, 84, 45, 49, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 53, 54, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 49, 53, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 49, 52, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 52, 53, 54, 128, - 83, 69, 88, 84, 65, 78, 84, 45, 49, 52, 53, 128, 83, 69, 88, 84, 65, 78, - 84, 45, 49, 52, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 51, 54, 128, 83, - 69, 88, 84, 65, 78, 84, 45, 49, 51, 53, 54, 128, 83, 69, 88, 84, 65, 78, - 84, 45, 49, 51, 52, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 51, 52, - 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 51, 52, 53, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 49, 51, 52, 128, 83, 69, 88, 84, 65, 78, 84, 45, - 49, 51, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 54, 128, 83, 69, 88, - 84, 65, 78, 84, 45, 49, 50, 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, - 49, 50, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 52, 54, 128, 83, - 69, 88, 84, 65, 78, 84, 45, 49, 50, 52, 53, 54, 128, 83, 69, 88, 84, 65, - 78, 84, 45, 49, 50, 52, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, - 52, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 51, 54, 128, 83, 69, 88, - 84, 65, 78, 84, 45, 49, 50, 51, 53, 54, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 49, 50, 51, 53, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 51, 52, - 54, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 51, 52, 53, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 49, 50, 51, 52, 128, 83, 69, 88, 84, 65, 78, 84, - 45, 49, 50, 51, 128, 83, 69, 88, 84, 65, 78, 84, 45, 49, 50, 128, 83, 69, - 88, 84, 65, 78, 84, 45, 49, 128, 83, 69, 88, 84, 65, 78, 211, 83, 69, 87, - 73, 78, 199, 83, 69, 86, 69, 82, 65, 78, 67, 69, 128, 83, 69, 86, 69, 78, - 84, 89, 128, 83, 69, 86, 69, 78, 84, 217, 83, 69, 86, 69, 78, 84, 72, - 128, 83, 69, 86, 69, 78, 84, 69, 69, 78, 128, 83, 69, 86, 69, 78, 84, 69, - 69, 206, 83, 69, 86, 69, 78, 45, 84, 72, 73, 82, 84, 89, 128, 83, 69, 86, - 69, 206, 83, 69, 85, 88, 128, 83, 69, 85, 78, 89, 65, 77, 128, 83, 69, - 85, 65, 69, 81, 128, 83, 69, 84, 70, 79, 78, 128, 83, 69, 83, 84, 69, 82, - 84, 73, 85, 211, 83, 69, 83, 81, 85, 73, 81, 85, 65, 68, 82, 65, 84, 69, - 128, 83, 69, 83, 65, 77, 197, 83, 69, 82, 86, 73, 67, 197, 83, 69, 82, - 73, 79, 85, 211, 83, 69, 82, 73, 70, 83, 128, 83, 69, 82, 73, 70, 211, - 83, 69, 82, 73, 70, 128, 83, 69, 81, 85, 69, 78, 84, 73, 65, 76, 128, 83, - 69, 81, 85, 69, 78, 67, 197, 83, 69, 80, 84, 85, 80, 76, 197, 83, 69, 80, - 84, 69, 77, 66, 69, 82, 128, 83, 69, 80, 65, 82, 65, 84, 79, 82, 128, 83, - 69, 80, 65, 82, 65, 84, 79, 210, 83, 69, 80, 65, 82, 65, 84, 69, 196, 83, - 69, 78, 84, 79, 128, 83, 69, 78, 84, 73, 128, 83, 69, 78, 84, 65, 71, 79, - 78, 128, 83, 69, 77, 85, 78, 67, 73, 193, 83, 69, 77, 75, 65, 84, 72, - 128, 83, 69, 77, 75, 128, 83, 69, 77, 73, 86, 79, 87, 69, 204, 83, 69, - 77, 73, 83, 79, 70, 212, 83, 69, 77, 73, 83, 69, 88, 84, 73, 76, 69, 128, - 83, 69, 77, 73, 77, 73, 78, 73, 77, 193, 83, 69, 77, 73, 68, 73, 82, 69, - 67, 212, 83, 69, 77, 73, 67, 79, 76, 79, 78, 128, 83, 69, 77, 73, 67, 79, - 76, 79, 206, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 210, 83, 69, 77, - 73, 67, 73, 82, 67, 76, 197, 83, 69, 77, 73, 66, 82, 69, 86, 73, 211, 83, - 69, 77, 73, 45, 86, 79, 73, 67, 69, 196, 83, 69, 76, 70, 73, 69, 128, 83, - 69, 76, 70, 128, 83, 69, 76, 69, 78, 65, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 57, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 57, 56, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 57, 55, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 57, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 57, 53, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 57, 52, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 57, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 57, 50, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 57, 49, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 57, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 57, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 56, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 56, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 56, 55, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 56, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 56, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 56, 52, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 56, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 56, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 56, 49, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 56, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 55, 57, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 55, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 55, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 55, 54, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 55, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 55, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 55, 51, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 55, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 55, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 55, 48, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, - 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, 56, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 54, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, - 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, 53, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 54, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, - 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, 50, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 54, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, - 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 54, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 53, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 56, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 55, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 53, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 53, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 52, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 53, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 50, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 49, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 53, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 53, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 57, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 52, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 55, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 54, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 52, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 52, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 51, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 52, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 49, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 52, 48, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 51, 57, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 51, 56, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 51, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 51, 54, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 51, 53, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 51, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 51, 51, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 51, 50, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 51, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 51, 48, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 56, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 50, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 53, 54, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 50, 53, 53, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 50, 53, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 53, - 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 53, 50, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 50, 53, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 53, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 53, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, 57, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 50, 52, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, - 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, 54, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 50, 52, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 52, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, 51, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, 50, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 50, 52, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, - 52, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 52, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 50, 51, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 51, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 51, 55, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 51, 54, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 50, 51, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, - 51, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 51, 51, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 50, 51, 50, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 50, 51, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 51, 48, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 51, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 50, 50, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, - 50, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 50, 55, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 50, 50, 54, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 50, 50, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 50, 52, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 50, 51, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 50, 50, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 50, 50, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 50, 48, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 50, 50, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 50, 49, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, 56, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, 55, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 50, 49, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 50, 49, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, 52, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, 51, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 50, 49, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, - 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 49, 48, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 50, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 50, 48, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, 56, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, 55, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 50, 48, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, - 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, 52, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 50, 48, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 50, 48, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, 49, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 48, 48, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 50, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 50, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 57, 57, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 57, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 57, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 57, 54, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 57, 53, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 57, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 57, 51, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 57, 50, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 57, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 57, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 57, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 56, 57, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 56, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, 55, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, 54, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 56, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 56, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, 51, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, 50, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 56, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, - 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 56, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 55, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 55, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, 55, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, 54, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 55, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, - 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, 51, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 49, 55, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 55, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, 48, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 55, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 54, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 54, - 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 54, 55, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 49, 54, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 54, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 54, 52, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 54, 51, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 54, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 54, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 54, 48, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 53, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 53, 56, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 53, 55, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 53, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 53, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 53, 52, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 53, 51, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 53, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 53, 49, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 53, 48, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 53, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 52, 57, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 52, 56, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 52, 55, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 52, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 52, 53, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 52, 52, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 52, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 52, 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 52, 49, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 49, 52, 48, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, 57, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, 56, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 51, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 51, 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, 53, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, 52, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 51, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, - 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 51, 49, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 49, 51, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, 57, 128, 83, - 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, 56, 128, 83, 69, 76, 69, 67, 84, - 79, 82, 45, 49, 50, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, - 54, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, 53, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 49, 50, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 50, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, 50, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 50, 49, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 50, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 50, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 49, 57, 128, 83, 69, 76, - 69, 67, 84, 79, 82, 45, 49, 49, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, - 45, 49, 49, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 49, 54, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 49, 53, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 49, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 49, 51, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 49, 50, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 49, 49, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 49, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 49, 128, - 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 48, 57, 128, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 49, 48, 56, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, - 48, 55, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 48, 54, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 48, 53, 128, 83, 69, 76, 69, 67, 84, 79, - 82, 45, 49, 48, 52, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 48, 51, - 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 48, 50, 128, 83, 69, 76, 69, - 67, 84, 79, 82, 45, 49, 48, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, - 49, 48, 48, 128, 83, 69, 76, 69, 67, 84, 79, 82, 45, 49, 48, 128, 83, 69, - 76, 69, 67, 84, 79, 82, 45, 49, 128, 83, 69, 76, 69, 67, 84, 79, 82, 128, - 83, 69, 76, 69, 67, 84, 79, 210, 83, 69, 76, 69, 67, 84, 69, 196, 83, 69, - 73, 83, 77, 65, 128, 83, 69, 73, 83, 77, 193, 83, 69, 72, 128, 83, 69, - 71, 79, 76, 128, 83, 69, 71, 78, 79, 128, 83, 69, 71, 77, 69, 78, 84, 69, - 196, 83, 69, 71, 77, 69, 78, 84, 128, 83, 69, 69, 86, 128, 83, 69, 69, - 78, 85, 128, 83, 69, 69, 78, 128, 83, 69, 69, 206, 83, 69, 69, 68, 76, - 73, 78, 71, 128, 83, 69, 69, 45, 78, 79, 45, 69, 86, 73, 204, 83, 69, 68, - 78, 65, 128, 83, 69, 67, 84, 79, 82, 128, 83, 69, 67, 84, 73, 79, 78, - 128, 83, 69, 67, 84, 73, 79, 206, 83, 69, 67, 82, 69, 84, 128, 83, 69, - 67, 65, 78, 84, 128, 83, 69, 66, 65, 84, 66, 69, 73, 212, 83, 69, 65, 84, - 128, 83, 69, 65, 76, 128, 83, 69, 65, 71, 85, 76, 204, 83, 69, 45, 53, - 128, 83, 69, 45, 52, 128, 83, 69, 45, 51, 128, 83, 68, 79, 78, 199, 83, - 68, 128, 83, 67, 87, 65, 128, 83, 67, 82, 85, 80, 76, 69, 128, 83, 67, - 82, 79, 76, 76, 128, 83, 67, 82, 73, 80, 84, 128, 83, 67, 82, 69, 87, 68, - 82, 73, 86, 69, 82, 128, 83, 67, 82, 69, 69, 78, 128, 83, 67, 82, 69, 69, - 206, 83, 67, 82, 69, 65, 77, 73, 78, 199, 83, 67, 79, 84, 211, 83, 67, - 79, 82, 80, 73, 85, 83, 128, 83, 67, 79, 82, 80, 73, 79, 78, 128, 83, 67, - 79, 82, 69, 128, 83, 67, 79, 79, 84, 69, 82, 128, 83, 67, 73, 83, 83, 79, - 82, 83, 128, 83, 67, 73, 128, 83, 67, 72, 87, 65, 128, 83, 67, 72, 87, - 193, 83, 67, 72, 82, 79, 69, 68, 69, 82, 128, 83, 67, 72, 79, 79, 76, - 128, 83, 67, 72, 79, 79, 204, 83, 67, 72, 79, 76, 65, 82, 128, 83, 67, - 72, 69, 77, 193, 83, 67, 69, 80, 84, 69, 210, 83, 67, 65, 82, 70, 128, - 83, 67, 65, 78, 68, 73, 67, 85, 83, 128, 83, 67, 65, 78, 68, 73, 67, 85, - 211, 83, 67, 65, 206, 83, 67, 65, 76, 69, 83, 128, 83, 66, 85, 194, 83, - 66, 82, 85, 204, 83, 65, 89, 73, 83, 201, 83, 65, 89, 65, 78, 78, 65, - 128, 83, 65, 89, 128, 83, 65, 88, 79, 80, 72, 79, 78, 69, 128, 83, 65, - 88, 73, 77, 65, 84, 65, 128, 83, 65, 87, 65, 78, 128, 83, 65, 87, 128, - 83, 65, 86, 79, 85, 82, 73, 78, 199, 83, 65, 85, 82, 79, 80, 79, 68, 128, - 83, 65, 85, 82, 65, 83, 72, 84, 82, 193, 83, 65, 85, 73, 76, 128, 83, 65, - 85, 67, 69, 82, 128, 83, 65, 84, 85, 82, 78, 128, 83, 65, 84, 75, 65, 65, - 78, 75, 85, 85, 128, 83, 65, 84, 75, 65, 65, 78, 128, 83, 65, 84, 69, 76, - 76, 73, 84, 69, 128, 83, 65, 84, 69, 76, 76, 73, 84, 197, 83, 65, 84, 67, - 72, 69, 76, 128, 83, 65, 84, 65, 78, 71, 65, 128, 83, 65, 83, 72, 128, - 83, 65, 83, 65, 75, 128, 83, 65, 82, 73, 128, 83, 65, 82, 193, 83, 65, - 82, 128, 83, 65, 81, 128, 83, 65, 80, 65, 128, 83, 65, 78, 89, 79, 79, - 71, 193, 83, 65, 78, 89, 65, 75, 193, 83, 65, 78, 84, 73, 73, 77, 85, - 128, 83, 65, 78, 83, 75, 82, 73, 212, 83, 65, 78, 78, 89, 65, 128, 83, - 65, 78, 71, 65, 50, 128, 83, 65, 78, 68, 87, 73, 67, 72, 128, 83, 65, 78, - 68, 72, 201, 83, 65, 78, 68, 65, 76, 128, 83, 65, 78, 65, 72, 128, 83, - 65, 78, 128, 83, 65, 77, 89, 79, 203, 83, 65, 77, 86, 65, 84, 128, 83, - 65, 77, 80, 73, 128, 83, 65, 77, 80, 72, 65, 79, 128, 83, 65, 77, 75, 65, - 128, 83, 65, 77, 69, 75, 72, 128, 83, 65, 77, 69, 75, 200, 83, 65, 77, - 66, 65, 128, 83, 65, 77, 65, 82, 73, 84, 65, 206, 83, 65, 77, 128, 83, - 65, 76, 85, 84, 73, 78, 199, 83, 65, 76, 84, 73, 82, 69, 128, 83, 65, 76, - 84, 73, 82, 197, 83, 65, 76, 84, 73, 76, 76, 79, 128, 83, 65, 76, 84, 45, - 50, 128, 83, 65, 76, 84, 128, 83, 65, 76, 212, 83, 65, 76, 76, 65, 76, - 76, 65, 72, 213, 83, 65, 76, 76, 65, 76, 76, 65, 72, 79, 213, 83, 65, 76, - 76, 65, 76, 76, 65, 65, 72, 213, 83, 65, 76, 76, 193, 83, 65, 76, 65, - 205, 83, 65, 76, 65, 68, 128, 83, 65, 76, 65, 65, 77, 85, 72, 213, 83, - 65, 76, 65, 128, 83, 65, 76, 45, 65, 77, 77, 79, 78, 73, 65, 67, 128, 83, - 65, 76, 128, 83, 65, 75, 84, 65, 128, 83, 65, 75, 79, 84, 128, 83, 65, - 75, 73, 78, 128, 83, 65, 75, 72, 193, 83, 65, 75, 69, 85, 65, 69, 128, - 83, 65, 75, 197, 83, 65, 74, 68, 65, 72, 128, 83, 65, 73, 76, 66, 79, 65, - 84, 128, 83, 65, 73, 76, 128, 83, 65, 73, 75, 85, 82, 85, 128, 83, 65, - 72, 128, 83, 65, 71, 73, 84, 84, 65, 82, 73, 85, 83, 128, 83, 65, 71, 65, - 128, 83, 65, 71, 128, 83, 65, 199, 83, 65, 70, 72, 65, 128, 83, 65, 70, - 69, 84, 217, 83, 65, 68, 72, 69, 128, 83, 65, 68, 72, 197, 83, 65, 68, - 69, 128, 83, 65, 68, 128, 83, 65, 196, 83, 65, 67, 82, 73, 70, 73, 67, - 73, 65, 204, 83, 65, 65, 73, 128, 83, 65, 65, 68, 72, 85, 128, 83, 65, - 45, 73, 128, 83, 65, 45, 56, 128, 83, 65, 45, 55, 128, 83, 65, 45, 54, - 128, 83, 65, 45, 53, 128, 83, 65, 45, 52, 128, 83, 65, 45, 51, 128, 83, - 65, 45, 50, 128, 83, 65, 45, 49, 128, 83, 48, 52, 54, 128, 83, 48, 52, - 53, 128, 83, 48, 52, 52, 128, 83, 48, 52, 51, 128, 83, 48, 52, 50, 128, - 83, 48, 52, 49, 128, 83, 48, 52, 48, 128, 83, 48, 51, 57, 128, 83, 48, - 51, 56, 128, 83, 48, 51, 55, 128, 83, 48, 51, 54, 128, 83, 48, 51, 53, - 65, 128, 83, 48, 51, 53, 128, 83, 48, 51, 52, 128, 83, 48, 51, 51, 128, - 83, 48, 51, 50, 128, 83, 48, 51, 49, 128, 83, 48, 51, 48, 128, 83, 48, - 50, 57, 128, 83, 48, 50, 56, 128, 83, 48, 50, 55, 128, 83, 48, 50, 54, - 66, 128, 83, 48, 50, 54, 65, 128, 83, 48, 50, 54, 128, 83, 48, 50, 53, - 128, 83, 48, 50, 52, 128, 83, 48, 50, 51, 128, 83, 48, 50, 50, 128, 83, - 48, 50, 49, 128, 83, 48, 50, 48, 128, 83, 48, 49, 57, 128, 83, 48, 49, - 56, 128, 83, 48, 49, 55, 65, 128, 83, 48, 49, 55, 128, 83, 48, 49, 54, - 128, 83, 48, 49, 53, 128, 83, 48, 49, 52, 66, 128, 83, 48, 49, 52, 65, - 128, 83, 48, 49, 52, 128, 83, 48, 49, 51, 128, 83, 48, 49, 50, 128, 83, - 48, 49, 49, 128, 83, 48, 49, 48, 128, 83, 48, 48, 57, 128, 83, 48, 48, - 56, 128, 83, 48, 48, 55, 128, 83, 48, 48, 54, 65, 128, 83, 48, 48, 54, - 128, 83, 48, 48, 53, 128, 83, 48, 48, 52, 128, 83, 48, 48, 51, 128, 83, - 48, 48, 50, 65, 128, 83, 48, 48, 50, 128, 83, 48, 48, 49, 128, 83, 45, - 87, 128, 83, 45, 83, 72, 65, 80, 69, 196, 82, 89, 89, 128, 82, 89, 88, - 128, 82, 89, 84, 128, 82, 89, 82, 88, 128, 82, 89, 82, 128, 82, 89, 80, - 128, 82, 87, 79, 79, 128, 82, 87, 79, 128, 82, 87, 73, 73, 128, 82, 87, - 73, 128, 82, 87, 69, 69, 128, 82, 87, 69, 128, 82, 87, 65, 72, 65, 128, - 82, 87, 65, 65, 128, 82, 87, 65, 128, 82, 85, 88, 128, 82, 85, 85, 66, - 85, 82, 85, 128, 82, 85, 85, 128, 82, 85, 84, 128, 82, 85, 83, 83, 73, - 65, 206, 82, 85, 83, 73, 128, 82, 85, 82, 88, 128, 82, 85, 82, 128, 82, - 85, 80, 73, 73, 128, 82, 85, 80, 69, 197, 82, 85, 80, 128, 82, 85, 79, - 88, 128, 82, 85, 79, 80, 128, 82, 85, 79, 128, 82, 85, 78, 79, 85, 84, - 128, 82, 85, 78, 78, 73, 78, 199, 82, 85, 78, 78, 69, 82, 128, 82, 85, - 78, 73, 195, 82, 85, 78, 128, 82, 85, 77, 201, 82, 85, 77, 65, 201, 82, - 85, 77, 128, 82, 85, 205, 82, 85, 76, 69, 82, 128, 82, 85, 76, 69, 45, - 68, 69, 76, 65, 89, 69, 68, 128, 82, 85, 76, 69, 128, 82, 85, 76, 65, 73, - 128, 82, 85, 75, 75, 65, 75, 72, 65, 128, 82, 85, 73, 83, 128, 82, 85, - 71, 66, 217, 82, 85, 68, 73, 77, 69, 78, 84, 193, 82, 85, 66, 76, 197, - 82, 85, 194, 82, 85, 65, 128, 82, 85, 45, 54, 128, 82, 85, 45, 53, 128, - 82, 85, 45, 52, 128, 82, 85, 45, 51, 128, 82, 85, 45, 50, 128, 82, 85, - 45, 49, 128, 82, 84, 72, 65, 78, 199, 82, 84, 69, 128, 82, 84, 65, 71, - 83, 128, 82, 84, 65, 71, 211, 82, 82, 89, 88, 128, 82, 82, 89, 84, 128, - 82, 82, 89, 82, 88, 128, 82, 82, 89, 82, 128, 82, 82, 89, 80, 128, 82, - 82, 85, 88, 128, 82, 82, 85, 85, 128, 82, 82, 85, 84, 128, 82, 82, 85, - 82, 88, 128, 82, 82, 85, 82, 128, 82, 82, 85, 80, 128, 82, 82, 85, 79, - 88, 128, 82, 82, 85, 79, 128, 82, 82, 85, 128, 82, 82, 82, 65, 128, 82, - 82, 79, 88, 128, 82, 82, 79, 84, 128, 82, 82, 79, 80, 128, 82, 82, 79, - 79, 128, 82, 82, 79, 128, 82, 82, 73, 73, 128, 82, 82, 73, 128, 82, 82, - 69, 88, 128, 82, 82, 69, 84, 128, 82, 82, 69, 80, 128, 82, 82, 69, 72, - 128, 82, 82, 69, 200, 82, 82, 69, 69, 128, 82, 82, 69, 128, 82, 82, 65, - 88, 128, 82, 82, 65, 85, 128, 82, 82, 65, 73, 128, 82, 82, 65, 65, 128, - 82, 79, 87, 66, 79, 65, 84, 128, 82, 79, 85, 78, 68, 69, 196, 82, 79, 85, - 78, 68, 45, 84, 73, 80, 80, 69, 196, 82, 79, 84, 85, 78, 68, 65, 128, 82, - 79, 84, 65, 84, 73, 79, 78, 83, 128, 82, 79, 84, 65, 84, 73, 79, 78, 45, - 87, 65, 76, 76, 80, 76, 65, 78, 197, 82, 79, 84, 65, 84, 73, 79, 78, 45, - 70, 76, 79, 79, 82, 80, 76, 65, 78, 197, 82, 79, 84, 65, 84, 73, 79, 78, - 128, 82, 79, 84, 65, 84, 73, 79, 206, 82, 79, 84, 65, 84, 69, 196, 82, - 79, 83, 72, 128, 82, 79, 83, 69, 84, 84, 69, 128, 82, 79, 83, 69, 128, - 82, 79, 79, 84, 128, 82, 79, 79, 83, 84, 69, 82, 128, 82, 79, 79, 77, - 128, 82, 79, 79, 75, 128, 82, 79, 79, 203, 82, 79, 79, 70, 128, 82, 79, - 77, 65, 78, 73, 65, 206, 82, 79, 77, 65, 206, 82, 79, 77, 128, 82, 79, - 76, 76, 73, 78, 199, 82, 79, 76, 76, 69, 210, 82, 79, 76, 76, 69, 68, 45, - 85, 208, 82, 79, 76, 204, 82, 79, 72, 73, 78, 71, 89, 193, 82, 79, 71, - 79, 77, 128, 82, 79, 71, 128, 82, 79, 196, 82, 79, 67, 75, 69, 84, 128, - 82, 79, 67, 203, 82, 79, 67, 128, 82, 79, 66, 79, 212, 82, 79, 66, 65, - 84, 128, 82, 79, 65, 83, 84, 69, 196, 82, 79, 65, 82, 128, 82, 79, 65, - 128, 82, 79, 45, 54, 128, 82, 79, 45, 53, 128, 82, 79, 45, 52, 128, 82, - 79, 45, 51, 128, 82, 79, 45, 50, 128, 82, 79, 45, 49, 128, 82, 78, 89, - 73, 78, 199, 82, 78, 79, 79, 78, 128, 82, 78, 79, 79, 206, 82, 78, 65, - 205, 82, 77, 84, 128, 82, 76, 79, 128, 82, 76, 77, 128, 82, 76, 73, 128, - 82, 76, 69, 128, 82, 74, 69, 211, 82, 74, 69, 128, 82, 74, 197, 82, 73, - 84, 85, 65, 76, 128, 82, 73, 84, 84, 79, 82, 85, 128, 82, 73, 84, 83, 73, - 128, 82, 73, 83, 73, 78, 199, 82, 73, 83, 72, 128, 82, 73, 82, 65, 128, - 82, 73, 80, 80, 76, 197, 82, 73, 80, 128, 82, 73, 78, 71, 211, 82, 73, - 78, 71, 73, 78, 199, 82, 73, 78, 71, 69, 196, 82, 73, 78, 70, 79, 82, 90, - 65, 78, 68, 79, 128, 82, 73, 206, 82, 73, 77, 71, 66, 65, 128, 82, 73, - 77, 128, 82, 73, 75, 82, 73, 75, 128, 82, 73, 71, 86, 69, 68, 73, 195, - 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 128, 82, 73, 71, 72, 84, 72, 65, - 78, 196, 82, 73, 71, 72, 84, 45, 84, 79, 45, 76, 69, 70, 212, 82, 73, 71, - 72, 84, 45, 83, 73, 68, 197, 82, 73, 71, 72, 84, 45, 83, 72, 65, 68, 79, - 87, 69, 196, 82, 73, 71, 72, 84, 45, 83, 72, 65, 68, 69, 196, 82, 73, 71, - 72, 84, 45, 80, 79, 73, 78, 84, 73, 78, 199, 82, 73, 71, 72, 84, 45, 76, - 73, 71, 72, 84, 69, 196, 82, 73, 71, 72, 84, 45, 72, 65, 78, 68, 69, 196, - 82, 73, 71, 72, 84, 45, 72, 65, 78, 196, 82, 73, 71, 72, 84, 45, 70, 65, - 67, 73, 78, 199, 82, 73, 70, 76, 69, 128, 82, 73, 69, 85, 76, 45, 89, 69, - 83, 73, 69, 85, 78, 71, 128, 82, 73, 69, 85, 76, 45, 89, 69, 79, 82, 73, - 78, 72, 73, 69, 85, 72, 45, 72, 73, 69, 85, 72, 128, 82, 73, 69, 85, 76, - 45, 89, 69, 79, 82, 73, 78, 72, 73, 69, 85, 72, 128, 82, 73, 69, 85, 76, - 45, 84, 73, 75, 69, 85, 84, 45, 72, 73, 69, 85, 72, 128, 82, 73, 69, 85, - 76, 45, 84, 73, 75, 69, 85, 84, 128, 82, 73, 69, 85, 76, 45, 84, 72, 73, - 69, 85, 84, 72, 128, 82, 73, 69, 85, 76, 45, 83, 83, 65, 78, 71, 84, 73, - 75, 69, 85, 84, 128, 82, 73, 69, 85, 76, 45, 83, 83, 65, 78, 71, 83, 73, - 79, 83, 128, 82, 73, 69, 85, 76, 45, 83, 83, 65, 78, 71, 80, 73, 69, 85, - 80, 128, 82, 73, 69, 85, 76, 45, 83, 83, 65, 78, 71, 75, 73, 89, 69, 79, - 75, 128, 82, 73, 69, 85, 76, 45, 83, 73, 79, 83, 128, 82, 73, 69, 85, 76, - 45, 80, 73, 69, 85, 80, 45, 84, 73, 75, 69, 85, 84, 128, 82, 73, 69, 85, - 76, 45, 80, 73, 69, 85, 80, 45, 83, 73, 79, 83, 128, 82, 73, 69, 85, 76, - 45, 80, 73, 69, 85, 80, 45, 80, 72, 73, 69, 85, 80, 72, 128, 82, 73, 69, - 85, 76, 45, 80, 73, 69, 85, 80, 45, 72, 73, 69, 85, 72, 128, 82, 73, 69, - 85, 76, 45, 80, 73, 69, 85, 80, 128, 82, 73, 69, 85, 76, 45, 80, 72, 73, - 69, 85, 80, 72, 128, 82, 73, 69, 85, 76, 45, 80, 65, 78, 83, 73, 79, 83, - 128, 82, 73, 69, 85, 76, 45, 78, 73, 69, 85, 78, 128, 82, 73, 69, 85, 76, - 45, 77, 73, 69, 85, 77, 45, 83, 73, 79, 83, 128, 82, 73, 69, 85, 76, 45, - 77, 73, 69, 85, 77, 45, 75, 73, 89, 69, 79, 75, 128, 82, 73, 69, 85, 76, - 45, 77, 73, 69, 85, 77, 45, 72, 73, 69, 85, 72, 128, 82, 73, 69, 85, 76, - 45, 77, 73, 69, 85, 77, 128, 82, 73, 69, 85, 76, 45, 75, 73, 89, 69, 79, - 75, 45, 83, 73, 79, 83, 128, 82, 73, 69, 85, 76, 45, 75, 73, 89, 69, 79, - 75, 45, 72, 73, 69, 85, 72, 128, 82, 73, 69, 85, 76, 45, 75, 73, 89, 69, - 79, 75, 128, 82, 73, 69, 85, 76, 45, 75, 65, 80, 89, 69, 79, 85, 78, 80, - 73, 69, 85, 80, 128, 82, 73, 69, 85, 76, 45, 72, 73, 69, 85, 72, 128, 82, - 73, 69, 85, 76, 45, 67, 73, 69, 85, 67, 128, 82, 73, 69, 85, 204, 82, 73, - 69, 76, 128, 82, 73, 69, 69, 128, 82, 73, 67, 75, 83, 72, 65, 87, 128, - 82, 73, 67, 69, 77, 128, 82, 73, 67, 69, 128, 82, 73, 67, 197, 82, 73, - 66, 66, 79, 78, 128, 82, 73, 66, 66, 79, 206, 82, 73, 65, 204, 82, 73, - 45, 55, 128, 82, 73, 45, 54, 128, 82, 73, 45, 53, 128, 82, 73, 45, 52, - 128, 82, 73, 45, 51, 128, 82, 73, 45, 50, 128, 82, 73, 45, 49, 128, 82, - 72, 79, 84, 73, 195, 82, 72, 79, 128, 82, 72, 207, 82, 72, 73, 78, 79, - 67, 69, 82, 79, 83, 128, 82, 72, 65, 128, 82, 72, 128, 82, 71, 89, 73, - 78, 71, 83, 128, 82, 71, 89, 65, 78, 128, 82, 71, 89, 193, 82, 69, 86, - 79, 76, 86, 73, 78, 199, 82, 69, 86, 79, 76, 85, 84, 73, 79, 78, 128, 82, - 69, 86, 77, 65, 128, 82, 69, 86, 73, 65, 128, 82, 69, 86, 69, 82, 83, 69, - 68, 45, 83, 67, 72, 87, 65, 128, 82, 69, 86, 69, 82, 83, 69, 68, 128, 82, - 69, 86, 69, 82, 83, 69, 196, 82, 69, 86, 69, 82, 83, 197, 82, 69, 85, 88, - 128, 82, 69, 85, 128, 82, 69, 84, 85, 82, 78, 128, 82, 69, 84, 85, 82, - 206, 82, 69, 84, 82, 79, 70, 76, 69, 216, 82, 69, 84, 82, 69, 65, 84, - 128, 82, 69, 84, 79, 82, 84, 128, 82, 69, 83, 85, 80, 73, 78, 85, 83, - 128, 82, 69, 83, 84, 82, 79, 79, 77, 128, 82, 69, 83, 84, 82, 73, 67, 84, - 69, 196, 82, 69, 83, 84, 128, 82, 69, 83, 80, 79, 78, 83, 69, 128, 82, - 69, 83, 79, 85, 82, 67, 69, 128, 82, 69, 83, 79, 76, 85, 84, 73, 79, 78, - 128, 82, 69, 83, 73, 83, 84, 65, 78, 67, 69, 128, 82, 69, 83, 73, 68, 69, - 78, 67, 69, 128, 82, 69, 83, 72, 45, 65, 89, 73, 78, 45, 68, 65, 76, 69, - 84, 72, 128, 82, 69, 83, 72, 45, 65, 89, 73, 78, 128, 82, 69, 83, 200, - 82, 69, 82, 69, 78, 71, 71, 65, 78, 128, 82, 69, 82, 69, 75, 65, 78, 128, - 82, 69, 80, 82, 69, 83, 69, 78, 84, 128, 82, 69, 80, 76, 65, 67, 69, 77, - 69, 78, 212, 82, 69, 80, 72, 65, 128, 82, 69, 80, 72, 128, 82, 69, 80, - 69, 84, 73, 84, 73, 79, 206, 82, 69, 80, 69, 65, 84, 69, 196, 82, 69, 80, - 69, 65, 84, 128, 82, 69, 80, 69, 65, 212, 82, 69, 80, 65, 89, 65, 128, - 82, 69, 80, 65, 128, 82, 69, 80, 193, 82, 69, 78, 84, 79, 71, 69, 78, - 128, 82, 69, 78, 128, 82, 69, 206, 82, 69, 77, 85, 128, 82, 69, 77, 73, - 78, 68, 69, 210, 82, 69, 77, 69, 68, 89, 128, 82, 69, 76, 73, 71, 73, 79, - 78, 128, 82, 69, 76, 73, 69, 86, 69, 196, 82, 69, 76, 69, 65, 83, 69, - 128, 82, 69, 76, 65, 88, 69, 68, 128, 82, 69, 76, 65, 84, 73, 79, 78, 65, - 204, 82, 69, 76, 65, 84, 73, 79, 78, 128, 82, 69, 76, 65, 65, 128, 82, - 69, 74, 65, 78, 199, 82, 69, 73, 87, 65, 128, 82, 69, 73, 196, 82, 69, - 73, 128, 82, 69, 71, 85, 76, 85, 83, 45, 52, 128, 82, 69, 71, 85, 76, 85, - 83, 45, 51, 128, 82, 69, 71, 85, 76, 85, 83, 45, 50, 128, 82, 69, 71, 85, - 76, 85, 83, 128, 82, 69, 71, 85, 76, 85, 211, 82, 69, 71, 73, 83, 84, 69, - 82, 69, 196, 82, 69, 71, 73, 79, 78, 65, 204, 82, 69, 71, 73, 65, 45, 50, - 128, 82, 69, 71, 73, 65, 128, 82, 69, 70, 79, 82, 77, 69, 196, 82, 69, - 70, 76, 69, 67, 84, 73, 79, 78, 128, 82, 69, 70, 69, 82, 69, 78, 67, 197, - 82, 69, 68, 85, 80, 76, 73, 67, 65, 84, 73, 79, 78, 128, 82, 69, 67, 89, - 67, 76, 73, 78, 199, 82, 69, 67, 89, 67, 76, 69, 196, 82, 69, 67, 84, 73, - 76, 73, 78, 69, 65, 210, 82, 69, 67, 84, 65, 78, 71, 85, 76, 65, 210, 82, - 69, 67, 84, 65, 78, 71, 76, 69, 128, 82, 69, 67, 84, 65, 78, 71, 76, 197, - 82, 69, 67, 82, 69, 65, 84, 73, 79, 78, 65, 204, 82, 69, 67, 79, 82, 68, - 73, 78, 199, 82, 69, 67, 79, 82, 68, 69, 82, 128, 82, 69, 67, 79, 82, 68, - 128, 82, 69, 67, 79, 82, 196, 82, 69, 67, 73, 84, 65, 84, 73, 86, 197, - 82, 69, 67, 69, 80, 84, 73, 86, 197, 82, 69, 67, 69, 73, 86, 69, 82, 128, - 82, 69, 67, 69, 73, 86, 69, 210, 82, 69, 67, 69, 73, 80, 84, 128, 82, 69, - 65, 76, 71, 65, 82, 45, 50, 128, 82, 69, 65, 76, 71, 65, 82, 128, 82, 69, - 65, 72, 77, 85, 75, 128, 82, 69, 65, 68, 73, 78, 199, 82, 69, 65, 67, 72, - 128, 82, 69, 45, 52, 128, 82, 69, 45, 51, 128, 82, 69, 45, 50, 128, 82, - 69, 45, 49, 128, 82, 68, 207, 82, 68, 69, 204, 82, 66, 65, 83, 193, 82, - 65, 90, 83, 69, 75, 65, 128, 82, 65, 90, 79, 82, 128, 82, 65, 89, 83, - 128, 82, 65, 89, 211, 82, 65, 89, 65, 78, 78, 65, 128, 82, 65, 86, 78, - 79, 128, 82, 65, 84, 73, 79, 128, 82, 65, 84, 72, 65, 128, 82, 65, 84, - 72, 193, 82, 65, 84, 65, 128, 82, 65, 84, 128, 82, 65, 83, 87, 65, 68, - 73, 128, 82, 65, 83, 79, 85, 204, 82, 65, 83, 72, 65, 128, 82, 65, 81, - 128, 82, 65, 80, 73, 83, 77, 65, 128, 82, 65, 78, 71, 197, 82, 65, 78, - 65, 128, 82, 65, 78, 128, 82, 65, 77, 211, 82, 65, 77, 66, 65, 84, 128, - 82, 65, 75, 72, 65, 78, 71, 128, 82, 65, 75, 65, 65, 82, 65, 65, 78, 83, - 65, 89, 65, 128, 82, 65, 73, 83, 73, 78, 199, 82, 65, 73, 83, 69, 68, - 128, 82, 65, 73, 83, 69, 196, 82, 65, 73, 78, 66, 79, 87, 128, 82, 65, - 73, 76, 87, 65, 89, 128, 82, 65, 73, 76, 87, 65, 217, 82, 65, 73, 76, - 128, 82, 65, 73, 68, 207, 82, 65, 73, 68, 65, 128, 82, 65, 72, 77, 65, - 84, 85, 76, 76, 65, 200, 82, 65, 72, 73, 77, 65, 72, 85, 205, 82, 65, 72, - 73, 77, 65, 72, 213, 82, 65, 70, 69, 128, 82, 65, 69, 77, 128, 82, 65, - 68, 73, 79, 65, 67, 84, 73, 86, 197, 82, 65, 68, 73, 79, 128, 82, 65, 68, - 73, 207, 82, 65, 68, 201, 82, 65, 68, 128, 82, 65, 196, 82, 65, 67, 81, - 85, 69, 212, 82, 65, 67, 73, 78, 71, 128, 82, 65, 67, 73, 78, 199, 82, - 65, 67, 67, 79, 79, 78, 128, 82, 65, 66, 66, 73, 84, 128, 82, 65, 66, 66, - 73, 212, 82, 65, 66, 128, 82, 65, 65, 73, 128, 82, 65, 51, 128, 82, 65, - 50, 128, 82, 65, 45, 75, 65, 82, 65, 128, 82, 65, 45, 52, 128, 82, 65, - 45, 51, 128, 82, 65, 45, 50, 128, 82, 65, 45, 49, 128, 82, 48, 50, 57, - 128, 82, 48, 50, 56, 128, 82, 48, 50, 55, 128, 82, 48, 50, 54, 128, 82, - 48, 50, 53, 128, 82, 48, 50, 52, 128, 82, 48, 50, 51, 128, 82, 48, 50, - 50, 128, 82, 48, 50, 49, 128, 82, 48, 50, 48, 128, 82, 48, 49, 57, 128, - 82, 48, 49, 56, 128, 82, 48, 49, 55, 128, 82, 48, 49, 54, 65, 128, 82, - 48, 49, 54, 128, 82, 48, 49, 53, 128, 82, 48, 49, 52, 128, 82, 48, 49, - 51, 128, 82, 48, 49, 50, 128, 82, 48, 49, 49, 128, 82, 48, 49, 48, 65, - 128, 82, 48, 49, 48, 128, 82, 48, 48, 57, 128, 82, 48, 48, 56, 128, 82, - 48, 48, 55, 128, 82, 48, 48, 54, 128, 82, 48, 48, 53, 128, 82, 48, 48, - 52, 128, 82, 48, 48, 51, 66, 128, 82, 48, 48, 51, 65, 128, 82, 48, 48, - 51, 128, 82, 48, 48, 50, 65, 128, 82, 48, 48, 50, 128, 82, 48, 48, 49, - 128, 82, 45, 67, 82, 69, 197, 81, 89, 88, 128, 81, 89, 85, 128, 81, 89, - 84, 128, 81, 89, 82, 88, 128, 81, 89, 82, 128, 81, 89, 80, 128, 81, 89, - 79, 128, 81, 89, 73, 128, 81, 89, 69, 69, 128, 81, 89, 69, 128, 81, 89, - 65, 65, 128, 81, 89, 65, 128, 81, 89, 128, 81, 87, 73, 128, 81, 87, 69, - 69, 128, 81, 87, 69, 128, 81, 87, 65, 65, 128, 81, 87, 65, 128, 81, 85, - 88, 128, 81, 85, 86, 128, 81, 85, 85, 86, 128, 81, 85, 85, 128, 81, 85, - 84, 128, 81, 85, 83, 72, 83, 72, 65, 89, 65, 128, 81, 85, 82, 88, 128, - 81, 85, 82, 128, 81, 85, 80, 128, 81, 85, 79, 88, 128, 81, 85, 79, 84, - 197, 81, 85, 79, 84, 65, 84, 73, 79, 206, 81, 85, 79, 84, 128, 81, 85, - 79, 80, 128, 81, 85, 79, 128, 81, 85, 75, 128, 81, 85, 73, 78, 84, 73, - 76, 69, 128, 81, 85, 73, 78, 84, 69, 83, 83, 69, 78, 67, 69, 128, 81, 85, - 73, 78, 68, 73, 67, 69, 83, 73, 77, 193, 81, 85, 73, 78, 67, 85, 78, 88, - 128, 81, 85, 73, 78, 65, 82, 73, 85, 211, 81, 85, 73, 76, 212, 81, 85, - 73, 76, 76, 128, 81, 85, 73, 67, 203, 81, 85, 73, 128, 81, 85, 70, 128, - 81, 85, 69, 83, 84, 73, 79, 78, 69, 196, 81, 85, 69, 83, 84, 73, 79, 78, - 128, 81, 85, 69, 83, 84, 73, 79, 206, 81, 85, 69, 69, 78, 128, 81, 85, - 69, 69, 206, 81, 85, 69, 128, 81, 85, 68, 68, 73, 83, 193, 81, 85, 66, - 85, 84, 83, 128, 81, 85, 65, 84, 69, 82, 78, 73, 79, 206, 81, 85, 65, 82, - 84, 69, 82, 83, 128, 81, 85, 65, 82, 84, 69, 82, 211, 81, 85, 65, 82, 84, - 69, 82, 128, 81, 85, 65, 79, 65, 82, 128, 81, 85, 65, 78, 84, 73, 84, - 217, 81, 85, 65, 68, 82, 85, 80, 76, 197, 81, 85, 65, 68, 82, 65, 78, 84, - 128, 81, 85, 65, 68, 82, 65, 78, 212, 81, 85, 65, 68, 67, 79, 76, 79, 78, - 128, 81, 85, 65, 68, 128, 81, 85, 65, 196, 81, 85, 65, 128, 81, 85, 128, - 81, 208, 81, 79, 88, 128, 81, 79, 84, 128, 81, 79, 80, 72, 128, 81, 79, - 80, 65, 128, 81, 79, 80, 128, 81, 79, 79, 128, 81, 79, 207, 81, 79, 70, - 128, 81, 79, 198, 81, 79, 65, 128, 81, 79, 128, 81, 78, 128, 81, 73, 88, - 128, 81, 73, 84, 83, 65, 128, 81, 73, 84, 128, 81, 73, 80, 128, 81, 73, - 73, 128, 81, 73, 70, 128, 81, 73, 69, 88, 128, 81, 73, 69, 84, 128, 81, - 73, 69, 80, 128, 81, 73, 69, 128, 81, 73, 128, 81, 72, 87, 73, 128, 81, - 72, 87, 69, 69, 128, 81, 72, 87, 69, 128, 81, 72, 87, 65, 65, 128, 81, - 72, 87, 65, 128, 81, 72, 85, 128, 81, 72, 79, 80, 72, 128, 81, 72, 79, - 128, 81, 72, 73, 128, 81, 72, 69, 69, 128, 81, 72, 69, 128, 81, 72, 65, - 85, 128, 81, 72, 65, 65, 128, 81, 72, 65, 128, 81, 71, 65, 128, 81, 69, - 84, 65, 78, 65, 128, 81, 69, 69, 128, 81, 69, 128, 81, 65, 89, 128, 81, - 65, 85, 128, 81, 65, 84, 65, 78, 128, 81, 65, 83, 82, 128, 81, 65, 82, - 78, 69, 217, 81, 65, 82, 128, 81, 65, 81, 128, 81, 65, 80, 72, 128, 81, - 65, 77, 65, 84, 83, 128, 81, 65, 77, 65, 84, 211, 81, 65, 76, 193, 81, - 65, 73, 82, 84, 72, 82, 65, 128, 81, 65, 73, 128, 81, 65, 70, 128, 81, - 65, 198, 81, 65, 68, 77, 65, 128, 81, 65, 65, 73, 128, 81, 65, 65, 70, - 85, 128, 81, 65, 65, 70, 128, 81, 48, 48, 55, 128, 81, 48, 48, 54, 128, - 81, 48, 48, 53, 128, 81, 48, 48, 52, 128, 81, 48, 48, 51, 128, 81, 48, - 48, 50, 128, 81, 48, 48, 49, 128, 80, 90, 128, 80, 89, 88, 128, 80, 89, - 84, 128, 80, 89, 82, 88, 128, 80, 89, 82, 128, 80, 89, 80, 128, 80, 87, - 79, 89, 128, 80, 87, 79, 79, 128, 80, 87, 79, 128, 80, 87, 207, 80, 87, - 73, 73, 128, 80, 87, 73, 128, 80, 87, 69, 69, 128, 80, 87, 69, 128, 80, - 87, 65, 65, 128, 80, 87, 128, 80, 86, 128, 80, 85, 90, 90, 76, 197, 80, - 85, 88, 128, 80, 85, 85, 84, 128, 80, 85, 85, 128, 80, 85, 84, 82, 69, - 70, 65, 67, 84, 73, 79, 78, 128, 80, 85, 84, 78, 65, 89, 65, 128, 80, 85, - 84, 128, 80, 85, 212, 80, 85, 83, 72, 80, 73, 78, 128, 80, 85, 83, 72, - 80, 73, 75, 65, 128, 80, 85, 83, 72, 73, 78, 199, 80, 85, 82, 88, 128, - 80, 85, 82, 83, 69, 128, 80, 85, 82, 80, 76, 197, 80, 85, 82, 78, 65, 77, - 65, 128, 80, 85, 82, 73, 84, 89, 128, 80, 85, 82, 73, 70, 89, 128, 80, - 85, 82, 128, 80, 85, 81, 128, 80, 85, 80, 128, 80, 85, 79, 88, 128, 80, - 85, 79, 80, 128, 80, 85, 79, 128, 80, 85, 78, 71, 65, 65, 77, 128, 80, - 85, 78, 71, 128, 80, 85, 78, 67, 84, 85, 211, 80, 85, 78, 67, 84, 85, 65, - 84, 73, 79, 78, 128, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 206, 80, 85, - 77, 80, 128, 80, 85, 77, 128, 80, 85, 70, 70, 69, 68, 128, 80, 85, 69, - 128, 80, 85, 67, 75, 128, 80, 85, 66, 76, 73, 195, 80, 85, 194, 80, 85, - 65, 81, 128, 80, 85, 65, 69, 128, 80, 85, 65, 67, 72, 85, 197, 80, 85, - 50, 128, 80, 85, 49, 128, 80, 85, 128, 80, 84, 72, 65, 72, 193, 80, 84, - 69, 128, 80, 83, 73, 76, 201, 80, 83, 73, 70, 73, 83, 84, 79, 83, 89, 78, - 65, 71, 77, 65, 128, 80, 83, 73, 70, 73, 83, 84, 79, 80, 65, 82, 65, 75, - 65, 76, 69, 83, 77, 65, 128, 80, 83, 73, 70, 73, 83, 84, 79, 206, 80, 83, - 73, 70, 73, 83, 84, 79, 76, 89, 71, 73, 83, 77, 65, 128, 80, 83, 73, 128, - 80, 83, 65, 76, 84, 69, 210, 80, 83, 128, 80, 82, 79, 86, 69, 128, 80, - 82, 79, 84, 79, 86, 65, 82, 89, 211, 80, 82, 79, 84, 79, 211, 80, 82, 79, - 84, 69, 67, 84, 69, 196, 80, 82, 79, 83, 84, 65, 89, 65, 128, 80, 82, 79, - 83, 71, 69, 71, 82, 65, 77, 77, 69, 78, 73, 128, 80, 82, 79, 83, 69, 82, - 80, 73, 78, 65, 128, 80, 82, 79, 80, 79, 82, 84, 73, 79, 78, 65, 204, 80, - 82, 79, 80, 79, 82, 84, 73, 79, 78, 128, 80, 82, 79, 80, 69, 82, 84, 217, - 80, 82, 79, 80, 69, 76, 76, 69, 210, 80, 82, 79, 79, 70, 128, 80, 82, 79, - 76, 79, 78, 71, 69, 196, 80, 82, 79, 76, 65, 84, 73, 79, 78, 197, 80, 82, - 79, 74, 69, 67, 84, 79, 82, 128, 80, 82, 79, 74, 69, 67, 84, 73, 86, 69, - 128, 80, 82, 79, 74, 69, 67, 84, 73, 79, 78, 128, 80, 82, 79, 72, 73, 66, - 73, 84, 69, 196, 80, 82, 79, 71, 82, 69, 83, 83, 128, 80, 82, 79, 71, 82, - 65, 205, 80, 82, 79, 70, 79, 85, 78, 68, 128, 80, 82, 79, 68, 85, 67, 84, - 128, 80, 82, 79, 68, 85, 67, 212, 80, 82, 79, 66, 73, 78, 199, 80, 82, - 73, 90, 78, 65, 203, 80, 82, 73, 86, 65, 84, 69, 128, 80, 82, 73, 86, 65, - 84, 197, 80, 82, 73, 86, 65, 67, 217, 80, 82, 73, 83, 72, 84, 72, 65, 77, - 65, 84, 82, 193, 80, 82, 73, 78, 84, 83, 128, 80, 82, 73, 78, 84, 69, 82, - 128, 80, 82, 73, 78, 84, 69, 210, 80, 82, 73, 78, 84, 128, 80, 82, 73, - 78, 212, 80, 82, 73, 78, 67, 69, 83, 83, 128, 80, 82, 73, 78, 67, 69, - 128, 80, 82, 73, 77, 69, 128, 80, 82, 73, 77, 197, 80, 82, 69, 86, 73, - 79, 85, 211, 80, 82, 69, 84, 90, 69, 76, 128, 80, 82, 69, 83, 83, 69, - 196, 80, 82, 69, 83, 69, 84, 128, 80, 82, 69, 83, 69, 78, 84, 65, 84, 73, - 79, 206, 80, 82, 69, 83, 67, 82, 73, 80, 84, 73, 79, 206, 80, 82, 69, 80, - 79, 78, 68, 69, 82, 65, 78, 67, 69, 128, 80, 82, 69, 78, 75, 72, 65, 128, - 80, 82, 69, 71, 78, 65, 78, 212, 80, 82, 69, 70, 73, 88, 69, 196, 80, 82, - 69, 70, 65, 67, 197, 80, 82, 69, 67, 73, 80, 73, 84, 65, 84, 69, 128, 80, - 82, 69, 67, 69, 68, 73, 78, 199, 80, 82, 69, 67, 69, 68, 69, 83, 128, 80, - 82, 69, 67, 69, 68, 69, 211, 80, 82, 69, 67, 69, 68, 69, 196, 80, 82, 69, - 67, 69, 68, 69, 128, 80, 82, 69, 67, 69, 68, 197, 80, 82, 65, 89, 69, - 210, 80, 82, 65, 77, 45, 80, 73, 73, 128, 80, 82, 65, 77, 45, 80, 73, - 201, 80, 82, 65, 77, 45, 77, 85, 79, 89, 128, 80, 82, 65, 77, 45, 77, 85, - 79, 217, 80, 82, 65, 77, 45, 66, 85, 79, 78, 128, 80, 82, 65, 77, 45, 66, - 85, 79, 206, 80, 82, 65, 77, 45, 66, 69, 73, 128, 80, 82, 65, 77, 45, 66, - 69, 201, 80, 82, 65, 77, 128, 80, 82, 65, 205, 80, 82, 128, 80, 80, 86, - 128, 80, 80, 77, 128, 80, 80, 65, 128, 80, 79, 89, 128, 80, 79, 88, 128, - 80, 79, 87, 69, 82, 211, 80, 79, 87, 69, 82, 128, 80, 79, 87, 69, 210, - 80, 79, 87, 68, 69, 82, 69, 196, 80, 79, 87, 68, 69, 82, 128, 80, 79, 86, - 89, 83, 72, 69, 128, 80, 79, 86, 89, 83, 72, 197, 80, 79, 86, 79, 68, 78, - 89, 128, 80, 79, 85, 82, 73, 78, 199, 80, 79, 85, 78, 196, 80, 79, 85, - 76, 84, 82, 217, 80, 79, 85, 67, 72, 128, 80, 79, 84, 84, 69, 196, 80, - 79, 84, 65, 84, 79, 128, 80, 79, 84, 65, 66, 76, 197, 80, 79, 212, 80, - 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 206, 80, 79, 83, 84, 66, 79, 88, - 128, 80, 79, 83, 84, 65, 204, 80, 79, 83, 84, 128, 80, 79, 83, 212, 80, - 79, 83, 83, 69, 83, 83, 73, 79, 78, 128, 80, 79, 83, 83, 69, 83, 83, 73, - 79, 206, 80, 79, 83, 73, 84, 73, 79, 78, 83, 128, 80, 79, 83, 73, 84, 73, - 79, 78, 128, 80, 79, 83, 69, 73, 68, 79, 78, 128, 80, 79, 82, 84, 65, 66, - 76, 197, 80, 79, 82, 82, 69, 67, 84, 85, 83, 128, 80, 79, 82, 82, 69, 67, - 84, 85, 211, 80, 79, 80, 80, 73, 78, 199, 80, 79, 80, 80, 69, 82, 128, - 80, 79, 80, 67, 79, 82, 78, 128, 80, 79, 80, 128, 80, 79, 208, 80, 79, - 79, 68, 76, 69, 128, 80, 79, 79, 128, 80, 79, 78, 68, 79, 128, 80, 79, - 206, 80, 79, 77, 77, 69, 69, 128, 80, 79, 77, 77, 69, 197, 80, 79, 76, - 85, 80, 79, 86, 79, 68, 78, 65, 89, 65, 128, 80, 79, 76, 79, 128, 80, 79, - 76, 78, 65, 89, 65, 128, 80, 79, 76, 76, 85, 128, 80, 79, 76, 75, 85, 76, - 73, 90, 77, 89, 128, 80, 79, 76, 73, 83, 72, 128, 80, 79, 76, 73, 83, - 200, 80, 79, 76, 73, 67, 197, 80, 79, 76, 201, 80, 79, 76, 69, 128, 80, - 79, 76, 197, 80, 79, 75, 82, 89, 84, 73, 69, 128, 80, 79, 75, 79, 74, 73, - 128, 80, 79, 73, 78, 84, 211, 80, 79, 73, 78, 84, 79, 128, 80, 79, 73, - 78, 84, 69, 82, 128, 80, 79, 73, 78, 84, 69, 196, 80, 79, 73, 78, 84, - 128, 80, 79, 73, 78, 212, 80, 79, 69, 84, 82, 217, 80, 79, 69, 84, 73, - 195, 80, 79, 68, 86, 69, 82, 84, 75, 65, 128, 80, 79, 68, 67, 72, 65, 83, - 72, 73, 69, 77, 128, 80, 79, 68, 67, 72, 65, 83, 72, 73, 69, 128, 80, 79, - 68, 67, 72, 65, 83, 72, 73, 197, 80, 79, 68, 65, 84, 85, 83, 128, 80, 79, - 67, 75, 69, 212, 80, 79, 65, 128, 80, 207, 80, 78, 69, 85, 77, 65, 84, - 65, 128, 80, 76, 85, 84, 207, 80, 76, 85, 84, 65, 128, 80, 76, 85, 83, - 45, 77, 73, 78, 85, 211, 80, 76, 85, 83, 128, 80, 76, 85, 82, 65, 76, - 128, 80, 76, 85, 78, 71, 69, 82, 128, 80, 76, 85, 77, 69, 196, 80, 76, - 85, 77, 128, 80, 76, 85, 75, 128, 80, 76, 85, 71, 128, 80, 76, 85, 128, - 80, 76, 79, 87, 128, 80, 76, 79, 80, 72, 85, 128, 80, 76, 72, 65, 85, - 128, 80, 76, 69, 84, 72, 82, 79, 78, 128, 80, 76, 69, 65, 68, 73, 78, - 199, 80, 76, 68, 128, 80, 76, 65, 89, 73, 78, 199, 80, 76, 65, 89, 71, - 82, 79, 85, 78, 196, 80, 76, 65, 84, 69, 128, 80, 76, 65, 83, 84, 73, 67, - 83, 128, 80, 76, 65, 78, 84, 128, 80, 76, 65, 78, 69, 84, 128, 80, 76, - 65, 78, 69, 128, 80, 76, 65, 78, 67, 203, 80, 76, 65, 75, 128, 80, 76, - 65, 71, 73, 79, 211, 80, 76, 65, 67, 69, 72, 79, 76, 68, 69, 82, 128, 80, - 76, 65, 67, 69, 72, 79, 76, 68, 69, 210, 80, 76, 65, 67, 197, 80, 76, 65, - 67, 65, 82, 68, 128, 80, 76, 65, 128, 80, 73, 90, 90, 73, 67, 65, 84, 79, - 128, 80, 73, 90, 90, 65, 128, 80, 73, 88, 128, 80, 73, 87, 82, 128, 80, - 73, 84, 67, 72, 70, 79, 82, 75, 128, 80, 73, 84, 67, 72, 70, 79, 82, 203, - 80, 73, 84, 128, 80, 73, 83, 84, 79, 76, 128, 80, 73, 83, 69, 76, 69, 72, - 128, 80, 73, 83, 67, 69, 83, 128, 80, 73, 82, 73, 71, 128, 80, 73, 82, - 73, 199, 80, 73, 82, 73, 69, 69, 78, 128, 80, 73, 82, 65, 67, 89, 128, - 80, 73, 82, 50, 128, 80, 73, 80, 73, 78, 71, 128, 80, 73, 80, 65, 69, 77, - 71, 66, 73, 69, 69, 128, 80, 73, 80, 65, 69, 77, 66, 65, 128, 80, 73, 80, - 128, 80, 73, 78, 87, 72, 69, 69, 204, 80, 73, 78, 203, 80, 73, 78, 69, - 65, 80, 80, 76, 69, 128, 80, 73, 78, 197, 80, 73, 78, 67, 72, 73, 78, - 199, 80, 73, 78, 67, 72, 69, 196, 80, 73, 78, 65, 84, 65, 128, 80, 73, - 78, 65, 82, 66, 79, 82, 65, 83, 128, 80, 73, 76, 76, 128, 80, 73, 76, - 197, 80, 73, 76, 67, 82, 79, 215, 80, 73, 75, 85, 82, 85, 128, 80, 73, - 75, 79, 128, 80, 73, 71, 128, 80, 73, 199, 80, 73, 69, 88, 128, 80, 73, - 69, 85, 80, 45, 84, 72, 73, 69, 85, 84, 72, 128, 80, 73, 69, 85, 80, 45, - 83, 83, 65, 78, 71, 83, 73, 79, 83, 128, 80, 73, 69, 85, 80, 45, 83, 73, - 79, 83, 45, 84, 73, 75, 69, 85, 84, 128, 80, 73, 69, 85, 80, 45, 83, 73, - 79, 83, 45, 84, 72, 73, 69, 85, 84, 72, 128, 80, 73, 69, 85, 80, 45, 83, - 73, 79, 83, 45, 80, 73, 69, 85, 80, 128, 80, 73, 69, 85, 80, 45, 83, 73, - 79, 83, 45, 75, 73, 89, 69, 79, 75, 128, 80, 73, 69, 85, 80, 45, 83, 73, - 79, 83, 45, 67, 73, 69, 85, 67, 128, 80, 73, 69, 85, 80, 45, 82, 73, 69, - 85, 76, 45, 80, 72, 73, 69, 85, 80, 72, 128, 80, 73, 69, 85, 80, 45, 82, - 73, 69, 85, 76, 128, 80, 73, 69, 85, 80, 45, 78, 73, 69, 85, 78, 128, 80, - 73, 69, 85, 80, 45, 77, 73, 69, 85, 77, 128, 80, 73, 69, 85, 80, 45, 75, - 72, 73, 69, 85, 75, 72, 128, 80, 73, 69, 85, 80, 45, 67, 73, 69, 85, 67, - 128, 80, 73, 69, 85, 80, 45, 67, 72, 73, 69, 85, 67, 72, 128, 80, 73, 69, - 85, 208, 80, 73, 69, 84, 128, 80, 73, 69, 80, 128, 80, 73, 69, 69, 84, - 128, 80, 73, 69, 69, 81, 128, 80, 73, 69, 67, 69, 128, 80, 73, 69, 128, - 80, 73, 67, 84, 85, 82, 69, 128, 80, 73, 67, 75, 85, 208, 80, 73, 67, 75, - 69, 84, 128, 80, 73, 67, 75, 128, 80, 73, 65, 83, 85, 84, 79, 82, 85, - 128, 80, 73, 65, 83, 84, 82, 197, 80, 73, 65, 83, 77, 193, 80, 73, 65, - 78, 79, 128, 80, 201, 80, 72, 87, 65, 128, 80, 72, 85, 84, 72, 65, 79, - 128, 80, 72, 85, 210, 80, 72, 85, 78, 71, 128, 80, 72, 82, 65, 83, 69, - 128, 80, 72, 79, 78, 69, 83, 128, 80, 72, 79, 76, 85, 83, 128, 80, 72, - 79, 69, 78, 73, 67, 73, 65, 206, 80, 72, 79, 65, 128, 80, 72, 79, 128, - 80, 72, 207, 80, 72, 78, 65, 69, 203, 80, 72, 73, 78, 84, 72, 85, 128, - 80, 72, 73, 76, 79, 83, 79, 80, 72, 69, 82, 211, 80, 72, 73, 76, 73, 80, - 80, 73, 78, 197, 80, 72, 73, 69, 85, 80, 72, 45, 84, 72, 73, 69, 85, 84, - 72, 128, 80, 72, 73, 69, 85, 80, 72, 45, 83, 73, 79, 83, 128, 80, 72, 73, - 69, 85, 80, 72, 45, 80, 73, 69, 85, 80, 128, 80, 72, 73, 69, 85, 80, 72, - 45, 72, 73, 69, 85, 72, 128, 80, 72, 73, 69, 85, 80, 200, 80, 72, 73, - 128, 80, 72, 201, 80, 72, 69, 69, 128, 80, 72, 69, 128, 80, 72, 65, 83, - 69, 45, 198, 80, 72, 65, 83, 69, 45, 195, 80, 72, 65, 83, 69, 45, 194, - 80, 72, 65, 83, 69, 45, 193, 80, 72, 65, 82, 89, 78, 71, 69, 65, 204, 80, - 72, 65, 82, 128, 80, 72, 65, 78, 128, 80, 72, 65, 77, 128, 80, 72, 65, - 73, 83, 84, 79, 211, 80, 72, 65, 71, 83, 45, 80, 193, 80, 72, 65, 66, - 128, 80, 72, 65, 65, 82, 75, 65, 65, 128, 80, 72, 65, 65, 128, 80, 71, - 128, 80, 70, 128, 80, 69, 85, 88, 128, 80, 69, 85, 84, 65, 69, 128, 80, - 69, 85, 84, 128, 80, 69, 84, 82, 201, 80, 69, 84, 65, 83, 84, 79, 75, 79, - 85, 70, 73, 83, 77, 65, 128, 80, 69, 84, 65, 83, 84, 73, 128, 80, 69, 84, - 65, 83, 77, 65, 128, 80, 69, 84, 65, 76, 76, 69, 196, 80, 69, 83, 79, - 128, 80, 69, 83, 207, 80, 69, 83, 72, 50, 128, 80, 69, 83, 72, 178, 80, - 69, 83, 69, 84, 193, 80, 69, 211, 80, 69, 82, 84, 72, 207, 80, 69, 82, - 83, 80, 69, 67, 84, 73, 86, 69, 128, 80, 69, 82, 83, 79, 78, 65, 204, 80, - 69, 82, 83, 79, 78, 128, 80, 69, 82, 83, 79, 206, 80, 69, 82, 83, 73, 65, - 206, 80, 69, 82, 83, 69, 86, 69, 82, 73, 78, 199, 80, 69, 82, 80, 69, 78, - 68, 73, 67, 85, 76, 65, 82, 128, 80, 69, 82, 80, 69, 78, 68, 73, 67, 85, - 76, 65, 210, 80, 69, 82, 78, 73, 206, 80, 69, 82, 77, 73, 84, 84, 69, - 196, 80, 69, 82, 77, 73, 195, 80, 69, 82, 77, 65, 78, 69, 78, 212, 80, - 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, 128, 80, 69, 82, 73, 83, 80, 79, - 77, 69, 78, 201, 80, 69, 82, 70, 79, 82, 77, 73, 78, 199, 80, 69, 82, 70, - 69, 67, 84, 85, 205, 80, 69, 82, 70, 69, 67, 84, 65, 128, 80, 69, 82, 70, - 69, 67, 84, 193, 80, 69, 82, 69, 86, 79, 68, 75, 65, 128, 80, 69, 82, 69, - 86, 79, 68, 75, 193, 80, 69, 82, 67, 85, 83, 83, 73, 86, 69, 128, 80, 69, - 82, 67, 69, 78, 212, 80, 69, 80, 80, 69, 82, 128, 80, 69, 80, 69, 84, - 128, 80, 69, 80, 69, 212, 80, 69, 79, 82, 84, 200, 80, 69, 79, 80, 76, - 69, 128, 80, 69, 79, 80, 76, 197, 80, 69, 78, 84, 65, 84, 72, 76, 79, 78, - 128, 80, 69, 78, 84, 65, 83, 69, 77, 69, 128, 80, 69, 78, 84, 65, 71, 82, - 65, 77, 128, 80, 69, 78, 84, 65, 71, 79, 78, 128, 80, 69, 78, 83, 85, - 128, 80, 69, 78, 83, 73, 86, 197, 80, 69, 78, 78, 217, 80, 69, 78, 78, - 65, 78, 84, 128, 80, 69, 78, 73, 72, 73, 128, 80, 69, 78, 71, 85, 73, 78, - 128, 80, 69, 78, 71, 75, 65, 76, 128, 80, 69, 78, 69, 84, 82, 65, 84, 73, - 79, 78, 128, 80, 69, 78, 67, 73, 76, 128, 80, 69, 206, 80, 69, 76, 65, - 83, 84, 79, 78, 128, 80, 69, 76, 65, 83, 84, 79, 206, 80, 69, 73, 84, 72, - 128, 80, 69, 72, 69, 72, 128, 80, 69, 72, 69, 200, 80, 69, 72, 128, 80, - 69, 200, 80, 69, 69, 90, 73, 128, 80, 69, 69, 83, 72, 73, 128, 80, 69, - 69, 80, 128, 80, 69, 69, 77, 128, 80, 69, 69, 75, 73, 78, 199, 80, 69, - 69, 73, 128, 80, 69, 69, 128, 80, 69, 68, 69, 83, 84, 82, 73, 65, 78, 83, - 128, 80, 69, 68, 69, 83, 84, 82, 73, 65, 78, 128, 80, 69, 68, 69, 83, 84, - 65, 76, 128, 80, 69, 68, 69, 83, 84, 65, 204, 80, 69, 68, 65, 204, 80, - 69, 65, 78, 85, 84, 83, 128, 80, 69, 65, 75, 211, 80, 69, 65, 67, 79, 67, - 75, 128, 80, 69, 65, 67, 72, 128, 80, 69, 65, 67, 69, 128, 80, 69, 65, - 67, 197, 80, 69, 193, 80, 68, 73, 128, 80, 68, 70, 128, 80, 68, 128, 80, - 67, 128, 80, 65, 90, 69, 82, 128, 80, 65, 89, 69, 82, 79, 75, 128, 80, - 65, 89, 65, 78, 78, 65, 128, 80, 65, 89, 128, 80, 65, 88, 128, 80, 65, - 87, 78, 128, 80, 65, 87, 206, 80, 65, 215, 80, 65, 86, 73, 89, 65, 78, - 73, 128, 80, 65, 85, 83, 197, 80, 65, 85, 75, 128, 80, 65, 85, 128, 80, - 65, 213, 80, 65, 84, 84, 217, 80, 65, 84, 84, 69, 82, 78, 128, 80, 65, - 84, 72, 65, 77, 65, 83, 65, 84, 128, 80, 65, 84, 72, 65, 75, 75, 85, 128, - 80, 65, 84, 200, 80, 65, 84, 65, 75, 128, 80, 65, 84, 65, 72, 128, 80, - 65, 84, 128, 80, 65, 83, 85, 81, 128, 80, 65, 83, 83, 80, 79, 82, 212, - 80, 65, 83, 83, 73, 86, 69, 45, 80, 85, 76, 76, 45, 85, 80, 45, 79, 85, - 84, 80, 85, 212, 80, 65, 83, 83, 73, 86, 69, 45, 80, 85, 76, 76, 45, 68, - 79, 87, 78, 45, 79, 85, 84, 80, 85, 212, 80, 65, 83, 83, 73, 77, 66, 65, - 78, 71, 128, 80, 65, 83, 83, 69, 78, 71, 69, 210, 80, 65, 83, 83, 69, - 196, 80, 65, 83, 72, 84, 65, 128, 80, 65, 83, 72, 65, 69, 128, 80, 65, - 83, 69, 81, 128, 80, 65, 83, 65, 78, 71, 65, 206, 80, 65, 82, 85, 77, - 128, 80, 65, 82, 84, 217, 80, 65, 82, 84, 78, 69, 82, 83, 72, 73, 208, - 80, 65, 82, 84, 73, 65, 76, 76, 89, 45, 82, 69, 67, 89, 67, 76, 69, 196, - 80, 65, 82, 84, 73, 65, 204, 80, 65, 82, 84, 72, 73, 65, 206, 80, 65, 82, - 212, 80, 65, 82, 82, 79, 84, 128, 80, 65, 82, 75, 128, 80, 65, 82, 73, - 67, 72, 79, 78, 128, 80, 65, 82, 69, 83, 84, 73, 71, 77, 69, 78, 79, 206, - 80, 65, 82, 69, 82, 69, 78, 128, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, - 83, 128, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 211, 80, 65, 82, 69, 78, - 84, 72, 69, 83, 69, 211, 80, 65, 82, 65, 80, 72, 82, 65, 83, 197, 80, 65, - 82, 65, 76, 76, 69, 76, 79, 71, 82, 65, 77, 128, 80, 65, 82, 65, 76, 76, - 69, 76, 128, 80, 65, 82, 65, 76, 76, 69, 204, 80, 65, 82, 65, 75, 76, 73, - 84, 73, 75, 73, 128, 80, 65, 82, 65, 75, 76, 73, 84, 73, 75, 201, 80, 65, - 82, 65, 75, 76, 73, 84, 128, 80, 65, 82, 65, 75, 65, 76, 69, 83, 77, 193, - 80, 65, 82, 65, 71, 82, 65, 80, 72, 85, 211, 80, 65, 82, 65, 71, 82, 65, - 80, 72, 79, 83, 128, 80, 65, 82, 65, 71, 82, 65, 80, 72, 128, 80, 65, 82, - 65, 71, 82, 65, 80, 200, 80, 65, 82, 65, 67, 72, 85, 84, 69, 128, 80, 65, - 82, 65, 128, 80, 65, 82, 128, 80, 65, 80, 89, 82, 85, 83, 128, 80, 65, - 80, 69, 82, 67, 76, 73, 80, 83, 128, 80, 65, 80, 69, 82, 67, 76, 73, 80, - 128, 80, 65, 80, 69, 82, 128, 80, 65, 80, 69, 210, 80, 65, 80, 128, 80, - 65, 208, 80, 65, 207, 80, 65, 78, 89, 85, 75, 85, 128, 80, 65, 78, 89, - 73, 75, 85, 128, 80, 65, 78, 89, 69, 67, 69, 75, 128, 80, 65, 78, 89, 65, - 78, 71, 71, 65, 128, 80, 65, 78, 89, 65, 75, 82, 65, 128, 80, 65, 78, 84, - 73, 128, 80, 65, 78, 84, 201, 80, 65, 78, 83, 73, 79, 83, 45, 80, 73, 69, - 85, 80, 128, 80, 65, 78, 83, 73, 79, 83, 45, 75, 65, 80, 89, 69, 79, 85, - 78, 80, 73, 69, 85, 80, 128, 80, 65, 78, 79, 78, 71, 79, 78, 65, 78, 128, - 80, 65, 78, 79, 76, 79, 78, 71, 128, 80, 65, 78, 71, 87, 73, 83, 65, 68, - 128, 80, 65, 78, 71, 82, 65, 78, 71, 75, 69, 80, 128, 80, 65, 78, 71, 79, - 76, 65, 84, 128, 80, 65, 78, 71, 76, 79, 78, 71, 128, 80, 65, 78, 71, 76, - 65, 89, 65, 82, 128, 80, 65, 78, 71, 75, 79, 78, 128, 80, 65, 78, 71, 75, - 65, 84, 128, 80, 65, 78, 71, 72, 85, 76, 85, 128, 80, 65, 78, 71, 128, - 80, 65, 78, 69, 85, 76, 69, 85, 78, 71, 128, 80, 65, 78, 68, 193, 80, 65, - 78, 67, 65, 75, 69, 83, 128, 80, 65, 78, 65, 77, 128, 80, 65, 78, 65, 69, - 76, 65, 69, 78, 71, 128, 80, 65, 78, 128, 80, 65, 206, 80, 65, 77, 85, - 78, 71, 75, 65, 72, 128, 80, 65, 77, 85, 68, 80, 79, 68, 128, 80, 65, 77, - 83, 72, 65, 69, 128, 80, 65, 77, 80, 72, 89, 76, 73, 65, 206, 80, 65, 77, - 73, 78, 71, 75, 65, 76, 128, 80, 65, 77, 69, 80, 69, 84, 128, 80, 65, 77, - 69, 78, 69, 78, 71, 128, 80, 65, 77, 65, 68, 65, 128, 80, 65, 77, 65, 68, - 193, 80, 65, 77, 65, 65, 69, 72, 128, 80, 65, 76, 85, 84, 65, 128, 80, - 65, 76, 79, 67, 72, 75, 65, 128, 80, 65, 76, 77, 89, 82, 69, 78, 197, 80, - 65, 76, 77, 211, 80, 65, 76, 77, 128, 80, 65, 76, 205, 80, 65, 76, 76, - 65, 87, 65, 128, 80, 65, 76, 76, 65, 83, 128, 80, 65, 76, 75, 65, 128, - 80, 65, 76, 201, 80, 65, 76, 69, 84, 84, 69, 128, 80, 65, 76, 65, 85, 78, - 199, 80, 65, 76, 65, 84, 65, 76, 73, 90, 69, 196, 80, 65, 76, 65, 84, 65, - 76, 73, 90, 65, 84, 73, 79, 78, 128, 80, 65, 76, 65, 84, 65, 204, 80, 65, - 75, 80, 65, 203, 80, 65, 73, 89, 65, 78, 78, 79, 73, 128, 80, 65, 73, 82, - 84, 72, 82, 65, 128, 80, 65, 73, 82, 69, 196, 80, 65, 73, 78, 84, 66, 82, - 85, 83, 72, 128, 80, 65, 73, 128, 80, 65, 72, 76, 65, 86, 201, 80, 65, - 72, 128, 80, 65, 71, 79, 68, 65, 128, 80, 65, 71, 69, 83, 128, 80, 65, - 71, 69, 82, 128, 80, 65, 71, 197, 80, 65, 68, 77, 193, 80, 65, 68, 68, - 76, 197, 80, 65, 68, 68, 73, 78, 199, 80, 65, 68, 193, 80, 65, 68, 128, - 80, 65, 67, 75, 73, 78, 71, 128, 80, 65, 67, 75, 65, 71, 69, 128, 80, 65, - 65, 84, 85, 128, 80, 65, 65, 83, 69, 78, 84, 79, 128, 80, 65, 65, 82, 65, - 77, 128, 80, 65, 65, 82, 65, 69, 128, 80, 65, 65, 77, 128, 80, 65, 65, - 73, 128, 80, 65, 65, 45, 80, 73, 76, 76, 65, 128, 80, 65, 65, 128, 80, - 50, 128, 80, 48, 49, 49, 128, 80, 48, 49, 48, 128, 80, 48, 48, 57, 128, - 80, 48, 48, 56, 128, 80, 48, 48, 55, 128, 80, 48, 48, 54, 128, 80, 48, - 48, 53, 128, 80, 48, 48, 52, 128, 80, 48, 48, 51, 65, 128, 80, 48, 48, - 51, 128, 80, 48, 48, 50, 128, 80, 48, 48, 49, 65, 128, 80, 48, 48, 49, - 128, 79, 90, 128, 79, 89, 83, 84, 69, 82, 128, 79, 89, 82, 65, 78, 73, - 83, 77, 193, 79, 89, 65, 78, 78, 65, 128, 79, 88, 73, 65, 128, 79, 88, - 73, 193, 79, 88, 69, 73, 65, 201, 79, 88, 69, 73, 193, 79, 87, 76, 128, - 79, 86, 69, 82, 83, 84, 82, 85, 67, 203, 79, 86, 69, 82, 82, 73, 68, 69, - 128, 79, 86, 69, 82, 76, 79, 78, 199, 79, 86, 69, 82, 76, 73, 78, 69, - 128, 79, 86, 69, 82, 76, 65, 89, 128, 79, 86, 69, 82, 76, 65, 217, 79, - 86, 69, 82, 76, 65, 80, 80, 73, 78, 199, 79, 86, 69, 82, 76, 65, 80, 128, - 79, 86, 69, 82, 76, 65, 73, 68, 128, 79, 86, 69, 82, 76, 65, 73, 196, 79, - 86, 69, 82, 72, 69, 65, 84, 69, 196, 79, 86, 69, 82, 66, 65, 82, 128, 79, - 86, 65, 76, 128, 79, 86, 65, 204, 79, 85, 84, 76, 73, 78, 69, 196, 79, - 85, 84, 76, 73, 78, 69, 128, 79, 85, 84, 69, 210, 79, 85, 84, 66, 79, - 216, 79, 85, 78, 75, 73, 193, 79, 85, 78, 67, 69, 128, 79, 85, 78, 67, - 197, 79, 84, 85, 128, 79, 84, 84, 79, 77, 65, 206, 79, 84, 84, 69, 82, - 128, 79, 84, 84, 65, 86, 193, 79, 84, 84, 128, 79, 84, 83, 69, 67, 72, - 75, 65, 128, 79, 84, 72, 69, 82, 211, 79, 84, 72, 69, 210, 79, 84, 72, - 65, 76, 65, 206, 79, 84, 72, 65, 76, 128, 79, 83, 79, 75, 65, 128, 79, - 83, 79, 75, 193, 79, 83, 77, 65, 78, 89, 193, 79, 83, 67, 128, 79, 83, - 65, 71, 197, 79, 82, 84, 72, 79, 71, 79, 78, 65, 204, 79, 82, 84, 72, 79, - 68, 79, 216, 79, 82, 78, 65, 84, 197, 79, 82, 78, 65, 77, 69, 78, 84, 83, - 128, 79, 82, 78, 65, 77, 69, 78, 84, 128, 79, 82, 78, 65, 77, 69, 78, - 212, 79, 82, 75, 72, 79, 206, 79, 82, 73, 89, 193, 79, 82, 73, 71, 73, - 78, 65, 204, 79, 82, 73, 71, 73, 78, 128, 79, 82, 69, 45, 50, 128, 79, - 82, 68, 73, 78, 65, 204, 79, 82, 68, 69, 210, 79, 82, 67, 85, 83, 128, - 79, 82, 67, 72, 73, 68, 128, 79, 82, 65, 78, 71, 85, 84, 65, 78, 128, 79, - 82, 65, 78, 71, 197, 79, 80, 84, 73, 79, 206, 79, 80, 84, 73, 67, 65, - 204, 79, 80, 80, 82, 69, 83, 83, 73, 79, 78, 128, 79, 80, 80, 79, 83, 73, - 84, 73, 79, 78, 128, 79, 80, 80, 79, 83, 73, 78, 199, 79, 80, 80, 79, 83, - 69, 128, 79, 80, 72, 73, 85, 67, 72, 85, 83, 128, 79, 80, 69, 82, 65, 84, - 79, 82, 128, 79, 80, 69, 82, 65, 84, 79, 210, 79, 80, 69, 82, 65, 84, 73, - 78, 199, 79, 80, 69, 78, 73, 78, 199, 79, 80, 69, 78, 45, 80, 128, 79, - 80, 69, 78, 45, 79, 85, 84, 76, 73, 78, 69, 196, 79, 80, 69, 78, 45, 79, - 128, 79, 80, 69, 78, 45, 207, 79, 80, 69, 78, 45, 72, 69, 65, 68, 69, - 196, 79, 80, 69, 78, 45, 67, 73, 82, 67, 85, 73, 84, 45, 79, 85, 84, 80, - 85, 212, 79, 80, 69, 78, 128, 79, 80, 69, 206, 79, 79, 90, 69, 128, 79, - 79, 89, 65, 78, 78, 65, 128, 79, 79, 85, 128, 79, 79, 77, 85, 128, 79, - 79, 72, 128, 79, 79, 69, 128, 79, 79, 66, 79, 79, 70, 73, 76, 73, 128, - 79, 78, 85, 128, 79, 78, 83, 85, 128, 79, 78, 78, 128, 79, 78, 75, 65, - 82, 128, 79, 78, 73, 79, 78, 128, 79, 78, 69, 83, 69, 76, 70, 128, 79, - 78, 69, 45, 87, 65, 217, 79, 78, 69, 45, 84, 72, 73, 82, 84, 89, 128, 79, - 78, 69, 45, 80, 73, 69, 67, 197, 79, 78, 69, 45, 76, 73, 78, 197, 79, 78, - 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, 84, - 73, 69, 84, 72, 128, 79, 78, 67, 79, 77, 73, 78, 199, 79, 78, 65, 80, - 128, 79, 78, 45, 79, 70, 198, 79, 77, 73, 83, 83, 73, 79, 206, 79, 77, - 73, 67, 82, 79, 78, 128, 79, 77, 73, 67, 82, 79, 206, 79, 77, 69, 84, - 128, 79, 77, 69, 71, 65, 128, 79, 77, 69, 71, 193, 79, 77, 65, 76, 79, - 78, 128, 79, 76, 73, 86, 69, 128, 79, 76, 73, 71, 79, 206, 79, 76, 68, - 128, 79, 75, 84, 207, 79, 75, 65, 82, 65, 128, 79, 75, 65, 82, 193, 79, - 74, 79, 68, 128, 79, 74, 73, 66, 87, 65, 217, 79, 74, 69, 79, 78, 128, - 79, 73, 78, 128, 79, 73, 76, 128, 79, 73, 204, 79, 72, 77, 128, 79, 72, - 205, 79, 71, 82, 69, 128, 79, 71, 79, 78, 69, 75, 128, 79, 71, 79, 78, - 69, 203, 79, 71, 72, 65, 205, 79, 70, 70, 73, 67, 69, 82, 128, 79, 70, - 70, 73, 67, 69, 128, 79, 70, 70, 73, 67, 197, 79, 70, 70, 128, 79, 69, - 89, 128, 79, 69, 82, 128, 79, 69, 75, 128, 79, 69, 69, 128, 79, 68, 69, - 78, 128, 79, 68, 68, 128, 79, 68, 196, 79, 67, 84, 79, 80, 85, 83, 128, - 79, 67, 84, 79, 66, 69, 82, 128, 79, 67, 84, 69, 212, 79, 67, 84, 65, 71, - 79, 78, 65, 204, 79, 67, 84, 65, 71, 79, 78, 128, 79, 67, 210, 79, 67, - 76, 79, 67, 75, 128, 79, 67, 72, 75, 79, 77, 128, 79, 67, 67, 85, 76, 84, - 65, 84, 73, 79, 78, 128, 79, 67, 67, 76, 85, 83, 73, 79, 78, 128, 79, 66, - 83, 84, 82, 85, 67, 84, 73, 79, 78, 128, 79, 66, 83, 69, 82, 86, 69, 210, - 79, 66, 79, 76, 211, 79, 66, 79, 204, 79, 66, 79, 70, 73, 76, 73, 128, - 79, 66, 76, 73, 81, 85, 197, 79, 66, 76, 65, 75, 79, 128, 79, 66, 76, 65, - 67, 72, 75, 79, 128, 79, 66, 74, 69, 67, 212, 79, 66, 69, 76, 85, 83, - 128, 79, 66, 69, 76, 79, 83, 128, 79, 66, 128, 79, 65, 89, 128, 79, 65, - 75, 128, 79, 65, 66, 79, 65, 70, 73, 76, 73, 128, 79, 193, 79, 48, 53, - 49, 128, 79, 48, 53, 48, 66, 128, 79, 48, 53, 48, 65, 128, 79, 48, 53, - 48, 128, 79, 48, 52, 57, 128, 79, 48, 52, 56, 128, 79, 48, 52, 55, 128, - 79, 48, 52, 54, 128, 79, 48, 52, 53, 128, 79, 48, 52, 52, 128, 79, 48, - 52, 51, 128, 79, 48, 52, 50, 128, 79, 48, 52, 49, 128, 79, 48, 52, 48, - 128, 79, 48, 51, 57, 128, 79, 48, 51, 56, 128, 79, 48, 51, 55, 128, 79, - 48, 51, 54, 68, 128, 79, 48, 51, 54, 67, 128, 79, 48, 51, 54, 66, 128, - 79, 48, 51, 54, 65, 128, 79, 48, 51, 54, 128, 79, 48, 51, 53, 128, 79, - 48, 51, 52, 128, 79, 48, 51, 51, 65, 128, 79, 48, 51, 51, 128, 79, 48, - 51, 50, 128, 79, 48, 51, 49, 128, 79, 48, 51, 48, 65, 128, 79, 48, 51, - 48, 128, 79, 48, 50, 57, 65, 128, 79, 48, 50, 57, 128, 79, 48, 50, 56, - 128, 79, 48, 50, 55, 128, 79, 48, 50, 54, 128, 79, 48, 50, 53, 65, 128, - 79, 48, 50, 53, 128, 79, 48, 50, 52, 65, 128, 79, 48, 50, 52, 128, 79, - 48, 50, 51, 128, 79, 48, 50, 50, 128, 79, 48, 50, 49, 128, 79, 48, 50, - 48, 65, 128, 79, 48, 50, 48, 128, 79, 48, 49, 57, 65, 128, 79, 48, 49, - 57, 128, 79, 48, 49, 56, 128, 79, 48, 49, 55, 128, 79, 48, 49, 54, 128, - 79, 48, 49, 53, 128, 79, 48, 49, 52, 128, 79, 48, 49, 51, 128, 79, 48, - 49, 50, 128, 79, 48, 49, 49, 128, 79, 48, 49, 48, 67, 128, 79, 48, 49, - 48, 66, 128, 79, 48, 49, 48, 65, 128, 79, 48, 49, 48, 128, 79, 48, 48, - 57, 128, 79, 48, 48, 56, 128, 79, 48, 48, 55, 128, 79, 48, 48, 54, 70, - 128, 79, 48, 48, 54, 69, 128, 79, 48, 48, 54, 68, 128, 79, 48, 48, 54, - 67, 128, 79, 48, 48, 54, 66, 128, 79, 48, 48, 54, 65, 128, 79, 48, 48, - 54, 128, 79, 48, 48, 53, 65, 128, 79, 48, 48, 53, 128, 79, 48, 48, 52, - 128, 79, 48, 48, 51, 128, 79, 48, 48, 50, 128, 79, 48, 48, 49, 65, 128, - 79, 48, 48, 49, 128, 79, 45, 89, 69, 128, 79, 45, 79, 45, 73, 128, 79, - 45, 69, 128, 78, 90, 89, 88, 128, 78, 90, 89, 84, 128, 78, 90, 89, 82, - 88, 128, 78, 90, 89, 82, 128, 78, 90, 89, 80, 128, 78, 90, 89, 128, 78, - 90, 85, 88, 128, 78, 90, 85, 82, 88, 128, 78, 90, 85, 82, 128, 78, 90, - 85, 81, 128, 78, 90, 85, 80, 128, 78, 90, 85, 79, 88, 128, 78, 90, 85, - 79, 128, 78, 90, 85, 206, 78, 90, 85, 128, 78, 90, 79, 88, 128, 78, 90, - 79, 80, 128, 78, 90, 73, 88, 128, 78, 90, 73, 84, 128, 78, 90, 73, 80, - 128, 78, 90, 73, 69, 88, 128, 78, 90, 73, 69, 80, 128, 78, 90, 73, 69, - 128, 78, 90, 73, 128, 78, 90, 69, 88, 128, 78, 90, 69, 85, 77, 128, 78, - 90, 69, 128, 78, 90, 65, 88, 128, 78, 90, 65, 84, 128, 78, 90, 65, 81, - 128, 78, 90, 65, 80, 128, 78, 90, 65, 128, 78, 90, 193, 78, 89, 87, 65, - 128, 78, 89, 85, 88, 128, 78, 89, 85, 85, 128, 78, 89, 85, 84, 128, 78, - 89, 85, 80, 128, 78, 89, 85, 79, 88, 128, 78, 89, 85, 79, 80, 128, 78, - 89, 85, 79, 128, 78, 89, 85, 78, 128, 78, 89, 85, 69, 128, 78, 89, 85, - 128, 78, 89, 79, 88, 128, 78, 89, 79, 84, 128, 78, 89, 79, 80, 128, 78, - 89, 79, 79, 128, 78, 89, 79, 78, 128, 78, 89, 79, 65, 128, 78, 89, 79, - 128, 78, 89, 74, 65, 128, 78, 89, 73, 88, 128, 78, 89, 73, 84, 128, 78, - 89, 73, 212, 78, 89, 73, 211, 78, 89, 73, 210, 78, 89, 73, 80, 128, 78, - 89, 73, 78, 45, 68, 79, 128, 78, 89, 73, 78, 128, 78, 89, 73, 73, 128, - 78, 89, 73, 69, 88, 128, 78, 89, 73, 69, 84, 128, 78, 89, 73, 69, 80, - 128, 78, 89, 73, 69, 128, 78, 89, 73, 65, 75, 69, 78, 199, 78, 89, 73, - 128, 78, 89, 201, 78, 89, 72, 65, 128, 78, 89, 69, 84, 128, 78, 89, 69, - 212, 78, 89, 69, 78, 128, 78, 89, 69, 72, 128, 78, 89, 69, 200, 78, 89, - 69, 69, 128, 78, 89, 69, 128, 78, 89, 196, 78, 89, 67, 65, 128, 78, 89, - 65, 85, 128, 78, 89, 65, 74, 128, 78, 89, 65, 73, 128, 78, 89, 65, 72, - 128, 78, 89, 65, 69, 77, 65, 69, 128, 78, 89, 65, 65, 128, 78, 87, 79, - 79, 128, 78, 87, 79, 128, 78, 87, 73, 73, 128, 78, 87, 73, 128, 78, 87, - 69, 128, 78, 87, 65, 65, 128, 78, 87, 65, 128, 78, 87, 128, 78, 86, 128, - 78, 85, 88, 128, 78, 85, 85, 78, 128, 78, 85, 85, 128, 78, 85, 84, 73, - 76, 76, 85, 128, 78, 85, 84, 128, 78, 85, 212, 78, 85, 82, 88, 128, 78, - 85, 82, 128, 78, 85, 80, 128, 78, 85, 79, 88, 128, 78, 85, 79, 80, 128, - 78, 85, 79, 128, 78, 85, 78, 85, 90, 128, 78, 85, 78, 85, 218, 78, 85, - 78, 71, 128, 78, 85, 78, 65, 86, 85, 212, 78, 85, 78, 65, 86, 73, 203, - 78, 85, 78, 128, 78, 85, 206, 78, 85, 77, 69, 82, 207, 78, 85, 77, 69, - 82, 65, 84, 79, 210, 78, 85, 77, 69, 82, 65, 204, 78, 85, 77, 66, 69, 82, - 83, 128, 78, 85, 77, 66, 69, 82, 128, 78, 85, 77, 128, 78, 85, 76, 76, - 128, 78, 85, 76, 204, 78, 85, 76, 128, 78, 85, 75, 84, 65, 128, 78, 85, - 75, 84, 193, 78, 85, 69, 78, 71, 128, 78, 85, 69, 128, 78, 85, 66, 73, - 65, 206, 78, 85, 65, 69, 128, 78, 85, 49, 49, 128, 78, 85, 49, 177, 78, - 85, 48, 50, 50, 65, 128, 78, 85, 48, 50, 50, 128, 78, 85, 48, 50, 49, - 128, 78, 85, 48, 50, 48, 128, 78, 85, 48, 49, 57, 128, 78, 85, 48, 49, - 56, 65, 128, 78, 85, 48, 49, 56, 128, 78, 85, 48, 49, 55, 128, 78, 85, - 48, 49, 54, 128, 78, 85, 48, 49, 53, 128, 78, 85, 48, 49, 52, 128, 78, - 85, 48, 49, 51, 128, 78, 85, 48, 49, 50, 128, 78, 85, 48, 49, 49, 65, - 128, 78, 85, 48, 49, 49, 128, 78, 85, 48, 49, 48, 65, 128, 78, 85, 48, - 49, 48, 128, 78, 85, 48, 48, 57, 128, 78, 85, 48, 48, 56, 128, 78, 85, - 48, 48, 55, 128, 78, 85, 48, 48, 54, 128, 78, 85, 48, 48, 53, 128, 78, - 85, 48, 48, 52, 128, 78, 85, 48, 48, 51, 128, 78, 85, 48, 48, 50, 128, - 78, 85, 48, 48, 49, 128, 78, 85, 45, 51, 128, 78, 85, 45, 50, 128, 78, - 85, 45, 49, 128, 78, 84, 88, 73, 86, 128, 78, 84, 88, 65, 128, 78, 84, - 85, 85, 128, 78, 84, 85, 77, 128, 78, 84, 85, 74, 128, 78, 84, 213, 78, - 84, 83, 65, 85, 128, 78, 84, 83, 65, 128, 78, 84, 79, 81, 80, 69, 78, - 128, 78, 84, 79, 71, 128, 78, 84, 79, 199, 78, 84, 73, 69, 197, 78, 84, - 72, 65, 85, 128, 78, 84, 69, 85, 78, 71, 66, 65, 128, 78, 84, 69, 85, 77, - 128, 78, 84, 69, 78, 128, 78, 84, 69, 69, 128, 78, 84, 65, 80, 128, 78, - 84, 65, 208, 78, 84, 65, 65, 128, 78, 84, 65, 128, 78, 83, 85, 79, 212, - 78, 83, 85, 78, 128, 78, 83, 85, 77, 128, 78, 83, 79, 77, 128, 78, 83, - 73, 69, 69, 84, 128, 78, 83, 73, 69, 69, 80, 128, 78, 83, 73, 69, 69, - 128, 78, 83, 72, 85, 84, 128, 78, 83, 72, 85, 212, 78, 83, 72, 85, 79, - 80, 128, 78, 83, 72, 85, 69, 128, 78, 83, 72, 73, 69, 69, 128, 78, 83, - 72, 69, 69, 128, 78, 83, 72, 65, 81, 128, 78, 83, 72, 65, 128, 78, 83, - 69, 85, 65, 69, 78, 128, 78, 83, 69, 78, 128, 78, 83, 65, 128, 78, 82, - 89, 88, 128, 78, 82, 89, 84, 128, 78, 82, 89, 82, 88, 128, 78, 82, 89, - 82, 128, 78, 82, 89, 80, 128, 78, 82, 89, 128, 78, 82, 85, 88, 128, 78, - 82, 85, 84, 128, 78, 82, 85, 82, 88, 128, 78, 82, 85, 82, 128, 78, 82, - 85, 80, 128, 78, 82, 85, 65, 128, 78, 82, 85, 128, 78, 82, 79, 88, 128, - 78, 82, 79, 80, 128, 78, 82, 79, 128, 78, 82, 69, 88, 128, 78, 82, 69, - 84, 128, 78, 82, 69, 211, 78, 82, 69, 80, 128, 78, 82, 69, 128, 78, 82, - 65, 88, 128, 78, 82, 65, 84, 128, 78, 82, 65, 80, 128, 78, 82, 65, 128, - 78, 81, 73, 71, 128, 78, 81, 65, 128, 78, 80, 76, 65, 128, 78, 80, 65, - 128, 78, 79, 90, 72, 75, 65, 128, 78, 79, 89, 128, 78, 79, 88, 128, 78, - 79, 87, 67, 128, 78, 79, 86, 73, 76, 69, 128, 78, 79, 86, 69, 77, 66, 69, - 82, 128, 78, 79, 84, 84, 79, 128, 78, 79, 84, 69, 83, 128, 78, 79, 84, - 69, 72, 69, 65, 68, 128, 78, 79, 84, 69, 72, 69, 65, 196, 78, 79, 84, 69, - 66, 79, 79, 75, 128, 78, 79, 84, 69, 66, 79, 79, 203, 78, 79, 84, 69, - 128, 78, 79, 84, 197, 78, 79, 84, 67, 72, 69, 196, 78, 79, 84, 67, 72, - 128, 78, 79, 84, 65, 84, 73, 79, 206, 78, 79, 84, 128, 78, 79, 212, 78, - 79, 83, 69, 128, 78, 79, 83, 197, 78, 79, 82, 84, 72, 87, 69, 83, 212, - 78, 79, 82, 84, 72, 69, 82, 206, 78, 79, 82, 84, 72, 69, 65, 83, 84, 45, - 80, 79, 73, 78, 84, 73, 78, 199, 78, 79, 82, 77, 65, 204, 78, 79, 82, 68, - 73, 195, 78, 79, 210, 78, 79, 80, 128, 78, 79, 79, 78, 85, 128, 78, 79, - 79, 128, 78, 79, 78, 70, 79, 82, 75, 73, 78, 71, 128, 78, 79, 78, 45, 80, - 79, 84, 65, 66, 76, 197, 78, 79, 78, 45, 74, 79, 73, 78, 69, 82, 128, 78, - 79, 78, 45, 66, 82, 69, 65, 75, 73, 78, 199, 78, 79, 78, 128, 78, 79, 77, - 73, 83, 77, 193, 78, 79, 77, 73, 78, 65, 204, 78, 79, 75, 72, 85, 75, - 128, 78, 79, 68, 69, 128, 78, 79, 65, 128, 78, 79, 45, 66, 82, 69, 65, - 203, 78, 79, 45, 53, 128, 78, 79, 45, 52, 128, 78, 79, 45, 51, 128, 78, - 79, 45, 50, 128, 78, 79, 45, 49, 128, 78, 78, 85, 85, 128, 78, 78, 85, - 128, 78, 78, 79, 79, 128, 78, 78, 78, 85, 85, 128, 78, 78, 78, 85, 128, - 78, 78, 78, 79, 79, 128, 78, 78, 78, 79, 128, 78, 78, 78, 73, 73, 128, - 78, 78, 78, 73, 128, 78, 78, 78, 69, 69, 128, 78, 78, 78, 69, 128, 78, - 78, 78, 65, 85, 128, 78, 78, 78, 65, 73, 128, 78, 78, 78, 65, 65, 128, - 78, 78, 78, 65, 128, 78, 78, 78, 128, 78, 78, 72, 65, 128, 78, 78, 71, - 79, 79, 128, 78, 78, 71, 79, 128, 78, 78, 71, 73, 73, 128, 78, 78, 71, - 73, 128, 78, 78, 71, 65, 65, 128, 78, 78, 71, 65, 128, 78, 78, 71, 128, - 78, 78, 66, 83, 80, 128, 78, 77, 128, 78, 76, 65, 85, 128, 78, 76, 48, - 50, 48, 128, 78, 76, 48, 49, 57, 128, 78, 76, 48, 49, 56, 128, 78, 76, - 48, 49, 55, 65, 128, 78, 76, 48, 49, 55, 128, 78, 76, 48, 49, 54, 128, - 78, 76, 48, 49, 53, 128, 78, 76, 48, 49, 52, 128, 78, 76, 48, 49, 51, - 128, 78, 76, 48, 49, 50, 128, 78, 76, 48, 49, 49, 128, 78, 76, 48, 49, - 48, 128, 78, 76, 48, 48, 57, 128, 78, 76, 48, 48, 56, 128, 78, 76, 48, - 48, 55, 128, 78, 76, 48, 48, 54, 128, 78, 76, 48, 48, 53, 65, 128, 78, - 76, 48, 48, 53, 128, 78, 76, 48, 48, 52, 128, 78, 76, 48, 48, 51, 128, - 78, 76, 48, 48, 50, 128, 78, 76, 48, 48, 49, 128, 78, 76, 128, 78, 75, - 79, 77, 128, 78, 75, 207, 78, 75, 73, 78, 68, 73, 128, 78, 75, 65, 85, - 128, 78, 75, 65, 65, 82, 65, 69, 128, 78, 75, 65, 128, 78, 74, 89, 88, - 128, 78, 74, 89, 84, 128, 78, 74, 89, 82, 88, 128, 78, 74, 89, 82, 128, - 78, 74, 89, 80, 128, 78, 74, 89, 128, 78, 74, 85, 88, 128, 78, 74, 85, - 82, 88, 128, 78, 74, 85, 82, 128, 78, 74, 85, 81, 65, 128, 78, 74, 85, - 80, 128, 78, 74, 85, 79, 88, 128, 78, 74, 85, 79, 128, 78, 74, 85, 69, - 81, 128, 78, 74, 85, 65, 69, 128, 78, 74, 85, 128, 78, 74, 79, 88, 128, - 78, 74, 79, 84, 128, 78, 74, 79, 80, 128, 78, 74, 79, 79, 128, 78, 74, - 79, 128, 78, 74, 73, 88, 128, 78, 74, 73, 84, 128, 78, 74, 73, 80, 128, - 78, 74, 73, 69, 88, 128, 78, 74, 73, 69, 84, 128, 78, 74, 73, 69, 80, - 128, 78, 74, 73, 69, 69, 128, 78, 74, 73, 69, 128, 78, 74, 73, 128, 78, - 74, 201, 78, 74, 69, 85, 88, 128, 78, 74, 69, 85, 84, 128, 78, 74, 69, - 85, 65, 69, 78, 65, 128, 78, 74, 69, 85, 65, 69, 77, 128, 78, 74, 69, 69, - 69, 69, 128, 78, 74, 69, 69, 128, 78, 74, 69, 197, 78, 74, 69, 128, 78, - 74, 65, 81, 128, 78, 74, 65, 80, 128, 78, 74, 65, 69, 77, 76, 73, 128, - 78, 74, 65, 69, 77, 128, 78, 74, 65, 65, 128, 78, 73, 90, 75, 207, 78, - 73, 88, 128, 78, 73, 84, 82, 69, 128, 78, 73, 83, 65, 71, 128, 78, 73, - 82, 85, 71, 85, 128, 78, 73, 80, 128, 78, 73, 78, 84, 72, 128, 78, 73, - 78, 74, 65, 128, 78, 73, 78, 69, 84, 89, 128, 78, 73, 78, 69, 84, 217, - 78, 73, 78, 69, 84, 69, 69, 78, 128, 78, 73, 78, 69, 84, 69, 69, 206, 78, - 73, 78, 69, 45, 84, 72, 73, 82, 84, 89, 128, 78, 73, 78, 69, 45, 76, 73, - 75, 197, 78, 73, 78, 197, 78, 73, 78, 68, 65, 50, 128, 78, 73, 78, 68, - 65, 178, 78, 73, 78, 57, 128, 78, 73, 78, 128, 78, 73, 77, 128, 78, 73, - 205, 78, 73, 75, 79, 76, 83, 66, 85, 82, 199, 78, 73, 75, 72, 65, 72, 73, - 84, 128, 78, 73, 75, 65, 72, 73, 84, 128, 78, 73, 75, 65, 128, 78, 73, - 72, 83, 72, 86, 65, 83, 65, 128, 78, 73, 71, 73, 68, 65, 77, 73, 78, 128, - 78, 73, 71, 73, 68, 65, 69, 83, 72, 128, 78, 73, 71, 72, 84, 128, 78, 73, - 71, 72, 212, 78, 73, 71, 71, 65, 72, 73, 84, 65, 128, 78, 73, 69, 88, - 128, 78, 73, 69, 85, 78, 45, 84, 73, 75, 69, 85, 84, 128, 78, 73, 69, 85, - 78, 45, 84, 72, 73, 69, 85, 84, 72, 128, 78, 73, 69, 85, 78, 45, 83, 73, - 79, 83, 128, 78, 73, 69, 85, 78, 45, 82, 73, 69, 85, 76, 128, 78, 73, 69, - 85, 78, 45, 80, 73, 69, 85, 80, 128, 78, 73, 69, 85, 78, 45, 80, 65, 78, - 83, 73, 79, 83, 128, 78, 73, 69, 85, 78, 45, 75, 73, 89, 69, 79, 75, 128, - 78, 73, 69, 85, 78, 45, 72, 73, 69, 85, 72, 128, 78, 73, 69, 85, 78, 45, - 67, 73, 69, 85, 67, 128, 78, 73, 69, 85, 78, 45, 67, 72, 73, 69, 85, 67, - 72, 128, 78, 73, 69, 85, 206, 78, 73, 69, 80, 128, 78, 73, 69, 128, 78, - 73, 66, 128, 78, 73, 65, 128, 78, 73, 50, 128, 78, 73, 45, 84, 69, 128, - 78, 73, 45, 55, 128, 78, 73, 45, 54, 128, 78, 73, 45, 53, 128, 78, 73, - 45, 52, 128, 78, 73, 45, 51, 128, 78, 73, 45, 50, 128, 78, 73, 45, 49, - 128, 78, 72, 85, 69, 128, 78, 72, 74, 65, 128, 78, 72, 65, 89, 128, 78, - 72, 128, 78, 71, 89, 69, 128, 78, 71, 86, 69, 128, 78, 71, 85, 85, 128, - 78, 71, 85, 79, 88, 128, 78, 71, 85, 79, 84, 128, 78, 71, 85, 79, 128, - 78, 71, 85, 65, 78, 128, 78, 71, 85, 65, 69, 84, 128, 78, 71, 85, 65, 69, - 128, 78, 71, 79, 88, 128, 78, 71, 79, 85, 128, 78, 71, 79, 213, 78, 71, - 79, 84, 128, 78, 71, 79, 81, 128, 78, 71, 79, 80, 128, 78, 71, 79, 78, - 128, 78, 71, 79, 77, 128, 78, 71, 79, 69, 72, 128, 78, 71, 79, 69, 200, - 78, 71, 207, 78, 71, 75, 89, 69, 69, 128, 78, 71, 75, 87, 65, 69, 78, - 128, 78, 71, 75, 85, 80, 128, 78, 71, 75, 85, 78, 128, 78, 71, 75, 85, - 77, 128, 78, 71, 75, 85, 69, 78, 90, 69, 85, 77, 128, 78, 71, 75, 85, - 197, 78, 71, 75, 73, 78, 68, 201, 78, 71, 75, 73, 69, 69, 128, 78, 71, - 75, 69, 85, 88, 128, 78, 71, 75, 69, 85, 82, 73, 128, 78, 71, 75, 69, 85, - 65, 69, 81, 128, 78, 71, 75, 69, 85, 65, 69, 77, 128, 78, 71, 75, 65, 81, - 128, 78, 71, 75, 65, 80, 128, 78, 71, 75, 65, 65, 77, 73, 128, 78, 71, - 75, 65, 128, 78, 71, 73, 69, 88, 128, 78, 71, 73, 69, 80, 128, 78, 71, - 73, 69, 128, 78, 71, 72, 65, 128, 78, 71, 71, 87, 65, 69, 78, 128, 78, - 71, 71, 85, 82, 65, 69, 128, 78, 71, 71, 85, 80, 128, 78, 71, 71, 85, 79, - 81, 128, 78, 71, 71, 85, 79, 209, 78, 71, 71, 85, 79, 78, 128, 78, 71, - 71, 85, 79, 77, 128, 78, 71, 71, 85, 77, 128, 78, 71, 71, 85, 69, 69, 84, - 128, 78, 71, 71, 85, 65, 69, 83, 72, 65, 197, 78, 71, 71, 85, 65, 69, - 206, 78, 71, 71, 85, 65, 128, 78, 71, 71, 85, 128, 78, 71, 71, 79, 79, - 128, 78, 71, 71, 79, 128, 78, 71, 71, 73, 128, 78, 71, 71, 69, 85, 88, - 128, 78, 71, 71, 69, 85, 65, 69, 84, 128, 78, 71, 71, 69, 85, 65, 69, - 128, 78, 71, 71, 69, 213, 78, 71, 71, 69, 78, 128, 78, 71, 71, 69, 69, - 84, 128, 78, 71, 71, 69, 69, 69, 69, 128, 78, 71, 71, 69, 69, 128, 78, - 71, 71, 69, 128, 78, 71, 71, 65, 80, 128, 78, 71, 71, 65, 65, 77, 65, 69, - 128, 78, 71, 71, 65, 65, 77, 128, 78, 71, 71, 65, 65, 128, 78, 71, 71, - 128, 78, 71, 69, 88, 128, 78, 71, 69, 85, 82, 69, 85, 84, 128, 78, 71, - 69, 80, 128, 78, 71, 69, 78, 128, 78, 71, 69, 69, 128, 78, 71, 69, 65, - 68, 65, 76, 128, 78, 71, 65, 88, 128, 78, 71, 65, 85, 128, 78, 71, 65, - 84, 128, 78, 71, 65, 211, 78, 71, 65, 81, 128, 78, 71, 65, 80, 128, 78, - 71, 65, 78, 71, 85, 128, 78, 71, 65, 78, 128, 78, 71, 65, 73, 128, 78, - 71, 65, 72, 128, 78, 71, 65, 65, 73, 128, 78, 71, 193, 78, 70, 128, 78, - 69, 88, 212, 78, 69, 88, 128, 78, 69, 87, 83, 80, 65, 80, 69, 82, 128, - 78, 69, 87, 76, 73, 78, 69, 128, 78, 69, 87, 76, 73, 78, 197, 78, 69, 87, - 193, 78, 69, 87, 128, 78, 69, 215, 78, 69, 85, 84, 82, 65, 76, 128, 78, - 69, 85, 84, 82, 65, 204, 78, 69, 85, 84, 69, 82, 128, 78, 69, 84, 87, 79, - 82, 75, 69, 196, 78, 69, 212, 78, 69, 83, 84, 73, 78, 199, 78, 69, 83, - 84, 69, 196, 78, 69, 83, 84, 128, 78, 69, 83, 212, 78, 69, 83, 83, 85, - 83, 128, 78, 69, 82, 196, 78, 69, 81, 85, 68, 65, 65, 128, 78, 69, 80, - 84, 85, 78, 69, 128, 78, 69, 80, 84, 85, 78, 197, 78, 69, 80, 79, 83, 84, - 79, 89, 65, 78, 78, 65, 89, 65, 128, 78, 69, 80, 128, 78, 69, 79, 128, - 78, 69, 207, 78, 69, 78, 79, 69, 128, 78, 69, 78, 65, 78, 79, 128, 78, - 69, 78, 128, 78, 69, 77, 75, 65, 128, 78, 69, 76, 128, 78, 69, 73, 84, - 72, 69, 210, 78, 69, 71, 65, 84, 73, 86, 197, 78, 69, 71, 65, 84, 73, 79, - 206, 78, 69, 71, 65, 84, 69, 196, 78, 69, 69, 68, 76, 69, 128, 78, 69, - 67, 75, 84, 73, 69, 128, 78, 69, 67, 75, 128, 78, 69, 66, 69, 78, 83, 84, - 73, 77, 77, 69, 128, 78, 69, 45, 75, 79, 128, 78, 68, 85, 88, 128, 78, - 68, 85, 84, 128, 78, 68, 85, 82, 88, 128, 78, 68, 85, 82, 128, 78, 68, - 85, 80, 128, 78, 68, 85, 78, 128, 78, 68, 213, 78, 68, 79, 88, 128, 78, - 68, 79, 84, 128, 78, 68, 79, 80, 128, 78, 68, 79, 79, 128, 78, 68, 79, - 78, 128, 78, 68, 79, 77, 66, 85, 128, 78, 68, 79, 76, 197, 78, 68, 73, - 88, 128, 78, 68, 73, 84, 128, 78, 68, 73, 81, 128, 78, 68, 73, 80, 128, - 78, 68, 73, 69, 88, 128, 78, 68, 73, 69, 128, 78, 68, 73, 68, 65, 128, - 78, 68, 73, 65, 81, 128, 78, 68, 69, 88, 128, 78, 68, 69, 85, 88, 128, - 78, 68, 69, 85, 84, 128, 78, 68, 69, 85, 65, 69, 82, 69, 69, 128, 78, 68, - 69, 80, 128, 78, 68, 69, 69, 128, 78, 68, 69, 128, 78, 68, 65, 88, 128, - 78, 68, 65, 84, 128, 78, 68, 65, 80, 128, 78, 68, 65, 77, 128, 78, 68, - 65, 65, 78, 71, 71, 69, 85, 65, 69, 84, 128, 78, 68, 65, 65, 128, 78, 68, - 65, 193, 78, 67, 72, 65, 85, 128, 78, 67, 65, 128, 78, 66, 89, 88, 128, - 78, 66, 89, 84, 128, 78, 66, 89, 82, 88, 128, 78, 66, 89, 82, 128, 78, - 66, 89, 80, 128, 78, 66, 89, 128, 78, 66, 85, 88, 128, 78, 66, 85, 84, - 128, 78, 66, 85, 82, 88, 128, 78, 66, 85, 82, 128, 78, 66, 85, 80, 128, - 78, 66, 85, 128, 78, 66, 79, 88, 128, 78, 66, 79, 84, 128, 78, 66, 79, - 80, 128, 78, 66, 79, 128, 78, 66, 73, 88, 128, 78, 66, 73, 84, 128, 78, - 66, 73, 80, 128, 78, 66, 73, 69, 88, 128, 78, 66, 73, 69, 80, 128, 78, - 66, 73, 69, 128, 78, 66, 73, 128, 78, 66, 72, 128, 78, 66, 65, 88, 128, - 78, 66, 65, 84, 128, 78, 66, 65, 80, 128, 78, 66, 65, 128, 78, 65, 90, - 65, 210, 78, 65, 89, 65, 78, 78, 65, 128, 78, 65, 89, 128, 78, 65, 88, - 73, 65, 206, 78, 65, 88, 128, 78, 65, 85, 84, 72, 83, 128, 78, 65, 85, - 83, 69, 65, 84, 69, 196, 78, 65, 85, 68, 73, 218, 78, 65, 84, 85, 82, 65, - 204, 78, 65, 84, 84, 73, 76, 73, 203, 78, 65, 84, 73, 79, 78, 65, 204, - 78, 65, 83, 75, 65, 80, 201, 78, 65, 83, 72, 73, 128, 78, 65, 83, 65, 76, - 73, 90, 69, 196, 78, 65, 83, 65, 76, 73, 90, 65, 84, 73, 79, 78, 128, 78, - 65, 83, 65, 76, 73, 90, 65, 84, 73, 79, 206, 78, 65, 83, 65, 204, 78, 65, - 82, 82, 79, 215, 78, 65, 82, 128, 78, 65, 81, 128, 78, 65, 79, 211, 78, - 65, 78, 83, 65, 78, 65, 81, 128, 78, 65, 78, 71, 77, 79, 78, 84, 72, 79, - 128, 78, 65, 78, 68, 73, 78, 65, 71, 65, 82, 201, 78, 65, 78, 68, 128, - 78, 65, 78, 65, 128, 78, 65, 77, 69, 128, 78, 65, 77, 197, 78, 65, 77, - 50, 128, 78, 65, 75, 65, 65, 82, 193, 78, 65, 75, 128, 78, 65, 73, 82, - 193, 78, 65, 73, 204, 78, 65, 71, 82, 201, 78, 65, 71, 65, 82, 128, 78, - 65, 71, 65, 128, 78, 65, 71, 193, 78, 65, 71, 128, 78, 65, 199, 78, 65, - 69, 128, 78, 65, 66, 76, 65, 128, 78, 65, 66, 65, 84, 65, 69, 65, 206, - 78, 65, 65, 83, 73, 75, 89, 65, 89, 65, 128, 78, 65, 65, 75, 83, 73, 75, - 89, 65, 89, 65, 128, 78, 65, 65, 73, 128, 78, 65, 193, 78, 65, 52, 128, - 78, 65, 50, 128, 78, 65, 45, 57, 128, 78, 65, 45, 56, 128, 78, 65, 45, - 55, 128, 78, 65, 45, 54, 128, 78, 65, 45, 53, 128, 78, 65, 45, 52, 128, - 78, 65, 45, 51, 128, 78, 65, 45, 50, 128, 78, 65, 45, 49, 128, 78, 48, - 52, 50, 128, 78, 48, 52, 49, 128, 78, 48, 52, 48, 128, 78, 48, 51, 57, - 128, 78, 48, 51, 56, 128, 78, 48, 51, 55, 65, 128, 78, 48, 51, 55, 128, - 78, 48, 51, 54, 128, 78, 48, 51, 53, 65, 128, 78, 48, 51, 53, 128, 78, - 48, 51, 52, 65, 128, 78, 48, 51, 52, 128, 78, 48, 51, 51, 65, 128, 78, - 48, 51, 51, 128, 78, 48, 51, 50, 128, 78, 48, 51, 49, 128, 78, 48, 51, - 48, 128, 78, 48, 50, 57, 128, 78, 48, 50, 56, 128, 78, 48, 50, 55, 128, - 78, 48, 50, 54, 128, 78, 48, 50, 53, 65, 128, 78, 48, 50, 53, 128, 78, - 48, 50, 52, 128, 78, 48, 50, 51, 128, 78, 48, 50, 50, 128, 78, 48, 50, - 49, 128, 78, 48, 50, 48, 128, 78, 48, 49, 57, 128, 78, 48, 49, 56, 66, - 128, 78, 48, 49, 56, 65, 128, 78, 48, 49, 56, 128, 78, 48, 49, 55, 128, - 78, 48, 49, 54, 128, 78, 48, 49, 53, 128, 78, 48, 49, 52, 128, 78, 48, - 49, 51, 128, 78, 48, 49, 50, 128, 78, 48, 49, 49, 128, 78, 48, 49, 48, - 128, 78, 48, 48, 57, 128, 78, 48, 48, 56, 128, 78, 48, 48, 55, 128, 78, - 48, 48, 54, 128, 78, 48, 48, 53, 128, 78, 48, 48, 52, 128, 78, 48, 48, - 51, 128, 78, 48, 48, 50, 128, 78, 48, 48, 49, 128, 78, 45, 77, 85, 45, - 77, 79, 45, 50, 128, 78, 45, 77, 85, 45, 77, 79, 45, 49, 128, 78, 45, 67, - 82, 69, 197, 78, 45, 65, 82, 217, 77, 90, 128, 77, 89, 88, 128, 77, 89, - 84, 128, 77, 89, 83, 76, 73, 84, 69, 128, 77, 89, 80, 128, 77, 89, 65, - 128, 77, 89, 193, 77, 87, 79, 79, 128, 77, 87, 79, 128, 77, 87, 73, 73, - 128, 77, 87, 73, 128, 77, 87, 69, 69, 128, 77, 87, 69, 128, 77, 87, 65, - 65, 128, 77, 87, 65, 128, 77, 87, 128, 77, 215, 77, 86, 83, 128, 77, 86, - 79, 80, 128, 77, 86, 73, 128, 77, 86, 69, 85, 65, 69, 78, 71, 65, 77, - 128, 77, 86, 128, 77, 214, 77, 85, 88, 128, 77, 85, 85, 86, 85, 90, 72, - 65, 75, 75, 85, 128, 77, 85, 85, 83, 73, 75, 65, 84, 79, 65, 78, 128, 77, - 85, 85, 82, 68, 72, 65, 74, 193, 77, 85, 85, 128, 77, 85, 84, 72, 65, 76, - 73, 89, 65, 128, 77, 85, 84, 128, 77, 85, 83, 73, 67, 128, 77, 85, 83, - 73, 195, 77, 85, 83, 72, 82, 79, 79, 77, 128, 77, 85, 83, 72, 51, 128, - 77, 85, 83, 72, 179, 77, 85, 83, 72, 128, 77, 85, 83, 200, 77, 85, 83, - 128, 77, 85, 82, 88, 128, 77, 85, 82, 71, 85, 50, 128, 77, 85, 82, 69, - 128, 77, 85, 82, 68, 65, 128, 77, 85, 82, 68, 193, 77, 85, 82, 128, 77, - 85, 81, 68, 65, 77, 128, 77, 85, 80, 128, 77, 85, 79, 88, 128, 77, 85, - 79, 84, 128, 77, 85, 79, 80, 128, 77, 85, 79, 77, 65, 69, 128, 77, 85, - 79, 128, 77, 85, 78, 83, 85, 66, 128, 77, 85, 78, 68, 65, 82, 201, 77, - 85, 78, 65, 72, 128, 77, 85, 78, 128, 77, 85, 76, 84, 73, 83, 69, 84, - 128, 77, 85, 76, 84, 73, 83, 69, 212, 77, 85, 76, 84, 73, 80, 76, 73, 67, - 65, 84, 73, 79, 78, 128, 77, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, - 79, 206, 77, 85, 76, 84, 73, 80, 76, 69, 128, 77, 85, 76, 84, 73, 80, 76, - 197, 77, 85, 76, 84, 73, 79, 67, 85, 76, 65, 210, 77, 85, 76, 84, 73, 77, - 65, 80, 128, 77, 85, 76, 84, 201, 77, 85, 76, 84, 65, 78, 201, 77, 85, - 75, 80, 72, 82, 69, 78, 71, 128, 77, 85, 75, 75, 85, 82, 85, 78, 73, 128, - 77, 85, 73, 78, 128, 77, 85, 72, 79, 82, 128, 77, 85, 71, 83, 128, 77, - 85, 71, 128, 77, 85, 199, 77, 85, 69, 78, 128, 77, 85, 69, 128, 77, 85, - 67, 72, 128, 77, 85, 67, 200, 77, 85, 67, 65, 65, 68, 128, 77, 85, 65, - 83, 128, 77, 85, 65, 78, 128, 77, 85, 65, 69, 128, 77, 85, 45, 71, 65, - 65, 72, 76, 65, 193, 77, 85, 45, 52, 128, 77, 85, 45, 51, 128, 77, 85, - 45, 50, 128, 77, 85, 45, 49, 128, 77, 213, 77, 84, 65, 86, 82, 85, 76, - 201, 77, 83, 128, 77, 82, 207, 77, 82, 65, 67, 72, 78, 89, 128, 77, 82, - 65, 67, 72, 78, 79, 84, 73, 75, 72, 65, 89, 65, 128, 77, 82, 65, 67, 72, - 78, 79, 128, 77, 82, 65, 67, 72, 78, 65, 89, 65, 128, 77, 210, 77, 81, - 128, 77, 80, 65, 128, 77, 79, 89, 65, 73, 128, 77, 79, 88, 128, 77, 79, - 86, 73, 197, 77, 79, 86, 69, 211, 77, 79, 86, 69, 77, 69, 78, 84, 45, 87, - 65, 76, 76, 80, 76, 65, 78, 197, 77, 79, 86, 69, 77, 69, 78, 84, 45, 72, - 73, 78, 71, 197, 77, 79, 86, 69, 77, 69, 78, 84, 45, 70, 76, 79, 79, 82, - 80, 76, 65, 78, 197, 77, 79, 86, 69, 77, 69, 78, 84, 45, 68, 73, 65, 71, - 79, 78, 65, 204, 77, 79, 86, 69, 77, 69, 78, 84, 128, 77, 79, 86, 69, 77, - 69, 78, 212, 77, 79, 86, 69, 196, 77, 79, 86, 69, 128, 77, 79, 85, 84, - 72, 128, 77, 79, 85, 83, 69, 128, 77, 79, 85, 83, 197, 77, 79, 85, 78, - 84, 65, 73, 78, 83, 128, 77, 79, 85, 78, 84, 65, 73, 78, 128, 77, 79, 85, - 78, 84, 65, 73, 206, 77, 79, 85, 78, 212, 77, 79, 85, 78, 68, 128, 77, - 79, 85, 78, 196, 77, 79, 84, 79, 82, 87, 65, 89, 128, 77, 79, 84, 79, 82, - 73, 90, 69, 196, 77, 79, 84, 79, 82, 67, 89, 67, 76, 69, 128, 77, 79, 84, - 79, 210, 77, 79, 84, 72, 69, 82, 128, 77, 79, 84, 72, 69, 210, 77, 79, - 84, 128, 77, 79, 83, 81, 85, 73, 84, 79, 128, 77, 79, 83, 81, 85, 69, - 128, 77, 79, 82, 84, 85, 85, 77, 128, 77, 79, 82, 84, 65, 82, 128, 77, - 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 204, 77, 79, 82, 78, 73, 78, - 71, 128, 77, 79, 80, 128, 77, 79, 79, 83, 69, 45, 67, 82, 69, 197, 77, - 79, 79, 83, 69, 128, 77, 79, 79, 78, 128, 77, 79, 79, 206, 77, 79, 79, - 77, 80, 85, 81, 128, 77, 79, 79, 77, 69, 85, 84, 128, 77, 79, 79, 68, - 128, 77, 79, 79, 196, 77, 79, 79, 128, 77, 79, 78, 84, 73, 69, 69, 78, - 128, 77, 79, 78, 84, 72, 128, 77, 79, 78, 84, 200, 77, 79, 78, 83, 84, - 69, 82, 128, 77, 79, 78, 79, 83, 84, 65, 66, 76, 197, 77, 79, 78, 79, 83, - 80, 65, 67, 197, 77, 79, 78, 79, 82, 65, 73, 76, 128, 77, 79, 78, 79, 71, - 82, 65, 80, 200, 77, 79, 78, 79, 71, 82, 65, 77, 77, 79, 211, 77, 79, 78, - 79, 71, 82, 65, 205, 77, 79, 78, 79, 70, 79, 78, 73, 65, 83, 128, 77, 79, - 78, 79, 67, 85, 76, 65, 210, 77, 79, 78, 79, 67, 76, 69, 128, 77, 79, 78, - 75, 69, 89, 128, 77, 79, 78, 75, 69, 217, 77, 79, 78, 73, 128, 77, 79, - 78, 71, 75, 69, 85, 65, 69, 81, 128, 77, 79, 78, 69, 89, 45, 77, 79, 85, - 84, 200, 77, 79, 78, 69, 217, 77, 79, 78, 128, 77, 79, 206, 77, 79, 76, - 128, 77, 79, 75, 72, 65, 83, 83, 65, 83, 128, 77, 79, 72, 65, 77, 77, 65, - 196, 77, 79, 68, 85, 76, 207, 77, 79, 68, 73, 70, 73, 69, 82, 45, 57, - 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 56, 128, 77, 79, 68, 73, 70, 73, - 69, 82, 45, 55, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 54, 128, 77, 79, - 68, 73, 70, 73, 69, 82, 45, 53, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, - 52, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 51, 128, 77, 79, 68, 73, 70, - 73, 69, 82, 45, 50, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 49, 54, 128, - 77, 79, 68, 73, 70, 73, 69, 82, 45, 49, 53, 128, 77, 79, 68, 73, 70, 73, - 69, 82, 45, 49, 52, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 49, 51, 128, - 77, 79, 68, 73, 70, 73, 69, 82, 45, 49, 50, 128, 77, 79, 68, 73, 70, 73, - 69, 82, 45, 49, 49, 128, 77, 79, 68, 73, 70, 73, 69, 82, 45, 49, 48, 128, - 77, 79, 68, 73, 70, 73, 69, 82, 128, 77, 79, 68, 201, 77, 79, 68, 69, 83, - 84, 89, 128, 77, 79, 68, 69, 82, 206, 77, 79, 68, 69, 77, 128, 77, 79, - 68, 69, 76, 83, 128, 77, 79, 68, 69, 76, 128, 77, 79, 68, 69, 128, 77, - 79, 66, 73, 76, 197, 77, 79, 65, 128, 77, 79, 45, 54, 128, 77, 79, 45, - 53, 128, 77, 79, 45, 52, 128, 77, 79, 45, 51, 128, 77, 207, 77, 78, 89, - 65, 205, 77, 78, 65, 83, 128, 77, 77, 83, 80, 128, 77, 77, 128, 77, 205, - 77, 76, 65, 128, 77, 76, 128, 77, 75, 80, 65, 82, 65, 209, 77, 73, 88, - 128, 77, 73, 84, 128, 77, 73, 83, 82, 65, 128, 77, 73, 82, 82, 79, 82, - 128, 77, 73, 82, 82, 79, 210, 77, 73, 82, 73, 66, 65, 65, 82, 85, 128, - 77, 73, 82, 73, 128, 77, 73, 82, 69, 68, 128, 77, 73, 80, 128, 77, 73, - 78, 89, 128, 77, 73, 78, 85, 83, 45, 79, 82, 45, 80, 76, 85, 211, 77, 73, - 78, 85, 83, 128, 77, 73, 78, 78, 65, 206, 77, 73, 78, 73, 83, 84, 69, 82, - 128, 77, 73, 78, 73, 77, 73, 90, 69, 128, 77, 73, 78, 73, 77, 65, 128, - 77, 73, 78, 73, 68, 73, 83, 67, 128, 77, 73, 78, 73, 66, 85, 83, 128, 77, - 73, 78, 68, 85, 128, 77, 73, 77, 69, 128, 77, 73, 77, 128, 77, 73, 76, - 76, 73, 79, 78, 83, 128, 77, 73, 76, 76, 73, 79, 78, 211, 77, 73, 76, 76, - 69, 84, 128, 77, 73, 76, 76, 197, 77, 73, 76, 204, 77, 73, 76, 75, 217, - 77, 73, 76, 75, 128, 77, 73, 76, 73, 84, 65, 82, 217, 77, 73, 76, 128, - 77, 73, 75, 85, 82, 79, 78, 128, 77, 73, 75, 82, 79, 206, 77, 73, 75, 82, - 73, 128, 77, 73, 73, 78, 128, 77, 73, 73, 77, 128, 77, 73, 73, 128, 77, - 73, 199, 77, 73, 69, 88, 128, 77, 73, 69, 85, 77, 45, 84, 73, 75, 69, 85, - 84, 128, 77, 73, 69, 85, 77, 45, 83, 83, 65, 78, 71, 83, 73, 79, 83, 128, - 77, 73, 69, 85, 77, 45, 83, 83, 65, 78, 71, 78, 73, 69, 85, 78, 128, 77, - 73, 69, 85, 77, 45, 82, 73, 69, 85, 76, 128, 77, 73, 69, 85, 77, 45, 80, - 73, 69, 85, 80, 45, 83, 73, 79, 83, 128, 77, 73, 69, 85, 77, 45, 80, 73, - 69, 85, 80, 128, 77, 73, 69, 85, 77, 45, 80, 65, 78, 83, 73, 79, 83, 128, - 77, 73, 69, 85, 77, 45, 78, 73, 69, 85, 78, 128, 77, 73, 69, 85, 77, 45, - 67, 73, 69, 85, 67, 128, 77, 73, 69, 85, 77, 45, 67, 72, 73, 69, 85, 67, - 72, 128, 77, 73, 69, 85, 205, 77, 73, 69, 80, 128, 77, 73, 69, 69, 128, - 77, 73, 69, 128, 77, 73, 68, 76, 73, 78, 197, 77, 73, 68, 68, 76, 69, 45, - 87, 69, 76, 83, 200, 77, 73, 68, 68, 76, 69, 128, 77, 73, 68, 45, 76, 69, - 86, 69, 204, 77, 73, 68, 45, 72, 69, 73, 71, 72, 212, 77, 73, 196, 77, - 73, 67, 82, 79, 83, 67, 79, 80, 69, 128, 77, 73, 67, 82, 79, 80, 72, 79, - 78, 69, 128, 77, 73, 67, 82, 79, 66, 69, 128, 77, 73, 67, 82, 207, 77, - 73, 67, 210, 77, 73, 45, 55, 128, 77, 73, 45, 54, 128, 77, 73, 45, 53, - 128, 77, 73, 45, 52, 128, 77, 73, 45, 51, 128, 77, 73, 45, 50, 128, 77, - 73, 45, 49, 128, 77, 72, 90, 128, 77, 72, 65, 128, 77, 72, 128, 77, 71, - 85, 88, 128, 77, 71, 85, 84, 128, 77, 71, 85, 82, 88, 128, 77, 71, 85, - 82, 128, 77, 71, 85, 80, 128, 77, 71, 85, 79, 88, 128, 77, 71, 85, 79, - 80, 128, 77, 71, 85, 79, 128, 77, 71, 85, 128, 77, 71, 79, 88, 128, 77, - 71, 79, 84, 128, 77, 71, 79, 80, 128, 77, 71, 79, 128, 77, 71, 207, 77, - 71, 73, 69, 88, 128, 77, 71, 73, 69, 128, 77, 71, 69, 88, 128, 77, 71, - 69, 80, 128, 77, 71, 69, 128, 77, 71, 66, 85, 128, 77, 71, 66, 79, 79, - 128, 77, 71, 66, 79, 70, 85, 77, 128, 77, 71, 66, 79, 128, 77, 71, 66, - 73, 128, 77, 71, 66, 69, 85, 78, 128, 77, 71, 66, 69, 78, 128, 77, 71, - 66, 69, 69, 128, 77, 71, 66, 69, 128, 77, 71, 66, 65, 83, 65, 81, 128, - 77, 71, 66, 65, 83, 65, 128, 77, 71, 65, 88, 128, 77, 71, 65, 84, 128, - 77, 71, 65, 80, 128, 77, 71, 65, 128, 77, 71, 128, 77, 70, 79, 78, 128, - 77, 70, 79, 206, 77, 70, 79, 128, 77, 70, 73, 89, 65, 81, 128, 77, 70, - 73, 69, 69, 128, 77, 70, 69, 85, 84, 128, 77, 70, 69, 85, 81, 128, 77, - 70, 69, 85, 65, 69, 128, 77, 70, 65, 65, 128, 77, 69, 90, 90, 79, 128, - 77, 69, 88, 128, 77, 69, 85, 212, 77, 69, 85, 81, 128, 77, 69, 85, 78, - 74, 79, 77, 78, 68, 69, 85, 81, 128, 77, 69, 85, 78, 128, 77, 69, 84, 82, - 79, 128, 77, 69, 84, 82, 73, 67, 65, 204, 77, 69, 84, 82, 73, 65, 128, - 77, 69, 84, 82, 69, 84, 69, 211, 77, 69, 84, 79, 66, 69, 76, 85, 83, 128, - 77, 69, 84, 69, 75, 128, 77, 69, 84, 69, 71, 128, 77, 69, 84, 65, 76, - 128, 77, 69, 84, 193, 77, 69, 83, 83, 69, 78, 73, 65, 206, 77, 69, 83, - 83, 65, 71, 69, 128, 77, 69, 83, 83, 65, 71, 197, 77, 69, 83, 79, 128, - 77, 69, 83, 73, 128, 77, 69, 83, 72, 128, 77, 69, 82, 80, 69, 82, 83, 79, - 78, 128, 77, 69, 82, 75, 72, 65, 128, 77, 69, 82, 75, 72, 193, 77, 69, - 82, 73, 68, 73, 65, 78, 83, 128, 77, 69, 82, 73, 128, 77, 69, 82, 71, 69, - 128, 77, 69, 82, 67, 85, 82, 89, 128, 77, 69, 82, 67, 85, 82, 217, 77, - 69, 78, 79, 82, 65, 200, 77, 69, 78, 79, 69, 128, 77, 69, 78, 68, 85, 84, - 128, 77, 69, 78, 128, 77, 69, 77, 79, 128, 77, 69, 77, 66, 69, 82, 83, - 72, 73, 80, 128, 77, 69, 77, 66, 69, 82, 128, 77, 69, 77, 66, 69, 210, - 77, 69, 77, 45, 81, 79, 80, 72, 128, 77, 69, 77, 128, 77, 69, 205, 77, - 69, 76, 84, 73, 78, 199, 77, 69, 76, 79, 68, 73, 195, 77, 69, 76, 73, 75, - 128, 77, 69, 73, 90, 73, 128, 77, 69, 71, 65, 84, 79, 78, 128, 77, 69, - 71, 65, 80, 72, 79, 78, 69, 128, 77, 69, 71, 65, 76, 73, 128, 77, 69, 69, - 84, 79, 82, 85, 128, 77, 69, 69, 84, 69, 201, 77, 69, 69, 84, 128, 77, - 69, 69, 77, 85, 128, 77, 69, 69, 77, 128, 77, 69, 69, 202, 77, 69, 69, - 69, 69, 128, 77, 69, 68, 73, 85, 77, 128, 77, 69, 68, 73, 85, 205, 77, - 69, 68, 73, 69, 86, 65, 204, 77, 69, 68, 73, 67, 73, 78, 69, 128, 77, 69, - 68, 73, 67, 65, 204, 77, 69, 68, 73, 65, 204, 77, 69, 68, 69, 70, 65, 73, - 68, 82, 73, 206, 77, 69, 68, 65, 76, 128, 77, 69, 67, 72, 73, 75, 128, - 77, 69, 67, 72, 73, 203, 77, 69, 67, 72, 65, 78, 73, 67, 65, 204, 77, 69, - 65, 84, 128, 77, 69, 65, 212, 77, 69, 65, 83, 85, 82, 69, 196, 77, 69, - 65, 83, 85, 82, 69, 128, 77, 69, 65, 83, 85, 82, 197, 77, 69, 45, 77, 65, - 128, 77, 69, 45, 50, 128, 77, 69, 45, 49, 128, 77, 68, 85, 206, 77, 196, - 77, 67, 72, 213, 77, 67, 72, 65, 206, 77, 67, 128, 77, 195, 77, 66, 85, - 85, 128, 77, 66, 85, 79, 81, 128, 77, 66, 85, 79, 128, 77, 66, 85, 69, - 128, 77, 66, 85, 65, 69, 77, 128, 77, 66, 85, 65, 69, 128, 77, 66, 79, - 79, 128, 77, 66, 79, 128, 77, 66, 73, 84, 128, 77, 66, 73, 212, 77, 66, - 73, 82, 73, 69, 69, 78, 128, 77, 66, 73, 128, 77, 66, 69, 85, 88, 128, - 77, 66, 69, 85, 82, 73, 128, 77, 66, 69, 85, 77, 128, 77, 66, 69, 82, 65, - 69, 128, 77, 66, 69, 78, 128, 77, 66, 69, 69, 75, 69, 69, 84, 128, 77, - 66, 69, 69, 128, 77, 66, 69, 128, 77, 66, 65, 81, 128, 77, 66, 65, 78, - 89, 73, 128, 77, 66, 65, 65, 82, 65, 69, 128, 77, 66, 65, 65, 75, 69, 84, - 128, 77, 66, 65, 65, 128, 77, 66, 65, 193, 77, 66, 193, 77, 66, 52, 128, - 77, 66, 51, 128, 77, 66, 50, 128, 77, 65, 89, 69, 203, 77, 65, 89, 65, - 78, 78, 65, 128, 77, 65, 89, 65, 206, 77, 65, 89, 128, 77, 65, 88, 73, - 77, 73, 90, 69, 128, 77, 65, 88, 73, 77, 65, 128, 77, 65, 88, 128, 77, - 65, 85, 128, 77, 65, 84, 84, 79, 67, 75, 128, 77, 65, 84, 82, 73, 88, - 128, 77, 65, 84, 69, 82, 73, 65, 76, 83, 128, 77, 65, 84, 128, 77, 65, - 83, 213, 77, 65, 83, 83, 73, 78, 71, 128, 77, 65, 83, 83, 65, 71, 69, - 128, 77, 65, 83, 79, 82, 193, 77, 65, 83, 75, 128, 77, 65, 83, 203, 77, - 65, 83, 72, 70, 65, 65, 84, 128, 77, 65, 83, 72, 50, 128, 77, 65, 83, 67, - 85, 76, 73, 78, 197, 77, 65, 83, 65, 82, 65, 205, 77, 65, 82, 89, 128, - 77, 65, 82, 87, 65, 82, 201, 77, 65, 82, 85, 75, 85, 128, 77, 65, 82, 84, - 89, 82, 73, 193, 77, 65, 82, 84, 73, 65, 204, 77, 65, 82, 82, 89, 73, 78, - 199, 77, 65, 82, 82, 73, 65, 71, 197, 77, 65, 82, 82, 65, 84, 65, 78, - 128, 77, 65, 82, 75, 211, 77, 65, 82, 75, 69, 82, 128, 77, 65, 82, 75, - 45, 52, 128, 77, 65, 82, 75, 45, 51, 128, 77, 65, 82, 75, 45, 50, 128, - 77, 65, 82, 75, 45, 49, 128, 77, 65, 82, 69, 128, 77, 65, 82, 67, 72, 69, - 206, 77, 65, 82, 67, 72, 128, 77, 65, 82, 67, 65, 84, 79, 45, 83, 84, 65, - 67, 67, 65, 84, 79, 128, 77, 65, 82, 67, 65, 84, 79, 128, 77, 65, 82, 67, - 65, 83, 73, 84, 69, 128, 77, 65, 82, 66, 85, 84, 65, 128, 77, 65, 82, 66, - 85, 84, 193, 77, 65, 82, 65, 67, 65, 83, 128, 77, 65, 82, 128, 77, 65, - 81, 65, 70, 128, 77, 65, 81, 128, 77, 65, 80, 76, 197, 77, 65, 80, 73, - 81, 128, 77, 65, 208, 77, 65, 79, 128, 77, 65, 78, 85, 65, 204, 77, 65, - 78, 84, 69, 76, 80, 73, 69, 67, 197, 77, 65, 78, 83, 89, 79, 78, 128, 77, - 65, 78, 83, 85, 65, 69, 128, 77, 65, 78, 78, 65, 218, 77, 65, 78, 78, 65, - 128, 77, 65, 78, 73, 67, 72, 65, 69, 65, 206, 77, 65, 78, 71, 79, 128, - 77, 65, 78, 71, 65, 76, 65, 77, 128, 77, 65, 78, 68, 65, 82, 73, 78, 128, - 77, 65, 78, 68, 65, 73, 76, 73, 78, 199, 77, 65, 78, 68, 65, 73, 195, 77, - 65, 78, 67, 72, 213, 77, 65, 78, 65, 212, 77, 65, 78, 65, 67, 76, 69, 83, - 128, 77, 65, 77, 77, 79, 84, 72, 128, 77, 65, 76, 84, 69, 83, 197, 77, - 65, 76, 207, 77, 65, 76, 69, 69, 82, 73, 128, 77, 65, 76, 197, 77, 65, - 76, 65, 75, 79, 206, 77, 65, 75, 83, 85, 82, 65, 128, 77, 65, 75, 83, 85, - 82, 193, 77, 65, 75, 69, 77, 65, 75, 69, 128, 77, 65, 75, 65, 83, 65, - 210, 77, 65, 73, 90, 69, 128, 77, 65, 73, 89, 65, 77, 79, 75, 128, 77, - 65, 73, 84, 65, 73, 75, 72, 85, 128, 77, 65, 73, 82, 85, 128, 77, 65, 73, - 77, 85, 65, 78, 128, 77, 65, 73, 77, 65, 76, 65, 73, 128, 77, 65, 73, 76, - 66, 79, 216, 77, 65, 73, 75, 85, 82, 79, 128, 77, 65, 73, 68, 69, 78, - 128, 77, 65, 73, 128, 77, 65, 72, 74, 79, 78, 199, 77, 65, 72, 72, 65, - 128, 77, 65, 72, 65, 80, 82, 65, 78, 65, 128, 77, 65, 72, 65, 80, 65, 75, - 72, 128, 77, 65, 72, 65, 74, 65, 78, 201, 77, 65, 72, 65, 65, 80, 82, 65, - 65, 78, 193, 77, 65, 72, 128, 77, 65, 71, 78, 73, 70, 89, 73, 78, 199, - 77, 65, 71, 78, 69, 84, 128, 77, 65, 71, 73, 195, 77, 65, 71, 69, 128, - 77, 65, 69, 83, 73, 128, 77, 65, 69, 78, 89, 73, 128, 77, 65, 69, 78, 74, - 69, 84, 128, 77, 65, 69, 77, 86, 69, 85, 88, 128, 77, 65, 69, 77, 75, 80, - 69, 78, 128, 77, 65, 69, 77, 71, 66, 73, 69, 69, 128, 77, 65, 69, 77, 66, - 71, 66, 73, 69, 69, 128, 77, 65, 69, 77, 66, 65, 128, 77, 65, 69, 77, - 128, 77, 65, 69, 76, 69, 69, 128, 77, 65, 69, 75, 69, 85, 80, 128, 77, - 65, 68, 89, 65, 128, 77, 65, 68, 85, 128, 77, 65, 68, 68, 65, 72, 128, - 77, 65, 68, 68, 65, 200, 77, 65, 68, 68, 65, 128, 77, 65, 68, 68, 193, - 77, 65, 67, 82, 79, 78, 45, 71, 82, 65, 86, 69, 128, 77, 65, 67, 82, 79, - 78, 45, 66, 82, 69, 86, 69, 128, 77, 65, 67, 82, 79, 78, 45, 65, 67, 85, - 84, 69, 128, 77, 65, 67, 82, 79, 78, 128, 77, 65, 67, 82, 79, 206, 77, - 65, 67, 72, 73, 78, 69, 128, 77, 65, 65, 89, 89, 65, 65, 128, 77, 65, 65, - 73, 128, 77, 65, 65, 128, 77, 65, 50, 128, 77, 65, 45, 55, 128, 77, 65, - 45, 54, 128, 77, 65, 45, 53, 128, 77, 65, 45, 52, 128, 77, 65, 45, 51, - 128, 77, 65, 45, 50, 128, 77, 65, 45, 49, 128, 77, 49, 57, 183, 77, 49, - 57, 182, 77, 49, 57, 181, 77, 49, 57, 180, 77, 49, 57, 179, 77, 49, 57, - 178, 77, 49, 57, 177, 77, 49, 57, 176, 77, 49, 56, 185, 77, 49, 56, 184, - 77, 49, 56, 183, 77, 49, 56, 182, 77, 49, 56, 181, 77, 49, 56, 180, 77, - 49, 56, 179, 77, 49, 56, 178, 77, 49, 56, 177, 77, 49, 56, 176, 77, 49, - 55, 185, 77, 49, 55, 184, 77, 49, 55, 183, 77, 49, 55, 182, 77, 49, 55, - 181, 77, 49, 55, 180, 77, 49, 55, 179, 77, 49, 55, 178, 77, 49, 55, 177, - 77, 49, 55, 176, 77, 49, 54, 185, 77, 49, 54, 184, 77, 49, 54, 183, 77, - 49, 54, 182, 77, 49, 54, 181, 77, 49, 54, 180, 77, 49, 54, 179, 77, 49, - 54, 178, 77, 49, 54, 177, 77, 49, 54, 176, 77, 49, 53, 185, 77, 49, 53, - 184, 77, 49, 53, 183, 77, 49, 53, 182, 77, 49, 53, 181, 77, 49, 53, 180, - 77, 49, 53, 179, 77, 49, 53, 178, 77, 49, 53, 177, 77, 49, 53, 176, 77, - 49, 52, 185, 77, 49, 52, 184, 77, 49, 52, 183, 77, 49, 52, 182, 77, 49, - 52, 181, 77, 49, 52, 180, 77, 49, 52, 179, 77, 49, 52, 178, 77, 49, 52, - 177, 77, 49, 52, 176, 77, 49, 51, 185, 77, 49, 51, 184, 77, 49, 51, 183, - 77, 49, 51, 182, 77, 49, 51, 181, 77, 49, 51, 180, 77, 49, 51, 179, 77, - 49, 51, 178, 77, 49, 51, 177, 77, 49, 51, 176, 77, 49, 50, 185, 77, 49, - 50, 184, 77, 49, 50, 183, 77, 49, 50, 182, 77, 49, 50, 181, 77, 49, 50, - 180, 77, 49, 50, 179, 77, 49, 50, 178, 77, 49, 50, 177, 77, 49, 50, 176, - 77, 49, 49, 185, 77, 49, 49, 184, 77, 49, 49, 183, 77, 49, 49, 182, 77, - 49, 49, 181, 77, 49, 49, 180, 77, 49, 49, 179, 77, 49, 49, 178, 77, 49, - 49, 177, 77, 49, 49, 176, 77, 49, 48, 185, 77, 49, 48, 184, 77, 49, 48, - 183, 77, 49, 48, 182, 77, 49, 48, 181, 77, 49, 48, 180, 77, 49, 48, 179, - 77, 49, 48, 178, 77, 49, 48, 177, 77, 49, 48, 176, 77, 48, 57, 185, 77, - 48, 57, 184, 77, 48, 57, 183, 77, 48, 57, 182, 77, 48, 57, 181, 77, 48, - 57, 180, 77, 48, 57, 179, 77, 48, 57, 178, 77, 48, 57, 177, 77, 48, 57, - 176, 77, 48, 56, 185, 77, 48, 56, 184, 77, 48, 56, 183, 77, 48, 56, 182, - 77, 48, 56, 181, 77, 48, 56, 180, 77, 48, 56, 179, 77, 48, 56, 178, 77, - 48, 56, 177, 77, 48, 56, 176, 77, 48, 55, 185, 77, 48, 55, 184, 77, 48, - 55, 183, 77, 48, 55, 182, 77, 48, 55, 181, 77, 48, 55, 180, 77, 48, 55, - 179, 77, 48, 55, 178, 77, 48, 55, 177, 77, 48, 55, 176, 77, 48, 54, 185, - 77, 48, 54, 184, 77, 48, 54, 183, 77, 48, 54, 182, 77, 48, 54, 181, 77, - 48, 54, 180, 77, 48, 54, 179, 77, 48, 54, 178, 77, 48, 54, 177, 77, 48, - 54, 176, 77, 48, 53, 185, 77, 48, 53, 184, 77, 48, 53, 183, 77, 48, 53, - 182, 77, 48, 53, 181, 77, 48, 53, 180, 77, 48, 53, 179, 77, 48, 53, 178, - 77, 48, 53, 177, 77, 48, 53, 176, 77, 48, 52, 185, 77, 48, 52, 184, 77, - 48, 52, 183, 77, 48, 52, 182, 77, 48, 52, 181, 77, 48, 52, 52, 128, 77, - 48, 52, 180, 77, 48, 52, 51, 128, 77, 48, 52, 179, 77, 48, 52, 50, 128, - 77, 48, 52, 178, 77, 48, 52, 49, 128, 77, 48, 52, 177, 77, 48, 52, 48, - 65, 128, 77, 48, 52, 48, 128, 77, 48, 52, 176, 77, 48, 51, 57, 128, 77, - 48, 51, 185, 77, 48, 51, 56, 128, 77, 48, 51, 184, 77, 48, 51, 55, 128, - 77, 48, 51, 183, 77, 48, 51, 54, 128, 77, 48, 51, 182, 77, 48, 51, 53, - 128, 77, 48, 51, 181, 77, 48, 51, 52, 128, 77, 48, 51, 180, 77, 48, 51, - 51, 66, 128, 77, 48, 51, 51, 65, 128, 77, 48, 51, 51, 128, 77, 48, 51, - 179, 77, 48, 51, 50, 128, 77, 48, 51, 178, 77, 48, 51, 49, 65, 128, 77, - 48, 51, 49, 128, 77, 48, 51, 177, 77, 48, 51, 48, 128, 77, 48, 51, 176, - 77, 48, 50, 57, 128, 77, 48, 50, 185, 77, 48, 50, 56, 65, 128, 77, 48, - 50, 56, 128, 77, 48, 50, 184, 77, 48, 50, 55, 128, 77, 48, 50, 183, 77, - 48, 50, 54, 128, 77, 48, 50, 182, 77, 48, 50, 53, 128, 77, 48, 50, 181, - 77, 48, 50, 52, 65, 128, 77, 48, 50, 52, 128, 77, 48, 50, 180, 77, 48, - 50, 51, 128, 77, 48, 50, 179, 77, 48, 50, 50, 65, 128, 77, 48, 50, 50, - 128, 77, 48, 50, 178, 77, 48, 50, 49, 128, 77, 48, 50, 177, 77, 48, 50, - 48, 128, 77, 48, 50, 176, 77, 48, 49, 57, 128, 77, 48, 49, 185, 77, 48, - 49, 56, 128, 77, 48, 49, 184, 77, 48, 49, 55, 65, 128, 77, 48, 49, 55, - 128, 77, 48, 49, 183, 77, 48, 49, 54, 65, 128, 77, 48, 49, 54, 128, 77, - 48, 49, 182, 77, 48, 49, 53, 65, 128, 77, 48, 49, 53, 128, 77, 48, 49, - 181, 77, 48, 49, 52, 128, 77, 48, 49, 180, 77, 48, 49, 51, 128, 77, 48, - 49, 179, 77, 48, 49, 50, 72, 128, 77, 48, 49, 50, 71, 128, 77, 48, 49, - 50, 70, 128, 77, 48, 49, 50, 69, 128, 77, 48, 49, 50, 68, 128, 77, 48, - 49, 50, 67, 128, 77, 48, 49, 50, 66, 128, 77, 48, 49, 50, 65, 128, 77, - 48, 49, 50, 128, 77, 48, 49, 178, 77, 48, 49, 49, 128, 77, 48, 49, 177, - 77, 48, 49, 48, 65, 128, 77, 48, 49, 48, 128, 77, 48, 49, 176, 77, 48, - 48, 57, 128, 77, 48, 48, 185, 77, 48, 48, 56, 128, 77, 48, 48, 184, 77, - 48, 48, 55, 128, 77, 48, 48, 183, 77, 48, 48, 54, 128, 77, 48, 48, 182, - 77, 48, 48, 53, 128, 77, 48, 48, 181, 77, 48, 48, 52, 128, 77, 48, 48, - 180, 77, 48, 48, 51, 65, 128, 77, 48, 48, 51, 128, 77, 48, 48, 179, 77, - 48, 48, 50, 128, 77, 48, 48, 178, 77, 48, 48, 49, 66, 128, 77, 48, 48, - 49, 65, 128, 77, 48, 48, 49, 128, 77, 48, 48, 177, 76, 218, 76, 89, 89, - 128, 76, 89, 88, 128, 76, 89, 84, 128, 76, 89, 82, 88, 128, 76, 89, 82, - 128, 76, 89, 80, 128, 76, 89, 73, 84, 128, 76, 89, 73, 78, 199, 76, 89, - 68, 73, 65, 206, 76, 89, 67, 73, 65, 206, 76, 88, 128, 76, 87, 79, 79, - 128, 76, 87, 79, 128, 76, 87, 73, 73, 128, 76, 87, 73, 128, 76, 87, 69, - 128, 76, 87, 65, 65, 128, 76, 87, 65, 128, 76, 85, 88, 128, 76, 85, 85, - 128, 76, 85, 84, 128, 76, 85, 82, 88, 128, 76, 85, 80, 128, 76, 85, 79, - 88, 128, 76, 85, 79, 84, 128, 76, 85, 79, 80, 128, 76, 85, 79, 128, 76, - 85, 78, 71, 83, 73, 128, 76, 85, 78, 71, 83, 128, 76, 85, 78, 65, 84, - 197, 76, 85, 78, 65, 210, 76, 85, 205, 76, 85, 76, 128, 76, 85, 73, 83, - 128, 76, 85, 72, 85, 82, 128, 76, 85, 72, 128, 76, 85, 200, 76, 85, 71, - 71, 65, 71, 69, 128, 76, 85, 71, 65, 76, 128, 76, 85, 71, 65, 204, 76, - 85, 69, 128, 76, 85, 197, 76, 85, 66, 128, 76, 85, 65, 69, 80, 128, 76, - 85, 51, 128, 76, 85, 50, 128, 76, 85, 178, 76, 82, 79, 128, 76, 82, 77, - 128, 76, 82, 73, 128, 76, 82, 69, 128, 76, 79, 90, 69, 78, 71, 69, 128, - 76, 79, 90, 69, 78, 71, 197, 76, 79, 88, 128, 76, 79, 87, 69, 82, 69, - 196, 76, 79, 87, 45, 82, 69, 86, 69, 82, 83, 69, 68, 45, 185, 76, 79, 87, - 45, 77, 73, 196, 76, 79, 87, 45, 70, 65, 76, 76, 73, 78, 199, 76, 79, 87, - 45, 185, 76, 79, 86, 197, 76, 79, 85, 82, 69, 128, 76, 79, 85, 68, 83, - 80, 69, 65, 75, 69, 82, 128, 76, 79, 85, 68, 76, 217, 76, 79, 84, 85, 83, - 128, 76, 79, 84, 85, 211, 76, 79, 84, 73, 79, 206, 76, 79, 84, 128, 76, - 79, 83, 212, 76, 79, 83, 83, 76, 69, 83, 83, 128, 76, 79, 82, 82, 89, - 128, 76, 79, 82, 82, 65, 73, 78, 69, 128, 76, 79, 81, 128, 76, 79, 80, - 128, 76, 79, 79, 84, 128, 76, 79, 79, 80, 69, 196, 76, 79, 79, 80, 128, - 76, 79, 79, 208, 76, 79, 79, 78, 128, 76, 79, 79, 203, 76, 79, 79, 128, - 76, 79, 78, 83, 85, 77, 128, 76, 79, 78, 71, 65, 128, 76, 79, 78, 71, - 193, 76, 79, 78, 71, 45, 76, 69, 71, 71, 69, 196, 76, 79, 78, 71, 45, 66, - 82, 65, 78, 67, 72, 45, 89, 82, 128, 76, 79, 78, 71, 45, 66, 82, 65, 78, - 67, 72, 45, 83, 79, 204, 76, 79, 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, - 79, 83, 211, 76, 79, 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, 77, 65, 68, - 210, 76, 79, 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, 72, 65, 71, 65, 76, - 204, 76, 79, 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, 65, 210, 76, 79, 77, - 77, 65, 69, 128, 76, 79, 77, 75, 65, 128, 76, 79, 77, 128, 76, 79, 205, - 76, 79, 76, 76, 73, 80, 79, 80, 128, 76, 79, 76, 76, 128, 76, 79, 71, - 210, 76, 79, 71, 79, 84, 89, 80, 197, 76, 79, 71, 79, 71, 82, 65, 205, - 76, 79, 71, 128, 76, 79, 68, 69, 83, 84, 79, 78, 69, 128, 76, 79, 67, 79, - 77, 79, 84, 73, 86, 69, 128, 76, 79, 67, 75, 73, 78, 71, 45, 83, 72, 73, - 70, 212, 76, 79, 67, 65, 84, 73, 86, 69, 128, 76, 79, 67, 65, 84, 73, 79, - 78, 45, 87, 65, 76, 76, 80, 76, 65, 78, 197, 76, 79, 67, 65, 84, 73, 79, - 78, 45, 70, 76, 79, 79, 82, 80, 76, 65, 78, 197, 76, 79, 67, 65, 84, 73, - 79, 78, 128, 76, 79, 67, 65, 84, 73, 79, 206, 76, 79, 66, 83, 84, 69, 82, - 128, 76, 79, 65, 128, 76, 78, 128, 76, 76, 85, 85, 128, 76, 76, 79, 79, - 128, 76, 76, 76, 85, 85, 128, 76, 76, 76, 85, 128, 76, 76, 76, 79, 79, - 128, 76, 76, 76, 79, 128, 76, 76, 76, 73, 73, 128, 76, 76, 76, 73, 128, - 76, 76, 76, 69, 69, 128, 76, 76, 76, 69, 128, 76, 76, 76, 65, 85, 128, - 76, 76, 76, 65, 73, 128, 76, 76, 76, 65, 65, 128, 76, 76, 76, 65, 128, - 76, 76, 76, 128, 76, 76, 72, 65, 128, 76, 76, 65, 77, 65, 128, 76, 74, - 85, 68, 73, 74, 69, 128, 76, 74, 69, 128, 76, 74, 128, 76, 73, 90, 65, - 82, 68, 128, 76, 73, 88, 128, 76, 73, 87, 78, 128, 76, 73, 86, 82, 197, - 76, 73, 84, 84, 76, 69, 128, 76, 73, 84, 84, 76, 197, 76, 73, 84, 84, 69, - 210, 76, 73, 84, 82, 193, 76, 73, 84, 200, 76, 73, 83, 213, 76, 73, 83, - 128, 76, 73, 82, 193, 76, 73, 81, 85, 73, 68, 128, 76, 73, 81, 85, 73, - 196, 76, 73, 81, 128, 76, 73, 80, 83, 84, 73, 67, 75, 128, 76, 73, 80, - 211, 76, 73, 208, 76, 73, 78, 75, 73, 78, 199, 76, 73, 78, 75, 69, 196, - 76, 73, 78, 203, 76, 73, 78, 71, 83, 65, 128, 76, 73, 78, 69, 83, 128, - 76, 73, 78, 69, 211, 76, 73, 78, 69, 45, 57, 128, 76, 73, 78, 69, 45, 55, - 128, 76, 73, 78, 69, 45, 51, 128, 76, 73, 78, 69, 45, 49, 128, 76, 73, - 77, 77, 85, 52, 128, 76, 73, 77, 77, 85, 50, 128, 76, 73, 77, 77, 85, - 128, 76, 73, 77, 77, 213, 76, 73, 77, 73, 84, 69, 196, 76, 73, 77, 73, - 84, 65, 84, 73, 79, 78, 128, 76, 73, 77, 73, 84, 128, 76, 73, 77, 69, - 128, 76, 73, 77, 66, 213, 76, 73, 77, 66, 211, 76, 73, 77, 194, 76, 73, - 76, 89, 128, 76, 73, 76, 73, 84, 72, 128, 76, 73, 76, 128, 76, 73, 71, - 72, 84, 78, 73, 78, 71, 128, 76, 73, 71, 72, 84, 78, 73, 78, 199, 76, 73, - 71, 72, 84, 72, 79, 85, 83, 69, 128, 76, 73, 71, 72, 84, 128, 76, 73, 71, - 65, 84, 73, 78, 199, 76, 73, 70, 84, 69, 82, 128, 76, 73, 70, 69, 128, - 76, 73, 69, 88, 128, 76, 73, 69, 84, 128, 76, 73, 69, 80, 128, 76, 73, - 69, 69, 128, 76, 73, 69, 128, 76, 73, 68, 128, 76, 73, 67, 75, 73, 78, - 199, 76, 73, 66, 82, 65, 128, 76, 73, 66, 69, 82, 84, 89, 128, 76, 73, - 65, 66, 73, 76, 73, 84, 217, 76, 72, 73, 73, 128, 76, 72, 65, 86, 73, 89, - 65, 78, 73, 128, 76, 72, 65, 199, 76, 72, 65, 65, 128, 76, 72, 128, 76, - 69, 90, 72, 128, 76, 69, 90, 200, 76, 69, 88, 128, 76, 69, 86, 73, 84, - 65, 84, 73, 78, 71, 128, 76, 69, 86, 69, 76, 45, 51, 128, 76, 69, 86, 69, - 76, 45, 50, 128, 76, 69, 85, 77, 128, 76, 69, 85, 65, 69, 80, 128, 76, - 69, 85, 65, 69, 77, 128, 76, 69, 85, 128, 76, 69, 213, 76, 69, 84, 84, - 69, 82, 83, 128, 76, 69, 84, 84, 69, 82, 128, 76, 69, 212, 76, 69, 83, - 83, 69, 210, 76, 69, 83, 83, 45, 84, 72, 65, 78, 128, 76, 69, 83, 83, 45, - 84, 72, 65, 206, 76, 69, 83, 72, 128, 76, 69, 80, 67, 72, 193, 76, 69, - 80, 128, 76, 69, 79, 80, 65, 82, 68, 128, 76, 69, 79, 128, 76, 69, 78, - 84, 73, 67, 85, 76, 65, 210, 76, 69, 78, 73, 83, 128, 76, 69, 78, 73, - 211, 76, 69, 78, 71, 84, 72, 69, 78, 69, 82, 128, 76, 69, 78, 71, 84, 72, - 45, 55, 128, 76, 69, 78, 71, 84, 72, 45, 54, 128, 76, 69, 78, 71, 84, 72, - 45, 53, 128, 76, 69, 78, 71, 84, 72, 45, 52, 128, 76, 69, 78, 71, 84, 72, - 45, 51, 128, 76, 69, 78, 71, 84, 72, 45, 50, 128, 76, 69, 78, 71, 84, 72, - 45, 49, 128, 76, 69, 78, 71, 84, 200, 76, 69, 78, 71, 65, 128, 76, 69, - 78, 71, 193, 76, 69, 77, 79, 78, 128, 76, 69, 77, 79, 73, 128, 76, 69, - 76, 69, 84, 128, 76, 69, 76, 69, 212, 76, 69, 203, 76, 69, 73, 77, 77, - 65, 128, 76, 69, 73, 77, 77, 193, 76, 69, 73, 128, 76, 69, 71, 83, 128, - 76, 69, 71, 73, 79, 78, 128, 76, 69, 71, 69, 84, 79, 211, 76, 69, 71, - 128, 76, 69, 199, 76, 69, 70, 84, 87, 65, 82, 68, 83, 128, 76, 69, 70, - 84, 45, 84, 79, 45, 82, 73, 71, 72, 212, 76, 69, 70, 84, 45, 83, 84, 69, - 205, 76, 69, 70, 84, 45, 83, 73, 68, 197, 76, 69, 70, 84, 45, 83, 72, 65, - 68, 69, 196, 76, 69, 70, 84, 45, 80, 79, 73, 78, 84, 73, 78, 199, 76, 69, - 70, 84, 45, 76, 73, 71, 72, 84, 69, 196, 76, 69, 70, 84, 45, 72, 65, 78, - 68, 69, 196, 76, 69, 70, 84, 45, 72, 65, 78, 196, 76, 69, 70, 84, 45, 70, - 65, 67, 73, 78, 199, 76, 69, 70, 84, 128, 76, 69, 69, 82, 65, 69, 87, 65, - 128, 76, 69, 69, 75, 128, 76, 69, 69, 69, 69, 128, 76, 69, 68, 71, 69, - 82, 128, 76, 69, 65, 84, 72, 69, 82, 128, 76, 69, 65, 78, 73, 78, 199, - 76, 69, 65, 70, 217, 76, 69, 65, 70, 128, 76, 69, 65, 198, 76, 69, 65, - 68, 69, 82, 128, 76, 69, 65, 196, 76, 68, 65, 78, 128, 76, 68, 50, 128, - 76, 67, 201, 76, 67, 197, 76, 65, 90, 217, 76, 65, 89, 65, 78, 78, 65, - 128, 76, 65, 88, 128, 76, 65, 87, 128, 76, 65, 215, 76, 65, 85, 76, 65, - 128, 76, 65, 85, 75, 65, 218, 76, 65, 85, 74, 128, 76, 65, 85, 71, 72, - 73, 78, 71, 128, 76, 65, 84, 73, 78, 65, 84, 197, 76, 65, 84, 73, 75, - 128, 76, 65, 84, 69, 82, 65, 204, 76, 65, 84, 197, 76, 65, 83, 212, 76, - 65, 82, 89, 78, 71, 69, 65, 204, 76, 65, 82, 201, 76, 65, 82, 71, 69, 83, - 84, 128, 76, 65, 82, 71, 69, 210, 76, 65, 82, 71, 69, 128, 76, 65, 82, - 71, 197, 76, 65, 81, 128, 76, 65, 80, 65, 81, 128, 76, 65, 207, 76, 65, - 78, 84, 69, 82, 78, 128, 76, 65, 78, 84, 65, 78, 71, 128, 76, 65, 78, 71, - 85, 65, 71, 197, 76, 65, 78, 69, 83, 128, 76, 65, 78, 196, 76, 65, 78, - 128, 76, 65, 77, 80, 128, 76, 65, 77, 69, 68, 72, 128, 76, 65, 77, 69, - 68, 128, 76, 65, 77, 69, 196, 76, 65, 77, 69, 128, 76, 65, 77, 197, 76, - 65, 77, 68, 65, 128, 76, 65, 77, 68, 128, 76, 65, 77, 66, 68, 193, 76, - 65, 77, 65, 68, 72, 128, 76, 65, 76, 128, 76, 65, 204, 76, 65, 75, 75, - 72, 65, 78, 71, 89, 65, 79, 128, 76, 65, 75, 72, 65, 78, 128, 76, 65, 75, - 72, 128, 76, 65, 75, 200, 76, 65, 75, 45, 55, 52, 57, 128, 76, 65, 75, - 45, 55, 50, 52, 128, 76, 65, 75, 45, 54, 54, 56, 128, 76, 65, 75, 45, 54, - 52, 56, 128, 76, 65, 75, 45, 54, 52, 184, 76, 65, 75, 45, 54, 51, 54, - 128, 76, 65, 75, 45, 54, 49, 55, 128, 76, 65, 75, 45, 54, 49, 183, 76, - 65, 75, 45, 54, 48, 56, 128, 76, 65, 75, 45, 53, 53, 48, 128, 76, 65, 75, - 45, 52, 57, 53, 128, 76, 65, 75, 45, 52, 57, 51, 128, 76, 65, 75, 45, 52, - 57, 50, 128, 76, 65, 75, 45, 52, 57, 48, 128, 76, 65, 75, 45, 52, 56, 51, - 128, 76, 65, 75, 45, 52, 55, 48, 128, 76, 65, 75, 45, 52, 53, 55, 128, - 76, 65, 75, 45, 52, 53, 48, 128, 76, 65, 75, 45, 52, 52, 57, 128, 76, 65, - 75, 45, 52, 52, 185, 76, 65, 75, 45, 52, 52, 49, 128, 76, 65, 75, 45, 51, - 57, 48, 128, 76, 65, 75, 45, 51, 56, 52, 128, 76, 65, 75, 45, 51, 56, 51, - 128, 76, 65, 75, 45, 51, 52, 56, 128, 76, 65, 75, 45, 51, 52, 55, 128, - 76, 65, 75, 45, 51, 52, 51, 128, 76, 65, 75, 45, 50, 54, 54, 128, 76, 65, - 75, 45, 50, 54, 53, 128, 76, 65, 75, 45, 50, 51, 56, 128, 76, 65, 75, 45, - 50, 50, 56, 128, 76, 65, 75, 45, 50, 50, 53, 128, 76, 65, 75, 45, 50, 50, - 48, 128, 76, 65, 75, 45, 50, 49, 57, 128, 76, 65, 75, 45, 50, 49, 48, - 128, 76, 65, 75, 45, 49, 52, 50, 128, 76, 65, 75, 45, 49, 51, 48, 128, - 76, 65, 75, 45, 48, 57, 50, 128, 76, 65, 75, 45, 48, 56, 49, 128, 76, 65, - 75, 45, 48, 56, 177, 76, 65, 75, 45, 48, 56, 48, 128, 76, 65, 75, 45, 48, - 55, 185, 76, 65, 75, 45, 48, 54, 50, 128, 76, 65, 75, 45, 48, 53, 49, - 128, 76, 65, 75, 45, 48, 53, 48, 128, 76, 65, 75, 45, 48, 51, 48, 128, - 76, 65, 75, 45, 48, 50, 53, 128, 76, 65, 75, 45, 48, 50, 49, 128, 76, 65, - 75, 45, 48, 50, 48, 128, 76, 65, 75, 45, 48, 48, 51, 128, 76, 65, 74, 65, - 78, 89, 65, 76, 65, 78, 128, 76, 65, 73, 78, 199, 76, 65, 201, 76, 65, - 72, 83, 72, 85, 128, 76, 65, 72, 128, 76, 65, 71, 85, 83, 128, 76, 65, - 71, 213, 76, 65, 71, 65, 82, 128, 76, 65, 71, 65, 210, 76, 65, 71, 65, - 66, 128, 76, 65, 71, 65, 194, 76, 65, 69, 86, 128, 76, 65, 69, 128, 76, - 65, 68, 217, 76, 65, 68, 68, 69, 82, 128, 76, 65, 67, 82, 79, 83, 83, - 197, 76, 65, 67, 75, 128, 76, 65, 67, 65, 128, 76, 65, 66, 79, 85, 82, - 73, 78, 71, 128, 76, 65, 66, 79, 82, 128, 76, 65, 66, 73, 65, 76, 73, 90, - 65, 84, 73, 79, 206, 76, 65, 66, 73, 65, 204, 76, 65, 66, 69, 76, 128, - 76, 65, 66, 65, 84, 128, 76, 65, 194, 76, 65, 65, 78, 65, 69, 128, 76, - 65, 65, 78, 128, 76, 65, 65, 77, 85, 128, 76, 65, 65, 73, 128, 76, 54, - 128, 76, 52, 128, 76, 51, 128, 76, 50, 128, 76, 48, 48, 54, 65, 128, 76, - 48, 48, 50, 65, 128, 76, 45, 84, 89, 80, 197, 76, 45, 83, 72, 65, 80, 69, - 196, 75, 89, 85, 82, 73, 73, 128, 75, 89, 85, 128, 75, 89, 79, 128, 75, - 89, 76, 73, 83, 77, 65, 128, 75, 89, 73, 128, 75, 89, 69, 128, 75, 89, - 65, 84, 72, 79, 211, 75, 89, 65, 65, 128, 75, 89, 65, 128, 75, 88, 87, - 73, 128, 75, 88, 87, 69, 69, 128, 75, 88, 87, 69, 128, 75, 88, 87, 65, - 65, 128, 75, 88, 87, 65, 128, 75, 88, 85, 128, 75, 88, 79, 128, 75, 88, - 73, 128, 75, 88, 69, 69, 128, 75, 88, 69, 128, 75, 88, 65, 65, 128, 75, - 88, 65, 128, 75, 87, 86, 128, 75, 87, 85, 51, 49, 56, 128, 75, 87, 79, - 79, 128, 75, 87, 79, 128, 75, 87, 77, 128, 75, 87, 73, 73, 128, 75, 87, - 73, 128, 75, 87, 69, 69, 128, 75, 87, 69, 128, 75, 87, 66, 128, 75, 87, - 65, 89, 128, 75, 87, 65, 69, 84, 128, 75, 87, 65, 65, 128, 75, 86, 65, - 128, 75, 86, 128, 75, 85, 90, 72, 73, 128, 75, 85, 88, 128, 75, 85, 86, - 128, 75, 85, 85, 72, 128, 75, 85, 84, 128, 75, 85, 83, 77, 65, 128, 75, - 85, 83, 72, 85, 50, 128, 75, 85, 83, 72, 85, 178, 75, 85, 82, 88, 128, - 75, 85, 82, 85, 90, 69, 73, 82, 79, 128, 75, 85, 82, 84, 128, 75, 85, 82, - 79, 79, 78, 69, 128, 75, 85, 82, 128, 75, 85, 210, 75, 85, 81, 128, 75, - 85, 80, 78, 65, 89, 65, 128, 75, 85, 79, 88, 128, 75, 85, 79, 80, 128, - 75, 85, 79, 208, 75, 85, 79, 77, 128, 75, 85, 79, 128, 75, 85, 78, 71, - 128, 75, 85, 78, 68, 68, 65, 76, 73, 89, 65, 128, 75, 85, 76, 128, 75, - 85, 204, 75, 85, 71, 128, 75, 85, 70, 73, 83, 77, 65, 128, 75, 85, 69, - 84, 128, 75, 85, 66, 128, 75, 85, 65, 86, 128, 75, 85, 65, 66, 128, 75, - 85, 65, 128, 75, 85, 55, 128, 75, 85, 52, 128, 75, 85, 180, 75, 85, 51, - 128, 75, 85, 179, 75, 85, 45, 55, 128, 75, 85, 45, 54, 128, 75, 85, 45, - 53, 128, 75, 85, 45, 52, 128, 75, 85, 45, 51, 128, 75, 85, 45, 50, 128, - 75, 85, 45, 49, 128, 75, 84, 128, 75, 83, 83, 85, 85, 128, 75, 83, 83, - 85, 128, 75, 83, 83, 79, 79, 128, 75, 83, 83, 79, 128, 75, 83, 83, 73, - 73, 128, 75, 83, 83, 73, 128, 75, 83, 83, 69, 69, 128, 75, 83, 83, 69, - 128, 75, 83, 83, 65, 85, 128, 75, 83, 83, 65, 73, 128, 75, 83, 83, 65, - 65, 128, 75, 83, 83, 65, 128, 75, 83, 83, 128, 75, 83, 73, 128, 75, 82, - 89, 90, 72, 69, 86, 65, 89, 65, 128, 75, 82, 89, 90, 72, 69, 77, 128, 75, - 82, 89, 90, 72, 69, 205, 75, 82, 89, 90, 72, 128, 75, 82, 89, 90, 200, - 75, 82, 89, 85, 75, 79, 86, 65, 89, 65, 128, 75, 82, 89, 85, 75, 79, 86, - 65, 89, 193, 75, 82, 89, 85, 75, 128, 75, 82, 89, 85, 203, 75, 82, 79, - 78, 79, 83, 128, 75, 82, 69, 77, 65, 83, 84, 73, 128, 75, 82, 65, 84, 73, - 77, 79, 89, 80, 79, 82, 82, 79, 79, 78, 128, 75, 82, 65, 84, 73, 77, 79, - 75, 79, 85, 70, 73, 83, 77, 65, 128, 75, 82, 65, 84, 73, 77, 65, 84, 65, - 128, 75, 82, 65, 84, 73, 77, 193, 75, 80, 85, 128, 75, 80, 79, 81, 128, - 75, 80, 79, 79, 128, 75, 80, 79, 128, 75, 80, 73, 128, 75, 80, 69, 85, - 88, 128, 75, 80, 69, 69, 128, 75, 80, 69, 128, 75, 80, 65, 82, 65, 81, - 128, 75, 80, 65, 78, 128, 75, 80, 65, 72, 128, 75, 80, 65, 128, 75, 80, - 128, 75, 79, 88, 128, 75, 79, 86, 85, 85, 128, 75, 79, 86, 128, 75, 79, - 84, 79, 128, 75, 79, 82, 85, 78, 65, 128, 75, 79, 82, 79, 78, 73, 83, - 128, 75, 79, 82, 79, 78, 128, 75, 79, 82, 69, 65, 206, 75, 79, 82, 65, - 78, 73, 195, 75, 79, 81, 78, 68, 79, 78, 128, 75, 79, 80, 80, 65, 128, - 75, 79, 80, 128, 75, 79, 79, 86, 128, 75, 79, 79, 80, 79, 128, 75, 79, - 79, 77, 85, 85, 84, 128, 75, 79, 79, 66, 128, 75, 79, 79, 128, 75, 79, - 78, 84, 69, 86, 77, 65, 128, 75, 79, 78, 84, 69, 86, 77, 193, 75, 79, 77, - 201, 75, 79, 77, 66, 85, 86, 65, 128, 75, 79, 77, 66, 85, 86, 193, 75, - 79, 77, 66, 213, 75, 79, 75, 79, 128, 75, 79, 75, 69, 128, 75, 79, 75, - 128, 75, 79, 203, 75, 79, 73, 78, 73, 128, 75, 79, 73, 128, 75, 79, 201, - 75, 79, 72, 128, 75, 79, 71, 72, 79, 77, 128, 75, 79, 69, 84, 128, 75, - 79, 66, 89, 76, 65, 128, 75, 79, 66, 128, 75, 79, 65, 76, 65, 128, 75, - 79, 65, 128, 75, 79, 45, 75, 73, 128, 75, 79, 45, 51, 128, 75, 79, 45, - 50, 128, 75, 79, 45, 49, 128, 75, 78, 85, 67, 75, 76, 69, 83, 128, 75, - 78, 85, 67, 75, 76, 69, 128, 75, 78, 79, 84, 128, 75, 78, 79, 66, 83, - 128, 75, 78, 73, 71, 72, 84, 45, 82, 79, 79, 75, 128, 75, 78, 73, 71, 72, - 84, 45, 81, 85, 69, 69, 78, 128, 75, 78, 73, 71, 72, 84, 45, 66, 73, 83, - 72, 79, 80, 128, 75, 78, 73, 71, 72, 84, 128, 75, 78, 73, 71, 72, 212, - 75, 78, 73, 70, 69, 128, 75, 78, 73, 70, 197, 75, 78, 69, 69, 76, 73, 78, - 199, 75, 77, 128, 75, 205, 75, 76, 89, 85, 67, 72, 69, 86, 79, 89, 128, - 75, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 128, 75, 76, 89, 85, 67, 72, - 69, 86, 65, 89, 193, 75, 76, 89, 85, 67, 72, 69, 80, 79, 86, 79, 68, 78, - 89, 128, 75, 76, 89, 85, 67, 72, 69, 80, 79, 86, 79, 68, 78, 65, 89, 65, - 128, 75, 76, 89, 85, 67, 72, 69, 78, 69, 80, 79, 83, 84, 79, 89, 65, 78, - 78, 89, 128, 75, 76, 89, 85, 67, 72, 69, 78, 69, 80, 79, 83, 84, 79, 89, - 65, 78, 78, 65, 89, 65, 128, 75, 76, 89, 85, 67, 72, 128, 75, 76, 73, 84, - 79, 78, 128, 75, 76, 65, 83, 77, 65, 128, 75, 76, 65, 83, 77, 193, 75, - 76, 65, 128, 75, 76, 128, 75, 75, 79, 128, 75, 75, 73, 128, 75, 75, 69, - 69, 128, 75, 75, 69, 128, 75, 75, 65, 128, 75, 75, 128, 75, 74, 69, 128, - 75, 73, 89, 69, 79, 75, 45, 84, 73, 75, 69, 85, 84, 128, 75, 73, 89, 69, - 79, 75, 45, 83, 73, 79, 83, 45, 75, 73, 89, 69, 79, 75, 128, 75, 73, 89, - 69, 79, 75, 45, 82, 73, 69, 85, 76, 128, 75, 73, 89, 69, 79, 75, 45, 80, - 73, 69, 85, 80, 128, 75, 73, 89, 69, 79, 75, 45, 78, 73, 69, 85, 78, 128, - 75, 73, 89, 69, 79, 75, 45, 75, 72, 73, 69, 85, 75, 72, 128, 75, 73, 89, - 69, 79, 75, 45, 67, 72, 73, 69, 85, 67, 72, 128, 75, 73, 89, 69, 79, 203, - 75, 73, 88, 128, 75, 73, 87, 73, 70, 82, 85, 73, 84, 128, 75, 73, 87, - 128, 75, 73, 86, 128, 75, 73, 84, 69, 128, 75, 73, 84, 128, 75, 73, 83, - 83, 73, 78, 199, 75, 73, 83, 83, 128, 75, 73, 83, 211, 75, 73, 83, 73, - 77, 53, 128, 75, 73, 83, 73, 77, 181, 75, 73, 83, 72, 128, 75, 73, 83, - 65, 76, 128, 75, 73, 82, 79, 87, 65, 84, 84, 79, 128, 75, 73, 82, 79, 77, - 69, 69, 84, 79, 82, 85, 128, 75, 73, 82, 79, 71, 85, 82, 65, 77, 85, 128, - 75, 73, 82, 79, 128, 75, 73, 82, 71, 72, 73, 218, 75, 73, 81, 128, 75, - 73, 80, 128, 75, 73, 208, 75, 73, 78, 83, 72, 73, 80, 128, 75, 73, 78, - 78, 193, 75, 73, 78, 68, 69, 82, 71, 65, 82, 84, 69, 78, 128, 75, 73, 77, - 79, 78, 79, 128, 75, 73, 76, 76, 69, 82, 128, 75, 73, 73, 90, 72, 128, - 75, 73, 73, 128, 75, 73, 72, 128, 75, 73, 69, 88, 128, 75, 73, 69, 86, - 65, 206, 75, 73, 69, 80, 128, 75, 73, 69, 69, 77, 128, 75, 73, 69, 128, - 75, 73, 68, 128, 75, 73, 196, 75, 73, 67, 75, 128, 75, 73, 66, 128, 75, - 73, 65, 86, 128, 75, 73, 65, 66, 128, 75, 73, 45, 56, 128, 75, 73, 45, - 55, 128, 75, 73, 45, 54, 128, 75, 73, 45, 53, 128, 75, 73, 45, 52, 128, - 75, 73, 45, 51, 128, 75, 73, 45, 50, 128, 75, 73, 45, 49, 128, 75, 72, - 90, 128, 75, 72, 87, 65, 73, 128, 75, 72, 85, 69, 78, 45, 76, 85, 197, - 75, 72, 85, 69, 206, 75, 72, 85, 68, 65, 87, 65, 68, 201, 75, 72, 85, 68, - 65, 77, 128, 75, 72, 85, 65, 84, 128, 75, 72, 79, 85, 128, 75, 72, 79, - 212, 75, 72, 79, 78, 78, 65, 128, 75, 72, 79, 78, 128, 75, 72, 79, 77, - 85, 84, 128, 75, 72, 79, 75, 72, 76, 79, 205, 75, 72, 79, 74, 75, 201, - 75, 72, 79, 128, 75, 72, 207, 75, 72, 77, 213, 75, 72, 73, 84, 128, 75, - 72, 73, 78, 89, 65, 128, 75, 72, 73, 69, 85, 75, 200, 75, 72, 73, 128, - 75, 72, 201, 75, 72, 72, 79, 128, 75, 72, 72, 65, 128, 75, 72, 69, 84, - 72, 128, 75, 72, 69, 73, 128, 75, 72, 69, 69, 128, 75, 72, 69, 128, 75, - 72, 65, 86, 128, 75, 72, 65, 82, 79, 83, 72, 84, 72, 201, 75, 72, 65, 82, - 128, 75, 72, 65, 80, 72, 128, 75, 72, 65, 78, 199, 75, 72, 65, 78, 68, - 65, 128, 75, 72, 65, 78, 68, 193, 75, 72, 65, 77, 84, 201, 75, 72, 65, - 77, 73, 76, 79, 128, 75, 72, 65, 75, 65, 83, 83, 73, 65, 206, 75, 72, 65, - 73, 128, 75, 72, 65, 72, 128, 75, 72, 65, 200, 75, 72, 65, 70, 128, 75, - 72, 65, 66, 128, 75, 72, 65, 65, 128, 75, 71, 128, 75, 69, 89, 67, 65, - 80, 128, 75, 69, 89, 67, 65, 208, 75, 69, 89, 66, 79, 65, 82, 68, 128, - 75, 69, 89, 66, 79, 65, 82, 196, 75, 69, 88, 128, 75, 69, 86, 128, 75, - 69, 85, 89, 69, 85, 88, 128, 75, 69, 85, 83, 72, 69, 85, 65, 69, 80, 128, - 75, 69, 85, 83, 69, 85, 88, 128, 75, 69, 85, 80, 85, 81, 128, 75, 69, 85, - 79, 212, 75, 69, 85, 77, 128, 75, 69, 85, 75, 69, 85, 84, 78, 68, 65, - 128, 75, 69, 85, 75, 65, 81, 128, 75, 69, 85, 65, 69, 84, 77, 69, 85, 78, - 128, 75, 69, 85, 65, 69, 82, 73, 128, 75, 69, 84, 84, 201, 75, 69, 83, - 72, 50, 128, 75, 69, 82, 69, 84, 128, 75, 69, 79, 87, 128, 75, 69, 78, - 84, 73, 77, 65, 84, 65, 128, 75, 69, 78, 84, 73, 77, 65, 84, 193, 75, 69, - 78, 84, 73, 77, 193, 75, 69, 78, 65, 84, 128, 75, 69, 78, 128, 75, 69, - 206, 75, 69, 77, 80, 85, 76, 128, 75, 69, 77, 80, 85, 204, 75, 69, 77, - 80, 76, 73, 128, 75, 69, 77, 80, 76, 201, 75, 69, 77, 80, 72, 82, 69, 78, - 71, 128, 75, 69, 77, 66, 65, 78, 71, 128, 75, 69, 76, 86, 73, 206, 75, - 69, 72, 69, 72, 128, 75, 69, 72, 69, 200, 75, 69, 72, 128, 75, 69, 70, - 85, 76, 65, 128, 75, 69, 69, 86, 128, 75, 69, 69, 83, 85, 128, 75, 69, - 69, 80, 73, 78, 199, 75, 69, 69, 78, 71, 128, 75, 69, 69, 66, 128, 75, - 69, 66, 128, 75, 69, 65, 65, 69, 128, 75, 67, 65, 76, 128, 75, 66, 128, - 75, 65, 90, 65, 75, 200, 75, 65, 89, 65, 78, 78, 65, 128, 75, 65, 89, 65, - 200, 75, 65, 88, 128, 75, 65, 87, 86, 128, 75, 65, 87, 73, 128, 75, 65, - 87, 201, 75, 65, 87, 66, 128, 75, 65, 86, 89, 75, 65, 128, 75, 65, 86, - 89, 75, 193, 75, 65, 86, 128, 75, 65, 85, 86, 128, 75, 65, 85, 78, 65, - 128, 75, 65, 85, 206, 75, 65, 85, 66, 128, 75, 65, 84, 79, 128, 75, 65, - 84, 72, 73, 83, 84, 73, 128, 75, 65, 84, 72, 65, 75, 193, 75, 65, 84, 65, - 86, 65, 83, 77, 65, 128, 75, 65, 84, 65, 86, 193, 75, 65, 84, 65, 75, 65, - 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 193, 75, 65, 83, 82, 65, 84, 65, - 78, 128, 75, 65, 83, 82, 65, 84, 65, 206, 75, 65, 83, 82, 65, 128, 75, - 65, 83, 82, 193, 75, 65, 83, 75, 65, 76, 128, 75, 65, 83, 75, 65, 204, - 75, 65, 83, 72, 77, 73, 82, 201, 75, 65, 82, 83, 72, 65, 78, 65, 128, 75, - 65, 82, 79, 82, 73, 73, 128, 75, 65, 82, 79, 82, 65, 78, 128, 75, 65, 82, - 79, 82, 128, 75, 65, 82, 207, 75, 65, 82, 69, 206, 75, 65, 82, 65, 84, - 84, 79, 128, 75, 65, 82, 65, 78, 128, 75, 65, 80, 89, 69, 79, 85, 78, 83, - 83, 65, 78, 71, 80, 73, 69, 85, 80, 128, 75, 65, 80, 89, 69, 79, 85, 78, - 82, 73, 69, 85, 76, 128, 75, 65, 80, 89, 69, 79, 85, 78, 80, 72, 73, 69, - 85, 80, 72, 128, 75, 65, 80, 89, 69, 79, 85, 78, 77, 73, 69, 85, 77, 128, - 75, 65, 80, 80, 65, 128, 75, 65, 80, 80, 193, 75, 65, 80, 79, 128, 75, - 65, 80, 72, 128, 75, 65, 80, 65, 76, 128, 75, 65, 80, 65, 128, 75, 65, - 208, 75, 65, 78, 84, 65, 74, 193, 75, 65, 78, 78, 65, 68, 193, 75, 65, - 78, 71, 65, 82, 79, 79, 128, 75, 65, 78, 71, 128, 75, 65, 78, 199, 75, - 65, 78, 65, 75, 79, 128, 75, 65, 77, 52, 128, 75, 65, 77, 50, 128, 75, - 65, 77, 128, 75, 65, 75, 84, 79, 86, 73, 203, 75, 65, 75, 79, 128, 75, - 65, 75, 65, 66, 65, 84, 128, 75, 65, 75, 128, 75, 65, 203, 75, 65, 73, - 86, 128, 75, 65, 73, 84, 72, 201, 75, 65, 73, 82, 73, 128, 75, 65, 73, - 66, 128, 75, 65, 73, 128, 75, 65, 201, 75, 65, 70, 65, 128, 75, 65, 70, - 128, 75, 65, 198, 75, 65, 68, 53, 128, 75, 65, 68, 181, 75, 65, 68, 52, - 128, 75, 65, 68, 51, 128, 75, 65, 68, 179, 75, 65, 68, 50, 128, 75, 65, - 68, 128, 75, 65, 67, 72, 75, 65, 128, 75, 65, 66, 193, 75, 65, 66, 128, - 75, 65, 65, 86, 128, 75, 65, 65, 73, 128, 75, 65, 65, 70, 85, 128, 75, - 65, 65, 70, 128, 75, 65, 65, 67, 85, 128, 75, 65, 65, 66, 65, 128, 75, - 65, 65, 66, 128, 75, 65, 50, 128, 75, 65, 178, 75, 65, 45, 75, 69, 128, - 75, 65, 45, 57, 128, 75, 65, 45, 56, 128, 75, 65, 45, 55, 128, 75, 65, - 45, 54, 128, 75, 65, 45, 53, 128, 75, 65, 45, 52, 128, 75, 65, 45, 51, - 128, 75, 65, 45, 50, 128, 75, 65, 45, 49, 49, 128, 75, 65, 45, 49, 48, - 128, 75, 65, 45, 49, 128, 75, 48, 48, 56, 128, 75, 48, 48, 55, 128, 75, - 48, 48, 54, 128, 75, 48, 48, 53, 128, 75, 48, 48, 52, 128, 75, 48, 48, - 51, 128, 75, 48, 48, 50, 128, 75, 48, 48, 49, 128, 74, 87, 65, 128, 74, - 85, 85, 128, 74, 85, 84, 128, 74, 85, 83, 84, 73, 70, 73, 67, 65, 84, 73, - 79, 78, 128, 74, 85, 80, 73, 84, 69, 82, 128, 74, 85, 79, 84, 128, 74, - 85, 79, 80, 128, 74, 85, 78, 79, 128, 74, 85, 78, 71, 83, 69, 79, 78, - 199, 74, 85, 78, 69, 128, 74, 85, 76, 89, 128, 74, 85, 71, 71, 76, 73, - 78, 71, 128, 74, 85, 69, 85, 73, 128, 74, 85, 68, 85, 76, 128, 74, 85, - 68, 71, 69, 128, 74, 85, 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 200, 74, - 79, 89, 83, 84, 73, 67, 75, 128, 74, 79, 89, 79, 85, 211, 74, 79, 89, - 128, 74, 79, 86, 69, 128, 74, 79, 212, 74, 79, 78, 71, 128, 74, 79, 78, - 193, 74, 79, 75, 69, 82, 128, 74, 79, 73, 78, 84, 83, 128, 74, 79, 73, - 78, 69, 68, 128, 74, 79, 73, 78, 128, 74, 79, 65, 128, 74, 78, 89, 65, - 128, 74, 74, 89, 88, 128, 74, 74, 89, 84, 128, 74, 74, 89, 80, 128, 74, - 74, 89, 128, 74, 74, 85, 88, 128, 74, 74, 85, 84, 128, 74, 74, 85, 82, - 88, 128, 74, 74, 85, 82, 128, 74, 74, 85, 80, 128, 74, 74, 85, 79, 88, - 128, 74, 74, 85, 79, 80, 128, 74, 74, 85, 79, 128, 74, 74, 85, 128, 74, - 74, 79, 88, 128, 74, 74, 79, 84, 128, 74, 74, 79, 80, 128, 74, 74, 79, - 128, 74, 74, 73, 88, 128, 74, 74, 73, 84, 128, 74, 74, 73, 80, 128, 74, - 74, 73, 69, 88, 128, 74, 74, 73, 69, 84, 128, 74, 74, 73, 69, 80, 128, - 74, 74, 73, 69, 128, 74, 74, 73, 128, 74, 74, 69, 69, 128, 74, 74, 69, - 128, 74, 74, 65, 128, 74, 73, 76, 128, 74, 73, 73, 77, 128, 74, 73, 73, - 128, 74, 73, 72, 86, 65, 77, 85, 76, 73, 89, 65, 128, 74, 73, 71, 83, 65, - 215, 74, 73, 65, 128, 74, 72, 79, 88, 128, 74, 72, 79, 128, 74, 72, 69, - 72, 128, 74, 72, 65, 89, 73, 78, 128, 74, 72, 65, 78, 128, 74, 72, 65, - 77, 128, 74, 72, 65, 65, 128, 74, 72, 65, 128, 74, 69, 85, 128, 74, 69, - 82, 85, 83, 65, 76, 69, 77, 128, 74, 69, 82, 65, 206, 74, 69, 82, 65, - 128, 74, 69, 82, 128, 74, 69, 76, 76, 89, 70, 73, 83, 72, 128, 74, 69, - 72, 128, 74, 69, 200, 74, 69, 71, 79, 71, 65, 78, 128, 74, 69, 69, 77, - 128, 74, 69, 69, 205, 74, 69, 65, 78, 83, 128, 74, 65, 89, 78, 128, 74, - 65, 89, 73, 78, 128, 74, 65, 89, 65, 78, 78, 65, 128, 74, 65, 87, 128, - 74, 65, 86, 73, 89, 65, 78, 73, 128, 74, 65, 86, 65, 78, 69, 83, 197, 74, - 65, 85, 128, 74, 65, 82, 128, 74, 65, 80, 65, 78, 69, 83, 197, 74, 65, - 80, 65, 78, 128, 74, 65, 78, 85, 65, 82, 89, 128, 74, 65, 76, 76, 65, 74, - 65, 76, 65, 76, 79, 85, 72, 79, 85, 128, 74, 65, 76, 76, 128, 74, 65, 73, - 206, 74, 65, 73, 128, 74, 65, 72, 128, 74, 65, 68, 69, 128, 74, 65, 67, - 75, 83, 128, 74, 65, 67, 75, 45, 79, 45, 76, 65, 78, 84, 69, 82, 78, 128, - 74, 65, 67, 203, 74, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 196, 73, 90, - 72, 73, 84, 83, 65, 128, 73, 90, 72, 73, 84, 83, 193, 73, 90, 72, 69, - 128, 73, 90, 65, 75, 65, 89, 193, 73, 89, 69, 75, 128, 73, 89, 65, 78, - 78, 65, 128, 73, 85, 74, 65, 128, 73, 84, 211, 73, 84, 69, 82, 65, 84, - 73, 79, 206, 73, 84, 69, 77, 128, 73, 83, 83, 72, 65, 82, 128, 73, 83, - 79, 83, 67, 69, 76, 69, 211, 73, 83, 79, 78, 128, 73, 83, 79, 206, 73, - 83, 79, 76, 65, 84, 69, 128, 73, 83, 76, 65, 78, 68, 128, 73, 83, 72, 77, - 65, 65, 77, 128, 73, 83, 69, 78, 45, 73, 83, 69, 78, 128, 73, 83, 65, 75, - 73, 193, 73, 83, 45, 80, 73, 76, 76, 65, 128, 73, 82, 85, 89, 65, 78, 78, - 65, 128, 73, 82, 85, 85, 89, 65, 78, 78, 65, 128, 73, 82, 79, 78, 45, 67, - 79, 80, 80, 69, 210, 73, 82, 79, 78, 128, 73, 82, 66, 128, 73, 79, 84, - 73, 70, 73, 69, 196, 73, 79, 84, 65, 84, 69, 196, 73, 79, 84, 65, 128, - 73, 79, 84, 193, 73, 79, 82, 128, 73, 79, 78, 71, 128, 73, 79, 68, 72, - 65, 68, 72, 128, 73, 78, 86, 73, 83, 73, 66, 76, 197, 73, 78, 86, 69, 82, - 84, 69, 68, 128, 73, 78, 86, 69, 82, 84, 69, 196, 73, 78, 86, 69, 82, 84, - 69, 66, 82, 65, 84, 69, 128, 73, 78, 86, 69, 82, 83, 197, 73, 78, 84, 82, - 79, 68, 85, 67, 69, 82, 128, 73, 78, 84, 73, 128, 73, 78, 84, 69, 82, 83, - 89, 76, 76, 65, 66, 73, 195, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, - 78, 128, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 206, 73, 78, 84, 69, - 82, 83, 69, 67, 84, 73, 78, 199, 73, 78, 84, 69, 82, 82, 79, 66, 65, 78, - 71, 128, 73, 78, 84, 69, 82, 82, 79, 66, 65, 78, 199, 73, 78, 84, 69, 82, - 80, 79, 76, 65, 84, 73, 79, 206, 73, 78, 84, 69, 82, 76, 79, 67, 75, 69, - 196, 73, 78, 84, 69, 82, 76, 73, 78, 69, 65, 210, 73, 78, 84, 69, 82, 76, - 65, 67, 69, 196, 73, 78, 84, 69, 82, 73, 79, 210, 73, 78, 84, 69, 82, 69, - 83, 212, 73, 78, 84, 69, 82, 67, 65, 76, 65, 84, 69, 128, 73, 78, 84, 69, - 71, 82, 65, 84, 73, 79, 78, 128, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, - 206, 73, 78, 84, 69, 71, 82, 65, 76, 128, 73, 78, 84, 69, 71, 82, 65, - 204, 73, 78, 83, 85, 76, 65, 210, 73, 78, 83, 84, 82, 85, 77, 69, 78, 84, - 65, 204, 73, 78, 83, 73, 68, 69, 128, 73, 78, 83, 73, 68, 197, 73, 78, - 83, 69, 82, 84, 73, 79, 206, 73, 78, 83, 69, 82, 212, 73, 78, 83, 69, 67, - 84, 128, 73, 78, 83, 67, 82, 73, 80, 84, 73, 79, 78, 65, 204, 73, 78, 80, - 85, 212, 73, 78, 78, 79, 67, 69, 78, 67, 69, 128, 73, 78, 78, 78, 128, - 73, 78, 78, 69, 82, 128, 73, 78, 78, 69, 210, 73, 78, 78, 128, 73, 78, - 73, 78, 71, 85, 128, 73, 78, 72, 73, 66, 73, 212, 73, 78, 72, 69, 82, 69, - 78, 212, 73, 78, 72, 65, 76, 69, 128, 73, 78, 71, 87, 65, 90, 128, 73, - 78, 70, 79, 82, 77, 65, 84, 73, 79, 206, 73, 78, 70, 76, 85, 69, 78, 67, - 69, 128, 73, 78, 70, 73, 78, 73, 84, 89, 128, 73, 78, 70, 73, 78, 73, 84, - 217, 73, 78, 68, 85, 83, 84, 82, 73, 65, 204, 73, 78, 68, 73, 82, 69, 67, - 212, 73, 78, 68, 73, 67, 84, 73, 79, 206, 73, 78, 68, 73, 67, 65, 84, 79, - 82, 128, 73, 78, 68, 73, 67, 65, 84, 79, 210, 73, 78, 68, 73, 195, 73, - 78, 68, 73, 65, 206, 73, 78, 68, 69, 88, 128, 73, 78, 68, 69, 80, 69, 78, - 68, 69, 78, 212, 73, 78, 67, 82, 69, 77, 69, 78, 84, 128, 73, 78, 67, 82, - 69, 65, 83, 69, 211, 73, 78, 67, 82, 69, 65, 83, 69, 128, 73, 78, 67, 82, - 69, 65, 83, 197, 73, 78, 67, 79, 77, 80, 76, 69, 84, 197, 73, 78, 67, 79, - 77, 73, 78, 199, 73, 78, 67, 76, 85, 68, 73, 78, 199, 73, 78, 67, 72, - 128, 73, 78, 66, 79, 216, 73, 78, 65, 80, 128, 73, 78, 45, 65, 76, 65, - 70, 128, 73, 77, 80, 69, 82, 73, 65, 204, 73, 77, 80, 69, 82, 70, 69, 67, - 84, 85, 205, 73, 77, 80, 69, 82, 70, 69, 67, 84, 65, 128, 73, 77, 80, 69, - 82, 70, 69, 67, 84, 193, 73, 77, 78, 128, 73, 77, 73, 83, 69, 79, 211, - 73, 77, 73, 78, 51, 128, 73, 77, 73, 78, 128, 73, 77, 73, 206, 73, 77, - 73, 70, 84, 72, 79, 82, 79, 78, 128, 73, 77, 73, 70, 84, 72, 79, 82, 65, - 128, 73, 77, 73, 70, 79, 78, 79, 78, 128, 73, 77, 73, 68, 73, 65, 82, 71, - 79, 78, 128, 73, 77, 65, 71, 197, 73, 77, 65, 65, 76, 65, 128, 73, 76, - 85, 89, 65, 78, 78, 65, 128, 73, 76, 85, 89, 128, 73, 76, 85, 85, 89, 65, - 78, 78, 65, 128, 73, 76, 85, 84, 128, 73, 76, 73, 77, 77, 85, 52, 128, - 73, 76, 73, 77, 77, 85, 51, 128, 73, 76, 73, 77, 77, 85, 128, 73, 76, 73, - 77, 77, 213, 73, 76, 50, 128, 73, 75, 73, 82, 128, 73, 75, 65, 82, 65, - 128, 73, 75, 65, 82, 193, 73, 74, 128, 73, 73, 89, 65, 78, 78, 65, 128, - 73, 71, 73, 128, 73, 71, 201, 73, 71, 71, 87, 83, 128, 73, 70, 73, 78, - 128, 73, 69, 85, 78, 71, 45, 84, 73, 75, 69, 85, 84, 128, 73, 69, 85, 78, - 71, 45, 84, 72, 73, 69, 85, 84, 72, 128, 73, 69, 85, 78, 71, 45, 82, 73, - 69, 85, 76, 128, 73, 69, 85, 78, 71, 45, 80, 73, 69, 85, 80, 128, 73, 69, - 85, 78, 71, 45, 80, 72, 73, 69, 85, 80, 72, 128, 73, 69, 85, 78, 71, 45, - 67, 73, 69, 85, 67, 128, 73, 69, 85, 78, 71, 45, 67, 72, 73, 69, 85, 67, - 72, 128, 73, 69, 85, 78, 199, 73, 68, 76, 69, 128, 73, 68, 73, 77, 128, - 73, 68, 73, 205, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 68, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 68, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 68, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 68, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 68, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 67, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 67, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 67, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 67, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 67, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 67, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 67, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 67, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 67, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 66, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 66, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 66, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 66, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 66, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 66, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 66, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 66, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 66, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 66, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 65, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 65, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 65, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 65, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 65, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 65, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 65, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 65, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 57, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 57, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 57, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 57, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 57, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 57, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 57, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 57, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 57, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 56, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 56, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 56, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 56, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 56, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 56, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 56, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 56, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 56, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 56, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 55, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 55, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 55, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 55, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 55, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 55, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 55, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 55, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 54, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 54, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 54, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 54, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 54, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 54, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 54, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 54, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 54, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 53, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 53, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 53, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 53, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 53, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 53, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 53, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 53, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 52, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 52, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 52, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 52, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 52, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 52, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 52, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 52, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 52, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 51, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 51, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 51, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 51, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 51, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 51, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 51, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 51, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 51, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 51, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 50, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 50, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 50, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 50, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 50, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 50, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 50, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 50, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 49, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 49, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 49, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 49, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 49, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 49, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 49, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 49, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 49, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 48, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 48, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 48, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 48, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 48, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 48, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 65, 48, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 65, 48, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 65, 48, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 65, 48, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 70, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 70, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 70, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 70, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 70, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 70, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 70, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 70, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 69, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 69, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 69, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 69, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 69, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 69, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 69, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 69, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 69, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 68, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 68, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 68, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 68, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 68, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 68, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 68, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 68, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 68, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 68, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 67, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 67, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 67, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 67, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 67, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 67, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 67, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 67, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 66, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 66, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 66, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 66, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 66, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 66, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 66, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 66, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 66, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 65, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 65, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 65, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 65, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 65, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 65, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 65, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 65, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 65, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 65, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 57, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 57, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 57, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 57, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 57, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 57, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 57, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 57, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 56, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 56, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 56, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 56, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 56, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 56, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 56, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 56, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 56, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 55, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 55, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 55, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 55, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 55, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 55, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 55, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 55, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 55, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 55, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 54, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 54, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 54, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 54, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 54, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 54, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 54, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 54, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 53, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 53, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 53, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 53, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 53, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 53, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 53, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 53, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 53, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 52, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 52, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 52, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 52, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 52, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 52, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 52, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 52, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 52, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 52, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 51, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 51, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 51, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 51, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 51, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 51, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 51, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 51, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 50, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 50, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 50, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 50, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 50, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 54, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 50, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 50, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 50, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 50, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 49, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 49, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 49, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 65, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 49, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 49, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 49, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 52, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 49, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 49, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 49, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 49, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 69, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 48, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 48, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 48, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 56, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 48, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 70, 57, 48, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, - 57, 48, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 50, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 70, 57, 48, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 70, 57, 48, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 57, 49, 52, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 57, - 48, 52, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 56, 68, 55, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 56, 67, 65, 57, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 56, 57, 69, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 55, 68, 52, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 55, 65, 55, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 55, - 57, 56, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 55, 54, 68, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 55, 53, 51, 51, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 55, 53, 49, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 55, 49, 50, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 55, 48, 66, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, - 70, 49, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, 69, 56, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, 55, 50, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 54, 55, 48, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 54, 55, 48, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 54, 54, 50, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, - 53, 66, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, 53, 57, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, 53, 53, 55, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 54, 51, 53, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 54, 51, 48, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 54, 50, 57, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, - 50, 53, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 54, 50, 52, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, 70, 56, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 53, 68, 69, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 53, 66, 56, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 53, 66, 53, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, - 57, 50, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, 57, 49, 65, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, 56, 70, 48, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 53, 53, 66, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 53, 52, 51, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 53, 52, 48, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, - 51, 70, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, 51, 67, 67, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, 50, 68, 68, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 53, 50, 55, 50, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 53, 50, 52, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 53, 50, 49, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 53, - 49, 56, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 52, 69, 65, 52, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 52, 69, 56, 67, 128, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 45, 52, 69, 50, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 52, 69, 48, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, - 72, 45, 52, 69, 48, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 65, 49, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, - 49, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 65, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 57, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 56, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 65, 49, 55, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 65, 49, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 65, 49, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 65, 49, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, - 49, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 50, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 49, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 49, 48, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 65, 48, 69, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 65, 48, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 65, 48, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 65, 48, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, - 48, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 56, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 55, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 65, 48, 53, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 65, 48, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 65, 48, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 65, 48, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, - 48, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 65, 48, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 70, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 69, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 70, 67, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 70, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 70, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 70, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 70, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 54, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 53, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 70, 52, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 70, 51, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 70, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 70, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 70, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 69, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 69, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 68, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 67, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 69, 65, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 69, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 69, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 69, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 69, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 52, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 51, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 69, 50, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 69, 49, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 69, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 68, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 68, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 68, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 67, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 66, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 65, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 68, 56, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 68, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 68, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 68, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 68, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 50, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 49, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 68, 48, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 67, 70, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 67, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 67, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 67, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 67, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 65, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 57, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 56, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 67, 54, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 67, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 67, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 67, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 67, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 67, 48, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 70, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 69, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 66, 68, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 66, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 66, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 66, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 66, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 56, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 55, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 54, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 66, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 66, 52, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 66, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 66, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 66, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 66, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 69, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 68, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 67, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 65, 66, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 65, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 65, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 65, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 65, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 54, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 53, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 52, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 65, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 65, 50, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 65, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 65, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 57, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 57, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 67, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 66, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 65, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 57, 57, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 57, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 57, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 57, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 57, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 52, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 51, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 50, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 57, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 57, 48, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 56, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 56, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 56, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 56, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 65, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 57, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 56, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 56, 55, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 56, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 56, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 56, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 56, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 50, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 49, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 56, 48, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 55, 69, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 55, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 55, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 55, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 55, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 56, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 55, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 55, 53, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 55, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 55, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 55, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 55, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 55, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 70, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 69, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 54, 67, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 54, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 54, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 54, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 54, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 54, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 53, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 54, 52, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 54, 51, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 54, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 54, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 54, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 53, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 69, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 68, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 67, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 53, 65, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 53, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 53, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 53, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 53, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 52, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 51, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 53, 50, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 53, 49, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 53, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 52, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 52, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 52, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 67, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 66, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 65, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 52, 56, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 52, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 52, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 52, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 52, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 50, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 49, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 52, 48, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 51, 70, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 51, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 51, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 51, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 51, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 65, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 57, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 56, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 51, 54, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 51, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 51, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 51, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 51, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 51, 48, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 70, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 69, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 50, 68, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 50, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 50, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 50, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 50, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 56, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 55, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 54, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 50, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 50, 52, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 50, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 50, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 50, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 50, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 69, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 68, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 67, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 49, 66, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 49, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 49, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 49, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 49, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 54, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 53, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 52, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 49, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 49, 50, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 49, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 49, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 48, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 48, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 67, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 66, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 65, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 48, 57, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 57, 48, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 57, 48, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 57, 48, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, - 48, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 52, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 51, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 50, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 57, 48, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 57, 48, 48, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 70, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 70, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 70, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 70, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 65, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 57, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 56, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 70, 55, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 70, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 70, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 70, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 70, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 50, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 49, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 70, 48, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 69, 69, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 69, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 69, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 69, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 69, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 56, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 55, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 69, 53, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 69, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 69, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 69, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 69, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 69, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 70, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 69, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 68, 67, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 68, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 68, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 68, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 68, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 54, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 53, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 68, 52, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 68, 51, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 68, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 68, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 68, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 67, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 69, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 68, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 67, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 67, 65, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 67, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 67, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 67, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 67, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 52, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 51, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 67, 50, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 67, 49, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 67, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 66, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 66, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 66, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 67, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 66, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 65, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 66, 56, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 66, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 66, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 66, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 66, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 50, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 49, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 66, 48, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 65, 70, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 65, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 65, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 65, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 65, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 65, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 57, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 56, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 65, 54, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 65, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 65, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 65, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 65, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 65, 48, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 70, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 69, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 57, 68, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 57, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 57, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 57, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 57, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 56, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 55, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 54, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 57, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 57, 52, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 57, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 57, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 57, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 57, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 70, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 69, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 68, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 67, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 56, 66, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 56, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 56, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 56, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 56, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 54, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 53, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 52, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 56, 51, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 56, 50, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 56, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 56, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 55, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 55, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 68, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 67, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 66, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 65, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 55, 57, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 55, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 55, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 55, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 55, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 52, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 51, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 50, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 55, 49, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 55, 48, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 54, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 54, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 54, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 54, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 66, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 65, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 57, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 56, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 54, 55, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 54, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 54, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 54, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 54, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 50, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 49, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 54, 48, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 70, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 53, 69, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 53, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 53, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 53, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 53, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 57, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 56, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 55, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 54, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 53, 53, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 53, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 53, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 53, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 53, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 53, 48, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 70, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 69, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 68, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 52, 67, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 52, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 52, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 52, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 52, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 55, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 54, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 53, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 52, 52, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 52, 51, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 52, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 52, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 52, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 51, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 69, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 68, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 67, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 66, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 51, 65, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 51, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 51, 56, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 51, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 51, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 53, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 52, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 51, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 51, 50, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 51, 49, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 51, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 50, 70, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 50, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 50, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 67, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 66, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 65, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 57, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 50, 56, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 50, 55, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 50, 54, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 50, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 50, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 51, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 50, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 49, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 50, 48, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 49, 70, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 49, 69, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 49, 68, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 49, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 49, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 65, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 57, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 56, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 55, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 49, 54, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 49, 53, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 49, 52, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 49, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 49, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 49, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 49, 48, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 70, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 69, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 48, 68, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 48, 67, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 48, 66, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 48, 65, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 48, 57, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 56, - 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 55, 128, 73, - 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 54, 128, 73, 68, 69, - 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, 48, 53, 128, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 45, 50, 70, 56, 48, 52, 128, 73, 68, 69, 79, 71, 82, 65, - 80, 72, 45, 50, 70, 56, 48, 51, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, - 45, 50, 70, 56, 48, 50, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, - 70, 56, 48, 49, 128, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 50, 70, 56, - 48, 48, 128, 73, 68, 69, 79, 71, 82, 65, 80, 200, 73, 68, 69, 78, 84, 73, - 70, 73, 67, 65, 84, 73, 79, 78, 128, 73, 68, 69, 78, 84, 73, 70, 73, 67, - 65, 84, 73, 79, 206, 73, 68, 69, 78, 84, 73, 67, 65, 204, 73, 68, 68, - 128, 73, 67, 79, 78, 128, 73, 67, 72, 79, 85, 128, 73, 67, 72, 79, 83, - 128, 73, 67, 72, 73, 77, 65, 84, 79, 83, 128, 73, 67, 72, 65, 68, 73, 78, - 128, 73, 67, 69, 76, 65, 78, 68, 73, 67, 45, 89, 82, 128, 73, 66, 73, 70, - 73, 76, 73, 128, 73, 65, 85, 68, 65, 128, 73, 48, 49, 53, 128, 73, 48, - 49, 52, 128, 73, 48, 49, 51, 128, 73, 48, 49, 50, 128, 73, 48, 49, 49, - 65, 128, 73, 48, 49, 49, 128, 73, 48, 49, 48, 65, 128, 73, 48, 49, 48, - 128, 73, 48, 48, 57, 65, 128, 73, 48, 48, 57, 128, 73, 48, 48, 56, 128, - 73, 48, 48, 55, 128, 73, 48, 48, 54, 128, 73, 48, 48, 53, 65, 128, 73, - 48, 48, 53, 128, 73, 48, 48, 52, 128, 73, 48, 48, 51, 128, 73, 48, 48, - 50, 128, 73, 48, 48, 49, 128, 73, 45, 89, 85, 128, 73, 45, 89, 79, 128, - 73, 45, 89, 69, 79, 128, 73, 45, 89, 69, 128, 73, 45, 89, 65, 69, 128, - 73, 45, 89, 65, 45, 79, 128, 73, 45, 89, 65, 128, 73, 45, 79, 45, 73, - 128, 73, 45, 79, 128, 73, 45, 69, 85, 128, 73, 45, 66, 69, 65, 77, 128, - 73, 45, 65, 82, 65, 69, 65, 128, 73, 45, 65, 128, 72, 90, 90, 90, 71, - 128, 72, 90, 90, 90, 128, 72, 90, 90, 80, 128, 72, 90, 90, 128, 72, 90, - 87, 71, 128, 72, 90, 87, 128, 72, 90, 84, 128, 72, 90, 71, 128, 72, 89, - 83, 84, 69, 82, 69, 83, 73, 211, 72, 89, 80, 79, 68, 73, 65, 83, 84, 79, - 76, 69, 128, 72, 89, 80, 72, 69, 78, 65, 84, 73, 79, 206, 72, 89, 80, 72, - 69, 78, 45, 77, 73, 78, 85, 83, 128, 72, 89, 80, 72, 69, 78, 128, 72, 89, - 80, 72, 69, 206, 72, 89, 71, 73, 69, 73, 65, 128, 72, 89, 71, 73, 69, 65, - 128, 72, 89, 65, 67, 73, 78, 84, 72, 128, 72, 88, 87, 71, 128, 72, 88, - 85, 79, 88, 128, 72, 88, 85, 79, 84, 128, 72, 88, 85, 79, 80, 128, 72, - 88, 85, 79, 128, 72, 88, 79, 88, 128, 72, 88, 79, 84, 128, 72, 88, 79, - 80, 128, 72, 88, 79, 128, 72, 88, 73, 88, 128, 72, 88, 73, 84, 128, 72, - 88, 73, 80, 128, 72, 88, 73, 69, 88, 128, 72, 88, 73, 69, 84, 128, 72, - 88, 73, 69, 80, 128, 72, 88, 73, 69, 128, 72, 88, 73, 128, 72, 88, 69, - 88, 128, 72, 88, 69, 80, 128, 72, 88, 69, 128, 72, 88, 65, 88, 128, 72, - 88, 65, 84, 128, 72, 88, 65, 80, 128, 72, 88, 65, 128, 72, 87, 85, 128, - 72, 87, 65, 73, 82, 128, 72, 87, 65, 72, 128, 72, 85, 86, 65, 128, 72, - 85, 83, 72, 69, 196, 72, 85, 83, 72, 128, 72, 85, 82, 65, 78, 128, 72, - 85, 79, 84, 128, 72, 85, 78, 68, 82, 69, 68, 83, 128, 72, 85, 78, 68, 82, - 69, 68, 211, 72, 85, 78, 68, 82, 69, 68, 128, 72, 85, 78, 68, 82, 69, - 196, 72, 85, 78, 128, 72, 85, 77, 208, 72, 85, 77, 65, 78, 128, 72, 85, - 77, 65, 206, 72, 85, 76, 50, 128, 72, 85, 73, 73, 84, 79, 128, 72, 85, - 71, 71, 73, 78, 71, 128, 72, 85, 71, 71, 73, 78, 199, 72, 85, 66, 50, - 128, 72, 85, 66, 178, 72, 85, 66, 128, 72, 85, 65, 82, 65, 68, 68, 79, - 128, 72, 85, 65, 78, 128, 72, 85, 45, 51, 128, 72, 85, 45, 50, 128, 72, - 85, 45, 49, 128, 72, 84, 84, 65, 128, 72, 84, 83, 128, 72, 84, 74, 128, - 72, 82, 89, 86, 78, 73, 193, 72, 80, 87, 71, 128, 72, 80, 65, 128, 72, - 80, 128, 72, 79, 85, 83, 197, 72, 79, 85, 82, 71, 76, 65, 83, 83, 128, - 72, 79, 85, 82, 71, 76, 65, 83, 211, 72, 79, 85, 82, 128, 72, 79, 85, - 210, 72, 79, 84, 69, 76, 128, 72, 79, 84, 65, 128, 72, 79, 83, 80, 73, - 84, 65, 76, 128, 72, 79, 82, 83, 69, 128, 72, 79, 82, 83, 197, 72, 79, - 82, 82, 128, 72, 79, 82, 78, 83, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, - 76, 76, 89, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 76, 217, 72, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 54, 45, 48, 54, 128, 72, 79, 82, - 73, 90, 79, 78, 84, 65, 76, 45, 48, 54, 45, 48, 53, 128, 72, 79, 82, 73, - 90, 79, 78, 84, 65, 76, 45, 48, 54, 45, 48, 52, 128, 72, 79, 82, 73, 90, - 79, 78, 84, 65, 76, 45, 48, 54, 45, 48, 51, 128, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 45, 48, 54, 45, 48, 50, 128, 72, 79, 82, 73, 90, 79, 78, - 84, 65, 76, 45, 48, 54, 45, 48, 49, 128, 72, 79, 82, 73, 90, 79, 78, 84, - 65, 76, 45, 48, 54, 45, 48, 48, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, - 76, 45, 48, 53, 45, 48, 54, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, - 45, 48, 53, 45, 48, 53, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, - 48, 53, 45, 48, 52, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, - 53, 45, 48, 51, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 53, - 45, 48, 50, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 53, 45, - 48, 49, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 53, 45, 48, - 48, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 54, - 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 53, 128, - 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 52, 128, 72, - 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 51, 128, 72, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 50, 128, 72, 79, 82, - 73, 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 49, 128, 72, 79, 82, 73, - 90, 79, 78, 84, 65, 76, 45, 48, 52, 45, 48, 48, 128, 72, 79, 82, 73, 90, - 79, 78, 84, 65, 76, 45, 48, 51, 45, 48, 54, 128, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 45, 48, 51, 45, 48, 53, 128, 72, 79, 82, 73, 90, 79, 78, - 84, 65, 76, 45, 48, 51, 45, 48, 52, 128, 72, 79, 82, 73, 90, 79, 78, 84, - 65, 76, 45, 48, 51, 45, 48, 51, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, - 76, 45, 48, 51, 45, 48, 50, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, - 45, 48, 51, 45, 48, 49, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, - 48, 51, 45, 48, 48, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, - 50, 45, 48, 54, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, - 45, 48, 53, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, 45, - 48, 52, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, 45, 48, - 51, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, 45, 48, 50, - 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, 45, 48, 49, 128, - 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 50, 45, 48, 48, 128, 72, - 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 49, 45, 48, 54, 128, 72, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 49, 45, 48, 53, 128, 72, 79, 82, - 73, 90, 79, 78, 84, 65, 76, 45, 48, 49, 45, 48, 52, 128, 72, 79, 82, 73, - 90, 79, 78, 84, 65, 76, 45, 48, 49, 45, 48, 51, 128, 72, 79, 82, 73, 90, - 79, 78, 84, 65, 76, 45, 48, 49, 45, 48, 50, 128, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 45, 48, 49, 45, 48, 49, 128, 72, 79, 82, 73, 90, 79, 78, - 84, 65, 76, 45, 48, 49, 45, 48, 48, 128, 72, 79, 82, 73, 90, 79, 78, 84, - 65, 76, 45, 48, 48, 45, 48, 54, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, - 76, 45, 48, 48, 45, 48, 53, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, - 45, 48, 48, 45, 48, 52, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, - 48, 48, 45, 48, 51, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, - 48, 45, 48, 50, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 48, - 45, 48, 49, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 45, 48, 48, 45, - 48, 48, 128, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 128, 72, 79, 82, 73, - 90, 79, 78, 84, 65, 204, 72, 79, 82, 73, 128, 72, 79, 82, 193, 72, 79, - 79, 85, 128, 72, 79, 79, 82, 85, 128, 72, 79, 79, 80, 128, 72, 79, 79, - 78, 128, 72, 79, 79, 75, 69, 68, 128, 72, 79, 79, 75, 69, 196, 72, 79, - 78, 69, 89, 66, 69, 69, 128, 72, 79, 78, 69, 217, 72, 79, 77, 79, 84, 72, - 69, 84, 73, 67, 128, 72, 79, 77, 79, 84, 72, 69, 84, 73, 195, 72, 79, 76, - 79, 128, 72, 79, 76, 76, 79, 215, 72, 79, 76, 69, 128, 72, 79, 76, 68, - 73, 78, 199, 72, 79, 76, 65, 77, 128, 72, 79, 76, 65, 205, 72, 79, 75, - 65, 128, 72, 79, 67, 75, 69, 217, 72, 79, 67, 72, 79, 128, 72, 79, 45, - 56, 128, 72, 79, 45, 55, 128, 72, 79, 45, 54, 128, 72, 79, 45, 53, 128, - 72, 79, 45, 52, 128, 72, 79, 45, 51, 128, 72, 79, 45, 50, 128, 72, 79, - 45, 49, 128, 72, 78, 85, 84, 128, 72, 78, 85, 79, 88, 128, 72, 78, 85, - 79, 128, 72, 78, 85, 66, 128, 72, 78, 79, 88, 128, 72, 78, 79, 84, 128, - 72, 78, 79, 80, 128, 72, 78, 73, 88, 128, 72, 78, 73, 84, 128, 72, 78, - 73, 80, 128, 72, 78, 73, 69, 88, 128, 72, 78, 73, 69, 84, 128, 72, 78, - 73, 69, 80, 128, 72, 78, 73, 69, 128, 72, 78, 73, 128, 72, 78, 69, 88, - 128, 72, 78, 69, 80, 128, 72, 78, 69, 128, 72, 78, 65, 88, 128, 72, 78, - 65, 85, 128, 72, 78, 65, 84, 128, 72, 78, 65, 80, 128, 72, 78, 65, 128, - 72, 77, 89, 88, 128, 72, 77, 89, 82, 88, 128, 72, 77, 89, 82, 128, 72, - 77, 89, 80, 128, 72, 77, 89, 128, 72, 77, 85, 88, 128, 72, 77, 85, 84, - 128, 72, 77, 85, 82, 88, 128, 72, 77, 85, 82, 128, 72, 77, 85, 80, 128, - 72, 77, 85, 79, 88, 128, 72, 77, 85, 79, 80, 128, 72, 77, 85, 79, 128, - 72, 77, 85, 128, 72, 77, 79, 88, 128, 72, 77, 79, 84, 128, 72, 77, 79, - 80, 128, 72, 77, 79, 128, 72, 77, 73, 88, 128, 72, 77, 73, 84, 128, 72, - 77, 73, 80, 128, 72, 77, 73, 69, 88, 128, 72, 77, 73, 69, 80, 128, 72, - 77, 73, 69, 128, 72, 77, 73, 128, 72, 77, 69, 128, 72, 77, 65, 88, 128, - 72, 77, 65, 84, 128, 72, 77, 65, 80, 128, 72, 77, 65, 128, 72, 76, 89, - 88, 128, 72, 76, 89, 84, 128, 72, 76, 89, 82, 88, 128, 72, 76, 89, 82, - 128, 72, 76, 89, 80, 128, 72, 76, 89, 128, 72, 76, 85, 88, 128, 72, 76, - 85, 84, 128, 72, 76, 85, 82, 88, 128, 72, 76, 85, 82, 128, 72, 76, 85, - 80, 128, 72, 76, 85, 79, 88, 128, 72, 76, 85, 79, 80, 128, 72, 76, 85, - 79, 128, 72, 76, 85, 128, 72, 76, 79, 88, 128, 72, 76, 79, 80, 128, 72, - 76, 79, 128, 72, 76, 73, 88, 128, 72, 76, 73, 84, 128, 72, 76, 73, 80, - 128, 72, 76, 73, 69, 88, 128, 72, 76, 73, 69, 80, 128, 72, 76, 73, 69, - 128, 72, 76, 73, 128, 72, 76, 69, 88, 128, 72, 76, 69, 80, 128, 72, 76, - 69, 128, 72, 76, 65, 88, 128, 72, 76, 65, 85, 128, 72, 76, 65, 84, 128, - 72, 76, 65, 80, 128, 72, 76, 65, 128, 72, 76, 128, 72, 75, 128, 72, 73, - 90, 66, 128, 72, 73, 89, 79, 128, 72, 73, 84, 84, 73, 78, 199, 72, 73, - 83, 84, 79, 82, 73, 195, 72, 73, 82, 73, 81, 128, 72, 73, 80, 80, 79, 80, - 79, 84, 65, 77, 85, 83, 128, 72, 73, 78, 71, 69, 68, 128, 72, 73, 78, 71, - 69, 196, 72, 73, 78, 71, 69, 128, 72, 73, 78, 68, 213, 72, 73, 75, 73, - 78, 199, 72, 73, 71, 72, 45, 83, 80, 69, 69, 196, 72, 73, 71, 72, 45, 82, - 69, 86, 69, 82, 83, 69, 68, 45, 185, 72, 73, 71, 72, 45, 76, 79, 215, 72, - 73, 71, 72, 45, 72, 69, 69, 76, 69, 196, 72, 73, 69, 88, 128, 72, 73, 69, - 85, 72, 45, 83, 73, 79, 83, 128, 72, 73, 69, 85, 72, 45, 82, 73, 69, 85, - 76, 128, 72, 73, 69, 85, 72, 45, 80, 73, 69, 85, 80, 128, 72, 73, 69, 85, - 72, 45, 78, 73, 69, 85, 78, 128, 72, 73, 69, 85, 72, 45, 77, 73, 69, 85, - 77, 128, 72, 73, 69, 85, 200, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 73, - 195, 72, 73, 68, 73, 78, 199, 72, 73, 68, 69, 84, 128, 72, 73, 68, 69, - 128, 72, 73, 66, 73, 83, 67, 85, 83, 128, 72, 73, 45, 82, 69, 83, 128, - 72, 73, 45, 55, 128, 72, 73, 45, 54, 128, 72, 73, 45, 53, 128, 72, 73, - 45, 52, 128, 72, 73, 45, 51, 128, 72, 73, 45, 50, 128, 72, 73, 45, 49, - 128, 72, 72, 89, 85, 128, 72, 72, 89, 79, 128, 72, 72, 89, 73, 128, 72, - 72, 89, 69, 69, 128, 72, 72, 89, 69, 128, 72, 72, 89, 65, 65, 128, 72, - 72, 89, 65, 128, 72, 72, 87, 73, 128, 72, 72, 87, 69, 69, 128, 72, 72, - 87, 69, 128, 72, 72, 87, 65, 128, 72, 72, 85, 128, 72, 72, 73, 128, 72, - 72, 69, 69, 128, 72, 72, 69, 128, 72, 72, 65, 65, 128, 72, 71, 128, 72, - 69, 89, 84, 128, 72, 69, 88, 73, 70, 79, 82, 205, 72, 69, 88, 65, 71, 82, - 65, 205, 72, 69, 88, 65, 71, 79, 78, 128, 72, 69, 82, 85, 84, 85, 128, - 72, 69, 82, 85, 128, 72, 69, 82, 77, 73, 84, 73, 65, 206, 72, 69, 82, 77, - 73, 79, 78, 73, 65, 206, 72, 69, 82, 77, 69, 83, 128, 72, 69, 82, 69, - 128, 72, 69, 82, 66, 128, 72, 69, 82, 65, 69, 85, 205, 72, 69, 78, 71, - 128, 72, 69, 78, 199, 72, 69, 77, 80, 128, 72, 69, 76, 77, 69, 84, 128, - 72, 69, 76, 77, 69, 212, 72, 69, 76, 205, 72, 69, 76, 76, 83, 67, 72, 82, - 69, 73, 66, 69, 210, 72, 69, 76, 73, 88, 128, 72, 69, 76, 73, 67, 79, 80, - 84, 69, 82, 128, 72, 69, 75, 85, 84, 65, 65, 82, 85, 128, 72, 69, 73, 83, - 69, 73, 128, 72, 69, 73, 71, 72, 84, 128, 72, 69, 69, 73, 128, 72, 69, - 68, 71, 69, 72, 79, 71, 128, 72, 69, 65, 86, 89, 128, 72, 69, 65, 86, 69, - 78, 76, 217, 72, 69, 65, 86, 69, 78, 128, 72, 69, 65, 86, 69, 206, 72, - 69, 65, 82, 84, 83, 128, 72, 69, 65, 82, 84, 45, 83, 72, 65, 80, 69, 196, - 72, 69, 65, 82, 84, 128, 72, 69, 65, 82, 212, 72, 69, 65, 82, 73, 78, - 199, 72, 69, 65, 82, 45, 78, 79, 45, 69, 86, 73, 204, 72, 69, 65, 68, 83, - 84, 82, 79, 75, 69, 128, 72, 69, 65, 68, 83, 84, 79, 78, 69, 128, 72, 69, - 65, 68, 83, 84, 79, 78, 197, 72, 69, 65, 68, 83, 67, 65, 82, 70, 128, 72, - 69, 65, 68, 80, 72, 79, 78, 69, 128, 72, 69, 65, 68, 73, 78, 71, 128, 72, - 69, 65, 68, 45, 66, 65, 78, 68, 65, 71, 69, 128, 72, 69, 45, 55, 128, 72, - 69, 45, 54, 128, 72, 69, 45, 53, 128, 72, 69, 45, 52, 128, 72, 69, 45, - 51, 128, 72, 69, 45, 50, 128, 72, 69, 45, 49, 128, 72, 68, 82, 128, 72, - 67, 128, 72, 66, 65, 83, 65, 45, 69, 83, 65, 83, 193, 72, 66, 65, 83, - 193, 72, 65, 89, 65, 78, 78, 65, 128, 72, 65, 87, 74, 128, 72, 65, 86, - 69, 128, 72, 65, 85, 80, 84, 83, 84, 73, 77, 77, 69, 128, 72, 65, 85, 77, - 69, 65, 128, 72, 65, 213, 72, 65, 84, 82, 65, 206, 72, 65, 84, 72, 73, - 128, 72, 65, 84, 69, 128, 72, 65, 84, 67, 72, 73, 78, 199, 72, 65, 84, - 65, 198, 72, 65, 83, 69, 210, 72, 65, 83, 65, 78, 84, 65, 128, 72, 65, - 82, 80, 79, 79, 78, 128, 72, 65, 82, 80, 79, 79, 206, 72, 65, 82, 77, 79, - 78, 73, 67, 128, 72, 65, 82, 75, 76, 69, 65, 206, 72, 65, 82, 68, 78, 69, - 83, 83, 128, 72, 65, 82, 196, 72, 65, 82, 66, 65, 72, 65, 89, 128, 72, - 65, 80, 80, 217, 72, 65, 78, 85, 78, 79, 207, 72, 65, 78, 73, 70, 201, - 72, 65, 78, 71, 90, 72, 79, 213, 72, 65, 78, 68, 83, 72, 65, 75, 69, 128, - 72, 65, 78, 68, 83, 128, 72, 65, 78, 68, 211, 72, 65, 78, 68, 76, 69, 83, - 128, 72, 65, 78, 68, 76, 69, 128, 72, 65, 78, 68, 66, 65, 76, 76, 128, - 72, 65, 78, 68, 66, 65, 71, 128, 72, 65, 78, 68, 45, 79, 86, 65, 76, 128, - 72, 65, 78, 68, 45, 79, 86, 65, 204, 72, 65, 78, 68, 45, 72, 79, 79, 75, - 128, 72, 65, 78, 68, 45, 72, 79, 79, 203, 72, 65, 78, 68, 45, 72, 73, 78, - 71, 69, 128, 72, 65, 78, 68, 45, 72, 73, 78, 71, 197, 72, 65, 78, 68, 45, - 70, 76, 65, 84, 128, 72, 65, 78, 68, 45, 70, 76, 65, 212, 72, 65, 78, 68, - 45, 70, 73, 83, 84, 128, 72, 65, 78, 68, 45, 67, 85, 82, 76, 73, 67, 85, - 69, 128, 72, 65, 78, 68, 45, 67, 85, 82, 76, 73, 67, 85, 197, 72, 65, 78, - 68, 45, 67, 85, 80, 128, 72, 65, 78, 68, 45, 67, 85, 208, 72, 65, 78, 68, - 45, 67, 76, 65, 87, 128, 72, 65, 78, 68, 45, 67, 76, 65, 215, 72, 65, 78, - 68, 45, 67, 73, 82, 67, 76, 69, 128, 72, 65, 78, 68, 45, 67, 73, 82, 67, - 76, 197, 72, 65, 78, 68, 45, 65, 78, 71, 76, 69, 128, 72, 65, 78, 68, 45, - 65, 78, 71, 76, 197, 72, 65, 78, 68, 128, 72, 65, 78, 45, 65, 75, 65, 84, - 128, 72, 65, 77, 90, 65, 128, 72, 65, 77, 90, 193, 72, 65, 77, 83, 84, - 69, 210, 72, 65, 77, 83, 65, 128, 72, 65, 77, 77, 69, 82, 128, 72, 65, - 77, 77, 69, 210, 72, 65, 77, 66, 85, 82, 71, 69, 82, 128, 72, 65, 76, 81, - 65, 128, 72, 65, 76, 79, 128, 72, 65, 76, 70, 45, 67, 73, 82, 67, 76, - 197, 72, 65, 76, 70, 45, 50, 128, 72, 65, 76, 70, 45, 49, 128, 72, 65, - 76, 70, 128, 72, 65, 76, 66, 69, 82, 68, 128, 72, 65, 76, 65, 78, 84, 65, - 128, 72, 65, 73, 84, 85, 128, 72, 65, 73, 211, 72, 65, 73, 82, 67, 85, - 84, 128, 72, 65, 71, 76, 65, 218, 72, 65, 71, 76, 128, 72, 65, 70, 85, - 75, 72, 65, 128, 72, 65, 70, 85, 75, 72, 128, 72, 65, 69, 71, 204, 72, - 65, 68, 69, 83, 128, 72, 65, 65, 82, 85, 128, 72, 65, 65, 77, 128, 72, - 65, 193, 72, 65, 45, 72, 65, 128, 72, 65, 45, 57, 128, 72, 65, 45, 56, - 128, 72, 65, 45, 55, 128, 72, 65, 45, 54, 128, 72, 65, 45, 53, 128, 72, - 65, 45, 52, 128, 72, 65, 45, 51, 128, 72, 65, 45, 50, 128, 72, 65, 45, - 49, 49, 128, 72, 65, 45, 49, 48, 128, 72, 65, 45, 49, 128, 72, 48, 48, - 56, 128, 72, 48, 48, 55, 128, 72, 48, 48, 54, 65, 128, 72, 48, 48, 54, - 128, 72, 48, 48, 53, 128, 72, 48, 48, 52, 128, 72, 48, 48, 51, 128, 72, - 48, 48, 50, 128, 72, 48, 48, 49, 128, 72, 45, 84, 89, 80, 197, 71, 89, - 85, 128, 71, 89, 79, 78, 128, 71, 89, 79, 128, 71, 89, 73, 128, 71, 89, - 70, 213, 71, 89, 69, 69, 128, 71, 89, 65, 83, 128, 71, 89, 65, 65, 128, - 71, 89, 65, 128, 71, 89, 128, 71, 87, 85, 128, 71, 87, 73, 128, 71, 87, - 69, 69, 128, 71, 87, 69, 128, 71, 87, 65, 65, 128, 71, 87, 65, 128, 71, - 87, 128, 71, 86, 65, 78, 71, 128, 71, 86, 128, 71, 85, 82, 85, 83, 72, - 128, 71, 85, 82, 85, 78, 128, 71, 85, 82, 77, 85, 75, 72, 201, 71, 85, - 82, 65, 77, 85, 84, 79, 78, 128, 71, 85, 82, 65, 71, 197, 71, 85, 82, 55, - 128, 71, 85, 78, 85, 128, 71, 85, 78, 213, 71, 85, 78, 74, 65, 76, 193, - 71, 85, 205, 71, 85, 76, 128, 71, 85, 74, 65, 82, 65, 84, 201, 71, 85, - 73, 84, 65, 82, 128, 71, 85, 73, 68, 197, 71, 85, 199, 71, 85, 69, 73, - 128, 71, 85, 69, 72, 128, 71, 85, 69, 200, 71, 85, 68, 128, 71, 85, 196, - 71, 85, 65, 82, 68, 83, 77, 65, 78, 128, 71, 85, 65, 82, 68, 69, 68, 78, - 69, 83, 83, 128, 71, 85, 65, 82, 68, 69, 196, 71, 85, 65, 82, 68, 128, - 71, 85, 65, 82, 65, 78, 201, 71, 85, 193, 71, 85, 178, 71, 84, 69, 210, - 71, 83, 85, 77, 128, 71, 83, 85, 205, 71, 82, 213, 71, 82, 79, 87, 73, - 78, 199, 71, 82, 79, 85, 78, 68, 128, 71, 82, 79, 78, 84, 72, 73, 83, 77, - 65, 84, 65, 128, 71, 82, 79, 77, 79, 80, 79, 86, 79, 68, 78, 65, 89, 65, - 128, 71, 82, 79, 77, 79, 80, 79, 86, 79, 68, 78, 65, 89, 193, 71, 82, 79, - 77, 79, 75, 82, 89, 90, 72, 69, 86, 65, 89, 65, 128, 71, 82, 79, 77, 79, - 75, 82, 89, 90, 72, 69, 86, 65, 89, 193, 71, 82, 79, 77, 78, 65, 89, 65, - 128, 71, 82, 79, 77, 78, 65, 89, 193, 71, 82, 73, 78, 78, 73, 78, 199, - 71, 82, 73, 77, 65, 67, 73, 78, 199, 71, 82, 69, 217, 71, 82, 69, 71, 79, - 82, 73, 65, 206, 71, 82, 69, 69, 78, 128, 71, 82, 69, 69, 206, 71, 82, - 69, 65, 84, 78, 69, 83, 83, 128, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, - 65, 78, 128, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 206, 71, 82, 69, - 65, 84, 69, 210, 71, 82, 69, 65, 212, 71, 82, 65, 86, 69, 89, 65, 82, - 196, 71, 82, 65, 86, 69, 45, 77, 65, 67, 82, 79, 78, 128, 71, 82, 65, 86, - 69, 45, 65, 67, 85, 84, 69, 45, 71, 82, 65, 86, 69, 128, 71, 82, 65, 86, - 197, 71, 82, 65, 84, 69, 82, 128, 71, 82, 65, 83, 83, 128, 71, 82, 65, - 83, 211, 71, 82, 65, 83, 208, 71, 82, 65, 80, 72, 69, 77, 197, 71, 82, - 65, 80, 69, 83, 128, 71, 82, 65, 78, 84, 72, 193, 71, 82, 65, 77, 77, - 193, 71, 82, 65, 73, 78, 128, 71, 82, 65, 70, 128, 71, 82, 65, 68, 85, - 65, 84, 73, 79, 206, 71, 82, 65, 68, 85, 65, 76, 128, 71, 82, 65, 67, 69, - 128, 71, 82, 65, 67, 197, 71, 80, 65, 128, 71, 79, 82, 84, 72, 77, 73, - 75, 79, 206, 71, 79, 82, 84, 128, 71, 79, 82, 73, 76, 76, 65, 128, 71, - 79, 82, 71, 79, 84, 69, 82, 73, 128, 71, 79, 82, 71, 79, 83, 89, 78, 84, - 72, 69, 84, 79, 78, 128, 71, 79, 82, 71, 79, 206, 71, 79, 82, 71, 73, - 128, 71, 79, 82, 65, 90, 68, 207, 71, 79, 82, 65, 128, 71, 79, 79, 83, - 69, 128, 71, 79, 79, 196, 71, 79, 78, 71, 71, 79, 78, 71, 128, 71, 79, - 76, 85, 66, 67, 72, 73, 203, 71, 79, 76, 70, 69, 82, 128, 71, 79, 76, 68, - 128, 71, 79, 75, 128, 71, 79, 73, 78, 199, 71, 79, 71, 71, 76, 69, 83, - 128, 71, 79, 66, 76, 73, 78, 128, 71, 79, 65, 76, 128, 71, 79, 65, 204, - 71, 79, 65, 128, 71, 78, 89, 73, 83, 128, 71, 78, 65, 86, 73, 89, 65, 78, - 73, 128, 71, 76, 79, 87, 73, 78, 199, 71, 76, 79, 86, 69, 83, 128, 71, - 76, 79, 86, 69, 128, 71, 76, 79, 84, 84, 65, 204, 71, 76, 79, 66, 197, - 71, 76, 73, 83, 83, 65, 78, 68, 207, 71, 76, 69, 73, 67, 200, 71, 76, 65, - 71, 79, 76, 73, 128, 71, 76, 65, 128, 71, 74, 69, 128, 71, 73, 88, 128, - 71, 73, 84, 128, 71, 73, 83, 72, 128, 71, 73, 83, 200, 71, 73, 83, 65, - 76, 128, 71, 73, 82, 85, 68, 65, 65, 128, 71, 73, 82, 76, 211, 71, 73, - 82, 76, 128, 71, 73, 82, 65, 70, 70, 197, 71, 73, 82, 51, 128, 71, 73, - 82, 179, 71, 73, 82, 50, 128, 71, 73, 82, 178, 71, 73, 80, 128, 71, 73, - 78, 73, 73, 128, 71, 73, 78, 71, 69, 210, 71, 73, 77, 69, 76, 45, 72, 69, - 84, 72, 128, 71, 73, 77, 69, 76, 128, 71, 73, 77, 69, 204, 71, 73, 77, - 128, 71, 73, 71, 65, 128, 71, 73, 71, 128, 71, 73, 70, 212, 71, 73, 69, - 84, 128, 71, 73, 68, 73, 77, 128, 71, 73, 66, 66, 79, 85, 211, 71, 73, - 66, 65, 128, 71, 73, 52, 128, 71, 73, 180, 71, 72, 90, 128, 71, 72, 87, - 65, 128, 71, 72, 85, 78, 78, 65, 128, 71, 72, 85, 78, 78, 193, 71, 72, - 85, 128, 71, 72, 79, 85, 128, 71, 72, 79, 83, 84, 128, 71, 72, 79, 128, - 71, 72, 73, 77, 69, 76, 128, 71, 72, 73, 128, 71, 72, 72, 65, 128, 71, - 72, 69, 89, 83, 128, 71, 72, 69, 85, 88, 128, 71, 72, 69, 85, 78, 128, - 71, 72, 69, 85, 71, 72, 69, 85, 65, 69, 77, 128, 71, 72, 69, 85, 71, 72, - 69, 78, 128, 71, 72, 69, 85, 65, 69, 82, 65, 69, 128, 71, 72, 69, 85, 65, - 69, 71, 72, 69, 85, 65, 69, 128, 71, 72, 69, 84, 128, 71, 72, 69, 69, - 128, 71, 72, 69, 128, 71, 72, 197, 71, 72, 65, 89, 78, 128, 71, 72, 65, - 82, 65, 69, 128, 71, 72, 65, 80, 128, 71, 72, 65, 78, 128, 71, 72, 65, - 77, 77, 65, 128, 71, 72, 65, 77, 65, 76, 128, 71, 72, 65, 73, 78, 85, - 128, 71, 72, 65, 73, 78, 128, 71, 72, 65, 73, 206, 71, 72, 65, 68, 128, - 71, 72, 65, 65, 77, 65, 69, 128, 71, 72, 65, 65, 128, 71, 71, 87, 73, - 128, 71, 71, 87, 69, 69, 128, 71, 71, 87, 69, 128, 71, 71, 87, 65, 65, - 128, 71, 71, 87, 65, 128, 71, 71, 85, 88, 128, 71, 71, 85, 84, 128, 71, - 71, 85, 82, 88, 128, 71, 71, 85, 82, 128, 71, 71, 85, 79, 88, 128, 71, - 71, 85, 79, 84, 128, 71, 71, 85, 79, 80, 128, 71, 71, 85, 79, 128, 71, - 71, 79, 88, 128, 71, 71, 79, 84, 128, 71, 71, 79, 80, 128, 71, 71, 73, - 88, 128, 71, 71, 73, 84, 128, 71, 71, 73, 69, 88, 128, 71, 71, 73, 69, - 80, 128, 71, 71, 73, 69, 128, 71, 71, 69, 88, 128, 71, 71, 69, 84, 128, - 71, 71, 69, 80, 128, 71, 71, 65, 88, 128, 71, 71, 65, 84, 128, 71, 69, - 84, 193, 71, 69, 83, 84, 85, 82, 69, 128, 71, 69, 83, 72, 85, 128, 71, - 69, 83, 72, 84, 73, 78, 128, 71, 69, 83, 72, 84, 73, 206, 71, 69, 83, 72, - 50, 128, 71, 69, 82, 83, 72, 65, 89, 73, 77, 128, 71, 69, 82, 77, 65, - 206, 71, 69, 82, 69, 83, 72, 128, 71, 69, 82, 69, 83, 200, 71, 69, 79, - 77, 69, 84, 82, 73, 67, 65, 76, 76, 217, 71, 69, 79, 77, 69, 84, 82, 73, - 195, 71, 69, 78, 84, 76, 197, 71, 69, 78, 73, 84, 73, 86, 69, 128, 71, - 69, 78, 73, 75, 201, 71, 69, 78, 73, 69, 128, 71, 69, 78, 69, 82, 73, - 195, 71, 69, 78, 69, 82, 65, 76, 128, 71, 69, 77, 73, 78, 73, 128, 71, - 69, 77, 73, 78, 65, 84, 73, 79, 206, 71, 69, 77, 73, 78, 65, 84, 197, 71, - 69, 205, 71, 69, 69, 77, 128, 71, 69, 68, 79, 76, 65, 128, 71, 69, 68, - 69, 128, 71, 69, 66, 207, 71, 69, 66, 193, 71, 69, 65, 82, 128, 71, 69, - 65, 210, 71, 69, 50, 50, 128, 71, 68, 65, 78, 128, 71, 67, 73, 71, 128, - 71, 67, 65, 206, 71, 66, 79, 78, 128, 71, 66, 73, 69, 197, 71, 66, 69, - 85, 88, 128, 71, 66, 69, 84, 128, 71, 66, 65, 89, 73, 128, 71, 66, 65, - 75, 85, 82, 85, 78, 69, 78, 128, 71, 66, 128, 71, 65, 89, 65, 78, 85, 75, - 73, 84, 84, 65, 128, 71, 65, 89, 65, 78, 78, 65, 128, 71, 65, 89, 128, - 71, 65, 85, 78, 84, 76, 69, 84, 128, 71, 65, 84, 72, 69, 82, 73, 78, 71, - 128, 71, 65, 84, 72, 69, 82, 73, 78, 199, 71, 65, 84, 69, 128, 71, 65, - 83, 72, 65, 78, 128, 71, 65, 82, 83, 72, 85, 78, 73, 128, 71, 65, 82, 79, - 78, 128, 71, 65, 82, 77, 69, 78, 84, 128, 71, 65, 82, 76, 73, 67, 128, - 71, 65, 82, 68, 69, 78, 128, 71, 65, 82, 51, 128, 71, 65, 80, 80, 69, - 196, 71, 65, 208, 71, 65, 78, 77, 65, 128, 71, 65, 78, 71, 73, 65, 128, - 71, 65, 78, 68, 193, 71, 65, 78, 50, 128, 71, 65, 78, 178, 71, 65, 77, - 77, 65, 128, 71, 65, 77, 76, 65, 128, 71, 65, 77, 76, 128, 71, 65, 77, - 69, 128, 71, 65, 77, 197, 71, 65, 77, 65, 78, 128, 71, 65, 77, 65, 76, - 128, 71, 65, 77, 65, 204, 71, 65, 76, 201, 71, 65, 71, 128, 71, 65, 70, - 128, 71, 65, 198, 71, 65, 69, 84, 84, 65, 45, 80, 73, 76, 76, 65, 128, - 71, 65, 68, 79, 76, 128, 71, 65, 68, 128, 71, 65, 196, 71, 65, 66, 65, - 128, 71, 65, 66, 193, 71, 65, 65, 70, 85, 128, 71, 65, 178, 71, 48, 53, - 52, 128, 71, 48, 53, 51, 128, 71, 48, 53, 50, 128, 71, 48, 53, 49, 128, - 71, 48, 53, 48, 128, 71, 48, 52, 57, 128, 71, 48, 52, 56, 128, 71, 48, - 52, 55, 128, 71, 48, 52, 54, 128, 71, 48, 52, 53, 65, 128, 71, 48, 52, - 53, 128, 71, 48, 52, 52, 128, 71, 48, 52, 51, 65, 128, 71, 48, 52, 51, - 128, 71, 48, 52, 50, 128, 71, 48, 52, 49, 128, 71, 48, 52, 48, 128, 71, - 48, 51, 57, 128, 71, 48, 51, 56, 128, 71, 48, 51, 55, 65, 128, 71, 48, - 51, 55, 128, 71, 48, 51, 54, 65, 128, 71, 48, 51, 54, 128, 71, 48, 51, - 53, 128, 71, 48, 51, 52, 128, 71, 48, 51, 51, 128, 71, 48, 51, 50, 128, - 71, 48, 51, 49, 128, 71, 48, 51, 48, 128, 71, 48, 50, 57, 128, 71, 48, - 50, 56, 128, 71, 48, 50, 55, 128, 71, 48, 50, 54, 65, 128, 71, 48, 50, - 54, 128, 71, 48, 50, 53, 128, 71, 48, 50, 52, 128, 71, 48, 50, 51, 128, - 71, 48, 50, 50, 128, 71, 48, 50, 49, 128, 71, 48, 50, 48, 65, 128, 71, - 48, 50, 48, 128, 71, 48, 49, 57, 128, 71, 48, 49, 56, 128, 71, 48, 49, - 55, 128, 71, 48, 49, 54, 128, 71, 48, 49, 53, 128, 71, 48, 49, 52, 128, - 71, 48, 49, 51, 128, 71, 48, 49, 50, 128, 71, 48, 49, 49, 65, 128, 71, - 48, 49, 49, 128, 71, 48, 49, 48, 128, 71, 48, 48, 57, 128, 71, 48, 48, - 56, 128, 71, 48, 48, 55, 66, 128, 71, 48, 48, 55, 65, 128, 71, 48, 48, - 55, 128, 71, 48, 48, 54, 65, 128, 71, 48, 48, 54, 128, 71, 48, 48, 53, - 128, 71, 48, 48, 52, 128, 71, 48, 48, 51, 128, 71, 48, 48, 50, 128, 71, - 48, 48, 49, 128, 70, 89, 88, 128, 70, 89, 84, 128, 70, 89, 80, 128, 70, - 89, 65, 128, 70, 87, 73, 128, 70, 87, 69, 69, 128, 70, 87, 69, 128, 70, - 87, 65, 65, 128, 70, 87, 65, 128, 70, 86, 83, 52, 128, 70, 86, 83, 51, - 128, 70, 86, 83, 50, 128, 70, 86, 83, 49, 128, 70, 85, 88, 128, 70, 85, - 84, 128, 70, 85, 83, 69, 128, 70, 85, 83, 193, 70, 85, 82, 88, 128, 70, - 85, 80, 128, 70, 85, 78, 69, 82, 65, 204, 70, 85, 78, 67, 84, 73, 79, 78, - 65, 204, 70, 85, 78, 67, 84, 73, 79, 78, 128, 70, 85, 76, 76, 87, 73, 68, - 84, 200, 70, 85, 76, 76, 78, 69, 83, 83, 128, 70, 85, 76, 204, 70, 85, - 74, 73, 128, 70, 85, 69, 84, 128, 70, 85, 69, 204, 70, 85, 69, 128, 70, - 85, 65, 128, 70, 84, 72, 79, 82, 193, 70, 83, 73, 128, 70, 82, 79, 87, - 78, 73, 78, 71, 128, 70, 82, 79, 87, 78, 73, 78, 199, 70, 82, 79, 87, 78, - 128, 70, 82, 79, 87, 206, 70, 82, 79, 78, 84, 45, 84, 73, 76, 84, 69, - 196, 70, 82, 79, 78, 84, 45, 70, 65, 67, 73, 78, 199, 70, 82, 79, 78, - 212, 70, 82, 79, 205, 70, 82, 79, 71, 128, 70, 82, 79, 199, 70, 82, 73, - 84, 85, 128, 70, 82, 73, 69, 83, 128, 70, 82, 73, 69, 196, 70, 82, 73, - 67, 65, 84, 73, 86, 69, 128, 70, 82, 69, 84, 66, 79, 65, 82, 68, 128, 70, - 82, 69, 78, 67, 200, 70, 82, 69, 69, 90, 73, 78, 199, 70, 82, 69, 69, - 128, 70, 82, 69, 197, 70, 82, 65, 78, 75, 211, 70, 82, 65, 78, 195, 70, - 82, 65, 77, 69, 83, 128, 70, 82, 65, 77, 69, 128, 70, 82, 65, 77, 197, - 70, 82, 65, 75, 84, 85, 210, 70, 82, 65, 71, 82, 65, 78, 84, 128, 70, 82, - 65, 71, 77, 69, 78, 84, 128, 70, 82, 65, 67, 84, 73, 79, 206, 70, 79, 88, - 128, 70, 79, 216, 70, 79, 85, 82, 84, 69, 69, 78, 128, 70, 79, 85, 82, - 84, 69, 69, 206, 70, 79, 85, 82, 45, 84, 72, 73, 82, 84, 89, 128, 70, 79, - 85, 82, 45, 83, 84, 82, 73, 78, 199, 70, 79, 85, 82, 45, 80, 69, 82, 45, - 69, 205, 70, 79, 85, 82, 45, 76, 73, 78, 197, 70, 79, 85, 210, 70, 79, - 85, 78, 84, 65, 73, 78, 128, 70, 79, 85, 78, 84, 65, 73, 206, 70, 79, 83, - 84, 69, 82, 73, 78, 71, 128, 70, 79, 82, 87, 65, 82, 68, 128, 70, 79, 82, - 87, 65, 82, 196, 70, 79, 82, 84, 89, 45, 70, 73, 86, 197, 70, 79, 82, 84, - 89, 128, 70, 79, 82, 84, 217, 70, 79, 82, 84, 85, 78, 69, 128, 70, 79, - 82, 84, 85, 78, 197, 70, 79, 82, 84, 73, 69, 84, 72, 128, 70, 79, 82, 84, - 69, 128, 70, 79, 82, 77, 211, 70, 79, 82, 77, 69, 69, 128, 70, 79, 82, - 77, 69, 197, 70, 79, 82, 77, 65, 84, 84, 73, 78, 71, 128, 70, 79, 82, 77, - 65, 212, 70, 79, 82, 75, 69, 196, 70, 79, 82, 69, 72, 69, 65, 196, 70, - 79, 82, 67, 69, 83, 128, 70, 79, 82, 67, 69, 128, 70, 79, 80, 128, 70, - 79, 79, 84, 83, 84, 79, 79, 76, 128, 70, 79, 79, 84, 80, 82, 73, 78, 84, - 83, 128, 70, 79, 79, 84, 78, 79, 84, 197, 70, 79, 79, 84, 66, 65, 76, 76, - 128, 70, 79, 79, 84, 128, 70, 79, 79, 76, 128, 70, 79, 79, 68, 128, 70, - 79, 79, 128, 70, 79, 78, 212, 70, 79, 78, 71, 77, 65, 78, 128, 70, 79, - 78, 68, 85, 69, 128, 70, 79, 77, 128, 70, 79, 76, 76, 89, 128, 70, 79, - 76, 76, 79, 87, 73, 78, 71, 128, 70, 79, 76, 68, 73, 78, 199, 70, 79, 76, - 68, 69, 82, 128, 70, 79, 76, 68, 69, 196, 70, 79, 71, 71, 89, 128, 70, - 79, 71, 128, 70, 207, 70, 77, 128, 70, 76, 89, 73, 78, 199, 70, 76, 89, - 128, 70, 76, 85, 84, 84, 69, 82, 73, 78, 71, 128, 70, 76, 85, 84, 84, 69, - 82, 73, 78, 199, 70, 76, 85, 84, 69, 128, 70, 76, 85, 83, 72, 69, 196, - 70, 76, 79, 87, 73, 78, 199, 70, 76, 79, 87, 69, 82, 83, 128, 70, 76, 79, - 87, 69, 210, 70, 76, 79, 85, 82, 73, 83, 72, 128, 70, 76, 79, 82, 69, 84, - 84, 69, 128, 70, 76, 79, 82, 65, 204, 70, 76, 79, 80, 80, 217, 70, 76, - 79, 79, 82, 128, 70, 76, 79, 79, 210, 70, 76, 73, 80, 128, 70, 76, 73, - 71, 72, 84, 128, 70, 76, 73, 67, 203, 70, 76, 69, 88, 85, 83, 128, 70, - 76, 69, 88, 69, 196, 70, 76, 69, 88, 128, 70, 76, 69, 85, 82, 79, 78, - 128, 70, 76, 69, 85, 82, 45, 68, 69, 45, 76, 73, 83, 128, 70, 76, 65, 84, - 84, 69, 78, 69, 196, 70, 76, 65, 84, 78, 69, 83, 83, 128, 70, 76, 65, 84, - 66, 82, 69, 65, 68, 128, 70, 76, 65, 83, 72, 128, 70, 76, 65, 77, 73, 78, - 71, 79, 128, 70, 76, 65, 77, 69, 128, 70, 76, 65, 71, 83, 128, 70, 76, - 65, 71, 45, 53, 128, 70, 76, 65, 71, 45, 52, 128, 70, 76, 65, 71, 45, 51, - 128, 70, 76, 65, 71, 45, 50, 128, 70, 76, 65, 71, 45, 49, 128, 70, 76, - 65, 71, 128, 70, 76, 65, 199, 70, 76, 65, 128, 70, 76, 128, 70, 73, 88, - 69, 68, 45, 70, 79, 82, 205, 70, 73, 88, 128, 70, 73, 86, 69, 45, 84, 72, - 73, 82, 84, 89, 128, 70, 73, 86, 69, 45, 76, 73, 78, 197, 70, 73, 86, 69, - 45, 76, 73, 75, 197, 70, 73, 84, 90, 80, 65, 84, 82, 73, 67, 203, 70, 73, - 84, 65, 128, 70, 73, 84, 128, 70, 73, 83, 84, 69, 196, 70, 73, 83, 72, - 73, 78, 199, 70, 73, 83, 72, 72, 79, 79, 75, 128, 70, 73, 83, 72, 72, 79, - 79, 203, 70, 73, 83, 72, 69, 89, 69, 128, 70, 73, 83, 200, 70, 73, 82, - 83, 212, 70, 73, 82, 73, 128, 70, 73, 82, 69, 87, 79, 82, 75, 83, 128, - 70, 73, 82, 69, 87, 79, 82, 203, 70, 73, 82, 69, 67, 82, 65, 67, 75, 69, - 82, 128, 70, 73, 82, 69, 128, 70, 73, 82, 197, 70, 73, 80, 128, 70, 73, - 78, 73, 84, 197, 70, 73, 78, 71, 69, 82, 83, 128, 70, 73, 78, 71, 69, 82, - 211, 70, 73, 78, 71, 69, 82, 78, 65, 73, 76, 83, 128, 70, 73, 78, 71, 69, - 82, 69, 196, 70, 73, 78, 71, 69, 82, 45, 80, 79, 83, 212, 70, 73, 78, 71, - 69, 82, 128, 70, 73, 78, 71, 69, 210, 70, 73, 78, 65, 78, 67, 73, 65, 76, - 128, 70, 73, 78, 65, 76, 128, 70, 73, 76, 205, 70, 73, 76, 76, 69, 82, - 45, 50, 128, 70, 73, 76, 76, 69, 82, 45, 49, 128, 70, 73, 76, 76, 69, 82, - 128, 70, 73, 76, 76, 69, 196, 70, 73, 76, 76, 128, 70, 73, 76, 204, 70, - 73, 76, 197, 70, 73, 73, 128, 70, 73, 71, 85, 82, 69, 45, 51, 128, 70, - 73, 71, 85, 82, 69, 45, 50, 128, 70, 73, 71, 85, 82, 69, 45, 49, 128, 70, - 73, 71, 85, 82, 69, 128, 70, 73, 71, 85, 82, 197, 70, 73, 71, 72, 84, - 128, 70, 73, 70, 84, 89, 128, 70, 73, 70, 84, 217, 70, 73, 70, 84, 72, - 83, 128, 70, 73, 70, 84, 72, 128, 70, 73, 70, 84, 69, 69, 78, 128, 70, - 73, 70, 84, 69, 69, 206, 70, 73, 69, 76, 68, 128, 70, 73, 69, 76, 196, - 70, 72, 84, 79, 82, 193, 70, 70, 76, 128, 70, 70, 73, 128, 70, 69, 85, - 88, 128, 70, 69, 85, 70, 69, 85, 65, 69, 84, 128, 70, 69, 84, 72, 128, - 70, 69, 83, 84, 73, 86, 65, 76, 128, 70, 69, 82, 82, 89, 128, 70, 69, 82, - 82, 73, 211, 70, 69, 82, 77, 65, 84, 65, 128, 70, 69, 82, 77, 65, 84, - 193, 70, 69, 79, 200, 70, 69, 78, 199, 70, 69, 78, 67, 69, 82, 128, 70, - 69, 78, 67, 69, 128, 70, 69, 77, 73, 78, 73, 78, 197, 70, 69, 77, 65, 76, - 69, 128, 70, 69, 77, 65, 76, 197, 70, 69, 76, 76, 79, 87, 83, 72, 73, 80, - 128, 70, 69, 73, 128, 70, 69, 72, 213, 70, 69, 72, 128, 70, 69, 200, 70, - 69, 69, 78, 71, 128, 70, 69, 69, 77, 128, 70, 69, 69, 68, 128, 70, 69, - 69, 196, 70, 69, 69, 128, 70, 69, 66, 82, 85, 65, 82, 89, 128, 70, 69, - 65, 84, 72, 69, 82, 128, 70, 69, 65, 84, 72, 69, 210, 70, 69, 65, 82, 78, - 128, 70, 69, 65, 82, 70, 85, 204, 70, 69, 65, 82, 128, 70, 65, 89, 65, - 78, 78, 65, 128, 70, 65, 89, 128, 70, 65, 88, 128, 70, 65, 216, 70, 65, - 84, 73, 71, 85, 69, 128, 70, 65, 84, 72, 69, 82, 128, 70, 65, 84, 72, 69, - 210, 70, 65, 84, 72, 65, 84, 65, 78, 128, 70, 65, 84, 72, 65, 84, 65, - 206, 70, 65, 84, 72, 65, 128, 70, 65, 84, 72, 193, 70, 65, 84, 128, 70, - 65, 83, 84, 128, 70, 65, 82, 83, 201, 70, 65, 82, 128, 70, 65, 81, 128, - 70, 65, 80, 128, 70, 65, 78, 71, 128, 70, 65, 78, 69, 82, 79, 83, 73, - 211, 70, 65, 78, 128, 70, 65, 77, 73, 76, 89, 128, 70, 65, 77, 128, 70, - 65, 76, 76, 69, 206, 70, 65, 76, 65, 70, 69, 76, 128, 70, 65, 74, 128, - 70, 65, 73, 82, 89, 128, 70, 65, 73, 76, 85, 82, 69, 128, 70, 65, 73, 72, - 85, 128, 70, 65, 73, 66, 128, 70, 65, 72, 82, 69, 78, 72, 69, 73, 84, - 128, 70, 65, 67, 84, 79, 82, 89, 128, 70, 65, 67, 84, 79, 210, 70, 65, - 67, 83, 73, 77, 73, 76, 197, 70, 65, 67, 73, 78, 71, 83, 128, 70, 65, 67, - 69, 45, 54, 128, 70, 65, 67, 69, 45, 53, 128, 70, 65, 67, 69, 45, 52, - 128, 70, 65, 67, 69, 45, 51, 128, 70, 65, 67, 69, 45, 50, 128, 70, 65, - 67, 69, 45, 49, 128, 70, 65, 65, 77, 65, 69, 128, 70, 65, 65, 73, 128, - 70, 65, 65, 70, 85, 128, 70, 48, 53, 51, 128, 70, 48, 53, 50, 128, 70, - 48, 53, 49, 67, 128, 70, 48, 53, 49, 66, 128, 70, 48, 53, 49, 65, 128, - 70, 48, 53, 49, 128, 70, 48, 53, 48, 128, 70, 48, 52, 57, 128, 70, 48, - 52, 56, 128, 70, 48, 52, 55, 65, 128, 70, 48, 52, 55, 128, 70, 48, 52, - 54, 65, 128, 70, 48, 52, 54, 128, 70, 48, 52, 53, 65, 128, 70, 48, 52, - 53, 128, 70, 48, 52, 52, 128, 70, 48, 52, 51, 128, 70, 48, 52, 50, 128, - 70, 48, 52, 49, 128, 70, 48, 52, 48, 128, 70, 48, 51, 57, 128, 70, 48, - 51, 56, 65, 128, 70, 48, 51, 56, 128, 70, 48, 51, 55, 65, 128, 70, 48, - 51, 55, 128, 70, 48, 51, 54, 128, 70, 48, 51, 53, 128, 70, 48, 51, 52, - 128, 70, 48, 51, 51, 128, 70, 48, 51, 50, 128, 70, 48, 51, 49, 65, 128, - 70, 48, 51, 49, 128, 70, 48, 51, 48, 128, 70, 48, 50, 57, 128, 70, 48, - 50, 56, 128, 70, 48, 50, 55, 128, 70, 48, 50, 54, 128, 70, 48, 50, 53, - 128, 70, 48, 50, 52, 128, 70, 48, 50, 51, 128, 70, 48, 50, 50, 128, 70, - 48, 50, 49, 65, 128, 70, 48, 50, 49, 128, 70, 48, 50, 48, 128, 70, 48, - 49, 57, 128, 70, 48, 49, 56, 128, 70, 48, 49, 55, 128, 70, 48, 49, 54, - 128, 70, 48, 49, 53, 128, 70, 48, 49, 52, 128, 70, 48, 49, 51, 65, 128, - 70, 48, 49, 51, 128, 70, 48, 49, 50, 128, 70, 48, 49, 49, 128, 70, 48, - 49, 48, 128, 70, 48, 48, 57, 128, 70, 48, 48, 56, 128, 70, 48, 48, 55, - 128, 70, 48, 48, 54, 128, 70, 48, 48, 53, 128, 70, 48, 48, 52, 128, 70, - 48, 48, 51, 128, 70, 48, 48, 50, 128, 70, 48, 48, 49, 65, 128, 70, 48, - 48, 49, 128, 69, 90, 83, 128, 69, 90, 69, 78, 128, 69, 90, 69, 206, 69, - 89, 89, 89, 128, 69, 89, 69, 83, 128, 69, 89, 69, 211, 69, 89, 69, 76, - 65, 83, 72, 69, 211, 69, 89, 69, 71, 76, 65, 83, 83, 69, 83, 128, 69, 89, - 69, 71, 65, 90, 69, 45, 87, 65, 76, 76, 80, 76, 65, 78, 197, 69, 89, 69, - 71, 65, 90, 69, 45, 70, 76, 79, 79, 82, 80, 76, 65, 78, 197, 69, 89, 69, - 66, 82, 79, 87, 211, 69, 89, 69, 66, 82, 79, 215, 69, 89, 197, 69, 89, - 66, 69, 89, 70, 73, 76, 73, 128, 69, 89, 65, 78, 78, 65, 128, 69, 88, 84, - 82, 69, 77, 69, 76, 217, 69, 88, 84, 82, 65, 84, 69, 82, 82, 69, 83, 84, - 82, 73, 65, 204, 69, 88, 84, 82, 65, 45, 76, 79, 215, 69, 88, 84, 82, 65, - 45, 72, 73, 71, 200, 69, 88, 84, 82, 193, 69, 88, 84, 73, 78, 71, 85, 73, - 83, 72, 69, 82, 128, 69, 88, 84, 69, 78, 83, 73, 79, 78, 128, 69, 88, 84, - 69, 78, 68, 69, 68, 128, 69, 88, 84, 69, 78, 68, 69, 196, 69, 88, 80, 82, - 69, 83, 83, 73, 79, 78, 76, 69, 83, 211, 69, 88, 80, 79, 78, 69, 78, 212, - 69, 88, 80, 76, 79, 68, 73, 78, 199, 69, 88, 79, 128, 69, 88, 207, 69, - 88, 73, 83, 84, 83, 128, 69, 88, 73, 83, 84, 128, 69, 88, 72, 65, 85, 83, - 84, 73, 79, 78, 128, 69, 88, 72, 65, 76, 69, 128, 69, 88, 67, 76, 65, 77, - 65, 84, 73, 79, 78, 128, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 206, 69, - 88, 67, 73, 84, 69, 77, 69, 78, 84, 128, 69, 88, 67, 72, 65, 78, 71, 69, - 128, 69, 88, 67, 69, 83, 83, 128, 69, 88, 67, 69, 76, 76, 69, 78, 84, - 128, 69, 87, 69, 128, 69, 86, 69, 82, 217, 69, 86, 69, 82, 71, 82, 69, - 69, 206, 69, 86, 69, 78, 73, 78, 71, 128, 69, 85, 82, 79, 80, 69, 65, - 206, 69, 85, 82, 79, 80, 69, 45, 65, 70, 82, 73, 67, 65, 128, 69, 85, 82, - 79, 45, 67, 85, 82, 82, 69, 78, 67, 217, 69, 85, 82, 207, 69, 85, 76, 69, - 210, 69, 85, 45, 85, 128, 69, 85, 45, 79, 128, 69, 85, 45, 69, 85, 128, - 69, 85, 45, 69, 79, 128, 69, 85, 45, 69, 128, 69, 85, 45, 65, 128, 69, - 84, 88, 128, 69, 84, 84, 128, 69, 84, 78, 65, 72, 84, 65, 128, 69, 84, - 72, 69, 204, 69, 84, 69, 82, 79, 206, 69, 84, 69, 82, 78, 73, 84, 89, - 128, 69, 84, 69, 82, 78, 73, 84, 217, 69, 84, 66, 128, 69, 83, 90, 128, - 69, 83, 85, 75, 85, 85, 68, 79, 128, 69, 83, 84, 73, 77, 65, 84, 69, 83, - 128, 69, 83, 84, 73, 77, 65, 84, 69, 196, 69, 83, 72, 69, 51, 128, 69, - 83, 72, 50, 49, 128, 69, 83, 72, 49, 54, 128, 69, 83, 67, 65, 80, 69, - 128, 69, 83, 67, 128, 69, 83, 65, 128, 69, 83, 45, 84, 69, 128, 69, 83, - 45, 51, 128, 69, 83, 45, 50, 128, 69, 83, 45, 49, 128, 69, 82, 82, 79, - 82, 45, 66, 65, 82, 82, 69, 196, 69, 82, 82, 128, 69, 82, 73, 211, 69, - 82, 73, 78, 50, 128, 69, 82, 73, 78, 178, 69, 82, 71, 128, 69, 82, 65, - 83, 197, 69, 81, 85, 73, 86, 65, 76, 69, 78, 212, 69, 81, 85, 73, 76, 65, - 84, 69, 82, 65, 204, 69, 81, 85, 73, 72, 79, 80, 80, 69, 82, 128, 69, 81, - 85, 73, 72, 79, 80, 80, 69, 210, 69, 81, 85, 73, 68, 128, 69, 81, 85, 73, - 65, 78, 71, 85, 76, 65, 210, 69, 81, 85, 65, 76, 83, 128, 69, 81, 85, 65, - 76, 211, 69, 81, 85, 65, 76, 128, 69, 81, 85, 65, 204, 69, 80, 83, 73, - 76, 79, 78, 128, 69, 80, 83, 73, 76, 79, 206, 69, 80, 79, 67, 72, 128, - 69, 80, 73, 71, 82, 65, 80, 72, 73, 195, 69, 80, 73, 68, 65, 85, 82, 69, - 65, 206, 69, 80, 69, 78, 84, 72, 69, 84, 73, 195, 69, 80, 69, 71, 69, 82, - 77, 65, 128, 69, 80, 65, 67, 212, 69, 79, 84, 128, 69, 79, 77, 128, 69, - 79, 76, 72, 88, 128, 69, 79, 76, 128, 69, 79, 72, 128, 69, 78, 89, 128, - 69, 78, 86, 69, 76, 79, 80, 69, 128, 69, 78, 86, 69, 76, 79, 80, 197, 69, - 78, 85, 77, 69, 82, 65, 84, 73, 79, 206, 69, 78, 84, 82, 89, 45, 50, 128, - 69, 78, 84, 82, 89, 45, 49, 128, 69, 78, 84, 82, 89, 128, 69, 78, 84, 82, - 217, 69, 78, 84, 72, 85, 83, 73, 65, 83, 77, 128, 69, 78, 84, 69, 82, 80, - 82, 73, 83, 69, 128, 69, 78, 84, 69, 82, 73, 78, 199, 69, 78, 84, 69, 82, - 128, 69, 78, 84, 69, 210, 69, 78, 84, 45, 83, 72, 65, 80, 69, 196, 69, - 78, 81, 85, 73, 82, 89, 128, 69, 78, 81, 128, 69, 78, 79, 211, 69, 78, - 78, 73, 128, 69, 78, 78, 128, 69, 78, 76, 65, 82, 71, 69, 77, 69, 78, 84, - 128, 69, 78, 71, 73, 78, 69, 128, 69, 78, 68, 79, 70, 79, 78, 79, 78, - 128, 69, 78, 68, 73, 78, 199, 69, 78, 68, 69, 80, 128, 69, 78, 68, 69, - 65, 86, 79, 85, 82, 128, 69, 78, 67, 79, 85, 78, 84, 69, 82, 83, 128, 69, - 78, 67, 76, 79, 83, 85, 82, 69, 83, 128, 69, 78, 67, 76, 79, 83, 85, 82, - 69, 128, 69, 78, 67, 76, 79, 83, 73, 78, 199, 69, 78, 67, 128, 69, 78, - 65, 82, 88, 73, 211, 69, 78, 65, 82, 77, 79, 78, 73, 79, 211, 69, 77, 80, - 84, 217, 69, 77, 80, 72, 65, 84, 73, 195, 69, 77, 80, 72, 65, 83, 73, - 211, 69, 77, 79, 74, 201, 69, 77, 66, 82, 79, 73, 68, 69, 82, 89, 128, - 69, 77, 66, 76, 69, 77, 128, 69, 77, 66, 69, 76, 76, 73, 83, 72, 77, 69, - 78, 84, 128, 69, 77, 66, 69, 68, 68, 73, 78, 71, 128, 69, 76, 89, 77, 65, - 73, 195, 69, 76, 89, 128, 69, 76, 84, 128, 69, 76, 76, 73, 80, 84, 73, - 195, 69, 76, 76, 73, 80, 83, 73, 83, 128, 69, 76, 76, 73, 80, 83, 69, - 128, 69, 76, 73, 70, 73, 128, 69, 76, 73, 70, 128, 69, 76, 69, 86, 69, - 78, 45, 84, 72, 73, 82, 84, 89, 128, 69, 76, 69, 86, 69, 78, 128, 69, 76, - 69, 86, 69, 206, 69, 76, 69, 86, 65, 84, 85, 211, 69, 76, 69, 86, 65, 84, - 79, 82, 128, 69, 76, 69, 80, 72, 65, 78, 84, 128, 69, 76, 69, 77, 69, 78, - 212, 69, 76, 69, 67, 84, 82, 79, 78, 73, 67, 83, 128, 69, 76, 69, 67, 84, - 82, 73, 67, 65, 204, 69, 76, 69, 67, 84, 82, 73, 195, 69, 76, 66, 65, 83, - 65, 206, 69, 76, 65, 77, 73, 84, 69, 128, 69, 76, 65, 77, 73, 84, 197, - 69, 76, 65, 70, 82, 79, 78, 128, 69, 75, 83, 84, 82, 69, 80, 84, 79, 78, - 128, 69, 75, 83, 128, 69, 75, 70, 79, 78, 73, 84, 73, 75, 79, 78, 128, - 69, 75, 65, 82, 65, 128, 69, 75, 65, 77, 128, 69, 74, 69, 67, 212, 69, - 73, 83, 128, 69, 73, 71, 72, 84, 89, 128, 69, 73, 71, 72, 84, 217, 69, - 73, 71, 72, 84, 73, 69, 84, 72, 83, 128, 69, 73, 71, 72, 84, 73, 69, 84, - 72, 128, 69, 73, 71, 72, 84, 72, 83, 128, 69, 73, 71, 72, 84, 72, 211, - 69, 73, 71, 72, 84, 72, 128, 69, 73, 71, 72, 84, 69, 69, 78, 128, 69, 73, - 71, 72, 84, 69, 69, 206, 69, 73, 71, 72, 84, 45, 84, 72, 73, 82, 84, 89, - 128, 69, 73, 69, 128, 69, 72, 87, 65, 218, 69, 72, 84, 83, 65, 128, 69, - 72, 84, 65, 128, 69, 72, 80, 65, 128, 69, 72, 75, 65, 128, 69, 72, 67, - 72, 65, 128, 69, 71, 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 204, 69, 71, - 89, 128, 69, 71, 73, 82, 128, 69, 71, 71, 83, 128, 69, 71, 71, 128, 69, - 69, 89, 65, 78, 78, 65, 128, 69, 69, 75, 65, 65, 128, 69, 69, 72, 128, - 69, 69, 66, 69, 69, 70, 73, 76, 73, 128, 69, 68, 73, 84, 79, 82, 73, 65, - 204, 69, 68, 73, 78, 128, 69, 68, 68, 128, 69, 67, 83, 128, 69, 67, 76, - 73, 80, 83, 69, 128, 69, 66, 69, 70, 73, 76, 73, 128, 69, 65, 83, 84, 69, - 82, 206, 69, 65, 83, 84, 128, 69, 65, 83, 212, 69, 65, 82, 84, 72, 76, - 217, 69, 65, 82, 84, 72, 128, 69, 65, 82, 84, 200, 69, 65, 82, 76, 217, - 69, 65, 77, 72, 65, 78, 67, 72, 79, 76, 76, 128, 69, 65, 71, 76, 69, 128, - 69, 65, 68, 72, 65, 68, 72, 128, 69, 65, 66, 72, 65, 68, 72, 128, 69, - 178, 69, 48, 51, 56, 128, 69, 48, 51, 55, 128, 69, 48, 51, 54, 128, 69, - 48, 51, 52, 65, 128, 69, 48, 51, 52, 128, 69, 48, 51, 51, 128, 69, 48, - 51, 50, 128, 69, 48, 51, 49, 128, 69, 48, 51, 48, 128, 69, 48, 50, 57, - 128, 69, 48, 50, 56, 65, 128, 69, 48, 50, 56, 128, 69, 48, 50, 55, 128, - 69, 48, 50, 54, 128, 69, 48, 50, 53, 128, 69, 48, 50, 52, 128, 69, 48, - 50, 51, 128, 69, 48, 50, 50, 128, 69, 48, 50, 49, 128, 69, 48, 50, 48, - 65, 128, 69, 48, 50, 48, 128, 69, 48, 49, 57, 128, 69, 48, 49, 56, 128, - 69, 48, 49, 55, 65, 128, 69, 48, 49, 55, 128, 69, 48, 49, 54, 65, 128, - 69, 48, 49, 54, 128, 69, 48, 49, 53, 128, 69, 48, 49, 52, 128, 69, 48, - 49, 51, 128, 69, 48, 49, 50, 128, 69, 48, 49, 49, 128, 69, 48, 49, 48, - 128, 69, 48, 48, 57, 65, 128, 69, 48, 48, 57, 128, 69, 48, 48, 56, 65, - 128, 69, 48, 48, 56, 128, 69, 48, 48, 55, 128, 69, 48, 48, 54, 128, 69, - 48, 48, 53, 128, 69, 48, 48, 52, 128, 69, 48, 48, 51, 128, 69, 48, 48, - 50, 128, 69, 48, 48, 49, 128, 69, 45, 77, 65, 73, 204, 68, 90, 90, 72, - 69, 128, 68, 90, 90, 69, 128, 68, 90, 90, 65, 128, 68, 90, 89, 73, 128, - 68, 90, 89, 65, 89, 128, 68, 90, 87, 69, 128, 68, 90, 85, 128, 68, 90, - 79, 128, 68, 90, 74, 69, 128, 68, 90, 73, 84, 65, 128, 68, 90, 73, 128, - 68, 90, 72, 79, 73, 128, 68, 90, 72, 69, 128, 68, 90, 72, 65, 128, 68, - 90, 69, 76, 79, 128, 68, 90, 69, 69, 128, 68, 90, 69, 128, 68, 90, 65, - 89, 128, 68, 90, 65, 65, 128, 68, 90, 65, 128, 68, 90, 128, 68, 218, 68, - 89, 79, 128, 68, 89, 207, 68, 89, 78, 65, 77, 73, 195, 68, 89, 69, 72, - 128, 68, 89, 69, 200, 68, 89, 65, 78, 128, 68, 87, 79, 128, 68, 87, 69, - 128, 68, 87, 65, 128, 68, 86, 85, 77, 89, 193, 68, 86, 79, 69, 84, 79, - 67, 72, 73, 69, 128, 68, 86, 79, 69, 67, 72, 69, 76, 78, 79, 80, 79, 86, - 79, 68, 78, 65, 89, 65, 128, 68, 86, 79, 69, 67, 72, 69, 76, 78, 79, 80, - 79, 86, 79, 68, 78, 65, 89, 193, 68, 86, 79, 69, 67, 72, 69, 76, 78, 79, - 75, 82, 89, 90, 72, 69, 86, 65, 89, 65, 128, 68, 86, 79, 69, 67, 72, 69, - 76, 78, 65, 89, 65, 128, 68, 86, 79, 69, 67, 72, 69, 76, 78, 65, 89, 193, - 68, 86, 73, 83, 86, 65, 82, 65, 128, 68, 86, 68, 128, 68, 86, 193, 68, - 86, 128, 68, 85, 84, 73, 69, 83, 128, 68, 85, 83, 75, 128, 68, 85, 83, - 72, 69, 78, 78, 65, 128, 68, 85, 82, 65, 84, 73, 79, 78, 128, 68, 85, 82, - 50, 128, 68, 85, 80, 79, 78, 68, 73, 85, 211, 68, 85, 79, 88, 128, 68, - 85, 79, 128, 68, 85, 78, 52, 128, 68, 85, 78, 51, 128, 68, 85, 78, 179, - 68, 85, 77, 80, 76, 73, 78, 71, 128, 68, 85, 77, 128, 68, 85, 204, 68, - 85, 72, 128, 68, 85, 71, 85, 68, 128, 68, 85, 199, 68, 85, 68, 65, 128, - 68, 85, 67, 75, 128, 68, 85, 66, 50, 128, 68, 85, 66, 128, 68, 85, 194, - 68, 82, 89, 128, 68, 82, 217, 68, 82, 85, 77, 83, 84, 73, 67, 75, 83, - 128, 68, 82, 85, 77, 128, 68, 82, 85, 205, 68, 82, 79, 80, 83, 128, 68, - 82, 79, 80, 76, 69, 84, 128, 68, 82, 79, 80, 45, 83, 72, 65, 68, 79, 87, - 69, 196, 68, 82, 79, 208, 68, 82, 79, 79, 76, 73, 78, 199, 68, 82, 79, - 77, 69, 68, 65, 82, 217, 68, 82, 73, 86, 69, 128, 68, 82, 73, 86, 197, - 68, 82, 73, 78, 75, 128, 68, 82, 73, 204, 68, 82, 69, 83, 83, 128, 68, - 82, 69, 65, 77, 217, 68, 82, 65, 85, 71, 72, 84, 211, 68, 82, 65, 77, - 128, 68, 82, 65, 205, 68, 82, 65, 71, 79, 78, 128, 68, 82, 65, 71, 79, - 206, 68, 82, 65, 70, 84, 73, 78, 199, 68, 82, 65, 67, 72, 77, 65, 83, - 128, 68, 82, 65, 67, 72, 77, 65, 128, 68, 82, 65, 67, 72, 77, 193, 68, - 79, 87, 78, 87, 65, 82, 68, 83, 128, 68, 79, 87, 78, 87, 65, 82, 68, 211, - 68, 79, 87, 78, 87, 65, 82, 196, 68, 79, 87, 78, 83, 67, 65, 76, 73, 78, - 199, 68, 79, 87, 78, 45, 80, 79, 73, 78, 84, 73, 78, 199, 68, 79, 87, 78, - 128, 68, 79, 86, 69, 128, 68, 79, 86, 197, 68, 79, 85, 71, 72, 78, 85, - 84, 128, 68, 79, 85, 66, 84, 128, 68, 79, 85, 66, 76, 69, 196, 68, 79, - 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 203, 68, 79, 85, 66, 76, 69, 45, - 76, 73, 78, 69, 196, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 197, 68, 79, - 85, 66, 76, 69, 45, 69, 78, 68, 69, 196, 68, 79, 85, 66, 76, 69, 128, 68, - 79, 84, 84, 69, 68, 45, 80, 128, 68, 79, 84, 84, 69, 68, 45, 78, 128, 68, - 79, 84, 84, 69, 68, 45, 76, 128, 68, 79, 84, 84, 69, 68, 128, 68, 79, 84, - 84, 69, 196, 68, 79, 84, 83, 45, 56, 128, 68, 79, 84, 83, 45, 55, 56, - 128, 68, 79, 84, 83, 45, 55, 128, 68, 79, 84, 83, 45, 54, 56, 128, 68, - 79, 84, 83, 45, 54, 55, 56, 128, 68, 79, 84, 83, 45, 54, 55, 128, 68, 79, - 84, 83, 45, 54, 128, 68, 79, 84, 83, 45, 53, 56, 128, 68, 79, 84, 83, 45, - 53, 55, 56, 128, 68, 79, 84, 83, 45, 53, 55, 128, 68, 79, 84, 83, 45, 53, - 54, 56, 128, 68, 79, 84, 83, 45, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, - 53, 54, 55, 128, 68, 79, 84, 83, 45, 53, 54, 128, 68, 79, 84, 83, 45, 53, - 128, 68, 79, 84, 83, 45, 52, 56, 128, 68, 79, 84, 83, 45, 52, 55, 56, - 128, 68, 79, 84, 83, 45, 52, 55, 128, 68, 79, 84, 83, 45, 52, 54, 56, - 128, 68, 79, 84, 83, 45, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 52, 54, - 55, 128, 68, 79, 84, 83, 45, 52, 54, 128, 68, 79, 84, 83, 45, 52, 53, 56, - 128, 68, 79, 84, 83, 45, 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, 52, 53, - 55, 128, 68, 79, 84, 83, 45, 52, 53, 54, 56, 128, 68, 79, 84, 83, 45, 52, - 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 52, 53, 54, 55, 128, 68, 79, 84, - 83, 45, 52, 53, 54, 128, 68, 79, 84, 83, 45, 52, 53, 128, 68, 79, 84, 83, - 45, 52, 128, 68, 79, 84, 83, 45, 51, 56, 128, 68, 79, 84, 83, 45, 51, 55, - 56, 128, 68, 79, 84, 83, 45, 51, 55, 128, 68, 79, 84, 83, 45, 51, 54, 56, - 128, 68, 79, 84, 83, 45, 51, 54, 55, 56, 128, 68, 79, 84, 83, 45, 51, 54, - 55, 128, 68, 79, 84, 83, 45, 51, 54, 128, 68, 79, 84, 83, 45, 51, 53, 56, - 128, 68, 79, 84, 83, 45, 51, 53, 55, 56, 128, 68, 79, 84, 83, 45, 51, 53, - 55, 128, 68, 79, 84, 83, 45, 51, 53, 54, 56, 128, 68, 79, 84, 83, 45, 51, - 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 51, 53, 54, 55, 128, 68, 79, 84, - 83, 45, 51, 53, 54, 128, 68, 79, 84, 83, 45, 51, 53, 128, 68, 79, 84, 83, - 45, 51, 52, 56, 128, 68, 79, 84, 83, 45, 51, 52, 55, 56, 128, 68, 79, 84, - 83, 45, 51, 52, 55, 128, 68, 79, 84, 83, 45, 51, 52, 54, 56, 128, 68, 79, - 84, 83, 45, 51, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 51, 52, 54, 55, - 128, 68, 79, 84, 83, 45, 51, 52, 54, 128, 68, 79, 84, 83, 45, 51, 52, 53, - 56, 128, 68, 79, 84, 83, 45, 51, 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, - 51, 52, 53, 55, 128, 68, 79, 84, 83, 45, 51, 52, 53, 54, 56, 128, 68, 79, - 84, 83, 45, 51, 52, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 51, 52, 53, - 54, 55, 128, 68, 79, 84, 83, 45, 51, 52, 53, 54, 128, 68, 79, 84, 83, 45, - 51, 52, 53, 128, 68, 79, 84, 83, 45, 51, 52, 128, 68, 79, 84, 83, 45, 51, - 128, 68, 79, 84, 83, 45, 50, 56, 128, 68, 79, 84, 83, 45, 50, 55, 56, - 128, 68, 79, 84, 83, 45, 50, 55, 128, 68, 79, 84, 83, 45, 50, 54, 56, - 128, 68, 79, 84, 83, 45, 50, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 54, - 55, 128, 68, 79, 84, 83, 45, 50, 54, 128, 68, 79, 84, 83, 45, 50, 53, 56, - 128, 68, 79, 84, 83, 45, 50, 53, 55, 56, 128, 68, 79, 84, 83, 45, 50, 53, - 55, 128, 68, 79, 84, 83, 45, 50, 53, 54, 56, 128, 68, 79, 84, 83, 45, 50, - 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 53, 54, 55, 128, 68, 79, 84, - 83, 45, 50, 53, 54, 128, 68, 79, 84, 83, 45, 50, 53, 128, 68, 79, 84, 83, - 45, 50, 52, 56, 128, 68, 79, 84, 83, 45, 50, 52, 55, 56, 128, 68, 79, 84, - 83, 45, 50, 52, 55, 128, 68, 79, 84, 83, 45, 50, 52, 54, 56, 128, 68, 79, - 84, 83, 45, 50, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 52, 54, 55, - 128, 68, 79, 84, 83, 45, 50, 52, 54, 128, 68, 79, 84, 83, 45, 50, 52, 53, - 56, 128, 68, 79, 84, 83, 45, 50, 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, - 50, 52, 53, 55, 128, 68, 79, 84, 83, 45, 50, 52, 53, 54, 56, 128, 68, 79, - 84, 83, 45, 50, 52, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 52, 53, - 54, 55, 128, 68, 79, 84, 83, 45, 50, 52, 53, 54, 128, 68, 79, 84, 83, 45, - 50, 52, 53, 128, 68, 79, 84, 83, 45, 50, 52, 128, 68, 79, 84, 83, 45, 50, - 51, 56, 128, 68, 79, 84, 83, 45, 50, 51, 55, 56, 128, 68, 79, 84, 83, 45, - 50, 51, 55, 128, 68, 79, 84, 83, 45, 50, 51, 54, 56, 128, 68, 79, 84, 83, - 45, 50, 51, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, 54, 55, 128, 68, - 79, 84, 83, 45, 50, 51, 54, 128, 68, 79, 84, 83, 45, 50, 51, 53, 56, 128, - 68, 79, 84, 83, 45, 50, 51, 53, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, - 53, 55, 128, 68, 79, 84, 83, 45, 50, 51, 53, 54, 56, 128, 68, 79, 84, 83, - 45, 50, 51, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, 53, 54, 55, - 128, 68, 79, 84, 83, 45, 50, 51, 53, 54, 128, 68, 79, 84, 83, 45, 50, 51, - 53, 128, 68, 79, 84, 83, 45, 50, 51, 52, 56, 128, 68, 79, 84, 83, 45, 50, - 51, 52, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, 52, 55, 128, 68, 79, 84, - 83, 45, 50, 51, 52, 54, 56, 128, 68, 79, 84, 83, 45, 50, 51, 52, 54, 55, - 56, 128, 68, 79, 84, 83, 45, 50, 51, 52, 54, 55, 128, 68, 79, 84, 83, 45, - 50, 51, 52, 54, 128, 68, 79, 84, 83, 45, 50, 51, 52, 53, 56, 128, 68, 79, - 84, 83, 45, 50, 51, 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, 52, - 53, 55, 128, 68, 79, 84, 83, 45, 50, 51, 52, 53, 54, 56, 128, 68, 79, 84, - 83, 45, 50, 51, 52, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 50, 51, 52, - 53, 54, 55, 128, 68, 79, 84, 83, 45, 50, 51, 52, 53, 54, 128, 68, 79, 84, - 83, 45, 50, 51, 52, 53, 128, 68, 79, 84, 83, 45, 50, 51, 52, 128, 68, 79, - 84, 83, 45, 50, 51, 128, 68, 79, 84, 83, 45, 50, 128, 68, 79, 84, 83, 45, - 49, 56, 128, 68, 79, 84, 83, 45, 49, 55, 56, 128, 68, 79, 84, 83, 45, 49, - 55, 128, 68, 79, 84, 83, 45, 49, 54, 56, 128, 68, 79, 84, 83, 45, 49, 54, - 55, 56, 128, 68, 79, 84, 83, 45, 49, 54, 55, 128, 68, 79, 84, 83, 45, 49, - 54, 128, 68, 79, 84, 83, 45, 49, 53, 56, 128, 68, 79, 84, 83, 45, 49, 53, - 55, 56, 128, 68, 79, 84, 83, 45, 49, 53, 55, 128, 68, 79, 84, 83, 45, 49, - 53, 54, 56, 128, 68, 79, 84, 83, 45, 49, 53, 54, 55, 56, 128, 68, 79, 84, - 83, 45, 49, 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 53, 54, 128, 68, 79, - 84, 83, 45, 49, 53, 128, 68, 79, 84, 83, 45, 49, 52, 56, 128, 68, 79, 84, - 83, 45, 49, 52, 55, 56, 128, 68, 79, 84, 83, 45, 49, 52, 55, 128, 68, 79, - 84, 83, 45, 49, 52, 54, 56, 128, 68, 79, 84, 83, 45, 49, 52, 54, 55, 56, - 128, 68, 79, 84, 83, 45, 49, 52, 54, 55, 128, 68, 79, 84, 83, 45, 49, 52, - 54, 128, 68, 79, 84, 83, 45, 49, 52, 53, 56, 128, 68, 79, 84, 83, 45, 49, - 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, 49, 52, 53, 55, 128, 68, 79, 84, - 83, 45, 49, 52, 53, 54, 56, 128, 68, 79, 84, 83, 45, 49, 52, 53, 54, 55, - 56, 128, 68, 79, 84, 83, 45, 49, 52, 53, 54, 55, 128, 68, 79, 84, 83, 45, - 49, 52, 53, 54, 128, 68, 79, 84, 83, 45, 49, 52, 53, 128, 68, 79, 84, 83, - 45, 49, 52, 128, 68, 79, 84, 83, 45, 49, 51, 56, 128, 68, 79, 84, 83, 45, - 49, 51, 55, 56, 128, 68, 79, 84, 83, 45, 49, 51, 55, 128, 68, 79, 84, 83, - 45, 49, 51, 54, 56, 128, 68, 79, 84, 83, 45, 49, 51, 54, 55, 56, 128, 68, - 79, 84, 83, 45, 49, 51, 54, 55, 128, 68, 79, 84, 83, 45, 49, 51, 54, 128, - 68, 79, 84, 83, 45, 49, 51, 53, 56, 128, 68, 79, 84, 83, 45, 49, 51, 53, - 55, 56, 128, 68, 79, 84, 83, 45, 49, 51, 53, 55, 128, 68, 79, 84, 83, 45, - 49, 51, 53, 54, 56, 128, 68, 79, 84, 83, 45, 49, 51, 53, 54, 55, 56, 128, - 68, 79, 84, 83, 45, 49, 51, 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 51, - 53, 54, 128, 68, 79, 84, 83, 45, 49, 51, 53, 128, 68, 79, 84, 83, 45, 49, - 51, 52, 56, 128, 68, 79, 84, 83, 45, 49, 51, 52, 55, 56, 128, 68, 79, 84, - 83, 45, 49, 51, 52, 55, 128, 68, 79, 84, 83, 45, 49, 51, 52, 54, 56, 128, - 68, 79, 84, 83, 45, 49, 51, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, - 51, 52, 54, 55, 128, 68, 79, 84, 83, 45, 49, 51, 52, 54, 128, 68, 79, 84, - 83, 45, 49, 51, 52, 53, 56, 128, 68, 79, 84, 83, 45, 49, 51, 52, 53, 55, - 56, 128, 68, 79, 84, 83, 45, 49, 51, 52, 53, 55, 128, 68, 79, 84, 83, 45, - 49, 51, 52, 53, 54, 56, 128, 68, 79, 84, 83, 45, 49, 51, 52, 53, 54, 55, - 56, 128, 68, 79, 84, 83, 45, 49, 51, 52, 53, 54, 55, 128, 68, 79, 84, 83, - 45, 49, 51, 52, 53, 54, 128, 68, 79, 84, 83, 45, 49, 51, 52, 53, 128, 68, - 79, 84, 83, 45, 49, 51, 52, 128, 68, 79, 84, 83, 45, 49, 51, 128, 68, 79, - 84, 83, 45, 49, 50, 56, 128, 68, 79, 84, 83, 45, 49, 50, 55, 56, 128, 68, - 79, 84, 83, 45, 49, 50, 55, 128, 68, 79, 84, 83, 45, 49, 50, 54, 56, 128, - 68, 79, 84, 83, 45, 49, 50, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, - 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 54, 128, 68, 79, 84, 83, 45, 49, - 50, 53, 56, 128, 68, 79, 84, 83, 45, 49, 50, 53, 55, 56, 128, 68, 79, 84, - 83, 45, 49, 50, 53, 55, 128, 68, 79, 84, 83, 45, 49, 50, 53, 54, 56, 128, - 68, 79, 84, 83, 45, 49, 50, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, - 50, 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 53, 54, 128, 68, 79, 84, - 83, 45, 49, 50, 53, 128, 68, 79, 84, 83, 45, 49, 50, 52, 56, 128, 68, 79, - 84, 83, 45, 49, 50, 52, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 52, 55, - 128, 68, 79, 84, 83, 45, 49, 50, 52, 54, 56, 128, 68, 79, 84, 83, 45, 49, - 50, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 52, 54, 55, 128, 68, - 79, 84, 83, 45, 49, 50, 52, 54, 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, - 56, 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, 55, 56, 128, 68, 79, 84, 83, - 45, 49, 50, 52, 53, 55, 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, 54, 56, - 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, 54, 55, 56, 128, 68, 79, 84, 83, - 45, 49, 50, 52, 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, 54, - 128, 68, 79, 84, 83, 45, 49, 50, 52, 53, 128, 68, 79, 84, 83, 45, 49, 50, - 52, 128, 68, 79, 84, 83, 45, 49, 50, 51, 56, 128, 68, 79, 84, 83, 45, 49, - 50, 51, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, 55, 128, 68, 79, 84, - 83, 45, 49, 50, 51, 54, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, 54, 55, - 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, 54, 55, 128, 68, 79, 84, 83, 45, - 49, 50, 51, 54, 128, 68, 79, 84, 83, 45, 49, 50, 51, 53, 56, 128, 68, 79, - 84, 83, 45, 49, 50, 51, 53, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, - 53, 55, 128, 68, 79, 84, 83, 45, 49, 50, 51, 53, 54, 56, 128, 68, 79, 84, - 83, 45, 49, 50, 51, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, - 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 51, 53, 54, 128, 68, 79, 84, - 83, 45, 49, 50, 51, 53, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, 56, 128, - 68, 79, 84, 83, 45, 49, 50, 51, 52, 55, 56, 128, 68, 79, 84, 83, 45, 49, - 50, 51, 52, 55, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, 54, 56, 128, 68, - 79, 84, 83, 45, 49, 50, 51, 52, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, - 50, 51, 52, 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, 54, 128, 68, - 79, 84, 83, 45, 49, 50, 51, 52, 53, 56, 128, 68, 79, 84, 83, 45, 49, 50, - 51, 52, 53, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, 53, 55, 128, - 68, 79, 84, 83, 45, 49, 50, 51, 52, 53, 54, 56, 128, 68, 79, 84, 83, 45, - 49, 50, 51, 52, 53, 54, 55, 56, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, - 53, 54, 55, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, 53, 54, 128, 68, 79, - 84, 83, 45, 49, 50, 51, 52, 53, 128, 68, 79, 84, 83, 45, 49, 50, 51, 52, - 128, 68, 79, 84, 83, 45, 49, 50, 51, 128, 68, 79, 84, 83, 45, 49, 50, - 128, 68, 79, 84, 83, 45, 49, 128, 68, 79, 84, 83, 128, 68, 79, 84, 76, - 69, 83, 211, 68, 79, 82, 85, 128, 68, 79, 82, 79, 77, 197, 68, 79, 79, - 82, 128, 68, 79, 79, 78, 71, 128, 68, 79, 78, 75, 69, 89, 128, 68, 79, - 78, 71, 128, 68, 79, 77, 73, 78, 207, 68, 79, 77, 65, 73, 206, 68, 79, - 76, 80, 72, 73, 78, 128, 68, 79, 76, 76, 83, 128, 68, 79, 76, 76, 65, - 210, 68, 79, 76, 73, 85, 77, 128, 68, 79, 75, 77, 65, 73, 128, 68, 79, - 73, 84, 128, 68, 79, 73, 78, 199, 68, 79, 73, 128, 68, 79, 71, 82, 193, - 68, 79, 71, 128, 68, 79, 199, 68, 79, 69, 211, 68, 79, 68, 79, 128, 68, - 79, 68, 69, 75, 65, 84, 65, 128, 68, 79, 67, 85, 77, 69, 78, 84, 128, 68, - 79, 67, 85, 77, 69, 78, 212, 68, 79, 66, 82, 79, 128, 68, 79, 65, 67, 72, - 65, 83, 72, 77, 69, 69, 128, 68, 79, 65, 67, 72, 65, 83, 72, 77, 69, 197, - 68, 79, 65, 128, 68, 79, 45, 79, 128, 68, 78, 193, 68, 77, 128, 68, 205, - 68, 76, 85, 128, 68, 76, 79, 128, 68, 76, 73, 128, 68, 76, 72, 89, 65, - 128, 68, 76, 72, 65, 128, 68, 76, 69, 69, 128, 68, 76, 65, 128, 68, 76, - 128, 68, 75, 65, 82, 128, 68, 75, 65, 210, 68, 74, 69, 82, 86, 73, 128, - 68, 74, 69, 82, 86, 128, 68, 74, 69, 128, 68, 74, 65, 128, 68, 73, 90, - 90, 217, 68, 73, 89, 193, 68, 73, 86, 79, 82, 67, 197, 68, 73, 86, 73, - 83, 73, 79, 78, 128, 68, 73, 86, 73, 83, 73, 79, 206, 68, 73, 86, 73, 78, - 199, 68, 73, 86, 73, 78, 65, 84, 73, 79, 78, 128, 68, 73, 86, 73, 68, 69, - 83, 128, 68, 73, 86, 73, 68, 69, 82, 83, 128, 68, 73, 86, 73, 68, 69, 82, - 128, 68, 73, 86, 73, 68, 69, 196, 68, 73, 86, 73, 68, 69, 128, 68, 73, - 86, 73, 68, 197, 68, 73, 86, 69, 211, 68, 73, 86, 69, 82, 71, 69, 78, 67, - 69, 128, 68, 73, 84, 84, 207, 68, 73, 83, 84, 79, 82, 84, 73, 79, 78, - 128, 68, 73, 83, 84, 73, 78, 71, 85, 73, 83, 72, 128, 68, 73, 83, 84, 73, - 76, 76, 128, 68, 73, 83, 83, 79, 76, 86, 69, 45, 50, 128, 68, 73, 83, 83, - 79, 76, 86, 69, 128, 68, 73, 83, 80, 85, 84, 69, 196, 68, 73, 83, 80, 69, - 82, 83, 73, 79, 78, 128, 68, 73, 83, 75, 128, 68, 73, 83, 73, 77, 79, 85, - 128, 68, 73, 83, 72, 128, 68, 73, 83, 71, 85, 73, 83, 69, 196, 68, 73, - 83, 67, 79, 78, 84, 73, 78, 85, 79, 85, 211, 68, 73, 83, 195, 68, 73, 83, - 65, 80, 80, 79, 73, 78, 84, 69, 196, 68, 73, 83, 65, 66, 76, 69, 196, 68, - 73, 82, 71, 193, 68, 73, 82, 69, 67, 84, 76, 217, 68, 73, 82, 69, 67, 84, - 73, 79, 78, 65, 204, 68, 73, 82, 69, 67, 84, 73, 79, 206, 68, 73, 80, 84, - 69, 128, 68, 73, 80, 80, 69, 82, 128, 68, 73, 80, 76, 79, 85, 78, 128, - 68, 73, 80, 76, 73, 128, 68, 73, 80, 76, 201, 68, 73, 78, 71, 66, 65, - 212, 68, 73, 206, 68, 73, 77, 77, 73, 78, 71, 128, 68, 73, 77, 73, 78, - 85, 84, 73, 79, 78, 45, 51, 128, 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, - 45, 50, 128, 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, 45, 49, 128, 68, 73, - 77, 73, 78, 73, 83, 72, 77, 69, 78, 84, 128, 68, 73, 77, 73, 68, 73, 193, - 68, 73, 77, 69, 78, 83, 73, 79, 78, 65, 204, 68, 73, 77, 69, 78, 83, 73, - 79, 206, 68, 73, 77, 50, 128, 68, 73, 77, 178, 68, 73, 76, 128, 68, 73, - 71, 82, 65, 80, 72, 128, 68, 73, 71, 82, 65, 80, 200, 68, 73, 71, 82, 65, - 77, 77, 79, 211, 68, 73, 71, 82, 65, 77, 77, 193, 68, 73, 71, 82, 65, - 205, 68, 73, 71, 79, 82, 71, 79, 78, 128, 68, 73, 71, 79, 82, 71, 79, - 206, 68, 73, 71, 73, 84, 83, 128, 68, 73, 71, 65, 77, 77, 65, 128, 68, - 73, 71, 193, 68, 73, 70, 84, 79, 71, 71, 79, 211, 68, 73, 70, 79, 78, 73, - 65, 83, 128, 68, 73, 70, 70, 73, 67, 85, 76, 84, 217, 68, 73, 70, 70, 73, - 67, 85, 76, 84, 73, 69, 83, 128, 68, 73, 70, 70, 69, 82, 69, 78, 84, 73, - 65, 76, 128, 68, 73, 70, 70, 69, 82, 69, 78, 67, 197, 68, 73, 70, 65, 84, - 128, 68, 73, 69, 83, 73, 83, 128, 68, 73, 69, 83, 73, 211, 68, 73, 69, - 83, 69, 204, 68, 73, 69, 80, 128, 68, 73, 197, 68, 73, 66, 128, 68, 73, - 65, 84, 79, 78, 79, 206, 68, 73, 65, 84, 79, 78, 73, 75, 201, 68, 73, 65, - 83, 84, 79, 76, 201, 68, 73, 65, 77, 79, 78, 68, 83, 128, 68, 73, 65, 77, - 79, 78, 68, 128, 68, 73, 65, 77, 79, 78, 196, 68, 73, 65, 77, 69, 84, 69, - 210, 68, 73, 65, 76, 89, 84, 73, 75, 65, 128, 68, 73, 65, 76, 89, 84, 73, - 75, 193, 68, 73, 65, 76, 69, 67, 84, 45, 208, 68, 73, 65, 71, 79, 78, 65, - 76, 128, 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 196, 68, 73, 65, 69, 82, - 69, 83, 73, 83, 45, 82, 73, 78, 71, 128, 68, 73, 65, 69, 82, 69, 83, 73, - 83, 128, 68, 73, 65, 69, 82, 69, 83, 73, 211, 68, 72, 79, 85, 128, 68, - 72, 79, 79, 128, 68, 72, 79, 128, 68, 72, 73, 73, 128, 68, 72, 72, 85, - 128, 68, 72, 72, 79, 79, 128, 68, 72, 72, 79, 128, 68, 72, 72, 73, 128, - 68, 72, 72, 69, 69, 128, 68, 72, 72, 69, 128, 68, 72, 72, 65, 128, 68, - 72, 69, 69, 128, 68, 72, 65, 82, 77, 65, 128, 68, 72, 65, 77, 69, 68, 72, - 128, 68, 72, 65, 76, 69, 84, 72, 128, 68, 72, 65, 76, 65, 84, 72, 128, - 68, 72, 65, 76, 128, 68, 72, 65, 68, 72, 69, 128, 68, 72, 65, 65, 76, 85, - 128, 68, 72, 65, 65, 128, 68, 72, 65, 128, 68, 69, 90, 200, 68, 69, 89, - 84, 69, 82, 79, 213, 68, 69, 89, 84, 69, 82, 79, 211, 68, 69, 88, 73, 65, - 128, 68, 69, 86, 73, 67, 197, 68, 69, 86, 69, 76, 79, 80, 77, 69, 78, 84, - 128, 68, 69, 85, 78, 71, 128, 68, 69, 83, 75, 84, 79, 208, 68, 69, 83, - 203, 68, 69, 83, 73, 71, 78, 128, 68, 69, 83, 73, 128, 68, 69, 83, 69, - 82, 84, 128, 68, 69, 83, 69, 82, 212, 68, 69, 83, 69, 82, 69, 212, 68, - 69, 83, 67, 82, 73, 80, 84, 73, 79, 206, 68, 69, 83, 67, 69, 78, 68, 73, - 78, 199, 68, 69, 83, 67, 69, 78, 68, 69, 82, 128, 68, 69, 82, 69, 84, 45, - 72, 73, 68, 69, 84, 128, 68, 69, 82, 69, 84, 128, 68, 69, 82, 69, 76, 73, - 67, 212, 68, 69, 82, 66, 73, 84, 83, 65, 128, 68, 69, 80, 84, 72, 128, - 68, 69, 80, 65, 82, 84, 85, 82, 69, 128, 68, 69, 80, 65, 82, 84, 77, 69, - 78, 212, 68, 69, 80, 65, 82, 84, 73, 78, 199, 68, 69, 78, 84, 73, 83, 84, - 82, 217, 68, 69, 78, 84, 65, 204, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, - 82, 128, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 210, 68, 69, 78, 78, 69, - 78, 128, 68, 69, 78, 71, 128, 68, 69, 78, 197, 68, 69, 78, 65, 82, 73, - 85, 211, 68, 69, 77, 69, 83, 84, 86, 69, 78, 78, 217, 68, 69, 76, 84, 65, - 128, 68, 69, 76, 84, 193, 68, 69, 76, 84, 128, 68, 69, 76, 80, 72, 73, - 195, 68, 69, 76, 73, 86, 69, 82, 217, 68, 69, 76, 73, 86, 69, 82, 65, 78, - 67, 69, 128, 68, 69, 76, 73, 77, 73, 84, 69, 82, 128, 68, 69, 76, 73, 77, - 73, 84, 69, 210, 68, 69, 76, 73, 67, 73, 79, 85, 211, 68, 69, 76, 69, 84, - 73, 79, 206, 68, 69, 76, 69, 84, 69, 128, 68, 69, 76, 69, 84, 197, 68, - 69, 75, 65, 128, 68, 69, 75, 128, 68, 69, 73, 128, 68, 69, 72, 73, 128, - 68, 69, 71, 82, 69, 69, 83, 128, 68, 69, 71, 82, 69, 197, 68, 69, 70, 73, - 78, 73, 84, 73, 79, 78, 128, 68, 69, 70, 69, 67, 84, 73, 86, 69, 78, 69, - 83, 211, 68, 69, 69, 82, 128, 68, 69, 69, 80, 76, 89, 128, 68, 69, 69, - 76, 128, 68, 69, 67, 82, 69, 83, 67, 69, 78, 68, 79, 128, 68, 69, 67, 82, - 69, 65, 83, 69, 128, 68, 69, 67, 82, 69, 65, 83, 197, 68, 69, 67, 79, 82, - 65, 84, 73, 86, 197, 68, 69, 67, 79, 82, 65, 84, 73, 79, 78, 128, 68, 69, - 67, 73, 83, 73, 86, 69, 78, 69, 83, 83, 128, 68, 69, 67, 73, 77, 65, 204, - 68, 69, 67, 73, 68, 85, 79, 85, 211, 68, 69, 67, 69, 77, 66, 69, 82, 128, - 68, 69, 67, 65, 89, 69, 68, 128, 68, 69, 66, 73, 212, 68, 69, 65, 84, 72, - 128, 68, 69, 65, 198, 68, 69, 65, 68, 128, 68, 68, 87, 65, 128, 68, 68, - 85, 88, 128, 68, 68, 85, 84, 128, 68, 68, 85, 82, 88, 128, 68, 68, 85, - 82, 128, 68, 68, 85, 80, 128, 68, 68, 85, 79, 88, 128, 68, 68, 85, 79, - 80, 128, 68, 68, 85, 79, 128, 68, 68, 85, 128, 68, 68, 79, 88, 128, 68, - 68, 79, 84, 128, 68, 68, 79, 80, 128, 68, 68, 79, 65, 128, 68, 68, 73, - 88, 128, 68, 68, 73, 84, 128, 68, 68, 73, 80, 128, 68, 68, 73, 69, 88, - 128, 68, 68, 73, 69, 80, 128, 68, 68, 73, 69, 128, 68, 68, 73, 128, 68, - 68, 72, 85, 128, 68, 68, 72, 79, 128, 68, 68, 72, 69, 69, 128, 68, 68, - 72, 69, 128, 68, 68, 72, 65, 65, 128, 68, 68, 72, 65, 128, 68, 68, 69, - 88, 128, 68, 68, 69, 80, 128, 68, 68, 69, 69, 128, 68, 68, 69, 128, 68, - 68, 68, 72, 65, 128, 68, 68, 68, 65, 128, 68, 68, 65, 89, 65, 78, 78, 65, - 128, 68, 68, 65, 88, 128, 68, 68, 65, 84, 128, 68, 68, 65, 80, 128, 68, - 68, 65, 76, 128, 68, 68, 65, 204, 68, 68, 65, 72, 65, 76, 128, 68, 68, - 65, 72, 65, 204, 68, 68, 65, 65, 128, 68, 67, 83, 128, 68, 67, 72, 69, - 128, 68, 67, 52, 128, 68, 67, 51, 128, 68, 67, 50, 128, 68, 67, 49, 128, - 68, 194, 68, 65, 89, 45, 78, 73, 71, 72, 84, 128, 68, 65, 217, 68, 65, - 87, 66, 128, 68, 65, 86, 73, 89, 65, 78, 73, 128, 68, 65, 86, 73, 68, - 128, 68, 65, 84, 197, 68, 65, 83, 73, 65, 128, 68, 65, 83, 73, 193, 68, - 65, 83, 72, 69, 196, 68, 65, 83, 72, 128, 68, 65, 83, 200, 68, 65, 83, - 69, 73, 65, 128, 68, 65, 82, 84, 128, 68, 65, 82, 75, 69, 78, 73, 78, 71, - 128, 68, 65, 82, 75, 69, 78, 73, 78, 199, 68, 65, 82, 203, 68, 65, 82, - 71, 65, 128, 68, 65, 82, 65, 52, 128, 68, 65, 82, 65, 51, 128, 68, 65, - 82, 128, 68, 65, 80, 45, 80, 82, 65, 205, 68, 65, 80, 45, 80, 73, 201, - 68, 65, 80, 45, 77, 85, 79, 217, 68, 65, 80, 45, 66, 85, 79, 206, 68, 65, - 80, 45, 66, 69, 201, 68, 65, 208, 68, 65, 78, 84, 65, 89, 65, 76, 65, 78, - 128, 68, 65, 78, 84, 65, 74, 193, 68, 65, 78, 71, 79, 128, 68, 65, 78, - 71, 128, 68, 65, 78, 199, 68, 65, 78, 68, 65, 128, 68, 65, 78, 67, 73, - 78, 71, 128, 68, 65, 78, 67, 69, 82, 128, 68, 65, 77, 80, 128, 68, 65, - 77, 208, 68, 65, 77, 77, 65, 84, 65, 78, 128, 68, 65, 77, 77, 65, 84, 65, - 206, 68, 65, 77, 77, 65, 128, 68, 65, 77, 77, 193, 68, 65, 77, 65, 82, - 85, 128, 68, 65, 77, 65, 71, 69, 68, 128, 68, 65, 77, 65, 71, 69, 196, - 68, 65, 76, 69, 84, 72, 45, 82, 69, 83, 72, 128, 68, 65, 76, 69, 84, 128, - 68, 65, 76, 69, 212, 68, 65, 76, 68, 65, 128, 68, 65, 76, 65, 84, 72, - 128, 68, 65, 76, 65, 84, 200, 68, 65, 76, 65, 84, 128, 68, 65, 73, 82, - 128, 68, 65, 73, 78, 71, 128, 68, 65, 73, 128, 68, 65, 72, 89, 65, 65, - 85, 83, 72, 45, 50, 128, 68, 65, 72, 89, 65, 65, 85, 83, 72, 128, 68, 65, - 71, 83, 128, 68, 65, 71, 71, 69, 82, 128, 68, 65, 71, 71, 69, 210, 68, - 65, 71, 69, 83, 72, 128, 68, 65, 71, 69, 83, 200, 68, 65, 71, 66, 65, 83, - 73, 78, 78, 65, 128, 68, 65, 71, 65, 218, 68, 65, 71, 65, 76, 71, 65, - 128, 68, 65, 71, 51, 128, 68, 65, 199, 68, 65, 69, 78, 71, 128, 68, 65, - 69, 199, 68, 65, 68, 128, 68, 65, 196, 68, 65, 65, 83, 85, 128, 68, 65, - 65, 76, 73, 128, 68, 65, 65, 68, 72, 85, 128, 68, 48, 54, 55, 72, 128, - 68, 48, 54, 55, 71, 128, 68, 48, 54, 55, 70, 128, 68, 48, 54, 55, 69, - 128, 68, 48, 54, 55, 68, 128, 68, 48, 54, 55, 67, 128, 68, 48, 54, 55, - 66, 128, 68, 48, 54, 55, 65, 128, 68, 48, 54, 55, 128, 68, 48, 54, 54, - 128, 68, 48, 54, 53, 128, 68, 48, 54, 52, 128, 68, 48, 54, 51, 128, 68, - 48, 54, 50, 128, 68, 48, 54, 49, 128, 68, 48, 54, 48, 128, 68, 48, 53, - 57, 128, 68, 48, 53, 56, 128, 68, 48, 53, 55, 128, 68, 48, 53, 54, 128, - 68, 48, 53, 53, 128, 68, 48, 53, 52, 65, 128, 68, 48, 53, 52, 128, 68, - 48, 53, 51, 128, 68, 48, 53, 50, 65, 128, 68, 48, 53, 50, 128, 68, 48, - 53, 49, 128, 68, 48, 53, 48, 73, 128, 68, 48, 53, 48, 72, 128, 68, 48, - 53, 48, 71, 128, 68, 48, 53, 48, 70, 128, 68, 48, 53, 48, 69, 128, 68, - 48, 53, 48, 68, 128, 68, 48, 53, 48, 67, 128, 68, 48, 53, 48, 66, 128, - 68, 48, 53, 48, 65, 128, 68, 48, 53, 48, 128, 68, 48, 52, 57, 128, 68, - 48, 52, 56, 65, 128, 68, 48, 52, 56, 128, 68, 48, 52, 55, 128, 68, 48, - 52, 54, 65, 128, 68, 48, 52, 54, 128, 68, 48, 52, 53, 128, 68, 48, 52, - 52, 128, 68, 48, 52, 51, 128, 68, 48, 52, 50, 128, 68, 48, 52, 49, 128, - 68, 48, 52, 48, 128, 68, 48, 51, 57, 128, 68, 48, 51, 56, 128, 68, 48, - 51, 55, 128, 68, 48, 51, 54, 128, 68, 48, 51, 53, 128, 68, 48, 51, 52, - 65, 128, 68, 48, 51, 52, 128, 68, 48, 51, 51, 128, 68, 48, 51, 50, 128, - 68, 48, 51, 49, 65, 128, 68, 48, 51, 49, 128, 68, 48, 51, 48, 128, 68, - 48, 50, 57, 128, 68, 48, 50, 56, 128, 68, 48, 50, 55, 65, 128, 68, 48, - 50, 55, 128, 68, 48, 50, 54, 128, 68, 48, 50, 53, 128, 68, 48, 50, 52, - 128, 68, 48, 50, 51, 128, 68, 48, 50, 50, 128, 68, 48, 50, 49, 128, 68, - 48, 50, 48, 128, 68, 48, 49, 57, 128, 68, 48, 49, 56, 128, 68, 48, 49, - 55, 128, 68, 48, 49, 54, 128, 68, 48, 49, 53, 128, 68, 48, 49, 52, 128, - 68, 48, 49, 51, 128, 68, 48, 49, 50, 128, 68, 48, 49, 49, 128, 68, 48, - 49, 48, 128, 68, 48, 48, 57, 128, 68, 48, 48, 56, 65, 128, 68, 48, 48, - 56, 128, 68, 48, 48, 55, 128, 68, 48, 48, 54, 128, 68, 48, 48, 53, 128, - 68, 48, 48, 52, 128, 68, 48, 48, 51, 128, 68, 48, 48, 50, 128, 68, 48, - 48, 49, 128, 67, 89, 88, 128, 67, 89, 84, 128, 67, 89, 82, 88, 128, 67, - 89, 82, 69, 78, 65, 73, 195, 67, 89, 82, 128, 67, 89, 80, 82, 79, 45, 77, - 73, 78, 79, 65, 206, 67, 89, 80, 82, 73, 79, 212, 67, 89, 80, 69, 82, 85, - 83, 128, 67, 89, 80, 128, 67, 89, 76, 73, 78, 68, 82, 73, 67, 73, 84, 89, - 128, 67, 89, 67, 76, 79, 78, 69, 128, 67, 89, 65, 89, 128, 67, 89, 65, - 87, 128, 67, 89, 65, 128, 67, 87, 79, 79, 128, 67, 87, 79, 128, 67, 87, - 73, 73, 128, 67, 87, 73, 128, 67, 87, 69, 79, 82, 84, 72, 128, 67, 87, - 69, 128, 67, 87, 65, 65, 128, 67, 85, 88, 128, 67, 85, 85, 128, 67, 85, - 212, 67, 85, 83, 84, 79, 77, 83, 128, 67, 85, 83, 84, 79, 77, 69, 210, - 67, 85, 83, 84, 65, 82, 68, 128, 67, 85, 83, 80, 128, 67, 85, 82, 88, - 128, 67, 85, 82, 86, 73, 78, 199, 67, 85, 82, 86, 69, 68, 128, 67, 85, - 82, 86, 69, 196, 67, 85, 82, 86, 69, 128, 67, 85, 82, 86, 197, 67, 85, - 82, 83, 73, 86, 197, 67, 85, 82, 82, 217, 67, 85, 82, 82, 69, 78, 84, - 128, 67, 85, 82, 82, 69, 78, 212, 67, 85, 82, 76, 217, 67, 85, 82, 76, - 73, 78, 199, 67, 85, 82, 76, 69, 196, 67, 85, 82, 76, 128, 67, 85, 82, - 128, 67, 85, 80, 80, 69, 68, 128, 67, 85, 80, 80, 69, 196, 67, 85, 80, - 73, 68, 79, 128, 67, 85, 80, 67, 65, 75, 69, 128, 67, 85, 79, 88, 128, - 67, 85, 79, 80, 128, 67, 85, 79, 128, 67, 85, 205, 67, 85, 76, 84, 73, - 86, 65, 84, 73, 79, 206, 67, 85, 67, 85, 77, 66, 69, 82, 128, 67, 85, 66, - 69, 68, 128, 67, 85, 66, 69, 128, 67, 85, 66, 197, 67, 85, 65, 84, 82, - 73, 76, 76, 79, 128, 67, 85, 65, 84, 82, 73, 76, 76, 207, 67, 85, 65, - 205, 67, 83, 73, 128, 67, 82, 89, 83, 84, 65, 204, 67, 82, 89, 80, 84, - 79, 71, 82, 65, 77, 77, 73, 195, 67, 82, 89, 73, 78, 199, 67, 82, 85, 90, - 69, 73, 82, 207, 67, 82, 85, 84, 67, 72, 128, 67, 82, 85, 67, 73, 70, 79, - 82, 205, 67, 82, 85, 67, 73, 66, 76, 69, 45, 53, 128, 67, 82, 85, 67, 73, - 66, 76, 69, 45, 52, 128, 67, 82, 85, 67, 73, 66, 76, 69, 45, 51, 128, 67, - 82, 85, 67, 73, 66, 76, 69, 45, 50, 128, 67, 82, 85, 67, 73, 66, 76, 69, - 128, 67, 82, 79, 87, 78, 128, 67, 82, 79, 83, 83, 73, 78, 71, 128, 67, - 82, 79, 83, 83, 73, 78, 199, 67, 82, 79, 83, 83, 72, 65, 84, 67, 200, 67, - 82, 79, 83, 83, 69, 68, 45, 84, 65, 73, 76, 128, 67, 82, 79, 83, 83, 69, - 68, 128, 67, 82, 79, 83, 83, 69, 196, 67, 82, 79, 83, 83, 66, 79, 78, 69, - 83, 128, 67, 82, 79, 83, 83, 66, 65, 82, 128, 67, 82, 79, 83, 83, 128, - 67, 82, 79, 83, 211, 67, 82, 79, 80, 128, 67, 82, 79, 73, 88, 128, 67, - 82, 79, 73, 83, 83, 65, 78, 84, 128, 67, 82, 79, 67, 85, 211, 67, 82, 79, - 67, 79, 68, 73, 76, 69, 128, 67, 82, 73, 67, 75, 69, 84, 128, 67, 82, 73, - 67, 75, 69, 212, 67, 82, 69, 83, 67, 69, 78, 84, 83, 128, 67, 82, 69, 83, - 67, 69, 78, 84, 128, 67, 82, 69, 83, 67, 69, 78, 212, 67, 82, 69, 68, 73, - 212, 67, 82, 69, 65, 84, 73, 86, 197, 67, 82, 69, 65, 77, 128, 67, 82, - 65, 89, 79, 78, 128, 67, 82, 65, 66, 128, 67, 82, 128, 67, 79, 88, 128, - 67, 79, 87, 66, 79, 217, 67, 79, 87, 128, 67, 79, 215, 67, 79, 86, 69, - 82, 73, 78, 199, 67, 79, 86, 69, 82, 128, 67, 79, 85, 80, 76, 197, 67, - 79, 85, 78, 84, 73, 78, 199, 67, 79, 85, 78, 84, 69, 82, 83, 73, 78, 75, - 128, 67, 79, 85, 78, 84, 69, 82, 66, 79, 82, 69, 128, 67, 79, 85, 78, 67, - 73, 204, 67, 79, 85, 67, 200, 67, 79, 84, 128, 67, 79, 82, 82, 69, 83, - 80, 79, 78, 68, 211, 67, 79, 82, 82, 69, 67, 84, 128, 67, 79, 82, 80, 83, - 69, 128, 67, 79, 82, 80, 79, 82, 65, 84, 73, 79, 78, 128, 67, 79, 82, 79, - 78, 73, 83, 128, 67, 79, 82, 78, 73, 83, 200, 67, 79, 82, 78, 69, 82, 83, - 128, 67, 79, 82, 78, 69, 82, 128, 67, 79, 82, 78, 69, 210, 67, 79, 82, - 75, 128, 67, 79, 82, 65, 76, 128, 67, 79, 80, 89, 82, 73, 71, 72, 84, - 128, 67, 79, 80, 89, 82, 73, 71, 72, 212, 67, 79, 80, 89, 76, 69, 70, - 212, 67, 79, 80, 89, 128, 67, 79, 80, 82, 79, 68, 85, 67, 84, 128, 67, - 79, 80, 80, 69, 82, 45, 50, 128, 67, 79, 80, 80, 69, 82, 128, 67, 79, 80, - 128, 67, 79, 79, 76, 128, 67, 79, 79, 75, 73, 78, 71, 128, 67, 79, 79, - 75, 73, 69, 128, 67, 79, 79, 75, 69, 196, 67, 79, 79, 128, 67, 79, 78, - 86, 69, 82, 71, 73, 78, 199, 67, 79, 78, 86, 69, 78, 73, 69, 78, 67, 197, - 67, 79, 78, 84, 82, 79, 76, 128, 67, 79, 78, 84, 82, 79, 204, 67, 79, 78, - 84, 82, 65, 82, 73, 69, 84, 89, 128, 67, 79, 78, 84, 82, 65, 67, 84, 73, - 79, 78, 128, 67, 79, 78, 84, 79, 85, 82, 69, 196, 67, 79, 78, 84, 79, 85, - 210, 67, 79, 78, 84, 73, 78, 85, 73, 78, 199, 67, 79, 78, 84, 73, 78, 85, - 65, 84, 73, 79, 206, 67, 79, 78, 84, 69, 78, 84, 73, 79, 78, 128, 67, 79, - 78, 84, 69, 77, 80, 76, 65, 84, 73, 79, 78, 128, 67, 79, 78, 84, 65, 73, - 78, 211, 67, 79, 78, 84, 65, 73, 78, 73, 78, 199, 67, 79, 78, 84, 65, 73, - 206, 67, 79, 78, 84, 65, 67, 84, 128, 67, 79, 78, 83, 84, 82, 85, 67, 84, - 73, 79, 78, 128, 67, 79, 78, 83, 84, 82, 85, 67, 84, 73, 79, 206, 67, 79, - 78, 83, 84, 65, 78, 84, 128, 67, 79, 78, 83, 84, 65, 78, 212, 67, 79, 78, - 83, 84, 65, 78, 67, 89, 128, 67, 79, 78, 83, 69, 67, 85, 84, 73, 86, 197, - 67, 79, 78, 74, 85, 78, 67, 84, 73, 79, 78, 128, 67, 79, 78, 74, 85, 71, - 65, 84, 197, 67, 79, 78, 74, 79, 73, 78, 73, 78, 199, 67, 79, 78, 74, 79, - 73, 78, 69, 82, 128, 67, 79, 78, 74, 79, 73, 78, 69, 68, 128, 67, 79, 78, - 74, 79, 73, 78, 69, 196, 67, 79, 78, 73, 67, 65, 204, 67, 79, 78, 71, 82, - 85, 69, 78, 212, 67, 79, 78, 71, 82, 65, 84, 85, 76, 65, 84, 73, 79, 78, - 128, 67, 79, 78, 70, 85, 83, 69, 196, 67, 79, 78, 70, 79, 85, 78, 68, 69, - 196, 67, 79, 78, 70, 76, 73, 67, 84, 128, 67, 79, 78, 70, 69, 84, 84, - 201, 67, 79, 78, 67, 65, 86, 69, 45, 83, 73, 68, 69, 196, 67, 79, 78, 67, - 65, 86, 69, 45, 80, 79, 73, 78, 84, 69, 196, 67, 79, 77, 80, 85, 84, 69, - 82, 83, 128, 67, 79, 77, 80, 85, 84, 69, 82, 128, 67, 79, 77, 80, 82, 69, - 83, 83, 73, 79, 78, 128, 67, 79, 77, 80, 82, 69, 83, 83, 69, 196, 67, 79, - 77, 80, 79, 83, 73, 84, 73, 79, 78, 128, 67, 79, 77, 80, 79, 83, 73, 84, - 73, 79, 206, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 54, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 54, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 54, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 54, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 54, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 54, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 54, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 54, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 54, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 53, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 53, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 53, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 53, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 53, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 53, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 53, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 53, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 53, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 53, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 52, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 52, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 52, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 52, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 52, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 52, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 52, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 52, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 52, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 52, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 51, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 51, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 51, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 51, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 51, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 51, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 51, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 51, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 51, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 51, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 50, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 50, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 50, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 50, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 50, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 50, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 50, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 50, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 50, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 50, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 49, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 49, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 49, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 49, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 49, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 49, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 49, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 49, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 49, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 49, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 48, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 55, 48, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 55, 48, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, - 48, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 48, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 48, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 55, 48, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 55, 48, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 48, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 55, 48, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 57, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 57, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 57, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 57, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 57, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 57, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 57, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 57, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 57, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 57, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 56, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 56, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 56, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 56, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 56, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 56, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 56, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 56, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 56, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 56, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 55, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 55, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 55, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 55, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 55, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 55, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 55, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 55, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 55, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 55, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 54, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 54, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 54, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 54, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 54, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 54, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 54, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 54, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 54, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 54, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 53, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 53, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 53, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 53, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 53, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 53, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 53, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 53, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 53, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 53, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 52, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 52, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 52, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 52, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 52, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 52, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 52, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 52, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 52, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 52, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 51, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 51, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 51, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 51, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 51, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 51, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 51, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 51, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 51, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 51, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 50, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 50, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 50, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 50, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 50, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 50, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 50, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 50, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 50, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 50, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 49, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 49, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 49, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 49, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 49, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 49, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 49, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 49, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 49, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 49, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 48, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 54, 48, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, - 48, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 48, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 48, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 54, 48, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 54, 48, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 48, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 54, 48, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 54, 48, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 57, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 57, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 57, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 57, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 57, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 57, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 57, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 57, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 57, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 57, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 56, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 56, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 56, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 56, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 56, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 56, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 56, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 56, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 56, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 56, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 55, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 55, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 55, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 55, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 55, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 55, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 55, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 55, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 55, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 55, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 54, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 54, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 54, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 54, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 54, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 54, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 54, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 54, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 54, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 54, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 53, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 53, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 53, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 53, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 53, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 53, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 53, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 53, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 53, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 53, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 52, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 52, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 52, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 52, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 52, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 52, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 52, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 52, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 52, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 52, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 51, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 51, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 51, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 51, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 51, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 51, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 51, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 51, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 51, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 51, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 50, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 50, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 50, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 50, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 50, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 50, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 50, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 50, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 50, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 50, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 49, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 49, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 49, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 49, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 49, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 49, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 49, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 49, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 49, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 49, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 48, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, - 48, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 48, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 48, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 53, 48, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 53, 48, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 48, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 53, 48, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 53, 48, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 53, 48, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 57, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 57, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 57, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 57, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 57, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 57, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 57, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 57, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 57, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 57, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 56, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 56, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 56, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 56, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 56, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 56, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 56, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 56, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 56, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 56, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 55, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 55, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 55, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 55, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 55, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 55, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 55, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 55, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 55, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 55, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 54, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 54, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 54, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 54, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 54, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 54, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 54, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 54, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 54, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 54, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 53, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 53, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 53, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 53, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 53, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 53, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 53, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 53, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 53, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 53, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 52, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 52, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 52, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 52, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 52, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 52, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 52, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 52, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 52, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 52, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 51, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 51, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 51, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 51, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 51, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 51, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 51, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 51, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 51, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 51, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 50, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 50, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 50, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 50, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 50, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 50, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 50, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 50, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 50, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 50, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 49, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 49, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 49, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 49, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 49, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 49, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 49, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 49, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 49, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 49, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 48, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 48, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 48, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 52, 48, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 52, 48, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 48, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, 48, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 52, 48, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 52, 48, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 52, - 48, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 57, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 57, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 57, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 57, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 57, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 57, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 57, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 57, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 57, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 57, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 56, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 56, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 56, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 56, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 56, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 56, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 56, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 56, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 56, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 56, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 55, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 55, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 55, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 55, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 55, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 55, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 55, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 55, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 55, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 55, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 54, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 54, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 54, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 54, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 54, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 54, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 54, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 54, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 54, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 54, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 53, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 53, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 53, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 53, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 53, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 53, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 53, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 53, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 53, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 53, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 52, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 52, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 52, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 52, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 52, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 52, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 52, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 52, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 52, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 52, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 51, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 51, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 51, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 51, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 51, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 51, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 51, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 51, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 51, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 51, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 50, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 50, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 50, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 50, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 50, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 50, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 50, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 50, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 50, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 50, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 49, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 49, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 49, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 49, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 49, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 49, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 49, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 49, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 49, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 49, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 48, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 48, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 51, 48, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 51, 48, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 48, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 48, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 51, 48, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 51, 48, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, - 48, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 51, 48, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 57, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 57, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 57, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 57, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 57, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 57, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 57, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 57, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 57, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 57, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 56, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 56, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 56, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 56, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 56, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 56, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 56, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 56, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 56, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 56, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 55, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 55, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 55, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 55, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 55, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 55, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 55, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 55, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 55, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 55, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 54, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 54, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 54, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 54, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 54, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 54, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 54, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 54, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 54, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 54, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 53, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 53, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 53, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 53, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 53, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 53, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 53, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 53, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 53, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 53, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 52, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 52, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 52, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 52, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 52, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 52, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 52, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 52, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 52, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 52, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 51, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 51, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 51, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 51, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 51, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 51, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 51, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 51, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 51, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 51, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 50, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 50, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 50, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 50, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 50, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 50, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 50, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 50, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 50, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 50, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 49, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 49, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 49, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 49, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 49, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 49, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 49, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 49, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 49, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 49, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 48, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 50, 48, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 50, 48, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 48, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 48, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 50, 48, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 50, 48, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, - 48, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 48, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 50, 48, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 57, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 57, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 57, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 57, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 57, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 57, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 57, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 57, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 57, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 57, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 56, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 56, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 56, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 56, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 56, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 56, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 56, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 56, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 56, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 56, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 55, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 55, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 55, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 55, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 55, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 55, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 55, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 55, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 55, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 55, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 54, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 54, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 54, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 54, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 54, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 54, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 54, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 54, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 54, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 54, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 53, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 53, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 53, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 53, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 53, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 53, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 53, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 53, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 53, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 53, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 52, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 52, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 52, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 52, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 52, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 52, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 52, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 52, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 52, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 52, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 51, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 51, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 51, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 51, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 51, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 51, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 51, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 51, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 51, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 51, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 50, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 50, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 50, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 50, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 50, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 50, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 50, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 50, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 50, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 50, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 49, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 49, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 49, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 49, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 49, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 49, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 49, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 49, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 49, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 49, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 48, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 49, 48, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 48, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 48, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 49, 48, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 49, 48, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, - 48, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 48, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 49, 48, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 49, 48, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 57, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 57, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 57, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 57, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 57, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 57, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 57, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 57, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 57, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 57, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 56, 57, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 56, 56, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 56, 55, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 56, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 56, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 56, 52, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 56, 51, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 56, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 56, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 56, 48, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 55, 57, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 55, 56, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 55, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 55, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 55, 53, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 55, 52, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 55, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 55, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 55, 49, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 55, 48, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 54, 57, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 54, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 54, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 54, 54, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 54, 53, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 54, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 54, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 54, 50, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 54, 49, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 54, 48, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 53, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 53, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 53, 55, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 53, 54, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 53, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 53, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 53, 51, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 53, 50, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 53, 49, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 53, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 52, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 52, 56, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 52, 55, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 52, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 52, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 52, 52, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 52, 51, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 52, 50, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 52, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 52, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 51, 57, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 51, 56, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 51, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 51, 54, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 51, 53, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 51, 52, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 51, 51, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 51, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 51, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 51, 48, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 50, 57, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 50, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 50, 55, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 50, 54, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 50, 53, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 50, 52, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 50, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 50, 50, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 50, 49, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 50, 48, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 49, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 49, 56, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 49, 55, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 49, 54, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 49, 53, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 49, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 49, 51, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 49, 50, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 49, 49, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 49, 48, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, - 45, 48, 48, 57, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 48, 56, - 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 48, 55, 128, 67, 79, 77, - 80, 79, 78, 69, 78, 84, 45, 48, 48, 54, 128, 67, 79, 77, 80, 79, 78, 69, - 78, 84, 45, 48, 48, 53, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, - 48, 52, 128, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 48, 51, 128, 67, - 79, 77, 80, 79, 78, 69, 78, 84, 45, 48, 48, 50, 128, 67, 79, 77, 80, 79, - 78, 69, 78, 84, 45, 48, 48, 49, 128, 67, 79, 77, 80, 79, 78, 69, 78, 212, - 67, 79, 77, 80, 76, 73, 65, 78, 67, 69, 128, 67, 79, 77, 80, 76, 69, 84, - 73, 79, 78, 128, 67, 79, 77, 80, 76, 69, 84, 69, 68, 128, 67, 79, 77, 80, - 76, 69, 77, 69, 78, 84, 128, 67, 79, 77, 80, 65, 83, 83, 128, 67, 79, 77, - 80, 65, 82, 69, 128, 67, 79, 77, 77, 79, 206, 67, 79, 77, 77, 69, 82, 67, - 73, 65, 204, 67, 79, 77, 77, 65, 78, 68, 128, 67, 79, 77, 77, 65, 128, - 67, 79, 77, 77, 193, 67, 79, 77, 69, 84, 128, 67, 79, 77, 66, 73, 78, 69, - 68, 128, 67, 79, 77, 66, 73, 78, 65, 84, 73, 79, 78, 128, 67, 79, 77, 66, - 128, 67, 79, 76, 85, 77, 78, 128, 67, 79, 76, 79, 82, 128, 67, 79, 76, - 76, 73, 83, 73, 79, 206, 67, 79, 76, 76, 128, 67, 79, 76, 196, 67, 79, - 73, 78, 128, 67, 79, 70, 70, 73, 78, 128, 67, 79, 69, 78, 71, 128, 67, - 79, 69, 78, 199, 67, 79, 68, 65, 128, 67, 79, 67, 79, 78, 85, 84, 128, - 67, 79, 67, 75, 84, 65, 73, 204, 67, 79, 67, 75, 82, 79, 65, 67, 72, 128, - 67, 79, 65, 84, 128, 67, 79, 65, 83, 84, 69, 82, 128, 67, 79, 65, 128, - 67, 77, 51, 48, 50, 128, 67, 77, 51, 48, 49, 128, 67, 77, 49, 49, 52, - 128, 67, 77, 49, 49, 50, 128, 67, 77, 49, 49, 48, 128, 67, 77, 49, 48, - 57, 128, 67, 77, 49, 48, 56, 128, 67, 77, 49, 48, 55, 128, 67, 77, 49, - 48, 53, 128, 67, 77, 49, 48, 52, 128, 67, 77, 49, 48, 51, 128, 67, 77, - 49, 48, 50, 128, 67, 77, 49, 48, 49, 128, 67, 77, 49, 48, 48, 128, 67, - 77, 48, 57, 57, 128, 67, 77, 48, 57, 56, 128, 67, 77, 48, 57, 55, 128, - 67, 77, 48, 57, 54, 128, 67, 77, 48, 57, 53, 128, 67, 77, 48, 57, 52, - 128, 67, 77, 48, 57, 50, 128, 67, 77, 48, 57, 49, 128, 67, 77, 48, 57, - 48, 128, 67, 77, 48, 56, 57, 128, 67, 77, 48, 56, 56, 128, 67, 77, 48, - 56, 55, 128, 67, 77, 48, 56, 54, 128, 67, 77, 48, 56, 53, 128, 67, 77, - 48, 56, 52, 128, 67, 77, 48, 56, 51, 128, 67, 77, 48, 56, 50, 128, 67, - 77, 48, 56, 49, 128, 67, 77, 48, 56, 48, 128, 67, 77, 48, 55, 57, 128, - 67, 77, 48, 55, 56, 128, 67, 77, 48, 55, 54, 128, 67, 77, 48, 55, 53, 66, - 128, 67, 77, 48, 55, 53, 128, 67, 77, 48, 55, 52, 128, 67, 77, 48, 55, - 51, 128, 67, 77, 48, 55, 50, 128, 67, 77, 48, 55, 49, 128, 67, 77, 48, - 55, 48, 128, 67, 77, 48, 54, 57, 128, 67, 77, 48, 54, 56, 128, 67, 77, - 48, 54, 55, 128, 67, 77, 48, 54, 54, 128, 67, 77, 48, 54, 52, 128, 67, - 77, 48, 54, 51, 128, 67, 77, 48, 54, 50, 128, 67, 77, 48, 54, 49, 128, - 67, 77, 48, 54, 48, 128, 67, 77, 48, 53, 57, 128, 67, 77, 48, 53, 56, - 128, 67, 77, 48, 53, 54, 128, 67, 77, 48, 53, 53, 128, 67, 77, 48, 53, - 52, 128, 67, 77, 48, 53, 51, 128, 67, 77, 48, 53, 50, 128, 67, 77, 48, - 53, 49, 128, 67, 77, 48, 53, 48, 128, 67, 77, 48, 52, 57, 128, 67, 77, - 48, 52, 55, 128, 67, 77, 48, 52, 54, 128, 67, 77, 48, 52, 52, 128, 67, - 77, 48, 52, 49, 128, 67, 77, 48, 52, 48, 128, 67, 77, 48, 51, 57, 128, - 67, 77, 48, 51, 56, 128, 67, 77, 48, 51, 55, 128, 67, 77, 48, 51, 54, - 128, 67, 77, 48, 51, 53, 128, 67, 77, 48, 51, 52, 128, 67, 77, 48, 51, - 51, 128, 67, 77, 48, 51, 48, 128, 67, 77, 48, 50, 57, 128, 67, 77, 48, - 50, 56, 128, 67, 77, 48, 50, 55, 128, 67, 77, 48, 50, 54, 128, 67, 77, - 48, 50, 53, 128, 67, 77, 48, 50, 52, 128, 67, 77, 48, 50, 51, 128, 67, - 77, 48, 50, 49, 128, 67, 77, 48, 49, 57, 128, 67, 77, 48, 49, 55, 128, - 67, 77, 48, 49, 53, 128, 67, 77, 48, 49, 51, 128, 67, 77, 48, 49, 50, 66, - 128, 67, 77, 48, 49, 50, 128, 67, 77, 48, 49, 49, 128, 67, 77, 48, 49, - 48, 128, 67, 77, 48, 48, 57, 128, 67, 77, 48, 48, 56, 128, 67, 77, 48, - 48, 55, 128, 67, 77, 48, 48, 54, 128, 67, 77, 48, 48, 53, 128, 67, 77, - 48, 48, 52, 128, 67, 77, 48, 48, 50, 128, 67, 77, 48, 48, 49, 128, 67, - 77, 128, 67, 205, 67, 76, 85, 83, 84, 69, 82, 45, 73, 78, 73, 84, 73, 65, - 204, 67, 76, 85, 83, 84, 69, 82, 45, 70, 73, 78, 65, 204, 67, 76, 85, 83, - 84, 69, 210, 67, 76, 85, 66, 83, 128, 67, 76, 85, 66, 45, 83, 80, 79, 75, - 69, 196, 67, 76, 85, 66, 128, 67, 76, 85, 194, 67, 76, 79, 87, 206, 67, - 76, 79, 86, 69, 82, 128, 67, 76, 79, 85, 68, 128, 67, 76, 79, 85, 196, - 67, 76, 79, 84, 72, 69, 83, 128, 67, 76, 79, 84, 72, 128, 67, 76, 79, 83, - 69, 84, 128, 67, 76, 79, 83, 69, 78, 69, 83, 83, 128, 67, 76, 79, 83, 69, - 68, 128, 67, 76, 79, 83, 197, 67, 76, 79, 67, 75, 87, 73, 83, 197, 67, - 76, 79, 67, 203, 67, 76, 73, 86, 73, 83, 128, 67, 76, 73, 80, 66, 79, 65, - 82, 68, 128, 67, 76, 73, 78, 75, 73, 78, 199, 67, 76, 73, 78, 71, 73, 78, - 199, 67, 76, 73, 77, 66, 73, 78, 71, 128, 67, 76, 73, 77, 65, 67, 85, 83, - 128, 67, 76, 73, 70, 70, 128, 67, 76, 73, 67, 75, 128, 67, 76, 73, 67, - 203, 67, 76, 69, 70, 45, 50, 128, 67, 76, 69, 70, 45, 49, 128, 67, 76, - 69, 70, 128, 67, 76, 69, 198, 67, 76, 69, 65, 86, 69, 82, 128, 67, 76, - 69, 65, 210, 67, 76, 65, 83, 83, 73, 67, 65, 204, 67, 76, 65, 80, 80, 73, - 78, 199, 67, 76, 65, 80, 80, 69, 210, 67, 76, 65, 78, 128, 67, 76, 65, - 206, 67, 76, 65, 77, 83, 72, 69, 76, 204, 67, 76, 65, 73, 77, 128, 67, - 76, 128, 67, 73, 88, 128, 67, 73, 86, 73, 76, 73, 65, 78, 128, 67, 73, - 84, 89, 83, 67, 65, 80, 69, 128, 67, 73, 84, 89, 83, 67, 65, 80, 197, 67, - 73, 84, 201, 67, 73, 84, 65, 84, 73, 79, 206, 67, 73, 84, 128, 67, 73, - 82, 67, 85, 211, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 128, 67, 73, 82, - 67, 85, 77, 70, 76, 69, 216, 67, 73, 82, 67, 85, 76, 65, 84, 73, 79, 206, - 67, 73, 82, 67, 76, 73, 78, 71, 128, 67, 73, 82, 67, 76, 73, 78, 199, 67, - 73, 82, 67, 76, 69, 83, 128, 67, 73, 82, 67, 76, 69, 211, 67, 73, 82, 67, - 76, 69, 68, 128, 67, 73, 80, 128, 67, 73, 78, 78, 65, 66, 65, 82, 128, - 67, 73, 78, 69, 77, 65, 128, 67, 73, 206, 67, 73, 77, 128, 67, 73, 205, - 67, 73, 73, 128, 67, 73, 69, 88, 128, 67, 73, 69, 85, 67, 45, 83, 83, 65, - 78, 71, 80, 73, 69, 85, 80, 128, 67, 73, 69, 85, 67, 45, 80, 73, 69, 85, - 80, 128, 67, 73, 69, 85, 67, 45, 73, 69, 85, 78, 71, 128, 67, 73, 69, 85, - 195, 67, 73, 69, 84, 128, 67, 73, 69, 80, 128, 67, 73, 69, 128, 67, 72, - 89, 88, 128, 67, 72, 89, 84, 128, 67, 72, 89, 82, 88, 128, 67, 72, 89, - 82, 128, 67, 72, 89, 80, 128, 67, 72, 87, 86, 128, 67, 72, 85, 88, 128, - 67, 72, 85, 82, 88, 128, 67, 72, 85, 82, 67, 72, 128, 67, 72, 85, 82, - 128, 67, 72, 85, 80, 128, 67, 72, 85, 79, 88, 128, 67, 72, 85, 79, 84, - 128, 67, 72, 85, 79, 80, 128, 67, 72, 85, 79, 128, 67, 72, 85, 76, 65, - 128, 67, 72, 85, 128, 67, 72, 82, 89, 83, 65, 78, 84, 72, 69, 77, 85, 77, - 128, 67, 72, 82, 79, 78, 79, 85, 128, 67, 72, 82, 79, 78, 79, 78, 128, - 67, 72, 82, 79, 77, 193, 67, 72, 82, 79, 193, 67, 72, 82, 73, 86, 73, - 128, 67, 72, 82, 73, 83, 84, 77, 65, 83, 128, 67, 72, 82, 73, 83, 84, 77, - 65, 211, 67, 72, 79, 89, 128, 67, 72, 79, 88, 128, 67, 72, 79, 84, 128, - 67, 72, 79, 82, 69, 86, 77, 193, 67, 72, 79, 82, 65, 83, 77, 73, 65, 206, - 67, 72, 79, 80, 83, 84, 73, 67, 75, 83, 128, 67, 72, 79, 80, 128, 67, 72, - 79, 75, 69, 128, 67, 72, 79, 69, 128, 67, 72, 79, 67, 79, 76, 65, 84, - 197, 67, 72, 79, 65, 128, 67, 72, 73, 84, 85, 69, 85, 77, 83, 83, 65, 78, - 71, 83, 73, 79, 83, 128, 67, 72, 73, 84, 85, 69, 85, 77, 83, 83, 65, 78, - 71, 67, 73, 69, 85, 67, 128, 67, 72, 73, 84, 85, 69, 85, 77, 83, 73, 79, - 83, 128, 67, 72, 73, 84, 85, 69, 85, 77, 67, 73, 69, 85, 67, 128, 67, 72, - 73, 84, 85, 69, 85, 77, 67, 72, 73, 69, 85, 67, 72, 128, 67, 72, 73, 82, - 79, 78, 128, 67, 72, 73, 82, 69, 84, 128, 67, 72, 73, 80, 77, 85, 78, 75, - 128, 67, 72, 73, 78, 79, 79, 203, 67, 72, 73, 78, 71, 128, 67, 72, 73, - 78, 69, 83, 197, 67, 72, 73, 78, 128, 67, 72, 73, 77, 69, 128, 67, 72, - 73, 77, 128, 67, 72, 73, 76, 76, 213, 67, 72, 73, 76, 68, 82, 69, 206, - 67, 72, 73, 76, 68, 128, 67, 72, 73, 76, 128, 67, 72, 73, 75, 201, 67, - 72, 73, 69, 85, 67, 72, 45, 75, 72, 73, 69, 85, 75, 72, 128, 67, 72, 73, - 69, 85, 67, 72, 45, 72, 73, 69, 85, 72, 128, 67, 72, 73, 69, 85, 67, 200, - 67, 72, 73, 67, 75, 69, 78, 128, 67, 72, 73, 67, 75, 128, 67, 72, 73, - 128, 67, 72, 201, 67, 72, 72, 73, 77, 128, 67, 72, 72, 65, 128, 67, 72, - 69, 88, 128, 67, 72, 69, 86, 82, 79, 78, 128, 67, 72, 69, 86, 82, 79, - 206, 67, 72, 69, 84, 128, 67, 72, 69, 83, 84, 78, 85, 84, 128, 67, 72, - 69, 83, 84, 128, 67, 72, 69, 83, 211, 67, 72, 69, 82, 89, 128, 67, 72, - 69, 82, 82, 217, 67, 72, 69, 82, 82, 73, 69, 83, 128, 67, 72, 69, 81, 85, - 69, 82, 69, 196, 67, 72, 69, 80, 128, 67, 72, 69, 76, 89, 85, 83, 84, 75, - 65, 128, 67, 72, 69, 76, 78, 85, 128, 67, 72, 69, 73, 78, 65, 80, 128, - 67, 72, 69, 73, 75, 72, 69, 73, 128, 67, 72, 69, 73, 75, 72, 65, 78, 128, - 67, 72, 69, 69, 83, 197, 67, 72, 69, 69, 82, 73, 78, 199, 67, 72, 69, 69, - 77, 128, 67, 72, 69, 69, 75, 211, 67, 72, 69, 69, 75, 128, 67, 72, 69, - 69, 128, 67, 72, 69, 67, 75, 69, 210, 67, 72, 69, 67, 75, 128, 67, 72, - 69, 67, 203, 67, 72, 197, 67, 72, 65, 88, 128, 67, 72, 65, 86, 73, 89, - 65, 78, 73, 128, 67, 72, 65, 84, 84, 65, 87, 65, 128, 67, 72, 65, 84, - 128, 67, 72, 65, 83, 72, 75, 65, 128, 67, 72, 65, 83, 72, 75, 193, 67, - 72, 65, 82, 84, 128, 67, 72, 65, 82, 212, 67, 72, 65, 82, 73, 79, 84, - 128, 67, 72, 65, 82, 73, 79, 212, 67, 72, 65, 82, 65, 67, 84, 69, 82, 83, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 70, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 70, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 70, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 70, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 70, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 70, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 69, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 69, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 69, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 69, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 69, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 69, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 69, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 69, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 69, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 69, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 69, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 68, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 68, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 68, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 68, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 68, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 68, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 68, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 68, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 68, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 68, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 68, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 67, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 67, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 67, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 67, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 67, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 67, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 67, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 67, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 67, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 66, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 66, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 66, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 66, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 66, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 66, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 66, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 66, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 66, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 65, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 65, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 65, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 65, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 65, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 65, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 65, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 65, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 65, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 65, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 57, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 57, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 57, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 57, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 57, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 57, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 57, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 57, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 57, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 57, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 57, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 70, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 69, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 68, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 56, 67, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 56, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 56, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 56, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 56, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 55, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 54, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 53, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 56, 52, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 56, 51, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 56, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 56, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 56, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 55, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 69, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 68, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 67, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 66, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 55, 65, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 55, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 55, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 55, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 55, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 53, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 52, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 51, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 55, 50, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 55, 49, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 55, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 54, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 54, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 54, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 67, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 54, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 54, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 54, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 54, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 54, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 54, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 53, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 53, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 53, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 53, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 53, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 53, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 53, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 53, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 53, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 53, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 53, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 52, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 52, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 52, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 52, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 52, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 52, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 52, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 52, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 52, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 52, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 52, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 51, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 51, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 51, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 51, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 51, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 51, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 51, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 51, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 51, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 50, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 50, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 50, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 50, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 50, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 50, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 50, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 50, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 50, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 49, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 49, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 49, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 49, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 49, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 49, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 49, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 49, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 49, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 49, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 48, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 48, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 48, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 48, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 48, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 50, 48, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 50, 48, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 50, 48, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 50, 48, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, - 48, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 50, 48, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 70, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 69, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 68, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 70, 67, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 70, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 70, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 70, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 70, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 55, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 54, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 53, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 70, 52, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 70, 51, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 70, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 70, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 70, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 69, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 69, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 68, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 67, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 66, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 69, 65, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 69, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 69, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 69, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 69, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 53, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 52, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 51, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 69, 50, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 69, 49, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 69, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 68, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 68, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 68, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 67, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 68, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 68, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 68, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 68, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 68, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 68, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 67, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 67, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 67, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 67, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 67, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 67, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 67, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 67, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 67, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 67, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 67, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 66, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 66, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 66, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 66, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 66, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 66, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 66, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 66, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 66, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 66, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 66, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 65, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 65, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 65, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 65, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 65, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 65, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 65, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 65, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 65, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 57, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 57, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 57, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 57, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 57, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 57, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 57, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 57, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 57, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 56, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 56, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 56, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 56, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 56, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 56, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 56, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 56, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 56, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 56, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 55, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 55, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 55, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 55, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 55, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 66, 49, 55, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 66, 49, 55, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 66, 49, 55, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 66, 49, 55, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, - 55, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 49, 55, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 68, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 68, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 68, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 68, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 68, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 68, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 67, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 67, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 67, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 67, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 67, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 67, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 67, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 67, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 67, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 66, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 66, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 66, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 66, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 66, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 66, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 66, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 66, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 66, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 66, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 65, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 65, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 65, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 65, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 65, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 65, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 65, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 65, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 65, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 65, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 65, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 70, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 69, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 68, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 57, 67, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 57, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 57, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 57, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 57, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 55, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 54, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 53, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 57, 52, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 57, 51, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 57, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 57, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 57, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 56, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 69, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 68, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 67, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 66, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 56, 65, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 56, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 56, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 56, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 56, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 53, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 52, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 51, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 56, 50, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 56, 49, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 56, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 55, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 55, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 55, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 67, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 55, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 55, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 55, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 55, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 55, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 55, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 54, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 54, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 54, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 54, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 54, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 54, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 54, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 54, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 54, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 54, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 54, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 53, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 53, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 53, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 53, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 53, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 53, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 53, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 53, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 53, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 53, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 53, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 52, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 52, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 52, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 52, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 52, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 52, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 52, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 52, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 52, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 51, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 51, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 51, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 51, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 51, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 51, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 51, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 51, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 51, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 50, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 50, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 50, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 50, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 50, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 50, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 50, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 50, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 50, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 50, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 49, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 49, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 49, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 49, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 49, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 49, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 49, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 49, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 49, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 49, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 49, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 70, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 69, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 68, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 48, 67, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 48, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 48, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 48, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, - 48, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 55, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 54, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 53, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 67, 48, 52, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 67, 48, 51, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 67, 48, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 67, 48, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 67, 48, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 70, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 69, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 68, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 67, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 66, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 70, 65, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 70, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 70, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 70, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 70, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 53, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 52, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 51, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 70, 50, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 70, 49, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 70, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 69, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 69, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 69, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 67, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 69, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 69, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 69, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 69, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 69, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 69, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 68, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 68, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 68, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 68, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 68, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 68, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 68, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 68, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 68, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 68, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 68, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 67, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 67, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 67, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 67, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 67, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 67, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 67, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 67, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 67, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 67, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 67, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 66, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 66, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 66, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 66, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 66, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 66, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 66, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 66, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 66, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 65, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 65, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 65, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 65, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 65, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 65, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 65, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 65, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 65, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 57, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 57, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 57, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 57, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 57, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 57, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 57, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 57, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 57, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 57, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 70, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 56, 69, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 56, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 56, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 56, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 56, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 57, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 56, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 55, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 54, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 56, 53, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 56, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 56, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 56, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 56, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 56, 48, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 70, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 69, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 68, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 55, 67, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 55, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 55, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 55, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 55, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 55, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 54, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 53, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 55, 52, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 55, 51, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 55, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 55, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 55, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 54, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 69, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 68, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 67, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 66, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 54, 65, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 54, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 54, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 54, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 54, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 53, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 52, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 51, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 54, 50, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 54, 49, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 54, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 53, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 53, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 53, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 67, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 66, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 65, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 57, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 53, 56, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 53, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 53, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 53, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 53, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 51, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 50, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 49, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 53, 48, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 52, 70, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 52, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 52, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 52, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 52, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 65, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 57, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 56, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 55, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 52, 54, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 52, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 52, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 52, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 52, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 49, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 52, 48, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 70, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 69, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 51, 68, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 51, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 51, 66, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 51, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 51, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 56, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 55, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 54, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 51, 53, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 51, 52, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 51, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 51, 50, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 51, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 51, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 70, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 69, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 68, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 67, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 50, 66, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 50, 65, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 50, 57, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 50, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 50, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 54, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 53, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 52, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 50, 51, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 50, 50, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 50, 49, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 50, 48, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 49, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 49, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 68, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 67, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 66, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 65, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 49, 57, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 49, 56, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 49, 55, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 49, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 49, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 52, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 51, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 50, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 49, 49, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 49, 48, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 48, 70, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 48, 69, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 48, 68, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 48, 67, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 66, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 65, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 57, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 56, 128, 67, 72, 65, 82, 65, - 67, 84, 69, 82, 45, 49, 56, 66, 48, 55, 128, 67, 72, 65, 82, 65, 67, 84, - 69, 82, 45, 49, 56, 66, 48, 54, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 45, 49, 56, 66, 48, 53, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, - 56, 66, 48, 52, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, - 48, 51, 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 50, - 128, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 49, 128, 67, - 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 66, 48, 48, 128, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 128, 67, 72, 65, 82, 65, 67, 84, 69, 210, 67, 72, - 65, 82, 128, 67, 72, 65, 80, 84, 69, 82, 128, 67, 72, 65, 80, 128, 67, - 72, 65, 78, 71, 128, 67, 72, 65, 78, 128, 67, 72, 65, 77, 75, 79, 128, - 67, 72, 65, 77, 73, 76, 79, 78, 128, 67, 72, 65, 77, 73, 76, 73, 128, 67, - 72, 65, 205, 67, 72, 65, 75, 77, 193, 67, 72, 65, 73, 78, 83, 128, 67, - 72, 65, 68, 65, 128, 67, 72, 65, 196, 67, 72, 65, 65, 128, 67, 71, 74, - 128, 67, 69, 88, 128, 67, 69, 86, 73, 84, 85, 128, 67, 69, 82, 69, 83, - 128, 67, 69, 82, 69, 77, 79, 78, 89, 128, 67, 69, 82, 69, 75, 128, 67, - 69, 82, 45, 87, 65, 128, 67, 69, 80, 128, 67, 69, 79, 78, 71, 67, 72, 73, - 69, 85, 77, 83, 83, 65, 78, 71, 83, 73, 79, 83, 128, 67, 69, 79, 78, 71, - 67, 72, 73, 69, 85, 77, 83, 83, 65, 78, 71, 67, 73, 69, 85, 67, 128, 67, - 69, 79, 78, 71, 67, 72, 73, 69, 85, 77, 83, 73, 79, 83, 128, 67, 69, 79, - 78, 71, 67, 72, 73, 69, 85, 77, 67, 73, 69, 85, 67, 128, 67, 69, 79, 78, - 71, 67, 72, 73, 69, 85, 77, 67, 72, 73, 69, 85, 67, 72, 128, 67, 69, 78, - 84, 85, 82, 73, 65, 204, 67, 69, 78, 84, 82, 69, 76, 73, 78, 197, 67, 69, - 78, 84, 82, 69, 68, 128, 67, 69, 78, 84, 82, 69, 196, 67, 69, 78, 84, 82, - 69, 128, 67, 69, 78, 84, 82, 197, 67, 69, 78, 84, 82, 65, 76, 73, 90, 65, - 84, 73, 79, 206, 67, 69, 78, 128, 67, 69, 76, 84, 73, 195, 67, 69, 76, - 83, 73, 85, 83, 128, 67, 69, 76, 69, 66, 82, 65, 84, 73, 79, 78, 128, 67, - 69, 73, 82, 84, 128, 67, 69, 73, 76, 73, 78, 71, 128, 67, 69, 73, 76, 73, - 78, 199, 67, 69, 69, 86, 128, 67, 69, 69, 66, 128, 67, 69, 69, 128, 67, - 69, 68, 73, 76, 76, 65, 128, 67, 69, 68, 73, 76, 76, 193, 67, 69, 68, - 201, 67, 69, 67, 69, 75, 128, 67, 69, 67, 65, 75, 128, 67, 69, 67, 65, - 203, 67, 69, 65, 76, 67, 128, 67, 67, 85, 128, 67, 67, 79, 128, 67, 67, - 73, 128, 67, 67, 72, 85, 128, 67, 67, 72, 79, 128, 67, 67, 72, 73, 128, - 67, 67, 72, 72, 85, 128, 67, 67, 72, 72, 79, 128, 67, 67, 72, 72, 73, - 128, 67, 67, 72, 72, 69, 69, 128, 67, 67, 72, 72, 69, 128, 67, 67, 72, - 72, 65, 65, 128, 67, 67, 72, 72, 65, 128, 67, 67, 72, 69, 69, 128, 67, - 67, 72, 69, 128, 67, 67, 72, 65, 65, 128, 67, 67, 72, 65, 128, 67, 67, - 72, 128, 67, 67, 69, 69, 128, 67, 67, 65, 65, 128, 67, 65, 89, 78, 128, - 67, 65, 89, 65, 78, 78, 65, 128, 67, 65, 88, 128, 67, 65, 86, 69, 128, - 67, 65, 85, 84, 73, 79, 206, 67, 65, 85, 76, 68, 82, 79, 78, 128, 67, 65, - 85, 68, 65, 84, 197, 67, 65, 85, 68, 65, 128, 67, 65, 85, 67, 65, 83, 73, - 65, 206, 67, 65, 85, 128, 67, 65, 84, 65, 87, 65, 128, 67, 65, 84, 128, - 67, 65, 212, 67, 65, 83, 84, 76, 69, 128, 67, 65, 83, 75, 69, 212, 67, - 65, 82, 89, 83, 84, 73, 65, 206, 67, 65, 82, 84, 87, 72, 69, 69, 76, 128, - 67, 65, 82, 84, 82, 73, 68, 71, 69, 128, 67, 65, 82, 84, 128, 67, 65, 82, - 211, 67, 65, 82, 82, 79, 84, 128, 67, 65, 82, 82, 73, 65, 71, 197, 67, - 65, 82, 80, 69, 78, 84, 82, 217, 67, 65, 82, 208, 67, 65, 82, 79, 85, 83, - 69, 204, 67, 65, 82, 79, 78, 128, 67, 65, 82, 79, 206, 67, 65, 82, 73, - 203, 67, 65, 82, 73, 65, 206, 67, 65, 82, 69, 84, 128, 67, 65, 82, 69, - 212, 67, 65, 82, 197, 67, 65, 82, 68, 83, 128, 67, 65, 82, 196, 67, 65, - 82, 128, 67, 65, 210, 67, 65, 80, 85, 212, 67, 65, 80, 84, 73, 86, 69, - 128, 67, 65, 80, 82, 73, 67, 79, 82, 78, 128, 67, 65, 80, 80, 69, 196, - 67, 65, 80, 79, 128, 67, 65, 80, 73, 84, 85, 76, 85, 77, 128, 67, 65, 80, - 73, 84, 65, 76, 128, 67, 65, 78, 84, 73, 76, 76, 65, 84, 73, 79, 206, 67, - 65, 78, 79, 69, 128, 67, 65, 78, 78, 79, 78, 128, 67, 65, 78, 78, 69, - 196, 67, 65, 78, 199, 67, 65, 78, 69, 128, 67, 65, 78, 68, 89, 128, 67, - 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 128, 67, 65, 78, 68, 82, 65, 66, - 73, 78, 68, 213, 67, 65, 78, 68, 82, 65, 128, 67, 65, 78, 68, 82, 193, - 67, 65, 78, 68, 76, 69, 128, 67, 65, 78, 67, 69, 82, 128, 67, 65, 78, 67, - 69, 76, 76, 65, 84, 73, 79, 206, 67, 65, 78, 67, 69, 76, 128, 67, 65, 78, - 67, 69, 204, 67, 65, 78, 128, 67, 65, 77, 80, 73, 78, 71, 128, 67, 65, - 77, 78, 85, 195, 67, 65, 77, 69, 82, 65, 128, 67, 65, 77, 69, 82, 193, - 67, 65, 77, 69, 76, 128, 67, 65, 76, 89, 65, 128, 67, 65, 76, 89, 193, - 67, 65, 76, 88, 128, 67, 65, 76, 76, 128, 67, 65, 76, 204, 67, 65, 76, - 69, 78, 68, 65, 82, 128, 67, 65, 76, 69, 78, 68, 65, 210, 67, 65, 76, 67, - 85, 76, 65, 84, 79, 82, 128, 67, 65, 76, 67, 128, 67, 65, 75, 82, 65, - 128, 67, 65, 75, 197, 67, 65, 73, 128, 67, 65, 72, 128, 67, 65, 69, 83, - 85, 82, 65, 128, 67, 65, 68, 85, 67, 69, 85, 83, 128, 67, 65, 68, 193, - 67, 65, 67, 84, 85, 83, 128, 67, 65, 66, 76, 69, 87, 65, 89, 128, 67, 65, - 66, 73, 78, 69, 84, 128, 67, 65, 66, 66, 65, 71, 69, 45, 84, 82, 69, 69, - 128, 67, 65, 65, 78, 71, 128, 67, 65, 65, 73, 128, 67, 193, 67, 48, 50, - 52, 128, 67, 48, 50, 51, 128, 67, 48, 50, 50, 128, 67, 48, 50, 49, 128, - 67, 48, 50, 48, 128, 67, 48, 49, 57, 128, 67, 48, 49, 56, 128, 67, 48, - 49, 55, 128, 67, 48, 49, 54, 128, 67, 48, 49, 53, 128, 67, 48, 49, 52, - 128, 67, 48, 49, 51, 128, 67, 48, 49, 50, 128, 67, 48, 49, 49, 128, 67, - 48, 49, 48, 65, 128, 67, 48, 49, 48, 128, 67, 48, 48, 57, 128, 67, 48, - 48, 56, 128, 67, 48, 48, 55, 128, 67, 48, 48, 54, 128, 67, 48, 48, 53, - 128, 67, 48, 48, 52, 128, 67, 48, 48, 51, 128, 67, 48, 48, 50, 67, 128, - 67, 48, 48, 50, 66, 128, 67, 48, 48, 50, 65, 128, 67, 48, 48, 50, 128, - 67, 48, 48, 49, 128, 67, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 196, 67, - 45, 51, 57, 128, 67, 45, 49, 56, 128, 66, 90, 85, 78, 199, 66, 90, 72, - 201, 66, 89, 84, 197, 66, 89, 69, 76, 79, 82, 85, 83, 83, 73, 65, 78, 45, - 85, 75, 82, 65, 73, 78, 73, 65, 206, 66, 88, 71, 128, 66, 87, 73, 128, - 66, 87, 69, 69, 128, 66, 87, 69, 128, 66, 87, 65, 128, 66, 85, 85, 77, - 73, 83, 72, 128, 66, 85, 84, 84, 79, 78, 128, 66, 85, 84, 84, 79, 206, - 66, 85, 84, 84, 69, 82, 70, 76, 89, 128, 66, 85, 84, 84, 69, 82, 128, 66, - 85, 212, 66, 85, 83, 84, 211, 66, 85, 83, 212, 66, 85, 83, 83, 89, 69, - 82, 85, 128, 66, 85, 83, 73, 78, 69, 83, 211, 66, 85, 211, 66, 85, 82, - 213, 66, 85, 82, 82, 73, 84, 79, 128, 66, 85, 82, 50, 128, 66, 85, 210, - 66, 85, 79, 89, 128, 66, 85, 79, 88, 128, 66, 85, 79, 80, 128, 66, 85, - 78, 78, 217, 66, 85, 78, 71, 128, 66, 85, 77, 80, 217, 66, 85, 76, 85, - 71, 128, 66, 85, 76, 85, 199, 66, 85, 76, 76, 83, 69, 89, 69, 128, 66, - 85, 76, 76, 211, 66, 85, 76, 76, 72, 79, 82, 78, 128, 66, 85, 76, 76, 72, - 79, 82, 206, 66, 85, 76, 76, 69, 84, 128, 66, 85, 76, 76, 69, 212, 66, - 85, 76, 76, 128, 66, 85, 76, 66, 128, 66, 85, 75, 89, 128, 66, 85, 73, - 76, 68, 73, 78, 71, 83, 128, 66, 85, 73, 76, 68, 73, 78, 71, 128, 66, 85, - 73, 76, 68, 73, 78, 199, 66, 85, 72, 73, 196, 66, 85, 71, 73, 78, 69, 83, - 197, 66, 85, 71, 128, 66, 85, 70, 70, 65, 76, 79, 128, 66, 85, 68, 128, - 66, 85, 67, 75, 76, 69, 128, 66, 85, 67, 75, 69, 84, 128, 66, 85, 66, 66, - 76, 69, 83, 128, 66, 85, 66, 66, 76, 69, 128, 66, 85, 66, 66, 76, 197, - 66, 83, 84, 65, 82, 128, 66, 83, 75, 85, 210, 66, 83, 75, 65, 173, 66, - 83, 68, 85, 211, 66, 82, 85, 83, 200, 66, 82, 79, 87, 206, 66, 82, 79, - 79, 77, 128, 66, 82, 79, 78, 90, 69, 128, 66, 82, 79, 75, 69, 206, 66, - 82, 79, 67, 67, 79, 76, 73, 128, 66, 82, 79, 65, 196, 66, 82, 73, 83, 84, - 76, 69, 128, 66, 82, 73, 71, 72, 84, 78, 69, 83, 211, 66, 82, 73, 69, 70, - 83, 128, 66, 82, 73, 69, 70, 67, 65, 83, 69, 128, 66, 82, 73, 68, 71, - 197, 66, 82, 73, 68, 197, 66, 82, 73, 67, 75, 128, 66, 82, 73, 128, 66, - 82, 69, 86, 73, 83, 128, 66, 82, 69, 86, 69, 45, 77, 65, 67, 82, 79, 78, - 128, 66, 82, 69, 86, 197, 66, 82, 69, 65, 84, 72, 217, 66, 82, 69, 65, - 84, 200, 66, 82, 69, 65, 83, 84, 45, 70, 69, 69, 68, 73, 78, 71, 128, 66, - 82, 69, 65, 75, 84, 72, 82, 79, 85, 71, 72, 128, 66, 82, 68, 193, 66, 82, - 65, 78, 67, 72, 73, 78, 199, 66, 82, 65, 78, 67, 72, 69, 83, 128, 66, 82, - 65, 78, 67, 72, 128, 66, 82, 65, 78, 67, 200, 66, 82, 65, 75, 67, 69, 84, - 128, 66, 82, 65, 73, 78, 128, 66, 82, 65, 67, 75, 69, 84, 211, 66, 82, - 65, 67, 75, 69, 84, 69, 196, 66, 82, 65, 67, 75, 69, 84, 128, 66, 82, 65, - 67, 75, 69, 212, 66, 82, 65, 67, 69, 128, 66, 81, 128, 66, 80, 72, 128, - 66, 79, 89, 211, 66, 79, 89, 128, 66, 79, 88, 73, 78, 199, 66, 79, 87, - 84, 73, 69, 128, 66, 79, 87, 84, 73, 197, 66, 79, 87, 76, 73, 78, 71, - 128, 66, 79, 87, 76, 128, 66, 79, 87, 204, 66, 79, 87, 73, 78, 199, 66, - 79, 215, 66, 79, 85, 81, 85, 69, 84, 128, 66, 79, 85, 81, 85, 69, 212, - 66, 79, 85, 78, 68, 65, 82, 217, 66, 79, 84, 84, 79, 77, 45, 83, 72, 65, - 68, 69, 196, 66, 79, 84, 84, 79, 77, 45, 76, 73, 71, 72, 84, 69, 196, 66, - 79, 84, 84, 79, 77, 128, 66, 79, 84, 84, 79, 205, 66, 79, 84, 84, 76, 69, - 128, 66, 79, 84, 84, 76, 197, 66, 79, 84, 200, 66, 79, 82, 90, 89, 128, - 66, 79, 82, 90, 65, 89, 65, 128, 66, 79, 82, 85, 84, 79, 128, 66, 79, 82, - 65, 88, 45, 51, 128, 66, 79, 82, 65, 88, 45, 50, 128, 66, 79, 82, 65, 88, - 128, 66, 79, 80, 79, 77, 79, 70, 207, 66, 79, 79, 84, 83, 128, 66, 79, - 79, 84, 128, 66, 79, 79, 77, 69, 82, 65, 78, 71, 128, 66, 79, 79, 75, 83, - 128, 66, 79, 79, 75, 77, 65, 82, 75, 128, 66, 79, 79, 75, 77, 65, 82, - 203, 66, 79, 78, 69, 128, 66, 79, 77, 66, 128, 66, 79, 77, 128, 66, 79, - 76, 84, 128, 66, 79, 76, 212, 66, 79, 72, 65, 73, 82, 73, 195, 66, 79, - 68, 89, 128, 66, 79, 68, 217, 66, 79, 65, 82, 128, 66, 79, 65, 128, 66, - 76, 85, 69, 66, 69, 82, 82, 73, 69, 83, 128, 66, 76, 85, 69, 128, 66, 76, - 85, 197, 66, 76, 79, 87, 73, 78, 199, 66, 76, 79, 87, 70, 73, 83, 72, - 128, 66, 76, 79, 215, 66, 76, 79, 83, 83, 79, 77, 128, 66, 76, 79, 79, - 68, 128, 66, 76, 79, 78, 196, 66, 76, 79, 67, 75, 45, 55, 128, 66, 76, - 79, 67, 75, 45, 54, 128, 66, 76, 79, 67, 75, 45, 53, 128, 66, 76, 79, 67, - 75, 45, 52, 128, 66, 76, 79, 67, 75, 45, 51, 128, 66, 76, 79, 67, 75, 45, - 50, 128, 66, 76, 79, 67, 75, 45, 49, 51, 53, 56, 128, 66, 76, 79, 67, 75, - 128, 66, 76, 73, 78, 203, 66, 76, 65, 78, 75, 128, 66, 76, 65, 78, 203, - 66, 76, 65, 68, 197, 66, 76, 65, 67, 75, 76, 69, 84, 84, 69, 210, 66, 76, - 65, 67, 75, 70, 79, 79, 212, 66, 76, 65, 67, 75, 45, 76, 69, 84, 84, 69, - 210, 66, 76, 65, 67, 75, 45, 70, 69, 65, 84, 72, 69, 82, 69, 196, 66, 76, - 65, 67, 75, 128, 66, 75, 65, 173, 66, 73, 84, 84, 69, 82, 128, 66, 73, - 84, 73, 78, 199, 66, 73, 84, 197, 66, 73, 84, 67, 79, 73, 206, 66, 73, - 83, 79, 78, 128, 66, 73, 83, 77, 85, 84, 200, 66, 73, 83, 77, 73, 76, 76, - 65, 200, 66, 73, 83, 72, 79, 208, 66, 73, 83, 69, 67, 84, 73, 78, 199, - 66, 73, 83, 65, 72, 128, 66, 73, 82, 85, 128, 66, 73, 82, 84, 72, 68, 65, - 217, 66, 73, 82, 71, 65, 128, 66, 73, 82, 71, 193, 66, 73, 82, 68, 128, - 66, 73, 79, 72, 65, 90, 65, 82, 196, 66, 73, 78, 79, 86, 73, 76, 69, 128, - 66, 73, 78, 79, 67, 85, 76, 65, 210, 66, 73, 78, 68, 73, 78, 199, 66, 73, - 78, 68, 73, 128, 66, 73, 78, 65, 82, 217, 66, 73, 76, 76, 73, 79, 78, 83, - 128, 66, 73, 76, 76, 73, 65, 82, 68, 83, 128, 66, 73, 76, 76, 69, 196, - 66, 73, 76, 65, 66, 73, 65, 204, 66, 73, 75, 73, 78, 73, 128, 66, 73, 71, - 128, 66, 73, 199, 66, 73, 69, 84, 128, 66, 73, 68, 69, 78, 84, 65, 204, - 66, 73, 68, 65, 75, 85, 79, 206, 66, 73, 67, 89, 67, 76, 73, 83, 84, 128, - 66, 73, 67, 89, 67, 76, 69, 83, 128, 66, 73, 67, 89, 67, 76, 69, 128, 66, - 73, 67, 69, 80, 83, 128, 66, 73, 66, 76, 69, 45, 67, 82, 69, 197, 66, 73, - 66, 128, 66, 201, 66, 72, 85, 128, 66, 72, 79, 79, 128, 66, 72, 79, 128, - 66, 72, 73, 128, 66, 72, 69, 84, 72, 128, 66, 72, 69, 69, 128, 66, 72, - 69, 128, 66, 72, 65, 84, 84, 73, 80, 82, 79, 76, 213, 66, 72, 65, 77, - 128, 66, 72, 65, 76, 69, 128, 66, 72, 65, 76, 197, 66, 72, 65, 73, 75, - 83, 85, 75, 201, 66, 72, 65, 65, 128, 66, 72, 65, 128, 66, 69, 89, 89, - 65, 76, 128, 66, 69, 88, 128, 66, 69, 86, 69, 82, 65, 71, 69, 128, 66, - 69, 86, 69, 82, 65, 71, 197, 66, 69, 84, 87, 69, 69, 78, 128, 66, 69, 84, - 87, 69, 69, 206, 66, 69, 84, 72, 128, 66, 69, 84, 65, 128, 66, 69, 84, - 193, 66, 69, 212, 66, 69, 83, 73, 68, 197, 66, 69, 82, 75, 65, 78, 65, - 206, 66, 69, 82, 66, 69, 210, 66, 69, 80, 128, 66, 69, 79, 82, 195, 66, - 69, 78, 90, 69, 78, 197, 66, 69, 78, 84, 207, 66, 69, 78, 84, 128, 66, - 69, 78, 212, 66, 69, 78, 71, 65, 76, 201, 66, 69, 78, 68, 69, 128, 66, - 69, 78, 68, 128, 66, 69, 78, 196, 66, 69, 206, 66, 69, 76, 84, 128, 66, - 69, 76, 212, 66, 69, 76, 79, 215, 66, 69, 76, 76, 72, 79, 208, 66, 69, - 76, 76, 128, 66, 69, 76, 204, 66, 69, 76, 71, 84, 72, 79, 210, 66, 69, - 73, 84, 72, 128, 66, 69, 72, 73, 78, 196, 66, 69, 72, 69, 72, 128, 66, - 69, 72, 69, 200, 66, 69, 72, 128, 66, 69, 200, 66, 69, 71, 73, 78, 78, - 73, 78, 71, 128, 66, 69, 71, 73, 78, 78, 69, 82, 128, 66, 69, 71, 73, - 206, 66, 69, 70, 79, 82, 197, 66, 69, 69, 84, 76, 69, 128, 66, 69, 69, - 84, 65, 128, 66, 69, 69, 210, 66, 69, 69, 72, 73, 86, 69, 128, 66, 69, - 69, 72, 128, 66, 69, 69, 200, 66, 69, 67, 65, 85, 83, 69, 128, 66, 69, - 65, 86, 69, 82, 128, 66, 69, 65, 86, 69, 210, 66, 69, 65, 84, 73, 78, - 199, 66, 69, 65, 84, 128, 66, 69, 65, 82, 68, 69, 196, 66, 69, 65, 82, - 128, 66, 69, 65, 210, 66, 69, 65, 78, 83, 128, 66, 69, 65, 78, 128, 66, - 69, 65, 77, 69, 196, 66, 69, 65, 68, 83, 128, 66, 69, 65, 67, 200, 66, - 67, 65, 68, 128, 66, 67, 65, 196, 66, 66, 89, 88, 128, 66, 66, 89, 84, - 128, 66, 66, 89, 80, 128, 66, 66, 89, 128, 66, 66, 85, 88, 128, 66, 66, - 85, 84, 128, 66, 66, 85, 82, 88, 128, 66, 66, 85, 82, 128, 66, 66, 85, - 80, 128, 66, 66, 85, 79, 88, 128, 66, 66, 85, 79, 80, 128, 66, 66, 85, - 79, 128, 66, 66, 85, 128, 66, 66, 79, 88, 128, 66, 66, 79, 84, 128, 66, - 66, 79, 80, 128, 66, 66, 79, 128, 66, 66, 73, 88, 128, 66, 66, 73, 80, - 128, 66, 66, 73, 69, 88, 128, 66, 66, 73, 69, 84, 128, 66, 66, 73, 69, - 80, 128, 66, 66, 73, 69, 128, 66, 66, 73, 128, 66, 66, 69, 88, 128, 66, - 66, 69, 80, 128, 66, 66, 69, 69, 128, 66, 66, 65, 88, 128, 66, 66, 65, - 84, 128, 66, 66, 65, 80, 128, 66, 66, 65, 65, 128, 66, 65, 89, 65, 78, - 78, 65, 128, 66, 65, 85, 128, 66, 65, 84, 84, 69, 82, 89, 128, 66, 65, - 84, 72, 84, 85, 66, 128, 66, 65, 84, 72, 65, 77, 65, 83, 65, 84, 128, 66, - 65, 84, 72, 128, 66, 65, 84, 200, 66, 65, 84, 65, 203, 66, 65, 83, 83, - 65, 128, 66, 65, 83, 83, 193, 66, 65, 83, 75, 69, 84, 66, 65, 76, 204, - 66, 65, 83, 72, 75, 73, 210, 66, 65, 83, 72, 128, 66, 65, 83, 69, 76, 73, - 78, 197, 66, 65, 83, 69, 66, 65, 76, 76, 128, 66, 65, 83, 69, 128, 66, - 65, 83, 197, 66, 65, 82, 83, 128, 66, 65, 82, 211, 66, 65, 82, 82, 73, - 69, 82, 128, 66, 65, 82, 82, 69, 75, 72, 128, 66, 65, 82, 82, 69, 69, - 128, 66, 65, 82, 82, 69, 197, 66, 65, 82, 76, 73, 78, 69, 128, 66, 65, - 82, 76, 69, 89, 128, 66, 65, 82, 73, 89, 79, 79, 83, 65, 78, 128, 66, 65, - 82, 66, 69, 210, 66, 65, 82, 65, 50, 128, 66, 65, 210, 66, 65, 78, 84, - 79, 67, 128, 66, 65, 78, 75, 78, 79, 84, 197, 66, 65, 78, 75, 128, 66, - 65, 78, 203, 66, 65, 78, 74, 79, 128, 66, 65, 78, 68, 128, 66, 65, 78, - 65, 78, 65, 128, 66, 65, 78, 50, 128, 66, 65, 78, 178, 66, 65, 77, 66, - 79, 79, 83, 128, 66, 65, 77, 66, 79, 79, 128, 66, 65, 76, 85, 68, 65, - 128, 66, 65, 76, 76, 80, 79, 73, 78, 212, 66, 65, 76, 76, 79, 84, 128, - 66, 65, 76, 76, 79, 212, 66, 65, 76, 76, 79, 79, 78, 45, 83, 80, 79, 75, - 69, 196, 66, 65, 76, 76, 79, 79, 78, 128, 66, 65, 76, 76, 69, 212, 66, - 65, 76, 68, 128, 66, 65, 76, 65, 71, 128, 66, 65, 76, 128, 66, 65, 204, - 66, 65, 73, 82, 75, 65, 78, 128, 66, 65, 73, 77, 65, 73, 128, 66, 65, 72, - 84, 128, 66, 65, 72, 73, 82, 71, 79, 77, 85, 75, 72, 65, 128, 66, 65, 72, - 65, 82, 50, 128, 66, 65, 72, 65, 82, 178, 66, 65, 72, 128, 66, 65, 71, - 85, 69, 84, 84, 197, 66, 65, 71, 83, 128, 66, 65, 71, 71, 65, 71, 197, - 66, 65, 71, 69, 76, 128, 66, 65, 71, 65, 128, 66, 65, 71, 51, 128, 66, - 65, 199, 66, 65, 68, 77, 73, 78, 84, 79, 206, 66, 65, 68, 71, 69, 82, - 128, 66, 65, 68, 71, 69, 128, 66, 65, 196, 66, 65, 67, 84, 82, 73, 65, - 206, 66, 65, 67, 79, 78, 128, 66, 65, 67, 75, 87, 65, 82, 68, 128, 66, - 65, 67, 75, 83, 80, 65, 67, 69, 128, 66, 65, 67, 75, 83, 76, 65, 83, 72, - 128, 66, 65, 67, 75, 83, 76, 65, 83, 200, 66, 65, 67, 75, 83, 76, 65, 78, - 84, 69, 196, 66, 65, 67, 75, 72, 65, 78, 196, 66, 65, 67, 75, 45, 84, 73, - 76, 84, 69, 196, 66, 65, 67, 75, 128, 66, 65, 67, 203, 66, 65, 66, 89, - 128, 66, 65, 66, 217, 66, 65, 65, 82, 69, 82, 85, 128, 66, 65, 45, 50, - 128, 66, 51, 48, 53, 128, 66, 50, 53, 180, 66, 50, 52, 183, 66, 50, 52, - 179, 66, 50, 52, 178, 66, 50, 52, 177, 66, 50, 52, 176, 66, 50, 51, 179, - 66, 50, 51, 177, 66, 50, 51, 176, 66, 50, 50, 181, 66, 50, 50, 176, 66, - 49, 57, 177, 66, 49, 55, 182, 66, 49, 55, 179, 66, 49, 54, 57, 128, 66, - 49, 54, 56, 128, 66, 49, 54, 55, 128, 66, 49, 54, 54, 128, 66, 49, 54, - 53, 128, 66, 49, 54, 52, 128, 66, 49, 54, 179, 66, 49, 54, 178, 66, 49, - 54, 49, 128, 66, 49, 54, 48, 128, 66, 49, 53, 185, 66, 49, 53, 56, 128, - 66, 49, 53, 55, 128, 66, 49, 53, 182, 66, 49, 53, 53, 128, 66, 49, 53, - 52, 128, 66, 49, 53, 51, 128, 66, 49, 53, 50, 128, 66, 49, 53, 177, 66, - 49, 53, 48, 128, 66, 49, 52, 54, 128, 66, 49, 52, 181, 66, 49, 52, 50, - 128, 66, 49, 52, 177, 66, 49, 52, 176, 66, 49, 51, 181, 66, 49, 51, 179, - 66, 49, 51, 50, 128, 66, 49, 51, 177, 66, 49, 51, 176, 66, 49, 50, 184, - 66, 49, 50, 183, 66, 49, 50, 181, 66, 49, 50, 179, 66, 49, 50, 178, 66, - 49, 50, 177, 66, 49, 50, 176, 66, 49, 48, 57, 205, 66, 49, 48, 57, 198, - 66, 49, 48, 56, 205, 66, 49, 48, 56, 198, 66, 49, 48, 55, 205, 66, 49, - 48, 55, 198, 66, 49, 48, 54, 205, 66, 49, 48, 54, 198, 66, 49, 48, 53, - 205, 66, 49, 48, 53, 198, 66, 49, 48, 181, 66, 49, 48, 180, 66, 49, 48, - 178, 66, 49, 48, 176, 66, 48, 57, 177, 66, 48, 57, 176, 66, 48, 56, 57, - 128, 66, 48, 56, 183, 66, 48, 56, 54, 128, 66, 48, 56, 181, 66, 48, 56, - 51, 128, 66, 48, 56, 50, 128, 66, 48, 56, 177, 66, 48, 56, 176, 66, 48, - 55, 57, 128, 66, 48, 55, 184, 66, 48, 55, 183, 66, 48, 55, 182, 66, 48, - 55, 181, 66, 48, 55, 180, 66, 48, 55, 179, 66, 48, 55, 178, 66, 48, 55, - 177, 66, 48, 55, 176, 66, 48, 54, 185, 66, 48, 54, 184, 66, 48, 54, 183, - 66, 48, 54, 182, 66, 48, 54, 181, 66, 48, 54, 52, 128, 66, 48, 54, 51, - 128, 66, 48, 54, 178, 66, 48, 54, 177, 66, 48, 54, 176, 66, 48, 53, 185, - 66, 48, 53, 184, 66, 48, 53, 183, 66, 48, 53, 54, 128, 66, 48, 53, 181, - 66, 48, 53, 180, 66, 48, 53, 179, 66, 48, 53, 178, 66, 48, 53, 177, 66, - 48, 53, 176, 66, 48, 52, 57, 128, 66, 48, 52, 184, 66, 48, 52, 55, 128, - 66, 48, 52, 182, 66, 48, 52, 181, 66, 48, 52, 180, 66, 48, 52, 179, 66, - 48, 52, 178, 66, 48, 52, 177, 66, 48, 52, 176, 66, 48, 51, 185, 66, 48, - 51, 184, 66, 48, 51, 183, 66, 48, 51, 182, 66, 48, 51, 52, 128, 66, 48, - 51, 179, 66, 48, 51, 178, 66, 48, 51, 177, 66, 48, 51, 176, 66, 48, 50, - 185, 66, 48, 50, 184, 66, 48, 50, 183, 66, 48, 50, 182, 66, 48, 50, 181, - 66, 48, 50, 180, 66, 48, 50, 179, 66, 48, 50, 50, 128, 66, 48, 50, 177, - 66, 48, 50, 176, 66, 48, 49, 57, 128, 66, 48, 49, 56, 128, 66, 48, 49, - 183, 66, 48, 49, 182, 66, 48, 49, 181, 66, 48, 49, 180, 66, 48, 49, 179, - 66, 48, 49, 178, 66, 48, 49, 177, 66, 48, 49, 176, 66, 48, 48, 57, 128, - 66, 48, 48, 185, 66, 48, 48, 56, 128, 66, 48, 48, 184, 66, 48, 48, 55, - 128, 66, 48, 48, 183, 66, 48, 48, 54, 128, 66, 48, 48, 182, 66, 48, 48, - 53, 65, 128, 66, 48, 48, 53, 128, 66, 48, 48, 181, 66, 48, 48, 52, 128, - 66, 48, 48, 180, 66, 48, 48, 51, 128, 66, 48, 48, 179, 66, 48, 48, 50, - 128, 66, 48, 48, 178, 66, 48, 48, 49, 128, 66, 48, 48, 177, 65, 90, 90, - 193, 65, 90, 85, 128, 65, 89, 66, 128, 65, 89, 65, 72, 128, 65, 88, 69, - 128, 65, 87, 90, 128, 65, 87, 88, 128, 65, 87, 81, 128, 65, 87, 69, 128, - 65, 87, 67, 128, 65, 87, 65, 217, 65, 86, 79, 67, 65, 68, 79, 128, 65, - 86, 69, 83, 84, 65, 206, 65, 86, 69, 82, 65, 71, 197, 65, 86, 65, 75, 82, - 65, 72, 65, 83, 65, 78, 89, 65, 128, 65, 86, 65, 71, 82, 65, 72, 65, 128, - 65, 85, 89, 65, 78, 78, 65, 128, 65, 85, 84, 85, 77, 78, 128, 65, 85, 84, - 79, 77, 79, 66, 73, 76, 69, 128, 65, 85, 84, 79, 77, 65, 84, 69, 196, 65, - 85, 84, 207, 65, 85, 83, 84, 82, 65, 204, 65, 85, 82, 73, 80, 73, 71, 77, - 69, 78, 84, 128, 65, 85, 82, 65, 77, 65, 90, 68, 65, 65, 72, 65, 128, 65, - 85, 82, 65, 77, 65, 90, 68, 65, 65, 45, 50, 128, 65, 85, 82, 65, 77, 65, - 90, 68, 65, 65, 128, 65, 85, 78, 78, 128, 65, 85, 71, 85, 83, 84, 128, - 65, 85, 71, 77, 69, 78, 84, 65, 84, 73, 79, 206, 65, 85, 69, 128, 65, 85, - 66, 69, 82, 71, 73, 78, 69, 128, 65, 84, 84, 73, 195, 65, 84, 84, 72, 65, - 67, 65, 78, 128, 65, 84, 84, 69, 78, 84, 73, 79, 78, 128, 65, 84, 84, 65, - 203, 65, 84, 84, 65, 67, 72, 73, 78, 199, 65, 84, 84, 65, 67, 72, 69, - 196, 65, 84, 79, 205, 65, 84, 78, 65, 200, 65, 84, 77, 65, 65, 85, 128, - 65, 84, 73, 89, 65, 128, 65, 84, 73, 85, 128, 65, 84, 73, 75, 82, 65, 77, - 65, 128, 65, 84, 72, 76, 69, 84, 73, 195, 65, 84, 72, 65, 82, 86, 65, 86, - 69, 68, 73, 195, 65, 84, 72, 65, 80, 65, 83, 67, 65, 206, 65, 84, 72, 45, - 84, 72, 65, 76, 65, 84, 72, 65, 128, 65, 83, 90, 128, 65, 83, 89, 85, 82, - 193, 65, 83, 89, 77, 80, 84, 79, 84, 73, 67, 65, 76, 76, 217, 65, 83, 84, - 82, 79, 78, 79, 77, 73, 67, 65, 204, 65, 83, 84, 82, 79, 76, 79, 71, 73, - 67, 65, 204, 65, 83, 84, 82, 65, 69, 65, 128, 65, 83, 84, 79, 78, 73, 83, - 72, 69, 196, 65, 83, 84, 69, 82, 73, 83, 77, 128, 65, 83, 84, 69, 82, 73, - 83, 75, 211, 65, 83, 84, 69, 82, 73, 83, 75, 128, 65, 83, 84, 69, 82, 73, - 83, 203, 65, 83, 84, 69, 82, 73, 83, 67, 85, 83, 128, 65, 83, 83, 89, 82, - 73, 65, 206, 65, 83, 83, 69, 82, 84, 73, 79, 78, 128, 65, 83, 80, 73, 82, - 65, 84, 73, 79, 78, 128, 65, 83, 80, 73, 82, 65, 84, 69, 196, 65, 83, 80, - 69, 82, 128, 65, 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 73, 65, 128, - 65, 83, 72, 71, 65, 66, 128, 65, 83, 72, 69, 83, 128, 65, 83, 72, 57, - 128, 65, 83, 72, 51, 128, 65, 83, 72, 178, 65, 83, 67, 73, 193, 65, 83, - 67, 69, 78, 84, 128, 65, 83, 67, 69, 78, 68, 73, 78, 199, 65, 83, 65, 76, - 50, 128, 65, 83, 45, 83, 65, 76, 65, 65, 84, 213, 65, 83, 45, 83, 65, 74, - 68, 65, 128, 65, 82, 85, 72, 85, 65, 128, 65, 82, 84, 211, 65, 82, 84, - 73, 83, 212, 65, 82, 84, 73, 67, 85, 76, 65, 84, 69, 196, 65, 82, 84, 65, - 66, 197, 65, 82, 84, 65, 128, 65, 82, 83, 69, 79, 83, 128, 65, 82, 83, - 69, 79, 211, 65, 82, 83, 69, 78, 73, 67, 128, 65, 82, 82, 79, 87, 83, - 128, 65, 82, 82, 79, 87, 211, 65, 82, 82, 79, 87, 72, 69, 65, 68, 83, - 128, 65, 82, 82, 79, 87, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, 196, 65, - 82, 82, 79, 87, 72, 69, 65, 68, 128, 65, 82, 82, 79, 87, 72, 69, 65, 196, - 65, 82, 82, 79, 87, 45, 84, 65, 73, 76, 128, 65, 82, 82, 73, 86, 73, 78, - 71, 128, 65, 82, 82, 73, 86, 69, 128, 65, 82, 82, 65, 89, 128, 65, 82, - 80, 69, 71, 71, 73, 65, 84, 207, 65, 82, 79, 85, 83, 73, 78, 199, 65, 82, - 79, 85, 82, 193, 65, 82, 79, 85, 78, 68, 45, 80, 82, 79, 70, 73, 76, 69, - 128, 65, 82, 79, 85, 78, 196, 65, 82, 77, 89, 128, 65, 82, 77, 211, 65, - 82, 77, 79, 85, 82, 128, 65, 82, 77, 69, 78, 73, 65, 206, 65, 82, 77, - 128, 65, 82, 205, 65, 82, 76, 65, 85, 199, 65, 82, 75, 84, 73, 75, 207, - 65, 82, 75, 65, 66, 128, 65, 82, 75, 65, 65, 78, 85, 128, 65, 82, 73, 83, - 84, 69, 82, 65, 128, 65, 82, 73, 83, 84, 69, 82, 193, 65, 82, 73, 69, 83, - 128, 65, 82, 71, 79, 84, 69, 82, 73, 128, 65, 82, 71, 79, 83, 89, 78, 84, - 72, 69, 84, 79, 78, 128, 65, 82, 71, 73, 128, 65, 82, 69, 80, 65, 128, - 65, 82, 69, 65, 128, 65, 82, 68, 72, 65, 86, 73, 83, 65, 82, 71, 65, 128, - 65, 82, 68, 72, 65, 67, 65, 78, 68, 82, 65, 128, 65, 82, 67, 72, 65, 73, - 79, 78, 128, 65, 82, 67, 72, 65, 73, 79, 206, 65, 82, 67, 72, 65, 73, - 195, 65, 82, 67, 200, 65, 82, 67, 128, 65, 82, 195, 65, 82, 65, 77, 65, - 73, 195, 65, 82, 65, 69, 65, 69, 128, 65, 82, 65, 69, 65, 45, 85, 128, - 65, 82, 65, 69, 65, 45, 73, 128, 65, 82, 65, 69, 65, 45, 69, 79, 128, 65, - 82, 65, 69, 65, 45, 69, 128, 65, 82, 65, 69, 65, 45, 65, 128, 65, 82, 65, - 68, 128, 65, 82, 65, 196, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, - 195, 65, 82, 65, 66, 73, 65, 206, 65, 82, 45, 82, 85, 66, 128, 65, 82, - 45, 82, 65, 72, 77, 65, 206, 65, 82, 45, 82, 65, 72, 69, 69, 77, 128, 65, - 81, 85, 65, 82, 73, 85, 83, 128, 65, 81, 85, 65, 70, 79, 82, 84, 73, 83, - 128, 65, 81, 85, 193, 65, 80, 85, 206, 65, 80, 82, 73, 76, 128, 65, 80, - 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, 217, 65, 80, 80, 82, 79, 88, 73, - 77, 65, 84, 69, 128, 65, 80, 80, 82, 79, 65, 67, 72, 69, 211, 65, 80, 80, - 82, 79, 65, 67, 72, 128, 65, 80, 80, 76, 73, 67, 65, 84, 73, 79, 78, 128, - 65, 80, 80, 76, 73, 67, 65, 84, 73, 79, 206, 65, 80, 79, 84, 72, 69, 83, - 128, 65, 80, 79, 84, 72, 69, 77, 65, 128, 65, 80, 79, 83, 84, 82, 79, 80, - 72, 69, 128, 65, 80, 79, 83, 84, 82, 79, 70, 79, 83, 128, 65, 80, 79, 83, - 84, 82, 79, 70, 79, 211, 65, 80, 79, 83, 84, 82, 79, 70, 79, 201, 65, 80, - 79, 76, 76, 79, 78, 128, 65, 80, 79, 68, 69, 88, 73, 65, 128, 65, 80, 79, - 68, 69, 82, 77, 193, 65, 80, 76, 79, 85, 78, 128, 65, 80, 76, 201, 65, - 80, 204, 65, 80, 73, 78, 128, 65, 80, 69, 83, 207, 65, 80, 67, 128, 65, - 80, 65, 82, 84, 128, 65, 80, 65, 65, 84, 79, 128, 65, 79, 85, 128, 65, - 79, 82, 128, 65, 78, 89, 128, 65, 78, 85, 83, 86, 65, 82, 65, 89, 65, - 128, 65, 78, 85, 83, 86, 65, 82, 65, 128, 65, 78, 85, 83, 86, 65, 82, - 193, 65, 78, 85, 68, 65, 84, 84, 65, 128, 65, 78, 85, 68, 65, 84, 84, - 193, 65, 78, 84, 73, 82, 69, 83, 84, 82, 73, 67, 84, 73, 79, 78, 128, 65, - 78, 84, 73, 77, 79, 78, 89, 45, 50, 128, 65, 78, 84, 73, 77, 79, 78, 89, - 128, 65, 78, 84, 73, 77, 79, 78, 217, 65, 78, 84, 73, 77, 79, 78, 73, 65, - 84, 69, 128, 65, 78, 84, 73, 75, 69, 78, 79, 77, 65, 128, 65, 78, 84, 73, - 75, 69, 78, 79, 75, 89, 76, 73, 83, 77, 65, 128, 65, 78, 84, 73, 70, 79, - 78, 73, 65, 128, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 45, - 82, 79, 84, 65, 84, 69, 196, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, - 83, 69, 128, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 197, 65, 78, - 84, 69, 78, 78, 65, 128, 65, 78, 84, 69, 78, 78, 193, 65, 78, 84, 65, 82, - 71, 79, 77, 85, 75, 72, 65, 128, 65, 78, 83, 85, 218, 65, 78, 83, 72, 69, - 128, 65, 78, 80, 69, 65, 128, 65, 78, 207, 65, 78, 78, 85, 73, 84, 217, - 65, 78, 78, 79, 84, 65, 84, 73, 79, 206, 65, 78, 78, 65, 65, 85, 128, 65, - 78, 75, 72, 128, 65, 78, 74, 73, 128, 65, 78, 73, 77, 65, 76, 128, 65, - 78, 72, 85, 78, 78, 65, 128, 65, 78, 72, 85, 77, 65, 65, 128, 65, 78, 72, - 85, 77, 128, 65, 78, 72, 85, 128, 65, 78, 72, 65, 65, 128, 65, 78, 72, - 128, 65, 78, 71, 85, 76, 65, 82, 128, 65, 78, 71, 85, 73, 83, 72, 69, - 196, 65, 78, 71, 83, 84, 82, 79, 205, 65, 78, 71, 82, 217, 65, 78, 71, - 76, 73, 67, 65, 78, 193, 65, 78, 71, 76, 69, 68, 128, 65, 78, 71, 76, 69, - 196, 65, 78, 71, 75, 72, 65, 78, 75, 72, 85, 128, 65, 78, 71, 75, 65, - 128, 65, 78, 71, 69, 210, 65, 78, 71, 69, 76, 128, 65, 78, 71, 69, 68, - 128, 65, 78, 68, 65, 80, 128, 65, 78, 67, 79, 82, 65, 128, 65, 78, 67, - 72, 79, 82, 128, 65, 78, 65, 84, 82, 73, 67, 72, 73, 83, 77, 65, 128, 65, - 78, 65, 84, 79, 77, 73, 67, 65, 204, 65, 78, 65, 80, 128, 65, 78, 45, 78, - 73, 83, 70, 128, 65, 77, 85, 76, 69, 84, 128, 65, 77, 80, 83, 128, 65, - 77, 80, 72, 79, 82, 65, 128, 65, 77, 80, 69, 82, 83, 65, 78, 68, 128, 65, - 77, 80, 69, 82, 83, 65, 78, 196, 65, 77, 79, 85, 78, 212, 65, 77, 69, 82, - 73, 67, 65, 83, 128, 65, 77, 69, 82, 73, 67, 65, 206, 65, 77, 66, 85, 76, - 65, 78, 67, 69, 128, 65, 77, 66, 193, 65, 77, 66, 128, 65, 77, 65, 82, - 128, 65, 77, 65, 210, 65, 77, 65, 76, 71, 65, 77, 65, 84, 73, 79, 206, - 65, 77, 65, 76, 71, 65, 77, 128, 65, 76, 86, 69, 79, 76, 65, 210, 65, 76, - 85, 77, 128, 65, 76, 84, 69, 82, 78, 65, 84, 73, 86, 197, 65, 76, 84, 69, - 82, 78, 65, 84, 73, 79, 206, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, - 128, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 199, 65, 76, 84, 69, 82, 78, - 65, 84, 69, 128, 65, 76, 84, 69, 82, 78, 65, 84, 197, 65, 76, 84, 65, - 128, 65, 76, 80, 72, 65, 128, 65, 76, 80, 72, 193, 65, 76, 80, 65, 80, - 82, 65, 78, 65, 128, 65, 76, 80, 65, 80, 82, 65, 65, 78, 193, 65, 76, 80, - 65, 128, 65, 76, 77, 79, 83, 212, 65, 76, 76, 79, 128, 65, 76, 76, 73, - 65, 78, 67, 69, 128, 65, 76, 76, 201, 65, 76, 76, 65, 200, 65, 76, 76, - 65, 65, 72, 128, 65, 76, 75, 65, 76, 73, 45, 50, 128, 65, 76, 75, 65, 76, - 73, 128, 65, 76, 73, 71, 78, 69, 196, 65, 76, 73, 70, 85, 128, 65, 76, - 73, 70, 128, 65, 76, 73, 198, 65, 76, 73, 69, 78, 128, 65, 76, 73, 69, - 206, 65, 76, 71, 73, 218, 65, 76, 70, 65, 128, 65, 76, 69, 85, 212, 65, - 76, 69, 82, 84, 128, 65, 76, 69, 80, 72, 128, 65, 76, 69, 77, 66, 73, 67, - 128, 65, 76, 69, 70, 128, 65, 76, 66, 65, 78, 73, 65, 206, 65, 76, 65, - 89, 78, 65, 65, 128, 65, 76, 65, 89, 72, 73, 77, 65, 193, 65, 76, 65, 89, - 72, 73, 205, 65, 76, 65, 89, 72, 201, 65, 76, 65, 89, 72, 69, 128, 65, - 76, 65, 89, 72, 197, 65, 76, 65, 89, 72, 65, 193, 65, 76, 65, 82, 205, - 65, 76, 65, 80, 72, 128, 65, 76, 45, 76, 65, 75, 85, 78, 65, 128, 65, 76, - 45, 74, 85, 90, 128, 65, 75, 85, 82, 213, 65, 75, 84, 73, 69, 83, 69, 76, - 83, 75, 65, 66, 128, 65, 75, 83, 65, 128, 65, 75, 72, 77, 73, 77, 73, - 195, 65, 75, 66, 65, 210, 65, 75, 65, 82, 65, 128, 65, 75, 65, 82, 193, - 65, 73, 89, 65, 78, 78, 65, 128, 65, 73, 86, 73, 76, 73, 203, 65, 73, 86, - 65, 128, 65, 73, 84, 79, 206, 65, 73, 82, 80, 76, 65, 78, 69, 128, 65, - 73, 82, 80, 76, 65, 78, 197, 65, 73, 78, 213, 65, 73, 78, 78, 128, 65, - 73, 76, 77, 128, 65, 73, 75, 65, 82, 65, 128, 65, 73, 72, 86, 85, 83, - 128, 65, 72, 83, 68, 65, 128, 65, 72, 83, 65, 128, 65, 72, 79, 205, 65, - 72, 65, 78, 199, 65, 72, 65, 71, 71, 65, 210, 65, 72, 65, 68, 128, 65, - 71, 85, 78, 71, 128, 65, 71, 79, 71, 201, 65, 71, 71, 82, 65, 86, 65, 84, - 73, 79, 78, 128, 65, 71, 71, 82, 65, 86, 65, 84, 69, 196, 65, 71, 65, 73, - 78, 83, 212, 65, 71, 65, 73, 78, 128, 65, 70, 84, 69, 210, 65, 70, 83, - 65, 65, 81, 128, 65, 70, 82, 73, 67, 65, 206, 65, 70, 79, 82, 69, 77, 69, - 78, 84, 73, 79, 78, 69, 68, 128, 65, 70, 71, 72, 65, 78, 201, 65, 70, 70, - 82, 73, 67, 65, 84, 73, 79, 206, 65, 70, 70, 73, 216, 65, 69, 89, 65, 78, - 78, 65, 128, 65, 69, 89, 128, 65, 69, 83, 67, 85, 76, 65, 80, 73, 85, 83, - 128, 65, 69, 83, 67, 128, 65, 69, 83, 128, 65, 69, 82, 73, 65, 204, 65, - 69, 82, 128, 65, 69, 76, 65, 45, 80, 73, 76, 76, 65, 128, 65, 69, 76, - 128, 65, 69, 75, 128, 65, 69, 71, 69, 65, 206, 65, 69, 71, 128, 65, 69, - 69, 89, 65, 78, 78, 65, 128, 65, 69, 69, 128, 65, 69, 68, 65, 45, 80, 73, - 76, 76, 65, 128, 65, 69, 68, 128, 65, 69, 66, 128, 65, 68, 86, 65, 78, - 84, 65, 71, 69, 128, 65, 68, 86, 65, 78, 67, 69, 128, 65, 68, 85, 76, 84, - 128, 65, 68, 77, 73, 83, 83, 73, 79, 206, 65, 68, 77, 69, 84, 79, 83, - 128, 65, 68, 76, 65, 205, 65, 68, 72, 69, 83, 73, 86, 197, 65, 68, 69, - 71, 128, 65, 68, 69, 199, 65, 68, 68, 82, 69, 83, 83, 69, 196, 65, 68, - 68, 82, 69, 83, 211, 65, 68, 68, 65, 75, 128, 65, 68, 65, 203, 65, 67, - 85, 84, 69, 45, 77, 65, 67, 82, 79, 78, 128, 65, 67, 85, 84, 69, 45, 71, - 82, 65, 86, 69, 45, 65, 67, 85, 84, 69, 128, 65, 67, 85, 84, 197, 65, 67, - 84, 85, 65, 76, 76, 217, 65, 67, 84, 73, 86, 65, 84, 197, 65, 67, 82, 79, - 80, 72, 79, 78, 73, 195, 65, 67, 75, 78, 79, 87, 76, 69, 68, 71, 69, 128, - 65, 67, 67, 85, 77, 85, 76, 65, 84, 73, 79, 78, 128, 65, 67, 67, 79, 85, - 78, 212, 65, 67, 67, 79, 82, 68, 73, 79, 78, 128, 65, 67, 67, 79, 77, 77, - 79, 68, 65, 84, 73, 79, 78, 128, 65, 67, 67, 69, 80, 84, 128, 65, 67, 67, - 69, 78, 84, 45, 83, 84, 65, 67, 67, 65, 84, 79, 128, 65, 67, 67, 69, 78, - 84, 128, 65, 67, 67, 69, 78, 212, 65, 67, 65, 68, 69, 77, 217, 65, 66, - 89, 83, 77, 65, 204, 65, 66, 85, 78, 68, 65, 78, 67, 69, 128, 65, 66, 75, - 72, 65, 83, 73, 65, 206, 65, 66, 66, 82, 69, 86, 73, 65, 84, 73, 79, 206, - 65, 66, 65, 70, 73, 76, 73, 128, 65, 66, 65, 67, 85, 83, 128, 65, 66, - 178, 65, 66, 49, 57, 49, 128, 65, 66, 49, 56, 56, 128, 65, 66, 49, 56, - 48, 128, 65, 66, 49, 55, 49, 128, 65, 66, 49, 54, 52, 128, 65, 66, 49, - 51, 49, 66, 128, 65, 66, 49, 51, 49, 65, 128, 65, 66, 49, 50, 51, 128, - 65, 66, 49, 50, 50, 128, 65, 66, 49, 50, 48, 128, 65, 66, 49, 49, 56, - 128, 65, 66, 48, 56, 55, 128, 65, 66, 48, 56, 54, 128, 65, 66, 48, 56, - 53, 128, 65, 66, 48, 56, 50, 128, 65, 66, 48, 56, 49, 128, 65, 66, 48, - 56, 48, 128, 65, 66, 48, 55, 57, 128, 65, 66, 48, 55, 56, 128, 65, 66, - 48, 55, 55, 128, 65, 66, 48, 55, 54, 128, 65, 66, 48, 55, 52, 128, 65, - 66, 48, 55, 51, 128, 65, 66, 48, 55, 48, 128, 65, 66, 48, 54, 57, 128, - 65, 66, 48, 54, 55, 128, 65, 66, 48, 54, 54, 128, 65, 66, 48, 54, 53, - 128, 65, 66, 48, 54, 49, 128, 65, 66, 48, 54, 48, 128, 65, 66, 48, 53, - 57, 128, 65, 66, 48, 53, 56, 128, 65, 66, 48, 53, 55, 128, 65, 66, 48, - 53, 54, 128, 65, 66, 48, 53, 53, 128, 65, 66, 48, 53, 52, 128, 65, 66, - 48, 53, 51, 128, 65, 66, 48, 53, 49, 128, 65, 66, 48, 53, 48, 128, 65, - 66, 48, 52, 57, 128, 65, 66, 48, 52, 56, 128, 65, 66, 48, 52, 55, 128, - 65, 66, 48, 52, 54, 128, 65, 66, 48, 52, 53, 128, 65, 66, 48, 52, 52, - 128, 65, 66, 48, 52, 49, 128, 65, 66, 48, 52, 48, 128, 65, 66, 48, 51, - 57, 128, 65, 66, 48, 51, 56, 128, 65, 66, 48, 51, 55, 128, 65, 66, 48, - 51, 52, 128, 65, 66, 48, 51, 49, 128, 65, 66, 48, 51, 48, 128, 65, 66, - 48, 50, 57, 128, 65, 66, 48, 50, 56, 128, 65, 66, 48, 50, 55, 128, 65, - 66, 48, 50, 54, 128, 65, 66, 48, 50, 52, 128, 65, 66, 48, 50, 51, 77, - 128, 65, 66, 48, 50, 51, 128, 65, 66, 48, 50, 50, 77, 128, 65, 66, 48, - 50, 50, 70, 128, 65, 66, 48, 50, 50, 128, 65, 66, 48, 50, 49, 77, 128, - 65, 66, 48, 50, 49, 70, 128, 65, 66, 48, 50, 49, 128, 65, 66, 48, 50, 48, - 128, 65, 66, 48, 49, 55, 128, 65, 66, 48, 49, 54, 128, 65, 66, 48, 49, - 51, 128, 65, 66, 48, 49, 49, 128, 65, 66, 48, 49, 48, 128, 65, 66, 48, - 48, 57, 128, 65, 66, 48, 48, 56, 128, 65, 66, 48, 48, 55, 128, 65, 66, - 48, 48, 54, 128, 65, 66, 48, 48, 53, 128, 65, 66, 48, 48, 52, 128, 65, - 66, 48, 48, 51, 128, 65, 66, 48, 48, 50, 128, 65, 66, 48, 48, 49, 128, - 65, 65, 90, 72, 65, 65, 75, 75, 85, 128, 65, 65, 89, 73, 78, 128, 65, 65, - 89, 65, 78, 78, 65, 128, 65, 65, 89, 128, 65, 65, 87, 128, 65, 65, 79, - 128, 65, 65, 74, 128, 65, 65, 66, 65, 65, 70, 73, 76, 73, 128, 65, 65, - 48, 51, 50, 128, 65, 65, 48, 51, 49, 128, 65, 65, 48, 51, 48, 128, 65, - 65, 48, 50, 57, 128, 65, 65, 48, 50, 56, 128, 65, 65, 48, 50, 55, 128, - 65, 65, 48, 50, 54, 128, 65, 65, 48, 50, 53, 128, 65, 65, 48, 50, 52, - 128, 65, 65, 48, 50, 51, 128, 65, 65, 48, 50, 50, 128, 65, 65, 48, 50, - 49, 128, 65, 65, 48, 50, 48, 128, 65, 65, 48, 49, 57, 128, 65, 65, 48, - 49, 56, 128, 65, 65, 48, 49, 55, 128, 65, 65, 48, 49, 54, 128, 65, 65, - 48, 49, 53, 128, 65, 65, 48, 49, 52, 128, 65, 65, 48, 49, 51, 128, 65, - 65, 48, 49, 50, 128, 65, 65, 48, 49, 49, 128, 65, 65, 48, 49, 48, 128, - 65, 65, 48, 48, 57, 128, 65, 65, 48, 48, 56, 128, 65, 65, 48, 48, 55, 66, - 128, 65, 65, 48, 48, 55, 65, 128, 65, 65, 48, 48, 55, 128, 65, 65, 48, - 48, 54, 128, 65, 65, 48, 48, 53, 128, 65, 65, 48, 48, 52, 128, 65, 65, - 48, 48, 51, 128, 65, 65, 48, 48, 50, 128, 65, 65, 48, 48, 49, 128, 65, - 56, 48, 55, 128, 65, 56, 48, 54, 128, 65, 56, 48, 53, 128, 65, 56, 48, - 52, 128, 65, 56, 48, 51, 128, 65, 56, 48, 50, 128, 65, 56, 48, 49, 128, - 65, 56, 48, 48, 128, 65, 55, 51, 178, 65, 55, 50, 182, 65, 55, 49, 183, - 65, 55, 49, 181, 65, 55, 49, 180, 65, 55, 49, 179, 65, 55, 49, 178, 65, - 55, 49, 177, 65, 55, 49, 176, 65, 55, 48, 57, 45, 182, 65, 55, 48, 57, - 45, 180, 65, 55, 48, 57, 45, 179, 65, 55, 48, 57, 45, 178, 65, 55, 48, - 185, 65, 55, 48, 184, 65, 55, 48, 183, 65, 55, 48, 182, 65, 55, 48, 181, - 65, 55, 48, 180, 65, 55, 48, 179, 65, 55, 48, 178, 65, 55, 48, 177, 65, - 54, 54, 52, 128, 65, 54, 54, 51, 128, 65, 54, 54, 50, 128, 65, 54, 54, - 49, 128, 65, 54, 54, 48, 128, 65, 54, 53, 57, 128, 65, 54, 53, 56, 128, - 65, 54, 53, 55, 128, 65, 54, 53, 54, 128, 65, 54, 53, 53, 128, 65, 54, - 53, 52, 128, 65, 54, 53, 51, 128, 65, 54, 53, 50, 128, 65, 54, 53, 49, - 128, 65, 54, 52, 57, 128, 65, 54, 52, 56, 128, 65, 54, 52, 54, 128, 65, - 54, 52, 53, 128, 65, 54, 52, 52, 128, 65, 54, 52, 51, 128, 65, 54, 52, - 50, 128, 65, 54, 52, 48, 128, 65, 54, 51, 56, 128, 65, 54, 51, 55, 128, - 65, 54, 51, 52, 128, 65, 54, 50, 57, 128, 65, 54, 50, 56, 128, 65, 54, - 50, 55, 128, 65, 54, 50, 54, 128, 65, 54, 50, 52, 128, 65, 54, 50, 51, - 128, 65, 54, 50, 50, 128, 65, 54, 50, 49, 128, 65, 54, 50, 48, 128, 65, - 54, 49, 57, 128, 65, 54, 49, 56, 128, 65, 54, 49, 55, 128, 65, 54, 49, - 54, 128, 65, 54, 49, 53, 128, 65, 54, 49, 52, 128, 65, 54, 49, 51, 128, - 65, 54, 49, 50, 128, 65, 54, 49, 49, 128, 65, 54, 49, 48, 128, 65, 54, - 48, 57, 128, 65, 54, 48, 56, 128, 65, 54, 48, 54, 128, 65, 54, 48, 52, - 128, 65, 54, 48, 51, 128, 65, 54, 48, 50, 128, 65, 54, 48, 49, 128, 65, - 54, 48, 48, 128, 65, 53, 57, 56, 128, 65, 53, 57, 54, 128, 65, 53, 57, - 53, 128, 65, 53, 57, 52, 128, 65, 53, 57, 50, 128, 65, 53, 57, 49, 128, - 65, 53, 56, 57, 128, 65, 53, 56, 56, 128, 65, 53, 56, 55, 128, 65, 53, - 56, 54, 128, 65, 53, 56, 53, 128, 65, 53, 56, 52, 128, 65, 53, 56, 51, - 128, 65, 53, 56, 50, 128, 65, 53, 56, 49, 128, 65, 53, 56, 48, 128, 65, - 53, 55, 57, 128, 65, 53, 55, 56, 128, 65, 53, 55, 55, 128, 65, 53, 55, - 54, 128, 65, 53, 55, 53, 128, 65, 53, 55, 52, 128, 65, 53, 55, 51, 128, - 65, 53, 55, 50, 128, 65, 53, 55, 49, 128, 65, 53, 55, 48, 128, 65, 53, - 54, 57, 128, 65, 53, 54, 56, 128, 65, 53, 54, 54, 128, 65, 53, 54, 53, - 128, 65, 53, 54, 52, 128, 65, 53, 54, 51, 128, 65, 53, 53, 57, 128, 65, - 53, 53, 55, 128, 65, 53, 53, 54, 128, 65, 53, 53, 53, 128, 65, 53, 53, - 52, 128, 65, 53, 53, 51, 128, 65, 53, 53, 50, 128, 65, 53, 53, 49, 128, - 65, 53, 53, 48, 128, 65, 53, 52, 57, 128, 65, 53, 52, 56, 128, 65, 53, - 52, 55, 128, 65, 53, 52, 53, 128, 65, 53, 52, 50, 128, 65, 53, 52, 49, - 128, 65, 53, 52, 48, 128, 65, 53, 51, 57, 128, 65, 53, 51, 56, 128, 65, - 53, 51, 55, 128, 65, 53, 51, 54, 128, 65, 53, 51, 53, 128, 65, 53, 51, - 52, 128, 65, 53, 51, 50, 128, 65, 53, 51, 49, 128, 65, 53, 51, 48, 128, - 65, 53, 50, 57, 128, 65, 53, 50, 56, 128, 65, 53, 50, 55, 128, 65, 53, - 50, 54, 128, 65, 53, 50, 53, 128, 65, 53, 50, 52, 128, 65, 53, 50, 51, - 128, 65, 53, 50, 50, 128, 65, 53, 50, 49, 128, 65, 53, 50, 48, 128, 65, - 53, 49, 57, 128, 65, 53, 49, 56, 128, 65, 53, 49, 55, 128, 65, 53, 49, - 54, 128, 65, 53, 49, 53, 128, 65, 53, 49, 52, 128, 65, 53, 49, 51, 128, - 65, 53, 49, 50, 128, 65, 53, 49, 49, 128, 65, 53, 49, 48, 128, 65, 53, - 48, 57, 128, 65, 53, 48, 56, 128, 65, 53, 48, 55, 128, 65, 53, 48, 54, - 128, 65, 53, 48, 53, 128, 65, 53, 48, 52, 128, 65, 53, 48, 51, 128, 65, - 53, 48, 50, 128, 65, 53, 48, 49, 128, 65, 52, 57, 55, 128, 65, 52, 57, - 54, 128, 65, 52, 57, 53, 128, 65, 52, 57, 52, 128, 65, 52, 57, 51, 128, - 65, 52, 57, 50, 128, 65, 52, 57, 49, 128, 65, 52, 57, 48, 128, 65, 52, - 56, 57, 128, 65, 52, 56, 56, 128, 65, 52, 56, 55, 128, 65, 52, 56, 54, - 128, 65, 52, 56, 53, 128, 65, 52, 56, 52, 128, 65, 52, 56, 51, 128, 65, - 52, 56, 50, 128, 65, 52, 56, 49, 128, 65, 52, 56, 48, 128, 65, 52, 55, - 57, 128, 65, 52, 55, 56, 128, 65, 52, 55, 55, 128, 65, 52, 55, 54, 128, - 65, 52, 55, 53, 128, 65, 52, 55, 52, 128, 65, 52, 55, 51, 128, 65, 52, - 55, 50, 128, 65, 52, 55, 49, 128, 65, 52, 55, 48, 128, 65, 52, 54, 57, - 128, 65, 52, 54, 56, 128, 65, 52, 54, 55, 128, 65, 52, 54, 54, 128, 65, - 52, 54, 53, 128, 65, 52, 54, 52, 128, 65, 52, 54, 51, 128, 65, 52, 54, - 50, 128, 65, 52, 54, 49, 128, 65, 52, 54, 48, 128, 65, 52, 53, 57, 128, - 65, 52, 53, 56, 128, 65, 52, 53, 55, 65, 128, 65, 52, 53, 55, 128, 65, - 52, 53, 54, 128, 65, 52, 53, 53, 128, 65, 52, 53, 52, 128, 65, 52, 53, - 51, 128, 65, 52, 53, 50, 128, 65, 52, 53, 49, 128, 65, 52, 53, 48, 65, - 128, 65, 52, 53, 48, 128, 65, 52, 52, 57, 128, 65, 52, 52, 56, 128, 65, - 52, 52, 55, 128, 65, 52, 52, 54, 128, 65, 52, 52, 53, 128, 65, 52, 52, - 52, 128, 65, 52, 52, 51, 128, 65, 52, 52, 50, 128, 65, 52, 52, 49, 128, - 65, 52, 52, 48, 128, 65, 52, 51, 57, 128, 65, 52, 51, 56, 128, 65, 52, - 51, 55, 128, 65, 52, 51, 54, 128, 65, 52, 51, 53, 128, 65, 52, 51, 52, - 128, 65, 52, 51, 51, 128, 65, 52, 51, 50, 128, 65, 52, 51, 49, 128, 65, - 52, 51, 48, 128, 65, 52, 50, 57, 128, 65, 52, 50, 56, 128, 65, 52, 50, - 55, 128, 65, 52, 50, 54, 128, 65, 52, 50, 53, 128, 65, 52, 50, 52, 128, - 65, 52, 50, 51, 128, 65, 52, 50, 50, 128, 65, 52, 50, 49, 128, 65, 52, - 50, 48, 128, 65, 52, 49, 57, 128, 65, 52, 49, 56, 45, 86, 65, 83, 128, - 65, 52, 49, 56, 128, 65, 52, 49, 55, 45, 86, 65, 83, 128, 65, 52, 49, 55, - 128, 65, 52, 49, 54, 45, 86, 65, 83, 128, 65, 52, 49, 54, 128, 65, 52, - 49, 53, 45, 86, 65, 83, 128, 65, 52, 49, 53, 128, 65, 52, 49, 52, 45, 86, - 65, 83, 128, 65, 52, 49, 52, 128, 65, 52, 49, 51, 45, 86, 65, 83, 128, - 65, 52, 49, 51, 128, 65, 52, 49, 50, 45, 86, 65, 83, 128, 65, 52, 49, 50, - 128, 65, 52, 49, 49, 45, 86, 65, 83, 128, 65, 52, 49, 49, 128, 65, 52, - 49, 48, 193, 65, 52, 49, 48, 45, 86, 65, 83, 128, 65, 52, 49, 176, 65, - 52, 48, 57, 45, 86, 65, 83, 128, 65, 52, 48, 57, 128, 65, 52, 48, 56, 45, - 86, 65, 83, 128, 65, 52, 48, 56, 128, 65, 52, 48, 55, 45, 86, 65, 83, - 128, 65, 52, 48, 55, 128, 65, 52, 48, 54, 45, 86, 65, 83, 128, 65, 52, - 48, 54, 128, 65, 52, 48, 53, 45, 86, 65, 83, 128, 65, 52, 48, 53, 128, - 65, 52, 48, 52, 45, 86, 65, 83, 128, 65, 52, 48, 52, 128, 65, 52, 48, 51, - 45, 86, 65, 83, 128, 65, 52, 48, 51, 128, 65, 52, 48, 50, 45, 86, 65, 83, - 128, 65, 52, 48, 50, 128, 65, 52, 48, 49, 45, 86, 65, 83, 128, 65, 52, - 48, 49, 128, 65, 52, 48, 48, 45, 86, 65, 83, 128, 65, 52, 48, 48, 128, - 65, 51, 57, 57, 128, 65, 51, 57, 56, 128, 65, 51, 57, 55, 128, 65, 51, - 57, 54, 128, 65, 51, 57, 53, 128, 65, 51, 57, 52, 128, 65, 51, 57, 179, - 65, 51, 57, 50, 128, 65, 51, 57, 49, 128, 65, 51, 57, 48, 128, 65, 51, - 56, 57, 128, 65, 51, 56, 56, 128, 65, 51, 56, 55, 128, 65, 51, 56, 54, - 65, 128, 65, 51, 56, 54, 128, 65, 51, 56, 53, 128, 65, 51, 56, 52, 128, - 65, 51, 56, 51, 65, 128, 65, 51, 56, 179, 65, 51, 56, 50, 128, 65, 51, - 56, 49, 65, 128, 65, 51, 56, 49, 128, 65, 51, 56, 48, 128, 65, 51, 55, - 57, 128, 65, 51, 55, 56, 128, 65, 51, 55, 55, 128, 65, 51, 55, 54, 128, - 65, 51, 55, 53, 128, 65, 51, 55, 52, 128, 65, 51, 55, 51, 128, 65, 51, - 55, 50, 128, 65, 51, 55, 49, 65, 128, 65, 51, 55, 49, 128, 65, 51, 55, - 48, 128, 65, 51, 54, 57, 128, 65, 51, 54, 56, 65, 128, 65, 51, 54, 56, - 128, 65, 51, 54, 55, 128, 65, 51, 54, 54, 128, 65, 51, 54, 53, 128, 65, - 51, 54, 52, 65, 128, 65, 51, 54, 52, 128, 65, 51, 54, 51, 128, 65, 51, - 54, 50, 128, 65, 51, 54, 49, 128, 65, 51, 54, 48, 128, 65, 51, 53, 57, - 65, 128, 65, 51, 53, 57, 128, 65, 51, 53, 56, 128, 65, 51, 53, 55, 128, - 65, 51, 53, 54, 128, 65, 51, 53, 53, 128, 65, 51, 53, 52, 128, 65, 51, - 53, 51, 128, 65, 51, 53, 50, 128, 65, 51, 53, 49, 128, 65, 51, 53, 48, - 128, 65, 51, 52, 57, 128, 65, 51, 52, 56, 128, 65, 51, 52, 55, 128, 65, - 51, 52, 54, 128, 65, 51, 52, 53, 128, 65, 51, 52, 52, 128, 65, 51, 52, - 51, 128, 65, 51, 52, 50, 128, 65, 51, 52, 49, 128, 65, 51, 52, 48, 128, - 65, 51, 51, 57, 128, 65, 51, 51, 56, 128, 65, 51, 51, 55, 128, 65, 51, - 51, 54, 67, 128, 65, 51, 51, 54, 66, 128, 65, 51, 51, 54, 65, 128, 65, - 51, 51, 54, 128, 65, 51, 51, 53, 128, 65, 51, 51, 52, 128, 65, 51, 51, - 51, 128, 65, 51, 51, 50, 67, 128, 65, 51, 51, 50, 66, 128, 65, 51, 51, - 50, 65, 128, 65, 51, 51, 50, 128, 65, 51, 51, 49, 128, 65, 51, 51, 48, - 128, 65, 51, 50, 57, 65, 128, 65, 51, 50, 57, 128, 65, 51, 50, 56, 128, - 65, 51, 50, 55, 128, 65, 51, 50, 54, 128, 65, 51, 50, 53, 128, 65, 51, - 50, 52, 128, 65, 51, 50, 51, 128, 65, 51, 50, 50, 128, 65, 51, 50, 49, - 128, 65, 51, 50, 48, 128, 65, 51, 49, 57, 128, 65, 51, 49, 56, 128, 65, - 51, 49, 55, 128, 65, 51, 49, 54, 128, 65, 51, 49, 53, 128, 65, 51, 49, - 52, 128, 65, 51, 49, 51, 67, 128, 65, 51, 49, 51, 66, 128, 65, 51, 49, - 51, 65, 128, 65, 51, 49, 51, 128, 65, 51, 49, 50, 128, 65, 51, 49, 49, - 128, 65, 51, 49, 48, 128, 65, 51, 48, 57, 67, 128, 65, 51, 48, 57, 66, - 128, 65, 51, 48, 57, 65, 128, 65, 51, 48, 57, 128, 65, 51, 48, 56, 128, - 65, 51, 48, 55, 128, 65, 51, 48, 54, 128, 65, 51, 48, 53, 128, 65, 51, - 48, 52, 128, 65, 51, 48, 51, 128, 65, 51, 48, 50, 128, 65, 51, 48, 49, - 128, 65, 51, 48, 48, 128, 65, 50, 57, 57, 65, 128, 65, 50, 57, 57, 128, - 65, 50, 57, 56, 128, 65, 50, 57, 55, 128, 65, 50, 57, 54, 128, 65, 50, - 57, 53, 128, 65, 50, 57, 52, 65, 128, 65, 50, 57, 52, 128, 65, 50, 57, - 51, 128, 65, 50, 57, 50, 128, 65, 50, 57, 49, 128, 65, 50, 57, 48, 128, - 65, 50, 56, 57, 65, 128, 65, 50, 56, 57, 128, 65, 50, 56, 56, 128, 65, - 50, 56, 55, 128, 65, 50, 56, 54, 128, 65, 50, 56, 53, 128, 65, 50, 56, - 52, 128, 65, 50, 56, 51, 128, 65, 50, 56, 50, 128, 65, 50, 56, 49, 128, - 65, 50, 56, 48, 128, 65, 50, 55, 57, 128, 65, 50, 55, 56, 128, 65, 50, - 55, 55, 128, 65, 50, 55, 54, 128, 65, 50, 55, 53, 128, 65, 50, 55, 52, - 128, 65, 50, 55, 51, 128, 65, 50, 55, 50, 128, 65, 50, 55, 49, 128, 65, - 50, 55, 48, 128, 65, 50, 54, 57, 128, 65, 50, 54, 56, 128, 65, 50, 54, - 55, 65, 128, 65, 50, 54, 55, 128, 65, 50, 54, 54, 128, 65, 50, 54, 53, - 128, 65, 50, 54, 52, 128, 65, 50, 54, 51, 128, 65, 50, 54, 50, 128, 65, - 50, 54, 49, 128, 65, 50, 54, 48, 128, 65, 50, 53, 57, 128, 65, 50, 53, - 56, 128, 65, 50, 53, 55, 128, 65, 50, 53, 54, 128, 65, 50, 53, 53, 128, - 65, 50, 53, 52, 128, 65, 50, 53, 51, 128, 65, 50, 53, 50, 128, 65, 50, - 53, 49, 128, 65, 50, 53, 48, 128, 65, 50, 52, 57, 128, 65, 50, 52, 56, - 128, 65, 50, 52, 55, 128, 65, 50, 52, 54, 128, 65, 50, 52, 53, 128, 65, - 50, 52, 52, 128, 65, 50, 52, 51, 128, 65, 50, 52, 50, 128, 65, 50, 52, - 49, 128, 65, 50, 52, 48, 128, 65, 50, 51, 57, 128, 65, 50, 51, 56, 128, - 65, 50, 51, 55, 128, 65, 50, 51, 54, 128, 65, 50, 51, 53, 128, 65, 50, - 51, 52, 128, 65, 50, 51, 51, 128, 65, 50, 51, 50, 128, 65, 50, 51, 49, - 128, 65, 50, 51, 48, 128, 65, 50, 50, 57, 128, 65, 50, 50, 56, 128, 65, - 50, 50, 55, 65, 128, 65, 50, 50, 55, 128, 65, 50, 50, 54, 128, 65, 50, - 50, 53, 128, 65, 50, 50, 52, 128, 65, 50, 50, 51, 128, 65, 50, 50, 50, - 128, 65, 50, 50, 49, 128, 65, 50, 50, 48, 128, 65, 50, 49, 57, 128, 65, - 50, 49, 56, 128, 65, 50, 49, 55, 128, 65, 50, 49, 54, 65, 128, 65, 50, - 49, 54, 128, 65, 50, 49, 53, 65, 128, 65, 50, 49, 53, 128, 65, 50, 49, - 52, 128, 65, 50, 49, 51, 128, 65, 50, 49, 50, 128, 65, 50, 49, 49, 128, - 65, 50, 49, 48, 128, 65, 50, 48, 57, 65, 128, 65, 50, 48, 57, 128, 65, - 50, 48, 56, 128, 65, 50, 48, 55, 65, 128, 65, 50, 48, 55, 128, 65, 50, - 48, 54, 128, 65, 50, 48, 53, 128, 65, 50, 48, 52, 128, 65, 50, 48, 51, - 128, 65, 50, 48, 50, 66, 128, 65, 50, 48, 50, 65, 128, 65, 50, 48, 50, - 128, 65, 50, 48, 49, 128, 65, 50, 48, 48, 128, 65, 49, 57, 57, 128, 65, - 49, 57, 56, 128, 65, 49, 57, 55, 128, 65, 49, 57, 54, 128, 65, 49, 57, - 53, 128, 65, 49, 57, 52, 128, 65, 49, 57, 51, 128, 65, 49, 57, 50, 128, - 65, 49, 57, 49, 128, 65, 49, 57, 48, 128, 65, 49, 56, 57, 128, 65, 49, - 56, 56, 128, 65, 49, 56, 55, 128, 65, 49, 56, 54, 128, 65, 49, 56, 53, - 128, 65, 49, 56, 52, 128, 65, 49, 56, 51, 128, 65, 49, 56, 50, 128, 65, - 49, 56, 49, 128, 65, 49, 56, 48, 128, 65, 49, 55, 57, 128, 65, 49, 55, - 56, 128, 65, 49, 55, 55, 128, 65, 49, 55, 54, 128, 65, 49, 55, 53, 128, - 65, 49, 55, 52, 128, 65, 49, 55, 51, 128, 65, 49, 55, 50, 128, 65, 49, - 55, 49, 128, 65, 49, 55, 48, 128, 65, 49, 54, 57, 128, 65, 49, 54, 56, - 128, 65, 49, 54, 55, 128, 65, 49, 54, 54, 128, 65, 49, 54, 53, 128, 65, - 49, 54, 52, 128, 65, 49, 54, 51, 128, 65, 49, 54, 50, 128, 65, 49, 54, - 49, 128, 65, 49, 54, 48, 128, 65, 49, 53, 57, 128, 65, 49, 53, 56, 128, - 65, 49, 53, 55, 128, 65, 49, 53, 54, 128, 65, 49, 53, 53, 128, 65, 49, - 53, 52, 128, 65, 49, 53, 51, 128, 65, 49, 53, 50, 128, 65, 49, 53, 49, - 128, 65, 49, 53, 48, 128, 65, 49, 52, 57, 128, 65, 49, 52, 56, 128, 65, - 49, 52, 55, 128, 65, 49, 52, 54, 128, 65, 49, 52, 53, 128, 65, 49, 52, - 52, 128, 65, 49, 52, 51, 128, 65, 49, 52, 50, 128, 65, 49, 52, 49, 128, - 65, 49, 52, 48, 128, 65, 49, 51, 57, 128, 65, 49, 51, 56, 128, 65, 49, - 51, 55, 128, 65, 49, 51, 54, 128, 65, 49, 51, 53, 65, 128, 65, 49, 51, - 53, 128, 65, 49, 51, 52, 128, 65, 49, 51, 51, 128, 65, 49, 51, 50, 128, - 65, 49, 51, 49, 67, 128, 65, 49, 51, 49, 128, 65, 49, 51, 48, 128, 65, - 49, 50, 57, 128, 65, 49, 50, 56, 128, 65, 49, 50, 55, 128, 65, 49, 50, - 54, 128, 65, 49, 50, 53, 65, 128, 65, 49, 50, 53, 128, 65, 49, 50, 52, - 128, 65, 49, 50, 51, 128, 65, 49, 50, 50, 128, 65, 49, 50, 49, 128, 65, - 49, 50, 48, 66, 128, 65, 49, 50, 48, 128, 65, 49, 49, 57, 128, 65, 49, - 49, 56, 128, 65, 49, 49, 55, 128, 65, 49, 49, 54, 128, 65, 49, 49, 53, - 65, 128, 65, 49, 49, 53, 128, 65, 49, 49, 52, 128, 65, 49, 49, 51, 128, - 65, 49, 49, 50, 128, 65, 49, 49, 49, 128, 65, 49, 49, 48, 66, 128, 65, - 49, 49, 48, 65, 128, 65, 49, 49, 48, 128, 65, 49, 48, 57, 128, 65, 49, - 48, 56, 128, 65, 49, 48, 55, 67, 128, 65, 49, 48, 55, 66, 128, 65, 49, - 48, 55, 65, 128, 65, 49, 48, 55, 128, 65, 49, 48, 54, 128, 65, 49, 48, - 53, 66, 128, 65, 49, 48, 53, 65, 128, 65, 49, 48, 53, 128, 65, 49, 48, - 52, 67, 128, 65, 49, 48, 52, 66, 128, 65, 49, 48, 52, 65, 128, 65, 49, - 48, 52, 128, 65, 49, 48, 51, 128, 65, 49, 48, 50, 65, 128, 65, 49, 48, - 50, 128, 65, 49, 48, 49, 65, 128, 65, 49, 48, 49, 128, 65, 49, 48, 48, - 65, 128, 65, 49, 48, 48, 45, 49, 48, 50, 128, 65, 49, 48, 48, 128, 65, - 48, 57, 57, 128, 65, 48, 57, 56, 65, 128, 65, 48, 57, 56, 128, 65, 48, - 57, 55, 65, 128, 65, 48, 57, 55, 128, 65, 48, 57, 54, 128, 65, 48, 57, - 53, 128, 65, 48, 57, 52, 128, 65, 48, 57, 51, 128, 65, 48, 57, 50, 128, - 65, 48, 57, 49, 128, 65, 48, 57, 48, 128, 65, 48, 56, 57, 128, 65, 48, - 56, 56, 128, 65, 48, 56, 55, 128, 65, 48, 56, 54, 128, 65, 48, 56, 53, - 128, 65, 48, 56, 52, 128, 65, 48, 56, 51, 128, 65, 48, 56, 50, 128, 65, - 48, 56, 49, 128, 65, 48, 56, 48, 128, 65, 48, 55, 57, 128, 65, 48, 55, - 56, 128, 65, 48, 55, 55, 128, 65, 48, 55, 54, 128, 65, 48, 55, 53, 128, - 65, 48, 55, 52, 128, 65, 48, 55, 51, 128, 65, 48, 55, 50, 128, 65, 48, - 55, 49, 128, 65, 48, 55, 48, 128, 65, 48, 54, 57, 128, 65, 48, 54, 56, - 128, 65, 48, 54, 55, 128, 65, 48, 54, 54, 67, 128, 65, 48, 54, 54, 66, - 128, 65, 48, 54, 54, 65, 128, 65, 48, 54, 54, 128, 65, 48, 54, 53, 128, - 65, 48, 54, 52, 128, 65, 48, 54, 51, 128, 65, 48, 54, 50, 128, 65, 48, - 54, 49, 128, 65, 48, 54, 48, 128, 65, 48, 53, 57, 128, 65, 48, 53, 56, - 128, 65, 48, 53, 55, 128, 65, 48, 53, 54, 128, 65, 48, 53, 53, 128, 65, - 48, 53, 52, 128, 65, 48, 53, 51, 128, 65, 48, 53, 50, 128, 65, 48, 53, - 49, 128, 65, 48, 53, 48, 128, 65, 48, 52, 57, 128, 65, 48, 52, 56, 128, - 65, 48, 52, 55, 128, 65, 48, 52, 54, 66, 128, 65, 48, 52, 54, 65, 128, - 65, 48, 52, 54, 128, 65, 48, 52, 53, 65, 128, 65, 48, 52, 53, 128, 65, - 48, 52, 52, 128, 65, 48, 52, 51, 65, 128, 65, 48, 52, 51, 128, 65, 48, - 52, 50, 65, 128, 65, 48, 52, 50, 128, 65, 48, 52, 49, 65, 128, 65, 48, - 52, 49, 128, 65, 48, 52, 48, 65, 128, 65, 48, 52, 48, 128, 65, 48, 51, - 57, 65, 128, 65, 48, 51, 57, 128, 65, 48, 51, 56, 128, 65, 48, 51, 55, - 128, 65, 48, 51, 54, 128, 65, 48, 51, 53, 128, 65, 48, 51, 52, 128, 65, - 48, 51, 51, 128, 65, 48, 51, 50, 65, 128, 65, 48, 50, 56, 66, 128, 65, - 48, 50, 54, 65, 128, 65, 48, 49, 55, 65, 128, 65, 48, 49, 52, 65, 128, - 65, 48, 49, 48, 65, 128, 65, 48, 48, 54, 66, 128, 65, 48, 48, 54, 65, - 128, 65, 48, 48, 53, 65, 128, 65, 45, 87, 79, 128, 65, 45, 69, 85, 128, - 45, 85, 205, 45, 80, 72, 82, 85, 128, 45, 75, 72, 89, 85, 196, 45, 75, - 72, 89, 73, 76, 128, 45, 68, 90, 85, 196, 45, 67, 72, 65, 210, 45, 67, - 72, 65, 76, 128, -}; - -static const unsigned int lexicon_offset[] = { - 0, 0, 6, 11, 15, 19, 27, 34, 44, 49, 55, 64, 66, 69, 81, 89, 102, 108, - 113, 118, 124, 129, 137, 146, 149, 160, 165, 170, 176, 180, 189, 195, - 201, 207, 216, 224, 229, 237, 244, 177, 252, 255, 261, 262, 268, 273, - 277, 282, 289, 296, 306, 311, 317, 325, 330, 333, 339, 344, 350, 356, - 359, 365, 375, 380, 385, 390, 392, 394, 403, 405, 412, 354, 419, 427, - 436, 438, 441, 449, 454, 455, 457, 464, 472, 478, 484, 491, 496, 503, - 507, 512, 519, 524, 527, 531, 536, 542, 547, 557, 565, 572, 575, 585, - 593, 598, 606, 615, 618, 625, 629, 633, 637, 642, 645, 652, 659, 666, - 671, 676, 683, 692, 694, 703, 707, 715, 719, 727, 281, 736, 749, 753, - 758, 761, 765, 769, 771, 776, 786, 792, 796, 802, 806, 809, 814, 823, - 828, 832, 774, 838, 846, 857, 865, 870, 879, 888, 896, 902, 905, 910, - 918, 925, 928, 938, 943, 949, 953, 957, 960, 967, 974, 977, 981, 984, - 657, 990, 993, 996, 1002, 1007, 1016, 1021, 1027, 1031, 1034, 1038, 1041, - 1046, 1050, 617, 1055, 1058, 1067, 1070, 1075, 1080, 1086, 1091, 1096, - 1101, 1105, 1110, 1116, 1121, 1126, 1130, 1136, 1141, 1146, 1151, 1155, - 1160, 1165, 1170, 1176, 1182, 1188, 1193, 1197, 1202, 1207, 1212, 1216, - 1221, 1226, 1231, 1236, 1071, 1076, 1081, 1087, 1092, 1240, 1102, 1246, - 1251, 1256, 1263, 1267, 1270, 1279, 1106, 1283, 1111, 1117, 1122, 1287, - 1292, 1297, 1301, 1305, 1311, 1315, 1127, 1318, 1320, 1137, 1325, 1329, - 1142, 1335, 1147, 1339, 1343, 1350, 1152, 1354, 1362, 1367, 1371, 1374, - 1378, 1156, 1161, 1383, 1389, 1166, 1401, 1407, 1413, 1419, 1171, 1183, - 1189, 1423, 1427, 1431, 1434, 1194, 1438, 1440, 1445, 1450, 1456, 1461, - 1466, 1470, 1475, 1480, 1485, 1490, 1496, 1501, 1506, 1512, 1518, 1523, - 1527, 1532, 1537, 1542, 1547, 1552, 1556, 1564, 1569, 1573, 1578, 1583, - 1588, 1593, 1597, 1600, 1607, 1612, 1617, 1622, 1627, 1633, 1638, 1642, - 1198, 1645, 1651, 1656, 1661, 1666, 1203, 1670, 1674, 1681, 1688, 1208, - 1693, 1698, 1213, 1702, 1704, 1709, 1720, 1726, 1217, 1731, 1740, 1222, - 1745, 1751, 1756, 1761, 1771, 1780, 1788, 1227, 1798, 1807, 1816, 1821, - 1825, 1828, 1837, 1847, 1856, 1861, 1865, 1869, 1873, 1876, 1880, 1885, - 1232, 1895, 1237, 1899, 1901, 1907, 1913, 1919, 1925, 1931, 1937, 1943, - 1949, 1954, 1960, 1966, 1972, 1978, 1984, 1990, 1996, 2002, 2008, 2013, - 2018, 2023, 2028, 2033, 2038, 2043, 2048, 2053, 2058, 2064, 2069, 2075, - 2080, 2086, 2092, 2097, 2103, 2109, 2115, 2121, 2126, 2131, 2133, 2134, - 2138, 2142, 2147, 2151, 2155, 2159, 2164, 2168, 2171, 2176, 2180, 2185, - 2189, 2193, 2198, 2202, 2205, 2209, 2215, 2229, 2233, 2237, 2241, 2244, - 2249, 2253, 2257, 2260, 2264, 2269, 2274, 2279, 2284, 2288, 2292, 2296, - 2300, 2304, 2309, 2313, 2318, 2322, 2327, 2333, 2340, 2346, 2351, 2356, - 2361, 2367, 2372, 2378, 2383, 2388, 2393, 2398, 2403, 2406, 2408, 1088, - 2412, 2419, 2427, 2437, 2446, 2460, 2464, 2468, 2473, 2486, 2494, 2497, - 2501, 2504, 2509, 2513, 2516, 2520, 2524, 2529, 1715, 2534, 2538, 2541, - 2545, 2551, 2558, 2565, 2571, 2576, 2581, 2587, 2593, 2598, 2603, 2608, - 2613, 2618, 2623, 2548, 2628, 1706, 2630, 2636, 2640, 2645, 2649, 2653, - 1603, 1728, 2658, 2662, 2666, 2669, 2674, 2679, 2684, 2689, 2693, 2700, - 2705, 2708, 2712, 2716, 2723, 2729, 2733, 2739, 2743, 2747, 2752, 2759, - 2764, 2769, 2776, 2782, 2788, 2794, 2815, 2829, 2846, 2861, 2877, 2894, - 2909, 2918, 2923, 2927, 2932, 2937, 2941, 2953, 2960, 2966, 2336, 2972, - 2979, 2985, 2989, 2992, 2999, 3005, 3010, 3014, 3019, 3023, 3027, 2156, - 3031, 3036, 3041, 3045, 3050, 3058, 3062, 3069, 3074, 3078, 3082, 3086, - 3091, 3096, 3101, 3105, 3110, 3115, 3119, 3124, 3129, 3133, 3136, 3140, - 3144, 3152, 3157, 3161, 3165, 3171, 3180, 3184, 3188, 3194, 3199, 3206, - 3210, 3220, 3224, 3228, 3233, 3237, 3242, 3248, 3253, 3257, 3261, 3265, - 2561, 3273, 3278, 3284, 3289, 3293, 3298, 3303, 3307, 3313, 3318, 2160, - 3324, 3330, 3335, 3340, 3345, 3350, 3355, 3360, 3365, 3370, 3375, 3380, - 3385, 3390, 3395, 3400, 3406, 3411, 1103, 101, 3417, 3421, 3425, 3429, - 3434, 3438, 3442, 3448, 3453, 3457, 3461, 3466, 3471, 3475, 3480, 3484, - 3487, 3491, 3496, 3500, 3505, 3509, 3512, 3514, 3518, 3522, 3527, 3531, - 3534, 3547, 3551, 3555, 3559, 3564, 3568, 3572, 3575, 3579, 3583, 3588, - 3592, 3597, 3602, 3607, 3611, 3618, 3623, 3626, 3632, 3635, 3640, 3646, - 3650, 3654, 3657, 3662, 3666, 3671, 3675, 3679, 3682, 3688, 3693, 3698, - 3704, 3709, 3714, 3720, 3726, 3731, 3736, 3741, 3746, 3749, 979, 644, - 3755, 3758, 3763, 3767, 3771, 3775, 3779, 3782, 3786, 3791, 3796, 3800, - 3805, 3809, 3814, 3818, 3822, 3826, 3832, 3838, 3841, 3844, 153, 3850, - 3855, 3864, 3872, 3881, 3891, 3898, 3904, 3911, 3916, 3920, 3924, 3932, - 3939, 3944, 3949, 3956, 3961, 3965, 3975, 3979, 3983, 3988, 3993, 4003, - 2172, 4008, 4012, 4015, 4021, 4026, 4032, 4038, 4043, 4050, 4054, 4058, - 4062, 4067, 4072, 4077, 4082, 4087, 4092, 634, 616, 1264, 4097, 4104, - 4111, 4117, 4126, 4131, 4138, 4145, 4150, 4156, 4162, 4167, 4172, 4176, - 4182, 4189, 4194, 4198, 4202, 2181, 4208, 4216, 4222, 4230, 874, 4236, - 4244, 4255, 4259, 4269, 4275, 4280, 4285, 4290, 4295, 2186, 4300, 4305, - 4320, 4326, 4333, 4344, 4354, 4360, 4365, 4371, 4377, 4380, 4383, 4387, - 4392, 4395, 4402, 4411, 4416, 4420, 4424, 4428, 4432, 4437, 4443, 4454, - 4458, 3492, 4463, 4475, 4481, 4489, 4493, 4498, 4505, 4510, 4515, 4520, - 1472, 4525, 4528, 4531, 4535, 4538, 4544, 4548, 4562, 4566, 4569, 4573, - 4579, 4585, 4590, 4594, 4598, 4604, 4615, 4621, 4626, 4632, 4636, 4644, - 4656, 4666, 4672, 4677, 4686, 4694, 4705, 4712, 4718, 4724, 4728, 4734, - 4743, 4752, 4757, 4763, 4767, 4776, 4782, 4787, 4791, 4796, 4800, 4808, - 4814, 4818, 4825, 4830, 4834, 4840, 4846, 4853, 2194, 4862, 4873, 4883, - 4892, 4897, 4902, 4907, 4912, 1280, 4917, 4919, 4924, 4930, 4935, 4940, - 4945, 4950, 4955, 4960, 4966, 4971, 4977, 4982, 4987, 4992, 4998, 5003, - 5008, 5013, 5018, 5024, 5029, 5035, 5040, 5045, 5050, 5055, 5060, 5065, - 5071, 5076, 5081, 348, 389, 5086, 5092, 5095, 5099, 5103, 5110, 5116, - 5121, 5125, 5129, 5132, 5135, 5139, 5143, 5146, 5150, 5154, 5158, 5163, - 5167, 5171, 5177, 5186, 4843, 5191, 5195, 5198, 5203, 5208, 5213, 5218, - 5223, 5228, 5233, 5238, 5243, 5248, 5252, 5257, 5262, 5267, 5272, 5277, - 5282, 5287, 5292, 5297, 5302, 5306, 5311, 5316, 5321, 5326, 5331, 5336, - 5341, 5346, 5351, 5356, 5360, 5365, 5370, 5375, 5380, 5385, 5390, 5395, - 5400, 5405, 5410, 5414, 5419, 5424, 5429, 5434, 5439, 5444, 5449, 5454, - 5459, 5464, 5468, 5473, 5478, 5483, 5488, 5493, 5498, 5503, 5508, 5513, - 5518, 5522, 5527, 5532, 5537, 5542, 5547, 5552, 5557, 5562, 5567, 5572, - 5576, 5581, 5586, 5591, 5596, 5602, 5608, 5614, 5620, 5626, 5632, 5638, - 5643, 5649, 5655, 5661, 5667, 5673, 5679, 5685, 5691, 5697, 5703, 5708, - 5714, 5720, 5726, 5732, 5738, 5744, 5750, 5756, 5762, 5768, 5773, 5779, - 5785, 5791, 5797, 5803, 5809, 5815, 5821, 5827, 5833, 5838, 5844, 5850, - 5856, 5862, 5868, 5874, 5880, 5886, 5892, 5898, 5903, 5909, 5915, 5921, - 5927, 5933, 5939, 5945, 5951, 5957, 5963, 5968, 5972, 5978, 5984, 5990, - 5996, 6002, 6008, 6014, 6020, 6026, 6032, 6037, 6043, 6049, 6055, 6061, - 6067, 6073, 6079, 6085, 6091, 6097, 6102, 6108, 6114, 6120, 6126, 6132, - 6138, 6144, 6150, 6156, 6162, 6167, 6173, 6179, 6185, 6191, 6197, 6203, - 6209, 6215, 6221, 6227, 6232, 6238, 6244, 6250, 6256, 6262, 6268, 6274, - 6280, 6286, 6292, 6297, 6303, 6309, 6315, 6321, 6327, 6333, 6339, 6345, - 6351, 6357, 6362, 6368, 6374, 6380, 6386, 6392, 6398, 6404, 6410, 6416, - 6422, 6427, 6433, 6439, 6445, 6451, 6457, 6463, 6469, 6475, 6481, 6487, - 6492, 6498, 6504, 6510, 6516, 6522, 6528, 6534, 6540, 6546, 6552, 6557, - 6563, 6569, 6575, 6581, 6587, 6593, 6599, 6605, 6611, 6617, 6622, 6626, - 6629, 6637, 6644, 6647, 6651, 6664, 6668, 6672, 6676, 6679, 6683, 6688, - 6692, 6701, 6705, 6711, 6718, 6729, 6737, 6744, 6750, 6754, 6762, 6771, - 6777, 6781, 6793, 6798, 6801, 6806, 6810, 6820, 6828, 6836, 6844, 6850, - 6854, 6864, 6874, 6882, 6889, 6896, 6902, 6908, 6915, 6919, 6926, 6936, - 6946, 6954, 6961, 6966, 6970, 6974, 6982, 6986, 6996, 7001, 7008, 7015, - 7023, 7033, 7038, 7042, 7047, 7051, 7058, 7063, 7077, 7082, 7087, 7094, - 3768, 7103, 7107, 7111, 7116, 7120, 7124, 7127, 7132, 7137, 7146, 7152, - 7158, 7163, 7169, 7173, 7184, 7194, 7209, 7224, 7239, 7254, 7269, 7284, - 7299, 7314, 7329, 7344, 7359, 7374, 7389, 7404, 7419, 7434, 7449, 7464, - 7479, 7494, 7509, 7524, 7539, 7554, 7569, 7584, 7599, 7614, 7629, 7644, - 7659, 7674, 7689, 7704, 7719, 7734, 7749, 7764, 7779, 7794, 7809, 7824, - 7839, 7854, 7869, 7884, 7899, 7914, 7929, 7938, 7947, 7952, 7958, 7968, - 7972, 7976, 7981, 7986, 7991, 7999, 8003, 8006, 8010, 3215, 8013, 8018, - 353, 534, 8024, 8027, 8035, 8039, 8043, 8046, 8050, 8056, 8060, 8068, - 8074, 8079, 8086, 8094, 8101, 8107, 8112, 8119, 8125, 8134, 8142, 8146, - 8151, 8159, 8171, 8182, 8189, 8200, 8204, 8208, 8212, 8215, 8221, 3519, - 8225, 8227, 8233, 8238, 8243, 8248, 8254, 8259, 8264, 8269, 8274, 8280, - 8285, 8290, 8296, 8301, 8307, 8312, 8318, 8323, 8329, 8334, 8339, 8344, - 8349, 8354, 8360, 8365, 8370, 8375, 8381, 8387, 8393, 8399, 8405, 8411, - 8417, 8423, 8429, 8435, 8441, 8447, 8452, 8457, 8462, 8467, 8472, 8477, - 8482, 8487, 8493, 8499, 8504, 8510, 8516, 8522, 8528, 8533, 8538, 8543, - 8548, 8554, 8560, 8565, 8570, 8575, 8580, 8585, 8591, 8596, 8602, 8608, - 8614, 8620, 8626, 8632, 8638, 8644, 8650, 2203, 8045, 8655, 8659, 8667, - 8671, 8674, 8677, 8683, 8690, 1107, 8693, 8697, 8705, 8710, 8715, 8706, - 8720, 2230, 8724, 8730, 8736, 8741, 8746, 8753, 8761, 8766, 8770, 8773, - 8777, 8783, 8789, 8793, 1653, 631, 8796, 8800, 8805, 8811, 8816, 8820, - 8823, 8827, 8833, 8838, 8842, 8849, 8853, 8857, 8861, 773, 8681, 2254, - 8864, 8872, 8879, 8886, 8892, 8899, 8907, 8914, 8925, 8932, 8938, 8950, - 1123, 1288, 1293, 8961, 8965, 1298, 8969, 8973, 8982, 8990, 8994, 9003, - 9009, 9015, 9020, 9024, 9030, 9035, 9043, 9050, 2914, 9057, 9063, 9067, - 9076, 9085, 9094, 9103, 9109, 9114, 9119, 9130, 9139, 9151, 9156, 9164, - 2289, 9168, 9170, 9175, 9179, 9188, 9196, 1302, 168, 3810, 3815, 9202, - 9206, 9215, 9221, 9226, 9229, 9233, 9237, 9242, 9247, 9252, 9257, 9261, - 9270, 9276, 2301, 9280, 2906, 9284, 9292, 9296, 9300, 2305, 9304, 9308, - 9312, 9316, 9320, 2310, 9324, 9329, 9336, 9342, 9349, 9355, 9358, 9254, - 9360, 9368, 9376, 9384, 9387, 9392, 2323, 9397, 8717, 9400, 9402, 9407, - 9412, 9417, 9422, 9427, 9432, 9437, 9442, 9447, 9452, 9458, 9463, 9468, - 9473, 9479, 9484, 9489, 9494, 9499, 9504, 9509, 9515, 9520, 9525, 9530, - 9535, 9540, 9545, 9550, 9555, 9560, 9565, 9570, 9575, 9580, 9585, 9590, - 9595, 9600, 9606, 9612, 9617, 9622, 9627, 9632, 9637, 2334, 2341, 2347, - 9642, 9650, 9656, 9664, 2373, 2379, 9672, 2384, 2389, 2394, 2399, 9676, - 9680, 9685, 9689, 9693, 9697, 9702, 9706, 9711, 9715, 9718, 9721, 9727, - 9734, 9740, 9747, 9753, 9760, 9766, 9773, 9779, 9785, 9794, 9800, 9804, - 9808, 9812, 9816, 9821, 9825, 9830, 9834, 9840, 9844, 9849, 9856, 9867, - 9875, 9885, 9891, 9901, 9910, 9917, 9922, 9926, 9937, 9947, 9960, 9971, - 9984, 9995, 10007, 10019, 10031, 10042, 10055, 10068, 10075, 10081, - 10092, 10102, 10116, 10123, 10129, 10138, 10146, 10150, 10155, 10159, - 10166, 10174, 10181, 10185, 10191, 10195, 10201, 10211, 10215, 10220, - 10225, 10232, 10238, 8894, 10248, 10252, 10259, 10265, 10272, 10279, - 10283, 10286, 10292, 10296, 10301, 10306, 10311, 10315, 10321, 10329, - 10336, 10342, 10346, 10349, 10355, 10365, 10369, 10375, 10380, 10384, - 10389, 10393, 10399, 10405, 10410, 10416, 10421, 10426, 10431, 2226, - 10436, 10438, 10443, 10451, 10460, 10464, 10470, 10475, 10480, 10485, - 10490, 10496, 10501, 10506, 4600, 10511, 10516, 10520, 10526, 10531, - 10537, 10542, 10547, 10553, 10558, 10465, 10564, 10568, 10575, 10581, - 10586, 10590, 7073, 10595, 10604, 10609, 10614, 9332, 9339, 10619, 3088, - 10623, 10628, 10633, 10638, 10476, 10642, 10647, 10652, 10481, 10656, - 10486, 10661, 10668, 10675, 10681, 10688, 10694, 10700, 10705, 10712, - 10717, 10722, 10727, 10733, 10491, 10497, 10739, 10744, 10750, 10755, - 10760, 10768, 1358, 10773, 1048, 10776, 10784, 10800, 10816, 10831, - 10839, 10845, 10851, 10860, 10868, 10876, 10884, 10892, 10900, 10908, - 10916, 10924, 10933, 10942, 10950, 10959, 10968, 10977, 10986, 10995, - 11004, 11013, 11022, 11031, 11040, 11048, 11053, 11057, 11063, 11071, - 11078, 11093, 11110, 11129, 11138, 11146, 11161, 11172, 11180, 11186, - 11196, 11206, 11214, 11220, 11232, 11241, 11249, 11256, 11263, 11270, - 11276, 11281, 11291, 11297, 11305, 11315, 11322, 11332, 11342, 11352, - 11360, 11367, 11376, 11386, 11400, 11415, 11424, 11432, 11437, 11441, - 11451, 11461, 11473, 11482, 11488, 11493, 11503, 11513, 11523, 11528, - 11532, 11542, 11551, 11556, 11572, 11589, 11599, 11604, 11615, 11628, - 11639, 11647, 11660, 11672, 11680, 11685, 11689, 11695, 11700, 11708, - 11716, 11723, 11734, 11739, 11747, 11757, 11763, 11767, 11770, 11776, - 11780, 11786, 11793, 11797, 11805, 11814, 11822, 11829, 11834, 11838, - 11843, 11847, 11851, 11859, 11874, 11890, 11896, 11904, 11913, 11921, - 11927, 11931, 11938, 11949, 11953, 11956, 11967, 11973, 11978, 10507, - 11986, 11992, 11999, 12005, 12010, 12017, 12024, 12031, 12038, 12045, - 12052, 12059, 12066, 12073, 12080, 12087, 12094, 12101, 12108, 12115, - 12120, 11106, 12125, 12131, 12138, 12145, 12150, 12157, 12166, 12170, - 12177, 12189, 12193, 12199, 12204, 12209, 12214, 12219, 12224, 12229, - 12232, 12236, 11457, 12240, 12244, 12250, 12256, 12261, 12267, 12272, - 12277, 12283, 12288, 12293, 10228, 12298, 12302, 12306, 12310, 12315, - 12320, 12325, 12333, 12339, 12344, 12348, 12352, 12359, 12364, 12372, - 12379, 12384, 12388, 12391, 12397, 12404, 12408, 12411, 12416, 12420, - 4639, 12426, 12435, 46, 12443, 12449, 12454, 12459, 12467, 12474, 12479, - 6991, 12485, 12491, 12496, 12500, 12503, 12509, 12517, 12524, 12539, - 12558, 12570, 12583, 12596, 12609, 12623, 12636, 12651, 12658, 10512, - 12664, 12678, 12683, 12689, 12694, 12702, 12707, 9072, 12712, 12715, - 12723, 12730, 12735, 12739, 12745, 12749, 12754, 12759, 12764, 12769, - 12774, 12779, 3093, 11194, 12784, 12788, 12794, 12800, 12805, 12811, - 12816, 10521, 12822, 12828, 12833, 12838, 12846, 12852, 12865, 12873, - 12880, 12886, 10527, 12892, 12900, 12908, 12915, 12928, 12941, 12953, - 12963, 12975, 13003, 13011, 13020, 13027, 13039, 13046, 13056, 13065, - 13073, 13080, 13085, 13091, 10532, 13096, 13102, 13107, 13112, 13117, - 10538, 13122, 13125, 13132, 13138, 13152, 13165, 13176, 9860, 13187, - 13193, 13202, 13210, 13217, 13223, 13234, 13240, 13245, 13253, 4113, - 13259, 13264, 12531, 13270, 13277, 13282, 10543, 13288, 13293, 13300, - 13306, 13312, 13317, 13325, 13333, 13340, 13344, 13356, 13370, 13380, - 13385, 13389, 13400, 13406, 13411, 13416, 10548, 13420, 10554, 13425, - 13428, 13433, 13445, 13452, 13457, 13461, 13469, 13474, 13478, 13483, - 13487, 13494, 13500, 10559, 10466, 13507, 3098, 17, 13514, 13519, 13523, - 13527, 13533, 13541, 13551, 13556, 13561, 13568, 13575, 13579, 13590, - 13600, 13609, 13618, 13630, 13635, 13639, 13647, 13661, 13665, 13668, - 13672, 13680, 13687, 13695, 13699, 13710, 13718, 13722, 13729, 13734, - 13738, 13744, 13749, 13755, 13760, 13765, 13769, 13775, 13780, 13791, - 13795, 13798, 13804, 13811, 13817, 13822, 13828, 13834, 13841, 13852, - 13862, 13872, 13881, 13888, 13897, 13901, 10569, 10576, 10582, 10587, - 13907, 13913, 13919, 13924, 13930, 10591, 13936, 13939, 13946, 13951, - 13957, 13962, 13977, 13993, 14008, 14016, 14021, 14028, 14034, 14038, - 14043, 14048, 14053, 14058, 14063, 14068, 14073, 14078, 14083, 1561, 383, - 14088, 14096, 14103, 14109, 14114, 14119, 10596, 14121, 14125, 14130, - 14134, 14144, 14149, 14153, 14156, 14165, 14169, 14172, 14179, 10605, - 14184, 14187, 14195, 14202, 14210, 14214, 14220, 14228, 14232, 14239, - 14248, 14255, 14251, 14262, 14266, 14272, 14276, 14280, 14284, 14290, - 14296, 14306, 14314, 14321, 14325, 14333, 14338, 14342, 14349, 14354, - 14361, 14365, 14370, 14375, 14379, 14386, 14392, 14400, 14406, 14411, - 14421, 14428, 14433, 14438, 14442, 14446, 14454, 4469, 14462, 14467, - 10610, 14471, 14478, 14482, 14485, 14493, 14500, 14504, 14507, 6846, - 14511, 14516, 14521, 14525, 14536, 14546, 14551, 14557, 14562, 14571, - 14575, 14578, 14586, 14591, 14596, 14603, 14608, 4865, 10615, 14613, - 14617, 14624, 14629, 14634, 14639, 7021, 14644, 14649, 14654, 14659, - 14665, 14670, 14676, 14681, 14686, 14691, 14696, 14701, 14706, 14711, - 14716, 14721, 14726, 14731, 14736, 14741, 14746, 14751, 14756, 14762, - 14767, 14772, 14777, 14782, 14787, 14793, 14798, 14803, 14809, 14814, - 14820, 14825, 14831, 14836, 14841, 14846, 14851, 14857, 14862, 14867, - 14872, 14880, 988, 112, 14886, 14890, 14895, 14900, 14904, 14908, 14912, - 14917, 14921, 14926, 14930, 14933, 14937, 14941, 14947, 14952, 14962, - 14968, 14976, 14982, 14986, 14990, 14997, 15005, 15014, 15025, 15035, - 15042, 15049, 15053, 15062, 15071, 15079, 15086, 15095, 15104, 15113, - 15122, 15132, 15142, 15152, 15162, 15172, 15181, 15191, 15201, 15211, - 15221, 15231, 15241, 15251, 15260, 15270, 15280, 15290, 15300, 15310, - 15320, 15329, 15339, 15349, 15359, 15369, 15379, 15389, 15399, 15409, - 15419, 15428, 15438, 15448, 15458, 15468, 15478, 15488, 15498, 15508, - 15518, 15528, 15537, 15543, 1132, 15547, 15550, 15554, 15559, 15566, - 15572, 15577, 15581, 15586, 15595, 15604, 15612, 15617, 15621, 15625, - 15631, 15636, 15642, 10624, 15647, 15652, 15661, 15666, 10634, 15671, - 11444, 11454, 11464, 15674, 15680, 15688, 10639, 15695, 15699, 15703, - 15709, 15714, 15718, 15728, 15734, 15740, 15745, 15754, 15762, 15769, - 15776, 15781, 15788, 15793, 15797, 15800, 15811, 15821, 15834, 15843, - 15851, 15862, 15874, 15884, 15894, 15899, 15903, 15908, 15913, 15917, - 15923, 15931, 15938, 15949, 15954, 15964, 15973, 15977, 15980, 15987, - 15997, 16006, 16013, 16017, 16024, 16030, 16035, 16040, 16044, 15590, - 16053, 16057, 16063, 16067, 16072, 16076, 16083, 16090, 16094, 16103, - 16111, 16119, 16126, 16134, 16146, 16158, 16169, 16179, 16186, 16192, - 16201, 16212, 16221, 16233, 16245, 16257, 16267, 16276, 16286, 16295, - 16303, 16310, 16320, 16329, 16337, 16341, 16346, 16352, 16358, 16363, - 16368, 16372, 16377, 16382, 16387, 16392, 16397, 16402, 16407, 8738, - 16412, 16414, 16418, 16423, 16429, 16436, 16442, 16448, 16457, 16461, - 16467, 16475, 16482, 16491, 16500, 16509, 16518, 16527, 16536, 16545, - 16554, 16564, 16574, 16583, 16589, 16596, 16603, 16609, 16623, 16629, - 16636, 16644, 16653, 16661, 16667, 16676, 16682, 16691, 16702, 16708, - 16718, 16726, 16733, 16741, 16749, 16756, 16765, 16778, 16787, 16795, - 16802, 16815, 16821, 16827, 16837, 16846, 16855, 16864, 16872, 16877, - 16881, 16887, 16893, 16898, 16905, 16912, 10242, 16917, 16922, 16929, - 16937, 16942, 16954, 16961, 16966, 16978, 14943, 16983, 16989, 16997, - 17003, 17008, 17016, 17024, 17031, 17039, 17046, 17052, 17058, 17066, - 17074, 17080, 17088, 17094, 17099, 17105, 17112, 17118, 17123, 17127, - 17138, 17146, 17154, 17160, 17165, 17172, 17181, 17187, 17192, 17200, - 17207, 17216, 17230, 4413, 17234, 17239, 17244, 17250, 17255, 17260, - 17264, 17269, 17274, 17279, 8737, 17284, 17289, 17294, 17299, 17304, - 17308, 17313, 17318, 17323, 17328, 17334, 17340, 14216, 17345, 17351, - 17356, 17361, 17366, 10643, 17371, 17376, 17381, 17386, 17391, 17405, - 17422, 17440, 17452, 17465, 17482, 17498, 17515, 17525, 17544, 17555, - 17566, 17577, 2803, 17588, 17599, 17610, 17627, 17638, 17649, 17654, - 10648, 17659, 17663, 2483, 17667, 17673, 17676, 17682, 17690, 17698, - 17704, 17713, 17720, 17725, 17733, 17741, 17748, 17752, 17757, 17763, - 17770, 17778, 17785, 17797, 17804, 17810, 17818, 17823, 17829, 17835, - 17840, 13971, 17847, 17851, 17860, 17866, 17871, 17879, 17888, 17896, - 17903, 17909, 17917, 17924, 17930, 17936, 17943, 17950, 17956, 17962, - 17966, 17975, 17983, 17988, 17998, 18005, 18011, 18019, 18025, 18033, - 18041, 18048, 18061, 18065, 18072, 18081, 18090, 18099, 18107, 18117, - 18124, 18129, 3969, 18136, 18141, 1248, 18145, 18152, 17285, 18156, - 18162, 18166, 18174, 18186, 18191, 18198, 18204, 18209, 18216, 17290, - 18220, 18224, 18232, 18237, 18241, 17295, 18245, 17300, 18249, 18256, - 18261, 18265, 18272, 18276, 18279, 18287, 18294, 18299, 18307, 18311, - 18318, 18335, 18344, 18353, 18357, 18360, 18366, 18374, 18380, 18385, - 18389, 18394, 18399, 18404, 18409, 18414, 18419, 4047, 18424, 18426, - 18434, 18441, 18451, 18463, 18468, 18472, 18478, 18483, 18491, 18495, - 18501, 18506, 18512, 18515, 18522, 18530, 18537, 18543, 18548, 18554, - 18559, 18566, 18572, 18577, 18587, 18596, 18603, 18608, 18612, 18618, - 18624, 18628, 18635, 18641, 18646, 18652, 18660, 18668, 18675, 18681, - 18687, 18692, 18698, 18704, 18712, 18717, 18722, 18730, 18736, 18742, - 18747, 18754, 18759, 18763, 18769, 18775, 18780, 18786, 18793, 18798, - 18804, 18807, 18813, 18824, 18830, 18839, 18842, 18846, 18850, 18864, - 18877, 18889, 18895, 18900, 18907, 18913, 18919, 18930, 18942, 18954, - 18964, 18973, 18981, 18988, 18999, 19009, 19019, 19027, 19030, 17314, - 19035, 19040, 19047, 17319, 17470, 19055, 19068, 19083, 19094, 17487, - 19112, 19125, 19138, 19149, 12546, 19160, 19173, 19192, 19203, 19214, - 19225, 2824, 19238, 19242, 19250, 19261, 19272, 19280, 19295, 19310, - 19321, 19328, 19334, 19342, 19346, 19352, 19356, 19359, 19372, 19384, - 19394, 19402, 19409, 19417, 19427, 19432, 19439, 19444, 19451, 19462, - 19472, 19478, 19483, 19488, 17324, 19492, 19498, 19505, 19511, 19516, - 19521, 19526, 19530, 17329, 17335, 19534, 17341, 19539, 19547, 19552, - 19556, 19563, 19571, 19578, 19587, 19594, 19598, 19602, 19607, 19612, - 19617, 19622, 19627, 10487, 19632, 19634, 19639, 19644, 19650, 19655, - 19660, 19665, 19670, 19674, 19680, 19686, 19691, 19697, 19702, 19707, - 19711, 19717, 19722, 19726, 19731, 19736, 19748, 19753, 19759, 19764, - 19769, 19775, 19781, 19786, 19791, 19796, 19803, 19809, 19820, 19827, - 19836, 19841, 19845, 279, 19849, 19857, 19862, 19868, 19874, 19879, - 19886, 19893, 19899, 19904, 19910, 19915, 19920, 19925, 19932, 19942, - 19950, 19955, 19960, 19967, 19973, 19982, 19992, 20002, 20016, 20030, - 20044, 20058, 20073, 20088, 20105, 20123, 20136, 20142, 20147, 20152, - 20156, 20164, 20169, 20177, 20183, 20189, 20194, 20199, 20203, 20209, - 20214, 20218, 20225, 20230, 20234, 20245, 20251, 20256, 20261, 20268, - 20273, 20277, 3927, 20282, 20288, 20295, 17346, 20301, 20305, 20311, - 20316, 20321, 20325, 20331, 20336, 20341, 20348, 20353, 15730, 20357, - 20362, 20366, 20371, 20377, 20383, 20390, 20400, 20408, 20415, 20420, - 20424, 20433, 20441, 20448, 20455, 20461, 20466, 20472, 20477, 20482, - 20488, 20493, 20499, 20504, 20510, 20516, 20523, 20529, 20534, 20539, - 10713, 20548, 20551, 20559, 20565, 20570, 20575, 20585, 20592, 20598, - 20603, 20608, 20614, 20619, 20625, 20630, 20636, 20643, 20649, 20655, - 20660, 20668, 20675, 20680, 20685, 20691, 20696, 20700, 20709, 20720, - 20727, 20734, 20742, 20749, 20756, 20761, 20766, 20772, 20777, 20785, - 20791, 20797, 20802, 20809, 20815, 20820, 20824, 20830, 20835, 20840, - 20844, 20849, 1321, 8762, 3112, 20853, 20857, 20861, 20865, 20869, 20873, - 20876, 20881, 20888, 20896, 20906, 20917, 20927, 20938, 20950, 20961, - 20971, 20982, 20994, 21005, 21017, 21030, 21042, 21053, 21063, 21074, - 21086, 21097, 21110, 21122, 21133, 21145, 21158, 21170, 21183, 21197, - 21210, 21222, 21233, 21243, 21254, 21266, 21277, 21289, 21302, 21314, - 21325, 21337, 21350, 21363, 21377, 21390, 21402, 21413, 21425, 21438, - 21450, 21463, 21477, 21490, 21502, 21515, 21529, 21542, 21556, 21570, - 21583, 21595, 21606, 21616, 17357, 21623, 21629, 21639, 21647, 21654, - 21662, 21672, 21681, 21694, 21699, 21704, 21712, 21719, 15839, 15848, - 21726, 21736, 21751, 21757, 21764, 21771, 21778, 21784, 21790, 21801, - 21809, 21817, 21827, 21837, 21846, 17362, 21855, 21861, 21867, 21876, - 21884, 21892, 21897, 21906, 21914, 21926, 21936, 21946, 21956, 21965, - 21977, 21987, 21997, 22008, 22015, 22020, 22027, 22039, 22051, 22063, - 22075, 22087, 22099, 22111, 22123, 22135, 22147, 22158, 22170, 22182, - 22194, 22206, 22218, 22230, 22242, 22254, 22266, 22278, 22289, 22301, - 22313, 22325, 22337, 22349, 22361, 22373, 22385, 22397, 22409, 22420, - 22432, 22444, 22456, 22468, 22480, 22492, 22504, 22516, 22528, 22540, - 22551, 22563, 22575, 22587, 22599, 22611, 22623, 22635, 22647, 22659, - 22671, 22682, 22694, 22706, 22718, 22730, 22742, 22754, 22766, 22778, - 22790, 22802, 22813, 22825, 22837, 22849, 22861, 22873, 22885, 22897, - 22909, 22921, 22933, 22944, 22956, 22968, 22980, 22992, 23005, 23018, - 23031, 23044, 23057, 23070, 23083, 23095, 23108, 23121, 23134, 23147, - 23160, 23173, 23186, 23199, 23212, 23225, 23237, 23250, 23263, 23276, - 23289, 23302, 23315, 23328, 23341, 23354, 23367, 23379, 23392, 23405, - 23418, 23431, 23444, 23457, 23470, 23483, 23496, 23509, 23521, 23534, - 23547, 23560, 23573, 23586, 23599, 23612, 23625, 23638, 23651, 23663, - 23676, 23689, 23702, 23715, 23728, 23741, 23754, 23767, 23780, 23793, - 23805, 23816, 23829, 23842, 23855, 23868, 23881, 23894, 23907, 23920, - 23933, 23946, 23958, 23971, 23984, 23997, 24010, 24023, 24036, 24049, - 24062, 24075, 24088, 24100, 24113, 24126, 24139, 24152, 24165, 24178, - 24191, 24204, 24217, 24230, 24242, 24255, 24268, 24281, 24294, 24307, - 24320, 24333, 24346, 24359, 24372, 24384, 24397, 24410, 24423, 24436, - 24449, 24462, 24475, 24488, 24501, 24514, 24526, 24539, 24552, 24565, - 24578, 24591, 24604, 24617, 24630, 24643, 24656, 24668, 24681, 24694, - 24707, 24720, 24733, 24746, 24759, 24772, 24785, 24798, 24810, 24823, - 24836, 24849, 24862, 24875, 24888, 24901, 24914, 24927, 24940, 24952, - 24965, 24978, 24991, 25004, 25017, 25030, 25043, 25056, 25069, 25082, - 25094, 25107, 25120, 25133, 25146, 25159, 25172, 25185, 25198, 25211, - 25224, 25236, 25247, 25256, 25264, 25272, 25279, 25285, 25289, 25295, - 25301, 25310, 25318, 25323, 25329, 25334, 25338, 25347, 10492, 25358, - 25364, 25371, 25379, 25386, 13145, 13159, 25393, 25400, 25409, 25414, - 25419, 25426, 25431, 25436, 8778, 8784, 8790, 25441, 25446, 25449, 25454, - 25462, 25469, 25476, 25488, 25495, 25501, 25510, 25515, 25524, 25533, - 25539, 25547, 25556, 25560, 25566, 25571, 25581, 25588, 25594, 25602, - 25608, 25615, 25621, 25631, 25640, 25644, 25651, 25655, 25660, 25666, - 25674, 25678, 25688, 17372, 25697, 25703, 25707, 25716, 25725, 25735, - 25741, 17377, 25748, 25755, 25766, 25774, 25784, 25793, 25801, 10207, - 25809, 25814, 25820, 25825, 25829, 25833, 25837, 11301, 25842, 25850, - 25857, 25866, 25874, 25881, 25888, 25897, 25903, 1062, 25910, 25916, - 25920, 25926, 25933, 25939, 25947, 25953, 25960, 25966, 25972, 25981, - 25985, 25993, 26001, 26008, 26017, 26024, 26029, 26033, 26043, 26054, - 26065, 26070, 26075, 26081, 26090, 26095, 26108, 9000, 26112, 26118, - 26124, 26130, 26135, 26143, 26147, 26154, 26163, 26168, 17650, 26176, - 26180, 26192, 26197, 26201, 26204, 26210, 26216, 26222, 26227, 26232, - 26236, 26239, 26250, 26255, 10769, 26262, 26267, 26272, 26277, 26282, - 26287, 26292, 26297, 26302, 10774, 26307, 26312, 26317, 26322, 26327, - 26332, 26337, 26342, 26347, 26352, 26357, 26362, 26368, 26373, 26378, - 26383, 26388, 26393, 26398, 26403, 26408, 26413, 26419, 26425, 26430, - 26435, 26440, 26445, 26450, 26455, 26460, 26465, 26470, 26476, 26481, - 26486, 26491, 26497, 26503, 26508, 26513, 26518, 26523, 26528, 26533, - 26538, 26543, 26549, 26554, 26559, 26564, 26569, 26575, 26580, 26585, - 26589, 1244, 145, 26597, 26601, 26605, 26609, 26614, 26618, 15736, 2409, - 26622, 26627, 26631, 26636, 26640, 26645, 26649, 26655, 26660, 26664, - 26668, 26676, 26680, 26684, 26691, 26696, 26701, 26705, 26711, 26716, - 26720, 26725, 26730, 26734, 26741, 26748, 26755, 26760, 26764, 26768, - 26773, 26777, 26780, 26786, 26799, 26804, 26810, 26819, 26824, 11049, - 26829, 26838, 26843, 26846, 26850, 26855, 26860, 26865, 26870, 26875, - 2920, 2925, 26880, 26886, 26890, 26896, 3888, 26901, 26906, 26911, 26917, - 26922, 16698, 26927, 26932, 26937, 26942, 26948, 26953, 26958, 26964, - 26969, 26973, 26978, 26983, 26988, 26993, 26998, 27002, 27007, 27011, - 27016, 27021, 27026, 27031, 27035, 27040, 27044, 27049, 27054, 27059, - 26974, 3121, 26979, 27064, 27072, 27079, 11395, 27091, 27099, 27109, - 27127, 27146, 27155, 27163, 26984, 27170, 27175, 27183, 26989, 27188, - 27193, 27201, 27206, 27211, 27215, 19870, 27220, 27228, 27233, 27237, - 27244, 27250, 27259, 27263, 27271, 27277, 27281, 27284, 20704, 27291, - 27295, 27299, 27304, 27310, 27317, 27322, 10234, 27326, 27331, 27336, - 27341, 27346, 27351, 1663, 1668, 27356, 27362, 27368, 27373, 27377, - 27381, 27385, 27389, 27393, 27397, 27401, 27405, 25482, 27408, 27415, - 27423, 27429, 27435, 27440, 27445, 27451, 27455, 27460, 27467, 16598, - 16605, 27473, 27485, 27488, 27495, 27499, 19895, 27506, 27514, 27525, - 27534, 27547, 27557, 27571, 27583, 27597, 27610, 27622, 27632, 27644, - 27650, 27665, 27689, 27707, 27726, 27739, 27753, 27771, 27787, 27804, - 27822, 27833, 27852, 27869, 27889, 27907, 27919, 27933, 27947, 27959, - 27976, 27995, 28013, 28025, 28043, 28062, 17530, 28075, 28095, 28107, - 12577, 28119, 28124, 28129, 28134, 28143, 28149, 28154, 28158, 28165, - 28171, 28175, 28180, 28185, 28190, 28195, 28200, 28205, 2506, 28210, - 28216, 28220, 28223, 28234, 28238, 28241, 28249, 28255, 14882, 28259, - 28268, 28279, 28285, 28291, 28306, 28315, 28323, 28330, 28335, 28339, - 28346, 28352, 28361, 28369, 28376, 28386, 28395, 28405, 28410, 28419, - 28428, 28439, 28450, 28460, 28477, 4557, 28487, 28491, 28501, 28509, - 28519, 28530, 28536, 28541, 28551, 28559, 28566, 28572, 28579, 28584, - 27022, 28588, 28597, 28601, 28604, 28609, 28617, 28624, 28633, 28641, - 28649, 28657, 28667, 28676, 28682, 28688, 28694, 28698, 27027, 27032, - 28702, 28712, 28722, 28732, 28740, 28747, 28757, 28765, 28773, 28779, - 28787, 28798, 798, 28807, 17737, 649, 28821, 28830, 28838, 28849, 28860, - 28870, 28879, 28891, 28900, 28909, 28916, 28922, 28932, 28941, 28950, - 28958, 28966, 28976, 28984, 28992, 28999, 29005, 29010, 29015, 29020, - 8156, 29025, 29028, 29032, 29037, 29045, 29051, 29056, 29060, 3751, - 27045, 29068, 27050, 29074, 29080, 29086, 29091, 29096, 29100, 29108, - 29114, 29120, 29124, 3912, 29132, 29137, 29142, 29146, 29150, 11681, - 29157, 29165, 29179, 29186, 29193, 29199, 11690, 11696, 29207, 29215, - 29222, 29227, 29232, 27055, 29238, 29249, 29258, 19043, 29266, 29271, - 2755, 29276, 29287, 29293, 29298, 29302, 29306, 29309, 29316, 29323, - 29329, 29337, 29344, 29350, 29354, 8196, 29359, 29363, 29367, 29375, - 29380, 29385, 29390, 1696, 29395, 29400, 29405, 29410, 29415, 29420, - 29425, 29430, 29435, 29440, 29445, 29450, 29455, 29460, 29466, 29471, - 29476, 29481, 29486, 29491, 29496, 29502, 29507, 29512, 29517, 29522, - 29527, 29532, 29537, 29543, 29549, 29554, 29560, 29565, 29570, 5, 29576, - 29580, 29584, 29588, 29593, 29597, 29601, 29605, 29609, 29614, 29618, - 29623, 29627, 29630, 29634, 29639, 29643, 29648, 29652, 29656, 29660, - 29665, 29669, 29673, 29683, 29688, 29692, 29696, 29701, 29706, 29715, - 29720, 29725, 29729, 29733, 29742, 29755, 29767, 29776, 29785, 29790, - 29796, 29801, 29805, 29809, 29819, 29828, 29836, 29842, 29847, 29851, - 29858, 29865, 29875, 29884, 29892, 12934, 29900, 29907, 29915, 29924, - 29933, 29941, 29951, 29956, 29960, 29964, 29967, 29969, 29973, 29977, - 29982, 29987, 29991, 29995, 29998, 30002, 30005, 30009, 30012, 30015, - 30019, 30025, 30029, 30033, 30037, 30041, 30046, 30051, 30056, 30060, - 30063, 30068, 30074, 30079, 30085, 30090, 30094, 30100, 30104, 30108, - 30113, 30117, 30122, 30127, 30131, 30135, 30142, 30146, 30149, 30153, - 30157, 30163, 30168, 30174, 30178, 30182, 30187, 30194, 30200, 30204, - 30213, 30217, 30221, 30224, 30230, 30235, 30241, 1385, 1748, 30246, - 30251, 30256, 30261, 30266, 30271, 30276, 2213, 827, 30281, 30284, 30288, - 30292, 30297, 30301, 17749, 30305, 30310, 30315, 30319, 30322, 30327, - 30331, 30336, 30340, 17753, 30345, 30348, 30351, 30357, 30361, 30366, - 30370, 30383, 30391, 30395, 30398, 30406, 30415, 30422, 30427, 30433, - 30439, 30447, 30454, 30461, 30465, 30469, 30473, 30478, 30483, 30487, - 30495, 30500, 30507, 30519, 30530, 30535, 30539, 30546, 30550, 30555, - 30561, 30564, 30569, 30574, 30581, 30585, 30589, 30592, 30598, 8900, - 2413, 30602, 30607, 30623, 11100, 30643, 30652, 30668, 30672, 30679, - 30682, 30688, 30698, 30704, 30713, 30722, 30737, 30748, 30760, 30771, - 30779, 30788, 30794, 30803, 30813, 30823, 30834, 30845, 30855, 30864, - 30871, 30880, 30888, 30895, 30902, 30909, 30917, 30924, 30931, 30944, - 30951, 30959, 30966, 30972, 30977, 30986, 30993, 30999, 31004, 31012, - 31020, 31027, 31034, 28511, 31046, 31058, 31072, 31080, 31088, 31096, - 31103, 31115, 31124, 31133, 31141, 31149, 31157, 31164, 31170, 31179, - 31187, 31197, 31206, 31216, 31225, 31234, 31242, 31247, 31251, 31254, - 31258, 31262, 31266, 31270, 31274, 31280, 31286, 31291, 31299, 31306, - 31314, 31321, 10806, 17811, 31329, 31336, 31341, 31348, 31354, 31360, - 31367, 14024, 31374, 31377, 31389, 31397, 31403, 31408, 31412, 31423, - 31433, 31443, 11620, 31452, 31461, 31469, 31479, 31488, 31495, 31502, - 31510, 31514, 17830, 31517, 31524, 31528, 4501, 31534, 31537, 31544, - 31550, 31564, 31569, 31577, 31583, 31594, 31601, 31607, 31613, 31617, - 31622, 31626, 31635, 31642, 31648, 8953, 31655, 31663, 31670, 31676, - 31681, 31687, 31693, 31703, 31715, 31726, 31736, 11252, 31744, 31750, - 17848, 31754, 31756, 31259, 11633, 31765, 31770, 31776, 31786, 31791, - 31798, 31806, 31812, 31817, 31822, 31827, 31831, 31836, 31843, 31849, - 31858, 31866, 31870, 31877, 31887, 31893, 31902, 31908, 31915, 4771, - 31921, 31927, 31932, 31939, 31951, 31962, 31967, 31975, 31979, 31989, - 31995, 31999, 32004, 32014, 32023, 32027, 32034, 32042, 32049, 32055, - 32060, 32068, 32075, 32080, 32087, 32099, 32108, 32112, 32120, 15656, - 32124, 32134, 32138, 32146, 32153, 32160, 30402, 32171, 32176, 32180, - 32187, 32194, 26707, 31184, 32199, 32203, 32206, 27839, 32211, 32225, - 32241, 32259, 32278, 32295, 32313, 27858, 32330, 32350, 27875, 32362, - 32374, 19099, 32386, 27895, 32400, 32412, 12590, 32426, 32431, 32436, - 32441, 32447, 32453, 32459, 32463, 32471, 32477, 32484, 32489, 32499, - 32506, 32512, 12128, 32518, 32520, 32525, 32533, 32537, 31839, 32543, - 32550, 13866, 13876, 32557, 32564, 32574, 32579, 32583, 32586, 32592, - 32600, 32612, 32622, 32638, 32651, 32665, 19117, 32679, 32686, 32690, - 32693, 32698, 32702, 32709, 32716, 32723, 32730, 32740, 32745, 32750, - 32755, 32763, 32771, 32776, 32785, 28532, 3561, 32790, 32793, 32796, - 32801, 32808, 32813, 32818, 32834, 32842, 32850, 10864, 32858, 32863, - 32867, 32873, 32878, 32884, 32887, 32893, 32905, 32913, 32920, 32926, - 32933, 32944, 32958, 32971, 32977, 32986, 32992, 33001, 33013, 33024, - 33034, 33043, 33052, 33060, 33070, 33079, 33090, 673, 33097, 33104, - 33110, 33115, 33121, 33128, 33134, 33145, 33155, 33165, 33174, 33180, - 33187, 33192, 33200, 33207, 33215, 33223, 33235, 7142, 33242, 33245, - 33254, 33262, 33268, 33274, 33279, 33283, 33286, 33292, 33299, 33304, - 33309, 33316, 33321, 33325, 33337, 33348, 33357, 33365, 18020, 33370, - 33378, 33383, 33391, 33397, 33403, 33408, 13859, 9802, 33411, 33415, - 33419, 33422, 33425, 33431, 33439, 33447, 33451, 33455, 33460, 33464, - 33467, 33476, 33481, 33486, 33490, 33493, 33498, 33506, 33517, 33526, - 33530, 33536, 33542, 33546, 33552, 33560, 33582, 33606, 33617, 33626, - 33632, 33639, 33646, 33652, 33660, 33666, 33671, 33682, 33700, 33707, - 33715, 33719, 33726, 33731, 33740, 33753, 33761, 33773, 33784, 33795, - 33805, 33819, 33828, 33836, 33848, 33859, 11117, 33868, 33879, 33890, - 33902, 33912, 33921, 33931, 33936, 33940, 33948, 33959, 33969, 33975, - 33980, 33984, 33987, 33990, 33998, 34006, 34015, 34025, 34034, 34040, - 34045, 34059, 2838, 34081, 34092, 34101, 34111, 34123, 34132, 34141, - 34151, 34159, 34167, 34176, 34181, 34192, 34197, 34206, 34212, 34223, - 34227, 34230, 34240, 34249, 34257, 34267, 34277, 34285, 34294, 34301, - 34307, 34315, 34322, 34331, 34340, 34345, 34350, 34354, 34362, 34369, - 34375, 34379, 34387, 34394, 34405, 34420, 34427, 34433, 34443, 34452, - 34458, 34469, 34473, 34480, 34484, 34491, 34497, 16850, 34503, 34507, - 34512, 34518, 34525, 34529, 34533, 34541, 34549, 34555, 34564, 34571, - 34578, 34583, 34588, 34598, 28586, 34602, 34605, 34610, 34615, 34620, - 34625, 34630, 34635, 34640, 34645, 34651, 34656, 34661, 34667, 1094, 770, - 34672, 34675, 34682, 34691, 1777, 34698, 34703, 34707, 34713, 1143, 643, - 34718, 347, 34722, 34732, 34741, 34749, 34758, 34766, 34773, 34784, - 34792, 34801, 34809, 34819, 34827, 34832, 11794, 34836, 34844, 34852, - 34857, 17766, 4101, 34863, 34869, 34875, 6669, 34880, 34884, 34891, - 34897, 34903, 34907, 34916, 34922, 34927, 34934, 1336, 34940, 34946, - 34951, 34958, 34962, 1243, 6677, 34967, 34977, 34985, 34991, 35001, - 35010, 35018, 35024, 35029, 35037, 35044, 13376, 35050, 35057, 35062, - 35068, 35075, 35085, 1404, 253, 2212, 35091, 35097, 35104, 35115, 35126, - 35134, 35141, 35151, 35160, 35168, 35177, 35184, 35191, 35204, 35211, - 35217, 35228, 35247, 35252, 1148, 35256, 35261, 35269, 3984, 35273, - 35278, 35282, 35286, 1340, 29996, 35296, 35300, 35305, 35309, 35315, - 3846, 35321, 35329, 35336, 35347, 35356, 35364, 35389, 35397, 35402, - 3985, 401, 35408, 35416, 35424, 35431, 35436, 35442, 35447, 2281, 12792, - 35454, 35460, 31618, 31957, 35466, 656, 106, 35470, 35474, 35480, 595, - 10679, 35485, 35490, 35497, 35503, 35507, 35511, 1549, 35514, 35518, - 18308, 35521, 35526, 35533, 35539, 8966, 35544, 35552, 35559, 35565, - 27217, 35569, 35573, 35577, 35581, 1834, 20216, 35585, 35590, 35594, - 35597, 35605, 35613, 35618, 35627, 35635, 35638, 35645, 35652, 35664, - 27296, 35674, 35686, 35694, 35699, 35703, 35711, 35718, 35725, 35734, - 35740, 35747, 35754, 35757, 35761, 35765, 1351, 35775, 35777, 35782, - 35788, 35794, 35799, 35804, 35809, 35814, 35819, 35824, 35829, 35834, - 35839, 35844, 35849, 35854, 35859, 35864, 35870, 35876, 35882, 35888, - 35893, 35898, 35903, 35909, 35914, 35919, 35924, 35930, 35935, 35941, - 35946, 35951, 35956, 35961, 35967, 35972, 35978, 35983, 35988, 35993, - 35998, 36004, 36009, 36015, 36020, 36025, 36030, 36035, 36040, 36045, - 36050, 36055, 36060, 36066, 36072, 36078, 36083, 36088, 36093, 36098, - 36104, 36110, 36116, 36122, 36128, 36134, 36139, 36145, 36150, 36155, - 36160, 36165, 36171, 2552, 36176, 2559, 2566, 2962, 36181, 2572, 2582, - 36187, 2614, 2619, 2624, 36191, 36196, 36201, 36207, 36212, 36217, 36221, - 36226, 36232, 36237, 36242, 36247, 36253, 36258, 36262, 36266, 36271, - 36276, 36281, 36286, 36291, 36297, 36303, 36308, 36312, 36317, 36323, - 36327, 36332, 36337, 36342, 36347, 36351, 36354, 36359, 36364, 36369, - 36374, 36379, 36385, 36391, 36396, 36401, 36406, 36410, 36415, 36420, - 36425, 36430, 36435, 36440, 36444, 36449, 36454, 36459, 36463, 36467, - 36471, 36476, 36484, 36489, 36494, 36500, 36506, 36512, 36517, 36525, - 36529, 36532, 36537, 36542, 36546, 36551, 36556, 36560, 36565, 36569, - 36572, 36577, 4211, 21707, 36582, 36587, 36592, 36597, 36605, 25877, - 34955, 10318, 36610, 36615, 36619, 36624, 36628, 36632, 36637, 36641, - 36644, 36647, 36651, 36656, 36660, 36668, 36672, 36675, 36680, 36684, - 36688, 36693, 36698, 36702, 36708, 36713, 36718, 36725, 36732, 36736, - 36739, 36745, 36754, 36761, 36769, 36776, 36780, 36785, 36789, 36793, - 36799, 36804, 36810, 36814, 36820, 36825, 36830, 36834, 36841, 36847, - 36853, 36859, 36865, 36872, 36878, 36884, 36890, 36896, 36902, 36908, - 36914, 36921, 36927, 36934, 36940, 36946, 36952, 36958, 36964, 36970, - 36976, 36982, 36988, 36994, 36999, 37004, 13731, 37009, 37015, 37020, - 37025, 37030, 37035, 37038, 37044, 37049, 37057, 37062, 37066, 37071, - 37077, 37086, 37092, 37097, 37102, 37107, 37111, 37116, 37120, 37125, - 37130, 37135, 37140, 37147, 37154, 37160, 37166, 37171, 19813, 37178, - 37184, 37191, 37197, 37203, 37208, 37216, 37221, 11288, 37225, 37230, - 37235, 37241, 37246, 37251, 37255, 37260, 37265, 37271, 37276, 37281, - 37286, 37290, 37295, 37300, 37304, 37309, 37314, 37318, 37323, 37327, - 37332, 37337, 37342, 37346, 37351, 37355, 37360, 37364, 37371, 37375, - 37379, 18464, 37384, 37391, 37400, 37406, 37412, 37421, 37429, 37438, - 37446, 37451, 37455, 37462, 37468, 37476, 37480, 37483, 37488, 37492, - 37501, 37509, 37527, 37533, 1403, 37539, 37542, 37546, 27363, 27369, - 37552, 37556, 37567, 37578, 37589, 37601, 37605, 37612, 37619, 37626, - 37631, 37635, 37643, 37648, 37653, 37658, 37663, 6734, 16754, 25876, - 37668, 37673, 37677, 16745, 37682, 37688, 37693, 37699, 37704, 37710, - 37715, 37721, 37726, 37732, 37738, 37744, 37749, 37705, 37711, 37753, - 37758, 37764, 37769, 37775, 37780, 37786, 37791, 37716, 12422, 37795, - 37727, 37733, 37739, 3054, 3760, 37801, 37804, 37809, 37815, 37821, - 37827, 37834, 37840, 37846, 37852, 37858, 37864, 37870, 37876, 37882, - 37888, 37894, 37900, 37906, 37913, 37919, 37925, 37931, 37937, 37943, - 37946, 37951, 37954, 37961, 37966, 37974, 37978, 37983, 37988, 37994, - 37999, 38004, 38008, 38013, 38019, 38024, 38030, 38035, 38041, 38046, - 38052, 38058, 38062, 38067, 38072, 38077, 38082, 38086, 38091, 38096, - 38101, 38107, 38113, 38119, 38125, 38130, 38134, 38137, 38143, 38149, - 38158, 38166, 38173, 38178, 38182, 38186, 38191, 18251, 38196, 38204, - 38210, 4152, 1253, 38215, 38220, 38224, 9016, 38230, 38236, 38243, 9025, - 38247, 38253, 38259, 38266, 38272, 38281, 38289, 38301, 38310, 38314, - 38321, 38327, 38332, 38336, 38340, 38343, 38353, 38362, 38370, 37706, - 38375, 38385, 38395, 38405, 38411, 38416, 38426, 38431, 38444, 38458, - 38469, 38481, 38493, 38507, 38520, 38532, 38544, 17571, 38558, 38563, - 38568, 38572, 38576, 38580, 38584, 38590, 38595, 38600, 38605, 38610, - 38615, 38620, 1737, 33022, 38625, 38630, 38635, 37754, 38640, 38643, - 38648, 38653, 38658, 38664, 38670, 19423, 11994, 38675, 38681, 38688, - 19051, 38694, 38699, 38704, 38708, 38713, 38718, 37759, 38723, 38728, - 38733, 38739, 37765, 38744, 38747, 38754, 38762, 38768, 38774, 38780, - 38791, 38796, 38803, 38810, 38817, 38825, 38834, 38843, 38849, 38855, - 38863, 37770, 38868, 38874, 38880, 37776, 38885, 38890, 38898, 38906, - 38912, 38919, 38925, 38932, 38939, 38945, 38953, 38963, 38970, 38976, - 38981, 38987, 38992, 38997, 39004, 39013, 39021, 39026, 39032, 39039, - 39047, 39053, 39058, 39064, 39073, 39080, 34020, 39086, 39090, 39095, - 39104, 39109, 39114, 39119, 14972, 39127, 39132, 39137, 39142, 39146, - 39151, 39156, 39163, 39168, 39173, 39178, 37781, 25805, 39184, 2655, 158, - 39187, 39190, 39194, 39198, 39208, 39216, 39223, 39227, 39231, 39234, - 39242, 39249, 39256, 31911, 39265, 39268, 39275, 39281, 39286, 39290, - 39297, 39301, 39309, 39317, 39324, 39339, 39343, 39347, 39350, 39356, - 39363, 39367, 39373, 39377, 39384, 39392, 39400, 39407, 37717, 39414, - 39422, 39427, 39439, 12075, 12082, 12089, 12096, 12103, 12110, 626, 434, - 39445, 39450, 39455, 39461, 39466, 39471, 4178, 39476, 39479, 39484, - 39489, 39494, 39499, 39504, 39511, 27481, 39516, 39521, 39526, 39531, - 39536, 39542, 39547, 39553, 37957, 39559, 39564, 39570, 39576, 39586, - 39591, 39596, 39600, 39605, 39610, 39615, 39620, 39633, 39638, 27095, - 20298, 1064, 39642, 39648, 39652, 39657, 39662, 39668, 39673, 39678, - 39682, 39687, 39692, 39698, 39703, 39708, 1258, 39712, 39717, 39722, - 39727, 39731, 39736, 39741, 39746, 39752, 39758, 39763, 39767, 39771, - 39776, 39781, 39786, 39790, 39795, 39803, 39807, 39813, 39817, 39824, - 39833, 20069, 37728, 39839, 39846, 39854, 39862, 39869, 39875, 39884, - 39897, 39909, 39914, 39920, 39924, 2981, 39928, 39932, 39358, 39941, - 39952, 39963, 39968, 34088, 39973, 39978, 39982, 34208, 27374, 39987, - 39994, 39998, 40003, 37734, 25912, 40007, 40012, 40018, 40023, 40027, - 40031, 40034, 40038, 40044, 40053, 40064, 40076, 37740, 40081, 40084, - 40088, 40092, 40097, 40102, 40107, 40112, 40117, 40122, 40127, 40132, - 373, 40137, 40142, 40147, 40152, 40157, 40162, 40168, 40173, 40178, - 40184, 40189, 40195, 40200, 40206, 40211, 40216, 40221, 40226, 40231, - 40236, 40241, 40246, 40252, 40257, 40262, 40267, 40272, 40277, 40282, - 40287, 40293, 40299, 40304, 40309, 40314, 40319, 40324, 40329, 40334, - 40339, 40344, 40349, 40354, 40359, 40364, 40369, 40374, 40379, 40384, - 40389, 40399, 40409, 40415, 342, 14, 40420, 40423, 40427, 40431, 40439, - 40443, 40447, 31591, 16987, 1818, 40450, 40455, 40459, 40464, 40468, - 40473, 40477, 40482, 40486, 40489, 40491, 40495, 40500, 40504, 40515, - 40518, 40520, 40524, 40536, 40548, 40557, 40561, 40571, 40575, 40581, - 40586, 40595, 40601, 40606, 40611, 40615, 40619, 40624, 40631, 40636, - 40642, 40647, 40651, 40658, 31192, 31202, 40662, 40667, 40672, 40677, - 40684, 40688, 40695, 40702, 40708, 9171, 40712, 40721, 40729, 40744, - 40758, 40767, 40775, 40786, 40795, 40800, 40807, 40817, 8165, 40827, - 40832, 40838, 40843, 40847, 40850, 40855, 40859, 40864, 40868, 40875, - 40880, 40885, 40890, 40900, 40905, 40910, 40915, 10188, 40920, 40922, - 40930, 40933, 40936, 40944, 40959, 40967, 40977, 40979, 40982, 40986, - 40992, 40996, 41001, 41006, 41024, 41038, 41057, 41074, 41083, 41091, - 41096, 41101, 1396, 41107, 41113, 41118, 41128, 41137, 41145, 41150, - 41156, 41161, 41170, 41179, 41190, 41195, 41202, 41208, 41212, 41221, - 41228, 41236, 41243, 41256, 41264, 41268, 41278, 41284, 41289, 41293, - 41301, 41309, 41314, 41318, 41322, 41331, 41337, 41342, 41350, 41360, - 41369, 41378, 41387, 41398, 41406, 41417, 41426, 41434, 41441, 41447, - 41452, 41463, 41474, 41479, 41483, 41486, 41490, 41500, 41508, 41514, - 41525, 41536, 41547, 41558, 41569, 41580, 41591, 41602, 41614, 41626, - 41638, 41650, 41662, 41674, 41686, 41695, 41699, 41707, 41713, 41719, - 41726, 41732, 41737, 41743, 41747, 41752, 41757, 41762, 40394, 40404, - 2526, 41767, 41769, 41774, 41779, 41784, 41787, 41789, 41793, 41796, - 41803, 41807, 11644, 41811, 41817, 41824, 41830, 41840, 41845, 41851, - 41855, 41860, 41873, 31781, 41879, 41885, 41894, 41903, 21930, 41910, - 41919, 41927, 38391, 41933, 41938, 41942, 41951, 41959, 41966, 41971, - 41975, 41980, 41985, 41993, 41997, 42005, 42011, 42017, 42022, 42027, - 42031, 42034, 42039, 42052, 42068, 27965, 42085, 42097, 42114, 42126, - 42140, 27982, 28001, 42152, 42164, 2855, 42178, 42183, 42188, 42193, - 42197, 42204, 42216, 42223, 42232, 42242, 42245, 42256, 42267, 42275, - 42280, 42284, 42289, 42294, 42299, 42304, 42309, 42314, 1768, 947, 42319, - 42323, 42327, 42330, 42335, 42340, 42346, 42351, 42356, 42362, 42368, - 42373, 42377, 42382, 42387, 42392, 42396, 42399, 42405, 42410, 42415, - 42420, 42424, 42429, 42435, 42443, 32092, 42448, 42453, 42460, 42466, - 42472, 42477, 42485, 27490, 42492, 42497, 42502, 42507, 42511, 42514, - 42519, 42523, 42527, 42534, 42540, 42546, 42552, 42559, 42564, 42570, - 41304, 42574, 42578, 42583, 42596, 42601, 42607, 42615, 42622, 42630, - 42640, 42646, 42652, 42658, 42662, 42671, 42679, 42686, 42691, 42696, - 12445, 42701, 42711, 42718, 42724, 42734, 42739, 42745, 42753, 4017, - 42760, 42767, 42773, 42780, 4023, 42784, 42789, 42800, 42807, 42813, - 42822, 42826, 42829, 4609, 42836, 42843, 42849, 42855, 42863, 42873, - 35437, 42880, 42888, 42894, 42899, 42905, 42910, 42914, 31540, 42920, - 42927, 42933, 42941, 42950, 42957, 42963, 42974, 28784, 42980, 42987, - 42993, 43003, 43008, 43012, 43020, 43028, 43035, 43041, 43046, 11246, - 941, 43051, 43055, 43057, 43061, 43066, 43069, 43071, 43076, 43082, - 43087, 43092, 43099, 39507, 43105, 43110, 43114, 43119, 43123, 43132, - 43136, 43142, 43149, 43155, 43162, 43167, 43176, 43181, 43185, 43190, - 43197, 43205, 43213, 43218, 25968, 43222, 43225, 43229, 43233, 12889, - 1005, 43237, 43242, 43250, 43255, 43259, 43268, 43275, 43279, 43283, - 43291, 43298, 16272, 43308, 43312, 43316, 43324, 43332, 43338, 43343, - 43347, 43356, 16008, 43362, 43371, 43378, 43383, 43390, 43397, 43405, - 43412, 43420, 43428, 43437, 43442, 43449, 43456, 43463, 43470, 43477, - 43482, 43489, 43495, 43512, 43520, 43530, 43538, 43545, 43553, 461, - 43557, 43563, 43567, 43572, 40791, 43578, 43581, 43585, 43591, 43602, - 43610, 4028, 43618, 43624, 43630, 43640, 43646, 43655, 43664, 43674, - 43681, 43687, 43692, 4034, 4040, 43701, 43709, 43716, 43720, 14356, - 43728, 43732, 43739, 43747, 43754, 43763, 43770, 43776, 43785, 43795, - 43801, 43809, 43818, 43825, 43833, 43840, 26770, 43844, 43851, 43857, - 43867, 43876, 43884, 43895, 43899, 43909, 43916, 43921, 43926, 43932, - 43939, 43947, 43956, 43965, 43975, 43986, 43993, 43998, 44005, 3269, - 44013, 44019, 44024, 44031, 44037, 44043, 44048, 44061, 44074, 44087, - 44094, 44100, 44108, 44116, 44121, 44125, 44129, 44134, 44139, 44144, - 44149, 44154, 44159, 1365, 44164, 44168, 44172, 44176, 44180, 44184, - 44188, 44192, 44196, 44200, 44204, 44208, 44212, 44216, 44220, 44224, - 44228, 44232, 44236, 44240, 44244, 44248, 44252, 44256, 44260, 44264, - 44268, 44272, 44276, 44280, 44284, 44288, 44292, 44296, 44300, 44304, - 44308, 44312, 44316, 44320, 44324, 44328, 44332, 44336, 44340, 44344, - 44348, 44352, 44356, 44360, 44364, 44368, 44372, 44376, 44380, 44384, - 44388, 44392, 44396, 44400, 44404, 44408, 44412, 44416, 44420, 44424, - 44428, 44432, 44436, 44440, 44444, 44448, 44452, 44456, 44460, 44464, - 44468, 44472, 44476, 44480, 44484, 44488, 44492, 44496, 44500, 44504, - 44508, 44512, 44516, 44520, 44524, 44528, 44532, 44536, 44540, 44544, - 44548, 44552, 44556, 44560, 44564, 44568, 44572, 44576, 44580, 44584, - 44588, 44592, 44596, 44600, 44604, 44608, 44612, 44616, 44620, 44624, - 44628, 44632, 44636, 44640, 44644, 44648, 44652, 44656, 44660, 44664, - 44668, 44672, 44676, 44680, 44684, 44688, 44692, 44696, 44700, 44704, - 44708, 44712, 44716, 44720, 44724, 44728, 44732, 44736, 44740, 44744, - 44748, 44752, 44756, 44760, 44764, 44768, 44772, 44776, 44781, 44785, - 44790, 44794, 44799, 44803, 44808, 44812, 44818, 44823, 44827, 44832, - 44836, 44841, 44845, 44850, 44854, 44859, 44863, 44868, 44872, 44877, - 44881, 44887, 44893, 44898, 44902, 44907, 44911, 44917, 44922, 44926, - 44931, 44935, 44940, 44944, 44950, 44955, 44959, 44964, 44968, 44973, - 44977, 44982, 44986, 44992, 44997, 45001, 45006, 45010, 45016, 45021, - 45025, 45030, 45034, 45039, 45043, 45048, 45052, 45057, 45061, 45067, - 45072, 45076, 45082, 45087, 45091, 45097, 45102, 45106, 45111, 45115, - 45120, 45124, 45130, 45136, 45142, 45148, 45154, 45160, 45166, 45172, - 45177, 45181, 45186, 45190, 45196, 45201, 45205, 45210, 45214, 45219, - 45223, 45228, 45232, 45237, 45241, 45246, 45250, 45255, 45259, 45265, - 45270, 45274, 45279, 45283, 45289, 45295, 45300, 127, 63, 45304, 45306, - 45310, 45314, 45318, 45323, 45327, 45331, 45336, 11153, 45341, 45347, - 1677, 7181, 45353, 45356, 45361, 45365, 45370, 45374, 45378, 45383, - 12233, 45387, 45391, 45395, 630, 45399, 18573, 45404, 45408, 45413, - 45418, 45423, 45427, 45434, 45440, 45446, 31813, 45451, 45454, 45458, - 45463, 45469, 45473, 45476, 45484, 45490, 45495, 45499, 45502, 45506, - 45512, 45516, 45520, 3811, 3816, 15084, 45523, 45527, 45531, 45535, - 45539, 45547, 45554, 45558, 15958, 45565, 45579, 45586, 45597, 361, - 45602, 45606, 45612, 45624, 45630, 45636, 45641, 45647, 18625, 45651, - 45655, 35750, 45664, 45670, 45679, 45683, 45687, 45692, 45698, 45703, - 45707, 45712, 45716, 45720, 45727, 45733, 45738, 45749, 45764, 45779, - 45794, 45810, 45828, 12140, 45842, 45849, 45855, 45859, 45862, 45871, - 45876, 45880, 45888, 19254, 45896, 45900, 45910, 45921, 35640, 1042, - 45934, 45943, 45961, 45980, 45989, 45997, 46005, 1690, 12342, 46009, - 27386, 46012, 31579, 46017, 11478, 46022, 46028, 46033, 46039, 46044, - 46050, 46055, 46061, 46066, 46072, 46078, 46084, 46089, 46045, 46051, - 46093, 46056, 46062, 46067, 46098, 46073, 46079, 9184, 4434, 46104, - 46112, 46116, 46119, 46126, 46130, 46135, 46140, 46147, 46153, 46159, - 46164, 17862, 46168, 31596, 46172, 46176, 46180, 46187, 46193, 46197, - 33954, 46206, 10351, 46210, 10780, 46213, 46220, 46226, 46230, 14381, - 46237, 46243, 46248, 46255, 46262, 46269, 34753, 9081, 46276, 46283, - 46290, 46296, 46301, 46308, 46319, 46325, 46330, 46335, 46340, 46344, - 46349, 46356, 46046, 46360, 46370, 46379, 46390, 46396, 46404, 46411, - 46416, 46421, 46426, 46431, 46436, 46440, 46444, 46451, 46457, 46465, - 2416, 30605, 12245, 12257, 12262, 12268, 46474, 12273, 12278, 12284, - 46479, 46489, 46493, 12289, 46498, 20496, 46501, 46506, 46510, 46514, - 46525, 46533, 42227, 46541, 46546, 46553, 46560, 46564, 46567, 46575, - 12153, 46582, 46585, 46591, 46601, 6767, 46610, 46615, 46621, 46625, - 46633, 46637, 46647, 46653, 46658, 46669, 46678, 46687, 46696, 46705, - 46714, 46723, 46732, 46738, 46744, 46749, 46755, 46761, 46767, 46772, - 46775, 46782, 46788, 46792, 46797, 46804, 46811, 46815, 46818, 46828, - 46841, 46850, 46859, 46870, 46883, 46895, 46906, 46915, 46926, 46931, - 46940, 46945, 12294, 46951, 46958, 46966, 46973, 46978, 46983, 31859, - 46987, 46994, 4374, 25, 46998, 47003, 20345, 47007, 47010, 47013, 34145, - 47017, 34762, 47025, 47029, 47033, 47036, 47042, 47048, 47053, 37805, - 47062, 47070, 47076, 47083, 34128, 47087, 34365, 47091, 47100, 47104, - 47112, 47118, 47124, 47129, 47133, 34788, 47139, 47142, 47150, 47158, - 47166, 4772, 47172, 47176, 47180, 47185, 47192, 47198, 47203, 47208, - 47212, 47218, 47223, 47229, 4662, 820, 47236, 47240, 47243, 47255, 47262, - 47267, 18446, 47271, 47279, 47287, 47295, 47303, 47310, 47318, 47326, - 47333, 47341, 47349, 47357, 47365, 47373, 47381, 47389, 47397, 47405, - 47413, 47421, 47428, 47436, 47444, 47452, 47460, 47468, 47476, 47484, - 47492, 47500, 47508, 47516, 47524, 47532, 47540, 47548, 47556, 47564, - 47572, 47580, 47587, 47595, 47602, 47610, 47618, 47626, 47634, 47642, - 47650, 47658, 47666, 47677, 26806, 47682, 47685, 47692, 47696, 47702, - 47706, 47712, 47717, 47723, 47728, 47733, 47737, 47741, 47748, 47756, - 47761, 47766, 47776, 47782, 47795, 47801, 47807, 47813, 47816, 47823, - 47828, 4700, 47834, 4869, 965, 47839, 47842, 47845, 47848, 37889, 37895, - 47851, 37901, 37914, 37920, 37926, 47857, 37932, 37938, 47863, 47869, 10, - 47877, 47884, 47888, 47892, 47900, 38749, 47904, 47908, 47915, 47920, - 47924, 47929, 47935, 47940, 47946, 47951, 47955, 47959, 47963, 47968, - 47972, 47977, 47981, 47985, 47992, 47997, 48001, 48005, 48010, 48014, - 48019, 48023, 48027, 48032, 48038, 18755, 18760, 48043, 48047, 48050, - 48056, 48060, 48064, 25762, 48069, 48073, 48079, 48086, 48092, 48097, - 40820, 48107, 48112, 48120, 48124, 48127, 48131, 38764, 48139, 4738, - 48144, 48149, 48153, 48158, 48162, 48167, 16026, 48178, 48182, 48185, - 48189, 48197, 48202, 48206, 48211, 48216, 48220, 48224, 48228, 48231, - 48235, 48238, 48243, 48248, 48253, 48258, 48263, 48268, 8664, 16042, - 48273, 48276, 48282, 48287, 48293, 48298, 48304, 48309, 48315, 48320, - 48326, 48332, 48338, 48343, 48347, 48351, 48362, 48370, 48377, 48383, - 48388, 48399, 48409, 48415, 48420, 48427, 48436, 48452, 48468, 48478, - 34030, 48485, 48489, 48494, 48499, 48503, 48507, 43960, 48513, 48518, - 48522, 48529, 48534, 48539, 48543, 48546, 48550, 48556, 32825, 48560, - 26120, 48565, 48572, 48580, 48586, 48592, 48599, 48607, 48613, 48617, - 48622, 48628, 48636, 48641, 48645, 48654, 11134, 48662, 48666, 48674, - 48681, 48686, 48691, 48696, 48700, 48703, 48709, 48713, 48716, 48720, - 48727, 48732, 48739, 48743, 48749, 48753, 48759, 48764, 48769, 5107, - 5114, 48774, 48783, 48791, 48796, 48802, 48814, 48827, 48841, 48848, - 48854, 48860, 48865, 48873, 48876, 48878, 48889, 48901, 48912, 48927, - 48944, 48964, 48986, 48993, 49000, 49007, 49013, 49017, 8663, 49020, - 49024, 49028, 49033, 49037, 49041, 49044, 49048, 49062, 28031, 49081, - 49094, 49107, 49120, 28049, 49135, 2808, 49150, 49156, 49160, 49170, - 49174, 49178, 49183, 49187, 49194, 49199, 49203, 49210, 49216, 49221, - 49227, 49237, 49249, 49260, 49265, 49272, 49276, 49280, 49283, 49291, - 19275, 4141, 49296, 18794, 49309, 49316, 49323, 49329, 49333, 49337, - 49342, 49348, 49353, 49359, 49363, 49367, 49370, 49375, 49379, 49384, - 49389, 49394, 49399, 49404, 49409, 49414, 49419, 49424, 8727, 18805, - 49429, 49433, 49439, 49448, 49453, 49462, 49469, 43791, 49475, 49480, - 49484, 49491, 49496, 49503, 49511, 49517, 49521, 49524, 49528, 49533, - 2886, 49540, 49547, 49551, 49554, 49559, 49564, 49570, 49575, 49580, - 49584, 49589, 49599, 49604, 49610, 49615, 49622, 47257, 49628, 49634, - 49642, 49652, 49657, 49662, 49666, 49671, 49676, 8167, 8179, 49681, - 49684, 49691, 49697, 49706, 10268, 41444, 49714, 49718, 49722, 38812, - 49730, 49741, 49749, 44008, 49756, 49761, 49766, 49777, 49784, 49795, - 38836, 26137, 49803, 4652, 49808, 16471, 49814, 34119, 49820, 49825, - 49835, 49844, 49851, 49857, 49861, 49864, 49871, 49877, 49884, 49890, - 49900, 49908, 49914, 49920, 49925, 49929, 49936, 49941, 49947, 49954, - 49960, 49029, 49965, 49969, 16513, 16522, 16531, 16540, 16549, 16578, - 622, 16587, 49975, 49980, 49983, 49989, 49997, 1275, 50002, 50006, 50011, - 50016, 50020, 50025, 50032, 50038, 50042, 50047, 50053, 50057, 37962, - 50062, 50067, 50076, 50083, 50093, 50099, 34163, 50116, 50125, 50133, - 50139, 50144, 50151, 50157, 50165, 50174, 50182, 50190, 50196, 50200, - 50205, 50213, 35311, 38845, 50219, 50238, 19178, 50252, 50268, 50282, - 50288, 50293, 50298, 50303, 50309, 38851, 50314, 50317, 50324, 50331, - 50340, 50345, 50349, 423, 3176, 50356, 50361, 50366, 33219, 50154, 50370, - 50378, 50383, 50391, 50395, 50398, 50403, 50409, 50415, 50420, 50424, - 34236, 50427, 50432, 50436, 50439, 50444, 50448, 50453, 50458, 50462, - 50467, 50471, 50478, 50482, 50486, 25758, 25769, 50491, 50496, 50502, - 50507, 50513, 50519, 32781, 50524, 50528, 50531, 50537, 50542, 50547, - 50552, 50557, 50562, 50567, 50572, 50577, 50583, 50589, 14569, 19485, - 50594, 50599, 50604, 50609, 50614, 50619, 50624, 50629, 452, 68, 37979, - 37984, 37989, 37995, 38000, 38005, 50634, 38009, 50638, 50642, 50646, - 38014, 38020, 50660, 38031, 38036, 50668, 50673, 38042, 50678, 50683, - 50692, 50697, 50702, 50711, 50717, 50723, 50729, 38059, 50742, 50751, - 50757, 38063, 50761, 38068, 50766, 38073, 38078, 50769, 50774, 50778, - 50784, 16279, 50791, 16289, 50798, 50803, 38083, 50807, 50812, 50817, - 50822, 50827, 50831, 50836, 50841, 50847, 50852, 50857, 50863, 50869, - 50874, 50878, 50883, 50888, 50893, 50897, 50902, 50907, 50912, 50918, - 50924, 50930, 50935, 50939, 50944, 50948, 38087, 38092, 38097, 50952, - 50956, 50961, 50965, 50977, 38102, 38108, 38114, 38126, 50983, 31639, - 50987, 50992, 50996, 51001, 51008, 51013, 51018, 51023, 51027, 51031, - 51041, 51046, 51051, 51055, 51065, 51069, 51072, 51080, 51085, 38174, - 51089, 1375, 51095, 51100, 51106, 51114, 51118, 51127, 51135, 51139, - 51143, 51151, 51157, 51165, 51181, 51186, 51190, 51194, 51198, 51203, - 51209, 51224, 38211, 1685, 14601, 51228, 1254, 1269, 51240, 51248, 51255, - 51260, 9230, 51267, 51272, 10765, 978, 2641, 12321, 51279, 10658, 51284, - 51287, 51296, 1162, 51301, 49200, 51308, 51317, 51322, 51326, 51334, - 51341, 27436, 2697, 51349, 12842, 51359, 51365, 2434, 2444, 51374, 51383, - 51393, 51404, 3584, 41841, 51409, 4341, 4352, 9258, 1167, 51413, 51421, - 51428, 51433, 51437, 51441, 51446, 29077, 49535, 12412, 51454, 51463, - 51472, 51480, 51493, 51500, 51511, 51516, 51529, 51542, 51554, 51566, - 51578, 51589, 51602, 51613, 51624, 51634, 51642, 51650, 51662, 51674, - 51685, 51694, 51702, 51709, 51721, 51728, 51734, 51743, 51749, 51756, - 51769, 51774, 51784, 51789, 51795, 51800, 32121, 51804, 48705, 51811, - 51818, 51826, 51833, 2654, 51840, 51851, 51861, 51870, 51878, 51888, - 51896, 51905, 51915, 51924, 51929, 51935, 51941, 4190, 51952, 51962, - 51971, 51980, 51988, 51998, 52006, 52015, 52020, 52025, 52030, 1604, 47, - 52038, 52046, 52057, 52068, 19889, 52078, 52082, 52089, 52095, 52100, - 52104, 52115, 52125, 52134, 52145, 52150, 20318, 20323, 52157, 52166, - 52171, 52181, 52186, 52194, 52202, 52209, 52215, 1566, 271, 52219, 52224, - 52230, 46108, 52235, 52238, 2182, 2663, 52246, 52250, 52253, 1420, 52259, - 16799, 1172, 52264, 52277, 2797, 2818, 52291, 52303, 52315, 2832, 2849, - 2864, 2880, 2897, 52329, 52341, 2912, 52355, 1178, 1184, 1190, 12713, - 52360, 52365, 52370, 52374, 52389, 52404, 52419, 52434, 52449, 52464, - 52479, 52494, 52509, 52524, 52539, 52554, 52569, 52584, 52599, 52614, - 52629, 52644, 52659, 52674, 52689, 52704, 52719, 52734, 52749, 52764, - 52779, 52794, 52809, 52824, 52839, 52854, 52869, 52884, 52899, 52914, - 52929, 52944, 52959, 52974, 52989, 53004, 53019, 53034, 53049, 53064, - 53079, 53094, 53109, 53124, 53139, 53154, 53169, 53184, 53199, 53214, - 53229, 53244, 53259, 53274, 53289, 53304, 53319, 53334, 53349, 53364, - 53379, 53394, 53409, 53424, 53439, 53454, 53469, 53484, 53499, 53514, - 53529, 53544, 53559, 53574, 53589, 53604, 53619, 53634, 53649, 53664, - 53679, 53694, 53709, 53724, 53739, 53754, 53769, 53784, 53799, 53814, - 53829, 53844, 53859, 53874, 53889, 53904, 53919, 53934, 53949, 53964, - 53979, 53994, 54009, 54024, 54039, 54054, 54069, 54084, 54099, 54114, - 54129, 54144, 54159, 54174, 54189, 54204, 54219, 54234, 54249, 54264, - 54279, 54294, 54309, 54324, 54339, 54354, 54369, 54384, 54399, 54414, - 54429, 54444, 54459, 54474, 54489, 54504, 54519, 54534, 54549, 54564, - 54579, 54594, 54609, 54624, 54639, 54654, 54669, 54684, 54699, 54714, - 54729, 54744, 54759, 54774, 54789, 54804, 54819, 54834, 54849, 54864, - 54879, 54894, 54909, 54924, 54939, 54954, 54969, 54984, 54999, 55014, - 55029, 55044, 55059, 55074, 55089, 55104, 55119, 55134, 55149, 55164, - 55179, 55194, 55209, 55224, 55239, 55254, 55269, 55284, 55299, 55314, - 55329, 55344, 55359, 55374, 55389, 55404, 55419, 55434, 55449, 55464, - 55479, 55494, 55509, 55524, 55539, 55554, 55569, 55584, 55599, 55614, - 55629, 55644, 55659, 55674, 55689, 55704, 55719, 55734, 55749, 55764, - 55779, 55794, 55809, 55824, 55839, 55854, 55869, 55884, 55899, 55914, - 55929, 55944, 55959, 55974, 55989, 56004, 56019, 56034, 56049, 56064, - 56079, 56094, 56109, 56124, 56139, 56154, 56169, 56184, 56199, 56214, - 56229, 56244, 56259, 56274, 56289, 56304, 56319, 56334, 56349, 56364, - 56379, 56394, 56409, 56424, 56439, 56454, 56469, 56484, 56499, 56514, - 56529, 56544, 56559, 56574, 56589, 56604, 56619, 56634, 56649, 56664, - 56679, 56694, 56709, 56724, 56739, 56754, 56769, 56784, 56799, 56814, - 56829, 56844, 56859, 56874, 56889, 56904, 56919, 56934, 56949, 56964, - 56979, 56994, 57009, 57024, 57039, 57054, 57069, 57084, 57099, 57114, - 57129, 57144, 57159, 57174, 57189, 57204, 57219, 57234, 57249, 57264, - 57279, 57294, 57309, 57324, 57339, 57354, 57369, 57384, 57399, 57414, - 57429, 57444, 57459, 57474, 57489, 57504, 57519, 57534, 57549, 57564, - 57579, 57594, 57609, 57624, 57639, 57654, 57669, 57684, 57699, 57714, - 57729, 57744, 57759, 57774, 57789, 57804, 57819, 57834, 57849, 57864, - 57879, 57894, 57909, 57924, 57939, 57954, 57969, 57984, 57999, 58014, - 58029, 58044, 58059, 58074, 58089, 58104, 58119, 58134, 58149, 58164, - 58179, 58194, 58209, 58224, 58239, 58254, 58269, 58284, 58299, 58314, - 58329, 58344, 58359, 58374, 58389, 58404, 58419, 58434, 58449, 58464, - 58479, 58494, 58509, 58524, 58539, 58554, 58569, 58584, 58599, 58614, - 58629, 58644, 58659, 58674, 58689, 58704, 58719, 58734, 58749, 58764, - 58779, 58794, 58809, 58824, 58839, 58854, 58869, 58884, 58899, 58914, - 58929, 58944, 58959, 58974, 58989, 59004, 59019, 59034, 59049, 59064, - 59079, 59094, 59109, 59124, 59139, 59154, 59169, 59184, 59199, 59214, - 59229, 59244, 59259, 59274, 59289, 59304, 59319, 59334, 59349, 59364, - 59379, 59394, 59409, 59424, 59439, 59454, 59469, 59484, 59499, 59514, - 59529, 59544, 59559, 59574, 59589, 59604, 59619, 59634, 59649, 59664, - 59679, 59694, 59709, 59724, 59739, 59754, 59769, 59784, 59799, 59814, - 59829, 59844, 59859, 59874, 59889, 59904, 59919, 59934, 59949, 59964, - 59979, 59994, 60009, 60024, 60039, 60054, 60069, 60084, 60099, 60114, - 60129, 60144, 60159, 60174, 60189, 60205, 60221, 60237, 60253, 60269, - 60285, 60301, 60317, 60333, 60349, 60365, 60381, 60397, 60413, 60429, - 60445, 60461, 60477, 60493, 60509, 60525, 60541, 60557, 60573, 60589, - 60605, 60621, 60637, 60653, 60669, 60685, 60701, 60717, 60733, 60749, - 60765, 60781, 60797, 60813, 60829, 60845, 60861, 60877, 60893, 60909, - 60925, 60941, 60957, 60973, 60989, 61005, 61021, 61037, 61053, 61069, - 61085, 61101, 61117, 61133, 61149, 61165, 61181, 61197, 61213, 61229, - 61245, 61261, 61277, 61293, 61309, 61325, 61341, 61357, 61373, 61389, - 61405, 61421, 61437, 61453, 61469, 61485, 61501, 61517, 61533, 61549, - 61565, 61581, 61597, 61613, 61629, 61645, 61661, 61677, 61693, 61709, - 61725, 61741, 61757, 61773, 61789, 61805, 61821, 61837, 61853, 61869, - 61885, 61901, 61917, 61933, 61949, 61965, 61981, 61997, 62013, 62029, - 62045, 62061, 62077, 62093, 62109, 62125, 62141, 62157, 62173, 62189, - 62205, 62221, 62237, 62253, 62269, 62285, 62301, 62317, 62333, 62349, - 62365, 62381, 62397, 62413, 62429, 62445, 62461, 62477, 62493, 62509, - 62525, 62541, 62557, 62573, 62589, 62605, 62621, 62637, 62653, 62669, - 62685, 62701, 62717, 62733, 62749, 62765, 62781, 62797, 62813, 62829, - 62845, 62861, 62877, 62893, 62909, 62925, 62941, 62957, 62973, 62989, - 63005, 63021, 63037, 63053, 63069, 63085, 63101, 63117, 63133, 63149, - 63165, 63181, 63197, 63213, 63229, 63245, 63261, 63277, 63293, 63309, - 63325, 63341, 63357, 63373, 63389, 63405, 63421, 63437, 63453, 63469, - 63485, 63501, 63517, 63533, 63549, 63565, 63581, 63597, 63613, 63629, - 63645, 63661, 63677, 63693, 63709, 63725, 63741, 63757, 63773, 63789, - 63805, 63821, 63837, 63853, 63869, 63885, 63901, 63917, 63933, 63949, - 63965, 63981, 63997, 64013, 64029, 64045, 64061, 64077, 64093, 64109, - 64125, 64141, 64157, 64173, 64189, 64205, 64221, 64237, 64253, 64269, - 64285, 64301, 64317, 64333, 64349, 64365, 64381, 64397, 64413, 64429, - 64445, 64461, 64477, 64493, 64509, 64525, 64541, 64557, 64573, 64589, - 64605, 64621, 64637, 64653, 64669, 64685, 64701, 64717, 64733, 64749, - 64765, 64781, 64797, 64813, 64829, 64845, 64861, 64877, 64893, 64909, - 64925, 64941, 64957, 64973, 64989, 65005, 65021, 65037, 65053, 65069, - 65085, 65101, 65117, 65133, 65149, 65165, 65181, 65197, 65213, 65229, - 65245, 65261, 65277, 65293, 65309, 65325, 65341, 65357, 65373, 65389, - 65405, 65421, 65437, 65453, 65469, 65485, 65501, 65517, 65533, 65549, - 65565, 65581, 65597, 65613, 65629, 65645, 65661, 65677, 65693, 65709, - 65725, 65741, 65757, 65773, 65789, 65805, 65821, 65837, 65853, 65869, - 65885, 65901, 65917, 65933, 65949, 65965, 65981, 65997, 66013, 66029, - 66045, 66061, 66077, 66093, 66109, 66125, 66141, 66157, 66173, 66189, - 66205, 66221, 66237, 66253, 66269, 66285, 66301, 66317, 66333, 66349, - 66365, 66381, 66397, 66413, 66429, 66445, 66461, 66477, 66493, 66509, - 66525, 66541, 66557, 66573, 66589, 66605, 66621, 66637, 66653, 66669, - 66685, 66701, 66717, 66733, 66749, 66765, 66781, 66797, 66813, 66829, - 66845, 66861, 66877, 66893, 66909, 66925, 66941, 66957, 66973, 66989, - 67005, 67021, 67037, 67053, 67069, 67085, 67101, 67117, 67133, 67149, - 67165, 67181, 67197, 67213, 67229, 67245, 67261, 67277, 67293, 67309, - 67325, 67341, 67357, 67373, 67389, 67405, 67421, 67437, 67453, 67469, - 67485, 67501, 67517, 67533, 67549, 67565, 67581, 67597, 67613, 67629, - 67645, 67661, 67677, 67693, 67709, 67725, 67741, 67757, 67773, 67789, - 67805, 67821, 67837, 67853, 67869, 67885, 67901, 67917, 67933, 67949, - 67965, 67981, 67997, 68013, 68029, 68045, 68061, 68077, 68093, 68109, - 68125, 68141, 68157, 68173, 68189, 68205, 68221, 68237, 68253, 68269, - 68285, 68301, 68317, 68333, 68349, 68365, 68381, 68397, 68413, 68429, - 68445, 68461, 68477, 68493, 68509, 68525, 68541, 68557, 68573, 68589, - 68605, 68621, 68637, 68653, 68669, 68685, 68701, 68717, 68733, 68749, - 68765, 68781, 68797, 68813, 68829, 68845, 68861, 68870, 68885, 68899, - 68908, 17701, 68912, 68917, 68923, 68929, 68939, 68947, 17958, 18689, - 9277, 68960, 1428, 1432, 68968, 4270, 33344, 8104, 68974, 68979, 68984, - 68989, 68994, 69000, 69005, 69011, 69016, 69022, 69027, 69032, 69037, - 69042, 69048, 69053, 69058, 69063, 69068, 69073, 69078, 69083, 69089, - 69094, 69100, 69107, 2701, 69112, 69118, 9652, 69122, 69127, 69134, - 69142, 4281, 4286, 4291, 4296, 65, 69146, 69152, 69157, 69162, 69166, - 69171, 69175, 69179, 12785, 69183, 69193, 69206, 69217, 69230, 69237, - 69243, 69251, 69258, 12246, 69267, 69272, 69278, 69284, 69290, 69295, - 69300, 69305, 69310, 69314, 69319, 69324, 69329, 69335, 69341, 69347, - 69352, 69356, 69361, 69366, 69370, 69375, 69380, 69385, 69389, 12801, - 12812, 12817, 1471, 69393, 69399, 1476, 19723, 69404, 19732, 1486, 69409, - 69415, 69420, 1507, 69426, 1513, 1519, 12847, 69431, 69440, 69448, 69456, - 69463, 69467, 69471, 69477, 69482, 37622, 69487, 69494, 69502, 69509, - 69514, 69518, 69522, 69531, 69536, 69541, 69546, 1524, 280, 69551, 69556, - 69560, 19858, 987, 69564, 69571, 69576, 69580, 19916, 1528, 46384, 69583, - 69588, 69598, 69607, 69612, 69616, 69622, 1533, 49481, 69627, 69636, - 69642, 69647, 69652, 13086, 13092, 69658, 69671, 69683, 69700, 69717, - 69734, 69751, 69768, 69785, 69802, 69819, 69836, 69853, 69870, 69887, - 69904, 69921, 69938, 69955, 69972, 69989, 70006, 70023, 70040, 70057, - 70074, 70091, 70108, 70125, 70142, 70159, 70176, 70193, 70210, 70227, - 70244, 70261, 70278, 70295, 70312, 70329, 70346, 70363, 70380, 70397, - 70414, 70431, 70448, 70465, 70482, 70499, 70516, 70527, 70537, 70542, - 1538, 70546, 70551, 70557, 70562, 70567, 70574, 10677, 1543, 70580, - 70589, 33736, 70594, 70605, 13108, 70615, 70620, 70626, 70631, 70638, - 70644, 70649, 1548, 20210, 70654, 70660, 13118, 70666, 70671, 70676, - 70681, 70686, 70691, 70696, 70701, 1553, 4761, 70706, 70711, 70717, - 70722, 70727, 70732, 70737, 70742, 70747, 70752, 70757, 70763, 70769, - 70775, 70780, 70784, 70789, 70794, 70798, 70803, 70808, 70813, 70818, - 70822, 70827, 70833, 70838, 70843, 70847, 70852, 70857, 70863, 70868, - 70873, 70879, 70885, 70890, 70894, 70899, 70904, 70909, 70913, 70918, - 70923, 70928, 70934, 70940, 70945, 70949, 70953, 70958, 70963, 70968, - 35515, 70972, 70977, 70982, 70988, 70993, 70998, 71002, 71007, 71012, - 71018, 71023, 71028, 71034, 71040, 71045, 71049, 71054, 71059, 71063, - 71068, 71073, 71078, 71084, 71090, 71095, 71099, 71104, 71109, 71113, - 71118, 71123, 71128, 71133, 71137, 71140, 71143, 71148, 71153, 38358, - 71160, 71168, 50108, 71174, 3928, 33679, 71187, 71194, 71200, 71206, - 4107, 71211, 13260, 71217, 71227, 71242, 71250, 13265, 71261, 71266, - 71277, 71289, 71301, 71313, 2903, 71325, 71330, 31722, 71342, 71348, - 71354, 71359, 71368, 71375, 71380, 71385, 71390, 71395, 71400, 71405, - 1570, 19350, 71410, 71415, 71420, 71425, 71431, 71436, 71442, 71447, - 71452, 71458, 71463, 71468, 49555, 71472, 71476, 71481, 71485, 20358, - 71490, 71493, 71498, 71506, 71514, 1574, 13301, 13307, 1579, 71522, - 71529, 71534, 71543, 71553, 71560, 71565, 71570, 1584, 71577, 71582, - 20478, 71586, 71591, 71598, 71604, 71608, 71621, 71627, 71638, 71648, - 71655, 20500, 10571, 10578, 4355, 4361, 71662, 1589, 71667, 71676, 71682, - 71690, 71697, 71703, 71710, 71722, 71728, 71733, 71740, 71752, 71763, - 71773, 71782, 71792, 71802, 4249, 71810, 37416, 37425, 20540, 71823, - 71828, 71833, 71838, 71843, 71848, 71853, 1594, 1598, 71858, 71862, - 71865, 71876, 71881, 20566, 1608, 71889, 71894, 71899, 71911, 20599, - 71918, 71921, 71927, 71933, 71938, 71946, 1613, 71951, 71956, 71964, - 71972, 71979, 71988, 71996, 72005, 72009, 1618, 72018, 1623, 25943, - 72023, 72030, 72036, 20686, 72044, 72054, 72060, 72065, 72073, 72080, - 72089, 72097, 72107, 72116, 72126, 72135, 72146, 72156, 72166, 72175, - 72185, 72199, 72212, 72221, 72229, 72239, 72248, 72260, 72271, 72282, - 72292, 19978, 72297, 13453, 72306, 72312, 72317, 72324, 72330, 72337, - 72343, 19567, 72353, 72359, 72364, 72375, 72382, 72389, 72394, 72402, - 13470, 13475, 72410, 72416, 72420, 4339, 4350, 20762, 49658, 72428, - 72434, 72439, 72447, 72454, 14582, 72459, 72465, 72471, 1634, 72476, - 72479, 72485, 72490, 72495, 72500, 72505, 72510, 72515, 72520, 72525, - 72531, 72537, 1333, 72542, 72547, 72552, 72558, 72563, 72568, 72573, - 72578, 72583, 72588, 1643, 18, 72594, 72598, 72603, 72607, 72611, 72615, - 38644, 72620, 28250, 72625, 72630, 72634, 72637, 72641, 72645, 72650, - 72654, 72659, 72663, 72666, 72672, 42331, 42336, 42341, 72675, 72682, - 72688, 72696, 49253, 72706, 72712, 42347, 38908, 38659, 38665, 42363, - 38671, 72717, 72722, 72726, 38941, 72733, 72736, 72740, 72748, 72755, - 72760, 72763, 72768, 72773, 72777, 72781, 72784, 72794, 72806, 72813, - 72819, 38676, 72826, 40627, 72829, 9669, 13815, 72832, 72836, 72841, - 4159, 72845, 72848, 16332, 72855, 72862, 72875, 72890, 72904, 72920, - 72935, 72944, 72952, 72960, 72969, 72973, 72982, 72988, 72993, 73003, - 73016, 73028, 73035, 73040, 73049, 73062, 44055, 73080, 73085, 73092, - 73098, 73103, 850, 73108, 73116, 73123, 73130, 33160, 914, 73136, 73142, - 73147, 73157, 73165, 73171, 73176, 38695, 6858, 38709, 73180, 73190, - 73195, 73203, 73213, 73228, 73234, 73240, 73247, 38719, 73252, 73258, - 37760, 73262, 73266, 73271, 73280, 73287, 73292, 73296, 73301, 73309, - 20543, 73316, 73321, 73325, 6899, 38745, 73329, 73335, 341, 73345, 73352, - 73359, 73365, 73372, 73377, 73386, 15941, 69592, 69602, 73392, 73400, - 73404, 73408, 73412, 73416, 73421, 73425, 73431, 73439, 73444, 73449, - 73456, 73461, 73465, 73470, 73474, 73478, 73484, 73490, 73501, 73507, - 73512, 73516, 73521, 73525, 38869, 73529, 38875, 38881, 73534, 73540, - 73547, 73552, 73556, 37777, 20197, 73559, 73563, 73568, 73575, 73581, - 73585, 73590, 48722, 73596, 73600, 73607, 73611, 73616, 73622, 73628, - 73634, 73646, 73655, 73665, 73671, 73678, 73683, 73688, 73692, 73695, - 73701, 73708, 73713, 73718, 73725, 73732, 73739, 73745, 73750, 73755, - 73763, 38886, 2531, 73768, 73773, 73779, 73784, 73790, 73795, 73800, - 73805, 73811, 38907, 73816, 73822, 73828, 73834, 38977, 73839, 73844, - 73849, 38988, 73854, 73859, 73864, 73870, 73876, 38993, 73881, 73886, - 73891, 39048, 39054, 73896, 73901, 39059, 39081, 34021, 39087, 39091, - 73906, 14258, 73910, 73918, 73924, 73932, 73939, 73945, 73955, 73961, - 73968, 12685, 39105, 73974, 73987, 73996, 74002, 74011, 74017, 74023, - 74030, 28593, 74038, 74045, 74055, 74063, 74066, 39049, 74071, 74078, - 74083, 74087, 74091, 74096, 74100, 4478, 74105, 74110, 74115, 42425, - 42430, 74119, 42444, 74124, 42449, 74129, 74135, 42461, 42467, 42473, - 74140, 74146, 27491, 74157, 74160, 74172, 74180, 39128, 74184, 74193, - 74203, 74212, 39138, 74217, 74224, 74233, 74239, 74247, 74254, 74261, - 6950, 5174, 74266, 39060, 74272, 74275, 74281, 74288, 74293, 74298, - 28497, 74302, 74308, 74314, 74319, 74324, 74328, 74334, 74340, 40511, - 74345, 43649, 45486, 45492, 39169, 39174, 74349, 74353, 74357, 74360, - 74373, 74379, 74383, 74386, 74391, 40893, 74395, 37782, 25884, 74401, - 6879, 6887, 10377, 74404, 74409, 74414, 74419, 74424, 74429, 74434, - 74439, 74444, 74449, 74455, 74460, 74465, 74471, 74476, 74481, 74486, - 74491, 74496, 74501, 74507, 74512, 74518, 74523, 74528, 74533, 74538, - 74543, 74548, 74553, 74558, 74563, 74568, 74574, 74579, 74584, 74589, - 74594, 74599, 74604, 74610, 74615, 74620, 74625, 74630, 74635, 74640, - 74645, 74650, 74655, 74661, 74666, 74671, 74676, 74681, 74687, 74693, - 74698, 74704, 74709, 74714, 74719, 74724, 74729, 1421, 159, 74734, 74738, - 74742, 74746, 30458, 74750, 74754, 74759, 74763, 74768, 74772, 74777, - 74782, 74787, 74792, 74796, 74800, 74805, 74809, 16020, 74814, 74818, - 74825, 74835, 18327, 74844, 74853, 74862, 74866, 74871, 74876, 74880, - 74884, 30238, 3259, 74888, 74894, 21775, 74898, 74907, 74915, 74921, - 74926, 74938, 74950, 74955, 74959, 74964, 74968, 74974, 74980, 74985, - 74995, 75005, 75011, 75019, 75024, 75028, 75034, 75039, 75046, 75052, - 75057, 75064, 75073, 75082, 75090, 75094, 18883, 75097, 75106, 75114, - 75126, 75137, 75148, 75157, 75161, 75170, 75178, 75188, 75196, 75203, - 75213, 75219, 75224, 75232, 75239, 75248, 75254, 75259, 75266, 75272, - 75283, 60, 37559, 75289, 32009, 32019, 75295, 75303, 75310, 75316, 75320, - 75330, 75341, 75349, 75358, 75363, 75368, 75373, 75377, 75381, 75389, - 21722, 75396, 75400, 75406, 75416, 75423, 75430, 75436, 75442, 42524, - 75446, 75448, 75451, 75457, 75461, 75472, 75482, 75488, 75495, 75502, - 15957, 75510, 75516, 75525, 75534, 75540, 11579, 75546, 75552, 75557, - 75562, 75569, 75574, 75581, 75587, 75592, 75600, 75613, 75622, 75631, - 72161, 72171, 75641, 75647, 75656, 75662, 75668, 75675, 75682, 75689, - 75696, 75703, 75708, 75712, 75716, 75719, 75729, 75733, 75745, 75754, - 10038, 75763, 75774, 75779, 75783, 72180, 75789, 75796, 75805, 75813, - 51060, 75821, 75825, 75830, 75835, 75845, 75853, 75865, 75870, 75874, - 75878, 75884, 75892, 75899, 75911, 75919, 75930, 75937, 75943, 75953, - 75959, 75963, 75972, 75981, 75988, 75994, 75999, 76003, 76007, 76011, - 76020, 76029, 76038, 76045, 76051, 76057, 76063, 76068, 76075, 76081, - 76089, 76096, 76102, 15046, 76107, 76113, 76117, 17184, 76121, 76126, - 76136, 76141, 76150, 76156, 76162, 76170, 76177, 76181, 76185, 76192, - 76198, 76206, 76213, 76219, 76230, 76234, 76238, 76242, 76245, 76251, - 76256, 76261, 76265, 76269, 76278, 76286, 76293, 76299, 76306, 29268, - 48863, 76311, 76319, 76323, 76327, 76330, 76338, 76345, 76351, 76360, - 76368, 76374, 76379, 76383, 76388, 76393, 76397, 76401, 76405, 76410, - 76419, 76423, 76430, 45590, 76434, 76440, 76448, 76452, 76458, 76466, - 76472, 76477, 76488, 76496, 76502, 76511, 27638, 76519, 76526, 76533, - 76540, 76547, 76554, 52549, 15772, 76561, 76568, 76573, 42560, 4721, - 76579, 76584, 76589, 76595, 76601, 76607, 76612, 76617, 76622, 76627, - 76633, 76638, 76644, 76649, 76655, 76660, 76665, 76670, 76675, 76680, - 76685, 76690, 76696, 76701, 76707, 76712, 76717, 76722, 76727, 76732, - 76737, 76743, 76748, 76753, 76758, 76763, 76768, 76773, 76778, 76783, - 76788, 76793, 76799, 76804, 76809, 76814, 76819, 76824, 76829, 76834, - 76839, 76845, 76850, 76855, 76860, 76865, 76870, 76875, 76880, 76885, - 76890, 76895, 76900, 76905, 76911, 1883, 305, 76916, 46502, 46507, 76920, - 76925, 9293, 76929, 3628, 76934, 76939, 76943, 76952, 76963, 76980, - 76998, 77006, 75817, 77013, 77016, 77026, 77033, 77042, 77058, 77067, - 77077, 77082, 77095, 77105, 77114, 77122, 77136, 77144, 77153, 77157, - 77160, 77167, 77173, 77184, 77191, 77203, 77214, 77225, 77234, 77241, - 1173, 812, 77251, 2744, 77255, 77260, 77269, 997, 9059, 25320, 77277, - 77285, 77299, 77312, 77316, 77321, 77326, 77331, 77337, 77343, 77348, - 9661, 18370, 77353, 77357, 77361, 77369, 10098, 77374, 77380, 77389, - 77397, 1657, 13314, 1179, 4393, 77401, 77405, 77414, 77424, 2482, 32859, - 77433, 77439, 20450, 32874, 77445, 4558, 13696, 77451, 77458, 71871, - 77462, 77466, 77472, 77477, 77482, 3861, 163, 3887, 77487, 77499, 77503, - 77507, 77513, 77518, 33756, 77522, 13684, 2938, 4, 77527, 77537, 77548, - 77559, 77569, 77575, 77586, 77593, 77599, 77605, 2306, 77610, 77618, - 77625, 77631, 77641, 77651, 77661, 77670, 28580, 1185, 77675, 77679, - 77683, 77689, 77693, 2961, 2967, 9658, 2337, 77697, 77701, 77710, 77718, - 77729, 77737, 77745, 77751, 77756, 77767, 77778, 77786, 77792, 77797, - 11356, 77807, 77815, 77819, 77823, 77828, 77832, 77844, 34219, 18269, - 77851, 77861, 77867, 77873, 7977, 11490, 77883, 77894, 77905, 77915, - 77924, 77928, 77935, 999, 2731, 77945, 77950, 77958, 71587, 77966, 77971, - 77982, 77989, 78003, 16980, 529, 78013, 78020, 78024, 78028, 78036, - 78045, 4433, 78053, 78059, 20495, 78064, 78078, 78085, 78091, 78099, - 78108, 78117, 78124, 78136, 78146, 78154, 78161, 78169, 78176, 4357, 116, - 78184, 78195, 78199, 78211, 78217, 1804, 227, 78222, 10709, 78227, 3006, - 78231, 78238, 78244, 78255, 78265, 78273, 78280, 10049, 78287, 78296, - 78304, 4438, 78317, 4455, 78321, 78326, 78332, 78337, 78342, 78347, 3011, - 573, 78353, 78366, 78370, 78375, 78380, 3016, 1882, 763, 78384, 4459, - 78392, 78398, 78402, 799, 78412, 78421, 78426, 3878, 78430, 78434, 18001, - 18008, 9317, 78442, 4490, 4367, 15644, 78450, 78457, 78462, 28644, 78466, - 78473, 78479, 13952, 78484, 14017, 204, 78489, 78501, 78507, 78515, 3028, - 1699, 78523, 78525, 78530, 78535, 78540, 78546, 78551, 78556, 78561, - 78566, 78571, 78576, 78582, 78587, 78592, 78597, 78602, 78607, 78612, - 78617, 78622, 78628, 78633, 78638, 78643, 78649, 78654, 78660, 78665, - 78670, 78675, 78680, 78685, 78690, 78695, 78701, 78706, 78712, 78717, - 78722, 78727, 78732, 78737, 78742, 78747, 78752, 9730, 9743, 4506, 4511, - 4516, 4521, 26, 78758, 78764, 78769, 78774, 78779, 78785, 78790, 78794, - 78798, 78803, 78809, 78813, 78819, 78824, 78829, 78835, 78840, 78844, - 78849, 78854, 78858, 78861, 78863, 78867, 78870, 78877, 78882, 78886, - 78891, 78895, 78899, 78903, 78909, 78920, 78940, 78959, 78980, 78993, - 79005, 79014, 79018, 79021, 39446, 79024, 39451, 79031, 79036, 39456, - 79045, 79054, 39462, 79059, 39467, 79068, 79073, 13941, 79077, 79082, - 79087, 39472, 79091, 79100, 50719, 79104, 79107, 79111, 9325, 79117, - 79120, 79125, 79130, 79135, 79139, 4179, 39477, 79142, 79146, 79149, - 79160, 79165, 79169, 79175, 79183, 79196, 79200, 79208, 79217, 79223, - 79228, 79234, 79238, 79244, 79250, 79258, 79263, 79267, 79274, 79280, - 79288, 79297, 79305, 39480, 79312, 79322, 79331, 79339, 79350, 79363, - 79368, 79373, 79377, 79386, 79392, 79399, 79412, 79424, 79435, 79447, - 79454, 79463, 79472, 79481, 79488, 79494, 79501, 79509, 79516, 79524, - 79533, 79541, 79548, 79556, 79565, 79573, 79582, 79592, 79601, 79609, - 79616, 79624, 79633, 79641, 79650, 79660, 79669, 79677, 79686, 79696, - 79705, 79715, 79726, 79736, 79745, 79753, 79760, 79768, 79777, 79785, - 79794, 79804, 79813, 79821, 79830, 79840, 79849, 79859, 79870, 79880, - 79889, 79897, 79906, 79916, 79925, 79935, 79946, 79956, 79965, 79975, - 79986, 79996, 80007, 80019, 80030, 80040, 80049, 80057, 80064, 80072, - 80081, 80089, 80098, 80108, 80117, 80125, 80134, 80144, 80153, 80163, - 80174, 80184, 80193, 80201, 80210, 80220, 80229, 80239, 80250, 80260, - 80269, 80279, 80290, 80300, 80311, 80323, 80334, 80344, 80353, 80361, - 80370, 80380, 80389, 80399, 80410, 80420, 80429, 80439, 80450, 80460, - 80471, 80483, 80494, 80504, 80513, 80523, 80534, 80544, 80555, 80567, - 80578, 80588, 80599, 80611, 80622, 80634, 80647, 80659, 80670, 80680, - 80689, 80697, 80704, 80712, 80721, 80729, 80738, 80748, 80757, 80765, - 80774, 80784, 80793, 80803, 80814, 80824, 80833, 80841, 80850, 80860, - 80869, 80879, 80890, 80900, 80909, 80919, 80930, 80940, 80951, 80963, - 80974, 80984, 80993, 81001, 81010, 81020, 81029, 81039, 81050, 81060, - 81069, 81079, 81090, 81100, 81111, 81123, 81134, 81144, 81153, 81163, - 81174, 81184, 81195, 81207, 81218, 81228, 81239, 81251, 81262, 81274, - 81287, 81299, 81310, 81320, 81329, 81337, 81346, 81356, 81365, 81375, - 81386, 81396, 81405, 81415, 81426, 81436, 81447, 81459, 81470, 81480, - 81489, 81499, 81510, 81520, 81531, 81543, 81554, 81564, 81575, 81587, - 81598, 81610, 81623, 81635, 81646, 81656, 81665, 81675, 81686, 81696, - 81707, 81719, 81730, 81740, 81751, 81763, 81774, 81786, 81799, 81811, - 81822, 81832, 81843, 81855, 81866, 81878, 81891, 81903, 81914, 81926, - 81939, 81951, 81964, 81978, 81991, 82003, 82014, 82024, 82033, 82041, - 82048, 82053, 9090, 82060, 82065, 39490, 82071, 82076, 39495, 82082, - 82089, 25442, 31457, 82094, 82100, 82106, 82114, 82120, 82126, 82133, - 82140, 82145, 82150, 82154, 82159, 82163, 82166, 82170, 82175, 82184, - 82193, 82201, 82207, 82219, 82230, 82234, 3321, 9065, 82239, 82242, - 82245, 82247, 82251, 82255, 82259, 82265, 82270, 31520, 82275, 82279, - 82282, 82287, 82291, 82298, 82304, 82308, 7060, 82312, 82317, 39517, - 82321, 82328, 82337, 82345, 82351, 82362, 82370, 82379, 82387, 82394, - 82401, 82407, 82412, 82423, 39522, 82428, 82439, 82451, 82459, 82470, - 82479, 82487, 82498, 82503, 82511, 2696, 82516, 82525, 41914, 82538, - 82542, 82554, 82562, 82567, 82575, 82586, 21940, 82595, 82601, 82608, - 82616, 82622, 39532, 82627, 4484, 68943, 82634, 82637, 82645, 82658, - 82671, 82684, 82697, 82704, 82715, 82724, 82729, 52366, 52371, 82733, - 82737, 82745, 82752, 82761, 82769, 82775, 82784, 82792, 82799, 82807, - 82811, 82820, 82829, 82839, 82852, 82865, 82875, 39537, 82881, 82888, - 82894, 82900, 39543, 82905, 82908, 82912, 82920, 82929, 52137, 82937, - 82946, 82954, 82961, 82969, 82979, 82988, 82997, 41066, 83006, 83017, - 83032, 83042, 10747, 26258, 83051, 83056, 83061, 83065, 19559, 83070, - 83075, 83081, 83086, 83091, 83097, 83102, 83107, 26218, 83112, 83119, - 83127, 83135, 83143, 83148, 83155, 83162, 83167, 1717, 83171, 83175, - 83183, 83191, 39560, 83197, 83203, 83215, 83221, 83228, 83232, 83239, - 83244, 83251, 83257, 83264, 83275, 83285, 83295, 83307, 83313, 83321, - 83330, 83336, 83346, 83356, 39587, 83365, 83374, 83380, 83392, 83403, - 83410, 83415, 83419, 83427, 83438, 83444, 83449, 83454, 83461, 83469, - 83481, 83491, 83500, 83509, 83517, 83524, 41728, 29029, 83530, 83535, - 83539, 83543, 83548, 83556, 83562, 83573, 83586, 83591, 83598, 39592, - 83603, 83615, 83624, 83632, 83642, 83653, 83666, 83673, 83682, 83691, - 83699, 83704, 83710, 83714, 1410, 83719, 83724, 83729, 83734, 83740, - 83745, 83750, 83756, 83762, 83767, 83771, 83776, 83781, 83786, 69527, - 83791, 83796, 83801, 83806, 83812, 83818, 83823, 83827, 83832, 19558, - 83837, 83843, 83848, 83854, 83859, 83864, 83869, 83874, 83878, 83884, - 83889, 83898, 83903, 83908, 83913, 83918, 83922, 83929, 83935, 4836, - 20812, 3286, 83940, 83944, 83949, 83953, 83957, 83961, 56166, 83965, - 83890, 83967, 83977, 39601, 83980, 83985, 83994, 84000, 7019, 39606, - 84004, 84010, 84015, 84021, 84026, 84030, 84037, 84042, 84052, 84061, - 84065, 84071, 84077, 84083, 84087, 84095, 84102, 84110, 84118, 39611, - 84125, 84128, 84139, 84146, 84152, 84157, 84161, 84167, 84175, 84182, - 84187, 84191, 84200, 84208, 84214, 84219, 84226, 84234, 39616, 84241, - 28470, 84253, 84259, 84264, 84270, 84277, 84283, 25906, 33367, 84289, - 84294, 84300, 84304, 84316, 83923, 83930, 26150, 84326, 84331, 84338, - 84344, 84351, 84357, 84368, 84373, 84381, 10447, 84386, 84389, 84395, - 84399, 84403, 84406, 84412, 84418, 39305, 4837, 1425, 16069, 84425, - 84431, 84437, 84443, 84449, 84455, 84461, 84467, 84473, 84478, 84483, - 84488, 84493, 84498, 84503, 84508, 84513, 84518, 84523, 84528, 84533, - 84538, 84544, 84549, 84554, 84560, 84565, 84570, 84576, 84582, 84588, - 84594, 84600, 84606, 84612, 84618, 84624, 84629, 84634, 84640, 84645, - 84650, 84656, 84661, 84666, 84671, 84676, 84681, 84686, 84691, 84696, - 84701, 84706, 84711, 84716, 84722, 84727, 84732, 84737, 84743, 84748, - 84753, 84758, 84763, 84769, 84774, 84779, 84784, 84789, 84794, 84799, - 84804, 84809, 84814, 84819, 84824, 84829, 84834, 84839, 84844, 84849, - 84854, 84859, 84864, 84870, 84875, 84880, 84885, 84890, 84895, 84900, - 84905, 1065, 148, 84910, 84914, 84918, 84923, 84931, 84935, 84947, 84954, - 84962, 84966, 84979, 84987, 84992, 84997, 32072, 85001, 85006, 85010, - 85015, 85019, 85027, 85031, 25450, 85036, 85040, 72424, 85044, 85047, - 85055, 85063, 85071, 85076, 85081, 85088, 85095, 85101, 85107, 85112, - 85119, 85124, 85132, 77304, 85139, 85144, 72190, 85151, 85157, 85162, - 85166, 85173, 85179, 85186, 72217, 14031, 85194, 85199, 85204, 85208, - 85211, 85222, 85231, 85237, 85242, 85246, 85256, 85265, 50510, 85269, - 85273, 85280, 85293, 85299, 85307, 85314, 85323, 85334, 85345, 85356, - 85367, 85376, 85382, 85391, 85399, 85409, 85422, 85430, 85437, 85448, - 85457, 85463, 85468, 85473, 85479, 85489, 85495, 85505, 85513, 85520, - 85530, 85539, 83605, 85547, 85553, 85561, 85567, 75857, 85574, 85579, - 85582, 85586, 85592, 85596, 85599, 85607, 85613, 85619, 85627, 85639, - 85651, 85658, 85663, 85667, 85678, 85686, 85693, 85705, 85713, 85720, - 85728, 85735, 85741, 85746, 85752, 85762, 85771, 85779, 85784, 85794, - 85803, 51398, 85810, 85814, 85819, 85827, 85834, 85840, 85844, 85854, - 85865, 85873, 85880, 85892, 85904, 85913, 82528, 85920, 85930, 85942, - 85953, 85967, 85975, 85985, 85992, 86000, 86013, 86025, 86034, 86042, - 86052, 86063, 86075, 86084, 86094, 86104, 86114, 86123, 86130, 86139, - 86154, 86162, 86172, 86181, 86189, 86202, 68913, 86217, 86227, 86236, - 86248, 86258, 86270, 86281, 86295, 86309, 86323, 86337, 86351, 86365, - 86379, 86393, 86407, 86421, 86435, 86449, 86463, 86477, 86491, 86505, - 86519, 86533, 86547, 86561, 86575, 86589, 86603, 86617, 86631, 86645, - 86659, 86673, 86687, 86701, 86715, 86729, 86743, 86757, 86771, 86785, - 86799, 86813, 86827, 86841, 86855, 86869, 86883, 86897, 86911, 86925, - 86939, 86953, 86967, 86981, 86995, 87009, 87023, 87037, 87051, 87065, - 87079, 87093, 87107, 87121, 87135, 87149, 87163, 87177, 87191, 87205, - 87219, 87233, 87247, 87261, 87275, 87289, 87303, 87317, 87331, 87345, - 87359, 87373, 87387, 87401, 87415, 87429, 87443, 87457, 87471, 87485, - 87499, 87513, 87527, 87541, 87555, 87569, 87583, 87597, 87611, 87625, - 87639, 87653, 87667, 87681, 87695, 87709, 87723, 87737, 87751, 87765, - 87779, 87793, 87807, 87821, 87835, 87849, 87863, 87877, 87891, 87905, - 87919, 87933, 87947, 87961, 87975, 87989, 88003, 88017, 88031, 88045, - 88059, 88073, 88087, 88101, 88115, 88129, 88143, 88157, 88171, 88185, - 88199, 88213, 88227, 88241, 88255, 88269, 88283, 88297, 88311, 88325, - 88339, 88353, 88367, 88381, 88395, 88409, 88423, 88437, 88451, 88465, - 88479, 88493, 88507, 88521, 88535, 88549, 88563, 88577, 88591, 88605, - 88619, 88633, 88647, 88661, 88675, 88689, 88703, 88717, 88731, 88745, - 88759, 88773, 88787, 88801, 88815, 88829, 88843, 88857, 88871, 88885, - 88899, 88913, 88927, 88941, 88955, 88969, 88983, 88997, 89011, 89025, - 89039, 89053, 89067, 89081, 89095, 89109, 89123, 89137, 89151, 89165, - 89179, 89193, 89207, 89221, 89235, 89249, 89263, 89277, 89291, 89305, - 89319, 89333, 89347, 89361, 89375, 89389, 89403, 89417, 89431, 89445, - 89459, 89473, 89487, 89501, 89515, 89529, 89543, 89557, 89571, 89585, - 89599, 89613, 89627, 89641, 89655, 89669, 89683, 89697, 89711, 89725, - 89739, 89753, 89767, 89781, 89795, 89809, 89823, 89837, 89851, 89865, - 89879, 89893, 89907, 89921, 89935, 89949, 89963, 89977, 89991, 90005, - 90019, 90033, 90047, 90061, 90075, 90089, 90103, 90117, 90131, 90145, - 90159, 90173, 90187, 90201, 90215, 90229, 90243, 90257, 90271, 90285, - 90299, 90313, 90327, 90341, 90355, 90369, 90383, 90397, 90411, 90425, - 90439, 90453, 90467, 90481, 90495, 90509, 90523, 90537, 90551, 90565, - 90579, 90593, 90607, 90621, 90635, 90649, 90663, 90677, 90691, 90705, - 90719, 90733, 90747, 90761, 90775, 90789, 90803, 90817, 90831, 90845, - 90859, 90873, 90887, 90901, 90915, 90929, 90943, 90957, 90971, 90985, - 90999, 91013, 91027, 91041, 91055, 91069, 91083, 91097, 91111, 91125, - 91139, 91153, 91167, 91181, 91195, 91209, 91223, 91237, 91251, 91265, - 91279, 91293, 91307, 91321, 91335, 91349, 91363, 91377, 91391, 91405, - 91419, 91433, 91447, 91461, 91475, 91489, 91503, 91517, 91531, 91545, - 91559, 91573, 91587, 91601, 91615, 91629, 91643, 91657, 91671, 91685, - 91699, 91713, 91727, 91741, 91755, 91769, 91783, 91797, 91811, 91825, - 91839, 91853, 91867, 91881, 91895, 91909, 91923, 91937, 91951, 91965, - 91979, 91993, 92007, 92021, 92035, 92049, 92063, 92077, 92091, 92105, - 92119, 92133, 92147, 92161, 92175, 92189, 92203, 92217, 92231, 92245, - 92259, 92273, 92287, 92301, 92315, 92329, 92343, 92357, 92371, 92385, - 92399, 92413, 92427, 92441, 92455, 92469, 92483, 92497, 92511, 92525, - 92539, 92553, 92567, 92581, 92595, 92609, 92623, 92637, 92651, 92665, - 92679, 92693, 92707, 92721, 92735, 92749, 92763, 92777, 92791, 92805, - 92819, 92833, 92847, 92861, 92875, 92889, 92903, 92917, 92931, 92945, - 92959, 92973, 92987, 93001, 93015, 93029, 93043, 93057, 93071, 93085, - 93099, 93113, 93127, 93141, 93155, 93169, 93183, 93197, 93211, 93225, - 93239, 93253, 93267, 93281, 93295, 93309, 93323, 93337, 93351, 93365, - 93379, 93393, 93407, 93421, 93435, 93449, 93463, 93477, 93491, 93505, - 93519, 93533, 93547, 93561, 93575, 93589, 93603, 93617, 93631, 93645, - 93659, 93673, 93687, 93701, 93715, 93729, 93743, 93757, 93771, 93785, - 93799, 93813, 93827, 93841, 93855, 93869, 93883, 93897, 93911, 93925, - 93939, 93953, 93967, 93981, 93995, 94009, 94023, 94037, 94051, 94065, - 94079, 94093, 94107, 94121, 94135, 94149, 94163, 94177, 94191, 94205, - 94219, 94233, 94247, 94261, 94275, 94289, 94303, 94317, 94331, 94345, - 94359, 94373, 94387, 94401, 94415, 94429, 94443, 94457, 94471, 94485, - 94499, 94513, 94527, 94541, 94555, 94569, 94583, 94597, 94611, 94625, - 94639, 94653, 94667, 94681, 94695, 94709, 94723, 94737, 94751, 94765, - 94779, 94793, 94807, 94821, 94835, 94849, 94863, 94877, 94891, 94905, - 94919, 94933, 94947, 94961, 94975, 94989, 95003, 95017, 95031, 95045, - 95059, 95073, 95087, 95101, 95115, 95129, 95143, 95157, 95171, 95185, - 95199, 95213, 95227, 95241, 95255, 95269, 95283, 95297, 95311, 95325, - 95339, 95353, 95367, 95381, 95395, 95409, 95423, 95437, 95451, 95465, - 95479, 95493, 95507, 95521, 95535, 95549, 95563, 95577, 95591, 95605, - 95619, 95633, 95647, 95661, 95675, 95689, 95703, 95717, 95731, 95745, - 95759, 95773, 95787, 95801, 95815, 95829, 95843, 95857, 95871, 95885, - 95899, 95913, 95927, 95941, 95955, 95969, 95983, 95997, 96011, 96025, - 96039, 96053, 96067, 96081, 96095, 96109, 96123, 96137, 96151, 96165, - 96179, 96193, 96207, 96221, 96235, 96249, 96263, 96277, 96291, 96305, - 96319, 96333, 96347, 96361, 96375, 96389, 96403, 96417, 96431, 96445, - 96459, 96473, 96487, 96501, 96515, 96529, 96543, 96557, 96571, 96585, - 96599, 96613, 96627, 96641, 96655, 96669, 96683, 96697, 96711, 96725, - 96739, 96753, 96767, 96781, 96795, 96809, 96823, 96837, 96851, 96865, - 96879, 96893, 96907, 96921, 96935, 96949, 96963, 96977, 96991, 97005, - 97019, 97033, 97042, 97053, 97064, 97074, 97085, 97093, 97101, 97107, - 97117, 97125, 97131, 35391, 97136, 97142, 97151, 97163, 97168, 97175, - 11370, 21960, 97181, 97190, 97195, 97199, 97204, 97211, 97217, 97222, - 97227, 97235, 97243, 97253, 97258, 97266, 14513, 97270, 97276, 97282, - 97288, 97294, 97300, 97306, 97312, 97318, 97324, 97330, 97336, 97342, - 97348, 97354, 97360, 97366, 97372, 97378, 97384, 97390, 97396, 97402, - 97408, 97414, 97420, 97426, 97432, 97438, 97444, 97450, 97456, 97462, - 97468, 97474, 97480, 97486, 97493, 97499, 97505, 97511, 97517, 97523, - 97529, 97535, 97541, 97547, 97553, 97559, 97565, 97571, 97577, 97583, - 97589, 97595, 97601, 97607, 97613, 97619, 97625, 97631, 97637, 97643, - 97649, 97655, 97661, 97667, 97673, 97679, 97685, 97691, 97697, 97703, - 97709, 97715, 97721, 97727, 97733, 97739, 97745, 97751, 97757, 97763, - 97769, 97775, 97781, 97787, 97793, 97800, 97806, 97812, 97818, 97824, - 97830, 97836, 97842, 97848, 97854, 97860, 97866, 97869, 97871, 97886, - 97899, 97906, 97912, 97923, 97928, 97932, 97937, 97944, 97950, 97955, - 97963, 77907, 77917, 97969, 97976, 97986, 12672, 97993, 97998, 35639, - 98007, 98012, 98019, 98029, 98037, 98045, 98054, 98063, 98069, 98075, - 98080, 98087, 98094, 98099, 98103, 98111, 72234, 98116, 98125, 98133, - 98140, 98145, 98149, 98158, 98164, 98167, 98171, 98180, 98190, 84974, - 98199, 98203, 98211, 98215, 98221, 98232, 98242, 21969, 98253, 98262, - 98270, 98278, 98285, 72253, 9895, 98293, 98297, 98306, 98313, 98316, - 98320, 33238, 98323, 98327, 98332, 98349, 98361, 12630, 98373, 98378, - 98383, 98388, 25557, 98392, 98397, 98402, 98408, 98413, 6640, 98418, - 25561, 98423, 98428, 98434, 98441, 98446, 98451, 98457, 98463, 98469, - 98474, 98480, 98484, 98498, 98506, 98514, 98520, 98525, 98532, 98542, - 98551, 98556, 98561, 98566, 98574, 98584, 98595, 98600, 98606, 98611, - 98620, 70662, 4760, 98625, 98643, 98662, 98675, 98689, 98705, 98712, - 98719, 98728, 98735, 98741, 98748, 98753, 98759, 98764, 98770, 98778, - 98784, 98789, 98794, 98810, 12643, 98824, 98831, 98839, 98845, 98849, - 98852, 98858, 98863, 98868, 98876, 98883, 98888, 98897, 98903, 98908, - 98914, 98920, 98929, 98938, 43485, 98943, 98954, 98961, 98969, 98978, - 14104, 98987, 98993, 99001, 99007, 99013, 99019, 99024, 99031, 99037, - 14115, 99042, 99045, 99050, 39643, 99060, 99069, 99074, 99082, 99089, - 99095, 99100, 99108, 99115, 99126, 99142, 99158, 99174, 99190, 99206, - 99222, 99238, 99254, 99270, 99286, 99302, 99318, 99334, 99350, 99366, - 99382, 99398, 99414, 99430, 99446, 99462, 99478, 99494, 99510, 99526, - 99542, 99558, 99574, 99590, 99606, 99622, 99638, 99654, 99670, 99686, - 99702, 99718, 99734, 99750, 99766, 99782, 99798, 99814, 99830, 99846, - 99862, 99878, 99894, 99910, 99926, 99942, 99958, 99974, 99990, 100006, - 100022, 100038, 100054, 100070, 100086, 100102, 100118, 100134, 100150, - 100166, 100182, 100198, 100214, 100230, 100246, 100262, 100278, 100294, - 100310, 100326, 100342, 100358, 100374, 100390, 100406, 100422, 100438, - 100454, 100470, 100486, 100502, 100518, 100534, 100550, 100566, 100582, - 100598, 100614, 100630, 100646, 100662, 100678, 100694, 100710, 100726, - 100742, 100758, 100774, 100790, 100806, 100822, 100838, 100854, 100870, - 100886, 100902, 100918, 100934, 100950, 100966, 100982, 100998, 101014, - 101030, 101046, 101062, 101078, 101094, 101110, 101126, 101142, 101158, - 101174, 101190, 101206, 101222, 101238, 101254, 101270, 101286, 101302, - 101318, 101334, 101350, 101366, 101382, 101398, 101414, 101430, 101446, - 101462, 101478, 101494, 101510, 101526, 101542, 101558, 101574, 101590, - 101606, 101622, 101638, 101654, 101670, 101686, 101702, 101718, 101734, - 101750, 101766, 101782, 101798, 101814, 101830, 101846, 101862, 101878, - 101894, 101910, 101926, 101942, 101958, 101974, 101990, 102006, 102022, - 102038, 102054, 102070, 102086, 102102, 102118, 102134, 102150, 102166, - 102182, 102198, 102214, 102230, 102246, 102262, 102278, 102294, 102310, - 102326, 102342, 102358, 102374, 102390, 102406, 102422, 102438, 102454, - 102470, 102486, 102502, 102518, 102534, 102550, 102566, 102582, 102598, - 102614, 102630, 102646, 102662, 102678, 102694, 102710, 102726, 102742, - 102758, 102774, 102790, 102806, 102822, 102838, 102854, 102870, 102886, - 102902, 102918, 102934, 102950, 102966, 102982, 102998, 103014, 103030, - 103046, 103062, 103078, 103094, 103110, 103126, 103142, 103158, 103174, - 103190, 103206, 103222, 103238, 103254, 103270, 103286, 103302, 103318, - 103334, 103350, 103366, 103382, 103398, 103414, 103430, 103446, 103462, - 103478, 103494, 103510, 103526, 103542, 103558, 103574, 103590, 103606, - 103622, 103638, 103654, 103670, 103686, 103702, 103718, 103734, 103750, - 103766, 103782, 103798, 103814, 103830, 103846, 103862, 103878, 103894, - 103910, 103926, 103942, 103958, 103974, 103990, 104006, 104022, 104038, - 104054, 104070, 104086, 104102, 104118, 104134, 104150, 104166, 104182, - 104198, 104214, 104230, 104246, 104262, 104278, 104294, 104310, 104326, - 104342, 104358, 104374, 104390, 104406, 104422, 104438, 104454, 104470, - 104486, 104502, 104518, 104534, 104550, 104566, 104582, 104598, 104614, - 104630, 104646, 104662, 104678, 104694, 104710, 104726, 104742, 104758, - 104774, 104790, 104806, 104822, 104838, 104854, 104870, 104886, 104902, - 104918, 104934, 104950, 104966, 104982, 104998, 105014, 105030, 105046, - 105062, 105078, 105094, 105110, 105126, 105142, 105158, 105174, 105190, - 105206, 105222, 105238, 105254, 105270, 105286, 105302, 105318, 105334, - 105350, 105366, 105382, 105398, 105414, 105430, 105446, 105462, 105478, - 105494, 105510, 105526, 105542, 105558, 105574, 105590, 105606, 105622, - 105638, 105654, 105670, 105686, 105702, 105718, 105734, 105750, 105766, - 105782, 105798, 105814, 105830, 105846, 105862, 105878, 105894, 105910, - 105926, 105942, 105958, 105974, 105990, 106006, 106022, 106038, 106054, - 106070, 106086, 106102, 106118, 106134, 106150, 106166, 106182, 106198, - 106214, 106230, 106246, 106262, 106278, 106294, 106310, 106326, 106342, - 106358, 106374, 106390, 106406, 106422, 106438, 106454, 106470, 106486, - 106502, 106518, 106534, 106550, 106566, 106582, 106598, 106614, 106630, - 106646, 106662, 106678, 106694, 106710, 106726, 106742, 106758, 106774, - 106790, 106806, 106822, 106838, 106854, 106870, 106886, 106902, 106918, - 106934, 106950, 106966, 106982, 106998, 107014, 107030, 107046, 107062, - 107078, 107094, 107110, 107126, 107142, 107158, 107174, 107190, 107206, - 107222, 107238, 107254, 107270, 107286, 107302, 107318, 107334, 107350, - 107366, 107382, 107398, 107414, 107430, 107446, 107462, 107478, 107494, - 107510, 107526, 107542, 107558, 107574, 107590, 107606, 107622, 107638, - 107654, 107670, 107686, 107702, 107718, 107734, 107750, 107766, 107782, - 107798, 107814, 107830, 107846, 107862, 107878, 107894, 107910, 107926, - 107942, 107958, 107974, 107990, 108006, 108022, 108038, 108054, 108070, - 108086, 108102, 108118, 108134, 108150, 108166, 108182, 108198, 108214, - 108230, 108246, 108262, 108278, 108294, 108310, 108326, 108342, 108358, - 108374, 108390, 108406, 108422, 108438, 108454, 108470, 108486, 108502, - 108518, 108534, 108550, 108566, 108582, 108598, 108614, 108630, 108646, - 108662, 108678, 108694, 108710, 108726, 108742, 108758, 108774, 108790, - 108806, 108822, 108838, 108854, 108870, 108886, 108902, 108918, 108934, - 108950, 108966, 108982, 108998, 109014, 109030, 109046, 109062, 109078, - 109094, 109110, 109126, 109142, 109158, 109174, 109190, 109206, 109222, - 109238, 109254, 109270, 109286, 109302, 109318, 109334, 109350, 109366, - 109382, 109398, 109414, 109430, 109446, 109462, 109478, 109494, 109510, - 109526, 109542, 109558, 109574, 109590, 109606, 109622, 109638, 109654, - 109670, 109686, 109702, 109718, 109734, 109750, 109766, 109782, 109798, - 109814, 109830, 109846, 109862, 109878, 109894, 109910, 109926, 109942, - 109958, 109974, 109990, 110006, 110022, 110038, 110054, 110070, 110086, - 110102, 110118, 110134, 110150, 110166, 110182, 110198, 110214, 110230, - 110246, 110262, 110278, 110294, 110310, 110326, 110342, 110358, 110374, - 110390, 110406, 110422, 110438, 110454, 110470, 110486, 110502, 110518, - 110534, 110550, 110566, 110582, 110598, 110614, 110630, 110646, 110662, - 110678, 110694, 110710, 110726, 110742, 110758, 110774, 110790, 110806, - 110822, 110838, 110854, 110870, 110886, 110902, 110918, 110934, 110950, - 110966, 110982, 110998, 111014, 111030, 111046, 111062, 111078, 111094, - 111110, 111126, 111142, 111158, 111174, 111190, 111206, 111222, 111238, - 111254, 111270, 111286, 111302, 111318, 111334, 111350, 111366, 111382, - 111398, 111414, 111430, 111446, 111462, 111478, 111494, 111510, 111526, - 111542, 111558, 111574, 111590, 111606, 111622, 111638, 111654, 111670, - 111686, 111702, 111718, 111734, 111750, 111766, 111782, 111798, 111814, - 111830, 111846, 111862, 111878, 111894, 111910, 111926, 111942, 111958, - 111974, 111990, 112006, 112022, 112038, 112054, 112070, 112086, 112102, - 112118, 112134, 112150, 112166, 112182, 112198, 112214, 112230, 112246, - 112262, 112278, 112294, 112310, 112326, 112342, 112358, 112374, 112390, - 112406, 112422, 112438, 112454, 112470, 112486, 112502, 112518, 112534, - 112550, 112566, 112582, 112598, 112614, 112630, 112646, 112662, 112678, - 112694, 112710, 112726, 112742, 112758, 112774, 112790, 112806, 112822, - 112838, 112854, 112870, 112886, 112902, 112918, 112934, 112950, 112966, - 112982, 112992, 113001, 113006, 113014, 77227, 113019, 113025, 113030, - 113037, 113046, 113054, 113058, 4338, 113064, 113071, 113077, 113081, - 20561, 46618, 3295, 113086, 113090, 113094, 113101, 113107, 113116, - 113122, 113129, 113133, 113154, 113176, 113192, 113209, 113228, 113237, - 113247, 113255, 113262, 113269, 113275, 33093, 113289, 113293, 113299, - 113307, 113319, 113325, 113333, 113340, 113345, 113350, 113354, 113362, - 113369, 113373, 113379, 113385, 113390, 3972, 52566, 113396, 113400, - 113404, 113408, 113413, 113418, 113423, 113429, 113435, 113441, 113448, - 113454, 113461, 113467, 113473, 113478, 113484, 113489, 113493, 105586, - 113498, 105650, 52581, 113503, 113508, 113516, 113520, 113525, 113532, - 113541, 113548, 113554, 113563, 113567, 113574, 113578, 113581, 113588, - 113594, 113603, 113613, 113623, 113628, 113632, 113639, 113647, 113656, - 113660, 113668, 113674, 113679, 113684, 113690, 113696, 113701, 113705, - 31970, 113711, 113715, 113719, 113722, 113727, 113735, 113745, 113751, - 113756, 113766, 49687, 113774, 113786, 113792, 113799, 113805, 113809, - 113814, 113820, 113832, 113843, 113850, 113856, 113863, 113870, 113882, - 113889, 113895, 25641, 113899, 113907, 113913, 113920, 113926, 113932, - 113938, 113943, 113948, 113953, 113957, 113966, 113974, 113985, 7934, - 113990, 19997, 113996, 114000, 114004, 114008, 114016, 114025, 114029, - 114036, 114045, 114053, 114066, 114072, 106162, 36574, 114077, 114079, - 114084, 114089, 114094, 114099, 114104, 114109, 114114, 114119, 114124, - 114129, 114134, 114139, 114144, 114149, 114155, 114160, 114165, 114170, - 114175, 114180, 114185, 114190, 114195, 114201, 114207, 114213, 114218, - 114223, 114235, 114240, 1935, 54, 114245, 114250, 39653, 114254, 39658, - 39663, 39669, 39674, 114258, 39679, 26827, 114280, 114284, 114288, - 114293, 114297, 39683, 114301, 114309, 114316, 114322, 114332, 39688, - 114339, 114342, 114347, 114351, 114360, 11168, 114368, 39693, 26671, - 114371, 114375, 114383, 1307, 114388, 39704, 114391, 114396, 114401, - 31211, 31221, 43083, 114406, 114411, 114416, 114421, 114427, 114432, - 114441, 114446, 114455, 114463, 114470, 114476, 114481, 114486, 114491, - 114501, 114510, 114518, 114523, 114531, 114535, 114543, 114547, 114554, - 114561, 114569, 114576, 39508, 46333, 114582, 114588, 114593, 114598, - 14548, 11961, 114603, 114608, 114613, 114619, 114626, 114632, 114641, - 114646, 114654, 114664, 114671, 114681, 114687, 114692, 114698, 114702, - 21991, 114709, 44068, 114722, 114727, 114734, 114740, 114755, 37638, - 75635, 114768, 114772, 114781, 114790, 114797, 114803, 114811, 114817, - 114825, 114834, 114842, 114849, 46453, 114855, 114858, 114862, 114866, - 114870, 11982, 114876, 114883, 114889, 114897, 114902, 114906, 29203, - 114912, 114915, 114923, 114930, 114938, 114951, 114965, 114972, 114978, - 114985, 114991, 39718, 114995, 115001, 115009, 115016, 115024, 115032, - 115038, 39723, 115046, 115052, 115057, 115067, 115073, 115082, 37433, - 42431, 115090, 115095, 115100, 115104, 115109, 115113, 115121, 115126, - 17993, 18818, 49709, 115130, 115135, 39728, 18150, 115139, 115151, - 115156, 115160, 115167, 115176, 115180, 115188, 115194, 115199, 115207, - 115215, 115223, 115231, 115239, 115247, 115258, 115264, 9132, 115269, - 115275, 115280, 115285, 115296, 115305, 115317, 115332, 40040, 115338, - 20116, 39732, 115342, 115349, 115355, 115359, 29340, 115366, 115372, - 115379, 48834, 115388, 115394, 115403, 115409, 115414, 115422, 115428, - 115433, 39742, 115438, 115447, 115456, 113838, 115465, 115472, 115478, - 115484, 115493, 115503, 115509, 115517, 115524, 115528, 39747, 115531, - 39753, 1346, 115536, 115544, 115552, 115562, 115571, 115579, 115586, - 115596, 39764, 115600, 115602, 115606, 115611, 115615, 115619, 115625, - 115630, 115634, 115645, 115650, 115656, 115661, 115670, 115675, 3300, - 115679, 115686, 115690, 115699, 115707, 115715, 115722, 115727, 115732, - 74136, 115736, 115739, 115745, 115753, 115759, 115763, 115768, 115775, - 115780, 115785, 115789, 115796, 115802, 115807, 42462, 115811, 115814, - 115819, 115823, 115828, 115835, 115840, 115844, 47803, 115852, 31230, - 31239, 115858, 115864, 115870, 115875, 115879, 115882, 115892, 115901, - 115906, 115912, 115919, 115925, 115929, 115937, 115942, 42468, 85233, - 115946, 115954, 115961, 115967, 115974, 115979, 115986, 115991, 115995, - 116001, 116006, 69129, 116012, 116018, 10386, 116023, 116028, 116032, - 116037, 116042, 116047, 116051, 116056, 116061, 116067, 116072, 116077, - 116083, 116089, 116094, 116098, 116103, 116108, 116113, 116117, 29339, - 116122, 116127, 116133, 116139, 116145, 116150, 116154, 116159, 116164, - 109938, 116169, 116174, 116179, 116184, 110002, 52836, 116189, 39772, - 116197, 116201, 116209, 116217, 116228, 116233, 116237, 27306, 82631, - 116242, 116248, 116253, 4649, 116263, 116270, 116275, 116283, 116292, - 116297, 116301, 116306, 116310, 116318, 116326, 116333, 77493, 116339, - 116347, 116354, 116365, 116371, 116377, 39782, 116380, 116387, 116395, - 116400, 116404, 33612, 71815, 116410, 116415, 116422, 116427, 10275, - 116431, 116439, 116446, 116453, 116462, 116469, 116475, 116489, 116497, - 6724, 116259, 116503, 116508, 116514, 116518, 116521, 116529, 116536, - 116541, 116554, 116561, 116567, 116571, 116579, 116584, 116591, 116597, - 116602, 72093, 116607, 116610, 116619, 116626, 110210, 116632, 116635, - 116643, 116649, 116658, 116668, 116678, 116687, 116698, 116706, 116717, - 116722, 116726, 116731, 116735, 43214, 116743, 18783, 43223, 116748, - 101729, 101745, 101761, 101777, 101793, 116753, 101825, 101841, 101857, - 101873, 101985, 102001, 116757, 102033, 102049, 116761, 116765, 116769, - 116773, 102289, 102321, 116777, 102353, 116781, 116785, 102497, 102513, - 102529, 102545, 116789, 102609, 102625, 116793, 102753, 102769, 102785, - 102801, 102817, 102833, 102849, 102865, 102881, 102897, 103009, 103025, - 103041, 103057, 103073, 103089, 103105, 103121, 103137, 103153, 116797, - 104945, 105057, 105121, 105137, 105153, 105169, 105185, 105201, 105313, - 105329, 105345, 116801, 105393, 116805, 105425, 105441, 105457, 116809, - 116814, 116819, 116824, 116829, 116834, 116839, 116843, 116847, 116852, - 116857, 116861, 116866, 116871, 116875, 116880, 116885, 116890, 116895, - 116899, 116904, 116909, 116913, 116918, 116922, 116926, 116930, 116934, - 116939, 116943, 116947, 116951, 116955, 116959, 116963, 116967, 116971, - 116975, 116980, 116985, 116990, 116995, 117000, 117005, 117010, 117015, - 117020, 117025, 117029, 117033, 117037, 117041, 117045, 117049, 117054, - 117058, 117063, 117067, 117072, 117077, 117081, 117085, 117090, 117094, - 117098, 117102, 117106, 117110, 117114, 117118, 117122, 117126, 117130, - 117134, 117138, 117142, 117146, 117151, 117156, 117160, 117164, 117168, - 117172, 117176, 117180, 117185, 117189, 117193, 117197, 117201, 117205, - 117209, 117214, 117218, 117223, 117227, 117231, 117235, 117239, 117243, - 117247, 117251, 117255, 117259, 117263, 117267, 117272, 117276, 117280, - 117284, 117288, 117292, 117296, 117300, 117304, 117308, 117312, 117316, - 117321, 117325, 117329, 117334, 117339, 117343, 117347, 117351, 117355, - 117359, 117363, 117367, 117371, 117376, 117380, 117385, 117389, 117394, - 117398, 117403, 117407, 117413, 117418, 117422, 117427, 117431, 117436, - 117440, 117445, 117449, 117454, 1429, 117458, 117462, 3042, 1705, 28465, - 1602, 31166, 117466, 3051, 117470, 1276, 117475, 1218, 117479, 117483, - 117487, 117491, 117495, 117499, 3075, 117503, 117511, 117518, 117525, - 117539, 3079, 8044, 117548, 117556, 117563, 117574, 117583, 117587, - 117594, 117606, 117619, 117632, 117643, 117648, 117655, 117667, 117671, - 3083, 14185, 117681, 117686, 117695, 117705, 117710, 117719, 3087, - 117727, 117731, 117736, 117743, 117749, 117754, 117763, 117771, 117783, - 117793, 1223, 15645, 117806, 117810, 117816, 117830, 117842, 117854, - 117862, 117872, 117881, 117890, 117899, 117907, 117918, 117926, 4657, - 117936, 117947, 117956, 117962, 117977, 117984, 117990, 117995, 43357, - 118000, 3111, 15649, 118004, 118009, 118016, 10206, 118025, 118031, 4695, - 118041, 3116, 39144, 118050, 71705, 118057, 118061, 118067, 118078, - 118084, 118089, 118096, 118102, 118110, 118117, 118123, 118134, 118150, - 118160, 118169, 118180, 118189, 118196, 118202, 118212, 118220, 118226, - 118241, 118247, 118252, 118256, 118263, 118271, 118275, 118278, 118284, - 118291, 118297, 118305, 118314, 118322, 118328, 118337, 52139, 118351, - 118356, 118362, 17744, 118367, 118380, 118392, 118401, 118409, 118416, - 118420, 118424, 118427, 118434, 118441, 118449, 118457, 118466, 118474, - 17643, 118482, 118487, 118491, 118503, 118510, 118517, 118526, 954, - 118536, 118545, 118556, 3137, 118560, 118564, 118570, 118583, 118595, - 118605, 118614, 118626, 32128, 118637, 118645, 118654, 118665, 118676, - 118686, 118696, 118704, 118713, 118721, 13604, 118728, 118732, 118735, - 118740, 118745, 118749, 118755, 1228, 118762, 118766, 14281, 118770, - 118774, 118785, 118794, 118802, 118811, 118819, 118835, 118846, 118855, - 118863, 118875, 118886, 118902, 118912, 118933, 118947, 118960, 118968, - 118975, 8090, 118988, 118993, 118999, 6733, 119005, 119008, 119015, - 119025, 9266, 119032, 119037, 119042, 119049, 119057, 119065, 119071, - 119076, 119082, 119086, 119094, 119103, 119111, 119116, 119125, 119132, - 11418, 11427, 119138, 119149, 119155, 119160, 119166, 3153, 3158, 119172, - 1063, 119178, 119185, 119192, 119205, 119215, 119220, 2324, 87, 119228, - 119235, 119240, 119248, 119258, 119267, 119273, 119282, 119290, 119300, - 119304, 119308, 119313, 119317, 119329, 3181, 119337, 119345, 119350, - 119361, 119372, 119384, 119395, 119405, 119414, 26025, 119419, 119425, - 119430, 119440, 119450, 119455, 34346, 119461, 119466, 119475, 26047, - 119479, 26058, 119484, 4783, 8, 119491, 119500, 119507, 119514, 119520, - 119525, 119529, 119535, 34376, 119540, 119545, 72390, 119550, 119555, - 119561, 119567, 119575, 119580, 119588, 119596, 119605, 119612, 119618, - 119625, 119631, 119638, 119643, 47672, 52033, 119649, 119659, 1822, 32, - 119666, 119671, 119684, 119689, 119697, 119702, 119708, 3207, 30907, - 3221, 119713, 119721, 119728, 119733, 119738, 119747, 4340, 4351, 73734, - 119755, 119759, 1629, 1862, 119764, 119769, 119776, 34797, 1866, 331, - 119783, 119789, 119794, 3229, 119798, 119803, 119810, 1870, 119815, - 119821, 119826, 119838, 6978, 119848, 119855, 1877, 119861, 119866, - 119873, 119880, 119895, 119902, 119913, 119918, 119926, 2772, 119930, - 119942, 119947, 119951, 119957, 34218, 2329, 119961, 119972, 119976, - 119980, 119986, 119990, 119999, 120003, 120014, 120018, 2375, 38961, - 120022, 120032, 120040, 3320, 120046, 120055, 120063, 10752, 120068, - 120076, 120081, 120085, 120094, 120101, 120107, 3290, 17808, 120111, - 120124, 44081, 120142, 120147, 120155, 120163, 120173, 11759, 15773, - 120185, 120198, 120205, 120215, 120229, 120236, 120252, 120259, 120265, - 26105, 14980, 120272, 120279, 120289, 120298, 52835, 120310, 120318, - 52970, 120325, 120328, 120334, 120340, 120346, 120352, 120358, 120365, - 120372, 120378, 120384, 120390, 120396, 120402, 120408, 120414, 120420, - 120426, 120432, 120438, 120444, 120450, 120456, 120462, 120468, 120474, - 120480, 120486, 120492, 120498, 120504, 120510, 120516, 120522, 120528, - 120534, 120540, 120546, 120552, 120558, 120564, 120570, 120576, 120582, - 120588, 120594, 120600, 120606, 120612, 120618, 120624, 120630, 120636, - 120642, 120648, 120654, 120660, 120666, 120672, 120678, 120685, 120691, - 120698, 120705, 120711, 120718, 120725, 120731, 120737, 120743, 120749, - 120755, 120761, 120767, 120773, 120779, 120785, 120791, 120797, 120803, - 120809, 120815, 3304, 10720, 120821, 120831, 120837, 120845, 120849, - 117739, 3308, 120853, 114067, 25770, 4701, 4265, 120857, 3314, 120861, - 120871, 120877, 120883, 120889, 120895, 120901, 120907, 120913, 120919, - 120925, 120931, 120937, 120943, 120949, 120955, 120961, 120967, 120973, - 120979, 120985, 120991, 120997, 121003, 121009, 121015, 121021, 121028, - 121035, 121041, 121047, 121053, 121059, 121065, 121071, 1233, 121077, - 121082, 121087, 121092, 121097, 121102, 121107, 121112, 121117, 121121, - 121125, 121129, 121133, 121137, 121141, 121145, 121149, 121153, 121159, - 121165, 121171, 121177, 121181, 121185, 121189, 121193, 121197, 121201, - 121205, 121209, 121213, 121218, 121223, 121228, 121233, 121238, 121243, - 121248, 121253, 121258, 121263, 121268, 121273, 121278, 121283, 121288, - 121293, 121298, 121303, 121308, 121313, 121318, 121323, 121328, 121333, - 121338, 121343, 121348, 121353, 121358, 121363, 121368, 121373, 121378, - 121383, 121388, 121393, 121398, 121403, 121408, 121413, 121418, 121423, - 121428, 121433, 121438, 121443, 121448, 121453, 121458, 121463, 121468, - 121473, 121478, 121483, 121488, 121493, 121498, 121503, 121508, 121513, - 121518, 121523, 121528, 121533, 121538, 121543, 121548, 121553, 121558, - 121563, 121568, 121573, 121578, 121583, 121588, 121593, 121598, 121603, - 121608, 121613, 121618, 121623, 121628, 121633, 121638, 121643, 121648, - 121653, 121658, 121663, 121668, 121673, 121678, 121683, 121688, 121693, - 121698, 121703, 121708, 121713, 121718, 121723, 121728, 121733, 121738, - 121743, 121748, 121753, 121758, 121763, 121768, 121773, 121778, 121783, - 121788, 121793, 121798, 121803, 121808, 121813, 121818, 121823, 121828, - 121833, 121838, 121843, 121848, 121853, 121858, 121863, 121868, 121873, - 121878, 121883, 121888, 121893, 121898, 121903, 121908, 121913, 121918, - 121923, 121928, 121933, 121938, 121943, 121948, 121953, 121958, 121963, - 121968, 121973, 121978, 121983, 121988, 121993, 121998, 122003, 122008, - 122013, 122018, 122023, 122028, 122033, 122038, 122043, 122048, 122053, - 122058, 122063, 122068, 122073, 122078, 122083, 122088, 122093, 122098, - 122103, 122109, 122114, 122119, 122124, 122129, 122134, 122139, 122144, - 122150, 122155, 122160, 122165, 122170, 122175, 122180, 122185, 122190, - 122195, 122200, 122205, 122210, 122215, 122220, 122225, 122230, 122235, - 122240, 122245, 122250, 122255, 122260, 122265, 122270, 122275, 122280, - 122285, 122290, 122295, 122300, 122305, 122310, 122319, 122324, 122333, - 122338, 122347, 122352, 122361, 122366, 122375, 122380, 122389, 122394, - 122403, 122408, 122417, 122422, 122427, 122436, 122440, 122449, 122454, - 122463, 122468, 122477, 122482, 122491, 122496, 122505, 122510, 122519, - 122524, 122533, 122538, 122547, 122552, 122561, 122566, 122575, 122580, - 122585, 122590, 122595, 122600, 122605, 122610, 122614, 122619, 122624, - 122629, 122634, 122639, 122644, 122650, 122655, 122660, 122665, 122671, - 122675, 122680, 122686, 122691, 122696, 122701, 122706, 122711, 122716, - 122721, 122726, 122731, 122736, 122742, 122747, 122752, 122757, 122763, - 122768, 122773, 122778, 122783, 122789, 122794, 122799, 122804, 122809, - 122814, 122820, 122825, 122830, 122835, 122840, 122845, 122850, 122855, - 122860, 122865, 122870, 122875, 122880, 122885, 122890, 122895, 122900, - 122905, 122910, 122915, 122920, 122925, 122930, 122935, 122941, 122947, - 122953, 122958, 122963, 122968, 122973, 122979, 122985, 122991, 122996, - 123001, 123006, 123012, 123017, 123022, 123027, 123032, 123037, 123042, - 123047, 123052, 123057, 123062, 123067, 123072, 123077, 123082, 123087, - 123092, 123098, 123104, 123110, 123115, 123120, 123125, 123130, 123136, - 123142, 123148, 123153, 123158, 123163, 123168, 123173, 123178, 123183, - 123188, 123193, 19475, 123198, 123204, 123209, 123214, 123219, 123224, - 123229, 123235, 123240, 123245, 123250, 123255, 123260, 123266, 123271, - 123276, 123281, 123286, 123291, 123296, 123301, 123306, 123311, 123316, - 123321, 123326, 123331, 123336, 123341, 123346, 123351, 123356, 123361, - 123366, 123371, 123376, 123382, 123387, 123392, 123397, 123402, 123407, - 123412, 123417, 123422, 123427, 123432, 123437, 123442, 123447, 123452, - 123457, 123462, 123467, 123472, 123477, 123482, 123487, 123492, 123497, - 123502, 123507, 123512, 123517, 123522, 123527, 123532, 123537, 123542, - 123547, 123552, 123557, 123562, 123567, 123572, 123577, 123582, 123588, - 123593, 123598, 123603, 123608, 123613, 123618, 123623, 123628, 123633, - 123638, 123643, 123649, 123654, 123660, 123665, 123670, 123675, 123680, - 123685, 123690, 123696, 123701, 123706, 123712, 123717, 123722, 123727, - 123732, 123737, 123743, 123749, 123754, 123759, 14614, 123764, 123769, - 123774, 123779, 123784, 123789, 123794, 123799, 123804, 123809, 123814, - 123819, 123824, 123829, 123834, 123839, 123844, 123849, 123854, 123859, - 123864, 123869, 123874, 123879, 123884, 123889, 123894, 123899, 123904, - 123909, 123914, 123919, 123924, 123929, 123934, 123939, 123944, 123949, - 123954, 123959, 123964, 123969, 123974, 123979, 123984, 123989, 123994, - 123999, 124004, 124009, 124014, 124019, 124024, 124029, 124034, 124039, - 124044, 124049, 124054, 124059, 124064, 124069, 124074, 124079, 124084, - 124090, 124095, 124100, 124105, 124110, 124116, 124121, 124126, 124131, - 124136, 124141, 124146, 124152, 124157, 124162, 124167, 124172, 124177, - 124183, 124188, 124193, 124198, 124203, 124208, 124214, 124219, 124224, - 124229, 124234, 124239, 124245, 124251, 124256, 124261, 124266, 124272, - 124278, 124284, 124289, 124294, 124300, 124306, 124311, 124317, 124323, - 124329, 124334, 124339, 124345, 124350, 124356, 124361, 124367, 124376, - 124381, 124386, 124392, 124397, 124403, 124408, 124413, 124418, 124423, - 124428, 124433, 124438, 124443, 124448, 124453, 124458, 124463, 124468, - 124473, 124478, 124483, 124488, 124493, 124498, 124503, 124508, 124513, - 124518, 124523, 124528, 124533, 124538, 124543, 124548, 124553, 124558, - 124564, 124570, 124576, 124581, 124586, 124591, 124596, 124601, 124606, - 124611, 124616, 124621, 124626, 124631, 124636, 124641, 124646, 124651, - 124656, 124661, 124666, 124671, 124676, 124682, 124688, 124693, 124699, - 124704, 124709, 124715, 124720, 124726, 124731, 124737, 124742, 124748, - 124753, 124759, 124764, 124769, 124774, 124779, 124784, 124789, 124794, - 120872, 120878, 120884, 120890, 124800, 120896, 120902, 124806, 120908, - 120914, 120920, 120926, 120932, 120938, 120944, 120950, 120956, 124812, - 120962, 120968, 120974, 124818, 120980, 120986, 120992, 120998, 124824, - 121004, 121010, 121016, 121036, 124830, 124836, 121042, 124842, 121048, - 121054, 121060, 121066, 121072, 124848, 3331, 3336, 124853, 3351, 3356, - 3361, 124858, 124861, 124867, 124873, 124880, 124885, 124890, 2380, +/* name->code dictionary */ +static const unsigned char packed_name_dawg[] = { + 228, 174, 4, 250, 2, 65, 206, 254, 2, 66, 222, 254, 2, 67, 198, 245, 4, + 68, 222, 134, 1, 69, 250, 151, 1, 70, 206, 52, 71, 186, 154, 1, 72, 242, + 223, 1, 73, 146, 65, 74, 138, 23, 75, 230, 134, 1, 76, 146, 130, 3, 77, + 162, 200, 3, 78, 162, 107, 79, 226, 124, 80, 128, 159, 1, 2, 81, 85, 194, + 7, 82, 182, 169, 1, 83, 166, 175, 4, 84, 170, 203, 2, 85, 222, 90, 86, + 182, 80, 87, 182, 111, 88, 254, 4, 89, 179, 48, 90, 182, 40, 222, 2, 67, + 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, 82, + 80, 76, 65, 78, 69, 82, 76, 210, 32, 77, 198, 1, 78, 142, 26, 80, 142, + 20, 82, 158, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 172, 213, 11, 4, + 70, 71, 72, 65, 206, 192, 9, 66, 220, 219, 2, 9, 75, 84, 73, 69, 83, 69, + 76, 83, 75, 164, 187, 2, 2, 81, 85, 199, 135, 10, 88, 18, 132, 1, 2, 67, + 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 140, 230, 16, 6, 84, 73, 86, 65, + 84, 69, 225, 150, 22, 5, 32, 67, 85, 82, 82, 4, 162, 185, 25, 85, 237, + 182, 13, 2, 82, 68, 5, 147, 175, 33, 78, 4, 178, 207, 34, 67, 167, 207, + 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 216, 141, 13, 7, + 72, 69, 83, 73, 86, 69, 32, 168, 151, 4, 19, 68, 82, 69, 83, 83, 69, 68, + 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 234, 178, 13, 85, 161, + 186, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, 77, 65, + 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, 84, 73, + 65, 76, 32, 38, 78, 178, 209, 2, 72, 220, 244, 22, 4, 65, 76, 73, 70, 0, + 5, 86, 79, 87, 69, 76, 131, 130, 12, 68, 70, 40, 5, 65, 80, 73, 84, 65, + 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 142, 2, + 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 210, 158, 29, 81, 190, + 205, 4, 76, 130, 40, 67, 202, 154, 4, 90, 212, 7, 2, 65, 76, 170, 2, 66, + 2, 89, 146, 1, 87, 182, 89, 84, 150, 14, 80, 158, 20, 70, 2, 72, 2, 82, + 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 250, 186, 5, 65, 247, 251, 33, + 72, 4, 174, 162, 39, 66, 215, 22, 65, 2, 171, 182, 38, 73, 6, 194, 183, + 38, 65, 154, 98, 80, 191, 28, 72, 6, 206, 232, 38, 85, 158, 77, 72, 3, + 89, 4, 188, 181, 2, 6, 73, 78, 78, 89, 73, 73, 135, 128, 37, 72, 4, 40, + 4, 69, 32, 67, 79, 151, 206, 38, 73, 2, 173, 174, 38, 13, 78, 83, 79, 78, + 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 218, 222, 32, 69, 191, 141, 4, 81, + 4, 166, 151, 25, 65, 199, 185, 10, 85, 4, 182, 215, 26, 69, 185, 202, 11, + 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, 80, 5, 71, 69, + 65, 78, 32, 249, 144, 33, 9, 82, 73, 65, 76, 32, 84, 82, 65, 77, 114, + 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, 77, 69, 65, + 83, 85, 82, 69, 32, 26, 87, 130, 185, 26, 78, 147, 185, 11, 67, 2, 153, + 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, 83, 31, 84, + 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, 32, 83, 69, + 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, 65, 5, 66, + 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, 3, 82, 83, + 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, 2, 25, 4, + 32, 83, 85, 66, 2, 133, 148, 39, 2, 85, 78, 4, 130, 151, 38, 76, 247, 72, + 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, + 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, 84, 69, 82, + 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 140, 234, 31, 8, 78, 85, 77, 66, 69, 82, 32, 84, 219, 209, 5, 68, 6, 26, + 76, 159, 171, 39, 82, 4, 132, 227, 38, 7, 73, 71, 65, 84, 73, 78, 71, + 207, 74, 65, 68, 154, 1, 65, 210, 195, 35, 68, 46, 84, 222, 225, 1, 76, + 246, 189, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, + 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, 32, 6, + 146, 169, 39, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 241, 193, 22, + 5, 89, 77, 66, 79, 76, 8, 50, 83, 182, 130, 25, 75, 145, 233, 9, 2, 82, + 85, 4, 148, 183, 36, 4, 77, 65, 76, 76, 207, 162, 2, 69, 22, 66, 65, 162, + 218, 35, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 11, 186, 169, 39, + 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 252, 190, 34, 3, 68, 69, 80, 141, + 153, 4, 5, 65, 82, 82, 73, 86, 130, 2, 232, 1, 20, 67, 72, 69, 77, 73, + 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 166, 28, 69, + 60, 2, 76, 32, 82, 77, 108, 6, 84, 69, 82, 78, 65, 84, 208, 179, 28, 7, + 73, 69, 78, 32, 77, 79, 78, 225, 230, 6, 3, 65, 82, 77, 232, 1, 158, 2, + 65, 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, + 82, 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, + 73, 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 214, + 245, 33, 85, 190, 206, 1, 69, 202, 153, 1, 79, 195, 27, 70, 30, 194, 1, + 76, 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 148, 137, + 3, 3, 77, 65, 76, 176, 222, 8, 5, 85, 82, 73, 80, 73, 188, 129, 12, 3, + 82, 83, 69, 234, 156, 14, 83, 159, 91, 73, 8, 224, 24, 2, 69, 77, 252, + 209, 25, 4, 75, 65, 76, 73, 179, 167, 13, 85, 10, 38, 32, 149, 139, 24, + 3, 70, 79, 82, 8, 216, 7, 4, 86, 73, 84, 65, 225, 194, 22, 4, 82, 69, 71, + 73, 16, 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, + 84, 72, 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 203, 232, + 37, 82, 4, 230, 182, 16, 77, 253, 218, 21, 5, 86, 65, 80, 79, 85, 28, 82, + 65, 92, 6, 79, 80, 80, 69, 82, 32, 34, 82, 217, 206, 36, 4, 73, 78, 78, + 65, 6, 216, 218, 5, 2, 68, 85, 196, 156, 32, 9, 80, 85, 84, 32, 77, 79, + 82, 84, 85, 235, 27, 76, 4, 230, 13, 65, 179, 240, 37, 79, 16, 72, 8, 79, + 67, 85, 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, + 67, 79, 80, 80, 69, 219, 232, 37, 73, 11, 11, 45, 8, 138, 157, 39, 50, 2, + 51, 2, 52, 3, 53, 8, 44, 2, 73, 83, 185, 133, 5, 3, 65, 89, 45, 6, 144, + 2, 4, 83, 79, 76, 86, 203, 169, 36, 84, 4, 234, 182, 29, 79, 191, 212, 9, + 85, 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 240, 191, 34, 2, 79, 85, 255, + 161, 3, 68, 4, 132, 245, 2, 4, 82, 83, 69, 32, 183, 237, 35, 85, 6, 56, + 3, 32, 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 147, 226, 25, 69, 4, + 48, 3, 69, 65, 68, 221, 158, 13, 3, 79, 68, 69, 2, 135, 225, 34, 32, 10, + 120, 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, + 176, 133, 16, 4, 65, 82, 67, 65, 135, 148, 21, 79, 7, 207, 151, 39, 45, + 4, 142, 248, 37, 84, 235, 117, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, + 80, 72, 69, 82, 83, 34, 79, 98, 85, 181, 155, 29, 6, 82, 69, 67, 73, 80, + 73, 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 161, 145, 35, + 3, 84, 32, 65, 5, 237, 231, 37, 5, 69, 68, 32, 66, 82, 4, 180, 143, 33, + 4, 84, 82, 69, 70, 161, 245, 5, 3, 82, 73, 70, 4, 132, 185, 34, 5, 78, + 84, 69, 83, 83, 249, 196, 3, 4, 67, 75, 32, 76, 24, 58, 69, 213, 220, 25, + 8, 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, + 3, 65, 76, 71, 179, 172, 25, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, + 228, 219, 25, 8, 65, 78, 84, 73, 77, 79, 78, 89, 255, 147, 12, 73, 6, + 170, 147, 39, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, + 1, 85, 184, 139, 17, 2, 80, 73, 200, 255, 11, 2, 73, 76, 194, 243, 8, 79, + 133, 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 197, + 234, 37, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, + 52, 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 237, 149, 29, 7, + 78, 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, + 169, 212, 25, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, + 82, 65, 84, 85, 77, 12, 44, 6, 66, 76, 73, 77, 65, 84, 155, 1, 76, 10, + 44, 5, 69, 32, 79, 70, 32, 215, 190, 38, 73, 8, 68, 8, 83, 65, 76, 84, + 32, 79, 70, 32, 130, 3, 65, 175, 209, 26, 67, 4, 254, 2, 65, 175, 209, + 26, 67, 2, 135, 149, 38, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, + 82, 189, 162, 37, 2, 85, 84, 4, 11, 65, 4, 219, 212, 25, 82, 4, 222, 162, + 34, 67, 167, 49, 32, 2, 145, 201, 38, 2, 73, 68, 14, 50, 73, 245, 238, + 36, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 149, 211, + 25, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 251, 137, 39, + 45, 2, 221, 188, 23, 3, 78, 84, 73, 4, 234, 204, 38, 84, 227, 61, 88, 6, + 38, 77, 246, 246, 37, 70, 231, 118, 82, 2, 151, 227, 37, 66, 4, 134, 227, + 32, 69, 233, 136, 4, 11, 65, 82, 79, 85, 78, 68, 45, 80, 82, 79, 70, 9, + 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, 2, 84, 79, 235, + 176, 31, 79, 5, 147, 205, 6, 32, 4, 240, 192, 10, 3, 73, 86, 69, 253, + 159, 27, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 180, 194, 28, 6, 69, 82, + 73, 67, 65, 78, 200, 232, 5, 3, 66, 85, 76, 209, 204, 3, 16, 65, 76, 71, + 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, 158, 205, 35, 69, + 171, 78, 72, 192, 9, 92, 3, 65, 84, 79, 174, 20, 71, 174, 1, 84, 150, + 167, 24, 67, 182, 203, 11, 68, 155, 252, 2, 75, 144, 9, 104, 17, 76, 73, + 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, 65, 133, 241, 12, + 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, 162, 3, 51, 174, + 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, 102, 57, 182, 7, + 51, 234, 190, 11, 49, 202, 174, 22, 48, 242, 1, 53, 2, 55, 3, 56, 22, + 238, 193, 33, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 3, 57, 28, 250, 135, 12, 54, 146, 185, 21, 49, 2, 53, 174, + 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, 210, 238, 11, + 54, 250, 146, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, + 57, 24, 186, 191, 33, 55, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 254, 242, 11, 50, 2, + 51, 158, 129, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, 162, 236, + 11, 52, 2, 55, 190, 24, 53, 146, 185, 21, 48, 2, 49, 2, 50, 174, 193, 5, + 51, 2, 54, 2, 56, 3, 57, 26, 238, 131, 12, 48, 146, 185, 21, 53, 174, + 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 222, 1, 102, + 48, 110, 49, 102, 57, 210, 1, 56, 170, 225, 11, 50, 2, 54, 250, 141, 22, + 51, 2, 52, 2, 53, 3, 55, 28, 150, 130, 12, 50, 146, 185, 21, 55, 2, 57, + 174, 193, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, 24, 186, 186, + 33, 53, 2, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, + 3, 57, 24, 214, 185, 33, 52, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, + 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, 102, 51, 110, + 54, 102, 56, 162, 1, 57, 174, 201, 11, 55, 162, 162, 22, 49, 3, 52, 22, + 134, 184, 33, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 3, 56, 30, 214, 229, 11, 54, 242, 150, 8, 50, 138, 252, 18, + 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 182, 182, 33, + 52, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, + 57, 26, 98, 51, 242, 180, 33, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 52, + 2, 53, 2, 55, 2, 56, 3, 57, 4, 144, 173, 20, 6, 32, 82, 65, 32, 79, 82, + 139, 201, 18, 65, 20, 140, 203, 38, 3, 51, 32, 69, 210, 42, 48, 2, 49, 2, + 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, 49, 210, 1, + 53, 130, 132, 20, 57, 166, 228, 13, 48, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 55, 3, 56, 22, 90, 48, 174, 243, 38, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, 5, 65, 32, + 69, 78, 68, 2, 129, 248, 15, 8, 32, 76, 79, 71, 79, 71, 82, 65, 24, 138, + 177, 33, 48, 2, 55, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 56, 3, 57, 60, 218, 142, 32, 51, 246, 214, 1, 48, 242, 1, 49, 3, 50, + 14, 96, 2, 76, 69, 250, 161, 2, 85, 208, 173, 31, 2, 83, 84, 230, 192, 3, + 69, 249, 162, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, + 133, 31, 83, 191, 161, 4, 85, 29, 104, 11, 73, 67, 76, 79, 67, 75, 87, + 73, 83, 69, 32, 233, 219, 35, 9, 69, 78, 78, 65, 32, 87, 73, 84, 72, 24, + 90, 84, 254, 220, 6, 67, 58, 68, 122, 71, 138, 4, 79, 253, 173, 27, 5, + 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, + 65, 68, 69, 68, 32, 187, 225, 6, 79, 10, 112, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, 80, 223, 224, 6, + 79, 2, 11, 84, 2, 169, 152, 36, 7, 32, 85, 45, 83, 72, 65, 80, 158, 1, + 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, 76, 32, 83, 89, + 77, 66, 79, 76, 32, 210, 15, 79, 38, 80, 167, 219, 38, 67, 140, 1, 222, + 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, 4, 79, 77, 69, + 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, 81, 85, 222, 3, + 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, 83, 76, 76, 2, + 85, 80, 208, 216, 15, 12, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, + 0, 5, 84, 73, 76, 68, 69, 191, 195, 19, 90, 14, 64, 6, 73, 82, 67, 76, + 69, 32, 165, 152, 36, 4, 79, 77, 77, 65, 12, 80, 2, 83, 84, 202, 138, 19, + 68, 170, 246, 5, 66, 214, 157, 10, 85, 175, 248, 2, 74, 4, 246, 197, 37, + 73, 215, 104, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, 253, 140, 31, 6, + 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, 6, 182, 137, 19, + 68, 174, 155, 16, 84, 159, 163, 1, 83, 4, 254, 156, 35, 85, 207, 170, 1, + 83, 10, 22, 32, 171, 9, 87, 8, 52, 5, 84, 65, 67, 75, 32, 182, 1, 83, + 231, 6, 67, 4, 138, 156, 35, 85, 175, 248, 2, 74, 6, 40, 2, 79, 84, 173, + 155, 31, 2, 45, 66, 4, 11, 65, 5, 219, 139, 31, 32, 4, 158, 135, 19, 68, + 255, 147, 16, 85, 4, 28, 2, 32, 83, 191, 7, 87, 2, 149, 197, 36, 4, 72, + 79, 69, 32, 46, 44, 2, 65, 68, 137, 3, 4, 79, 84, 69, 32, 43, 11, 32, 40, + 178, 1, 67, 34, 68, 92, 2, 85, 80, 44, 2, 76, 69, 186, 158, 6, 81, 200, + 170, 13, 3, 78, 79, 84, 22, 69, 182, 176, 5, 66, 198, 133, 7, 83, 234, + 209, 4, 71, 162, 111, 82, 183, 79, 74, 4, 158, 146, 37, 73, 147, 127, 79, + 12, 54, 73, 36, 3, 79, 87, 78, 197, 188, 35, 2, 69, 76, 4, 218, 246, 30, + 86, 219, 174, 6, 65, 4, 152, 167, 10, 2, 32, 67, 215, 153, 27, 87, 4, + 174, 209, 36, 83, 255, 110, 70, 4, 242, 248, 11, 81, 139, 158, 23, 85, 4, + 184, 3, 5, 73, 71, 72, 84, 87, 131, 189, 38, 72, 10, 88, 5, 69, 77, 73, + 67, 79, 34, 76, 34, 84, 129, 247, 11, 7, 81, 85, 73, 83, 72, 32, 81, 2, + 229, 133, 31, 3, 76, 79, 78, 2, 177, 142, 36, 3, 65, 83, 72, 4, 248, 216, + 15, 2, 65, 82, 239, 133, 7, 73, 12, 22, 32, 171, 1, 87, 10, 78, 67, 36, + 5, 84, 65, 67, 75, 32, 253, 139, 38, 6, 83, 72, 79, 69, 32, 74, 2, 229, + 221, 22, 4, 65, 82, 69, 84, 6, 210, 255, 18, 68, 162, 195, 16, 79, 139, + 201, 2, 74, 2, 169, 207, 35, 6, 65, 82, 68, 83, 32, 86, 4, 242, 237, 32, + 83, 235, 157, 5, 76, 12, 92, 2, 82, 79, 153, 200, 25, 15, 76, 73, 67, 65, + 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, 88, 73, 77, 65, + 84, 69, 76, 89, 32, 153, 189, 38, 13, 65, 67, 72, 69, 83, 32, 84, 72, 69, + 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 201, 161, 24, 7, 66, + 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 255, 128, 31, 79, 5, 181, 201, + 25, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, 69, 230, 22, 148, + 1, 4, 65, 66, 73, 67, 208, 245, 1, 7, 77, 69, 78, 73, 65, 78, 32, 224, + 12, 3, 82, 79, 87, 236, 3, 2, 84, 73, 254, 188, 35, 73, 227, 147, 1, 67, + 148, 21, 54, 32, 141, 244, 1, 7, 45, 73, 78, 68, 73, 67, 32, 248, 20, + 178, 3, 67, 158, 1, 68, 242, 2, 69, 198, 1, 70, 212, 1, 2, 72, 65, 104, + 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 220, 18, 7, 78, + 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, 214, 1, 82, + 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, 69, 82, 84, + 98, 83, 154, 33, 84, 202, 4, 86, 188, 245, 16, 9, 87, 65, 86, 89, 32, 72, + 65, 77, 90, 138, 164, 17, 81, 201, 190, 1, 6, 90, 87, 65, 82, 65, 75, 14, + 44, 5, 85, 82, 76, 89, 32, 191, 135, 36, 79, 12, 72, 4, 68, 65, 77, 77, + 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, 5, 199, 138, 37, + 84, 24, 154, 1, 65, 108, 5, 79, 85, 66, 76, 69, 152, 2, 16, 73, 83, 80, + 85, 84, 69, 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 137, 253, 22, 5, 69, + 67, 73, 77, 65, 14, 36, 3, 77, 77, 65, 171, 189, 32, 84, 13, 22, 32, 211, + 5, 84, 6, 158, 226, 1, 73, 58, 77, 167, 202, 35, 87, 6, 202, 4, 68, 157, + 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, + 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, 69, 78, 84, 82, 69, 32, + 57, 6, 78, 68, 32, 79, 70, 32, 4, 184, 131, 36, 4, 72, 73, 71, 72, 1, 3, + 76, 79, 87, 4, 26, 65, 139, 249, 28, 84, 2, 255, 148, 31, 89, 22, 84, 4, + 65, 84, 72, 65, 230, 209, 1, 79, 164, 196, 4, 3, 73, 86, 69, 223, 235, + 29, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, 32, 162, 222, + 1, 73, 59, 77, 6, 170, 254, 36, 84, 174, 77, 68, 191, 47, 82, 6, 72, 13, + 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 231, 16, 77, 2, 181, + 157, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 230, 72, 87, 146, 148, 1, 73, + 59, 77, 4, 161, 82, 2, 65, 78, 224, 15, 84, 5, 65, 82, 71, 69, 32, 146, + 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, 64, 10, 82, 79, 85, + 78, 68, 32, 68, 79, 84, 32, 251, 218, 10, 67, 6, 220, 218, 10, 6, 73, 78, + 83, 73, 68, 69, 238, 213, 25, 66, 223, 155, 1, 65, 252, 7, 80, 5, 84, 84, + 69, 82, 32, 157, 251, 6, 9, 70, 84, 32, 65, 82, 82, 79, 87, 72, 248, 7, + 206, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 214, 5, 70, 246, 4, 71, 226, + 2, 72, 156, 9, 2, 74, 69, 122, 75, 246, 5, 76, 222, 2, 77, 154, 1, 78, + 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, 214, 1, 82, 230, 3, 83, + 158, 9, 84, 186, 7, 85, 148, 2, 2, 86, 69, 28, 3, 87, 65, 87, 202, 1, 89, + 246, 3, 79, 248, 2, 2, 90, 65, 31, 69, 112, 92, 7, 70, 82, 73, 67, 65, + 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 195, 193, 38, 69, 8, 52, 3, + 81, 65, 70, 214, 144, 32, 70, 243, 161, 3, 78, 5, 221, 64, 5, 32, 87, 73, + 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, 32, 84, 154, 160, 1, 70, + 230, 43, 73, 203, 9, 77, 10, 60, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, + 32, 131, 55, 87, 6, 216, 204, 14, 17, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 68, 79, 87, 78, 87, 65, 82, 68, 166, 221, 21, 66, 223, 155, 1, 65, 83, + 11, 32, 80, 70, 87, 196, 45, 6, 77, 65, 75, 83, 85, 82, 202, 112, 70, + 247, 52, 73, 70, 48, 4, 73, 84, 72, 32, 145, 45, 3, 65, 83, 76, 64, 192, + 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, 82, 73, 71, 72, 84, + 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, 69, 188, 1, 6, 72, + 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, 77, 254, 69, 69, + 204, 1, 4, 77, 65, 68, 68, 196, 139, 35, 9, 76, 69, 70, 84, 32, 77, 73, + 68, 68, 243, 230, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, 79, 77, 32, 82, + 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, 32, 82, 73, 71, + 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, 186, 201, 1, 75, + 223, 160, 31, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, 32, 4, 196, 231, + 18, 3, 76, 69, 70, 255, 211, 18, 68, 8, 52, 3, 73, 71, 72, 253, 214, 1, + 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 162, 176, 1, 82, 227, 37, 72, 12, + 146, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 211, 143, 36, 90, 50, 22, 72, + 219, 63, 69, 41, 22, 32, 195, 63, 69, 28, 68, 5, 87, 73, 84, 72, 32, 198, + 150, 1, 70, 230, 43, 73, 203, 9, 77, 20, 116, 6, 83, 77, 65, 76, 76, 32, + 34, 84, 154, 19, 73, 200, 33, 10, 68, 79, 84, 32, 66, 69, 76, 79, 87, 32, + 215, 19, 72, 6, 226, 39, 77, 175, 232, 22, 86, 8, 88, 10, 72, 82, 69, 69, + 32, 68, 79, 84, 83, 32, 177, 49, 7, 87, 79, 32, 68, 79, 84, 83, 6, 144, + 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, 82, 68, 83, + 32, 66, 69, 76, 79, 87, 133, 29, 8, 72, 79, 82, 73, 90, 79, 78, 84, 5, + 227, 191, 14, 32, 76, 90, 65, 164, 4, 2, 68, 65, 40, 7, 79, 84, 76, 69, + 83, 83, 32, 190, 28, 89, 255, 24, 85, 42, 34, 76, 154, 4, 72, 187, 46, + 68, 25, 11, 32, 22, 56, 5, 87, 73, 84, 72, 32, 226, 145, 1, 70, 247, 52, + 73, 18, 132, 1, 9, 68, 79, 84, 32, 66, 69, 76, 79, 87, 16, 9, 73, 78, 86, + 69, 82, 84, 69, 68, 32, 34, 84, 202, 215, 11, 70, 203, 138, 26, 82, 5, + 243, 1, 32, 4, 226, 14, 83, 199, 164, 38, 86, 6, 160, 1, 10, 72, 82, 69, + 69, 32, 68, 79, 84, 83, 32, 33, 25, 87, 79, 32, 68, 79, 84, 83, 32, 86, + 69, 82, 84, 73, 67, 65, 76, 76, 89, 32, 66, 69, 76, 79, 87, 32, 4, 234, + 53, 65, 195, 227, 35, 66, 2, 253, 235, 14, 10, 65, 78, 68, 32, 83, 77, + 65, 76, 76, 32, 12, 22, 72, 207, 62, 76, 6, 199, 53, 65, 6, 190, 253, 31, + 66, 2, 70, 139, 178, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, + 169, 2, 2, 69, 72, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 194, 140, + 1, 70, 230, 43, 73, 203, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, + 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, + 71, 73, 84, 32, 30, 84, 163, 40, 73, 6, 250, 8, 70, 219, 49, 84, 4, 226, + 245, 6, 72, 131, 194, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, + 154, 138, 1, 70, 230, 43, 73, 203, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, + 84, 6, 254, 40, 66, 249, 142, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, + 69, 69, 32, 68, 79, 84, 83, 32, 203, 132, 36, 87, 4, 198, 17, 80, 227, + 129, 36, 66, 44, 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 178, 21, 85, + 143, 147, 37, 82, 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 194, 135, 1, + 70, 230, 43, 73, 203, 9, 77, 8, 162, 24, 84, 134, 193, 37, 82, 229, 32, + 8, 73, 78, 86, 69, 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, + 32, 162, 134, 1, 70, 230, 43, 73, 203, 9, 77, 4, 254, 37, 84, 171, 140, + 26, 68, 82, 78, 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, + 77, 90, 65, 34, 34, 72, 213, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, + 87, 73, 84, 72, 32, 186, 132, 1, 70, 230, 43, 73, 203, 9, 77, 20, 132, 2, + 29, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, + 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, + 65, 76, 76, 32, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, + 84, 65, 72, 32, 62, 84, 135, 52, 72, 2, 145, 186, 1, 2, 79, 85, 2, 45, 9, + 78, 86, 69, 82, 84, 69, 68, 32, 83, 2, 221, 253, 35, 6, 77, 65, 76, 76, + 32, 86, 6, 26, 65, 199, 139, 36, 66, 4, 174, 31, 78, 255, 135, 37, 66, 8, + 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, + 79, 84, 83, 32, 4, 230, 8, 80, 191, 157, 37, 65, 4, 232, 165, 37, 8, 86, + 69, 82, 84, 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, + 65, 76, 88, 5, 87, 73, 84, 72, 32, 220, 46, 10, 68, 79, 65, 67, 72, 65, + 83, 72, 77, 69, 242, 78, 70, 230, 43, 73, 203, 9, 77, 13, 11, 32, 10, + 156, 49, 6, 87, 73, 84, 72, 32, 72, 250, 76, 70, 230, 43, 73, 203, 9, 77, + 8, 218, 26, 73, 245, 18, 3, 89, 69, 72, 9, 11, 32, 6, 166, 254, 22, 65, + 178, 238, 8, 89, 135, 181, 5, 87, 22, 28, 2, 69, 77, 131, 45, 72, 17, 11, + 32, 14, 140, 31, 6, 87, 73, 84, 72, 32, 84, 170, 93, 70, 230, 43, 73, + 203, 9, 77, 68, 98, 65, 208, 1, 4, 69, 72, 69, 72, 160, 3, 7, 73, 82, 71, + 72, 73, 90, 32, 205, 30, 2, 72, 65, 22, 46, 70, 177, 162, 1, 5, 83, 72, + 77, 73, 82, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 138, 122, 70, 230, + 43, 73, 203, 9, 77, 10, 42, 84, 130, 174, 34, 68, 199, 157, 3, 82, 4, + 174, 166, 14, 87, 207, 206, 21, 72, 25, 11, 32, 22, 64, 5, 87, 73, 84, + 72, 32, 234, 120, 70, 230, 43, 73, 203, 9, 77, 14, 38, 84, 226, 41, 83, + 143, 241, 36, 68, 10, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, + 121, 8, 87, 79, 32, 68, 79, 84, 83, 32, 6, 42, 80, 226, 129, 36, 66, 223, + 155, 1, 65, 2, 165, 243, 35, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, + 87, 65, 82, 68, 4, 48, 6, 86, 69, 82, 84, 73, 67, 191, 156, 37, 65, 2, + 145, 188, 11, 3, 65, 76, 76, 12, 198, 38, 79, 13, 2, 89, 85, 26, 40, 2, + 65, 77, 245, 169, 1, 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, + 32, 254, 116, 70, 230, 43, 73, 203, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, + 77, 65, 76, 76, 32, 216, 162, 32, 2, 84, 72, 223, 209, 4, 66, 4, 166, + 161, 17, 85, 231, 248, 19, 84, 4, 204, 26, 16, 65, 82, 65, 66, 73, 67, + 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 143, 252, 37, 86, 18, 36, 3, 69, + 69, 77, 195, 212, 37, 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 166, + 114, 70, 230, 43, 73, 203, 9, 77, 6, 130, 18, 84, 199, 148, 34, 68, 60, + 38, 71, 26, 89, 17, 3, 79, 79, 78, 21, 22, 79, 255, 108, 32, 10, 247, 25, + 69, 31, 11, 32, 28, 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, + 186, 112, 70, 230, 43, 73, 203, 9, 77, 6, 131, 33, 65, 14, 116, 6, 83, + 77, 65, 76, 76, 32, 38, 84, 184, 23, 8, 73, 78, 86, 69, 82, 84, 69, 68, + 238, 131, 26, 68, 227, 165, 11, 82, 4, 214, 217, 30, 84, 203, 184, 7, 86, + 4, 238, 217, 6, 72, 247, 144, 29, 87, 25, 22, 32, 151, 23, 69, 12, 88, + 11, 87, 73, 84, 72, 32, 83, 77, 65, 76, 76, 32, 134, 110, 70, 230, 43, + 73, 203, 9, 77, 4, 26, 77, 203, 144, 38, 86, 2, 209, 147, 37, 3, 69, 69, + 77, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 250, 108, 70, 230, 43, 73, + 203, 9, 77, 8, 36, 4, 68, 79, 84, 32, 179, 12, 84, 6, 44, 5, 66, 69, 76, + 79, 87, 167, 146, 37, 65, 5, 241, 152, 14, 6, 32, 65, 78, 68, 32, 78, 52, + 108, 2, 69, 72, 192, 27, 3, 82, 69, 72, 136, 3, 4, 78, 79, 79, 78, 197, + 118, 7, 79, 72, 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, + 72, 32, 194, 106, 70, 247, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, + 73, 250, 21, 72, 154, 148, 11, 70, 148, 226, 6, 5, 68, 79, 84, 32, 66, + 234, 39, 76, 207, 128, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 251, 219, + 37, 84, 8, 194, 6, 65, 182, 198, 6, 78, 215, 150, 16, 86, 4, 37, 7, 79, + 32, 68, 79, 84, 83, 32, 4, 162, 8, 86, 147, 134, 37, 65, 60, 184, 1, 2, + 65, 68, 120, 3, 69, 69, 78, 132, 6, 4, 72, 69, 69, 78, 220, 147, 1, 4, + 85, 80, 69, 82, 156, 145, 4, 7, 84, 82, 65, 73, 71, 72, 84, 169, 220, 31, + 6, 87, 65, 83, 72, 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, + 130, 102, 70, 230, 43, 73, 203, 9, 77, 6, 218, 148, 32, 72, 239, 204, 3, + 87, 27, 11, 32, 24, 64, 5, 87, 73, 84, 72, 32, 142, 101, 70, 230, 43, 73, + 203, 9, 77, 16, 232, 1, 3, 68, 79, 84, 50, 73, 44, 7, 83, 77, 65, 76, 76, + 32, 65, 110, 84, 250, 168, 11, 70, 177, 137, 5, 31, 69, 88, 84, 69, 78, + 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, + 73, 71, 73, 84, 32, 70, 79, 85, 2, 249, 140, 18, 7, 32, 66, 69, 76, 79, + 87, 32, 2, 169, 20, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, + 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 159, + 167, 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, + 9, 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, + 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 243, 202, 6, 72, 2, 209, + 165, 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 158, + 95, 70, 230, 43, 73, 203, 9, 77, 2, 161, 139, 26, 5, 73, 84, 72, 32, 68, + 118, 92, 2, 65, 72, 144, 1, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, + 72, 61, 3, 84, 69, 72, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 198, + 93, 70, 230, 43, 73, 203, 9, 77, 8, 26, 84, 179, 137, 26, 68, 6, 250, + 137, 14, 87, 139, 130, 18, 72, 25, 22, 32, 163, 5, 69, 12, 64, 5, 87, 73, + 84, 72, 32, 170, 92, 70, 230, 43, 73, 203, 9, 77, 4, 198, 13, 83, 143, + 241, 36, 68, 37, 22, 32, 167, 4, 69, 24, 62, 77, 116, 5, 87, 73, 84, 72, + 32, 190, 90, 70, 231, 43, 73, 10, 48, 6, 65, 82, 66, 85, 84, 65, 183, + 144, 1, 69, 9, 11, 32, 6, 238, 90, 70, 246, 52, 73, 197, 228, 35, 2, 71, + 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, 84, 72, 82, 69, 69, 32, + 68, 79, 84, 83, 32, 65, 131, 171, 37, 82, 4, 32, 2, 84, 69, 143, 252, 37, + 86, 2, 151, 255, 36, 72, 2, 237, 135, 27, 4, 66, 79, 86, 69, 18, 42, 65, + 126, 69, 213, 129, 1, 2, 73, 78, 6, 131, 9, 76, 23, 18, 32, 91, 69, 10, + 60, 4, 87, 73, 84, 72, 230, 87, 70, 230, 43, 73, 203, 9, 77, 2, 129, 9, + 2, 32, 83, 10, 143, 11, 72, 15, 158, 1, 32, 181, 80, 33, 73, 71, 72, 85, + 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, 71, 72, 73, 90, 32, 65, + 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, 96, 16, 87, 73, 84, 72, + 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 186, 85, 70, 247, 52, 73, + 5, 155, 75, 32, 17, 234, 8, 72, 167, 76, 32, 25, 11, 32, 22, 52, 5, 87, + 73, 84, 72, 32, 202, 84, 70, 247, 52, 73, 18, 80, 4, 68, 79, 84, 32, 162, + 2, 69, 182, 1, 72, 234, 252, 13, 84, 163, 165, 23, 82, 4, 192, 178, 30, + 3, 87, 73, 84, 183, 199, 6, 65, 54, 28, 2, 69, 72, 227, 3, 85, 49, 11, + 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, 2, 5, 87, 73, 84, 72, 32, + 182, 79, 70, 230, 43, 73, 203, 9, 77, 17, 11, 32, 14, 52, 5, 87, 73, 84, + 72, 32, 238, 81, 70, 247, 52, 73, 10, 22, 69, 183, 1, 72, 4, 121, 28, 88, + 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, + 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 160, 229, 34, 3, 72, 82, 69, 161, + 145, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, 11, 65, 6, 17, 2, 32, 65, 6, + 21, 3, 66, 79, 86, 6, 11, 69, 7, 171, 79, 32, 22, 64, 10, 72, 65, 77, 90, + 65, 32, 65, 66, 79, 86, 18, 83, 51, 84, 10, 179, 2, 69, 2, 25, 4, 77, 65, + 76, 76, 2, 255, 150, 31, 32, 10, 108, 18, 87, 79, 32, 68, 79, 84, 83, 32, + 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 174, 195, 35, 65, 183, 5, 72, 6, + 70, 72, 232, 176, 6, 7, 83, 77, 65, 76, 76, 32, 78, 191, 190, 30, 68, 2, + 205, 248, 32, 3, 65, 77, 90, 18, 26, 72, 17, 2, 73, 78, 11, 243, 71, 32, + 9, 11, 32, 6, 158, 76, 70, 246, 52, 73, 161, 11, 13, 87, 73, 84, 72, 32, + 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, 65, 158, 5, 66, 176, 2, + 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, 69, 72, 32, 87, 73, 84, + 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, 84, 72, 32, 154, 1, 72, + 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, 32, 87, 73, 84, 72, 32, + 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 182, 2, + 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, 7, 87, 65, 83, 65, 76, + 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, 216, 3, 53, 85, 73, + 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, 89, 69, 72, 32, 87, + 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, + 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 205, 6, 14, + 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, 32, 54, 128, 1, 8, + 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, 75, 66, 65, 82, 205, + 168, 35, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, 19, 4, 77, 69, 69, + 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, 65, 89, 72, 216, 1, + 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, 56, 69, 8, 52, 9, + 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, 84, 85, 32, 87, + 139, 229, 37, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, 149, 153, 32, + 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, 85, 82, 65, + 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, + 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, 65, 84, 65, + 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 141, 135, 25, 25, 73, + 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, 65, 78, 32, + 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, 52, 65, 250, + 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, 65, 171, 56, + 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, 250, 3, 74, + 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, 3, 74, 146, + 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, 250, 50, 65, + 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 235, 103, 73, 6, + 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, 87, 73, 84, + 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, 77, 69, 69, + 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, 69, 69, 77, + 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, 74, 53, 16, + 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 8, 40, 5, + 87, 73, 84, 72, 32, 215, 100, 73, 4, 254, 43, 74, 3, 77, 30, 104, 9, 69, + 69, 77, 32, 87, 73, 84, 72, 32, 173, 230, 3, 11, 65, 76, 76, 65, 74, 65, + 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, 32, 186, 45, 65, + 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, 191, 69, 73, 12, + 40, 5, 87, 73, 84, 72, 32, 171, 98, 73, 8, 166, 37, 72, 242, 3, 65, 203, + 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, 9, 72, 65, 72, 32, + 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, 74, 96, 2, 76, 65, + 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, 250, 42, 65, 250, 3, + 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, 65, 76, 69, 70, 32, + 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, 5, 77, 69, 69, 77, + 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, 44, 77, 222, 6, 70, + 247, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, 7, 77, 65, 68, 68, + 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, 229, 45, 3, 66, 79, + 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, 40, 5, 87, 73, 84, + 72, 32, 231, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, 89, 14, 40, 5, 87, + 73, 84, 72, 32, 151, 92, 73, 10, 130, 47, 74, 2, 77, 75, 89, 8, 129, 15, + 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 230, 43, 73, 203, 9, 77, 48, + 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, 7, 79, 72, 65, 77, + 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, 2, 75, 72, 104, 5, + 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, 4, 22, 77, 219, + 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, 65, 72, 32, 10, + 40, 5, 87, 73, 84, 72, 32, 211, 88, 73, 6, 250, 31, 74, 2, 77, 143, 12, + 89, 12, 40, 5, 87, 73, 84, 72, 32, 131, 88, 73, 8, 170, 31, 77, 194, 7, + 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 231, 43, 73, 2, 11, + 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, 32, 92, 5, 77, + 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, 75, 238, 1, 82, + 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, 84, 72, 32, 191, + 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, 89, 12, 206, 15, + 87, 218, 25, 70, 230, 43, 73, 203, 9, 77, 28, 70, 65, 197, 146, 30, 11, + 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, 70, 32, 87, 73, + 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, 69, 77, 32, + 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, 72, 32, 223, + 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, 69, 69, 77, + 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, 65, 76, 76, + 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, 85, 161, 26, + 4, 83, 79, 85, 76, 11, 26, 85, 163, 197, 37, 65, 6, 26, 77, 251, 140, 36, + 78, 5, 131, 197, 37, 65, 4, 26, 32, 1, 2, 77, 32, 2, 193, 142, 30, 4, 65, + 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, 32, 87, 73, 84, + 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, 65, 72, 85, 32, + 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, 1, 76, 32, 76, 4, + 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, 4, 82, 3, 89, 10, + 22, 87, 195, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, 16, 72, 187, 16, + 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, 73, 84, 72, 32, + 77, 147, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, 77, 193, 189, 35, + 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, 32, 133, 1, 3, + 76, 76, 65, 4, 22, 85, 159, 84, 73, 2, 217, 8, 23, 83, 69, 68, 32, 65, + 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, 83, 73, 71, + 6, 98, 72, 169, 191, 37, 18, 65, 72, 85, 32, 65, 76, 65, 89, 72, 73, 32, + 87, 65, 45, 65, 65, 76, 73, 4, 220, 63, 12, 79, 85, 32, 65, 76, 65, 89, + 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, 32, 87, 65, + 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, 100, 5, 74, + 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, 77, 32, + 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, 69, 8, + 156, 14, 6, 87, 73, 84, 72, 32, 74, 214, 56, 73, 203, 9, 77, 10, 52, 5, + 87, 73, 84, 72, 32, 254, 69, 73, 203, 9, 77, 4, 234, 12, 65, 139, 8, 72, + 10, 34, 87, 190, 69, 73, 203, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, 142, + 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 206, 68, 73, 203, 9, + 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, 68, 65, + 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, 72, 32, + 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, 237, 41, 6, + 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 203, 52, 32, 2, 163, + 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, 69, 69, 77, + 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, 89, 18, 194, + 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 230, 43, 73, 203, + 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 230, 43, 73, 203, 9, + 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, 8, 69, 72, + 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, 82, 65, 75, + 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, 177, 163, 27, + 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, 6, 72, 139, 2, + 89, 14, 52, 5, 87, 73, 84, 72, 32, 166, 61, 73, 203, 9, 77, 8, 34, 72, + 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, 1, 72, 108, 3, + 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, 32, 238, 4, 65, + 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, 72, 32, 227, 9, + 69, 10, 40, 5, 87, 73, 84, 72, 32, 143, 59, 73, 6, 182, 2, 77, 199, 11, + 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, 32, 175, 58, 73, + 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, 84, 72, 32, 226, + 13, 70, 230, 43, 73, 203, 9, 77, 10, 50, 65, 62, 74, 194, 7, 75, 14, 72, + 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, + 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, 69, 72, 32, 87, 73, + 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, 197, 1, 15, 85, 80, + 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 26, 108, 3, 74, 69, + 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, 42, 89, 233, 53, 5, + 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 143, 63, 73, 114, 82, 65, + 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, 1, 82, 43, 89, + 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, 72, 80, 15, 77, + 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, 147, 5, 72, 66, + 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, 22, 79, 16, 2, + 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, 247, 6, 69, 8, + 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 247, 52, 73, 4, 181, 6, 6, 65, + 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 231, 43, 73, 8, 22, 69, + 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 163, 54, 77, 4, 251, 48, + 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, 17, 2, 69, + 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 230, 43, 73, 203, 9, 77, 2, 93, + 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, 85, 2, 17, 2, + 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, 22, 87, 219, + 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, 65, 4, 135, + 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, + 194, 1, 70, 230, 43, 73, 203, 9, 77, 6, 18, 77, 75, 89, 4, 21, 3, 69, 69, + 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 239, 44, 78, 2, 17, 2, 69, 72, 2, + 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, 247, 52, 73, + 2, 205, 53, 2, 73, 78, 6, 202, 43, 73, 203, 9, 77, 166, 2, 92, 3, 68, 68, + 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, 73, 67, 65, 76, + 32, 4, 148, 231, 35, 5, 32, 87, 65, 65, 74, 191, 61, 72, 4, 58, 78, 1, + 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 221, 254, 33, 7, 79, 79, + 78, 32, 71, 72, 85, 158, 2, 178, 2, 68, 144, 3, 8, 73, 78, 73, 84, 73, + 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, 79, 82, 32, + 166, 1, 83, 146, 3, 84, 158, 205, 23, 65, 102, 75, 110, 90, 246, 104, 74, + 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, 89, 174, 57, + 71, 198, 232, 2, 78, 154, 144, 2, 81, 255, 2, 87, 62, 26, 79, 187, 155, + 36, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, 69, 45, + 83, 84, 82, 85, 67, 75, 32, 8, 214, 233, 30, 66, 2, 70, 242, 161, 3, 78, + 155, 144, 2, 81, 50, 190, 12, 83, 186, 20, 75, 158, 183, 23, 84, 74, 90, + 246, 104, 74, 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, + 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, + 81, 222, 1, 65, 163, 1, 87, 40, 182, 1, 84, 174, 9, 83, 150, 203, 23, 72, + 30, 75, 226, 105, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 89, 174, 57, 71, + 198, 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 222, 1, 65, 159, 57, 68, + 4, 134, 230, 30, 72, 147, 178, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, 32, + 251, 135, 37, 65, 54, 210, 8, 83, 186, 20, 75, 150, 182, 23, 65, 74, 72, + 66, 84, 74, 90, 246, 104, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 82, 2, + 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, + 81, 255, 2, 87, 4, 160, 154, 28, 23, 77, 69, 69, 77, 32, 87, 73, 84, 72, + 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 177, 233, 7, 9, + 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, + 69, 68, 32, 134, 196, 35, 72, 14, 69, 227, 138, 1, 65, 46, 182, 1, 68, + 86, 84, 254, 2, 83, 150, 203, 23, 72, 30, 75, 226, 105, 74, 2, 77, 166, + 162, 5, 90, 170, 133, 1, 66, 2, 70, 2, 89, 174, 57, 71, 198, 232, 2, 78, + 154, 144, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 131, + 205, 36, 65, 4, 242, 223, 30, 66, 3, 70, 6, 214, 223, 30, 72, 146, 178, + 6, 65, 3, 69, 38, 42, 65, 158, 206, 23, 72, 247, 194, 13, 69, 32, 44, 5, + 73, 76, 69, 68, 32, 243, 145, 37, 72, 30, 150, 1, 68, 94, 83, 186, 20, + 75, 218, 160, 24, 74, 166, 162, 5, 72, 170, 133, 1, 89, 174, 57, 71, 198, + 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 223, 1, 65, 6, 52, 7, 79, 84, + 76, 69, 83, 83, 32, 143, 202, 36, 65, 4, 238, 254, 33, 78, 155, 144, 2, + 81, 6, 250, 190, 35, 72, 14, 69, 227, 138, 1, 65, 4, 238, 252, 17, 77, + 147, 140, 18, 83, 6, 128, 149, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, + 1, 4, 70, 65, 84, 72, 10, 130, 1, 79, 212, 156, 12, 11, 76, 65, 67, 69, + 32, 79, 70, 32, 83, 65, 74, 144, 222, 5, 6, 73, 65, 83, 84, 82, 69, 175, + 173, 13, 69, 4, 148, 141, 12, 8, 69, 84, 73, 67, 32, 86, 69, 82, 205, + 237, 5, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, 83, 36, 15, + 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, 201, 206, 22, + 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, 79, 80, 32, + 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, 69, 68, + 239, 138, 37, 89, 2, 17, 2, 32, 82, 2, 165, 193, 35, 3, 79, 85, 78, 2, + 133, 135, 21, 4, 69, 68, 32, 68, 6, 26, 65, 195, 241, 34, 66, 4, 193, + 230, 35, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, 68, + 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, 85, + 244, 2, 6, 89, 77, 66, 79, 76, 32, 209, 178, 35, 18, 84, 65, 82, 84, 32, + 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, 85, + 69, 78, 67, 69, 32, 223, 209, 33, 77, 18, 184, 1, 26, 89, 69, 72, 32, 87, + 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, + 84, 72, 32, 209, 210, 30, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 75, + 69, 72, 16, 70, 65, 142, 135, 36, 87, 182, 89, 89, 150, 14, 79, 214, 22, + 69, 3, 85, 6, 36, 3, 76, 69, 70, 131, 133, 37, 69, 5, 159, 13, 32, 7, 11, + 32, 4, 246, 22, 73, 59, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, 1, 7, + 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 148, 33, 6, 84, 65, + 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, 72, 32, + 65, 76, 65, 213, 192, 32, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, 85, 32, + 65, 78, 2, 175, 213, 36, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, 72, 73, + 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, 76, 65, + 89, 72, 69, 32, 87, 166, 200, 29, 78, 142, 193, 5, 70, 237, 99, 2, 77, + 86, 2, 17, 2, 65, 83, 2, 229, 173, 15, 3, 83, 65, 76, 4, 140, 249, 19, + 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 233, 205, 13, 2, 65, + 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, 76, + 79, 87, 32, 110, 75, 170, 238, 20, 68, 146, 208, 9, 89, 135, 181, 5, 87, + 4, 190, 3, 82, 219, 131, 35, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, 83, + 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, 77, + 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, 72, + 228, 128, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, 78, + 71, 85, 76, 65, 196, 180, 8, 7, 82, 79, 85, 78, 68, 69, 68, 130, 122, 90, + 246, 104, 74, 214, 217, 11, 81, 223, 1, 65, 2, 251, 194, 29, 72, 4, 24, + 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 181, 220, 25, 6, 79, 84, + 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, 2, + 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, + 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 153, + 140, 35, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 197, 30, 2, 32, 89, 2, + 89, 20, 32, 87, 73, 84, 72, 32, 76, 65, 77, 32, 87, 73, 84, 72, 32, 65, + 76, 69, 70, 32, 2, 225, 193, 21, 3, 77, 65, 75, 6, 26, 69, 215, 201, 15, + 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 197, 9, + 2, 73, 84, 5, 207, 4, 32, 6, 252, 254, 34, 7, 73, 71, 78, 32, 83, 65, 70, + 202, 38, 69, 227, 138, 1, 65, 4, 194, 243, 22, 72, 159, 129, 14, 65, 20, + 40, 4, 79, 82, 68, 32, 147, 247, 35, 65, 18, 74, 65, 172, 1, 2, 83, 65, + 200, 214, 24, 3, 87, 65, 81, 143, 155, 11, 81, 10, 228, 225, 6, 3, 76, + 45, 74, 236, 253, 17, 3, 82, 45, 82, 132, 196, 6, 7, 84, 72, 45, 84, 72, + 65, 76, 240, 207, 4, 5, 78, 45, 78, 73, 83, 137, 92, 5, 83, 45, 83, 65, + 74, 4, 218, 206, 36, 75, 207, 36, 72, 5, 149, 150, 10, 12, 32, 66, 65, + 82, 82, 69, 69, 32, 87, 73, 84, 72, 20, 68, 5, 78, 79, 79, 78, 32, 70, + 87, 206, 150, 24, 77, 239, 137, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, + 2, 11, 65, 2, 151, 173, 36, 83, 14, 40, 4, 79, 82, 68, 32, 255, 242, 35, + 65, 12, 106, 73, 198, 194, 15, 77, 204, 135, 9, 3, 84, 65, 83, 212, 192, + 8, 2, 83, 65, 237, 172, 3, 3, 81, 65, 83, 4, 252, 230, 12, 2, 77, 65, + 177, 189, 18, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 177, 152, + 24, 22, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 32, 77, + 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 229, 204, + 21, 2, 32, 65, 9, 11, 32, 6, 34, 73, 58, 77, 255, 212, 34, 66, 2, 11, 83, + 2, 141, 239, 35, 6, 79, 76, 65, 84, 69, 68, 2, 11, 69, 2, 11, 68, 2, 11, + 73, 2, 189, 238, 35, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 224, 145, + 5, 8, 83, 77, 65, 76, 76, 32, 84, 65, 156, 229, 25, 4, 70, 79, 85, 82, + 228, 124, 4, 87, 65, 83, 76, 147, 166, 4, 82, 6, 88, 16, 85, 66, 76, 69, + 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 159, 210, 34, 84, 2, 151, + 196, 34, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, + 87, 79, 32, 68, 79, 84, 83, 32, 8, 184, 246, 30, 17, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 154, 219, 3, 66, 223, + 155, 1, 65, 8, 176, 208, 34, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, + 42, 66, 223, 155, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 244, + 209, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 152, 164, 6, + 8, 85, 82, 78, 69, 68, 32, 68, 65, 149, 222, 13, 8, 72, 79, 85, 83, 65, + 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 157, 180, 9, 6, 73, 76, 32, + 70, 82, 65, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 112, 11, 79, 86, 69, + 82, 83, 84, 82, 85, 67, 75, 32, 184, 166, 5, 7, 70, 65, 84, 72, 65, 84, + 65, 131, 226, 4, 84, 4, 26, 72, 223, 230, 35, 87, 2, 197, 254, 28, 2, 65, + 77, 12, 68, 3, 79, 78, 69, 234, 159, 29, 84, 205, 171, 5, 4, 76, 79, 79, + 80, 4, 213, 245, 32, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 243, 178, 23, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, + 32, 83, 2, 83, 147, 236, 24, 68, 2, 25, 4, 77, 65, 76, 76, 2, 213, 229, + 35, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 136, 214, 5, 3, 67, 85, 66, + 208, 128, 5, 5, 70, 79, 85, 82, 84, 211, 156, 24, 68, 4, 250, 182, 24, + 77, 31, 84, 188, 1, 222, 1, 65, 34, 67, 138, 3, 69, 60, 7, 83, 77, 65, + 76, 76, 32, 76, 240, 176, 20, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, + 69, 84, 84, 69, 82, 32, 76, 222, 225, 8, 72, 184, 168, 2, 3, 68, 82, 65, + 218, 213, 2, 70, 83, 81, 4, 214, 215, 30, 66, 135, 26, 80, 78, 80, 14, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 187, 147, 34, 79, + 76, 250, 1, 84, 36, 2, 89, 73, 150, 3, 67, 38, 82, 34, 69, 42, 71, 34, + 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 254, 161, 30, + 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, 54, 78, 226, 26, + 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 4, 178, 139, 35, 73, 163, 209, + 1, 79, 5, 219, 140, 36, 87, 4, 216, 135, 25, 6, 77, 80, 72, 65, 83, 73, + 191, 233, 5, 88, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, + 65, 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, + 38, 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, + 254, 161, 30, 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, + 54, 78, 226, 26, 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, + 250, 216, 36, 65, 3, 79, 4, 230, 215, 36, 69, 147, 1, 65, 6, 198, 215, + 36, 67, 146, 1, 72, 3, 84, 4, 142, 146, 36, 72, 203, 53, 73, 4, 238, 164, + 30, 72, 163, 179, 6, 65, 4, 207, 225, 32, 69, 4, 166, 217, 3, 73, 163, + 253, 32, 69, 4, 250, 212, 36, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, + 68, 32, 65, 174, 133, 35, 73, 163, 209, 1, 79, 2, 167, 156, 35, 89, 4, + 166, 216, 35, 69, 147, 126, 79, 7, 222, 216, 20, 32, 231, 173, 15, 87, + 12, 84, 5, 69, 67, 72, 32, 89, 20, 4, 77, 69, 78, 32, 133, 136, 29, 4, + 86, 69, 87, 32, 2, 251, 131, 35, 73, 8, 230, 161, 30, 88, 170, 227, 1, + 73, 238, 181, 3, 78, 231, 119, 69, 14, 116, 10, 32, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 177, 151, 35, 13, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, + 68, 32, 80, 12, 156, 2, 24, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, + 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, + 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 78, 79, 82, 201, 248, 33, 22, 68, + 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, + 73, 78, 71, 6, 44, 3, 83, 79, 85, 210, 239, 25, 68, 35, 85, 2, 173, 173, + 28, 4, 84, 72, 32, 87, 4, 144, 135, 30, 10, 67, 85, 76, 65, 84, 69, 68, + 32, 76, 79, 137, 133, 5, 6, 83, 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, + 146, 200, 22, 89, 241, 156, 9, 2, 83, 69, 4, 222, 141, 8, 69, 175, 128, + 11, 73, 12, 48, 4, 69, 82, 73, 83, 36, 2, 79, 78, 31, 82, 6, 238, 192, 2, + 75, 247, 141, 34, 77, 2, 209, 199, 13, 2, 73, 83, 4, 226, 254, 29, 65, + 189, 204, 4, 22, 79, 78, 79, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, + 76, 32, 70, 79, 82, 32, 85, 82, 4, 216, 219, 18, 6, 72, 76, 69, 84, 73, + 67, 189, 222, 16, 2, 79, 77, 10, 68, 2, 84, 79, 240, 142, 21, 2, 83, 84, + 185, 162, 2, 3, 66, 69, 82, 6, 54, 77, 201, 205, 35, 7, 32, 82, 73, 67, + 75, 83, 72, 4, 218, 176, 23, 79, 153, 182, 5, 12, 65, 84, 69, 68, 32, 84, + 69, 76, 76, 69, 82, 32, 112, 56, 6, 69, 83, 84, 65, 78, 32, 133, 196, 4, + 2, 79, 67, 110, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, 194, 30, 65, 108, + 234, 1, 65, 58, 71, 42, 72, 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 246, + 246, 32, 85, 158, 144, 1, 79, 182, 56, 73, 198, 134, 1, 66, 2, 68, 2, 90, + 246, 63, 69, 194, 41, 67, 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, + 82, 3, 86, 17, 254, 130, 34, 65, 158, 133, 2, 69, 150, 64, 78, 3, 79, 6, + 162, 177, 36, 71, 2, 72, 215, 22, 69, 4, 250, 176, 36, 77, 215, 22, 69, + 12, 46, 71, 174, 176, 36, 78, 2, 89, 215, 22, 69, 6, 170, 176, 36, 86, 2, + 89, 215, 22, 69, 8, 38, 72, 166, 153, 36, 83, 143, 45, 69, 4, 218, 175, + 36, 89, 215, 22, 69, 6, 186, 175, 36, 72, 2, 84, 215, 22, 69, 134, 37, + 178, 1, 65, 182, 169, 1, 69, 132, 16, 9, 72, 65, 73, 75, 83, 85, 75, 73, + 32, 198, 6, 73, 130, 3, 76, 186, 34, 79, 130, 55, 82, 138, 19, 85, 138, + 8, 89, 222, 139, 34, 80, 147, 1, 83, 182, 14, 132, 1, 2, 66, 89, 94, 67, + 206, 2, 68, 146, 1, 71, 106, 76, 140, 28, 4, 77, 85, 77, 32, 230, 114, + 78, 178, 1, 82, 98, 83, 223, 6, 84, 11, 11, 32, 8, 238, 188, 12, 67, 134, + 212, 5, 66, 204, 209, 13, 3, 65, 78, 71, 227, 205, 3, 83, 16, 62, 75, + 172, 223, 8, 5, 84, 82, 73, 65, 78, 139, 147, 27, 79, 12, 42, 32, 46, 83, + 229, 186, 10, 2, 45, 84, 4, 178, 220, 9, 87, 161, 145, 20, 2, 79, 70, 6, + 132, 1, 26, 76, 65, 78, 84, 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, + 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, 175, 131, 36, 80, 4, 150, + 215, 28, 82, 245, 130, 5, 2, 79, 75, 4, 188, 215, 33, 27, 77, 73, 78, 84, + 79, 78, 32, 82, 65, 67, 81, 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, + 84, 84, 76, 69, 67, 223, 170, 2, 71, 6, 228, 247, 29, 6, 85, 69, 84, 84, + 69, 32, 234, 189, 5, 69, 229, 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 150, + 2, 44, 6, 73, 78, 69, 83, 69, 32, 131, 24, 76, 248, 1, 188, 2, 6, 67, 65, + 82, 73, 75, 32, 104, 7, 76, 69, 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, + 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, + 120, 5, 83, 73, 71, 78, 32, 180, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 192, 246, 13, 5, 65, 68, 69, 71, 32, 162, 204, 18, 87, 219, + 247, 1, 68, 6, 32, 2, 80, 65, 255, 230, 2, 83, 4, 156, 130, 29, 5, 77, + 85, 78, 71, 75, 213, 231, 5, 3, 82, 69, 82, 110, 166, 2, 65, 112, 2, 66, + 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, 30, 73, 2, 79, + 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, 2, 80, 65, 36, + 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, 90, 65, 76, + 170, 175, 36, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, 3, 5, 83, 89, + 85, 82, 65, 172, 243, 14, 8, 82, 67, 72, 65, 73, 67, 32, 74, 243, 217, 3, + 73, 5, 213, 165, 18, 3, 32, 75, 69, 5, 221, 192, 22, 3, 32, 76, 65, 9, + 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 175, 208, 32, 65, 4, 166, + 129, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 163, 237, 35, 75, 5, 185, + 202, 33, 2, 32, 71, 4, 11, 75, 4, 229, 14, 2, 65, 82, 5, 133, 169, 34, 2, + 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 247, 252, 13, + 32, 7, 199, 12, 32, 8, 34, 65, 182, 177, 36, 71, 3, 89, 5, 221, 154, 26, + 4, 32, 82, 65, 77, 5, 157, 170, 35, 4, 32, 75, 65, 80, 7, 235, 11, 32, 7, + 21, 3, 32, 83, 65, 4, 194, 176, 36, 71, 3, 80, 10, 30, 65, 97, 3, 90, 73, + 82, 9, 11, 32, 6, 196, 253, 13, 6, 77, 85, 82, 68, 65, 32, 208, 7, 3, 76, + 65, 84, 151, 232, 17, 84, 2, 229, 249, 13, 2, 32, 83, 56, 168, 1, 10, 67, + 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, 10, 76, 69, 70, + 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, 45, 72, 65, 78, + 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 180, 181, 19, 2, 66, 69, + 136, 159, 7, 3, 69, 78, 68, 232, 166, 3, 3, 84, 69, 71, 195, 246, 3, 71, + 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, 84, 72, 32, 74, 2, + 225, 243, 13, 3, 69, 71, 79, 20, 50, 65, 90, 69, 174, 220, 35, 73, 2, 79, + 3, 85, 10, 40, 2, 78, 71, 218, 220, 35, 69, 3, 73, 7, 11, 32, 4, 158, 4, + 83, 227, 203, 10, 71, 4, 170, 220, 35, 85, 155, 80, 78, 10, 76, 6, 79, + 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, 32, 80, 76, 6, 186, + 219, 35, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, 69, 68, 32, 84, 29, + 6, 79, 80, 69, 78, 32, 68, 4, 158, 168, 36, 65, 3, 85, 4, 198, 170, 36, + 65, 3, 85, 10, 30, 77, 49, 3, 78, 84, 73, 6, 44, 3, 65, 68, 65, 129, 238, + 33, 2, 69, 78, 5, 233, 209, 35, 5, 32, 76, 65, 78, 84, 12, 110, 83, 20, + 4, 85, 76, 85, 32, 188, 240, 28, 3, 66, 73, 83, 150, 134, 1, 67, 153, + 248, 1, 4, 82, 69, 82, 69, 2, 247, 146, 24, 85, 4, 130, 245, 19, 67, 213, + 161, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, + 83, 85, 75, 85, 34, 84, 104, 5, 80, 69, 80, 69, 84, 53, 3, 85, 76, 85, 4, + 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 233, 173, 35, 3, 32, 73, 76, + 10, 36, 5, 65, 76, 73, 78, 71, 99, 69, 9, 11, 32, 6, 18, 82, 55, 84, 4, + 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 131, 245, 29, + 68, 5, 133, 157, 29, 3, 32, 83, 65, 30, 86, 79, 164, 196, 22, 5, 32, 79, + 70, 32, 89, 245, 204, 12, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, + 21, 2, 84, 32, 5, 199, 147, 34, 45, 22, 44, 2, 66, 79, 246, 1, 83, 159, + 162, 36, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, + 73, 84, 72, 32, 14, 82, 66, 86, 83, 140, 167, 12, 4, 76, 73, 71, 72, 210, + 145, 10, 67, 195, 233, 13, 88, 6, 52, 4, 79, 76, 68, 32, 141, 210, 35, 3, + 65, 76, 76, 4, 26, 83, 223, 184, 22, 67, 2, 137, 167, 12, 4, 67, 82, 73, + 80, 162, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, + 91, 3, 78, 74, 65, 186, 234, 27, 83, 146, 142, 5, 70, 83, 81, 8, 26, 77, + 255, 208, 35, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, + 32, 219, 157, 36, 77, 4, 168, 137, 21, 6, 84, 85, 75, 87, 69, 78, 241, + 165, 3, 4, 75, 79, 81, 78, 146, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, + 77, 102, 78, 234, 1, 80, 206, 103, 82, 102, 83, 134, 1, 84, 54, 89, 202, + 193, 2, 87, 166, 215, 32, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 226, + 79, 65, 222, 183, 35, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, + 174, 176, 34, 89, 254, 233, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 254, 144, + 35, 85, 142, 140, 1, 78, 3, 84, 7, 218, 173, 34, 86, 161, 222, 1, 2, 71, + 72, 10, 194, 102, 69, 190, 162, 25, 79, 182, 147, 10, 65, 2, 73, 3, 85, + 19, 62, 69, 166, 101, 66, 146, 182, 35, 65, 2, 73, 2, 79, 3, 85, 4, 254, + 174, 34, 69, 183, 236, 1, 78, 30, 102, 71, 50, 74, 50, 84, 150, 101, 85, + 250, 188, 33, 83, 246, 7, 68, 242, 219, 1, 89, 218, 19, 65, 3, 73, 6, + 178, 223, 31, 75, 190, 184, 4, 71, 187, 2, 65, 6, 222, 236, 25, 85, 198, + 150, 10, 69, 171, 4, 65, 4, 222, 244, 35, 85, 151, 14, 69, 254, 8, 76, 5, + 72, 65, 83, 69, 45, 210, 100, 69, 138, 2, 85, 254, 177, 35, 65, 3, 73, + 242, 8, 116, 2, 65, 32, 160, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, + 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 174, 1, 158, 1, 71, + 106, 75, 130, 1, 76, 102, 77, 190, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, + 84, 154, 1, 85, 144, 253, 28, 2, 70, 73, 182, 156, 5, 86, 211, 219, 1, + 82, 6, 60, 5, 72, 69, 85, 65, 69, 173, 46, 5, 66, 73, 69, 69, 32, 4, 168, + 27, 2, 71, 72, 179, 205, 25, 82, 12, 38, 65, 34, 69, 142, 67, 80, 3, 85, + 4, 234, 146, 36, 70, 187, 2, 81, 4, 132, 192, 26, 5, 85, 75, 69, 85, 84, + 255, 212, 9, 84, 10, 78, 85, 222, 65, 79, 224, 190, 25, 2, 65, 80, 129, + 154, 9, 4, 69, 84, 32, 75, 5, 243, 185, 26, 65, 34, 142, 1, 65, 168, 1, + 2, 66, 65, 38, 79, 76, 6, 86, 69, 85, 65, 69, 78, 244, 70, 8, 69, 85, 78, + 74, 79, 77, 78, 68, 157, 152, 33, 2, 71, 66, 18, 62, 69, 164, 49, 2, 78, + 83, 149, 253, 30, 4, 80, 32, 80, 73, 14, 58, 77, 242, 254, 11, 75, 246, + 134, 7, 78, 163, 248, 16, 83, 9, 178, 24, 66, 206, 44, 86, 183, 248, 26, + 75, 4, 182, 133, 19, 78, 251, 139, 17, 81, 6, 36, 2, 79, 77, 237, 1, 2, + 78, 32, 4, 250, 207, 12, 80, 171, 199, 22, 69, 2, 131, 215, 34, 71, 48, + 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, 84, 102, 89, + 74, 90, 210, 210, 34, 74, 234, 105, 69, 239, 70, 73, 4, 250, 8, 65, 183, + 134, 36, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, 71, 8, 52, 3, + 75, 85, 69, 222, 202, 31, 65, 215, 242, 2, 71, 4, 250, 7, 32, 181, 239, + 11, 2, 78, 90, 4, 246, 25, 65, 147, 253, 18, 73, 4, 166, 161, 34, 73, + 183, 236, 1, 65, 8, 40, 2, 65, 80, 229, 184, 27, 2, 79, 81, 7, 11, 32, 4, + 180, 157, 34, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 143, 240, 35, 69, 5, + 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 245, 243, 11, 4, + 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 166, 238, 35, 69, 163, 28, + 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 218, 221, 25, 82, + 155, 173, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, 32, 78, 74, 73, + 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, 231, 246, 35, 66, + 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 158, 160, 35, 85, 167, 34, + 73, 10, 74, 73, 70, 85, 129, 224, 34, 10, 79, 81, 32, 78, 83, 72, 85, 84, + 32, 89, 4, 158, 219, 25, 82, 233, 194, 6, 8, 78, 68, 65, 32, 80, 65, 32, + 78, 4, 188, 11, 4, 69, 78, 83, 72, 183, 252, 35, 77, 6, 132, 206, 34, 2, + 78, 74, 190, 185, 1, 81, 3, 84, 10, 44, 2, 69, 85, 44, 3, 73, 84, 65, 31, + 85, 4, 234, 181, 34, 65, 1, 4, 84, 69, 85, 87, 2, 11, 32, 2, 223, 30, 77, + 4, 186, 26, 32, 215, 145, 26, 65, 4, 228, 3, 5, 32, 89, 85, 81, 32, 145, + 130, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, 69, 85, 71, 72, 69, 34, + 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, 83, 178, 1, 84, 106, + 89, 190, 22, 87, 204, 46, 2, 70, 69, 147, 157, 11, 86, 4, 198, 56, 85, + 203, 203, 35, 78, 12, 40, 2, 69, 85, 50, 73, 191, 242, 35, 65, 6, 230, + 54, 89, 246, 139, 12, 80, 191, 174, 23, 65, 4, 230, 240, 35, 69, 175, 18, + 81, 8, 46, 65, 240, 69, 2, 79, 77, 171, 160, 35, 69, 4, 50, 65, 129, 61, + 7, 77, 32, 78, 83, 72, 85, 84, 2, 247, 212, 25, 78, 26, 70, 65, 74, 66, + 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, 84, 7, 21, 3, 32, 78, 74, + 4, 194, 134, 18, 85, 137, 192, 16, 3, 69, 85, 65, 10, 90, 65, 154, 46, + 85, 144, 201, 28, 2, 69, 85, 173, 182, 6, 7, 73, 84, 32, 77, 66, 65, 65, + 4, 150, 12, 65, 245, 242, 18, 4, 32, 77, 65, 69, 4, 252, 155, 31, 5, 84, + 32, 78, 71, 71, 227, 227, 4, 81, 4, 48, 4, 79, 78, 32, 84, 181, 235, 25, + 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, 83, 50, 89, 152, 23, 8, + 84, 73, 69, 69, 32, 83, 72, 69, 193, 245, 33, 5, 68, 85, 32, 78, 74, 12, + 70, 71, 168, 132, 35, 8, 75, 73, 78, 68, 73, 32, 77, 86, 175, 104, 79, 8, + 60, 3, 85, 79, 81, 216, 133, 19, 2, 69, 85, 219, 189, 15, 65, 5, 137, + 208, 27, 2, 32, 76, 4, 26, 72, 227, 172, 35, 69, 2, 167, 199, 35, 85, 4, + 248, 45, 2, 65, 69, 211, 17, 73, 8, 152, 155, 10, 2, 69, 69, 210, 130, + 12, 65, 160, 146, 8, 3, 85, 78, 71, 199, 175, 5, 73, 12, 88, 2, 65, 75, + 16, 2, 72, 69, 160, 236, 18, 2, 69, 84, 230, 161, 15, 73, 227, 213, 1, + 85, 2, 211, 25, 69, 4, 244, 230, 25, 4, 84, 32, 78, 74, 129, 239, 4, 4, + 85, 65, 69, 81, 6, 32, 2, 85, 32, 151, 192, 34, 65, 4, 36, 4, 77, 65, 69, + 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, 65, 70, 85, 32, 229, + 4, 2, 69, 85, 2, 221, 179, 31, 6, 76, 69, 69, 82, 65, 69, 196, 1, 170, 1, + 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, 6, 80, 78, 83, 118, 84, + 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 250, 179, 29, 66, 166, 208, 2, 70, + 247, 187, 3, 82, 6, 40, 2, 72, 65, 221, 234, 18, 2, 66, 65, 4, 186, 201, + 25, 82, 155, 173, 10, 80, 22, 66, 69, 182, 1, 85, 248, 224, 25, 3, 80, + 65, 82, 139, 145, 10, 65, 14, 40, 2, 78, 32, 54, 85, 143, 245, 35, 84, 4, + 212, 148, 32, 4, 70, 65, 84, 73, 207, 226, 2, 76, 8, 42, 83, 162, 225, + 25, 75, 195, 147, 10, 77, 4, 232, 14, 3, 72, 69, 85, 239, 53, 69, 4, 48, + 6, 79, 80, 32, 78, 75, 65, 135, 244, 35, 84, 2, 11, 65, 2, 223, 198, 25, + 82, 8, 202, 47, 65, 230, 176, 25, 73, 183, 147, 10, 85, 38, 82, 65, 130, + 1, 66, 202, 196, 25, 85, 216, 25, 4, 71, 66, 65, 83, 143, 167, 8, 73, 8, + 18, 32, 79, 69, 4, 42, 78, 209, 159, 17, 4, 75, 69, 85, 65, 2, 11, 83, 2, + 223, 133, 34, 73, 4, 214, 196, 35, 77, 211, 25, 83, 24, 34, 65, 114, 69, + 82, 73, 35, 85, 6, 32, 2, 65, 32, 163, 229, 18, 78, 4, 204, 219, 30, 8, + 67, 65, 66, 66, 65, 71, 69, 45, 129, 195, 4, 2, 80, 73, 8, 50, 85, 130, + 195, 25, 82, 185, 201, 5, 2, 69, 75, 4, 150, 240, 35, 77, 3, 88, 7, 226, + 7, 82, 155, 232, 35, 84, 4, 174, 221, 35, 65, 175, 18, 69, 72, 130, 1, + 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, 102, 90, 237, 7, + 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, 244, 218, 25, 4, + 78, 83, 65, 78, 195, 147, 10, 81, 12, 60, 2, 69, 85, 174, 41, 65, 146, + 205, 18, 79, 135, 210, 16, 73, 4, 128, 129, 34, 2, 65, 69, 195, 236, 1, + 84, 20, 50, 71, 98, 75, 210, 216, 25, 65, 223, 130, 10, 79, 12, 26, 85, + 247, 156, 35, 69, 11, 180, 39, 3, 65, 69, 78, 190, 245, 34, 79, 202, 26, + 69, 143, 53, 77, 4, 36, 3, 85, 69, 32, 171, 216, 25, 65, 2, 145, 230, 18, + 3, 77, 65, 69, 10, 34, 65, 34, 69, 255, 169, 12, 85, 4, 190, 218, 35, 69, + 219, 16, 77, 4, 194, 254, 33, 69, 171, 96, 85, 12, 78, 85, 202, 214, 25, + 72, 216, 216, 5, 2, 69, 85, 146, 170, 4, 79, 219, 16, 65, 4, 212, 214, + 35, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 232, 204, 35, 4, 85, 32, + 77, 66, 131, 26, 65, 4, 204, 225, 18, 2, 85, 78, 211, 135, 17, 78, 4, + 254, 192, 34, 69, 247, 167, 1, 65, 6, 26, 73, 227, 152, 35, 69, 4, 26, + 82, 155, 232, 35, 78, 2, 151, 151, 34, 73, 10, 30, 69, 50, 72, 255, 6, + 85, 4, 26, 84, 191, 219, 34, 85, 2, 231, 151, 35, 70, 4, 182, 166, 12, + 85, 203, 173, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, 2, 11, + 83, 2, 191, 185, 25, 72, 4, 228, 25, 2, 85, 84, 223, 204, 35, 84, 4, 38, + 85, 153, 133, 32, 3, 65, 32, 89, 2, 235, 139, 26, 65, 4, 180, 145, 27, 2, + 65, 69, 171, 212, 8, 88, 4, 40, 4, 65, 78, 71, 75, 255, 228, 35, 85, 2, + 143, 19, 85, 10, 42, 85, 230, 163, 12, 69, 179, 190, 23, 65, 6, 210, 18, + 87, 212, 3, 4, 32, 77, 85, 79, 167, 206, 35, 77, 234, 1, 134, 1, 70, 68, + 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, 80, 122, + 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, 70, 167, + 224, 35, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 191, 222, 35, 65, + 22, 46, 69, 146, 1, 85, 42, 87, 135, 244, 33, 89, 10, 26, 85, 215, 225, + 35, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 246, 224, 35, + 77, 3, 80, 2, 251, 205, 11, 77, 2, 219, 179, 25, 85, 9, 134, 208, 35, 79, + 218, 16, 78, 3, 81, 2, 171, 171, 35, 65, 14, 58, 69, 182, 204, 25, 79, + 130, 167, 8, 73, 223, 219, 1, 85, 8, 42, 85, 138, 243, 33, 69, 183, 236, + 1, 84, 4, 178, 133, 26, 65, 139, 218, 9, 77, 41, 94, 65, 58, 66, 66, 69, + 38, 70, 80, 2, 71, 66, 214, 140, 31, 79, 246, 214, 3, 86, 135, 121, 85, + 4, 228, 139, 17, 2, 76, 69, 197, 157, 18, 3, 69, 78, 74, 6, 32, 2, 65, + 65, 235, 190, 35, 85, 5, 237, 130, 28, 2, 32, 83, 6, 250, 226, 17, 85, + 147, 142, 16, 69, 10, 44, 2, 69, 85, 238, 228, 33, 79, 207, 11, 73, 4, + 150, 198, 35, 65, 215, 22, 84, 6, 150, 240, 33, 73, 192, 68, 2, 79, 70, + 223, 39, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, 89, + 210, 215, 35, 73, 3, 85, 8, 194, 39, 69, 178, 228, 34, 79, 255, 62, 65, + 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 245, 30, 87, 135, 170, + 4, 85, 6, 152, 38, 2, 65, 77, 183, 180, 35, 80, 12, 68, 2, 69, 85, 174, + 237, 33, 73, 2, 89, 226, 156, 1, 85, 203, 79, 65, 4, 226, 152, 12, 65, + 167, 173, 23, 82, 12, 60, 2, 69, 85, 230, 15, 73, 158, 136, 12, 85, 243, + 192, 23, 65, 4, 206, 198, 35, 65, 175, 18, 84, 10, 46, 72, 32, 3, 73, 69, + 69, 183, 199, 35, 85, 4, 254, 187, 35, 85, 219, 5, 69, 4, 138, 216, 35, + 80, 3, 84, 6, 130, 14, 69, 135, 165, 35, 85, 8, 162, 187, 35, 69, 218, 5, + 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 182, 214, 35, 65, 3, 73, + 4, 208, 169, 25, 2, 85, 84, 155, 173, 10, 69, 4, 146, 186, 35, 85, 175, + 28, 81, 8, 48, 3, 69, 78, 32, 150, 194, 35, 73, 175, 1, 65, 4, 214, 240, + 25, 79, 227, 194, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, 174, 6, 69, + 167, 161, 35, 79, 12, 54, 69, 162, 193, 25, 79, 222, 255, 9, 73, 219, 19, + 85, 6, 214, 7, 85, 255, 204, 35, 69, 6, 222, 189, 35, 65, 214, 22, 69, 3, + 85, 18, 62, 69, 74, 85, 210, 191, 25, 79, 226, 252, 9, 65, 215, 22, 73, + 8, 26, 85, 255, 230, 33, 69, 6, 186, 130, 34, 65, 246, 208, 1, 78, 3, 84, + 5, 215, 182, 35, 79, 4, 134, 152, 31, 85, 223, 186, 4, 65, 10, 40, 2, 65, + 69, 18, 85, 191, 130, 35, 69, 2, 251, 3, 77, 6, 22, 87, 227, 13, 79, 2, + 195, 190, 25, 79, 186, 2, 178, 1, 70, 154, 1, 71, 202, 1, 75, 170, 1, 76, + 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, 78, 83, 166, 1, 84, 246, + 1, 86, 66, 87, 34, 89, 226, 186, 35, 65, 2, 73, 3, 79, 18, 54, 85, 218, + 172, 22, 65, 202, 140, 13, 69, 255, 5, 79, 10, 26, 32, 231, 154, 30, 69, + 6, 148, 143, 23, 4, 82, 69, 77, 69, 134, 188, 10, 67, 191, 132, 2, 73, + 16, 24, 2, 66, 69, 39, 72, 4, 234, 194, 34, 85, 143, 140, 1, 84, 12, 34, + 65, 34, 69, 187, 189, 35, 79, 2, 11, 65, 2, 139, 161, 25, 77, 8, 26, 85, + 247, 205, 35, 84, 6, 158, 183, 35, 65, 214, 22, 78, 3, 88, 18, 50, 69, + 62, 80, 18, 85, 206, 204, 35, 73, 3, 79, 6, 26, 85, 255, 204, 35, 84, 4, + 166, 182, 35, 65, 215, 22, 88, 2, 211, 28, 69, 6, 158, 176, 35, 69, 162, + 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, 79, 183, 203, 35, 85, 6, + 150, 187, 35, 65, 218, 16, 80, 3, 81, 2, 155, 185, 35, 65, 8, 238, 208, + 17, 79, 198, 250, 17, 77, 3, 81, 32, 110, 65, 44, 2, 66, 69, 34, 70, 20, + 2, 71, 66, 34, 73, 130, 156, 25, 85, 198, 221, 9, 69, 2, 79, 255, 59, 86, + 11, 234, 164, 17, 69, 174, 165, 18, 80, 3, 81, 4, 146, 185, 35, 85, 219, + 16, 69, 2, 227, 136, 12, 69, 4, 226, 249, 34, 69, 215, 79, 65, 5, 195, + 178, 35, 69, 80, 114, 68, 170, 1, 71, 138, 3, 74, 112, 2, 83, 72, 58, 84, + 32, 3, 89, 73, 32, 54, 90, 210, 129, 35, 65, 179, 47, 75, 12, 34, 65, 98, + 73, 187, 247, 34, 85, 6, 32, 2, 65, 32, 203, 199, 35, 80, 4, 128, 215, + 17, 3, 77, 89, 32, 249, 140, 13, 3, 83, 79, 70, 4, 214, 179, 25, 65, 183, + 147, 10, 81, 36, 78, 71, 50, 85, 122, 75, 118, 79, 188, 177, 11, 3, 69, + 85, 82, 195, 146, 24, 65, 12, 18, 69, 51, 85, 6, 26, 85, 183, 217, 33, + 69, 4, 199, 187, 23, 65, 6, 64, 6, 65, 69, 83, 72, 65, 69, 250, 151, 25, + 82, 155, 173, 10, 80, 2, 11, 32, 2, 203, 231, 22, 78, 12, 34, 65, 20, 2, + 69, 85, 35, 85, 5, 231, 176, 35, 65, 4, 138, 178, 35, 65, 175, 18, 88, 4, + 150, 196, 35, 77, 3, 80, 4, 250, 195, 35, 80, 3, 81, 8, 18, 65, 31, 69, + 2, 201, 244, 30, 2, 69, 77, 6, 26, 69, 139, 183, 34, 85, 5, 233, 160, 35, + 4, 32, 69, 80, 79, 6, 26, 85, 163, 214, 33, 73, 4, 198, 194, 35, 79, 15, + 69, 4, 222, 177, 35, 85, 207, 16, 65, 4, 128, 190, 25, 4, 67, 76, 69, 65, + 247, 240, 1, 66, 4, 174, 174, 25, 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, + 50, 79, 23, 85, 8, 32, 2, 32, 80, 207, 155, 17, 65, 4, 152, 176, 27, 2, + 69, 79, 169, 135, 7, 2, 76, 85, 6, 26, 85, 211, 175, 35, 69, 4, 166, 192, + 35, 84, 3, 88, 7, 11, 69, 4, 202, 172, 25, 69, 183, 147, 10, 84, 5, 135, + 240, 34, 79, 11, 82, 65, 246, 190, 35, 69, 3, 77, 8, 46, 65, 238, 14, 69, + 185, 168, 18, 2, 73, 77, 4, 242, 190, 35, 69, 3, 81, 18, 62, 69, 30, 72, + 150, 131, 31, 85, 146, 170, 4, 79, 163, 14, 65, 4, 150, 190, 35, 69, 3, + 84, 8, 42, 69, 182, 154, 22, 79, 235, 143, 3, 73, 2, 209, 252, 11, 2, 85, + 65, 28, 34, 65, 94, 69, 50, 79, 39, 85, 10, 56, 2, 69, 78, 186, 153, 22, + 65, 158, 163, 13, 77, 3, 81, 2, 221, 165, 11, 3, 32, 78, 84, 6, 26, 85, + 155, 188, 35, 78, 5, 155, 251, 11, 65, 6, 130, 196, 33, 79, 131, 248, 1, + 81, 6, 170, 7, 77, 227, 157, 35, 65, 6, 26, 69, 207, 170, 35, 79, 4, 138, + 142, 25, 85, 155, 173, 10, 69, 6, 246, 10, 69, 255, 203, 31, 85, 26, 68, + 2, 69, 85, 46, 73, 32, 3, 79, 81, 32, 54, 85, 143, 185, 35, 65, 8, 246, + 175, 23, 65, 162, 138, 12, 77, 3, 88, 4, 150, 163, 35, 69, 215, 22, 84, + 4, 248, 150, 12, 4, 83, 87, 73, 77, 223, 162, 9, 67, 8, 226, 165, 25, 69, + 194, 194, 8, 65, 246, 208, 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, + 98, 78, 190, 1, 80, 66, 82, 50, 83, 110, 84, 38, 89, 170, 196, 2, 87, + 252, 172, 9, 2, 86, 85, 170, 170, 23, 69, 242, 5, 70, 231, 16, 85, 14, + 242, 188, 17, 69, 146, 142, 16, 89, 254, 233, 1, 80, 186, 2, 65, 2, 79, + 3, 85, 6, 178, 163, 25, 79, 182, 147, 10, 65, 3, 73, 13, 42, 66, 34, 69, + 242, 181, 35, 65, 3, 79, 4, 186, 230, 34, 69, 159, 77, 65, 2, 187, 201, + 33, 69, 22, 94, 71, 38, 74, 38, 85, 250, 188, 33, 83, 246, 7, 68, 150, 3, + 84, 222, 216, 1, 89, 219, 19, 73, 4, 134, 250, 30, 75, 191, 184, 4, 71, + 4, 190, 135, 25, 85, 239, 154, 10, 65, 5, 223, 157, 35, 65, 6, 26, 69, + 239, 134, 25, 85, 4, 246, 167, 34, 85, 143, 140, 1, 69, 12, 186, 2, 69, + 154, 231, 8, 73, 143, 202, 26, 85, 14, 66, 72, 230, 2, 69, 194, 170, 18, + 65, 226, 224, 16, 85, 235, 36, 73, 6, 146, 159, 35, 73, 218, 19, 79, 3, + 85, 6, 186, 190, 20, 65, 223, 215, 14, 69, 4, 234, 158, 25, 79, 183, 147, + 10, 65, 4, 166, 155, 35, 65, 215, 22, 69, 14, 54, 69, 154, 231, 8, 73, + 186, 179, 26, 65, 215, 22, 85, 6, 150, 165, 34, 85, 142, 140, 1, 69, 3, + 78, 16, 62, 72, 50, 69, 194, 170, 18, 65, 226, 224, 16, 85, 235, 36, 73, + 8, 46, 69, 178, 156, 35, 73, 218, 19, 79, 3, 85, 2, 251, 163, 34, 85, 10, + 174, 181, 17, 69, 190, 134, 3, 65, 139, 244, 14, 73, 6, 138, 156, 25, 79, + 2, 85, 183, 147, 10, 65, 14, 42, 75, 166, 244, 33, 65, 211, 155, 1, 74, + 11, 49, 10, 78, 79, 84, 69, 32, 87, 73, 84, 72, 32, 8, 224, 173, 9, 2, + 80, 79, 190, 1, 89, 188, 131, 12, 2, 69, 85, 147, 156, 6, 68, 6, 38, 32, + 237, 172, 16, 3, 66, 69, 82, 4, 170, 226, 28, 67, 189, 186, 5, 5, 79, 70, + 32, 83, 79, 78, 72, 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, + 183, 167, 33, 69, 5, 241, 218, 15, 10, 66, 65, 76, 76, 32, 65, 78, 68, + 32, 72, 72, 104, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 235, 222, 32, 70, 10, 52, 4, 72, 73, 71, 72, + 44, 3, 76, 79, 87, 39, 77, 4, 172, 140, 21, 2, 45, 76, 251, 207, 10, 32, + 4, 32, 2, 45, 77, 219, 219, 31, 32, 2, 201, 219, 31, 2, 73, 68, 60, 238, + 1, 68, 38, 69, 38, 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 128, 238, + 27, 2, 72, 87, 154, 196, 2, 77, 178, 181, 2, 79, 154, 154, 2, 86, 246, 5, + 74, 2, 84, 2, 90, 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, + 65, 3, 73, 4, 222, 235, 32, 72, 159, 188, 2, 79, 7, 222, 215, 30, 78, + 251, 207, 4, 69, 4, 202, 130, 35, 66, 219, 35, 65, 4, 202, 238, 27, 80, + 203, 184, 7, 65, 5, 135, 130, 35, 87, 5, 215, 129, 35, 68, 4, 138, 167, + 34, 69, 235, 104, 73, 121, 48, 3, 65, 75, 32, 230, 10, 72, 179, 218, 21, + 84, 112, 196, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, + 78, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, + 141, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 214, 163, 35, + 78, 87, 72, 76, 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 232, 144, 18, + 4, 75, 65, 82, 79, 214, 141, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, + 2, 76, 2, 82, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 251, + 159, 35, 66, 9, 45, 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 198, 159, + 35, 72, 2, 78, 3, 83, 10, 152, 2, 2, 79, 82, 138, 157, 35, 68, 2, 71, 2, + 89, 187, 2, 65, 5, 189, 165, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, + 65, 76, 85, 78, 71, 85, 78, 32, 96, 2, 79, 85, 195, 159, 35, 65, 20, 230, + 157, 35, 71, 2, 72, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, + 2, 65, 2, 145, 145, 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 161, 145, + 18, 4, 79, 78, 71, 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, + 68, 85, 32, 193, 140, 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 152, 243, + 10, 3, 74, 85, 68, 249, 166, 24, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, + 78, 71, 133, 199, 22, 7, 73, 78, 65, 82, 66, 79, 82, 2, 139, 131, 25, 79, + 18, 122, 85, 228, 173, 25, 6, 80, 65, 75, 80, 65, 75, 180, 248, 2, 5, 75, + 65, 82, 79, 32, 182, 181, 6, 69, 150, 64, 73, 3, 79, 5, 177, 200, 29, 15, + 32, 70, 79, 82, 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 143, 134, + 23, 84, 254, 1, 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 198, 10, 84, + 144, 220, 31, 2, 67, 65, 204, 162, 2, 5, 86, 69, 82, 65, 71, 191, 140, 1, + 68, 20, 136, 1, 4, 77, 69, 68, 32, 170, 1, 82, 164, 143, 3, 10, 67, 72, + 32, 87, 73, 84, 72, 32, 85, 77, 210, 144, 7, 84, 234, 185, 24, 86, 19, + 78, 8, 86, 65, 0, 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, + 69, 69, 78, 2, 201, 214, 19, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 193, + 214, 19, 2, 84, 72, 4, 128, 196, 28, 3, 68, 69, 68, 179, 150, 6, 32, 4, + 176, 166, 8, 3, 82, 32, 77, 183, 207, 25, 84, 13, 11, 76, 11, 38, 32, + 217, 187, 24, 3, 72, 79, 80, 6, 162, 140, 12, 80, 212, 232, 15, 6, 87, + 73, 84, 72, 32, 67, 187, 142, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, + 162, 9, 84, 37, 9, 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, + 65, 44, 9, 67, 85, 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, + 69, 82, 32, 204, 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 242, 128, 22, 73, + 130, 4, 86, 242, 153, 11, 68, 141, 105, 3, 71, 65, 78, 6, 190, 147, 30, + 85, 138, 149, 1, 66, 51, 78, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, + 82, 32, 221, 212, 22, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, + 83, 73, 10, 52, 3, 79, 78, 69, 214, 170, 30, 70, 155, 250, 2, 84, 5, 165, + 201, 28, 19, 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, + 68, 69, 78, 79, 108, 226, 1, 75, 90, 82, 226, 232, 20, 86, 234, 239, 7, + 89, 150, 205, 2, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, + 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 254, 68, 72, 2, + 76, 2, 77, 186, 2, 69, 3, 79, 8, 26, 72, 235, 142, 35, 65, 6, 26, 65, + 155, 208, 13, 73, 5, 157, 128, 18, 3, 78, 68, 65, 10, 34, 65, 210, 139, + 35, 72, 3, 82, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 244, 179, 24, 5, 76, + 79, 87, 69, 82, 1, 6, 77, 73, 68, 68, 76, 69, 4, 202, 134, 34, 83, 179, + 69, 77, 20, 116, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, + 69, 84, 84, 69, 82, 32, 154, 154, 25, 65, 187, 138, 6, 73, 6, 150, 131, + 22, 82, 147, 134, 13, 89, 4, 182, 173, 25, 32, 251, 209, 8, 79, 5, 137, + 247, 31, 4, 32, 87, 73, 84, 4, 150, 186, 33, 87, 255, 61, 32, 194, 1, + 180, 2, 7, 76, 69, 84, 84, 69, 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, + 32, 72, 5, 83, 73, 71, 78, 32, 44, 11, 86, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 250, 143, 25, 68, 130, 229, 3, 87, 136, 97, 10, 71, 65, 80, + 32, 70, 73, 76, 76, 69, 82, 153, 229, 4, 12, 72, 85, 78, 68, 82, 69, 68, + 83, 32, 85, 78, 73, 92, 210, 1, 86, 154, 158, 31, 65, 38, 68, 46, 84, + 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, + 69, 3, 79, 8, 234, 1, 79, 255, 132, 35, 65, 36, 138, 100, 69, 38, 70, 66, + 78, 26, 83, 250, 241, 20, 84, 183, 150, 12, 79, 10, 142, 159, 31, 65, 74, + 67, 171, 163, 3, 86, 24, 84, 2, 86, 79, 166, 160, 31, 65, 190, 21, 85, + 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, + 32, 6, 178, 161, 31, 82, 167, 227, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, + 36, 2, 71, 32, 28, 2, 76, 76, 46, 82, 66, 84, 212, 216, 18, 4, 79, 72, + 65, 90, 164, 217, 11, 2, 75, 73, 155, 128, 4, 83, 4, 210, 187, 31, 73, + 203, 199, 3, 69, 4, 186, 240, 32, 82, 67, 83, 4, 164, 175, 9, 2, 69, 68, + 219, 251, 22, 73, 4, 176, 188, 31, 7, 84, 72, 68, 65, 89, 32, 67, 255, + 197, 3, 68, 4, 160, 165, 13, 2, 67, 79, 149, 219, 21, 4, 73, 78, 71, 32, + 230, 3, 42, 65, 222, 29, 79, 249, 3, 2, 85, 69, 228, 2, 32, 2, 67, 75, + 135, 159, 17, 78, 226, 2, 22, 32, 143, 28, 45, 210, 2, 214, 1, 67, 254, + 4, 68, 174, 2, 70, 102, 72, 82, 76, 170, 2, 77, 250, 2, 80, 46, 82, 202, + 3, 83, 210, 3, 84, 82, 85, 248, 2, 3, 86, 69, 82, 198, 215, 11, 79, 208, + 240, 19, 3, 66, 79, 87, 154, 227, 1, 78, 179, 1, 81, 98, 196, 1, 5, 73, + 82, 67, 76, 69, 200, 1, 6, 85, 82, 86, 69, 68, 32, 192, 150, 25, 12, 82, + 79, 83, 83, 32, 79, 78, 32, 83, 72, 73, 69, 240, 155, 3, 5, 69, 78, 84, + 82, 69, 178, 238, 4, 72, 203, 13, 76, 11, 11, 32, 8, 72, 5, 87, 73, 84, + 72, 32, 145, 164, 16, 7, 70, 79, 82, 32, 82, 69, 67, 6, 224, 51, 8, 87, + 72, 73, 84, 69, 32, 68, 79, 238, 181, 18, 68, 221, 144, 2, 8, 84, 87, 79, + 32, 87, 72, 73, 84, 16, 84, 4, 68, 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, + 69, 70, 1, 4, 82, 73, 71, 72, 4, 141, 139, 31, 9, 87, 65, 82, 68, 83, 32, + 65, 78, 68, 4, 53, 11, 84, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, + 178, 164, 10, 85, 179, 237, 21, 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, + 152, 1, 3, 79, 87, 78, 43, 82, 13, 11, 32, 10, 210, 15, 67, 152, 208, 12, + 10, 77, 73, 78, 85, 83, 32, 87, 72, 73, 84, 184, 134, 6, 6, 87, 73, 84, + 72, 32, 68, 242, 212, 14, 79, 131, 11, 83, 12, 142, 16, 32, 62, 45, 175, + 185, 30, 87, 6, 196, 155, 33, 2, 79, 80, 175, 20, 65, 8, 18, 76, 39, 79, + 4, 150, 134, 27, 79, 191, 240, 7, 65, 4, 172, 192, 2, 2, 85, 82, 231, + 146, 30, 76, 12, 38, 69, 146, 179, 33, 65, 239, 2, 79, 6, 216, 180, 33, + 2, 65, 82, 215, 10, 88, 32, 60, 3, 69, 70, 84, 202, 1, 79, 161, 228, 32, + 3, 65, 82, 71, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 64, 6, + 87, 65, 82, 68, 83, 32, 155, 183, 33, 32, 12, 162, 6, 68, 246, 7, 73, + 174, 170, 33, 80, 166, 24, 83, 51, 84, 4, 226, 142, 32, 69, 143, 137, 1, + 66, 6, 230, 13, 87, 143, 178, 33, 90, 30, 76, 6, 69, 68, 73, 85, 77, 32, + 197, 145, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, 36, 4, + 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 219, 182, + 33, 73, 6, 32, 2, 69, 70, 235, 189, 33, 79, 4, 11, 84, 4, 81, 18, 45, 80, + 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 217, + 7, 2, 32, 67, 8, 130, 12, 77, 207, 196, 33, 81, 8, 142, 169, 24, 85, 222, + 141, 9, 65, 87, 69, 32, 52, 4, 73, 71, 72, 84, 178, 228, 32, 79, 251, + 105, 69, 28, 100, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 244, 1, 6, + 87, 65, 82, 68, 83, 32, 159, 199, 33, 32, 16, 90, 68, 88, 8, 84, 82, 73, + 65, 78, 71, 76, 69, 158, 7, 73, 162, 175, 33, 80, 179, 19, 83, 4, 65, 14, + 79, 85, 66, 76, 69, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 247, 146, 12, + 32, 5, 197, 153, 20, 11, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, + 226, 211, 18, 65, 134, 180, 13, 69, 143, 137, 1, 66, 38, 150, 2, 77, 84, + 5, 81, 85, 65, 82, 69, 172, 156, 25, 3, 85, 78, 32, 196, 223, 1, 5, 75, + 85, 76, 76, 32, 208, 164, 3, 3, 78, 79, 87, 208, 195, 1, 5, 65, 70, 69, + 84, 89, 164, 118, 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, + 146, 90, 67, 50, 72, 226, 1, 80, 143, 17, 84, 10, 40, 4, 65, 76, 76, 32, + 239, 182, 33, 73, 8, 134, 175, 33, 68, 134, 7, 76, 71, 83, 9, 11, 32, 6, + 54, 67, 228, 158, 32, 3, 70, 79, 82, 243, 152, 1, 66, 2, 245, 137, 30, 3, + 69, 78, 84, 14, 180, 4, 3, 73, 78, 89, 134, 221, 22, 82, 226, 221, 10, + 79, 54, 69, 135, 2, 87, 18, 38, 80, 133, 208, 31, 3, 78, 73, 86, 16, 46, + 32, 62, 45, 170, 1, 80, 135, 184, 30, 87, 2, 185, 194, 33, 10, 80, 79, + 73, 78, 84, 73, 78, 71, 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 8, 66, 73, 236, 190, 33, 5, 68, 79, 85, 66, 76, 230, 3, 83, 51, 84, + 2, 129, 199, 32, 8, 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, + 4, 190, 222, 23, 76, 143, 232, 8, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, + 29, 4, 89, 32, 83, 77, 4, 134, 195, 33, 69, 51, 82, 6, 21, 3, 65, 76, 76, + 6, 11, 32, 6, 182, 169, 33, 68, 134, 7, 76, 227, 19, 83, 16, 84, 15, 76, + 69, 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 155, 234, 10, 70, + 10, 218, 227, 34, 67, 2, 72, 2, 73, 2, 82, 3, 90, 124, 84, 11, 67, 75, + 32, 83, 69, 88, 84, 65, 78, 84, 45, 238, 128, 2, 83, 135, 181, 10, 87, + 120, 62, 49, 214, 1, 50, 46, 51, 38, 52, 42, 53, 199, 223, 34, 54, 61, + 50, 50, 118, 51, 126, 52, 42, 53, 199, 223, 34, 54, 31, 46, 51, 194, 1, + 52, 42, 53, 199, 223, 34, 54, 15, 38, 52, 194, 1, 53, 199, 223, 34, 54, + 7, 130, 225, 34, 53, 3, 54, 15, 122, 52, 190, 248, 27, 53, 179, 231, 6, + 54, 31, 42, 51, 66, 52, 14, 53, 199, 223, 34, 54, 17, 34, 52, 42, 53, + 199, 223, 34, 54, 9, 38, 53, 199, 223, 34, 54, 7, 11, 53, 5, 195, 223, + 34, 54, 6, 178, 207, 21, 32, 241, 166, 8, 4, 66, 69, 82, 82, 144, 4, 200, + 1, 3, 76, 68, 32, 78, 79, 104, 7, 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, + 84, 84, 140, 3, 5, 85, 81, 85, 69, 84, 54, 87, 198, 1, 88, 210, 204, 5, + 77, 194, 153, 3, 89, 218, 177, 25, 65, 255, 33, 78, 14, 246, 169, 8, 83, + 210, 164, 7, 69, 134, 252, 16, 70, 234, 2, 87, 207, 10, 71, 10, 26, 75, + 155, 198, 22, 77, 9, 40, 4, 77, 65, 82, 75, 155, 220, 34, 83, 5, 197, + 224, 22, 3, 32, 84, 65, 150, 1, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, + 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, 84, 69, 82, 32, 10, 142, 219, 34, + 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, 234, 1, 65, 54, 85, 22, 69, 82, + 71, 46, 73, 70, 78, 38, 79, 98, 90, 226, 138, 7, 75, 162, 215, 23, 67, 2, + 76, 2, 83, 198, 36, 66, 210, 200, 1, 74, 206, 134, 2, 68, 2, 70, 2, 72, + 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, 3, 88, 21, 50, 73, 2, 85, 74, + 78, 242, 215, 34, 72, 3, 77, 5, 227, 136, 34, 78, 17, 50, 78, 242, 215, + 34, 69, 2, 72, 2, 73, 3, 82, 7, 238, 215, 34, 71, 3, 78, 11, 210, 215, + 34, 72, 2, 78, 2, 85, 3, 87, 15, 164, 223, 32, 2, 78, 78, 130, 248, 1, + 72, 2, 77, 2, 82, 3, 85, 9, 206, 222, 32, 71, 151, 248, 1, 78, 17, 66, + 78, 146, 213, 33, 32, 238, 128, 1, 69, 2, 77, 2, 79, 3, 85, 4, 250, 213, + 34, 71, 3, 78, 9, 222, 213, 34, 72, 2, 73, 3, 89, 34, 104, 3, 79, 77, 32, + 197, 238, 20, 17, 76, 69, 32, 87, 73, 84, 72, 32, 80, 79, 80, 80, 73, 78, + 71, 32, 67, 32, 168, 1, 5, 72, 65, 76, 70, 32, 52, 14, 83, 81, 85, 65, + 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 134, 157, 15, 65, 166, 217, 15, + 67, 134, 2, 80, 126, 76, 22, 82, 175, 1, 84, 8, 226, 247, 30, 66, 46, 76, + 26, 82, 255, 214, 1, 73, 5, 193, 250, 30, 11, 32, 79, 86, 69, 82, 32, 84, + 79, 80, 32, 83, 5, 229, 195, 33, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, + 58, 76, 116, 3, 84, 73, 69, 181, 252, 31, 3, 32, 65, 78, 6, 26, 32, 187, + 128, 34, 73, 4, 160, 158, 9, 7, 79, 70, 32, 72, 89, 71, 73, 197, 161, 22, + 6, 87, 73, 84, 72, 32, 83, 7, 235, 196, 31, 32, 162, 2, 88, 10, 32, 68, + 82, 65, 87, 73, 78, 71, 83, 32, 137, 211, 33, 6, 73, 78, 71, 32, 71, 76, + 160, 2, 176, 1, 2, 68, 79, 148, 4, 6, 72, 69, 65, 86, 89, 32, 194, 2, 76, + 156, 17, 6, 82, 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 58, 48, 5, 85, 66, 76, 69, 32, 89, 3, 87, + 78, 32, 22, 68, 4, 68, 79, 87, 78, 0, 2, 85, 80, 250, 21, 86, 155, 177, + 28, 72, 6, 171, 22, 32, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, + 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 230, 29, 68, + 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 26, 2, 85, + 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 250, 23, 85, 227, 136, 34, 76, 12, + 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 176, 27, 2, 85, 80, 235, 4, 72, + 4, 17, 2, 84, 32, 4, 218, 23, 85, 135, 9, 72, 46, 120, 4, 76, 69, 70, 84, + 68, 2, 85, 80, 176, 14, 2, 68, 79, 106, 81, 34, 84, 234, 1, 86, 154, 177, + 28, 72, 199, 219, 5, 82, 5, 45, 9, 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, + 207, 250, 32, 84, 11, 29, 5, 32, 65, 78, 68, 32, 8, 42, 76, 238, 193, 28, + 72, 199, 219, 5, 82, 4, 204, 223, 23, 4, 73, 71, 72, 84, 211, 189, 10, + 69, 108, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, 32, 16, 244, + 18, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, + 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, + 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, 71, 72, 97, + 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, + 92, 172, 1, 4, 65, 82, 67, 32, 30, 68, 172, 9, 4, 76, 69, 70, 84, 62, 81, + 34, 84, 112, 2, 85, 80, 122, 86, 176, 134, 34, 10, 72, 79, 82, 73, 90, + 79, 78, 84, 65, 76, 175, 6, 82, 8, 158, 143, 17, 68, 67, 85, 48, 52, 8, + 73, 65, 71, 79, 78, 65, 76, 32, 203, 8, 79, 36, 100, 7, 77, 73, 68, 68, + 76, 69, 32, 216, 2, 6, 85, 80, 80, 69, 82, 32, 194, 129, 33, 67, 191, 2, + 68, 12, 88, 8, 76, 69, 70, 84, 32, 84, 79, 32, 161, 1, 9, 82, 73, 71, 72, + 84, 32, 84, 79, 32, 8, 136, 1, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, + 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, + 84, 195, 3, 76, 5, 143, 132, 20, 32, 4, 232, 4, 15, 85, 80, 80, 69, 82, + 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 175, 255, 19, 76, 20, 196, 1, 17, + 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 228, + 239, 22, 7, 76, 69, 70, 84, 32, 84, 79, 193, 155, 11, 14, 82, 73, 71, 72, + 84, 32, 84, 79, 32, 76, 79, 87, 69, 82, 16, 56, 4, 76, 69, 70, 84, 165, + 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 60, 4, 84, 79, 32, 76, 137, 128, + 20, 5, 65, 78, 68, 32, 77, 4, 53, 11, 79, 87, 69, 82, 32, 67, 69, 78, 84, + 82, 69, 5, 169, 198, 31, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, 78, 68, + 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, 69, 2, + 249, 254, 19, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, 165, + 187, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 12, 164, 1, 2, 85, 66, + 241, 1, 2, 87, 78, 5, 221, 235, 32, 10, 32, 65, 78, 68, 32, 72, 69, 65, + 86, 89, 4, 49, 5, 85, 65, 68, 82, 85, 4, 17, 2, 82, 73, 4, 11, 80, 4, 41, + 8, 76, 69, 32, 68, 65, 83, 72, 32, 4, 230, 130, 16, 86, 207, 175, 12, 72, + 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 246, 140, 34, 76, 31, 82, 4, + 172, 207, 23, 4, 69, 65, 86, 89, 171, 226, 4, 79, 8, 37, 7, 69, 82, 84, + 73, 67, 65, 76, 9, 11, 32, 6, 25, 4, 65, 78, 68, 32, 6, 198, 176, 28, 72, + 170, 219, 5, 76, 31, 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, + 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, + 65, 78, 68, 32, 76, 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, + 32, 76, 69, 70, 97, 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, + 32, 76, 69, 70, 2, 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, + 3, 68, 245, 4, 6, 86, 69, 82, 84, 73, 67, 2, 227, 157, 32, 80, 6, 17, 2, + 84, 32, 6, 58, 85, 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, + 8, 80, 2, 185, 2, 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, + 65, 78, 68, 32, 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, + 2, 68, 131, 4, 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, + 82, 73, 71, 72, 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, + 26, 68, 179, 133, 34, 76, 2, 177, 154, 32, 3, 79, 87, 78, 12, 80, 4, 68, + 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, + 4, 2, 32, 72, 4, 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, + 87, 78, 24, 130, 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, + 32, 144, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, + 49, 10, 79, 85, 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, + 4, 82, 73, 71, 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, + 84, 2, 189, 157, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, + 73, 71, 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 189, 149, 32, 2, 65, + 76, 2, 163, 149, 32, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, + 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, + 17, 2, 32, 72, 2, 253, 151, 34, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, + 69, 32, 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 165, + 225, 25, 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 175, 225, 25, 84, 134, + 6, 46, 65, 186, 14, 69, 150, 1, 73, 191, 1, 79, 232, 5, 36, 4, 72, 77, + 73, 32, 255, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, + 2, 7, 78, 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, + 84, 73, 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 126, 86, 215, 169, 24, 68, + 108, 210, 1, 79, 178, 187, 30, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, + 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, + 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, + 45, 9, 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 226, 165, 28, 76, 186, + 153, 2, 83, 190, 66, 78, 199, 221, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, + 83, 250, 241, 20, 84, 174, 213, 5, 79, 155, 156, 7, 74, 4, 193, 175, 30, + 4, 73, 71, 72, 84, 8, 26, 79, 135, 218, 20, 73, 4, 134, 184, 32, 82, 167, + 177, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, + 5, 203, 143, 34, 84, 10, 46, 76, 170, 188, 12, 68, 169, 16, 2, 67, 82, 4, + 178, 138, 19, 79, 139, 253, 13, 73, 12, 204, 155, 16, 9, 79, 76, 68, 32, + 84, 65, 77, 73, 76, 182, 158, 14, 67, 198, 180, 1, 74, 158, 2, 86, 122, + 85, 231, 233, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 247, 236, 31, 73, 32, 142, 1, 79, 140, 213, 28, 11, 66, 72, 65, 84, 84, + 73, 80, 82, 79, 76, 85, 250, 227, 1, 65, 106, 86, 214, 20, 85, 210, 200, + 1, 73, 207, 134, 2, 69, 7, 165, 185, 30, 10, 76, 68, 32, 84, 65, 77, 73, + 76, 32, 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, + 32, 211, 156, 34, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 215, 243, 6, 66, + 254, 3, 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, + 34, 56, 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, + 34, 56, 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, + 65, 50, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, 33, 42, 53, 38, 54, + 30, 55, 167, 153, 34, 56, 17, 34, 54, 30, 55, 167, 153, 34, 56, 9, 26, + 55, 167, 153, 34, 56, 5, 163, 153, 34, 56, 8, 26, 65, 163, 130, 34, 86, + 6, 212, 131, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, + 135, 8, 6, 83, 84, 45, 70, 69, 69, 219, 141, 6, 68, 10, 42, 68, 108, 2, + 69, 70, 207, 147, 34, 67, 4, 84, 6, 71, 69, 32, 65, 84, 32, 133, 234, 31, + 9, 69, 32, 87, 73, 84, 72, 32, 86, 69, 2, 167, 236, 33, 78, 4, 190, 134, + 31, 67, 167, 144, 3, 83, 12, 84, 4, 75, 69, 78, 32, 232, 177, 8, 2, 67, + 67, 220, 220, 24, 2, 87, 78, 207, 118, 79, 6, 140, 227, 26, 17, 67, 73, + 82, 67, 76, 69, 32, 87, 73, 84, 72, 32, 78, 79, 82, 84, 72, 178, 144, 6, + 66, 155, 27, 72, 134, 1, 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, + 72, 73, 68, 32, 36, 2, 76, 76, 122, 83, 56, 4, 84, 84, 69, 82, 220, 230, + 10, 10, 73, 76, 68, 73, 78, 71, 32, 67, 79, 78, 208, 154, 16, 2, 82, 82, + 155, 185, 6, 67, 4, 216, 195, 27, 2, 32, 84, 195, 207, 6, 83, 63, 33, 6, + 73, 78, 69, 83, 69, 32, 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, + 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 252, 15, 69, 237, + 205, 13, 4, 80, 65, 76, 76, 46, 154, 1, 77, 34, 78, 198, 141, 34, 66, 2, + 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, + 2, 86, 2, 89, 187, 2, 65, 4, 226, 141, 34, 80, 187, 2, 65, 12, 46, 71, + 34, 89, 246, 140, 34, 82, 187, 2, 65, 4, 146, 141, 34, 75, 187, 2, 65, 4, + 242, 140, 34, 67, 187, 2, 65, 10, 182, 248, 33, 65, 214, 22, 69, 2, 73, + 2, 79, 3, 85, 40, 134, 134, 10, 76, 183, 169, 18, 86, 10, 56, 2, 69, 84, + 20, 4, 72, 79, 82, 78, 151, 149, 9, 83, 5, 211, 227, 30, 32, 5, 221, 237, + 26, 5, 32, 87, 73, 84, 72, 9, 26, 84, 203, 194, 31, 32, 4, 214, 237, 26, + 83, 15, 32, 5, 207, 233, 33, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, + 78, 69, 32, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, + 173, 254, 19, 5, 84, 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, + 72, 170, 1, 68, 158, 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, + 75, 142, 5, 76, 190, 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, + 5, 84, 196, 5, 2, 86, 65, 250, 1, 89, 238, 162, 31, 78, 149, 136, 2, 9, + 88, 73, 82, 79, 78, 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, + 1, 78, 118, 80, 199, 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, + 5, 80, 79, 76, 73, 32, 4, 17, 2, 82, 71, 4, 200, 181, 15, 2, 79, 84, 211, + 210, 18, 73, 4, 134, 138, 9, 84, 147, 234, 24, 83, 4, 26, 65, 1, 2, 71, + 79, 2, 223, 150, 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 169, 229, 33, + 6, 65, 84, 82, 73, 67, 72, 4, 168, 32, 2, 75, 89, 239, 227, 33, 77, 22, + 52, 5, 69, 83, 79, 32, 69, 38, 79, 247, 241, 33, 76, 4, 204, 1, 2, 88, + 79, 147, 55, 75, 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, + 82, 181, 128, 1, 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, + 78, 68, 69, 83, 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, + 214, 60, 68, 251, 213, 21, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, + 79, 32, 4, 170, 21, 83, 135, 238, 33, 78, 14, 194, 242, 26, 86, 218, 241, + 6, 90, 162, 8, 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, + 79, 65, 32, 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, + 204, 161, 19, 2, 83, 80, 152, 184, 3, 3, 90, 89, 71, 161, 246, 9, 3, 75, + 76, 73, 42, 50, 73, 220, 205, 8, 2, 65, 83, 143, 148, 25, 89, 38, 122, + 65, 200, 1, 5, 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 185, + 235, 26, 8, 70, 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, + 73, 32, 187, 200, 32, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, + 177, 159, 22, 5, 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 197, 245, 26, 2, + 73, 75, 2, 171, 175, 29, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, + 194, 10, 69, 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 181, 252, 33, 7, + 82, 65, 77, 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, + 71, 77, 69, 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, + 84, 69, 82, 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, + 78, 32, 188, 44, 4, 88, 79, 32, 69, 252, 236, 31, 4, 80, 69, 71, 69, 196, + 46, 6, 75, 83, 84, 82, 69, 80, 209, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, + 79, 70, 157, 211, 26, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, + 72, 79, 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, + 44, 180, 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, + 65, 32, 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, + 32, 67, 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, + 84, 82, 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, + 73, 65, 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, + 83, 32, 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, + 77, 65, 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 133, 157, 18, + 18, 69, 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, + 78, 5, 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 211, + 227, 26, 72, 14, 62, 78, 218, 212, 33, 90, 162, 8, 75, 254, 2, 68, 163, + 17, 80, 6, 242, 39, 73, 151, 145, 32, 65, 2, 237, 42, 4, 84, 65, 82, 84, + 4, 18, 68, 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, + 166, 19, 65, 193, 218, 31, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, + 32, 205, 225, 16, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 181, 229, + 16, 4, 83, 89, 78, 65, 2, 159, 212, 31, 65, 22, 80, 6, 69, 78, 73, 75, + 73, 32, 44, 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 140, 150, 30, + 2, 68, 73, 1, 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, + 75, 79, 78, 32, 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, + 65, 82, 69, 83, 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, + 78, 69, 79, 4, 214, 24, 68, 233, 201, 31, 5, 65, 82, 73, 83, 84, 2, 165, + 188, 32, 5, 89, 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, + 77, 73, 114, 83, 213, 236, 32, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, + 68, 73, 65, 6, 40, 4, 84, 72, 79, 82, 183, 226, 32, 79, 4, 218, 156, 33, + 79, 215, 79, 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 145, + 141, 21, 12, 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, + 65, 84, 96, 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, + 79, 98, 82, 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, + 29, 5, 32, 84, 82, 79, 77, 227, 171, 33, 83, 18, 24, 2, 84, 65, 15, 32, + 11, 11, 32, 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 220, 144, 25, 2, + 77, 69, 166, 247, 4, 75, 235, 183, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, + 84, 69, 86, 77, 65, 250, 198, 8, 82, 195, 255, 24, 85, 5, 165, 254, 13, + 2, 32, 65, 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, + 79, 8, 182, 23, 32, 159, 205, 33, 84, 4, 134, 12, 75, 181, 201, 30, 5, + 89, 80, 79, 82, 82, 2, 187, 194, 26, 83, 2, 251, 196, 33, 76, 14, 22, 69, + 147, 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 167, 163, 29, 77, 10, 72, 2, + 69, 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 173, 212, + 26, 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, + 2, 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 235, 217, 32, + 79, 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 241, 143, 27, 7, 73, 75, + 82, 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, + 1, 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, + 69, 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, + 84, 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, + 2, 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, + 2, 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, + 70, 35, 84, 2, 205, 137, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, + 2, 32, 73, 2, 195, 178, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, + 82, 65, 78, 73, 210, 14, 88, 141, 254, 32, 2, 77, 65, 4, 211, 1, 78, 42, + 60, 2, 65, 82, 100, 2, 73, 65, 90, 69, 169, 1, 2, 83, 73, 12, 40, 2, 65, + 75, 201, 246, 16, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, + 73, 75, 73, 4, 11, 83, 4, 17, 2, 77, 65, 4, 23, 32, 7, 11, 32, 4, 198, + 15, 65, 151, 219, 21, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, + 84, 79, 78, 239, 202, 8, 82, 6, 26, 84, 211, 216, 33, 77, 4, 32, 2, 79, + 75, 231, 218, 33, 73, 2, 173, 185, 33, 2, 79, 85, 14, 36, 5, 70, 73, 83, + 84, 79, 67, 76, 10, 46, 80, 214, 1, 78, 170, 8, 76, 187, 1, 83, 2, 135, + 11, 65, 4, 138, 138, 33, 79, 215, 79, 73, 4, 178, 5, 69, 229, 178, 33, 2, + 65, 80, 42, 120, 5, 69, 73, 83, 77, 65, 32, 8, 73, 77, 65, 78, 83, 73, + 83, 32, 182, 1, 84, 154, 1, 89, 249, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, + 219, 230, 21, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, + 79, 83, 9, 11, 32, 6, 18, 84, 39, 68, 4, 34, 82, 13, 4, 69, 84, 82, 65, + 2, 11, 73, 2, 153, 198, 26, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, + 52, 4, 82, 65, 71, 71, 175, 147, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, + 175, 206, 8, 69, 2, 145, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, + 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 150, 219, + 21, 78, 233, 224, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 251, + 178, 33, 86, 5, 11, 84, 2, 255, 193, 16, 73, 40, 42, 69, 70, 72, 218, 1, + 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 162, 148, 8, 76, 137, 167, 23, + 2, 83, 83, 12, 26, 69, 131, 174, 33, 73, 10, 64, 2, 77, 65, 133, 190, 32, + 8, 83, 32, 75, 65, 73, 32, 65, 80, 9, 56, 2, 32, 65, 33, 8, 84, 73, 83, + 77, 79, 83, 32, 69, 2, 177, 209, 32, 3, 80, 76, 79, 4, 182, 178, 33, 83, + 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 182, 129, + 29, 80, 131, 207, 4, 65, 2, 191, 153, 32, 82, 14, 42, 76, 32, 2, 78, 32, + 62, 80, 95, 83, 2, 11, 89, 2, 191, 174, 33, 71, 6, 26, 65, 135, 222, 21, + 78, 4, 250, 2, 82, 239, 169, 33, 76, 4, 46, 65, 141, 157, 32, 5, 83, 73, + 70, 73, 83, 2, 201, 173, 33, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, + 141, 140, 16, 2, 78, 65, 10, 26, 82, 183, 172, 33, 84, 8, 21, 3, 69, 73, + 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 251, 213, 21, 78, + 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 141, 253, 32, 2, 73, 75, 2, + 11, 65, 2, 11, 82, 2, 129, 252, 32, 3, 67, 72, 65, 22, 28, 2, 70, 69, + 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 210, 234, + 29, 75, 235, 183, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, + 133, 2, 6, 80, 76, 73, 32, 68, 89, 2, 233, 1, 11, 73, 71, 82, 65, 77, 77, + 79, 83, 32, 69, 88, 2, 173, 1, 18, 79, 78, 79, 71, 82, 65, 77, 77, 79, + 83, 32, 84, 69, 83, 83, 69, 82, 65, 4, 11, 73, 4, 60, 11, 71, 82, 65, 77, + 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 189, 174, 2, 6, 32, 68, + 79, 68, 69, 75, 2, 129, 248, 32, 4, 73, 77, 79, 82, 8, 26, 79, 243, 248, + 28, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 157, 133, 29, 2, 82, 82, 5, 17, + 2, 32, 68, 2, 11, 73, 2, 159, 248, 28, 80, 162, 82, 182, 1, 65, 134, 72, + 69, 230, 1, 72, 154, 31, 73, 240, 41, 3, 74, 75, 32, 238, 25, 76, 186, + 18, 79, 194, 113, 82, 162, 8, 85, 130, 128, 2, 89, 166, 221, 18, 71, 226, + 155, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, + 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 198, 240, 17, 67, 139, 181, 6, + 83, 4, 34, 85, 221, 189, 26, 2, 65, 32, 2, 201, 193, 31, 2, 67, 69, 4, + 144, 189, 9, 3, 76, 32, 77, 131, 130, 19, 69, 6, 36, 3, 69, 82, 65, 207, + 242, 32, 80, 5, 217, 226, 26, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, + 148, 1, 16, 65, 68, 73, 65, 78, 32, 83, 89, 76, 76, 65, 66, 73, 67, 83, + 32, 148, 49, 2, 67, 69, 94, 68, 172, 152, 23, 3, 78, 69, 68, 139, 225, 9, + 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, 198, + 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, 178, + 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, 5, + 89, 223, 152, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, 83, + 67, 65, 78, 32, 242, 190, 33, 78, 3, 89, 7, 178, 191, 33, 73, 3, 89, 5, + 133, 150, 23, 5, 86, 73, 76, 73, 75, 4, 238, 190, 33, 77, 3, 83, 42, 156, + 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 226, 18, 9, 73, 66, 76, + 69, 45, 67, 82, 69, 69, 253, 180, 7, 10, 69, 65, 86, 69, 82, 32, 68, 69, + 78, 69, 36, 82, 87, 158, 208, 11, 75, 2, 78, 198, 236, 21, 65, 2, 69, 2, + 73, 2, 79, 3, 83, 11, 222, 188, 33, 65, 2, 69, 2, 73, 3, 79, 141, 3, 82, + 65, 186, 42, 87, 206, 155, 8, 72, 218, 182, 22, 79, 182, 56, 73, 207, + 134, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 219, 180, 31, 65, 234, + 2, 170, 1, 68, 122, 71, 94, 72, 46, 73, 46, 74, 82, 75, 30, 78, 66, 83, + 66, 80, 2, 90, 58, 84, 38, 76, 132, 1, 2, 67, 72, 2, 77, 2, 82, 2, 87, 2, + 89, 171, 157, 33, 69, 32, 46, 69, 202, 5, 76, 2, 90, 255, 179, 33, 73, 6, + 26, 78, 171, 185, 33, 69, 4, 166, 210, 16, 69, 181, 191, 6, 2, 84, 65, + 30, 254, 4, 72, 174, 234, 20, 87, 166, 136, 7, 65, 154, 129, 5, 69, 150, + 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 170, 157, 33, 69, 215, 22, 73, 5, + 209, 223, 13, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 234, 243, 32, + 69, 222, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, + 72, 14, 198, 246, 32, 69, 150, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, + 62, 72, 202, 245, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 15, 198, 245, + 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, + 83, 24, 130, 1, 72, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, + 24, 62, 83, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 12, 230, + 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, + 69, 215, 149, 33, 78, 51, 74, 65, 22, 73, 190, 231, 30, 85, 250, 11, 79, + 186, 79, 87, 203, 239, 1, 69, 7, 251, 171, 31, 65, 33, 40, 4, 78, 65, 76, + 32, 139, 178, 33, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 150, + 221, 13, 71, 186, 2, 65, 132, 165, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, + 84, 79, 80, 202, 205, 12, 80, 211, 203, 2, 77, 6, 44, 5, 85, 66, 76, 69, + 32, 195, 212, 23, 87, 4, 232, 193, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, + 82, 84, 73, 67, 131, 159, 7, 65, 6, 38, 73, 253, 229, 31, 3, 65, 73, 83, + 4, 218, 132, 17, 71, 139, 170, 16, 78, 4, 184, 199, 24, 4, 77, 65, 76, + 76, 129, 202, 6, 4, 72, 79, 82, 84, 4, 198, 229, 25, 89, 139, 201, 7, 75, + 7, 170, 174, 33, 73, 3, 78, 39, 66, 87, 234, 27, 65, 230, 210, 30, 79, + 182, 56, 73, 207, 134, 2, 69, 19, 182, 250, 11, 65, 150, 244, 18, 79, + 182, 56, 73, 207, 134, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 166, + 210, 30, 79, 182, 56, 73, 207, 134, 2, 69, 43, 70, 69, 38, 79, 238, 25, + 65, 66, 87, 218, 138, 31, 73, 207, 134, 2, 72, 7, 229, 133, 26, 4, 68, + 73, 65, 76, 7, 11, 79, 5, 161, 233, 31, 8, 83, 69, 45, 67, 82, 69, 69, + 32, 149, 1, 164, 1, 8, 45, 67, 82, 69, 69, 32, 84, 72, 38, 65, 250, 2, + 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 218, 138, + 31, 73, 206, 134, 2, 69, 3, 72, 6, 150, 163, 31, 73, 207, 134, 2, 69, 63, + 104, 6, 83, 75, 65, 80, 73, 32, 188, 1, 7, 84, 84, 73, 76, 73, 75, 32, + 206, 160, 31, 65, 207, 134, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, + 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 220, 230, 27, 2, 75, 87, 178, 124, + 67, 2, 80, 2, 84, 182, 213, 2, 87, 187, 113, 45, 4, 194, 136, 33, 79, + 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 202, 229, 27, 65, 174, + 130, 3, 79, 183, 56, 73, 19, 38, 65, 162, 231, 30, 79, 183, 56, 73, 9, + 210, 159, 31, 65, 207, 134, 2, 73, 15, 206, 228, 27, 65, 174, 130, 3, 79, + 183, 56, 73, 18, 224, 9, 3, 73, 75, 32, 149, 243, 22, 2, 85, 84, 33, 68, + 7, 74, 73, 66, 87, 65, 89, 32, 210, 164, 33, 78, 2, 79, 3, 89, 24, 74, + 78, 166, 176, 29, 83, 226, 243, 3, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, + 11, 87, 8, 246, 228, 30, 79, 183, 56, 73, 39, 206, 4, 87, 166, 13, 65, + 38, 79, 246, 138, 31, 73, 207, 134, 2, 69, 39, 104, 7, 45, 67, 82, 69, + 69, 32, 82, 146, 14, 87, 182, 2, 65, 230, 210, 30, 79, 182, 56, 73, 207, + 134, 2, 69, 4, 210, 139, 33, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, + 138, 1, 79, 238, 2, 87, 154, 175, 11, 80, 250, 229, 19, 73, 207, 134, 2, + 69, 43, 26, 89, 195, 154, 31, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, + 54, 74, 218, 4, 83, 198, 135, 33, 89, 202, 18, 84, 147, 1, 77, 10, 166, + 225, 30, 79, 130, 191, 2, 65, 2, 69, 3, 73, 6, 214, 208, 29, 85, 159, + 207, 3, 73, 37, 70, 87, 202, 13, 79, 174, 222, 11, 65, 202, 172, 19, 73, + 207, 134, 2, 69, 16, 198, 13, 79, 150, 208, 27, 65, 226, 186, 3, 73, 207, + 134, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, + 246, 157, 33, 79, 3, 89, 8, 226, 156, 33, 65, 2, 69, 2, 73, 3, 79, 117, + 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 226, 164, 11, 89, 198, + 173, 19, 79, 182, 56, 73, 207, 134, 2, 69, 39, 108, 7, 45, 67, 82, 69, + 69, 32, 84, 226, 4, 87, 222, 213, 27, 65, 174, 130, 3, 79, 182, 56, 73, + 207, 134, 2, 69, 16, 11, 72, 17, 174, 218, 27, 65, 174, 130, 3, 79, 182, + 56, 73, 207, 134, 2, 69, 12, 11, 72, 12, 142, 220, 30, 79, 174, 168, 2, + 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 158, 154, 33, 65, 2, 69, 2, + 73, 3, 79, 17, 238, 216, 27, 65, 174, 130, 3, 79, 174, 168, 2, 87, 214, + 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 174, 226, 11, 65, 203, 172, + 19, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, + 251, 2, 69, 180, 1, 110, 76, 66, 77, 2, 80, 2, 89, 16, 2, 78, 87, 38, 82, + 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 250, + 207, 27, 65, 174, 130, 3, 79, 131, 191, 2, 69, 17, 243, 5, 87, 6, 218, + 213, 27, 65, 175, 193, 5, 69, 13, 154, 167, 31, 87, 202, 239, 1, 65, 2, + 69, 2, 73, 3, 79, 28, 22, 72, 239, 4, 87, 14, 235, 4, 87, 16, 22, 72, + 199, 4, 87, 2, 159, 166, 31, 87, 2, 135, 240, 22, 82, 31, 11, 79, 29, 41, + 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 201, 149, 31, 6, + 70, 73, 78, 65, 76, 32, 25, 50, 87, 154, 148, 33, 65, 2, 69, 2, 73, 3, + 79, 14, 234, 210, 27, 65, 174, 130, 3, 79, 182, 56, 73, 251, 239, 1, 69, + 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 218, 138, + 31, 73, 207, 134, 2, 69, 24, 110, 80, 194, 209, 30, 67, 2, 75, 2, 76, 2, + 77, 2, 78, 2, 83, 2, 84, 2, 89, 214, 161, 2, 79, 247, 30, 87, 4, 190, + 162, 31, 87, 215, 208, 1, 79, 9, 150, 139, 31, 65, 207, 134, 2, 89, 7, + 190, 145, 33, 79, 3, 89, 14, 246, 207, 27, 65, 174, 130, 3, 79, 182, 56, + 73, 207, 134, 2, 69, 10, 26, 76, 203, 144, 33, 82, 9, 26, 32, 239, 162, + 9, 76, 4, 222, 217, 21, 67, 207, 208, 9, 84, 4, 178, 249, 32, 76, 215, + 22, 89, 4, 180, 247, 14, 3, 73, 84, 85, 253, 241, 7, 3, 82, 73, 67, 124, + 140, 1, 2, 68, 32, 102, 69, 72, 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, + 82, 32, 210, 3, 79, 62, 80, 90, 82, 181, 250, 26, 4, 32, 83, 76, 73, 6, + 52, 5, 73, 78, 68, 69, 88, 157, 129, 32, 2, 70, 73, 5, 153, 255, 31, 6, + 32, 68, 73, 86, 73, 68, 6, 26, 84, 147, 131, 21, 32, 5, 209, 136, 10, 6, + 32, 73, 78, 83, 69, 82, 98, 196, 1, 2, 67, 45, 38, 76, 22, 77, 50, 78, + 38, 83, 46, 84, 22, 85, 230, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, + 80, 214, 208, 25, 82, 218, 220, 1, 73, 206, 134, 2, 66, 2, 79, 2, 81, 3, + 88, 4, 226, 163, 26, 49, 159, 148, 3, 51, 7, 139, 215, 3, 68, 11, 11, 66, + 9, 226, 138, 33, 50, 2, 51, 3, 52, 9, 190, 138, 33, 68, 2, 71, 3, 78, 13, + 162, 214, 3, 72, 2, 84, 251, 179, 29, 83, 7, 247, 213, 3, 84, 13, 11, 85, + 11, 11, 85, 9, 194, 137, 33, 50, 2, 51, 3, 85, 4, 148, 128, 32, 6, 85, + 83, 69, 76, 32, 72, 139, 137, 1, 78, 4, 152, 230, 26, 6, 32, 83, 84, 82, + 69, 65, 173, 164, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 198, 240, 26, 73, + 155, 251, 5, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 132, 229, 6, 2, 84, 69, 157, 132, 24, 6, 87, 82, 89, 32, 83, + 77, 108, 88, 16, 67, 65, 83, 73, 65, 78, 32, 65, 76, 66, 65, 78, 73, 65, + 78, 32, 159, 172, 29, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 189, + 160, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, + 46, 73, 46, 74, 34, 75, 34, 76, 34, 80, 34, 83, 74, 84, 62, 89, 46, 90, + 140, 189, 16, 2, 81, 65, 166, 173, 6, 77, 208, 156, 6, 3, 86, 69, 89, + 182, 189, 2, 70, 220, 18, 3, 78, 79, 87, 146, 19, 82, 234, 21, 88, 150, + 46, 79, 202, 26, 66, 225, 24, 3, 72, 69, 89, 4, 194, 202, 32, 79, 167, + 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 250, 178, 32, 89, 215, 79, + 82, 8, 186, 205, 27, 65, 174, 163, 5, 79, 203, 17, 73, 4, 254, 129, 33, + 87, 3, 89, 8, 42, 90, 134, 186, 31, 89, 139, 171, 1, 65, 4, 202, 245, 31, + 89, 167, 122, 65, 4, 194, 177, 32, 89, 215, 79, 66, 4, 136, 214, 31, 2, + 72, 69, 151, 154, 1, 73, 6, 166, 198, 31, 82, 210, 106, 87, 251, 76, 78, + 4, 134, 244, 31, 72, 191, 14, 65, 4, 238, 129, 32, 73, 195, 69, 65, 4, + 138, 176, 32, 65, 159, 51, 89, 4, 142, 1, 73, 223, 174, 32, 69, 8, 34, + 72, 229, 251, 32, 2, 69, 89, 6, 142, 220, 19, 65, 167, 145, 13, 79, 6, + 38, 73, 198, 242, 31, 89, 159, 78, 65, 2, 223, 197, 32, 87, 4, 132, 184, + 32, 2, 65, 89, 1, 2, 79, 87, 6, 142, 223, 15, 72, 199, 177, 4, 65, 16, + 72, 2, 68, 73, 32, 2, 78, 84, 180, 155, 31, 3, 76, 84, 73, 207, 77, 82, + 4, 162, 246, 31, 32, 199, 100, 76, 8, 32, 2, 82, 69, 227, 245, 31, 32, 6, + 44, 5, 76, 73, 78, 69, 32, 207, 226, 4, 32, 4, 174, 154, 21, 76, 255, + 252, 9, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 140, + 152, 4, 6, 82, 73, 83, 84, 77, 65, 195, 222, 18, 85, 198, 2, 66, 73, 32, + 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 226, 188, 32, 78, + 211, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, + 98, 86, 170, 255, 22, 68, 170, 171, 7, 81, 236, 90, 5, 77, 65, 65, 89, + 89, 144, 174, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, + 89, 142, 162, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 182, 228, + 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 146, 237, 1, 65, 186, 2, 69, 2, + 73, 3, 85, 8, 166, 163, 28, 68, 182, 228, 2, 72, 147, 237, 1, 65, 8, 166, + 135, 31, 71, 2, 78, 2, 89, 147, 237, 1, 65, 8, 190, 162, 28, 84, 182, + 228, 2, 72, 147, 237, 1, 65, 4, 190, 134, 31, 89, 147, 237, 1, 65, 8, 40, + 4, 73, 71, 78, 32, 195, 188, 21, 69, 6, 142, 143, 29, 67, 226, 182, 1, + 86, 223, 234, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 203, 194, 30, 73, 24, 238, 139, 29, 65, 182, 25, 85, 210, 200, 1, 69, 2, + 73, 3, 79, 166, 1, 236, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, + 83, 73, 71, 78, 32, 112, 7, 76, 69, 84, 84, 69, 82, 32, 152, 4, 12, 80, + 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 60, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 175, 254, 30, 68, 14, 72, 6, 70, 73, 78, 65, 76, + 32, 138, 239, 32, 76, 2, 82, 2, 87, 3, 89, 6, 234, 240, 32, 78, 86, 72, + 3, 77, 104, 132, 2, 6, 70, 73, 78, 65, 76, 32, 110, 78, 50, 77, 78, 80, + 146, 195, 11, 66, 198, 231, 5, 68, 234, 191, 2, 83, 166, 252, 10, 65, + 152, 191, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 254, 68, 72, 2, 76, + 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 254, 246, 30, + 78, 198, 186, 1, 83, 194, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, + 2, 84, 3, 89, 14, 46, 71, 34, 72, 134, 215, 32, 85, 215, 22, 65, 4, 162, + 215, 32, 85, 215, 22, 65, 6, 130, 215, 32, 85, 158, 20, 74, 187, 2, 65, + 6, 242, 234, 32, 72, 2, 80, 187, 2, 65, 8, 190, 136, 11, 83, 246, 142, + 12, 68, 45, 4, 84, 82, 73, 80, 20, 238, 135, 29, 65, 142, 222, 1, 73, + 186, 198, 1, 79, 2, 85, 191, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, + 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 140, 223, 9, 3, 65, 66, 85, + 211, 245, 22, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, + 69, 78, 68, 149, 214, 24, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, + 82, 5, 233, 234, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, + 69, 90, 82, 224, 247, 4, 2, 83, 84, 241, 219, 14, 2, 81, 85, 6, 56, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 151, 167, 32, 32, 5, 223, 247, 29, 32, 4, + 184, 215, 26, 4, 83, 69, 32, 87, 177, 231, 4, 9, 82, 73, 78, 71, 32, 77, + 69, 71, 65, 220, 2, 40, 5, 79, 75, 69, 69, 32, 143, 5, 82, 216, 2, 46, + 76, 1, 7, 83, 77, 65, 76, 76, 32, 76, 172, 1, 33, 6, 69, 84, 84, 69, 82, + 32, 172, 1, 170, 1, 68, 74, 72, 74, 78, 70, 83, 62, 84, 54, 71, 2, 76, 2, + 77, 0, 2, 81, 85, 2, 87, 2, 89, 158, 224, 32, 75, 186, 2, 65, 2, 69, 2, + 73, 2, 79, 2, 85, 3, 86, 14, 222, 226, 32, 76, 186, 2, 65, 2, 69, 2, 73, + 2, 79, 2, 85, 3, 86, 14, 150, 226, 32, 78, 186, 2, 65, 2, 69, 2, 73, 2, + 79, 2, 85, 3, 86, 14, 166, 240, 28, 65, 226, 243, 3, 69, 2, 73, 2, 79, 2, + 85, 3, 86, 15, 194, 227, 32, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, + 50, 76, 2, 83, 214, 226, 32, 65, 2, 69, 3, 73, 12, 210, 226, 32, 65, 2, + 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 135, + 206, 31, 73, 2, 255, 208, 32, 83, 12, 96, 2, 76, 68, 174, 222, 7, 32, + 232, 239, 2, 2, 80, 77, 228, 193, 20, 2, 67, 75, 151, 129, 1, 82, 5, 213, + 128, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, + 65, 78, 32, 222, 253, 4, 80, 249, 203, 11, 5, 67, 79, 76, 65, 84, 56, 52, + 7, 76, 69, 84, 84, 69, 82, 32, 243, 134, 21, 78, 42, 224, 1, 6, 67, 85, + 82, 76, 69, 68, 30, 83, 202, 131, 21, 68, 34, 76, 254, 194, 1, 82, 134, + 212, 2, 65, 50, 71, 90, 90, 98, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, + 66, 190, 173, 4, 78, 254, 1, 84, 2, 87, 202, 103, 80, 171, 4, 77, 2, 161, + 223, 31, 2, 32, 87, 6, 176, 131, 21, 6, 77, 65, 76, 76, 32, 65, 172, 210, + 5, 2, 65, 77, 139, 136, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 233, 40, 7, + 84, 89, 83, 67, 65, 80, 69, 2, 247, 186, 32, 69, 226, 4, 28, 2, 76, 69, + 239, 39, 85, 220, 4, 30, 32, 181, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, + 84, 72, 32, 161, 133, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, + 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 65, 78, 68, + 32, 84, 79, 80, 32, 72, 65, 76, 70, 32, 68, 73, 86, 73, 68, 69, 68, 32, + 66, 89, 22, 142, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, + 90, 79, 78, 84, 65, 76, 32, 83, 84, 82, 79, 75, 69, 83, 44, 6, 85, 80, + 80, 69, 82, 32, 44, 17, 65, 76, 76, 32, 66, 85, 84, 32, 85, 80, 80, 69, + 82, 32, 76, 69, 70, 238, 230, 25, 86, 190, 227, 3, 82, 231, 58, 72, 4, + 138, 204, 29, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, 32, + 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, 83, + 69, 2, 173, 225, 29, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, 72, + 227, 202, 29, 72, 2, 245, 202, 29, 10, 84, 32, 81, 85, 65, 68, 82, 65, + 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, 72, + 250, 2, 73, 210, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, 66, + 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, 32, + 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 224, 228, 9, 4, 90, + 69, 82, 79, 222, 253, 15, 69, 250, 129, 4, 86, 130, 65, 71, 190, 110, 83, + 199, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, + 69, 249, 166, 29, 7, 83, 84, 69, 82, 73, 83, 75, 4, 160, 220, 8, 11, 45, + 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 211, 218, 22, 32, 4, 32, 2, 79, + 76, 139, 244, 30, 85, 2, 175, 225, 13, 68, 16, 60, 4, 82, 79, 83, 83, + 210, 2, 32, 142, 205, 32, 67, 3, 68, 10, 26, 32, 219, 193, 1, 73, 8, 64, + 6, 70, 79, 82, 77, 69, 69, 201, 226, 30, 4, 80, 79, 77, 77, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 254, 177, 30, 70, 251, 78, 84, 30, 34, 73, 70, 79, + 183, 232, 30, 65, 24, 152, 188, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, + 247, 163, 26, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 227, + 162, 29, 84, 2, 197, 230, 18, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, + 65, 73, 68, 64, 212, 1, 6, 65, 78, 71, 85, 76, 32, 184, 151, 12, 20, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, + 78, 152, 181, 9, 4, 69, 65, 86, 89, 189, 158, 6, 8, 85, 77, 65, 78, 32, + 70, 73, 71, 58, 110, 67, 160, 128, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, + 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 27, 84, 8, 254, 255, 19, 72, + 157, 3, 4, 73, 69, 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, + 72, 32, 212, 8, 27, 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 176, 26, 9, + 78, 70, 79, 82, 77, 65, 84, 73, 79, 110, 178, 1, 65, 110, 67, 90, 69, 86, + 70, 62, 72, 38, 75, 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 246, + 128, 20, 84, 50, 87, 146, 204, 1, 73, 172, 24, 2, 80, 82, 231, 193, 9, + 79, 8, 130, 129, 20, 76, 180, 132, 1, 3, 67, 67, 69, 196, 212, 6, 3, 84, + 84, 69, 245, 17, 5, 68, 86, 65, 78, 84, 8, 26, 79, 159, 208, 29, 69, 6, + 168, 180, 10, 2, 82, 82, 146, 204, 9, 78, 175, 179, 12, 80, 8, 206, 128, + 20, 78, 246, 253, 8, 65, 148, 131, 3, 5, 88, 67, 69, 76, 76, 219, 24, 73, + 10, 218, 128, 20, 73, 128, 169, 5, 2, 69, 77, 131, 162, 6, 79, 4, 214, + 203, 28, 73, 203, 251, 2, 65, 4, 240, 141, 7, 8, 73, 78, 68, 69, 82, 71, + 65, 82, 199, 233, 21, 79, 6, 234, 255, 19, 65, 130, 197, 11, 79, 179, 83, + 69, 8, 38, 69, 130, 161, 31, 65, 199, 81, 79, 4, 128, 169, 31, 3, 68, 73, + 67, 251, 15, 84, 6, 26, 73, 139, 170, 31, 65, 4, 162, 151, 32, 71, 231, + 19, 78, 2, 237, 218, 27, 2, 85, 69, 8, 26, 69, 195, 150, 32, 73, 6, 242, + 254, 19, 83, 205, 225, 7, 2, 76, 73, 22, 90, 69, 38, 85, 164, 230, 13, 2, + 67, 72, 166, 152, 6, 79, 22, 80, 34, 84, 211, 180, 11, 73, 4, 178, 132, + 10, 67, 215, 234, 20, 86, 6, 198, 255, 19, 80, 180, 177, 4, 3, 73, 84, + 65, 223, 142, 8, 78, 4, 150, 191, 32, 67, 3, 82, 98, 116, 8, 65, 84, 65, + 75, 65, 78, 65, 32, 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, + 65, 67, 84, 69, 82, 32, 94, 194, 208, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, + 2, 83, 2, 84, 126, 87, 46, 89, 154, 236, 21, 65, 2, 69, 2, 73, 2, 79, 3, + 85, 4, 216, 214, 15, 3, 74, 85, 69, 165, 199, 16, 4, 67, 72, 65, 77, 106, + 166, 211, 26, 65, 203, 218, 3, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, + 73, 67, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 191, 184, 30, 73, 2, + 129, 84, 4, 87, 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 151, 251, + 19, 78, 6, 180, 1, 3, 73, 71, 72, 191, 222, 24, 76, 30, 28, 3, 73, 70, + 84, 35, 79, 6, 214, 1, 89, 203, 231, 30, 69, 24, 142, 2, 82, 255, 250, + 19, 85, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, + 32, 191, 231, 30, 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, + 87, 69, 5, 11, 32, 2, 135, 252, 24, 79, 24, 74, 89, 223, 230, 30, 69, 24, + 26, 78, 163, 187, 31, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, + 78, 218, 201, 30, 70, 30, 83, 42, 84, 190, 83, 78, 211, 110, 69, 4, 226, + 250, 24, 32, 151, 188, 7, 69, 2, 213, 163, 30, 8, 32, 80, 79, 73, 78, 84, + 69, 68, 8, 184, 146, 17, 3, 79, 83, 84, 246, 223, 6, 65, 188, 189, 2, 8, + 69, 82, 80, 69, 78, 68, 73, 67, 203, 131, 4, 76, 4, 128, 139, 29, 3, 73, + 78, 71, 255, 151, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 151, 224, + 25, 73, 5, 163, 204, 21, 32, 6, 44, 5, 72, 73, 84, 69, 32, 135, 180, 32, + 90, 4, 206, 215, 30, 66, 211, 24, 83, 6, 226, 198, 26, 77, 252, 138, 4, + 6, 76, 65, 84, 73, 79, 78, 233, 158, 1, 3, 83, 32, 84, 5, 233, 240, 30, + 6, 32, 65, 84, 32, 68, 85, 154, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, + 73, 66, 73, 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, + 144, 3, 8, 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, + 75, 69, 32, 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 194, 225, 10, + 56, 3, 57, 60, 202, 1, 49, 143, 182, 18, 48, 176, 7, 26, 65, 131, 225, + 10, 57, 176, 3, 134, 1, 54, 142, 182, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 231, 238, 8, 68, 28, + 130, 175, 32, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, + 2, 57, 2, 65, 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, + 68, 62, 69, 50, 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, + 80, 46, 82, 78, 83, 182, 3, 84, 82, 87, 200, 197, 10, 2, 78, 69, 252, 16, + 4, 75, 78, 73, 70, 159, 225, 11, 79, 14, 74, 79, 192, 229, 10, 4, 82, 85, + 83, 72, 154, 218, 4, 65, 179, 139, 13, 76, 6, 230, 140, 10, 76, 170, 136, + 22, 78, 215, 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, + 68, 32, 162, 3, 73, 50, 76, 234, 168, 31, 79, 175, 78, 72, 44, 114, 69, + 38, 70, 50, 71, 38, 76, 34, 83, 94, 84, 214, 146, 14, 87, 42, 68, 246, + 149, 16, 66, 234, 117, 72, 135, 2, 67, 4, 222, 216, 30, 86, 203, 180, 1, + 65, 6, 146, 196, 30, 73, 214, 211, 1, 76, 235, 16, 82, 4, 234, 195, 22, + 79, 235, 160, 8, 65, 4, 194, 167, 31, 69, 175, 48, 79, 10, 174, 204, 6, + 73, 242, 253, 16, 65, 142, 194, 1, 72, 232, 249, 6, 3, 80, 69, 69, 215, + 11, 69, 6, 214, 10, 85, 176, 155, 10, 2, 65, 78, 143, 255, 3, 79, 4, 156, + 150, 20, 3, 86, 73, 76, 195, 255, 11, 84, 4, 214, 161, 17, 73, 171, 241, + 13, 79, 6, 220, 201, 23, 2, 73, 86, 194, 226, 2, 69, 227, 249, 5, 79, 10, + 196, 6, 2, 65, 84, 254, 136, 32, 87, 3, 89, 4, 198, 133, 31, 73, 247, 79, + 79, 8, 244, 5, 4, 82, 65, 83, 83, 255, 215, 28, 72, 10, 48, 2, 69, 65, + 218, 198, 27, 79, 175, 208, 3, 65, 6, 186, 8, 82, 171, 156, 32, 68, 10, + 72, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 135, 151, 30, 65, + 8, 42, 84, 234, 142, 14, 68, 251, 167, 16, 69, 4, 194, 6, 85, 191, 154, + 14, 79, 12, 132, 221, 10, 3, 79, 78, 71, 197, 176, 15, 3, 65, 77, 69, 14, + 18, 69, 35, 79, 4, 174, 134, 32, 65, 159, 27, 83, 10, 148, 220, 10, 3, + 85, 78, 68, 158, 236, 12, 84, 166, 150, 7, 82, 147, 116, 79, 6, 200, 219, + 10, 2, 65, 87, 143, 243, 14, 69, 8, 34, 65, 233, 252, 27, 2, 69, 80, 6, + 202, 209, 31, 73, 214, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, + 196, 1, 4, 80, 73, 82, 73, 132, 214, 10, 4, 77, 65, 76, 76, 250, 222, 11, + 72, 219, 151, 9, 85, 8, 40, 4, 67, 79, 78, 68, 183, 150, 31, 65, 6, 11, + 32, 6, 230, 178, 30, 84, 203, 83, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, + 69, 68, 32, 207, 155, 32, 76, 10, 34, 72, 50, 87, 159, 161, 10, 89, 4, + 184, 139, 10, 3, 65, 76, 70, 251, 180, 17, 79, 4, 130, 194, 6, 65, 215, + 183, 21, 72, 6, 192, 1, 2, 79, 85, 211, 236, 31, 65, 8, 58, 85, 174, 142, + 24, 65, 230, 199, 1, 72, 135, 143, 3, 73, 2, 243, 187, 23, 82, 12, 26, + 65, 49, 2, 69, 83, 8, 176, 214, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 131, + 214, 10, 84, 72, 110, 72, 162, 1, 80, 38, 83, 226, 161, 30, 84, 172, 247, + 1, 2, 66, 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 29, 50, 80, 22, 90, + 136, 154, 32, 2, 88, 87, 87, 71, 5, 155, 154, 32, 87, 19, 50, 90, 150, + 162, 30, 87, 150, 248, 1, 71, 3, 84, 9, 146, 162, 30, 90, 151, 248, 1, + 80, 9, 130, 154, 32, 68, 2, 71, 3, 90, 21, 50, 87, 30, 90, 146, 153, 32, + 71, 2, 80, 3, 84, 7, 170, 153, 32, 71, 3, 90, 7, 186, 152, 32, 87, 87, + 90, 124, 62, 65, 194, 1, 73, 114, 79, 173, 181, 20, 4, 69, 65, 82, 32, 8, + 132, 1, 2, 80, 80, 252, 177, 18, 6, 83, 83, 73, 67, 65, 76, 241, 187, 12, + 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, 32, 4, 144, 130, + 19, 5, 73, 78, 71, 32, 72, 199, 226, 1, 69, 6, 48, 6, 78, 75, 73, 78, 71, + 32, 207, 255, 30, 80, 4, 234, 198, 24, 71, 229, 169, 3, 6, 66, 69, 69, + 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, 236, 3, 2, 85, 68, + 227, 154, 29, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, 237, 2, 5, 87, 73, + 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, 18, 83, 51, 84, 8, + 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, 25, 3, 79, 85, 82, + 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, 86, 69, 16, 38, 69, + 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, 76, 27, 79, 4, 11, + 86, 4, 11, 69, 4, 196, 164, 28, 2, 32, 79, 213, 131, 2, 3, 45, 84, 72, + 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, 226, 2, 84, 122, + 79, 247, 135, 30, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, 4, 79, 78, 84, + 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, + 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, 65, 80, 80, + 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, 69, 70, 84, + 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 76, + 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, 73, 82, 67, + 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 129, 184, 29, 6, 67, 73, 82, 67, + 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, 67, 76, 69, + 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, + 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, 21, 3, 80, + 69, 78, 2, 11, 32, 2, 161, 182, 29, 4, 67, 73, 82, 67, 26, 32, 2, 68, 32, + 167, 149, 31, 32, 24, 216, 1, 2, 83, 85, 66, 85, 254, 243, 18, 77, 180, + 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 192, 11, 66, 145, 147, 1, + 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, 84, 72, + 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 145, 177, 24, 3, + 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, 32, + 83, 69, 82, 73, 70, 83, 2, 197, 148, 24, 3, 66, 82, 69, 5, 221, 248, 30, + 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, + 8, 72, 4, 84, 79, 82, 78, 190, 162, 5, 76, 146, 151, 19, 83, 215, 161, 6, + 82, 2, 155, 140, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, 82, 78, + 220, 5, 2, 79, 75, 58, 80, 154, 19, 82, 138, 1, 85, 186, 165, 27, 87, + 160, 242, 2, 2, 70, 70, 234, 47, 73, 171, 51, 65, 6, 26, 75, 139, 149, 4, + 79, 4, 186, 214, 8, 82, 129, 151, 22, 4, 84, 65, 73, 76, 8, 44, 2, 79, + 78, 209, 187, 24, 3, 76, 73, 83, 7, 11, 32, 4, 146, 178, 25, 69, 171, + 203, 5, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, 94, + 80, 239, 149, 31, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, 246, + 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, 216, + 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, 80, + 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, 238, + 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, 183, + 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 168, 185, 24, 6, 83, 84, + 69, 82, 73, 83, 201, 172, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, 65, + 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 180, 164, 12, 6, 71, 82, 65, + 86, 69, 45, 195, 193, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 177, 180, 3, + 4, 78, 85, 73, 84, 12, 42, 82, 129, 215, 29, 4, 73, 78, 68, 85, 10, 48, + 3, 69, 86, 69, 197, 228, 29, 3, 73, 68, 71, 7, 162, 252, 11, 45, 183, + 218, 17, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, + 76, 73, 67, 32, 184, 28, 11, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, + 175, 238, 11, 69, 6, 162, 18, 82, 199, 131, 28, 78, 4, 41, 8, 79, 67, 75, + 87, 73, 83, 69, 32, 4, 184, 131, 12, 4, 82, 73, 78, 71, 239, 160, 16, 65, + 10, 76, 4, 77, 77, 65, 32, 189, 16, 10, 78, 74, 79, 73, 78, 73, 78, 71, + 32, 77, 6, 250, 229, 12, 65, 247, 251, 16, 66, 116, 252, 1, 8, 72, 85, + 78, 68, 82, 69, 68, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 166, 5, 80, + 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 206, 243, 2, 75, 132, 214, + 12, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 217, + 241, 15, 2, 86, 90, 4, 194, 7, 77, 215, 158, 3, 84, 84, 238, 1, 66, 38, + 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 214, 137, 3, 76, + 140, 5, 5, 77, 79, 78, 79, 71, 142, 18, 72, 202, 174, 12, 84, 190, 163, + 15, 90, 254, 82, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, + 187, 2, 65, 4, 174, 147, 6, 73, 199, 226, 25, 69, 4, 168, 155, 25, 3, 74, + 69, 82, 167, 218, 6, 69, 14, 58, 83, 230, 244, 31, 70, 2, 76, 2, 77, 2, + 78, 3, 82, 5, 183, 176, 30, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, + 32, 151, 244, 31, 69, 6, 194, 145, 6, 66, 210, 226, 25, 65, 3, 69, 5, + 239, 243, 24, 77, 6, 26, 72, 155, 209, 15, 79, 4, 250, 251, 29, 67, 191, + 247, 1, 65, 5, 193, 153, 3, 2, 75, 82, 8, 142, 230, 27, 69, 190, 240, 3, + 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 133, 212, 28, + 4, 79, 75, 82, 89, 4, 170, 246, 2, 89, 241, 175, 12, 3, 76, 65, 84, 2, + 145, 13, 5, 32, 80, 78, 69, 85, 10, 80, 2, 69, 78, 0, 7, 72, 79, 85, 83, + 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 169, 164, 21, + 5, 73, 76, 76, 73, 79, 110, 62, 69, 244, 1, 8, 73, 65, 69, 82, 69, 83, + 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 249, 133, + 31, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 240, 179, 25, 6, + 83, 73, 71, 78, 32, 65, 167, 204, 4, 68, 14, 214, 218, 31, 86, 162, 17, + 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 246, 140, 27, 45, 151, + 186, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, + 38, 32, 149, 215, 15, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 241, + 165, 6, 5, 66, 69, 76, 79, 87, 7, 151, 193, 31, 32, 48, 22, 32, 191, 4, + 68, 46, 230, 1, 66, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 26, + 77, 54, 79, 34, 80, 68, 2, 82, 73, 48, 5, 84, 73, 76, 68, 69, 80, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 174, 210, 15, 65, 42, 71, 166, 179, 4, + 76, 187, 228, 10, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, + 181, 193, 16, 2, 79, 78, 4, 230, 35, 80, 255, 224, 29, 86, 6, 238, 36, + 76, 205, 206, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, + 71, 72, 84, 87, 139, 228, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, + 82, 73, 71, 72, 2, 199, 205, 29, 84, 6, 178, 239, 11, 83, 211, 255, 13, + 76, 2, 141, 205, 15, 2, 32, 67, 4, 128, 41, 2, 32, 84, 227, 158, 30, 87, + 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 129, 193, 25, 4, 81, 85, + 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 216, 145, 6, 3, + 75, 69, 89, 160, 241, 1, 7, 85, 80, 87, 65, 82, 68, 32, 227, 166, 22, 68, + 5, 219, 254, 17, 32, 4, 134, 138, 25, 67, 195, 186, 5, 81, 4, 32, 2, 69, + 82, 135, 137, 5, 79, 2, 143, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, + 84, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 143, 3, 82, 76, 238, 1, 68, + 30, 73, 46, 83, 50, 84, 182, 250, 5, 65, 22, 66, 94, 67, 134, 1, 70, 38, + 71, 238, 2, 77, 32, 2, 76, 74, 34, 78, 88, 2, 80, 79, 30, 82, 194, 2, 86, + 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 154, 253, 4, 79, 235, + 5, 75, 4, 238, 252, 5, 74, 31, 79, 11, 250, 253, 5, 78, 54, 79, 135, 182, + 25, 90, 8, 218, 254, 5, 77, 130, 4, 76, 243, 204, 20, 72, 4, 142, 131, 6, + 86, 255, 201, 25, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, + 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 153, 228, 27, 5, 80, 72, 69, 77, + 69, 24, 68, 6, 68, 73, 71, 73, 84, 32, 65, 7, 76, 69, 84, 84, 69, 82, 32, + 14, 202, 177, 16, 83, 142, 192, 13, 70, 70, 84, 62, 90, 143, 83, 79, 10, + 222, 202, 31, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, + 45, 6, 26, 65, 215, 242, 15, 84, 4, 245, 143, 4, 4, 67, 67, 69, 78, 4, + 208, 220, 11, 6, 65, 67, 85, 84, 69, 45, 195, 231, 3, 77, 14, 148, 1, 8, + 77, 85, 83, 73, 67, 65, 76, 32, 150, 151, 4, 75, 250, 182, 2, 80, 174, 4, + 89, 249, 227, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, + 84, 131, 135, 15, 80, 4, 218, 135, 15, 69, 35, 82, 6, 206, 200, 12, 79, + 216, 149, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 163, 45, 82, 14, 26, 78, + 207, 179, 29, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 143, 222, 27, + 70, 10, 64, 2, 66, 82, 45, 10, 68, 79, 85, 66, 76, 69, 32, 65, 82, 67, 6, + 22, 69, 235, 37, 73, 4, 211, 181, 11, 86, 4, 167, 192, 29, 72, 8, 128, 1, + 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 153, + 180, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 146, 237, 9, 83, 35, + 86, 158, 1, 80, 5, 65, 84, 73, 78, 32, 156, 8, 3, 69, 70, 84, 184, 3, 2, + 73, 71, 83, 79, 106, 156, 1, 21, 76, 69, 84, 84, 69, 82, 32, 83, 77, 65, + 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 53, 13, 83, 77, 65, 76, 76, + 32, 76, 69, 84, 84, 69, 82, 32, 10, 218, 213, 31, 71, 2, 76, 2, 77, 2, + 78, 3, 82, 96, 226, 1, 65, 70, 67, 34, 69, 30, 70, 78, 73, 86, 76, 106, + 79, 2, 85, 134, 1, 82, 50, 84, 206, 131, 12, 83, 194, 134, 2, 66, 226, + 156, 2, 87, 158, 168, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, + 2, 86, 2, 88, 3, 90, 13, 142, 244, 2, 32, 146, 235, 12, 76, 166, 244, 15, + 69, 2, 79, 3, 86, 5, 221, 227, 11, 3, 32, 67, 69, 7, 206, 209, 31, 83, 3, + 84, 5, 185, 219, 26, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, + 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 206, 209, 31, 68, 2, + 71, 2, 82, 3, 84, 7, 136, 210, 15, 14, 32, 87, 73, 84, 72, 32, 68, 79, + 85, 66, 76, 69, 32, 77, 165, 172, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 194, 243, 11, 68, 201, 187, 12, 15, 76, 73, 71, 72, + 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 146, 210, 11, + 82, 227, 228, 17, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, + 213, 2, 6, 87, 65, 82, 68, 83, 32, 28, 154, 1, 65, 112, 12, 80, 65, 82, + 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 177, 165, + 29, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, + 79, 87, 233, 179, 29, 2, 78, 71, 8, 26, 72, 243, 179, 29, 32, 4, 225, + 179, 29, 3, 69, 65, 68, 4, 244, 204, 7, 4, 65, 66, 79, 86, 245, 204, 23, + 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 133, 198, 11, 5, 65, 82, 82, 79, + 87, 10, 52, 6, 65, 84, 85, 82, 69, 32, 169, 17, 2, 72, 84, 8, 238, 1, 76, + 23, 82, 10, 36, 3, 78, 71, 32, 251, 176, 30, 87, 8, 172, 7, 7, 68, 79, + 85, 66, 76, 69, 32, 246, 7, 83, 71, 86, 20, 52, 5, 65, 67, 82, 79, 78, + 145, 164, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 183, + 176, 29, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 181, 160, 16, 6, + 84, 32, 72, 65, 76, 70, 6, 166, 246, 11, 71, 186, 2, 65, 139, 157, 3, 66, + 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 177, 173, 11, 2, 79, + 84, 18, 136, 1, 17, 76, 68, 32, 80, 69, 82, 77, 73, 67, 32, 76, 69, 84, + 84, 69, 82, 32, 82, 80, 248, 243, 11, 4, 71, 79, 78, 69, 135, 237, 17, + 86, 10, 246, 43, 90, 206, 218, 17, 78, 186, 253, 8, 68, 138, 134, 2, 83, + 203, 236, 1, 65, 2, 213, 128, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, + 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 141, 248, 23, + 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 197, 171, 29, 7, 85, 83, 32, + 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 133, + 254, 23, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, + 8, 2, 83, 79, 32, 40, 3, 71, 72, 84, 157, 5, 2, 78, 71, 26, 50, 32, 149, + 4, 7, 87, 65, 82, 68, 83, 32, 72, 24, 104, 5, 65, 82, 82, 79, 87, 218, 1, + 72, 124, 12, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 32, 59, 84, 12, + 44, 5, 72, 69, 65, 68, 32, 155, 168, 29, 32, 8, 26, 65, 155, 168, 29, 66, + 6, 36, 3, 78, 68, 32, 219, 195, 30, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, + 85, 80, 2, 157, 203, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, + 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 163, 166, 29, 71, 2, + 17, 2, 79, 79, 2, 231, 193, 30, 78, 4, 174, 198, 9, 65, 137, 170, 20, 5, + 66, 69, 76, 79, 87, 2, 149, 249, 23, 2, 65, 67, 2, 233, 201, 20, 16, 65, + 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 11, 32, 6, + 182, 197, 11, 79, 166, 223, 17, 66, 223, 155, 1, 65, 18, 188, 1, 5, 72, + 79, 82, 84, 32, 160, 1, 7, 81, 85, 65, 82, 69, 32, 66, 56, 5, 84, 82, 79, + 78, 71, 232, 193, 3, 2, 85, 83, 148, 179, 20, 2, 78, 65, 233, 157, 5, 6, + 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 223, 194, 11, 84, + 2, 253, 194, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, + 65, 76, 32, 76, 73, 2, 151, 194, 11, 78, 4, 148, 196, 7, 5, 82, 65, 67, + 75, 69, 191, 221, 21, 69, 2, 225, 244, 23, 17, 32, 67, 69, 78, 84, 82, + 65, 76, 73, 90, 65, 84, 73, 79, 78, 32, 83, 20, 98, 72, 32, 4, 73, 76, + 68, 69, 136, 1, 6, 82, 73, 80, 76, 69, 32, 69, 5, 85, 82, 78, 69, 68, 2, + 137, 194, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 182, 191, 11, 79, 167, 223, 17, 66, 2, 169, 144, 29, 6, 84, + 32, 72, 65, 76, 70, 6, 186, 161, 15, 65, 184, 132, 15, 5, 85, 78, 68, 69, + 82, 199, 64, 68, 2, 145, 252, 14, 2, 32, 67, 10, 34, 80, 238, 184, 30, + 82, 3, 83, 6, 42, 87, 141, 240, 23, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, + 68, 83, 32, 65, 82, 82, 79, 2, 159, 142, 29, 87, 6, 158, 188, 25, 76, + 171, 183, 2, 84, 6, 52, 3, 68, 69, 32, 133, 189, 11, 4, 71, 71, 76, 89, + 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 161, 165, 28, + 5, 66, 82, 73, 68, 71, 2, 241, 238, 23, 2, 68, 71, 6, 178, 196, 11, 45, + 147, 214, 17, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 191, 178, 31, 65, + 4, 206, 152, 28, 77, 195, 253, 2, 65, 8, 178, 140, 8, 82, 144, 221, 15, + 3, 79, 83, 73, 246, 225, 2, 76, 171, 208, 3, 65, 38, 214, 1, 70, 84, 10, + 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 172, 236, 2, 8, 86, 69, + 78, 73, 69, 78, 67, 69, 172, 237, 17, 6, 73, 67, 65, 76, 32, 84, 170, + 243, 8, 74, 133, 97, 7, 71, 82, 85, 69, 78, 84, 32, 6, 168, 187, 26, 4, + 69, 84, 84, 73, 162, 214, 1, 85, 229, 141, 2, 4, 79, 85, 78, 68, 4, 240, + 144, 20, 2, 87, 79, 219, 151, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, + 1, 79, 28, 4, 82, 79, 76, 32, 199, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, + 5, 87, 73, 84, 72, 32, 6, 242, 205, 23, 77, 253, 7, 15, 78, 79, 82, 77, + 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 162, 190, 4, 76, 218, 144, + 19, 86, 223, 196, 4, 79, 2, 193, 168, 29, 2, 85, 82, 4, 232, 176, 19, 3, + 75, 78, 79, 173, 242, 3, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, + 175, 158, 2, 69, 4, 250, 170, 31, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, + 32, 175, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, + 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 129, + 133, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, + 9, 79, 77, 66, 73, 78, 73, 78, 71, 32, 161, 3, 5, 65, 80, 73, 84, 65, 6, + 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 189, 171, 30, 2, 78, 73, 4, + 196, 135, 6, 2, 76, 69, 161, 198, 16, 2, 65, 83, 56, 184, 211, 19, 8, 84, + 72, 79, 85, 83, 65, 78, 68, 166, 146, 1, 78, 195, 190, 2, 68, 4, 230, + 241, 17, 82, 203, 234, 10, 85, 8, 58, 68, 0, 3, 73, 78, 68, 146, 15, 86, + 183, 204, 28, 70, 2, 145, 220, 28, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, + 56, 3, 77, 65, 76, 197, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, + 32, 76, 69, 84, 84, 69, 82, 32, 120, 138, 2, 65, 48, 6, 66, 79, 72, 65, + 73, 82, 32, 2, 67, 82, 170, 1, 68, 160, 1, 2, 71, 65, 34, 72, 38, 75, 46, + 70, 34, 76, 66, 79, 138, 3, 83, 90, 84, 46, 90, 146, 197, 5, 86, 210, 55, + 80, 206, 203, 3, 73, 138, 179, 21, 82, 198, 3, 69, 218, 7, 77, 2, 78, + 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 143, 134, 19, 76, 2, 177, 1, + 4, 73, 67, 32, 75, 10, 92, 12, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, + 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 170, + 129, 31, 69, 219, 7, 78, 2, 215, 144, 24, 72, 12, 80, 9, 73, 65, 76, 69, + 67, 84, 45, 80, 32, 204, 251, 30, 2, 65, 76, 175, 17, 69, 8, 222, 137, 8, + 72, 162, 245, 7, 65, 240, 252, 11, 2, 75, 65, 139, 145, 3, 78, 4, 190, 3, + 78, 183, 251, 30, 77, 4, 250, 150, 24, 79, 191, 196, 5, 65, 8, 42, 72, + 182, 250, 27, 65, 139, 145, 3, 83, 4, 186, 139, 31, 69, 219, 19, 73, 4, + 224, 131, 19, 7, 45, 83, 72, 65, 80, 69, 68, 207, 212, 7, 65, 35, 36, 3, + 76, 68, 32, 167, 249, 30, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 193, + 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 188, 1, 2, 83, 72, + 238, 169, 14, 68, 190, 224, 9, 79, 178, 171, 5, 69, 227, 101, 65, 2, 17, + 2, 65, 78, 2, 171, 199, 15, 71, 8, 198, 147, 24, 79, 246, 211, 1, 65, + 159, 161, 5, 69, 8, 50, 78, 176, 253, 15, 2, 83, 72, 163, 139, 5, 87, 4, + 238, 135, 31, 71, 3, 89, 10, 54, 72, 178, 137, 6, 65, 222, 236, 24, 79, + 219, 3, 73, 4, 226, 249, 30, 73, 187, 13, 69, 4, 196, 237, 30, 3, 72, 69, + 84, 167, 8, 65, 2, 219, 245, 30, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, + 253, 146, 27, 3, 84, 65, 85, 4, 26, 72, 227, 133, 31, 65, 2, 143, 147, + 27, 73, 4, 136, 251, 15, 6, 72, 73, 77, 65, 32, 83, 145, 246, 3, 3, 84, + 65, 85, 4, 220, 252, 23, 3, 76, 69, 70, 157, 133, 2, 4, 82, 73, 71, 72, + 6, 96, 6, 78, 73, 83, 72, 32, 86, 252, 240, 24, 8, 82, 69, 83, 80, 79, + 78, 68, 83, 163, 157, 5, 65, 2, 157, 246, 27, 4, 69, 82, 83, 69, 44, 104, + 2, 78, 84, 208, 132, 3, 6, 67, 72, 32, 65, 78, 68, 181, 138, 27, 8, 80, + 76, 69, 32, 87, 73, 84, 72, 40, 56, 2, 69, 82, 33, 8, 73, 78, 71, 32, 82, + 79, 68, 32, 4, 158, 221, 26, 66, 235, 121, 83, 36, 48, 4, 84, 69, 78, 83, + 1, 4, 85, 78, 73, 84, 18, 185, 145, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, + 67, 75, 69, 84, 34, 79, 254, 5, 85, 50, 89, 239, 210, 29, 65, 4, 220, + 148, 8, 3, 68, 73, 84, 153, 238, 19, 7, 83, 67, 69, 78, 84, 32, 77, 5, + 237, 158, 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 228, 244, 28, 3, 67, 79, + 68, 188, 148, 1, 3, 73, 83, 83, 243, 57, 87, 22, 46, 32, 240, 2, 3, 69, + 68, 32, 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 151, 208, 30, 77, 4, + 236, 202, 27, 6, 74, 69, 82, 85, 83, 65, 245, 173, 2, 5, 76, 79, 82, 82, + 65, 8, 76, 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 97, 5, 79, 77, 77, + 69, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 241, 192, 28, 7, + 84, 32, 67, 82, 79, 83, 83, 5, 233, 161, 3, 11, 32, 87, 73, 84, 72, 32, + 72, 65, 76, 70, 45, 6, 196, 181, 24, 37, 78, 69, 71, 65, 84, 73, 86, 69, + 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, + 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 160, 212, 2, 2, 70, 76, 137, + 169, 2, 3, 83, 87, 79, 2, 141, 160, 23, 5, 78, 71, 32, 76, 65, 4, 228, + 145, 17, 3, 90, 69, 73, 179, 217, 13, 84, 6, 194, 159, 19, 73, 137, 249, + 6, 4, 83, 84, 65, 76, 204, 19, 154, 1, 66, 20, 8, 78, 69, 73, 70, 79, 82, + 77, 32, 138, 250, 1, 80, 114, 82, 172, 3, 2, 83, 84, 254, 165, 15, 67, + 209, 194, 9, 6, 84, 32, 79, 70, 32, 77, 2, 207, 160, 5, 69, 168, 19, 176, + 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, 184, 20, 17, + 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 233, + 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, 3, 2, 78, 73, + 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, 84, 32, 153, + 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 214, 113, 85, 238, 50, + 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 85, + 83, 83, 85, 186, 223, 25, 68, 171, 217, 1, 65, 8, 136, 197, 20, 5, 79, + 78, 69, 32, 84, 250, 214, 8, 70, 175, 14, 84, 58, 48, 4, 73, 86, 69, 32, + 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, 82, 71, 250, + 109, 85, 175, 202, 26, 68, 6, 182, 15, 72, 165, 212, 18, 5, 73, 88, 84, + 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, + 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, 226, 109, + 85, 175, 202, 26, 68, 6, 188, 123, 3, 65, 78, 50, 139, 252, 25, 85, 9, + 194, 156, 12, 32, 195, 231, 18, 52, 24, 44, 4, 71, 73, 68, 65, 33, 3, 78, + 69, 32, 4, 146, 158, 29, 69, 227, 101, 77, 20, 156, 1, 19, 86, 65, 82, + 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, 7, + 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, 9, 182, + 255, 30, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, 89, + 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 170, 134, 18, + 83, 203, 250, 10, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, 82, + 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 224, 244, 23, 2, 83, + 72, 241, 136, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 189, 164, 1, + 5, 73, 71, 72, 84, 72, 4, 202, 153, 29, 65, 219, 108, 71, 38, 144, 1, 5, + 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, 83, + 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, 1, + 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, 78, + 218, 1, 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, + 6, 218, 148, 12, 32, 195, 231, 18, 51, 4, 178, 189, 27, 68, 139, 191, 2, + 77, 16, 142, 1, 83, 142, 3, 65, 202, 110, 85, 238, 50, 71, 220, 130, 26, + 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, 231, + 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, 87, 79, + 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, 73, 65, + 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 167, 184, + 27, 68, 6, 248, 111, 3, 85, 82, 85, 199, 10, 65, 8, 210, 111, 50, 3, 85, + 4, 250, 144, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, 22, + 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 209, 184, 1, 2, 83, 72, 4, + 238, 120, 65, 199, 241, 25, 85, 2, 239, 147, 24, 72, 4, 53, 3, 69, 83, + 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 174, 246, 30, 50, 3, 85, 4, 11, 32, + 4, 128, 226, 26, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 195, + 85, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 188, 241, 24, + 6, 86, 69, 82, 84, 73, 67, 169, 225, 2, 14, 79, 76, 68, 32, 65, 83, 83, + 89, 82, 73, 65, 78, 32, 87, 6, 192, 190, 27, 2, 84, 82, 204, 229, 2, 4, + 81, 85, 65, 68, 15, 67, 192, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, + 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 138, 20, + 76, 134, 37, 77, 134, 6, 78, 226, 17, 80, 210, 3, 82, 42, 83, 238, 19, + 84, 186, 8, 85, 207, 20, 90, 141, 1, 160, 1, 7, 32, 84, 73, 77, 69, 83, + 32, 142, 1, 66, 250, 3, 68, 38, 75, 90, 76, 212, 1, 3, 77, 65, 82, 78, + 78, 134, 2, 82, 42, 83, 210, 230, 29, 80, 191, 127, 50, 16, 128, 173, 1, + 5, 76, 65, 71, 65, 82, 226, 23, 71, 166, 187, 12, 73, 198, 144, 13, 77, + 142, 250, 1, 83, 226, 159, 1, 66, 234, 67, 72, 187, 2, 65, 45, 22, 32, + 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 251, 202, 1, 71, 26, 180, + 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 240, 112, 8, 85, 32, 80, 76, 85, 83, + 32, 85, 188, 56, 4, 68, 85, 78, 51, 166, 2, 65, 204, 2, 3, 78, 85, 78, + 242, 27, 76, 191, 160, 29, 72, 4, 178, 210, 1, 78, 151, 155, 29, 76, 4, + 194, 169, 1, 71, 167, 196, 28, 77, 4, 170, 137, 24, 85, 187, 254, 4, 69, + 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 190, 133, 1, 77, 130, 59, 71, + 194, 9, 83, 234, 10, 84, 192, 177, 27, 2, 66, 65, 143, 230, 1, 65, 5, + 249, 51, 5, 32, 84, 73, 77, 69, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, + 252, 27, 5, 83, 72, 73, 84, 65, 199, 80, 69, 23, 68, 7, 32, 84, 73, 77, + 69, 83, 32, 202, 240, 24, 69, 247, 169, 5, 65, 16, 102, 75, 140, 173, 1, + 2, 68, 73, 218, 253, 25, 71, 170, 217, 1, 85, 130, 92, 65, 146, 92, 83, + 215, 42, 72, 4, 242, 157, 1, 65, 183, 203, 29, 73, 7, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 4, 130, 248, 3, 75, 207, 195, 26, 83, 13, 26, 32, 147, + 187, 30, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 148, + 164, 8, 7, 84, 72, 82, 69, 69, 32, 84, 245, 220, 5, 4, 79, 86, 69, 82, 4, + 128, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, + 85, 83, 163, 226, 22, 83, 6, 200, 49, 2, 65, 68, 175, 219, 17, 75, 18, + 26, 72, 239, 245, 11, 65, 17, 42, 32, 138, 140, 18, 71, 191, 217, 12, 50, + 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 158, 97, 90, 135, 112, 75, + 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 141, 143, 1, 29, 84, 85, + 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 32, 84, 85, 71, 50, 32, + 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 133, 145, 1, 27, 32, 67, 82, 79, + 83, 83, 73, 78, 71, 32, 65, 83, 72, 32, 79, 86, 69, 82, 32, 65, 83, 72, + 32, 79, 86, 69, 82, 52, 30, 65, 158, 2, 73, 95, 85, 27, 66, 68, 44, 4, + 72, 65, 82, 50, 90, 76, 66, 82, 183, 140, 27, 71, 5, 129, 208, 1, 6, 32, + 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 222, 164, 1, + 65, 218, 168, 29, 78, 163, 17, 90, 7, 180, 215, 29, 7, 32, 79, 86, 69, + 82, 32, 66, 215, 136, 1, 65, 5, 203, 252, 23, 65, 9, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 6, 246, 155, 1, 73, 198, 161, 28, 71, 171, 162, 1, 65, + 19, 50, 32, 168, 1, 3, 76, 85, 71, 223, 169, 1, 82, 8, 88, 8, 79, 86, 69, + 82, 32, 66, 85, 32, 217, 222, 26, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, + 144, 152, 12, 7, 84, 73, 77, 69, 83, 32, 78, 234, 139, 17, 65, 211, 106, + 85, 5, 237, 236, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, + 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 163, 166, + 30, 77, 55, 26, 32, 171, 220, 30, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, + 32, 84, 73, 77, 69, 83, 32, 211, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, + 71, 112, 2, 73, 82, 42, 76, 94, 85, 166, 131, 1, 80, 162, 61, 84, 234, + 232, 26, 75, 146, 152, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 175, + 183, 29, 77, 4, 154, 244, 28, 65, 143, 230, 1, 73, 10, 34, 65, 58, 73, + 151, 147, 30, 85, 5, 11, 32, 2, 169, 244, 28, 6, 80, 76, 85, 83, 32, 77, + 5, 211, 245, 23, 82, 5, 181, 203, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, + 211, 216, 30, 65, 7, 144, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 191, + 193, 29, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 203, 214, 30, 83, 4, + 128, 27, 2, 71, 73, 175, 215, 28, 77, 7, 171, 147, 12, 65, 25, 54, 77, + 150, 1, 78, 76, 2, 83, 72, 135, 213, 30, 66, 13, 44, 7, 32, 84, 73, 77, + 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 134, 160, 13, 73, 199, 195, + 16, 83, 5, 153, 228, 12, 6, 32, 84, 73, 77, 69, 83, 5, 173, 179, 18, 14, + 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 229, 157, 1, + 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 190, 234, + 23, 82, 215, 227, 6, 72, 6, 138, 176, 1, 71, 170, 3, 83, 249, 144, 12, 4, + 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 159, 211, 30, 50, + 4, 234, 145, 1, 69, 167, 148, 29, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, + 32, 175, 140, 30, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, + 77, 50, 83, 242, 79, 69, 218, 58, 73, 250, 139, 16, 72, 158, 164, 13, 78, + 3, 80, 6, 32, 2, 83, 72, 139, 164, 30, 78, 5, 143, 154, 14, 32, 4, 182, + 129, 30, 73, 3, 85, 8, 26, 73, 215, 208, 30, 65, 7, 252, 140, 1, 2, 82, + 50, 199, 194, 29, 83, 8, 26, 85, 195, 185, 1, 65, 6, 40, 4, 83, 72, 85, + 50, 227, 207, 30, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, + 26, 77, 197, 207, 23, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, + 6, 198, 233, 28, 65, 206, 167, 1, 69, 211, 61, 73, 4, 222, 138, 1, 73, + 131, 173, 29, 72, 11, 26, 51, 247, 205, 30, 52, 7, 143, 9, 32, 117, 130, + 1, 32, 90, 50, 210, 1, 78, 202, 1, 82, 60, 3, 83, 72, 50, 52, 3, 90, 69, + 78, 230, 164, 18, 71, 226, 163, 11, 68, 191, 127, 76, 4, 168, 25, 11, 79, + 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 237, 94, 4, 84, 73, 77, 69, 19, + 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 128, 168, 1, 10, 65, + 32, 80, 76, 85, 83, 32, 72, 65, 32, 178, 128, 28, 71, 214, 17, 80, 146, + 24, 75, 238, 100, 77, 219, 19, 85, 4, 186, 193, 29, 65, 203, 114, 72, 15, + 11, 32, 12, 96, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 36, 6, 84, 73, + 77, 69, 83, 32, 179, 208, 23, 83, 2, 189, 222, 13, 4, 83, 73, 78, 71, 6, + 188, 138, 1, 4, 71, 65, 78, 50, 147, 168, 29, 77, 6, 36, 3, 73, 78, 50, + 247, 248, 29, 69, 5, 187, 188, 29, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, + 83, 73, 78, 71, 63, 11, 32, 60, 96, 14, 83, 72, 69, 83, 72, 73, 71, 32, + 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 162, 131, 1, + 73, 130, 219, 2, 77, 210, 131, 25, 65, 128, 105, 2, 76, 65, 182, 87, 83, + 147, 17, 72, 44, 134, 1, 65, 68, 5, 68, 85, 78, 51, 32, 26, 75, 58, 76, + 62, 83, 34, 85, 206, 127, 73, 204, 31, 2, 72, 65, 226, 249, 27, 71, 163, + 100, 66, 9, 228, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 195, 238, 29, + 78, 4, 217, 32, 2, 71, 85, 6, 192, 203, 23, 5, 65, 83, 75, 65, 76, 159, + 165, 3, 85, 8, 34, 65, 242, 195, 30, 73, 3, 85, 5, 185, 85, 2, 76, 32, 4, + 130, 173, 30, 72, 215, 22, 85, 4, 182, 195, 30, 50, 3, 68, 160, 2, 42, + 65, 166, 19, 69, 122, 73, 135, 5, 85, 191, 1, 114, 50, 144, 16, 2, 66, + 65, 94, 68, 22, 76, 38, 78, 190, 140, 1, 32, 154, 6, 82, 218, 143, 27, + 83, 167, 142, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, + 32, 137, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, + 154, 1, 68, 178, 1, 69, 58, 71, 178, 1, 72, 230, 1, 73, 70, 75, 194, 1, + 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 206, 154, 1, 84, 248, 179, + 22, 3, 90, 73, 90, 159, 225, 6, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, + 32, 50, 78, 52, 2, 83, 72, 161, 228, 17, 14, 66, 50, 32, 84, 69, 78, 85, + 32, 80, 76, 85, 83, 32, 84, 6, 186, 58, 68, 182, 147, 13, 73, 155, 238, + 16, 72, 5, 169, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, + 245, 89, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, + 44, 2, 82, 32, 183, 188, 30, 68, 2, 11, 65, 2, 255, 216, 23, 82, 5, 11, + 32, 2, 229, 241, 29, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 187, 187, 30, + 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 191, 129, 1, 71, 6, 56, 8, 71, 32, + 84, 73, 77, 69, 83, 32, 255, 186, 30, 66, 4, 142, 119, 73, 151, 45, 75, + 10, 38, 78, 254, 2, 76, 203, 230, 28, 82, 5, 243, 28, 32, 18, 18, 65, 99, + 73, 11, 26, 82, 231, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, + 254, 162, 30, 78, 255, 2, 68, 9, 158, 122, 52, 193, 176, 12, 7, 82, 50, + 32, 80, 76, 85, 83, 12, 62, 65, 138, 124, 85, 173, 174, 12, 6, 73, 32, + 80, 76, 85, 83, 8, 40, 6, 32, 80, 76, 85, 83, 32, 83, 76, 4, 48, 6, 76, + 85, 32, 80, 76, 85, 219, 183, 30, 65, 2, 11, 83, 2, 199, 97, 32, 5, 201, + 169, 13, 5, 32, 80, 76, 85, 83, 4, 160, 100, 10, 83, 72, 32, 80, 76, 85, + 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 218, + 252, 23, 83, 211, 185, 6, 75, 5, 209, 77, 2, 32, 80, 4, 60, 5, 83, 72, + 85, 50, 32, 181, 127, 5, 51, 32, 80, 76, 85, 2, 189, 158, 1, 3, 80, 76, + 85, 8, 26, 65, 231, 179, 30, 85, 7, 11, 77, 5, 211, 159, 1, 32, 6, 234, + 77, 69, 206, 129, 28, 85, 159, 229, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, + 33, 6, 32, 80, 76, 85, 83, 32, 4, 142, 208, 23, 69, 255, 207, 6, 71, 5, + 11, 32, 2, 171, 101, 79, 14, 42, 72, 250, 140, 23, 65, 171, 149, 7, 85, + 8, 18, 69, 51, 73, 5, 157, 186, 29, 7, 32, 80, 76, 85, 83, 32, 84, 4, + 178, 178, 30, 68, 3, 77, 7, 11, 68, 5, 161, 162, 13, 5, 32, 80, 76, 85, + 83, 7, 11, 32, 4, 170, 216, 28, 82, 133, 172, 1, 11, 67, 82, 79, 83, 83, + 73, 78, 71, 32, 71, 65, 5, 175, 131, 1, 32, 7, 246, 130, 1, 32, 167, 157, + 29, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, + 0, 4, 79, 86, 69, 82, 255, 221, 25, 84, 2, 181, 49, 3, 32, 71, 65, 8, 44, + 5, 83, 72, 84, 73, 78, 199, 203, 23, 50, 7, 37, 7, 32, 84, 73, 77, 69, + 83, 32, 4, 170, 182, 29, 75, 199, 120, 85, 49, 70, 32, 94, 52, 114, 82, + 190, 1, 83, 202, 168, 29, 68, 187, 130, 1, 71, 6, 172, 232, 8, 6, 84, 73, + 77, 69, 83, 32, 253, 212, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, + 32, 4, 64, 8, 67, 82, 79, 83, 83, 73, 78, 71, 1, 4, 79, 86, 69, 82, 2, + 225, 199, 23, 3, 32, 71, 73, 16, 26, 51, 131, 136, 1, 50, 13, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 190, 127, 71, 166, 187, + 12, 73, 155, 238, 16, 80, 2, 221, 186, 13, 7, 32, 80, 76, 85, 83, 32, 73, + 14, 26, 72, 203, 161, 29, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, + 5, 73, 77, 69, 83, 32, 159, 133, 30, 69, 6, 192, 20, 6, 71, 73, 83, 72, + 32, 67, 130, 126, 84, 175, 209, 28, 66, 43, 110, 50, 174, 1, 68, 218, 1, + 77, 54, 82, 176, 183, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, + 237, 16, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 151, 132, 1, + 71, 10, 252, 24, 3, 75, 65, 75, 178, 75, 73, 152, 20, 10, 83, 65, 76, 32, + 80, 76, 85, 83, 32, 84, 191, 175, 28, 78, 11, 11, 32, 8, 128, 1, 10, 80, + 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 161, + 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 247, 120, 84, 4, 156, 145, 1, + 5, 65, 32, 80, 76, 85, 175, 156, 28, 75, 5, 17, 2, 32, 84, 2, 225, 42, 4, + 73, 77, 69, 83, 9, 26, 85, 139, 165, 30, 55, 4, 246, 163, 30, 83, 147, 1, + 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 175, 164, 30, 76, 6, 32, + 2, 84, 69, 131, 128, 1, 71, 4, 219, 127, 78, 23, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 20, 104, 3, 65, 83, 72, 162, 228, 26, 68, 174, 190, 2, 78, + 94, 75, 158, 57, 66, 2, 71, 150, 25, 83, 143, 45, 85, 7, 208, 55, 8, 32, + 79, 86, 69, 82, 32, 72, 73, 147, 235, 29, 50, 19, 48, 2, 66, 50, 158, + 190, 23, 76, 199, 226, 6, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 238, 138, 1, 75, 150, 216, 25, 76, 158, 181, 2, 72, 214, 57, 65, 195, 9, + 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 238, 154, 30, 32, + 170, 1, 83, 146, 1, 66, 2, 78, 3, 82, 7, 53, 11, 32, 79, 86, 69, 82, 32, + 73, 68, 73, 77, 32, 4, 226, 166, 23, 83, 179, 128, 6, 66, 13, 11, 73, 11, + 11, 32, 8, 146, 123, 71, 168, 178, 11, 31, 79, 86, 69, 82, 32, 73, 71, + 73, 32, 83, 72, 73, 82, 32, 79, 86, 69, 82, 32, 83, 72, 73, 82, 32, 85, + 68, 32, 79, 86, 69, 82, 178, 183, 16, 68, 219, 166, 1, 82, 7, 26, 32, + 199, 157, 30, 50, 2, 153, 72, 4, 84, 73, 77, 69, 13, 26, 32, 183, 205, + 29, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 146, 111, 84, + 183, 180, 22, 83, 2, 253, 153, 29, 5, 83, 73, 78, 71, 32, 222, 1, 78, 65, + 130, 15, 73, 242, 1, 85, 158, 73, 69, 237, 217, 22, 4, 87, 85, 51, 49, + 175, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, + 75, 50, 76, 82, 77, 28, 4, 83, 75, 65, 76, 152, 140, 15, 6, 80, 32, 69, + 76, 65, 77, 235, 129, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, + 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 198, 8, + 72, 246, 51, 78, 154, 157, 16, 80, 142, 147, 13, 82, 147, 17, 90, 13, 46, + 68, 154, 235, 29, 78, 165, 44, 2, 83, 72, 5, 213, 50, 7, 32, 80, 76, 85, + 83, 32, 75, 10, 34, 65, 226, 151, 30, 73, 3, 85, 6, 222, 177, 28, 76, + 130, 230, 1, 68, 3, 82, 4, 250, 24, 82, 151, 61, 83, 26, 30, 65, 98, 73, + 147, 1, 85, 11, 38, 82, 190, 123, 78, 151, 155, 29, 76, 5, 233, 21, 10, + 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 163, 178, 23, + 82, 7, 11, 32, 4, 26, 67, 239, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, + 78, 71, 2, 221, 214, 26, 2, 32, 71, 7, 242, 193, 26, 82, 151, 211, 3, 68, + 4, 138, 129, 30, 71, 219, 19, 77, 8, 234, 8, 73, 239, 232, 16, 65, 6, + 246, 238, 11, 85, 175, 165, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, + 76, 85, 83, 32, 6, 222, 238, 29, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, + 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 251, 155, 26, 85, 18, 62, 72, 246, + 174, 26, 65, 224, 234, 2, 2, 85, 72, 243, 119, 73, 10, 250, 149, 29, 85, + 138, 54, 73, 150, 70, 65, 3, 69, 4, 146, 217, 29, 65, 211, 56, 85, 17, + 110, 32, 222, 92, 82, 156, 182, 25, 9, 77, 85, 77, 32, 84, 73, 77, 69, + 83, 206, 252, 3, 83, 146, 1, 50, 3, 68, 2, 207, 190, 20, 85, 5, 177, 202, + 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 214, 143, + 30, 50, 2, 51, 3, 52, 5, 249, 169, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, + 68, 5, 173, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 7, 11, 32, 4, 166, + 105, 84, 233, 235, 22, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 4, 154, + 142, 30, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, 82, 32, 75, + 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, 32, 84, 73, + 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 66, 32, 21, + 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 198, 139, 30, 68, 3, 78, 6, + 26, 85, 187, 197, 29, 66, 5, 215, 139, 30, 68, 8, 52, 3, 73, 77, 53, 242, + 129, 29, 65, 159, 137, 1, 72, 5, 177, 165, 23, 11, 32, 79, 86, 69, 82, + 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, 32, 72, 73, + 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, 79, 86, 69, + 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 162, 164, 29, 51, 2, + 55, 3, 78, 2, 155, 71, 73, 5, 157, 138, 29, 8, 32, 86, 65, 82, 73, 65, + 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, 71, 240, 2, 30, + 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, 2, 75, 45, 222, + 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, 109, 11, 32, 106, + 48, 6, 84, 73, 77, 69, 83, 32, 207, 141, 23, 83, 104, 190, 1, 65, 186, 1, + 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, 46, 83, 138, 2, 84, + 126, 85, 200, 150, 4, 8, 90, 85, 32, 79, 86, 69, 82, 32, 138, 196, 24, + 68, 214, 82, 69, 131, 57, 78, 15, 80, 6, 32, 80, 76, 85, 83, 32, 76, 4, + 83, 72, 32, 90, 226, 131, 30, 76, 3, 78, 6, 38, 68, 222, 225, 28, 71, + 131, 25, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, 112, 2, 73, 68, + 4, 178, 189, 29, 65, 151, 70, 73, 10, 48, 2, 85, 68, 210, 159, 26, 65, + 167, 227, 3, 73, 5, 191, 105, 32, 6, 244, 177, 17, 7, 73, 32, 84, 73, 77, + 69, 83, 211, 212, 11, 65, 8, 22, 77, 175, 62, 71, 7, 33, 6, 32, 80, 76, + 85, 83, 32, 4, 254, 220, 29, 76, 179, 34, 72, 10, 26, 85, 175, 137, 28, + 73, 6, 26, 76, 147, 129, 30, 51, 5, 41, 8, 32, 80, 76, 85, 83, 32, 72, + 73, 2, 219, 65, 32, 8, 198, 100, 65, 242, 182, 27, 73, 131, 105, 85, 6, + 26, 69, 243, 154, 28, 85, 5, 175, 25, 32, 12, 26, 72, 139, 239, 29, 85, + 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, 72, 32, 96, + 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, 69, 223, 172, + 25, 84, 2, 11, 82, 2, 11, 73, 2, 215, 154, 23, 78, 5, 189, 73, 5, 32, 80, + 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, 83, 32, 65, 32, + 80, 76, 85, 83, 32, 83, 85, 4, 162, 152, 23, 75, 251, 228, 6, 71, 13, 72, + 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 226, 209, 29, 83, 147, 1, 68, 4, + 26, 85, 147, 252, 29, 65, 2, 255, 40, 32, 11, 11, 32, 8, 68, 4, 71, 85, + 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, 73, 16, 32, 79, 86, + 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, 2, 199, 205, 29, 32, + 5, 11, 32, 2, 229, 143, 14, 4, 80, 76, 85, 83, 136, 1, 82, 48, 146, 2, + 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, 243, 24, 53, 22, + 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, 82, 32, 76, 65, + 75, 45, 48, 55, 57, 242, 199, 10, 53, 146, 152, 12, 54, 2, 57, 94, 51, + 143, 143, 3, 48, 4, 162, 248, 29, 49, 3, 53, 4, 144, 148, 23, 12, 49, 32, + 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 247, 227, 6, 48, 4, 222, 147, 23, + 52, 95, 51, 16, 46, 50, 38, 54, 230, 140, 23, 49, 159, 2, 51, 6, 230, + 246, 29, 48, 2, 53, 3, 56, 4, 194, 246, 29, 53, 3, 54, 12, 42, 52, 250, + 177, 11, 56, 143, 225, 11, 57, 6, 250, 245, 29, 51, 2, 55, 3, 56, 30, 62, + 52, 214, 1, 53, 30, 57, 178, 144, 23, 55, 143, 143, 3, 56, 14, 26, 57, + 255, 244, 29, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, 116, 9, 80, + 65, 80, 32, 80, 76, 85, 83, 32, 208, 227, 12, 7, 85, 50, 32, 80, 76, 85, + 83, 190, 31, 73, 235, 203, 16, 71, 4, 210, 13, 80, 51, 76, 4, 194, 243, + 29, 48, 3, 55, 8, 166, 243, 29, 48, 2, 50, 2, 51, 3, 53, 46, 60, 2, 49, + 55, 240, 1, 2, 52, 56, 138, 137, 23, 48, 23, 51, 23, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, 68, 85, 78, + 51, 32, 71, 85, 78, 85, 186, 222, 27, 65, 254, 158, 1, 66, 235, 67, 76, + 4, 198, 184, 29, 65, 211, 56, 69, 6, 250, 146, 14, 82, 254, 221, 15, 50, + 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, 85, 220, 8, 4, + 80, 65, 80, 32, 146, 45, 73, 164, 167, 12, 10, 83, 72, 69, 83, 72, 32, + 80, 76, 85, 83, 250, 252, 4, 68, 170, 129, 12, 78, 163, 17, 71, 4, 138, + 145, 14, 82, 255, 221, 15, 68, 4, 234, 137, 23, 50, 211, 145, 3, 52, 5, + 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, 77, 69, 83, + 32, 75, 85, 82, 5, 221, 160, 11, 5, 32, 80, 76, 85, 83, 9, 200, 160, 11, + 2, 77, 77, 222, 203, 18, 83, 147, 1, 76, 93, 90, 50, 212, 7, 3, 71, 65, + 76, 142, 1, 77, 130, 62, 32, 198, 165, 29, 51, 2, 72, 3, 76, 69, 11, 32, + 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, 71, 85, 78, 85, + 38, 83, 35, 84, 2, 205, 158, 11, 6, 83, 73, 78, 71, 32, 76, 2, 193, 24, + 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 139, 173, 22, 81, 54, 44, 5, 73, + 77, 69, 83, 32, 171, 197, 29, 69, 52, 188, 1, 4, 69, 83, 72, 50, 94, 72, + 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, 83, 90, 84, 250, + 56, 71, 158, 230, 7, 78, 150, 135, 18, 68, 170, 181, 2, 65, 254, 66, 66, + 203, 53, 73, 7, 11, 32, 4, 26, 80, 207, 150, 25, 84, 2, 17, 2, 76, 85, 2, + 245, 222, 28, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, 171, 156, 28, 65, 8, + 32, 2, 65, 68, 183, 231, 29, 73, 6, 226, 19, 51, 211, 211, 29, 50, 4, + 174, 20, 32, 171, 249, 16, 71, 2, 11, 69, 2, 11, 32, 2, 209, 251, 12, 4, + 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, 32, 80, 2, 45, 9, + 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 235, 145, 26, 85, 6, 26, 73, 195, + 192, 29, 72, 4, 194, 18, 32, 157, 211, 25, 7, 75, 50, 32, 80, 76, 85, 83, + 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, 207, 67, 83, 4, 56, 7, + 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, 21, 3, 32, 76, 85, 2, + 175, 218, 28, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, 76, 85, 77, 5, 187, + 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, 45, 32, 188, 1, 2, + 83, 72, 250, 179, 29, 50, 2, 72, 3, 82, 7, 187, 238, 25, 83, 7, 240, 142, + 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 151, 211, 3, 78, 43, 104, 2, 83, + 72, 186, 60, 71, 140, 215, 10, 5, 32, 79, 86, 69, 82, 40, 2, 82, 71, 237, + 183, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, 1, 9, 79, 86, 69, 82, + 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, 173, 254, 25, 10, 67, + 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 6, 42, 65, 254, 175, 27, 75, 175, 172, 2, 71, 2, 237, 142, 11, 5, 32, 80, + 76, 85, 83, 6, 242, 229, 28, 75, 142, 118, 90, 187, 2, 65, 13, 11, 32, + 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, 8, 38, 65, 210, 201, 29, + 68, 163, 17, 90, 5, 173, 205, 12, 5, 32, 80, 76, 85, 83, 158, 1, 42, 65, + 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, 65, 166, 1, 77, 246, 218, + 29, 50, 3, 52, 11, 26, 32, 255, 219, 29, 82, 6, 100, 8, 79, 80, 80, 79, + 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, 77, 69, 83, 32, 83, 72, + 85, 2, 217, 152, 29, 3, 32, 78, 65, 7, 204, 21, 2, 32, 78, 167, 197, 29, + 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, 179, 57, 83, 4, 250, + 147, 29, 85, 151, 70, 65, 73, 90, 77, 78, 78, 204, 234, 19, 6, 32, 84, + 73, 77, 69, 83, 254, 135, 8, 83, 131, 230, 1, 50, 7, 45, 9, 32, 84, 73, + 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, 59, 36, 3, 68, 65, + 50, 163, 216, 29, 57, 55, 37, 7, 32, 84, 73, 77, 69, 83, 32, 52, 162, 1, + 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, 77, 86, 78, 34, 80, + 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, 174, 186, 28, 66, 211, + 117, 72, 6, 246, 2, 83, 223, 211, 29, 78, 8, 26, 73, 255, 211, 28, 85, 5, + 135, 213, 29, 83, 2, 211, 20, 69, 2, 251, 242, 22, 53, 4, 26, 69, 171, + 240, 27, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, 41, 3, 83, 32, 71, 4, 182, + 133, 29, 85, 215, 79, 69, 2, 33, 6, 65, 80, 32, 80, 76, 85, 2, 11, 83, 2, + 229, 195, 28, 2, 32, 80, 9, 37, 7, 32, 80, 76, 85, 83, 32, 65, 6, 26, 83, + 171, 140, 28, 32, 4, 11, 72, 5, 107, 32, 11, 50, 32, 34, 50, 246, 244, + 13, 82, 239, 220, 15, 83, 2, 233, 217, 13, 3, 80, 76, 85, 2, 11, 32, 2, + 21, 3, 80, 76, 85, 2, 11, 83, 2, 151, 237, 27, 32, 57, 24, 2, 49, 49, 99, + 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, 82, 32, 78, 85, 49, 49, 210, + 230, 24, 84, 191, 248, 2, 82, 47, 30, 32, 169, 3, 2, 85, 90, 18, 144, 1, + 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, 78, 85, 78, 72, 12, 76, 65, 71, + 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, 1, 79, 163, 252, 24, 84, 5, 209, + 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, 86, 69, 82, 32, 76, 65, 10, 56, + 3, 83, 65, 76, 210, 233, 27, 77, 14, 85, 247, 66, 71, 5, 205, 197, 28, + 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, 76, 65, 71, 65, 82, 32, 84, + 73, 77, 69, 83, 32, 83, 2, 153, 253, 16, 3, 86, 69, 82, 27, 11, 32, 24, + 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, 32, 205, 37, 15, 75, 73, 83, + 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, 73, 20, 168, 1, 2, 75, 65, + 202, 7, 73, 212, 39, 2, 65, 83, 194, 171, 2, 68, 164, 166, 8, 3, 83, 73, + 76, 226, 230, 11, 85, 150, 185, 5, 71, 238, 147, 1, 78, 254, 2, 66, 163, + 17, 76, 2, 199, 247, 25, 68, 46, 42, 65, 36, 4, 69, 83, 72, 50, 23, 73, + 9, 242, 202, 29, 68, 2, 78, 3, 80, 5, 135, 186, 27, 32, 35, 22, 32, 151, + 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, 205, 199, 17, 8, 67, 82, 79, + 83, 83, 73, 78, 71, 18, 214, 21, 85, 146, 48, 65, 2, 73, 190, 195, 28, + 66, 175, 64, 69, 12, 32, 2, 73, 71, 239, 200, 29, 50, 11, 11, 32, 8, 96, + 6, 84, 73, 77, 69, 83, 32, 173, 193, 25, 12, 79, 80, 80, 79, 83, 73, 78, + 71, 32, 80, 73, 82, 6, 210, 190, 28, 75, 150, 67, 85, 223, 67, 90, 8, + 230, 67, 65, 234, 131, 29, 73, 3, 85, 202, 1, 46, 65, 250, 5, 72, 150, + 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, 122, 78, 147, 193, 29, 82, + 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, 32, 6, 84, 73, 77, 69, 83, + 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, 84, 73, 77, 69, 2, 167, + 170, 17, 83, 2, 177, 203, 24, 3, 85, 84, 73, 42, 150, 1, 73, 42, 75, 34, + 83, 64, 2, 84, 65, 38, 85, 216, 62, 2, 68, 85, 246, 219, 27, 76, 222, 39, + 78, 198, 48, 69, 254, 59, 77, 162, 17, 72, 187, 2, 65, 2, 11, 71, 2, 11, + 73, 2, 191, 31, 32, 4, 242, 138, 29, 85, 187, 53, 65, 6, 26, 72, 239, + 185, 28, 65, 4, 234, 139, 13, 69, 139, 241, 15, 73, 4, 234, 221, 22, 75, + 251, 228, 6, 66, 10, 174, 193, 29, 83, 146, 1, 50, 2, 66, 2, 77, 3, 82, + 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 2, 11, 65, 2, 11, 83, 2, 207, 221, 22, 72, 2, 131, 251, 10, 71, 106, 46, + 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, 50, 51, 182, 1, 54, 182, 215, + 22, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 90, 85, + 146, 25, 83, 194, 231, 25, 71, 130, 198, 2, 84, 166, 50, 66, 206, 47, 78, + 215, 22, 65, 5, 11, 32, 2, 129, 171, 25, 4, 80, 76, 85, 83, 5, 175, 45, + 32, 27, 62, 32, 140, 2, 2, 83, 72, 226, 232, 25, 71, 171, 211, 3, 78, 14, + 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, 5, 80, 76, 85, 83, 32, 255, + 151, 29, 72, 7, 11, 32, 4, 190, 15, 71, 141, 6, 12, 84, 65, 66, 32, 79, + 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, 20, 2, 78, 65, 247, 153, + 28, 83, 2, 219, 216, 22, 66, 2, 199, 216, 22, 77, 7, 202, 130, 28, 76, + 191, 185, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, 182, 178, 25, 78, 155, + 132, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 162, 170, 29, 73, + 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, 22, 114, 66, 38, 73, + 130, 19, 75, 198, 181, 2, 77, 190, 198, 2, 76, 246, 135, 23, 71, 130, 25, + 83, 238, 9, 68, 191, 127, 65, 4, 218, 200, 2, 85, 203, 231, 25, 65, 4, + 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, 82, 32, 83, 72, 73, + 82, 211, 230, 24, 84, 2, 241, 191, 28, 11, 32, 66, 85, 82, 32, 79, 86, + 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, 73, 78, 86, 69, 82, + 84, 69, 68, 34, 50, 187, 190, 28, 66, 2, 11, 32, 2, 167, 245, 24, 83, 7, + 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, 84, 73, 77, 69, 173, + 234, 10, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, 235, 207, 12, 83, 17, + 50, 32, 30, 71, 230, 234, 10, 76, 227, 230, 11, 75, 4, 138, 8, 84, 163, + 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, 83, 73, 71, 52, 2, + 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 252, 164, 12, 5, 32, 79, 86, 69, + 82, 147, 150, 16, 72, 5, 247, 179, 29, 50, 5, 199, 206, 27, 65, 5, 207, + 179, 29, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, 227, 8, 69, 37, 66, 32, + 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 183, 175, 29, 82, 8, 60, 6, 84, + 73, 77, 69, 83, 32, 130, 14, 71, 207, 147, 27, 65, 4, 174, 158, 29, 72, + 3, 77, 7, 11, 32, 4, 188, 143, 17, 29, 79, 86, 69, 82, 32, 84, 65, 66, + 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, 73, 83, 72, 32, 79, + 86, 69, 82, 175, 169, 5, 83, 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, + 74, 84, 252, 233, 7, 2, 83, 72, 154, 155, 20, 71, 174, 100, 85, 191, 50, + 66, 2, 11, 85, 2, 243, 203, 22, 71, 5, 29, 5, 32, 80, 76, 85, 83, 2, 145, + 201, 27, 2, 32, 83, 17, 46, 82, 150, 29, 32, 182, 145, 29, 50, 3, 76, 9, + 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, 99, 84, 5, 11, 32, 2, + 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, 69, 82, 32, 71, 65, + 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, 77, 114, 82, 138, 200, + 22, 71, 215, 227, 6, 75, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 46, 71, + 213, 137, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, 2, 65, 78, 5, 137, 221, + 10, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, 90, 65, 32, 79, 86, 69, + 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, 46, 77, 158, 2, 82, 128, + 9, 2, 83, 72, 174, 1, 90, 192, 133, 12, 2, 84, 85, 238, 145, 17, 50, 3, + 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, 2, 85, 215, 252, 27, 71, + 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, 80, 65, 168, 247, 8, 19, + 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, 86, 69, 82, 32, 85, 32, + 82, 249, 183, 19, 10, 83, 85, 82, 32, 79, 86, 69, 82, 32, 83, 2, 11, 32, + 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, 255, 132, 28, 71, 5, + 171, 130, 29, 32, 21, 26, 32, 147, 166, 29, 85, 16, 70, 75, 44, 2, 83, + 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, 85, 2, 11, 83, 2, + 151, 217, 10, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, 32, 2, 11, 84, 2, + 149, 223, 28, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, 32, 80, 76, 85, + 83, 32, 85, 32, 80, 76, 85, 83, 32, 142, 222, 28, 66, 203, 50, 77, 4, 11, + 85, 5, 11, 32, 2, 11, 71, 2, 147, 255, 28, 85, 21, 72, 7, 32, 84, 73, 77, + 69, 83, 32, 136, 1, 2, 85, 77, 231, 162, 28, 66, 10, 50, 76, 16, 2, 77, + 69, 50, 83, 183, 162, 29, 85, 2, 231, 6, 65, 5, 11, 32, 2, 177, 153, 25, + 4, 80, 76, 85, 83, 2, 255, 215, 10, 72, 7, 37, 7, 32, 84, 73, 77, 69, 83, + 32, 4, 158, 11, 75, 163, 148, 29, 80, 93, 54, 32, 102, 50, 194, 2, 73, + 22, 85, 235, 157, 29, 52, 4, 62, 83, 157, 168, 28, 9, 67, 82, 79, 83, 83, + 73, 78, 71, 32, 2, 141, 154, 25, 4, 72, 69, 83, 72, 23, 11, 32, 20, 42, + 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, 168, 23, 4, 78, 86, 69, 82, + 18, 46, 65, 82, 85, 202, 158, 28, 78, 227, 125, 72, 6, 48, 6, 32, 80, 76, + 85, 83, 32, 255, 158, 29, 76, 4, 194, 156, 29, 72, 3, 78, 8, 26, 50, 199, + 158, 29, 68, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 130, 185, 27, 65, 199, + 209, 1, 66, 5, 251, 157, 29, 51, 59, 56, 7, 32, 84, 73, 77, 69, 83, 32, + 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, 38, 71, 82, 73, 46, 76, 78, + 83, 46, 85, 186, 247, 27, 66, 238, 34, 77, 214, 90, 84, 146, 17, 75, 162, + 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, 207, 194, 16, 71, 4, 186, 208, + 10, 65, 235, 251, 17, 85, 10, 26, 65, 187, 155, 29, 85, 9, 34, 78, 150, + 155, 29, 76, 3, 82, 2, 211, 9, 50, 6, 170, 135, 29, 71, 202, 18, 83, 147, + 1, 77, 6, 46, 85, 229, 178, 22, 5, 65, 75, 45, 54, 54, 4, 166, 154, 29, + 51, 3, 77, 4, 144, 181, 22, 2, 73, 71, 167, 206, 6, 72, 6, 42, 32, 186, + 187, 13, 82, 255, 221, 15, 68, 2, 241, 238, 27, 6, 80, 76, 85, 83, 32, + 71, 5, 11, 32, 2, 213, 159, 13, 4, 84, 73, 77, 69, 17, 84, 7, 32, 84, 73, + 77, 69, 83, 32, 244, 139, 28, 2, 85, 77, 142, 140, 1, 50, 3, 88, 8, 50, + 84, 168, 180, 25, 2, 75, 85, 167, 227, 3, 65, 2, 11, 65, 2, 199, 178, 22, + 75, 6, 26, 51, 147, 151, 29, 85, 5, 29, 5, 32, 84, 73, 77, 69, 2, 21, 3, + 83, 32, 75, 2, 11, 65, 2, 251, 220, 22, 83, 42, 50, 65, 190, 1, 73, 146, + 1, 85, 231, 175, 22, 69, 13, 50, 32, 198, 137, 28, 77, 142, 140, 1, 55, + 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, 68, 32, 175, 195, 24, 84, 2, 11, + 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, 157, 156, 28, 2, 32, 75, 15, 86, + 66, 132, 152, 22, 6, 32, 79, 86, 69, 82, 32, 154, 24, 90, 214, 227, 6, + 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, 66, 2, 135, 2, 65, 15, 84, 10, + 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, 42, 53, 246, 153, 28, 66, 199, + 120, 77, 2, 181, 240, 27, 5, 76, 85, 83, 32, 83, 7, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, 69, 227, 145, 29, 65, 2, 29, 5, + 32, 68, 73, 83, 72, 2, 11, 32, 2, 203, 191, 24, 84, 8, 42, 32, 194, 150, + 22, 73, 187, 180, 3, 67, 4, 202, 212, 21, 79, 141, 190, 6, 8, 87, 73, 84, + 72, 32, 83, 84, 82, 18, 134, 1, 76, 154, 1, 82, 189, 202, 28, 23, 86, 69, + 68, 32, 83, 84, 69, 77, 32, 80, 65, 82, 65, 71, 82, 65, 80, 72, 32, 83, + 73, 71, 78, 10, 48, 2, 89, 32, 253, 147, 3, 4, 73, 78, 71, 32, 8, 60, 2, + 76, 79, 177, 128, 6, 7, 66, 82, 65, 67, 75, 69, 84, 6, 210, 226, 25, 71, + 215, 171, 3, 79, 6, 52, 5, 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, + 248, 158, 24, 4, 69, 88, 67, 72, 143, 232, 3, 83, 2, 133, 160, 19, 3, 68, + 32, 82, 4, 174, 246, 27, 65, 133, 89, 2, 79, 77, 236, 8, 128, 1, 2, 80, + 82, 140, 9, 7, 82, 73, 76, 76, 73, 67, 32, 244, 134, 25, 7, 76, 73, 78, + 68, 82, 73, 67, 173, 210, 2, 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, + 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, + 79, 65, 78, 32, 83, 73, 71, 78, 32, 67, 77, 110, 254, 156, 7, 75, 2, 76, + 2, 77, 2, 78, 2, 80, 2, 82, 2, 83, 2, 84, 126, 87, 254, 41, 74, 2, 90, + 194, 206, 6, 88, 138, 244, 14, 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, + 46, 48, 146, 5, 49, 141, 212, 23, 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, + 50, 78, 51, 78, 52, 62, 53, 86, 55, 190, 140, 10, 57, 190, 3, 54, 179, + 234, 13, 56, 16, 246, 135, 29, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, + 56, 3, 57, 16, 194, 3, 50, 234, 131, 29, 48, 2, 49, 2, 51, 2, 53, 2, 55, + 3, 57, 16, 226, 134, 29, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, + 57, 16, 150, 134, 29, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, + 57, 12, 202, 133, 29, 48, 2, 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 142, + 133, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, + 82, 53, 234, 131, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, + 57, 5, 231, 131, 29, 66, 24, 18, 48, 87, 49, 18, 190, 131, 29, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 6, 234, 130, 29, 48, 2, + 50, 3, 52, 180, 6, 140, 1, 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 194, 4, + 75, 32, 7, 76, 69, 84, 84, 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, + 203, 41, 84, 240, 2, 44, 6, 69, 84, 84, 69, 82, 32, 159, 43, 73, 234, 2, + 150, 2, 76, 46, 78, 34, 80, 34, 82, 58, 84, 54, 85, 190, 5, 65, 214, 1, + 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, + 73, 230, 3, 75, 226, 3, 77, 218, 1, 79, 226, 2, 83, 210, 7, 89, 166, 1, + 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 214, 25, + 73, 186, 206, 28, 74, 159, 20, 72, 4, 254, 26, 69, 231, 204, 28, 74, 8, + 246, 33, 69, 203, 200, 28, 83, 12, 212, 9, 3, 79, 85, 78, 226, 18, 69, + 139, 223, 28, 72, 18, 158, 33, 69, 106, 83, 174, 174, 28, 67, 187, 22, + 87, 13, 190, 34, 32, 115, 75, 2, 137, 244, 28, 3, 65, 86, 89, 6, 248, 10, + 5, 77, 85, 76, 84, 73, 156, 205, 12, 2, 80, 65, 229, 195, 11, 14, 83, 77, + 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 2, 237, 209, 28, 2, 69, + 82, 184, 3, 136, 1, 6, 77, 65, 76, 76, 32, 76, 193, 37, 22, 85, 66, 83, + 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 132, 3, 44, 6, 69, 84, 84, 69, 82, 32, 143, 36, 73, 254, 2, 154, 2, + 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, + 72, 134, 1, 73, 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, + 110, 80, 50, 82, 198, 1, 83, 218, 2, 84, 210, 2, 85, 246, 1, 87, 54, 89, + 166, 1, 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, + 6, 32, 87, 73, 84, 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, + 229, 228, 11, 4, 76, 69, 85, 84, 4, 138, 154, 9, 68, 151, 170, 3, 66, 8, + 128, 209, 12, 3, 67, 72, 69, 250, 189, 6, 68, 139, 229, 9, 72, 18, 110, + 65, 78, 73, 32, 3, 82, 79, 65, 128, 146, 3, 6, 76, 69, 78, 68, 69, 68, + 178, 187, 9, 89, 243, 166, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, + 233, 205, 11, 5, 83, 72, 75, 73, 82, 4, 230, 2, 78, 187, 143, 3, 71, 2, + 179, 133, 10, 68, 14, 76, 2, 72, 69, 246, 9, 76, 164, 222, 12, 4, 82, 79, + 83, 83, 155, 222, 15, 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 142, 29, + 68, 195, 165, 28, 86, 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, + 66, 90, 182, 196, 28, 67, 186, 22, 87, 215, 22, 69, 5, 239, 151, 22, 82, + 4, 36, 3, 77, 79, 78, 219, 241, 28, 79, 2, 153, 13, 2, 79, 67, 12, 46, + 69, 138, 241, 27, 90, 182, 105, 72, 3, 87, 5, 155, 210, 28, 76, 41, 86, + 76, 98, 78, 110, 82, 166, 15, 32, 254, 185, 12, 83, 250, 247, 13, 77, + 195, 173, 2, 70, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 158, 20, 77, 222, + 182, 12, 68, 134, 248, 13, 84, 163, 92, 72, 13, 33, 6, 32, 87, 73, 84, + 72, 32, 10, 190, 19, 77, 222, 182, 12, 68, 202, 52, 76, 190, 195, 13, 84, + 163, 92, 72, 5, 229, 191, 27, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, + 187, 215, 28, 74, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 134, 18, 77, + 146, 15, 85, 206, 167, 12, 68, 143, 31, 83, 12, 26, 65, 195, 214, 28, 87, + 11, 48, 6, 32, 87, 73, 84, 72, 32, 143, 135, 27, 82, 6, 218, 199, 12, 68, + 166, 212, 14, 72, 231, 160, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, + 69, 74, 79, 201, 1, 6, 90, 72, 73, 84, 83, 65, 6, 254, 142, 9, 68, 214, + 10, 71, 239, 184, 3, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, 152, + 9, 71, 195, 159, 3, 66, 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, + 159, 234, 28, 65, 12, 80, 2, 67, 76, 38, 76, 214, 134, 3, 66, 130, 207, + 24, 89, 210, 147, 1, 65, 3, 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, + 4, 73, 5, 145, 212, 12, 14, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, + 69, 32, 71, 34, 102, 65, 98, 79, 244, 185, 28, 11, 72, 65, 75, 65, 83, + 83, 73, 65, 78, 32, 67, 186, 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, + 84, 72, 32, 8, 182, 194, 12, 68, 166, 212, 14, 72, 154, 160, 1, 86, 79, + 83, 18, 36, 3, 77, 73, 32, 251, 193, 25, 80, 16, 58, 68, 198, 236, 11, + 76, 2, 78, 2, 83, 2, 84, 3, 90, 6, 194, 236, 11, 90, 134, 227, 16, 74, + 215, 22, 69, 8, 94, 73, 236, 154, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, + 69, 68, 206, 179, 18, 74, 159, 20, 72, 2, 233, 130, 3, 4, 84, 84, 76, 69, + 4, 21, 3, 79, 78, 79, 4, 18, 67, 39, 71, 2, 205, 188, 18, 4, 85, 76, 65, + 82, 2, 237, 10, 4, 82, 65, 80, 72, 6, 62, 69, 204, 187, 18, 5, 65, 82, + 82, 79, 87, 155, 145, 10, 74, 2, 249, 192, 12, 5, 85, 84, 82, 65, 76, 11, + 52, 4, 77, 69, 71, 65, 166, 3, 32, 183, 223, 28, 84, 5, 233, 185, 28, 8, + 32, 87, 73, 84, 72, 32, 84, 73, 10, 130, 6, 69, 230, 183, 12, 65, 231, + 144, 16, 83, 14, 50, 69, 100, 4, 79, 85, 78, 68, 167, 222, 28, 72, 8, 37, + 7, 86, 69, 82, 83, 69, 68, 32, 8, 214, 249, 18, 68, 206, 222, 8, 84, 142, + 100, 89, 151, 14, 90, 4, 250, 241, 9, 32, 241, 237, 1, 2, 69, 68, 34, + 108, 4, 67, 72, 87, 65, 34, 72, 132, 1, 4, 79, 70, 84, 32, 170, 186, 12, + 84, 213, 1, 5, 69, 77, 73, 83, 79, 5, 11, 32, 2, 183, 218, 5, 87, 16, 92, + 4, 79, 82, 84, 32, 136, 185, 12, 2, 72, 65, 254, 173, 14, 67, 234, 224, + 1, 87, 215, 22, 65, 6, 254, 176, 26, 73, 195, 173, 2, 85, 8, 38, 69, 166, + 215, 27, 83, 255, 111, 68, 4, 242, 221, 28, 76, 3, 77, 26, 132, 1, 4, 65, + 76, 76, 32, 50, 69, 106, 83, 212, 211, 11, 11, 72, 82, 69, 69, 45, 76, + 69, 71, 71, 69, 68, 218, 218, 16, 67, 187, 22, 87, 6, 178, 179, 12, 72, + 206, 149, 15, 89, 255, 124, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, + 77, 223, 182, 12, 68, 2, 185, 169, 21, 5, 73, 68, 68, 76, 69, 8, 226, + 196, 28, 72, 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, + 66, 76, 69, 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 255, + 192, 12, 77, 4, 172, 255, 8, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, + 82, 2, 213, 4, 6, 65, 73, 78, 73, 65, 78, 2, 159, 144, 24, 32, 4, 176, + 197, 27, 4, 73, 68, 69, 32, 227, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, + 85, 134, 216, 28, 73, 2, 78, 3, 85, 7, 158, 216, 28, 69, 3, 84, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 138, 251, 8, 68, 131, 186, 3, 66, 18, 18, 69, + 71, 72, 9, 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 173, 211, 28, 2, 77, + 76, 10, 26, 69, 247, 191, 28, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, + 26, 68, 203, 163, 12, 66, 4, 222, 131, 9, 73, 179, 173, 3, 69, 6, 37, 7, + 71, 65, 84, 85, 82, 69, 32, 6, 66, 65, 176, 140, 15, 2, 84, 69, 189, 155, + 13, 4, 69, 78, 32, 71, 2, 199, 185, 28, 32, 52, 198, 1, 66, 38, 68, 38, + 69, 36, 3, 71, 72, 69, 38, 72, 246, 188, 21, 89, 222, 157, 5, 83, 202, + 110, 84, 238, 8, 90, 246, 63, 73, 138, 19, 67, 186, 22, 80, 2, 86, 158, + 20, 75, 186, 2, 65, 2, 79, 3, 85, 4, 170, 172, 12, 89, 243, 166, 16, 69, + 6, 238, 210, 27, 90, 139, 128, 1, 69, 6, 210, 210, 28, 70, 2, 76, 3, 83, + 5, 201, 5, 5, 32, 87, 73, 84, 72, 4, 11, 65, 5, 171, 236, 26, 82, 2, 209, + 188, 15, 4, 72, 79, 85, 83, 184, 15, 178, 1, 65, 138, 4, 67, 54, 69, 198, + 35, 73, 250, 22, 79, 142, 44, 82, 174, 3, 85, 180, 180, 12, 13, 78, 65, + 32, 68, 79, 85, 66, 76, 69, 32, 72, 69, 76, 154, 231, 14, 86, 195, 47, + 76, 30, 116, 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, + 83, 72, 200, 177, 21, 2, 76, 69, 155, 133, 1, 84, 9, 11, 32, 6, 44, 5, + 87, 73, 84, 72, 32, 143, 190, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, + 71, 72, 2, 201, 183, 27, 4, 84, 32, 71, 85, 4, 166, 144, 28, 67, 239, 30, + 71, 4, 254, 253, 20, 85, 175, 224, 4, 72, 10, 30, 32, 105, 3, 69, 68, 32, + 4, 60, 9, 87, 73, 84, 72, 32, 76, 69, 70, 84, 239, 185, 27, 83, 2, 17, 2, + 32, 85, 2, 175, 181, 22, 80, 6, 170, 215, 4, 84, 182, 147, 12, 76, 255, + 252, 9, 79, 10, 134, 204, 28, 49, 2, 50, 2, 51, 2, 52, 3, 83, 176, 4, + 182, 2, 67, 132, 2, 5, 71, 82, 69, 69, 32, 98, 76, 80, 21, 78, 84, 73, + 83, 84, 82, 89, 32, 83, 89, 77, 66, 79, 76, 32, 76, 73, 71, 72, 84, 32, + 160, 3, 8, 80, 65, 82, 84, 77, 69, 78, 84, 34, 83, 174, 6, 86, 188, 214, + 14, 11, 82, 69, 76, 73, 67, 84, 32, 72, 79, 85, 83, 132, 146, 7, 2, 65, + 70, 235, 154, 6, 69, 8, 50, 73, 217, 229, 5, 6, 82, 69, 65, 83, 69, 32, + 6, 48, 4, 68, 85, 79, 85, 21, 4, 77, 65, 76, 32, 2, 219, 178, 23, 83, 4, + 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, 82, 197, 171, 21, 7, 69, 88, 80, + 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, 151, 180, 27, 89, 6, 200, 242, + 21, 4, 67, 69, 76, 83, 250, 205, 5, 83, 153, 106, 8, 70, 65, 72, 82, 69, + 78, 72, 69, 9, 176, 190, 16, 5, 73, 86, 69, 82, 89, 128, 225, 5, 2, 84, + 65, 215, 226, 4, 69, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, + 79, 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, + 234, 219, 26, 87, 170, 25, 67, 191, 43, 84, 14, 52, 4, 65, 78, 68, 32, + 85, 5, 87, 73, 84, 72, 32, 10, 200, 158, 17, 2, 84, 79, 190, 188, 9, 87, + 165, 189, 1, 6, 66, 79, 84, 84, 79, 77, 4, 218, 243, 26, 67, 191, 43, 84, + 2, 245, 137, 24, 3, 32, 83, 84, 170, 1, 64, 2, 67, 69, 48, 2, 69, 82, + 129, 5, 5, 75, 84, 79, 80, 32, 2, 217, 180, 26, 7, 78, 68, 73, 78, 71, + 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, 160, 1, 56, 6, 67, 65, 80, + 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, 71, 0, 5, 83, 72, 79, 82, 84, + 74, 79, 30, 84, 2, 90, 246, 178, 11, 67, 206, 157, 15, 66, 2, 68, 2, 74, + 2, 80, 2, 86, 2, 89, 202, 96, 71, 2, 75, 134, 103, 87, 162, 19, 65, 203, + 17, 72, 20, 214, 202, 24, 83, 206, 251, 1, 78, 134, 247, 1, 84, 146, 1, + 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, 32, 12, 234, 201, 24, 65, 226, + 180, 1, 79, 130, 191, 2, 69, 3, 73, 4, 142, 189, 28, 73, 3, 87, 4, 190, + 208, 26, 72, 227, 213, 1, 69, 5, 225, 174, 27, 4, 32, 73, 83, 76, 4, 186, + 153, 16, 67, 229, 136, 11, 4, 87, 73, 78, 68, 202, 2, 100, 8, 65, 78, 65, + 71, 65, 82, 73, 32, 185, 18, 12, 73, 67, 69, 32, 67, 79, 78, 84, 82, 79, + 76, 32, 192, 2, 222, 1, 65, 38, 67, 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, + 76, 69, 84, 84, 69, 82, 32, 198, 5, 83, 176, 7, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 158, 181, 18, 68, 148, 245, 5, 4, 74, 65, 73, 78, + 243, 239, 3, 79, 4, 138, 164, 12, 67, 191, 170, 12, 66, 2, 187, 253, 5, + 65, 4, 250, 163, 12, 82, 215, 236, 1, 65, 6, 44, 5, 32, 77, 65, 82, 75, + 163, 136, 28, 83, 5, 209, 195, 18, 7, 32, 87, 73, 84, 72, 32, 72, 158, 1, + 158, 2, 65, 54, 67, 62, 68, 50, 71, 62, 72, 52, 2, 77, 65, 46, 83, 154, + 10, 79, 218, 254, 6, 66, 82, 74, 218, 220, 4, 75, 238, 194, 3, 82, 166, + 212, 6, 89, 170, 109, 85, 238, 30, 78, 230, 46, 76, 238, 146, 1, 84, 46, + 86, 138, 225, 1, 73, 154, 191, 1, 80, 2, 90, 254, 68, 70, 2, 81, 187, 2, + 69, 13, 198, 181, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 10, 26, 65, 195, + 178, 28, 72, 9, 185, 2, 4, 78, 68, 82, 65, 12, 170, 204, 24, 68, 246, + 229, 3, 72, 187, 2, 65, 10, 182, 185, 12, 76, 190, 179, 15, 72, 254, 68, + 71, 187, 2, 65, 4, 232, 150, 16, 4, 69, 65, 86, 89, 131, 157, 12, 65, 5, + 229, 133, 7, 6, 82, 87, 65, 82, 73, 32, 12, 38, 72, 174, 176, 28, 83, + 187, 2, 65, 8, 36, 3, 79, 82, 84, 191, 178, 28, 65, 6, 163, 212, 11, 32, + 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, + 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, 32, 141, 128, 26, 10, 84, 82, 69, + 83, 83, 32, 83, 73, 71, 78, 16, 226, 202, 3, 71, 2, 75, 220, 238, 22, 3, + 68, 68, 68, 2, 82, 226, 244, 1, 89, 38, 70, 2, 81, 3, 90, 48, 210, 3, 66, + 0, 10, 69, 88, 84, 69, 78, 68, 69, 68, 32, 66, 36, 11, 67, 65, 78, 68, + 82, 65, 66, 73, 78, 68, 85, 60, 8, 87, 69, 83, 84, 69, 82, 78, 32, 32, + 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 148, 231, 4, 5, 80, 85, 83, + 72, 80, 182, 204, 1, 83, 156, 243, 3, 18, 68, 79, 85, 66, 76, 69, 32, 67, + 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 182, 148, 8, 73, 186, 137, 6, 65, + 126, 77, 46, 78, 204, 212, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, 67, 73, + 78, 71, 255, 77, 86, 4, 133, 170, 12, 4, 72, 65, 76, 69, 11, 11, 32, 8, + 206, 241, 21, 65, 210, 89, 86, 151, 244, 3, 84, 4, 30, 78, 21, 3, 70, 73, + 86, 2, 17, 2, 73, 78, 2, 217, 144, 21, 8, 69, 45, 76, 73, 75, 69, 32, 66, + 50, 146, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, 190, 186, 18, + 80, 254, 165, 4, 85, 138, 228, 1, 83, 70, 86, 166, 221, 1, 73, 207, 134, + 2, 69, 10, 226, 169, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 6, 176, 187, 18, + 4, 76, 79, 78, 71, 254, 237, 9, 69, 3, 79, 7, 158, 146, 28, 79, 215, 22, + 69, 10, 222, 193, 23, 70, 136, 6, 2, 83, 84, 146, 244, 2, 84, 203, 83, + 79, 228, 2, 182, 2, 65, 134, 2, 69, 74, 71, 224, 4, 6, 78, 71, 66, 65, + 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, 178, 2, 86, 200, 7, 2, 89, + 65, 32, 4, 90, 90, 89, 32, 192, 152, 5, 2, 84, 84, 216, 229, 14, 10, 70, + 70, 69, 82, 69, 78, 67, 69, 32, 66, 229, 147, 7, 12, 77, 69, 78, 83, 73, + 79, 78, 32, 79, 82, 73, 71, 18, 26, 77, 167, 211, 8, 69, 16, 40, 4, 79, + 78, 68, 32, 195, 138, 3, 69, 14, 128, 1, 5, 87, 73, 84, 72, 32, 218, 220, + 17, 84, 128, 221, 2, 12, 83, 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, + 159, 192, 4, 79, 8, 202, 177, 21, 66, 162, 2, 84, 250, 228, 3, 76, 27, + 82, 14, 164, 240, 4, 5, 32, 70, 65, 67, 69, 225, 229, 16, 4, 83, 69, 76, + 32, 78, 64, 3, 73, 84, 32, 149, 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, + 98, 70, 44, 2, 78, 73, 2, 79, 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, + 69, 73, 71, 72, 84, 12, 128, 1, 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, + 12, 96, 4, 69, 86, 69, 78, 1, 2, 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, + 6, 23, 69, 6, 11, 79, 7, 207, 186, 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, + 64, 5, 71, 82, 69, 65, 84, 0, 4, 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, + 89, 32, 72, 4, 238, 213, 23, 85, 187, 131, 1, 69, 4, 229, 207, 12, 4, 69, + 82, 32, 89, 4, 172, 129, 5, 7, 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, + 65, 78, 64, 120, 17, 78, 69, 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, + 76, 69, 68, 32, 41, 9, 67, 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, + 162, 32, 78, 139, 249, 19, 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, + 73, 70, 32, 22, 234, 31, 78, 203, 142, 26, 68, 4, 140, 143, 22, 15, 67, + 85, 82, 82, 69, 78, 84, 32, 83, 89, 77, 66, 79, 76, 32, 235, 240, 5, 72, + 10, 70, 65, 144, 1, 5, 67, 79, 78, 84, 73, 213, 251, 24, 3, 71, 85, 73, + 6, 76, 9, 80, 80, 79, 73, 78, 84, 69, 68, 32, 217, 255, 14, 4, 66, 76, + 69, 68, 4, 232, 226, 16, 7, 66, 85, 84, 32, 82, 69, 76, 211, 250, 10, 70, + 2, 53, 11, 78, 85, 79, 85, 83, 32, 85, 78, 68, 69, 82, 2, 213, 197, 16, + 3, 76, 73, 78, 156, 1, 80, 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, + 73, 233, 190, 16, 2, 79, 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, + 69, 82, 32, 236, 1, 5, 83, 73, 71, 78, 32, 58, 86, 252, 243, 8, 6, 77, + 69, 68, 73, 65, 76, 198, 247, 4, 71, 198, 211, 4, 69, 172, 189, 4, 12, + 80, 82, 69, 70, 73, 88, 69, 68, 32, 78, 65, 83, 241, 204, 4, 7, 73, 78, + 73, 84, 73, 65, 76, 22, 246, 192, 18, 79, 175, 231, 7, 73, 84, 186, 142, + 10, 84, 242, 209, 11, 89, 186, 116, 65, 130, 217, 1, 68, 146, 25, 85, + 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, + 2, 75, 2, 80, 254, 68, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, + 3, 79, 8, 234, 128, 24, 72, 146, 45, 67, 98, 78, 223, 160, 3, 65, 18, 64, + 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 179, 225, 25, 73, 16, 178, + 219, 14, 65, 218, 232, 9, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, + 10, 68, 5, 83, 73, 79, 78, 32, 204, 234, 1, 2, 78, 71, 235, 147, 25, 68, + 6, 26, 83, 247, 206, 5, 84, 4, 206, 172, 26, 76, 231, 94, 73, 2, 173, + 184, 18, 3, 32, 76, 65, 4, 190, 254, 26, 83, 255, 85, 70, 190, 5, 188, 2, + 6, 67, 85, 77, 69, 78, 84, 124, 7, 69, 83, 32, 78, 79, 84, 32, 194, 3, + 71, 210, 3, 76, 36, 10, 77, 73, 78, 79, 32, 84, 73, 76, 69, 32, 226, 1, + 78, 38, 84, 246, 2, 85, 204, 17, 2, 87, 78, 220, 128, 16, 8, 32, 78, 79, + 84, 32, 76, 73, 84, 136, 178, 11, 8, 86, 69, 32, 79, 70, 32, 80, 69, 174, + 4, 79, 223, 25, 68, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, + 88, 84, 191, 133, 2, 80, 5, 185, 133, 2, 6, 32, 65, 78, 68, 32, 80, 22, + 164, 1, 11, 67, 79, 78, 84, 65, 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, + 86, 73, 68, 69, 112, 2, 80, 82, 48, 7, 83, 85, 67, 67, 69, 69, 68, 233, + 251, 21, 2, 70, 79, 6, 248, 1, 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, + 66, 71, 82, 79, 85, 80, 195, 170, 20, 77, 5, 209, 170, 21, 23, 32, 87, + 73, 84, 72, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 69, 71, 65, 84, + 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, 69, 243, 141, 27, 79, 5, 217, 240, + 8, 2, 32, 79, 125, 36, 3, 82, 65, 32, 235, 204, 27, 32, 120, 120, 7, 76, + 69, 84, 84, 69, 82, 32, 212, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 158, 199, 22, 65, 191, 2, 83, 88, 214, 232, 23, 82, 206, 55, 65, + 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, + 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, + 86, 2, 89, 186, 2, 69, 3, 79, 22, 162, 193, 18, 86, 166, 225, 5, 65, 190, + 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 182, 194, 20, 80, + 243, 173, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, 79, 78, 84, 1, 6, 86, + 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, 45, 48, 191, 162, 23, + 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 14, 197, 219, + 19, 2, 45, 48, 4, 254, 237, 25, 75, 223, 143, 1, 71, 26, 34, 32, 57, 4, + 84, 69, 68, 32, 8, 254, 166, 24, 80, 34, 77, 202, 50, 79, 187, 173, 2, + 65, 18, 214, 1, 67, 34, 83, 144, 213, 15, 3, 76, 73, 78, 192, 132, 1, 4, + 79, 66, 69, 76, 160, 9, 9, 84, 82, 65, 78, 83, 80, 79, 83, 73, 134, 194, + 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, 78, 84, 73, 78, + 71, 4, 238, 178, 26, 73, 167, 17, 82, 4, 182, 241, 25, 79, 223, 111, 81, + 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, 42, 32, 190, 10, + 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, 72, 32, 4, 73, + 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, 74, 80, 66, + 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 132, 140, 6, + 6, 79, 66, 76, 73, 81, 85, 172, 167, 2, 9, 82, 73, 71, 72, 84, 32, 65, + 82, 67, 194, 174, 3, 65, 186, 189, 9, 69, 190, 141, 4, 81, 213, 101, 6, + 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, 32, 252, 166, + 7, 4, 85, 82, 76, 89, 145, 191, 1, 4, 79, 76, 79, 78, 20, 26, 78, 139, + 249, 19, 68, 2, 137, 199, 2, 5, 85, 77, 66, 69, 82, 4, 186, 160, 18, 79, + 131, 164, 6, 65, 4, 186, 243, 19, 73, 215, 63, 89, 4, 230, 136, 25, 82, + 223, 233, 1, 71, 12, 54, 79, 141, 222, 16, 7, 69, 70, 84, 32, 65, 82, 67, + 10, 26, 87, 163, 207, 24, 71, 6, 26, 45, 187, 225, 26, 32, 4, 162, 242, + 19, 82, 211, 1, 57, 6, 196, 204, 16, 9, 76, 69, 83, 83, 45, 84, 72, 65, + 78, 255, 158, 9, 71, 8, 26, 82, 247, 246, 25, 76, 6, 154, 152, 1, 69, + 139, 180, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, 50, 85, 185, + 161, 13, 4, 79, 76, 73, 68, 4, 238, 221, 24, 73, 55, 85, 4, 252, 138, 12, + 3, 65, 67, 75, 187, 192, 4, 82, 8, 58, 83, 186, 150, 1, 67, 170, 182, 20, + 80, 191, 194, 4, 66, 2, 209, 142, 27, 4, 80, 69, 78, 83, 4, 210, 155, 18, + 80, 179, 139, 9, 78, 10, 36, 3, 66, 65, 82, 211, 221, 26, 76, 9, 11, 32, + 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 163, 215, 25, 76, 4, 158, 215, 25, + 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, 84, 82, 85, + 67, 75, 32, 189, 182, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, 48, 8, 83, + 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, 69, 65, 84, + 69, 82, 1, 4, 76, 69, 83, 83, 4, 177, 232, 19, 9, 81, 85, 65, 76, 32, 84, + 79, 32, 79, 2, 193, 239, 13, 5, 45, 84, 72, 65, 78, 34, 160, 1, 8, 67, + 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, 124, 6, + 83, 77, 65, 76, 76, 32, 153, 155, 13, 8, 78, 45, 65, 82, 89, 32, 83, 85, + 18, 234, 238, 11, 71, 230, 252, 13, 80, 206, 134, 2, 67, 2, 72, 2, 78, 2, + 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 185, 151, 21, 7, 67, + 65, 80, 73, 84, 65, 76, 8, 238, 240, 27, 68, 2, 69, 2, 73, 3, 74, 4, 146, + 237, 11, 71, 219, 239, 15, 80, 6, 146, 237, 9, 70, 26, 77, 255, 239, 16, + 83, 2, 179, 246, 26, 78, 144, 1, 86, 32, 156, 1, 10, 45, 80, 79, 73, 78, + 84, 73, 78, 71, 32, 65, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, + 250, 222, 24, 70, 30, 82, 251, 64, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 11, 32, 2, 11, 67, 2, 201, 169, 20, 4, 73, 82, 67, 76, 8, 178, 225, 24, + 82, 0, 7, 83, 77, 65, 76, 76, 32, 82, 35, 84, 126, 56, 8, 32, 70, 65, 67, + 73, 78, 71, 32, 89, 2, 83, 32, 8, 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, + 68, 32, 72, 4, 137, 201, 26, 3, 79, 79, 75, 118, 158, 1, 65, 202, 2, 84, + 168, 192, 8, 2, 87, 72, 186, 138, 8, 90, 198, 184, 8, 66, 154, 1, 68, 50, + 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 243, 7, 80, 34, 40, 4, + 82, 82, 79, 87, 187, 255, 24, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, + 72, 32, 208, 147, 3, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, + 32, 85, 254, 235, 21, 65, 170, 10, 70, 175, 5, 84, 22, 180, 163, 5, 6, + 67, 79, 82, 78, 69, 82, 182, 222, 19, 68, 58, 76, 42, 77, 38, 78, 58, 83, + 66, 69, 246, 12, 84, 139, 59, 72, 36, 36, 2, 82, 73, 161, 2, 2, 87, 79, + 32, 44, 5, 65, 78, 71, 76, 69, 171, 146, 25, 80, 30, 56, 8, 45, 72, 69, + 65, 68, 69, 68, 32, 215, 152, 25, 32, 28, 68, 5, 65, 82, 82, 79, 87, 134, + 201, 16, 90, 162, 199, 8, 68, 39, 80, 23, 11, 32, 20, 216, 139, 25, 15, + 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 80, 98, 84, 19, + 87, 4, 162, 145, 25, 32, 105, 15, 45, 72, 69, 65, 68, 69, 68, 32, 65, 82, + 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, 1, 12, 85, 77, 32, 87, + 73, 84, 72, 32, 68, 82, 85, 77, 152, 182, 16, 6, 73, 86, 69, 32, 83, 76, + 179, 149, 10, 69, 8, 238, 189, 2, 67, 236, 243, 5, 10, 70, 84, 73, 78, + 71, 32, 80, 79, 73, 78, 201, 192, 15, 3, 71, 79, 78, 8, 56, 6, 77, 69, + 68, 65, 82, 89, 34, 80, 251, 174, 26, 79, 2, 217, 160, 20, 3, 32, 67, 65, + 4, 240, 234, 17, 6, 32, 79, 70, 32, 66, 76, 191, 194, 9, 76, 2, 217, 165, + 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, 89, 65, 78, 32, 128, 187, + 19, 2, 77, 80, 247, 162, 8, 67, 158, 2, 212, 2, 6, 65, 70, 70, 73, 88, + 32, 152, 7, 7, 76, 69, 84, 84, 69, 82, 32, 208, 188, 1, 16, 84, 72, 73, + 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, 76, 236, 160, 3, 4, 68, + 79, 85, 66, 204, 172, 16, 19, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, + 32, 67, 72, 73, 78, 79, 79, 75, 153, 143, 5, 11, 83, 73, 71, 78, 32, 79, + 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, + 184, 1, 5, 72, 73, 71, 72, 32, 102, 76, 40, 4, 82, 73, 71, 72, 169, 2, 4, + 77, 73, 68, 32, 14, 112, 2, 84, 65, 224, 4, 13, 76, 69, 70, 84, 45, 84, + 79, 45, 82, 73, 71, 72, 84, 26, 83, 146, 165, 20, 69, 3, 73, 6, 44, 5, + 78, 71, 69, 78, 84, 227, 210, 26, 73, 5, 227, 169, 20, 32, 20, 166, 2, + 76, 50, 84, 38, 86, 238, 134, 8, 71, 186, 2, 65, 154, 231, 17, 87, 170, + 25, 67, 223, 126, 68, 24, 36, 2, 69, 70, 29, 3, 79, 87, 32, 2, 209, 2, 3, + 84, 32, 72, 22, 90, 65, 38, 76, 50, 84, 38, 86, 238, 134, 8, 71, 210, + 233, 17, 87, 170, 25, 67, 223, 126, 68, 4, 202, 150, 21, 67, 131, 169, 5, + 82, 4, 220, 216, 7, 3, 79, 78, 71, 207, 231, 18, 73, 2, 173, 254, 7, 4, + 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, 65, 76, 5, 131, 1, 32, 4, + 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, 7, 79, 82, 73, 90, 79, 78, + 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 237, 205, 26, 2, 69, 67, + 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, 71, 22, 74, 162, 1, 75, + 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, 82, 74, 83, 210, 2, 84, + 66, 85, 42, 86, 38, 87, 70, 88, 230, 206, 26, 72, 138, 60, 73, 194, 41, + 89, 215, 22, 66, 5, 203, 176, 27, 79, 7, 206, 151, 27, 32, 211, 61, 72, + 7, 254, 212, 27, 69, 3, 85, 5, 143, 133, 27, 32, 5, 207, 129, 24, 32, 19, + 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, 230, 5, 77, 2, 78, + 215, 170, 26, 83, 5, 233, 214, 26, 12, 83, 32, 73, 78, 83, 73, 68, 69, + 32, 65, 78, 68, 9, 26, 32, 131, 211, 27, 75, 4, 138, 128, 24, 82, 247, + 210, 3, 77, 9, 248, 243, 21, 3, 79, 78, 71, 227, 222, 5, 72, 11, 11, 32, + 8, 162, 4, 78, 238, 170, 26, 87, 147, 163, 1, 83, 19, 38, 32, 49, 5, 65, + 83, 65, 76, 32, 8, 202, 3, 77, 238, 170, 26, 87, 147, 163, 1, 83, 8, 150, + 209, 27, 65, 2, 73, 2, 79, 3, 85, 11, 218, 207, 27, 79, 146, 1, 65, 2, + 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 179, 128, 27, 32, 4, + 130, 208, 27, 77, 3, 78, 11, 248, 240, 21, 6, 79, 77, 65, 78, 73, 65, + 158, 161, 5, 32, 211, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, 32, + 195, 158, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, 84, + 72, 32, 68, 79, 214, 205, 27, 77, 2, 78, 3, 83, 5, 171, 144, 27, 32, 5, + 151, 149, 27, 32, 4, 183, 165, 12, 84, 18, 74, 69, 138, 176, 5, 79, 154, + 205, 21, 65, 198, 78, 68, 146, 1, 74, 3, 85, 6, 242, 204, 27, 69, 2, 72, + 3, 78, 9, 26, 32, 183, 204, 27, 72, 4, 190, 249, 23, 82, 247, 210, 3, 83, + 9, 186, 252, 26, 32, 214, 79, 72, 3, 73, 5, 217, 237, 6, 4, 79, 67, 65, + 76, 17, 66, 79, 178, 146, 27, 32, 250, 36, 69, 218, 19, 65, 2, 72, 3, 73, + 5, 255, 202, 27, 87, 250, 28, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, 82, + 73, 65, 76, 32, 67, 22, 71, 240, 75, 4, 73, 71, 72, 84, 198, 1, 76, 194, + 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, 83, + 118, 84, 242, 31, 85, 198, 1, 88, 252, 3, 2, 89, 69, 152, 72, 4, 45, 77, + 65, 73, 132, 207, 18, 3, 74, 69, 67, 168, 134, 2, 8, 86, 69, 82, 71, 82, + 69, 69, 78, 159, 147, 5, 80, 22, 38, 82, 130, 230, 25, 83, 179, 64, 71, + 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 221, 210, 4, 13, 87, + 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 216, 138, 11, 2, 77, + 65, 227, 206, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, + 239, 165, 24, 82, 6, 70, 65, 149, 207, 13, 11, 69, 85, 82, 79, 80, 69, + 45, 65, 70, 82, 73, 4, 128, 214, 9, 4, 77, 69, 82, 73, 133, 154, 2, 11, + 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 191, 163, 2, 79, 174, 17, + 96, 18, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, + 72, 32, 199, 195, 27, 71, 172, 17, 146, 3, 65, 178, 4, 66, 52, 2, 67, 48, + 224, 1, 2, 68, 48, 170, 4, 69, 198, 3, 70, 164, 4, 2, 71, 48, 210, 3, 72, + 222, 1, 73, 238, 2, 76, 122, 77, 134, 9, 78, 210, 5, 79, 140, 5, 2, 80, + 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, 190, 3, 84, 220, 2, 2, 85, 48, + 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, 48, 88, 3, 89, 48, 48, 84, 2, + 90, 48, 224, 196, 3, 3, 75, 48, 48, 177, 218, 14, 3, 81, 48, 48, 228, 1, + 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, 48, 98, 49, 102, 52, 234, 55, 51, + 138, 163, 20, 55, 230, 216, 1, 50, 2, 53, 3, 54, 24, 238, 68, 54, 146, + 185, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, + 24, 158, 253, 21, 52, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, + 2, 54, 2, 56, 3, 57, 28, 186, 252, 21, 48, 2, 50, 2, 51, 2, 53, 174, 193, + 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 68, 46, 48, 186, 55, 51, 138, + 251, 21, 49, 3, 50, 22, 150, 66, 55, 190, 250, 26, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, 148, 9, 4, 69, 71, 73, 78, 253, 25, + 2, 48, 48, 56, 34, 48, 90, 49, 207, 199, 8, 50, 24, 170, 40, 50, 250, + 146, 27, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 158, + 249, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, 102, 52, 102, 53, 106, 54, 134, 30, + 50, 251, 141, 22, 49, 20, 238, 247, 21, 56, 174, 193, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 24, 146, 247, 21, 49, 2, 52, 174, + 193, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 174, + 246, 21, 54, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 55, 3, 57, 42, 154, 61, 48, 178, 184, 21, 50, 2, 52, 174, 193, 5, 49, + 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, 134, 61, 55, 138, 249, 26, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 94, 30, 48, 189, 2, 2, 78, + 68, 88, 38, 48, 94, 50, 102, 51, 235, 7, 49, 22, 202, 243, 21, 56, 2, 57, + 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 24, 238, 242, + 21, 48, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 3, 57, 18, 138, 242, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 54, + 2, 55, 3, 56, 6, 11, 32, 6, 68, 2, 83, 69, 136, 160, 5, 6, 87, 65, 76, + 76, 69, 68, 191, 19, 69, 2, 231, 238, 26, 71, 132, 1, 42, 48, 133, 9, 5, + 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, 102, 53, + 215, 1, 50, 20, 226, 239, 21, 49, 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 22, 134, 239, 21, 51, 174, 193, 5, 48, 2, 49, + 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 162, 238, 21, 49, 2, + 55, 2, 56, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 26, + 190, 237, 21, 53, 2, 54, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 56, 3, 57, 14, 142, 27, 49, 250, 146, 27, 48, 2, 50, 3, 51, 128, + 1, 62, 48, 98, 49, 102, 51, 102, 52, 130, 28, 50, 239, 155, 8, 53, 24, + 214, 50, 55, 146, 185, 21, 54, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 56, 3, 57, 22, 134, 235, 21, 49, 174, 193, 5, 48, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 162, 234, 21, 54, 2, 55, 174, + 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, 190, + 233, 21, 51, 2, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, 55, + 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 209, 40, 7, 79, + 82, 73, 90, 79, 78, 84, 18, 134, 232, 21, 54, 174, 193, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 175, 43, 76, 2, 237, 233, 23, + 2, 76, 65, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, + 18, 48, 95, 49, 22, 174, 230, 21, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 210, 229, 21, 48, 2, 49, 174, 193, 5, + 50, 2, 51, 2, 52, 3, 53, 14, 68, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, + 80, 239, 138, 19, 77, 7, 11, 32, 4, 134, 152, 26, 69, 221, 6, 2, 83, 84, + 22, 32, 2, 48, 48, 215, 162, 15, 79, 20, 238, 227, 21, 50, 2, 54, 174, + 193, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 158, 1, 48, 128, + 4, 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 189, + 239, 23, 14, 73, 82, 82, 79, 82, 32, 72, 79, 82, 73, 90, 79, 78, 84, 132, + 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, 52, 24, 182, 40, 49, 146, 185, + 21, 51, 174, 193, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 44, + 138, 41, 50, 222, 183, 21, 48, 2, 53, 2, 54, 2, 55, 174, 193, 5, 49, 2, + 51, 2, 52, 2, 56, 3, 57, 26, 254, 223, 21, 50, 2, 52, 2, 56, 174, 193, 5, + 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, 26, 138, 38, 51, 146, 185, + 21, 49, 174, 193, 5, 48, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, + 12, 178, 222, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 3, 52, 31, 25, 4, + 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, 77, 120, 5, 83, 84, 65, + 82, 84, 68, 3, 84, 79, 80, 139, 143, 26, 69, 11, 11, 32, 8, 56, 5, 83, + 84, 65, 82, 84, 186, 1, 65, 199, 142, 26, 69, 5, 129, 2, 8, 32, 65, 78, + 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, 4, 146, 212, 23, 66, + 247, 207, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, 84, 65, 82, 84, 179, + 142, 26, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, 78, 68, 32, 66, 79, 84, + 84, 79, 77, 2, 235, 135, 19, 32, 194, 1, 50, 48, 128, 2, 2, 76, 48, 229, + 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, 150, 6, 52, 155, 249, 21, + 48, 24, 146, 32, 56, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 3, 57, 28, 194, 216, 21, 51, 2, 52, 2, 53, 2, 55, 174, 193, + 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, 34, 48, 94, 49, 151, 181, + 20, 50, 20, 186, 215, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, + 54, 2, 55, 2, 56, 3, 57, 22, 222, 214, 21, 55, 174, 193, 5, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 52, 34, 49, 102, 50, 159, + 138, 22, 48, 26, 214, 213, 21, 48, 2, 49, 2, 56, 174, 193, 5, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 242, 212, 21, 50, 174, 193, 5, 48, + 3, 49, 152, 1, 50, 48, 229, 172, 18, 6, 86, 69, 82, 76, 65, 89, 150, 1, + 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, 53, 247, 134, 22, 52, 34, + 90, 54, 238, 210, 21, 49, 2, 53, 174, 193, 5, 50, 2, 51, 2, 52, 2, 55, 2, + 56, 3, 57, 15, 150, 148, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 28, + 98, 48, 206, 209, 21, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 3, 56, 9, 246, 146, 27, 65, 2, 66, 3, 67, 28, 166, 209, 21, + 48, 2, 52, 2, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, + 56, 32, 134, 23, 54, 190, 185, 21, 48, 2, 51, 174, 193, 5, 49, 2, 50, 2, + 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, 48, 191, 250, 26, 49, 26, 26, + 48, 235, 148, 8, 49, 22, 158, 207, 21, 49, 2, 51, 174, 193, 5, 50, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, 48, 98, 49, 219, 132, 22, 50, + 24, 142, 21, 51, 146, 185, 21, 50, 174, 193, 5, 49, 2, 52, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 24, 190, 205, 21, 48, 2, 54, 174, 193, 5, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 108, 50, 48, 94, 49, 106, 50, + 98, 51, 155, 226, 18, 52, 22, 166, 204, 21, 50, 2, 54, 174, 193, 5, 49, + 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 26, 186, 18, 52, 146, 185, 21, + 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 24, + 210, 17, 54, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, + 2, 56, 3, 57, 22, 130, 202, 21, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, 34, 48, 249, 12, 3, 65, 76, 76, + 88, 42, 48, 94, 49, 102, 51, 171, 254, 21, 50, 26, 206, 200, 21, 51, 2, + 55, 2, 56, 2, 57, 174, 193, 5, 49, 2, 50, 2, 52, 2, 53, 3, 54, 24, 242, + 199, 21, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, + 2, 56, 3, 57, 18, 142, 199, 21, 50, 2, 51, 174, 193, 5, 48, 2, 49, 2, 52, + 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, 51, 102, 52, 139, 251, 21, 49, 22, + 254, 12, 54, 190, 250, 26, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, + 3, 57, 24, 182, 197, 21, 51, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, + 2, 53, 2, 54, 2, 55, 3, 56, 22, 210, 196, 21, 50, 174, 193, 5, 48, 2, 49, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 154, 133, 27, 48, 2, + 49, 3, 50, 158, 1, 42, 48, 185, 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, + 98, 49, 98, 50, 210, 1, 51, 201, 191, 21, 2, 52, 48, 42, 198, 9, 55, 98, + 49, 178, 184, 21, 50, 174, 193, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, + 32, 186, 8, 49, 46, 50, 190, 250, 26, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 50, 98, 48, 182, 192, 21, 51, 2, 56, 2, 57, 174, 193, + 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 55, 27, 222, 129, 27, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, + 198, 191, 21, 48, 2, 49, 2, 51, 2, 55, 174, 193, 5, 50, 2, 52, 2, 53, 2, + 54, 2, 56, 3, 57, 2, 169, 134, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, + 68, 69, 64, 26, 48, 94, 49, 103, 50, 22, 134, 190, 21, 51, 2, 57, 174, + 193, 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 170, 189, 21, + 48, 2, 52, 2, 55, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, + 57, 14, 198, 188, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, + 17, 2, 32, 76, 2, 211, 250, 14, 79, 24, 202, 2, 52, 146, 185, 21, 54, 2, + 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 130, 187, 21, 49, + 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, + 167, 1, 49, 34, 90, 50, 46, 51, 146, 185, 21, 52, 2, 53, 174, 193, 5, 49, + 2, 54, 2, 55, 2, 56, 3, 57, 11, 230, 250, 26, 65, 2, 66, 2, 67, 3, 68, 7, + 186, 250, 26, 65, 3, 66, 48, 66, 53, 86, 54, 138, 249, 26, 48, 2, 49, 2, + 50, 2, 51, 3, 52, 21, 218, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, + 70, 2, 71, 2, 72, 3, 73, 19, 134, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, + 69, 2, 70, 2, 71, 3, 72, 14, 26, 32, 159, 171, 15, 72, 12, 42, 80, 242, + 133, 3, 84, 135, 225, 21, 83, 8, 242, 132, 3, 79, 213, 129, 16, 22, 69, + 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, 76, + 65, 67, 75, 160, 1, 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, + 69, 82, 32, 166, 3, 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 155, 238, + 26, 70, 80, 230, 1, 71, 78, 76, 34, 78, 50, 82, 158, 236, 24, 69, 198, + 134, 1, 67, 2, 68, 2, 75, 2, 83, 2, 84, 2, 90, 182, 105, 66, 2, 70, 2, + 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, + 2, 85, 3, 89, 8, 38, 72, 138, 221, 26, 74, 215, 22, 69, 4, 194, 168, 24, + 65, 155, 203, 2, 69, 4, 226, 220, 26, 76, 215, 22, 69, 8, 194, 220, 26, + 68, 2, 74, 214, 22, 65, 3, 69, 4, 146, 220, 26, 82, 215, 22, 69, 32, 96, + 5, 67, 84, 82, 73, 67, 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 246, 230, + 25, 80, 223, 80, 86, 10, 26, 32, 215, 165, 23, 65, 8, 98, 80, 244, 134, + 17, 2, 84, 79, 204, 175, 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 131, + 32, 65, 2, 11, 76, 2, 131, 240, 26, 85, 19, 11, 32, 16, 72, 5, 87, 73, + 84, 72, 32, 157, 142, 16, 7, 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, + 32, 12, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 186, 144, 19, 86, + 186, 149, 4, 85, 166, 47, 79, 131, 153, 2, 68, 2, 141, 209, 24, 3, 79, + 78, 71, 2, 233, 158, 14, 7, 65, 76, 32, 83, 84, 82, 79, 46, 170, 154, 4, + 69, 153, 147, 15, 15, 73, 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, + 45, 89, 53, 48, 4, 79, 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, + 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, + 73, 69, 82, 32, 70, 73, 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, + 80, 69, 45, 8, 140, 200, 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, + 87, 72, 73, 84, 69, 173, 190, 2, 2, 66, 65, 10, 152, 135, 20, 2, 49, 45, + 214, 227, 6, 51, 2, 52, 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 225, 229, 3, + 2, 72, 65, 24, 82, 78, 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 177, 165, + 26, 4, 68, 79, 67, 85, 8, 36, 3, 79, 84, 69, 187, 161, 23, 69, 7, 131, + 232, 12, 32, 4, 139, 229, 24, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, + 234, 203, 13, 82, 24, 3, 76, 69, 70, 176, 166, 1, 2, 83, 77, 207, 220, 8, + 79, 38, 86, 32, 64, 2, 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, + 235, 164, 15, 84, 6, 42, 81, 186, 129, 25, 68, 139, 167, 1, 83, 2, 191, + 160, 26, 85, 20, 32, 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, + 146, 149, 20, 71, 56, 2, 83, 69, 194, 59, 77, 30, 84, 139, 251, 4, 76, 4, + 210, 149, 20, 84, 239, 206, 5, 79, 2, 241, 169, 19, 7, 73, 84, 72, 32, + 76, 69, 70, 5, 167, 209, 20, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, + 76, 221, 140, 23, 4, 68, 79, 87, 78, 2, 209, 248, 21, 4, 73, 71, 72, 84, + 6, 238, 227, 26, 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, + 32, 133, 2, 2, 83, 32, 12, 56, 3, 84, 79, 32, 237, 220, 12, 5, 65, 78, + 68, 32, 80, 10, 68, 3, 79, 82, 32, 217, 144, 26, 8, 66, 89, 32, 68, 69, + 70, 73, 78, 8, 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 170, 210, 24, 71, + 39, 76, 2, 133, 230, 14, 2, 67, 69, 2, 221, 132, 25, 3, 67, 69, 69, 26, + 72, 4, 83, 73, 71, 78, 232, 207, 24, 4, 87, 73, 84, 72, 223, 192, 1, 67, + 23, 11, 32, 20, 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, + 79, 86, 69, 32, 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, + 65, 82, 65, 76, 76, 69, 76, 8, 158, 178, 20, 80, 154, 5, 84, 150, 184, 2, + 76, 231, 207, 2, 82, 5, 135, 205, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, + 20, 7, 73, 78, 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, + 83, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 155, 230, 14, 68, 2, 163, + 224, 25, 89, 2, 159, 182, 24, 89, 2, 17, 2, 32, 84, 2, 195, 181, 24, 87, + 6, 80, 7, 86, 65, 76, 69, 78, 84, 32, 185, 181, 20, 7, 65, 78, 71, 85, + 76, 65, 82, 4, 48, 6, 87, 73, 84, 72, 32, 70, 207, 188, 26, 84, 2, 11, + 79, 2, 177, 229, 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, + 82, 82, 69, 68, 32, 216, 147, 5, 7, 73, 83, 32, 70, 79, 82, 77, 205, 162, + 7, 9, 65, 83, 69, 32, 84, 79, 32, 84, 72, 12, 240, 200, 5, 4, 87, 72, 73, + 84, 13, 5, 66, 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, + 239, 216, 26, 65, 5, 143, 148, 25, 65, 4, 218, 197, 25, 68, 147, 147, 1, + 83, 154, 8, 60, 7, 72, 73, 79, 80, 73, 67, 32, 134, 216, 26, 66, 3, 88, + 150, 8, 204, 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, + 80, 54, 83, 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 174, + 183, 18, 68, 218, 184, 5, 70, 82, 81, 137, 140, 2, 4, 87, 79, 82, 68, 10, + 26, 77, 183, 134, 26, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 167, 211, + 26, 77, 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, + 44, 5, 65, 78, 68, 32, 86, 207, 147, 26, 77, 2, 129, 213, 21, 4, 79, 87, + 69, 76, 22, 66, 84, 226, 171, 21, 72, 210, 188, 3, 69, 30, 70, 42, 78, + 39, 83, 8, 130, 134, 16, 69, 222, 227, 8, 72, 27, 87, 4, 242, 103, 65, + 201, 155, 25, 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, + 65, 66, 76, 69, 32, 4, 154, 157, 23, 77, 187, 204, 2, 67, 194, 7, 210, 1, + 66, 90, 67, 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, + 77, 90, 78, 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, + 2, 76, 138, 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, + 230, 8, 66, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, + 78, 94, 67, 254, 15, 72, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, + 61, 87, 186, 2, 73, 3, 85, 42, 70, 72, 198, 141, 21, 65, 154, 129, 5, 69, + 150, 64, 73, 2, 79, 3, 85, 28, 166, 19, 72, 158, 250, 20, 65, 154, 129, + 5, 69, 150, 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 198, 253, 20, + 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, + 72, 198, 253, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, + 85, 24, 190, 8, 87, 130, 131, 21, 65, 154, 129, 5, 69, 222, 61, 89, 186, + 2, 73, 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, + 84, 65, 76, 32, 158, 2, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, + 129, 5, 69, 151, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, + 70, 2, 77, 2, 80, 46, 71, 2, 75, 2, 81, 191, 210, 11, 72, 4, 11, 87, 4, + 182, 179, 26, 69, 215, 22, 73, 6, 11, 87, 6, 202, 137, 26, 69, 151, 64, + 73, 52, 70, 72, 206, 135, 21, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, + 85, 36, 202, 4, 87, 230, 8, 89, 158, 250, 20, 65, 154, 129, 5, 69, 150, + 64, 73, 2, 79, 3, 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 158, 250, + 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 26, 142, 3, 87, 130, + 131, 21, 65, 2, 79, 154, 129, 5, 69, 222, 61, 89, 186, 2, 73, 3, 85, 36, + 166, 7, 89, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, + 2, 73, 3, 85, 56, 82, 72, 142, 1, 87, 130, 131, 21, 65, 2, 79, 154, 129, + 5, 69, 150, 64, 73, 3, 85, 32, 74, 65, 194, 131, 21, 79, 154, 129, 5, 69, + 222, 61, 87, 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, + 76, 32, 203, 187, 26, 65, 8, 150, 132, 26, 69, 150, 64, 65, 3, 73, 64, + 94, 72, 134, 6, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, + 150, 64, 73, 3, 85, 24, 130, 6, 87, 246, 251, 20, 65, 154, 129, 5, 69, + 150, 64, 73, 2, 79, 3, 85, 20, 170, 129, 21, 65, 2, 79, 154, 129, 5, 69, + 222, 61, 87, 2, 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, + 90, 78, 83, 158, 250, 20, 65, 2, 79, 246, 190, 5, 87, 186, 2, 73, 3, 85, + 13, 56, 8, 66, 65, 84, 66, 69, 73, 84, 32, 227, 192, 26, 69, 8, 226, 251, + 21, 66, 2, 70, 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, + 83, 234, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, + 85, 18, 142, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, + 73, 3, 85, 12, 218, 254, 25, 69, 222, 61, 65, 186, 2, 73, 2, 79, 3, 85, + 16, 134, 253, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 40, 82, + 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, + 85, 10, 242, 251, 20, 65, 154, 129, 5, 69, 151, 64, 73, 48, 90, 72, 78, + 90, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, + 85, 16, 230, 250, 20, 65, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 2, + 79, 3, 85, 14, 154, 250, 20, 65, 154, 129, 5, 69, 150, 64, 73, 2, 79, 3, + 85, 20, 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, + 82, 176, 253, 3, 3, 67, 72, 73, 189, 134, 22, 3, 89, 73, 90, 6, 48, 4, + 69, 82, 69, 84, 129, 166, 25, 2, 73, 70, 5, 17, 2, 45, 72, 2, 173, 132, + 26, 2, 73, 68, 4, 204, 165, 25, 2, 69, 78, 219, 12, 85, 2, 217, 139, 4, + 3, 73, 75, 82, 10, 68, 2, 82, 79, 205, 174, 25, 9, 76, 69, 82, 32, 67, + 79, 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 172, 153, 23, 8, 45, 67, + 85, 82, 82, 69, 78, 67, 171, 151, 2, 32, 4, 186, 248, 3, 67, 19, 80, 42, + 82, 67, 102, 84, 201, 248, 25, 12, 80, 82, 69, 83, 83, 73, 79, 78, 76, + 69, 83, 83, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, 203, 159, 25, + 69, 4, 202, 235, 23, 81, 151, 137, 2, 77, 34, 98, 82, 217, 223, 20, 18, + 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, + 14, 140, 1, 12, 69, 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 245, 226, + 24, 16, 65, 84, 69, 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, + 12, 50, 83, 214, 160, 24, 70, 234, 2, 87, 207, 10, 71, 4, 230, 161, 24, + 65, 43, 73, 7, 194, 227, 18, 71, 215, 207, 7, 83, 130, 4, 142, 1, 65, + 206, 18, 69, 194, 1, 73, 186, 7, 76, 214, 4, 79, 238, 6, 82, 142, 5, 85, + 136, 232, 19, 2, 86, 83, 254, 150, 4, 83, 207, 134, 2, 70, 90, 122, 67, + 254, 14, 76, 176, 2, 2, 88, 32, 230, 129, 3, 77, 252, 213, 5, 2, 82, 83, + 168, 250, 1, 2, 84, 72, 255, 192, 9, 73, 68, 72, 2, 69, 32, 236, 235, 15, + 4, 83, 73, 77, 73, 249, 183, 4, 2, 84, 79, 64, 226, 1, 83, 160, 1, 4, 87, + 73, 84, 72, 168, 203, 12, 2, 80, 65, 176, 130, 2, 2, 77, 65, 132, 201, + 10, 13, 84, 72, 82, 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, + 72, 79, 76, 68, 73, 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 212, 182, 16, + 18, 65, 86, 79, 85, 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, + 83, 161, 243, 4, 13, 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, + 52, 38, 32, 221, 190, 21, 3, 79, 85, 84, 50, 144, 4, 2, 67, 79, 44, 5, + 72, 69, 65, 68, 45, 38, 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, + 238, 1, 80, 164, 1, 16, 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, + 78, 71, 85, 69, 110, 84, 168, 140, 18, 22, 70, 73, 78, 71, 69, 82, 32, + 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, 83, 69, 68, 172, 67, 3, + 82, 79, 76, 180, 217, 1, 13, 76, 79, 79, 75, 32, 79, 70, 32, 84, 82, 73, + 85, 77, 184, 139, 1, 8, 68, 73, 65, 71, 79, 78, 65, 76, 1, 20, 85, 78, + 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, 68, 32, 87, 65, 86, 89, + 4, 252, 4, 3, 87, 66, 79, 163, 208, 18, 76, 2, 225, 205, 21, 4, 66, 65, + 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 217, 216, 24, 3, 79, 78, 79, 2, + 165, 229, 24, 3, 32, 77, 65, 12, 90, 75, 36, 4, 80, 69, 78, 32, 165, 218, + 19, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, 87, 2, 217, 188, 21, 4, 32, + 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, 161, 183, 21, 18, 69, 89, 69, + 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, 79, 86, 69, 82, 7, 11, 32, 4, + 228, 206, 9, 3, 86, 79, 77, 241, 130, 9, 5, 65, 78, 68, 32, 67, 6, 132, + 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, 32, 65, 78, 68, 32, 80, 65, + 82, 84, 100, 2, 69, 69, 133, 208, 18, 4, 76, 69, 65, 68, 2, 213, 230, 21, + 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, 36, 3, 87, 73, 78, 159, 208, + 18, 84, 2, 153, 170, 1, 4, 75, 73, 78, 71, 4, 50, 69, 149, 234, 21, 6, + 72, 69, 82, 77, 79, 77, 2, 197, 144, 26, 8, 65, 82, 83, 32, 79, 70, 32, + 74, 10, 34, 76, 237, 192, 21, 2, 65, 70, 8, 84, 13, 73, 78, 71, 32, 68, + 73, 65, 71, 79, 78, 65, 76, 32, 197, 236, 22, 2, 69, 78, 6, 128, 1, 9, + 67, 82, 79, 83, 83, 73, 78, 71, 32, 181, 227, 18, 16, 73, 78, 32, 87, 72, + 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, 4, 224, 197, 15, 3, 82, 73, + 83, 135, 165, 3, 78, 4, 166, 191, 14, 73, 175, 251, 3, 77, 14, 50, 65, + 54, 77, 44, 2, 82, 82, 147, 149, 18, 78, 4, 214, 196, 17, 84, 245, 156, + 8, 4, 82, 70, 85, 76, 4, 144, 188, 8, 2, 73, 78, 239, 157, 7, 65, 4, 168, + 199, 7, 2, 73, 83, 215, 214, 18, 89, 54, 200, 1, 5, 71, 85, 82, 69, 32, + 38, 76, 186, 1, 82, 182, 2, 83, 208, 12, 4, 86, 69, 32, 68, 172, 233, 4, + 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, 89, 233, 157, 19, 9, 78, 73, 84, + 69, 32, 80, 65, 82, 84, 4, 222, 182, 24, 68, 139, 167, 1, 83, 10, 32, 2, + 69, 32, 65, 2, 77, 32, 6, 174, 133, 15, 70, 172, 59, 4, 67, 65, 66, 73, + 247, 193, 7, 83, 4, 52, 4, 80, 82, 79, 74, 149, 198, 19, 3, 70, 82, 65, + 2, 221, 225, 25, 2, 69, 67, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, + 32, 69, 68, 4, 87, 79, 82, 75, 207, 244, 14, 67, 4, 214, 254, 12, 78, + 237, 192, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 220, 209, 22, 6, 32, + 83, 80, 65, 82, 75, 171, 199, 3, 83, 10, 222, 137, 5, 81, 168, 146, 10, + 8, 83, 84, 82, 79, 78, 71, 32, 73, 219, 197, 6, 80, 10, 38, 72, 177, 224, + 22, 3, 84, 69, 68, 9, 236, 234, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, + 32, 65, 78, 68, 32, 182, 171, 5, 69, 209, 250, 15, 19, 32, 67, 65, 75, + 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 36, 50, 65, + 134, 1, 69, 98, 79, 194, 1, 85, 39, 89, 10, 78, 84, 220, 148, 7, 6, 71, + 32, 73, 78, 32, 72, 229, 246, 16, 3, 77, 73, 78, 6, 214, 163, 8, 32, 202, + 170, 11, 66, 183, 176, 5, 78, 4, 132, 245, 22, 8, 88, 69, 68, 32, 66, 73, + 67, 69, 241, 130, 1, 7, 85, 82, 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, + 87, 69, 82, 231, 208, 24, 80, 2, 173, 140, 25, 2, 65, 76, 7, 17, 2, 32, + 80, 4, 58, 85, 141, 187, 23, 8, 76, 65, 89, 73, 78, 71, 32, 67, 2, 193, + 169, 25, 4, 78, 67, 84, 85, 4, 214, 139, 3, 83, 155, 240, 22, 84, 9, 25, + 4, 73, 78, 71, 32, 6, 242, 213, 9, 68, 148, 179, 8, 2, 83, 65, 139, 181, + 1, 69, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, 1, 85, + 172, 173, 22, 2, 78, 68, 203, 163, 3, 88, 5, 151, 255, 25, 71, 4, 196, + 148, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 183, 195, 16, 69, 5, 147, 224, + 13, 80, 16, 102, 75, 218, 252, 19, 77, 250, 141, 4, 32, 222, 112, 67, + 177, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, 78, + 68, 32, 75, 78, 73, 70, 69, 218, 225, 14, 69, 247, 219, 10, 73, 5, 133, + 146, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 159, 200, 22, 78, 20, + 38, 32, 218, 2, 84, 255, 154, 18, 45, 16, 110, 67, 174, 1, 68, 234, 154, + 2, 66, 204, 240, 9, 7, 76, 69, 65, 70, 32, 67, 76, 242, 105, 84, 171, + 130, 11, 80, 4, 244, 246, 12, 3, 76, 85, 66, 229, 85, 32, 79, 82, 78, 69, + 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, 78, 71, 32, + 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, 4, 186, + 222, 22, 80, 183, 235, 2, 77, 2, 255, 31, 72, 28, 78, 65, 226, 1, 69, 98, + 79, 165, 174, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, 67, 84, + 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 214, 168, 19, + 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, 166, 155, + 2, 65, 213, 196, 20, 2, 84, 73, 2, 213, 158, 21, 2, 73, 67, 6, 48, 6, 78, + 67, 72, 32, 70, 82, 171, 185, 18, 69, 4, 150, 244, 24, 73, 249, 12, 3, + 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 143, 201, 25, 71, 4, + 70, 84, 145, 128, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, 66, 89, 2, + 253, 132, 12, 6, 73, 76, 84, 69, 68, 32, 5, 181, 176, 18, 15, 73, 78, 71, + 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 79, 224, 1, 80, 2, 76, 76, + 250, 5, 78, 216, 165, 16, 5, 69, 76, 32, 80, 85, 215, 194, 9, 83, 216, 1, + 42, 32, 73, 6, 87, 73, 68, 84, 72, 32, 10, 158, 210, 11, 77, 160, 159, 3, + 2, 79, 85, 194, 170, 8, 66, 131, 30, 83, 206, 1, 238, 1, 67, 42, 76, 78, + 78, 30, 80, 66, 82, 134, 1, 83, 38, 89, 130, 230, 9, 77, 130, 171, 10, + 65, 158, 2, 68, 58, 69, 98, 71, 118, 72, 190, 4, 81, 214, 139, 2, 87, + 182, 24, 84, 220, 113, 6, 66, 82, 79, 75, 69, 78, 202, 4, 70, 159, 177, + 1, 86, 10, 238, 148, 20, 73, 62, 79, 227, 75, 69, 116, 42, 69, 162, 152, + 20, 65, 171, 132, 4, 79, 10, 162, 1, 70, 139, 154, 20, 83, 4, 182, 234, + 20, 79, 35, 85, 6, 42, 79, 138, 155, 20, 69, 247, 203, 2, 76, 2, 239, + 154, 24, 85, 10, 36, 3, 73, 71, 72, 231, 237, 23, 69, 8, 17, 2, 84, 32, + 8, 182, 163, 22, 67, 240, 1, 5, 87, 72, 73, 84, 69, 22, 80, 155, 2, 83, + 4, 218, 201, 22, 69, 203, 165, 1, 79, 2, 247, 136, 24, 69, 4, 236, 146, + 11, 8, 67, 84, 73, 79, 78, 32, 65, 80, 253, 212, 8, 5, 69, 82, 65, 76, + 32, 228, 18, 110, 65, 62, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, + 6, 82, 158, 93, 85, 226, 175, 21, 72, 215, 199, 3, 83, 4, 196, 214, 24, + 2, 82, 76, 189, 139, 1, 4, 77, 69, 32, 68, 242, 2, 112, 2, 65, 82, 110, + 77, 50, 79, 224, 220, 22, 9, 82, 77, 65, 78, 32, 80, 69, 78, 78, 130, + 153, 2, 84, 215, 105, 78, 7, 29, 5, 32, 87, 73, 84, 72, 4, 220, 230, 13, + 5, 79, 85, 84, 32, 72, 177, 236, 8, 5, 32, 72, 65, 78, 68, 4, 26, 32, + 131, 171, 21, 73, 2, 199, 172, 22, 83, 226, 2, 64, 6, 77, 69, 84, 82, 73, + 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 185, 19, 4, 65, 76, 76, 89, + 137, 214, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, 1, 6, 67, 65, 80, 73, + 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, 84, 84, 69, 82, 32, 180, + 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, 65, 80, 73, 84, 65, 76, + 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, 65, 80, 45, 9, 76, 32, 76, + 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, 72, 166, 5, 67, 118, 71, 130, + 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 176, 155, 5, 2, 84, 65, 234, 250, + 7, 76, 214, 168, 2, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, 37, 78, 2, + 81, 238, 34, 86, 222, 47, 68, 14, 69, 2, 73, 2, 79, 2, 85, 2, 89, 131, + 57, 87, 4, 210, 165, 25, 69, 215, 79, 78, 10, 46, 65, 134, 222, 25, 73, + 2, 79, 215, 22, 69, 4, 214, 244, 25, 69, 3, 82, 94, 254, 1, 85, 178, 2, + 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, 80, 34, + 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, + 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 89, 130, 57, + 87, 255, 2, 70, 4, 252, 174, 21, 4, 45, 66, 82, 74, 191, 195, 4, 78, 92, + 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, + 80, 34, 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, + 88, 142, 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 85, + 2, 89, 130, 57, 87, 255, 2, 70, 6, 182, 160, 25, 69, 2, 73, 215, 79, 78, + 8, 38, 72, 162, 230, 24, 73, 203, 57, 65, 4, 230, 159, 25, 73, 135, 23, + 65, 4, 180, 227, 8, 2, 76, 73, 231, 139, 17, 78, 4, 186, 167, 24, 72, + 227, 119, 65, 12, 46, 65, 206, 215, 25, 73, 2, 79, 215, 22, 69, 6, 26, + 82, 135, 238, 25, 69, 5, 155, 231, 24, 68, 4, 186, 166, 24, 72, 155, 62, + 73, 4, 166, 203, 24, 72, 215, 82, 65, 4, 11, 65, 4, 134, 154, 15, 66, + 159, 211, 10, 83, 4, 214, 202, 24, 72, 219, 105, 65, 4, 162, 237, 24, 72, + 235, 47, 65, 6, 220, 177, 3, 6, 85, 82, 78, 69, 68, 32, 143, 234, 1, 65, + 4, 218, 201, 24, 72, 215, 82, 69, 2, 165, 219, 19, 7, 82, 65, 71, 82, 65, + 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, 82, 31, 82, 2, 239, 215, + 24, 76, 2, 141, 252, 17, 2, 32, 82, 6, 38, 76, 237, 189, 13, 3, 65, 70, + 70, 5, 151, 215, 24, 83, 202, 1, 66, 65, 214, 13, 79, 217, 154, 25, 7, + 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, 79, 76, 73, 84, 73, 67, + 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, 192, 1, 56, 6, 67, 65, + 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, 68, 94, 70, 38, 71, 46, + 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, 58, 82, 30, 83, 186, 1, + 84, 110, 86, 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 131, 131, + 5, 75, 2, 199, 193, 25, 90, 4, 214, 3, 73, 253, 208, 25, 2, 85, 75, 4, + 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, 72, 2, 11, 72, 2, 197, 252, 8, + 2, 82, 73, 6, 42, 74, 30, 79, 129, 194, 25, 2, 90, 69, 2, 253, 251, 8, 2, + 69, 82, 2, 187, 248, 23, 66, 4, 238, 191, 20, 82, 131, 128, 5, 73, 2, 21, + 3, 76, 65, 71, 2, 255, 148, 21, 79, 13, 38, 78, 54, 79, 141, 1, 2, 90, + 72, 2, 181, 182, 25, 8, 73, 84, 73, 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, + 84, 69, 68, 32, 4, 26, 66, 25, 2, 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, + 65, 76, 76, 2, 177, 223, 23, 2, 32, 89, 4, 178, 223, 25, 73, 211, 2, 69, + 4, 52, 9, 65, 84, 73, 78, 65, 84, 69, 32, 77, 35, 74, 2, 193, 223, 10, 3, + 89, 83, 76, 2, 177, 231, 8, 3, 85, 68, 73, 2, 11, 65, 2, 239, 188, 20, + 83, 4, 226, 187, 25, 78, 3, 84, 4, 26, 79, 151, 224, 25, 69, 2, 225, 245, + 21, 2, 75, 79, 2, 225, 161, 21, 2, 73, 84, 14, 106, 72, 58, 76, 168, 195, + 13, 6, 80, 73, 68, 69, 82, 89, 209, 237, 9, 8, 77, 65, 76, 76, 32, 89, + 85, 83, 6, 32, 2, 84, 65, 207, 222, 25, 65, 5, 223, 183, 24, 80, 2, 251, + 218, 8, 79, 6, 78, 86, 208, 201, 21, 9, 82, 79, 75, 85, 84, 65, 83, 84, + 73, 175, 128, 4, 83, 2, 157, 227, 18, 2, 82, 73, 2, 195, 208, 23, 69, 12, + 50, 69, 246, 184, 18, 65, 254, 163, 7, 79, 3, 85, 6, 174, 184, 20, 83, + 151, 228, 4, 82, 4, 192, 235, 8, 3, 69, 77, 76, 221, 172, 15, 4, 72, 73, + 86, 69, 2, 243, 216, 25, 76, 6, 212, 199, 11, 13, 66, 69, 32, 87, 73, 84, + 72, 32, 77, 69, 82, 73, 68, 146, 177, 4, 87, 179, 207, 8, 86, 68, 162, 1, + 65, 44, 12, 84, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 178, 174, 17, + 71, 182, 55, 82, 252, 182, 5, 3, 78, 71, 71, 238, 179, 1, 79, 149, 75, 2, + 76, 70, 4, 132, 255, 14, 2, 76, 32, 231, 218, 10, 84, 54, 202, 2, 65, 50, + 72, 46, 73, 46, 78, 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 240, 227, 8, 2, + 87, 73, 142, 233, 11, 85, 160, 70, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, + 82, 248, 21, 2, 79, 84, 202, 30, 68, 218, 99, 77, 144, 117, 3, 83, 65, + 85, 138, 16, 69, 128, 26, 3, 76, 65, 71, 142, 197, 1, 74, 184, 16, 2, 71, + 73, 141, 12, 2, 75, 85, 4, 132, 212, 23, 3, 73, 72, 86, 171, 128, 2, 72, + 4, 154, 179, 13, 87, 141, 154, 11, 2, 65, 71, 4, 142, 229, 8, 85, 201, + 155, 14, 2, 71, 71, 6, 194, 149, 15, 73, 137, 195, 8, 2, 65, 85, 2, 137, + 145, 25, 5, 65, 73, 82, 84, 72, 2, 147, 176, 25, 73, 4, 136, 231, 20, 2, + 72, 73, 185, 152, 2, 2, 69, 73, 234, 9, 46, 65, 190, 6, 69, 206, 82, 73, + 139, 3, 79, 152, 1, 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, + 65, 32, 226, 232, 19, 86, 215, 214, 4, 80, 2, 11, 32, 2, 211, 194, 24, + 67, 146, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, + 78, 32, 104, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 234, 191, 7, + 65, 247, 253, 17, 79, 100, 214, 1, 86, 202, 231, 21, 65, 38, 68, 46, 84, + 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, 158, 20, 72, 2, + 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, 246, 235, 21, 79, 231, + 227, 3, 65, 4, 212, 160, 23, 6, 68, 79, 85, 66, 76, 69, 151, 234, 1, 65, + 16, 66, 67, 162, 191, 21, 80, 202, 40, 65, 170, 1, 78, 203, 162, 3, 86, + 4, 238, 200, 7, 79, 207, 159, 14, 65, 26, 218, 233, 21, 65, 106, 86, 214, + 20, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, 69, 192, 8, 76, 10, 65, + 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 159, 190, 24, 89, 56, + 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, 84, + 79, 32, 79, 82, 218, 173, 6, 67, 138, 4, 87, 175, 141, 18, 83, 16, 44, 5, + 66, 79, 86, 69, 32, 183, 178, 6, 78, 12, 150, 1, 83, 180, 1, 19, 68, 79, + 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, 65, 220, + 172, 6, 4, 76, 69, 83, 83, 207, 252, 17, 82, 6, 148, 1, 7, 73, 77, 73, + 76, 65, 82, 32, 221, 175, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, + 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, 179, 175, + 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 155, 176, 6, 85, + 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, 120, 16, 83, + 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, 226, 177, 6, + 65, 150, 213, 12, 69, 191, 177, 4, 76, 9, 49, 10, 32, 87, 73, 84, 72, 32, + 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 171, 220, 17, 73, 5, 251, + 147, 25, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 235, 158, 19, 32, 2, + 253, 182, 23, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, 2, 78, 32, 254, + 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, 32, 182, 11, 68, + 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, 3, 78, 66, 77, + 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, 130, 22, 84, + 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, 32, 150, 1, + 86, 142, 2, 89, 254, 186, 7, 66, 164, 174, 3, 4, 71, 82, 65, 77, 204, 23, + 2, 90, 69, 167, 177, 11, 81, 112, 92, 10, 67, 82, 79, 80, 72, 79, 78, 73, + 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, 188, 2, 6, 65, + 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, 68, 69, 76, + 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, 84, 73, 65, + 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, 82, 69, 65, + 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, 78, 73, 65, + 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, 205, 1, 4, 84, + 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, 32, 8, 22, 84, + 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, 65, 5, 231, 3, + 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, 84, 143, 3, + 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, 2, 65, 5, + 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, 78, 230, + 186, 23, 81, 229, 222, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, 78, 68, + 82, 69, 203, 186, 23, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, 83, 2, 95, + 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, 85, 83, 65, + 78, 68, 32, 2, 171, 241, 15, 76, 2, 11, 83, 2, 201, 187, 23, 2, 84, 65, + 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, 73, 67, 32, + 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, 77, 2, 147, + 226, 12, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 221, 248, 14, 2, 73, 86, 5, + 11, 32, 2, 161, 234, 9, 5, 68, 82, 65, 67, 72, 8, 112, 8, 77, 73, 79, 78, + 73, 65, 78, 32, 253, 145, 24, 14, 65, 69, 85, 77, 32, 79, 78, 69, 32, 80, + 76, 69, 84, 72, 6, 150, 140, 12, 70, 254, 216, 11, 84, 227, 55, 79, 2, + 11, 32, 2, 215, 228, 23, 84, 32, 92, 8, 72, 69, 83, 80, 73, 65, 78, 32, + 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, 40, 2, 70, 73, + 38, 84, 239, 217, 17, 79, 6, 222, 139, 20, 86, 219, 190, 3, 70, 8, 246, + 244, 14, 72, 190, 239, 9, 69, 227, 48, 87, 12, 36, 2, 70, 73, 209, 24, 2, + 84, 69, 8, 142, 243, 17, 86, 145, 193, 6, 3, 70, 84, 89, 2, 231, 221, 9, + 69, 4, 252, 246, 21, 2, 79, 85, 149, 164, 1, 3, 84, 65, 66, 154, 2, 66, + 76, 174, 45, 82, 66, 68, 168, 188, 7, 2, 75, 65, 135, 6, 84, 144, 2, 44, + 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, 65, 190, 1, 69, + 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, 72, 79, 46, 83, + 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, 84, 138, 157, 5, + 68, 156, 164, 2, 2, 75, 65, 146, 192, 1, 71, 194, 223, 10, 67, 166, 255, + 1, 66, 2, 72, 2, 90, 166, 1, 76, 254, 210, 2, 89, 198, 43, 77, 2, 78, + 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, 67, 72, 65, 73, + 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, 2, 68, 30, 80, + 226, 29, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, 62, 186, 1, + 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, 5, 68, 136, + 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, + 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, 87, 73, 84, + 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 251, 225, 3, 84, 16, 65, + 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, 17, 29, 5, 32, + 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, 80, 4, 81, 2, + 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, 175, 15, 82, + 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, 32, 87, 73, 84, + 72, 32, 68, 6, 170, 230, 7, 73, 206, 242, 16, 65, 227, 48, 72, 23, 33, 6, + 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, 5, 79, 142, 226, 3, + 84, 147, 140, 5, 77, 10, 130, 24, 65, 133, 203, 1, 5, 73, 65, 76, 89, 84, + 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, 3, 82, 65, 67, 227, + 22, 65, 8, 174, 24, 65, 223, 232, 3, 84, 2, 235, 146, 11, 72, 4, 40, 3, + 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, 76, 144, 1, 27, 83, + 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, + 32, 83, 89, 77, 66, 79, 76, 45, 165, 202, 21, 2, 68, 73, 74, 70, 49, 70, + 50, 62, 51, 62, 52, 170, 37, 53, 250, 252, 24, 55, 3, 56, 17, 218, 163, + 25, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 150, 163, 25, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 218, 162, 25, 48, 2, 50, 2, 54, 2, + 55, 2, 56, 3, 57, 17, 158, 162, 25, 48, 2, 50, 2, 51, 2, 53, 2, 55, 2, + 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, 84, 72, 79, 83, 4, 210, + 216, 7, 80, 195, 181, 16, 73, 2, 11, 82, 2, 11, 79, 2, 251, 131, 23, 78, + 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, 6, 79, 87, 69, 82, 32, 78, + 32, 6, 85, 78, 65, 84, 69, 32, 157, 225, 21, 2, 73, 84, 24, 94, 83, 140, + 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, 75, 182, 11, 68, 135, 182, + 24, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, + 210, 12, 65, 195, 205, 7, 84, 12, 74, 80, 154, 154, 9, 71, 174, 131, 9, + 79, 238, 195, 2, 82, 243, 152, 1, 76, 4, 238, 137, 25, 83, 219, 19, 73, + 2, 249, 223, 9, 3, 85, 77, 69, 4, 222, 25, 83, 163, 186, 7, 69, 4, 80, 4, + 69, 84, 82, 69, 253, 208, 22, 10, 85, 83, 73, 67, 65, 76, 32, 76, 69, 73, + 2, 139, 206, 2, 84, 12, 88, 3, 78, 69, 32, 142, 198, 9, 88, 200, 136, 5, + 3, 85, 78, 75, 161, 181, 5, 2, 66, 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, + 73, 71, 21, 4, 81, 85, 65, 82, 4, 207, 155, 24, 78, 2, 159, 132, 20, 84, + 16, 62, 82, 206, 11, 83, 114, 69, 230, 197, 7, 72, 203, 180, 16, 73, 2, + 217, 29, 2, 79, 83, 6, 100, 3, 72, 79, 32, 241, 207, 7, 16, 69, 86, 69, + 82, 83, 69, 68, 32, 76, 85, 78, 65, 84, 69, 32, 69, 4, 140, 197, 13, 10, + 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 243, 192, 10, 83, 226, 2, 220, 1, + 5, 77, 65, 76, 76, 32, 192, 19, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, + 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 10, 89, 77, 66, + 79, 76, 32, 84, 65, 85, 32, 237, 156, 23, 6, 73, 78, 85, 83, 79, 73, 212, + 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, 202, 17, 82, 67, 68, 206, 2, 178, + 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, 79, 84, 65, 0, 7, 85, 80, 83, 73, + 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, 182, 5, 80, 112, 3, 82, 72, 79, + 94, 83, 94, 84, 192, 192, 7, 2, 70, 73, 246, 192, 1, 71, 194, 223, 10, + 67, 166, 255, 1, 66, 2, 72, 2, 90, 166, 1, 76, 194, 254, 2, 77, 2, 78, + 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, 1, 7, 82, 67, 72, 65, 73, + 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, 82, 86, 226, 6, 68, 30, + 80, 114, 79, 142, 1, 89, 250, 227, 3, 84, 147, 140, 5, 77, 6, 154, 5, 82, + 155, 3, 65, 4, 18, 75, 23, 83, 2, 163, 206, 7, 79, 2, 11, 65, 2, 203, + 143, 13, 77, 4, 214, 184, 7, 73, 191, 180, 14, 69, 70, 22, 80, 215, 4, + 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, 84, 72, 32, 38, 78, + 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, + 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, 8, 153, 1, 3, 83, 73, + 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, 154, 6, 79, 22, 86, + 251, 225, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, 76, 73, 9, 17, 2, 32, + 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, 23, 86, 2, 11, 69, 2, + 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, 147, 15, 65, 2, 177, + 235, 24, 2, 65, 67, 4, 154, 201, 7, 65, 3, 79, 70, 28, 2, 69, 71, 151, 3, + 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, 48, 58, 68, 30, 80, + 114, 79, 62, 86, 82, 89, 251, 227, 3, 84, 16, 61, 4, 65, 83, 73, 65, 20, + 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, 68, 32, 14, 42, 79, + 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, 82, 73, 83, 80, 79, + 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, 65, 5, 33, 6, 32, 65, + 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, 20, 17, 2, 79, 78, 21, + 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, 83, 73, 65, 0, 5, 80, + 83, 73, 76, 73, 54, 79, 22, 86, 251, 225, 3, 84, 7, 29, 5, 32, 65, 78, + 68, 32, 4, 18, 79, 23, 86, 2, 151, 178, 9, 88, 2, 179, 9, 65, 8, 88, 11, + 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 218, 242, 24, 72, 2, 83, 219, + 19, 73, 2, 215, 173, 7, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 34, 68, + 201, 182, 20, 2, 80, 83, 2, 215, 159, 8, 65, 10, 54, 65, 134, 193, 7, 84, + 230, 1, 73, 175, 163, 17, 72, 4, 142, 131, 13, 77, 251, 129, 12, 78, 4, + 150, 224, 21, 72, 231, 255, 2, 65, 4, 41, 8, 69, 86, 69, 82, 83, 69, 68, + 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, 32, 76, 2, 11, 85, + 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 245, 193, 7, 3, 73, 71, 77, 10, 214, + 255, 8, 71, 194, 223, 10, 67, 2, 80, 218, 103, 82, 207, 151, 1, 66, 2, + 167, 198, 20, 82, 16, 106, 72, 104, 7, 82, 89, 66, 76, 73, 79, 78, 44, 3, + 87, 79, 32, 150, 219, 3, 79, 141, 132, 16, 2, 65, 76, 6, 40, 4, 82, 69, + 69, 32, 219, 191, 7, 69, 4, 146, 1, 79, 185, 230, 21, 7, 81, 85, 65, 82, + 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 232, 22, 83, 4, 42, 79, 253, + 234, 11, 4, 84, 72, 73, 82, 2, 229, 148, 19, 2, 66, 79, 6, 80, 5, 65, 67, + 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, 83, 39, 72, 2, 33, 6, + 32, 65, 78, 68, 32, 72, 2, 145, 157, 7, 2, 79, 79, 60, 102, 65, 21, 21, + 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, + 79, 76, 45, 2, 207, 168, 9, 82, 58, 90, 50, 2, 53, 166, 195, 22, 49, 214, + 185, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, 246, 252, 24, 48, 2, + 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 139, 230, 19, 69, 2, 11, 79, 2, 33, + 6, 71, 69, 71, 82, 65, 77, 2, 253, 171, 20, 2, 77, 69, 8, 238, 189, 13, + 65, 142, 237, 9, 66, 210, 73, 72, 241, 64, 3, 83, 65, 76, 12, 60, 6, 78, + 78, 73, 78, 71, 32, 205, 242, 23, 3, 77, 65, 67, 10, 100, 4, 70, 65, 67, + 69, 217, 166, 17, 15, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, + 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, 23, 79, 78, 69, 32, 76, + 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, 32, 83, 77, 65, 76, 76, + 35, 83, 2, 11, 32, 2, 159, 247, 7, 69, 4, 174, 165, 17, 77, 145, 1, 3, + 84, 65, 82, 6, 28, 3, 85, 80, 32, 39, 87, 4, 250, 222, 21, 83, 211, 215, + 2, 77, 2, 251, 211, 17, 73, 232, 3, 164, 1, 2, 65, 82, 70, 73, 52, 7, 74, + 65, 82, 65, 84, 73, 32, 212, 6, 12, 78, 74, 65, 76, 65, 32, 71, 79, 78, + 68, 73, 32, 137, 3, 7, 82, 77, 85, 75, 72, 73, 32, 4, 34, 65, 157, 172, + 20, 2, 68, 83, 2, 11, 78, 2, 239, 238, 23, 73, 4, 154, 211, 23, 84, 201, + 161, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, 32, 226, + 175, 19, 65, 210, 24, 82, 154, 185, 3, 68, 199, 221, 1, 79, 98, 142, 139, + 21, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, + 234, 14, 90, 142, 175, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, + 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 24, 98, 67, + 28, 3, 77, 65, 68, 22, 84, 250, 194, 3, 83, 138, 199, 17, 65, 170, 1, 78, + 203, 162, 3, 86, 4, 118, 73, 143, 138, 21, 65, 2, 187, 184, 17, 68, 4, + 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, 84, 2, 25, + 4, 82, 67, 76, 69, 2, 129, 249, 19, 5, 32, 78, 85, 75, 84, 34, 36, 5, 83, + 73, 71, 78, 32, 91, 67, 30, 86, 67, 142, 138, 21, 65, 106, 86, 214, 20, + 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 201, 138, 21, 5, 65, 78, + 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, + 71, 78, 32, 38, 86, 186, 253, 22, 68, 199, 221, 1, 79, 80, 250, 132, 21, + 65, 38, 68, 46, 84, 230, 24, 85, 22, 78, 190, 200, 1, 73, 42, 76, 242, + 190, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, + 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 174, 188, 22, 86, 223, + 234, 1, 65, 20, 230, 6, 79, 247, 178, 22, 73, 172, 1, 194, 1, 65, 44, 7, + 76, 69, 84, 84, 69, 82, 32, 242, 1, 83, 228, 2, 2, 86, 79, 244, 226, 12, + 3, 84, 73, 80, 146, 249, 4, 73, 172, 130, 3, 5, 69, 75, 32, 79, 78, 162, + 152, 2, 68, 235, 169, 1, 85, 4, 138, 255, 20, 66, 253, 187, 1, 2, 68, 68, + 96, 202, 156, 8, 71, 2, 75, 218, 172, 12, 82, 206, 55, 65, 38, 68, 46, + 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 126, 66, 2, + 67, 2, 74, 2, 80, 2, 83, 194, 40, 79, 162, 8, 69, 158, 20, 70, 2, 72, 2, + 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, + 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, 71, 78, 32, 12, 70, + 71, 2, 75, 222, 238, 22, 83, 166, 213, 1, 76, 226, 31, 70, 3, 90, 2, 219, + 238, 22, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, 66, 2, 66, 164, 211, 14, + 2, 85, 68, 150, 172, 6, 78, 160, 128, 2, 3, 89, 65, 75, 171, 162, 1, 86, + 2, 139, 238, 7, 73, 18, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 18, + 226, 255, 20, 65, 190, 21, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, + 69, 142, 23, 110, 65, 162, 95, 69, 206, 103, 73, 146, 9, 79, 142, 11, 84, + 30, 85, 130, 1, 89, 137, 186, 12, 4, 82, 89, 86, 78, 136, 11, 248, 1, 2, + 73, 82, 60, 8, 76, 70, 87, 73, 68, 84, 72, 32, 254, 10, 77, 210, 1, 78, + 236, 76, 21, 80, 80, 89, 32, 80, 69, 82, 83, 79, 78, 32, 82, 65, 73, 83, + 73, 78, 71, 32, 79, 78, 22, 84, 180, 183, 17, 2, 85, 77, 128, 141, 5, 2, + 82, 68, 227, 46, 68, 6, 26, 32, 163, 231, 23, 67, 4, 134, 178, 23, 80, + 187, 112, 83, 244, 1, 132, 2, 7, 72, 65, 78, 71, 85, 76, 32, 216, 4, 8, + 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 58, 85, 228, 159, 6, 11, 70, 79, 82, 77, 83, 32, 76, 73, 71, 72, 84, + 194, 207, 6, 73, 174, 170, 4, 66, 190, 185, 4, 87, 167, 26, 68, 104, 52, + 7, 76, 69, 84, 84, 69, 82, 32, 255, 181, 10, 70, 102, 206, 1, 75, 28, 5, + 78, 73, 69, 85, 78, 42, 80, 24, 5, 82, 73, 69, 85, 76, 86, 83, 98, 89, + 222, 61, 67, 54, 69, 30, 73, 242, 4, 77, 138, 1, 84, 206, 3, 87, 198, 1, + 72, 214, 208, 23, 65, 2, 79, 151, 64, 85, 6, 242, 65, 72, 155, 3, 73, 7, + 11, 45, 4, 154, 72, 67, 131, 3, 72, 6, 166, 69, 72, 35, 73, 17, 11, 45, + 14, 226, 49, 84, 226, 14, 80, 130, 4, 77, 194, 3, 75, 218, 2, 72, 99, 83, + 12, 40, 4, 83, 65, 78, 71, 215, 178, 13, 73, 10, 230, 70, 67, 42, 75, 74, + 80, 34, 84, 211, 2, 83, 14, 154, 155, 22, 69, 238, 254, 1, 65, 150, 64, + 73, 2, 79, 3, 85, 118, 70, 32, 149, 238, 2, 11, 45, 72, 73, 82, 65, 71, + 65, 78, 65, 32, 80, 116, 76, 7, 76, 69, 84, 84, 69, 82, 32, 234, 237, 2, + 83, 34, 86, 147, 217, 20, 77, 110, 146, 1, 83, 222, 231, 2, 78, 150, 2, + 72, 2, 75, 2, 77, 2, 82, 2, 84, 170, 1, 89, 210, 41, 87, 202, 194, 21, + 65, 2, 69, 2, 73, 2, 79, 3, 85, 28, 76, 5, 77, 65, 76, 76, 32, 226, 214, + 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 18, 198, 234, 2, 89, 178, 199, 21, + 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 11, 84, 4, 236, 243, 12, + 2, 32, 67, 231, 194, 10, 87, 2, 155, 182, 23, 80, 14, 56, 3, 77, 69, 82, + 106, 83, 173, 156, 21, 3, 66, 85, 82, 9, 29, 5, 32, 65, 78, 68, 32, 6, + 172, 128, 12, 3, 87, 82, 69, 180, 213, 3, 2, 83, 73, 175, 208, 7, 80, 4, + 136, 151, 24, 3, 84, 69, 82, 151, 61, 65, 194, 8, 118, 68, 202, 2, 71, + 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, 89, 65, 32, 165, 5, + 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, 72, 32, 73, 78, 68, + 69, 88, 32, 188, 1, 2, 66, 65, 217, 138, 21, 2, 83, 72, 4, 156, 1, 18, + 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, 78, 71, 69, 82, 83, + 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, 84, 72, 85, 77, 66, 2, + 145, 206, 15, 2, 32, 67, 4, 182, 199, 23, 76, 159, 137, 1, 71, 170, 7, + 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, 78, 85, 77, 69, 82, + 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, 79, 78, 71, 32, 244, + 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, 180, 31, 7, 76, 69, + 84, 84, 69, 82, 32, 215, 246, 9, 70, 250, 1, 246, 1, 67, 172, 2, 5, 73, + 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, 77, 56, 5, 78, 73, + 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, 210, 1, 83, 166, 3, + 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, 139, 238, 9, 70, 30, + 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, 121, 4, 73, 69, 85, 67, + 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, 45, 4, 202, 31, 75, 243, + 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, 46, 67, 6, 40, 4, 83, 65, + 78, 71, 179, 162, 13, 73, 4, 194, 54, 67, 227, 3, 83, 5, 215, 30, 45, 27, + 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, 194, 5, 77, 138, 1, 84, + 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, 214, 3, 73, 207, 2, 65, + 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, 5, 73, 89, 69, 79, 75, + 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, 231, 3, 77, 11, 11, + 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, 11, 45, 12, 190, 51, + 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, 68, 6, 72, 73, 69, + 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, 45, 4, 158, 51, + 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, 7, 75, 162, 12, + 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, 79, 83, 251, 3, + 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, 80, 29, 11, 45, + 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, 80, 34, 84, 243, + 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, 4, 83, 65, 78, 71, + 151, 155, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, 48, 3, 73, 79, 83, + 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, 44, 2, 83, 83, + 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, 84, 226, 2, 78, + 178, 2, 72, 223, 223, 17, 73, 6, 184, 23, 4, 73, 69, 85, 80, 179, 19, 72, + 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, 85, 67, 172, 3, 4, + 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, 2, 75, 42, 78, 34, + 80, 146, 2, 72, 223, 223, 17, 73, 16, 40, 5, 73, 75, 69, 85, 84, 195, 41, + 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, 3, 67, 42, 75, 74, 80, + 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 165, 211, 8, 6, 76, 69, 32, + 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, 78, 71, 32, 165, 21, + 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, 1, 67, 80, 5, 72, 73, + 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, 1, 5, 77, 73, 69, 85, + 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, 5, 82, 73, 69, 85, 76, + 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, 4, 73, 69, 85, 67, 239, + 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, 11, 11, 45, 8, 174, 16, + 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, 6, 194, 17, 75, 57, 2, + 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, 5, 73, 89, 69, 79, 75, + 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, 77, 19, 11, 45, 16, + 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, 146, 9, 78, 34, 80, + 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, 82, 242, 13, 67, + 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, 65, 6, 40, 4, 83, + 65, 78, 71, 143, 143, 13, 73, 4, 238, 35, 78, 147, 3, 83, 21, 11, 45, 18, + 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, 218, 2, 72, 62, 80, 39, + 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, 73, 69, 85, 80, 72, 53, + 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, 65, 195, 26, 80, 9, 11, + 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, 11, 45, 20, 112, 5, 82, + 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, 80, 246, 19, 67, 194, 5, + 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, 80, 5, 221, 32, 2, 45, 84, + 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, 85, 77, 50, 80, 126, 83, 74, + 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, 10, 52, 5, 73, 89, 69, 79, + 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, 254, 32, 72, 99, 83, 9, 11, + 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, 48, 4, 73, 69, 85, 80, 174, + 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, 222, 29, 84, 242, 1, 72, 99, + 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, 71, 255, 135, 13, 73, 4, 182, + 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, 85, 84, 151, 25, 72, 6, 56, 9, + 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, 3, 83, 5, 255, 29, 45, 52, 48, + 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, 25, 11, 45, 22, 82, 75, 162, 3, + 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, 138, 1, 84, 147, 5, 72, 4, 22, + 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, 69, 79, 85, 78, 28, 160, 1, 5, + 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, 85, 84, 16, 3, 89, 69, 83, + 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, 42, 78, 34, 80, 239, 225, + 17, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, 255, 16, 45, 2, 171, 250, + 17, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, 72, 19, 11, 45, 16, 58, + 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, 80, 2, 17, 2, 73, 69, 2, + 255, 160, 23, 85, 4, 21, 3, 73, 79, 83, 5, 223, 1, 45, 2, 255, 19, 72, + 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, 17, 11, 45, 14, 50, 75, + 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, 14, 72, 135, 7, 73, 4, + 26, 83, 175, 128, 13, 73, 2, 21, 3, 65, 78, 71, 2, 207, 20, 75, 190, 1, + 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, 71, 65, 82, 106, 79, + 138, 1, 85, 102, 89, 174, 15, 87, 175, 234, 9, 70, 23, 48, 4, 82, 65, 69, + 65, 98, 45, 239, 165, 24, 69, 13, 11, 45, 10, 190, 231, 21, 69, 130, 191, + 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, 45, 6, 130, 129, 24, 69, + 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, 142, 156, 24, 65, 2, 79, + 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, 79, 142, 254, 23, 69, 234, + 36, 73, 3, 85, 5, 11, 82, 2, 235, 212, 17, 65, 14, 50, 65, 230, 228, 21, + 69, 130, 191, 2, 79, 3, 85, 7, 238, 132, 24, 45, 247, 30, 69, 23, 26, 45, + 171, 163, 24, 69, 18, 50, 79, 22, 89, 226, 227, 21, 69, 131, 191, 2, 85, + 5, 155, 143, 24, 45, 8, 222, 227, 21, 69, 239, 254, 1, 65, 17, 11, 45, + 14, 158, 19, 89, 234, 136, 5, 73, 192, 170, 4, 3, 69, 79, 45, 214, 155, + 14, 65, 151, 64, 85, 62, 42, 65, 70, 69, 66, 73, 22, 79, 107, 85, 11, 26, + 45, 147, 161, 24, 69, 6, 154, 130, 24, 89, 246, 30, 79, 3, 85, 11, 11, + 79, 9, 11, 45, 6, 150, 158, 24, 89, 186, 2, 79, 3, 85, 5, 191, 251, 23, + 45, 19, 11, 45, 16, 58, 89, 186, 223, 23, 65, 162, 33, 69, 246, 30, 73, + 3, 79, 6, 182, 223, 23, 65, 163, 33, 69, 21, 11, 45, 18, 142, 16, 89, + 146, 208, 21, 69, 238, 254, 1, 65, 150, 64, 73, 2, 79, 3, 85, 186, 1, + 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, 5, 77, 73, 69, 85, + 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, 73, 69, 85, 76, + 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 214, 208, 23, 79, + 151, 64, 85, 9, 156, 13, 3, 82, 65, 69, 207, 143, 24, 69, 4, 22, 72, 207, + 8, 73, 2, 241, 249, 23, 2, 73, 69, 7, 138, 156, 24, 79, 3, 85, 5, 239, + 234, 17, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, 155, 3, 73, + 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, 25, 4, 83, + 65, 78, 71, 2, 207, 7, 80, 2, 253, 59, 2, 73, 69, 9, 11, 45, 6, 22, 80, + 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, 67, 146, + 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, 170, 2, 72, + 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, 43, 75, 6, + 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, 45, 24, 68, + 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, 83, 4, 149, + 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 238, 22, 69, 8, 30, 72, 34, 73, + 131, 6, 65, 2, 245, 156, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, 80, 5, 243, + 5, 45, 4, 22, 72, 151, 3, 73, 2, 229, 167, 19, 2, 73, 69, 2, 17, 2, 69, + 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, 71, 13, 11, + 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, 42, 75, 42, + 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 255, 222, 17, 73, 2, 11, 73, 2, + 249, 236, 22, 2, 69, 85, 2, 11, 73, 2, 209, 233, 23, 2, 89, 69, 2, 11, + 73, 2, 243, 146, 23, 69, 2, 11, 73, 2, 135, 157, 23, 69, 2, 11, 73, 2, + 11, 75, 2, 255, 152, 23, 69, 10, 170, 211, 21, 69, 238, 254, 1, 65, 151, + 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 166, 207, 23, 65, 151, 64, 73, + 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, 2, 73, 78, + 2, 11, 72, 2, 157, 200, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, 39, 83, 2, + 11, 65, 2, 11, 78, 2, 11, 83, 2, 139, 232, 12, 73, 9, 11, 45, 6, 26, 89, + 207, 143, 24, 73, 4, 183, 207, 23, 65, 9, 11, 45, 6, 26, 89, 147, 143, + 24, 73, 4, 143, 208, 21, 69, 24, 178, 223, 10, 84, 170, 194, 11, 70, 30, + 83, 230, 83, 78, 14, 79, 199, 110, 69, 100, 156, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, 78, 32, + 196, 156, 2, 6, 86, 79, 87, 69, 76, 32, 167, 255, 19, 68, 58, 202, 1, 68, + 34, 75, 222, 249, 18, 84, 174, 113, 82, 246, 168, 2, 78, 246, 175, 1, 83, + 254, 68, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 80, 2, + 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 138, 137, 24, 68, 187, 2, 65, 8, + 56, 5, 73, 78, 78, 65, 32, 178, 136, 24, 72, 187, 2, 65, 4, 174, 136, 24, + 87, 3, 89, 4, 208, 232, 20, 6, 78, 65, 32, 75, 72, 79, 189, 162, 2, 3, + 83, 65, 75, 8, 52, 2, 84, 65, 209, 255, 21, 5, 72, 65, 82, 66, 65, 6, 42, + 72, 142, 203, 19, 83, 223, 187, 4, 78, 2, 135, 231, 23, 65, 42, 62, 76, + 172, 167, 18, 6, 83, 73, 71, 78, 32, 80, 139, 2, 86, 36, 33, 6, 69, 84, + 84, 69, 82, 32, 36, 158, 185, 20, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, + 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, + 2, 65, 2, 73, 3, 85, 2, 135, 249, 22, 69, 54, 52, 5, 67, 72, 73, 78, 71, + 41, 4, 82, 65, 78, 32, 2, 17, 2, 32, 67, 2, 207, 215, 22, 72, 52, 52, 7, + 76, 69, 84, 84, 69, 82, 32, 211, 145, 6, 78, 42, 218, 1, 65, 226, 132, + 11, 90, 210, 46, 84, 150, 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, + 198, 150, 4, 71, 122, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, + 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 4, 222, 193, + 16, 76, 207, 242, 6, 89, 170, 9, 230, 1, 65, 184, 32, 5, 66, 82, 69, 87, + 32, 230, 32, 76, 200, 1, 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, + 84, 84, 69, 82, 32, 174, 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, + 79, 82, 32, 137, 177, 23, 4, 68, 71, 69, 72, 212, 1, 42, 68, 98, 82, 201, + 1, 3, 86, 89, 32, 6, 44, 5, 83, 84, 79, 78, 69, 223, 215, 22, 80, 5, 205, + 219, 2, 7, 32, 71, 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 143, 222, 16, + 45, 10, 60, 5, 87, 73, 84, 72, 32, 210, 250, 11, 68, 139, 220, 8, 72, 6, + 76, 9, 84, 73, 80, 32, 79, 78, 32, 84, 72, 130, 200, 12, 82, 139, 157, + 10, 65, 2, 243, 204, 23, 69, 194, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, + 154, 3, 68, 162, 1, 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, + 162, 1, 82, 142, 2, 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, + 80, 174, 1, 87, 146, 215, 13, 73, 146, 223, 3, 80, 242, 220, 3, 86, 215, + 74, 71, 12, 108, 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, + 68, 84, 72, 32, 154, 217, 17, 77, 215, 146, 4, 83, 8, 36, 3, 79, 78, 69, + 235, 158, 22, 84, 7, 11, 32, 4, 182, 185, 13, 84, 151, 194, 8, 72, 14, + 48, 4, 65, 76, 76, 79, 21, 4, 76, 65, 67, 75, 2, 143, 140, 5, 84, 12, 30, + 32, 153, 1, 2, 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 151, 243, + 22, 72, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 217, 22, 8, 87, + 65, 82, 68, 83, 32, 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, + 6, 194, 159, 13, 83, 210, 164, 3, 78, 163, 149, 6, 82, 14, 116, 2, 72, + 69, 52, 5, 73, 82, 67, 76, 69, 245, 159, 17, 14, 79, 78, 67, 65, 86, 69, + 45, 80, 79, 73, 78, 84, 69, 68, 4, 156, 134, 20, 4, 86, 82, 79, 78, 247, + 176, 2, 67, 9, 64, 6, 32, 87, 73, 84, 72, 32, 165, 229, 21, 4, 68, 32, + 83, 65, 4, 52, 7, 83, 84, 82, 79, 75, 69, 32, 159, 227, 4, 67, 2, 29, 5, + 65, 78, 68, 32, 84, 2, 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, + 84, 2, 231, 248, 22, 83, 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, + 31, 79, 2, 175, 16, 82, 2, 157, 201, 18, 2, 86, 73, 10, 192, 16, 3, 87, + 78, 87, 206, 214, 13, 85, 135, 162, 4, 76, 16, 120, 5, 73, 71, 72, 84, + 32, 156, 2, 16, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, + 75, 32, 231, 133, 18, 81, 10, 40, 2, 80, 79, 126, 84, 135, 225, 21, 83, + 6, 33, 6, 73, 78, 84, 69, 68, 32, 6, 174, 130, 16, 80, 128, 221, 5, 11, + 82, 69, 67, 84, 73, 76, 73, 78, 69, 65, 82, 23, 66, 2, 173, 225, 21, 24, + 69, 65, 82, 68, 82, 79, 80, 45, 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, + 80, 69, 76, 76, 69, 82, 4, 150, 222, 22, 83, 187, 79, 79, 6, 44, 5, 79, + 85, 82, 32, 66, 227, 226, 4, 73, 2, 225, 218, 10, 6, 65, 76, 76, 79, 79, + 78, 4, 210, 254, 16, 79, 157, 170, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, + 65, 44, 3, 69, 70, 84, 28, 2, 79, 87, 237, 221, 4, 3, 73, 71, 65, 4, 212, + 236, 20, 2, 82, 71, 167, 196, 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, + 32, 179, 236, 9, 69, 4, 162, 225, 13, 68, 25, 4, 83, 73, 78, 71, 4, 56, + 8, 85, 76, 84, 73, 80, 76, 73, 67, 227, 211, 20, 73, 2, 25, 4, 65, 84, + 73, 79, 2, 163, 254, 4, 78, 6, 172, 234, 5, 9, 80, 69, 78, 32, 67, 69, + 78, 84, 82, 204, 152, 10, 13, 86, 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, + 65, 76, 213, 214, 5, 5, 85, 84, 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, + 149, 203, 22, 9, 79, 85, 78, 68, 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, + 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, + 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, 32, 4, 130, 196, 6, 66, 247, 153, + 7, 81, 7, 139, 6, 32, 26, 106, 65, 78, 73, 44, 2, 79, 85, 168, 229, 13, + 7, 67, 82, 73, 80, 84, 32, 76, 225, 131, 1, 3, 80, 65, 82, 4, 244, 220, + 13, 10, 78, 83, 45, 83, 69, 82, 73, 70, 32, 73, 195, 250, 7, 76, 8, 224, + 218, 13, 2, 78, 71, 203, 249, 7, 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, + 69, 65, 83, 84, 32, 29, 6, 87, 69, 83, 84, 32, 80, 6, 26, 80, 235, 204, + 22, 65, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 178, 181, 16, 86, + 239, 134, 6, 66, 12, 88, 9, 69, 65, 82, 68, 82, 79, 80, 45, 83, 130, 1, + 82, 169, 192, 6, 4, 87, 69, 76, 86, 6, 64, 6, 80, 79, 75, 69, 68, 32, + 145, 197, 22, 4, 72, 65, 78, 75, 4, 196, 212, 21, 8, 80, 73, 78, 87, 72, + 69, 69, 76, 27, 65, 2, 137, 3, 5, 73, 65, 78, 71, 76, 6, 26, 87, 159, + 226, 9, 80, 4, 53, 11, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, + 5, 87, 73, 84, 72, 32, 4, 204, 181, 19, 5, 76, 65, 82, 71, 69, 187, 200, + 1, 69, 10, 72, 5, 72, 73, 84, 69, 32, 112, 2, 73, 68, 177, 254, 21, 3, + 69, 68, 71, 6, 56, 5, 68, 79, 87, 78, 45, 130, 147, 22, 67, 199, 46, 83, + 2, 165, 190, 22, 8, 80, 79, 73, 78, 84, 73, 78, 71, 2, 145, 153, 20, 2, + 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, 32, 146, 8, 76, 164, 14, + 5, 77, 65, 82, 75, 32, 114, 80, 245, 188, 20, 2, 89, 79, 60, 132, 2, 9, + 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, 71, 69, 82, 74, 77, + 128, 1, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, 58, 83, 58, 84, + 156, 1, 2, 89, 69, 78, 90, 152, 132, 8, 3, 82, 69, 86, 246, 179, 14, 79, + 221, 143, 1, 3, 73, 76, 85, 2, 255, 214, 17, 85, 4, 178, 186, 18, 69, + 175, 225, 4, 65, 6, 32, 3, 69, 83, 72, 195, 28, 83, 5, 237, 187, 6, 4, + 32, 77, 85, 81, 8, 88, 5, 69, 82, 75, 72, 65, 148, 164, 16, 2, 85, 78, + 145, 177, 1, 5, 65, 72, 65, 80, 65, 5, 149, 150, 19, 4, 32, 75, 69, 70, + 4, 26, 83, 191, 158, 23, 90, 2, 207, 183, 23, 72, 4, 196, 151, 23, 6, 82, + 78, 69, 89, 32, 80, 179, 35, 68, 4, 194, 24, 69, 241, 230, 21, 6, 72, 65, + 76, 83, 72, 69, 8, 38, 69, 181, 227, 21, 3, 73, 80, 69, 6, 48, 6, 76, 73, + 83, 72, 65, 32, 163, 183, 11, 86, 4, 136, 177, 21, 3, 81, 69, 84, 153, + 135, 2, 4, 71, 69, 68, 79, 4, 176, 171, 8, 10, 82, 65, 72, 32, 66, 69, + 78, 32, 89, 79, 167, 214, 2, 84, 8, 34, 65, 177, 160, 23, 2, 73, 78, 6, + 40, 4, 81, 69, 70, 32, 139, 222, 5, 82, 4, 194, 222, 10, 81, 197, 231, + 11, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, 32, 201, 11, 8, 73, + 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, 1, 3, 66, 69, 84, 0, + 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, 76, 32, 92, 2, 81, + 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, 73, 18, 83, 48, 3, + 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, 71, 73, 77, 69, 76, + 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, 86, 65, 86, 80, 5, 87, + 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, 251, 212, 22, 89, 12, + 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, 86, 69, 32, 9, 33, 6, + 32, 87, 73, 84, 72, 32, 6, 250, 12, 77, 130, 1, 80, 35, 81, 4, 254, 145, + 16, 65, 187, 148, 1, 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 154, 15, + 82, 243, 172, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, 80, 69, 136, 152, 1, + 2, 84, 83, 218, 182, 21, 78, 239, 109, 77, 4, 235, 2, 70, 7, 236, 10, 5, + 32, 87, 73, 84, 72, 231, 198, 23, 84, 4, 167, 2, 78, 16, 44, 4, 65, 77, + 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, 6, 32, 87, 73, 84, 72, + 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, 33, 6, 32, 65, 78, 68, + 32, 83, 4, 210, 221, 5, 72, 167, 192, 2, 73, 12, 50, 69, 12, 2, 65, 86, + 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 161, 184, 13, 7, 32, 87, 73, 84, 72, + 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 176, 251, 1, 2, 72, 79, 159, + 188, 11, 68, 16, 174, 2, 76, 250, 169, 8, 65, 246, 188, 2, 84, 158, 206, + 2, 82, 164, 186, 8, 2, 68, 65, 134, 91, 75, 198, 106, 72, 169, 4, 7, 70, + 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 168, 7, 2, + 72, 73, 191, 174, 13, 68, 10, 72, 6, 65, 76, 69, 70, 32, 76, 29, 8, 89, + 73, 68, 68, 73, 83, 72, 32, 2, 133, 236, 18, 2, 65, 77, 8, 120, 7, 68, + 79, 85, 66, 76, 69, 32, 224, 4, 9, 89, 79, 68, 32, 89, 79, 68, 32, 80, + 237, 163, 20, 5, 86, 65, 86, 32, 89, 4, 210, 229, 10, 86, 251, 194, 9, + 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, 153, 250, 21, 6, 77, 65, 83, + 79, 82, 65, 2, 185, 184, 22, 2, 69, 82, 50, 84, 5, 79, 73, 78, 84, 32, + 241, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 38, 220, 1, 9, + 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, 106, 80, 164, 1, 17, 74, 85, + 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, 82, 22, 81, 86, + 82, 22, 83, 168, 174, 9, 2, 84, 83, 173, 207, 13, 3, 77, 69, 84, 2, 17, + 2, 32, 77, 2, 201, 1, 2, 65, 80, 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, + 33, 4, 79, 76, 65, 77, 6, 38, 80, 34, 81, 165, 2, 2, 83, 69, 2, 11, 65, + 2, 251, 140, 16, 84, 2, 153, 177, 22, 3, 65, 77, 65, 2, 11, 82, 2, 207, + 177, 13, 73, 5, 145, 224, 10, 12, 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, + 32, 86, 2, 195, 187, 23, 73, 6, 52, 5, 65, 77, 65, 84, 83, 133, 150, 11, + 2, 85, 66, 5, 193, 201, 10, 2, 32, 81, 2, 243, 191, 6, 65, 8, 34, 69, 22, + 72, 151, 145, 8, 73, 2, 179, 176, 22, 71, 4, 146, 145, 8, 73, 171, 187, + 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, 7, 83, 79, 70, + 32, 80, 65, 83, 224, 142, 21, 7, 78, 85, 78, 32, 72, 65, 70, 241, 176, 1, + 3, 77, 65, 81, 4, 26, 83, 131, 220, 21, 69, 2, 225, 190, 22, 3, 72, 65, + 89, 2, 199, 173, 13, 69, 2, 179, 173, 13, 85, 8, 114, 77, 248, 235, 11, + 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, 83, 221, 193, + 5, 3, 73, 67, 79, 4, 252, 188, 5, 12, 69, 84, 32, 87, 73, 84, 72, 32, 87, + 72, 73, 84, 223, 239, 16, 32, 188, 4, 164, 1, 2, 65, 45, 50, 72, 70, 75, + 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, 1, 87, 62, 89, + 110, 69, 186, 206, 8, 85, 250, 148, 5, 79, 167, 178, 3, 73, 8, 234, 158, + 23, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 182, 157, 8, 65, 222, + 201, 5, 85, 162, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, 2, 79, 45, 178, + 4, 73, 226, 3, 69, 211, 214, 14, 85, 24, 210, 193, 11, 49, 250, 227, 11, + 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 8, 222, + 167, 23, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, 45, 154, 7, 79, + 210, 214, 14, 65, 2, 73, 135, 191, 2, 85, 6, 134, 184, 23, 77, 186, 2, + 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, 246, 208, 8, + 79, 250, 148, 5, 85, 165, 160, 4, 6, 45, 77, 85, 45, 77, 79, 14, 170, + 154, 23, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, 130, + 162, 23, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 54, + 222, 4, 79, 2, 85, 210, 214, 14, 73, 134, 191, 2, 65, 3, 69, 68, 62, 65, + 2, 85, 226, 3, 73, 186, 206, 8, 69, 155, 136, 6, 79, 16, 11, 45, 16, 250, + 182, 23, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, 74, 69, + 20, 2, 79, 45, 72, 2, 85, 45, 178, 216, 14, 73, 135, 191, 2, 65, 18, 223, + 169, 18, 45, 14, 150, 179, 23, 82, 186, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 3, 54, 10, 146, 150, 23, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, 42, + 222, 207, 8, 65, 2, 73, 154, 136, 6, 79, 135, 191, 2, 69, 32, 40, 2, 65, + 45, 66, 79, 215, 149, 17, 85, 12, 242, 148, 23, 89, 246, 30, 49, 2, 50, + 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 154, 179, 23, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 54, 4, 176, 133, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, 79, + 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 175, 173, 15, 66, 128, 1, + 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, 71, 40, + 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 112, 2, 77, 79, 58, 79, 122, 80, + 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 216, 227, 14, 4, 76, 73, + 77, 73, 184, 137, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, 79, 133, + 159, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, 82, 124, + 4, 70, 84, 69, 82, 245, 209, 18, 4, 66, 85, 78, 68, 2, 149, 140, 23, 2, + 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 192, 193, 3, 6, 73, 84, 73, 78, 71, + 32, 1, 4, 82, 69, 65, 75, 2, 253, 220, 22, 7, 32, 67, 79, 77, 80, 76, 69, + 6, 26, 78, 239, 200, 18, 77, 4, 196, 236, 19, 4, 84, 69, 77, 80, 217, + 177, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 134, 205, 18, 85, 173, 200, + 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, 6, + 150, 201, 18, 67, 192, 6, 2, 76, 73, 201, 152, 4, 5, 86, 69, 76, 79, 80, + 4, 184, 191, 18, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, 84, + 72, 69, 32, 66, 69, 71, 73, 78, 137, 210, 2, 4, 83, 80, 69, 82, 4, 148, + 196, 18, 2, 79, 76, 221, 144, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, 65, + 84, 72, 69, 82, 35, 82, 2, 213, 206, 14, 3, 73, 78, 71, 10, 40, 4, 69, + 65, 84, 32, 211, 235, 22, 65, 8, 22, 80, 207, 5, 84, 6, 22, 79, 143, 5, + 82, 4, 168, 2, 2, 83, 83, 135, 232, 22, 87, 8, 50, 78, 134, 197, 18, 67, + 217, 5, 3, 70, 76, 85, 4, 184, 185, 18, 5, 69, 82, 32, 84, 82, 161, 17, + 2, 79, 67, 4, 128, 189, 21, 3, 68, 69, 83, 221, 59, 3, 85, 84, 72, 6, 26, + 66, 37, 2, 80, 80, 2, 209, 213, 22, 4, 83, 84, 82, 85, 4, 26, 82, 171, + 212, 22, 79, 2, 169, 140, 21, 2, 69, 83, 6, 200, 211, 14, 9, 85, 83, 72, + 73, 78, 71, 32, 85, 80, 168, 134, 2, 3, 82, 79, 71, 219, 142, 6, 69, 6, + 26, 84, 235, 180, 18, 86, 4, 186, 198, 18, 85, 215, 57, 82, 8, 132, 1, 5, + 77, 65, 76, 76, 32, 204, 178, 20, 6, 84, 65, 78, 68, 83, 84, 209, 233, 1, + 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, 80, 82, 43, 84, + 2, 157, 199, 18, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, 215, 209, 22, 77, + 30, 36, 3, 72, 69, 32, 171, 148, 17, 82, 28, 186, 2, 65, 130, 1, 67, 132, + 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, 86, 69, 30, 87, 156, + 21, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, 73, 232, 242, 4, 6, + 71, 69, 78, 84, 76, 69, 204, 199, 10, 13, 75, 69, 69, 80, 73, 78, 71, 32, + 83, 84, 73, 76, 76, 177, 220, 3, 7, 74, 79, 89, 79, 85, 83, 32, 6, 58, + 82, 141, 140, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, 4, 220, 217, 19, 8, + 79, 85, 83, 73, 78, 71, 32, 84, 247, 179, 3, 77, 6, 160, 129, 1, 2, 65, + 85, 152, 215, 18, 9, 82, 69, 65, 84, 73, 86, 69, 32, 72, 233, 180, 1, 9, + 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 163, 250, 22, 73, 2, 129, 215, 19, + 2, 32, 69, 4, 242, 152, 21, 69, 209, 198, 1, 5, 65, 78, 68, 69, 82, 4, + 174, 198, 6, 65, 217, 146, 6, 14, 79, 82, 75, 32, 79, 78, 32, 84, 72, 69, + 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, 82, 65, 71, 65, 78, + 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 184, 130, 15, 7, 78, 68, + 85, 32, 84, 69, 77, 236, 33, 4, 75, 73, 78, 71, 150, 235, 1, 66, 241, + 128, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, 119, 45, 6, 144, + 234, 4, 2, 66, 82, 156, 152, 16, 6, 86, 79, 76, 84, 65, 71, 149, 47, 9, + 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, 69, 69, 68, 32, + 84, 82, 65, 73, 78, 233, 166, 5, 6, 72, 69, 69, 76, 69, 68, 5, 133, 145, + 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, 32, 78, 200, 1, + 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, 69, 84, 84, 69, + 82, 32, 234, 170, 1, 86, 215, 207, 19, 73, 2, 135, 142, 16, 79, 194, 1, + 194, 1, 65, 74, 83, 218, 161, 1, 66, 162, 3, 78, 150, 2, 68, 2, 71, 2, + 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, 89, 178, 199, + 21, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, 72, 65, 73, + 67, 32, 4, 130, 240, 22, 87, 151, 14, 89, 42, 76, 5, 77, 65, 76, 76, 32, + 254, 147, 23, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 182, 167, 1, 87, 46, + 89, 130, 142, 5, 75, 178, 185, 16, 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, + 3, 85, 2, 171, 145, 8, 83, 92, 166, 1, 76, 52, 3, 78, 69, 89, 46, 82, + 130, 6, 84, 138, 1, 85, 210, 130, 16, 83, 138, 203, 2, 67, 172, 149, 3, + 6, 77, 79, 84, 72, 69, 84, 214, 163, 1, 79, 155, 3, 80, 6, 180, 212, 15, + 4, 76, 79, 87, 32, 163, 189, 7, 69, 4, 238, 164, 21, 66, 137, 156, 1, 2, + 32, 80, 50, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 137, 5, 2, 83, 69, 44, + 198, 1, 66, 108, 4, 76, 73, 78, 69, 28, 17, 79, 78, 69, 32, 69, 73, 71, + 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 88, 10, 83, 67, 65, 78, 32, 76, + 73, 78, 69, 45, 46, 84, 210, 239, 20, 69, 251, 4, 77, 6, 44, 5, 76, 65, + 67, 75, 32, 243, 213, 22, 65, 4, 38, 79, 153, 215, 21, 3, 72, 69, 88, 2, + 139, 215, 21, 67, 2, 173, 244, 20, 2, 32, 69, 14, 160, 166, 16, 3, 49, + 51, 53, 198, 231, 6, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, 142, 141, + 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 215, 246, 20, 82, 8, 26, + 85, 215, 245, 20, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, 32, 4, + 208, 163, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 151, 179, 13, 83, 7, 11, + 32, 4, 140, 159, 7, 2, 82, 65, 155, 175, 15, 70, 10, 26, 32, 195, 129, + 22, 69, 8, 86, 80, 188, 175, 18, 5, 66, 69, 86, 69, 82, 204, 80, 3, 83, + 80, 82, 159, 137, 4, 68, 2, 175, 175, 14, 69, 12, 48, 6, 82, 71, 76, 65, + 83, 83, 77, 2, 83, 69, 5, 153, 208, 19, 14, 32, 87, 73, 84, 72, 32, 70, + 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, 71, + 65, 82, 169, 132, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 147, 183, 21, + 68, 7, 242, 135, 23, 74, 3, 83, 8, 106, 83, 208, 243, 21, 11, 78, 68, 82, + 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 139, 136, 1, 84, 2, + 171, 246, 21, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 220, 180, + 16, 2, 71, 73, 193, 207, 4, 2, 65, 67, 12, 60, 3, 72, 69, 78, 129, 133, + 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 227, 167, 19, 45, 4, + 26, 87, 203, 168, 21, 66, 2, 21, 3, 73, 84, 72, 2, 141, 168, 3, 2, 32, + 68, 2, 193, 169, 12, 6, 84, 73, 79, 78, 32, 80, 2, 253, 240, 21, 2, 83, + 73, 202, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 214, + 2, 78, 132, 37, 7, 90, 65, 75, 65, 89, 65, 32, 237, 137, 19, 9, 32, 76, + 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 252, 249, 10, 18, 72, 79, 67, 75, + 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 155, 179, 1, 83, + 4, 250, 184, 15, 82, 187, 212, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, + 9, 79, 71, 82, 65, 80, 72, 73, 67, 32, 8, 64, 4, 67, 65, 76, 32, 105, 8, + 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 175, 255, 21, 87, 5, + 129, 188, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 229, + 232, 21, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, + 79, 78, 32, 242, 3, 67, 72, 2, 68, 69, 248, 5, 5, 78, 85, 77, 66, 69, 22, + 84, 220, 204, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 144, 186, 1, 5, 69, + 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 236, 240, 8, + 5, 86, 65, 82, 73, 65, 158, 175, 4, 70, 230, 46, 73, 131, 222, 1, 83, 32, + 232, 1, 5, 66, 79, 84, 84, 79, 22, 70, 82, 77, 62, 84, 140, 1, 4, 76, 73, + 78, 75, 188, 142, 1, 4, 83, 69, 67, 79, 150, 255, 5, 79, 228, 238, 5, 6, + 82, 69, 86, 69, 82, 83, 148, 147, 9, 5, 72, 69, 65, 86, 69, 165, 39, 3, + 69, 65, 82, 2, 175, 185, 22, 77, 6, 48, 3, 79, 85, 82, 229, 175, 22, 3, + 73, 82, 83, 4, 218, 184, 22, 84, 27, 32, 4, 36, 3, 73, 68, 68, 235, 144, + 22, 65, 2, 211, 253, 12, 76, 8, 34, 72, 46, 87, 131, 226, 7, 79, 4, 222, + 236, 8, 82, 181, 192, 13, 2, 73, 82, 2, 191, 183, 22, 79, 4, 36, 3, 76, + 79, 83, 167, 173, 20, 79, 2, 129, 183, 22, 3, 73, 78, 71, 36, 104, 20, + 83, 67, 82, 73, 80, 84, 73, 79, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, + 82, 32, 247, 139, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, + 32, 84, 8, 76, 69, 70, 84, 32, 84, 79, 32, 76, 5, 79, 86, 69, 82, 76, 20, + 2, 83, 85, 202, 186, 14, 82, 152, 153, 5, 8, 70, 85, 76, 76, 32, 83, 85, + 82, 173, 207, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, + 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 167, 220, 20, 66, + 2, 247, 205, 20, 68, 4, 180, 166, 21, 10, 77, 73, 68, 68, 76, 69, 32, 65, + 78, 68, 195, 163, 1, 82, 2, 231, 162, 16, 65, 18, 72, 12, 82, 82, 79, 85, + 78, 68, 32, 70, 82, 79, 77, 32, 163, 236, 16, 66, 16, 82, 76, 200, 163, + 11, 3, 85, 80, 80, 202, 182, 9, 66, 222, 155, 1, 65, 143, 82, 82, 6, 186, + 163, 11, 79, 175, 164, 11, 69, 2, 195, 180, 8, 82, 148, 1, 92, 10, 65, + 76, 76, 89, 32, 77, 65, 82, 75, 32, 41, 9, 69, 76, 69, 71, 82, 65, 80, + 72, 32, 10, 138, 132, 21, 70, 70, 84, 203, 83, 79, 138, 1, 140, 1, 11, + 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 205, 179, 12, 17, 76, 73, 78, + 69, 32, 70, 69, 69, 68, 32, 83, 69, 80, 65, 82, 65, 84, 136, 1, 182, 1, + 65, 58, 68, 200, 2, 5, 72, 79, 85, 82, 32, 214, 1, 74, 28, 4, 70, 69, 66, + 82, 64, 2, 77, 65, 144, 137, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, + 4, 79, 67, 84, 79, 4, 232, 166, 19, 3, 85, 71, 85, 213, 154, 1, 2, 80, + 82, 64, 44, 3, 65, 89, 32, 217, 141, 15, 2, 69, 67, 62, 66, 84, 130, 177, + 5, 69, 66, 70, 70, 78, 26, 83, 219, 161, 16, 79, 34, 34, 72, 90, 87, 187, + 156, 22, 69, 8, 36, 3, 73, 82, 84, 139, 128, 21, 82, 6, 26, 89, 175, 155, + 21, 69, 5, 247, 194, 21, 45, 24, 26, 69, 243, 235, 22, 79, 22, 36, 3, 78, + 84, 89, 135, 239, 21, 76, 21, 147, 232, 14, 45, 50, 78, 84, 222, 174, 5, + 69, 66, 70, 70, 78, 26, 83, 206, 206, 15, 90, 143, 83, 79, 20, 42, 87, + 186, 176, 5, 72, 175, 234, 16, 69, 14, 26, 69, 159, 234, 22, 79, 12, 36, + 3, 78, 84, 89, 179, 237, 21, 76, 11, 159, 212, 16, 45, 6, 24, 2, 65, 78, + 35, 85, 2, 11, 85, 2, 131, 221, 16, 65, 4, 206, 210, 22, 78, 143, 5, 76, + 4, 218, 198, 22, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, + 80, 5, 233, 225, 8, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, + 65, 84, 69, 65, 65, 14, 69, 82, 73, 65, 76, 32, 65, 82, 65, 77, 65, 73, + 67, 32, 62, 64, 7, 78, 85, 77, 66, 69, 82, 32, 234, 18, 76, 239, 249, 18, + 83, 16, 26, 84, 171, 140, 15, 79, 10, 138, 143, 11, 87, 186, 137, 1, 69, + 195, 225, 8, 72, 132, 3, 202, 1, 67, 234, 1, 68, 218, 6, 70, 132, 2, 5, + 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, 76, + 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 175, 211, + 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 144, 144, 16, 3, 73, 78, 71, + 129, 216, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 151, 160, 22, + 77, 4, 34, 32, 161, 141, 10, 2, 83, 32, 2, 161, 143, 11, 8, 70, 79, 78, + 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 205, 201, 21, 20, + 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, 86, + 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 209, 185, 17, 4, + 65, 78, 32, 82, 136, 1, 200, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, 69, + 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, 85, + 77, 66, 69, 82, 32, 222, 209, 8, 82, 201, 246, 2, 6, 80, 76, 65, 67, 69, + 72, 2, 237, 157, 22, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 217, 224, 20, + 7, 84, 72, 82, 69, 69, 32, 81, 4, 178, 222, 20, 72, 43, 81, 122, 212, 1, + 10, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 5, 75, 65, 82, 79, 82, 0, + 4, 76, 65, 75, 72, 250, 226, 9, 69, 66, 70, 94, 78, 26, 83, 78, 84, 220, + 243, 4, 8, 80, 82, 69, 70, 73, 88, 69, 68, 167, 41, 79, 6, 26, 84, 175, + 195, 21, 79, 4, 236, 128, 6, 2, 69, 78, 223, 188, 16, 87, 5, 191, 140, + 22, 65, 16, 72, 5, 73, 78, 73, 84, 89, 85, 9, 79, 82, 77, 65, 84, 73, 79, + 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 237, 134, 8, 4, + 87, 73, 84, 72, 12, 42, 83, 253, 134, 16, 4, 68, 69, 83, 75, 10, 156, + 245, 6, 5, 69, 80, 65, 82, 65, 191, 214, 9, 79, 4, 11, 32, 4, 232, 136, + 22, 15, 65, 82, 65, 66, 73, 67, 32, 70, 79, 82, 77, 32, 83, 72, 65, 1, + 14, 83, 89, 77, 77, 69, 84, 82, 73, 67, 32, 83, 87, 65, 80, 10, 80, 6, + 76, 65, 84, 73, 78, 32, 234, 185, 14, 83, 221, 143, 7, 4, 78, 85, 77, 66, + 6, 64, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 27, 76, 2, 21, 3, + 76, 32, 76, 2, 205, 217, 20, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, + 73, 79, 78, 65, 76, 32, 80, 65, 233, 141, 15, 2, 69, 82, 114, 72, 6, 72, + 76, 65, 86, 73, 32, 225, 1, 7, 82, 84, 72, 73, 65, 78, 32, 54, 48, 7, 76, + 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 234, 132, 10, 84, 226, 118, 65, + 22, 68, 34, 76, 22, 77, 50, 87, 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, + 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 199, 105, 80, + 60, 22, 76, 251, 1, 78, 44, 11, 69, 44, 29, 5, 84, 84, 69, 82, 32, 44, + 250, 130, 10, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, + 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, + 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 16, + 33, 6, 85, 77, 66, 69, 82, 32, 16, 246, 249, 10, 84, 202, 253, 3, 79, + 171, 243, 2, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, + 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 219, 181, 20, 69, 4, 96, 6, 86, + 69, 82, 65, 71, 69, 173, 237, 15, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, + 79, 73, 78, 2, 209, 238, 15, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, + 73, 77, 20, 2, 85, 78, 200, 203, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 182, 231, 12, 73, 198, 1, 79, 251, 49, + 68, 2, 191, 180, 19, 69, 4, 202, 180, 19, 68, 203, 201, 2, 73, 28, 94, + 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 202, 6, 82, 144, 200, 11, 2, + 67, 65, 43, 73, 8, 164, 1, 17, 73, 78, 69, 65, 82, 32, 65, 78, 78, 79, + 84, 65, 84, 73, 79, 78, 32, 201, 200, 4, 17, 79, 67, 75, 69, 68, 32, 70, + 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 232, 130, 8, 3, 65, 78, 67, + 170, 128, 8, 84, 199, 175, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, + 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 133, 174, 19, 22, 66, 69, 83, 73, + 68, 69, 32, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, + 4, 244, 174, 19, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 158, + 143, 4, 76, 190, 160, 15, 79, 139, 201, 2, 68, 36, 56, 2, 69, 82, 245, 4, + 7, 73, 83, 73, 66, 76, 69, 32, 30, 48, 3, 83, 69, 32, 153, 2, 4, 84, 69, + 68, 32, 12, 180, 1, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, 78, 87, + 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 84, 73, + 80, 166, 195, 19, 87, 230, 18, 77, 183, 147, 1, 66, 4, 196, 213, 19, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 195, 175, 2, 32, 2, 229, 237, 19, 2, 32, + 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 150, 195, 11, 80, 170, + 169, 4, 69, 148, 183, 1, 2, 79, 72, 136, 131, 2, 4, 85, 78, 68, 69, 163, + 83, 81, 2, 151, 179, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, + 187, 196, 6, 65, 5, 225, 204, 17, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, + 89, 75, 6, 38, 84, 138, 230, 18, 80, 143, 68, 83, 2, 207, 238, 15, 73, + 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 146, 11, 10, 73, 71, + 83, 65, 87, 32, 80, 85, 90, 90, 202, 1, 120, 5, 67, 75, 45, 79, 45, 32, + 7, 80, 65, 78, 69, 83, 69, 32, 184, 2, 7, 86, 65, 78, 69, 83, 69, 32, + 183, 190, 22, 82, 2, 233, 226, 17, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, + 128, 153, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 140, 196, 3, 3, + 66, 65, 78, 250, 171, 3, 68, 244, 171, 12, 3, 71, 79, 66, 204, 105, 2, + 79, 71, 181, 98, 16, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 66, 69, + 71, 73, 78, 2, 199, 32, 65, 2, 197, 209, 12, 7, 79, 83, 84, 32, 79, 70, + 70, 182, 1, 172, 2, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, + 71, 78, 32, 76, 2, 76, 69, 40, 4, 82, 73, 71, 72, 224, 6, 2, 80, 65, 176, + 3, 14, 84, 85, 82, 78, 69, 68, 32, 80, 65, 68, 65, 32, 80, 73, 88, 5, 83, + 73, 71, 78, 32, 140, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 255, 192, 20, 68, 6, 52, 2, 75, 69, 228, 129, 16, 2, 80, 69, 175, 5, 67, + 2, 187, 134, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, + 32, 82, 69, 82, 69, 78, 71, 2, 171, 243, 20, 71, 94, 230, 1, 68, 26, 73, + 48, 2, 75, 65, 66, 78, 130, 1, 66, 2, 67, 2, 71, 16, 2, 80, 65, 56, 2, + 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 186, 174, 20, 65, 150, 132, 2, 72, + 2, 76, 2, 77, 2, 87, 2, 89, 186, 2, 69, 2, 79, 3, 85, 8, 222, 3, 68, 15, + 65, 7, 164, 167, 5, 3, 32, 75, 65, 167, 145, 17, 73, 7, 11, 32, 4, 22, + 83, 215, 2, 77, 2, 145, 137, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, + 167, 1, 65, 7, 33, 6, 32, 76, 69, 76, 69, 84, 5, 29, 5, 32, 82, 65, 83, + 87, 2, 135, 192, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 129, + 178, 22, 3, 67, 69, 82, 5, 145, 133, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, + 4, 70, 85, 71, 65, 8, 18, 65, 55, 84, 5, 17, 2, 32, 77, 2, 11, 85, 2, + 147, 144, 22, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, + 2, 233, 232, 16, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, + 174, 1, 65, 90, 76, 86, 80, 168, 240, 6, 10, 84, 73, 82, 84, 65, 32, 84, + 85, 77, 69, 138, 219, 11, 87, 128, 149, 2, 7, 73, 83, 69, 78, 45, 73, 83, + 153, 206, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 205, 161, 21, 2, 78, + 68, 5, 11, 32, 2, 181, 236, 21, 2, 65, 68, 6, 38, 85, 141, 175, 22, 3, + 73, 78, 71, 4, 184, 243, 17, 2, 78, 71, 211, 197, 3, 72, 4, 38, 73, 149, + 236, 17, 3, 65, 78, 71, 2, 217, 253, 15, 3, 83, 69, 76, 4, 208, 214, 12, + 5, 82, 65, 78, 71, 75, 171, 138, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, + 204, 165, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 239, 212, 10, + 76, 5, 233, 218, 17, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, + 64, 4, 87, 85, 76, 85, 182, 251, 15, 80, 165, 210, 1, 7, 68, 73, 82, 71, + 65, 32, 77, 5, 221, 180, 21, 5, 32, 77, 69, 78, 68, 6, 26, 65, 243, 252, + 15, 79, 4, 218, 252, 15, 82, 139, 224, 5, 76, 5, 25, 4, 32, 77, 69, 76, + 2, 131, 170, 22, 73, 4, 36, 3, 76, 76, 89, 207, 254, 18, 65, 2, 155, 238, + 18, 70, 4, 228, 253, 20, 2, 89, 83, 147, 95, 73, 6, 148, 134, 14, 2, 71, + 71, 138, 230, 4, 80, 147, 161, 3, 78, 192, 24, 74, 65, 178, 78, 69, 202, + 1, 72, 178, 50, 73, 178, 2, 78, 50, 79, 111, 82, 198, 10, 200, 1, 5, 73, + 84, 72, 73, 32, 230, 4, 78, 168, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, + 3, 87, 73, 32, 240, 8, 7, 89, 65, 72, 32, 76, 73, 32, 252, 163, 4, 6, 75, + 84, 79, 86, 73, 75, 219, 143, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 174, 2, 83, 234, 115, 68, 156, 206, 3, 2, 86, 79, 168, 136, + 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 154, 155, 9, 65, 197, + 213, 1, 6, 69, 78, 85, 77, 69, 82, 90, 210, 1, 68, 210, 189, 18, 65, 82, + 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, + 89, 186, 2, 69, 3, 79, 10, 38, 68, 194, 163, 22, 72, 187, 2, 65, 6, 186, + 174, 20, 68, 134, 245, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, 167, + 236, 10, 69, 10, 242, 190, 18, 67, 98, 78, 222, 160, 3, 65, 239, 1, 86, + 230, 4, 42, 71, 221, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, 73, + 32, 82, 65, 68, 73, 67, 65, 76, 32, 153, 227, 19, 2, 65, 82, 172, 3, 130, + 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, 2, + 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, + 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 210, 6, 84, 226, 2, 86, + 50, 87, 210, 2, 89, 131, 244, 20, 85, 10, 56, 2, 82, 82, 238, 245, 20, + 71, 158, 101, 78, 195, 47, 88, 4, 138, 163, 21, 79, 207, 1, 73, 34, 58, + 65, 38, 73, 46, 76, 54, 79, 102, 82, 167, 215, 20, 69, 4, 190, 180, 5, + 77, 143, 180, 13, 68, 6, 206, 233, 20, 84, 186, 112, 82, 151, 70, 71, 6, + 138, 254, 18, 79, 154, 243, 1, 65, 251, 151, 1, 85, 10, 62, 76, 210, 130, + 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 173, 193, 3, 4, 84, 32, + 79, 70, 6, 42, 73, 246, 201, 9, 65, 159, 239, 10, 85, 2, 151, 189, 13, + 83, 28, 58, 65, 70, 76, 74, 79, 190, 135, 10, 72, 163, 171, 10, 73, 6, + 38, 85, 138, 129, 22, 82, 219, 5, 86, 2, 165, 249, 20, 2, 76, 68, 8, 42, + 65, 206, 151, 7, 73, 171, 241, 13, 79, 4, 234, 156, 22, 78, 3, 87, 10, + 214, 250, 20, 82, 204, 1, 2, 77, 80, 206, 98, 86, 134, 5, 76, 223, 56, + 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 214, 156, 20, 65, 183, + 198, 1, 69, 8, 38, 83, 158, 190, 13, 86, 175, 2, 80, 4, 200, 220, 18, 5, + 84, 73, 78, 71, 85, 199, 190, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 214, + 191, 12, 87, 172, 137, 9, 2, 32, 78, 222, 23, 79, 211, 56, 71, 7, 25, 4, + 84, 69, 68, 32, 4, 184, 148, 7, 3, 67, 76, 73, 135, 194, 14, 84, 2, 171, + 170, 3, 69, 6, 162, 226, 20, 65, 158, 166, 1, 85, 219, 16, 89, 20, 106, + 65, 38, 78, 32, 3, 86, 69, 78, 216, 214, 8, 6, 77, 66, 82, 79, 73, 68, + 158, 150, 13, 73, 243, 19, 89, 6, 202, 152, 16, 82, 179, 255, 5, 84, 4, + 158, 25, 67, 219, 192, 21, 84, 5, 203, 198, 21, 73, 28, 74, 65, 50, 73, + 62, 76, 38, 82, 150, 20, 69, 138, 177, 21, 79, 223, 23, 85, 6, 206, 188, + 13, 84, 182, 195, 8, 67, 131, 22, 78, 8, 146, 177, 12, 69, 222, 186, 9, + 71, 230, 19, 82, 199, 21, 83, 4, 190, 209, 20, 85, 175, 196, 1, 89, 4, + 220, 232, 20, 2, 65, 71, 151, 172, 1, 79, 14, 58, 79, 52, 2, 82, 65, 218, + 204, 18, 72, 171, 131, 2, 65, 7, 202, 206, 21, 76, 229, 34, 5, 32, 83, + 76, 79, 87, 4, 214, 196, 21, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, + 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 150, 205, 21, 78, 199, 13, 73, + 4, 34, 70, 177, 252, 20, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, + 82, 2, 255, 211, 18, 85, 6, 26, 65, 159, 146, 22, 77, 4, 250, 245, 21, + 82, 175, 28, 68, 2, 175, 19, 32, 6, 26, 82, 199, 142, 22, 79, 4, 134, + 251, 21, 83, 215, 22, 78, 6, 26, 78, 207, 250, 21, 67, 4, 26, 83, 247, + 143, 22, 67, 2, 171, 130, 21, 69, 4, 154, 250, 21, 68, 215, 22, 82, 2, + 253, 140, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, + 174, 249, 21, 77, 191, 19, 67, 8, 38, 65, 238, 209, 21, 71, 187, 58, 69, + 4, 178, 181, 13, 84, 139, 218, 8, 70, 6, 194, 248, 21, 70, 2, 78, 215, + 22, 68, 5, 209, 146, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 135, + 190, 21, 65, 4, 194, 190, 21, 76, 183, 51, 65, 4, 156, 185, 11, 2, 78, + 73, 203, 248, 8, 76, 12, 34, 82, 34, 85, 195, 189, 21, 79, 4, 138, 235, + 20, 84, 179, 81, 78, 6, 26, 78, 235, 139, 22, 84, 4, 198, 225, 20, 84, + 179, 171, 1, 68, 6, 26, 79, 143, 240, 21, 69, 4, 226, 245, 21, 83, 215, + 22, 84, 10, 40, 2, 78, 69, 22, 80, 199, 197, 21, 76, 5, 139, 247, 5, 83, + 4, 170, 132, 10, 80, 215, 173, 2, 69, 10, 54, 82, 242, 240, 20, 76, 166, + 1, 79, 151, 152, 1, 73, 4, 152, 144, 12, 2, 73, 86, 229, 218, 6, 2, 79, + 70, 18, 62, 65, 42, 73, 214, 255, 9, 79, 186, 186, 11, 85, 195, 9, 69, 6, + 178, 186, 21, 73, 214, 79, 80, 3, 84, 6, 232, 175, 12, 3, 71, 72, 84, + 150, 156, 9, 86, 143, 39, 67, 74, 170, 1, 65, 86, 67, 46, 69, 62, 72, + 166, 1, 73, 46, 76, 62, 80, 106, 84, 234, 159, 17, 87, 182, 145, 1, 78, + 190, 50, 79, 182, 155, 1, 77, 134, 100, 81, 254, 32, 75, 235, 47, 85, 6, + 192, 168, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 138, 195, 18, 76, + 175, 28, 89, 4, 150, 198, 10, 82, 229, 186, 5, 2, 72, 79, 8, 242, 203, + 20, 67, 234, 49, 65, 146, 8, 76, 143, 129, 1, 69, 10, 18, 69, 39, 79, 4, + 138, 253, 20, 76, 147, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 175, 233, + 21, 79, 4, 222, 190, 15, 72, 137, 200, 4, 7, 65, 73, 76, 69, 68, 32, 66, + 4, 148, 162, 17, 2, 67, 75, 235, 223, 4, 76, 6, 26, 65, 227, 199, 21, 73, + 4, 250, 237, 21, 86, 199, 21, 83, 10, 50, 69, 34, 73, 178, 177, 18, 82, + 207, 130, 3, 79, 4, 170, 203, 21, 65, 171, 22, 69, 2, 163, 231, 21, 82, + 12, 34, 69, 34, 79, 167, 245, 20, 65, 4, 202, 242, 21, 65, 219, 16, 80, + 6, 26, 80, 151, 236, 21, 78, 5, 239, 177, 21, 80, 30, 82, 65, 98, 73, 34, + 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 183, 176, 21, 69, 6, 38, 78, + 182, 224, 20, 66, 139, 24, 76, 2, 33, 6, 78, 69, 68, 32, 76, 69, 2, 159, + 167, 13, 65, 4, 190, 195, 21, 71, 143, 39, 76, 4, 150, 160, 18, 78, 231, + 225, 1, 79, 6, 214, 159, 16, 73, 186, 178, 4, 65, 251, 151, 1, 69, 4, + 242, 184, 20, 66, 147, 38, 84, 5, 215, 214, 18, 32, 4, 170, 157, 12, 65, + 129, 136, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, 72, 32, 2, 73, 78, + 30, 79, 39, 82, 6, 32, 2, 76, 75, 135, 193, 21, 84, 5, 11, 32, 2, 11, 69, + 2, 17, 2, 78, 67, 2, 201, 158, 17, 2, 76, 79, 4, 184, 174, 21, 2, 65, 80, + 183, 51, 83, 4, 198, 185, 20, 73, 223, 48, 69, 4, 210, 253, 21, 68, 3, + 69, 4, 130, 182, 20, 77, 159, 196, 1, 82, 4, 166, 172, 21, 79, 227, 80, + 65, 2, 197, 228, 19, 2, 69, 76, 184, 1, 92, 2, 76, 69, 148, 2, 5, 83, 73, + 71, 78, 32, 238, 249, 16, 65, 174, 9, 86, 175, 136, 3, 68, 110, 44, 5, + 84, 84, 69, 82, 32, 235, 185, 21, 78, 108, 214, 128, 17, 76, 38, 78, 174, + 90, 82, 206, 55, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, + 79, 182, 56, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, + 80, 162, 7, 69, 222, 61, 70, 2, 72, 2, 77, 3, 89, 22, 94, 67, 138, 1, 83, + 138, 145, 18, 65, 170, 1, 78, 230, 179, 1, 74, 150, 3, 85, 211, 235, 1, + 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, 32, 65, 78, 85, 83, 86, + 65, 82, 65, 32, 65, 255, 145, 18, 65, 2, 185, 131, 19, 3, 66, 79, 86, 4, + 244, 135, 12, 6, 80, 65, 67, 73, 78, 71, 155, 249, 4, 73, 162, 2, 62, 32, + 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 154, 2, 140, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 236, 159, 18, 10, 68, 73, 71, 82, + 65, 80, 72, 32, 75, 79, 234, 175, 1, 73, 247, 138, 1, 77, 146, 2, 186, 1, + 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, 83, 226, 1, 68, 2, 71, 2, 72, 2, + 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, 126, 87, 46, 89, 154, 236, 21, 69, + 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, 78, 85, 32, 45, 7, 82, 67, 72, 65, + 73, 67, 32, 8, 130, 7, 84, 142, 214, 21, 67, 215, 22, 80, 8, 38, 89, 170, + 206, 21, 87, 235, 36, 69, 4, 142, 243, 21, 69, 3, 73, 20, 50, 73, 194, + 242, 21, 65, 2, 69, 2, 79, 3, 85, 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, + 32, 78, 71, 36, 50, 73, 218, 241, 21, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, + 78, 78, 65, 78, 32, 26, 96, 15, 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, + 84, 79, 78, 69, 45, 69, 5, 84, 79, 78, 69, 45, 14, 210, 240, 21, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 12, 142, 240, 21, 50, 2, 51, 2, + 52, 2, 53, 2, 55, 3, 56, 13, 210, 239, 21, 65, 2, 69, 2, 73, 2, 79, 3, + 85, 76, 76, 5, 77, 65, 76, 76, 32, 210, 238, 21, 65, 2, 69, 2, 73, 2, 79, + 3, 85, 66, 142, 1, 72, 2, 82, 54, 75, 46, 84, 30, 87, 46, 89, 254, 156, + 18, 78, 242, 142, 3, 83, 198, 27, 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, + 3, 85, 10, 190, 237, 21, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 138, 237, 21, + 65, 2, 69, 2, 79, 3, 85, 4, 222, 236, 21, 79, 3, 85, 8, 194, 236, 21, 65, + 2, 69, 2, 73, 3, 79, 6, 150, 236, 21, 65, 2, 79, 3, 85, 2, 189, 207, 19, + 5, 79, 73, 67, 69, 68, 8, 52, 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, + 86, 2, 251, 216, 7, 69, 2, 89, 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, + 77, 73, 45, 86, 2, 21, 3, 79, 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, + 2, 239, 157, 21, 78, 172, 1, 228, 1, 7, 76, 69, 84, 84, 69, 82, 32, 132, + 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, + 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 178, 236, + 11, 68, 205, 183, 9, 3, 67, 79, 78, 94, 230, 1, 65, 214, 128, 4, 74, 154, + 253, 13, 68, 46, 84, 146, 4, 86, 214, 20, 85, 210, 200, 1, 73, 158, 190, + 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, + 77, 2, 82, 2, 87, 2, 89, 186, 2, 69, 3, 79, 7, 150, 230, 21, 65, 3, 73, + 22, 122, 67, 90, 68, 58, 70, 54, 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, + 84, 69, 32, 83, 69, 149, 210, 20, 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, + 83, 73, 78, 71, 32, 83, 203, 149, 20, 73, 2, 141, 219, 20, 2, 80, 73, 4, + 11, 79, 4, 132, 211, 20, 2, 85, 66, 151, 145, 1, 84, 4, 204, 148, 20, 5, + 73, 76, 76, 69, 68, 175, 54, 76, 6, 18, 69, 23, 80, 2, 203, 196, 10, 67, + 4, 156, 241, 3, 2, 65, 67, 195, 232, 16, 73, 10, 192, 238, 5, 2, 82, 69, + 166, 204, 1, 75, 218, 193, 10, 67, 226, 182, 1, 86, 223, 234, 1, 65, 20, + 66, 65, 134, 255, 3, 86, 170, 147, 14, 69, 2, 85, 211, 200, 1, 73, 6, + 244, 152, 16, 8, 76, 84, 69, 82, 78, 65, 84, 69, 210, 200, 5, 65, 3, 73, + 96, 148, 1, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 5, 83, 73, 71, 78, 32, + 44, 5, 84, 79, 78, 69, 32, 92, 6, 86, 79, 87, 69, 76, 32, 247, 238, 19, + 68, 56, 154, 206, 16, 72, 190, 173, 1, 79, 230, 236, 1, 78, 246, 175, 1, + 75, 2, 80, 2, 83, 2, 84, 254, 68, 66, 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, + 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, 186, 2, 65, 3, 73, 4, 242, 204, 4, 67, + 201, 142, 17, 2, 83, 72, 6, 36, 5, 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, + 32, 80, 2, 197, 155, 17, 3, 76, 79, 80, 10, 250, 156, 21, 69, 2, 85, 151, + 64, 79, 34, 24, 2, 76, 86, 23, 89, 2, 175, 230, 19, 73, 33, 52, 5, 66, + 79, 65, 82, 68, 37, 4, 67, 65, 80, 32, 5, 213, 172, 18, 4, 32, 65, 78, + 68, 26, 142, 197, 16, 78, 146, 134, 3, 65, 162, 34, 68, 171, 29, 84, 186, + 13, 202, 1, 65, 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, + 83, 67, 82, 73, 80, 84, 32, 184, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, + 74, 75, 73, 32, 145, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, + 82, 79, 83, 72, 84, 72, 73, 32, 143, 180, 21, 78, 136, 1, 220, 1, 4, 68, + 73, 71, 73, 20, 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 36, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, + 2, 5, 83, 73, 71, 78, 32, 198, 1, 86, 207, 155, 8, 70, 8, 203, 193, 15, + 84, 74, 182, 1, 75, 42, 84, 150, 229, 6, 78, 250, 135, 11, 68, 170, 160, + 3, 83, 82, 66, 2, 67, 2, 71, 2, 80, 2, 86, 254, 68, 72, 2, 74, 2, 76, 2, + 77, 2, 82, 2, 89, 2, 90, 187, 2, 65, 6, 166, 211, 21, 72, 2, 75, 187, 2, + 65, 12, 182, 237, 17, 84, 202, 229, 3, 72, 187, 2, 65, 8, 146, 149, 14, + 84, 159, 151, 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, + 65, 175, 131, 20, 83, 4, 26, 82, 135, 133, 20, 73, 2, 177, 132, 19, 6, + 69, 83, 67, 69, 78, 84, 6, 26, 79, 191, 254, 11, 65, 4, 134, 254, 11, 85, + 179, 213, 9, 84, 4, 246, 188, 6, 79, 199, 168, 7, 73, 2, 179, 153, 20, + 76, 12, 72, 2, 66, 65, 22, 67, 20, 2, 68, 79, 162, 162, 19, 86, 223, 234, + 1, 65, 2, 155, 213, 20, 82, 2, 255, 243, 5, 65, 4, 56, 8, 85, 66, 76, 69, + 32, 82, 73, 78, 203, 170, 19, 84, 2, 199, 170, 19, 71, 14, 44, 5, 79, 87, + 69, 76, 32, 243, 158, 19, 73, 12, 44, 5, 83, 73, 71, 78, 32, 203, 142, + 21, 76, 10, 242, 237, 3, 86, 198, 226, 17, 69, 2, 73, 2, 79, 3, 85, 174, + 7, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 175, 167, 7, + 70, 172, 7, 22, 66, 147, 1, 67, 128, 4, 146, 214, 7, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, + 2, 69, 3, 70, 172, 3, 122, 68, 138, 212, 7, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 3, 67, 12, 254, 204, + 21, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, 67, 240, 2, + 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, 87, 69, 76, + 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, 12, 6, 86, + 79, 87, 69, 76, 32, 195, 199, 19, 68, 70, 176, 1, 20, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 233, 232, 16, + 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 82, 73, + 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, 76, 198, + 250, 20, 83, 146, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, 72, 8, + 190, 169, 21, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, 11, 83, + 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, 65, 44, 6, + 79, 79, 32, 84, 89, 80, 34, 85, 198, 191, 19, 73, 207, 134, 2, 69, 8, + 218, 198, 21, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 159, 231, 9, 32, 9, + 214, 152, 7, 85, 187, 173, 14, 75, 8, 18, 81, 31, 82, 4, 214, 197, 21, + 69, 3, 85, 4, 195, 210, 17, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, 80, + 30, 83, 46, 84, 54, 68, 2, 76, 214, 163, 21, 77, 2, 82, 2, 86, 2, 89, + 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 202, 194, 21, 65, 3, 79, 8, + 254, 164, 21, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 203, 194, 21, + 79, 6, 178, 164, 21, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, + 72, 202, 194, 21, 65, 3, 79, 4, 198, 194, 21, 65, 3, 79, 130, 1, 60, 4, + 73, 71, 78, 32, 221, 6, 6, 89, 77, 66, 79, 76, 32, 46, 202, 2, 65, 104, + 10, 83, 65, 77, 89, 79, 75, 32, 83, 65, 78, 22, 66, 118, 67, 86, 75, 74, + 82, 54, 84, 244, 191, 4, 3, 76, 69, 75, 244, 10, 6, 80, 72, 78, 65, 69, + 75, 180, 204, 11, 11, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 78, 152, + 97, 3, 78, 73, 75, 244, 251, 2, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, + 249, 13, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, + 65, 152, 189, 17, 4, 84, 84, 72, 65, 153, 220, 3, 2, 72, 83, 2, 215, 187, + 21, 78, 8, 38, 65, 209, 180, 20, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, + 78, 84, 189, 238, 19, 6, 82, 73, 89, 79, 79, 83, 4, 224, 244, 6, 12, 65, + 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 155, 140, 12, 79, 6, 188, + 163, 11, 2, 65, 75, 230, 209, 8, 72, 245, 77, 4, 79, 79, 77, 85, 4, 130, + 163, 11, 79, 133, 208, 5, 4, 69, 65, 72, 77, 4, 136, 168, 20, 8, 79, 65, + 78, 68, 65, 75, 72, 73, 253, 2, 4, 82, 73, 73, 83, 84, 128, 1, 3, 68, 65, + 80, 92, 10, 76, 69, 75, 32, 65, 84, 84, 65, 75, 32, 182, 1, 80, 72, 5, + 84, 85, 84, 69, 89, 78, 66, 47, 77, 24, 22, 45, 223, 3, 32, 20, 30, 80, + 238, 2, 66, 47, 77, 8, 138, 3, 73, 37, 3, 82, 65, 77, 20, 50, 80, 98, 66, + 130, 221, 10, 77, 187, 139, 10, 83, 12, 36, 3, 82, 65, 77, 251, 164, 21, + 73, 11, 11, 45, 8, 42, 66, 130, 221, 10, 77, 255, 158, 8, 80, 4, 182, + 232, 20, 85, 139, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, + 2, 21, 3, 72, 65, 77, 2, 223, 237, 15, 65, 20, 18, 45, 119, 32, 16, 34, + 66, 32, 2, 80, 73, 15, 77, 8, 30, 69, 37, 3, 85, 79, 78, 4, 35, 73, 4, + 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 229, 128, 21, 2, 75, 79, 2, 143, + 143, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, + 83, 73, 71, 78, 32, 4, 138, 181, 21, 65, 3, 81, 38, 102, 65, 54, 73, 30, + 79, 38, 89, 180, 228, 2, 5, 67, 79, 69, 78, 71, 250, 150, 7, 85, 235, + 183, 11, 69, 10, 218, 142, 3, 65, 174, 165, 18, 69, 2, 73, 3, 85, 7, 210, + 179, 21, 69, 3, 73, 6, 182, 179, 21, 69, 2, 77, 3, 79, 7, 146, 179, 21, + 65, 3, 89, 130, 1, 146, 1, 68, 88, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, + 83, 164, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 226, 156, 15, + 87, 219, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 179, 220, 11, 65, 4, + 166, 248, 9, 83, 255, 227, 1, 68, 90, 234, 1, 83, 130, 5, 66, 42, 71, + 246, 230, 5, 68, 222, 137, 4, 74, 214, 208, 7, 65, 82, 84, 222, 225, 1, + 76, 246, 189, 1, 78, 126, 67, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 81, 2, + 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 131, 175, + 21, 65, 2, 249, 238, 20, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 191, + 245, 9, 69, 10, 58, 83, 138, 205, 15, 86, 170, 251, 1, 78, 223, 160, 3, + 65, 4, 26, 72, 243, 222, 16, 85, 2, 11, 65, 2, 203, 136, 21, 68, 18, 246, + 202, 3, 86, 238, 253, 13, 65, 142, 222, 1, 73, 206, 134, 2, 69, 2, 79, 3, + 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 216, 2, 5, 83, 73, 71, + 78, 32, 170, 236, 15, 86, 227, 206, 3, 68, 94, 222, 1, 66, 42, 71, 42, + 74, 206, 230, 5, 68, 230, 162, 11, 82, 206, 55, 65, 82, 84, 230, 24, 85, + 210, 200, 1, 73, 158, 190, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 254, 68, + 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 226, 167, 21, 66, + 2, 72, 187, 2, 65, 6, 186, 167, 21, 71, 2, 72, 187, 2, 65, 6, 146, 167, + 21, 72, 2, 74, 187, 2, 65, 6, 178, 200, 15, 86, 170, 251, 1, 78, 223, + 160, 3, 65, 20, 92, 2, 83, 83, 140, 245, 19, 4, 87, 73, 70, 82, 158, 44, + 80, 216, 93, 2, 77, 79, 191, 18, 84, 13, 40, 4, 73, 78, 71, 32, 187, 230, + 20, 32, 8, 96, 4, 70, 65, 67, 69, 209, 212, 13, 14, 67, 65, 84, 32, 70, + 65, 67, 69, 32, 87, 73, 84, 72, 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, + 198, 211, 13, 83, 115, 67, 4, 236, 210, 14, 3, 69, 69, 76, 167, 183, 6, + 79, 4, 40, 4, 82, 69, 65, 78, 207, 131, 21, 65, 2, 33, 6, 32, 83, 84, 65, + 78, 68, 2, 129, 141, 18, 2, 65, 82, 2, 11, 79, 2, 155, 253, 9, 78, 128, + 41, 154, 1, 65, 190, 216, 1, 69, 242, 56, 73, 230, 83, 79, 146, 22, 85, + 94, 89, 182, 206, 7, 82, 168, 249, 8, 5, 32, 66, 32, 66, 65, 174, 157, 1, + 76, 227, 66, 70, 130, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, + 68, 56, 2, 79, 32, 222, 11, 82, 208, 6, 4, 83, 84, 32, 81, 76, 4, 84, 73, + 78, 32, 161, 168, 19, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 212, 131, 14, + 2, 32, 67, 183, 149, 6, 69, 2, 153, 173, 16, 6, 32, 83, 84, 73, 67, 75, + 4, 184, 192, 12, 5, 89, 32, 66, 69, 69, 183, 163, 8, 68, 174, 1, 200, 2, + 3, 72, 79, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, + 84, 79, 78, 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 128, 137, 4, 2, 75, 79, 188, 211, 12, 2, 89, 65, 162, + 157, 2, 69, 250, 46, 68, 248, 132, 1, 7, 67, 65, 78, 67, 69, 76, 76, 205, + 68, 6, 78, 73, 71, 71, 65, 72, 4, 210, 255, 20, 77, 3, 78, 94, 176, 1, 3, + 70, 79, 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, + 206, 212, 15, 78, 214, 165, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, + 89, 247, 30, 79, 8, 42, 70, 206, 235, 14, 83, 199, 247, 4, 84, 4, 246, + 204, 20, 79, 143, 62, 65, 10, 26, 72, 147, 156, 21, 79, 8, 32, 3, 77, 85, + 32, 231, 2, 79, 4, 182, 195, 20, 78, 195, 57, 71, 7, 17, 2, 32, 76, 4, + 202, 202, 20, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 151, + 153, 21, 79, 24, 190, 165, 6, 68, 34, 84, 206, 6, 78, 254, 246, 12, 66, + 2, 67, 2, 71, 2, 74, 167, 213, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, + 84, 32, 83, 71, 79, 4, 146, 151, 21, 72, 3, 83, 6, 26, 72, 151, 153, 21, + 79, 4, 11, 79, 4, 11, 32, 4, 250, 231, 14, 83, 199, 247, 4, 84, 6, 112, + 14, 69, 77, 73, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 137, 147, 3, + 8, 73, 71, 78, 32, 80, 65, 76, 73, 4, 174, 191, 20, 78, 195, 57, 76, 8, + 50, 84, 252, 209, 16, 2, 67, 65, 255, 193, 4, 69, 4, 154, 248, 20, 72, + 247, 30, 73, 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 226, 162, 17, 89, + 222, 35, 85, 210, 200, 1, 69, 2, 73, 207, 134, 2, 79, 11, 130, 150, 21, + 65, 2, 73, 2, 77, 3, 89, 4, 130, 198, 20, 65, 3, 79, 56, 32, 2, 71, 69, + 179, 142, 20, 73, 54, 26, 32, 231, 188, 13, 82, 50, 254, 1, 66, 44, 4, + 71, 82, 69, 69, 22, 79, 250, 1, 84, 186, 180, 11, 68, 36, 2, 85, 80, 168, + 180, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 224, 151, 4, + 5, 80, 85, 82, 80, 76, 12, 3, 82, 69, 68, 0, 6, 89, 69, 76, 76, 79, 87, + 207, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 251, 130, 19, + 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 200, 248, 7, 5, + 68, 79, 84, 32, 79, 173, 221, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, + 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 230, 193, + 19, 67, 226, 19, 68, 231, 26, 83, 6, 104, 3, 87, 79, 32, 161, 229, 17, + 17, 82, 73, 80, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, + 4, 162, 162, 17, 68, 205, 64, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, + 32, 79, 78, 69, 32, 82, 73, 78, 71, 6, 53, 11, 85, 65, 82, 84, 69, 82, + 32, 77, 79, 79, 78, 7, 167, 220, 6, 32, 138, 20, 150, 1, 67, 244, 45, 18, + 69, 80, 73, 71, 82, 65, 80, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, + 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 192, 7, 56, 8, 65, + 80, 73, 84, 65, 76, 32, 76, 239, 206, 19, 82, 190, 7, 76, 6, 69, 84, 84, + 69, 82, 32, 189, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 186, 7, 154, 2, + 65, 158, 3, 66, 154, 1, 67, 254, 1, 68, 250, 1, 69, 174, 2, 70, 82, 71, + 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 222, 2, 77, 114, + 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 174, 2, 83, 254, 2, 84, 142, + 3, 85, 234, 1, 86, 146, 1, 87, 118, 88, 50, 89, 167, 1, 90, 93, 172, 1, + 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 182, 65, 78, 166, 209, 4, 76, 236, + 154, 9, 6, 70, 82, 73, 67, 65, 78, 178, 136, 4, 86, 138, 209, 2, 65, 2, + 79, 2, 85, 3, 89, 66, 182, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, + 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, + 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 142, 183, 1, 65, 183, 182, 3, 77, 21, 60, 6, 32, 87, 73, 84, + 72, 32, 254, 67, 82, 175, 157, 20, 69, 14, 218, 66, 84, 146, 60, 70, 202, + 56, 76, 134, 224, 15, 68, 222, 157, 2, 72, 231, 160, 1, 83, 33, 108, 6, + 32, 87, 73, 84, 72, 32, 140, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, + 154, 238, 19, 79, 255, 59, 72, 20, 94, 67, 134, 180, 1, 65, 222, 215, 3, + 80, 142, 167, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, 8, 134, 68, + 69, 190, 112, 73, 255, 170, 18, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, + 179, 1, 90, 24, 78, 83, 182, 15, 67, 214, 47, 84, 218, 116, 76, 134, 224, + 15, 68, 223, 157, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 90, 246, 136, 1, 72, 223, 200, 19, 84, 5, 177, 72, 2, 32, 87, + 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 134, 76, 71, 194, + 191, 16, 84, 210, 242, 3, 83, 63, 78, 70, 254, 73, 67, 158, 2, 68, 194, + 16, 84, 162, 23, 77, 250, 1, 86, 234, 44, 72, 158, 1, 79, 198, 10, 71, + 186, 2, 65, 206, 156, 3, 73, 62, 66, 199, 130, 16, 83, 7, 11, 32, 4, 138, + 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 246, 173, 19, + 72, 174, 80, 68, 187, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 190, 81, + 76, 138, 225, 17, 65, 227, 200, 2, 72, 20, 154, 84, 67, 250, 89, 65, 138, + 157, 3, 66, 174, 25, 77, 142, 133, 4, 79, 170, 195, 10, 72, 174, 80, 68, + 187, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 138, 126, 65, 238, 218, + 7, 87, 191, 210, 11, 69, 20, 186, 82, 66, 34, 67, 46, 68, 182, 216, 18, + 72, 231, 160, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 132, 88, 2, 78, + 83, 186, 254, 19, 79, 207, 36, 83, 40, 138, 1, 68, 162, 83, 67, 242, 1, + 77, 142, 1, 84, 130, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, + 156, 3, 73, 62, 66, 199, 130, 16, 83, 10, 22, 79, 191, 83, 73, 6, 234, + 120, 85, 155, 232, 17, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, + 206, 183, 17, 84, 175, 145, 3, 83, 4, 234, 169, 1, 73, 199, 205, 3, 82, + 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 182, 89, 67, 34, 68, 50, 83, 222, + 78, 65, 138, 1, 76, 186, 186, 7, 79, 171, 195, 10, 72, 37, 48, 6, 32, 87, + 73, 84, 72, 32, 163, 247, 20, 74, 32, 138, 1, 66, 36, 2, 68, 79, 52, 7, + 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 198, 91, 72, 226, 71, 65, + 138, 1, 76, 155, 141, 16, 84, 4, 246, 151, 12, 69, 207, 165, 8, 65, 6, + 22, 85, 135, 92, 84, 2, 205, 223, 4, 2, 66, 76, 4, 146, 180, 17, 84, 255, + 240, 2, 68, 4, 214, 2, 77, 211, 194, 20, 84, 19, 44, 6, 32, 87, 73, 84, + 72, 32, 223, 94, 73, 10, 254, 164, 1, 65, 142, 225, 15, 68, 150, 45, 84, + 203, 240, 1, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 215, 243, 20, 74, 28, + 102, 67, 44, 2, 83, 77, 194, 96, 76, 130, 64, 71, 186, 2, 65, 102, 68, + 222, 186, 7, 79, 227, 210, 8, 84, 6, 202, 131, 1, 69, 22, 73, 131, 203, + 18, 65, 2, 249, 194, 10, 10, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 101, + 108, 6, 32, 87, 73, 84, 72, 32, 194, 103, 76, 202, 140, 4, 80, 150, 253, + 8, 77, 250, 255, 6, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, 79, 34, 77, 242, + 96, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, 218, 43, 72, 242, + 12, 71, 186, 2, 65, 206, 156, 3, 73, 63, 66, 4, 174, 99, 78, 199, 140, + 20, 79, 8, 170, 99, 65, 171, 141, 4, 73, 17, 33, 6, 32, 87, 73, 84, 72, + 32, 14, 146, 104, 70, 34, 83, 162, 55, 65, 234, 254, 17, 72, 175, 80, 68, + 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 230, 104, 68, 39, 83, 41, 78, 32, + 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 145, 111, 3, 85, 77, 32, 28, + 40, 5, 87, 73, 84, 72, 32, 143, 112, 82, 26, 174, 78, 67, 234, 28, 68, + 170, 2, 84, 158, 48, 65, 138, 1, 76, 198, 155, 3, 73, 246, 158, 4, 79, + 143, 228, 11, 83, 8, 146, 110, 72, 218, 138, 4, 79, 162, 246, 12, 67, + 171, 253, 2, 69, 47, 136, 1, 6, 32, 87, 73, 84, 72, 32, 132, 1, 5, 77, + 65, 76, 76, 32, 156, 115, 2, 65, 76, 234, 1, 72, 152, 2, 2, 73, 71, 235, + 248, 3, 67, 30, 82, 67, 194, 111, 65, 118, 68, 126, 83, 174, 1, 86, 178, + 227, 7, 79, 171, 195, 10, 72, 10, 154, 112, 65, 218, 10, 69, 82, 79, 203, + 31, 73, 4, 200, 188, 18, 11, 81, 32, 87, 73, 84, 72, 32, 72, 79, 79, 75, + 149, 237, 1, 7, 67, 65, 80, 73, 84, 65, 76, 59, 136, 1, 6, 32, 87, 73, + 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, 32, 232, 122, 2, 72, 79, 176, + 1, 2, 79, 78, 74, 82, 231, 233, 19, 90, 22, 82, 67, 50, 68, 170, 152, 1, + 76, 186, 220, 3, 82, 170, 161, 14, 72, 231, 160, 1, 83, 8, 246, 119, 69, + 22, 73, 62, 79, 199, 202, 18, 65, 6, 234, 141, 1, 73, 207, 234, 15, 79, + 18, 162, 40, 73, 146, 249, 2, 65, 146, 197, 17, 72, 2, 75, 2, 76, 2, 77, + 2, 84, 3, 86, 79, 26, 32, 179, 245, 4, 80, 74, 44, 5, 87, 73, 84, 72, 32, + 251, 194, 19, 66, 72, 210, 131, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, + 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, + 186, 136, 14, 82, 143, 250, 1, 83, 23, 88, 6, 32, 87, 73, 84, 72, 32, + 222, 138, 1, 73, 66, 79, 174, 202, 18, 69, 227, 141, 1, 89, 8, 142, 138, + 1, 68, 242, 151, 16, 84, 203, 240, 1, 72, 19, 48, 6, 32, 87, 73, 84, 72, + 32, 159, 132, 7, 89, 14, 234, 143, 1, 67, 18, 68, 70, 71, 186, 2, 65, + 235, 254, 17, 72, 7, 245, 139, 1, 7, 32, 87, 73, 84, 72, 32, 68, 29, 48, + 6, 32, 87, 73, 84, 72, 32, 147, 233, 16, 79, 24, 198, 142, 1, 67, 18, 68, + 70, 71, 22, 72, 42, 76, 254, 1, 65, 182, 182, 3, 77, 238, 215, 12, 84, + 175, 145, 3, 83, 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 170, 57, 67, 150, + 87, 65, 102, 68, 38, 76, 34, 83, 182, 214, 3, 80, 143, 167, 14, 72, 4, + 250, 175, 10, 73, 231, 152, 10, 79, 12, 128, 1, 5, 65, 82, 67, 72, 65, + 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, 32, 197, 173, 10, 7, 83, + 73, 68, 69, 87, 65, 89, 2, 213, 199, 13, 2, 73, 67, 4, 136, 107, 5, 78, + 86, 69, 82, 84, 149, 156, 14, 3, 32, 76, 79, 4, 174, 221, 20, 70, 3, 80, + 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, 73, 78, 86, 69, 82, 84, + 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 72, 2, 82, + 69, 222, 1, 83, 162, 28, 87, 132, 33, 3, 84, 87, 79, 204, 246, 3, 2, 68, + 69, 140, 5, 2, 76, 65, 206, 11, 71, 248, 174, 1, 21, 80, 72, 65, 82, 89, + 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, 32, 70, 82, 73, 192, 181, + 13, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, 89, 78, 71, 69, 65, 76, + 32, 83, 80, 73, 155, 116, 89, 4, 150, 169, 4, 76, 223, 224, 15, 73, 6, + 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, 78, 84, 65, 76, 32, 80, + 4, 26, 80, 143, 188, 4, 67, 2, 153, 208, 15, 6, 69, 82, 67, 85, 83, 83, + 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 225, 4, 67, 231, 198, 15, 83, + 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 149, 225, 4, 13, 84, 82, 79, 70, + 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, 72, 193, 61, 12, + 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 189, 132, 1, 2, 32, + 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, + 220, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, 225, 211, 18, 10, + 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, 30, 73, 22, 76, + 74, 79, 42, 82, 122, 84, 186, 164, 4, 66, 202, 41, 71, 166, 194, 15, 65, + 150, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, 2, 78, 2, 80, 2, + 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 190, 209, 20, 84, 3, 90, + 5, 191, 201, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 198, 199, 8, 66, + 187, 218, 11, 83, 9, 214, 94, 80, 130, 243, 19, 69, 3, 85, 11, 88, 8, 69, + 86, 69, 82, 83, 69, 68, 32, 216, 127, 5, 32, 87, 73, 84, 72, 167, 192, + 19, 85, 4, 210, 208, 20, 78, 3, 82, 13, 33, 6, 85, 82, 78, 69, 68, 32, + 10, 146, 208, 20, 69, 2, 71, 2, 75, 2, 77, 3, 82, 180, 11, 136, 1, 5, 77, + 65, 76, 76, 32, 253, 129, 1, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, + 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 144, 11, 76, 15, 67, 65, + 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 27, 76, 4, 182, 53, + 73, 3, 85, 140, 11, 76, 6, 69, 84, 84, 69, 82, 32, 161, 127, 8, 73, 71, + 65, 84, 85, 82, 69, 32, 250, 10, 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, + 4, 68, 182, 4, 69, 174, 9, 70, 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, + 7, 74, 182, 1, 75, 178, 2, 76, 210, 6, 77, 178, 2, 78, 218, 3, 79, 242, + 8, 80, 230, 1, 81, 174, 1, 82, 142, 8, 83, 234, 10, 84, 246, 13, 85, 218, + 9, 86, 130, 3, 87, 110, 88, 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, + 69, 82, 78, 132, 208, 4, 4, 76, 80, 72, 65, 190, 164, 13, 86, 138, 209, + 2, 65, 2, 79, 2, 85, 3, 89, 72, 88, 5, 87, 73, 84, 72, 32, 209, 209, 5, + 11, 82, 69, 86, 69, 82, 83, 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, + 54, 82, 170, 34, 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, + 186, 2, 65, 206, 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 12, 161, + 105, 4, 82, 69, 86, 69, 12, 22, 79, 155, 57, 73, 8, 218, 57, 84, 223, 12, + 85, 10, 26, 73, 255, 212, 4, 69, 8, 26, 78, 131, 156, 4, 71, 6, 17, 2, + 71, 32, 6, 240, 58, 4, 65, 66, 79, 86, 159, 243, 17, 66, 9, 33, 6, 32, + 87, 73, 84, 72, 32, 6, 242, 115, 71, 186, 2, 65, 183, 182, 3, 77, 2, 133, + 160, 10, 7, 71, 76, 73, 67, 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, + 72, 32, 130, 1, 65, 108, 11, 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, + 38, 82, 242, 182, 4, 79, 191, 230, 15, 69, 18, 110, 84, 146, 60, 70, 202, + 56, 76, 170, 207, 3, 77, 174, 7, 80, 178, 137, 12, 68, 222, 157, 2, 72, + 231, 160, 1, 83, 2, 255, 7, 79, 8, 64, 5, 82, 82, 69, 68, 32, 161, 80, 6, + 83, 69, 76, 73, 78, 69, 6, 182, 32, 65, 142, 162, 20, 69, 3, 79, 6, 186, + 177, 4, 79, 227, 144, 16, 69, 2, 133, 148, 10, 4, 79, 75, 69, 78, 47, + 108, 6, 32, 87, 73, 84, 72, 32, 196, 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, + 68, 32, 90, 85, 155, 238, 19, 79, 24, 102, 67, 182, 112, 65, 222, 215, 3, + 80, 230, 5, 82, 170, 161, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, + 10, 54, 69, 190, 112, 73, 238, 225, 6, 85, 147, 201, 11, 65, 4, 249, 51, + 5, 68, 73, 76, 76, 65, 7, 49, 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, + 4, 162, 106, 82, 45, 4, 76, 69, 70, 84, 8, 34, 73, 18, 79, 227, 181, 4, + 82, 2, 163, 87, 78, 4, 210, 202, 4, 80, 199, 243, 8, 77, 4, 37, 7, 65, + 84, 82, 73, 76, 76, 79, 5, 153, 224, 12, 5, 32, 87, 73, 84, 72, 73, 96, + 6, 32, 87, 73, 84, 72, 32, 182, 1, 69, 34, 79, 178, 1, 90, 246, 194, 4, + 66, 235, 229, 15, 85, 32, 98, 83, 34, 84, 130, 33, 67, 206, 44, 77, 170, + 31, 76, 206, 199, 3, 72, 138, 15, 80, 179, 137, 12, 68, 4, 134, 67, 72, + 223, 200, 19, 84, 4, 26, 79, 167, 142, 18, 65, 2, 215, 235, 17, 80, 8, + 246, 77, 90, 195, 200, 19, 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, + 85, 66, 76, 69, 32, 8, 26, 74, 139, 186, 20, 73, 7, 235, 179, 4, 32, 8, + 42, 87, 210, 183, 4, 82, 187, 219, 5, 84, 2, 163, 219, 6, 89, 11, 11, 32, + 8, 26, 87, 219, 179, 4, 68, 2, 169, 89, 5, 73, 84, 72, 32, 67, 119, 112, + 6, 32, 87, 73, 84, 72, 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, + 1, 2, 90, 72, 147, 189, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, + 15, 84, 162, 23, 77, 250, 1, 86, 194, 3, 70, 170, 41, 72, 158, 1, 79, + 198, 10, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 186, 64, 82, 143, 194, + 15, 83, 24, 92, 6, 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, + 76, 69, 88, 179, 145, 19, 65, 5, 129, 131, 4, 3, 32, 65, 78, 19, 11, 32, + 16, 40, 4, 65, 78, 68, 32, 207, 156, 18, 66, 14, 162, 85, 67, 130, 2, 72, + 226, 11, 71, 186, 2, 65, 182, 182, 3, 77, 190, 162, 4, 68, 179, 181, 8, + 84, 12, 22, 79, 227, 97, 73, 10, 28, 2, 84, 32, 243, 50, 85, 8, 192, 87, + 5, 65, 66, 79, 86, 69, 239, 195, 17, 66, 2, 183, 232, 13, 79, 4, 237, + 237, 6, 13, 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 226, 176, 4, 67, 231, 9, 80, 13, 33, 6, 32, + 87, 73, 84, 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, + 170, 185, 4, 80, 142, 1, 67, 219, 4, 82, 5, 157, 186, 4, 4, 32, 65, 78, + 68, 15, 11, 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 209, 228, 13, + 4, 69, 86, 69, 82, 10, 54, 67, 246, 183, 4, 80, 230, 5, 82, 139, 197, 13, + 84, 4, 194, 194, 7, 85, 147, 201, 11, 65, 17, 84, 6, 32, 87, 73, 84, 72, + 32, 73, 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 202, 175, 4, + 77, 174, 7, 80, 142, 167, 14, 72, 174, 80, 68, 187, 80, 83, 5, 197, 189, + 17, 8, 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, + 126, 76, 138, 225, 17, 65, 227, 200, 2, 72, 22, 218, 3, 67, 250, 89, 65, + 138, 157, 3, 66, 174, 25, 77, 170, 33, 80, 230, 227, 3, 79, 170, 195, 10, + 72, 174, 80, 68, 187, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 178, + 225, 17, 83, 202, 202, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, + 84, 72, 32, 166, 44, 65, 216, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, + 87, 65, 32, 242, 225, 3, 69, 207, 130, 16, 86, 24, 86, 66, 34, 67, 46, + 68, 214, 90, 76, 214, 214, 3, 80, 142, 167, 14, 72, 231, 160, 1, 83, 2, + 157, 229, 12, 3, 82, 69, 86, 6, 158, 58, 69, 154, 32, 73, 255, 170, 18, + 65, 8, 234, 86, 73, 178, 173, 3, 69, 203, 182, 12, 79, 75, 80, 6, 32, 87, + 73, 84, 72, 32, 222, 4, 78, 188, 1, 2, 79, 84, 251, 161, 20, 83, 48, 178, + 1, 67, 34, 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 130, + 70, 72, 226, 11, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 136, 64, 6, 83, + 84, 82, 79, 75, 69, 51, 82, 4, 210, 87, 73, 255, 170, 18, 65, 14, 18, 73, + 47, 79, 4, 221, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, + 231, 36, 85, 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 227, 140, + 18, 66, 6, 150, 83, 71, 186, 2, 65, 163, 142, 16, 84, 4, 29, 5, 65, 67, + 82, 79, 78, 5, 233, 35, 4, 32, 65, 78, 68, 7, 145, 72, 15, 32, 65, 78, + 68, 32, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, + 171, 251, 4, 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, + 5, 85, 76, 65, 82, 32, 12, 226, 162, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, + 3, 84, 4, 26, 65, 187, 139, 20, 79, 2, 231, 173, 4, 76, 6, 150, 145, 4, + 65, 229, 162, 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, + 32, 10, 94, 67, 216, 161, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, + 65, 78, 68, 235, 206, 15, 83, 6, 178, 81, 73, 198, 205, 3, 82, 187, 221, + 14, 65, 29, 48, 6, 32, 87, 73, 84, 72, 32, 163, 157, 20, 82, 24, 98, 67, + 34, 68, 50, 83, 222, 78, 65, 138, 1, 76, 214, 214, 3, 80, 230, 227, 3, + 79, 171, 195, 10, 72, 4, 210, 47, 69, 151, 203, 18, 65, 6, 214, 69, 73, + 134, 180, 3, 69, 175, 174, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, + 24, 6, 32, 65, 78, 68, 32, 68, 77, 156, 1, 6, 32, 87, 73, 84, 72, 32, + 248, 3, 5, 65, 77, 66, 68, 65, 22, 69, 48, 5, 79, 78, 71, 32, 83, 194, + 161, 4, 83, 2, 90, 234, 229, 15, 85, 219, 16, 74, 50, 178, 1, 66, 86, 67, + 56, 2, 68, 79, 100, 3, 77, 73, 68, 242, 1, 72, 226, 71, 65, 138, 1, 76, + 218, 206, 3, 73, 162, 1, 82, 222, 2, 70, 130, 4, 80, 198, 182, 12, 84, + 175, 145, 3, 83, 6, 36, 3, 69, 76, 84, 147, 226, 19, 65, 5, 241, 162, 4, + 6, 32, 65, 78, 68, 32, 80, 8, 146, 43, 69, 22, 73, 242, 129, 7, 85, 147, + 201, 11, 65, 8, 38, 84, 25, 5, 85, 66, 76, 69, 32, 4, 137, 24, 2, 32, 66, + 4, 162, 154, 4, 77, 243, 220, 14, 66, 8, 36, 4, 68, 76, 69, 32, 187, 43, + 45, 6, 198, 215, 16, 84, 178, 240, 2, 82, 79, 68, 2, 139, 155, 4, 32, 6, + 222, 162, 4, 90, 161, 192, 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, + 72, 32, 6, 18, 68, 35, 72, 4, 202, 62, 73, 215, 216, 18, 79, 2, 133, 154, + 4, 2, 73, 71, 27, 56, 6, 32, 87, 73, 84, 72, 32, 102, 73, 151, 133, 20, + 85, 16, 134, 71, 65, 250, 205, 3, 67, 186, 2, 77, 174, 7, 80, 178, 137, + 12, 68, 150, 45, 84, 203, 240, 1, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, + 45, 87, 69, 76, 83, 72, 32, 141, 194, 16, 6, 32, 83, 67, 79, 84, 83, 4, + 226, 139, 19, 76, 159, 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 242, + 128, 20, 85, 219, 16, 74, 40, 88, 5, 87, 73, 84, 72, 32, 193, 250, 5, 11, + 80, 82, 69, 67, 69, 68, 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 154, 36, + 77, 234, 27, 71, 186, 2, 65, 102, 68, 250, 214, 3, 80, 230, 5, 82, 130, + 222, 3, 79, 227, 210, 8, 84, 10, 166, 35, 69, 22, 73, 202, 237, 3, 82, + 170, 148, 3, 85, 147, 201, 11, 65, 6, 128, 65, 3, 79, 78, 71, 202, 2, 73, + 135, 222, 3, 69, 2, 25, 4, 87, 73, 84, 72, 2, 149, 131, 17, 5, 32, 84, + 73, 76, 68, 115, 116, 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, + 69, 78, 32, 170, 137, 13, 77, 250, 255, 6, 73, 2, 79, 3, 85, 86, 154, 1, + 67, 70, 68, 152, 1, 2, 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, + 218, 43, 72, 242, 12, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 187, 64, + 82, 14, 168, 48, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 187, 186, 18, 65, + 14, 18, 73, 47, 79, 4, 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, + 84, 167, 46, 85, 6, 11, 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 247, 232, + 17, 66, 6, 66, 78, 188, 162, 12, 6, 87, 32, 82, 73, 78, 71, 139, 234, 7, + 79, 2, 155, 20, 71, 6, 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, + 5, 71, 79, 78, 69, 75, 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 209, 48, + 2, 32, 65, 8, 25, 4, 73, 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, + 158, 46, 68, 142, 13, 65, 183, 182, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, + 65, 76, 32, 76, 73, 78, 69, 32, 66, 69, 76, 79, 87, 7, 217, 42, 4, 32, + 65, 78, 68, 2, 229, 225, 9, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, + 79, 195, 147, 4, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 211, 136, 20, 69, + 8, 206, 54, 71, 186, 2, 65, 194, 221, 3, 82, 143, 194, 15, 83, 23, 48, 6, + 32, 87, 73, 84, 72, 32, 139, 244, 19, 72, 18, 86, 70, 34, 83, 162, 55, + 65, 178, 208, 3, 77, 174, 7, 80, 142, 167, 14, 72, 175, 80, 68, 2, 181, + 135, 14, 3, 76, 79, 85, 6, 202, 27, 84, 185, 130, 12, 6, 81, 85, 73, 82, + 82, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 191, 143, 4, 80, 8, 42, 68, + 16, 4, 72, 79, 79, 75, 23, 83, 2, 211, 44, 73, 5, 179, 216, 17, 32, 2, + 181, 26, 6, 84, 82, 79, 75, 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, + 82, 83, 69, 68, 32, 148, 2, 2, 85, 77, 231, 129, 4, 65, 46, 36, 4, 87, + 73, 84, 72, 235, 6, 82, 44, 26, 32, 219, 253, 14, 79, 42, 166, 1, 67, 50, + 68, 200, 1, 8, 70, 73, 83, 72, 72, 79, 79, 75, 66, 76, 34, 84, 254, 17, + 77, 162, 30, 65, 206, 156, 3, 73, 146, 59, 80, 230, 227, 3, 79, 143, 228, + 11, 83, 6, 154, 19, 69, 222, 237, 3, 82, 187, 221, 14, 65, 8, 11, 79, 8, + 24, 2, 84, 32, 111, 85, 6, 26, 66, 239, 132, 19, 65, 4, 25, 4, 69, 76, + 79, 87, 5, 29, 5, 32, 65, 78, 68, 32, 2, 247, 231, 3, 77, 2, 21, 3, 66, + 76, 69, 2, 11, 32, 2, 211, 46, 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 138, + 129, 4, 77, 175, 7, 80, 4, 206, 49, 73, 211, 218, 3, 79, 4, 238, 210, 17, + 65, 251, 22, 73, 22, 162, 1, 72, 40, 6, 79, 80, 69, 78, 32, 69, 140, 130, + 4, 8, 82, 32, 87, 73, 84, 72, 32, 70, 168, 2, 3, 83, 67, 82, 218, 129, + 14, 69, 198, 84, 67, 171, 163, 1, 75, 2, 11, 65, 2, 189, 213, 9, 2, 76, + 70, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 139, 4, 82, 171, 161, 14, + 72, 5, 11, 32, 2, 11, 82, 2, 153, 168, 10, 3, 79, 84, 85, 83, 180, 1, 6, + 32, 87, 73, 84, 72, 32, 202, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, + 13, 81, 85, 65, 84, 32, 82, 69, 86, 69, 82, 83, 69, 68, 173, 198, 9, 6, + 84, 73, 82, 82, 85, 80, 38, 106, 65, 34, 67, 86, 68, 126, 83, 174, 1, 86, + 210, 9, 77, 254, 245, 3, 80, 230, 227, 3, 79, 171, 195, 10, 72, 4, 193, + 1, 4, 67, 85, 84, 69, 12, 58, 65, 218, 10, 69, 82, 79, 202, 31, 73, 239, + 225, 6, 85, 4, 101, 3, 82, 79, 78, 6, 21, 3, 79, 84, 32, 6, 26, 66, 211, + 252, 18, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 185, 248, 18, 3, 65, + 78, 68, 4, 22, 72, 203, 42, 87, 2, 21, 3, 79, 82, 84, 2, 17, 2, 32, 83, + 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 237, + 235, 18, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, + 40, 2, 32, 76, 4, 46, 76, 249, 226, 18, 5, 75, 72, 65, 32, 89, 2, 247, + 12, 84, 18, 48, 3, 72, 87, 65, 97, 5, 82, 73, 80, 84, 32, 11, 33, 6, 32, + 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 194, 221, 3, 82, 171, + 161, 14, 72, 8, 26, 82, 147, 243, 3, 71, 5, 173, 141, 11, 5, 32, 87, 73, + 84, 72, 2, 233, 161, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, + 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 224, 160, 10, 11, + 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 135, 210, 9, 85, 7, 26, 80, + 203, 245, 3, 32, 2, 145, 203, 9, 2, 69, 78, 2, 173, 220, 12, 2, 69, 68, + 2, 209, 159, 16, 4, 77, 79, 73, 68, 2, 227, 219, 9, 32, 147, 1, 188, 1, + 6, 32, 87, 73, 84, 72, 32, 192, 3, 2, 69, 83, 70, 72, 130, 2, 79, 102, + 82, 54, 85, 242, 241, 3, 67, 202, 1, 83, 136, 210, 10, 9, 65, 73, 76, 76, + 69, 83, 83, 32, 80, 243, 163, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, + 170, 31, 76, 214, 214, 3, 80, 180, 5, 4, 72, 79, 79, 75, 50, 82, 143, + 194, 15, 83, 10, 58, 69, 22, 73, 62, 79, 182, 129, 7, 85, 147, 201, 11, + 65, 2, 195, 251, 11, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 255, + 199, 17, 88, 2, 17, 2, 77, 77, 2, 215, 199, 17, 65, 8, 32, 2, 73, 65, + 215, 255, 15, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, + 45, 195, 238, 3, 68, 2, 229, 252, 3, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, + 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 183, 244, 3, 32, 8, 64, 12, 32, + 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 245, 243, 15, 5, + 84, 72, 82, 79, 85, 6, 17, 2, 82, 78, 7, 41, 8, 32, 87, 73, 84, 72, 32, + 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 213, 197, 3, 6, 84, 72, + 82, 79, 85, 71, 8, 26, 78, 143, 244, 3, 80, 6, 17, 2, 69, 32, 6, 234, + 188, 4, 83, 190, 160, 9, 84, 215, 132, 1, 70, 2, 17, 2, 69, 83, 2, 11, + 73, 2, 243, 198, 19, 76, 76, 44, 5, 82, 78, 69, 68, 32, 131, 233, 19, 77, + 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 138, + 240, 3, 65, 50, 77, 198, 2, 89, 246, 221, 15, 85, 218, 19, 69, 2, 71, 2, + 75, 2, 76, 3, 87, 2, 191, 195, 16, 69, 7, 209, 224, 3, 11, 32, 87, 73, + 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 149, 181, 9, 5, 83, 85, 76, + 65, 82, 12, 66, 69, 176, 213, 3, 7, 32, 79, 80, 69, 78, 45, 79, 167, 29, + 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 166, 200, 17, 72, 203, 237, 1, + 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 138, 230, 3, 77, 174, 7, 80, + 142, 5, 76, 226, 197, 13, 84, 163, 92, 72, 5, 203, 237, 3, 32, 7, 11, 32, + 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 224, 210, 3, 6, 80, 83, 73, 76, + 79, 78, 226, 144, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, + 5, 87, 73, 84, 72, 32, 5, 245, 17, 8, 32, 87, 73, 84, 72, 32, 83, 72, 78, + 142, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 106, 82, 38, 84, + 186, 9, 71, 82, 83, 234, 1, 65, 206, 156, 3, 73, 62, 66, 135, 66, 76, 6, + 172, 185, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 147, 132, 14, 65, 18, + 52, 8, 73, 65, 69, 82, 69, 83, 73, 83, 131, 1, 79, 13, 11, 32, 10, 40, 4, + 65, 78, 68, 32, 223, 199, 17, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, + 183, 182, 3, 77, 2, 211, 187, 18, 65, 6, 26, 85, 207, 184, 17, 84, 4, 21, + 3, 66, 76, 69, 4, 11, 32, 4, 138, 13, 71, 187, 2, 65, 14, 11, 79, 14, 28, + 2, 82, 78, 251, 75, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, + 11, 71, 186, 2, 65, 242, 216, 7, 68, 179, 181, 8, 84, 2, 145, 75, 2, 79, + 79, 10, 29, 5, 65, 67, 82, 79, 78, 11, 29, 5, 32, 65, 78, 68, 32, 8, 50, + 68, 214, 10, 71, 186, 2, 65, 163, 142, 16, 84, 2, 171, 10, 73, 6, 29, 5, + 71, 79, 78, 69, 75, 7, 11, 32, 4, 25, 4, 65, 78, 68, 32, 4, 178, 12, 65, + 163, 142, 16, 84, 4, 222, 233, 3, 69, 131, 200, 13, 73, 6, 25, 4, 73, 76, + 68, 69, 7, 11, 32, 4, 26, 65, 191, 194, 17, 66, 2, 17, 2, 78, 68, 2, 11, + 32, 2, 139, 11, 65, 29, 84, 6, 32, 87, 73, 84, 72, 32, 162, 1, 73, 66, + 79, 174, 202, 18, 69, 227, 141, 1, 89, 14, 82, 68, 174, 225, 3, 80, 142, + 1, 67, 158, 7, 82, 158, 174, 12, 84, 203, 240, 1, 72, 4, 26, 73, 179, + 226, 7, 79, 2, 17, 2, 65, 71, 2, 189, 168, 19, 2, 79, 78, 2, 41, 8, 83, + 73, 71, 79, 84, 72, 73, 67, 2, 175, 225, 15, 32, 6, 33, 6, 76, 65, 80, + 85, 75, 32, 6, 146, 193, 19, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, + 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 194, 165, 17, 82, + 171, 89, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, + 79, 255, 221, 3, 80, 4, 222, 3, 73, 131, 210, 18, 79, 8, 60, 11, 78, 71, + 32, 76, 69, 70, 84, 32, 76, 69, 71, 79, 87, 7, 11, 32, 4, 60, 7, 65, 78, + 68, 32, 76, 79, 87, 65, 4, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 21, 3, + 73, 71, 72, 2, 215, 236, 10, 84, 2, 197, 210, 18, 4, 32, 83, 69, 82, 33, + 48, 6, 32, 87, 73, 84, 72, 32, 187, 219, 15, 79, 28, 110, 67, 18, 68, 70, + 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 182, 182, 3, 77, 238, 215, 12, + 84, 163, 151, 1, 82, 2, 203, 3, 73, 6, 26, 73, 251, 227, 15, 79, 2, 17, + 2, 65, 69, 2, 243, 247, 15, 82, 2, 207, 233, 17, 82, 4, 17, 2, 79, 79, 4, + 187, 246, 4, 75, 2, 191, 216, 18, 79, 4, 26, 72, 167, 161, 19, 84, 2, 21, + 3, 79, 82, 84, 2, 245, 226, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, + 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 138, 207, + 3, 77, 174, 7, 80, 230, 5, 82, 171, 161, 14, 72, 2, 175, 140, 13, 67, 6, + 42, 73, 238, 225, 6, 85, 147, 201, 11, 65, 2, 177, 158, 11, 4, 82, 67, + 85, 77, 6, 234, 169, 3, 69, 203, 182, 12, 79, 2, 11, 73, 2, 243, 137, 12, + 78, 4, 26, 87, 151, 158, 19, 84, 2, 195, 190, 16, 65, 18, 90, 70, 222, + 157, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 137, 10, 83, 219, 5, + 79, 10, 34, 70, 242, 204, 19, 73, 3, 76, 7, 238, 204, 19, 73, 3, 76, 36, + 150, 1, 83, 190, 203, 19, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 139, + 212, 4, 67, 160, 5, 200, 1, 2, 65, 70, 104, 2, 70, 84, 186, 33, 79, 20, + 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, 142, + 231, 15, 68, 248, 22, 6, 86, 69, 76, 32, 83, 76, 158, 209, 2, 77, 227, + 79, 71, 4, 180, 180, 1, 14, 32, 70, 76, 85, 84, 84, 69, 82, 73, 78, 71, + 32, 73, 78, 169, 186, 11, 3, 89, 32, 71, 200, 3, 58, 32, 186, 17, 45, + 169, 1, 6, 87, 65, 82, 68, 83, 32, 218, 1, 190, 1, 65, 174, 3, 66, 202, + 1, 67, 96, 2, 68, 79, 112, 2, 72, 65, 250, 2, 76, 46, 77, 38, 79, 38, 82, + 146, 3, 83, 82, 84, 174, 1, 87, 190, 159, 8, 70, 210, 3, 78, 202, 1, 80, + 187, 11, 86, 26, 22, 78, 211, 2, 82, 20, 28, 2, 68, 32, 131, 2, 71, 14, + 72, 3, 76, 79, 87, 0, 3, 85, 80, 80, 93, 6, 82, 73, 71, 72, 84, 32, 4, + 21, 3, 69, 82, 32, 4, 132, 190, 16, 9, 65, 78, 68, 32, 82, 73, 71, 72, + 84, 175, 30, 79, 6, 50, 84, 161, 166, 17, 6, 68, 79, 85, 66, 76, 69, 4, + 138, 190, 16, 82, 131, 216, 1, 65, 6, 202, 163, 8, 69, 185, 1, 4, 76, 69, + 32, 66, 6, 26, 67, 167, 167, 8, 82, 2, 129, 167, 8, 5, 32, 76, 69, 83, + 83, 12, 40, 4, 65, 82, 66, 32, 191, 167, 8, 76, 8, 40, 4, 68, 79, 87, 78, + 1, 2, 85, 80, 4, 57, 12, 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, + 4, 208, 176, 16, 4, 68, 79, 87, 78, 1, 2, 85, 80, 14, 130, 167, 8, 85, + 230, 21, 79, 250, 222, 2, 69, 217, 153, 2, 8, 76, 79, 83, 69, 68, 32, 69, + 78, 10, 44, 5, 85, 66, 76, 69, 32, 175, 167, 8, 84, 8, 226, 168, 8, 87, + 234, 24, 65, 198, 248, 2, 81, 151, 172, 4, 80, 26, 36, 3, 76, 70, 32, + 203, 170, 8, 78, 24, 216, 1, 6, 67, 73, 82, 67, 76, 69, 190, 167, 8, 66, + 90, 70, 34, 82, 128, 147, 8, 30, 73, 78, 86, 69, 82, 83, 69, 32, 77, 69, + 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 82, 73, 71, + 72, 84, 139, 19, 77, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 84, 254, + 160, 17, 70, 175, 204, 1, 68, 4, 166, 187, 5, 72, 219, 180, 12, 87, 4, + 252, 148, 1, 2, 85, 71, 207, 147, 7, 79, 2, 161, 172, 18, 4, 85, 76, 84, + 73, 6, 214, 169, 8, 85, 215, 150, 8, 78, 30, 44, 5, 73, 71, 72, 84, 32, + 219, 170, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, 1, 12, 68, 79, + 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 218, 209, 8, 79, 234, + 215, 7, 66, 54, 83, 159, 53, 84, 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, + 32, 193, 234, 17, 7, 84, 72, 82, 79, 85, 71, 72, 6, 146, 248, 15, 68, + 190, 145, 3, 86, 79, 83, 7, 197, 201, 8, 2, 32, 87, 4, 202, 220, 8, 65, + 255, 248, 7, 72, 34, 238, 168, 8, 45, 70, 69, 78, 73, 168, 1, 3, 80, 69, + 69, 26, 81, 227, 2, 85, 18, 58, 82, 246, 173, 8, 72, 178, 178, 7, 79, + 179, 169, 2, 65, 6, 40, 4, 73, 65, 78, 71, 143, 175, 8, 65, 4, 208, 227, + 4, 8, 76, 69, 32, 66, 69, 83, 73, 68, 235, 230, 11, 85, 16, 174, 177, 8, + 72, 202, 1, 73, 141, 246, 9, 2, 82, 73, 30, 90, 83, 230, 178, 8, 70, 206, + 1, 72, 130, 1, 80, 189, 3, 7, 84, 79, 45, 82, 73, 71, 72, 4, 44, 5, 73, + 68, 69, 32, 65, 171, 183, 8, 72, 2, 209, 138, 1, 2, 82, 67, 208, 1, 172, + 1, 5, 65, 82, 82, 79, 87, 134, 5, 66, 70, 72, 134, 4, 84, 178, 2, 87, + 230, 182, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, 50, 82, 70, 83, + 134, 136, 8, 67, 47, 81, 71, 26, 32, 247, 134, 17, 45, 66, 102, 65, 138, + 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 238, 184, 8, 70, 173, 215, 9, 4, + 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 219, 186, 8, 78, 10, 64, + 4, 83, 72, 79, 82, 210, 185, 8, 82, 202, 208, 4, 65, 55, 84, 2, 175, 146, + 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, 79, 32, 66, 6, 242, + 135, 13, 83, 134, 155, 4, 76, 223, 142, 2, 88, 6, 32, 2, 65, 82, 199, + 187, 8, 76, 5, 221, 128, 12, 23, 32, 79, 86, 69, 82, 32, 82, 73, 71, 72, + 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, 154, 188, 8, 68, + 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, 78, 122, 69, 226, + 148, 1, 72, 155, 160, 1, 86, 8, 194, 189, 8, 65, 224, 10, 5, 79, 84, 84, + 79, 77, 251, 129, 8, 76, 30, 26, 65, 163, 207, 16, 69, 26, 48, 6, 82, 80, + 79, 79, 78, 32, 243, 231, 18, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, + 65, 82, 66, 32, 213, 191, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, + 40, 4, 68, 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 251, 212, 16, 87, 8, + 226, 192, 8, 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 130, 138, 8, 70, + 175, 5, 84, 12, 26, 32, 135, 212, 16, 87, 10, 60, 6, 65, 66, 79, 86, 69, + 32, 238, 203, 16, 70, 175, 5, 84, 6, 42, 76, 205, 192, 8, 4, 82, 73, 71, + 72, 4, 242, 190, 8, 69, 207, 201, 4, 79, 54, 44, 2, 82, 73, 194, 196, 8, + 79, 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 247, 200, 8, 80, 30, 56, + 8, 45, 72, 69, 65, 68, 69, 68, 32, 243, 218, 16, 32, 28, 52, 5, 65, 82, + 82, 79, 87, 210, 210, 16, 68, 39, 80, 25, 11, 32, 22, 184, 197, 8, 9, 79, + 86, 69, 82, 32, 82, 73, 71, 72, 22, 87, 155, 137, 8, 84, 6, 26, 72, 131, + 203, 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 165, 216, + 16, 2, 32, 87, 5, 215, 144, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, + 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, + 82, 32, 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, + 1, 83, 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 175, 177, + 17, 68, 18, 66, 75, 22, 78, 202, 164, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, + 5, 239, 211, 18, 65, 5, 145, 170, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, + 2, 80, 198, 134, 7, 68, 250, 149, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, + 254, 14, 78, 202, 175, 1, 84, 46, 67, 2, 83, 254, 68, 74, 2, 76, 2, 82, + 2, 86, 2, 87, 2, 89, 187, 2, 65, 6, 166, 160, 19, 72, 2, 76, 187, 2, 65, + 10, 82, 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, + 65, 43, 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 255, 233, 6, 45, 5, + 17, 2, 32, 67, 2, 149, 220, 14, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, + 37, 15, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, + 202, 186, 15, 78, 151, 158, 2, 82, 4, 11, 32, 4, 170, 157, 19, 82, 3, 89, + 14, 170, 208, 15, 85, 158, 144, 1, 79, 202, 188, 2, 65, 186, 2, 69, 3, + 73, 52, 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, + 82, 200, 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 175, 141, 18, 83, + 16, 40, 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, + 84, 69, 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, + 69, 32, 69, 81, 85, 65, 76, 32, 65, 207, 170, 15, 76, 2, 181, 5, 23, 45, + 84, 72, 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, + 76, 73, 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, + 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, + 82, 69, 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, + 241, 138, 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, + 32, 4, 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 129, 238, 12, 11, + 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 253, 215, 12, 5, 84, 32, + 78, 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, + 86, 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, + 69, 68, 2, 17, 2, 32, 69, 2, 183, 186, 14, 81, 18, 40, 2, 82, 32, 193, + 190, 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, + 69, 81, 85, 65, 76, 32, 84, 79, 230, 212, 12, 69, 155, 177, 4, 71, 2, + 201, 154, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, + 72, 32, 68, 79, 84, 32, 6, 26, 65, 175, 170, 11, 73, 4, 25, 4, 66, 79, + 86, 69, 5, 139, 198, 17, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, + 81, 85, 69, 83, 84, 73, 79, 78, 203, 194, 18, 68, 2, 133, 169, 11, 5, 73, + 82, 67, 76, 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 215, 149, 18, 75, + 136, 11, 190, 1, 71, 186, 5, 77, 162, 7, 78, 168, 65, 2, 80, 83, 20, 3, + 83, 85, 32, 130, 135, 15, 82, 152, 34, 11, 86, 82, 69, 32, 84, 79, 85, + 82, 78, 79, 73, 234, 44, 79, 130, 214, 1, 90, 175, 82, 66, 44, 26, 65, + 57, 2, 72, 84, 2, 165, 142, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, + 38, 32, 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, + 3, 76, 69, 70, 86, 83, 42, 84, 152, 184, 7, 3, 66, 76, 85, 134, 224, 8, + 87, 134, 36, 86, 214, 74, 71, 195, 68, 67, 2, 201, 253, 16, 4, 73, 71, + 72, 84, 8, 94, 73, 201, 201, 17, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, + 69, 68, 32, 66, 76, 65, 67, 75, 4, 221, 249, 16, 2, 86, 69, 4, 36, 3, 73, + 71, 72, 171, 224, 16, 65, 2, 241, 230, 1, 16, 84, 32, 84, 79, 82, 84, 79, + 73, 83, 69, 32, 83, 72, 69, 76, 76, 6, 138, 157, 16, 72, 242, 93, 65, 43, + 73, 4, 216, 216, 15, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, + 79, 68, 5, 163, 130, 8, 32, 138, 1, 80, 3, 66, 85, 32, 229, 136, 8, 11, + 73, 84, 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, + 84, 84, 69, 82, 32, 218, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 206, 173, + 12, 69, 190, 141, 4, 81, 155, 92, 68, 60, 166, 1, 71, 174, 254, 5, 84, + 174, 192, 8, 89, 214, 122, 78, 134, 135, 3, 83, 82, 66, 2, 67, 2, 68, 2, + 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 192, + 17, 89, 254, 196, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, + 12, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 193, 236, 6, 2, 85, + 66, 8, 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 227, 197, 16, + 76, 2, 149, 202, 16, 3, 80, 72, 82, 2, 207, 242, 18, 45, 18, 138, 183, + 15, 78, 158, 138, 3, 65, 182, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, + 20, 84, 6, 32, 83, 73, 71, 78, 32, 253, 82, 10, 45, 67, 65, 82, 82, 73, + 69, 82, 32, 76, 18, 142, 199, 5, 65, 214, 254, 10, 79, 238, 254, 1, 69, + 150, 64, 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, + 3, 65, 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, + 32, 190, 240, 12, 70, 178, 249, 2, 83, 193, 130, 1, 4, 84, 65, 66, 85, 6, + 108, 5, 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, + 85, 68, 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, + 1, 8, 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, + 65, 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 151, 223, + 17, 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, + 32, 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, + 55, 148, 4, 2, 56, 48, 78, 66, 201, 179, 17, 3, 48, 50, 56, 6, 140, 155, + 12, 5, 48, 48, 45, 49, 48, 176, 169, 5, 2, 50, 48, 197, 19, 2, 51, 49, + 150, 1, 78, 48, 90, 49, 130, 1, 55, 210, 241, 13, 50, 2, 51, 2, 52, 2, + 53, 3, 54, 22, 178, 1, 57, 138, 252, 18, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 3, 56, 24, 90, 51, 138, 252, 18, 48, 2, 49, 2, 50, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 134, 252, 18, 65, 2, 66, 3, 67, 4, + 226, 251, 18, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 161, 164, 6, 2, 45, + 86, 160, 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, + 250, 159, 2, 57, 190, 192, 11, 55, 3, 56, 16, 134, 249, 18, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 186, 248, 18, 48, 2, 49, 2, + 50, 2, 51, 2, 53, 3, 54, 18, 254, 247, 18, 48, 2, 49, 2, 50, 2, 52, 2, + 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 170, 247, 18, 48, 2, 49, 2, 50, 2, + 53, 2, 55, 2, 56, 3, 57, 18, 230, 246, 18, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 146, 246, 18, 51, 2, 52, 2, 53, 2, + 54, 2, 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 254, 230, + 13, 53, 243, 1, 49, 16, 142, 245, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 54, 2, 56, 3, 57, 18, 194, 244, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 54, 2, 55, 2, 56, 3, 57, 6, 238, 243, 18, 52, 2, 55, 3, 56, 16, 202, 243, + 18, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 254, 242, + 18, 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 140, 246, 1, + 2, 51, 50, 157, 232, 16, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 194, + 193, 8, 55, 182, 6, 50, 62, 54, 218, 58, 52, 198, 148, 3, 51, 110, 56, + 202, 196, 2, 49, 215, 94, 53, 10, 26, 45, 207, 231, 17, 32, 8, 96, 2, 50, + 32, 212, 136, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 141, 145, 3, 3, + 51, 32, 76, 2, 175, 140, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, + 0, 2, 53, 32, 198, 147, 4, 50, 202, 181, 4, 48, 141, 209, 6, 2, 55, 32, + 2, 243, 226, 17, 32, 2, 11, 32, 2, 219, 238, 11, 79, 2, 167, 180, 17, 66, + 16, 182, 238, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 162, 1, 22, 48, 155, 5, 49, 140, 1, 82, 49, 54, 50, 118, 51, 62, 52, 78, + 53, 86, 54, 62, 55, 70, 56, 243, 220, 13, 48, 10, 246, 236, 18, 48, 2, + 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 194, 70, 51, 174, 165, 18, + 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 234, 235, 18, 70, 3, 77, 12, + 206, 235, 18, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 146, 235, 18, + 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 198, 234, 18, + 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 233, + 18, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 182, 233, 18, 48, 2, 51, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 232, 18, 48, 2, 49, 2, 50, 2, + 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 226, 255, 11, 49, + 206, 2, 54, 146, 1, 55, 3, 57, 6, 226, 231, 18, 48, 2, 50, 3, 51, 4, 190, + 231, 18, 65, 3, 66, 4, 162, 231, 18, 48, 3, 56, 166, 3, 116, 9, 73, 68, + 69, 79, 71, 82, 65, 77, 32, 216, 17, 10, 77, 79, 78, 79, 71, 82, 65, 77, + 32, 66, 249, 1, 2, 83, 89, 234, 1, 54, 66, 249, 15, 8, 86, 69, 83, 83, + 69, 76, 32, 66, 176, 1, 22, 49, 255, 10, 50, 126, 86, 48, 210, 3, 50, + 162, 1, 51, 86, 52, 122, 53, 114, 54, 146, 1, 55, 118, 56, 71, 57, 28, + 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 148, 247, 6, 3, 50, 32, 87, 234, + 84, 48, 237, 215, 10, 4, 52, 32, 68, 69, 6, 252, 245, 6, 2, 32, 69, 188, + 205, 10, 3, 70, 32, 77, 245, 79, 7, 77, 32, 83, 84, 65, 76, 76, 4, 224, + 233, 1, 3, 70, 32, 69, 221, 191, 15, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, + 2, 77, 32, 2, 213, 195, 11, 4, 72, 69, 45, 71, 4, 228, 250, 8, 3, 77, 32, + 66, 145, 205, 8, 3, 70, 32, 83, 4, 148, 221, 16, 4, 77, 32, 66, 85, 165, + 106, 3, 70, 32, 67, 10, 236, 243, 8, 4, 51, 32, 83, 80, 224, 10, 5, 49, + 32, 66, 65, 82, 160, 218, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, + 80, 69, 209, 99, 4, 48, 32, 87, 72, 6, 54, 49, 192, 178, 16, 3, 48, 32, + 79, 247, 172, 2, 50, 2, 145, 198, 17, 2, 32, 87, 10, 224, 5, 3, 53, 32, + 87, 244, 241, 8, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, + 150, 229, 9, 50, 3, 54, 16, 82, 57, 150, 226, 2, 49, 186, 251, 15, 48, 2, + 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 253, 90, 3, 32, 67, 76, 20, 184, 246, + 13, 5, 51, 32, 65, 82, 77, 204, 163, 4, 5, 50, 32, 71, 65, 82, 170, 67, + 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 202, 198, 13, + 54, 196, 150, 3, 4, 51, 32, 77, 79, 146, 255, 1, 48, 2, 49, 2, 50, 2, 52, + 2, 55, 2, 56, 3, 57, 14, 170, 219, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 3, 57, 4, 240, 157, 2, 3, 49, 32, 72, 247, 188, 16, 48, 50, 42, 50, + 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, 84, 83, + 84, 181, 196, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 219, 198, 17, 79, 12, + 104, 4, 51, 32, 83, 87, 248, 211, 13, 4, 48, 32, 83, 80, 130, 234, 3, 49, + 190, 154, 1, 50, 2, 52, 3, 54, 2, 223, 193, 17, 79, 16, 168, 1, 9, 48, + 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 240, 202, 16, 12, 50, 32, + 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 230, 139, 2, 53, 2, 54, 2, 56, 3, + 57, 2, 133, 205, 17, 3, 32, 67, 72, 2, 183, 176, 6, 32, 18, 184, 207, 17, + 3, 52, 32, 68, 134, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, + 3, 57, 58, 50, 50, 204, 239, 11, 2, 49, 53, 1, 2, 51, 48, 54, 50, 50, + 250, 241, 11, 53, 230, 216, 1, 48, 3, 49, 12, 238, 212, 18, 49, 2, 50, 2, + 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, 55, 32, 68, 73, + 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, 252, 167, 3, 3, + 55, 32, 75, 197, 140, 15, 5, 56, 32, 75, 65, 78, 4, 56, 3, 53, 32, 77, + 241, 173, 15, 5, 51, 32, 65, 82, 69, 2, 231, 201, 11, 69, 176, 1, 84, 9, + 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, 79, 76, 32, 66, + 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, 174, 1, 52, 162, + 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, 118, 54, 250, 191, + 1, 55, 150, 8, 52, 162, 14, 57, 138, 211, 10, 53, 186, 146, 2, 56, 226, + 10, 49, 254, 2, 50, 207, 8, 51, 2, 171, 149, 17, 32, 16, 122, 54, 18, 55, + 158, 201, 1, 49, 142, 2, 50, 158, 26, 52, 196, 186, 1, 2, 53, 32, 198, + 220, 5, 48, 177, 186, 8, 2, 51, 32, 2, 135, 84, 32, 2, 159, 232, 10, 32, + 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, 53, 32, 136, 168, + 3, 2, 48, 32, 228, 253, 13, 2, 55, 32, 140, 7, 2, 52, 32, 146, 89, 56, + 173, 44, 3, 49, 32, 81, 2, 151, 250, 13, 32, 2, 151, 192, 14, 32, 2, 151, + 233, 11, 85, 16, 134, 1, 48, 20, 2, 51, 32, 254, 188, 1, 55, 254, 34, 54, + 194, 233, 4, 57, 138, 148, 2, 56, 210, 154, 4, 49, 185, 180, 5, 3, 50, + 32, 81, 2, 215, 251, 13, 32, 2, 135, 1, 82, 16, 116, 2, 51, 32, 22, 53, + 242, 184, 1, 48, 222, 1, 49, 134, 6, 50, 222, 10, 52, 146, 5, 54, 233, + 180, 12, 3, 56, 32, 78, 2, 199, 246, 14, 65, 2, 171, 189, 16, 32, 18, + 130, 1, 51, 162, 185, 1, 49, 150, 1, 56, 42, 57, 102, 55, 214, 5, 48, + 158, 140, 2, 52, 200, 210, 14, 2, 50, 32, 157, 4, 2, 53, 32, 2, 159, 192, + 11, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, 54, 32, 84, 150, + 180, 1, 55, 226, 3, 57, 146, 2, 53, 158, 228, 6, 49, 163, 221, 9, 48, 2, + 191, 131, 17, 80, 2, 129, 228, 11, 2, 82, 79, 18, 172, 1, 3, 54, 32, 82, + 182, 180, 1, 55, 158, 8, 48, 142, 16, 53, 12, 3, 49, 32, 68, 228, 145, 7, + 2, 52, 32, 214, 235, 3, 50, 152, 229, 5, 3, 56, 32, 81, 241, 2, 2, 51, + 32, 2, 183, 226, 11, 65, 8, 178, 180, 1, 49, 160, 24, 3, 55, 32, 84, 172, + 230, 6, 2, 53, 32, 211, 129, 6, 48, 4, 158, 184, 12, 49, 21, 3, 48, 32, + 68, 28, 90, 52, 30, 54, 30, 56, 186, 220, 11, 53, 158, 2, 49, 30, 51, + 166, 1, 50, 175, 144, 3, 55, 4, 158, 196, 18, 55, 3, 57, 4, 130, 196, 18, + 51, 3, 52, 8, 230, 195, 18, 50, 2, 51, 2, 54, 3, 57, 4, 156, 173, 10, 9, + 69, 68, 32, 80, 65, 80, 69, 82, 67, 155, 131, 7, 32, 5, 155, 148, 17, 84, + 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 253, 219, 6, 11, 80, 85, 78, 67, + 84, 85, 65, 84, 73, 79, 78, 94, 234, 1, 84, 186, 234, 2, 68, 226, 249, 9, + 85, 222, 140, 2, 69, 22, 78, 214, 135, 3, 67, 2, 71, 2, 72, 2, 75, 2, 80, + 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 222, 61, 66, 2, 70, 2, 74, 2, 76, + 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, 79, 78, 69, 32, 154, 248, 17, + 83, 254, 68, 72, 187, 2, 65, 12, 48, 4, 77, 89, 65, 32, 229, 185, 1, 2, + 78, 65, 10, 174, 227, 3, 74, 246, 188, 14, 66, 158, 11, 84, 254, 16, 67, + 39, 78, 208, 1, 244, 1, 2, 67, 75, 132, 1, 6, 71, 73, 67, 65, 76, 32, + 236, 3, 3, 78, 71, 32, 230, 4, 84, 116, 3, 86, 69, 32, 70, 87, 144, 8, 5, + 90, 69, 78, 71, 69, 218, 212, 7, 66, 220, 204, 9, 8, 85, 68, 76, 89, 32, + 67, 82, 89, 133, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, 78, 71, 45, 83, + 72, 73, 70, 84, 32, 149, 16, 9, 32, 87, 73, 84, 72, 32, 73, 78, 75, 4, + 210, 207, 16, 90, 143, 83, 79, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, + 33, 6, 32, 87, 73, 84, 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 187, 239, + 14, 85, 15, 11, 32, 12, 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, + 71, 32, 76, 49, 5, 87, 73, 84, 72, 32, 2, 129, 172, 17, 7, 79, 71, 73, + 67, 65, 76, 32, 10, 26, 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, + 76, 69, 32, 247, 187, 17, 84, 4, 142, 240, 14, 85, 167, 47, 79, 2, 149, + 144, 15, 9, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 213, 196, 7, 5, 73, + 68, 68, 76, 69, 34, 66, 68, 236, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, + 71, 72, 84, 6, 172, 1, 30, 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, + 70, 84, 32, 77, 69, 77, 66, 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, + 69, 196, 180, 13, 2, 73, 86, 191, 217, 3, 82, 2, 17, 2, 32, 86, 2, 181, + 172, 17, 5, 69, 82, 84, 73, 67, 16, 18, 32, 119, 87, 6, 48, 6, 82, 73, + 71, 72, 84, 32, 215, 170, 15, 84, 4, 130, 165, 15, 68, 203, 245, 1, 65, + 12, 26, 87, 215, 216, 8, 32, 10, 29, 5, 65, 82, 68, 83, 32, 10, 82, 65, + 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, 153, 205, 7, 4, 83, 81, 85, 73, 4, + 25, 4, 82, 82, 79, 87, 5, 233, 211, 15, 2, 32, 70, 6, 92, 5, 73, 79, 78, + 32, 66, 144, 153, 17, 9, 32, 79, 70, 32, 70, 79, 82, 84, 85, 199, 91, 85, + 2, 131, 209, 9, 79, 4, 38, 76, 145, 209, 13, 3, 72, 79, 84, 2, 153, 251, + 16, 2, 69, 84, 118, 34, 32, 229, 1, 3, 69, 82, 32, 14, 138, 1, 66, 140, + 169, 10, 11, 68, 79, 85, 66, 76, 69, 32, 80, 82, 73, 77, 214, 246, 5, 65, + 148, 109, 6, 75, 65, 86, 89, 75, 65, 139, 10, 76, 4, 38, 82, 149, 239, 4, + 3, 65, 84, 84, 2, 213, 156, 17, 7, 73, 71, 72, 84, 78, 69, 83, 104, 152, + 1, 5, 72, 65, 76, 70, 32, 92, 5, 76, 69, 70, 84, 32, 220, 2, 6, 82, 73, + 71, 72, 84, 32, 210, 164, 15, 66, 74, 70, 154, 9, 79, 150, 13, 83, 63, + 84, 8, 208, 137, 1, 7, 73, 78, 86, 69, 82, 83, 69, 238, 180, 14, 77, 162, + 7, 66, 195, 153, 1, 67, 40, 174, 1, 66, 60, 8, 70, 79, 85, 78, 84, 65, + 73, 78, 22, 80, 52, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, + 202, 1, 84, 232, 166, 11, 3, 67, 82, 65, 147, 144, 4, 81, 24, 56, 8, 65, + 76, 76, 80, 79, 73, 78, 84, 227, 176, 15, 76, 2, 183, 215, 9, 32, 4, 176, + 203, 14, 5, 65, 73, 78, 84, 66, 171, 109, 69, 2, 177, 173, 7, 5, 32, 65, + 78, 84, 73, 40, 78, 83, 82, 84, 158, 168, 15, 66, 238, 3, 67, 130, 9, 68, + 206, 1, 80, 39, 81, 6, 172, 172, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, + 76, 65, 82, 131, 140, 8, 72, 4, 139, 185, 15, 82, 5, 149, 136, 17, 25, + 32, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, + 78, 84, 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 231, 205, 13, 71, 4, + 140, 134, 17, 5, 65, 82, 32, 69, 67, 191, 100, 71, 114, 104, 12, 67, 73, + 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, + 159, 157, 17, 73, 58, 206, 1, 77, 194, 130, 5, 75, 142, 175, 9, 66, 50, + 84, 178, 251, 1, 65, 2, 69, 2, 78, 130, 248, 1, 68, 2, 71, 2, 72, 2, 73, + 2, 74, 2, 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, + 171, 165, 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 253, 177, 14, 9, + 84, 82, 73, 65, 78, 71, 85, 76, 65, 52, 198, 239, 12, 84, 254, 193, 1, + 76, 154, 239, 1, 83, 226, 11, 65, 2, 69, 2, 78, 130, 248, 1, 66, 2, 67, + 2, 68, 2, 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, + 86, 3, 89, 194, 53, 162, 1, 65, 242, 103, 69, 138, 103, 73, 254, 24, 79, + 136, 112, 3, 82, 79, 32, 234, 3, 85, 164, 70, 7, 89, 65, 78, 77, 65, 82, + 32, 246, 193, 14, 86, 186, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, + 130, 11, 75, 234, 3, 76, 156, 14, 2, 77, 77, 22, 78, 150, 17, 80, 118, + 82, 174, 7, 83, 194, 7, 84, 196, 36, 3, 89, 65, 78, 132, 129, 1, 3, 88, + 73, 77, 243, 151, 15, 67, 6, 156, 146, 17, 4, 73, 67, 32, 87, 226, 88, + 78, 143, 53, 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 133, 3, 10, 74, + 79, 78, 71, 32, 84, 73, 76, 69, 32, 78, 38, 76, 166, 2, 83, 167, 220, 12, + 65, 72, 88, 6, 69, 84, 84, 69, 82, 32, 137, 149, 11, 10, 73, 71, 65, 84, + 85, 82, 69, 32, 83, 72, 70, 202, 173, 3, 78, 138, 208, 10, 82, 242, 55, + 68, 46, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, + 72, 2, 76, 2, 77, 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, + 4, 174, 227, 6, 69, 177, 211, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, + 66, 65, 38, 69, 46, 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, + 87, 112, 5, 71, 82, 69, 69, 78, 0, 3, 82, 69, 68, 168, 128, 6, 3, 65, 85, + 84, 210, 21, 74, 141, 215, 10, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, + 69, 77, 4, 254, 173, 1, 77, 235, 232, 16, 67, 8, 228, 2, 4, 73, 71, 72, + 84, 195, 1, 65, 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, + 79, 82, 65, 2, 73, 78, 8, 218, 1, 78, 237, 197, 11, 3, 82, 67, 72, 2, + 187, 240, 16, 76, 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, + 162, 244, 11, 85, 235, 193, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, + 82, 69, 13, 2, 87, 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, + 173, 238, 6, 5, 66, 65, 77, 66, 79, 4, 180, 132, 6, 2, 73, 82, 145, 148, + 10, 5, 72, 65, 82, 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 163, 222, + 16, 73, 2, 17, 2, 83, 84, 2, 17, 2, 32, 87, 2, 171, 135, 17, 73, 2, 17, + 2, 32, 68, 2, 227, 221, 16, 82, 52, 52, 5, 65, 83, 65, 82, 32, 157, 206, + 14, 2, 69, 77, 50, 162, 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, + 5, 80, 65, 83, 83, 73, 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 173, 136, 18, 3, 65, 78, 71, 2, 213, 159, 15, 5, 78, 68, 32, 79, 70, + 36, 162, 155, 16, 78, 242, 244, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, + 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, + 11, 77, 2, 227, 184, 17, 66, 8, 146, 145, 18, 69, 2, 73, 2, 79, 3, 85, + 246, 1, 80, 7, 65, 89, 65, 76, 65, 77, 32, 208, 11, 2, 69, 32, 225, 1, 3, + 84, 69, 83, 236, 1, 190, 1, 65, 22, 68, 44, 9, 70, 82, 65, 67, 84, 73, + 79, 78, 32, 240, 1, 7, 76, 69, 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, + 66, 69, 82, 32, 36, 5, 83, 73, 71, 78, 32, 143, 144, 13, 86, 2, 227, 142, + 13, 85, 22, 172, 146, 8, 2, 65, 84, 251, 141, 8, 73, 26, 56, 4, 79, 78, + 69, 32, 121, 6, 84, 72, 82, 69, 69, 32, 18, 82, 84, 142, 128, 5, 83, 202, + 219, 7, 70, 58, 79, 170, 176, 3, 69, 46, 72, 43, 81, 4, 254, 222, 12, 87, + 239, 174, 3, 69, 8, 166, 128, 5, 83, 174, 223, 7, 69, 110, 84, 163, 174, + 3, 81, 132, 1, 226, 1, 65, 82, 67, 158, 1, 68, 78, 84, 82, 86, 222, 221, + 12, 78, 230, 46, 76, 210, 90, 82, 130, 81, 85, 158, 144, 1, 79, 182, 56, + 73, 202, 190, 1, 83, 82, 66, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, + 61, 72, 2, 77, 3, 89, 11, 148, 206, 15, 7, 82, 67, 72, 65, 73, 67, 32, + 158, 188, 2, 65, 2, 73, 3, 85, 22, 26, 72, 203, 137, 18, 65, 20, 44, 5, + 73, 76, 76, 85, 32, 155, 137, 18, 65, 18, 226, 211, 12, 76, 182, 189, 3, + 78, 178, 191, 1, 82, 210, 56, 75, 2, 77, 3, 89, 10, 164, 198, 10, 4, 79, + 84, 32, 82, 238, 250, 6, 68, 254, 68, 72, 187, 2, 65, 10, 38, 84, 158, + 133, 18, 72, 187, 2, 65, 6, 154, 133, 18, 72, 2, 84, 187, 2, 65, 12, 250, + 225, 3, 69, 206, 193, 10, 79, 231, 227, 3, 65, 6, 158, 222, 12, 79, 231, + 215, 3, 84, 18, 50, 67, 114, 86, 194, 158, 14, 65, 179, 162, 3, 80, 6, + 54, 79, 120, 5, 73, 82, 67, 85, 76, 215, 158, 14, 65, 2, 185, 142, 13, 9, + 77, 66, 73, 78, 73, 78, 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, + 32, 66, 255, 193, 17, 73, 2, 17, 2, 65, 82, 2, 233, 163, 12, 2, 32, 86, + 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 69, 32, 70, 65, 219, + 252, 16, 83, 4, 64, 10, 65, 78, 68, 32, 77, 65, 76, 69, 32, 65, 219, 252, + 16, 83, 2, 25, 4, 78, 68, 32, 70, 2, 11, 69, 2, 11, 77, 2, 199, 190, 7, + 65, 2, 231, 196, 16, 69, 2, 199, 131, 16, 79, 185, 1, 210, 1, 32, 220, 2, + 5, 68, 65, 73, 67, 32, 220, 3, 8, 73, 67, 72, 65, 69, 65, 78, 32, 222, 8, + 83, 200, 165, 2, 3, 85, 65, 76, 166, 181, 10, 65, 220, 168, 1, 8, 84, 69, + 76, 80, 73, 69, 67, 69, 147, 207, 3, 71, 12, 132, 1, 3, 73, 78, 32, 128, + 1, 5, 87, 73, 84, 72, 32, 196, 146, 2, 3, 68, 65, 78, 169, 194, 12, 8, + 65, 78, 68, 32, 87, 79, 77, 65, 4, 208, 240, 9, 19, 66, 85, 83, 73, 78, + 69, 83, 83, 32, 83, 85, 73, 84, 32, 76, 69, 86, 73, 84, 209, 148, 1, 4, + 84, 85, 88, 69, 4, 204, 195, 13, 8, 71, 85, 65, 32, 80, 73, 32, 77, 213, + 243, 2, 4, 84, 85, 82, 66, 58, 136, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 230, 209, 10, 71, 176, 184, 3, 3, 86, 79, 67, 146, 70, 80, 177, 195, 2, + 6, 65, 70, 70, 82, 73, 67, 50, 82, 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, + 72, 65, 76, 22, 73, 167, 180, 17, 75, 38, 154, 1, 75, 150, 198, 12, 84, + 226, 193, 1, 83, 194, 163, 3, 73, 214, 79, 66, 2, 68, 2, 71, 2, 72, 2, + 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 3, 90, 5, 207, 248, 17, 83, 2, + 253, 239, 7, 2, 83, 72, 2, 159, 248, 17, 81, 4, 194, 250, 17, 78, 3, 84, + 102, 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 4, 78, 80, 12, 80, 85, + 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 220, 1, 4, 83, 73, 71, 78, 185, + 172, 10, 16, 65, 66, 66, 82, 69, 86, 73, 65, 84, 73, 79, 78, 32, 77, 65, + 82, 72, 218, 1, 65, 46, 66, 38, 68, 30, 71, 30, 74, 2, 90, 30, 75, 30, + 81, 38, 83, 50, 84, 54, 88, 238, 154, 6, 76, 254, 194, 1, 82, 238, 213, + 2, 89, 154, 193, 1, 72, 166, 255, 4, 78, 254, 1, 87, 202, 103, 70, 2, 80, + 171, 4, 77, 6, 206, 180, 10, 76, 122, 65, 239, 193, 6, 89, 4, 238, 200, + 12, 72, 211, 174, 3, 69, 4, 206, 156, 6, 65, 23, 72, 4, 246, 179, 10, 72, + 15, 73, 4, 178, 180, 10, 72, 15, 65, 4, 246, 251, 11, 72, 15, 65, 4, 162, + 156, 6, 72, 199, 223, 5, 79, 8, 250, 176, 10, 83, 154, 3, 65, 191, 193, + 6, 72, 6, 158, 155, 6, 72, 186, 218, 9, 69, 247, 128, 1, 65, 4, 222, 250, + 11, 65, 3, 79, 10, 33, 6, 85, 77, 66, 69, 82, 32, 10, 186, 179, 10, 79, + 58, 84, 219, 183, 2, 70, 14, 80, 2, 68, 79, 116, 3, 76, 73, 78, 138, 159, + 5, 70, 142, 133, 11, 84, 255, 9, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, + 32, 68, 79, 84, 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 163, 192, 2, + 73, 2, 223, 201, 3, 69, 2, 159, 199, 16, 32, 2, 11, 32, 2, 221, 218, 17, + 2, 83, 72, 4, 92, 17, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, + 73, 71, 72, 84, 175, 191, 10, 76, 2, 175, 193, 14, 72, 142, 1, 142, 1, + 65, 20, 5, 67, 72, 69, 78, 32, 160, 155, 6, 4, 82, 73, 65, 71, 165, 213, + 10, 13, 84, 73, 65, 76, 32, 65, 82, 84, 83, 32, 85, 78, 73, 2, 175, 153, + 5, 67, 136, 1, 152, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, + 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 157, 17, 72, 225, 5, + 4, 77, 65, 82, 75, 60, 254, 3, 84, 146, 148, 2, 68, 178, 222, 13, 78, + 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 254, 68, 45, 2, 66, 2, 71, + 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, + 73, 71, 78, 32, 37, 16, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, + 84, 69, 82, 32, 4, 154, 133, 14, 67, 191, 161, 3, 65, 58, 182, 1, 84, + 146, 148, 2, 68, 178, 222, 13, 78, 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, + 2, 90, 254, 68, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, + 89, 187, 2, 65, 8, 178, 162, 17, 83, 254, 68, 72, 187, 2, 65, 10, 130, + 231, 17, 65, 186, 2, 69, 2, 73, 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, + 65, 77, 32, 71, 79, 78, 68, 73, 32, 236, 5, 3, 67, 85, 76, 64, 5, 75, 32, + 87, 79, 82, 163, 160, 17, 85, 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, + 178, 2, 82, 56, 5, 83, 73, 71, 78, 32, 82, 86, 199, 245, 15, 68, 94, 210, + 1, 74, 42, 84, 206, 252, 13, 65, 38, 68, 146, 25, 85, 210, 200, 1, 73, + 42, 76, 210, 189, 1, 75, 38, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, + 254, 68, 72, 2, 77, 2, 82, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 230, 226, + 17, 78, 38, 72, 187, 2, 65, 10, 230, 157, 17, 84, 254, 68, 72, 2, 82, + 187, 2, 65, 4, 32, 2, 65, 45, 163, 240, 1, 69, 2, 131, 160, 17, 75, 10, + 174, 176, 1, 67, 202, 160, 12, 72, 242, 45, 78, 130, 182, 1, 86, 223, + 234, 1, 65, 22, 26, 79, 207, 177, 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 20, 78, 86, 238, 253, 13, 65, 190, 21, 85, 210, 200, 1, + 73, 206, 134, 2, 69, 3, 79, 2, 201, 182, 7, 6, 79, 67, 65, 76, 73, 67, 2, + 253, 168, 17, 11, 73, 78, 69, 32, 79, 82, 68, 73, 78, 65, 76, 2, 199, + 206, 16, 75, 226, 15, 76, 10, 72, 69, 77, 65, 84, 73, 67, 65, 76, 32, + 129, 162, 14, 3, 69, 32, 68, 224, 15, 252, 1, 5, 66, 79, 76, 68, 32, 168, + 6, 14, 68, 79, 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 75, 32, 238, 1, + 70, 164, 2, 7, 73, 84, 65, 76, 73, 67, 32, 180, 3, 10, 77, 79, 78, 79, + 83, 80, 65, 67, 69, 32, 40, 2, 82, 73, 36, 3, 76, 69, 70, 167, 1, 83, + 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 130, 2, 83, 210, 14, + 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, 98, 82, 242, 5, 84, 56, 7, 70, + 82, 65, 75, 84, 85, 82, 131, 211, 15, 68, 104, 190, 4, 68, 174, 15, 84, + 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, + 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, + 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, + 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, 213, 25, 5, 67, 82, 73, 80, 84, + 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, + 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, + 196, 1, 67, 214, 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, + 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, 26, 73, 191, 180, 14, 69, 2, 167, 213, + 1, 71, 110, 68, 8, 67, 65, 80, 73, 84, 65, 76, 32, 162, 23, 83, 195, 210, + 15, 68, 38, 250, 215, 17, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, + 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, + 88, 3, 89, 96, 52, 7, 82, 65, 75, 84, 85, 82, 32, 139, 252, 6, 65, 94, + 52, 8, 67, 65, 80, 73, 84, 65, 76, 32, 143, 21, 83, 42, 230, 213, 17, 65, + 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, + 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, + 1, 100, 6, 83, 77, 65, 76, 76, 32, 222, 7, 67, 230, 2, 69, 38, 75, 38, + 78, 30, 80, 98, 82, 243, 5, 84, 104, 242, 1, 68, 198, 12, 65, 22, 66, 2, + 90, 42, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, + 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, 78, + 210, 200, 1, 88, 206, 134, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 52, 7, + 79, 84, 76, 69, 83, 83, 32, 131, 173, 14, 69, 4, 154, 209, 17, 73, 3, 74, + 124, 130, 16, 67, 34, 83, 195, 210, 15, 68, 12, 32, 2, 71, 72, 179, 246, + 6, 83, 10, 17, 2, 84, 32, 10, 112, 6, 87, 72, 73, 84, 69, 32, 170, 236, + 5, 68, 234, 99, 65, 197, 164, 7, 9, 70, 76, 65, 84, 84, 69, 78, 69, 68, + 4, 254, 246, 13, 83, 39, 84, 130, 6, 84, 10, 65, 78, 83, 45, 83, 69, 82, + 73, 70, 32, 133, 14, 6, 67, 82, 73, 80, 84, 32, 176, 5, 96, 5, 66, 79, + 76, 68, 32, 176, 12, 6, 73, 84, 65, 76, 73, 67, 34, 67, 34, 83, 195, 210, + 15, 68, 204, 3, 98, 73, 122, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, + 82, 30, 83, 214, 5, 84, 187, 211, 15, 68, 220, 1, 33, 6, 84, 65, 76, 73, + 67, 32, 220, 1, 74, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, + 83, 215, 5, 84, 102, 37, 7, 65, 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, + 186, 4, 65, 22, 66, 2, 90, 22, 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, + 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, + 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, + 86, 2, 87, 3, 89, 9, 40, 4, 72, 69, 84, 65, 183, 164, 17, 65, 5, 151, + 182, 16, 32, 2, 185, 212, 15, 4, 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, + 159, 7, 80, 2, 165, 166, 17, 2, 65, 66, 6, 74, 72, 168, 141, 5, 8, 65, + 82, 84, 73, 65, 76, 32, 68, 163, 167, 11, 73, 2, 199, 180, 16, 73, 2, + 177, 180, 16, 2, 72, 79, 102, 29, 5, 77, 65, 76, 76, 32, 102, 246, 1, 65, + 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, + 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, + 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, 74, 2, 81, 2, + 86, 2, 87, 3, 89, 5, 235, 208, 1, 76, 5, 175, 160, 17, 69, 5, 203, 160, + 14, 69, 7, 178, 212, 1, 80, 235, 237, 15, 84, 5, 11, 73, 2, 29, 5, 78, + 65, 76, 32, 83, 2, 227, 1, 73, 5, 219, 248, 14, 65, 5, 147, 159, 17, 79, + 5, 11, 65, 2, 223, 158, 14, 80, 5, 143, 160, 14, 65, 7, 11, 77, 4, 230, + 170, 1, 73, 239, 213, 15, 69, 9, 142, 175, 17, 72, 2, 83, 219, 19, 73, 5, + 203, 163, 17, 72, 5, 11, 73, 2, 143, 161, 17, 71, 7, 190, 157, 14, 72, + 231, 255, 2, 65, 5, 199, 209, 1, 80, 2, 11, 72, 2, 11, 69, 2, 11, 84, 2, + 159, 174, 16, 65, 104, 11, 32, 104, 18, 67, 35, 83, 52, 53, 5, 65, 80, + 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 183, 216, 11, 76, 82, 76, 8, 67, + 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, 76, 76, 32, 36, 222, + 191, 17, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 78, 2, 79, 2, 80, 2, + 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 46, 194, 190, + 17, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, + 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, + 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, 76, 32, 40, 70, 69, + 66, 70, 70, 78, 26, 83, 66, 84, 142, 206, 15, 90, 143, 83, 79, 6, 40, 4, + 73, 71, 72, 84, 243, 224, 9, 76, 5, 231, 234, 15, 69, 8, 30, 73, 105, 3, + 79, 85, 82, 4, 202, 254, 4, 70, 139, 166, 12, 86, 4, 65, 3, 73, 78, 69, + 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 191, 233, 15, 84, 10, 42, 72, + 190, 224, 9, 87, 243, 137, 7, 69, 4, 150, 253, 4, 73, 183, 208, 10, 82, + 248, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, 73, 67, 65, 76, 32, 34, + 68, 180, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, 69, 75, 32, 178, 11, 76, + 34, 78, 162, 50, 82, 172, 15, 8, 83, 83, 65, 71, 69, 32, 87, 65, 20, 2, + 84, 82, 183, 183, 16, 77, 26, 72, 6, 83, 85, 82, 69, 68, 32, 249, 141, + 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, 71, 76, 69, 200, 247, + 9, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 175, 173, 7, 66, 21, 11, 32, + 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, 78, 32, 65, 82, 77, 32, + 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, 82, 79, 87, 32, 80, 79, + 73, 78, 84, 73, 78, 71, 32, 161, 130, 17, 7, 79, 80, 69, 78, 73, 78, 71, + 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 43, 85, 4, 73, 3, + 79, 87, 78, 4, 173, 249, 7, 5, 84, 32, 65, 78, 68, 4, 11, 80, 4, 145, + 190, 11, 3, 32, 65, 78, 4, 254, 180, 16, 65, 211, 56, 76, 250, 1, 56, 9, + 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, 1, 174, 1, 67, 52, + 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, + 83, 132, 255, 6, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 215, + 228, 7, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 255, 226, 14, 79, 26, 82, + 84, 48, 2, 79, 78, 194, 194, 15, 70, 30, 83, 102, 90, 130, 83, 78, 211, + 110, 69, 8, 44, 3, 72, 82, 69, 209, 176, 16, 2, 87, 79, 4, 207, 176, 16, + 69, 20, 50, 84, 226, 240, 4, 69, 46, 70, 42, 78, 31, 83, 6, 162, 242, 4, + 72, 204, 227, 4, 2, 87, 69, 215, 137, 7, 69, 70, 68, 3, 77, 65, 76, 217, + 183, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 146, 221, 3, 72, 2, 75, 162, 255, + 9, 89, 138, 143, 3, 79, 150, 64, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, + 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, + 86, 2, 87, 2, 88, 3, 90, 7, 162, 237, 3, 84, 207, 190, 13, 73, 7, 202, + 171, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, 45, 3, 85, 77, 32, 6, + 138, 211, 10, 69, 206, 140, 4, 67, 115, 81, 54, 210, 1, 66, 46, 70, 164, + 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, 11, 77, 65, 84, 72, 69, + 77, 65, 84, 73, 67, 65, 22, 83, 104, 4, 84, 72, 82, 69, 186, 212, 14, 86, + 230, 59, 69, 166, 4, 87, 207, 10, 71, 4, 164, 4, 3, 79, 76, 68, 163, 201, + 13, 76, 10, 84, 9, 76, 65, 84, 84, 69, 78, 69, 68, 32, 204, 3, 3, 79, 85, + 82, 219, 145, 15, 73, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, + 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, 32, 41, 15, 45, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 65, 78, 71, 76, 69, 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, + 2, 17, 2, 32, 66, 2, 237, 163, 7, 4, 82, 65, 67, 75, 2, 153, 226, 16, 10, + 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 2, 251, 230, 16, 76, 10, 72, 4, + 77, 65, 76, 76, 154, 181, 14, 72, 188, 91, 2, 73, 88, 183, 2, 65, 2, 133, + 162, 14, 2, 32, 87, 4, 11, 69, 4, 45, 9, 32, 80, 79, 73, 78, 84, 69, 68, + 32, 4, 206, 179, 9, 80, 151, 221, 5, 66, 158, 1, 146, 1, 65, 104, 6, 67, + 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, 76, 76, 65, 66, 76, 69, 0, + 4, 87, 79, 82, 68, 50, 86, 223, 172, 15, 68, 6, 80, 8, 72, 65, 78, 71, + 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 155, 177, 13, 78, 2, 175, 232, 15, + 68, 4, 130, 210, 16, 65, 255, 59, 69, 94, 52, 6, 69, 84, 84, 69, 82, 32, + 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, 67, 72, 38, 68, 50, 71, 38, 74, + 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, 54, 73, 0, 3, 76, 65, 73, 0, 3, + 77, 73, 84, 230, 210, 12, 72, 166, 10, 82, 2, 87, 188, 144, 2, 2, 65, 84, + 230, 213, 1, 89, 242, 8, 85, 214, 79, 69, 3, 79, 4, 198, 229, 15, 72, + 191, 185, 1, 65, 4, 194, 149, 16, 73, 159, 137, 1, 65, 8, 238, 141, 10, + 72, 178, 135, 6, 73, 235, 65, 68, 4, 190, 141, 10, 72, 183, 141, 7, 79, + 4, 170, 228, 15, 72, 163, 48, 73, 6, 216, 1, 2, 79, 75, 163, 139, 10, 72, + 12, 178, 1, 65, 0, 3, 71, 79, 85, 182, 153, 17, 78, 3, 89, 6, 118, 65, + 179, 226, 15, 72, 6, 238, 139, 17, 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, + 76, 162, 139, 10, 72, 155, 201, 6, 84, 5, 157, 177, 1, 4, 32, 76, 79, 78, + 2, 161, 152, 17, 3, 32, 73, 89, 2, 141, 178, 16, 7, 32, 82, 69, 80, 69, + 84, 73, 30, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 191, 232, 14, + 73, 28, 130, 1, 65, 44, 4, 67, 72, 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, + 89, 69, 22, 73, 38, 85, 206, 231, 10, 78, 187, 129, 4, 86, 8, 218, 136, + 16, 78, 170, 80, 65, 175, 64, 85, 2, 175, 136, 16, 78, 4, 154, 136, 16, + 78, 215, 144, 1, 73, 4, 246, 135, 16, 78, 215, 144, 1, 85, 4, 158, 144, + 16, 84, 183, 56, 79, 174, 3, 160, 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, + 85, 73, 32, 244, 130, 16, 20, 79, 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, + 73, 78, 69, 32, 66, 82, 65, 78, 67, 95, 83, 170, 3, 148, 1, 17, 67, 79, + 77, 66, 73, 78, 73, 78, 71, 32, 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, + 83, 89, 76, 76, 65, 66, 76, 69, 32, 77, 199, 144, 9, 68, 14, 62, 84, 56, + 7, 72, 85, 78, 68, 82, 69, 68, 243, 177, 4, 77, 8, 26, 69, 187, 178, 4, + 72, 6, 26, 78, 187, 230, 13, 69, 4, 140, 178, 4, 2, 32, 84, 191, 226, 12, + 83, 138, 3, 22, 48, 167, 21, 49, 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, + 138, 2, 51, 254, 1, 52, 150, 2, 53, 158, 2, 54, 238, 1, 55, 142, 2, 56, + 187, 2, 57, 18, 142, 1, 49, 34, 50, 22, 51, 22, 52, 218, 149, 2, 53, 142, + 230, 3, 56, 180, 149, 10, 3, 57, 32, 77, 92, 3, 55, 32, 77, 213, 90, 3, + 54, 32, 87, 2, 11, 32, 2, 147, 254, 16, 75, 2, 239, 136, 17, 32, 2, 183, + 239, 11, 32, 2, 11, 32, 2, 203, 253, 16, 87, 20, 130, 1, 49, 22, 54, 18, + 56, 22, 57, 200, 28, 2, 48, 32, 138, 161, 7, 53, 130, 190, 5, 52, 226, + 10, 55, 222, 9, 50, 199, 191, 3, 51, 2, 227, 226, 16, 32, 2, 143, 25, 32, + 2, 139, 170, 13, 32, 2, 191, 209, 12, 32, 20, 106, 49, 22, 50, 22, 51, + 22, 52, 22, 53, 22, 54, 22, 55, 22, 57, 158, 186, 11, 48, 253, 232, 1, 2, + 56, 32, 2, 155, 144, 10, 32, 2, 195, 234, 9, 32, 2, 223, 233, 16, 32, 2, + 215, 233, 11, 32, 2, 131, 191, 12, 32, 2, 215, 235, 16, 32, 2, 143, 185, + 12, 32, 2, 191, 28, 32, 20, 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, + 2, 56, 32, 230, 186, 4, 57, 238, 51, 50, 22, 53, 188, 201, 7, 2, 54, 32, + 244, 211, 3, 3, 55, 32, 78, 213, 90, 3, 51, 32, 89, 2, 187, 37, 32, 2, + 159, 248, 16, 89, 2, 139, 248, 16, 70, 2, 155, 196, 15, 78, 20, 184, 1, + 2, 48, 32, 20, 2, 51, 32, 30, 53, 22, 56, 164, 8, 3, 52, 32, 75, 252, 21, + 3, 54, 32, 72, 230, 2, 55, 170, 205, 4, 49, 212, 164, 4, 3, 57, 32, 87, + 245, 181, 5, 3, 50, 32, 72, 2, 203, 157, 15, 72, 2, 177, 199, 16, 2, 78, + 71, 2, 223, 252, 10, 32, 2, 187, 234, 16, 32, 20, 178, 1, 48, 18, 53, 20, + 2, 54, 32, 20, 2, 56, 32, 22, 57, 172, 190, 9, 2, 50, 32, 236, 4, 2, 51, + 32, 166, 200, 1, 49, 164, 187, 3, 3, 52, 32, 76, 213, 137, 1, 3, 55, 32, + 78, 2, 143, 4, 32, 2, 183, 195, 15, 32, 2, 167, 218, 16, 71, 2, 227, 236, + 12, 78, 2, 177, 182, 15, 2, 32, 77, 20, 192, 1, 2, 50, 32, 22, 54, 242, + 25, 51, 164, 2, 3, 55, 32, 78, 52, 3, 49, 32, 87, 156, 215, 1, 2, 52, 32, + 206, 143, 3, 56, 156, 255, 4, 3, 48, 32, 78, 174, 92, 57, 137, 186, 4, 3, + 53, 32, 75, 2, 159, 216, 16, 77, 2, 195, 254, 9, 32, 20, 184, 1, 3, 52, + 32, 75, 20, 2, 53, 32, 20, 2, 55, 32, 22, 57, 178, 9, 51, 234, 11, 50, + 144, 2, 3, 49, 32, 71, 40, 3, 48, 32, 71, 144, 171, 14, 3, 54, 32, 75, + 149, 84, 3, 56, 32, 70, 2, 251, 222, 16, 80, 2, 251, 236, 16, 70, 2, 199, + 228, 16, 86, 2, 139, 171, 8, 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, + 20, 2, 52, 32, 22, 53, 248, 13, 6, 54, 32, 76, 79, 78, 71, 152, 12, 2, + 57, 32, 160, 167, 3, 3, 51, 32, 72, 132, 252, 8, 4, 50, 32, 78, 71, 164, + 195, 3, 3, 55, 32, 72, 165, 97, 3, 56, 32, 70, 2, 215, 234, 16, 89, 2, + 227, 148, 15, 80, 2, 207, 148, 15, 76, 2, 171, 132, 16, 32, 20, 230, 1, + 53, 164, 14, 4, 48, 32, 78, 71, 172, 8, 3, 51, 32, 71, 244, 243, 11, 2, + 55, 32, 214, 118, 57, 252, 88, 3, 50, 32, 75, 172, 184, 1, 3, 49, 32, 84, + 192, 27, 4, 56, 32, 78, 89, 148, 129, 1, 3, 52, 32, 77, 213, 34, 2, 54, + 32, 2, 155, 207, 16, 32, 196, 1, 118, 48, 154, 2, 49, 210, 2, 50, 170, 2, + 51, 222, 1, 52, 162, 2, 53, 214, 2, 54, 166, 2, 55, 238, 2, 56, 231, 2, + 57, 20, 222, 1, 49, 30, 52, 172, 11, 6, 54, 32, 76, 79, 78, 71, 164, 4, + 6, 53, 32, 76, 79, 78, 71, 24, 2, 51, 32, 162, 229, 4, 50, 230, 152, 2, + 48, 204, 249, 4, 3, 55, 32, 71, 252, 135, 3, 3, 57, 32, 89, 197, 39, 3, + 56, 32, 75, 2, 205, 180, 15, 2, 32, 70, 2, 157, 187, 14, 2, 32, 84, 20, + 208, 1, 6, 48, 32, 76, 79, 78, 71, 22, 51, 34, 54, 22, 56, 32, 2, 57, 32, + 156, 18, 4, 53, 32, 78, 71, 224, 206, 2, 2, 55, 32, 192, 145, 2, 3, 50, + 32, 75, 140, 198, 9, 3, 52, 32, 87, 145, 254, 1, 2, 49, 32, 2, 211, 216, + 15, 32, 2, 11, 32, 2, 131, 227, 16, 74, 2, 207, 240, 15, 32, 2, 11, 32, + 2, 207, 226, 16, 87, 2, 155, 236, 14, 78, 20, 230, 1, 50, 32, 2, 51, 32, + 232, 3, 3, 52, 32, 71, 182, 10, 48, 220, 147, 7, 2, 57, 32, 176, 196, 2, + 4, 56, 32, 72, 79, 248, 207, 2, 5, 55, 32, 78, 71, 71, 236, 90, 2, 53, + 32, 180, 249, 1, 3, 54, 32, 87, 129, 148, 1, 2, 49, 32, 2, 11, 32, 2, + 203, 247, 12, 77, 2, 11, 78, 2, 143, 227, 16, 68, 20, 172, 8, 3, 52, 32, + 78, 204, 2, 2, 56, 32, 224, 233, 4, 3, 50, 32, 75, 194, 195, 2, 49, 2, + 53, 216, 173, 2, 2, 55, 32, 152, 208, 4, 3, 51, 32, 70, 0, 3, 54, 32, 83, + 164, 50, 2, 48, 32, 225, 145, 1, 3, 57, 32, 87, 20, 232, 1, 3, 53, 32, + 77, 22, 54, 224, 2, 3, 57, 32, 78, 172, 4, 6, 50, 32, 76, 79, 78, 71, + 130, 196, 6, 55, 140, 151, 3, 3, 51, 32, 87, 200, 212, 2, 2, 48, 32, 220, + 60, 3, 56, 32, 71, 244, 190, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 179, + 223, 16, 66, 2, 129, 191, 15, 3, 32, 78, 71, 20, 192, 1, 2, 50, 32, 34, + 52, 26, 53, 34, 54, 36, 2, 55, 32, 244, 7, 2, 48, 32, 180, 225, 9, 3, 56, + 32, 75, 148, 182, 2, 2, 49, 32, 148, 225, 2, 5, 57, 32, 78, 71, 71, 249, + 204, 1, 2, 51, 32, 2, 11, 78, 2, 207, 238, 16, 74, 2, 213, 5, 2, 32, 77, + 2, 11, 32, 2, 247, 220, 16, 71, 2, 241, 175, 14, 4, 32, 78, 71, 71, 2, + 227, 131, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, 49, 32, 76, 79, 78, 71, + 30, 56, 204, 2, 2, 55, 32, 128, 170, 9, 3, 52, 32, 78, 200, 48, 4, 54, + 32, 71, 85, 152, 208, 4, 2, 53, 32, 148, 84, 3, 50, 32, 83, 0, 2, 51, 32, + 185, 98, 2, 57, 32, 2, 219, 173, 14, 74, 2, 221, 249, 11, 2, 32, 77, 2, + 183, 170, 12, 32, 20, 198, 1, 50, 46, 52, 40, 6, 53, 32, 76, 79, 78, 71, + 28, 3, 55, 32, 78, 34, 56, 152, 199, 11, 2, 54, 32, 156, 99, 3, 57, 32, + 75, 160, 212, 2, 3, 51, 32, 86, 156, 108, 4, 48, 32, 78, 89, 207, 53, 49, + 2, 11, 32, 2, 11, 77, 2, 183, 171, 14, 66, 2, 17, 2, 32, 77, 2, 227, 204, + 16, 66, 2, 205, 204, 16, 2, 32, 74, 2, 11, 71, 2, 231, 163, 15, 85, 2, + 147, 154, 15, 32, 20, 208, 1, 2, 48, 32, 22, 49, 22, 50, 32, 6, 51, 32, + 76, 79, 78, 71, 34, 56, 252, 241, 8, 2, 53, 32, 152, 126, 3, 52, 32, 78, + 188, 182, 2, 2, 54, 32, 228, 210, 2, 4, 55, 32, 77, 66, 145, 30, 4, 57, + 32, 77, 85, 2, 215, 168, 14, 68, 2, 215, 238, 9, 32, 2, 11, 32, 2, 147, + 213, 16, 86, 2, 253, 222, 14, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 239, + 251, 14, 66, 16, 142, 1, 48, 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, + 51, 32, 20, 3, 52, 32, 87, 22, 53, 20, 2, 54, 32, 169, 165, 12, 3, 55, + 32, 70, 2, 11, 32, 2, 191, 250, 14, 71, 2, 171, 250, 14, 68, 2, 227, 193, + 16, 74, 2, 215, 150, 16, 72, 2, 203, 210, 16, 85, 2, 219, 158, 15, 32, 2, + 227, 144, 1, 83, 248, 1, 68, 6, 79, 73, 84, 73, 67, 32, 250, 145, 10, 80, + 241, 70, 2, 67, 85, 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, + 10, 13, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, + 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, + 66, 69, 82, 32, 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, + 84, 61, 3, 78, 73, 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, + 192, 1, 2, 73, 86, 13, 3, 79, 85, 82, 4, 232, 225, 14, 4, 84, 87, 69, 76, + 19, 72, 4, 26, 69, 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, + 82, 69, 13, 2, 87, 79, 2, 23, 78, 2, 11, 69, 2, 197, 227, 14, 5, 32, 84, + 87, 69, 76, 52, 76, 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, + 82, 65, 77, 32, 48, 182, 1, 65, 46, 84, 146, 234, 1, 78, 2, 83, 206, 252, + 12, 72, 138, 176, 1, 75, 254, 68, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, + 2, 82, 2, 87, 2, 89, 186, 2, 69, 2, 73, 3, 79, 5, 149, 139, 11, 6, 82, + 67, 72, 65, 73, 67, 6, 150, 222, 16, 65, 2, 69, 3, 79, 4, 174, 201, 4, + 73, 153, 248, 11, 2, 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, + 4, 78, 73, 78, 69, 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, + 32, 24, 18, 73, 35, 79, 12, 142, 2, 86, 139, 228, 3, 70, 12, 148, 2, 2, + 85, 82, 227, 227, 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 231, 141, 6, + 32, 24, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 227, 227, 3, + 84, 28, 34, 72, 50, 87, 159, 140, 6, 69, 12, 32, 2, 82, 69, 215, 227, 3, + 73, 8, 39, 69, 12, 26, 79, 215, 227, 3, 69, 9, 11, 32, 6, 194, 138, 6, + 72, 171, 167, 5, 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 229, 160, 3, + 11, 83, 89, 77, 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, + 22, 78, 30, 83, 38, 84, 174, 223, 14, 72, 138, 176, 1, 75, 254, 68, 68, + 2, 76, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, + 79, 4, 179, 159, 3, 65, 8, 158, 159, 3, 65, 3, 69, 6, 130, 159, 3, 65, + 139, 184, 13, 69, 10, 222, 158, 3, 65, 2, 69, 139, 184, 13, 79, 2, 175, + 252, 11, 73, 20, 44, 5, 73, 67, 65, 76, 32, 251, 213, 16, 79, 18, 116, + 10, 76, 79, 78, 71, 32, 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, + 83, 72, 79, 82, 84, 32, 79, 187, 32, 66, 4, 188, 244, 2, 2, 83, 72, 185, + 180, 10, 7, 84, 87, 79, 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, + 34, 82, 41, 10, 87, 79, 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, + 2, 23, 65, 2, 11, 73, 2, 217, 187, 15, 2, 83, 69, 4, 26, 79, 179, 231, 7, + 74, 2, 137, 162, 10, 4, 86, 69, 82, 32, 224, 2, 100, 3, 65, 79, 32, 136, + 18, 2, 67, 82, 142, 1, 68, 118, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, + 82, 170, 2, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, + 71, 78, 32, 212, 1, 5, 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, + 76, 50, 78, 106, 82, 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, + 118, 90, 238, 247, 12, 81, 230, 200, 1, 80, 242, 190, 1, 72, 2, 77, 254, + 68, 70, 2, 75, 2, 87, 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 179, + 206, 16, 72, 8, 254, 231, 8, 90, 170, 151, 4, 78, 211, 204, 3, 77, 4, + 162, 186, 16, 82, 219, 19, 65, 16, 50, 90, 154, 4, 76, 218, 198, 16, 68, + 187, 2, 65, 8, 206, 185, 16, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 190, + 133, 16, 72, 183, 71, 65, 8, 238, 150, 10, 72, 174, 179, 6, 89, 187, 2, + 65, 18, 54, 65, 186, 132, 16, 71, 2, 78, 2, 89, 255, 68, 72, 5, 11, 83, + 2, 169, 159, 13, 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, + 68, 32, 84, 171, 180, 16, 84, 14, 40, 4, 79, 78, 69, 45, 151, 211, 14, + 83, 12, 206, 202, 16, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 186, 182, + 16, 89, 162, 17, 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, + 45, 70, 83, 130, 198, 16, 84, 186, 2, 65, 3, 69, 4, 171, 147, 10, 72, 14, + 250, 200, 16, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 254, 197, + 16, 72, 2, 83, 186, 2, 65, 3, 69, 4, 206, 197, 16, 70, 187, 2, 65, 16, + 70, 84, 228, 207, 14, 2, 68, 90, 250, 60, 78, 142, 184, 1, 75, 3, 80, 8, + 234, 255, 15, 83, 254, 68, 84, 187, 2, 65, 16, 50, 90, 142, 255, 15, 83, + 254, 68, 72, 187, 2, 65, 8, 218, 144, 10, 83, 174, 179, 6, 89, 187, 2, + 65, 8, 144, 1, 9, 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 145, 245, + 13, 18, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, + 69, 82, 4, 30, 65, 189, 88, 2, 86, 79, 2, 225, 230, 11, 3, 83, 80, 73, 8, + 130, 172, 14, 66, 168, 74, 3, 84, 79, 80, 182, 81, 65, 143, 82, 82, 104, + 146, 1, 65, 78, 69, 62, 73, 78, 79, 74, 85, 74, 89, 180, 250, 6, 9, 82, + 79, 85, 78, 68, 69, 68, 32, 69, 206, 134, 7, 87, 238, 70, 78, 135, 122, + 86, 19, 206, 202, 14, 78, 130, 184, 1, 69, 134, 63, 72, 146, 1, 65, 2, + 73, 3, 85, 15, 242, 222, 12, 82, 146, 235, 1, 78, 150, 248, 1, 65, 3, 73, + 23, 186, 201, 14, 65, 54, 79, 154, 247, 1, 78, 86, 69, 2, 71, 2, 73, 3, + 85, 13, 42, 69, 230, 192, 16, 71, 2, 79, 3, 85, 4, 226, 192, 16, 82, 3, + 89, 19, 166, 200, 14, 65, 202, 228, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, + 85, 7, 166, 172, 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 172, 172, + 15, 4, 79, 78, 32, 85, 13, 4, 68, 65, 83, 72, 8, 190, 189, 11, 83, 158, + 216, 3, 80, 218, 34, 32, 139, 112, 66, 6, 64, 4, 68, 76, 69, 32, 173, + 215, 4, 6, 76, 73, 78, 69, 32, 72, 4, 152, 180, 5, 3, 84, 72, 73, 203, + 185, 10, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 172, 153, 10, 3, 75, 89, + 32, 255, 156, 5, 76, 4, 26, 72, 187, 134, 12, 77, 2, 163, 170, 4, 69, 24, + 42, 73, 76, 2, 85, 83, 235, 187, 16, 89, 6, 34, 68, 22, 77, 147, 185, 14, + 66, 2, 195, 137, 6, 73, 2, 187, 212, 6, 73, 16, 46, 32, 201, 142, 10, 5, + 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 231, 249, 12, 84, 13, 11, + 32, 10, 44, 5, 87, 73, 84, 72, 32, 239, 153, 6, 73, 8, 66, 67, 182, 195, + 4, 68, 206, 135, 8, 82, 21, 4, 70, 65, 76, 76, 2, 129, 195, 11, 3, 79, + 77, 77, 5, 163, 181, 14, 32, 182, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, + 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 174, 27, 79, 156, 1, 3, 83, 81, + 85, 38, 84, 250, 1, 85, 242, 140, 11, 89, 217, 178, 2, 5, 86, 73, 69, 32, + 67, 7, 11, 32, 4, 108, 21, 87, 73, 84, 72, 32, 82, 73, 71, 72, 84, 87, + 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 255, 177, 1, 79, 2, 11, 32, 2, + 213, 132, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, + 32, 84, 87, 79, 4, 196, 230, 15, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, + 72, 159, 18, 76, 148, 6, 42, 32, 221, 1, 5, 70, 73, 69, 82, 32, 158, 1, + 88, 5, 83, 73, 71, 78, 32, 210, 172, 3, 86, 178, 146, 3, 68, 182, 1, 76, + 167, 180, 4, 65, 10, 42, 65, 234, 189, 8, 72, 203, 179, 7, 86, 4, 44, 5, + 82, 68, 72, 65, 67, 163, 239, 15, 78, 2, 201, 239, 15, 3, 65, 78, 68, + 246, 4, 92, 12, 66, 82, 69, 86, 69, 32, 87, 73, 84, 72, 32, 73, 89, 7, + 76, 69, 84, 84, 69, 82, 32, 2, 33, 6, 78, 86, 69, 82, 84, 69, 2, 21, 3, + 68, 32, 66, 2, 249, 181, 15, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, + 67, 190, 13, 68, 194, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, + 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 214, 33, + 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 150, 27, 67, 183, 167, 10, 80, + 2, 237, 19, 6, 86, 69, 79, 76, 65, 82, 6, 176, 19, 5, 73, 76, 65, 66, 73, + 201, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, + 32, 192, 2, 7, 69, 78, 84, 82, 69, 68, 32, 100, 13, 72, 73, 78, 69, 83, + 69, 32, 84, 79, 78, 69, 32, 89, 108, 8, 89, 82, 73, 76, 76, 73, 67, 32, + 166, 188, 10, 73, 244, 2, 4, 82, 79, 83, 83, 191, 154, 5, 79, 56, 206, 1, + 66, 42, 82, 250, 26, 72, 198, 193, 12, 79, 138, 143, 3, 65, 150, 64, 67, + 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, + 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 237, 130, 6, 5, 65, 82, 82, 69, + 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 170, 171, 16, 69, 3, 78, + 4, 30, 76, 21, 3, 82, 73, 71, 2, 29, 2, 69, 70, 2, 11, 72, 2, 11, 84, 2, + 173, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, + 170, 226, 11, 83, 214, 246, 3, 80, 146, 44, 81, 3, 82, 78, 34, 72, 30, + 83, 175, 217, 15, 69, 2, 197, 195, 14, 2, 65, 82, 74, 40, 5, 77, 65, 76, + 76, 32, 183, 6, 79, 72, 186, 1, 66, 138, 1, 68, 38, 69, 150, 1, 80, 58, + 83, 94, 84, 34, 89, 158, 163, 15, 90, 246, 63, 73, 138, 19, 67, 2, 71, + 186, 22, 74, 2, 86, 158, 20, 72, 2, 75, 186, 2, 65, 2, 79, 3, 85, 6, 38, + 89, 186, 27, 65, 187, 139, 16, 69, 2, 221, 201, 1, 19, 69, 76, 79, 82, + 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 206, 190, + 6, 90, 195, 231, 9, 69, 15, 50, 83, 186, 165, 16, 70, 2, 76, 2, 77, 3, + 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 205, + 223, 12, 3, 83, 67, 69, 4, 26, 65, 191, 164, 16, 69, 2, 241, 247, 15, 2, + 76, 79, 8, 42, 84, 198, 172, 1, 67, 251, 244, 14, 72, 4, 145, 19, 8, 82, + 65, 73, 71, 72, 84, 32, 85, 4, 238, 140, 16, 83, 215, 22, 69, 6, 36, 3, + 69, 82, 85, 255, 162, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, + 21, 3, 65, 67, 75, 2, 217, 228, 15, 2, 32, 89, 2, 191, 139, 11, 70, 16, + 18, 69, 27, 79, 2, 161, 5, 2, 78, 84, 14, 64, 2, 84, 32, 44, 5, 85, 66, + 76, 69, 32, 137, 52, 2, 87, 78, 6, 178, 192, 9, 83, 194, 144, 4, 72, 43, + 86, 4, 254, 135, 2, 65, 215, 240, 2, 80, 24, 40, 5, 88, 84, 82, 65, 45, + 139, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, 32, 10, + 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, 76, 79, + 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, 68, 70, + 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, 9, 82, + 226, 3, 76, 237, 238, 14, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, 10, 248, + 5, 4, 73, 71, 72, 32, 135, 41, 65, 44, 46, 65, 84, 2, 79, 87, 201, 11, 2, + 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, 67, 2, 211, + 237, 14, 76, 36, 86, 32, 245, 231, 11, 15, 69, 82, 32, 82, 73, 71, 72, + 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, 79, 40, 4, 76, + 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, 71, 170, 2, + 73, 136, 196, 13, 2, 85, 80, 235, 180, 1, 86, 2, 131, 173, 10, 73, 6, + 238, 2, 84, 205, 200, 13, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, 45, 4, 25, + 4, 82, 82, 79, 87, 5, 151, 203, 13, 72, 2, 111, 65, 4, 180, 202, 13, 3, + 71, 72, 84, 255, 205, 2, 78, 4, 194, 2, 79, 155, 128, 14, 73, 18, 18, 65, + 23, 73, 2, 151, 244, 14, 67, 16, 26, 68, 163, 254, 12, 78, 14, 38, 32, + 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, 2, 79, 84, + 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, 84, 45, 2, + 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, 2, 187, + 198, 13, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, 4, 18, + 65, 43, 71, 2, 11, 67, 2, 229, 170, 10, 2, 85, 84, 2, 11, 82, 2, 179, + 170, 10, 65, 2, 207, 148, 14, 69, 4, 246, 250, 12, 76, 247, 129, 2, 82, + 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, 72, + 173, 222, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 234, 186, 9, 69, + 180, 189, 5, 2, 85, 80, 199, 74, 67, 2, 129, 197, 14, 2, 79, 87, 2, 209, + 186, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, 69, 68, + 32, 133, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 32, + 6, 26, 71, 255, 197, 13, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, 65, 76, + 32, 83, 84, 79, 80, 5, 167, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, 250, + 133, 13, 84, 231, 59, 65, 2, 185, 168, 7, 3, 65, 76, 70, 156, 2, 138, 1, + 72, 48, 5, 77, 65, 76, 76, 32, 156, 31, 8, 84, 82, 69, 83, 83, 32, 65, + 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 216, 130, 8, + 3, 79, 82, 84, 251, 139, 6, 69, 144, 2, 154, 2, 65, 46, 66, 130, 1, 67, + 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, 34, 73, + 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, 94, 80, + 22, 82, 146, 2, 83, 190, 1, 84, 130, 8, 85, 142, 1, 86, 134, 240, 15, 75, + 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 23, 76, 210, 164, 15, 73, 215, 79, + 69, 11, 46, 65, 50, 79, 222, 8, 32, 227, 221, 15, 69, 2, 17, 2, 82, 82, + 2, 181, 227, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, 100, 7, 65, + 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, 190, 17, 32, + 187, 227, 15, 72, 30, 114, 73, 214, 6, 71, 238, 16, 76, 250, 218, 15, 79, + 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, 7, 22, 78, + 191, 11, 32, 2, 237, 220, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, 175, 136, + 9, 79, 2, 229, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, 104, 6, 32, + 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, 32, 74, 32, + 105, 3, 90, 32, 68, 6, 26, 72, 247, 217, 13, 84, 4, 21, 3, 79, 79, 75, 5, + 201, 160, 13, 3, 32, 65, 78, 4, 238, 15, 90, 131, 210, 15, 76, 4, 33, 6, + 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, 211, 8, 4, + 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 138, 14, 67, 219, 4, 82, 11, 190, 131, 16, 83, 2, 84, 2, + 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, 69, 75, 32, + 162, 1, 32, 151, 183, 13, 65, 4, 26, 71, 195, 223, 10, 80, 2, 151, 184, + 13, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 142, 178, 14, 72, + 231, 160, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 239, 207, 8, 87, 4, 222, + 4, 32, 239, 216, 15, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, 11, 82, + 2, 181, 212, 13, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, 73, 84, + 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 241, 222, 15, 8, 73, 71, 65, + 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 187, 237, 3, + 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 189, 172, 12, + 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, 185, 189, + 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, 72, 79, 79, + 75, 5, 217, 12, 4, 32, 65, 78, 68, 4, 222, 11, 82, 207, 1, 76, 9, 18, 32, + 47, 80, 2, 21, 3, 87, 73, 84, 2, 139, 205, 15, 72, 4, 241, 152, 12, 2, + 69, 78, 5, 151, 233, 15, 72, 15, 80, 6, 32, 87, 73, 84, 72, 32, 62, 65, + 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 239, 206, 13, 84, 2, + 209, 201, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 247, 212, 5, 32, 6, 38, + 71, 182, 7, 79, 203, 243, 15, 69, 2, 11, 76, 2, 129, 176, 13, 4, 79, 84, + 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, 68, 69, 87, + 65, 89, 4, 158, 3, 67, 131, 166, 14, 72, 4, 26, 82, 203, 180, 11, 72, 2, + 221, 199, 5, 3, 73, 80, 84, 2, 159, 167, 6, 83, 51, 106, 32, 102, 67, + 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, 68, 32, + 207, 208, 12, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 231, 5, 82, 2, + 197, 197, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, 71, 82, 65, + 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, 195, 137, 3, + 85, 2, 11, 72, 2, 157, 241, 9, 3, 32, 68, 73, 2, 177, 206, 5, 5, 32, 72, + 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, 72, 5, 11, 32, + 2, 153, 3, 4, 87, 73, 84, 72, 32, 86, 65, 50, 77, 74, 79, 42, 82, 214, 1, + 89, 206, 241, 15, 72, 2, 73, 2, 86, 3, 87, 7, 26, 76, 167, 244, 15, 69, + 2, 231, 252, 13, 80, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, 221, + 133, 4, 2, 78, 71, 2, 11, 80, 2, 189, 133, 6, 2, 69, 78, 9, 33, 6, 32, + 87, 73, 84, 72, 32, 6, 26, 76, 131, 162, 14, 72, 4, 37, 7, 79, 78, 71, + 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, 69, 2, + 213, 191, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, 84, 72, + 2, 245, 230, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, 84, 72, + 171, 206, 14, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, 177, + 160, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, 231, + 158, 14, 72, 2, 21, 3, 73, 71, 72, 2, 135, 189, 8, 84, 4, 11, 68, 4, 11, + 32, 4, 182, 208, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, 76, + 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 237, 157, 15, 8, + 73, 65, 78, 71, 85, 76, 65, 82, 2, 161, 144, 8, 2, 69, 68, 8, 70, 80, + 140, 245, 9, 7, 78, 65, 83, 80, 73, 82, 65, 223, 247, 5, 83, 4, 11, 32, + 4, 190, 226, 12, 84, 231, 59, 65, 4, 26, 79, 199, 210, 14, 69, 2, 11, 73, + 2, 155, 155, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, + 68, 69, 80, 2, 21, 3, 65, 82, 84, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 84, + 2, 11, 79, 2, 227, 238, 5, 78, 2, 11, 32, 2, 215, 194, 14, 83, 234, 2, + 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 210, 24, 79, 209, 222, + 11, 3, 75, 69, 89, 6, 26, 32, 251, 171, 15, 45, 4, 184, 223, 11, 6, 87, + 73, 84, 72, 32, 87, 255, 163, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, + 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 73, 82, 71, 65, 16, 7, 76, + 69, 84, 84, 69, 82, 32, 142, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, + 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 158, 233, 3, 67, 164, 1, + 6, 77, 65, 78, 67, 72, 85, 196, 162, 7, 4, 78, 73, 82, 85, 171, 166, 2, + 69, 22, 236, 20, 3, 79, 85, 66, 183, 227, 13, 73, 12, 112, 19, 82, 69, + 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 158, + 154, 13, 85, 175, 46, 79, 8, 177, 207, 9, 3, 84, 79, 82, 5, 231, 19, 32, + 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 250, 1, + 84, 234, 3, 90, 190, 229, 13, 72, 150, 168, 1, 75, 2, 76, 162, 7, 69, 2, + 79, 2, 85, 222, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, + 2, 82, 2, 87, 2, 89, 187, 2, 73, 59, 56, 8, 76, 73, 32, 71, 65, 76, 73, + 32, 207, 225, 15, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, + 68, 76, 5, 72, 65, 76, 70, 32, 34, 73, 50, 85, 34, 78, 30, 84, 66, 66, + 222, 150, 15, 80, 2, 90, 242, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, + 83, 86, 65, 82, 191, 224, 15, 72, 2, 131, 134, 9, 65, 8, 26, 65, 215, + 221, 15, 68, 7, 214, 222, 8, 77, 253, 190, 6, 3, 71, 65, 76, 4, 162, 221, + 15, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 169, + 156, 15, 3, 66, 65, 68, 4, 178, 220, 15, 71, 3, 78, 8, 60, 6, 72, 82, 69, + 69, 32, 66, 222, 150, 15, 84, 183, 71, 65, 2, 17, 2, 65, 76, 2, 247, 184, + 15, 85, 6, 26, 65, 207, 221, 15, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 201, 143, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, + 76, 73, 32, 71, 65, 76, 73, 32, 202, 228, 13, 90, 134, 245, 1, 70, 2, 75, + 2, 82, 187, 2, 73, 28, 122, 68, 166, 165, 9, 67, 158, 164, 1, 84, 142, + 154, 3, 66, 2, 71, 2, 74, 2, 76, 138, 176, 1, 90, 254, 4, 78, 247, 63, + 83, 4, 202, 227, 13, 68, 135, 245, 1, 72, 48, 52, 4, 73, 66, 69, 32, 246, + 215, 15, 72, 187, 2, 65, 44, 186, 215, 2, 84, 162, 243, 2, 71, 2, 72, + 218, 156, 6, 73, 190, 251, 1, 67, 2, 83, 246, 7, 82, 222, 158, 1, 65, + 186, 9, 90, 162, 7, 85, 222, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, + 69, 60, 52, 4, 79, 68, 79, 32, 254, 213, 15, 83, 187, 2, 65, 56, 250, 1, + 65, 98, 68, 34, 74, 34, 78, 176, 163, 1, 8, 76, 79, 78, 71, 32, 86, 79, + 87, 250, 174, 1, 84, 162, 243, 2, 71, 150, 152, 8, 67, 246, 7, 72, 182, + 175, 1, 79, 2, 85, 222, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, + 186, 2, 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 171, 213, + 15, 78, 4, 190, 222, 13, 90, 135, 245, 1, 84, 4, 158, 211, 15, 90, 187, + 2, 65, 4, 254, 210, 15, 73, 187, 2, 65, 2, 223, 210, 15, 73, 6, 170, 193, + 15, 72, 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 149, 181, + 4, 21, 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, + 78, 68, 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 150, 2, 68, 171, 141, 15, 79, 6, 152, 1, 3, + 82, 73, 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, + 73, 82, 71, 65, 32, 245, 190, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, + 209, 141, 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, + 84, 72, 32, 68, 2, 145, 141, 15, 5, 79, 85, 66, 76, 69, 2, 131, 192, 9, + 76, 10, 96, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 224, 251, 3, 5, 83, + 84, 65, 66, 76, 235, 166, 9, 82, 6, 26, 89, 243, 136, 12, 69, 4, 190, + 254, 14, 65, 155, 1, 73, 8, 48, 2, 78, 32, 162, 197, 4, 68, 231, 242, 10, + 83, 4, 72, 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 147, 136, + 12, 67, 2, 177, 174, 15, 2, 77, 79, 4, 230, 129, 12, 73, 139, 204, 3, 69, + 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, 84, + 2, 147, 247, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 155, 192, 14, 87, + 4, 230, 173, 8, 66, 133, 232, 5, 4, 83, 67, 79, 79, 2, 205, 168, 3, 7, + 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 247, + 200, 15, 84, 10, 48, 3, 65, 73, 78, 185, 224, 11, 3, 32, 70, 85, 9, 11, + 32, 6, 222, 166, 9, 82, 24, 5, 67, 65, 66, 76, 69, 137, 235, 1, 6, 66, + 73, 67, 89, 67, 76, 7, 11, 32, 4, 160, 185, 14, 2, 84, 82, 219, 83, 70, + 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 191, 211, 5, 68, 62, 198, 1, 75, + 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 246, 104, 68, 220, 231, 7, 2, 72, + 65, 2, 82, 158, 180, 1, 69, 190, 29, 83, 150, 90, 76, 246, 7, 67, 246, + 162, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 180, 154, 5, 2, 69, 65, + 182, 238, 5, 72, 231, 190, 4, 79, 4, 230, 180, 15, 65, 215, 1, 73, 4, + 158, 247, 14, 73, 255, 59, 71, 7, 210, 198, 15, 76, 3, 79, 4, 222, 178, + 15, 72, 219, 19, 65, 6, 214, 246, 8, 72, 170, 204, 6, 69, 155, 3, 65, + 204, 4, 44, 2, 76, 84, 238, 6, 83, 159, 175, 13, 67, 102, 36, 4, 65, 78, + 73, 32, 223, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 139, 4, + 83, 74, 210, 1, 68, 222, 82, 78, 158, 230, 1, 82, 230, 208, 1, 74, 166, + 209, 7, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, + 76, 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, + 68, 162, 192, 15, 72, 187, 2, 65, 6, 158, 192, 15, 68, 2, 72, 187, 2, 65, + 26, 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 183, 174, 14, 77, 18, 50, 69, + 89, 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, + 65, 76, 2, 21, 3, 32, 78, 79, 2, 155, 173, 14, 84, 16, 40, 4, 83, 73, 71, + 78, 191, 192, 15, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, + 84, 72, 32, 8, 166, 199, 3, 76, 22, 82, 140, 246, 8, 5, 68, 79, 85, 66, + 76, 235, 222, 1, 84, 4, 210, 246, 11, 85, 167, 200, 2, 68, 7, 11, 32, 4, + 232, 82, 5, 77, 85, 76, 84, 73, 207, 209, 11, 85, 226, 3, 32, 2, 73, 67, + 247, 152, 3, 72, 224, 3, 30, 32, 101, 3, 65, 76, 32, 6, 76, 4, 78, 65, + 84, 85, 200, 43, 2, 70, 76, 181, 229, 10, 4, 83, 72, 65, 82, 2, 239, 165, + 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, 65, 82, 68, 74, 83, 155, 239, + 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, 74, 65, 2, 209, 254, 14, 2, 67, + 75, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 135, 131, 11, 67, 210, 3, 230, + 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, 1, 70, 166, 2, 71, 112, + 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, 72, 65, 126, 75, 198, + 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, 80, 200, 2, 2, 81, 85, + 146, 2, 82, 190, 2, 83, 230, 5, 84, 182, 10, 86, 42, 88, 42, 87, 180, + 233, 8, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 163, 151, 6, 90, 20, 36, + 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, 83, 182, 224, 7, 66, + 187, 173, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 218, 130, 15, 75, 143, 53, + 69, 8, 26, 86, 223, 245, 14, 65, 6, 32, 2, 73, 83, 139, 183, 15, 69, 5, + 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, 194, 7, 82, 234, 11, 32, + 224, 32, 7, 73, 82, 67, 76, 69, 32, 88, 145, 200, 5, 2, 85, 84, 2, 197, + 241, 14, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, 83, 84, 69, 82, 4, 26, 77, + 175, 152, 13, 86, 2, 207, 177, 9, 65, 64, 26, 77, 179, 178, 15, 68, 62, + 64, 7, 66, 73, 78, 73, 78, 71, 32, 185, 251, 5, 3, 77, 79, 78, 60, 202, + 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, 68, 79, 56, 2, 85, 80, + 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, 142, 2, 84, 202, 154, + 7, 66, 140, 180, 3, 2, 76, 79, 183, 222, 4, 82, 6, 76, 5, 67, 67, 69, 78, + 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, 79, 5, 205, 2, 5, 45, 83, + 84, 65, 67, 2, 231, 160, 14, 78, 6, 52, 2, 87, 78, 168, 3, 2, 85, 66, + 199, 145, 15, 73, 2, 137, 151, 14, 2, 32, 66, 12, 136, 76, 2, 65, 71, + 231, 228, 14, 73, 2, 231, 137, 14, 78, 12, 132, 1, 9, 78, 65, 80, 32, 80, + 73, 90, 90, 73, 22, 84, 236, 187, 4, 11, 80, 82, 69, 67, 72, 71, 69, 83, + 65, 78, 71, 255, 238, 5, 77, 2, 191, 206, 11, 67, 6, 44, 5, 65, 67, 67, + 65, 84, 159, 158, 15, 69, 4, 40, 4, 73, 83, 83, 73, 203, 174, 15, 79, 2, + 211, 143, 15, 77, 10, 34, 82, 253, 225, 11, 2, 69, 78, 8, 28, 2, 73, 80, + 191, 6, 69, 2, 133, 205, 11, 6, 76, 69, 32, 84, 79, 78, 4, 22, 79, 191, + 2, 69, 2, 159, 161, 14, 73, 24, 98, 65, 142, 1, 69, 84, 6, 79, 85, 66, + 76, 69, 32, 213, 249, 9, 7, 82, 85, 77, 32, 67, 76, 69, 10, 96, 2, 32, + 67, 20, 2, 77, 80, 196, 29, 4, 83, 72, 69, 68, 233, 228, 14, 5, 76, 32, + 83, 69, 71, 2, 223, 164, 8, 65, 5, 255, 166, 13, 32, 4, 52, 3, 67, 82, + 69, 241, 201, 8, 4, 71, 82, 69, 69, 2, 157, 25, 3, 83, 67, 69, 6, 246, + 21, 83, 250, 6, 66, 131, 244, 4, 70, 14, 32, 3, 78, 68, 32, 155, 16, 73, + 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 230, 210, 7, 66, 187, 173, 7, + 84, 2, 145, 153, 12, 2, 72, 82, 2, 215, 176, 14, 76, 34, 104, 6, 69, 82, + 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 144, 36, 3, 85, 83, 65, 157, + 221, 3, 2, 76, 65, 5, 195, 129, 13, 32, 10, 22, 78, 135, 34, 86, 8, 56, + 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 235, 20, 65, 6, 185, 215, 5, 4, + 77, 79, 76, 79, 6, 208, 25, 3, 85, 82, 45, 147, 201, 13, 82, 18, 54, 32, + 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, 69, 70, 7, + 161, 13, 3, 32, 79, 84, 4, 163, 235, 5, 79, 8, 84, 9, 65, 67, 69, 32, 78, + 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, 132, 196, 8, 2, + 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, 32, 209, 8, 3, + 85, 80, 84, 6, 52, 3, 80, 69, 68, 246, 214, 3, 78, 163, 169, 3, 82, 2, + 135, 175, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 147, 255, 13, 79, 22, + 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, 62, 70, 184, 211, + 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 253, 208, 1, 9, 82, 69, + 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, 171, 144, 5, 76, 6, + 64, 5, 73, 71, 72, 84, 72, 253, 235, 13, 5, 78, 68, 32, 79, 70, 4, 165, + 230, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, 4, 214, 14, 76, 225, + 196, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, 213, 13, 2, 69, 70, + 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, 69, 82, 70, 69, 67, + 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, 78, 73, 77, 65, 48, + 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 135, 253, 14, 73, 2, 143, 255, + 14, 90, 7, 11, 32, 4, 146, 250, 6, 82, 171, 153, 5, 66, 4, 185, 17, 2, + 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, 82, 69, 15, 32, 2, + 11, 32, 2, 139, 249, 6, 82, 10, 120, 4, 69, 66, 69, 78, 208, 26, 3, 85, + 76, 76, 148, 198, 5, 3, 65, 84, 85, 173, 176, 6, 7, 79, 84, 69, 72, 69, + 65, 68, 2, 245, 131, 14, 4, 83, 84, 73, 77, 32, 88, 2, 78, 69, 120, 14, + 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, 69, 107, 84, 6, 92, + 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, 78, 84, 89, 45, 69, + 73, 155, 20, 45, 4, 177, 13, 2, 71, 72, 22, 11, 45, 22, 250, 158, 3, 49, + 206, 250, 11, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, + 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, 72, 69, 83, 73, 0, + 2, 76, 85, 18, 69, 142, 1, 79, 247, 145, 13, 73, 2, 239, 22, 83, 6, 68, + 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, 78, 67, 4, 26, 85, + 219, 213, 14, 77, 2, 191, 213, 14, 80, 2, 247, 249, 12, 84, 6, 48, 2, 68, + 65, 237, 5, 5, 82, 82, 69, 67, 84, 2, 175, 147, 13, 84, 12, 76, 6, 65, + 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, 83, 73, 77, 8, 60, 5, + 84, 79, 78, 69, 32, 138, 200, 3, 78, 163, 169, 3, 82, 4, 26, 83, 251, + 250, 4, 70, 2, 205, 148, 15, 3, 72, 65, 82, 4, 17, 2, 65, 32, 4, 138, + 240, 11, 65, 217, 161, 3, 3, 66, 65, 83, 14, 22, 69, 167, 1, 73, 10, 72, + 4, 80, 69, 65, 84, 73, 10, 86, 69, 82, 83, 69, 32, 70, 73, 78, 65, 8, + 152, 195, 5, 9, 69, 68, 32, 70, 73, 71, 85, 82, 69, 191, 130, 8, 32, 2, + 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, 90, 65, 2, 33, 6, + 84, 32, 82, 69, 80, 69, 2, 131, 251, 9, 65, 2, 155, 151, 8, 78, 48, 136, + 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, 54, 73, 252, 1, 6, 81, + 85, 65, 82, 69, 32, 204, 128, 8, 2, 85, 66, 239, 2, 79, 4, 17, 2, 85, 83, + 5, 173, 141, 13, 5, 32, 70, 76, 69, 88, 14, 32, 2, 77, 73, 195, 230, 14, + 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, 77, 73, 78, 73, 77, 65, 6, + 11, 32, 6, 138, 13, 87, 166, 222, 6, 82, 171, 153, 5, 66, 6, 84, 3, 79, + 82, 84, 169, 211, 5, 3, 65, 82, 80, 14, 32, 4, 78, 71, 76, 69, 43, 88, 2, + 17, 2, 32, 66, 2, 155, 169, 13, 65, 12, 18, 45, 79, 84, 4, 242, 7, 76, + 213, 238, 13, 11, 83, 84, 82, 73, 78, 71, 32, 70, 82, 69, 84, 8, 52, 3, + 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, 193, 12, 2, 84, 72, 6, 26, + 78, 135, 140, 15, 66, 4, 229, 9, 7, 79, 84, 69, 72, 69, 65, 68, 60, 132, + 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, 88, 2, 87, 79, 84, 7, 79, 82, + 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, 82, 78, 16, 232, 1, 27, 73, + 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, + 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, 80, 69, 82, 70, 69, 67, 84, + 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, + 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, 84, 65, 131, 1, 80, 9, 169, + 184, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, 6, 60, 3, 73, 77, + 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, 2, 213, 226, 14, 5, 69, 82, + 70, 69, 67, 5, 145, 163, 8, 12, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, + 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, 84, 89, 45, 83, 69, 67, 79, + 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, 37, 7, 73, 78, 69, 32, 83, + 84, 65, 2, 147, 132, 14, 70, 5, 173, 168, 11, 6, 32, 82, 69, 83, 85, 80, + 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, 10, 32, 78, 79, 84, 69, + 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, 68, 32, 78, 79, 84, 69, + 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, 80, 32, 38, 82, 25, 3, 76, + 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, 87, 207, 247, 11, 66, 4, 21, + 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, 87, 207, 247, 11, 66, 2, 11, + 72, 2, 187, 189, 13, 73, 7, 11, 32, 4, 222, 160, 8, 83, 223, 224, 6, 85, + 4, 36, 3, 79, 73, 68, 187, 190, 14, 73, 2, 221, 178, 12, 5, 32, 78, 79, + 84, 69, 6, 92, 4, 72, 79, 76, 69, 157, 174, 8, 13, 73, 84, 72, 32, 70, + 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, 246, 178, 3, 78, 163, 169, + 3, 82, 192, 3, 252, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, + 73, 71, 78, 32, 254, 1, 76, 216, 14, 16, 77, 79, 68, 73, 70, 73, 69, 82, + 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 166, 10, 84, 216, 1, 11, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 223, 241, 12, 68, 16, 66, 77, 165, 1, + 11, 83, 72, 65, 78, 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, + 65, 76, 32, 45, 10, 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 238, 249, + 14, 72, 2, 82, 2, 87, 3, 89, 6, 194, 249, 14, 76, 2, 77, 3, 78, 2, 219, + 182, 10, 32, 238, 1, 104, 6, 69, 84, 84, 69, 82, 32, 189, 13, 15, 79, 71, + 79, 71, 82, 65, 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 242, 1, 65, + 50, 69, 146, 1, 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, + 3, 84, 198, 1, 87, 242, 133, 11, 68, 190, 4, 86, 214, 20, 85, 210, 200, + 1, 73, 42, 76, 242, 190, 1, 66, 2, 67, 2, 74, 2, 80, 254, 68, 72, 2, 89, + 187, 2, 79, 7, 140, 174, 14, 4, 73, 84, 79, 78, 207, 74, 85, 9, 77, 17, + 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, + 42, 71, 182, 178, 10, 89, 187, 138, 3, 78, 2, 179, 178, 10, 72, 6, 214, + 163, 9, 82, 142, 209, 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 203, 246, 14, + 65, 43, 25, 4, 77, 84, 73, 32, 40, 134, 1, 68, 34, 84, 162, 191, 8, 78, + 178, 238, 5, 67, 2, 72, 2, 74, 158, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, + 83, 2, 88, 3, 90, 6, 238, 173, 14, 68, 255, 68, 72, 4, 207, 173, 14, 84, + 12, 36, 3, 79, 78, 32, 203, 244, 14, 65, 10, 60, 2, 66, 66, 206, 252, 12, + 74, 134, 181, 1, 78, 187, 66, 69, 4, 134, 244, 14, 65, 3, 69, 10, 134, + 190, 8, 78, 174, 179, 6, 71, 2, 89, 187, 2, 65, 4, 196, 214, 2, 12, 85, + 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 239, 156, 12, 65, 50, 90, 72, + 136, 215, 2, 9, 71, 65, 87, 32, 75, 65, 82, 69, 78, 198, 152, 12, 83, + 187, 2, 65, 44, 66, 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, + 71, 32, 41, 17, 2, 78, 32, 38, 134, 1, 78, 234, 248, 12, 74, 2, 80, 2, + 84, 138, 176, 1, 66, 2, 67, 2, 71, 2, 75, 254, 68, 68, 2, 70, 2, 72, 2, + 90, 187, 2, 65, 6, 234, 237, 14, 78, 2, 89, 187, 2, 65, 4, 190, 248, 12, + 67, 3, 83, 36, 38, 65, 134, 168, 14, 84, 255, 68, 72, 31, 41, 8, 73, 32, + 76, 65, 73, 78, 71, 32, 28, 82, 78, 142, 134, 11, 68, 250, 160, 3, 66, 2, + 71, 2, 74, 158, 37, 76, 227, 31, 70, 4, 254, 235, 14, 78, 3, 89, 6, 92, + 17, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, + 191, 237, 14, 65, 4, 190, 168, 10, 80, 195, 205, 2, 84, 6, 170, 225, 13, + 79, 154, 60, 81, 255, 62, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, + 72, 65, 78, 2, 29, 5, 32, 82, 69, 68, 85, 2, 133, 132, 1, 2, 80, 76, 90, + 88, 4, 73, 71, 78, 32, 232, 6, 6, 89, 77, 66, 79, 76, 32, 145, 143, 9, 3, + 72, 65, 78, 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, + 78, 69, 45, 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, + 84, 79, 78, 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, + 32, 75, 65, 82, 69, 78, 32, 84, 79, 78, 69, 154, 238, 2, 68, 128, 145, 5, + 19, 82, 85, 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, + 69, 45, 156, 151, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 160, 91, 6, + 76, 73, 84, 84, 76, 69, 159, 176, 2, 86, 4, 250, 211, 13, 83, 243, 78, + 78, 4, 170, 231, 14, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 151, 150, 14, + 69, 16, 84, 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, + 159, 231, 13, 83, 6, 204, 151, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, + 189, 205, 3, 4, 84, 79, 78, 69, 8, 182, 229, 14, 50, 2, 51, 2, 53, 3, 54, + 4, 138, 229, 14, 50, 3, 53, 10, 11, 45, 10, 226, 228, 14, 49, 2, 50, 2, + 51, 2, 52, 3, 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, + 78, 32, 232, 148, 8, 4, 71, 69, 78, 73, 201, 85, 6, 67, 79, 77, 80, 76, + 69, 8, 84, 5, 73, 84, 79, 78, 32, 217, 152, 6, 10, 70, 79, 82, 69, 77, + 69, 78, 84, 73, 79, 6, 98, 69, 130, 213, 8, 84, 223, 243, 4, 79, 2, 129, + 149, 8, 2, 67, 65, 4, 26, 69, 223, 200, 13, 79, 2, 205, 12, 4, 88, 67, + 76, 65, 24, 140, 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, + 87, 32, 75, 65, 82, 69, 78, 32, 161, 139, 9, 8, 65, 73, 32, 76, 65, 73, + 78, 71, 4, 42, 72, 253, 163, 10, 4, 75, 69, 32, 80, 2, 161, 188, 9, 2, + 65, 84, 56, 154, 2, 65, 76, 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, + 75, 65, 89, 65, 72, 32, 40, 4, 77, 79, 78, 32, 34, 83, 154, 1, 69, 40, + 18, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, + 32, 144, 146, 9, 2, 84, 65, 246, 228, 1, 86, 214, 20, 85, 211, 200, 1, + 73, 8, 26, 73, 195, 221, 14, 65, 7, 25, 4, 84, 79, 78, 32, 4, 215, 214, + 12, 65, 2, 139, 157, 14, 78, 6, 166, 198, 14, 69, 2, 79, 215, 22, 85, 4, + 250, 200, 14, 73, 219, 19, 79, 10, 72, 10, 71, 65, 87, 32, 75, 65, 82, + 69, 78, 32, 21, 4, 72, 65, 78, 32, 2, 255, 182, 14, 69, 8, 54, 69, 20, 5, + 70, 73, 78, 65, 76, 211, 216, 14, 65, 5, 179, 222, 13, 32, 2, 191, 201, + 14, 32, 4, 138, 182, 14, 69, 151, 14, 85, 216, 17, 128, 2, 5, 45, 65, 82, + 89, 32, 214, 4, 65, 242, 16, 66, 30, 69, 206, 31, 73, 136, 1, 3, 75, 79, + 32, 154, 10, 79, 162, 24, 85, 224, 8, 22, 89, 73, 65, 75, 69, 78, 71, 32, + 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, 78, 71, 32, 232, 246, 13, 2, + 78, 66, 27, 76, 32, 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, + 79, 78, 108, 4, 87, 72, 73, 84, 174, 168, 11, 76, 254, 16, 73, 135, 140, + 2, 80, 8, 52, 7, 73, 82, 67, 76, 69, 68, 32, 147, 200, 13, 79, 6, 40, 2, + 80, 76, 14, 84, 167, 244, 7, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 147, + 172, 11, 83, 6, 40, 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, + 84, 69, 82, 83, 69, 67, 84, 1, 2, 85, 78, 2, 255, 146, 11, 73, 2, 11, 77, + 2, 235, 168, 11, 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, + 87, 73, 84, 72, 32, 4, 198, 247, 10, 80, 211, 140, 3, 68, 2, 11, 69, 2, + 145, 132, 12, 2, 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, + 78, 68, 65, 82, 73, 32, 154, 4, 73, 52, 2, 78, 68, 216, 4, 7, 84, 73, 79, + 78, 65, 76, 32, 240, 206, 1, 2, 85, 83, 136, 236, 7, 5, 77, 69, 32, 66, + 65, 148, 172, 3, 7, 90, 65, 82, 32, 65, 77, 85, 196, 157, 1, 8, 82, 82, + 79, 87, 32, 78, 79, 45, 219, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, + 32, 219, 206, 14, 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, + 7, 78, 85, 77, 66, 69, 82, 32, 193, 207, 9, 16, 67, 82, 85, 67, 73, 70, + 79, 82, 77, 32, 78, 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, + 76, 32, 174, 253, 1, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, + 134, 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, + 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, + 18, 138, 244, 2, 65, 54, 76, 174, 149, 4, 83, 190, 3, 89, 130, 199, 1, + 75, 130, 76, 66, 190, 173, 4, 78, 198, 105, 72, 171, 4, 77, 16, 154, 245, + 2, 84, 186, 151, 4, 79, 143, 211, 5, 70, 84, 84, 7, 76, 69, 84, 84, 69, + 82, 32, 164, 2, 5, 83, 73, 71, 78, 32, 183, 219, 12, 68, 54, 42, 65, 50, + 69, 66, 73, 50, 79, 47, 85, 11, 254, 185, 14, 78, 202, 17, 66, 2, 72, 3, + 74, 15, 150, 211, 12, 78, 230, 110, 76, 242, 108, 84, 174, 28, 71, 3, 77, + 11, 194, 132, 14, 68, 150, 70, 72, 2, 83, 3, 84, 11, 210, 201, 14, 78, + 86, 76, 2, 80, 3, 89, 11, 250, 201, 14, 67, 2, 68, 2, 75, 3, 82, 10, 112, + 2, 77, 85, 20, 3, 83, 85, 84, 172, 165, 2, 2, 73, 75, 184, 129, 9, 2, 79, + 74, 137, 233, 2, 3, 84, 79, 89, 2, 255, 143, 14, 72, 2, 183, 199, 14, 85, + 4, 236, 137, 11, 5, 76, 32, 80, 79, 76, 231, 2, 82, 133, 1, 41, 8, 73, + 78, 65, 71, 65, 82, 73, 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, + 32, 252, 1, 5, 83, 73, 71, 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 199, 207, 4, 72, 94, 214, 1, 86, 202, 164, 10, 82, 206, + 55, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, + 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, + 77, 2, 89, 186, 2, 69, 3, 79, 6, 242, 254, 4, 79, 243, 197, 9, 65, 10, + 202, 205, 9, 83, 134, 144, 1, 65, 243, 163, 3, 86, 24, 210, 213, 4, 80, + 190, 40, 86, 166, 225, 5, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, + 69, 3, 79, 4, 202, 47, 68, 191, 210, 13, 80, 4, 250, 194, 14, 83, 15, 72, + 246, 4, 216, 1, 3, 71, 65, 84, 128, 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, + 83, 136, 1, 2, 85, 84, 242, 1, 87, 144, 16, 3, 88, 84, 32, 140, 150, 8, + 4, 80, 84, 85, 78, 240, 239, 2, 2, 67, 75, 150, 141, 2, 82, 183, 144, 1, + 76, 154, 1, 152, 1, 4, 73, 86, 69, 32, 245, 160, 12, 27, 69, 68, 32, 68, + 79, 85, 66, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, + 32, 68, 79, 85, 66, 152, 1, 152, 1, 8, 67, 73, 82, 67, 76, 69, 68, 32, + 216, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 196, 1, 8, 83, 81, 85, 65, + 82, 69, 68, 32, 147, 170, 8, 65, 78, 104, 5, 68, 73, 71, 73, 84, 28, 7, + 78, 85, 77, 66, 69, 82, 32, 142, 3, 76, 238, 150, 13, 84, 139, 3, 83, 2, + 189, 209, 12, 2, 32, 90, 20, 50, 84, 218, 254, 1, 69, 46, 70, 42, 78, 31, + 83, 6, 154, 128, 2, 72, 47, 87, 6, 38, 77, 234, 254, 12, 67, 191, 2, 68, + 2, 49, 10, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, + 25, 4, 84, 79, 32, 76, 2, 25, 4, 79, 87, 69, 82, 2, 169, 198, 11, 2, 32, + 67, 66, 118, 76, 248, 229, 2, 4, 67, 82, 79, 83, 142, 138, 9, 81, 146, + 144, 1, 65, 198, 19, 73, 2, 87, 170, 164, 1, 80, 3, 83, 52, 221, 209, 8, + 6, 65, 84, 73, 78, 32, 67, 18, 158, 1, 65, 216, 1, 17, 71, 82, 69, 65, + 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 37, 14, 76, 69, 83, + 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 6, 92, 3, 32, 83, 85, 85, 16, + 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, 89, 32, 78, 79, 82, 4, 30, + 66, 1, 3, 80, 69, 82, 2, 253, 223, 6, 8, 83, 69, 84, 32, 79, 70, 32, 78, + 2, 213, 48, 5, 32, 65, 67, 84, 85, 6, 250, 246, 7, 69, 191, 177, 4, 76, + 6, 214, 246, 7, 69, 155, 177, 4, 71, 6, 26, 84, 199, 179, 12, 83, 4, 76, + 5, 73, 78, 71, 32, 68, 145, 176, 10, 8, 32, 87, 73, 84, 72, 32, 69, 71, + 2, 133, 228, 7, 2, 79, 76, 64, 40, 4, 82, 65, 76, 32, 203, 252, 13, 69, + 62, 48, 6, 67, 72, 69, 83, 83, 32, 235, 247, 13, 70, 60, 74, 75, 134, + 218, 12, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, 131, 2, 84, 20, 44, 5, + 78, 73, 71, 72, 84, 151, 219, 12, 73, 15, 195, 219, 12, 32, 246, 2, 70, + 32, 184, 8, 2, 65, 32, 248, 6, 3, 76, 73, 78, 183, 206, 3, 83, 174, 1, + 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, 73, 32, 76, 85, 69, 32, + 183, 152, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, 234, 244, 7, 87, 171, 170, + 5, 83, 2, 183, 154, 9, 69, 166, 1, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, 22, 84, 76, 11, 86, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 191, 190, 12, 68, 102, 76, 6, 70, 73, 78, + 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 14, 198, 183, 12, + 78, 150, 248, 1, 66, 2, 68, 2, 75, 2, 77, 3, 86, 44, 11, 32, 44, 146, 1, + 75, 2, 88, 34, 83, 210, 222, 10, 78, 134, 135, 3, 84, 82, 80, 254, 68, + 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 86, 3, 89, 4, 190, 171, + 14, 86, 187, 2, 65, 4, 158, 171, 14, 85, 187, 2, 65, 5, 183, 173, 14, 86, + 6, 224, 133, 7, 3, 79, 78, 69, 253, 76, 8, 72, 65, 77, 32, 68, 73, 71, + 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, 216, 231, 11, 11, 86, 79, 87, + 69, 76, 32, 83, 72, 79, 82, 84, 167, 195, 2, 69, 8, 238, 184, 10, 65, + 250, 242, 3, 69, 3, 89, 4, 186, 171, 14, 73, 3, 89, 9, 166, 184, 10, 65, + 251, 242, 3, 89, 11, 130, 184, 10, 69, 250, 242, 3, 85, 3, 89, 194, 1, + 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, 69, 82, 32, 214, 2, 83, 190, + 23, 80, 202, 133, 1, 86, 138, 200, 7, 65, 144, 230, 1, 5, 73, 78, 83, 69, + 82, 210, 142, 1, 67, 207, 186, 2, 79, 26, 64, 6, 79, 85, 66, 76, 69, 32, + 214, 211, 4, 65, 239, 230, 7, 73, 4, 198, 211, 4, 68, 231, 137, 7, 67, 2, + 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, 213, 224, 10, 2, 73, 76, 108, + 218, 1, 78, 62, 86, 254, 188, 10, 65, 38, 68, 46, 84, 230, 24, 85, 210, + 200, 1, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 80, 2, 82, 254, 68, 72, 2, 87, 2, 89, 186, 2, 69, 3, 79, 14, + 210, 222, 13, 71, 2, 89, 254, 68, 72, 2, 78, 187, 2, 65, 10, 26, 69, 207, + 193, 10, 79, 2, 173, 246, 11, 3, 68, 73, 67, 22, 26, 73, 139, 179, 4, 65, + 20, 44, 3, 71, 78, 32, 213, 128, 9, 2, 68, 68, 18, 210, 189, 10, 65, 74, + 67, 98, 78, 230, 179, 1, 74, 228, 2, 5, 70, 73, 78, 65, 76, 50, 85, 211, + 235, 1, 86, 4, 203, 248, 13, 69, 4, 250, 200, 9, 80, 171, 193, 3, 76, 6, + 70, 78, 245, 141, 13, 11, 71, 72, 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, + 156, 217, 7, 7, 69, 32, 80, 79, 73, 78, 84, 131, 199, 6, 74, 124, 152, 1, + 3, 67, 79, 77, 242, 2, 68, 78, 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, + 89, 77, 66, 79, 76, 32, 238, 192, 7, 69, 133, 178, 1, 3, 84, 65, 77, 20, + 52, 7, 66, 73, 78, 73, 78, 71, 32, 143, 158, 14, 77, 18, 116, 5, 76, 79, + 78, 71, 32, 76, 2, 78, 65, 20, 6, 83, 72, 79, 82, 84, 32, 165, 158, 13, + 6, 68, 79, 85, 66, 76, 69, 8, 130, 1, 72, 34, 76, 194, 207, 10, 82, 21, + 7, 68, 69, 83, 67, 69, 78, 68, 2, 155, 172, 10, 83, 6, 34, 72, 34, 76, + 195, 207, 10, 82, 2, 137, 208, 10, 3, 73, 71, 72, 2, 233, 207, 10, 2, 79, + 87, 24, 152, 1, 4, 65, 78, 84, 65, 152, 133, 12, 4, 79, 82, 79, 77, 203, + 41, 73, 70, 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, + 2, 79, 87, 2, 185, 213, 12, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, + 90, 78, 206, 249, 9, 82, 158, 225, 1, 79, 198, 71, 67, 170, 183, 1, 69, + 238, 18, 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, + 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 237, 248, 10, 5, + 71, 66, 65, 83, 73, 8, 40, 4, 79, 78, 65, 32, 151, 154, 14, 65, 6, 214, + 162, 12, 67, 134, 245, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 173, + 193, 5, 5, 32, 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, + 32, 65, 2, 135, 171, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, + 79, 79, 32, 68, 69, 78, 2, 159, 199, 12, 78, 180, 1, 90, 32, 156, 3, 2, + 77, 73, 118, 78, 150, 1, 82, 238, 7, 84, 186, 203, 13, 45, 135, 40, 83, + 18, 202, 1, 66, 96, 5, 69, 78, 84, 82, 89, 22, 80, 160, 180, 2, 15, 79, + 78, 69, 32, 85, 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 220, 242, 3, 9, + 77, 79, 66, 73, 76, 69, 32, 80, 72, 181, 30, 3, 83, 77, 79, 4, 52, 4, 82, + 69, 65, 75, 173, 131, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 231, 244, + 12, 69, 5, 151, 142, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 161, 167, + 9, 3, 73, 82, 65, 2, 137, 230, 10, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, + 68, 59, 83, 2, 249, 255, 12, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, + 195, 215, 10, 77, 6, 38, 45, 185, 183, 9, 3, 70, 79, 82, 4, 76, 8, 66, + 82, 69, 65, 75, 73, 78, 71, 245, 162, 2, 5, 80, 79, 84, 65, 66, 2, 129, + 201, 6, 2, 32, 72, 93, 56, 2, 84, 72, 146, 11, 77, 173, 185, 10, 3, 68, + 73, 67, 84, 78, 32, 213, 131, 11, 13, 69, 65, 83, 84, 45, 80, 79, 73, 78, + 84, 73, 78, 71, 82, 96, 5, 69, 65, 83, 84, 32, 132, 2, 6, 73, 78, 68, 73, + 67, 32, 217, 1, 5, 87, 69, 83, 84, 32, 30, 82, 65, 182, 221, 6, 80, 130, + 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, 87, 12, 40, 4, 82, 82, + 79, 87, 175, 217, 6, 78, 11, 11, 32, 8, 80, 9, 67, 82, 79, 83, 83, 73, + 78, 71, 32, 244, 2, 2, 65, 78, 255, 216, 6, 87, 4, 218, 180, 3, 83, 207, + 166, 3, 78, 20, 50, 80, 60, 3, 81, 85, 65, 66, 82, 215, 126, 70, 2, 37, + 7, 76, 65, 67, 69, 72, 79, 76, 2, 135, 158, 4, 68, 4, 240, 157, 4, 2, 82, + 84, 237, 173, 9, 5, 78, 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 187, 144, 4, + 69, 32, 82, 65, 218, 217, 6, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, + 83, 203, 43, 87, 14, 34, 78, 33, 4, 82, 82, 79, 87, 2, 229, 177, 3, 3, + 68, 32, 83, 13, 11, 32, 10, 68, 3, 84, 79, 32, 138, 213, 6, 67, 40, 3, + 65, 78, 68, 219, 2, 87, 4, 146, 214, 6, 67, 229, 228, 4, 4, 76, 79, 78, + 71, 56, 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, + 65, 132, 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 250, 195, 7, + 69, 196, 25, 7, 73, 68, 69, 78, 84, 73, 67, 214, 151, 4, 71, 38, 76, 243, + 77, 67, 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, + 83, 89, 139, 223, 7, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, + 2, 11, 78, 2, 243, 119, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, + 2, 65, 76, 2, 201, 223, 7, 2, 76, 89, 4, 181, 174, 6, 14, 65, 76, 32, 83, + 85, 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 153, 223, 7, 6, 65, 82, 65, + 76, 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 195, 254, 12, 73, 4, 68, 5, + 73, 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, + 79, 70, 2, 183, 172, 6, 32, 4, 178, 163, 10, 82, 227, 202, 1, 73, 8, 58, + 76, 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, + 1, 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, + 32, 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 207, 180, 12, 69, 2, + 11, 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 153, 131, 3, 6, 83, + 72, 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, + 4, 146, 234, 13, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, + 69, 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 223, 251, 3, 79, 186, 6, + 102, 77, 176, 3, 4, 83, 72, 85, 32, 168, 157, 5, 8, 84, 32, 65, 78, 68, + 32, 66, 79, 131, 226, 7, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, + 58, 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, + 71, 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, + 73, 85, 4, 69, 86, 69, 78, 4, 82, 88, 239, 172, 13, 71, 8, 40, 2, 72, 73, + 46, 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 143, + 175, 7, 78, 4, 248, 174, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 155, 221, + 6, 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, + 215, 222, 11, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, + 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 143, 1, 70, 32, 246, 248, 13, 48, 2, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, + 69, 3, 70, 24, 234, 247, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 108, 8, 67, 73, 82, 67, + 76, 69, 68, 32, 22, 76, 226, 3, 83, 160, 2, 5, 84, 79, 78, 69, 45, 223, + 129, 12, 68, 2, 211, 243, 13, 67, 92, 88, 6, 69, 84, 84, 69, 82, 32, 141, + 157, 1, 10, 79, 71, 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, + 84, 222, 188, 7, 88, 186, 116, 65, 146, 161, 1, 82, 188, 44, 2, 72, 65, + 226, 180, 1, 79, 222, 56, 68, 2, 77, 2, 80, 146, 198, 1, 69, 222, 61, 67, + 2, 70, 2, 71, 2, 75, 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, + 73, 2, 85, 3, 87, 22, 86, 84, 158, 236, 11, 80, 238, 131, 2, 67, 2, 75, + 2, 81, 2, 82, 2, 89, 187, 2, 65, 6, 134, 240, 13, 83, 2, 88, 187, 2, 65, + 14, 64, 4, 73, 71, 78, 32, 185, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, + 4, 70, 79, 82, 32, 137, 243, 12, 4, 88, 87, 32, 88, 10, 152, 9, 2, 76, + 79, 190, 147, 3, 84, 132, 89, 8, 73, 78, 86, 69, 82, 84, 69, 66, 154, + 168, 3, 80, 237, 86, 3, 65, 78, 73, 2, 221, 172, 11, 4, 32, 76, 69, 78, + 14, 246, 239, 13, 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 214, 13, + 178, 2, 66, 226, 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, + 76, 32, 68, 22, 76, 222, 63, 78, 166, 5, 80, 214, 7, 82, 190, 10, 83, + 212, 8, 2, 84, 84, 154, 8, 85, 180, 1, 3, 86, 69, 82, 234, 159, 2, 89, + 138, 180, 5, 72, 190, 233, 1, 75, 174, 230, 1, 68, 230, 61, 77, 246, 9, + 87, 159, 137, 1, 88, 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 140, 152, 2, + 9, 83, 69, 82, 86, 69, 82, 32, 69, 89, 221, 28, 8, 74, 69, 67, 84, 32, + 82, 69, 80, 6, 160, 177, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, + 73, 78, 71, 191, 241, 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 172, 5, + 3, 67, 85, 76, 22, 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, + 67, 22, 66, 198, 1, 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, + 69, 68, 32, 70, 175, 149, 12, 72, 2, 207, 158, 5, 72, 6, 136, 1, 15, 82, + 65, 78, 67, 72, 32, 66, 65, 78, 75, 32, 73, 68, 69, 78, 144, 233, 4, 5, + 69, 76, 84, 32, 66, 201, 225, 5, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, + 11, 73, 2, 199, 187, 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, + 65, 67, 67, 79, 85, 78, 84, 32, 78, 227, 195, 1, 72, 2, 159, 135, 6, 85, + 4, 44, 5, 79, 85, 66, 76, 69, 171, 129, 12, 65, 2, 11, 32, 2, 11, 66, 2, + 181, 133, 7, 3, 65, 67, 75, 2, 247, 164, 13, 79, 4, 184, 206, 8, 4, 65, + 71, 79, 78, 209, 148, 3, 2, 79, 80, 2, 11, 69, 2, 213, 215, 7, 5, 32, 66, + 85, 73, 76, 60, 48, 4, 72, 65, 77, 32, 177, 225, 13, 2, 79, 78, 58, 122, + 70, 0, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, + 69, 82, 32, 161, 231, 3, 3, 83, 80, 65, 2, 177, 244, 3, 4, 69, 65, 84, + 72, 52, 196, 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, + 70, 69, 22, 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, + 150, 178, 1, 77, 170, 9, 68, 149, 138, 11, 3, 84, 73, 78, 2, 147, 209, + 13, 76, 2, 11, 69, 2, 199, 226, 11, 73, 4, 138, 239, 7, 69, 255, 237, 3, + 79, 6, 138, 1, 66, 2, 68, 201, 247, 3, 6, 77, 72, 65, 78, 67, 72, 2, 199, + 130, 9, 65, 2, 199, 217, 12, 79, 4, 32, 2, 79, 68, 207, 224, 12, 70, 2, + 235, 224, 7, 72, 2, 135, 195, 11, 85, 4, 176, 204, 12, 3, 71, 69, 65, + 207, 67, 73, 4, 222, 143, 13, 78, 215, 79, 82, 4, 158, 178, 11, 65, 161, + 171, 1, 3, 84, 82, 65, 6, 60, 5, 73, 76, 76, 69, 65, 162, 223, 11, 65, + 135, 255, 1, 82, 2, 207, 142, 13, 78, 2, 155, 182, 12, 82, 220, 7, 64, 7, + 32, 67, 72, 73, 75, 73, 32, 206, 6, 68, 167, 218, 12, 73, 96, 132, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, 62, 71, 74, 80, 164, 198, + 3, 2, 82, 69, 214, 163, 8, 68, 243, 167, 1, 65, 60, 50, 65, 98, 69, 54, + 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 242, 218, 13, 78, 86, 71, 2, 76, + 3, 84, 8, 194, 219, 13, 74, 2, 75, 2, 77, 3, 87, 8, 130, 149, 13, 68, + 198, 13, 82, 210, 56, 78, 3, 80, 8, 154, 201, 13, 78, 202, 17, 72, 2, 82, + 3, 83, 12, 134, 153, 8, 65, 174, 193, 5, 69, 2, 73, 2, 79, 3, 85, 8, 202, + 189, 13, 84, 174, 28, 66, 2, 72, 3, 86, 8, 242, 137, 13, 78, 214, 79, 67, + 2, 68, 3, 89, 4, 56, 2, 45, 71, 221, 242, 11, 6, 32, 84, 84, 85, 68, 68, + 2, 217, 242, 11, 13, 65, 65, 72, 76, 65, 65, 32, 84, 84, 85, 68, 68, 65, + 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 137, 197, 6, 4, + 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, 69, 32, 77, 3, 77, 2, 229, + 144, 13, 3, 85, 67, 65, 250, 6, 34, 32, 225, 55, 3, 69, 82, 32, 244, 6, + 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, 10, 72, 85, 78, 71, 65, + 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, 73, 67, 32, 212, 4, 14, + 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, 78, 32, 196, 4, 3, 80, + 69, 82, 164, 12, 2, 83, 79, 144, 13, 14, 84, 85, 82, 75, 73, 67, 32, 76, + 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, 72, 85, 82, 32, 151, 140, + 11, 75, 4, 166, 183, 11, 73, 205, 91, 3, 72, 79, 79, 216, 1, 96, 6, 67, + 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, 5, 7, 78, 85, 77, 66, 69, + 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 102, 226, 1, 65, + 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, 85, 82, 71, 0, 9, 82, 85, + 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, 132, 223, 3, 5, 67, 76, 79, + 83, 69, 178, 231, 7, 73, 237, 200, 1, 6, 83, 72, 79, 82, 84, 32, 11, 250, + 149, 12, 77, 146, 116, 78, 150, 70, 65, 3, 75, 63, 174, 1, 77, 22, 78, + 78, 83, 146, 218, 9, 67, 86, 71, 2, 76, 2, 84, 154, 239, 1, 90, 226, 131, + 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, 2, 75, 2, 80, 2, 82, 3, 86, 5, + 187, 206, 13, 80, 11, 34, 84, 134, 206, 13, 67, 3, 89, 5, 165, 145, 2, 5, + 45, 83, 72, 65, 80, 5, 219, 205, 13, 90, 4, 11, 32, 4, 230, 182, 13, 79, + 3, 85, 7, 202, 182, 13, 69, 215, 22, 79, 9, 230, 201, 13, 78, 154, 3, 83, + 3, 85, 12, 210, 4, 70, 234, 237, 5, 79, 171, 137, 6, 84, 78, 80, 7, 76, + 69, 84, 84, 69, 82, 32, 169, 3, 8, 78, 85, 77, 69, 82, 65, 76, 32, 70, + 190, 1, 69, 90, 75, 50, 83, 36, 3, 78, 79, 82, 206, 249, 9, 85, 210, 200, + 1, 73, 198, 134, 1, 80, 2, 84, 254, 82, 67, 186, 22, 66, 2, 68, 2, 72, 2, + 86, 2, 89, 2, 90, 214, 22, 65, 3, 79, 23, 198, 214, 9, 83, 238, 239, 1, + 82, 146, 198, 1, 75, 210, 61, 70, 2, 76, 2, 77, 3, 78, 8, 230, 178, 13, + 72, 214, 22, 65, 2, 69, 3, 85, 4, 32, 2, 79, 85, 151, 178, 13, 72, 2, 29, + 5, 84, 72, 69, 82, 78, 2, 209, 191, 12, 2, 32, 84, 8, 38, 70, 146, 247, + 11, 84, 227, 55, 79, 4, 11, 73, 4, 130, 222, 11, 70, 163, 211, 1, 86, 64, + 76, 7, 76, 69, 84, 84, 69, 82, 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, + 58, 210, 1, 65, 38, 71, 38, 72, 30, 75, 38, 84, 74, 90, 246, 104, 77, + 244, 138, 3, 2, 69, 83, 218, 156, 3, 66, 2, 70, 2, 82, 2, 89, 242, 161, + 3, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, 81, 254, 2, 87, 255, 55, 83, + 4, 138, 180, 3, 76, 211, 193, 9, 73, 4, 218, 153, 12, 72, 135, 153, 1, + 69, 4, 214, 195, 13, 65, 3, 69, 4, 130, 140, 6, 72, 191, 183, 6, 65, 8, + 34, 72, 246, 194, 13, 65, 3, 69, 4, 230, 186, 12, 65, 143, 136, 1, 69, 4, + 11, 65, 4, 254, 243, 12, 73, 215, 79, 72, 6, 190, 131, 6, 84, 227, 166, + 6, 79, 178, 1, 64, 11, 77, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, + 5, 83, 76, 228, 1, 2, 67, 72, 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, + 80, 22, 83, 70, 84, 50, 86, 38, 89, 94, 90, 218, 172, 6, 76, 2, 82, 254, + 209, 2, 71, 230, 255, 1, 79, 190, 198, 1, 66, 242, 40, 65, 242, 31, 75, + 174, 45, 72, 187, 2, 85, 2, 171, 180, 7, 69, 6, 26, 90, 219, 172, 13, 79, + 4, 138, 254, 8, 72, 219, 157, 4, 73, 5, 139, 192, 13, 70, 7, 246, 191, + 13, 65, 3, 69, 2, 249, 168, 13, 2, 69, 78, 2, 223, 174, 6, 69, 6, 26, 72, + 187, 171, 13, 73, 4, 232, 252, 8, 3, 67, 72, 79, 3, 79, 4, 26, 83, 247, + 170, 13, 65, 2, 227, 153, 13, 73, 4, 146, 252, 8, 79, 215, 137, 4, 69, + 14, 60, 2, 69, 82, 226, 136, 8, 65, 174, 163, 5, 82, 203, 17, 85, 7, 210, + 189, 13, 73, 3, 85, 4, 146, 251, 8, 72, 219, 157, 4, 65, 102, 52, 4, 73, + 65, 78, 32, 213, 153, 1, 3, 79, 78, 65, 100, 80, 7, 78, 85, 77, 66, 69, + 82, 32, 80, 5, 83, 73, 71, 78, 32, 139, 154, 10, 87, 10, 42, 84, 186, + 147, 8, 72, 147, 143, 4, 79, 6, 162, 228, 1, 87, 231, 135, 11, 69, 88, + 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 226, 109, 71, + 2, 75, 2, 78, 2, 82, 250, 228, 8, 77, 214, 156, 3, 83, 206, 69, 67, 2, + 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, 85, + 82, 65, 77, 65, 90, 68, 65, 65, 7, 222, 213, 6, 45, 159, 225, 6, 72, 6, + 38, 65, 165, 250, 9, 3, 85, 85, 77, 5, 175, 182, 13, 71, 10, 34, 65, 178, + 184, 13, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 179, 212, 6, + 45, 4, 242, 183, 13, 65, 3, 73, 6, 158, 181, 13, 72, 186, 2, 65, 3, 85, + 4, 132, 137, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 171, 174, 2, 65, 144, + 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, 65, 82, 65, + 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, 77, 66, 69, + 82, 32, 2, 11, 82, 2, 157, 155, 11, 10, 65, 67, 84, 73, 79, 78, 32, 79, + 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, 65, 84, 85, + 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, 200, 1, 5, 82, + 69, 83, 72, 45, 230, 214, 1, 76, 178, 151, 4, 71, 90, 90, 34, 83, 66, 89, + 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, + 87, 202, 103, 80, 171, 4, 77, 6, 26, 76, 243, 178, 12, 89, 4, 244, 240, + 5, 8, 84, 69, 82, 78, 65, 84, 69, 32, 239, 199, 1, 69, 18, 116, 3, 78, + 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 130, 215, 1, 65, 158, + 172, 6, 66, 131, 151, 5, 72, 5, 41, 8, 32, 87, 73, 84, 72, 32, 86, 69, 2, + 209, 199, 5, 4, 82, 84, 73, 67, 2, 193, 214, 1, 6, 65, 89, 73, 78, 45, + 68, 18, 42, 84, 158, 239, 5, 79, 143, 211, 5, 70, 10, 42, 72, 230, 215, + 1, 87, 231, 135, 11, 69, 4, 230, 194, 11, 82, 159, 2, 73, 64, 60, 7, 76, + 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, 65, 38, 68, + 74, 71, 34, 75, 34, 83, 78, 84, 218, 43, 90, 230, 165, 1, 76, 50, 81, + 206, 194, 1, 82, 238, 213, 2, 89, 154, 193, 1, 72, 234, 81, 66, 190, 173, + 4, 78, 254, 1, 87, 202, 103, 70, 171, 4, 77, 4, 202, 155, 3, 76, 211, + 193, 9, 89, 6, 32, 2, 72, 65, 219, 210, 1, 65, 4, 210, 254, 7, 76, 203, + 128, 5, 68, 4, 226, 44, 72, 163, 189, 5, 73, 4, 154, 178, 7, 65, 247, 75, + 72, 8, 26, 65, 239, 171, 12, 72, 6, 226, 163, 7, 77, 186, 218, 5, 68, + 143, 45, 84, 8, 166, 90, 72, 206, 209, 10, 69, 247, 128, 1, 65, 6, 56, 4, + 66, 69, 82, 32, 245, 240, 12, 4, 69, 82, 73, 67, 4, 26, 70, 223, 144, 12, + 79, 2, 207, 190, 11, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, 32, 197, + 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, 122, 73, + 30, 79, 203, 192, 11, 66, 45, 106, 69, 130, 180, 9, 83, 226, 243, 3, 66, + 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, 222, 167, 13, + 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, 3, 89, 20, + 74, 78, 142, 138, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, 80, 3, 90, + 8, 182, 166, 13, 67, 2, 71, 2, 84, 3, 89, 7, 138, 166, 13, 67, 3, 81, 13, + 130, 3, 69, 238, 162, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, 1, 69, 86, + 73, 23, 79, 39, 98, 69, 166, 163, 13, 83, 62, 78, 86, 66, 2, 68, 2, 71, + 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 158, 172, 11, 78, 150, 248, 1, 66, + 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 178, 162, 13, 83, 146, 1, 67, 3, + 90, 6, 190, 163, 13, 67, 2, 84, 3, 89, 5, 155, 163, 13, 81, 6, 26, 69, + 239, 162, 13, 81, 5, 235, 162, 13, 75, 52, 148, 1, 10, 67, 79, 77, 66, + 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, 233, 1, 12, + 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 146, 221, 5, 84, 243, + 213, 3, 68, 36, 170, 199, 1, 65, 178, 195, 1, 82, 214, 212, 2, 76, 58, + 90, 34, 83, 66, 89, 252, 195, 1, 6, 70, 73, 78, 65, 76, 32, 0, 6, 71, 73, + 77, 69, 76, 45, 134, 3, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, + 87, 202, 103, 80, 171, 4, 77, 8, 52, 4, 84, 87, 79, 32, 166, 130, 11, 70, + 179, 122, 66, 4, 150, 138, 12, 66, 71, 68, 6, 170, 180, 1, 87, 152, 140, + 3, 3, 65, 68, 85, 235, 150, 7, 77, 22, 212, 1, 34, 32, 87, 73, 84, 72, + 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, + 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, 79, 77, 73, 78, + 71, 32, 210, 1, 69, 239, 202, 12, 73, 2, 21, 3, 73, 71, 72, 2, 151, 197, + 9, 84, 10, 144, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, 73, 82, 69, 32, + 69, 78, 20, 6, 80, 79, 76, 73, 67, 69, 184, 212, 2, 2, 84, 65, 191, 195, + 8, 66, 2, 135, 253, 10, 66, 2, 211, 129, 12, 71, 2, 135, 206, 9, 32, 8, + 70, 32, 245, 230, 11, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, + 40, 4, 68, 79, 84, 32, 147, 234, 9, 66, 4, 26, 79, 235, 235, 9, 76, 2, + 169, 221, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, + 168, 6, 2, 84, 73, 148, 143, 11, 5, 72, 73, 85, 67, 72, 167, 177, 1, 80, + 36, 70, 78, 181, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, + 34, 22, 32, 247, 3, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, + 69, 32, 114, 70, 50, 72, 42, 77, 202, 233, 6, 83, 239, 194, 3, 76, 4, + 190, 147, 13, 79, 155, 3, 88, 8, 50, 84, 238, 130, 11, 66, 222, 2, 65, + 219, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 207, 132, 11, 45, + 4, 180, 255, 1, 5, 73, 76, 69, 32, 70, 15, 79, 2, 17, 2, 65, 78, 2, 187, + 251, 9, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, 87, 73, 84, 72, 32, 4, + 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, 2, 69, 82, 2, 205, 173, + 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, 73, 84, 45, 79, 85, 84, + 80, 85, 84, 32, 217, 242, 11, 6, 79, 85, 84, 76, 73, 78, 4, 18, 72, 3, + 76, 2, 229, 190, 1, 4, 45, 84, 89, 80, 2, 189, 132, 12, 6, 77, 32, 67, + 79, 77, 77, 6, 64, 2, 79, 78, 153, 177, 1, 8, 67, 65, 76, 32, 68, 73, 83, + 67, 2, 235, 250, 10, 32, 204, 1, 132, 1, 3, 65, 78, 71, 66, 73, 212, 8, + 5, 78, 65, 84, 69, 32, 192, 225, 3, 5, 84, 72, 79, 68, 79, 254, 186, 1, + 32, 227, 231, 5, 67, 6, 28, 2, 69, 32, 239, 21, 85, 4, 154, 191, 11, 66, + 211, 73, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, + 151, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, + 84, 69, 82, 32, 142, 2, 83, 214, 1, 86, 243, 153, 11, 68, 4, 182, 142, 8, + 73, 3, 85, 12, 41, 8, 82, 65, 67, 84, 73, 79, 78, 32, 12, 56, 4, 79, 78, + 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 170, 140, 11, 69, 46, + 72, 43, 81, 2, 177, 141, 11, 4, 73, 88, 84, 69, 4, 26, 83, 187, 142, 11, + 81, 2, 249, 223, 7, 4, 73, 88, 84, 69, 2, 225, 233, 11, 3, 83, 83, 72, + 104, 226, 1, 82, 166, 212, 6, 89, 150, 205, 2, 65, 38, 68, 46, 84, 46, + 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, + 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 87, 186, 2, + 69, 3, 79, 6, 206, 135, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, + 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, + 29, 4, 73, 71, 78, 32, 4, 178, 134, 13, 72, 3, 82, 14, 222, 161, 9, 65, + 74, 67, 98, 78, 234, 128, 2, 79, 227, 161, 1, 86, 26, 49, 10, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 26, 254, 162, 9, 65, 106, 86, 214, 20, 85, + 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 242, 171, 9, 76, 27, 82, 226, + 1, 76, 4, 65, 71, 69, 32, 164, 4, 6, 77, 65, 78, 89, 65, 32, 247, 129, + 13, 67, 144, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, + 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 114, + 75, 42, 79, 22, 84, 178, 204, 6, 72, 170, 190, 4, 67, 2, 68, 2, 71, 138, + 176, 1, 83, 2, 90, 130, 3, 66, 254, 65, 76, 2, 77, 2, 78, 2, 80, 2, 87, + 186, 2, 73, 3, 85, 9, 226, 139, 11, 73, 131, 248, 1, 72, 15, 26, 72, 211, + 179, 12, 73, 10, 50, 84, 182, 139, 11, 67, 134, 245, 1, 75, 3, 80, 4, + 182, 128, 13, 83, 187, 2, 65, 6, 150, 128, 13, 72, 2, 89, 187, 2, 65, 5, + 211, 178, 12, 73, 6, 222, 186, 12, 83, 183, 71, 65, 80, 52, 7, 76, 69, + 84, 84, 69, 82, 32, 163, 147, 11, 68, 60, 246, 1, 65, 38, 67, 22, 68, 38, + 75, 34, 83, 30, 77, 130, 229, 2, 81, 174, 218, 7, 79, 204, 119, 2, 76, + 65, 156, 71, 2, 78, 85, 254, 1, 87, 138, 62, 69, 222, 61, 66, 2, 70, 2, + 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, 73, 3, 85, 7, 158, + 238, 2, 76, 167, 145, 10, 65, 2, 195, 129, 12, 65, 4, 178, 158, 8, 69, + 163, 222, 4, 72, 4, 206, 253, 11, 65, 227, 126, 72, 4, 26, 72, 175, 254, + 12, 65, 2, 239, 254, 11, 73, 124, 68, 11, 79, 77, 65, 78, 32, 83, 73, 89, + 65, 81, 32, 131, 197, 12, 69, 122, 172, 1, 17, 65, 76, 84, 69, 82, 78, + 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, 13, 70, 82, 65, 67, + 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, 82, 47, 78, 26, 54, + 70, 50, 83, 46, 84, 238, 225, 11, 78, 211, 110, 69, 6, 208, 187, 5, 3, + 79, 85, 82, 211, 195, 6, 73, 6, 160, 187, 5, 2, 73, 88, 171, 211, 5, 69, + 10, 138, 173, 2, 69, 12, 2, 87, 79, 183, 225, 8, 72, 4, 26, 83, 163, 250, + 10, 72, 2, 139, 251, 10, 73, 2, 11, 82, 2, 11, 65, 2, 219, 178, 11, 84, + 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, 70, 94, 78, 26, 83, + 78, 84, 131, 157, 5, 79, 10, 25, 4, 73, 71, 72, 84, 11, 138, 171, 2, 89, + 191, 165, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, 147, 156, 5, 86, 10, + 134, 2, 82, 157, 156, 5, 2, 85, 82, 10, 65, 3, 73, 78, 69, 20, 40, 4, 69, + 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 223, 205, 7, 32, 24, 34, 72, + 50, 87, 203, 168, 2, 69, 10, 34, 73, 197, 156, 5, 2, 82, 69, 4, 51, 82, + 10, 26, 69, 171, 156, 5, 79, 4, 11, 78, 4, 11, 84, 4, 159, 168, 2, 89, + 12, 34, 84, 133, 222, 10, 2, 78, 67, 10, 42, 66, 37, 6, 76, 73, 78, 69, + 68, 32, 2, 181, 233, 11, 4, 79, 88, 32, 84, 8, 254, 252, 2, 76, 254, 174, + 3, 87, 250, 181, 4, 66, 239, 13, 71, 12, 18, 72, 35, 76, 2, 245, 227, 11, + 3, 69, 65, 84, 10, 32, 2, 65, 80, 227, 218, 11, 73, 9, 29, 5, 80, 73, 78, + 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, 4, 44, 5, 65, 78, 68, + 32, 66, 247, 199, 9, 83, 2, 221, 199, 9, 4, 76, 65, 67, 75, 144, 13, 194, + 1, 65, 174, 65, 68, 30, 69, 194, 13, 72, 134, 25, 73, 206, 4, 76, 150, + 15, 79, 234, 8, 82, 208, 15, 15, 83, 65, 76, 84, 69, 82, 32, 80, 65, 72, + 76, 65, 86, 73, 32, 214, 5, 85, 151, 215, 11, 77, 154, 6, 134, 2, 68, 38, + 71, 212, 1, 11, 72, 65, 87, 72, 32, 72, 77, 79, 78, 71, 32, 138, 23, 76, + 130, 6, 78, 58, 82, 156, 22, 2, 83, 83, 132, 2, 10, 85, 32, 67, 73, 78, + 32, 72, 65, 85, 32, 180, 7, 3, 87, 32, 80, 172, 213, 7, 2, 67, 75, 145, + 217, 4, 4, 80, 69, 82, 67, 5, 157, 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, + 191, 201, 12, 79, 13, 34, 32, 158, 238, 12, 82, 3, 83, 6, 72, 6, 87, 73, + 84, 72, 32, 67, 149, 139, 4, 6, 70, 65, 67, 73, 78, 71, 4, 50, 85, 181, + 197, 7, 6, 73, 82, 67, 76, 69, 68, 2, 255, 227, 11, 82, 254, 1, 190, 1, + 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, 73, 77, 32, 160, 1, 7, 78, 85, 77, + 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, 78, 32, 148, 10, 7, 86, 79, 87, + 69, 76, 32, 75, 227, 233, 10, 68, 78, 92, 9, 76, 65, 78, 32, 83, 73, 71, + 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, 78, 84, 32, 38, 132, 1, 2, 72, + 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, 89, 148, 9, 2, 88, 89, 176, 5, 2, + 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, 4, 162, 186, 2, 87, 187, 175, 10, + 77, 6, 250, 15, 72, 202, 186, 12, 79, 159, 14, 87, 4, 214, 12, 65, 231, + 158, 12, 73, 8, 22, 83, 247, 9, 72, 6, 152, 185, 2, 3, 72, 69, 69, 150, + 245, 8, 65, 3, 87, 4, 226, 184, 2, 65, 3, 87, 4, 198, 184, 2, 65, 223, + 233, 9, 69, 40, 122, 67, 38, 72, 46, 78, 60, 2, 80, 76, 2, 81, 222, 210, + 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, 147, 238, 9, 65, 4, 230, 211, + 2, 72, 147, 238, 9, 65, 6, 194, 211, 2, 76, 2, 78, 147, 238, 9, 65, 12, + 58, 67, 22, 84, 202, 210, 2, 75, 2, 76, 147, 238, 9, 65, 2, 219, 210, 2, + 72, 4, 198, 210, 2, 72, 3, 83, 14, 42, 75, 50, 83, 38, 84, 195, 211, 12, + 72, 4, 26, 72, 143, 167, 12, 69, 2, 183, 138, 6, 65, 4, 138, 171, 11, 85, + 191, 185, 1, 79, 4, 254, 169, 11, 85, 179, 18, 65, 14, 52, 7, 72, 85, 78, + 68, 82, 69, 68, 38, 84, 79, 77, 4, 108, 2, 32, 77, 223, 226, 12, 83, 8, + 24, 2, 69, 78, 51, 82, 6, 26, 32, 243, 226, 12, 83, 4, 18, 66, 35, 84, 2, + 145, 194, 4, 3, 73, 76, 76, 2, 11, 72, 2, 137, 185, 9, 3, 79, 85, 83, 72, + 188, 1, 4, 67, 73, 77, 32, 250, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, + 50, 84, 124, 4, 86, 79, 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, + 84, 72, 150, 176, 1, 76, 219, 205, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, + 69, 83, 32, 22, 84, 148, 164, 11, 7, 80, 85, 66, 32, 68, 65, 87, 197, + 185, 1, 16, 72, 65, 73, 83, 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, + 84, 4, 48, 7, 85, 65, 77, 32, 84, 83, 72, 131, 4, 72, 2, 11, 79, 2, 167, + 175, 2, 79, 2, 207, 182, 1, 84, 6, 56, 3, 88, 87, 86, 157, 221, 12, 5, + 83, 79, 86, 32, 82, 5, 213, 131, 6, 4, 32, 67, 72, 87, 4, 214, 72, 78, + 171, 129, 12, 76, 2, 251, 163, 11, 89, 6, 40, 4, 69, 69, 74, 32, 171, + 159, 12, 85, 4, 128, 3, 2, 84, 83, 57, 2, 83, 85, 4, 26, 84, 231, 213, 8, + 81, 2, 251, 172, 2, 85, 6, 196, 1, 7, 88, 72, 69, 69, 74, 32, 67, 216, + 139, 8, 12, 72, 73, 82, 68, 45, 83, 84, 65, 71, 69, 32, 72, 155, 170, 4, + 65, 14, 54, 70, 22, 83, 30, 84, 190, 69, 76, 163, 212, 7, 78, 2, 191, + 200, 12, 69, 2, 177, 128, 6, 2, 69, 69, 6, 42, 72, 29, 6, 83, 72, 65, 66, + 32, 67, 4, 82, 73, 231, 200, 12, 79, 2, 223, 188, 5, 69, 14, 34, 73, 22, + 89, 191, 214, 10, 65, 2, 151, 159, 11, 65, 10, 40, 4, 69, 69, 77, 32, + 139, 186, 12, 79, 8, 84, 3, 78, 84, 88, 128, 254, 5, 2, 84, 79, 142, 158, + 2, 82, 233, 129, 3, 2, 70, 65, 2, 255, 253, 5, 73, 2, 215, 168, 2, 65, + 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, 79, 39, 87, 20, 170, 1, 65, 2, 73, + 2, 85, 2, 87, 158, 214, 12, 66, 3, 86, 8, 106, 69, 158, 214, 12, 66, 3, + 86, 8, 70, 65, 158, 214, 12, 66, 3, 86, 8, 34, 79, 158, 214, 12, 66, 3, + 86, 4, 154, 214, 12, 66, 3, 86, 76, 18, 76, 23, 77, 2, 155, 152, 12, 65, + 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, 32, 241, 249, 3, 4, 83, 32, 85, + 80, 8, 80, 3, 66, 82, 65, 250, 231, 10, 84, 192, 94, 4, 68, 79, 87, 78, + 1, 2, 85, 80, 2, 143, 178, 12, 78, 64, 80, 2, 76, 69, 40, 4, 82, 73, 71, + 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, 32, 48, 38, 70, 89, 5, 84, 84, 69, + 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 70, 2, + 137, 209, 5, 2, 76, 69, 46, 224, 1, 5, 70, 73, 78, 65, 76, 30, 84, 246, + 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, 212, 2, 65, 50, 71, 90, + 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, + 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 2, 209, 208, 11, 2, 32, 78, + 4, 202, 209, 10, 69, 247, 128, 1, 65, 14, 198, 120, 84, 198, 234, 9, 70, + 143, 84, 79, 4, 32, 2, 67, 65, 183, 144, 12, 68, 2, 251, 187, 11, 75, + 184, 2, 94, 65, 216, 1, 11, 69, 78, 84, 72, 69, 83, 73, 90, 69, 68, 32, + 242, 16, 84, 243, 235, 11, 82, 14, 80, 5, 71, 82, 65, 80, 72, 48, 5, 76, + 76, 69, 76, 32, 201, 138, 6, 2, 67, 72, 6, 206, 121, 85, 198, 187, 8, 32, + 175, 219, 2, 79, 6, 44, 5, 87, 73, 84, 72, 32, 191, 174, 12, 84, 4, 214, + 165, 6, 84, 151, 138, 4, 72, 150, 2, 252, 1, 7, 72, 65, 78, 71, 85, 76, + 32, 188, 4, 10, 73, 68, 69, 79, 71, 82, 65, 80, 72, 32, 188, 7, 18, 75, + 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 79, 44, + 7, 78, 85, 77, 66, 69, 82, 32, 138, 187, 4, 68, 197, 154, 2, 2, 76, 65, + 58, 102, 67, 110, 72, 30, 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 26, + 84, 73, 5, 73, 69, 85, 78, 71, 10, 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, + 3, 4, 73, 69, 85, 67, 7, 11, 32, 4, 206, 201, 12, 65, 3, 85, 4, 197, 2, + 3, 73, 69, 85, 8, 168, 2, 5, 72, 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, + 75, 4, 245, 1, 4, 73, 69, 85, 77, 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, + 1, 5, 72, 73, 69, 85, 80, 13, 4, 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, + 76, 4, 93, 3, 73, 79, 83, 8, 56, 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, + 69, 85, 84, 4, 11, 72, 5, 167, 196, 12, 32, 72, 148, 1, 2, 65, 76, 30, + 67, 74, 69, 82, 70, 112, 2, 76, 65, 22, 77, 38, 78, 32, 2, 82, 69, 78, + 83, 138, 2, 84, 50, 87, 134, 215, 10, 72, 159, 79, 79, 2, 225, 233, 7, 2, + 76, 73, 4, 32, 2, 79, 78, 199, 192, 10, 65, 2, 137, 212, 7, 4, 71, 82, + 65, 84, 6, 42, 78, 246, 253, 8, 65, 239, 155, 3, 73, 2, 169, 4, 5, 84, + 69, 82, 80, 82, 10, 58, 73, 156, 186, 11, 5, 69, 83, 84, 73, 86, 231, 16, + 79, 6, 208, 2, 3, 78, 65, 78, 158, 170, 12, 82, 3, 86, 2, 179, 138, 12, + 66, 4, 174, 245, 9, 69, 239, 253, 1, 79, 4, 194, 169, 11, 73, 195, 1, 65, + 8, 38, 83, 170, 178, 11, 80, 195, 109, 65, 4, 182, 179, 6, 79, 219, 142, + 6, 84, 18, 58, 69, 34, 79, 22, 80, 34, 84, 50, 85, 163, 180, 11, 73, 4, + 186, 240, 10, 86, 235, 79, 76, 2, 231, 212, 7, 67, 2, 11, 69, 2, 247, + 161, 4, 67, 4, 26, 85, 247, 145, 11, 79, 2, 247, 174, 12, 68, 4, 26, 80, + 147, 192, 12, 78, 2, 21, 3, 69, 82, 86, 2, 131, 183, 11, 73, 6, 162, 211, + 10, 72, 238, 156, 1, 69, 227, 48, 87, 4, 206, 157, 9, 79, 155, 235, 1, + 65, 4, 166, 253, 7, 32, 137, 242, 3, 2, 74, 69, 22, 42, 69, 46, 70, 42, + 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, 71, 72, 243, 225, 4, 76, 4, 160, 1, + 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, + 86, 69, 78, 2, 71, 88, 8, 34, 72, 46, 87, 247, 236, 11, 69, 2, 11, 73, 2, + 11, 82, 2, 219, 235, 10, 84, 4, 11, 69, 4, 198, 210, 10, 78, 179, 109, + 76, 18, 128, 1, 3, 73, 65, 76, 216, 1, 3, 89, 32, 80, 144, 141, 8, 6, 78, + 69, 82, 83, 72, 73, 189, 167, 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, + 189, 122, 10, 76, 89, 45, 82, 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, + 76, 73, 78, 69, 32, 2, 157, 155, 4, 5, 73, 70, 70, 69, 82, 8, 134, 186, + 3, 68, 242, 45, 70, 20, 4, 66, 65, 67, 75, 243, 209, 8, 85, 2, 235, 222, + 3, 79, 10, 98, 69, 52, 9, 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, + 79, 82, 84, 32, 67, 79, 78, 84, 4, 182, 165, 11, 68, 145, 62, 5, 78, 71, + 69, 82, 32, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 233, 155, 5, 6, + 45, 79, 85, 84, 80, 85, 2, 227, 164, 11, 82, 114, 192, 1, 12, 71, 76, 79, + 84, 84, 65, 76, 32, 83, 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, + 14, 77, 73, 68, 45, 76, 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, + 65, 78, 68, 72, 73, 32, 7, 11, 32, 4, 206, 5, 70, 197, 166, 11, 4, 86, + 65, 82, 73, 82, 72, 6, 69, 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, + 65, 76, 76, 74, 206, 1, 70, 226, 241, 6, 73, 2, 85, 166, 242, 1, 78, 214, + 135, 3, 67, 2, 75, 2, 80, 2, 84, 254, 68, 66, 2, 68, 2, 71, 2, 72, 2, 76, + 2, 77, 2, 82, 2, 83, 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, + 73, 78, 65, 76, 32, 223, 178, 12, 65, 18, 198, 186, 10, 78, 150, 248, 1, + 75, 2, 76, 2, 77, 2, 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, + 32, 84, 7, 11, 32, 4, 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, + 73, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, + 69, 9, 11, 32, 6, 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, + 135, 181, 3, 73, 2, 223, 161, 4, 82, 4, 222, 175, 12, 70, 3, 73, 80, 130, + 1, 65, 122, 78, 162, 1, 82, 222, 9, 83, 44, 3, 84, 82, 73, 254, 16, 68, + 173, 192, 11, 9, 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, + 218, 204, 6, 32, 135, 225, 5, 82, 6, 190, 255, 10, 79, 190, 27, 69, 147, + 147, 1, 72, 2, 199, 153, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 190, + 171, 1, 84, 224, 172, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 175, + 167, 4, 67, 2, 147, 173, 11, 85, 2, 167, 239, 11, 69, 48, 186, 1, 32, + 116, 10, 80, 69, 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 198, 136, 7, 67, + 224, 10, 10, 77, 65, 78, 69, 78, 84, 32, 80, 65, 80, 153, 234, 1, 8, 70, + 79, 82, 77, 73, 78, 71, 32, 6, 34, 77, 30, 84, 223, 163, 11, 83, 2, 153, + 230, 1, 2, 73, 76, 2, 189, 196, 10, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, + 129, 215, 8, 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 182, 75, 80, 213, + 213, 10, 4, 69, 86, 69, 82, 28, 22, 32, 243, 5, 65, 26, 216, 2, 10, 68, + 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 108, 5, 87, 73, + 84, 72, 32, 208, 185, 7, 4, 70, 82, 79, 87, 200, 13, 27, 82, 65, 73, 83, + 73, 78, 71, 32, 66, 79, 84, 72, 32, 72, 65, 78, 68, 83, 32, 73, 78, 32, + 67, 69, 76, 69, 66, 128, 141, 4, 5, 67, 76, 73, 77, 66, 201, 45, 11, 66, + 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 179, 169, 3, 72, 4, + 76, 7, 83, 84, 69, 65, 77, 89, 32, 217, 211, 11, 6, 76, 79, 84, 85, 83, + 32, 2, 197, 148, 12, 2, 82, 79, 12, 154, 1, 66, 212, 134, 2, 3, 80, 79, + 85, 160, 157, 1, 2, 67, 82, 196, 214, 5, 6, 70, 79, 76, 68, 69, 68, 177, + 168, 2, 8, 72, 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 159, + 159, 10, 65, 2, 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 203, 234, 11, + 73, 2, 21, 3, 76, 32, 67, 2, 161, 236, 10, 4, 79, 77, 80, 85, 4, 232, + 230, 8, 2, 69, 84, 251, 180, 2, 79, 2, 217, 227, 8, 2, 32, 68, 140, 2, + 66, 65, 180, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, + 108, 6, 71, 83, 45, 80, 65, 32, 189, 7, 16, 73, 83, 84, 79, 83, 32, 68, + 73, 83, 67, 32, 83, 73, 71, 78, 32, 112, 100, 7, 76, 69, 84, 84, 69, 82, + 32, 248, 4, 5, 77, 65, 82, 75, 32, 30, 83, 33, 4, 68, 79, 85, 66, 96, + 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 174, + 204, 8, 71, 218, 134, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, + 222, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, + 88, 2, 89, 186, 2, 73, 2, 79, 3, 85, 7, 80, 8, 76, 84, 69, 82, 78, 65, + 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 163, 154, 12, 32, 2, + 11, 32, 2, 167, 154, 12, 70, 6, 26, 65, 251, 153, 12, 72, 5, 159, 182, 8, + 78, 6, 226, 153, 12, 68, 2, 90, 187, 2, 65, 6, 236, 135, 8, 4, 77, 65, + 76, 76, 206, 145, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, + 211, 163, 10, 32, 2, 243, 247, 8, 32, 4, 130, 212, 11, 68, 59, 83, 10, + 28, 3, 73, 78, 71, 31, 85, 2, 241, 205, 11, 2, 76, 69, 8, 58, 66, 225, + 206, 11, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 229, 150, 8, 13, 74, 79, + 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 92, 238, 1, 66, 146, 1, 67, + 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 66, 80, + 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 168, 203, 5, 2, 70, 76, 172, 224, + 1, 2, 79, 88, 188, 193, 3, 2, 69, 65, 142, 7, 65, 135, 1, 86, 10, 52, 2, + 69, 69, 22, 79, 205, 40, 4, 85, 76, 76, 83, 5, 199, 142, 7, 72, 4, 32, 2, + 79, 77, 175, 150, 12, 87, 2, 11, 69, 2, 219, 189, 11, 82, 16, 34, 65, 86, + 72, 22, 76, 23, 79, 6, 194, 200, 5, 80, 200, 192, 3, 8, 82, 80, 69, 78, + 84, 82, 89, 32, 223, 140, 3, 84, 2, 255, 175, 2, 73, 2, 219, 218, 10, 85, + 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 239, 196, 11, 77, 5, 37, 7, 73, + 78, 73, 78, 71, 32, 79, 2, 237, 172, 9, 5, 66, 76, 73, 81, 85, 4, 222, + 235, 10, 76, 175, 145, 1, 86, 4, 42, 82, 245, 182, 10, 4, 65, 85, 78, 84, + 2, 195, 220, 10, 65, 6, 42, 69, 198, 180, 7, 79, 155, 209, 2, 73, 2, 183, + 221, 11, 76, 4, 242, 128, 12, 76, 203, 17, 68, 4, 40, 2, 78, 65, 181, + 169, 9, 2, 84, 84, 2, 159, 233, 8, 67, 8, 52, 2, 69, 68, 50, 76, 237, + 136, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 211, 201, 10, 73, 4, + 184, 167, 4, 2, 85, 77, 229, 211, 2, 3, 65, 78, 69, 4, 230, 132, 10, 79, + 255, 250, 1, 65, 12, 108, 2, 72, 73, 154, 145, 11, 65, 154, 45, 76, 128, + 19, 3, 84, 82, 65, 165, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 146, 170, + 2, 69, 151, 229, 9, 80, 6, 200, 165, 4, 5, 65, 84, 84, 79, 79, 250, 164, + 7, 73, 195, 36, 85, 4, 246, 196, 7, 79, 221, 187, 3, 5, 65, 86, 89, 32, + 66, 4, 218, 232, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, + 73, 65, 78, 32, 183, 138, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, + 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 199, 248, 5, 87, 44, 234, 1, 65, + 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 202, 246, 1, 87, 150, 212, 5, 90, + 130, 157, 1, 89, 200, 188, 1, 2, 82, 79, 128, 92, 3, 71, 65, 77, 134, 8, + 75, 130, 1, 78, 132, 58, 3, 76, 65, 77, 138, 17, 66, 186, 30, 80, 171, 4, + 77, 4, 198, 137, 11, 76, 187, 49, 73, 2, 151, 172, 3, 69, 4, 171, 213, 6, + 69, 2, 255, 136, 11, 79, 6, 130, 253, 9, 65, 186, 141, 1, 72, 165, 124, + 2, 69, 77, 4, 214, 228, 11, 65, 191, 8, 69, 12, 186, 49, 84, 187, 151, 4, + 79, 40, 104, 2, 67, 75, 66, 71, 62, 76, 90, 78, 178, 1, 83, 32, 7, 84, + 67, 72, 70, 79, 82, 75, 243, 132, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, + 84, 82, 2, 155, 217, 10, 85, 7, 11, 32, 4, 26, 78, 179, 202, 11, 70, 2, + 183, 254, 10, 79, 6, 52, 4, 69, 32, 79, 70, 238, 90, 67, 247, 171, 11, + 76, 2, 11, 32, 2, 139, 198, 9, 80, 14, 68, 2, 67, 72, 46, 69, 186, 149, + 4, 87, 242, 232, 6, 75, 219, 98, 65, 4, 144, 140, 3, 2, 69, 68, 211, 235, + 7, 73, 4, 28, 2, 32, 68, 195, 71, 65, 2, 185, 241, 4, 2, 69, 67, 4, 174, + 241, 10, 67, 139, 1, 84, 5, 165, 186, 9, 10, 32, 87, 73, 84, 72, 32, 84, + 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 179, 250, 11, 68, 176, 1, 78, 67, + 128, 1, 12, 78, 67, 75, 32, 67, 79, 78, 83, 84, 65, 78, 84, 83, 89, 6, + 44, 5, 69, 32, 79, 70, 32, 179, 236, 10, 65, 4, 56, 6, 73, 78, 84, 69, + 82, 69, 189, 173, 11, 2, 87, 79, 2, 231, 235, 6, 83, 5, 45, 9, 32, 79, + 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 159, 238, 11, 80, 166, 1, 96, + 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 149, 151, 4, 9, 71, 82, 79, 85, + 78, 68, 32, 83, 76, 164, 1, 182, 1, 66, 44, 3, 82, 69, 68, 0, 5, 87, 72, + 73, 84, 69, 42, 70, 74, 75, 38, 69, 34, 83, 36, 3, 81, 85, 69, 14, 84, + 92, 2, 65, 67, 0, 3, 78, 73, 78, 13, 4, 74, 65, 67, 75, 4, 40, 4, 76, 65, + 67, 75, 203, 208, 10, 65, 2, 17, 2, 32, 74, 2, 203, 225, 1, 79, 18, 30, + 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 175, 243, 10, 79, 16, 34, + 78, 185, 1, 3, 73, 78, 71, 8, 181, 1, 4, 73, 71, 72, 84, 16, 32, 2, 69, + 86, 117, 2, 73, 88, 8, 91, 69, 66, 78, 69, 12, 3, 72, 82, 69, 12, 2, 87, + 79, 161, 1, 5, 82, 85, 77, 80, 45, 8, 23, 78, 8, 11, 69, 8, 25, 4, 32, + 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 158, 207, 8, 72, + 137, 3, 5, 68, 73, 65, 77, 79, 2, 255, 189, 11, 66, 2, 219, 231, 10, 68, + 42, 90, 50, 250, 192, 9, 49, 214, 185, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 7, 202, 250, 11, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, 79, + 239, 189, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 221, 223, 8, 2, 45, 77, + 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 219, 235, + 3, 65, 6, 34, 76, 22, 82, 247, 212, 10, 84, 2, 41, 2, 69, 70, 2, 21, 3, + 73, 71, 72, 2, 133, 169, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, 68, + 34, 83, 204, 221, 9, 4, 84, 73, 76, 68, 140, 117, 5, 66, 76, 65, 67, 75, + 185, 35, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, 69, + 78, 2, 11, 79, 2, 231, 207, 9, 84, 4, 54, 77, 165, 157, 5, 7, 85, 66, 83, + 67, 82, 73, 80, 2, 197, 231, 8, 3, 65, 76, 76, 11, 33, 6, 32, 70, 79, 82, + 77, 32, 8, 154, 136, 10, 70, 71, 84, 58, 232, 1, 5, 76, 73, 67, 69, 32, + 122, 80, 136, 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, + 158, 1, 84, 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 202, 231, 6, 79, 245, + 204, 4, 11, 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, + 82, 141, 234, 3, 4, 79, 70, 70, 73, 5, 173, 221, 9, 11, 83, 32, 82, 69, + 86, 79, 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, + 79, 78, 65, 76, 32, 187, 203, 1, 67, 4, 166, 117, 73, 185, 162, 6, 6, 70, + 79, 82, 77, 65, 84, 2, 191, 210, 11, 69, 12, 40, 2, 69, 73, 22, 84, 195, + 246, 4, 73, 2, 147, 161, 11, 68, 8, 36, 3, 65, 76, 32, 163, 228, 10, 66, + 6, 130, 202, 1, 72, 217, 180, 6, 4, 77, 65, 82, 75, 8, 66, 65, 210, 247, + 1, 32, 173, 238, 8, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 179, 208, 11, + 84, 2, 29, 5, 76, 69, 32, 87, 65, 2, 135, 47, 84, 12, 108, 4, 76, 84, 82, + 89, 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 250, 135, 10, 78, + 199, 228, 1, 67, 2, 165, 168, 11, 2, 32, 76, 2, 209, 156, 5, 2, 81, 85, + 4, 197, 137, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 128, 218, 10, 4, + 45, 79, 70, 70, 15, 32, 4, 160, 213, 8, 3, 76, 69, 69, 219, 132, 2, 89, + 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 237, 139, 9, 6, 65, 89, 69, 82, + 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 156, 167, 5, 4, + 67, 69, 68, 69, 144, 193, 1, 5, 86, 73, 79, 85, 83, 157, 33, 2, 84, 90, + 6, 42, 87, 146, 151, 5, 80, 243, 139, 5, 77, 2, 235, 160, 7, 79, 70, 176, + 1, 27, 69, 78, 84, 65, 84, 73, 79, 78, 32, 70, 79, 82, 77, 32, 70, 79, + 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 241, 162, 8, 10, 67, 82, 73, + 80, 84, 73, 79, 78, 32, 84, 68, 198, 1, 67, 22, 69, 46, 72, 50, 73, 94, + 76, 188, 1, 6, 82, 73, 71, 72, 84, 32, 192, 2, 6, 87, 65, 86, 89, 32, 76, + 202, 136, 4, 83, 192, 171, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 163, + 99, 81, 4, 203, 170, 2, 79, 6, 138, 252, 5, 88, 214, 194, 2, 77, 3, 78, + 2, 233, 215, 8, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, + 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 206, 154, 9, 67, 35, 70, 22, 40, 4, + 69, 70, 84, 32, 191, 128, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, + 142, 1, 66, 42, 68, 218, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, + 39, 84, 4, 162, 2, 67, 159, 93, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, + 73, 84, 69, 32, 182, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, 39, + 84, 2, 177, 94, 6, 76, 65, 67, 75, 32, 76, 2, 209, 99, 5, 79, 85, 66, 76, + 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, 66, + 82, 65, 2, 131, 93, 79, 4, 198, 143, 11, 67, 177, 29, 2, 75, 67, 2, 235, + 252, 9, 79, 22, 46, 78, 156, 1, 2, 86, 65, 171, 201, 11, 77, 10, 34, 84, + 173, 171, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, 67, 82, + 2, 253, 235, 9, 2, 69, 69, 5, 17, 2, 32, 73, 2, 175, 144, 11, 67, 10, 60, + 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 241, 132, 7, 2, + 83, 83, 8, 26, 32, 191, 175, 6, 45, 4, 146, 210, 5, 84, 223, 243, 4, 79, + 14, 130, 1, 74, 30, 80, 228, 32, 5, 72, 73, 66, 73, 84, 148, 176, 8, 6, + 66, 73, 78, 71, 32, 67, 133, 210, 1, 5, 83, 69, 82, 80, 73, 2, 181, 144, + 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 197, 195, 10, 4, 69, 82, + 84, 89, 5, 179, 182, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, 32, 68, 79, + 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 230, + 2, 78, 178, 31, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, 210, 157, + 10, 67, 187, 109, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, 50, 87, + 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, + 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 203, 103, 80, 2, 207, 151, 4, + 76, 2, 11, 65, 2, 251, 171, 6, 76, 2, 235, 151, 4, 65, 2, 25, 4, 69, 77, + 45, 81, 2, 195, 223, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, 141, + 194, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, 186, + 151, 4, 79, 187, 217, 2, 70, 8, 42, 87, 250, 234, 9, 72, 239, 156, 1, 69, + 4, 182, 237, 9, 69, 131, 234, 1, 79, 18, 252, 1, 4, 78, 67, 84, 85, 106, + 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, 84, 83, + 32, 80, 76, 65, 170, 140, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, 65, 68, + 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 157, 10, 49, 3, + 50, 4, 80, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 137, 150, 11, 5, 65, + 84, 73, 79, 78, 2, 135, 147, 11, 83, 4, 32, 2, 80, 76, 183, 189, 11, 83, + 2, 227, 204, 10, 69, 2, 11, 67, 2, 215, 192, 10, 69, 40, 98, 65, 148, 6, + 6, 69, 83, 84, 73, 79, 78, 214, 231, 5, 79, 253, 216, 4, 5, 73, 78, 67, + 85, 78, 30, 104, 2, 68, 82, 208, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, + 73, 48, 4, 82, 84, 69, 82, 223, 170, 10, 79, 24, 56, 4, 65, 78, 84, 32, + 253, 3, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, + 76, 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, + 32, 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, + 80, 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 184, 1, 5, + 76, 69, 70, 84, 32, 195, 163, 11, 82, 4, 11, 79, 4, 11, 87, 4, 241, 163, + 11, 2, 69, 82, 7, 69, 15, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, + 69, 70, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 229, 255, 9, 6, 32, 76, + 79, 87, 69, 82, 4, 22, 73, 155, 37, 80, 2, 133, 163, 8, 7, 78, 84, 69, + 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 195, 136, 10, 79, 6, 34, 32, 185, + 165, 5, 2, 69, 68, 4, 198, 244, 4, 69, 211, 150, 6, 77, 152, 9, 114, 65, + 182, 7, 69, 224, 27, 6, 72, 73, 78, 79, 67, 69, 34, 73, 182, 78, 76, 46, + 79, 246, 18, 85, 151, 198, 10, 83, 60, 110, 67, 104, 2, 68, 73, 130, 1, + 73, 202, 4, 84, 160, 210, 7, 4, 66, 66, 73, 84, 130, 185, 3, 90, 223, 56, + 77, 6, 40, 4, 73, 78, 71, 32, 211, 184, 8, 67, 4, 140, 251, 9, 7, 77, 79, + 84, 79, 82, 67, 89, 203, 44, 67, 8, 66, 79, 245, 255, 7, 10, 67, 65, 76, + 32, 83, 89, 77, 66, 79, 76, 7, 200, 145, 6, 4, 65, 67, 84, 73, 197, 133, + 4, 2, 32, 66, 34, 60, 5, 76, 87, 65, 89, 32, 46, 78, 21, 4, 83, 69, 68, + 32, 4, 228, 189, 8, 2, 84, 82, 247, 231, 1, 67, 5, 203, 173, 10, 66, 26, + 156, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, 65, 78, 68, 182, 1, 77, 198, + 139, 7, 70, 186, 237, 1, 67, 226, 170, 1, 83, 197, 18, 7, 66, 65, 67, 75, + 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, 165, 39, 8, 78, 84, 69, 82, + 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 130, 27, 70, 201, + 176, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, 69, 69, 78, 32, 77, 73, + 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, 71, 6, 182, 189, 10, 67, + 2, 68, 3, 82, 5, 135, 165, 11, 73, 248, 1, 226, 1, 67, 232, 3, 2, 68, 32, + 64, 2, 71, 73, 144, 1, 5, 74, 65, 78, 71, 32, 254, 4, 76, 32, 8, 77, 73, + 78, 68, 69, 82, 32, 82, 34, 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, + 42, 86, 225, 184, 1, 5, 70, 69, 82, 69, 78, 24, 114, 69, 32, 3, 89, 67, + 76, 158, 176, 5, 79, 181, 237, 3, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, + 32, 86, 69, 72, 2, 11, 73, 2, 211, 164, 11, 80, 18, 78, 69, 53, 15, 73, + 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, 32, + 80, 65, 80, 2, 239, 223, 9, 69, 16, 100, 5, 84, 89, 80, 69, 45, 157, 237, + 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, 58, + 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 229, 177, 5, 6, 32, 80, + 76, 65, 83, 84, 4, 42, 65, 229, 233, 4, 4, 71, 73, 70, 84, 2, 143, 173, + 3, 80, 54, 120, 4, 83, 84, 69, 82, 165, 212, 5, 20, 79, 78, 65, 76, 32, + 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 135, 215, + 9, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, + 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 150, 2, 83, 35, 86, 8, + 174, 195, 9, 78, 150, 248, 1, 72, 3, 82, 46, 154, 1, 77, 34, 78, 166, + 183, 11, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, + 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 4, 194, 183, 11, 66, 187, 2, + 65, 12, 54, 89, 158, 234, 7, 71, 210, 204, 3, 68, 187, 2, 65, 4, 234, + 182, 11, 74, 187, 2, 65, 2, 11, 69, 2, 223, 207, 10, 67, 18, 64, 10, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 171, 134, 9, 73, 16, 54, 69, 190, + 247, 10, 65, 174, 64, 73, 2, 79, 3, 85, 7, 230, 183, 11, 65, 3, 85, 2, + 137, 167, 10, 3, 73, 69, 86, 2, 201, 231, 10, 3, 73, 66, 66, 2, 41, 8, + 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 253, 255, 9, 5, 72, + 65, 82, 65, 67, 8, 32, 2, 84, 82, 187, 217, 6, 80, 6, 168, 134, 6, 16, + 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 243, 158, + 5, 79, 6, 162, 162, 10, 83, 142, 104, 76, 31, 82, 70, 64, 4, 69, 82, 83, + 69, 173, 228, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, 68, + 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, 79, + 76, 73, 68, 85, 83, 196, 193, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, 69, + 82, 65, 84, 79, 82, 32, 65, 147, 203, 2, 73, 2, 221, 155, 10, 2, 82, 32, + 4, 136, 194, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, + 84, 69, 68, 32, 80, 239, 221, 1, 78, 9, 11, 32, 6, 200, 135, 5, 9, 80, + 82, 69, 67, 69, 68, 73, 78, 71, 226, 255, 2, 79, 167, 140, 1, 87, 48, + 232, 2, 5, 65, 78, 71, 76, 69, 20, 7, 68, 79, 85, 66, 76, 69, 32, 118, + 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 102, 84, 212, + 201, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, + 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 158, 148, 2, + 67, 114, 81, 212, 61, 3, 86, 73, 67, 157, 36, 6, 69, 77, 80, 84, 89, 32, + 5, 159, 213, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, + 77, 69, 5, 195, 166, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 147, 150, 6, + 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 175, 4, 72, 4, + 36, 3, 73, 76, 67, 151, 148, 10, 82, 2, 11, 82, 2, 253, 164, 10, 2, 79, + 87, 6, 156, 1, 17, 65, 73, 83, 69, 68, 32, 72, 65, 78, 68, 32, 87, 73, + 84, 72, 32, 70, 252, 96, 8, 79, 84, 65, 84, 69, 68, 32, 70, 229, 149, 6, + 4, 73, 71, 72, 84, 2, 185, 102, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, + 132, 124, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, 84, + 65, 76, 151, 248, 6, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, 80, + 76, 69, 32, 80, 173, 251, 6, 3, 72, 85, 77, 5, 189, 214, 4, 2, 32, 69, 2, + 187, 144, 10, 82, 2, 11, 82, 2, 171, 234, 10, 79, 213, 3, 228, 1, 4, 66, + 66, 79, 78, 164, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 236, 71, 2, 78, 71, + 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 67, + 82, 79, 83, 83, 73, 78, 71, 32, 134, 195, 5, 65, 143, 246, 3, 70, 19, 37, + 7, 32, 65, 82, 82, 79, 87, 32, 16, 80, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 22, 85, 213, 249, 10, 3, 68, 79, 87, 4, 175, 234, 1, 84, 4, 223, 249, + 10, 80, 4, 26, 67, 131, 160, 9, 66, 2, 133, 135, 1, 3, 82, 65, 67, 162, + 3, 110, 32, 158, 31, 45, 132, 8, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, + 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 176, 1, 198, 1, 65, 132, 6, 2, 66, + 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 212, 2, 2, 76, + 79, 66, 78, 82, 79, 122, 80, 168, 1, 2, 82, 65, 58, 83, 154, 6, 84, 194, + 3, 86, 143, 1, 87, 28, 22, 78, 163, 4, 82, 22, 24, 2, 68, 32, 39, 71, 4, + 214, 41, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, 2, 76, 69, 2, 175, 23, + 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, 84, 69, 68, 32, 83, 2, + 83, 112, 5, 87, 73, 84, 72, 32, 189, 253, 9, 12, 86, 65, 82, 73, 65, 78, + 84, 32, 87, 73, 84, 72, 4, 133, 252, 9, 6, 82, 65, 67, 75, 69, 84, 2, 37, + 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, 73, 79, 78, 2, 21, 3, 32, + 77, 65, 2, 243, 128, 1, 82, 4, 68, 11, 68, 79, 87, 78, 87, 65, 82, 68, + 83, 32, 90, 203, 171, 8, 65, 2, 141, 131, 10, 5, 73, 71, 90, 65, 71, 6, + 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, 69, 82, 2, 189, 20, 4, + 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, 72, 32, 4, 132, 191, 7, + 7, 67, 73, 82, 67, 76, 69, 68, 207, 141, 2, 83, 4, 25, 4, 65, 67, 75, 32, + 4, 194, 22, 76, 243, 172, 7, 84, 12, 38, 85, 230, 21, 79, 251, 222, 2, + 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 9, 11, 32, 6, + 44, 4, 77, 73, 68, 68, 138, 8, 76, 23, 85, 2, 129, 229, 9, 2, 76, 69, 12, + 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 173, 14, 6, 84, 69, 68, 32, 83, 85, + 10, 50, 65, 94, 87, 174, 145, 3, 81, 151, 172, 4, 80, 4, 206, 25, 78, + 217, 149, 3, 15, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 82, 79, 85, 78, + 68, 2, 203, 19, 73, 6, 26, 73, 227, 241, 2, 76, 4, 146, 146, 8, 86, 163, + 88, 83, 14, 32, 3, 76, 70, 32, 223, 1, 78, 12, 42, 66, 90, 70, 34, 82, + 139, 166, 8, 77, 6, 11, 76, 6, 40, 4, 65, 67, 75, 32, 231, 231, 9, 79, 4, + 166, 199, 9, 67, 167, 11, 83, 2, 11, 79, 2, 147, 242, 8, 76, 2, 33, 6, + 85, 78, 78, 73, 78, 71, 2, 131, 204, 6, 32, 2, 149, 145, 1, 16, 68, 32, + 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 2, 241, 193, 10, + 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, 65, 83, 2, 197, 4, 16, 79, 82, + 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, 32, 83, 69, 77, 8, 74, 85, 214, + 150, 8, 78, 237, 178, 1, 8, 80, 69, 78, 32, 83, 81, 85, 65, 2, 201, 147, + 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, 10, 65, 82, 69, 78, 84, 72, 69, 83, + 73, 83, 9, 11, 32, 6, 34, 76, 22, 85, 147, 248, 8, 69, 2, 37, 2, 79, 87, + 2, 17, 2, 80, 80, 2, 169, 223, 3, 2, 69, 82, 2, 249, 8, 10, 73, 83, 69, + 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, 73, 68, 2, 80, 69, + 126, 81, 227, 2, 85, 2, 153, 208, 7, 12, 83, 72, 65, 80, 69, 68, 32, 66, + 65, 71, 32, 68, 4, 26, 77, 199, 160, 8, 86, 2, 205, 128, 10, 7, 73, 68, + 73, 82, 69, 67, 84, 4, 134, 136, 3, 78, 193, 180, 7, 8, 68, 69, 87, 65, + 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 186, 237, 3, 79, 75, 84, 2, 201, 4, 2, 67, 72, 20, 57, 12, + 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, 11, 32, 18, 84, 3, + 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, 32, 223, 242, 8, 69, + 2, 133, 216, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, 75, 32, 73, 78, 32, + 162, 5, 81, 214, 189, 7, 85, 158, 97, 68, 195, 183, 2, 83, 4, 132, 215, + 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, 197, 2, 6, 66, 83, 84, + 73, 84, 85, 20, 54, 72, 162, 1, 82, 146, 177, 7, 79, 179, 169, 2, 65, 8, + 54, 73, 60, 5, 79, 85, 71, 72, 84, 131, 155, 8, 82, 2, 201, 197, 1, 10, + 82, 68, 32, 87, 72, 73, 84, 69, 32, 82, 2, 11, 32, 2, 205, 250, 2, 3, 66, + 85, 66, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, 6, 78, 83, 80, 79, 83, + 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 191, 181, 10, 78, 6, 32, 2, 76, 69, + 183, 154, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, 32, 76, 2, 161, 232, + 8, 2, 69, 70, 4, 45, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 4, 68, 9, 65, + 82, 32, 87, 73, 84, 72, 32, 81, 245, 236, 9, 2, 79, 88, 2, 215, 149, 8, + 85, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, 69, 32, 12, 54, 67, 54, + 76, 182, 170, 7, 80, 154, 2, 83, 39, 84, 4, 26, 79, 215, 168, 7, 85, 2, + 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, 85, 76, 65, 2, 231, 177, + 10, 82, 2, 221, 167, 6, 6, 71, 71, 76, 89, 32, 70, 30, 82, 70, 206, 1, + 72, 130, 1, 80, 250, 1, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 8, 33, 6, + 65, 67, 73, 78, 71, 32, 8, 144, 1, 14, 65, 82, 77, 69, 78, 73, 65, 78, + 32, 69, 84, 69, 82, 78, 144, 196, 3, 11, 83, 86, 65, 83, 84, 73, 32, 83, + 73, 71, 78, 151, 133, 3, 70, 2, 239, 227, 7, 73, 2, 81, 18, 65, 78, 68, + 69, 68, 32, 73, 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, + 78, 84, 2, 171, 199, 9, 65, 8, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 8, + 140, 1, 6, 67, 85, 82, 86, 69, 68, 22, 65, 224, 247, 2, 6, 68, 79, 85, + 66, 76, 69, 253, 239, 6, 10, 77, 65, 71, 78, 73, 70, 89, 73, 78, 71, 2, + 17, 2, 32, 65, 2, 11, 78, 2, 157, 172, 10, 2, 71, 76, 4, 46, 72, 85, 7, + 73, 68, 69, 32, 65, 82, 67, 2, 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 165, + 222, 9, 6, 32, 87, 72, 73, 84, 69, 2, 11, 32, 2, 193, 168, 8, 8, 67, 76, + 79, 67, 75, 87, 73, 83, 8, 17, 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, + 82, 196, 238, 4, 5, 69, 77, 66, 69, 68, 187, 204, 5, 77, 2, 17, 2, 83, + 79, 2, 207, 129, 1, 76, 2, 179, 146, 3, 82, 2, 249, 236, 9, 2, 79, 82, + 210, 1, 160, 1, 5, 65, 82, 82, 79, 87, 182, 8, 66, 86, 68, 210, 1, 70, + 122, 72, 230, 6, 76, 26, 79, 34, 80, 50, 82, 70, 83, 94, 84, 202, 8, 87, + 226, 254, 7, 67, 47, 81, 71, 26, 32, 159, 205, 8, 45, 66, 94, 65, 170, 2, + 70, 82, 84, 184, 1, 5, 87, 73, 84, 72, 32, 149, 152, 1, 4, 79, 86, 69, + 82, 12, 40, 5, 66, 79, 86, 69, 32, 143, 1, 78, 10, 70, 82, 208, 155, 1, + 5, 83, 72, 79, 82, 84, 250, 180, 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, + 83, 69, 32, 4, 158, 208, 4, 65, 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, + 69, 82, 32, 65, 78, 68, 32, 76, 2, 17, 2, 79, 87, 2, 137, 142, 8, 2, 69, + 82, 4, 37, 7, 82, 79, 77, 32, 66, 65, 82, 5, 189, 1, 6, 32, 84, 79, 32, + 66, 76, 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 144, + 203, 4, 3, 83, 85, 80, 130, 156, 4, 71, 131, 143, 2, 88, 4, 26, 76, 235, + 188, 10, 65, 2, 213, 171, 9, 3, 65, 67, 75, 38, 140, 1, 6, 67, 79, 82, + 78, 69, 82, 26, 68, 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, + 78, 122, 69, 226, 148, 1, 72, 155, 160, 1, 86, 2, 213, 18, 2, 32, 68, 4, + 11, 79, 4, 40, 4, 84, 84, 69, 68, 203, 177, 7, 85, 2, 17, 2, 32, 83, 2, + 247, 224, 10, 84, 4, 150, 140, 8, 65, 247, 237, 1, 79, 2, 249, 203, 8, 2, + 76, 85, 6, 218, 140, 8, 77, 135, 182, 2, 84, 10, 206, 16, 73, 171, 3, 65, + 8, 58, 65, 128, 12, 5, 79, 84, 84, 79, 77, 219, 128, 8, 76, 2, 145, 2, 2, + 67, 75, 14, 48, 6, 79, 85, 66, 76, 69, 32, 235, 154, 8, 65, 12, 40, 5, + 65, 82, 82, 79, 87, 135, 19, 68, 11, 26, 32, 183, 195, 8, 45, 6, 26, 87, + 243, 144, 8, 70, 4, 25, 4, 73, 84, 72, 32, 4, 190, 191, 10, 86, 79, 83, + 4, 40, 4, 82, 79, 78, 84, 143, 140, 8, 73, 2, 141, 139, 8, 14, 45, 84, + 73, 76, 84, 69, 68, 32, 83, 72, 65, 68, 79, 87, 30, 26, 65, 195, 143, 8, + 69, 26, 48, 6, 82, 80, 79, 79, 78, 32, 147, 168, 10, 78, 24, 88, 8, 79, + 86, 69, 82, 32, 76, 69, 70, 45, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, + 32, 2, 237, 219, 7, 6, 84, 87, 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, + 173, 1, 2, 85, 80, 10, 26, 32, 247, 148, 8, 87, 8, 76, 8, 65, 66, 79, 86, + 69, 32, 76, 69, 18, 66, 190, 140, 8, 70, 175, 5, 84, 2, 227, 2, 70, 2, + 185, 201, 4, 7, 69, 76, 79, 87, 32, 76, 79, 12, 26, 32, 203, 147, 8, 87, + 10, 60, 6, 65, 66, 79, 86, 69, 32, 178, 139, 8, 70, 175, 5, 84, 6, 22, + 76, 155, 1, 82, 4, 32, 2, 69, 70, 247, 199, 4, 79, 2, 233, 135, 2, 24, + 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, + 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, 71, 72, 2, 105, 24, 84, 87, 65, 82, + 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, + 82, 66, 2, 11, 32, 2, 163, 232, 1, 68, 2, 141, 1, 2, 69, 70, 2, 221, 141, + 8, 3, 80, 69, 78, 4, 218, 145, 8, 65, 221, 199, 1, 3, 85, 83, 72, 4, 36, + 3, 73, 71, 72, 187, 148, 10, 79, 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, + 85, 235, 136, 8, 65, 4, 26, 73, 131, 137, 8, 65, 2, 245, 144, 8, 2, 71, + 71, 54, 38, 79, 60, 2, 82, 73, 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 149, + 129, 8, 4, 83, 72, 65, 68, 34, 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, + 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 251, 149, 8, 32, 28, 52, 5, 65, + 82, 82, 79, 87, 218, 141, 8, 68, 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, + 69, 82, 32, 76, 69, 70, 22, 87, 155, 137, 8, 84, 2, 203, 136, 8, 84, 18, + 25, 4, 73, 84, 72, 32, 18, 128, 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, + 76, 79, 78, 71, 32, 84, 73, 246, 136, 8, 66, 158, 1, 77, 34, 78, 34, 86, + 35, 72, 4, 186, 196, 8, 72, 255, 236, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, + 68, 35, 85, 2, 197, 137, 8, 3, 79, 87, 78, 2, 167, 137, 8, 80, 4, 21, 3, + 76, 69, 32, 4, 138, 3, 68, 183, 195, 9, 65, 18, 11, 79, 18, 56, 8, 45, + 72, 69, 65, 68, 69, 68, 32, 191, 139, 8, 32, 16, 76, 6, 65, 82, 82, 79, + 87, 32, 213, 1, 8, 84, 82, 73, 80, 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, + 72, 32, 203, 255, 7, 70, 12, 42, 84, 206, 156, 7, 68, 191, 145, 3, 86, 8, + 26, 65, 131, 139, 8, 82, 6, 17, 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 246, 155, 7, 68, 191, 145, 3, 86, 2, 153, 127, 2, 65, 83, 8, 58, + 65, 21, 10, 72, 73, 84, 69, 32, 65, 82, 82, 79, 87, 2, 227, 135, 8, 86, + 7, 11, 32, 4, 164, 185, 2, 4, 70, 82, 79, 77, 147, 212, 5, 87, 19, 66, + 32, 136, 1, 6, 69, 68, 32, 80, 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, + 80, 132, 180, 4, 2, 73, 78, 26, 69, 214, 252, 2, 79, 187, 173, 2, 65, 2, + 195, 201, 10, 85, 2, 11, 79, 2, 231, 151, 10, 73, 2, 215, 165, 10, 78, 2, + 189, 190, 3, 2, 32, 66, 4, 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, + 21, 3, 73, 78, 71, 2, 205, 222, 1, 2, 32, 68, 2, 225, 164, 3, 2, 79, 85, + 8, 178, 217, 10, 69, 2, 73, 2, 77, 3, 79, 124, 136, 2, 2, 67, 75, 20, 2, + 76, 76, 134, 2, 79, 56, 4, 77, 65, 78, 32, 192, 8, 2, 83, 69, 20, 6, 84, + 65, 84, 69, 68, 32, 152, 3, 3, 85, 78, 68, 230, 169, 3, 87, 228, 189, 3, + 15, 65, 83, 84, 69, 68, 32, 83, 87, 69, 69, 84, 32, 80, 79, 84, 165, 253, + 1, 2, 66, 79, 5, 207, 186, 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, + 101, 21, 73, 78, 71, 32, 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, + 32, 76, 65, 85, 71, 6, 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, + 82, 32, 2, 11, 80, 2, 207, 250, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, + 151, 158, 9, 83, 2, 239, 89, 75, 2, 199, 131, 10, 72, 72, 140, 1, 6, 67, + 69, 78, 84, 85, 82, 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, + 82, 65, 76, 32, 182, 4, 83, 114, 85, 159, 179, 7, 65, 2, 223, 187, 5, 73, + 6, 98, 69, 232, 5, 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, + 32, 83, 69, 88, 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, + 3, 79, 78, 69, 134, 1, 83, 66, 84, 152, 14, 10, 82, 69, 86, 69, 82, 83, + 69, 68, 32, 79, 142, 229, 2, 69, 235, 193, 6, 78, 14, 26, 73, 155, 216, + 9, 79, 12, 36, 3, 70, 84, 89, 167, 246, 2, 86, 7, 11, 32, 4, 158, 168, 5, + 84, 185, 169, 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, + 72, 79, 85, 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 205, + 245, 3, 2, 32, 67, 6, 32, 2, 73, 88, 223, 225, 8, 69, 5, 221, 207, 9, 2, + 32, 76, 10, 42, 69, 190, 244, 2, 87, 135, 237, 5, 72, 4, 11, 78, 5, 11, + 32, 2, 223, 165, 5, 84, 10, 46, 69, 197, 145, 7, 5, 73, 76, 73, 81, 85, + 8, 60, 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, + 67, 2, 239, 144, 7, 73, 2, 231, 178, 7, 73, 4, 18, 65, 23, 85, 2, 203, + 178, 7, 78, 2, 159, 144, 7, 76, 5, 191, 135, 9, 84, 10, 166, 2, 70, 32, + 11, 72, 69, 65, 86, 89, 32, 66, 76, 65, 67, 75, 52, 24, 76, 73, 71, 72, + 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, 66, 76, 65, + 67, 75, 0, 18, 87, 72, 73, 84, 69, 32, 70, 79, 85, 82, 32, 80, 79, 73, + 78, 84, 69, 68, 169, 53, 8, 67, 65, 80, 73, 84, 65, 76, 32, 2, 29, 5, 76, + 79, 82, 65, 76, 2, 217, 236, 8, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, + 225, 132, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, + 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 141, + 200, 9, 2, 72, 80, 2, 175, 210, 3, 65, 12, 68, 2, 83, 72, 242, 135, 6, + 67, 254, 153, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, + 161, 10, 79, 2, 207, 178, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, + 40, 5, 76, 69, 45, 68, 69, 40, 3, 77, 73, 32, 154, 5, 78, 156, 26, 26, + 83, 83, 73, 65, 78, 32, 65, 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, + 32, 83, 89, 77, 66, 79, 76, 32, 247, 252, 4, 80, 2, 211, 172, 8, 76, 2, + 205, 191, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 161, 228, 5, 2, 65, 89, + 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 195, 190, 2, 68, + 8, 40, 4, 79, 78, 69, 32, 203, 229, 8, 84, 6, 34, 84, 150, 194, 8, 72, + 43, 81, 2, 191, 195, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, + 69, 73, 71, 72, 84, 38, 70, 92, 2, 78, 73, 22, 79, 18, 83, 83, 84, 4, + 158, 129, 3, 32, 139, 192, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 251, + 212, 8, 70, 4, 136, 2, 2, 85, 82, 211, 212, 8, 82, 4, 77, 2, 78, 69, 2, + 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 206, 255, 2, 32, + 195, 174, 7, 84, 10, 34, 72, 50, 87, 143, 239, 9, 69, 4, 32, 2, 82, 69, + 215, 212, 8, 73, 2, 39, 69, 4, 26, 79, 199, 212, 8, 69, 2, 187, 254, 2, + 32, 182, 1, 32, 3, 73, 67, 32, 147, 25, 78, 178, 1, 220, 1, 6, 66, 69, + 76, 71, 84, 72, 20, 4, 67, 82, 79, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, + 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 188, 141, 7, 5, 65, 82, 76, + 65, 85, 205, 183, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 151, 220, 8, 79, + 2, 167, 143, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, + 222, 1, 71, 104, 2, 72, 65, 50, 73, 220, 1, 5, 74, 69, 82, 65, 78, 34, + 75, 58, 76, 234, 1, 79, 128, 1, 13, 82, 65, 73, 68, 79, 32, 82, 65, 68, + 32, 82, 69, 73, 34, 83, 176, 2, 16, 66, 69, 82, 75, 65, 78, 65, 78, 32, + 66, 69, 79, 82, 67, 32, 66, 144, 1, 12, 78, 65, 85, 68, 73, 90, 32, 78, + 89, 68, 32, 78, 110, 84, 194, 1, 87, 244, 83, 7, 85, 82, 85, 90, 32, 85, + 82, 208, 187, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 184, 60, 13, + 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 218, 246, 2, 89, 166, + 227, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 202, 158, 6, 67, 0, 4, + 78, 83, 85, 90, 249, 135, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, + 46, 69, 30, 65, 197, 239, 6, 3, 87, 69, 79, 4, 26, 65, 247, 181, 10, 78, + 2, 147, 143, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 209, 219, 3, 9, + 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 181, 10, 76, 2, 78, 3, 80, + 11, 228, 70, 7, 72, 87, 65, 90, 32, 69, 72, 174, 181, 9, 65, 194, 55, 84, + 63, 78, 12, 120, 13, 82, 65, 78, 75, 83, 32, 67, 65, 83, 75, 69, 84, 32, + 233, 253, 6, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, + 154, 245, 9, 73, 2, 79, 195, 60, 69, 4, 26, 69, 207, 178, 10, 67, 2, 235, + 139, 9, 83, 9, 26, 69, 207, 249, 9, 65, 4, 52, 7, 66, 79, 32, 71, 89, 70, + 85, 231, 177, 10, 82, 2, 143, 177, 10, 32, 4, 236, 8, 2, 69, 71, 13, 4, + 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, 83, 32, + 73, 83, 20, 5, 87, 65, 90, 32, 69, 136, 247, 9, 10, 67, 69, 76, 65, 78, + 68, 73, 67, 45, 89, 3, 79, 5, 131, 185, 6, 87, 2, 231, 239, 9, 83, 2, + 199, 174, 10, 79, 2, 11, 32, 2, 183, 175, 10, 74, 7, 21, 3, 65, 85, 78, + 4, 242, 171, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, 32, 76, + 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, 65, 78, + 67, 72, 45, 2, 211, 164, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, 72, 62, + 77, 66, 79, 179, 239, 9, 89, 2, 195, 150, 10, 65, 15, 150, 5, 83, 0, 12, + 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 167, 10, 4, 80, 69, + 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 167, 243, 9, 32, 26, 150, 1, + 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, 82, 65, 78, + 67, 72, 45, 83, 228, 212, 6, 5, 79, 87, 73, 76, 79, 195, 139, 2, 84, 21, + 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, 72, 62, 77, + 30, 78, 38, 79, 42, 83, 186, 1, 84, 156, 146, 6, 2, 65, 82, 183, 219, 3, + 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 235, 238, 8, 32, 2, 25, 4, 65, + 71, 65, 76, 2, 11, 76, 2, 195, 167, 10, 32, 2, 253, 145, 3, 2, 65, 68, 2, + 205, 216, 9, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, 136, 10, 32, 2, + 11, 79, 2, 215, 212, 6, 76, 4, 116, 15, 72, 85, 82, 73, 83, 65, 90, 32, + 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, 84, 73, 82, 32, + 84, 2, 11, 72, 2, 167, 200, 5, 79, 2, 17, 2, 89, 82, 2, 223, 137, 10, 32, + 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, 175, 167, 9, 32, + 2, 21, 3, 73, 78, 71, 2, 169, 248, 6, 2, 76, 69, 4, 204, 191, 8, 16, 73, + 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, 83, 219, 172, 1, + 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, 86, 73, 71, 134, + 234, 6, 83, 249, 154, 1, 6, 84, 82, 69, 68, 69, 67, 2, 149, 133, 8, 2, + 79, 86, 2, 237, 132, 8, 2, 73, 78, 214, 37, 244, 1, 2, 32, 73, 22, 65, + 238, 25, 67, 138, 5, 69, 146, 7, 72, 186, 33, 73, 186, 233, 1, 75, 154, + 2, 76, 130, 9, 77, 170, 21, 78, 174, 2, 79, 142, 39, 80, 180, 11, 2, 81, + 85, 158, 67, 83, 38, 84, 142, 16, 85, 150, 35, 87, 186, 1, 89, 187, 251, + 4, 71, 2, 131, 253, 8, 78, 196, 2, 140, 2, 5, 70, 69, 84, 89, 32, 36, 4, + 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, 65, 78, 32, 142, 14, + 78, 190, 3, 84, 100, 2, 85, 82, 132, 236, 2, 2, 73, 76, 198, 87, 88, 180, + 209, 5, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, 65, 78, 68, 32, 67, + 143, 98, 82, 4, 234, 250, 1, 86, 151, 164, 7, 80, 2, 205, 201, 3, 2, 65, + 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 163, 253, 8, 73, 2, 11, 65, + 2, 211, 223, 9, 75, 2, 151, 149, 9, 84, 122, 184, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, + 137, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, 38, 75, 22, 76, 34, + 83, 46, 84, 182, 2, 65, 208, 204, 5, 2, 71, 65, 134, 139, 1, 82, 210, + 237, 1, 90, 190, 78, 77, 172, 1, 2, 81, 85, 118, 78, 218, 6, 89, 235, + 101, 70, 4, 214, 253, 9, 73, 247, 25, 65, 2, 147, 134, 9, 76, 6, 206, + 153, 10, 78, 2, 84, 3, 89, 2, 147, 152, 9, 65, 2, 11, 65, 2, 187, 133, 9, + 66, 4, 152, 7, 3, 73, 78, 71, 171, 202, 8, 72, 6, 178, 151, 9, 65, 238, + 100, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, 82, 75, 32, 165, + 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 12, + 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 173, 249, 7, 5, 79, 67, 67, + 76, 85, 2, 17, 2, 65, 71, 2, 163, 177, 8, 69, 5, 17, 2, 45, 65, 2, 255, + 148, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 143, 10, 73, 2, 145, 156, + 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, 130, 1, 65, 154, + 1, 66, 22, 78, 30, 83, 130, 1, 90, 238, 155, 3, 84, 200, 208, 6, 9, 77, + 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, 65, 34, 78, 28, + 2, 84, 77, 145, 238, 9, 4, 82, 75, 65, 65, 2, 11, 65, 2, 179, 147, 10, + 81, 4, 26, 78, 199, 179, 5, 71, 2, 11, 65, 2, 143, 238, 9, 65, 2, 177, + 132, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, 32, 77, 65, + 83, 72, 70, 2, 17, 2, 65, 65, 2, 167, 162, 8, 76, 2, 139, 254, 8, 65, 4, + 34, 65, 221, 161, 8, 2, 73, 81, 2, 151, 144, 9, 69, 30, 92, 5, 76, 79, + 78, 71, 32, 54, 79, 66, 83, 146, 206, 4, 65, 174, 193, 5, 69, 2, 73, 3, + 85, 10, 130, 207, 4, 65, 174, 193, 5, 69, 2, 73, 3, 85, 7, 41, 8, 86, 69, + 82, 76, 79, 78, 71, 32, 4, 163, 206, 4, 65, 4, 26, 72, 183, 192, 5, 85, + 2, 153, 251, 5, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, 69, 82, 73, 70, 32, + 145, 236, 9, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, 86, 89, 32, 139, 2, 73, + 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, 4, 11, 66, 4, 21, 3, 76, + 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, 67, 2, 21, 3, 79, 85, 66, + 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, 79, 77, 77, 65, 32, 81, 2, + 153, 197, 8, 3, 85, 79, 84, 2, 169, 200, 9, 10, 78, 84, 69, 82, 82, 79, + 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, 69, 135, 173, 5, 85, 5, 25, + 4, 32, 65, 78, 84, 2, 251, 232, 6, 69, 166, 1, 52, 7, 65, 83, 72, 84, 82, + 65, 32, 163, 169, 4, 79, 164, 1, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 212, 1, 5, 83, 73, 71, 78, 32, 186, 17, 68, 156, 244, 2, 16, 67, 79, 78, + 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 72, 251, 138, 2, 86, 100, + 150, 160, 6, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, 79, + 182, 56, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 162, 7, 69, 222, 61, 72, 2, 77, 2, 82, 3, 89, 8, 234, + 160, 6, 67, 190, 161, 3, 65, 239, 1, 86, 52, 66, 65, 32, 4, 72, 79, 79, + 76, 46, 79, 74, 82, 143, 133, 10, 73, 4, 190, 242, 8, 76, 215, 18, 82, 5, + 165, 165, 5, 6, 32, 83, 65, 84, 67, 72, 6, 36, 3, 82, 80, 73, 227, 206, + 8, 79, 4, 214, 181, 9, 79, 135, 18, 85, 36, 66, 69, 72, 4, 73, 80, 84, + 32, 166, 243, 1, 85, 175, 140, 6, 79, 4, 36, 3, 87, 68, 82, 207, 180, 9, + 69, 2, 11, 73, 2, 179, 198, 9, 86, 28, 80, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, 18, 170, 131, 10, 66, 2, 69, + 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, 82, 2, 37, 7, 73, 71, 65, + 84, 85, 82, 69, 2, 11, 32, 2, 205, 190, 9, 2, 69, 84, 8, 134, 130, 10, + 69, 2, 71, 2, 76, 3, 79, 64, 250, 1, 65, 30, 67, 74, 69, 36, 5, 71, 77, + 69, 78, 84, 28, 2, 77, 73, 174, 1, 82, 158, 1, 83, 68, 2, 84, 32, 200, + 245, 4, 8, 87, 73, 78, 71, 32, 78, 69, 69, 172, 237, 1, 6, 80, 65, 82, + 65, 84, 69, 178, 122, 88, 202, 99, 68, 149, 159, 1, 2, 76, 70, 4, 222, + 255, 9, 76, 3, 84, 6, 34, 84, 157, 200, 5, 2, 79, 78, 4, 194, 165, 6, 73, + 143, 161, 3, 79, 4, 238, 216, 1, 68, 223, 131, 1, 45, 23, 181, 169, 4, 2, + 69, 68, 6, 128, 77, 28, 68, 73, 82, 69, 67, 84, 32, 80, 82, 79, 68, 85, + 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, 84, 84, 79, 77, 32, 67, 172, 147, + 7, 3, 83, 69, 88, 151, 206, 1, 67, 4, 120, 2, 86, 73, 217, 171, 2, 22, + 73, 79, 85, 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, + 66, 79, 76, 83, 2, 11, 67, 2, 207, 186, 9, 69, 4, 52, 7, 81, 85, 73, 81, + 85, 65, 68, 155, 198, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, + 77, 73, 84, 32, 83, 147, 158, 6, 77, 2, 11, 84, 2, 179, 182, 8, 65, 240, + 2, 102, 65, 254, 19, 73, 102, 79, 250, 10, 69, 70, 82, 184, 202, 8, 5, + 85, 70, 70, 76, 69, 143, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, + 68, 32, 87, 72, 73, 84, 69, 32, 68, 9, 76, 76, 79, 87, 32, 80, 65, 78, + 32, 62, 82, 206, 9, 86, 172, 133, 7, 2, 77, 82, 183, 224, 1, 75, 6, 38, + 76, 134, 169, 8, 67, 167, 11, 83, 2, 249, 185, 8, 2, 65, 84, 2, 17, 2, + 79, 70, 2, 17, 2, 32, 70, 2, 227, 213, 6, 79, 194, 1, 40, 4, 65, 68, 65, + 32, 135, 247, 9, 75, 192, 1, 162, 1, 68, 42, 69, 110, 72, 34, 76, 246, 1, + 83, 208, 2, 6, 86, 79, 87, 69, 76, 32, 226, 175, 4, 65, 196, 213, 1, 7, + 67, 79, 78, 84, 73, 78, 85, 235, 217, 3, 79, 24, 158, 32, 79, 66, 65, + 239, 230, 7, 73, 4, 84, 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, + 86, 79, 87, 69, 163, 187, 8, 75, 2, 163, 179, 9, 76, 2, 189, 196, 9, 3, + 69, 65, 68, 96, 33, 6, 69, 84, 84, 69, 82, 32, 96, 178, 139, 6, 65, 38, + 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, + 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, + 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, + 78, 32, 141, 236, 8, 3, 85, 84, 82, 2, 177, 176, 9, 4, 78, 68, 72, 73, 6, + 132, 202, 2, 5, 67, 84, 73, 79, 78, 231, 142, 4, 80, 20, 102, 73, 182, + 249, 4, 83, 134, 144, 1, 65, 74, 67, 98, 78, 230, 179, 1, 74, 150, 3, 85, + 211, 235, 1, 86, 2, 37, 7, 78, 86, 69, 82, 84, 69, 68, 2, 213, 137, 6, 2, + 32, 67, 30, 60, 6, 77, 79, 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, + 227, 252, 5, 69, 28, 86, 80, 226, 137, 6, 65, 106, 86, 214, 20, 85, 210, + 200, 1, 73, 206, 134, 2, 69, 3, 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, + 65, 77, 65, 84, 82, 65, 2, 167, 215, 9, 32, 98, 72, 3, 69, 68, 32, 21, + 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 2, 171, 176, 9, 73, 96, + 158, 2, 65, 120, 3, 67, 72, 85, 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, + 38, 79, 94, 80, 18, 82, 22, 83, 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, + 208, 221, 4, 2, 74, 85, 162, 186, 2, 68, 202, 13, 90, 150, 84, 70, 182, + 6, 71, 210, 43, 66, 218, 11, 75, 206, 20, 86, 246, 25, 78, 143, 128, 1, + 85, 16, 82, 82, 198, 177, 9, 73, 222, 25, 68, 162, 8, 71, 2, 87, 198, 21, + 83, 147, 1, 72, 4, 166, 222, 8, 82, 239, 139, 1, 69, 2, 199, 199, 9, 82, + 8, 38, 65, 230, 176, 9, 82, 255, 55, 71, 4, 178, 233, 9, 82, 3, 84, 4, + 216, 241, 7, 2, 65, 45, 211, 166, 1, 85, 6, 150, 153, 9, 65, 130, 57, 67, + 215, 22, 70, 2, 143, 228, 7, 79, 4, 178, 136, 5, 69, 167, 200, 3, 73, 12, + 70, 79, 166, 222, 8, 73, 242, 108, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, + 235, 208, 9, 90, 2, 163, 13, 69, 2, 243, 196, 8, 79, 4, 230, 198, 8, 85, + 163, 160, 1, 79, 6, 26, 72, 159, 202, 9, 79, 4, 186, 238, 5, 73, 199, + 230, 3, 69, 4, 134, 221, 8, 79, 159, 137, 1, 69, 4, 254, 229, 9, 65, 3, + 87, 10, 78, 69, 222, 214, 3, 70, 192, 121, 6, 78, 84, 79, 32, 83, 72, + 251, 148, 5, 80, 2, 255, 158, 9, 76, 44, 236, 1, 2, 79, 84, 32, 6, 80, + 80, 73, 78, 71, 32, 72, 2, 82, 84, 180, 8, 7, 85, 76, 68, 69, 82, 69, 68, + 208, 240, 1, 24, 67, 75, 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, + 32, 69, 88, 80, 76, 79, 68, 73, 78, 71, 199, 171, 7, 87, 2, 145, 208, 7, + 3, 73, 78, 71, 4, 36, 3, 84, 82, 79, 239, 220, 5, 66, 2, 11, 76, 2, 247, + 203, 7, 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, + 65, 84, 32, 206, 149, 6, 67, 255, 197, 3, 83, 20, 154, 2, 66, 92, 11, 83, + 76, 65, 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, + 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, + 7, 85, 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 133, 160, 2, 9, + 68, 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, + 78, 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 255, + 195, 8, 72, 2, 213, 37, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, + 32, 78, 79, 82, 84, 72, 32, 69, 2, 189, 237, 5, 2, 32, 76, 7, 11, 32, 4, + 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 191, 131, 2, + 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 209, 6, 84, 8, + 120, 10, 67, 79, 78, 84, 73, 78, 85, 73, 78, 71, 0, 6, 76, 69, 84, 84, + 69, 82, 44, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 201, 8, 6, 32, 79, + 86, 69, 82, 76, 2, 21, 3, 32, 83, 84, 2, 251, 217, 9, 69, 2, 25, 4, 32, + 79, 80, 69, 2, 167, 205, 8, 78, 4, 26, 73, 215, 216, 9, 85, 2, 155, 217, + 9, 77, 159, 14, 174, 1, 68, 132, 20, 2, 71, 78, 232, 181, 1, 6, 77, 73, + 76, 65, 82, 32, 158, 2, 78, 198, 24, 88, 225, 171, 6, 15, 76, 72, 79, 85, + 69, 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, + 77, 32, 225, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 194, 1, 68, 106, 69, + 68, 7, 76, 69, 84, 84, 69, 82, 32, 172, 4, 15, 82, 69, 80, 69, 84, 73, + 84, 73, 79, 78, 32, 77, 65, 82, 75, 50, 83, 165, 8, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 4, 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, + 25, 4, 76, 69, 32, 68, 2, 11, 65, 2, 247, 175, 9, 78, 2, 37, 7, 78, 68, + 32, 79, 70, 32, 84, 2, 129, 138, 9, 2, 69, 88, 102, 210, 1, 65, 98, 84, + 198, 233, 5, 68, 90, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, + 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, + 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, + 84, 69, 134, 210, 9, 65, 2, 73, 3, 85, 2, 155, 173, 9, 32, 14, 134, 1, + 72, 196, 212, 5, 19, 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, + 69, 82, 78, 65, 84, 69, 242, 180, 3, 84, 183, 71, 65, 4, 224, 144, 9, 20, + 82, 69, 69, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, + 84, 69, 135, 64, 65, 6, 11, 45, 6, 234, 207, 9, 49, 2, 50, 3, 51, 44, 38, + 69, 181, 7, 4, 73, 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, + 65, 82, 75, 32, 177, 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, + 68, 79, 85, 66, 76, 69, 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, + 11, 32, 2, 225, 162, 8, 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, + 73, 82, 67, 76, 69, 83, 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, + 0, 4, 83, 69, 80, 84, 12, 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, + 84, 84, 69, 68, 32, 46, 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, + 0, 3, 84, 87, 79, 163, 160, 8, 82, 2, 169, 160, 6, 8, 32, 69, 78, 67, 76, + 79, 83, 85, 2, 83, 85, 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, + 1, 3, 79, 85, 66, 2, 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, + 68, 32, 103, 80, 4, 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, + 72, 65, 80, 69, 68, 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, + 32, 67, 2, 25, 4, 82, 69, 83, 67, 2, 151, 186, 1, 69, 4, 226, 165, 8, 66, + 255, 81, 68, 12, 246, 208, 4, 83, 206, 144, 1, 67, 98, 78, 222, 160, 3, + 65, 239, 1, 86, 26, 74, 65, 94, 86, 226, 246, 5, 85, 210, 200, 1, 73, + 206, 134, 2, 69, 3, 79, 10, 184, 247, 5, 10, 76, 84, 69, 82, 78, 65, 84, + 69, 32, 85, 158, 207, 3, 65, 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, + 76, 73, 67, 32, 4, 167, 226, 5, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, + 87, 72, 73, 84, 69, 8, 11, 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, + 32, 80, 79, 73, 78, 84, 194, 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, + 217, 238, 1, 10, 32, 79, 70, 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, + 4, 65, 73, 82, 32, 192, 1, 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, + 242, 6, 70, 244, 3, 4, 87, 65, 76, 76, 138, 1, 72, 238, 77, 76, 224, 6, + 2, 77, 79, 222, 42, 78, 230, 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, + 71, 82, 65, 83, 80, 184, 5, 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, + 32, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, + 74, 79, 211, 179, 4, 80, 8, 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, + 4, 58, 32, 195, 207, 4, 73, 4, 30, 32, 61, 3, 73, 78, 71, 2, 157, 158, 1, + 10, 83, 77, 65, 76, 76, 32, 82, 79, 84, 65, 2, 251, 190, 8, 32, 10, 52, + 5, 69, 65, 84, 72, 32, 149, 170, 1, 2, 85, 83, 4, 152, 163, 2, 2, 69, 88, + 1, 2, 73, 78, 10, 40, 6, 72, 69, 69, 75, 83, 32, 63, 79, 6, 170, 107, 83, + 146, 38, 78, 153, 204, 3, 4, 80, 85, 70, 70, 4, 246, 236, 8, 76, 211, 46, + 77, 28, 108, 15, 82, 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, + 32, 165, 1, 7, 89, 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, + 2, 85, 80, 29, 4, 78, 69, 85, 84, 2, 169, 143, 1, 2, 32, 78, 4, 21, 3, + 82, 65, 76, 4, 11, 32, 4, 194, 58, 68, 247, 255, 8, 85, 20, 180, 1, 11, + 69, 86, 69, 82, 89, 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 144, 122, 9, + 65, 82, 82, 79, 87, 72, 69, 65, 68, 138, 39, 82, 232, 186, 3, 2, 84, 69, + 21, 4, 71, 82, 65, 68, 2, 137, 161, 8, 2, 32, 84, 2, 135, 241, 5, 65, 6, + 68, 11, 73, 77, 85, 76, 84, 65, 78, 69, 79, 85, 83, 235, 157, 8, 76, 5, + 251, 151, 1, 32, 54, 64, 2, 89, 69, 188, 208, 4, 4, 88, 67, 73, 84, 167, + 216, 3, 65, 50, 166, 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, + 65, 73, 71, 72, 84, 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, + 72, 69, 83, 32, 65, 2, 83, 32, 6, 140, 151, 1, 5, 66, 76, 73, 78, 75, + 131, 224, 4, 87, 6, 186, 53, 68, 170, 84, 78, 207, 171, 8, 85, 18, 100, + 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, + 80, 76, 65, 78, 69, 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, + 254, 45, 85, 241, 95, 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, + 84, 7, 11, 32, 4, 250, 163, 1, 65, 55, 68, 6, 130, 51, 68, 196, 154, 4, + 4, 70, 76, 85, 84, 179, 229, 4, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, + 67, 28, 4, 87, 73, 68, 69, 246, 92, 79, 233, 244, 3, 6, 83, 81, 85, 69, + 69, 90, 4, 22, 67, 147, 93, 79, 2, 205, 229, 2, 2, 76, 79, 4, 162, 67, + 32, 233, 61, 4, 78, 73, 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, + 82, 69, 67, 84, 73, 79, 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, + 79, 83, 69, 32, 138, 1, 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, + 68, 32, 163, 225, 6, 85, 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, + 78, 13, 8, 70, 79, 82, 87, 65, 82, 68, 32, 5, 11, 32, 2, 169, 212, 4, 3, + 84, 73, 76, 12, 248, 210, 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, + 82, 171, 163, 2, 78, 12, 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, + 174, 141, 1, 76, 35, 83, 2, 169, 176, 8, 20, 80, 76, 65, 78, 69, 32, 83, + 72, 79, 85, 76, 68, 69, 82, 32, 72, 73, 80, 32, 77, 6, 182, 89, 87, 222, + 38, 67, 47, 78, 156, 4, 34, 65, 137, 75, 3, 69, 65, 68, 140, 4, 36, 3, + 78, 68, 45, 211, 242, 8, 73, 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, + 67, 150, 10, 70, 186, 42, 72, 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, + 188, 1, 5, 73, 78, 68, 69, 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, + 136, 1, 5, 82, 73, 78, 71, 32, 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, + 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, + 77, 73, 68, 68, 76, 69, 32, 228, 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, + 84, 76, 69, 241, 42, 5, 84, 72, 85, 77, 66, 4, 178, 70, 76, 183, 144, 8, + 82, 8, 44, 5, 73, 78, 68, 69, 88, 135, 167, 9, 85, 7, 145, 24, 18, 32, + 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, + 108, 22, 68, 79, 87, 78, 32, 77, 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, + 66, 32, 73, 78, 68, 69, 159, 68, 76, 2, 207, 231, 7, 88, 88, 64, 5, 73, + 82, 67, 76, 69, 184, 3, 3, 76, 65, 87, 183, 2, 85, 35, 11, 32, 32, 116, + 5, 73, 78, 68, 69, 88, 200, 1, 7, 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, + 73, 68, 68, 76, 69, 32, 167, 64, 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, + 68, 76, 69, 198, 26, 72, 246, 38, 82, 195, 158, 8, 66, 13, 11, 32, 10, + 64, 5, 67, 82, 79, 83, 83, 194, 64, 84, 90, 76, 183, 144, 8, 82, 4, 138, + 65, 32, 167, 155, 8, 69, 4, 170, 253, 7, 73, 239, 164, 1, 85, 6, 252, 47, + 10, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 247, 241, 8, 85, 17, 11, 32, + 14, 144, 2, 28, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, + 84, 84, 76, 69, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, + 79, 194, 11, 78, 170, 1, 84, 237, 118, 23, 73, 78, 68, 69, 88, 32, 84, + 72, 85, 77, 66, 32, 67, 85, 82, 86, 69, 32, 84, 72, 85, 77, 66, 5, 163, + 181, 1, 32, 38, 46, 80, 149, 2, 6, 82, 76, 73, 67, 85, 69, 31, 11, 32, + 28, 188, 1, 5, 73, 78, 68, 69, 88, 56, 19, 70, 73, 86, 69, 32, 70, 73, + 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 196, 50, 7, 77, 73, 68, + 68, 76, 69, 32, 18, 79, 214, 7, 78, 171, 1, 84, 9, 11, 32, 6, 40, 5, 84, + 72, 85, 77, 66, 247, 58, 82, 5, 215, 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, + 68, 69, 88, 0, 6, 77, 73, 68, 68, 76, 69, 195, 71, 79, 2, 133, 204, 8, + 13, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 172, 2, 44, 3, + 73, 83, 84, 177, 33, 3, 76, 65, 84, 247, 1, 11, 32, 244, 1, 160, 2, 5, + 73, 78, 68, 69, 88, 192, 16, 7, 76, 73, 84, 84, 76, 69, 32, 196, 2, 7, + 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, 82, 73, 78, 71, 32, 132, 2, 5, 84, + 72, 85, 77, 66, 138, 2, 72, 205, 9, 22, 70, 79, 85, 82, 32, 70, 73, 78, + 71, 69, 82, 83, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 137, 1, 11, 32, + 134, 1, 232, 1, 4, 66, 69, 78, 84, 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, + 77, 73, 68, 68, 76, 69, 168, 7, 2, 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, + 160, 5, 16, 85, 80, 32, 77, 73, 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, + 68, 151, 4, 82, 5, 217, 45, 5, 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, + 8, 77, 73, 68, 68, 76, 69, 32, 85, 251, 230, 7, 76, 71, 11, 32, 68, 236, + 1, 4, 66, 69, 78, 84, 42, 67, 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, + 68, 32, 100, 6, 72, 73, 78, 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, + 66, 188, 28, 14, 83, 84, 82, 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, + 231, 17, 76, 5, 229, 92, 6, 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, + 74, 79, 73, 78, 69, 68, 233, 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, + 1, 6, 67, 85, 80, 80, 69, 68, 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, + 73, 78, 71, 69, 166, 22, 73, 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, + 32, 2, 203, 27, 84, 8, 160, 1, 4, 83, 73, 68, 69, 235, 61, 70, 6, 22, 69, + 163, 47, 32, 4, 223, 15, 68, 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, + 8, 32, 3, 73, 78, 71, 247, 19, 65, 7, 11, 32, 4, 138, 36, 67, 199, 168, + 8, 66, 19, 11, 32, 16, 64, 6, 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, + 231, 202, 8, 66, 5, 207, 188, 5, 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, + 79, 78, 74, 79, 73, 78, 69, 68, 32, 72, 2, 217, 174, 4, 2, 80, 80, 4, + 194, 33, 73, 233, 26, 2, 79, 79, 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, + 32, 46, 67, 220, 1, 14, 70, 79, 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, + 88, 32, 32, 4, 72, 79, 79, 75, 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, + 78, 1, 3, 79, 85, 84, 12, 36, 5, 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, + 8, 32, 4, 80, 80, 69, 68, 39, 82, 2, 221, 40, 5, 32, 77, 73, 68, 68, 6, + 56, 9, 86, 69, 32, 84, 72, 85, 77, 66, 32, 143, 37, 76, 4, 206, 160, 1, + 73, 219, 164, 4, 85, 4, 218, 84, 83, 183, 242, 7, 66, 7, 157, 8, 9, 69, + 68, 32, 77, 73, 68, 68, 76, 69, 15, 11, 32, 12, 92, 6, 73, 78, 68, 69, + 88, 32, 156, 13, 5, 84, 72, 85, 77, 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, + 72, 183, 197, 8, 66, 2, 207, 212, 7, 73, 7, 37, 7, 32, 84, 72, 85, 77, + 66, 32, 4, 178, 28, 67, 251, 129, 1, 83, 22, 88, 4, 68, 79, 87, 78, 186, + 1, 84, 158, 6, 82, 130, 21, 73, 170, 167, 8, 66, 147, 67, 85, 9, 11, 32, + 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, 67, 73, 25, 7, 82, 73, 80, 80, 76, + 69, 32, 2, 241, 51, 2, 82, 67, 4, 22, 67, 187, 80, 83, 2, 11, 85, 2, 185, + 166, 4, 2, 82, 86, 4, 192, 35, 6, 79, 85, 67, 72, 69, 83, 39, 72, 30, + 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, 32, 138, 3, 85, 134, 1, 68, 214, + 30, 76, 161, 188, 7, 9, 66, 69, 78, 84, 32, 79, 86, 69, 82, 4, 238, 4, + 65, 235, 29, 73, 16, 80, 7, 65, 78, 71, 76, 69, 68, 32, 126, 67, 124, 4, + 72, 79, 79, 75, 151, 32, 76, 6, 60, 10, 79, 85, 84, 32, 73, 78, 68, 69, + 88, 32, 215, 1, 73, 4, 26, 67, 211, 130, 9, 85, 2, 241, 182, 2, 3, 82, + 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, 32, 73, 78, 68, 69, 88, 45, 3, + 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, 191, 236, 8, 85, 2, 25, 4, 69, 68, + 32, 73, 2, 229, 30, 4, 78, 68, 69, 88, 4, 11, 80, 5, 175, 15, 32, 18, + 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, 42, 82, 194, 29, 84, 90, 76, 182, + 188, 7, 73, 239, 164, 1, 85, 2, 223, 174, 7, 79, 7, 11, 32, 4, 206, 3, + 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, 73, 83, 69, 68, 32, 75, 78, 2, 11, + 85, 2, 11, 67, 2, 211, 221, 7, 75, 35, 11, 32, 32, 148, 1, 8, 66, 69, 84, + 87, 69, 69, 78, 32, 102, 72, 20, 5, 79, 86, 69, 82, 32, 120, 4, 83, 73, + 68, 69, 100, 6, 85, 78, 68, 69, 82, 32, 223, 40, 70, 8, 80, 12, 73, 78, + 68, 69, 88, 32, 77, 73, 68, 68, 76, 69, 246, 20, 77, 159, 6, 82, 5, 151, + 70, 32, 2, 131, 156, 4, 69, 4, 52, 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, + 84, 87, 2, 11, 65, 2, 133, 81, 9, 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, + 11, 32, 6, 38, 68, 190, 15, 67, 199, 168, 8, 66, 2, 25, 4, 73, 65, 71, + 79, 2, 215, 241, 7, 78, 10, 54, 73, 34, 84, 48, 4, 70, 79, 85, 82, 255, + 22, 76, 2, 161, 7, 4, 78, 68, 69, 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, + 2, 11, 79, 2, 153, 235, 7, 5, 32, 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, + 70, 172, 3, 4, 72, 69, 69, 76, 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, + 6, 84, 72, 85, 77, 66, 32, 153, 231, 4, 16, 66, 69, 84, 87, 69, 69, 78, + 32, 80, 65, 76, 77, 32, 70, 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, + 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, + 82, 32, 70, 73, 78, 71, 69, 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, + 71, 69, 68, 42, 84, 186, 2, 70, 143, 176, 8, 66, 7, 11, 32, 4, 190, 4, + 84, 151, 15, 78, 2, 205, 35, 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, + 72, 9, 67, 79, 78, 74, 79, 73, 78, 69, 68, 154, 8, 72, 175, 169, 8, 66, + 5, 153, 216, 8, 3, 32, 83, 80, 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, + 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, + 11, 32, 4, 26, 70, 143, 176, 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, + 10, 72, 6, 67, 69, 78, 84, 82, 69, 96, 5, 73, 78, 68, 69, 88, 171, 16, + 76, 7, 49, 10, 32, 84, 72, 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, + 32, 2, 203, 174, 8, 66, 2, 11, 32, 2, 11, 84, 2, 197, 135, 1, 5, 72, 85, + 77, 66, 32, 6, 130, 31, 70, 170, 104, 83, 207, 166, 7, 66, 82, 48, 4, 73, + 78, 71, 69, 181, 9, 3, 79, 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, + 73, 78, 68, 69, 88, 196, 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, + 68, 76, 69, 30, 79, 64, 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, + 32, 154, 6, 78, 215, 128, 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, + 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 32, 19, 79, 2, 207, 25, 79, + 2, 249, 1, 11, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, + 20, 86, 72, 40, 7, 77, 73, 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, + 66, 223, 9, 82, 2, 11, 73, 2, 249, 140, 4, 2, 78, 71, 6, 36, 4, 82, 73, + 78, 71, 207, 10, 76, 5, 11, 32, 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 191, + 33, 79, 11, 11, 32, 8, 34, 83, 226, 22, 79, 203, 39, 76, 4, 146, 222, 6, + 73, 187, 8, 77, 9, 11, 32, 6, 22, 73, 195, 8, 84, 4, 25, 4, 78, 68, 69, + 88, 5, 151, 8, 32, 5, 11, 32, 2, 175, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, + 32, 6, 174, 7, 78, 171, 1, 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, + 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 72, 79, 79, 75, 2, 205, 77, 2, + 32, 77, 6, 68, 9, 66, 69, 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, + 69, 2, 29, 5, 73, 68, 68, 76, 69, 2, 235, 134, 4, 32, 5, 33, 6, 32, 84, + 79, 85, 67, 72, 2, 145, 194, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, + 67, 85, 82, 76, 28, 18, 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, + 73, 84, 84, 76, 69, 32, 48, 7, 77, 73, 68, 68, 76, 69, 32, 221, 2, 4, 82, + 73, 78, 71, 2, 229, 132, 5, 2, 73, 67, 6, 254, 159, 5, 85, 234, 203, 2, + 79, 239, 41, 73, 8, 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, + 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 139, 2, 84, 7, 231, 213, 2, 32, + 17, 11, 32, 14, 130, 1, 76, 66, 78, 78, 82, 94, 84, 136, 154, 2, 15, 70, + 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 131, 162, 5, 73, + 2, 21, 3, 73, 84, 84, 2, 17, 2, 76, 69, 2, 215, 236, 7, 32, 2, 11, 79, 2, + 11, 32, 2, 11, 84, 2, 11, 72, 2, 209, 167, 7, 2, 85, 77, 2, 11, 73, 2, + 21, 3, 78, 71, 32, 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 239, 191, 7, 84, + 4, 29, 5, 72, 85, 77, 66, 32, 4, 206, 14, 70, 171, 104, 83, 17, 11, 32, + 14, 56, 8, 77, 79, 86, 69, 77, 69, 78, 84, 199, 221, 7, 82, 12, 26, 45, + 207, 144, 7, 32, 10, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, + 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 235, 40, 83, + 6, 38, 67, 28, 2, 84, 73, 207, 40, 83, 2, 241, 225, 7, 2, 85, 82, 2, 239, + 193, 8, 76, 38, 50, 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, + 77, 66, 32, 171, 1, 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, + 84, 72, 2, 189, 176, 5, 2, 73, 78, 14, 11, 45, 14, 202, 220, 8, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, + 82, 69, 83, 83, 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, + 82, 32, 85, 213, 193, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, + 2, 11, 80, 2, 231, 156, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, + 2, 167, 156, 8, 72, 16, 22, 32, 215, 1, 45, 12, 140, 1, 2, 72, 69, 48, 3, + 84, 79, 82, 152, 217, 6, 3, 68, 69, 80, 0, 3, 87, 73, 68, 241, 106, 10, + 76, 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 216, 13, 4, 65, 68, 32, 78, + 175, 160, 8, 73, 2, 167, 185, 8, 83, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, + 87, 65, 76, 76, 2, 149, 153, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, + 84, 72, 32, 161, 5, 7, 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, + 70, 82, 79, 87, 78, 0, 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, + 5, 79, 80, 69, 78, 32, 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, + 73, 78, 75, 8, 48, 6, 76, 79, 83, 69, 68, 32, 243, 166, 7, 79, 6, 222, 2, + 70, 142, 38, 67, 47, 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 179, 131, + 7, 80, 7, 11, 32, 4, 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, + 9, 82, 69, 67, 84, 65, 78, 71, 76, 69, 42, 87, 82, 70, 139, 131, 7, 67, + 7, 11, 32, 4, 26, 87, 183, 129, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 131, + 243, 3, 76, 7, 11, 32, 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 155, 187, + 7, 87, 2, 17, 2, 85, 67, 2, 147, 242, 3, 75, 230, 2, 192, 1, 9, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, + 78, 69, 32, 184, 14, 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, + 22, 65, 31, 84, 16, 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, + 28, 6, 79, 87, 65, 82, 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, + 73, 82, 67, 76, 69, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, + 32, 38, 66, 34, 67, 224, 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, + 85, 77, 80, 32, 184, 3, 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, + 82, 73, 80, 76, 69, 32, 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, + 159, 175, 7, 74, 12, 178, 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 243, + 172, 7, 69, 28, 86, 72, 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, + 78, 69, 82, 195, 239, 6, 82, 2, 163, 156, 7, 69, 18, 80, 4, 67, 79, 77, + 66, 158, 8, 72, 230, 15, 76, 198, 157, 2, 77, 243, 178, 1, 83, 2, 11, 73, + 2, 187, 234, 3, 78, 6, 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, + 18, 56, 8, 72, 73, 84, 84, 73, 78, 71, 32, 167, 231, 3, 83, 16, 72, 8, + 67, 69, 73, 76, 73, 78, 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, + 76, 65, 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 238, 54, 84, + 135, 2, 68, 8, 88, 4, 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, + 84, 82, 73, 80, 76, 69, 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, + 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 2, 157, 53, 2, 32, 84, 18, 56, 8, + 72, 73, 84, 84, 73, 78, 71, 32, 239, 227, 3, 83, 16, 64, 7, 67, 69, 73, + 76, 73, 78, 71, 1, 5, 70, 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, + 83, 4, 249, 22, 4, 65, 82, 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, + 243, 16, 73, 2, 21, 3, 32, 80, 65, 2, 149, 227, 3, 4, 82, 65, 76, 76, 8, + 76, 12, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, + 83, 4, 134, 18, 87, 159, 21, 77, 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, + 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, + 76, 8, 37, 7, 73, 84, 84, 73, 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, + 25, 5, 70, 76, 79, 79, 82, 4, 138, 251, 4, 78, 195, 193, 1, 77, 4, 209, + 5, 10, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, + 184, 1, 6, 68, 79, 87, 78, 32, 83, 245, 183, 5, 10, 83, 73, 68, 69, 32, + 84, 79, 32, 83, 73, 10, 40, 5, 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, + 65, 76, 84, 69, 82, 78, 65, 84, 230, 17, 76, 143, 203, 3, 83, 4, 21, 3, + 73, 78, 71, 4, 11, 32, 4, 190, 17, 76, 143, 203, 3, 83, 2, 219, 30, 69, + 162, 1, 148, 2, 11, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, + 54, 67, 174, 4, 68, 100, 8, 70, 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, + 85, 77, 80, 32, 60, 5, 76, 79, 79, 80, 32, 102, 80, 34, 83, 216, 1, 7, + 84, 82, 73, 80, 76, 69, 32, 218, 1, 87, 202, 2, 90, 159, 175, 7, 74, 8, + 18, 77, 39, 83, 4, 225, 13, 5, 69, 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, + 3, 65, 76, 76, 12, 34, 79, 185, 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, + 100, 6, 79, 82, 78, 69, 82, 32, 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, + 69, 67, 75, 195, 239, 6, 82, 8, 54, 82, 194, 12, 76, 158, 152, 2, 77, + 243, 178, 1, 83, 2, 11, 79, 2, 223, 140, 5, 84, 30, 50, 32, 137, 2, 7, + 68, 32, 67, 82, 79, 83, 83, 26, 66, 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, + 65, 82, 84, 69, 82, 12, 196, 5, 10, 65, 76, 70, 45, 67, 73, 82, 67, 76, + 69, 159, 14, 73, 6, 96, 2, 69, 78, 29, 18, 82, 69, 69, 45, 81, 85, 65, + 82, 84, 69, 82, 32, 67, 73, 82, 67, 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, + 11, 32, 4, 242, 161, 2, 77, 243, 178, 1, 83, 8, 33, 6, 79, 85, 66, 76, + 69, 32, 8, 30, 83, 150, 4, 65, 75, 87, 2, 253, 138, 8, 3, 84, 82, 65, 6, + 32, 3, 73, 82, 67, 151, 9, 79, 4, 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, + 178, 8, 72, 238, 143, 2, 77, 243, 178, 1, 83, 12, 68, 5, 83, 77, 65, 76, + 76, 142, 7, 76, 178, 8, 72, 239, 143, 2, 77, 5, 11, 32, 2, 239, 36, 68, + 6, 181, 6, 4, 69, 65, 75, 83, 12, 22, 73, 243, 36, 72, 10, 29, 5, 78, 71, + 76, 69, 32, 10, 52, 8, 83, 84, 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, + 32, 8, 42, 76, 198, 157, 2, 77, 243, 178, 1, 83, 4, 25, 4, 65, 82, 71, + 69, 5, 191, 149, 8, 83, 8, 26, 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, + 82, 78, 65, 84, 73, 78, 71, 5, 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, + 32, 2, 237, 139, 7, 2, 70, 76, 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, + 171, 20, 32, 26, 104, 4, 65, 86, 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, + 67, 73, 82, 67, 76, 69, 32, 70, 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, + 86, 69, 32, 140, 1, 13, 68, 73, 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, + 72, 235, 8, 72, 12, 48, 4, 68, 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, + 2, 76, 69, 4, 11, 32, 4, 222, 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, + 71, 6, 11, 32, 6, 42, 76, 158, 152, 2, 77, 243, 178, 1, 83, 2, 143, 240, + 6, 65, 10, 40, 4, 79, 83, 69, 32, 215, 253, 6, 69, 8, 26, 67, 46, 78, 35, + 87, 2, 11, 79, 2, 157, 157, 7, 3, 78, 84, 65, 2, 161, 162, 7, 3, 69, 85, + 84, 4, 36, 2, 73, 71, 21, 3, 82, 73, 78, 2, 191, 130, 5, 71, 2, 171, 130, + 5, 75, 72, 56, 7, 79, 84, 65, 84, 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, + 10, 32, 77, 79, 68, 73, 70, 73, 69, 82, 45, 155, 1, 45, 30, 82, 49, 154, + 169, 8, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 150, + 169, 8, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 36, 104, 11, 70, + 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 133, 2, 10, 87, 65, 76, 76, 80, + 76, 65, 78, 69, 32, 18, 100, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, + 11, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, + 9, 32, 72, 73, 84, 84, 73, 78, 71, 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, + 2, 11, 73, 2, 147, 213, 7, 76, 2, 155, 237, 7, 79, 18, 88, 8, 65, 76, 84, + 69, 82, 78, 65, 84, 44, 4, 68, 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, + 4, 73, 78, 71, 32, 191, 164, 8, 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, + 72, 4, 11, 73, 4, 33, 6, 84, 84, 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, + 70, 82, 79, 78, 84, 2, 135, 220, 4, 69, 2, 253, 158, 6, 2, 32, 87, 30, + 172, 1, 8, 72, 79, 85, 76, 68, 69, 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, + 69, 32, 240, 1, 7, 85, 82, 70, 65, 67, 69, 32, 240, 10, 5, 84, 82, 73, + 75, 69, 243, 221, 4, 69, 6, 100, 4, 72, 73, 80, 32, 173, 232, 3, 15, 84, + 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, + 83, 73, 181, 135, 7, 2, 83, 80, 2, 11, 84, 2, 177, 242, 4, 2, 73, 79, 12, + 48, 6, 70, 76, 73, 67, 75, 32, 18, 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, + 4, 65, 82, 71, 69, 6, 34, 69, 65, 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, + 21, 3, 69, 78, 84, 2, 207, 149, 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, + 83, 4, 22, 83, 135, 11, 66, 2, 201, 204, 1, 4, 89, 77, 66, 79, 82, 58, + 69, 218, 2, 79, 137, 8, 6, 82, 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, + 185, 1, 11, 78, 83, 69, 32, 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, + 56, 3, 79, 78, 32, 86, 77, 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, + 80, 83, 1, 6, 84, 79, 78, 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 245, 180, + 3, 2, 79, 86, 6, 50, 77, 252, 162, 4, 2, 72, 73, 131, 222, 2, 76, 2, 157, + 149, 3, 2, 73, 68, 28, 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, + 45, 129, 2, 2, 85, 67, 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, + 73, 78, 83, 73, 68, 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, + 80, 32, 88, 7, 76, 73, 67, 75, 73, 78, 71, 128, 244, 6, 14, 83, 84, 73, + 67, 75, 73, 78, 71, 32, 79, 85, 84, 32, 70, 133, 159, 1, 17, 77, 79, 86, + 69, 83, 32, 65, 71, 65, 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, + 193, 166, 3, 5, 83, 84, 73, 67, 75, 2, 225, 183, 3, 4, 69, 76, 65, 88, 4, + 84, 7, 66, 69, 84, 87, 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, + 71, 32, 73, 2, 11, 32, 2, 189, 246, 4, 2, 76, 73, 2, 237, 167, 3, 5, 78, + 83, 73, 68, 69, 6, 120, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 169, + 186, 3, 14, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, + 108, 8, 67, 85, 82, 86, 69, 68, 32, 66, 241, 200, 1, 13, 83, 84, 82, 65, + 73, 71, 72, 84, 32, 83, 84, 82, 69, 2, 139, 134, 7, 69, 6, 11, 72, 6, 11, + 32, 6, 30, 66, 34, 77, 187, 4, 83, 2, 157, 194, 6, 3, 69, 84, 87, 2, 149, + 2, 3, 85, 76, 84, 34, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, + 32, 29, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, + 2, 83, 20, 72, 11, 65, 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, + 151, 2, 83, 6, 30, 84, 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 183, + 239, 6, 80, 12, 41, 8, 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, + 79, 79, 82, 1, 4, 87, 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, + 26, 65, 54, 68, 43, 83, 2, 29, 5, 76, 84, 69, 82, 78, 2, 247, 180, 3, 65, + 2, 17, 2, 79, 85, 2, 183, 237, 6, 66, 2, 139, 237, 6, 73, 2, 11, 72, 2, + 247, 178, 3, 65, 2, 11, 73, 2, 243, 249, 6, 78, 10, 100, 6, 65, 66, 79, + 86, 69, 32, 162, 1, 79, 197, 133, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, + 77, 73, 4, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, + 37, 7, 45, 84, 72, 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 157, 228, + 1, 2, 32, 69, 4, 203, 252, 5, 82, 250, 1, 64, 3, 71, 76, 69, 212, 4, 5, + 72, 65, 76, 65, 32, 135, 102, 69, 20, 50, 32, 221, 137, 8, 6, 45, 83, 72, + 73, 70, 84, 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, + 116, 2, 72, 73, 74, 76, 36, 4, 82, 73, 71, 72, 145, 208, 4, 4, 83, 72, + 73, 70, 2, 41, 8, 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, + 84, 82, 79, 68, 2, 11, 85, 2, 143, 203, 7, 67, 2, 25, 4, 71, 72, 45, 82, + 2, 185, 1, 7, 69, 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, + 87, 2, 49, 10, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, + 78, 2, 17, 2, 71, 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, + 2, 205, 161, 2, 2, 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, + 32, 224, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, + 32, 114, 76, 188, 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 229, 212, 2, 17, 80, 85, 78, 67, 84, 85, 65, + 84, 73, 79, 78, 32, 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, + 66, 69, 82, 32, 18, 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 146, 150, 6, + 70, 30, 83, 42, 84, 190, 83, 78, 14, 79, 199, 110, 69, 22, 150, 218, 2, + 79, 134, 189, 3, 69, 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, + 4, 56, 6, 65, 75, 65, 65, 82, 65, 245, 254, 7, 2, 69, 80, 2, 241, 254, 7, + 3, 65, 78, 83, 138, 1, 60, 6, 69, 84, 84, 69, 82, 32, 217, 171, 2, 3, 73, + 84, 72, 118, 246, 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, + 65, 78, 84, 58, 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, + 2, 82, 2, 86, 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, + 65, 78, 84, 65, 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, + 5, 77, 66, 65, 32, 66, 14, 65, 2, 73, 2, 85, 175, 216, 4, 89, 4, 230, 3, + 69, 175, 216, 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 175, 216, 4, 89, 4, + 11, 85, 4, 138, 3, 85, 175, 216, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, + 82, 68, 72, 22, 32, 2, 72, 65, 203, 218, 4, 89, 20, 41, 8, 65, 80, 82, + 65, 65, 78, 65, 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, + 2, 75, 3, 80, 4, 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, + 76, 2, 78, 3, 83, 4, 86, 79, 175, 216, 4, 89, 8, 26, 68, 22, 71, 3, 74, + 4, 18, 68, 15, 65, 2, 11, 65, 2, 171, 216, 4, 89, 6, 26, 78, 21, 2, 83, + 65, 2, 89, 2, 65, 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, + 75, 163, 215, 4, 89, 2, 185, 246, 7, 4, 83, 73, 75, 89, 8, 66, 65, 150, + 146, 4, 67, 193, 227, 3, 6, 86, 73, 83, 65, 82, 71, 4, 220, 113, 5, 76, + 45, 76, 65, 75, 245, 131, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, + 73, 71, 65, 32, 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, + 77, 66, 118, 65, 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, + 89, 2, 11, 85, 2, 131, 244, 7, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, + 79, 77, 66, 85, 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, + 1, 83, 2, 175, 1, 65, 10, 40, 2, 86, 65, 173, 198, 7, 2, 32, 68, 9, 29, + 5, 32, 72, 65, 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, + 71, 65, 89, 2, 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, + 183, 209, 7, 76, 2, 193, 202, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, + 80, 206, 2, 45, 137, 223, 5, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, + 8, 140, 1, 23, 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, + 65, 78, 68, 32, 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, + 25, 4, 32, 70, 76, 79, 2, 207, 172, 6, 82, 6, 78, 80, 150, 221, 5, 66, + 217, 129, 1, 9, 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, + 2, 149, 221, 5, 4, 72, 69, 69, 76, 2, 179, 194, 3, 80, 12, 46, 73, 102, + 85, 213, 215, 6, 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, + 163, 182, 7, 69, 2, 17, 2, 32, 66, 2, 175, 158, 7, 79, 6, 32, 2, 76, 76, + 143, 235, 7, 78, 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, + 83, 66, 2, 11, 79, 2, 247, 217, 6, 78, 42, 46, 65, 198, 4, 69, 210, 1, + 73, 155, 1, 79, 14, 64, 5, 78, 84, 69, 68, 32, 253, 219, 5, 5, 86, 79, + 78, 73, 67, 12, 148, 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, + 32, 229, 1, 19, 78, 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, + 84, 72, 32, 72, 79, 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, + 83, 83, 4, 29, 5, 45, 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, + 72, 2, 25, 4, 32, 68, 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, + 243, 220, 5, 73, 4, 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 135, 155, + 5, 32, 2, 29, 5, 73, 90, 79, 78, 84, 2, 11, 65, 2, 203, 187, 5, 76, 12, + 80, 2, 69, 80, 180, 214, 7, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, + 17, 68, 8, 40, 4, 73, 78, 71, 32, 191, 170, 7, 89, 6, 152, 187, 4, 8, 65, + 67, 67, 79, 77, 77, 79, 68, 190, 153, 2, 83, 255, 85, 70, 6, 76, 9, 67, + 69, 32, 79, 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 247, + 227, 7, 90, 4, 196, 162, 6, 2, 70, 82, 145, 16, 3, 83, 77, 73, 10, 18, + 80, 75, 84, 6, 140, 186, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 199, + 171, 3, 69, 4, 26, 32, 243, 228, 7, 72, 2, 11, 77, 2, 193, 203, 6, 3, 65, + 67, 72, 148, 1, 34, 65, 210, 12, 73, 159, 8, 79, 114, 32, 2, 76, 76, 227, + 212, 6, 83, 112, 30, 32, 193, 11, 2, 69, 82, 108, 178, 2, 65, 44, 3, 66, + 76, 85, 0, 5, 79, 82, 65, 78, 71, 20, 2, 67, 79, 158, 1, 68, 22, 69, 140, + 2, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 28, 2, 76, 69, 34, 82, + 218, 3, 83, 22, 84, 36, 4, 86, 69, 69, 32, 208, 238, 1, 3, 71, 82, 69, + 34, 72, 138, 4, 80, 154, 79, 78, 202, 203, 2, 70, 83, 81, 6, 226, 167, 4, + 77, 218, 44, 73, 187, 124, 83, 2, 151, 151, 6, 69, 12, 68, 7, 78, 84, 65, + 73, 78, 83, 32, 218, 243, 1, 77, 231, 156, 5, 76, 6, 36, 4, 65, 83, 32, + 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 251, 161, 7, 66, 2, 235, 243, 1, + 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, 178, 243, 1, 81, 42, + 88, 215, 194, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, 84, 72, 32, 4, 26, + 86, 223, 196, 4, 79, 2, 181, 192, 5, 21, 69, 82, 84, 73, 67, 65, 76, 32, + 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, 2, 129, 146, 5, 2, + 32, 67, 8, 130, 1, 70, 255, 245, 1, 83, 40, 96, 3, 73, 71, 72, 64, 13, + 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 155, 201, 5, 69, 6, + 17, 2, 84, 32, 6, 170, 255, 3, 67, 134, 2, 80, 191, 2, 84, 32, 62, 69, + 50, 70, 62, 79, 46, 84, 238, 236, 5, 83, 231, 83, 78, 4, 26, 76, 171, + 176, 7, 73, 2, 255, 237, 5, 69, 8, 26, 73, 251, 225, 6, 79, 6, 154, 26, + 86, 175, 214, 5, 70, 6, 11, 78, 6, 11, 69, 7, 203, 177, 2, 32, 8, 42, 87, + 134, 237, 5, 72, 239, 156, 1, 69, 4, 26, 69, 171, 217, 7, 79, 2, 227, + 220, 6, 76, 2, 183, 163, 4, 69, 4, 178, 235, 3, 87, 211, 215, 1, 73, 2, + 11, 87, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 211, 143, 4, 85, 4, 29, 5, + 32, 84, 72, 65, 78, 5, 11, 32, 2, 11, 79, 2, 223, 176, 1, 82, 32, 34, 76, + 161, 207, 6, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 255, 214, 7, 69, 28, + 112, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 65, 10, + 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 4, 30, 79, 157, 1, 2, 72, 69, 2, + 213, 231, 2, 3, 80, 69, 78, 24, 86, 72, 104, 10, 79, 80, 69, 78, 32, 77, + 79, 85, 84, 72, 178, 2, 83, 159, 205, 2, 84, 6, 34, 69, 50, 79, 207, 177, + 7, 65, 2, 189, 2, 8, 65, 82, 84, 45, 83, 72, 65, 80, 2, 135, 166, 4, 82, + 9, 29, 5, 32, 65, 78, 68, 32, 6, 26, 67, 58, 83, 63, 84, 2, 17, 2, 79, + 76, 2, 253, 174, 3, 4, 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, + 2, 117, 3, 73, 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 11, 67, + 2, 21, 3, 76, 79, 83, 2, 17, 2, 69, 68, 2, 149, 190, 6, 3, 32, 69, 89, 8, + 64, 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, + 32, 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, + 2, 169, 226, 2, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 225, 163, 4, 2, + 32, 72, 2, 17, 2, 78, 71, 2, 241, 187, 6, 4, 76, 65, 83, 83, 2, 195, 183, + 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 210, 197, 6, 73, 203, + 114, 75, 2, 199, 198, 6, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, + 3, 77, 65, 78, 212, 203, 2, 3, 66, 79, 65, 195, 144, 1, 70, 2, 233, 135, + 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 247, + 178, 6, 78, 151, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, + 78, 32, 244, 10, 3, 76, 73, 68, 140, 2, 12, 79, 78, 32, 87, 73, 84, 72, + 32, 82, 73, 71, 72, 20, 11, 82, 65, 32, 83, 79, 77, 80, 69, 78, 71, 32, + 238, 2, 85, 172, 8, 6, 89, 79, 77, 66, 79, 32, 162, 169, 6, 77, 234, 134, + 1, 72, 3, 83, 4, 128, 214, 2, 3, 67, 69, 82, 247, 182, 4, 75, 10, 70, 32, + 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 155, 196, 5, 66, 6, 58, 72, + 44, 6, 73, 67, 69, 32, 67, 82, 231, 133, 6, 83, 2, 11, 89, 2, 11, 80, 2, + 135, 248, 5, 72, 2, 171, 143, 6, 69, 2, 11, 84, 2, 251, 211, 5, 73, 84, + 248, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 168, 2, 13, 73, 78, + 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 20, 7, 76, 69, 84, 84, 69, + 82, 32, 188, 3, 7, 78, 85, 77, 66, 69, 82, 32, 113, 12, 80, 85, 78, 67, + 84, 85, 65, 84, 73, 79, 78, 32, 22, 140, 1, 3, 72, 79, 79, 20, 4, 76, 79, + 78, 71, 50, 83, 58, 84, 242, 213, 3, 68, 216, 199, 1, 4, 82, 69, 83, 72, + 249, 13, 4, 67, 85, 82, 86, 4, 199, 172, 5, 75, 2, 25, 4, 32, 72, 79, 79, + 2, 147, 158, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 219, 157, 5, 69, + 4, 189, 208, 1, 2, 87, 79, 2, 211, 196, 6, 72, 42, 190, 1, 65, 50, 71, + 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 154, 193, 1, + 72, 234, 5, 75, 130, 76, 66, 2, 70, 190, 173, 4, 78, 254, 1, 84, 2, 87, + 202, 103, 80, 171, 4, 77, 4, 26, 76, 231, 194, 6, 89, 2, 215, 200, 1, 69, + 2, 11, 73, 2, 167, 225, 2, 77, 4, 26, 65, 183, 220, 5, 69, 2, 189, 194, + 1, 2, 77, 69, 2, 11, 65, 2, 235, 193, 6, 89, 6, 26, 65, 191, 193, 6, 72, + 4, 178, 185, 1, 77, 187, 218, 5, 68, 2, 195, 193, 1, 79, 8, 18, 79, 59, + 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 227, 151, 2, 72, 4, 230, 213, 5, + 87, 187, 154, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, + 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 201, 193, 4, 2, 76, 69, 6, 100, 13, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 83, 13, 8, 67, 73, 82, + 67, 76, 69, 83, 32, 5, 11, 32, 2, 153, 240, 5, 4, 87, 73, 84, 72, 8, 30, + 32, 189, 1, 2, 85, 83, 4, 93, 21, 81, 85, 73, 76, 84, 32, 83, 81, 85, 65, + 82, 69, 32, 79, 82, 78, 65, 77, 69, 78, 84, 5, 11, 32, 2, 11, 73, 2, 17, + 2, 78, 32, 2, 11, 66, 2, 173, 155, 6, 4, 76, 65, 67, 75, 5, 173, 162, 4, + 7, 32, 87, 73, 84, 72, 32, 79, 2, 251, 227, 3, 84, 70, 52, 7, 76, 69, 84, + 84, 69, 82, 32, 207, 204, 5, 68, 50, 190, 1, 69, 32, 2, 77, 65, 30, 78, + 38, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, + 83, 2, 84, 2, 86, 2, 89, 186, 183, 7, 65, 2, 73, 2, 79, 3, 85, 4, 150, + 184, 7, 69, 147, 1, 72, 4, 134, 185, 7, 69, 3, 72, 6, 34, 71, 2, 89, 187, + 183, 7, 65, 2, 183, 183, 7, 65, 54, 104, 3, 84, 72, 32, 197, 233, 5, 17, + 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 52, + 60, 5, 69, 65, 83, 84, 32, 237, 2, 5, 87, 69, 83, 84, 32, 28, 96, 5, 65, + 82, 82, 79, 87, 162, 4, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, + 203, 43, 87, 11, 11, 32, 8, 52, 2, 65, 78, 38, 67, 120, 2, 84, 79, 139, + 2, 87, 2, 237, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, 79, 83, 83, 73, + 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 221, 209, 4, 5, 84, 72, + 32, 69, 65, 2, 17, 2, 32, 67, 2, 239, 219, 3, 79, 24, 96, 5, 65, 82, 82, + 79, 87, 182, 1, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, + 87, 7, 11, 32, 4, 36, 5, 65, 78, 68, 32, 78, 75, 87, 2, 17, 2, 79, 82, 2, + 21, 3, 84, 72, 32, 2, 201, 207, 4, 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, + 11, 32, 2, 183, 225, 5, 72, 6, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 6, + 42, 86, 154, 253, 3, 76, 215, 137, 2, 66, 2, 17, 2, 73, 78, 2, 239, 252, + 3, 69, 4, 89, 20, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, + 32, 65, 82, 82, 79, 87, 5, 11, 32, 2, 147, 214, 4, 84, 166, 1, 212, 3, + 23, 67, 76, 85, 83, 84, 69, 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, + 69, 84, 84, 69, 82, 32, 40, 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, + 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 71, 32, 27, 72, 69, 65, + 68, 32, 77, 65, 82, 75, 32, 87, 73, 84, 72, 32, 77, 79, 79, 78, 32, 65, + 78, 68, 32, 83, 85, 78, 104, 7, 76, 69, 84, 84, 69, 82, 32, 180, 1, 5, + 77, 65, 82, 75, 32, 54, 83, 100, 8, 84, 69, 82, 77, 73, 78, 65, 76, 41, + 6, 86, 79, 87, 69, 76, 32, 8, 246, 228, 6, 83, 254, 68, 76, 3, 82, 24, + 162, 184, 3, 83, 206, 251, 1, 78, 222, 245, 1, 45, 186, 2, 66, 2, 68, 2, + 71, 2, 75, 2, 76, 2, 77, 3, 82, 2, 157, 166, 6, 3, 69, 77, 73, 7, 29, 5, + 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, 76, 69, 32, 70, 2, + 187, 158, 5, 76, 82, 234, 174, 2, 68, 206, 178, 4, 75, 38, 78, 46, 83, + 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 254, 68, 45, 2, 72, 2, + 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 158, 153, 3, 80, 214, 200, + 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, 211, 229, 6, 85, 8, + 146, 246, 4, 74, 158, 2, 86, 122, 85, 231, 233, 1, 65, 4, 233, 247, 1, 5, + 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, 251, 228, 6, 76, 20, + 88, 7, 86, 79, 67, 65, 76, 73, 67, 226, 229, 6, 65, 26, 79, 2, 85, 150, + 64, 69, 3, 73, 4, 11, 32, 4, 254, 165, 7, 76, 3, 82, 61, 166, 1, 65, 142, + 1, 69, 136, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, 78, 71, 76, 69, + 82, 73, 220, 1, 12, 76, 65, 83, 72, 73, 78, 71, 32, 83, 87, 69, 65, 23, + 79, 13, 60, 4, 71, 72, 69, 84, 20, 3, 82, 75, 76, 151, 141, 7, 67, 2, + 163, 144, 7, 84, 6, 26, 73, 239, 159, 5, 69, 2, 165, 156, 6, 2, 78, 71, + 18, 52, 2, 65, 75, 250, 3, 69, 69, 4, 83, 77, 73, 76, 12, 70, 45, 64, 2, + 69, 82, 173, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 221, 139, 5, 11, + 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, 73, 84, 72, + 32, 6, 26, 67, 78, 79, 75, 84, 2, 33, 6, 65, 78, 67, 69, 76, 76, 2, 245, + 240, 6, 5, 65, 84, 73, 79, 78, 2, 41, 8, 78, 69, 32, 83, 79, 85, 78, 68, + 2, 133, 184, 5, 2, 32, 87, 2, 25, 4, 72, 82, 69, 69, 2, 157, 140, 6, 10, + 32, 83, 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 245, 218, 5, 9, 73, + 78, 32, 83, 73, 76, 72, 79, 85, 4, 34, 68, 241, 232, 2, 2, 67, 72, 2, 11, + 66, 2, 255, 138, 6, 79, 2, 211, 151, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, + 73, 78, 71, 32, 4, 138, 243, 6, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, + 41, 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 239, 226, 5, 69, 6, 80, 8, + 67, 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, + 137, 214, 6, 2, 32, 80, 2, 223, 151, 5, 69, 2, 235, 136, 6, 84, 10, 90, + 79, 60, 7, 85, 84, 73, 78, 71, 32, 87, 128, 228, 2, 3, 82, 84, 83, 223, + 130, 3, 78, 4, 144, 84, 7, 76, 32, 79, 70, 32, 84, 72, 251, 198, 6, 78, + 2, 11, 72, 2, 159, 249, 5, 65, 206, 5, 26, 65, 255, 211, 6, 73, 204, 5, + 28, 2, 82, 69, 175, 66, 84, 202, 5, 30, 32, 129, 44, 2, 68, 32, 218, 3, + 246, 1, 65, 116, 2, 86, 32, 58, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, + 2, 70, 74, 71, 222, 1, 72, 214, 2, 73, 94, 75, 206, 4, 76, 102, 77, 210, + 5, 78, 94, 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 186, 1, 84, 74, 87, + 218, 6, 89, 175, 158, 6, 85, 14, 114, 32, 152, 71, 2, 78, 80, 236, 142, + 2, 3, 82, 85, 72, 238, 51, 65, 240, 43, 2, 80, 65, 250, 224, 3, 77, 3, + 85, 2, 21, 3, 79, 86, 69, 2, 11, 82, 2, 219, 133, 7, 32, 14, 98, 65, 36, + 4, 85, 83, 83, 89, 178, 136, 3, 73, 212, 64, 2, 79, 82, 202, 39, 69, 207, + 164, 3, 81, 4, 32, 2, 65, 82, 155, 149, 7, 82, 2, 175, 136, 3, 69, 22, + 98, 65, 30, 79, 230, 18, 77, 204, 128, 7, 7, 32, 79, 86, 69, 82, 32, 75, + 74, 85, 14, 67, 3, 68, 4, 158, 148, 7, 76, 3, 80, 5, 17, 2, 82, 80, 2, + 247, 181, 2, 79, 20, 82, 65, 234, 17, 77, 142, 195, 2, 69, 174, 49, 79, + 234, 140, 4, 66, 2, 74, 3, 76, 5, 207, 20, 65, 18, 66, 69, 22, 82, 236, + 23, 5, 83, 85, 75, 85, 85, 183, 250, 6, 86, 2, 235, 162, 5, 75, 12, 52, + 7, 65, 32, 78, 65, 77, 69, 32, 235, 145, 7, 71, 10, 132, 1, 4, 72, 69, + 73, 83, 20, 5, 84, 65, 73, 83, 89, 224, 20, 3, 77, 69, 73, 240, 182, 2, + 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 135, 253, 6, 69, 2, 227, 235, 6, + 79, 6, 26, 79, 159, 144, 7, 77, 4, 128, 226, 5, 2, 85, 82, 239, 145, 1, + 79, 24, 102, 65, 46, 73, 188, 12, 5, 85, 82, 65, 77, 85, 178, 139, 3, 72, + 186, 244, 3, 80, 186, 2, 66, 3, 89, 6, 166, 191, 6, 82, 210, 46, 78, 147, + 33, 76, 6, 42, 82, 150, 210, 4, 78, 231, 185, 2, 71, 2, 229, 158, 5, 2, + 85, 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 226, 202, 1, 80, 196, + 184, 5, 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, + 207, 232, 1, 73, 4, 184, 11, 3, 75, 85, 84, 129, 221, 1, 2, 82, 85, 6, + 26, 79, 175, 140, 7, 78, 4, 194, 231, 6, 82, 235, 36, 78, 6, 54, 73, 160, + 17, 4, 65, 82, 65, 68, 131, 179, 5, 82, 2, 203, 191, 3, 73, 14, 54, 78, + 228, 12, 4, 77, 65, 71, 69, 167, 254, 6, 85, 7, 190, 199, 2, 73, 243, + 175, 4, 84, 58, 230, 1, 65, 64, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, + 79, 60, 2, 85, 82, 144, 8, 2, 69, 69, 202, 135, 3, 72, 202, 237, 2, 67, + 208, 120, 3, 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, + 84, 2, 86, 3, 87, 9, 18, 73, 23, 82, 2, 143, 245, 6, 82, 4, 22, 79, 143, + 20, 65, 2, 159, 204, 4, 82, 9, 252, 16, 3, 77, 69, 69, 248, 2, 2, 87, 65, + 169, 161, 2, 3, 71, 85, 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, + 179, 6, 85, 2, 209, 185, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 251, 231, 6, + 80, 2, 151, 204, 5, 85, 4, 176, 154, 5, 4, 85, 90, 69, 73, 239, 66, 79, + 12, 62, 79, 244, 13, 2, 69, 70, 230, 247, 6, 77, 2, 78, 3, 88, 4, 202, + 209, 5, 90, 143, 180, 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, + 73, 110, 77, 68, 2, 85, 32, 78, 86, 2, 87, 162, 137, 3, 72, 214, 237, 2, + 79, 230, 134, 1, 80, 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, + 139, 8, 83, 6, 198, 10, 86, 207, 232, 6, 72, 13, 54, 73, 44, 2, 78, 83, + 146, 82, 82, 159, 185, 4, 72, 4, 252, 150, 5, 2, 75, 85, 175, 199, 1, 82, + 2, 135, 179, 6, 89, 5, 243, 160, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, + 191, 178, 6, 84, 8, 42, 75, 20, 2, 82, 73, 207, 129, 7, 76, 2, 211, 221, + 5, 85, 5, 11, 66, 2, 11, 65, 2, 203, 244, 2, 65, 7, 11, 32, 4, 22, 67, + 139, 8, 83, 2, 11, 85, 2, 175, 161, 2, 66, 16, 218, 128, 7, 65, 2, 70, 2, + 71, 2, 76, 2, 77, 2, 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 191, 189, + 6, 69, 16, 70, 65, 238, 10, 79, 178, 244, 6, 70, 2, 77, 2, 83, 2, 86, 3, + 87, 5, 167, 224, 6, 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, + 170, 171, 2, 79, 255, 210, 4, 86, 2, 207, 217, 6, 83, 6, 21, 3, 32, 79, + 70, 7, 25, 4, 32, 79, 82, 32, 4, 214, 86, 78, 51, 69, 46, 118, 65, 82, + 69, 94, 73, 74, 79, 150, 234, 6, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, + 82, 2, 83, 2, 86, 3, 87, 9, 38, 65, 221, 220, 3, 3, 32, 65, 77, 4, 236, + 1, 2, 83, 69, 227, 213, 6, 84, 8, 34, 69, 22, 78, 231, 220, 6, 83, 2, + 147, 232, 6, 90, 4, 230, 215, 1, 73, 139, 255, 4, 83, 6, 34, 75, 233, 3, + 3, 65, 83, 85, 4, 166, 238, 2, 85, 235, 140, 4, 79, 6, 34, 73, 22, 78, + 21, 2, 83, 73, 2, 191, 174, 3, 78, 2, 191, 219, 6, 68, 2, 133, 193, 6, 4, + 84, 73, 79, 78, 22, 60, 2, 65, 68, 114, 69, 62, 73, 134, 1, 85, 227, 166, + 6, 79, 7, 21, 3, 32, 79, 86, 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, + 2, 11, 81, 2, 153, 153, 2, 2, 85, 65, 4, 36, 3, 78, 84, 79, 191, 211, 6, + 77, 2, 175, 167, 5, 71, 6, 40, 2, 71, 72, 62, 84, 243, 244, 6, 82, 2, + 181, 205, 3, 10, 84, 32, 79, 80, 69, 78, 32, 66, 79, 88, 2, 189, 234, 2, + 2, 84, 79, 4, 172, 1, 2, 85, 66, 195, 185, 4, 80, 14, 82, 65, 72, 3, 69, + 78, 84, 128, 178, 2, 3, 73, 82, 73, 202, 195, 4, 82, 3, 86, 4, 48, 2, 73, + 75, 225, 162, 2, 4, 78, 84, 73, 73, 2, 243, 232, 2, 85, 4, 198, 245, 6, + 73, 3, 79, 6, 42, 65, 146, 254, 2, 72, 159, 167, 3, 79, 2, 241, 191, 6, + 2, 82, 71, 34, 50, 65, 20, 4, 73, 84, 72, 32, 159, 244, 6, 66, 2, 167, + 168, 3, 84, 30, 222, 1, 66, 40, 3, 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, + 84, 2, 72, 79, 42, 76, 126, 84, 28, 6, 85, 80, 80, 69, 82, 32, 186, 1, + 86, 190, 227, 3, 82, 225, 241, 1, 13, 67, 79, 78, 84, 79, 85, 82, 69, 68, + 32, 79, 85, 84, 2, 209, 231, 3, 5, 79, 84, 84, 79, 77, 2, 165, 129, 4, + 16, 71, 79, 78, 65, 76, 32, 67, 82, 79, 83, 83, 72, 65, 84, 67, 72, 2, + 149, 3, 6, 82, 73, 90, 79, 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 211, + 229, 3, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, + 32, 68, 2, 177, 229, 3, 2, 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, + 82, 73, 71, 72, 84, 32, 4, 70, 68, 253, 241, 3, 2, 84, 79, 4, 34, 68, + 205, 253, 3, 2, 84, 79, 2, 141, 228, 3, 7, 73, 65, 71, 79, 78, 65, 76, 2, + 29, 5, 69, 82, 84, 73, 67, 2, 185, 253, 3, 2, 65, 76, 6, 32, 2, 65, 65, + 183, 166, 5, 85, 4, 254, 200, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, + 9, 68, 34, 70, 88, 3, 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, + 77, 38, 78, 34, 79, 94, 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, + 72, 84, 22, 85, 90, 86, 166, 199, 4, 65, 222, 105, 87, 207, 96, 73, 90, + 128, 1, 21, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, + 71, 82, 65, 80, 72, 45, 134, 216, 5, 79, 219, 146, 1, 76, 86, 68, 2, 52, + 69, 82, 53, 206, 2, 54, 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, + 4, 65, 154, 190, 5, 56, 207, 96, 50, 4, 222, 233, 6, 48, 3, 57, 30, 150, + 1, 50, 42, 51, 38, 52, 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, + 56, 70, 162, 143, 3, 66, 248, 172, 2, 2, 70, 56, 205, 96, 2, 49, 56, 6, + 214, 4, 55, 194, 157, 6, 49, 3, 52, 4, 150, 148, 3, 70, 143, 173, 2, 67, + 4, 26, 48, 159, 148, 3, 51, 2, 195, 231, 6, 56, 2, 175, 231, 6, 54, 4, + 242, 147, 3, 50, 243, 208, 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, + 2, 55, 48, 28, 2, 70, 49, 129, 2, 2, 69, 56, 6, 70, 57, 238, 145, 3, 53, + 203, 153, 2, 52, 4, 26, 53, 199, 146, 3, 48, 2, 215, 229, 6, 53, 4, 202, + 2, 66, 211, 143, 3, 57, 2, 171, 2, 50, 4, 146, 229, 6, 56, 3, 57, 2, 247, + 228, 6, 52, 12, 74, 53, 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 157, 225, + 6, 2, 65, 55, 4, 170, 144, 3, 51, 227, 210, 2, 49, 2, 11, 56, 2, 231, + 227, 6, 49, 2, 211, 227, 6, 50, 6, 50, 57, 20, 2, 68, 55, 209, 143, 3, 2, + 67, 65, 2, 159, 143, 3, 69, 2, 247, 226, 6, 48, 4, 204, 156, 6, 2, 49, + 52, 221, 67, 2, 48, 52, 2, 11, 79, 2, 251, 183, 3, 84, 10, 84, 3, 65, 76, + 76, 104, 4, 79, 85, 82, 32, 240, 5, 3, 73, 86, 69, 179, 238, 4, 82, 2, + 57, 12, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, + 83, 2, 191, 251, 4, 76, 4, 210, 179, 3, 68, 131, 173, 3, 75, 8, 192, 181, + 3, 2, 73, 45, 158, 242, 2, 68, 210, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, + 75, 65, 78, 65, 32, 239, 205, 6, 69, 6, 150, 172, 6, 75, 202, 28, 68, + 159, 20, 83, 60, 36, 5, 65, 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, + 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 227, 112, 67, 6, 178, 178, 3, + 71, 157, 149, 2, 3, 83, 83, 76, 4, 234, 218, 4, 73, 251, 130, 2, 86, 4, + 174, 223, 5, 69, 147, 126, 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, + 82, 69, 68, 32, 84, 87, 69, 78, 147, 218, 6, 75, 4, 158, 2, 80, 187, 215, + 4, 76, 20, 110, 69, 146, 1, 72, 20, 2, 73, 88, 198, 200, 4, 65, 150, 66, + 77, 130, 47, 81, 234, 98, 79, 210, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, + 68, 32, 83, 67, 21, 3, 86, 69, 78, 2, 247, 137, 5, 82, 2, 29, 5, 32, 80, + 79, 73, 78, 2, 11, 84, 2, 143, 177, 5, 32, 2, 163, 218, 6, 86, 2, 17, 2, + 84, 89, 2, 239, 217, 6, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, + 2, 163, 147, 6, 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, + 79, 73, 78, 19, 79, 2, 215, 75, 84, 2, 179, 213, 6, 32, 4, 48, 6, 80, 32, + 87, 73, 84, 72, 243, 145, 6, 72, 2, 17, 2, 32, 69, 2, 235, 108, 88, 4, + 202, 145, 6, 79, 151, 70, 83, 2, 201, 183, 5, 6, 32, 66, 76, 65, 67, 75, + 6, 146, 215, 6, 50, 2, 51, 3, 65, 85, 134, 1, 65, 138, 7, 69, 236, 1, 10, + 73, 67, 75, 32, 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 234, 2, 85, + 130, 200, 6, 83, 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 80, + 2, 78, 68, 90, 82, 206, 3, 84, 175, 167, 5, 68, 4, 60, 8, 65, 69, 83, 67, + 85, 76, 65, 80, 21, 3, 72, 69, 82, 2, 219, 209, 4, 73, 2, 211, 192, 5, + 77, 2, 21, 3, 80, 69, 68, 2, 17, 2, 32, 69, 2, 173, 210, 1, 4, 78, 86, + 69, 76, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, + 171, 131, 6, 83, 24, 42, 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, + 54, 79, 240, 224, 1, 8, 65, 78, 68, 32, 67, 82, 69, 83, 131, 229, 1, 87, + 2, 17, 2, 81, 85, 2, 11, 65, 2, 135, 148, 6, 76, 4, 44, 5, 70, 32, 68, + 65, 86, 143, 167, 3, 80, 2, 255, 138, 6, 73, 12, 66, 71, 30, 83, 40, 4, + 80, 82, 79, 84, 142, 66, 72, 155, 102, 84, 2, 89, 4, 85, 65, 82, 68, 4, + 26, 69, 155, 239, 1, 84, 2, 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, + 68, 32, 65, 82, 2, 135, 205, 6, 69, 4, 144, 229, 4, 10, 85, 69, 32, 79, + 70, 32, 76, 73, 66, 69, 187, 154, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, + 79, 71, 82, 65, 80, 72, 73, 67, 193, 203, 1, 4, 84, 72, 79, 83, 4, 54, + 32, 173, 196, 5, 7, 73, 78, 71, 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, + 77, 79, 2, 135, 197, 1, 84, 2, 137, 130, 4, 2, 32, 70, 11, 11, 32, 8, 60, + 5, 87, 73, 84, 72, 32, 193, 153, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, + 77, 83, 51, 68, 2, 25, 4, 32, 82, 65, 73, 2, 167, 236, 1, 83, 2, 143, + 181, 5, 82, 4, 44, 4, 67, 75, 32, 67, 21, 3, 80, 87, 65, 2, 139, 196, 5, + 72, 2, 211, 168, 6, 84, 12, 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, + 76, 73, 78, 79, 73, 6, 44, 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 188, + 130, 3, 2, 32, 82, 243, 176, 2, 78, 2, 147, 61, 82, 2, 17, 2, 69, 68, 2, + 17, 2, 32, 87, 2, 145, 182, 4, 4, 72, 73, 84, 69, 4, 52, 4, 78, 71, 32, + 84, 233, 7, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 169, 143, 6, 3, 77, + 73, 78, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, 20, 9, 70, 70, 69, 68, + 32, 70, 76, 65, 84, 207, 196, 6, 80, 2, 215, 157, 5, 79, 2, 11, 66, 2, + 231, 248, 3, 82, 202, 2, 150, 1, 66, 196, 2, 5, 67, 67, 69, 69, 68, 192, + 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 132, 15, 3, 80, 69, 82, + 148, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 72, 6, 67, 82, 73, 80, 84, + 32, 124, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 222, 22, 69, 162, 1, 80, + 150, 208, 2, 77, 202, 1, 76, 26, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, + 90, 130, 83, 78, 15, 79, 28, 56, 6, 65, 66, 79, 86, 69, 32, 246, 25, 79, + 159, 3, 87, 6, 142, 24, 83, 203, 139, 5, 82, 2, 235, 254, 4, 85, 22, 11, + 83, 23, 11, 32, 20, 128, 1, 6, 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, + 84, 32, 78, 79, 84, 32, 2, 79, 82, 229, 255, 2, 5, 85, 78, 68, 69, 82, + 12, 100, 4, 78, 79, 84, 32, 28, 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, + 78, 69, 32, 218, 24, 65, 39, 69, 4, 242, 24, 65, 167, 1, 69, 4, 250, 24, + 69, 83, 78, 2, 93, 5, 32, 69, 81, 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, + 85, 4, 22, 73, 167, 25, 65, 2, 173, 25, 6, 86, 65, 76, 69, 78, 84, 6, + 202, 246, 2, 66, 228, 196, 1, 4, 87, 73, 84, 72, 147, 139, 1, 84, 161, 1, + 166, 1, 32, 72, 7, 68, 65, 78, 69, 83, 69, 32, 200, 12, 4, 82, 73, 83, + 69, 200, 166, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, + 68, 255, 240, 2, 70, 4, 50, 87, 197, 146, 5, 6, 66, 69, 72, 73, 78, 68, + 2, 247, 255, 5, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, + 66, 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 200, 1, 13, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 255, 195, 4, 68, 2, 235, + 211, 2, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, + 78, 32, 2, 17, 2, 78, 71, 2, 167, 176, 5, 75, 4, 142, 5, 65, 163, 146, 1, + 73, 4, 226, 182, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, + 83, 202, 230, 2, 69, 246, 215, 1, 78, 246, 175, 1, 66, 2, 75, 254, 68, + 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, + 2, 88, 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 128, 247, 5, 6, 82, 67, + 72, 65, 73, 67, 135, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 163, 182, 6, + 65, 4, 158, 182, 6, 75, 3, 77, 4, 154, 145, 6, 69, 235, 36, 65, 4, 170, + 179, 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, + 67, 34, 80, 189, 177, 6, 3, 83, 85, 82, 2, 11, 65, 2, 137, 94, 5, 32, 83, + 65, 84, 65, 2, 11, 65, 2, 247, 239, 5, 75, 4, 168, 3, 3, 65, 78, 71, 141, + 238, 5, 3, 85, 82, 78, 10, 28, 2, 80, 65, 211, 82, 86, 8, 28, 3, 77, 65, + 65, 23, 78, 2, 143, 178, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 177, 236, + 5, 3, 87, 73, 83, 2, 165, 144, 5, 2, 65, 89, 2, 141, 175, 6, 2, 69, 67, + 12, 18, 77, 23, 78, 2, 179, 244, 2, 69, 10, 96, 3, 69, 85, 76, 34, 79, + 22, 89, 140, 220, 1, 3, 71, 72, 85, 157, 152, 2, 4, 65, 69, 76, 65, 2, + 11, 69, 2, 147, 224, 5, 85, 2, 139, 244, 3, 76, 2, 203, 142, 1, 85, 5, + 173, 130, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 72, + 54, 83, 166, 195, 4, 72, 241, 64, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, + 73, 80, 84, 32, 173, 2, 3, 69, 84, 32, 34, 118, 69, 34, 76, 130, 1, 80, + 150, 208, 2, 77, 226, 1, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, 90, + 130, 83, 78, 15, 79, 4, 226, 66, 81, 235, 192, 5, 73, 6, 96, 18, 65, 84, + 73, 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 135, 210, + 2, 69, 4, 138, 173, 6, 73, 3, 78, 2, 155, 147, 3, 76, 34, 148, 1, 6, 65, + 66, 79, 86, 69, 32, 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, + 3, 87, 153, 148, 4, 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, + 231, 187, 2, 76, 4, 11, 85, 4, 26, 80, 191, 194, 4, 66, 2, 185, 194, 4, + 2, 69, 82, 4, 108, 23, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, + 89, 32, 68, 65, 83, 72, 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, + 153, 193, 4, 2, 85, 66, 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, + 86, 69, 32, 132, 1, 4, 87, 73, 84, 72, 181, 166, 5, 11, 79, 82, 32, 69, + 81, 85, 65, 76, 32, 84, 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, + 2, 113, 3, 77, 79, 83, 2, 187, 60, 81, 2, 21, 3, 73, 76, 68, 2, 147, 253, + 2, 69, 2, 17, 2, 32, 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, + 17, 2, 81, 85, 2, 11, 65, 2, 11, 76, 2, 179, 218, 2, 32, 6, 25, 4, 73, + 84, 72, 32, 6, 96, 2, 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, + 65, 84, 73, 79, 78, 231, 212, 5, 68, 2, 11, 85, 2, 11, 83, 2, 169, 254, + 3, 5, 32, 83, 73, 71, 78, 4, 196, 160, 4, 2, 65, 67, 211, 203, 1, 69, 4, + 60, 9, 80, 69, 78, 83, 73, 79, 78, 32, 82, 167, 144, 6, 72, 2, 21, 3, 65, + 73, 76, 2, 247, 151, 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, + 32, 65, 77, 135, 163, 6, 78, 2, 173, 223, 5, 7, 80, 69, 82, 83, 65, 78, + 68, 2, 11, 77, 2, 231, 228, 5, 77, 2, 129, 250, 2, 2, 78, 71, 230, 2, 92, + 11, 76, 79, 84, 73, 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 182, 15, 78, + 97, 2, 82, 73, 90, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, + 79, 69, 84, 82, 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, + 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, + 82, 34, 84, 154, 89, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 246, 165, 5, + 72, 2, 76, 2, 77, 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, + 8, 226, 89, 68, 246, 165, 5, 72, 247, 30, 79, 4, 170, 255, 5, 82, 247, + 30, 79, 8, 150, 89, 84, 246, 165, 5, 72, 247, 30, 79, 8, 11, 45, 8, 198, + 157, 6, 49, 2, 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 213, 215, 5, 3, 68, + 86, 73, 4, 64, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 203, 215, 5, + 78, 2, 129, 137, 2, 2, 65, 83, 10, 150, 253, 5, 79, 246, 30, 65, 2, 69, + 2, 73, 3, 85, 82, 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 213, 14, 2, 77, + 69, 80, 238, 2, 66, 48, 2, 67, 65, 118, 68, 222, 1, 69, 130, 2, 70, 56, + 8, 72, 79, 82, 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, + 76, 73, 78, 52, 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, + 68, 3, 82, 69, 67, 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, + 83, 205, 3, 15, 84, 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, + 78, 4, 242, 147, 4, 69, 133, 198, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, + 237, 182, 1, 2, 78, 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, + 135, 185, 1, 85, 14, 24, 2, 65, 84, 55, 69, 2, 161, 3, 9, 65, 32, 76, 73, + 78, 75, 32, 69, 83, 12, 76, 12, 86, 73, 67, 69, 32, 67, 79, 78, 84, 82, + 79, 76, 157, 8, 2, 76, 69, 8, 11, 32, 8, 222, 174, 1, 70, 154, 250, 2, + 84, 203, 83, 79, 12, 22, 78, 207, 1, 83, 10, 48, 5, 68, 32, 79, 70, 32, + 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, 193, 236, 4, 2, 69, 68, 6, 64, + 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, 79, 78, 159, 108, 69, 5, 143, + 171, 3, 32, 2, 155, 7, 73, 2, 205, 206, 4, 2, 67, 65, 4, 36, 2, 73, 76, + 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 129, 162, 1, 6, 65, 76, 32, 84, 65, + 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 223, 178, 1, 69, 2, 191, 219, 4, 80, + 6, 26, 69, 175, 141, 4, 85, 4, 56, 8, 71, 65, 84, 73, 86, 69, 32, 65, + 247, 247, 4, 87, 2, 21, 3, 67, 75, 78, 2, 21, 3, 79, 87, 76, 2, 191, 136, + 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 191, 247, 2, 32, 18, 176, 1, 7, + 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, 52, 8, 84, 65, 82, 84, 32, + 79, 70, 32, 64, 7, 85, 66, 83, 84, 73, 84, 85, 204, 1, 3, 89, 78, 67, + 255, 206, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, 11, 85, 2, 219, 209, 5, 82, + 4, 17, 2, 84, 32, 4, 226, 148, 5, 79, 239, 41, 73, 4, 22, 72, 155, 102, + 84, 2, 17, 2, 69, 65, 2, 223, 188, 5, 68, 4, 11, 84, 4, 11, 69, 5, 11, + 32, 2, 25, 4, 70, 79, 82, 77, 2, 17, 2, 32, 84, 2, 251, 237, 5, 87, 2, + 137, 207, 5, 2, 73, 67, 2, 11, 84, 2, 235, 250, 5, 82, 7, 38, 67, 145, + 171, 2, 3, 65, 71, 79, 2, 141, 134, 1, 9, 72, 82, 79, 78, 79, 85, 83, 32, + 73, 180, 1, 36, 3, 65, 67, 32, 163, 215, 4, 78, 178, 1, 150, 3, 65, 52, + 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, 84, 69, 68, 32, 90, + 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, 69, 84, 84, 69, 82, + 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, 28, 4, 80, 84, 72, 65, + 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, 2, 84, 216, 205, 4, 3, + 77, 85, 83, 228, 21, 7, 70, 69, 77, 73, 78, 73, 78, 229, 136, 1, 5, 81, + 85, 83, 72, 83, 2, 11, 66, 2, 197, 158, 5, 5, 66, 82, 69, 86, 73, 2, 11, + 69, 2, 167, 134, 6, 75, 6, 38, 78, 165, 17, 4, 76, 79, 78, 32, 2, 17, 2, + 84, 82, 2, 151, 182, 5, 65, 4, 28, 3, 65, 78, 71, 35, 72, 2, 11, 85, 2, + 255, 227, 4, 76, 2, 11, 79, 2, 185, 184, 3, 5, 82, 73, 90, 79, 78, 6, 60, + 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, 14, 83, 2, 201, 11, 2, + 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, 32, 132, 1, 4, 66, 65, + 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, 5, 65, 83, 84, 69, 82, + 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, 219, 128, 4, 67, 2, + 181, 166, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, 65, 83, 65, 32, 68, + 159, 223, 3, 32, 2, 197, 178, 5, 2, 65, 76, 92, 154, 2, 68, 102, 72, 32, + 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, 78, 83, 144, 1, 8, + 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, 81, 34, 84, 40, 5, + 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 146, 75, 66, 166, 130, 4, 90, + 154, 43, 78, 254, 1, 87, 159, 126, 69, 4, 76, 14, 79, 84, 76, 69, 83, 83, + 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, 139, 193, 2, 82, 4, 11, + 69, 5, 159, 254, 5, 84, 2, 11, 65, 2, 255, 253, 5, 68, 24, 60, 9, 65, 76, + 65, 89, 65, 76, 65, 77, 32, 231, 237, 5, 73, 22, 78, 76, 22, 78, 158, + 134, 4, 66, 242, 210, 1, 84, 138, 34, 83, 14, 74, 3, 82, 4, 199, 247, 3, + 76, 8, 182, 84, 78, 234, 166, 5, 71, 3, 89, 9, 33, 6, 82, 83, 73, 65, 78, + 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, 72, 65, 2, 131, 79, 72, 2, + 147, 2, 76, 2, 143, 243, 4, 77, 4, 52, 7, 69, 86, 69, 82, 83, 69, 68, + 211, 150, 4, 73, 2, 251, 182, 4, 32, 14, 142, 1, 69, 40, 7, 79, 71, 68, + 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, + 166, 250, 4, 72, 177, 82, 2, 65, 68, 2, 17, 2, 77, 75, 2, 175, 251, 3, + 65, 6, 42, 90, 32, 2, 75, 72, 131, 227, 5, 70, 2, 195, 206, 4, 72, 2, 11, + 76, 2, 11, 65, 2, 183, 248, 5, 80, 6, 36, 3, 69, 84, 72, 255, 250, 4, 65, + 5, 237, 85, 6, 32, 71, 65, 82, 83, 72, 5, 215, 203, 5, 32, 4, 225, 223, + 3, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, 186, 223, 3, 66, 223, 155, + 1, 65, 2, 17, 2, 79, 84, 2, 155, 152, 1, 84, 8, 58, 66, 210, 144, 2, 87, + 217, 189, 1, 4, 85, 75, 75, 65, 4, 173, 204, 3, 2, 65, 83, 14, 88, 8, 66, + 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, 65, 76, 73, 78, 69, 65, 82, + 32, 8, 44, 5, 67, 79, 76, 79, 78, 227, 170, 3, 70, 7, 11, 32, 4, 29, 5, + 83, 75, 69, 87, 69, 4, 171, 202, 5, 68, 6, 44, 5, 67, 79, 76, 79, 78, + 243, 169, 3, 70, 5, 185, 194, 5, 7, 32, 83, 75, 69, 87, 69, 68, 8, 62, + 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, 67, 65, 76, 4, 21, 3, 82, 69, + 69, 4, 25, 4, 32, 68, 79, 84, 4, 239, 218, 3, 83, 148, 41, 102, 45, 58, + 65, 166, 106, 69, 254, 45, 72, 174, 46, 73, 254, 72, 79, 182, 20, 82, + 226, 16, 85, 191, 9, 87, 4, 32, 2, 83, 72, 203, 205, 4, 82, 2, 179, 235, + 4, 73, 244, 26, 182, 1, 66, 82, 71, 192, 17, 2, 73, 32, 182, 29, 75, 152, + 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, 32, 34, 77, 226, 37, 78, 156, 13, + 3, 80, 69, 32, 94, 85, 150, 233, 4, 67, 159, 11, 88, 5, 249, 123, 16, 76, + 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, 65, 68, 68, 76, 69, 144, 2, 78, + 32, 224, 11, 5, 65, 76, 79, 71, 32, 253, 2, 6, 66, 65, 78, 87, 65, 32, + 190, 1, 158, 1, 65, 102, 67, 186, 1, 68, 58, 69, 98, 71, 118, 72, 46, 76, + 222, 3, 80, 54, 81, 62, 82, 106, 83, 194, 77, 78, 166, 213, 1, 84, 166, + 118, 70, 159, 177, 1, 86, 6, 42, 80, 170, 180, 2, 77, 147, 169, 1, 83, 2, + 17, 2, 79, 83, 2, 197, 192, 5, 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, + 25, 4, 82, 67, 85, 77, 2, 213, 2, 4, 70, 76, 69, 88, 6, 26, 77, 231, 156, + 5, 76, 4, 11, 77, 4, 228, 216, 4, 7, 69, 82, 67, 73, 65, 76, 32, 211, + 147, 1, 65, 22, 26, 79, 207, 253, 3, 73, 2, 11, 76, 2, 175, 85, 76, 4, + 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, 207, 209, 2, 76, 2, 221, 129, 5, 4, + 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, 55, 69, 2, 11, 86, 2, 11, 69, 2, + 209, 121, 3, 32, 65, 67, 2, 137, 4, 4, 65, 84, 69, 82, 2, 129, 141, 2, 6, + 89, 80, 72, 69, 78, 45, 114, 38, 65, 246, 2, 69, 183, 129, 4, 79, 104, + 25, 4, 84, 73, 78, 32, 104, 34, 67, 33, 4, 83, 77, 65, 76, 52, 29, 5, 65, + 80, 73, 84, 65, 52, 41, 8, 76, 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, + 52, 254, 231, 5, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, + 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, + 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 8, 22, 83, 195, 1, 70, + 2, 11, 83, 2, 145, 64, 3, 45, 84, 72, 4, 26, 69, 247, 203, 2, 76, 2, 185, + 68, 2, 82, 67, 4, 11, 85, 4, 26, 79, 235, 154, 3, 69, 2, 227, 251, 4, 84, + 8, 36, 3, 73, 71, 72, 159, 210, 3, 69, 6, 17, 2, 84, 32, 6, 238, 135, 2, + 67, 134, 2, 80, 155, 2, 83, 6, 174, 174, 2, 69, 202, 165, 1, 79, 155, + 211, 1, 80, 46, 80, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, + 78, 32, 159, 2, 86, 38, 162, 1, 65, 218, 146, 2, 78, 210, 204, 3, 66, 2, + 68, 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, + 2, 89, 186, 2, 73, 3, 85, 5, 145, 151, 5, 6, 82, 67, 72, 65, 73, 67, 4, + 18, 80, 51, 86, 2, 25, 4, 65, 77, 85, 68, 2, 159, 191, 2, 80, 2, 247, + 174, 3, 73, 36, 48, 7, 76, 69, 84, 84, 69, 82, 32, 147, 1, 86, 32, 158, + 145, 2, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, + 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, + 76, 32, 83, 73, 71, 4, 11, 78, 4, 179, 158, 5, 32, 212, 3, 112, 10, 76, + 69, 32, 76, 69, 84, 84, 69, 82, 32, 248, 2, 5, 84, 72, 65, 77, 32, 193, + 18, 5, 86, 73, 69, 84, 32, 70, 186, 1, 65, 34, 69, 30, 84, 214, 140, 2, + 78, 138, 144, 1, 79, 206, 247, 1, 75, 2, 80, 162, 7, 85, 222, 61, 70, 2, + 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, 2, 88, 2, 89, 187, 2, 73, 7, 194, + 197, 5, 85, 215, 22, 73, 7, 246, 219, 5, 69, 3, 72, 18, 60, 3, 79, 78, + 69, 234, 147, 5, 83, 254, 68, 72, 187, 2, 65, 10, 11, 45, 10, 142, 219, + 5, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, 1, 196, 1, 2, 67, 79, 240, 3, 4, + 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, 28, 7, 76, 69, 84, 84, 69, 82, 32, + 220, 4, 5, 83, 73, 71, 78, 32, 217, 5, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, + 71, 78, 32, 237, 198, 4, 21, 77, 66, 73, 78, 73, 78, 71, 32, 67, 82, 89, + 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, 18, 148, 1, 6, 70, 73, 78, 65, + 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, 32, 82, 65, 84, 72, 65, 32, 79, + 82, 32, 76, 79, 22, 77, 186, 211, 5, 66, 3, 83, 2, 151, 148, 5, 78, 4, + 54, 79, 149, 151, 1, 7, 65, 32, 84, 65, 78, 71, 32, 2, 167, 216, 1, 87, + 6, 48, 6, 69, 68, 73, 65, 76, 32, 191, 213, 5, 65, 4, 130, 211, 5, 76, 3, + 82, 20, 129, 231, 3, 2, 32, 68, 106, 188, 1, 2, 71, 82, 44, 5, 72, 73, + 71, 72, 32, 94, 76, 222, 1, 82, 198, 129, 2, 85, 210, 200, 1, 73, 166, + 15, 78, 182, 216, 1, 79, 162, 8, 69, 158, 20, 66, 2, 68, 2, 77, 2, 87, + 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 231, 208, 5, 32, 32, 242, 1, 75, 42, + 82, 250, 136, 5, 83, 82, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 3, 89, 36, + 60, 3, 79, 87, 32, 234, 145, 5, 65, 194, 41, 85, 159, 20, 76, 28, 86, 75, + 42, 82, 202, 137, 5, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 2, 83, 3, 89, + 6, 234, 206, 5, 72, 2, 88, 187, 2, 65, 2, 189, 217, 3, 2, 65, 84, 8, 26, + 65, 243, 185, 5, 85, 7, 138, 206, 5, 78, 3, 84, 50, 182, 1, 72, 34, 75, + 176, 1, 4, 77, 65, 73, 32, 82, 82, 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, + 78, 71, 152, 27, 4, 84, 79, 78, 69, 224, 100, 3, 68, 79, 75, 177, 242, 3, + 2, 67, 65, 4, 130, 254, 4, 65, 167, 63, 79, 14, 52, 4, 72, 85, 69, 78, + 134, 3, 65, 139, 177, 4, 69, 8, 80, 6, 32, 84, 79, 78, 69, 45, 145, 134, + 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, 6, 194, 205, 5, 51, 2, 52, 3, 53, + 8, 56, 4, 75, 65, 78, 71, 194, 136, 1, 89, 235, 138, 3, 83, 5, 243, 141, + 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, 69, 86, 69, 82, 83, 69, 68, 32, 82, + 79, 84, 65, 84, 69, 68, 32, 2, 179, 146, 4, 65, 2, 171, 162, 3, 82, 8, + 48, 3, 84, 75, 65, 226, 131, 4, 87, 139, 119, 75, 4, 17, 2, 65, 78, 5, + 251, 219, 3, 75, 5, 133, 156, 3, 2, 87, 65, 38, 90, 65, 36, 4, 77, 65, + 73, 32, 22, 79, 46, 84, 86, 85, 242, 193, 3, 73, 207, 134, 2, 69, 9, 242, + 201, 5, 65, 2, 69, 3, 73, 2, 255, 181, 4, 83, 11, 218, 176, 3, 65, 226, + 152, 2, 79, 3, 89, 4, 42, 65, 129, 138, 1, 4, 72, 65, 77, 32, 2, 17, 2, + 76, 76, 2, 135, 217, 3, 32, 9, 166, 136, 5, 85, 151, 64, 69, 144, 1, 184, + 1, 7, 76, 69, 84, 84, 69, 82, 32, 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, + 83, 89, 77, 66, 79, 76, 32, 124, 9, 84, 79, 78, 69, 32, 77, 65, 73, 32, + 81, 6, 86, 79, 87, 69, 76, 32, 96, 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, + 87, 48, 11, 32, 48, 154, 1, 75, 30, 67, 2, 80, 2, 84, 34, 78, 214, 165, + 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, + 89, 247, 30, 79, 6, 26, 72, 235, 196, 5, 79, 4, 242, 165, 5, 72, 247, 30, + 79, 6, 210, 165, 5, 71, 2, 89, 247, 30, 79, 4, 178, 243, 4, 65, 183, 52, + 72, 10, 72, 2, 75, 79, 110, 78, 164, 128, 1, 4, 72, 79, 32, 72, 235, 136, + 3, 83, 4, 140, 129, 1, 3, 73, 32, 75, 167, 194, 4, 78, 8, 58, 78, 178, + 134, 1, 84, 186, 255, 1, 83, 203, 185, 2, 69, 2, 151, 134, 3, 85, 26, 50, + 65, 58, 85, 30, 73, 174, 193, 5, 69, 3, 79, 10, 170, 171, 5, 85, 214, 22, + 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 175, 193, 5, 65, 5, 171, 193, 5, 65, + 138, 1, 52, 3, 82, 73, 32, 173, 180, 4, 4, 69, 79, 85, 84, 136, 1, 82, + 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, 83, 78, 86, 227, 206, 3, + 68, 2, 163, 213, 1, 66, 88, 214, 1, 65, 254, 157, 1, 82, 242, 55, 68, 46, + 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 126, 66, 2, 67, 2, 71, + 2, 74, 2, 75, 2, 80, 2, 83, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, + 2, 69, 3, 79, 11, 176, 149, 3, 7, 82, 67, 72, 65, 73, 67, 32, 230, 168, + 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, 32, 8, 226, 215, 1, 78, 222, + 160, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 18, 238, 215, 1, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, + 3, 79, 4, 226, 51, 70, 139, 239, 3, 79, 188, 6, 36, 3, 73, 76, 32, 147, + 154, 4, 65, 186, 6, 170, 3, 65, 102, 67, 194, 2, 68, 72, 3, 87, 69, 84, + 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 136, 8, 9, 73, 78, 32, 80, 79, + 83, 83, 69, 83, 22, 76, 204, 2, 7, 78, 85, 77, 66, 69, 82, 32, 152, 1, + 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 69, 78, 68, 32, 79, + 70, 54, 82, 42, 83, 188, 14, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 78, 84, 228, 1, 2, 89, 69, 184, 143, 4, 5, 77, 79, 78, 84, 72, 147, + 118, 79, 6, 76, 5, 83, 32, 65, 66, 79, 206, 55, 85, 149, 154, 3, 5, 78, + 68, 32, 79, 68, 2, 219, 159, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, 65, + 78, 84, 32, 192, 21, 3, 85, 82, 82, 143, 10, 82, 48, 130, 1, 75, 22, 76, + 22, 78, 46, 84, 170, 193, 1, 83, 170, 16, 82, 166, 227, 3, 67, 2, 72, 2, + 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 247, 247, 4, 83, 7, 151, 185, 4, 76, + 11, 158, 189, 3, 78, 130, 248, 1, 71, 3, 89, 5, 243, 180, 5, 84, 26, 68, + 2, 82, 89, 172, 29, 2, 69, 66, 234, 248, 1, 65, 247, 175, 1, 73, 2, 213, + 201, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, 32, + 252, 4, 6, 84, 72, 82, 69, 69, 32, 165, 172, 5, 22, 68, 79, 87, 78, 83, + 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, 30, + 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 18, 79, 88, 4, 83, + 73, 88, 84, 110, 84, 187, 175, 3, 81, 4, 242, 3, 73, 215, 173, 5, 72, 4, + 188, 3, 2, 79, 82, 191, 173, 3, 73, 4, 171, 1, 70, 2, 133, 3, 18, 78, 69, + 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, 6, 64, 5, + 69, 69, 78, 84, 72, 165, 233, 1, 5, 89, 45, 70, 79, 85, 4, 11, 45, 4, + 198, 175, 5, 49, 3, 50, 8, 38, 72, 138, 1, 87, 239, 174, 3, 69, 4, 132, + 1, 18, 82, 69, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, + 84, 87, 249, 242, 3, 8, 73, 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, + 2, 17, 2, 84, 73, 2, 207, 174, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, + 82, 84, 163, 174, 3, 81, 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 175, + 3, 6, 89, 45, 70, 79, 85, 82, 2, 145, 175, 3, 2, 69, 78, 2, 21, 3, 87, + 69, 78, 2, 221, 174, 3, 3, 84, 73, 69, 2, 247, 209, 1, 83, 72, 48, 6, 69, + 84, 84, 69, 82, 32, 187, 197, 3, 65, 70, 198, 1, 78, 138, 24, 84, 222, + 22, 76, 210, 90, 82, 206, 55, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, + 73, 202, 190, 1, 83, 242, 7, 69, 222, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, + 2, 80, 2, 86, 3, 89, 10, 46, 78, 234, 166, 5, 71, 2, 89, 187, 2, 65, 4, + 230, 166, 5, 78, 187, 2, 65, 8, 38, 79, 230, 215, 3, 84, 167, 74, 83, 4, + 11, 78, 4, 17, 2, 69, 32, 4, 18, 72, 31, 84, 2, 213, 72, 3, 85, 78, 68, + 2, 229, 238, 1, 3, 72, 79, 85, 2, 17, 2, 32, 84, 2, 11, 69, 2, 155, 139, + 5, 88, 2, 17, 2, 85, 80, 2, 199, 143, 3, 69, 194, 4, 152, 1, 5, 65, 76, + 84, 32, 80, 20, 4, 73, 71, 78, 32, 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, + 78, 71, 32, 70, 41, 8, 89, 76, 76, 65, 66, 76, 69, 32, 2, 207, 175, 3, + 65, 40, 90, 65, 48, 3, 67, 69, 86, 34, 75, 80, 2, 77, 85, 102, 85, 14, + 80, 118, 86, 163, 65, 78, 4, 216, 2, 4, 65, 90, 72, 65, 195, 221, 4, 78, + 2, 11, 73, 2, 215, 255, 4, 84, 6, 38, 85, 157, 255, 4, 3, 65, 65, 67, 4, + 18, 90, 87, 82, 2, 151, 144, 5, 72, 6, 60, 4, 75, 75, 85, 82, 16, 2, 84, + 72, 21, 3, 85, 86, 85, 2, 163, 83, 85, 2, 203, 241, 2, 65, 2, 75, 90, 8, + 26, 65, 255, 210, 4, 79, 6, 34, 84, 230, 232, 3, 65, 15, 78, 2, 11, 72, + 2, 17, 2, 65, 75, 2, 163, 253, 4, 75, 10, 38, 65, 210, 82, 69, 143, 140, + 4, 73, 4, 176, 103, 3, 82, 65, 65, 213, 202, 2, 6, 75, 65, 73, 89, 65, + 82, 2, 11, 69, 2, 159, 10, 78, 2, 17, 2, 82, 79, 2, 243, 153, 4, 77, 148, + 4, 122, 75, 170, 1, 76, 174, 1, 78, 194, 1, 82, 90, 83, 186, 1, 84, 86, + 67, 2, 72, 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 46, 88, 2, 83, 83, 150, + 186, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, + 24, 154, 182, 1, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, + 1, 69, 66, 82, 76, 246, 184, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, + 56, 73, 187, 198, 1, 69, 44, 250, 4, 76, 250, 179, 1, 65, 190, 21, 85, + 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 110, 102, 78, 190, 3, 71, + 2, 89, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, + 198, 1, 69, 44, 186, 3, 78, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, + 79, 182, 56, 73, 187, 198, 1, 69, 44, 226, 2, 82, 250, 179, 1, 65, 190, + 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 68, 94, 72, 174, + 1, 83, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, + 198, 1, 69, 24, 162, 181, 1, 65, 190, 21, 85, 158, 144, 1, 79, 230, 2, + 82, 210, 53, 73, 187, 198, 1, 69, 44, 82, 84, 250, 179, 1, 65, 190, 21, + 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 22, 246, 179, 1, 65, + 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 68, 2, 79, + 84, 33, 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, + 199, 144, 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, + 2, 247, 143, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 23, 69, 2, 11, 65, 2, + 171, 143, 4, 82, 184, 13, 36, 5, 65, 66, 65, 84, 65, 35, 71, 2, 11, 32, + 2, 135, 169, 3, 84, 182, 13, 54, 69, 20, 3, 83, 65, 32, 241, 6, 3, 85, + 84, 32, 2, 215, 251, 3, 82, 178, 1, 52, 7, 76, 69, 84, 84, 69, 82, 32, + 143, 166, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, 72, 62, 76, 58, 77, 54, + 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, 86, 158, 198, 4, 84, 82, + 67, 2, 68, 2, 71, 2, 75, 2, 80, 254, 68, 66, 2, 82, 2, 87, 2, 88, 2, 89, + 3, 90, 16, 174, 4, 87, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 4, 168, 193, + 4, 5, 73, 78, 65, 76, 32, 239, 80, 65, 6, 26, 84, 203, 145, 5, 65, 4, + 142, 143, 5, 84, 187, 2, 65, 4, 192, 236, 3, 5, 79, 78, 71, 32, 85, 231, + 164, 1, 65, 10, 238, 144, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 130, 142, + 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 243, 143, 5, 65, 6, 40, 4, + 79, 82, 84, 32, 199, 143, 5, 65, 4, 208, 152, 1, 2, 85, 69, 229, 234, 2, + 2, 65, 87, 32, 58, 73, 54, 69, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 16, + 50, 85, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 8, 158, 142, 5, 67, 2, 81, + 2, 88, 3, 90, 130, 12, 64, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, + 143, 241, 2, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 95, 55, 198, 1, 86, 48, 242, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 18, 130, 140, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 200, 1, 150, 1, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 138, 1, 58, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 95, 54, 20, 146, 138, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 182, 137, 5, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 4, 48, 6, 67, 65, 82, 84, 82, + 73, 21, 2, 68, 82, 2, 187, 212, 3, 68, 2, 215, 139, 4, 73, 2, 155, 133, + 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 234, + 14, 78, 144, 1, 5, 83, 84, 32, 84, 85, 21, 12, 84, 82, 65, 71, 82, 65, + 77, 32, 70, 79, 82, 32, 10, 68, 9, 67, 85, 80, 32, 87, 73, 84, 72, 79, + 58, 82, 171, 181, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 195, 228, + 3, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, + 80, 45, 2, 221, 226, 3, 2, 78, 68, 4, 234, 243, 2, 83, 245, 112, 4, 66, + 65, 82, 66, 2, 147, 226, 3, 69, 216, 1, 38, 69, 225, 2, 4, 85, 71, 85, + 32, 16, 52, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 35, 86, 12, 132, 1, 3, + 82, 69, 67, 252, 151, 1, 3, 76, 79, 67, 242, 227, 2, 83, 165, 116, 13, + 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, 69, 73, 86, + 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 159, 39, 32, 2, 243, + 221, 2, 82, 2, 11, 67, 2, 211, 188, 3, 79, 2, 247, 231, 2, 73, 200, 1, + 162, 1, 65, 40, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, + 84, 32, 164, 2, 2, 76, 69, 156, 4, 5, 83, 73, 71, 78, 32, 198, 2, 86, + 175, 136, 3, 68, 2, 11, 73, 2, 245, 189, 4, 2, 32, 76, 14, 74, 84, 40, 2, + 79, 78, 81, 10, 90, 69, 82, 79, 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, + 69, 13, 2, 87, 79, 4, 11, 69, 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, + 21, 4, 69, 86, 69, 78, 2, 17, 2, 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, + 82, 83, 32, 79, 70, 2, 201, 22, 2, 32, 70, 114, 44, 5, 84, 84, 69, 82, + 32, 159, 187, 4, 78, 112, 214, 1, 68, 54, 76, 38, 78, 106, 82, 34, 84, + 242, 144, 1, 65, 126, 86, 186, 24, 85, 158, 144, 1, 79, 182, 56, 73, 202, + 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, + 61, 72, 2, 77, 3, 89, 10, 234, 179, 4, 68, 254, 68, 72, 2, 90, 187, 2, + 65, 6, 198, 244, 2, 76, 167, 134, 2, 65, 10, 42, 65, 230, 247, 4, 71, 2, + 78, 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 163, 37, 76, 6, + 194, 89, 82, 159, 160, 4, 65, 10, 138, 178, 4, 84, 254, 68, 72, 2, 83, + 187, 2, 65, 20, 86, 67, 194, 1, 83, 248, 35, 3, 84, 85, 85, 142, 108, 65, + 170, 1, 78, 203, 162, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 219, 145, 1, 65, 4, 70, 65, 185, 250, 3, 11, 67, 65, 78, 68, 82, 65, + 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 147, 250, 3, 65, + 2, 11, 73, 2, 169, 189, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, + 83, 73, 71, 78, 32, 30, 186, 145, 1, 65, 106, 86, 214, 20, 85, 158, 144, + 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, + 67, 81, 85, 69, 150, 221, 2, 71, 219, 151, 2, 84, 2, 11, 84, 2, 25, 4, + 32, 65, 78, 68, 2, 251, 239, 2, 32, 2, 219, 221, 4, 66, 162, 1, 210, 2, + 65, 134, 1, 66, 174, 1, 67, 130, 3, 68, 198, 3, 69, 154, 2, 70, 182, 1, + 71, 198, 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, + 75, 98, 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, + 1, 14, 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, + 65, 202, 94, 85, 143, 234, 3, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 132, + 21, 2, 68, 86, 157, 175, 1, 5, 71, 71, 82, 65, 86, 2, 183, 176, 1, 85, 2, + 183, 173, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, + 67, 72, 221, 230, 3, 3, 65, 82, 82, 2, 153, 159, 4, 3, 79, 76, 85, 2, 11, + 73, 2, 133, 157, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 102, + 79, 211, 248, 1, 69, 2, 219, 186, 3, 65, 6, 26, 69, 143, 206, 3, 85, 4, + 26, 68, 207, 215, 3, 78, 2, 21, 3, 32, 77, 79, 2, 243, 238, 2, 85, 12, + 28, 3, 77, 80, 76, 35, 78, 4, 242, 17, 73, 143, 139, 4, 69, 8, 32, 4, 83, + 84, 65, 78, 23, 84, 2, 183, 219, 4, 67, 6, 46, 69, 20, 3, 82, 65, 82, + 211, 221, 3, 65, 2, 247, 155, 4, 78, 2, 171, 129, 3, 73, 20, 80, 4, 65, + 82, 75, 69, 22, 69, 206, 1, 73, 74, 85, 237, 204, 4, 3, 79, 85, 66, 2, + 199, 154, 4, 78, 6, 132, 1, 19, 70, 69, 67, 84, 73, 86, 69, 78, 69, 83, + 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, 80, 133, 7, 6, 67, 73, 83, 73, + 86, 69, 2, 11, 79, 2, 211, 153, 4, 82, 2, 17, 2, 65, 82, 2, 143, 10, 84, + 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, 77, 161, 12, 4, 86, 69, 82, 71, 2, + 11, 84, 2, 171, 213, 3, 73, 4, 140, 1, 2, 73, 78, 243, 150, 4, 77, 14, + 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, 4, 88, 72, 65, 85, 236, 106, 3, + 84, 69, 82, 239, 242, 2, 65, 2, 189, 164, 4, 3, 73, 83, 72, 6, 80, 4, 68, + 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, 233, 2, 4, 67, 79, 85, 78, 2, + 175, 238, 3, 79, 2, 183, 163, 4, 69, 2, 135, 150, 4, 83, 12, 70, 79, 76, + 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, 147, 181, 4, 76, 4, 18, 76, 35, 83, + 2, 225, 148, 4, 3, 76, 79, 87, 2, 205, 4, 2, 84, 69, 4, 142, 150, 3, 32, + 211, 56, 78, 10, 96, 8, 65, 84, 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, + 82, 69, 65, 65, 5, 85, 65, 82, 68, 69, 5, 227, 228, 3, 32, 2, 41, 8, 73, + 78, 71, 32, 84, 79, 32, 77, 2, 211, 174, 4, 69, 2, 75, 84, 4, 48, 2, 65, + 82, 33, 6, 79, 76, 68, 73, 78, 71, 2, 11, 68, 2, 187, 204, 3, 78, 2, 11, + 32, 2, 195, 216, 1, 66, 4, 26, 67, 211, 164, 4, 78, 2, 143, 5, 82, 4, 60, + 7, 69, 69, 80, 73, 78, 71, 32, 225, 140, 4, 2, 73, 78, 2, 11, 83, 2, 151, + 221, 2, 77, 6, 18, 65, 107, 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, + 82, 32, 77, 79, 68, 2, 243, 143, 4, 82, 2, 187, 215, 3, 69, 2, 139, 144, + 4, 71, 6, 26, 65, 30, 69, 47, 73, 2, 153, 143, 4, 2, 83, 83, 2, 11, 65, + 2, 11, 83, 2, 191, 191, 3, 85, 2, 11, 82, 2, 171, 153, 4, 69, 4, 204, + 162, 3, 7, 78, 32, 84, 72, 69, 32, 86, 139, 107, 80, 8, 54, 65, 64, 4, + 69, 78, 69, 84, 165, 98, 2, 85, 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, + 69, 2, 183, 142, 4, 82, 2, 195, 177, 1, 82, 12, 30, 69, 157, 1, 2, 73, + 84, 10, 34, 76, 22, 83, 231, 186, 4, 65, 2, 251, 204, 1, 69, 6, 18, 73, + 51, 80, 4, 30, 68, 137, 1, 2, 83, 84, 2, 147, 1, 69, 2, 11, 79, 2, 199, + 211, 3, 78, 2, 255, 210, 3, 85, 10, 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, + 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, 211, 158, 4, 78, 2, 183, 138, 4, 75, + 6, 42, 79, 237, 219, 2, 4, 82, 69, 78, 71, 4, 26, 80, 131, 196, 4, 86, 2, + 11, 80, 2, 211, 166, 3, 65, 2, 39, 83, 4, 26, 73, 231, 183, 4, 84, 2, + 147, 137, 4, 84, 224, 2, 86, 65, 180, 31, 2, 69, 82, 246, 1, 73, 206, 1, + 79, 100, 3, 82, 69, 69, 179, 8, 85, 146, 2, 44, 4, 65, 78, 65, 32, 145, + 10, 2, 73, 32, 100, 122, 65, 38, 69, 76, 7, 76, 69, 84, 84, 69, 82, 32, + 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, 85, 66, 85, 45, 2, 83, 85, 4, 184, + 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, 3, 69, 66, 69, 245, 7, 4, 89, 66, + 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, 42, 68, 82, 70, 2, 81, 14, 71, 50, + 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, 20, 2, 67, 72, 2, 74, 2, 80, 18, + 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, 90, 130, 225, 2, 66, 2, 82, 3, + 89, 4, 252, 1, 2, 76, 73, 191, 174, 4, 73, 6, 30, 65, 29, 3, 72, 65, 65, + 4, 218, 2, 65, 255, 1, 86, 2, 227, 175, 4, 76, 2, 123, 65, 6, 110, 65, + 86, 78, 245, 173, 4, 3, 72, 65, 73, 4, 178, 228, 2, 72, 147, 237, 1, 65, + 4, 26, 65, 247, 227, 2, 72, 2, 11, 65, 2, 199, 174, 4, 70, 4, 18, 65, 35, + 72, 2, 11, 65, 2, 147, 174, 4, 77, 2, 211, 1, 65, 4, 224, 173, 4, 2, 79, + 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, 72, 23, 69, 2, 163, 16, 68, 4, 18, + 69, 87, 65, 2, 243, 172, 4, 69, 10, 62, 65, 16, 3, 72, 65, 65, 190, 225, + 2, 84, 203, 239, 1, 79, 2, 131, 1, 86, 5, 155, 172, 4, 76, 2, 17, 2, 65, + 65, 2, 243, 171, 4, 86, 6, 26, 65, 175, 208, 4, 79, 4, 26, 86, 147, 208, + 4, 65, 2, 21, 3, 73, 89, 65, 2, 159, 188, 4, 78, 6, 48, 3, 65, 66, 79, + 14, 66, 1, 3, 79, 66, 79, 2, 23, 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, + 167, 187, 4, 76, 2, 211, 206, 3, 75, 174, 1, 26, 67, 159, 224, 2, 68, + 154, 1, 132, 1, 9, 72, 65, 82, 65, 67, 84, 69, 82, 32, 245, 162, 4, 17, + 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 66, 65, 152, + 1, 176, 2, 6, 66, 79, 32, 66, 65, 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, + 3, 68, 79, 32, 44, 2, 70, 79, 76, 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, + 1, 77, 252, 1, 7, 65, 78, 71, 75, 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, + 82, 70, 83, 238, 2, 84, 224, 2, 3, 87, 79, 32, 34, 89, 145, 226, 3, 2, + 79, 32, 2, 143, 12, 77, 8, 204, 210, 2, 2, 65, 78, 170, 167, 1, 73, 155, + 58, 79, 4, 164, 165, 4, 3, 67, 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, + 2, 78, 71, 4, 219, 209, 2, 65, 2, 147, 130, 3, 77, 4, 40, 4, 78, 79, 75, + 72, 223, 199, 4, 72, 2, 239, 197, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, + 79, 32, 75, 12, 26, 32, 239, 206, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, + 65, 75, 8, 158, 9, 87, 150, 171, 3, 85, 254, 67, 79, 255, 59, 65, 2, 135, + 239, 3, 72, 8, 76, 2, 79, 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, + 139, 187, 4, 85, 4, 32, 2, 67, 72, 207, 245, 3, 76, 2, 171, 164, 4, 85, + 16, 28, 2, 65, 73, 235, 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, + 89, 10, 76, 4, 67, 72, 65, 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 191, + 193, 4, 69, 2, 11, 84, 2, 11, 65, 2, 195, 194, 4, 87, 2, 151, 177, 3, 75, + 4, 222, 165, 4, 72, 159, 11, 82, 2, 203, 2, 75, 2, 193, 154, 4, 2, 65, + 77, 8, 60, 3, 71, 79, 32, 32, 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, + 78, 2, 211, 158, 4, 71, 2, 237, 166, 4, 2, 65, 72, 4, 182, 243, 3, 69, + 215, 79, 85, 12, 68, 6, 65, 73, 89, 65, 78, 78, 22, 72, 249, 159, 4, 3, + 79, 32, 80, 2, 203, 174, 4, 79, 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, + 255, 156, 4, 72, 6, 44, 2, 80, 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, + 240, 3, 85, 155, 1, 65, 4, 32, 2, 79, 32, 227, 192, 4, 85, 2, 11, 82, 2, + 155, 190, 4, 85, 36, 44, 4, 65, 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, + 65, 130, 1, 85, 134, 184, 2, 73, 206, 134, 2, 69, 3, 79, 13, 64, 6, 73, + 32, 77, 65, 73, 77, 142, 191, 4, 65, 2, 69, 3, 77, 4, 26, 65, 191, 247, + 2, 85, 2, 11, 76, 2, 139, 171, 4, 65, 9, 186, 254, 3, 69, 151, 64, 85, 8, + 24, 2, 82, 85, 23, 83, 2, 187, 170, 4, 83, 6, 230, 155, 4, 65, 226, 31, + 85, 187, 2, 79, 18, 30, 72, 133, 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, + 65, 75, 21, 2, 79, 32, 2, 175, 169, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, + 79, 78, 84, 20, 4, 80, 72, 85, 84, 13, 2, 84, 72, 2, 163, 157, 4, 72, 2, + 123, 72, 8, 34, 65, 234, 234, 3, 79, 3, 85, 4, 158, 244, 2, 72, 183, 199, + 1, 78, 4, 38, 84, 165, 140, 2, 3, 80, 65, 84, 2, 147, 156, 4, 65, 2, 11, + 87, 2, 243, 233, 2, 65, 6, 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, + 75, 2, 231, 242, 2, 75, 4, 154, 233, 3, 73, 215, 77, 65, 10, 30, 69, 145, + 1, 2, 77, 79, 6, 18, 32, 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, + 84, 32, 69, 88, 201, 164, 3, 4, 69, 88, 73, 83, 2, 147, 113, 73, 2, 171, + 152, 3, 79, 4, 46, 77, 157, 145, 3, 5, 68, 89, 78, 65, 77, 2, 187, 129, + 3, 69, 10, 18, 78, 95, 82, 8, 26, 32, 191, 175, 3, 75, 6, 26, 83, 255, + 177, 2, 71, 4, 218, 165, 2, 65, 211, 212, 1, 80, 2, 21, 3, 68, 32, 80, 2, + 25, 4, 76, 65, 67, 69, 2, 17, 2, 32, 77, 2, 143, 163, 3, 69, 4, 56, 4, + 85, 71, 72, 84, 173, 162, 3, 4, 78, 71, 32, 83, 2, 185, 164, 1, 5, 32, + 66, 65, 76, 76, 46, 22, 32, 199, 3, 45, 30, 154, 1, 68, 90, 76, 114, 82, + 130, 131, 1, 66, 86, 67, 242, 5, 83, 170, 149, 1, 80, 253, 21, 15, 78, + 69, 84, 87, 79, 82, 75, 69, 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, + 77, 69, 78, 83, 73, 79, 78, 65, 76, 219, 134, 1, 79, 2, 131, 146, 3, 32, + 6, 76, 12, 73, 78, 69, 83, 32, 67, 79, 78, 86, 69, 82, 71, 237, 127, 2, + 69, 70, 4, 217, 135, 4, 3, 73, 78, 71, 10, 36, 4, 65, 89, 83, 32, 147, + 127, 73, 8, 230, 153, 2, 66, 222, 155, 1, 65, 242, 81, 76, 31, 82, 16, + 44, 2, 68, 32, 254, 3, 80, 131, 133, 1, 69, 12, 196, 2, 12, 84, 79, 80, + 45, 76, 73, 71, 72, 84, 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, + 71, 72, 84, 69, 68, 32, 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, + 76, 73, 71, 72, 84, 69, 68, 32, 85, 80, 209, 223, 1, 25, 66, 79, 84, 84, + 79, 77, 45, 76, 73, 71, 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, + 82, 68, 83, 6, 76, 4, 76, 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, + 82, 68, 83, 32, 2, 11, 87, 2, 25, 4, 65, 82, 68, 83, 2, 181, 200, 1, 2, + 32, 69, 4, 154, 200, 1, 69, 139, 23, 65, 2, 129, 239, 3, 5, 69, 82, 45, + 69, 77, 8, 34, 77, 85, 4, 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, + 85, 149, 182, 2, 3, 68, 79, 87, 2, 199, 165, 3, 80, 4, 212, 128, 3, 10, + 32, 67, 76, 79, 85, 68, 32, 65, 78, 68, 141, 45, 2, 83, 84, 230, 5, 204, + 1, 6, 66, 69, 84, 65, 78, 32, 240, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, + 70, 73, 78, 65, 71, 72, 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 248, 2, 2, + 77, 69, 56, 2, 78, 89, 78, 82, 223, 154, 3, 67, 160, 3, 228, 2, 18, 65, + 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, + 1, 18, 67, 65, 78, 84, 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, + 78, 32, 208, 1, 6, 68, 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, + 32, 75, 72, 65, 26, 76, 204, 3, 5, 77, 65, 82, 75, 32, 246, 18, 83, 181, + 15, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 129, 41, + 6, 45, 75, 72, 89, 85, 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, + 229, 21, 13, 71, 82, 65, 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 159, + 32, 85, 8, 144, 1, 5, 72, 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, + 7, 83, 66, 85, 66, 32, 45, 67, 189, 255, 3, 8, 67, 65, 78, 71, 32, 84, + 69, 45, 2, 17, 2, 32, 66, 2, 231, 144, 3, 69, 2, 251, 154, 3, 72, 40, + 156, 182, 2, 4, 72, 65, 76, 70, 82, 70, 30, 83, 42, 84, 62, 90, 130, 83, + 78, 14, 79, 199, 110, 69, 5, 245, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, + 69, 82, 32, 181, 2, 13, 79, 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, + 32, 88, 226, 1, 75, 50, 82, 208, 213, 3, 10, 70, 73, 88, 69, 68, 45, 70, + 79, 82, 77, 202, 1, 68, 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, + 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, + 65, 8, 134, 158, 4, 83, 14, 72, 2, 75, 187, 2, 65, 4, 226, 157, 4, 82, + 187, 2, 65, 4, 248, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, 68, 80, 250, 2, + 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, 172, 2, 13, 89, + 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, 71, 212, 2, 9, + 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, 246, 1, 78, + 220, 1, 2, 80, 65, 30, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 233, + 250, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, 83, 72, 69, 71, + 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, 27, 83, 4, 142, 1, + 71, 67, 89, 6, 34, 75, 229, 21, 3, 68, 85, 83, 4, 56, 6, 65, 45, 32, 83, + 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, 2, 41, 8, 73, 32, + 77, 71, 79, 32, 82, 71, 2, 163, 211, 2, 89, 2, 221, 2, 2, 73, 71, 12, 84, + 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, 79, 83, 73, 78, 71, + 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, 71, 83, 32, 97, 12, + 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, 42, 66, 37, 6, 77, 69, + 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, 2, 231, 22, 71, 2, 225, + 3, 3, 32, 83, 72, 2, 137, 142, 2, 2, 32, 77, 4, 68, 13, 66, 82, 68, 65, + 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 213, 5, 11, 73, 71, 32, 77, + 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, 69, 82, 32, 141, 2, 8, 85, 71, + 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, 71, 32, 77, 71, 79, 32, 211, + 207, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, 9, 84, 82, 85, 78, 67, 65, 84, + 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, 67, 245, 2, 10, 71, 84, 69, 82, + 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, 68, 2, 203, 145, 4, 32, 4, 21, + 3, 32, 71, 89, 4, 130, 196, 3, 79, 135, 18, 65, 2, 17, 2, 65, 76, 2, 213, + 238, 3, 2, 65, 78, 6, 92, 6, 73, 84, 73, 65, 76, 32, 165, 204, 3, 11, 84, + 69, 82, 83, 89, 76, 76, 65, 66, 73, 67, 4, 68, 13, 66, 82, 68, 65, 32, + 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 53, 11, 73, 71, 32, 77, 71, 79, + 32, 77, 68, 85, 78, 2, 151, 240, 3, 32, 10, 72, 10, 71, 65, 83, 32, 66, + 90, 85, 78, 71, 32, 77, 4, 89, 73, 83, 32, 4, 56, 3, 83, 71, 79, 245, + 237, 3, 5, 78, 89, 73, 32, 90, 2, 151, 10, 82, 6, 44, 5, 84, 83, 72, 69, + 71, 163, 201, 3, 83, 5, 147, 201, 3, 32, 2, 233, 234, 3, 2, 76, 85, 4, + 212, 200, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, + 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 224, 199, 3, 4, 66, 82, 85, + 76, 39, 72, 6, 32, 4, 82, 65, 73, 76, 55, 83, 2, 225, 7, 9, 73, 78, 71, + 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 165, 198, 3, 3, 72, + 69, 71, 2, 255, 231, 3, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, + 66, 74, 79, 73, 78, 69, 68, 32, 143, 4, 89, 42, 176, 1, 4, 71, 82, 85, + 32, 96, 2, 76, 67, 30, 77, 36, 11, 78, 89, 73, 32, 90, 76, 65, 32, 78, + 65, 65, 22, 82, 252, 2, 2, 89, 65, 130, 4, 73, 165, 4, 5, 83, 78, 65, 32, + 76, 4, 40, 3, 67, 65, 78, 1, 3, 77, 69, 68, 2, 25, 4, 32, 82, 71, 89, 2, + 169, 4, 2, 73, 78, 4, 238, 3, 73, 179, 4, 69, 4, 136, 4, 2, 65, 82, 231, + 3, 67, 2, 159, 228, 3, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, + 69, 83, 32, 83, 85, 32, 78, 71, 65, 137, 192, 3, 6, 78, 65, 77, 32, 66, + 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 129, + 161, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, + 68, 6, 46, 67, 228, 233, 1, 2, 78, 89, 251, 116, 83, 2, 251, 133, 4, 73, + 2, 147, 228, 2, 75, 2, 147, 154, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, + 21, 3, 32, 82, 84, 2, 11, 65, 2, 255, 199, 3, 71, 2, 17, 2, 32, 84, 2, + 187, 241, 2, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, + 73, 71, 78, 32, 88, 220, 1, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, + 234, 185, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, + 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, + 187, 2, 65, 6, 11, 32, 6, 166, 128, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, + 77, 33, 3, 76, 67, 69, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 77, 2, + 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 231, + 185, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, + 2, 227, 239, 3, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, + 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 211, + 218, 3, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, + 4, 78, 89, 73, 83, 2, 217, 209, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, + 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 187, 182, 2, 68, 2, 251, 88, + 32, 5, 245, 195, 2, 6, 32, 82, 71, 89, 65, 32, 30, 120, 8, 82, 69, 86, + 69, 82, 83, 69, 68, 182, 24, 86, 214, 20, 85, 158, 144, 1, 79, 182, 56, + 73, 186, 198, 1, 69, 223, 61, 65, 4, 213, 245, 1, 2, 32, 73, 2, 25, 4, + 32, 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 223, 145, 2, 73, 118, 216, 1, 9, + 67, 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, + 236, 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, + 32, 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 211, 187, + 3, 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, + 69, 77, 89, 32, 89, 65, 30, 84, 223, 1, 89, 4, 84, 6, 89, 69, 82, 32, 89, + 65, 241, 246, 3, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 251, 246, 3, + 71, 4, 246, 247, 3, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, + 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 239, + 246, 3, 90, 16, 62, 71, 190, 2, 75, 210, 242, 3, 90, 62, 78, 86, 72, 3, + 81, 4, 154, 246, 3, 72, 3, 78, 86, 54, 65, 210, 2, 69, 250, 242, 3, 73, + 2, 79, 3, 85, 77, 190, 1, 68, 30, 71, 2, 75, 14, 66, 2, 72, 22, 83, 30, + 84, 30, 90, 242, 15, 82, 150, 226, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, + 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 227, 243, + 3, 72, 7, 11, 72, 5, 223, 243, 3, 72, 7, 202, 243, 3, 72, 3, 83, 7, 174, + 243, 3, 72, 3, 84, 7, 146, 243, 3, 72, 3, 90, 5, 247, 242, 3, 89, 2, 189, + 137, 3, 4, 65, 76, 73, 90, 2, 135, 177, 3, 82, 6, 76, 2, 69, 82, 21, 13, + 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 211, 180, 3, 32, + 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 167, 43, 76, 19, 11, 32, 16, 72, 8, + 79, 80, 69, 82, 65, 84, 79, 82, 229, 1, 5, 87, 73, 84, 72, 32, 11, 11, + 32, 8, 38, 65, 97, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, + 11, 32, 4, 26, 76, 231, 207, 2, 82, 2, 249, 207, 2, 2, 69, 70, 4, 26, 82, + 219, 238, 2, 68, 2, 17, 2, 73, 83, 2, 173, 161, 2, 3, 73, 78, 71, 6, 26, + 68, 183, 196, 1, 82, 4, 11, 79, 4, 207, 213, 1, 84, 6, 22, 82, 155, 98, + 83, 2, 11, 32, 2, 175, 133, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, 79, + 32, 68, 2, 249, 63, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, 32, + 156, 9, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 223, 210, 2, 69, + 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 232, 2, 2, 83, 73, + 140, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 247, 1, 68, + 216, 149, 1, 2, 71, 86, 239, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, 82, + 69, 86, 73, 2, 203, 16, 65, 2, 215, 214, 3, 74, 94, 202, 1, 65, 38, 68, + 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, + 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, + 82, 2, 89, 186, 2, 69, 3, 79, 9, 206, 232, 3, 65, 2, 73, 3, 85, 8, 246, + 160, 3, 68, 254, 68, 72, 187, 2, 65, 8, 202, 160, 3, 84, 254, 68, 72, + 187, 2, 65, 10, 238, 3, 79, 231, 227, 3, 65, 12, 21, 3, 71, 78, 32, 12, + 42, 65, 74, 67, 98, 78, 203, 162, 3, 86, 4, 26, 86, 247, 161, 3, 78, 2, + 21, 3, 65, 71, 82, 2, 255, 238, 1, 65, 2, 11, 65, 2, 11, 78, 2, 25, 4, + 68, 82, 65, 66, 2, 11, 73, 2, 11, 78, 2, 243, 192, 3, 68, 2, 11, 85, 2, + 239, 192, 3, 75, 30, 78, 65, 38, 83, 70, 86, 214, 20, 85, 210, 200, 1, + 73, 206, 134, 2, 69, 3, 79, 6, 214, 228, 3, 65, 2, 73, 3, 85, 4, 25, 4, + 72, 79, 82, 84, 4, 11, 32, 4, 138, 228, 3, 69, 3, 79, 8, 11, 79, 8, 33, + 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 139, 231, 2, 76, 5, 163, 227, 3, + 82, 4, 128, 174, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 227, 24, 69, 138, + 1, 210, 1, 77, 18, 78, 34, 79, 80, 2, 80, 32, 152, 9, 23, 82, 84, 79, 73, + 83, 69, 32, 83, 72, 69, 76, 76, 32, 66, 82, 65, 67, 75, 69, 84, 69, 68, + 32, 130, 4, 84, 198, 246, 1, 73, 169, 67, 5, 75, 89, 79, 32, 84, 2, 239, + 20, 65, 2, 11, 71, 2, 135, 202, 3, 85, 6, 32, 2, 84, 72, 131, 212, 2, 76, + 5, 11, 66, 2, 11, 82, 2, 239, 250, 1, 85, 40, 140, 1, 4, 65, 82, 67, 32, + 162, 2, 67, 44, 2, 72, 65, 218, 1, 80, 126, 76, 22, 82, 138, 1, 83, 38, + 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, 19, 65, 78, 84, 73, 67, + 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 73, 21, 67, 76, + 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, + 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 80, 2, 143, 218, 1, 76, + 2, 11, 77, 2, 227, 217, 1, 73, 2, 11, 85, 2, 213, 137, 3, 3, 82, 76, 89, + 12, 36, 3, 76, 70, 32, 247, 219, 3, 84, 10, 50, 66, 46, 76, 26, 82, 114, + 83, 143, 214, 1, 73, 2, 11, 76, 2, 145, 140, 2, 3, 65, 67, 75, 2, 11, 69, + 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 33, 6, + 65, 82, 69, 78, 84, 72, 2, 167, 189, 1, 69, 2, 11, 69, 2, 11, 67, 2, 11, + 84, 2, 11, 73, 2, 191, 227, 1, 79, 6, 41, 2, 69, 70, 6, 21, 3, 73, 71, + 72, 6, 17, 2, 84, 32, 6, 42, 67, 249, 133, 3, 4, 72, 65, 76, 70, 4, 26, + 79, 191, 223, 2, 82, 2, 211, 154, 3, 82, 2, 185, 133, 3, 4, 81, 85, 65, + 82, 2, 11, 79, 2, 149, 133, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, + 69, 76, 76, 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 141, 218, + 2, 4, 82, 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, + 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, + 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, + 40, 2, 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 172, 2, 56, 4, + 30, 50, 141, 1, 2, 66, 56, 2, 159, 142, 3, 68, 6, 48, 2, 50, 53, 22, 53, + 249, 172, 2, 2, 55, 50, 2, 235, 211, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, + 21, 2, 54, 68, 2, 167, 211, 3, 57, 2, 147, 211, 3, 55, 2, 11, 82, 2, 163, + 149, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 235, + 216, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 229, 2, 6, 83, 73, 71, + 78, 32, 82, 60, 202, 1, 66, 102, 73, 22, 78, 138, 144, 1, 69, 198, 71, + 67, 170, 183, 1, 65, 222, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, + 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, 52, + 7, 82, 69, 65, 84, 72, 89, 32, 203, 207, 3, 65, 10, 42, 73, 158, 144, 1, + 69, 175, 168, 2, 65, 5, 155, 207, 3, 85, 4, 206, 204, 3, 71, 187, 2, 65, + 2, 17, 2, 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 135, 165, 2, 84, + 98, 46, 65, 138, 2, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, + 69, 34, 77, 32, 2, 78, 83, 243, 252, 2, 73, 4, 222, 200, 1, 75, 239, 203, + 1, 84, 2, 153, 16, 4, 32, 77, 65, 82, 5, 11, 32, 2, 175, 170, 2, 67, 4, + 44, 2, 80, 76, 33, 5, 86, 69, 82, 83, 65, 2, 11, 85, 2, 147, 173, 3, 84, + 2, 231, 48, 76, 66, 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, + 69, 77, 66, 20, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, + 69, 32, 255, 244, 2, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, + 32, 10, 44, 6, 32, 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, + 73, 70, 83, 32, 65, 84, 54, 85, 150, 136, 2, 82, 147, 64, 68, 2, 17, 2, + 32, 66, 2, 145, 184, 3, 3, 79, 84, 84, 2, 173, 47, 2, 78, 68, 2, 177, + 168, 2, 4, 72, 69, 65, 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, + 34, 82, 131, 235, 1, 66, 2, 11, 79, 2, 155, 171, 3, 83, 2, 11, 85, 2, + 199, 137, 3, 76, 2, 231, 180, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, + 46, 84, 42, 87, 227, 179, 1, 70, 2, 11, 65, 2, 171, 199, 1, 82, 2, 11, + 69, 2, 163, 217, 1, 65, 2, 163, 150, 3, 65, 2, 21, 3, 79, 85, 78, 2, 159, + 154, 2, 84, 2, 17, 2, 72, 85, 2, 183, 161, 1, 78, 4, 198, 142, 2, 65, + 239, 40, 73, 30, 176, 2, 3, 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, + 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, + 20, 13, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, + 79, 76, 73, 68, 85, 83, 32, 66, 232, 8, 2, 68, 65, 224, 168, 1, 6, 78, + 69, 83, 84, 69, 68, 163, 11, 73, 2, 145, 24, 2, 79, 78, 4, 34, 68, 33, 4, + 84, 82, 73, 80, 2, 17, 2, 79, 85, 2, 11, 66, 2, 133, 145, 3, 2, 76, 69, + 4, 218, 190, 1, 76, 143, 107, 82, 2, 171, 171, 1, 73, 8, 42, 66, 74, 68, + 198, 161, 1, 82, 111, 87, 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, + 69, 2, 251, 19, 76, 2, 25, 4, 69, 76, 73, 77, 2, 183, 137, 2, 73, 10, 24, + 2, 76, 76, 35, 80, 5, 229, 188, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, + 76, 32, 191, 173, 3, 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 191, + 187, 3, 78, 2, 167, 217, 1, 73, 6, 18, 69, 79, 77, 5, 205, 153, 3, 14, + 32, 76, 73, 71, 72, 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 195, 136, 3, + 80, 56, 80, 3, 71, 82, 73, 22, 82, 148, 165, 2, 5, 77, 66, 76, 69, 82, + 151, 150, 1, 76, 2, 255, 181, 2, 75, 50, 50, 75, 84, 4, 78, 69, 68, 32, + 171, 154, 2, 84, 4, 48, 6, 73, 83, 72, 32, 76, 73, 167, 170, 3, 69, 2, + 11, 82, 2, 247, 180, 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, + 32, 40, 7, 87, 72, 73, 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, + 2, 79, 75, 30, 83, 185, 146, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, + 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 147, 152, 2, + 78, 2, 17, 2, 80, 69, 2, 11, 82, 2, 179, 171, 2, 83, 4, 18, 80, 23, 83, + 2, 223, 129, 2, 69, 2, 207, 131, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, + 167, 106, 79, 6, 30, 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 147, 250, 2, + 71, 4, 129, 203, 1, 3, 84, 32, 84, 6, 162, 2, 82, 183, 174, 2, 84, 2, + 177, 209, 1, 2, 32, 72, 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, + 70, 32, 67, 65, 80, 73, 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, + 79, 85, 6, 254, 181, 3, 71, 2, 76, 3, 89, 2, 11, 77, 2, 203, 229, 2, 73, + 2, 11, 76, 2, 163, 180, 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, + 2, 87, 69, 2, 53, 11, 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, + 2, 32, 76, 2, 251, 178, 2, 69, 36, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, + 82, 73, 63, 79, 2, 17, 2, 76, 86, 2, 209, 159, 1, 3, 69, 32, 80, 2, 17, + 2, 71, 72, 2, 153, 93, 6, 84, 87, 65, 82, 68, 83, 32, 30, 32, 249, 9, 2, + 45, 69, 30, 200, 2, 24, 65, 83, 84, 69, 82, 73, 83, 75, 83, 32, 65, 76, + 73, 71, 78, 69, 68, 32, 86, 69, 82, 84, 73, 67, 34, 66, 86, 67, 116, 3, + 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, + 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, 64, 8, 76, 79, 71, 73, 67, + 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 119, 83, 2, 11, 65, 2, 171, 140, 3, + 76, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, 11, 79, 2, 147, + 166, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, 86, 69, 32, 69, + 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 195, 240, 2, 78, 6, 18, 32, 55, + 83, 4, 22, 76, 131, 1, 80, 2, 225, 137, 1, 2, 69, 65, 2, 53, 11, 32, 79, + 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, 2, 17, 2, 32, + 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 239, 219, 2, 65, 2, 11, 69, 2, 11, + 65, 2, 243, 151, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, 67, 65, 76, 4, + 11, 32, 4, 214, 157, 2, 65, 147, 85, 79, 2, 17, 2, 32, 83, 2, 21, 3, 81, + 85, 65, 2, 139, 151, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, 2, 11, 82, 2, + 11, 32, 2, 11, 79, 2, 11, 80, 2, 183, 17, 69, 2, 11, 69, 2, 45, 9, 78, + 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 207, + 204, 1, 78, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, 66, 66, 2, 239, + 148, 2, 76, 2, 11, 77, 2, 133, 195, 1, 2, 32, 68, 140, 3, 140, 1, 8, 71, + 65, 82, 73, 84, 73, 67, 32, 148, 6, 7, 77, 66, 82, 69, 76, 76, 65, 166, + 1, 78, 154, 8, 80, 174, 148, 1, 82, 135, 131, 2, 83, 62, 48, 7, 76, 69, + 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, 22, 66, + 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, 83, 74, + 84, 74, 89, 22, 90, 186, 161, 2, 78, 178, 91, 80, 246, 5, 87, 202, 12, + 77, 174, 18, 73, 3, 85, 4, 26, 76, 143, 213, 2, 73, 2, 167, 162, 3, 80, + 2, 255, 255, 2, 69, 4, 26, 69, 247, 154, 2, 72, 2, 207, 255, 2, 76, 4, + 214, 248, 1, 72, 153, 137, 1, 2, 65, 77, 5, 163, 161, 3, 84, 4, 186, 162, + 2, 65, 215, 126, 72, 2, 11, 65, 2, 151, 254, 2, 77, 2, 203, 171, 1, 83, + 8, 38, 65, 146, 163, 2, 72, 215, 90, 83, 4, 246, 139, 3, 68, 239, 13, 77, + 6, 38, 72, 218, 133, 3, 69, 175, 28, 79, 2, 11, 65, 2, 179, 231, 1, 78, + 2, 207, 219, 2, 79, 4, 130, 253, 2, 69, 207, 36, 85, 2, 21, 3, 79, 82, + 68, 2, 25, 4, 32, 68, 73, 86, 2, 139, 125, 73, 7, 11, 32, 4, 84, 4, 79, + 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, 68, 82, 79, + 2, 11, 82, 2, 11, 79, 2, 143, 146, 2, 85, 2, 139, 226, 2, 80, 32, 160, 1, + 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, 73, 241, 5, + 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, 82, 83, 72, + 73, 2, 227, 141, 2, 83, 2, 11, 84, 2, 155, 151, 2, 89, 4, 26, 69, 199, + 138, 2, 79, 2, 11, 82, 2, 151, 130, 3, 84, 22, 60, 2, 79, 78, 230, 3, 84, + 106, 86, 201, 43, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, 65, 66, 79, + 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, + 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, 32, 4, 52, 9, + 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, 73, 2, 229, 39, + 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 139, 202, 2, 78, 6, 66, 77, 58, + 79, 217, 224, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, 73, 2, 11, 78, + 2, 11, 85, 2, 223, 146, 2, 83, 2, 11, 86, 2, 229, 73, 2, 69, 82, 4, 18, + 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, 219, 223, 2, 82, + 2, 171, 133, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, 82, 69, 67, 89, + 67, 76, 2, 17, 2, 73, 78, 2, 199, 132, 2, 71, 2, 179, 132, 2, 80, 164, 2, + 170, 1, 32, 128, 8, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 248, 3, + 4, 80, 69, 82, 32, 168, 29, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, + 65, 82, 68, 83, 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 240, 1, 5, 66, 65, + 82, 66, 32, 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 89, 4, 84, + 65, 67, 75, 8, 60, 7, 32, 84, 72, 82, 79, 85, 71, 21, 4, 72, 69, 65, 68, + 2, 143, 197, 1, 72, 7, 11, 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, + 32, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 159, 134, 2, + 73, 2, 181, 254, 1, 2, 32, 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, + 72, 4, 57, 12, 84, 32, 68, 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, + 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, + 11, 80, 2, 207, 193, 2, 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, + 68, 18, 83, 202, 43, 87, 215, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, + 66, 2, 175, 135, 2, 65, 2, 209, 245, 1, 4, 76, 65, 67, 75, 2, 147, 44, + 79, 2, 131, 50, 65, 2, 11, 73, 2, 155, 98, 83, 2, 53, 11, 73, 71, 72, 84, + 32, 68, 73, 65, 71, 79, 78, 2, 225, 113, 4, 65, 76, 32, 69, 5, 29, 5, 32, + 87, 73, 84, 72, 2, 33, 6, 32, 67, 73, 82, 67, 76, 2, 159, 145, 2, 69, 14, + 96, 8, 77, 73, 76, 73, 84, 65, 82, 89, 20, 6, 83, 77, 65, 76, 76, 32, 22, + 65, 66, 82, 35, 84, 2, 37, 2, 32, 65, 4, 18, 65, 67, 82, 2, 11, 73, 2, + 11, 82, 2, 17, 2, 80, 76, 2, 179, 243, 1, 65, 2, 11, 69, 2, 199, 232, 1, + 68, 4, 37, 7, 82, 73, 65, 78, 71, 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, + 73, 84, 72, 32, 4, 18, 76, 27, 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, + 71, 72, 2, 11, 84, 2, 17, 2, 32, 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, + 66, 2, 11, 76, 2, 207, 219, 1, 65, 110, 172, 1, 4, 65, 78, 68, 32, 186, + 2, 66, 74, 70, 36, 5, 72, 65, 76, 70, 32, 148, 3, 5, 76, 69, 70, 84, 32, + 226, 5, 79, 60, 6, 82, 73, 71, 72, 84, 32, 218, 12, 83, 63, 84, 8, 34, + 76, 53, 4, 82, 73, 71, 72, 6, 48, 2, 69, 70, 149, 1, 5, 79, 87, 69, 82, + 32, 2, 53, 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 217, 25, + 19, 32, 84, 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, + 32, 81, 4, 28, 2, 84, 82, 179, 29, 79, 2, 205, 3, 7, 73, 65, 78, 71, 85, + 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 177, 208, 1, 3, 32, 83, + 67, 2, 17, 2, 73, 86, 2, 191, 22, 69, 12, 96, 5, 66, 76, 79, 67, 75, 108, + 8, 73, 78, 86, 69, 82, 83, 69, 32, 254, 19, 77, 227, 160, 1, 67, 5, 209, + 20, 23, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, + 73, 78, 86, 69, 82, 83, 69, 4, 100, 21, 77, 69, 68, 73, 85, 77, 32, 83, + 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, 87, 2, 11, 82, 2, + 141, 26, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, 2, 147, 179, 1, 69, + 34, 106, 66, 238, 3, 67, 54, 84, 176, 9, 13, 79, 82, 32, 76, 79, 87, 69, + 82, 32, 82, 73, 71, 72, 143, 1, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, + 73, 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, + 173, 6, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, + 70, 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, + 69, 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 169, 8, 18, + 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, + 6, 70, 76, 193, 6, 3, 85, 80, 80, 6, 34, 76, 153, 7, 3, 85, 80, 80, 2, + 217, 7, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 139, 218, 1, 32, 6, + 22, 79, 187, 12, 82, 2, 129, 12, 11, 32, 76, 79, 87, 69, 82, 32, 82, 73, + 71, 72, 4, 11, 78, 4, 17, 2, 69, 32, 4, 150, 15, 81, 139, 4, 69, 40, 128, + 1, 2, 66, 76, 186, 6, 68, 100, 12, 79, 82, 32, 76, 79, 87, 69, 82, 32, + 76, 69, 70, 106, 80, 38, 81, 104, 2, 83, 72, 51, 84, 22, 61, 13, 79, 67, + 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, 87, + 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, + 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, 246, + 3, 67, 211, 202, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, 84, + 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, 68, + 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, 76, + 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 171, 202, 2, 82, + 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, 43, + 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, 69, + 82, 32, 6, 34, 67, 42, 77, 171, 202, 2, 82, 2, 11, 69, 2, 241, 212, 1, 2, + 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 203, 166, 1, 69, 2, 65, 14, 82, 79, + 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 32, 87, 2, 225, 181, 1, 3, 72, + 73, 84, 2, 69, 15, 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, + 69, 84, 2, 11, 32, 2, 179, 162, 2, 83, 2, 11, 69, 2, 249, 69, 2, 78, 67, + 2, 81, 18, 85, 65, 68, 82, 65, 78, 84, 32, 67, 73, 82, 67, 85, 76, 65, + 82, 32, 65, 2, 163, 203, 1, 82, 4, 245, 97, 8, 65, 68, 79, 87, 69, 68, + 32, 87, 6, 18, 79, 107, 82, 2, 49, 10, 32, 76, 79, 87, 69, 82, 32, 76, + 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, 70, 2, 163, 108, 73, 4, 25, 4, 73, + 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, 171, 217, 2, 76, 2, 17, 2, 32, 77, + 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, 72, 2, 199, 98, 65, 2, 17, 2, 69, + 86, 2, 17, 2, 69, 78, 2, 117, 2, 32, 69, 6, 52, 2, 72, 82, 129, 1, 6, 82, + 73, 65, 78, 71, 85, 4, 21, 3, 69, 69, 32, 4, 18, 69, 35, 81, 2, 65, 5, + 73, 71, 72, 84, 72, 2, 33, 6, 85, 65, 82, 84, 69, 82, 2, 231, 4, 83, 2, + 45, 9, 76, 65, 82, 32, 79, 78, 69, 32, 81, 2, 165, 4, 6, 85, 65, 82, 84, + 69, 82, 2, 203, 175, 2, 78, 128, 1, 146, 1, 65, 174, 6, 66, 154, 1, 68, + 50, 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 142, 7, 80, + 173, 3, 6, 87, 72, 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, + 11, 67, 2, 207, 166, 2, 79, 31, 11, 32, 28, 134, 1, 65, 180, 1, 14, 76, + 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, + 72, 32, 214, 8, 70, 175, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, + 2, 17, 2, 32, 79, 2, 25, 4, 78, 69, 32, 69, 2, 29, 5, 73, 71, 72, 84, 72, + 2, 11, 32, 2, 11, 66, 2, 11, 76, 2, 239, 185, 1, 79, 2, 233, 200, 1, 3, + 79, 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 246, 12, + 84, 139, 59, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 131, 183, 2, 69, 2, + 11, 65, 2, 233, 23, 3, 82, 71, 69, 2, 205, 23, 5, 69, 68, 73, 85, 77, 2, + 25, 4, 79, 84, 67, 72, 2, 11, 69, 2, 247, 56, 68, 4, 11, 77, 4, 25, 4, + 65, 76, 76, 32, 4, 22, 69, 203, 22, 84, 2, 237, 22, 10, 81, 85, 73, 76, + 65, 84, 69, 82, 65, 76, 4, 11, 76, 4, 25, 4, 65, 67, 75, 32, 4, 44, 5, + 67, 73, 82, 67, 76, 143, 202, 1, 65, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, + 2, 229, 14, 2, 73, 84, 4, 22, 79, 187, 13, 65, 2, 169, 14, 2, 85, 66, 2, + 11, 73, 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 205, 200, 1, 2, 83, 84, + 20, 88, 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, + 66, 32, 255, 2, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, + 84, 10, 22, 32, 231, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, + 70, 175, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 193, 146, 1, + 23, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, + 72, 32, 66, 65, 82, 66, 6, 22, 32, 243, 7, 87, 4, 22, 70, 175, 5, 84, 2, + 229, 15, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 191, 196, 1, + 65, 2, 185, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 133, 9, 6, 85, 65, 68, + 82, 85, 80, 2, 195, 139, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 153, + 195, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 255, 8, 82, 36, 36, 2, 82, + 73, 225, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 151, 7, 80, 28, 52, + 8, 45, 72, 69, 65, 68, 69, 68, 32, 199, 13, 32, 26, 48, 5, 65, 82, 82, + 79, 87, 170, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, + 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 19, 87, 2, 37, 7, + 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 215, + 5, 45, 2, 187, 10, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, 76, + 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 57, 6, 68, + 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, 11, + 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, 11, + 87, 2, 135, 123, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, 82, + 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, 237, + 171, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, 65, 2, + 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 239, 151, 2, 87, + 2, 11, 76, 2, 227, 186, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, 68, 69, + 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, 69, 2, + 219, 185, 1, 68, 4, 42, 87, 129, 132, 1, 4, 70, 82, 79, 77, 2, 33, 6, 73, + 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, 82, + 79, 87, 72, 2, 169, 118, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, 133, + 4, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, 12, + 104, 8, 70, 82, 79, 77, 32, 66, 65, 82, 40, 11, 79, 78, 32, 80, 69, 68, + 69, 83, 84, 65, 76, 183, 1, 87, 5, 93, 7, 32, 87, 73, 84, 72, 32, 72, 7, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 18, 72, 43, 86, 2, 69, 7, 79, 82, 73, + 90, 79, 78, 84, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, + 32, 2, 151, 173, 1, 66, 2, 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, + 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, + 82, 79, 87, 2, 11, 72, 2, 239, 135, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, + 69, 68, 69, 83, 2, 163, 196, 1, 84, 226, 15, 86, 65, 254, 16, 83, 174, 3, + 69, 162, 42, 73, 178, 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, + 73, 32, 128, 16, 17, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, + 84, 79, 82, 45, 173, 42, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, + 2, 83, 89, 243, 91, 68, 2, 11, 79, 2, 135, 170, 2, 77, 2, 11, 85, 2, 11, + 76, 2, 11, 76, 2, 11, 32, 2, 11, 83, 2, 195, 209, 1, 84, 2, 17, 2, 85, + 69, 2, 135, 225, 1, 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, + 11, 5, 77, 66, 79, 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, + 90, 70, 71, 122, 72, 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, + 2, 70, 2, 74, 2, 80, 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 158, + 70, 65, 2, 73, 3, 85, 42, 66, 72, 162, 8, 79, 238, 254, 1, 69, 150, 64, + 65, 2, 73, 3, 85, 28, 230, 7, 72, 58, 79, 238, 254, 1, 69, 150, 64, 65, + 2, 73, 3, 85, 34, 62, 66, 202, 6, 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, + 85, 18, 106, 79, 222, 5, 69, 214, 191, 2, 65, 2, 73, 3, 85, 24, 50, 79, + 222, 5, 69, 214, 71, 65, 2, 73, 3, 85, 7, 174, 197, 2, 78, 3, 79, 34, 70, + 80, 206, 5, 79, 130, 71, 65, 238, 183, 1, 69, 150, 64, 73, 3, 85, 18, + 246, 4, 69, 86, 79, 130, 71, 65, 130, 248, 1, 73, 3, 85, 16, 54, 69, 218, + 4, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 26, 78, 191, 195, 2, 69, 2, 21, + 3, 71, 84, 72, 2, 183, 133, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, + 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, + 74, 2, 89, 58, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 24, 54, + 79, 150, 129, 2, 69, 150, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, + 131, 193, 2, 79, 10, 54, 83, 214, 161, 2, 68, 190, 28, 70, 2, 75, 3, 77, + 2, 211, 161, 2, 79, 25, 42, 71, 182, 240, 1, 65, 2, 69, 3, 79, 16, 50, + 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 210, 191, 2, 69, 3, 78, 14, + 54, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 5, 255, 190, 2, 79, + 28, 46, 69, 34, 79, 158, 70, 65, 2, 73, 3, 85, 9, 186, 70, 69, 131, 248, + 1, 78, 9, 154, 70, 79, 131, 248, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, + 75, 50, 78, 22, 84, 199, 227, 1, 66, 6, 26, 79, 179, 236, 1, 65, 4, 174, + 236, 1, 79, 251, 49, 45, 4, 74, 69, 251, 185, 2, 65, 2, 243, 235, 1, 79, + 4, 26, 69, 199, 235, 1, 85, 2, 195, 235, 1, 69, 2, 195, 168, 2, 73, 6, + 154, 235, 1, 73, 2, 79, 183, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 94, 53, 250, 184, 2, 54, 2, 55, 2, 56, 3, 57, 23, + 210, 185, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, + 3, 57, 17, 246, 184, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, + 180, 1, 116, 4, 68, 73, 67, 32, 222, 19, 82, 208, 148, 1, 13, 67, 84, 79, + 82, 32, 79, 82, 32, 67, 82, 79, 83, 83, 195, 106, 83, 86, 60, 5, 83, 73, + 71, 78, 32, 153, 10, 5, 84, 79, 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, + 68, 79, 85, 66, 76, 69, 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, + 74, 52, 6, 78, 73, 72, 83, 72, 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, + 79, 82, 77, 22, 76, 52, 4, 84, 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, + 82, 71, 65, 32, 237, 6, 17, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, + 77, 73, 68, 76, 73, 78, 14, 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, + 1, 3, 84, 73, 75, 151, 2, 82, 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, + 9, 85, 66, 72, 65, 89, 65, 84, 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, + 79, 77, 85, 75, 72, 2, 21, 3, 78, 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, + 77, 2, 163, 9, 85, 2, 151, 239, 1, 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, + 2, 151, 3, 76, 2, 235, 174, 2, 65, 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, + 68, 32, 86, 73, 83, 65, 82, 71, 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, + 65, 82, 57, 5, 84, 72, 65, 78, 71, 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, + 25, 4, 68, 72, 65, 86, 2, 245, 236, 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, + 21, 3, 79, 78, 71, 2, 145, 234, 1, 2, 32, 65, 2, 203, 171, 2, 65, 2, 37, + 7, 80, 65, 68, 72, 77, 65, 78, 2, 203, 171, 2, 73, 10, 40, 3, 65, 78, 85, + 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, + 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, 84, 2, 11, 65, 2, 215, 163, 1, 73, + 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, 2, 68, 79, 96, 2, 75, 65, 136, 1, + 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, 84, 164, 1, 11, 89, 65, 74, 85, + 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, 65, 84, 72, 65, 82, 86, 65, 86, + 69, 68, 73, 67, 213, 224, 1, 2, 83, 72, 4, 251, 17, 65, 6, 40, 5, 85, 66, + 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, 5, 83, 2, 11, 73, 2, 251, 1, 78, + 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, 75, 65, 32, 65, 2, 235, 110, 65, + 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, 171, 132, 2, 84, 2, 167, 49, 75, + 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, 73, 67, 32, 75, 65, 83, 72, 77, + 73, 82, 73, 2, 139, 171, 1, 71, 6, 42, 72, 24, 4, 82, 73, 80, 76, 19, 87, + 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, + 2, 11, 83, 2, 11, 32, 2, 159, 14, 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, + 86, 65, 84, 69, 68, 22, 73, 105, 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, + 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, + 2, 17, 2, 32, 73, 2, 49, 10, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, + 17, 2, 32, 83, 2, 221, 255, 1, 3, 86, 65, 82, 5, 37, 7, 32, 83, 67, 72, + 82, 79, 69, 2, 143, 230, 1, 68, 90, 62, 83, 16, 6, 84, 73, 67, 65, 76, + 32, 221, 13, 2, 89, 32, 2, 171, 84, 73, 62, 208, 2, 4, 66, 65, 82, 32, + 166, 3, 69, 62, 70, 36, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, + 48, 12, 75, 65, 78, 65, 32, 82, 69, 80, 69, 65, 84, 32, 252, 1, 4, 76, + 73, 78, 69, 238, 1, 77, 88, 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, + 32, 66, 76, 79, 67, 75, 45, 62, 84, 192, 73, 3, 83, 73, 88, 241, 32, 5, + 90, 73, 71, 90, 65, 8, 108, 6, 66, 69, 83, 73, 68, 69, 60, 8, 68, 79, 85, + 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 139, 1, 87, 2, 17, 2, 32, 82, + 2, 21, 3, 73, 71, 72, 2, 187, 123, 84, 2, 69, 2, 69, 70, 2, 25, 4, 76, + 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, 85, 82, + 78, 2, 11, 83, 2, 11, 84, 2, 219, 124, 73, 2, 21, 3, 73, 84, 72, 2, 17, + 2, 32, 72, 2, 153, 237, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, 25, 4, 76, + 76, 73, 80, 2, 11, 83, 2, 155, 223, 1, 73, 2, 11, 79, 2, 129, 79, 2, 85, + 82, 2, 17, 2, 32, 73, 2, 237, 178, 1, 2, 84, 69, 10, 120, 4, 77, 65, 82, + 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, 79, 85, + 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, 87, 5, + 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 129, 26, 2, 32, 72, + 11, 11, 32, 8, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, + 78, 2, 255, 200, 1, 83, 6, 40, 4, 67, 73, 82, 67, 207, 135, 1, 77, 4, 11, + 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 223, 155, 1, 65, 2, 11, 69, 2, 159, + 126, 76, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, + 75, 2, 239, 144, 1, 69, 12, 194, 151, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 3, 55, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, + 2, 251, 132, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 199, 45, 32, 2, + 135, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 249, 234, 1, 2, 32, 76, 26, + 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, 3, 77, 85, 67, 233, 9, 3, 66, 79, + 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, 83, 246, 1, 87, 207, 10, 71, 4, + 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, 80, 223, 2, 83, 2, 25, 4, 79, + 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, 66, 2, 25, 4, 76, 65, 67, 75, + 2, 11, 32, 2, 139, 79, 83, 2, 11, 73, 2, 185, 1, 2, 86, 69, 2, 11, 69, 2, + 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, 83, 2, 231, 1, 79, 6, 30, 65, + 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, 2, 147, 113, 73, 2, 11, 88, 2, + 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, 2, 17, 2, 69, 68, 2, 11, 32, 2, + 11, 65, 2, 11, 83, 2, 129, 78, 3, 84, 69, 82, 2, 165, 13, 3, 76, 73, 68, + 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, 32, 4, 190, 64, 67, 199, 46, 83, + 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, 2, 69, 6, 82, 69, 65, 84, 69, + 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, 2, 45, 84, 2, 239, 70, 72, + 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, 32, 77, 34, 67, 32, 3, 68, + 69, 79, 126, 69, 218, 1, 79, 22, 82, 21, 7, 84, 72, 75, 85, 81, 73, 32, + 2, 11, 79, 2, 159, 246, 1, 68, 2, 221, 126, 4, 84, 79, 82, 89, 6, 30, 32, + 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, 17, 2, 65, 77, 2, 195, 199, 1, + 69, 2, 143, 116, 65, 2, 147, 71, 83, 6, 168, 1, 31, 84, 78, 65, 77, 69, + 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 82, 69, 65, 68, 73, + 78, 71, 32, 77, 65, 82, 75, 32, 193, 105, 5, 87, 68, 65, 84, 65, 4, 26, + 78, 199, 135, 2, 67, 2, 143, 126, 72, 2, 175, 138, 1, 76, 2, 227, 234, 1, + 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 70, 45, + 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 70, 230, 1, 66, 34, 69, 22, 73, + 22, 76, 34, 78, 254, 133, 1, 67, 2, 68, 2, 83, 2, 84, 214, 56, 72, 226, + 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, 86, 2, 88, 2, 90, 158, 20, 71, 2, + 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, 89, 4, 150, 240, 1, 66, 215, 22, + 69, 5, 203, 134, 2, 73, 5, 227, 239, 1, 74, 4, 234, 131, 2, 76, 187, 2, + 65, 4, 174, 239, 1, 74, 215, 22, 69, 8, 28, 3, 73, 68, 69, 59, 76, 2, 21, + 3, 68, 32, 71, 2, 177, 71, 4, 82, 69, 69, 75, 6, 46, 67, 20, 3, 76, 69, + 89, 41, 2, 85, 77, 2, 203, 219, 1, 65, 2, 11, 66, 2, 11, 65, 2, 147, 123, + 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, 201, 122, 4, 78, 84, 69, 71, 5, 223, + 131, 2, 83, 40, 70, 67, 45, 13, 71, 65, 82, 32, 70, 82, 65, 67, 84, 73, + 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, 155, 197, 1, 85, 38, 106, 70, 96, 4, + 79, 78, 69, 32, 174, 2, 84, 80, 7, 83, 69, 86, 69, 78, 32, 69, 145, 33, + 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, 32, 249, 3, 5, 79, 85, 82, 32, 70, + 4, 158, 3, 69, 113, 3, 83, 73, 88, 18, 66, 69, 28, 2, 70, 73, 18, 72, 30, + 78, 14, 81, 30, 83, 55, 84, 2, 193, 1, 3, 73, 71, 72, 2, 167, 1, 70, 2, + 11, 65, 2, 131, 127, 76, 2, 111, 73, 2, 165, 73, 3, 85, 65, 82, 4, 24, 2, + 69, 86, 15, 73, 2, 43, 69, 2, 43, 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, + 243, 253, 1, 84, 2, 155, 104, 73, 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, + 87, 79, 32, 6, 26, 69, 26, 81, 67, 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, + 85, 65, 82, 2, 151, 111, 84, 4, 22, 70, 211, 32, 84, 2, 11, 73, 2, 11, + 70, 2, 205, 191, 1, 2, 84, 72, 156, 6, 86, 65, 242, 23, 69, 158, 3, 72, + 142, 66, 73, 234, 8, 79, 134, 6, 82, 167, 144, 1, 74, 204, 2, 122, 78, + 220, 5, 2, 88, 73, 158, 1, 82, 210, 10, 84, 194, 1, 86, 252, 70, 2, 70, + 70, 233, 78, 6, 83, 84, 69, 66, 65, 83, 122, 36, 4, 67, 72, 79, 32, 183, + 5, 73, 118, 100, 7, 76, 69, 84, 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, + 16, 5, 84, 79, 78, 69, 32, 243, 7, 68, 88, 210, 1, 65, 38, 79, 34, 69, + 22, 73, 22, 76, 50, 78, 42, 84, 50, 85, 22, 89, 138, 175, 1, 75, 2, 80, + 2, 83, 254, 68, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, + 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, 206, 247, 1, 78, 87, 85, 7, 11, 78, + 5, 147, 248, 1, 71, 5, 255, 247, 1, 78, 5, 151, 247, 1, 78, 4, 26, 76, + 191, 247, 1, 65, 2, 131, 245, 1, 72, 6, 238, 244, 1, 71, 2, 89, 187, 2, + 65, 8, 198, 244, 1, 72, 2, 82, 2, 83, 187, 2, 65, 5, 251, 166, 1, 69, 4, + 170, 245, 1, 73, 147, 1, 65, 2, 179, 111, 78, 8, 40, 3, 75, 79, 73, 1, 3, + 84, 85, 80, 5, 135, 226, 1, 78, 4, 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, + 69, 83, 67, 69, 78, 84, 1, 7, 71, 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, + 77, 79, 2, 11, 79, 2, 175, 97, 78, 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, + 84, 73, 32, 249, 108, 4, 78, 73, 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, + 73, 84, 65, 0, 4, 83, 77, 65, 76, 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 171, 219, 1, 79, 64, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 64, 174, 1, 65, 38, 69, 42, 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, + 89, 170, 42, 84, 248, 107, 2, 86, 73, 206, 51, 66, 2, 80, 246, 5, 75, + 158, 11, 73, 2, 87, 162, 17, 68, 3, 71, 9, 162, 240, 1, 78, 86, 77, 3, + 84, 7, 11, 78, 4, 198, 240, 1, 78, 3, 89, 8, 38, 79, 210, 151, 1, 73, + 231, 31, 65, 4, 178, 183, 1, 82, 223, 25, 76, 4, 26, 71, 223, 158, 1, 85, + 2, 143, 237, 1, 65, 5, 159, 169, 1, 68, 4, 26, 83, 175, 219, 1, 73, 2, + 155, 202, 1, 85, 4, 238, 238, 1, 67, 3, 85, 8, 34, 85, 178, 238, 1, 65, + 3, 79, 5, 175, 238, 1, 74, 20, 11, 73, 20, 11, 71, 20, 17, 2, 73, 84, 20, + 11, 32, 20, 66, 70, 30, 83, 42, 84, 62, 90, 130, 83, 78, 14, 79, 199, + 110, 69, 4, 218, 112, 73, 131, 4, 79, 4, 22, 69, 227, 96, 73, 2, 247, 27, + 86, 4, 26, 72, 207, 205, 1, 87, 2, 11, 82, 2, 223, 213, 1, 69, 2, 11, 69, + 2, 159, 205, 1, 82, 18, 42, 69, 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, + 1, 3, 73, 71, 72, 4, 22, 73, 139, 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, + 2, 135, 1, 69, 4, 92, 2, 69, 86, 25, 2, 73, 88, 6, 34, 72, 26, 87, 187, + 154, 1, 69, 2, 11, 73, 2, 35, 82, 2, 11, 69, 2, 11, 78, 2, 171, 216, 1, + 84, 12, 32, 2, 69, 82, 175, 232, 1, 67, 10, 34, 32, 173, 153, 1, 2, 77, + 69, 8, 80, 3, 67, 76, 79, 22, 87, 224, 197, 1, 5, 66, 85, 70, 70, 65, 1, + 2, 80, 79, 2, 183, 179, 1, 83, 2, 235, 107, 65, 20, 60, 2, 69, 32, 124, + 4, 73, 78, 71, 32, 161, 1, 2, 89, 32, 6, 182, 2, 68, 157, 186, 1, 23, 65, + 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, + 67, 84, 76, 89, 6, 64, 5, 66, 76, 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, + 55, 72, 2, 17, 2, 32, 70, 2, 11, 76, 2, 171, 229, 1, 65, 2, 11, 65, 2, + 11, 78, 2, 235, 94, 68, 8, 26, 68, 34, 76, 43, 79, 2, 11, 65, 2, 139, + 228, 1, 83, 4, 22, 79, 223, 75, 73, 2, 183, 75, 87, 2, 11, 86, 2, 11, 69, + 2, 155, 75, 82, 14, 48, 3, 65, 82, 89, 70, 68, 70, 73, 171, 1, 83, 4, 11, + 32, 4, 32, 2, 67, 65, 219, 166, 1, 70, 2, 191, 166, 1, 84, 4, 26, 71, + 175, 146, 1, 68, 2, 149, 67, 6, 69, 45, 84, 65, 73, 76, 4, 116, 17, 69, + 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, 76, 73, 80, 84, 73, 67, 185, 43, + 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, 32, 70, 2, 157, 145, 1, 2, 85, + 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, 2, 179, 35, 67, 180, 2, 52, 3, + 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, 65, 7, 60, 7, 32, 79, 70, 32, + 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 143, 191, 1, 82, 2, 251, 76, 82, + 172, 2, 54, 32, 181, 63, 8, 45, 70, 69, 65, 84, 72, 69, 82, 170, 2, 148, + 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, + 72, 32, 114, 66, 46, 67, 194, 15, 68, 166, 5, 69, 50, 70, 178, 3, 72, + 222, 3, 76, 218, 3, 77, 226, 1, 78, 34, 80, 146, 1, 81, 78, 82, 250, 1, + 83, 142, 12, 84, 228, 3, 2, 85, 80, 217, 3, 3, 86, 69, 82, 4, 22, 84, + 251, 67, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, 2, 72, 73, 2, 11, + 82, 2, 247, 158, 1, 68, 2, 11, 85, 2, 11, 76, 2, 143, 167, 1, 76, 92, + 146, 1, 72, 168, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, 25, 20, 79, 78, + 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, 77, 79, 78, 68, + 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, 118, 75, 142, 4, 80, 22, + 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, 72, 79, 80, 4, 45, 9, 81, + 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, 82, 2, 129, 6, 8, 79, 84, + 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, 73, 71, 72, 84, 6, 177, 4, + 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, 82, 79, 84, 65, 84, 69, 68, + 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, 32, 72, 85, 78, 68, 82, 69, + 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, 207, 1, 82, 6, 148, 1, 11, 87, + 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 133, 3, 20, 72, 82, 69, 69, 32, + 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, 70, 84, 69, 69, 78, 4, 36, 4, 84, + 87, 69, 78, 175, 2, 83, 2, 217, 2, 7, 84, 89, 45, 70, 73, 86, 69, 6, 166, + 3, 66, 94, 81, 47, 82, 6, 41, 2, 65, 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, + 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, 82, 79, 84, 65, 84, 69, 68, 32, 4, + 70, 78, 25, 13, 84, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 83, 2, + 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, 69, 78, 2, 17, 2, 84, 89, 2, 253, + 62, 6, 32, 68, 69, 71, 82, 69, 12, 33, 6, 85, 82, 78, 69, 68, 32, 12, 42, + 66, 30, 75, 34, 80, 34, 81, 47, 82, 2, 225, 88, 3, 73, 83, 72, 4, 222, + 128, 1, 73, 159, 38, 78, 2, 11, 65, 2, 203, 129, 1, 87, 2, 11, 85, 2, 11, + 69, 2, 159, 129, 1, 69, 2, 255, 166, 1, 79, 19, 11, 32, 16, 100, 16, 67, + 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, + 73, 84, 72, 32, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, + 32, 2, 11, 67, 2, 11, 73, 2, 11, 82, 2, 203, 45, 67, 14, 56, 2, 68, 79, + 70, 84, 148, 30, 2, 76, 79, 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, + 179, 163, 1, 82, 2, 179, 51, 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 155, + 57, 68, 2, 141, 26, 2, 85, 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, + 29, 2, 76, 69, 49, 2, 82, 73, 28, 76, 6, 73, 65, 77, 79, 78, 68, 216, 2, + 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, 67, 79, 78, + 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, 1, 9, 87, + 73, 84, 72, 32, 67, 69, 78, 84, 198, 22, 83, 173, 18, 2, 73, 78, 6, 74, + 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, 77, 2, 25, + 4, 77, 65, 76, 76, 2, 181, 14, 2, 32, 68, 2, 11, 82, 2, 11, 69, 2, 179, + 56, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 53, 10, 45, + 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 250, 34, 66, 24, 5, 76, 69, 70, + 84, 32, 51, 73, 4, 138, 36, 83, 51, 84, 4, 33, 6, 85, 71, 72, 84, 83, 32, + 4, 22, 77, 191, 118, 75, 2, 223, 119, 65, 2, 29, 5, 88, 67, 76, 65, 77, + 2, 155, 14, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, 79, 73, 78, + 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, 34, 32, 87, + 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 77, 73, 68, + 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, 155, 174, 1, + 80, 4, 26, 82, 227, 134, 1, 87, 2, 17, 2, 69, 84, 2, 215, 173, 1, 84, 4, + 11, 32, 4, 18, 67, 23, 83, 2, 223, 195, 1, 85, 2, 187, 33, 84, 2, 183, + 59, 79, 16, 34, 65, 150, 1, 69, 219, 1, 79, 2, 25, 4, 82, 68, 32, 83, 2, + 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, 2, 17, 2, + 32, 68, 2, 11, 73, 2, 135, 191, 1, 83, 10, 22, 65, 247, 10, 88, 8, 30, + 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 183, 14, 32, 4, 74, 67, 41, 14, 83, + 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 21, 3, 72, 69, 67, + 2, 155, 127, 75, 2, 139, 18, 79, 4, 152, 30, 10, 82, 73, 90, 79, 78, 84, + 65, 76, 32, 69, 229, 10, 2, 85, 82, 18, 170, 1, 65, 116, 3, 69, 70, 84, + 165, 9, 31, 79, 90, 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, + 78, 71, 32, 66, 76, 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 24, 2, + 82, 71, 19, 84, 2, 199, 29, 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, + 2, 11, 82, 2, 159, 39, 79, 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 6, 44, 6, 76, 65, 78, 69, 32, 77, 223, 22, 80, 2, 11, 69, + 2, 219, 8, 82, 6, 30, 80, 166, 24, 83, 51, 84, 2, 187, 5, 79, 12, 68, 6, + 69, 68, 73, 85, 77, 32, 121, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, + 54, 83, 211, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 163, 45, 79, 6, + 202, 24, 84, 186, 1, 77, 59, 81, 2, 11, 69, 2, 139, 184, 1, 78, 2, 11, + 73, 2, 163, 186, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, + 76, 79, 2, 11, 71, 2, 11, 82, 2, 227, 168, 1, 65, 4, 11, 78, 4, 154, 2, + 84, 191, 45, 78, 2, 21, 3, 85, 69, 83, 2, 145, 117, 9, 84, 73, 79, 78, + 32, 77, 65, 82, 75, 14, 36, 4, 73, 71, 72, 84, 179, 22, 69, 12, 60, 10, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 195, 17, 32, 8, 30, 80, 178, 19, + 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, 147, + 103, 71, 2, 11, 73, 2, 11, 78, 2, 235, 120, 84, 56, 118, 67, 32, 3, 69, + 83, 65, 18, 72, 58, 77, 170, 1, 80, 68, 5, 81, 85, 65, 82, 69, 168, 6, 2, + 85, 78, 163, 10, 84, 2, 133, 39, 4, 73, 83, 83, 79, 2, 155, 36, 77, 2, + 21, 3, 79, 71, 73, 2, 137, 120, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, 76, + 32, 119, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, 11, + 78, 2, 159, 157, 1, 71, 4, 182, 17, 84, 243, 1, 81, 2, 187, 43, 76, 2, + 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 203, 150, 1, 85, 29, 11, 32, + 26, 114, 66, 36, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, + 65, 67, 75, 32, 73, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 251, 97, + 84, 6, 184, 16, 4, 86, 69, 82, 89, 22, 83, 37, 6, 77, 69, 68, 73, 85, 77, + 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, 84, 73, 67, + 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 221, 18, 6, 67, 69, 78, 84, + 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, 73, 115, 79, + 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, 32, 2, 11, 84, + 2, 11, 73, 2, 179, 171, 1, 67, 2, 29, 5, 85, 78, 68, 69, 68, 2, 21, 3, + 32, 67, 79, 2, 181, 31, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, 3, 69, 82, + 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, 32, 81, 85, + 65, 68, 2, 163, 35, 82, 2, 159, 19, 71, 11, 11, 32, 8, 84, 12, 66, 69, + 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, 72, 32, 5, 29, + 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 243, 43, 65, 4, 38, 82, 29, + 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 155, 109, 89, 2, 25, 4, 32, 67, 76, + 79, 2, 171, 100, 85, 10, 38, 79, 54, 69, 62, 82, 203, 1, 87, 2, 49, 10, + 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, 76, 69, 2, 11, 80, 2, + 11, 72, 2, 147, 16, 79, 4, 148, 1, 4, 65, 80, 69, 90, 33, 28, 73, 65, 78, + 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 83, 77, 65, + 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 155, 151, 1, 85, 2, 255, 3, 69, + 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, 69, 70, 84, 32, 87, 65, 89, + 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 223, 166, 1, 67, 12, 62, 32, 185, + 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 11, 80, 4, 41, 8, 79, + 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, 73, 2, 21, 3, 65, 67, 75, 2, + 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, 17, 2, 78, 68, 2, 215, 24, + 69, 8, 68, 4, 67, 72, 69, 86, 18, 83, 73, 7, 84, 82, 73, 65, 78, 71, 76, + 2, 171, 84, 82, 2, 25, 4, 77, 65, 76, 76, 2, 17, 2, 32, 84, 2, 129, 2, 2, + 82, 73, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 145, 18, 3, 73, 84, 72, 8, + 44, 6, 84, 73, 67, 65, 76, 32, 183, 1, 89, 6, 26, 66, 18, 69, 51, 82, 2, + 215, 105, 65, 2, 11, 76, 2, 17, 2, 76, 73, 2, 131, 25, 80, 2, 11, 69, 2, + 17, 2, 67, 84, 2, 11, 65, 2, 11, 78, 2, 11, 71, 2, 207, 138, 1, 76, 2, + 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, 2, 11, + 81, 2, 11, 85, 2, 11, 65, 2, 203, 137, 1, 82, 2, 11, 69, 2, 11, 68, 2, + 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, 4, 4, + 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, 68, 32, + 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, 78, 189, + 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 12, 4, 68, + 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, 10, 11, + 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, 32, 2, 69, + 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, 0, 6, 86, + 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, 68, 73, 85, + 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, 21, 3, 65, 82, + 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 251, 27, 79, 2, 17, 2, + 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, 11, 32, 2, 11, + 76, 2, 11, 73, 2, 191, 130, 1, 78, 2, 11, 76, 2, 11, 79, 2, 135, 91, 87, + 12, 46, 68, 106, 69, 186, 15, 75, 139, 136, 1, 71, 6, 22, 32, 131, 26, + 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, 2, 255, 128, + 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, 75, 69, 89, + 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 191, 80, 82, 2, 11, 69, 2, 231, + 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 214, 64, 79, 129, 9, 2, 76, + 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, 87, 73, 84, 72, + 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, 167, 6, 65, 8, + 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, 2, 195, 86, 84, + 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 143, 86, 69, 2, 163, 119, 65, 2, 17, + 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, 83, 2, 11, 89, 2, + 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, 1, 3, 76, 68, 32, + 32, 2, 82, 73, 195, 144, 1, 77, 4, 56, 9, 83, 69, 80, 65, 82, 65, 84, 79, + 82, 191, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, 68, 2, 11, 76, 2, + 11, 69, 2, 11, 32, 2, 195, 64, 68, 2, 11, 77, 2, 199, 144, 1, 65, 2, 11, + 69, 2, 159, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, 80, 30, 69, 163, + 1, 73, 2, 201, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, 3, 83, 84, 76, + 163, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, 82, 79, 68, 85, + 2, 171, 114, 67, 2, 11, 69, 2, 235, 80, 82, 2, 11, 84, 2, 11, 73, 2, 17, + 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 203, 71, 78, 34, 76, 2, 32, + 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 143, 83, 79, 2, 53, + 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, 2, 11, 69, 2, + 11, 32, 2, 11, 66, 2, 11, 79, 2, 139, 140, 1, 88, 2, 11, 82, 2, 163, 122, + 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, 11, 32, 14, 130, + 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, 72, 36, 4, 83, + 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, 65, 78, 19, 72, + 2, 155, 58, 78, 2, 189, 57, 3, 65, 82, 73, 2, 11, 72, 2, 179, 70, 65, 2, + 11, 82, 2, 11, 65, 2, 155, 137, 1, 76, 2, 17, 2, 79, 82, 2, 159, 114, 83, + 2, 131, 75, 73, 242, 19, 50, 65, 58, 69, 218, 11, 73, 217, 35, 2, 79, 45, + 2, 17, 2, 87, 78, 2, 11, 73, 2, 229, 74, 2, 78, 71, 98, 60, 4, 76, 76, + 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, 32, 72, 2, 11, 69, + 2, 11, 65, 2, 203, 106, 82, 2, 11, 32, 2, 11, 83, 2, 11, 73, 2, 239, 54, + 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 76, 5, 72, 89, + 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, 3, 77, 65, 68, 13, + 4, 72, 65, 77, 90, 2, 11, 68, 2, 203, 67, 65, 2, 207, 27, 78, 88, 250, 1, + 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, 2, 81, 32, 3, 76, 65, + 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, 118, 87, 14, 79, 18, + 88, 52, 3, 89, 79, 84, 154, 1, 90, 130, 53, 82, 226, 48, 66, 254, 5, 85, + 162, 14, 70, 3, 74, 6, 22, 72, 251, 113, 73, 4, 22, 72, 227, 113, 73, 2, + 223, 113, 73, 4, 11, 65, 4, 154, 130, 1, 68, 3, 76, 8, 42, 76, 130, 50, + 89, 214, 79, 84, 3, 87, 2, 71, 73, 4, 254, 111, 65, 147, 15, 72, 4, 18, + 72, 15, 65, 2, 11, 65, 2, 139, 129, 1, 70, 5, 11, 32, 2, 11, 87, 2, 21, + 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 179, 3, 84, 2, 195, 48, + 85, 4, 178, 105, 72, 215, 22, 69, 8, 46, 72, 234, 47, 73, 194, 9, 65, + 151, 70, 69, 2, 231, 47, 73, 4, 214, 104, 72, 215, 22, 65, 5, 37, 7, 32, + 65, 76, 84, 69, 82, 78, 2, 21, 3, 65, 84, 69, 2, 11, 32, 2, 11, 70, 2, + 11, 79, 2, 211, 109, 82, 2, 11, 65, 2, 143, 126, 87, 4, 22, 72, 235, 125, + 65, 2, 11, 69, 2, 135, 46, 89, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, + 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, 2, 11, 65, 2, 11, + 66, 2, 11, 79, 2, 239, 101, 86, 6, 22, 65, 159, 124, 69, 5, 155, 124, 76, + 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, 88, 8, 82, 65, 68, 73, + 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, 76, 69, 32, 110, 170, + 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, 75, 30, 76, 30, 77, + 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, 90, 246, 35, 81, 186, + 49, 87, 235, 30, 79, 4, 22, 66, 243, 64, 85, 2, 147, 93, 85, 12, 42, 85, + 18, 89, 162, 98, 72, 203, 22, 73, 2, 247, 120, 79, 7, 242, 120, 80, 3, + 84, 4, 22, 68, 199, 120, 85, 2, 243, 63, 85, 10, 42, 71, 222, 91, 79, + 162, 28, 69, 15, 65, 4, 146, 89, 85, 235, 30, 79, 8, 22, 88, 227, 88, 77, + 6, 222, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, 151, 119, 79, 6, 230, + 90, 85, 218, 5, 73, 215, 22, 89, 4, 190, 90, 73, 175, 28, 69, 6, 186, 54, + 73, 199, 7, 89, 4, 166, 118, 79, 15, 73, 8, 30, 89, 26, 90, 183, 90, 66, + 4, 238, 117, 73, 3, 79, 2, 215, 117, 85, 4, 166, 89, 85, 3, 89, 10, 22, + 72, 207, 97, 83, 8, 210, 60, 85, 166, 28, 65, 162, 28, 79, 15, 89, 4, + 198, 88, 65, 175, 28, 85, 4, 134, 60, 85, 199, 56, 69, 4, 142, 88, 73, + 175, 28, 79, 10, 54, 85, 220, 62, 2, 90, 73, 226, 24, 79, 175, 28, 65, 4, + 230, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, 162, 1, 68, 110, + 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, 54, 76, 62, 77, + 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, 83, 134, 1, 84, + 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, 65, 2, 79, 107, + 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, + 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, 89, 122, 66, + 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, 54, 46, 85, + 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, 214, 88, 80, + 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, 3, 79, 54, + 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, 66, 65, 2, + 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, 65, 2, 69, + 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, 150, 19, + 69, 254, 89, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, 110, 88, + 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 215, 88, 84, 64, 238, + 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, 58, 73, + 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, 73, 43, + 65, 6, 226, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 215, 88, 84, 46, 46, + 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, 84, 130, + 16, 69, 190, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, 11, 84, + 2, 11, 73, 2, 11, 79, 2, 187, 39, 78, 106, 50, 74, 190, 10, 85, 146, 4, + 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, 56, 242, + 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, 3, 69, 26, + 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, 73, 158, 1, + 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, 65, 2, 79, + 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, 82, 50, 89, + 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, 10, 73, 158, + 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, 65, 2, 79, 67, + 85, 13, 234, 11, 69, 214, 88, 80, 2, 84, 3, 88, 34, 60, 2, 85, 79, 218, + 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 210, 99, 84, 3, 88, 50, 230, 1, + 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, 69, 66, 85, 3, + 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 254, 89, 80, 2, 84, + 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, 89, 43, 69, + 15, 254, 8, 79, 2, 82, 214, 88, 80, 3, 88, 60, 150, 6, 85, 58, 73, 158, + 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, 67, 89, 100, + 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, 46, 85, 162, + 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 214, 88, 80, 2, 84, + 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, 66, 65, 2, 79, + 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, 89, 56, 46, 85, + 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, 82, 214, 88, 80, + 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, 85, 3, 89, 4, + 134, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, 9, 203, 2, 79, + 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, 72, 50, 85, + 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, 214, 1, 65, 2, + 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 214, 88, 80, 2, 84, 3, 88, + 15, 90, 69, 254, 89, 80, 2, 84, 3, 88, 58, 50, 69, 2, 79, 26, 73, 42, 65, + 34, 85, 35, 89, 7, 250, 89, 80, 3, 88, 17, 38, 69, 190, 89, 80, 2, 84, 3, + 88, 9, 186, 89, 80, 2, 84, 3, 88, 11, 70, 82, 214, 88, 80, 3, 88, 13, 38, + 82, 214, 88, 80, 2, 84, 3, 88, 5, 211, 88, 88, 2, 215, 7, 65, 2, 191, 57, + 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 220, 6, + 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, 85, 65, 82, 69, 32, 210, + 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, 8, 78, 65, 77, 69, 78, 78, + 89, 32, 176, 33, 3, 79, 77, 66, 235, 26, 87, 26, 144, 1, 9, 66, 65, 71, + 32, 77, 69, 77, 66, 69, 38, 82, 104, 6, 68, 79, 77, 65, 73, 78, 60, 3, + 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, 2, 11, 82, 2, 209, 83, 2, + 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, 73, 71, 72, 221, 1, 11, 69, 76, + 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, 173, 3, 11, 32, 65, 78, 84, 73, + 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, 4, 60, 4, 73, 77, 65, 71, 13, + 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, 2, 25, 4, 32, 66, 82, 65, 2, + 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, 69, 77, 65, 32, 211, 1, 80, 6, + 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, 80, 2, 11, 79, 2, 11, 83, 2, + 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, 11, 80, 2, 11, 73, 2, 151, 80, + 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, 87, 73, 2, 255, 51, 79, 2, 11, + 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, 11, 76, 2, 11, 79, 2, 211, 79, + 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, 78, 73, 84, 73, 65, 76, 204, 2, + 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 16, 7, 76, + 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, 82, 75, 32, 206, 1, 83, 181, + 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, 83, 73, 78, 71, 137, 1, 6, + 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, 68, 79, 85, 66, 76, 69, 45, + 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, 2, 17, 2, 69, 65, 2, 215, + 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, + 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 226, 72, 76, 2, 82, 2, 86, 3, 89, + 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, 32, 2, 131, 9, 84, 82, + 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, + 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, + 2, 65, 12, 206, 1, 68, 2, 90, 254, 68, 72, 187, 2, 65, 6, 142, 70, 83, + 14, 72, 187, 2, 65, 8, 246, 69, 71, 2, 78, 2, 89, 187, 2, 65, 6, 202, 69, + 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 254, 68, 72, 187, 2, 65, 4, + 250, 68, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, 76, 79, 78, 71, 23, + 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, 83, 2, 11, 72, 2, 11, + 65, 2, 147, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, 72, 2, 135, 69, 69, + 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, 72, 6, 67, 65, 78, 68, + 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, 2, 83, 86, 2, 11, 65, 2, + 251, 65, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, 32, 5, 11, 32, 2, 25, 4, + 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, 78, 65, 2, 11, 77, 2, 11, + 69, 2, 227, 38, 78, 4, 11, 73, 4, 18, 82, 19, 83, 2, 207, 33, 65, 2, 11, + 65, 2, 11, 82, 2, 255, 63, 71, 2, 151, 4, 66, 20, 38, 76, 109, 5, 83, 73, + 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, 11, 84, 2, 11, 72, 2, 11, 32, + 2, 11, 77, 2, 11, 65, 2, 251, 61, 82, 18, 86, 65, 26, 79, 2, 85, 16, 8, + 82, 69, 86, 69, 82, 83, 69, 68, 134, 64, 69, 3, 73, 4, 170, 64, 73, 3, + 85, 5, 147, 64, 69, 2, 171, 44, 32, 12, 72, 2, 66, 82, 16, 9, 82, 79, 32, + 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, 2, 65, 8, 32, 2, 78, 79, 86, + 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, 5, 66, 82, 69, 65, 75, 2, + 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, 11, 79, 2, 11, 73, 2, 11, + 78, 2, 143, 5, 69, 2, 207, 61, 83, 2, 21, 3, 77, 79, 85, 2, 17, 2, 84, + 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 155, 38, 67, 242, 2, 168, 1, 10, + 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 212, 18, 6, 78, 69, 85, 77, 69, + 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, 32, 77, 79, 68, 73, 70, 73, + 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, 69, 82, 32, 84, 79, 78, 65, + 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, 82, 75, 32, 148, 3, 18, 65, + 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, 82, 84, 73, 67, 65, 76, 221, + 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 32, 77, 65, 82, 75, + 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, 65, 2, 11, 84, 2, 11, 79, 2, + 207, 56, 82, 118, 182, 2, 67, 142, 1, 68, 104, 8, 71, 79, 82, 65, 90, 68, + 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, 80, 79, 140, 2, 8, 77, 65, 76, + 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, 83, 186, 1, 84, 108, 7, 86, 89, + 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, 2, 76, 79, 224, 22, 4, 85, 68, + 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, 6, 72, 65, 83, 72, 75, 65, 29, + 5, 85, 82, 86, 69, 68, 5, 185, 47, 3, 32, 80, 79, 2, 17, 2, 32, 79, 2, + 11, 77, 2, 223, 24, 69, 4, 228, 10, 13, 69, 77, 69, 83, 84, 86, 69, 78, + 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, 84, 79, 67, 10, 30, 78, 89, + 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, 8, 152, 8, 8, 83, 32, 75, + 82, 89, 90, 72, 69, 35, 79, 2, 155, 20, 79, 8, 56, 4, 82, 89, 90, 72, + 138, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, 78, 2, 11, 32, 2, 191, + 7, 76, 6, 208, 5, 3, 84, 83, 69, 200, 13, 5, 66, 76, 65, 67, 72, 131, 31, + 78, 18, 22, 68, 247, 1, 86, 6, 60, 7, 67, 72, 65, 83, 72, 73, 69, 189, + 22, 3, 86, 69, 82, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, + 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, + 2, 11, 82, 2, 11, 79, 2, 131, 25, 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, + 22, 83, 251, 3, 79, 8, 154, 3, 32, 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, + 90, 83, 183, 5, 86, 2, 219, 37, 69, 10, 136, 1, 2, 75, 79, 16, 16, 84, + 82, 65, 78, 78, 79, 32, 77, 65, 76, 79, 32, 80, 79, 86, 89, 236, 1, 5, + 82, 69, 68, 78, 69, 231, 26, 79, 2, 239, 42, 66, 2, 11, 83, 2, 183, 22, + 72, 10, 50, 79, 16, 5, 83, 65, 84, 65, 32, 139, 36, 73, 2, 187, 28, 67, + 6, 158, 1, 79, 201, 24, 3, 83, 32, 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, + 75, 6, 44, 6, 72, 79, 75, 72, 76, 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, + 32, 79, 4, 11, 78, 4, 11, 32, 4, 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, + 70, 2, 11, 73, 2, 11, 71, 2, 139, 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, + 3, 68, 69, 82, 4, 22, 86, 239, 10, 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, + 65, 67, 72, 18, 83, 1, 4, 84, 82, 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, + 69, 2, 155, 6, 84, 232, 1, 128, 2, 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, + 71, 79, 76, 85, 66, 67, 72, 73, 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, + 67, 72, 73, 75, 184, 1, 2, 78, 69, 18, 79, 138, 2, 80, 146, 2, 82, 114, + 83, 180, 20, 9, 86, 82, 65, 75, 72, 73, 89, 65, 32, 151, 2, 90, 4, 246, + 11, 69, 209, 10, 2, 65, 83, 10, 74, 69, 98, 85, 16, 9, 86, 65, 32, 86, + 32, 67, 72, 69, 76, 187, 27, 79, 4, 76, 2, 82, 66, 213, 2, 12, 77, 69, + 83, 84, 86, 69, 78, 78, 89, 32, 75, 76, 2, 195, 34, 73, 2, 203, 34, 68, + 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, 2, 147, 34, 84, 10, 78, 84, 38, 83, + 240, 3, 5, 77, 82, 65, 67, 72, 145, 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, + 69, 83, 251, 1, 73, 2, 21, 3, 86, 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, + 65, 77, 73, 18, 76, 40, 3, 79, 66, 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, + 76, 2, 11, 89, 2, 11, 85, 2, 151, 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, + 75, 219, 32, 90, 5, 21, 3, 32, 84, 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, + 70, 2, 11, 73, 2, 11, 83, 2, 215, 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, + 89, 85, 67, 72, 69, 83, 80, 6, 64, 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, + 14, 80, 163, 14, 86, 2, 39, 78, 2, 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, + 2, 223, 22, 77, 14, 40, 2, 66, 76, 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, + 11, 75, 2, 243, 30, 79, 13, 11, 32, 10, 30, 75, 142, 26, 84, 39, 83, 6, + 104, 11, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 185, 25, 10, 82, 89, + 85, 75, 79, 86, 65, 89, 65, 32, 4, 202, 16, 78, 251, 8, 83, 14, 58, 65, + 88, 8, 69, 82, 69, 86, 79, 68, 75, 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, + 19, 76, 211, 5, 85, 2, 11, 76, 2, 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, + 32, 78, 4, 68, 5, 68, 67, 72, 65, 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, + 77, 2, 11, 72, 2, 219, 4, 73, 4, 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, + 67, 72, 69, 139, 26, 79, 2, 25, 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, + 96, 9, 75, 65, 77, 69, 89, 84, 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, + 84, 73, 69, 119, 84, 22, 128, 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, + 65, 89, 65, 32, 44, 7, 75, 76, 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, + 119, 83, 8, 218, 10, 75, 110, 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, + 89, 65, 32, 243, 10, 78, 4, 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, + 4, 82, 9, 11, 32, 6, 48, 2, 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, + 166, 4, 75, 95, 90, 2, 11, 79, 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, + 172, 4, 6, 79, 80, 73, 84, 83, 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, + 32, 20, 76, 2, 83, 32, 236, 2, 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, + 159, 5, 78, 14, 160, 1, 14, 68, 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, + 65, 84, 89, 28, 7, 75, 82, 89, 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, + 65, 80, 89, 65, 84, 79, 89, 2, 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, + 73, 2, 203, 2, 71, 7, 21, 3, 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, + 67, 72, 65, 83, 72, 73, 2, 11, 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, + 5, 17, 2, 32, 83, 2, 17, 2, 32, 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, + 79, 2, 199, 17, 89, 7, 11, 32, 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, + 87, 73, 84, 72, 32, 83, 79, 2, 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, + 67, 72, 89, 65, 32, 78, 79, 2, 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, + 86, 79, 69, 67, 72, 69, 76, 78, 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, + 8, 77, 82, 65, 67, 72, 78, 79, 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, + 64, 7, 80, 79, 86, 79, 68, 78, 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, + 89, 65, 5, 17, 2, 32, 75, 2, 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, + 89, 85, 67, 72, 69, 77, 2, 82, 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, + 69, 80, 79, 83, 84, 79, 89, 65, 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, + 65, 195, 3, 90, 21, 11, 32, 18, 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, + 179, 2, 80, 14, 21, 3, 82, 79, 77, 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, + 237, 1, 7, 32, 87, 73, 84, 72, 32, 83, 10, 92, 10, 75, 82, 89, 90, 72, + 69, 86, 65, 89, 65, 17, 9, 80, 79, 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, + 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, + 85, 66, 2, 21, 3, 73, 78, 71, 2, 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, + 175, 6, 75, 2, 237, 5, 3, 72, 69, 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, + 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, 76, 85, 187, 1, 86, 2, 163, 1, 80, + 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, 10, 73, 75, 72, 65, 89, 65, 32, 80, + 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, 69, 173, 1, 4, 71, 76, 65, 83, 2, + 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, 171, 1, 76, 16, 88, 12, 75, 76, + 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 38, 77, 46, 80, 38, 84, 39, 83, + 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, 4, 82, 65, 67, 72, 2, 247, 1, + 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, 82, 2, 11, 69, 2, 11, 83, 2, + 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, 3, 77, 69, 89, 4, 32, 4, 78, + 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, 75, 2, 17, 2, 89, 65, 2, 11, + 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, 83, 2, 183, 2, 65, 10, 108, + 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, 70, 28, 3, 75, 82, 89, 28, 5, + 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, 159, 1, 73, 2, 11, 90, 2, 143, + 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, 11, 79, 2, 83, 71, 8, 26, 78, 34, + 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, 83, 2, 11, 80, 3, 0, }; -/* code->name phrasebook */ -#define phrasebook_shift 7 -#define phrasebook_short 190 -static const unsigned char phrasebook[] = { - 0, 201, 248, 233, 218, 77, 207, 254, 77, 31, 56, 236, 157, 56, 210, 15, - 56, 251, 139, 251, 51, 45, 210, 115, 50, 210, 115, 250, 195, 108, 56, - 242, 76, 228, 89, 232, 82, 201, 64, 202, 24, 17, 191, 77, 17, 107, 17, - 109, 17, 138, 17, 134, 17, 150, 17, 169, 17, 175, 17, 171, 17, 178, 242, - 85, 204, 26, 219, 182, 56, 234, 45, 56, 230, 206, 56, 208, 15, 77, 242, - 74, 250, 184, 8, 6, 1, 65, 8, 6, 1, 250, 122, 8, 6, 1, 247, 195, 8, 6, 1, - 238, 129, 8, 6, 1, 71, 8, 6, 1, 233, 177, 8, 6, 1, 232, 53, 8, 6, 1, 230, - 118, 8, 6, 1, 68, 8, 6, 1, 223, 37, 8, 6, 1, 222, 154, 8, 6, 1, 172, 8, - 6, 1, 218, 170, 8, 6, 1, 215, 63, 8, 6, 1, 74, 8, 6, 1, 210, 238, 8, 6, - 1, 208, 106, 8, 6, 1, 146, 8, 6, 1, 206, 9, 8, 6, 1, 200, 43, 8, 6, 1, - 66, 8, 6, 1, 196, 12, 8, 6, 1, 193, 224, 8, 6, 1, 192, 235, 8, 6, 1, 192, - 159, 8, 6, 1, 191, 166, 45, 51, 248, 55, 207, 20, 202, 24, 50, 51, 248, - 55, 243, 4, 252, 62, 130, 219, 114, 230, 213, 252, 62, 8, 2, 1, 65, 8, 2, - 1, 250, 122, 8, 2, 1, 247, 195, 8, 2, 1, 238, 129, 8, 2, 1, 71, 8, 2, 1, - 233, 177, 8, 2, 1, 232, 53, 8, 2, 1, 230, 118, 8, 2, 1, 68, 8, 2, 1, 223, - 37, 8, 2, 1, 222, 154, 8, 2, 1, 172, 8, 2, 1, 218, 170, 8, 2, 1, 215, 63, - 8, 2, 1, 74, 8, 2, 1, 210, 238, 8, 2, 1, 208, 106, 8, 2, 1, 146, 8, 2, 1, - 206, 9, 8, 2, 1, 200, 43, 8, 2, 1, 66, 8, 2, 1, 196, 12, 8, 2, 1, 193, - 224, 8, 2, 1, 192, 235, 8, 2, 1, 192, 159, 8, 2, 1, 191, 166, 45, 238, - 173, 248, 55, 81, 219, 114, 50, 238, 173, 248, 55, 198, 152, 213, 39, - 201, 248, 223, 95, 233, 218, 77, 247, 26, 56, 209, 10, 56, 238, 172, 56, - 192, 71, 56, 248, 24, 164, 205, 55, 56, 237, 44, 239, 8, 56, 233, 42, - 211, 52, 223, 146, 219, 221, 55, 251, 118, 207, 254, 77, 213, 14, 56, - 202, 33, 228, 90, 207, 79, 56, 217, 148, 237, 127, 56, 209, 82, 56, 200, - 182, 109, 200, 182, 138, 252, 49, 252, 62, 216, 93, 56, 209, 144, 56, 82, - 236, 142, 247, 37, 200, 182, 107, 217, 42, 211, 52, 223, 146, 206, 204, - 55, 251, 118, 207, 254, 77, 193, 246, 232, 120, 91, 208, 24, 193, 246, - 232, 120, 91, 230, 72, 193, 246, 232, 120, 115, 208, 22, 223, 95, 208, - 15, 77, 8, 6, 1, 42, 4, 230, 212, 8, 6, 1, 42, 4, 252, 48, 8, 6, 1, 42, - 4, 243, 3, 8, 6, 1, 42, 4, 198, 152, 8, 6, 1, 42, 4, 237, 44, 8, 6, 1, - 42, 4, 206, 190, 58, 8, 6, 1, 252, 27, 8, 6, 1, 247, 196, 4, 247, 37, 8, - 6, 1, 235, 17, 4, 230, 212, 8, 6, 1, 235, 17, 4, 252, 48, 8, 6, 1, 235, - 17, 4, 243, 3, 8, 6, 1, 235, 17, 4, 237, 44, 8, 6, 1, 228, 76, 4, 230, - 212, 8, 6, 1, 228, 76, 4, 252, 48, 8, 6, 1, 228, 76, 4, 243, 3, 8, 6, 1, - 228, 76, 4, 237, 44, 8, 6, 1, 233, 250, 8, 6, 1, 215, 64, 4, 198, 152, 8, - 6, 1, 187, 4, 230, 212, 8, 6, 1, 187, 4, 252, 48, 8, 6, 1, 187, 4, 243, - 3, 8, 6, 1, 187, 4, 198, 152, 8, 6, 1, 187, 4, 237, 44, 215, 129, 56, 8, - 6, 1, 187, 4, 106, 8, 6, 1, 126, 4, 230, 212, 8, 6, 1, 126, 4, 252, 48, - 8, 6, 1, 126, 4, 243, 3, 8, 6, 1, 126, 4, 237, 44, 8, 6, 1, 192, 160, 4, - 252, 48, 8, 6, 1, 198, 233, 8, 2, 1, 203, 128, 206, 9, 8, 2, 1, 42, 4, - 230, 212, 8, 2, 1, 42, 4, 252, 48, 8, 2, 1, 42, 4, 243, 3, 8, 2, 1, 42, - 4, 198, 152, 8, 2, 1, 42, 4, 237, 44, 8, 2, 1, 42, 4, 206, 190, 58, 8, 2, - 1, 252, 27, 8, 2, 1, 247, 196, 4, 247, 37, 8, 2, 1, 235, 17, 4, 230, 212, - 8, 2, 1, 235, 17, 4, 252, 48, 8, 2, 1, 235, 17, 4, 243, 3, 8, 2, 1, 235, - 17, 4, 237, 44, 8, 2, 1, 228, 76, 4, 230, 212, 8, 2, 1, 228, 76, 4, 252, - 48, 8, 2, 1, 228, 76, 4, 243, 3, 8, 2, 1, 228, 76, 4, 237, 44, 8, 2, 1, - 233, 250, 8, 2, 1, 215, 64, 4, 198, 152, 8, 2, 1, 187, 4, 230, 212, 8, 2, - 1, 187, 4, 252, 48, 8, 2, 1, 187, 4, 243, 3, 8, 2, 1, 187, 4, 198, 152, - 8, 2, 1, 187, 4, 237, 44, 236, 202, 56, 8, 2, 1, 187, 4, 106, 8, 2, 1, - 126, 4, 230, 212, 8, 2, 1, 126, 4, 252, 48, 8, 2, 1, 126, 4, 243, 3, 8, - 2, 1, 126, 4, 237, 44, 8, 2, 1, 192, 160, 4, 252, 48, 8, 2, 1, 198, 233, - 8, 2, 1, 192, 160, 4, 237, 44, 8, 6, 1, 42, 4, 217, 148, 8, 2, 1, 42, 4, - 217, 148, 8, 6, 1, 42, 4, 248, 38, 8, 2, 1, 42, 4, 248, 38, 8, 6, 1, 42, - 4, 211, 140, 8, 2, 1, 42, 4, 211, 140, 8, 6, 1, 247, 196, 4, 252, 48, 8, - 2, 1, 247, 196, 4, 252, 48, 8, 6, 1, 247, 196, 4, 243, 3, 8, 2, 1, 247, - 196, 4, 243, 3, 8, 6, 1, 247, 196, 4, 75, 58, 8, 2, 1, 247, 196, 4, 75, - 58, 8, 6, 1, 247, 196, 4, 247, 94, 8, 2, 1, 247, 196, 4, 247, 94, 8, 6, - 1, 238, 130, 4, 247, 94, 8, 2, 1, 238, 130, 4, 247, 94, 8, 6, 1, 238, - 130, 4, 106, 8, 2, 1, 238, 130, 4, 106, 8, 6, 1, 235, 17, 4, 217, 148, 8, - 2, 1, 235, 17, 4, 217, 148, 8, 6, 1, 235, 17, 4, 248, 38, 8, 2, 1, 235, - 17, 4, 248, 38, 8, 6, 1, 235, 17, 4, 75, 58, 8, 2, 1, 235, 17, 4, 75, 58, - 8, 6, 1, 235, 17, 4, 211, 140, 8, 2, 1, 235, 17, 4, 211, 140, 8, 6, 1, - 235, 17, 4, 247, 94, 8, 2, 1, 235, 17, 4, 247, 94, 8, 6, 1, 232, 54, 4, - 243, 3, 8, 2, 1, 232, 54, 4, 243, 3, 8, 6, 1, 232, 54, 4, 248, 38, 8, 2, - 1, 232, 54, 4, 248, 38, 8, 6, 1, 232, 54, 4, 75, 58, 8, 2, 1, 232, 54, 4, - 75, 58, 8, 6, 1, 232, 54, 4, 247, 37, 8, 2, 1, 232, 54, 4, 247, 37, 8, 6, - 1, 230, 119, 4, 243, 3, 8, 2, 1, 230, 119, 4, 243, 3, 8, 6, 1, 230, 119, - 4, 106, 8, 2, 1, 230, 119, 4, 106, 8, 6, 1, 228, 76, 4, 198, 152, 8, 2, - 1, 228, 76, 4, 198, 152, 8, 6, 1, 228, 76, 4, 217, 148, 8, 2, 1, 228, 76, - 4, 217, 148, 8, 6, 1, 228, 76, 4, 248, 38, 8, 2, 1, 228, 76, 4, 248, 38, - 8, 6, 1, 228, 76, 4, 211, 140, 8, 2, 1, 228, 76, 4, 211, 140, 8, 6, 1, - 228, 76, 4, 75, 58, 8, 2, 1, 236, 141, 68, 8, 6, 34, 223, 199, 8, 2, 34, - 223, 199, 8, 6, 1, 223, 38, 4, 243, 3, 8, 2, 1, 223, 38, 4, 243, 3, 8, 6, - 1, 222, 155, 4, 247, 37, 8, 2, 1, 222, 155, 4, 247, 37, 8, 2, 1, 221, 10, - 8, 6, 1, 220, 145, 4, 252, 48, 8, 2, 1, 220, 145, 4, 252, 48, 8, 6, 1, - 220, 145, 4, 247, 37, 8, 2, 1, 220, 145, 4, 247, 37, 8, 6, 1, 220, 145, - 4, 247, 94, 8, 2, 1, 220, 145, 4, 247, 94, 8, 6, 1, 220, 145, 4, 82, 236, - 142, 8, 2, 1, 220, 145, 4, 82, 236, 142, 8, 6, 1, 220, 145, 4, 106, 8, 2, - 1, 220, 145, 4, 106, 8, 6, 1, 215, 64, 4, 252, 48, 8, 2, 1, 215, 64, 4, - 252, 48, 8, 6, 1, 215, 64, 4, 247, 37, 8, 2, 1, 215, 64, 4, 247, 37, 8, - 6, 1, 215, 64, 4, 247, 94, 8, 2, 1, 215, 64, 4, 247, 94, 8, 2, 1, 215, - 64, 208, 235, 247, 207, 251, 51, 8, 6, 1, 234, 90, 8, 2, 1, 234, 90, 8, - 6, 1, 187, 4, 217, 148, 8, 2, 1, 187, 4, 217, 148, 8, 6, 1, 187, 4, 248, - 38, 8, 2, 1, 187, 4, 248, 38, 8, 6, 1, 187, 4, 55, 252, 48, 8, 2, 1, 187, - 4, 55, 252, 48, 8, 6, 34, 211, 153, 8, 2, 34, 211, 153, 8, 6, 1, 207, - 224, 4, 252, 48, 8, 2, 1, 207, 224, 4, 252, 48, 8, 6, 1, 207, 224, 4, - 247, 37, 8, 2, 1, 207, 224, 4, 247, 37, 8, 6, 1, 207, 224, 4, 247, 94, 8, - 2, 1, 207, 224, 4, 247, 94, 8, 6, 1, 206, 10, 4, 252, 48, 8, 2, 1, 206, - 10, 4, 252, 48, 8, 6, 1, 206, 10, 4, 243, 3, 8, 2, 1, 206, 10, 4, 243, 3, - 8, 6, 1, 206, 10, 4, 247, 37, 8, 2, 1, 206, 10, 4, 247, 37, 8, 6, 1, 206, - 10, 4, 247, 94, 8, 2, 1, 206, 10, 4, 247, 94, 8, 6, 1, 200, 44, 4, 247, - 37, 8, 2, 1, 200, 44, 4, 247, 37, 8, 6, 1, 200, 44, 4, 247, 94, 8, 2, 1, - 200, 44, 4, 247, 94, 8, 6, 1, 200, 44, 4, 106, 8, 2, 1, 200, 44, 4, 106, - 8, 6, 1, 126, 4, 198, 152, 8, 2, 1, 126, 4, 198, 152, 8, 6, 1, 126, 4, - 217, 148, 8, 2, 1, 126, 4, 217, 148, 8, 6, 1, 126, 4, 248, 38, 8, 2, 1, - 126, 4, 248, 38, 8, 6, 1, 126, 4, 206, 190, 58, 8, 2, 1, 126, 4, 206, - 190, 58, 8, 6, 1, 126, 4, 55, 252, 48, 8, 2, 1, 126, 4, 55, 252, 48, 8, - 6, 1, 126, 4, 211, 140, 8, 2, 1, 126, 4, 211, 140, 8, 6, 1, 193, 225, 4, - 243, 3, 8, 2, 1, 193, 225, 4, 243, 3, 8, 6, 1, 192, 160, 4, 243, 3, 8, 2, - 1, 192, 160, 4, 243, 3, 8, 6, 1, 192, 160, 4, 237, 44, 8, 6, 1, 191, 167, - 4, 252, 48, 8, 2, 1, 191, 167, 4, 252, 48, 8, 6, 1, 191, 167, 4, 75, 58, - 8, 2, 1, 191, 167, 4, 75, 58, 8, 6, 1, 191, 167, 4, 247, 94, 8, 2, 1, - 191, 167, 4, 247, 94, 8, 2, 1, 180, 206, 9, 8, 2, 1, 78, 4, 106, 8, 6, 1, - 78, 4, 102, 8, 6, 1, 78, 4, 198, 51, 8, 2, 1, 78, 4, 198, 51, 8, 6, 1, - 163, 169, 8, 2, 1, 163, 169, 8, 6, 1, 211, 79, 74, 8, 6, 1, 247, 196, 4, - 102, 8, 2, 1, 247, 196, 4, 102, 8, 6, 1, 252, 2, 238, 129, 8, 6, 1, 238, - 130, 4, 102, 8, 6, 1, 238, 130, 4, 198, 51, 8, 2, 1, 238, 130, 4, 198, - 51, 8, 2, 1, 154, 237, 108, 8, 6, 1, 207, 19, 71, 8, 6, 1, 205, 87, 8, 6, - 1, 211, 79, 71, 8, 6, 1, 233, 178, 4, 102, 8, 2, 1, 233, 178, 4, 102, 8, - 6, 1, 232, 54, 4, 102, 8, 6, 1, 231, 213, 8, 2, 1, 228, 128, 8, 6, 1, - 223, 85, 8, 6, 1, 228, 76, 4, 106, 8, 6, 1, 222, 155, 4, 102, 8, 2, 1, - 222, 155, 4, 102, 8, 2, 1, 220, 145, 4, 164, 8, 2, 1, 220, 35, 4, 106, 8, - 6, 1, 154, 218, 170, 8, 6, 1, 215, 64, 4, 45, 102, 8, 2, 1, 215, 64, 4, - 180, 50, 219, 214, 8, 6, 1, 187, 4, 82, 198, 152, 8, 6, 1, 187, 4, 228, - 189, 8, 2, 1, 187, 4, 228, 189, 8, 6, 1, 211, 135, 8, 2, 1, 211, 135, 8, - 6, 1, 210, 239, 4, 102, 8, 2, 1, 210, 239, 4, 102, 8, 1, 191, 228, 8, 6, - 1, 163, 109, 8, 2, 1, 163, 109, 8, 6, 1, 234, 14, 8, 1, 207, 19, 234, 15, - 219, 6, 8, 2, 1, 200, 44, 4, 210, 194, 102, 8, 6, 1, 200, 44, 4, 102, 8, - 2, 1, 200, 44, 4, 102, 8, 6, 1, 200, 44, 4, 207, 25, 102, 8, 6, 1, 126, - 4, 228, 189, 8, 2, 1, 126, 4, 228, 189, 8, 6, 1, 196, 70, 8, 6, 1, 196, - 13, 4, 102, 8, 6, 1, 192, 160, 4, 102, 8, 2, 1, 192, 160, 4, 102, 8, 6, - 1, 191, 167, 4, 106, 8, 2, 1, 191, 167, 4, 106, 8, 6, 1, 233, 180, 8, 6, - 1, 233, 181, 207, 18, 8, 2, 1, 233, 181, 207, 18, 8, 2, 1, 233, 181, 4, - 199, 215, 8, 1, 105, 4, 106, 8, 6, 1, 163, 150, 8, 2, 1, 163, 150, 8, 1, - 223, 95, 231, 13, 201, 65, 4, 106, 8, 1, 192, 238, 8, 1, 237, 100, 242, - 233, 8, 1, 220, 5, 242, 233, 8, 1, 251, 152, 242, 233, 8, 1, 207, 25, - 242, 233, 8, 6, 1, 235, 39, 4, 247, 94, 8, 6, 1, 238, 130, 4, 2, 1, 191, - 167, 4, 247, 94, 8, 2, 1, 235, 39, 4, 247, 94, 8, 6, 1, 219, 79, 8, 6, 1, - 220, 145, 4, 2, 1, 223, 37, 8, 2, 1, 219, 79, 8, 6, 1, 213, 160, 8, 6, 1, - 215, 64, 4, 2, 1, 223, 37, 8, 2, 1, 213, 160, 8, 6, 1, 42, 4, 247, 94, 8, - 2, 1, 42, 4, 247, 94, 8, 6, 1, 228, 76, 4, 247, 94, 8, 2, 1, 228, 76, 4, - 247, 94, 8, 6, 1, 187, 4, 247, 94, 8, 2, 1, 187, 4, 247, 94, 8, 6, 1, - 126, 4, 247, 94, 8, 2, 1, 126, 4, 247, 94, 8, 6, 1, 126, 4, 237, 45, 23, - 217, 148, 8, 2, 1, 126, 4, 237, 45, 23, 217, 148, 8, 6, 1, 126, 4, 237, - 45, 23, 252, 48, 8, 2, 1, 126, 4, 237, 45, 23, 252, 48, 8, 6, 1, 126, 4, - 237, 45, 23, 247, 94, 8, 2, 1, 126, 4, 237, 45, 23, 247, 94, 8, 6, 1, - 126, 4, 237, 45, 23, 230, 212, 8, 2, 1, 126, 4, 237, 45, 23, 230, 212, 8, - 2, 1, 154, 71, 8, 6, 1, 42, 4, 237, 45, 23, 217, 148, 8, 2, 1, 42, 4, - 237, 45, 23, 217, 148, 8, 6, 1, 42, 4, 75, 93, 23, 217, 148, 8, 2, 1, 42, - 4, 75, 93, 23, 217, 148, 8, 6, 1, 252, 28, 4, 217, 148, 8, 2, 1, 252, 28, - 4, 217, 148, 8, 6, 1, 232, 54, 4, 106, 8, 2, 1, 232, 54, 4, 106, 8, 6, 1, - 232, 54, 4, 247, 94, 8, 2, 1, 232, 54, 4, 247, 94, 8, 6, 1, 222, 155, 4, - 247, 94, 8, 2, 1, 222, 155, 4, 247, 94, 8, 6, 1, 187, 4, 211, 140, 8, 2, - 1, 187, 4, 211, 140, 8, 6, 1, 187, 4, 211, 141, 23, 217, 148, 8, 2, 1, - 187, 4, 211, 141, 23, 217, 148, 8, 6, 1, 233, 181, 4, 247, 94, 8, 2, 1, - 233, 181, 4, 247, 94, 8, 2, 1, 223, 38, 4, 247, 94, 8, 6, 1, 235, 38, 8, - 6, 1, 238, 130, 4, 2, 1, 191, 166, 8, 2, 1, 235, 38, 8, 6, 1, 232, 54, 4, - 252, 48, 8, 2, 1, 232, 54, 4, 252, 48, 8, 6, 1, 228, 125, 8, 6, 1, 192, - 238, 8, 6, 1, 215, 64, 4, 230, 212, 8, 2, 1, 215, 64, 4, 230, 212, 8, 6, - 1, 42, 4, 206, 190, 93, 23, 252, 48, 8, 2, 1, 42, 4, 206, 190, 93, 23, - 252, 48, 8, 6, 1, 252, 28, 4, 252, 48, 8, 2, 1, 252, 28, 4, 252, 48, 8, - 6, 1, 187, 4, 201, 29, 23, 252, 48, 8, 2, 1, 187, 4, 201, 29, 23, 252, - 48, 8, 6, 1, 42, 4, 55, 230, 212, 8, 2, 1, 42, 4, 55, 230, 212, 8, 6, 1, - 42, 4, 223, 95, 248, 38, 8, 2, 1, 42, 4, 223, 95, 248, 38, 8, 6, 1, 235, - 17, 4, 55, 230, 212, 8, 2, 1, 235, 17, 4, 55, 230, 212, 8, 6, 1, 235, 17, - 4, 223, 95, 248, 38, 8, 2, 1, 235, 17, 4, 223, 95, 248, 38, 8, 6, 1, 228, - 76, 4, 55, 230, 212, 8, 2, 1, 228, 76, 4, 55, 230, 212, 8, 6, 1, 228, 76, - 4, 223, 95, 248, 38, 8, 2, 1, 228, 76, 4, 223, 95, 248, 38, 8, 6, 1, 187, - 4, 55, 230, 212, 8, 2, 1, 187, 4, 55, 230, 212, 8, 6, 1, 187, 4, 223, 95, - 248, 38, 8, 2, 1, 187, 4, 223, 95, 248, 38, 8, 6, 1, 207, 224, 4, 55, - 230, 212, 8, 2, 1, 207, 224, 4, 55, 230, 212, 8, 6, 1, 207, 224, 4, 223, - 95, 248, 38, 8, 2, 1, 207, 224, 4, 223, 95, 248, 38, 8, 6, 1, 126, 4, 55, - 230, 212, 8, 2, 1, 126, 4, 55, 230, 212, 8, 6, 1, 126, 4, 223, 95, 248, - 38, 8, 2, 1, 126, 4, 223, 95, 248, 38, 8, 6, 1, 206, 10, 4, 242, 77, 60, - 8, 2, 1, 206, 10, 4, 242, 77, 60, 8, 6, 1, 200, 44, 4, 242, 77, 60, 8, 2, - 1, 200, 44, 4, 242, 77, 60, 8, 6, 1, 191, 248, 8, 2, 1, 191, 248, 8, 6, - 1, 230, 119, 4, 247, 94, 8, 2, 1, 230, 119, 4, 247, 94, 8, 6, 1, 215, 64, - 4, 180, 50, 219, 214, 8, 2, 1, 238, 130, 4, 238, 177, 8, 6, 1, 211, 21, - 8, 2, 1, 211, 21, 8, 6, 1, 191, 167, 4, 102, 8, 2, 1, 191, 167, 4, 102, - 8, 6, 1, 42, 4, 75, 58, 8, 2, 1, 42, 4, 75, 58, 8, 6, 1, 235, 17, 4, 247, - 37, 8, 2, 1, 235, 17, 4, 247, 37, 8, 6, 1, 187, 4, 237, 45, 23, 217, 148, - 8, 2, 1, 187, 4, 237, 45, 23, 217, 148, 8, 6, 1, 187, 4, 198, 153, 23, - 217, 148, 8, 2, 1, 187, 4, 198, 153, 23, 217, 148, 8, 6, 1, 187, 4, 75, - 58, 8, 2, 1, 187, 4, 75, 58, 8, 6, 1, 187, 4, 75, 93, 23, 217, 148, 8, 2, - 1, 187, 4, 75, 93, 23, 217, 148, 8, 6, 1, 192, 160, 4, 217, 148, 8, 2, 1, - 192, 160, 4, 217, 148, 8, 2, 1, 220, 145, 4, 238, 177, 8, 2, 1, 215, 64, - 4, 238, 177, 8, 2, 1, 200, 44, 4, 238, 177, 8, 2, 1, 236, 141, 223, 37, - 8, 2, 1, 237, 203, 237, 4, 8, 2, 1, 208, 36, 237, 4, 8, 6, 1, 42, 4, 106, - 8, 6, 1, 247, 196, 4, 106, 8, 2, 1, 247, 196, 4, 106, 8, 6, 1, 220, 145, - 4, 164, 8, 6, 1, 200, 44, 4, 237, 41, 106, 8, 2, 1, 206, 10, 4, 200, 146, - 199, 215, 8, 2, 1, 191, 167, 4, 200, 146, 199, 215, 8, 6, 1, 231, 13, - 201, 64, 8, 2, 1, 231, 13, 201, 64, 8, 6, 1, 78, 4, 106, 8, 6, 1, 126, - 164, 8, 6, 1, 154, 196, 12, 8, 6, 1, 235, 17, 4, 106, 8, 2, 1, 235, 17, - 4, 106, 8, 6, 1, 223, 38, 4, 106, 8, 2, 1, 223, 38, 4, 106, 8, 6, 1, 2, - 208, 107, 4, 228, 253, 199, 215, 8, 2, 1, 208, 107, 4, 228, 253, 199, - 215, 8, 6, 1, 207, 224, 4, 106, 8, 2, 1, 207, 224, 4, 106, 8, 6, 1, 192, - 160, 4, 106, 8, 2, 1, 192, 160, 4, 106, 8, 2, 1, 154, 65, 8, 2, 1, 251, - 162, 8, 2, 1, 154, 251, 162, 8, 2, 1, 78, 4, 102, 8, 2, 1, 211, 79, 74, - 8, 2, 1, 247, 196, 4, 238, 177, 8, 2, 1, 238, 130, 4, 199, 215, 8, 2, 1, - 238, 130, 4, 102, 8, 2, 1, 207, 19, 71, 8, 2, 1, 205, 87, 8, 2, 1, 205, - 88, 4, 102, 8, 2, 1, 211, 79, 71, 8, 2, 1, 207, 19, 211, 79, 71, 8, 2, 1, - 207, 19, 211, 79, 235, 17, 4, 102, 8, 2, 1, 242, 221, 207, 19, 211, 79, - 71, 8, 2, 1, 236, 141, 223, 38, 4, 106, 8, 2, 1, 232, 54, 4, 102, 8, 2, - 1, 27, 232, 53, 8, 1, 2, 6, 232, 53, 8, 2, 1, 231, 213, 8, 2, 1, 207, - 142, 228, 189, 8, 2, 1, 154, 230, 118, 8, 2, 1, 230, 119, 4, 102, 8, 2, - 1, 229, 199, 4, 102, 8, 2, 1, 228, 76, 4, 106, 8, 2, 1, 223, 85, 8, 1, 2, - 6, 68, 8, 2, 1, 220, 145, 4, 82, 198, 152, 8, 2, 1, 220, 145, 4, 248, - 233, 8, 2, 1, 220, 145, 4, 207, 25, 102, 8, 2, 1, 219, 164, 8, 2, 1, 154, - 218, 170, 8, 2, 1, 154, 218, 171, 4, 180, 219, 214, 8, 2, 1, 218, 171, 4, - 102, 8, 2, 1, 215, 64, 4, 45, 102, 8, 2, 1, 215, 64, 4, 207, 25, 102, 8, - 1, 2, 6, 215, 63, 8, 2, 1, 249, 84, 74, 8, 1, 2, 6, 211, 153, 8, 2, 1, - 242, 221, 211, 112, 8, 2, 1, 209, 213, 8, 2, 1, 154, 146, 8, 2, 1, 154, - 207, 224, 4, 180, 219, 214, 8, 2, 1, 154, 207, 224, 4, 102, 8, 2, 1, 207, - 224, 4, 180, 219, 214, 8, 2, 1, 207, 224, 4, 199, 215, 8, 2, 1, 207, 224, - 4, 232, 235, 8, 2, 1, 207, 19, 207, 224, 4, 232, 235, 8, 1, 2, 6, 146, 8, - 1, 2, 6, 223, 95, 146, 8, 2, 1, 206, 10, 4, 102, 8, 2, 1, 234, 14, 8, 2, - 1, 236, 141, 223, 38, 4, 201, 29, 23, 102, 8, 2, 1, 201, 188, 207, 19, - 234, 14, 8, 2, 1, 234, 15, 4, 238, 177, 8, 2, 1, 154, 200, 43, 8, 2, 1, - 200, 44, 4, 207, 25, 102, 8, 2, 1, 126, 164, 8, 2, 1, 196, 70, 8, 2, 1, - 196, 13, 4, 102, 8, 2, 1, 154, 196, 12, 8, 2, 1, 154, 193, 224, 8, 2, 1, - 154, 192, 159, 8, 1, 2, 6, 192, 159, 8, 2, 1, 191, 167, 4, 207, 25, 102, - 8, 2, 1, 191, 167, 4, 238, 177, 8, 2, 1, 233, 180, 8, 2, 1, 233, 181, 4, - 238, 177, 8, 1, 231, 13, 201, 64, 8, 1, 209, 221, 195, 20, 232, 106, 8, - 1, 223, 95, 231, 13, 201, 64, 8, 1, 201, 37, 247, 195, 8, 1, 248, 174, - 242, 233, 8, 1, 2, 6, 250, 122, 8, 2, 1, 242, 221, 211, 79, 71, 8, 1, 2, - 6, 232, 54, 4, 102, 8, 1, 2, 6, 230, 118, 8, 2, 1, 223, 38, 4, 238, 214, - 8, 2, 1, 154, 222, 154, 8, 1, 2, 6, 172, 8, 2, 1, 208, 107, 4, 102, 8, 1, - 231, 13, 201, 65, 4, 106, 8, 1, 207, 19, 231, 13, 201, 65, 4, 106, 8, 2, - 1, 235, 39, 237, 4, 8, 2, 1, 237, 72, 237, 4, 8, 2, 1, 235, 39, 237, 5, - 4, 238, 177, 8, 2, 1, 197, 170, 237, 4, 8, 2, 1, 199, 79, 237, 4, 8, 2, - 1, 199, 152, 237, 5, 4, 238, 177, 8, 2, 1, 233, 39, 237, 4, 8, 2, 1, 218, - 229, 237, 4, 8, 2, 1, 218, 172, 237, 4, 8, 1, 248, 174, 210, 14, 8, 1, - 248, 182, 210, 14, 8, 2, 1, 154, 230, 119, 4, 232, 235, 8, 2, 1, 154, - 230, 119, 4, 232, 236, 23, 199, 215, 52, 1, 2, 230, 118, 52, 1, 2, 230, - 119, 4, 102, 52, 1, 2, 223, 37, 52, 1, 2, 146, 52, 1, 2, 154, 146, 52, 1, - 2, 154, 207, 224, 4, 102, 52, 1, 2, 6, 223, 95, 146, 52, 1, 2, 193, 224, - 52, 1, 2, 192, 159, 52, 1, 208, 217, 52, 1, 55, 208, 217, 52, 1, 154, - 242, 76, 52, 1, 251, 51, 52, 1, 207, 19, 242, 76, 52, 1, 50, 132, 206, - 189, 52, 1, 45, 132, 206, 189, 52, 1, 231, 13, 201, 64, 52, 1, 207, 19, - 231, 13, 201, 64, 52, 1, 45, 250, 237, 52, 1, 50, 250, 237, 52, 1, 133, - 250, 237, 52, 1, 144, 250, 237, 52, 1, 243, 4, 252, 62, 247, 94, 52, 1, - 81, 219, 114, 52, 1, 217, 148, 52, 1, 252, 49, 252, 62, 52, 1, 230, 213, - 252, 62, 52, 1, 130, 81, 219, 114, 52, 1, 130, 217, 148, 52, 1, 130, 230, - 213, 252, 62, 52, 1, 130, 252, 49, 252, 62, 52, 1, 197, 238, 242, 85, 52, - 1, 132, 197, 238, 242, 85, 52, 1, 247, 22, 50, 132, 206, 189, 52, 1, 247, - 22, 45, 132, 206, 189, 52, 1, 133, 199, 228, 52, 1, 144, 199, 228, 52, 1, - 108, 56, 52, 1, 216, 37, 56, 248, 38, 75, 58, 206, 190, 58, 211, 140, 2, - 198, 152, 55, 252, 49, 252, 62, 52, 1, 207, 3, 102, 52, 1, 238, 220, 252, - 62, 52, 1, 2, 231, 213, 52, 1, 2, 172, 52, 1, 2, 206, 9, 52, 1, 2, 192, - 235, 52, 1, 2, 207, 19, 231, 13, 201, 64, 52, 1, 233, 202, 163, 164, 52, - 1, 137, 163, 164, 52, 1, 216, 89, 163, 164, 52, 1, 130, 163, 164, 52, 1, - 233, 201, 163, 164, 52, 1, 192, 22, 237, 97, 163, 77, 52, 1, 192, 107, - 237, 97, 163, 77, 52, 1, 195, 18, 52, 1, 196, 109, 52, 1, 55, 251, 51, - 52, 1, 130, 144, 250, 237, 52, 1, 130, 133, 250, 237, 52, 1, 130, 45, - 250, 237, 52, 1, 130, 50, 250, 237, 52, 1, 130, 206, 189, 52, 1, 82, 230, - 213, 252, 62, 52, 1, 82, 55, 230, 213, 252, 62, 52, 1, 82, 55, 252, 49, - 252, 62, 52, 1, 130, 198, 152, 52, 1, 207, 149, 242, 85, 52, 1, 248, 251, - 137, 198, 79, 52, 1, 234, 97, 137, 198, 79, 52, 1, 248, 251, 130, 198, - 79, 52, 1, 234, 97, 130, 198, 79, 52, 1, 203, 105, 52, 1, 211, 79, 203, - 105, 52, 1, 130, 45, 57, 33, 230, 213, 252, 62, 33, 252, 49, 252, 62, 33, - 243, 4, 252, 62, 33, 198, 152, 33, 217, 148, 33, 211, 0, 33, 248, 38, 33, - 75, 58, 33, 237, 44, 33, 228, 253, 58, 33, 206, 190, 58, 33, 55, 252, 49, - 252, 62, 33, 247, 94, 33, 81, 219, 115, 58, 33, 55, 81, 219, 115, 58, 33, - 55, 230, 213, 252, 62, 33, 247, 121, 33, 223, 95, 248, 38, 33, 154, 242, - 77, 58, 33, 242, 77, 58, 33, 207, 19, 242, 77, 58, 33, 242, 77, 93, 179, - 33, 230, 213, 252, 63, 60, 33, 252, 49, 252, 63, 60, 33, 45, 199, 229, - 60, 33, 50, 199, 229, 60, 33, 45, 251, 118, 58, 33, 228, 189, 33, 45, - 132, 206, 190, 60, 33, 133, 199, 229, 60, 33, 144, 199, 229, 60, 33, 108, - 3, 60, 33, 216, 37, 3, 60, 33, 210, 192, 228, 253, 60, 33, 207, 25, 228, - 253, 60, 33, 75, 60, 33, 237, 45, 60, 33, 206, 190, 60, 33, 242, 77, 60, - 33, 247, 37, 33, 211, 140, 33, 81, 219, 115, 60, 33, 248, 31, 60, 33, - 223, 95, 55, 251, 17, 60, 33, 247, 95, 60, 33, 243, 4, 252, 63, 60, 33, - 248, 39, 60, 33, 223, 95, 248, 39, 60, 33, 198, 153, 60, 33, 217, 149, - 60, 33, 130, 219, 114, 33, 55, 130, 219, 114, 33, 198, 153, 211, 1, 33, - 203, 41, 201, 29, 211, 1, 33, 180, 201, 29, 211, 1, 33, 203, 41, 202, 25, - 211, 1, 33, 180, 202, 25, 211, 1, 33, 50, 132, 206, 190, 60, 33, 223, 95, - 248, 31, 60, 33, 51, 60, 33, 205, 63, 60, 33, 192, 236, 58, 33, 81, 198, - 152, 33, 55, 211, 0, 33, 230, 213, 163, 77, 33, 252, 49, 163, 77, 33, 35, - 210, 6, 33, 35, 221, 32, 33, 35, 237, 38, 198, 60, 33, 35, 191, 233, 33, - 248, 31, 58, 33, 234, 45, 3, 60, 33, 55, 81, 219, 115, 60, 33, 45, 251, - 118, 60, 33, 213, 14, 198, 153, 58, 33, 229, 3, 58, 33, 251, 167, 234, - 47, 119, 58, 33, 45, 50, 64, 60, 33, 196, 66, 64, 60, 33, 230, 219, 222, - 198, 33, 50, 250, 238, 58, 33, 45, 132, 206, 190, 58, 33, 233, 36, 33, - 192, 236, 60, 33, 45, 250, 238, 60, 33, 50, 250, 238, 60, 33, 50, 250, - 238, 23, 133, 250, 238, 60, 33, 50, 132, 206, 190, 58, 33, 75, 93, 179, - 33, 250, 196, 60, 33, 55, 206, 190, 60, 33, 191, 21, 58, 33, 55, 248, 39, - 60, 33, 55, 248, 38, 33, 55, 217, 148, 33, 55, 217, 149, 60, 33, 55, 198, - 152, 33, 55, 223, 95, 248, 38, 33, 55, 96, 64, 60, 33, 8, 2, 1, 65, 33, - 8, 2, 1, 71, 33, 8, 2, 1, 68, 33, 8, 2, 1, 74, 33, 8, 2, 1, 66, 33, 8, 2, - 1, 247, 195, 33, 8, 2, 1, 238, 129, 33, 8, 2, 1, 230, 118, 33, 8, 2, 1, - 218, 170, 33, 8, 2, 1, 146, 33, 8, 2, 1, 200, 43, 33, 8, 2, 1, 196, 12, - 33, 8, 2, 1, 192, 235, 35, 6, 1, 229, 187, 35, 2, 1, 229, 187, 35, 6, 1, - 251, 16, 205, 146, 35, 2, 1, 251, 16, 205, 146, 35, 212, 136, 56, 35, - 110, 212, 136, 56, 35, 6, 1, 210, 173, 237, 12, 35, 2, 1, 210, 173, 237, - 12, 35, 191, 233, 35, 2, 207, 19, 218, 208, 202, 198, 113, 35, 2, 235, - 140, 218, 208, 202, 198, 113, 35, 2, 207, 19, 235, 140, 218, 208, 202, - 198, 113, 35, 208, 15, 77, 35, 6, 1, 191, 240, 35, 198, 60, 35, 237, 38, - 198, 60, 35, 6, 1, 251, 163, 4, 198, 60, 35, 251, 96, 199, 108, 35, 6, 1, - 234, 50, 4, 198, 60, 35, 6, 1, 234, 0, 4, 198, 60, 35, 6, 1, 223, 86, 4, - 198, 60, 35, 6, 1, 211, 110, 4, 198, 60, 35, 6, 1, 196, 71, 4, 198, 60, - 35, 6, 1, 211, 113, 4, 198, 60, 35, 2, 1, 223, 86, 4, 237, 38, 23, 198, - 60, 35, 6, 1, 251, 162, 35, 6, 1, 248, 214, 35, 6, 1, 231, 213, 35, 6, 1, - 237, 108, 35, 6, 1, 234, 49, 35, 6, 1, 191, 76, 35, 6, 1, 233, 255, 35, - 6, 1, 199, 15, 35, 6, 1, 223, 85, 35, 6, 1, 222, 74, 35, 6, 1, 220, 33, - 35, 6, 1, 215, 157, 35, 6, 1, 212, 180, 35, 6, 1, 192, 207, 35, 6, 1, - 211, 109, 35, 6, 1, 209, 187, 35, 6, 1, 207, 4, 35, 6, 1, 202, 197, 35, - 6, 1, 199, 166, 35, 6, 1, 196, 70, 35, 6, 1, 209, 213, 35, 6, 1, 243, 97, - 35, 6, 1, 208, 178, 35, 6, 1, 211, 112, 35, 6, 1, 223, 86, 4, 237, 37, - 35, 6, 1, 196, 71, 4, 237, 37, 35, 2, 1, 251, 163, 4, 198, 60, 35, 2, 1, - 234, 50, 4, 198, 60, 35, 2, 1, 234, 0, 4, 198, 60, 35, 2, 1, 223, 86, 4, - 198, 60, 35, 2, 1, 196, 71, 4, 237, 38, 23, 198, 60, 35, 2, 1, 251, 162, - 35, 2, 1, 248, 214, 35, 2, 1, 231, 213, 35, 2, 1, 237, 108, 35, 2, 1, - 234, 49, 35, 2, 1, 191, 76, 35, 2, 1, 233, 255, 35, 2, 1, 199, 15, 35, 2, - 1, 223, 85, 35, 2, 1, 222, 74, 35, 2, 1, 220, 33, 35, 2, 1, 215, 157, 35, - 2, 1, 212, 180, 35, 2, 1, 192, 207, 35, 2, 1, 211, 109, 35, 2, 1, 209, - 187, 35, 2, 1, 207, 4, 35, 2, 1, 53, 202, 197, 35, 2, 1, 202, 197, 35, 2, - 1, 199, 166, 35, 2, 1, 196, 70, 35, 2, 1, 209, 213, 35, 2, 1, 243, 97, - 35, 2, 1, 208, 178, 35, 2, 1, 211, 112, 35, 2, 1, 223, 86, 4, 237, 37, - 35, 2, 1, 196, 71, 4, 237, 37, 35, 2, 1, 211, 110, 4, 198, 60, 35, 2, 1, - 196, 71, 4, 198, 60, 35, 2, 1, 211, 113, 4, 198, 60, 35, 6, 222, 105, - 113, 35, 248, 215, 113, 35, 199, 16, 113, 35, 196, 71, 4, 228, 253, 113, - 35, 196, 71, 4, 252, 49, 23, 228, 253, 113, 35, 196, 71, 4, 237, 45, 23, - 228, 253, 113, 35, 209, 214, 113, 35, 209, 188, 113, 35, 222, 105, 113, - 35, 1, 251, 16, 221, 37, 35, 2, 1, 251, 16, 221, 37, 35, 1, 201, 74, 35, - 2, 1, 201, 74, 35, 1, 237, 12, 35, 2, 1, 237, 12, 35, 1, 221, 37, 35, 2, - 1, 221, 37, 35, 1, 205, 146, 35, 2, 1, 205, 146, 94, 6, 1, 203, 106, 94, - 2, 1, 203, 106, 94, 6, 1, 233, 46, 94, 2, 1, 233, 46, 94, 6, 1, 221, 197, - 94, 2, 1, 221, 197, 94, 6, 1, 228, 244, 94, 2, 1, 228, 244, 94, 6, 1, - 231, 208, 94, 2, 1, 231, 208, 94, 6, 1, 203, 72, 94, 2, 1, 203, 72, 94, - 6, 1, 237, 124, 94, 2, 1, 237, 124, 35, 222, 75, 113, 35, 207, 5, 113, - 35, 218, 208, 202, 198, 113, 35, 1, 191, 240, 35, 6, 199, 16, 113, 35, - 218, 208, 234, 50, 113, 35, 207, 19, 218, 208, 234, 50, 113, 35, 6, 1, - 203, 57, 35, 2, 1, 203, 57, 35, 6, 218, 208, 202, 198, 113, 35, 6, 1, - 205, 143, 35, 2, 1, 205, 143, 35, 207, 5, 4, 201, 29, 113, 35, 6, 207, - 19, 218, 208, 202, 198, 113, 35, 6, 235, 140, 218, 208, 202, 198, 113, - 35, 6, 207, 19, 235, 140, 218, 208, 202, 198, 113, 38, 6, 1, 223, 229, 4, - 230, 212, 38, 6, 1, 223, 90, 38, 6, 1, 236, 194, 38, 6, 1, 231, 22, 38, - 6, 1, 196, 125, 223, 228, 38, 6, 1, 235, 34, 38, 6, 1, 247, 205, 68, 38, - 6, 1, 192, 33, 38, 6, 1, 223, 12, 38, 6, 1, 219, 78, 38, 6, 1, 213, 152, - 38, 6, 1, 197, 155, 38, 6, 1, 221, 105, 38, 6, 1, 228, 76, 4, 230, 212, - 38, 6, 1, 203, 41, 66, 38, 6, 1, 235, 30, 38, 6, 1, 65, 38, 6, 1, 249, - 19, 38, 6, 1, 195, 153, 38, 6, 1, 231, 79, 38, 6, 1, 237, 148, 38, 6, 1, - 223, 228, 38, 6, 1, 191, 62, 38, 6, 1, 191, 87, 38, 6, 1, 68, 38, 6, 1, - 203, 41, 68, 38, 6, 1, 155, 38, 6, 1, 234, 142, 38, 6, 1, 234, 116, 38, - 6, 1, 234, 105, 38, 6, 1, 74, 38, 6, 1, 210, 65, 38, 6, 1, 234, 36, 38, - 6, 1, 234, 24, 38, 6, 1, 199, 145, 38, 6, 1, 66, 38, 6, 1, 234, 183, 38, - 6, 1, 140, 38, 6, 1, 197, 161, 38, 6, 1, 243, 129, 38, 6, 1, 203, 166, - 38, 6, 1, 203, 117, 38, 6, 1, 230, 19, 56, 38, 6, 1, 192, 58, 38, 6, 1, - 202, 33, 56, 38, 6, 1, 71, 38, 6, 1, 191, 225, 38, 6, 1, 170, 38, 2, 1, - 65, 38, 2, 1, 249, 19, 38, 2, 1, 195, 153, 38, 2, 1, 231, 79, 38, 2, 1, - 237, 148, 38, 2, 1, 223, 228, 38, 2, 1, 191, 62, 38, 2, 1, 191, 87, 38, - 2, 1, 68, 38, 2, 1, 203, 41, 68, 38, 2, 1, 155, 38, 2, 1, 234, 142, 38, - 2, 1, 234, 116, 38, 2, 1, 234, 105, 38, 2, 1, 74, 38, 2, 1, 210, 65, 38, - 2, 1, 234, 36, 38, 2, 1, 234, 24, 38, 2, 1, 199, 145, 38, 2, 1, 66, 38, - 2, 1, 234, 183, 38, 2, 1, 140, 38, 2, 1, 197, 161, 38, 2, 1, 243, 129, - 38, 2, 1, 203, 166, 38, 2, 1, 203, 117, 38, 2, 1, 230, 19, 56, 38, 2, 1, - 192, 58, 38, 2, 1, 202, 33, 56, 38, 2, 1, 71, 38, 2, 1, 191, 225, 38, 2, - 1, 170, 38, 2, 1, 223, 229, 4, 230, 212, 38, 2, 1, 223, 90, 38, 2, 1, - 236, 194, 38, 2, 1, 231, 22, 38, 2, 1, 196, 125, 223, 228, 38, 2, 1, 235, - 34, 38, 2, 1, 247, 205, 68, 38, 2, 1, 192, 33, 38, 2, 1, 223, 12, 38, 2, - 1, 219, 78, 38, 2, 1, 213, 152, 38, 2, 1, 197, 155, 38, 2, 1, 221, 105, - 38, 2, 1, 228, 76, 4, 230, 212, 38, 2, 1, 203, 41, 66, 38, 2, 1, 235, 30, - 38, 6, 1, 211, 112, 38, 2, 1, 211, 112, 38, 6, 1, 192, 95, 38, 2, 1, 192, - 95, 38, 6, 1, 223, 83, 71, 38, 2, 1, 223, 83, 71, 38, 6, 1, 219, 85, 191, - 190, 38, 2, 1, 219, 85, 191, 190, 38, 6, 1, 223, 83, 219, 85, 191, 190, - 38, 2, 1, 223, 83, 219, 85, 191, 190, 38, 6, 1, 248, 177, 191, 190, 38, - 2, 1, 248, 177, 191, 190, 38, 6, 1, 223, 83, 248, 177, 191, 190, 38, 2, - 1, 223, 83, 248, 177, 191, 190, 38, 6, 1, 220, 250, 38, 2, 1, 220, 250, - 38, 6, 1, 208, 178, 38, 2, 1, 208, 178, 38, 6, 1, 232, 230, 38, 2, 1, - 232, 230, 38, 6, 1, 223, 39, 38, 2, 1, 223, 39, 38, 6, 1, 223, 40, 4, 55, - 230, 213, 252, 62, 38, 2, 1, 223, 40, 4, 55, 230, 213, 252, 62, 38, 6, 1, - 196, 128, 38, 2, 1, 196, 128, 38, 6, 1, 206, 116, 211, 112, 38, 2, 1, - 206, 116, 211, 112, 38, 6, 1, 211, 113, 4, 198, 122, 38, 2, 1, 211, 113, - 4, 198, 122, 38, 6, 1, 211, 32, 38, 2, 1, 211, 32, 38, 6, 1, 221, 37, 38, - 2, 1, 221, 37, 38, 198, 229, 56, 33, 38, 198, 122, 33, 38, 210, 193, 33, - 38, 237, 215, 209, 77, 33, 38, 208, 172, 209, 77, 33, 38, 209, 56, 33, - 38, 228, 143, 198, 229, 56, 33, 38, 216, 50, 56, 38, 6, 1, 203, 41, 228, - 76, 4, 199, 215, 38, 2, 1, 203, 41, 228, 76, 4, 199, 215, 38, 6, 1, 204, - 22, 56, 38, 2, 1, 204, 22, 56, 38, 6, 1, 234, 37, 4, 198, 182, 38, 2, 1, - 234, 37, 4, 198, 182, 38, 6, 1, 231, 80, 4, 196, 69, 38, 2, 1, 231, 80, - 4, 196, 69, 38, 6, 1, 231, 80, 4, 106, 38, 2, 1, 231, 80, 4, 106, 38, 6, - 1, 231, 80, 4, 82, 102, 38, 2, 1, 231, 80, 4, 82, 102, 38, 6, 1, 191, 63, - 4, 237, 89, 38, 2, 1, 191, 63, 4, 237, 89, 38, 6, 1, 191, 88, 4, 237, 89, - 38, 2, 1, 191, 88, 4, 237, 89, 38, 6, 1, 222, 144, 4, 237, 89, 38, 2, 1, - 222, 144, 4, 237, 89, 38, 6, 1, 222, 144, 4, 81, 106, 38, 2, 1, 222, 144, - 4, 81, 106, 38, 6, 1, 222, 144, 4, 106, 38, 2, 1, 222, 144, 4, 106, 38, - 6, 1, 249, 72, 155, 38, 2, 1, 249, 72, 155, 38, 6, 1, 234, 106, 4, 237, - 89, 38, 2, 1, 234, 106, 4, 237, 89, 38, 6, 34, 234, 106, 231, 79, 38, 2, - 34, 234, 106, 231, 79, 38, 6, 1, 210, 66, 4, 82, 102, 38, 2, 1, 210, 66, - 4, 82, 102, 38, 6, 1, 252, 69, 140, 38, 2, 1, 252, 69, 140, 38, 6, 1, - 234, 25, 4, 237, 89, 38, 2, 1, 234, 25, 4, 237, 89, 38, 6, 1, 199, 146, - 4, 237, 89, 38, 2, 1, 199, 146, 4, 237, 89, 38, 6, 1, 201, 54, 66, 38, 2, - 1, 201, 54, 66, 38, 6, 1, 201, 54, 126, 4, 106, 38, 2, 1, 201, 54, 126, - 4, 106, 38, 6, 1, 230, 107, 4, 237, 89, 38, 2, 1, 230, 107, 4, 237, 89, - 38, 6, 34, 199, 146, 197, 161, 38, 2, 34, 199, 146, 197, 161, 38, 6, 1, - 243, 130, 4, 237, 89, 38, 2, 1, 243, 130, 4, 237, 89, 38, 6, 1, 243, 130, - 4, 81, 106, 38, 2, 1, 243, 130, 4, 81, 106, 38, 6, 1, 203, 83, 38, 2, 1, - 203, 83, 38, 6, 1, 252, 69, 243, 129, 38, 2, 1, 252, 69, 243, 129, 38, 6, - 1, 252, 69, 243, 130, 4, 237, 89, 38, 2, 1, 252, 69, 243, 130, 4, 237, - 89, 38, 1, 210, 181, 38, 6, 1, 191, 63, 4, 248, 38, 38, 2, 1, 191, 63, 4, - 248, 38, 38, 6, 1, 222, 144, 4, 102, 38, 2, 1, 222, 144, 4, 102, 38, 6, - 1, 234, 143, 4, 199, 215, 38, 2, 1, 234, 143, 4, 199, 215, 38, 6, 1, 234, - 106, 4, 102, 38, 2, 1, 234, 106, 4, 102, 38, 6, 1, 234, 106, 4, 199, 215, - 38, 2, 1, 234, 106, 4, 199, 215, 38, 6, 1, 221, 210, 243, 129, 38, 2, 1, - 221, 210, 243, 129, 38, 6, 1, 234, 117, 4, 199, 215, 38, 2, 1, 234, 117, - 4, 199, 215, 38, 2, 1, 210, 181, 38, 6, 1, 42, 4, 248, 38, 38, 2, 1, 42, - 4, 248, 38, 38, 6, 1, 42, 4, 237, 44, 38, 2, 1, 42, 4, 237, 44, 38, 6, - 34, 42, 223, 228, 38, 2, 34, 42, 223, 228, 38, 6, 1, 223, 229, 4, 248, - 38, 38, 2, 1, 223, 229, 4, 248, 38, 38, 6, 1, 205, 87, 38, 2, 1, 205, 87, - 38, 6, 1, 205, 88, 4, 237, 44, 38, 2, 1, 205, 88, 4, 237, 44, 38, 6, 1, - 191, 63, 4, 237, 44, 38, 2, 1, 191, 63, 4, 237, 44, 38, 6, 1, 191, 88, 4, - 237, 44, 38, 2, 1, 191, 88, 4, 237, 44, 38, 6, 1, 252, 69, 235, 34, 38, - 2, 1, 252, 69, 235, 34, 38, 6, 1, 228, 76, 4, 217, 148, 38, 2, 1, 228, - 76, 4, 217, 148, 38, 6, 1, 228, 76, 4, 237, 44, 38, 2, 1, 228, 76, 4, - 237, 44, 38, 6, 1, 187, 4, 237, 44, 38, 2, 1, 187, 4, 237, 44, 38, 6, 1, - 249, 84, 74, 38, 2, 1, 249, 84, 74, 38, 6, 1, 249, 84, 187, 4, 237, 44, - 38, 2, 1, 249, 84, 187, 4, 237, 44, 38, 6, 1, 235, 17, 4, 237, 44, 38, 2, - 1, 235, 17, 4, 237, 44, 38, 6, 1, 126, 4, 217, 148, 38, 2, 1, 126, 4, - 217, 148, 38, 6, 1, 126, 4, 237, 44, 38, 2, 1, 126, 4, 237, 44, 38, 6, 1, - 126, 4, 55, 252, 48, 38, 2, 1, 126, 4, 55, 252, 48, 38, 6, 1, 243, 130, - 4, 237, 44, 38, 2, 1, 243, 130, 4, 237, 44, 38, 6, 1, 231, 80, 4, 237, - 89, 38, 2, 1, 231, 80, 4, 237, 89, 38, 6, 1, 192, 59, 4, 237, 44, 38, 2, - 1, 192, 59, 4, 237, 44, 38, 6, 1, 231, 80, 4, 201, 29, 23, 102, 38, 2, 1, - 231, 80, 4, 201, 29, 23, 102, 38, 6, 1, 230, 107, 4, 102, 38, 2, 1, 230, - 107, 4, 102, 38, 6, 1, 230, 107, 4, 106, 38, 2, 1, 230, 107, 4, 106, 38, - 6, 1, 221, 47, 237, 148, 38, 2, 1, 221, 47, 237, 148, 38, 6, 1, 221, 47, - 236, 194, 38, 2, 1, 221, 47, 236, 194, 38, 6, 1, 221, 47, 191, 12, 38, 2, - 1, 221, 47, 191, 12, 38, 6, 1, 221, 47, 235, 26, 38, 2, 1, 221, 47, 235, - 26, 38, 6, 1, 221, 47, 219, 78, 38, 2, 1, 221, 47, 219, 78, 38, 6, 1, - 221, 47, 213, 152, 38, 2, 1, 221, 47, 213, 152, 38, 6, 1, 221, 47, 202, - 116, 38, 2, 1, 221, 47, 202, 116, 38, 6, 1, 221, 47, 198, 116, 38, 2, 1, - 221, 47, 198, 116, 38, 6, 1, 207, 19, 191, 87, 38, 2, 1, 207, 19, 191, - 87, 38, 6, 1, 234, 143, 4, 102, 38, 2, 1, 234, 143, 4, 102, 38, 6, 1, - 219, 161, 38, 2, 1, 219, 161, 38, 6, 1, 207, 7, 38, 2, 1, 207, 7, 38, 6, - 1, 192, 129, 38, 2, 1, 192, 129, 38, 6, 1, 208, 98, 38, 2, 1, 208, 98, - 38, 6, 1, 193, 125, 38, 2, 1, 193, 125, 38, 6, 1, 251, 190, 155, 38, 2, - 1, 251, 190, 155, 38, 6, 1, 234, 143, 4, 82, 102, 38, 2, 1, 234, 143, 4, - 82, 102, 38, 6, 1, 234, 106, 4, 82, 102, 38, 2, 1, 234, 106, 4, 82, 102, - 38, 6, 1, 210, 66, 4, 237, 89, 38, 2, 1, 210, 66, 4, 237, 89, 38, 6, 1, - 203, 84, 4, 237, 89, 38, 2, 1, 203, 84, 4, 237, 89, 38, 6, 1, 234, 106, - 4, 45, 102, 38, 2, 1, 234, 106, 4, 45, 102, 38, 6, 1, 235, 18, 38, 2, 1, - 235, 18, 38, 6, 1, 237, 197, 38, 2, 1, 237, 197, 38, 6, 1, 234, 143, 4, - 237, 89, 38, 2, 1, 234, 143, 4, 237, 89, 250, 251, 6, 1, 250, 130, 250, - 251, 6, 1, 248, 231, 250, 251, 6, 1, 231, 42, 250, 251, 6, 1, 238, 34, - 250, 251, 6, 1, 234, 197, 250, 251, 6, 1, 191, 123, 250, 251, 6, 1, 234, - 175, 250, 251, 6, 1, 234, 1, 250, 251, 6, 1, 159, 250, 251, 6, 1, 191, - 62, 250, 251, 6, 1, 223, 133, 250, 251, 6, 1, 219, 82, 250, 251, 6, 1, - 192, 212, 250, 251, 6, 1, 247, 162, 250, 251, 6, 1, 221, 253, 250, 251, - 6, 1, 229, 25, 250, 251, 6, 1, 223, 34, 250, 251, 6, 1, 231, 90, 250, - 251, 6, 1, 243, 119, 250, 251, 6, 1, 216, 188, 250, 251, 6, 1, 192, 33, - 250, 251, 6, 1, 212, 255, 250, 251, 6, 1, 203, 166, 250, 251, 6, 1, 195, - 24, 250, 251, 6, 1, 247, 3, 250, 251, 6, 1, 210, 43, 250, 251, 6, 1, 222, - 249, 250, 251, 6, 1, 165, 250, 251, 6, 1, 205, 40, 250, 251, 6, 1, 195, - 74, 250, 251, 6, 1, 198, 119, 250, 251, 6, 1, 207, 72, 250, 251, 6, 1, - 242, 101, 250, 251, 6, 1, 192, 17, 250, 251, 6, 1, 209, 116, 250, 251, 6, - 1, 222, 8, 250, 251, 6, 1, 211, 138, 250, 251, 6, 1, 233, 48, 250, 251, - 52, 1, 45, 132, 206, 189, 250, 251, 251, 51, 250, 251, 234, 109, 77, 250, - 251, 233, 218, 77, 250, 251, 242, 76, 250, 251, 208, 15, 77, 250, 251, - 252, 70, 77, 250, 251, 2, 1, 154, 250, 130, 250, 251, 2, 1, 250, 130, - 250, 251, 2, 1, 248, 231, 250, 251, 2, 1, 231, 42, 250, 251, 2, 1, 238, - 34, 250, 251, 2, 1, 234, 197, 250, 251, 2, 1, 191, 123, 250, 251, 2, 1, - 234, 175, 250, 251, 2, 1, 234, 1, 250, 251, 2, 1, 159, 250, 251, 2, 1, - 191, 62, 250, 251, 2, 1, 223, 133, 250, 251, 2, 1, 219, 82, 250, 251, 2, - 1, 192, 212, 250, 251, 2, 1, 247, 162, 250, 251, 2, 1, 221, 253, 250, - 251, 2, 1, 229, 25, 250, 251, 2, 1, 223, 34, 250, 251, 2, 1, 231, 90, - 250, 251, 2, 1, 243, 119, 250, 251, 2, 1, 216, 188, 250, 251, 2, 1, 192, - 33, 250, 251, 2, 1, 212, 255, 250, 251, 2, 1, 203, 166, 250, 251, 2, 1, - 195, 24, 250, 251, 2, 1, 247, 3, 250, 251, 2, 1, 210, 43, 250, 251, 2, 1, - 222, 249, 250, 251, 2, 1, 165, 250, 251, 2, 1, 205, 40, 250, 251, 2, 1, - 195, 74, 250, 251, 2, 1, 198, 119, 250, 251, 2, 1, 207, 72, 250, 251, 2, - 1, 242, 101, 250, 251, 2, 1, 192, 17, 250, 251, 2, 1, 209, 116, 250, 251, - 2, 1, 222, 8, 250, 251, 2, 1, 211, 138, 250, 251, 2, 1, 233, 48, 250, - 251, 2, 34, 234, 198, 192, 17, 250, 251, 2, 1, 11, 4, 106, 250, 251, 232, - 82, 201, 64, 250, 251, 228, 90, 206, 208, 250, 251, 233, 253, 56, 219, - 225, 250, 251, 233, 253, 56, 250, 251, 235, 112, 56, 136, 252, 63, 233, - 248, 136, 252, 63, 205, 41, 136, 252, 63, 203, 142, 136, 252, 63, 191, - 99, 208, 80, 136, 252, 63, 191, 99, 231, 232, 136, 252, 63, 198, 134, - 136, 252, 63, 207, 16, 136, 252, 63, 191, 97, 136, 252, 63, 210, 99, 136, - 252, 63, 192, 48, 136, 252, 63, 199, 56, 136, 252, 63, 231, 141, 136, - 252, 63, 231, 142, 215, 112, 136, 252, 63, 231, 139, 136, 252, 63, 208, - 82, 210, 132, 136, 252, 63, 199, 103, 231, 160, 136, 252, 63, 210, 71, - 136, 252, 63, 250, 175, 230, 87, 136, 252, 63, 215, 123, 136, 252, 63, - 217, 119, 136, 252, 63, 216, 177, 136, 252, 63, 216, 178, 222, 9, 136, - 252, 63, 237, 224, 136, 252, 63, 208, 93, 136, 252, 63, 199, 103, 208, - 75, 136, 252, 63, 192, 61, 248, 232, 191, 247, 136, 252, 63, 211, 119, - 136, 252, 63, 223, 185, 136, 252, 63, 237, 125, 136, 252, 63, 191, 19, - 136, 87, 217, 36, 243, 12, 136, 209, 64, 203, 86, 136, 209, 64, 230, 10, - 205, 41, 136, 209, 64, 230, 10, 210, 90, 136, 209, 64, 230, 10, 208, 86, - 136, 209, 64, 229, 121, 136, 209, 64, 197, 158, 136, 209, 64, 205, 41, - 136, 209, 64, 210, 90, 136, 209, 64, 208, 86, 136, 209, 64, 229, 9, 136, - 209, 64, 229, 10, 230, 12, 40, 195, 158, 136, 209, 64, 208, 20, 136, 209, - 64, 238, 19, 211, 59, 217, 72, 136, 209, 64, 216, 166, 136, 208, 154, - 217, 69, 136, 209, 64, 207, 163, 136, 208, 154, 210, 101, 136, 209, 64, - 203, 71, 236, 142, 136, 209, 64, 202, 176, 236, 142, 136, 208, 154, 202, - 34, 210, 92, 136, 87, 116, 236, 142, 136, 87, 110, 236, 142, 136, 208, - 154, 212, 133, 230, 86, 136, 209, 64, 208, 87, 208, 80, 136, 1, 251, 194, - 136, 1, 248, 216, 136, 1, 231, 40, 136, 1, 237, 255, 136, 1, 229, 247, - 136, 1, 195, 158, 136, 1, 191, 91, 136, 1, 229, 188, 136, 1, 199, 73, - 136, 1, 191, 250, 136, 1, 53, 222, 108, 136, 1, 222, 108, 136, 1, 220, - 29, 136, 1, 53, 216, 195, 136, 1, 216, 195, 136, 1, 53, 212, 132, 136, 1, - 212, 132, 136, 1, 205, 149, 136, 1, 250, 128, 136, 1, 53, 210, 65, 136, - 1, 210, 65, 136, 1, 53, 197, 163, 136, 1, 197, 163, 136, 1, 208, 44, 136, - 1, 207, 39, 136, 1, 203, 70, 136, 1, 199, 162, 136, 191, 251, 197, 241, - 136, 34, 192, 31, 55, 195, 158, 136, 34, 192, 31, 195, 159, 191, 250, - 136, 34, 192, 31, 55, 191, 250, 136, 208, 154, 231, 141, 136, 208, 154, - 231, 139, 9, 31, 56, 9, 3, 205, 142, 9, 232, 159, 217, 53, 9, 3, 205, - 188, 9, 3, 205, 145, 9, 31, 87, 58, 251, 30, 238, 193, 206, 129, 251, 30, - 232, 123, 206, 129, 9, 207, 124, 251, 30, 210, 16, 216, 52, 56, 251, 30, - 210, 16, 199, 96, 198, 230, 56, 252, 4, 56, 9, 242, 76, 9, 237, 211, 204, - 11, 9, 209, 66, 195, 137, 56, 9, 3, 216, 27, 9, 3, 205, 162, 251, 201, - 193, 149, 9, 3, 251, 201, 250, 200, 9, 3, 207, 159, 251, 200, 9, 3, 207, - 169, 251, 172, 251, 107, 9, 3, 199, 206, 9, 2, 137, 199, 219, 9, 2, 137, - 34, 131, 4, 220, 38, 4, 192, 75, 9, 2, 137, 191, 113, 9, 2, 233, 72, 9, - 2, 237, 247, 9, 2, 222, 54, 9, 204, 26, 9, 1, 77, 9, 234, 97, 79, 199, - 54, 77, 9, 197, 225, 75, 208, 154, 77, 9, 208, 15, 77, 9, 1, 222, 58, - 192, 75, 9, 1, 230, 59, 9, 1, 131, 4, 217, 144, 58, 9, 1, 131, 4, 230, - 60, 58, 9, 1, 193, 134, 4, 230, 60, 58, 9, 1, 131, 4, 230, 60, 60, 9, 1, - 99, 4, 230, 60, 58, 9, 1, 251, 194, 9, 1, 248, 247, 9, 1, 199, 115, 217, - 64, 9, 1, 199, 114, 9, 1, 199, 29, 9, 1, 223, 8, 9, 1, 230, 83, 9, 1, - 221, 212, 9, 1, 238, 5, 9, 1, 199, 41, 9, 1, 207, 72, 9, 1, 191, 113, 9, - 1, 205, 47, 9, 1, 203, 110, 9, 1, 205, 193, 9, 1, 238, 28, 9, 1, 199, - 219, 9, 1, 191, 116, 9, 1, 251, 232, 9, 1, 231, 88, 9, 1, 222, 7, 4, 105, - 185, 58, 9, 1, 222, 7, 4, 115, 185, 60, 9, 1, 233, 76, 99, 4, 223, 95, - 196, 12, 9, 1, 233, 76, 99, 4, 105, 185, 58, 9, 1, 233, 76, 99, 4, 115, - 185, 58, 9, 199, 168, 9, 1, 233, 48, 9, 1, 208, 91, 9, 1, 222, 108, 9, 1, - 220, 37, 9, 1, 216, 210, 9, 1, 213, 26, 9, 1, 229, 212, 9, 1, 193, 133, - 9, 1, 131, 217, 101, 9, 1, 192, 75, 9, 233, 70, 9, 237, 245, 9, 222, 52, - 9, 233, 72, 9, 237, 247, 9, 222, 54, 9, 203, 156, 9, 200, 206, 9, 217, - 142, 58, 9, 230, 60, 58, 9, 230, 60, 60, 9, 200, 231, 251, 194, 9, 223, - 95, 237, 247, 9, 87, 213, 27, 231, 59, 9, 190, 237, 9, 18, 3, 2, 196, 13, - 58, 9, 18, 3, 223, 95, 2, 196, 13, 58, 9, 18, 3, 75, 60, 9, 207, 19, 237, - 247, 9, 233, 73, 4, 105, 236, 140, 9, 193, 135, 230, 60, 60, 251, 30, 17, - 191, 77, 251, 30, 17, 107, 251, 30, 17, 109, 251, 30, 17, 138, 251, 30, - 17, 134, 251, 30, 17, 150, 251, 30, 17, 169, 251, 30, 17, 175, 251, 30, - 17, 171, 251, 30, 17, 178, 9, 210, 15, 56, 9, 237, 140, 204, 11, 9, 198, - 229, 204, 11, 9, 232, 228, 209, 62, 201, 103, 9, 1, 236, 141, 248, 247, - 9, 1, 236, 141, 208, 91, 9, 1, 200, 182, 251, 194, 9, 1, 131, 193, 150, - 9, 1, 131, 4, 193, 135, 230, 60, 58, 9, 1, 131, 4, 193, 135, 230, 60, 60, - 9, 1, 137, 230, 59, 9, 1, 137, 230, 60, 251, 194, 9, 1, 137, 230, 60, - 193, 133, 9, 1, 126, 4, 230, 60, 58, 9, 1, 137, 230, 60, 192, 75, 9, 1, - 197, 124, 9, 1, 197, 122, 9, 1, 249, 1, 9, 1, 199, 115, 4, 206, 189, 9, - 1, 199, 115, 4, 115, 185, 93, 235, 120, 9, 1, 210, 43, 9, 1, 199, 112, 9, - 1, 248, 245, 9, 1, 183, 4, 230, 60, 58, 9, 1, 183, 4, 105, 185, 81, 58, - 9, 1, 212, 89, 9, 1, 235, 43, 9, 1, 183, 4, 115, 185, 58, 9, 1, 199, 149, - 9, 1, 199, 147, 9, 1, 237, 188, 9, 1, 238, 6, 4, 206, 189, 9, 1, 238, 6, - 4, 75, 60, 9, 1, 238, 6, 4, 75, 248, 235, 23, 2, 199, 219, 9, 1, 238, 12, - 9, 1, 237, 190, 9, 1, 235, 80, 9, 1, 238, 6, 4, 115, 185, 93, 235, 120, - 9, 1, 238, 6, 4, 232, 130, 185, 58, 9, 1, 206, 102, 9, 1, 207, 73, 4, 2, - 196, 12, 9, 1, 207, 73, 4, 206, 189, 9, 1, 207, 73, 4, 75, 60, 9, 1, 207, - 73, 4, 2, 196, 13, 60, 9, 1, 207, 73, 4, 75, 248, 235, 23, 75, 58, 9, 1, - 207, 73, 4, 105, 185, 58, 9, 1, 223, 5, 9, 1, 207, 73, 4, 232, 130, 185, - 58, 9, 1, 205, 48, 4, 75, 248, 235, 23, 75, 58, 9, 1, 205, 48, 4, 115, - 185, 60, 9, 1, 205, 48, 4, 115, 185, 248, 235, 23, 115, 185, 58, 9, 1, - 205, 194, 4, 105, 185, 60, 9, 1, 205, 194, 4, 115, 185, 58, 9, 1, 199, - 220, 4, 115, 185, 58, 9, 1, 251, 233, 4, 115, 185, 58, 9, 1, 236, 141, - 233, 48, 9, 1, 233, 49, 4, 75, 215, 179, 60, 9, 1, 233, 49, 4, 75, 60, 9, - 1, 195, 146, 9, 1, 233, 49, 4, 115, 185, 60, 9, 1, 210, 41, 9, 1, 208, - 92, 4, 75, 58, 9, 1, 208, 92, 4, 115, 185, 58, 9, 1, 222, 6, 9, 1, 200, - 146, 222, 108, 9, 1, 222, 109, 4, 206, 189, 9, 1, 222, 109, 4, 75, 58, 9, - 1, 214, 72, 9, 1, 222, 109, 4, 115, 185, 60, 9, 1, 231, 229, 9, 1, 231, - 230, 4, 206, 189, 9, 1, 213, 249, 9, 1, 231, 230, 4, 105, 185, 60, 9, 1, - 230, 168, 9, 1, 231, 230, 4, 115, 185, 58, 9, 1, 220, 38, 4, 2, 196, 12, - 9, 1, 220, 38, 4, 75, 58, 9, 1, 220, 38, 4, 115, 185, 58, 9, 1, 220, 38, - 4, 115, 185, 60, 9, 1, 213, 27, 4, 75, 60, 9, 1, 213, 27, 231, 59, 9, 1, - 206, 166, 9, 1, 213, 27, 4, 206, 189, 9, 1, 213, 27, 4, 115, 185, 58, 9, - 1, 229, 213, 236, 172, 9, 1, 199, 150, 4, 75, 58, 9, 1, 229, 213, 4, 99, - 58, 9, 1, 229, 213, 231, 2, 9, 1, 229, 213, 231, 3, 4, 230, 60, 58, 9, 1, - 199, 115, 217, 65, 231, 2, 9, 1, 193, 134, 4, 206, 189, 9, 1, 221, 134, - 211, 153, 9, 1, 211, 153, 9, 1, 66, 9, 1, 191, 225, 9, 1, 221, 134, 191, - 225, 9, 1, 193, 134, 4, 105, 185, 58, 9, 1, 195, 153, 9, 1, 233, 76, 192, - 75, 9, 1, 99, 4, 199, 215, 9, 1, 99, 4, 2, 196, 12, 9, 1, 193, 134, 4, - 75, 58, 9, 1, 71, 9, 1, 99, 4, 115, 185, 60, 9, 1, 99, 249, 82, 9, 1, 99, - 249, 83, 4, 230, 60, 58, 9, 232, 82, 201, 64, 9, 1, 252, 27, 9, 2, 137, - 34, 205, 194, 4, 220, 38, 4, 131, 217, 101, 9, 2, 137, 34, 208, 92, 4, - 220, 38, 4, 131, 217, 101, 9, 2, 137, 92, 89, 20, 9, 2, 137, 220, 38, - 251, 194, 9, 2, 137, 223, 8, 9, 2, 137, 115, 236, 140, 9, 2, 137, 205, - 47, 9, 234, 97, 79, 250, 132, 9, 201, 99, 79, 206, 61, 234, 143, 229, - 116, 9, 2, 137, 206, 114, 191, 77, 9, 2, 137, 196, 73, 207, 93, 191, 77, - 9, 2, 137, 236, 141, 229, 238, 79, 221, 212, 9, 2, 137, 92, 76, 20, 9, 2, - 130, 205, 47, 9, 2, 137, 217, 143, 9, 2, 193, 133, 9, 2, 192, 75, 9, 2, - 137, 192, 75, 9, 2, 137, 213, 26, 9, 209, 110, 79, 205, 178, 9, 234, 107, - 247, 24, 130, 201, 64, 9, 234, 107, 247, 24, 137, 201, 64, 9, 206, 114, - 137, 201, 65, 4, 233, 6, 247, 23, 9, 2, 130, 216, 210, 9, 1, 238, 6, 4, - 223, 95, 196, 12, 9, 1, 207, 73, 4, 223, 95, 196, 12, 233, 207, 251, 30, - 17, 191, 77, 233, 207, 251, 30, 17, 107, 233, 207, 251, 30, 17, 109, 233, - 207, 251, 30, 17, 138, 233, 207, 251, 30, 17, 134, 233, 207, 251, 30, 17, - 150, 233, 207, 251, 30, 17, 169, 233, 207, 251, 30, 17, 175, 233, 207, - 251, 30, 17, 171, 233, 207, 251, 30, 17, 178, 9, 1, 203, 111, 4, 75, 60, - 9, 1, 238, 29, 4, 75, 60, 9, 1, 231, 89, 4, 75, 60, 9, 3, 202, 174, 251, - 139, 9, 3, 202, 174, 209, 18, 216, 188, 9, 1, 229, 213, 4, 223, 95, 196, - 12, 200, 63, 234, 97, 79, 210, 129, 200, 63, 200, 177, 232, 82, 201, 64, - 200, 63, 200, 233, 232, 82, 201, 64, 200, 63, 200, 177, 242, 85, 200, 63, - 200, 233, 242, 85, 200, 63, 228, 243, 242, 85, 200, 63, 242, 86, 202, - 111, 219, 226, 200, 63, 242, 86, 202, 111, 179, 200, 63, 200, 177, 242, - 86, 202, 111, 219, 226, 200, 63, 200, 233, 242, 86, 202, 111, 179, 200, - 63, 239, 26, 200, 63, 230, 17, 211, 178, 200, 63, 230, 17, 216, 164, 200, - 63, 230, 17, 250, 197, 200, 63, 252, 70, 77, 200, 63, 1, 251, 204, 200, - 63, 1, 200, 182, 251, 204, 200, 63, 1, 248, 213, 200, 63, 1, 231, 219, - 200, 63, 1, 231, 220, 231, 196, 200, 63, 1, 238, 2, 200, 63, 1, 236, 141, - 238, 3, 206, 182, 200, 63, 1, 229, 247, 200, 63, 1, 193, 133, 200, 63, 1, - 191, 113, 200, 63, 1, 229, 186, 200, 63, 1, 199, 69, 200, 63, 1, 199, 70, - 231, 196, 200, 63, 1, 191, 208, 200, 63, 1, 191, 209, 229, 247, 200, 63, - 1, 222, 77, 200, 63, 1, 220, 36, 200, 63, 1, 216, 48, 200, 63, 1, 212, - 132, 200, 63, 1, 204, 19, 200, 63, 1, 53, 204, 19, 200, 63, 1, 71, 200, - 63, 1, 210, 65, 200, 63, 1, 207, 19, 210, 65, 200, 63, 1, 205, 190, 200, - 63, 1, 208, 85, 200, 63, 1, 206, 182, 200, 63, 1, 203, 70, 200, 63, 1, - 199, 159, 200, 63, 1, 209, 254, 248, 196, 200, 63, 1, 209, 254, 231, 86, - 200, 63, 1, 209, 254, 237, 65, 200, 63, 208, 168, 58, 200, 63, 208, 168, - 60, 200, 63, 208, 168, 235, 139, 200, 63, 191, 0, 58, 200, 63, 191, 0, - 60, 200, 63, 191, 0, 235, 139, 200, 63, 207, 118, 58, 200, 63, 207, 118, - 60, 200, 63, 235, 140, 191, 9, 228, 242, 200, 63, 235, 140, 191, 9, 251, - 110, 200, 63, 229, 252, 58, 200, 63, 229, 252, 60, 200, 63, 229, 251, - 235, 139, 200, 63, 234, 18, 58, 200, 63, 234, 18, 60, 200, 63, 206, 25, - 200, 63, 233, 42, 236, 142, 200, 63, 207, 248, 200, 63, 206, 55, 200, 63, - 105, 81, 185, 58, 200, 63, 105, 81, 185, 60, 200, 63, 115, 185, 58, 200, - 63, 115, 185, 60, 200, 63, 211, 174, 219, 115, 58, 200, 63, 211, 174, - 219, 115, 60, 200, 63, 215, 98, 200, 63, 249, 81, 200, 63, 1, 202, 29, - 191, 69, 200, 63, 1, 202, 29, 221, 203, 200, 63, 1, 202, 29, 233, 61, 9, - 1, 248, 248, 4, 115, 185, 228, 192, 60, 9, 1, 248, 248, 4, 75, 248, 235, - 23, 115, 185, 58, 9, 1, 248, 248, 4, 115, 185, 209, 60, 196, 66, 60, 9, - 1, 248, 248, 4, 115, 185, 209, 60, 196, 66, 248, 235, 23, 105, 185, 58, - 9, 1, 248, 248, 4, 105, 185, 248, 235, 23, 75, 58, 9, 1, 248, 248, 4, - 223, 95, 2, 196, 13, 60, 9, 1, 248, 248, 4, 2, 196, 12, 9, 1, 183, 4, - 105, 185, 58, 9, 1, 183, 4, 115, 185, 209, 60, 196, 66, 60, 9, 1, 238, 6, - 4, 105, 185, 195, 85, 248, 235, 23, 2, 199, 219, 9, 1, 238, 6, 4, 223, - 95, 2, 196, 13, 60, 9, 1, 207, 73, 4, 106, 9, 1, 205, 48, 4, 232, 130, - 185, 58, 9, 1, 251, 233, 4, 105, 185, 58, 9, 1, 251, 233, 4, 115, 185, - 209, 60, 235, 121, 58, 9, 1, 251, 233, 4, 105, 185, 195, 85, 58, 9, 1, - 233, 49, 4, 105, 185, 60, 9, 1, 233, 49, 4, 115, 185, 209, 60, 196, 66, - 60, 9, 1, 222, 7, 4, 75, 58, 9, 1, 222, 7, 4, 115, 185, 58, 9, 1, 222, 7, - 4, 115, 185, 209, 60, 196, 66, 60, 9, 1, 92, 4, 75, 58, 9, 1, 92, 4, 75, - 60, 9, 1, 213, 27, 4, 105, 185, 60, 9, 1, 213, 27, 4, 2, 199, 219, 9, 1, - 213, 27, 4, 2, 196, 12, 9, 1, 220, 38, 4, 164, 9, 1, 207, 73, 4, 105, - 185, 195, 85, 58, 9, 1, 207, 73, 4, 230, 60, 58, 9, 1, 205, 48, 4, 105, - 185, 195, 85, 58, 9, 1, 183, 4, 2, 9, 1, 199, 220, 60, 9, 1, 183, 4, 2, - 9, 1, 199, 220, 23, 105, 236, 140, 9, 1, 205, 48, 4, 2, 9, 1, 199, 220, - 23, 105, 236, 140, 9, 1, 207, 73, 4, 2, 9, 1, 199, 220, 23, 105, 236, - 140, 9, 1, 183, 4, 2, 9, 1, 199, 220, 58, 9, 1, 131, 4, 233, 207, 251, - 30, 17, 105, 58, 9, 1, 131, 4, 233, 207, 251, 30, 17, 115, 58, 9, 1, 233, - 76, 99, 4, 233, 207, 251, 30, 17, 105, 58, 9, 1, 233, 76, 99, 4, 233, - 207, 251, 30, 17, 115, 58, 9, 1, 233, 76, 99, 4, 233, 207, 251, 30, 17, - 232, 130, 60, 9, 1, 193, 134, 4, 233, 207, 251, 30, 17, 105, 58, 9, 1, - 193, 134, 4, 233, 207, 251, 30, 17, 115, 58, 9, 1, 99, 249, 83, 4, 233, - 207, 251, 30, 17, 105, 58, 9, 1, 99, 249, 83, 4, 233, 207, 251, 30, 17, - 115, 58, 9, 1, 183, 4, 233, 207, 251, 30, 17, 232, 130, 60, 9, 1, 205, - 48, 4, 233, 207, 251, 30, 17, 232, 130, 58, 9, 1, 205, 48, 4, 223, 95, - 196, 12, 9, 1, 222, 109, 4, 105, 185, 58, 199, 46, 1, 230, 93, 199, 46, - 1, 203, 120, 199, 46, 1, 213, 25, 199, 46, 1, 207, 180, 199, 46, 1, 249, - 153, 199, 46, 1, 219, 158, 199, 46, 1, 222, 124, 199, 46, 1, 251, 181, - 199, 46, 1, 195, 186, 199, 46, 1, 216, 209, 199, 46, 1, 233, 109, 199, - 46, 1, 237, 68, 199, 46, 1, 199, 48, 199, 46, 1, 220, 124, 199, 46, 1, - 231, 238, 199, 46, 1, 231, 8, 199, 46, 1, 205, 46, 199, 46, 1, 237, 209, - 199, 46, 1, 191, 94, 199, 46, 1, 199, 161, 199, 46, 1, 192, 140, 199, 46, - 1, 210, 79, 199, 46, 1, 223, 17, 199, 46, 1, 243, 132, 199, 46, 1, 197, - 131, 199, 46, 1, 229, 178, 199, 46, 1, 221, 216, 199, 46, 1, 199, 47, - 199, 46, 1, 191, 121, 199, 46, 1, 203, 109, 199, 46, 1, 205, 197, 199, - 46, 1, 238, 32, 199, 46, 1, 159, 199, 46, 1, 191, 7, 199, 46, 1, 251, - 229, 199, 46, 1, 231, 87, 199, 46, 1, 208, 95, 199, 46, 1, 193, 178, 199, - 46, 252, 72, 199, 46, 252, 173, 199, 46, 228, 31, 199, 46, 234, 189, 199, - 46, 196, 161, 199, 46, 211, 88, 199, 46, 234, 200, 199, 46, 233, 197, - 199, 46, 211, 173, 199, 46, 211, 183, 199, 46, 200, 206, 199, 46, 1, 214, - 252, 213, 109, 17, 191, 77, 213, 109, 17, 107, 213, 109, 17, 109, 213, - 109, 17, 138, 213, 109, 17, 134, 213, 109, 17, 150, 213, 109, 17, 169, - 213, 109, 17, 175, 213, 109, 17, 171, 213, 109, 17, 178, 213, 109, 1, 65, - 213, 109, 1, 234, 190, 213, 109, 1, 68, 213, 109, 1, 71, 213, 109, 1, 66, - 213, 109, 1, 211, 89, 213, 109, 1, 74, 213, 109, 1, 238, 20, 213, 109, 1, - 215, 63, 213, 109, 1, 249, 155, 213, 109, 1, 168, 213, 109, 1, 190, 190, - 213, 109, 1, 223, 34, 213, 109, 1, 247, 3, 213, 109, 1, 238, 34, 213, - 109, 1, 165, 213, 109, 1, 206, 110, 213, 109, 1, 188, 213, 109, 1, 231, - 184, 213, 109, 1, 233, 111, 213, 109, 1, 155, 213, 109, 1, 173, 213, 109, - 1, 215, 9, 193, 37, 213, 109, 1, 174, 213, 109, 1, 212, 103, 213, 109, 1, - 181, 213, 109, 1, 140, 213, 109, 1, 193, 190, 213, 109, 1, 170, 213, 109, - 1, 212, 104, 193, 37, 213, 109, 1, 222, 195, 223, 34, 213, 109, 1, 222, - 195, 247, 3, 213, 109, 1, 222, 195, 165, 213, 109, 33, 203, 41, 137, 198, - 79, 213, 109, 33, 203, 41, 130, 198, 79, 213, 109, 33, 203, 41, 206, 181, - 198, 79, 213, 109, 33, 180, 237, 88, 198, 79, 213, 109, 33, 180, 137, - 198, 79, 213, 109, 33, 180, 130, 198, 79, 213, 109, 33, 180, 206, 181, - 198, 79, 213, 109, 33, 214, 215, 77, 213, 109, 33, 55, 75, 58, 213, 109, - 137, 163, 251, 51, 213, 109, 130, 163, 251, 51, 213, 109, 16, 211, 90, - 237, 103, 213, 109, 16, 231, 183, 213, 109, 242, 76, 213, 109, 233, 218, - 77, 213, 109, 220, 96, 213, 109, 237, 235, 213, 109, 236, 144, 56, 213, - 109, 199, 195, 56, 205, 152, 1, 251, 206, 205, 152, 1, 248, 150, 205, - 152, 1, 231, 218, 205, 152, 1, 238, 4, 205, 152, 1, 223, 46, 205, 152, 1, - 249, 153, 205, 152, 1, 191, 80, 205, 152, 1, 223, 55, 205, 152, 1, 198, - 125, 205, 152, 1, 191, 189, 205, 152, 1, 222, 125, 205, 152, 1, 220, 120, - 205, 152, 1, 216, 48, 205, 152, 1, 212, 132, 205, 152, 1, 202, 172, 205, - 152, 1, 223, 164, 205, 152, 1, 233, 25, 205, 152, 1, 197, 166, 205, 152, - 1, 208, 12, 205, 152, 1, 206, 182, 205, 152, 1, 203, 139, 205, 152, 1, - 199, 243, 205, 152, 87, 223, 164, 205, 152, 87, 223, 163, 205, 152, 87, - 211, 167, 205, 152, 87, 238, 18, 205, 152, 52, 1, 234, 54, 191, 189, 205, - 152, 87, 234, 54, 191, 189, 205, 152, 18, 3, 180, 71, 205, 152, 18, 3, - 71, 205, 152, 18, 3, 210, 255, 252, 208, 205, 152, 18, 3, 180, 252, 208, - 205, 152, 18, 3, 252, 208, 205, 152, 18, 3, 210, 255, 65, 205, 152, 18, - 3, 180, 65, 205, 152, 18, 3, 65, 205, 152, 52, 1, 203, 41, 65, 205, 152, - 18, 3, 203, 41, 65, 205, 152, 18, 3, 180, 66, 205, 152, 18, 3, 66, 205, - 152, 52, 1, 68, 205, 152, 18, 3, 180, 68, 205, 152, 18, 3, 68, 205, 152, - 18, 3, 74, 205, 152, 18, 3, 200, 206, 205, 152, 87, 214, 95, 205, 152, - 208, 154, 214, 95, 205, 152, 208, 154, 252, 1, 205, 152, 208, 154, 251, - 123, 205, 152, 208, 154, 249, 59, 205, 152, 208, 154, 250, 176, 205, 152, - 208, 154, 203, 58, 205, 152, 252, 70, 77, 205, 152, 208, 154, 216, 199, - 208, 50, 205, 152, 208, 154, 191, 16, 205, 152, 208, 154, 208, 50, 205, - 152, 208, 154, 191, 119, 205, 152, 208, 154, 197, 54, 205, 152, 208, 154, - 251, 1, 205, 152, 208, 154, 202, 34, 217, 39, 205, 152, 208, 154, 251, - 99, 217, 88, 1, 230, 67, 217, 88, 1, 252, 157, 217, 88, 1, 251, 255, 217, - 88, 1, 252, 44, 217, 88, 1, 251, 247, 217, 88, 1, 196, 36, 217, 88, 1, - 250, 125, 217, 88, 1, 223, 55, 217, 88, 1, 250, 173, 217, 88, 1, 251, - 213, 217, 88, 1, 251, 218, 217, 88, 1, 251, 209, 217, 88, 1, 251, 151, - 217, 88, 1, 251, 134, 217, 88, 1, 250, 221, 217, 88, 1, 223, 164, 217, - 88, 1, 251, 67, 217, 88, 1, 250, 186, 217, 88, 1, 251, 39, 217, 88, 1, - 251, 35, 217, 88, 1, 250, 211, 217, 88, 1, 250, 184, 217, 88, 1, 235, 64, - 217, 88, 1, 222, 116, 217, 88, 1, 251, 232, 217, 88, 252, 5, 77, 217, 88, - 195, 22, 77, 217, 88, 231, 155, 77, 217, 88, 208, 153, 200, 63, 1, 142, - 214, 70, 200, 63, 1, 142, 223, 34, 200, 63, 1, 142, 212, 103, 200, 63, 1, - 142, 197, 132, 200, 63, 1, 142, 213, 81, 200, 63, 1, 142, 213, 63, 200, - 63, 1, 142, 248, 205, 200, 63, 1, 142, 165, 200, 63, 1, 142, 219, 75, - 200, 63, 1, 142, 219, 64, 200, 63, 1, 142, 201, 176, 9, 1, 131, 4, 250, - 172, 233, 72, 9, 1, 131, 4, 250, 172, 198, 54, 50, 233, 72, 9, 1, 131, 4, - 50, 82, 106, 9, 1, 131, 4, 45, 82, 106, 9, 1, 131, 4, 250, 172, 222, 54, - 9, 1, 131, 4, 250, 172, 248, 79, 50, 222, 54, 9, 1, 131, 4, 250, 172, - 206, 116, 75, 58, 9, 1, 131, 4, 250, 172, 50, 206, 116, 236, 142, 9, 1, - 131, 4, 250, 172, 45, 206, 116, 236, 142, 9, 1, 131, 4, 250, 172, 206, - 116, 75, 60, 9, 1, 131, 4, 75, 58, 9, 1, 131, 4, 250, 172, 198, 54, 50, - 233, 73, 23, 75, 58, 9, 1, 131, 4, 50, 82, 201, 29, 23, 75, 58, 9, 1, - 131, 4, 250, 172, 248, 79, 50, 222, 55, 23, 75, 58, 9, 1, 131, 4, 250, - 172, 198, 54, 50, 233, 73, 23, 45, 206, 189, 9, 1, 131, 4, 50, 82, 201, - 29, 23, 45, 206, 189, 9, 1, 131, 4, 250, 172, 248, 79, 50, 222, 55, 23, - 45, 206, 189, 9, 1, 131, 4, 250, 172, 50, 230, 59, 9, 1, 131, 4, 250, - 172, 45, 230, 59, 9, 199, 169, 4, 210, 253, 230, 59, 9, 199, 169, 4, 210, - 253, 193, 133, 9, 199, 169, 4, 105, 185, 60, 9, 1, 199, 4, 192, 75, 9, - 249, 74, 206, 116, 236, 142, 9, 207, 149, 206, 116, 236, 142, 9, 1, 213, - 27, 4, 223, 95, 2, 196, 12, 9, 1, 183, 4, 223, 95, 2, 196, 13, 60, 9, 1, - 199, 220, 4, 75, 60, 9, 1, 199, 220, 4, 115, 185, 60, 9, 1, 222, 7, 4, - 105, 185, 195, 85, 60, 9, 81, 199, 215, 9, 209, 10, 87, 58, 9, 209, 184, - 87, 58, 9, 2, 137, 193, 23, 251, 208, 9, 2, 130, 193, 23, 223, 63, 9, 2, - 130, 193, 23, 223, 181, 9, 2, 130, 193, 23, 199, 173, 9, 217, 144, 193, - 179, 9, 200, 182, 131, 215, 236, 9, 235, 130, 217, 143, 9, 132, 217, 144, - 139, 217, 143, 9, 1, 248, 248, 4, 2, 196, 13, 60, 9, 1, 248, 248, 4, 230, - 60, 58, 9, 1, 223, 9, 4, 105, 185, 58, 9, 1, 199, 220, 4, 105, 185, 58, - 9, 1, 233, 49, 4, 75, 248, 235, 23, 115, 185, 58, 9, 1, 208, 92, 4, 75, - 60, 9, 1, 220, 38, 4, 55, 164, 9, 1, 92, 4, 115, 185, 58, 9, 1, 99, 4, - 105, 185, 248, 235, 23, 230, 60, 58, 9, 1, 99, 4, 105, 185, 248, 235, 23, - 75, 58, 9, 1, 207, 73, 4, 219, 6, 9, 1, 193, 134, 4, 75, 193, 52, 9, 1, - 206, 143, 192, 75, 9, 1, 130, 251, 194, 9, 1, 238, 6, 4, 115, 185, 60, 9, - 1, 205, 194, 4, 115, 185, 60, 9, 1, 231, 230, 4, 223, 95, 106, 9, 1, 201, - 54, 193, 133, 9, 1, 191, 114, 4, 223, 95, 196, 13, 58, 9, 1, 251, 233, 4, - 115, 185, 60, 9, 1, 222, 109, 4, 75, 60, 9, 1, 208, 92, 4, 75, 248, 235, - 23, 213, 46, 185, 58, 9, 1, 248, 248, 4, 2, 92, 58, 9, 1, 210, 44, 4, 2, - 92, 58, 9, 1, 199, 115, 4, 2, 199, 115, 58, 9, 1, 207, 73, 4, 2, 213, 27, - 58, 9, 1, 99, 4, 105, 185, 248, 235, 23, 2, 213, 27, 58, 9, 1, 252, 2, - 233, 48, 9, 1, 252, 2, 208, 91, 9, 1, 252, 2, 213, 26, 9, 1, 210, 44, 4, - 2, 196, 12, 9, 1, 199, 115, 4, 2, 196, 12, 9, 1, 197, 125, 4, 2, 196, 12, - 9, 1, 199, 150, 4, 2, 196, 12, 9, 1, 222, 7, 4, 2, 196, 12, 9, 1, 231, - 89, 4, 115, 185, 58, 9, 1, 252, 2, 208, 92, 4, 115, 185, 58, 9, 1, 223, - 9, 4, 115, 185, 58, 9, 1, 223, 9, 4, 115, 185, 60, 9, 1, 220, 38, 4, 2, - 9, 1, 199, 220, 58, 9, 1, 230, 226, 9, 2, 233, 76, 192, 75, 9, 2, 137, - 233, 76, 192, 75, 9, 2, 137, 99, 249, 83, 4, 105, 185, 60, 9, 2, 137, - 193, 23, 205, 183, 9, 2, 137, 191, 116, 9, 220, 15, 206, 116, 75, 58, 9, - 220, 15, 206, 116, 75, 60, 9, 200, 207, 60, 9, 220, 15, 243, 13, 60, 9, - 220, 15, 206, 116, 75, 223, 120, 243, 13, 60, 9, 2, 130, 193, 133, 9, 2, - 137, 193, 23, 251, 32, 9, 2, 137, 205, 193, 9, 2, 137, 251, 232, 9, 2, - 137, 208, 91, 9, 2, 137, 213, 27, 4, 222, 54, 9, 2, 130, 213, 27, 4, 222, - 54, 9, 2, 137, 193, 23, 250, 183, 9, 2, 137, 193, 23, 250, 220, 9, 2, - 137, 193, 23, 251, 133, 9, 2, 137, 193, 23, 205, 172, 9, 2, 137, 193, 23, - 208, 54, 9, 2, 137, 193, 23, 193, 157, 9, 2, 137, 232, 159, 217, 53, 9, - 2, 137, 3, 205, 188, 9, 236, 220, 234, 97, 79, 250, 132, 9, 154, 237, - 248, 60, 9, 238, 173, 233, 72, 9, 238, 173, 237, 247, 9, 238, 173, 222, - 54, 9, 238, 173, 233, 70, 9, 238, 173, 237, 245, 9, 238, 173, 222, 52, 9, - 163, 91, 75, 58, 9, 163, 105, 185, 58, 9, 163, 219, 7, 58, 9, 163, 91, - 75, 60, 9, 163, 105, 185, 60, 9, 163, 219, 7, 60, 9, 211, 79, 233, 70, 9, - 211, 79, 237, 245, 9, 211, 79, 222, 52, 9, 2, 137, 193, 133, 9, 233, 73, - 4, 206, 189, 9, 233, 73, 4, 75, 58, 9, 222, 55, 4, 75, 60, 9, 45, 250, - 238, 58, 9, 50, 250, 238, 58, 9, 45, 250, 238, 60, 9, 50, 250, 238, 60, - 9, 55, 50, 250, 238, 58, 9, 55, 50, 250, 238, 93, 4, 236, 142, 9, 50, - 250, 238, 93, 4, 236, 142, 9, 237, 248, 4, 236, 142, 9, 87, 202, 207, - 213, 27, 231, 59, 100, 3, 223, 95, 247, 121, 100, 3, 247, 121, 100, 3, - 251, 73, 100, 3, 195, 35, 100, 1, 203, 41, 65, 100, 1, 65, 100, 1, 252, - 208, 100, 1, 68, 100, 1, 223, 201, 100, 1, 66, 100, 1, 196, 30, 100, 1, - 117, 146, 100, 1, 117, 172, 100, 1, 247, 124, 71, 100, 1, 203, 41, 71, - 100, 1, 71, 100, 1, 251, 238, 100, 1, 247, 124, 74, 100, 1, 203, 41, 74, - 100, 1, 74, 100, 1, 250, 165, 100, 1, 155, 100, 1, 221, 217, 100, 1, 231, - 242, 100, 1, 231, 93, 100, 1, 214, 70, 100, 1, 247, 162, 100, 1, 247, 3, - 100, 1, 223, 34, 100, 1, 222, 254, 100, 1, 212, 103, 100, 1, 197, 132, - 100, 1, 197, 120, 100, 1, 237, 193, 100, 1, 237, 177, 100, 1, 213, 81, - 100, 1, 190, 190, 100, 1, 199, 49, 100, 1, 238, 34, 100, 1, 237, 70, 100, - 1, 181, 100, 1, 213, 63, 100, 1, 168, 100, 1, 209, 230, 100, 1, 249, 155, - 100, 1, 248, 205, 100, 1, 174, 100, 1, 170, 100, 1, 165, 100, 1, 206, - 110, 100, 1, 173, 100, 1, 219, 75, 100, 1, 219, 64, 100, 1, 195, 188, - 100, 1, 203, 166, 100, 1, 201, 176, 100, 1, 188, 100, 1, 140, 100, 18, 3, - 211, 153, 100, 18, 3, 211, 87, 100, 3, 212, 143, 100, 3, 250, 147, 100, - 18, 3, 252, 208, 100, 18, 3, 68, 100, 18, 3, 223, 201, 100, 18, 3, 66, - 100, 18, 3, 196, 30, 100, 18, 3, 117, 146, 100, 18, 3, 117, 206, 111, - 100, 18, 3, 247, 124, 71, 100, 18, 3, 203, 41, 71, 100, 18, 3, 71, 100, - 18, 3, 251, 238, 100, 18, 3, 247, 124, 74, 100, 18, 3, 203, 41, 74, 100, - 18, 3, 74, 100, 18, 3, 250, 165, 100, 3, 195, 40, 100, 18, 3, 208, 209, - 71, 100, 18, 3, 250, 142, 100, 211, 115, 100, 201, 39, 3, 196, 154, 100, - 201, 39, 3, 251, 75, 100, 230, 213, 252, 62, 100, 252, 49, 252, 62, 100, - 18, 3, 247, 124, 180, 71, 100, 18, 3, 196, 152, 100, 18, 3, 196, 29, 100, - 1, 208, 98, 100, 1, 221, 195, 100, 1, 231, 68, 100, 1, 191, 123, 100, 1, - 237, 182, 100, 1, 207, 7, 100, 1, 233, 111, 100, 1, 191, 175, 100, 1, - 117, 206, 111, 100, 1, 117, 219, 76, 100, 18, 3, 117, 172, 100, 18, 3, - 117, 219, 76, 100, 237, 240, 100, 55, 237, 240, 100, 17, 191, 77, 100, - 17, 107, 100, 17, 109, 100, 17, 138, 100, 17, 134, 100, 17, 150, 100, 17, - 169, 100, 17, 175, 100, 17, 171, 100, 17, 178, 100, 252, 70, 56, 100, 3, - 137, 201, 247, 236, 142, 100, 1, 247, 124, 65, 100, 1, 211, 153, 100, 1, - 211, 87, 100, 1, 250, 142, 100, 1, 196, 152, 100, 1, 196, 29, 100, 1, - 217, 45, 237, 193, 100, 1, 191, 71, 100, 1, 88, 170, 100, 1, 231, 129, - 100, 1, 222, 232, 100, 1, 231, 13, 201, 64, 100, 1, 237, 183, 100, 1, - 249, 55, 248, 227, 251, 102, 248, 227, 3, 247, 121, 248, 227, 3, 251, 73, - 248, 227, 3, 195, 35, 248, 227, 1, 65, 248, 227, 1, 252, 208, 248, 227, - 1, 68, 248, 227, 1, 223, 201, 248, 227, 1, 66, 248, 227, 1, 196, 30, 248, - 227, 1, 117, 146, 248, 227, 1, 117, 172, 248, 227, 1, 71, 248, 227, 1, - 251, 238, 248, 227, 1, 74, 248, 227, 1, 250, 165, 248, 227, 1, 155, 248, - 227, 1, 221, 217, 248, 227, 1, 231, 242, 248, 227, 1, 231, 93, 248, 227, - 1, 214, 70, 248, 227, 1, 247, 162, 248, 227, 1, 247, 3, 248, 227, 1, 223, - 34, 248, 227, 1, 222, 254, 248, 227, 1, 212, 103, 248, 227, 1, 197, 132, - 248, 227, 1, 197, 120, 248, 227, 1, 237, 193, 248, 227, 1, 237, 177, 248, - 227, 1, 213, 81, 248, 227, 1, 190, 190, 248, 227, 1, 199, 49, 248, 227, - 1, 238, 34, 248, 227, 1, 237, 70, 248, 227, 1, 181, 248, 227, 1, 168, - 248, 227, 1, 209, 230, 248, 227, 1, 249, 155, 248, 227, 1, 248, 205, 248, - 227, 1, 174, 248, 227, 1, 170, 248, 227, 1, 165, 248, 227, 1, 173, 248, - 227, 1, 203, 166, 248, 227, 1, 201, 176, 248, 227, 1, 188, 248, 227, 1, - 140, 248, 227, 3, 212, 143, 248, 227, 3, 250, 147, 248, 227, 18, 3, 252, - 208, 248, 227, 18, 3, 68, 248, 227, 18, 3, 223, 201, 248, 227, 18, 3, 66, - 248, 227, 18, 3, 196, 30, 248, 227, 18, 3, 117, 146, 248, 227, 18, 3, - 117, 206, 111, 248, 227, 18, 3, 71, 248, 227, 18, 3, 251, 238, 248, 227, - 18, 3, 74, 248, 227, 18, 3, 250, 165, 248, 227, 3, 195, 40, 248, 227, 1, - 221, 206, 190, 190, 248, 227, 250, 166, 219, 200, 77, 248, 227, 1, 206, - 110, 248, 227, 1, 207, 7, 248, 227, 1, 191, 175, 248, 227, 1, 117, 206, - 111, 248, 227, 1, 117, 219, 76, 248, 227, 18, 3, 117, 172, 248, 227, 18, - 3, 117, 219, 76, 248, 227, 17, 191, 77, 248, 227, 17, 107, 248, 227, 17, - 109, 248, 227, 17, 138, 248, 227, 17, 134, 248, 227, 17, 150, 248, 227, - 17, 169, 248, 227, 17, 175, 248, 227, 17, 171, 248, 227, 17, 178, 248, - 227, 1, 207, 188, 4, 82, 237, 40, 248, 227, 1, 207, 188, 4, 110, 237, 40, - 248, 227, 206, 37, 77, 248, 227, 206, 37, 56, 248, 227, 238, 172, 212, - 135, 107, 248, 227, 238, 172, 212, 135, 109, 248, 227, 238, 172, 212, - 135, 138, 248, 227, 238, 172, 212, 135, 134, 248, 227, 238, 172, 212, - 135, 91, 219, 183, 199, 39, 199, 34, 237, 101, 248, 227, 238, 172, 237, - 102, 202, 131, 248, 227, 223, 56, 248, 227, 231, 209, 77, 248, 227, 1, - 195, 150, 251, 73, 248, 227, 252, 70, 56, 248, 227, 205, 139, 77, 230, - 146, 3, 252, 43, 248, 169, 230, 146, 3, 248, 169, 230, 146, 3, 195, 35, - 230, 146, 1, 65, 230, 146, 1, 252, 208, 230, 146, 1, 68, 230, 146, 1, - 223, 201, 230, 146, 1, 66, 230, 146, 1, 196, 30, 230, 146, 1, 234, 190, - 230, 146, 1, 251, 238, 230, 146, 1, 211, 89, 230, 146, 1, 250, 165, 230, - 146, 1, 155, 230, 146, 1, 221, 217, 230, 146, 1, 231, 242, 230, 146, 1, - 231, 93, 230, 146, 1, 214, 70, 230, 146, 1, 247, 162, 230, 146, 1, 247, - 3, 230, 146, 1, 223, 34, 230, 146, 1, 222, 254, 230, 146, 1, 212, 103, - 230, 146, 1, 197, 132, 230, 146, 1, 197, 120, 230, 146, 1, 237, 193, 230, - 146, 1, 237, 177, 230, 146, 1, 213, 81, 230, 146, 1, 190, 190, 230, 146, - 1, 199, 49, 230, 146, 1, 238, 34, 230, 146, 1, 237, 70, 230, 146, 1, 181, - 230, 146, 1, 168, 230, 146, 1, 209, 230, 230, 146, 1, 249, 155, 230, 146, - 1, 248, 205, 230, 146, 1, 174, 230, 146, 1, 170, 230, 146, 1, 165, 230, - 146, 1, 173, 230, 146, 1, 219, 75, 230, 146, 1, 195, 188, 230, 146, 1, - 203, 166, 230, 146, 1, 188, 230, 146, 1, 140, 230, 146, 3, 212, 143, 230, - 146, 18, 3, 252, 208, 230, 146, 18, 3, 68, 230, 146, 18, 3, 223, 201, - 230, 146, 18, 3, 66, 230, 146, 18, 3, 196, 30, 230, 146, 18, 3, 234, 190, - 230, 146, 18, 3, 251, 238, 230, 146, 18, 3, 211, 89, 230, 146, 18, 3, - 250, 165, 230, 146, 3, 195, 40, 230, 146, 3, 196, 157, 230, 146, 1, 221, - 195, 230, 146, 1, 231, 68, 230, 146, 1, 191, 123, 230, 146, 1, 206, 110, - 230, 146, 1, 233, 111, 230, 146, 17, 191, 77, 230, 146, 17, 107, 230, - 146, 17, 109, 230, 146, 17, 138, 230, 146, 17, 134, 230, 146, 17, 150, - 230, 146, 17, 169, 230, 146, 17, 175, 230, 146, 17, 171, 230, 146, 17, - 178, 230, 146, 198, 133, 230, 146, 252, 42, 230, 146, 223, 77, 230, 146, - 196, 58, 230, 146, 234, 150, 211, 94, 230, 146, 3, 192, 115, 230, 146, - 252, 70, 56, 230, 163, 3, 247, 121, 230, 163, 3, 251, 73, 230, 163, 3, - 195, 35, 230, 163, 1, 65, 230, 163, 1, 252, 208, 230, 163, 1, 68, 230, - 163, 1, 223, 201, 230, 163, 1, 66, 230, 163, 1, 196, 30, 230, 163, 1, - 117, 146, 230, 163, 1, 117, 172, 230, 163, 18, 247, 124, 71, 230, 163, 1, - 71, 230, 163, 1, 251, 238, 230, 163, 18, 247, 124, 74, 230, 163, 1, 74, - 230, 163, 1, 250, 165, 230, 163, 1, 155, 230, 163, 1, 221, 217, 230, 163, - 1, 231, 242, 230, 163, 1, 231, 93, 230, 163, 1, 214, 70, 230, 163, 1, - 247, 162, 230, 163, 1, 247, 3, 230, 163, 1, 223, 34, 230, 163, 1, 222, - 254, 230, 163, 1, 212, 103, 230, 163, 1, 197, 132, 230, 163, 1, 197, 120, - 230, 163, 1, 237, 193, 230, 163, 1, 237, 177, 230, 163, 1, 213, 81, 230, - 163, 1, 190, 190, 230, 163, 1, 199, 49, 230, 163, 1, 238, 34, 230, 163, - 1, 237, 70, 230, 163, 1, 181, 230, 163, 1, 168, 230, 163, 1, 209, 230, - 230, 163, 1, 249, 155, 230, 163, 1, 248, 205, 230, 163, 1, 174, 230, 163, - 1, 170, 230, 163, 1, 165, 230, 163, 1, 173, 230, 163, 1, 219, 75, 230, - 163, 1, 195, 188, 230, 163, 1, 203, 166, 230, 163, 1, 201, 176, 230, 163, - 1, 188, 230, 163, 1, 140, 230, 163, 3, 212, 143, 230, 163, 3, 250, 147, - 230, 163, 18, 3, 252, 208, 230, 163, 18, 3, 68, 230, 163, 18, 3, 223, - 201, 230, 163, 18, 3, 66, 230, 163, 18, 3, 196, 30, 230, 163, 18, 3, 117, - 146, 230, 163, 18, 3, 117, 206, 111, 230, 163, 18, 3, 247, 124, 71, 230, - 163, 18, 3, 71, 230, 163, 18, 3, 251, 238, 230, 163, 18, 3, 247, 124, 74, - 230, 163, 18, 3, 74, 230, 163, 18, 3, 250, 165, 230, 163, 3, 195, 40, - 230, 163, 211, 115, 230, 163, 1, 117, 206, 111, 230, 163, 1, 117, 219, - 76, 230, 163, 18, 3, 117, 172, 230, 163, 18, 3, 117, 219, 76, 230, 163, - 17, 191, 77, 230, 163, 17, 107, 230, 163, 17, 109, 230, 163, 17, 138, - 230, 163, 17, 134, 230, 163, 17, 150, 230, 163, 17, 169, 230, 163, 17, - 175, 230, 163, 17, 171, 230, 163, 17, 178, 230, 163, 252, 70, 56, 230, - 163, 206, 37, 56, 230, 163, 1, 191, 71, 230, 163, 3, 200, 206, 230, 163, - 3, 203, 156, 230, 163, 3, 217, 141, 230, 163, 3, 198, 224, 212, 144, 58, - 230, 163, 3, 243, 13, 212, 144, 58, 230, 163, 3, 197, 15, 212, 144, 58, - 211, 47, 3, 247, 121, 211, 47, 3, 251, 73, 211, 47, 3, 195, 35, 211, 47, - 1, 65, 211, 47, 1, 252, 208, 211, 47, 1, 68, 211, 47, 1, 223, 201, 211, - 47, 1, 66, 211, 47, 1, 196, 30, 211, 47, 1, 117, 146, 211, 47, 1, 117, - 172, 211, 47, 1, 71, 211, 47, 1, 251, 238, 211, 47, 1, 74, 211, 47, 1, - 250, 165, 211, 47, 1, 155, 211, 47, 1, 221, 217, 211, 47, 1, 231, 242, - 211, 47, 1, 231, 93, 211, 47, 1, 214, 70, 211, 47, 1, 247, 162, 211, 47, - 1, 247, 3, 211, 47, 1, 223, 34, 211, 47, 1, 222, 254, 211, 47, 1, 212, - 103, 211, 47, 1, 197, 132, 211, 47, 1, 197, 120, 211, 47, 1, 237, 193, - 211, 47, 1, 237, 177, 211, 47, 1, 213, 81, 211, 47, 1, 190, 190, 211, 47, - 1, 199, 49, 211, 47, 1, 238, 34, 211, 47, 1, 237, 70, 211, 47, 1, 181, - 211, 47, 1, 168, 211, 47, 1, 209, 230, 211, 47, 1, 249, 155, 211, 47, 1, - 248, 205, 211, 47, 1, 174, 211, 47, 1, 170, 211, 47, 1, 165, 211, 47, 1, - 173, 211, 47, 1, 219, 75, 211, 47, 1, 195, 188, 211, 47, 1, 203, 166, - 211, 47, 1, 201, 176, 211, 47, 1, 188, 211, 47, 1, 140, 211, 47, 3, 212, - 143, 211, 47, 3, 250, 147, 211, 47, 18, 3, 252, 208, 211, 47, 18, 3, 68, - 211, 47, 18, 3, 223, 201, 211, 47, 18, 3, 66, 211, 47, 18, 3, 196, 30, - 211, 47, 18, 3, 117, 146, 211, 47, 18, 3, 117, 206, 111, 211, 47, 18, 3, - 71, 211, 47, 18, 3, 251, 238, 211, 47, 18, 3, 74, 211, 47, 18, 3, 250, - 165, 211, 47, 3, 195, 40, 211, 47, 3, 211, 0, 211, 47, 251, 239, 219, - 200, 77, 211, 47, 250, 166, 219, 200, 77, 211, 47, 1, 206, 110, 211, 47, - 1, 207, 7, 211, 47, 1, 191, 175, 211, 47, 1, 117, 206, 111, 211, 47, 1, - 117, 219, 76, 211, 47, 18, 3, 117, 172, 211, 47, 18, 3, 117, 219, 76, - 211, 47, 17, 191, 77, 211, 47, 17, 107, 211, 47, 17, 109, 211, 47, 17, - 138, 211, 47, 17, 134, 211, 47, 17, 150, 211, 47, 17, 169, 211, 47, 17, - 175, 211, 47, 17, 171, 211, 47, 17, 178, 211, 47, 223, 56, 211, 47, 1, - 193, 190, 211, 47, 232, 120, 91, 208, 24, 211, 47, 232, 120, 91, 230, 72, - 211, 47, 232, 120, 115, 208, 22, 211, 47, 232, 120, 91, 202, 129, 211, - 47, 232, 120, 91, 234, 161, 211, 47, 232, 120, 115, 202, 126, 44, 3, 251, - 73, 44, 3, 195, 35, 44, 1, 65, 44, 1, 252, 208, 44, 1, 68, 44, 1, 223, - 201, 44, 1, 66, 44, 1, 196, 30, 44, 1, 71, 44, 1, 234, 190, 44, 1, 251, - 238, 44, 1, 74, 44, 1, 211, 89, 44, 1, 250, 165, 44, 1, 155, 44, 1, 214, - 70, 44, 1, 247, 162, 44, 1, 223, 34, 44, 1, 212, 103, 44, 1, 197, 132, - 44, 1, 213, 81, 44, 1, 190, 190, 44, 1, 181, 44, 1, 213, 63, 44, 1, 168, - 44, 1, 174, 44, 1, 170, 44, 1, 165, 44, 1, 206, 110, 44, 1, 173, 44, 1, - 219, 75, 44, 1, 219, 64, 44, 1, 195, 188, 44, 1, 203, 166, 44, 1, 201, - 176, 44, 1, 188, 44, 1, 140, 44, 18, 3, 252, 208, 44, 18, 3, 68, 44, 18, - 3, 223, 201, 44, 18, 3, 66, 44, 18, 3, 196, 30, 44, 18, 3, 71, 44, 18, 3, - 234, 190, 44, 18, 3, 251, 238, 44, 18, 3, 74, 44, 18, 3, 211, 89, 44, 18, - 3, 250, 165, 44, 3, 195, 40, 44, 211, 115, 44, 250, 166, 219, 200, 77, - 44, 17, 191, 77, 44, 17, 107, 44, 17, 109, 44, 17, 138, 44, 17, 134, 44, - 17, 150, 44, 17, 169, 44, 17, 175, 44, 17, 171, 44, 17, 178, 44, 31, 199, - 95, 44, 31, 91, 228, 142, 44, 31, 91, 189, 44, 237, 206, 56, 44, 215, - 216, 56, 44, 192, 78, 56, 44, 237, 144, 56, 44, 238, 232, 56, 44, 250, - 222, 93, 56, 44, 206, 37, 56, 44, 31, 56, 199, 99, 3, 33, 247, 122, 58, - 199, 99, 3, 247, 121, 199, 99, 3, 251, 73, 199, 99, 3, 195, 35, 199, 99, - 3, 33, 251, 74, 58, 199, 99, 1, 65, 199, 99, 1, 252, 208, 199, 99, 1, 68, - 199, 99, 1, 223, 201, 199, 99, 1, 66, 199, 99, 1, 196, 30, 199, 99, 1, - 117, 146, 199, 99, 1, 117, 172, 199, 99, 1, 71, 199, 99, 1, 234, 190, - 199, 99, 1, 251, 238, 199, 99, 1, 74, 199, 99, 1, 211, 89, 199, 99, 1, - 250, 165, 199, 99, 1, 155, 199, 99, 1, 221, 217, 199, 99, 1, 231, 242, - 199, 99, 1, 231, 93, 199, 99, 1, 214, 70, 199, 99, 1, 247, 162, 199, 99, - 1, 247, 3, 199, 99, 1, 223, 34, 199, 99, 1, 222, 254, 199, 99, 1, 212, - 103, 199, 99, 1, 197, 132, 199, 99, 1, 197, 120, 199, 99, 1, 237, 193, - 199, 99, 1, 237, 177, 199, 99, 1, 213, 81, 199, 99, 1, 190, 190, 199, 99, - 1, 199, 49, 199, 99, 1, 238, 34, 199, 99, 1, 237, 70, 199, 99, 1, 181, - 199, 99, 1, 168, 199, 99, 1, 209, 230, 199, 99, 1, 249, 155, 199, 99, 1, - 248, 205, 199, 99, 1, 174, 199, 99, 1, 170, 199, 99, 1, 165, 199, 99, 1, - 206, 110, 199, 99, 1, 173, 199, 99, 1, 219, 75, 199, 99, 1, 219, 64, 199, - 99, 1, 195, 188, 199, 99, 1, 203, 166, 199, 99, 1, 201, 176, 199, 99, 1, - 188, 199, 99, 1, 140, 199, 99, 3, 212, 143, 199, 99, 3, 250, 147, 199, - 99, 18, 3, 252, 208, 199, 99, 18, 3, 68, 199, 99, 18, 3, 223, 201, 199, - 99, 18, 3, 66, 199, 99, 18, 3, 196, 30, 199, 99, 18, 3, 117, 146, 199, - 99, 18, 3, 117, 206, 111, 199, 99, 18, 3, 71, 199, 99, 18, 3, 234, 190, - 199, 99, 18, 3, 251, 238, 199, 99, 18, 3, 74, 199, 99, 18, 3, 211, 89, - 199, 99, 18, 3, 250, 165, 199, 99, 3, 195, 40, 199, 99, 219, 200, 77, - 199, 99, 251, 239, 219, 200, 77, 199, 99, 1, 197, 168, 199, 99, 1, 235, - 37, 199, 99, 1, 206, 91, 199, 99, 1, 214, 234, 209, 48, 199, 99, 1, 117, - 206, 111, 199, 99, 1, 117, 219, 76, 199, 99, 18, 3, 117, 172, 199, 99, - 18, 3, 117, 219, 76, 199, 99, 17, 191, 77, 199, 99, 17, 107, 199, 99, 17, - 109, 199, 99, 17, 138, 199, 99, 17, 134, 199, 99, 17, 150, 199, 99, 17, - 169, 199, 99, 17, 175, 199, 99, 17, 171, 199, 99, 17, 178, 199, 99, 3, - 202, 211, 199, 99, 232, 120, 17, 191, 78, 40, 211, 157, 208, 255, 79, - 134, 199, 99, 232, 120, 17, 91, 40, 211, 157, 208, 255, 79, 134, 199, 99, - 232, 120, 17, 105, 40, 211, 157, 208, 255, 79, 134, 199, 99, 232, 120, - 17, 115, 40, 211, 157, 208, 255, 79, 134, 199, 99, 232, 120, 17, 91, 40, - 233, 231, 208, 255, 79, 134, 199, 99, 232, 120, 17, 105, 40, 233, 231, - 208, 255, 79, 134, 199, 99, 232, 120, 17, 115, 40, 233, 231, 208, 255, - 79, 134, 199, 99, 3, 197, 48, 222, 83, 3, 201, 247, 247, 121, 222, 83, 3, - 247, 121, 222, 83, 3, 251, 73, 222, 83, 3, 195, 35, 222, 83, 3, 202, 211, - 222, 83, 1, 65, 222, 83, 1, 252, 208, 222, 83, 1, 68, 222, 83, 1, 223, - 201, 222, 83, 1, 66, 222, 83, 1, 196, 30, 222, 83, 1, 117, 146, 222, 83, - 1, 117, 172, 222, 83, 1, 71, 222, 83, 1, 234, 190, 222, 83, 1, 251, 238, - 222, 83, 1, 74, 222, 83, 1, 211, 89, 222, 83, 1, 250, 165, 222, 83, 1, - 155, 222, 83, 1, 221, 217, 222, 83, 1, 231, 242, 222, 83, 1, 231, 93, - 222, 83, 1, 214, 70, 222, 83, 1, 247, 162, 222, 83, 1, 247, 3, 222, 83, - 1, 223, 34, 222, 83, 1, 222, 254, 222, 83, 1, 212, 103, 222, 83, 1, 197, - 132, 222, 83, 1, 197, 120, 222, 83, 1, 237, 193, 222, 83, 1, 237, 177, - 222, 83, 1, 213, 81, 222, 83, 1, 190, 190, 222, 83, 1, 199, 49, 222, 83, - 1, 238, 34, 222, 83, 1, 237, 70, 222, 83, 1, 181, 222, 83, 1, 168, 222, - 83, 1, 209, 230, 222, 83, 1, 249, 155, 222, 83, 1, 248, 205, 222, 83, 1, - 174, 222, 83, 1, 170, 222, 83, 1, 165, 222, 83, 1, 206, 110, 222, 83, 1, - 173, 222, 83, 1, 219, 75, 222, 83, 1, 195, 188, 222, 83, 1, 203, 166, - 222, 83, 1, 201, 176, 222, 83, 1, 188, 222, 83, 1, 140, 222, 83, 3, 212, - 143, 222, 83, 3, 250, 147, 222, 83, 18, 3, 252, 208, 222, 83, 18, 3, 68, - 222, 83, 18, 3, 223, 201, 222, 83, 18, 3, 66, 222, 83, 18, 3, 196, 30, - 222, 83, 18, 3, 117, 146, 222, 83, 18, 3, 117, 206, 111, 222, 83, 18, 3, - 71, 222, 83, 18, 3, 234, 190, 222, 83, 18, 3, 251, 238, 222, 83, 18, 3, - 74, 222, 83, 18, 3, 211, 89, 222, 83, 18, 3, 250, 165, 222, 83, 3, 195, - 40, 222, 83, 219, 200, 77, 222, 83, 251, 239, 219, 200, 77, 222, 83, 1, - 214, 234, 209, 48, 222, 83, 1, 233, 111, 222, 83, 1, 117, 206, 111, 222, - 83, 1, 117, 219, 76, 222, 83, 18, 3, 117, 172, 222, 83, 18, 3, 117, 219, - 76, 222, 83, 17, 191, 77, 222, 83, 17, 107, 222, 83, 17, 109, 222, 83, - 17, 138, 222, 83, 17, 134, 222, 83, 17, 150, 222, 83, 17, 169, 222, 83, - 17, 175, 222, 83, 17, 171, 222, 83, 17, 178, 222, 83, 3, 222, 239, 222, - 83, 3, 196, 75, 222, 83, 3, 33, 251, 74, 93, 179, 142, 3, 33, 251, 74, - 58, 142, 3, 247, 121, 142, 3, 251, 73, 142, 3, 195, 35, 142, 1, 195, 150, - 251, 73, 142, 1, 65, 142, 1, 252, 208, 142, 1, 68, 142, 1, 223, 201, 142, - 1, 66, 142, 1, 196, 30, 142, 1, 117, 146, 142, 1, 117, 172, 142, 1, 71, - 142, 1, 234, 190, 142, 1, 251, 238, 142, 1, 74, 142, 1, 211, 89, 142, 1, - 250, 165, 142, 1, 155, 142, 1, 221, 217, 142, 1, 231, 242, 142, 1, 231, - 93, 142, 1, 214, 70, 142, 1, 247, 162, 142, 1, 247, 3, 142, 1, 223, 34, - 142, 1, 222, 254, 142, 1, 212, 103, 142, 1, 197, 132, 142, 1, 197, 120, - 142, 1, 237, 193, 142, 1, 237, 177, 142, 1, 213, 81, 142, 1, 190, 190, - 142, 1, 199, 49, 142, 1, 238, 34, 142, 1, 237, 70, 142, 1, 181, 142, 1, - 213, 63, 142, 1, 168, 142, 1, 209, 230, 142, 1, 249, 155, 142, 1, 248, - 205, 142, 1, 174, 142, 1, 170, 142, 1, 165, 142, 1, 206, 110, 142, 1, - 173, 142, 1, 219, 75, 142, 1, 219, 64, 142, 1, 195, 188, 142, 1, 203, - 166, 142, 1, 201, 176, 142, 1, 188, 142, 1, 140, 142, 1, 197, 101, 142, - 3, 81, 249, 90, 195, 40, 142, 3, 243, 6, 195, 40, 142, 3, 250, 147, 142, - 18, 3, 252, 208, 142, 18, 3, 68, 142, 18, 3, 223, 201, 142, 18, 3, 66, - 142, 18, 3, 196, 30, 142, 18, 3, 117, 146, 142, 18, 3, 117, 206, 111, - 142, 18, 3, 71, 142, 18, 3, 234, 190, 142, 18, 3, 251, 238, 142, 18, 3, - 74, 142, 18, 3, 211, 89, 142, 18, 3, 250, 165, 142, 3, 195, 40, 142, 1, - 75, 207, 46, 142, 3, 210, 132, 142, 1, 243, 86, 218, 170, 142, 1, 243, - 86, 192, 159, 142, 1, 243, 86, 219, 65, 142, 250, 166, 219, 200, 77, 142, - 232, 120, 91, 211, 102, 142, 232, 120, 91, 232, 141, 142, 232, 120, 115, - 234, 157, 142, 232, 120, 91, 197, 35, 142, 232, 120, 91, 199, 86, 142, - 232, 120, 115, 197, 34, 142, 232, 120, 91, 233, 20, 142, 1, 251, 16, 223, - 201, 142, 1, 117, 206, 111, 142, 1, 117, 219, 76, 142, 18, 3, 117, 172, - 142, 18, 3, 117, 219, 76, 142, 17, 191, 77, 142, 17, 107, 142, 17, 109, - 142, 17, 138, 142, 17, 134, 142, 17, 150, 142, 17, 169, 142, 17, 175, - 142, 17, 171, 142, 17, 178, 142, 31, 199, 95, 142, 31, 91, 228, 142, 142, - 31, 91, 189, 142, 232, 120, 91, 208, 24, 142, 232, 120, 91, 230, 72, 142, - 232, 120, 115, 208, 22, 142, 232, 120, 91, 202, 129, 142, 232, 120, 91, - 234, 161, 142, 232, 120, 115, 202, 126, 142, 237, 211, 77, 142, 1, 243, - 86, 213, 82, 142, 1, 243, 86, 215, 63, 142, 1, 243, 86, 206, 111, 142, 1, - 243, 86, 172, 142, 1, 243, 86, 219, 76, 142, 1, 243, 86, 222, 154, 166, - 3, 247, 121, 166, 3, 251, 72, 166, 3, 195, 34, 166, 1, 250, 131, 166, 1, - 252, 161, 166, 1, 252, 7, 166, 1, 252, 22, 166, 1, 223, 45, 166, 1, 223, - 200, 166, 1, 196, 20, 166, 1, 196, 24, 166, 1, 223, 72, 166, 1, 223, 73, - 166, 1, 223, 184, 166, 1, 223, 186, 166, 1, 233, 198, 166, 1, 234, 185, - 166, 1, 251, 221, 166, 1, 210, 243, 166, 1, 211, 82, 166, 1, 250, 150, - 166, 1, 251, 165, 222, 29, 166, 1, 217, 121, 222, 29, 166, 1, 251, 165, - 231, 187, 166, 1, 217, 121, 231, 187, 166, 1, 222, 82, 214, 249, 166, 1, - 205, 133, 231, 187, 166, 1, 251, 165, 247, 70, 166, 1, 217, 121, 247, 70, - 166, 1, 251, 165, 223, 15, 166, 1, 217, 121, 223, 15, 166, 1, 199, 241, - 214, 249, 166, 1, 199, 241, 205, 132, 214, 250, 166, 1, 205, 133, 223, - 15, 166, 1, 251, 165, 197, 128, 166, 1, 217, 121, 197, 128, 166, 1, 251, - 165, 237, 184, 166, 1, 217, 121, 237, 184, 166, 1, 215, 94, 214, 199, - 166, 1, 205, 133, 237, 184, 166, 1, 251, 165, 199, 153, 166, 1, 217, 121, - 199, 153, 166, 1, 251, 165, 237, 204, 166, 1, 217, 121, 237, 204, 166, 1, - 237, 236, 214, 199, 166, 1, 205, 133, 237, 204, 166, 1, 251, 165, 210, - 73, 166, 1, 217, 121, 210, 73, 166, 1, 251, 165, 249, 57, 166, 1, 217, - 121, 249, 57, 166, 1, 217, 21, 166, 1, 251, 145, 249, 57, 166, 1, 192, - 85, 166, 1, 207, 123, 166, 1, 237, 236, 219, 249, 166, 1, 195, 156, 166, - 1, 199, 241, 205, 103, 166, 1, 215, 94, 205, 103, 166, 1, 237, 236, 205, - 103, 166, 1, 229, 253, 166, 1, 215, 94, 219, 249, 166, 1, 233, 63, 166, - 3, 251, 207, 166, 18, 3, 252, 17, 166, 18, 3, 221, 242, 252, 24, 166, 18, - 3, 237, 13, 252, 24, 166, 18, 3, 221, 242, 223, 69, 166, 18, 3, 237, 13, - 223, 69, 166, 18, 3, 221, 242, 210, 221, 166, 18, 3, 237, 13, 210, 221, - 166, 18, 3, 231, 231, 166, 18, 3, 221, 48, 166, 18, 3, 237, 13, 221, 48, - 166, 18, 3, 221, 50, 237, 122, 166, 18, 3, 221, 49, 230, 94, 252, 17, - 166, 18, 3, 221, 49, 230, 94, 237, 13, 252, 17, 166, 18, 3, 221, 49, 230, - 94, 231, 186, 166, 18, 3, 231, 186, 166, 219, 88, 17, 191, 77, 166, 219, - 88, 17, 107, 166, 219, 88, 17, 109, 166, 219, 88, 17, 138, 166, 219, 88, - 17, 134, 166, 219, 88, 17, 150, 166, 219, 88, 17, 169, 166, 219, 88, 17, - 175, 166, 219, 88, 17, 171, 166, 219, 88, 17, 178, 166, 18, 3, 237, 13, - 231, 231, 166, 18, 3, 237, 13, 231, 186, 166, 208, 154, 220, 211, 199, - 44, 246, 242, 221, 70, 222, 104, 199, 44, 246, 242, 221, 186, 221, 211, - 199, 44, 246, 242, 221, 186, 221, 176, 199, 44, 246, 242, 221, 186, 221, - 171, 199, 44, 246, 242, 221, 186, 221, 181, 199, 44, 246, 242, 221, 186, - 207, 145, 199, 44, 246, 242, 213, 252, 213, 239, 199, 44, 246, 242, 243, - 71, 246, 248, 199, 44, 246, 242, 243, 71, 243, 81, 199, 44, 246, 242, - 243, 71, 246, 247, 199, 44, 246, 242, 202, 48, 202, 47, 199, 44, 246, - 242, 243, 71, 243, 67, 199, 44, 246, 242, 192, 13, 192, 20, 199, 44, 246, - 242, 236, 177, 247, 0, 199, 44, 246, 242, 119, 210, 89, 199, 44, 246, - 242, 198, 242, 199, 38, 199, 44, 246, 242, 198, 242, 214, 224, 199, 44, - 246, 242, 198, 242, 209, 190, 199, 44, 246, 242, 213, 46, 214, 104, 199, - 44, 246, 242, 236, 177, 237, 123, 199, 44, 246, 242, 119, 199, 184, 199, - 44, 246, 242, 198, 242, 198, 207, 199, 44, 246, 242, 198, 242, 199, 45, - 199, 44, 246, 242, 198, 242, 198, 236, 199, 44, 246, 242, 213, 46, 212, - 180, 199, 44, 246, 242, 248, 114, 249, 120, 199, 44, 246, 242, 209, 76, - 209, 112, 199, 44, 246, 242, 209, 202, 209, 192, 199, 44, 246, 242, 232, - 178, 233, 111, 199, 44, 246, 242, 209, 202, 209, 223, 199, 44, 246, 242, - 232, 178, 233, 82, 199, 44, 246, 242, 209, 202, 205, 147, 199, 44, 246, - 242, 216, 15, 174, 199, 44, 246, 242, 192, 13, 192, 116, 199, 44, 246, - 242, 206, 164, 206, 62, 199, 44, 246, 242, 206, 69, 199, 44, 246, 242, - 219, 46, 219, 107, 199, 44, 246, 242, 218, 227, 199, 44, 246, 242, 193, - 49, 193, 175, 199, 44, 246, 242, 202, 48, 205, 168, 199, 44, 246, 242, - 202, 48, 206, 33, 199, 44, 246, 242, 202, 48, 200, 252, 199, 44, 246, - 242, 229, 26, 229, 124, 199, 44, 246, 242, 219, 46, 243, 49, 199, 44, - 246, 242, 187, 251, 124, 199, 44, 246, 242, 229, 26, 213, 36, 199, 44, - 246, 242, 210, 196, 199, 44, 246, 242, 205, 127, 65, 199, 44, 246, 242, - 217, 115, 230, 57, 199, 44, 246, 242, 205, 127, 252, 208, 199, 44, 246, - 242, 205, 127, 251, 151, 199, 44, 246, 242, 205, 127, 68, 199, 44, 246, - 242, 205, 127, 223, 201, 199, 44, 246, 242, 205, 127, 196, 152, 199, 44, - 246, 242, 205, 127, 196, 149, 199, 44, 246, 242, 205, 127, 66, 199, 44, - 246, 242, 205, 127, 196, 30, 199, 44, 246, 242, 209, 204, 199, 44, 238, - 172, 16, 249, 121, 199, 44, 246, 242, 205, 127, 71, 199, 44, 246, 242, - 205, 127, 252, 27, 199, 44, 246, 242, 205, 127, 74, 199, 44, 246, 242, - 205, 127, 251, 239, 217, 109, 199, 44, 246, 242, 205, 127, 251, 239, 217, - 110, 199, 44, 246, 242, 220, 41, 199, 44, 246, 242, 217, 106, 199, 44, - 246, 242, 217, 107, 199, 44, 246, 242, 217, 115, 234, 149, 199, 44, 246, - 242, 217, 115, 198, 241, 199, 44, 246, 242, 217, 115, 197, 244, 199, 44, - 246, 242, 217, 115, 243, 134, 199, 44, 246, 242, 199, 36, 199, 44, 246, - 242, 213, 185, 199, 44, 246, 242, 192, 110, 199, 44, 246, 242, 232, 166, - 199, 44, 17, 191, 77, 199, 44, 17, 107, 199, 44, 17, 109, 199, 44, 17, - 138, 199, 44, 17, 134, 199, 44, 17, 150, 199, 44, 17, 169, 199, 44, 17, - 175, 199, 44, 17, 171, 199, 44, 17, 178, 199, 44, 246, 242, 251, 119, - 199, 44, 246, 242, 221, 182, 220, 19, 1, 221, 69, 220, 19, 1, 221, 186, - 200, 195, 220, 19, 1, 221, 186, 199, 197, 220, 19, 1, 210, 189, 231, 93, - 220, 19, 1, 213, 251, 220, 19, 1, 242, 101, 220, 19, 1, 210, 189, 247, 3, - 220, 19, 1, 202, 48, 199, 197, 220, 19, 1, 210, 189, 222, 254, 220, 19, - 1, 212, 67, 220, 19, 1, 210, 189, 212, 103, 220, 19, 1, 210, 189, 197, - 132, 220, 19, 1, 210, 189, 197, 120, 220, 19, 1, 210, 189, 237, 193, 220, - 19, 1, 210, 189, 237, 177, 220, 19, 1, 210, 189, 213, 81, 220, 19, 1, - 236, 176, 220, 19, 1, 159, 220, 19, 1, 198, 242, 200, 195, 220, 19, 1, - 198, 242, 199, 197, 220, 19, 1, 210, 189, 237, 70, 220, 19, 1, 213, 45, - 220, 19, 1, 248, 113, 220, 19, 1, 209, 75, 220, 19, 1, 209, 202, 200, - 195, 220, 19, 1, 232, 178, 199, 197, 220, 19, 1, 209, 202, 199, 197, 220, - 19, 1, 232, 178, 200, 195, 220, 19, 1, 210, 189, 248, 205, 220, 19, 1, - 216, 14, 220, 19, 1, 192, 12, 220, 19, 1, 219, 46, 219, 107, 220, 19, 1, - 219, 46, 219, 4, 220, 19, 1, 193, 48, 220, 19, 1, 205, 135, 203, 166, - 220, 19, 1, 205, 135, 201, 176, 220, 19, 1, 202, 48, 200, 195, 220, 19, - 1, 229, 26, 200, 195, 220, 19, 1, 210, 189, 219, 75, 220, 19, 1, 74, 220, - 19, 1, 229, 26, 199, 197, 220, 19, 234, 122, 220, 19, 18, 3, 65, 220, 19, - 18, 3, 217, 115, 222, 89, 220, 19, 18, 3, 252, 208, 220, 19, 18, 3, 251, - 151, 220, 19, 18, 3, 68, 220, 19, 18, 3, 223, 201, 220, 19, 18, 3, 192, - 159, 220, 19, 18, 3, 191, 176, 220, 19, 18, 3, 66, 220, 19, 18, 3, 196, - 30, 220, 19, 3, 210, 189, 195, 40, 220, 19, 18, 3, 217, 115, 221, 46, - 220, 19, 204, 21, 3, 219, 45, 220, 19, 204, 21, 3, 212, 67, 220, 19, 18, - 3, 71, 220, 19, 18, 3, 234, 168, 220, 19, 18, 3, 74, 220, 19, 18, 3, 250, - 133, 220, 19, 18, 3, 251, 238, 220, 19, 221, 70, 173, 220, 19, 163, 217, - 115, 234, 149, 220, 19, 163, 217, 115, 198, 241, 220, 19, 163, 217, 115, - 198, 193, 220, 19, 163, 217, 115, 247, 79, 220, 19, 247, 127, 77, 220, - 19, 213, 194, 220, 19, 192, 110, 220, 19, 17, 191, 77, 220, 19, 17, 107, - 220, 19, 17, 109, 220, 19, 17, 138, 220, 19, 17, 134, 220, 19, 17, 150, - 220, 19, 17, 169, 220, 19, 17, 175, 220, 19, 17, 171, 220, 19, 17, 178, - 220, 19, 229, 26, 213, 45, 220, 19, 229, 26, 216, 14, 220, 19, 1, 221, - 187, 231, 5, 220, 19, 1, 221, 187, 212, 67, 86, 5, 211, 115, 86, 87, 230, - 183, 192, 25, 216, 120, 197, 178, 65, 86, 87, 230, 183, 192, 25, 216, - 120, 255, 209, 206, 168, 249, 21, 174, 86, 87, 230, 183, 192, 25, 216, - 120, 255, 209, 230, 183, 197, 153, 174, 86, 87, 89, 192, 25, 216, 120, - 216, 236, 174, 86, 87, 242, 217, 192, 25, 216, 120, 203, 173, 174, 86, - 87, 247, 99, 192, 25, 216, 120, 209, 191, 203, 159, 174, 86, 87, 192, 25, - 216, 120, 197, 153, 203, 159, 174, 86, 87, 205, 101, 203, 158, 86, 87, - 248, 15, 192, 25, 216, 119, 86, 87, 248, 143, 203, 51, 192, 25, 216, 119, - 86, 87, 223, 100, 197, 152, 86, 87, 237, 115, 197, 153, 248, 14, 86, 87, - 203, 158, 86, 87, 212, 72, 203, 158, 86, 87, 197, 153, 203, 158, 86, 87, - 212, 72, 197, 153, 203, 158, 86, 87, 206, 192, 243, 113, 201, 194, 203, - 158, 86, 87, 207, 11, 230, 224, 203, 158, 86, 87, 247, 99, 255, 213, 206, - 74, 216, 235, 180, 247, 130, 86, 87, 230, 183, 197, 152, 86, 219, 29, 3, - 247, 1, 206, 73, 86, 219, 29, 3, 219, 159, 206, 73, 86, 250, 190, 3, 203, - 169, 231, 170, 255, 214, 206, 73, 86, 250, 190, 3, 255, 211, 168, 86, - 250, 190, 3, 205, 70, 197, 147, 86, 3, 207, 117, 236, 191, 231, 169, 86, - 3, 207, 117, 236, 191, 231, 7, 86, 3, 207, 117, 236, 191, 230, 184, 86, - 3, 207, 117, 214, 245, 231, 169, 86, 3, 207, 117, 214, 245, 231, 7, 86, - 3, 207, 117, 236, 191, 207, 117, 214, 244, 86, 17, 191, 77, 86, 17, 107, - 86, 17, 109, 86, 17, 138, 86, 17, 134, 86, 17, 150, 86, 17, 169, 86, 17, - 175, 86, 17, 171, 86, 17, 178, 86, 17, 132, 107, 86, 17, 132, 109, 86, - 17, 132, 138, 86, 17, 132, 134, 86, 17, 132, 150, 86, 17, 132, 169, 86, - 17, 132, 175, 86, 17, 132, 171, 86, 17, 132, 178, 86, 17, 132, 191, 77, - 86, 87, 248, 17, 206, 73, 86, 87, 214, 61, 247, 197, 212, 84, 191, 10, - 86, 87, 247, 99, 255, 213, 206, 74, 247, 198, 216, 64, 247, 130, 86, 87, - 214, 61, 247, 197, 203, 170, 206, 73, 86, 87, 243, 130, 216, 119, 86, 87, - 197, 169, 255, 210, 86, 87, 230, 166, 206, 74, 230, 121, 86, 87, 230, - 166, 206, 74, 230, 127, 86, 87, 251, 125, 221, 204, 230, 121, 86, 87, - 251, 125, 221, 204, 230, 127, 86, 3, 192, 102, 197, 151, 86, 3, 217, 68, - 197, 151, 86, 1, 155, 86, 1, 221, 217, 86, 1, 231, 242, 86, 1, 231, 93, - 86, 1, 214, 70, 86, 1, 247, 162, 86, 1, 247, 3, 86, 1, 223, 34, 86, 1, - 212, 103, 86, 1, 197, 132, 86, 1, 197, 120, 86, 1, 237, 193, 86, 1, 237, - 177, 86, 1, 213, 81, 86, 1, 190, 190, 86, 1, 199, 49, 86, 1, 238, 34, 86, - 1, 237, 70, 86, 1, 181, 86, 1, 168, 86, 1, 209, 230, 86, 1, 249, 155, 86, - 1, 248, 205, 86, 1, 174, 86, 1, 197, 168, 86, 1, 197, 157, 86, 1, 235, - 37, 86, 1, 235, 31, 86, 1, 193, 190, 86, 1, 191, 71, 86, 1, 191, 123, 86, - 1, 255, 216, 86, 1, 170, 86, 1, 165, 86, 1, 173, 86, 1, 203, 166, 86, 1, - 201, 176, 86, 1, 188, 86, 1, 140, 86, 1, 65, 86, 1, 220, 248, 86, 1, 232, - 223, 165, 86, 1, 221, 103, 86, 1, 206, 110, 86, 18, 3, 252, 208, 86, 18, - 3, 68, 86, 18, 3, 223, 201, 86, 18, 3, 66, 86, 18, 3, 196, 30, 86, 18, 3, - 117, 146, 86, 18, 3, 117, 206, 111, 86, 18, 3, 117, 172, 86, 18, 3, 117, - 219, 76, 86, 18, 3, 71, 86, 18, 3, 234, 190, 86, 18, 3, 74, 86, 18, 3, - 211, 89, 86, 3, 206, 174, 201, 6, 214, 71, 206, 163, 86, 3, 206, 168, - 249, 20, 86, 18, 3, 207, 19, 68, 86, 18, 3, 207, 19, 223, 201, 86, 3, - 212, 84, 191, 11, 214, 253, 238, 34, 86, 3, 202, 62, 219, 242, 86, 87, - 230, 74, 86, 87, 210, 180, 86, 3, 219, 245, 206, 73, 86, 3, 192, 107, - 206, 73, 86, 3, 219, 246, 197, 169, 247, 130, 86, 3, 216, 238, 247, 130, - 86, 3, 230, 187, 247, 131, 207, 9, 86, 3, 230, 187, 216, 222, 207, 9, 86, - 3, 223, 95, 216, 238, 247, 130, 86, 200, 240, 3, 219, 246, 197, 169, 247, - 130, 86, 200, 240, 3, 216, 238, 247, 130, 86, 200, 240, 3, 223, 95, 216, - 238, 247, 130, 86, 200, 240, 1, 155, 86, 200, 240, 1, 221, 217, 86, 200, - 240, 1, 231, 242, 86, 200, 240, 1, 231, 93, 86, 200, 240, 1, 214, 70, 86, - 200, 240, 1, 247, 162, 86, 200, 240, 1, 247, 3, 86, 200, 240, 1, 223, 34, - 86, 200, 240, 1, 212, 103, 86, 200, 240, 1, 197, 132, 86, 200, 240, 1, - 197, 120, 86, 200, 240, 1, 237, 193, 86, 200, 240, 1, 237, 177, 86, 200, - 240, 1, 213, 81, 86, 200, 240, 1, 190, 190, 86, 200, 240, 1, 199, 49, 86, - 200, 240, 1, 238, 34, 86, 200, 240, 1, 237, 70, 86, 200, 240, 1, 181, 86, - 200, 240, 1, 168, 86, 200, 240, 1, 209, 230, 86, 200, 240, 1, 249, 155, - 86, 200, 240, 1, 248, 205, 86, 200, 240, 1, 174, 86, 200, 240, 1, 197, - 168, 86, 200, 240, 1, 197, 157, 86, 200, 240, 1, 235, 37, 86, 200, 240, - 1, 235, 31, 86, 200, 240, 1, 193, 190, 86, 200, 240, 1, 191, 71, 86, 200, - 240, 1, 191, 123, 86, 200, 240, 1, 255, 216, 86, 200, 240, 1, 170, 86, - 200, 240, 1, 165, 86, 200, 240, 1, 173, 86, 200, 240, 1, 203, 166, 86, - 200, 240, 1, 201, 176, 86, 200, 240, 1, 188, 86, 200, 240, 1, 140, 86, - 200, 240, 1, 65, 86, 200, 240, 1, 220, 248, 86, 200, 240, 1, 232, 223, - 193, 190, 86, 200, 240, 1, 232, 223, 170, 86, 200, 240, 1, 232, 223, 165, - 86, 220, 235, 206, 70, 221, 217, 86, 220, 235, 206, 70, 221, 218, 247, - 198, 216, 64, 247, 130, 86, 247, 114, 3, 88, 249, 9, 86, 247, 114, 3, - 156, 249, 9, 86, 247, 114, 3, 247, 118, 199, 135, 86, 247, 114, 3, 205, - 100, 255, 215, 86, 16, 235, 107, 248, 12, 86, 16, 207, 116, 206, 175, 86, - 16, 210, 208, 231, 168, 86, 16, 207, 116, 206, 176, 207, 11, 230, 223, - 86, 16, 209, 191, 168, 86, 16, 213, 23, 248, 12, 86, 16, 213, 23, 248, - 13, 212, 72, 255, 212, 86, 16, 213, 23, 248, 13, 230, 185, 255, 212, 86, - 16, 213, 23, 248, 13, 247, 198, 255, 212, 86, 3, 207, 117, 214, 245, 207, - 117, 236, 190, 86, 3, 207, 117, 214, 245, 230, 184, 86, 87, 248, 16, 203, - 51, 231, 56, 216, 120, 207, 10, 86, 87, 216, 16, 192, 25, 231, 56, 216, - 120, 207, 10, 86, 87, 212, 72, 197, 152, 86, 87, 89, 248, 46, 206, 165, - 192, 25, 216, 120, 216, 236, 174, 86, 87, 242, 217, 248, 46, 206, 165, - 192, 25, 216, 120, 203, 173, 174, 206, 208, 200, 155, 56, 219, 225, 200, - 155, 56, 206, 208, 200, 155, 3, 4, 236, 140, 219, 225, 200, 155, 3, 4, - 236, 140, 86, 87, 219, 237, 216, 239, 206, 73, 86, 87, 198, 18, 216, 239, - 206, 73, 80, 1, 155, 80, 1, 221, 217, 80, 1, 231, 242, 80, 1, 231, 93, - 80, 1, 214, 70, 80, 1, 247, 162, 80, 1, 247, 3, 80, 1, 223, 34, 80, 1, - 222, 254, 80, 1, 212, 103, 80, 1, 213, 47, 80, 1, 197, 132, 80, 1, 197, - 120, 80, 1, 237, 193, 80, 1, 237, 177, 80, 1, 213, 81, 80, 1, 190, 190, - 80, 1, 199, 49, 80, 1, 238, 34, 80, 1, 237, 70, 80, 1, 181, 80, 1, 168, - 80, 1, 209, 230, 80, 1, 249, 155, 80, 1, 248, 205, 80, 1, 174, 80, 1, - 170, 80, 1, 165, 80, 1, 173, 80, 1, 193, 190, 80, 1, 188, 80, 1, 140, 80, - 1, 219, 75, 80, 1, 65, 80, 1, 203, 140, 65, 80, 1, 68, 80, 1, 223, 201, - 80, 1, 66, 80, 1, 196, 30, 80, 1, 71, 80, 1, 215, 234, 71, 80, 1, 74, 80, - 1, 250, 165, 80, 18, 3, 199, 200, 252, 208, 80, 18, 3, 252, 208, 80, 18, - 3, 68, 80, 18, 3, 223, 201, 80, 18, 3, 66, 80, 18, 3, 196, 30, 80, 18, 3, - 71, 80, 18, 3, 251, 238, 80, 18, 3, 215, 234, 223, 201, 80, 18, 3, 215, - 234, 74, 80, 18, 3, 235, 17, 58, 80, 3, 251, 73, 80, 3, 75, 60, 80, 3, - 195, 35, 80, 3, 195, 40, 80, 3, 250, 216, 80, 120, 3, 216, 219, 170, 80, - 120, 3, 216, 219, 165, 80, 120, 3, 216, 219, 193, 190, 80, 120, 3, 216, - 219, 140, 80, 1, 230, 208, 188, 80, 17, 191, 77, 80, 17, 107, 80, 17, - 109, 80, 17, 138, 80, 17, 134, 80, 17, 150, 80, 17, 169, 80, 17, 175, 80, - 17, 171, 80, 17, 178, 80, 3, 219, 85, 205, 54, 80, 3, 205, 54, 80, 16, - 219, 38, 80, 16, 242, 69, 80, 16, 252, 3, 80, 16, 231, 148, 80, 1, 203, - 166, 80, 1, 201, 176, 80, 1, 117, 146, 80, 1, 117, 206, 111, 80, 1, 117, - 172, 80, 1, 117, 219, 76, 80, 18, 3, 117, 146, 80, 18, 3, 117, 206, 111, - 80, 18, 3, 117, 172, 80, 18, 3, 117, 219, 76, 80, 1, 215, 234, 214, 70, - 80, 1, 215, 234, 222, 254, 80, 1, 215, 234, 249, 55, 80, 1, 215, 234, - 249, 50, 80, 120, 3, 215, 234, 216, 219, 181, 80, 120, 3, 215, 234, 216, - 219, 174, 80, 120, 3, 215, 234, 216, 219, 173, 80, 1, 203, 172, 222, 64, - 203, 166, 80, 18, 3, 203, 172, 222, 64, 233, 244, 80, 163, 87, 203, 172, - 222, 64, 230, 7, 80, 163, 87, 203, 172, 222, 64, 222, 25, 209, 201, 80, - 1, 193, 102, 208, 118, 222, 64, 199, 49, 80, 1, 193, 102, 208, 118, 222, - 64, 208, 124, 80, 18, 3, 193, 102, 208, 118, 222, 64, 233, 244, 80, 18, - 3, 193, 102, 208, 118, 222, 64, 196, 152, 80, 3, 193, 102, 208, 118, 222, - 64, 198, 78, 80, 3, 193, 102, 208, 118, 222, 64, 198, 77, 80, 3, 193, - 102, 208, 118, 222, 64, 198, 76, 80, 3, 193, 102, 208, 118, 222, 64, 198, - 75, 80, 3, 193, 102, 208, 118, 222, 64, 198, 74, 80, 1, 234, 204, 208, - 118, 222, 64, 213, 81, 80, 1, 234, 204, 208, 118, 222, 64, 191, 183, 80, - 1, 234, 204, 208, 118, 222, 64, 231, 58, 80, 18, 3, 231, 163, 222, 64, - 68, 80, 18, 3, 222, 30, 211, 153, 80, 18, 3, 222, 30, 66, 80, 18, 3, 222, - 30, 234, 190, 80, 1, 203, 140, 155, 80, 1, 203, 140, 221, 217, 80, 1, - 203, 140, 231, 242, 80, 1, 203, 140, 247, 162, 80, 1, 203, 140, 191, 123, - 80, 1, 203, 140, 212, 103, 80, 1, 203, 140, 238, 34, 80, 1, 203, 140, - 181, 80, 1, 203, 140, 209, 230, 80, 1, 203, 140, 233, 111, 80, 1, 203, - 140, 249, 155, 80, 1, 203, 140, 199, 49, 80, 1, 203, 140, 140, 80, 120, - 3, 203, 140, 216, 219, 193, 190, 80, 18, 3, 203, 140, 252, 208, 80, 18, - 3, 203, 140, 71, 80, 18, 3, 203, 140, 235, 17, 58, 80, 18, 3, 203, 140, - 53, 192, 159, 80, 3, 203, 140, 198, 77, 80, 3, 203, 140, 198, 76, 80, 3, - 203, 140, 198, 74, 80, 3, 203, 140, 198, 73, 80, 3, 203, 140, 238, 249, - 198, 77, 80, 3, 203, 140, 238, 249, 198, 76, 80, 3, 203, 140, 238, 249, - 234, 108, 198, 79, 80, 1, 206, 48, 210, 191, 233, 111, 80, 3, 206, 48, - 210, 191, 198, 74, 80, 203, 140, 17, 191, 77, 80, 203, 140, 17, 107, 80, - 203, 140, 17, 109, 80, 203, 140, 17, 138, 80, 203, 140, 17, 134, 80, 203, - 140, 17, 150, 80, 203, 140, 17, 169, 80, 203, 140, 17, 175, 80, 203, 140, - 17, 171, 80, 203, 140, 17, 178, 80, 3, 221, 208, 198, 78, 80, 3, 221, - 208, 198, 76, 80, 18, 3, 251, 224, 65, 80, 18, 3, 251, 224, 251, 238, 80, - 16, 203, 140, 107, 80, 16, 203, 140, 233, 217, 101, 6, 1, 251, 134, 101, - 6, 1, 249, 103, 101, 6, 1, 231, 212, 101, 6, 1, 236, 152, 101, 6, 1, 234, - 105, 101, 6, 1, 195, 49, 101, 6, 1, 191, 80, 101, 6, 1, 199, 193, 101, 6, - 1, 223, 164, 101, 6, 1, 222, 89, 101, 6, 1, 220, 9, 101, 6, 1, 217, 92, - 101, 6, 1, 214, 218, 101, 6, 1, 211, 106, 101, 6, 1, 210, 133, 101, 6, 1, - 191, 67, 101, 6, 1, 207, 165, 101, 6, 1, 205, 143, 101, 6, 1, 199, 179, - 101, 6, 1, 196, 113, 101, 6, 1, 209, 222, 101, 6, 1, 221, 202, 101, 6, 1, - 231, 84, 101, 6, 1, 208, 83, 101, 6, 1, 203, 70, 101, 6, 1, 243, 83, 101, - 6, 1, 247, 130, 101, 6, 1, 222, 236, 101, 6, 1, 243, 20, 101, 6, 1, 246, - 243, 101, 6, 1, 192, 218, 101, 6, 1, 222, 251, 101, 6, 1, 230, 89, 101, - 6, 1, 229, 247, 101, 6, 1, 229, 147, 101, 6, 1, 193, 125, 101, 6, 1, 230, - 21, 101, 6, 1, 229, 13, 101, 6, 1, 192, 14, 101, 6, 1, 252, 16, 101, 1, - 251, 134, 101, 1, 249, 103, 101, 1, 231, 212, 101, 1, 236, 152, 101, 1, - 234, 105, 101, 1, 195, 49, 101, 1, 191, 80, 101, 1, 199, 193, 101, 1, - 223, 164, 101, 1, 222, 89, 101, 1, 220, 9, 101, 1, 217, 92, 101, 1, 214, - 218, 101, 1, 211, 106, 101, 1, 210, 133, 101, 1, 191, 67, 101, 1, 207, - 165, 101, 1, 205, 143, 101, 1, 199, 179, 101, 1, 196, 113, 101, 1, 209, - 222, 101, 1, 221, 202, 101, 1, 231, 84, 101, 1, 208, 83, 101, 1, 203, 70, - 101, 1, 243, 83, 101, 1, 247, 130, 101, 1, 222, 236, 101, 1, 243, 20, - 101, 1, 246, 243, 101, 1, 192, 218, 101, 1, 222, 251, 101, 1, 230, 89, - 101, 1, 229, 247, 101, 1, 229, 147, 101, 1, 193, 125, 101, 1, 230, 21, - 101, 1, 229, 13, 101, 1, 233, 25, 101, 1, 192, 14, 101, 1, 234, 125, 101, - 1, 154, 231, 212, 101, 1, 251, 232, 101, 210, 130, 204, 11, 52, 1, 101, - 214, 218, 101, 1, 252, 16, 101, 1, 230, 19, 56, 101, 1, 220, 118, 56, 30, - 147, 221, 115, 30, 147, 201, 168, 30, 147, 213, 206, 30, 147, 198, 168, - 30, 147, 201, 157, 30, 147, 206, 240, 30, 147, 216, 79, 30, 147, 209, - 171, 30, 147, 201, 165, 30, 147, 202, 161, 30, 147, 201, 162, 30, 147, - 223, 224, 30, 147, 243, 26, 30, 147, 201, 172, 30, 147, 243, 93, 30, 147, - 221, 190, 30, 147, 199, 7, 30, 147, 209, 211, 30, 147, 229, 144, 30, 147, - 213, 202, 30, 147, 201, 166, 30, 147, 213, 196, 30, 147, 213, 200, 30, - 147, 198, 165, 30, 147, 206, 228, 30, 147, 201, 164, 30, 147, 206, 238, - 30, 147, 222, 70, 30, 147, 216, 72, 30, 147, 222, 73, 30, 147, 209, 166, - 30, 147, 209, 164, 30, 147, 209, 152, 30, 147, 209, 160, 30, 147, 209, - 158, 30, 147, 209, 155, 30, 147, 209, 157, 30, 147, 209, 154, 30, 147, - 209, 159, 30, 147, 209, 169, 30, 147, 209, 170, 30, 147, 209, 153, 30, - 147, 209, 163, 30, 147, 222, 71, 30, 147, 222, 69, 30, 147, 202, 154, 30, - 147, 202, 152, 30, 147, 202, 144, 30, 147, 202, 147, 30, 147, 202, 153, - 30, 147, 202, 149, 30, 147, 202, 148, 30, 147, 202, 146, 30, 147, 202, - 157, 30, 147, 202, 159, 30, 147, 202, 160, 30, 147, 202, 155, 30, 147, - 202, 145, 30, 147, 202, 150, 30, 147, 202, 158, 30, 147, 243, 74, 30, - 147, 243, 72, 30, 147, 247, 16, 30, 147, 247, 14, 30, 147, 210, 151, 30, - 147, 223, 219, 30, 147, 223, 210, 30, 147, 223, 218, 30, 147, 223, 215, - 30, 147, 223, 213, 30, 147, 223, 217, 30, 147, 201, 169, 30, 147, 223, - 222, 30, 147, 223, 223, 30, 147, 223, 211, 30, 147, 223, 216, 30, 147, - 192, 57, 30, 147, 243, 25, 30, 147, 243, 75, 30, 147, 243, 73, 30, 147, - 247, 17, 30, 147, 247, 15, 30, 147, 243, 91, 30, 147, 243, 92, 30, 147, - 243, 76, 30, 147, 247, 18, 30, 147, 209, 209, 30, 147, 222, 72, 30, 147, - 201, 170, 30, 147, 192, 63, 30, 147, 221, 106, 30, 147, 213, 198, 30, - 147, 213, 204, 30, 147, 213, 203, 30, 147, 198, 162, 30, 147, 233, 5, 30, - 222, 176, 233, 5, 30, 222, 176, 65, 30, 222, 176, 252, 27, 30, 222, 176, - 170, 30, 222, 176, 192, 129, 30, 222, 176, 234, 67, 30, 222, 176, 71, 30, - 222, 176, 192, 67, 30, 222, 176, 192, 80, 30, 222, 176, 74, 30, 222, 176, - 193, 190, 30, 222, 176, 193, 176, 30, 222, 176, 211, 153, 30, 222, 176, - 192, 12, 30, 222, 176, 66, 30, 222, 176, 193, 107, 30, 222, 176, 193, - 125, 30, 222, 176, 193, 86, 30, 222, 176, 191, 225, 30, 222, 176, 233, - 244, 30, 222, 176, 192, 33, 30, 222, 176, 68, 30, 222, 176, 255, 204, 30, - 222, 176, 255, 203, 30, 222, 176, 192, 143, 30, 222, 176, 192, 141, 30, - 222, 176, 234, 65, 30, 222, 176, 234, 64, 30, 222, 176, 234, 66, 30, 222, - 176, 192, 66, 30, 222, 176, 192, 65, 30, 222, 176, 212, 12, 30, 222, 176, - 212, 13, 30, 222, 176, 212, 6, 30, 222, 176, 212, 11, 30, 222, 176, 212, - 9, 30, 222, 176, 192, 0, 30, 222, 176, 191, 255, 30, 222, 176, 191, 254, - 30, 222, 176, 192, 1, 30, 222, 176, 192, 2, 30, 222, 176, 196, 226, 30, - 222, 176, 196, 225, 30, 222, 176, 196, 223, 30, 222, 176, 196, 219, 30, - 222, 176, 196, 220, 30, 222, 176, 191, 220, 30, 222, 176, 191, 217, 30, - 222, 176, 191, 218, 30, 222, 176, 191, 212, 30, 222, 176, 191, 213, 30, - 222, 176, 191, 214, 30, 222, 176, 191, 216, 30, 222, 176, 233, 238, 30, - 222, 176, 233, 240, 30, 222, 176, 192, 32, 30, 222, 176, 228, 71, 30, - 222, 176, 228, 63, 30, 222, 176, 228, 66, 30, 222, 176, 228, 64, 30, 222, - 176, 228, 68, 30, 222, 176, 228, 70, 30, 222, 176, 251, 27, 30, 222, 176, - 251, 24, 30, 222, 176, 251, 22, 30, 222, 176, 251, 23, 30, 222, 176, 201, - 173, 30, 222, 176, 255, 205, 30, 222, 176, 192, 142, 30, 222, 176, 192, - 64, 30, 222, 176, 212, 8, 30, 222, 176, 212, 7, 30, 125, 221, 115, 30, - 125, 201, 168, 30, 125, 221, 108, 30, 125, 213, 206, 30, 125, 213, 204, - 30, 125, 213, 203, 30, 125, 198, 168, 30, 125, 206, 240, 30, 125, 206, - 235, 30, 125, 206, 232, 30, 125, 206, 225, 30, 125, 206, 220, 30, 125, - 206, 215, 30, 125, 206, 226, 30, 125, 206, 238, 30, 125, 216, 79, 30, - 125, 209, 171, 30, 125, 209, 160, 30, 125, 202, 161, 30, 125, 201, 162, - 30, 125, 223, 224, 30, 125, 243, 26, 30, 125, 243, 93, 30, 125, 221, 190, - 30, 125, 199, 7, 30, 125, 209, 211, 30, 125, 229, 144, 30, 125, 221, 109, - 30, 125, 221, 107, 30, 125, 213, 202, 30, 125, 213, 196, 30, 125, 213, - 198, 30, 125, 213, 201, 30, 125, 213, 197, 30, 125, 198, 165, 30, 125, - 198, 162, 30, 125, 206, 233, 30, 125, 206, 228, 30, 125, 206, 214, 30, - 125, 206, 213, 30, 125, 201, 164, 30, 125, 206, 230, 30, 125, 206, 229, - 30, 125, 206, 222, 30, 125, 206, 224, 30, 125, 206, 237, 30, 125, 206, - 217, 30, 125, 206, 227, 30, 125, 206, 236, 30, 125, 206, 212, 30, 125, - 216, 75, 30, 125, 216, 70, 30, 125, 216, 72, 30, 125, 216, 69, 30, 125, - 216, 67, 30, 125, 216, 73, 30, 125, 216, 78, 30, 125, 216, 76, 30, 125, - 222, 73, 30, 125, 209, 162, 30, 125, 209, 163, 30, 125, 209, 168, 30, - 125, 222, 71, 30, 125, 202, 154, 30, 125, 202, 144, 30, 125, 202, 147, - 30, 125, 202, 149, 30, 125, 210, 151, 30, 125, 223, 219, 30, 125, 223, - 212, 30, 125, 201, 169, 30, 125, 223, 220, 30, 125, 192, 57, 30, 125, - 192, 51, 30, 125, 192, 52, 30, 125, 209, 209, 30, 125, 222, 72, 30, 125, - 229, 142, 30, 125, 229, 140, 30, 125, 229, 143, 30, 125, 229, 141, 30, - 125, 192, 63, 30, 125, 221, 111, 30, 125, 221, 110, 30, 125, 221, 114, - 30, 125, 221, 112, 30, 125, 221, 113, 30, 125, 201, 166, 36, 5, 140, 36, - 5, 228, 161, 36, 5, 229, 160, 36, 5, 230, 93, 36, 5, 229, 217, 36, 5, - 229, 247, 36, 5, 229, 25, 36, 5, 229, 16, 36, 5, 173, 36, 5, 218, 227, - 36, 5, 219, 148, 36, 5, 220, 127, 36, 5, 219, 230, 36, 5, 219, 240, 36, - 5, 219, 45, 36, 5, 218, 193, 36, 5, 229, 179, 36, 5, 229, 173, 36, 5, - 229, 175, 36, 5, 229, 178, 36, 5, 229, 176, 36, 5, 229, 177, 36, 5, 229, - 174, 36, 5, 229, 172, 36, 5, 174, 36, 5, 215, 157, 36, 5, 216, 102, 36, - 5, 217, 153, 36, 5, 216, 213, 36, 5, 216, 234, 36, 5, 216, 14, 36, 5, - 215, 82, 36, 5, 200, 54, 36, 5, 200, 48, 36, 5, 200, 50, 36, 5, 200, 53, - 36, 5, 200, 51, 36, 5, 200, 52, 36, 5, 200, 49, 36, 5, 200, 47, 36, 5, - 165, 36, 5, 206, 69, 36, 5, 207, 2, 36, 5, 207, 180, 36, 5, 207, 86, 36, - 5, 207, 115, 36, 5, 206, 163, 36, 5, 206, 27, 36, 5, 188, 36, 5, 201, 5, - 36, 5, 202, 223, 36, 5, 205, 198, 36, 5, 205, 51, 36, 5, 205, 69, 36, 5, - 202, 47, 36, 5, 200, 150, 36, 5, 203, 166, 36, 5, 203, 6, 36, 5, 203, 82, - 36, 5, 203, 161, 36, 5, 203, 112, 36, 5, 203, 114, 36, 5, 203, 57, 36, 5, - 202, 241, 36, 5, 208, 98, 36, 5, 208, 35, 36, 5, 208, 59, 36, 5, 208, 97, - 36, 5, 208, 76, 36, 5, 208, 77, 36, 5, 208, 47, 36, 5, 208, 46, 36, 5, - 207, 242, 36, 5, 207, 238, 36, 5, 207, 241, 36, 5, 207, 239, 36, 5, 207, - 240, 36, 5, 208, 73, 36, 5, 208, 65, 36, 5, 208, 68, 36, 5, 208, 72, 36, - 5, 208, 69, 36, 5, 208, 70, 36, 5, 208, 67, 36, 5, 208, 64, 36, 5, 208, - 60, 36, 5, 208, 63, 36, 5, 208, 61, 36, 5, 208, 62, 36, 5, 249, 155, 36, - 5, 248, 12, 36, 5, 248, 190, 36, 5, 249, 153, 36, 5, 249, 3, 36, 5, 249, - 19, 36, 5, 248, 113, 36, 5, 247, 212, 36, 5, 195, 188, 36, 5, 193, 249, - 36, 5, 195, 69, 36, 5, 195, 187, 36, 5, 195, 148, 36, 5, 195, 153, 36, 5, - 195, 24, 36, 5, 193, 238, 36, 5, 190, 190, 36, 5, 197, 94, 36, 5, 198, - 193, 36, 5, 199, 245, 36, 5, 199, 121, 36, 5, 199, 145, 36, 5, 159, 36, - 5, 197, 43, 36, 5, 247, 162, 36, 5, 238, 197, 36, 5, 243, 31, 36, 5, 247, - 161, 36, 5, 247, 36, 36, 5, 247, 44, 36, 5, 242, 101, 36, 5, 238, 153, - 36, 5, 192, 220, 36, 5, 192, 188, 36, 5, 192, 207, 36, 5, 192, 219, 36, - 5, 192, 213, 36, 5, 192, 214, 36, 5, 192, 196, 36, 5, 192, 195, 36, 5, - 192, 181, 36, 5, 192, 177, 36, 5, 192, 180, 36, 5, 192, 178, 36, 5, 192, - 179, 36, 5, 181, 36, 5, 212, 180, 36, 5, 213, 221, 36, 5, 214, 252, 36, - 5, 214, 112, 36, 5, 214, 123, 36, 5, 213, 45, 36, 5, 212, 112, 36, 5, - 212, 103, 36, 5, 212, 60, 36, 5, 212, 83, 36, 5, 212, 102, 36, 5, 212, - 91, 36, 5, 212, 92, 36, 5, 212, 67, 36, 5, 212, 50, 36, 5, 231, 13, 65, - 36, 5, 231, 13, 66, 36, 5, 231, 13, 68, 36, 5, 231, 13, 252, 208, 36, 5, - 231, 13, 234, 190, 36, 5, 231, 13, 71, 36, 5, 231, 13, 74, 36, 5, 231, - 13, 193, 190, 36, 5, 155, 36, 5, 220, 234, 36, 5, 221, 168, 36, 5, 222, - 129, 36, 5, 222, 15, 36, 5, 222, 24, 36, 5, 221, 69, 36, 5, 221, 64, 36, - 5, 220, 181, 36, 5, 220, 174, 36, 5, 220, 180, 36, 5, 220, 175, 36, 5, - 220, 176, 36, 5, 220, 167, 36, 5, 220, 161, 36, 5, 220, 163, 36, 5, 220, - 166, 36, 5, 220, 164, 36, 5, 220, 165, 36, 5, 220, 162, 36, 5, 220, 160, - 36, 5, 220, 156, 36, 5, 220, 159, 36, 5, 220, 157, 36, 5, 220, 158, 36, - 5, 193, 190, 36, 5, 193, 0, 36, 5, 193, 86, 36, 5, 193, 181, 36, 5, 193, - 114, 36, 5, 193, 125, 36, 5, 193, 48, 36, 5, 193, 40, 36, 5, 209, 221, - 65, 36, 5, 209, 221, 66, 36, 5, 209, 221, 68, 36, 5, 209, 221, 252, 208, - 36, 5, 209, 221, 234, 190, 36, 5, 209, 221, 71, 36, 5, 209, 221, 74, 36, - 5, 191, 123, 36, 5, 190, 251, 36, 5, 191, 30, 36, 5, 191, 121, 36, 5, - 191, 84, 36, 5, 191, 87, 36, 5, 191, 7, 36, 5, 190, 238, 36, 5, 191, 71, - 36, 5, 191, 48, 36, 5, 191, 57, 36, 5, 191, 70, 36, 5, 191, 61, 36, 5, - 191, 62, 36, 5, 191, 54, 36, 5, 191, 39, 36, 5, 170, 36, 5, 191, 225, 36, - 5, 192, 33, 36, 5, 192, 140, 36, 5, 192, 77, 36, 5, 192, 80, 36, 5, 192, - 12, 36, 5, 191, 252, 36, 5, 238, 34, 36, 5, 235, 91, 36, 5, 237, 46, 36, - 5, 238, 33, 36, 5, 237, 133, 36, 5, 237, 148, 36, 5, 236, 176, 36, 5, - 235, 48, 36, 5, 237, 193, 36, 5, 237, 158, 36, 5, 237, 170, 36, 5, 237, - 192, 36, 5, 237, 180, 36, 5, 237, 181, 36, 5, 237, 163, 36, 5, 237, 149, - 36, 5, 223, 34, 36, 5, 222, 184, 36, 5, 222, 246, 36, 5, 223, 33, 36, 5, - 223, 10, 36, 5, 223, 12, 36, 5, 222, 203, 36, 5, 222, 162, 36, 5, 231, - 242, 36, 5, 230, 181, 36, 5, 231, 55, 36, 5, 231, 239, 36, 5, 231, 159, - 36, 5, 231, 167, 36, 5, 231, 5, 36, 5, 231, 4, 36, 5, 230, 137, 36, 5, - 230, 133, 36, 5, 230, 136, 36, 5, 230, 134, 36, 5, 230, 135, 36, 5, 231, - 129, 36, 5, 231, 109, 36, 5, 231, 119, 36, 5, 231, 128, 36, 5, 231, 123, - 36, 5, 231, 124, 36, 5, 231, 113, 36, 5, 231, 98, 36, 5, 199, 49, 36, 5, - 198, 213, 36, 5, 199, 11, 36, 5, 199, 48, 36, 5, 199, 31, 36, 5, 199, 33, - 36, 5, 198, 241, 36, 5, 198, 204, 36, 5, 247, 3, 36, 5, 243, 50, 36, 5, - 243, 97, 36, 5, 247, 2, 36, 5, 243, 125, 36, 5, 243, 129, 36, 5, 243, 70, - 36, 5, 243, 39, 36, 5, 209, 230, 36, 5, 209, 193, 36, 5, 209, 213, 36, 5, - 209, 229, 36, 5, 209, 215, 36, 5, 209, 216, 36, 5, 209, 201, 36, 5, 209, - 189, 36, 5, 197, 168, 36, 5, 197, 140, 36, 5, 197, 146, 36, 5, 197, 167, - 36, 5, 197, 160, 36, 5, 197, 161, 36, 5, 197, 144, 36, 5, 197, 138, 36, - 5, 196, 240, 36, 5, 196, 232, 36, 5, 196, 236, 36, 5, 196, 239, 36, 5, - 196, 237, 36, 5, 196, 238, 36, 5, 196, 234, 36, 5, 196, 233, 36, 5, 233, - 111, 36, 5, 232, 88, 36, 5, 233, 25, 36, 5, 233, 110, 36, 5, 233, 54, 36, - 5, 233, 61, 36, 5, 232, 177, 36, 5, 232, 64, 36, 5, 168, 36, 5, 208, 167, - 36, 5, 209, 187, 36, 5, 210, 222, 36, 5, 210, 51, 36, 5, 210, 65, 36, 5, - 209, 75, 36, 5, 208, 124, 36, 5, 206, 17, 36, 5, 215, 70, 36, 5, 232, 58, - 36, 33, 231, 155, 23, 18, 219, 200, 77, 36, 33, 18, 219, 200, 77, 36, 33, - 231, 155, 77, 36, 205, 55, 77, 36, 193, 22, 36, 232, 82, 201, 64, 36, - 242, 76, 36, 204, 26, 36, 242, 85, 36, 208, 230, 242, 85, 36, 208, 15, - 77, 36, 210, 130, 204, 11, 36, 17, 107, 36, 17, 109, 36, 17, 138, 36, 17, - 134, 36, 17, 150, 36, 17, 169, 36, 17, 175, 36, 17, 171, 36, 17, 178, 36, - 31, 199, 95, 36, 31, 197, 32, 36, 31, 198, 249, 36, 31, 232, 137, 36, 31, - 233, 17, 36, 31, 202, 121, 36, 31, 203, 242, 36, 31, 234, 155, 36, 31, - 213, 171, 36, 31, 228, 142, 36, 31, 199, 96, 189, 36, 5, 205, 60, 215, - 82, 36, 5, 215, 78, 36, 5, 215, 79, 36, 5, 215, 80, 36, 5, 205, 60, 247, - 212, 36, 5, 247, 209, 36, 5, 247, 210, 36, 5, 247, 211, 36, 5, 205, 60, - 232, 64, 36, 5, 232, 60, 36, 5, 232, 61, 36, 5, 232, 62, 36, 5, 205, 60, - 208, 124, 36, 5, 208, 120, 36, 5, 208, 121, 36, 5, 208, 122, 36, 198, 80, - 87, 192, 15, 36, 198, 80, 87, 237, 91, 36, 198, 80, 87, 206, 195, 36, - 198, 80, 87, 203, 41, 206, 195, 36, 198, 80, 87, 237, 20, 36, 198, 80, - 87, 221, 252, 36, 198, 80, 87, 243, 78, 36, 198, 80, 87, 229, 149, 36, - 198, 80, 87, 237, 90, 36, 198, 80, 87, 220, 197, 103, 1, 65, 103, 1, 71, - 103, 1, 68, 103, 1, 74, 103, 1, 66, 103, 1, 196, 12, 103, 1, 231, 242, - 103, 1, 155, 103, 1, 231, 167, 103, 1, 231, 55, 103, 1, 231, 5, 103, 1, - 230, 181, 103, 1, 230, 140, 103, 1, 140, 103, 1, 229, 247, 103, 1, 229, - 160, 103, 1, 229, 25, 103, 1, 228, 161, 103, 1, 228, 128, 103, 1, 173, - 103, 1, 219, 240, 103, 1, 219, 148, 103, 1, 219, 45, 103, 1, 218, 227, - 103, 1, 218, 194, 103, 1, 174, 103, 1, 216, 234, 103, 1, 216, 102, 103, - 1, 216, 14, 103, 1, 215, 157, 103, 1, 181, 103, 1, 229, 49, 103, 1, 214, - 239, 103, 1, 214, 123, 103, 1, 213, 221, 103, 1, 213, 45, 103, 1, 212, - 180, 103, 1, 212, 114, 103, 1, 208, 34, 103, 1, 208, 18, 103, 1, 208, 11, - 103, 1, 208, 1, 103, 1, 207, 246, 103, 1, 207, 244, 103, 1, 188, 103, 1, - 206, 9, 103, 1, 205, 69, 103, 1, 202, 223, 103, 1, 202, 47, 103, 1, 201, - 5, 103, 1, 200, 158, 103, 1, 238, 34, 103, 1, 190, 190, 103, 1, 237, 148, - 103, 1, 199, 145, 103, 1, 237, 46, 103, 1, 198, 193, 103, 1, 236, 176, - 103, 1, 235, 91, 103, 1, 235, 59, 103, 1, 236, 188, 103, 1, 198, 115, - 103, 1, 198, 114, 103, 1, 198, 103, 103, 1, 198, 102, 103, 1, 198, 101, - 103, 1, 198, 100, 103, 1, 197, 168, 103, 1, 197, 161, 103, 1, 197, 146, - 103, 1, 197, 144, 103, 1, 197, 140, 103, 1, 197, 139, 103, 1, 193, 190, - 103, 1, 193, 125, 103, 1, 193, 86, 103, 1, 193, 48, 103, 1, 193, 0, 103, - 1, 192, 243, 103, 1, 170, 103, 1, 192, 80, 103, 1, 192, 33, 103, 1, 192, - 12, 103, 1, 191, 225, 103, 1, 191, 184, 103, 1, 215, 89, 103, 2, 1, 192, - 80, 103, 2, 1, 192, 33, 103, 2, 1, 192, 12, 103, 2, 1, 191, 225, 103, 2, - 1, 191, 184, 103, 2, 1, 215, 89, 21, 22, 228, 90, 21, 22, 71, 21, 22, - 252, 172, 21, 22, 68, 21, 22, 223, 201, 21, 22, 74, 21, 22, 211, 89, 21, - 22, 192, 158, 211, 89, 21, 22, 98, 234, 190, 21, 22, 98, 68, 21, 22, 65, - 21, 22, 252, 208, 21, 22, 193, 125, 21, 22, 193, 103, 193, 125, 21, 22, - 193, 86, 21, 22, 193, 103, 193, 86, 21, 22, 193, 70, 21, 22, 193, 103, - 193, 70, 21, 22, 193, 48, 21, 22, 193, 103, 193, 48, 21, 22, 193, 29, 21, - 22, 193, 103, 193, 29, 21, 22, 214, 211, 193, 29, 21, 22, 193, 190, 21, - 22, 193, 103, 193, 190, 21, 22, 193, 181, 21, 22, 193, 103, 193, 181, 21, - 22, 214, 211, 193, 181, 21, 22, 251, 238, 21, 22, 192, 158, 193, 224, 21, - 22, 231, 13, 201, 64, 21, 22, 53, 252, 48, 21, 22, 53, 230, 212, 21, 22, - 53, 248, 79, 132, 206, 189, 21, 22, 53, 198, 54, 132, 206, 189, 21, 22, - 53, 50, 132, 206, 189, 21, 22, 53, 206, 189, 21, 22, 53, 55, 252, 48, 21, - 22, 53, 55, 203, 41, 81, 201, 16, 21, 22, 53, 82, 236, 142, 21, 22, 53, - 203, 41, 228, 243, 106, 21, 22, 53, 209, 83, 21, 22, 53, 144, 199, 228, - 21, 22, 234, 105, 21, 22, 223, 164, 21, 22, 211, 106, 21, 22, 251, 134, - 21, 22, 210, 65, 21, 22, 210, 220, 21, 22, 209, 187, 21, 22, 209, 147, - 21, 22, 209, 75, 21, 22, 209, 39, 21, 22, 192, 158, 209, 39, 21, 22, 98, - 229, 217, 21, 22, 98, 229, 160, 21, 22, 168, 21, 22, 210, 222, 21, 22, - 208, 122, 21, 22, 193, 103, 208, 122, 21, 22, 208, 120, 21, 22, 193, 103, - 208, 120, 21, 22, 208, 119, 21, 22, 193, 103, 208, 119, 21, 22, 208, 117, - 21, 22, 193, 103, 208, 117, 21, 22, 208, 116, 21, 22, 193, 103, 208, 116, - 21, 22, 208, 124, 21, 22, 193, 103, 208, 124, 21, 22, 208, 123, 21, 22, - 193, 103, 208, 123, 21, 22, 192, 158, 208, 123, 21, 22, 210, 238, 21, 22, - 193, 103, 210, 238, 21, 22, 98, 230, 118, 21, 22, 199, 145, 21, 22, 199, - 242, 21, 22, 198, 193, 21, 22, 198, 170, 21, 22, 159, 21, 22, 198, 59, - 21, 22, 192, 158, 198, 59, 21, 22, 98, 237, 133, 21, 22, 98, 237, 46, 21, - 22, 190, 190, 21, 22, 199, 245, 21, 22, 197, 41, 21, 22, 193, 103, 197, - 41, 21, 22, 197, 19, 21, 22, 193, 103, 197, 19, 21, 22, 197, 18, 21, 22, - 193, 103, 197, 18, 21, 22, 109, 21, 22, 193, 103, 109, 21, 22, 197, 9, - 21, 22, 193, 103, 197, 9, 21, 22, 197, 43, 21, 22, 193, 103, 197, 43, 21, - 22, 197, 42, 21, 22, 193, 103, 197, 42, 21, 22, 214, 211, 197, 42, 21, - 22, 200, 43, 21, 22, 197, 127, 21, 22, 197, 111, 21, 22, 197, 109, 21, - 22, 197, 132, 21, 22, 222, 24, 21, 22, 222, 123, 21, 22, 221, 168, 21, - 22, 221, 147, 21, 22, 221, 69, 21, 22, 221, 43, 21, 22, 192, 158, 221, - 43, 21, 22, 155, 21, 22, 222, 129, 21, 22, 220, 176, 21, 22, 193, 103, - 220, 176, 21, 22, 220, 174, 21, 22, 193, 103, 220, 174, 21, 22, 220, 173, - 21, 22, 193, 103, 220, 173, 21, 22, 220, 171, 21, 22, 193, 103, 220, 171, - 21, 22, 220, 170, 21, 22, 193, 103, 220, 170, 21, 22, 220, 181, 21, 22, - 193, 103, 220, 181, 21, 22, 220, 180, 21, 22, 193, 103, 220, 180, 21, 22, - 214, 211, 220, 180, 21, 22, 222, 154, 21, 22, 220, 182, 21, 22, 202, 2, - 222, 8, 21, 22, 202, 2, 221, 148, 21, 22, 202, 2, 221, 58, 21, 22, 202, - 2, 222, 106, 21, 22, 247, 44, 21, 22, 247, 160, 21, 22, 243, 31, 21, 22, - 243, 21, 21, 22, 242, 101, 21, 22, 239, 20, 21, 22, 192, 158, 239, 20, - 21, 22, 247, 162, 21, 22, 247, 161, 21, 22, 238, 151, 21, 22, 193, 103, - 238, 151, 21, 22, 238, 149, 21, 22, 193, 103, 238, 149, 21, 22, 238, 148, - 21, 22, 193, 103, 238, 148, 21, 22, 238, 147, 21, 22, 193, 103, 238, 147, - 21, 22, 238, 146, 21, 22, 193, 103, 238, 146, 21, 22, 238, 153, 21, 22, - 193, 103, 238, 153, 21, 22, 238, 152, 21, 22, 193, 103, 238, 152, 21, 22, - 214, 211, 238, 152, 21, 22, 247, 195, 21, 22, 205, 102, 199, 51, 21, 22, - 216, 234, 21, 22, 217, 152, 21, 22, 216, 102, 21, 22, 216, 63, 21, 22, - 216, 14, 21, 22, 215, 213, 21, 22, 192, 158, 215, 213, 21, 22, 174, 21, - 22, 217, 153, 21, 22, 215, 80, 21, 22, 193, 103, 215, 80, 21, 22, 215, - 78, 21, 22, 193, 103, 215, 78, 21, 22, 215, 77, 21, 22, 193, 103, 215, - 77, 21, 22, 215, 76, 21, 22, 193, 103, 215, 76, 21, 22, 215, 75, 21, 22, - 193, 103, 215, 75, 21, 22, 215, 82, 21, 22, 193, 103, 215, 82, 21, 22, - 215, 81, 21, 22, 193, 103, 215, 81, 21, 22, 214, 211, 215, 81, 21, 22, - 218, 170, 21, 22, 193, 103, 218, 170, 21, 22, 216, 106, 21, 22, 250, 182, - 218, 170, 21, 22, 205, 102, 218, 170, 21, 22, 214, 123, 21, 22, 214, 251, - 21, 22, 213, 221, 21, 22, 213, 188, 21, 22, 213, 45, 21, 22, 213, 28, 21, - 22, 192, 158, 213, 28, 21, 22, 181, 21, 22, 214, 252, 21, 22, 212, 110, - 21, 22, 193, 103, 212, 110, 21, 22, 212, 112, 21, 22, 193, 103, 212, 112, - 21, 22, 212, 111, 21, 22, 193, 103, 212, 111, 21, 22, 214, 211, 212, 111, - 21, 22, 215, 63, 21, 22, 98, 214, 72, 21, 22, 213, 227, 21, 22, 219, 240, - 21, 22, 220, 126, 21, 22, 219, 148, 21, 22, 219, 130, 21, 22, 219, 45, - 21, 22, 219, 10, 21, 22, 192, 158, 219, 10, 21, 22, 173, 21, 22, 220, - 127, 21, 22, 218, 191, 21, 22, 193, 103, 218, 191, 21, 22, 218, 190, 21, - 22, 193, 103, 218, 190, 21, 22, 218, 189, 21, 22, 193, 103, 218, 189, 21, - 22, 218, 188, 21, 22, 193, 103, 218, 188, 21, 22, 218, 187, 21, 22, 193, - 103, 218, 187, 21, 22, 218, 193, 21, 22, 193, 103, 218, 193, 21, 22, 218, - 192, 21, 22, 193, 103, 218, 192, 21, 22, 172, 21, 22, 193, 103, 172, 21, - 22, 216, 219, 172, 21, 22, 205, 69, 21, 22, 205, 196, 21, 22, 202, 223, - 21, 22, 202, 194, 21, 22, 202, 47, 21, 22, 202, 17, 21, 22, 192, 158, - 202, 17, 21, 22, 188, 21, 22, 205, 198, 21, 22, 200, 145, 21, 22, 193, - 103, 200, 145, 21, 22, 200, 139, 21, 22, 193, 103, 200, 139, 21, 22, 200, - 138, 21, 22, 193, 103, 200, 138, 21, 22, 200, 133, 21, 22, 193, 103, 200, - 133, 21, 22, 200, 132, 21, 22, 193, 103, 200, 132, 21, 22, 200, 150, 21, - 22, 193, 103, 200, 150, 21, 22, 200, 149, 21, 22, 193, 103, 200, 149, 21, - 22, 214, 211, 200, 149, 21, 22, 206, 9, 21, 22, 250, 182, 206, 9, 21, 22, - 200, 151, 21, 22, 248, 138, 206, 9, 21, 22, 215, 205, 202, 115, 21, 22, - 214, 211, 202, 102, 21, 22, 214, 211, 206, 7, 21, 22, 214, 211, 201, 193, - 21, 22, 214, 211, 201, 8, 21, 22, 214, 211, 202, 101, 21, 22, 214, 211, - 205, 72, 21, 22, 203, 114, 21, 22, 203, 82, 21, 22, 203, 77, 21, 22, 203, - 57, 21, 22, 203, 49, 21, 22, 203, 166, 21, 22, 203, 161, 21, 22, 202, - 238, 21, 22, 193, 103, 202, 238, 21, 22, 202, 237, 21, 22, 193, 103, 202, - 237, 21, 22, 202, 236, 21, 22, 193, 103, 202, 236, 21, 22, 202, 235, 21, - 22, 193, 103, 202, 235, 21, 22, 202, 234, 21, 22, 193, 103, 202, 234, 21, - 22, 202, 241, 21, 22, 193, 103, 202, 241, 21, 22, 202, 240, 21, 22, 193, - 103, 202, 240, 21, 22, 203, 168, 21, 22, 192, 80, 21, 22, 192, 138, 21, - 22, 192, 33, 21, 22, 192, 23, 21, 22, 192, 12, 21, 22, 191, 246, 21, 22, - 192, 158, 191, 246, 21, 22, 170, 21, 22, 192, 140, 21, 22, 191, 181, 21, - 22, 193, 103, 191, 181, 21, 22, 191, 180, 21, 22, 193, 103, 191, 180, 21, - 22, 191, 179, 21, 22, 193, 103, 191, 179, 21, 22, 191, 178, 21, 22, 193, - 103, 191, 178, 21, 22, 191, 177, 21, 22, 193, 103, 191, 177, 21, 22, 191, - 183, 21, 22, 193, 103, 191, 183, 21, 22, 191, 182, 21, 22, 193, 103, 191, - 182, 21, 22, 214, 211, 191, 182, 21, 22, 192, 159, 21, 22, 248, 188, 192, - 159, 21, 22, 193, 103, 192, 159, 21, 22, 205, 102, 192, 33, 21, 22, 207, - 115, 21, 22, 207, 223, 207, 115, 21, 22, 193, 103, 219, 240, 21, 22, 207, - 179, 21, 22, 207, 2, 21, 22, 206, 196, 21, 22, 206, 163, 21, 22, 206, - 135, 21, 22, 193, 103, 219, 45, 21, 22, 165, 21, 22, 207, 180, 21, 22, - 193, 103, 173, 21, 22, 206, 26, 21, 22, 193, 103, 206, 26, 21, 22, 146, - 21, 22, 193, 103, 146, 21, 22, 216, 219, 146, 21, 22, 233, 61, 21, 22, - 233, 108, 21, 22, 233, 25, 21, 22, 233, 10, 21, 22, 232, 177, 21, 22, - 232, 164, 21, 22, 233, 111, 21, 22, 233, 110, 21, 22, 232, 63, 21, 22, - 193, 103, 232, 63, 21, 22, 233, 177, 21, 22, 199, 33, 21, 22, 215, 61, - 199, 33, 21, 22, 199, 11, 21, 22, 215, 61, 199, 11, 21, 22, 199, 5, 21, - 22, 215, 61, 199, 5, 21, 22, 198, 241, 21, 22, 198, 235, 21, 22, 199, 49, - 21, 22, 199, 48, 21, 22, 198, 203, 21, 22, 193, 103, 198, 203, 21, 22, - 199, 51, 21, 22, 197, 118, 21, 22, 197, 116, 21, 22, 197, 115, 21, 22, - 197, 120, 21, 22, 197, 121, 21, 22, 197, 2, 21, 22, 197, 1, 21, 22, 197, - 0, 21, 22, 197, 4, 21, 22, 212, 131, 229, 247, 21, 22, 212, 131, 229, - 160, 21, 22, 212, 131, 229, 132, 21, 22, 212, 131, 229, 25, 21, 22, 212, - 131, 228, 254, 21, 22, 212, 131, 140, 21, 22, 212, 131, 230, 93, 21, 22, - 212, 131, 230, 118, 21, 22, 212, 130, 230, 118, 21, 22, 229, 115, 21, 22, - 208, 94, 21, 22, 208, 59, 21, 22, 208, 53, 21, 22, 208, 47, 21, 22, 208, - 42, 21, 22, 208, 98, 21, 22, 208, 97, 21, 22, 208, 106, 21, 22, 198, 111, - 21, 22, 198, 109, 21, 22, 198, 108, 21, 22, 198, 112, 21, 22, 193, 103, - 207, 115, 21, 22, 193, 103, 207, 2, 21, 22, 193, 103, 206, 163, 21, 22, - 193, 103, 165, 21, 22, 214, 68, 21, 22, 214, 18, 21, 22, 214, 14, 21, 22, - 213, 251, 21, 22, 213, 246, 21, 22, 214, 70, 21, 22, 214, 69, 21, 22, - 214, 72, 21, 22, 213, 74, 21, 22, 205, 102, 203, 114, 21, 22, 205, 102, - 203, 82, 21, 22, 205, 102, 203, 57, 21, 22, 205, 102, 203, 166, 21, 22, - 193, 27, 199, 33, 21, 22, 193, 27, 199, 11, 21, 22, 193, 27, 198, 241, - 21, 22, 193, 27, 199, 49, 21, 22, 193, 27, 199, 51, 21, 22, 219, 155, 21, - 22, 219, 154, 21, 22, 219, 153, 21, 22, 219, 152, 21, 22, 219, 161, 21, - 22, 219, 160, 21, 22, 219, 162, 21, 22, 199, 50, 199, 33, 21, 22, 199, - 50, 199, 11, 21, 22, 199, 50, 199, 5, 21, 22, 199, 50, 198, 241, 21, 22, - 199, 50, 198, 235, 21, 22, 199, 50, 199, 49, 21, 22, 199, 50, 199, 48, - 21, 22, 199, 50, 199, 51, 21, 22, 251, 222, 250, 122, 21, 22, 248, 138, - 71, 21, 22, 248, 138, 68, 21, 22, 248, 138, 74, 21, 22, 248, 138, 65, 21, - 22, 248, 138, 193, 125, 21, 22, 248, 138, 193, 86, 21, 22, 248, 138, 193, - 48, 21, 22, 248, 138, 193, 190, 21, 22, 248, 138, 214, 123, 21, 22, 248, - 138, 213, 221, 21, 22, 248, 138, 213, 45, 21, 22, 248, 138, 181, 21, 22, - 248, 138, 222, 24, 21, 22, 248, 138, 221, 168, 21, 22, 248, 138, 221, 69, - 21, 22, 248, 138, 155, 21, 22, 205, 102, 229, 247, 21, 22, 205, 102, 229, - 160, 21, 22, 205, 102, 229, 25, 21, 22, 205, 102, 140, 21, 22, 98, 231, - 61, 21, 22, 98, 231, 65, 21, 22, 98, 231, 79, 21, 22, 98, 231, 78, 21, - 22, 98, 231, 67, 21, 22, 98, 231, 93, 21, 22, 98, 206, 69, 21, 22, 98, - 206, 163, 21, 22, 98, 207, 115, 21, 22, 98, 207, 86, 21, 22, 98, 207, 2, - 21, 22, 98, 165, 21, 22, 98, 193, 0, 21, 22, 98, 193, 48, 21, 22, 98, - 193, 125, 21, 22, 98, 193, 114, 21, 22, 98, 193, 86, 21, 22, 98, 193, - 190, 21, 22, 98, 228, 120, 21, 22, 98, 228, 121, 21, 22, 98, 228, 124, - 21, 22, 98, 228, 123, 21, 22, 98, 228, 122, 21, 22, 98, 228, 127, 21, 22, - 98, 198, 213, 21, 22, 98, 198, 241, 21, 22, 98, 199, 33, 21, 22, 98, 199, - 31, 21, 22, 98, 199, 11, 21, 22, 98, 199, 49, 21, 22, 98, 197, 99, 21, - 22, 98, 197, 109, 21, 22, 98, 197, 127, 21, 22, 98, 197, 126, 21, 22, 98, - 197, 111, 21, 22, 98, 197, 132, 21, 22, 98, 208, 167, 21, 22, 98, 209, - 75, 21, 22, 98, 210, 65, 21, 22, 98, 210, 51, 21, 22, 98, 209, 187, 21, - 22, 98, 168, 21, 22, 98, 210, 238, 21, 22, 98, 230, 181, 21, 22, 98, 231, - 5, 21, 22, 98, 231, 167, 21, 22, 98, 231, 159, 21, 22, 98, 231, 55, 21, - 22, 98, 231, 242, 21, 22, 98, 221, 177, 21, 22, 98, 221, 185, 21, 22, 98, - 221, 199, 21, 22, 98, 221, 198, 21, 22, 98, 221, 192, 21, 22, 98, 221, - 217, 21, 22, 98, 221, 98, 21, 22, 98, 221, 99, 21, 22, 98, 221, 102, 21, - 22, 98, 221, 101, 21, 22, 98, 221, 100, 21, 22, 98, 221, 103, 21, 22, 98, - 221, 104, 21, 22, 98, 212, 180, 21, 22, 98, 213, 45, 21, 22, 98, 214, - 123, 21, 22, 98, 214, 112, 21, 22, 98, 213, 221, 21, 22, 98, 181, 21, 22, - 98, 215, 157, 21, 22, 98, 216, 14, 21, 22, 98, 216, 234, 21, 22, 98, 216, - 213, 21, 22, 98, 216, 102, 21, 22, 98, 174, 21, 22, 98, 191, 225, 21, 22, - 98, 192, 12, 21, 22, 98, 192, 80, 21, 22, 98, 192, 77, 21, 22, 98, 192, - 33, 21, 22, 98, 170, 21, 22, 98, 222, 184, 21, 22, 205, 102, 222, 184, - 21, 22, 98, 222, 203, 21, 22, 98, 223, 12, 21, 22, 98, 223, 10, 21, 22, - 98, 222, 246, 21, 22, 205, 102, 222, 246, 21, 22, 98, 223, 34, 21, 22, - 98, 222, 217, 21, 22, 98, 222, 221, 21, 22, 98, 222, 231, 21, 22, 98, - 222, 230, 21, 22, 98, 222, 229, 21, 22, 98, 222, 232, 21, 22, 98, 218, - 227, 21, 22, 98, 219, 45, 21, 22, 98, 219, 240, 21, 22, 98, 219, 230, 21, - 22, 98, 219, 148, 21, 22, 98, 173, 21, 22, 98, 236, 181, 21, 22, 98, 236, - 182, 21, 22, 98, 236, 187, 21, 22, 98, 236, 186, 21, 22, 98, 236, 183, - 21, 22, 98, 236, 188, 21, 22, 98, 219, 151, 21, 22, 98, 219, 153, 21, 22, - 98, 219, 157, 21, 22, 98, 219, 156, 21, 22, 98, 219, 155, 21, 22, 98, - 219, 161, 21, 22, 98, 198, 106, 21, 22, 98, 198, 108, 21, 22, 98, 198, - 111, 21, 22, 98, 198, 110, 21, 22, 98, 198, 109, 21, 22, 98, 198, 112, - 21, 22, 98, 198, 101, 21, 22, 98, 198, 102, 21, 22, 98, 198, 114, 21, 22, - 98, 198, 113, 21, 22, 98, 198, 103, 21, 22, 98, 198, 115, 21, 22, 98, - 190, 251, 21, 22, 98, 191, 7, 21, 22, 98, 191, 87, 21, 22, 98, 191, 84, - 21, 22, 98, 191, 30, 21, 22, 98, 191, 123, 21, 22, 98, 191, 166, 21, 22, - 98, 89, 191, 166, 21, 22, 98, 235, 24, 21, 22, 98, 235, 25, 21, 22, 98, - 235, 34, 21, 22, 98, 235, 33, 21, 22, 98, 235, 28, 21, 22, 98, 235, 37, - 21, 22, 98, 201, 5, 21, 22, 98, 202, 47, 21, 22, 98, 205, 69, 21, 22, 98, - 205, 51, 21, 22, 98, 202, 223, 21, 22, 98, 188, 21, 22, 98, 203, 6, 21, - 22, 98, 203, 57, 21, 22, 98, 203, 114, 21, 22, 98, 203, 112, 21, 22, 98, - 203, 82, 21, 22, 98, 203, 166, 21, 22, 98, 203, 168, 21, 22, 98, 197, - 140, 21, 22, 98, 197, 144, 21, 22, 98, 197, 161, 21, 22, 98, 197, 160, - 21, 22, 98, 197, 146, 21, 22, 98, 197, 168, 21, 22, 98, 243, 50, 21, 22, - 98, 243, 70, 21, 22, 98, 243, 129, 21, 22, 98, 243, 125, 21, 22, 98, 243, - 97, 21, 22, 98, 247, 3, 21, 22, 98, 197, 102, 21, 22, 98, 197, 103, 21, - 22, 98, 197, 106, 21, 22, 98, 197, 105, 21, 22, 98, 197, 104, 21, 22, 98, - 197, 107, 21, 22, 243, 98, 56, 21, 22, 232, 82, 201, 64, 21, 22, 208, 90, - 21, 22, 214, 66, 21, 22, 213, 71, 21, 22, 213, 70, 21, 22, 213, 69, 21, - 22, 213, 68, 21, 22, 213, 73, 21, 22, 213, 72, 21, 22, 193, 27, 198, 201, - 21, 22, 193, 27, 198, 200, 21, 22, 193, 27, 198, 199, 21, 22, 193, 27, - 198, 198, 21, 22, 193, 27, 198, 197, 21, 22, 193, 27, 198, 204, 21, 22, - 193, 27, 198, 203, 21, 22, 193, 27, 53, 199, 51, 21, 22, 248, 138, 193, - 224, 211, 142, 201, 249, 77, 211, 142, 1, 248, 241, 211, 142, 1, 218, - 213, 211, 142, 1, 233, 58, 211, 142, 1, 205, 180, 211, 142, 1, 213, 168, - 211, 142, 1, 196, 165, 211, 142, 1, 238, 7, 211, 142, 1, 198, 139, 211, - 142, 1, 242, 88, 211, 142, 1, 247, 31, 211, 142, 1, 215, 139, 211, 142, - 1, 230, 236, 211, 142, 1, 214, 56, 211, 142, 1, 201, 55, 211, 142, 1, - 206, 56, 211, 142, 1, 251, 234, 211, 142, 1, 211, 93, 211, 142, 1, 196, - 62, 211, 142, 1, 234, 217, 211, 142, 1, 223, 89, 211, 142, 1, 234, 218, - 211, 142, 1, 211, 58, 211, 142, 1, 196, 136, 211, 142, 1, 223, 207, 211, - 142, 1, 234, 215, 211, 142, 1, 210, 40, 211, 142, 233, 57, 77, 211, 142, - 207, 19, 233, 57, 77, 206, 45, 1, 233, 47, 233, 38, 233, 62, 233, 177, - 206, 45, 1, 196, 12, 206, 45, 1, 196, 47, 196, 63, 66, 206, 45, 1, 191, - 228, 206, 45, 1, 192, 159, 206, 45, 1, 193, 224, 206, 45, 1, 198, 206, - 198, 205, 198, 233, 206, 45, 1, 233, 250, 206, 45, 1, 251, 92, 65, 206, - 45, 1, 211, 39, 74, 206, 45, 1, 252, 66, 65, 206, 45, 1, 252, 11, 206, - 45, 1, 219, 17, 74, 206, 45, 1, 203, 34, 74, 206, 45, 1, 74, 206, 45, 1, - 211, 153, 206, 45, 1, 211, 106, 206, 45, 1, 207, 156, 207, 171, 207, 70, - 146, 206, 45, 1, 222, 41, 206, 45, 1, 247, 27, 206, 45, 1, 222, 42, 222, - 154, 206, 45, 1, 232, 53, 206, 45, 1, 234, 90, 206, 45, 1, 231, 162, 230, - 124, 232, 53, 206, 45, 1, 231, 202, 206, 45, 1, 192, 248, 192, 239, 193, - 224, 206, 45, 1, 230, 84, 230, 118, 206, 45, 1, 230, 88, 230, 118, 206, - 45, 1, 219, 19, 230, 118, 206, 45, 1, 203, 37, 230, 118, 206, 45, 1, 214, - 205, 212, 93, 214, 206, 215, 63, 206, 45, 1, 203, 35, 215, 63, 206, 45, - 1, 235, 137, 206, 45, 1, 223, 67, 223, 71, 223, 57, 68, 206, 45, 1, 71, - 206, 45, 1, 223, 1, 223, 37, 206, 45, 1, 231, 143, 206, 45, 1, 219, 20, - 252, 27, 206, 45, 1, 203, 39, 65, 206, 45, 1, 223, 49, 234, 63, 206, 45, - 1, 209, 249, 210, 20, 210, 238, 206, 45, 1, 251, 187, 234, 61, 206, 45, - 1, 201, 255, 206, 9, 206, 45, 1, 202, 199, 219, 16, 206, 9, 206, 45, 1, - 203, 33, 206, 9, 206, 45, 1, 247, 195, 206, 45, 1, 191, 166, 206, 45, 1, - 198, 120, 198, 132, 196, 242, 200, 43, 206, 45, 1, 203, 32, 200, 43, 206, - 45, 1, 238, 129, 206, 45, 1, 248, 219, 248, 222, 248, 144, 250, 122, 206, - 45, 1, 203, 38, 250, 122, 206, 45, 1, 235, 136, 206, 45, 1, 211, 72, 206, - 45, 1, 234, 169, 234, 176, 71, 206, 45, 1, 217, 81, 217, 93, 218, 170, - 206, 45, 1, 219, 18, 218, 170, 206, 45, 1, 203, 36, 218, 170, 206, 45, 1, - 219, 255, 220, 103, 219, 28, 172, 206, 45, 1, 235, 138, 206, 45, 1, 223, - 137, 206, 45, 1, 223, 138, 206, 45, 1, 238, 21, 238, 27, 238, 129, 206, - 45, 1, 211, 30, 233, 249, 74, 206, 45, 1, 234, 213, 206, 45, 1, 223, 87, - 206, 45, 1, 238, 150, 206, 45, 1, 247, 145, 206, 45, 1, 247, 43, 206, 45, - 1, 201, 109, 206, 45, 1, 219, 15, 206, 45, 1, 203, 31, 206, 45, 1, 228, - 27, 206, 45, 1, 208, 106, 206, 45, 1, 192, 235, 206, 45, 202, 171, 208, - 153, 206, 45, 215, 131, 208, 153, 206, 45, 238, 220, 208, 153, 206, 45, - 250, 254, 113, 206, 45, 197, 45, 113, 206, 45, 248, 239, 113, 206, 45, 1, - 222, 154, 206, 45, 1, 203, 168, 206, 45, 1, 211, 89, 206, 45, 1, 232, - 112, 247, 83, 211, 38, 206, 45, 1, 232, 112, 247, 83, 223, 70, 206, 45, - 1, 232, 112, 247, 83, 234, 175, 206, 45, 1, 232, 112, 247, 83, 252, 65, - 206, 45, 1, 232, 112, 247, 83, 252, 11, 199, 222, 1, 65, 199, 222, 1, 68, - 199, 222, 1, 66, 199, 222, 1, 155, 199, 222, 1, 231, 242, 199, 222, 1, - 214, 70, 199, 222, 1, 190, 190, 199, 222, 1, 238, 34, 199, 222, 1, 181, - 199, 222, 1, 168, 199, 222, 1, 249, 155, 199, 222, 1, 174, 199, 222, 1, - 170, 199, 222, 1, 165, 199, 222, 1, 173, 199, 222, 1, 193, 190, 199, 222, - 1, 188, 199, 222, 1, 140, 199, 222, 18, 3, 68, 199, 222, 18, 3, 66, 199, - 222, 3, 195, 40, 199, 222, 3, 210, 171, 199, 222, 1, 251, 16, 165, 230, - 25, 1, 65, 230, 25, 1, 68, 230, 25, 1, 66, 230, 25, 1, 155, 230, 25, 1, - 231, 242, 230, 25, 1, 214, 70, 230, 25, 1, 190, 190, 230, 25, 1, 238, 34, - 230, 25, 1, 181, 230, 25, 1, 168, 230, 25, 1, 249, 155, 230, 25, 1, 174, - 230, 25, 1, 170, 230, 25, 1, 165, 230, 25, 1, 173, 230, 25, 1, 193, 190, - 230, 25, 1, 188, 230, 25, 1, 140, 230, 25, 18, 3, 68, 230, 25, 18, 3, 66, - 230, 25, 3, 210, 171, 209, 206, 202, 171, 208, 153, 209, 206, 55, 208, - 153, 248, 2, 1, 65, 248, 2, 1, 68, 248, 2, 1, 66, 248, 2, 1, 155, 248, 2, - 1, 231, 242, 248, 2, 1, 214, 70, 248, 2, 1, 190, 190, 248, 2, 1, 238, 34, - 248, 2, 1, 181, 248, 2, 1, 168, 248, 2, 1, 249, 155, 248, 2, 1, 174, 248, - 2, 1, 170, 248, 2, 1, 165, 248, 2, 1, 173, 248, 2, 1, 193, 190, 248, 2, - 1, 188, 248, 2, 1, 140, 248, 2, 18, 3, 68, 248, 2, 18, 3, 66, 199, 221, - 1, 65, 199, 221, 1, 68, 199, 221, 1, 66, 199, 221, 1, 155, 199, 221, 1, - 231, 242, 199, 221, 1, 214, 70, 199, 221, 1, 190, 190, 199, 221, 1, 238, - 34, 199, 221, 1, 181, 199, 221, 1, 168, 199, 221, 1, 249, 155, 199, 221, - 1, 174, 199, 221, 1, 170, 199, 221, 1, 173, 199, 221, 1, 193, 190, 199, - 221, 1, 188, 199, 221, 18, 3, 68, 199, 221, 18, 3, 66, 95, 1, 155, 95, 1, - 221, 217, 95, 1, 221, 69, 95, 1, 221, 185, 95, 1, 213, 251, 95, 1, 247, - 162, 95, 1, 247, 3, 95, 1, 242, 101, 95, 1, 243, 70, 95, 1, 212, 67, 95, - 1, 238, 34, 95, 1, 197, 120, 95, 1, 236, 176, 95, 1, 197, 115, 95, 1, - 213, 51, 95, 1, 190, 190, 95, 1, 199, 49, 95, 1, 159, 95, 1, 198, 241, - 95, 1, 213, 45, 95, 1, 249, 155, 95, 1, 209, 230, 95, 1, 209, 75, 95, 1, - 209, 201, 95, 1, 216, 14, 95, 1, 192, 12, 95, 1, 206, 163, 95, 1, 219, - 45, 95, 1, 195, 24, 95, 1, 203, 166, 95, 1, 201, 135, 95, 1, 188, 95, 1, - 140, 95, 1, 173, 95, 1, 208, 98, 95, 223, 151, 18, 208, 84, 95, 223, 151, - 18, 208, 97, 95, 223, 151, 18, 208, 59, 95, 223, 151, 18, 208, 53, 95, - 223, 151, 18, 208, 35, 95, 223, 151, 18, 208, 2, 95, 223, 151, 18, 207, - 246, 95, 223, 151, 18, 207, 245, 95, 223, 151, 18, 206, 18, 95, 223, 151, - 18, 206, 11, 95, 223, 151, 18, 218, 185, 95, 223, 151, 18, 218, 173, 95, - 223, 151, 18, 208, 77, 95, 223, 151, 18, 208, 90, 95, 223, 151, 18, 208, - 43, 196, 255, 107, 95, 223, 151, 18, 208, 43, 196, 255, 109, 95, 223, - 151, 18, 208, 79, 95, 18, 223, 135, 251, 39, 95, 18, 223, 135, 252, 208, - 95, 18, 3, 252, 208, 95, 18, 3, 68, 95, 18, 3, 223, 201, 95, 18, 3, 192, - 159, 95, 18, 3, 191, 176, 95, 18, 3, 66, 95, 18, 3, 196, 30, 95, 18, 3, - 196, 168, 95, 18, 3, 211, 153, 95, 18, 3, 170, 95, 18, 3, 223, 228, 95, - 18, 3, 71, 95, 18, 3, 252, 27, 95, 18, 3, 251, 238, 95, 18, 3, 211, 89, - 95, 18, 3, 250, 165, 95, 3, 213, 186, 95, 3, 207, 108, 95, 3, 191, 187, - 95, 3, 215, 93, 95, 3, 197, 229, 95, 3, 249, 92, 95, 3, 206, 152, 95, 3, - 198, 90, 95, 3, 222, 97, 95, 3, 251, 240, 95, 3, 205, 144, 205, 136, 95, - 3, 195, 37, 95, 3, 242, 92, 95, 3, 249, 62, 95, 3, 221, 207, 95, 3, 249, - 87, 95, 3, 247, 133, 209, 148, 220, 188, 95, 3, 219, 207, 198, 59, 95, 3, - 248, 207, 95, 3, 209, 203, 215, 150, 95, 3, 221, 41, 95, 238, 172, 16, - 206, 242, 95, 3, 250, 146, 95, 3, 250, 168, 95, 17, 191, 77, 95, 17, 107, - 95, 17, 109, 95, 17, 138, 95, 17, 134, 95, 17, 150, 95, 17, 169, 95, 17, - 175, 95, 17, 171, 95, 17, 178, 95, 16, 219, 207, 250, 170, 202, 20, 95, - 16, 219, 207, 250, 170, 215, 114, 95, 16, 219, 207, 250, 170, 209, 147, - 95, 16, 219, 207, 250, 170, 248, 242, 95, 16, 219, 207, 250, 170, 247, - 238, 95, 16, 219, 207, 250, 170, 208, 247, 95, 16, 219, 207, 250, 170, - 208, 241, 95, 16, 219, 207, 250, 170, 208, 239, 95, 16, 219, 207, 250, - 170, 208, 245, 95, 16, 219, 207, 250, 170, 208, 243, 104, 248, 160, 104, - 234, 122, 104, 242, 76, 104, 232, 82, 201, 64, 104, 242, 85, 104, 232, - 130, 236, 140, 104, 198, 88, 202, 33, 228, 90, 104, 202, 215, 5, 248, 75, - 217, 53, 104, 217, 89, 242, 76, 104, 217, 89, 232, 82, 201, 64, 104, 213, - 166, 104, 232, 111, 67, 205, 36, 107, 104, 232, 111, 67, 205, 36, 109, - 104, 232, 111, 67, 205, 36, 138, 104, 18, 204, 11, 104, 232, 111, 67, - 205, 36, 134, 104, 17, 191, 77, 104, 17, 107, 104, 17, 109, 104, 17, 138, - 104, 17, 134, 104, 17, 150, 104, 17, 169, 104, 17, 175, 104, 17, 171, - 104, 17, 178, 104, 1, 65, 104, 1, 71, 104, 1, 68, 104, 1, 74, 104, 1, 66, - 104, 1, 211, 153, 104, 1, 196, 152, 104, 1, 234, 190, 104, 1, 181, 104, - 1, 251, 124, 104, 1, 249, 155, 104, 1, 168, 104, 1, 208, 98, 104, 1, 231, - 242, 104, 1, 174, 104, 1, 173, 104, 1, 188, 104, 1, 203, 166, 104, 1, - 190, 190, 104, 1, 238, 34, 104, 1, 247, 3, 104, 1, 223, 34, 104, 1, 170, - 104, 1, 165, 104, 1, 193, 190, 104, 1, 233, 111, 104, 1, 155, 104, 1, - 221, 217, 104, 1, 197, 168, 104, 1, 191, 123, 104, 1, 230, 93, 104, 1, - 190, 255, 104, 1, 219, 161, 104, 1, 191, 57, 104, 1, 243, 97, 104, 1, - 198, 88, 180, 18, 56, 104, 1, 198, 88, 71, 104, 1, 198, 88, 68, 104, 1, - 198, 88, 74, 104, 1, 198, 88, 66, 104, 1, 198, 88, 211, 153, 104, 1, 198, - 88, 196, 152, 104, 1, 198, 88, 251, 124, 104, 1, 198, 88, 249, 155, 104, - 1, 198, 88, 168, 104, 1, 198, 88, 208, 98, 104, 1, 198, 88, 231, 242, - 104, 1, 198, 88, 174, 104, 1, 198, 88, 190, 190, 104, 1, 198, 88, 238, - 34, 104, 1, 198, 88, 247, 3, 104, 1, 198, 88, 223, 34, 104, 1, 198, 88, - 197, 168, 104, 1, 198, 88, 170, 104, 1, 198, 88, 193, 190, 104, 1, 198, - 88, 155, 104, 1, 198, 88, 231, 239, 104, 1, 198, 88, 230, 93, 104, 1, - 198, 88, 222, 245, 104, 1, 198, 88, 213, 211, 104, 1, 198, 88, 235, 37, - 104, 1, 202, 215, 71, 104, 1, 202, 215, 68, 104, 1, 202, 215, 223, 46, - 104, 1, 202, 215, 196, 152, 104, 1, 202, 215, 66, 104, 1, 202, 215, 251, - 124, 104, 1, 202, 215, 155, 104, 1, 202, 215, 231, 242, 104, 1, 202, 215, - 140, 104, 1, 202, 215, 168, 104, 1, 202, 215, 203, 166, 104, 1, 202, 215, - 190, 190, 104, 1, 202, 215, 238, 34, 104, 1, 202, 215, 223, 34, 104, 1, - 202, 215, 233, 111, 104, 1, 202, 215, 231, 239, 104, 1, 202, 215, 230, - 93, 104, 1, 202, 215, 197, 168, 104, 1, 202, 215, 191, 123, 104, 1, 202, - 215, 207, 180, 104, 1, 202, 215, 247, 3, 104, 1, 202, 215, 191, 71, 104, - 1, 217, 89, 68, 104, 1, 217, 89, 155, 104, 1, 217, 89, 165, 104, 1, 217, - 89, 233, 111, 104, 1, 217, 89, 191, 71, 104, 1, 247, 4, 4, 105, 236, 140, - 104, 1, 251, 186, 231, 222, 251, 74, 107, 104, 1, 251, 186, 231, 222, - 195, 36, 107, 104, 1, 251, 186, 231, 222, 237, 249, 104, 1, 251, 186, - 231, 222, 196, 163, 104, 1, 251, 186, 231, 222, 223, 95, 196, 163, 104, - 1, 251, 186, 231, 222, 249, 106, 104, 1, 251, 186, 231, 222, 115, 249, - 106, 104, 1, 251, 186, 231, 222, 65, 104, 1, 251, 186, 231, 222, 68, 104, - 1, 251, 186, 231, 222, 155, 104, 1, 251, 186, 231, 222, 214, 70, 104, 1, - 251, 186, 231, 222, 247, 162, 104, 1, 251, 186, 231, 222, 197, 132, 104, - 1, 251, 186, 231, 222, 197, 120, 104, 1, 251, 186, 231, 222, 237, 193, - 104, 1, 251, 186, 231, 222, 213, 81, 104, 1, 251, 186, 231, 222, 190, - 190, 104, 1, 251, 186, 231, 222, 238, 34, 104, 1, 251, 186, 231, 222, - 168, 104, 1, 251, 186, 231, 222, 209, 230, 104, 1, 251, 186, 231, 222, - 201, 176, 104, 1, 251, 186, 231, 222, 191, 71, 104, 1, 251, 186, 231, - 222, 191, 123, 104, 1, 251, 186, 231, 222, 251, 247, 104, 1, 198, 88, - 251, 186, 231, 222, 190, 190, 104, 1, 198, 88, 251, 186, 231, 222, 191, - 71, 104, 1, 217, 89, 251, 186, 231, 222, 231, 93, 104, 1, 217, 89, 251, - 186, 231, 222, 214, 70, 104, 1, 217, 89, 251, 186, 231, 222, 247, 162, - 104, 1, 217, 89, 251, 186, 231, 222, 222, 254, 104, 1, 217, 89, 251, 186, - 231, 222, 197, 132, 104, 1, 217, 89, 251, 186, 231, 222, 237, 177, 104, - 1, 217, 89, 251, 186, 231, 222, 190, 190, 104, 1, 217, 89, 251, 186, 231, - 222, 237, 70, 104, 1, 217, 89, 251, 186, 231, 222, 201, 176, 104, 1, 217, - 89, 251, 186, 231, 222, 238, 144, 104, 1, 217, 89, 251, 186, 231, 222, - 191, 71, 104, 1, 217, 89, 251, 186, 231, 222, 191, 123, 104, 1, 251, 186, - 231, 222, 132, 66, 104, 1, 251, 186, 231, 222, 132, 170, 104, 1, 217, 89, - 251, 186, 231, 222, 248, 205, 104, 1, 251, 186, 231, 222, 238, 22, 104, - 1, 217, 89, 251, 186, 231, 222, 219, 161, 21, 22, 210, 244, 21, 22, 250, - 133, 21, 22, 252, 162, 21, 22, 193, 128, 21, 22, 208, 253, 21, 22, 210, - 74, 21, 22, 208, 115, 21, 22, 199, 154, 21, 22, 222, 31, 21, 22, 220, - 178, 21, 22, 217, 23, 21, 22, 212, 252, 21, 22, 214, 200, 21, 22, 219, - 250, 21, 22, 201, 253, 21, 22, 205, 104, 21, 22, 203, 19, 21, 22, 203, - 118, 21, 22, 202, 233, 21, 22, 191, 234, 21, 22, 192, 86, 21, 22, 207, - 124, 21, 22, 212, 109, 21, 22, 211, 130, 212, 109, 21, 22, 212, 108, 21, - 22, 211, 130, 212, 108, 21, 22, 212, 107, 21, 22, 211, 130, 212, 107, 21, - 22, 212, 106, 21, 22, 211, 130, 212, 106, 21, 22, 206, 23, 21, 22, 206, - 22, 21, 22, 206, 21, 21, 22, 206, 20, 21, 22, 206, 19, 21, 22, 206, 27, - 21, 22, 211, 130, 210, 238, 21, 22, 211, 130, 200, 43, 21, 22, 211, 130, - 222, 154, 21, 22, 211, 130, 247, 195, 21, 22, 211, 130, 218, 170, 21, 22, - 211, 130, 215, 63, 21, 22, 211, 130, 206, 9, 21, 22, 211, 130, 203, 168, - 21, 22, 234, 204, 193, 224, 21, 22, 193, 102, 193, 224, 21, 22, 53, 2, - 206, 189, 21, 22, 53, 207, 149, 236, 142, 21, 22, 207, 223, 206, 24, 21, - 22, 193, 103, 219, 10, 21, 22, 193, 103, 220, 127, 21, 22, 198, 202, 21, - 22, 198, 204, 21, 22, 197, 112, 21, 22, 197, 114, 21, 22, 197, 119, 21, - 22, 198, 105, 21, 22, 198, 107, 21, 22, 205, 102, 202, 238, 21, 22, 205, - 102, 203, 49, 21, 22, 205, 102, 228, 254, 21, 22, 98, 230, 132, 21, 22, - 98, 237, 105, 231, 159, 21, 22, 98, 231, 239, 21, 22, 98, 230, 137, 21, - 22, 205, 102, 222, 164, 21, 22, 98, 222, 162, 21, 22, 249, 7, 237, 105, - 172, 21, 22, 249, 7, 237, 105, 146, 21, 22, 98, 237, 100, 206, 9, 219, - 124, 195, 1, 219, 177, 219, 124, 1, 155, 219, 124, 1, 221, 217, 219, 124, - 1, 231, 242, 219, 124, 1, 231, 93, 219, 124, 1, 214, 70, 219, 124, 1, - 247, 162, 219, 124, 1, 247, 3, 219, 124, 1, 223, 34, 219, 124, 1, 222, - 254, 219, 124, 1, 192, 108, 219, 124, 1, 190, 190, 219, 124, 1, 199, 49, - 219, 124, 1, 238, 34, 219, 124, 1, 237, 70, 219, 124, 1, 181, 219, 124, - 1, 168, 219, 124, 1, 209, 230, 219, 124, 1, 249, 155, 219, 124, 1, 248, - 205, 219, 124, 1, 174, 219, 124, 1, 170, 219, 124, 1, 165, 219, 124, 1, - 173, 219, 124, 1, 193, 190, 219, 124, 1, 203, 166, 219, 124, 1, 201, 176, - 219, 124, 1, 188, 219, 124, 1, 140, 219, 124, 1, 230, 128, 219, 124, 1, - 198, 26, 219, 124, 18, 3, 65, 219, 124, 18, 3, 68, 219, 124, 18, 3, 66, - 219, 124, 18, 3, 234, 190, 219, 124, 18, 3, 251, 238, 219, 124, 18, 3, - 211, 89, 219, 124, 18, 3, 250, 165, 219, 124, 18, 3, 71, 219, 124, 18, 3, - 74, 219, 124, 200, 240, 1, 170, 219, 124, 200, 240, 1, 165, 219, 124, - 200, 240, 1, 193, 190, 219, 124, 2, 1, 155, 219, 124, 2, 1, 214, 70, 219, - 124, 2, 1, 251, 73, 219, 124, 2, 1, 190, 190, 219, 124, 2, 1, 181, 219, - 124, 2, 1, 168, 219, 124, 2, 1, 174, 219, 124, 2, 1, 165, 219, 124, 2, 1, - 173, 219, 124, 3, 215, 136, 219, 124, 3, 222, 3, 219, 124, 3, 205, 199, - 219, 124, 3, 219, 10, 219, 124, 233, 218, 77, 219, 124, 208, 15, 77, 219, - 124, 17, 191, 77, 219, 124, 17, 107, 219, 124, 17, 109, 219, 124, 17, - 138, 219, 124, 17, 134, 219, 124, 17, 150, 219, 124, 17, 169, 219, 124, - 17, 175, 219, 124, 17, 171, 219, 124, 17, 178, 54, 219, 241, 1, 155, 54, - 219, 241, 1, 192, 220, 54, 219, 241, 1, 214, 70, 54, 219, 241, 1, 197, - 168, 54, 219, 241, 1, 188, 54, 219, 241, 1, 170, 54, 219, 241, 1, 190, - 190, 54, 219, 241, 1, 199, 49, 54, 219, 241, 1, 173, 54, 219, 241, 1, - 168, 54, 219, 241, 1, 209, 230, 54, 219, 241, 1, 174, 54, 219, 241, 1, - 233, 111, 54, 219, 241, 1, 195, 188, 54, 219, 241, 1, 140, 54, 219, 241, - 1, 208, 98, 54, 219, 241, 1, 221, 217, 54, 219, 241, 1, 197, 157, 54, - 219, 241, 1, 181, 54, 219, 241, 1, 65, 54, 219, 241, 1, 68, 54, 219, 241, - 1, 234, 190, 54, 219, 241, 1, 234, 175, 54, 219, 241, 1, 66, 54, 219, - 241, 1, 211, 89, 54, 219, 241, 1, 74, 54, 219, 241, 1, 196, 152, 54, 219, - 241, 1, 71, 54, 219, 241, 1, 250, 163, 54, 219, 241, 1, 251, 238, 54, - 219, 241, 1, 198, 77, 54, 219, 241, 1, 198, 76, 54, 219, 241, 1, 198, 75, - 54, 219, 241, 1, 198, 74, 54, 219, 241, 1, 198, 73, 214, 82, 54, 218, - 221, 1, 137, 208, 98, 214, 82, 54, 218, 221, 1, 130, 208, 98, 214, 82, - 54, 218, 221, 1, 137, 155, 214, 82, 54, 218, 221, 1, 137, 192, 220, 214, - 82, 54, 218, 221, 1, 137, 214, 70, 214, 82, 54, 218, 221, 1, 130, 155, - 214, 82, 54, 218, 221, 1, 130, 192, 220, 214, 82, 54, 218, 221, 1, 130, - 214, 70, 214, 82, 54, 218, 221, 1, 137, 197, 168, 214, 82, 54, 218, 221, - 1, 137, 188, 214, 82, 54, 218, 221, 1, 137, 170, 214, 82, 54, 218, 221, - 1, 130, 197, 168, 214, 82, 54, 218, 221, 1, 130, 188, 214, 82, 54, 218, - 221, 1, 130, 170, 214, 82, 54, 218, 221, 1, 137, 190, 190, 214, 82, 54, - 218, 221, 1, 137, 199, 49, 214, 82, 54, 218, 221, 1, 137, 181, 214, 82, - 54, 218, 221, 1, 130, 190, 190, 214, 82, 54, 218, 221, 1, 130, 199, 49, - 214, 82, 54, 218, 221, 1, 130, 181, 214, 82, 54, 218, 221, 1, 137, 168, - 214, 82, 54, 218, 221, 1, 137, 209, 230, 214, 82, 54, 218, 221, 1, 137, - 174, 214, 82, 54, 218, 221, 1, 130, 168, 214, 82, 54, 218, 221, 1, 130, - 209, 230, 214, 82, 54, 218, 221, 1, 130, 174, 214, 82, 54, 218, 221, 1, - 137, 233, 111, 214, 82, 54, 218, 221, 1, 137, 195, 188, 214, 82, 54, 218, - 221, 1, 137, 173, 214, 82, 54, 218, 221, 1, 130, 233, 111, 214, 82, 54, - 218, 221, 1, 130, 195, 188, 214, 82, 54, 218, 221, 1, 130, 173, 214, 82, - 54, 218, 221, 1, 137, 140, 214, 82, 54, 218, 221, 1, 137, 238, 34, 214, - 82, 54, 218, 221, 1, 137, 249, 155, 214, 82, 54, 218, 221, 1, 130, 140, - 214, 82, 54, 218, 221, 1, 130, 238, 34, 214, 82, 54, 218, 221, 1, 130, - 249, 155, 214, 82, 54, 218, 221, 1, 137, 220, 183, 214, 82, 54, 218, 221, - 1, 137, 192, 185, 214, 82, 54, 218, 221, 1, 130, 220, 183, 214, 82, 54, - 218, 221, 1, 130, 192, 185, 214, 82, 54, 218, 221, 1, 137, 200, 252, 214, - 82, 54, 218, 221, 1, 130, 200, 252, 214, 82, 54, 218, 221, 18, 3, 18, - 203, 29, 214, 82, 54, 218, 221, 18, 3, 252, 208, 214, 82, 54, 218, 221, - 18, 3, 223, 201, 214, 82, 54, 218, 221, 18, 3, 66, 214, 82, 54, 218, 221, - 18, 3, 196, 30, 214, 82, 54, 218, 221, 18, 3, 71, 214, 82, 54, 218, 221, - 18, 3, 252, 27, 214, 82, 54, 218, 221, 18, 3, 74, 214, 82, 54, 218, 221, - 18, 3, 211, 184, 214, 82, 54, 218, 221, 18, 3, 196, 152, 214, 82, 54, - 218, 221, 18, 3, 250, 133, 214, 82, 54, 218, 221, 18, 3, 252, 162, 214, - 82, 54, 218, 221, 18, 3, 196, 21, 214, 82, 54, 218, 221, 18, 3, 210, 244, - 214, 82, 54, 218, 221, 18, 3, 211, 181, 214, 82, 54, 218, 221, 18, 3, - 196, 144, 214, 82, 54, 218, 221, 18, 3, 223, 46, 214, 82, 54, 218, 221, - 1, 53, 196, 12, 214, 82, 54, 218, 221, 1, 53, 214, 72, 214, 82, 54, 218, - 221, 1, 53, 215, 63, 214, 82, 54, 218, 221, 1, 53, 218, 170, 214, 82, 54, - 218, 221, 1, 53, 222, 154, 214, 82, 54, 218, 221, 1, 53, 238, 129, 214, - 82, 54, 218, 221, 1, 53, 250, 122, 214, 82, 54, 218, 221, 163, 217, 57, - 214, 82, 54, 218, 221, 163, 217, 56, 214, 82, 54, 218, 221, 17, 191, 77, - 214, 82, 54, 218, 221, 17, 107, 214, 82, 54, 218, 221, 17, 109, 214, 82, - 54, 218, 221, 17, 138, 214, 82, 54, 218, 221, 17, 134, 214, 82, 54, 218, - 221, 17, 150, 214, 82, 54, 218, 221, 17, 169, 214, 82, 54, 218, 221, 17, - 175, 214, 82, 54, 218, 221, 17, 171, 214, 82, 54, 218, 221, 17, 178, 214, - 82, 54, 218, 221, 128, 17, 107, 214, 82, 54, 218, 221, 3, 220, 109, 214, - 82, 54, 218, 221, 3, 220, 108, 95, 16, 210, 86, 95, 16, 215, 115, 221, - 60, 95, 16, 209, 148, 221, 60, 95, 16, 248, 243, 221, 60, 95, 16, 247, - 239, 221, 60, 95, 16, 208, 248, 221, 60, 95, 16, 208, 242, 221, 60, 95, - 16, 208, 240, 221, 60, 95, 16, 208, 246, 221, 60, 95, 16, 208, 244, 221, - 60, 95, 16, 237, 234, 221, 60, 95, 16, 237, 230, 221, 60, 95, 16, 237, - 229, 221, 60, 95, 16, 237, 232, 221, 60, 95, 16, 237, 231, 221, 60, 95, - 16, 237, 228, 221, 60, 95, 16, 197, 51, 95, 16, 215, 115, 206, 150, 95, - 16, 209, 148, 206, 150, 95, 16, 248, 243, 206, 150, 95, 16, 247, 239, - 206, 150, 95, 16, 208, 248, 206, 150, 95, 16, 208, 242, 206, 150, 95, 16, - 208, 240, 206, 150, 95, 16, 208, 246, 206, 150, 95, 16, 208, 244, 206, - 150, 95, 16, 237, 234, 206, 150, 95, 16, 237, 230, 206, 150, 95, 16, 237, - 229, 206, 150, 95, 16, 237, 232, 206, 150, 95, 16, 237, 231, 206, 150, - 95, 16, 237, 228, 206, 150, 248, 3, 1, 155, 248, 3, 1, 231, 242, 248, 3, - 1, 214, 70, 248, 3, 1, 214, 13, 248, 3, 1, 168, 248, 3, 1, 249, 155, 248, - 3, 1, 174, 248, 3, 1, 215, 168, 248, 3, 1, 190, 190, 248, 3, 1, 238, 34, - 248, 3, 1, 181, 248, 3, 1, 212, 246, 248, 3, 1, 247, 162, 248, 3, 1, 223, - 34, 248, 3, 1, 212, 103, 248, 3, 1, 212, 94, 248, 3, 1, 170, 248, 3, 1, - 165, 248, 3, 1, 173, 248, 3, 1, 195, 188, 248, 3, 1, 188, 248, 3, 1, 65, - 248, 3, 1, 140, 248, 3, 18, 3, 68, 248, 3, 18, 3, 66, 248, 3, 18, 3, 71, - 248, 3, 18, 3, 74, 248, 3, 18, 3, 252, 27, 248, 3, 210, 186, 248, 3, 234, - 97, 79, 205, 54, 54, 128, 1, 137, 155, 54, 128, 1, 137, 221, 217, 54, - 128, 1, 137, 220, 167, 54, 128, 1, 130, 155, 54, 128, 1, 130, 220, 167, - 54, 128, 1, 130, 221, 217, 54, 128, 1, 214, 70, 54, 128, 1, 137, 247, - 162, 54, 128, 1, 137, 247, 3, 54, 128, 1, 130, 247, 162, 54, 128, 1, 130, - 188, 54, 128, 1, 130, 247, 3, 54, 128, 1, 212, 103, 54, 128, 1, 207, 131, - 54, 128, 1, 137, 207, 129, 54, 128, 1, 238, 34, 54, 128, 1, 130, 207, - 129, 54, 128, 1, 207, 140, 54, 128, 1, 137, 190, 190, 54, 128, 1, 137, - 199, 49, 54, 128, 1, 130, 190, 190, 54, 128, 1, 130, 199, 49, 54, 128, 1, - 181, 54, 128, 1, 249, 155, 54, 128, 1, 137, 168, 54, 128, 1, 137, 209, - 230, 54, 128, 1, 137, 233, 111, 54, 128, 1, 130, 168, 54, 128, 1, 130, - 233, 111, 54, 128, 1, 130, 209, 230, 54, 128, 1, 174, 54, 128, 1, 130, - 170, 54, 128, 1, 137, 170, 54, 128, 1, 165, 54, 128, 1, 206, 58, 54, 128, - 1, 173, 54, 128, 1, 218, 220, 54, 128, 1, 193, 190, 54, 128, 1, 137, 203, - 166, 54, 128, 1, 137, 201, 176, 54, 128, 1, 137, 188, 54, 128, 1, 137, - 140, 54, 128, 1, 219, 75, 54, 128, 1, 65, 54, 128, 1, 130, 140, 54, 128, - 1, 68, 54, 128, 1, 223, 201, 54, 128, 1, 66, 54, 128, 1, 196, 30, 54, - 128, 1, 234, 190, 54, 128, 1, 211, 89, 54, 128, 1, 220, 109, 54, 128, 1, - 230, 208, 188, 54, 128, 120, 3, 216, 219, 165, 54, 128, 120, 3, 216, 219, - 173, 54, 128, 120, 3, 220, 128, 199, 190, 220, 98, 54, 128, 3, 217, 115, - 222, 86, 220, 98, 54, 128, 120, 3, 53, 214, 70, 54, 128, 120, 3, 130, - 168, 54, 128, 120, 3, 137, 207, 130, 211, 59, 130, 168, 54, 128, 120, 3, - 174, 54, 128, 120, 3, 249, 155, 54, 128, 120, 3, 188, 54, 128, 3, 205, - 173, 54, 128, 18, 3, 65, 54, 128, 18, 3, 217, 115, 205, 123, 54, 128, 18, - 3, 252, 208, 54, 128, 18, 3, 199, 200, 252, 208, 54, 128, 18, 3, 68, 54, - 128, 18, 3, 223, 201, 54, 128, 18, 3, 196, 152, 54, 128, 18, 3, 196, 29, - 54, 128, 18, 3, 66, 54, 128, 18, 3, 196, 30, 54, 128, 18, 3, 74, 54, 128, - 18, 3, 211, 185, 60, 54, 128, 18, 3, 210, 244, 54, 128, 18, 3, 71, 54, - 128, 18, 3, 252, 27, 54, 128, 18, 3, 211, 89, 54, 128, 18, 3, 251, 238, - 54, 128, 18, 3, 128, 251, 238, 54, 128, 18, 3, 211, 185, 58, 54, 128, 3, - 217, 115, 222, 85, 54, 128, 3, 198, 78, 54, 128, 3, 198, 77, 54, 128, 3, - 221, 173, 198, 76, 54, 128, 3, 221, 173, 198, 75, 54, 128, 3, 221, 173, - 198, 74, 54, 128, 3, 207, 188, 230, 92, 54, 128, 3, 217, 115, 205, 153, - 54, 128, 3, 221, 172, 222, 66, 54, 128, 33, 238, 200, 236, 142, 54, 128, - 228, 245, 17, 191, 77, 54, 128, 228, 245, 17, 107, 54, 128, 228, 245, 17, - 109, 54, 128, 228, 245, 17, 138, 54, 128, 228, 245, 17, 134, 54, 128, - 228, 245, 17, 150, 54, 128, 228, 245, 17, 169, 54, 128, 228, 245, 17, - 175, 54, 128, 228, 245, 17, 171, 54, 128, 228, 245, 17, 178, 54, 128, - 128, 17, 191, 77, 54, 128, 128, 17, 107, 54, 128, 128, 17, 109, 54, 128, - 128, 17, 138, 54, 128, 128, 17, 134, 54, 128, 128, 17, 150, 54, 128, 128, - 17, 169, 54, 128, 128, 17, 175, 54, 128, 128, 17, 171, 54, 128, 128, 17, - 178, 54, 128, 3, 193, 80, 54, 128, 3, 193, 79, 54, 128, 3, 205, 108, 54, - 128, 3, 221, 248, 54, 128, 3, 228, 172, 54, 128, 3, 236, 159, 54, 128, 3, - 207, 19, 206, 123, 207, 140, 54, 128, 3, 217, 115, 192, 109, 54, 128, 3, - 222, 122, 54, 128, 3, 222, 121, 54, 128, 3, 205, 118, 54, 128, 3, 205, - 117, 54, 128, 3, 230, 28, 54, 128, 3, 247, 159, 33, 235, 130, 243, 4, - 252, 62, 33, 237, 43, 33, 223, 141, 33, 235, 121, 57, 33, 197, 225, 236, - 142, 33, 192, 233, 60, 33, 193, 72, 219, 115, 60, 33, 211, 79, 87, 60, - 33, 55, 211, 79, 87, 60, 33, 156, 247, 25, 201, 29, 60, 33, 201, 15, 247, - 25, 201, 29, 60, 33, 210, 117, 58, 33, 55, 210, 117, 58, 33, 210, 117, - 60, 33, 210, 117, 211, 1, 33, 8, 2, 1, 193, 225, 60, 33, 8, 2, 1, 154, - 193, 225, 60, 33, 45, 210, 116, 93, 219, 226, 33, 50, 210, 116, 93, 179, - 33, 45, 210, 116, 248, 235, 219, 226, 33, 50, 210, 116, 248, 235, 179, - 33, 51, 248, 53, 58, 33, 31, 3, 58, 33, 223, 95, 55, 251, 17, 58, 33, - 108, 3, 58, 33, 55, 108, 3, 58, 33, 55, 108, 3, 60, 33, 197, 225, 252, - 49, 252, 62, 33, 8, 2, 1, 223, 117, 232, 53, 33, 8, 2, 1, 223, 117, 146, - 33, 8, 2, 1, 223, 117, 200, 43, 149, 3, 196, 123, 206, 245, 149, 3, 196, - 123, 247, 123, 149, 3, 247, 40, 149, 3, 200, 173, 149, 3, 248, 157, 149, - 1, 251, 216, 149, 1, 251, 217, 199, 123, 149, 1, 223, 196, 149, 1, 223, - 197, 199, 123, 149, 1, 196, 126, 149, 1, 196, 127, 199, 123, 149, 1, 207, - 188, 207, 52, 149, 1, 207, 188, 207, 53, 199, 123, 149, 1, 220, 128, 219, - 201, 149, 1, 220, 128, 219, 202, 199, 123, 149, 1, 234, 147, 149, 1, 251, - 235, 149, 1, 211, 125, 149, 1, 211, 126, 199, 123, 149, 1, 155, 149, 1, - 222, 144, 217, 118, 149, 1, 231, 242, 149, 1, 231, 243, 230, 243, 149, 1, - 214, 70, 149, 1, 247, 162, 149, 1, 247, 163, 220, 114, 149, 1, 223, 34, - 149, 1, 223, 35, 223, 2, 149, 1, 212, 103, 149, 1, 199, 252, 220, 4, 149, - 1, 199, 252, 215, 110, 217, 118, 149, 1, 238, 35, 215, 110, 251, 164, - 149, 1, 238, 35, 215, 110, 217, 118, 149, 1, 215, 9, 207, 143, 149, 1, - 190, 190, 149, 1, 199, 252, 199, 158, 149, 1, 238, 34, 149, 1, 238, 35, - 217, 140, 149, 1, 181, 149, 1, 168, 149, 1, 210, 223, 222, 78, 149, 1, - 249, 155, 149, 1, 249, 156, 222, 4, 149, 1, 174, 149, 1, 170, 149, 1, - 165, 149, 1, 173, 149, 1, 193, 190, 149, 1, 205, 208, 205, 185, 149, 1, - 205, 208, 205, 130, 149, 1, 188, 149, 1, 140, 149, 3, 207, 42, 149, 18, - 3, 199, 123, 149, 18, 3, 196, 122, 149, 18, 3, 196, 123, 205, 126, 149, - 18, 3, 200, 208, 149, 18, 3, 200, 209, 223, 187, 149, 18, 3, 207, 188, - 207, 52, 149, 18, 3, 207, 188, 207, 53, 199, 123, 149, 18, 3, 220, 128, - 219, 201, 149, 18, 3, 220, 128, 219, 202, 199, 123, 149, 18, 3, 199, 201, - 149, 18, 3, 199, 202, 207, 52, 149, 18, 3, 199, 202, 199, 123, 149, 18, - 3, 199, 202, 207, 53, 199, 123, 149, 18, 3, 210, 18, 149, 18, 3, 210, 19, - 199, 123, 149, 252, 39, 252, 38, 149, 1, 222, 109, 205, 125, 149, 1, 221, - 179, 205, 125, 149, 1, 196, 235, 205, 125, 149, 1, 234, 184, 205, 125, - 149, 1, 195, 154, 205, 125, 149, 1, 191, 109, 205, 125, 149, 1, 250, 187, - 205, 125, 149, 1, 251, 16, 222, 204, 149, 17, 191, 77, 149, 17, 107, 149, - 17, 109, 149, 17, 138, 149, 17, 134, 149, 17, 150, 149, 17, 169, 149, 17, - 175, 149, 17, 171, 149, 17, 178, 149, 210, 147, 149, 210, 177, 149, 193, - 64, 149, 247, 96, 210, 170, 149, 247, 96, 202, 191, 149, 247, 96, 210, - 114, 149, 210, 176, 149, 37, 16, 236, 150, 149, 37, 16, 237, 104, 149, - 37, 16, 235, 73, 149, 37, 16, 237, 238, 149, 37, 16, 237, 239, 200, 173, - 149, 37, 16, 236, 244, 149, 37, 16, 238, 26, 149, 37, 16, 237, 79, 149, - 37, 16, 238, 8, 149, 37, 16, 237, 239, 231, 161, 149, 37, 16, 33, 199, - 116, 149, 37, 16, 33, 234, 94, 149, 37, 16, 33, 221, 255, 149, 37, 16, - 33, 222, 1, 149, 37, 16, 33, 223, 7, 149, 37, 16, 33, 222, 0, 4, 223, 7, - 149, 37, 16, 33, 222, 2, 4, 223, 7, 149, 37, 16, 33, 248, 228, 149, 37, - 16, 33, 230, 249, 149, 37, 16, 206, 207, 211, 79, 235, 84, 149, 37, 16, - 206, 207, 211, 79, 238, 24, 149, 37, 16, 206, 207, 242, 221, 197, 80, - 149, 37, 16, 206, 207, 242, 221, 199, 211, 149, 37, 16, 219, 224, 211, - 79, 210, 162, 149, 37, 16, 219, 224, 211, 79, 208, 151, 149, 37, 16, 219, - 224, 242, 221, 209, 106, 149, 37, 16, 219, 224, 242, 221, 209, 88, 149, - 37, 16, 219, 224, 211, 79, 209, 134, 149, 210, 148, 220, 21, 149, 210, - 178, 220, 21, 200, 197, 3, 210, 144, 200, 197, 3, 210, 158, 200, 197, 3, - 210, 154, 200, 197, 1, 65, 200, 197, 1, 68, 200, 197, 1, 66, 200, 197, 1, - 252, 27, 200, 197, 1, 74, 200, 197, 1, 71, 200, 197, 1, 233, 244, 200, - 197, 1, 155, 200, 197, 1, 208, 98, 200, 197, 1, 231, 242, 200, 197, 1, - 214, 70, 200, 197, 1, 247, 162, 200, 197, 1, 223, 34, 200, 197, 1, 191, - 123, 200, 197, 1, 212, 103, 200, 197, 1, 190, 190, 200, 197, 1, 238, 34, - 200, 197, 1, 181, 200, 197, 1, 168, 200, 197, 1, 233, 111, 200, 197, 1, - 195, 188, 200, 197, 1, 249, 155, 200, 197, 1, 174, 200, 197, 1, 170, 200, - 197, 1, 165, 200, 197, 1, 173, 200, 197, 1, 193, 190, 200, 197, 1, 188, - 200, 197, 1, 192, 220, 200, 197, 1, 140, 200, 197, 120, 3, 210, 174, 200, - 197, 120, 3, 210, 146, 200, 197, 120, 3, 210, 143, 200, 197, 18, 3, 210, - 161, 200, 197, 18, 3, 210, 142, 200, 197, 18, 3, 210, 167, 200, 197, 18, - 3, 210, 153, 200, 197, 18, 3, 210, 175, 200, 197, 18, 3, 210, 163, 200, - 197, 3, 210, 179, 200, 197, 3, 195, 40, 200, 197, 120, 3, 210, 102, 174, - 200, 197, 120, 3, 210, 102, 193, 190, 200, 197, 1, 221, 217, 200, 197, 1, - 200, 126, 200, 197, 17, 191, 77, 200, 197, 17, 107, 200, 197, 17, 109, - 200, 197, 17, 138, 200, 197, 17, 134, 200, 197, 17, 150, 200, 197, 17, - 169, 200, 197, 17, 175, 200, 197, 17, 171, 200, 197, 17, 178, 200, 197, - 250, 147, 200, 197, 1, 207, 22, 200, 197, 1, 219, 174, 200, 197, 1, 248, - 205, 200, 197, 1, 53, 222, 154, 200, 197, 1, 53, 218, 170, 249, 65, 1, - 65, 249, 65, 1, 202, 183, 65, 249, 65, 1, 140, 249, 65, 1, 202, 183, 140, - 249, 65, 1, 217, 87, 140, 249, 65, 1, 249, 155, 249, 65, 1, 222, 63, 249, - 155, 249, 65, 1, 168, 249, 65, 1, 202, 183, 168, 249, 65, 1, 181, 249, - 65, 1, 217, 87, 181, 249, 65, 1, 193, 190, 249, 65, 1, 202, 183, 193, - 190, 249, 65, 1, 210, 195, 193, 190, 249, 65, 1, 231, 242, 249, 65, 1, - 202, 183, 231, 242, 249, 65, 1, 223, 34, 249, 65, 1, 238, 34, 249, 65, 1, - 165, 249, 65, 1, 202, 183, 165, 249, 65, 1, 174, 249, 65, 1, 202, 183, - 174, 249, 65, 1, 202, 1, 190, 190, 249, 65, 1, 213, 18, 190, 190, 249, - 65, 1, 188, 249, 65, 1, 202, 183, 188, 249, 65, 1, 217, 87, 188, 249, 65, - 1, 170, 249, 65, 1, 202, 183, 170, 249, 65, 1, 214, 70, 249, 65, 1, 173, - 249, 65, 1, 202, 183, 173, 249, 65, 1, 212, 103, 249, 65, 1, 247, 162, - 249, 65, 1, 214, 164, 249, 65, 1, 217, 13, 249, 65, 1, 68, 249, 65, 1, - 66, 249, 65, 3, 198, 82, 249, 65, 18, 3, 71, 249, 65, 18, 3, 210, 195, - 71, 249, 65, 18, 3, 234, 190, 249, 65, 18, 3, 68, 249, 65, 18, 3, 222, - 63, 68, 249, 65, 18, 3, 74, 249, 65, 18, 3, 222, 63, 74, 249, 65, 18, 3, - 66, 249, 65, 18, 3, 126, 40, 202, 183, 188, 249, 65, 120, 3, 214, 72, - 249, 65, 120, 3, 230, 118, 249, 65, 210, 156, 249, 65, 210, 152, 249, 65, - 16, 248, 167, 215, 9, 216, 165, 249, 65, 16, 248, 167, 209, 140, 249, 65, - 16, 248, 167, 222, 181, 249, 65, 16, 248, 167, 210, 156, 219, 185, 1, - 155, 219, 185, 1, 221, 96, 219, 185, 1, 221, 217, 219, 185, 1, 231, 242, - 219, 185, 1, 231, 21, 219, 185, 1, 214, 70, 219, 185, 1, 247, 162, 219, - 185, 1, 247, 3, 219, 185, 1, 223, 34, 219, 185, 1, 212, 103, 219, 185, 1, - 190, 190, 219, 185, 1, 199, 49, 219, 185, 1, 238, 34, 219, 185, 1, 181, - 219, 185, 1, 168, 219, 185, 1, 209, 112, 219, 185, 1, 209, 230, 219, 185, - 1, 233, 111, 219, 185, 1, 232, 221, 219, 185, 1, 249, 155, 219, 185, 1, - 248, 142, 219, 185, 1, 174, 219, 185, 1, 216, 21, 219, 185, 1, 197, 168, - 219, 185, 1, 197, 157, 219, 185, 1, 235, 37, 219, 185, 1, 170, 219, 185, - 1, 165, 219, 185, 1, 173, 219, 185, 1, 140, 219, 185, 1, 229, 113, 219, - 185, 1, 195, 188, 219, 185, 1, 188, 219, 185, 1, 203, 166, 219, 185, 1, - 193, 190, 219, 185, 1, 65, 219, 185, 200, 240, 1, 170, 219, 185, 200, - 240, 1, 165, 219, 185, 18, 3, 252, 208, 219, 185, 18, 3, 68, 219, 185, - 18, 3, 74, 219, 185, 18, 3, 211, 89, 219, 185, 18, 3, 66, 219, 185, 18, - 3, 196, 30, 219, 185, 18, 3, 71, 219, 185, 120, 3, 222, 154, 219, 185, - 120, 3, 218, 170, 219, 185, 120, 3, 172, 219, 185, 120, 3, 215, 63, 219, - 185, 120, 3, 210, 238, 219, 185, 120, 3, 146, 219, 185, 120, 3, 200, 43, - 219, 185, 120, 3, 212, 75, 219, 185, 120, 3, 222, 85, 219, 185, 3, 207, - 141, 219, 185, 3, 212, 143, 219, 185, 208, 154, 199, 247, 219, 185, 208, - 154, 212, 87, 198, 196, 199, 247, 219, 185, 208, 154, 247, 12, 219, 185, - 208, 154, 197, 149, 247, 12, 219, 185, 208, 154, 197, 148, 219, 185, 17, - 191, 77, 219, 185, 17, 107, 219, 185, 17, 109, 219, 185, 17, 138, 219, - 185, 17, 134, 219, 185, 17, 150, 219, 185, 17, 169, 219, 185, 17, 175, - 219, 185, 17, 171, 219, 185, 17, 178, 219, 185, 1, 197, 132, 219, 185, 1, - 197, 120, 219, 185, 1, 237, 193, 211, 123, 243, 90, 17, 191, 77, 211, - 123, 243, 90, 17, 107, 211, 123, 243, 90, 17, 109, 211, 123, 243, 90, 17, - 138, 211, 123, 243, 90, 17, 134, 211, 123, 243, 90, 17, 150, 211, 123, - 243, 90, 17, 169, 211, 123, 243, 90, 17, 175, 211, 123, 243, 90, 17, 171, - 211, 123, 243, 90, 17, 178, 211, 123, 243, 90, 1, 173, 211, 123, 243, 90, - 1, 250, 184, 211, 123, 243, 90, 1, 251, 255, 211, 123, 243, 90, 1, 251, - 124, 211, 123, 243, 90, 1, 251, 209, 211, 123, 243, 90, 1, 220, 127, 211, - 123, 243, 90, 1, 252, 170, 211, 123, 243, 90, 1, 252, 171, 211, 123, 243, - 90, 1, 252, 169, 211, 123, 243, 90, 1, 252, 163, 211, 123, 243, 90, 1, - 219, 148, 211, 123, 243, 90, 1, 223, 70, 211, 123, 243, 90, 1, 223, 202, - 211, 123, 243, 90, 1, 223, 92, 211, 123, 243, 90, 1, 223, 79, 211, 123, - 243, 90, 1, 218, 227, 211, 123, 243, 90, 1, 196, 160, 211, 123, 243, 90, - 1, 196, 158, 211, 123, 243, 90, 1, 196, 83, 211, 123, 243, 90, 1, 196, - 21, 211, 123, 243, 90, 1, 219, 240, 211, 123, 243, 90, 1, 234, 58, 211, - 123, 243, 90, 1, 234, 193, 211, 123, 243, 90, 1, 234, 105, 211, 123, 243, - 90, 1, 234, 28, 211, 123, 243, 90, 1, 219, 45, 211, 123, 243, 90, 1, 211, - 26, 211, 123, 243, 90, 1, 211, 180, 211, 123, 243, 90, 1, 211, 11, 211, - 123, 243, 90, 1, 211, 138, 211, 123, 243, 90, 215, 158, 197, 97, 211, - 123, 243, 90, 231, 237, 197, 98, 211, 123, 243, 90, 215, 152, 197, 98, - 211, 123, 243, 90, 207, 67, 211, 123, 243, 90, 209, 228, 211, 123, 243, - 90, 251, 246, 211, 123, 243, 90, 208, 154, 215, 148, 211, 123, 243, 90, - 208, 154, 55, 215, 148, 38, 2, 1, 206, 114, 195, 153, 38, 2, 1, 219, 14, - 237, 148, 38, 2, 1, 214, 217, 74, 38, 2, 1, 193, 78, 234, 24, 38, 2, 1, - 199, 200, 199, 145, 38, 2, 1, 198, 221, 199, 145, 38, 2, 1, 199, 200, - 230, 19, 56, 38, 2, 1, 199, 200, 192, 95, 38, 2, 1, 196, 108, 196, 128, - 101, 215, 159, 6, 1, 251, 134, 101, 215, 159, 6, 1, 249, 103, 101, 215, - 159, 6, 1, 231, 212, 101, 215, 159, 6, 1, 236, 152, 101, 215, 159, 6, 1, - 234, 105, 101, 215, 159, 6, 1, 195, 49, 101, 215, 159, 6, 1, 191, 80, - 101, 215, 159, 6, 1, 199, 193, 101, 215, 159, 6, 1, 223, 164, 101, 215, - 159, 6, 1, 222, 89, 101, 215, 159, 6, 1, 220, 9, 101, 215, 159, 6, 1, - 217, 92, 101, 215, 159, 6, 1, 214, 218, 101, 215, 159, 6, 1, 211, 106, - 101, 215, 159, 6, 1, 210, 133, 101, 215, 159, 6, 1, 191, 67, 101, 215, - 159, 6, 1, 207, 165, 101, 215, 159, 6, 1, 205, 143, 101, 215, 159, 6, 1, - 199, 179, 101, 215, 159, 6, 1, 196, 113, 101, 215, 159, 6, 1, 209, 222, - 101, 215, 159, 6, 1, 221, 202, 101, 215, 159, 6, 1, 231, 84, 101, 215, - 159, 6, 1, 208, 83, 101, 215, 159, 6, 1, 203, 70, 101, 215, 159, 6, 1, - 243, 83, 101, 215, 159, 6, 1, 247, 130, 101, 215, 159, 6, 1, 222, 236, - 101, 215, 159, 6, 1, 243, 20, 101, 215, 159, 6, 1, 246, 243, 101, 215, - 159, 6, 1, 192, 218, 101, 215, 159, 6, 1, 222, 251, 101, 215, 159, 6, 1, - 230, 89, 101, 215, 159, 6, 1, 229, 247, 101, 215, 159, 6, 1, 229, 147, - 101, 215, 159, 6, 1, 193, 125, 101, 215, 159, 6, 1, 230, 21, 101, 215, - 159, 6, 1, 229, 13, 101, 215, 159, 6, 1, 233, 25, 101, 215, 159, 6, 1, - 192, 14, 101, 215, 159, 6, 1, 234, 125, 101, 215, 159, 6, 1, 154, 231, - 212, 101, 215, 159, 6, 1, 251, 232, 101, 215, 159, 6, 1, 252, 16, 101, - 215, 159, 6, 1, 230, 19, 56, 101, 215, 159, 6, 1, 220, 118, 56, 200, 197, - 208, 154, 248, 167, 200, 166, 200, 197, 208, 154, 248, 167, 210, 157, - 200, 197, 208, 154, 248, 167, 208, 141, 200, 197, 208, 154, 248, 167, - 247, 147, 200, 197, 208, 154, 248, 167, 219, 175, 205, 122, 200, 197, - 208, 154, 248, 167, 222, 144, 205, 122, 200, 197, 208, 154, 248, 167, - 238, 35, 205, 122, 200, 197, 208, 154, 248, 167, 249, 156, 205, 122, 195, - 150, 163, 222, 59, 195, 150, 163, 203, 131, 195, 150, 163, 208, 227, 195, - 150, 3, 213, 189, 195, 150, 3, 192, 117, 216, 84, 200, 156, 195, 150, - 163, 192, 117, 251, 251, 223, 151, 200, 156, 195, 150, 163, 192, 117, - 223, 151, 200, 156, 195, 150, 163, 192, 117, 222, 47, 223, 151, 200, 156, - 195, 150, 163, 247, 124, 60, 195, 150, 163, 192, 117, 222, 47, 223, 151, - 200, 157, 205, 89, 195, 150, 163, 55, 200, 156, 195, 150, 163, 197, 225, - 200, 156, 195, 150, 163, 222, 47, 251, 75, 195, 150, 163, 75, 60, 195, - 150, 163, 105, 185, 60, 195, 150, 163, 115, 185, 60, 195, 150, 163, 206, - 197, 222, 58, 223, 151, 200, 156, 195, 150, 163, 250, 181, 223, 151, 200, - 156, 195, 150, 3, 195, 36, 200, 156, 195, 150, 3, 195, 36, 196, 154, 195, - 150, 3, 207, 19, 195, 36, 196, 154, 195, 150, 3, 195, 36, 251, 75, 195, - 150, 3, 207, 19, 195, 36, 251, 75, 195, 150, 3, 195, 36, 196, 155, 4, - 199, 215, 195, 150, 3, 195, 36, 251, 76, 4, 199, 215, 195, 150, 3, 251, - 74, 251, 90, 195, 150, 3, 251, 74, 249, 122, 195, 150, 3, 251, 74, 195, - 178, 195, 150, 3, 251, 74, 195, 179, 4, 199, 215, 195, 150, 3, 198, 126, - 195, 150, 3, 229, 182, 180, 251, 73, 195, 150, 3, 180, 251, 73, 195, 150, - 3, 206, 71, 180, 251, 73, 195, 150, 3, 251, 74, 196, 162, 215, 138, 195, - 150, 3, 251, 12, 195, 150, 3, 206, 123, 251, 12, 195, 150, 163, 247, 124, - 58, 195, 150, 3, 222, 239, 195, 150, 3, 196, 75, 195, 150, 3, 250, 179, - 195, 150, 163, 206, 190, 58, 195, 150, 163, 55, 206, 190, 58, 195, 150, - 3, 55, 251, 74, 251, 90, 8, 1, 2, 6, 65, 8, 1, 2, 6, 252, 27, 8, 2, 1, - 154, 252, 27, 8, 1, 2, 6, 249, 84, 250, 122, 8, 1, 2, 6, 247, 195, 8, 1, - 2, 6, 238, 129, 8, 1, 2, 6, 233, 250, 8, 1, 2, 6, 71, 8, 2, 1, 154, 211, - 79, 71, 8, 2, 1, 154, 68, 8, 1, 2, 6, 223, 37, 8, 1, 2, 6, 222, 154, 8, - 1, 2, 6, 220, 145, 4, 106, 8, 1, 2, 6, 218, 170, 8, 1, 2, 6, 207, 19, - 215, 63, 8, 1, 2, 6, 74, 8, 1, 2, 6, 211, 79, 74, 8, 2, 1, 202, 207, 74, - 8, 2, 1, 202, 207, 211, 79, 74, 8, 2, 1, 202, 207, 187, 4, 106, 8, 2, 1, - 154, 211, 153, 8, 1, 2, 6, 211, 21, 8, 2, 1, 198, 54, 132, 74, 8, 2, 1, - 248, 79, 132, 74, 8, 1, 2, 6, 210, 238, 8, 1, 2, 6, 207, 19, 146, 8, 1, - 2, 6, 154, 146, 8, 1, 2, 6, 200, 43, 8, 1, 2, 6, 66, 8, 2, 1, 202, 207, - 66, 8, 2, 1, 202, 207, 237, 42, 66, 8, 2, 1, 202, 207, 154, 218, 170, 8, - 1, 2, 6, 196, 12, 8, 1, 2, 6, 193, 224, 8, 1, 2, 6, 191, 166, 8, 1, 2, 6, - 233, 180, 8, 1, 195, 20, 220, 10, 201, 217, 8, 1, 251, 232, 35, 1, 2, 6, - 231, 213, 35, 1, 2, 6, 220, 33, 35, 1, 2, 6, 209, 187, 35, 1, 2, 6, 207, - 4, 35, 1, 2, 6, 208, 178, 38, 1, 2, 6, 234, 142, 52, 1, 6, 65, 52, 1, 6, - 252, 27, 52, 1, 6, 250, 122, 52, 1, 6, 249, 84, 250, 122, 52, 1, 6, 238, - 129, 52, 1, 6, 71, 52, 1, 6, 207, 19, 71, 52, 1, 6, 232, 53, 52, 1, 6, - 230, 118, 52, 1, 6, 68, 52, 1, 6, 223, 37, 52, 1, 6, 222, 154, 52, 1, 6, - 172, 52, 1, 6, 218, 170, 52, 1, 6, 215, 63, 52, 1, 6, 207, 19, 215, 63, - 52, 1, 6, 74, 52, 1, 6, 211, 21, 52, 1, 6, 210, 238, 52, 1, 6, 146, 52, - 1, 6, 200, 43, 52, 1, 6, 66, 52, 1, 6, 193, 224, 52, 1, 2, 65, 52, 1, 2, - 154, 65, 52, 1, 2, 251, 162, 52, 1, 2, 154, 252, 27, 52, 1, 2, 250, 122, - 52, 1, 2, 238, 129, 52, 1, 2, 71, 52, 1, 2, 205, 87, 52, 1, 2, 211, 79, - 71, 52, 1, 2, 154, 211, 79, 71, 52, 1, 2, 232, 53, 52, 1, 2, 154, 68, 52, - 1, 2, 222, 154, 52, 1, 2, 218, 170, 52, 1, 2, 234, 90, 52, 1, 2, 74, 52, - 1, 2, 211, 79, 74, 52, 1, 2, 198, 54, 132, 74, 52, 1, 2, 248, 79, 132, - 74, 52, 1, 2, 210, 238, 52, 1, 2, 200, 43, 52, 1, 2, 66, 52, 1, 2, 202, - 207, 66, 52, 1, 2, 154, 218, 170, 52, 1, 2, 196, 12, 52, 1, 2, 251, 232, - 52, 1, 2, 248, 214, 52, 1, 2, 35, 231, 213, 52, 1, 2, 237, 108, 52, 1, 2, - 35, 209, 213, 52, 1, 2, 243, 97, 8, 200, 231, 2, 1, 68, 8, 200, 231, 2, - 1, 146, 8, 200, 231, 2, 1, 66, 8, 200, 231, 2, 1, 196, 12, 35, 200, 231, - 2, 1, 248, 214, 35, 200, 231, 2, 1, 231, 213, 35, 200, 231, 2, 1, 207, 4, - 35, 200, 231, 2, 1, 209, 213, 35, 200, 231, 2, 1, 243, 97, 8, 2, 1, 196, - 152, 8, 2, 1, 78, 4, 82, 198, 152, 8, 2, 1, 238, 130, 4, 82, 198, 152, 8, - 2, 1, 233, 178, 4, 82, 198, 152, 8, 2, 1, 218, 171, 4, 82, 198, 152, 8, - 2, 1, 215, 64, 4, 82, 198, 152, 8, 2, 1, 210, 239, 4, 82, 198, 152, 8, 2, - 1, 207, 224, 4, 82, 198, 152, 8, 2, 1, 207, 224, 4, 232, 236, 23, 82, - 198, 152, 8, 2, 1, 206, 10, 4, 82, 198, 152, 8, 2, 1, 200, 44, 4, 82, - 198, 152, 8, 2, 1, 191, 167, 4, 82, 198, 152, 8, 2, 1, 154, 232, 53, 52, - 1, 38, 234, 105, 8, 2, 1, 223, 117, 232, 53, 8, 2, 1, 199, 52, 4, 201, - 33, 8, 2, 6, 1, 228, 76, 4, 106, 8, 2, 1, 223, 86, 4, 106, 8, 2, 1, 210, - 239, 4, 106, 8, 2, 6, 1, 126, 4, 106, 8, 2, 1, 196, 71, 4, 106, 8, 2, 1, - 78, 4, 210, 194, 102, 8, 2, 1, 238, 130, 4, 210, 194, 102, 8, 2, 1, 233, - 178, 4, 210, 194, 102, 8, 2, 1, 232, 54, 4, 210, 194, 102, 8, 2, 1, 222, - 155, 4, 210, 194, 102, 8, 2, 1, 220, 145, 4, 210, 194, 102, 8, 2, 1, 218, - 171, 4, 210, 194, 102, 8, 2, 1, 215, 64, 4, 210, 194, 102, 8, 2, 1, 210, - 239, 4, 210, 194, 102, 8, 2, 1, 207, 224, 4, 210, 194, 102, 8, 2, 1, 206, - 10, 4, 210, 194, 102, 8, 2, 1, 234, 15, 4, 210, 194, 102, 8, 2, 1, 196, - 13, 4, 210, 194, 102, 8, 2, 1, 192, 236, 4, 210, 194, 102, 8, 2, 1, 191, - 167, 4, 210, 194, 102, 8, 2, 1, 42, 4, 207, 25, 102, 8, 2, 1, 251, 163, - 4, 207, 25, 102, 8, 2, 1, 238, 130, 4, 228, 253, 23, 199, 215, 8, 2, 1, - 235, 17, 4, 207, 25, 102, 8, 2, 1, 211, 79, 235, 17, 4, 207, 25, 102, 8, - 2, 1, 207, 19, 211, 79, 235, 17, 4, 207, 25, 102, 8, 2, 1, 205, 88, 4, - 207, 25, 102, 8, 2, 1, 228, 76, 4, 207, 25, 102, 8, 2, 1, 211, 79, 187, - 4, 207, 25, 102, 8, 2, 1, 234, 15, 4, 207, 25, 102, 8, 2, 1, 126, 4, 207, - 25, 102, 8, 2, 1, 233, 181, 4, 207, 25, 102, 52, 1, 2, 154, 251, 162, 52, - 1, 2, 247, 195, 52, 1, 2, 247, 196, 4, 238, 177, 52, 1, 2, 233, 250, 52, - 1, 2, 207, 19, 211, 79, 71, 52, 1, 2, 233, 177, 52, 1, 2, 236, 141, 223, - 38, 4, 106, 52, 1, 2, 27, 232, 53, 52, 1, 2, 154, 230, 118, 52, 1, 2, - 228, 76, 4, 106, 52, 1, 2, 223, 85, 52, 1, 2, 6, 68, 52, 1, 2, 6, 228, - 76, 4, 106, 52, 1, 2, 223, 38, 4, 238, 214, 52, 1, 2, 220, 145, 4, 207, - 25, 102, 52, 1, 2, 220, 145, 4, 210, 194, 102, 52, 1, 2, 6, 172, 52, 1, - 2, 218, 171, 4, 102, 52, 1, 2, 154, 218, 171, 4, 180, 219, 214, 52, 1, 2, - 215, 64, 4, 45, 102, 52, 1, 2, 215, 64, 4, 207, 25, 102, 52, 1, 2, 6, - 215, 63, 52, 1, 2, 249, 84, 74, 52, 1, 2, 209, 213, 52, 1, 2, 206, 10, 4, - 102, 52, 1, 2, 234, 14, 52, 1, 2, 200, 44, 4, 210, 194, 102, 52, 1, 2, - 126, 164, 52, 1, 2, 196, 70, 52, 1, 2, 6, 66, 52, 1, 2, 196, 13, 4, 102, - 52, 1, 2, 154, 196, 12, 52, 1, 2, 191, 166, 52, 1, 2, 191, 167, 4, 207, - 25, 102, 52, 1, 2, 191, 167, 4, 238, 177, 52, 1, 2, 233, 180, 52, 1, 2, - 199, 15, 33, 235, 140, 230, 213, 252, 62, 33, 235, 140, 252, 49, 252, 62, - 33, 202, 60, 60, 33, 200, 164, 77, 33, 217, 147, 33, 230, 210, 33, 217, - 145, 33, 252, 46, 33, 230, 211, 33, 252, 47, 33, 8, 2, 1, 207, 224, 60, - 33, 248, 37, 33, 217, 146, 33, 55, 243, 4, 58, 33, 211, 141, 58, 33, 191, - 21, 60, 33, 223, 71, 60, 33, 196, 63, 58, 33, 196, 46, 58, 33, 8, 2, 1, - 232, 205, 211, 79, 42, 58, 33, 8, 2, 1, 252, 27, 33, 8, 2, 1, 251, 70, - 33, 8, 2, 1, 250, 148, 33, 8, 2, 1, 247, 196, 247, 37, 33, 8, 2, 1, 223, - 117, 238, 129, 33, 8, 2, 1, 233, 250, 33, 8, 2, 1, 232, 53, 33, 8, 1, 2, - 6, 232, 53, 33, 8, 2, 1, 222, 154, 33, 8, 2, 1, 172, 33, 8, 1, 2, 6, 172, - 33, 8, 1, 2, 6, 218, 170, 33, 8, 2, 1, 215, 63, 33, 8, 1, 2, 6, 215, 63, - 33, 8, 1, 2, 6, 146, 33, 8, 2, 1, 207, 224, 206, 117, 33, 8, 2, 1, 206, - 9, 33, 8, 2, 1, 180, 206, 9, 33, 8, 2, 1, 191, 166, 33, 8, 2, 1, 251, - 162, 33, 8, 2, 1, 250, 122, 33, 8, 2, 1, 248, 214, 33, 8, 2, 1, 205, 87, - 33, 8, 2, 1, 233, 177, 33, 8, 2, 1, 220, 145, 4, 55, 82, 198, 152, 33, 8, - 2, 1, 187, 4, 156, 247, 25, 106, 33, 8, 2, 1, 210, 238, 33, 8, 2, 1, 234, - 14, 33, 8, 2, 1, 126, 4, 156, 247, 25, 106, 33, 8, 2, 1, 193, 224, 33, 8, - 2, 1, 42, 4, 237, 44, 33, 8, 2, 1, 187, 4, 237, 44, 33, 8, 2, 1, 126, 4, - 237, 44, 33, 133, 199, 229, 58, 33, 222, 38, 93, 179, 33, 222, 38, 93, - 219, 226, 33, 75, 93, 219, 226, 33, 193, 78, 223, 95, 248, 31, 60, 33, - 75, 248, 235, 219, 226, 33, 237, 117, 77, 33, 55, 223, 95, 248, 39, 60, - 33, 251, 167, 234, 47, 119, 60, 33, 45, 250, 238, 58, 33, 50, 250, 238, - 23, 144, 250, 238, 60, 8, 6, 1, 42, 4, 206, 190, 60, 8, 2, 1, 42, 4, 206, - 190, 60, 8, 6, 1, 78, 4, 75, 58, 8, 2, 1, 78, 4, 75, 58, 8, 6, 1, 78, 4, - 75, 60, 8, 2, 1, 78, 4, 75, 60, 8, 6, 1, 78, 4, 219, 115, 60, 8, 2, 1, - 78, 4, 219, 115, 60, 8, 6, 1, 247, 196, 4, 247, 38, 23, 252, 48, 8, 2, 1, - 247, 196, 4, 247, 38, 23, 252, 48, 8, 6, 1, 238, 130, 4, 75, 58, 8, 2, 1, - 238, 130, 4, 75, 58, 8, 6, 1, 238, 130, 4, 75, 60, 8, 2, 1, 238, 130, 4, - 75, 60, 8, 6, 1, 238, 130, 4, 219, 115, 60, 8, 2, 1, 238, 130, 4, 219, - 115, 60, 8, 6, 1, 238, 130, 4, 247, 37, 8, 2, 1, 238, 130, 4, 247, 37, 8, - 6, 1, 238, 130, 4, 243, 4, 60, 8, 2, 1, 238, 130, 4, 243, 4, 60, 8, 6, 1, - 235, 17, 4, 217, 149, 23, 230, 212, 8, 2, 1, 235, 17, 4, 217, 149, 23, - 230, 212, 8, 6, 1, 235, 17, 4, 217, 149, 23, 252, 48, 8, 2, 1, 235, 17, - 4, 217, 149, 23, 252, 48, 8, 6, 1, 235, 17, 4, 243, 4, 60, 8, 2, 1, 235, - 17, 4, 243, 4, 60, 8, 6, 1, 235, 17, 4, 198, 153, 60, 8, 2, 1, 235, 17, - 4, 198, 153, 60, 8, 6, 1, 235, 17, 4, 247, 38, 23, 248, 38, 8, 2, 1, 235, - 17, 4, 247, 38, 23, 248, 38, 8, 6, 1, 233, 178, 4, 75, 58, 8, 2, 1, 233, - 178, 4, 75, 58, 8, 6, 1, 232, 54, 4, 217, 148, 8, 2, 1, 232, 54, 4, 217, - 148, 8, 6, 1, 230, 119, 4, 75, 58, 8, 2, 1, 230, 119, 4, 75, 58, 8, 6, 1, - 230, 119, 4, 75, 60, 8, 2, 1, 230, 119, 4, 75, 60, 8, 6, 1, 230, 119, 4, - 237, 44, 8, 2, 1, 230, 119, 4, 237, 44, 8, 6, 1, 230, 119, 4, 247, 37, 8, - 2, 1, 230, 119, 4, 247, 37, 8, 6, 1, 230, 119, 4, 248, 39, 60, 8, 2, 1, - 230, 119, 4, 248, 39, 60, 8, 6, 1, 228, 76, 4, 198, 153, 60, 8, 2, 1, - 228, 76, 4, 198, 153, 60, 8, 6, 1, 228, 76, 4, 237, 45, 23, 252, 48, 8, - 2, 1, 228, 76, 4, 237, 45, 23, 252, 48, 8, 6, 1, 222, 155, 4, 252, 48, 8, - 2, 1, 222, 155, 4, 252, 48, 8, 6, 1, 222, 155, 4, 75, 60, 8, 2, 1, 222, - 155, 4, 75, 60, 8, 6, 1, 222, 155, 4, 219, 115, 60, 8, 2, 1, 222, 155, 4, - 219, 115, 60, 8, 6, 1, 220, 145, 4, 75, 60, 8, 2, 1, 220, 145, 4, 75, 60, - 8, 6, 1, 220, 145, 4, 75, 248, 235, 23, 217, 148, 8, 2, 1, 220, 145, 4, - 75, 248, 235, 23, 217, 148, 8, 6, 1, 220, 145, 4, 219, 115, 60, 8, 2, 1, - 220, 145, 4, 219, 115, 60, 8, 6, 1, 220, 145, 4, 243, 4, 60, 8, 2, 1, - 220, 145, 4, 243, 4, 60, 8, 6, 1, 218, 171, 4, 252, 48, 8, 2, 1, 218, - 171, 4, 252, 48, 8, 6, 1, 218, 171, 4, 75, 58, 8, 2, 1, 218, 171, 4, 75, - 58, 8, 6, 1, 218, 171, 4, 75, 60, 8, 2, 1, 218, 171, 4, 75, 60, 8, 6, 1, - 215, 64, 4, 75, 58, 8, 2, 1, 215, 64, 4, 75, 58, 8, 6, 1, 215, 64, 4, 75, - 60, 8, 2, 1, 215, 64, 4, 75, 60, 8, 6, 1, 215, 64, 4, 219, 115, 60, 8, 2, - 1, 215, 64, 4, 219, 115, 60, 8, 6, 1, 215, 64, 4, 243, 4, 60, 8, 2, 1, - 215, 64, 4, 243, 4, 60, 8, 6, 1, 187, 4, 198, 153, 23, 252, 48, 8, 2, 1, - 187, 4, 198, 153, 23, 252, 48, 8, 6, 1, 187, 4, 198, 153, 23, 237, 44, 8, - 2, 1, 187, 4, 198, 153, 23, 237, 44, 8, 6, 1, 187, 4, 217, 149, 23, 230, - 212, 8, 2, 1, 187, 4, 217, 149, 23, 230, 212, 8, 6, 1, 187, 4, 217, 149, - 23, 252, 48, 8, 2, 1, 187, 4, 217, 149, 23, 252, 48, 8, 6, 1, 210, 239, - 4, 252, 48, 8, 2, 1, 210, 239, 4, 252, 48, 8, 6, 1, 210, 239, 4, 75, 58, - 8, 2, 1, 210, 239, 4, 75, 58, 8, 6, 1, 207, 224, 4, 75, 58, 8, 2, 1, 207, - 224, 4, 75, 58, 8, 6, 1, 207, 224, 4, 75, 60, 8, 2, 1, 207, 224, 4, 75, - 60, 8, 6, 1, 207, 224, 4, 75, 248, 235, 23, 217, 148, 8, 2, 1, 207, 224, - 4, 75, 248, 235, 23, 217, 148, 8, 6, 1, 207, 224, 4, 219, 115, 60, 8, 2, - 1, 207, 224, 4, 219, 115, 60, 8, 6, 1, 206, 10, 4, 75, 58, 8, 2, 1, 206, - 10, 4, 75, 58, 8, 6, 1, 206, 10, 4, 75, 60, 8, 2, 1, 206, 10, 4, 75, 60, - 8, 6, 1, 206, 10, 4, 252, 49, 23, 75, 58, 8, 2, 1, 206, 10, 4, 252, 49, - 23, 75, 58, 8, 6, 1, 206, 10, 4, 247, 95, 23, 75, 58, 8, 2, 1, 206, 10, - 4, 247, 95, 23, 75, 58, 8, 6, 1, 206, 10, 4, 75, 248, 235, 23, 75, 58, 8, - 2, 1, 206, 10, 4, 75, 248, 235, 23, 75, 58, 8, 6, 1, 200, 44, 4, 75, 58, - 8, 2, 1, 200, 44, 4, 75, 58, 8, 6, 1, 200, 44, 4, 75, 60, 8, 2, 1, 200, - 44, 4, 75, 60, 8, 6, 1, 200, 44, 4, 219, 115, 60, 8, 2, 1, 200, 44, 4, - 219, 115, 60, 8, 6, 1, 200, 44, 4, 243, 4, 60, 8, 2, 1, 200, 44, 4, 243, - 4, 60, 8, 6, 1, 126, 4, 237, 45, 60, 8, 2, 1, 126, 4, 237, 45, 60, 8, 6, - 1, 126, 4, 198, 153, 60, 8, 2, 1, 126, 4, 198, 153, 60, 8, 6, 1, 126, 4, - 243, 4, 60, 8, 2, 1, 126, 4, 243, 4, 60, 8, 6, 1, 126, 4, 198, 153, 23, - 252, 48, 8, 2, 1, 126, 4, 198, 153, 23, 252, 48, 8, 6, 1, 126, 4, 217, - 149, 23, 237, 44, 8, 2, 1, 126, 4, 217, 149, 23, 237, 44, 8, 6, 1, 196, - 13, 4, 198, 152, 8, 2, 1, 196, 13, 4, 198, 152, 8, 6, 1, 196, 13, 4, 75, - 60, 8, 2, 1, 196, 13, 4, 75, 60, 8, 6, 1, 193, 225, 4, 230, 212, 8, 2, 1, - 193, 225, 4, 230, 212, 8, 6, 1, 193, 225, 4, 252, 48, 8, 2, 1, 193, 225, - 4, 252, 48, 8, 6, 1, 193, 225, 4, 237, 44, 8, 2, 1, 193, 225, 4, 237, 44, - 8, 6, 1, 193, 225, 4, 75, 58, 8, 2, 1, 193, 225, 4, 75, 58, 8, 6, 1, 193, - 225, 4, 75, 60, 8, 2, 1, 193, 225, 4, 75, 60, 8, 6, 1, 192, 236, 4, 75, - 58, 8, 2, 1, 192, 236, 4, 75, 58, 8, 6, 1, 192, 236, 4, 237, 44, 8, 2, 1, - 192, 236, 4, 237, 44, 8, 6, 1, 192, 160, 4, 75, 58, 8, 2, 1, 192, 160, 4, - 75, 58, 8, 6, 1, 191, 167, 4, 243, 3, 8, 2, 1, 191, 167, 4, 243, 3, 8, 6, - 1, 191, 167, 4, 75, 60, 8, 2, 1, 191, 167, 4, 75, 60, 8, 6, 1, 191, 167, - 4, 219, 115, 60, 8, 2, 1, 191, 167, 4, 219, 115, 60, 8, 2, 1, 230, 119, - 4, 219, 115, 60, 8, 2, 1, 200, 44, 4, 237, 44, 8, 2, 1, 193, 225, 4, 206, - 190, 58, 8, 2, 1, 192, 160, 4, 206, 190, 58, 8, 2, 1, 42, 4, 50, 132, - 206, 189, 8, 2, 1, 180, 206, 10, 4, 75, 58, 8, 2, 1, 180, 206, 10, 4, - 237, 41, 106, 8, 2, 1, 180, 206, 10, 4, 137, 106, 8, 6, 1, 203, 128, 206, - 9, 8, 2, 1, 237, 108, 8, 6, 1, 42, 4, 75, 60, 8, 2, 1, 42, 4, 75, 60, 8, - 6, 1, 42, 4, 228, 253, 58, 8, 2, 1, 42, 4, 228, 253, 58, 8, 6, 1, 42, 4, - 243, 4, 23, 252, 48, 8, 2, 1, 42, 4, 243, 4, 23, 252, 48, 8, 6, 1, 42, 4, - 243, 4, 23, 230, 212, 8, 2, 1, 42, 4, 243, 4, 23, 230, 212, 8, 6, 1, 42, - 4, 243, 4, 23, 228, 253, 58, 8, 2, 1, 42, 4, 243, 4, 23, 228, 253, 58, 8, - 6, 1, 42, 4, 243, 4, 23, 198, 152, 8, 2, 1, 42, 4, 243, 4, 23, 198, 152, - 8, 6, 1, 42, 4, 243, 4, 23, 75, 60, 8, 2, 1, 42, 4, 243, 4, 23, 75, 60, - 8, 6, 1, 42, 4, 248, 39, 23, 252, 48, 8, 2, 1, 42, 4, 248, 39, 23, 252, - 48, 8, 6, 1, 42, 4, 248, 39, 23, 230, 212, 8, 2, 1, 42, 4, 248, 39, 23, - 230, 212, 8, 6, 1, 42, 4, 248, 39, 23, 228, 253, 58, 8, 2, 1, 42, 4, 248, - 39, 23, 228, 253, 58, 8, 6, 1, 42, 4, 248, 39, 23, 198, 152, 8, 2, 1, 42, - 4, 248, 39, 23, 198, 152, 8, 6, 1, 42, 4, 248, 39, 23, 75, 60, 8, 2, 1, - 42, 4, 248, 39, 23, 75, 60, 8, 6, 1, 235, 17, 4, 75, 60, 8, 2, 1, 235, - 17, 4, 75, 60, 8, 6, 1, 235, 17, 4, 228, 253, 58, 8, 2, 1, 235, 17, 4, - 228, 253, 58, 8, 6, 1, 235, 17, 4, 198, 152, 8, 2, 1, 235, 17, 4, 198, - 152, 8, 6, 1, 235, 17, 4, 243, 4, 23, 252, 48, 8, 2, 1, 235, 17, 4, 243, - 4, 23, 252, 48, 8, 6, 1, 235, 17, 4, 243, 4, 23, 230, 212, 8, 2, 1, 235, - 17, 4, 243, 4, 23, 230, 212, 8, 6, 1, 235, 17, 4, 243, 4, 23, 228, 253, - 58, 8, 2, 1, 235, 17, 4, 243, 4, 23, 228, 253, 58, 8, 6, 1, 235, 17, 4, - 243, 4, 23, 198, 152, 8, 2, 1, 235, 17, 4, 243, 4, 23, 198, 152, 8, 6, 1, - 235, 17, 4, 243, 4, 23, 75, 60, 8, 2, 1, 235, 17, 4, 243, 4, 23, 75, 60, - 8, 6, 1, 228, 76, 4, 228, 253, 58, 8, 2, 1, 228, 76, 4, 228, 253, 58, 8, - 6, 1, 228, 76, 4, 75, 60, 8, 2, 1, 228, 76, 4, 75, 60, 8, 6, 1, 187, 4, - 75, 60, 8, 2, 1, 187, 4, 75, 60, 8, 6, 1, 187, 4, 228, 253, 58, 8, 2, 1, - 187, 4, 228, 253, 58, 8, 6, 1, 187, 4, 243, 4, 23, 252, 48, 8, 2, 1, 187, - 4, 243, 4, 23, 252, 48, 8, 6, 1, 187, 4, 243, 4, 23, 230, 212, 8, 2, 1, - 187, 4, 243, 4, 23, 230, 212, 8, 6, 1, 187, 4, 243, 4, 23, 228, 253, 58, - 8, 2, 1, 187, 4, 243, 4, 23, 228, 253, 58, 8, 6, 1, 187, 4, 243, 4, 23, - 198, 152, 8, 2, 1, 187, 4, 243, 4, 23, 198, 152, 8, 6, 1, 187, 4, 243, 4, - 23, 75, 60, 8, 2, 1, 187, 4, 243, 4, 23, 75, 60, 8, 6, 1, 187, 4, 228, - 190, 23, 252, 48, 8, 2, 1, 187, 4, 228, 190, 23, 252, 48, 8, 6, 1, 187, - 4, 228, 190, 23, 230, 212, 8, 2, 1, 187, 4, 228, 190, 23, 230, 212, 8, 6, - 1, 187, 4, 228, 190, 23, 228, 253, 58, 8, 2, 1, 187, 4, 228, 190, 23, - 228, 253, 58, 8, 6, 1, 187, 4, 228, 190, 23, 198, 152, 8, 2, 1, 187, 4, - 228, 190, 23, 198, 152, 8, 6, 1, 187, 4, 228, 190, 23, 75, 60, 8, 2, 1, - 187, 4, 228, 190, 23, 75, 60, 8, 6, 1, 126, 4, 75, 60, 8, 2, 1, 126, 4, - 75, 60, 8, 6, 1, 126, 4, 228, 253, 58, 8, 2, 1, 126, 4, 228, 253, 58, 8, - 6, 1, 126, 4, 228, 190, 23, 252, 48, 8, 2, 1, 126, 4, 228, 190, 23, 252, - 48, 8, 6, 1, 126, 4, 228, 190, 23, 230, 212, 8, 2, 1, 126, 4, 228, 190, - 23, 230, 212, 8, 6, 1, 126, 4, 228, 190, 23, 228, 253, 58, 8, 2, 1, 126, - 4, 228, 190, 23, 228, 253, 58, 8, 6, 1, 126, 4, 228, 190, 23, 198, 152, - 8, 2, 1, 126, 4, 228, 190, 23, 198, 152, 8, 6, 1, 126, 4, 228, 190, 23, - 75, 60, 8, 2, 1, 126, 4, 228, 190, 23, 75, 60, 8, 6, 1, 192, 160, 4, 230, - 212, 8, 2, 1, 192, 160, 4, 230, 212, 8, 6, 1, 192, 160, 4, 75, 60, 8, 2, - 1, 192, 160, 4, 75, 60, 8, 6, 1, 192, 160, 4, 228, 253, 58, 8, 2, 1, 192, - 160, 4, 228, 253, 58, 8, 6, 1, 192, 160, 4, 198, 152, 8, 2, 1, 192, 160, - 4, 198, 152, 8, 6, 1, 216, 85, 219, 76, 8, 2, 1, 216, 85, 219, 76, 8, 6, - 1, 216, 85, 196, 12, 8, 2, 1, 216, 85, 196, 12, 8, 6, 1, 192, 160, 4, - 219, 6, 8, 2, 1, 192, 160, 4, 219, 6, 35, 2, 1, 251, 163, 4, 208, 171, - 35, 2, 1, 251, 163, 4, 237, 214, 35, 2, 1, 251, 163, 4, 208, 172, 23, - 195, 169, 35, 2, 1, 251, 163, 4, 237, 215, 23, 195, 169, 35, 2, 1, 251, - 163, 4, 208, 172, 23, 210, 245, 35, 2, 1, 251, 163, 4, 237, 215, 23, 210, - 245, 35, 2, 1, 251, 163, 4, 208, 172, 23, 210, 6, 35, 2, 1, 251, 163, 4, - 237, 215, 23, 210, 6, 35, 6, 1, 251, 163, 4, 208, 171, 35, 6, 1, 251, - 163, 4, 237, 214, 35, 6, 1, 251, 163, 4, 208, 172, 23, 195, 169, 35, 6, - 1, 251, 163, 4, 237, 215, 23, 195, 169, 35, 6, 1, 251, 163, 4, 208, 172, - 23, 210, 245, 35, 6, 1, 251, 163, 4, 237, 215, 23, 210, 245, 35, 6, 1, - 251, 163, 4, 208, 172, 23, 210, 6, 35, 6, 1, 251, 163, 4, 237, 215, 23, - 210, 6, 35, 2, 1, 234, 50, 4, 208, 171, 35, 2, 1, 234, 50, 4, 237, 214, - 35, 2, 1, 234, 50, 4, 208, 172, 23, 195, 169, 35, 2, 1, 234, 50, 4, 237, - 215, 23, 195, 169, 35, 2, 1, 234, 50, 4, 208, 172, 23, 210, 245, 35, 2, - 1, 234, 50, 4, 237, 215, 23, 210, 245, 35, 6, 1, 234, 50, 4, 208, 171, - 35, 6, 1, 234, 50, 4, 237, 214, 35, 6, 1, 234, 50, 4, 208, 172, 23, 195, - 169, 35, 6, 1, 234, 50, 4, 237, 215, 23, 195, 169, 35, 6, 1, 234, 50, 4, - 208, 172, 23, 210, 245, 35, 6, 1, 234, 50, 4, 237, 215, 23, 210, 245, 35, - 2, 1, 234, 0, 4, 208, 171, 35, 2, 1, 234, 0, 4, 237, 214, 35, 2, 1, 234, - 0, 4, 208, 172, 23, 195, 169, 35, 2, 1, 234, 0, 4, 237, 215, 23, 195, - 169, 35, 2, 1, 234, 0, 4, 208, 172, 23, 210, 245, 35, 2, 1, 234, 0, 4, - 237, 215, 23, 210, 245, 35, 2, 1, 234, 0, 4, 208, 172, 23, 210, 6, 35, 2, - 1, 234, 0, 4, 237, 215, 23, 210, 6, 35, 6, 1, 234, 0, 4, 208, 171, 35, 6, - 1, 234, 0, 4, 237, 214, 35, 6, 1, 234, 0, 4, 208, 172, 23, 195, 169, 35, - 6, 1, 234, 0, 4, 237, 215, 23, 195, 169, 35, 6, 1, 234, 0, 4, 208, 172, - 23, 210, 245, 35, 6, 1, 234, 0, 4, 237, 215, 23, 210, 245, 35, 6, 1, 234, - 0, 4, 208, 172, 23, 210, 6, 35, 6, 1, 234, 0, 4, 237, 215, 23, 210, 6, - 35, 2, 1, 223, 86, 4, 208, 171, 35, 2, 1, 223, 86, 4, 237, 214, 35, 2, 1, - 223, 86, 4, 208, 172, 23, 195, 169, 35, 2, 1, 223, 86, 4, 237, 215, 23, - 195, 169, 35, 2, 1, 223, 86, 4, 208, 172, 23, 210, 245, 35, 2, 1, 223, - 86, 4, 237, 215, 23, 210, 245, 35, 2, 1, 223, 86, 4, 208, 172, 23, 210, - 6, 35, 2, 1, 223, 86, 4, 237, 215, 23, 210, 6, 35, 6, 1, 223, 86, 4, 208, - 171, 35, 6, 1, 223, 86, 4, 237, 214, 35, 6, 1, 223, 86, 4, 208, 172, 23, - 195, 169, 35, 6, 1, 223, 86, 4, 237, 215, 23, 195, 169, 35, 6, 1, 223, - 86, 4, 208, 172, 23, 210, 245, 35, 6, 1, 223, 86, 4, 237, 215, 23, 210, - 245, 35, 6, 1, 223, 86, 4, 208, 172, 23, 210, 6, 35, 6, 1, 223, 86, 4, - 237, 215, 23, 210, 6, 35, 2, 1, 211, 110, 4, 208, 171, 35, 2, 1, 211, - 110, 4, 237, 214, 35, 2, 1, 211, 110, 4, 208, 172, 23, 195, 169, 35, 2, - 1, 211, 110, 4, 237, 215, 23, 195, 169, 35, 2, 1, 211, 110, 4, 208, 172, - 23, 210, 245, 35, 2, 1, 211, 110, 4, 237, 215, 23, 210, 245, 35, 6, 1, - 211, 110, 4, 208, 171, 35, 6, 1, 211, 110, 4, 237, 214, 35, 6, 1, 211, - 110, 4, 208, 172, 23, 195, 169, 35, 6, 1, 211, 110, 4, 237, 215, 23, 195, - 169, 35, 6, 1, 211, 110, 4, 208, 172, 23, 210, 245, 35, 6, 1, 211, 110, - 4, 237, 215, 23, 210, 245, 35, 2, 1, 196, 71, 4, 208, 171, 35, 2, 1, 196, - 71, 4, 237, 214, 35, 2, 1, 196, 71, 4, 208, 172, 23, 195, 169, 35, 2, 1, - 196, 71, 4, 237, 215, 23, 195, 169, 35, 2, 1, 196, 71, 4, 208, 172, 23, - 210, 245, 35, 2, 1, 196, 71, 4, 237, 215, 23, 210, 245, 35, 2, 1, 196, - 71, 4, 208, 172, 23, 210, 6, 35, 2, 1, 196, 71, 4, 237, 215, 23, 210, 6, - 35, 6, 1, 196, 71, 4, 237, 214, 35, 6, 1, 196, 71, 4, 237, 215, 23, 195, - 169, 35, 6, 1, 196, 71, 4, 237, 215, 23, 210, 245, 35, 6, 1, 196, 71, 4, - 237, 215, 23, 210, 6, 35, 2, 1, 211, 113, 4, 208, 171, 35, 2, 1, 211, - 113, 4, 237, 214, 35, 2, 1, 211, 113, 4, 208, 172, 23, 195, 169, 35, 2, - 1, 211, 113, 4, 237, 215, 23, 195, 169, 35, 2, 1, 211, 113, 4, 208, 172, - 23, 210, 245, 35, 2, 1, 211, 113, 4, 237, 215, 23, 210, 245, 35, 2, 1, - 211, 113, 4, 208, 172, 23, 210, 6, 35, 2, 1, 211, 113, 4, 237, 215, 23, - 210, 6, 35, 6, 1, 211, 113, 4, 208, 171, 35, 6, 1, 211, 113, 4, 237, 214, - 35, 6, 1, 211, 113, 4, 208, 172, 23, 195, 169, 35, 6, 1, 211, 113, 4, - 237, 215, 23, 195, 169, 35, 6, 1, 211, 113, 4, 208, 172, 23, 210, 245, - 35, 6, 1, 211, 113, 4, 237, 215, 23, 210, 245, 35, 6, 1, 211, 113, 4, - 208, 172, 23, 210, 6, 35, 6, 1, 211, 113, 4, 237, 215, 23, 210, 6, 35, 2, - 1, 251, 163, 4, 195, 169, 35, 2, 1, 251, 163, 4, 210, 245, 35, 2, 1, 234, - 50, 4, 195, 169, 35, 2, 1, 234, 50, 4, 210, 245, 35, 2, 1, 234, 0, 4, - 195, 169, 35, 2, 1, 234, 0, 4, 210, 245, 35, 2, 1, 223, 86, 4, 195, 169, - 35, 2, 1, 223, 86, 4, 210, 245, 35, 2, 1, 211, 110, 4, 195, 169, 35, 2, - 1, 211, 110, 4, 210, 245, 35, 2, 1, 196, 71, 4, 195, 169, 35, 2, 1, 196, - 71, 4, 210, 245, 35, 2, 1, 211, 113, 4, 195, 169, 35, 2, 1, 211, 113, 4, - 210, 245, 35, 2, 1, 251, 163, 4, 208, 172, 23, 191, 233, 35, 2, 1, 251, - 163, 4, 237, 215, 23, 191, 233, 35, 2, 1, 251, 163, 4, 208, 172, 23, 195, - 170, 23, 191, 233, 35, 2, 1, 251, 163, 4, 237, 215, 23, 195, 170, 23, - 191, 233, 35, 2, 1, 251, 163, 4, 208, 172, 23, 210, 246, 23, 191, 233, - 35, 2, 1, 251, 163, 4, 237, 215, 23, 210, 246, 23, 191, 233, 35, 2, 1, - 251, 163, 4, 208, 172, 23, 210, 7, 23, 191, 233, 35, 2, 1, 251, 163, 4, - 237, 215, 23, 210, 7, 23, 191, 233, 35, 6, 1, 251, 163, 4, 208, 172, 23, - 208, 186, 35, 6, 1, 251, 163, 4, 237, 215, 23, 208, 186, 35, 6, 1, 251, - 163, 4, 208, 172, 23, 195, 170, 23, 208, 186, 35, 6, 1, 251, 163, 4, 237, - 215, 23, 195, 170, 23, 208, 186, 35, 6, 1, 251, 163, 4, 208, 172, 23, - 210, 246, 23, 208, 186, 35, 6, 1, 251, 163, 4, 237, 215, 23, 210, 246, - 23, 208, 186, 35, 6, 1, 251, 163, 4, 208, 172, 23, 210, 7, 23, 208, 186, - 35, 6, 1, 251, 163, 4, 237, 215, 23, 210, 7, 23, 208, 186, 35, 2, 1, 234, - 0, 4, 208, 172, 23, 191, 233, 35, 2, 1, 234, 0, 4, 237, 215, 23, 191, - 233, 35, 2, 1, 234, 0, 4, 208, 172, 23, 195, 170, 23, 191, 233, 35, 2, 1, - 234, 0, 4, 237, 215, 23, 195, 170, 23, 191, 233, 35, 2, 1, 234, 0, 4, - 208, 172, 23, 210, 246, 23, 191, 233, 35, 2, 1, 234, 0, 4, 237, 215, 23, - 210, 246, 23, 191, 233, 35, 2, 1, 234, 0, 4, 208, 172, 23, 210, 7, 23, - 191, 233, 35, 2, 1, 234, 0, 4, 237, 215, 23, 210, 7, 23, 191, 233, 35, 6, - 1, 234, 0, 4, 208, 172, 23, 208, 186, 35, 6, 1, 234, 0, 4, 237, 215, 23, - 208, 186, 35, 6, 1, 234, 0, 4, 208, 172, 23, 195, 170, 23, 208, 186, 35, - 6, 1, 234, 0, 4, 237, 215, 23, 195, 170, 23, 208, 186, 35, 6, 1, 234, 0, - 4, 208, 172, 23, 210, 246, 23, 208, 186, 35, 6, 1, 234, 0, 4, 237, 215, - 23, 210, 246, 23, 208, 186, 35, 6, 1, 234, 0, 4, 208, 172, 23, 210, 7, - 23, 208, 186, 35, 6, 1, 234, 0, 4, 237, 215, 23, 210, 7, 23, 208, 186, - 35, 2, 1, 211, 113, 4, 208, 172, 23, 191, 233, 35, 2, 1, 211, 113, 4, - 237, 215, 23, 191, 233, 35, 2, 1, 211, 113, 4, 208, 172, 23, 195, 170, - 23, 191, 233, 35, 2, 1, 211, 113, 4, 237, 215, 23, 195, 170, 23, 191, - 233, 35, 2, 1, 211, 113, 4, 208, 172, 23, 210, 246, 23, 191, 233, 35, 2, - 1, 211, 113, 4, 237, 215, 23, 210, 246, 23, 191, 233, 35, 2, 1, 211, 113, - 4, 208, 172, 23, 210, 7, 23, 191, 233, 35, 2, 1, 211, 113, 4, 237, 215, - 23, 210, 7, 23, 191, 233, 35, 6, 1, 211, 113, 4, 208, 172, 23, 208, 186, - 35, 6, 1, 211, 113, 4, 237, 215, 23, 208, 186, 35, 6, 1, 211, 113, 4, - 208, 172, 23, 195, 170, 23, 208, 186, 35, 6, 1, 211, 113, 4, 237, 215, - 23, 195, 170, 23, 208, 186, 35, 6, 1, 211, 113, 4, 208, 172, 23, 210, - 246, 23, 208, 186, 35, 6, 1, 211, 113, 4, 237, 215, 23, 210, 246, 23, - 208, 186, 35, 6, 1, 211, 113, 4, 208, 172, 23, 210, 7, 23, 208, 186, 35, - 6, 1, 211, 113, 4, 237, 215, 23, 210, 7, 23, 208, 186, 35, 2, 1, 251, - 163, 4, 194, 254, 35, 2, 1, 251, 163, 4, 217, 148, 35, 2, 1, 251, 163, 4, - 195, 170, 23, 191, 233, 35, 2, 1, 251, 163, 4, 191, 233, 35, 2, 1, 251, - 163, 4, 210, 246, 23, 191, 233, 35, 2, 1, 251, 163, 4, 210, 6, 35, 2, 1, - 251, 163, 4, 210, 7, 23, 191, 233, 35, 6, 1, 251, 163, 4, 194, 254, 35, - 6, 1, 251, 163, 4, 217, 148, 35, 6, 1, 251, 163, 4, 195, 169, 35, 6, 1, - 251, 163, 4, 210, 245, 35, 6, 1, 251, 163, 4, 208, 186, 35, 221, 32, 35, - 208, 186, 35, 208, 171, 35, 210, 6, 35, 237, 38, 23, 210, 6, 35, 2, 1, - 234, 0, 4, 195, 170, 23, 191, 233, 35, 2, 1, 234, 0, 4, 191, 233, 35, 2, - 1, 234, 0, 4, 210, 246, 23, 191, 233, 35, 2, 1, 234, 0, 4, 210, 6, 35, 2, - 1, 234, 0, 4, 210, 7, 23, 191, 233, 35, 6, 1, 234, 50, 4, 195, 169, 35, - 6, 1, 234, 50, 4, 210, 245, 35, 6, 1, 234, 0, 4, 195, 169, 35, 6, 1, 234, - 0, 4, 210, 245, 35, 6, 1, 234, 0, 4, 208, 186, 35, 208, 172, 23, 195, - 169, 35, 208, 172, 23, 210, 245, 35, 208, 172, 23, 210, 6, 35, 2, 1, 223, - 86, 4, 194, 254, 35, 2, 1, 223, 86, 4, 217, 148, 35, 2, 1, 223, 86, 4, - 237, 38, 23, 195, 169, 35, 2, 1, 223, 86, 4, 237, 38, 23, 210, 245, 35, - 2, 1, 223, 86, 4, 210, 6, 35, 2, 1, 223, 86, 4, 237, 38, 23, 210, 6, 35, - 6, 1, 223, 86, 4, 194, 254, 35, 6, 1, 223, 86, 4, 217, 148, 35, 6, 1, - 223, 86, 4, 195, 169, 35, 6, 1, 223, 86, 4, 210, 245, 35, 237, 215, 23, - 195, 169, 35, 237, 215, 23, 210, 245, 35, 237, 215, 23, 210, 6, 35, 2, 1, - 196, 71, 4, 194, 254, 35, 2, 1, 196, 71, 4, 217, 148, 35, 2, 1, 196, 71, - 4, 237, 38, 23, 195, 169, 35, 2, 1, 196, 71, 4, 237, 38, 23, 210, 245, - 35, 2, 1, 207, 5, 4, 208, 171, 35, 2, 1, 207, 5, 4, 237, 214, 35, 2, 1, - 196, 71, 4, 210, 6, 35, 2, 1, 196, 71, 4, 237, 38, 23, 210, 6, 35, 6, 1, - 196, 71, 4, 194, 254, 35, 6, 1, 196, 71, 4, 217, 148, 35, 6, 1, 196, 71, - 4, 195, 169, 35, 6, 1, 196, 71, 4, 210, 245, 35, 6, 1, 207, 5, 4, 237, - 214, 35, 237, 38, 23, 195, 169, 35, 237, 38, 23, 210, 245, 35, 195, 169, - 35, 2, 1, 211, 113, 4, 195, 170, 23, 191, 233, 35, 2, 1, 211, 113, 4, - 191, 233, 35, 2, 1, 211, 113, 4, 210, 246, 23, 191, 233, 35, 2, 1, 211, - 113, 4, 210, 6, 35, 2, 1, 211, 113, 4, 210, 7, 23, 191, 233, 35, 6, 1, - 211, 110, 4, 195, 169, 35, 6, 1, 211, 110, 4, 210, 245, 35, 6, 1, 211, - 113, 4, 195, 169, 35, 6, 1, 211, 113, 4, 210, 245, 35, 6, 1, 211, 113, 4, - 208, 186, 35, 210, 245, 35, 237, 214, 234, 106, 208, 32, 234, 117, 208, - 32, 234, 106, 201, 248, 234, 117, 201, 248, 198, 219, 201, 248, 232, 128, - 201, 248, 202, 135, 201, 248, 233, 15, 201, 248, 208, 154, 201, 248, 199, - 4, 201, 248, 230, 81, 201, 248, 191, 78, 193, 75, 201, 248, 191, 78, 193, - 75, 213, 31, 191, 78, 193, 75, 222, 198, 219, 217, 77, 206, 200, 77, 228, - 90, 213, 32, 228, 90, 233, 15, 237, 217, 234, 106, 237, 217, 234, 117, - 237, 217, 228, 243, 164, 55, 81, 219, 114, 55, 130, 219, 114, 45, 202, - 171, 207, 254, 77, 50, 202, 171, 207, 254, 77, 202, 171, 218, 242, 207, - 254, 77, 202, 171, 229, 134, 207, 254, 77, 45, 55, 207, 254, 77, 50, 55, - 207, 254, 77, 55, 218, 242, 207, 254, 77, 55, 229, 134, 207, 254, 77, - 238, 16, 55, 238, 16, 247, 250, 197, 238, 247, 250, 91, 75, 219, 238, - 105, 75, 219, 238, 228, 243, 234, 122, 228, 88, 209, 63, 219, 115, 204, - 11, 210, 130, 204, 11, 219, 217, 234, 115, 206, 200, 234, 115, 209, 36, - 236, 234, 232, 146, 219, 217, 210, 254, 206, 200, 210, 254, 214, 217, - 213, 39, 201, 248, 210, 16, 216, 52, 56, 210, 16, 199, 96, 198, 230, 56, - 208, 217, 55, 208, 217, 197, 225, 208, 217, 207, 19, 208, 217, 207, 19, - 55, 208, 217, 207, 19, 197, 225, 208, 217, 247, 98, 202, 171, 219, 221, - 251, 118, 207, 254, 77, 202, 171, 206, 204, 251, 118, 207, 254, 77, 207, - 85, 77, 55, 233, 218, 77, 223, 104, 211, 0, 196, 100, 246, 242, 198, 178, - 247, 99, 223, 121, 209, 63, 250, 193, 228, 91, 247, 250, 232, 120, 202, - 98, 45, 51, 248, 56, 4, 208, 9, 50, 51, 248, 56, 4, 208, 9, 55, 208, 15, - 77, 208, 15, 233, 218, 77, 233, 218, 208, 15, 77, 198, 128, 3, 234, 1, - 207, 19, 209, 144, 56, 62, 118, 247, 250, 62, 96, 247, 250, 130, 250, - 195, 207, 19, 204, 26, 242, 222, 196, 77, 105, 250, 194, 251, 180, 195, - 84, 242, 74, 216, 37, 56, 200, 129, 237, 217, 223, 95, 196, 100, 232, - 189, 208, 154, 77, 115, 75, 208, 153, 208, 28, 208, 217, 232, 130, 75, - 208, 153, 232, 228, 75, 208, 153, 105, 75, 208, 153, 232, 130, 75, 77, - 235, 140, 238, 219, 197, 237, 81, 232, 130, 236, 140, 216, 215, 13, 201, - 248, 193, 23, 222, 198, 232, 79, 251, 46, 223, 93, 198, 144, 223, 93, - 204, 11, 223, 93, 209, 83, 219, 217, 223, 61, 206, 200, 223, 61, 232, - 240, 201, 15, 223, 61, 209, 36, 236, 234, 223, 61, 223, 134, 200, 75, - 200, 147, 252, 51, 200, 75, 200, 147, 223, 134, 9, 232, 148, 203, 134, - 252, 51, 9, 232, 148, 203, 134, 214, 210, 17, 203, 135, 213, 35, 17, 203, - 135, 200, 182, 191, 77, 200, 182, 8, 2, 1, 68, 200, 182, 134, 200, 182, - 150, 200, 182, 169, 200, 182, 175, 200, 182, 171, 200, 182, 178, 200, - 182, 108, 56, 200, 182, 216, 36, 200, 182, 234, 45, 56, 200, 182, 45, - 210, 115, 200, 182, 50, 210, 115, 200, 182, 8, 2, 1, 215, 63, 200, 231, - 191, 77, 200, 231, 107, 200, 231, 109, 200, 231, 138, 200, 231, 134, 200, - 231, 150, 200, 231, 169, 200, 231, 175, 200, 231, 171, 200, 231, 178, - 200, 231, 108, 56, 200, 231, 216, 36, 200, 231, 234, 45, 56, 200, 231, - 45, 210, 115, 200, 231, 50, 210, 115, 8, 200, 231, 2, 1, 65, 8, 200, 231, - 2, 1, 71, 8, 200, 231, 2, 1, 74, 8, 200, 231, 2, 1, 192, 235, 8, 200, - 231, 2, 1, 205, 87, 8, 200, 231, 2, 1, 230, 118, 8, 200, 231, 2, 1, 222, - 154, 8, 200, 231, 2, 1, 172, 8, 200, 231, 2, 1, 218, 170, 8, 200, 231, 2, - 1, 215, 63, 8, 200, 231, 2, 1, 210, 238, 8, 200, 231, 2, 1, 206, 9, 8, - 200, 231, 2, 1, 200, 43, 233, 235, 56, 242, 86, 56, 238, 202, 56, 232, - 108, 232, 113, 56, 219, 93, 56, 216, 53, 56, 214, 236, 56, 209, 247, 56, - 206, 37, 56, 193, 31, 56, 214, 82, 203, 100, 56, 236, 151, 56, 233, 236, - 56, 221, 137, 56, 197, 81, 56, 235, 118, 56, 231, 140, 210, 29, 56, 209, - 244, 56, 230, 176, 56, 250, 155, 56, 228, 168, 56, 247, 39, 56, 219, 83, - 198, 33, 56, 201, 227, 56, 199, 93, 56, 223, 149, 206, 37, 56, 197, 60, - 219, 93, 56, 213, 21, 87, 56, 217, 90, 56, 206, 60, 56, 220, 11, 56, 248, - 149, 56, 202, 23, 56, 33, 45, 230, 15, 58, 33, 50, 230, 15, 58, 33, 180, - 81, 219, 115, 211, 1, 33, 203, 41, 81, 219, 115, 211, 1, 33, 251, 87, 64, - 58, 33, 242, 223, 64, 58, 33, 45, 64, 58, 33, 50, 64, 58, 33, 206, 190, - 211, 1, 33, 242, 223, 206, 190, 211, 1, 33, 251, 87, 206, 190, 211, 1, - 33, 115, 185, 58, 33, 232, 130, 185, 58, 33, 234, 101, 243, 12, 33, 234, - 101, 201, 191, 33, 234, 101, 237, 34, 33, 234, 101, 243, 13, 249, 143, - 33, 45, 50, 64, 58, 33, 234, 101, 205, 77, 33, 234, 101, 221, 220, 33, - 234, 101, 196, 68, 209, 60, 197, 241, 33, 207, 20, 202, 25, 211, 1, 33, - 55, 81, 201, 29, 211, 1, 33, 251, 97, 113, 33, 197, 225, 196, 102, 33, - 193, 78, 248, 31, 58, 33, 118, 64, 211, 1, 33, 180, 55, 202, 25, 211, 1, - 33, 96, 230, 15, 4, 182, 235, 120, 33, 118, 230, 15, 4, 182, 235, 120, - 33, 45, 64, 60, 33, 50, 64, 60, 33, 250, 196, 58, 252, 57, 211, 148, 252, - 40, 119, 199, 34, 200, 241, 235, 131, 6, 247, 195, 237, 127, 247, 29, - 247, 24, 219, 115, 113, 247, 100, 211, 148, 247, 154, 196, 112, 233, 237, - 239, 40, 205, 73, 237, 127, 233, 93, 27, 2, 232, 53, 27, 6, 230, 118, - 248, 139, 6, 230, 118, 235, 131, 6, 230, 118, 209, 105, 239, 40, 209, - 105, 239, 41, 139, 105, 209, 187, 27, 6, 68, 248, 139, 6, 68, 27, 6, 172, - 27, 2, 172, 220, 145, 78, 249, 90, 113, 235, 131, 6, 215, 63, 212, 134, - 56, 202, 6, 207, 97, 239, 7, 27, 6, 210, 238, 235, 131, 6, 210, 238, 235, - 131, 6, 208, 106, 27, 6, 146, 248, 139, 6, 146, 235, 131, 6, 146, 208, - 225, 199, 208, 207, 32, 204, 2, 77, 199, 107, 56, 198, 22, 87, 56, 195, - 136, 235, 131, 6, 191, 166, 211, 20, 56, 211, 137, 56, 223, 95, 211, 137, - 56, 248, 139, 6, 191, 166, 154, 35, 2, 1, 223, 85, 222, 5, 56, 251, 112, - 56, 27, 6, 250, 122, 248, 139, 6, 247, 195, 234, 6, 113, 27, 2, 71, 27, - 6, 71, 27, 6, 233, 177, 154, 6, 233, 177, 27, 6, 218, 170, 27, 2, 74, - 131, 113, 248, 217, 113, 231, 41, 113, 238, 0, 113, 223, 139, 202, 4, - 206, 123, 6, 208, 106, 233, 96, 56, 235, 131, 2, 209, 187, 235, 131, 2, - 231, 213, 235, 131, 6, 231, 213, 235, 131, 6, 209, 187, 235, 131, 215, - 62, 200, 201, 154, 49, 6, 232, 53, 154, 49, 6, 172, 207, 19, 49, 6, 172, - 154, 49, 6, 192, 159, 235, 131, 43, 6, 238, 129, 235, 131, 43, 2, 238, - 129, 235, 131, 43, 2, 71, 235, 131, 43, 2, 68, 235, 131, 43, 2, 223, 37, - 208, 190, 219, 114, 154, 251, 139, 210, 16, 56, 251, 212, 154, 2, 233, - 177, 16, 40, 205, 152, 202, 4, 193, 246, 232, 120, 91, 203, 244, 193, - 246, 232, 120, 91, 213, 169, 193, 246, 232, 120, 91, 199, 86, 193, 246, - 232, 120, 91, 199, 0, 193, 246, 232, 120, 105, 198, 253, 193, 246, 232, - 120, 91, 233, 20, 193, 246, 232, 120, 105, 233, 19, 193, 246, 232, 120, - 115, 233, 19, 193, 246, 232, 120, 232, 130, 233, 19, 193, 246, 232, 120, - 91, 202, 125, 193, 246, 232, 120, 232, 228, 202, 123, 193, 246, 232, 120, - 91, 234, 161, 193, 246, 232, 120, 115, 234, 159, 193, 246, 232, 120, 232, - 228, 234, 159, 193, 246, 232, 120, 203, 248, 234, 159, 232, 120, 212, - 135, 107, 206, 137, 212, 136, 107, 206, 137, 212, 136, 109, 206, 137, - 212, 136, 138, 206, 137, 212, 136, 134, 206, 137, 212, 136, 150, 206, - 137, 212, 136, 169, 206, 137, 212, 136, 175, 206, 137, 212, 136, 171, - 206, 137, 212, 136, 178, 206, 137, 212, 136, 199, 95, 206, 137, 212, 136, - 234, 129, 206, 137, 212, 136, 197, 37, 206, 137, 212, 136, 233, 17, 206, - 137, 212, 136, 91, 228, 142, 206, 137, 212, 136, 232, 228, 228, 142, 206, - 137, 212, 136, 91, 189, 2, 206, 137, 212, 136, 107, 2, 206, 137, 212, - 136, 109, 2, 206, 137, 212, 136, 138, 2, 206, 137, 212, 136, 134, 2, 206, - 137, 212, 136, 150, 2, 206, 137, 212, 136, 169, 2, 206, 137, 212, 136, - 175, 2, 206, 137, 212, 136, 171, 2, 206, 137, 212, 136, 178, 2, 206, 137, - 212, 136, 199, 95, 2, 206, 137, 212, 136, 234, 129, 2, 206, 137, 212, - 136, 197, 37, 2, 206, 137, 212, 136, 233, 17, 2, 206, 137, 212, 136, 91, - 228, 142, 2, 206, 137, 212, 136, 232, 228, 228, 142, 2, 206, 137, 212, - 136, 91, 189, 206, 137, 212, 136, 91, 198, 230, 247, 196, 238, 129, 206, - 137, 212, 136, 232, 228, 189, 206, 137, 212, 136, 199, 96, 189, 206, 137, - 212, 136, 207, 19, 91, 228, 142, 8, 2, 1, 207, 19, 247, 195, 206, 137, - 212, 136, 202, 137, 220, 6, 20, 206, 137, 212, 136, 233, 18, 234, 212, - 20, 206, 137, 212, 136, 233, 18, 189, 206, 137, 212, 136, 91, 228, 143, - 189, 193, 246, 232, 120, 191, 78, 198, 253, 154, 17, 109, 154, 17, 138, - 118, 57, 196, 66, 57, 96, 57, 235, 121, 57, 45, 50, 57, 133, 144, 57, - 186, 193, 105, 57, 186, 234, 206, 57, 202, 3, 234, 206, 57, 202, 3, 193, - 105, 57, 118, 64, 4, 106, 96, 64, 4, 106, 118, 193, 139, 57, 96, 193, - 139, 57, 118, 105, 229, 235, 57, 196, 66, 105, 229, 235, 57, 96, 105, - 229, 235, 57, 235, 121, 105, 229, 235, 57, 118, 64, 4, 199, 215, 96, 64, - 4, 199, 215, 118, 64, 232, 100, 164, 196, 66, 64, 232, 100, 164, 96, 64, - 232, 100, 164, 235, 121, 64, 232, 100, 164, 133, 144, 64, 4, 249, 76, - 118, 64, 4, 102, 96, 64, 4, 102, 118, 64, 4, 219, 6, 96, 64, 4, 219, 6, - 45, 50, 193, 139, 57, 45, 50, 64, 4, 106, 235, 121, 191, 21, 57, 196, 66, - 64, 4, 198, 136, 219, 216, 196, 66, 64, 4, 198, 136, 206, 198, 235, 121, - 64, 4, 198, 136, 219, 216, 235, 121, 64, 4, 198, 136, 206, 198, 96, 64, - 4, 239, 4, 235, 120, 235, 121, 64, 4, 239, 4, 219, 216, 251, 87, 198, 54, - 204, 29, 57, 242, 223, 198, 54, 204, 29, 57, 186, 193, 105, 64, 119, 180, - 164, 118, 64, 119, 249, 90, 139, 96, 64, 119, 164, 251, 87, 211, 79, 243, - 13, 57, 242, 223, 211, 79, 243, 13, 57, 118, 230, 15, 4, 182, 196, 65, - 118, 230, 15, 4, 182, 235, 120, 196, 66, 230, 15, 4, 182, 206, 198, 196, - 66, 230, 15, 4, 182, 219, 216, 96, 230, 15, 4, 182, 196, 65, 96, 230, 15, - 4, 182, 235, 120, 235, 121, 230, 15, 4, 182, 206, 198, 235, 121, 230, 15, - 4, 182, 219, 216, 96, 64, 139, 118, 57, 196, 66, 64, 118, 79, 235, 121, - 57, 118, 64, 139, 96, 57, 118, 210, 198, 250, 233, 196, 66, 210, 198, - 250, 233, 96, 210, 198, 250, 233, 235, 121, 210, 198, 250, 233, 118, 230, - 15, 139, 96, 230, 14, 96, 230, 15, 139, 118, 230, 14, 118, 55, 64, 4, - 106, 45, 50, 55, 64, 4, 106, 96, 55, 64, 4, 106, 118, 55, 57, 196, 66, - 55, 57, 96, 55, 57, 235, 121, 55, 57, 45, 50, 55, 57, 133, 144, 55, 57, - 186, 193, 105, 55, 57, 186, 234, 206, 55, 57, 202, 3, 234, 206, 55, 57, - 202, 3, 193, 105, 55, 57, 118, 197, 225, 57, 96, 197, 225, 57, 118, 201, - 184, 57, 96, 201, 184, 57, 196, 66, 64, 4, 55, 106, 235, 121, 64, 4, 55, - 106, 118, 237, 216, 57, 196, 66, 237, 216, 57, 96, 237, 216, 57, 235, - 121, 237, 216, 57, 118, 64, 119, 164, 96, 64, 119, 164, 118, 63, 57, 196, - 66, 63, 57, 96, 63, 57, 235, 121, 63, 57, 196, 66, 63, 64, 232, 100, 164, - 196, 66, 63, 64, 211, 107, 210, 54, 196, 66, 63, 64, 211, 107, 210, 55, - 4, 228, 243, 164, 196, 66, 63, 64, 211, 107, 210, 55, 4, 81, 164, 196, - 66, 63, 55, 57, 196, 66, 63, 55, 64, 211, 107, 210, 54, 96, 63, 64, 232, - 100, 193, 167, 186, 193, 105, 64, 119, 239, 3, 202, 3, 234, 206, 64, 119, - 239, 3, 133, 144, 63, 57, 50, 64, 4, 2, 243, 12, 235, 121, 64, 118, 79, - 196, 66, 57, 115, 96, 250, 233, 118, 64, 4, 81, 106, 96, 64, 4, 81, 106, - 45, 50, 64, 4, 81, 106, 118, 64, 4, 55, 81, 106, 96, 64, 4, 55, 81, 106, - 45, 50, 64, 4, 55, 81, 106, 118, 211, 76, 57, 96, 211, 76, 57, 45, 50, - 211, 76, 57, 40, 251, 176, 242, 70, 210, 107, 237, 18, 199, 24, 233, 213, - 199, 24, 236, 166, 213, 14, 233, 214, 234, 107, 203, 253, 223, 153, 214, - 247, 234, 134, 211, 148, 213, 14, 251, 135, 234, 134, 211, 148, 2, 234, - 134, 211, 148, 239, 34, 250, 222, 216, 192, 236, 166, 213, 14, 239, 36, - 250, 222, 216, 192, 2, 239, 34, 250, 222, 216, 192, 234, 97, 79, 208, - 192, 215, 62, 208, 202, 215, 62, 239, 11, 215, 62, 200, 201, 216, 37, 56, - 216, 35, 56, 75, 209, 83, 236, 202, 202, 98, 203, 254, 216, 36, 250, 196, - 211, 68, 206, 190, 211, 68, 247, 251, 211, 68, 51, 206, 129, 238, 193, - 206, 129, 232, 123, 206, 129, 208, 188, 159, 223, 141, 50, 251, 117, 251, - 117, 216, 228, 251, 117, 201, 226, 251, 117, 236, 205, 236, 166, 213, 14, - 236, 209, 210, 121, 159, 213, 14, 210, 121, 159, 219, 31, 251, 127, 219, - 31, 211, 58, 223, 101, 196, 92, 223, 115, 55, 223, 115, 197, 225, 223, - 115, 239, 28, 223, 115, 200, 171, 223, 115, 195, 11, 223, 115, 242, 223, - 223, 115, 242, 223, 239, 28, 223, 115, 251, 87, 239, 28, 223, 115, 199, - 23, 249, 5, 207, 128, 208, 189, 75, 216, 36, 233, 221, 231, 146, 208, - 189, 229, 2, 198, 153, 211, 68, 207, 19, 198, 152, 223, 95, 219, 247, - 206, 9, 202, 173, 193, 138, 193, 10, 208, 202, 213, 14, 198, 152, 216, - 37, 198, 152, 250, 188, 234, 47, 159, 213, 14, 250, 188, 234, 47, 159, - 251, 42, 234, 47, 159, 251, 42, 247, 220, 213, 14, 252, 50, 234, 47, 159, - 214, 107, 251, 42, 213, 23, 252, 50, 234, 47, 159, 251, 167, 234, 47, - 159, 213, 14, 251, 167, 234, 47, 159, 251, 167, 234, 47, 211, 59, 234, - 47, 159, 197, 225, 198, 152, 251, 177, 234, 47, 159, 234, 38, 159, 231, - 145, 234, 38, 159, 237, 19, 248, 211, 251, 44, 199, 34, 219, 122, 231, - 145, 234, 47, 159, 251, 42, 234, 47, 119, 211, 59, 199, 34, 223, 180, - 211, 148, 223, 180, 79, 211, 59, 251, 42, 234, 47, 159, 242, 86, 234, 44, - 234, 45, 242, 85, 206, 190, 223, 165, 234, 47, 159, 206, 190, 234, 47, - 159, 238, 252, 159, 234, 5, 234, 43, 159, 201, 104, 234, 44, 237, 109, - 234, 47, 159, 234, 47, 119, 247, 207, 237, 128, 216, 228, 247, 206, 208, - 13, 234, 47, 159, 213, 14, 234, 47, 159, 228, 19, 159, 213, 14, 228, 19, - 159, 201, 36, 234, 38, 159, 219, 182, 211, 59, 234, 47, 159, 230, 206, - 211, 59, 234, 47, 159, 219, 182, 139, 234, 47, 159, 230, 206, 139, 234, - 47, 159, 219, 182, 247, 220, 213, 14, 234, 47, 159, 230, 206, 247, 220, - 213, 14, 234, 47, 159, 215, 147, 219, 181, 215, 147, 230, 205, 248, 211, - 213, 14, 234, 38, 159, 213, 14, 219, 181, 213, 14, 230, 205, 214, 107, - 219, 182, 213, 23, 234, 47, 159, 214, 107, 230, 206, 213, 23, 234, 47, - 159, 219, 182, 211, 59, 234, 38, 159, 230, 206, 211, 59, 234, 38, 159, - 214, 107, 219, 182, 213, 23, 234, 38, 159, 214, 107, 230, 206, 213, 23, - 234, 38, 159, 219, 182, 211, 59, 230, 205, 230, 206, 211, 59, 219, 181, - 214, 107, 219, 182, 213, 23, 230, 205, 214, 107, 230, 206, 213, 23, 219, - 181, 208, 233, 200, 220, 208, 234, 211, 59, 234, 47, 159, 200, 221, 211, - 59, 234, 47, 159, 208, 234, 211, 59, 234, 38, 159, 200, 221, 211, 59, - 234, 38, 159, 236, 166, 213, 14, 208, 236, 236, 166, 213, 14, 200, 222, - 200, 230, 211, 148, 200, 181, 211, 148, 213, 14, 42, 200, 230, 211, 148, - 213, 14, 42, 200, 181, 211, 148, 200, 230, 79, 211, 59, 234, 47, 159, - 200, 181, 79, 211, 59, 234, 47, 159, 214, 107, 42, 200, 230, 79, 213, 23, - 234, 47, 159, 214, 107, 42, 200, 181, 79, 213, 23, 234, 47, 159, 200, - 230, 79, 4, 213, 14, 234, 47, 159, 200, 181, 79, 4, 213, 14, 234, 47, - 159, 215, 126, 215, 127, 215, 128, 215, 127, 196, 92, 51, 223, 180, 211, - 148, 51, 211, 48, 211, 148, 51, 223, 180, 79, 211, 59, 234, 47, 159, 51, - 211, 48, 79, 211, 59, 234, 47, 159, 51, 247, 113, 51, 238, 183, 47, 209, - 83, 47, 216, 36, 47, 198, 144, 47, 236, 202, 202, 98, 47, 75, 211, 68, - 47, 206, 190, 211, 68, 47, 250, 196, 211, 68, 47, 234, 44, 47, 237, 217, - 112, 209, 83, 112, 216, 36, 112, 198, 144, 112, 75, 211, 68, 50, 199, - 228, 45, 199, 228, 144, 199, 228, 133, 199, 228, 250, 199, 216, 3, 197, - 201, 232, 154, 197, 225, 81, 249, 90, 50, 197, 57, 55, 81, 249, 90, 55, - 50, 197, 57, 236, 166, 213, 14, 208, 181, 213, 14, 197, 201, 236, 166, - 213, 14, 232, 155, 214, 110, 55, 81, 249, 90, 55, 50, 197, 57, 208, 234, - 196, 105, 207, 66, 200, 221, 196, 105, 207, 66, 213, 20, 200, 245, 211, - 148, 239, 34, 250, 222, 213, 20, 200, 244, 213, 20, 200, 245, 79, 211, - 59, 234, 47, 159, 239, 34, 250, 222, 213, 20, 200, 245, 211, 59, 234, 47, - 159, 211, 48, 211, 148, 223, 180, 211, 148, 215, 133, 229, 191, 239, 45, - 217, 29, 223, 112, 192, 192, 214, 226, 213, 22, 50, 251, 118, 4, 251, 18, - 50, 197, 241, 215, 62, 219, 31, 251, 127, 215, 62, 219, 31, 211, 58, 215, - 62, 223, 101, 215, 62, 196, 92, 237, 35, 211, 68, 75, 211, 68, 201, 104, - 211, 68, 236, 202, 198, 144, 248, 65, 45, 213, 20, 233, 95, 204, 25, 208, - 202, 50, 213, 20, 233, 95, 204, 25, 208, 202, 45, 204, 25, 208, 202, 50, - 204, 25, 208, 202, 207, 19, 198, 153, 234, 44, 238, 173, 219, 31, 211, - 58, 238, 173, 219, 31, 251, 127, 55, 200, 229, 55, 200, 180, 55, 223, - 101, 55, 196, 92, 209, 117, 234, 47, 23, 210, 121, 159, 219, 182, 4, 236, - 142, 230, 206, 4, 236, 142, 195, 83, 215, 147, 219, 181, 195, 83, 215, - 147, 230, 205, 219, 182, 234, 47, 119, 211, 59, 230, 205, 230, 206, 234, - 47, 119, 211, 59, 219, 181, 234, 47, 119, 211, 59, 219, 181, 234, 47, - 119, 211, 59, 230, 205, 234, 47, 119, 211, 59, 208, 233, 234, 47, 119, - 211, 59, 200, 220, 236, 166, 213, 14, 208, 237, 211, 59, 234, 46, 236, - 166, 213, 14, 200, 223, 211, 59, 234, 46, 213, 14, 51, 223, 180, 79, 211, - 59, 234, 47, 159, 213, 14, 51, 211, 48, 79, 211, 59, 234, 47, 159, 51, - 223, 180, 79, 211, 59, 213, 14, 234, 47, 159, 51, 211, 48, 79, 211, 59, - 213, 14, 234, 47, 159, 219, 182, 247, 220, 213, 14, 234, 38, 159, 230, - 206, 247, 220, 213, 14, 234, 38, 159, 208, 234, 247, 220, 213, 14, 234, - 38, 159, 200, 221, 247, 220, 213, 14, 234, 38, 159, 213, 14, 213, 20, - 200, 245, 211, 148, 236, 166, 213, 14, 239, 36, 250, 222, 213, 20, 200, - 244, 213, 14, 213, 20, 200, 245, 79, 211, 59, 234, 47, 159, 236, 166, - 213, 14, 239, 36, 250, 222, 213, 20, 200, 245, 211, 59, 234, 46, 81, 234, - 122, 216, 84, 228, 243, 234, 122, 133, 50, 237, 41, 234, 122, 144, 50, - 237, 41, 234, 122, 234, 134, 79, 4, 180, 228, 243, 106, 234, 134, 79, 4, - 81, 249, 90, 250, 185, 234, 97, 79, 228, 243, 106, 2, 234, 134, 79, 4, - 81, 249, 90, 250, 185, 234, 97, 79, 228, 243, 106, 234, 134, 79, 4, 75, - 58, 234, 134, 79, 4, 211, 8, 2, 234, 134, 79, 4, 211, 8, 234, 134, 79, 4, - 196, 103, 234, 134, 79, 4, 105, 228, 243, 201, 16, 239, 34, 4, 180, 228, - 243, 106, 239, 34, 4, 81, 249, 90, 250, 185, 234, 97, 79, 228, 243, 106, - 2, 239, 34, 4, 81, 249, 90, 250, 185, 234, 97, 79, 228, 243, 106, 239, - 34, 4, 211, 8, 2, 239, 34, 4, 211, 8, 191, 167, 213, 12, 249, 133, 216, - 191, 237, 36, 56, 234, 137, 57, 228, 174, 133, 250, 237, 144, 250, 237, - 208, 196, 209, 250, 193, 135, 219, 114, 45, 247, 32, 50, 247, 32, 45, - 232, 195, 50, 232, 195, 248, 79, 50, 238, 221, 248, 79, 45, 238, 221, - 198, 54, 50, 238, 221, 198, 54, 45, 238, 221, 207, 19, 213, 14, 56, 51, - 218, 234, 251, 18, 205, 44, 205, 53, 199, 107, 207, 98, 209, 27, 223, - 146, 195, 56, 201, 191, 209, 110, 79, 223, 111, 56, 154, 213, 14, 56, - 193, 145, 228, 176, 198, 54, 45, 239, 3, 198, 54, 50, 239, 3, 248, 79, - 45, 239, 3, 248, 79, 50, 239, 3, 198, 54, 132, 223, 115, 248, 79, 132, - 223, 115, 232, 95, 202, 66, 133, 250, 238, 248, 212, 105, 228, 243, 249, - 78, 211, 61, 221, 224, 234, 34, 119, 199, 34, 179, 192, 236, 223, 165, - 42, 207, 95, 248, 64, 221, 222, 219, 221, 251, 118, 248, 55, 206, 204, - 251, 118, 248, 55, 234, 34, 119, 199, 34, 219, 226, 248, 223, 206, 189, - 238, 140, 251, 177, 250, 246, 200, 74, 198, 39, 206, 42, 236, 254, 211, - 49, 239, 50, 199, 181, 202, 82, 238, 248, 238, 247, 251, 61, 232, 77, 16, - 228, 69, 251, 61, 232, 77, 16, 201, 182, 208, 32, 251, 61, 232, 77, 16, - 208, 33, 234, 46, 251, 61, 232, 77, 16, 208, 33, 236, 209, 251, 61, 232, - 77, 16, 208, 33, 237, 34, 251, 61, 232, 77, 16, 208, 33, 222, 190, 251, - 61, 232, 77, 16, 208, 33, 243, 12, 251, 61, 232, 77, 16, 243, 13, 201, - 72, 251, 61, 232, 77, 16, 243, 13, 222, 190, 251, 61, 232, 77, 16, 202, - 99, 164, 251, 61, 232, 77, 16, 249, 144, 164, 251, 61, 232, 77, 16, 208, - 33, 202, 98, 251, 61, 232, 77, 16, 208, 33, 249, 143, 251, 61, 232, 77, - 16, 208, 33, 219, 181, 251, 61, 232, 77, 16, 208, 33, 230, 205, 251, 61, - 232, 77, 16, 118, 195, 176, 251, 61, 232, 77, 16, 96, 195, 176, 251, 61, - 232, 77, 16, 208, 33, 118, 57, 251, 61, 232, 77, 16, 208, 33, 96, 57, - 251, 61, 232, 77, 16, 243, 13, 249, 143, 251, 61, 232, 77, 16, 144, 199, - 229, 196, 103, 251, 61, 232, 77, 16, 237, 109, 201, 72, 251, 61, 232, 77, - 16, 208, 33, 144, 247, 98, 251, 61, 232, 77, 16, 208, 33, 237, 108, 251, - 61, 232, 77, 16, 144, 199, 229, 222, 190, 251, 61, 232, 77, 16, 196, 66, - 195, 176, 251, 61, 232, 77, 16, 208, 33, 196, 66, 57, 251, 61, 232, 77, - 16, 133, 199, 229, 211, 8, 251, 61, 232, 77, 16, 237, 121, 201, 72, 251, - 61, 232, 77, 16, 208, 33, 133, 247, 98, 251, 61, 232, 77, 16, 208, 33, - 237, 120, 251, 61, 232, 77, 16, 133, 199, 229, 222, 190, 251, 61, 232, - 77, 16, 235, 121, 195, 176, 251, 61, 232, 77, 16, 208, 33, 235, 121, 57, - 251, 61, 232, 77, 16, 207, 253, 196, 103, 251, 61, 232, 77, 16, 237, 109, - 196, 103, 251, 61, 232, 77, 16, 237, 35, 196, 103, 251, 61, 232, 77, 16, - 222, 191, 196, 103, 251, 61, 232, 77, 16, 243, 13, 196, 103, 251, 61, - 232, 77, 16, 133, 203, 54, 222, 190, 251, 61, 232, 77, 16, 207, 253, 208, - 32, 251, 61, 232, 77, 16, 243, 13, 201, 103, 251, 61, 232, 77, 16, 208, - 33, 242, 85, 251, 61, 232, 77, 16, 133, 199, 229, 237, 44, 251, 61, 232, - 77, 16, 237, 121, 237, 44, 251, 61, 232, 77, 16, 201, 104, 237, 44, 251, - 61, 232, 77, 16, 222, 191, 237, 44, 251, 61, 232, 77, 16, 243, 13, 237, - 44, 251, 61, 232, 77, 16, 144, 203, 54, 201, 72, 251, 61, 232, 77, 16, - 45, 203, 54, 201, 72, 251, 61, 232, 77, 16, 198, 153, 237, 44, 251, 61, - 232, 77, 16, 230, 206, 237, 44, 251, 61, 232, 77, 16, 242, 77, 164, 251, - 61, 232, 77, 16, 237, 121, 198, 152, 251, 61, 232, 77, 16, 191, 20, 251, - 61, 232, 77, 16, 201, 73, 198, 152, 251, 61, 232, 77, 16, 204, 27, 196, - 103, 251, 61, 232, 77, 16, 208, 33, 213, 14, 234, 46, 251, 61, 232, 77, - 16, 208, 33, 208, 14, 251, 61, 232, 77, 16, 144, 247, 99, 198, 152, 251, - 61, 232, 77, 16, 133, 247, 99, 198, 152, 251, 61, 232, 77, 16, 223, 85, - 251, 61, 232, 77, 16, 207, 4, 251, 61, 232, 77, 16, 211, 112, 251, 61, - 232, 77, 16, 251, 163, 196, 103, 251, 61, 232, 77, 16, 234, 50, 196, 103, - 251, 61, 232, 77, 16, 223, 86, 196, 103, 251, 61, 232, 77, 16, 211, 113, - 196, 103, 251, 61, 232, 77, 16, 251, 162, 213, 14, 243, 128, 77, 50, 251, - 118, 4, 235, 121, 191, 21, 57, 203, 22, 211, 79, 248, 64, 248, 238, 113, - 81, 219, 115, 4, 82, 236, 142, 223, 121, 113, 239, 29, 196, 101, 113, - 236, 227, 196, 101, 113, 234, 109, 113, 239, 65, 113, 63, 51, 4, 247, 24, - 81, 219, 114, 234, 80, 113, 251, 154, 221, 225, 113, 229, 204, 113, 47, - 228, 243, 249, 90, 4, 213, 11, 47, 197, 242, 235, 125, 248, 24, 243, 13, - 4, 213, 17, 57, 196, 99, 113, 215, 218, 113, 228, 86, 113, 211, 77, 230, - 117, 113, 211, 77, 220, 143, 113, 210, 95, 113, 210, 94, 113, 236, 236, - 238, 171, 16, 232, 148, 109, 202, 30, 113, 251, 61, 232, 77, 16, 208, 32, - 237, 140, 204, 12, 221, 225, 113, 208, 219, 210, 206, 214, 75, 210, 206, - 208, 214, 205, 78, 113, 242, 240, 205, 78, 113, 45, 210, 116, 116, 102, - 45, 210, 116, 233, 205, 45, 210, 116, 110, 102, 50, 210, 116, 116, 102, - 50, 210, 116, 233, 205, 50, 210, 116, 110, 102, 45, 51, 248, 56, 116, - 239, 3, 45, 51, 248, 56, 233, 205, 45, 51, 248, 56, 110, 239, 3, 50, 51, - 248, 56, 116, 239, 3, 50, 51, 248, 56, 233, 205, 50, 51, 248, 56, 110, - 239, 3, 45, 238, 173, 248, 56, 116, 102, 45, 238, 173, 248, 56, 82, 209, - 177, 45, 238, 173, 248, 56, 110, 102, 238, 173, 248, 56, 233, 205, 50, - 238, 173, 248, 56, 116, 102, 50, 238, 173, 248, 56, 82, 209, 177, 50, - 238, 173, 248, 56, 110, 102, 223, 116, 233, 205, 228, 243, 219, 115, 233, - 205, 116, 45, 211, 59, 110, 50, 238, 173, 248, 56, 205, 54, 116, 50, 211, - 59, 110, 45, 238, 173, 248, 56, 205, 54, 200, 202, 198, 53, 200, 202, - 248, 78, 198, 54, 51, 248, 55, 248, 79, 51, 248, 55, 248, 79, 51, 248, - 56, 139, 198, 54, 51, 248, 55, 48, 16, 248, 78, 45, 81, 111, 219, 114, - 50, 81, 111, 219, 114, 228, 243, 205, 98, 219, 113, 228, 243, 205, 98, - 219, 112, 228, 243, 205, 98, 219, 111, 228, 243, 205, 98, 219, 110, 237, - 99, 16, 156, 81, 23, 198, 54, 179, 237, 99, 16, 156, 81, 23, 248, 79, - 179, 237, 99, 16, 156, 81, 4, 243, 12, 237, 99, 16, 156, 144, 23, 228, - 243, 4, 243, 12, 237, 99, 16, 156, 133, 23, 228, 243, 4, 243, 12, 237, - 99, 16, 156, 81, 4, 197, 241, 237, 99, 16, 156, 144, 23, 228, 243, 4, - 197, 241, 237, 99, 16, 156, 133, 23, 228, 243, 4, 197, 241, 237, 99, 16, - 156, 81, 23, 193, 138, 237, 99, 16, 156, 144, 23, 228, 243, 4, 193, 138, - 237, 99, 16, 156, 133, 23, 228, 243, 4, 193, 138, 237, 99, 16, 156, 144, - 23, 228, 242, 237, 99, 16, 156, 133, 23, 228, 242, 237, 99, 16, 156, 81, - 23, 198, 54, 219, 226, 237, 99, 16, 156, 81, 23, 248, 79, 219, 226, 51, - 232, 161, 207, 24, 113, 234, 151, 113, 81, 219, 115, 233, 205, 216, 161, - 248, 38, 216, 161, 180, 139, 203, 40, 216, 161, 203, 41, 139, 219, 21, - 216, 161, 180, 139, 105, 203, 26, 216, 161, 105, 203, 27, 139, 219, 21, - 216, 161, 105, 203, 27, 222, 199, 216, 161, 197, 221, 216, 161, 199, 65, - 216, 161, 210, 24, 234, 210, 230, 190, 232, 71, 198, 54, 210, 115, 248, - 79, 210, 115, 198, 54, 238, 173, 248, 55, 248, 79, 238, 173, 248, 55, - 198, 54, 198, 42, 203, 104, 248, 55, 248, 79, 198, 42, 203, 104, 248, 55, - 63, 198, 5, 248, 223, 206, 190, 4, 243, 12, 201, 52, 232, 206, 252, 66, - 238, 170, 234, 136, 223, 101, 237, 140, 233, 209, 113, 62, 206, 204, 55, - 197, 241, 62, 219, 221, 55, 197, 241, 62, 196, 76, 55, 197, 241, 62, 235, - 124, 55, 197, 241, 62, 206, 204, 55, 197, 242, 4, 81, 164, 62, 219, 221, - 55, 197, 242, 4, 81, 164, 62, 206, 204, 197, 242, 4, 55, 81, 164, 251, - 203, 242, 224, 201, 59, 198, 145, 242, 224, 228, 177, 4, 232, 186, 205, - 141, 62, 216, 215, 219, 221, 197, 241, 62, 216, 215, 206, 204, 197, 241, - 62, 216, 215, 196, 76, 197, 241, 62, 216, 215, 235, 124, 197, 241, 55, - 81, 164, 62, 51, 40, 201, 64, 62, 243, 13, 40, 207, 99, 209, 1, 113, 209, - 1, 211, 105, 113, 209, 1, 211, 107, 113, 209, 1, 202, 94, 113, 211, 170, - 233, 196, 113, 16, 40, 212, 140, 16, 40, 201, 99, 79, 229, 234, 16, 40, - 201, 99, 79, 199, 53, 16, 40, 234, 97, 79, 199, 53, 16, 40, 234, 97, 79, - 198, 11, 16, 40, 234, 83, 16, 40, 252, 53, 16, 40, 248, 237, 16, 40, 249, - 142, 16, 40, 228, 243, 199, 230, 16, 40, 219, 115, 233, 52, 16, 40, 81, - 199, 230, 16, 40, 232, 148, 233, 52, 16, 40, 247, 90, 207, 23, 16, 40, - 203, 78, 211, 16, 16, 40, 203, 78, 223, 164, 16, 40, 237, 212, 219, 105, - 234, 16, 16, 40, 237, 77, 239, 24, 107, 16, 40, 237, 77, 239, 24, 109, - 16, 40, 237, 77, 239, 24, 138, 16, 40, 237, 77, 239, 24, 134, 16, 40, - 214, 108, 252, 53, 16, 40, 200, 69, 223, 230, 16, 40, 234, 97, 79, 198, - 12, 248, 131, 16, 40, 247, 128, 16, 40, 234, 97, 79, 216, 214, 16, 40, - 200, 227, 16, 40, 234, 16, 16, 40, 233, 9, 204, 11, 16, 40, 230, 189, - 204, 11, 16, 40, 207, 100, 204, 11, 16, 40, 196, 91, 204, 11, 16, 40, - 201, 248, 16, 40, 237, 118, 248, 135, 113, 211, 79, 248, 64, 16, 40, 214, - 78, 16, 40, 237, 119, 232, 148, 109, 16, 40, 200, 228, 232, 148, 109, - 211, 163, 102, 211, 163, 246, 254, 211, 163, 232, 151, 211, 163, 223, 95, - 232, 151, 211, 163, 248, 234, 248, 7, 211, 163, 248, 72, 198, 178, 211, - 163, 248, 50, 249, 95, 228, 17, 211, 163, 251, 141, 79, 243, 127, 211, - 163, 237, 217, 211, 163, 238, 159, 252, 57, 212, 138, 211, 163, 55, 249, - 143, 47, 17, 107, 47, 17, 109, 47, 17, 138, 47, 17, 134, 47, 17, 150, 47, - 17, 169, 47, 17, 175, 47, 17, 171, 47, 17, 178, 47, 31, 199, 95, 47, 31, - 234, 129, 47, 31, 197, 37, 47, 31, 198, 251, 47, 31, 232, 124, 47, 31, - 233, 21, 47, 31, 202, 131, 47, 31, 203, 245, 47, 31, 234, 163, 47, 31, - 213, 173, 47, 31, 197, 32, 127, 17, 107, 127, 17, 109, 127, 17, 138, 127, - 17, 134, 127, 17, 150, 127, 17, 169, 127, 17, 175, 127, 17, 171, 127, 17, - 178, 127, 31, 199, 95, 127, 31, 234, 129, 127, 31, 197, 37, 127, 31, 198, - 251, 127, 31, 232, 124, 127, 31, 233, 21, 127, 31, 202, 131, 127, 31, - 203, 245, 127, 31, 234, 163, 127, 31, 213, 173, 127, 31, 197, 32, 17, 91, - 232, 82, 201, 64, 17, 105, 232, 82, 201, 64, 17, 115, 232, 82, 201, 64, - 17, 232, 130, 232, 82, 201, 64, 17, 232, 228, 232, 82, 201, 64, 17, 202, - 137, 232, 82, 201, 64, 17, 203, 248, 232, 82, 201, 64, 17, 234, 166, 232, - 82, 201, 64, 17, 213, 177, 232, 82, 201, 64, 31, 199, 96, 232, 82, 201, - 64, 31, 234, 130, 232, 82, 201, 64, 31, 197, 38, 232, 82, 201, 64, 31, - 198, 252, 232, 82, 201, 64, 31, 232, 125, 232, 82, 201, 64, 31, 233, 22, - 232, 82, 201, 64, 31, 202, 132, 232, 82, 201, 64, 31, 203, 246, 232, 82, - 201, 64, 31, 234, 164, 232, 82, 201, 64, 31, 213, 174, 232, 82, 201, 64, - 31, 197, 33, 232, 82, 201, 64, 127, 8, 2, 1, 65, 127, 8, 2, 1, 250, 122, - 127, 8, 2, 1, 247, 195, 127, 8, 2, 1, 238, 129, 127, 8, 2, 1, 71, 127, 8, - 2, 1, 233, 177, 127, 8, 2, 1, 232, 53, 127, 8, 2, 1, 230, 118, 127, 8, 2, - 1, 68, 127, 8, 2, 1, 223, 37, 127, 8, 2, 1, 222, 154, 127, 8, 2, 1, 172, - 127, 8, 2, 1, 218, 170, 127, 8, 2, 1, 215, 63, 127, 8, 2, 1, 74, 127, 8, - 2, 1, 210, 238, 127, 8, 2, 1, 208, 106, 127, 8, 2, 1, 146, 127, 8, 2, 1, - 206, 9, 127, 8, 2, 1, 200, 43, 127, 8, 2, 1, 66, 127, 8, 2, 1, 196, 12, - 127, 8, 2, 1, 193, 224, 127, 8, 2, 1, 192, 235, 127, 8, 2, 1, 192, 159, - 127, 8, 2, 1, 191, 166, 47, 8, 6, 1, 65, 47, 8, 6, 1, 250, 122, 47, 8, 6, - 1, 247, 195, 47, 8, 6, 1, 238, 129, 47, 8, 6, 1, 71, 47, 8, 6, 1, 233, - 177, 47, 8, 6, 1, 232, 53, 47, 8, 6, 1, 230, 118, 47, 8, 6, 1, 68, 47, 8, - 6, 1, 223, 37, 47, 8, 6, 1, 222, 154, 47, 8, 6, 1, 172, 47, 8, 6, 1, 218, - 170, 47, 8, 6, 1, 215, 63, 47, 8, 6, 1, 74, 47, 8, 6, 1, 210, 238, 47, 8, - 6, 1, 208, 106, 47, 8, 6, 1, 146, 47, 8, 6, 1, 206, 9, 47, 8, 6, 1, 200, - 43, 47, 8, 6, 1, 66, 47, 8, 6, 1, 196, 12, 47, 8, 6, 1, 193, 224, 47, 8, - 6, 1, 192, 235, 47, 8, 6, 1, 192, 159, 47, 8, 6, 1, 191, 166, 47, 8, 2, - 1, 65, 47, 8, 2, 1, 250, 122, 47, 8, 2, 1, 247, 195, 47, 8, 2, 1, 238, - 129, 47, 8, 2, 1, 71, 47, 8, 2, 1, 233, 177, 47, 8, 2, 1, 232, 53, 47, 8, - 2, 1, 230, 118, 47, 8, 2, 1, 68, 47, 8, 2, 1, 223, 37, 47, 8, 2, 1, 222, - 154, 47, 8, 2, 1, 172, 47, 8, 2, 1, 218, 170, 47, 8, 2, 1, 215, 63, 47, - 8, 2, 1, 74, 47, 8, 2, 1, 210, 238, 47, 8, 2, 1, 208, 106, 47, 8, 2, 1, - 146, 47, 8, 2, 1, 206, 9, 47, 8, 2, 1, 200, 43, 47, 8, 2, 1, 66, 47, 8, - 2, 1, 196, 12, 47, 8, 2, 1, 193, 224, 47, 8, 2, 1, 192, 235, 47, 8, 2, 1, - 192, 159, 47, 8, 2, 1, 191, 166, 47, 17, 191, 77, 214, 108, 47, 31, 234, - 129, 214, 108, 47, 31, 197, 37, 214, 108, 47, 31, 198, 251, 214, 108, 47, - 31, 232, 124, 214, 108, 47, 31, 233, 21, 214, 108, 47, 31, 202, 131, 214, - 108, 47, 31, 203, 245, 214, 108, 47, 31, 234, 163, 214, 108, 47, 31, 213, - 173, 214, 108, 47, 31, 197, 32, 55, 47, 17, 107, 55, 47, 17, 109, 55, 47, - 17, 138, 55, 47, 17, 134, 55, 47, 17, 150, 55, 47, 17, 169, 55, 47, 17, - 175, 55, 47, 17, 171, 55, 47, 17, 178, 55, 47, 31, 199, 95, 214, 108, 47, - 17, 191, 77, 111, 122, 156, 228, 242, 111, 122, 88, 228, 242, 111, 122, - 156, 195, 135, 111, 122, 88, 195, 135, 111, 122, 156, 197, 225, 237, 218, - 228, 242, 111, 122, 88, 197, 225, 237, 218, 228, 242, 111, 122, 156, 197, - 225, 237, 218, 195, 135, 111, 122, 88, 197, 225, 237, 218, 195, 135, 111, - 122, 156, 208, 28, 237, 218, 228, 242, 111, 122, 88, 208, 28, 237, 218, - 228, 242, 111, 122, 156, 208, 28, 237, 218, 195, 135, 111, 122, 88, 208, - 28, 237, 218, 195, 135, 111, 122, 156, 144, 23, 179, 111, 122, 144, 156, - 23, 50, 229, 219, 111, 122, 144, 88, 23, 50, 219, 134, 111, 122, 88, 144, - 23, 179, 111, 122, 156, 144, 23, 219, 226, 111, 122, 144, 156, 23, 45, - 229, 219, 111, 122, 144, 88, 23, 45, 219, 134, 111, 122, 88, 144, 23, - 219, 226, 111, 122, 156, 133, 23, 179, 111, 122, 133, 156, 23, 50, 229, - 219, 111, 122, 133, 88, 23, 50, 219, 134, 111, 122, 88, 133, 23, 179, - 111, 122, 156, 133, 23, 219, 226, 111, 122, 133, 156, 23, 45, 229, 219, - 111, 122, 133, 88, 23, 45, 219, 134, 111, 122, 88, 133, 23, 219, 226, - 111, 122, 156, 81, 23, 179, 111, 122, 81, 156, 23, 50, 229, 219, 111, - 122, 133, 88, 23, 50, 144, 219, 134, 111, 122, 144, 88, 23, 50, 133, 219, - 134, 111, 122, 81, 88, 23, 50, 219, 134, 111, 122, 144, 156, 23, 50, 133, - 229, 219, 111, 122, 133, 156, 23, 50, 144, 229, 219, 111, 122, 88, 81, - 23, 179, 111, 122, 156, 81, 23, 219, 226, 111, 122, 81, 156, 23, 45, 229, - 219, 111, 122, 133, 88, 23, 45, 144, 219, 134, 111, 122, 144, 88, 23, 45, - 133, 219, 134, 111, 122, 81, 88, 23, 45, 219, 134, 111, 122, 144, 156, - 23, 45, 133, 229, 219, 111, 122, 133, 156, 23, 45, 144, 229, 219, 111, - 122, 88, 81, 23, 219, 226, 111, 122, 156, 144, 23, 228, 242, 111, 122, - 45, 88, 23, 50, 144, 219, 134, 111, 122, 50, 88, 23, 45, 144, 219, 134, - 111, 122, 144, 156, 23, 228, 243, 229, 219, 111, 122, 144, 88, 23, 228, - 243, 219, 134, 111, 122, 50, 156, 23, 45, 144, 229, 219, 111, 122, 45, - 156, 23, 50, 144, 229, 219, 111, 122, 88, 144, 23, 228, 242, 111, 122, - 156, 133, 23, 228, 242, 111, 122, 45, 88, 23, 50, 133, 219, 134, 111, - 122, 50, 88, 23, 45, 133, 219, 134, 111, 122, 133, 156, 23, 228, 243, - 229, 219, 111, 122, 133, 88, 23, 228, 243, 219, 134, 111, 122, 50, 156, - 23, 45, 133, 229, 219, 111, 122, 45, 156, 23, 50, 133, 229, 219, 111, - 122, 88, 133, 23, 228, 242, 111, 122, 156, 81, 23, 228, 242, 111, 122, - 45, 88, 23, 50, 81, 219, 134, 111, 122, 50, 88, 23, 45, 81, 219, 134, - 111, 122, 81, 156, 23, 228, 243, 229, 219, 111, 122, 133, 88, 23, 144, - 228, 243, 219, 134, 111, 122, 144, 88, 23, 133, 228, 243, 219, 134, 111, - 122, 81, 88, 23, 228, 243, 219, 134, 111, 122, 45, 133, 88, 23, 50, 144, - 219, 134, 111, 122, 50, 133, 88, 23, 45, 144, 219, 134, 111, 122, 45, - 144, 88, 23, 50, 133, 219, 134, 111, 122, 50, 144, 88, 23, 45, 133, 219, - 134, 111, 122, 144, 156, 23, 133, 228, 243, 229, 219, 111, 122, 133, 156, - 23, 144, 228, 243, 229, 219, 111, 122, 50, 156, 23, 45, 81, 229, 219, - 111, 122, 45, 156, 23, 50, 81, 229, 219, 111, 122, 88, 81, 23, 228, 242, - 111, 122, 156, 55, 237, 218, 228, 242, 111, 122, 88, 55, 237, 218, 228, - 242, 111, 122, 156, 55, 237, 218, 195, 135, 111, 122, 88, 55, 237, 218, - 195, 135, 111, 122, 55, 228, 242, 111, 122, 55, 195, 135, 111, 122, 144, - 202, 171, 23, 50, 235, 135, 111, 122, 144, 55, 23, 50, 202, 170, 111, - 122, 55, 144, 23, 179, 111, 122, 144, 202, 171, 23, 45, 235, 135, 111, - 122, 144, 55, 23, 45, 202, 170, 111, 122, 55, 144, 23, 219, 226, 111, - 122, 133, 202, 171, 23, 50, 235, 135, 111, 122, 133, 55, 23, 50, 202, - 170, 111, 122, 55, 133, 23, 179, 111, 122, 133, 202, 171, 23, 45, 235, - 135, 111, 122, 133, 55, 23, 45, 202, 170, 111, 122, 55, 133, 23, 219, - 226, 111, 122, 81, 202, 171, 23, 50, 235, 135, 111, 122, 81, 55, 23, 50, - 202, 170, 111, 122, 55, 81, 23, 179, 111, 122, 81, 202, 171, 23, 45, 235, - 135, 111, 122, 81, 55, 23, 45, 202, 170, 111, 122, 55, 81, 23, 219, 226, - 111, 122, 144, 202, 171, 23, 228, 243, 235, 135, 111, 122, 144, 55, 23, - 228, 243, 202, 170, 111, 122, 55, 144, 23, 228, 242, 111, 122, 133, 202, - 171, 23, 228, 243, 235, 135, 111, 122, 133, 55, 23, 228, 243, 202, 170, - 111, 122, 55, 133, 23, 228, 242, 111, 122, 81, 202, 171, 23, 228, 243, - 235, 135, 111, 122, 81, 55, 23, 228, 243, 202, 170, 111, 122, 55, 81, 23, - 228, 242, 111, 122, 156, 251, 19, 144, 23, 179, 111, 122, 156, 251, 19, - 144, 23, 219, 226, 111, 122, 156, 251, 19, 133, 23, 219, 226, 111, 122, - 156, 251, 19, 133, 23, 179, 111, 122, 156, 237, 41, 116, 50, 119, 110, - 219, 226, 111, 122, 156, 237, 41, 116, 45, 119, 110, 179, 111, 122, 156, - 237, 41, 238, 219, 111, 122, 156, 219, 226, 111, 122, 156, 196, 77, 111, - 122, 156, 179, 111, 122, 156, 235, 125, 111, 122, 88, 219, 226, 111, 122, - 88, 196, 77, 111, 122, 88, 179, 111, 122, 88, 235, 125, 111, 122, 156, - 45, 23, 88, 179, 111, 122, 156, 133, 23, 88, 235, 125, 111, 122, 88, 45, - 23, 156, 179, 111, 122, 88, 133, 23, 156, 235, 125, 116, 132, 248, 131, - 110, 91, 234, 162, 248, 131, 110, 91, 208, 25, 248, 131, 110, 115, 234, - 160, 248, 131, 110, 132, 248, 131, 110, 232, 228, 234, 160, 248, 131, - 110, 115, 208, 23, 248, 131, 110, 203, 248, 234, 160, 248, 131, 232, 82, - 248, 131, 45, 203, 248, 234, 160, 248, 131, 45, 115, 208, 23, 248, 131, - 45, 232, 228, 234, 160, 248, 131, 45, 132, 248, 131, 45, 115, 234, 160, - 248, 131, 45, 91, 208, 25, 248, 131, 45, 91, 234, 162, 248, 131, 50, 132, - 248, 131, 156, 203, 154, 216, 215, 203, 154, 237, 223, 203, 154, 116, 91, - 234, 162, 248, 131, 50, 91, 234, 162, 248, 131, 208, 30, 110, 219, 226, - 208, 30, 110, 179, 208, 30, 116, 219, 226, 208, 30, 116, 45, 23, 110, 45, - 23, 110, 179, 208, 30, 116, 45, 23, 110, 179, 208, 30, 116, 45, 23, 116, - 50, 23, 110, 219, 226, 208, 30, 116, 45, 23, 116, 50, 23, 110, 179, 208, - 30, 116, 179, 208, 30, 116, 50, 23, 110, 219, 226, 208, 30, 116, 50, 23, - 110, 45, 23, 110, 179, 62, 201, 191, 63, 201, 191, 63, 51, 4, 206, 114, - 239, 2, 63, 51, 239, 35, 62, 2, 201, 191, 51, 4, 228, 243, 233, 7, 51, 4, - 81, 233, 7, 51, 4, 211, 40, 238, 213, 233, 7, 51, 4, 116, 45, 119, 110, - 50, 233, 7, 51, 4, 116, 50, 119, 110, 45, 233, 7, 51, 4, 237, 41, 238, - 213, 233, 7, 62, 2, 201, 191, 63, 2, 201, 191, 62, 207, 94, 63, 207, 94, - 62, 81, 207, 94, 63, 81, 207, 94, 62, 210, 119, 63, 210, 119, 62, 196, - 76, 197, 241, 63, 196, 76, 197, 241, 62, 196, 76, 2, 197, 241, 63, 196, - 76, 2, 197, 241, 62, 206, 204, 197, 241, 63, 206, 204, 197, 241, 62, 206, - 204, 2, 197, 241, 63, 206, 204, 2, 197, 241, 62, 206, 204, 209, 61, 63, - 206, 204, 209, 61, 62, 235, 124, 197, 241, 63, 235, 124, 197, 241, 62, - 235, 124, 2, 197, 241, 63, 235, 124, 2, 197, 241, 62, 219, 221, 197, 241, - 63, 219, 221, 197, 241, 62, 219, 221, 2, 197, 241, 63, 219, 221, 2, 197, - 241, 62, 219, 221, 209, 61, 63, 219, 221, 209, 61, 62, 237, 34, 63, 237, - 34, 63, 237, 35, 239, 35, 62, 2, 237, 34, 232, 237, 218, 234, 63, 243, - 12, 235, 140, 243, 12, 243, 13, 4, 81, 233, 7, 247, 246, 62, 243, 12, - 243, 13, 4, 45, 132, 248, 141, 243, 13, 4, 50, 132, 248, 141, 243, 13, 4, - 110, 132, 248, 141, 243, 13, 4, 116, 132, 248, 141, 243, 13, 4, 116, 50, - 208, 30, 248, 141, 243, 13, 4, 251, 177, 247, 220, 116, 45, 208, 30, 248, - 141, 45, 132, 62, 243, 12, 50, 132, 62, 243, 12, 223, 97, 247, 250, 223, - 97, 63, 243, 12, 116, 132, 223, 97, 63, 243, 12, 110, 132, 223, 97, 63, - 243, 12, 116, 45, 208, 30, 243, 6, 251, 18, 116, 50, 208, 30, 243, 6, - 251, 18, 110, 50, 208, 30, 243, 6, 251, 18, 110, 45, 208, 30, 243, 6, - 251, 18, 116, 132, 243, 12, 110, 132, 243, 12, 62, 110, 50, 197, 241, 62, - 110, 45, 197, 241, 62, 116, 45, 197, 241, 62, 116, 50, 197, 241, 63, 247, - 250, 51, 4, 45, 132, 248, 141, 51, 4, 50, 132, 248, 141, 51, 4, 116, 45, - 237, 41, 132, 248, 141, 51, 4, 110, 50, 237, 41, 132, 248, 141, 63, 51, - 4, 81, 248, 156, 219, 114, 63, 196, 76, 197, 242, 4, 236, 142, 196, 76, - 197, 242, 4, 45, 132, 248, 141, 196, 76, 197, 242, 4, 50, 132, 248, 141, - 220, 15, 243, 12, 63, 51, 4, 116, 45, 208, 29, 63, 51, 4, 110, 45, 208, - 29, 63, 51, 4, 110, 50, 208, 29, 63, 51, 4, 116, 50, 208, 29, 63, 243, - 13, 4, 116, 45, 208, 29, 63, 243, 13, 4, 110, 45, 208, 29, 63, 243, 13, - 4, 110, 50, 208, 29, 63, 243, 13, 4, 116, 50, 208, 29, 116, 45, 197, 241, - 116, 50, 197, 241, 110, 45, 197, 241, 63, 216, 215, 201, 191, 62, 216, - 215, 201, 191, 63, 216, 215, 2, 201, 191, 62, 216, 215, 2, 201, 191, 110, - 50, 197, 241, 62, 200, 199, 4, 207, 121, 242, 212, 196, 117, 202, 49, - 242, 79, 62, 201, 103, 63, 201, 103, 219, 131, 198, 208, 200, 198, 250, - 215, 213, 37, 237, 88, 213, 37, 239, 44, 211, 64, 62, 199, 106, 63, 199, - 106, 249, 109, 248, 64, 249, 109, 111, 4, 243, 127, 249, 109, 111, 4, - 192, 235, 205, 155, 196, 118, 4, 207, 152, 235, 98, 228, 183, 248, 209, - 63, 203, 50, 209, 177, 62, 203, 50, 209, 177, 203, 141, 207, 19, 206, - 123, 232, 192, 229, 226, 247, 250, 62, 45, 209, 60, 223, 150, 62, 50, - 209, 60, 223, 150, 63, 45, 209, 60, 223, 150, 63, 133, 209, 60, 223, 150, - 63, 50, 209, 60, 223, 150, 63, 144, 209, 60, 223, 150, 202, 105, 23, 238, - 217, 247, 73, 56, 207, 166, 56, 248, 164, 56, 247, 153, 251, 101, 211, - 41, 238, 219, 243, 98, 207, 4, 238, 220, 79, 219, 1, 238, 220, 79, 223, - 0, 201, 104, 23, 238, 229, 233, 76, 113, 252, 36, 203, 144, 230, 64, 23, - 202, 214, 210, 63, 113, 192, 22, 192, 106, 197, 231, 40, 229, 221, 197, - 231, 40, 220, 45, 197, 231, 40, 232, 245, 197, 231, 40, 198, 209, 197, - 231, 40, 193, 66, 197, 231, 40, 193, 143, 197, 231, 40, 215, 186, 197, - 231, 40, 234, 209, 193, 94, 79, 237, 62, 63, 232, 94, 233, 105, 63, 202, - 65, 233, 105, 62, 202, 65, 233, 105, 63, 200, 199, 4, 207, 121, 232, 240, - 208, 25, 215, 207, 220, 8, 208, 25, 215, 207, 216, 182, 233, 44, 56, 234, - 209, 217, 99, 56, 222, 169, 205, 116, 196, 57, 214, 96, 209, 79, 251, 4, - 199, 164, 231, 154, 247, 126, 219, 188, 195, 38, 219, 145, 205, 81, 205, - 184, 247, 108, 251, 36, 209, 122, 63, 243, 107, 221, 140, 63, 243, 107, - 208, 16, 63, 243, 107, 206, 132, 63, 243, 107, 248, 154, 63, 243, 107, - 221, 78, 63, 243, 107, 210, 76, 62, 243, 107, 221, 140, 62, 243, 107, - 208, 16, 62, 243, 107, 206, 132, 62, 243, 107, 248, 154, 62, 243, 107, - 221, 78, 62, 243, 107, 210, 76, 62, 201, 246, 200, 211, 63, 229, 226, - 200, 211, 63, 237, 35, 200, 211, 62, 242, 209, 200, 211, 63, 201, 246, - 200, 211, 62, 229, 226, 200, 211, 62, 237, 35, 200, 211, 63, 242, 209, - 200, 211, 228, 183, 201, 196, 208, 25, 213, 8, 234, 162, 213, 8, 249, 15, - 234, 162, 213, 3, 249, 15, 202, 130, 213, 3, 215, 99, 232, 209, 56, 215, - 99, 214, 208, 56, 215, 99, 203, 128, 56, 193, 105, 200, 63, 238, 219, - 234, 206, 200, 63, 238, 219, 196, 87, 207, 90, 113, 207, 90, 16, 40, 196, - 254, 209, 100, 207, 90, 16, 40, 196, 252, 209, 100, 207, 90, 16, 40, 196, - 251, 209, 100, 207, 90, 16, 40, 196, 249, 209, 100, 207, 90, 16, 40, 196, - 247, 209, 100, 207, 90, 16, 40, 196, 245, 209, 100, 207, 90, 16, 40, 196, - 243, 209, 100, 207, 90, 16, 40, 231, 151, 217, 30, 62, 196, 87, 207, 90, - 113, 207, 91, 210, 138, 113, 210, 106, 210, 138, 113, 210, 5, 210, 138, - 56, 193, 92, 113, 237, 27, 233, 104, 237, 27, 233, 103, 237, 27, 233, - 102, 237, 27, 233, 101, 237, 27, 233, 100, 237, 27, 233, 99, 63, 243, 13, - 4, 75, 179, 63, 243, 13, 4, 105, 236, 140, 62, 243, 13, 4, 63, 75, 179, - 62, 243, 13, 4, 105, 63, 236, 140, 215, 223, 40, 192, 106, 215, 223, 40, - 192, 21, 237, 8, 40, 230, 207, 192, 106, 237, 8, 40, 219, 180, 192, 21, - 237, 8, 40, 219, 180, 192, 106, 237, 8, 40, 230, 207, 192, 21, 63, 232, - 219, 62, 232, 219, 230, 64, 23, 209, 182, 251, 129, 238, 216, 200, 130, - 201, 113, 79, 252, 10, 205, 99, 251, 193, 232, 188, 231, 164, 201, 113, - 79, 229, 193, 250, 174, 113, 232, 204, 211, 12, 63, 201, 103, 115, 219, - 109, 239, 21, 179, 115, 219, 109, 239, 21, 219, 226, 193, 155, 56, 137, - 195, 12, 56, 235, 130, 233, 44, 56, 235, 130, 217, 99, 56, 223, 107, 233, - 44, 23, 217, 99, 56, 217, 99, 23, 233, 44, 56, 217, 99, 4, 201, 29, 56, - 217, 99, 4, 201, 29, 23, 217, 99, 23, 233, 44, 56, 81, 217, 99, 4, 201, - 29, 56, 228, 243, 217, 99, 4, 201, 29, 56, 216, 215, 63, 243, 12, 216, - 215, 62, 243, 12, 216, 215, 2, 63, 243, 12, 217, 50, 113, 236, 200, 113, - 196, 84, 210, 105, 113, 242, 91, 232, 76, 196, 53, 214, 85, 247, 9, 210, - 187, 222, 175, 195, 80, 243, 77, 62, 215, 208, 219, 128, 203, 177, 204, - 23, 208, 6, 204, 0, 202, 37, 249, 113, 249, 75, 112, 221, 224, 63, 235, - 110, 217, 92, 63, 235, 110, 221, 140, 62, 235, 110, 217, 92, 62, 235, - 110, 221, 140, 202, 50, 193, 51, 202, 53, 200, 199, 248, 244, 242, 212, - 207, 151, 62, 202, 49, 198, 210, 242, 213, 23, 207, 151, 154, 63, 203, - 50, 209, 177, 154, 62, 203, 50, 209, 177, 63, 237, 35, 223, 165, 201, - 191, 238, 212, 220, 23, 236, 231, 247, 104, 211, 67, 209, 182, 247, 105, - 202, 86, 229, 203, 4, 63, 238, 219, 47, 238, 212, 220, 23, 246, 255, 213, - 46, 234, 74, 251, 159, 211, 98, 45, 193, 129, 198, 19, 62, 197, 10, 45, - 193, 129, 198, 19, 63, 197, 10, 45, 193, 129, 198, 19, 62, 45, 220, 24, - 216, 181, 63, 45, 220, 24, 216, 181, 235, 105, 202, 77, 56, 88, 63, 235, - 124, 197, 241, 45, 242, 221, 234, 74, 112, 205, 155, 233, 85, 237, 41, - 223, 165, 63, 243, 13, 223, 165, 62, 201, 191, 62, 197, 203, 207, 30, 45, - 234, 73, 207, 30, 45, 234, 72, 250, 189, 16, 40, 196, 57, 88, 243, 13, 4, - 201, 29, 23, 105, 185, 58, 210, 25, 206, 206, 223, 109, 210, 25, 219, - 223, 223, 109, 210, 25, 223, 95, 210, 25, 62, 238, 220, 211, 107, 203, - 79, 203, 67, 203, 13, 243, 42, 247, 82, 229, 120, 202, 138, 231, 165, - 193, 51, 228, 155, 231, 165, 4, 230, 32, 217, 74, 16, 40, 219, 133, 215, - 186, 196, 118, 211, 107, 230, 190, 232, 131, 232, 220, 223, 165, 229, 7, - 233, 34, 205, 179, 51, 232, 130, 239, 2, 202, 109, 228, 29, 202, 113, - 209, 253, 4, 249, 113, 199, 87, 223, 21, 249, 95, 113, 229, 231, 230, - 209, 113, 232, 85, 208, 155, 238, 184, 211, 107, 62, 201, 191, 63, 232, - 220, 4, 228, 243, 82, 62, 201, 30, 62, 205, 189, 205, 85, 116, 248, 136, - 205, 85, 62, 205, 85, 110, 248, 136, 205, 85, 63, 205, 85, 63, 88, 243, - 128, 77, 199, 107, 219, 42, 56, 199, 182, 235, 104, 251, 225, 234, 69, - 207, 149, 232, 233, 207, 149, 230, 55, 195, 67, 230, 55, 193, 3, 230, 55, - 110, 50, 210, 35, 210, 35, 116, 50, 210, 35, 63, 213, 210, 62, 213, 210, - 243, 128, 77, 88, 243, 128, 77, 215, 129, 192, 235, 88, 215, 129, 192, - 235, 249, 109, 192, 235, 88, 249, 109, 192, 235, 211, 12, 35, 238, 219, - 88, 35, 238, 219, 211, 79, 247, 24, 238, 219, 88, 211, 79, 247, 24, 238, - 219, 8, 238, 219, 203, 152, 63, 8, 238, 219, 211, 12, 8, 238, 219, 217, - 95, 238, 219, 201, 104, 79, 237, 210, 232, 130, 199, 127, 250, 195, 232, - 130, 249, 110, 250, 195, 88, 232, 130, 249, 110, 250, 195, 232, 130, 242, - 207, 250, 195, 62, 232, 130, 209, 62, 201, 103, 63, 232, 130, 209, 62, - 201, 103, 201, 241, 201, 39, 211, 12, 63, 201, 103, 47, 63, 201, 103, - 211, 79, 247, 24, 62, 201, 103, 62, 247, 24, 63, 201, 103, 211, 12, 62, - 201, 103, 88, 211, 12, 62, 201, 103, 209, 132, 201, 103, 203, 152, 63, - 201, 103, 88, 250, 195, 211, 79, 247, 24, 250, 195, 234, 166, 201, 207, - 250, 195, 234, 166, 209, 62, 62, 201, 103, 234, 166, 209, 62, 209, 132, - 201, 103, 202, 137, 209, 62, 62, 201, 103, 234, 166, 209, 62, 207, 92, - 62, 201, 103, 88, 234, 166, 209, 62, 207, 92, 62, 201, 103, 197, 38, 209, - 62, 62, 201, 103, 202, 132, 209, 62, 250, 195, 199, 127, 250, 195, 211, - 79, 247, 24, 199, 127, 250, 195, 88, 199, 127, 250, 195, 202, 137, 209, - 241, 62, 23, 63, 232, 191, 62, 232, 191, 63, 232, 191, 234, 166, 209, - 241, 211, 12, 62, 232, 191, 47, 211, 79, 247, 24, 234, 166, 209, 62, 201, - 103, 88, 199, 127, 209, 132, 250, 195, 202, 51, 198, 172, 197, 234, 202, - 51, 88, 243, 103, 202, 51, 201, 243, 88, 201, 243, 249, 110, 250, 195, - 234, 166, 199, 127, 208, 191, 250, 195, 88, 234, 166, 199, 127, 208, 191, - 250, 195, 238, 220, 77, 203, 152, 63, 243, 12, 214, 108, 112, 238, 220, - 77, 110, 50, 235, 100, 63, 201, 191, 116, 50, 235, 100, 63, 201, 191, - 110, 50, 203, 152, 63, 201, 191, 116, 50, 203, 152, 63, 201, 191, 62, - 208, 15, 87, 211, 44, 63, 208, 15, 87, 211, 44, 63, 233, 218, 87, 211, - 44, 62, 237, 35, 216, 37, 63, 192, 235, 88, 233, 218, 87, 113, 156, 81, - 164, 216, 215, 81, 164, 88, 81, 164, 88, 202, 171, 154, 242, 77, 207, - 254, 87, 211, 44, 88, 202, 171, 242, 77, 207, 254, 87, 211, 44, 88, 55, - 154, 242, 77, 207, 254, 87, 211, 44, 88, 55, 242, 77, 207, 254, 87, 211, - 44, 88, 130, 202, 171, 242, 77, 207, 254, 87, 211, 44, 88, 130, 55, 242, - 77, 207, 254, 87, 211, 44, 238, 165, 201, 82, 210, 130, 3, 211, 44, 88, - 233, 218, 87, 211, 44, 88, 229, 226, 233, 218, 87, 211, 44, 88, 62, 229, - 225, 206, 123, 88, 62, 229, 226, 247, 250, 232, 192, 229, 225, 206, 123, - 232, 192, 229, 226, 247, 250, 216, 215, 45, 210, 116, 211, 44, 216, 215, - 50, 210, 116, 211, 44, 216, 215, 232, 205, 45, 210, 116, 211, 44, 216, - 215, 232, 205, 50, 210, 116, 211, 44, 216, 215, 219, 221, 251, 118, 248, - 56, 211, 44, 216, 215, 206, 204, 251, 118, 248, 56, 211, 44, 88, 219, - 221, 251, 118, 207, 254, 87, 211, 44, 88, 206, 204, 251, 118, 207, 254, - 87, 211, 44, 88, 219, 221, 251, 118, 248, 56, 211, 44, 88, 206, 204, 251, - 118, 248, 56, 211, 44, 156, 45, 198, 42, 203, 104, 248, 56, 211, 44, 156, - 50, 198, 42, 203, 104, 248, 56, 211, 44, 216, 215, 45, 238, 173, 248, 56, - 211, 44, 216, 215, 50, 238, 173, 248, 56, 211, 44, 236, 243, 214, 108, - 47, 17, 107, 236, 243, 214, 108, 47, 17, 109, 236, 243, 214, 108, 47, 17, - 138, 236, 243, 214, 108, 47, 17, 134, 236, 243, 214, 108, 47, 17, 150, - 236, 243, 214, 108, 47, 17, 169, 236, 243, 214, 108, 47, 17, 175, 236, - 243, 214, 108, 47, 17, 171, 236, 243, 214, 108, 47, 17, 178, 236, 243, - 214, 108, 47, 31, 199, 95, 236, 243, 47, 49, 17, 107, 236, 243, 47, 49, - 17, 109, 236, 243, 47, 49, 17, 138, 236, 243, 47, 49, 17, 134, 236, 243, - 47, 49, 17, 150, 236, 243, 47, 49, 17, 169, 236, 243, 47, 49, 17, 175, - 236, 243, 47, 49, 17, 171, 236, 243, 47, 49, 17, 178, 236, 243, 47, 49, - 31, 199, 95, 236, 243, 214, 108, 47, 49, 17, 107, 236, 243, 214, 108, 47, - 49, 17, 109, 236, 243, 214, 108, 47, 49, 17, 138, 236, 243, 214, 108, 47, - 49, 17, 134, 236, 243, 214, 108, 47, 49, 17, 150, 236, 243, 214, 108, 47, - 49, 17, 169, 236, 243, 214, 108, 47, 49, 17, 175, 236, 243, 214, 108, 47, - 49, 17, 171, 236, 243, 214, 108, 47, 49, 17, 178, 236, 243, 214, 108, 47, - 49, 31, 199, 95, 88, 193, 77, 96, 57, 88, 108, 56, 88, 216, 37, 56, 88, - 236, 202, 56, 88, 202, 3, 234, 206, 57, 88, 96, 57, 88, 186, 234, 206, - 57, 235, 115, 209, 64, 96, 57, 88, 206, 115, 96, 57, 197, 240, 96, 57, - 88, 197, 240, 96, 57, 237, 216, 197, 240, 96, 57, 88, 237, 216, 197, 240, - 96, 57, 62, 96, 57, 198, 225, 198, 52, 96, 250, 237, 198, 225, 248, 77, - 96, 250, 237, 62, 96, 250, 237, 88, 62, 238, 165, 235, 121, 23, 96, 57, - 88, 62, 238, 165, 196, 66, 23, 96, 57, 201, 188, 62, 96, 57, 88, 239, 58, - 62, 96, 57, 206, 203, 63, 96, 57, 219, 220, 63, 96, 57, 249, 147, 203, - 152, 63, 96, 57, 232, 97, 203, 152, 63, 96, 57, 88, 110, 206, 202, 63, - 96, 57, 88, 116, 206, 202, 63, 96, 57, 213, 10, 110, 206, 202, 63, 96, - 57, 238, 173, 219, 6, 213, 10, 116, 206, 202, 63, 96, 57, 47, 88, 63, 96, - 57, 193, 88, 96, 57, 248, 140, 202, 3, 234, 206, 57, 248, 140, 96, 57, - 248, 140, 186, 234, 206, 57, 88, 248, 140, 202, 3, 234, 206, 57, 88, 248, - 140, 96, 57, 88, 248, 140, 186, 234, 206, 57, 199, 129, 96, 57, 88, 199, - 128, 96, 57, 193, 115, 96, 57, 88, 193, 115, 96, 57, 211, 73, 96, 57, 55, - 238, 173, 219, 6, 115, 236, 253, 251, 117, 63, 197, 242, 239, 35, 2, 63, - 197, 241, 210, 0, 211, 79, 200, 229, 211, 79, 200, 180, 45, 206, 8, 249, - 133, 237, 114, 50, 206, 8, 249, 133, 237, 114, 211, 59, 4, 75, 223, 119, - 207, 20, 202, 25, 208, 232, 200, 229, 200, 181, 208, 232, 202, 24, 81, - 249, 90, 4, 228, 243, 106, 13, 206, 181, 237, 40, 180, 236, 201, 13, 233, - 85, 237, 40, 112, 219, 31, 251, 127, 112, 219, 31, 211, 58, 63, 237, 35, - 4, 247, 22, 236, 142, 23, 4, 236, 142, 234, 134, 79, 211, 71, 196, 65, - 110, 50, 239, 4, 4, 236, 142, 116, 45, 239, 4, 4, 236, 142, 45, 211, 14, - 222, 201, 50, 211, 14, 222, 201, 232, 82, 211, 14, 222, 201, 220, 15, - 133, 199, 228, 220, 15, 144, 199, 228, 45, 23, 50, 55, 197, 57, 45, 23, - 50, 199, 228, 45, 215, 133, 180, 50, 199, 228, 180, 45, 199, 228, 133, - 199, 229, 4, 243, 13, 58, 218, 235, 236, 208, 247, 207, 228, 243, 206, - 53, 63, 239, 57, 237, 34, 63, 239, 57, 237, 35, 4, 118, 198, 182, 63, - 239, 57, 237, 35, 4, 96, 198, 182, 63, 51, 4, 118, 198, 182, 63, 51, 4, - 96, 198, 182, 13, 45, 63, 51, 248, 55, 13, 50, 63, 51, 248, 55, 13, 45, - 251, 118, 248, 55, 13, 50, 251, 118, 248, 55, 13, 45, 55, 251, 118, 248, - 55, 13, 50, 55, 251, 118, 248, 55, 13, 45, 63, 198, 42, 203, 104, 248, - 55, 13, 50, 63, 198, 42, 203, 104, 248, 55, 13, 45, 232, 205, 210, 115, - 13, 50, 232, 205, 210, 115, 196, 66, 208, 28, 57, 235, 121, 208, 28, 57, - 251, 87, 231, 204, 243, 13, 57, 242, 223, 231, 204, 243, 13, 57, 50, 64, - 4, 47, 209, 83, 180, 118, 57, 180, 96, 57, 180, 45, 50, 57, 180, 118, 55, - 57, 180, 96, 55, 57, 180, 45, 50, 55, 57, 180, 118, 64, 232, 100, 164, - 180, 96, 64, 232, 100, 164, 180, 118, 55, 64, 232, 100, 164, 180, 96, 55, - 64, 232, 100, 164, 180, 96, 201, 184, 57, 69, 70, 248, 134, 69, 70, 236, - 139, 69, 70, 236, 11, 69, 70, 236, 138, 69, 70, 235, 203, 69, 70, 236, - 74, 69, 70, 236, 10, 69, 70, 236, 137, 69, 70, 235, 171, 69, 70, 236, 42, - 69, 70, 235, 234, 69, 70, 236, 105, 69, 70, 235, 202, 69, 70, 236, 73, - 69, 70, 236, 9, 69, 70, 236, 136, 69, 70, 235, 155, 69, 70, 236, 26, 69, - 70, 235, 218, 69, 70, 236, 89, 69, 70, 235, 186, 69, 70, 236, 57, 69, 70, - 235, 249, 69, 70, 236, 120, 69, 70, 235, 170, 69, 70, 236, 41, 69, 70, - 235, 233, 69, 70, 236, 104, 69, 70, 235, 201, 69, 70, 236, 72, 69, 70, - 236, 8, 69, 70, 236, 135, 69, 70, 235, 147, 69, 70, 236, 18, 69, 70, 235, - 210, 69, 70, 236, 81, 69, 70, 235, 178, 69, 70, 236, 49, 69, 70, 235, - 241, 69, 70, 236, 112, 69, 70, 235, 162, 69, 70, 236, 33, 69, 70, 235, - 225, 69, 70, 236, 96, 69, 70, 235, 193, 69, 70, 236, 64, 69, 70, 236, 0, - 69, 70, 236, 127, 69, 70, 235, 154, 69, 70, 236, 25, 69, 70, 235, 217, - 69, 70, 236, 88, 69, 70, 235, 185, 69, 70, 236, 56, 69, 70, 235, 248, 69, - 70, 236, 119, 69, 70, 235, 169, 69, 70, 236, 40, 69, 70, 235, 232, 69, - 70, 236, 103, 69, 70, 235, 200, 69, 70, 236, 71, 69, 70, 236, 7, 69, 70, - 236, 134, 69, 70, 235, 143, 69, 70, 236, 14, 69, 70, 235, 206, 69, 70, - 236, 77, 69, 70, 235, 174, 69, 70, 236, 45, 69, 70, 235, 237, 69, 70, - 236, 108, 69, 70, 235, 158, 69, 70, 236, 29, 69, 70, 235, 221, 69, 70, - 236, 92, 69, 70, 235, 189, 69, 70, 236, 60, 69, 70, 235, 252, 69, 70, - 236, 123, 69, 70, 235, 150, 69, 70, 236, 21, 69, 70, 235, 213, 69, 70, - 236, 84, 69, 70, 235, 181, 69, 70, 236, 52, 69, 70, 235, 244, 69, 70, - 236, 115, 69, 70, 235, 165, 69, 70, 236, 36, 69, 70, 235, 228, 69, 70, - 236, 99, 69, 70, 235, 196, 69, 70, 236, 67, 69, 70, 236, 3, 69, 70, 236, - 130, 69, 70, 235, 146, 69, 70, 236, 17, 69, 70, 235, 209, 69, 70, 236, - 80, 69, 70, 235, 177, 69, 70, 236, 48, 69, 70, 235, 240, 69, 70, 236, - 111, 69, 70, 235, 161, 69, 70, 236, 32, 69, 70, 235, 224, 69, 70, 236, - 95, 69, 70, 235, 192, 69, 70, 236, 63, 69, 70, 235, 255, 69, 70, 236, - 126, 69, 70, 235, 153, 69, 70, 236, 24, 69, 70, 235, 216, 69, 70, 236, - 87, 69, 70, 235, 184, 69, 70, 236, 55, 69, 70, 235, 247, 69, 70, 236, - 118, 69, 70, 235, 168, 69, 70, 236, 39, 69, 70, 235, 231, 69, 70, 236, - 102, 69, 70, 235, 199, 69, 70, 236, 70, 69, 70, 236, 6, 69, 70, 236, 133, - 69, 70, 235, 141, 69, 70, 236, 12, 69, 70, 235, 204, 69, 70, 236, 75, 69, - 70, 235, 172, 69, 70, 236, 43, 69, 70, 235, 235, 69, 70, 236, 106, 69, - 70, 235, 156, 69, 70, 236, 27, 69, 70, 235, 219, 69, 70, 236, 90, 69, 70, - 235, 187, 69, 70, 236, 58, 69, 70, 235, 250, 69, 70, 236, 121, 69, 70, - 235, 148, 69, 70, 236, 19, 69, 70, 235, 211, 69, 70, 236, 82, 69, 70, - 235, 179, 69, 70, 236, 50, 69, 70, 235, 242, 69, 70, 236, 113, 69, 70, - 235, 163, 69, 70, 236, 34, 69, 70, 235, 226, 69, 70, 236, 97, 69, 70, - 235, 194, 69, 70, 236, 65, 69, 70, 236, 1, 69, 70, 236, 128, 69, 70, 235, - 144, 69, 70, 236, 15, 69, 70, 235, 207, 69, 70, 236, 78, 69, 70, 235, - 175, 69, 70, 236, 46, 69, 70, 235, 238, 69, 70, 236, 109, 69, 70, 235, - 159, 69, 70, 236, 30, 69, 70, 235, 222, 69, 70, 236, 93, 69, 70, 235, - 190, 69, 70, 236, 61, 69, 70, 235, 253, 69, 70, 236, 124, 69, 70, 235, - 151, 69, 70, 236, 22, 69, 70, 235, 214, 69, 70, 236, 85, 69, 70, 235, - 182, 69, 70, 236, 53, 69, 70, 235, 245, 69, 70, 236, 116, 69, 70, 235, - 166, 69, 70, 236, 37, 69, 70, 235, 229, 69, 70, 236, 100, 69, 70, 235, - 197, 69, 70, 236, 68, 69, 70, 236, 4, 69, 70, 236, 131, 69, 70, 235, 142, - 69, 70, 236, 13, 69, 70, 235, 205, 69, 70, 236, 76, 69, 70, 235, 173, 69, - 70, 236, 44, 69, 70, 235, 236, 69, 70, 236, 107, 69, 70, 235, 157, 69, - 70, 236, 28, 69, 70, 235, 220, 69, 70, 236, 91, 69, 70, 235, 188, 69, 70, - 236, 59, 69, 70, 235, 251, 69, 70, 236, 122, 69, 70, 235, 149, 69, 70, - 236, 20, 69, 70, 235, 212, 69, 70, 236, 83, 69, 70, 235, 180, 69, 70, - 236, 51, 69, 70, 235, 243, 69, 70, 236, 114, 69, 70, 235, 164, 69, 70, - 236, 35, 69, 70, 235, 227, 69, 70, 236, 98, 69, 70, 235, 195, 69, 70, - 236, 66, 69, 70, 236, 2, 69, 70, 236, 129, 69, 70, 235, 145, 69, 70, 236, - 16, 69, 70, 235, 208, 69, 70, 236, 79, 69, 70, 235, 176, 69, 70, 236, 47, - 69, 70, 235, 239, 69, 70, 236, 110, 69, 70, 235, 160, 69, 70, 236, 31, - 69, 70, 235, 223, 69, 70, 236, 94, 69, 70, 235, 191, 69, 70, 236, 62, 69, - 70, 235, 254, 69, 70, 236, 125, 69, 70, 235, 152, 69, 70, 236, 23, 69, - 70, 235, 215, 69, 70, 236, 86, 69, 70, 235, 183, 69, 70, 236, 54, 69, 70, - 235, 246, 69, 70, 236, 117, 69, 70, 235, 167, 69, 70, 236, 38, 69, 70, - 235, 230, 69, 70, 236, 101, 69, 70, 235, 198, 69, 70, 236, 69, 69, 70, - 236, 5, 69, 70, 236, 132, 96, 197, 13, 64, 4, 81, 106, 96, 197, 13, 64, - 4, 55, 81, 106, 118, 55, 64, 4, 81, 106, 96, 55, 64, 4, 81, 106, 45, 50, - 55, 64, 4, 81, 106, 96, 197, 13, 64, 232, 100, 164, 118, 55, 64, 232, - 100, 164, 96, 55, 64, 232, 100, 164, 235, 121, 64, 4, 228, 243, 106, 196, - 66, 64, 4, 228, 243, 106, 196, 66, 197, 225, 57, 235, 121, 197, 225, 57, - 118, 55, 237, 218, 57, 96, 55, 237, 218, 57, 118, 197, 225, 237, 218, 57, - 96, 197, 225, 237, 218, 57, 96, 197, 13, 197, 225, 237, 218, 57, 96, 64, - 4, 235, 140, 201, 81, 196, 66, 64, 119, 164, 235, 121, 64, 119, 164, 96, - 64, 4, 199, 216, 4, 81, 106, 96, 64, 4, 199, 216, 4, 55, 81, 106, 96, - 197, 13, 64, 4, 199, 215, 96, 197, 13, 64, 4, 199, 216, 4, 81, 106, 96, - 197, 13, 64, 4, 199, 216, 4, 55, 81, 106, 118, 250, 239, 96, 250, 239, - 118, 55, 250, 239, 96, 55, 250, 239, 118, 64, 119, 62, 237, 34, 96, 64, - 119, 62, 237, 34, 118, 64, 232, 100, 249, 90, 119, 62, 237, 34, 96, 64, - 232, 100, 249, 90, 119, 62, 237, 34, 186, 193, 105, 23, 202, 3, 234, 206, - 57, 186, 234, 206, 23, 202, 3, 193, 105, 57, 186, 193, 105, 64, 4, 102, - 186, 234, 206, 64, 4, 102, 202, 3, 234, 206, 64, 4, 102, 202, 3, 193, - 105, 64, 4, 102, 186, 193, 105, 64, 23, 186, 234, 206, 57, 186, 234, 206, - 64, 23, 202, 3, 234, 206, 57, 202, 3, 234, 206, 64, 23, 202, 3, 193, 105, - 57, 202, 3, 193, 105, 64, 23, 186, 193, 105, 57, 206, 181, 237, 41, 238, - 212, 233, 85, 237, 40, 233, 85, 237, 41, 238, 212, 206, 181, 237, 40, - 202, 3, 234, 206, 64, 238, 212, 186, 234, 206, 57, 186, 234, 206, 64, - 238, 212, 202, 3, 234, 206, 57, 233, 85, 237, 41, 238, 212, 186, 234, - 206, 57, 206, 181, 237, 41, 238, 212, 202, 3, 234, 206, 57, 186, 234, - 206, 64, 238, 212, 186, 193, 105, 57, 186, 193, 105, 64, 238, 212, 186, - 234, 206, 57, 193, 139, 64, 209, 60, 236, 233, 179, 64, 209, 60, 96, 199, - 25, 238, 163, 196, 65, 64, 209, 60, 96, 199, 25, 238, 163, 235, 120, 64, - 209, 60, 235, 121, 199, 25, 238, 163, 219, 216, 64, 209, 60, 235, 121, - 199, 25, 238, 163, 206, 198, 206, 201, 251, 19, 242, 223, 57, 219, 219, - 251, 19, 251, 87, 57, 198, 54, 251, 19, 251, 87, 57, 248, 79, 251, 19, - 251, 87, 57, 198, 54, 251, 19, 242, 223, 64, 4, 216, 36, 198, 54, 251, - 19, 251, 87, 64, 4, 209, 83, 110, 50, 204, 28, 242, 223, 57, 110, 45, - 204, 28, 251, 87, 57, 251, 87, 242, 221, 243, 13, 57, 242, 223, 242, 221, - 243, 13, 57, 96, 64, 93, 203, 41, 118, 57, 118, 64, 93, 203, 41, 96, 57, - 203, 41, 96, 64, 93, 118, 57, 96, 64, 4, 108, 60, 118, 64, 4, 108, 60, - 96, 64, 198, 216, 192, 235, 45, 50, 64, 198, 216, 2, 243, 12, 196, 66, - 197, 13, 64, 232, 100, 2, 243, 12, 45, 182, 133, 50, 182, 144, 230, 14, - 45, 182, 144, 50, 182, 133, 230, 14, 133, 182, 50, 144, 182, 45, 230, 14, - 133, 182, 45, 144, 182, 50, 230, 14, 45, 182, 133, 50, 182, 133, 230, 14, - 133, 182, 50, 144, 182, 50, 230, 14, 45, 182, 144, 50, 182, 144, 230, 14, - 133, 182, 45, 144, 182, 45, 230, 14, 118, 230, 15, 4, 182, 133, 119, 164, - 96, 230, 15, 4, 182, 133, 119, 164, 196, 66, 230, 15, 4, 182, 50, 119, - 164, 235, 121, 230, 15, 4, 182, 50, 119, 164, 118, 230, 15, 4, 182, 144, - 119, 164, 96, 230, 15, 4, 182, 144, 119, 164, 196, 66, 230, 15, 4, 182, - 45, 119, 164, 235, 121, 230, 15, 4, 182, 45, 119, 164, 118, 230, 15, 4, - 182, 133, 232, 100, 164, 96, 230, 15, 4, 182, 133, 232, 100, 164, 196, - 66, 230, 15, 4, 182, 50, 232, 100, 164, 235, 121, 230, 15, 4, 182, 50, - 232, 100, 164, 118, 230, 15, 4, 182, 144, 232, 100, 164, 96, 230, 15, 4, - 182, 144, 232, 100, 164, 196, 66, 230, 15, 4, 182, 45, 232, 100, 164, - 235, 121, 230, 15, 4, 182, 45, 232, 100, 164, 118, 230, 15, 4, 182, 133, - 93, 118, 230, 15, 4, 182, 235, 125, 196, 66, 230, 15, 4, 182, 45, 248, - 218, 196, 66, 230, 15, 4, 182, 179, 96, 230, 15, 4, 182, 133, 93, 96, - 230, 15, 4, 182, 235, 125, 235, 121, 230, 15, 4, 182, 45, 248, 218, 235, - 121, 230, 15, 4, 182, 179, 118, 230, 15, 4, 182, 133, 93, 96, 230, 15, 4, - 182, 196, 77, 118, 230, 15, 4, 182, 144, 93, 96, 230, 15, 4, 182, 235, - 125, 96, 230, 15, 4, 182, 133, 93, 118, 230, 15, 4, 182, 196, 77, 96, - 230, 15, 4, 182, 144, 93, 118, 230, 15, 4, 182, 235, 125, 118, 230, 15, - 4, 182, 133, 93, 180, 237, 217, 118, 230, 15, 4, 182, 144, 248, 235, 180, - 237, 217, 96, 230, 15, 4, 182, 133, 93, 180, 237, 217, 96, 230, 15, 4, - 182, 144, 248, 235, 180, 237, 217, 196, 66, 230, 15, 4, 182, 45, 248, - 218, 235, 121, 230, 15, 4, 182, 179, 235, 121, 230, 15, 4, 182, 45, 248, - 218, 196, 66, 230, 15, 4, 182, 179, 50, 55, 64, 4, 206, 114, 229, 237, - 234, 45, 3, 93, 96, 57, 198, 153, 211, 69, 93, 96, 57, 118, 64, 93, 198, - 153, 211, 68, 96, 64, 93, 198, 153, 211, 68, 96, 64, 93, 251, 167, 234, - 47, 159, 219, 182, 93, 118, 57, 118, 64, 198, 216, 219, 181, 230, 206, - 93, 96, 57, 200, 230, 93, 96, 57, 118, 64, 198, 216, 200, 229, 200, 181, - 93, 118, 57, 45, 232, 239, 199, 215, 50, 232, 239, 199, 215, 133, 232, - 239, 199, 215, 144, 232, 239, 199, 215, 197, 225, 81, 249, 90, 237, 114, - 191, 167, 213, 12, 201, 202, 191, 167, 213, 12, 196, 255, 242, 85, 45, - 63, 238, 173, 248, 55, 50, 63, 238, 173, 248, 55, 45, 63, 210, 115, 50, - 63, 210, 115, 191, 167, 213, 12, 45, 223, 180, 248, 55, 191, 167, 213, - 12, 50, 223, 180, 248, 55, 191, 167, 213, 12, 45, 248, 168, 248, 55, 191, - 167, 213, 12, 50, 248, 168, 248, 55, 45, 51, 248, 56, 4, 196, 103, 50, - 51, 248, 56, 4, 196, 103, 45, 51, 248, 56, 4, 198, 183, 223, 165, 198, - 54, 239, 3, 50, 51, 248, 56, 4, 198, 183, 223, 165, 248, 79, 239, 3, 45, - 51, 248, 56, 4, 198, 183, 223, 165, 248, 79, 239, 3, 50, 51, 248, 56, 4, - 198, 183, 223, 165, 198, 54, 239, 3, 45, 251, 118, 248, 56, 4, 236, 142, - 50, 251, 118, 248, 56, 4, 236, 142, 45, 251, 19, 219, 182, 248, 55, 50, - 251, 19, 230, 206, 248, 55, 55, 45, 251, 19, 230, 206, 248, 55, 55, 50, - 251, 19, 219, 182, 248, 55, 45, 62, 198, 42, 203, 104, 248, 55, 50, 62, - 198, 42, 203, 104, 248, 55, 235, 140, 233, 41, 81, 191, 21, 219, 114, - 216, 228, 251, 118, 211, 71, 219, 226, 50, 251, 118, 195, 168, 4, 201, - 191, 216, 228, 50, 251, 118, 4, 236, 142, 251, 118, 4, 206, 10, 223, 119, - 252, 49, 251, 117, 201, 226, 251, 118, 211, 71, 219, 226, 201, 226, 251, - 118, 211, 71, 196, 77, 154, 251, 117, 207, 19, 251, 117, 251, 118, 4, - 196, 103, 207, 19, 251, 118, 4, 196, 103, 211, 174, 251, 118, 211, 71, - 196, 77, 211, 174, 251, 118, 211, 71, 235, 125, 216, 228, 251, 118, 4, - 211, 79, 250, 253, 234, 93, 223, 165, 64, 209, 60, 133, 23, 179, 216, - 228, 251, 118, 4, 211, 79, 250, 253, 234, 93, 223, 165, 64, 209, 60, 133, - 23, 219, 226, 216, 228, 251, 118, 4, 211, 79, 250, 253, 234, 93, 223, - 165, 64, 209, 60, 144, 23, 179, 216, 228, 251, 118, 4, 211, 79, 250, 253, - 234, 93, 223, 165, 64, 209, 60, 144, 23, 219, 226, 216, 228, 251, 118, 4, - 211, 79, 250, 253, 234, 93, 223, 165, 64, 209, 60, 50, 23, 196, 77, 216, - 228, 251, 118, 4, 211, 79, 250, 253, 234, 93, 223, 165, 64, 209, 60, 45, - 23, 196, 77, 216, 228, 251, 118, 4, 211, 79, 250, 253, 234, 93, 223, 165, - 64, 209, 60, 50, 23, 235, 125, 216, 228, 251, 118, 4, 211, 79, 250, 253, - 234, 93, 223, 165, 64, 209, 60, 45, 23, 235, 125, 207, 19, 234, 107, 203, - 253, 234, 107, 203, 254, 4, 211, 8, 234, 107, 203, 254, 4, 2, 243, 13, - 58, 234, 107, 203, 254, 4, 50, 64, 58, 234, 107, 203, 254, 4, 45, 64, 58, - 243, 13, 4, 228, 243, 164, 47, 81, 164, 47, 210, 120, 47, 207, 20, 202, - 24, 47, 210, 0, 243, 13, 236, 208, 247, 207, 228, 243, 249, 90, 23, 198, - 54, 132, 236, 208, 247, 207, 81, 164, 243, 13, 4, 200, 183, 192, 235, 47, - 251, 85, 236, 202, 56, 133, 64, 198, 216, 243, 12, 47, 63, 247, 250, 47, - 247, 250, 47, 219, 181, 47, 230, 205, 243, 13, 4, 2, 243, 13, 119, 199, - 34, 179, 243, 13, 4, 105, 228, 243, 201, 17, 119, 199, 34, 179, 112, 206, - 181, 237, 41, 202, 98, 112, 233, 85, 237, 41, 202, 98, 112, 250, 195, - 112, 2, 243, 12, 112, 201, 191, 105, 222, 200, 201, 189, 197, 242, 4, 75, - 58, 197, 242, 4, 196, 103, 206, 10, 223, 165, 197, 241, 197, 242, 4, 204, - 5, 250, 185, 248, 78, 50, 197, 242, 93, 45, 197, 241, 45, 197, 242, 248, - 218, 81, 164, 81, 249, 90, 248, 218, 50, 197, 241, 248, 66, 4, 45, 132, - 248, 141, 248, 66, 4, 50, 132, 248, 141, 62, 248, 65, 25, 4, 45, 132, - 248, 141, 25, 4, 50, 132, 248, 141, 63, 228, 176, 62, 228, 176, 45, 193, - 72, 233, 41, 50, 193, 72, 233, 41, 45, 55, 193, 72, 233, 41, 50, 55, 193, - 72, 233, 41, 223, 157, 223, 141, 198, 179, 139, 223, 141, 223, 142, 214, - 110, 4, 81, 164, 235, 134, 215, 133, 51, 4, 239, 27, 211, 13, 223, 154, - 250, 221, 202, 255, 208, 202, 234, 45, 3, 23, 202, 100, 210, 120, 234, - 45, 3, 23, 202, 100, 210, 121, 4, 198, 153, 58, 228, 19, 119, 23, 202, - 100, 210, 120, 231, 16, 201, 102, 199, 22, 235, 124, 197, 242, 4, 45, - 132, 248, 141, 235, 124, 197, 242, 4, 50, 132, 248, 141, 62, 237, 35, 4, - 144, 57, 62, 218, 234, 63, 243, 13, 4, 144, 57, 62, 243, 13, 4, 144, 57, - 234, 27, 63, 201, 191, 234, 27, 62, 201, 191, 234, 27, 63, 237, 34, 234, - 27, 62, 237, 34, 234, 27, 63, 243, 12, 234, 27, 62, 243, 12, 206, 52, - 207, 20, 202, 25, 211, 68, 202, 25, 4, 211, 8, 207, 20, 202, 25, 4, 228, - 243, 106, 248, 177, 202, 24, 248, 177, 207, 20, 202, 24, 55, 209, 83, - 197, 225, 209, 83, 219, 221, 238, 165, 251, 118, 248, 55, 206, 204, 238, - 165, 251, 118, 248, 55, 198, 137, 216, 34, 215, 62, 47, 75, 211, 68, 215, - 62, 47, 108, 211, 68, 215, 62, 47, 25, 211, 68, 215, 62, 196, 93, 211, - 69, 4, 236, 142, 215, 62, 196, 93, 211, 69, 4, 209, 83, 215, 62, 51, 223, - 102, 211, 68, 215, 62, 51, 196, 93, 211, 68, 105, 219, 31, 23, 211, 68, - 105, 219, 31, 211, 59, 211, 68, 215, 62, 25, 211, 68, 215, 238, 105, 200, - 204, 200, 202, 4, 223, 115, 208, 28, 223, 116, 211, 68, 232, 248, 210, - 109, 223, 115, 223, 116, 4, 55, 106, 223, 116, 250, 145, 4, 202, 98, 243, - 5, 232, 78, 251, 87, 223, 113, 219, 115, 223, 114, 4, 207, 93, 210, 88, - 250, 247, 209, 54, 219, 115, 223, 114, 4, 204, 28, 210, 88, 250, 247, - 209, 54, 219, 115, 223, 114, 213, 14, 223, 159, 199, 34, 209, 54, 223, - 116, 250, 247, 42, 209, 64, 211, 68, 208, 21, 223, 116, 211, 68, 223, - 116, 4, 118, 64, 4, 102, 223, 116, 4, 25, 56, 223, 116, 4, 223, 101, 223, - 116, 4, 196, 92, 223, 116, 4, 211, 8, 223, 116, 4, 196, 103, 222, 201, - 220, 15, 45, 197, 242, 211, 68, 191, 167, 213, 12, 205, 93, 239, 64, 191, - 167, 213, 12, 205, 93, 209, 128, 191, 167, 213, 12, 205, 93, 208, 197, - 108, 3, 4, 2, 243, 13, 58, 108, 3, 4, 243, 4, 252, 63, 58, 108, 3, 4, - 198, 153, 58, 108, 3, 4, 75, 60, 108, 3, 4, 198, 153, 60, 108, 3, 4, 200, - 231, 109, 108, 3, 4, 62, 197, 241, 216, 37, 3, 4, 242, 77, 58, 216, 37, - 3, 4, 75, 60, 216, 37, 3, 4, 233, 85, 236, 140, 216, 37, 3, 4, 206, 181, - 236, 140, 108, 3, 223, 165, 45, 132, 243, 12, 108, 3, 223, 165, 50, 132, - 243, 12, 195, 152, 211, 59, 238, 220, 208, 202, 215, 129, 3, 4, 75, 58, - 215, 129, 3, 4, 196, 103, 204, 25, 208, 203, 4, 248, 79, 242, 220, 202, - 69, 208, 202, 215, 129, 3, 223, 165, 45, 132, 243, 12, 215, 129, 3, 223, - 165, 50, 132, 243, 12, 47, 215, 129, 3, 4, 243, 4, 252, 62, 215, 129, 3, - 223, 165, 55, 243, 12, 47, 236, 202, 56, 108, 3, 223, 165, 197, 241, 216, - 37, 3, 223, 165, 197, 241, 215, 129, 3, 223, 165, 197, 241, 223, 110, - 208, 202, 206, 199, 223, 110, 208, 202, 191, 167, 213, 12, 207, 65, 239, - 64, 251, 149, 211, 59, 239, 11, 223, 102, 4, 236, 142, 196, 93, 4, 216, - 37, 56, 196, 93, 4, 211, 8, 223, 102, 4, 211, 8, 223, 102, 4, 219, 31, - 251, 127, 196, 93, 4, 219, 31, 211, 58, 196, 93, 93, 223, 101, 223, 102, - 93, 196, 92, 196, 93, 93, 249, 90, 93, 223, 101, 223, 102, 93, 249, 90, - 93, 196, 92, 196, 93, 248, 218, 23, 222, 200, 4, 196, 92, 223, 102, 248, - 218, 23, 222, 200, 4, 223, 101, 242, 221, 196, 93, 4, 204, 4, 242, 221, - 223, 102, 4, 204, 4, 55, 51, 223, 101, 55, 51, 196, 92, 242, 221, 196, - 93, 4, 204, 5, 23, 202, 69, 208, 202, 219, 31, 23, 4, 75, 58, 219, 31, - 211, 59, 4, 75, 58, 55, 219, 31, 251, 127, 55, 219, 31, 211, 58, 105, - 223, 103, 219, 31, 251, 127, 105, 223, 103, 219, 31, 211, 58, 202, 81, - 220, 15, 211, 58, 202, 81, 220, 15, 251, 127, 219, 31, 211, 59, 211, 3, - 219, 31, 251, 127, 219, 31, 23, 4, 82, 201, 81, 219, 31, 211, 59, 4, 82, - 201, 81, 219, 31, 23, 4, 228, 243, 237, 217, 219, 31, 211, 59, 4, 228, - 243, 237, 217, 219, 31, 23, 4, 55, 211, 8, 219, 31, 23, 4, 196, 103, 219, - 31, 23, 4, 55, 196, 103, 2, 195, 149, 4, 196, 103, 219, 31, 211, 59, 4, - 55, 211, 8, 219, 31, 211, 59, 4, 55, 196, 103, 191, 167, 213, 12, 236, - 154, 251, 77, 191, 167, 213, 12, 207, 139, 251, 77, 234, 45, 3, 4, 75, - 60, 228, 19, 4, 75, 58, 197, 225, 228, 243, 249, 90, 4, 55, 81, 106, 197, - 225, 228, 243, 249, 90, 4, 197, 225, 81, 106, 198, 153, 211, 69, 4, 75, - 58, 198, 153, 211, 69, 4, 206, 181, 236, 140, 202, 181, 216, 37, 202, - 180, 239, 51, 4, 75, 58, 234, 45, 4, 250, 195, 251, 167, 234, 47, 119, 4, - 243, 4, 252, 62, 251, 42, 234, 47, 211, 59, 234, 47, 159, 234, 45, 3, 93, - 108, 56, 108, 3, 93, 234, 45, 56, 234, 45, 3, 93, 198, 153, 211, 68, 55, - 242, 86, 234, 46, 105, 239, 43, 234, 45, 202, 195, 115, 239, 43, 234, 45, - 202, 195, 234, 45, 3, 4, 105, 185, 93, 23, 105, 185, 60, 234, 38, 4, 232, - 130, 185, 58, 219, 182, 4, 243, 13, 223, 119, 230, 206, 4, 243, 13, 223, - 119, 219, 182, 4, 208, 15, 87, 58, 230, 206, 4, 208, 15, 87, 58, 219, - 182, 211, 59, 202, 100, 234, 47, 159, 230, 206, 211, 59, 202, 100, 234, - 47, 159, 219, 182, 211, 59, 202, 100, 234, 47, 119, 4, 75, 223, 119, 230, - 206, 211, 59, 202, 100, 234, 47, 119, 4, 75, 223, 119, 219, 182, 211, 59, - 202, 100, 234, 47, 119, 4, 75, 58, 230, 206, 211, 59, 202, 100, 234, 47, - 119, 4, 75, 58, 219, 182, 211, 59, 202, 100, 234, 47, 119, 4, 75, 93, - 179, 230, 206, 211, 59, 202, 100, 234, 47, 119, 4, 75, 93, 219, 226, 219, - 182, 211, 59, 251, 43, 230, 206, 211, 59, 251, 43, 219, 182, 23, 202, - 169, 213, 14, 234, 47, 159, 230, 206, 23, 202, 169, 213, 14, 234, 47, - 159, 219, 182, 23, 213, 14, 251, 43, 230, 206, 23, 213, 14, 251, 43, 219, - 182, 93, 235, 133, 234, 47, 93, 230, 205, 230, 206, 93, 235, 133, 234, - 47, 93, 219, 181, 219, 182, 93, 202, 181, 211, 59, 234, 46, 230, 206, 93, - 202, 181, 211, 59, 234, 46, 219, 182, 93, 202, 181, 93, 230, 205, 230, - 206, 93, 202, 181, 93, 219, 181, 219, 182, 93, 230, 206, 93, 235, 133, - 234, 46, 230, 206, 93, 219, 182, 93, 235, 133, 234, 46, 219, 182, 93, - 202, 100, 234, 47, 93, 230, 206, 93, 202, 100, 234, 46, 230, 206, 93, - 202, 100, 234, 47, 93, 219, 182, 93, 202, 100, 234, 46, 202, 100, 234, - 47, 119, 211, 59, 219, 181, 202, 100, 234, 47, 119, 211, 59, 230, 205, - 202, 100, 234, 47, 119, 211, 59, 219, 182, 4, 75, 223, 119, 202, 100, - 234, 47, 119, 211, 59, 230, 206, 4, 75, 223, 119, 235, 133, 234, 47, 119, - 211, 59, 219, 181, 235, 133, 234, 47, 119, 211, 59, 230, 205, 235, 133, - 202, 100, 234, 47, 119, 211, 59, 219, 181, 235, 133, 202, 100, 234, 47, - 119, 211, 59, 230, 205, 202, 181, 211, 59, 219, 181, 202, 181, 211, 59, - 230, 205, 202, 181, 93, 219, 182, 93, 234, 45, 56, 202, 181, 93, 230, - 206, 93, 234, 45, 56, 55, 214, 90, 219, 181, 55, 214, 90, 230, 205, 55, - 214, 90, 219, 182, 4, 196, 103, 230, 206, 211, 3, 219, 181, 230, 206, - 248, 218, 219, 181, 219, 182, 242, 221, 247, 207, 238, 166, 230, 206, - 242, 221, 247, 207, 238, 166, 219, 182, 242, 221, 247, 207, 238, 167, 93, - 202, 100, 234, 46, 230, 206, 242, 221, 247, 207, 238, 167, 93, 202, 100, - 234, 46, 202, 70, 199, 38, 220, 13, 199, 38, 202, 70, 199, 39, 211, 59, - 234, 47, 159, 220, 13, 199, 39, 211, 59, 234, 47, 159, 234, 45, 3, 4, - 247, 243, 58, 208, 234, 93, 202, 169, 234, 45, 56, 200, 221, 93, 202, - 169, 234, 45, 56, 208, 234, 93, 202, 169, 213, 14, 234, 47, 159, 200, - 221, 93, 202, 169, 213, 14, 234, 47, 159, 208, 234, 93, 234, 45, 56, 200, - 221, 93, 234, 45, 56, 208, 234, 93, 213, 14, 234, 47, 159, 200, 221, 93, - 213, 14, 234, 47, 159, 208, 234, 93, 251, 167, 234, 47, 159, 200, 221, - 93, 251, 167, 234, 47, 159, 208, 234, 93, 213, 14, 251, 167, 234, 47, - 159, 200, 221, 93, 213, 14, 251, 167, 234, 47, 159, 55, 208, 233, 55, - 200, 220, 200, 230, 4, 236, 142, 200, 181, 4, 236, 142, 200, 230, 4, 108, - 3, 60, 200, 181, 4, 108, 3, 60, 200, 230, 4, 215, 129, 3, 60, 200, 181, - 4, 215, 129, 3, 60, 200, 230, 79, 211, 59, 234, 47, 119, 4, 75, 58, 200, - 181, 79, 211, 59, 234, 47, 119, 4, 75, 58, 200, 230, 79, 93, 234, 45, 56, - 200, 181, 79, 93, 234, 45, 56, 200, 230, 79, 93, 198, 153, 211, 68, 200, - 181, 79, 93, 198, 153, 211, 68, 200, 230, 79, 93, 251, 167, 234, 47, 159, - 200, 181, 79, 93, 251, 167, 234, 47, 159, 200, 230, 79, 93, 213, 14, 234, - 47, 159, 200, 181, 79, 93, 213, 14, 234, 47, 159, 51, 45, 211, 79, 111, - 211, 68, 51, 50, 211, 79, 111, 211, 68, 242, 221, 200, 229, 242, 221, - 200, 180, 242, 221, 200, 230, 211, 59, 234, 47, 159, 242, 221, 200, 181, - 211, 59, 234, 47, 159, 200, 230, 93, 200, 180, 200, 181, 93, 200, 229, - 200, 230, 93, 200, 229, 200, 181, 93, 200, 180, 200, 181, 248, 218, 200, - 229, 200, 181, 248, 218, 23, 222, 200, 247, 207, 237, 218, 4, 200, 229, - 234, 134, 79, 211, 71, 235, 120, 209, 118, 4, 199, 122, 198, 53, 198, 7, - 223, 101, 232, 149, 213, 29, 203, 41, 45, 199, 228, 203, 41, 144, 199, - 228, 203, 41, 133, 199, 228, 210, 1, 4, 206, 9, 81, 249, 90, 197, 225, - 50, 197, 57, 55, 81, 249, 90, 45, 197, 57, 81, 249, 90, 55, 45, 197, 57, - 55, 81, 249, 90, 55, 45, 197, 57, 180, 237, 218, 232, 100, 45, 216, 193, - 79, 55, 195, 135, 203, 41, 144, 199, 229, 4, 211, 8, 203, 41, 133, 199, - 229, 4, 196, 103, 203, 41, 133, 199, 229, 93, 203, 41, 144, 199, 228, 55, - 144, 199, 228, 55, 133, 199, 228, 55, 201, 29, 213, 14, 56, 207, 19, 55, - 201, 29, 213, 14, 56, 236, 166, 213, 14, 236, 210, 4, 207, 19, 214, 109, - 202, 98, 81, 219, 115, 4, 243, 13, 58, 81, 219, 115, 4, 243, 13, 60, 144, - 199, 229, 4, 243, 13, 60, 210, 121, 4, 228, 243, 106, 210, 121, 4, 198, - 153, 211, 68, 197, 225, 81, 249, 90, 248, 170, 207, 66, 197, 225, 81, - 249, 90, 4, 228, 243, 106, 197, 225, 242, 86, 211, 68, 197, 225, 214, 90, - 219, 181, 197, 225, 214, 90, 230, 205, 235, 133, 202, 100, 219, 182, 211, - 59, 234, 47, 159, 235, 133, 202, 100, 230, 206, 211, 59, 234, 47, 159, - 197, 225, 202, 25, 248, 170, 207, 66, 220, 15, 197, 225, 81, 249, 90, - 211, 68, 55, 202, 25, 211, 68, 63, 81, 164, 215, 62, 63, 81, 164, 186, - 234, 206, 63, 57, 186, 193, 105, 63, 57, 202, 3, 234, 206, 63, 57, 202, - 3, 193, 105, 63, 57, 45, 50, 63, 57, 118, 62, 57, 196, 66, 62, 57, 235, - 121, 62, 57, 186, 234, 206, 62, 57, 186, 193, 105, 62, 57, 202, 3, 234, - 206, 62, 57, 202, 3, 193, 105, 62, 57, 45, 50, 62, 57, 133, 144, 62, 57, - 96, 64, 4, 198, 136, 235, 120, 96, 64, 4, 198, 136, 196, 65, 118, 64, 4, - 198, 136, 235, 120, 118, 64, 4, 198, 136, 196, 65, 51, 4, 198, 54, 132, - 248, 141, 51, 4, 248, 79, 132, 248, 141, 51, 4, 116, 50, 237, 41, 132, - 248, 141, 51, 4, 110, 45, 237, 41, 132, 248, 141, 237, 35, 4, 45, 132, - 248, 141, 237, 35, 4, 50, 132, 248, 141, 237, 35, 4, 198, 54, 132, 248, - 141, 237, 35, 4, 248, 79, 132, 248, 141, 235, 140, 201, 191, 62, 220, 15, - 201, 191, 63, 220, 15, 201, 191, 62, 195, 83, 2, 201, 191, 63, 195, 83, - 2, 201, 191, 62, 210, 26, 63, 210, 26, 63, 229, 184, 62, 229, 184, 228, - 243, 62, 229, 184, 62, 220, 15, 243, 12, 62, 216, 215, 237, 34, 63, 216, - 215, 237, 34, 62, 216, 215, 218, 234, 63, 216, 215, 218, 234, 62, 2, 237, - 34, 62, 2, 218, 234, 63, 2, 218, 234, 62, 228, 243, 234, 123, 63, 228, - 243, 234, 123, 62, 81, 234, 123, 63, 81, 234, 123, 45, 64, 4, 2, 243, 12, - 115, 118, 250, 233, 45, 64, 4, 47, 209, 83, 180, 118, 201, 184, 57, 118, - 197, 13, 64, 4, 81, 106, 118, 197, 13, 64, 4, 55, 81, 106, 118, 197, 13, - 64, 232, 100, 164, 118, 197, 13, 197, 225, 237, 218, 57, 118, 64, 4, 235, - 140, 201, 81, 118, 64, 4, 199, 216, 4, 81, 106, 118, 64, 4, 199, 216, 4, - 55, 81, 106, 118, 197, 13, 64, 4, 199, 215, 118, 197, 13, 64, 4, 199, - 216, 4, 81, 106, 118, 197, 13, 64, 4, 199, 216, 4, 55, 81, 106, 118, 64, - 198, 216, 192, 235, 193, 139, 64, 209, 60, 236, 233, 219, 226, 234, 45, - 3, 93, 118, 57, 207, 20, 198, 153, 211, 69, 93, 118, 57, 118, 64, 93, - 207, 20, 251, 167, 234, 47, 159, 96, 64, 198, 216, 230, 205, 96, 64, 198, - 216, 200, 180, 118, 208, 28, 57, 96, 208, 28, 57, 207, 20, 198, 153, 211, - 69, 93, 96, 57, 96, 64, 93, 207, 20, 251, 167, 234, 47, 159, 198, 153, - 211, 69, 93, 118, 57, 118, 64, 93, 251, 167, 234, 47, 159, 118, 64, 93, - 207, 20, 198, 153, 211, 68, 96, 64, 93, 207, 20, 198, 153, 211, 68, 235, - 121, 197, 240, 191, 21, 57, 203, 41, 202, 100, 186, 57, 203, 41, 249, - 145, 202, 3, 57, 63, 216, 215, 201, 103, 62, 2, 201, 103, 63, 2, 201, - 103, 62, 206, 204, 210, 26, 63, 206, 204, 210, 26, 88, 220, 15, 243, 12, - 88, 211, 10, 4, 211, 10, 223, 119, 88, 243, 13, 4, 243, 13, 223, 119, 88, - 243, 12, 88, 47, 205, 155, 202, 100, 186, 64, 4, 228, 252, 229, 237, 249, - 145, 202, 3, 64, 4, 228, 252, 199, 215, 202, 100, 186, 64, 4, 228, 243, - 199, 215, 249, 145, 202, 3, 64, 4, 228, 243, 199, 215, 248, 226, 64, 209, - 60, 235, 121, 199, 25, 186, 234, 205, 203, 41, 248, 226, 64, 209, 60, - 235, 121, 199, 25, 186, 234, 205, 118, 197, 240, 57, 196, 66, 197, 240, - 57, 96, 197, 240, 57, 235, 121, 197, 240, 57, 45, 50, 197, 240, 57, 133, - 144, 197, 240, 57, 186, 193, 105, 197, 240, 57, 186, 234, 206, 197, 240, - 57, 202, 3, 234, 206, 197, 240, 57, 202, 3, 193, 105, 197, 240, 57, 118, - 197, 240, 237, 216, 57, 196, 66, 197, 240, 237, 216, 57, 96, 197, 240, - 237, 216, 57, 235, 121, 197, 240, 237, 216, 57, 242, 223, 197, 240, 211, - 79, 243, 13, 57, 251, 87, 197, 240, 211, 79, 243, 13, 57, 118, 197, 240, - 64, 119, 164, 196, 66, 197, 240, 64, 119, 164, 96, 197, 240, 64, 119, - 164, 235, 121, 197, 240, 64, 119, 164, 186, 193, 105, 197, 240, 64, 119, - 164, 186, 234, 206, 197, 240, 64, 119, 164, 202, 3, 234, 206, 197, 240, - 64, 119, 164, 202, 3, 193, 105, 197, 240, 64, 119, 164, 118, 197, 240, - 64, 4, 55, 228, 243, 106, 196, 66, 197, 240, 64, 4, 55, 228, 243, 106, - 96, 197, 240, 64, 4, 55, 228, 243, 106, 235, 121, 197, 240, 64, 4, 55, - 228, 243, 106, 228, 243, 199, 237, 221, 224, 81, 199, 237, 221, 224, 118, - 197, 240, 64, 139, 96, 197, 240, 57, 196, 66, 197, 240, 64, 118, 79, 235, - 121, 197, 240, 57, 96, 197, 240, 64, 139, 118, 197, 240, 57, 235, 121, - 197, 240, 64, 118, 79, 196, 66, 197, 240, 57, 118, 197, 240, 210, 198, - 250, 233, 196, 66, 197, 240, 210, 198, 250, 233, 96, 197, 240, 210, 198, - 250, 233, 235, 121, 197, 240, 210, 198, 250, 233, 118, 62, 47, 63, 57, - 196, 66, 62, 47, 63, 57, 96, 62, 47, 63, 57, 235, 121, 62, 47, 63, 57, - 251, 87, 197, 240, 50, 196, 221, 57, 251, 87, 197, 240, 248, 79, 196, - 221, 57, 251, 87, 197, 240, 45, 196, 221, 57, 251, 87, 197, 240, 198, 54, - 196, 221, 57, 207, 24, 219, 226, 207, 24, 179, 214, 79, 219, 226, 214, - 79, 179, 232, 130, 239, 4, 250, 234, 243, 8, 251, 86, 96, 62, 57, 16, 40, - 196, 255, 42, 234, 135, 198, 225, 198, 52, 118, 234, 39, 250, 237, 198, - 225, 206, 205, 196, 66, 234, 39, 250, 237, 198, 225, 198, 52, 96, 234, - 39, 250, 237, 198, 225, 219, 222, 235, 121, 234, 39, 250, 237, 62, 118, - 234, 39, 250, 237, 62, 196, 66, 234, 39, 250, 237, 62, 96, 234, 39, 250, - 237, 62, 235, 121, 234, 39, 250, 237, 235, 121, 197, 240, 64, 4, 180, - 198, 136, 219, 216, 235, 121, 197, 240, 64, 4, 180, 198, 136, 206, 198, - 196, 66, 197, 240, 64, 4, 180, 198, 136, 219, 216, 196, 66, 197, 240, 64, - 4, 180, 198, 136, 206, 198, 118, 197, 240, 64, 4, 180, 198, 136, 196, 65, - 96, 197, 240, 64, 4, 180, 198, 136, 196, 65, 118, 197, 240, 64, 4, 180, - 198, 136, 235, 120, 96, 197, 240, 64, 4, 180, 198, 136, 235, 120, 62, - 238, 165, 235, 121, 23, 118, 57, 62, 238, 165, 235, 121, 23, 96, 57, 62, - 238, 165, 196, 66, 23, 118, 57, 62, 238, 165, 196, 66, 23, 96, 57, 62, - 238, 165, 118, 23, 196, 66, 57, 62, 238, 165, 96, 23, 196, 66, 57, 62, - 238, 165, 118, 23, 235, 121, 57, 62, 238, 165, 96, 23, 235, 121, 57, 206, - 249, 64, 144, 219, 226, 206, 249, 64, 144, 179, 206, 249, 64, 133, 219, - 226, 206, 249, 64, 133, 179, 206, 249, 64, 45, 196, 77, 206, 249, 64, 50, - 196, 77, 206, 249, 64, 45, 235, 125, 206, 249, 64, 50, 235, 125, 196, 66, - 63, 64, 232, 100, 249, 90, 4, 228, 243, 164, 133, 250, 238, 223, 165, 42, - 207, 95, 248, 64, 211, 3, 63, 201, 189, 211, 3, 63, 23, 62, 201, 189, - 211, 3, 62, 201, 189, 249, 109, 111, 4, 156, 192, 235, 47, 192, 235, 47, - 28, 192, 235, 62, 51, 247, 21, 62, 237, 35, 247, 21, 154, 62, 210, 26, - 228, 243, 62, 211, 162, 62, 211, 162, 62, 216, 215, 196, 76, 197, 242, - 247, 21, 62, 216, 215, 235, 124, 197, 242, 247, 21, 62, 216, 215, 219, - 221, 197, 242, 247, 21, 62, 216, 215, 206, 204, 197, 242, 247, 21, 214, - 97, 232, 148, 109, 198, 54, 132, 62, 243, 12, 248, 79, 132, 62, 243, 12, - 156, 232, 130, 209, 62, 62, 238, 161, 206, 123, 156, 232, 130, 209, 62, - 62, 238, 161, 63, 232, 130, 209, 62, 238, 161, 206, 123, 63, 232, 130, - 209, 62, 238, 161, 51, 209, 27, 223, 146, 196, 107, 56, 230, 189, 77, - 209, 80, 232, 148, 109, 209, 80, 232, 148, 138, 209, 80, 232, 148, 134, - 209, 80, 232, 148, 150, 198, 9, 208, 187, 250, 191, 228, 93, 209, 198, - 214, 93, 63, 215, 208, 204, 34, 62, 237, 35, 211, 107, 238, 219, 197, - 202, 156, 215, 208, 250, 229, 238, 181, 230, 90, 191, 75, 221, 4, 251, - 55, 252, 34, 193, 247, 209, 28, 45, 132, 62, 201, 103, 50, 132, 62, 201, - 103, 201, 104, 4, 45, 132, 248, 141, 201, 104, 4, 50, 132, 248, 141, 118, - 197, 13, 64, 4, 197, 242, 250, 235, 196, 66, 197, 13, 64, 4, 197, 242, - 250, 235, 96, 197, 13, 64, 4, 197, 242, 250, 235, 235, 121, 197, 13, 64, - 4, 197, 242, 250, 235, 234, 29, 232, 148, 107, 234, 29, 232, 148, 109, - 205, 52, 206, 32, 250, 190, 16, 195, 52, 206, 32, 250, 190, 16, 213, 0, - 206, 32, 250, 190, 16, 208, 3, 206, 32, 250, 190, 16, 248, 165, 206, 32, - 250, 190, 16, 204, 17, 206, 32, 250, 190, 16, 198, 0, 234, 45, 3, 4, 223, - 142, 60, 196, 89, 113, 204, 13, 113, 235, 130, 113, 210, 98, 113, 207, - 19, 50, 251, 117, 229, 205, 210, 80, 113, 135, 6, 1, 250, 124, 135, 6, 1, - 247, 254, 135, 6, 1, 195, 151, 135, 6, 1, 231, 20, 135, 6, 1, 236, 171, - 135, 6, 1, 192, 49, 135, 6, 1, 191, 55, 135, 6, 1, 235, 32, 135, 6, 1, - 191, 82, 135, 6, 1, 223, 41, 135, 6, 1, 89, 223, 41, 135, 6, 1, 68, 135, - 6, 1, 236, 192, 135, 6, 1, 222, 96, 135, 6, 1, 219, 77, 135, 6, 1, 215, - 68, 135, 6, 1, 214, 212, 135, 6, 1, 211, 91, 135, 6, 1, 209, 57, 135, 6, - 1, 206, 180, 135, 6, 1, 202, 78, 135, 6, 1, 197, 44, 135, 6, 1, 196, 124, - 135, 6, 1, 232, 103, 135, 6, 1, 229, 190, 135, 6, 1, 211, 22, 135, 6, 1, - 210, 65, 135, 6, 1, 203, 9, 135, 6, 1, 197, 146, 135, 6, 1, 243, 56, 135, - 6, 1, 203, 166, 135, 6, 1, 192, 58, 135, 6, 1, 192, 60, 135, 6, 1, 192, - 93, 135, 6, 1, 201, 221, 140, 135, 6, 1, 191, 225, 135, 6, 1, 2, 191, - 190, 135, 6, 1, 2, 191, 191, 4, 199, 215, 135, 6, 1, 192, 12, 135, 6, 1, - 223, 84, 2, 191, 190, 135, 6, 1, 248, 177, 191, 190, 135, 6, 1, 223, 84, - 248, 177, 191, 190, 135, 6, 1, 232, 230, 135, 6, 1, 223, 39, 135, 6, 1, - 203, 8, 135, 6, 1, 197, 215, 65, 135, 6, 1, 220, 3, 215, 68, 135, 6, 1, - 247, 75, 243, 56, 135, 2, 1, 250, 124, 135, 2, 1, 247, 254, 135, 2, 1, - 195, 151, 135, 2, 1, 231, 20, 135, 2, 1, 236, 171, 135, 2, 1, 192, 49, - 135, 2, 1, 191, 55, 135, 2, 1, 235, 32, 135, 2, 1, 191, 82, 135, 2, 1, - 223, 41, 135, 2, 1, 89, 223, 41, 135, 2, 1, 68, 135, 2, 1, 236, 192, 135, - 2, 1, 222, 96, 135, 2, 1, 219, 77, 135, 2, 1, 215, 68, 135, 2, 1, 214, - 212, 135, 2, 1, 211, 91, 135, 2, 1, 209, 57, 135, 2, 1, 206, 180, 135, 2, - 1, 202, 78, 135, 2, 1, 197, 44, 135, 2, 1, 196, 124, 135, 2, 1, 232, 103, - 135, 2, 1, 229, 190, 135, 2, 1, 211, 22, 135, 2, 1, 210, 65, 135, 2, 1, - 203, 9, 135, 2, 1, 197, 146, 135, 2, 1, 243, 56, 135, 2, 1, 203, 166, - 135, 2, 1, 192, 58, 135, 2, 1, 192, 60, 135, 2, 1, 192, 93, 135, 2, 1, - 201, 221, 140, 135, 2, 1, 191, 225, 135, 2, 1, 2, 191, 190, 135, 2, 1, 2, - 191, 191, 4, 199, 215, 135, 2, 1, 192, 12, 135, 2, 1, 223, 84, 2, 191, - 190, 135, 2, 1, 248, 177, 191, 190, 135, 2, 1, 223, 84, 248, 177, 191, - 190, 135, 2, 1, 232, 230, 135, 2, 1, 223, 39, 135, 2, 1, 203, 8, 135, 2, - 1, 197, 215, 65, 135, 2, 1, 220, 3, 215, 68, 135, 2, 1, 247, 75, 243, 56, - 8, 6, 1, 220, 145, 4, 55, 164, 8, 2, 1, 220, 145, 4, 55, 164, 8, 6, 1, - 220, 145, 4, 82, 198, 152, 8, 6, 1, 210, 239, 4, 106, 8, 6, 1, 207, 224, - 4, 199, 215, 8, 2, 1, 42, 4, 106, 8, 2, 1, 200, 44, 4, 237, 41, 106, 8, - 6, 1, 230, 119, 4, 237, 89, 8, 2, 1, 230, 119, 4, 237, 89, 8, 6, 1, 222, - 155, 4, 237, 89, 8, 2, 1, 222, 155, 4, 237, 89, 8, 6, 1, 191, 167, 4, - 237, 89, 8, 2, 1, 191, 167, 4, 237, 89, 8, 6, 1, 251, 162, 8, 6, 1, 218, - 171, 4, 102, 8, 6, 1, 154, 65, 8, 6, 1, 154, 251, 162, 8, 2, 1, 196, 13, - 4, 50, 102, 8, 6, 1, 193, 225, 4, 102, 8, 2, 1, 193, 225, 4, 102, 8, 2, - 1, 196, 13, 4, 238, 177, 8, 6, 1, 132, 230, 118, 8, 2, 1, 132, 230, 118, - 8, 2, 1, 199, 213, 209, 213, 8, 2, 1, 235, 17, 4, 213, 11, 8, 2, 1, 154, - 207, 224, 4, 199, 215, 8, 2, 1, 187, 4, 130, 206, 190, 223, 119, 8, 1, 2, - 6, 154, 71, 8, 200, 231, 2, 1, 223, 37, 52, 1, 6, 196, 12, 8, 6, 1, 206, - 10, 4, 200, 146, 199, 215, 8, 6, 1, 191, 167, 4, 200, 146, 199, 215, 94, - 6, 1, 251, 188, 94, 2, 1, 251, 188, 94, 6, 1, 195, 66, 94, 2, 1, 195, 66, - 94, 6, 1, 231, 213, 94, 2, 1, 231, 213, 94, 6, 1, 238, 1, 94, 2, 1, 238, - 1, 94, 6, 1, 234, 167, 94, 2, 1, 234, 167, 94, 6, 1, 202, 8, 94, 2, 1, - 202, 8, 94, 6, 1, 191, 95, 94, 2, 1, 191, 95, 94, 6, 1, 230, 8, 94, 2, 1, - 230, 8, 94, 6, 1, 199, 13, 94, 2, 1, 199, 13, 94, 6, 1, 228, 34, 94, 2, - 1, 228, 34, 94, 6, 1, 222, 79, 94, 2, 1, 222, 79, 94, 6, 1, 219, 254, 94, - 2, 1, 219, 254, 94, 6, 1, 216, 102, 94, 2, 1, 216, 102, 94, 6, 1, 213, - 221, 94, 2, 1, 213, 221, 94, 6, 1, 220, 250, 94, 2, 1, 220, 250, 94, 6, - 1, 74, 94, 2, 1, 74, 94, 6, 1, 209, 187, 94, 2, 1, 209, 187, 94, 6, 1, - 206, 163, 94, 2, 1, 206, 163, 94, 6, 1, 202, 184, 94, 2, 1, 202, 184, 94, - 6, 1, 199, 166, 94, 2, 1, 199, 166, 94, 6, 1, 196, 168, 94, 2, 1, 196, - 168, 94, 6, 1, 233, 25, 94, 2, 1, 233, 25, 94, 6, 1, 221, 192, 94, 2, 1, - 221, 192, 94, 6, 1, 208, 178, 94, 2, 1, 208, 178, 94, 6, 1, 211, 83, 94, - 2, 1, 211, 83, 94, 6, 1, 237, 39, 251, 194, 94, 2, 1, 237, 39, 251, 194, - 94, 6, 1, 39, 94, 251, 232, 94, 2, 1, 39, 94, 251, 232, 94, 6, 1, 238, - 200, 234, 167, 94, 2, 1, 238, 200, 234, 167, 94, 6, 1, 237, 39, 222, 79, - 94, 2, 1, 237, 39, 222, 79, 94, 6, 1, 237, 39, 213, 221, 94, 2, 1, 237, - 39, 213, 221, 94, 6, 1, 238, 200, 213, 221, 94, 2, 1, 238, 200, 213, 221, - 94, 6, 1, 39, 94, 211, 83, 94, 2, 1, 39, 94, 211, 83, 94, 6, 1, 205, 146, - 94, 2, 1, 205, 146, 94, 6, 1, 238, 216, 203, 106, 94, 2, 1, 238, 216, - 203, 106, 94, 6, 1, 39, 94, 203, 106, 94, 2, 1, 39, 94, 203, 106, 94, 6, - 1, 39, 94, 234, 14, 94, 2, 1, 39, 94, 234, 14, 94, 6, 1, 251, 214, 221, - 197, 94, 2, 1, 251, 214, 221, 197, 94, 6, 1, 237, 39, 228, 244, 94, 2, 1, - 237, 39, 228, 244, 94, 6, 1, 39, 94, 228, 244, 94, 2, 1, 39, 94, 228, - 244, 94, 6, 1, 39, 94, 140, 94, 2, 1, 39, 94, 140, 94, 6, 1, 220, 144, - 140, 94, 2, 1, 220, 144, 140, 94, 6, 1, 39, 94, 229, 211, 94, 2, 1, 39, - 94, 229, 211, 94, 6, 1, 39, 94, 230, 11, 94, 2, 1, 39, 94, 230, 11, 94, - 6, 1, 39, 94, 231, 208, 94, 2, 1, 39, 94, 231, 208, 94, 6, 1, 39, 94, - 236, 195, 94, 2, 1, 39, 94, 236, 195, 94, 6, 1, 39, 94, 203, 72, 94, 2, - 1, 39, 94, 203, 72, 94, 6, 1, 39, 212, 147, 203, 72, 94, 2, 1, 39, 212, - 147, 203, 72, 94, 6, 1, 39, 212, 147, 214, 18, 94, 2, 1, 39, 212, 147, - 214, 18, 94, 6, 1, 39, 212, 147, 212, 83, 94, 2, 1, 39, 212, 147, 212, - 83, 94, 6, 1, 39, 212, 147, 193, 140, 94, 2, 1, 39, 212, 147, 193, 140, - 94, 16, 222, 104, 94, 16, 216, 103, 206, 163, 94, 16, 209, 188, 206, 163, - 94, 16, 201, 90, 94, 16, 199, 167, 206, 163, 94, 16, 221, 193, 206, 163, - 94, 16, 203, 73, 202, 184, 94, 6, 1, 238, 200, 203, 106, 94, 2, 1, 238, - 200, 203, 106, 94, 6, 1, 238, 200, 231, 208, 94, 2, 1, 238, 200, 231, - 208, 94, 33, 213, 222, 58, 94, 33, 201, 214, 250, 203, 94, 33, 201, 214, - 219, 190, 94, 6, 1, 248, 105, 221, 197, 94, 2, 1, 248, 105, 221, 197, 94, - 39, 212, 147, 232, 82, 201, 64, 94, 39, 212, 147, 236, 236, 208, 15, 77, - 94, 39, 212, 147, 223, 144, 208, 15, 77, 94, 39, 212, 147, 195, 137, 236, - 207, 94, 232, 120, 91, 230, 72, 94, 232, 82, 201, 64, 94, 215, 202, 236, - 207, 101, 2, 1, 251, 134, 101, 2, 1, 249, 103, 101, 2, 1, 231, 212, 101, - 2, 1, 236, 152, 101, 2, 1, 234, 105, 101, 2, 1, 195, 49, 101, 2, 1, 191, - 80, 101, 2, 1, 199, 193, 101, 2, 1, 223, 164, 101, 2, 1, 222, 89, 101, 2, - 1, 220, 9, 101, 2, 1, 217, 92, 101, 2, 1, 214, 218, 101, 2, 1, 211, 106, - 101, 2, 1, 210, 133, 101, 2, 1, 191, 67, 101, 2, 1, 207, 165, 101, 2, 1, - 205, 143, 101, 2, 1, 199, 179, 101, 2, 1, 196, 113, 101, 2, 1, 209, 222, - 101, 2, 1, 221, 202, 101, 2, 1, 231, 84, 101, 2, 1, 208, 83, 101, 2, 1, - 203, 70, 101, 2, 1, 243, 83, 101, 2, 1, 247, 130, 101, 2, 1, 222, 236, - 101, 2, 1, 243, 20, 101, 2, 1, 246, 243, 101, 2, 1, 192, 218, 101, 2, 1, - 222, 251, 101, 2, 1, 230, 89, 101, 2, 1, 229, 247, 101, 2, 1, 229, 147, - 101, 2, 1, 193, 125, 101, 2, 1, 230, 21, 101, 2, 1, 229, 13, 101, 2, 1, - 192, 14, 101, 2, 1, 252, 16, 198, 175, 1, 170, 198, 175, 1, 192, 136, - 198, 175, 1, 192, 135, 198, 175, 1, 192, 125, 198, 175, 1, 192, 123, 198, - 175, 1, 248, 220, 252, 64, 192, 118, 198, 175, 1, 192, 118, 198, 175, 1, - 192, 133, 198, 175, 1, 192, 130, 198, 175, 1, 192, 132, 198, 175, 1, 192, - 131, 198, 175, 1, 192, 40, 198, 175, 1, 192, 127, 198, 175, 1, 192, 116, - 198, 175, 1, 197, 86, 192, 116, 198, 175, 1, 192, 113, 198, 175, 1, 192, - 121, 198, 175, 1, 248, 220, 252, 64, 192, 121, 198, 175, 1, 197, 86, 192, - 121, 198, 175, 1, 192, 120, 198, 175, 1, 192, 140, 198, 175, 1, 192, 114, - 198, 175, 1, 197, 86, 192, 114, 198, 175, 1, 192, 103, 198, 175, 1, 197, - 86, 192, 103, 198, 175, 1, 192, 33, 198, 175, 1, 192, 82, 198, 175, 1, - 251, 245, 192, 82, 198, 175, 1, 197, 86, 192, 82, 198, 175, 1, 192, 112, - 198, 175, 1, 192, 111, 198, 175, 1, 192, 108, 198, 175, 1, 197, 86, 192, - 122, 198, 175, 1, 197, 86, 192, 106, 198, 175, 1, 192, 104, 198, 175, 1, - 191, 225, 198, 175, 1, 192, 101, 198, 175, 1, 192, 99, 198, 175, 1, 192, - 124, 198, 175, 1, 197, 86, 192, 124, 198, 175, 1, 250, 129, 192, 124, - 198, 175, 1, 192, 98, 198, 175, 1, 192, 96, 198, 175, 1, 192, 97, 198, - 175, 1, 192, 95, 198, 175, 1, 192, 94, 198, 175, 1, 192, 134, 198, 175, - 1, 192, 92, 198, 175, 1, 192, 90, 198, 175, 1, 192, 89, 198, 175, 1, 192, - 86, 198, 175, 1, 192, 83, 198, 175, 1, 199, 157, 192, 83, 198, 175, 1, - 192, 81, 198, 175, 1, 192, 80, 198, 175, 1, 192, 12, 198, 175, 52, 1, - 220, 117, 77, 198, 175, 204, 12, 77, 198, 175, 120, 222, 198, 36, 5, 219, - 44, 36, 5, 216, 7, 36, 5, 206, 155, 36, 5, 202, 39, 36, 5, 203, 56, 36, - 5, 248, 112, 36, 5, 198, 91, 36, 5, 242, 100, 36, 5, 213, 38, 36, 5, 212, - 66, 36, 5, 231, 13, 211, 184, 36, 5, 191, 6, 36, 5, 236, 174, 36, 5, 237, - 162, 36, 5, 222, 202, 36, 5, 198, 240, 36, 5, 243, 69, 36, 5, 209, 200, - 36, 5, 209, 74, 36, 5, 231, 99, 36, 5, 231, 95, 36, 5, 231, 96, 36, 5, - 231, 97, 36, 5, 201, 176, 36, 5, 201, 130, 36, 5, 201, 143, 36, 5, 201, - 175, 36, 5, 201, 148, 36, 5, 201, 149, 36, 5, 201, 135, 36, 5, 247, 67, - 36, 5, 247, 46, 36, 5, 247, 48, 36, 5, 247, 66, 36, 5, 247, 64, 36, 5, - 247, 65, 36, 5, 247, 47, 36, 5, 190, 224, 36, 5, 190, 202, 36, 5, 190, - 215, 36, 5, 190, 223, 36, 5, 190, 218, 36, 5, 190, 219, 36, 5, 190, 207, - 36, 5, 247, 62, 36, 5, 247, 49, 36, 5, 247, 51, 36, 5, 247, 61, 36, 5, - 247, 59, 36, 5, 247, 60, 36, 5, 247, 50, 36, 5, 207, 236, 36, 5, 207, - 226, 36, 5, 207, 232, 36, 5, 207, 235, 36, 5, 207, 233, 36, 5, 207, 234, - 36, 5, 207, 231, 36, 5, 220, 155, 36, 5, 220, 147, 36, 5, 220, 150, 36, - 5, 220, 154, 36, 5, 220, 151, 36, 5, 220, 152, 36, 5, 220, 148, 36, 5, - 192, 175, 36, 5, 192, 162, 36, 5, 192, 170, 36, 5, 192, 174, 36, 5, 192, - 172, 36, 5, 192, 173, 36, 5, 192, 169, 36, 5, 230, 130, 36, 5, 230, 120, - 36, 5, 230, 123, 36, 5, 230, 129, 36, 5, 230, 125, 36, 5, 230, 126, 36, - 5, 230, 122, 33, 38, 1, 249, 19, 33, 38, 1, 195, 153, 33, 38, 1, 231, 79, - 33, 38, 1, 237, 148, 33, 38, 1, 191, 62, 33, 38, 1, 191, 87, 33, 38, 1, - 155, 33, 38, 1, 234, 142, 33, 38, 1, 234, 116, 33, 38, 1, 234, 105, 33, - 38, 1, 74, 33, 38, 1, 210, 65, 33, 38, 1, 234, 36, 33, 38, 1, 234, 24, - 33, 38, 1, 199, 145, 33, 38, 1, 140, 33, 38, 1, 197, 161, 33, 38, 1, 243, - 129, 33, 38, 1, 203, 166, 33, 38, 1, 203, 117, 33, 38, 1, 232, 230, 33, - 38, 1, 234, 20, 33, 38, 1, 65, 33, 38, 1, 223, 228, 33, 38, 1, 236, 193, - 33, 38, 1, 215, 221, 196, 128, 33, 38, 1, 192, 95, 33, 38, 1, 191, 225, - 33, 38, 1, 223, 83, 65, 33, 38, 1, 219, 85, 191, 190, 33, 38, 1, 248, - 177, 191, 190, 33, 38, 1, 223, 83, 248, 177, 191, 190, 50, 251, 118, 200, - 226, 217, 53, 50, 251, 118, 235, 140, 200, 226, 217, 53, 45, 200, 226, - 248, 55, 50, 200, 226, 248, 55, 45, 235, 140, 200, 226, 248, 55, 50, 235, - 140, 200, 226, 248, 55, 207, 149, 223, 106, 217, 53, 207, 149, 235, 140, - 223, 106, 217, 53, 235, 140, 198, 8, 217, 53, 45, 198, 8, 248, 55, 50, - 198, 8, 248, 55, 207, 149, 201, 191, 45, 207, 149, 211, 108, 248, 55, 50, - 207, 149, 211, 108, 248, 55, 234, 191, 239, 0, 210, 128, 232, 150, 210, - 128, 207, 19, 232, 150, 210, 128, 228, 87, 235, 140, 211, 179, 235, 121, - 251, 128, 196, 66, 251, 128, 235, 140, 206, 204, 251, 117, 55, 211, 174, - 228, 90, 223, 95, 223, 104, 210, 185, 248, 49, 228, 91, 4, 237, 44, 198, - 153, 4, 206, 190, 58, 45, 130, 210, 118, 248, 55, 50, 130, 210, 118, 248, - 55, 198, 153, 4, 75, 58, 198, 153, 4, 75, 60, 45, 81, 249, 90, 4, 208, 9, - 50, 81, 249, 90, 4, 208, 9, 198, 54, 45, 132, 248, 55, 198, 54, 50, 132, - 248, 55, 248, 79, 45, 132, 248, 55, 248, 79, 50, 132, 248, 55, 45, 202, - 207, 126, 248, 55, 50, 202, 207, 126, 248, 55, 45, 55, 210, 115, 50, 55, - 210, 115, 105, 185, 139, 91, 75, 208, 153, 91, 75, 139, 105, 185, 208, - 153, 112, 232, 130, 75, 208, 153, 232, 228, 75, 77, 207, 19, 208, 15, 77, - 81, 198, 152, 206, 190, 209, 63, 193, 23, 204, 12, 82, 236, 142, 154, - 242, 76, 207, 149, 236, 142, 207, 149, 242, 76, 154, 204, 26, 238, 17, 4, - 45, 230, 175, 238, 17, 4, 50, 230, 175, 154, 238, 16, 198, 54, 132, 205, - 55, 56, 197, 14, 237, 217, 198, 223, 237, 217, 201, 80, 232, 82, 201, 64, - 81, 202, 137, 236, 140, 193, 72, 81, 219, 114, 247, 111, 55, 228, 90, - 207, 19, 242, 76, 55, 218, 239, 207, 254, 77, 237, 218, 4, 45, 196, 69, - 55, 200, 164, 77, 223, 95, 130, 222, 37, 223, 95, 130, 222, 38, 4, 222, - 38, 58, 130, 222, 37, 130, 222, 38, 4, 236, 142, 55, 201, 115, 242, 76, - 235, 140, 202, 24, 197, 225, 238, 16, 216, 216, 242, 76, 210, 127, 77, - 208, 152, 234, 131, 77, 239, 1, 195, 137, 236, 207, 238, 220, 210, 84, 4, - 50, 238, 218, 238, 220, 210, 84, 4, 45, 238, 218, 198, 128, 3, 6, 234, 1, - 216, 216, 233, 218, 77, 216, 216, 208, 15, 77, 45, 51, 248, 56, 4, 106, - 50, 51, 248, 56, 4, 106, 45, 51, 248, 56, 4, 55, 106, 50, 51, 248, 56, 4, - 55, 106, 198, 54, 132, 45, 210, 115, 198, 54, 132, 50, 210, 115, 248, 79, - 132, 45, 210, 115, 248, 79, 132, 50, 210, 115, 211, 174, 228, 90, 12, 48, - 207, 49, 12, 48, 242, 232, 12, 48, 205, 58, 107, 12, 48, 205, 58, 109, - 12, 48, 205, 58, 138, 12, 48, 209, 252, 12, 48, 248, 64, 12, 48, 199, - 233, 12, 48, 221, 81, 107, 12, 48, 221, 81, 109, 12, 48, 236, 204, 12, - 48, 205, 62, 12, 48, 2, 107, 12, 48, 2, 109, 12, 48, 220, 32, 107, 12, - 48, 220, 32, 109, 12, 48, 220, 32, 138, 12, 48, 220, 32, 134, 12, 48, - 202, 59, 12, 48, 198, 227, 12, 48, 202, 56, 107, 12, 48, 202, 56, 109, - 12, 48, 229, 226, 107, 12, 48, 229, 226, 109, 12, 48, 230, 55, 12, 48, - 207, 138, 12, 48, 243, 66, 12, 48, 200, 198, 12, 48, 215, 207, 12, 48, - 237, 145, 12, 48, 215, 195, 12, 48, 242, 251, 12, 48, 193, 144, 107, 12, - 48, 193, 144, 109, 12, 48, 232, 245, 12, 48, 210, 78, 107, 12, 48, 210, - 78, 109, 12, 48, 202, 179, 132, 197, 255, 197, 177, 12, 48, 238, 241, 12, - 48, 236, 164, 12, 48, 223, 29, 12, 48, 248, 104, 79, 242, 215, 12, 48, - 233, 195, 12, 48, 201, 216, 107, 12, 48, 201, 216, 109, 12, 48, 249, 105, - 12, 48, 202, 186, 12, 48, 247, 192, 202, 186, 12, 48, 214, 88, 107, 12, - 48, 214, 88, 109, 12, 48, 214, 88, 138, 12, 48, 214, 88, 134, 12, 48, - 216, 174, 12, 48, 203, 108, 12, 48, 207, 144, 12, 48, 233, 225, 12, 48, - 211, 121, 12, 48, 248, 20, 107, 12, 48, 248, 20, 109, 12, 48, 216, 226, - 12, 48, 215, 201, 12, 48, 230, 216, 107, 12, 48, 230, 216, 109, 12, 48, - 230, 216, 138, 12, 48, 198, 173, 12, 48, 242, 214, 12, 48, 193, 105, 107, - 12, 48, 193, 105, 109, 12, 48, 247, 192, 205, 51, 12, 48, 202, 179, 228, - 189, 12, 48, 228, 189, 12, 48, 247, 192, 201, 230, 12, 48, 247, 192, 203, - 103, 12, 48, 232, 161, 12, 48, 247, 192, 247, 87, 12, 48, 202, 179, 193, - 169, 12, 48, 193, 170, 107, 12, 48, 193, 170, 109, 12, 48, 242, 254, 12, - 48, 247, 192, 230, 252, 12, 48, 180, 107, 12, 48, 180, 109, 12, 48, 247, - 192, 219, 21, 12, 48, 247, 192, 231, 193, 12, 48, 215, 190, 107, 12, 48, - 215, 190, 109, 12, 48, 207, 151, 12, 48, 248, 116, 12, 48, 247, 192, 199, - 185, 219, 232, 12, 48, 247, 192, 219, 235, 12, 48, 247, 192, 193, 66, 12, - 48, 247, 192, 232, 181, 12, 48, 234, 203, 107, 12, 48, 234, 203, 109, 12, - 48, 234, 203, 138, 12, 48, 247, 192, 234, 202, 12, 48, 229, 237, 12, 48, - 247, 192, 228, 185, 12, 48, 248, 100, 12, 48, 231, 63, 12, 48, 247, 192, - 232, 238, 12, 48, 247, 192, 248, 162, 12, 48, 247, 192, 205, 159, 12, 48, - 202, 179, 193, 95, 12, 48, 202, 179, 192, 72, 12, 48, 247, 192, 232, 101, - 12, 48, 223, 36, 233, 230, 12, 48, 247, 192, 233, 230, 12, 48, 223, 36, - 198, 56, 12, 48, 247, 192, 198, 56, 12, 48, 223, 36, 235, 113, 12, 48, - 247, 192, 235, 113, 12, 48, 197, 55, 12, 48, 223, 36, 197, 55, 12, 48, - 247, 192, 197, 55, 83, 48, 107, 83, 48, 219, 114, 83, 48, 236, 142, 83, - 48, 202, 98, 83, 48, 205, 57, 83, 48, 102, 83, 48, 109, 83, 48, 219, 143, - 83, 48, 217, 92, 83, 48, 219, 211, 83, 48, 234, 79, 83, 48, 171, 83, 48, - 144, 248, 64, 83, 48, 238, 244, 83, 48, 228, 28, 83, 48, 199, 233, 83, - 48, 211, 79, 248, 64, 83, 48, 221, 80, 83, 48, 209, 0, 83, 48, 193, 12, - 83, 48, 201, 204, 83, 48, 50, 211, 79, 248, 64, 83, 48, 229, 148, 234, - 100, 83, 48, 199, 95, 83, 48, 236, 204, 83, 48, 205, 62, 83, 48, 242, - 232, 83, 48, 208, 206, 83, 48, 251, 254, 83, 48, 215, 181, 83, 48, 234, - 100, 83, 48, 234, 209, 83, 48, 205, 92, 83, 48, 231, 5, 83, 48, 231, 6, - 202, 75, 83, 48, 233, 229, 83, 48, 248, 176, 83, 48, 193, 35, 83, 48, - 243, 88, 83, 48, 206, 134, 83, 48, 223, 160, 83, 48, 202, 71, 83, 48, - 220, 31, 83, 48, 238, 254, 83, 48, 201, 195, 83, 48, 215, 186, 83, 48, - 206, 177, 83, 48, 193, 20, 83, 48, 211, 97, 83, 48, 197, 63, 83, 48, 235, - 93, 83, 48, 203, 41, 198, 227, 83, 48, 235, 140, 242, 232, 83, 48, 180, - 201, 35, 83, 48, 105, 230, 30, 83, 48, 203, 47, 83, 48, 248, 71, 83, 48, - 202, 55, 83, 48, 248, 27, 83, 48, 201, 79, 83, 48, 229, 225, 83, 48, 230, - 73, 83, 48, 236, 146, 83, 48, 230, 55, 83, 48, 248, 49, 83, 48, 207, 138, - 83, 48, 205, 75, 83, 48, 236, 238, 83, 48, 250, 134, 83, 48, 201, 191, - 83, 48, 213, 13, 83, 48, 200, 198, 83, 48, 205, 104, 83, 48, 215, 207, - 83, 48, 197, 254, 83, 48, 220, 113, 83, 48, 201, 64, 83, 48, 237, 145, - 83, 48, 193, 120, 83, 48, 236, 177, 213, 13, 83, 48, 242, 72, 83, 48, - 232, 74, 83, 48, 242, 245, 83, 48, 201, 85, 83, 48, 193, 143, 83, 48, - 232, 245, 83, 48, 242, 241, 83, 48, 233, 68, 83, 48, 55, 192, 235, 83, - 48, 132, 197, 255, 197, 177, 83, 48, 202, 89, 83, 48, 233, 80, 83, 48, - 238, 241, 83, 48, 236, 164, 83, 48, 208, 201, 83, 48, 223, 29, 83, 48, - 216, 198, 83, 48, 198, 151, 83, 48, 200, 141, 83, 48, 219, 137, 83, 48, - 196, 43, 83, 48, 233, 23, 83, 48, 248, 104, 79, 242, 215, 83, 48, 202, - 213, 83, 48, 235, 140, 199, 87, 83, 48, 193, 89, 83, 48, 202, 108, 83, - 48, 236, 224, 83, 48, 233, 195, 83, 48, 201, 233, 83, 48, 57, 83, 48, - 201, 66, 83, 48, 201, 215, 83, 48, 198, 25, 83, 48, 230, 225, 83, 48, - 247, 72, 83, 48, 201, 108, 83, 48, 249, 105, 83, 48, 206, 246, 83, 48, - 202, 186, 83, 48, 223, 20, 83, 48, 214, 87, 83, 48, 203, 108, 83, 48, - 233, 56, 83, 48, 211, 121, 83, 48, 251, 127, 83, 48, 209, 91, 83, 48, - 234, 213, 83, 48, 248, 19, 83, 48, 216, 226, 83, 48, 216, 39, 83, 48, - 204, 33, 83, 48, 250, 241, 83, 48, 215, 201, 83, 48, 198, 61, 83, 48, - 211, 66, 83, 48, 248, 108, 83, 48, 201, 60, 83, 48, 242, 84, 83, 48, 230, - 215, 83, 48, 198, 173, 83, 48, 223, 123, 83, 48, 248, 122, 83, 48, 193, - 170, 234, 100, 83, 48, 242, 214, 83, 48, 193, 104, 83, 48, 205, 51, 83, - 48, 228, 189, 83, 48, 201, 230, 83, 48, 195, 180, 83, 48, 249, 14, 83, - 48, 209, 149, 83, 48, 249, 135, 83, 48, 203, 103, 83, 48, 207, 88, 83, - 48, 206, 46, 83, 48, 232, 161, 83, 48, 248, 106, 83, 48, 247, 87, 83, 48, - 248, 146, 83, 48, 215, 203, 83, 48, 193, 169, 83, 48, 242, 254, 83, 48, - 193, 62, 83, 48, 236, 216, 83, 48, 195, 50, 83, 48, 230, 252, 83, 48, - 219, 21, 83, 48, 231, 193, 83, 48, 215, 189, 83, 48, 202, 97, 83, 48, - 203, 41, 199, 214, 248, 162, 83, 48, 207, 151, 83, 48, 248, 116, 83, 48, - 193, 2, 83, 48, 233, 105, 83, 48, 219, 232, 83, 48, 199, 185, 219, 232, - 83, 48, 219, 228, 83, 48, 202, 5, 83, 48, 219, 235, 83, 48, 193, 66, 83, - 48, 232, 181, 83, 48, 234, 202, 83, 48, 229, 237, 83, 48, 232, 118, 83, - 48, 228, 185, 83, 48, 248, 100, 83, 48, 199, 199, 83, 48, 230, 80, 83, - 48, 233, 16, 83, 48, 205, 195, 193, 62, 83, 48, 247, 74, 83, 48, 231, 63, - 83, 48, 232, 238, 83, 48, 248, 162, 83, 48, 205, 159, 83, 48, 237, 130, - 83, 48, 193, 95, 83, 48, 229, 201, 83, 48, 192, 72, 83, 48, 216, 51, 83, - 48, 248, 141, 83, 48, 234, 112, 83, 48, 232, 101, 83, 48, 197, 222, 83, - 48, 235, 96, 83, 48, 207, 132, 83, 48, 213, 15, 83, 48, 233, 230, 83, 48, - 198, 56, 83, 48, 235, 113, 83, 48, 197, 55, 83, 48, 232, 184, 148, 237, - 87, 246, 242, 45, 119, 179, 148, 237, 87, 246, 242, 93, 119, 60, 148, - 237, 87, 246, 242, 45, 119, 82, 23, 179, 148, 237, 87, 246, 242, 93, 119, - 82, 23, 60, 148, 237, 87, 246, 242, 232, 82, 200, 168, 148, 237, 87, 246, - 242, 200, 169, 232, 100, 58, 148, 237, 87, 246, 242, 200, 169, 232, 100, - 60, 148, 237, 87, 246, 242, 200, 169, 232, 100, 219, 226, 148, 237, 87, - 246, 242, 200, 169, 232, 100, 116, 219, 226, 148, 237, 87, 246, 242, 200, - 169, 232, 100, 116, 179, 148, 237, 87, 246, 242, 200, 169, 232, 100, 110, - 219, 226, 148, 237, 87, 246, 242, 211, 5, 148, 237, 87, 246, 242, 200, - 169, 232, 100, 179, 148, 237, 87, 246, 242, 200, 169, 232, 100, 110, 179, - 148, 237, 87, 246, 242, 228, 243, 207, 84, 148, 237, 87, 246, 242, 206, - 121, 148, 201, 248, 148, 242, 76, 148, 232, 82, 201, 64, 236, 213, 77, - 223, 21, 223, 143, 201, 107, 113, 148, 223, 53, 77, 148, 242, 217, 77, - 148, 31, 191, 77, 45, 251, 118, 248, 55, 50, 251, 118, 248, 55, 45, 55, - 251, 118, 248, 55, 50, 55, 251, 118, 248, 55, 45, 239, 4, 248, 55, 50, - 239, 4, 248, 55, 45, 63, 239, 4, 248, 55, 50, 63, 239, 4, 248, 55, 45, - 62, 219, 189, 248, 55, 50, 62, 219, 189, 248, 55, 209, 20, 77, 231, 132, - 77, 45, 198, 42, 203, 104, 248, 55, 50, 198, 42, 203, 104, 248, 55, 45, - 63, 219, 189, 248, 55, 50, 63, 219, 189, 248, 55, 45, 63, 198, 42, 203, - 104, 248, 55, 50, 63, 198, 42, 203, 104, 248, 55, 45, 63, 51, 248, 55, - 50, 63, 51, 248, 55, 193, 139, 237, 217, 207, 19, 55, 208, 218, 207, 254, - 77, 55, 208, 218, 207, 254, 77, 130, 55, 208, 218, 207, 254, 77, 209, 20, - 87, 233, 105, 230, 27, 212, 136, 107, 230, 27, 212, 136, 109, 230, 27, - 212, 136, 138, 230, 27, 212, 136, 134, 230, 27, 212, 136, 150, 230, 27, - 212, 136, 169, 230, 27, 212, 136, 175, 230, 27, 212, 136, 171, 230, 27, - 212, 136, 178, 148, 219, 170, 163, 77, 148, 206, 181, 163, 77, 148, 237, - 97, 163, 77, 148, 234, 78, 163, 77, 30, 202, 171, 75, 163, 77, 30, 55, - 75, 163, 77, 193, 135, 237, 217, 81, 222, 88, 207, 50, 77, 81, 222, 88, - 207, 50, 4, 195, 20, 202, 6, 77, 81, 222, 88, 207, 50, 87, 116, 230, 72, - 81, 222, 88, 207, 50, 4, 195, 20, 202, 6, 87, 116, 230, 72, 81, 222, 88, - 207, 50, 87, 110, 230, 72, 47, 209, 20, 77, 148, 199, 109, 219, 115, 233, - 53, 204, 12, 113, 230, 27, 212, 136, 199, 95, 230, 27, 212, 136, 197, 32, - 230, 27, 212, 136, 198, 249, 81, 148, 223, 53, 77, 217, 33, 77, 210, 109, - 251, 155, 77, 148, 67, 223, 146, 148, 132, 233, 8, 201, 248, 229, 122, 1, - 2, 65, 229, 122, 1, 65, 229, 122, 1, 2, 68, 229, 122, 1, 68, 229, 122, 1, - 2, 66, 229, 122, 1, 66, 229, 122, 1, 2, 71, 229, 122, 1, 71, 229, 122, 1, - 2, 74, 229, 122, 1, 74, 229, 122, 1, 155, 229, 122, 1, 231, 242, 229, - 122, 1, 221, 168, 229, 122, 1, 231, 55, 229, 122, 1, 220, 234, 229, 122, - 1, 230, 181, 229, 122, 1, 222, 24, 229, 122, 1, 231, 167, 229, 122, 1, - 221, 69, 229, 122, 1, 231, 5, 229, 122, 1, 188, 229, 122, 1, 191, 123, - 229, 122, 1, 202, 223, 229, 122, 1, 191, 30, 229, 122, 1, 201, 5, 229, - 122, 1, 190, 251, 229, 122, 1, 205, 69, 229, 122, 1, 191, 87, 229, 122, - 1, 202, 47, 229, 122, 1, 191, 7, 229, 122, 1, 190, 190, 229, 122, 1, 238, - 34, 229, 122, 1, 198, 193, 229, 122, 1, 237, 46, 229, 122, 1, 2, 197, 94, - 229, 122, 1, 197, 94, 229, 122, 1, 235, 91, 229, 122, 1, 199, 145, 229, - 122, 1, 237, 148, 229, 122, 1, 159, 229, 122, 1, 236, 176, 229, 122, 1, - 181, 229, 122, 1, 213, 221, 229, 122, 1, 212, 180, 229, 122, 1, 214, 123, - 229, 122, 1, 213, 45, 229, 122, 1, 140, 229, 122, 1, 249, 155, 229, 122, - 1, 168, 229, 122, 1, 229, 160, 229, 122, 1, 248, 190, 229, 122, 1, 209, - 187, 229, 122, 1, 228, 161, 229, 122, 1, 248, 12, 229, 122, 1, 208, 167, - 229, 122, 1, 229, 247, 229, 122, 1, 249, 19, 229, 122, 1, 210, 65, 229, - 122, 1, 229, 25, 229, 122, 1, 248, 113, 229, 122, 1, 209, 75, 229, 122, - 1, 174, 229, 122, 1, 216, 102, 229, 122, 1, 215, 157, 229, 122, 1, 216, - 234, 229, 122, 1, 216, 14, 229, 122, 1, 2, 170, 229, 122, 1, 170, 229, - 122, 1, 2, 191, 225, 229, 122, 1, 191, 225, 229, 122, 1, 2, 192, 12, 229, - 122, 1, 192, 12, 229, 122, 1, 165, 229, 122, 1, 207, 2, 229, 122, 1, 206, - 69, 229, 122, 1, 207, 115, 229, 122, 1, 206, 163, 229, 122, 1, 2, 193, - 190, 229, 122, 1, 193, 190, 229, 122, 1, 193, 86, 229, 122, 1, 193, 125, - 229, 122, 1, 193, 48, 229, 122, 1, 215, 63, 229, 122, 1, 193, 249, 229, - 122, 1, 2, 155, 229, 122, 1, 2, 222, 24, 33, 222, 50, 195, 20, 202, 6, - 77, 33, 222, 50, 204, 31, 202, 6, 77, 222, 50, 195, 20, 202, 6, 77, 222, - 50, 204, 31, 202, 6, 77, 229, 122, 223, 53, 77, 229, 122, 195, 20, 223, - 53, 77, 229, 122, 237, 5, 191, 242, 222, 50, 55, 228, 90, 72, 1, 2, 65, - 72, 1, 65, 72, 1, 2, 68, 72, 1, 68, 72, 1, 2, 66, 72, 1, 66, 72, 1, 2, - 71, 72, 1, 71, 72, 1, 2, 74, 72, 1, 74, 72, 1, 155, 72, 1, 231, 242, 72, - 1, 221, 168, 72, 1, 231, 55, 72, 1, 220, 234, 72, 1, 230, 181, 72, 1, - 222, 24, 72, 1, 231, 167, 72, 1, 221, 69, 72, 1, 231, 5, 72, 1, 188, 72, - 1, 191, 123, 72, 1, 202, 223, 72, 1, 191, 30, 72, 1, 201, 5, 72, 1, 190, - 251, 72, 1, 205, 69, 72, 1, 191, 87, 72, 1, 202, 47, 72, 1, 191, 7, 72, - 1, 190, 190, 72, 1, 238, 34, 72, 1, 198, 193, 72, 1, 237, 46, 72, 1, 2, - 197, 94, 72, 1, 197, 94, 72, 1, 235, 91, 72, 1, 199, 145, 72, 1, 237, - 148, 72, 1, 159, 72, 1, 236, 176, 72, 1, 181, 72, 1, 213, 221, 72, 1, - 212, 180, 72, 1, 214, 123, 72, 1, 213, 45, 72, 1, 140, 72, 1, 249, 155, - 72, 1, 168, 72, 1, 229, 160, 72, 1, 248, 190, 72, 1, 209, 187, 72, 1, - 228, 161, 72, 1, 248, 12, 72, 1, 208, 167, 72, 1, 229, 247, 72, 1, 249, - 19, 72, 1, 210, 65, 72, 1, 229, 25, 72, 1, 248, 113, 72, 1, 209, 75, 72, - 1, 174, 72, 1, 216, 102, 72, 1, 215, 157, 72, 1, 216, 234, 72, 1, 216, - 14, 72, 1, 2, 170, 72, 1, 170, 72, 1, 2, 191, 225, 72, 1, 191, 225, 72, - 1, 2, 192, 12, 72, 1, 192, 12, 72, 1, 165, 72, 1, 207, 2, 72, 1, 206, 69, - 72, 1, 207, 115, 72, 1, 206, 163, 72, 1, 2, 193, 190, 72, 1, 193, 190, - 72, 1, 193, 86, 72, 1, 193, 125, 72, 1, 193, 48, 72, 1, 215, 63, 72, 1, - 193, 249, 72, 1, 2, 155, 72, 1, 2, 222, 24, 72, 1, 195, 188, 72, 1, 195, - 69, 72, 1, 195, 153, 72, 1, 195, 24, 72, 82, 236, 142, 222, 50, 208, 193, - 202, 6, 77, 72, 223, 53, 77, 72, 195, 20, 223, 53, 77, 72, 237, 5, 221, - 29, 248, 90, 1, 250, 122, 248, 90, 1, 210, 238, 248, 90, 1, 218, 170, - 248, 90, 1, 233, 177, 248, 90, 1, 238, 129, 248, 90, 1, 200, 43, 248, 90, - 1, 215, 63, 248, 90, 1, 172, 248, 90, 1, 232, 53, 248, 90, 1, 222, 154, - 248, 90, 1, 230, 118, 248, 90, 1, 223, 37, 248, 90, 1, 208, 106, 248, 90, - 1, 192, 235, 248, 90, 1, 191, 72, 248, 90, 1, 247, 5, 248, 90, 1, 203, - 168, 248, 90, 1, 146, 248, 90, 1, 191, 166, 248, 90, 1, 247, 195, 248, - 90, 1, 206, 9, 248, 90, 1, 65, 248, 90, 1, 74, 248, 90, 1, 71, 248, 90, - 1, 234, 175, 248, 90, 1, 251, 238, 248, 90, 1, 234, 168, 248, 90, 1, 250, - 165, 248, 90, 1, 211, 21, 248, 90, 1, 251, 134, 248, 90, 1, 234, 105, - 248, 90, 1, 251, 124, 248, 90, 1, 234, 90, 248, 90, 1, 234, 36, 248, 90, - 1, 68, 248, 90, 1, 66, 248, 90, 1, 223, 51, 248, 90, 1, 196, 12, 248, 90, - 1, 214, 72, 248, 90, 1, 231, 9, 248, 90, 1, 223, 202, 248, 90, 1, 187, 4, - 75, 58, 248, 90, 1, 213, 82, 30, 1, 221, 115, 30, 1, 201, 168, 30, 1, - 221, 108, 30, 1, 213, 206, 30, 1, 213, 204, 30, 1, 213, 203, 30, 1, 198, - 168, 30, 1, 201, 157, 30, 1, 206, 240, 30, 1, 206, 235, 30, 1, 206, 232, - 30, 1, 206, 225, 30, 1, 206, 220, 30, 1, 206, 215, 30, 1, 206, 226, 30, - 1, 206, 238, 30, 1, 216, 79, 30, 1, 209, 171, 30, 1, 201, 165, 30, 1, - 209, 160, 30, 1, 202, 161, 30, 1, 201, 162, 30, 1, 223, 224, 30, 1, 243, - 26, 30, 1, 201, 172, 30, 1, 243, 93, 30, 1, 221, 190, 30, 1, 199, 7, 30, - 1, 209, 211, 30, 1, 229, 144, 30, 1, 65, 30, 1, 252, 27, 30, 1, 170, 30, - 1, 192, 129, 30, 1, 234, 67, 30, 1, 71, 30, 1, 192, 67, 30, 1, 192, 80, - 30, 1, 74, 30, 1, 193, 190, 30, 1, 193, 176, 30, 1, 211, 153, 30, 1, 192, - 12, 30, 1, 66, 30, 1, 193, 107, 30, 1, 193, 125, 30, 1, 193, 86, 30, 1, - 191, 225, 30, 1, 233, 244, 30, 1, 192, 33, 30, 1, 68, 30, 233, 5, 30, 1, - 201, 166, 30, 1, 213, 196, 30, 1, 213, 198, 30, 1, 213, 201, 30, 1, 206, - 233, 30, 1, 206, 214, 30, 1, 206, 222, 30, 1, 206, 227, 30, 1, 206, 212, - 30, 1, 216, 72, 30, 1, 216, 69, 30, 1, 216, 73, 30, 1, 222, 73, 30, 1, - 209, 166, 30, 1, 209, 152, 30, 1, 209, 158, 30, 1, 209, 155, 30, 1, 209, - 169, 30, 1, 209, 153, 30, 1, 222, 71, 30, 1, 222, 69, 30, 1, 202, 154, - 30, 1, 202, 152, 30, 1, 202, 144, 30, 1, 202, 149, 30, 1, 202, 159, 30, - 1, 210, 151, 30, 1, 201, 169, 30, 1, 192, 57, 30, 1, 192, 51, 30, 1, 192, - 52, 30, 1, 222, 72, 30, 1, 201, 170, 30, 1, 192, 63, 30, 1, 192, 0, 30, - 1, 191, 255, 30, 1, 192, 2, 30, 1, 191, 212, 30, 1, 191, 213, 30, 1, 191, - 216, 30, 1, 251, 27, 30, 1, 251, 21, 148, 251, 98, 219, 103, 77, 148, - 251, 98, 207, 20, 77, 148, 251, 98, 91, 77, 148, 251, 98, 105, 77, 148, - 251, 98, 115, 77, 148, 251, 98, 232, 130, 77, 148, 251, 98, 198, 54, 77, - 148, 251, 98, 82, 77, 148, 251, 98, 248, 79, 77, 148, 251, 98, 232, 240, - 77, 148, 251, 98, 205, 58, 77, 148, 251, 98, 199, 1, 77, 148, 251, 98, - 232, 123, 77, 148, 251, 98, 229, 222, 77, 148, 251, 98, 234, 210, 77, - 148, 251, 98, 217, 93, 77, 248, 90, 1, 248, 12, 248, 90, 1, 191, 30, 248, - 90, 1, 222, 246, 248, 90, 1, 230, 181, 248, 90, 1, 234, 190, 248, 90, 1, - 234, 87, 248, 90, 1, 211, 89, 248, 90, 1, 211, 93, 248, 90, 1, 223, 79, - 248, 90, 1, 251, 100, 248, 90, 1, 223, 130, 248, 90, 1, 196, 83, 248, 90, - 1, 223, 182, 248, 90, 1, 214, 50, 248, 90, 1, 251, 231, 248, 90, 1, 250, - 160, 248, 90, 1, 251, 151, 248, 90, 1, 211, 115, 248, 90, 1, 211, 96, - 248, 90, 1, 223, 127, 248, 90, 53, 1, 210, 238, 248, 90, 53, 1, 200, 43, - 248, 90, 53, 1, 222, 154, 248, 90, 53, 1, 230, 118, 248, 90, 1, 231, 94, - 248, 90, 1, 219, 162, 248, 90, 1, 190, 231, 248, 90, 53, 1, 232, 53, 248, - 90, 1, 230, 138, 248, 90, 1, 220, 182, 248, 90, 1, 211, 153, 248, 90, 1, - 251, 247, 12, 201, 29, 200, 43, 12, 201, 29, 193, 98, 12, 201, 29, 192, - 209, 12, 201, 29, 247, 208, 12, 201, 29, 200, 151, 12, 201, 29, 228, 80, - 12, 201, 29, 228, 84, 12, 201, 29, 228, 171, 12, 201, 29, 228, 81, 12, - 201, 29, 200, 46, 12, 201, 29, 228, 83, 12, 201, 29, 228, 79, 12, 201, - 29, 228, 169, 12, 201, 29, 228, 82, 12, 201, 29, 228, 78, 12, 201, 29, - 215, 63, 12, 201, 29, 230, 118, 12, 201, 29, 206, 9, 12, 201, 29, 210, - 238, 12, 201, 29, 201, 251, 12, 201, 29, 238, 129, 12, 201, 29, 228, 85, - 12, 201, 29, 229, 180, 12, 201, 29, 200, 55, 12, 201, 29, 200, 128, 12, - 201, 29, 201, 119, 12, 201, 29, 203, 174, 12, 201, 29, 210, 69, 12, 201, - 29, 208, 108, 12, 201, 29, 198, 99, 12, 201, 29, 200, 45, 12, 201, 29, - 200, 140, 12, 201, 29, 228, 96, 12, 201, 29, 228, 77, 12, 201, 29, 209, - 232, 12, 201, 29, 208, 106, 148, 237, 87, 246, 242, 200, 225, 72, 1, 2, - 220, 234, 72, 1, 2, 202, 223, 72, 1, 2, 201, 5, 72, 1, 2, 159, 72, 1, 2, - 212, 180, 72, 1, 2, 140, 72, 1, 2, 229, 160, 72, 1, 2, 228, 161, 72, 1, - 2, 229, 247, 72, 1, 2, 229, 25, 72, 1, 2, 215, 157, 72, 1, 2, 165, 72, 1, - 2, 207, 2, 72, 1, 2, 206, 69, 72, 1, 2, 207, 115, 72, 1, 2, 206, 163, - 127, 30, 221, 115, 127, 30, 213, 206, 127, 30, 198, 168, 127, 30, 206, - 240, 127, 30, 216, 79, 127, 30, 209, 171, 127, 30, 202, 161, 127, 30, - 223, 224, 127, 30, 243, 26, 127, 30, 243, 93, 127, 30, 221, 190, 127, 30, - 199, 7, 127, 30, 209, 211, 127, 30, 229, 144, 127, 30, 221, 116, 65, 127, - 30, 213, 207, 65, 127, 30, 198, 169, 65, 127, 30, 206, 241, 65, 127, 30, - 216, 80, 65, 127, 30, 209, 172, 65, 127, 30, 202, 162, 65, 127, 30, 223, - 225, 65, 127, 30, 243, 27, 65, 127, 30, 243, 94, 65, 127, 30, 221, 191, - 65, 127, 30, 199, 8, 65, 127, 30, 209, 212, 65, 127, 30, 229, 145, 65, - 127, 30, 243, 27, 66, 127, 221, 34, 246, 242, 211, 131, 127, 221, 34, - 246, 242, 187, 228, 161, 127, 228, 16, 107, 127, 228, 16, 109, 127, 228, - 16, 138, 127, 228, 16, 134, 127, 228, 16, 150, 127, 228, 16, 169, 127, - 228, 16, 175, 127, 228, 16, 171, 127, 228, 16, 178, 127, 228, 16, 199, - 95, 127, 228, 16, 215, 207, 127, 228, 16, 232, 245, 127, 228, 16, 193, - 143, 127, 228, 16, 193, 28, 127, 228, 16, 216, 167, 127, 228, 16, 234, - 209, 127, 228, 16, 200, 198, 127, 228, 16, 201, 67, 127, 228, 16, 230, 1, - 127, 228, 16, 202, 36, 127, 228, 16, 214, 229, 127, 228, 16, 201, 232, - 127, 228, 16, 233, 0, 127, 228, 16, 239, 52, 127, 228, 16, 220, 116, 127, - 228, 16, 207, 43, 127, 228, 16, 247, 140, 127, 228, 16, 201, 11, 127, - 228, 16, 200, 178, 127, 228, 16, 234, 77, 127, 228, 16, 207, 33, 127, - 228, 16, 251, 170, 127, 228, 16, 233, 33, 127, 228, 16, 207, 31, 127, - 228, 16, 204, 33, 127, 228, 16, 207, 110, 47, 228, 16, 208, 14, 47, 228, - 16, 221, 142, 47, 228, 16, 205, 90, 47, 228, 16, 221, 29, 47, 31, 199, - 96, 211, 107, 62, 201, 191, 47, 31, 197, 33, 211, 107, 62, 201, 191, 47, - 31, 198, 250, 211, 107, 62, 201, 191, 47, 31, 232, 138, 211, 107, 62, - 201, 191, 47, 31, 233, 18, 211, 107, 62, 201, 191, 47, 31, 202, 122, 211, - 107, 62, 201, 191, 47, 31, 203, 243, 211, 107, 62, 201, 191, 47, 31, 234, - 156, 211, 107, 62, 201, 191, 210, 105, 56, 47, 31, 197, 33, 107, 47, 31, - 197, 33, 109, 47, 31, 197, 33, 138, 47, 31, 197, 33, 134, 47, 31, 197, - 33, 150, 47, 31, 197, 33, 169, 47, 31, 197, 33, 175, 47, 31, 197, 33, - 171, 47, 31, 197, 33, 178, 47, 31, 198, 249, 47, 31, 198, 250, 107, 47, - 31, 198, 250, 109, 47, 31, 198, 250, 138, 47, 31, 198, 250, 134, 47, 31, - 198, 250, 150, 47, 30, 221, 115, 47, 30, 213, 206, 47, 30, 198, 168, 47, - 30, 206, 240, 47, 30, 216, 79, 47, 30, 209, 171, 47, 30, 202, 161, 47, - 30, 223, 224, 47, 30, 243, 26, 47, 30, 243, 93, 47, 30, 221, 190, 47, 30, - 199, 7, 47, 30, 209, 211, 47, 30, 229, 144, 47, 30, 221, 116, 65, 47, 30, - 213, 207, 65, 47, 30, 198, 169, 65, 47, 30, 206, 241, 65, 47, 30, 216, - 80, 65, 47, 30, 209, 172, 65, 47, 30, 202, 162, 65, 47, 30, 223, 225, 65, - 47, 30, 243, 27, 65, 47, 30, 243, 94, 65, 47, 30, 221, 191, 65, 47, 30, - 199, 8, 65, 47, 30, 209, 212, 65, 47, 30, 229, 145, 65, 47, 221, 34, 246, - 242, 246, 249, 47, 221, 34, 246, 242, 222, 180, 47, 30, 223, 225, 66, - 221, 34, 201, 107, 113, 47, 228, 16, 107, 47, 228, 16, 109, 47, 228, 16, - 138, 47, 228, 16, 134, 47, 228, 16, 150, 47, 228, 16, 169, 47, 228, 16, - 175, 47, 228, 16, 171, 47, 228, 16, 178, 47, 228, 16, 199, 95, 47, 228, - 16, 215, 207, 47, 228, 16, 232, 245, 47, 228, 16, 193, 143, 47, 228, 16, - 193, 28, 47, 228, 16, 216, 167, 47, 228, 16, 234, 209, 47, 228, 16, 200, - 198, 47, 228, 16, 201, 67, 47, 228, 16, 230, 1, 47, 228, 16, 202, 36, 47, - 228, 16, 214, 229, 47, 228, 16, 201, 232, 47, 228, 16, 233, 0, 47, 228, - 16, 239, 52, 47, 228, 16, 220, 116, 47, 228, 16, 205, 56, 47, 228, 16, - 217, 98, 47, 228, 16, 233, 43, 47, 228, 16, 200, 210, 47, 228, 16, 233, - 222, 47, 228, 16, 208, 213, 47, 228, 16, 250, 169, 47, 228, 16, 223, 54, - 47, 228, 16, 207, 31, 47, 228, 16, 239, 10, 47, 228, 16, 238, 253, 47, - 228, 16, 229, 137, 47, 228, 16, 247, 23, 47, 228, 16, 218, 243, 47, 228, - 16, 219, 226, 47, 228, 16, 179, 47, 228, 16, 216, 217, 47, 228, 16, 207, - 61, 47, 228, 16, 201, 11, 47, 228, 16, 200, 178, 47, 228, 16, 234, 77, - 47, 228, 16, 207, 33, 47, 228, 16, 251, 170, 47, 228, 16, 213, 192, 47, - 31, 198, 250, 169, 47, 31, 198, 250, 175, 47, 31, 198, 250, 171, 47, 31, - 198, 250, 178, 47, 31, 232, 137, 47, 31, 232, 138, 107, 47, 31, 232, 138, - 109, 47, 31, 232, 138, 138, 47, 31, 232, 138, 134, 47, 31, 232, 138, 150, - 47, 31, 232, 138, 169, 47, 31, 232, 138, 175, 47, 31, 232, 138, 171, 47, - 31, 232, 138, 178, 47, 31, 233, 17, 148, 199, 109, 16, 40, 223, 23, 148, - 199, 109, 16, 40, 233, 55, 148, 199, 109, 16, 40, 217, 60, 148, 199, 109, - 16, 40, 251, 41, 148, 199, 109, 16, 40, 217, 23, 148, 199, 109, 16, 40, - 222, 177, 148, 199, 109, 16, 40, 222, 178, 148, 199, 109, 16, 40, 250, - 161, 148, 199, 109, 16, 40, 204, 10, 148, 199, 109, 16, 40, 211, 159, - 148, 199, 109, 16, 40, 213, 1, 148, 199, 109, 16, 40, 237, 142, 51, 229, - 180, 51, 234, 32, 51, 233, 232, 219, 120, 219, 147, 56, 47, 72, 65, 47, - 72, 68, 47, 72, 66, 47, 72, 71, 47, 72, 74, 47, 72, 155, 47, 72, 221, - 168, 47, 72, 220, 234, 47, 72, 222, 24, 47, 72, 221, 69, 47, 72, 188, 47, - 72, 202, 223, 47, 72, 201, 5, 47, 72, 205, 69, 47, 72, 202, 47, 47, 72, - 190, 190, 47, 72, 198, 193, 47, 72, 197, 94, 47, 72, 199, 145, 47, 72, - 159, 47, 72, 181, 47, 72, 213, 221, 47, 72, 212, 180, 47, 72, 214, 123, - 47, 72, 213, 45, 47, 72, 140, 47, 72, 229, 160, 47, 72, 228, 161, 47, 72, - 229, 247, 47, 72, 229, 25, 47, 72, 174, 47, 72, 216, 102, 47, 72, 215, - 157, 47, 72, 216, 234, 47, 72, 216, 14, 47, 72, 170, 47, 72, 191, 225, - 47, 72, 192, 12, 47, 72, 165, 47, 72, 207, 2, 47, 72, 206, 69, 47, 72, - 207, 115, 47, 72, 206, 163, 47, 72, 193, 190, 47, 72, 193, 86, 47, 72, - 193, 125, 47, 72, 193, 48, 51, 234, 35, 214, 230, 207, 69, 51, 251, 66, - 51, 250, 223, 51, 251, 94, 51, 252, 165, 51, 223, 132, 51, 223, 99, 51, - 196, 80, 51, 234, 4, 51, 234, 187, 51, 211, 92, 51, 211, 85, 51, 222, - 102, 51, 222, 65, 51, 222, 60, 51, 231, 197, 51, 231, 207, 51, 231, 43, - 51, 231, 37, 51, 220, 146, 51, 231, 28, 51, 221, 133, 51, 221, 132, 51, - 221, 131, 51, 221, 130, 51, 230, 148, 51, 230, 147, 51, 220, 195, 51, - 220, 198, 51, 222, 11, 51, 221, 31, 51, 221, 40, 51, 205, 181, 51, 205, - 134, 51, 202, 142, 51, 204, 16, 51, 204, 15, 51, 238, 30, 51, 237, 83, - 51, 236, 143, 51, 198, 81, 51, 214, 223, 51, 213, 2, 51, 230, 77, 51, - 210, 216, 51, 210, 215, 51, 249, 152, 51, 209, 183, 51, 209, 145, 51, - 209, 146, 51, 248, 158, 51, 228, 156, 51, 228, 150, 51, 247, 223, 51, - 228, 134, 51, 229, 208, 51, 209, 243, 51, 210, 31, 51, 229, 189, 51, 210, - 27, 51, 210, 45, 51, 248, 254, 51, 209, 59, 51, 248, 86, 51, 229, 1, 51, - 209, 40, 51, 228, 248, 51, 228, 250, 51, 217, 112, 51, 217, 108, 51, 217, - 117, 51, 217, 46, 51, 217, 78, 51, 216, 58, 51, 216, 31, 51, 216, 30, 51, - 216, 205, 51, 216, 202, 51, 216, 206, 51, 192, 139, 51, 192, 137, 51, - 191, 210, 51, 206, 179, 51, 206, 183, 51, 206, 36, 51, 206, 29, 51, 207, - 58, 51, 207, 55, 51, 193, 141, 148, 199, 109, 16, 40, 228, 179, 191, 77, - 148, 199, 109, 16, 40, 228, 179, 107, 148, 199, 109, 16, 40, 228, 179, - 109, 148, 199, 109, 16, 40, 228, 179, 138, 148, 199, 109, 16, 40, 228, - 179, 134, 148, 199, 109, 16, 40, 228, 179, 150, 148, 199, 109, 16, 40, - 228, 179, 169, 148, 199, 109, 16, 40, 228, 179, 175, 148, 199, 109, 16, - 40, 228, 179, 171, 148, 199, 109, 16, 40, 228, 179, 178, 148, 199, 109, - 16, 40, 228, 179, 199, 95, 148, 199, 109, 16, 40, 228, 179, 234, 129, - 148, 199, 109, 16, 40, 228, 179, 197, 37, 148, 199, 109, 16, 40, 228, - 179, 198, 251, 148, 199, 109, 16, 40, 228, 179, 232, 124, 148, 199, 109, - 16, 40, 228, 179, 233, 21, 148, 199, 109, 16, 40, 228, 179, 202, 131, - 148, 199, 109, 16, 40, 228, 179, 203, 245, 148, 199, 109, 16, 40, 228, - 179, 234, 163, 148, 199, 109, 16, 40, 228, 179, 213, 173, 148, 199, 109, - 16, 40, 228, 179, 197, 32, 148, 199, 109, 16, 40, 228, 179, 197, 25, 148, - 199, 109, 16, 40, 228, 179, 197, 20, 148, 199, 109, 16, 40, 228, 179, - 197, 22, 148, 199, 109, 16, 40, 228, 179, 197, 27, 51, 228, 170, 51, 238, - 34, 51, 250, 165, 51, 164, 51, 211, 11, 51, 210, 70, 51, 236, 179, 51, - 236, 180, 201, 190, 51, 236, 180, 238, 191, 51, 223, 51, 51, 234, 35, - 214, 230, 229, 209, 51, 234, 35, 214, 230, 200, 66, 51, 234, 35, 214, - 230, 199, 212, 51, 234, 35, 214, 230, 216, 201, 51, 238, 255, 51, 210, - 223, 251, 137, 51, 181, 51, 215, 158, 65, 51, 174, 51, 155, 51, 222, 27, - 51, 217, 18, 51, 231, 185, 51, 247, 146, 51, 222, 26, 51, 209, 233, 51, - 214, 74, 51, 215, 158, 233, 177, 51, 215, 158, 232, 53, 51, 216, 143, 51, - 221, 219, 51, 228, 85, 51, 221, 170, 51, 216, 104, 51, 231, 57, 51, 198, - 195, 51, 215, 158, 172, 51, 216, 22, 51, 236, 189, 51, 221, 97, 51, 232, - 179, 51, 213, 83, 51, 215, 158, 218, 170, 51, 216, 19, 51, 242, 201, 51, - 221, 83, 51, 216, 20, 201, 190, 51, 242, 202, 201, 190, 51, 218, 171, - 201, 190, 51, 221, 84, 201, 190, 51, 216, 20, 238, 191, 51, 242, 202, - 238, 191, 51, 218, 171, 238, 191, 51, 221, 84, 238, 191, 51, 218, 171, - 139, 206, 9, 51, 218, 171, 139, 206, 10, 201, 190, 51, 168, 51, 221, 23, - 51, 215, 168, 51, 230, 231, 51, 207, 170, 51, 207, 171, 139, 206, 9, 51, - 207, 171, 139, 206, 10, 201, 190, 51, 208, 180, 51, 212, 221, 51, 215, - 158, 206, 9, 51, 215, 160, 51, 208, 126, 51, 212, 114, 51, 215, 158, 196, - 12, 51, 215, 89, 51, 220, 184, 51, 215, 90, 216, 205, 51, 208, 125, 51, - 212, 113, 51, 215, 158, 193, 224, 51, 215, 83, 51, 220, 182, 51, 215, 84, - 216, 205, 51, 222, 155, 211, 136, 51, 218, 171, 211, 136, 51, 251, 151, - 51, 248, 59, 51, 247, 68, 51, 247, 45, 51, 247, 196, 139, 221, 219, 51, - 242, 101, 51, 237, 202, 51, 230, 131, 51, 140, 51, 228, 171, 51, 223, - 164, 51, 221, 104, 51, 221, 84, 247, 112, 51, 220, 236, 51, 219, 48, 51, - 219, 47, 51, 219, 32, 51, 218, 186, 51, 217, 19, 202, 71, 51, 216, 57, - 51, 215, 235, 51, 209, 231, 51, 209, 78, 51, 208, 251, 51, 208, 249, 51, - 201, 181, 51, 200, 158, 51, 193, 127, 51, 196, 13, 139, 218, 170, 51, 42, - 139, 218, 170, 148, 199, 109, 16, 40, 237, 206, 107, 148, 199, 109, 16, - 40, 237, 206, 109, 148, 199, 109, 16, 40, 237, 206, 138, 148, 199, 109, - 16, 40, 237, 206, 134, 148, 199, 109, 16, 40, 237, 206, 150, 148, 199, - 109, 16, 40, 237, 206, 169, 148, 199, 109, 16, 40, 237, 206, 175, 148, - 199, 109, 16, 40, 237, 206, 171, 148, 199, 109, 16, 40, 237, 206, 178, - 148, 199, 109, 16, 40, 237, 206, 199, 95, 148, 199, 109, 16, 40, 237, - 206, 234, 129, 148, 199, 109, 16, 40, 237, 206, 197, 37, 148, 199, 109, - 16, 40, 237, 206, 198, 251, 148, 199, 109, 16, 40, 237, 206, 232, 124, - 148, 199, 109, 16, 40, 237, 206, 233, 21, 148, 199, 109, 16, 40, 237, - 206, 202, 131, 148, 199, 109, 16, 40, 237, 206, 203, 245, 148, 199, 109, - 16, 40, 237, 206, 234, 163, 148, 199, 109, 16, 40, 237, 206, 213, 173, - 148, 199, 109, 16, 40, 237, 206, 197, 32, 148, 199, 109, 16, 40, 237, - 206, 197, 25, 148, 199, 109, 16, 40, 237, 206, 197, 20, 148, 199, 109, - 16, 40, 237, 206, 197, 22, 148, 199, 109, 16, 40, 237, 206, 197, 27, 148, - 199, 109, 16, 40, 237, 206, 197, 28, 148, 199, 109, 16, 40, 237, 206, - 197, 23, 148, 199, 109, 16, 40, 237, 206, 197, 24, 148, 199, 109, 16, 40, - 237, 206, 197, 31, 148, 199, 109, 16, 40, 237, 206, 197, 26, 148, 199, - 109, 16, 40, 237, 206, 198, 249, 148, 199, 109, 16, 40, 237, 206, 198, - 247, 51, 231, 224, 229, 183, 40, 199, 34, 238, 233, 229, 221, 229, 183, - 40, 199, 34, 207, 102, 234, 209, 229, 183, 40, 237, 16, 250, 185, 199, - 34, 248, 249, 229, 183, 40, 191, 238, 232, 170, 229, 183, 40, 193, 171, - 229, 183, 40, 239, 55, 229, 183, 40, 199, 34, 250, 248, 229, 183, 40, - 229, 8, 198, 87, 229, 183, 40, 2, 199, 194, 229, 183, 40, 198, 1, 229, - 183, 40, 210, 62, 229, 183, 40, 201, 105, 229, 183, 40, 233, 45, 229, - 183, 40, 230, 208, 209, 23, 229, 183, 40, 216, 0, 229, 183, 40, 234, 76, - 229, 183, 40, 232, 171, 229, 183, 40, 193, 21, 211, 107, 199, 34, 237, - 143, 229, 183, 40, 251, 45, 229, 183, 40, 239, 33, 229, 183, 40, 248, - 147, 198, 215, 229, 183, 40, 230, 229, 229, 183, 40, 201, 209, 251, 65, - 229, 183, 40, 207, 23, 229, 183, 40, 223, 126, 229, 183, 40, 230, 208, - 199, 194, 229, 183, 40, 215, 182, 239, 2, 229, 183, 40, 230, 208, 208, - 226, 229, 183, 40, 199, 34, 252, 67, 193, 143, 229, 183, 40, 199, 34, - 242, 229, 232, 245, 229, 183, 40, 223, 140, 229, 183, 40, 235, 66, 229, - 183, 40, 207, 26, 229, 183, 40, 230, 208, 209, 0, 229, 183, 40, 208, 199, - 229, 183, 40, 237, 222, 79, 199, 34, 219, 134, 229, 183, 40, 199, 34, - 233, 83, 229, 183, 40, 211, 64, 229, 183, 40, 211, 169, 229, 183, 40, - 237, 113, 229, 183, 40, 237, 135, 229, 183, 40, 223, 155, 229, 183, 40, - 248, 43, 229, 183, 40, 242, 78, 119, 216, 208, 229, 183, 40, 231, 192, - 198, 87, 229, 183, 40, 208, 137, 196, 67, 229, 183, 40, 211, 63, 229, - 183, 40, 199, 34, 193, 109, 229, 183, 40, 207, 14, 229, 183, 40, 199, 34, - 247, 74, 229, 183, 40, 199, 34, 250, 244, 198, 209, 229, 183, 40, 199, - 34, 222, 12, 201, 71, 215, 186, 229, 183, 40, 237, 78, 229, 183, 40, 199, - 34, 217, 49, 217, 113, 229, 183, 40, 252, 68, 229, 183, 40, 199, 34, 193, - 161, 229, 183, 40, 199, 34, 231, 147, 193, 66, 229, 183, 40, 199, 34, - 222, 186, 220, 45, 229, 183, 40, 236, 221, 229, 183, 40, 219, 121, 229, - 183, 40, 223, 129, 197, 176, 229, 183, 40, 2, 208, 226, 229, 183, 40, - 252, 0, 242, 68, 229, 183, 40, 248, 252, 242, 68, 11, 5, 223, 55, 11, 5, - 223, 47, 11, 5, 68, 11, 5, 223, 82, 11, 5, 223, 226, 11, 5, 223, 209, 11, - 5, 223, 228, 11, 5, 223, 227, 11, 5, 250, 184, 11, 5, 250, 135, 11, 5, - 65, 11, 5, 251, 67, 11, 5, 196, 78, 11, 5, 196, 82, 11, 5, 196, 79, 11, - 5, 211, 32, 11, 5, 210, 249, 11, 5, 74, 11, 5, 211, 80, 11, 5, 233, 223, - 11, 5, 71, 11, 5, 193, 0, 11, 5, 248, 150, 11, 5, 248, 145, 11, 5, 248, - 190, 11, 5, 248, 163, 11, 5, 248, 179, 11, 5, 248, 178, 11, 5, 248, 181, - 11, 5, 248, 180, 11, 5, 249, 66, 11, 5, 249, 58, 11, 5, 249, 155, 11, 5, - 249, 91, 11, 5, 247, 236, 11, 5, 247, 240, 11, 5, 247, 237, 11, 5, 248, - 83, 11, 5, 248, 64, 11, 5, 248, 113, 11, 5, 248, 91, 11, 5, 248, 208, 11, - 5, 249, 19, 11, 5, 248, 221, 11, 5, 247, 219, 11, 5, 247, 213, 11, 5, - 248, 12, 11, 5, 247, 234, 11, 5, 247, 227, 11, 5, 247, 232, 11, 5, 247, - 201, 11, 5, 247, 199, 11, 5, 247, 206, 11, 5, 247, 204, 11, 5, 247, 202, - 11, 5, 247, 203, 11, 5, 209, 119, 11, 5, 209, 115, 11, 5, 209, 187, 11, - 5, 209, 131, 11, 5, 209, 151, 11, 5, 209, 178, 11, 5, 209, 174, 11, 5, - 210, 91, 11, 5, 210, 75, 11, 5, 168, 11, 5, 210, 139, 11, 5, 208, 147, - 11, 5, 208, 149, 11, 5, 208, 148, 11, 5, 209, 16, 11, 5, 208, 254, 11, 5, - 209, 75, 11, 5, 209, 35, 11, 5, 208, 133, 11, 5, 208, 128, 11, 5, 208, - 167, 11, 5, 208, 146, 11, 5, 208, 138, 11, 5, 208, 144, 11, 5, 208, 110, - 11, 5, 208, 109, 11, 5, 208, 114, 11, 5, 208, 113, 11, 5, 208, 111, 11, - 5, 208, 112, 11, 5, 249, 40, 11, 5, 249, 39, 11, 5, 249, 46, 11, 5, 249, - 41, 11, 5, 249, 43, 11, 5, 249, 42, 11, 5, 249, 45, 11, 5, 249, 44, 11, - 5, 249, 52, 11, 5, 249, 51, 11, 5, 249, 55, 11, 5, 249, 53, 11, 5, 249, - 31, 11, 5, 249, 33, 11, 5, 249, 32, 11, 5, 249, 36, 11, 5, 249, 35, 11, - 5, 249, 38, 11, 5, 249, 37, 11, 5, 249, 47, 11, 5, 249, 50, 11, 5, 249, - 48, 11, 5, 249, 27, 11, 5, 249, 26, 11, 5, 249, 34, 11, 5, 249, 30, 11, - 5, 249, 28, 11, 5, 249, 29, 11, 5, 249, 23, 11, 5, 249, 22, 11, 5, 249, - 25, 11, 5, 249, 24, 11, 5, 214, 187, 11, 5, 214, 186, 11, 5, 214, 192, - 11, 5, 214, 188, 11, 5, 214, 189, 11, 5, 214, 191, 11, 5, 214, 190, 11, - 5, 214, 195, 11, 5, 214, 194, 11, 5, 214, 197, 11, 5, 214, 196, 11, 5, - 214, 183, 11, 5, 214, 182, 11, 5, 214, 185, 11, 5, 214, 184, 11, 5, 214, - 176, 11, 5, 214, 175, 11, 5, 214, 180, 11, 5, 214, 179, 11, 5, 214, 177, - 11, 5, 214, 178, 11, 5, 214, 170, 11, 5, 214, 169, 11, 5, 214, 174, 11, - 5, 214, 173, 11, 5, 214, 171, 11, 5, 214, 172, 11, 5, 229, 69, 11, 5, - 229, 68, 11, 5, 229, 74, 11, 5, 229, 70, 11, 5, 229, 71, 11, 5, 229, 73, - 11, 5, 229, 72, 11, 5, 229, 77, 11, 5, 229, 76, 11, 5, 229, 79, 11, 5, - 229, 78, 11, 5, 229, 60, 11, 5, 229, 62, 11, 5, 229, 61, 11, 5, 229, 65, - 11, 5, 229, 64, 11, 5, 229, 67, 11, 5, 229, 66, 11, 5, 229, 56, 11, 5, - 229, 55, 11, 5, 229, 63, 11, 5, 229, 59, 11, 5, 229, 57, 11, 5, 229, 58, - 11, 5, 229, 50, 11, 5, 229, 54, 11, 5, 229, 53, 11, 5, 229, 51, 11, 5, - 229, 52, 11, 5, 216, 25, 11, 5, 216, 24, 11, 5, 216, 102, 11, 5, 216, 33, - 11, 5, 216, 65, 11, 5, 216, 83, 11, 5, 216, 81, 11, 5, 217, 32, 11, 5, - 217, 26, 11, 5, 174, 11, 5, 217, 73, 11, 5, 215, 117, 11, 5, 215, 116, - 11, 5, 215, 120, 11, 5, 215, 118, 11, 5, 215, 197, 11, 5, 215, 170, 11, - 5, 216, 14, 11, 5, 215, 204, 11, 5, 216, 154, 11, 5, 216, 234, 11, 5, - 215, 97, 11, 5, 215, 91, 11, 5, 215, 157, 11, 5, 215, 113, 11, 5, 215, - 106, 11, 5, 215, 111, 11, 5, 215, 67, 11, 5, 215, 66, 11, 5, 215, 72, 11, - 5, 215, 69, 11, 5, 232, 231, 11, 5, 232, 224, 11, 5, 233, 25, 11, 5, 232, - 247, 11, 5, 233, 74, 11, 5, 233, 65, 11, 5, 233, 111, 11, 5, 233, 79, 11, - 5, 232, 121, 11, 5, 232, 177, 11, 5, 232, 156, 11, 5, 232, 70, 11, 5, - 232, 69, 11, 5, 232, 88, 11, 5, 232, 75, 11, 5, 232, 73, 11, 5, 232, 74, - 11, 5, 232, 56, 11, 5, 232, 55, 11, 5, 232, 59, 11, 5, 232, 57, 11, 5, - 195, 32, 11, 5, 195, 26, 11, 5, 195, 69, 11, 5, 195, 41, 11, 5, 195, 58, - 11, 5, 195, 53, 11, 5, 195, 61, 11, 5, 195, 60, 11, 5, 195, 162, 11, 5, - 195, 157, 11, 5, 195, 188, 11, 5, 195, 175, 11, 5, 195, 4, 11, 5, 195, 0, - 11, 5, 195, 24, 11, 5, 195, 6, 11, 5, 195, 73, 11, 5, 195, 141, 11, 5, - 193, 242, 11, 5, 193, 240, 11, 5, 193, 249, 11, 5, 193, 245, 11, 5, 193, - 243, 11, 5, 193, 244, 11, 5, 193, 229, 11, 5, 193, 228, 11, 5, 193, 235, - 11, 5, 193, 234, 11, 5, 193, 232, 11, 5, 193, 233, 11, 5, 236, 214, 11, - 5, 236, 199, 11, 5, 237, 46, 11, 5, 236, 242, 11, 5, 237, 21, 11, 5, 237, - 26, 11, 5, 237, 25, 11, 5, 237, 213, 11, 5, 237, 207, 11, 5, 238, 34, 11, - 5, 237, 233, 11, 5, 235, 71, 11, 5, 235, 72, 11, 5, 236, 142, 11, 5, 235, - 119, 11, 5, 236, 176, 11, 5, 236, 145, 11, 5, 237, 76, 11, 5, 237, 148, - 11, 5, 237, 98, 11, 5, 235, 62, 11, 5, 235, 60, 11, 5, 235, 91, 11, 5, - 235, 70, 11, 5, 235, 65, 11, 5, 235, 68, 11, 5, 198, 125, 11, 5, 198, - 117, 11, 5, 198, 193, 11, 5, 198, 135, 11, 5, 198, 176, 11, 5, 198, 178, - 11, 5, 198, 177, 11, 5, 199, 171, 11, 5, 199, 156, 11, 5, 190, 190, 11, - 5, 199, 183, 11, 5, 197, 69, 11, 5, 197, 68, 11, 5, 197, 71, 11, 5, 197, - 70, 11, 5, 198, 40, 11, 5, 198, 29, 11, 5, 159, 11, 5, 198, 53, 11, 5, - 199, 55, 11, 5, 199, 145, 11, 5, 199, 82, 11, 5, 197, 52, 11, 5, 197, 47, - 11, 5, 197, 94, 11, 5, 197, 67, 11, 5, 197, 53, 11, 5, 197, 64, 11, 5, - 237, 165, 11, 5, 237, 164, 11, 5, 237, 170, 11, 5, 237, 166, 11, 5, 237, - 167, 11, 5, 237, 169, 11, 5, 237, 168, 11, 5, 237, 186, 11, 5, 237, 185, - 11, 5, 237, 193, 11, 5, 237, 187, 11, 5, 237, 155, 11, 5, 237, 157, 11, - 5, 237, 156, 11, 5, 237, 160, 11, 5, 237, 159, 11, 5, 237, 163, 11, 5, - 237, 161, 11, 5, 237, 178, 11, 5, 237, 181, 11, 5, 237, 179, 11, 5, 237, - 151, 11, 5, 237, 150, 11, 5, 237, 158, 11, 5, 237, 154, 11, 5, 237, 152, - 11, 5, 237, 153, 11, 5, 214, 142, 11, 5, 214, 141, 11, 5, 214, 149, 11, - 5, 214, 144, 11, 5, 214, 145, 11, 5, 214, 146, 11, 5, 214, 158, 11, 5, - 214, 157, 11, 5, 214, 164, 11, 5, 214, 159, 11, 5, 214, 134, 11, 5, 214, - 133, 11, 5, 214, 140, 11, 5, 214, 135, 11, 5, 214, 150, 11, 5, 214, 156, - 11, 5, 214, 154, 11, 5, 214, 126, 11, 5, 214, 125, 11, 5, 214, 131, 11, - 5, 214, 129, 11, 5, 214, 127, 11, 5, 214, 128, 11, 5, 229, 35, 11, 5, - 229, 34, 11, 5, 229, 41, 11, 5, 229, 36, 11, 5, 229, 38, 11, 5, 229, 37, - 11, 5, 229, 40, 11, 5, 229, 39, 11, 5, 229, 47, 11, 5, 229, 45, 11, 5, - 229, 49, 11, 5, 229, 48, 11, 5, 229, 28, 11, 5, 229, 29, 11, 5, 229, 32, - 11, 5, 229, 31, 11, 5, 229, 33, 11, 5, 229, 42, 11, 5, 229, 44, 11, 5, - 229, 43, 11, 5, 229, 27, 11, 5, 213, 164, 11, 5, 213, 162, 11, 5, 213, - 221, 11, 5, 213, 167, 11, 5, 213, 195, 11, 5, 213, 209, 11, 5, 213, 208, - 11, 5, 214, 202, 11, 5, 181, 11, 5, 214, 220, 11, 5, 212, 124, 11, 5, - 212, 126, 11, 5, 212, 125, 11, 5, 213, 13, 11, 5, 212, 253, 11, 5, 213, - 45, 11, 5, 213, 24, 11, 5, 214, 76, 11, 5, 214, 123, 11, 5, 214, 99, 11, - 5, 212, 119, 11, 5, 212, 115, 11, 5, 212, 180, 11, 5, 212, 123, 11, 5, - 212, 121, 11, 5, 212, 122, 11, 5, 229, 100, 11, 5, 229, 99, 11, 5, 229, - 105, 11, 5, 229, 101, 11, 5, 229, 102, 11, 5, 229, 104, 11, 5, 229, 103, - 11, 5, 229, 111, 11, 5, 229, 109, 11, 5, 229, 113, 11, 5, 229, 112, 11, - 5, 229, 92, 11, 5, 229, 94, 11, 5, 229, 93, 11, 5, 229, 96, 11, 5, 229, - 98, 11, 5, 229, 97, 11, 5, 229, 106, 11, 5, 229, 108, 11, 5, 229, 107, - 11, 5, 229, 88, 11, 5, 229, 87, 11, 5, 229, 95, 11, 5, 229, 91, 11, 5, - 229, 89, 11, 5, 229, 90, 11, 5, 229, 82, 11, 5, 229, 81, 11, 5, 229, 86, - 11, 5, 229, 85, 11, 5, 229, 83, 11, 5, 229, 84, 11, 5, 219, 89, 11, 5, - 219, 81, 11, 5, 219, 148, 11, 5, 219, 100, 11, 5, 219, 139, 11, 5, 219, - 138, 11, 5, 219, 142, 11, 5, 219, 140, 11, 5, 220, 7, 11, 5, 219, 251, - 11, 5, 173, 11, 5, 220, 18, 11, 5, 218, 203, 11, 5, 218, 202, 11, 5, 218, - 205, 11, 5, 218, 204, 11, 5, 218, 251, 11, 5, 218, 236, 11, 5, 219, 45, - 11, 5, 219, 3, 11, 5, 219, 166, 11, 5, 219, 240, 11, 5, 219, 186, 11, 5, - 218, 197, 11, 5, 218, 195, 11, 5, 218, 227, 11, 5, 218, 201, 11, 5, 218, - 199, 11, 5, 218, 200, 11, 5, 218, 175, 11, 5, 218, 174, 11, 5, 218, 185, - 11, 5, 218, 178, 11, 5, 218, 176, 11, 5, 218, 177, 11, 5, 231, 24, 11, 5, - 231, 23, 11, 5, 231, 55, 11, 5, 231, 36, 11, 5, 231, 47, 11, 5, 231, 46, - 11, 5, 231, 49, 11, 5, 231, 48, 11, 5, 231, 194, 11, 5, 231, 189, 11, 5, - 231, 242, 11, 5, 231, 205, 11, 5, 230, 154, 11, 5, 230, 153, 11, 5, 230, - 156, 11, 5, 230, 155, 11, 5, 230, 234, 11, 5, 230, 232, 11, 5, 231, 5, - 11, 5, 230, 244, 11, 5, 231, 133, 11, 5, 231, 131, 11, 5, 231, 167, 11, - 5, 231, 144, 11, 5, 230, 142, 11, 5, 230, 141, 11, 5, 230, 181, 11, 5, - 230, 152, 11, 5, 230, 143, 11, 5, 230, 151, 11, 5, 221, 122, 11, 5, 221, - 117, 11, 5, 221, 168, 11, 5, 221, 136, 11, 5, 221, 149, 11, 5, 221, 153, - 11, 5, 221, 151, 11, 5, 222, 51, 11, 5, 222, 32, 11, 5, 155, 11, 5, 222, - 80, 11, 5, 220, 204, 11, 5, 220, 209, 11, 5, 220, 206, 11, 5, 221, 30, - 11, 5, 221, 25, 11, 5, 221, 69, 11, 5, 221, 38, 11, 5, 221, 243, 11, 5, - 221, 226, 11, 5, 222, 24, 11, 5, 221, 247, 11, 5, 220, 190, 11, 5, 220, - 186, 11, 5, 220, 234, 11, 5, 220, 203, 11, 5, 220, 194, 11, 5, 220, 199, - 11, 5, 231, 115, 11, 5, 231, 114, 11, 5, 231, 119, 11, 5, 231, 116, 11, - 5, 231, 118, 11, 5, 231, 117, 11, 5, 231, 126, 11, 5, 231, 125, 11, 5, - 231, 129, 11, 5, 231, 127, 11, 5, 231, 106, 11, 5, 231, 105, 11, 5, 231, - 108, 11, 5, 231, 107, 11, 5, 231, 111, 11, 5, 231, 110, 11, 5, 231, 113, - 11, 5, 231, 112, 11, 5, 231, 121, 11, 5, 231, 120, 11, 5, 231, 124, 11, - 5, 231, 122, 11, 5, 231, 101, 11, 5, 231, 100, 11, 5, 231, 109, 11, 5, - 231, 104, 11, 5, 231, 102, 11, 5, 231, 103, 11, 5, 216, 121, 11, 5, 216, - 122, 11, 5, 216, 140, 11, 5, 216, 139, 11, 5, 216, 142, 11, 5, 216, 141, - 11, 5, 216, 112, 11, 5, 216, 114, 11, 5, 216, 113, 11, 5, 216, 117, 11, - 5, 216, 116, 11, 5, 216, 119, 11, 5, 216, 118, 11, 5, 216, 123, 11, 5, - 216, 125, 11, 5, 216, 124, 11, 5, 216, 108, 11, 5, 216, 107, 11, 5, 216, - 115, 11, 5, 216, 111, 11, 5, 216, 109, 11, 5, 216, 110, 11, 5, 228, 106, - 11, 5, 228, 105, 11, 5, 228, 112, 11, 5, 228, 107, 11, 5, 228, 109, 11, - 5, 228, 108, 11, 5, 228, 111, 11, 5, 228, 110, 11, 5, 228, 117, 11, 5, - 228, 116, 11, 5, 228, 119, 11, 5, 228, 118, 11, 5, 228, 98, 11, 5, 228, - 97, 11, 5, 228, 100, 11, 5, 228, 99, 11, 5, 228, 102, 11, 5, 228, 101, - 11, 5, 228, 104, 11, 5, 228, 103, 11, 5, 228, 113, 11, 5, 228, 115, 11, - 5, 228, 114, 11, 5, 214, 15, 11, 5, 214, 17, 11, 5, 214, 16, 11, 5, 214, - 60, 11, 5, 214, 58, 11, 5, 214, 70, 11, 5, 214, 63, 11, 5, 213, 232, 11, - 5, 213, 231, 11, 5, 213, 233, 11, 5, 213, 243, 11, 5, 213, 240, 11, 5, - 213, 251, 11, 5, 213, 245, 11, 5, 214, 51, 11, 5, 214, 57, 11, 5, 214, - 53, 11, 5, 229, 119, 11, 5, 229, 138, 11, 5, 229, 147, 11, 5, 230, 11, - 11, 5, 229, 255, 11, 5, 140, 11, 5, 230, 23, 11, 5, 228, 136, 11, 5, 228, - 135, 11, 5, 228, 138, 11, 5, 228, 137, 11, 5, 228, 182, 11, 5, 228, 173, - 11, 5, 229, 25, 11, 5, 228, 246, 11, 5, 229, 185, 11, 5, 229, 247, 11, 5, - 229, 197, 11, 5, 193, 146, 11, 5, 193, 131, 11, 5, 193, 190, 11, 5, 193, - 158, 11, 5, 192, 245, 11, 5, 192, 247, 11, 5, 192, 246, 11, 5, 193, 13, - 11, 5, 193, 48, 11, 5, 193, 24, 11, 5, 193, 99, 11, 5, 193, 125, 11, 5, - 193, 106, 11, 5, 191, 15, 11, 5, 191, 14, 11, 5, 191, 30, 11, 5, 191, 18, - 11, 5, 191, 23, 11, 5, 191, 25, 11, 5, 191, 24, 11, 5, 191, 96, 11, 5, - 191, 93, 11, 5, 191, 123, 11, 5, 191, 104, 11, 5, 190, 244, 11, 5, 190, - 246, 11, 5, 190, 245, 11, 5, 191, 2, 11, 5, 191, 1, 11, 5, 191, 7, 11, 5, - 191, 3, 11, 5, 191, 73, 11, 5, 191, 87, 11, 5, 191, 79, 11, 5, 190, 240, - 11, 5, 190, 239, 11, 5, 190, 251, 11, 5, 190, 243, 11, 5, 190, 241, 11, - 5, 190, 242, 11, 5, 190, 226, 11, 5, 190, 225, 11, 5, 190, 231, 11, 5, - 190, 229, 11, 5, 190, 227, 11, 5, 190, 228, 11, 5, 243, 1, 11, 5, 242, - 250, 11, 5, 243, 31, 11, 5, 243, 14, 11, 5, 243, 28, 11, 5, 243, 22, 11, - 5, 243, 30, 11, 5, 243, 29, 11, 5, 247, 80, 11, 5, 247, 71, 11, 5, 247, - 162, 11, 5, 247, 113, 11, 5, 238, 185, 11, 5, 238, 187, 11, 5, 238, 186, - 11, 5, 238, 251, 11, 5, 238, 239, 11, 5, 242, 101, 11, 5, 239, 15, 11, 5, - 247, 7, 11, 5, 247, 44, 11, 5, 247, 13, 11, 5, 238, 156, 11, 5, 238, 154, - 11, 5, 238, 197, 11, 5, 238, 183, 11, 5, 238, 162, 11, 5, 238, 178, 11, - 5, 238, 132, 11, 5, 238, 131, 11, 5, 238, 145, 11, 5, 238, 139, 11, 5, - 238, 133, 11, 5, 238, 135, 11, 5, 190, 209, 11, 5, 190, 208, 11, 5, 190, - 215, 11, 5, 190, 210, 11, 5, 190, 212, 11, 5, 190, 211, 11, 5, 190, 214, - 11, 5, 190, 213, 11, 5, 190, 221, 11, 5, 190, 220, 11, 5, 190, 224, 11, - 5, 190, 222, 11, 5, 190, 205, 11, 5, 190, 207, 11, 5, 190, 206, 11, 5, - 190, 216, 11, 5, 190, 219, 11, 5, 190, 217, 11, 5, 190, 198, 11, 5, 190, - 202, 11, 5, 190, 201, 11, 5, 190, 199, 11, 5, 190, 200, 11, 5, 190, 192, - 11, 5, 190, 191, 11, 5, 190, 197, 11, 5, 190, 195, 11, 5, 190, 193, 11, - 5, 190, 194, 11, 5, 212, 35, 11, 5, 212, 34, 11, 5, 212, 40, 11, 5, 212, - 36, 11, 5, 212, 37, 11, 5, 212, 39, 11, 5, 212, 38, 11, 5, 212, 45, 11, - 5, 212, 44, 11, 5, 212, 48, 11, 5, 212, 47, 11, 5, 212, 28, 11, 5, 212, - 29, 11, 5, 212, 32, 11, 5, 212, 33, 11, 5, 212, 41, 11, 5, 212, 43, 11, - 5, 212, 23, 11, 5, 212, 31, 11, 5, 212, 27, 11, 5, 212, 24, 11, 5, 212, - 25, 11, 5, 212, 18, 11, 5, 212, 17, 11, 5, 212, 22, 11, 5, 212, 21, 11, - 5, 212, 19, 11, 5, 212, 20, 11, 5, 202, 139, 11, 5, 169, 11, 5, 202, 223, - 11, 5, 202, 143, 11, 5, 202, 203, 11, 5, 202, 206, 11, 5, 202, 204, 11, - 5, 205, 123, 11, 5, 205, 107, 11, 5, 188, 11, 5, 205, 131, 11, 5, 200, - 188, 11, 5, 200, 190, 11, 5, 200, 189, 11, 5, 202, 9, 11, 5, 201, 254, - 11, 5, 202, 47, 11, 5, 202, 15, 11, 5, 203, 239, 11, 5, 205, 69, 11, 5, - 204, 14, 11, 5, 200, 163, 11, 5, 200, 159, 11, 5, 201, 5, 11, 5, 200, - 187, 11, 5, 200, 167, 11, 5, 200, 175, 11, 5, 200, 57, 11, 5, 200, 56, - 11, 5, 200, 127, 11, 5, 200, 65, 11, 5, 200, 59, 11, 5, 200, 64, 11, 5, - 201, 137, 11, 5, 201, 136, 11, 5, 201, 143, 11, 5, 201, 138, 11, 5, 201, - 140, 11, 5, 201, 142, 11, 5, 201, 141, 11, 5, 201, 152, 11, 5, 201, 150, - 11, 5, 201, 176, 11, 5, 201, 153, 11, 5, 201, 132, 11, 5, 201, 131, 11, - 5, 201, 135, 11, 5, 201, 133, 11, 5, 201, 146, 11, 5, 201, 149, 11, 5, - 201, 147, 11, 5, 201, 128, 11, 5, 201, 126, 11, 5, 201, 130, 11, 5, 201, - 129, 11, 5, 201, 121, 11, 5, 201, 120, 11, 5, 201, 125, 11, 5, 201, 124, - 11, 5, 201, 122, 11, 5, 201, 123, 11, 5, 191, 66, 11, 5, 191, 65, 11, 5, - 191, 71, 11, 5, 191, 68, 11, 5, 191, 45, 11, 5, 191, 47, 11, 5, 191, 46, - 11, 5, 191, 50, 11, 5, 191, 49, 11, 5, 191, 54, 11, 5, 191, 51, 11, 5, - 191, 59, 11, 5, 191, 58, 11, 5, 191, 62, 11, 5, 191, 60, 11, 5, 191, 41, - 11, 5, 191, 40, 11, 5, 191, 48, 11, 5, 191, 44, 11, 5, 191, 42, 11, 5, - 191, 43, 11, 5, 191, 33, 11, 5, 191, 32, 11, 5, 191, 37, 11, 5, 191, 36, - 11, 5, 191, 34, 11, 5, 191, 35, 11, 5, 243, 135, 11, 5, 243, 131, 11, 5, - 247, 3, 11, 5, 246, 245, 11, 5, 243, 46, 11, 5, 243, 45, 11, 5, 243, 48, - 11, 5, 243, 47, 11, 5, 243, 61, 11, 5, 243, 60, 11, 5, 243, 70, 11, 5, - 243, 65, 11, 5, 243, 104, 11, 5, 243, 101, 11, 5, 243, 129, 11, 5, 243, - 112, 11, 5, 243, 40, 11, 5, 243, 50, 11, 5, 243, 44, 11, 5, 243, 41, 11, - 5, 243, 43, 11, 5, 243, 33, 11, 5, 243, 32, 11, 5, 243, 37, 11, 5, 243, - 36, 11, 5, 243, 34, 11, 5, 243, 35, 11, 5, 206, 106, 11, 5, 206, 110, 11, - 5, 206, 88, 11, 5, 206, 89, 11, 5, 206, 93, 11, 5, 206, 92, 11, 5, 206, - 96, 11, 5, 206, 94, 11, 5, 206, 100, 11, 5, 206, 99, 11, 5, 206, 105, 11, - 5, 206, 101, 11, 5, 206, 84, 11, 5, 206, 82, 11, 5, 206, 90, 11, 5, 206, - 87, 11, 5, 206, 85, 11, 5, 206, 86, 11, 5, 206, 77, 11, 5, 206, 76, 11, - 5, 206, 81, 11, 5, 206, 80, 11, 5, 206, 78, 11, 5, 206, 79, 11, 5, 212, - 244, 11, 5, 212, 243, 11, 5, 212, 246, 11, 5, 212, 245, 11, 5, 212, 235, - 11, 5, 212, 237, 11, 5, 212, 236, 11, 5, 212, 239, 11, 5, 212, 238, 11, - 5, 212, 242, 11, 5, 212, 241, 11, 5, 212, 229, 11, 5, 212, 228, 11, 5, - 212, 234, 11, 5, 212, 232, 11, 5, 212, 230, 11, 5, 212, 231, 11, 5, 212, - 223, 11, 5, 212, 222, 11, 5, 212, 227, 11, 5, 212, 226, 11, 5, 212, 224, - 11, 5, 212, 225, 11, 5, 203, 124, 11, 5, 203, 119, 11, 5, 203, 166, 11, - 5, 203, 137, 11, 5, 202, 250, 11, 5, 202, 252, 11, 5, 202, 251, 11, 5, - 203, 25, 11, 5, 203, 20, 11, 5, 203, 57, 11, 5, 203, 45, 11, 5, 203, 92, - 11, 5, 203, 85, 11, 5, 203, 114, 11, 5, 203, 101, 11, 5, 202, 246, 11, 5, - 202, 243, 11, 5, 203, 6, 11, 5, 202, 249, 11, 5, 202, 247, 11, 5, 202, - 248, 11, 5, 202, 226, 11, 5, 202, 225, 11, 5, 202, 232, 11, 5, 202, 229, - 11, 5, 202, 227, 11, 5, 202, 228, 11, 5, 207, 132, 11, 5, 207, 125, 11, - 5, 165, 11, 5, 207, 138, 11, 5, 206, 39, 11, 5, 206, 41, 11, 5, 206, 40, - 11, 5, 206, 124, 11, 5, 206, 112, 11, 5, 206, 163, 11, 5, 206, 128, 11, - 5, 207, 12, 11, 5, 207, 115, 11, 5, 207, 54, 11, 5, 206, 31, 11, 5, 206, - 28, 11, 5, 206, 69, 11, 5, 206, 38, 11, 5, 206, 34, 11, 5, 206, 35, 11, - 5, 206, 13, 11, 5, 206, 12, 11, 5, 206, 18, 11, 5, 206, 16, 11, 5, 206, - 14, 11, 5, 206, 15, 11, 5, 222, 234, 11, 5, 222, 233, 11, 5, 222, 246, - 11, 5, 222, 235, 11, 5, 222, 242, 11, 5, 222, 241, 11, 5, 222, 244, 11, - 5, 222, 243, 11, 5, 222, 172, 11, 5, 222, 171, 11, 5, 222, 174, 11, 5, - 222, 173, 11, 5, 222, 190, 11, 5, 222, 188, 11, 5, 222, 203, 11, 5, 222, - 192, 11, 5, 222, 165, 11, 5, 222, 163, 11, 5, 222, 184, 11, 5, 222, 170, - 11, 5, 222, 167, 11, 5, 222, 168, 11, 5, 222, 157, 11, 5, 222, 156, 11, - 5, 222, 161, 11, 5, 222, 160, 11, 5, 222, 158, 11, 5, 222, 159, 11, 5, - 208, 51, 11, 5, 208, 49, 11, 5, 208, 59, 11, 5, 208, 52, 11, 5, 208, 56, - 11, 5, 208, 55, 11, 5, 208, 58, 11, 5, 208, 57, 11, 5, 207, 255, 11, 5, - 207, 252, 11, 5, 208, 1, 11, 5, 208, 0, 11, 5, 208, 38, 11, 5, 208, 37, - 11, 5, 208, 47, 11, 5, 208, 41, 11, 5, 207, 247, 11, 5, 207, 243, 11, 5, - 208, 35, 11, 5, 207, 251, 11, 5, 207, 249, 11, 5, 207, 250, 11, 5, 207, - 227, 11, 5, 207, 225, 11, 5, 207, 237, 11, 5, 207, 230, 11, 5, 207, 228, - 11, 5, 207, 229, 11, 5, 222, 223, 11, 5, 222, 222, 11, 5, 222, 229, 11, - 5, 222, 224, 11, 5, 222, 226, 11, 5, 222, 225, 11, 5, 222, 228, 11, 5, - 222, 227, 11, 5, 222, 214, 11, 5, 222, 216, 11, 5, 222, 215, 11, 5, 222, - 219, 11, 5, 222, 218, 11, 5, 222, 221, 11, 5, 222, 220, 11, 5, 222, 210, - 11, 5, 222, 209, 11, 5, 222, 217, 11, 5, 222, 213, 11, 5, 222, 211, 11, - 5, 222, 212, 11, 5, 222, 206, 11, 5, 222, 205, 11, 5, 222, 208, 11, 5, - 222, 207, 11, 5, 213, 136, 11, 5, 213, 135, 11, 5, 213, 143, 11, 5, 213, - 137, 11, 5, 213, 139, 11, 5, 213, 138, 11, 5, 213, 142, 11, 5, 213, 140, - 11, 5, 213, 125, 11, 5, 213, 126, 11, 5, 213, 131, 11, 5, 213, 130, 11, - 5, 213, 134, 11, 5, 213, 132, 11, 5, 213, 120, 11, 5, 213, 129, 11, 5, - 213, 124, 11, 5, 213, 121, 11, 5, 213, 122, 11, 5, 213, 115, 11, 5, 213, - 114, 11, 5, 213, 119, 11, 5, 213, 118, 11, 5, 213, 116, 11, 5, 213, 117, - 11, 5, 212, 70, 11, 5, 212, 69, 11, 5, 212, 83, 11, 5, 212, 74, 11, 5, - 212, 79, 11, 5, 212, 78, 11, 5, 212, 81, 11, 5, 212, 80, 11, 5, 212, 55, - 11, 5, 212, 57, 11, 5, 212, 56, 11, 5, 212, 62, 11, 5, 212, 61, 11, 5, - 212, 67, 11, 5, 212, 63, 11, 5, 212, 53, 11, 5, 212, 51, 11, 5, 212, 60, - 11, 5, 212, 54, 11, 5, 192, 198, 11, 5, 192, 197, 11, 5, 192, 207, 11, 5, - 192, 200, 11, 5, 192, 202, 11, 5, 192, 201, 11, 5, 192, 204, 11, 5, 192, - 203, 11, 5, 192, 186, 11, 5, 192, 187, 11, 5, 192, 191, 11, 5, 192, 190, - 11, 5, 192, 196, 11, 5, 192, 194, 11, 5, 192, 163, 11, 5, 192, 161, 11, - 5, 192, 176, 11, 5, 192, 166, 11, 5, 192, 164, 11, 5, 192, 165, 11, 5, - 192, 18, 11, 5, 192, 16, 11, 5, 192, 33, 11, 5, 192, 19, 11, 5, 192, 27, - 11, 5, 192, 26, 11, 5, 192, 30, 11, 5, 192, 28, 11, 5, 191, 198, 11, 5, - 191, 197, 11, 5, 191, 201, 11, 5, 191, 199, 11, 5, 191, 240, 11, 5, 191, - 235, 11, 5, 192, 12, 11, 5, 191, 245, 11, 5, 191, 189, 11, 5, 191, 185, - 11, 5, 191, 225, 11, 5, 191, 196, 11, 5, 191, 192, 11, 5, 191, 193, 11, - 5, 191, 169, 11, 5, 191, 168, 11, 5, 191, 176, 11, 5, 191, 172, 11, 5, - 191, 170, 11, 5, 191, 171, 11, 48, 208, 38, 11, 48, 219, 148, 11, 48, - 221, 122, 11, 48, 212, 74, 11, 48, 238, 139, 11, 48, 201, 143, 11, 48, - 231, 112, 11, 48, 231, 144, 11, 48, 216, 102, 11, 48, 228, 106, 11, 48, - 218, 177, 11, 48, 249, 27, 11, 48, 215, 204, 11, 48, 192, 12, 11, 48, - 208, 133, 11, 48, 228, 100, 11, 48, 199, 171, 11, 48, 231, 242, 11, 48, - 190, 243, 11, 48, 238, 132, 11, 48, 237, 153, 11, 48, 247, 232, 11, 48, - 231, 108, 11, 48, 212, 63, 11, 48, 197, 94, 11, 48, 211, 80, 11, 48, 222, - 210, 11, 48, 191, 2, 11, 48, 208, 110, 11, 48, 229, 67, 11, 48, 192, 18, - 11, 48, 193, 244, 11, 48, 202, 232, 11, 48, 195, 141, 11, 48, 191, 123, - 11, 48, 222, 203, 11, 48, 212, 27, 11, 48, 222, 208, 11, 48, 230, 234, - 11, 48, 222, 228, 11, 48, 193, 48, 11, 48, 235, 91, 11, 48, 202, 248, 11, - 48, 219, 142, 11, 48, 238, 145, 11, 48, 238, 186, 11, 48, 243, 14, 11, - 48, 228, 103, 11, 48, 203, 124, 11, 48, 190, 242, 11, 48, 203, 45, 11, - 48, 243, 129, 11, 48, 190, 212, 11, 48, 214, 191, 11, 48, 222, 24, 219, - 90, 1, 249, 155, 219, 90, 1, 168, 219, 90, 1, 209, 230, 219, 90, 1, 238, - 34, 219, 90, 1, 190, 190, 219, 90, 1, 199, 49, 219, 90, 1, 231, 242, 219, - 90, 1, 155, 219, 90, 1, 221, 217, 219, 90, 1, 223, 34, 219, 90, 1, 247, - 162, 219, 90, 1, 247, 3, 219, 90, 1, 235, 37, 219, 90, 1, 197, 168, 219, - 90, 1, 197, 157, 219, 90, 1, 174, 219, 90, 1, 181, 219, 90, 1, 173, 219, - 90, 1, 188, 219, 90, 1, 191, 71, 219, 90, 1, 191, 123, 219, 90, 1, 214, - 70, 219, 90, 1, 140, 219, 90, 1, 192, 220, 219, 90, 1, 229, 179, 219, 90, - 1, 233, 111, 219, 90, 1, 193, 190, 219, 90, 1, 203, 166, 219, 90, 1, 170, - 219, 90, 1, 231, 93, 219, 90, 1, 65, 219, 90, 1, 252, 27, 219, 90, 1, 71, - 219, 90, 1, 233, 244, 219, 90, 1, 68, 219, 90, 1, 74, 219, 90, 1, 66, - 219, 90, 1, 196, 152, 219, 90, 1, 196, 141, 219, 90, 1, 211, 153, 219, - 90, 1, 163, 215, 71, 198, 193, 219, 90, 1, 163, 215, 9, 209, 75, 219, 90, - 1, 163, 215, 71, 238, 144, 219, 90, 1, 163, 215, 71, 248, 113, 219, 90, - 1, 163, 215, 71, 181, 219, 90, 1, 163, 215, 71, 222, 255, 219, 90, 208, - 154, 242, 76, 219, 90, 208, 154, 232, 82, 201, 64, 59, 5, 234, 190, 59, - 5, 234, 186, 59, 5, 229, 217, 59, 5, 193, 114, 59, 5, 193, 113, 59, 5, - 210, 51, 59, 5, 248, 197, 59, 5, 249, 3, 59, 5, 217, 5, 59, 5, 221, 18, - 59, 5, 216, 134, 59, 5, 231, 180, 59, 5, 233, 54, 59, 5, 195, 148, 59, 5, - 199, 121, 59, 5, 199, 31, 59, 5, 237, 60, 59, 5, 237, 57, 59, 5, 219, - 230, 59, 5, 207, 86, 59, 5, 237, 133, 59, 5, 214, 155, 59, 5, 205, 51, - 59, 5, 203, 112, 59, 5, 191, 84, 59, 5, 191, 61, 59, 5, 247, 36, 59, 5, - 223, 10, 59, 5, 213, 150, 59, 5, 192, 77, 59, 5, 222, 15, 59, 5, 214, 43, - 59, 5, 231, 159, 59, 5, 216, 213, 59, 5, 214, 112, 59, 5, 212, 91, 59, 5, - 68, 59, 5, 223, 164, 59, 5, 229, 160, 59, 5, 229, 130, 59, 5, 193, 86, - 59, 5, 193, 68, 59, 5, 209, 187, 59, 5, 248, 195, 59, 5, 248, 190, 59, 5, - 216, 254, 59, 5, 221, 15, 59, 5, 216, 131, 59, 5, 231, 176, 59, 5, 233, - 25, 59, 5, 195, 69, 59, 5, 198, 193, 59, 5, 199, 11, 59, 5, 237, 52, 59, - 5, 237, 56, 59, 5, 219, 148, 59, 5, 207, 2, 59, 5, 237, 46, 59, 5, 214, - 149, 59, 5, 202, 223, 59, 5, 203, 82, 59, 5, 191, 30, 59, 5, 191, 57, 59, - 5, 243, 31, 59, 5, 222, 246, 59, 5, 213, 143, 59, 5, 192, 33, 59, 5, 221, - 168, 59, 5, 214, 35, 59, 5, 231, 55, 59, 5, 216, 102, 59, 5, 213, 221, - 59, 5, 212, 83, 59, 5, 65, 59, 5, 251, 134, 59, 5, 214, 65, 59, 5, 140, - 59, 5, 230, 58, 59, 5, 193, 190, 59, 5, 193, 164, 59, 5, 168, 59, 5, 248, - 205, 59, 5, 249, 155, 59, 5, 217, 13, 59, 5, 221, 23, 59, 5, 221, 21, 59, - 5, 216, 138, 59, 5, 231, 184, 59, 5, 233, 111, 59, 5, 195, 188, 59, 5, - 190, 190, 59, 5, 199, 49, 59, 5, 237, 70, 59, 5, 237, 59, 59, 5, 173, 59, - 5, 165, 59, 5, 238, 34, 59, 5, 214, 164, 59, 5, 188, 59, 5, 203, 166, 59, - 5, 191, 123, 59, 5, 191, 71, 59, 5, 247, 162, 59, 5, 223, 34, 59, 5, 213, - 159, 59, 5, 170, 59, 5, 155, 59, 5, 222, 89, 59, 5, 214, 49, 59, 5, 231, - 242, 59, 5, 174, 59, 5, 181, 59, 5, 212, 103, 59, 5, 211, 89, 59, 5, 211, - 84, 59, 5, 228, 254, 59, 5, 193, 29, 59, 5, 193, 25, 59, 5, 209, 39, 59, - 5, 248, 193, 59, 5, 248, 99, 59, 5, 216, 249, 59, 5, 221, 13, 59, 5, 216, - 127, 59, 5, 231, 172, 59, 5, 232, 164, 59, 5, 195, 8, 59, 5, 198, 59, 59, - 5, 198, 235, 59, 5, 237, 49, 59, 5, 237, 54, 59, 5, 219, 10, 59, 5, 206, - 135, 59, 5, 236, 148, 59, 5, 214, 136, 59, 5, 202, 17, 59, 5, 203, 49, - 59, 5, 191, 4, 59, 5, 191, 52, 59, 5, 239, 20, 59, 5, 222, 193, 59, 5, - 213, 133, 59, 5, 191, 246, 59, 5, 221, 43, 59, 5, 214, 33, 59, 5, 230, - 247, 59, 5, 215, 213, 59, 5, 213, 28, 59, 5, 212, 64, 59, 5, 66, 59, 5, - 196, 113, 59, 5, 228, 161, 59, 5, 228, 144, 59, 5, 193, 0, 59, 5, 192, - 249, 59, 5, 208, 167, 59, 5, 248, 192, 59, 5, 248, 12, 59, 5, 216, 248, - 59, 5, 221, 11, 59, 5, 216, 126, 59, 5, 231, 171, 59, 5, 232, 88, 59, 5, - 193, 249, 59, 5, 197, 94, 59, 5, 198, 213, 59, 5, 237, 47, 59, 5, 237, - 53, 59, 5, 218, 227, 59, 5, 206, 69, 59, 5, 235, 91, 59, 5, 214, 131, 59, - 5, 201, 5, 59, 5, 203, 6, 59, 5, 190, 251, 59, 5, 191, 48, 59, 5, 238, - 197, 59, 5, 222, 184, 59, 5, 213, 129, 59, 5, 191, 225, 59, 5, 220, 234, - 59, 5, 214, 32, 59, 5, 230, 181, 59, 5, 215, 157, 59, 5, 212, 180, 59, 5, - 212, 60, 59, 5, 74, 59, 5, 211, 106, 59, 5, 213, 247, 59, 5, 229, 25, 59, - 5, 229, 1, 59, 5, 193, 48, 59, 5, 193, 30, 59, 5, 209, 75, 59, 5, 248, - 194, 59, 5, 248, 113, 59, 5, 216, 250, 59, 5, 221, 14, 59, 5, 216, 129, - 59, 5, 231, 174, 59, 5, 231, 173, 59, 5, 232, 177, 59, 5, 195, 24, 59, 5, - 159, 59, 5, 198, 241, 59, 5, 237, 50, 59, 5, 237, 55, 59, 5, 219, 45, 59, - 5, 206, 163, 59, 5, 236, 176, 59, 5, 214, 140, 59, 5, 202, 47, 59, 5, - 203, 57, 59, 5, 191, 7, 59, 5, 191, 54, 59, 5, 242, 101, 59, 5, 222, 203, - 59, 5, 213, 134, 59, 5, 192, 12, 59, 5, 221, 69, 59, 5, 214, 34, 59, 5, - 231, 5, 59, 5, 216, 14, 59, 5, 213, 45, 59, 5, 212, 67, 59, 5, 71, 59, 5, - 234, 105, 59, 5, 214, 54, 59, 5, 229, 247, 59, 5, 229, 200, 59, 5, 193, - 125, 59, 5, 193, 108, 59, 5, 210, 65, 59, 5, 248, 198, 59, 5, 249, 19, - 59, 5, 217, 6, 59, 5, 221, 19, 59, 5, 221, 17, 59, 5, 216, 135, 59, 5, - 231, 181, 59, 5, 231, 179, 59, 5, 233, 61, 59, 5, 195, 153, 59, 5, 199, - 145, 59, 5, 199, 33, 59, 5, 237, 61, 59, 5, 237, 58, 59, 5, 219, 240, 59, - 5, 207, 115, 59, 5, 237, 148, 59, 5, 214, 156, 59, 5, 205, 69, 59, 5, - 203, 114, 59, 5, 191, 87, 59, 5, 191, 62, 59, 5, 247, 44, 59, 5, 223, 12, - 59, 5, 213, 152, 59, 5, 192, 80, 59, 5, 222, 24, 59, 5, 214, 44, 59, 5, - 214, 40, 59, 5, 231, 167, 59, 5, 231, 153, 59, 5, 216, 234, 59, 5, 214, - 123, 59, 5, 212, 92, 59, 5, 214, 72, 59, 5, 219, 192, 59, 242, 76, 59, - 232, 82, 201, 64, 59, 208, 15, 77, 59, 5, 214, 139, 233, 111, 59, 5, 214, - 139, 155, 59, 5, 214, 139, 202, 17, 59, 16, 233, 50, 59, 16, 222, 13, 59, - 16, 198, 140, 59, 16, 213, 188, 59, 16, 249, 97, 59, 16, 233, 110, 59, - 16, 199, 245, 59, 16, 237, 238, 59, 16, 236, 147, 59, 16, 220, 210, 59, - 16, 198, 63, 59, 16, 236, 175, 59, 16, 222, 194, 59, 17, 191, 77, 59, 17, - 107, 59, 17, 109, 59, 17, 138, 59, 17, 134, 59, 17, 150, 59, 17, 169, 59, - 17, 175, 59, 17, 171, 59, 17, 178, 59, 5, 214, 139, 174, 59, 5, 214, 139, - 236, 176, 38, 6, 1, 191, 81, 38, 2, 1, 191, 81, 38, 6, 1, 235, 32, 38, 2, - 1, 235, 32, 38, 6, 1, 207, 19, 235, 34, 38, 2, 1, 207, 19, 235, 34, 38, - 6, 1, 223, 85, 38, 2, 1, 223, 85, 38, 6, 1, 236, 193, 38, 2, 1, 236, 193, - 38, 6, 1, 215, 221, 196, 128, 38, 2, 1, 215, 221, 196, 128, 38, 6, 1, - 248, 26, 211, 112, 38, 2, 1, 248, 26, 211, 112, 38, 6, 1, 214, 84, 192, - 62, 38, 2, 1, 214, 84, 192, 62, 38, 6, 1, 192, 59, 4, 249, 149, 192, 62, - 38, 2, 1, 192, 59, 4, 249, 149, 192, 62, 38, 6, 1, 223, 83, 192, 95, 38, - 2, 1, 223, 83, 192, 95, 38, 6, 1, 207, 19, 191, 225, 38, 2, 1, 207, 19, - 191, 225, 38, 6, 1, 223, 83, 65, 38, 2, 1, 223, 83, 65, 38, 6, 1, 242, - 221, 219, 85, 191, 190, 38, 2, 1, 242, 221, 219, 85, 191, 190, 38, 6, 1, - 248, 133, 191, 190, 38, 2, 1, 248, 133, 191, 190, 38, 6, 1, 223, 83, 242, - 221, 219, 85, 191, 190, 38, 2, 1, 223, 83, 242, 221, 219, 85, 191, 190, - 38, 6, 1, 192, 14, 38, 2, 1, 192, 14, 38, 6, 1, 207, 19, 197, 161, 38, 2, - 1, 207, 19, 197, 161, 38, 6, 1, 202, 33, 237, 148, 38, 2, 1, 202, 33, - 237, 148, 38, 6, 1, 202, 33, 234, 142, 38, 2, 1, 202, 33, 234, 142, 38, - 6, 1, 202, 33, 234, 116, 38, 2, 1, 202, 33, 234, 116, 38, 6, 1, 215, 225, - 74, 38, 2, 1, 215, 225, 74, 38, 6, 1, 248, 166, 74, 38, 2, 1, 248, 166, - 74, 38, 6, 1, 55, 215, 225, 74, 38, 2, 1, 55, 215, 225, 74, 38, 1, 215, - 132, 74, 33, 38, 193, 226, 33, 38, 199, 96, 216, 50, 56, 33, 38, 228, - 143, 216, 50, 56, 33, 38, 198, 230, 216, 50, 56, 202, 96, 250, 195, 33, - 38, 1, 196, 125, 223, 228, 33, 38, 1, 68, 33, 38, 1, 192, 33, 33, 38, 1, - 66, 33, 38, 1, 230, 19, 56, 33, 38, 1, 192, 58, 33, 38, 1, 202, 33, 56, - 33, 38, 1, 211, 112, 33, 38, 222, 37, 33, 38, 210, 72, 38, 222, 37, 38, - 210, 72, 38, 6, 1, 235, 47, 38, 2, 1, 235, 47, 38, 6, 1, 235, 23, 38, 2, - 1, 235, 23, 38, 6, 1, 191, 38, 38, 2, 1, 191, 38, 38, 6, 1, 247, 60, 38, - 2, 1, 247, 60, 38, 6, 1, 235, 19, 38, 2, 1, 235, 19, 38, 6, 1, 199, 146, - 4, 82, 102, 38, 2, 1, 199, 146, 4, 82, 102, 38, 6, 1, 197, 41, 38, 2, 1, - 197, 41, 38, 6, 1, 197, 136, 38, 2, 1, 197, 136, 38, 6, 1, 197, 141, 38, - 2, 1, 197, 141, 38, 6, 1, 199, 151, 38, 2, 1, 199, 151, 38, 6, 1, 228, - 124, 38, 2, 1, 228, 124, 38, 6, 1, 202, 238, 38, 2, 1, 202, 238, 38, 6, - 1, 55, 74, 38, 2, 1, 55, 74, 38, 6, 1, 238, 216, 74, 38, 2, 1, 238, 216, - 74, 52, 1, 38, 230, 19, 56, 52, 1, 38, 202, 33, 56, 33, 38, 1, 234, 183, - 33, 38, 1, 223, 83, 71, 26, 1, 65, 26, 1, 155, 26, 1, 66, 26, 1, 220, - 234, 26, 1, 234, 190, 26, 1, 207, 86, 26, 1, 199, 226, 26, 1, 74, 26, 1, - 212, 83, 26, 1, 68, 26, 1, 173, 26, 1, 168, 26, 1, 206, 196, 26, 1, 206, - 243, 26, 1, 219, 229, 26, 1, 216, 212, 26, 1, 199, 245, 26, 1, 214, 162, - 26, 1, 213, 157, 26, 1, 218, 170, 26, 1, 200, 160, 26, 1, 215, 157, 26, - 1, 203, 77, 26, 1, 202, 223, 26, 1, 203, 87, 26, 1, 203, 249, 26, 1, 220, - 151, 26, 1, 221, 243, 26, 1, 212, 148, 26, 1, 212, 180, 26, 1, 213, 128, - 26, 1, 191, 243, 26, 1, 203, 6, 26, 1, 191, 194, 26, 1, 170, 26, 1, 212, - 217, 26, 1, 221, 229, 26, 1, 209, 234, 26, 1, 213, 150, 26, 1, 212, 197, - 26, 1, 208, 158, 26, 1, 192, 253, 26, 1, 210, 51, 26, 1, 233, 54, 26, 1, - 206, 69, 26, 1, 218, 227, 26, 1, 216, 102, 26, 1, 213, 221, 26, 1, 207, - 21, 26, 1, 207, 165, 26, 1, 221, 253, 26, 1, 213, 254, 26, 1, 214, 49, - 26, 1, 214, 70, 26, 1, 203, 57, 26, 1, 208, 163, 26, 1, 232, 88, 26, 1, - 232, 169, 26, 1, 193, 190, 26, 1, 181, 26, 1, 219, 148, 26, 1, 209, 187, - 26, 1, 219, 2, 26, 1, 221, 69, 26, 1, 217, 3, 26, 1, 207, 56, 26, 1, 216, - 188, 26, 1, 174, 26, 1, 198, 193, 26, 1, 221, 168, 26, 1, 216, 14, 26, 1, - 217, 11, 26, 1, 199, 73, 26, 1, 221, 23, 26, 1, 199, 95, 26, 1, 212, 183, - 26, 1, 205, 151, 26, 1, 233, 107, 26, 1, 221, 26, 26, 1, 221, 59, 26, 33, - 87, 221, 36, 26, 33, 87, 197, 79, 26, 213, 156, 26, 232, 82, 201, 64, 26, - 242, 85, 26, 242, 76, 26, 204, 26, 26, 208, 15, 77, 52, 1, 243, 82, 163, - 192, 22, 209, 134, 52, 1, 243, 82, 163, 192, 107, 209, 134, 52, 1, 243, - 82, 163, 192, 22, 203, 138, 52, 1, 243, 82, 163, 192, 107, 203, 138, 52, - 1, 243, 82, 163, 192, 22, 208, 35, 52, 1, 243, 82, 163, 192, 107, 208, - 35, 52, 1, 243, 82, 163, 192, 22, 206, 69, 52, 1, 243, 82, 163, 192, 107, - 206, 69, 52, 1, 233, 202, 235, 140, 163, 164, 52, 1, 137, 235, 140, 163, - 164, 52, 1, 216, 89, 235, 140, 163, 164, 52, 1, 130, 235, 140, 163, 164, - 52, 1, 233, 201, 235, 140, 163, 164, 52, 1, 233, 202, 235, 140, 219, 218, - 163, 164, 52, 1, 137, 235, 140, 219, 218, 163, 164, 52, 1, 216, 89, 235, - 140, 219, 218, 163, 164, 52, 1, 130, 235, 140, 219, 218, 163, 164, 52, 1, - 233, 201, 235, 140, 219, 218, 163, 164, 52, 1, 233, 202, 219, 218, 163, - 164, 52, 1, 137, 219, 218, 163, 164, 52, 1, 216, 89, 219, 218, 163, 164, - 52, 1, 130, 219, 218, 163, 164, 52, 1, 233, 201, 219, 218, 163, 164, 52, - 1, 75, 81, 164, 52, 1, 75, 202, 98, 52, 1, 75, 228, 243, 164, 52, 1, 110, - 50, 239, 4, 251, 117, 52, 1, 207, 149, 133, 57, 52, 1, 207, 149, 144, 57, - 52, 1, 207, 149, 233, 218, 77, 52, 1, 207, 149, 223, 95, 233, 218, 77, - 52, 1, 130, 223, 95, 233, 218, 77, 52, 1, 201, 39, 23, 137, 198, 79, 52, - 1, 201, 39, 23, 130, 198, 79, 8, 6, 1, 234, 177, 251, 194, 8, 2, 1, 234, - 177, 251, 194, 8, 6, 1, 234, 177, 251, 232, 8, 2, 1, 234, 177, 251, 232, - 8, 6, 1, 229, 198, 8, 2, 1, 229, 198, 8, 6, 1, 196, 241, 8, 2, 1, 196, - 241, 8, 6, 1, 197, 248, 8, 2, 1, 197, 248, 8, 6, 1, 238, 194, 8, 2, 1, - 238, 194, 8, 6, 1, 238, 195, 4, 242, 76, 8, 2, 1, 238, 195, 4, 242, 76, - 8, 1, 2, 6, 233, 177, 8, 1, 2, 6, 206, 9, 8, 6, 1, 252, 208, 8, 2, 1, - 252, 208, 8, 6, 1, 251, 70, 8, 2, 1, 251, 70, 8, 6, 1, 250, 165, 8, 2, 1, - 250, 165, 8, 6, 1, 250, 148, 8, 2, 1, 250, 148, 8, 6, 1, 250, 149, 4, - 228, 243, 164, 8, 2, 1, 250, 149, 4, 228, 243, 164, 8, 6, 1, 250, 133, 8, - 2, 1, 250, 133, 8, 6, 1, 207, 19, 247, 196, 4, 236, 142, 8, 2, 1, 207, - 19, 247, 196, 4, 236, 142, 8, 6, 1, 222, 155, 4, 106, 8, 2, 1, 222, 155, - 4, 106, 8, 6, 1, 222, 155, 4, 237, 41, 106, 8, 2, 1, 222, 155, 4, 237, - 41, 106, 8, 6, 1, 222, 155, 4, 201, 29, 23, 237, 41, 106, 8, 2, 1, 222, - 155, 4, 201, 29, 23, 237, 41, 106, 8, 6, 1, 248, 24, 172, 8, 2, 1, 248, - 24, 172, 8, 6, 1, 220, 145, 4, 137, 106, 8, 2, 1, 220, 145, 4, 137, 106, - 8, 6, 1, 187, 4, 180, 201, 29, 211, 1, 8, 2, 1, 187, 4, 180, 201, 29, - 211, 1, 8, 6, 1, 187, 4, 219, 6, 8, 2, 1, 187, 4, 219, 6, 8, 6, 1, 211, - 89, 8, 2, 1, 211, 89, 8, 6, 1, 210, 239, 4, 201, 29, 198, 216, 237, 89, - 8, 2, 1, 210, 239, 4, 201, 29, 198, 216, 237, 89, 8, 6, 1, 210, 239, 4, - 232, 190, 8, 2, 1, 210, 239, 4, 232, 190, 8, 6, 1, 210, 239, 4, 201, 183, - 199, 215, 8, 2, 1, 210, 239, 4, 201, 183, 199, 215, 8, 6, 1, 208, 107, 4, - 201, 29, 198, 216, 237, 89, 8, 2, 1, 208, 107, 4, 201, 29, 198, 216, 237, - 89, 8, 6, 1, 208, 107, 4, 237, 41, 106, 8, 2, 1, 208, 107, 4, 237, 41, - 106, 8, 6, 1, 207, 224, 206, 117, 8, 2, 1, 207, 224, 206, 117, 8, 6, 1, - 206, 50, 206, 117, 8, 2, 1, 206, 50, 206, 117, 8, 6, 1, 196, 13, 4, 237, - 41, 106, 8, 2, 1, 196, 13, 4, 237, 41, 106, 8, 6, 1, 193, 235, 8, 2, 1, - 193, 235, 8, 6, 1, 195, 33, 191, 166, 8, 2, 1, 195, 33, 191, 166, 8, 6, - 1, 198, 234, 4, 106, 8, 2, 1, 198, 234, 4, 106, 8, 6, 1, 198, 234, 4, - 201, 29, 198, 216, 237, 89, 8, 2, 1, 198, 234, 4, 201, 29, 198, 216, 237, - 89, 8, 6, 1, 195, 142, 8, 2, 1, 195, 142, 8, 6, 1, 234, 1, 8, 2, 1, 234, - 1, 8, 6, 1, 223, 70, 8, 2, 1, 223, 70, 8, 6, 1, 239, 59, 8, 2, 1, 239, - 59, 52, 1, 196, 45, 8, 2, 1, 235, 79, 8, 2, 1, 218, 210, 8, 2, 1, 215, - 125, 8, 2, 1, 212, 139, 8, 2, 1, 206, 49, 8, 1, 2, 6, 206, 49, 8, 2, 1, - 197, 76, 8, 2, 1, 196, 120, 8, 6, 1, 223, 117, 238, 129, 8, 2, 1, 223, - 117, 238, 129, 8, 6, 1, 223, 117, 233, 177, 8, 2, 1, 223, 117, 233, 177, - 8, 6, 1, 223, 117, 232, 53, 8, 6, 1, 154, 223, 117, 232, 53, 8, 2, 1, - 154, 223, 117, 232, 53, 8, 6, 1, 154, 172, 8, 2, 1, 154, 172, 8, 6, 1, - 223, 117, 146, 8, 2, 1, 223, 117, 146, 8, 6, 1, 223, 117, 206, 9, 8, 2, - 1, 223, 117, 206, 9, 8, 6, 1, 223, 117, 200, 43, 8, 2, 1, 223, 117, 200, - 43, 52, 1, 130, 243, 4, 252, 62, 52, 1, 242, 85, 52, 1, 203, 41, 234, 45, - 56, 8, 6, 1, 205, 157, 8, 2, 1, 205, 157, 8, 6, 1, 154, 230, 118, 8, 2, - 1, 220, 145, 4, 207, 25, 228, 253, 23, 248, 233, 8, 1, 202, 164, 236, - 142, 8, 6, 1, 215, 64, 4, 237, 89, 8, 2, 1, 215, 64, 4, 237, 89, 8, 6, 1, - 247, 196, 4, 164, 8, 2, 1, 247, 196, 4, 164, 8, 2, 1, 247, 196, 4, 210, - 194, 102, 8, 2, 1, 230, 119, 4, 210, 194, 102, 8, 6, 1, 78, 4, 232, 190, - 8, 2, 1, 78, 4, 232, 190, 8, 6, 1, 233, 178, 4, 106, 8, 2, 1, 233, 178, - 4, 106, 8, 6, 1, 195, 15, 252, 27, 8, 2, 1, 195, 15, 252, 27, 8, 6, 1, - 195, 15, 211, 153, 8, 2, 1, 195, 15, 211, 153, 8, 6, 1, 195, 15, 196, - 152, 8, 2, 1, 195, 15, 196, 152, 8, 6, 1, 232, 54, 4, 211, 174, 106, 8, - 2, 1, 232, 54, 4, 211, 174, 106, 8, 6, 1, 222, 155, 4, 211, 174, 106, 8, - 2, 1, 222, 155, 4, 211, 174, 106, 8, 6, 1, 215, 64, 4, 211, 174, 106, 8, - 2, 1, 215, 64, 4, 211, 174, 106, 8, 6, 1, 207, 224, 4, 211, 174, 106, 8, - 2, 1, 207, 224, 4, 211, 174, 106, 8, 6, 1, 206, 10, 4, 211, 174, 106, 8, - 2, 1, 206, 10, 4, 211, 174, 106, 8, 6, 1, 230, 119, 4, 102, 8, 6, 1, 207, - 19, 211, 79, 71, 8, 6, 1, 27, 232, 53, 8, 6, 1, 220, 145, 4, 248, 233, 8, - 6, 1, 2, 6, 68, 8, 1, 2, 6, 208, 106, 8, 6, 1, 154, 222, 154, 8, 6, 1, - 154, 200, 43, 8, 6, 1, 223, 38, 4, 238, 214, 8, 6, 1, 243, 97, 8, 6, 1, - 248, 214, 8, 2, 1, 248, 214, 8, 6, 1, 211, 112, 8, 2, 1, 211, 112, 8, 6, - 1, 126, 4, 106, 8, 2, 1, 126, 4, 106, 8, 6, 1, 231, 13, 65, 8, 2, 1, 231, - 13, 65, 8, 6, 1, 231, 13, 68, 8, 2, 1, 231, 13, 68, 8, 6, 1, 231, 13, 66, - 8, 2, 1, 231, 13, 66, 8, 6, 1, 39, 209, 51, 74, 8, 2, 1, 39, 209, 51, 74, - 8, 6, 1, 251, 114, 193, 224, 8, 2, 1, 251, 114, 193, 224, 8, 6, 1, 247, - 196, 4, 210, 194, 102, 8, 6, 1, 206, 10, 4, 102, 8, 6, 1, 191, 167, 4, - 210, 194, 102, 8, 6, 1, 238, 130, 4, 203, 41, 201, 29, 211, 1, 8, 2, 1, - 238, 130, 4, 203, 41, 201, 29, 211, 1, 8, 6, 1, 206, 10, 4, 203, 41, 201, - 29, 211, 1, 8, 2, 1, 206, 10, 4, 203, 41, 201, 29, 211, 1, 8, 6, 1, 242, - 221, 223, 117, 232, 53, 8, 2, 1, 242, 221, 223, 117, 232, 53, 8, 2, 1, - 55, 198, 233, 8, 2, 1, 55, 192, 238, 8, 6, 1, 82, 205, 80, 206, 9, 8, 2, - 1, 82, 205, 80, 206, 9, 8, 6, 1, 202, 196, 206, 9, 8, 2, 1, 202, 196, - 206, 9, 52, 1, 6, 247, 195, 52, 1, 6, 233, 177, 52, 1, 6, 208, 106, 8, 6, - 1, 207, 19, 132, 230, 118, 8, 2, 1, 207, 19, 132, 230, 118, 8, 234, 52, - 1, 202, 207, 68, 52, 1, 6, 230, 119, 4, 106, 52, 1, 2, 34, 211, 153, 8, - 1, 2, 6, 154, 218, 170, 8, 234, 52, 1, 207, 19, 233, 177, 8, 234, 52, 1, - 207, 19, 210, 238, 8, 234, 52, 1, 223, 95, 218, 170, 8, 234, 52, 1, 228, - 76, 219, 12, 8, 234, 52, 1, 251, 16, 218, 170, 200, 124, 214, 240, 1, 65, - 200, 124, 214, 240, 1, 68, 200, 124, 214, 240, 3, 235, 56, 200, 124, 214, - 240, 1, 66, 200, 124, 214, 240, 1, 71, 200, 124, 214, 240, 1, 74, 200, - 124, 214, 240, 3, 230, 13, 200, 124, 214, 240, 1, 221, 69, 200, 124, 214, - 240, 1, 221, 185, 200, 124, 214, 240, 1, 231, 5, 200, 124, 214, 240, 1, - 231, 65, 200, 124, 214, 240, 3, 251, 73, 200, 124, 214, 240, 1, 242, 101, - 200, 124, 214, 240, 1, 243, 70, 200, 124, 214, 240, 1, 222, 203, 200, - 124, 214, 240, 1, 222, 248, 200, 124, 214, 240, 1, 197, 109, 200, 124, - 214, 240, 1, 197, 115, 200, 124, 214, 240, 1, 237, 163, 200, 124, 214, - 240, 1, 237, 172, 200, 124, 214, 240, 1, 159, 200, 124, 214, 240, 1, 198, - 241, 200, 124, 214, 240, 1, 236, 176, 200, 124, 214, 240, 1, 237, 50, - 200, 124, 214, 240, 1, 213, 45, 200, 124, 214, 240, 1, 209, 75, 200, 124, - 214, 240, 1, 209, 201, 200, 124, 214, 240, 1, 248, 113, 200, 124, 214, - 240, 1, 248, 194, 200, 124, 214, 240, 1, 216, 14, 200, 124, 214, 240, 1, - 206, 163, 200, 124, 214, 240, 1, 219, 45, 200, 124, 214, 240, 1, 206, 96, - 200, 124, 214, 240, 1, 202, 47, 200, 124, 214, 240, 1, 229, 25, 200, 124, - 214, 240, 18, 3, 65, 200, 124, 214, 240, 18, 3, 68, 200, 124, 214, 240, - 18, 3, 66, 200, 124, 214, 240, 18, 3, 71, 200, 124, 214, 240, 18, 3, 211, - 89, 200, 124, 214, 240, 209, 65, 217, 57, 200, 124, 214, 240, 209, 65, - 217, 56, 200, 124, 214, 240, 209, 65, 217, 55, 200, 124, 214, 240, 209, - 65, 217, 54, 200, 124, 214, 240, 3, 251, 159, 230, 13, 186, 223, 148, - 232, 120, 91, 208, 24, 186, 223, 148, 232, 120, 91, 230, 72, 186, 223, - 148, 232, 120, 115, 208, 22, 186, 223, 148, 232, 120, 91, 202, 129, 186, - 223, 148, 232, 120, 91, 234, 161, 186, 223, 148, 232, 120, 115, 202, 126, - 186, 223, 148, 208, 25, 77, 186, 223, 148, 209, 109, 77, 186, 223, 148, - 206, 37, 77, 186, 223, 148, 208, 27, 77, 209, 226, 1, 155, 209, 226, 1, - 221, 217, 209, 226, 1, 231, 242, 209, 226, 1, 214, 70, 209, 226, 1, 247, - 162, 209, 226, 1, 247, 3, 209, 226, 1, 223, 34, 209, 226, 1, 212, 103, - 209, 226, 1, 190, 190, 209, 226, 1, 199, 49, 209, 226, 1, 238, 34, 209, - 226, 1, 181, 209, 226, 1, 168, 209, 226, 1, 209, 230, 209, 226, 1, 249, - 155, 209, 226, 1, 174, 209, 226, 1, 197, 168, 209, 226, 1, 197, 157, 209, - 226, 1, 235, 37, 209, 226, 1, 193, 190, 209, 226, 1, 191, 71, 209, 226, - 1, 191, 123, 209, 226, 1, 2, 65, 209, 226, 1, 170, 209, 226, 1, 165, 209, - 226, 1, 173, 209, 226, 1, 203, 166, 209, 226, 1, 188, 209, 226, 1, 140, - 209, 226, 1, 65, 209, 226, 1, 68, 209, 226, 1, 66, 209, 226, 1, 71, 209, - 226, 1, 74, 209, 226, 1, 208, 98, 209, 226, 1, 192, 220, 209, 226, 1, - 233, 111, 209, 226, 1, 231, 129, 209, 226, 1, 234, 190, 209, 226, 200, - 240, 1, 193, 190, 209, 226, 200, 240, 1, 170, 209, 226, 1, 197, 132, 209, - 226, 1, 197, 120, 209, 226, 1, 237, 193, 209, 226, 1, 213, 81, 209, 226, - 1, 251, 159, 170, 209, 226, 1, 195, 19, 203, 166, 209, 226, 1, 195, 20, - 140, 209, 226, 1, 250, 202, 233, 111, 209, 226, 200, 240, 1, 165, 209, - 226, 200, 185, 1, 165, 209, 226, 1, 247, 121, 209, 226, 202, 171, 229, - 238, 77, 209, 226, 55, 229, 238, 77, 209, 226, 87, 203, 158, 209, 226, - 87, 55, 203, 158, 205, 112, 3, 251, 73, 205, 112, 3, 195, 35, 205, 112, - 1, 65, 205, 112, 1, 252, 208, 205, 112, 1, 68, 205, 112, 1, 223, 201, - 205, 112, 1, 66, 205, 112, 1, 196, 30, 205, 112, 1, 117, 146, 205, 112, - 1, 117, 206, 111, 205, 112, 1, 117, 172, 205, 112, 1, 117, 219, 76, 205, - 112, 1, 71, 205, 112, 1, 234, 190, 205, 112, 1, 251, 238, 205, 112, 1, - 74, 205, 112, 1, 211, 89, 205, 112, 1, 250, 165, 205, 112, 1, 155, 205, - 112, 1, 221, 217, 205, 112, 1, 231, 242, 205, 112, 1, 231, 93, 205, 112, - 1, 214, 70, 205, 112, 1, 247, 162, 205, 112, 1, 247, 3, 205, 112, 1, 223, - 34, 205, 112, 1, 222, 254, 205, 112, 1, 212, 103, 205, 112, 1, 197, 132, - 205, 112, 1, 197, 120, 205, 112, 1, 237, 193, 205, 112, 1, 237, 177, 205, - 112, 1, 213, 81, 205, 112, 1, 190, 190, 205, 112, 1, 199, 49, 205, 112, - 1, 238, 34, 205, 112, 1, 237, 70, 205, 112, 1, 181, 205, 112, 1, 168, - 205, 112, 1, 209, 230, 205, 112, 1, 249, 155, 205, 112, 1, 248, 205, 205, - 112, 1, 174, 205, 112, 1, 170, 205, 112, 1, 165, 205, 112, 1, 173, 205, - 112, 1, 195, 188, 205, 112, 1, 203, 166, 205, 112, 1, 201, 176, 205, 112, - 1, 188, 205, 112, 1, 140, 205, 112, 1, 219, 75, 205, 112, 120, 3, 230, - 91, 205, 112, 18, 3, 252, 208, 205, 112, 18, 3, 68, 205, 112, 18, 3, 223, - 201, 205, 112, 18, 3, 66, 205, 112, 18, 3, 196, 30, 205, 112, 18, 3, 117, - 146, 205, 112, 18, 3, 117, 206, 111, 205, 112, 18, 3, 117, 172, 205, 112, - 18, 3, 117, 219, 76, 205, 112, 18, 3, 71, 205, 112, 18, 3, 234, 190, 205, - 112, 18, 3, 251, 238, 205, 112, 18, 3, 74, 205, 112, 18, 3, 211, 89, 205, - 112, 18, 3, 250, 165, 205, 112, 3, 195, 40, 205, 112, 3, 247, 121, 205, - 112, 237, 240, 205, 112, 55, 237, 240, 205, 112, 17, 191, 77, 205, 112, - 17, 107, 205, 112, 17, 109, 205, 112, 17, 138, 205, 112, 17, 134, 205, - 112, 17, 150, 205, 112, 17, 169, 205, 112, 17, 175, 205, 112, 17, 171, - 205, 112, 17, 178, 33, 100, 17, 191, 77, 33, 100, 17, 107, 33, 100, 17, - 109, 33, 100, 17, 138, 33, 100, 17, 134, 33, 100, 17, 150, 33, 100, 17, - 169, 33, 100, 17, 175, 33, 100, 17, 171, 33, 100, 17, 178, 33, 100, 1, - 65, 33, 100, 1, 66, 33, 100, 1, 155, 33, 100, 1, 181, 33, 100, 1, 168, - 33, 100, 1, 165, 33, 100, 1, 195, 69, 33, 100, 3, 250, 147, 100, 3, 201, - 247, 247, 121, 100, 3, 247, 122, 195, 40, 100, 3, 55, 247, 122, 195, 40, - 100, 3, 247, 122, 109, 100, 3, 247, 122, 138, 100, 3, 247, 122, 250, 147, - 100, 3, 208, 136, 100, 231, 206, 233, 5, 100, 247, 98, 100, 229, 229, - 100, 3, 202, 211, 100, 223, 26, 211, 115, 100, 1, 250, 133, 100, 18, 3, - 250, 133, 222, 30, 219, 149, 17, 191, 77, 222, 30, 219, 149, 17, 107, - 222, 30, 219, 149, 17, 109, 222, 30, 219, 149, 17, 138, 222, 30, 219, - 149, 17, 134, 222, 30, 219, 149, 17, 150, 222, 30, 219, 149, 17, 169, - 222, 30, 219, 149, 17, 175, 222, 30, 219, 149, 17, 171, 222, 30, 219, - 149, 17, 178, 222, 30, 219, 149, 1, 155, 222, 30, 219, 149, 1, 221, 217, - 222, 30, 219, 149, 1, 231, 242, 222, 30, 219, 149, 1, 214, 70, 222, 30, - 219, 149, 1, 188, 222, 30, 219, 149, 1, 203, 166, 222, 30, 219, 149, 1, - 191, 123, 222, 30, 219, 149, 1, 212, 103, 222, 30, 219, 149, 1, 190, 190, - 222, 30, 219, 149, 1, 228, 166, 222, 30, 219, 149, 1, 181, 222, 30, 219, - 149, 1, 168, 222, 30, 219, 149, 1, 209, 230, 222, 30, 219, 149, 1, 174, - 222, 30, 219, 149, 1, 238, 34, 222, 30, 219, 149, 1, 249, 155, 222, 30, - 219, 149, 1, 165, 222, 30, 219, 149, 1, 170, 222, 30, 219, 149, 1, 173, - 222, 30, 219, 149, 1, 193, 190, 222, 30, 219, 149, 1, 199, 49, 222, 30, - 219, 149, 1, 140, 222, 30, 219, 149, 1, 195, 188, 222, 30, 219, 149, 1, - 247, 162, 222, 30, 219, 149, 1, 65, 222, 30, 219, 149, 1, 211, 153, 222, - 30, 219, 149, 1, 68, 222, 30, 219, 149, 1, 211, 89, 222, 30, 219, 149, - 18, 196, 152, 222, 30, 219, 149, 18, 71, 222, 30, 219, 149, 18, 66, 222, - 30, 219, 149, 18, 234, 190, 222, 30, 219, 149, 18, 74, 222, 30, 219, 149, - 163, 209, 92, 222, 30, 219, 149, 163, 247, 137, 222, 30, 219, 149, 163, - 247, 138, 209, 92, 222, 30, 219, 149, 3, 238, 149, 222, 30, 219, 149, 3, - 202, 231, 207, 68, 1, 155, 207, 68, 1, 231, 242, 207, 68, 1, 214, 70, - 207, 68, 1, 190, 190, 207, 68, 1, 238, 34, 207, 68, 1, 181, 207, 68, 1, - 168, 207, 68, 1, 249, 155, 207, 68, 1, 174, 207, 68, 1, 247, 162, 207, - 68, 1, 223, 34, 207, 68, 1, 212, 103, 207, 68, 1, 188, 207, 68, 1, 165, - 207, 68, 1, 173, 207, 68, 1, 170, 207, 68, 1, 193, 190, 207, 68, 1, 140, - 207, 68, 1, 217, 13, 207, 68, 1, 214, 49, 207, 68, 1, 214, 164, 207, 68, - 1, 212, 68, 207, 68, 1, 65, 207, 68, 18, 3, 68, 207, 68, 18, 3, 66, 207, - 68, 18, 3, 71, 207, 68, 18, 3, 251, 238, 207, 68, 18, 3, 74, 207, 68, 18, - 3, 250, 165, 207, 68, 18, 3, 233, 244, 207, 68, 18, 3, 234, 219, 207, 68, - 120, 3, 214, 72, 207, 68, 120, 3, 215, 63, 207, 68, 120, 3, 146, 207, 68, - 120, 3, 230, 118, 207, 68, 195, 40, 207, 68, 205, 55, 77, 30, 147, 198, - 164, 30, 147, 198, 163, 30, 147, 198, 161, 30, 147, 198, 166, 30, 147, - 206, 235, 30, 147, 206, 219, 30, 147, 206, 214, 30, 147, 206, 216, 30, - 147, 206, 232, 30, 147, 206, 225, 30, 147, 206, 218, 30, 147, 206, 237, - 30, 147, 206, 220, 30, 147, 206, 239, 30, 147, 206, 236, 30, 147, 216, - 75, 30, 147, 216, 66, 30, 147, 216, 69, 30, 147, 209, 156, 30, 147, 209, - 167, 30, 147, 209, 168, 30, 147, 201, 160, 30, 147, 223, 214, 30, 147, - 223, 221, 30, 147, 201, 171, 30, 147, 201, 158, 30, 147, 209, 210, 30, - 147, 229, 139, 30, 147, 201, 155, 223, 18, 3, 210, 145, 223, 18, 3, 247, - 41, 223, 18, 3, 219, 248, 223, 18, 3, 193, 71, 223, 18, 1, 65, 223, 18, - 1, 228, 76, 222, 34, 223, 18, 1, 68, 223, 18, 1, 223, 201, 223, 18, 1, - 66, 223, 18, 1, 210, 223, 247, 11, 223, 18, 1, 214, 71, 219, 205, 223, - 18, 1, 214, 71, 219, 206, 207, 133, 223, 18, 1, 71, 223, 18, 1, 251, 238, - 223, 18, 1, 74, 223, 18, 1, 155, 223, 18, 1, 222, 144, 205, 125, 223, 18, - 1, 222, 144, 215, 109, 223, 18, 1, 231, 242, 223, 18, 1, 231, 243, 215, - 109, 223, 18, 1, 214, 70, 223, 18, 1, 247, 162, 223, 18, 1, 247, 163, - 215, 109, 223, 18, 1, 223, 34, 223, 18, 1, 212, 104, 215, 109, 223, 18, - 1, 223, 35, 217, 118, 223, 18, 1, 212, 103, 223, 18, 1, 197, 132, 223, - 18, 1, 197, 133, 217, 118, 223, 18, 1, 237, 193, 223, 18, 1, 237, 194, - 217, 118, 223, 18, 1, 215, 9, 215, 109, 223, 18, 1, 190, 190, 223, 18, 1, - 199, 252, 215, 109, 223, 18, 1, 238, 34, 223, 18, 1, 238, 35, 217, 118, - 223, 18, 1, 181, 223, 18, 1, 168, 223, 18, 1, 210, 223, 215, 109, 223, - 18, 1, 249, 155, 223, 18, 1, 249, 156, 215, 109, 223, 18, 1, 174, 223, - 18, 1, 170, 223, 18, 1, 165, 223, 18, 1, 207, 188, 251, 248, 223, 18, 1, - 173, 223, 18, 1, 193, 190, 223, 18, 1, 205, 208, 215, 109, 223, 18, 1, - 205, 208, 217, 118, 223, 18, 1, 188, 223, 18, 1, 140, 223, 18, 3, 247, - 42, 199, 100, 223, 18, 18, 3, 199, 175, 223, 18, 18, 3, 198, 84, 223, 18, - 18, 3, 192, 250, 223, 18, 18, 3, 192, 251, 216, 200, 223, 18, 18, 3, 200, - 208, 223, 18, 18, 3, 200, 209, 216, 187, 223, 18, 18, 3, 199, 201, 223, - 18, 18, 3, 236, 232, 215, 108, 223, 18, 18, 3, 210, 18, 223, 18, 120, 3, - 221, 246, 223, 18, 120, 3, 210, 33, 223, 18, 120, 3, 247, 147, 223, 18, - 210, 159, 223, 18, 45, 207, 41, 223, 18, 50, 207, 41, 223, 18, 210, 211, - 251, 126, 223, 18, 210, 211, 217, 139, 223, 18, 210, 211, 218, 214, 223, - 18, 210, 211, 193, 64, 223, 18, 210, 211, 210, 160, 223, 18, 210, 211, - 219, 106, 223, 18, 210, 211, 218, 206, 223, 18, 210, 211, 252, 38, 223, - 18, 210, 211, 252, 39, 252, 38, 223, 18, 210, 211, 209, 121, 223, 18, - 154, 210, 211, 209, 121, 223, 18, 210, 155, 223, 18, 17, 191, 77, 223, - 18, 17, 107, 223, 18, 17, 109, 223, 18, 17, 138, 223, 18, 17, 134, 223, - 18, 17, 150, 223, 18, 17, 169, 223, 18, 17, 175, 223, 18, 17, 171, 223, - 18, 17, 178, 223, 18, 210, 211, 198, 127, 197, 73, 223, 18, 210, 211, - 223, 66, 80, 1, 203, 140, 231, 93, 80, 1, 203, 140, 247, 3, 80, 1, 203, - 140, 222, 254, 80, 1, 203, 140, 213, 81, 80, 1, 203, 140, 248, 205, 80, - 3, 203, 140, 205, 109, 80, 52, 1, 203, 140, 207, 87, 80, 1, 54, 220, 97, - 212, 103, 80, 1, 54, 220, 97, 233, 111, 80, 1, 54, 220, 97, 231, 242, 80, - 1, 54, 220, 97, 231, 93, 80, 1, 54, 220, 97, 223, 34, 80, 1, 54, 220, 97, - 222, 254, 80, 1, 54, 220, 97, 237, 193, 80, 1, 54, 220, 97, 237, 177, 80, - 1, 54, 220, 97, 213, 81, 80, 54, 220, 97, 17, 191, 77, 80, 54, 220, 97, - 17, 107, 80, 54, 220, 97, 17, 109, 80, 54, 220, 97, 17, 138, 80, 54, 220, - 97, 17, 134, 80, 54, 220, 97, 17, 150, 80, 54, 220, 97, 17, 169, 80, 54, - 220, 97, 17, 175, 80, 54, 220, 97, 17, 171, 80, 54, 220, 97, 17, 178, 80, - 1, 54, 220, 97, 219, 75, 80, 1, 54, 220, 97, 238, 34, 80, 1, 54, 220, 97, - 237, 70, 80, 1, 54, 220, 97, 249, 155, 80, 1, 54, 220, 97, 248, 205, 246, - 252, 1, 65, 246, 252, 1, 68, 246, 252, 1, 66, 246, 252, 1, 71, 246, 252, - 1, 251, 238, 246, 252, 1, 74, 246, 252, 1, 155, 246, 252, 1, 221, 217, - 246, 252, 1, 231, 242, 246, 252, 1, 231, 93, 246, 252, 1, 213, 235, 246, - 252, 1, 214, 70, 246, 252, 1, 247, 3, 246, 252, 1, 243, 100, 246, 252, 1, - 223, 34, 246, 252, 1, 222, 254, 246, 252, 1, 213, 223, 246, 252, 1, 213, - 226, 246, 252, 1, 213, 224, 246, 252, 1, 190, 190, 246, 252, 1, 199, 49, - 246, 252, 1, 238, 34, 246, 252, 1, 237, 70, 246, 252, 1, 212, 146, 246, - 252, 1, 181, 246, 252, 1, 237, 193, 246, 252, 1, 168, 246, 252, 1, 208, - 252, 246, 252, 1, 209, 230, 246, 252, 1, 249, 155, 246, 252, 1, 248, 205, - 246, 252, 1, 215, 145, 246, 252, 1, 174, 246, 252, 1, 249, 55, 246, 252, - 1, 170, 246, 252, 1, 165, 246, 252, 1, 173, 246, 252, 1, 195, 188, 246, - 252, 1, 201, 176, 246, 252, 1, 188, 246, 252, 1, 140, 246, 252, 18, 3, - 252, 208, 246, 252, 18, 3, 68, 246, 252, 18, 3, 223, 201, 246, 252, 18, - 3, 234, 168, 246, 252, 18, 3, 66, 246, 252, 18, 3, 211, 153, 246, 252, - 18, 3, 74, 246, 252, 18, 3, 251, 238, 246, 252, 18, 3, 250, 165, 246, - 252, 18, 3, 196, 152, 246, 252, 120, 3, 170, 246, 252, 120, 3, 165, 246, - 252, 120, 3, 173, 246, 252, 120, 3, 193, 190, 246, 252, 1, 53, 222, 154, - 246, 252, 1, 53, 232, 53, 246, 252, 1, 53, 214, 72, 246, 252, 120, 3, 53, - 214, 72, 246, 252, 1, 53, 247, 5, 246, 252, 1, 53, 200, 43, 246, 252, 1, - 53, 215, 63, 246, 252, 1, 53, 210, 238, 246, 252, 1, 53, 192, 159, 246, - 252, 1, 53, 146, 246, 252, 1, 53, 172, 246, 252, 1, 53, 201, 179, 246, - 252, 120, 3, 53, 218, 170, 246, 252, 120, 3, 53, 230, 118, 246, 252, 17, - 191, 77, 246, 252, 17, 107, 246, 252, 17, 109, 246, 252, 17, 138, 246, - 252, 17, 134, 246, 252, 17, 150, 246, 252, 17, 169, 246, 252, 17, 175, - 246, 252, 17, 171, 246, 252, 17, 178, 246, 252, 208, 154, 201, 218, 246, - 252, 208, 154, 237, 240, 246, 252, 208, 154, 55, 237, 240, 246, 252, 208, - 154, 197, 225, 237, 240, 80, 1, 221, 208, 231, 242, 80, 1, 221, 208, 247, - 162, 80, 1, 221, 208, 247, 3, 80, 1, 221, 208, 223, 34, 80, 1, 221, 208, - 222, 254, 80, 1, 221, 208, 212, 103, 80, 1, 221, 208, 197, 132, 80, 1, - 221, 208, 197, 120, 80, 1, 221, 208, 237, 193, 80, 1, 221, 208, 237, 177, - 80, 1, 221, 208, 237, 70, 80, 1, 221, 208, 181, 80, 1, 221, 208, 188, 80, - 1, 221, 208, 140, 80, 1, 221, 208, 229, 179, 80, 1, 221, 208, 233, 111, - 80, 52, 1, 221, 208, 207, 87, 80, 1, 221, 208, 192, 220, 80, 1, 221, 208, - 191, 123, 80, 1, 221, 208, 165, 80, 219, 30, 221, 208, 211, 181, 80, 219, - 30, 221, 208, 208, 48, 80, 219, 30, 221, 208, 229, 80, 80, 16, 251, 224, - 233, 217, 80, 16, 251, 224, 107, 80, 16, 251, 224, 109, 80, 1, 251, 224, - 165, 80, 3, 210, 141, 222, 64, 198, 79, 80, 3, 54, 220, 97, 198, 77, 80, - 3, 54, 220, 97, 198, 74, 80, 1, 202, 239, 210, 191, 247, 3, 80, 1, 202, - 239, 210, 191, 203, 166, 54, 195, 59, 1, 130, 221, 69, 54, 195, 59, 1, - 137, 221, 69, 54, 195, 59, 1, 130, 221, 185, 54, 195, 59, 1, 137, 221, - 185, 54, 195, 59, 1, 130, 221, 194, 54, 195, 59, 1, 137, 221, 194, 54, - 195, 59, 1, 130, 231, 5, 54, 195, 59, 1, 137, 231, 5, 54, 195, 59, 1, - 130, 213, 251, 54, 195, 59, 1, 137, 213, 251, 54, 195, 59, 1, 130, 242, - 101, 54, 195, 59, 1, 137, 242, 101, 54, 195, 59, 1, 130, 243, 70, 54, - 195, 59, 1, 137, 243, 70, 54, 195, 59, 1, 130, 202, 47, 54, 195, 59, 1, - 137, 202, 47, 54, 195, 59, 1, 130, 212, 67, 54, 195, 59, 1, 137, 212, 67, - 54, 195, 59, 1, 130, 236, 176, 54, 195, 59, 1, 137, 236, 176, 54, 195, - 59, 1, 130, 159, 54, 195, 59, 1, 137, 159, 54, 195, 59, 1, 130, 198, 241, - 54, 195, 59, 1, 137, 198, 241, 54, 195, 59, 1, 130, 213, 45, 54, 195, 59, - 1, 137, 213, 45, 54, 195, 59, 1, 130, 248, 113, 54, 195, 59, 1, 137, 248, - 113, 54, 195, 59, 1, 130, 209, 75, 54, 195, 59, 1, 137, 209, 75, 54, 195, - 59, 1, 130, 209, 201, 54, 195, 59, 1, 137, 209, 201, 54, 195, 59, 1, 130, - 232, 177, 54, 195, 59, 1, 137, 232, 177, 54, 195, 59, 1, 130, 216, 14, - 54, 195, 59, 1, 137, 216, 14, 54, 195, 59, 1, 130, 192, 12, 54, 195, 59, - 1, 137, 192, 12, 54, 195, 59, 1, 130, 206, 163, 54, 195, 59, 1, 137, 206, - 163, 54, 195, 59, 1, 130, 219, 45, 54, 195, 59, 1, 137, 219, 45, 54, 195, - 59, 1, 130, 195, 24, 54, 195, 59, 1, 137, 195, 24, 54, 195, 59, 1, 130, - 229, 25, 54, 195, 59, 1, 137, 229, 25, 54, 195, 59, 1, 130, 74, 54, 195, - 59, 1, 137, 74, 54, 195, 59, 217, 115, 222, 85, 54, 195, 59, 18, 252, - 208, 54, 195, 59, 18, 68, 54, 195, 59, 18, 196, 152, 54, 195, 59, 18, 66, - 54, 195, 59, 18, 71, 54, 195, 59, 18, 74, 54, 195, 59, 217, 115, 221, - 188, 54, 195, 59, 18, 228, 37, 54, 195, 59, 18, 196, 151, 54, 195, 59, - 18, 196, 168, 54, 195, 59, 18, 250, 163, 54, 195, 59, 18, 250, 133, 54, - 195, 59, 18, 251, 134, 54, 195, 59, 18, 251, 151, 54, 195, 59, 163, 217, - 115, 234, 149, 54, 195, 59, 163, 217, 115, 212, 145, 54, 195, 59, 163, - 217, 115, 198, 241, 54, 195, 59, 163, 217, 115, 202, 19, 54, 195, 59, 16, - 221, 46, 54, 195, 59, 16, 212, 145, 54, 195, 59, 16, 205, 153, 54, 195, - 59, 16, 229, 26, 229, 12, 54, 195, 59, 16, 221, 57, 221, 56, 216, 207, - 217, 20, 1, 71, 216, 207, 217, 20, 1, 74, 216, 207, 217, 20, 1, 247, 3, - 216, 207, 217, 20, 1, 212, 103, 216, 207, 217, 20, 1, 197, 132, 216, 207, - 217, 20, 1, 197, 120, 216, 207, 217, 20, 1, 237, 193, 216, 207, 217, 20, - 1, 237, 177, 216, 207, 217, 20, 1, 213, 81, 216, 207, 217, 20, 1, 203, - 166, 216, 207, 217, 20, 1, 201, 176, 216, 207, 217, 20, 18, 3, 223, 201, - 216, 207, 217, 20, 18, 3, 196, 30, 216, 207, 217, 20, 18, 3, 252, 172, - 216, 207, 217, 20, 18, 3, 250, 165, 216, 207, 217, 20, 18, 3, 252, 164, - 216, 207, 217, 20, 243, 118, 216, 207, 217, 20, 251, 244, 221, 175, 216, - 207, 217, 20, 251, 102, 216, 207, 217, 20, 5, 207, 47, 77, 216, 207, 217, - 20, 193, 23, 207, 47, 77, 216, 207, 217, 20, 18, 3, 195, 35, 216, 207, - 217, 20, 195, 40, 36, 5, 197, 113, 36, 5, 197, 116, 36, 5, 197, 119, 36, - 5, 197, 117, 36, 5, 197, 118, 36, 5, 197, 115, 36, 5, 237, 171, 36, 5, - 237, 173, 36, 5, 237, 176, 36, 5, 237, 174, 36, 5, 237, 175, 36, 5, 237, - 172, 36, 5, 235, 24, 36, 5, 235, 28, 36, 5, 235, 36, 36, 5, 235, 33, 36, - 5, 235, 34, 36, 5, 235, 25, 36, 5, 247, 58, 36, 5, 247, 52, 36, 5, 247, - 54, 36, 5, 247, 57, 36, 5, 247, 55, 36, 5, 247, 56, 36, 5, 247, 53, 36, - 5, 249, 55, 36, 5, 249, 34, 36, 5, 249, 46, 36, 5, 249, 54, 36, 5, 249, - 49, 36, 5, 249, 50, 36, 5, 249, 38, 8, 2, 1, 249, 84, 251, 162, 8, 2, 1, - 42, 207, 17, 8, 2, 1, 248, 137, 71, 8, 2, 1, 249, 84, 71, 8, 2, 1, 235, - 17, 4, 232, 190, 8, 2, 1, 219, 191, 233, 177, 8, 2, 1, 27, 232, 54, 4, - 238, 214, 8, 2, 1, 220, 145, 4, 223, 95, 219, 247, 206, 9, 8, 2, 1, 220, - 145, 4, 55, 82, 198, 152, 8, 2, 1, 220, 145, 4, 82, 206, 189, 8, 2, 1, - 218, 171, 4, 238, 214, 8, 2, 1, 215, 64, 4, 238, 214, 8, 2, 1, 234, 91, - 4, 238, 214, 8, 2, 1, 248, 137, 74, 8, 2, 1, 248, 137, 187, 4, 106, 8, 2, - 1, 211, 79, 187, 4, 106, 8, 2, 1, 223, 95, 211, 153, 8, 2, 1, 154, 211, - 154, 4, 106, 8, 2, 1, 154, 211, 154, 4, 228, 243, 106, 8, 2, 1, 154, 187, - 211, 74, 8, 2, 1, 154, 187, 211, 75, 4, 106, 8, 2, 1, 201, 69, 146, 8, 1, - 2, 6, 207, 224, 4, 50, 219, 214, 8, 2, 1, 207, 224, 193, 51, 230, 33, 8, - 2, 1, 55, 146, 8, 2, 1, 207, 224, 4, 238, 214, 8, 2, 1, 55, 207, 224, 4, - 238, 214, 8, 2, 1, 27, 146, 8, 2, 1, 27, 207, 224, 4, 206, 189, 8, 2, 1, - 249, 74, 234, 14, 8, 2, 1, 126, 4, 203, 41, 50, 219, 214, 8, 2, 1, 126, - 249, 90, 4, 203, 41, 50, 219, 214, 8, 2, 1, 196, 139, 8, 2, 1, 154, 196, - 139, 8, 2, 1, 126, 4, 45, 102, 8, 2, 1, 243, 97, 8, 2, 1, 243, 98, 4, - 130, 50, 206, 189, 8, 2, 1, 243, 98, 4, 130, 45, 204, 6, 8, 2, 1, 192, - 236, 4, 130, 50, 206, 189, 8, 2, 1, 192, 236, 4, 180, 45, 219, 214, 8, 2, - 1, 192, 236, 4, 180, 45, 219, 215, 23, 130, 50, 206, 189, 8, 2, 1, 192, - 236, 4, 180, 45, 219, 215, 4, 204, 6, 8, 2, 1, 192, 160, 4, 203, 41, 50, - 219, 214, 52, 248, 39, 4, 223, 95, 248, 38, 52, 1, 2, 229, 198, 52, 1, 2, - 220, 145, 4, 223, 95, 219, 247, 206, 9, 52, 1, 2, 220, 145, 4, 82, 198, - 152, 52, 1, 2, 126, 4, 45, 102, 8, 2, 1, 205, 175, 192, 95, 8, 2, 1, 223, - 83, 71, 8, 2, 1, 211, 79, 211, 153, 8, 2, 1, 196, 82, 8, 2, 1, 223, 95, - 251, 162, 35, 1, 2, 6, 211, 112, 8, 2, 1, 235, 39, 237, 5, 4, 207, 25, - 102, 8, 2, 1, 197, 170, 237, 5, 4, 207, 25, 102, 8, 2, 1, 154, 207, 224, - 4, 82, 198, 152, 52, 1, 2, 154, 193, 224, 52, 1, 45, 199, 228, 52, 1, 50, - 199, 228, 103, 2, 1, 65, 103, 2, 1, 71, 103, 2, 1, 68, 103, 2, 1, 74, - 103, 2, 1, 66, 103, 2, 1, 196, 12, 103, 2, 1, 231, 242, 103, 2, 1, 155, - 103, 2, 1, 231, 167, 103, 2, 1, 231, 55, 103, 2, 1, 231, 5, 103, 2, 1, - 230, 181, 103, 2, 1, 230, 140, 103, 2, 1, 140, 103, 2, 1, 229, 247, 103, - 2, 1, 229, 160, 103, 2, 1, 229, 25, 103, 2, 1, 228, 161, 103, 2, 1, 228, - 128, 103, 2, 1, 173, 103, 2, 1, 219, 240, 103, 2, 1, 219, 148, 103, 2, 1, - 219, 45, 103, 2, 1, 218, 227, 103, 2, 1, 218, 194, 103, 2, 1, 174, 103, - 2, 1, 216, 234, 103, 2, 1, 216, 102, 103, 2, 1, 216, 14, 103, 2, 1, 215, - 157, 103, 2, 1, 181, 103, 2, 1, 229, 49, 103, 2, 1, 214, 239, 103, 2, 1, - 214, 123, 103, 2, 1, 213, 221, 103, 2, 1, 213, 45, 103, 2, 1, 212, 180, - 103, 2, 1, 212, 114, 103, 2, 1, 208, 34, 103, 2, 1, 208, 18, 103, 2, 1, - 208, 11, 103, 2, 1, 208, 1, 103, 2, 1, 207, 246, 103, 2, 1, 207, 244, - 103, 2, 1, 188, 103, 2, 1, 206, 9, 103, 2, 1, 205, 69, 103, 2, 1, 202, - 223, 103, 2, 1, 202, 47, 103, 2, 1, 201, 5, 103, 2, 1, 200, 158, 103, 2, - 1, 238, 34, 103, 2, 1, 190, 190, 103, 2, 1, 237, 148, 103, 2, 1, 199, - 145, 103, 2, 1, 237, 46, 103, 2, 1, 198, 193, 103, 2, 1, 236, 176, 103, - 2, 1, 235, 91, 103, 2, 1, 235, 59, 103, 2, 1, 236, 188, 103, 2, 1, 198, - 115, 103, 2, 1, 198, 114, 103, 2, 1, 198, 103, 103, 2, 1, 198, 102, 103, - 2, 1, 198, 101, 103, 2, 1, 198, 100, 103, 2, 1, 197, 168, 103, 2, 1, 197, - 161, 103, 2, 1, 197, 146, 103, 2, 1, 197, 144, 103, 2, 1, 197, 140, 103, - 2, 1, 197, 139, 103, 2, 1, 193, 190, 103, 2, 1, 193, 125, 103, 2, 1, 193, - 86, 103, 2, 1, 193, 48, 103, 2, 1, 193, 0, 103, 2, 1, 192, 243, 103, 2, - 1, 170, 216, 207, 217, 20, 1, 221, 53, 216, 207, 217, 20, 1, 205, 153, - 216, 207, 217, 20, 1, 220, 98, 216, 207, 217, 20, 1, 216, 25, 216, 207, - 217, 20, 1, 168, 216, 207, 217, 20, 1, 181, 216, 207, 217, 20, 1, 243, - 89, 216, 207, 217, 20, 1, 198, 154, 216, 207, 217, 20, 1, 221, 178, 216, - 207, 217, 20, 1, 213, 241, 216, 207, 217, 20, 1, 198, 232, 216, 207, 217, - 20, 1, 193, 173, 216, 207, 217, 20, 1, 192, 106, 216, 207, 217, 20, 1, - 228, 149, 216, 207, 217, 20, 1, 196, 113, 216, 207, 217, 20, 1, 68, 216, - 207, 217, 20, 1, 209, 224, 216, 207, 217, 20, 1, 250, 177, 216, 207, 217, - 20, 1, 230, 253, 216, 207, 217, 20, 1, 222, 252, 216, 207, 217, 20, 1, - 207, 158, 216, 207, 217, 20, 1, 249, 155, 216, 207, 217, 20, 1, 222, 236, - 216, 207, 217, 20, 1, 237, 3, 216, 207, 217, 20, 1, 231, 62, 216, 207, - 217, 20, 1, 237, 48, 216, 207, 217, 20, 1, 248, 200, 216, 207, 217, 20, - 1, 221, 54, 219, 11, 216, 207, 217, 20, 1, 220, 99, 219, 11, 216, 207, - 217, 20, 1, 216, 26, 219, 11, 216, 207, 217, 20, 1, 210, 223, 219, 11, - 216, 207, 217, 20, 1, 215, 9, 219, 11, 216, 207, 217, 20, 1, 198, 155, - 219, 11, 216, 207, 217, 20, 1, 213, 242, 219, 11, 216, 207, 217, 20, 1, - 228, 76, 219, 11, 216, 207, 217, 20, 18, 3, 211, 104, 216, 207, 217, 20, - 18, 3, 223, 162, 216, 207, 217, 20, 18, 3, 251, 132, 216, 207, 217, 20, - 18, 3, 192, 69, 216, 207, 217, 20, 18, 3, 202, 7, 216, 207, 217, 20, 18, - 3, 196, 110, 216, 207, 217, 20, 18, 3, 243, 116, 216, 207, 217, 20, 18, - 3, 212, 129, 216, 207, 217, 20, 243, 117, 216, 207, 217, 20, 218, 211, - 223, 44, 216, 207, 217, 20, 251, 40, 223, 44, 216, 207, 217, 20, 17, 191, - 77, 216, 207, 217, 20, 17, 107, 216, 207, 217, 20, 17, 109, 216, 207, - 217, 20, 17, 138, 216, 207, 217, 20, 17, 134, 216, 207, 217, 20, 17, 150, - 216, 207, 217, 20, 17, 169, 216, 207, 217, 20, 17, 175, 216, 207, 217, - 20, 17, 171, 216, 207, 217, 20, 17, 178, 30, 222, 176, 212, 5, 30, 222, - 176, 212, 10, 30, 222, 176, 192, 5, 30, 222, 176, 192, 4, 30, 222, 176, - 192, 3, 30, 222, 176, 196, 218, 30, 222, 176, 196, 222, 30, 222, 176, - 191, 219, 30, 222, 176, 191, 215, 30, 222, 176, 233, 243, 30, 222, 176, - 233, 241, 30, 222, 176, 233, 242, 30, 222, 176, 233, 239, 30, 222, 176, - 228, 62, 30, 222, 176, 228, 61, 30, 222, 176, 228, 59, 30, 222, 176, 228, - 60, 30, 222, 176, 228, 65, 30, 222, 176, 228, 58, 30, 222, 176, 228, 57, - 30, 222, 176, 228, 67, 30, 222, 176, 251, 26, 30, 222, 176, 251, 25, 30, - 125, 213, 199, 30, 125, 213, 205, 30, 125, 201, 157, 30, 125, 201, 156, - 30, 125, 198, 163, 30, 125, 198, 161, 30, 125, 198, 160, 30, 125, 198, - 166, 30, 125, 198, 167, 30, 125, 198, 159, 30, 125, 206, 219, 30, 125, - 206, 234, 30, 125, 201, 163, 30, 125, 206, 231, 30, 125, 206, 221, 30, - 125, 206, 223, 30, 125, 206, 210, 30, 125, 206, 211, 30, 125, 222, 70, - 30, 125, 216, 74, 30, 125, 216, 68, 30, 125, 201, 167, 30, 125, 216, 71, - 30, 125, 216, 77, 30, 125, 209, 152, 30, 125, 209, 161, 30, 125, 209, - 165, 30, 125, 201, 165, 30, 125, 209, 155, 30, 125, 209, 169, 30, 125, - 209, 170, 30, 125, 202, 153, 30, 125, 202, 156, 30, 125, 201, 161, 30, - 125, 201, 159, 30, 125, 202, 151, 30, 125, 202, 159, 30, 125, 202, 160, - 30, 125, 202, 145, 30, 125, 202, 158, 30, 125, 210, 149, 30, 125, 210, - 150, 30, 125, 192, 53, 30, 125, 192, 56, 30, 125, 243, 24, 30, 125, 243, - 23, 30, 125, 201, 172, 30, 125, 209, 208, 30, 125, 209, 207, 12, 15, 225, - 192, 12, 15, 225, 191, 12, 15, 225, 190, 12, 15, 225, 189, 12, 15, 225, - 188, 12, 15, 225, 187, 12, 15, 225, 186, 12, 15, 225, 185, 12, 15, 225, - 184, 12, 15, 225, 183, 12, 15, 225, 182, 12, 15, 225, 181, 12, 15, 225, - 180, 12, 15, 225, 179, 12, 15, 225, 178, 12, 15, 225, 177, 12, 15, 225, - 176, 12, 15, 225, 175, 12, 15, 225, 174, 12, 15, 225, 173, 12, 15, 225, - 172, 12, 15, 225, 171, 12, 15, 225, 170, 12, 15, 225, 169, 12, 15, 225, - 168, 12, 15, 225, 167, 12, 15, 225, 166, 12, 15, 225, 165, 12, 15, 225, - 164, 12, 15, 225, 163, 12, 15, 225, 162, 12, 15, 225, 161, 12, 15, 225, - 160, 12, 15, 225, 159, 12, 15, 225, 158, 12, 15, 225, 157, 12, 15, 225, - 156, 12, 15, 225, 155, 12, 15, 225, 154, 12, 15, 225, 153, 12, 15, 225, - 152, 12, 15, 225, 151, 12, 15, 225, 150, 12, 15, 225, 149, 12, 15, 225, - 148, 12, 15, 225, 147, 12, 15, 225, 146, 12, 15, 225, 145, 12, 15, 225, - 144, 12, 15, 225, 143, 12, 15, 225, 142, 12, 15, 225, 141, 12, 15, 225, - 140, 12, 15, 225, 139, 12, 15, 225, 138, 12, 15, 225, 137, 12, 15, 225, - 136, 12, 15, 225, 135, 12, 15, 225, 134, 12, 15, 225, 133, 12, 15, 225, - 132, 12, 15, 225, 131, 12, 15, 225, 130, 12, 15, 225, 129, 12, 15, 225, - 128, 12, 15, 225, 127, 12, 15, 225, 126, 12, 15, 225, 125, 12, 15, 225, - 124, 12, 15, 225, 123, 12, 15, 225, 122, 12, 15, 225, 121, 12, 15, 225, - 120, 12, 15, 225, 119, 12, 15, 225, 118, 12, 15, 225, 117, 12, 15, 225, - 116, 12, 15, 225, 115, 12, 15, 225, 114, 12, 15, 225, 113, 12, 15, 225, - 112, 12, 15, 225, 111, 12, 15, 225, 110, 12, 15, 225, 109, 12, 15, 225, - 108, 12, 15, 225, 107, 12, 15, 225, 106, 12, 15, 225, 105, 12, 15, 225, - 104, 12, 15, 225, 103, 12, 15, 225, 102, 12, 15, 225, 101, 12, 15, 225, - 100, 12, 15, 225, 99, 12, 15, 225, 98, 12, 15, 225, 97, 12, 15, 225, 96, - 12, 15, 225, 95, 12, 15, 225, 94, 12, 15, 225, 93, 12, 15, 225, 92, 12, - 15, 225, 91, 12, 15, 225, 90, 12, 15, 225, 89, 12, 15, 225, 88, 12, 15, - 225, 87, 12, 15, 225, 86, 12, 15, 225, 85, 12, 15, 225, 84, 12, 15, 225, - 83, 12, 15, 225, 82, 12, 15, 225, 81, 12, 15, 225, 80, 12, 15, 225, 79, - 12, 15, 225, 78, 12, 15, 225, 77, 12, 15, 225, 76, 12, 15, 225, 75, 12, - 15, 225, 74, 12, 15, 225, 73, 12, 15, 225, 72, 12, 15, 225, 71, 12, 15, - 225, 70, 12, 15, 225, 69, 12, 15, 225, 68, 12, 15, 225, 67, 12, 15, 225, - 66, 12, 15, 225, 65, 12, 15, 225, 64, 12, 15, 225, 63, 12, 15, 225, 62, - 12, 15, 225, 61, 12, 15, 225, 60, 12, 15, 225, 59, 12, 15, 225, 58, 12, - 15, 225, 57, 12, 15, 225, 56, 12, 15, 225, 55, 12, 15, 225, 54, 12, 15, - 225, 53, 12, 15, 225, 52, 12, 15, 225, 51, 12, 15, 225, 50, 12, 15, 225, - 49, 12, 15, 225, 48, 12, 15, 225, 47, 12, 15, 225, 46, 12, 15, 225, 45, - 12, 15, 225, 44, 12, 15, 225, 43, 12, 15, 225, 42, 12, 15, 225, 41, 12, - 15, 225, 40, 12, 15, 225, 39, 12, 15, 225, 38, 12, 15, 225, 37, 12, 15, - 225, 36, 12, 15, 225, 35, 12, 15, 225, 34, 12, 15, 225, 33, 12, 15, 225, - 32, 12, 15, 225, 31, 12, 15, 225, 30, 12, 15, 225, 29, 12, 15, 225, 28, - 12, 15, 225, 27, 12, 15, 225, 26, 12, 15, 225, 25, 12, 15, 225, 24, 12, - 15, 225, 23, 12, 15, 225, 22, 12, 15, 225, 21, 12, 15, 225, 20, 12, 15, - 225, 19, 12, 15, 225, 18, 12, 15, 225, 17, 12, 15, 225, 16, 12, 15, 225, - 15, 12, 15, 225, 14, 12, 15, 225, 13, 12, 15, 225, 12, 12, 15, 225, 11, - 12, 15, 225, 10, 12, 15, 225, 9, 12, 15, 225, 8, 12, 15, 225, 7, 12, 15, - 225, 6, 12, 15, 225, 5, 12, 15, 225, 4, 12, 15, 225, 3, 12, 15, 225, 2, - 12, 15, 225, 1, 12, 15, 225, 0, 12, 15, 224, 255, 12, 15, 224, 254, 12, - 15, 224, 253, 12, 15, 224, 252, 12, 15, 224, 251, 12, 15, 224, 250, 12, - 15, 224, 249, 12, 15, 224, 248, 12, 15, 224, 247, 12, 15, 224, 246, 12, - 15, 224, 245, 12, 15, 224, 244, 12, 15, 224, 243, 12, 15, 224, 242, 12, - 15, 224, 241, 12, 15, 224, 240, 12, 15, 224, 239, 12, 15, 224, 238, 12, - 15, 224, 237, 12, 15, 224, 236, 12, 15, 224, 235, 12, 15, 224, 234, 12, - 15, 224, 233, 12, 15, 224, 232, 12, 15, 224, 231, 12, 15, 224, 230, 12, - 15, 224, 229, 12, 15, 224, 228, 12, 15, 224, 227, 12, 15, 224, 226, 12, - 15, 224, 225, 12, 15, 224, 224, 12, 15, 224, 223, 12, 15, 224, 222, 12, - 15, 224, 221, 12, 15, 224, 220, 12, 15, 224, 219, 12, 15, 224, 218, 12, - 15, 224, 217, 12, 15, 224, 216, 12, 15, 224, 215, 12, 15, 224, 214, 12, - 15, 224, 213, 12, 15, 224, 212, 12, 15, 224, 211, 12, 15, 224, 210, 12, - 15, 224, 209, 12, 15, 224, 208, 12, 15, 224, 207, 12, 15, 224, 206, 12, - 15, 224, 205, 12, 15, 224, 204, 12, 15, 224, 203, 12, 15, 224, 202, 12, - 15, 224, 201, 12, 15, 224, 200, 12, 15, 224, 199, 12, 15, 224, 198, 12, - 15, 224, 197, 12, 15, 224, 196, 12, 15, 224, 195, 12, 15, 224, 194, 12, - 15, 224, 193, 12, 15, 224, 192, 12, 15, 224, 191, 12, 15, 224, 190, 12, - 15, 224, 189, 12, 15, 224, 188, 12, 15, 224, 187, 12, 15, 224, 186, 12, - 15, 224, 185, 12, 15, 224, 184, 12, 15, 224, 183, 12, 15, 224, 182, 12, - 15, 224, 181, 12, 15, 224, 180, 12, 15, 224, 179, 12, 15, 224, 178, 12, - 15, 224, 177, 12, 15, 224, 176, 12, 15, 224, 175, 12, 15, 224, 174, 12, - 15, 224, 173, 12, 15, 224, 172, 12, 15, 224, 171, 12, 15, 224, 170, 12, - 15, 224, 169, 12, 15, 224, 168, 12, 15, 224, 167, 12, 15, 224, 166, 12, - 15, 224, 165, 12, 15, 224, 164, 12, 15, 224, 163, 12, 15, 224, 162, 12, - 15, 224, 161, 12, 15, 224, 160, 12, 15, 224, 159, 12, 15, 224, 158, 12, - 15, 224, 157, 12, 15, 224, 156, 12, 15, 224, 155, 12, 15, 224, 154, 12, - 15, 224, 153, 12, 15, 224, 152, 12, 15, 224, 151, 12, 15, 224, 150, 12, - 15, 224, 149, 12, 15, 224, 148, 12, 15, 224, 147, 12, 15, 224, 146, 12, - 15, 224, 145, 12, 15, 224, 144, 12, 15, 224, 143, 12, 15, 224, 142, 12, - 15, 224, 141, 12, 15, 224, 140, 12, 15, 224, 139, 12, 15, 224, 138, 12, - 15, 224, 137, 12, 15, 224, 136, 12, 15, 224, 135, 12, 15, 224, 134, 12, - 15, 224, 133, 12, 15, 224, 132, 12, 15, 224, 131, 12, 15, 224, 130, 12, - 15, 224, 129, 12, 15, 224, 128, 12, 15, 224, 127, 12, 15, 224, 126, 12, - 15, 224, 125, 12, 15, 224, 124, 12, 15, 224, 123, 12, 15, 224, 122, 12, - 15, 224, 121, 12, 15, 224, 120, 12, 15, 224, 119, 12, 15, 224, 118, 12, - 15, 224, 117, 12, 15, 224, 116, 12, 15, 224, 115, 12, 15, 224, 114, 12, - 15, 224, 113, 12, 15, 224, 112, 12, 15, 224, 111, 12, 15, 224, 110, 12, - 15, 224, 109, 12, 15, 224, 108, 12, 15, 224, 107, 12, 15, 224, 106, 12, - 15, 224, 105, 12, 15, 224, 104, 12, 15, 224, 103, 12, 15, 224, 102, 12, - 15, 224, 101, 12, 15, 224, 100, 12, 15, 224, 99, 12, 15, 224, 98, 12, 15, - 224, 97, 12, 15, 224, 96, 12, 15, 224, 95, 12, 15, 224, 94, 12, 15, 224, - 93, 12, 15, 224, 92, 12, 15, 224, 91, 12, 15, 224, 90, 12, 15, 224, 89, - 12, 15, 224, 88, 12, 15, 224, 87, 12, 15, 224, 86, 12, 15, 224, 85, 12, - 15, 224, 84, 12, 15, 224, 83, 12, 15, 224, 82, 12, 15, 224, 81, 12, 15, - 224, 80, 12, 15, 224, 79, 12, 15, 224, 78, 12, 15, 224, 77, 12, 15, 224, - 76, 12, 15, 224, 75, 12, 15, 224, 74, 12, 15, 224, 73, 12, 15, 224, 72, - 12, 15, 224, 71, 12, 15, 224, 70, 12, 15, 224, 69, 12, 15, 224, 68, 12, - 15, 224, 67, 12, 15, 224, 66, 12, 15, 224, 65, 12, 15, 224, 64, 12, 15, - 224, 63, 12, 15, 224, 62, 12, 15, 224, 61, 12, 15, 224, 60, 12, 15, 224, - 59, 12, 15, 224, 58, 12, 15, 224, 57, 12, 15, 224, 56, 12, 15, 224, 55, - 12, 15, 224, 54, 12, 15, 224, 53, 12, 15, 224, 52, 12, 15, 224, 51, 12, - 15, 224, 50, 12, 15, 224, 49, 12, 15, 224, 48, 12, 15, 224, 47, 12, 15, - 224, 46, 12, 15, 224, 45, 12, 15, 224, 44, 12, 15, 224, 43, 12, 15, 224, - 42, 12, 15, 224, 41, 12, 15, 224, 40, 12, 15, 224, 39, 12, 15, 224, 38, - 12, 15, 224, 37, 12, 15, 224, 36, 12, 15, 224, 35, 12, 15, 224, 34, 12, - 15, 224, 33, 12, 15, 224, 32, 12, 15, 224, 31, 12, 15, 224, 30, 12, 15, - 224, 29, 12, 15, 224, 28, 12, 15, 224, 27, 12, 15, 224, 26, 12, 15, 224, - 25, 12, 15, 224, 24, 12, 15, 224, 23, 12, 15, 224, 22, 12, 15, 224, 21, - 12, 15, 224, 20, 12, 15, 224, 19, 12, 15, 224, 18, 12, 15, 224, 17, 12, - 15, 224, 16, 12, 15, 224, 15, 12, 15, 224, 14, 12, 15, 224, 13, 12, 15, - 224, 12, 12, 15, 224, 11, 12, 15, 224, 10, 12, 15, 224, 9, 12, 15, 224, - 8, 12, 15, 224, 7, 12, 15, 224, 6, 12, 15, 224, 5, 12, 15, 224, 4, 12, - 15, 224, 3, 12, 15, 224, 2, 12, 15, 224, 1, 12, 15, 224, 0, 12, 15, 223, - 255, 12, 15, 223, 254, 12, 15, 223, 253, 12, 15, 223, 252, 12, 15, 223, - 251, 12, 15, 223, 250, 12, 15, 223, 249, 12, 15, 223, 248, 12, 15, 223, - 247, 12, 15, 223, 246, 12, 15, 223, 245, 12, 15, 223, 244, 12, 15, 223, - 243, 12, 15, 223, 242, 12, 15, 223, 241, 12, 15, 223, 240, 12, 15, 223, - 239, 12, 15, 223, 238, 12, 15, 223, 237, 12, 15, 223, 236, 12, 15, 223, - 235, 12, 15, 223, 234, 12, 15, 223, 233, 8, 2, 34, 233, 29, 8, 2, 34, - 233, 25, 8, 2, 34, 232, 222, 8, 2, 34, 233, 28, 8, 2, 34, 233, 27, 8, 2, - 34, 180, 206, 10, 200, 43, 8, 2, 34, 201, 119, 250, 251, 2, 34, 216, 189, - 212, 255, 250, 251, 2, 34, 216, 189, 234, 197, 250, 251, 2, 34, 216, 189, - 223, 133, 250, 251, 2, 34, 195, 75, 212, 255, 250, 251, 2, 34, 216, 189, - 192, 212, 136, 1, 191, 251, 4, 229, 121, 136, 209, 64, 222, 183, 195, - 166, 136, 34, 192, 31, 191, 251, 191, 251, 210, 90, 136, 1, 251, 154, - 250, 128, 136, 1, 193, 78, 251, 194, 136, 1, 193, 78, 237, 255, 136, 1, - 193, 78, 229, 247, 136, 1, 193, 78, 222, 108, 136, 1, 193, 78, 220, 29, - 136, 1, 193, 78, 53, 216, 195, 136, 1, 193, 78, 207, 39, 136, 1, 193, 78, - 199, 162, 136, 1, 251, 154, 108, 56, 136, 1, 203, 71, 4, 203, 71, 236, - 142, 136, 1, 203, 71, 4, 202, 176, 236, 142, 136, 1, 203, 71, 4, 238, 19, - 23, 203, 71, 236, 142, 136, 1, 203, 71, 4, 238, 19, 23, 202, 176, 236, - 142, 136, 1, 131, 4, 210, 90, 136, 1, 131, 4, 208, 86, 136, 1, 131, 4, - 217, 72, 136, 1, 248, 217, 4, 238, 18, 136, 1, 231, 41, 4, 238, 18, 136, - 1, 238, 0, 4, 238, 18, 136, 1, 229, 248, 4, 217, 72, 136, 1, 195, 159, 4, - 238, 18, 136, 1, 191, 92, 4, 238, 18, 136, 1, 199, 74, 4, 238, 18, 136, - 1, 191, 251, 4, 238, 18, 136, 1, 53, 222, 109, 4, 238, 18, 136, 1, 222, - 109, 4, 238, 18, 136, 1, 220, 30, 4, 238, 18, 136, 1, 216, 196, 4, 238, - 18, 136, 1, 212, 133, 4, 238, 18, 136, 1, 205, 150, 4, 238, 18, 136, 1, - 53, 210, 66, 4, 238, 18, 136, 1, 210, 66, 4, 238, 18, 136, 1, 197, 164, - 4, 238, 18, 136, 1, 208, 45, 4, 238, 18, 136, 1, 207, 40, 4, 238, 18, - 136, 1, 203, 71, 4, 238, 18, 136, 1, 199, 163, 4, 238, 18, 136, 1, 195, - 159, 4, 229, 9, 136, 1, 248, 217, 4, 207, 163, 136, 1, 222, 109, 4, 207, - 163, 136, 1, 210, 66, 4, 207, 163, 136, 34, 131, 220, 29, 9, 1, 131, 193, - 151, 76, 20, 9, 1, 131, 193, 151, 53, 20, 9, 1, 249, 2, 76, 20, 9, 1, - 249, 2, 53, 20, 9, 1, 249, 2, 89, 20, 9, 1, 249, 2, 216, 219, 20, 9, 1, - 210, 44, 76, 20, 9, 1, 210, 44, 53, 20, 9, 1, 210, 44, 89, 20, 9, 1, 210, - 44, 216, 219, 20, 9, 1, 248, 246, 76, 20, 9, 1, 248, 246, 53, 20, 9, 1, - 248, 246, 89, 20, 9, 1, 248, 246, 216, 219, 20, 9, 1, 197, 123, 76, 20, - 9, 1, 197, 123, 53, 20, 9, 1, 197, 123, 89, 20, 9, 1, 197, 123, 216, 219, - 20, 9, 1, 199, 113, 76, 20, 9, 1, 199, 113, 53, 20, 9, 1, 199, 113, 89, - 20, 9, 1, 199, 113, 216, 219, 20, 9, 1, 197, 125, 76, 20, 9, 1, 197, 125, - 53, 20, 9, 1, 197, 125, 89, 20, 9, 1, 197, 125, 216, 219, 20, 9, 1, 195, - 147, 76, 20, 9, 1, 195, 147, 53, 20, 9, 1, 195, 147, 89, 20, 9, 1, 195, - 147, 216, 219, 20, 9, 1, 210, 42, 76, 20, 9, 1, 210, 42, 53, 20, 9, 1, - 210, 42, 89, 20, 9, 1, 210, 42, 216, 219, 20, 9, 1, 235, 44, 76, 20, 9, - 1, 235, 44, 53, 20, 9, 1, 235, 44, 89, 20, 9, 1, 235, 44, 216, 219, 20, - 9, 1, 212, 90, 76, 20, 9, 1, 212, 90, 53, 20, 9, 1, 212, 90, 89, 20, 9, - 1, 212, 90, 216, 219, 20, 9, 1, 199, 150, 76, 20, 9, 1, 199, 150, 53, 20, - 9, 1, 199, 150, 89, 20, 9, 1, 199, 150, 216, 219, 20, 9, 1, 199, 148, 76, - 20, 9, 1, 199, 148, 53, 20, 9, 1, 199, 148, 89, 20, 9, 1, 199, 148, 216, - 219, 20, 9, 1, 237, 191, 76, 20, 9, 1, 237, 191, 53, 20, 9, 1, 238, 13, - 76, 20, 9, 1, 238, 13, 53, 20, 9, 1, 235, 81, 76, 20, 9, 1, 235, 81, 53, - 20, 9, 1, 237, 189, 76, 20, 9, 1, 237, 189, 53, 20, 9, 1, 223, 6, 76, 20, - 9, 1, 223, 6, 53, 20, 9, 1, 206, 103, 76, 20, 9, 1, 206, 103, 53, 20, 9, - 1, 222, 7, 76, 20, 9, 1, 222, 7, 53, 20, 9, 1, 222, 7, 89, 20, 9, 1, 222, - 7, 216, 219, 20, 9, 1, 231, 230, 76, 20, 9, 1, 231, 230, 53, 20, 9, 1, - 231, 230, 89, 20, 9, 1, 231, 230, 216, 219, 20, 9, 1, 230, 169, 76, 20, - 9, 1, 230, 169, 53, 20, 9, 1, 230, 169, 89, 20, 9, 1, 230, 169, 216, 219, - 20, 9, 1, 213, 250, 76, 20, 9, 1, 213, 250, 53, 20, 9, 1, 213, 250, 89, - 20, 9, 1, 213, 250, 216, 219, 20, 9, 1, 213, 27, 231, 60, 76, 20, 9, 1, - 213, 27, 231, 60, 53, 20, 9, 1, 206, 167, 76, 20, 9, 1, 206, 167, 53, 20, - 9, 1, 206, 167, 89, 20, 9, 1, 206, 167, 216, 219, 20, 9, 1, 229, 213, 4, - 99, 93, 76, 20, 9, 1, 229, 213, 4, 99, 93, 53, 20, 9, 1, 229, 213, 231, - 3, 76, 20, 9, 1, 229, 213, 231, 3, 53, 20, 9, 1, 229, 213, 231, 3, 89, - 20, 9, 1, 229, 213, 231, 3, 216, 219, 20, 9, 1, 229, 213, 236, 173, 76, - 20, 9, 1, 229, 213, 236, 173, 53, 20, 9, 1, 229, 213, 236, 173, 89, 20, - 9, 1, 229, 213, 236, 173, 216, 219, 20, 9, 1, 99, 249, 83, 76, 20, 9, 1, - 99, 249, 83, 53, 20, 9, 1, 99, 249, 83, 4, 230, 60, 93, 76, 20, 9, 1, 99, - 249, 83, 4, 230, 60, 93, 53, 20, 9, 16, 75, 58, 9, 16, 75, 60, 9, 16, - 105, 185, 58, 9, 16, 105, 185, 60, 9, 16, 115, 185, 58, 9, 16, 115, 185, - 60, 9, 16, 115, 185, 209, 60, 235, 121, 58, 9, 16, 115, 185, 209, 60, - 235, 121, 60, 9, 16, 232, 130, 185, 58, 9, 16, 232, 130, 185, 60, 9, 16, - 55, 81, 249, 90, 60, 9, 16, 105, 185, 195, 85, 58, 9, 16, 105, 185, 195, - 85, 60, 9, 16, 206, 189, 9, 16, 2, 199, 220, 58, 9, 16, 2, 199, 220, 60, - 9, 16, 193, 151, 58, 9, 1, 214, 73, 76, 20, 9, 1, 214, 73, 53, 20, 9, 1, - 214, 73, 89, 20, 9, 1, 214, 73, 216, 219, 20, 9, 1, 126, 76, 20, 9, 1, - 126, 53, 20, 9, 1, 211, 154, 76, 20, 9, 1, 211, 154, 53, 20, 9, 1, 191, - 226, 76, 20, 9, 1, 191, 226, 53, 20, 9, 1, 126, 4, 230, 60, 93, 76, 20, - 9, 1, 195, 154, 76, 20, 9, 1, 195, 154, 53, 20, 9, 1, 221, 134, 211, 154, - 76, 20, 9, 1, 221, 134, 211, 154, 53, 20, 9, 1, 221, 134, 191, 226, 76, - 20, 9, 1, 221, 134, 191, 226, 53, 20, 9, 1, 235, 17, 76, 20, 9, 1, 235, - 17, 53, 20, 9, 1, 235, 17, 89, 20, 9, 1, 235, 17, 216, 219, 20, 9, 1, - 196, 137, 222, 28, 221, 134, 131, 217, 102, 89, 20, 9, 1, 196, 137, 222, - 28, 221, 134, 131, 217, 102, 216, 219, 20, 9, 34, 99, 4, 230, 60, 93, 4, - 131, 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 131, 53, 20, 9, 34, 99, 4, - 230, 60, 93, 4, 252, 28, 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 252, 28, - 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, 193, 134, 76, 20, 9, 34, 99, 4, - 230, 60, 93, 4, 193, 134, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, 126, 76, - 20, 9, 34, 99, 4, 230, 60, 93, 4, 126, 53, 20, 9, 34, 99, 4, 230, 60, 93, - 4, 211, 154, 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 211, 154, 53, 20, 9, - 34, 99, 4, 230, 60, 93, 4, 191, 226, 76, 20, 9, 34, 99, 4, 230, 60, 93, - 4, 191, 226, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, 235, 17, 76, 20, 9, - 34, 99, 4, 230, 60, 93, 4, 235, 17, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, - 235, 17, 89, 20, 9, 34, 196, 137, 221, 134, 99, 4, 230, 60, 93, 4, 131, - 217, 102, 76, 20, 9, 34, 196, 137, 221, 134, 99, 4, 230, 60, 93, 4, 131, - 217, 102, 53, 20, 9, 34, 196, 137, 221, 134, 99, 4, 230, 60, 93, 4, 131, - 217, 102, 89, 20, 9, 1, 233, 76, 99, 76, 20, 9, 1, 233, 76, 99, 53, 20, - 9, 1, 233, 76, 99, 89, 20, 9, 1, 233, 76, 99, 216, 219, 20, 9, 34, 99, 4, - 230, 60, 93, 4, 223, 9, 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 183, 76, - 20, 9, 34, 99, 4, 230, 60, 93, 4, 92, 76, 20, 9, 34, 99, 4, 230, 60, 93, - 4, 131, 217, 102, 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 99, 76, 20, 9, - 34, 248, 248, 4, 223, 9, 76, 20, 9, 34, 248, 248, 4, 183, 76, 20, 9, 34, - 248, 248, 4, 221, 213, 76, 20, 9, 34, 248, 248, 4, 92, 76, 20, 9, 34, - 248, 248, 4, 131, 217, 102, 76, 20, 9, 34, 248, 248, 4, 99, 76, 20, 9, - 34, 199, 115, 4, 223, 9, 76, 20, 9, 34, 199, 115, 4, 183, 76, 20, 9, 34, - 199, 115, 4, 221, 213, 76, 20, 9, 34, 199, 115, 4, 92, 76, 20, 9, 34, - 199, 115, 4, 131, 217, 102, 76, 20, 9, 34, 199, 115, 4, 99, 76, 20, 9, - 34, 199, 30, 4, 223, 9, 76, 20, 9, 34, 199, 30, 4, 92, 76, 20, 9, 34, - 199, 30, 4, 131, 217, 102, 76, 20, 9, 34, 199, 30, 4, 99, 76, 20, 9, 34, - 223, 9, 4, 183, 76, 20, 9, 34, 223, 9, 4, 92, 76, 20, 9, 34, 183, 4, 223, - 9, 76, 20, 9, 34, 183, 4, 92, 76, 20, 9, 34, 221, 213, 4, 223, 9, 76, 20, - 9, 34, 221, 213, 4, 183, 76, 20, 9, 34, 221, 213, 4, 92, 76, 20, 9, 34, - 205, 48, 4, 223, 9, 76, 20, 9, 34, 205, 48, 4, 183, 76, 20, 9, 34, 205, - 48, 4, 221, 213, 76, 20, 9, 34, 205, 48, 4, 92, 76, 20, 9, 34, 205, 194, - 4, 183, 76, 20, 9, 34, 205, 194, 4, 92, 76, 20, 9, 34, 238, 29, 4, 223, - 9, 76, 20, 9, 34, 238, 29, 4, 183, 76, 20, 9, 34, 238, 29, 4, 221, 213, - 76, 20, 9, 34, 238, 29, 4, 92, 76, 20, 9, 34, 199, 220, 4, 183, 76, 20, - 9, 34, 199, 220, 4, 92, 76, 20, 9, 34, 191, 117, 4, 92, 76, 20, 9, 34, - 251, 233, 4, 223, 9, 76, 20, 9, 34, 251, 233, 4, 92, 76, 20, 9, 34, 231, - 89, 4, 223, 9, 76, 20, 9, 34, 231, 89, 4, 92, 76, 20, 9, 34, 233, 49, 4, - 223, 9, 76, 20, 9, 34, 233, 49, 4, 183, 76, 20, 9, 34, 233, 49, 4, 221, - 213, 76, 20, 9, 34, 233, 49, 4, 92, 76, 20, 9, 34, 233, 49, 4, 131, 217, - 102, 76, 20, 9, 34, 233, 49, 4, 99, 76, 20, 9, 34, 208, 92, 4, 183, 76, - 20, 9, 34, 208, 92, 4, 92, 76, 20, 9, 34, 208, 92, 4, 131, 217, 102, 76, - 20, 9, 34, 208, 92, 4, 99, 76, 20, 9, 34, 222, 109, 4, 131, 76, 20, 9, - 34, 222, 109, 4, 223, 9, 76, 20, 9, 34, 222, 109, 4, 183, 76, 20, 9, 34, - 222, 109, 4, 221, 213, 76, 20, 9, 34, 222, 109, 4, 220, 38, 76, 20, 9, - 34, 222, 109, 4, 92, 76, 20, 9, 34, 222, 109, 4, 131, 217, 102, 76, 20, - 9, 34, 222, 109, 4, 99, 76, 20, 9, 34, 220, 38, 4, 223, 9, 76, 20, 9, 34, - 220, 38, 4, 183, 76, 20, 9, 34, 220, 38, 4, 221, 213, 76, 20, 9, 34, 220, - 38, 4, 92, 76, 20, 9, 34, 220, 38, 4, 131, 217, 102, 76, 20, 9, 34, 220, - 38, 4, 99, 76, 20, 9, 34, 92, 4, 223, 9, 76, 20, 9, 34, 92, 4, 183, 76, - 20, 9, 34, 92, 4, 221, 213, 76, 20, 9, 34, 92, 4, 92, 76, 20, 9, 34, 92, - 4, 131, 217, 102, 76, 20, 9, 34, 92, 4, 99, 76, 20, 9, 34, 213, 27, 4, - 223, 9, 76, 20, 9, 34, 213, 27, 4, 183, 76, 20, 9, 34, 213, 27, 4, 221, - 213, 76, 20, 9, 34, 213, 27, 4, 92, 76, 20, 9, 34, 213, 27, 4, 131, 217, - 102, 76, 20, 9, 34, 213, 27, 4, 99, 76, 20, 9, 34, 229, 213, 4, 223, 9, - 76, 20, 9, 34, 229, 213, 4, 92, 76, 20, 9, 34, 229, 213, 4, 131, 217, - 102, 76, 20, 9, 34, 229, 213, 4, 99, 76, 20, 9, 34, 99, 4, 223, 9, 76, - 20, 9, 34, 99, 4, 183, 76, 20, 9, 34, 99, 4, 221, 213, 76, 20, 9, 34, 99, - 4, 92, 76, 20, 9, 34, 99, 4, 131, 217, 102, 76, 20, 9, 34, 99, 4, 99, 76, - 20, 9, 34, 199, 42, 4, 200, 182, 131, 76, 20, 9, 34, 207, 73, 4, 200, - 182, 131, 76, 20, 9, 34, 131, 217, 102, 4, 200, 182, 131, 76, 20, 9, 34, - 203, 157, 4, 237, 246, 76, 20, 9, 34, 203, 157, 4, 222, 53, 76, 20, 9, - 34, 203, 157, 4, 233, 73, 76, 20, 9, 34, 203, 157, 4, 237, 248, 76, 20, - 9, 34, 203, 157, 4, 222, 55, 76, 20, 9, 34, 203, 157, 4, 200, 182, 131, - 76, 20, 9, 34, 99, 4, 230, 60, 93, 4, 207, 73, 53, 20, 9, 34, 99, 4, 230, - 60, 93, 4, 191, 114, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, 92, 53, 20, 9, - 34, 99, 4, 230, 60, 93, 4, 213, 27, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, - 131, 217, 102, 53, 20, 9, 34, 99, 4, 230, 60, 93, 4, 99, 53, 20, 9, 34, - 248, 248, 4, 207, 73, 53, 20, 9, 34, 248, 248, 4, 191, 114, 53, 20, 9, - 34, 248, 248, 4, 92, 53, 20, 9, 34, 248, 248, 4, 213, 27, 53, 20, 9, 34, - 248, 248, 4, 131, 217, 102, 53, 20, 9, 34, 248, 248, 4, 99, 53, 20, 9, - 34, 199, 115, 4, 207, 73, 53, 20, 9, 34, 199, 115, 4, 191, 114, 53, 20, - 9, 34, 199, 115, 4, 92, 53, 20, 9, 34, 199, 115, 4, 213, 27, 53, 20, 9, - 34, 199, 115, 4, 131, 217, 102, 53, 20, 9, 34, 199, 115, 4, 99, 53, 20, - 9, 34, 199, 30, 4, 207, 73, 53, 20, 9, 34, 199, 30, 4, 191, 114, 53, 20, - 9, 34, 199, 30, 4, 92, 53, 20, 9, 34, 199, 30, 4, 213, 27, 53, 20, 9, 34, - 199, 30, 4, 131, 217, 102, 53, 20, 9, 34, 199, 30, 4, 99, 53, 20, 9, 34, - 233, 49, 4, 131, 217, 102, 53, 20, 9, 34, 233, 49, 4, 99, 53, 20, 9, 34, - 208, 92, 4, 131, 217, 102, 53, 20, 9, 34, 208, 92, 4, 99, 53, 20, 9, 34, - 222, 109, 4, 131, 53, 20, 9, 34, 222, 109, 4, 220, 38, 53, 20, 9, 34, - 222, 109, 4, 92, 53, 20, 9, 34, 222, 109, 4, 131, 217, 102, 53, 20, 9, - 34, 222, 109, 4, 99, 53, 20, 9, 34, 220, 38, 4, 92, 53, 20, 9, 34, 220, - 38, 4, 131, 217, 102, 53, 20, 9, 34, 220, 38, 4, 99, 53, 20, 9, 34, 92, - 4, 131, 53, 20, 9, 34, 92, 4, 92, 53, 20, 9, 34, 213, 27, 4, 207, 73, 53, - 20, 9, 34, 213, 27, 4, 191, 114, 53, 20, 9, 34, 213, 27, 4, 92, 53, 20, - 9, 34, 213, 27, 4, 213, 27, 53, 20, 9, 34, 213, 27, 4, 131, 217, 102, 53, - 20, 9, 34, 213, 27, 4, 99, 53, 20, 9, 34, 131, 217, 102, 4, 200, 182, - 131, 53, 20, 9, 34, 99, 4, 207, 73, 53, 20, 9, 34, 99, 4, 191, 114, 53, - 20, 9, 34, 99, 4, 92, 53, 20, 9, 34, 99, 4, 213, 27, 53, 20, 9, 34, 99, - 4, 131, 217, 102, 53, 20, 9, 34, 99, 4, 99, 53, 20, 9, 34, 99, 4, 230, - 60, 93, 4, 223, 9, 89, 20, 9, 34, 99, 4, 230, 60, 93, 4, 183, 89, 20, 9, - 34, 99, 4, 230, 60, 93, 4, 221, 213, 89, 20, 9, 34, 99, 4, 230, 60, 93, - 4, 92, 89, 20, 9, 34, 99, 4, 230, 60, 93, 4, 229, 213, 89, 20, 9, 34, - 248, 248, 4, 223, 9, 89, 20, 9, 34, 248, 248, 4, 183, 89, 20, 9, 34, 248, - 248, 4, 221, 213, 89, 20, 9, 34, 248, 248, 4, 92, 89, 20, 9, 34, 248, - 248, 4, 229, 213, 89, 20, 9, 34, 199, 115, 4, 223, 9, 89, 20, 9, 34, 199, - 115, 4, 183, 89, 20, 9, 34, 199, 115, 4, 221, 213, 89, 20, 9, 34, 199, - 115, 4, 92, 89, 20, 9, 34, 199, 115, 4, 229, 213, 89, 20, 9, 34, 199, 30, - 4, 92, 89, 20, 9, 34, 223, 9, 4, 183, 89, 20, 9, 34, 223, 9, 4, 92, 89, - 20, 9, 34, 183, 4, 223, 9, 89, 20, 9, 34, 183, 4, 92, 89, 20, 9, 34, 221, - 213, 4, 223, 9, 89, 20, 9, 34, 221, 213, 4, 92, 89, 20, 9, 34, 205, 48, - 4, 223, 9, 89, 20, 9, 34, 205, 48, 4, 183, 89, 20, 9, 34, 205, 48, 4, - 221, 213, 89, 20, 9, 34, 205, 48, 4, 92, 89, 20, 9, 34, 205, 194, 4, 183, - 89, 20, 9, 34, 205, 194, 4, 221, 213, 89, 20, 9, 34, 205, 194, 4, 92, 89, - 20, 9, 34, 238, 29, 4, 223, 9, 89, 20, 9, 34, 238, 29, 4, 183, 89, 20, 9, - 34, 238, 29, 4, 221, 213, 89, 20, 9, 34, 238, 29, 4, 92, 89, 20, 9, 34, - 199, 220, 4, 183, 89, 20, 9, 34, 191, 117, 4, 92, 89, 20, 9, 34, 251, - 233, 4, 223, 9, 89, 20, 9, 34, 251, 233, 4, 92, 89, 20, 9, 34, 231, 89, - 4, 223, 9, 89, 20, 9, 34, 231, 89, 4, 92, 89, 20, 9, 34, 233, 49, 4, 223, - 9, 89, 20, 9, 34, 233, 49, 4, 183, 89, 20, 9, 34, 233, 49, 4, 221, 213, - 89, 20, 9, 34, 233, 49, 4, 92, 89, 20, 9, 34, 208, 92, 4, 183, 89, 20, 9, - 34, 208, 92, 4, 92, 89, 20, 9, 34, 222, 109, 4, 223, 9, 89, 20, 9, 34, - 222, 109, 4, 183, 89, 20, 9, 34, 222, 109, 4, 221, 213, 89, 20, 9, 34, - 222, 109, 4, 220, 38, 89, 20, 9, 34, 222, 109, 4, 92, 89, 20, 9, 34, 220, - 38, 4, 223, 9, 89, 20, 9, 34, 220, 38, 4, 183, 89, 20, 9, 34, 220, 38, 4, - 221, 213, 89, 20, 9, 34, 220, 38, 4, 92, 89, 20, 9, 34, 220, 38, 4, 229, - 213, 89, 20, 9, 34, 92, 4, 223, 9, 89, 20, 9, 34, 92, 4, 183, 89, 20, 9, - 34, 92, 4, 221, 213, 89, 20, 9, 34, 92, 4, 92, 89, 20, 9, 34, 213, 27, 4, - 223, 9, 89, 20, 9, 34, 213, 27, 4, 183, 89, 20, 9, 34, 213, 27, 4, 221, - 213, 89, 20, 9, 34, 213, 27, 4, 92, 89, 20, 9, 34, 213, 27, 4, 229, 213, - 89, 20, 9, 34, 229, 213, 4, 223, 9, 89, 20, 9, 34, 229, 213, 4, 92, 89, - 20, 9, 34, 229, 213, 4, 200, 182, 131, 89, 20, 9, 34, 99, 4, 223, 9, 89, - 20, 9, 34, 99, 4, 183, 89, 20, 9, 34, 99, 4, 221, 213, 89, 20, 9, 34, 99, - 4, 92, 89, 20, 9, 34, 99, 4, 229, 213, 89, 20, 9, 34, 99, 4, 230, 60, 93, - 4, 92, 216, 219, 20, 9, 34, 99, 4, 230, 60, 93, 4, 229, 213, 216, 219, - 20, 9, 34, 248, 248, 4, 92, 216, 219, 20, 9, 34, 248, 248, 4, 229, 213, - 216, 219, 20, 9, 34, 199, 115, 4, 92, 216, 219, 20, 9, 34, 199, 115, 4, - 229, 213, 216, 219, 20, 9, 34, 199, 30, 4, 92, 216, 219, 20, 9, 34, 199, - 30, 4, 229, 213, 216, 219, 20, 9, 34, 205, 48, 4, 92, 216, 219, 20, 9, - 34, 205, 48, 4, 229, 213, 216, 219, 20, 9, 34, 203, 111, 4, 92, 216, 219, - 20, 9, 34, 203, 111, 4, 229, 213, 216, 219, 20, 9, 34, 222, 109, 4, 220, - 38, 216, 219, 20, 9, 34, 222, 109, 4, 92, 216, 219, 20, 9, 34, 220, 38, - 4, 92, 216, 219, 20, 9, 34, 213, 27, 4, 92, 216, 219, 20, 9, 34, 213, 27, - 4, 229, 213, 216, 219, 20, 9, 34, 99, 4, 92, 216, 219, 20, 9, 34, 99, 4, - 229, 213, 216, 219, 20, 9, 34, 203, 157, 4, 233, 73, 216, 219, 20, 9, 34, - 203, 157, 4, 237, 248, 216, 219, 20, 9, 34, 203, 157, 4, 222, 55, 216, - 219, 20, 9, 34, 199, 220, 4, 131, 217, 102, 76, 20, 9, 34, 199, 220, 4, - 99, 76, 20, 9, 34, 251, 233, 4, 131, 217, 102, 76, 20, 9, 34, 251, 233, - 4, 99, 76, 20, 9, 34, 231, 89, 4, 131, 217, 102, 76, 20, 9, 34, 231, 89, - 4, 99, 76, 20, 9, 34, 205, 48, 4, 131, 217, 102, 76, 20, 9, 34, 205, 48, - 4, 99, 76, 20, 9, 34, 203, 111, 4, 131, 217, 102, 76, 20, 9, 34, 203, - 111, 4, 99, 76, 20, 9, 34, 183, 4, 131, 217, 102, 76, 20, 9, 34, 183, 4, - 99, 76, 20, 9, 34, 223, 9, 4, 131, 217, 102, 76, 20, 9, 34, 223, 9, 4, - 99, 76, 20, 9, 34, 221, 213, 4, 131, 217, 102, 76, 20, 9, 34, 221, 213, - 4, 99, 76, 20, 9, 34, 205, 194, 4, 131, 217, 102, 76, 20, 9, 34, 205, - 194, 4, 99, 76, 20, 9, 34, 238, 29, 4, 131, 217, 102, 76, 20, 9, 34, 238, - 29, 4, 99, 76, 20, 9, 34, 203, 111, 4, 223, 9, 76, 20, 9, 34, 203, 111, - 4, 183, 76, 20, 9, 34, 203, 111, 4, 221, 213, 76, 20, 9, 34, 203, 111, 4, - 92, 76, 20, 9, 34, 203, 111, 4, 207, 73, 76, 20, 9, 34, 205, 48, 4, 207, - 73, 76, 20, 9, 34, 205, 194, 4, 207, 73, 76, 20, 9, 34, 238, 29, 4, 207, - 73, 76, 20, 9, 34, 199, 220, 4, 131, 217, 102, 53, 20, 9, 34, 199, 220, - 4, 99, 53, 20, 9, 34, 251, 233, 4, 131, 217, 102, 53, 20, 9, 34, 251, - 233, 4, 99, 53, 20, 9, 34, 231, 89, 4, 131, 217, 102, 53, 20, 9, 34, 231, - 89, 4, 99, 53, 20, 9, 34, 205, 48, 4, 131, 217, 102, 53, 20, 9, 34, 205, - 48, 4, 99, 53, 20, 9, 34, 203, 111, 4, 131, 217, 102, 53, 20, 9, 34, 203, - 111, 4, 99, 53, 20, 9, 34, 183, 4, 131, 217, 102, 53, 20, 9, 34, 183, 4, - 99, 53, 20, 9, 34, 223, 9, 4, 131, 217, 102, 53, 20, 9, 34, 223, 9, 4, - 99, 53, 20, 9, 34, 221, 213, 4, 131, 217, 102, 53, 20, 9, 34, 221, 213, - 4, 99, 53, 20, 9, 34, 205, 194, 4, 131, 217, 102, 53, 20, 9, 34, 205, - 194, 4, 99, 53, 20, 9, 34, 238, 29, 4, 131, 217, 102, 53, 20, 9, 34, 238, - 29, 4, 99, 53, 20, 9, 34, 203, 111, 4, 223, 9, 53, 20, 9, 34, 203, 111, - 4, 183, 53, 20, 9, 34, 203, 111, 4, 221, 213, 53, 20, 9, 34, 203, 111, 4, - 92, 53, 20, 9, 34, 203, 111, 4, 207, 73, 53, 20, 9, 34, 205, 48, 4, 207, - 73, 53, 20, 9, 34, 205, 194, 4, 207, 73, 53, 20, 9, 34, 238, 29, 4, 207, - 73, 53, 20, 9, 34, 203, 111, 4, 223, 9, 89, 20, 9, 34, 203, 111, 4, 183, - 89, 20, 9, 34, 203, 111, 4, 221, 213, 89, 20, 9, 34, 203, 111, 4, 92, 89, - 20, 9, 34, 205, 48, 4, 229, 213, 89, 20, 9, 34, 203, 111, 4, 229, 213, - 89, 20, 9, 34, 199, 220, 4, 92, 89, 20, 9, 34, 205, 48, 4, 223, 9, 216, - 219, 20, 9, 34, 205, 48, 4, 183, 216, 219, 20, 9, 34, 205, 48, 4, 221, - 213, 216, 219, 20, 9, 34, 203, 111, 4, 223, 9, 216, 219, 20, 9, 34, 203, - 111, 4, 183, 216, 219, 20, 9, 34, 203, 111, 4, 221, 213, 216, 219, 20, 9, - 34, 199, 220, 4, 92, 216, 219, 20, 9, 34, 191, 117, 4, 92, 216, 219, 20, - 9, 34, 131, 4, 233, 71, 53, 20, 9, 34, 131, 4, 233, 71, 76, 20, 211, 42, - 45, 210, 115, 211, 42, 50, 210, 115, 9, 34, 207, 161, 251, 175, 9, 34, - 207, 169, 251, 174, 251, 109, 9, 34, 207, 169, 251, 174, 251, 108, 9, 34, - 207, 169, 251, 174, 251, 106, 9, 34, 207, 169, 251, 174, 251, 105, 9, 34, - 207, 169, 251, 174, 251, 104, 9, 34, 205, 163, 251, 199, 193, 184, 9, 34, - 251, 199, 250, 219, 9, 34, 251, 198, 250, 219, 9, 34, 251, 197, 250, 219, - 9, 34, 251, 199, 250, 218, 193, 154, 9, 34, 208, 19, 202, 141, 9, 34, - 205, 161, 251, 199, 193, 180, 193, 183, 9, 34, 251, 202, 250, 219, 9, 34, - 199, 235, 193, 182, 9, 34, 207, 160, 251, 175, 9, 34, 199, 115, 4, 223, - 9, 4, 92, 89, 20, 9, 34, 199, 115, 4, 183, 4, 223, 9, 53, 20, 9, 34, 199, - 115, 4, 183, 4, 223, 9, 89, 20, 9, 34, 199, 115, 4, 183, 4, 92, 89, 20, - 9, 34, 199, 115, 4, 221, 213, 4, 92, 89, 20, 9, 34, 199, 115, 4, 92, 4, - 223, 9, 89, 20, 9, 34, 199, 115, 4, 92, 4, 183, 89, 20, 9, 34, 199, 115, - 4, 92, 4, 221, 213, 89, 20, 9, 34, 223, 9, 4, 92, 4, 183, 53, 20, 9, 34, - 223, 9, 4, 92, 4, 183, 89, 20, 9, 34, 183, 4, 92, 4, 99, 53, 20, 9, 34, - 183, 4, 92, 4, 131, 217, 102, 53, 20, 9, 34, 205, 48, 4, 183, 4, 223, 9, - 89, 20, 9, 34, 205, 48, 4, 223, 9, 4, 183, 89, 20, 9, 34, 205, 48, 4, - 223, 9, 4, 131, 217, 102, 53, 20, 9, 34, 205, 48, 4, 92, 4, 183, 53, 20, - 9, 34, 205, 48, 4, 92, 4, 183, 89, 20, 9, 34, 205, 48, 4, 92, 4, 223, 9, - 89, 20, 9, 34, 205, 48, 4, 92, 4, 92, 53, 20, 9, 34, 205, 48, 4, 92, 4, - 92, 89, 20, 9, 34, 205, 194, 4, 183, 4, 183, 53, 20, 9, 34, 205, 194, 4, - 183, 4, 183, 89, 20, 9, 34, 205, 194, 4, 92, 4, 92, 53, 20, 9, 34, 203, - 111, 4, 183, 4, 92, 53, 20, 9, 34, 203, 111, 4, 183, 4, 92, 89, 20, 9, - 34, 203, 111, 4, 223, 9, 4, 99, 53, 20, 9, 34, 203, 111, 4, 92, 4, 221, - 213, 53, 20, 9, 34, 203, 111, 4, 92, 4, 221, 213, 89, 20, 9, 34, 203, - 111, 4, 92, 4, 92, 53, 20, 9, 34, 203, 111, 4, 92, 4, 92, 89, 20, 9, 34, - 238, 29, 4, 183, 4, 131, 217, 102, 53, 20, 9, 34, 238, 29, 4, 221, 213, - 4, 92, 53, 20, 9, 34, 238, 29, 4, 221, 213, 4, 92, 89, 20, 9, 34, 199, - 220, 4, 92, 4, 183, 53, 20, 9, 34, 199, 220, 4, 92, 4, 183, 89, 20, 9, - 34, 199, 220, 4, 92, 4, 92, 89, 20, 9, 34, 199, 220, 4, 92, 4, 99, 53, - 20, 9, 34, 251, 233, 4, 223, 9, 4, 92, 53, 20, 9, 34, 251, 233, 4, 92, 4, - 92, 53, 20, 9, 34, 251, 233, 4, 92, 4, 92, 89, 20, 9, 34, 251, 233, 4, - 92, 4, 131, 217, 102, 53, 20, 9, 34, 231, 89, 4, 92, 4, 92, 53, 20, 9, - 34, 231, 89, 4, 92, 4, 99, 53, 20, 9, 34, 231, 89, 4, 92, 4, 131, 217, - 102, 53, 20, 9, 34, 233, 49, 4, 221, 213, 4, 92, 53, 20, 9, 34, 233, 49, - 4, 221, 213, 4, 92, 89, 20, 9, 34, 208, 92, 4, 92, 4, 183, 53, 20, 9, 34, - 208, 92, 4, 92, 4, 92, 53, 20, 9, 34, 220, 38, 4, 183, 4, 92, 53, 20, 9, - 34, 220, 38, 4, 183, 4, 99, 53, 20, 9, 34, 220, 38, 4, 183, 4, 131, 217, - 102, 53, 20, 9, 34, 220, 38, 4, 223, 9, 4, 223, 9, 89, 20, 9, 34, 220, - 38, 4, 223, 9, 4, 223, 9, 53, 20, 9, 34, 220, 38, 4, 221, 213, 4, 92, 53, - 20, 9, 34, 220, 38, 4, 221, 213, 4, 92, 89, 20, 9, 34, 220, 38, 4, 92, 4, - 183, 53, 20, 9, 34, 220, 38, 4, 92, 4, 183, 89, 20, 9, 34, 92, 4, 183, 4, - 223, 9, 89, 20, 9, 34, 92, 4, 183, 4, 92, 89, 20, 9, 34, 92, 4, 183, 4, - 99, 53, 20, 9, 34, 92, 4, 223, 9, 4, 183, 89, 20, 9, 34, 92, 4, 223, 9, - 4, 92, 89, 20, 9, 34, 92, 4, 221, 213, 4, 223, 9, 89, 20, 9, 34, 92, 4, - 221, 213, 4, 92, 89, 20, 9, 34, 92, 4, 223, 9, 4, 221, 213, 89, 20, 9, - 34, 229, 213, 4, 92, 4, 223, 9, 89, 20, 9, 34, 229, 213, 4, 92, 4, 92, - 89, 20, 9, 34, 213, 27, 4, 183, 4, 92, 89, 20, 9, 34, 213, 27, 4, 183, 4, - 131, 217, 102, 53, 20, 9, 34, 213, 27, 4, 223, 9, 4, 92, 53, 20, 9, 34, - 213, 27, 4, 223, 9, 4, 92, 89, 20, 9, 34, 213, 27, 4, 223, 9, 4, 131, - 217, 102, 53, 20, 9, 34, 213, 27, 4, 92, 4, 99, 53, 20, 9, 34, 213, 27, - 4, 92, 4, 131, 217, 102, 53, 20, 9, 34, 99, 4, 92, 4, 92, 53, 20, 9, 34, - 99, 4, 92, 4, 92, 89, 20, 9, 34, 248, 248, 4, 221, 213, 4, 99, 53, 20, 9, - 34, 199, 115, 4, 223, 9, 4, 99, 53, 20, 9, 34, 199, 115, 4, 223, 9, 4, - 131, 217, 102, 53, 20, 9, 34, 199, 115, 4, 221, 213, 4, 99, 53, 20, 9, - 34, 199, 115, 4, 221, 213, 4, 131, 217, 102, 53, 20, 9, 34, 199, 115, 4, - 92, 4, 99, 53, 20, 9, 34, 199, 115, 4, 92, 4, 131, 217, 102, 53, 20, 9, - 34, 223, 9, 4, 92, 4, 99, 53, 20, 9, 34, 223, 9, 4, 183, 4, 131, 217, - 102, 53, 20, 9, 34, 223, 9, 4, 92, 4, 131, 217, 102, 53, 20, 9, 34, 205, - 48, 4, 221, 213, 4, 131, 217, 102, 53, 20, 9, 34, 205, 194, 4, 183, 4, - 99, 53, 20, 9, 34, 203, 111, 4, 183, 4, 99, 53, 20, 9, 34, 238, 29, 4, - 183, 4, 99, 53, 20, 9, 34, 220, 38, 4, 223, 9, 4, 99, 53, 20, 9, 34, 220, - 38, 4, 92, 4, 99, 53, 20, 9, 34, 99, 4, 183, 4, 99, 53, 20, 9, 34, 99, 4, - 223, 9, 4, 99, 53, 20, 9, 34, 99, 4, 92, 4, 99, 53, 20, 9, 34, 92, 4, 92, - 4, 99, 53, 20, 9, 34, 208, 92, 4, 92, 4, 99, 53, 20, 9, 34, 213, 27, 4, - 183, 4, 99, 53, 20, 9, 34, 208, 92, 4, 92, 4, 183, 89, 20, 9, 34, 220, - 38, 4, 183, 4, 92, 89, 20, 9, 34, 251, 233, 4, 92, 4, 99, 53, 20, 9, 34, - 222, 109, 4, 92, 4, 99, 53, 20, 9, 34, 213, 27, 4, 223, 9, 4, 183, 89, - 20, 9, 34, 92, 4, 221, 213, 4, 99, 53, 20, 9, 34, 220, 38, 4, 223, 9, 4, - 92, 89, 20, 9, 34, 222, 109, 4, 92, 4, 92, 53, 20, 9, 34, 220, 38, 4, - 223, 9, 4, 92, 53, 20, 9, 34, 213, 27, 4, 223, 9, 4, 183, 53, 20, 9, 34, - 223, 9, 4, 183, 4, 99, 53, 20, 9, 34, 183, 4, 223, 9, 4, 99, 53, 20, 9, - 34, 92, 4, 223, 9, 4, 99, 53, 20, 9, 34, 233, 49, 4, 92, 4, 99, 53, 20, - 9, 34, 248, 248, 4, 183, 4, 99, 53, 20, 9, 34, 222, 109, 4, 92, 4, 92, - 89, 20, 9, 34, 251, 233, 4, 223, 9, 4, 92, 89, 20, 9, 34, 205, 194, 4, - 92, 4, 92, 89, 20, 9, 34, 205, 48, 4, 221, 213, 4, 99, 53, 20, 9, 34, - 213, 27, 4, 223, 9, 4, 99, 53, 20, 9, 34, 205, 167, 251, 196, 9, 34, 205, - 164, 196, 40, 250, 222, 221, 35, 201, 65, 3, 76, 20, 9, 34, 208, 88, 196, - 40, 250, 222, 221, 35, 201, 65, 3, 76, 20, 9, 34, 251, 173, 76, 20, 9, - 34, 251, 215, 76, 20, 9, 34, 215, 237, 76, 20, 9, 34, 205, 165, 76, 20, - 9, 34, 207, 134, 76, 20, 9, 34, 251, 201, 76, 20, 9, 34, 193, 153, 76, - 20, 9, 34, 205, 164, 76, 20, 9, 34, 205, 162, 251, 201, 193, 152, 9, 34, - 223, 24, 206, 250, 56, 9, 34, 248, 153, 251, 33, 251, 34, 9, 34, 200, - 243, 193, 191, 199, 244, 9, 34, 250, 123, 193, 191, 223, 25, 67, 205, 34, - 67, 204, 179, 67, 204, 111, 67, 204, 100, 67, 204, 89, 67, 204, 78, 67, - 204, 67, 67, 204, 56, 67, 204, 45, 67, 205, 33, 67, 205, 22, 67, 205, 11, - 67, 205, 0, 67, 204, 245, 67, 204, 234, 67, 204, 223, 208, 223, 232, 148, - 40, 81, 242, 76, 208, 223, 232, 148, 40, 81, 148, 242, 76, 208, 223, 232, - 148, 40, 81, 148, 232, 82, 201, 64, 208, 223, 232, 148, 40, 81, 242, 85, - 208, 223, 232, 148, 40, 81, 204, 26, 208, 223, 232, 148, 40, 81, 233, - 218, 77, 208, 223, 232, 148, 40, 81, 208, 15, 77, 208, 223, 232, 148, 40, - 81, 45, 63, 219, 189, 248, 55, 208, 223, 232, 148, 40, 81, 50, 63, 219, - 189, 248, 51, 208, 223, 232, 148, 40, 81, 228, 243, 234, 122, 33, 34, 45, - 230, 72, 33, 34, 50, 230, 72, 33, 55, 198, 153, 45, 230, 72, 33, 55, 198, - 153, 50, 230, 72, 33, 217, 149, 45, 230, 72, 33, 217, 149, 50, 230, 72, - 33, 239, 46, 217, 148, 33, 34, 45, 132, 60, 33, 34, 50, 132, 60, 33, 198, - 153, 45, 132, 60, 33, 198, 153, 50, 132, 60, 33, 217, 149, 45, 132, 60, - 33, 217, 149, 50, 132, 60, 33, 239, 46, 217, 149, 60, 33, 38, 198, 123, - 45, 230, 72, 33, 38, 198, 123, 50, 230, 72, 208, 223, 232, 148, 40, 81, - 105, 75, 219, 238, 208, 223, 232, 148, 40, 81, 234, 117, 237, 217, 208, - 223, 232, 148, 40, 81, 234, 106, 237, 217, 208, 223, 232, 148, 40, 81, - 130, 219, 114, 208, 223, 232, 148, 40, 81, 193, 135, 130, 219, 114, 208, - 223, 232, 148, 40, 81, 45, 210, 115, 208, 223, 232, 148, 40, 81, 50, 210, - 115, 208, 223, 232, 148, 40, 81, 45, 238, 173, 248, 55, 208, 223, 232, - 148, 40, 81, 50, 238, 173, 248, 55, 208, 223, 232, 148, 40, 81, 45, 198, - 42, 203, 104, 248, 55, 208, 223, 232, 148, 40, 81, 50, 198, 42, 203, 104, - 248, 55, 208, 223, 232, 148, 40, 81, 45, 62, 219, 189, 248, 55, 208, 223, - 232, 148, 40, 81, 50, 62, 219, 189, 248, 55, 208, 223, 232, 148, 40, 81, - 45, 55, 251, 118, 248, 55, 208, 223, 232, 148, 40, 81, 50, 55, 251, 118, - 248, 55, 208, 223, 232, 148, 40, 81, 45, 251, 118, 248, 55, 208, 223, - 232, 148, 40, 81, 50, 251, 118, 248, 55, 208, 223, 232, 148, 40, 81, 45, - 239, 4, 248, 55, 208, 223, 232, 148, 40, 81, 50, 239, 4, 248, 55, 208, - 223, 232, 148, 40, 81, 45, 63, 239, 4, 248, 55, 208, 223, 232, 148, 40, - 81, 50, 63, 239, 4, 248, 55, 204, 1, 236, 142, 63, 204, 1, 236, 142, 208, - 223, 232, 148, 40, 81, 45, 51, 248, 55, 208, 223, 232, 148, 40, 81, 50, - 51, 248, 55, 237, 216, 211, 0, 247, 20, 211, 0, 193, 135, 211, 0, 55, - 193, 135, 211, 0, 237, 216, 130, 219, 114, 247, 20, 130, 219, 114, 193, - 135, 130, 219, 114, 2, 242, 76, 2, 148, 242, 76, 2, 232, 82, 201, 64, 2, - 204, 26, 2, 242, 85, 2, 208, 15, 77, 2, 233, 218, 77, 2, 234, 117, 237, - 217, 2, 45, 210, 115, 2, 50, 210, 115, 2, 45, 238, 173, 248, 55, 2, 50, - 238, 173, 248, 55, 2, 45, 198, 42, 203, 104, 248, 55, 2, 50, 198, 42, - 203, 104, 248, 55, 2, 31, 56, 2, 251, 139, 2, 250, 195, 2, 108, 56, 2, - 228, 89, 2, 219, 182, 56, 2, 230, 206, 56, 2, 234, 45, 56, 2, 207, 20, - 202, 24, 2, 236, 157, 56, 2, 210, 15, 56, 2, 242, 74, 250, 184, 9, 233, - 71, 76, 20, 9, 199, 169, 4, 233, 71, 58, 9, 237, 246, 76, 20, 9, 199, - 216, 232, 119, 9, 222, 53, 76, 20, 9, 233, 73, 76, 20, 9, 233, 73, 216, - 219, 20, 9, 237, 248, 76, 20, 9, 237, 248, 216, 219, 20, 9, 222, 55, 76, - 20, 9, 222, 55, 216, 219, 20, 9, 203, 157, 76, 20, 9, 203, 157, 216, 219, - 20, 9, 200, 207, 76, 20, 9, 200, 207, 216, 219, 20, 9, 1, 230, 60, 76, - 20, 9, 1, 131, 4, 217, 144, 93, 76, 20, 9, 1, 131, 4, 217, 144, 93, 53, - 20, 9, 1, 131, 4, 230, 60, 93, 76, 20, 9, 1, 131, 4, 230, 60, 93, 53, 20, - 9, 1, 193, 134, 4, 230, 60, 93, 76, 20, 9, 1, 193, 134, 4, 230, 60, 93, - 53, 20, 9, 1, 131, 4, 230, 60, 248, 235, 76, 20, 9, 1, 131, 4, 230, 60, - 248, 235, 53, 20, 9, 1, 99, 4, 230, 60, 93, 76, 20, 9, 1, 99, 4, 230, 60, - 93, 53, 20, 9, 1, 99, 4, 230, 60, 93, 89, 20, 9, 1, 99, 4, 230, 60, 93, - 216, 219, 20, 9, 1, 131, 76, 20, 9, 1, 131, 53, 20, 9, 1, 248, 248, 76, - 20, 9, 1, 248, 248, 53, 20, 9, 1, 248, 248, 89, 20, 9, 1, 248, 248, 216, - 219, 20, 9, 1, 199, 115, 217, 65, 76, 20, 9, 1, 199, 115, 217, 65, 53, - 20, 9, 1, 199, 115, 76, 20, 9, 1, 199, 115, 53, 20, 9, 1, 199, 115, 89, - 20, 9, 1, 199, 115, 216, 219, 20, 9, 1, 199, 30, 76, 20, 9, 1, 199, 30, - 53, 20, 9, 1, 199, 30, 89, 20, 9, 1, 199, 30, 216, 219, 20, 9, 1, 223, 9, - 76, 20, 9, 1, 223, 9, 53, 20, 9, 1, 223, 9, 89, 20, 9, 1, 223, 9, 216, - 219, 20, 9, 1, 183, 76, 20, 9, 1, 183, 53, 20, 9, 1, 183, 89, 20, 9, 1, - 183, 216, 219, 20, 9, 1, 221, 213, 76, 20, 9, 1, 221, 213, 53, 20, 9, 1, - 221, 213, 89, 20, 9, 1, 221, 213, 216, 219, 20, 9, 1, 238, 6, 76, 20, 9, - 1, 238, 6, 53, 20, 9, 1, 199, 42, 76, 20, 9, 1, 199, 42, 53, 20, 9, 1, - 207, 73, 76, 20, 9, 1, 207, 73, 53, 20, 9, 1, 191, 114, 76, 20, 9, 1, - 191, 114, 53, 20, 9, 1, 205, 48, 76, 20, 9, 1, 205, 48, 53, 20, 9, 1, - 205, 48, 89, 20, 9, 1, 205, 48, 216, 219, 20, 9, 1, 203, 111, 76, 20, 9, - 1, 203, 111, 53, 20, 9, 1, 203, 111, 89, 20, 9, 1, 203, 111, 216, 219, - 20, 9, 1, 205, 194, 76, 20, 9, 1, 205, 194, 53, 20, 9, 1, 205, 194, 89, - 20, 9, 1, 205, 194, 216, 219, 20, 9, 1, 238, 29, 76, 20, 9, 1, 238, 29, - 53, 20, 9, 1, 238, 29, 89, 20, 9, 1, 238, 29, 216, 219, 20, 9, 1, 199, - 220, 76, 20, 9, 1, 199, 220, 53, 20, 9, 1, 199, 220, 89, 20, 9, 1, 199, - 220, 216, 219, 20, 9, 1, 191, 117, 76, 20, 9, 1, 191, 117, 53, 20, 9, 1, - 191, 117, 89, 20, 9, 1, 191, 117, 216, 219, 20, 9, 1, 251, 233, 76, 20, - 9, 1, 251, 233, 53, 20, 9, 1, 251, 233, 89, 20, 9, 1, 251, 233, 216, 219, - 20, 9, 1, 231, 89, 76, 20, 9, 1, 231, 89, 53, 20, 9, 1, 231, 89, 89, 20, - 9, 1, 231, 89, 216, 219, 20, 9, 1, 233, 49, 76, 20, 9, 1, 233, 49, 53, - 20, 9, 1, 233, 49, 89, 20, 9, 1, 233, 49, 216, 219, 20, 9, 1, 208, 92, - 76, 20, 9, 1, 208, 92, 53, 20, 9, 1, 208, 92, 89, 20, 9, 1, 208, 92, 216, - 219, 20, 9, 1, 222, 109, 76, 20, 9, 1, 222, 109, 53, 20, 9, 1, 222, 109, - 89, 20, 9, 1, 222, 109, 216, 219, 20, 9, 1, 220, 38, 76, 20, 9, 1, 220, - 38, 53, 20, 9, 1, 220, 38, 89, 20, 9, 1, 220, 38, 216, 219, 20, 9, 1, 92, - 76, 20, 9, 1, 92, 53, 20, 9, 1, 92, 89, 20, 9, 1, 92, 216, 219, 20, 9, 1, - 213, 27, 76, 20, 9, 1, 213, 27, 53, 20, 9, 1, 213, 27, 89, 20, 9, 1, 213, - 27, 216, 219, 20, 9, 1, 229, 213, 76, 20, 9, 1, 229, 213, 53, 20, 9, 1, - 229, 213, 89, 20, 9, 1, 229, 213, 216, 219, 20, 9, 1, 193, 134, 76, 20, - 9, 1, 193, 134, 53, 20, 9, 1, 131, 217, 102, 76, 20, 9, 1, 131, 217, 102, - 53, 20, 9, 1, 99, 76, 20, 9, 1, 99, 53, 20, 9, 1, 99, 89, 20, 9, 1, 99, - 216, 219, 20, 9, 34, 220, 38, 4, 131, 4, 217, 144, 93, 76, 20, 9, 34, - 220, 38, 4, 131, 4, 217, 144, 93, 53, 20, 9, 34, 220, 38, 4, 131, 4, 230, - 60, 93, 76, 20, 9, 34, 220, 38, 4, 131, 4, 230, 60, 93, 53, 20, 9, 34, - 220, 38, 4, 131, 4, 230, 60, 248, 235, 76, 20, 9, 34, 220, 38, 4, 131, 4, - 230, 60, 248, 235, 53, 20, 9, 34, 220, 38, 4, 131, 76, 20, 9, 34, 220, - 38, 4, 131, 53, 20, 191, 78, 193, 75, 213, 39, 201, 248, 232, 80, 233, - 218, 77, 232, 80, 207, 254, 77, 232, 80, 31, 56, 232, 80, 236, 157, 56, - 232, 80, 210, 15, 56, 232, 80, 251, 139, 232, 80, 251, 51, 232, 80, 45, - 210, 115, 232, 80, 50, 210, 115, 232, 80, 250, 195, 232, 80, 108, 56, - 232, 80, 242, 76, 232, 80, 228, 89, 232, 80, 232, 82, 201, 64, 232, 80, - 202, 24, 232, 80, 17, 191, 77, 232, 80, 17, 107, 232, 80, 17, 109, 232, - 80, 17, 138, 232, 80, 17, 134, 232, 80, 17, 150, 232, 80, 17, 169, 232, - 80, 17, 175, 232, 80, 17, 171, 232, 80, 17, 178, 232, 80, 242, 85, 232, - 80, 204, 26, 232, 80, 219, 182, 56, 232, 80, 234, 45, 56, 232, 80, 230, - 206, 56, 232, 80, 208, 15, 77, 232, 80, 242, 74, 250, 184, 232, 80, 8, 6, - 1, 65, 232, 80, 8, 6, 1, 250, 122, 232, 80, 8, 6, 1, 247, 195, 232, 80, - 8, 6, 1, 238, 129, 232, 80, 8, 6, 1, 71, 232, 80, 8, 6, 1, 233, 177, 232, - 80, 8, 6, 1, 232, 53, 232, 80, 8, 6, 1, 230, 118, 232, 80, 8, 6, 1, 68, - 232, 80, 8, 6, 1, 223, 37, 232, 80, 8, 6, 1, 222, 154, 232, 80, 8, 6, 1, - 172, 232, 80, 8, 6, 1, 218, 170, 232, 80, 8, 6, 1, 215, 63, 232, 80, 8, - 6, 1, 74, 232, 80, 8, 6, 1, 210, 238, 232, 80, 8, 6, 1, 208, 106, 232, - 80, 8, 6, 1, 146, 232, 80, 8, 6, 1, 206, 9, 232, 80, 8, 6, 1, 200, 43, - 232, 80, 8, 6, 1, 66, 232, 80, 8, 6, 1, 196, 12, 232, 80, 8, 6, 1, 193, - 224, 232, 80, 8, 6, 1, 192, 235, 232, 80, 8, 6, 1, 192, 159, 232, 80, 8, - 6, 1, 191, 166, 232, 80, 45, 51, 248, 55, 232, 80, 207, 20, 202, 24, 232, - 80, 50, 51, 248, 55, 232, 80, 243, 4, 252, 62, 232, 80, 130, 219, 114, - 232, 80, 230, 213, 252, 62, 232, 80, 8, 2, 1, 65, 232, 80, 8, 2, 1, 250, - 122, 232, 80, 8, 2, 1, 247, 195, 232, 80, 8, 2, 1, 238, 129, 232, 80, 8, - 2, 1, 71, 232, 80, 8, 2, 1, 233, 177, 232, 80, 8, 2, 1, 232, 53, 232, 80, - 8, 2, 1, 230, 118, 232, 80, 8, 2, 1, 68, 232, 80, 8, 2, 1, 223, 37, 232, - 80, 8, 2, 1, 222, 154, 232, 80, 8, 2, 1, 172, 232, 80, 8, 2, 1, 218, 170, - 232, 80, 8, 2, 1, 215, 63, 232, 80, 8, 2, 1, 74, 232, 80, 8, 2, 1, 210, - 238, 232, 80, 8, 2, 1, 208, 106, 232, 80, 8, 2, 1, 146, 232, 80, 8, 2, 1, - 206, 9, 232, 80, 8, 2, 1, 200, 43, 232, 80, 8, 2, 1, 66, 232, 80, 8, 2, - 1, 196, 12, 232, 80, 8, 2, 1, 193, 224, 232, 80, 8, 2, 1, 192, 235, 232, - 80, 8, 2, 1, 192, 159, 232, 80, 8, 2, 1, 191, 166, 232, 80, 45, 238, 173, - 248, 55, 232, 80, 81, 219, 114, 232, 80, 50, 238, 173, 248, 55, 232, 80, - 198, 152, 232, 80, 45, 63, 210, 115, 232, 80, 50, 63, 210, 115, 151, 148, - 232, 82, 201, 64, 151, 45, 239, 4, 248, 55, 151, 50, 239, 4, 248, 55, - 151, 148, 242, 76, 151, 72, 82, 236, 142, 151, 72, 1, 193, 48, 151, 72, - 1, 2, 65, 151, 72, 1, 2, 68, 151, 72, 1, 2, 66, 151, 72, 1, 2, 71, 151, - 72, 1, 2, 74, 151, 72, 1, 2, 170, 151, 72, 1, 2, 191, 225, 151, 72, 1, 2, - 192, 12, 151, 72, 1, 2, 197, 94, 151, 222, 50, 208, 193, 202, 6, 77, 151, - 72, 1, 65, 151, 72, 1, 68, 151, 72, 1, 66, 151, 72, 1, 71, 151, 72, 1, - 74, 151, 72, 1, 155, 151, 72, 1, 221, 168, 151, 72, 1, 220, 234, 151, 72, - 1, 222, 24, 151, 72, 1, 221, 69, 151, 72, 1, 188, 151, 72, 1, 202, 223, - 151, 72, 1, 201, 5, 151, 72, 1, 205, 69, 151, 72, 1, 202, 47, 151, 72, 1, - 190, 190, 151, 72, 1, 198, 193, 151, 72, 1, 197, 94, 151, 72, 1, 199, - 145, 151, 72, 1, 159, 151, 72, 1, 181, 151, 72, 1, 213, 221, 151, 72, 1, - 212, 180, 151, 72, 1, 214, 123, 151, 72, 1, 213, 45, 151, 72, 1, 140, - 151, 72, 1, 229, 160, 151, 72, 1, 228, 161, 151, 72, 1, 229, 247, 151, - 72, 1, 229, 25, 151, 72, 1, 174, 151, 72, 1, 216, 102, 151, 72, 1, 215, - 157, 151, 72, 1, 216, 234, 151, 72, 1, 216, 14, 151, 72, 1, 170, 151, 72, - 1, 191, 225, 151, 72, 1, 192, 12, 151, 72, 1, 165, 151, 72, 1, 207, 2, - 151, 72, 1, 206, 69, 151, 72, 1, 207, 115, 151, 72, 1, 206, 163, 151, 72, - 1, 193, 190, 151, 72, 1, 215, 63, 151, 72, 195, 20, 202, 6, 77, 151, 72, - 204, 31, 202, 6, 77, 151, 30, 233, 5, 151, 30, 1, 221, 115, 151, 30, 1, - 201, 168, 151, 30, 1, 221, 108, 151, 30, 1, 213, 206, 151, 30, 1, 213, - 204, 151, 30, 1, 213, 203, 151, 30, 1, 198, 168, 151, 30, 1, 201, 157, - 151, 30, 1, 206, 240, 151, 30, 1, 206, 235, 151, 30, 1, 206, 232, 151, - 30, 1, 206, 225, 151, 30, 1, 206, 220, 151, 30, 1, 206, 215, 151, 30, 1, - 206, 226, 151, 30, 1, 206, 238, 151, 30, 1, 216, 79, 151, 30, 1, 209, - 171, 151, 30, 1, 201, 165, 151, 30, 1, 209, 160, 151, 30, 1, 202, 161, - 151, 30, 1, 201, 162, 151, 30, 1, 223, 224, 151, 30, 1, 243, 26, 151, 30, - 1, 201, 172, 151, 30, 1, 243, 93, 151, 30, 1, 221, 190, 151, 30, 1, 199, - 7, 151, 30, 1, 209, 211, 151, 30, 1, 229, 144, 151, 30, 1, 65, 151, 30, - 1, 252, 27, 151, 30, 1, 170, 151, 30, 1, 192, 129, 151, 30, 1, 234, 67, - 151, 30, 1, 71, 151, 30, 1, 192, 67, 151, 30, 1, 192, 80, 151, 30, 1, 74, - 151, 30, 1, 193, 190, 151, 30, 1, 193, 176, 151, 30, 1, 211, 153, 151, - 30, 1, 192, 12, 151, 30, 1, 66, 151, 30, 1, 193, 107, 151, 30, 1, 193, - 125, 151, 30, 1, 193, 86, 151, 30, 1, 191, 225, 151, 30, 1, 233, 244, - 151, 30, 1, 192, 33, 151, 30, 1, 68, 232, 80, 247, 26, 56, 232, 80, 209, - 10, 56, 232, 80, 213, 14, 56, 232, 80, 217, 148, 232, 80, 248, 24, 164, - 232, 80, 192, 71, 56, 232, 80, 193, 31, 56, 151, 232, 143, 156, 195, 135, - 151, 118, 57, 151, 196, 66, 57, 151, 96, 57, 151, 235, 121, 57, 151, 62, - 201, 191, 151, 63, 243, 12, 223, 108, 251, 98, 251, 129, 223, 108, 251, - 98, 204, 11, 223, 108, 251, 98, 199, 80, 211, 177, 207, 44, 246, 241, - 207, 44, 246, 241, 32, 78, 5, 250, 106, 65, 32, 78, 5, 250, 75, 71, 32, - 78, 5, 250, 84, 68, 32, 78, 5, 250, 52, 74, 32, 78, 5, 250, 102, 66, 32, - 78, 5, 250, 121, 238, 34, 32, 78, 5, 250, 68, 237, 148, 32, 78, 5, 250, - 108, 237, 46, 32, 78, 5, 250, 98, 236, 176, 32, 78, 5, 250, 62, 235, 91, - 32, 78, 5, 250, 56, 223, 34, 32, 78, 5, 250, 67, 223, 12, 32, 78, 5, 250, - 77, 222, 203, 32, 78, 5, 250, 48, 222, 184, 32, 78, 5, 250, 36, 155, 32, - 78, 5, 250, 69, 222, 24, 32, 78, 5, 250, 46, 221, 168, 32, 78, 5, 250, - 43, 221, 69, 32, 78, 5, 250, 32, 220, 234, 32, 78, 5, 250, 33, 174, 32, - 78, 5, 250, 99, 216, 234, 32, 78, 5, 250, 40, 216, 102, 32, 78, 5, 250, - 97, 216, 14, 32, 78, 5, 250, 89, 215, 157, 32, 78, 5, 250, 110, 181, 32, - 78, 5, 250, 88, 214, 123, 32, 78, 5, 250, 82, 213, 221, 32, 78, 5, 250, - 61, 213, 45, 32, 78, 5, 250, 58, 212, 180, 32, 78, 5, 250, 117, 168, 32, - 78, 5, 250, 41, 210, 65, 32, 78, 5, 250, 74, 209, 187, 32, 78, 5, 250, - 101, 209, 75, 32, 78, 5, 250, 63, 208, 167, 32, 78, 5, 250, 96, 208, 98, - 32, 78, 5, 250, 35, 208, 77, 32, 78, 5, 250, 91, 208, 59, 32, 78, 5, 250, - 80, 208, 47, 32, 78, 5, 250, 53, 165, 32, 78, 5, 250, 85, 207, 115, 32, - 78, 5, 250, 60, 207, 2, 32, 78, 5, 250, 119, 206, 163, 32, 78, 5, 250, - 86, 206, 69, 32, 78, 5, 250, 81, 188, 32, 78, 5, 250, 104, 205, 69, 32, - 78, 5, 250, 72, 202, 223, 32, 78, 5, 250, 100, 202, 47, 32, 78, 5, 250, - 55, 201, 5, 32, 78, 5, 250, 54, 190, 190, 32, 78, 5, 250, 115, 199, 145, - 32, 78, 5, 250, 76, 198, 193, 32, 78, 5, 250, 113, 159, 32, 78, 5, 250, - 44, 197, 94, 32, 78, 5, 250, 59, 193, 190, 32, 78, 5, 250, 38, 193, 125, - 32, 78, 5, 250, 73, 193, 86, 32, 78, 5, 250, 71, 193, 48, 32, 78, 5, 250, - 95, 191, 123, 32, 78, 5, 250, 39, 191, 87, 32, 78, 5, 250, 92, 191, 7, - 32, 78, 5, 250, 87, 254, 217, 32, 78, 5, 250, 70, 254, 105, 32, 78, 5, - 250, 29, 250, 165, 32, 78, 5, 250, 42, 235, 47, 32, 78, 5, 250, 25, 235, - 46, 32, 78, 5, 250, 65, 212, 112, 32, 78, 5, 250, 83, 208, 165, 32, 78, - 5, 250, 51, 208, 169, 32, 78, 5, 250, 37, 207, 182, 32, 78, 5, 250, 79, - 207, 181, 32, 78, 5, 250, 45, 206, 156, 32, 78, 5, 250, 47, 199, 246, 32, - 78, 5, 250, 27, 197, 41, 32, 78, 5, 250, 24, 109, 32, 78, 16, 250, 94, - 32, 78, 16, 250, 93, 32, 78, 16, 250, 90, 32, 78, 16, 250, 78, 32, 78, - 16, 250, 66, 32, 78, 16, 250, 64, 32, 78, 16, 250, 57, 32, 78, 16, 250, - 50, 32, 78, 16, 250, 49, 32, 78, 16, 250, 34, 32, 78, 16, 250, 31, 32, - 78, 16, 250, 30, 32, 78, 16, 250, 28, 32, 78, 16, 250, 26, 32, 78, 157, - 250, 23, 217, 92, 32, 78, 157, 250, 22, 193, 35, 32, 78, 157, 250, 21, - 237, 130, 32, 78, 157, 250, 20, 234, 42, 32, 78, 157, 250, 19, 217, 58, - 32, 78, 157, 250, 18, 201, 111, 32, 78, 157, 250, 17, 233, 225, 32, 78, - 157, 250, 16, 207, 144, 32, 78, 157, 250, 15, 203, 113, 32, 78, 157, 250, - 14, 229, 239, 32, 78, 157, 250, 13, 202, 0, 32, 78, 157, 250, 12, 248, - 111, 32, 78, 157, 250, 11, 238, 241, 32, 78, 157, 250, 10, 247, 252, 32, - 78, 157, 250, 9, 193, 95, 32, 78, 157, 250, 8, 249, 86, 32, 78, 157, 250, - 7, 211, 117, 32, 78, 157, 250, 6, 201, 224, 32, 78, 157, 250, 5, 238, - 138, 32, 78, 215, 223, 250, 4, 222, 76, 32, 78, 215, 223, 250, 3, 222, - 87, 32, 78, 157, 250, 2, 211, 133, 32, 78, 157, 250, 1, 193, 62, 32, 78, - 157, 250, 0, 32, 78, 215, 223, 249, 255, 251, 9, 32, 78, 215, 223, 249, - 254, 216, 180, 32, 78, 157, 249, 253, 248, 23, 32, 78, 157, 249, 252, - 230, 252, 32, 78, 157, 249, 251, 32, 78, 157, 249, 250, 193, 26, 32, 78, - 157, 249, 249, 32, 78, 157, 249, 248, 32, 78, 157, 249, 247, 228, 189, - 32, 78, 157, 249, 246, 32, 78, 157, 249, 245, 32, 78, 157, 249, 244, 32, - 78, 215, 223, 249, 242, 197, 56, 32, 78, 157, 249, 241, 32, 78, 157, 249, - 240, 32, 78, 157, 249, 239, 242, 215, 32, 78, 157, 249, 238, 32, 78, 157, - 249, 237, 32, 78, 157, 249, 236, 231, 198, 32, 78, 157, 249, 235, 250, - 250, 32, 78, 157, 249, 234, 32, 78, 157, 249, 233, 32, 78, 157, 249, 232, - 32, 78, 157, 249, 231, 32, 78, 157, 249, 230, 32, 78, 157, 249, 229, 32, - 78, 157, 249, 228, 32, 78, 157, 249, 227, 32, 78, 157, 249, 226, 32, 78, - 157, 249, 225, 215, 215, 32, 78, 157, 249, 224, 32, 78, 157, 249, 223, - 197, 254, 32, 78, 157, 249, 222, 32, 78, 157, 249, 221, 32, 78, 157, 249, - 220, 32, 78, 157, 249, 219, 32, 78, 157, 249, 218, 32, 78, 157, 249, 217, - 32, 78, 157, 249, 216, 32, 78, 157, 249, 215, 32, 78, 157, 249, 214, 32, - 78, 157, 249, 213, 32, 78, 157, 249, 212, 32, 78, 157, 249, 211, 229, - 202, 32, 78, 157, 249, 190, 232, 157, 32, 78, 157, 249, 187, 249, 61, 32, - 78, 157, 249, 182, 201, 233, 32, 78, 157, 249, 181, 57, 32, 78, 157, 249, - 180, 32, 78, 157, 249, 179, 200, 131, 32, 78, 157, 249, 178, 32, 78, 157, - 249, 177, 32, 78, 157, 249, 176, 193, 90, 243, 140, 32, 78, 157, 249, - 175, 243, 140, 32, 78, 157, 249, 174, 243, 141, 232, 115, 32, 78, 157, - 249, 173, 193, 93, 32, 78, 157, 249, 172, 32, 78, 157, 249, 171, 32, 78, - 215, 223, 249, 170, 236, 237, 32, 78, 157, 249, 169, 32, 78, 157, 249, - 168, 32, 78, 157, 249, 166, 32, 78, 157, 249, 165, 32, 78, 157, 249, 164, - 32, 78, 157, 249, 163, 237, 220, 32, 78, 157, 249, 162, 32, 78, 157, 249, - 161, 32, 78, 157, 249, 160, 32, 78, 157, 249, 159, 32, 78, 157, 249, 158, - 32, 78, 157, 195, 82, 249, 243, 32, 78, 157, 195, 82, 249, 210, 32, 78, - 157, 195, 82, 249, 209, 32, 78, 157, 195, 82, 249, 208, 32, 78, 157, 195, - 82, 249, 207, 32, 78, 157, 195, 82, 249, 206, 32, 78, 157, 195, 82, 249, - 205, 32, 78, 157, 195, 82, 249, 204, 32, 78, 157, 195, 82, 249, 203, 32, - 78, 157, 195, 82, 249, 202, 32, 78, 157, 195, 82, 249, 201, 32, 78, 157, - 195, 82, 249, 200, 32, 78, 157, 195, 82, 249, 199, 32, 78, 157, 195, 82, - 249, 198, 32, 78, 157, 195, 82, 249, 197, 32, 78, 157, 195, 82, 249, 196, - 32, 78, 157, 195, 82, 249, 195, 32, 78, 157, 195, 82, 249, 194, 32, 78, - 157, 195, 82, 249, 193, 32, 78, 157, 195, 82, 249, 192, 32, 78, 157, 195, - 82, 249, 191, 32, 78, 157, 195, 82, 249, 189, 32, 78, 157, 195, 82, 249, - 188, 32, 78, 157, 195, 82, 249, 186, 32, 78, 157, 195, 82, 249, 185, 32, - 78, 157, 195, 82, 249, 184, 32, 78, 157, 195, 82, 249, 183, 32, 78, 157, - 195, 82, 249, 167, 32, 78, 157, 195, 82, 249, 157, 252, 20, 193, 23, 204, - 12, 219, 114, 252, 20, 193, 23, 204, 12, 236, 142, 252, 20, 243, 128, 77, - 252, 20, 31, 107, 252, 20, 31, 109, 252, 20, 31, 138, 252, 20, 31, 134, - 252, 20, 31, 150, 252, 20, 31, 169, 252, 20, 31, 175, 252, 20, 31, 171, - 252, 20, 31, 178, 252, 20, 31, 199, 95, 252, 20, 31, 197, 32, 252, 20, - 31, 198, 249, 252, 20, 31, 232, 137, 252, 20, 31, 233, 17, 252, 20, 31, - 202, 121, 252, 20, 31, 203, 242, 252, 20, 31, 234, 155, 252, 20, 31, 213, - 171, 252, 20, 31, 91, 228, 142, 252, 20, 31, 105, 228, 142, 252, 20, 31, - 115, 228, 142, 252, 20, 31, 232, 130, 228, 142, 252, 20, 31, 232, 228, - 228, 142, 252, 20, 31, 202, 137, 228, 142, 252, 20, 31, 203, 248, 228, - 142, 252, 20, 31, 234, 166, 228, 142, 252, 20, 31, 213, 177, 228, 142, - 252, 20, 31, 91, 189, 252, 20, 31, 105, 189, 252, 20, 31, 115, 189, 252, - 20, 31, 232, 130, 189, 252, 20, 31, 232, 228, 189, 252, 20, 31, 202, 137, - 189, 252, 20, 31, 203, 248, 189, 252, 20, 31, 234, 166, 189, 252, 20, 31, - 213, 177, 189, 252, 20, 31, 199, 96, 189, 252, 20, 31, 197, 33, 189, 252, - 20, 31, 198, 250, 189, 252, 20, 31, 232, 138, 189, 252, 20, 31, 233, 18, - 189, 252, 20, 31, 202, 122, 189, 252, 20, 31, 203, 243, 189, 252, 20, 31, - 234, 156, 189, 252, 20, 31, 213, 172, 189, 252, 20, 193, 110, 249, 77, - 196, 90, 252, 20, 193, 110, 232, 240, 200, 224, 252, 20, 193, 110, 205, - 58, 200, 224, 252, 20, 193, 110, 199, 1, 200, 224, 252, 20, 193, 110, - 232, 123, 200, 224, 252, 20, 235, 94, 216, 230, 232, 240, 200, 224, 252, - 20, 219, 95, 216, 230, 232, 240, 200, 224, 252, 20, 216, 230, 205, 58, - 200, 224, 252, 20, 216, 230, 199, 1, 200, 224, 35, 252, 52, 250, 167, 91, - 208, 24, 35, 252, 52, 250, 167, 91, 230, 72, 35, 252, 52, 250, 167, 91, - 235, 117, 35, 252, 52, 250, 167, 150, 35, 252, 52, 250, 167, 233, 17, 35, - 252, 52, 250, 167, 232, 228, 228, 142, 35, 252, 52, 250, 167, 232, 228, - 189, 35, 252, 52, 250, 167, 233, 18, 189, 35, 252, 52, 250, 167, 232, - 228, 199, 203, 35, 252, 52, 250, 167, 199, 96, 199, 203, 35, 252, 52, - 250, 167, 233, 18, 199, 203, 35, 252, 52, 250, 167, 91, 228, 143, 199, - 203, 35, 252, 52, 250, 167, 232, 228, 228, 143, 199, 203, 35, 252, 52, - 250, 167, 91, 198, 230, 199, 203, 35, 252, 52, 250, 167, 232, 228, 198, - 230, 199, 203, 35, 252, 52, 250, 167, 232, 228, 201, 95, 35, 252, 52, - 250, 167, 199, 96, 201, 95, 35, 252, 52, 250, 167, 233, 18, 201, 95, 35, - 252, 52, 250, 167, 91, 228, 143, 201, 95, 35, 252, 52, 250, 167, 232, - 228, 228, 143, 201, 95, 35, 252, 52, 250, 167, 91, 198, 230, 201, 95, 35, - 252, 52, 250, 167, 199, 96, 198, 230, 201, 95, 35, 252, 52, 250, 167, - 233, 18, 198, 230, 201, 95, 35, 252, 52, 250, 167, 199, 96, 216, 17, 35, - 252, 52, 229, 196, 91, 209, 94, 35, 252, 52, 199, 17, 107, 35, 252, 52, - 229, 192, 107, 35, 252, 52, 234, 53, 109, 35, 252, 52, 199, 17, 109, 35, - 252, 52, 238, 134, 105, 235, 116, 35, 252, 52, 234, 53, 105, 235, 116, - 35, 252, 52, 197, 216, 150, 35, 252, 52, 197, 216, 199, 95, 35, 252, 52, - 197, 216, 199, 96, 251, 159, 20, 35, 252, 52, 229, 192, 199, 95, 35, 252, - 52, 216, 169, 199, 95, 35, 252, 52, 199, 17, 199, 95, 35, 252, 52, 199, - 17, 198, 249, 35, 252, 52, 197, 216, 233, 17, 35, 252, 52, 197, 216, 233, - 18, 251, 159, 20, 35, 252, 52, 229, 192, 233, 17, 35, 252, 52, 199, 17, - 233, 17, 35, 252, 52, 199, 17, 91, 228, 142, 35, 252, 52, 199, 17, 115, - 228, 142, 35, 252, 52, 234, 53, 232, 228, 228, 142, 35, 252, 52, 197, - 216, 232, 228, 228, 142, 35, 252, 52, 199, 17, 232, 228, 228, 142, 35, - 252, 52, 247, 84, 232, 228, 228, 142, 35, 252, 52, 214, 201, 232, 228, - 228, 142, 35, 252, 52, 199, 17, 91, 189, 35, 252, 52, 199, 17, 232, 228, - 189, 35, 252, 52, 237, 111, 232, 228, 216, 17, 35, 252, 52, 201, 48, 233, - 18, 216, 17, 35, 91, 132, 56, 35, 91, 132, 3, 251, 159, 20, 35, 105, 198, - 254, 56, 35, 115, 208, 23, 56, 35, 192, 78, 56, 35, 199, 204, 56, 35, - 235, 118, 56, 35, 211, 172, 56, 35, 105, 211, 171, 56, 35, 115, 211, 171, - 56, 35, 232, 130, 211, 171, 56, 35, 232, 228, 211, 171, 56, 35, 216, 163, - 56, 35, 220, 153, 249, 77, 56, 35, 219, 87, 56, 35, 211, 18, 56, 35, 192, - 211, 56, 35, 250, 228, 56, 35, 250, 245, 56, 35, 230, 222, 56, 35, 197, - 171, 249, 77, 56, 35, 191, 78, 56, 35, 91, 208, 25, 56, 35, 202, 163, 56, - 35, 223, 145, 56, 213, 34, 56, 206, 137, 203, 238, 56, 206, 137, 196, - 106, 56, 206, 137, 204, 18, 56, 206, 137, 203, 176, 56, 206, 137, 236, - 252, 203, 176, 56, 206, 137, 202, 187, 56, 206, 137, 237, 106, 56, 206, - 137, 208, 7, 56, 206, 137, 203, 255, 56, 206, 137, 235, 69, 56, 206, 137, - 250, 222, 56, 206, 137, 247, 19, 56, 250, 213, 113, 35, 16, 199, 167, - 207, 4, 209, 225, 236, 229, 3, 210, 53, 209, 225, 236, 229, 3, 209, 86, - 229, 237, 209, 225, 236, 229, 3, 199, 170, 229, 237, 209, 225, 236, 229, - 3, 247, 107, 209, 225, 236, 229, 3, 243, 88, 209, 225, 236, 229, 3, 193, - 35, 209, 225, 236, 229, 3, 229, 202, 209, 225, 236, 229, 3, 231, 190, - 209, 225, 236, 229, 3, 198, 184, 209, 225, 236, 229, 3, 57, 209, 225, - 236, 229, 3, 248, 71, 209, 225, 236, 229, 3, 203, 79, 209, 225, 236, 229, - 3, 242, 208, 209, 225, 236, 229, 3, 217, 91, 209, 225, 236, 229, 3, 217, - 28, 209, 225, 236, 229, 3, 205, 109, 209, 225, 236, 229, 3, 219, 143, - 209, 225, 236, 229, 3, 248, 94, 209, 225, 236, 229, 3, 247, 91, 209, 103, - 209, 225, 236, 229, 3, 236, 158, 209, 225, 236, 229, 3, 242, 82, 209, - 225, 236, 229, 3, 202, 84, 209, 225, 236, 229, 3, 242, 83, 209, 225, 236, - 229, 3, 249, 0, 209, 225, 236, 229, 3, 203, 66, 209, 225, 236, 229, 3, - 228, 189, 209, 225, 236, 229, 3, 229, 150, 209, 225, 236, 229, 3, 247, - 247, 219, 214, 209, 225, 236, 229, 3, 247, 80, 209, 225, 236, 229, 3, - 207, 144, 209, 225, 236, 229, 3, 234, 216, 209, 225, 236, 229, 3, 235, - 126, 209, 225, 236, 229, 3, 197, 72, 209, 225, 236, 229, 3, 249, 3, 209, - 225, 236, 229, 3, 209, 104, 197, 254, 209, 225, 236, 229, 3, 195, 47, - 209, 225, 236, 229, 3, 210, 134, 209, 225, 236, 229, 3, 206, 126, 209, - 225, 236, 229, 3, 219, 127, 209, 225, 236, 229, 3, 210, 250, 249, 148, - 209, 225, 236, 229, 3, 232, 184, 209, 225, 236, 229, 3, 230, 214, 209, - 225, 236, 229, 3, 201, 51, 209, 225, 236, 229, 3, 2, 250, 134, 209, 225, - 236, 229, 3, 193, 135, 249, 99, 209, 225, 236, 229, 3, 33, 211, 174, 106, - 218, 183, 1, 65, 218, 183, 1, 71, 218, 183, 1, 250, 122, 218, 183, 1, - 248, 206, 218, 183, 1, 232, 53, 218, 183, 1, 238, 129, 218, 183, 1, 68, - 218, 183, 1, 193, 224, 218, 183, 1, 191, 166, 218, 183, 1, 199, 51, 218, - 183, 1, 223, 37, 218, 183, 1, 222, 154, 218, 183, 1, 208, 106, 218, 183, - 1, 172, 218, 183, 1, 218, 170, 218, 183, 1, 215, 63, 218, 183, 1, 216, - 19, 218, 183, 1, 213, 82, 218, 183, 1, 66, 218, 183, 1, 210, 238, 218, - 183, 1, 221, 104, 218, 183, 1, 146, 218, 183, 1, 206, 9, 218, 183, 1, - 200, 43, 218, 183, 1, 197, 135, 218, 183, 1, 251, 134, 218, 183, 1, 234, - 105, 218, 183, 1, 230, 118, 218, 183, 1, 192, 235, 247, 97, 1, 65, 247, - 97, 1, 210, 224, 247, 97, 1, 238, 129, 247, 97, 1, 172, 247, 97, 1, 196, - 28, 247, 97, 1, 146, 247, 97, 1, 219, 244, 247, 97, 1, 254, 217, 247, 97, - 1, 208, 106, 247, 97, 1, 250, 122, 247, 97, 1, 218, 170, 247, 97, 1, 74, - 247, 97, 1, 238, 36, 247, 97, 1, 200, 43, 247, 97, 1, 203, 168, 247, 97, - 1, 203, 167, 247, 97, 1, 206, 9, 247, 97, 1, 247, 194, 247, 97, 1, 66, - 247, 97, 1, 213, 82, 247, 97, 1, 192, 235, 247, 97, 1, 215, 63, 247, 97, - 1, 197, 134, 247, 97, 1, 210, 238, 247, 97, 1, 201, 179, 247, 97, 1, 68, - 247, 97, 1, 71, 247, 97, 1, 196, 25, 247, 97, 1, 222, 154, 247, 97, 1, - 222, 145, 247, 97, 1, 214, 166, 247, 97, 1, 196, 30, 247, 97, 1, 232, 53, - 247, 97, 1, 231, 244, 247, 97, 1, 201, 119, 247, 97, 1, 201, 118, 247, - 97, 1, 214, 72, 247, 97, 1, 223, 201, 247, 97, 1, 247, 193, 247, 97, 1, - 197, 135, 247, 97, 1, 196, 27, 247, 97, 1, 206, 111, 247, 97, 1, 217, 18, - 247, 97, 1, 217, 17, 247, 97, 1, 217, 16, 247, 97, 1, 217, 15, 247, 97, - 1, 219, 243, 247, 97, 1, 234, 220, 247, 97, 1, 196, 26, 94, 234, 56, 198, - 229, 77, 94, 234, 56, 17, 107, 94, 234, 56, 17, 109, 94, 234, 56, 17, - 138, 94, 234, 56, 17, 134, 94, 234, 56, 17, 150, 94, 234, 56, 17, 169, - 94, 234, 56, 17, 175, 94, 234, 56, 17, 171, 94, 234, 56, 17, 178, 94, - 234, 56, 31, 199, 95, 94, 234, 56, 31, 197, 32, 94, 234, 56, 31, 198, - 249, 94, 234, 56, 31, 232, 137, 94, 234, 56, 31, 233, 17, 94, 234, 56, - 31, 202, 121, 94, 234, 56, 31, 203, 242, 94, 234, 56, 31, 234, 155, 94, - 234, 56, 31, 213, 171, 94, 234, 56, 31, 91, 228, 142, 94, 234, 56, 31, - 105, 228, 142, 94, 234, 56, 31, 115, 228, 142, 94, 234, 56, 31, 232, 130, - 228, 142, 94, 234, 56, 31, 232, 228, 228, 142, 94, 234, 56, 31, 202, 137, - 228, 142, 94, 234, 56, 31, 203, 248, 228, 142, 94, 234, 56, 31, 234, 166, - 228, 142, 94, 234, 56, 31, 213, 177, 228, 142, 39, 43, 1, 65, 39, 43, 1, - 249, 19, 39, 43, 1, 222, 24, 39, 43, 1, 237, 148, 39, 43, 1, 71, 39, 43, - 1, 195, 153, 39, 43, 1, 191, 87, 39, 43, 1, 229, 247, 39, 43, 1, 199, 33, - 39, 43, 1, 68, 39, 43, 1, 155, 39, 43, 1, 234, 142, 39, 43, 1, 234, 116, - 39, 43, 1, 234, 105, 39, 43, 1, 234, 14, 39, 43, 1, 74, 39, 43, 1, 210, - 65, 39, 43, 1, 203, 114, 39, 43, 1, 220, 234, 39, 43, 1, 234, 36, 39, 43, - 1, 234, 24, 39, 43, 1, 199, 145, 39, 43, 1, 66, 39, 43, 1, 234, 145, 39, - 43, 1, 209, 216, 39, 43, 1, 221, 199, 39, 43, 1, 234, 183, 39, 43, 1, - 234, 26, 39, 43, 1, 243, 129, 39, 43, 1, 223, 201, 39, 43, 1, 196, 30, - 39, 43, 1, 234, 7, 39, 43, 212, 136, 107, 39, 43, 212, 136, 150, 39, 43, - 212, 136, 199, 95, 39, 43, 212, 136, 233, 17, 39, 43, 1, 192, 80, 39, 43, - 1, 213, 18, 197, 161, 39, 43, 1, 202, 1, 197, 161, 230, 233, 1, 251, 241, - 230, 233, 1, 249, 119, 230, 233, 1, 231, 52, 230, 233, 1, 238, 15, 230, - 233, 1, 251, 236, 230, 233, 1, 208, 89, 230, 233, 1, 223, 50, 230, 233, - 1, 230, 85, 230, 233, 1, 198, 243, 230, 233, 1, 234, 153, 230, 233, 1, - 220, 191, 230, 233, 1, 220, 102, 230, 233, 1, 217, 82, 230, 233, 1, 214, - 203, 230, 233, 1, 223, 3, 230, 233, 1, 196, 48, 230, 233, 1, 210, 197, - 230, 233, 1, 213, 171, 230, 233, 1, 207, 157, 230, 233, 1, 205, 113, 230, - 233, 1, 199, 111, 230, 233, 1, 193, 59, 230, 233, 1, 233, 91, 230, 233, - 1, 223, 205, 230, 233, 1, 228, 125, 230, 233, 1, 211, 31, 230, 233, 1, - 213, 177, 228, 142, 39, 210, 4, 1, 251, 134, 39, 210, 4, 1, 247, 232, 39, - 210, 4, 1, 231, 226, 39, 210, 4, 1, 236, 162, 39, 210, 4, 1, 71, 39, 210, - 4, 1, 191, 53, 39, 210, 4, 1, 235, 29, 39, 210, 4, 1, 191, 95, 39, 210, - 4, 1, 235, 27, 39, 210, 4, 1, 68, 39, 210, 4, 1, 221, 52, 39, 210, 4, 1, - 219, 210, 39, 210, 4, 1, 216, 186, 39, 210, 4, 1, 214, 102, 39, 210, 4, - 1, 195, 7, 39, 210, 4, 1, 210, 50, 39, 210, 4, 1, 207, 71, 39, 210, 4, 1, - 202, 194, 39, 210, 4, 1, 199, 217, 39, 210, 4, 1, 66, 39, 210, 4, 1, 243, - 108, 39, 210, 4, 1, 203, 48, 39, 210, 4, 1, 203, 116, 39, 210, 4, 1, 191, - 227, 39, 210, 4, 1, 192, 58, 39, 210, 4, 1, 74, 39, 210, 4, 1, 211, 89, - 39, 210, 4, 1, 234, 183, 39, 210, 4, 1, 140, 39, 210, 4, 1, 197, 145, 39, - 210, 4, 1, 195, 140, 39, 210, 4, 1, 192, 62, 39, 210, 4, 1, 192, 60, 39, - 210, 4, 1, 192, 95, 39, 210, 4, 1, 223, 228, 39, 210, 4, 1, 191, 225, 39, - 210, 4, 1, 170, 39, 210, 4, 1, 228, 37, 33, 39, 210, 4, 1, 251, 134, 33, - 39, 210, 4, 1, 236, 162, 33, 39, 210, 4, 1, 191, 95, 33, 39, 210, 4, 1, - 214, 102, 33, 39, 210, 4, 1, 202, 194, 196, 142, 1, 251, 166, 196, 142, - 1, 248, 214, 196, 142, 1, 231, 214, 196, 142, 1, 221, 217, 196, 142, 1, - 237, 108, 196, 142, 1, 229, 25, 196, 142, 1, 193, 48, 196, 142, 1, 191, - 76, 196, 142, 1, 228, 181, 196, 142, 1, 199, 73, 196, 142, 1, 191, 250, - 196, 142, 1, 222, 108, 196, 142, 1, 203, 70, 196, 142, 1, 220, 33, 196, - 142, 1, 216, 195, 196, 142, 1, 237, 66, 196, 142, 1, 212, 132, 196, 142, - 1, 190, 251, 196, 142, 1, 205, 148, 196, 142, 1, 251, 232, 196, 142, 1, - 208, 167, 196, 142, 1, 205, 192, 196, 142, 1, 208, 40, 196, 142, 1, 207, - 135, 196, 142, 1, 199, 37, 196, 142, 1, 231, 88, 196, 142, 1, 159, 196, - 142, 1, 68, 196, 142, 1, 66, 196, 142, 1, 201, 130, 196, 142, 193, 23, - 236, 207, 39, 209, 254, 3, 65, 39, 209, 254, 3, 68, 39, 209, 254, 3, 66, - 39, 209, 254, 3, 155, 39, 209, 254, 3, 220, 234, 39, 209, 254, 3, 231, - 242, 39, 209, 254, 3, 230, 181, 39, 209, 254, 3, 192, 220, 39, 209, 254, - 3, 247, 162, 39, 209, 254, 3, 223, 34, 39, 209, 254, 3, 222, 246, 39, - 209, 254, 3, 190, 190, 39, 209, 254, 3, 197, 94, 39, 209, 254, 3, 238, - 34, 39, 209, 254, 3, 237, 46, 39, 209, 254, 3, 235, 91, 39, 209, 254, 3, - 199, 49, 39, 209, 254, 3, 168, 39, 209, 254, 3, 249, 155, 39, 209, 254, - 3, 233, 111, 39, 209, 254, 3, 181, 39, 209, 254, 3, 212, 180, 39, 209, - 254, 3, 174, 39, 209, 254, 3, 216, 102, 39, 209, 254, 3, 215, 157, 39, - 209, 254, 3, 170, 39, 209, 254, 3, 195, 188, 39, 209, 254, 3, 195, 69, - 39, 209, 254, 3, 165, 39, 209, 254, 3, 206, 69, 39, 209, 254, 3, 173, 39, - 209, 254, 3, 188, 39, 209, 254, 3, 191, 123, 39, 209, 254, 3, 203, 166, - 39, 209, 254, 3, 201, 176, 39, 209, 254, 3, 140, 39, 209, 254, 3, 250, - 159, 39, 209, 254, 3, 250, 158, 39, 209, 254, 3, 250, 157, 39, 209, 254, - 3, 192, 189, 39, 209, 254, 3, 238, 11, 39, 209, 254, 3, 238, 10, 39, 209, - 254, 3, 249, 130, 39, 209, 254, 3, 247, 214, 39, 209, 254, 193, 23, 236, - 207, 39, 209, 254, 31, 107, 39, 209, 254, 31, 109, 39, 209, 254, 31, 199, - 95, 39, 209, 254, 31, 197, 32, 39, 209, 254, 31, 228, 142, 237, 86, 6, 1, - 180, 68, 237, 86, 6, 1, 180, 71, 237, 86, 6, 1, 180, 65, 237, 86, 6, 1, - 180, 251, 247, 237, 86, 6, 1, 180, 74, 237, 86, 6, 1, 180, 211, 89, 237, - 86, 6, 1, 203, 41, 68, 237, 86, 6, 1, 203, 41, 71, 237, 86, 6, 1, 203, - 41, 65, 237, 86, 6, 1, 203, 41, 251, 247, 237, 86, 6, 1, 203, 41, 74, - 237, 86, 6, 1, 203, 41, 211, 89, 237, 86, 6, 1, 250, 133, 237, 86, 6, 1, - 210, 252, 237, 86, 6, 1, 193, 0, 237, 86, 6, 1, 192, 77, 237, 86, 6, 1, - 230, 118, 237, 86, 6, 1, 210, 51, 237, 86, 6, 1, 249, 3, 237, 86, 6, 1, - 199, 121, 237, 86, 6, 1, 237, 133, 237, 86, 6, 1, 243, 125, 237, 86, 6, - 1, 223, 10, 237, 86, 6, 1, 222, 31, 237, 86, 6, 1, 231, 188, 237, 86, 6, - 1, 234, 183, 237, 86, 6, 1, 195, 148, 237, 86, 6, 1, 233, 250, 237, 86, - 6, 1, 199, 31, 237, 86, 6, 1, 234, 24, 237, 86, 6, 1, 191, 84, 237, 86, - 6, 1, 234, 14, 237, 86, 6, 1, 191, 61, 237, 86, 6, 1, 234, 36, 237, 86, - 6, 1, 234, 142, 237, 86, 6, 1, 234, 116, 237, 86, 6, 1, 234, 105, 237, - 86, 6, 1, 234, 90, 237, 86, 6, 1, 211, 135, 237, 86, 6, 1, 233, 226, 237, - 86, 2, 1, 180, 68, 237, 86, 2, 1, 180, 71, 237, 86, 2, 1, 180, 65, 237, - 86, 2, 1, 180, 251, 247, 237, 86, 2, 1, 180, 74, 237, 86, 2, 1, 180, 211, - 89, 237, 86, 2, 1, 203, 41, 68, 237, 86, 2, 1, 203, 41, 71, 237, 86, 2, - 1, 203, 41, 65, 237, 86, 2, 1, 203, 41, 251, 247, 237, 86, 2, 1, 203, 41, - 74, 237, 86, 2, 1, 203, 41, 211, 89, 237, 86, 2, 1, 250, 133, 237, 86, 2, - 1, 210, 252, 237, 86, 2, 1, 193, 0, 237, 86, 2, 1, 192, 77, 237, 86, 2, - 1, 230, 118, 237, 86, 2, 1, 210, 51, 237, 86, 2, 1, 249, 3, 237, 86, 2, - 1, 199, 121, 237, 86, 2, 1, 237, 133, 237, 86, 2, 1, 243, 125, 237, 86, - 2, 1, 223, 10, 237, 86, 2, 1, 222, 31, 237, 86, 2, 1, 231, 188, 237, 86, - 2, 1, 234, 183, 237, 86, 2, 1, 195, 148, 237, 86, 2, 1, 233, 250, 237, - 86, 2, 1, 199, 31, 237, 86, 2, 1, 234, 24, 237, 86, 2, 1, 191, 84, 237, - 86, 2, 1, 234, 14, 237, 86, 2, 1, 191, 61, 237, 86, 2, 1, 234, 36, 237, - 86, 2, 1, 234, 142, 237, 86, 2, 1, 234, 116, 237, 86, 2, 1, 234, 105, - 237, 86, 2, 1, 234, 90, 237, 86, 2, 1, 211, 135, 237, 86, 2, 1, 233, 226, - 203, 121, 1, 210, 47, 203, 121, 1, 198, 40, 203, 121, 1, 221, 156, 203, - 121, 1, 233, 54, 203, 121, 1, 199, 6, 203, 121, 1, 202, 47, 203, 121, 1, - 200, 172, 203, 121, 1, 243, 42, 203, 121, 1, 192, 79, 203, 121, 1, 228, - 139, 203, 121, 1, 248, 189, 203, 121, 1, 237, 147, 203, 121, 1, 231, 228, - 203, 121, 1, 195, 2, 203, 121, 1, 199, 12, 203, 121, 1, 191, 4, 203, 121, - 1, 216, 229, 203, 121, 1, 222, 182, 203, 121, 1, 193, 39, 203, 121, 1, - 230, 95, 203, 121, 1, 219, 27, 203, 121, 1, 216, 47, 203, 121, 1, 223, - 208, 203, 121, 1, 234, 181, 203, 121, 1, 250, 211, 203, 121, 1, 252, 32, - 203, 121, 1, 211, 106, 203, 121, 1, 193, 26, 203, 121, 1, 211, 16, 203, - 121, 1, 251, 247, 203, 121, 1, 206, 154, 203, 121, 1, 212, 132, 203, 121, - 1, 234, 202, 203, 121, 1, 251, 252, 203, 121, 1, 228, 28, 203, 121, 1, - 196, 77, 203, 121, 1, 211, 182, 203, 121, 1, 211, 81, 203, 121, 1, 211, - 133, 203, 121, 1, 250, 139, 203, 121, 1, 251, 11, 203, 121, 1, 211, 58, - 203, 121, 1, 251, 227, 203, 121, 1, 234, 28, 203, 121, 1, 250, 242, 203, - 121, 1, 234, 213, 203, 121, 1, 228, 36, 203, 121, 1, 192, 41, 211, 35, 1, - 251, 194, 211, 35, 1, 249, 155, 211, 35, 1, 190, 190, 211, 35, 1, 223, - 34, 211, 35, 1, 192, 220, 211, 35, 1, 221, 217, 211, 35, 1, 237, 132, - 211, 35, 1, 165, 211, 35, 1, 188, 211, 35, 1, 203, 76, 211, 35, 1, 237, - 70, 211, 35, 1, 247, 69, 211, 35, 1, 231, 242, 211, 35, 1, 233, 111, 211, - 35, 1, 208, 96, 211, 35, 1, 222, 125, 211, 35, 1, 220, 123, 211, 35, 1, - 216, 61, 211, 35, 1, 212, 116, 211, 35, 1, 193, 133, 211, 35, 1, 140, - 211, 35, 1, 170, 211, 35, 1, 65, 211, 35, 1, 71, 211, 35, 1, 68, 211, 35, - 1, 74, 211, 35, 1, 66, 211, 35, 1, 252, 208, 211, 35, 1, 234, 190, 211, - 35, 1, 211, 89, 211, 35, 17, 191, 77, 211, 35, 17, 107, 211, 35, 17, 109, - 211, 35, 17, 138, 211, 35, 17, 134, 211, 35, 17, 150, 211, 35, 17, 169, - 211, 35, 17, 175, 211, 35, 17, 171, 211, 35, 17, 178, 211, 37, 6, 1, 65, - 211, 37, 6, 1, 251, 238, 211, 37, 6, 1, 251, 232, 211, 37, 6, 1, 251, - 247, 211, 37, 6, 1, 248, 58, 211, 37, 6, 1, 247, 3, 211, 37, 6, 1, 234, - 174, 211, 37, 6, 1, 71, 211, 37, 6, 1, 234, 154, 211, 37, 6, 1, 140, 211, - 37, 6, 1, 228, 95, 211, 37, 6, 1, 68, 211, 37, 6, 1, 155, 211, 37, 6, 1, - 234, 173, 211, 37, 6, 1, 220, 155, 211, 37, 6, 1, 173, 211, 37, 6, 1, - 174, 211, 37, 6, 1, 181, 211, 37, 6, 1, 74, 211, 37, 6, 1, 211, 132, 211, - 37, 6, 1, 168, 211, 37, 6, 1, 234, 172, 211, 37, 6, 1, 188, 211, 37, 6, - 1, 203, 166, 211, 37, 6, 1, 190, 190, 211, 37, 6, 1, 234, 171, 211, 37, - 6, 1, 197, 168, 211, 37, 6, 1, 234, 170, 211, 37, 6, 1, 197, 157, 211, - 37, 6, 1, 237, 70, 211, 37, 6, 1, 66, 211, 37, 6, 1, 193, 190, 211, 37, - 6, 1, 221, 217, 211, 37, 6, 1, 231, 93, 211, 37, 6, 1, 191, 123, 211, 37, - 6, 1, 191, 71, 211, 37, 2, 1, 65, 211, 37, 2, 1, 251, 238, 211, 37, 2, 1, - 251, 232, 211, 37, 2, 1, 251, 247, 211, 37, 2, 1, 248, 58, 211, 37, 2, 1, - 247, 3, 211, 37, 2, 1, 234, 174, 211, 37, 2, 1, 71, 211, 37, 2, 1, 234, - 154, 211, 37, 2, 1, 140, 211, 37, 2, 1, 228, 95, 211, 37, 2, 1, 68, 211, - 37, 2, 1, 155, 211, 37, 2, 1, 234, 173, 211, 37, 2, 1, 220, 155, 211, 37, - 2, 1, 173, 211, 37, 2, 1, 174, 211, 37, 2, 1, 181, 211, 37, 2, 1, 74, - 211, 37, 2, 1, 211, 132, 211, 37, 2, 1, 168, 211, 37, 2, 1, 234, 172, - 211, 37, 2, 1, 188, 211, 37, 2, 1, 203, 166, 211, 37, 2, 1, 190, 190, - 211, 37, 2, 1, 234, 171, 211, 37, 2, 1, 197, 168, 211, 37, 2, 1, 234, - 170, 211, 37, 2, 1, 197, 157, 211, 37, 2, 1, 237, 70, 211, 37, 2, 1, 66, - 211, 37, 2, 1, 193, 190, 211, 37, 2, 1, 221, 217, 211, 37, 2, 1, 231, 93, - 211, 37, 2, 1, 191, 123, 211, 37, 2, 1, 191, 71, 234, 138, 1, 65, 234, - 138, 1, 249, 19, 234, 138, 1, 247, 44, 234, 138, 1, 243, 129, 234, 138, - 1, 237, 148, 234, 138, 1, 214, 156, 234, 138, 1, 237, 61, 234, 138, 1, - 234, 168, 234, 138, 1, 71, 234, 138, 1, 233, 61, 234, 138, 1, 231, 167, - 234, 138, 1, 231, 22, 234, 138, 1, 229, 247, 234, 138, 1, 68, 234, 138, - 1, 223, 12, 234, 138, 1, 222, 24, 234, 138, 1, 219, 240, 234, 138, 1, - 219, 70, 234, 138, 1, 216, 234, 234, 138, 1, 214, 123, 234, 138, 1, 181, - 234, 138, 1, 213, 152, 234, 138, 1, 74, 234, 138, 1, 210, 65, 234, 138, - 1, 208, 77, 234, 138, 1, 207, 115, 234, 138, 1, 206, 105, 234, 138, 1, - 205, 69, 234, 138, 1, 203, 114, 234, 138, 1, 199, 145, 234, 138, 1, 199, - 33, 234, 138, 1, 66, 234, 138, 1, 195, 153, 234, 138, 1, 192, 214, 234, - 138, 1, 192, 159, 234, 138, 1, 191, 87, 234, 138, 1, 191, 62, 234, 138, - 1, 231, 79, 234, 138, 1, 231, 85, 234, 138, 1, 221, 199, 247, 77, 251, - 195, 1, 251, 161, 247, 77, 251, 195, 1, 248, 216, 247, 77, 251, 195, 1, - 231, 42, 247, 77, 251, 195, 1, 237, 213, 247, 77, 251, 195, 1, 234, 201, - 247, 77, 251, 195, 1, 191, 98, 247, 77, 251, 195, 1, 233, 186, 247, 77, - 251, 195, 1, 191, 56, 247, 77, 251, 195, 1, 199, 174, 247, 77, 251, 195, - 1, 247, 3, 247, 77, 251, 195, 1, 191, 236, 247, 77, 251, 195, 1, 191, 71, - 247, 77, 251, 195, 1, 223, 78, 247, 77, 251, 195, 1, 203, 166, 247, 77, - 251, 195, 1, 220, 26, 247, 77, 251, 195, 1, 223, 91, 247, 77, 251, 195, - 1, 192, 210, 247, 77, 251, 195, 1, 235, 45, 247, 77, 251, 195, 1, 247, - 104, 247, 77, 251, 195, 1, 222, 247, 247, 77, 251, 195, 1, 222, 67, 247, - 77, 251, 195, 1, 218, 179, 247, 77, 251, 195, 1, 229, 181, 247, 77, 251, - 195, 1, 208, 78, 247, 77, 251, 195, 1, 251, 69, 247, 77, 251, 195, 1, - 243, 59, 247, 77, 251, 195, 1, 243, 97, 247, 77, 251, 195, 1, 238, 142, - 247, 77, 251, 195, 1, 217, 70, 247, 77, 251, 195, 1, 208, 83, 247, 77, - 251, 195, 1, 212, 254, 247, 77, 251, 195, 1, 235, 22, 247, 77, 251, 195, - 1, 203, 148, 247, 77, 251, 195, 1, 223, 13, 247, 77, 251, 195, 1, 211, - 106, 247, 77, 251, 195, 1, 197, 3, 247, 77, 251, 195, 1, 233, 84, 247, - 77, 251, 195, 1, 235, 35, 247, 77, 251, 195, 1, 243, 135, 247, 77, 251, - 195, 1, 210, 36, 247, 77, 251, 195, 1, 231, 69, 247, 77, 251, 195, 1, - 207, 132, 247, 77, 251, 195, 1, 203, 175, 247, 77, 251, 195, 1, 195, 72, - 247, 77, 251, 195, 1, 198, 118, 247, 77, 251, 195, 1, 203, 19, 247, 77, - 251, 195, 1, 223, 48, 247, 77, 251, 195, 1, 238, 143, 247, 77, 251, 195, - 1, 247, 69, 247, 77, 251, 195, 1, 192, 84, 247, 77, 251, 195, 1, 209, - 116, 247, 77, 251, 195, 1, 221, 119, 247, 77, 251, 195, 243, 0, 77, 195, - 29, 6, 1, 65, 195, 29, 6, 1, 249, 50, 195, 29, 6, 1, 249, 19, 195, 29, 6, - 1, 247, 44, 195, 29, 6, 1, 243, 129, 195, 29, 6, 1, 237, 148, 195, 29, 6, - 1, 237, 61, 195, 29, 6, 1, 234, 168, 195, 29, 6, 1, 71, 195, 29, 6, 1, - 233, 61, 195, 29, 6, 1, 231, 242, 195, 29, 6, 1, 140, 195, 29, 6, 1, 229, - 179, 195, 29, 6, 1, 68, 195, 29, 6, 1, 223, 198, 195, 29, 6, 1, 223, 12, - 195, 29, 6, 1, 155, 195, 29, 6, 1, 173, 195, 29, 6, 1, 219, 75, 195, 29, - 6, 1, 216, 234, 195, 29, 6, 1, 214, 123, 195, 29, 6, 1, 213, 152, 195, - 29, 6, 1, 74, 195, 29, 6, 1, 210, 65, 195, 29, 6, 1, 208, 98, 195, 29, 6, - 1, 207, 115, 195, 29, 6, 1, 205, 69, 195, 29, 6, 1, 203, 114, 195, 29, 6, - 1, 199, 145, 195, 29, 6, 1, 199, 33, 195, 29, 6, 1, 66, 195, 29, 6, 1, - 195, 153, 195, 29, 6, 1, 192, 214, 195, 29, 6, 1, 192, 159, 195, 29, 6, - 1, 191, 87, 195, 29, 2, 1, 65, 195, 29, 2, 1, 249, 50, 195, 29, 2, 1, - 249, 19, 195, 29, 2, 1, 247, 44, 195, 29, 2, 1, 243, 129, 195, 29, 2, 1, - 237, 148, 195, 29, 2, 1, 237, 61, 195, 29, 2, 1, 234, 168, 195, 29, 2, 1, - 71, 195, 29, 2, 1, 233, 61, 195, 29, 2, 1, 231, 242, 195, 29, 2, 1, 140, - 195, 29, 2, 1, 229, 179, 195, 29, 2, 1, 68, 195, 29, 2, 1, 223, 198, 195, - 29, 2, 1, 223, 12, 195, 29, 2, 1, 155, 195, 29, 2, 1, 173, 195, 29, 2, 1, - 219, 75, 195, 29, 2, 1, 216, 234, 195, 29, 2, 1, 214, 123, 195, 29, 2, 1, - 213, 152, 195, 29, 2, 1, 74, 195, 29, 2, 1, 210, 65, 195, 29, 2, 1, 208, - 98, 195, 29, 2, 1, 207, 115, 195, 29, 2, 1, 205, 69, 195, 29, 2, 1, 203, - 114, 195, 29, 2, 1, 199, 145, 195, 29, 2, 1, 199, 33, 195, 29, 2, 1, 66, - 195, 29, 2, 1, 195, 153, 195, 29, 2, 1, 192, 214, 195, 29, 2, 1, 192, - 159, 195, 29, 2, 1, 191, 87, 32, 42, 3, 252, 156, 32, 42, 3, 252, 155, - 32, 42, 3, 252, 154, 32, 42, 3, 252, 153, 32, 42, 3, 252, 152, 32, 42, 3, - 252, 151, 32, 42, 3, 252, 150, 32, 42, 3, 252, 149, 32, 42, 3, 252, 148, - 32, 42, 3, 252, 147, 32, 42, 3, 252, 146, 32, 42, 3, 252, 145, 32, 42, 3, - 252, 144, 32, 42, 3, 252, 143, 32, 42, 3, 252, 142, 32, 42, 3, 252, 141, - 32, 42, 3, 252, 140, 32, 42, 3, 252, 139, 32, 42, 3, 252, 138, 32, 42, 3, - 252, 137, 32, 42, 3, 252, 136, 32, 42, 3, 252, 135, 32, 42, 3, 252, 134, - 32, 42, 3, 252, 133, 32, 42, 3, 252, 132, 32, 42, 3, 252, 131, 32, 42, 3, - 252, 130, 32, 42, 3, 255, 166, 32, 42, 3, 252, 129, 32, 42, 3, 252, 128, - 32, 42, 3, 252, 127, 32, 42, 3, 252, 126, 32, 42, 3, 252, 125, 32, 42, 3, - 252, 124, 32, 42, 3, 252, 123, 32, 42, 3, 252, 122, 32, 42, 3, 252, 121, - 32, 42, 3, 252, 120, 32, 42, 3, 252, 119, 32, 42, 3, 252, 118, 32, 42, 3, - 252, 117, 32, 42, 3, 252, 116, 32, 42, 3, 252, 115, 32, 42, 3, 252, 114, - 32, 42, 3, 252, 113, 32, 42, 3, 252, 112, 32, 42, 3, 252, 111, 32, 42, 3, - 252, 110, 32, 42, 3, 252, 109, 32, 42, 3, 252, 108, 32, 42, 3, 252, 107, - 32, 42, 3, 252, 106, 32, 42, 3, 252, 105, 32, 42, 3, 252, 104, 32, 42, 3, - 252, 103, 32, 42, 3, 252, 102, 32, 42, 3, 252, 101, 32, 42, 3, 252, 100, - 32, 42, 3, 252, 99, 32, 42, 3, 252, 98, 32, 42, 3, 252, 97, 32, 42, 3, - 252, 96, 32, 42, 3, 252, 95, 32, 42, 3, 252, 94, 32, 42, 3, 252, 93, 32, - 42, 3, 252, 92, 32, 42, 3, 252, 91, 32, 42, 3, 252, 90, 32, 42, 3, 252, - 89, 32, 42, 3, 252, 88, 32, 42, 3, 252, 87, 32, 42, 3, 255, 79, 32, 42, - 3, 252, 86, 32, 42, 3, 252, 85, 32, 42, 3, 255, 44, 32, 42, 3, 252, 84, - 32, 42, 3, 252, 83, 32, 42, 3, 252, 82, 32, 42, 3, 252, 81, 32, 42, 3, - 255, 31, 32, 42, 3, 252, 80, 32, 42, 3, 252, 79, 32, 42, 3, 252, 78, 32, - 42, 3, 252, 77, 32, 42, 3, 252, 76, 32, 42, 3, 254, 103, 32, 42, 3, 254, - 102, 32, 42, 3, 254, 101, 32, 42, 3, 254, 100, 32, 42, 3, 254, 99, 32, - 42, 3, 254, 98, 32, 42, 3, 254, 97, 32, 42, 3, 254, 96, 32, 42, 3, 254, - 94, 32, 42, 3, 254, 93, 32, 42, 3, 254, 92, 32, 42, 3, 254, 91, 32, 42, - 3, 254, 90, 32, 42, 3, 254, 89, 32, 42, 3, 254, 87, 32, 42, 3, 254, 86, - 32, 42, 3, 254, 85, 32, 42, 3, 254, 84, 32, 42, 3, 254, 83, 32, 42, 3, - 254, 82, 32, 42, 3, 254, 81, 32, 42, 3, 254, 80, 32, 42, 3, 254, 79, 32, - 42, 3, 254, 78, 32, 42, 3, 254, 77, 32, 42, 3, 254, 76, 32, 42, 3, 254, - 75, 32, 42, 3, 254, 74, 32, 42, 3, 254, 73, 32, 42, 3, 254, 72, 32, 42, - 3, 254, 71, 32, 42, 3, 254, 70, 32, 42, 3, 254, 69, 32, 42, 3, 254, 67, - 32, 42, 3, 254, 66, 32, 42, 3, 254, 65, 32, 42, 3, 254, 61, 32, 42, 3, - 254, 60, 32, 42, 3, 254, 59, 32, 42, 3, 254, 58, 32, 42, 3, 254, 54, 32, - 42, 3, 254, 53, 32, 42, 3, 254, 52, 32, 42, 3, 254, 51, 32, 42, 3, 254, - 50, 32, 42, 3, 254, 49, 32, 42, 3, 254, 48, 32, 42, 3, 254, 47, 32, 42, - 3, 254, 46, 32, 42, 3, 254, 45, 32, 42, 3, 254, 44, 32, 42, 3, 254, 43, - 32, 42, 3, 254, 42, 32, 42, 3, 254, 41, 32, 42, 3, 254, 40, 32, 42, 3, - 254, 39, 32, 42, 3, 254, 38, 32, 42, 3, 254, 37, 32, 42, 3, 254, 36, 32, - 42, 3, 254, 35, 32, 42, 3, 254, 34, 32, 42, 3, 254, 33, 32, 42, 3, 254, - 32, 32, 42, 3, 254, 30, 32, 42, 3, 254, 29, 32, 42, 3, 254, 28, 32, 42, - 3, 254, 27, 32, 42, 3, 254, 26, 32, 42, 3, 254, 24, 32, 42, 3, 254, 23, - 32, 42, 3, 254, 22, 32, 42, 3, 254, 21, 32, 42, 3, 254, 19, 32, 42, 3, - 254, 18, 32, 42, 3, 254, 17, 32, 42, 3, 253, 239, 32, 42, 3, 253, 237, - 32, 42, 3, 253, 235, 32, 42, 3, 253, 233, 32, 42, 3, 253, 231, 32, 42, 3, - 253, 229, 32, 42, 3, 253, 227, 32, 42, 3, 253, 225, 32, 42, 3, 253, 223, - 32, 42, 3, 253, 221, 32, 42, 3, 253, 219, 32, 42, 3, 253, 216, 32, 42, 3, - 253, 214, 32, 42, 3, 253, 212, 32, 42, 3, 253, 210, 32, 42, 3, 253, 208, - 32, 42, 3, 253, 206, 32, 42, 3, 253, 204, 32, 42, 3, 253, 202, 32, 42, 3, - 253, 120, 32, 42, 3, 253, 119, 32, 42, 3, 253, 118, 32, 42, 3, 253, 117, - 32, 42, 3, 253, 116, 32, 42, 3, 253, 115, 32, 42, 3, 253, 113, 32, 42, 3, - 253, 112, 32, 42, 3, 253, 111, 32, 42, 3, 253, 110, 32, 42, 3, 253, 109, - 32, 42, 3, 253, 108, 32, 42, 3, 253, 106, 32, 42, 3, 253, 105, 32, 42, 3, - 253, 101, 32, 42, 3, 253, 100, 32, 42, 3, 253, 98, 32, 42, 3, 253, 97, - 32, 42, 3, 253, 96, 32, 42, 3, 253, 95, 32, 42, 3, 253, 94, 32, 42, 3, - 253, 93, 32, 42, 3, 253, 92, 32, 42, 3, 253, 91, 32, 42, 3, 253, 90, 32, - 42, 3, 253, 89, 32, 42, 3, 253, 88, 32, 42, 3, 253, 87, 32, 42, 3, 253, - 86, 32, 42, 3, 253, 85, 32, 42, 3, 253, 84, 32, 42, 3, 253, 83, 32, 42, - 3, 253, 82, 32, 42, 3, 253, 81, 32, 42, 3, 253, 80, 32, 42, 3, 253, 79, - 32, 42, 3, 253, 78, 32, 42, 3, 253, 77, 32, 42, 3, 253, 76, 32, 42, 3, - 253, 75, 32, 42, 3, 253, 74, 32, 42, 3, 253, 73, 32, 42, 3, 253, 72, 32, - 42, 3, 253, 71, 32, 42, 3, 253, 70, 32, 42, 3, 253, 69, 32, 42, 3, 253, - 68, 32, 42, 3, 253, 67, 32, 42, 3, 253, 66, 32, 42, 3, 253, 65, 32, 42, - 3, 253, 64, 32, 42, 3, 253, 63, 32, 42, 3, 253, 62, 32, 42, 3, 253, 61, - 32, 42, 3, 253, 60, 32, 42, 3, 253, 59, 32, 42, 3, 253, 58, 32, 42, 3, - 253, 57, 32, 42, 3, 253, 56, 32, 42, 3, 253, 55, 32, 42, 3, 253, 54, 32, - 42, 3, 253, 53, 32, 42, 3, 253, 52, 32, 42, 3, 253, 51, 32, 42, 3, 253, - 50, 32, 42, 3, 253, 49, 32, 42, 3, 253, 48, 32, 42, 3, 253, 47, 32, 42, - 3, 253, 46, 32, 42, 3, 253, 45, 32, 42, 3, 253, 44, 32, 42, 3, 253, 43, - 32, 42, 3, 253, 42, 32, 42, 3, 253, 41, 32, 42, 3, 253, 40, 32, 42, 3, - 253, 39, 32, 42, 3, 253, 38, 32, 42, 3, 253, 37, 32, 42, 3, 253, 36, 32, - 42, 3, 253, 35, 32, 42, 3, 253, 34, 32, 42, 3, 253, 33, 32, 42, 3, 253, - 32, 32, 42, 3, 253, 31, 32, 42, 3, 253, 30, 32, 42, 3, 253, 29, 32, 42, - 3, 253, 28, 32, 42, 3, 253, 27, 32, 42, 3, 253, 26, 32, 42, 3, 253, 25, - 32, 42, 3, 253, 24, 32, 42, 3, 253, 23, 32, 42, 3, 253, 22, 32, 42, 3, - 253, 21, 32, 42, 3, 253, 20, 32, 42, 3, 253, 19, 32, 42, 3, 253, 18, 32, - 42, 3, 253, 17, 32, 42, 3, 253, 16, 32, 42, 3, 253, 15, 32, 42, 3, 253, - 14, 32, 42, 3, 253, 13, 32, 42, 3, 253, 12, 32, 42, 3, 253, 11, 32, 42, - 3, 253, 10, 32, 42, 3, 253, 9, 32, 42, 3, 253, 8, 32, 42, 3, 253, 7, 32, - 42, 3, 253, 6, 32, 42, 3, 253, 5, 32, 42, 3, 253, 4, 32, 42, 3, 253, 3, - 32, 42, 3, 253, 2, 32, 42, 3, 253, 1, 32, 42, 3, 253, 0, 32, 42, 3, 252, - 255, 32, 42, 3, 252, 254, 32, 42, 3, 252, 253, 32, 42, 3, 252, 252, 32, - 42, 3, 252, 251, 32, 42, 3, 252, 250, 32, 42, 3, 252, 249, 32, 42, 3, - 252, 248, 32, 42, 3, 252, 247, 32, 42, 3, 252, 246, 32, 42, 3, 252, 245, - 32, 42, 3, 252, 244, 32, 42, 3, 252, 243, 32, 42, 3, 252, 242, 32, 42, 3, - 252, 241, 32, 42, 3, 252, 240, 32, 42, 3, 252, 239, 32, 42, 3, 252, 238, - 65, 32, 42, 3, 252, 237, 250, 122, 32, 42, 3, 252, 236, 238, 129, 32, 42, - 3, 252, 235, 71, 32, 42, 3, 252, 234, 233, 177, 32, 42, 3, 252, 233, 230, - 118, 32, 42, 3, 252, 232, 223, 37, 32, 42, 3, 252, 231, 222, 154, 32, 42, - 3, 252, 230, 172, 32, 42, 3, 252, 229, 220, 132, 32, 42, 3, 252, 228, - 220, 131, 32, 42, 3, 252, 227, 220, 130, 32, 42, 3, 252, 226, 220, 129, - 32, 42, 3, 252, 225, 193, 224, 32, 42, 3, 252, 224, 192, 235, 32, 42, 3, - 252, 223, 192, 159, 32, 42, 3, 252, 222, 211, 112, 32, 42, 3, 252, 221, - 252, 71, 32, 42, 3, 252, 220, 249, 56, 32, 42, 3, 252, 219, 237, 195, 32, - 42, 3, 252, 218, 233, 185, 32, 42, 3, 252, 217, 223, 12, 32, 42, 3, 252, - 216, 32, 42, 3, 252, 215, 32, 42, 3, 252, 214, 32, 42, 3, 252, 213, 32, - 42, 3, 252, 212, 32, 42, 3, 252, 211, 32, 42, 3, 252, 210, 32, 42, 3, - 252, 209, 52, 1, 2, 6, 252, 208, 52, 1, 200, 182, 197, 238, 242, 85, 52, - 1, 200, 182, 132, 197, 238, 242, 85, 52, 1, 2, 252, 27, 52, 1, 2, 6, 250, - 122, 52, 1, 2, 78, 4, 102, 52, 1, 2, 235, 39, 237, 4, 52, 1, 2, 235, 39, - 237, 5, 4, 207, 25, 102, 52, 1, 2, 235, 39, 237, 5, 4, 238, 177, 52, 1, - 2, 237, 72, 237, 4, 52, 1, 2, 238, 130, 4, 199, 215, 52, 1, 2, 238, 130, - 4, 102, 52, 1, 2, 238, 130, 4, 228, 253, 23, 199, 215, 52, 1, 2, 207, 19, - 71, 52, 1, 2, 242, 221, 207, 19, 211, 79, 71, 52, 1, 2, 233, 39, 237, 4, - 52, 1, 2, 207, 142, 228, 189, 52, 1, 2, 6, 232, 53, 52, 1, 2, 232, 54, 4, - 102, 52, 1, 2, 6, 232, 54, 4, 102, 52, 1, 2, 230, 119, 4, 106, 52, 1, 2, - 6, 230, 118, 52, 1, 2, 229, 199, 4, 102, 52, 1, 2, 236, 141, 223, 38, 4, - 201, 29, 23, 102, 52, 1, 2, 218, 229, 237, 4, 52, 1, 2, 218, 172, 237, 4, - 52, 1, 2, 220, 145, 4, 248, 233, 52, 1, 2, 6, 220, 145, 4, 248, 233, 52, - 1, 2, 220, 145, 4, 207, 25, 228, 253, 23, 248, 233, 52, 1, 2, 219, 164, - 52, 1, 2, 219, 165, 4, 207, 25, 102, 52, 1, 2, 154, 192, 159, 52, 1, 2, - 154, 192, 160, 4, 248, 233, 52, 1, 2, 187, 4, 106, 52, 1, 2, 6, 211, 153, - 52, 1, 2, 242, 221, 211, 112, 52, 1, 2, 208, 106, 52, 1, 2, 154, 207, - 224, 4, 180, 219, 214, 52, 1, 2, 154, 207, 224, 4, 180, 219, 215, 23, - 207, 25, 102, 52, 1, 2, 207, 224, 4, 199, 215, 52, 1, 2, 207, 224, 4, - 232, 235, 52, 1, 2, 6, 146, 52, 1, 2, 199, 152, 237, 5, 4, 238, 177, 52, - 1, 2, 197, 170, 237, 4, 52, 1, 2, 197, 170, 237, 5, 4, 207, 25, 102, 52, - 1, 2, 199, 79, 237, 4, 52, 1, 2, 200, 44, 4, 207, 25, 102, 52, 1, 2, 196, - 13, 4, 50, 102, 52, 1, 2, 6, 192, 159, 52, 1, 231, 13, 201, 65, 4, 106, - 52, 1, 207, 19, 231, 13, 201, 65, 4, 106, 52, 1, 248, 174, 242, 233, 52, - 1, 237, 100, 242, 233, 52, 1, 220, 5, 242, 233, 52, 1, 251, 152, 242, - 233, 52, 1, 207, 25, 242, 234, 4, 207, 25, 102, 52, 1, 2, 206, 10, 4, - 238, 177, 238, 137, 5, 65, 238, 137, 5, 71, 238, 137, 5, 68, 238, 137, 5, - 74, 238, 137, 5, 66, 238, 137, 5, 223, 34, 238, 137, 5, 222, 203, 238, - 137, 5, 155, 238, 137, 5, 222, 24, 238, 137, 5, 221, 168, 238, 137, 5, - 221, 69, 238, 137, 5, 220, 234, 238, 137, 5, 173, 238, 137, 5, 219, 240, - 238, 137, 5, 219, 148, 238, 137, 5, 219, 45, 238, 137, 5, 218, 227, 238, - 137, 5, 174, 238, 137, 5, 216, 234, 238, 137, 5, 216, 102, 238, 137, 5, - 216, 14, 238, 137, 5, 215, 157, 238, 137, 5, 181, 238, 137, 5, 214, 123, - 238, 137, 5, 213, 221, 238, 137, 5, 213, 45, 238, 137, 5, 212, 180, 238, - 137, 5, 168, 238, 137, 5, 210, 65, 238, 137, 5, 209, 187, 238, 137, 5, - 209, 75, 238, 137, 5, 208, 167, 238, 137, 5, 165, 238, 137, 5, 207, 115, - 238, 137, 5, 207, 2, 238, 137, 5, 206, 163, 238, 137, 5, 206, 69, 238, - 137, 5, 188, 238, 137, 5, 205, 69, 238, 137, 5, 202, 223, 238, 137, 5, - 202, 47, 238, 137, 5, 201, 5, 238, 137, 5, 190, 190, 238, 137, 5, 199, - 145, 238, 137, 5, 198, 193, 238, 137, 5, 159, 238, 137, 5, 197, 94, 238, - 137, 5, 193, 190, 238, 137, 5, 193, 125, 238, 137, 5, 193, 86, 238, 137, - 5, 193, 48, 238, 137, 5, 192, 220, 238, 137, 5, 192, 214, 238, 137, 5, - 191, 123, 238, 137, 5, 191, 7, 223, 166, 251, 20, 1, 251, 192, 223, 166, - 251, 20, 1, 248, 213, 223, 166, 251, 20, 1, 231, 40, 223, 166, 251, 20, - 1, 237, 254, 223, 166, 251, 20, 1, 229, 247, 223, 166, 251, 20, 1, 193, - 133, 223, 166, 251, 20, 1, 191, 91, 223, 166, 251, 20, 1, 229, 186, 223, - 166, 251, 20, 1, 199, 69, 223, 166, 251, 20, 1, 191, 249, 223, 166, 251, - 20, 1, 222, 77, 223, 166, 251, 20, 1, 220, 28, 223, 166, 251, 20, 1, 216, - 195, 223, 166, 251, 20, 1, 212, 132, 223, 166, 251, 20, 1, 205, 149, 223, - 166, 251, 20, 1, 250, 128, 223, 166, 251, 20, 1, 210, 65, 223, 166, 251, - 20, 1, 205, 190, 223, 166, 251, 20, 1, 208, 39, 223, 166, 251, 20, 1, - 207, 39, 223, 166, 251, 20, 1, 203, 70, 223, 166, 251, 20, 1, 199, 159, - 223, 166, 251, 20, 205, 55, 56, 223, 166, 251, 20, 31, 107, 223, 166, - 251, 20, 31, 109, 223, 166, 251, 20, 31, 138, 223, 166, 251, 20, 31, 199, - 95, 223, 166, 251, 20, 31, 197, 32, 223, 166, 251, 20, 31, 91, 228, 142, - 223, 166, 251, 20, 31, 91, 189, 223, 166, 251, 20, 31, 199, 96, 189, 210, - 182, 1, 251, 192, 210, 182, 1, 248, 213, 210, 182, 1, 231, 40, 210, 182, - 1, 237, 254, 210, 182, 1, 229, 247, 210, 182, 1, 193, 133, 210, 182, 1, - 191, 91, 210, 182, 1, 229, 186, 210, 182, 1, 199, 69, 210, 182, 1, 191, - 249, 210, 182, 1, 222, 77, 210, 182, 1, 220, 28, 210, 182, 1, 216, 195, - 210, 182, 1, 53, 212, 132, 210, 182, 1, 212, 132, 210, 182, 1, 205, 149, - 210, 182, 1, 250, 128, 210, 182, 1, 210, 65, 210, 182, 1, 205, 190, 210, - 182, 1, 208, 39, 210, 182, 1, 207, 39, 210, 182, 1, 203, 70, 210, 182, 1, - 199, 159, 210, 182, 219, 221, 232, 203, 210, 182, 206, 204, 232, 203, - 210, 182, 31, 107, 210, 182, 31, 109, 210, 182, 31, 138, 210, 182, 31, - 134, 210, 182, 31, 150, 210, 182, 31, 199, 95, 210, 182, 31, 197, 32, - 214, 248, 1, 53, 251, 192, 214, 248, 1, 251, 192, 214, 248, 1, 53, 248, - 213, 214, 248, 1, 248, 213, 214, 248, 1, 231, 40, 214, 248, 1, 237, 254, - 214, 248, 1, 53, 229, 247, 214, 248, 1, 229, 247, 214, 248, 1, 193, 133, - 214, 248, 1, 191, 91, 214, 248, 1, 229, 186, 214, 248, 1, 199, 69, 214, - 248, 1, 53, 191, 249, 214, 248, 1, 191, 249, 214, 248, 1, 53, 222, 77, - 214, 248, 1, 222, 77, 214, 248, 1, 53, 220, 28, 214, 248, 1, 220, 28, - 214, 248, 1, 53, 216, 195, 214, 248, 1, 216, 195, 214, 248, 1, 53, 212, - 132, 214, 248, 1, 212, 132, 214, 248, 1, 205, 149, 214, 248, 1, 250, 128, - 214, 248, 1, 210, 65, 214, 248, 1, 205, 190, 214, 248, 1, 208, 39, 214, - 248, 1, 207, 39, 214, 248, 1, 53, 203, 70, 214, 248, 1, 203, 70, 214, - 248, 1, 199, 159, 214, 248, 31, 107, 214, 248, 31, 109, 214, 248, 31, - 138, 214, 248, 31, 134, 214, 248, 238, 204, 31, 134, 214, 248, 31, 150, - 214, 248, 31, 199, 95, 214, 248, 31, 197, 32, 214, 248, 31, 91, 228, 142, - 230, 6, 1, 251, 192, 230, 6, 1, 248, 213, 230, 6, 1, 231, 40, 230, 6, 1, - 237, 253, 230, 6, 1, 229, 247, 230, 6, 1, 193, 133, 230, 6, 1, 191, 89, - 230, 6, 1, 229, 186, 230, 6, 1, 199, 69, 230, 6, 1, 191, 249, 230, 6, 1, - 222, 77, 230, 6, 1, 220, 28, 230, 6, 1, 216, 195, 230, 6, 1, 212, 132, - 230, 6, 1, 205, 149, 230, 6, 1, 250, 126, 230, 6, 1, 210, 65, 230, 6, 1, - 205, 190, 230, 6, 1, 208, 39, 230, 6, 1, 203, 70, 230, 6, 1, 199, 159, - 230, 6, 31, 107, 230, 6, 31, 150, 230, 6, 31, 199, 95, 230, 6, 31, 197, - 32, 230, 6, 31, 91, 228, 142, 209, 199, 1, 251, 189, 209, 199, 1, 248, - 216, 209, 199, 1, 231, 215, 209, 199, 1, 237, 110, 209, 199, 1, 229, 247, - 209, 199, 1, 193, 140, 209, 199, 1, 191, 115, 209, 199, 1, 229, 188, 209, - 199, 1, 199, 73, 209, 199, 1, 191, 250, 209, 199, 1, 222, 108, 209, 199, - 1, 220, 34, 209, 199, 1, 216, 195, 209, 199, 1, 212, 132, 209, 199, 1, - 204, 20, 209, 199, 1, 251, 232, 209, 199, 1, 210, 65, 209, 199, 1, 205, - 192, 209, 199, 1, 208, 44, 209, 199, 1, 206, 125, 209, 199, 1, 203, 70, - 209, 199, 1, 199, 166, 209, 199, 31, 107, 209, 199, 31, 199, 95, 209, - 199, 31, 197, 32, 209, 199, 31, 91, 228, 142, 209, 199, 31, 109, 209, - 199, 31, 138, 209, 199, 193, 23, 204, 11, 218, 182, 1, 65, 218, 182, 1, - 250, 122, 218, 182, 1, 232, 53, 218, 182, 1, 238, 129, 218, 182, 1, 71, - 218, 182, 1, 196, 12, 218, 182, 1, 68, 218, 182, 1, 192, 159, 218, 182, - 1, 222, 154, 218, 182, 1, 172, 218, 182, 1, 218, 170, 218, 182, 1, 215, - 63, 218, 182, 1, 74, 218, 182, 1, 146, 218, 182, 1, 201, 179, 218, 182, - 1, 200, 43, 218, 182, 1, 66, 218, 182, 1, 233, 177, 218, 182, 1, 208, - 106, 218, 182, 1, 206, 9, 218, 182, 1, 197, 135, 218, 182, 1, 251, 134, - 218, 182, 1, 234, 105, 218, 182, 1, 218, 185, 218, 182, 1, 213, 82, 218, - 182, 1, 247, 195, 218, 182, 197, 238, 77, 153, 229, 146, 1, 65, 153, 229, - 146, 1, 71, 153, 229, 146, 1, 68, 153, 229, 146, 1, 74, 153, 229, 146, 1, - 170, 153, 229, 146, 1, 193, 190, 153, 229, 146, 1, 249, 155, 153, 229, - 146, 1, 249, 154, 153, 229, 146, 1, 168, 153, 229, 146, 1, 174, 153, 229, - 146, 1, 181, 153, 229, 146, 1, 215, 7, 153, 229, 146, 1, 214, 123, 153, - 229, 146, 1, 214, 121, 153, 229, 146, 1, 165, 153, 229, 146, 1, 207, 186, - 153, 229, 146, 1, 173, 153, 229, 146, 1, 221, 217, 153, 229, 146, 1, 229, - 179, 153, 229, 146, 1, 188, 153, 229, 146, 1, 205, 206, 153, 229, 146, 1, - 205, 69, 153, 229, 146, 1, 155, 153, 229, 146, 1, 208, 98, 153, 229, 146, - 1, 190, 190, 153, 229, 146, 1, 199, 250, 153, 229, 146, 1, 199, 145, 153, - 229, 146, 1, 199, 143, 153, 229, 146, 1, 159, 153, 229, 146, 1, 238, 34, - 153, 229, 146, 16, 195, 63, 153, 229, 146, 16, 195, 62, 153, 238, 168, 1, - 65, 153, 238, 168, 1, 71, 153, 238, 168, 1, 68, 153, 238, 168, 1, 74, - 153, 238, 168, 1, 170, 153, 238, 168, 1, 193, 190, 153, 238, 168, 1, 249, - 155, 153, 238, 168, 1, 168, 153, 238, 168, 1, 174, 153, 238, 168, 1, 181, - 153, 238, 168, 1, 214, 123, 153, 238, 168, 1, 165, 153, 238, 168, 1, 173, - 153, 238, 168, 1, 221, 217, 153, 238, 168, 1, 229, 179, 153, 238, 168, 1, - 188, 153, 238, 168, 1, 251, 16, 188, 153, 238, 168, 1, 205, 69, 153, 238, - 168, 1, 155, 153, 238, 168, 1, 208, 98, 153, 238, 168, 1, 190, 190, 153, - 238, 168, 1, 199, 145, 153, 238, 168, 1, 159, 153, 238, 168, 1, 238, 34, - 153, 238, 168, 232, 120, 234, 130, 197, 39, 153, 238, 168, 232, 120, 91, - 230, 72, 153, 238, 168, 219, 30, 206, 169, 153, 238, 168, 219, 30, 223, - 171, 153, 238, 168, 31, 107, 153, 238, 168, 31, 109, 153, 238, 168, 31, - 138, 153, 238, 168, 31, 134, 153, 238, 168, 31, 150, 153, 238, 168, 31, - 169, 153, 238, 168, 31, 175, 153, 238, 168, 31, 171, 153, 238, 168, 31, - 178, 153, 238, 168, 31, 199, 95, 153, 238, 168, 31, 197, 32, 153, 238, - 168, 31, 198, 249, 153, 238, 168, 31, 232, 137, 153, 238, 168, 31, 233, - 17, 153, 238, 168, 31, 202, 121, 153, 238, 168, 31, 203, 242, 153, 238, - 168, 31, 91, 228, 142, 153, 238, 168, 31, 105, 228, 142, 153, 238, 168, - 31, 115, 228, 142, 153, 238, 168, 31, 232, 130, 228, 142, 153, 238, 168, - 31, 232, 228, 228, 142, 153, 238, 168, 31, 202, 137, 228, 142, 153, 238, - 168, 31, 203, 248, 228, 142, 153, 238, 168, 31, 234, 166, 228, 142, 153, - 238, 168, 31, 213, 177, 228, 142, 153, 238, 168, 31, 91, 189, 153, 238, - 168, 31, 105, 189, 153, 238, 168, 31, 115, 189, 153, 238, 168, 31, 232, - 130, 189, 153, 238, 168, 31, 232, 228, 189, 153, 238, 168, 31, 202, 137, - 189, 153, 238, 168, 31, 203, 248, 189, 153, 238, 168, 31, 234, 166, 189, - 153, 238, 168, 31, 213, 177, 189, 153, 238, 168, 31, 199, 96, 189, 153, - 238, 168, 31, 197, 33, 189, 153, 238, 168, 31, 198, 250, 189, 153, 238, - 168, 31, 232, 138, 189, 153, 238, 168, 31, 233, 18, 189, 153, 238, 168, - 31, 202, 122, 189, 153, 238, 168, 31, 203, 243, 189, 153, 238, 168, 31, - 234, 156, 189, 153, 238, 168, 31, 213, 172, 189, 153, 238, 168, 31, 91, - 228, 143, 189, 153, 238, 168, 31, 105, 228, 143, 189, 153, 238, 168, 31, - 115, 228, 143, 189, 153, 238, 168, 31, 232, 130, 228, 143, 189, 153, 238, - 168, 31, 232, 228, 228, 143, 189, 153, 238, 168, 31, 202, 137, 228, 143, - 189, 153, 238, 168, 31, 203, 248, 228, 143, 189, 153, 238, 168, 31, 234, - 166, 228, 143, 189, 153, 238, 168, 31, 213, 177, 228, 143, 189, 153, 238, - 168, 232, 120, 91, 197, 40, 153, 238, 168, 232, 120, 105, 197, 39, 153, - 238, 168, 232, 120, 115, 197, 39, 153, 238, 168, 232, 120, 232, 130, 197, - 39, 153, 238, 168, 232, 120, 232, 228, 197, 39, 153, 238, 168, 232, 120, - 202, 137, 197, 39, 153, 238, 168, 232, 120, 203, 248, 197, 39, 153, 238, - 168, 232, 120, 234, 166, 197, 39, 153, 238, 168, 232, 120, 213, 177, 197, - 39, 153, 238, 168, 232, 120, 199, 96, 197, 39, 221, 201, 1, 65, 221, 201, - 18, 3, 68, 221, 201, 18, 3, 66, 221, 201, 18, 3, 117, 146, 221, 201, 18, - 3, 71, 221, 201, 18, 3, 74, 221, 201, 18, 219, 200, 77, 221, 201, 3, 55, - 206, 190, 60, 221, 201, 3, 251, 73, 221, 201, 3, 195, 35, 221, 201, 1, - 155, 221, 201, 1, 221, 217, 221, 201, 1, 231, 242, 221, 201, 1, 231, 93, - 221, 201, 1, 247, 162, 221, 201, 1, 247, 3, 221, 201, 1, 223, 34, 221, - 201, 1, 212, 103, 221, 201, 1, 197, 132, 221, 201, 1, 197, 120, 221, 201, - 1, 237, 193, 221, 201, 1, 237, 177, 221, 201, 1, 213, 81, 221, 201, 1, - 190, 190, 221, 201, 1, 199, 49, 221, 201, 1, 238, 34, 221, 201, 1, 237, - 70, 221, 201, 1, 181, 221, 201, 1, 168, 221, 201, 1, 209, 230, 221, 201, - 1, 249, 155, 221, 201, 1, 248, 205, 221, 201, 1, 174, 221, 201, 1, 170, - 221, 201, 1, 165, 221, 201, 1, 173, 221, 201, 1, 195, 188, 221, 201, 1, - 203, 166, 221, 201, 1, 201, 176, 221, 201, 1, 188, 221, 201, 1, 191, 123, - 221, 201, 1, 140, 221, 201, 1, 221, 103, 221, 201, 1, 197, 100, 221, 201, - 1, 197, 101, 221, 201, 1, 195, 70, 221, 201, 3, 249, 90, 58, 221, 201, 3, - 247, 76, 221, 201, 3, 75, 60, 221, 201, 195, 40, 221, 201, 17, 107, 221, - 201, 17, 109, 221, 201, 17, 138, 221, 201, 17, 134, 221, 201, 31, 199, - 95, 221, 201, 31, 197, 32, 221, 201, 31, 91, 228, 142, 221, 201, 31, 91, - 189, 221, 201, 232, 120, 91, 230, 72, 221, 201, 208, 154, 236, 142, 221, - 201, 208, 154, 2, 243, 12, 221, 201, 208, 154, 243, 12, 221, 201, 208, - 154, 238, 230, 164, 221, 201, 208, 154, 217, 85, 221, 201, 208, 154, 218, - 248, 221, 201, 208, 154, 237, 240, 221, 201, 208, 154, 55, 237, 240, 221, - 201, 208, 154, 219, 108, 39, 202, 3, 251, 31, 1, 229, 247, 39, 202, 3, - 251, 31, 1, 220, 28, 39, 202, 3, 251, 31, 1, 229, 186, 39, 202, 3, 251, - 31, 1, 216, 195, 39, 202, 3, 251, 31, 1, 208, 39, 39, 202, 3, 251, 31, 1, - 193, 133, 39, 202, 3, 251, 31, 1, 203, 70, 39, 202, 3, 251, 31, 1, 207, - 39, 39, 202, 3, 251, 31, 1, 248, 213, 39, 202, 3, 251, 31, 1, 199, 159, - 39, 202, 3, 251, 31, 1, 205, 123, 39, 202, 3, 251, 31, 1, 222, 77, 39, - 202, 3, 251, 31, 1, 212, 132, 39, 202, 3, 251, 31, 1, 221, 196, 39, 202, - 3, 251, 31, 1, 205, 190, 39, 202, 3, 251, 31, 1, 205, 149, 39, 202, 3, - 251, 31, 1, 233, 61, 39, 202, 3, 251, 31, 1, 251, 194, 39, 202, 3, 251, - 31, 1, 250, 126, 39, 202, 3, 251, 31, 1, 237, 67, 39, 202, 3, 251, 31, 1, - 231, 40, 39, 202, 3, 251, 31, 1, 237, 254, 39, 202, 3, 251, 31, 1, 231, - 81, 39, 202, 3, 251, 31, 1, 199, 69, 39, 202, 3, 251, 31, 1, 191, 89, 39, - 202, 3, 251, 31, 1, 237, 64, 39, 202, 3, 251, 31, 1, 191, 249, 39, 202, - 3, 251, 31, 1, 199, 35, 39, 202, 3, 251, 31, 1, 199, 14, 39, 202, 3, 251, - 31, 31, 107, 39, 202, 3, 251, 31, 31, 233, 17, 39, 202, 3, 251, 31, 167, - 223, 146, 39, 186, 251, 31, 1, 229, 212, 39, 186, 251, 31, 1, 220, 37, - 39, 186, 251, 31, 1, 230, 83, 39, 186, 251, 31, 1, 216, 210, 39, 186, - 251, 31, 1, 208, 91, 39, 186, 251, 31, 1, 193, 133, 39, 186, 251, 31, 1, - 234, 22, 39, 186, 251, 31, 1, 207, 72, 39, 186, 251, 31, 1, 248, 247, 39, - 186, 251, 31, 1, 199, 114, 39, 186, 251, 31, 1, 234, 23, 39, 186, 251, - 31, 1, 222, 108, 39, 186, 251, 31, 1, 213, 26, 39, 186, 251, 31, 1, 221, - 212, 39, 186, 251, 31, 1, 205, 193, 39, 186, 251, 31, 1, 234, 21, 39, - 186, 251, 31, 1, 233, 48, 39, 186, 251, 31, 1, 251, 194, 39, 186, 251, - 31, 1, 251, 232, 39, 186, 251, 31, 1, 238, 28, 39, 186, 251, 31, 1, 231, - 158, 39, 186, 251, 31, 1, 238, 5, 39, 186, 251, 31, 1, 231, 88, 39, 186, - 251, 31, 1, 199, 219, 39, 186, 251, 31, 1, 191, 113, 39, 186, 251, 31, 1, - 199, 41, 39, 186, 251, 31, 1, 192, 75, 39, 186, 251, 31, 1, 199, 29, 39, - 186, 251, 31, 1, 191, 116, 39, 186, 251, 31, 31, 107, 39, 186, 251, 31, - 31, 199, 95, 39, 186, 251, 31, 31, 197, 32, 217, 83, 1, 251, 192, 217, - 83, 1, 248, 213, 217, 83, 1, 248, 196, 217, 83, 1, 231, 40, 217, 83, 1, - 231, 66, 217, 83, 1, 237, 254, 217, 83, 1, 229, 247, 217, 83, 1, 193, - 133, 217, 83, 3, 196, 158, 217, 83, 1, 191, 91, 217, 83, 1, 191, 64, 217, - 83, 1, 223, 14, 217, 83, 1, 222, 250, 217, 83, 1, 229, 186, 217, 83, 1, - 199, 69, 217, 83, 1, 191, 249, 217, 83, 1, 222, 77, 217, 83, 1, 192, 217, - 217, 83, 1, 221, 203, 217, 83, 1, 220, 28, 217, 83, 1, 237, 63, 217, 83, - 1, 199, 40, 217, 83, 1, 216, 195, 217, 83, 1, 212, 132, 217, 83, 1, 205, - 149, 217, 83, 1, 250, 128, 217, 83, 1, 252, 160, 217, 83, 1, 210, 65, - 217, 83, 1, 233, 61, 217, 83, 1, 205, 190, 217, 83, 1, 208, 39, 217, 83, - 1, 192, 193, 217, 83, 1, 208, 66, 217, 83, 1, 207, 39, 217, 83, 1, 203, - 70, 217, 83, 1, 201, 144, 217, 83, 1, 199, 159, 217, 83, 252, 70, 87, 58, - 217, 83, 252, 70, 87, 60, 217, 83, 31, 107, 217, 83, 31, 150, 217, 83, - 31, 199, 95, 217, 83, 31, 197, 32, 217, 83, 31, 91, 228, 142, 217, 83, - 208, 154, 201, 103, 217, 83, 208, 154, 232, 203, 217, 83, 208, 154, 55, - 75, 193, 53, 236, 142, 217, 83, 208, 154, 75, 193, 53, 236, 142, 217, 83, - 208, 154, 236, 142, 217, 83, 208, 154, 105, 236, 140, 217, 83, 208, 154, - 219, 115, 233, 5, 250, 144, 1, 65, 250, 144, 1, 252, 208, 250, 144, 1, - 251, 70, 250, 144, 1, 252, 166, 250, 144, 1, 251, 134, 250, 144, 1, 252, - 168, 250, 144, 1, 252, 27, 250, 144, 1, 252, 23, 250, 144, 1, 71, 250, - 144, 1, 234, 190, 250, 144, 1, 74, 250, 144, 1, 211, 89, 250, 144, 1, 68, - 250, 144, 1, 223, 201, 250, 144, 1, 66, 250, 144, 1, 196, 30, 250, 144, - 1, 222, 24, 250, 144, 1, 192, 214, 250, 144, 1, 192, 173, 250, 144, 1, - 192, 184, 250, 144, 1, 231, 167, 250, 144, 1, 231, 124, 250, 144, 1, 231, - 79, 250, 144, 1, 247, 44, 250, 144, 1, 223, 12, 250, 144, 1, 199, 145, - 250, 144, 1, 199, 33, 250, 144, 1, 237, 148, 250, 144, 1, 237, 61, 250, - 144, 1, 197, 127, 250, 144, 1, 210, 65, 250, 144, 1, 233, 61, 250, 144, - 1, 249, 19, 250, 144, 1, 248, 198, 250, 144, 1, 214, 57, 250, 144, 1, - 213, 228, 250, 144, 1, 213, 229, 250, 144, 1, 214, 123, 250, 144, 1, 212, - 92, 250, 144, 1, 213, 76, 250, 144, 1, 216, 234, 250, 144, 1, 229, 75, - 250, 144, 1, 191, 173, 250, 144, 1, 192, 80, 250, 144, 1, 195, 153, 250, - 144, 1, 207, 115, 250, 144, 1, 219, 240, 250, 144, 1, 205, 69, 250, 144, - 1, 191, 87, 250, 144, 1, 203, 114, 250, 144, 1, 191, 62, 250, 144, 1, - 202, 230, 250, 144, 1, 201, 145, 250, 144, 1, 229, 247, 250, 144, 252, - 70, 77, 198, 138, 105, 185, 139, 91, 75, 208, 153, 2, 105, 185, 139, 91, - 75, 208, 153, 220, 15, 105, 185, 139, 91, 75, 208, 153, 220, 15, 91, 75, - 139, 105, 185, 208, 153, 220, 15, 105, 206, 186, 139, 91, 206, 190, 208, - 153, 220, 15, 91, 206, 190, 139, 105, 206, 186, 208, 153, 223, 124, 210, - 108, 1, 251, 192, 223, 124, 210, 108, 1, 248, 213, 223, 124, 210, 108, 1, - 231, 40, 223, 124, 210, 108, 1, 237, 254, 223, 124, 210, 108, 1, 229, - 247, 223, 124, 210, 108, 1, 193, 133, 223, 124, 210, 108, 1, 191, 91, - 223, 124, 210, 108, 1, 229, 186, 223, 124, 210, 108, 1, 199, 69, 223, - 124, 210, 108, 1, 191, 249, 223, 124, 210, 108, 1, 222, 77, 223, 124, - 210, 108, 1, 220, 28, 223, 124, 210, 108, 1, 216, 195, 223, 124, 210, - 108, 1, 212, 132, 223, 124, 210, 108, 1, 205, 149, 223, 124, 210, 108, 1, - 250, 128, 223, 124, 210, 108, 1, 210, 65, 223, 124, 210, 108, 1, 205, - 190, 223, 124, 210, 108, 1, 208, 39, 223, 124, 210, 108, 1, 207, 39, 223, - 124, 210, 108, 1, 203, 70, 223, 124, 210, 108, 1, 199, 159, 223, 124, - 210, 108, 31, 107, 223, 124, 210, 108, 31, 109, 223, 124, 210, 108, 31, - 138, 223, 124, 210, 108, 31, 134, 223, 124, 210, 108, 31, 199, 95, 223, - 124, 210, 108, 31, 197, 32, 223, 124, 210, 108, 31, 91, 228, 142, 223, - 124, 210, 108, 31, 91, 189, 223, 124, 210, 201, 1, 251, 192, 223, 124, - 210, 201, 1, 248, 213, 223, 124, 210, 201, 1, 231, 40, 223, 124, 210, - 201, 1, 237, 254, 223, 124, 210, 201, 1, 229, 247, 223, 124, 210, 201, 1, - 193, 132, 223, 124, 210, 201, 1, 191, 91, 223, 124, 210, 201, 1, 229, - 186, 223, 124, 210, 201, 1, 199, 69, 223, 124, 210, 201, 1, 191, 249, - 223, 124, 210, 201, 1, 222, 77, 223, 124, 210, 201, 1, 220, 28, 223, 124, - 210, 201, 1, 216, 194, 223, 124, 210, 201, 1, 212, 132, 223, 124, 210, - 201, 1, 205, 149, 223, 124, 210, 201, 1, 210, 65, 223, 124, 210, 201, 1, - 205, 190, 223, 124, 210, 201, 1, 203, 70, 223, 124, 210, 201, 1, 199, - 159, 223, 124, 210, 201, 31, 107, 223, 124, 210, 201, 31, 109, 223, 124, - 210, 201, 31, 138, 223, 124, 210, 201, 31, 134, 223, 124, 210, 201, 31, - 199, 95, 223, 124, 210, 201, 31, 197, 32, 223, 124, 210, 201, 31, 91, - 228, 142, 223, 124, 210, 201, 31, 91, 189, 208, 179, 210, 201, 1, 251, - 192, 208, 179, 210, 201, 1, 248, 213, 208, 179, 210, 201, 1, 231, 40, - 208, 179, 210, 201, 1, 237, 254, 208, 179, 210, 201, 1, 229, 247, 208, - 179, 210, 201, 1, 193, 132, 208, 179, 210, 201, 1, 191, 91, 208, 179, - 210, 201, 1, 229, 186, 208, 179, 210, 201, 1, 191, 249, 208, 179, 210, - 201, 1, 222, 77, 208, 179, 210, 201, 1, 220, 28, 208, 179, 210, 201, 1, - 216, 194, 208, 179, 210, 201, 1, 212, 132, 208, 179, 210, 201, 1, 205, - 149, 208, 179, 210, 201, 1, 210, 65, 208, 179, 210, 201, 1, 205, 190, - 208, 179, 210, 201, 1, 203, 70, 208, 179, 210, 201, 1, 199, 159, 208, - 179, 210, 201, 205, 55, 77, 208, 179, 210, 201, 154, 205, 55, 77, 208, - 179, 210, 201, 232, 130, 185, 4, 238, 219, 208, 179, 210, 201, 232, 130, - 185, 4, 236, 142, 208, 179, 210, 201, 31, 107, 208, 179, 210, 201, 31, - 109, 208, 179, 210, 201, 31, 138, 208, 179, 210, 201, 31, 134, 208, 179, - 210, 201, 31, 199, 95, 208, 179, 210, 201, 31, 197, 32, 208, 179, 210, - 201, 31, 91, 228, 142, 39, 197, 61, 1, 211, 46, 65, 39, 197, 61, 1, 192, - 68, 65, 39, 197, 61, 1, 192, 68, 252, 27, 39, 197, 61, 1, 211, 46, 68, - 39, 197, 61, 1, 192, 68, 68, 39, 197, 61, 1, 192, 68, 71, 39, 197, 61, 1, - 211, 46, 74, 39, 197, 61, 1, 211, 46, 211, 153, 39, 197, 61, 1, 192, 68, - 211, 153, 39, 197, 61, 1, 211, 46, 252, 157, 39, 197, 61, 1, 192, 68, - 252, 157, 39, 197, 61, 1, 211, 46, 252, 26, 39, 197, 61, 1, 192, 68, 252, - 26, 39, 197, 61, 1, 211, 46, 251, 255, 39, 197, 61, 1, 192, 68, 251, 255, - 39, 197, 61, 1, 211, 46, 252, 21, 39, 197, 61, 1, 192, 68, 252, 21, 39, - 197, 61, 1, 211, 46, 252, 44, 39, 197, 61, 1, 192, 68, 252, 44, 39, 197, - 61, 1, 211, 46, 252, 25, 39, 197, 61, 1, 211, 46, 233, 184, 39, 197, 61, - 1, 192, 68, 233, 184, 39, 197, 61, 1, 211, 46, 250, 133, 39, 197, 61, 1, - 192, 68, 250, 133, 39, 197, 61, 1, 211, 46, 252, 8, 39, 197, 61, 1, 192, - 68, 252, 8, 39, 197, 61, 1, 211, 46, 252, 19, 39, 197, 61, 1, 192, 68, - 252, 19, 39, 197, 61, 1, 211, 46, 211, 151, 39, 197, 61, 1, 192, 68, 211, - 151, 39, 197, 61, 1, 211, 46, 251, 209, 39, 197, 61, 1, 192, 68, 251, - 209, 39, 197, 61, 1, 211, 46, 252, 18, 39, 197, 61, 1, 211, 46, 234, 120, - 39, 197, 61, 1, 211, 46, 234, 116, 39, 197, 61, 1, 211, 46, 251, 134, 39, - 197, 61, 1, 211, 46, 252, 16, 39, 197, 61, 1, 192, 68, 252, 16, 39, 197, - 61, 1, 211, 46, 234, 82, 39, 197, 61, 1, 192, 68, 234, 82, 39, 197, 61, - 1, 211, 46, 234, 102, 39, 197, 61, 1, 192, 68, 234, 102, 39, 197, 61, 1, - 211, 46, 234, 68, 39, 197, 61, 1, 192, 68, 234, 68, 39, 197, 61, 1, 192, - 68, 251, 124, 39, 197, 61, 1, 211, 46, 234, 90, 39, 197, 61, 1, 192, 68, - 252, 15, 39, 197, 61, 1, 211, 46, 234, 58, 39, 197, 61, 1, 211, 46, 211, - 80, 39, 197, 61, 1, 211, 46, 228, 30, 39, 197, 61, 1, 211, 46, 234, 199, - 39, 197, 61, 1, 192, 68, 234, 199, 39, 197, 61, 1, 211, 46, 251, 39, 39, - 197, 61, 1, 192, 68, 251, 39, 39, 197, 61, 1, 211, 46, 223, 81, 39, 197, - 61, 1, 192, 68, 223, 81, 39, 197, 61, 1, 211, 46, 211, 60, 39, 197, 61, - 1, 192, 68, 211, 60, 39, 197, 61, 1, 211, 46, 251, 35, 39, 197, 61, 1, - 192, 68, 251, 35, 39, 197, 61, 1, 211, 46, 252, 14, 39, 197, 61, 1, 211, - 46, 250, 221, 39, 197, 61, 1, 211, 46, 252, 12, 39, 197, 61, 1, 211, 46, - 250, 211, 39, 197, 61, 1, 192, 68, 250, 211, 39, 197, 61, 1, 211, 46, - 234, 14, 39, 197, 61, 1, 192, 68, 234, 14, 39, 197, 61, 1, 211, 46, 250, - 184, 39, 197, 61, 1, 192, 68, 250, 184, 39, 197, 61, 1, 211, 46, 252, 9, - 39, 197, 61, 1, 192, 68, 252, 9, 39, 197, 61, 1, 211, 46, 211, 32, 39, - 197, 61, 1, 211, 46, 249, 73, 39, 177, 6, 1, 65, 39, 177, 6, 1, 252, 208, - 39, 177, 6, 1, 234, 201, 39, 177, 6, 1, 251, 146, 39, 177, 6, 1, 234, - 199, 39, 177, 6, 1, 234, 102, 39, 177, 6, 1, 234, 195, 39, 177, 6, 1, - 234, 194, 39, 177, 6, 1, 251, 127, 39, 177, 6, 1, 71, 39, 177, 6, 1, 242, - 222, 71, 39, 177, 6, 1, 234, 190, 39, 177, 6, 1, 234, 183, 39, 177, 6, 1, - 234, 182, 39, 177, 6, 1, 234, 178, 39, 177, 6, 1, 234, 175, 39, 177, 6, - 1, 68, 39, 177, 6, 1, 223, 201, 39, 177, 6, 1, 234, 152, 39, 177, 6, 1, - 234, 149, 39, 177, 6, 1, 251, 218, 39, 177, 6, 1, 196, 86, 39, 177, 6, 1, - 234, 142, 39, 177, 6, 1, 234, 119, 39, 177, 6, 1, 234, 116, 39, 177, 6, - 1, 234, 105, 39, 177, 6, 1, 234, 68, 39, 177, 6, 1, 74, 39, 177, 6, 1, - 211, 89, 39, 177, 6, 1, 213, 184, 211, 153, 39, 177, 6, 1, 206, 59, 211, - 153, 39, 177, 6, 1, 211, 152, 39, 177, 6, 1, 234, 58, 39, 177, 6, 1, 234, - 110, 39, 177, 6, 1, 234, 36, 39, 177, 6, 1, 203, 41, 234, 36, 39, 177, 6, - 1, 234, 24, 39, 177, 6, 1, 234, 3, 39, 177, 6, 1, 234, 1, 39, 177, 6, 1, - 234, 82, 39, 177, 6, 1, 233, 245, 39, 177, 6, 1, 234, 197, 39, 177, 6, 1, - 66, 39, 177, 6, 1, 196, 30, 39, 177, 6, 1, 213, 184, 196, 152, 39, 177, - 6, 1, 206, 59, 196, 152, 39, 177, 6, 1, 233, 232, 39, 177, 6, 1, 233, - 184, 39, 177, 6, 1, 233, 179, 39, 177, 6, 1, 234, 81, 56, 39, 177, 6, 1, - 196, 45, 39, 177, 2, 1, 65, 39, 177, 2, 1, 252, 208, 39, 177, 2, 1, 234, - 201, 39, 177, 2, 1, 251, 146, 39, 177, 2, 1, 234, 199, 39, 177, 2, 1, - 234, 102, 39, 177, 2, 1, 234, 195, 39, 177, 2, 1, 234, 194, 39, 177, 2, - 1, 251, 127, 39, 177, 2, 1, 71, 39, 177, 2, 1, 242, 222, 71, 39, 177, 2, - 1, 234, 190, 39, 177, 2, 1, 234, 183, 39, 177, 2, 1, 234, 182, 39, 177, - 2, 1, 234, 178, 39, 177, 2, 1, 234, 175, 39, 177, 2, 1, 68, 39, 177, 2, - 1, 223, 201, 39, 177, 2, 1, 234, 152, 39, 177, 2, 1, 234, 149, 39, 177, - 2, 1, 251, 218, 39, 177, 2, 1, 196, 86, 39, 177, 2, 1, 234, 142, 39, 177, - 2, 1, 234, 119, 39, 177, 2, 1, 234, 116, 39, 177, 2, 1, 234, 105, 39, - 177, 2, 1, 234, 68, 39, 177, 2, 1, 74, 39, 177, 2, 1, 211, 89, 39, 177, - 2, 1, 213, 184, 211, 153, 39, 177, 2, 1, 206, 59, 211, 153, 39, 177, 2, - 1, 211, 152, 39, 177, 2, 1, 234, 58, 39, 177, 2, 1, 234, 110, 39, 177, 2, - 1, 234, 36, 39, 177, 2, 1, 203, 41, 234, 36, 39, 177, 2, 1, 234, 24, 39, - 177, 2, 1, 234, 3, 39, 177, 2, 1, 234, 1, 39, 177, 2, 1, 234, 82, 39, - 177, 2, 1, 233, 245, 39, 177, 2, 1, 234, 197, 39, 177, 2, 1, 66, 39, 177, - 2, 1, 196, 30, 39, 177, 2, 1, 213, 184, 196, 152, 39, 177, 2, 1, 206, 59, - 196, 152, 39, 177, 2, 1, 233, 232, 39, 177, 2, 1, 233, 184, 39, 177, 2, - 1, 233, 179, 39, 177, 2, 1, 234, 81, 56, 39, 177, 2, 1, 196, 45, 39, 177, - 31, 107, 39, 177, 31, 150, 39, 177, 31, 199, 95, 39, 177, 31, 233, 17, - 39, 177, 31, 91, 228, 142, 39, 177, 31, 91, 189, 230, 26, 206, 143, 1, - 65, 230, 26, 206, 143, 1, 249, 155, 230, 26, 206, 143, 1, 168, 230, 26, - 206, 143, 1, 190, 190, 230, 26, 206, 143, 1, 197, 132, 230, 26, 206, 143, - 1, 223, 34, 230, 26, 206, 143, 1, 247, 162, 230, 26, 206, 143, 1, 140, - 230, 26, 206, 143, 1, 221, 217, 230, 26, 206, 143, 1, 233, 111, 230, 26, - 206, 143, 1, 238, 34, 230, 26, 206, 143, 1, 237, 193, 230, 26, 206, 143, - 1, 165, 230, 26, 206, 143, 1, 206, 110, 230, 26, 206, 143, 1, 191, 123, - 230, 26, 206, 143, 1, 188, 230, 26, 206, 143, 1, 203, 166, 230, 26, 206, - 143, 1, 155, 230, 26, 206, 143, 1, 231, 242, 230, 26, 206, 143, 1, 173, - 230, 26, 206, 143, 1, 174, 230, 26, 206, 143, 1, 181, 230, 26, 206, 143, - 1, 193, 190, 230, 26, 206, 143, 1, 221, 139, 193, 190, 230, 26, 206, 143, - 1, 170, 230, 26, 206, 143, 1, 221, 139, 170, 230, 26, 206, 143, 1, 214, - 70, 230, 26, 206, 143, 1, 212, 103, 230, 26, 206, 143, 1, 195, 188, 230, - 26, 206, 143, 18, 65, 230, 26, 206, 143, 18, 68, 230, 26, 206, 143, 18, - 66, 230, 26, 206, 143, 18, 71, 230, 26, 206, 143, 18, 74, 230, 26, 206, - 143, 87, 205, 174, 230, 26, 206, 143, 87, 215, 9, 221, 180, 230, 26, 206, - 143, 3, 230, 20, 230, 26, 206, 143, 3, 199, 218, 230, 26, 206, 143, 3, - 199, 192, 230, 26, 206, 143, 3, 199, 172, 230, 26, 206, 143, 17, 191, 77, - 230, 26, 206, 143, 17, 107, 230, 26, 206, 143, 17, 109, 230, 26, 206, - 143, 17, 138, 230, 26, 206, 143, 17, 134, 230, 26, 206, 143, 17, 150, - 230, 26, 206, 143, 17, 169, 230, 26, 206, 143, 17, 175, 230, 26, 206, - 143, 17, 171, 230, 26, 206, 143, 17, 178, 206, 47, 17, 107, 206, 47, 17, - 109, 206, 47, 17, 138, 206, 47, 17, 134, 206, 47, 17, 150, 206, 47, 17, - 169, 206, 47, 17, 175, 206, 47, 17, 171, 206, 47, 17, 178, 206, 47, 31, - 199, 95, 206, 47, 31, 197, 32, 206, 47, 31, 198, 249, 206, 47, 31, 232, - 137, 206, 47, 31, 233, 17, 206, 47, 31, 202, 121, 206, 47, 31, 203, 242, - 206, 47, 31, 234, 155, 206, 47, 31, 213, 171, 206, 47, 31, 91, 228, 142, - 206, 47, 31, 105, 228, 142, 206, 47, 31, 115, 228, 142, 206, 47, 31, 232, - 130, 228, 142, 206, 47, 31, 232, 228, 228, 142, 206, 47, 31, 202, 137, - 228, 142, 206, 47, 31, 203, 248, 228, 142, 206, 47, 31, 234, 166, 228, - 142, 206, 47, 31, 213, 177, 228, 142, 206, 47, 232, 120, 91, 230, 72, - 206, 47, 232, 120, 91, 208, 24, 206, 47, 232, 120, 91, 199, 0, 206, 47, - 232, 120, 105, 198, 253, 192, 39, 1, 234, 126, 192, 39, 1, 249, 19, 192, - 39, 1, 210, 65, 192, 39, 1, 209, 216, 192, 39, 1, 199, 33, 192, 39, 1, - 205, 69, 192, 39, 1, 243, 18, 192, 39, 1, 243, 85, 192, 39, 1, 243, 99, - 192, 39, 1, 229, 179, 192, 39, 1, 192, 220, 192, 39, 1, 238, 5, 192, 39, - 1, 191, 108, 192, 39, 1, 165, 192, 39, 1, 207, 7, 192, 39, 1, 191, 123, - 192, 39, 1, 223, 34, 192, 39, 1, 202, 175, 192, 39, 1, 203, 70, 192, 39, - 1, 205, 193, 192, 39, 1, 238, 28, 192, 39, 1, 190, 190, 192, 39, 1, 191, - 87, 192, 39, 1, 233, 186, 192, 39, 1, 192, 208, 192, 39, 1, 233, 111, - 192, 39, 1, 195, 188, 192, 39, 1, 195, 189, 251, 159, 20, 192, 39, 1, - 208, 91, 192, 39, 1, 222, 108, 192, 39, 1, 221, 214, 192, 39, 1, 231, - 229, 192, 39, 1, 220, 37, 192, 39, 1, 216, 48, 192, 39, 1, 212, 132, 192, - 39, 1, 196, 120, 192, 39, 1, 193, 133, 192, 39, 1, 210, 252, 192, 39, 1, - 233, 226, 192, 39, 1, 229, 254, 192, 39, 1, 191, 240, 192, 39, 1, 234, 1, - 192, 39, 33, 230, 60, 77, 192, 39, 33, 217, 144, 77, 192, 39, 228, 88, - 77, 192, 39, 1, 220, 38, 4, 75, 58, 192, 39, 1, 191, 241, 4, 243, 4, 58, - 9, 2, 130, 193, 23, 205, 172, 9, 2, 130, 193, 23, 208, 81, 9, 2, 130, - 193, 23, 217, 143, 39, 202, 29, 1, 251, 192, 39, 202, 29, 1, 53, 251, - 192, 39, 202, 29, 1, 248, 213, 39, 202, 29, 1, 53, 248, 213, 39, 202, 29, - 1, 231, 40, 39, 202, 29, 1, 229, 247, 39, 202, 29, 1, 53, 229, 247, 39, - 202, 29, 1, 193, 133, 39, 202, 29, 1, 191, 91, 39, 202, 29, 1, 229, 186, - 39, 202, 29, 1, 191, 249, 39, 202, 29, 1, 222, 77, 39, 202, 29, 1, 220, - 28, 39, 202, 29, 1, 216, 195, 39, 202, 29, 1, 212, 132, 39, 202, 29, 1, - 53, 212, 132, 39, 202, 29, 1, 53, 212, 133, 4, 81, 199, 215, 39, 202, 29, - 1, 205, 149, 39, 202, 29, 1, 250, 128, 39, 202, 29, 1, 251, 159, 250, - 128, 39, 202, 29, 1, 210, 65, 39, 202, 29, 1, 205, 190, 39, 202, 29, 1, - 53, 205, 190, 39, 202, 29, 1, 53, 205, 191, 4, 81, 199, 215, 39, 202, 29, - 1, 207, 37, 39, 202, 29, 1, 203, 70, 39, 202, 29, 1, 199, 159, 39, 202, - 29, 1, 53, 199, 159, 39, 202, 29, 1, 53, 199, 160, 4, 81, 199, 215, 39, - 202, 29, 31, 107, 39, 202, 29, 31, 109, 39, 202, 29, 31, 138, 39, 202, - 29, 31, 134, 39, 202, 29, 31, 150, 39, 202, 29, 31, 199, 95, 39, 202, 29, - 31, 197, 32, 39, 202, 29, 31, 198, 249, 39, 202, 29, 31, 91, 228, 142, - 39, 202, 29, 232, 120, 91, 230, 72, 39, 202, 29, 34, 250, 127, 202, 29, - 1, 251, 192, 202, 29, 1, 248, 213, 202, 29, 1, 231, 40, 202, 29, 1, 229, - 247, 202, 29, 1, 193, 133, 202, 29, 1, 191, 91, 202, 29, 1, 229, 186, - 202, 29, 1, 191, 249, 202, 29, 1, 222, 77, 202, 29, 1, 220, 28, 202, 29, - 1, 216, 195, 202, 29, 1, 212, 132, 202, 29, 1, 205, 149, 202, 29, 1, 250, - 128, 202, 29, 1, 210, 65, 202, 29, 1, 205, 190, 202, 29, 1, 207, 38, 202, - 29, 1, 203, 70, 202, 29, 1, 199, 159, 202, 29, 1, 233, 32, 202, 29, 1, - 219, 184, 202, 29, 223, 151, 203, 70, 202, 29, 33, 75, 60, 202, 29, 33, - 105, 185, 60, 202, 29, 33, 75, 58, 202, 29, 33, 105, 185, 58, 202, 29, - 33, 238, 167, 58, 202, 29, 33, 238, 167, 60, 202, 29, 33, 228, 253, 58, - 202, 29, 33, 228, 253, 60, 202, 29, 33, 180, 228, 253, 60, 202, 29, 33, - 207, 40, 60, 202, 29, 33, 201, 29, 60, 202, 29, 31, 107, 202, 29, 31, - 199, 95, 202, 29, 31, 197, 32, 202, 29, 31, 91, 228, 142, 202, 29, 208, - 154, 105, 81, 249, 78, 202, 29, 208, 154, 105, 81, 249, 79, 4, 236, 140, - 202, 29, 208, 154, 243, 13, 4, 236, 142, 202, 29, 208, 154, 105, 243, 10, - 4, 236, 140, 202, 29, 208, 154, 132, 243, 13, 4, 236, 142, 39, 196, 19, - 1, 251, 192, 39, 196, 19, 1, 248, 213, 39, 196, 19, 1, 231, 39, 39, 196, - 19, 1, 193, 133, 39, 196, 19, 1, 191, 91, 39, 196, 19, 1, 53, 229, 186, - 39, 196, 19, 1, 191, 249, 39, 196, 19, 1, 222, 77, 39, 196, 19, 1, 220, - 28, 39, 196, 19, 1, 216, 195, 39, 196, 19, 1, 212, 132, 39, 196, 19, 1, - 205, 149, 39, 196, 19, 1, 210, 65, 39, 196, 19, 1, 205, 190, 39, 196, 19, - 1, 207, 39, 39, 196, 19, 1, 203, 70, 39, 196, 19, 1, 199, 159, 39, 196, - 19, 1, 219, 184, 39, 196, 19, 33, 75, 58, 39, 196, 19, 33, 75, 60, 39, - 196, 19, 33, 105, 185, 58, 39, 196, 19, 33, 105, 185, 60, 39, 196, 19, - 208, 154, 164, 39, 196, 19, 208, 154, 105, 249, 78, 39, 196, 19, 208, - 154, 105, 236, 140, 39, 196, 19, 208, 154, 232, 130, 236, 140, 243, 63, - 1, 251, 192, 243, 63, 1, 2, 251, 192, 243, 63, 1, 248, 213, 243, 63, 1, - 231, 40, 243, 63, 1, 237, 254, 243, 63, 1, 229, 247, 243, 63, 1, 193, - 133, 243, 63, 1, 238, 176, 193, 133, 243, 63, 1, 191, 91, 243, 63, 1, - 229, 186, 243, 63, 1, 191, 249, 243, 63, 1, 222, 77, 243, 63, 1, 220, 28, - 243, 63, 1, 216, 195, 243, 63, 1, 212, 132, 243, 63, 1, 205, 149, 243, - 63, 1, 250, 128, 243, 63, 1, 210, 65, 243, 63, 1, 207, 39, 243, 63, 1, - 203, 70, 243, 63, 1, 199, 159, 243, 63, 31, 107, 243, 63, 31, 109, 243, - 63, 31, 138, 243, 63, 31, 134, 243, 63, 31, 199, 95, 243, 63, 31, 197, - 32, 243, 63, 31, 91, 228, 142, 234, 118, 1, 251, 192, 234, 118, 1, 248, - 213, 234, 118, 1, 231, 40, 234, 118, 1, 237, 254, 234, 118, 1, 229, 247, - 234, 118, 1, 193, 133, 234, 118, 1, 191, 91, 234, 118, 1, 229, 186, 234, - 118, 1, 199, 69, 234, 118, 1, 191, 249, 234, 118, 1, 222, 77, 234, 118, - 1, 220, 28, 234, 118, 1, 216, 195, 234, 118, 1, 212, 132, 234, 118, 1, - 205, 149, 234, 118, 1, 250, 128, 234, 118, 1, 210, 65, 234, 118, 1, 205, - 190, 234, 118, 1, 208, 39, 234, 118, 1, 207, 39, 234, 118, 1, 203, 70, - 234, 118, 1, 199, 159, 234, 118, 34, 191, 90, 162, 3, 247, 121, 162, 3, - 251, 73, 162, 3, 195, 35, 162, 3, 222, 239, 162, 3, 196, 75, 162, 1, 65, - 162, 1, 252, 208, 162, 1, 68, 162, 1, 223, 201, 162, 1, 66, 162, 1, 196, - 30, 162, 1, 117, 146, 162, 1, 117, 206, 111, 162, 1, 117, 172, 162, 1, - 117, 219, 76, 162, 1, 71, 162, 1, 251, 238, 162, 1, 74, 162, 1, 250, 165, - 162, 1, 155, 162, 1, 221, 217, 162, 1, 231, 242, 162, 1, 231, 93, 162, 1, - 214, 70, 162, 1, 247, 162, 162, 1, 247, 3, 162, 1, 223, 34, 162, 1, 222, - 254, 162, 1, 212, 103, 162, 1, 197, 132, 162, 1, 197, 120, 162, 1, 237, - 193, 162, 1, 237, 177, 162, 1, 213, 81, 162, 1, 190, 190, 162, 1, 199, - 49, 162, 1, 238, 34, 162, 1, 237, 70, 162, 1, 181, 162, 1, 168, 162, 1, - 209, 230, 162, 1, 249, 155, 162, 1, 248, 205, 162, 1, 174, 162, 1, 170, - 162, 1, 165, 162, 1, 173, 162, 1, 195, 188, 162, 1, 203, 166, 162, 1, - 201, 176, 162, 1, 188, 162, 1, 140, 162, 1, 219, 75, 162, 1, 39, 44, 219, - 64, 162, 1, 39, 44, 206, 110, 162, 1, 39, 44, 213, 63, 162, 18, 3, 252, - 208, 162, 18, 3, 248, 199, 252, 208, 162, 18, 3, 68, 162, 18, 3, 223, - 201, 162, 18, 3, 66, 162, 18, 3, 196, 30, 162, 18, 3, 117, 146, 162, 18, - 3, 117, 206, 111, 162, 18, 3, 117, 172, 162, 18, 3, 117, 219, 76, 162, - 18, 3, 71, 162, 18, 3, 251, 238, 162, 18, 3, 74, 162, 18, 3, 250, 165, - 162, 195, 40, 162, 237, 240, 162, 55, 237, 240, 162, 208, 154, 236, 142, - 162, 208, 154, 55, 236, 142, 162, 208, 154, 219, 114, 162, 208, 154, 238, - 230, 164, 162, 208, 154, 218, 248, 162, 31, 107, 162, 31, 109, 162, 31, - 138, 162, 31, 134, 162, 31, 150, 162, 31, 169, 162, 31, 175, 162, 31, - 171, 162, 31, 178, 162, 31, 199, 95, 162, 31, 197, 32, 162, 31, 198, 249, - 162, 31, 232, 137, 162, 31, 233, 17, 162, 31, 202, 121, 162, 31, 203, - 242, 162, 31, 234, 155, 162, 31, 213, 171, 162, 31, 91, 228, 142, 162, - 31, 91, 189, 162, 17, 191, 77, 162, 17, 107, 162, 17, 109, 162, 17, 138, - 162, 17, 134, 162, 17, 150, 162, 17, 169, 162, 17, 175, 162, 17, 171, - 162, 17, 178, 162, 3, 39, 44, 195, 40, 162, 1, 39, 44, 203, 41, 71, 162, - 1, 39, 44, 203, 41, 74, 162, 18, 3, 39, 44, 203, 41, 71, 162, 18, 3, 39, - 44, 203, 41, 74, 162, 1, 39, 44, 219, 75, 162, 31, 222, 198, 222, 101, 3, - 247, 121, 222, 101, 3, 251, 73, 222, 101, 3, 195, 35, 222, 101, 1, 65, - 222, 101, 1, 252, 208, 222, 101, 1, 68, 222, 101, 1, 223, 201, 222, 101, - 1, 66, 222, 101, 1, 196, 30, 222, 101, 1, 71, 222, 101, 1, 251, 238, 222, - 101, 1, 74, 222, 101, 1, 250, 165, 222, 101, 1, 155, 222, 101, 1, 221, - 217, 222, 101, 1, 231, 242, 222, 101, 1, 231, 93, 222, 101, 1, 214, 70, - 222, 101, 1, 247, 162, 222, 101, 1, 247, 3, 222, 101, 1, 223, 34, 222, - 101, 1, 222, 254, 222, 101, 1, 212, 103, 222, 101, 1, 197, 132, 222, 101, - 1, 197, 120, 222, 101, 1, 237, 193, 222, 101, 1, 237, 182, 222, 101, 1, - 237, 177, 222, 101, 1, 207, 7, 222, 101, 1, 213, 81, 222, 101, 1, 190, - 190, 222, 101, 1, 199, 49, 222, 101, 1, 238, 34, 222, 101, 1, 237, 70, - 222, 101, 1, 181, 222, 101, 1, 168, 222, 101, 1, 209, 230, 222, 101, 1, - 249, 155, 222, 101, 1, 248, 205, 222, 101, 1, 174, 222, 101, 1, 170, 222, - 101, 1, 165, 222, 101, 1, 173, 222, 101, 1, 195, 188, 222, 101, 1, 203, - 166, 222, 101, 1, 201, 176, 222, 101, 1, 188, 222, 101, 1, 140, 222, 101, - 18, 3, 252, 208, 222, 101, 18, 3, 68, 222, 101, 18, 3, 223, 201, 222, - 101, 18, 3, 66, 222, 101, 18, 3, 196, 30, 222, 101, 18, 3, 71, 222, 101, - 18, 3, 251, 238, 222, 101, 18, 3, 74, 222, 101, 18, 3, 250, 165, 222, - 101, 3, 195, 40, 222, 101, 3, 212, 143, 222, 101, 252, 70, 56, 222, 101, - 234, 71, 56, 222, 101, 31, 56, 222, 101, 205, 55, 77, 222, 101, 55, 205, - 55, 77, 222, 101, 237, 240, 222, 101, 55, 237, 240, 222, 101, 18, 3, 117, - 146, 222, 101, 31, 3, 58, 202, 13, 202, 21, 1, 205, 183, 202, 13, 202, - 21, 1, 199, 219, 202, 13, 202, 21, 1, 249, 125, 202, 13, 202, 21, 1, 247, - 151, 202, 13, 202, 21, 1, 238, 14, 202, 13, 202, 21, 1, 231, 227, 202, - 13, 202, 21, 1, 217, 122, 202, 13, 202, 21, 1, 214, 67, 202, 13, 202, 21, - 1, 220, 101, 202, 13, 202, 21, 1, 214, 239, 202, 13, 202, 21, 1, 195, - 184, 202, 13, 202, 21, 1, 210, 202, 202, 13, 202, 21, 1, 192, 121, 202, - 13, 202, 21, 1, 207, 162, 202, 13, 202, 21, 1, 230, 83, 202, 13, 202, 21, - 1, 222, 106, 202, 13, 202, 21, 1, 223, 28, 202, 13, 202, 21, 1, 212, 100, - 202, 13, 202, 21, 1, 251, 247, 202, 13, 202, 21, 1, 234, 188, 202, 13, - 202, 21, 1, 223, 202, 202, 13, 202, 21, 1, 196, 141, 202, 13, 202, 21, 1, - 211, 138, 202, 13, 202, 21, 1, 234, 175, 202, 13, 202, 21, 1, 217, 138, - 202, 13, 202, 21, 17, 191, 77, 202, 13, 202, 21, 17, 107, 202, 13, 202, - 21, 17, 109, 202, 13, 202, 21, 17, 138, 202, 13, 202, 21, 17, 134, 202, - 13, 202, 21, 17, 150, 202, 13, 202, 21, 17, 169, 202, 13, 202, 21, 17, - 175, 202, 13, 202, 21, 17, 171, 202, 13, 202, 21, 17, 178, 246, 253, 3, - 247, 121, 246, 253, 3, 251, 73, 246, 253, 3, 195, 35, 246, 253, 1, 252, - 208, 246, 253, 1, 68, 246, 253, 1, 66, 246, 253, 1, 71, 246, 253, 1, 222, - 129, 246, 253, 1, 221, 216, 246, 253, 1, 231, 239, 246, 253, 1, 231, 92, - 246, 253, 1, 214, 69, 246, 253, 1, 247, 161, 246, 253, 1, 247, 2, 246, - 253, 1, 223, 33, 246, 253, 1, 222, 253, 246, 253, 1, 212, 102, 246, 253, - 1, 197, 131, 246, 253, 1, 197, 119, 246, 253, 1, 237, 192, 246, 253, 1, - 237, 176, 246, 253, 1, 213, 80, 246, 253, 1, 199, 245, 246, 253, 1, 199, - 48, 246, 253, 1, 238, 33, 246, 253, 1, 237, 69, 246, 253, 1, 214, 252, - 246, 253, 1, 210, 222, 246, 253, 1, 209, 229, 246, 253, 1, 249, 153, 246, - 253, 1, 248, 204, 246, 253, 1, 217, 153, 246, 253, 1, 191, 174, 246, 253, - 1, 192, 140, 246, 253, 1, 207, 180, 246, 253, 1, 220, 127, 246, 253, 1, - 193, 181, 246, 253, 1, 205, 198, 246, 253, 1, 230, 93, 246, 253, 18, 3, - 65, 246, 253, 18, 3, 68, 246, 253, 18, 3, 223, 201, 246, 253, 18, 3, 66, - 246, 253, 18, 3, 196, 30, 246, 253, 18, 3, 71, 246, 253, 18, 3, 251, 238, - 246, 253, 18, 3, 74, 246, 253, 18, 3, 250, 165, 246, 253, 18, 3, 211, - 135, 246, 253, 187, 77, 246, 253, 250, 166, 77, 246, 253, 195, 40, 246, - 253, 217, 151, 246, 253, 17, 191, 77, 246, 253, 17, 107, 246, 253, 17, - 109, 246, 253, 17, 138, 246, 253, 17, 134, 246, 253, 17, 150, 246, 253, - 17, 169, 246, 253, 17, 175, 246, 253, 17, 171, 246, 253, 17, 178, 246, - 253, 205, 55, 77, 246, 253, 237, 240, 246, 253, 55, 237, 240, 246, 253, - 208, 15, 77, 246, 253, 1, 219, 160, 246, 253, 18, 3, 252, 208, 246, 253, - 18, 3, 234, 168, 246, 253, 1, 195, 187, 217, 120, 1, 65, 217, 120, 1, 68, - 217, 120, 1, 66, 217, 120, 1, 71, 217, 120, 1, 74, 217, 120, 1, 155, 217, - 120, 1, 221, 217, 217, 120, 1, 231, 242, 217, 120, 1, 231, 93, 217, 120, - 1, 247, 162, 217, 120, 1, 247, 3, 217, 120, 1, 223, 34, 217, 120, 1, 222, - 254, 217, 120, 1, 212, 103, 217, 120, 1, 197, 132, 217, 120, 1, 197, 120, - 217, 120, 1, 237, 193, 217, 120, 1, 237, 177, 217, 120, 1, 213, 81, 217, - 120, 1, 190, 190, 217, 120, 1, 199, 49, 217, 120, 1, 238, 34, 217, 120, - 1, 237, 70, 217, 120, 1, 181, 217, 120, 1, 168, 217, 120, 1, 209, 230, - 217, 120, 1, 249, 155, 217, 120, 1, 248, 205, 217, 120, 1, 174, 217, 120, - 1, 165, 217, 120, 1, 173, 217, 120, 1, 195, 188, 217, 120, 1, 188, 217, - 120, 1, 140, 217, 120, 1, 206, 110, 217, 120, 3, 212, 143, 217, 120, 252, - 70, 56, 217, 120, 205, 55, 77, 217, 120, 34, 203, 16, 203, 130, 3, 247, - 121, 203, 130, 3, 251, 73, 203, 130, 3, 195, 35, 203, 130, 1, 65, 203, - 130, 1, 252, 208, 203, 130, 1, 68, 203, 130, 1, 223, 201, 203, 130, 1, - 66, 203, 130, 1, 196, 30, 203, 130, 1, 117, 146, 203, 130, 1, 117, 206, - 111, 203, 130, 1, 117, 172, 203, 130, 1, 117, 219, 76, 203, 130, 1, 71, - 203, 130, 1, 251, 238, 203, 130, 1, 74, 203, 130, 1, 250, 165, 203, 130, - 1, 155, 203, 130, 1, 221, 217, 203, 130, 1, 231, 242, 203, 130, 1, 231, - 93, 203, 130, 1, 214, 70, 203, 130, 1, 247, 162, 203, 130, 1, 247, 3, - 203, 130, 1, 223, 34, 203, 130, 1, 222, 254, 203, 130, 1, 212, 103, 203, - 130, 1, 197, 132, 203, 130, 1, 197, 120, 203, 130, 1, 237, 193, 203, 130, - 1, 237, 177, 203, 130, 1, 213, 81, 203, 130, 1, 190, 190, 203, 130, 1, - 199, 49, 203, 130, 1, 238, 34, 203, 130, 1, 237, 70, 203, 130, 1, 181, - 203, 130, 1, 168, 203, 130, 1, 209, 230, 203, 130, 1, 249, 155, 203, 130, - 1, 248, 205, 203, 130, 1, 174, 203, 130, 1, 170, 203, 130, 1, 165, 203, - 130, 1, 173, 203, 130, 1, 219, 75, 203, 130, 1, 195, 188, 203, 130, 1, - 203, 166, 203, 130, 1, 201, 176, 203, 130, 1, 188, 203, 130, 1, 140, 203, - 130, 18, 3, 252, 208, 203, 130, 18, 3, 68, 203, 130, 18, 3, 223, 201, - 203, 130, 18, 3, 66, 203, 130, 18, 3, 196, 30, 203, 130, 18, 3, 117, 146, - 203, 130, 18, 3, 117, 206, 111, 203, 130, 18, 3, 117, 172, 203, 130, 18, - 3, 117, 219, 76, 203, 130, 18, 3, 71, 203, 130, 18, 3, 251, 238, 203, - 130, 18, 3, 74, 203, 130, 18, 3, 250, 165, 203, 130, 3, 195, 40, 203, - 130, 3, 250, 147, 203, 130, 3, 222, 239, 203, 130, 3, 196, 75, 203, 130, - 211, 115, 203, 130, 237, 240, 203, 130, 55, 237, 240, 203, 130, 252, 70, - 56, 203, 130, 204, 11, 203, 130, 205, 139, 77, 203, 130, 3, 212, 143, - 203, 130, 18, 52, 77, 203, 130, 233, 203, 203, 41, 18, 77, 203, 130, 200, - 162, 77, 203, 130, 18, 3, 208, 209, 71, 203, 130, 3, 223, 95, 247, 121, - 203, 130, 17, 191, 77, 203, 130, 17, 107, 203, 130, 17, 109, 203, 130, - 17, 138, 203, 130, 17, 134, 203, 130, 17, 150, 203, 130, 17, 169, 203, - 130, 17, 175, 203, 130, 17, 171, 203, 130, 17, 178, 203, 130, 234, 148, - 203, 130, 3, 202, 211, 203, 130, 229, 229, 203, 130, 239, 31, 56, 203, - 130, 205, 55, 217, 57, 203, 130, 205, 55, 217, 56, 166, 251, 16, 17, 107, - 166, 251, 16, 17, 109, 166, 251, 16, 17, 138, 166, 251, 16, 17, 134, 166, - 251, 16, 17, 150, 166, 251, 16, 17, 169, 166, 251, 16, 17, 175, 166, 251, - 16, 17, 171, 166, 251, 16, 17, 178, 166, 251, 16, 31, 199, 95, 166, 251, - 16, 31, 197, 32, 166, 251, 16, 31, 198, 249, 166, 251, 16, 31, 232, 137, - 166, 251, 16, 31, 233, 17, 166, 251, 16, 31, 202, 121, 166, 251, 16, 31, - 203, 242, 166, 251, 16, 31, 234, 155, 166, 251, 16, 31, 213, 171, 166, - 251, 16, 31, 91, 228, 142, 166, 251, 16, 31, 91, 189, 221, 184, 1, 65, - 221, 184, 1, 252, 208, 221, 184, 1, 68, 221, 184, 1, 66, 221, 184, 1, 71, - 221, 184, 1, 251, 238, 221, 184, 1, 74, 221, 184, 1, 250, 165, 221, 184, - 1, 155, 221, 184, 1, 221, 217, 221, 184, 1, 231, 242, 221, 184, 1, 231, - 129, 221, 184, 1, 231, 93, 221, 184, 1, 214, 70, 221, 184, 1, 247, 162, - 221, 184, 1, 247, 3, 221, 184, 1, 223, 34, 221, 184, 1, 222, 232, 221, - 184, 1, 212, 103, 221, 184, 1, 197, 132, 221, 184, 1, 197, 120, 221, 184, - 1, 237, 193, 221, 184, 1, 237, 177, 221, 184, 1, 213, 81, 221, 184, 1, - 190, 190, 221, 184, 1, 199, 49, 221, 184, 1, 238, 34, 221, 184, 1, 237, - 183, 221, 184, 1, 237, 70, 221, 184, 1, 181, 221, 184, 1, 168, 221, 184, - 1, 209, 230, 221, 184, 1, 249, 155, 221, 184, 1, 249, 55, 221, 184, 1, - 248, 205, 221, 184, 1, 174, 221, 184, 1, 170, 221, 184, 1, 165, 221, 184, - 1, 173, 221, 184, 1, 195, 188, 221, 184, 1, 188, 221, 184, 1, 140, 221, - 184, 1, 219, 75, 221, 184, 18, 3, 252, 208, 221, 184, 18, 3, 68, 221, - 184, 18, 3, 223, 201, 221, 184, 18, 3, 66, 221, 184, 18, 3, 71, 221, 184, - 18, 3, 251, 238, 221, 184, 18, 3, 74, 221, 184, 18, 3, 250, 165, 221, - 184, 3, 251, 73, 221, 184, 3, 195, 40, 221, 184, 3, 212, 143, 221, 184, - 3, 203, 156, 221, 184, 237, 240, 221, 184, 55, 237, 240, 221, 184, 193, - 23, 204, 11, 221, 184, 205, 55, 77, 221, 184, 55, 205, 55, 77, 221, 184, - 252, 70, 56, 221, 184, 3, 200, 206, 221, 184, 1, 208, 98, 221, 184, 1, - 203, 41, 68, 221, 184, 18, 3, 117, 146, 215, 135, 1, 65, 215, 135, 1, 68, - 215, 135, 1, 66, 215, 135, 1, 71, 215, 135, 1, 155, 215, 135, 1, 221, - 217, 215, 135, 1, 231, 242, 215, 135, 1, 231, 93, 215, 135, 1, 247, 162, - 215, 135, 1, 247, 3, 215, 135, 1, 223, 34, 215, 135, 1, 222, 232, 215, - 135, 1, 212, 103, 215, 135, 1, 197, 132, 215, 135, 1, 197, 120, 215, 135, - 1, 237, 193, 215, 135, 1, 237, 183, 215, 135, 1, 237, 177, 215, 135, 1, - 213, 81, 215, 135, 1, 190, 190, 215, 135, 1, 199, 49, 215, 135, 1, 238, - 34, 215, 135, 1, 237, 70, 215, 135, 1, 181, 215, 135, 1, 168, 215, 135, - 1, 209, 230, 215, 135, 1, 249, 155, 215, 135, 1, 248, 205, 215, 135, 1, - 174, 215, 135, 1, 170, 215, 135, 1, 165, 215, 135, 1, 173, 215, 135, 1, - 195, 188, 215, 135, 1, 188, 215, 135, 1, 140, 215, 135, 1, 206, 110, 215, - 135, 1, 207, 7, 215, 135, 205, 55, 77, 221, 174, 1, 65, 221, 174, 1, 252, - 208, 221, 174, 1, 68, 221, 174, 1, 223, 201, 221, 174, 1, 66, 221, 174, - 1, 196, 30, 221, 174, 1, 71, 221, 174, 1, 251, 238, 221, 174, 1, 74, 221, - 174, 1, 250, 165, 221, 174, 1, 155, 221, 174, 1, 221, 217, 221, 174, 1, - 231, 242, 221, 174, 1, 231, 129, 221, 174, 1, 231, 93, 221, 174, 1, 214, - 70, 221, 174, 1, 247, 162, 221, 174, 1, 247, 3, 221, 174, 1, 223, 34, - 221, 174, 1, 222, 232, 221, 174, 1, 222, 254, 221, 174, 1, 212, 103, 221, - 174, 1, 197, 132, 221, 174, 1, 197, 120, 221, 174, 1, 237, 193, 221, 174, - 1, 237, 183, 221, 174, 1, 206, 110, 221, 174, 1, 237, 177, 221, 174, 1, - 213, 81, 221, 174, 1, 190, 190, 221, 174, 1, 199, 49, 221, 174, 1, 238, - 34, 221, 174, 1, 237, 70, 221, 174, 1, 181, 221, 174, 1, 168, 221, 174, - 1, 209, 230, 221, 174, 1, 249, 155, 221, 174, 1, 249, 55, 221, 174, 1, - 248, 205, 221, 174, 1, 174, 221, 174, 1, 170, 221, 174, 1, 165, 221, 174, - 1, 173, 221, 174, 1, 195, 188, 221, 174, 1, 203, 166, 221, 174, 1, 188, - 221, 174, 1, 140, 221, 174, 3, 251, 73, 221, 174, 18, 3, 252, 208, 221, - 174, 18, 3, 68, 221, 174, 18, 3, 223, 201, 221, 174, 18, 3, 66, 221, 174, - 18, 3, 196, 30, 221, 174, 18, 3, 71, 221, 174, 18, 3, 251, 238, 221, 174, - 18, 3, 74, 221, 174, 18, 3, 250, 165, 221, 174, 3, 212, 143, 221, 174, 3, - 195, 40, 221, 174, 17, 191, 77, 221, 174, 17, 107, 221, 174, 17, 109, - 221, 174, 17, 138, 221, 174, 17, 134, 221, 174, 17, 150, 221, 174, 17, - 169, 221, 174, 17, 175, 221, 174, 17, 171, 221, 174, 17, 178, 230, 221, - 3, 33, 251, 74, 58, 230, 221, 3, 247, 121, 230, 221, 3, 251, 73, 230, - 221, 3, 195, 35, 230, 221, 1, 65, 230, 221, 1, 252, 208, 230, 221, 1, 68, - 230, 221, 1, 223, 201, 230, 221, 1, 66, 230, 221, 1, 196, 30, 230, 221, - 1, 117, 146, 230, 221, 1, 117, 172, 230, 221, 1, 234, 190, 230, 221, 1, - 251, 238, 230, 221, 1, 211, 89, 230, 221, 1, 250, 165, 230, 221, 1, 155, - 230, 221, 1, 221, 217, 230, 221, 1, 231, 242, 230, 221, 1, 231, 93, 230, - 221, 1, 214, 70, 230, 221, 1, 247, 162, 230, 221, 1, 247, 3, 230, 221, 1, - 223, 34, 230, 221, 1, 222, 254, 230, 221, 1, 212, 103, 230, 221, 1, 197, - 132, 230, 221, 1, 197, 120, 230, 221, 1, 237, 193, 230, 221, 1, 237, 177, - 230, 221, 1, 213, 81, 230, 221, 1, 190, 190, 230, 221, 1, 199, 49, 230, - 221, 1, 238, 34, 230, 221, 1, 237, 70, 230, 221, 1, 181, 230, 221, 1, - 168, 230, 221, 1, 209, 230, 230, 221, 1, 249, 155, 230, 221, 1, 248, 205, - 230, 221, 1, 174, 230, 221, 1, 170, 230, 221, 1, 165, 230, 221, 1, 173, - 230, 221, 1, 219, 75, 230, 221, 1, 195, 188, 230, 221, 1, 203, 166, 230, - 221, 1, 201, 176, 230, 221, 1, 188, 230, 221, 1, 140, 33, 248, 167, 60, - 230, 221, 3, 212, 143, 230, 221, 3, 250, 147, 230, 221, 18, 3, 252, 208, - 230, 221, 18, 3, 68, 230, 221, 18, 3, 223, 201, 230, 221, 18, 3, 66, 230, - 221, 18, 3, 196, 30, 230, 221, 18, 3, 117, 146, 230, 221, 18, 3, 117, - 206, 111, 230, 221, 18, 3, 234, 190, 230, 221, 18, 3, 251, 238, 230, 221, - 18, 3, 211, 89, 230, 221, 18, 3, 250, 165, 230, 221, 3, 195, 40, 230, - 221, 211, 115, 230, 221, 250, 166, 219, 200, 77, 230, 221, 3, 209, 81, - 230, 221, 1, 195, 150, 251, 73, 230, 221, 1, 195, 150, 55, 251, 73, 230, - 221, 1, 117, 206, 111, 230, 221, 1, 117, 219, 76, 230, 221, 18, 3, 117, - 172, 230, 221, 18, 3, 117, 219, 76, 33, 230, 221, 17, 191, 77, 33, 230, - 221, 17, 107, 33, 230, 221, 17, 109, 33, 230, 221, 17, 138, 33, 230, 221, - 17, 134, 33, 230, 221, 17, 150, 33, 230, 221, 17, 169, 33, 230, 221, 1, - 65, 33, 230, 221, 1, 155, 33, 230, 221, 1, 181, 33, 230, 221, 1, 195, 69, - 33, 230, 221, 1, 168, 214, 80, 1, 65, 214, 80, 1, 252, 208, 214, 80, 1, - 68, 214, 80, 1, 223, 201, 214, 80, 1, 66, 214, 80, 1, 196, 30, 214, 80, - 1, 117, 146, 214, 80, 1, 117, 206, 111, 214, 80, 1, 117, 172, 214, 80, 1, - 117, 219, 76, 214, 80, 1, 71, 214, 80, 1, 251, 238, 214, 80, 1, 74, 214, - 80, 1, 250, 165, 214, 80, 1, 155, 214, 80, 1, 221, 217, 214, 80, 1, 231, - 242, 214, 80, 1, 231, 93, 214, 80, 1, 214, 70, 214, 80, 1, 214, 19, 214, - 80, 1, 247, 162, 214, 80, 1, 247, 3, 214, 80, 1, 223, 34, 214, 80, 1, - 222, 254, 214, 80, 1, 212, 103, 214, 80, 1, 212, 85, 214, 80, 1, 197, - 132, 214, 80, 1, 197, 120, 214, 80, 1, 237, 193, 214, 80, 1, 237, 177, - 214, 80, 1, 213, 81, 214, 80, 1, 190, 190, 214, 80, 1, 199, 49, 214, 80, - 1, 238, 34, 214, 80, 1, 237, 70, 214, 80, 1, 181, 214, 80, 1, 213, 226, - 214, 80, 1, 168, 214, 80, 1, 209, 230, 214, 80, 1, 249, 155, 214, 80, 1, - 248, 205, 214, 80, 1, 174, 214, 80, 1, 216, 105, 214, 80, 1, 170, 214, - 80, 1, 165, 214, 80, 1, 207, 7, 214, 80, 1, 173, 214, 80, 1, 219, 161, - 214, 80, 1, 193, 190, 214, 80, 1, 203, 166, 214, 80, 1, 201, 176, 214, - 80, 1, 188, 214, 80, 1, 140, 214, 80, 18, 3, 252, 208, 214, 80, 18, 3, - 68, 214, 80, 18, 3, 223, 201, 214, 80, 18, 3, 66, 214, 80, 18, 3, 196, - 30, 214, 80, 18, 3, 117, 146, 214, 80, 18, 3, 117, 206, 111, 214, 80, 18, - 3, 117, 172, 214, 80, 18, 3, 117, 219, 76, 214, 80, 18, 3, 71, 214, 80, - 18, 3, 251, 238, 214, 80, 18, 3, 74, 214, 80, 18, 3, 250, 165, 214, 80, - 3, 195, 40, 214, 80, 3, 247, 121, 214, 80, 3, 251, 73, 214, 80, 3, 195, - 35, 214, 80, 3, 212, 143, 214, 80, 3, 250, 147, 214, 80, 3, 53, 251, 73, - 214, 80, 211, 115, 214, 80, 202, 210, 214, 80, 237, 240, 214, 80, 55, - 237, 240, 214, 80, 242, 76, 214, 80, 231, 206, 233, 5, 214, 80, 252, 70, - 56, 214, 80, 17, 191, 77, 214, 80, 17, 107, 214, 80, 17, 109, 214, 80, - 17, 138, 214, 80, 17, 134, 214, 80, 17, 150, 214, 80, 17, 169, 214, 80, - 17, 175, 214, 80, 17, 171, 214, 80, 17, 178, 214, 80, 55, 242, 76, 214, - 80, 209, 109, 77, 214, 80, 223, 121, 56, 214, 80, 205, 139, 77, 214, 80, - 1, 195, 150, 251, 73, 214, 80, 3, 222, 239, 214, 80, 3, 196, 75, 198, - 129, 251, 102, 198, 129, 1, 65, 198, 129, 1, 252, 208, 198, 129, 1, 68, - 198, 129, 1, 223, 201, 198, 129, 1, 66, 198, 129, 1, 196, 30, 198, 129, - 1, 117, 146, 198, 129, 1, 117, 206, 111, 198, 129, 1, 117, 172, 198, 129, - 1, 117, 219, 76, 198, 129, 1, 71, 198, 129, 1, 251, 238, 198, 129, 1, 74, - 198, 129, 1, 250, 165, 198, 129, 1, 155, 198, 129, 1, 221, 217, 198, 129, - 1, 231, 242, 198, 129, 1, 231, 93, 198, 129, 1, 214, 70, 198, 129, 1, - 247, 162, 198, 129, 1, 247, 3, 198, 129, 1, 223, 34, 198, 129, 1, 222, - 254, 198, 129, 1, 212, 103, 198, 129, 1, 197, 132, 198, 129, 1, 197, 120, - 198, 129, 1, 237, 193, 198, 129, 1, 237, 177, 198, 129, 1, 213, 81, 198, - 129, 1, 190, 190, 198, 129, 1, 199, 49, 198, 129, 1, 238, 34, 198, 129, - 1, 237, 70, 198, 129, 1, 181, 198, 129, 1, 168, 198, 129, 1, 209, 230, - 198, 129, 1, 249, 155, 198, 129, 1, 248, 205, 198, 129, 1, 174, 198, 129, - 1, 170, 198, 129, 1, 165, 198, 129, 1, 173, 198, 129, 1, 195, 188, 198, - 129, 1, 203, 166, 198, 129, 1, 201, 176, 198, 129, 1, 188, 198, 129, 1, - 140, 198, 129, 18, 3, 252, 208, 198, 129, 18, 3, 68, 198, 129, 18, 3, - 223, 201, 198, 129, 18, 3, 66, 198, 129, 18, 3, 196, 30, 198, 129, 18, 3, - 117, 146, 198, 129, 18, 3, 117, 206, 111, 198, 129, 18, 3, 117, 172, 198, - 129, 18, 3, 117, 219, 76, 198, 129, 18, 3, 71, 198, 129, 18, 3, 203, 41, - 71, 198, 129, 18, 3, 251, 238, 198, 129, 18, 3, 74, 198, 129, 18, 3, 203, - 41, 74, 198, 129, 18, 3, 250, 165, 198, 129, 3, 247, 121, 198, 129, 3, - 251, 73, 198, 129, 3, 195, 35, 198, 129, 3, 195, 40, 198, 129, 3, 212, - 143, 198, 129, 3, 250, 147, 198, 129, 230, 139, 198, 129, 252, 70, 56, - 198, 129, 211, 115, 198, 129, 17, 191, 77, 198, 129, 17, 107, 198, 129, - 17, 109, 198, 129, 17, 138, 198, 129, 17, 134, 198, 129, 17, 150, 198, - 129, 17, 169, 198, 129, 17, 175, 198, 129, 17, 171, 198, 129, 17, 178, - 202, 212, 1, 65, 202, 212, 1, 252, 208, 202, 212, 1, 68, 202, 212, 1, - 223, 201, 202, 212, 1, 66, 202, 212, 1, 196, 30, 202, 212, 1, 117, 146, - 202, 212, 1, 117, 206, 111, 202, 212, 1, 117, 172, 202, 212, 1, 117, 219, - 76, 202, 212, 1, 71, 202, 212, 1, 251, 238, 202, 212, 1, 74, 202, 212, 1, - 250, 165, 202, 212, 1, 155, 202, 212, 1, 221, 217, 202, 212, 1, 231, 242, - 202, 212, 1, 231, 93, 202, 212, 1, 214, 70, 202, 212, 1, 247, 162, 202, - 212, 1, 247, 3, 202, 212, 1, 223, 34, 202, 212, 1, 222, 254, 202, 212, 1, - 212, 103, 202, 212, 1, 197, 132, 202, 212, 1, 197, 120, 202, 212, 1, 237, - 193, 202, 212, 1, 237, 177, 202, 212, 1, 213, 81, 202, 212, 1, 190, 190, - 202, 212, 1, 199, 49, 202, 212, 1, 238, 34, 202, 212, 1, 237, 70, 202, - 212, 1, 181, 202, 212, 1, 168, 202, 212, 1, 209, 230, 202, 212, 1, 249, - 155, 202, 212, 1, 248, 205, 202, 212, 1, 174, 202, 212, 1, 170, 202, 212, - 1, 165, 202, 212, 1, 173, 202, 212, 1, 195, 188, 202, 212, 1, 203, 166, - 202, 212, 1, 201, 176, 202, 212, 1, 188, 202, 212, 1, 140, 202, 212, 18, - 3, 252, 208, 202, 212, 18, 3, 68, 202, 212, 18, 3, 223, 201, 202, 212, - 18, 3, 66, 202, 212, 18, 3, 196, 30, 202, 212, 18, 3, 117, 146, 202, 212, - 18, 3, 117, 206, 111, 202, 212, 18, 3, 71, 202, 212, 18, 3, 251, 238, - 202, 212, 18, 3, 74, 202, 212, 18, 3, 250, 165, 202, 212, 3, 247, 121, - 202, 212, 3, 251, 73, 202, 212, 3, 195, 35, 202, 212, 3, 195, 40, 202, - 212, 3, 212, 143, 202, 212, 3, 202, 211, 202, 212, 237, 240, 202, 212, - 55, 237, 240, 202, 212, 204, 12, 236, 142, 202, 212, 204, 12, 164, 202, - 212, 207, 47, 217, 57, 202, 212, 207, 47, 217, 56, 202, 212, 207, 47, - 217, 55, 202, 212, 234, 97, 79, 199, 54, 77, 202, 212, 205, 55, 87, 4, - 197, 236, 23, 196, 221, 211, 43, 202, 212, 205, 55, 87, 4, 197, 236, 23, - 235, 140, 238, 228, 202, 212, 205, 55, 87, 4, 207, 122, 23, 235, 140, - 238, 228, 202, 212, 205, 55, 87, 4, 207, 122, 23, 235, 140, 55, 238, 228, - 202, 212, 205, 55, 87, 4, 207, 122, 23, 235, 140, 197, 225, 238, 228, - 202, 212, 205, 55, 87, 55, 206, 189, 202, 212, 205, 55, 87, 55, 206, 190, - 4, 207, 121, 202, 212, 205, 55, 87, 4, 55, 238, 228, 202, 212, 205, 55, - 87, 4, 197, 225, 238, 228, 202, 212, 205, 55, 87, 4, 208, 28, 238, 228, - 202, 212, 205, 55, 87, 4, 204, 9, 238, 228, 202, 212, 205, 55, 87, 4, - 243, 10, 23, 207, 121, 202, 212, 205, 55, 87, 4, 243, 10, 23, 105, 234, - 99, 202, 212, 205, 55, 87, 4, 243, 10, 23, 232, 130, 234, 99, 202, 212, - 1, 198, 226, 251, 159, 68, 202, 212, 1, 197, 15, 251, 159, 68, 202, 212, - 1, 197, 15, 251, 159, 223, 201, 202, 212, 1, 251, 159, 66, 202, 212, 18, - 3, 251, 159, 66, 202, 212, 18, 3, 251, 159, 196, 30, 215, 255, 1, 65, - 215, 255, 1, 252, 208, 215, 255, 1, 68, 215, 255, 1, 223, 201, 215, 255, - 1, 66, 215, 255, 1, 196, 30, 215, 255, 1, 117, 146, 215, 255, 1, 117, - 206, 111, 215, 255, 1, 117, 172, 215, 255, 1, 117, 219, 76, 215, 255, 1, - 71, 215, 255, 1, 251, 238, 215, 255, 1, 74, 215, 255, 1, 250, 165, 215, - 255, 1, 155, 215, 255, 1, 221, 217, 215, 255, 1, 231, 242, 215, 255, 1, - 231, 93, 215, 255, 1, 214, 70, 215, 255, 1, 247, 162, 215, 255, 1, 247, - 3, 215, 255, 1, 223, 34, 215, 255, 1, 222, 254, 215, 255, 1, 212, 103, - 215, 255, 1, 197, 132, 215, 255, 1, 197, 120, 215, 255, 1, 237, 193, 215, - 255, 1, 237, 177, 215, 255, 1, 213, 81, 215, 255, 1, 190, 190, 215, 255, - 1, 199, 49, 215, 255, 1, 238, 34, 215, 255, 1, 237, 70, 215, 255, 1, 181, - 215, 255, 1, 168, 215, 255, 1, 209, 230, 215, 255, 1, 249, 155, 215, 255, - 1, 248, 205, 215, 255, 1, 174, 215, 255, 1, 170, 215, 255, 1, 165, 215, - 255, 1, 173, 215, 255, 1, 195, 188, 215, 255, 1, 203, 166, 215, 255, 1, - 201, 176, 215, 255, 1, 188, 215, 255, 1, 140, 215, 255, 1, 219, 75, 215, - 255, 18, 3, 252, 208, 215, 255, 18, 3, 68, 215, 255, 18, 3, 223, 201, - 215, 255, 18, 3, 66, 215, 255, 18, 3, 196, 30, 215, 255, 18, 3, 117, 146, - 215, 255, 18, 3, 117, 206, 111, 215, 255, 18, 3, 117, 172, 215, 255, 18, - 3, 117, 219, 76, 215, 255, 18, 3, 71, 215, 255, 18, 3, 251, 238, 215, - 255, 18, 3, 74, 215, 255, 18, 3, 250, 165, 215, 255, 3, 251, 73, 215, - 255, 3, 195, 35, 215, 255, 3, 195, 40, 215, 255, 3, 251, 13, 215, 255, - 237, 240, 215, 255, 55, 237, 240, 215, 255, 252, 70, 56, 215, 255, 3, - 228, 129, 215, 255, 17, 191, 77, 215, 255, 17, 107, 215, 255, 17, 109, - 215, 255, 17, 138, 215, 255, 17, 134, 215, 255, 17, 150, 215, 255, 17, - 169, 215, 255, 17, 175, 215, 255, 17, 171, 215, 255, 17, 178, 104, 248, - 161, 4, 211, 44, 104, 206, 123, 248, 160, 104, 55, 248, 161, 4, 211, 44, - 104, 197, 225, 248, 161, 4, 211, 44, 104, 248, 161, 4, 55, 211, 44, 104, - 206, 123, 248, 161, 4, 211, 44, 104, 206, 123, 248, 161, 4, 55, 211, 44, - 104, 223, 95, 248, 160, 104, 223, 95, 248, 161, 4, 55, 211, 44, 104, 200, - 134, 248, 160, 104, 200, 134, 248, 161, 4, 211, 44, 104, 200, 134, 248, - 161, 4, 55, 211, 44, 104, 154, 200, 134, 248, 161, 4, 55, 211, 44, 199, - 205, 1, 65, 199, 205, 1, 252, 208, 199, 205, 1, 68, 199, 205, 1, 223, - 201, 199, 205, 1, 66, 199, 205, 1, 196, 30, 199, 205, 1, 71, 199, 205, 1, - 251, 238, 199, 205, 1, 74, 199, 205, 1, 250, 165, 199, 205, 1, 155, 199, - 205, 1, 221, 217, 199, 205, 1, 231, 242, 199, 205, 1, 231, 93, 199, 205, - 1, 214, 70, 199, 205, 1, 247, 162, 199, 205, 1, 247, 3, 199, 205, 1, 223, - 34, 199, 205, 1, 222, 254, 199, 205, 1, 212, 103, 199, 205, 1, 197, 132, - 199, 205, 1, 197, 120, 199, 205, 1, 237, 193, 199, 205, 1, 237, 177, 199, - 205, 1, 213, 81, 199, 205, 1, 190, 190, 199, 205, 1, 199, 49, 199, 205, - 1, 238, 34, 199, 205, 1, 237, 70, 199, 205, 1, 181, 199, 205, 1, 168, - 199, 205, 1, 209, 230, 199, 205, 1, 249, 155, 199, 205, 1, 248, 205, 199, - 205, 1, 174, 199, 205, 1, 170, 199, 205, 1, 165, 199, 205, 1, 173, 199, - 205, 1, 195, 188, 199, 205, 1, 203, 166, 199, 205, 1, 188, 199, 205, 1, - 140, 199, 205, 1, 206, 110, 199, 205, 3, 251, 73, 199, 205, 3, 195, 35, - 199, 205, 18, 3, 252, 208, 199, 205, 18, 3, 68, 199, 205, 18, 3, 223, - 201, 199, 205, 18, 3, 66, 199, 205, 18, 3, 196, 30, 199, 205, 18, 3, 71, - 199, 205, 18, 3, 251, 238, 199, 205, 18, 3, 74, 199, 205, 18, 3, 250, - 165, 199, 205, 3, 195, 40, 199, 205, 3, 212, 143, 199, 205, 1, 251, 16, - 221, 217, 199, 205, 252, 70, 56, 199, 205, 17, 191, 77, 199, 205, 17, - 107, 199, 205, 17, 109, 199, 205, 17, 138, 199, 205, 17, 134, 199, 205, - 17, 150, 199, 205, 17, 169, 199, 205, 17, 175, 199, 205, 17, 171, 199, - 205, 17, 178, 251, 242, 1, 155, 251, 242, 1, 221, 217, 251, 242, 1, 214, - 70, 251, 242, 1, 181, 251, 242, 1, 190, 190, 251, 242, 1, 251, 159, 190, - 190, 251, 242, 1, 168, 251, 242, 1, 209, 230, 251, 242, 1, 249, 155, 251, - 242, 1, 174, 251, 242, 1, 223, 34, 251, 242, 1, 247, 3, 251, 242, 1, 199, - 49, 251, 242, 1, 165, 251, 242, 1, 173, 251, 242, 1, 188, 251, 242, 1, - 212, 103, 251, 242, 1, 140, 251, 242, 1, 65, 251, 242, 1, 238, 34, 251, - 242, 1, 237, 70, 251, 242, 1, 231, 242, 251, 242, 1, 251, 159, 231, 242, - 251, 242, 1, 231, 93, 251, 242, 1, 248, 205, 251, 242, 1, 222, 254, 251, - 242, 1, 251, 159, 249, 155, 251, 242, 120, 3, 216, 219, 173, 251, 242, - 120, 3, 216, 219, 165, 251, 242, 120, 3, 216, 219, 219, 135, 165, 251, - 242, 18, 3, 65, 251, 242, 18, 3, 252, 208, 251, 242, 18, 3, 68, 251, 242, - 18, 3, 223, 201, 251, 242, 18, 3, 66, 251, 242, 18, 3, 196, 30, 251, 242, - 18, 3, 71, 251, 242, 18, 3, 250, 142, 251, 242, 18, 3, 74, 251, 242, 18, - 3, 251, 238, 251, 242, 18, 3, 251, 151, 251, 242, 3, 221, 145, 251, 242, - 17, 191, 77, 251, 242, 17, 107, 251, 242, 17, 109, 251, 242, 17, 138, - 251, 242, 17, 134, 251, 242, 17, 150, 251, 242, 17, 169, 251, 242, 17, - 175, 251, 242, 17, 171, 251, 242, 17, 178, 251, 242, 31, 199, 95, 251, - 242, 31, 197, 32, 251, 242, 3, 2, 205, 54, 251, 242, 3, 205, 54, 251, - 242, 3, 206, 54, 251, 242, 16, 195, 69, 251, 242, 1, 247, 162, 251, 242, - 1, 197, 132, 251, 242, 1, 197, 120, 251, 242, 1, 237, 193, 251, 242, 1, - 237, 177, 251, 242, 1, 213, 81, 251, 242, 1, 219, 75, 236, 163, 1, 65, - 236, 163, 1, 252, 208, 236, 163, 1, 68, 236, 163, 1, 223, 201, 236, 163, - 1, 66, 236, 163, 1, 196, 30, 236, 163, 1, 71, 236, 163, 1, 251, 238, 236, - 163, 1, 74, 236, 163, 1, 250, 165, 236, 163, 1, 155, 236, 163, 1, 221, - 217, 236, 163, 1, 231, 242, 236, 163, 1, 231, 93, 236, 163, 1, 214, 70, - 236, 163, 1, 247, 162, 236, 163, 1, 247, 3, 236, 163, 1, 223, 34, 236, - 163, 1, 222, 254, 236, 163, 1, 212, 103, 236, 163, 1, 197, 132, 236, 163, - 1, 197, 120, 236, 163, 1, 237, 193, 236, 163, 1, 237, 177, 236, 163, 1, - 213, 81, 236, 163, 1, 190, 190, 236, 163, 1, 199, 49, 236, 163, 1, 238, - 34, 236, 163, 1, 237, 70, 236, 163, 1, 181, 236, 163, 1, 168, 236, 163, - 1, 209, 230, 236, 163, 1, 249, 155, 236, 163, 1, 248, 205, 236, 163, 1, - 174, 236, 163, 1, 170, 236, 163, 1, 165, 236, 163, 1, 173, 236, 163, 1, - 195, 188, 236, 163, 1, 203, 166, 236, 163, 1, 201, 176, 236, 163, 1, 188, - 236, 163, 1, 140, 236, 163, 1, 206, 110, 236, 163, 18, 3, 252, 208, 236, - 163, 18, 3, 68, 236, 163, 18, 3, 223, 201, 236, 163, 18, 3, 66, 236, 163, - 18, 3, 196, 30, 236, 163, 18, 3, 117, 146, 236, 163, 18, 3, 117, 206, - 111, 236, 163, 18, 3, 71, 236, 163, 18, 3, 251, 238, 236, 163, 18, 3, 74, - 236, 163, 18, 3, 250, 165, 236, 163, 3, 251, 73, 236, 163, 3, 195, 35, - 236, 163, 3, 195, 40, 236, 163, 3, 212, 143, 236, 163, 252, 70, 56, 193, - 156, 242, 255, 6, 1, 214, 69, 193, 156, 242, 255, 6, 1, 65, 193, 156, - 242, 255, 6, 1, 193, 86, 193, 156, 242, 255, 6, 1, 191, 225, 193, 156, - 242, 255, 6, 1, 170, 193, 156, 242, 255, 6, 1, 192, 12, 193, 156, 242, - 255, 6, 1, 223, 201, 193, 156, 242, 255, 6, 1, 196, 30, 193, 156, 242, - 255, 6, 1, 71, 193, 156, 242, 255, 6, 1, 74, 193, 156, 242, 255, 6, 1, - 251, 124, 193, 156, 242, 255, 6, 1, 231, 242, 193, 156, 242, 255, 6, 1, - 221, 69, 193, 156, 242, 255, 6, 1, 234, 68, 193, 156, 242, 255, 6, 1, - 191, 204, 193, 156, 242, 255, 6, 1, 196, 160, 193, 156, 242, 255, 6, 1, - 234, 87, 193, 156, 242, 255, 6, 1, 211, 156, 193, 156, 242, 255, 6, 1, - 197, 127, 193, 156, 242, 255, 6, 1, 212, 129, 193, 156, 242, 255, 6, 1, - 238, 34, 193, 156, 242, 255, 6, 1, 250, 184, 193, 156, 242, 255, 6, 1, - 251, 151, 193, 156, 242, 255, 6, 1, 248, 12, 193, 156, 242, 255, 6, 1, - 208, 167, 193, 156, 242, 255, 6, 1, 229, 117, 193, 156, 242, 255, 6, 1, - 229, 5, 193, 156, 242, 255, 6, 1, 228, 187, 193, 156, 242, 255, 6, 1, - 230, 21, 193, 156, 242, 255, 6, 1, 201, 127, 193, 156, 242, 255, 6, 1, - 202, 194, 193, 156, 242, 255, 6, 1, 195, 25, 193, 156, 242, 255, 2, 1, - 214, 69, 193, 156, 242, 255, 2, 1, 65, 193, 156, 242, 255, 2, 1, 193, 86, - 193, 156, 242, 255, 2, 1, 191, 225, 193, 156, 242, 255, 2, 1, 170, 193, - 156, 242, 255, 2, 1, 192, 12, 193, 156, 242, 255, 2, 1, 223, 201, 193, - 156, 242, 255, 2, 1, 196, 30, 193, 156, 242, 255, 2, 1, 71, 193, 156, - 242, 255, 2, 1, 74, 193, 156, 242, 255, 2, 1, 251, 124, 193, 156, 242, - 255, 2, 1, 231, 242, 193, 156, 242, 255, 2, 1, 221, 69, 193, 156, 242, - 255, 2, 1, 234, 68, 193, 156, 242, 255, 2, 1, 191, 204, 193, 156, 242, - 255, 2, 1, 196, 160, 193, 156, 242, 255, 2, 1, 234, 87, 193, 156, 242, - 255, 2, 1, 211, 156, 193, 156, 242, 255, 2, 1, 197, 127, 193, 156, 242, - 255, 2, 1, 212, 129, 193, 156, 242, 255, 2, 1, 238, 34, 193, 156, 242, - 255, 2, 1, 250, 184, 193, 156, 242, 255, 2, 1, 251, 151, 193, 156, 242, - 255, 2, 1, 248, 12, 193, 156, 242, 255, 2, 1, 208, 167, 193, 156, 242, - 255, 2, 1, 229, 117, 193, 156, 242, 255, 2, 1, 229, 5, 193, 156, 242, - 255, 2, 1, 228, 187, 193, 156, 242, 255, 2, 1, 230, 21, 193, 156, 242, - 255, 2, 1, 201, 127, 193, 156, 242, 255, 2, 1, 202, 194, 193, 156, 242, - 255, 2, 1, 195, 25, 193, 156, 242, 255, 17, 191, 77, 193, 156, 242, 255, - 17, 107, 193, 156, 242, 255, 17, 109, 193, 156, 242, 255, 17, 138, 193, - 156, 242, 255, 17, 134, 193, 156, 242, 255, 17, 150, 193, 156, 242, 255, - 17, 169, 193, 156, 242, 255, 17, 175, 193, 156, 242, 255, 17, 171, 193, - 156, 242, 255, 17, 178, 193, 156, 242, 255, 31, 199, 95, 193, 156, 242, - 255, 31, 197, 32, 193, 156, 242, 255, 31, 198, 249, 193, 156, 242, 255, - 31, 232, 137, 193, 156, 242, 255, 31, 233, 17, 193, 156, 242, 255, 31, - 202, 121, 193, 156, 242, 255, 31, 203, 242, 193, 156, 242, 255, 31, 234, - 155, 193, 156, 242, 255, 31, 213, 171, 193, 156, 242, 255, 211, 115, 236, - 211, 251, 211, 1, 65, 236, 211, 251, 211, 1, 252, 208, 236, 211, 251, - 211, 1, 68, 236, 211, 251, 211, 1, 223, 201, 236, 211, 251, 211, 1, 66, - 236, 211, 251, 211, 1, 196, 30, 236, 211, 251, 211, 1, 71, 236, 211, 251, - 211, 1, 74, 236, 211, 251, 211, 1, 155, 236, 211, 251, 211, 1, 221, 217, - 236, 211, 251, 211, 1, 231, 242, 236, 211, 251, 211, 1, 231, 93, 236, - 211, 251, 211, 1, 214, 70, 236, 211, 251, 211, 1, 247, 162, 236, 211, - 251, 211, 1, 247, 3, 236, 211, 251, 211, 1, 223, 34, 236, 211, 251, 211, - 1, 212, 103, 236, 211, 251, 211, 1, 197, 132, 236, 211, 251, 211, 1, 237, - 193, 236, 211, 251, 211, 1, 237, 177, 236, 211, 251, 211, 1, 213, 81, - 236, 211, 251, 211, 1, 190, 190, 236, 211, 251, 211, 1, 199, 49, 236, - 211, 251, 211, 1, 238, 34, 236, 211, 251, 211, 1, 237, 70, 236, 211, 251, - 211, 1, 181, 236, 211, 251, 211, 1, 168, 236, 211, 251, 211, 1, 209, 230, - 236, 211, 251, 211, 1, 249, 155, 236, 211, 251, 211, 1, 248, 205, 236, - 211, 251, 211, 1, 174, 236, 211, 251, 211, 1, 170, 236, 211, 251, 211, 1, - 191, 175, 236, 211, 251, 211, 1, 165, 236, 211, 251, 211, 1, 173, 236, - 211, 251, 211, 1, 195, 188, 236, 211, 251, 211, 1, 203, 166, 236, 211, - 251, 211, 1, 201, 176, 236, 211, 251, 211, 1, 188, 236, 211, 251, 211, 1, - 140, 236, 211, 251, 211, 1, 219, 75, 236, 211, 251, 211, 1, 191, 123, - 236, 211, 251, 211, 18, 3, 252, 208, 236, 211, 251, 211, 18, 3, 68, 236, - 211, 251, 211, 18, 3, 223, 201, 236, 211, 251, 211, 18, 3, 66, 236, 211, - 251, 211, 18, 3, 196, 30, 236, 211, 251, 211, 18, 3, 71, 236, 211, 251, - 211, 18, 3, 251, 238, 236, 211, 251, 211, 18, 3, 74, 236, 211, 251, 211, - 3, 251, 73, 236, 211, 251, 211, 3, 247, 121, 236, 211, 251, 211, 3, 230, - 74, 236, 211, 251, 211, 195, 40, 236, 211, 251, 211, 208, 229, 214, 216, - 56, 236, 211, 251, 211, 216, 219, 170, 236, 211, 251, 211, 89, 165, 236, - 211, 251, 211, 216, 219, 165, 236, 211, 251, 211, 3, 212, 143, 236, 211, - 251, 211, 55, 237, 240, 236, 211, 251, 211, 231, 206, 233, 5, 236, 211, - 251, 211, 234, 97, 79, 199, 54, 77, 236, 211, 251, 211, 17, 191, 77, 236, - 211, 251, 211, 17, 107, 236, 211, 251, 211, 17, 109, 236, 211, 251, 211, - 17, 138, 236, 211, 251, 211, 17, 134, 236, 211, 251, 211, 17, 150, 236, - 211, 251, 211, 17, 169, 236, 211, 251, 211, 17, 175, 236, 211, 251, 211, - 17, 171, 236, 211, 251, 211, 17, 178, 214, 225, 1, 65, 214, 225, 1, 252, - 208, 214, 225, 1, 68, 214, 225, 1, 223, 201, 214, 225, 1, 66, 214, 225, - 1, 196, 30, 214, 225, 1, 117, 146, 214, 225, 1, 117, 206, 111, 214, 225, - 1, 71, 214, 225, 1, 251, 238, 214, 225, 1, 74, 214, 225, 1, 250, 165, - 214, 225, 1, 155, 214, 225, 1, 221, 217, 214, 225, 1, 231, 242, 214, 225, - 1, 231, 93, 214, 225, 1, 214, 70, 214, 225, 1, 247, 162, 214, 225, 1, - 247, 3, 214, 225, 1, 223, 34, 214, 225, 1, 222, 254, 214, 225, 1, 212, - 103, 214, 225, 1, 197, 132, 214, 225, 1, 197, 120, 214, 225, 1, 237, 193, - 214, 225, 1, 237, 177, 214, 225, 1, 213, 81, 214, 225, 1, 190, 190, 214, - 225, 1, 199, 49, 214, 225, 1, 238, 34, 214, 225, 1, 237, 70, 214, 225, 1, - 181, 214, 225, 1, 168, 214, 225, 1, 209, 230, 214, 225, 1, 249, 155, 214, - 225, 1, 248, 205, 214, 225, 1, 174, 214, 225, 1, 170, 214, 225, 1, 165, - 214, 225, 1, 173, 214, 225, 1, 195, 188, 214, 225, 1, 203, 166, 214, 225, - 1, 201, 176, 214, 225, 1, 188, 214, 225, 1, 140, 214, 225, 1, 219, 75, - 214, 225, 1, 206, 110, 214, 225, 18, 3, 252, 208, 214, 225, 18, 3, 68, - 214, 225, 18, 3, 223, 201, 214, 225, 18, 3, 66, 214, 225, 18, 3, 196, 30, - 214, 225, 18, 3, 117, 146, 214, 225, 18, 3, 117, 206, 111, 214, 225, 18, - 3, 71, 214, 225, 18, 3, 251, 238, 214, 225, 18, 3, 74, 214, 225, 18, 3, - 250, 165, 214, 225, 3, 251, 73, 214, 225, 3, 195, 35, 214, 225, 3, 195, - 40, 214, 225, 3, 250, 147, 214, 225, 3, 202, 211, 214, 225, 229, 229, - 214, 225, 18, 3, 208, 209, 71, 191, 106, 51, 1, 65, 191, 106, 51, 18, 3, - 68, 191, 106, 51, 18, 3, 196, 152, 191, 106, 51, 18, 3, 66, 191, 106, 51, - 18, 3, 71, 191, 106, 51, 18, 3, 211, 153, 191, 106, 51, 18, 3, 74, 191, - 106, 51, 18, 3, 251, 238, 191, 106, 51, 18, 3, 250, 165, 191, 106, 51, - 18, 3, 207, 19, 68, 191, 106, 51, 18, 219, 200, 77, 191, 106, 51, 1, 155, - 191, 106, 51, 1, 221, 217, 191, 106, 51, 1, 231, 242, 191, 106, 51, 1, - 231, 93, 191, 106, 51, 1, 214, 70, 191, 106, 51, 1, 247, 162, 191, 106, - 51, 1, 247, 3, 191, 106, 51, 1, 223, 34, 191, 106, 51, 1, 212, 103, 191, - 106, 51, 1, 197, 132, 191, 106, 51, 1, 197, 120, 191, 106, 51, 1, 237, - 193, 191, 106, 51, 1, 237, 177, 191, 106, 51, 1, 213, 81, 191, 106, 51, - 1, 190, 190, 191, 106, 51, 1, 199, 49, 191, 106, 51, 1, 238, 34, 191, - 106, 51, 1, 237, 70, 191, 106, 51, 1, 181, 191, 106, 51, 1, 168, 191, - 106, 51, 1, 209, 230, 191, 106, 51, 1, 249, 155, 191, 106, 51, 1, 248, - 205, 191, 106, 51, 1, 174, 191, 106, 51, 1, 197, 168, 191, 106, 51, 1, - 197, 157, 191, 106, 51, 1, 235, 37, 191, 106, 51, 1, 235, 31, 191, 106, - 51, 1, 191, 71, 191, 106, 51, 1, 191, 123, 191, 106, 51, 1, 255, 216, - 191, 106, 51, 1, 170, 191, 106, 51, 1, 165, 191, 106, 51, 1, 173, 191, - 106, 51, 1, 195, 188, 191, 106, 51, 1, 203, 166, 191, 106, 51, 1, 201, - 176, 191, 106, 51, 1, 188, 191, 106, 51, 1, 140, 191, 106, 51, 1, 220, - 248, 191, 106, 51, 53, 120, 77, 191, 106, 51, 3, 195, 40, 191, 106, 51, - 3, 247, 121, 191, 106, 51, 3, 247, 122, 4, 211, 44, 191, 106, 51, 3, 247, - 124, 4, 211, 44, 191, 106, 51, 3, 251, 73, 191, 106, 51, 3, 195, 35, 191, - 106, 51, 242, 203, 1, 165, 191, 106, 51, 242, 204, 1, 170, 191, 106, 51, - 242, 204, 1, 165, 191, 106, 51, 242, 204, 1, 173, 191, 106, 51, 242, 204, - 1, 195, 188, 191, 106, 51, 89, 229, 238, 77, 191, 106, 51, 242, 217, 229, - 238, 77, 191, 106, 51, 87, 197, 152, 191, 106, 51, 87, 203, 158, 191, - 106, 51, 87, 55, 203, 158, 191, 106, 51, 87, 180, 197, 152, 191, 106, 51, - 89, 235, 132, 229, 238, 77, 191, 106, 51, 242, 217, 235, 132, 229, 238, - 77, 191, 106, 51, 200, 239, 201, 252, 1, 65, 201, 252, 18, 3, 68, 201, - 252, 18, 3, 196, 152, 201, 252, 18, 3, 66, 201, 252, 18, 3, 71, 201, 252, - 18, 3, 74, 201, 252, 18, 3, 211, 153, 201, 252, 18, 3, 251, 238, 201, - 252, 18, 3, 250, 165, 201, 252, 18, 3, 117, 146, 201, 252, 18, 3, 117, - 172, 201, 252, 18, 219, 200, 77, 201, 252, 1, 155, 201, 252, 1, 221, 217, - 201, 252, 1, 231, 242, 201, 252, 1, 231, 93, 201, 252, 1, 214, 70, 201, - 252, 1, 247, 162, 201, 252, 1, 247, 3, 201, 252, 1, 223, 34, 201, 252, 1, - 222, 254, 201, 252, 1, 212, 103, 201, 252, 1, 197, 132, 201, 252, 1, 197, - 120, 201, 252, 1, 237, 193, 201, 252, 1, 237, 177, 201, 252, 1, 213, 81, - 201, 252, 1, 190, 190, 201, 252, 1, 199, 49, 201, 252, 1, 238, 34, 201, - 252, 1, 237, 70, 201, 252, 1, 181, 201, 252, 1, 168, 201, 252, 1, 209, - 230, 201, 252, 1, 249, 155, 201, 252, 1, 248, 205, 201, 252, 1, 174, 201, - 252, 1, 197, 168, 201, 252, 1, 197, 157, 201, 252, 1, 235, 37, 201, 252, - 1, 191, 71, 201, 252, 1, 191, 123, 201, 252, 1, 255, 216, 201, 252, 1, - 170, 201, 252, 1, 165, 201, 252, 1, 173, 201, 252, 1, 195, 188, 201, 252, - 1, 203, 166, 201, 252, 1, 201, 176, 201, 252, 1, 188, 201, 252, 1, 140, - 201, 252, 1, 220, 248, 201, 252, 3, 222, 239, 201, 252, 3, 196, 75, 201, - 252, 242, 203, 1, 165, 201, 252, 242, 203, 1, 173, 201, 252, 242, 203, 1, - 203, 166, 201, 252, 242, 203, 1, 188, 201, 252, 53, 120, 3, 232, 53, 201, - 252, 53, 120, 3, 222, 154, 201, 252, 53, 120, 3, 214, 72, 201, 252, 53, - 120, 3, 238, 129, 201, 252, 53, 120, 3, 215, 63, 201, 252, 53, 120, 3, - 250, 122, 201, 252, 53, 120, 3, 218, 170, 201, 252, 53, 120, 3, 146, 201, - 252, 53, 120, 3, 172, 201, 252, 53, 120, 3, 203, 168, 201, 252, 53, 120, - 3, 206, 9, 201, 252, 53, 120, 3, 255, 216, 201, 252, 3, 251, 73, 201, - 252, 3, 195, 35, 201, 252, 231, 155, 77, 201, 252, 200, 239, 201, 252, - 87, 197, 152, 201, 252, 87, 203, 158, 201, 252, 87, 55, 203, 158, 201, - 252, 87, 209, 81, 201, 252, 229, 238, 87, 4, 215, 208, 23, 200, 199, 23, - 197, 225, 232, 212, 201, 252, 229, 238, 87, 4, 215, 208, 23, 200, 199, - 23, 232, 212, 201, 252, 229, 238, 87, 4, 215, 208, 23, 200, 198, 201, - 252, 199, 81, 217, 57, 201, 252, 199, 81, 217, 56, 21, 22, 214, 209, 229, - 160, 21, 22, 214, 209, 229, 132, 21, 22, 214, 209, 229, 25, 21, 22, 214, - 209, 228, 254, 21, 22, 214, 209, 140, 21, 22, 214, 209, 230, 93, 21, 22, - 214, 209, 203, 16, 21, 22, 214, 209, 203, 15, 21, 22, 214, 209, 203, 12, - 21, 22, 214, 209, 203, 11, 21, 22, 214, 209, 203, 18, 21, 22, 214, 209, - 203, 17, 21, 22, 201, 238, 21, 22, 201, 225, 21, 22, 201, 208, 21, 22, - 201, 250, 210, 83, 243, 17, 230, 5, 1, 168, 210, 83, 243, 17, 230, 5, 1, - 155, 210, 83, 243, 17, 230, 5, 1, 173, 210, 83, 243, 17, 230, 5, 1, 174, - 210, 83, 243, 17, 230, 5, 1, 238, 34, 210, 83, 243, 17, 230, 5, 1, 191, - 123, 210, 83, 243, 17, 230, 5, 1, 195, 188, 210, 83, 243, 17, 230, 5, 1, - 214, 70, 210, 83, 243, 17, 230, 5, 1, 140, 210, 83, 243, 17, 230, 5, 1, - 231, 242, 210, 83, 243, 17, 230, 5, 1, 221, 217, 210, 83, 243, 17, 230, - 5, 1, 188, 210, 83, 243, 17, 230, 5, 1, 249, 155, 210, 83, 243, 17, 230, - 5, 1, 247, 162, 210, 83, 243, 17, 230, 5, 1, 190, 190, 210, 83, 243, 17, - 230, 5, 1, 199, 49, 210, 83, 243, 17, 230, 5, 1, 181, 210, 83, 243, 17, - 230, 5, 1, 209, 230, 210, 83, 243, 17, 230, 5, 1, 165, 210, 83, 243, 17, - 230, 5, 1, 233, 111, 210, 83, 243, 17, 230, 5, 1, 247, 3, 210, 83, 243, - 17, 230, 5, 1, 65, 210, 83, 243, 17, 230, 5, 1, 71, 210, 83, 243, 17, - 230, 5, 1, 68, 210, 83, 243, 17, 230, 5, 1, 74, 210, 83, 243, 17, 230, 5, - 1, 66, 210, 83, 243, 17, 230, 5, 1, 196, 168, 210, 83, 243, 17, 230, 5, - 1, 228, 37, 210, 83, 243, 17, 230, 5, 1, 53, 210, 238, 210, 83, 243, 17, - 230, 5, 1, 53, 222, 154, 210, 83, 243, 17, 230, 5, 1, 53, 200, 43, 210, - 83, 243, 17, 230, 5, 1, 53, 218, 170, 210, 83, 243, 17, 230, 5, 1, 53, - 215, 63, 210, 83, 243, 17, 230, 5, 1, 53, 172, 210, 83, 243, 17, 230, 5, - 1, 53, 193, 224, 210, 83, 243, 17, 230, 5, 1, 53, 214, 72, 210, 83, 243, - 17, 230, 5, 1, 53, 192, 159, 210, 83, 243, 17, 230, 5, 206, 181, 163, - 219, 21, 210, 83, 243, 17, 230, 5, 206, 181, 198, 79, 210, 83, 243, 17, - 230, 5, 205, 139, 231, 13, 201, 64, 210, 83, 243, 17, 230, 5, 206, 181, - 163, 180, 233, 1, 210, 83, 243, 17, 230, 5, 206, 181, 163, 233, 1, 210, - 83, 243, 17, 230, 5, 205, 139, 231, 13, 201, 65, 233, 1, 210, 83, 243, - 17, 230, 5, 205, 139, 163, 219, 21, 210, 83, 243, 17, 230, 5, 205, 139, - 198, 79, 210, 83, 243, 17, 230, 5, 205, 139, 163, 180, 233, 1, 210, 83, - 243, 17, 230, 5, 205, 139, 163, 233, 1, 210, 83, 243, 17, 230, 5, 216, - 87, 198, 79, 210, 83, 243, 17, 230, 5, 231, 13, 201, 65, 195, 167, 210, - 83, 243, 17, 230, 5, 216, 87, 163, 180, 233, 1, 210, 83, 243, 17, 230, 5, - 216, 87, 163, 233, 1, 210, 83, 243, 17, 230, 5, 218, 241, 163, 219, 21, - 210, 83, 243, 17, 230, 5, 218, 241, 198, 79, 210, 83, 243, 17, 230, 5, - 231, 13, 201, 64, 210, 83, 243, 17, 230, 5, 218, 241, 163, 180, 233, 1, - 210, 83, 243, 17, 230, 5, 218, 241, 163, 233, 1, 210, 83, 243, 17, 230, - 5, 231, 13, 201, 65, 233, 1, 100, 229, 238, 77, 100, 229, 238, 87, 4, - 229, 229, 100, 3, 248, 201, 100, 3, 248, 202, 4, 102, 100, 3, 233, 207, - 248, 201, 100, 3, 233, 207, 248, 202, 4, 102, 100, 3, 193, 102, 232, 227, - 248, 201, 100, 3, 193, 102, 213, 176, 248, 201, 100, 3, 207, 19, 213, - 176, 248, 201, 100, 3, 216, 45, 248, 203, 1, 65, 248, 203, 1, 252, 208, - 248, 203, 1, 68, 248, 203, 1, 223, 201, 248, 203, 1, 66, 248, 203, 1, - 196, 30, 248, 203, 1, 117, 146, 248, 203, 1, 117, 206, 111, 248, 203, 1, - 117, 172, 248, 203, 1, 71, 248, 203, 1, 251, 238, 248, 203, 1, 74, 248, - 203, 1, 250, 165, 248, 203, 1, 155, 248, 203, 1, 221, 217, 248, 203, 1, - 231, 242, 248, 203, 1, 231, 93, 248, 203, 1, 214, 70, 248, 203, 1, 247, - 162, 248, 203, 1, 247, 3, 248, 203, 1, 223, 34, 248, 203, 1, 222, 254, - 248, 203, 1, 212, 103, 248, 203, 1, 197, 132, 248, 203, 1, 197, 120, 248, - 203, 1, 237, 193, 248, 203, 1, 237, 177, 248, 203, 1, 213, 81, 248, 203, - 1, 190, 190, 248, 203, 1, 199, 49, 248, 203, 1, 238, 34, 248, 203, 1, - 237, 70, 248, 203, 1, 181, 248, 203, 1, 168, 248, 203, 1, 209, 230, 248, - 203, 1, 249, 155, 248, 203, 1, 248, 205, 248, 203, 1, 174, 248, 203, 1, - 170, 248, 203, 1, 165, 248, 203, 1, 173, 248, 203, 1, 195, 188, 248, 203, - 1, 203, 166, 248, 203, 1, 201, 176, 248, 203, 1, 188, 248, 203, 1, 140, - 248, 203, 18, 3, 252, 208, 248, 203, 18, 3, 68, 248, 203, 18, 3, 223, - 201, 248, 203, 18, 3, 66, 248, 203, 18, 3, 196, 30, 248, 203, 18, 3, 117, - 146, 248, 203, 18, 3, 117, 206, 111, 248, 203, 18, 3, 117, 172, 248, 203, - 18, 3, 71, 248, 203, 18, 3, 251, 238, 248, 203, 18, 3, 74, 248, 203, 18, - 3, 250, 165, 248, 203, 3, 247, 121, 248, 203, 3, 251, 73, 248, 203, 3, - 195, 35, 248, 203, 3, 195, 40, 248, 203, 3, 250, 147, 248, 203, 237, 240, - 248, 203, 55, 237, 240, 248, 203, 193, 23, 204, 11, 248, 203, 231, 206, - 233, 4, 248, 203, 231, 206, 233, 3, 248, 203, 17, 191, 77, 248, 203, 17, - 107, 248, 203, 17, 109, 248, 203, 17, 138, 248, 203, 17, 134, 248, 203, - 17, 150, 248, 203, 17, 169, 248, 203, 17, 175, 248, 203, 17, 171, 248, - 203, 17, 178, 248, 203, 31, 107, 248, 203, 31, 109, 248, 203, 31, 138, - 248, 203, 31, 134, 248, 203, 31, 150, 248, 203, 31, 169, 248, 203, 31, - 175, 248, 203, 31, 171, 248, 203, 31, 178, 248, 203, 31, 199, 95, 248, - 203, 31, 197, 32, 248, 203, 31, 198, 249, 248, 203, 31, 232, 137, 248, - 203, 31, 233, 17, 248, 203, 31, 202, 121, 248, 203, 31, 203, 242, 248, - 203, 31, 234, 155, 248, 203, 31, 213, 171, 248, 203, 228, 141, 196, 91, - 77, 217, 59, 229, 238, 77, 217, 59, 87, 203, 158, 217, 59, 1, 155, 217, - 59, 1, 221, 217, 217, 59, 1, 231, 242, 217, 59, 1, 214, 70, 217, 59, 1, - 247, 162, 217, 59, 1, 247, 3, 217, 59, 1, 223, 34, 217, 59, 1, 212, 103, - 217, 59, 1, 190, 190, 217, 59, 1, 199, 49, 217, 59, 1, 238, 34, 217, 59, - 1, 181, 217, 59, 1, 168, 217, 59, 1, 209, 230, 217, 59, 1, 249, 155, 217, - 59, 1, 174, 217, 59, 1, 197, 168, 217, 59, 1, 197, 157, 217, 59, 1, 235, - 37, 217, 59, 1, 193, 190, 217, 59, 1, 191, 71, 217, 59, 1, 191, 123, 217, - 59, 1, 255, 216, 217, 59, 1, 170, 217, 59, 1, 165, 217, 59, 1, 173, 217, - 59, 1, 203, 166, 217, 59, 1, 188, 217, 59, 1, 140, 217, 59, 1, 65, 217, - 59, 200, 240, 1, 155, 217, 59, 200, 240, 1, 221, 217, 217, 59, 200, 240, - 1, 231, 242, 217, 59, 200, 240, 1, 214, 70, 217, 59, 200, 240, 1, 247, - 162, 217, 59, 200, 240, 1, 247, 3, 217, 59, 200, 240, 1, 223, 34, 217, - 59, 200, 240, 1, 212, 103, 217, 59, 200, 240, 1, 190, 190, 217, 59, 200, - 240, 1, 199, 49, 217, 59, 200, 240, 1, 238, 34, 217, 59, 200, 240, 1, - 181, 217, 59, 200, 240, 1, 168, 217, 59, 200, 240, 1, 209, 230, 217, 59, - 200, 240, 1, 249, 155, 217, 59, 200, 240, 1, 174, 217, 59, 200, 240, 1, - 197, 168, 217, 59, 200, 240, 1, 197, 157, 217, 59, 200, 240, 1, 235, 37, - 217, 59, 200, 240, 1, 193, 190, 217, 59, 200, 240, 1, 191, 71, 217, 59, - 200, 240, 1, 191, 123, 217, 59, 200, 240, 1, 170, 217, 59, 200, 240, 1, - 165, 217, 59, 200, 240, 1, 173, 217, 59, 200, 240, 1, 203, 166, 217, 59, - 200, 240, 1, 188, 217, 59, 200, 240, 1, 140, 217, 59, 200, 240, 1, 65, - 217, 59, 18, 3, 252, 208, 217, 59, 18, 3, 68, 217, 59, 18, 3, 66, 217, - 59, 18, 3, 71, 217, 59, 18, 3, 74, 217, 59, 3, 251, 73, 217, 59, 3, 247, - 121, 217, 43, 129, 1, 65, 217, 43, 129, 1, 252, 208, 217, 43, 129, 1, 68, - 217, 43, 129, 1, 223, 201, 217, 43, 129, 1, 66, 217, 43, 129, 1, 196, 30, - 217, 43, 129, 1, 71, 217, 43, 129, 1, 251, 238, 217, 43, 129, 1, 74, 217, - 43, 129, 1, 250, 165, 217, 43, 129, 1, 155, 217, 43, 129, 1, 221, 217, - 217, 43, 129, 1, 231, 242, 217, 43, 129, 1, 231, 93, 217, 43, 129, 1, - 214, 70, 217, 43, 129, 1, 247, 162, 217, 43, 129, 1, 247, 3, 217, 43, - 129, 1, 223, 34, 217, 43, 129, 1, 222, 254, 217, 43, 129, 1, 212, 103, - 217, 43, 129, 1, 197, 132, 217, 43, 129, 1, 197, 120, 217, 43, 129, 1, - 237, 193, 217, 43, 129, 1, 237, 177, 217, 43, 129, 1, 213, 81, 217, 43, - 129, 1, 190, 190, 217, 43, 129, 1, 199, 49, 217, 43, 129, 1, 238, 34, - 217, 43, 129, 1, 237, 70, 217, 43, 129, 1, 181, 217, 43, 129, 1, 168, - 217, 43, 129, 1, 209, 230, 217, 43, 129, 1, 249, 155, 217, 43, 129, 1, - 248, 205, 217, 43, 129, 1, 174, 217, 43, 129, 1, 170, 217, 43, 129, 1, - 165, 217, 43, 129, 1, 173, 217, 43, 129, 1, 195, 188, 217, 43, 129, 1, - 203, 166, 217, 43, 129, 1, 201, 176, 217, 43, 129, 1, 188, 217, 43, 129, - 1, 140, 217, 43, 129, 1, 219, 75, 217, 43, 129, 1, 220, 248, 217, 43, - 129, 1, 222, 204, 217, 43, 129, 1, 198, 26, 217, 43, 129, 18, 3, 252, - 208, 217, 43, 129, 18, 3, 68, 217, 43, 129, 18, 3, 223, 201, 217, 43, - 129, 18, 3, 66, 217, 43, 129, 18, 3, 196, 30, 217, 43, 129, 18, 3, 117, - 146, 217, 43, 129, 18, 3, 71, 217, 43, 129, 18, 3, 251, 238, 217, 43, - 129, 18, 3, 74, 217, 43, 129, 18, 3, 250, 165, 217, 43, 129, 3, 251, 73, - 217, 43, 129, 3, 195, 35, 217, 43, 129, 3, 212, 143, 217, 43, 129, 3, - 247, 123, 217, 43, 129, 3, 230, 74, 217, 43, 129, 195, 40, 217, 43, 129, - 207, 45, 217, 43, 129, 207, 183, 217, 43, 129, 17, 191, 77, 217, 43, 129, - 17, 107, 217, 43, 129, 17, 109, 217, 43, 129, 17, 138, 217, 43, 129, 17, - 134, 217, 43, 129, 17, 150, 217, 43, 129, 17, 169, 217, 43, 129, 17, 175, - 217, 43, 129, 17, 171, 217, 43, 129, 17, 178, 230, 159, 129, 1, 65, 230, - 159, 129, 1, 252, 208, 230, 159, 129, 1, 68, 230, 159, 129, 1, 223, 201, - 230, 159, 129, 1, 66, 230, 159, 129, 1, 196, 30, 230, 159, 129, 1, 234, - 190, 230, 159, 129, 1, 251, 238, 230, 159, 129, 1, 211, 89, 230, 159, - 129, 1, 250, 165, 230, 159, 129, 1, 170, 230, 159, 129, 1, 195, 188, 230, - 159, 129, 1, 249, 155, 230, 159, 129, 1, 248, 205, 230, 159, 129, 1, 174, - 230, 159, 129, 1, 155, 230, 159, 129, 1, 221, 217, 230, 159, 129, 1, 190, - 190, 230, 159, 129, 1, 199, 49, 230, 159, 129, 1, 173, 230, 159, 129, 1, - 231, 242, 230, 159, 129, 1, 231, 93, 230, 159, 129, 1, 238, 34, 230, 159, - 129, 1, 237, 70, 230, 159, 129, 1, 181, 230, 159, 129, 1, 247, 162, 230, - 159, 129, 1, 247, 3, 230, 159, 129, 1, 197, 132, 230, 159, 129, 1, 197, - 120, 230, 159, 129, 1, 219, 75, 230, 159, 129, 1, 223, 34, 230, 159, 129, - 1, 222, 254, 230, 159, 129, 1, 237, 193, 230, 159, 129, 1, 237, 177, 230, - 159, 129, 1, 214, 70, 230, 159, 129, 1, 168, 230, 159, 129, 1, 209, 230, - 230, 159, 129, 1, 140, 230, 159, 129, 1, 165, 230, 159, 129, 1, 188, 230, - 159, 129, 18, 3, 252, 208, 230, 159, 129, 18, 3, 68, 230, 159, 129, 18, - 3, 223, 201, 230, 159, 129, 18, 3, 66, 230, 159, 129, 18, 3, 196, 30, - 230, 159, 129, 18, 3, 234, 190, 230, 159, 129, 18, 3, 251, 238, 230, 159, - 129, 18, 3, 211, 89, 230, 159, 129, 18, 3, 250, 165, 230, 159, 129, 3, - 251, 73, 230, 159, 129, 3, 195, 35, 230, 159, 129, 195, 40, 230, 159, - 129, 211, 115, 230, 159, 129, 17, 191, 77, 230, 159, 129, 17, 107, 230, - 159, 129, 17, 109, 230, 159, 129, 17, 138, 230, 159, 129, 17, 134, 230, - 159, 129, 17, 150, 230, 159, 129, 17, 169, 230, 159, 129, 17, 175, 230, - 159, 129, 17, 171, 230, 159, 129, 17, 178, 217, 104, 1, 155, 217, 104, 1, - 231, 242, 217, 104, 1, 214, 70, 217, 104, 1, 168, 217, 104, 1, 249, 155, - 217, 104, 1, 174, 217, 104, 1, 190, 190, 217, 104, 1, 238, 34, 217, 104, - 1, 181, 217, 104, 1, 247, 162, 217, 104, 1, 223, 34, 217, 104, 1, 212, - 103, 217, 104, 1, 170, 217, 104, 1, 165, 217, 104, 1, 173, 217, 104, 1, - 195, 188, 217, 104, 1, 188, 217, 104, 1, 65, 217, 104, 251, 120, 217, - 104, 18, 3, 68, 217, 104, 18, 3, 66, 217, 104, 18, 3, 71, 217, 104, 18, - 3, 74, 217, 104, 210, 96, 217, 104, 234, 97, 79, 205, 54, 222, 35, 3, - 247, 121, 222, 35, 3, 251, 73, 222, 35, 3, 207, 45, 222, 35, 3, 195, 35, - 222, 35, 1, 65, 222, 35, 1, 252, 208, 222, 35, 1, 68, 222, 35, 1, 223, - 201, 222, 35, 1, 66, 222, 35, 1, 196, 30, 222, 35, 1, 117, 146, 222, 35, - 1, 117, 206, 111, 222, 35, 1, 117, 172, 222, 35, 1, 117, 219, 76, 222, - 35, 1, 71, 222, 35, 1, 251, 238, 222, 35, 1, 74, 222, 35, 1, 155, 222, - 35, 1, 221, 217, 222, 35, 1, 231, 242, 222, 35, 1, 231, 93, 222, 35, 1, - 214, 70, 222, 35, 1, 247, 162, 222, 35, 1, 247, 3, 222, 35, 1, 223, 34, - 222, 35, 1, 222, 254, 222, 35, 1, 212, 103, 222, 35, 1, 197, 132, 222, - 35, 1, 197, 120, 222, 35, 1, 237, 193, 222, 35, 1, 237, 177, 222, 35, 1, - 213, 81, 222, 35, 1, 190, 190, 222, 35, 1, 199, 49, 222, 35, 1, 238, 34, - 222, 35, 1, 237, 70, 222, 35, 1, 181, 222, 35, 1, 168, 222, 35, 1, 209, - 230, 222, 35, 1, 249, 155, 222, 35, 1, 248, 205, 222, 35, 1, 174, 222, - 35, 1, 170, 222, 35, 1, 165, 222, 35, 1, 173, 222, 35, 1, 193, 190, 222, - 35, 1, 203, 166, 222, 35, 1, 201, 176, 222, 35, 1, 188, 222, 35, 1, 140, - 222, 35, 1, 222, 204, 222, 35, 18, 3, 252, 208, 222, 35, 18, 3, 251, 159, - 252, 208, 222, 35, 18, 3, 68, 222, 35, 18, 3, 223, 201, 222, 35, 18, 3, - 66, 222, 35, 18, 3, 196, 30, 222, 35, 18, 3, 117, 146, 222, 35, 18, 3, - 71, 222, 35, 18, 3, 251, 238, 222, 35, 18, 3, 233, 244, 222, 35, 3, 221, - 145, 222, 35, 239, 47, 222, 35, 237, 240, 222, 35, 55, 237, 240, 222, 35, - 208, 154, 205, 55, 217, 53, 222, 35, 208, 154, 251, 159, 205, 55, 217, - 53, 222, 35, 208, 154, 232, 188, 222, 35, 208, 154, 201, 249, 233, 5, - 222, 35, 208, 154, 236, 142, 222, 35, 208, 154, 55, 236, 142, 222, 35, - 208, 154, 197, 225, 236, 142, 222, 35, 208, 154, 243, 12, 222, 35, 208, - 154, 233, 6, 243, 12, 222, 35, 208, 154, 201, 218, 222, 35, 208, 154, - 242, 217, 201, 218, 222, 35, 17, 191, 77, 222, 35, 17, 107, 222, 35, 17, - 109, 222, 35, 17, 138, 222, 35, 17, 134, 222, 35, 17, 150, 222, 35, 17, - 169, 222, 35, 17, 175, 222, 35, 17, 171, 222, 35, 17, 178, 219, 90, 1, - 192, 35, 44, 232, 120, 91, 198, 222, 44, 232, 120, 91, 211, 102, 44, 232, - 120, 91, 234, 158, 44, 232, 120, 91, 202, 119, 44, 232, 120, 91, 232, - 141, 44, 232, 120, 91, 198, 245, 44, 232, 120, 115, 234, 157, 44, 232, - 120, 115, 202, 118, 44, 232, 120, 91, 197, 35, 44, 232, 120, 91, 202, - 128, 44, 232, 120, 91, 202, 127, 44, 232, 120, 91, 199, 86, 44, 232, 120, - 91, 234, 161, 44, 232, 120, 115, 197, 34, 44, 232, 120, 115, 202, 126, - 44, 232, 120, 91, 233, 20, 44, 232, 120, 91, 208, 24, 44, 232, 120, 91, - 230, 71, 44, 232, 120, 91, 230, 70, 44, 232, 120, 115, 208, 22, 44, 232, - 120, 235, 123, 233, 95, 221, 146, 44, 3, 214, 106, 44, 3, 247, 8, 44, 3, - 252, 159, 44, 3, 196, 15, 44, 3, 215, 92, 44, 3, 220, 196, 44, 3, 210, - 87, 44, 3, 215, 137, 44, 3, 222, 126, 44, 3, 210, 166, 44, 3, 209, 41, - 44, 3, 195, 173, 44, 3, 210, 217, 44, 3, 220, 185, 44, 3, 195, 143, 44, - 193, 101, 238, 189, 56, 44, 235, 94, 238, 189, 56, 44, 220, 25, 56, 44, - 205, 160, 210, 169, 56, 44, 198, 21, 238, 232, 56, 44, 198, 21, 31, 56, - 44, 238, 171, 56, 44, 23, 211, 157, 56, 44, 201, 228, 56, 44, 198, 39, - 56, 44, 223, 165, 209, 24, 56, 44, 201, 97, 232, 100, 56, 44, 3, 215, 96, - 44, 3, 195, 181, 44, 208, 154, 234, 97, 79, 199, 53, 10, 3, 65, 10, 3, - 42, 25, 65, 10, 3, 42, 25, 249, 137, 10, 3, 42, 25, 231, 211, 199, 84, - 10, 3, 42, 25, 140, 10, 3, 42, 25, 223, 203, 10, 3, 42, 25, 220, 105, - 230, 157, 10, 3, 42, 25, 215, 103, 10, 3, 42, 25, 205, 186, 10, 3, 254, - 217, 10, 3, 252, 157, 10, 3, 252, 158, 25, 250, 209, 10, 3, 252, 158, 25, - 235, 76, 230, 157, 10, 3, 252, 158, 25, 231, 224, 10, 3, 252, 158, 25, - 231, 211, 199, 84, 10, 3, 252, 158, 25, 140, 10, 3, 252, 158, 25, 223, - 204, 230, 157, 10, 3, 252, 158, 25, 223, 174, 10, 3, 252, 158, 25, 220, - 106, 10, 3, 252, 158, 25, 203, 98, 10, 3, 252, 158, 25, 126, 108, 126, - 108, 66, 10, 3, 252, 158, 230, 157, 10, 3, 252, 74, 10, 3, 252, 75, 25, - 249, 116, 10, 3, 252, 75, 25, 231, 211, 199, 84, 10, 3, 252, 75, 25, 216, - 235, 108, 234, 105, 10, 3, 252, 75, 25, 203, 164, 10, 3, 252, 75, 25, - 199, 209, 10, 3, 252, 44, 10, 3, 251, 218, 10, 3, 251, 219, 25, 234, 30, - 10, 3, 251, 219, 25, 203, 60, 108, 231, 25, 10, 3, 251, 209, 10, 3, 251, - 210, 25, 251, 209, 10, 3, 251, 210, 25, 236, 255, 10, 3, 251, 210, 25, - 231, 25, 10, 3, 251, 210, 25, 140, 10, 3, 251, 210, 25, 222, 113, 10, 3, - 251, 210, 25, 221, 168, 10, 3, 251, 210, 25, 203, 114, 10, 3, 251, 210, - 25, 196, 38, 10, 3, 251, 205, 10, 3, 251, 192, 10, 3, 251, 147, 10, 3, - 251, 148, 25, 203, 114, 10, 3, 251, 134, 10, 3, 251, 135, 139, 251, 134, - 10, 3, 251, 135, 115, 198, 144, 10, 3, 251, 135, 108, 214, 243, 211, 65, - 251, 135, 108, 214, 242, 10, 3, 251, 135, 108, 214, 243, 201, 190, 10, 3, - 251, 93, 10, 3, 251, 62, 10, 3, 251, 28, 10, 3, 251, 29, 25, 220, 199, - 10, 3, 251, 0, 10, 3, 250, 217, 10, 3, 250, 211, 10, 3, 250, 212, 191, - 26, 199, 84, 10, 3, 250, 212, 222, 118, 199, 84, 10, 3, 250, 212, 139, - 250, 212, 197, 83, 139, 197, 83, 197, 83, 139, 197, 83, 210, 139, 10, 3, - 250, 212, 139, 250, 212, 139, 250, 211, 10, 3, 250, 212, 139, 250, 212, - 139, 250, 212, 238, 212, 250, 212, 139, 250, 212, 139, 250, 211, 10, 3, - 250, 209, 10, 3, 250, 205, 10, 3, 249, 155, 10, 3, 249, 137, 10, 3, 249, - 131, 10, 3, 249, 123, 10, 3, 249, 117, 10, 3, 249, 118, 139, 249, 117, - 10, 3, 249, 116, 10, 3, 164, 10, 3, 249, 89, 10, 3, 248, 190, 10, 3, 248, - 191, 25, 65, 10, 3, 248, 191, 25, 231, 202, 10, 3, 248, 191, 25, 223, - 204, 230, 157, 10, 3, 248, 12, 10, 3, 248, 13, 139, 248, 13, 252, 157, - 10, 3, 248, 13, 139, 248, 13, 196, 113, 10, 3, 248, 13, 238, 212, 248, - 12, 10, 3, 247, 244, 10, 3, 247, 245, 139, 247, 244, 10, 3, 247, 232, 10, - 3, 247, 231, 10, 3, 238, 34, 10, 3, 238, 24, 10, 3, 238, 25, 221, 127, - 25, 42, 108, 217, 41, 10, 3, 238, 25, 221, 127, 25, 251, 147, 10, 3, 238, - 25, 221, 127, 25, 249, 116, 10, 3, 238, 25, 221, 127, 25, 248, 190, 10, - 3, 238, 25, 221, 127, 25, 231, 242, 10, 3, 238, 25, 221, 127, 25, 231, - 243, 108, 217, 41, 10, 3, 238, 25, 221, 127, 25, 231, 55, 10, 3, 238, 25, - 221, 127, 25, 231, 34, 10, 3, 238, 25, 221, 127, 25, 230, 170, 10, 3, - 238, 25, 221, 127, 25, 140, 10, 3, 238, 25, 221, 127, 25, 223, 79, 10, 3, - 238, 25, 221, 127, 25, 223, 80, 108, 218, 227, 10, 3, 238, 25, 221, 127, - 25, 222, 98, 10, 3, 238, 25, 221, 127, 25, 173, 10, 3, 238, 25, 221, 127, - 25, 218, 227, 10, 3, 238, 25, 221, 127, 25, 218, 228, 108, 217, 40, 10, - 3, 238, 25, 221, 127, 25, 218, 210, 10, 3, 238, 25, 221, 127, 25, 214, - 123, 10, 3, 238, 25, 221, 127, 25, 210, 140, 108, 210, 139, 10, 3, 238, - 25, 221, 127, 25, 202, 223, 10, 3, 238, 25, 221, 127, 25, 199, 209, 10, - 3, 238, 25, 221, 127, 25, 196, 170, 108, 231, 34, 10, 3, 238, 25, 221, - 127, 25, 196, 38, 10, 3, 237, 252, 10, 3, 237, 227, 10, 3, 237, 226, 10, - 3, 237, 225, 10, 3, 237, 46, 10, 3, 237, 28, 10, 3, 237, 1, 10, 3, 237, - 2, 25, 203, 114, 10, 3, 236, 255, 10, 3, 236, 245, 10, 3, 236, 246, 222, - 57, 126, 230, 158, 236, 224, 10, 3, 236, 224, 10, 3, 235, 91, 10, 3, 235, - 92, 139, 235, 91, 10, 3, 235, 92, 230, 157, 10, 3, 235, 92, 203, 95, 10, - 3, 235, 89, 10, 3, 235, 90, 25, 234, 11, 10, 3, 235, 88, 10, 3, 235, 84, - 10, 3, 235, 83, 10, 3, 235, 82, 10, 3, 235, 77, 10, 3, 235, 75, 10, 3, - 235, 76, 230, 157, 10, 3, 235, 76, 230, 158, 230, 157, 10, 3, 235, 74, - 10, 3, 235, 67, 10, 3, 71, 10, 3, 235, 17, 25, 210, 139, 10, 3, 235, 17, - 139, 235, 17, 212, 133, 139, 212, 132, 10, 3, 234, 220, 10, 3, 234, 221, - 25, 42, 108, 230, 107, 108, 238, 34, 10, 3, 234, 221, 25, 231, 202, 10, - 3, 234, 221, 25, 216, 102, 10, 3, 234, 221, 25, 205, 170, 10, 3, 234, - 221, 25, 203, 114, 10, 3, 234, 221, 25, 66, 10, 3, 234, 192, 10, 3, 234, - 179, 10, 3, 234, 142, 10, 3, 234, 105, 10, 3, 234, 106, 25, 231, 210, 10, - 3, 234, 106, 25, 231, 211, 199, 84, 10, 3, 234, 106, 25, 216, 234, 10, 3, - 234, 106, 238, 212, 234, 105, 10, 3, 234, 106, 211, 65, 234, 105, 10, 3, - 234, 106, 201, 190, 10, 3, 234, 33, 10, 3, 234, 30, 10, 3, 234, 11, 10, - 3, 233, 182, 10, 3, 233, 183, 25, 65, 10, 3, 233, 183, 25, 42, 108, 220, - 39, 10, 3, 233, 183, 25, 42, 108, 220, 40, 25, 220, 39, 10, 3, 233, 183, - 25, 251, 134, 10, 3, 233, 183, 25, 249, 137, 10, 3, 233, 183, 25, 235, - 76, 230, 157, 10, 3, 233, 183, 25, 235, 76, 230, 158, 230, 157, 10, 3, - 233, 183, 25, 140, 10, 3, 233, 183, 25, 230, 107, 230, 157, 10, 3, 233, - 183, 25, 223, 204, 230, 157, 10, 3, 233, 183, 25, 222, 56, 10, 3, 233, - 183, 25, 222, 57, 201, 190, 10, 3, 233, 183, 25, 220, 225, 10, 3, 233, - 183, 25, 173, 10, 3, 233, 183, 25, 220, 40, 25, 220, 39, 10, 3, 233, 183, - 25, 219, 148, 10, 3, 233, 183, 25, 218, 227, 10, 3, 233, 183, 25, 196, - 169, 10, 3, 233, 183, 25, 196, 158, 10, 3, 231, 242, 10, 3, 231, 243, - 230, 157, 10, 3, 231, 240, 10, 3, 231, 241, 25, 42, 108, 238, 35, 108, - 140, 10, 3, 231, 241, 25, 42, 108, 140, 10, 3, 231, 241, 25, 42, 108, - 223, 203, 10, 3, 231, 241, 25, 252, 75, 199, 85, 108, 199, 236, 10, 3, - 231, 241, 25, 251, 134, 10, 3, 231, 241, 25, 250, 211, 10, 3, 231, 241, - 25, 250, 210, 108, 231, 224, 10, 3, 231, 241, 25, 249, 137, 10, 3, 231, - 241, 25, 249, 90, 108, 165, 10, 3, 231, 241, 25, 247, 232, 10, 3, 231, - 241, 25, 247, 233, 108, 165, 10, 3, 231, 241, 25, 238, 34, 10, 3, 231, - 241, 25, 237, 46, 10, 3, 231, 241, 25, 237, 2, 25, 203, 114, 10, 3, 231, - 241, 25, 235, 89, 10, 3, 231, 241, 25, 234, 142, 10, 3, 231, 241, 25, - 234, 143, 108, 173, 10, 3, 231, 241, 25, 234, 105, 10, 3, 231, 241, 25, - 234, 106, 25, 231, 211, 199, 84, 10, 3, 231, 241, 25, 231, 211, 199, 84, - 10, 3, 231, 241, 25, 231, 202, 10, 3, 231, 241, 25, 231, 55, 10, 3, 231, - 241, 25, 231, 53, 10, 3, 231, 241, 25, 231, 54, 108, 65, 10, 3, 231, 241, - 25, 231, 35, 108, 201, 5, 10, 3, 231, 241, 25, 230, 107, 108, 218, 228, - 108, 234, 11, 10, 3, 231, 241, 25, 230, 75, 10, 3, 231, 241, 25, 230, 76, - 108, 173, 10, 3, 231, 241, 25, 229, 161, 108, 219, 148, 10, 3, 231, 241, - 25, 228, 153, 10, 3, 231, 241, 25, 223, 204, 230, 157, 10, 3, 231, 241, - 25, 223, 65, 108, 228, 162, 108, 250, 211, 10, 3, 231, 241, 25, 222, 98, - 10, 3, 231, 241, 25, 222, 56, 10, 3, 231, 241, 25, 221, 154, 10, 3, 231, - 241, 25, 221, 155, 108, 220, 39, 10, 3, 231, 241, 25, 220, 226, 108, 251, - 134, 10, 3, 231, 241, 25, 173, 10, 3, 231, 241, 25, 216, 235, 108, 234, - 105, 10, 3, 231, 241, 25, 216, 102, 10, 3, 231, 241, 25, 212, 132, 10, 3, - 231, 241, 25, 212, 133, 139, 212, 132, 10, 3, 231, 241, 25, 168, 10, 3, - 231, 241, 25, 205, 170, 10, 3, 231, 241, 25, 205, 128, 10, 3, 231, 241, - 25, 203, 114, 10, 3, 231, 241, 25, 203, 115, 108, 197, 64, 10, 3, 231, - 241, 25, 203, 80, 10, 3, 231, 241, 25, 200, 204, 10, 3, 231, 241, 25, - 199, 209, 10, 3, 231, 241, 25, 66, 10, 3, 231, 241, 25, 196, 158, 10, 3, - 231, 241, 25, 196, 159, 108, 235, 91, 10, 3, 231, 241, 139, 231, 240, 10, - 3, 231, 235, 10, 3, 231, 236, 238, 212, 231, 235, 10, 3, 231, 233, 10, 3, - 231, 234, 139, 231, 234, 231, 203, 139, 231, 202, 10, 3, 231, 224, 10, 3, - 231, 225, 231, 234, 139, 231, 234, 231, 203, 139, 231, 202, 10, 3, 231, - 223, 10, 3, 231, 221, 10, 3, 231, 212, 10, 3, 231, 210, 10, 3, 231, 211, - 199, 84, 10, 3, 231, 211, 139, 231, 210, 10, 3, 231, 211, 238, 212, 231, - 210, 10, 3, 231, 202, 10, 3, 231, 201, 10, 3, 231, 195, 10, 3, 231, 136, - 10, 3, 231, 137, 25, 220, 199, 10, 3, 231, 55, 10, 3, 231, 56, 25, 71, - 10, 3, 231, 56, 25, 66, 10, 3, 231, 56, 238, 212, 231, 55, 10, 3, 231, - 53, 10, 3, 231, 54, 139, 231, 53, 10, 3, 231, 54, 238, 212, 231, 53, 10, - 3, 231, 50, 10, 3, 231, 34, 10, 3, 231, 35, 230, 157, 10, 3, 231, 32, 10, - 3, 231, 33, 25, 42, 108, 223, 203, 10, 3, 231, 33, 25, 231, 211, 199, 84, - 10, 3, 231, 33, 25, 223, 203, 10, 3, 231, 33, 25, 218, 228, 108, 223, - 203, 10, 3, 231, 33, 25, 168, 10, 3, 231, 27, 10, 3, 231, 25, 10, 3, 231, - 26, 238, 212, 231, 25, 10, 3, 231, 26, 25, 249, 137, 10, 3, 231, 26, 25, - 199, 209, 10, 3, 231, 26, 199, 84, 10, 3, 230, 181, 10, 3, 230, 182, 238, - 212, 230, 181, 10, 3, 230, 179, 10, 3, 230, 180, 25, 222, 98, 10, 3, 230, - 180, 25, 222, 99, 25, 223, 204, 230, 157, 10, 3, 230, 180, 25, 212, 132, - 10, 3, 230, 180, 25, 205, 171, 108, 197, 82, 10, 3, 230, 180, 230, 157, - 10, 3, 230, 170, 10, 3, 230, 171, 25, 42, 108, 220, 199, 10, 3, 230, 171, - 25, 220, 199, 10, 3, 230, 171, 139, 230, 171, 218, 218, 10, 3, 230, 162, - 10, 3, 230, 160, 10, 3, 230, 161, 25, 203, 114, 10, 3, 230, 151, 10, 3, - 230, 150, 10, 3, 230, 145, 10, 3, 230, 144, 10, 3, 140, 10, 3, 230, 107, - 199, 84, 10, 3, 230, 107, 230, 157, 10, 3, 230, 75, 10, 3, 229, 160, 10, - 3, 229, 161, 25, 250, 211, 10, 3, 229, 161, 25, 250, 209, 10, 3, 229, - 161, 25, 249, 137, 10, 3, 229, 161, 25, 236, 224, 10, 3, 229, 161, 25, - 231, 233, 10, 3, 229, 161, 25, 221, 143, 10, 3, 229, 161, 25, 212, 132, - 10, 3, 229, 161, 25, 203, 114, 10, 3, 229, 161, 25, 66, 10, 3, 228, 161, - 10, 3, 228, 153, 10, 3, 228, 154, 25, 251, 134, 10, 3, 228, 154, 25, 230, - 75, 10, 3, 228, 154, 25, 222, 56, 10, 3, 228, 154, 25, 219, 91, 10, 3, - 228, 154, 25, 196, 158, 10, 3, 228, 148, 10, 3, 68, 10, 3, 228, 76, 65, - 10, 3, 228, 32, 10, 3, 223, 231, 10, 3, 223, 232, 139, 223, 232, 247, - 232, 10, 3, 223, 232, 139, 223, 232, 201, 190, 10, 3, 223, 206, 10, 3, - 223, 203, 10, 3, 223, 204, 237, 28, 10, 3, 223, 204, 207, 2, 10, 3, 223, - 204, 139, 223, 204, 203, 64, 139, 203, 64, 196, 159, 139, 196, 158, 10, - 3, 223, 204, 230, 157, 10, 3, 223, 193, 10, 3, 223, 194, 25, 231, 211, - 199, 84, 10, 3, 223, 192, 10, 3, 223, 182, 10, 3, 223, 183, 25, 199, 209, - 10, 3, 223, 183, 238, 212, 223, 182, 10, 3, 223, 183, 211, 65, 223, 182, - 10, 3, 223, 183, 201, 190, 10, 3, 223, 174, 10, 3, 223, 164, 10, 3, 223, - 79, 10, 3, 223, 64, 10, 3, 155, 10, 3, 222, 144, 25, 65, 10, 3, 222, 144, - 25, 252, 44, 10, 3, 222, 144, 25, 252, 45, 108, 220, 225, 10, 3, 222, - 144, 25, 250, 209, 10, 3, 222, 144, 25, 249, 137, 10, 3, 222, 144, 25, - 249, 116, 10, 3, 222, 144, 25, 164, 10, 3, 222, 144, 25, 248, 190, 10, 3, - 222, 144, 25, 234, 30, 10, 3, 222, 144, 25, 234, 11, 10, 3, 222, 144, 25, - 231, 242, 10, 3, 222, 144, 25, 231, 224, 10, 3, 222, 144, 25, 231, 211, - 199, 84, 10, 3, 222, 144, 25, 231, 202, 10, 3, 222, 144, 25, 231, 203, - 108, 203, 165, 108, 65, 10, 3, 222, 144, 25, 231, 55, 10, 3, 222, 144, - 25, 231, 34, 10, 3, 222, 144, 25, 231, 26, 108, 205, 128, 10, 3, 222, - 144, 25, 231, 26, 238, 212, 231, 25, 10, 3, 222, 144, 25, 230, 181, 10, - 3, 222, 144, 25, 230, 150, 10, 3, 222, 144, 25, 223, 203, 10, 3, 222, - 144, 25, 223, 182, 10, 3, 222, 144, 25, 222, 98, 10, 3, 222, 144, 25, - 221, 168, 10, 3, 222, 144, 25, 221, 154, 10, 3, 222, 144, 25, 219, 148, - 10, 3, 222, 144, 25, 218, 227, 10, 3, 222, 144, 25, 216, 234, 10, 3, 222, - 144, 25, 216, 235, 108, 235, 91, 10, 3, 222, 144, 25, 216, 235, 108, 231, - 55, 10, 3, 222, 144, 25, 216, 235, 108, 199, 145, 10, 3, 222, 144, 25, - 216, 102, 10, 3, 222, 144, 25, 216, 103, 108, 212, 127, 10, 3, 222, 144, - 25, 214, 123, 10, 3, 222, 144, 25, 212, 132, 10, 3, 222, 144, 25, 209, - 187, 10, 3, 222, 144, 25, 206, 69, 10, 3, 222, 144, 25, 188, 10, 3, 222, - 144, 25, 205, 128, 10, 3, 222, 144, 25, 203, 166, 10, 3, 222, 144, 25, - 203, 114, 10, 3, 222, 144, 25, 203, 80, 10, 3, 222, 144, 25, 203, 6, 10, - 3, 222, 144, 25, 202, 202, 10, 3, 222, 144, 25, 200, 213, 10, 3, 222, - 144, 25, 199, 179, 10, 3, 222, 144, 25, 66, 10, 3, 222, 144, 25, 196, - 169, 10, 3, 222, 144, 25, 196, 158, 10, 3, 222, 144, 25, 196, 116, 25, - 168, 10, 3, 222, 144, 25, 196, 38, 10, 3, 222, 144, 25, 191, 30, 10, 3, - 222, 130, 10, 3, 222, 131, 238, 212, 222, 130, 10, 3, 222, 119, 10, 3, - 222, 115, 10, 3, 222, 113, 10, 3, 222, 112, 10, 3, 222, 110, 10, 3, 222, - 111, 139, 222, 110, 10, 3, 222, 98, 10, 3, 222, 99, 25, 223, 204, 230, - 157, 10, 3, 222, 93, 10, 3, 222, 94, 25, 249, 137, 10, 3, 222, 94, 238, - 212, 222, 93, 10, 3, 222, 91, 10, 3, 222, 90, 10, 3, 222, 56, 10, 3, 222, - 57, 220, 107, 25, 126, 139, 220, 107, 25, 66, 10, 3, 222, 57, 139, 222, - 57, 220, 107, 25, 126, 139, 220, 107, 25, 66, 10, 3, 221, 244, 10, 3, - 221, 168, 10, 3, 221, 169, 25, 249, 137, 10, 3, 221, 169, 25, 66, 10, 3, - 221, 169, 25, 196, 158, 10, 3, 221, 154, 10, 3, 221, 143, 10, 3, 221, - 129, 10, 3, 221, 128, 10, 3, 221, 126, 10, 3, 221, 127, 139, 221, 126, - 10, 3, 220, 234, 10, 3, 220, 235, 139, 229, 161, 25, 250, 210, 220, 235, - 139, 229, 161, 25, 250, 209, 10, 3, 220, 225, 10, 3, 220, 223, 10, 3, - 220, 224, 195, 168, 20, 10, 3, 220, 222, 10, 3, 220, 213, 10, 3, 220, - 214, 230, 157, 10, 3, 220, 212, 10, 3, 220, 199, 10, 3, 220, 200, 211, - 65, 220, 199, 10, 3, 220, 192, 10, 3, 220, 169, 10, 3, 173, 10, 3, 220, - 106, 10, 3, 220, 107, 25, 65, 10, 3, 220, 107, 25, 42, 108, 238, 35, 108, - 140, 10, 3, 220, 107, 25, 42, 108, 231, 202, 10, 3, 220, 107, 25, 42, - 108, 220, 39, 10, 3, 220, 107, 25, 251, 209, 10, 3, 220, 107, 25, 251, - 134, 10, 3, 220, 107, 25, 250, 212, 191, 26, 199, 84, 10, 3, 220, 107, - 25, 249, 137, 10, 3, 220, 107, 25, 248, 190, 10, 3, 220, 107, 25, 237, - 227, 10, 3, 220, 107, 25, 234, 105, 10, 3, 220, 107, 25, 231, 242, 10, 3, - 220, 107, 25, 231, 202, 10, 3, 220, 107, 25, 230, 170, 10, 3, 220, 107, - 25, 230, 171, 108, 230, 170, 10, 3, 220, 107, 25, 140, 10, 3, 220, 107, - 25, 230, 75, 10, 3, 220, 107, 25, 229, 161, 25, 212, 132, 10, 3, 220, - 107, 25, 223, 204, 230, 157, 10, 3, 220, 107, 25, 223, 182, 10, 3, 220, - 107, 25, 223, 183, 108, 140, 10, 3, 220, 107, 25, 223, 183, 108, 218, - 227, 10, 3, 220, 107, 25, 221, 168, 10, 3, 220, 107, 25, 221, 143, 10, 3, - 220, 107, 25, 220, 225, 10, 3, 220, 107, 25, 220, 213, 10, 3, 220, 107, - 25, 220, 214, 108, 229, 161, 108, 65, 10, 3, 220, 107, 25, 220, 106, 10, - 3, 220, 107, 25, 219, 91, 10, 3, 220, 107, 25, 218, 227, 10, 3, 220, 107, - 25, 218, 212, 10, 3, 220, 107, 25, 216, 234, 10, 3, 220, 107, 25, 216, - 235, 108, 234, 105, 10, 3, 220, 107, 25, 215, 103, 10, 3, 220, 107, 25, - 214, 123, 10, 3, 220, 107, 25, 203, 115, 108, 200, 204, 10, 3, 220, 107, - 25, 203, 60, 108, 231, 26, 108, 234, 30, 10, 3, 220, 107, 25, 203, 60, - 108, 231, 26, 199, 84, 10, 3, 220, 107, 25, 203, 4, 10, 3, 220, 107, 25, - 203, 5, 108, 203, 4, 10, 3, 220, 107, 25, 200, 204, 10, 3, 220, 107, 25, - 199, 223, 10, 3, 220, 107, 25, 199, 209, 10, 3, 220, 107, 25, 199, 146, - 108, 42, 108, 201, 6, 108, 181, 10, 3, 220, 107, 25, 66, 10, 3, 220, 107, - 25, 126, 108, 65, 10, 3, 220, 107, 25, 126, 108, 126, 108, 66, 10, 3, - 220, 107, 25, 196, 170, 108, 250, 211, 10, 3, 220, 107, 25, 196, 158, 10, - 3, 220, 107, 25, 196, 38, 10, 3, 220, 107, 201, 190, 10, 3, 220, 104, 10, - 3, 220, 105, 25, 203, 114, 10, 3, 220, 105, 25, 203, 115, 108, 200, 204, - 10, 3, 220, 105, 230, 157, 10, 3, 220, 105, 230, 158, 139, 220, 105, 230, - 158, 203, 114, 10, 3, 220, 100, 10, 3, 220, 39, 10, 3, 220, 40, 25, 220, - 39, 10, 3, 220, 37, 10, 3, 220, 38, 25, 220, 199, 10, 3, 220, 38, 25, - 220, 200, 108, 206, 69, 10, 3, 219, 148, 10, 3, 219, 129, 10, 3, 219, - 117, 10, 3, 219, 91, 10, 3, 218, 227, 10, 3, 218, 228, 25, 249, 137, 10, - 3, 218, 225, 10, 3, 218, 226, 25, 251, 209, 10, 3, 218, 226, 25, 249, - 137, 10, 3, 218, 226, 25, 234, 11, 10, 3, 218, 226, 25, 234, 12, 199, 84, - 10, 3, 218, 226, 25, 231, 211, 199, 84, 10, 3, 218, 226, 25, 229, 161, - 25, 249, 137, 10, 3, 218, 226, 25, 223, 182, 10, 3, 218, 226, 25, 222, - 115, 10, 3, 218, 226, 25, 222, 113, 10, 3, 218, 226, 25, 222, 114, 108, - 250, 211, 10, 3, 218, 226, 25, 221, 168, 10, 3, 218, 226, 25, 220, 128, - 108, 250, 211, 10, 3, 218, 226, 25, 220, 106, 10, 3, 218, 226, 25, 216, - 235, 108, 234, 105, 10, 3, 218, 226, 25, 214, 123, 10, 3, 218, 226, 25, - 212, 180, 10, 3, 218, 226, 25, 202, 224, 108, 250, 211, 10, 3, 218, 226, - 25, 202, 193, 108, 248, 12, 10, 3, 218, 226, 25, 197, 82, 10, 3, 218, - 226, 199, 84, 10, 3, 218, 226, 238, 212, 218, 225, 10, 3, 218, 226, 211, - 65, 218, 225, 10, 3, 218, 226, 201, 190, 10, 3, 218, 226, 203, 95, 10, 3, - 218, 224, 10, 3, 218, 218, 10, 3, 218, 219, 139, 218, 218, 10, 3, 218, - 219, 211, 65, 218, 218, 10, 3, 218, 219, 203, 95, 10, 3, 218, 215, 10, 3, - 218, 212, 10, 3, 218, 210, 10, 3, 218, 211, 139, 218, 210, 10, 3, 218, - 211, 139, 218, 211, 231, 203, 139, 231, 202, 10, 3, 174, 10, 3, 217, 162, - 25, 199, 209, 10, 3, 217, 162, 230, 157, 10, 3, 217, 154, 10, 3, 217, - 122, 10, 3, 217, 67, 10, 3, 217, 41, 10, 3, 217, 40, 10, 3, 216, 234, 10, - 3, 216, 175, 10, 3, 216, 102, 10, 3, 216, 46, 10, 3, 215, 157, 10, 3, - 215, 158, 139, 215, 157, 10, 3, 215, 142, 10, 3, 215, 143, 230, 157, 10, - 3, 215, 121, 10, 3, 215, 107, 10, 3, 215, 103, 10, 3, 215, 104, 25, 65, - 10, 3, 215, 104, 25, 220, 199, 10, 3, 215, 104, 25, 191, 123, 10, 3, 215, - 104, 139, 215, 103, 10, 3, 215, 104, 139, 215, 104, 25, 42, 108, 181, 10, - 3, 215, 104, 238, 212, 215, 103, 10, 3, 215, 101, 10, 3, 215, 102, 25, - 65, 10, 3, 215, 102, 25, 42, 108, 237, 46, 10, 3, 215, 102, 25, 237, 46, - 10, 3, 215, 102, 230, 157, 10, 3, 181, 10, 3, 214, 255, 10, 3, 214, 242, - 10, 3, 214, 243, 223, 94, 10, 3, 214, 243, 25, 203, 7, 199, 84, 10, 3, - 214, 243, 211, 65, 214, 242, 10, 3, 214, 241, 10, 3, 214, 233, 212, 118, - 10, 3, 214, 232, 10, 3, 214, 231, 10, 3, 214, 123, 10, 3, 214, 124, 25, - 65, 10, 3, 214, 124, 25, 196, 158, 10, 3, 214, 124, 203, 95, 10, 3, 213, - 221, 10, 3, 213, 222, 25, 71, 10, 3, 213, 212, 10, 3, 213, 182, 10, 3, - 213, 183, 25, 231, 211, 199, 84, 10, 3, 213, 183, 25, 231, 203, 108, 231, - 211, 199, 84, 10, 3, 213, 178, 10, 3, 213, 179, 25, 251, 134, 10, 3, 213, - 179, 25, 250, 211, 10, 3, 213, 179, 25, 250, 212, 108, 250, 211, 10, 3, - 213, 179, 25, 230, 170, 10, 3, 213, 179, 25, 216, 235, 108, 231, 211, - 199, 84, 10, 3, 213, 179, 25, 214, 123, 10, 3, 213, 179, 25, 212, 132, - 10, 3, 213, 179, 25, 203, 114, 10, 3, 213, 179, 25, 203, 115, 108, 42, - 251, 134, 10, 3, 213, 179, 25, 203, 115, 108, 250, 211, 10, 3, 213, 179, - 25, 203, 115, 108, 250, 212, 108, 250, 211, 10, 3, 213, 179, 25, 196, - 170, 108, 250, 211, 10, 3, 213, 179, 25, 196, 38, 10, 3, 213, 165, 10, 3, - 212, 180, 10, 3, 212, 149, 10, 3, 212, 132, 10, 3, 212, 133, 220, 105, - 25, 231, 202, 10, 3, 212, 133, 220, 105, 25, 217, 41, 10, 3, 212, 133, - 220, 105, 25, 205, 170, 10, 3, 212, 133, 220, 105, 25, 205, 171, 139, - 212, 133, 220, 105, 25, 205, 170, 10, 3, 212, 133, 220, 105, 25, 196, 38, - 10, 3, 212, 133, 199, 84, 10, 3, 212, 133, 139, 212, 132, 10, 3, 212, - 133, 238, 212, 212, 132, 10, 3, 212, 133, 238, 212, 212, 133, 220, 105, - 139, 220, 104, 10, 3, 212, 127, 10, 3, 212, 128, 252, 75, 25, 250, 205, - 10, 3, 212, 128, 252, 75, 25, 248, 190, 10, 3, 212, 128, 252, 75, 25, - 235, 84, 10, 3, 212, 128, 252, 75, 25, 230, 170, 10, 3, 212, 128, 252, - 75, 25, 223, 204, 230, 157, 10, 3, 212, 128, 252, 75, 25, 222, 113, 10, - 3, 212, 128, 252, 75, 25, 173, 10, 3, 212, 128, 252, 75, 25, 214, 123, - 10, 3, 212, 128, 252, 75, 25, 202, 190, 10, 3, 212, 128, 252, 75, 25, - 196, 169, 10, 3, 212, 128, 221, 127, 25, 248, 190, 10, 3, 212, 128, 221, - 127, 25, 248, 191, 66, 10, 3, 168, 10, 3, 210, 212, 10, 3, 210, 168, 10, - 3, 210, 139, 10, 3, 209, 245, 10, 3, 209, 187, 10, 3, 209, 188, 25, 65, - 10, 3, 209, 188, 25, 252, 157, 10, 3, 209, 188, 25, 248, 190, 10, 3, 209, - 188, 25, 248, 12, 10, 3, 209, 188, 25, 71, 10, 3, 209, 188, 25, 68, 10, - 3, 209, 188, 25, 228, 32, 10, 3, 209, 188, 25, 66, 10, 3, 209, 188, 25, - 196, 169, 10, 3, 209, 188, 238, 212, 209, 187, 10, 3, 209, 123, 10, 3, - 209, 124, 25, 222, 93, 10, 3, 209, 124, 25, 196, 158, 10, 3, 209, 124, - 25, 191, 123, 10, 3, 209, 124, 211, 65, 209, 123, 10, 3, 165, 10, 3, 207, - 178, 10, 3, 207, 2, 10, 3, 206, 69, 10, 3, 188, 10, 3, 205, 187, 212, - 118, 10, 3, 205, 186, 10, 3, 205, 187, 25, 65, 10, 3, 205, 187, 25, 235, - 91, 10, 3, 205, 187, 25, 235, 89, 10, 3, 205, 187, 25, 140, 10, 3, 205, - 187, 25, 222, 98, 10, 3, 205, 187, 25, 220, 199, 10, 3, 205, 187, 25, - 218, 210, 10, 3, 205, 187, 25, 216, 102, 10, 3, 205, 187, 25, 212, 132, - 10, 3, 205, 187, 25, 205, 170, 10, 3, 205, 187, 25, 203, 80, 10, 3, 205, - 187, 25, 199, 236, 10, 3, 205, 187, 25, 196, 169, 10, 3, 205, 187, 25, - 196, 164, 10, 3, 205, 187, 25, 196, 120, 10, 3, 205, 187, 25, 196, 62, - 10, 3, 205, 187, 25, 196, 38, 10, 3, 205, 187, 139, 205, 186, 10, 3, 205, - 187, 230, 157, 10, 3, 205, 170, 10, 3, 205, 171, 220, 107, 25, 250, 209, - 10, 3, 205, 137, 10, 3, 205, 128, 10, 3, 203, 166, 10, 3, 203, 164, 10, - 3, 203, 165, 25, 65, 10, 3, 203, 165, 25, 249, 137, 10, 3, 203, 165, 25, - 231, 25, 10, 3, 203, 165, 25, 214, 123, 10, 3, 203, 165, 25, 203, 4, 10, - 3, 203, 165, 25, 197, 64, 10, 3, 203, 165, 25, 66, 10, 3, 203, 165, 25, - 126, 108, 65, 10, 3, 203, 162, 10, 3, 203, 160, 10, 3, 203, 132, 10, 3, - 203, 114, 10, 3, 203, 115, 228, 161, 10, 3, 203, 115, 139, 203, 115, 231, - 234, 139, 231, 234, 231, 203, 139, 231, 202, 10, 3, 203, 115, 139, 203, - 115, 199, 237, 139, 199, 237, 231, 203, 139, 231, 202, 10, 3, 203, 107, - 10, 3, 203, 102, 10, 3, 203, 98, 10, 3, 203, 97, 10, 3, 203, 94, 10, 3, - 203, 80, 10, 3, 203, 81, 25, 65, 10, 3, 203, 81, 25, 223, 182, 10, 3, - 203, 74, 10, 3, 203, 75, 25, 65, 10, 3, 203, 75, 25, 249, 117, 10, 3, - 203, 75, 25, 247, 244, 10, 3, 203, 75, 25, 236, 245, 10, 3, 203, 75, 25, - 231, 202, 10, 3, 203, 75, 25, 223, 203, 10, 3, 203, 75, 25, 223, 204, - 230, 157, 10, 3, 203, 75, 25, 220, 192, 10, 3, 203, 75, 25, 218, 212, 10, - 3, 203, 75, 25, 215, 142, 10, 3, 203, 75, 25, 205, 170, 10, 3, 203, 68, - 10, 3, 203, 63, 10, 3, 203, 64, 199, 84, 10, 3, 203, 64, 139, 203, 64, - 247, 233, 139, 247, 232, 10, 3, 203, 59, 10, 3, 203, 6, 10, 3, 203, 7, - 139, 223, 95, 203, 6, 10, 3, 203, 4, 10, 3, 203, 2, 10, 3, 202, 223, 10, - 3, 202, 224, 230, 157, 10, 3, 202, 202, 10, 3, 202, 200, 10, 3, 202, 201, - 139, 202, 201, 203, 4, 10, 3, 202, 192, 10, 3, 202, 190, 10, 3, 201, 5, - 10, 3, 201, 6, 139, 201, 5, 10, 3, 200, 216, 10, 3, 200, 215, 10, 3, 200, - 213, 10, 3, 200, 204, 10, 3, 200, 203, 10, 3, 200, 175, 10, 3, 200, 174, - 10, 3, 190, 190, 10, 3, 199, 252, 250, 195, 10, 3, 199, 252, 25, 229, - 160, 10, 3, 199, 252, 25, 216, 102, 10, 3, 199, 252, 230, 157, 10, 3, - 199, 236, 10, 3, 199, 237, 139, 199, 237, 213, 222, 139, 213, 222, 236, - 225, 139, 236, 224, 10, 3, 199, 237, 201, 190, 10, 3, 199, 223, 10, 3, - 199, 224, 25, 248, 190, 10, 3, 199, 224, 25, 230, 170, 10, 3, 199, 224, - 25, 203, 114, 10, 3, 199, 224, 25, 203, 6, 10, 3, 199, 224, 25, 197, 82, - 10, 3, 199, 224, 25, 196, 158, 10, 3, 199, 209, 10, 3, 199, 179, 10, 3, - 199, 145, 10, 3, 199, 146, 230, 157, 10, 3, 198, 193, 10, 3, 198, 194, - 199, 84, 10, 3, 198, 154, 10, 3, 198, 131, 10, 3, 198, 132, 25, 199, 209, - 10, 3, 198, 132, 139, 198, 131, 10, 3, 198, 132, 139, 198, 132, 231, 234, - 139, 231, 234, 231, 203, 139, 231, 202, 10, 3, 197, 94, 10, 3, 197, 82, - 10, 3, 197, 80, 10, 3, 197, 76, 10, 3, 197, 64, 10, 3, 197, 65, 139, 197, - 65, 191, 124, 139, 191, 123, 10, 3, 66, 10, 3, 126, 230, 170, 10, 3, 126, - 126, 66, 10, 3, 126, 139, 126, 210, 223, 139, 210, 223, 231, 203, 139, - 231, 202, 10, 3, 126, 139, 126, 200, 176, 139, 200, 175, 10, 3, 126, 139, - 126, 126, 207, 19, 139, 126, 207, 18, 10, 3, 196, 169, 10, 3, 196, 164, - 10, 3, 196, 158, 10, 3, 196, 159, 220, 192, 10, 3, 196, 159, 25, 249, - 137, 10, 3, 196, 159, 25, 216, 102, 10, 3, 196, 159, 25, 126, 108, 126, - 108, 66, 10, 3, 196, 159, 25, 126, 108, 126, 108, 126, 230, 157, 10, 3, - 196, 159, 230, 157, 10, 3, 196, 159, 203, 95, 10, 3, 196, 159, 203, 96, - 25, 249, 137, 10, 3, 196, 153, 10, 3, 196, 120, 10, 3, 196, 121, 25, 220, - 106, 10, 3, 196, 121, 25, 216, 235, 108, 238, 34, 10, 3, 196, 121, 25, - 203, 164, 10, 3, 196, 121, 25, 66, 10, 3, 196, 119, 10, 3, 196, 115, 10, - 3, 196, 116, 25, 222, 56, 10, 3, 196, 116, 25, 168, 10, 3, 196, 113, 10, - 3, 196, 114, 230, 157, 10, 3, 196, 62, 10, 3, 196, 63, 238, 212, 196, 62, - 10, 3, 196, 63, 203, 95, 10, 3, 196, 60, 10, 3, 196, 61, 25, 42, 108, - 140, 10, 3, 196, 61, 25, 42, 108, 181, 10, 3, 196, 61, 25, 251, 209, 10, - 3, 196, 61, 25, 140, 10, 3, 196, 61, 25, 212, 132, 10, 3, 196, 61, 25, - 196, 169, 10, 3, 196, 61, 25, 196, 170, 108, 250, 211, 10, 3, 196, 61, - 25, 196, 170, 108, 248, 190, 10, 3, 196, 59, 10, 3, 196, 56, 10, 3, 196, - 55, 10, 3, 196, 51, 10, 3, 196, 52, 25, 65, 10, 3, 196, 52, 25, 250, 205, - 10, 3, 196, 52, 25, 164, 10, 3, 196, 52, 25, 235, 77, 10, 3, 196, 52, 25, - 231, 242, 10, 3, 196, 52, 25, 231, 224, 10, 3, 196, 52, 25, 231, 211, - 199, 84, 10, 3, 196, 52, 25, 231, 202, 10, 3, 196, 52, 25, 230, 181, 10, - 3, 196, 52, 25, 140, 10, 3, 196, 52, 25, 223, 203, 10, 3, 196, 52, 25, - 223, 182, 10, 3, 196, 52, 25, 223, 64, 10, 3, 196, 52, 25, 221, 168, 10, - 3, 196, 52, 25, 218, 210, 10, 3, 196, 52, 25, 216, 46, 10, 3, 196, 52, - 25, 168, 10, 3, 196, 52, 25, 203, 114, 10, 3, 196, 52, 25, 202, 200, 10, - 3, 196, 52, 25, 197, 94, 10, 3, 196, 52, 25, 126, 108, 230, 170, 10, 3, - 196, 52, 25, 196, 158, 10, 3, 196, 52, 25, 196, 49, 10, 3, 196, 49, 10, - 3, 196, 50, 25, 66, 10, 3, 196, 38, 10, 3, 196, 39, 25, 65, 10, 3, 196, - 39, 25, 220, 234, 10, 3, 196, 39, 25, 220, 199, 10, 3, 196, 39, 25, 199, - 209, 10, 3, 196, 34, 10, 3, 196, 37, 10, 3, 196, 35, 10, 3, 196, 31, 10, - 3, 196, 16, 10, 3, 196, 17, 25, 222, 56, 10, 3, 196, 14, 10, 3, 191, 123, - 10, 3, 191, 124, 199, 84, 10, 3, 191, 124, 112, 25, 220, 199, 10, 3, 191, - 118, 10, 3, 191, 107, 10, 3, 191, 86, 10, 3, 191, 30, 10, 3, 191, 31, - 139, 191, 30, 10, 3, 191, 29, 10, 3, 191, 27, 10, 3, 191, 28, 222, 118, - 199, 84, 10, 3, 191, 22, 10, 3, 191, 13, 10, 3, 190, 251, 10, 3, 190, - 249, 10, 3, 190, 250, 25, 65, 10, 3, 190, 248, 10, 3, 190, 247, 10, 3, - 222, 81, 234, 139, 10, 3, 252, 158, 25, 212, 132, 10, 3, 252, 75, 25, 65, - 10, 3, 251, 148, 25, 220, 215, 10, 3, 238, 25, 221, 127, 25, 196, 170, - 108, 217, 41, 10, 3, 238, 23, 10, 3, 236, 225, 108, 203, 6, 10, 3, 235, - 90, 25, 203, 114, 10, 3, 233, 183, 25, 230, 170, 10, 3, 233, 183, 25, - 203, 114, 10, 3, 231, 241, 25, 251, 135, 108, 222, 99, 108, 65, 10, 3, - 231, 241, 25, 250, 209, 10, 3, 231, 166, 10, 3, 231, 44, 10, 3, 228, 133, - 10, 3, 222, 144, 25, 251, 93, 10, 3, 222, 144, 25, 250, 208, 10, 3, 222, - 144, 25, 231, 25, 10, 3, 222, 144, 25, 230, 170, 10, 3, 222, 144, 25, - 229, 161, 25, 250, 209, 10, 3, 222, 144, 25, 218, 210, 10, 3, 222, 144, - 25, 168, 10, 3, 222, 144, 25, 202, 254, 10, 3, 222, 144, 25, 197, 94, 10, - 3, 222, 144, 25, 196, 60, 10, 3, 220, 107, 25, 231, 55, 10, 3, 218, 226, - 203, 96, 25, 249, 137, 10, 3, 218, 226, 25, 234, 12, 108, 220, 39, 10, 3, - 218, 226, 25, 203, 6, 10, 3, 216, 174, 10, 3, 215, 102, 25, 191, 123, 10, - 3, 214, 254, 10, 3, 213, 181, 10, 3, 213, 180, 10, 3, 213, 179, 25, 249, - 117, 10, 3, 213, 179, 25, 231, 55, 10, 3, 212, 150, 206, 123, 213, 172, - 237, 126, 10, 3, 209, 246, 250, 195, 10, 3, 209, 127, 10, 3, 205, 187, - 25, 223, 204, 230, 157, 10, 3, 198, 185, 10, 3, 196, 121, 25, 216, 234, - 10, 3, 126, 66, 10, 167, 3, 105, 250, 211, 10, 167, 3, 115, 250, 211, 10, - 167, 3, 232, 130, 250, 211, 10, 167, 3, 232, 228, 250, 211, 10, 167, 3, - 202, 137, 250, 211, 10, 167, 3, 203, 248, 250, 211, 10, 167, 3, 234, 166, - 250, 211, 10, 167, 3, 213, 177, 250, 211, 10, 167, 3, 115, 236, 224, 10, - 167, 3, 232, 130, 236, 224, 10, 167, 3, 232, 228, 236, 224, 10, 167, 3, - 202, 137, 236, 224, 10, 167, 3, 203, 248, 236, 224, 10, 167, 3, 234, 166, - 236, 224, 10, 167, 3, 213, 177, 236, 224, 10, 167, 3, 232, 130, 66, 10, - 167, 3, 232, 228, 66, 10, 167, 3, 202, 137, 66, 10, 167, 3, 203, 248, 66, - 10, 167, 3, 234, 166, 66, 10, 167, 3, 213, 177, 66, 10, 167, 3, 91, 231, - 138, 10, 167, 3, 105, 231, 138, 10, 167, 3, 115, 231, 138, 10, 167, 3, - 232, 130, 231, 138, 10, 167, 3, 232, 228, 231, 138, 10, 167, 3, 202, 137, - 231, 138, 10, 167, 3, 203, 248, 231, 138, 10, 167, 3, 234, 166, 231, 138, - 10, 167, 3, 213, 177, 231, 138, 10, 167, 3, 91, 231, 135, 10, 167, 3, - 105, 231, 135, 10, 167, 3, 115, 231, 135, 10, 167, 3, 232, 130, 231, 135, - 10, 167, 3, 232, 228, 231, 135, 10, 167, 3, 105, 203, 132, 10, 167, 3, - 115, 203, 132, 10, 167, 3, 115, 203, 133, 195, 168, 20, 10, 167, 3, 232, - 130, 203, 132, 10, 167, 3, 232, 228, 203, 132, 10, 167, 3, 202, 137, 203, - 132, 10, 167, 3, 203, 248, 203, 132, 10, 167, 3, 234, 166, 203, 132, 10, - 167, 3, 213, 177, 203, 132, 10, 167, 3, 91, 203, 125, 10, 167, 3, 105, - 203, 125, 10, 167, 3, 115, 203, 125, 10, 167, 3, 115, 203, 126, 195, 168, - 20, 10, 167, 3, 232, 130, 203, 125, 10, 167, 3, 232, 228, 203, 125, 10, - 167, 3, 203, 133, 25, 231, 225, 108, 236, 224, 10, 167, 3, 203, 133, 25, - 231, 225, 108, 216, 46, 10, 167, 3, 91, 247, 228, 10, 167, 3, 105, 247, - 228, 10, 167, 3, 115, 247, 228, 10, 167, 3, 115, 247, 229, 195, 168, 20, - 10, 167, 3, 232, 130, 247, 228, 10, 167, 3, 232, 228, 247, 228, 10, 167, - 3, 115, 195, 168, 232, 148, 234, 13, 10, 167, 3, 115, 195, 168, 232, 148, - 234, 10, 10, 167, 3, 232, 130, 195, 168, 232, 148, 219, 118, 10, 167, 3, - 232, 130, 195, 168, 232, 148, 219, 116, 10, 167, 3, 232, 130, 195, 168, - 232, 148, 219, 119, 65, 10, 167, 3, 232, 130, 195, 168, 232, 148, 219, - 119, 250, 122, 10, 167, 3, 202, 137, 195, 168, 232, 148, 250, 207, 10, - 167, 3, 203, 248, 195, 168, 232, 148, 223, 173, 10, 167, 3, 203, 248, - 195, 168, 232, 148, 223, 175, 65, 10, 167, 3, 203, 248, 195, 168, 232, - 148, 223, 175, 250, 122, 10, 167, 3, 234, 166, 195, 168, 232, 148, 196, - 33, 10, 167, 3, 234, 166, 195, 168, 232, 148, 196, 32, 10, 167, 3, 213, - 177, 195, 168, 232, 148, 223, 190, 10, 167, 3, 213, 177, 195, 168, 232, - 148, 223, 189, 10, 167, 3, 213, 177, 195, 168, 232, 148, 223, 188, 10, - 167, 3, 213, 177, 195, 168, 232, 148, 223, 191, 65, 10, 167, 3, 105, 250, - 212, 199, 84, 10, 167, 3, 115, 250, 212, 199, 84, 10, 167, 3, 232, 130, - 250, 212, 199, 84, 10, 167, 3, 232, 228, 250, 212, 199, 84, 10, 167, 3, - 202, 137, 250, 212, 199, 84, 10, 167, 3, 91, 249, 101, 10, 167, 3, 105, - 249, 101, 10, 167, 3, 115, 249, 101, 10, 167, 3, 232, 130, 249, 101, 10, - 167, 3, 232, 130, 249, 102, 195, 168, 20, 10, 167, 3, 232, 228, 249, 101, - 10, 167, 3, 232, 228, 249, 102, 195, 168, 20, 10, 167, 3, 213, 190, 10, - 167, 3, 213, 191, 10, 167, 3, 91, 234, 9, 10, 167, 3, 105, 234, 9, 10, - 167, 3, 91, 199, 1, 236, 224, 10, 167, 3, 105, 198, 254, 236, 224, 10, - 167, 3, 232, 228, 202, 124, 236, 224, 10, 167, 3, 91, 199, 1, 195, 168, - 232, 148, 65, 10, 167, 3, 105, 198, 254, 195, 168, 232, 148, 65, 10, 167, - 3, 91, 234, 162, 250, 211, 10, 167, 3, 91, 208, 25, 250, 211, 10, 167, 3, - 39, 250, 198, 91, 202, 125, 10, 167, 3, 39, 250, 198, 91, 208, 24, 10, - 167, 3, 91, 208, 25, 230, 151, 10, 167, 3, 91, 132, 230, 151, 10, 167, 3, - 234, 140, 91, 199, 0, 10, 167, 3, 234, 140, 105, 198, 253, 10, 167, 3, - 234, 140, 232, 137, 10, 167, 3, 234, 140, 233, 17, 10, 167, 3, 232, 130, - 126, 195, 168, 20, 10, 167, 3, 232, 228, 126, 195, 168, 20, 10, 167, 3, - 202, 137, 126, 195, 168, 20, 10, 167, 3, 203, 248, 126, 195, 168, 20, 10, - 167, 3, 234, 166, 126, 195, 168, 20, 10, 167, 3, 213, 177, 126, 195, 168, - 20, 10, 208, 154, 3, 39, 250, 198, 193, 23, 236, 207, 10, 208, 154, 3, - 81, 242, 85, 10, 208, 154, 3, 237, 41, 242, 85, 10, 208, 154, 3, 237, 41, - 197, 237, 10, 208, 154, 3, 237, 41, 208, 31, 10, 3, 252, 158, 25, 212, - 133, 199, 84, 10, 3, 252, 158, 25, 203, 4, 10, 3, 252, 45, 25, 234, 11, - 10, 3, 249, 138, 25, 236, 225, 199, 84, 10, 3, 249, 124, 25, 252, 74, 10, - 3, 249, 124, 25, 213, 221, 10, 3, 249, 124, 25, 191, 123, 10, 3, 248, 13, - 139, 248, 13, 25, 214, 255, 10, 3, 238, 35, 25, 199, 209, 10, 3, 238, 25, - 25, 220, 199, 10, 3, 237, 2, 25, 223, 203, 10, 3, 237, 2, 25, 126, 126, - 66, 10, 3, 237, 0, 25, 196, 158, 10, 3, 235, 85, 25, 251, 93, 10, 3, 235, - 85, 25, 250, 211, 10, 3, 235, 85, 25, 250, 212, 250, 185, 219, 226, 10, - 3, 235, 85, 25, 236, 245, 10, 3, 235, 85, 25, 235, 77, 10, 3, 235, 85, - 25, 234, 30, 10, 3, 235, 85, 25, 231, 242, 10, 3, 235, 85, 25, 231, 55, - 10, 3, 235, 85, 25, 231, 35, 230, 157, 10, 3, 235, 85, 25, 231, 25, 10, - 3, 235, 85, 25, 140, 10, 3, 235, 85, 25, 229, 160, 10, 3, 235, 85, 25, - 223, 204, 230, 157, 10, 3, 235, 85, 25, 222, 56, 10, 3, 235, 85, 25, 220, - 199, 10, 3, 235, 85, 25, 220, 192, 10, 3, 235, 85, 25, 220, 193, 108, - 222, 56, 10, 3, 235, 85, 25, 220, 94, 10, 3, 235, 85, 25, 220, 37, 10, 3, - 235, 85, 25, 220, 38, 25, 220, 199, 10, 3, 235, 85, 25, 218, 216, 108, - 231, 25, 10, 3, 235, 85, 25, 217, 41, 10, 3, 235, 85, 25, 216, 175, 10, - 3, 235, 85, 25, 216, 102, 10, 3, 235, 85, 25, 213, 221, 10, 3, 235, 85, - 25, 209, 187, 10, 3, 235, 85, 25, 203, 114, 10, 3, 235, 85, 25, 202, 224, - 230, 157, 10, 3, 234, 221, 25, 220, 199, 10, 3, 234, 221, 25, 210, 139, - 10, 3, 234, 31, 192, 235, 10, 3, 234, 12, 238, 212, 234, 11, 10, 3, 233, - 183, 203, 96, 25, 250, 211, 10, 3, 233, 183, 203, 96, 25, 229, 160, 10, - 3, 233, 183, 203, 96, 25, 223, 204, 230, 157, 10, 3, 233, 183, 203, 96, - 25, 173, 10, 3, 233, 183, 203, 96, 25, 220, 39, 10, 3, 233, 183, 203, 96, - 25, 216, 234, 10, 3, 233, 183, 203, 96, 25, 216, 175, 10, 3, 233, 183, - 203, 96, 25, 201, 5, 10, 3, 233, 183, 25, 201, 5, 10, 3, 231, 241, 25, - 249, 123, 10, 3, 231, 241, 25, 237, 2, 230, 157, 10, 3, 231, 241, 25, - 235, 85, 25, 223, 204, 230, 157, 10, 3, 231, 241, 25, 235, 85, 25, 222, - 56, 10, 3, 231, 241, 25, 234, 33, 10, 3, 231, 241, 25, 231, 242, 10, 3, - 231, 241, 25, 231, 203, 108, 237, 46, 10, 3, 231, 241, 25, 231, 203, 108, - 214, 123, 10, 3, 231, 241, 25, 230, 107, 108, 65, 10, 3, 231, 241, 25, - 220, 193, 108, 222, 56, 10, 3, 231, 241, 25, 220, 37, 10, 3, 231, 241, - 25, 220, 38, 25, 220, 199, 10, 3, 231, 241, 25, 218, 215, 10, 3, 231, - 241, 25, 215, 103, 10, 3, 231, 241, 25, 214, 123, 10, 3, 231, 241, 25, - 214, 124, 108, 234, 220, 10, 3, 231, 241, 25, 214, 124, 108, 231, 55, 10, - 3, 231, 241, 25, 203, 74, 10, 3, 231, 241, 25, 191, 13, 10, 3, 231, 236, - 206, 123, 213, 172, 237, 126, 10, 3, 231, 137, 25, 66, 10, 3, 231, 26, - 25, 231, 26, 238, 212, 231, 25, 10, 3, 230, 180, 25, 223, 204, 230, 157, - 10, 3, 230, 171, 108, 231, 26, 25, 199, 209, 10, 3, 230, 107, 199, 85, - 230, 157, 10, 3, 229, 161, 25, 250, 212, 139, 229, 161, 25, 250, 211, 10, - 3, 222, 144, 25, 248, 12, 10, 3, 222, 144, 25, 155, 10, 3, 222, 144, 25, - 126, 126, 66, 10, 3, 222, 144, 25, 196, 62, 10, 3, 220, 107, 25, 190, - 252, 139, 190, 251, 10, 3, 220, 95, 10, 3, 220, 93, 10, 3, 220, 92, 10, - 3, 220, 91, 10, 3, 220, 90, 10, 3, 220, 89, 10, 3, 220, 88, 10, 3, 220, - 87, 139, 220, 87, 230, 157, 10, 3, 220, 86, 10, 3, 220, 85, 139, 220, 84, - 10, 3, 220, 83, 10, 3, 220, 82, 10, 3, 220, 81, 10, 3, 220, 80, 10, 3, - 220, 79, 10, 3, 220, 78, 10, 3, 220, 77, 10, 3, 220, 76, 10, 3, 220, 75, - 10, 3, 220, 74, 10, 3, 220, 73, 10, 3, 220, 72, 10, 3, 220, 71, 10, 3, - 220, 70, 10, 3, 220, 69, 10, 3, 220, 68, 10, 3, 220, 67, 10, 3, 220, 66, - 10, 3, 220, 64, 10, 3, 220, 65, 25, 230, 181, 10, 3, 220, 65, 25, 223, - 203, 10, 3, 220, 65, 25, 210, 140, 108, 218, 224, 10, 3, 220, 65, 25, - 210, 140, 108, 210, 140, 108, 218, 224, 10, 3, 220, 65, 25, 196, 170, - 108, 249, 155, 10, 3, 220, 63, 10, 3, 220, 62, 10, 3, 220, 61, 10, 3, - 220, 60, 10, 3, 220, 59, 10, 3, 220, 58, 10, 3, 220, 57, 10, 3, 220, 56, - 10, 3, 220, 55, 10, 3, 220, 54, 10, 3, 220, 52, 10, 3, 220, 53, 25, 250, - 211, 10, 3, 220, 53, 25, 249, 137, 10, 3, 220, 53, 25, 235, 76, 230, 158, - 230, 157, 10, 3, 220, 53, 25, 220, 225, 10, 3, 220, 53, 25, 173, 10, 3, - 220, 53, 25, 199, 179, 10, 3, 220, 53, 25, 199, 145, 10, 3, 220, 53, 25, - 196, 169, 10, 3, 220, 53, 25, 196, 158, 10, 3, 220, 53, 25, 196, 49, 10, - 3, 220, 51, 10, 3, 220, 49, 10, 3, 220, 50, 25, 235, 89, 10, 3, 220, 50, - 25, 231, 242, 10, 3, 220, 50, 25, 223, 203, 10, 3, 220, 50, 25, 223, 204, - 230, 157, 10, 3, 220, 50, 25, 213, 221, 10, 3, 220, 50, 25, 210, 140, - 108, 210, 140, 108, 218, 224, 10, 3, 220, 50, 25, 203, 99, 108, 221, 168, - 10, 3, 220, 50, 25, 196, 158, 10, 3, 220, 50, 25, 196, 49, 10, 3, 220, - 47, 10, 3, 220, 46, 10, 3, 218, 226, 230, 158, 25, 250, 211, 10, 3, 218, - 226, 25, 236, 224, 10, 3, 218, 226, 25, 230, 75, 10, 3, 218, 226, 25, - 210, 139, 10, 3, 218, 226, 25, 210, 140, 108, 210, 140, 108, 218, 224, - 10, 3, 218, 226, 25, 199, 209, 10, 3, 216, 103, 108, 191, 122, 10, 3, - 215, 104, 139, 215, 104, 25, 231, 242, 10, 3, 215, 104, 139, 215, 104, - 25, 222, 98, 10, 3, 213, 179, 25, 237, 2, 230, 157, 10, 3, 213, 179, 25, - 231, 25, 10, 3, 213, 179, 25, 230, 162, 10, 3, 213, 179, 25, 229, 160, - 10, 3, 213, 179, 25, 221, 244, 10, 3, 213, 179, 25, 220, 90, 10, 3, 213, - 179, 25, 217, 41, 10, 3, 213, 179, 25, 210, 140, 108, 210, 139, 10, 3, - 213, 179, 25, 66, 10, 3, 213, 179, 25, 126, 108, 66, 10, 3, 213, 179, 25, - 196, 49, 10, 3, 205, 187, 230, 158, 25, 140, 10, 3, 205, 187, 25, 234, - 105, 10, 3, 205, 187, 25, 203, 115, 250, 185, 219, 226, 10, 3, 205, 187, - 25, 199, 209, 10, 3, 203, 163, 199, 84, 10, 3, 203, 115, 139, 203, 114, - 10, 3, 203, 115, 108, 228, 153, 10, 3, 203, 115, 108, 214, 231, 10, 3, - 203, 115, 108, 205, 128, 10, 3, 203, 5, 108, 235, 85, 25, 213, 221, 10, - 3, 203, 5, 108, 234, 221, 25, 251, 134, 10, 3, 202, 224, 25, 199, 209, - 10, 3, 199, 210, 108, 205, 186, 10, 3, 197, 77, 25, 231, 211, 199, 84, - 10, 3, 197, 77, 25, 115, 236, 224, 10, 3, 196, 61, 223, 94, 10, 3, 196, - 61, 25, 196, 158, 10, 3, 196, 52, 25, 237, 226, 10, 3, 196, 52, 25, 220, - 48, 10, 3, 196, 52, 25, 218, 224, 10, 3, 191, 122, 10, 3, 190, 252, 139, - 190, 252, 108, 205, 128, 10, 3, 190, 250, 25, 115, 236, 225, 199, 84, - 238, 136, 3, 242, 200, 238, 136, 3, 242, 199, 238, 136, 3, 242, 198, 238, - 136, 3, 242, 197, 238, 136, 3, 242, 196, 238, 136, 3, 242, 195, 238, 136, - 3, 242, 194, 238, 136, 3, 242, 193, 238, 136, 3, 242, 192, 238, 136, 3, - 242, 191, 238, 136, 3, 242, 190, 238, 136, 3, 242, 189, 238, 136, 3, 242, - 188, 238, 136, 3, 242, 187, 238, 136, 3, 242, 186, 238, 136, 3, 242, 185, - 238, 136, 3, 242, 184, 238, 136, 3, 242, 183, 238, 136, 3, 242, 182, 238, - 136, 3, 242, 181, 238, 136, 3, 242, 180, 238, 136, 3, 242, 179, 238, 136, - 3, 242, 178, 238, 136, 3, 242, 177, 238, 136, 3, 242, 176, 238, 136, 3, - 242, 175, 238, 136, 3, 242, 174, 238, 136, 3, 242, 173, 238, 136, 3, 242, - 172, 238, 136, 3, 242, 171, 238, 136, 3, 242, 170, 238, 136, 3, 242, 169, - 238, 136, 3, 242, 168, 238, 136, 3, 242, 167, 238, 136, 3, 242, 166, 238, - 136, 3, 242, 165, 238, 136, 3, 242, 164, 238, 136, 3, 242, 163, 238, 136, - 3, 242, 162, 238, 136, 3, 242, 161, 238, 136, 3, 242, 160, 238, 136, 3, - 242, 159, 238, 136, 3, 242, 158, 238, 136, 3, 242, 157, 238, 136, 3, 242, - 156, 238, 136, 3, 242, 155, 238, 136, 3, 242, 154, 238, 136, 3, 242, 153, - 238, 136, 3, 242, 152, 238, 136, 3, 242, 151, 238, 136, 3, 242, 150, 238, - 136, 3, 242, 149, 238, 136, 3, 242, 148, 238, 136, 3, 242, 147, 238, 136, - 3, 242, 146, 238, 136, 3, 242, 145, 238, 136, 3, 242, 144, 238, 136, 3, - 242, 143, 238, 136, 3, 242, 142, 238, 136, 3, 242, 141, 238, 136, 3, 242, - 140, 238, 136, 3, 242, 139, 238, 136, 3, 242, 138, 238, 136, 3, 242, 137, - 238, 136, 3, 242, 136, 238, 136, 3, 242, 135, 238, 136, 3, 242, 134, 238, - 136, 3, 242, 133, 238, 136, 3, 242, 132, 238, 136, 3, 242, 131, 238, 136, - 3, 242, 130, 238, 136, 3, 242, 129, 238, 136, 3, 242, 128, 238, 136, 3, - 242, 127, 238, 136, 3, 242, 126, 238, 136, 3, 242, 125, 238, 136, 3, 242, - 124, 238, 136, 3, 242, 123, 238, 136, 3, 242, 122, 238, 136, 3, 242, 121, - 238, 136, 3, 242, 120, 238, 136, 3, 242, 119, 238, 136, 3, 242, 118, 238, - 136, 3, 242, 117, 238, 136, 3, 242, 116, 238, 136, 3, 242, 115, 238, 136, - 3, 242, 114, 238, 136, 3, 242, 113, 238, 136, 3, 242, 112, 238, 136, 3, - 242, 111, 238, 136, 3, 242, 110, 238, 136, 3, 242, 109, 238, 136, 3, 242, - 108, 238, 136, 3, 242, 107, 238, 136, 3, 242, 106, 238, 136, 3, 242, 105, - 238, 136, 3, 242, 104, 238, 136, 3, 242, 103, 238, 136, 3, 242, 102, 14, - 7, 255, 201, 14, 7, 255, 200, 14, 7, 255, 199, 14, 7, 255, 198, 14, 7, - 255, 197, 14, 7, 255, 196, 14, 7, 255, 195, 14, 7, 255, 194, 14, 7, 255, - 193, 14, 7, 255, 192, 14, 7, 255, 191, 14, 7, 255, 190, 14, 7, 255, 189, - 14, 7, 255, 187, 14, 7, 255, 186, 14, 7, 255, 185, 14, 7, 255, 184, 14, - 7, 255, 183, 14, 7, 255, 182, 14, 7, 255, 181, 14, 7, 255, 180, 14, 7, - 255, 179, 14, 7, 255, 178, 14, 7, 255, 177, 14, 7, 255, 176, 14, 7, 255, - 175, 14, 7, 255, 174, 14, 7, 255, 173, 14, 7, 255, 172, 14, 7, 255, 171, - 14, 7, 255, 170, 14, 7, 255, 168, 14, 7, 255, 167, 14, 7, 255, 165, 14, - 7, 255, 164, 14, 7, 255, 163, 14, 7, 255, 162, 14, 7, 255, 161, 14, 7, - 255, 160, 14, 7, 255, 159, 14, 7, 255, 158, 14, 7, 255, 157, 14, 7, 255, - 156, 14, 7, 255, 155, 14, 7, 255, 154, 14, 7, 255, 152, 14, 7, 255, 151, - 14, 7, 255, 150, 14, 7, 255, 148, 14, 7, 255, 147, 14, 7, 255, 146, 14, - 7, 255, 145, 14, 7, 255, 144, 14, 7, 255, 143, 14, 7, 255, 142, 14, 7, - 255, 141, 14, 7, 255, 138, 14, 7, 255, 137, 14, 7, 255, 136, 14, 7, 255, - 135, 14, 7, 255, 134, 14, 7, 255, 133, 14, 7, 255, 132, 14, 7, 255, 131, - 14, 7, 255, 130, 14, 7, 255, 129, 14, 7, 255, 128, 14, 7, 255, 127, 14, - 7, 255, 126, 14, 7, 255, 125, 14, 7, 255, 124, 14, 7, 255, 123, 14, 7, - 255, 122, 14, 7, 255, 121, 14, 7, 255, 120, 14, 7, 255, 119, 14, 7, 255, - 115, 14, 7, 255, 114, 14, 7, 255, 113, 14, 7, 255, 112, 14, 7, 250, 120, - 14, 7, 250, 118, 14, 7, 250, 116, 14, 7, 250, 114, 14, 7, 250, 112, 14, - 7, 250, 111, 14, 7, 250, 109, 14, 7, 250, 107, 14, 7, 250, 105, 14, 7, - 250, 103, 14, 7, 247, 191, 14, 7, 247, 190, 14, 7, 247, 189, 14, 7, 247, - 188, 14, 7, 247, 187, 14, 7, 247, 186, 14, 7, 247, 185, 14, 7, 247, 184, - 14, 7, 247, 183, 14, 7, 247, 182, 14, 7, 247, 181, 14, 7, 247, 180, 14, - 7, 247, 179, 14, 7, 247, 178, 14, 7, 247, 177, 14, 7, 247, 176, 14, 7, - 247, 175, 14, 7, 247, 174, 14, 7, 247, 173, 14, 7, 247, 172, 14, 7, 247, - 171, 14, 7, 247, 170, 14, 7, 247, 169, 14, 7, 247, 168, 14, 7, 247, 167, - 14, 7, 247, 166, 14, 7, 247, 165, 14, 7, 247, 164, 14, 7, 238, 128, 14, - 7, 238, 127, 14, 7, 238, 126, 14, 7, 238, 125, 14, 7, 238, 124, 14, 7, - 238, 123, 14, 7, 238, 122, 14, 7, 238, 121, 14, 7, 238, 120, 14, 7, 238, - 119, 14, 7, 238, 118, 14, 7, 238, 117, 14, 7, 238, 116, 14, 7, 238, 115, - 14, 7, 238, 114, 14, 7, 238, 113, 14, 7, 238, 112, 14, 7, 238, 111, 14, - 7, 238, 110, 14, 7, 238, 109, 14, 7, 238, 108, 14, 7, 238, 107, 14, 7, - 238, 106, 14, 7, 238, 105, 14, 7, 238, 104, 14, 7, 238, 103, 14, 7, 238, - 102, 14, 7, 238, 101, 14, 7, 238, 100, 14, 7, 238, 99, 14, 7, 238, 98, - 14, 7, 238, 97, 14, 7, 238, 96, 14, 7, 238, 95, 14, 7, 238, 94, 14, 7, - 238, 93, 14, 7, 238, 92, 14, 7, 238, 91, 14, 7, 238, 90, 14, 7, 238, 89, - 14, 7, 238, 88, 14, 7, 238, 87, 14, 7, 238, 86, 14, 7, 238, 85, 14, 7, - 238, 84, 14, 7, 238, 83, 14, 7, 238, 82, 14, 7, 238, 81, 14, 7, 238, 80, - 14, 7, 238, 79, 14, 7, 238, 78, 14, 7, 238, 77, 14, 7, 238, 76, 14, 7, - 238, 75, 14, 7, 238, 74, 14, 7, 238, 73, 14, 7, 238, 72, 14, 7, 238, 71, - 14, 7, 238, 70, 14, 7, 238, 69, 14, 7, 238, 68, 14, 7, 238, 67, 14, 7, - 238, 66, 14, 7, 238, 65, 14, 7, 238, 64, 14, 7, 238, 63, 14, 7, 238, 62, - 14, 7, 238, 61, 14, 7, 238, 60, 14, 7, 238, 59, 14, 7, 238, 58, 14, 7, - 238, 57, 14, 7, 238, 56, 14, 7, 238, 55, 14, 7, 238, 54, 14, 7, 238, 53, - 14, 7, 238, 52, 14, 7, 238, 51, 14, 7, 238, 50, 14, 7, 238, 49, 14, 7, - 238, 48, 14, 7, 238, 47, 14, 7, 238, 46, 14, 7, 238, 45, 14, 7, 238, 44, - 14, 7, 238, 43, 14, 7, 238, 42, 14, 7, 238, 41, 14, 7, 238, 40, 14, 7, - 238, 39, 14, 7, 238, 38, 14, 7, 238, 37, 14, 7, 235, 9, 14, 7, 235, 8, - 14, 7, 235, 7, 14, 7, 235, 6, 14, 7, 235, 5, 14, 7, 235, 4, 14, 7, 235, - 3, 14, 7, 235, 2, 14, 7, 235, 1, 14, 7, 235, 0, 14, 7, 234, 255, 14, 7, - 234, 254, 14, 7, 234, 253, 14, 7, 234, 252, 14, 7, 234, 251, 14, 7, 234, - 250, 14, 7, 234, 249, 14, 7, 234, 248, 14, 7, 234, 247, 14, 7, 234, 246, - 14, 7, 234, 245, 14, 7, 234, 244, 14, 7, 234, 243, 14, 7, 234, 242, 14, - 7, 234, 241, 14, 7, 234, 240, 14, 7, 234, 239, 14, 7, 234, 238, 14, 7, - 234, 237, 14, 7, 234, 236, 14, 7, 234, 235, 14, 7, 234, 234, 14, 7, 234, - 233, 14, 7, 234, 232, 14, 7, 234, 231, 14, 7, 234, 230, 14, 7, 234, 229, - 14, 7, 234, 228, 14, 7, 234, 227, 14, 7, 234, 226, 14, 7, 234, 225, 14, - 7, 234, 224, 14, 7, 234, 223, 14, 7, 234, 222, 14, 7, 233, 176, 14, 7, - 233, 175, 14, 7, 233, 174, 14, 7, 233, 173, 14, 7, 233, 172, 14, 7, 233, - 171, 14, 7, 233, 170, 14, 7, 233, 169, 14, 7, 233, 168, 14, 7, 233, 167, - 14, 7, 233, 166, 14, 7, 233, 165, 14, 7, 233, 164, 14, 7, 233, 163, 14, - 7, 233, 162, 14, 7, 233, 161, 14, 7, 233, 160, 14, 7, 233, 159, 14, 7, - 233, 158, 14, 7, 233, 157, 14, 7, 233, 156, 14, 7, 233, 155, 14, 7, 233, - 154, 14, 7, 233, 153, 14, 7, 233, 152, 14, 7, 233, 151, 14, 7, 233, 150, - 14, 7, 233, 149, 14, 7, 233, 148, 14, 7, 233, 147, 14, 7, 233, 146, 14, - 7, 233, 145, 14, 7, 233, 144, 14, 7, 233, 143, 14, 7, 233, 142, 14, 7, - 233, 141, 14, 7, 233, 140, 14, 7, 233, 139, 14, 7, 233, 138, 14, 7, 233, - 137, 14, 7, 233, 136, 14, 7, 233, 135, 14, 7, 233, 134, 14, 7, 233, 133, - 14, 7, 233, 132, 14, 7, 233, 131, 14, 7, 233, 130, 14, 7, 233, 129, 14, - 7, 233, 128, 14, 7, 233, 127, 14, 7, 233, 126, 14, 7, 233, 125, 14, 7, - 233, 124, 14, 7, 233, 123, 14, 7, 233, 122, 14, 7, 233, 121, 14, 7, 233, - 120, 14, 7, 233, 119, 14, 7, 233, 118, 14, 7, 233, 117, 14, 7, 233, 116, - 14, 7, 233, 115, 14, 7, 233, 114, 14, 7, 233, 113, 14, 7, 233, 112, 14, - 7, 232, 52, 14, 7, 232, 51, 14, 7, 232, 50, 14, 7, 232, 49, 14, 7, 232, - 48, 14, 7, 232, 47, 14, 7, 232, 46, 14, 7, 232, 45, 14, 7, 232, 44, 14, - 7, 232, 43, 14, 7, 232, 42, 14, 7, 232, 41, 14, 7, 232, 40, 14, 7, 232, - 39, 14, 7, 232, 38, 14, 7, 232, 37, 14, 7, 232, 36, 14, 7, 232, 35, 14, - 7, 232, 34, 14, 7, 232, 33, 14, 7, 232, 32, 14, 7, 232, 31, 14, 7, 232, - 30, 14, 7, 232, 29, 14, 7, 232, 28, 14, 7, 232, 27, 14, 7, 232, 26, 14, - 7, 232, 25, 14, 7, 232, 24, 14, 7, 232, 23, 14, 7, 232, 22, 14, 7, 232, - 21, 14, 7, 232, 20, 14, 7, 232, 19, 14, 7, 232, 18, 14, 7, 232, 17, 14, - 7, 232, 16, 14, 7, 232, 15, 14, 7, 232, 14, 14, 7, 232, 13, 14, 7, 232, - 12, 14, 7, 232, 11, 14, 7, 232, 10, 14, 7, 232, 9, 14, 7, 232, 8, 14, 7, - 232, 7, 14, 7, 232, 6, 14, 7, 232, 5, 14, 7, 232, 4, 14, 7, 232, 3, 14, - 7, 232, 2, 14, 7, 232, 1, 14, 7, 232, 0, 14, 7, 231, 255, 14, 7, 231, - 254, 14, 7, 231, 253, 14, 7, 231, 252, 14, 7, 231, 251, 14, 7, 231, 250, - 14, 7, 231, 249, 14, 7, 231, 248, 14, 7, 231, 247, 14, 7, 231, 246, 14, - 7, 231, 245, 14, 7, 230, 116, 14, 7, 230, 115, 14, 7, 230, 114, 14, 7, - 230, 113, 14, 7, 230, 112, 14, 7, 230, 111, 14, 7, 230, 110, 14, 7, 230, - 109, 14, 7, 230, 108, 14, 7, 228, 56, 14, 7, 228, 55, 14, 7, 228, 54, 14, - 7, 228, 53, 14, 7, 228, 52, 14, 7, 228, 51, 14, 7, 228, 50, 14, 7, 228, - 49, 14, 7, 228, 48, 14, 7, 228, 47, 14, 7, 228, 46, 14, 7, 228, 45, 14, - 7, 228, 44, 14, 7, 228, 43, 14, 7, 228, 42, 14, 7, 228, 41, 14, 7, 228, - 40, 14, 7, 228, 39, 14, 7, 228, 38, 14, 7, 222, 153, 14, 7, 222, 152, 14, - 7, 222, 151, 14, 7, 222, 150, 14, 7, 222, 149, 14, 7, 222, 148, 14, 7, - 222, 147, 14, 7, 222, 146, 14, 7, 220, 142, 14, 7, 220, 141, 14, 7, 220, - 140, 14, 7, 220, 139, 14, 7, 220, 138, 14, 7, 220, 137, 14, 7, 220, 136, - 14, 7, 220, 135, 14, 7, 220, 134, 14, 7, 220, 133, 14, 7, 218, 168, 14, - 7, 218, 167, 14, 7, 218, 166, 14, 7, 218, 164, 14, 7, 218, 162, 14, 7, - 218, 161, 14, 7, 218, 159, 14, 7, 218, 157, 14, 7, 218, 155, 14, 7, 218, - 153, 14, 7, 218, 151, 14, 7, 218, 149, 14, 7, 218, 147, 14, 7, 218, 146, - 14, 7, 218, 144, 14, 7, 218, 142, 14, 7, 218, 141, 14, 7, 218, 140, 14, - 7, 218, 139, 14, 7, 218, 138, 14, 7, 218, 137, 14, 7, 218, 136, 14, 7, - 218, 135, 14, 7, 218, 134, 14, 7, 218, 132, 14, 7, 218, 130, 14, 7, 218, - 128, 14, 7, 218, 127, 14, 7, 218, 125, 14, 7, 218, 124, 14, 7, 218, 122, - 14, 7, 218, 121, 14, 7, 218, 119, 14, 7, 218, 117, 14, 7, 218, 115, 14, - 7, 218, 113, 14, 7, 218, 111, 14, 7, 218, 110, 14, 7, 218, 108, 14, 7, - 218, 106, 14, 7, 218, 105, 14, 7, 218, 103, 14, 7, 218, 101, 14, 7, 218, - 99, 14, 7, 218, 97, 14, 7, 218, 96, 14, 7, 218, 94, 14, 7, 218, 92, 14, - 7, 218, 90, 14, 7, 218, 89, 14, 7, 218, 87, 14, 7, 218, 85, 14, 7, 218, - 84, 14, 7, 218, 83, 14, 7, 218, 81, 14, 7, 218, 79, 14, 7, 218, 77, 14, - 7, 218, 75, 14, 7, 218, 73, 14, 7, 218, 71, 14, 7, 218, 69, 14, 7, 218, - 68, 14, 7, 218, 66, 14, 7, 218, 64, 14, 7, 218, 62, 14, 7, 218, 60, 14, - 7, 215, 58, 14, 7, 215, 57, 14, 7, 215, 56, 14, 7, 215, 55, 14, 7, 215, - 54, 14, 7, 215, 53, 14, 7, 215, 52, 14, 7, 215, 51, 14, 7, 215, 50, 14, - 7, 215, 49, 14, 7, 215, 48, 14, 7, 215, 47, 14, 7, 215, 46, 14, 7, 215, - 45, 14, 7, 215, 44, 14, 7, 215, 43, 14, 7, 215, 42, 14, 7, 215, 41, 14, - 7, 215, 40, 14, 7, 215, 39, 14, 7, 215, 38, 14, 7, 215, 37, 14, 7, 215, - 36, 14, 7, 215, 35, 14, 7, 215, 34, 14, 7, 215, 33, 14, 7, 215, 32, 14, - 7, 215, 31, 14, 7, 215, 30, 14, 7, 215, 29, 14, 7, 215, 28, 14, 7, 215, - 27, 14, 7, 215, 26, 14, 7, 215, 25, 14, 7, 215, 24, 14, 7, 215, 23, 14, - 7, 215, 22, 14, 7, 215, 21, 14, 7, 215, 20, 14, 7, 215, 19, 14, 7, 215, - 18, 14, 7, 215, 17, 14, 7, 215, 16, 14, 7, 215, 15, 14, 7, 215, 14, 14, - 7, 215, 13, 14, 7, 215, 12, 14, 7, 215, 11, 14, 7, 215, 10, 14, 7, 213, - 106, 14, 7, 213, 105, 14, 7, 213, 104, 14, 7, 213, 103, 14, 7, 213, 102, - 14, 7, 213, 101, 14, 7, 213, 100, 14, 7, 213, 99, 14, 7, 213, 98, 14, 7, - 213, 97, 14, 7, 213, 96, 14, 7, 213, 95, 14, 7, 213, 94, 14, 7, 213, 93, - 14, 7, 213, 92, 14, 7, 213, 91, 14, 7, 213, 90, 14, 7, 213, 89, 14, 7, - 213, 88, 14, 7, 213, 87, 14, 7, 213, 86, 14, 7, 213, 85, 14, 7, 212, 176, - 14, 7, 212, 175, 14, 7, 212, 174, 14, 7, 212, 173, 14, 7, 212, 172, 14, - 7, 212, 171, 14, 7, 212, 170, 14, 7, 212, 169, 14, 7, 212, 168, 14, 7, - 212, 167, 14, 7, 212, 166, 14, 7, 212, 165, 14, 7, 212, 164, 14, 7, 212, - 163, 14, 7, 212, 162, 14, 7, 212, 161, 14, 7, 212, 160, 14, 7, 212, 159, - 14, 7, 212, 158, 14, 7, 212, 157, 14, 7, 212, 156, 14, 7, 212, 155, 14, - 7, 212, 154, 14, 7, 212, 153, 14, 7, 212, 152, 14, 7, 212, 151, 14, 7, - 212, 4, 14, 7, 212, 3, 14, 7, 212, 2, 14, 7, 212, 1, 14, 7, 212, 0, 14, - 7, 211, 255, 14, 7, 211, 254, 14, 7, 211, 253, 14, 7, 211, 252, 14, 7, - 211, 251, 14, 7, 211, 250, 14, 7, 211, 249, 14, 7, 211, 248, 14, 7, 211, - 247, 14, 7, 211, 246, 14, 7, 211, 245, 14, 7, 211, 244, 14, 7, 211, 243, - 14, 7, 211, 242, 14, 7, 211, 241, 14, 7, 211, 240, 14, 7, 211, 239, 14, - 7, 211, 238, 14, 7, 211, 237, 14, 7, 211, 236, 14, 7, 211, 235, 14, 7, - 211, 234, 14, 7, 211, 233, 14, 7, 211, 232, 14, 7, 211, 231, 14, 7, 211, - 230, 14, 7, 211, 229, 14, 7, 211, 228, 14, 7, 211, 227, 14, 7, 211, 226, - 14, 7, 211, 225, 14, 7, 211, 224, 14, 7, 211, 223, 14, 7, 211, 222, 14, - 7, 211, 221, 14, 7, 211, 220, 14, 7, 211, 219, 14, 7, 211, 218, 14, 7, - 211, 217, 14, 7, 211, 216, 14, 7, 211, 215, 14, 7, 211, 214, 14, 7, 211, - 213, 14, 7, 211, 212, 14, 7, 211, 211, 14, 7, 211, 210, 14, 7, 211, 209, - 14, 7, 211, 208, 14, 7, 211, 207, 14, 7, 211, 206, 14, 7, 211, 205, 14, - 7, 211, 204, 14, 7, 211, 203, 14, 7, 211, 202, 14, 7, 211, 201, 14, 7, - 211, 200, 14, 7, 211, 199, 14, 7, 211, 198, 14, 7, 211, 197, 14, 7, 211, - 196, 14, 7, 211, 195, 14, 7, 211, 194, 14, 7, 211, 193, 14, 7, 211, 192, - 14, 7, 211, 191, 14, 7, 211, 190, 14, 7, 211, 189, 14, 7, 211, 188, 14, - 7, 211, 187, 14, 7, 211, 186, 14, 7, 210, 237, 14, 7, 210, 236, 14, 7, - 210, 235, 14, 7, 210, 234, 14, 7, 210, 233, 14, 7, 210, 232, 14, 7, 210, - 231, 14, 7, 210, 230, 14, 7, 210, 229, 14, 7, 210, 228, 14, 7, 210, 227, - 14, 7, 210, 226, 14, 7, 210, 225, 14, 7, 208, 105, 14, 7, 208, 104, 14, - 7, 208, 103, 14, 7, 208, 102, 14, 7, 208, 101, 14, 7, 208, 100, 14, 7, - 208, 99, 14, 7, 207, 222, 14, 7, 207, 221, 14, 7, 207, 220, 14, 7, 207, - 219, 14, 7, 207, 218, 14, 7, 207, 217, 14, 7, 207, 216, 14, 7, 207, 215, - 14, 7, 207, 214, 14, 7, 207, 213, 14, 7, 207, 212, 14, 7, 207, 211, 14, - 7, 207, 210, 14, 7, 207, 209, 14, 7, 207, 208, 14, 7, 207, 207, 14, 7, - 207, 206, 14, 7, 207, 205, 14, 7, 207, 204, 14, 7, 207, 203, 14, 7, 207, - 202, 14, 7, 207, 201, 14, 7, 207, 200, 14, 7, 207, 199, 14, 7, 207, 198, - 14, 7, 207, 197, 14, 7, 207, 196, 14, 7, 207, 195, 14, 7, 207, 194, 14, - 7, 207, 193, 14, 7, 207, 192, 14, 7, 207, 191, 14, 7, 207, 190, 14, 7, - 207, 189, 14, 7, 206, 6, 14, 7, 206, 5, 14, 7, 206, 4, 14, 7, 206, 3, 14, - 7, 206, 2, 14, 7, 206, 1, 14, 7, 206, 0, 14, 7, 205, 255, 14, 7, 205, - 254, 14, 7, 205, 253, 14, 7, 205, 252, 14, 7, 205, 251, 14, 7, 205, 250, - 14, 7, 205, 249, 14, 7, 205, 248, 14, 7, 205, 247, 14, 7, 205, 246, 14, - 7, 205, 245, 14, 7, 205, 244, 14, 7, 205, 243, 14, 7, 205, 242, 14, 7, - 205, 241, 14, 7, 205, 240, 14, 7, 205, 239, 14, 7, 205, 238, 14, 7, 205, - 237, 14, 7, 205, 236, 14, 7, 205, 235, 14, 7, 205, 234, 14, 7, 205, 233, - 14, 7, 205, 232, 14, 7, 205, 231, 14, 7, 205, 230, 14, 7, 205, 229, 14, - 7, 205, 228, 14, 7, 205, 227, 14, 7, 205, 226, 14, 7, 205, 225, 14, 7, - 205, 224, 14, 7, 205, 223, 14, 7, 205, 222, 14, 7, 205, 221, 14, 7, 205, - 220, 14, 7, 205, 219, 14, 7, 205, 218, 14, 7, 205, 217, 14, 7, 205, 216, - 14, 7, 205, 215, 14, 7, 205, 214, 14, 7, 205, 213, 14, 7, 205, 212, 14, - 7, 205, 211, 14, 7, 205, 210, 14, 7, 205, 209, 14, 7, 200, 40, 14, 7, - 200, 39, 14, 7, 200, 38, 14, 7, 200, 37, 14, 7, 200, 36, 14, 7, 200, 35, - 14, 7, 200, 34, 14, 7, 200, 33, 14, 7, 200, 32, 14, 7, 200, 31, 14, 7, - 200, 30, 14, 7, 200, 29, 14, 7, 200, 28, 14, 7, 200, 27, 14, 7, 200, 26, - 14, 7, 200, 25, 14, 7, 200, 24, 14, 7, 200, 23, 14, 7, 200, 22, 14, 7, - 200, 21, 14, 7, 200, 20, 14, 7, 200, 19, 14, 7, 200, 18, 14, 7, 200, 17, - 14, 7, 200, 16, 14, 7, 200, 15, 14, 7, 200, 14, 14, 7, 200, 13, 14, 7, - 200, 12, 14, 7, 200, 11, 14, 7, 200, 10, 14, 7, 200, 9, 14, 7, 200, 8, - 14, 7, 200, 7, 14, 7, 200, 6, 14, 7, 200, 5, 14, 7, 200, 4, 14, 7, 200, - 3, 14, 7, 200, 2, 14, 7, 200, 1, 14, 7, 200, 0, 14, 7, 199, 255, 14, 7, - 199, 254, 14, 7, 199, 253, 14, 7, 196, 217, 14, 7, 196, 216, 14, 7, 196, - 215, 14, 7, 196, 214, 14, 7, 196, 213, 14, 7, 196, 212, 14, 7, 196, 211, - 14, 7, 196, 210, 14, 7, 196, 209, 14, 7, 196, 208, 14, 7, 196, 207, 14, - 7, 196, 206, 14, 7, 196, 205, 14, 7, 196, 204, 14, 7, 196, 203, 14, 7, - 196, 202, 14, 7, 196, 201, 14, 7, 196, 200, 14, 7, 196, 199, 14, 7, 196, - 198, 14, 7, 196, 197, 14, 7, 196, 196, 14, 7, 196, 195, 14, 7, 196, 194, - 14, 7, 196, 193, 14, 7, 196, 192, 14, 7, 196, 191, 14, 7, 196, 190, 14, - 7, 196, 189, 14, 7, 196, 188, 14, 7, 196, 187, 14, 7, 196, 186, 14, 7, - 196, 185, 14, 7, 196, 184, 14, 7, 196, 183, 14, 7, 196, 182, 14, 7, 196, - 181, 14, 7, 196, 180, 14, 7, 196, 179, 14, 7, 196, 178, 14, 7, 196, 177, - 14, 7, 196, 176, 14, 7, 196, 175, 14, 7, 196, 174, 14, 7, 196, 173, 14, - 7, 196, 172, 14, 7, 196, 171, 14, 7, 196, 11, 14, 7, 196, 10, 14, 7, 196, - 9, 14, 7, 196, 8, 14, 7, 196, 7, 14, 7, 196, 6, 14, 7, 196, 5, 14, 7, - 196, 4, 14, 7, 196, 3, 14, 7, 196, 2, 14, 7, 196, 1, 14, 7, 196, 0, 14, - 7, 195, 255, 14, 7, 195, 254, 14, 7, 195, 253, 14, 7, 195, 252, 14, 7, - 195, 251, 14, 7, 195, 250, 14, 7, 195, 249, 14, 7, 195, 248, 14, 7, 195, - 247, 14, 7, 195, 246, 14, 7, 195, 245, 14, 7, 195, 244, 14, 7, 195, 243, - 14, 7, 195, 242, 14, 7, 195, 240, 14, 7, 195, 239, 14, 7, 195, 238, 14, - 7, 195, 237, 14, 7, 195, 236, 14, 7, 195, 235, 14, 7, 195, 234, 14, 7, - 195, 233, 14, 7, 195, 232, 14, 7, 195, 231, 14, 7, 195, 230, 14, 7, 195, - 229, 14, 7, 195, 228, 14, 7, 195, 227, 14, 7, 195, 226, 14, 7, 195, 225, - 14, 7, 195, 224, 14, 7, 195, 223, 14, 7, 195, 222, 14, 7, 195, 221, 14, - 7, 195, 220, 14, 7, 195, 219, 14, 7, 195, 218, 14, 7, 195, 217, 14, 7, - 195, 216, 14, 7, 195, 215, 14, 7, 195, 214, 14, 7, 195, 213, 14, 7, 195, - 212, 14, 7, 195, 211, 14, 7, 195, 210, 14, 7, 195, 209, 14, 7, 195, 208, - 14, 7, 195, 207, 14, 7, 195, 206, 14, 7, 195, 205, 14, 7, 195, 204, 14, - 7, 195, 203, 14, 7, 195, 202, 14, 7, 195, 201, 14, 7, 195, 200, 14, 7, - 195, 199, 14, 7, 195, 198, 14, 7, 195, 197, 14, 7, 195, 196, 14, 7, 195, - 195, 14, 7, 195, 194, 14, 7, 195, 193, 14, 7, 195, 192, 14, 7, 195, 191, - 14, 7, 195, 190, 14, 7, 193, 223, 14, 7, 193, 222, 14, 7, 193, 221, 14, - 7, 193, 220, 14, 7, 193, 219, 14, 7, 193, 218, 14, 7, 193, 217, 14, 7, - 193, 216, 14, 7, 193, 215, 14, 7, 193, 214, 14, 7, 193, 213, 14, 7, 193, - 212, 14, 7, 193, 211, 14, 7, 193, 210, 14, 7, 193, 209, 14, 7, 193, 208, - 14, 7, 193, 207, 14, 7, 193, 206, 14, 7, 193, 205, 14, 7, 193, 204, 14, - 7, 193, 203, 14, 7, 193, 202, 14, 7, 193, 201, 14, 7, 193, 200, 14, 7, - 193, 199, 14, 7, 193, 198, 14, 7, 193, 197, 14, 7, 193, 196, 14, 7, 193, - 195, 14, 7, 193, 194, 14, 7, 193, 193, 14, 7, 193, 192, 14, 7, 192, 232, - 14, 7, 192, 231, 14, 7, 192, 230, 14, 7, 192, 229, 14, 7, 192, 228, 14, - 7, 192, 227, 14, 7, 192, 226, 14, 7, 192, 225, 14, 7, 192, 224, 14, 7, - 192, 223, 14, 7, 192, 222, 14, 7, 192, 221, 14, 7, 192, 157, 14, 7, 192, - 156, 14, 7, 192, 155, 14, 7, 192, 154, 14, 7, 192, 153, 14, 7, 192, 152, - 14, 7, 192, 151, 14, 7, 192, 150, 14, 7, 192, 149, 14, 7, 191, 165, 14, - 7, 191, 164, 14, 7, 191, 163, 14, 7, 191, 162, 14, 7, 191, 161, 14, 7, - 191, 160, 14, 7, 191, 159, 14, 7, 191, 158, 14, 7, 191, 157, 14, 7, 191, - 156, 14, 7, 191, 155, 14, 7, 191, 154, 14, 7, 191, 153, 14, 7, 191, 152, - 14, 7, 191, 151, 14, 7, 191, 150, 14, 7, 191, 149, 14, 7, 191, 148, 14, - 7, 191, 147, 14, 7, 191, 146, 14, 7, 191, 145, 14, 7, 191, 144, 14, 7, - 191, 143, 14, 7, 191, 142, 14, 7, 191, 141, 14, 7, 191, 140, 14, 7, 191, - 139, 14, 7, 191, 138, 14, 7, 191, 137, 14, 7, 191, 136, 14, 7, 191, 135, - 14, 7, 191, 134, 14, 7, 191, 133, 14, 7, 191, 132, 14, 7, 191, 131, 14, - 7, 191, 130, 14, 7, 191, 129, 14, 7, 191, 128, 14, 7, 191, 127, 14, 7, - 191, 126, 14, 7, 191, 125, 14, 7, 252, 207, 14, 7, 252, 206, 14, 7, 252, - 205, 14, 7, 252, 204, 14, 7, 252, 203, 14, 7, 252, 202, 14, 7, 252, 201, - 14, 7, 252, 200, 14, 7, 252, 199, 14, 7, 252, 198, 14, 7, 252, 197, 14, - 7, 252, 196, 14, 7, 252, 195, 14, 7, 252, 194, 14, 7, 252, 193, 14, 7, - 252, 192, 14, 7, 252, 191, 14, 7, 252, 190, 14, 7, 252, 189, 14, 7, 252, - 188, 14, 7, 252, 187, 14, 7, 252, 186, 14, 7, 252, 185, 14, 7, 252, 184, - 14, 7, 252, 183, 14, 7, 252, 182, 14, 7, 252, 181, 14, 7, 252, 180, 14, - 7, 252, 179, 14, 7, 252, 178, 14, 7, 252, 177, 14, 7, 252, 176, 14, 7, - 252, 175, 14, 7, 252, 174, 14, 7, 195, 241, 14, 7, 81, 222, 198, 14, 7, - 228, 243, 222, 198, 14, 7, 223, 122, 250, 185, 198, 54, 201, 98, 14, 7, - 223, 122, 250, 185, 248, 79, 201, 98, 14, 7, 223, 122, 250, 185, 198, 54, - 234, 96, 14, 7, 223, 122, 250, 185, 248, 79, 234, 96, 14, 7, 211, 2, 216, - 86, 14, 7, 248, 251, 205, 44, 14, 7, 234, 97, 205, 44, 14, 7, 223, 122, - 250, 185, 216, 86, 14, 7, 223, 122, 250, 185, 198, 53, 14, 7, 223, 122, - 250, 185, 248, 78, 14, 7, 248, 251, 234, 100, 14, 7, 234, 97, 234, 100, - 14, 7, 248, 251, 193, 166, 234, 100, 14, 7, 234, 97, 193, 166, 234, 100, - 14, 7, 216, 29, 228, 191, 14, 7, 232, 82, 248, 134, 14, 7, 132, 248, 134, - 14, 7, 218, 253, 56, 14, 7, 132, 218, 253, 56, 14, 7, 199, 200, 218, 253, - 56, 14, 7, 193, 78, 218, 253, 56, 14, 7, 52, 237, 251, 250, 185, 198, 54, - 201, 98, 14, 7, 52, 237, 251, 250, 185, 248, 79, 201, 98, 14, 7, 52, 237, - 251, 250, 185, 201, 98, 14, 7, 52, 237, 251, 250, 185, 198, 54, 234, 96, - 14, 7, 52, 237, 251, 250, 185, 198, 53, 14, 7, 52, 237, 251, 250, 185, - 248, 79, 201, 99, 23, 198, 54, 234, 96, 14, 7, 52, 237, 251, 250, 185, - 201, 99, 23, 198, 53, 14, 7, 52, 237, 251, 250, 185, 248, 79, 234, 96, - 14, 7, 52, 237, 251, 250, 185, 198, 54, 201, 99, 23, 248, 79, 234, 96, - 14, 7, 52, 237, 251, 250, 185, 248, 78, 14, 7, 52, 237, 251, 250, 185, - 201, 99, 23, 248, 78, 14, 7, 52, 237, 251, 250, 185, 234, 96, 14, 7, 52, - 237, 251, 250, 185, 198, 54, 23, 234, 96, 14, 7, 52, 237, 251, 250, 185, - 248, 79, 23, 234, 96, 14, 7, 52, 237, 250, 29, 7, 255, 201, 29, 7, 255, - 200, 29, 7, 255, 199, 29, 7, 255, 198, 29, 7, 255, 197, 29, 7, 255, 195, - 29, 7, 255, 192, 29, 7, 255, 191, 29, 7, 255, 190, 29, 7, 255, 189, 29, - 7, 255, 188, 29, 7, 255, 187, 29, 7, 255, 186, 29, 7, 255, 185, 29, 7, - 255, 184, 29, 7, 255, 182, 29, 7, 255, 181, 29, 7, 255, 180, 29, 7, 255, - 178, 29, 7, 255, 177, 29, 7, 255, 176, 29, 7, 255, 175, 29, 7, 255, 174, - 29, 7, 255, 173, 29, 7, 255, 172, 29, 7, 255, 171, 29, 7, 255, 170, 29, - 7, 255, 169, 29, 7, 255, 168, 29, 7, 255, 167, 29, 7, 255, 165, 29, 7, - 255, 164, 29, 7, 255, 163, 29, 7, 255, 162, 29, 7, 255, 160, 29, 7, 255, - 159, 29, 7, 255, 158, 29, 7, 255, 157, 29, 7, 255, 156, 29, 7, 255, 155, - 29, 7, 255, 154, 29, 7, 255, 153, 29, 7, 255, 152, 29, 7, 255, 150, 29, - 7, 255, 149, 29, 7, 255, 148, 29, 7, 255, 146, 29, 7, 255, 144, 29, 7, - 255, 143, 29, 7, 255, 142, 29, 7, 255, 141, 29, 7, 255, 140, 29, 7, 255, - 139, 29, 7, 255, 138, 29, 7, 255, 137, 29, 7, 255, 136, 29, 7, 255, 135, - 29, 7, 255, 134, 29, 7, 255, 133, 29, 7, 255, 132, 29, 7, 255, 131, 29, - 7, 255, 130, 29, 7, 255, 129, 29, 7, 255, 128, 29, 7, 255, 127, 29, 7, - 255, 126, 29, 7, 255, 125, 29, 7, 255, 124, 29, 7, 255, 123, 29, 7, 255, - 122, 29, 7, 255, 121, 29, 7, 255, 120, 29, 7, 255, 119, 29, 7, 255, 118, - 29, 7, 255, 117, 29, 7, 255, 116, 29, 7, 255, 115, 29, 7, 255, 114, 29, - 7, 255, 113, 29, 7, 255, 112, 29, 7, 255, 111, 29, 7, 255, 110, 29, 7, - 255, 109, 29, 7, 255, 108, 29, 7, 255, 107, 29, 7, 255, 106, 29, 7, 255, - 105, 29, 7, 255, 104, 29, 7, 255, 103, 29, 7, 255, 102, 29, 7, 255, 101, - 29, 7, 255, 100, 29, 7, 255, 99, 29, 7, 255, 98, 29, 7, 255, 97, 29, 7, - 255, 96, 29, 7, 255, 95, 29, 7, 255, 94, 29, 7, 255, 93, 29, 7, 255, 92, - 29, 7, 255, 91, 29, 7, 255, 90, 29, 7, 255, 89, 29, 7, 255, 88, 29, 7, - 255, 87, 29, 7, 255, 86, 29, 7, 255, 85, 29, 7, 255, 84, 29, 7, 255, 83, - 29, 7, 255, 82, 29, 7, 255, 81, 29, 7, 255, 80, 29, 7, 255, 78, 29, 7, - 255, 77, 29, 7, 255, 76, 29, 7, 255, 75, 29, 7, 255, 74, 29, 7, 255, 73, - 29, 7, 255, 72, 29, 7, 255, 71, 29, 7, 255, 70, 29, 7, 255, 69, 29, 7, - 255, 68, 29, 7, 255, 67, 29, 7, 255, 66, 29, 7, 255, 65, 29, 7, 255, 64, - 29, 7, 255, 63, 29, 7, 255, 62, 29, 7, 255, 61, 29, 7, 255, 60, 29, 7, - 255, 59, 29, 7, 255, 58, 29, 7, 255, 57, 29, 7, 255, 56, 29, 7, 255, 55, - 29, 7, 255, 54, 29, 7, 255, 53, 29, 7, 255, 52, 29, 7, 255, 51, 29, 7, - 255, 50, 29, 7, 255, 49, 29, 7, 255, 48, 29, 7, 255, 47, 29, 7, 255, 46, - 29, 7, 255, 45, 29, 7, 255, 43, 29, 7, 255, 42, 29, 7, 255, 41, 29, 7, - 255, 40, 29, 7, 255, 39, 29, 7, 255, 38, 29, 7, 255, 37, 29, 7, 255, 36, - 29, 7, 255, 35, 29, 7, 255, 34, 29, 7, 255, 33, 29, 7, 255, 32, 29, 7, - 255, 30, 29, 7, 255, 29, 29, 7, 255, 28, 29, 7, 255, 27, 29, 7, 255, 26, - 29, 7, 255, 25, 29, 7, 255, 24, 29, 7, 255, 23, 29, 7, 255, 22, 29, 7, - 255, 21, 29, 7, 255, 20, 29, 7, 255, 19, 29, 7, 255, 18, 29, 7, 255, 17, - 29, 7, 255, 16, 29, 7, 255, 15, 29, 7, 255, 14, 29, 7, 255, 13, 29, 7, - 255, 12, 29, 7, 255, 11, 29, 7, 255, 10, 29, 7, 255, 9, 29, 7, 255, 8, - 29, 7, 255, 7, 29, 7, 255, 6, 29, 7, 255, 5, 29, 7, 255, 4, 29, 7, 255, - 3, 29, 7, 255, 2, 29, 7, 255, 1, 29, 7, 255, 0, 29, 7, 254, 255, 29, 7, - 254, 254, 29, 7, 254, 253, 29, 7, 254, 252, 29, 7, 254, 251, 29, 7, 254, - 250, 29, 7, 254, 249, 29, 7, 254, 248, 29, 7, 254, 247, 29, 7, 254, 246, - 29, 7, 254, 245, 29, 7, 254, 244, 29, 7, 254, 243, 29, 7, 254, 242, 29, - 7, 254, 241, 29, 7, 254, 240, 29, 7, 254, 239, 29, 7, 254, 238, 29, 7, - 254, 237, 29, 7, 254, 236, 29, 7, 254, 235, 29, 7, 254, 234, 29, 7, 254, - 233, 29, 7, 254, 232, 29, 7, 254, 231, 29, 7, 254, 230, 29, 7, 254, 229, - 29, 7, 254, 228, 29, 7, 254, 227, 29, 7, 254, 226, 29, 7, 254, 225, 29, - 7, 254, 224, 29, 7, 254, 223, 29, 7, 254, 222, 29, 7, 254, 221, 29, 7, - 254, 220, 29, 7, 254, 219, 29, 7, 254, 218, 29, 7, 254, 216, 29, 7, 254, - 215, 29, 7, 254, 214, 29, 7, 254, 213, 29, 7, 254, 212, 29, 7, 254, 211, - 29, 7, 254, 210, 29, 7, 254, 209, 29, 7, 254, 208, 29, 7, 254, 207, 29, - 7, 254, 206, 29, 7, 254, 205, 29, 7, 254, 204, 29, 7, 254, 203, 29, 7, - 254, 202, 29, 7, 254, 201, 29, 7, 254, 200, 29, 7, 254, 199, 29, 7, 254, - 198, 29, 7, 254, 197, 29, 7, 254, 196, 29, 7, 254, 195, 29, 7, 254, 194, - 29, 7, 254, 193, 29, 7, 254, 192, 29, 7, 254, 191, 29, 7, 254, 190, 29, - 7, 254, 189, 29, 7, 254, 188, 29, 7, 254, 187, 29, 7, 254, 186, 29, 7, - 254, 185, 29, 7, 254, 184, 29, 7, 254, 183, 29, 7, 254, 182, 29, 7, 254, - 181, 29, 7, 254, 180, 29, 7, 254, 179, 29, 7, 254, 178, 29, 7, 254, 177, - 29, 7, 254, 176, 29, 7, 254, 175, 29, 7, 254, 174, 29, 7, 254, 173, 29, - 7, 254, 172, 29, 7, 254, 171, 29, 7, 254, 170, 29, 7, 254, 169, 29, 7, - 254, 168, 29, 7, 254, 167, 29, 7, 254, 166, 29, 7, 254, 165, 29, 7, 254, - 164, 29, 7, 254, 163, 29, 7, 254, 162, 29, 7, 254, 161, 29, 7, 254, 160, - 29, 7, 254, 159, 29, 7, 254, 158, 29, 7, 254, 157, 29, 7, 254, 156, 29, - 7, 254, 155, 29, 7, 254, 154, 29, 7, 254, 153, 29, 7, 254, 152, 29, 7, - 254, 151, 29, 7, 254, 150, 29, 7, 254, 149, 29, 7, 254, 148, 29, 7, 254, - 147, 29, 7, 254, 146, 29, 7, 254, 145, 29, 7, 254, 144, 29, 7, 254, 143, - 29, 7, 254, 142, 29, 7, 254, 141, 29, 7, 254, 140, 29, 7, 254, 139, 29, - 7, 254, 138, 29, 7, 254, 137, 29, 7, 254, 136, 29, 7, 254, 135, 29, 7, - 254, 134, 29, 7, 254, 133, 29, 7, 254, 132, 29, 7, 254, 131, 29, 7, 254, - 130, 29, 7, 254, 129, 29, 7, 254, 128, 29, 7, 254, 127, 29, 7, 254, 126, - 29, 7, 254, 125, 29, 7, 254, 124, 29, 7, 254, 123, 29, 7, 254, 122, 29, - 7, 254, 121, 29, 7, 254, 120, 29, 7, 254, 119, 29, 7, 254, 118, 29, 7, - 254, 117, 29, 7, 254, 116, 29, 7, 254, 115, 29, 7, 254, 114, 29, 7, 254, - 113, 29, 7, 254, 112, 29, 7, 254, 111, 29, 7, 254, 110, 29, 7, 254, 109, - 29, 7, 254, 108, 29, 7, 254, 107, 29, 7, 254, 106, 29, 7, 254, 104, 29, - 7, 254, 103, 29, 7, 254, 102, 29, 7, 254, 101, 29, 7, 254, 100, 29, 7, - 254, 99, 29, 7, 254, 98, 29, 7, 254, 97, 29, 7, 254, 96, 29, 7, 254, 95, - 29, 7, 254, 94, 29, 7, 254, 91, 29, 7, 254, 90, 29, 7, 254, 89, 29, 7, - 254, 88, 29, 7, 254, 84, 29, 7, 254, 83, 29, 7, 254, 82, 29, 7, 254, 81, - 29, 7, 254, 80, 29, 7, 254, 79, 29, 7, 254, 78, 29, 7, 254, 77, 29, 7, - 254, 76, 29, 7, 254, 75, 29, 7, 254, 74, 29, 7, 254, 73, 29, 7, 254, 72, - 29, 7, 254, 71, 29, 7, 254, 70, 29, 7, 254, 69, 29, 7, 254, 68, 29, 7, - 254, 67, 29, 7, 254, 66, 29, 7, 254, 64, 29, 7, 254, 63, 29, 7, 254, 62, - 29, 7, 254, 61, 29, 7, 254, 60, 29, 7, 254, 59, 29, 7, 254, 58, 29, 7, - 254, 57, 29, 7, 254, 56, 29, 7, 254, 55, 29, 7, 254, 54, 29, 7, 254, 53, - 29, 7, 254, 52, 29, 7, 254, 51, 29, 7, 254, 50, 29, 7, 254, 49, 29, 7, - 254, 48, 29, 7, 254, 47, 29, 7, 254, 46, 29, 7, 254, 45, 29, 7, 254, 44, - 29, 7, 254, 43, 29, 7, 254, 42, 29, 7, 254, 41, 29, 7, 254, 40, 29, 7, - 254, 39, 29, 7, 254, 38, 29, 7, 254, 37, 29, 7, 254, 36, 29, 7, 254, 35, - 29, 7, 254, 34, 29, 7, 254, 33, 29, 7, 254, 32, 29, 7, 254, 31, 29, 7, - 254, 30, 29, 7, 254, 29, 29, 7, 254, 28, 29, 7, 254, 27, 29, 7, 254, 26, - 29, 7, 254, 25, 29, 7, 254, 24, 29, 7, 254, 23, 29, 7, 254, 22, 29, 7, - 254, 21, 29, 7, 254, 20, 29, 7, 254, 19, 29, 7, 254, 18, 29, 7, 254, 17, - 29, 7, 254, 16, 29, 7, 254, 15, 29, 7, 254, 14, 29, 7, 254, 13, 29, 7, - 254, 12, 29, 7, 254, 11, 29, 7, 254, 10, 29, 7, 254, 9, 29, 7, 254, 8, - 29, 7, 254, 7, 29, 7, 254, 6, 29, 7, 254, 5, 29, 7, 254, 4, 29, 7, 254, - 3, 207, 188, 211, 59, 207, 2, 29, 7, 254, 2, 29, 7, 254, 1, 29, 7, 254, - 0, 29, 7, 253, 255, 29, 7, 253, 254, 29, 7, 253, 253, 29, 7, 253, 252, - 29, 7, 253, 251, 29, 7, 253, 250, 29, 7, 253, 249, 29, 7, 253, 248, 29, - 7, 253, 247, 171, 29, 7, 253, 246, 29, 7, 253, 245, 29, 7, 253, 244, 29, - 7, 253, 243, 29, 7, 253, 242, 29, 7, 253, 241, 29, 7, 253, 240, 29, 7, - 253, 238, 29, 7, 253, 236, 29, 7, 253, 234, 29, 7, 253, 232, 29, 7, 253, - 230, 29, 7, 253, 228, 29, 7, 253, 226, 29, 7, 253, 224, 29, 7, 253, 222, - 29, 7, 253, 220, 248, 251, 219, 30, 77, 29, 7, 253, 218, 234, 97, 219, - 30, 77, 29, 7, 253, 217, 29, 7, 253, 215, 29, 7, 253, 213, 29, 7, 253, - 211, 29, 7, 253, 209, 29, 7, 253, 207, 29, 7, 253, 205, 29, 7, 253, 203, - 29, 7, 253, 201, 29, 7, 253, 200, 29, 7, 253, 199, 29, 7, 253, 198, 29, - 7, 253, 197, 29, 7, 253, 196, 29, 7, 253, 195, 29, 7, 253, 194, 29, 7, - 253, 193, 29, 7, 253, 192, 29, 7, 253, 191, 29, 7, 253, 190, 29, 7, 253, - 189, 29, 7, 253, 188, 29, 7, 253, 187, 29, 7, 253, 186, 29, 7, 253, 185, - 29, 7, 253, 184, 29, 7, 253, 183, 29, 7, 253, 182, 29, 7, 253, 181, 29, - 7, 253, 180, 29, 7, 253, 179, 29, 7, 253, 178, 29, 7, 253, 177, 29, 7, - 253, 176, 29, 7, 253, 175, 29, 7, 253, 174, 29, 7, 253, 173, 29, 7, 253, - 172, 29, 7, 253, 171, 29, 7, 253, 170, 29, 7, 253, 169, 29, 7, 253, 168, - 29, 7, 253, 167, 29, 7, 253, 166, 29, 7, 253, 165, 29, 7, 253, 164, 29, - 7, 253, 163, 29, 7, 253, 162, 29, 7, 253, 161, 29, 7, 253, 160, 29, 7, - 253, 159, 29, 7, 253, 158, 29, 7, 253, 157, 29, 7, 253, 156, 29, 7, 253, - 155, 29, 7, 253, 154, 29, 7, 253, 153, 29, 7, 253, 152, 29, 7, 253, 151, - 29, 7, 253, 150, 29, 7, 253, 149, 29, 7, 253, 148, 29, 7, 253, 147, 29, - 7, 253, 146, 29, 7, 253, 145, 29, 7, 253, 144, 29, 7, 253, 143, 29, 7, - 253, 142, 29, 7, 253, 141, 29, 7, 253, 140, 29, 7, 253, 139, 29, 7, 253, - 138, 29, 7, 253, 137, 29, 7, 253, 136, 29, 7, 253, 135, 29, 7, 253, 134, - 29, 7, 253, 133, 29, 7, 253, 132, 29, 7, 253, 131, 29, 7, 253, 130, 29, - 7, 253, 129, 29, 7, 253, 128, 29, 7, 253, 127, 29, 7, 253, 126, 29, 7, - 253, 125, 29, 7, 253, 124, 29, 7, 253, 123, 29, 7, 253, 122, 29, 7, 253, - 121, 29, 7, 253, 120, 29, 7, 253, 119, 29, 7, 253, 118, 29, 7, 253, 117, - 29, 7, 253, 116, 29, 7, 253, 115, 29, 7, 253, 114, 29, 7, 253, 113, 29, - 7, 253, 112, 29, 7, 253, 111, 29, 7, 253, 110, 29, 7, 253, 109, 29, 7, - 253, 108, 29, 7, 253, 107, 29, 7, 253, 106, 29, 7, 253, 105, 29, 7, 253, - 104, 29, 7, 253, 103, 29, 7, 253, 102, 29, 7, 253, 101, 29, 7, 253, 100, - 29, 7, 253, 99, 29, 7, 253, 98, 29, 7, 253, 97, 29, 7, 253, 96, 29, 7, - 253, 95, 29, 7, 253, 94, 29, 7, 253, 93, 29, 7, 253, 92, 29, 7, 253, 91, - 26, 1, 209, 220, 214, 3, 216, 144, 26, 1, 209, 220, 231, 175, 232, 168, - 26, 1, 209, 220, 209, 42, 216, 145, 209, 129, 26, 1, 209, 220, 209, 42, - 216, 145, 209, 130, 26, 1, 209, 220, 214, 253, 216, 144, 26, 1, 209, 220, - 203, 1, 26, 1, 209, 220, 198, 124, 216, 144, 26, 1, 209, 220, 212, 49, - 216, 144, 26, 1, 209, 220, 203, 69, 210, 223, 213, 143, 26, 1, 209, 220, - 209, 42, 210, 223, 213, 144, 209, 129, 26, 1, 209, 220, 209, 42, 210, - 223, 213, 144, 209, 130, 26, 1, 209, 220, 217, 133, 26, 1, 209, 220, 197, - 95, 217, 134, 26, 1, 209, 220, 214, 64, 26, 1, 209, 220, 217, 130, 26, 1, - 209, 220, 217, 79, 26, 1, 209, 220, 215, 88, 26, 1, 209, 220, 203, 250, - 26, 1, 209, 220, 212, 189, 26, 1, 209, 220, 221, 236, 26, 1, 209, 220, - 213, 110, 26, 1, 209, 220, 200, 160, 26, 1, 209, 220, 214, 2, 26, 1, 209, - 220, 220, 17, 26, 1, 209, 220, 219, 179, 220, 190, 26, 1, 209, 220, 212, - 199, 216, 152, 26, 1, 209, 220, 217, 137, 26, 1, 209, 220, 210, 100, 26, - 1, 209, 220, 231, 74, 26, 1, 209, 220, 210, 172, 26, 1, 209, 220, 215, - 234, 214, 37, 26, 1, 209, 220, 212, 30, 216, 155, 26, 1, 209, 220, 126, - 191, 195, 214, 246, 26, 1, 209, 220, 231, 75, 26, 1, 209, 220, 212, 199, - 212, 200, 26, 1, 209, 220, 202, 140, 26, 1, 209, 220, 216, 137, 26, 1, - 209, 220, 216, 158, 26, 1, 209, 220, 215, 209, 26, 1, 209, 220, 222, 107, - 26, 1, 209, 220, 210, 223, 219, 227, 26, 1, 209, 220, 214, 165, 219, 227, - 26, 1, 209, 220, 209, 242, 26, 1, 209, 220, 217, 131, 26, 1, 209, 220, - 213, 187, 26, 1, 209, 220, 208, 146, 26, 1, 209, 220, 197, 87, 26, 1, - 209, 220, 218, 223, 26, 1, 209, 220, 202, 18, 26, 1, 209, 220, 199, 58, - 26, 1, 209, 220, 217, 128, 26, 1, 209, 220, 221, 243, 26, 1, 209, 220, - 214, 161, 26, 1, 209, 220, 220, 205, 26, 1, 209, 220, 215, 210, 26, 1, - 209, 220, 202, 253, 26, 1, 209, 220, 219, 22, 26, 1, 209, 220, 232, 241, - 26, 1, 209, 220, 206, 138, 26, 1, 209, 220, 221, 12, 26, 1, 209, 220, - 202, 14, 26, 1, 209, 220, 217, 74, 209, 175, 26, 1, 209, 220, 203, 62, - 26, 1, 209, 220, 212, 198, 26, 1, 209, 220, 203, 43, 212, 210, 191, 203, - 26, 1, 209, 220, 212, 71, 215, 230, 26, 1, 209, 220, 210, 218, 26, 1, - 209, 220, 213, 112, 26, 1, 209, 220, 196, 85, 26, 1, 209, 220, 214, 40, - 26, 1, 209, 220, 217, 127, 26, 1, 209, 220, 213, 155, 26, 1, 209, 220, - 217, 8, 26, 1, 209, 220, 212, 86, 26, 1, 209, 220, 199, 62, 26, 1, 209, - 220, 202, 9, 26, 1, 209, 220, 210, 219, 26, 1, 209, 220, 212, 214, 26, 1, - 209, 220, 217, 135, 26, 1, 209, 220, 212, 83, 26, 1, 209, 220, 222, 68, - 26, 1, 209, 220, 212, 217, 26, 1, 209, 220, 195, 148, 26, 1, 209, 220, - 218, 227, 26, 1, 209, 220, 214, 104, 26, 1, 209, 220, 214, 219, 26, 1, - 209, 220, 217, 7, 26, 1, 209, 219, 212, 212, 26, 1, 209, 219, 197, 95, - 217, 132, 26, 1, 209, 219, 202, 205, 26, 1, 209, 219, 203, 254, 197, 94, - 26, 1, 209, 219, 219, 25, 212, 195, 26, 1, 209, 219, 217, 14, 217, 136, - 26, 1, 209, 219, 221, 152, 26, 1, 209, 219, 192, 43, 26, 1, 209, 219, - 217, 9, 26, 1, 209, 219, 222, 92, 26, 1, 209, 219, 210, 46, 26, 1, 209, - 219, 192, 126, 219, 227, 26, 1, 209, 219, 220, 38, 212, 210, 212, 97, 26, - 1, 209, 219, 212, 192, 203, 88, 26, 1, 209, 219, 214, 132, 213, 158, 26, - 1, 209, 219, 231, 72, 26, 1, 209, 219, 209, 119, 26, 1, 209, 219, 197, - 95, 212, 208, 26, 1, 209, 219, 203, 93, 213, 153, 26, 1, 209, 219, 203, - 89, 26, 1, 209, 219, 216, 145, 199, 61, 26, 1, 209, 219, 216, 252, 217, - 10, 26, 1, 209, 219, 212, 84, 212, 195, 26, 1, 209, 219, 221, 232, 26, 1, - 209, 219, 231, 73, 26, 1, 209, 219, 221, 228, 26, 1, 209, 219, 220, 122, - 26, 1, 209, 219, 210, 103, 26, 1, 209, 219, 195, 77, 26, 1, 209, 219, - 214, 4, 215, 86, 26, 1, 209, 219, 214, 39, 216, 248, 26, 1, 209, 219, - 192, 254, 26, 1, 209, 219, 205, 176, 26, 1, 209, 219, 199, 240, 26, 1, - 209, 219, 216, 157, 26, 1, 209, 219, 214, 23, 26, 1, 209, 219, 214, 24, - 220, 14, 26, 1, 209, 219, 216, 147, 26, 1, 209, 219, 200, 214, 26, 1, - 209, 219, 217, 0, 26, 1, 209, 219, 215, 214, 26, 1, 209, 219, 212, 101, - 26, 1, 209, 219, 208, 150, 26, 1, 209, 219, 216, 156, 214, 41, 26, 1, - 209, 219, 233, 30, 26, 1, 209, 219, 216, 243, 26, 1, 209, 219, 233, 54, - 26, 1, 209, 219, 221, 240, 26, 1, 209, 219, 217, 162, 213, 147, 26, 1, - 209, 219, 217, 162, 213, 123, 26, 1, 209, 219, 219, 178, 26, 1, 209, 219, - 214, 47, 26, 1, 209, 219, 212, 219, 26, 1, 209, 219, 174, 26, 1, 209, - 219, 221, 135, 26, 1, 209, 219, 213, 248, 26, 1, 209, 218, 214, 3, 217, - 134, 26, 1, 209, 218, 212, 48, 26, 1, 209, 218, 191, 203, 26, 1, 209, - 218, 193, 160, 26, 1, 209, 218, 214, 40, 26, 1, 209, 218, 214, 153, 26, - 1, 209, 218, 214, 10, 26, 1, 209, 218, 231, 82, 26, 1, 209, 218, 217, 4, - 26, 1, 209, 218, 231, 182, 26, 1, 209, 218, 212, 73, 216, 23, 216, 159, - 26, 1, 209, 218, 212, 186, 216, 251, 26, 1, 209, 218, 217, 1, 26, 1, 209, - 218, 209, 125, 26, 1, 209, 218, 214, 138, 26, 1, 209, 218, 217, 12, 247, - 158, 26, 1, 209, 218, 221, 230, 26, 1, 209, 218, 231, 83, 26, 1, 209, - 218, 221, 237, 26, 1, 209, 218, 191, 226, 215, 119, 26, 1, 209, 218, 212, - 42, 26, 1, 209, 218, 216, 245, 26, 1, 209, 218, 212, 218, 26, 1, 209, - 218, 216, 251, 26, 1, 209, 218, 192, 44, 26, 1, 209, 218, 221, 20, 26, 1, - 209, 218, 222, 129, 26, 1, 209, 218, 203, 249, 26, 1, 209, 218, 214, 147, - 26, 1, 209, 218, 199, 238, 26, 1, 209, 218, 213, 127, 26, 1, 209, 218, - 198, 124, 191, 207, 26, 1, 209, 218, 200, 248, 26, 1, 209, 218, 214, 30, - 212, 97, 26, 1, 209, 218, 195, 76, 26, 1, 209, 218, 214, 222, 26, 1, 209, - 218, 217, 162, 221, 239, 26, 1, 209, 218, 212, 200, 26, 1, 209, 218, 214, - 25, 26, 1, 209, 218, 220, 18, 26, 1, 209, 218, 216, 253, 26, 1, 209, 218, - 216, 136, 26, 1, 209, 218, 212, 194, 26, 1, 209, 218, 199, 57, 26, 1, - 209, 218, 214, 27, 26, 1, 209, 218, 232, 86, 26, 1, 209, 218, 214, 152, - 26, 1, 209, 218, 212, 220, 26, 1, 209, 218, 212, 216, 26, 1, 209, 218, - 247, 242, 26, 1, 209, 218, 195, 78, 26, 1, 209, 218, 217, 2, 26, 1, 209, - 218, 206, 69, 26, 1, 209, 218, 213, 157, 26, 1, 209, 218, 220, 37, 26, 1, - 209, 218, 198, 121, 26, 1, 209, 218, 212, 202, 213, 248, 26, 1, 209, 218, - 213, 149, 26, 1, 209, 218, 221, 243, 26, 1, 209, 218, 214, 32, 26, 1, - 209, 218, 217, 127, 26, 1, 209, 218, 216, 246, 26, 1, 209, 218, 218, 227, - 26, 1, 209, 218, 220, 190, 26, 1, 209, 218, 213, 155, 26, 1, 209, 218, - 213, 248, 26, 1, 209, 218, 192, 244, 26, 1, 209, 218, 214, 28, 26, 1, - 209, 218, 212, 205, 26, 1, 209, 218, 212, 196, 26, 1, 209, 218, 220, 207, - 213, 112, 26, 1, 209, 218, 212, 203, 26, 1, 209, 218, 214, 160, 26, 1, - 209, 218, 217, 162, 212, 208, 26, 1, 209, 218, 192, 140, 26, 1, 209, 218, - 214, 159, 26, 1, 209, 218, 203, 0, 26, 1, 209, 218, 203, 252, 26, 1, 209, - 218, 216, 254, 26, 1, 209, 218, 217, 134, 26, 1, 209, 218, 217, 8, 26, 1, - 209, 218, 221, 231, 26, 1, 209, 218, 216, 255, 26, 1, 209, 218, 221, 235, - 26, 1, 209, 218, 217, 12, 209, 181, 26, 1, 209, 218, 191, 186, 26, 1, - 209, 218, 213, 145, 26, 1, 209, 218, 216, 82, 26, 1, 209, 218, 215, 151, - 26, 1, 209, 218, 203, 65, 26, 1, 209, 218, 221, 254, 219, 252, 26, 1, - 209, 218, 221, 254, 233, 67, 26, 1, 209, 218, 214, 62, 26, 1, 209, 218, - 214, 219, 26, 1, 209, 218, 219, 96, 26, 1, 209, 218, 209, 141, 26, 1, - 209, 218, 210, 36, 26, 1, 209, 218, 199, 73, 26, 1, 158, 216, 244, 26, 1, - 158, 193, 158, 26, 1, 158, 213, 143, 26, 1, 158, 216, 144, 26, 1, 158, - 213, 141, 26, 1, 158, 219, 141, 26, 1, 158, 213, 146, 26, 1, 158, 212, - 215, 26, 1, 158, 214, 46, 26, 1, 158, 212, 97, 26, 1, 158, 192, 255, 26, - 1, 158, 214, 0, 26, 1, 158, 203, 112, 26, 1, 158, 214, 11, 26, 1, 158, - 221, 238, 26, 1, 158, 199, 59, 26, 1, 158, 203, 91, 26, 1, 158, 213, 154, - 26, 1, 158, 200, 214, 26, 1, 158, 221, 243, 26, 1, 158, 192, 128, 26, 1, - 158, 220, 208, 26, 1, 158, 205, 131, 26, 1, 158, 216, 149, 26, 1, 158, - 214, 151, 26, 1, 158, 217, 97, 26, 1, 158, 216, 155, 26, 1, 158, 203, - 251, 26, 1, 158, 192, 70, 26, 1, 158, 213, 148, 26, 1, 158, 221, 234, - 216, 247, 26, 1, 158, 214, 7, 26, 1, 158, 197, 94, 26, 1, 158, 231, 92, - 26, 1, 158, 213, 253, 26, 1, 158, 233, 31, 26, 1, 158, 214, 155, 26, 1, - 158, 216, 128, 26, 1, 158, 219, 172, 26, 1, 158, 214, 137, 26, 1, 158, - 215, 229, 26, 1, 158, 216, 132, 26, 1, 158, 208, 129, 26, 1, 158, 216, - 130, 26, 1, 158, 216, 146, 26, 1, 158, 218, 210, 26, 1, 158, 212, 207, - 26, 1, 158, 217, 11, 26, 1, 158, 220, 179, 26, 1, 158, 212, 86, 26, 1, - 158, 199, 62, 26, 1, 158, 202, 9, 26, 1, 158, 191, 186, 26, 1, 158, 221, - 235, 26, 1, 158, 207, 164, 26, 1, 158, 199, 120, 26, 1, 158, 214, 8, 26, - 1, 158, 216, 151, 26, 1, 158, 212, 206, 26, 1, 158, 221, 233, 26, 1, 158, - 209, 131, 26, 1, 158, 209, 235, 26, 1, 158, 212, 59, 26, 1, 158, 219, - 178, 26, 1, 158, 214, 47, 26, 1, 158, 216, 148, 26, 1, 158, 214, 20, 26, - 1, 158, 191, 200, 26, 1, 158, 210, 139, 26, 1, 158, 191, 199, 26, 1, 158, - 214, 160, 26, 1, 158, 212, 195, 26, 1, 158, 200, 250, 26, 1, 158, 220, - 212, 26, 1, 158, 214, 36, 26, 1, 158, 214, 5, 26, 1, 158, 197, 69, 26, 1, - 158, 216, 159, 26, 1, 158, 220, 201, 26, 1, 158, 212, 204, 26, 1, 158, - 199, 60, 26, 1, 158, 217, 129, 26, 1, 158, 214, 45, 26, 1, 158, 219, 171, - 26, 1, 158, 214, 26, 26, 1, 158, 212, 209, 26, 1, 158, 213, 127, 26, 1, - 158, 231, 76, 26, 1, 158, 220, 234, 26, 1, 158, 207, 57, 211, 121, 26, 1, - 158, 199, 226, 26, 1, 158, 198, 50, 26, 1, 158, 212, 83, 26, 1, 158, 206, - 196, 26, 1, 158, 219, 229, 26, 1, 158, 216, 212, 26, 1, 158, 218, 170, - 26, 1, 158, 200, 160, 26, 1, 158, 215, 157, 26, 1, 158, 203, 77, 26, 1, - 158, 203, 87, 26, 1, 158, 220, 151, 26, 1, 158, 212, 180, 26, 1, 158, - 203, 6, 26, 1, 158, 212, 197, 26, 1, 158, 210, 51, 26, 1, 158, 213, 221, - 26, 1, 158, 203, 42, 26, 1, 158, 208, 145, 26, 1, 158, 215, 86, 26, 1, - 158, 219, 2, 26, 1, 158, 207, 57, 215, 146, 26, 1, 158, 198, 193, 26, 1, - 158, 212, 183, 26, 1, 158, 217, 12, 175, 26, 1, 158, 205, 129, 26, 1, - 158, 233, 110, 26, 1, 114, 214, 159, 26, 1, 114, 198, 57, 26, 1, 114, - 217, 1, 26, 1, 114, 220, 18, 26, 1, 114, 195, 10, 26, 1, 114, 219, 8, 26, - 1, 114, 210, 222, 26, 1, 114, 202, 22, 26, 1, 114, 207, 136, 26, 1, 114, - 212, 211, 26, 1, 114, 214, 130, 26, 1, 114, 208, 163, 26, 1, 114, 199, - 197, 26, 1, 114, 214, 13, 26, 1, 114, 221, 16, 26, 1, 114, 192, 247, 26, - 1, 114, 205, 51, 26, 1, 114, 214, 37, 26, 1, 114, 210, 219, 26, 1, 114, - 198, 59, 26, 1, 114, 220, 206, 26, 1, 114, 219, 24, 26, 1, 114, 212, 214, - 26, 1, 114, 213, 245, 26, 1, 114, 217, 135, 26, 1, 114, 214, 6, 26, 1, - 114, 213, 244, 26, 1, 114, 212, 213, 26, 1, 114, 206, 193, 26, 1, 114, - 213, 145, 26, 1, 114, 210, 48, 26, 1, 114, 205, 198, 26, 1, 114, 214, 21, - 26, 1, 114, 216, 138, 26, 1, 114, 231, 70, 26, 1, 114, 214, 9, 26, 1, - 114, 213, 156, 26, 1, 114, 217, 73, 26, 1, 114, 219, 4, 26, 1, 114, 214, - 42, 26, 1, 114, 214, 143, 26, 1, 114, 199, 225, 212, 195, 26, 1, 114, - 203, 253, 26, 1, 114, 208, 156, 26, 1, 114, 214, 163, 202, 31, 26, 1, - 114, 214, 29, 212, 97, 26, 1, 114, 192, 29, 26, 1, 114, 231, 71, 26, 1, - 114, 197, 88, 26, 1, 114, 192, 47, 26, 1, 114, 209, 75, 26, 1, 114, 197, - 75, 26, 1, 114, 221, 241, 26, 1, 114, 200, 249, 26, 1, 114, 199, 61, 26, - 1, 114, 195, 79, 26, 1, 114, 193, 100, 26, 1, 114, 220, 125, 26, 1, 114, - 208, 167, 26, 1, 114, 199, 239, 26, 1, 114, 231, 91, 26, 1, 114, 214, 52, - 26, 1, 114, 203, 90, 26, 1, 114, 216, 133, 26, 1, 114, 217, 5, 26, 1, - 114, 212, 46, 26, 1, 114, 213, 108, 26, 1, 114, 231, 178, 26, 1, 114, - 197, 76, 26, 1, 114, 220, 217, 26, 1, 114, 192, 104, 26, 1, 114, 212, 84, - 242, 239, 26, 1, 114, 192, 18, 26, 1, 114, 216, 150, 26, 1, 114, 214, - 148, 26, 1, 114, 209, 176, 26, 1, 114, 191, 206, 26, 1, 114, 219, 173, - 26, 1, 114, 232, 86, 26, 1, 114, 231, 177, 26, 1, 114, 213, 255, 26, 1, - 114, 221, 243, 26, 1, 114, 217, 138, 26, 1, 114, 214, 12, 26, 1, 114, - 231, 77, 26, 1, 114, 233, 111, 26, 1, 114, 212, 184, 26, 1, 114, 209, - 236, 26, 1, 114, 192, 45, 26, 1, 114, 214, 38, 26, 1, 114, 212, 84, 248, - 211, 26, 1, 114, 212, 26, 26, 1, 114, 209, 37, 26, 1, 114, 216, 82, 26, - 1, 114, 232, 84, 26, 1, 114, 214, 246, 26, 1, 114, 215, 151, 26, 1, 114, - 231, 76, 26, 1, 114, 232, 89, 68, 26, 1, 114, 215, 87, 26, 1, 114, 208, - 162, 26, 1, 114, 214, 1, 26, 1, 114, 220, 190, 26, 1, 114, 209, 173, 26, - 1, 114, 212, 198, 26, 1, 114, 192, 46, 26, 1, 114, 214, 22, 26, 1, 114, - 210, 223, 210, 21, 26, 1, 114, 232, 89, 247, 140, 26, 1, 114, 232, 169, - 26, 1, 114, 213, 150, 26, 1, 114, 65, 26, 1, 114, 198, 50, 26, 1, 114, - 74, 26, 1, 114, 68, 26, 1, 114, 220, 16, 26, 1, 114, 210, 223, 209, 84, - 26, 1, 114, 199, 245, 26, 1, 114, 199, 180, 26, 1, 114, 214, 163, 215, - 73, 228, 174, 26, 1, 114, 203, 65, 26, 1, 114, 192, 42, 26, 1, 114, 213, - 238, 26, 1, 114, 191, 211, 26, 1, 114, 191, 244, 200, 136, 26, 1, 114, - 191, 244, 238, 244, 26, 1, 114, 191, 194, 26, 1, 114, 191, 202, 26, 1, - 114, 221, 229, 26, 1, 114, 209, 234, 26, 1, 114, 213, 151, 234, 51, 26, - 1, 114, 208, 158, 26, 1, 114, 192, 253, 26, 1, 114, 233, 54, 26, 1, 114, - 195, 148, 26, 1, 114, 218, 227, 26, 1, 114, 216, 102, 26, 1, 114, 207, - 21, 26, 1, 114, 207, 165, 26, 1, 114, 213, 237, 26, 1, 114, 214, 70, 26, - 1, 114, 203, 57, 26, 1, 114, 203, 42, 26, 1, 114, 232, 89, 207, 60, 26, - 1, 114, 181, 26, 1, 114, 209, 187, 26, 1, 114, 219, 2, 26, 1, 114, 221, - 69, 26, 1, 114, 216, 188, 26, 1, 114, 174, 26, 1, 114, 217, 70, 26, 1, - 114, 199, 63, 26, 1, 114, 221, 168, 26, 1, 114, 215, 233, 26, 1, 114, - 199, 95, 26, 1, 114, 233, 78, 26, 1, 114, 231, 64, 26, 1, 209, 217, 155, - 26, 1, 209, 217, 66, 26, 1, 209, 217, 220, 234, 26, 1, 209, 217, 234, - 190, 26, 1, 209, 217, 207, 86, 26, 1, 209, 217, 199, 226, 26, 1, 209, - 217, 212, 83, 26, 1, 209, 217, 173, 26, 1, 209, 217, 206, 196, 26, 1, - 209, 217, 206, 243, 26, 1, 209, 217, 216, 212, 26, 1, 209, 217, 199, 245, - 26, 1, 209, 217, 214, 162, 26, 1, 209, 217, 213, 157, 26, 1, 209, 217, - 218, 170, 26, 1, 209, 217, 200, 160, 26, 1, 209, 217, 203, 77, 26, 1, - 209, 217, 202, 223, 26, 1, 209, 217, 203, 249, 26, 1, 209, 217, 220, 151, - 26, 1, 209, 217, 221, 243, 26, 1, 209, 217, 212, 148, 26, 1, 209, 217, - 212, 180, 26, 1, 209, 217, 213, 128, 26, 1, 209, 217, 191, 243, 26, 1, - 209, 217, 203, 6, 26, 1, 209, 217, 170, 26, 1, 209, 217, 212, 217, 26, 1, - 209, 217, 209, 234, 26, 1, 209, 217, 212, 197, 26, 1, 209, 217, 192, 253, - 26, 1, 209, 217, 210, 51, 26, 1, 209, 217, 206, 69, 26, 1, 209, 217, 213, - 221, 26, 1, 209, 217, 207, 21, 26, 1, 209, 217, 221, 253, 26, 1, 209, - 217, 213, 254, 26, 1, 209, 217, 214, 49, 26, 1, 209, 217, 203, 57, 26, 1, - 209, 217, 208, 163, 26, 1, 209, 217, 232, 169, 26, 1, 209, 217, 193, 190, - 26, 1, 209, 217, 219, 148, 26, 1, 209, 217, 219, 2, 26, 1, 209, 217, 221, - 69, 26, 1, 209, 217, 217, 3, 26, 1, 209, 217, 207, 56, 26, 1, 209, 217, - 174, 26, 1, 209, 217, 216, 14, 26, 1, 209, 217, 217, 11, 26, 1, 209, 217, - 199, 73, 26, 1, 209, 217, 221, 23, 26, 1, 209, 217, 205, 151, 26, 1, 209, - 217, 193, 248, 215, 161, 1, 190, 190, 215, 161, 1, 214, 18, 215, 161, 1, - 192, 12, 215, 161, 1, 216, 48, 215, 161, 1, 249, 155, 215, 161, 1, 238, - 34, 215, 161, 1, 65, 215, 161, 1, 209, 213, 215, 161, 1, 221, 211, 215, - 161, 1, 230, 24, 215, 161, 1, 238, 9, 215, 161, 1, 243, 50, 215, 161, 1, - 222, 17, 215, 161, 1, 211, 122, 215, 161, 1, 217, 135, 215, 161, 1, 213, - 181, 215, 161, 1, 168, 215, 161, 1, 211, 89, 215, 161, 1, 74, 215, 161, - 1, 206, 163, 215, 161, 1, 203, 82, 215, 161, 1, 199, 32, 215, 161, 1, - 234, 219, 215, 161, 1, 193, 190, 215, 161, 1, 71, 215, 161, 1, 221, 69, - 215, 161, 1, 220, 26, 215, 161, 1, 173, 215, 161, 1, 230, 82, 215, 161, - 1, 207, 2, 215, 161, 1, 199, 110, 215, 161, 17, 191, 77, 215, 161, 17, - 107, 215, 161, 17, 109, 215, 161, 17, 138, 215, 161, 17, 134, 215, 161, - 17, 150, 215, 161, 17, 169, 215, 161, 17, 175, 215, 161, 17, 171, 215, - 161, 17, 178, 215, 161, 237, 240, 215, 161, 55, 237, 240, 199, 186, 1, - 210, 240, 199, 186, 1, 211, 168, 199, 186, 1, 211, 60, 199, 186, 1, 210, - 249, 199, 186, 1, 250, 125, 199, 186, 1, 252, 65, 199, 186, 1, 251, 39, - 199, 186, 1, 250, 135, 199, 186, 1, 193, 227, 199, 186, 1, 195, 155, 199, - 186, 1, 194, 255, 199, 186, 1, 193, 236, 199, 186, 1, 233, 184, 199, 186, - 1, 234, 199, 199, 186, 1, 234, 48, 199, 186, 1, 233, 223, 199, 186, 1, - 223, 43, 199, 186, 1, 228, 30, 199, 186, 1, 223, 81, 199, 186, 1, 223, - 47, 199, 186, 1, 196, 18, 199, 186, 1, 196, 160, 199, 186, 1, 196, 64, - 199, 186, 1, 196, 22, 199, 186, 1, 250, 136, 199, 186, 1, 250, 140, 199, - 186, 1, 250, 138, 199, 186, 1, 250, 137, 199, 186, 1, 196, 129, 199, 186, - 1, 196, 138, 199, 186, 1, 196, 135, 199, 186, 1, 196, 130, 199, 186, 1, - 53, 214, 72, 199, 186, 1, 180, 196, 145, 199, 186, 1, 203, 41, 196, 143, - 199, 186, 1, 203, 41, 250, 137, 199, 186, 1, 196, 150, 199, 186, 1, 196, - 143, 199, 186, 1, 196, 146, 199, 186, 1, 196, 145, 199, 186, 1, 196, 131, - 199, 186, 1, 196, 134, 199, 186, 1, 196, 133, 199, 186, 1, 196, 132, 199, - 186, 1, 215, 65, 199, 186, 1, 216, 240, 199, 186, 1, 215, 167, 199, 186, - 1, 215, 74, 199, 186, 1, 155, 199, 186, 1, 221, 217, 199, 186, 1, 231, - 242, 199, 186, 1, 214, 70, 199, 186, 1, 188, 199, 186, 1, 170, 199, 186, - 1, 193, 190, 199, 186, 1, 168, 199, 186, 1, 212, 103, 199, 186, 1, 209, - 230, 199, 186, 1, 249, 155, 199, 186, 1, 174, 199, 186, 1, 181, 199, 186, - 1, 140, 199, 186, 1, 173, 199, 186, 1, 228, 166, 199, 186, 1, 190, 190, - 199, 186, 1, 238, 34, 199, 186, 1, 165, 199, 186, 1, 213, 226, 199, 186, - 1, 203, 166, 199, 186, 1, 247, 162, 199, 186, 1, 197, 168, 199, 186, 1, - 231, 93, 199, 186, 1, 228, 163, 199, 186, 1, 199, 49, 199, 186, 1, 192, - 220, 199, 186, 1, 233, 111, 199, 186, 1, 237, 70, 199, 186, 1, 247, 3, - 199, 186, 1, 191, 123, 199, 186, 17, 191, 77, 199, 186, 17, 107, 199, - 186, 17, 109, 199, 186, 17, 138, 199, 186, 17, 134, 199, 186, 17, 150, - 199, 186, 17, 169, 199, 186, 17, 175, 199, 186, 17, 171, 199, 186, 17, - 178, 249, 69, 195, 185, 1, 234, 86, 249, 69, 195, 185, 1, 155, 249, 69, - 195, 185, 1, 205, 69, 249, 69, 195, 185, 1, 233, 111, 249, 69, 195, 185, - 1, 217, 6, 249, 69, 195, 185, 1, 192, 30, 249, 69, 195, 185, 1, 231, 227, - 249, 69, 195, 185, 1, 237, 51, 249, 69, 195, 185, 1, 221, 22, 249, 69, - 195, 185, 1, 222, 203, 249, 69, 195, 185, 1, 228, 126, 249, 69, 195, 185, - 1, 193, 190, 249, 69, 195, 185, 1, 191, 7, 249, 69, 195, 185, 1, 231, - 171, 249, 69, 195, 185, 1, 236, 176, 249, 69, 195, 185, 1, 247, 44, 249, - 69, 195, 185, 1, 196, 23, 249, 69, 195, 185, 1, 159, 249, 69, 195, 185, - 1, 249, 155, 249, 69, 195, 185, 1, 193, 249, 249, 69, 195, 185, 1, 192, - 74, 249, 69, 195, 185, 1, 168, 249, 69, 195, 185, 1, 193, 177, 249, 69, - 195, 185, 1, 65, 249, 69, 195, 185, 1, 74, 249, 69, 195, 185, 1, 211, 89, - 249, 69, 195, 185, 1, 66, 249, 69, 195, 185, 1, 234, 190, 249, 69, 195, - 185, 1, 71, 249, 69, 195, 185, 1, 68, 249, 69, 195, 185, 33, 137, 198, - 79, 249, 69, 195, 185, 33, 130, 198, 79, 249, 69, 195, 185, 33, 216, 89, - 198, 79, 249, 69, 195, 185, 33, 218, 240, 198, 79, 249, 69, 195, 185, 33, - 229, 135, 198, 79, 249, 69, 195, 185, 232, 82, 201, 64, 145, 90, 18, 222, - 14, 145, 90, 18, 222, 10, 145, 90, 18, 221, 157, 145, 90, 18, 221, 120, - 145, 90, 18, 222, 43, 145, 90, 18, 222, 40, 145, 90, 18, 220, 218, 145, - 90, 18, 220, 187, 145, 90, 18, 222, 16, 145, 90, 18, 221, 227, 145, 90, - 18, 222, 103, 145, 90, 18, 222, 100, 145, 90, 18, 221, 42, 145, 90, 18, - 221, 39, 145, 90, 18, 222, 36, 145, 90, 18, 222, 33, 145, 90, 18, 220, - 220, 145, 90, 18, 220, 219, 145, 90, 18, 221, 62, 145, 90, 18, 221, 27, - 145, 90, 18, 221, 159, 145, 90, 18, 221, 158, 145, 90, 18, 222, 119, 145, - 90, 18, 222, 39, 145, 90, 18, 220, 177, 145, 90, 18, 220, 168, 145, 90, - 18, 222, 128, 145, 90, 18, 222, 120, 145, 90, 120, 195, 160, 145, 90, - 120, 212, 187, 145, 90, 120, 220, 2, 145, 90, 120, 230, 4, 145, 90, 120, - 213, 84, 145, 90, 120, 207, 127, 145, 90, 120, 213, 111, 145, 90, 120, - 208, 71, 145, 90, 120, 192, 91, 145, 90, 120, 229, 110, 145, 90, 120, - 217, 27, 145, 90, 120, 243, 133, 145, 90, 120, 214, 167, 145, 90, 120, - 229, 46, 145, 90, 120, 209, 93, 145, 90, 120, 212, 193, 145, 90, 120, - 214, 207, 145, 90, 120, 250, 165, 145, 90, 120, 192, 216, 145, 90, 120, - 247, 78, 145, 90, 87, 243, 19, 197, 85, 145, 90, 87, 243, 19, 202, 47, - 145, 90, 87, 243, 19, 221, 245, 145, 90, 87, 243, 19, 221, 200, 145, 90, - 87, 243, 19, 200, 247, 145, 90, 87, 243, 19, 229, 4, 145, 90, 87, 243, - 19, 199, 165, 145, 90, 3, 195, 5, 198, 238, 145, 90, 3, 195, 5, 197, 156, - 247, 35, 145, 90, 3, 243, 19, 243, 122, 145, 90, 3, 195, 5, 199, 10, 145, - 90, 3, 195, 5, 233, 51, 145, 90, 3, 192, 171, 212, 181, 145, 90, 3, 192, - 171, 207, 4, 145, 90, 3, 192, 171, 198, 32, 145, 90, 3, 192, 171, 233, - 92, 145, 90, 3, 195, 5, 205, 45, 145, 90, 3, 216, 211, 200, 251, 145, 90, - 3, 195, 5, 212, 233, 145, 90, 3, 228, 33, 192, 111, 145, 90, 3, 192, 215, - 145, 90, 3, 243, 19, 197, 143, 206, 145, 145, 90, 17, 191, 77, 145, 90, - 17, 107, 145, 90, 17, 109, 145, 90, 17, 138, 145, 90, 17, 134, 145, 90, - 17, 150, 145, 90, 17, 169, 145, 90, 17, 175, 145, 90, 17, 171, 145, 90, - 17, 178, 145, 90, 31, 199, 90, 145, 90, 31, 228, 140, 145, 90, 31, 199, - 96, 198, 228, 145, 90, 31, 216, 49, 145, 90, 31, 228, 143, 216, 49, 145, - 90, 31, 199, 96, 248, 171, 145, 90, 31, 197, 227, 145, 90, 3, 195, 5, - 218, 222, 145, 90, 3, 192, 168, 145, 90, 3, 229, 105, 145, 90, 3, 198, - 255, 229, 105, 145, 90, 3, 190, 236, 199, 43, 145, 90, 3, 229, 30, 145, - 90, 3, 212, 247, 145, 90, 3, 192, 206, 145, 90, 3, 212, 185, 145, 90, 3, - 250, 148, 145, 90, 3, 197, 7, 247, 34, 145, 90, 3, 216, 211, 197, 159, - 145, 90, 3, 199, 166, 145, 90, 3, 218, 255, 145, 90, 3, 215, 105, 145, - 90, 3, 243, 19, 230, 78, 218, 198, 212, 191, 212, 190, 145, 90, 3, 243, - 19, 238, 196, 197, 150, 145, 90, 3, 243, 19, 197, 5, 145, 90, 3, 243, 19, - 197, 6, 243, 38, 145, 90, 3, 243, 19, 208, 161, 237, 208, 145, 90, 3, - 243, 19, 212, 240, 198, 41, 145, 90, 242, 246, 3, 197, 154, 145, 90, 242, - 246, 3, 192, 76, 145, 90, 242, 246, 3, 219, 92, 145, 90, 242, 246, 3, - 220, 0, 145, 90, 242, 246, 3, 192, 167, 145, 90, 242, 246, 3, 221, 43, - 145, 90, 242, 246, 3, 230, 0, 145, 90, 242, 246, 3, 215, 149, 145, 90, - 242, 246, 3, 198, 239, 145, 90, 242, 246, 3, 197, 165, 145, 90, 242, 246, - 3, 209, 227, 145, 90, 242, 246, 3, 221, 215, 145, 90, 242, 246, 3, 230, - 66, 145, 90, 242, 246, 3, 195, 182, 145, 90, 242, 246, 3, 233, 88, 145, - 90, 242, 246, 3, 192, 118, 145, 90, 242, 246, 3, 197, 137, 145, 90, 242, - 246, 3, 220, 172, 145, 90, 242, 246, 3, 193, 237, 216, 220, 6, 1, 218, - 170, 216, 220, 6, 1, 206, 9, 216, 220, 6, 1, 196, 12, 216, 220, 6, 1, - 193, 224, 216, 220, 6, 1, 250, 178, 216, 220, 6, 1, 191, 166, 216, 220, - 6, 1, 221, 24, 216, 220, 6, 1, 210, 238, 216, 220, 6, 1, 200, 43, 216, - 220, 6, 1, 232, 53, 216, 220, 6, 1, 233, 177, 216, 220, 6, 1, 68, 216, - 220, 6, 1, 222, 154, 216, 220, 6, 1, 65, 216, 220, 6, 1, 223, 37, 216, - 220, 6, 1, 71, 216, 220, 6, 1, 250, 122, 216, 220, 6, 1, 247, 195, 216, - 220, 6, 1, 66, 216, 220, 6, 1, 191, 225, 216, 220, 6, 1, 172, 216, 220, - 6, 1, 208, 106, 216, 220, 6, 1, 228, 171, 216, 220, 6, 1, 212, 105, 216, - 220, 6, 1, 192, 235, 216, 220, 6, 1, 238, 129, 216, 220, 6, 1, 211, 153, - 216, 220, 6, 1, 215, 63, 216, 220, 6, 1, 146, 216, 220, 6, 1, 74, 216, - 220, 6, 1, 251, 238, 216, 220, 6, 1, 192, 159, 216, 220, 2, 1, 218, 170, - 216, 220, 2, 1, 206, 9, 216, 220, 2, 1, 196, 12, 216, 220, 2, 1, 193, - 224, 216, 220, 2, 1, 250, 178, 216, 220, 2, 1, 191, 166, 216, 220, 2, 1, - 221, 24, 216, 220, 2, 1, 210, 238, 216, 220, 2, 1, 200, 43, 216, 220, 2, - 1, 232, 53, 216, 220, 2, 1, 233, 177, 216, 220, 2, 1, 68, 216, 220, 2, 1, - 222, 154, 216, 220, 2, 1, 65, 216, 220, 2, 1, 223, 37, 216, 220, 2, 1, - 71, 216, 220, 2, 1, 250, 122, 216, 220, 2, 1, 247, 195, 216, 220, 2, 1, - 66, 216, 220, 2, 1, 191, 225, 216, 220, 2, 1, 172, 216, 220, 2, 1, 208, - 106, 216, 220, 2, 1, 228, 171, 216, 220, 2, 1, 212, 105, 216, 220, 2, 1, - 192, 235, 216, 220, 2, 1, 238, 129, 216, 220, 2, 1, 211, 153, 216, 220, - 2, 1, 215, 63, 216, 220, 2, 1, 146, 216, 220, 2, 1, 74, 216, 220, 2, 1, - 251, 238, 216, 220, 2, 1, 192, 159, 216, 220, 17, 191, 77, 216, 220, 17, - 107, 216, 220, 17, 109, 216, 220, 17, 138, 216, 220, 17, 134, 216, 220, - 17, 150, 216, 220, 17, 169, 216, 220, 17, 175, 216, 220, 17, 171, 216, - 220, 17, 178, 216, 220, 31, 199, 95, 216, 220, 31, 234, 129, 216, 220, - 31, 197, 37, 216, 220, 31, 198, 251, 216, 220, 31, 232, 124, 216, 220, - 31, 233, 21, 216, 220, 31, 202, 131, 216, 220, 31, 203, 245, 216, 220, - 31, 234, 163, 216, 220, 31, 213, 173, 216, 220, 17, 91, 251, 159, 20, - 216, 220, 17, 105, 251, 159, 20, 216, 220, 17, 115, 251, 159, 20, 216, - 220, 242, 76, 216, 220, 232, 82, 201, 64, 216, 220, 16, 251, 223, 216, - 220, 233, 218, 211, 138, 121, 1, 168, 121, 1, 249, 155, 121, 1, 11, 168, - 121, 1, 209, 112, 121, 1, 174, 121, 1, 216, 105, 121, 1, 251, 16, 174, - 121, 1, 233, 111, 121, 1, 195, 188, 121, 1, 195, 71, 121, 1, 190, 190, - 121, 1, 238, 34, 121, 1, 11, 197, 132, 121, 1, 11, 190, 190, 121, 1, 197, - 132, 121, 1, 237, 193, 121, 1, 181, 121, 1, 213, 226, 121, 1, 11, 213, - 81, 121, 1, 251, 16, 181, 121, 1, 213, 81, 121, 1, 213, 67, 121, 1, 173, - 121, 1, 218, 184, 121, 1, 219, 161, 121, 1, 219, 150, 121, 1, 198, 112, - 121, 1, 236, 185, 121, 1, 198, 104, 121, 1, 236, 184, 121, 1, 155, 121, - 1, 231, 242, 121, 1, 11, 155, 121, 1, 208, 98, 121, 1, 208, 74, 121, 1, - 214, 70, 121, 1, 214, 19, 121, 1, 251, 16, 214, 70, 121, 1, 140, 121, 1, - 192, 220, 121, 1, 231, 93, 121, 1, 231, 68, 121, 1, 197, 142, 121, 1, - 235, 20, 121, 1, 212, 103, 121, 1, 212, 85, 121, 1, 197, 157, 121, 1, - 235, 31, 121, 1, 11, 197, 157, 121, 1, 11, 235, 31, 121, 1, 207, 83, 197, - 157, 121, 1, 203, 166, 121, 1, 201, 176, 121, 1, 191, 71, 121, 1, 190, - 253, 121, 1, 197, 168, 121, 1, 235, 37, 121, 1, 11, 197, 168, 121, 1, - 188, 121, 1, 191, 123, 121, 1, 190, 254, 121, 1, 190, 224, 121, 1, 190, - 204, 121, 1, 251, 16, 190, 224, 121, 1, 190, 196, 121, 1, 190, 203, 121, - 1, 193, 190, 121, 1, 251, 247, 121, 1, 229, 179, 121, 1, 248, 34, 121, 1, - 200, 125, 121, 1, 235, 21, 121, 1, 199, 145, 121, 1, 197, 161, 121, 1, - 206, 72, 121, 3, 120, 52, 164, 121, 1, 214, 214, 121, 3, 250, 201, 121, - 3, 207, 83, 195, 18, 121, 3, 207, 83, 250, 201, 121, 18, 3, 65, 121, 18, - 3, 252, 208, 121, 18, 3, 251, 243, 121, 18, 3, 251, 134, 121, 18, 3, 251, - 124, 121, 18, 3, 74, 121, 18, 3, 211, 89, 121, 18, 3, 193, 48, 121, 18, - 3, 193, 224, 121, 18, 3, 71, 121, 18, 3, 234, 105, 121, 18, 3, 234, 90, - 121, 18, 3, 211, 149, 121, 18, 3, 68, 121, 18, 3, 228, 37, 121, 18, 3, - 228, 36, 121, 18, 3, 228, 35, 121, 18, 3, 223, 90, 121, 18, 3, 223, 228, - 121, 18, 3, 223, 201, 121, 18, 3, 223, 51, 121, 18, 3, 223, 138, 121, 18, - 3, 66, 121, 18, 3, 196, 168, 121, 18, 3, 196, 167, 121, 18, 3, 196, 166, - 121, 18, 3, 196, 30, 121, 18, 3, 196, 148, 121, 18, 3, 196, 97, 121, 18, - 3, 192, 159, 121, 18, 3, 192, 33, 121, 18, 3, 252, 27, 121, 18, 3, 252, - 23, 121, 18, 3, 234, 28, 121, 18, 3, 206, 114, 234, 28, 121, 18, 3, 234, - 36, 121, 18, 3, 206, 114, 234, 36, 121, 18, 3, 251, 238, 121, 18, 3, 234, - 168, 121, 18, 3, 250, 165, 121, 18, 3, 211, 21, 121, 18, 3, 215, 63, 121, - 18, 3, 214, 72, 121, 18, 3, 196, 81, 121, 18, 3, 191, 205, 121, 18, 3, - 211, 143, 121, 18, 3, 211, 150, 121, 18, 3, 193, 239, 121, 18, 3, 223, - 206, 121, 18, 3, 234, 219, 121, 18, 3, 223, 88, 121, 18, 3, 196, 139, - 121, 163, 179, 121, 163, 198, 54, 179, 121, 163, 58, 121, 163, 60, 121, - 1, 198, 77, 121, 1, 198, 76, 121, 1, 198, 75, 121, 1, 198, 74, 121, 1, - 198, 73, 121, 1, 198, 72, 121, 1, 198, 71, 121, 1, 207, 83, 198, 78, 121, - 1, 207, 83, 198, 77, 121, 1, 207, 83, 198, 75, 121, 1, 207, 83, 198, 74, - 121, 1, 207, 83, 198, 73, 121, 1, 207, 83, 198, 71, 19, 223, 53, 77, 46, - 223, 53, 77, 39, 243, 82, 228, 253, 77, 39, 243, 82, 223, 53, 77, 41, 2, - 27, 233, 5, 195, 57, 251, 159, 207, 109, 87, 247, 162, 195, 57, 251, 159, - 207, 109, 87, 213, 225, 19, 242, 65, 19, 242, 64, 19, 242, 63, 19, 242, - 62, 19, 242, 61, 19, 242, 60, 19, 242, 59, 19, 242, 58, 19, 242, 57, 19, - 242, 56, 19, 242, 55, 19, 242, 54, 19, 242, 53, 19, 242, 52, 19, 242, 51, - 19, 242, 50, 19, 242, 49, 19, 242, 48, 19, 242, 47, 19, 242, 46, 19, 242, - 45, 19, 242, 44, 19, 242, 43, 19, 242, 42, 19, 242, 41, 19, 242, 40, 19, - 242, 39, 19, 242, 38, 19, 242, 37, 19, 242, 36, 19, 242, 35, 19, 242, 34, - 19, 242, 33, 19, 242, 32, 19, 242, 31, 19, 242, 30, 19, 242, 29, 19, 242, - 28, 19, 242, 27, 19, 242, 26, 19, 242, 25, 19, 242, 24, 19, 242, 23, 19, - 242, 22, 19, 242, 21, 19, 242, 20, 19, 242, 19, 19, 242, 18, 19, 242, 17, - 19, 242, 16, 19, 242, 15, 19, 242, 14, 19, 242, 13, 19, 242, 12, 19, 242, - 11, 19, 242, 10, 19, 242, 9, 19, 242, 8, 19, 242, 7, 19, 242, 6, 19, 242, - 5, 19, 242, 4, 19, 242, 3, 19, 242, 2, 19, 242, 1, 19, 242, 0, 19, 241, - 255, 19, 241, 254, 19, 241, 253, 19, 241, 252, 19, 241, 251, 19, 241, - 250, 19, 241, 249, 19, 241, 248, 19, 241, 247, 19, 241, 246, 19, 241, - 245, 19, 241, 244, 19, 241, 243, 19, 241, 242, 19, 241, 241, 19, 241, - 240, 19, 241, 239, 19, 241, 238, 19, 241, 237, 19, 241, 236, 19, 241, - 235, 19, 241, 234, 19, 241, 233, 19, 241, 232, 19, 241, 231, 19, 241, - 230, 19, 241, 229, 19, 241, 228, 19, 241, 227, 19, 241, 226, 19, 241, - 225, 19, 241, 224, 19, 241, 223, 19, 241, 222, 19, 241, 221, 19, 241, - 220, 19, 241, 219, 19, 241, 218, 19, 241, 217, 19, 241, 216, 19, 241, - 215, 19, 241, 214, 19, 241, 213, 19, 241, 212, 19, 241, 211, 19, 241, - 210, 19, 241, 209, 19, 241, 208, 19, 241, 207, 19, 241, 206, 19, 241, - 205, 19, 241, 204, 19, 241, 203, 19, 241, 202, 19, 241, 201, 19, 241, - 200, 19, 241, 199, 19, 241, 198, 19, 241, 197, 19, 241, 196, 19, 241, - 195, 19, 241, 194, 19, 241, 193, 19, 241, 192, 19, 241, 191, 19, 241, - 190, 19, 241, 189, 19, 241, 188, 19, 241, 187, 19, 241, 186, 19, 241, - 185, 19, 241, 184, 19, 241, 183, 19, 241, 182, 19, 241, 181, 19, 241, - 180, 19, 241, 179, 19, 241, 178, 19, 241, 177, 19, 241, 176, 19, 241, - 175, 19, 241, 174, 19, 241, 173, 19, 241, 172, 19, 241, 171, 19, 241, - 170, 19, 241, 169, 19, 241, 168, 19, 241, 167, 19, 241, 166, 19, 241, - 165, 19, 241, 164, 19, 241, 163, 19, 241, 162, 19, 241, 161, 19, 241, - 160, 19, 241, 159, 19, 241, 158, 19, 241, 157, 19, 241, 156, 19, 241, - 155, 19, 241, 154, 19, 241, 153, 19, 241, 152, 19, 241, 151, 19, 241, - 150, 19, 241, 149, 19, 241, 148, 19, 241, 147, 19, 241, 146, 19, 241, - 145, 19, 241, 144, 19, 241, 143, 19, 241, 142, 19, 241, 141, 19, 241, - 140, 19, 241, 139, 19, 241, 138, 19, 241, 137, 19, 241, 136, 19, 241, - 135, 19, 241, 134, 19, 241, 133, 19, 241, 132, 19, 241, 131, 19, 241, - 130, 19, 241, 129, 19, 241, 128, 19, 241, 127, 19, 241, 126, 19, 241, - 125, 19, 241, 124, 19, 241, 123, 19, 241, 122, 19, 241, 121, 19, 241, - 120, 19, 241, 119, 19, 241, 118, 19, 241, 117, 19, 241, 116, 19, 241, - 115, 19, 241, 114, 19, 241, 113, 19, 241, 112, 19, 241, 111, 19, 241, - 110, 19, 241, 109, 19, 241, 108, 19, 241, 107, 19, 241, 106, 19, 241, - 105, 19, 241, 104, 19, 241, 103, 19, 241, 102, 19, 241, 101, 19, 241, - 100, 19, 241, 99, 19, 241, 98, 19, 241, 97, 19, 241, 96, 19, 241, 95, 19, - 241, 94, 19, 241, 93, 19, 241, 92, 19, 241, 91, 19, 241, 90, 19, 241, 89, - 19, 241, 88, 19, 241, 87, 19, 241, 86, 19, 241, 85, 19, 241, 84, 19, 241, - 83, 19, 241, 82, 19, 241, 81, 19, 241, 80, 19, 241, 79, 19, 241, 78, 19, - 241, 77, 19, 241, 76, 19, 241, 75, 19, 241, 74, 19, 241, 73, 19, 241, 72, - 19, 241, 71, 19, 241, 70, 19, 241, 69, 19, 241, 68, 19, 241, 67, 19, 241, - 66, 19, 241, 65, 19, 241, 64, 19, 241, 63, 19, 241, 62, 19, 241, 61, 19, - 241, 60, 19, 241, 59, 19, 241, 58, 19, 241, 57, 19, 241, 56, 19, 241, 55, - 19, 241, 54, 19, 241, 53, 19, 241, 52, 19, 241, 51, 19, 241, 50, 19, 241, - 49, 19, 241, 48, 19, 241, 47, 19, 241, 46, 19, 241, 45, 19, 241, 44, 19, - 241, 43, 19, 241, 42, 19, 241, 41, 19, 241, 40, 19, 241, 39, 19, 241, 38, - 19, 241, 37, 19, 241, 36, 19, 241, 35, 19, 241, 34, 19, 241, 33, 19, 241, - 32, 19, 241, 31, 19, 241, 30, 19, 241, 29, 19, 241, 28, 19, 241, 27, 19, - 241, 26, 19, 241, 25, 19, 241, 24, 19, 241, 23, 19, 241, 22, 19, 241, 21, - 19, 241, 20, 19, 241, 19, 19, 241, 18, 19, 241, 17, 19, 241, 16, 19, 241, - 15, 19, 241, 14, 19, 241, 13, 19, 241, 12, 19, 241, 11, 19, 241, 10, 19, - 241, 9, 19, 241, 8, 19, 241, 7, 19, 241, 6, 19, 241, 5, 19, 241, 4, 19, - 241, 3, 19, 241, 2, 19, 241, 1, 19, 241, 0, 19, 240, 255, 19, 240, 254, - 19, 240, 253, 19, 240, 252, 19, 240, 251, 19, 240, 250, 19, 240, 249, 19, - 240, 248, 19, 240, 247, 19, 240, 246, 19, 240, 245, 19, 240, 244, 19, - 240, 243, 19, 240, 242, 19, 240, 241, 19, 240, 240, 19, 240, 239, 19, - 240, 238, 19, 240, 237, 19, 240, 236, 19, 240, 235, 19, 240, 234, 19, - 240, 233, 19, 240, 232, 19, 240, 231, 19, 240, 230, 19, 240, 229, 19, - 240, 228, 19, 240, 227, 19, 240, 226, 19, 240, 225, 19, 240, 224, 19, - 240, 223, 19, 240, 222, 19, 240, 221, 19, 240, 220, 19, 240, 219, 19, - 240, 218, 19, 240, 217, 19, 240, 216, 19, 240, 215, 19, 240, 214, 19, - 240, 213, 19, 240, 212, 19, 240, 211, 19, 240, 210, 19, 240, 209, 19, - 240, 208, 19, 240, 207, 19, 240, 206, 19, 240, 205, 19, 240, 204, 19, - 240, 203, 19, 240, 202, 19, 240, 201, 19, 240, 200, 19, 240, 199, 19, - 240, 198, 19, 240, 197, 19, 240, 196, 19, 240, 195, 19, 240, 194, 19, - 240, 193, 19, 240, 192, 19, 240, 191, 19, 240, 190, 19, 240, 189, 19, - 240, 188, 19, 240, 187, 19, 240, 186, 19, 240, 185, 19, 240, 184, 19, - 240, 183, 19, 240, 182, 19, 240, 181, 19, 240, 180, 19, 240, 179, 19, - 240, 178, 19, 240, 177, 19, 240, 176, 19, 240, 175, 19, 240, 174, 19, - 240, 173, 19, 240, 172, 19, 240, 171, 19, 240, 170, 19, 240, 169, 19, - 240, 168, 19, 240, 167, 19, 240, 166, 19, 240, 165, 19, 240, 164, 19, - 240, 163, 19, 240, 162, 19, 240, 161, 19, 240, 160, 19, 240, 159, 19, - 240, 158, 19, 240, 157, 19, 240, 156, 19, 240, 155, 19, 240, 154, 19, - 240, 153, 19, 240, 152, 19, 240, 151, 19, 240, 150, 19, 240, 149, 19, - 240, 148, 19, 240, 147, 19, 240, 146, 19, 240, 145, 19, 240, 144, 19, - 240, 143, 19, 240, 142, 19, 240, 141, 19, 240, 140, 19, 240, 139, 19, - 240, 138, 19, 240, 137, 19, 240, 136, 19, 240, 135, 19, 240, 134, 19, - 240, 133, 19, 240, 132, 19, 240, 131, 19, 240, 130, 19, 240, 129, 19, - 240, 128, 19, 240, 127, 19, 240, 126, 19, 240, 125, 19, 240, 124, 19, - 240, 123, 19, 240, 122, 19, 240, 121, 19, 240, 120, 19, 240, 119, 19, - 240, 118, 19, 240, 117, 19, 240, 116, 19, 240, 115, 19, 240, 114, 19, - 240, 113, 19, 240, 112, 19, 240, 111, 19, 240, 110, 19, 240, 109, 19, - 240, 108, 19, 240, 107, 19, 240, 106, 19, 240, 105, 19, 240, 104, 19, - 240, 103, 19, 240, 102, 19, 240, 101, 19, 240, 100, 19, 240, 99, 19, 240, - 98, 19, 240, 97, 19, 240, 96, 19, 240, 95, 19, 240, 94, 19, 240, 93, 19, - 240, 92, 19, 240, 91, 19, 240, 90, 19, 240, 89, 19, 240, 88, 19, 240, 87, - 19, 240, 86, 19, 240, 85, 19, 240, 84, 19, 240, 83, 19, 240, 82, 19, 240, - 81, 19, 240, 80, 19, 240, 79, 19, 240, 78, 19, 240, 77, 19, 240, 76, 19, - 240, 75, 19, 240, 74, 19, 240, 73, 19, 240, 72, 19, 240, 71, 19, 240, 70, - 19, 240, 69, 19, 240, 68, 19, 240, 67, 19, 240, 66, 19, 240, 65, 19, 240, - 64, 19, 240, 63, 19, 240, 62, 19, 240, 61, 19, 240, 60, 19, 240, 59, 19, - 240, 58, 19, 240, 57, 19, 240, 56, 19, 240, 55, 19, 240, 54, 19, 240, 53, - 19, 240, 52, 19, 240, 51, 19, 240, 50, 19, 240, 49, 19, 240, 48, 19, 240, - 47, 19, 240, 46, 19, 240, 45, 19, 240, 44, 19, 240, 43, 19, 240, 42, 19, - 240, 41, 19, 240, 40, 19, 240, 39, 19, 240, 38, 19, 240, 37, 19, 240, 36, - 19, 240, 35, 19, 240, 34, 19, 240, 33, 19, 240, 32, 19, 240, 31, 19, 240, - 30, 19, 240, 29, 19, 240, 28, 19, 240, 27, 19, 240, 26, 19, 240, 25, 19, - 240, 24, 19, 240, 23, 19, 240, 22, 19, 240, 21, 19, 240, 20, 19, 240, 19, - 19, 240, 18, 19, 240, 17, 19, 240, 16, 19, 240, 15, 19, 240, 14, 19, 240, - 13, 19, 240, 12, 19, 240, 11, 19, 240, 10, 19, 240, 9, 19, 240, 8, 19, - 240, 7, 19, 240, 6, 19, 240, 5, 19, 240, 4, 19, 240, 3, 19, 240, 2, 19, - 240, 1, 19, 240, 0, 19, 239, 255, 19, 239, 254, 19, 239, 253, 19, 239, - 252, 19, 239, 251, 19, 239, 250, 19, 239, 249, 19, 239, 248, 19, 239, - 247, 19, 239, 246, 19, 239, 245, 19, 239, 244, 19, 239, 243, 19, 239, - 242, 19, 239, 241, 19, 239, 240, 19, 239, 239, 19, 239, 238, 19, 239, - 237, 19, 239, 236, 19, 239, 235, 19, 239, 234, 19, 239, 233, 19, 239, - 232, 19, 239, 231, 19, 239, 230, 19, 239, 229, 19, 239, 228, 19, 239, - 227, 19, 239, 226, 19, 239, 225, 19, 239, 224, 19, 239, 223, 19, 239, - 222, 19, 239, 221, 19, 239, 220, 19, 239, 219, 19, 239, 218, 19, 239, - 217, 19, 239, 216, 19, 239, 215, 19, 239, 214, 19, 239, 213, 19, 239, - 212, 19, 239, 211, 19, 239, 210, 19, 239, 209, 19, 239, 208, 19, 239, - 207, 19, 239, 206, 19, 239, 205, 19, 239, 204, 19, 239, 203, 19, 239, - 202, 19, 239, 201, 19, 239, 200, 19, 239, 199, 19, 239, 198, 19, 239, - 197, 19, 239, 196, 19, 239, 195, 19, 239, 194, 19, 239, 193, 19, 239, - 192, 19, 239, 191, 19, 239, 190, 19, 239, 189, 19, 239, 188, 19, 239, - 187, 19, 239, 186, 19, 239, 185, 19, 239, 184, 19, 239, 183, 19, 239, - 182, 19, 239, 181, 19, 239, 180, 19, 239, 179, 19, 239, 178, 19, 239, - 177, 19, 239, 176, 19, 239, 175, 19, 239, 174, 19, 239, 173, 19, 239, - 172, 19, 239, 171, 19, 239, 170, 19, 239, 169, 19, 239, 168, 19, 239, - 167, 19, 239, 166, 19, 239, 165, 19, 239, 164, 19, 239, 163, 19, 239, - 162, 19, 239, 161, 19, 239, 160, 19, 239, 159, 19, 239, 158, 19, 239, - 157, 19, 239, 156, 19, 239, 155, 19, 239, 154, 19, 239, 153, 19, 239, - 152, 19, 239, 151, 19, 239, 150, 19, 239, 149, 19, 239, 148, 19, 239, - 147, 19, 239, 146, 19, 239, 145, 19, 239, 144, 19, 239, 143, 19, 239, - 142, 19, 239, 141, 19, 239, 140, 19, 239, 139, 19, 239, 138, 19, 239, - 137, 19, 239, 136, 19, 239, 135, 19, 239, 134, 19, 239, 133, 19, 239, - 132, 19, 239, 131, 19, 239, 130, 19, 239, 129, 19, 239, 128, 19, 239, - 127, 19, 239, 126, 19, 239, 125, 19, 239, 124, 19, 239, 123, 19, 239, - 122, 19, 239, 121, 19, 239, 120, 19, 239, 119, 19, 239, 118, 19, 239, - 117, 19, 239, 116, 19, 239, 115, 19, 239, 114, 19, 239, 113, 19, 239, - 112, 19, 239, 111, 19, 239, 110, 19, 239, 109, 19, 239, 108, 19, 239, - 107, 19, 239, 106, 19, 239, 105, 19, 239, 104, 19, 239, 103, 19, 239, - 102, 19, 239, 101, 19, 239, 100, 19, 239, 99, 19, 239, 98, 19, 239, 97, - 19, 239, 96, 19, 239, 95, 19, 239, 94, 19, 239, 93, 19, 239, 92, 19, 239, - 91, 19, 239, 90, 19, 239, 89, 19, 239, 88, 19, 239, 87, 19, 239, 86, 19, - 239, 85, 19, 239, 84, 19, 239, 83, 19, 239, 82, 19, 239, 81, 19, 239, 80, - 19, 239, 79, 19, 239, 78, 19, 239, 77, 19, 239, 76, 19, 239, 75, 19, 239, - 74, 19, 239, 73, 19, 239, 72, 19, 239, 71, 19, 239, 70, 19, 239, 69, 19, - 239, 68, 19, 239, 67, 19, 239, 66, 41, 2, 27, 246, 240, 41, 2, 27, 246, - 239, 41, 2, 27, 246, 238, 41, 2, 27, 246, 237, 41, 2, 27, 246, 236, 41, - 2, 27, 246, 235, 41, 2, 27, 246, 234, 41, 2, 27, 246, 233, 41, 2, 27, - 246, 232, 41, 2, 27, 246, 231, 41, 2, 27, 246, 230, 41, 2, 27, 246, 229, - 41, 2, 27, 246, 228, 41, 2, 27, 246, 227, 41, 2, 27, 246, 226, 41, 2, 27, - 246, 225, 41, 2, 27, 246, 224, 41, 2, 27, 246, 223, 41, 2, 27, 246, 222, - 41, 2, 27, 246, 221, 41, 2, 27, 246, 220, 41, 2, 27, 246, 219, 41, 2, 27, - 246, 218, 41, 2, 27, 246, 217, 41, 2, 27, 246, 216, 41, 2, 27, 246, 215, - 41, 2, 27, 246, 214, 41, 2, 27, 246, 213, 41, 2, 27, 246, 212, 41, 2, 27, - 246, 211, 41, 2, 27, 246, 210, 41, 2, 27, 246, 209, 41, 2, 27, 246, 208, - 41, 2, 27, 246, 207, 41, 2, 27, 246, 206, 41, 2, 27, 246, 205, 41, 2, 27, - 246, 204, 41, 2, 27, 246, 203, 41, 2, 27, 246, 202, 41, 2, 27, 246, 201, - 41, 2, 27, 246, 200, 41, 2, 27, 246, 199, 41, 2, 27, 246, 198, 41, 2, 27, - 246, 197, 41, 2, 27, 246, 196, 41, 2, 27, 246, 195, 41, 2, 27, 246, 194, - 41, 2, 27, 246, 193, 41, 2, 27, 246, 192, 41, 2, 27, 246, 191, 41, 2, 27, - 246, 190, 41, 2, 27, 246, 189, 41, 2, 27, 246, 188, 41, 2, 27, 246, 187, - 41, 2, 27, 246, 186, 41, 2, 27, 246, 185, 41, 2, 27, 246, 184, 41, 2, 27, - 246, 183, 41, 2, 27, 246, 182, 41, 2, 27, 246, 181, 41, 2, 27, 246, 180, - 41, 2, 27, 246, 179, 41, 2, 27, 246, 178, 41, 2, 27, 246, 177, 41, 2, 27, - 246, 176, 41, 2, 27, 246, 175, 41, 2, 27, 246, 174, 41, 2, 27, 246, 173, - 41, 2, 27, 246, 172, 41, 2, 27, 246, 171, 41, 2, 27, 246, 170, 41, 2, 27, - 246, 169, 41, 2, 27, 246, 168, 41, 2, 27, 246, 167, 41, 2, 27, 246, 166, - 41, 2, 27, 246, 165, 41, 2, 27, 246, 164, 41, 2, 27, 246, 163, 41, 2, 27, - 246, 162, 41, 2, 27, 246, 161, 41, 2, 27, 246, 160, 41, 2, 27, 246, 159, - 41, 2, 27, 246, 158, 41, 2, 27, 246, 157, 41, 2, 27, 246, 156, 41, 2, 27, - 246, 155, 41, 2, 27, 246, 154, 41, 2, 27, 246, 153, 41, 2, 27, 246, 152, - 41, 2, 27, 246, 151, 41, 2, 27, 246, 150, 41, 2, 27, 246, 149, 41, 2, 27, - 246, 148, 41, 2, 27, 246, 147, 41, 2, 27, 246, 146, 41, 2, 27, 246, 145, - 41, 2, 27, 246, 144, 41, 2, 27, 246, 143, 41, 2, 27, 246, 142, 41, 2, 27, - 246, 141, 41, 2, 27, 246, 140, 41, 2, 27, 246, 139, 41, 2, 27, 246, 138, - 41, 2, 27, 246, 137, 41, 2, 27, 246, 136, 41, 2, 27, 246, 135, 41, 2, 27, - 246, 134, 41, 2, 27, 246, 133, 41, 2, 27, 246, 132, 41, 2, 27, 246, 131, - 41, 2, 27, 246, 130, 41, 2, 27, 246, 129, 41, 2, 27, 246, 128, 41, 2, 27, - 246, 127, 41, 2, 27, 246, 126, 41, 2, 27, 246, 125, 41, 2, 27, 246, 124, - 41, 2, 27, 246, 123, 41, 2, 27, 246, 122, 41, 2, 27, 246, 121, 41, 2, 27, - 246, 120, 41, 2, 27, 246, 119, 41, 2, 27, 246, 118, 41, 2, 27, 246, 117, - 41, 2, 27, 246, 116, 41, 2, 27, 246, 115, 41, 2, 27, 246, 114, 41, 2, 27, - 246, 113, 41, 2, 27, 246, 112, 41, 2, 27, 246, 111, 41, 2, 27, 246, 110, - 41, 2, 27, 246, 109, 41, 2, 27, 246, 108, 41, 2, 27, 246, 107, 41, 2, 27, - 246, 106, 41, 2, 27, 246, 105, 41, 2, 27, 246, 104, 41, 2, 27, 246, 103, - 41, 2, 27, 246, 102, 41, 2, 27, 246, 101, 41, 2, 27, 246, 100, 41, 2, 27, - 246, 99, 41, 2, 27, 246, 98, 41, 2, 27, 246, 97, 41, 2, 27, 246, 96, 41, - 2, 27, 246, 95, 41, 2, 27, 246, 94, 41, 2, 27, 246, 93, 41, 2, 27, 246, - 92, 41, 2, 27, 246, 91, 41, 2, 27, 246, 90, 41, 2, 27, 246, 89, 41, 2, - 27, 246, 88, 41, 2, 27, 246, 87, 41, 2, 27, 246, 86, 41, 2, 27, 246, 85, - 41, 2, 27, 246, 84, 41, 2, 27, 246, 83, 41, 2, 27, 246, 82, 41, 2, 27, - 246, 81, 41, 2, 27, 246, 80, 41, 2, 27, 246, 79, 41, 2, 27, 246, 78, 41, - 2, 27, 246, 77, 41, 2, 27, 246, 76, 41, 2, 27, 246, 75, 41, 2, 27, 246, - 74, 41, 2, 27, 246, 73, 41, 2, 27, 246, 72, 41, 2, 27, 246, 71, 41, 2, - 27, 246, 70, 41, 2, 27, 246, 69, 41, 2, 27, 246, 68, 41, 2, 27, 246, 67, - 41, 2, 27, 246, 66, 41, 2, 27, 246, 65, 41, 2, 27, 246, 64, 41, 2, 27, - 246, 63, 41, 2, 27, 246, 62, 41, 2, 27, 246, 61, 41, 2, 27, 246, 60, 41, - 2, 27, 246, 59, 41, 2, 27, 246, 58, 41, 2, 27, 246, 57, 41, 2, 27, 246, - 56, 41, 2, 27, 246, 55, 41, 2, 27, 246, 54, 41, 2, 27, 246, 53, 41, 2, - 27, 246, 52, 41, 2, 27, 246, 51, 41, 2, 27, 246, 50, 41, 2, 27, 246, 49, - 41, 2, 27, 246, 48, 41, 2, 27, 246, 47, 41, 2, 27, 246, 46, 41, 2, 27, - 246, 45, 41, 2, 27, 246, 44, 41, 2, 27, 246, 43, 41, 2, 27, 246, 42, 41, - 2, 27, 246, 41, 41, 2, 27, 246, 40, 41, 2, 27, 246, 39, 41, 2, 27, 246, - 38, 41, 2, 27, 246, 37, 41, 2, 27, 246, 36, 41, 2, 27, 246, 35, 41, 2, - 27, 246, 34, 41, 2, 27, 246, 33, 41, 2, 27, 246, 32, 41, 2, 27, 246, 31, - 41, 2, 27, 246, 30, 41, 2, 27, 246, 29, 41, 2, 27, 246, 28, 41, 2, 27, - 246, 27, 41, 2, 27, 246, 26, 41, 2, 27, 246, 25, 41, 2, 27, 246, 24, 41, - 2, 27, 246, 23, 41, 2, 27, 246, 22, 41, 2, 27, 246, 21, 41, 2, 27, 246, - 20, 41, 2, 27, 246, 19, 41, 2, 27, 246, 18, 41, 2, 27, 246, 17, 41, 2, - 27, 246, 16, 41, 2, 27, 246, 15, 41, 2, 27, 246, 14, 41, 2, 27, 246, 13, - 41, 2, 27, 246, 12, 41, 2, 27, 246, 11, 41, 2, 27, 246, 10, 41, 2, 27, - 246, 9, 41, 2, 27, 246, 8, 41, 2, 27, 246, 7, 41, 2, 27, 246, 6, 41, 2, - 27, 246, 5, 41, 2, 27, 246, 4, 41, 2, 27, 246, 3, 41, 2, 27, 246, 2, 41, - 2, 27, 246, 1, 41, 2, 27, 246, 0, 41, 2, 27, 245, 255, 41, 2, 27, 245, - 254, 41, 2, 27, 245, 253, 41, 2, 27, 245, 252, 41, 2, 27, 245, 251, 41, - 2, 27, 245, 250, 41, 2, 27, 245, 249, 41, 2, 27, 245, 248, 41, 2, 27, - 245, 247, 41, 2, 27, 245, 246, 41, 2, 27, 245, 245, 41, 2, 27, 245, 244, - 41, 2, 27, 245, 243, 41, 2, 27, 245, 242, 41, 2, 27, 245, 241, 41, 2, 27, - 245, 240, 41, 2, 27, 245, 239, 41, 2, 27, 245, 238, 41, 2, 27, 245, 237, - 41, 2, 27, 245, 236, 41, 2, 27, 245, 235, 41, 2, 27, 245, 234, 41, 2, 27, - 245, 233, 41, 2, 27, 245, 232, 41, 2, 27, 245, 231, 41, 2, 27, 245, 230, - 41, 2, 27, 245, 229, 41, 2, 27, 245, 228, 41, 2, 27, 245, 227, 41, 2, 27, - 245, 226, 41, 2, 27, 245, 225, 41, 2, 27, 245, 224, 41, 2, 27, 245, 223, - 41, 2, 27, 245, 222, 41, 2, 27, 245, 221, 41, 2, 27, 245, 220, 41, 2, 27, - 245, 219, 41, 2, 27, 245, 218, 41, 2, 27, 245, 217, 41, 2, 27, 245, 216, - 41, 2, 27, 245, 215, 41, 2, 27, 245, 214, 41, 2, 27, 245, 213, 41, 2, 27, - 245, 212, 41, 2, 27, 245, 211, 41, 2, 27, 245, 210, 41, 2, 27, 245, 209, - 41, 2, 27, 245, 208, 41, 2, 27, 245, 207, 41, 2, 27, 245, 206, 41, 2, 27, - 245, 205, 41, 2, 27, 245, 204, 41, 2, 27, 245, 203, 41, 2, 27, 245, 202, - 41, 2, 27, 245, 201, 41, 2, 27, 245, 200, 41, 2, 27, 245, 199, 41, 2, 27, - 245, 198, 41, 2, 27, 245, 197, 41, 2, 27, 245, 196, 41, 2, 27, 245, 195, - 41, 2, 27, 245, 194, 41, 2, 27, 245, 193, 41, 2, 27, 245, 192, 41, 2, 27, - 245, 191, 41, 2, 27, 245, 190, 41, 2, 27, 245, 189, 41, 2, 27, 245, 188, - 41, 2, 27, 245, 187, 41, 2, 27, 245, 186, 41, 2, 27, 245, 185, 41, 2, 27, - 245, 184, 41, 2, 27, 245, 183, 41, 2, 27, 245, 182, 41, 2, 27, 245, 181, - 41, 2, 27, 245, 180, 41, 2, 27, 245, 179, 41, 2, 27, 245, 178, 41, 2, 27, - 245, 177, 41, 2, 27, 245, 176, 41, 2, 27, 245, 175, 41, 2, 27, 245, 174, - 41, 2, 27, 245, 173, 41, 2, 27, 245, 172, 41, 2, 27, 245, 171, 41, 2, 27, - 245, 170, 41, 2, 27, 245, 169, 41, 2, 27, 245, 168, 41, 2, 27, 245, 167, - 41, 2, 27, 245, 166, 41, 2, 27, 245, 165, 41, 2, 27, 245, 164, 41, 2, 27, - 245, 163, 41, 2, 27, 245, 162, 41, 2, 27, 245, 161, 41, 2, 27, 245, 160, - 41, 2, 27, 245, 159, 41, 2, 27, 245, 158, 41, 2, 27, 245, 157, 41, 2, 27, - 245, 156, 41, 2, 27, 245, 155, 41, 2, 27, 245, 154, 41, 2, 27, 245, 153, - 41, 2, 27, 245, 152, 41, 2, 27, 245, 151, 41, 2, 27, 245, 150, 41, 2, 27, - 245, 149, 41, 2, 27, 245, 148, 41, 2, 27, 245, 147, 41, 2, 27, 245, 146, - 41, 2, 27, 245, 145, 41, 2, 27, 245, 144, 41, 2, 27, 245, 143, 41, 2, 27, - 245, 142, 41, 2, 27, 245, 141, 41, 2, 27, 245, 140, 41, 2, 27, 245, 139, - 41, 2, 27, 245, 138, 41, 2, 27, 245, 137, 41, 2, 27, 245, 136, 41, 2, 27, - 245, 135, 41, 2, 27, 245, 134, 41, 2, 27, 245, 133, 41, 2, 27, 245, 132, - 41, 2, 27, 245, 131, 41, 2, 27, 245, 130, 41, 2, 27, 245, 129, 41, 2, 27, - 245, 128, 41, 2, 27, 245, 127, 41, 2, 27, 245, 126, 41, 2, 27, 245, 125, - 41, 2, 27, 245, 124, 41, 2, 27, 245, 123, 41, 2, 27, 245, 122, 41, 2, 27, - 245, 121, 41, 2, 27, 245, 120, 41, 2, 27, 245, 119, 41, 2, 27, 245, 118, - 41, 2, 27, 245, 117, 41, 2, 27, 245, 116, 41, 2, 27, 245, 115, 41, 2, 27, - 245, 114, 41, 2, 27, 245, 113, 41, 2, 27, 245, 112, 41, 2, 27, 245, 111, - 41, 2, 27, 245, 110, 41, 2, 27, 245, 109, 41, 2, 27, 245, 108, 41, 2, 27, - 245, 107, 41, 2, 27, 245, 106, 41, 2, 27, 245, 105, 41, 2, 27, 245, 104, - 41, 2, 27, 245, 103, 41, 2, 27, 245, 102, 41, 2, 27, 245, 101, 41, 2, 27, - 245, 100, 41, 2, 27, 245, 99, 41, 2, 27, 245, 98, 41, 2, 27, 245, 97, 41, - 2, 27, 245, 96, 41, 2, 27, 245, 95, 41, 2, 27, 245, 94, 41, 2, 27, 245, - 93, 41, 2, 27, 245, 92, 41, 2, 27, 245, 91, 41, 2, 27, 245, 90, 41, 2, - 27, 245, 89, 41, 2, 27, 245, 88, 41, 2, 27, 245, 87, 41, 2, 27, 245, 86, - 41, 2, 27, 245, 85, 41, 2, 27, 245, 84, 41, 2, 27, 245, 83, 41, 2, 27, - 245, 82, 41, 2, 27, 245, 81, 41, 2, 27, 245, 80, 41, 2, 27, 245, 79, 41, - 2, 27, 245, 78, 41, 2, 27, 245, 77, 41, 2, 27, 245, 76, 41, 2, 27, 245, - 75, 41, 2, 27, 245, 74, 41, 2, 27, 245, 73, 41, 2, 27, 245, 72, 41, 2, - 27, 245, 71, 41, 2, 27, 245, 70, 41, 2, 27, 245, 69, 41, 2, 27, 245, 68, - 41, 2, 27, 245, 67, 41, 2, 27, 245, 66, 41, 2, 27, 245, 65, 41, 2, 27, - 245, 64, 41, 2, 27, 245, 63, 41, 2, 27, 245, 62, 41, 2, 27, 245, 61, 41, - 2, 27, 245, 60, 41, 2, 27, 245, 59, 41, 2, 27, 245, 58, 41, 2, 27, 245, - 57, 41, 2, 27, 245, 56, 41, 2, 27, 245, 55, 41, 2, 27, 245, 54, 41, 2, - 27, 245, 53, 41, 2, 27, 245, 52, 41, 2, 27, 245, 51, 41, 2, 27, 245, 50, - 41, 2, 27, 245, 49, 41, 2, 27, 245, 48, 41, 2, 27, 245, 47, 41, 2, 27, - 245, 46, 41, 2, 27, 245, 45, 41, 2, 27, 245, 44, 41, 2, 27, 245, 43, 41, - 2, 27, 245, 42, 41, 2, 27, 245, 41, 41, 2, 27, 245, 40, 41, 2, 27, 245, - 39, 41, 2, 27, 245, 38, 41, 2, 27, 245, 37, 41, 2, 27, 245, 36, 41, 2, - 27, 245, 35, 41, 2, 27, 245, 34, 41, 2, 27, 245, 33, 41, 2, 27, 245, 32, - 41, 2, 27, 245, 31, 41, 2, 27, 245, 30, 41, 2, 27, 245, 29, 41, 2, 27, - 245, 28, 41, 2, 27, 245, 27, 72, 1, 216, 38, 198, 77, 72, 1, 216, 38, - 198, 76, 72, 1, 216, 38, 198, 75, 72, 1, 216, 38, 198, 74, 72, 1, 216, - 38, 198, 72, 72, 1, 216, 38, 198, 71, 72, 1, 216, 38, 214, 213, 198, 78, - 72, 1, 216, 38, 214, 213, 198, 77, 72, 1, 216, 38, 214, 213, 198, 76, 72, - 1, 216, 38, 214, 213, 198, 75, 72, 1, 216, 38, 214, 213, 198, 74, 72, 1, - 216, 38, 214, 213, 198, 72, 72, 1, 216, 38, 214, 213, 198, 71, 72, 1, - 251, 16, 71, 229, 122, 1, 251, 16, 192, 80, 61, 1, 255, 208, 61, 1, 255, - 207, 61, 1, 255, 206, 61, 1, 255, 202, 61, 1, 228, 75, 61, 1, 228, 74, - 61, 1, 228, 73, 61, 1, 228, 72, 61, 1, 196, 231, 61, 1, 196, 230, 61, 1, - 196, 229, 61, 1, 196, 228, 61, 1, 196, 227, 61, 1, 235, 15, 61, 1, 235, - 14, 61, 1, 235, 13, 61, 1, 235, 12, 61, 1, 235, 11, 61, 1, 212, 16, 61, - 1, 212, 15, 61, 1, 212, 14, 61, 1, 222, 143, 61, 1, 222, 140, 61, 1, 222, - 139, 61, 1, 222, 138, 61, 1, 222, 137, 61, 1, 222, 136, 61, 1, 222, 135, - 61, 1, 222, 134, 61, 1, 222, 133, 61, 1, 222, 142, 61, 1, 222, 141, 61, - 1, 222, 132, 61, 1, 221, 167, 61, 1, 221, 166, 61, 1, 221, 165, 61, 1, - 221, 164, 61, 1, 221, 163, 61, 1, 221, 162, 61, 1, 221, 161, 61, 1, 221, - 160, 61, 1, 220, 233, 61, 1, 220, 232, 61, 1, 220, 231, 61, 1, 220, 230, - 61, 1, 220, 229, 61, 1, 220, 228, 61, 1, 220, 227, 61, 1, 222, 23, 61, 1, - 222, 22, 61, 1, 222, 21, 61, 1, 222, 20, 61, 1, 222, 19, 61, 1, 222, 18, - 61, 1, 221, 68, 61, 1, 221, 67, 61, 1, 221, 66, 61, 1, 221, 65, 61, 1, - 205, 207, 61, 1, 205, 206, 61, 1, 205, 205, 61, 1, 205, 204, 61, 1, 205, - 203, 61, 1, 205, 202, 61, 1, 205, 201, 61, 1, 205, 200, 61, 1, 202, 222, - 61, 1, 202, 221, 61, 1, 202, 220, 61, 1, 202, 219, 61, 1, 202, 218, 61, - 1, 202, 217, 61, 1, 201, 4, 61, 1, 201, 3, 61, 1, 201, 2, 61, 1, 201, 1, - 61, 1, 201, 0, 61, 1, 200, 255, 61, 1, 200, 254, 61, 1, 200, 253, 61, 1, - 205, 68, 61, 1, 205, 67, 61, 1, 205, 66, 61, 1, 205, 65, 61, 1, 205, 64, - 61, 1, 202, 46, 61, 1, 202, 45, 61, 1, 202, 44, 61, 1, 202, 43, 61, 1, - 202, 42, 61, 1, 202, 41, 61, 1, 202, 40, 61, 1, 199, 251, 61, 1, 199, - 250, 61, 1, 199, 249, 61, 1, 199, 248, 61, 1, 198, 192, 61, 1, 198, 191, - 61, 1, 198, 190, 61, 1, 198, 189, 61, 1, 198, 188, 61, 1, 198, 187, 61, - 1, 198, 186, 61, 1, 197, 93, 61, 1, 197, 92, 61, 1, 197, 91, 61, 1, 197, - 90, 61, 1, 197, 89, 61, 1, 199, 144, 61, 1, 199, 143, 61, 1, 199, 142, - 61, 1, 199, 141, 61, 1, 199, 140, 61, 1, 199, 139, 61, 1, 199, 138, 61, - 1, 199, 137, 61, 1, 199, 136, 61, 1, 198, 98, 61, 1, 198, 97, 61, 1, 198, - 96, 61, 1, 198, 95, 61, 1, 198, 94, 61, 1, 198, 93, 61, 1, 198, 92, 61, - 1, 215, 8, 61, 1, 215, 7, 61, 1, 215, 6, 61, 1, 215, 5, 61, 1, 215, 4, - 61, 1, 215, 3, 61, 1, 215, 2, 61, 1, 215, 1, 61, 1, 215, 0, 61, 1, 213, - 220, 61, 1, 213, 219, 61, 1, 213, 218, 61, 1, 213, 217, 61, 1, 213, 216, - 61, 1, 213, 215, 61, 1, 213, 214, 61, 1, 213, 213, 61, 1, 212, 179, 61, - 1, 212, 178, 61, 1, 212, 177, 61, 1, 214, 122, 61, 1, 214, 121, 61, 1, - 214, 120, 61, 1, 214, 119, 61, 1, 214, 118, 61, 1, 214, 117, 61, 1, 214, - 116, 61, 1, 213, 44, 61, 1, 213, 43, 61, 1, 213, 42, 61, 1, 213, 41, 61, - 1, 213, 40, 61, 1, 230, 106, 61, 1, 230, 103, 61, 1, 230, 102, 61, 1, - 230, 101, 61, 1, 230, 100, 61, 1, 230, 99, 61, 1, 230, 98, 61, 1, 230, - 97, 61, 1, 230, 96, 61, 1, 230, 105, 61, 1, 230, 104, 61, 1, 229, 159, - 61, 1, 229, 158, 61, 1, 229, 157, 61, 1, 229, 156, 61, 1, 229, 155, 61, - 1, 229, 154, 61, 1, 229, 153, 61, 1, 228, 160, 61, 1, 228, 159, 61, 1, - 228, 158, 61, 1, 229, 246, 61, 1, 229, 245, 61, 1, 229, 244, 61, 1, 229, - 243, 61, 1, 229, 242, 61, 1, 229, 241, 61, 1, 229, 240, 61, 1, 229, 24, - 61, 1, 229, 23, 61, 1, 229, 22, 61, 1, 229, 21, 61, 1, 229, 20, 61, 1, - 229, 19, 61, 1, 229, 18, 61, 1, 229, 17, 61, 1, 217, 161, 61, 1, 217, - 160, 61, 1, 217, 159, 61, 1, 217, 158, 61, 1, 217, 157, 61, 1, 217, 156, - 61, 1, 217, 155, 61, 1, 216, 101, 61, 1, 216, 100, 61, 1, 216, 99, 61, 1, - 216, 98, 61, 1, 216, 97, 61, 1, 216, 96, 61, 1, 216, 95, 61, 1, 215, 156, - 61, 1, 215, 155, 61, 1, 215, 154, 61, 1, 215, 153, 61, 1, 216, 233, 61, - 1, 216, 232, 61, 1, 216, 231, 61, 1, 216, 13, 61, 1, 216, 12, 61, 1, 216, - 11, 61, 1, 216, 10, 61, 1, 216, 9, 61, 1, 216, 8, 61, 1, 192, 148, 61, 1, - 192, 147, 61, 1, 192, 146, 61, 1, 192, 145, 61, 1, 192, 144, 61, 1, 192, - 141, 61, 1, 191, 224, 61, 1, 191, 223, 61, 1, 191, 222, 61, 1, 191, 221, - 61, 1, 192, 11, 61, 1, 192, 10, 61, 1, 192, 9, 61, 1, 192, 8, 61, 1, 192, - 7, 61, 1, 192, 6, 61, 1, 207, 187, 61, 1, 207, 186, 61, 1, 207, 185, 61, - 1, 207, 184, 61, 1, 207, 1, 61, 1, 207, 0, 61, 1, 206, 255, 61, 1, 206, - 254, 61, 1, 206, 253, 61, 1, 206, 252, 61, 1, 206, 251, 61, 1, 206, 68, - 61, 1, 206, 67, 61, 1, 206, 66, 61, 1, 206, 65, 61, 1, 206, 64, 61, 1, - 206, 63, 61, 1, 207, 114, 61, 1, 207, 113, 61, 1, 207, 112, 61, 1, 207, - 111, 61, 1, 206, 162, 61, 1, 206, 161, 61, 1, 206, 160, 61, 1, 206, 159, - 61, 1, 206, 158, 61, 1, 206, 157, 61, 1, 193, 189, 61, 1, 193, 188, 61, - 1, 193, 187, 61, 1, 193, 186, 61, 1, 193, 185, 61, 1, 193, 85, 61, 1, - 193, 84, 61, 1, 193, 83, 61, 1, 193, 82, 61, 1, 193, 81, 61, 1, 193, 124, - 61, 1, 193, 123, 61, 1, 193, 122, 61, 1, 193, 121, 61, 1, 193, 47, 61, 1, - 193, 46, 61, 1, 193, 45, 61, 1, 193, 44, 61, 1, 193, 43, 61, 1, 193, 42, - 61, 1, 193, 41, 61, 1, 215, 60, 61, 1, 215, 59, 229, 122, 1, 251, 16, - 193, 0, 72, 1, 251, 16, 192, 33, 72, 1, 251, 16, 192, 80, 72, 1, 251, 16, - 193, 0, 229, 122, 1, 2, 221, 69, 229, 122, 1, 2, 193, 86, 229, 122, 1, 2, - 193, 125, 229, 122, 1, 2, 193, 48, 72, 1, 2, 221, 69, 72, 1, 2, 193, 86, - 72, 1, 2, 193, 125, 72, 1, 2, 193, 48, 72, 1, 2, 215, 63, 46, 245, 26, - 46, 245, 25, 46, 245, 24, 46, 245, 23, 46, 245, 22, 46, 245, 21, 46, 245, - 20, 46, 245, 19, 46, 245, 18, 46, 245, 17, 46, 245, 16, 46, 245, 15, 46, - 245, 14, 46, 245, 13, 46, 245, 12, 46, 245, 11, 46, 245, 10, 46, 245, 9, - 46, 245, 8, 46, 245, 7, 46, 245, 6, 46, 245, 5, 46, 245, 4, 46, 245, 3, - 46, 245, 2, 46, 245, 1, 46, 245, 0, 46, 244, 255, 46, 244, 254, 46, 244, - 253, 46, 244, 252, 46, 244, 251, 46, 244, 250, 46, 244, 249, 46, 244, - 248, 46, 244, 247, 46, 244, 246, 46, 244, 245, 46, 244, 244, 46, 244, - 243, 46, 244, 242, 46, 244, 241, 46, 244, 240, 46, 244, 239, 46, 244, - 238, 46, 244, 237, 46, 244, 236, 46, 244, 235, 46, 244, 234, 46, 244, - 233, 46, 244, 232, 46, 244, 231, 46, 244, 230, 46, 244, 229, 46, 244, - 228, 46, 244, 227, 46, 244, 226, 46, 244, 225, 46, 244, 224, 46, 244, - 223, 46, 244, 222, 46, 244, 221, 46, 244, 220, 46, 244, 219, 46, 244, - 218, 46, 244, 217, 46, 244, 216, 46, 244, 215, 46, 244, 214, 46, 244, - 213, 46, 244, 212, 46, 244, 211, 46, 244, 210, 46, 244, 209, 46, 244, - 208, 46, 244, 207, 46, 244, 206, 46, 244, 205, 46, 244, 204, 46, 244, - 203, 46, 244, 202, 46, 244, 201, 46, 244, 200, 46, 244, 199, 46, 244, - 198, 46, 244, 197, 46, 244, 196, 46, 244, 195, 46, 244, 194, 46, 244, - 193, 46, 244, 192, 46, 244, 191, 46, 244, 190, 46, 244, 189, 46, 244, - 188, 46, 244, 187, 46, 244, 186, 46, 244, 185, 46, 244, 184, 46, 244, - 183, 46, 244, 182, 46, 244, 181, 46, 244, 180, 46, 244, 179, 46, 244, - 178, 46, 244, 177, 46, 244, 176, 46, 244, 175, 46, 244, 174, 46, 244, - 173, 46, 244, 172, 46, 244, 171, 46, 244, 170, 46, 244, 169, 46, 244, - 168, 46, 244, 167, 46, 244, 166, 46, 244, 165, 46, 244, 164, 46, 244, - 163, 46, 244, 162, 46, 244, 161, 46, 244, 160, 46, 244, 159, 46, 244, - 158, 46, 244, 157, 46, 244, 156, 46, 244, 155, 46, 244, 154, 46, 244, - 153, 46, 244, 152, 46, 244, 151, 46, 244, 150, 46, 244, 149, 46, 244, - 148, 46, 244, 147, 46, 244, 146, 46, 244, 145, 46, 244, 144, 46, 244, - 143, 46, 244, 142, 46, 244, 141, 46, 244, 140, 46, 244, 139, 46, 244, - 138, 46, 244, 137, 46, 244, 136, 46, 244, 135, 46, 244, 134, 46, 244, - 133, 46, 244, 132, 46, 244, 131, 46, 244, 130, 46, 244, 129, 46, 244, - 128, 46, 244, 127, 46, 244, 126, 46, 244, 125, 46, 244, 124, 46, 244, - 123, 46, 244, 122, 46, 244, 121, 46, 244, 120, 46, 244, 119, 46, 244, - 118, 46, 244, 117, 46, 244, 116, 46, 244, 115, 46, 244, 114, 46, 244, - 113, 46, 244, 112, 46, 244, 111, 46, 244, 110, 46, 244, 109, 46, 244, - 108, 46, 244, 107, 46, 244, 106, 46, 244, 105, 46, 244, 104, 46, 244, - 103, 46, 244, 102, 46, 244, 101, 46, 244, 100, 46, 244, 99, 46, 244, 98, - 46, 244, 97, 46, 244, 96, 46, 244, 95, 46, 244, 94, 46, 244, 93, 46, 244, - 92, 46, 244, 91, 46, 244, 90, 46, 244, 89, 46, 244, 88, 46, 244, 87, 46, - 244, 86, 46, 244, 85, 46, 244, 84, 46, 244, 83, 46, 244, 82, 46, 244, 81, - 46, 244, 80, 46, 244, 79, 46, 244, 78, 46, 244, 77, 46, 244, 76, 46, 244, - 75, 46, 244, 74, 46, 244, 73, 46, 244, 72, 46, 244, 71, 46, 244, 70, 46, - 244, 69, 46, 244, 68, 46, 244, 67, 46, 244, 66, 46, 244, 65, 46, 244, 64, - 46, 244, 63, 46, 244, 62, 46, 244, 61, 46, 244, 60, 46, 244, 59, 46, 244, - 58, 46, 244, 57, 46, 244, 56, 46, 244, 55, 46, 244, 54, 46, 244, 53, 46, - 244, 52, 46, 244, 51, 46, 244, 50, 46, 244, 49, 46, 244, 48, 46, 244, 47, - 46, 244, 46, 46, 244, 45, 46, 244, 44, 46, 244, 43, 46, 244, 42, 46, 244, - 41, 46, 244, 40, 46, 244, 39, 46, 244, 38, 46, 244, 37, 46, 244, 36, 46, - 244, 35, 46, 244, 34, 46, 244, 33, 46, 244, 32, 46, 244, 31, 46, 244, 30, - 46, 244, 29, 46, 244, 28, 46, 244, 27, 46, 244, 26, 46, 244, 25, 46, 244, - 24, 46, 244, 23, 46, 244, 22, 46, 244, 21, 46, 244, 20, 46, 244, 19, 46, - 244, 18, 46, 244, 17, 46, 244, 16, 46, 244, 15, 46, 244, 14, 46, 244, 13, - 46, 244, 12, 46, 244, 11, 46, 244, 10, 46, 244, 9, 46, 244, 8, 46, 244, - 7, 46, 244, 6, 46, 244, 5, 46, 244, 4, 46, 244, 3, 46, 244, 2, 46, 244, - 1, 46, 244, 0, 46, 243, 255, 46, 243, 254, 46, 243, 253, 46, 243, 252, - 46, 243, 251, 46, 243, 250, 46, 243, 249, 46, 243, 248, 46, 243, 247, 46, - 243, 246, 46, 243, 245, 46, 243, 244, 46, 243, 243, 46, 243, 242, 46, - 243, 241, 46, 243, 240, 46, 243, 239, 46, 243, 238, 46, 243, 237, 46, - 243, 236, 46, 243, 235, 46, 243, 234, 46, 243, 233, 46, 243, 232, 46, - 243, 231, 46, 243, 230, 46, 243, 229, 46, 243, 228, 46, 243, 227, 46, - 243, 226, 46, 243, 225, 46, 243, 224, 46, 243, 223, 46, 243, 222, 46, - 243, 221, 46, 243, 220, 46, 243, 219, 46, 243, 218, 46, 243, 217, 46, - 243, 216, 46, 243, 215, 46, 243, 214, 46, 243, 213, 46, 243, 212, 46, - 243, 211, 46, 243, 210, 46, 243, 209, 46, 243, 208, 46, 243, 207, 46, - 243, 206, 46, 243, 205, 46, 243, 204, 46, 243, 203, 46, 243, 202, 46, - 243, 201, 46, 243, 200, 46, 243, 199, 46, 243, 198, 46, 243, 197, 46, - 243, 196, 46, 243, 195, 46, 243, 194, 46, 243, 193, 46, 243, 192, 46, - 243, 191, 46, 243, 190, 46, 243, 189, 46, 243, 188, 46, 243, 187, 46, - 243, 186, 46, 243, 185, 46, 243, 184, 46, 243, 183, 46, 243, 182, 46, - 243, 181, 46, 243, 180, 46, 243, 179, 46, 243, 178, 46, 243, 177, 46, - 243, 176, 46, 243, 175, 46, 243, 174, 46, 243, 173, 46, 243, 172, 46, - 243, 171, 46, 243, 170, 46, 243, 169, 46, 243, 168, 46, 243, 167, 46, - 243, 166, 46, 243, 165, 46, 243, 164, 46, 243, 163, 46, 243, 162, 46, - 243, 161, 46, 243, 160, 46, 243, 159, 46, 243, 158, 46, 243, 157, 46, - 243, 156, 46, 243, 155, 46, 243, 154, 46, 243, 153, 46, 243, 152, 46, - 243, 151, 46, 243, 150, 46, 243, 149, 46, 243, 148, 46, 243, 147, 46, - 243, 146, 46, 243, 145, 46, 243, 144, 46, 243, 143, 124, 1, 230, 118, - 124, 1, 192, 235, 124, 1, 210, 238, 124, 1, 200, 43, 124, 1, 233, 177, - 124, 1, 222, 154, 124, 1, 172, 124, 1, 250, 122, 124, 1, 238, 129, 124, - 1, 196, 12, 124, 1, 232, 53, 124, 1, 146, 124, 1, 210, 239, 215, 63, 124, - 1, 238, 130, 206, 9, 124, 1, 233, 178, 215, 63, 124, 1, 222, 155, 218, - 170, 124, 1, 207, 224, 206, 9, 124, 1, 199, 51, 124, 1, 202, 83, 237, 71, - 124, 1, 237, 71, 124, 1, 221, 104, 124, 1, 202, 83, 223, 37, 124, 1, 229, - 114, 124, 1, 219, 162, 124, 1, 207, 8, 124, 1, 218, 170, 124, 1, 215, 63, - 124, 1, 223, 37, 124, 1, 206, 9, 124, 1, 218, 171, 215, 63, 124, 1, 215, - 64, 218, 170, 124, 1, 223, 38, 218, 170, 124, 1, 206, 10, 223, 37, 124, - 1, 218, 171, 4, 236, 142, 124, 1, 215, 64, 4, 236, 142, 124, 1, 223, 38, - 4, 236, 142, 124, 1, 223, 38, 4, 185, 223, 120, 23, 58, 124, 1, 206, 10, - 4, 236, 142, 124, 1, 206, 10, 4, 75, 60, 124, 1, 218, 171, 206, 9, 124, - 1, 215, 64, 206, 9, 124, 1, 223, 38, 206, 9, 124, 1, 206, 10, 206, 9, - 124, 1, 218, 171, 215, 64, 206, 9, 124, 1, 215, 64, 218, 171, 206, 9, - 124, 1, 223, 38, 218, 171, 206, 9, 124, 1, 206, 10, 223, 38, 206, 9, 124, - 1, 223, 38, 206, 10, 4, 236, 142, 124, 1, 223, 38, 215, 63, 124, 1, 223, - 38, 215, 64, 206, 9, 124, 1, 206, 10, 200, 43, 124, 1, 206, 10, 200, 44, - 146, 124, 1, 206, 10, 210, 238, 124, 1, 206, 10, 210, 239, 146, 124, 1, - 200, 44, 206, 9, 124, 1, 200, 44, 207, 224, 206, 9, 124, 1, 193, 224, - 124, 1, 193, 97, 124, 1, 193, 225, 146, 124, 1, 206, 10, 215, 63, 124, 1, - 206, 10, 218, 170, 124, 1, 222, 155, 207, 224, 206, 9, 124, 1, 232, 54, - 207, 224, 206, 9, 124, 1, 206, 10, 222, 154, 124, 1, 206, 10, 222, 155, - 146, 124, 1, 65, 124, 1, 202, 83, 210, 252, 124, 1, 211, 184, 124, 1, 74, - 124, 1, 251, 68, 124, 1, 68, 124, 1, 71, 124, 1, 223, 228, 124, 1, 203, - 41, 68, 124, 1, 196, 139, 124, 1, 234, 190, 124, 1, 202, 83, 234, 175, - 124, 1, 206, 136, 68, 124, 1, 202, 83, 234, 190, 124, 1, 180, 68, 124, 1, - 192, 80, 124, 1, 66, 124, 1, 233, 244, 124, 1, 192, 182, 124, 1, 126, - 215, 63, 124, 1, 180, 66, 124, 1, 206, 136, 66, 124, 1, 196, 141, 124, 1, - 202, 83, 66, 124, 1, 211, 86, 124, 1, 210, 252, 124, 1, 211, 21, 124, 1, - 193, 190, 124, 1, 193, 48, 124, 1, 193, 86, 124, 1, 193, 112, 124, 1, - 193, 14, 124, 1, 214, 216, 66, 124, 1, 214, 216, 74, 124, 1, 214, 216, - 68, 124, 1, 214, 216, 65, 124, 1, 210, 2, 251, 134, 124, 1, 210, 2, 251, - 151, 124, 1, 202, 83, 234, 105, 124, 1, 202, 83, 251, 134, 124, 1, 202, - 83, 211, 106, 124, 1, 117, 218, 170, 124, 252, 6, 45, 228, 243, 205, 59, - 124, 252, 6, 216, 89, 228, 243, 205, 59, 124, 252, 6, 50, 228, 243, 205, - 59, 124, 252, 6, 130, 81, 205, 59, 124, 252, 6, 216, 89, 81, 205, 59, - 124, 252, 6, 137, 81, 205, 59, 124, 252, 6, 250, 172, 205, 59, 124, 252, - 6, 250, 172, 219, 217, 205, 59, 124, 252, 6, 250, 172, 199, 188, 124, - 252, 6, 250, 172, 199, 215, 124, 252, 6, 250, 172, 235, 17, 102, 124, - 252, 6, 250, 172, 228, 76, 102, 124, 252, 6, 250, 172, 199, 189, 102, - 124, 252, 6, 137, 252, 48, 124, 252, 6, 137, 198, 172, 252, 48, 124, 252, - 6, 137, 230, 212, 124, 252, 6, 137, 180, 230, 212, 124, 252, 6, 137, 236, - 142, 124, 252, 6, 137, 243, 12, 124, 252, 6, 137, 219, 114, 124, 252, 6, - 137, 193, 138, 124, 252, 6, 137, 195, 135, 124, 252, 6, 130, 252, 48, - 124, 252, 6, 130, 198, 172, 252, 48, 124, 252, 6, 130, 230, 212, 124, - 252, 6, 130, 180, 230, 212, 124, 252, 6, 130, 236, 142, 124, 252, 6, 130, - 243, 12, 124, 252, 6, 130, 219, 114, 124, 252, 6, 130, 193, 138, 124, - 252, 6, 130, 195, 135, 124, 252, 6, 130, 57, 124, 3, 187, 4, 238, 219, - 124, 199, 9, 1, 205, 35, 124, 55, 77, 124, 208, 154, 243, 80, 232, 82, - 201, 64, 203, 28, 232, 147, 1, 211, 4, 203, 28, 232, 147, 239, 30, 211, - 4, 203, 28, 232, 147, 144, 201, 79, 203, 28, 232, 147, 133, 201, 79, 97, - 33, 87, 230, 242, 213, 161, 206, 10, 220, 253, 211, 107, 219, 226, 97, - 33, 87, 213, 161, 206, 10, 220, 253, 211, 107, 219, 226, 97, 33, 87, 197, - 162, 211, 107, 219, 226, 97, 33, 87, 230, 242, 213, 161, 211, 107, 219, - 226, 97, 33, 87, 213, 161, 211, 107, 219, 226, 97, 33, 87, 201, 180, 211, - 107, 219, 226, 97, 33, 87, 217, 96, 209, 5, 211, 107, 219, 226, 97, 33, - 87, 209, 5, 211, 107, 219, 226, 97, 33, 87, 193, 231, 211, 107, 219, 226, - 97, 33, 87, 217, 96, 209, 5, 206, 10, 221, 183, 211, 107, 219, 226, 97, - 33, 87, 209, 5, 206, 10, 221, 183, 211, 107, 219, 226, 97, 33, 87, 193, - 231, 206, 10, 221, 183, 211, 107, 219, 226, 97, 33, 87, 230, 242, 213, - 161, 206, 10, 220, 253, 211, 107, 179, 97, 33, 87, 213, 161, 206, 10, - 220, 253, 211, 107, 179, 97, 33, 87, 197, 162, 211, 107, 179, 97, 33, 87, - 230, 242, 213, 161, 211, 107, 179, 97, 33, 87, 213, 161, 211, 107, 179, - 97, 33, 87, 201, 180, 211, 107, 179, 97, 33, 87, 217, 96, 209, 5, 211, - 107, 179, 97, 33, 87, 209, 5, 211, 107, 179, 97, 33, 87, 193, 231, 211, - 107, 179, 97, 33, 87, 217, 96, 209, 5, 206, 10, 221, 183, 211, 107, 179, - 97, 33, 87, 209, 5, 206, 10, 221, 183, 211, 107, 179, 97, 33, 87, 193, - 231, 206, 10, 221, 183, 211, 107, 179, 97, 33, 87, 197, 162, 206, 10, - 220, 252, 97, 33, 87, 217, 96, 209, 5, 206, 10, 220, 252, 97, 33, 87, - 201, 50, 217, 96, 209, 4, 97, 33, 87, 209, 5, 206, 10, 220, 252, 97, 33, - 87, 209, 5, 201, 49, 97, 33, 87, 193, 231, 206, 10, 220, 252, 97, 33, 87, - 217, 96, 209, 5, 201, 49, 97, 33, 87, 230, 242, 193, 230, 97, 33, 87, - 191, 83, 97, 33, 87, 211, 106, 97, 33, 87, 207, 126, 97, 33, 87, 198, - 157, 97, 33, 87, 248, 85, 97, 33, 87, 196, 156, 97, 33, 87, 209, 67, 97, - 33, 87, 219, 23, 97, 33, 87, 220, 202, 97, 33, 87, 222, 117, 97, 33, 87, - 191, 74, 97, 33, 87, 202, 106, 97, 33, 87, 207, 119, 97, 33, 87, 220, - 255, 211, 107, 219, 226, 97, 33, 198, 80, 207, 139, 87, 215, 164, 97, 33, - 198, 80, 207, 139, 87, 200, 153, 97, 33, 198, 80, 207, 139, 87, 197, 246, - 97, 33, 87, 191, 120, 97, 33, 87, 237, 107, 191, 120, 97, 33, 87, 211, - 27, 97, 33, 87, 209, 69, 97, 33, 87, 209, 70, 4, 81, 106, 97, 33, 87, - 243, 136, 97, 33, 87, 243, 137, 209, 47, 97, 33, 87, 211, 176, 97, 33, - 87, 202, 11, 212, 251, 97, 33, 87, 198, 89, 97, 33, 87, 235, 50, 97, 33, - 250, 171, 81, 211, 111, 97, 33, 87, 238, 165, 211, 111, 97, 33, 87, 220, - 254, 97, 33, 110, 198, 80, 207, 139, 223, 146, 97, 208, 205, 52, 219, - 169, 97, 208, 205, 52, 219, 168, 97, 208, 205, 52, 236, 235, 232, 197, - 97, 208, 205, 52, 220, 254, 97, 208, 205, 52, 206, 145, 97, 161, 221, 2, - 97, 161, 221, 3, 198, 156, 97, 161, 210, 124, 97, 161, 235, 58, 196, 13, - 243, 115, 97, 161, 221, 92, 97, 161, 191, 105, 97, 161, 201, 62, 97, 161, - 201, 63, 206, 10, 211, 165, 97, 161, 210, 12, 97, 161, 210, 13, 214, 98, - 97, 161, 201, 63, 4, 202, 11, 212, 251, 97, 161, 243, 114, 97, 161, 210, - 188, 97, 161, 191, 103, 97, 161, 230, 250, 248, 84, 97, 161, 230, 250, - 198, 156, 97, 161, 230, 250, 215, 162, 97, 161, 230, 250, 200, 152, 97, - 161, 230, 250, 197, 245, 97, 161, 194, 253, 208, 185, 97, 161, 194, 253, - 215, 165, 97, 161, 194, 253, 200, 154, 97, 161, 194, 253, 197, 247, 97, - 161, 194, 253, 221, 87, 208, 185, 97, 161, 194, 253, 221, 87, 215, 165, - 97, 161, 194, 253, 221, 87, 200, 154, 97, 161, 194, 253, 221, 87, 197, - 247, 97, 161, 55, 191, 103, 97, 161, 207, 19, 243, 114, 97, 161, 237, 93, - 97, 161, 221, 209, 97, 161, 243, 136, 97, 161, 209, 69, 97, 161, 202, - 114, 215, 165, 97, 161, 202, 114, 200, 154, 97, 161, 202, 114, 197, 247, - 97, 161, 202, 114, 198, 157, 97, 161, 237, 107, 221, 92, 97, 161, 202, - 114, 221, 87, 200, 154, 97, 161, 202, 114, 221, 91, 97, 161, 202, 114, - 221, 87, 198, 157, 97, 161, 202, 114, 235, 55, 208, 185, 97, 161, 202, - 114, 235, 55, 200, 154, 97, 161, 202, 114, 235, 55, 214, 98, 97, 161, - 202, 114, 235, 55, 221, 86, 97, 161, 202, 73, 97, 161, 202, 74, 206, 10, - 191, 101, 97, 161, 202, 74, 191, 110, 97, 161, 202, 74, 206, 10, 220, - 252, 97, 161, 220, 254, 97, 161, 206, 145, 97, 161, 232, 230, 97, 161, - 221, 61, 97, 161, 191, 8, 97, 161, 201, 91, 97, 161, 201, 92, 206, 10, - 191, 101, 97, 161, 201, 92, 206, 10, 220, 252, 97, 161, 201, 92, 206, 10, - 191, 102, 228, 76, 220, 252, 97, 161, 201, 92, 206, 10, 220, 253, 228, - 76, 191, 101, 97, 161, 201, 92, 191, 111, 97, 161, 201, 92, 191, 112, - 206, 10, 191, 101, 97, 161, 201, 92, 206, 10, 206, 144, 97, 161, 201, 92, - 206, 10, 235, 49, 191, 100, 97, 161, 201, 92, 206, 10, 191, 102, 228, 76, - 209, 68, 97, 161, 209, 49, 97, 161, 201, 92, 214, 98, 97, 161, 201, 41, - 208, 185, 97, 161, 201, 41, 215, 163, 97, 161, 201, 41, 220, 251, 97, - 161, 201, 41, 209, 45, 97, 161, 201, 41, 209, 7, 97, 161, 201, 41, 214, - 98, 97, 161, 201, 41, 221, 89, 97, 161, 201, 41, 221, 91, 97, 161, 201, - 41, 198, 158, 208, 132, 97, 161, 201, 41, 235, 54, 97, 161, 201, 41, 235, - 53, 97, 161, 201, 41, 235, 51, 97, 161, 201, 41, 235, 55, 221, 86, 97, - 161, 201, 41, 235, 52, 221, 86, 97, 161, 201, 41, 230, 197, 4, 202, 171, - 191, 103, 97, 161, 201, 41, 230, 193, 4, 202, 171, 191, 103, 97, 161, - 201, 41, 230, 196, 97, 161, 201, 41, 230, 192, 97, 161, 201, 41, 230, - 193, 4, 55, 191, 103, 97, 161, 201, 41, 230, 194, 97, 161, 201, 41, 230, - 195, 209, 7, 97, 161, 216, 223, 97, 161, 216, 224, 209, 6, 97, 161, 216, - 224, 221, 85, 97, 161, 216, 224, 221, 88, 97, 161, 216, 224, 221, 90, 97, - 161, 201, 41, 197, 174, 97, 161, 201, 41, 197, 173, 97, 161, 201, 41, - 197, 172, 97, 161, 211, 33, 97, 161, 211, 34, 200, 154, 97, 161, 211, 34, - 197, 247, 97, 161, 211, 34, 221, 1, 200, 154, 97, 161, 211, 34, 221, 87, - 200, 154, 97, 161, 211, 34, 221, 87, 214, 98, 97, 161, 201, 41, 221, 0, - 97, 161, 201, 41, 221, 1, 209, 7, 97, 161, 201, 41, 221, 1, 230, 197, 4, - 202, 171, 191, 103, 97, 161, 201, 41, 221, 1, 230, 193, 4, 202, 171, 191, - 103, 97, 161, 201, 41, 221, 1, 230, 196, 97, 161, 201, 41, 221, 1, 230, - 192, 97, 161, 201, 41, 221, 1, 230, 193, 4, 55, 191, 103, 97, 161, 201, - 41, 221, 1, 230, 194, 97, 161, 201, 41, 221, 1, 230, 195, 209, 7, 97, - 161, 201, 41, 221, 1, 197, 175, 97, 161, 220, 216, 97, 161, 211, 175, 97, - 161, 235, 86, 97, 161, 214, 105, 97, 161, 210, 81, 73, 37, 16, 208, 171, - 73, 37, 16, 237, 219, 73, 37, 16, 210, 6, 73, 37, 16, 210, 248, 234, 146, - 73, 37, 16, 210, 248, 236, 240, 73, 37, 16, 195, 172, 234, 146, 73, 37, - 16, 195, 172, 236, 240, 73, 37, 16, 222, 46, 73, 37, 16, 200, 60, 73, 37, - 16, 210, 122, 73, 37, 16, 191, 231, 73, 37, 16, 191, 232, 236, 240, 73, - 37, 16, 221, 5, 73, 37, 16, 251, 63, 234, 146, 73, 37, 16, 233, 212, 234, - 146, 73, 37, 16, 199, 108, 73, 37, 16, 221, 249, 73, 37, 16, 251, 52, 73, - 37, 16, 251, 53, 236, 240, 73, 37, 16, 200, 67, 73, 37, 16, 198, 244, 73, - 37, 16, 211, 118, 251, 14, 73, 37, 16, 230, 240, 251, 14, 73, 37, 16, - 208, 170, 73, 37, 16, 246, 250, 73, 37, 16, 195, 161, 73, 37, 16, 223, - 60, 251, 14, 73, 37, 16, 221, 251, 251, 14, 73, 37, 16, 221, 250, 251, - 14, 73, 37, 16, 205, 106, 73, 37, 16, 210, 112, 73, 37, 16, 201, 89, 251, - 56, 73, 37, 16, 210, 247, 251, 14, 73, 37, 16, 195, 171, 251, 14, 73, 37, - 16, 251, 57, 251, 14, 73, 37, 16, 251, 50, 73, 37, 16, 221, 94, 73, 37, - 16, 207, 15, 73, 37, 16, 209, 185, 251, 14, 73, 37, 16, 198, 142, 73, 37, - 16, 251, 130, 73, 37, 16, 205, 38, 73, 37, 16, 200, 71, 251, 14, 73, 37, - 16, 200, 71, 216, 168, 201, 87, 73, 37, 16, 210, 242, 251, 14, 73, 37, - 16, 199, 27, 73, 37, 16, 219, 204, 73, 37, 16, 235, 40, 73, 37, 16, 197, - 243, 73, 37, 16, 199, 76, 73, 37, 16, 221, 8, 73, 37, 16, 251, 63, 233, - 212, 214, 100, 73, 37, 16, 232, 90, 251, 14, 73, 37, 16, 223, 177, 73, - 37, 16, 197, 210, 251, 14, 73, 37, 16, 222, 49, 197, 209, 73, 37, 16, - 210, 38, 73, 37, 16, 208, 175, 73, 37, 16, 221, 44, 73, 37, 16, 243, 62, - 251, 14, 73, 37, 16, 207, 137, 73, 37, 16, 210, 126, 251, 14, 73, 37, 16, - 210, 123, 251, 14, 73, 37, 16, 228, 26, 73, 37, 16, 214, 227, 73, 37, 16, - 209, 240, 73, 37, 16, 221, 45, 251, 169, 73, 37, 16, 197, 210, 251, 169, - 73, 37, 16, 201, 56, 73, 37, 16, 230, 191, 73, 37, 16, 223, 60, 214, 100, - 73, 37, 16, 211, 118, 214, 100, 73, 37, 16, 210, 248, 214, 100, 73, 37, - 16, 209, 239, 73, 37, 16, 221, 28, 73, 37, 16, 209, 238, 73, 37, 16, 221, - 7, 73, 37, 16, 210, 39, 214, 100, 73, 37, 16, 221, 250, 214, 101, 251, - 95, 73, 37, 16, 221, 251, 214, 101, 251, 95, 73, 37, 16, 191, 229, 73, - 37, 16, 251, 53, 214, 100, 73, 37, 16, 251, 54, 200, 68, 214, 100, 73, - 37, 16, 191, 230, 73, 37, 16, 221, 6, 73, 37, 16, 234, 141, 73, 37, 16, - 246, 251, 73, 37, 16, 216, 59, 223, 59, 73, 37, 16, 195, 172, 214, 100, - 73, 37, 16, 209, 185, 214, 100, 73, 37, 16, 208, 176, 214, 100, 73, 37, - 16, 211, 114, 73, 37, 16, 251, 82, 73, 37, 16, 218, 181, 73, 37, 16, 210, - 123, 214, 100, 73, 37, 16, 210, 126, 214, 100, 73, 37, 16, 233, 251, 210, - 125, 73, 37, 16, 220, 149, 73, 37, 16, 251, 83, 73, 37, 16, 197, 210, - 214, 100, 73, 37, 16, 234, 144, 73, 37, 16, 200, 71, 214, 100, 73, 37, - 16, 200, 61, 73, 37, 16, 243, 62, 214, 100, 73, 37, 16, 234, 55, 73, 37, - 16, 205, 39, 214, 100, 73, 37, 16, 192, 199, 221, 94, 73, 37, 16, 197, - 207, 73, 37, 16, 208, 177, 73, 37, 16, 197, 211, 73, 37, 16, 197, 208, - 73, 37, 16, 208, 174, 73, 37, 16, 197, 206, 73, 37, 16, 208, 173, 73, 37, - 16, 230, 239, 73, 37, 16, 251, 6, 73, 37, 16, 233, 251, 251, 6, 73, 37, - 16, 210, 242, 214, 100, 73, 37, 16, 199, 26, 234, 8, 73, 37, 16, 199, 26, - 233, 211, 73, 37, 16, 199, 28, 251, 58, 73, 37, 16, 199, 20, 222, 105, - 251, 49, 73, 37, 16, 222, 48, 73, 37, 16, 234, 92, 73, 37, 16, 192, 38, - 222, 45, 73, 37, 16, 192, 38, 251, 95, 73, 37, 16, 201, 88, 73, 37, 16, - 221, 95, 251, 95, 73, 37, 16, 236, 241, 251, 14, 73, 37, 16, 221, 9, 251, - 14, 73, 37, 16, 221, 9, 251, 169, 73, 37, 16, 221, 9, 214, 100, 73, 37, - 16, 251, 57, 214, 100, 73, 37, 16, 251, 59, 73, 37, 16, 236, 240, 73, 37, - 16, 197, 223, 73, 37, 16, 199, 66, 73, 37, 16, 221, 32, 73, 37, 16, 219, - 209, 234, 85, 243, 52, 73, 37, 16, 219, 209, 235, 41, 243, 53, 73, 37, - 16, 219, 209, 197, 226, 243, 53, 73, 37, 16, 219, 209, 199, 78, 243, 53, - 73, 37, 16, 219, 209, 223, 172, 243, 52, 73, 37, 16, 230, 240, 214, 101, - 251, 95, 73, 37, 16, 230, 240, 210, 113, 251, 2, 73, 37, 16, 230, 240, - 210, 113, 237, 75, 73, 37, 16, 237, 9, 73, 37, 16, 237, 10, 210, 113, - 251, 3, 222, 45, 73, 37, 16, 237, 10, 210, 113, 251, 3, 251, 95, 73, 37, - 16, 237, 10, 210, 113, 237, 75, 73, 37, 16, 197, 232, 73, 37, 16, 251, 7, - 73, 37, 16, 223, 179, 73, 37, 16, 237, 32, 73, 37, 16, 251, 249, 209, 53, - 251, 8, 73, 37, 16, 251, 249, 251, 5, 73, 37, 16, 251, 249, 251, 8, 73, - 37, 16, 251, 249, 216, 162, 73, 37, 16, 251, 249, 216, 173, 73, 37, 16, - 251, 249, 230, 241, 73, 37, 16, 251, 249, 230, 238, 73, 37, 16, 251, 249, - 209, 53, 230, 241, 73, 37, 16, 217, 47, 208, 183, 228, 24, 73, 37, 16, - 217, 47, 251, 171, 208, 183, 228, 24, 73, 37, 16, 217, 47, 237, 74, 228, - 24, 73, 37, 16, 217, 47, 251, 171, 237, 74, 228, 24, 73, 37, 16, 217, 47, - 197, 218, 228, 24, 73, 37, 16, 217, 47, 197, 233, 73, 37, 16, 217, 47, - 199, 71, 228, 24, 73, 37, 16, 217, 47, 199, 71, 219, 213, 228, 24, 73, - 37, 16, 217, 47, 219, 213, 228, 24, 73, 37, 16, 217, 47, 209, 107, 228, - 24, 73, 37, 16, 223, 68, 199, 101, 228, 25, 73, 37, 16, 251, 54, 199, - 101, 228, 25, 73, 37, 16, 233, 81, 199, 68, 73, 37, 16, 233, 81, 215, - 224, 73, 37, 16, 233, 81, 237, 15, 73, 37, 16, 217, 47, 195, 165, 228, - 24, 73, 37, 16, 217, 47, 208, 182, 228, 24, 73, 37, 16, 217, 47, 209, - 107, 199, 71, 228, 24, 73, 37, 16, 230, 235, 215, 64, 251, 58, 73, 37, - 16, 230, 235, 215, 64, 236, 239, 73, 37, 16, 234, 103, 222, 105, 232, 90, - 195, 3, 73, 37, 16, 223, 178, 73, 37, 16, 223, 176, 73, 37, 16, 232, 90, - 251, 15, 237, 73, 228, 23, 73, 37, 16, 232, 90, 237, 30, 168, 73, 37, 16, - 232, 90, 237, 30, 214, 227, 73, 37, 16, 232, 90, 214, 221, 228, 24, 73, - 37, 16, 232, 90, 237, 30, 237, 46, 73, 37, 16, 232, 90, 202, 107, 237, - 29, 237, 46, 73, 37, 16, 232, 90, 237, 30, 222, 24, 73, 37, 16, 232, 90, - 237, 30, 191, 7, 73, 37, 16, 232, 90, 237, 30, 213, 222, 222, 45, 73, 37, - 16, 232, 90, 237, 30, 213, 222, 251, 95, 73, 37, 16, 232, 90, 217, 100, - 243, 54, 237, 15, 73, 37, 16, 232, 90, 217, 100, 243, 54, 215, 224, 73, - 37, 16, 233, 26, 202, 107, 243, 54, 195, 164, 73, 37, 16, 232, 90, 202, - 107, 243, 54, 200, 72, 73, 37, 16, 232, 90, 214, 103, 73, 37, 16, 243, - 55, 190, 230, 73, 37, 16, 243, 55, 221, 93, 73, 37, 16, 243, 55, 201, - 239, 73, 37, 16, 232, 90, 228, 76, 192, 37, 199, 72, 73, 37, 16, 232, 90, - 234, 104, 251, 84, 73, 37, 16, 192, 37, 197, 219, 73, 37, 16, 237, 23, - 197, 219, 73, 37, 16, 237, 23, 199, 72, 73, 37, 16, 237, 23, 251, 60, - 235, 41, 236, 168, 73, 37, 16, 237, 23, 215, 222, 199, 77, 236, 168, 73, - 37, 16, 237, 23, 237, 6, 233, 224, 236, 168, 73, 37, 16, 237, 23, 197, - 230, 211, 124, 236, 168, 73, 37, 16, 192, 37, 251, 60, 235, 41, 236, 168, - 73, 37, 16, 192, 37, 215, 222, 199, 77, 236, 168, 73, 37, 16, 192, 37, - 237, 6, 233, 224, 236, 168, 73, 37, 16, 192, 37, 197, 230, 211, 124, 236, - 168, 73, 37, 16, 231, 149, 237, 22, 73, 37, 16, 231, 149, 192, 36, 73, - 37, 16, 237, 31, 251, 60, 216, 60, 73, 37, 16, 237, 31, 251, 60, 216, - 204, 73, 37, 16, 237, 31, 236, 240, 73, 37, 16, 237, 31, 199, 18, 73, 37, - 16, 202, 182, 199, 18, 73, 37, 16, 202, 182, 199, 19, 236, 223, 73, 37, - 16, 202, 182, 199, 19, 197, 220, 73, 37, 16, 202, 182, 199, 19, 199, 64, - 73, 37, 16, 202, 182, 250, 230, 73, 37, 16, 202, 182, 250, 231, 236, 223, - 73, 37, 16, 202, 182, 250, 231, 197, 220, 73, 37, 16, 202, 182, 250, 231, - 199, 64, 73, 37, 16, 237, 7, 231, 130, 73, 37, 16, 237, 14, 211, 21, 73, - 37, 16, 201, 74, 73, 37, 16, 250, 255, 168, 73, 37, 16, 250, 255, 195, 3, - 73, 37, 16, 250, 255, 231, 242, 73, 37, 16, 250, 255, 237, 46, 73, 37, - 16, 250, 255, 222, 24, 73, 37, 16, 250, 255, 191, 7, 73, 37, 16, 250, - 255, 213, 221, 73, 37, 16, 221, 250, 214, 101, 216, 172, 73, 37, 16, 221, - 251, 214, 101, 216, 172, 73, 37, 16, 221, 250, 214, 101, 222, 45, 73, 37, - 16, 221, 251, 214, 101, 222, 45, 73, 37, 16, 221, 95, 222, 45, 73, 37, - 16, 230, 240, 214, 101, 222, 45, 37, 16, 202, 171, 249, 85, 37, 16, 55, - 249, 85, 37, 16, 53, 249, 85, 37, 16, 207, 20, 53, 249, 85, 37, 16, 237, - 216, 249, 85, 37, 16, 203, 41, 249, 85, 37, 16, 45, 207, 50, 56, 37, 16, - 50, 207, 50, 56, 37, 16, 207, 50, 236, 140, 37, 16, 238, 6, 205, 42, 37, - 16, 238, 35, 247, 110, 37, 16, 205, 42, 37, 16, 242, 94, 37, 16, 207, 48, - 233, 13, 37, 16, 207, 48, 233, 12, 37, 16, 207, 48, 233, 11, 37, 16, 233, - 36, 37, 16, 233, 37, 60, 37, 16, 248, 41, 77, 37, 16, 247, 152, 37, 16, - 248, 57, 37, 16, 248, 55, 37, 16, 211, 101, 201, 112, 37, 16, 197, 12, - 201, 112, 37, 16, 198, 220, 201, 112, 37, 16, 232, 129, 201, 112, 37, 16, - 232, 226, 201, 112, 37, 16, 202, 136, 201, 112, 37, 16, 202, 134, 232, - 107, 37, 16, 232, 127, 232, 107, 37, 16, 232, 54, 242, 237, 37, 16, 232, - 54, 242, 238, 211, 25, 251, 160, 37, 16, 232, 54, 242, 238, 211, 25, 249, - 68, 37, 16, 247, 196, 242, 237, 37, 16, 233, 178, 242, 237, 37, 16, 233, - 178, 242, 238, 211, 25, 251, 160, 37, 16, 233, 178, 242, 238, 211, 25, - 249, 68, 37, 16, 235, 97, 242, 236, 37, 16, 235, 97, 242, 235, 37, 16, - 215, 131, 216, 230, 207, 31, 37, 16, 55, 203, 127, 37, 16, 55, 232, 208, - 37, 16, 232, 209, 196, 77, 37, 16, 232, 209, 235, 125, 37, 16, 214, 208, - 196, 77, 37, 16, 214, 208, 235, 125, 37, 16, 203, 128, 196, 77, 37, 16, - 203, 128, 235, 125, 37, 16, 208, 25, 163, 203, 127, 37, 16, 208, 25, 163, - 232, 208, 37, 16, 242, 73, 198, 146, 37, 16, 238, 157, 198, 146, 37, 16, - 211, 25, 251, 160, 37, 16, 211, 25, 249, 68, 37, 16, 208, 5, 251, 160, - 37, 16, 208, 5, 249, 68, 37, 16, 215, 134, 207, 31, 37, 16, 193, 87, 207, - 31, 37, 16, 132, 207, 31, 37, 16, 208, 25, 207, 31, 37, 16, 234, 162, - 207, 31, 37, 16, 202, 130, 207, 31, 37, 16, 198, 246, 207, 31, 37, 16, - 202, 120, 207, 31, 37, 16, 91, 228, 143, 197, 30, 207, 31, 37, 16, 192, - 236, 213, 4, 37, 16, 108, 213, 4, 37, 16, 243, 13, 192, 236, 213, 4, 37, - 16, 51, 213, 5, 193, 89, 37, 16, 51, 213, 5, 248, 141, 37, 16, 197, 242, - 213, 5, 133, 193, 89, 37, 16, 197, 242, 213, 5, 133, 248, 141, 37, 16, - 197, 242, 213, 5, 45, 193, 89, 37, 16, 197, 242, 213, 5, 45, 248, 141, - 37, 16, 197, 242, 213, 5, 50, 193, 89, 37, 16, 197, 242, 213, 5, 50, 248, - 141, 37, 16, 197, 242, 213, 5, 144, 193, 89, 37, 16, 197, 242, 213, 5, - 144, 248, 141, 37, 16, 197, 242, 213, 5, 133, 50, 193, 89, 37, 16, 197, - 242, 213, 5, 133, 50, 248, 141, 37, 16, 215, 208, 213, 5, 193, 89, 37, - 16, 215, 208, 213, 5, 248, 141, 37, 16, 197, 239, 213, 5, 144, 193, 89, - 37, 16, 197, 239, 213, 5, 144, 248, 141, 37, 16, 210, 116, 213, 4, 37, - 16, 195, 17, 213, 4, 37, 16, 213, 5, 248, 141, 37, 16, 212, 141, 213, 4, - 37, 16, 242, 205, 213, 5, 193, 89, 37, 16, 242, 205, 213, 5, 248, 141, - 37, 16, 248, 38, 37, 16, 193, 87, 213, 8, 37, 16, 132, 213, 8, 37, 16, - 208, 25, 213, 8, 37, 16, 234, 162, 213, 8, 37, 16, 202, 130, 213, 8, 37, - 16, 198, 246, 213, 8, 37, 16, 202, 120, 213, 8, 37, 16, 91, 228, 143, - 197, 30, 213, 8, 37, 16, 33, 201, 81, 37, 16, 33, 201, 198, 201, 81, 37, - 16, 33, 197, 253, 37, 16, 33, 197, 252, 37, 16, 33, 197, 251, 37, 16, - 232, 252, 197, 253, 37, 16, 232, 252, 197, 252, 37, 16, 232, 252, 197, - 251, 37, 16, 33, 250, 162, 236, 142, 37, 16, 33, 232, 218, 37, 16, 33, - 232, 217, 37, 16, 33, 232, 216, 37, 16, 33, 232, 215, 37, 16, 33, 232, - 214, 37, 16, 248, 251, 249, 16, 37, 16, 234, 97, 249, 16, 37, 16, 248, - 251, 198, 178, 37, 16, 234, 97, 198, 178, 37, 16, 248, 251, 202, 72, 37, - 16, 234, 97, 202, 72, 37, 16, 248, 251, 209, 194, 37, 16, 234, 97, 209, - 194, 37, 16, 33, 252, 62, 37, 16, 33, 201, 116, 37, 16, 33, 199, 83, 37, - 16, 33, 201, 117, 37, 16, 33, 217, 62, 37, 16, 33, 217, 61, 37, 16, 33, - 252, 61, 37, 16, 33, 218, 245, 37, 16, 250, 243, 196, 77, 37, 16, 250, - 243, 235, 125, 37, 16, 33, 236, 160, 37, 16, 33, 206, 185, 37, 16, 33, - 232, 197, 37, 16, 33, 202, 68, 37, 16, 33, 248, 229, 37, 16, 33, 55, 198, - 61, 37, 16, 33, 197, 225, 198, 61, 37, 16, 206, 191, 37, 16, 200, 242, - 37, 16, 191, 166, 37, 16, 209, 186, 37, 16, 216, 153, 37, 16, 232, 142, - 37, 16, 238, 231, 37, 16, 237, 134, 37, 16, 230, 230, 213, 9, 202, 98, - 37, 16, 230, 230, 213, 9, 213, 46, 202, 98, 37, 16, 198, 27, 37, 16, 197, - 58, 37, 16, 223, 95, 197, 58, 37, 16, 197, 59, 202, 98, 37, 16, 197, 59, - 196, 77, 37, 16, 211, 45, 201, 28, 37, 16, 211, 45, 201, 25, 37, 16, 211, - 45, 201, 24, 37, 16, 211, 45, 201, 23, 37, 16, 211, 45, 201, 22, 37, 16, - 211, 45, 201, 21, 37, 16, 211, 45, 201, 20, 37, 16, 211, 45, 201, 19, 37, - 16, 211, 45, 201, 18, 37, 16, 211, 45, 201, 27, 37, 16, 211, 45, 201, 26, - 37, 16, 230, 2, 37, 16, 214, 115, 37, 16, 234, 97, 79, 201, 70, 37, 16, - 237, 127, 202, 98, 37, 16, 33, 144, 248, 71, 37, 16, 33, 133, 248, 71, - 37, 16, 33, 230, 16, 37, 16, 33, 202, 58, 209, 113, 37, 16, 210, 56, 77, - 37, 16, 210, 56, 133, 77, 37, 16, 132, 210, 56, 77, 37, 16, 231, 15, 196, - 77, 37, 16, 231, 15, 235, 125, 37, 16, 4, 232, 251, 37, 16, 237, 243, 37, - 16, 237, 244, 251, 176, 37, 16, 217, 25, 37, 16, 219, 12, 37, 16, 248, - 35, 37, 16, 204, 30, 193, 89, 37, 16, 204, 30, 248, 141, 37, 16, 216, 41, - 37, 16, 216, 42, 248, 141, 37, 16, 204, 24, 193, 89, 37, 16, 204, 24, - 248, 141, 37, 16, 232, 72, 193, 89, 37, 16, 232, 72, 248, 141, 37, 16, - 219, 13, 210, 11, 207, 31, 37, 16, 219, 13, 223, 169, 207, 31, 37, 16, - 248, 36, 207, 31, 37, 16, 204, 30, 207, 31, 37, 16, 216, 42, 207, 31, 37, - 16, 204, 24, 207, 31, 37, 16, 199, 97, 210, 9, 238, 188, 208, 194, 210, - 10, 37, 16, 199, 97, 210, 9, 238, 188, 208, 194, 223, 168, 37, 16, 199, - 97, 210, 9, 238, 188, 208, 194, 210, 11, 236, 250, 37, 16, 199, 97, 223, - 167, 238, 188, 208, 194, 210, 10, 37, 16, 199, 97, 223, 167, 238, 188, - 208, 194, 223, 168, 37, 16, 199, 97, 223, 167, 238, 188, 208, 194, 223, - 169, 236, 250, 37, 16, 199, 97, 223, 167, 238, 188, 208, 194, 223, 169, - 236, 249, 37, 16, 199, 97, 223, 167, 238, 188, 208, 194, 223, 169, 236, - 248, 37, 16, 238, 222, 37, 16, 230, 201, 247, 196, 242, 237, 37, 16, 230, - 201, 233, 178, 242, 237, 37, 16, 51, 250, 122, 37, 16, 195, 39, 37, 16, - 209, 71, 37, 16, 242, 226, 37, 16, 205, 96, 37, 16, 242, 231, 37, 16, - 198, 47, 37, 16, 209, 30, 37, 16, 209, 31, 232, 200, 37, 16, 205, 97, - 232, 200, 37, 16, 198, 48, 207, 28, 37, 16, 209, 248, 200, 232, 37, 16, - 221, 150, 247, 196, 242, 237, 37, 16, 221, 150, 234, 97, 79, 209, 177, - 37, 16, 221, 150, 53, 213, 8, 37, 16, 221, 150, 207, 101, 77, 37, 16, - 221, 150, 193, 87, 213, 8, 37, 16, 221, 150, 132, 213, 8, 37, 16, 221, - 150, 208, 25, 213, 9, 201, 82, 235, 125, 37, 16, 221, 150, 208, 25, 213, - 9, 201, 82, 196, 77, 37, 16, 221, 150, 234, 162, 213, 9, 201, 82, 235, - 125, 37, 16, 221, 150, 234, 162, 213, 9, 201, 82, 196, 77, 37, 16, 221, - 150, 232, 209, 56, 37, 16, 202, 12, 37, 16, 221, 33, 35, 195, 23, 213, - 12, 200, 123, 35, 195, 23, 213, 12, 200, 112, 35, 195, 23, 213, 12, 200, - 102, 35, 195, 23, 213, 12, 200, 95, 35, 195, 23, 213, 12, 200, 87, 35, - 195, 23, 213, 12, 200, 81, 35, 195, 23, 213, 12, 200, 80, 35, 195, 23, - 213, 12, 200, 79, 35, 195, 23, 213, 12, 200, 78, 35, 195, 23, 213, 12, - 200, 122, 35, 195, 23, 213, 12, 200, 121, 35, 195, 23, 213, 12, 200, 120, - 35, 195, 23, 213, 12, 200, 119, 35, 195, 23, 213, 12, 200, 118, 35, 195, - 23, 213, 12, 200, 117, 35, 195, 23, 213, 12, 200, 116, 35, 195, 23, 213, - 12, 200, 115, 35, 195, 23, 213, 12, 200, 114, 35, 195, 23, 213, 12, 200, - 113, 35, 195, 23, 213, 12, 200, 111, 35, 195, 23, 213, 12, 200, 110, 35, - 195, 23, 213, 12, 200, 109, 35, 195, 23, 213, 12, 200, 108, 35, 195, 23, - 213, 12, 200, 107, 35, 195, 23, 213, 12, 200, 86, 35, 195, 23, 213, 12, - 200, 85, 35, 195, 23, 213, 12, 200, 84, 35, 195, 23, 213, 12, 200, 83, - 35, 195, 23, 213, 12, 200, 82, 35, 223, 118, 213, 12, 200, 123, 35, 223, - 118, 213, 12, 200, 112, 35, 223, 118, 213, 12, 200, 95, 35, 223, 118, - 213, 12, 200, 87, 35, 223, 118, 213, 12, 200, 80, 35, 223, 118, 213, 12, - 200, 79, 35, 223, 118, 213, 12, 200, 121, 35, 223, 118, 213, 12, 200, - 120, 35, 223, 118, 213, 12, 200, 119, 35, 223, 118, 213, 12, 200, 118, - 35, 223, 118, 213, 12, 200, 115, 35, 223, 118, 213, 12, 200, 114, 35, - 223, 118, 213, 12, 200, 113, 35, 223, 118, 213, 12, 200, 108, 35, 223, - 118, 213, 12, 200, 107, 35, 223, 118, 213, 12, 200, 106, 35, 223, 118, - 213, 12, 200, 105, 35, 223, 118, 213, 12, 200, 104, 35, 223, 118, 213, - 12, 200, 103, 35, 223, 118, 213, 12, 200, 101, 35, 223, 118, 213, 12, - 200, 100, 35, 223, 118, 213, 12, 200, 99, 35, 223, 118, 213, 12, 200, 98, - 35, 223, 118, 213, 12, 200, 97, 35, 223, 118, 213, 12, 200, 96, 35, 223, - 118, 213, 12, 200, 94, 35, 223, 118, 213, 12, 200, 93, 35, 223, 118, 213, - 12, 200, 92, 35, 223, 118, 213, 12, 200, 91, 35, 223, 118, 213, 12, 200, - 90, 35, 223, 118, 213, 12, 200, 89, 35, 223, 118, 213, 12, 200, 88, 35, - 223, 118, 213, 12, 200, 86, 35, 223, 118, 213, 12, 200, 85, 35, 223, 118, - 213, 12, 200, 84, 35, 223, 118, 213, 12, 200, 83, 35, 223, 118, 213, 12, - 200, 82, 33, 35, 37, 197, 221, 33, 35, 37, 199, 65, 33, 35, 37, 210, 24, - 35, 37, 219, 208, 222, 95, 212, 136, 191, 77, 222, 95, 212, 136, 107, - 222, 95, 212, 136, 109, 222, 95, 212, 136, 138, 222, 95, 212, 136, 134, - 222, 95, 212, 136, 150, 222, 95, 212, 136, 169, 222, 95, 212, 136, 175, - 222, 95, 212, 136, 171, 222, 95, 212, 136, 178, 222, 95, 212, 136, 199, - 95, 222, 95, 212, 136, 234, 129, 222, 95, 212, 136, 197, 37, 222, 95, - 212, 136, 198, 251, 222, 95, 212, 136, 232, 124, 222, 95, 212, 136, 233, - 21, 222, 95, 212, 136, 202, 131, 222, 95, 212, 136, 203, 245, 222, 95, - 212, 136, 234, 163, 222, 95, 212, 136, 213, 173, 217, 22, 212, 136, 191, - 77, 217, 22, 212, 136, 107, 217, 22, 212, 136, 109, 217, 22, 212, 136, - 138, 217, 22, 212, 136, 134, 217, 22, 212, 136, 150, 217, 22, 212, 136, - 169, 217, 22, 212, 136, 175, 217, 22, 212, 136, 171, 217, 22, 212, 136, - 178, 217, 22, 212, 136, 199, 95, 217, 22, 212, 136, 234, 129, 217, 22, - 212, 136, 197, 37, 217, 22, 212, 136, 198, 251, 217, 22, 212, 136, 232, - 124, 217, 22, 212, 136, 233, 21, 217, 22, 212, 136, 202, 131, 217, 22, - 212, 136, 203, 245, 217, 22, 212, 136, 234, 163, 217, 22, 212, 136, 213, - 173, 215, 223, 40, 234, 209, 237, 8, 40, 229, 220, 234, 209, 237, 8, 40, - 228, 147, 234, 209, 237, 8, 40, 234, 208, 229, 221, 237, 8, 40, 234, 208, - 228, 146, 237, 8, 40, 234, 209, 199, 67, 40, 247, 23, 199, 67, 40, 232, - 82, 243, 12, 199, 67, 40, 216, 32, 199, 67, 40, 249, 80, 199, 67, 40, - 222, 12, 202, 71, 199, 67, 40, 239, 25, 199, 67, 40, 250, 214, 199, 67, - 40, 211, 64, 199, 67, 40, 248, 47, 211, 16, 199, 67, 40, 237, 129, 211, - 59, 236, 215, 199, 67, 40, 236, 212, 199, 67, 40, 191, 237, 199, 67, 40, - 223, 155, 199, 67, 40, 210, 34, 199, 67, 40, 207, 110, 199, 67, 40, 239, - 37, 199, 67, 40, 229, 8, 249, 148, 199, 67, 40, 193, 171, 199, 67, 40, - 232, 171, 199, 67, 40, 252, 30, 199, 67, 40, 207, 63, 199, 67, 40, 207, - 35, 199, 67, 40, 234, 207, 199, 67, 40, 222, 187, 199, 67, 40, 239, 32, - 199, 67, 40, 234, 95, 199, 67, 40, 235, 61, 199, 67, 40, 246, 246, 199, - 67, 40, 237, 139, 199, 67, 40, 28, 207, 34, 199, 67, 40, 210, 213, 199, - 67, 40, 219, 212, 199, 67, 40, 242, 219, 199, 67, 40, 221, 138, 199, 67, - 40, 231, 191, 199, 67, 40, 201, 40, 199, 67, 40, 208, 142, 199, 67, 40, - 232, 81, 199, 67, 40, 207, 36, 199, 67, 40, 219, 253, 211, 59, 216, 4, - 199, 67, 40, 207, 32, 199, 67, 40, 230, 254, 119, 216, 208, 199, 67, 40, - 234, 98, 199, 67, 40, 201, 57, 199, 67, 40, 230, 204, 199, 67, 40, 234, - 88, 199, 67, 40, 210, 85, 199, 67, 40, 206, 178, 199, 67, 40, 232, 198, - 199, 67, 40, 195, 163, 211, 59, 193, 147, 199, 67, 40, 239, 42, 199, 67, - 40, 216, 229, 199, 67, 40, 233, 252, 199, 67, 40, 196, 88, 199, 67, 40, - 236, 251, 199, 67, 40, 242, 221, 215, 181, 199, 67, 40, 230, 173, 199, - 67, 40, 231, 192, 223, 164, 199, 67, 40, 217, 34, 199, 67, 40, 252, 56, - 199, 67, 40, 234, 114, 199, 67, 40, 235, 129, 199, 67, 40, 193, 145, 199, - 67, 40, 202, 166, 199, 67, 40, 223, 128, 199, 67, 40, 237, 95, 199, 67, - 40, 237, 221, 199, 67, 40, 236, 247, 199, 67, 40, 233, 215, 199, 67, 40, - 203, 241, 199, 67, 40, 201, 61, 199, 67, 40, 230, 18, 199, 67, 40, 242, - 68, 199, 67, 40, 242, 216, 199, 67, 40, 233, 90, 199, 67, 40, 251, 250, - 199, 67, 40, 242, 67, 199, 67, 40, 211, 107, 199, 34, 195, 138, 199, 67, - 40, 237, 17, 199, 67, 40, 220, 115, 199, 67, 40, 232, 133, 238, 246, 206, - 146, 196, 91, 17, 107, 238, 246, 206, 146, 196, 91, 17, 109, 238, 246, - 206, 146, 196, 91, 17, 138, 238, 246, 206, 146, 196, 91, 17, 134, 238, - 246, 206, 146, 196, 91, 17, 150, 238, 246, 206, 146, 196, 91, 17, 169, - 238, 246, 206, 146, 196, 91, 17, 175, 238, 246, 206, 146, 196, 91, 17, - 171, 238, 246, 206, 146, 196, 91, 17, 178, 238, 246, 206, 146, 199, 91, - 17, 107, 238, 246, 206, 146, 199, 91, 17, 109, 238, 246, 206, 146, 199, - 91, 17, 138, 238, 246, 206, 146, 199, 91, 17, 134, 238, 246, 206, 146, - 199, 91, 17, 150, 238, 246, 206, 146, 199, 91, 17, 169, 238, 246, 206, - 146, 199, 91, 17, 175, 238, 246, 206, 146, 199, 91, 17, 171, 238, 246, - 206, 146, 199, 91, 17, 178, 148, 199, 198, 87, 107, 148, 199, 198, 87, - 109, 148, 199, 198, 87, 138, 148, 199, 198, 87, 134, 148, 199, 198, 87, - 150, 199, 198, 87, 107, 199, 198, 87, 150, 13, 28, 6, 65, 13, 28, 6, 250, - 122, 13, 28, 6, 247, 195, 13, 28, 6, 238, 129, 13, 28, 6, 71, 13, 28, 6, - 233, 177, 13, 28, 6, 232, 53, 13, 28, 6, 230, 118, 13, 28, 6, 68, 13, 28, - 6, 223, 37, 13, 28, 6, 222, 154, 13, 28, 6, 172, 13, 28, 6, 218, 170, 13, - 28, 6, 215, 63, 13, 28, 6, 74, 13, 28, 6, 210, 238, 13, 28, 6, 208, 106, - 13, 28, 6, 146, 13, 28, 6, 206, 9, 13, 28, 6, 200, 43, 13, 28, 6, 66, 13, - 28, 6, 196, 12, 13, 28, 6, 193, 224, 13, 28, 6, 192, 235, 13, 28, 6, 192, - 159, 13, 28, 6, 191, 166, 13, 28, 2, 65, 13, 28, 2, 250, 122, 13, 28, 2, - 247, 195, 13, 28, 2, 238, 129, 13, 28, 2, 71, 13, 28, 2, 233, 177, 13, - 28, 2, 232, 53, 13, 28, 2, 230, 118, 13, 28, 2, 68, 13, 28, 2, 223, 37, - 13, 28, 2, 222, 154, 13, 28, 2, 172, 13, 28, 2, 218, 170, 13, 28, 2, 215, - 63, 13, 28, 2, 74, 13, 28, 2, 210, 238, 13, 28, 2, 208, 106, 13, 28, 2, - 146, 13, 28, 2, 206, 9, 13, 28, 2, 200, 43, 13, 28, 2, 66, 13, 28, 2, - 196, 12, 13, 28, 2, 193, 224, 13, 28, 2, 192, 235, 13, 28, 2, 192, 159, - 13, 28, 2, 191, 166, 13, 43, 6, 65, 13, 43, 6, 250, 122, 13, 43, 6, 247, - 195, 13, 43, 6, 238, 129, 13, 43, 6, 71, 13, 43, 6, 233, 177, 13, 43, 6, - 232, 53, 13, 43, 6, 230, 118, 13, 43, 6, 68, 13, 43, 6, 223, 37, 13, 43, - 6, 222, 154, 13, 43, 6, 172, 13, 43, 6, 218, 170, 13, 43, 6, 215, 63, 13, - 43, 6, 74, 13, 43, 6, 210, 238, 13, 43, 6, 208, 106, 13, 43, 6, 146, 13, - 43, 6, 206, 9, 13, 43, 6, 200, 43, 13, 43, 6, 66, 13, 43, 6, 196, 12, 13, - 43, 6, 193, 224, 13, 43, 6, 192, 235, 13, 43, 6, 192, 159, 13, 43, 6, - 191, 166, 13, 43, 2, 65, 13, 43, 2, 250, 122, 13, 43, 2, 247, 195, 13, - 43, 2, 238, 129, 13, 43, 2, 71, 13, 43, 2, 233, 177, 13, 43, 2, 232, 53, - 13, 43, 2, 68, 13, 43, 2, 223, 37, 13, 43, 2, 222, 154, 13, 43, 2, 172, - 13, 43, 2, 218, 170, 13, 43, 2, 215, 63, 13, 43, 2, 74, 13, 43, 2, 210, - 238, 13, 43, 2, 208, 106, 13, 43, 2, 146, 13, 43, 2, 206, 9, 13, 43, 2, - 200, 43, 13, 43, 2, 66, 13, 43, 2, 196, 12, 13, 43, 2, 193, 224, 13, 43, - 2, 192, 235, 13, 43, 2, 192, 159, 13, 43, 2, 191, 166, 13, 28, 43, 6, 65, - 13, 28, 43, 6, 250, 122, 13, 28, 43, 6, 247, 195, 13, 28, 43, 6, 238, - 129, 13, 28, 43, 6, 71, 13, 28, 43, 6, 233, 177, 13, 28, 43, 6, 232, 53, - 13, 28, 43, 6, 230, 118, 13, 28, 43, 6, 68, 13, 28, 43, 6, 223, 37, 13, - 28, 43, 6, 222, 154, 13, 28, 43, 6, 172, 13, 28, 43, 6, 218, 170, 13, 28, - 43, 6, 215, 63, 13, 28, 43, 6, 74, 13, 28, 43, 6, 210, 238, 13, 28, 43, - 6, 208, 106, 13, 28, 43, 6, 146, 13, 28, 43, 6, 206, 9, 13, 28, 43, 6, - 200, 43, 13, 28, 43, 6, 66, 13, 28, 43, 6, 196, 12, 13, 28, 43, 6, 193, - 224, 13, 28, 43, 6, 192, 235, 13, 28, 43, 6, 192, 159, 13, 28, 43, 6, - 191, 166, 13, 28, 43, 2, 65, 13, 28, 43, 2, 250, 122, 13, 28, 43, 2, 247, - 195, 13, 28, 43, 2, 238, 129, 13, 28, 43, 2, 71, 13, 28, 43, 2, 233, 177, - 13, 28, 43, 2, 232, 53, 13, 28, 43, 2, 230, 118, 13, 28, 43, 2, 68, 13, - 28, 43, 2, 223, 37, 13, 28, 43, 2, 222, 154, 13, 28, 43, 2, 172, 13, 28, - 43, 2, 218, 170, 13, 28, 43, 2, 215, 63, 13, 28, 43, 2, 74, 13, 28, 43, - 2, 210, 238, 13, 28, 43, 2, 208, 106, 13, 28, 43, 2, 146, 13, 28, 43, 2, - 206, 9, 13, 28, 43, 2, 200, 43, 13, 28, 43, 2, 66, 13, 28, 43, 2, 196, - 12, 13, 28, 43, 2, 193, 224, 13, 28, 43, 2, 192, 235, 13, 28, 43, 2, 192, - 159, 13, 28, 43, 2, 191, 166, 13, 27, 6, 65, 13, 27, 6, 247, 195, 13, 27, - 6, 238, 129, 13, 27, 6, 232, 53, 13, 27, 6, 223, 37, 13, 27, 6, 222, 154, - 13, 27, 6, 215, 63, 13, 27, 6, 74, 13, 27, 6, 210, 238, 13, 27, 6, 208, - 106, 13, 27, 6, 206, 9, 13, 27, 6, 200, 43, 13, 27, 6, 66, 13, 27, 6, - 196, 12, 13, 27, 6, 193, 224, 13, 27, 6, 192, 235, 13, 27, 6, 192, 159, - 13, 27, 6, 191, 166, 13, 27, 2, 65, 13, 27, 2, 250, 122, 13, 27, 2, 247, - 195, 13, 27, 2, 238, 129, 13, 27, 2, 233, 177, 13, 27, 2, 230, 118, 13, - 27, 2, 68, 13, 27, 2, 223, 37, 13, 27, 2, 222, 154, 13, 27, 2, 172, 13, - 27, 2, 218, 170, 13, 27, 2, 215, 63, 13, 27, 2, 210, 238, 13, 27, 2, 208, - 106, 13, 27, 2, 146, 13, 27, 2, 206, 9, 13, 27, 2, 200, 43, 13, 27, 2, - 66, 13, 27, 2, 196, 12, 13, 27, 2, 193, 224, 13, 27, 2, 192, 235, 13, 27, - 2, 192, 159, 13, 27, 2, 191, 166, 13, 28, 27, 6, 65, 13, 28, 27, 6, 250, - 122, 13, 28, 27, 6, 247, 195, 13, 28, 27, 6, 238, 129, 13, 28, 27, 6, 71, - 13, 28, 27, 6, 233, 177, 13, 28, 27, 6, 232, 53, 13, 28, 27, 6, 230, 118, - 13, 28, 27, 6, 68, 13, 28, 27, 6, 223, 37, 13, 28, 27, 6, 222, 154, 13, - 28, 27, 6, 172, 13, 28, 27, 6, 218, 170, 13, 28, 27, 6, 215, 63, 13, 28, - 27, 6, 74, 13, 28, 27, 6, 210, 238, 13, 28, 27, 6, 208, 106, 13, 28, 27, - 6, 146, 13, 28, 27, 6, 206, 9, 13, 28, 27, 6, 200, 43, 13, 28, 27, 6, 66, - 13, 28, 27, 6, 196, 12, 13, 28, 27, 6, 193, 224, 13, 28, 27, 6, 192, 235, - 13, 28, 27, 6, 192, 159, 13, 28, 27, 6, 191, 166, 13, 28, 27, 2, 65, 13, - 28, 27, 2, 250, 122, 13, 28, 27, 2, 247, 195, 13, 28, 27, 2, 238, 129, - 13, 28, 27, 2, 71, 13, 28, 27, 2, 233, 177, 13, 28, 27, 2, 232, 53, 13, - 28, 27, 2, 230, 118, 13, 28, 27, 2, 68, 13, 28, 27, 2, 223, 37, 13, 28, - 27, 2, 222, 154, 13, 28, 27, 2, 172, 13, 28, 27, 2, 218, 170, 13, 28, 27, - 2, 215, 63, 13, 28, 27, 2, 74, 13, 28, 27, 2, 210, 238, 13, 28, 27, 2, - 208, 106, 13, 28, 27, 2, 146, 13, 28, 27, 2, 206, 9, 13, 28, 27, 2, 200, - 43, 13, 28, 27, 2, 66, 13, 28, 27, 2, 196, 12, 13, 28, 27, 2, 193, 224, - 13, 28, 27, 2, 192, 235, 13, 28, 27, 2, 192, 159, 13, 28, 27, 2, 191, - 166, 13, 232, 117, 6, 65, 13, 232, 117, 6, 250, 122, 13, 232, 117, 6, - 238, 129, 13, 232, 117, 6, 71, 13, 232, 117, 6, 233, 177, 13, 232, 117, - 6, 232, 53, 13, 232, 117, 6, 223, 37, 13, 232, 117, 6, 222, 154, 13, 232, - 117, 6, 172, 13, 232, 117, 6, 218, 170, 13, 232, 117, 6, 215, 63, 13, - 232, 117, 6, 74, 13, 232, 117, 6, 210, 238, 13, 232, 117, 6, 208, 106, - 13, 232, 117, 6, 206, 9, 13, 232, 117, 6, 200, 43, 13, 232, 117, 6, 66, - 13, 232, 117, 6, 196, 12, 13, 232, 117, 6, 193, 224, 13, 232, 117, 6, - 192, 235, 13, 232, 117, 6, 192, 159, 13, 232, 117, 2, 65, 13, 232, 117, - 2, 250, 122, 13, 232, 117, 2, 247, 195, 13, 232, 117, 2, 238, 129, 13, - 232, 117, 2, 71, 13, 232, 117, 2, 233, 177, 13, 232, 117, 2, 232, 53, 13, - 232, 117, 2, 230, 118, 13, 232, 117, 2, 68, 13, 232, 117, 2, 223, 37, 13, - 232, 117, 2, 222, 154, 13, 232, 117, 2, 172, 13, 232, 117, 2, 218, 170, - 13, 232, 117, 2, 215, 63, 13, 232, 117, 2, 74, 13, 232, 117, 2, 210, 238, - 13, 232, 117, 2, 208, 106, 13, 232, 117, 2, 146, 13, 232, 117, 2, 206, 9, - 13, 232, 117, 2, 200, 43, 13, 232, 117, 2, 66, 13, 232, 117, 2, 196, 12, - 13, 232, 117, 2, 193, 224, 13, 232, 117, 2, 192, 235, 13, 232, 117, 2, - 192, 159, 13, 232, 117, 2, 191, 166, 13, 235, 131, 6, 65, 13, 235, 131, - 6, 250, 122, 13, 235, 131, 6, 238, 129, 13, 235, 131, 6, 71, 13, 235, - 131, 6, 233, 177, 13, 235, 131, 6, 232, 53, 13, 235, 131, 6, 68, 13, 235, - 131, 6, 223, 37, 13, 235, 131, 6, 222, 154, 13, 235, 131, 6, 172, 13, - 235, 131, 6, 218, 170, 13, 235, 131, 6, 74, 13, 235, 131, 6, 206, 9, 13, - 235, 131, 6, 200, 43, 13, 235, 131, 6, 66, 13, 235, 131, 6, 196, 12, 13, - 235, 131, 6, 193, 224, 13, 235, 131, 6, 192, 235, 13, 235, 131, 6, 192, - 159, 13, 235, 131, 2, 65, 13, 235, 131, 2, 250, 122, 13, 235, 131, 2, - 247, 195, 13, 235, 131, 2, 238, 129, 13, 235, 131, 2, 71, 13, 235, 131, - 2, 233, 177, 13, 235, 131, 2, 232, 53, 13, 235, 131, 2, 230, 118, 13, - 235, 131, 2, 68, 13, 235, 131, 2, 223, 37, 13, 235, 131, 2, 222, 154, 13, - 235, 131, 2, 172, 13, 235, 131, 2, 218, 170, 13, 235, 131, 2, 215, 63, - 13, 235, 131, 2, 74, 13, 235, 131, 2, 210, 238, 13, 235, 131, 2, 208, - 106, 13, 235, 131, 2, 146, 13, 235, 131, 2, 206, 9, 13, 235, 131, 2, 200, - 43, 13, 235, 131, 2, 66, 13, 235, 131, 2, 196, 12, 13, 235, 131, 2, 193, - 224, 13, 235, 131, 2, 192, 235, 13, 235, 131, 2, 192, 159, 13, 235, 131, - 2, 191, 166, 13, 28, 232, 117, 6, 65, 13, 28, 232, 117, 6, 250, 122, 13, - 28, 232, 117, 6, 247, 195, 13, 28, 232, 117, 6, 238, 129, 13, 28, 232, - 117, 6, 71, 13, 28, 232, 117, 6, 233, 177, 13, 28, 232, 117, 6, 232, 53, - 13, 28, 232, 117, 6, 230, 118, 13, 28, 232, 117, 6, 68, 13, 28, 232, 117, - 6, 223, 37, 13, 28, 232, 117, 6, 222, 154, 13, 28, 232, 117, 6, 172, 13, - 28, 232, 117, 6, 218, 170, 13, 28, 232, 117, 6, 215, 63, 13, 28, 232, - 117, 6, 74, 13, 28, 232, 117, 6, 210, 238, 13, 28, 232, 117, 6, 208, 106, - 13, 28, 232, 117, 6, 146, 13, 28, 232, 117, 6, 206, 9, 13, 28, 232, 117, - 6, 200, 43, 13, 28, 232, 117, 6, 66, 13, 28, 232, 117, 6, 196, 12, 13, - 28, 232, 117, 6, 193, 224, 13, 28, 232, 117, 6, 192, 235, 13, 28, 232, - 117, 6, 192, 159, 13, 28, 232, 117, 6, 191, 166, 13, 28, 232, 117, 2, 65, - 13, 28, 232, 117, 2, 250, 122, 13, 28, 232, 117, 2, 247, 195, 13, 28, - 232, 117, 2, 238, 129, 13, 28, 232, 117, 2, 71, 13, 28, 232, 117, 2, 233, - 177, 13, 28, 232, 117, 2, 232, 53, 13, 28, 232, 117, 2, 230, 118, 13, 28, - 232, 117, 2, 68, 13, 28, 232, 117, 2, 223, 37, 13, 28, 232, 117, 2, 222, - 154, 13, 28, 232, 117, 2, 172, 13, 28, 232, 117, 2, 218, 170, 13, 28, - 232, 117, 2, 215, 63, 13, 28, 232, 117, 2, 74, 13, 28, 232, 117, 2, 210, - 238, 13, 28, 232, 117, 2, 208, 106, 13, 28, 232, 117, 2, 146, 13, 28, - 232, 117, 2, 206, 9, 13, 28, 232, 117, 2, 200, 43, 13, 28, 232, 117, 2, - 66, 13, 28, 232, 117, 2, 196, 12, 13, 28, 232, 117, 2, 193, 224, 13, 28, - 232, 117, 2, 192, 235, 13, 28, 232, 117, 2, 192, 159, 13, 28, 232, 117, - 2, 191, 166, 13, 49, 6, 65, 13, 49, 6, 250, 122, 13, 49, 6, 247, 195, 13, - 49, 6, 238, 129, 13, 49, 6, 71, 13, 49, 6, 233, 177, 13, 49, 6, 232, 53, - 13, 49, 6, 230, 118, 13, 49, 6, 68, 13, 49, 6, 223, 37, 13, 49, 6, 222, - 154, 13, 49, 6, 172, 13, 49, 6, 218, 170, 13, 49, 6, 215, 63, 13, 49, 6, - 74, 13, 49, 6, 210, 238, 13, 49, 6, 208, 106, 13, 49, 6, 146, 13, 49, 6, - 206, 9, 13, 49, 6, 200, 43, 13, 49, 6, 66, 13, 49, 6, 196, 12, 13, 49, 6, - 193, 224, 13, 49, 6, 192, 235, 13, 49, 6, 192, 159, 13, 49, 6, 191, 166, - 13, 49, 2, 65, 13, 49, 2, 250, 122, 13, 49, 2, 247, 195, 13, 49, 2, 238, - 129, 13, 49, 2, 71, 13, 49, 2, 233, 177, 13, 49, 2, 232, 53, 13, 49, 2, - 230, 118, 13, 49, 2, 68, 13, 49, 2, 223, 37, 13, 49, 2, 222, 154, 13, 49, - 2, 172, 13, 49, 2, 218, 170, 13, 49, 2, 215, 63, 13, 49, 2, 74, 13, 49, - 2, 210, 238, 13, 49, 2, 208, 106, 13, 49, 2, 146, 13, 49, 2, 206, 9, 13, - 49, 2, 200, 43, 13, 49, 2, 66, 13, 49, 2, 196, 12, 13, 49, 2, 193, 224, - 13, 49, 2, 192, 235, 13, 49, 2, 192, 159, 13, 49, 2, 191, 166, 13, 49, - 28, 6, 65, 13, 49, 28, 6, 250, 122, 13, 49, 28, 6, 247, 195, 13, 49, 28, - 6, 238, 129, 13, 49, 28, 6, 71, 13, 49, 28, 6, 233, 177, 13, 49, 28, 6, - 232, 53, 13, 49, 28, 6, 230, 118, 13, 49, 28, 6, 68, 13, 49, 28, 6, 223, - 37, 13, 49, 28, 6, 222, 154, 13, 49, 28, 6, 172, 13, 49, 28, 6, 218, 170, - 13, 49, 28, 6, 215, 63, 13, 49, 28, 6, 74, 13, 49, 28, 6, 210, 238, 13, - 49, 28, 6, 208, 106, 13, 49, 28, 6, 146, 13, 49, 28, 6, 206, 9, 13, 49, - 28, 6, 200, 43, 13, 49, 28, 6, 66, 13, 49, 28, 6, 196, 12, 13, 49, 28, 6, - 193, 224, 13, 49, 28, 6, 192, 235, 13, 49, 28, 6, 192, 159, 13, 49, 28, - 6, 191, 166, 13, 49, 28, 2, 65, 13, 49, 28, 2, 250, 122, 13, 49, 28, 2, - 247, 195, 13, 49, 28, 2, 238, 129, 13, 49, 28, 2, 71, 13, 49, 28, 2, 233, - 177, 13, 49, 28, 2, 232, 53, 13, 49, 28, 2, 230, 118, 13, 49, 28, 2, 68, - 13, 49, 28, 2, 223, 37, 13, 49, 28, 2, 222, 154, 13, 49, 28, 2, 172, 13, - 49, 28, 2, 218, 170, 13, 49, 28, 2, 215, 63, 13, 49, 28, 2, 74, 13, 49, - 28, 2, 210, 238, 13, 49, 28, 2, 208, 106, 13, 49, 28, 2, 146, 13, 49, 28, - 2, 206, 9, 13, 49, 28, 2, 200, 43, 13, 49, 28, 2, 66, 13, 49, 28, 2, 196, - 12, 13, 49, 28, 2, 193, 224, 13, 49, 28, 2, 192, 235, 13, 49, 28, 2, 192, - 159, 13, 49, 28, 2, 191, 166, 13, 49, 43, 6, 65, 13, 49, 43, 6, 250, 122, - 13, 49, 43, 6, 247, 195, 13, 49, 43, 6, 238, 129, 13, 49, 43, 6, 71, 13, - 49, 43, 6, 233, 177, 13, 49, 43, 6, 232, 53, 13, 49, 43, 6, 230, 118, 13, - 49, 43, 6, 68, 13, 49, 43, 6, 223, 37, 13, 49, 43, 6, 222, 154, 13, 49, - 43, 6, 172, 13, 49, 43, 6, 218, 170, 13, 49, 43, 6, 215, 63, 13, 49, 43, - 6, 74, 13, 49, 43, 6, 210, 238, 13, 49, 43, 6, 208, 106, 13, 49, 43, 6, - 146, 13, 49, 43, 6, 206, 9, 13, 49, 43, 6, 200, 43, 13, 49, 43, 6, 66, - 13, 49, 43, 6, 196, 12, 13, 49, 43, 6, 193, 224, 13, 49, 43, 6, 192, 235, - 13, 49, 43, 6, 192, 159, 13, 49, 43, 6, 191, 166, 13, 49, 43, 2, 65, 13, - 49, 43, 2, 250, 122, 13, 49, 43, 2, 247, 195, 13, 49, 43, 2, 238, 129, - 13, 49, 43, 2, 71, 13, 49, 43, 2, 233, 177, 13, 49, 43, 2, 232, 53, 13, - 49, 43, 2, 230, 118, 13, 49, 43, 2, 68, 13, 49, 43, 2, 223, 37, 13, 49, - 43, 2, 222, 154, 13, 49, 43, 2, 172, 13, 49, 43, 2, 218, 170, 13, 49, 43, - 2, 215, 63, 13, 49, 43, 2, 74, 13, 49, 43, 2, 210, 238, 13, 49, 43, 2, - 208, 106, 13, 49, 43, 2, 146, 13, 49, 43, 2, 206, 9, 13, 49, 43, 2, 200, - 43, 13, 49, 43, 2, 66, 13, 49, 43, 2, 196, 12, 13, 49, 43, 2, 193, 224, - 13, 49, 43, 2, 192, 235, 13, 49, 43, 2, 192, 159, 13, 49, 43, 2, 191, - 166, 13, 49, 28, 43, 6, 65, 13, 49, 28, 43, 6, 250, 122, 13, 49, 28, 43, - 6, 247, 195, 13, 49, 28, 43, 6, 238, 129, 13, 49, 28, 43, 6, 71, 13, 49, - 28, 43, 6, 233, 177, 13, 49, 28, 43, 6, 232, 53, 13, 49, 28, 43, 6, 230, - 118, 13, 49, 28, 43, 6, 68, 13, 49, 28, 43, 6, 223, 37, 13, 49, 28, 43, - 6, 222, 154, 13, 49, 28, 43, 6, 172, 13, 49, 28, 43, 6, 218, 170, 13, 49, - 28, 43, 6, 215, 63, 13, 49, 28, 43, 6, 74, 13, 49, 28, 43, 6, 210, 238, - 13, 49, 28, 43, 6, 208, 106, 13, 49, 28, 43, 6, 146, 13, 49, 28, 43, 6, - 206, 9, 13, 49, 28, 43, 6, 200, 43, 13, 49, 28, 43, 6, 66, 13, 49, 28, - 43, 6, 196, 12, 13, 49, 28, 43, 6, 193, 224, 13, 49, 28, 43, 6, 192, 235, - 13, 49, 28, 43, 6, 192, 159, 13, 49, 28, 43, 6, 191, 166, 13, 49, 28, 43, - 2, 65, 13, 49, 28, 43, 2, 250, 122, 13, 49, 28, 43, 2, 247, 195, 13, 49, - 28, 43, 2, 238, 129, 13, 49, 28, 43, 2, 71, 13, 49, 28, 43, 2, 233, 177, - 13, 49, 28, 43, 2, 232, 53, 13, 49, 28, 43, 2, 230, 118, 13, 49, 28, 43, - 2, 68, 13, 49, 28, 43, 2, 223, 37, 13, 49, 28, 43, 2, 222, 154, 13, 49, - 28, 43, 2, 172, 13, 49, 28, 43, 2, 218, 170, 13, 49, 28, 43, 2, 215, 63, - 13, 49, 28, 43, 2, 74, 13, 49, 28, 43, 2, 210, 238, 13, 49, 28, 43, 2, - 208, 106, 13, 49, 28, 43, 2, 146, 13, 49, 28, 43, 2, 206, 9, 13, 49, 28, - 43, 2, 200, 43, 13, 49, 28, 43, 2, 66, 13, 49, 28, 43, 2, 196, 12, 13, - 49, 28, 43, 2, 193, 224, 13, 49, 28, 43, 2, 192, 235, 13, 49, 28, 43, 2, - 192, 159, 13, 49, 28, 43, 2, 191, 166, 13, 215, 219, 6, 65, 13, 215, 219, - 6, 250, 122, 13, 215, 219, 6, 247, 195, 13, 215, 219, 6, 238, 129, 13, - 215, 219, 6, 71, 13, 215, 219, 6, 233, 177, 13, 215, 219, 6, 232, 53, 13, - 215, 219, 6, 230, 118, 13, 215, 219, 6, 68, 13, 215, 219, 6, 223, 37, 13, - 215, 219, 6, 222, 154, 13, 215, 219, 6, 172, 13, 215, 219, 6, 218, 170, - 13, 215, 219, 6, 215, 63, 13, 215, 219, 6, 74, 13, 215, 219, 6, 210, 238, - 13, 215, 219, 6, 208, 106, 13, 215, 219, 6, 146, 13, 215, 219, 6, 206, 9, - 13, 215, 219, 6, 200, 43, 13, 215, 219, 6, 66, 13, 215, 219, 6, 196, 12, - 13, 215, 219, 6, 193, 224, 13, 215, 219, 6, 192, 235, 13, 215, 219, 6, - 192, 159, 13, 215, 219, 6, 191, 166, 13, 215, 219, 2, 65, 13, 215, 219, - 2, 250, 122, 13, 215, 219, 2, 247, 195, 13, 215, 219, 2, 238, 129, 13, - 215, 219, 2, 71, 13, 215, 219, 2, 233, 177, 13, 215, 219, 2, 232, 53, 13, - 215, 219, 2, 230, 118, 13, 215, 219, 2, 68, 13, 215, 219, 2, 223, 37, 13, - 215, 219, 2, 222, 154, 13, 215, 219, 2, 172, 13, 215, 219, 2, 218, 170, - 13, 215, 219, 2, 215, 63, 13, 215, 219, 2, 74, 13, 215, 219, 2, 210, 238, - 13, 215, 219, 2, 208, 106, 13, 215, 219, 2, 146, 13, 215, 219, 2, 206, 9, - 13, 215, 219, 2, 200, 43, 13, 215, 219, 2, 66, 13, 215, 219, 2, 196, 12, - 13, 215, 219, 2, 193, 224, 13, 215, 219, 2, 192, 235, 13, 215, 219, 2, - 192, 159, 13, 215, 219, 2, 191, 166, 13, 43, 2, 236, 141, 68, 13, 43, 2, - 236, 141, 223, 37, 13, 28, 6, 251, 162, 13, 28, 6, 248, 214, 13, 28, 6, - 231, 213, 13, 28, 6, 237, 108, 13, 28, 6, 234, 49, 13, 28, 6, 191, 76, - 13, 28, 6, 233, 255, 13, 28, 6, 199, 15, 13, 28, 6, 223, 85, 13, 28, 6, - 222, 74, 13, 28, 6, 220, 33, 13, 28, 6, 215, 157, 13, 28, 6, 212, 180, - 13, 28, 6, 192, 207, 13, 28, 6, 211, 109, 13, 28, 6, 209, 187, 13, 28, 6, - 207, 4, 13, 28, 6, 199, 16, 113, 13, 28, 6, 202, 197, 13, 28, 6, 199, - 166, 13, 28, 6, 196, 70, 13, 28, 6, 209, 213, 13, 28, 6, 243, 97, 13, 28, - 6, 208, 178, 13, 28, 6, 211, 112, 13, 28, 214, 247, 13, 28, 2, 251, 162, - 13, 28, 2, 248, 214, 13, 28, 2, 231, 213, 13, 28, 2, 237, 108, 13, 28, 2, - 234, 49, 13, 28, 2, 191, 76, 13, 28, 2, 233, 255, 13, 28, 2, 199, 15, 13, - 28, 2, 223, 85, 13, 28, 2, 222, 74, 13, 28, 2, 220, 33, 13, 28, 2, 215, - 157, 13, 28, 2, 212, 180, 13, 28, 2, 192, 207, 13, 28, 2, 211, 109, 13, - 28, 2, 209, 187, 13, 28, 2, 207, 4, 13, 28, 2, 53, 202, 197, 13, 28, 2, - 202, 197, 13, 28, 2, 199, 166, 13, 28, 2, 196, 70, 13, 28, 2, 209, 213, - 13, 28, 2, 243, 97, 13, 28, 2, 208, 178, 13, 28, 2, 211, 112, 13, 28, - 210, 107, 237, 18, 13, 28, 234, 50, 113, 13, 28, 199, 16, 113, 13, 28, - 222, 75, 113, 13, 28, 209, 214, 113, 13, 28, 207, 5, 113, 13, 28, 209, - 188, 113, 13, 43, 6, 251, 162, 13, 43, 6, 248, 214, 13, 43, 6, 231, 213, - 13, 43, 6, 237, 108, 13, 43, 6, 234, 49, 13, 43, 6, 191, 76, 13, 43, 6, - 233, 255, 13, 43, 6, 199, 15, 13, 43, 6, 223, 85, 13, 43, 6, 222, 74, 13, - 43, 6, 220, 33, 13, 43, 6, 215, 157, 13, 43, 6, 212, 180, 13, 43, 6, 192, - 207, 13, 43, 6, 211, 109, 13, 43, 6, 209, 187, 13, 43, 6, 207, 4, 13, 43, - 6, 199, 16, 113, 13, 43, 6, 202, 197, 13, 43, 6, 199, 166, 13, 43, 6, - 196, 70, 13, 43, 6, 209, 213, 13, 43, 6, 243, 97, 13, 43, 6, 208, 178, - 13, 43, 6, 211, 112, 13, 43, 214, 247, 13, 43, 2, 251, 162, 13, 43, 2, - 248, 214, 13, 43, 2, 231, 213, 13, 43, 2, 237, 108, 13, 43, 2, 234, 49, - 13, 43, 2, 191, 76, 13, 43, 2, 233, 255, 13, 43, 2, 199, 15, 13, 43, 2, - 223, 85, 13, 43, 2, 222, 74, 13, 43, 2, 220, 33, 13, 43, 2, 215, 157, 13, - 43, 2, 212, 180, 13, 43, 2, 192, 207, 13, 43, 2, 211, 109, 13, 43, 2, - 209, 187, 13, 43, 2, 207, 4, 13, 43, 2, 53, 202, 197, 13, 43, 2, 202, - 197, 13, 43, 2, 199, 166, 13, 43, 2, 196, 70, 13, 43, 2, 209, 213, 13, - 43, 2, 243, 97, 13, 43, 2, 208, 178, 13, 43, 2, 211, 112, 13, 43, 210, - 107, 237, 18, 13, 43, 234, 50, 113, 13, 43, 199, 16, 113, 13, 43, 222, - 75, 113, 13, 43, 209, 214, 113, 13, 43, 207, 5, 113, 13, 43, 209, 188, - 113, 13, 28, 43, 6, 251, 162, 13, 28, 43, 6, 248, 214, 13, 28, 43, 6, - 231, 213, 13, 28, 43, 6, 237, 108, 13, 28, 43, 6, 234, 49, 13, 28, 43, 6, - 191, 76, 13, 28, 43, 6, 233, 255, 13, 28, 43, 6, 199, 15, 13, 28, 43, 6, - 223, 85, 13, 28, 43, 6, 222, 74, 13, 28, 43, 6, 220, 33, 13, 28, 43, 6, - 215, 157, 13, 28, 43, 6, 212, 180, 13, 28, 43, 6, 192, 207, 13, 28, 43, - 6, 211, 109, 13, 28, 43, 6, 209, 187, 13, 28, 43, 6, 207, 4, 13, 28, 43, - 6, 199, 16, 113, 13, 28, 43, 6, 202, 197, 13, 28, 43, 6, 199, 166, 13, - 28, 43, 6, 196, 70, 13, 28, 43, 6, 209, 213, 13, 28, 43, 6, 243, 97, 13, - 28, 43, 6, 208, 178, 13, 28, 43, 6, 211, 112, 13, 28, 43, 214, 247, 13, - 28, 43, 2, 251, 162, 13, 28, 43, 2, 248, 214, 13, 28, 43, 2, 231, 213, - 13, 28, 43, 2, 237, 108, 13, 28, 43, 2, 234, 49, 13, 28, 43, 2, 191, 76, - 13, 28, 43, 2, 233, 255, 13, 28, 43, 2, 199, 15, 13, 28, 43, 2, 223, 85, - 13, 28, 43, 2, 222, 74, 13, 28, 43, 2, 220, 33, 13, 28, 43, 2, 215, 157, - 13, 28, 43, 2, 212, 180, 13, 28, 43, 2, 192, 207, 13, 28, 43, 2, 211, - 109, 13, 28, 43, 2, 209, 187, 13, 28, 43, 2, 207, 4, 13, 28, 43, 2, 53, - 202, 197, 13, 28, 43, 2, 202, 197, 13, 28, 43, 2, 199, 166, 13, 28, 43, - 2, 196, 70, 13, 28, 43, 2, 209, 213, 13, 28, 43, 2, 243, 97, 13, 28, 43, - 2, 208, 178, 13, 28, 43, 2, 211, 112, 13, 28, 43, 210, 107, 237, 18, 13, - 28, 43, 234, 50, 113, 13, 28, 43, 199, 16, 113, 13, 28, 43, 222, 75, 113, - 13, 28, 43, 209, 214, 113, 13, 28, 43, 207, 5, 113, 13, 28, 43, 209, 188, - 113, 13, 49, 28, 6, 251, 162, 13, 49, 28, 6, 248, 214, 13, 49, 28, 6, - 231, 213, 13, 49, 28, 6, 237, 108, 13, 49, 28, 6, 234, 49, 13, 49, 28, 6, - 191, 76, 13, 49, 28, 6, 233, 255, 13, 49, 28, 6, 199, 15, 13, 49, 28, 6, - 223, 85, 13, 49, 28, 6, 222, 74, 13, 49, 28, 6, 220, 33, 13, 49, 28, 6, - 215, 157, 13, 49, 28, 6, 212, 180, 13, 49, 28, 6, 192, 207, 13, 49, 28, - 6, 211, 109, 13, 49, 28, 6, 209, 187, 13, 49, 28, 6, 207, 4, 13, 49, 28, - 6, 199, 16, 113, 13, 49, 28, 6, 202, 197, 13, 49, 28, 6, 199, 166, 13, - 49, 28, 6, 196, 70, 13, 49, 28, 6, 209, 213, 13, 49, 28, 6, 243, 97, 13, - 49, 28, 6, 208, 178, 13, 49, 28, 6, 211, 112, 13, 49, 28, 214, 247, 13, - 49, 28, 2, 251, 162, 13, 49, 28, 2, 248, 214, 13, 49, 28, 2, 231, 213, - 13, 49, 28, 2, 237, 108, 13, 49, 28, 2, 234, 49, 13, 49, 28, 2, 191, 76, - 13, 49, 28, 2, 233, 255, 13, 49, 28, 2, 199, 15, 13, 49, 28, 2, 223, 85, - 13, 49, 28, 2, 222, 74, 13, 49, 28, 2, 220, 33, 13, 49, 28, 2, 215, 157, - 13, 49, 28, 2, 212, 180, 13, 49, 28, 2, 192, 207, 13, 49, 28, 2, 211, - 109, 13, 49, 28, 2, 209, 187, 13, 49, 28, 2, 207, 4, 13, 49, 28, 2, 53, - 202, 197, 13, 49, 28, 2, 202, 197, 13, 49, 28, 2, 199, 166, 13, 49, 28, - 2, 196, 70, 13, 49, 28, 2, 209, 213, 13, 49, 28, 2, 243, 97, 13, 49, 28, - 2, 208, 178, 13, 49, 28, 2, 211, 112, 13, 49, 28, 210, 107, 237, 18, 13, - 49, 28, 234, 50, 113, 13, 49, 28, 199, 16, 113, 13, 49, 28, 222, 75, 113, - 13, 49, 28, 209, 214, 113, 13, 49, 28, 207, 5, 113, 13, 49, 28, 209, 188, - 113, 13, 49, 28, 43, 6, 251, 162, 13, 49, 28, 43, 6, 248, 214, 13, 49, - 28, 43, 6, 231, 213, 13, 49, 28, 43, 6, 237, 108, 13, 49, 28, 43, 6, 234, - 49, 13, 49, 28, 43, 6, 191, 76, 13, 49, 28, 43, 6, 233, 255, 13, 49, 28, - 43, 6, 199, 15, 13, 49, 28, 43, 6, 223, 85, 13, 49, 28, 43, 6, 222, 74, - 13, 49, 28, 43, 6, 220, 33, 13, 49, 28, 43, 6, 215, 157, 13, 49, 28, 43, - 6, 212, 180, 13, 49, 28, 43, 6, 192, 207, 13, 49, 28, 43, 6, 211, 109, - 13, 49, 28, 43, 6, 209, 187, 13, 49, 28, 43, 6, 207, 4, 13, 49, 28, 43, - 6, 199, 16, 113, 13, 49, 28, 43, 6, 202, 197, 13, 49, 28, 43, 6, 199, - 166, 13, 49, 28, 43, 6, 196, 70, 13, 49, 28, 43, 6, 209, 213, 13, 49, 28, - 43, 6, 243, 97, 13, 49, 28, 43, 6, 208, 178, 13, 49, 28, 43, 6, 211, 112, - 13, 49, 28, 43, 214, 247, 13, 49, 28, 43, 2, 251, 162, 13, 49, 28, 43, 2, - 248, 214, 13, 49, 28, 43, 2, 231, 213, 13, 49, 28, 43, 2, 237, 108, 13, - 49, 28, 43, 2, 234, 49, 13, 49, 28, 43, 2, 191, 76, 13, 49, 28, 43, 2, - 233, 255, 13, 49, 28, 43, 2, 199, 15, 13, 49, 28, 43, 2, 223, 85, 13, 49, - 28, 43, 2, 222, 74, 13, 49, 28, 43, 2, 220, 33, 13, 49, 28, 43, 2, 215, - 157, 13, 49, 28, 43, 2, 212, 180, 13, 49, 28, 43, 2, 192, 207, 13, 49, - 28, 43, 2, 211, 109, 13, 49, 28, 43, 2, 209, 187, 13, 49, 28, 43, 2, 207, - 4, 13, 49, 28, 43, 2, 53, 202, 197, 13, 49, 28, 43, 2, 202, 197, 13, 49, - 28, 43, 2, 199, 166, 13, 49, 28, 43, 2, 196, 70, 13, 49, 28, 43, 2, 209, - 213, 13, 49, 28, 43, 2, 243, 97, 13, 49, 28, 43, 2, 208, 178, 13, 49, 28, - 43, 2, 211, 112, 13, 49, 28, 43, 210, 107, 237, 18, 13, 49, 28, 43, 234, - 50, 113, 13, 49, 28, 43, 199, 16, 113, 13, 49, 28, 43, 222, 75, 113, 13, - 49, 28, 43, 209, 214, 113, 13, 49, 28, 43, 207, 5, 113, 13, 49, 28, 43, - 209, 188, 113, 13, 28, 6, 237, 12, 13, 28, 2, 237, 12, 13, 28, 17, 191, - 77, 13, 28, 17, 107, 13, 28, 17, 109, 13, 28, 17, 138, 13, 28, 17, 134, - 13, 28, 17, 150, 13, 28, 17, 169, 13, 28, 17, 175, 13, 28, 17, 171, 13, - 28, 17, 178, 13, 235, 131, 17, 191, 77, 13, 235, 131, 17, 107, 13, 235, - 131, 17, 109, 13, 235, 131, 17, 138, 13, 235, 131, 17, 134, 13, 235, 131, - 17, 150, 13, 235, 131, 17, 169, 13, 235, 131, 17, 175, 13, 235, 131, 17, - 171, 13, 235, 131, 17, 178, 13, 49, 17, 191, 77, 13, 49, 17, 107, 13, 49, - 17, 109, 13, 49, 17, 138, 13, 49, 17, 134, 13, 49, 17, 150, 13, 49, 17, - 169, 13, 49, 17, 175, 13, 49, 17, 171, 13, 49, 17, 178, 13, 49, 28, 17, - 191, 77, 13, 49, 28, 17, 107, 13, 49, 28, 17, 109, 13, 49, 28, 17, 138, - 13, 49, 28, 17, 134, 13, 49, 28, 17, 150, 13, 49, 28, 17, 169, 13, 49, - 28, 17, 175, 13, 49, 28, 17, 171, 13, 49, 28, 17, 178, 13, 215, 219, 17, - 191, 77, 13, 215, 219, 17, 107, 13, 215, 219, 17, 109, 13, 215, 219, 17, - 138, 13, 215, 219, 17, 134, 13, 215, 219, 17, 150, 13, 215, 219, 17, 169, - 13, 215, 219, 17, 175, 13, 215, 219, 17, 171, 13, 215, 219, 17, 178, 24, - 152, 223, 150, 24, 230, 52, 223, 150, 24, 230, 48, 223, 150, 24, 230, 37, - 223, 150, 24, 230, 41, 223, 150, 24, 230, 54, 223, 150, 24, 152, 141, - 248, 225, 24, 230, 52, 141, 248, 225, 24, 152, 176, 196, 105, 141, 248, - 225, 24, 152, 141, 207, 149, 221, 72, 24, 152, 141, 238, 179, 24, 152, - 141, 229, 126, 24, 152, 141, 229, 127, 218, 243, 24, 230, 52, 141, 229, - 128, 24, 152, 141, 216, 86, 24, 230, 52, 141, 216, 86, 24, 152, 141, 82, - 248, 225, 24, 152, 141, 82, 207, 149, 221, 71, 24, 152, 141, 82, 229, - 126, 24, 152, 141, 133, 82, 229, 126, 24, 152, 141, 229, 127, 82, 196, - 77, 24, 152, 141, 82, 239, 48, 24, 152, 141, 82, 239, 49, 141, 248, 225, - 24, 152, 141, 82, 239, 49, 82, 248, 225, 24, 152, 141, 82, 239, 49, 238, - 179, 24, 152, 141, 82, 239, 49, 229, 126, 24, 152, 141, 82, 238, 215, 24, - 230, 52, 141, 82, 238, 215, 24, 152, 82, 248, 226, 139, 223, 150, 24, - 152, 141, 248, 226, 139, 216, 86, 24, 152, 141, 82, 198, 212, 24, 230, - 52, 141, 82, 198, 212, 24, 152, 141, 82, 201, 54, 176, 248, 225, 24, 152, - 141, 82, 248, 226, 176, 201, 53, 24, 152, 141, 82, 176, 248, 225, 24, - 152, 141, 82, 229, 127, 201, 200, 176, 202, 208, 24, 152, 141, 133, 82, - 229, 127, 176, 202, 208, 24, 152, 141, 133, 82, 229, 127, 176, 239, 48, - 24, 152, 141, 229, 127, 82, 133, 176, 202, 208, 24, 152, 141, 82, 133, - 201, 200, 176, 232, 134, 24, 152, 141, 82, 176, 238, 179, 24, 152, 141, - 82, 176, 243, 11, 24, 152, 141, 82, 176, 228, 251, 24, 152, 141, 82, 176, - 229, 126, 24, 152, 176, 248, 212, 141, 82, 201, 53, 24, 152, 141, 82, - 239, 49, 176, 202, 208, 24, 152, 141, 82, 239, 49, 176, 202, 209, 239, - 48, 24, 152, 141, 82, 239, 49, 176, 202, 209, 248, 225, 24, 152, 82, 176, - 228, 252, 141, 196, 77, 24, 152, 141, 176, 228, 252, 82, 196, 77, 24, - 152, 141, 82, 239, 49, 229, 127, 176, 202, 208, 24, 152, 141, 82, 238, - 216, 176, 202, 208, 24, 152, 141, 82, 239, 49, 176, 232, 134, 24, 152, - 141, 82, 239, 49, 238, 180, 176, 232, 134, 24, 152, 82, 176, 238, 180, - 141, 196, 77, 24, 152, 141, 176, 238, 180, 82, 196, 77, 24, 152, 82, 176, - 47, 141, 196, 77, 24, 152, 82, 176, 47, 141, 229, 126, 24, 152, 141, 176, - 251, 116, 211, 17, 82, 196, 77, 24, 152, 141, 176, 251, 116, 223, 165, - 82, 196, 77, 24, 152, 141, 176, 47, 82, 196, 77, 24, 152, 141, 82, 176, - 239, 49, 229, 126, 24, 152, 141, 82, 176, 251, 116, 211, 16, 24, 152, - 141, 82, 176, 251, 115, 24, 152, 82, 176, 251, 116, 211, 17, 141, 196, - 77, 24, 152, 82, 176, 251, 116, 211, 17, 141, 238, 215, 24, 152, 82, 176, - 251, 116, 141, 196, 77, 24, 152, 141, 176, 228, 252, 82, 229, 126, 24, - 230, 43, 232, 130, 232, 249, 24, 230, 43, 232, 130, 232, 250, 248, 225, - 24, 230, 43, 232, 130, 232, 250, 229, 126, 24, 230, 43, 232, 130, 232, - 250, 239, 48, 24, 230, 43, 232, 130, 232, 250, 239, 49, 201, 210, 24, - 230, 50, 232, 130, 232, 250, 239, 48, 24, 152, 232, 130, 232, 250, 239, - 49, 248, 225, 24, 230, 41, 232, 130, 232, 250, 239, 48, 24, 230, 43, 232, - 228, 232, 250, 201, 199, 24, 230, 43, 229, 215, 232, 228, 232, 250, 201, - 199, 24, 230, 43, 232, 228, 232, 250, 201, 200, 232, 130, 248, 225, 24, - 230, 43, 229, 215, 232, 228, 232, 250, 201, 200, 232, 130, 248, 225, 24, - 230, 43, 232, 228, 232, 250, 201, 200, 248, 225, 24, 230, 43, 229, 215, - 232, 228, 232, 250, 201, 200, 248, 225, 24, 230, 43, 232, 228, 232, 250, - 201, 200, 176, 232, 134, 24, 230, 48, 232, 228, 232, 250, 201, 199, 24, - 230, 48, 232, 228, 232, 250, 201, 200, 211, 78, 24, 230, 41, 232, 228, - 232, 250, 201, 200, 211, 78, 24, 230, 37, 232, 228, 232, 250, 201, 199, - 24, 230, 43, 232, 228, 232, 250, 201, 200, 229, 126, 24, 230, 43, 232, - 228, 232, 250, 201, 200, 229, 127, 176, 202, 208, 24, 230, 43, 232, 228, - 232, 250, 201, 200, 229, 127, 213, 46, 198, 212, 24, 230, 42, 24, 230, - 43, 248, 212, 210, 185, 233, 97, 24, 230, 43, 229, 214, 24, 230, 43, 176, - 202, 208, 24, 230, 43, 229, 215, 176, 202, 208, 24, 230, 43, 176, 248, - 225, 24, 230, 43, 176, 232, 134, 24, 230, 43, 201, 211, 141, 176, 202, - 208, 24, 230, 43, 201, 211, 247, 23, 24, 230, 43, 201, 211, 247, 24, 176, - 202, 208, 24, 230, 43, 201, 211, 247, 24, 176, 202, 209, 248, 225, 24, - 230, 43, 201, 211, 219, 84, 24, 230, 49, 24, 230, 50, 176, 202, 208, 24, - 230, 50, 213, 46, 198, 212, 24, 230, 50, 176, 232, 134, 24, 230, 39, 238, - 175, 24, 230, 38, 24, 230, 48, 211, 78, 24, 230, 47, 24, 230, 48, 211, - 79, 176, 202, 208, 24, 230, 48, 176, 202, 208, 24, 230, 48, 211, 79, 213, - 46, 198, 212, 24, 230, 48, 213, 46, 198, 212, 24, 230, 48, 211, 79, 176, - 232, 134, 24, 230, 48, 176, 232, 134, 24, 230, 46, 211, 78, 24, 230, 45, - 24, 230, 51, 24, 230, 36, 24, 230, 37, 176, 202, 208, 24, 230, 37, 213, - 46, 198, 212, 24, 230, 37, 176, 232, 134, 24, 230, 41, 211, 78, 24, 230, - 41, 211, 79, 176, 232, 134, 24, 230, 40, 24, 230, 41, 202, 71, 24, 230, - 41, 211, 79, 176, 202, 208, 24, 230, 41, 176, 202, 208, 24, 230, 41, 211, - 79, 213, 46, 198, 212, 24, 230, 41, 213, 46, 198, 212, 24, 230, 41, 176, - 202, 209, 198, 35, 223, 150, 24, 230, 41, 176, 248, 212, 82, 206, 189, - 24, 230, 53, 24, 152, 141, 82, 206, 189, 24, 230, 52, 141, 82, 206, 189, - 24, 230, 41, 141, 82, 206, 189, 24, 230, 54, 141, 82, 206, 189, 24, 230, - 41, 219, 84, 24, 152, 141, 82, 206, 190, 248, 225, 24, 152, 141, 82, 206, - 190, 239, 48, 24, 230, 41, 141, 82, 206, 190, 239, 48, 24, 152, 219, 85, - 235, 125, 24, 152, 219, 85, 144, 206, 184, 201, 53, 24, 152, 219, 85, - 144, 206, 184, 238, 164, 24, 152, 219, 85, 144, 211, 28, 243, 11, 24, - 152, 219, 85, 196, 77, 24, 152, 176, 196, 105, 219, 85, 196, 77, 24, 230, - 52, 219, 85, 196, 77, 24, 230, 37, 219, 85, 196, 77, 24, 230, 54, 219, - 85, 196, 77, 24, 152, 219, 85, 207, 149, 221, 72, 24, 152, 219, 85, 248, - 225, 24, 152, 219, 85, 198, 36, 198, 212, 24, 152, 219, 85, 198, 212, 24, - 230, 41, 219, 85, 198, 212, 24, 152, 219, 85, 141, 198, 212, 24, 230, 41, - 219, 85, 141, 198, 212, 24, 230, 54, 219, 85, 141, 176, 141, 176, 211, - 16, 24, 230, 54, 219, 85, 141, 176, 141, 198, 212, 24, 152, 219, 85, 223, - 150, 24, 230, 52, 219, 85, 223, 150, 24, 230, 41, 219, 85, 223, 150, 24, - 230, 54, 219, 85, 223, 150, 24, 152, 141, 82, 219, 84, 24, 230, 52, 141, - 82, 219, 84, 24, 230, 41, 141, 82, 219, 84, 24, 230, 41, 206, 189, 24, - 230, 54, 141, 82, 219, 84, 24, 152, 141, 82, 238, 220, 219, 84, 24, 230, - 52, 141, 82, 238, 220, 219, 84, 24, 152, 206, 190, 235, 125, 24, 230, 41, - 206, 190, 144, 141, 176, 228, 253, 216, 86, 24, 230, 54, 206, 190, 144, - 82, 176, 141, 238, 219, 24, 152, 206, 190, 196, 77, 24, 152, 206, 190, - 207, 149, 221, 72, 24, 152, 206, 190, 219, 84, 24, 230, 52, 206, 190, - 219, 84, 24, 230, 37, 206, 190, 219, 84, 24, 230, 54, 206, 190, 219, 84, - 24, 152, 206, 190, 216, 86, 24, 152, 206, 190, 82, 239, 48, 24, 152, 206, - 190, 82, 207, 149, 221, 71, 24, 152, 206, 190, 223, 150, 24, 152, 206, - 190, 198, 212, 24, 230, 39, 206, 190, 198, 212, 24, 152, 141, 206, 190, - 219, 84, 24, 230, 52, 141, 206, 190, 219, 84, 24, 230, 46, 141, 206, 190, - 219, 85, 211, 106, 24, 230, 39, 141, 206, 190, 219, 85, 211, 16, 24, 230, - 39, 141, 206, 190, 219, 85, 223, 164, 24, 230, 39, 141, 206, 190, 219, - 85, 196, 104, 24, 230, 48, 141, 206, 190, 219, 84, 24, 230, 41, 141, 206, - 190, 219, 84, 24, 230, 54, 141, 206, 190, 219, 85, 211, 16, 24, 230, 54, - 141, 206, 190, 219, 84, 24, 152, 82, 235, 125, 24, 230, 41, 216, 86, 24, - 152, 82, 196, 77, 24, 230, 52, 82, 196, 77, 24, 152, 82, 207, 149, 221, - 72, 24, 152, 82, 133, 176, 202, 208, 24, 230, 39, 82, 198, 212, 24, 152, - 82, 176, 219, 84, 24, 152, 82, 219, 84, 24, 152, 82, 206, 190, 219, 84, - 24, 230, 52, 82, 206, 190, 219, 84, 24, 230, 46, 82, 206, 190, 219, 85, - 211, 106, 24, 230, 48, 82, 206, 190, 219, 84, 24, 230, 41, 82, 206, 190, - 219, 84, 24, 230, 54, 82, 206, 190, 219, 85, 211, 16, 24, 230, 54, 82, - 206, 190, 219, 85, 223, 164, 24, 230, 54, 82, 206, 190, 219, 84, 24, 230, - 52, 82, 206, 190, 219, 85, 248, 225, 24, 230, 50, 82, 206, 190, 219, 85, - 239, 48, 24, 230, 50, 82, 206, 190, 219, 85, 239, 49, 202, 208, 24, 230, - 39, 82, 206, 190, 219, 85, 239, 49, 211, 16, 24, 230, 39, 82, 206, 190, - 219, 85, 239, 49, 223, 164, 24, 230, 39, 82, 206, 190, 219, 85, 239, 48, - 24, 230, 41, 141, 229, 126, 24, 152, 141, 176, 202, 208, 24, 230, 41, - 141, 176, 202, 208, 24, 152, 141, 176, 202, 209, 176, 237, 40, 24, 152, - 141, 176, 202, 209, 176, 239, 48, 24, 152, 141, 176, 202, 209, 176, 248, - 225, 24, 152, 141, 176, 202, 209, 141, 248, 225, 24, 152, 141, 176, 202, - 209, 248, 82, 248, 225, 24, 152, 141, 176, 202, 209, 141, 229, 128, 24, - 152, 141, 176, 232, 135, 141, 201, 53, 24, 152, 141, 176, 232, 135, 141, - 248, 225, 24, 152, 141, 176, 102, 24, 152, 141, 176, 238, 175, 24, 152, - 141, 176, 238, 167, 176, 223, 119, 24, 230, 50, 141, 176, 238, 167, 176, - 223, 119, 24, 152, 141, 176, 238, 167, 176, 196, 104, 24, 152, 141, 176, - 243, 12, 24, 230, 48, 141, 198, 212, 24, 230, 48, 141, 176, 211, 78, 24, - 230, 41, 141, 176, 211, 78, 24, 230, 41, 141, 176, 220, 14, 24, 230, 41, - 141, 198, 212, 24, 230, 41, 141, 176, 202, 71, 24, 230, 54, 141, 176, - 211, 16, 24, 230, 54, 141, 176, 223, 164, 24, 230, 54, 141, 198, 212, 24, - 152, 198, 212, 24, 152, 176, 229, 214, 24, 152, 176, 202, 209, 237, 40, - 24, 152, 176, 202, 209, 239, 48, 24, 152, 176, 202, 209, 248, 225, 24, - 152, 176, 232, 134, 24, 152, 176, 248, 212, 141, 216, 86, 24, 152, 176, - 248, 212, 82, 206, 189, 24, 152, 176, 248, 212, 206, 190, 219, 84, 24, - 152, 176, 196, 105, 105, 232, 249, 24, 152, 176, 139, 105, 232, 249, 24, - 152, 176, 196, 105, 115, 232, 249, 24, 152, 176, 196, 105, 232, 130, 232, - 249, 24, 152, 176, 139, 232, 130, 207, 149, 221, 71, 24, 230, 44, 24, - 152, 229, 214, 24, 198, 37, 202, 170, 24, 198, 37, 215, 130, 24, 198, 37, - 248, 211, 24, 230, 217, 202, 170, 24, 230, 217, 215, 130, 24, 230, 217, - 248, 211, 24, 201, 34, 202, 170, 24, 201, 34, 215, 130, 24, 201, 34, 248, - 211, 24, 248, 20, 202, 170, 24, 248, 20, 215, 130, 24, 248, 20, 248, 211, - 24, 206, 61, 202, 170, 24, 206, 61, 215, 130, 24, 206, 61, 248, 211, 24, - 200, 171, 200, 76, 24, 200, 171, 248, 211, 24, 201, 187, 220, 15, 202, - 170, 24, 201, 187, 2, 202, 170, 24, 201, 187, 220, 15, 215, 130, 24, 201, - 187, 2, 215, 130, 24, 201, 187, 204, 7, 24, 232, 199, 220, 15, 202, 170, - 24, 232, 199, 2, 202, 170, 24, 232, 199, 220, 15, 215, 130, 24, 232, 199, - 2, 215, 130, 24, 232, 199, 204, 7, 24, 201, 187, 232, 199, 251, 156, 24, - 215, 174, 133, 144, 220, 14, 24, 215, 174, 133, 144, 202, 71, 24, 215, - 174, 133, 204, 7, 24, 215, 174, 144, 204, 7, 24, 215, 174, 133, 144, 251, - 157, 220, 14, 24, 215, 174, 133, 144, 251, 157, 202, 71, 24, 215, 174, - 202, 209, 119, 202, 209, 205, 85, 24, 215, 173, 232, 255, 239, 37, 24, - 215, 175, 232, 255, 239, 37, 24, 215, 173, 202, 171, 201, 54, 202, 71, - 24, 215, 173, 202, 171, 201, 54, 216, 214, 24, 215, 173, 202, 171, 201, - 54, 220, 14, 24, 215, 173, 202, 171, 201, 54, 220, 12, 24, 215, 173, 202, - 171, 193, 4, 232, 202, 24, 215, 173, 55, 201, 53, 24, 215, 173, 55, 193, - 4, 232, 202, 24, 215, 173, 55, 251, 156, 24, 215, 173, 55, 251, 157, 193, - 4, 232, 202, 24, 215, 173, 238, 219, 24, 215, 173, 197, 225, 201, 54, - 215, 177, 24, 215, 173, 197, 225, 193, 4, 232, 202, 24, 215, 173, 197, - 225, 251, 156, 24, 215, 173, 197, 225, 251, 157, 193, 4, 232, 202, 24, - 215, 173, 248, 230, 202, 71, 24, 215, 173, 248, 230, 216, 214, 24, 215, - 173, 248, 230, 220, 14, 24, 215, 173, 239, 4, 202, 71, 24, 215, 173, 239, - 4, 216, 214, 24, 215, 173, 239, 4, 220, 14, 24, 215, 173, 239, 4, 206, - 121, 24, 215, 173, 243, 128, 202, 71, 24, 215, 173, 243, 128, 216, 214, - 24, 215, 173, 243, 128, 220, 14, 24, 215, 173, 111, 202, 71, 24, 215, - 173, 111, 216, 214, 24, 215, 173, 111, 220, 14, 24, 215, 173, 191, 21, - 202, 71, 24, 215, 173, 191, 21, 216, 214, 24, 215, 173, 191, 21, 220, 14, - 24, 215, 173, 210, 59, 202, 71, 24, 215, 173, 210, 59, 216, 214, 24, 215, - 173, 210, 59, 220, 14, 24, 198, 3, 206, 119, 202, 170, 24, 198, 3, 206, - 119, 235, 135, 24, 198, 3, 206, 119, 251, 156, 24, 198, 3, 206, 120, 202, - 170, 24, 198, 3, 206, 120, 235, 135, 24, 198, 3, 206, 120, 251, 156, 24, - 198, 3, 203, 145, 24, 198, 3, 250, 253, 201, 219, 202, 170, 24, 198, 3, - 250, 253, 201, 219, 235, 135, 24, 198, 3, 250, 253, 201, 219, 197, 224, - 24, 215, 176, 250, 141, 202, 71, 24, 215, 176, 250, 141, 216, 214, 24, - 215, 176, 250, 141, 220, 14, 24, 215, 176, 250, 141, 220, 12, 24, 215, - 176, 198, 31, 202, 71, 24, 215, 176, 198, 31, 216, 214, 24, 215, 176, - 198, 31, 220, 14, 24, 215, 176, 198, 31, 220, 12, 24, 215, 176, 248, 212, - 250, 141, 202, 71, 24, 215, 176, 248, 212, 250, 141, 216, 214, 24, 215, - 176, 248, 212, 250, 141, 220, 14, 24, 215, 176, 248, 212, 250, 141, 220, - 12, 24, 215, 176, 248, 212, 198, 31, 202, 71, 24, 215, 176, 248, 212, - 198, 31, 216, 214, 24, 215, 176, 248, 212, 198, 31, 220, 14, 24, 215, - 176, 248, 212, 198, 31, 220, 12, 24, 215, 175, 202, 171, 201, 54, 202, - 71, 24, 215, 175, 202, 171, 201, 54, 216, 214, 24, 215, 175, 202, 171, - 201, 54, 220, 14, 24, 215, 175, 202, 171, 201, 54, 220, 12, 24, 215, 175, - 202, 171, 193, 4, 232, 202, 24, 215, 175, 55, 201, 53, 24, 215, 175, 55, - 193, 4, 232, 202, 24, 215, 175, 55, 251, 156, 24, 215, 175, 55, 251, 157, - 193, 4, 232, 202, 24, 215, 175, 238, 219, 24, 215, 175, 197, 225, 201, - 54, 215, 177, 24, 215, 175, 197, 225, 193, 4, 232, 202, 24, 215, 175, - 197, 225, 251, 157, 215, 177, 24, 215, 175, 197, 225, 251, 157, 193, 4, - 232, 202, 24, 215, 175, 248, 229, 24, 215, 175, 239, 4, 202, 71, 24, 215, - 175, 239, 4, 216, 214, 24, 215, 175, 239, 4, 220, 14, 24, 215, 175, 243, - 127, 24, 215, 175, 111, 202, 71, 24, 215, 175, 111, 216, 214, 24, 215, - 175, 111, 220, 14, 24, 215, 175, 191, 21, 202, 71, 24, 215, 175, 191, 21, - 216, 214, 24, 215, 175, 191, 21, 220, 14, 24, 215, 175, 210, 59, 202, 71, - 24, 215, 175, 210, 59, 216, 214, 24, 215, 175, 210, 59, 220, 14, 24, 198, - 4, 206, 120, 202, 170, 24, 198, 4, 206, 120, 235, 135, 24, 198, 4, 206, - 120, 251, 156, 24, 198, 4, 206, 119, 202, 170, 24, 198, 4, 206, 119, 235, - 135, 24, 198, 4, 206, 119, 251, 156, 24, 198, 4, 203, 145, 24, 215, 173, - 238, 167, 208, 25, 202, 71, 24, 215, 173, 238, 167, 208, 25, 216, 214, - 24, 215, 173, 238, 167, 208, 25, 220, 14, 24, 215, 173, 238, 167, 208, - 25, 220, 12, 24, 215, 173, 238, 167, 230, 69, 202, 71, 24, 215, 173, 238, - 167, 230, 69, 216, 214, 24, 215, 173, 238, 167, 230, 69, 220, 14, 24, - 215, 173, 238, 167, 230, 69, 220, 12, 24, 215, 173, 238, 167, 198, 218, - 243, 13, 202, 71, 24, 215, 173, 238, 167, 198, 218, 243, 13, 216, 214, - 24, 215, 173, 228, 145, 202, 71, 24, 215, 173, 228, 145, 216, 214, 24, - 215, 173, 228, 145, 220, 14, 24, 215, 173, 219, 7, 202, 71, 24, 215, 173, - 219, 7, 216, 214, 24, 215, 173, 219, 7, 220, 14, 24, 215, 173, 219, 7, 2, - 235, 135, 24, 215, 173, 193, 139, 238, 167, 55, 202, 71, 24, 215, 173, - 193, 139, 238, 167, 55, 216, 214, 24, 215, 173, 193, 139, 238, 167, 55, - 220, 14, 24, 215, 173, 193, 139, 238, 167, 197, 225, 202, 71, 24, 215, - 173, 193, 139, 238, 167, 197, 225, 216, 214, 24, 215, 173, 193, 139, 238, - 167, 197, 225, 220, 14, 24, 215, 173, 238, 167, 199, 25, 201, 53, 24, - 215, 173, 238, 165, 238, 220, 202, 71, 24, 215, 173, 238, 165, 238, 220, - 216, 214, 24, 206, 119, 202, 170, 24, 206, 119, 235, 135, 24, 206, 119, - 251, 158, 24, 215, 173, 203, 145, 24, 215, 173, 238, 167, 229, 118, 232, - 99, 193, 167, 24, 215, 173, 228, 145, 229, 118, 232, 99, 193, 167, 24, - 215, 173, 219, 7, 229, 118, 232, 99, 193, 167, 24, 215, 173, 193, 139, - 229, 118, 232, 99, 193, 167, 24, 206, 119, 202, 171, 229, 118, 232, 99, - 193, 167, 24, 206, 119, 55, 229, 118, 232, 99, 193, 167, 24, 206, 119, - 251, 157, 229, 118, 232, 99, 193, 167, 24, 215, 173, 238, 167, 229, 118, - 243, 106, 24, 215, 173, 228, 145, 229, 118, 243, 106, 24, 215, 173, 219, - 7, 229, 118, 243, 106, 24, 215, 173, 193, 139, 229, 118, 243, 106, 24, - 206, 119, 202, 171, 229, 118, 243, 106, 24, 206, 119, 55, 229, 118, 243, - 106, 24, 206, 119, 251, 157, 229, 118, 243, 106, 24, 215, 173, 193, 139, - 237, 41, 210, 88, 202, 71, 24, 215, 173, 193, 139, 237, 41, 210, 88, 216, - 214, 24, 215, 173, 193, 139, 237, 41, 210, 88, 220, 14, 24, 215, 175, - 238, 167, 229, 118, 247, 33, 202, 71, 24, 215, 175, 238, 167, 229, 118, - 247, 33, 220, 14, 24, 215, 175, 228, 145, 229, 118, 247, 33, 2, 235, 135, - 24, 215, 175, 228, 145, 229, 118, 247, 33, 220, 15, 235, 135, 24, 215, - 175, 228, 145, 229, 118, 247, 33, 2, 197, 224, 24, 215, 175, 228, 145, - 229, 118, 247, 33, 220, 15, 197, 224, 24, 215, 175, 219, 7, 229, 118, - 247, 33, 2, 202, 170, 24, 215, 175, 219, 7, 229, 118, 247, 33, 220, 15, - 202, 170, 24, 215, 175, 219, 7, 229, 118, 247, 33, 2, 235, 135, 24, 215, - 175, 219, 7, 229, 118, 247, 33, 220, 15, 235, 135, 24, 215, 175, 193, - 139, 229, 118, 247, 33, 202, 71, 24, 215, 175, 193, 139, 229, 118, 247, - 33, 220, 14, 24, 206, 120, 202, 171, 229, 118, 247, 32, 24, 206, 120, 55, - 229, 118, 247, 32, 24, 206, 120, 251, 157, 229, 118, 247, 32, 24, 215, - 175, 238, 167, 229, 118, 232, 196, 202, 71, 24, 215, 175, 238, 167, 229, - 118, 232, 196, 220, 14, 24, 215, 175, 228, 145, 229, 118, 232, 196, 2, - 235, 135, 24, 215, 175, 228, 145, 229, 118, 232, 196, 220, 15, 235, 135, - 24, 215, 175, 228, 145, 229, 118, 232, 196, 197, 225, 2, 197, 224, 24, - 215, 175, 228, 145, 229, 118, 232, 196, 197, 225, 220, 15, 197, 224, 24, - 215, 175, 219, 7, 229, 118, 232, 196, 2, 202, 170, 24, 215, 175, 219, 7, - 229, 118, 232, 196, 220, 15, 202, 170, 24, 215, 175, 219, 7, 229, 118, - 232, 196, 2, 235, 135, 24, 215, 175, 219, 7, 229, 118, 232, 196, 220, 15, - 235, 135, 24, 215, 175, 193, 139, 229, 118, 232, 196, 202, 71, 24, 215, - 175, 193, 139, 229, 118, 232, 196, 220, 14, 24, 206, 120, 202, 171, 229, - 118, 232, 195, 24, 206, 120, 55, 229, 118, 232, 195, 24, 206, 120, 251, - 157, 229, 118, 232, 195, 24, 215, 175, 238, 167, 202, 71, 24, 215, 175, - 238, 167, 216, 214, 24, 215, 175, 238, 167, 220, 14, 24, 215, 175, 238, - 167, 220, 12, 24, 215, 175, 238, 167, 242, 80, 24, 215, 175, 228, 145, - 202, 71, 24, 215, 175, 219, 7, 202, 71, 24, 215, 175, 193, 139, 202, 59, - 24, 215, 175, 193, 139, 202, 71, 24, 215, 175, 193, 139, 220, 14, 24, - 206, 120, 202, 170, 24, 206, 120, 235, 135, 24, 206, 120, 251, 156, 24, - 215, 175, 203, 146, 210, 120, 24, 215, 173, 250, 253, 243, 13, 2, 202, - 170, 24, 215, 173, 250, 253, 243, 13, 216, 215, 202, 170, 24, 215, 173, - 250, 253, 243, 13, 2, 235, 135, 24, 215, 173, 250, 253, 243, 13, 216, - 215, 235, 135, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 2, - 202, 170, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 216, 215, - 202, 170, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 220, 15, - 202, 170, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 2, 235, - 135, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 216, 215, 235, - 135, 24, 215, 175, 250, 253, 243, 13, 229, 118, 193, 168, 220, 15, 235, - 135, 24, 215, 173, 193, 4, 243, 13, 232, 99, 202, 170, 24, 215, 173, 193, - 4, 243, 13, 232, 99, 235, 135, 24, 215, 175, 193, 4, 243, 13, 229, 118, - 193, 168, 202, 170, 24, 215, 175, 193, 4, 243, 13, 229, 118, 193, 168, - 235, 135, 24, 215, 173, 232, 255, 243, 10, 202, 170, 24, 215, 173, 232, - 255, 243, 10, 235, 135, 24, 215, 175, 232, 255, 243, 10, 229, 118, 193, - 168, 202, 170, 24, 215, 175, 232, 255, 243, 10, 229, 118, 193, 168, 235, - 135, 24, 235, 42, 250, 238, 202, 71, 24, 235, 42, 250, 238, 220, 14, 24, - 235, 42, 233, 75, 24, 235, 42, 202, 76, 24, 235, 42, 199, 88, 24, 235, - 42, 207, 64, 24, 235, 42, 202, 177, 24, 235, 42, 202, 178, 251, 156, 24, - 235, 42, 233, 227, 211, 29, 198, 146, 24, 235, 42, 230, 228, 24, 229, - 237, 24, 229, 238, 206, 194, 24, 229, 238, 215, 173, 201, 53, 24, 229, - 238, 215, 173, 198, 149, 24, 229, 238, 215, 175, 201, 53, 24, 229, 238, - 215, 173, 238, 166, 24, 229, 238, 215, 175, 238, 166, 24, 229, 238, 215, - 178, 243, 12, 24, 233, 106, 236, 235, 209, 27, 213, 16, 232, 135, 198, - 147, 24, 233, 106, 236, 235, 209, 27, 213, 16, 133, 211, 59, 235, 125, - 24, 233, 106, 236, 235, 209, 27, 213, 16, 133, 211, 59, 144, 198, 147, - 24, 233, 193, 201, 54, 196, 77, 24, 233, 193, 201, 54, 214, 83, 24, 233, - 193, 201, 54, 235, 125, 24, 235, 109, 233, 193, 214, 84, 235, 125, 24, - 235, 109, 233, 193, 144, 214, 83, 24, 235, 109, 233, 193, 133, 214, 83, - 24, 235, 109, 233, 193, 214, 84, 196, 77, 24, 232, 153, 214, 83, 24, 232, - 153, 239, 37, 24, 232, 153, 193, 7, 24, 233, 188, 211, 78, 24, 233, 188, - 201, 186, 24, 233, 188, 242, 220, 24, 233, 196, 248, 132, 202, 170, 24, - 233, 196, 248, 132, 215, 130, 24, 233, 188, 132, 211, 78, 24, 233, 188, - 193, 78, 211, 78, 24, 233, 188, 132, 242, 220, 24, 233, 188, 193, 76, - 215, 177, 24, 233, 196, 193, 58, 24, 233, 189, 196, 77, 24, 233, 189, - 235, 125, 24, 233, 189, 232, 182, 24, 233, 191, 201, 53, 24, 233, 191, - 201, 54, 235, 135, 24, 233, 191, 201, 54, 251, 156, 24, 233, 192, 201, - 53, 24, 233, 192, 201, 54, 235, 135, 24, 233, 192, 201, 54, 251, 156, 24, - 233, 191, 238, 164, 24, 233, 192, 238, 164, 24, 233, 191, 243, 7, 24, - 243, 123, 208, 157, 24, 243, 123, 214, 83, 24, 243, 123, 200, 218, 24, - 199, 89, 243, 123, 229, 137, 24, 199, 89, 243, 123, 216, 86, 24, 199, 89, - 243, 123, 218, 243, 24, 234, 211, 24, 213, 16, 214, 83, 24, 213, 16, 239, - 37, 24, 213, 16, 193, 5, 24, 213, 16, 193, 73, 24, 251, 228, 248, 118, - 211, 16, 24, 251, 228, 200, 217, 223, 164, 24, 251, 228, 248, 120, 2, - 206, 118, 24, 251, 228, 200, 219, 2, 206, 118, 24, 248, 41, 223, 136, 24, - 248, 41, 233, 216, 24, 215, 182, 242, 221, 214, 83, 24, 215, 182, 242, - 221, 232, 134, 24, 215, 182, 242, 221, 239, 37, 24, 215, 182, 202, 66, - 24, 215, 182, 202, 67, 193, 7, 24, 215, 182, 202, 67, 211, 78, 24, 215, - 182, 232, 95, 24, 215, 182, 232, 96, 193, 7, 24, 215, 182, 232, 96, 211, - 78, 24, 215, 182, 211, 79, 243, 12, 24, 215, 182, 211, 79, 232, 134, 24, - 215, 182, 211, 79, 193, 7, 24, 215, 182, 211, 79, 211, 9, 24, 215, 182, - 211, 79, 211, 10, 193, 7, 24, 215, 182, 211, 79, 211, 10, 192, 88, 24, - 215, 182, 211, 79, 207, 94, 24, 215, 182, 211, 79, 207, 95, 193, 7, 24, - 215, 182, 211, 79, 207, 95, 192, 88, 24, 215, 182, 221, 124, 24, 215, - 182, 221, 125, 232, 134, 24, 215, 182, 221, 125, 193, 7, 24, 215, 182, - 199, 88, 24, 215, 182, 199, 89, 232, 134, 24, 215, 182, 199, 89, 200, - 218, 24, 219, 99, 208, 221, 198, 87, 24, 219, 101, 110, 139, 196, 74, 24, - 219, 101, 116, 139, 218, 238, 24, 215, 182, 239, 2, 24, 215, 182, 193, 6, - 202, 170, 24, 215, 182, 193, 6, 235, 135, 24, 198, 62, 201, 75, 211, 17, - 233, 77, 24, 198, 62, 219, 144, 219, 98, 24, 198, 62, 198, 136, 248, 212, - 219, 98, 24, 198, 62, 198, 136, 198, 35, 223, 120, 215, 181, 24, 198, 62, - 223, 120, 215, 182, 207, 64, 24, 198, 62, 215, 172, 251, 253, 243, 124, - 24, 198, 62, 247, 24, 201, 75, 211, 16, 24, 198, 62, 247, 24, 223, 120, - 215, 181, 24, 199, 117, 24, 199, 118, 215, 177, 24, 199, 118, 211, 107, - 198, 61, 24, 199, 118, 211, 107, 198, 62, 215, 177, 24, 199, 118, 211, - 107, 219, 98, 24, 199, 118, 211, 107, 219, 99, 215, 177, 24, 199, 118, - 248, 148, 219, 98, 24, 215, 173, 223, 16, 24, 215, 175, 223, 16, 24, 214, - 114, 24, 230, 80, 24, 233, 219, 24, 203, 23, 229, 125, 201, 220, 24, 203, - 23, 229, 125, 209, 25, 24, 193, 165, 203, 23, 229, 125, 215, 180, 24, - 232, 194, 203, 23, 229, 125, 215, 180, 24, 203, 23, 198, 148, 232, 100, - 193, 172, 24, 198, 43, 201, 54, 201, 38, 24, 198, 43, 238, 165, 248, 229, - 24, 198, 44, 197, 16, 24, 116, 248, 107, 198, 148, 232, 100, 229, 125, - 222, 197, 24, 219, 126, 242, 81, 24, 219, 126, 219, 199, 24, 219, 126, - 219, 198, 24, 219, 126, 219, 197, 24, 219, 126, 219, 196, 24, 219, 126, - 219, 195, 24, 219, 126, 219, 194, 24, 219, 126, 219, 193, 24, 232, 254, - 24, 219, 39, 201, 248, 24, 219, 40, 201, 248, 24, 219, 42, 229, 210, 24, - 219, 42, 193, 74, 24, 219, 42, 237, 94, 24, 219, 42, 229, 238, 214, 114, - 24, 219, 42, 198, 45, 24, 219, 42, 219, 125, 237, 11, 24, 242, 76, 24, - 232, 82, 201, 64, 24, 204, 26, 24, 242, 85, 24, 210, 115, 24, 233, 8, - 215, 246, 24, 233, 8, 215, 245, 24, 233, 8, 215, 244, 24, 233, 8, 215, - 243, 24, 233, 8, 215, 242, 24, 206, 122, 215, 246, 24, 206, 122, 215, - 245, 24, 206, 122, 215, 244, 24, 206, 122, 215, 243, 24, 206, 122, 215, - 242, 24, 206, 122, 215, 241, 24, 206, 122, 215, 240, 24, 206, 122, 215, - 239, 24, 206, 122, 215, 253, 24, 206, 122, 215, 252, 24, 206, 122, 215, - 251, 24, 206, 122, 215, 250, 24, 206, 122, 215, 249, 24, 206, 122, 215, - 248, 24, 206, 122, 215, 247, 8, 2, 1, 233, 39, 237, 5, 4, 197, 228, 8, 2, - 1, 207, 19, 27, 232, 53, 8, 1, 2, 6, 154, 232, 53, 8, 2, 1, 207, 19, 222, - 154, 8, 1, 2, 6, 220, 145, 4, 248, 233, 8, 2, 1, 219, 165, 4, 207, 25, - 102, 8, 2, 1, 154, 192, 160, 4, 248, 233, 8, 2, 1, 207, 19, 234, 90, 8, - 2, 1, 154, 207, 224, 4, 180, 219, 215, 23, 207, 25, 102, 8, 2, 1, 200, - 44, 4, 228, 253, 23, 207, 25, 102, 8, 1, 207, 25, 242, 234, 4, 207, 25, - 102, 8, 2, 1, 234, 15, 4, 55, 164, 8, 2, 1, 234, 15, 4, 55, 249, 90, 23, - 238, 177, 8, 2, 1, 154, 200, 44, 4, 238, 177, 8, 1, 223, 95, 231, 13, - 201, 65, 4, 238, 177, 8, 1, 201, 37, 247, 196, 4, 238, 177, 8, 1, 2, 6, - 154, 222, 154, 8, 2, 1, 220, 145, 4, 232, 235, 8, 2, 1, 237, 72, 237, 5, - 4, 210, 194, 102, 8, 2, 1, 220, 145, 4, 248, 234, 23, 210, 194, 102, 8, - 2, 1, 234, 91, 4, 210, 194, 102, 8, 2, 1, 154, 207, 224, 4, 210, 194, - 102, 8, 2, 1, 207, 224, 4, 232, 236, 23, 210, 194, 102, 8, 2, 1, 199, 79, - 237, 5, 4, 210, 194, 102, 8, 2, 1, 233, 181, 4, 210, 194, 102, 8, 2, 1, - 237, 72, 237, 5, 4, 207, 25, 102, 8, 2, 1, 228, 76, 4, 201, 29, 23, 207, - 25, 102, 8, 2, 1, 187, 4, 207, 25, 102, 8, 2, 1, 199, 79, 237, 5, 4, 207, - 25, 102, 8, 2, 1, 247, 196, 4, 207, 25, 102, 8, 2, 1, 206, 10, 4, 238, - 177, 8, 2, 1, 238, 130, 4, 216, 88, 45, 102, 8, 2, 1, 220, 145, 4, 216, - 88, 45, 102, 8, 2, 1, 215, 64, 4, 216, 88, 45, 102, 8, 2, 1, 207, 224, 4, - 216, 88, 45, 102, 8, 2, 1, 206, 10, 4, 216, 88, 45, 102, 8, 2, 1, 200, - 44, 4, 216, 88, 45, 102, 33, 135, 1, 250, 124, 33, 135, 1, 247, 254, 33, - 135, 1, 195, 151, 33, 135, 1, 231, 20, 33, 135, 1, 236, 171, 33, 135, 1, - 192, 49, 33, 135, 1, 191, 55, 33, 135, 1, 191, 82, 33, 135, 1, 223, 41, - 33, 135, 1, 89, 223, 41, 33, 135, 1, 68, 33, 135, 1, 236, 192, 33, 135, - 1, 222, 96, 33, 135, 1, 219, 77, 33, 135, 1, 215, 68, 33, 135, 1, 214, - 212, 33, 135, 1, 211, 91, 33, 135, 1, 209, 57, 33, 135, 1, 206, 180, 33, - 135, 1, 202, 78, 33, 135, 1, 197, 44, 33, 135, 1, 196, 124, 33, 135, 1, - 232, 103, 33, 135, 1, 229, 190, 33, 135, 1, 203, 9, 33, 135, 1, 197, 146, - 33, 135, 1, 243, 56, 33, 135, 1, 203, 166, 33, 135, 1, 192, 58, 33, 135, - 1, 192, 60, 33, 135, 1, 192, 93, 33, 135, 1, 191, 225, 33, 135, 1, 2, - 191, 190, 33, 135, 1, 192, 12, 33, 135, 1, 223, 84, 2, 191, 190, 33, 135, - 1, 248, 177, 191, 190, 33, 135, 1, 223, 84, 248, 177, 191, 190, 33, 135, - 1, 232, 230, 52, 1, 38, 2, 65, 52, 1, 38, 2, 249, 19, 52, 1, 38, 2, 195, - 153, 52, 1, 38, 2, 231, 79, 52, 1, 38, 2, 237, 148, 52, 1, 38, 2, 223, - 228, 52, 1, 38, 2, 191, 62, 52, 1, 38, 2, 191, 87, 52, 1, 38, 2, 68, 52, - 1, 38, 2, 155, 52, 1, 38, 2, 234, 142, 52, 1, 38, 2, 234, 116, 52, 1, 38, - 2, 74, 52, 1, 38, 2, 210, 65, 52, 1, 38, 2, 234, 36, 52, 1, 38, 2, 234, - 24, 52, 1, 38, 2, 199, 145, 52, 1, 38, 2, 66, 52, 1, 38, 2, 234, 183, 52, - 1, 38, 2, 140, 52, 1, 38, 2, 197, 161, 52, 1, 38, 2, 243, 129, 52, 1, 38, - 2, 203, 166, 52, 1, 38, 2, 192, 58, 52, 1, 38, 2, 71, 52, 1, 38, 2, 191, - 225, 52, 1, 38, 2, 235, 19, 52, 1, 38, 2, 205, 87, 52, 1, 38, 2, 247, - 205, 68, 52, 1, 38, 2, 223, 12, 52, 1, 38, 2, 249, 84, 74, 52, 1, 38, 2, - 201, 54, 66, 52, 1, 38, 2, 210, 181, 38, 200, 231, 2, 1, 65, 38, 200, - 231, 2, 1, 249, 19, 38, 200, 231, 2, 1, 195, 153, 38, 200, 231, 2, 1, - 231, 79, 38, 200, 231, 2, 1, 237, 148, 38, 200, 231, 2, 1, 223, 228, 38, - 200, 231, 2, 1, 191, 62, 38, 200, 231, 2, 1, 191, 87, 38, 200, 231, 2, 1, - 68, 38, 200, 231, 2, 1, 155, 38, 200, 231, 2, 1, 234, 142, 38, 200, 231, - 2, 1, 74, 38, 200, 231, 2, 1, 210, 65, 38, 200, 231, 2, 1, 234, 24, 38, - 200, 231, 2, 1, 66, 38, 200, 231, 2, 1, 234, 183, 38, 200, 231, 2, 1, - 140, 38, 200, 231, 2, 1, 197, 161, 38, 200, 231, 2, 1, 243, 129, 38, 200, - 231, 2, 1, 203, 166, 38, 200, 231, 2, 1, 230, 19, 56, 38, 200, 231, 2, 1, - 192, 58, 38, 200, 231, 2, 1, 231, 80, 4, 196, 69, 38, 200, 231, 2, 1, - 247, 205, 68, 38, 200, 231, 2, 1, 235, 34, 38, 200, 231, 2, 1, 235, 30, - 52, 1, 38, 2, 234, 25, 4, 237, 89, 52, 1, 38, 2, 192, 59, 4, 249, 149, - 192, 62, 52, 1, 38, 2, 201, 54, 126, 4, 106, 33, 38, 2, 1, 247, 205, 68, - 212, 82, 208, 164, 90, 1, 174, 212, 82, 208, 164, 90, 1, 197, 168, 212, - 82, 208, 164, 90, 1, 212, 201, 212, 82, 208, 164, 90, 1, 190, 190, 212, - 82, 208, 164, 90, 1, 140, 212, 82, 208, 164, 90, 1, 181, 212, 82, 208, - 164, 90, 1, 192, 220, 212, 82, 208, 164, 90, 1, 213, 113, 212, 82, 208, - 164, 90, 1, 247, 162, 212, 82, 208, 164, 90, 1, 173, 212, 82, 208, 164, - 90, 1, 188, 212, 82, 208, 164, 90, 1, 191, 123, 212, 82, 208, 164, 90, 1, - 214, 168, 212, 82, 208, 164, 90, 1, 212, 188, 212, 82, 208, 164, 90, 1, - 155, 212, 82, 208, 164, 90, 1, 238, 34, 212, 82, 208, 164, 90, 1, 212, - 103, 212, 82, 208, 164, 90, 1, 212, 246, 212, 82, 208, 164, 90, 1, 195, - 188, 212, 82, 208, 164, 90, 1, 212, 182, 212, 82, 208, 164, 90, 1, 197, - 8, 212, 82, 208, 164, 90, 1, 233, 111, 212, 82, 208, 164, 90, 1, 165, - 212, 82, 208, 164, 90, 1, 208, 98, 212, 82, 208, 164, 90, 1, 170, 212, - 82, 208, 164, 90, 1, 212, 248, 212, 82, 208, 164, 90, 1, 168, 212, 82, - 208, 164, 90, 1, 192, 175, 212, 82, 208, 164, 90, 1, 212, 250, 212, 82, - 208, 164, 90, 1, 236, 188, 212, 82, 208, 164, 90, 1, 212, 249, 212, 82, - 208, 164, 90, 1, 230, 83, 212, 82, 208, 164, 90, 1, 216, 21, 212, 82, - 208, 164, 90, 1, 209, 112, 212, 82, 208, 164, 90, 1, 231, 242, 212, 82, - 208, 164, 90, 1, 206, 110, 212, 82, 208, 164, 90, 1, 65, 212, 82, 208, - 164, 90, 1, 252, 208, 212, 82, 208, 164, 90, 1, 68, 212, 82, 208, 164, - 90, 1, 66, 212, 82, 208, 164, 90, 1, 74, 212, 82, 208, 164, 90, 1, 211, - 89, 212, 82, 208, 164, 90, 1, 71, 212, 82, 208, 164, 90, 1, 234, 190, - 212, 82, 208, 164, 90, 1, 193, 224, 212, 82, 208, 164, 90, 198, 70, 212, - 82, 208, 164, 90, 198, 66, 212, 82, 208, 164, 90, 198, 67, 212, 82, 208, - 164, 90, 198, 64, 212, 82, 208, 164, 90, 198, 65, 212, 82, 208, 164, 90, - 198, 68, 212, 82, 208, 164, 90, 198, 69, 212, 82, 208, 164, 90, 3, 40, - 209, 252, 212, 82, 208, 164, 90, 3, 40, 199, 3, 212, 82, 208, 164, 90, 3, - 40, 219, 41, 212, 82, 208, 164, 90, 3, 40, 251, 103, 212, 82, 208, 164, - 90, 3, 40, 223, 96, 212, 82, 208, 164, 90, 3, 192, 183, 192, 182, 212, - 82, 208, 164, 90, 5, 219, 192, 212, 82, 208, 164, 90, 17, 191, 77, 212, - 82, 208, 164, 90, 17, 107, 212, 82, 208, 164, 90, 17, 109, 212, 82, 208, - 164, 90, 17, 138, 212, 82, 208, 164, 90, 17, 134, 212, 82, 208, 164, 90, - 17, 150, 212, 82, 208, 164, 90, 17, 169, 212, 82, 208, 164, 90, 17, 175, - 212, 82, 208, 164, 90, 17, 171, 212, 82, 208, 164, 90, 17, 178, 212, 82, - 208, 164, 90, 219, 30, 212, 98, 212, 82, 208, 164, 90, 47, 247, 162, 198, - 38, 1, 168, 198, 38, 1, 249, 155, 198, 38, 1, 190, 190, 198, 38, 1, 238, - 34, 198, 38, 1, 155, 198, 38, 1, 231, 242, 198, 38, 1, 174, 198, 38, 1, - 181, 198, 38, 1, 214, 70, 198, 38, 1, 188, 198, 38, 1, 247, 3, 198, 38, - 1, 170, 198, 38, 1, 193, 190, 198, 38, 1, 223, 34, 198, 38, 1, 140, 198, - 38, 1, 165, 198, 38, 1, 173, 198, 38, 1, 68, 198, 38, 1, 248, 40, 68, - 198, 38, 1, 223, 51, 198, 38, 1, 248, 40, 223, 51, 198, 38, 1, 66, 198, - 38, 1, 71, 198, 38, 1, 248, 40, 71, 198, 38, 1, 234, 67, 198, 38, 1, 248, - 40, 234, 67, 198, 38, 1, 74, 198, 38, 1, 252, 27, 198, 38, 1, 248, 40, - 252, 27, 198, 38, 1, 65, 198, 38, 3, 206, 181, 198, 79, 193, 163, 1, 252, - 208, 193, 163, 1, 65, 193, 163, 1, 249, 155, 193, 163, 1, 247, 162, 193, - 163, 1, 238, 34, 193, 163, 1, 231, 242, 193, 163, 1, 170, 193, 163, 1, - 209, 230, 193, 163, 1, 173, 193, 163, 1, 181, 193, 163, 1, 168, 193, 163, - 1, 190, 190, 193, 163, 1, 199, 49, 193, 163, 1, 233, 111, 193, 163, 1, - 188, 193, 163, 1, 203, 166, 193, 163, 1, 223, 34, 193, 163, 1, 191, 123, - 193, 163, 1, 193, 190, 193, 163, 1, 195, 188, 193, 163, 1, 155, 193, 163, - 1, 74, 193, 163, 1, 250, 165, 193, 163, 1, 165, 193, 163, 1, 174, 193, - 163, 1, 221, 217, 193, 163, 1, 140, 193, 163, 1, 71, 193, 163, 1, 68, - 193, 163, 1, 214, 70, 193, 163, 1, 66, 193, 163, 1, 219, 68, 193, 163, 1, - 197, 168, 193, 163, 1, 198, 26, 193, 163, 1, 211, 96, 193, 163, 1, 252, - 167, 193, 163, 1, 251, 124, 193, 163, 1, 223, 138, 193, 163, 1, 211, 106, - 193, 163, 1, 234, 105, 193, 163, 1, 252, 168, 193, 163, 1, 212, 103, 193, - 163, 1, 196, 147, 193, 163, 1, 192, 24, 193, 163, 163, 197, 67, 193, 163, - 163, 197, 66, 193, 163, 163, 221, 56, 193, 163, 163, 221, 55, 193, 163, - 17, 191, 77, 193, 163, 17, 107, 193, 163, 17, 109, 193, 163, 17, 138, - 193, 163, 17, 134, 193, 163, 17, 150, 193, 163, 17, 169, 193, 163, 17, - 175, 193, 163, 17, 171, 193, 163, 17, 178, 193, 163, 213, 234, 56, 214, - 245, 215, 122, 1, 74, 214, 245, 215, 122, 1, 211, 80, 214, 245, 215, 122, - 1, 211, 122, 214, 245, 215, 122, 1, 210, 244, 214, 245, 215, 122, 1, 211, - 96, 214, 245, 215, 122, 1, 65, 214, 245, 215, 122, 1, 251, 220, 214, 245, - 215, 122, 1, 252, 157, 214, 245, 215, 122, 1, 251, 71, 214, 245, 215, - 122, 1, 251, 247, 214, 245, 215, 122, 1, 68, 214, 245, 215, 122, 1, 223, - 70, 214, 245, 215, 122, 1, 228, 20, 214, 245, 215, 122, 1, 223, 55, 214, - 245, 215, 122, 1, 223, 202, 214, 245, 215, 122, 1, 66, 214, 245, 215, - 122, 1, 196, 160, 214, 245, 215, 122, 1, 196, 158, 214, 245, 215, 122, 1, - 196, 128, 214, 245, 215, 122, 1, 196, 62, 214, 245, 215, 122, 1, 71, 214, - 245, 215, 122, 1, 234, 87, 214, 245, 215, 122, 1, 234, 182, 214, 245, - 215, 122, 1, 234, 116, 214, 245, 215, 122, 1, 234, 105, 214, 245, 215, - 122, 1, 233, 247, 214, 245, 215, 122, 1, 234, 124, 214, 245, 215, 122, 3, - 211, 129, 214, 245, 215, 122, 3, 215, 140, 214, 245, 215, 122, 3, 198, - 28, 214, 245, 215, 122, 3, 223, 195, 214, 245, 215, 122, 3, 200, 161, - 214, 245, 215, 122, 17, 191, 77, 214, 245, 215, 122, 17, 107, 214, 245, - 215, 122, 17, 109, 214, 245, 215, 122, 17, 138, 214, 245, 215, 122, 17, - 134, 214, 245, 215, 122, 17, 150, 214, 245, 215, 122, 17, 169, 214, 245, - 215, 122, 17, 175, 214, 245, 215, 122, 17, 171, 214, 245, 215, 122, 17, - 178, 36, 5, 229, 168, 36, 5, 229, 162, 36, 5, 229, 164, 36, 5, 229, 167, - 36, 5, 229, 165, 36, 5, 229, 166, 36, 5, 229, 163, 36, 5, 230, 149, 229, - 172, 36, 5, 229, 169, 36, 5, 229, 170, 36, 5, 229, 171, 36, 5, 230, 149, - 215, 78, 36, 5, 230, 149, 215, 79, 36, 5, 230, 149, 207, 238, 36, 5, 230, - 149, 207, 239, 36, 5, 230, 149, 207, 240, 36, 5, 230, 149, 247, 209, 36, - 5, 230, 149, 247, 210, 36, 5, 230, 149, 220, 174, 36, 5, 230, 149, 220, - 175, 36, 5, 230, 149, 220, 176, 36, 5, 230, 149, 230, 133, 36, 5, 230, - 149, 230, 134, 36, 5, 230, 149, 230, 135, 36, 5, 230, 149, 232, 60, 36, - 5, 230, 149, 232, 61, 36, 5, 230, 149, 208, 120, 36, 5, 230, 149, 208, - 121, 85, 84, 5, 218, 169, 221, 168, 85, 84, 5, 218, 165, 155, 85, 84, 5, - 218, 163, 220, 234, 85, 84, 5, 218, 39, 222, 15, 85, 84, 5, 218, 9, 222, - 24, 85, 84, 5, 218, 28, 221, 43, 85, 84, 5, 218, 56, 221, 69, 85, 84, 5, - 217, 181, 220, 221, 85, 84, 5, 218, 160, 193, 86, 85, 84, 5, 218, 158, - 193, 190, 85, 84, 5, 218, 156, 193, 0, 85, 84, 5, 217, 234, 193, 114, 85, - 84, 5, 217, 242, 193, 125, 85, 84, 5, 217, 246, 193, 29, 85, 84, 5, 218, - 59, 193, 48, 85, 84, 5, 217, 166, 192, 252, 85, 84, 5, 217, 217, 193, - 112, 85, 84, 5, 218, 43, 192, 240, 85, 84, 5, 218, 55, 192, 242, 85, 84, - 5, 217, 221, 192, 241, 85, 84, 5, 218, 154, 216, 46, 85, 84, 5, 218, 152, - 217, 92, 85, 84, 5, 218, 150, 215, 124, 85, 84, 5, 218, 45, 216, 188, 85, - 84, 5, 218, 10, 215, 233, 85, 84, 5, 217, 206, 215, 150, 85, 84, 5, 217, - 171, 215, 144, 85, 84, 5, 218, 148, 248, 190, 85, 84, 5, 218, 145, 249, - 155, 85, 84, 5, 218, 143, 248, 12, 85, 84, 5, 217, 210, 249, 3, 85, 84, - 5, 218, 7, 249, 19, 85, 84, 5, 218, 1, 248, 99, 85, 84, 5, 217, 222, 248, - 113, 85, 84, 5, 218, 133, 68, 85, 84, 5, 218, 131, 65, 85, 84, 5, 218, - 129, 66, 85, 84, 5, 217, 197, 234, 190, 85, 84, 5, 218, 4, 71, 85, 84, 5, - 217, 195, 211, 89, 85, 84, 5, 217, 213, 74, 85, 84, 5, 217, 223, 234, - 168, 85, 84, 5, 217, 229, 223, 164, 85, 84, 5, 217, 225, 223, 164, 85, - 84, 5, 217, 165, 251, 134, 85, 84, 5, 217, 182, 234, 105, 85, 84, 5, 218, - 118, 202, 223, 85, 84, 5, 218, 116, 188, 85, 84, 5, 218, 114, 201, 5, 85, - 84, 5, 217, 198, 205, 51, 85, 84, 5, 217, 244, 205, 69, 85, 84, 5, 217, - 224, 202, 17, 85, 84, 5, 218, 25, 202, 47, 85, 84, 5, 217, 164, 202, 216, - 85, 84, 5, 218, 104, 219, 148, 85, 84, 5, 218, 102, 173, 85, 84, 5, 218, - 100, 218, 227, 85, 84, 5, 218, 20, 219, 230, 85, 84, 5, 218, 31, 219, - 240, 85, 84, 5, 218, 50, 219, 10, 85, 84, 5, 217, 207, 219, 45, 85, 84, - 5, 217, 250, 180, 219, 240, 85, 84, 5, 218, 126, 237, 46, 85, 84, 5, 218, - 123, 238, 34, 85, 84, 5, 218, 120, 235, 91, 85, 84, 5, 218, 15, 237, 133, - 85, 84, 5, 217, 180, 236, 148, 85, 84, 5, 217, 179, 236, 176, 85, 84, 5, - 218, 112, 198, 193, 85, 84, 5, 218, 109, 190, 190, 85, 84, 5, 218, 107, - 197, 94, 85, 84, 5, 218, 13, 199, 121, 85, 84, 5, 218, 49, 199, 145, 85, - 84, 5, 218, 0, 198, 59, 85, 84, 5, 218, 35, 159, 85, 84, 5, 218, 98, 222, - 246, 85, 84, 5, 218, 95, 223, 34, 85, 84, 5, 218, 93, 222, 184, 85, 84, - 5, 217, 203, 223, 10, 85, 84, 5, 217, 247, 223, 12, 85, 84, 5, 217, 200, - 222, 193, 85, 84, 5, 218, 41, 222, 203, 85, 84, 5, 217, 185, 180, 222, - 203, 85, 84, 5, 218, 91, 192, 33, 85, 84, 5, 218, 88, 170, 85, 84, 5, - 218, 86, 191, 225, 85, 84, 5, 217, 251, 192, 77, 85, 84, 5, 218, 24, 192, - 80, 85, 84, 5, 217, 219, 191, 246, 85, 84, 5, 217, 239, 192, 12, 85, 84, - 5, 218, 82, 233, 25, 85, 84, 5, 218, 80, 233, 111, 85, 84, 5, 218, 78, - 232, 88, 85, 84, 5, 218, 26, 233, 54, 85, 84, 5, 218, 29, 233, 61, 85, - 84, 5, 217, 227, 232, 164, 85, 84, 5, 218, 16, 232, 177, 85, 84, 5, 217, - 163, 232, 87, 85, 84, 5, 218, 3, 233, 82, 85, 84, 5, 218, 76, 213, 181, - 85, 84, 5, 218, 74, 214, 228, 85, 84, 5, 218, 72, 212, 132, 85, 84, 5, - 217, 243, 214, 104, 85, 84, 5, 217, 191, 213, 33, 85, 84, 5, 217, 184, - 229, 160, 85, 84, 5, 218, 67, 140, 85, 84, 5, 217, 174, 228, 161, 85, 84, - 5, 218, 70, 229, 217, 85, 84, 5, 218, 8, 229, 247, 85, 84, 5, 218, 65, - 228, 254, 85, 84, 5, 217, 220, 229, 25, 85, 84, 5, 218, 21, 229, 216, 85, - 84, 5, 217, 232, 228, 247, 85, 84, 5, 218, 51, 229, 130, 85, 84, 5, 217, - 230, 230, 58, 85, 84, 5, 218, 17, 228, 144, 85, 84, 5, 218, 52, 229, 200, - 85, 84, 5, 217, 167, 229, 1, 85, 84, 5, 218, 58, 228, 157, 85, 84, 5, - 218, 14, 214, 35, 85, 84, 5, 218, 63, 214, 49, 85, 84, 5, 218, 22, 214, - 32, 85, 84, 5, 217, 245, 214, 43, 85, 84, 5, 217, 214, 214, 44, 85, 84, - 5, 217, 204, 214, 33, 85, 84, 5, 217, 240, 214, 34, 85, 84, 5, 217, 201, - 214, 48, 85, 84, 5, 217, 233, 214, 31, 85, 84, 5, 218, 18, 180, 214, 44, - 85, 84, 5, 217, 254, 180, 214, 33, 85, 84, 5, 217, 177, 180, 214, 34, 85, - 84, 5, 217, 205, 231, 55, 85, 84, 5, 217, 249, 231, 242, 85, 84, 5, 217, - 192, 230, 181, 85, 84, 5, 217, 170, 231, 159, 85, 84, 5, 217, 194, 230, - 167, 85, 84, 5, 217, 193, 230, 177, 85, 84, 5, 217, 176, 214, 54, 85, 84, - 5, 218, 47, 213, 247, 85, 84, 5, 217, 183, 213, 236, 85, 84, 5, 218, 36, - 209, 187, 85, 84, 5, 218, 5, 168, 85, 84, 5, 218, 54, 208, 167, 85, 84, - 5, 218, 23, 210, 51, 85, 84, 5, 218, 53, 210, 65, 85, 84, 5, 218, 2, 209, - 39, 85, 84, 5, 218, 38, 209, 75, 85, 84, 5, 217, 215, 216, 254, 85, 84, - 5, 218, 42, 217, 13, 85, 84, 5, 217, 238, 216, 248, 85, 84, 5, 218, 57, - 217, 5, 85, 84, 5, 217, 172, 217, 5, 85, 84, 5, 218, 32, 217, 6, 85, 84, - 5, 217, 188, 216, 249, 85, 84, 5, 217, 186, 216, 250, 85, 84, 5, 217, - 173, 216, 242, 85, 84, 5, 217, 199, 180, 217, 6, 85, 84, 5, 217, 255, - 180, 216, 249, 85, 84, 5, 217, 218, 180, 216, 250, 85, 84, 5, 217, 228, - 221, 15, 85, 84, 5, 218, 12, 221, 23, 85, 84, 5, 218, 30, 221, 11, 85, - 84, 5, 218, 61, 221, 18, 85, 84, 5, 217, 252, 221, 19, 85, 84, 5, 217, - 248, 221, 13, 85, 84, 5, 217, 202, 221, 14, 85, 84, 5, 217, 236, 231, - 176, 85, 84, 5, 218, 48, 231, 184, 85, 84, 5, 217, 212, 231, 171, 85, 84, - 5, 218, 11, 231, 180, 85, 84, 5, 217, 253, 231, 181, 85, 84, 5, 218, 33, - 231, 172, 85, 84, 5, 218, 34, 231, 174, 85, 84, 5, 217, 189, 165, 85, 84, - 5, 217, 237, 214, 149, 85, 84, 5, 217, 231, 214, 164, 85, 84, 5, 217, - 235, 214, 131, 85, 84, 5, 217, 169, 214, 155, 85, 84, 5, 217, 241, 214, - 156, 85, 84, 5, 218, 37, 214, 136, 85, 84, 5, 218, 40, 214, 140, 85, 84, - 5, 217, 208, 213, 159, 85, 84, 5, 217, 168, 213, 129, 85, 84, 5, 217, - 211, 213, 150, 85, 84, 5, 217, 226, 213, 133, 85, 84, 5, 217, 178, 195, - 69, 85, 84, 5, 217, 175, 195, 188, 85, 84, 5, 217, 209, 193, 249, 85, 84, - 5, 217, 187, 195, 148, 85, 84, 5, 218, 19, 195, 153, 85, 84, 5, 217, 216, - 195, 8, 85, 84, 5, 218, 27, 195, 24, 85, 84, 5, 217, 196, 212, 76, 85, - 84, 5, 218, 46, 212, 96, 85, 84, 5, 217, 190, 212, 58, 85, 84, 5, 218, 6, - 212, 88, 85, 84, 5, 218, 44, 212, 65, 85, 84, 17, 107, 85, 84, 17, 109, - 85, 84, 17, 138, 85, 84, 17, 134, 85, 84, 17, 150, 85, 84, 17, 169, 85, - 84, 17, 175, 85, 84, 17, 171, 85, 84, 17, 178, 85, 84, 33, 31, 199, 119, - 85, 84, 33, 31, 199, 90, 85, 84, 33, 31, 228, 140, 85, 84, 33, 31, 198, - 228, 85, 84, 33, 31, 199, 96, 198, 228, 85, 84, 33, 31, 228, 143, 198, - 228, 85, 84, 33, 31, 216, 49, 252, 35, 6, 1, 251, 182, 252, 35, 6, 1, - 238, 31, 252, 35, 6, 1, 220, 125, 252, 35, 6, 1, 216, 62, 252, 35, 6, 1, - 249, 155, 252, 35, 6, 1, 202, 165, 252, 35, 6, 1, 210, 65, 252, 35, 6, 1, - 248, 198, 252, 35, 6, 1, 165, 252, 35, 6, 1, 71, 252, 35, 6, 1, 233, 111, - 252, 35, 6, 1, 68, 252, 35, 6, 1, 74, 252, 35, 6, 1, 237, 70, 252, 35, 6, - 1, 192, 34, 252, 35, 6, 1, 193, 133, 252, 35, 6, 1, 212, 132, 252, 35, 6, - 1, 222, 108, 252, 35, 6, 1, 170, 252, 35, 6, 1, 66, 252, 35, 6, 1, 222, - 237, 252, 35, 6, 1, 243, 97, 252, 35, 6, 1, 140, 252, 35, 6, 1, 208, 96, - 252, 35, 6, 1, 231, 242, 252, 35, 6, 1, 212, 103, 252, 35, 6, 1, 197, 94, - 252, 35, 6, 1, 213, 226, 252, 35, 6, 1, 195, 188, 252, 35, 6, 1, 221, - 217, 252, 35, 6, 1, 231, 181, 252, 35, 6, 1, 191, 108, 252, 35, 6, 1, - 221, 14, 252, 35, 6, 1, 203, 166, 252, 35, 2, 1, 251, 182, 252, 35, 2, 1, - 238, 31, 252, 35, 2, 1, 220, 125, 252, 35, 2, 1, 216, 62, 252, 35, 2, 1, - 249, 155, 252, 35, 2, 1, 202, 165, 252, 35, 2, 1, 210, 65, 252, 35, 2, 1, - 248, 198, 252, 35, 2, 1, 165, 252, 35, 2, 1, 71, 252, 35, 2, 1, 233, 111, - 252, 35, 2, 1, 68, 252, 35, 2, 1, 74, 252, 35, 2, 1, 237, 70, 252, 35, 2, - 1, 192, 34, 252, 35, 2, 1, 193, 133, 252, 35, 2, 1, 212, 132, 252, 35, 2, - 1, 222, 108, 252, 35, 2, 1, 170, 252, 35, 2, 1, 66, 252, 35, 2, 1, 222, - 237, 252, 35, 2, 1, 243, 97, 252, 35, 2, 1, 140, 252, 35, 2, 1, 208, 96, - 252, 35, 2, 1, 231, 242, 252, 35, 2, 1, 212, 103, 252, 35, 2, 1, 197, 94, - 252, 35, 2, 1, 213, 226, 252, 35, 2, 1, 195, 188, 252, 35, 2, 1, 221, - 217, 252, 35, 2, 1, 231, 181, 252, 35, 2, 1, 191, 108, 252, 35, 2, 1, - 221, 14, 252, 35, 2, 1, 203, 166, 252, 35, 251, 183, 219, 192, 252, 35, - 18, 219, 192, 252, 35, 231, 155, 77, 252, 35, 230, 59, 252, 35, 120, 215, - 254, 252, 35, 231, 156, 120, 215, 254, 252, 35, 212, 143, 252, 35, 214, - 215, 77, 252, 35, 17, 191, 77, 252, 35, 17, 107, 252, 35, 17, 109, 252, - 35, 17, 138, 252, 35, 17, 134, 252, 35, 17, 150, 252, 35, 17, 169, 252, - 35, 17, 175, 252, 35, 17, 171, 252, 35, 17, 178, 252, 35, 89, 233, 218, - 77, 252, 35, 89, 208, 15, 77, 223, 148, 143, 31, 107, 223, 148, 143, 31, - 109, 223, 148, 143, 31, 138, 223, 148, 143, 31, 134, 223, 148, 143, 31, - 150, 223, 148, 143, 31, 169, 223, 148, 143, 31, 175, 223, 148, 143, 31, - 171, 223, 148, 143, 31, 178, 223, 148, 143, 31, 199, 95, 223, 148, 143, - 31, 197, 32, 223, 148, 143, 31, 198, 249, 223, 148, 143, 31, 232, 137, - 223, 148, 143, 31, 233, 17, 223, 148, 143, 31, 202, 121, 223, 148, 143, - 31, 203, 242, 223, 148, 143, 31, 234, 155, 223, 148, 143, 31, 213, 171, - 223, 148, 143, 31, 91, 228, 142, 223, 148, 143, 31, 105, 228, 142, 223, - 148, 143, 31, 115, 228, 142, 223, 148, 143, 31, 232, 130, 228, 142, 223, - 148, 143, 31, 232, 228, 228, 142, 223, 148, 143, 31, 202, 137, 228, 142, - 223, 148, 143, 31, 203, 248, 228, 142, 223, 148, 143, 31, 234, 166, 228, - 142, 223, 148, 143, 31, 213, 177, 228, 142, 223, 148, 143, 31, 91, 189, - 223, 148, 143, 31, 105, 189, 223, 148, 143, 31, 115, 189, 223, 148, 143, - 31, 232, 130, 189, 223, 148, 143, 31, 232, 228, 189, 223, 148, 143, 31, - 202, 137, 189, 223, 148, 143, 31, 203, 248, 189, 223, 148, 143, 31, 234, - 166, 189, 223, 148, 143, 31, 213, 177, 189, 223, 148, 143, 31, 199, 96, - 189, 223, 148, 143, 31, 197, 33, 189, 223, 148, 143, 31, 198, 250, 189, - 223, 148, 143, 31, 232, 138, 189, 223, 148, 143, 31, 233, 18, 189, 223, - 148, 143, 31, 202, 122, 189, 223, 148, 143, 31, 203, 243, 189, 223, 148, - 143, 31, 234, 156, 189, 223, 148, 143, 31, 213, 172, 189, 223, 148, 143, - 31, 220, 43, 223, 148, 143, 31, 220, 42, 223, 148, 143, 220, 44, 77, 223, - 148, 143, 31, 222, 62, 223, 148, 143, 31, 222, 61, 223, 148, 143, 31, - 208, 229, 107, 223, 148, 143, 31, 208, 229, 109, 223, 148, 143, 31, 208, - 229, 138, 223, 148, 143, 31, 208, 229, 134, 223, 148, 143, 31, 208, 229, - 150, 223, 148, 143, 31, 208, 229, 169, 223, 148, 143, 31, 208, 229, 175, - 223, 148, 143, 31, 208, 229, 171, 223, 148, 143, 31, 208, 229, 178, 223, - 148, 143, 209, 108, 223, 148, 143, 232, 120, 91, 208, 24, 223, 148, 143, - 232, 120, 91, 230, 72, 223, 148, 143, 232, 120, 115, 208, 22, 223, 148, - 143, 206, 37, 77, 223, 148, 143, 31, 251, 159, 107, 223, 148, 143, 31, - 251, 159, 109, 223, 148, 143, 31, 251, 159, 199, 96, 189, 223, 148, 143, - 251, 159, 220, 44, 77, 211, 23, 143, 31, 107, 211, 23, 143, 31, 109, 211, - 23, 143, 31, 138, 211, 23, 143, 31, 134, 211, 23, 143, 31, 150, 211, 23, - 143, 31, 169, 211, 23, 143, 31, 175, 211, 23, 143, 31, 171, 211, 23, 143, - 31, 178, 211, 23, 143, 31, 199, 95, 211, 23, 143, 31, 197, 32, 211, 23, - 143, 31, 198, 249, 211, 23, 143, 31, 232, 137, 211, 23, 143, 31, 233, 17, - 211, 23, 143, 31, 202, 121, 211, 23, 143, 31, 203, 242, 211, 23, 143, 31, - 234, 155, 211, 23, 143, 31, 213, 171, 211, 23, 143, 31, 91, 228, 142, - 211, 23, 143, 31, 105, 228, 142, 211, 23, 143, 31, 115, 228, 142, 211, - 23, 143, 31, 232, 130, 228, 142, 211, 23, 143, 31, 232, 228, 228, 142, - 211, 23, 143, 31, 202, 137, 228, 142, 211, 23, 143, 31, 203, 248, 228, - 142, 211, 23, 143, 31, 234, 166, 228, 142, 211, 23, 143, 31, 213, 177, - 228, 142, 211, 23, 143, 31, 91, 189, 211, 23, 143, 31, 105, 189, 211, 23, - 143, 31, 115, 189, 211, 23, 143, 31, 232, 130, 189, 211, 23, 143, 31, - 232, 228, 189, 211, 23, 143, 31, 202, 137, 189, 211, 23, 143, 31, 203, - 248, 189, 211, 23, 143, 31, 234, 166, 189, 211, 23, 143, 31, 213, 177, - 189, 211, 23, 143, 31, 199, 96, 189, 211, 23, 143, 31, 197, 33, 189, 211, - 23, 143, 31, 198, 250, 189, 211, 23, 143, 31, 232, 138, 189, 211, 23, - 143, 31, 233, 18, 189, 211, 23, 143, 31, 202, 122, 189, 211, 23, 143, 31, - 203, 243, 189, 211, 23, 143, 31, 234, 156, 189, 211, 23, 143, 31, 213, - 172, 189, 211, 23, 143, 217, 51, 211, 23, 143, 251, 159, 31, 109, 211, - 23, 143, 251, 159, 31, 138, 211, 23, 143, 251, 159, 31, 134, 211, 23, - 143, 251, 159, 31, 150, 211, 23, 143, 251, 159, 31, 169, 211, 23, 143, - 251, 159, 31, 175, 211, 23, 143, 251, 159, 31, 171, 211, 23, 143, 251, - 159, 31, 178, 211, 23, 143, 251, 159, 31, 199, 95, 211, 23, 143, 251, - 159, 31, 232, 130, 228, 142, 211, 23, 143, 251, 159, 31, 202, 137, 228, - 142, 211, 23, 143, 251, 159, 31, 105, 189, 211, 23, 143, 251, 159, 31, - 199, 96, 189, 211, 23, 143, 232, 120, 91, 230, 72, 211, 23, 143, 232, - 120, 91, 202, 125, 9, 13, 251, 194, 9, 13, 248, 247, 9, 13, 223, 8, 9, - 13, 238, 5, 9, 13, 193, 133, 9, 13, 191, 113, 9, 13, 230, 83, 9, 13, 199, - 219, 9, 13, 192, 75, 9, 13, 222, 108, 9, 13, 220, 37, 9, 13, 216, 210, 9, - 13, 213, 26, 9, 13, 205, 47, 9, 13, 251, 232, 9, 13, 233, 48, 9, 13, 205, - 193, 9, 13, 208, 91, 9, 13, 207, 72, 9, 13, 203, 110, 9, 13, 199, 114, 9, - 13, 199, 29, 9, 13, 221, 212, 9, 13, 199, 41, 9, 13, 238, 28, 9, 13, 191, - 116, 9, 13, 231, 88, 9, 13, 236, 141, 248, 247, 9, 13, 236, 141, 213, 26, - 9, 13, 236, 141, 233, 48, 9, 13, 236, 141, 208, 91, 9, 13, 89, 248, 247, - 9, 13, 89, 223, 8, 9, 13, 89, 229, 212, 9, 13, 89, 230, 83, 9, 13, 89, - 192, 75, 9, 13, 89, 222, 108, 9, 13, 89, 220, 37, 9, 13, 89, 216, 210, 9, - 13, 89, 213, 26, 9, 13, 89, 205, 47, 9, 13, 89, 251, 232, 9, 13, 89, 233, - 48, 9, 13, 89, 205, 193, 9, 13, 89, 208, 91, 9, 13, 89, 203, 110, 9, 13, - 89, 199, 114, 9, 13, 89, 199, 29, 9, 13, 89, 221, 212, 9, 13, 89, 238, - 28, 9, 13, 89, 231, 88, 9, 13, 199, 214, 223, 8, 9, 13, 199, 214, 230, - 83, 9, 13, 199, 214, 192, 75, 9, 13, 199, 214, 220, 37, 9, 13, 199, 214, - 213, 26, 9, 13, 199, 214, 205, 47, 9, 13, 199, 214, 251, 232, 9, 13, 199, - 214, 205, 193, 9, 13, 199, 214, 208, 91, 9, 13, 199, 214, 203, 110, 9, - 13, 199, 214, 221, 212, 9, 13, 199, 214, 238, 28, 9, 13, 199, 214, 231, - 88, 9, 13, 199, 214, 236, 141, 213, 26, 9, 13, 199, 214, 236, 141, 208, - 91, 9, 13, 201, 37, 248, 247, 9, 13, 201, 37, 223, 8, 9, 13, 201, 37, - 229, 212, 9, 13, 201, 37, 230, 83, 9, 13, 201, 37, 199, 219, 9, 13, 201, - 37, 192, 75, 9, 13, 201, 37, 222, 108, 9, 13, 201, 37, 216, 210, 9, 13, - 201, 37, 213, 26, 9, 13, 201, 37, 205, 47, 9, 13, 201, 37, 251, 232, 9, - 13, 201, 37, 233, 48, 9, 13, 201, 37, 205, 193, 9, 13, 201, 37, 208, 91, - 9, 13, 201, 37, 203, 110, 9, 13, 201, 37, 199, 114, 9, 13, 201, 37, 199, - 29, 9, 13, 201, 37, 221, 212, 9, 13, 201, 37, 238, 28, 9, 13, 201, 37, - 191, 116, 9, 13, 201, 37, 231, 88, 9, 13, 201, 37, 236, 141, 248, 247, 9, - 13, 201, 37, 236, 141, 233, 48, 9, 13, 219, 5, 251, 194, 9, 13, 219, 5, - 248, 247, 9, 13, 219, 5, 223, 8, 9, 13, 219, 5, 238, 5, 9, 13, 219, 5, - 229, 212, 9, 13, 219, 5, 193, 133, 9, 13, 219, 5, 191, 113, 9, 13, 219, - 5, 230, 83, 9, 13, 219, 5, 199, 219, 9, 13, 219, 5, 192, 75, 9, 13, 219, - 5, 220, 37, 9, 13, 219, 5, 216, 210, 9, 13, 219, 5, 213, 26, 9, 13, 219, - 5, 205, 47, 9, 13, 219, 5, 251, 232, 9, 13, 219, 5, 233, 48, 9, 13, 219, - 5, 205, 193, 9, 13, 219, 5, 208, 91, 9, 13, 219, 5, 207, 72, 9, 13, 219, - 5, 203, 110, 9, 13, 219, 5, 199, 114, 9, 13, 219, 5, 199, 29, 9, 13, 219, - 5, 221, 212, 9, 13, 219, 5, 199, 41, 9, 13, 219, 5, 238, 28, 9, 13, 219, - 5, 191, 116, 9, 13, 219, 5, 231, 88, 9, 13, 235, 131, 248, 247, 9, 13, - 235, 131, 223, 8, 9, 13, 235, 131, 238, 5, 9, 13, 235, 131, 193, 133, 9, - 13, 235, 131, 191, 113, 9, 13, 235, 131, 230, 83, 9, 13, 235, 131, 199, - 219, 9, 13, 235, 131, 192, 75, 9, 13, 235, 131, 220, 37, 9, 13, 235, 131, - 216, 210, 9, 13, 235, 131, 213, 26, 9, 13, 235, 131, 205, 47, 9, 13, 235, - 131, 251, 232, 9, 13, 235, 131, 233, 48, 9, 13, 235, 131, 205, 193, 9, - 13, 235, 131, 208, 91, 9, 13, 235, 131, 207, 72, 9, 13, 235, 131, 203, - 110, 9, 13, 235, 131, 199, 114, 9, 13, 235, 131, 199, 29, 9, 13, 235, - 131, 221, 212, 9, 13, 235, 131, 199, 41, 9, 13, 235, 131, 238, 28, 9, 13, - 235, 131, 191, 116, 9, 13, 235, 131, 231, 88, 9, 13, 211, 69, 92, 4, 183, - 4, 199, 168, 9, 13, 211, 69, 183, 4, 238, 5, 217, 116, 123, 234, 206, - 193, 66, 217, 116, 123, 202, 3, 193, 66, 217, 116, 123, 193, 105, 193, - 66, 217, 116, 123, 186, 193, 66, 217, 116, 123, 207, 89, 235, 113, 217, - 116, 123, 230, 203, 235, 113, 217, 116, 123, 63, 235, 113, 217, 116, 123, - 91, 79, 243, 142, 217, 116, 123, 105, 79, 243, 142, 217, 116, 123, 115, - 79, 243, 142, 217, 116, 123, 232, 130, 79, 243, 142, 217, 116, 123, 232, - 228, 79, 243, 142, 217, 116, 123, 202, 137, 79, 243, 142, 217, 116, 123, - 203, 248, 79, 243, 142, 217, 116, 123, 234, 166, 79, 243, 142, 217, 116, - 123, 213, 177, 79, 243, 142, 217, 116, 123, 91, 79, 249, 104, 217, 116, - 123, 105, 79, 249, 104, 217, 116, 123, 115, 79, 249, 104, 217, 116, 123, - 232, 130, 79, 249, 104, 217, 116, 123, 232, 228, 79, 249, 104, 217, 116, - 123, 202, 137, 79, 249, 104, 217, 116, 123, 203, 248, 79, 249, 104, 217, - 116, 123, 234, 166, 79, 249, 104, 217, 116, 123, 213, 177, 79, 249, 104, - 217, 116, 123, 91, 79, 243, 9, 217, 116, 123, 105, 79, 243, 9, 217, 116, - 123, 115, 79, 243, 9, 217, 116, 123, 232, 130, 79, 243, 9, 217, 116, 123, - 232, 228, 79, 243, 9, 217, 116, 123, 202, 137, 79, 243, 9, 217, 116, 123, - 203, 248, 79, 243, 9, 217, 116, 123, 234, 166, 79, 243, 9, 217, 116, 123, - 213, 177, 79, 243, 9, 217, 116, 123, 209, 87, 217, 116, 123, 211, 55, - 217, 116, 123, 249, 105, 217, 116, 123, 243, 51, 217, 116, 123, 201, 197, - 217, 116, 123, 200, 200, 217, 116, 123, 250, 151, 217, 116, 123, 193, 56, - 217, 116, 123, 222, 196, 217, 116, 123, 249, 148, 236, 153, 123, 228, - 243, 249, 148, 236, 153, 123, 228, 241, 236, 153, 123, 228, 240, 236, - 153, 123, 228, 239, 236, 153, 123, 228, 238, 236, 153, 123, 228, 237, - 236, 153, 123, 228, 236, 236, 153, 123, 228, 235, 236, 153, 123, 228, - 234, 236, 153, 123, 228, 233, 236, 153, 123, 228, 232, 236, 153, 123, - 228, 231, 236, 153, 123, 228, 230, 236, 153, 123, 228, 229, 236, 153, - 123, 228, 228, 236, 153, 123, 228, 227, 236, 153, 123, 228, 226, 236, - 153, 123, 228, 225, 236, 153, 123, 228, 224, 236, 153, 123, 228, 223, - 236, 153, 123, 228, 222, 236, 153, 123, 228, 221, 236, 153, 123, 228, - 220, 236, 153, 123, 228, 219, 236, 153, 123, 228, 218, 236, 153, 123, - 228, 217, 236, 153, 123, 228, 216, 236, 153, 123, 228, 215, 236, 153, - 123, 228, 214, 236, 153, 123, 228, 213, 236, 153, 123, 228, 212, 236, - 153, 123, 228, 211, 236, 153, 123, 228, 210, 236, 153, 123, 228, 209, - 236, 153, 123, 228, 208, 236, 153, 123, 228, 207, 236, 153, 123, 228, - 206, 236, 153, 123, 228, 205, 236, 153, 123, 228, 204, 236, 153, 123, - 228, 203, 236, 153, 123, 228, 202, 236, 153, 123, 228, 201, 236, 153, - 123, 228, 200, 236, 153, 123, 228, 199, 236, 153, 123, 228, 198, 236, - 153, 123, 228, 197, 236, 153, 123, 228, 196, 236, 153, 123, 228, 195, - 236, 153, 123, 228, 194, 236, 153, 123, 228, 193, 236, 153, 123, 81, 249, - 148, 236, 153, 123, 195, 134, 236, 153, 123, 195, 133, 236, 153, 123, - 195, 132, 236, 153, 123, 195, 131, 236, 153, 123, 195, 130, 236, 153, - 123, 195, 129, 236, 153, 123, 195, 128, 236, 153, 123, 195, 127, 236, - 153, 123, 195, 126, 236, 153, 123, 195, 125, 236, 153, 123, 195, 124, - 236, 153, 123, 195, 123, 236, 153, 123, 195, 122, 236, 153, 123, 195, - 121, 236, 153, 123, 195, 120, 236, 153, 123, 195, 119, 236, 153, 123, - 195, 118, 236, 153, 123, 195, 117, 236, 153, 123, 195, 116, 236, 153, - 123, 195, 115, 236, 153, 123, 195, 114, 236, 153, 123, 195, 113, 236, - 153, 123, 195, 112, 236, 153, 123, 195, 111, 236, 153, 123, 195, 110, - 236, 153, 123, 195, 109, 236, 153, 123, 195, 108, 236, 153, 123, 195, - 107, 236, 153, 123, 195, 106, 236, 153, 123, 195, 105, 236, 153, 123, - 195, 104, 236, 153, 123, 195, 103, 236, 153, 123, 195, 102, 236, 153, - 123, 195, 101, 236, 153, 123, 195, 100, 236, 153, 123, 195, 99, 236, 153, - 123, 195, 98, 236, 153, 123, 195, 97, 236, 153, 123, 195, 96, 236, 153, - 123, 195, 95, 236, 153, 123, 195, 94, 236, 153, 123, 195, 93, 236, 153, - 123, 195, 92, 236, 153, 123, 195, 91, 236, 153, 123, 195, 90, 236, 153, - 123, 195, 89, 236, 153, 123, 195, 88, 236, 153, 123, 195, 87, 236, 153, - 123, 195, 86, 209, 97, 247, 103, 249, 148, 209, 97, 247, 103, 252, 55, - 79, 201, 245, 209, 97, 247, 103, 105, 79, 201, 245, 209, 97, 247, 103, - 115, 79, 201, 245, 209, 97, 247, 103, 232, 130, 79, 201, 245, 209, 97, - 247, 103, 232, 228, 79, 201, 245, 209, 97, 247, 103, 202, 137, 79, 201, - 245, 209, 97, 247, 103, 203, 248, 79, 201, 245, 209, 97, 247, 103, 234, - 166, 79, 201, 245, 209, 97, 247, 103, 213, 177, 79, 201, 245, 209, 97, - 247, 103, 199, 96, 79, 201, 245, 209, 97, 247, 103, 223, 32, 79, 201, - 245, 209, 97, 247, 103, 221, 79, 79, 201, 245, 209, 97, 247, 103, 208, - 17, 79, 201, 245, 209, 97, 247, 103, 221, 141, 79, 201, 245, 209, 97, - 247, 103, 252, 55, 79, 229, 223, 209, 97, 247, 103, 105, 79, 229, 223, - 209, 97, 247, 103, 115, 79, 229, 223, 209, 97, 247, 103, 232, 130, 79, - 229, 223, 209, 97, 247, 103, 232, 228, 79, 229, 223, 209, 97, 247, 103, - 202, 137, 79, 229, 223, 209, 97, 247, 103, 203, 248, 79, 229, 223, 209, - 97, 247, 103, 234, 166, 79, 229, 223, 209, 97, 247, 103, 213, 177, 79, - 229, 223, 209, 97, 247, 103, 199, 96, 79, 229, 223, 209, 97, 247, 103, - 223, 32, 79, 229, 223, 209, 97, 247, 103, 221, 79, 79, 229, 223, 209, 97, - 247, 103, 208, 17, 79, 229, 223, 209, 97, 247, 103, 221, 141, 79, 229, - 223, 209, 97, 247, 103, 207, 89, 222, 196, 209, 97, 247, 103, 252, 55, - 79, 237, 33, 209, 97, 247, 103, 105, 79, 237, 33, 209, 97, 247, 103, 115, - 79, 237, 33, 209, 97, 247, 103, 232, 130, 79, 237, 33, 209, 97, 247, 103, - 232, 228, 79, 237, 33, 209, 97, 247, 103, 202, 137, 79, 237, 33, 209, 97, - 247, 103, 203, 248, 79, 237, 33, 209, 97, 247, 103, 234, 166, 79, 237, - 33, 209, 97, 247, 103, 213, 177, 79, 237, 33, 209, 97, 247, 103, 199, 96, - 79, 237, 33, 209, 97, 247, 103, 223, 32, 79, 237, 33, 209, 97, 247, 103, - 221, 79, 79, 237, 33, 209, 97, 247, 103, 208, 17, 79, 237, 33, 209, 97, - 247, 103, 221, 141, 79, 237, 33, 209, 97, 247, 103, 62, 222, 196, 209, - 97, 247, 103, 252, 55, 79, 242, 206, 209, 97, 247, 103, 105, 79, 242, - 206, 209, 97, 247, 103, 115, 79, 242, 206, 209, 97, 247, 103, 232, 130, - 79, 242, 206, 209, 97, 247, 103, 232, 228, 79, 242, 206, 209, 97, 247, - 103, 202, 137, 79, 242, 206, 209, 97, 247, 103, 203, 248, 79, 242, 206, - 209, 97, 247, 103, 234, 166, 79, 242, 206, 209, 97, 247, 103, 213, 177, - 79, 242, 206, 209, 97, 247, 103, 199, 96, 79, 242, 206, 209, 97, 247, - 103, 223, 32, 79, 242, 206, 209, 97, 247, 103, 221, 79, 79, 242, 206, - 209, 97, 247, 103, 208, 17, 79, 242, 206, 209, 97, 247, 103, 221, 141, - 79, 242, 206, 209, 97, 247, 103, 63, 222, 196, 209, 97, 247, 103, 232, - 162, 209, 97, 247, 103, 197, 200, 209, 97, 247, 103, 197, 189, 209, 97, - 247, 103, 197, 186, 209, 97, 247, 103, 197, 185, 209, 97, 247, 103, 197, - 184, 209, 97, 247, 103, 197, 183, 209, 97, 247, 103, 197, 182, 209, 97, - 247, 103, 197, 181, 209, 97, 247, 103, 197, 180, 209, 97, 247, 103, 197, - 199, 209, 97, 247, 103, 197, 198, 209, 97, 247, 103, 197, 197, 209, 97, - 247, 103, 197, 196, 209, 97, 247, 103, 197, 195, 209, 97, 247, 103, 197, - 194, 209, 97, 247, 103, 197, 193, 209, 97, 247, 103, 197, 192, 209, 97, - 247, 103, 197, 191, 209, 97, 247, 103, 197, 190, 209, 97, 247, 103, 197, - 188, 209, 97, 247, 103, 197, 187, 17, 191, 78, 232, 82, 201, 64, 17, 191, - 78, 242, 76, 17, 91, 242, 76, 17, 105, 242, 76, 17, 115, 242, 76, 17, - 232, 130, 242, 76, 17, 232, 228, 242, 76, 17, 202, 137, 242, 76, 17, 203, - 248, 242, 76, 17, 234, 166, 242, 76, 17, 213, 177, 242, 76, 236, 243, 47, - 49, 17, 191, 77, 236, 243, 214, 108, 47, 49, 17, 191, 77, 47, 191, 78, 4, - 202, 98, 47, 251, 87, 57, 47, 236, 157, 3, 4, 211, 6, 249, 143, 127, 8, - 6, 1, 65, 127, 8, 6, 1, 250, 122, 127, 8, 6, 1, 247, 195, 127, 8, 6, 1, - 238, 129, 127, 8, 6, 1, 71, 127, 8, 6, 1, 233, 177, 127, 8, 6, 1, 232, - 53, 127, 8, 6, 1, 230, 118, 127, 8, 6, 1, 68, 127, 8, 6, 1, 223, 37, 127, - 8, 6, 1, 222, 154, 127, 8, 6, 1, 172, 127, 8, 6, 1, 218, 170, 127, 8, 6, - 1, 215, 63, 127, 8, 6, 1, 74, 127, 8, 6, 1, 210, 238, 127, 8, 6, 1, 208, - 106, 127, 8, 6, 1, 146, 127, 8, 6, 1, 206, 9, 127, 8, 6, 1, 200, 43, 127, - 8, 6, 1, 66, 127, 8, 6, 1, 196, 12, 127, 8, 6, 1, 193, 224, 127, 8, 6, 1, - 192, 235, 127, 8, 6, 1, 192, 159, 127, 8, 6, 1, 191, 166, 198, 42, 203, - 104, 248, 54, 8, 6, 1, 206, 9, 47, 43, 8, 6, 1, 247, 195, 47, 43, 8, 6, - 1, 146, 47, 247, 45, 47, 192, 237, 239, 9, 113, 112, 8, 6, 1, 65, 112, 8, - 6, 1, 250, 122, 112, 8, 6, 1, 247, 195, 112, 8, 6, 1, 238, 129, 112, 8, - 6, 1, 71, 112, 8, 6, 1, 233, 177, 112, 8, 6, 1, 232, 53, 112, 8, 6, 1, - 230, 118, 112, 8, 6, 1, 68, 112, 8, 6, 1, 223, 37, 112, 8, 6, 1, 222, - 154, 112, 8, 6, 1, 172, 112, 8, 6, 1, 218, 170, 112, 8, 6, 1, 215, 63, - 112, 8, 6, 1, 74, 112, 8, 6, 1, 210, 238, 112, 8, 6, 1, 208, 106, 112, 8, - 6, 1, 146, 112, 8, 6, 1, 206, 9, 112, 8, 6, 1, 200, 43, 112, 8, 6, 1, 66, - 112, 8, 6, 1, 196, 12, 112, 8, 6, 1, 193, 224, 112, 8, 6, 1, 192, 235, - 112, 8, 6, 1, 192, 159, 112, 8, 6, 1, 191, 166, 112, 228, 128, 112, 215, - 89, 112, 205, 71, 112, 201, 179, 112, 208, 250, 112, 193, 126, 214, 108, - 47, 8, 6, 1, 65, 214, 108, 47, 8, 6, 1, 250, 122, 214, 108, 47, 8, 6, 1, - 247, 195, 214, 108, 47, 8, 6, 1, 238, 129, 214, 108, 47, 8, 6, 1, 71, - 214, 108, 47, 8, 6, 1, 233, 177, 214, 108, 47, 8, 6, 1, 232, 53, 214, - 108, 47, 8, 6, 1, 230, 118, 214, 108, 47, 8, 6, 1, 68, 214, 108, 47, 8, - 6, 1, 223, 37, 214, 108, 47, 8, 6, 1, 222, 154, 214, 108, 47, 8, 6, 1, - 172, 214, 108, 47, 8, 6, 1, 218, 170, 214, 108, 47, 8, 6, 1, 215, 63, - 214, 108, 47, 8, 6, 1, 74, 214, 108, 47, 8, 6, 1, 210, 238, 214, 108, 47, - 8, 6, 1, 208, 106, 214, 108, 47, 8, 6, 1, 146, 214, 108, 47, 8, 6, 1, - 206, 9, 214, 108, 47, 8, 6, 1, 200, 43, 214, 108, 47, 8, 6, 1, 66, 214, - 108, 47, 8, 6, 1, 196, 12, 214, 108, 47, 8, 6, 1, 193, 224, 214, 108, 47, - 8, 6, 1, 192, 235, 214, 108, 47, 8, 6, 1, 192, 159, 214, 108, 47, 8, 6, - 1, 191, 166, 207, 149, 216, 241, 56, 207, 149, 216, 237, 56, 207, 149, - 215, 166, 56, 47, 247, 68, 47, 247, 196, 4, 211, 6, 249, 143, 47, 228, - 147, 233, 14, 214, 108, 112, 8, 6, 1, 65, 214, 108, 112, 8, 6, 1, 250, - 122, 214, 108, 112, 8, 6, 1, 247, 195, 214, 108, 112, 8, 6, 1, 238, 129, - 214, 108, 112, 8, 6, 1, 71, 214, 108, 112, 8, 6, 1, 233, 177, 214, 108, - 112, 8, 6, 1, 232, 53, 214, 108, 112, 8, 6, 1, 230, 118, 214, 108, 112, - 8, 6, 1, 68, 214, 108, 112, 8, 6, 1, 223, 37, 214, 108, 112, 8, 6, 1, - 222, 154, 214, 108, 112, 8, 6, 1, 172, 214, 108, 112, 8, 6, 1, 218, 170, - 214, 108, 112, 8, 6, 1, 215, 63, 214, 108, 112, 8, 6, 1, 74, 214, 108, - 112, 8, 6, 1, 210, 238, 214, 108, 112, 8, 6, 1, 208, 106, 214, 108, 112, - 8, 6, 1, 146, 214, 108, 112, 8, 6, 1, 206, 9, 214, 108, 112, 8, 6, 1, - 200, 43, 214, 108, 112, 8, 6, 1, 66, 214, 108, 112, 8, 6, 1, 196, 12, - 214, 108, 112, 8, 6, 1, 193, 224, 214, 108, 112, 8, 6, 1, 192, 235, 214, - 108, 112, 8, 6, 1, 192, 159, 214, 108, 112, 8, 6, 1, 191, 166, 238, 216, - 214, 108, 112, 8, 6, 1, 210, 238, 214, 108, 112, 228, 30, 214, 108, 112, - 168, 214, 108, 112, 188, 214, 108, 112, 252, 157, 214, 108, 112, 193, - 126, 51, 236, 196, 112, 242, 249, 112, 239, 16, 112, 232, 110, 112, 228, - 21, 112, 214, 81, 112, 214, 72, 112, 211, 127, 112, 202, 10, 112, 133, 4, - 233, 218, 77, 112, 194, 252, 112, 115, 238, 129, 112, 205, 58, 205, 77, - 112, 105, 222, 154, 112, 232, 130, 222, 154, 112, 234, 166, 222, 154, - 112, 232, 228, 209, 64, 107, 112, 203, 248, 209, 64, 107, 112, 197, 21, - 209, 64, 109, 112, 202, 122, 210, 238, 112, 91, 228, 143, 197, 33, 210, - 238, 112, 8, 2, 1, 238, 129, 112, 229, 250, 112, 229, 249, 112, 229, 152, - 112, 218, 254, 112, 202, 242, 112, 196, 140, 112, 195, 21, 217, 38, 193, - 21, 113, 207, 80, 223, 147, 16, 1, 65, 207, 80, 223, 147, 16, 1, 250, - 122, 207, 80, 223, 147, 16, 1, 247, 195, 207, 80, 223, 147, 16, 1, 238, - 129, 207, 80, 223, 147, 16, 1, 71, 207, 80, 223, 147, 16, 1, 233, 177, - 207, 80, 223, 147, 16, 1, 232, 53, 207, 80, 223, 147, 16, 1, 230, 118, - 207, 80, 223, 147, 16, 1, 68, 207, 80, 223, 147, 16, 1, 223, 37, 207, 80, - 223, 147, 16, 1, 222, 154, 207, 80, 223, 147, 16, 1, 172, 207, 80, 223, - 147, 16, 1, 218, 170, 207, 80, 223, 147, 16, 1, 215, 63, 207, 80, 223, - 147, 16, 1, 74, 207, 80, 223, 147, 16, 1, 210, 238, 207, 80, 223, 147, - 16, 1, 208, 106, 207, 80, 223, 147, 16, 1, 146, 207, 80, 223, 147, 16, 1, - 206, 9, 207, 80, 223, 147, 16, 1, 200, 43, 207, 80, 223, 147, 16, 1, 66, - 207, 80, 223, 147, 16, 1, 196, 12, 207, 80, 223, 147, 16, 1, 193, 224, - 207, 80, 223, 147, 16, 1, 192, 235, 207, 80, 223, 147, 16, 1, 192, 159, - 207, 80, 223, 147, 16, 1, 191, 166, 51, 229, 122, 229, 11, 112, 72, 221, - 51, 112, 72, 188, 112, 12, 196, 95, 225, 219, 112, 12, 196, 95, 225, 223, - 112, 12, 196, 95, 225, 231, 112, 72, 237, 148, 112, 12, 196, 95, 225, - 238, 112, 12, 196, 95, 225, 225, 112, 12, 196, 95, 225, 197, 112, 12, - 196, 95, 225, 224, 112, 12, 196, 95, 225, 237, 112, 12, 196, 95, 225, - 211, 112, 12, 196, 95, 225, 204, 112, 12, 196, 95, 225, 213, 112, 12, - 196, 95, 225, 234, 112, 12, 196, 95, 225, 220, 112, 12, 196, 95, 225, - 236, 112, 12, 196, 95, 225, 212, 112, 12, 196, 95, 225, 235, 112, 12, - 196, 95, 225, 198, 112, 12, 196, 95, 225, 203, 112, 12, 196, 95, 225, - 196, 112, 12, 196, 95, 225, 226, 112, 12, 196, 95, 225, 228, 112, 12, - 196, 95, 225, 206, 112, 12, 196, 95, 225, 217, 112, 12, 196, 95, 225, - 215, 112, 12, 196, 95, 225, 241, 112, 12, 196, 95, 225, 240, 112, 12, - 196, 95, 225, 194, 112, 12, 196, 95, 225, 221, 112, 12, 196, 95, 225, - 239, 112, 12, 196, 95, 225, 230, 112, 12, 196, 95, 225, 216, 112, 12, - 196, 95, 225, 195, 112, 12, 196, 95, 225, 218, 112, 12, 196, 95, 225, - 200, 112, 12, 196, 95, 225, 199, 112, 12, 196, 95, 225, 229, 112, 12, - 196, 95, 225, 207, 112, 12, 196, 95, 225, 209, 112, 12, 196, 95, 225, - 210, 112, 12, 196, 95, 225, 202, 112, 12, 196, 95, 225, 233, 112, 12, - 196, 95, 225, 227, 112, 12, 196, 95, 225, 193, 198, 42, 203, 104, 248, - 54, 12, 196, 95, 225, 208, 198, 42, 203, 104, 248, 54, 12, 196, 95, 225, - 240, 198, 42, 203, 104, 248, 54, 12, 196, 95, 225, 238, 198, 42, 203, - 104, 248, 54, 12, 196, 95, 225, 222, 198, 42, 203, 104, 248, 54, 12, 196, - 95, 225, 205, 198, 42, 203, 104, 248, 54, 12, 196, 95, 225, 218, 198, 42, - 203, 104, 248, 54, 12, 196, 95, 225, 201, 198, 42, 203, 104, 248, 54, 12, - 196, 95, 225, 232, 198, 42, 203, 104, 248, 54, 12, 196, 95, 225, 214, 47, - 228, 16, 252, 29, 47, 228, 16, 252, 60, 206, 114, 16, 40, 232, 88, 206, - 114, 16, 40, 218, 227, 206, 114, 16, 40, 203, 24, 206, 114, 16, 40, 192, - 207, 206, 114, 16, 40, 203, 3, 206, 114, 16, 40, 247, 150, 238, 141, 232, - 175, 242, 221, 196, 117, 213, 193, 4, 201, 100, 200, 193, 139, 215, 185, - 200, 192, 242, 253, 250, 185, 235, 63, 200, 191, 139, 247, 255, 207, 150, - 248, 31, 250, 185, 213, 192, 193, 144, 193, 138, 195, 14, 216, 54, 193, - 128, 234, 210, 231, 14, 233, 234, 234, 210, 231, 14, 251, 142, 234, 210, - 231, 14, 250, 204, 231, 14, 4, 216, 179, 214, 82, 215, 208, 113, 193, - 130, 238, 230, 215, 208, 113, 232, 240, 208, 25, 215, 208, 113, 193, 130, - 231, 51, 215, 208, 113, 232, 82, 215, 208, 113, 193, 159, 231, 51, 215, - 208, 113, 220, 8, 208, 25, 215, 208, 113, 193, 159, 238, 230, 215, 208, - 113, 238, 230, 215, 207, 214, 82, 215, 208, 4, 233, 105, 232, 240, 208, - 25, 215, 208, 4, 233, 105, 220, 8, 208, 25, 215, 208, 4, 233, 105, 232, - 82, 215, 208, 4, 233, 105, 200, 199, 4, 233, 105, 231, 10, 201, 103, 203, - 46, 201, 103, 199, 21, 62, 235, 99, 63, 200, 198, 63, 200, 199, 4, 2, - 242, 212, 63, 200, 199, 248, 244, 242, 212, 63, 200, 199, 248, 244, 242, - 213, 4, 207, 151, 242, 213, 4, 207, 151, 242, 213, 4, 202, 53, 242, 213, - 4, 219, 131, 242, 213, 4, 198, 46, 232, 176, 193, 67, 248, 118, 233, 105, - 228, 183, 236, 164, 199, 227, 247, 230, 243, 105, 205, 49, 233, 228, 197, - 254, 237, 141, 197, 254, 210, 185, 197, 254, 247, 155, 228, 183, 210, 17, - 197, 78, 243, 109, 248, 121, 206, 127, 229, 151, 200, 196, 248, 121, 234, - 214, 79, 217, 105, 234, 214, 79, 206, 246, 229, 195, 232, 130, 219, 236, - 242, 211, 217, 71, 219, 235, 233, 86, 219, 235, 219, 236, 232, 183, 223, - 165, 193, 66, 215, 100, 198, 83, 250, 164, 230, 220, 216, 198, 193, 142, - 199, 187, 219, 203, 249, 100, 209, 135, 207, 89, 251, 48, 230, 203, 251, - 48, 210, 57, 210, 61, 243, 110, 201, 43, 230, 65, 202, 90, 79, 209, 114, - 216, 227, 211, 107, 248, 100, 209, 11, 219, 214, 206, 247, 238, 236, 206, - 247, 249, 113, 239, 19, 206, 246, 238, 169, 23, 206, 246, 201, 84, 248, - 68, 201, 244, 248, 45, 232, 108, 232, 104, 206, 153, 200, 142, 209, 14, - 237, 237, 211, 155, 200, 165, 232, 105, 203, 14, 232, 239, 247, 149, 4, - 200, 134, 237, 82, 202, 33, 228, 29, 238, 234, 203, 122, 228, 28, 228, - 29, 238, 234, 235, 128, 239, 18, 243, 68, 164, 247, 120, 219, 26, 238, - 160, 229, 0, 209, 16, 203, 30, 248, 224, 248, 64, 209, 17, 79, 232, 163, - 239, 17, 232, 152, 23, 221, 80, 199, 133, 193, 51, 230, 33, 205, 177, - 248, 81, 23, 238, 183, 193, 63, 231, 18, 242, 96, 231, 18, 197, 204, 235, - 106, 248, 255, 215, 142, 242, 228, 248, 255, 215, 141, 249, 151, 248, 80, - 232, 152, 23, 221, 81, 4, 209, 99, 248, 81, 4, 209, 32, 239, 5, 209, 34, - 206, 248, 193, 11, 208, 224, 248, 159, 247, 148, 223, 31, 243, 58, 197, - 254, 233, 69, 243, 57, 232, 242, 232, 243, 201, 242, 249, 111, 210, 104, - 209, 33, 239, 56, 249, 113, 199, 191, 197, 254, 238, 216, 232, 213, 209, - 136, 237, 138, 223, 21, 236, 156, 247, 92, 201, 42, 193, 67, 243, 84, - 215, 208, 195, 54, 247, 10, 205, 91, 205, 121, 230, 227, 247, 113, 229, - 226, 4, 198, 136, 211, 107, 199, 34, 219, 226, 248, 74, 79, 232, 187, - 216, 56, 216, 221, 207, 59, 206, 248, 37, 221, 223, 4, 223, 30, 201, 12, - 216, 91, 219, 170, 202, 87, 239, 24, 221, 74, 249, 15, 250, 215, 37, 213, - 3, 249, 15, 237, 88, 37, 213, 3, 233, 2, 232, 114, 252, 33, 198, 180, - 247, 93, 228, 185, 233, 35, 193, 93, 206, 140, 242, 99, 232, 234, 209, - 55, 23, 232, 238, 216, 91, 215, 171, 247, 134, 243, 16, 229, 233, 250, - 226, 210, 190, 198, 54, 230, 11, 243, 2, 199, 87, 198, 181, 242, 244, - 248, 109, 210, 8, 250, 224, 195, 65, 231, 216, 236, 236, 229, 119, 202, - 80, 217, 150, 248, 172, 231, 217, 237, 26, 248, 67, 232, 189, 209, 97, - 247, 101, 37, 213, 8, 215, 131, 37, 213, 3, 205, 105, 230, 164, 37, 221, - 222, 197, 179, 195, 42, 37, 205, 83, 206, 43, 203, 61, 4, 205, 124, 199, - 92, 207, 172, 23, 249, 113, 202, 110, 23, 202, 110, 248, 93, 249, 70, 23, - 228, 249, 243, 111, 232, 219, 202, 52, 206, 44, 200, 170, 201, 203, 216, - 221, 197, 205, 228, 186, 207, 173, 251, 143, 232, 160, 206, 57, 232, 160, - 200, 137, 193, 110, 219, 136, 230, 251, 207, 174, 215, 193, 207, 174, - 247, 104, 238, 227, 249, 67, 23, 249, 113, 195, 13, 233, 24, 229, 14, - 201, 76, 23, 249, 113, 228, 29, 229, 14, 201, 76, 23, 208, 159, 199, 234, - 199, 92, 210, 209, 23, 249, 113, 202, 54, 247, 109, 215, 186, 247, 132, - 249, 18, 4, 196, 117, 248, 1, 239, 38, 228, 175, 247, 255, 242, 252, 237, - 92, 228, 175, 248, 0, 242, 242, 248, 0, 237, 84, 237, 85, 223, 62, 214, - 210, 210, 111, 201, 114, 228, 175, 248, 0, 228, 175, 4, 231, 200, 211, - 146, 248, 0, 223, 21, 209, 22, 211, 145, 233, 233, 209, 22, 211, 145, - 228, 184, 249, 94, 250, 153, 199, 102, 217, 150, 228, 180, 218, 244, 228, - 180, 239, 22, 201, 58, 205, 90, 237, 96, 201, 58, 233, 94, 223, 42, 220, - 20, 223, 21, 247, 82, 233, 233, 247, 82, 63, 210, 30, 62, 210, 30, 193, - 136, 63, 232, 219, 193, 136, 62, 232, 219, 206, 126, 62, 206, 126, 220, - 119, 249, 134, 207, 172, 23, 202, 245, 248, 72, 23, 57, 251, 138, 234, - 111, 52, 232, 229, 196, 253, 234, 111, 52, 232, 229, 196, 250, 234, 111, - 52, 232, 229, 196, 248, 234, 111, 52, 232, 229, 196, 246, 234, 111, 52, - 232, 229, 196, 244, 207, 132, 215, 183, 210, 249, 193, 144, 248, 5, 238, - 241, 198, 173, 219, 187, 207, 176, 247, 80, 235, 113, 238, 225, 193, 96, - 202, 61, 202, 59, 228, 185, 207, 144, 231, 1, 203, 108, 215, 227, 206, - 130, 243, 95, 236, 164, 209, 149, 248, 111, 234, 133, 211, 158, 201, 219, - 203, 103, 248, 4, 251, 91, 228, 255, 220, 110, 248, 253, 232, 238, 197, - 204, 232, 238, 248, 119, 197, 55, 230, 9, 243, 96, 249, 151, 243, 96, - 232, 98, 249, 151, 243, 96, 248, 162, 210, 32, 221, 63, 209, 38, 235, - 103, 247, 136, 249, 139, 247, 136, 236, 155, 215, 184, 233, 105, 238, - 242, 233, 105, 198, 174, 233, 105, 207, 177, 233, 105, 247, 81, 233, 105, - 235, 114, 233, 105, 201, 201, 193, 96, 228, 186, 233, 105, 215, 228, 233, - 105, 236, 165, 233, 105, 209, 150, 233, 105, 232, 102, 233, 105, 230, 61, - 233, 105, 193, 38, 233, 105, 249, 12, 233, 105, 210, 164, 233, 105, 209, - 150, 213, 15, 210, 78, 208, 210, 243, 79, 233, 187, 233, 195, 234, 213, - 213, 15, 215, 181, 198, 61, 63, 133, 209, 60, 249, 146, 223, 150, 63, - 144, 209, 60, 249, 146, 223, 150, 63, 45, 209, 60, 249, 146, 223, 150, - 63, 50, 209, 60, 249, 146, 223, 150, 232, 232, 230, 56, 56, 193, 136, - 230, 56, 56, 211, 128, 230, 56, 56, 198, 211, 133, 56, 198, 211, 144, 56, - 242, 243, 230, 31, 56, 211, 79, 230, 31, 56, 238, 210, 193, 34, 230, 11, - 233, 190, 214, 113, 200, 41, 223, 11, 235, 108, 221, 144, 248, 175, 193, - 34, 242, 214, 208, 139, 230, 35, 209, 12, 217, 80, 203, 53, 250, 180, - 203, 53, 229, 136, 203, 53, 193, 34, 205, 140, 193, 34, 248, 92, 232, - 158, 247, 222, 223, 165, 202, 188, 247, 221, 223, 165, 202, 188, 248, 62, - 231, 30, 217, 92, 193, 35, 233, 83, 217, 93, 23, 193, 36, 229, 8, 230, - 30, 105, 216, 189, 229, 8, 230, 30, 105, 193, 33, 229, 8, 230, 30, 209, - 52, 211, 144, 193, 36, 4, 247, 241, 234, 211, 248, 32, 4, 195, 144, 209, - 253, 4, 248, 123, 230, 80, 217, 93, 4, 230, 178, 209, 188, 217, 75, 217, - 93, 4, 197, 63, 211, 120, 217, 92, 211, 120, 193, 35, 249, 150, 239, 39, - 193, 19, 208, 215, 223, 21, 211, 139, 223, 21, 231, 0, 231, 63, 249, 151, - 251, 122, 233, 200, 251, 184, 251, 185, 215, 217, 223, 170, 202, 104, - 223, 139, 237, 81, 209, 252, 230, 172, 237, 242, 219, 97, 214, 237, 209, - 50, 233, 106, 217, 35, 230, 79, 249, 88, 209, 54, 200, 62, 209, 142, 221, - 125, 77, 218, 244, 219, 177, 206, 189, 231, 157, 201, 66, 221, 124, 248, - 73, 238, 245, 4, 229, 225, 193, 117, 249, 8, 229, 225, 248, 24, 229, 225, - 105, 229, 223, 201, 240, 229, 225, 230, 188, 229, 225, 229, 226, 4, 57, - 248, 117, 229, 225, 230, 203, 229, 225, 192, 73, 229, 225, 208, 140, 229, - 225, 229, 226, 4, 206, 248, 207, 13, 229, 223, 229, 226, 237, 138, 237, - 35, 203, 136, 4, 42, 75, 223, 119, 234, 137, 156, 247, 253, 251, 121, - 113, 248, 101, 202, 93, 113, 242, 87, 113, 201, 213, 200, 144, 113, 235, - 99, 237, 218, 113, 209, 143, 79, 209, 39, 232, 201, 248, 187, 236, 197, - 113, 201, 231, 249, 111, 198, 231, 249, 111, 63, 232, 188, 228, 143, 209, - 58, 113, 215, 232, 249, 132, 238, 172, 233, 220, 88, 236, 157, 56, 238, - 232, 247, 102, 249, 93, 4, 192, 71, 56, 249, 93, 4, 236, 157, 56, 249, - 93, 4, 233, 236, 56, 249, 93, 4, 209, 10, 56, 215, 232, 4, 193, 60, 243, - 139, 4, 196, 66, 197, 250, 23, 192, 71, 56, 205, 61, 209, 251, 239, 61, - 248, 30, 216, 43, 232, 193, 236, 222, 211, 62, 236, 228, 235, 57, 233, 9, - 232, 173, 211, 79, 233, 9, 232, 173, 210, 207, 4, 238, 177, 210, 207, - 233, 98, 196, 77, 247, 142, 199, 130, 247, 142, 247, 103, 223, 150, 243, - 139, 4, 196, 66, 197, 249, 243, 139, 4, 235, 121, 197, 249, 249, 90, 243, - 138, 242, 227, 208, 135, 206, 116, 208, 135, 210, 136, 201, 54, 206, 51, - 197, 238, 206, 51, 248, 97, 199, 232, 219, 231, 213, 6, 213, 7, 4, 237, - 137, 238, 244, 242, 221, 248, 98, 211, 79, 248, 98, 230, 203, 248, 98, - 248, 117, 248, 98, 211, 57, 248, 98, 248, 95, 214, 230, 249, 136, 205, - 74, 216, 190, 199, 107, 207, 103, 210, 205, 233, 66, 217, 150, 205, 120, - 251, 88, 208, 160, 252, 41, 218, 246, 243, 121, 216, 203, 211, 15, 198, - 2, 223, 161, 198, 2, 210, 214, 235, 10, 113, 223, 158, 234, 69, 234, 70, - 4, 235, 121, 64, 58, 242, 221, 217, 111, 4, 218, 237, 232, 219, 242, 221, - 217, 111, 4, 207, 149, 232, 219, 211, 79, 217, 111, 4, 207, 149, 232, - 219, 211, 79, 217, 111, 4, 218, 237, 232, 219, 209, 19, 209, 20, 228, - 189, 214, 77, 216, 6, 209, 196, 216, 6, 209, 197, 4, 96, 64, 250, 185, - 219, 226, 195, 68, 216, 5, 216, 6, 209, 197, 211, 147, 213, 46, 216, 6, - 209, 195, 251, 89, 4, 249, 78, 247, 134, 247, 135, 4, 232, 210, 195, 65, - 247, 134, 199, 104, 207, 167, 195, 64, 233, 2, 208, 195, 209, 29, 201, - 78, 208, 238, 249, 17, 197, 17, 96, 250, 233, 242, 223, 96, 23, 118, 211, - 79, 243, 13, 250, 233, 242, 223, 96, 23, 118, 211, 79, 243, 13, 250, 234, - 4, 47, 91, 211, 1, 242, 223, 235, 121, 23, 196, 66, 211, 79, 243, 13, - 250, 233, 251, 87, 235, 121, 23, 196, 66, 211, 79, 243, 13, 250, 233, - 130, 248, 28, 113, 137, 248, 28, 113, 201, 236, 4, 247, 127, 106, 201, - 235, 201, 236, 4, 91, 202, 6, 193, 138, 201, 236, 4, 115, 202, 6, 193, - 137, 249, 60, 234, 137, 209, 89, 219, 221, 217, 123, 231, 18, 206, 204, - 217, 123, 231, 18, 219, 37, 4, 223, 131, 210, 36, 242, 221, 219, 37, 4, - 221, 224, 221, 224, 219, 36, 211, 79, 219, 36, 248, 237, 248, 238, 4, - 247, 127, 106, 248, 96, 219, 105, 113, 207, 168, 247, 215, 249, 149, 4, - 118, 64, 58, 234, 97, 4, 118, 64, 58, 211, 107, 4, 233, 218, 87, 4, 45, - 50, 64, 58, 202, 16, 4, 96, 64, 58, 198, 54, 4, 196, 66, 64, 58, 213, 46, - 91, 196, 105, 234, 164, 113, 221, 221, 199, 95, 223, 125, 16, 40, 8, 6, - 219, 176, 223, 125, 16, 40, 8, 2, 219, 176, 223, 125, 16, 40, 212, 137, - 223, 125, 16, 40, 200, 76, 223, 125, 16, 40, 8, 219, 176, 232, 245, 234, - 137, 198, 49, 193, 9, 230, 63, 212, 120, 23, 248, 103, 229, 15, 209, 120, - 216, 90, 199, 105, 238, 199, 249, 113, 202, 137, 209, 62, 201, 104, 4, - 82, 236, 142, 223, 21, 16, 40, 248, 250, 197, 236, 234, 113, 62, 51, 247, - 215, 63, 51, 247, 215, 220, 15, 207, 89, 243, 12, 220, 15, 248, 117, 243, - 12, 220, 15, 211, 57, 237, 34, 220, 15, 248, 117, 237, 34, 2, 211, 57, - 237, 34, 2, 248, 117, 237, 34, 196, 76, 207, 89, 197, 241, 235, 124, 207, - 89, 197, 241, 196, 76, 2, 207, 89, 197, 241, 235, 124, 2, 207, 89, 197, - 241, 110, 50, 203, 152, 63, 243, 12, 116, 50, 203, 152, 63, 243, 12, 47, - 238, 220, 209, 43, 238, 220, 209, 44, 4, 230, 69, 60, 238, 220, 209, 43, - 213, 10, 45, 204, 29, 4, 115, 236, 140, 213, 10, 50, 204, 29, 4, 115, - 236, 140, 16, 40, 217, 52, 246, 244, 63, 8, 238, 219, 88, 8, 238, 219, - 247, 28, 238, 219, 211, 116, 113, 235, 127, 79, 210, 62, 222, 127, 215, - 199, 200, 70, 216, 185, 4, 213, 177, 248, 48, 248, 69, 79, 228, 92, 242, - 225, 233, 106, 91, 211, 164, 242, 225, 233, 106, 105, 211, 164, 242, 225, - 233, 106, 115, 211, 164, 242, 225, 233, 106, 232, 130, 211, 164, 242, - 225, 233, 106, 232, 228, 211, 164, 242, 225, 233, 106, 202, 137, 211, - 164, 242, 225, 233, 106, 203, 248, 211, 164, 242, 225, 233, 106, 234, - 166, 211, 164, 242, 225, 233, 106, 213, 177, 211, 164, 242, 225, 233, - 106, 199, 96, 211, 164, 242, 225, 233, 106, 234, 130, 211, 164, 242, 225, - 233, 106, 197, 38, 211, 164, 242, 225, 233, 106, 211, 99, 242, 225, 233, - 106, 197, 11, 242, 225, 233, 106, 198, 217, 242, 225, 233, 106, 232, 126, - 242, 225, 233, 106, 232, 225, 242, 225, 233, 106, 202, 133, 242, 225, - 233, 106, 203, 247, 242, 225, 233, 106, 234, 165, 242, 225, 233, 106, - 213, 175, 242, 225, 233, 106, 199, 94, 242, 225, 233, 106, 234, 128, 242, - 225, 233, 106, 197, 36, 50, 201, 235, 50, 201, 236, 4, 91, 202, 6, 193, - 138, 50, 201, 236, 4, 115, 202, 6, 193, 137, 247, 248, 247, 249, 4, 202, - 6, 193, 137, 206, 187, 248, 237, 248, 98, 247, 125, 217, 77, 242, 224, - 62, 202, 105, 23, 238, 217, 213, 46, 209, 126, 229, 7, 217, 93, 223, 165, - 247, 224, 200, 212, 219, 167, 202, 91, 211, 59, 201, 192, 237, 223, 200, - 194, 201, 222, 201, 223, 193, 118, 222, 185, 217, 93, 237, 241, 45, 230, - 56, 199, 107, 207, 103, 199, 107, 207, 104, 4, 210, 206, 50, 230, 56, - 199, 107, 207, 103, 63, 198, 34, 199, 106, 62, 198, 34, 199, 106, 199, - 107, 211, 107, 198, 54, 79, 216, 2, 242, 247, 216, 6, 209, 196, 249, 149, - 79, 234, 69, 201, 110, 234, 69, 234, 70, 4, 219, 131, 232, 180, 234, 69, - 210, 37, 139, 201, 110, 234, 69, 219, 104, 210, 135, 62, 208, 135, 110, - 45, 210, 35, 110, 45, 249, 107, 210, 36, 110, 45, 232, 132, 210, 36, 110, - 45, 210, 199, 110, 45, 238, 235, 45, 193, 3, 230, 55, 154, 211, 128, 230, - 56, 56, 207, 149, 230, 56, 4, 232, 250, 201, 212, 207, 19, 207, 149, 230, - 56, 4, 232, 250, 201, 212, 207, 19, 198, 211, 133, 56, 207, 19, 198, 211, - 144, 56, 207, 19, 195, 67, 230, 55, 207, 19, 230, 56, 4, 82, 232, 255, - 233, 206, 207, 149, 230, 56, 4, 210, 109, 248, 212, 82, 23, 206, 190, - 232, 249, 63, 144, 209, 60, 45, 230, 56, 223, 150, 202, 207, 63, 45, 209, - 60, 223, 150, 202, 207, 63, 50, 209, 60, 223, 150, 202, 207, 62, 45, 209, - 60, 223, 150, 202, 207, 62, 50, 209, 60, 223, 150, 62, 45, 209, 60, 249, - 146, 223, 150, 62, 50, 209, 60, 249, 146, 223, 150, 202, 207, 63, 133, - 209, 60, 223, 150, 202, 207, 63, 144, 209, 60, 223, 150, 202, 207, 62, - 133, 209, 60, 223, 150, 202, 207, 62, 144, 209, 60, 223, 150, 62, 133, - 209, 60, 249, 146, 223, 150, 62, 144, 209, 60, 249, 146, 223, 150, 62, - 229, 225, 237, 80, 239, 61, 221, 223, 23, 215, 183, 115, 214, 86, 239, - 60, 208, 211, 209, 73, 247, 144, 62, 230, 19, 203, 104, 232, 193, 236, - 222, 63, 230, 19, 203, 104, 232, 193, 236, 222, 202, 33, 203, 104, 232, - 193, 236, 222, 199, 182, 247, 86, 193, 55, 221, 222, 91, 247, 216, 215, - 183, 105, 247, 216, 215, 183, 115, 247, 216, 215, 183, 198, 24, 39, 209, - 251, 239, 61, 230, 19, 236, 222, 205, 77, 208, 212, 228, 22, 233, 66, - 228, 22, 211, 62, 236, 229, 228, 22, 236, 170, 4, 199, 53, 236, 170, 4, - 199, 54, 23, 209, 179, 236, 170, 4, 209, 179, 232, 116, 4, 209, 179, 232, - 116, 4, 198, 150, 232, 116, 4, 251, 135, 192, 235, 62, 232, 173, 232, - 173, 211, 79, 232, 173, 247, 103, 141, 236, 206, 247, 103, 233, 9, 248, - 64, 233, 9, 247, 157, 234, 107, 213, 8, 234, 107, 213, 9, 210, 206, 234, - 107, 213, 9, 210, 212, 213, 8, 213, 9, 210, 206, 213, 9, 210, 212, 234, - 107, 236, 169, 234, 107, 210, 206, 234, 107, 210, 204, 236, 169, 210, - 206, 210, 204, 193, 148, 201, 219, 213, 9, 210, 212, 201, 219, 247, 143, - 210, 212, 237, 80, 193, 65, 216, 40, 217, 24, 211, 4, 242, 223, 50, 23, - 45, 204, 29, 250, 233, 247, 127, 192, 235, 223, 156, 232, 165, 202, 117, - 113, 237, 136, 232, 165, 202, 117, 113, 239, 62, 39, 221, 224, 206, 141, - 214, 77, 210, 207, 4, 47, 199, 53, 201, 68, 243, 138, 238, 17, 221, 80, - 219, 98, 201, 234, 229, 238, 223, 165, 202, 188, 115, 207, 122, 58, 115, - 207, 122, 60, 115, 207, 122, 219, 226, 115, 207, 122, 179, 45, 201, 231, - 248, 10, 50, 201, 231, 248, 10, 105, 201, 231, 248, 9, 115, 201, 231, - 248, 9, 45, 198, 231, 248, 10, 50, 198, 231, 248, 10, 45, 251, 121, 248, - 10, 50, 251, 121, 248, 10, 215, 212, 248, 10, 219, 132, 215, 212, 248, - 10, 219, 132, 215, 211, 249, 109, 111, 4, 249, 108, 249, 109, 27, 192, - 235, 249, 109, 111, 4, 27, 192, 235, 249, 109, 28, 27, 192, 235, 249, - 109, 111, 4, 28, 27, 192, 235, 156, 243, 128, 77, 249, 109, 111, 4, 28, - 243, 127, 193, 18, 217, 73, 215, 188, 232, 83, 198, 85, 198, 30, 201, 93, - 79, 219, 146, 202, 189, 79, 223, 22, 215, 169, 230, 198, 233, 105, 230, - 198, 233, 106, 4, 202, 65, 233, 187, 233, 106, 4, 199, 126, 79, 222, 187, - 202, 65, 233, 106, 4, 211, 79, 215, 181, 202, 65, 233, 106, 4, 211, 79, - 215, 182, 23, 202, 65, 233, 187, 202, 65, 233, 106, 4, 211, 79, 215, 182, - 23, 242, 89, 200, 143, 202, 65, 233, 106, 4, 211, 79, 215, 182, 23, 198, - 171, 233, 187, 202, 65, 233, 106, 4, 230, 68, 202, 65, 233, 106, 4, 228, - 188, 193, 57, 233, 105, 202, 65, 233, 106, 4, 202, 65, 233, 187, 233, - 106, 205, 110, 237, 116, 232, 163, 207, 62, 233, 105, 202, 65, 233, 106, - 4, 229, 224, 233, 187, 202, 65, 233, 106, 4, 200, 194, 202, 64, 233, 105, - 214, 84, 233, 105, 233, 208, 233, 105, 196, 111, 233, 105, 233, 106, 4, - 242, 89, 200, 143, 210, 28, 233, 105, 239, 53, 233, 105, 239, 54, 233, - 105, 221, 123, 233, 105, 233, 106, 198, 214, 42, 221, 124, 221, 123, 233, - 106, 4, 202, 65, 233, 187, 221, 123, 233, 106, 4, 242, 221, 233, 187, - 233, 106, 4, 201, 13, 198, 61, 233, 106, 4, 201, 13, 198, 62, 23, 193, - 57, 233, 195, 233, 106, 4, 201, 13, 198, 62, 23, 198, 171, 233, 187, 236, - 230, 233, 105, 193, 16, 233, 105, 251, 113, 233, 105, 209, 8, 233, 105, - 238, 201, 233, 105, 209, 255, 233, 105, 233, 106, 4, 219, 9, 79, 197, - 217, 236, 230, 247, 220, 207, 62, 233, 105, 232, 94, 233, 106, 4, 211, - 79, 215, 181, 251, 111, 233, 105, 233, 59, 233, 105, 193, 119, 233, 105, - 202, 92, 233, 105, 198, 130, 233, 105, 230, 199, 233, 105, 218, 247, 238, - 201, 233, 105, 233, 106, 4, 211, 79, 215, 181, 228, 132, 233, 105, 233, - 106, 4, 211, 79, 215, 182, 23, 242, 89, 200, 143, 233, 106, 205, 79, 223, - 165, 233, 60, 250, 192, 233, 105, 232, 185, 233, 105, 202, 93, 233, 105, - 236, 197, 233, 105, 233, 106, 193, 51, 215, 181, 233, 106, 4, 216, 218, - 217, 37, 230, 198, 247, 81, 233, 106, 4, 202, 65, 233, 187, 247, 81, 233, - 106, 4, 199, 126, 79, 222, 187, 202, 65, 247, 81, 233, 106, 4, 211, 79, - 215, 181, 202, 65, 247, 81, 233, 106, 4, 229, 224, 233, 187, 247, 81, - 233, 106, 4, 193, 1, 202, 66, 221, 123, 247, 81, 233, 106, 4, 242, 221, - 233, 187, 209, 8, 247, 81, 233, 105, 238, 201, 247, 81, 233, 105, 193, - 119, 247, 81, 233, 105, 202, 85, 232, 94, 233, 105, 202, 85, 202, 65, - 233, 105, 196, 72, 233, 105, 233, 106, 4, 206, 139, 233, 187, 233, 106, - 4, 213, 46, 230, 246, 231, 134, 233, 106, 4, 211, 128, 231, 134, 209, - 253, 248, 70, 237, 131, 205, 50, 215, 227, 229, 228, 215, 227, 201, 237, - 215, 227, 230, 22, 209, 253, 207, 147, 91, 230, 55, 209, 253, 207, 147, - 248, 82, 230, 31, 223, 165, 247, 30, 209, 253, 232, 93, 209, 253, 4, 209, - 8, 233, 105, 209, 253, 4, 232, 174, 230, 30, 186, 193, 105, 209, 60, 219, - 235, 202, 3, 193, 105, 209, 60, 219, 235, 186, 234, 206, 209, 60, 219, - 235, 202, 3, 234, 206, 209, 60, 219, 235, 154, 186, 193, 105, 209, 60, - 219, 235, 154, 202, 3, 193, 105, 209, 60, 219, 235, 154, 186, 234, 206, - 209, 60, 219, 235, 154, 202, 3, 234, 206, 209, 60, 219, 235, 186, 193, - 105, 209, 60, 195, 48, 219, 235, 202, 3, 193, 105, 209, 60, 195, 48, 219, - 235, 186, 234, 206, 209, 60, 195, 48, 219, 235, 202, 3, 234, 206, 209, - 60, 195, 48, 219, 235, 88, 186, 193, 105, 209, 60, 195, 48, 219, 235, 88, - 202, 3, 193, 105, 209, 60, 195, 48, 219, 235, 88, 186, 234, 206, 209, 60, - 195, 48, 219, 235, 88, 202, 3, 234, 206, 209, 60, 195, 48, 219, 235, 186, - 193, 105, 209, 60, 248, 6, 202, 3, 193, 105, 209, 60, 248, 6, 186, 234, - 206, 209, 60, 248, 6, 202, 3, 234, 206, 209, 60, 248, 6, 88, 186, 193, - 105, 209, 60, 248, 6, 88, 202, 3, 193, 105, 209, 60, 248, 6, 88, 186, - 234, 206, 209, 60, 248, 6, 88, 202, 3, 234, 206, 209, 60, 248, 6, 229, 6, - 208, 8, 51, 211, 44, 229, 6, 208, 8, 51, 211, 45, 223, 165, 62, 201, 191, - 202, 26, 208, 8, 51, 211, 44, 202, 26, 208, 8, 51, 211, 45, 223, 165, 62, - 201, 191, 118, 206, 147, 196, 66, 206, 147, 96, 206, 147, 235, 121, 206, - 147, 27, 34, 234, 2, 211, 44, 88, 27, 34, 234, 2, 211, 44, 34, 211, 79, - 234, 2, 211, 44, 88, 34, 211, 79, 234, 2, 211, 44, 88, 251, 140, 211, 44, - 200, 146, 251, 140, 211, 44, 49, 88, 55, 154, 242, 77, 207, 254, 87, 211, - 44, 49, 88, 55, 242, 77, 207, 254, 87, 211, 44, 49, 88, 130, 55, 242, 77, - 207, 254, 87, 211, 44, 88, 223, 105, 211, 44, 49, 223, 105, 211, 44, 88, - 49, 223, 105, 211, 44, 195, 83, 88, 202, 24, 195, 83, 88, 207, 20, 202, - 24, 243, 126, 248, 109, 207, 20, 243, 126, 248, 109, 206, 147, 229, 207, - 201, 86, 219, 34, 207, 154, 247, 104, 229, 133, 198, 16, 229, 133, 198, - 17, 4, 247, 251, 213, 15, 198, 16, 216, 160, 156, 207, 155, 201, 94, 198, - 14, 198, 15, 247, 104, 247, 225, 211, 103, 247, 225, 197, 212, 247, 226, - 201, 64, 216, 44, 251, 144, 232, 246, 234, 89, 209, 52, 247, 104, 211, - 103, 209, 52, 247, 104, 199, 155, 211, 103, 199, 155, 250, 152, 211, 103, - 250, 152, 207, 96, 195, 145, 237, 112, 197, 203, 250, 227, 219, 0, 198, - 23, 215, 220, 215, 187, 207, 153, 200, 164, 207, 153, 215, 187, 247, 156, - 252, 13, 198, 13, 203, 66, 206, 113, 201, 229, 228, 243, 198, 20, 219, - 134, 81, 198, 20, 219, 134, 239, 39, 56, 209, 52, 247, 88, 207, 13, 219, - 134, 197, 238, 232, 220, 211, 107, 209, 21, 236, 146, 213, 46, 234, 75, - 56, 202, 63, 113, 213, 46, 202, 63, 113, 208, 134, 219, 86, 223, 165, - 223, 52, 209, 110, 113, 236, 177, 213, 14, 219, 86, 113, 209, 15, 193, - 144, 113, 213, 30, 193, 144, 113, 248, 186, 213, 46, 248, 185, 248, 184, - 215, 187, 248, 184, 210, 53, 213, 46, 210, 52, 243, 87, 238, 211, 216, - 184, 113, 193, 32, 113, 207, 29, 249, 151, 113, 198, 86, 193, 144, 242, - 218, 203, 21, 249, 63, 249, 61, 210, 93, 239, 23, 238, 158, 249, 128, - 242, 248, 45, 218, 217, 197, 242, 4, 206, 114, 239, 2, 208, 198, 56, 47, - 223, 139, 202, 4, 248, 61, 113, 231, 29, 113, 238, 250, 23, 220, 27, 202, - 93, 252, 59, 203, 44, 249, 127, 248, 236, 248, 237, 249, 4, 209, 110, 79, - 193, 15, 211, 161, 56, 203, 44, 197, 213, 201, 9, 210, 203, 229, 129, - 199, 98, 228, 131, 234, 132, 193, 54, 209, 98, 202, 88, 193, 93, 206, - 190, 247, 235, 230, 64, 23, 193, 9, 203, 79, 211, 134, 235, 96, 215, 191, - 207, 154, 198, 25, 215, 194, 248, 108, 196, 76, 216, 56, 251, 225, 196, - 76, 251, 225, 196, 76, 2, 251, 225, 2, 251, 225, 213, 19, 251, 225, 251, - 226, 237, 95, 251, 226, 250, 240, 205, 119, 211, 103, 232, 246, 234, 89, - 237, 24, 219, 34, 210, 97, 203, 66, 205, 84, 215, 194, 205, 84, 247, 115, - 202, 95, 232, 180, 205, 114, 202, 112, 250, 154, 206, 244, 209, 180, 197, - 203, 206, 140, 202, 113, 160, 16, 40, 208, 4, 160, 16, 40, 251, 227, 160, - 16, 40, 232, 245, 160, 16, 40, 234, 209, 160, 16, 40, 193, 143, 160, 16, - 40, 251, 37, 160, 16, 40, 251, 38, 207, 82, 160, 16, 40, 251, 38, 207, - 81, 160, 16, 40, 251, 38, 195, 31, 160, 16, 40, 251, 38, 195, 30, 160, - 16, 40, 195, 45, 160, 16, 40, 195, 44, 160, 16, 40, 195, 43, 160, 16, 40, - 200, 205, 160, 16, 40, 209, 205, 200, 205, 160, 16, 40, 62, 200, 205, - 160, 16, 40, 216, 183, 200, 237, 160, 16, 40, 216, 183, 200, 236, 160, - 16, 40, 216, 183, 200, 235, 160, 16, 40, 243, 15, 160, 16, 40, 205, 159, - 160, 16, 40, 213, 163, 160, 16, 40, 195, 28, 160, 16, 40, 195, 27, 160, - 16, 40, 206, 149, 205, 159, 160, 16, 40, 206, 149, 205, 158, 160, 16, 40, - 230, 252, 160, 16, 40, 202, 185, 160, 16, 40, 223, 76, 211, 51, 160, 16, - 40, 223, 76, 211, 50, 160, 16, 40, 238, 224, 79, 223, 75, 160, 16, 40, - 207, 78, 79, 223, 75, 160, 16, 40, 239, 14, 211, 51, 160, 16, 40, 223, - 74, 211, 51, 160, 16, 40, 200, 238, 79, 239, 13, 160, 16, 40, 238, 224, - 79, 239, 13, 160, 16, 40, 238, 224, 79, 239, 12, 160, 16, 40, 239, 14, - 251, 81, 160, 16, 40, 205, 160, 79, 239, 14, 251, 81, 160, 16, 40, 200, - 238, 79, 205, 160, 79, 239, 13, 160, 16, 40, 195, 139, 160, 16, 40, 198, - 143, 211, 51, 160, 16, 40, 219, 239, 211, 51, 160, 16, 40, 251, 80, 211, - 51, 160, 16, 40, 200, 238, 79, 251, 79, 160, 16, 40, 205, 160, 79, 251, - 79, 160, 16, 40, 200, 238, 79, 205, 160, 79, 251, 79, 160, 16, 40, 195, - 46, 79, 251, 79, 160, 16, 40, 207, 78, 79, 251, 79, 160, 16, 40, 207, 78, - 79, 251, 78, 160, 16, 40, 207, 77, 160, 16, 40, 207, 76, 160, 16, 40, - 207, 75, 160, 16, 40, 207, 74, 160, 16, 40, 251, 179, 160, 16, 40, 251, - 178, 160, 16, 40, 217, 63, 160, 16, 40, 205, 169, 160, 16, 40, 250, 232, - 160, 16, 40, 207, 107, 160, 16, 40, 207, 106, 160, 16, 40, 250, 156, 160, - 16, 40, 248, 152, 211, 51, 160, 16, 40, 199, 177, 160, 16, 40, 199, 176, - 160, 16, 40, 208, 10, 219, 123, 160, 16, 40, 248, 89, 160, 16, 40, 248, - 88, 160, 16, 40, 248, 87, 160, 16, 40, 251, 153, 160, 16, 40, 211, 133, - 160, 16, 40, 201, 215, 160, 16, 40, 198, 141, 160, 16, 40, 230, 160, 160, - 16, 40, 193, 131, 160, 16, 40, 209, 3, 160, 16, 40, 247, 139, 160, 16, - 40, 197, 50, 160, 16, 40, 247, 106, 215, 200, 160, 16, 40, 205, 94, 79, - 222, 189, 160, 16, 40, 247, 153, 160, 16, 40, 197, 235, 160, 16, 40, 201, - 101, 197, 235, 160, 16, 40, 219, 33, 160, 16, 40, 202, 38, 160, 16, 40, - 196, 54, 160, 16, 40, 228, 186, 235, 73, 160, 16, 40, 250, 206, 160, 16, - 40, 209, 17, 250, 206, 160, 16, 40, 248, 33, 160, 16, 40, 209, 2, 248, - 33, 160, 16, 40, 251, 150, 160, 16, 40, 201, 47, 200, 186, 201, 46, 160, - 16, 40, 201, 47, 200, 186, 201, 45, 160, 16, 40, 200, 234, 160, 16, 40, - 208, 231, 160, 16, 40, 236, 217, 160, 16, 40, 236, 219, 160, 16, 40, 236, - 218, 160, 16, 40, 208, 143, 160, 16, 40, 208, 131, 160, 16, 40, 238, 209, - 160, 16, 40, 238, 208, 160, 16, 40, 238, 207, 160, 16, 40, 238, 206, 160, - 16, 40, 238, 205, 160, 16, 40, 251, 193, 160, 16, 40, 249, 64, 79, 217, - 44, 160, 16, 40, 249, 64, 79, 195, 174, 160, 16, 40, 207, 27, 160, 16, - 40, 228, 178, 160, 16, 40, 213, 192, 160, 16, 40, 237, 205, 160, 16, 40, - 215, 215, 160, 16, 40, 132, 235, 111, 160, 16, 40, 132, 211, 19, 218, - 252, 79, 232, 139, 211, 166, 218, 209, 234, 196, 230, 3, 217, 103, 230, - 248, 208, 26, 211, 54, 62, 219, 221, 223, 58, 50, 197, 241, 62, 196, 76, - 223, 58, 50, 197, 241, 62, 206, 204, 223, 58, 50, 197, 241, 62, 235, 124, - 223, 58, 50, 197, 241, 62, 202, 85, 2, 243, 12, 216, 215, 28, 63, 243, - 12, 28, 63, 243, 12, 88, 63, 243, 12, 195, 83, 88, 63, 243, 12, 233, 199, - 88, 63, 243, 12, 63, 243, 13, 239, 35, 62, 2, 243, 12, 206, 116, 199, - 178, 62, 198, 138, 201, 191, 62, 202, 85, 2, 201, 191, 156, 63, 201, 191, - 216, 215, 63, 201, 191, 28, 63, 201, 191, 88, 63, 201, 191, 195, 83, 88, - 63, 201, 191, 233, 199, 88, 63, 201, 191, 63, 51, 239, 35, 62, 195, 83, - 2, 201, 191, 63, 51, 239, 35, 62, 216, 215, 201, 191, 51, 199, 178, 62, - 198, 138, 237, 34, 62, 195, 83, 2, 237, 34, 62, 216, 215, 2, 237, 34, 63, - 237, 35, 239, 35, 62, 195, 83, 2, 237, 34, 63, 237, 35, 239, 35, 62, 216, - 215, 237, 34, 237, 35, 199, 178, 62, 198, 138, 218, 234, 62, 195, 83, 2, - 218, 234, 62, 216, 215, 2, 218, 234, 63, 218, 235, 239, 35, 62, 2, 218, - 234, 199, 4, 35, 238, 219, 156, 35, 238, 219, 216, 215, 35, 238, 219, 28, - 35, 238, 219, 195, 83, 28, 35, 238, 219, 195, 83, 88, 35, 238, 219, 233, - 199, 88, 35, 238, 219, 199, 4, 205, 155, 156, 205, 155, 216, 215, 205, - 155, 28, 205, 155, 88, 205, 155, 195, 83, 88, 205, 155, 233, 199, 88, - 205, 155, 156, 232, 228, 201, 207, 250, 195, 216, 215, 232, 228, 201, - 207, 250, 195, 28, 232, 228, 201, 207, 250, 195, 88, 232, 228, 201, 207, - 250, 195, 195, 83, 88, 232, 228, 201, 207, 250, 195, 233, 199, 88, 232, - 228, 201, 207, 250, 195, 156, 202, 137, 201, 207, 250, 195, 216, 215, - 202, 137, 201, 207, 250, 195, 28, 202, 137, 201, 207, 250, 195, 88, 202, - 137, 201, 207, 250, 195, 195, 83, 88, 202, 137, 201, 207, 250, 195, 233, - 199, 88, 202, 137, 201, 207, 250, 195, 156, 234, 166, 201, 207, 250, 195, - 216, 215, 234, 166, 201, 207, 250, 195, 28, 234, 166, 201, 207, 250, 195, - 88, 234, 166, 201, 207, 250, 195, 195, 83, 88, 234, 166, 201, 207, 250, - 195, 156, 115, 209, 62, 62, 201, 103, 216, 215, 115, 209, 62, 62, 201, - 103, 115, 209, 62, 62, 201, 103, 216, 215, 115, 209, 62, 209, 132, 201, - 103, 156, 232, 130, 209, 62, 62, 201, 103, 216, 215, 232, 130, 209, 62, - 62, 201, 103, 232, 130, 209, 62, 62, 201, 103, 216, 215, 232, 130, 209, - 62, 209, 132, 201, 103, 207, 20, 156, 232, 130, 209, 62, 209, 132, 201, - 103, 156, 232, 228, 209, 62, 62, 201, 103, 88, 232, 228, 209, 62, 62, - 201, 103, 216, 215, 202, 137, 209, 62, 62, 201, 103, 88, 202, 137, 209, - 62, 62, 201, 103, 202, 137, 209, 62, 209, 132, 201, 103, 216, 215, 234, - 166, 209, 62, 62, 201, 103, 88, 234, 166, 209, 62, 62, 201, 103, 195, 83, - 88, 234, 166, 209, 62, 62, 201, 103, 88, 234, 166, 209, 62, 209, 132, - 201, 103, 156, 197, 38, 209, 62, 62, 201, 103, 88, 197, 38, 209, 62, 62, - 201, 103, 88, 197, 38, 209, 62, 209, 132, 201, 103, 47, 197, 241, 214, - 108, 47, 197, 241, 47, 201, 191, 214, 108, 47, 201, 191, 213, 177, 209, - 62, 63, 201, 103, 220, 15, 211, 57, 243, 12, 220, 15, 192, 73, 243, 12, - 220, 15, 230, 203, 243, 12, 220, 15, 208, 140, 243, 12, 220, 15, 248, 21, - 243, 12, 220, 15, 207, 89, 201, 191, 220, 15, 248, 117, 201, 191, 220, - 15, 211, 57, 201, 191, 220, 15, 192, 73, 201, 191, 220, 15, 230, 203, - 201, 191, 220, 15, 208, 140, 201, 191, 220, 15, 248, 21, 201, 191, 88, - 234, 45, 56, 118, 64, 4, 2, 197, 242, 250, 237, 196, 66, 64, 4, 2, 197, - 242, 250, 237, 96, 64, 4, 2, 197, 242, 250, 237, 235, 121, 64, 4, 2, 197, - 242, 250, 237, 118, 64, 4, 216, 215, 197, 242, 250, 237, 196, 66, 64, 4, - 216, 215, 197, 242, 250, 237, 96, 64, 4, 216, 215, 197, 242, 250, 237, - 235, 121, 64, 4, 216, 215, 197, 242, 250, 237, 118, 64, 4, 220, 15, 197, - 242, 250, 237, 196, 66, 64, 4, 220, 15, 197, 242, 250, 237, 96, 64, 4, - 220, 15, 197, 242, 250, 237, 235, 121, 64, 4, 220, 15, 197, 242, 250, - 237, 118, 64, 4, 2, 234, 39, 250, 237, 196, 66, 64, 4, 2, 234, 39, 250, - 237, 96, 64, 4, 2, 234, 39, 250, 237, 235, 121, 64, 4, 2, 234, 39, 250, - 237, 118, 64, 4, 234, 39, 250, 237, 196, 66, 64, 4, 234, 39, 250, 237, - 96, 64, 4, 234, 39, 250, 237, 235, 121, 64, 4, 234, 39, 250, 237, 88, - 118, 64, 4, 234, 39, 250, 237, 88, 196, 66, 64, 4, 234, 39, 250, 237, 88, - 96, 64, 4, 234, 39, 250, 237, 88, 235, 121, 64, 4, 234, 39, 250, 237, 88, - 118, 64, 4, 220, 15, 234, 39, 250, 237, 88, 196, 66, 64, 4, 220, 15, 234, - 39, 250, 237, 88, 96, 64, 4, 220, 15, 234, 39, 250, 237, 88, 235, 121, - 64, 4, 220, 15, 234, 39, 250, 237, 118, 197, 240, 64, 4, 214, 217, 203, - 150, 196, 66, 197, 240, 64, 4, 214, 217, 203, 150, 96, 197, 240, 64, 4, - 214, 217, 203, 150, 235, 121, 197, 240, 64, 4, 214, 217, 203, 150, 118, - 197, 240, 64, 4, 216, 215, 203, 150, 196, 66, 197, 240, 64, 4, 216, 215, - 203, 150, 96, 197, 240, 64, 4, 216, 215, 203, 150, 235, 121, 197, 240, - 64, 4, 216, 215, 203, 150, 118, 197, 240, 64, 4, 28, 203, 150, 196, 66, - 197, 240, 64, 4, 28, 203, 150, 96, 197, 240, 64, 4, 28, 203, 150, 235, - 121, 197, 240, 64, 4, 28, 203, 150, 118, 197, 240, 64, 4, 88, 203, 150, - 196, 66, 197, 240, 64, 4, 88, 203, 150, 96, 197, 240, 64, 4, 88, 203, - 150, 235, 121, 197, 240, 64, 4, 88, 203, 150, 118, 197, 240, 64, 4, 195, - 83, 88, 203, 150, 196, 66, 197, 240, 64, 4, 195, 83, 88, 203, 150, 96, - 197, 240, 64, 4, 195, 83, 88, 203, 150, 235, 121, 197, 240, 64, 4, 195, - 83, 88, 203, 150, 118, 232, 253, 57, 196, 66, 232, 253, 57, 96, 232, 253, - 57, 235, 121, 232, 253, 57, 118, 112, 57, 196, 66, 112, 57, 96, 112, 57, - 235, 121, 112, 57, 118, 239, 63, 57, 196, 66, 239, 63, 57, 96, 239, 63, - 57, 235, 121, 239, 63, 57, 118, 88, 239, 63, 57, 196, 66, 88, 239, 63, - 57, 96, 88, 239, 63, 57, 235, 121, 88, 239, 63, 57, 118, 88, 57, 196, 66, - 88, 57, 96, 88, 57, 235, 121, 88, 57, 118, 49, 57, 196, 66, 49, 57, 96, - 49, 57, 235, 121, 49, 57, 186, 193, 105, 49, 57, 186, 234, 206, 49, 57, - 202, 3, 234, 206, 49, 57, 202, 3, 193, 105, 49, 57, 45, 50, 49, 57, 133, - 144, 49, 57, 193, 77, 118, 156, 182, 57, 193, 77, 196, 66, 156, 182, 57, - 193, 77, 96, 156, 182, 57, 193, 77, 235, 121, 156, 182, 57, 193, 77, 186, - 193, 105, 156, 182, 57, 193, 77, 186, 234, 206, 156, 182, 57, 193, 77, - 202, 3, 234, 206, 156, 182, 57, 193, 77, 202, 3, 193, 105, 156, 182, 57, - 193, 77, 118, 182, 57, 193, 77, 196, 66, 182, 57, 193, 77, 96, 182, 57, - 193, 77, 235, 121, 182, 57, 193, 77, 186, 193, 105, 182, 57, 193, 77, - 186, 234, 206, 182, 57, 193, 77, 202, 3, 234, 206, 182, 57, 193, 77, 202, - 3, 193, 105, 182, 57, 193, 77, 118, 216, 215, 182, 57, 193, 77, 196, 66, - 216, 215, 182, 57, 193, 77, 96, 216, 215, 182, 57, 193, 77, 235, 121, - 216, 215, 182, 57, 193, 77, 186, 193, 105, 216, 215, 182, 57, 193, 77, - 186, 234, 206, 216, 215, 182, 57, 193, 77, 202, 3, 234, 206, 216, 215, - 182, 57, 193, 77, 202, 3, 193, 105, 216, 215, 182, 57, 193, 77, 118, 88, - 182, 57, 193, 77, 196, 66, 88, 182, 57, 193, 77, 96, 88, 182, 57, 193, - 77, 235, 121, 88, 182, 57, 193, 77, 186, 193, 105, 88, 182, 57, 193, 77, - 186, 234, 206, 88, 182, 57, 193, 77, 202, 3, 234, 206, 88, 182, 57, 193, - 77, 202, 3, 193, 105, 88, 182, 57, 193, 77, 118, 195, 83, 88, 182, 57, - 193, 77, 196, 66, 195, 83, 88, 182, 57, 193, 77, 96, 195, 83, 88, 182, - 57, 193, 77, 235, 121, 195, 83, 88, 182, 57, 193, 77, 186, 193, 105, 195, - 83, 88, 182, 57, 193, 77, 186, 234, 206, 195, 83, 88, 182, 57, 193, 77, - 202, 3, 234, 206, 195, 83, 88, 182, 57, 193, 77, 202, 3, 193, 105, 195, - 83, 88, 182, 57, 118, 197, 242, 250, 237, 196, 66, 197, 242, 250, 237, - 96, 197, 242, 250, 237, 235, 121, 197, 242, 250, 237, 118, 63, 64, 193, - 53, 197, 242, 250, 237, 196, 66, 63, 64, 193, 53, 197, 242, 250, 237, 96, - 63, 64, 193, 53, 197, 242, 250, 237, 235, 121, 63, 64, 193, 53, 197, 242, - 250, 237, 118, 64, 4, 213, 10, 199, 215, 196, 66, 64, 4, 213, 10, 199, - 215, 96, 64, 4, 213, 10, 199, 215, 235, 121, 64, 4, 213, 10, 199, 215, - 88, 64, 203, 151, 193, 75, 107, 88, 64, 203, 151, 193, 75, 105, 198, 253, - 88, 64, 203, 151, 193, 75, 91, 230, 72, 88, 64, 203, 151, 193, 75, 91, - 199, 0, 118, 248, 76, 63, 57, 96, 248, 79, 203, 153, 63, 57, 118, 198, - 54, 203, 153, 63, 57, 96, 198, 54, 203, 153, 63, 57, 118, 219, 220, 63, - 57, 96, 206, 203, 63, 57, 118, 206, 203, 63, 57, 96, 219, 220, 63, 57, - 118, 249, 147, 203, 152, 63, 57, 96, 249, 147, 203, 152, 63, 57, 118, - 232, 97, 203, 152, 63, 57, 96, 232, 97, 203, 152, 63, 57, 63, 64, 203, - 151, 193, 75, 107, 63, 64, 203, 151, 193, 75, 105, 198, 253, 64, 209, 60, - 196, 66, 199, 25, 186, 193, 104, 64, 209, 60, 96, 199, 25, 238, 163, 202, - 3, 193, 104, 47, 238, 220, 232, 145, 4, 232, 130, 236, 140, 47, 238, 220, - 232, 145, 4, 105, 236, 140, 47, 238, 220, 232, 144, 45, 132, 243, 13, 4, - 232, 130, 236, 140, 45, 132, 243, 13, 4, 115, 236, 140, 45, 132, 243, 13, - 4, 105, 236, 140, 45, 132, 243, 13, 4, 236, 142, 45, 132, 243, 12, 235, - 122, 233, 98, 102, 235, 122, 233, 98, 213, 10, 102, 235, 122, 233, 98, - 228, 253, 4, 236, 142, 235, 122, 233, 98, 213, 10, 228, 253, 4, 236, 142, - 209, 138, 232, 249, 63, 229, 225, 248, 21, 229, 225, 209, 137, 230, 55, - 191, 17, 233, 105, 215, 231, 233, 105, 233, 106, 4, 199, 21, 214, 94, - 233, 105, 199, 2, 233, 105, 233, 106, 4, 229, 236, 206, 151, 233, 105, - 228, 152, 233, 105, 3, 79, 199, 34, 228, 188, 247, 141, 216, 235, 230, - 55, 207, 149, 249, 149, 79, 230, 55, 219, 225, 232, 233, 206, 208, 232, - 233, 230, 29, 230, 56, 4, 141, 23, 82, 232, 250, 238, 215, 228, 76, 218, - 244, 191, 239, 230, 56, 56, 233, 106, 4, 238, 240, 230, 11, 242, 210, - 233, 105, 214, 204, 233, 105, 206, 139, 211, 107, 199, 34, 232, 196, 220, - 1, 235, 102, 233, 105, 218, 180, 233, 105, 233, 106, 210, 184, 202, 57, - 233, 105, 233, 106, 4, 91, 233, 194, 207, 148, 230, 198, 233, 106, 4, - 201, 104, 233, 187, 230, 198, 233, 106, 4, 91, 220, 15, 23, 91, 2, 233, - 195, 233, 106, 4, 232, 255, 238, 243, 242, 221, 219, 98, 204, 3, 233, - 106, 4, 200, 77, 238, 243, 215, 181, 202, 65, 233, 106, 4, 202, 65, 233, - 188, 23, 230, 56, 238, 243, 215, 181, 233, 106, 4, 211, 79, 215, 182, - 195, 9, 203, 55, 233, 106, 4, 233, 210, 229, 237, 208, 228, 193, 35, 248, - 42, 210, 183, 133, 198, 87, 204, 32, 208, 216, 217, 93, 223, 165, 197, - 46, 215, 196, 243, 57, 203, 10, 209, 253, 236, 161, 247, 85, 222, 179, - 233, 40, 216, 1, 210, 23, 193, 8, 193, 144, 209, 46, 230, 34, 236, 203, - 217, 37, 193, 69, 232, 188, 235, 97, 4, 235, 95, 242, 228, 231, 17, 197, - 74, 231, 18, 201, 204, 231, 3, 214, 87, 206, 209, 232, 240, 209, 110, - 216, 221, 205, 58, 209, 110, 216, 221, 199, 1, 209, 110, 216, 221, 248, - 63, 231, 12, 217, 48, 250, 225, 196, 94, 238, 174, 201, 66, 220, 112, - 201, 76, 23, 249, 113, 202, 32, 232, 180, 236, 228, 238, 223, 250, 143, - 238, 190, 249, 140, 209, 14, 247, 89, 249, 126, 248, 45, 230, 203, 205, - 166, 203, 143, 210, 169, 79, 232, 163, 201, 10, 232, 207, 234, 181, 231, - 19, 79, 216, 55, 210, 58, 221, 118, 210, 165, 235, 78, 232, 140, 239, 18, - 199, 207, 248, 64, 243, 64, 248, 69, 4, 201, 204, 238, 184, 4, 201, 44, - 242, 95, 248, 25, 209, 178, 208, 220, 238, 157, 79, 216, 226, 205, 138, - 247, 117, 232, 163, 219, 234, 230, 202, 217, 84, 215, 208, 247, 148, 249, - 129, 202, 65, 233, 106, 4, 202, 65, 233, 188, 23, 115, 229, 223, 192, 87, - 233, 105, 202, 65, 233, 106, 4, 199, 131, 233, 106, 4, 210, 104, 228, - 190, 23, 210, 104, 230, 11, 233, 106, 4, 196, 98, 233, 188, 23, 193, 135, - 215, 181, 211, 7, 233, 105, 232, 109, 233, 105, 213, 170, 236, 226, 233, - 105, 233, 106, 229, 8, 249, 149, 199, 125, 233, 106, 4, 209, 95, 233, - 187, 205, 126, 220, 121, 242, 98, 230, 255, 229, 131, 248, 93, 232, 209, - 203, 53, 238, 237, 219, 102, 233, 105, 205, 82, 197, 62, 196, 96, 233, - 105, 234, 216, 235, 87, 249, 66, 203, 129, 210, 251, 232, 122, 233, 105, - 247, 217, 237, 130, 230, 237, 219, 80, 207, 6, 203, 14, 201, 185, 231, - 31, 233, 105, 191, 85, 233, 105, 229, 218, 205, 111, 200, 42, 238, 226, - 222, 84, 219, 72, 210, 60, 229, 123, 210, 110, 207, 175, 219, 43, 215, - 198, 216, 92, 249, 135, 200, 148, 217, 94, 236, 167, 202, 79, 211, 24, - 211, 56, 202, 103, 232, 211, 210, 241, 249, 6, 248, 151, 205, 62, 230, - 165, 236, 164, 208, 204, 247, 119, 234, 111, 242, 66, 207, 89, 230, 80, - 234, 111, 242, 66, 238, 173, 230, 80, 234, 111, 242, 66, 249, 115, 234, - 111, 242, 66, 63, 230, 80, 248, 100, 219, 214, 232, 161, 198, 56, 200, - 184, 200, 179, 205, 189, 195, 81, 234, 214, 4, 229, 227, 251, 237, 215, - 192, 193, 91, 217, 76, 193, 91, 216, 225, 250, 252, 216, 225, 219, 214, - 243, 120, 193, 116, 238, 182, 205, 160, 203, 147, 248, 210, 248, 64, 231, - 199, 211, 95, 233, 87, 193, 174, 247, 218, 217, 31, 235, 106, 228, 29, - 238, 192, 248, 11, 199, 134, 197, 214, 201, 106, 209, 252, 221, 82, 209, - 252, 237, 146, 209, 252, 233, 106, 4, 215, 226, 252, 31, 243, 88, 211, - 120, 252, 31, 249, 10, 209, 252, 209, 253, 4, 229, 232, 209, 253, 223, - 165, 201, 83, 206, 131, 209, 253, 242, 230, 209, 253, 223, 165, 218, 249, - 209, 26, 217, 126, 233, 89, 195, 177, 216, 176, 234, 127, 231, 150, 191, - 5, 248, 52, 211, 57, 229, 225, 248, 173, 247, 113, 205, 95, 231, 11, 242, - 98, 202, 35, 207, 89, 231, 45, 234, 69, 232, 244, 222, 240, 208, 127, - 209, 177, 199, 75, 197, 84, 209, 237, 236, 224, 236, 178, 55, 229, 206, - 242, 71, 252, 73, 232, 246, 233, 204, 198, 58, 248, 33, 217, 124, 218, - 217, 218, 250, 248, 80, 201, 205, 79, 198, 227, 249, 114, 79, 192, 100, - 205, 189, 209, 141, 199, 124, 249, 11, 248, 22, 249, 71, 206, 142, 79, - 210, 137, 249, 90, 79, 202, 38, 201, 206, 207, 105, 214, 198, 251, 136, - 214, 84, 243, 107, 221, 140, 214, 84, 243, 107, 208, 16, 214, 84, 243, - 107, 206, 132, 214, 84, 243, 107, 248, 154, 214, 84, 243, 107, 221, 78, - 214, 84, 243, 107, 210, 76, 63, 243, 107, 221, 79, 206, 123, 232, 136, - 237, 126, 62, 243, 107, 221, 79, 206, 123, 232, 136, 237, 126, 214, 84, - 243, 107, 221, 79, 206, 123, 232, 136, 237, 126, 63, 243, 107, 221, 141, - 206, 123, 213, 172, 237, 126, 63, 243, 107, 208, 17, 206, 123, 213, 172, - 237, 126, 63, 243, 107, 206, 133, 206, 123, 213, 172, 237, 126, 63, 243, - 107, 248, 155, 206, 123, 213, 172, 237, 126, 63, 243, 107, 221, 79, 206, - 123, 213, 172, 237, 126, 63, 243, 107, 210, 77, 206, 123, 213, 172, 237, - 126, 62, 243, 107, 221, 141, 206, 123, 213, 172, 237, 126, 62, 243, 107, - 208, 17, 206, 123, 213, 172, 237, 126, 62, 243, 107, 206, 133, 206, 123, - 213, 172, 237, 126, 62, 243, 107, 248, 155, 206, 123, 213, 172, 237, 126, - 62, 243, 107, 221, 79, 206, 123, 213, 172, 237, 126, 62, 243, 107, 210, - 77, 206, 123, 213, 172, 237, 126, 214, 84, 243, 107, 221, 141, 206, 123, - 213, 172, 237, 126, 214, 84, 243, 107, 208, 17, 206, 123, 213, 172, 237, - 126, 214, 84, 243, 107, 206, 133, 206, 123, 213, 172, 237, 126, 214, 84, - 243, 107, 248, 155, 206, 123, 213, 172, 237, 126, 214, 84, 243, 107, 221, - 79, 206, 123, 213, 172, 237, 126, 214, 84, 243, 107, 210, 77, 206, 123, - 213, 172, 237, 126, 63, 243, 107, 221, 79, 206, 123, 91, 228, 143, 198, - 248, 237, 126, 62, 243, 107, 221, 79, 206, 123, 91, 228, 143, 198, 248, - 237, 126, 214, 84, 243, 107, 221, 79, 206, 123, 91, 228, 143, 198, 248, - 237, 126, 63, 243, 107, 154, 221, 140, 63, 243, 107, 154, 208, 16, 63, - 243, 107, 154, 206, 132, 63, 243, 107, 154, 248, 154, 63, 243, 107, 154, - 221, 78, 63, 243, 107, 154, 210, 76, 62, 243, 107, 154, 221, 140, 62, - 243, 107, 154, 208, 16, 62, 243, 107, 154, 206, 132, 62, 243, 107, 154, - 248, 154, 62, 243, 107, 154, 221, 78, 62, 243, 107, 154, 210, 76, 214, - 84, 243, 107, 154, 221, 140, 214, 84, 243, 107, 154, 208, 16, 214, 84, - 243, 107, 154, 206, 132, 214, 84, 243, 107, 154, 248, 154, 214, 84, 243, - 107, 154, 221, 78, 214, 84, 243, 107, 154, 210, 76, 63, 243, 107, 221, - 79, 206, 123, 105, 228, 143, 197, 29, 237, 126, 62, 243, 107, 221, 79, - 206, 123, 105, 228, 143, 197, 29, 237, 126, 214, 84, 243, 107, 221, 79, - 206, 123, 105, 228, 143, 197, 29, 237, 126, 63, 243, 107, 221, 141, 206, - 123, 105, 228, 143, 203, 243, 237, 126, 63, 243, 107, 208, 17, 206, 123, - 105, 228, 143, 203, 243, 237, 126, 63, 243, 107, 206, 133, 206, 123, 105, - 228, 143, 203, 243, 237, 126, 63, 243, 107, 248, 155, 206, 123, 105, 228, - 143, 203, 243, 237, 126, 63, 243, 107, 221, 79, 206, 123, 105, 228, 143, - 203, 243, 237, 126, 63, 243, 107, 210, 77, 206, 123, 105, 228, 143, 203, - 243, 237, 126, 62, 243, 107, 221, 141, 206, 123, 105, 228, 143, 203, 243, - 237, 126, 62, 243, 107, 208, 17, 206, 123, 105, 228, 143, 203, 243, 237, - 126, 62, 243, 107, 206, 133, 206, 123, 105, 228, 143, 203, 243, 237, 126, - 62, 243, 107, 248, 155, 206, 123, 105, 228, 143, 203, 243, 237, 126, 62, - 243, 107, 221, 79, 206, 123, 105, 228, 143, 203, 243, 237, 126, 62, 243, - 107, 210, 77, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 221, 141, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 208, 17, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 206, 133, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 248, 155, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 221, 79, 206, 123, 105, 228, 143, 203, 243, 237, 126, 214, 84, 243, - 107, 210, 77, 206, 123, 105, 228, 143, 203, 243, 237, 126, 63, 243, 107, - 221, 79, 206, 123, 115, 228, 143, 233, 22, 237, 126, 62, 243, 107, 221, - 79, 206, 123, 115, 228, 143, 233, 22, 237, 126, 214, 84, 243, 107, 221, - 79, 206, 123, 115, 228, 143, 233, 22, 237, 126, 63, 243, 107, 234, 40, - 62, 243, 107, 234, 40, 214, 84, 243, 107, 234, 40, 63, 243, 107, 234, 41, - 206, 123, 213, 172, 237, 126, 62, 243, 107, 234, 41, 206, 123, 213, 172, - 237, 126, 214, 84, 243, 107, 234, 41, 206, 123, 213, 172, 237, 126, 63, - 243, 107, 221, 76, 63, 243, 107, 221, 75, 63, 243, 107, 221, 77, 62, 243, - 107, 221, 76, 62, 243, 107, 221, 75, 62, 243, 107, 221, 77, 192, 205, - 207, 89, 231, 152, 192, 205, 207, 89, 217, 86, 192, 205, 207, 89, 234, - 133, 192, 205, 207, 89, 228, 185, 192, 205, 207, 89, 243, 140, 192, 205, - 207, 89, 247, 116, 192, 205, 207, 89, 202, 27, 192, 205, 62, 231, 152, - 192, 205, 62, 217, 86, 192, 205, 62, 234, 133, 192, 205, 62, 228, 185, - 192, 205, 62, 243, 140, 192, 205, 62, 247, 116, 192, 205, 62, 202, 27, - 249, 112, 203, 52, 211, 100, 200, 135, 248, 29, 203, 26, 198, 237, 205, - 140, 156, 248, 117, 229, 225, 230, 200, 229, 225, 209, 133, 229, 225, - 235, 101, 79, 248, 122, 252, 37, 249, 98, 201, 77, 192, 234, 238, 203, - 191, 253, 221, 121, 210, 131, 248, 94, 217, 125, 193, 162, 209, 139, 214, - 89, 236, 156, 217, 66, 232, 184, 206, 188, 209, 102, 246, 254, 207, 120, - 250, 134, 236, 198, 220, 27, 249, 96, 216, 56, 229, 202, 252, 58, 180, - 235, 96, 242, 90, 247, 91, 205, 109, 205, 76, 220, 111, 102, 216, 28, - 193, 65, 209, 85, 203, 240, 214, 111, 221, 73, 248, 8, 215, 184, 198, 6, - 198, 55, 229, 230, 209, 111, 206, 148, 216, 29, 249, 113, 228, 18, 247, - 102, 130, 249, 60, 230, 62, 232, 172, 230, 56, 233, 82, 230, 81, 209, - 182, 221, 205, 232, 181, 193, 17, 248, 253, 242, 97, 209, 13, 209, 101, - 193, 28, 233, 56, 218, 248, 239, 6, 234, 107, 214, 91, 214, 92, 4, 234, - 180, 228, 94, 223, 4, 193, 61, 230, 245, 251, 131, 229, 225, 218, 207, - 210, 22, 228, 151, 208, 228, 217, 92, 208, 228, 209, 252, 209, 253, 4, - 238, 210, 215, 206, 236, 149, 248, 115, 248, 238, 210, 17, 211, 117, 232, - 207, 199, 196, 232, 167, 199, 132, 209, 9, 219, 94, 249, 13, 223, 20, - 231, 38, 206, 129, 210, 64, 209, 72, 216, 197, 233, 105, 205, 154, 233, - 105, 233, 106, 4, 211, 79, 233, 188, 23, 230, 56, 139, 215, 181, 233, - 106, 4, 210, 49, 233, 195, 233, 106, 4, 237, 41, 215, 181, 235, 140, 219, - 115, 233, 105, 248, 147, 219, 100, 248, 9, 203, 146, 233, 105, 230, 56, - 4, 141, 232, 255, 23, 176, 238, 215, 96, 230, 55, 118, 230, 55, 210, 185, - 144, 230, 55, 210, 185, 133, 230, 55, 141, 209, 60, 250, 185, 199, 34, - 195, 55, 229, 226, 230, 30, 118, 208, 137, 230, 55, 96, 208, 137, 230, - 55, 184, 203, 237, 184, 203, 207, 184, 203, 236, 184, 203, 192, 184, 203, - 221, 184, 203, 206, 184, 203, 235, 184, 203, 184, 184, 203, 214, 184, - 203, 198, 184, 203, 228, 184, 203, 191, 184, 203, 220, 184, 203, 205, - 184, 203, 234, 184, 203, 180, 184, 203, 210, 184, 203, 195, 184, 203, - 224, 184, 203, 187, 184, 203, 201, 184, 203, 231, 184, 203, 183, 184, - 203, 213, 184, 203, 197, 184, 203, 227, 184, 203, 190, 184, 203, 219, - 184, 203, 204, 184, 203, 233, 184, 203, 178, 184, 203, 208, 184, 203, - 193, 184, 203, 222, 184, 203, 185, 184, 203, 215, 184, 203, 199, 184, - 203, 229, 184, 203, 181, 184, 203, 211, 184, 203, 225, 184, 203, 188, - 184, 203, 217, 184, 203, 202, 184, 203, 232, 184, 203, 179, 184, 203, - 209, 184, 203, 194, 184, 203, 223, 184, 203, 186, 184, 203, 216, 184, - 203, 200, 184, 203, 230, 184, 203, 182, 184, 203, 212, 184, 203, 196, - 184, 203, 226, 184, 203, 189, 184, 203, 218, 184, 203, 203, 110, 45, 184, - 237, 41, 110, 82, 45, 119, 110, 247, 23, 110, 45, 184, 237, 41, 110, 82, - 45, 119, 110, 179, 110, 45, 184, 237, 41, 116, 82, 45, 119, 110, 247, 23, - 110, 45, 184, 237, 41, 116, 82, 45, 119, 110, 179, 110, 45, 184, 237, 41, - 116, 45, 119, 110, 247, 23, 110, 50, 184, 237, 41, 116, 82, 45, 119, 116, - 247, 23, 110, 50, 184, 237, 41, 116, 82, 45, 119, 116, 179, 110, 50, 184, - 237, 41, 110, 82, 45, 119, 116, 247, 23, 110, 50, 184, 237, 41, 110, 82, - 45, 119, 116, 179, 110, 50, 184, 237, 41, 110, 45, 119, 116, 247, 23, - 110, 50, 184, 237, 41, 110, 82, 45, 119, 116, 82, 179, 110, 50, 184, 237, - 41, 110, 247, 24, 119, 110, 82, 179, 110, 50, 184, 237, 41, 110, 45, 119, - 110, 82, 179, 110, 50, 184, 237, 41, 110, 247, 24, 119, 116, 82, 179, - 110, 50, 184, 237, 41, 110, 45, 119, 116, 82, 179, 110, 50, 184, 237, 41, - 110, 247, 24, 119, 116, 179, 110, 45, 184, 237, 41, 116, 247, 24, 119, - 116, 82, 179, 110, 45, 184, 237, 41, 116, 45, 119, 116, 82, 179, 110, 45, - 184, 237, 41, 116, 247, 24, 119, 110, 82, 179, 110, 45, 184, 237, 41, - 116, 45, 119, 110, 82, 179, 110, 45, 184, 237, 41, 116, 247, 24, 119, - 110, 179, 110, 45, 184, 237, 41, 116, 82, 45, 119, 110, 82, 179, 116, 50, - 184, 237, 41, 110, 82, 45, 119, 110, 247, 23, 116, 50, 184, 237, 41, 110, - 82, 45, 119, 110, 179, 116, 50, 184, 237, 41, 116, 82, 45, 119, 110, 247, - 23, 116, 50, 184, 237, 41, 116, 82, 45, 119, 110, 179, 116, 50, 184, 237, - 41, 116, 45, 119, 110, 247, 23, 116, 45, 184, 237, 41, 116, 82, 45, 119, - 116, 247, 23, 116, 45, 184, 237, 41, 116, 82, 45, 119, 116, 179, 116, 45, - 184, 237, 41, 110, 82, 45, 119, 116, 247, 23, 116, 45, 184, 237, 41, 110, - 82, 45, 119, 116, 179, 116, 45, 184, 237, 41, 110, 45, 119, 116, 247, 23, - 116, 45, 184, 237, 41, 110, 82, 45, 119, 116, 82, 179, 116, 45, 184, 237, - 41, 110, 247, 24, 119, 110, 82, 179, 116, 45, 184, 237, 41, 110, 45, 119, - 110, 82, 179, 116, 45, 184, 237, 41, 110, 247, 24, 119, 116, 82, 179, - 116, 45, 184, 237, 41, 110, 45, 119, 116, 82, 179, 116, 45, 184, 237, 41, - 110, 247, 24, 119, 116, 179, 116, 50, 184, 237, 41, 116, 247, 24, 119, - 116, 82, 179, 116, 50, 184, 237, 41, 116, 45, 119, 116, 82, 179, 116, 50, - 184, 237, 41, 116, 247, 24, 119, 110, 82, 179, 116, 50, 184, 237, 41, - 116, 45, 119, 110, 82, 179, 116, 50, 184, 237, 41, 116, 247, 24, 119, - 110, 179, 116, 50, 184, 237, 41, 116, 82, 45, 119, 110, 82, 179, 116, 23, - 50, 23, 110, 197, 238, 115, 208, 23, 248, 131, 45, 23, 110, 23, 50, 197, - 238, 115, 208, 23, 248, 131, 116, 23, 45, 23, 110, 197, 238, 115, 208, - 23, 248, 131, 45, 23, 116, 23, 50, 197, 238, 115, 208, 23, 248, 131, 45, - 197, 238, 91, 208, 25, 248, 131, 116, 197, 238, 91, 208, 25, 248, 131, - 50, 197, 238, 91, 208, 25, 248, 131, 110, 197, 238, 91, 208, 25, 248, - 131, 81, 91, 234, 162, 248, 129, 81, 91, 234, 162, 248, 128, 81, 91, 234, - 162, 248, 127, 81, 91, 234, 162, 248, 126, 81, 91, 234, 162, 248, 125, - 81, 91, 234, 162, 248, 124, 228, 243, 91, 234, 162, 248, 129, 228, 243, - 91, 234, 162, 248, 128, 228, 243, 91, 234, 162, 248, 127, 228, 243, 91, - 234, 162, 248, 126, 228, 243, 91, 234, 162, 248, 125, 228, 243, 91, 234, - 162, 248, 124, 45, 23, 110, 91, 234, 162, 248, 131, 45, 23, 116, 91, 234, - 162, 248, 131, 50, 23, 116, 91, 234, 162, 248, 131, 50, 23, 110, 91, 234, - 162, 248, 131, 116, 23, 110, 91, 234, 162, 248, 131, 228, 243, 91, 234, - 162, 248, 130, 116, 91, 208, 25, 248, 131, 116, 115, 234, 160, 248, 131, - 116, 232, 228, 234, 160, 248, 131, 116, 115, 208, 23, 248, 131, 116, 203, - 248, 234, 160, 248, 131, 50, 91, 208, 25, 248, 131, 50, 115, 234, 160, - 248, 131, 50, 232, 228, 234, 160, 248, 131, 50, 115, 208, 23, 248, 131, - 50, 203, 248, 234, 160, 248, 131, 45, 132, 216, 215, 203, 154, 50, 132, - 216, 215, 203, 154, 116, 132, 216, 215, 203, 154, 110, 132, 216, 215, - 203, 154, 223, 97, 216, 215, 203, 154, 116, 132, 184, 23, 110, 132, 223, - 97, 216, 215, 203, 154, 116, 132, 223, 97, 216, 215, 203, 155, 23, 110, - 132, 248, 131, 45, 132, 223, 97, 216, 215, 203, 155, 23, 50, 132, 248, - 131, 243, 126, 248, 110, 233, 7, 223, 97, 243, 126, 248, 110, 233, 7, 88, - 228, 243, 233, 7, 116, 45, 119, 110, 50, 233, 7, 116, 50, 119, 110, 45, - 233, 7, 116, 23, 110, 197, 238, 132, 248, 131, 45, 23, 50, 197, 238, 132, - 248, 131, 116, 45, 197, 238, 216, 215, 203, 154, 116, 50, 197, 238, 216, - 215, 203, 154, 110, 50, 197, 238, 216, 215, 203, 154, 110, 45, 197, 238, - 216, 215, 203, 154, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 219, - 226, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 179, 111, 122, 156, - 237, 41, 82, 45, 119, 110, 247, 23, 111, 122, 156, 237, 41, 82, 50, 119, - 110, 247, 23, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 45, 119, - 110, 247, 23, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 50, 119, - 110, 247, 23, 111, 122, 156, 237, 41, 82, 45, 119, 110, 247, 24, 119, 82, - 179, 111, 122, 156, 237, 41, 82, 45, 119, 116, 247, 24, 119, 82, 179, - 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 45, 23, 82, 50, 119, 110, - 247, 23, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 50, 23, 82, 45, - 119, 110, 247, 23, 111, 122, 156, 237, 41, 116, 247, 24, 119, 82, 50, - 119, 110, 247, 24, 119, 82, 219, 226, 111, 122, 156, 237, 41, 116, 247, - 24, 119, 82, 45, 119, 110, 247, 24, 119, 82, 179, 111, 122, 156, 237, 41, - 82, 45, 119, 116, 247, 24, 119, 82, 50, 119, 110, 247, 23, 111, 122, 156, - 237, 41, 82, 50, 119, 116, 247, 24, 119, 82, 45, 119, 110, 247, 23, 111, - 122, 156, 237, 41, 237, 34, 111, 122, 156, 228, 243, 4, 81, 106, 250, - 236, 209, 61, 223, 97, 243, 128, 77, 45, 132, 206, 43, 217, 92, 50, 132, - 206, 43, 217, 92, 223, 97, 235, 121, 64, 4, 198, 136, 219, 216, 118, 64, - 23, 116, 23, 110, 91, 234, 162, 248, 131, 96, 64, 23, 116, 23, 110, 91, - 234, 162, 248, 131, 235, 121, 64, 23, 50, 91, 234, 162, 248, 131, 196, - 66, 64, 23, 50, 91, 234, 162, 248, 131, 45, 132, 232, 173, 50, 132, 232, - 173, 195, 16, 35, 238, 219, 50, 211, 79, 112, 236, 142, 214, 108, 237, - 41, 238, 219, 214, 108, 237, 41, 82, 50, 119, 110, 247, 23, 214, 108, - 237, 41, 237, 34, 63, 88, 205, 156, 4, 206, 114, 239, 2, 45, 199, 1, 63, - 50, 209, 60, 223, 150, 82, 199, 1, 63, 50, 209, 60, 223, 150, 50, 199, 1, - 63, 50, 209, 60, 223, 150, 214, 108, 112, 208, 15, 77, 201, 76, 233, 14, - 201, 76, 233, 15, 4, 250, 249, 207, 148, 201, 76, 233, 15, 219, 233, 219, - 226, 201, 76, 233, 15, 219, 233, 179, 201, 76, 233, 15, 4, 235, 108, 63, - 196, 76, 243, 102, 205, 43, 17, 191, 77, 205, 43, 17, 107, 205, 43, 17, - 109, 205, 43, 17, 138, 205, 43, 17, 134, 205, 43, 17, 150, 205, 43, 17, - 169, 205, 43, 17, 175, 205, 43, 17, 171, 205, 43, 17, 178, 12, 15, 228, - 15, 12, 15, 228, 14, 12, 15, 228, 13, 12, 15, 228, 12, 12, 15, 228, 11, - 12, 15, 228, 10, 12, 15, 228, 9, 12, 15, 228, 8, 12, 15, 228, 7, 12, 15, - 228, 6, 12, 15, 228, 5, 12, 15, 228, 4, 12, 15, 228, 3, 12, 15, 228, 2, - 12, 15, 228, 1, 12, 15, 228, 0, 12, 15, 227, 255, 12, 15, 227, 254, 12, - 15, 227, 253, 12, 15, 227, 252, 12, 15, 227, 251, 12, 15, 227, 250, 12, - 15, 227, 249, 12, 15, 227, 248, 12, 15, 227, 247, 12, 15, 227, 246, 12, - 15, 227, 245, 12, 15, 227, 244, 12, 15, 227, 243, 12, 15, 227, 242, 12, - 15, 227, 241, 12, 15, 227, 240, 12, 15, 227, 239, 12, 15, 227, 238, 12, - 15, 227, 237, 12, 15, 227, 236, 12, 15, 227, 235, 12, 15, 227, 234, 12, - 15, 227, 233, 12, 15, 227, 232, 12, 15, 227, 231, 12, 15, 227, 230, 12, - 15, 227, 229, 12, 15, 227, 228, 12, 15, 227, 227, 12, 15, 227, 226, 12, - 15, 227, 225, 12, 15, 227, 224, 12, 15, 227, 223, 12, 15, 227, 222, 12, - 15, 227, 221, 12, 15, 227, 220, 12, 15, 227, 219, 12, 15, 227, 218, 12, - 15, 227, 217, 12, 15, 227, 216, 12, 15, 227, 215, 12, 15, 227, 214, 12, - 15, 227, 213, 12, 15, 227, 212, 12, 15, 227, 211, 12, 15, 227, 210, 12, - 15, 227, 209, 12, 15, 227, 208, 12, 15, 227, 207, 12, 15, 227, 206, 12, - 15, 227, 205, 12, 15, 227, 204, 12, 15, 227, 203, 12, 15, 227, 202, 12, - 15, 227, 201, 12, 15, 227, 200, 12, 15, 227, 199, 12, 15, 227, 198, 12, - 15, 227, 197, 12, 15, 227, 196, 12, 15, 227, 195, 12, 15, 227, 194, 12, - 15, 227, 193, 12, 15, 227, 192, 12, 15, 227, 191, 12, 15, 227, 190, 12, - 15, 227, 189, 12, 15, 227, 188, 12, 15, 227, 187, 12, 15, 227, 186, 12, - 15, 227, 185, 12, 15, 227, 184, 12, 15, 227, 183, 12, 15, 227, 182, 12, - 15, 227, 181, 12, 15, 227, 180, 12, 15, 227, 179, 12, 15, 227, 178, 12, - 15, 227, 177, 12, 15, 227, 176, 12, 15, 227, 175, 12, 15, 227, 174, 12, - 15, 227, 173, 12, 15, 227, 172, 12, 15, 227, 171, 12, 15, 227, 170, 12, - 15, 227, 169, 12, 15, 227, 168, 12, 15, 227, 167, 12, 15, 227, 166, 12, - 15, 227, 165, 12, 15, 227, 164, 12, 15, 227, 163, 12, 15, 227, 162, 12, - 15, 227, 161, 12, 15, 227, 160, 12, 15, 227, 159, 12, 15, 227, 158, 12, - 15, 227, 157, 12, 15, 227, 156, 12, 15, 227, 155, 12, 15, 227, 154, 12, - 15, 227, 153, 12, 15, 227, 152, 12, 15, 227, 151, 12, 15, 227, 150, 12, - 15, 227, 149, 12, 15, 227, 148, 12, 15, 227, 147, 12, 15, 227, 146, 12, - 15, 227, 145, 12, 15, 227, 144, 12, 15, 227, 143, 12, 15, 227, 142, 12, - 15, 227, 141, 12, 15, 227, 140, 12, 15, 227, 139, 12, 15, 227, 138, 12, - 15, 227, 137, 12, 15, 227, 136, 12, 15, 227, 135, 12, 15, 227, 134, 12, - 15, 227, 133, 12, 15, 227, 132, 12, 15, 227, 131, 12, 15, 227, 130, 12, - 15, 227, 129, 12, 15, 227, 128, 12, 15, 227, 127, 12, 15, 227, 126, 12, - 15, 227, 125, 12, 15, 227, 124, 12, 15, 227, 123, 12, 15, 227, 122, 12, - 15, 227, 121, 12, 15, 227, 120, 12, 15, 227, 119, 12, 15, 227, 118, 12, - 15, 227, 117, 12, 15, 227, 116, 12, 15, 227, 115, 12, 15, 227, 114, 12, - 15, 227, 113, 12, 15, 227, 112, 12, 15, 227, 111, 12, 15, 227, 110, 12, - 15, 227, 109, 12, 15, 227, 108, 12, 15, 227, 107, 12, 15, 227, 106, 12, - 15, 227, 105, 12, 15, 227, 104, 12, 15, 227, 103, 12, 15, 227, 102, 12, - 15, 227, 101, 12, 15, 227, 100, 12, 15, 227, 99, 12, 15, 227, 98, 12, 15, - 227, 97, 12, 15, 227, 96, 12, 15, 227, 95, 12, 15, 227, 94, 12, 15, 227, - 93, 12, 15, 227, 92, 12, 15, 227, 91, 12, 15, 227, 90, 12, 15, 227, 89, - 12, 15, 227, 88, 12, 15, 227, 87, 12, 15, 227, 86, 12, 15, 227, 85, 12, - 15, 227, 84, 12, 15, 227, 83, 12, 15, 227, 82, 12, 15, 227, 81, 12, 15, - 227, 80, 12, 15, 227, 79, 12, 15, 227, 78, 12, 15, 227, 77, 12, 15, 227, - 76, 12, 15, 227, 75, 12, 15, 227, 74, 12, 15, 227, 73, 12, 15, 227, 72, - 12, 15, 227, 71, 12, 15, 227, 70, 12, 15, 227, 69, 12, 15, 227, 68, 12, - 15, 227, 67, 12, 15, 227, 66, 12, 15, 227, 65, 12, 15, 227, 64, 12, 15, - 227, 63, 12, 15, 227, 62, 12, 15, 227, 61, 12, 15, 227, 60, 12, 15, 227, - 59, 12, 15, 227, 58, 12, 15, 227, 57, 12, 15, 227, 56, 12, 15, 227, 55, - 12, 15, 227, 54, 12, 15, 227, 53, 12, 15, 227, 52, 12, 15, 227, 51, 12, - 15, 227, 50, 12, 15, 227, 49, 12, 15, 227, 48, 12, 15, 227, 47, 12, 15, - 227, 46, 12, 15, 227, 45, 12, 15, 227, 44, 12, 15, 227, 43, 12, 15, 227, - 42, 12, 15, 227, 41, 12, 15, 227, 40, 12, 15, 227, 39, 12, 15, 227, 38, - 12, 15, 227, 37, 12, 15, 227, 36, 12, 15, 227, 35, 12, 15, 227, 34, 12, - 15, 227, 33, 12, 15, 227, 32, 12, 15, 227, 31, 12, 15, 227, 30, 12, 15, - 227, 29, 12, 15, 227, 28, 12, 15, 227, 27, 12, 15, 227, 26, 12, 15, 227, - 25, 12, 15, 227, 24, 12, 15, 227, 23, 12, 15, 227, 22, 12, 15, 227, 21, - 12, 15, 227, 20, 12, 15, 227, 19, 12, 15, 227, 18, 12, 15, 227, 17, 12, - 15, 227, 16, 12, 15, 227, 15, 12, 15, 227, 14, 12, 15, 227, 13, 12, 15, - 227, 12, 12, 15, 227, 11, 12, 15, 227, 10, 12, 15, 227, 9, 12, 15, 227, - 8, 12, 15, 227, 7, 12, 15, 227, 6, 12, 15, 227, 5, 12, 15, 227, 4, 12, - 15, 227, 3, 12, 15, 227, 2, 12, 15, 227, 1, 12, 15, 227, 0, 12, 15, 226, - 255, 12, 15, 226, 254, 12, 15, 226, 253, 12, 15, 226, 252, 12, 15, 226, - 251, 12, 15, 226, 250, 12, 15, 226, 249, 12, 15, 226, 248, 12, 15, 226, - 247, 12, 15, 226, 246, 12, 15, 226, 245, 12, 15, 226, 244, 12, 15, 226, - 243, 12, 15, 226, 242, 12, 15, 226, 241, 12, 15, 226, 240, 12, 15, 226, - 239, 12, 15, 226, 238, 12, 15, 226, 237, 12, 15, 226, 236, 12, 15, 226, - 235, 12, 15, 226, 234, 12, 15, 226, 233, 12, 15, 226, 232, 12, 15, 226, - 231, 12, 15, 226, 230, 12, 15, 226, 229, 12, 15, 226, 228, 12, 15, 226, - 227, 12, 15, 226, 226, 12, 15, 226, 225, 12, 15, 226, 224, 12, 15, 226, - 223, 12, 15, 226, 222, 12, 15, 226, 221, 12, 15, 226, 220, 12, 15, 226, - 219, 12, 15, 226, 218, 12, 15, 226, 217, 12, 15, 226, 216, 12, 15, 226, - 215, 12, 15, 226, 214, 12, 15, 226, 213, 12, 15, 226, 212, 12, 15, 226, - 211, 12, 15, 226, 210, 12, 15, 226, 209, 12, 15, 226, 208, 12, 15, 226, - 207, 12, 15, 226, 206, 12, 15, 226, 205, 12, 15, 226, 204, 12, 15, 226, - 203, 12, 15, 226, 202, 12, 15, 226, 201, 12, 15, 226, 200, 12, 15, 226, - 199, 12, 15, 226, 198, 12, 15, 226, 197, 12, 15, 226, 196, 12, 15, 226, - 195, 12, 15, 226, 194, 12, 15, 226, 193, 12, 15, 226, 192, 12, 15, 226, - 191, 12, 15, 226, 190, 12, 15, 226, 189, 12, 15, 226, 188, 12, 15, 226, - 187, 12, 15, 226, 186, 12, 15, 226, 185, 12, 15, 226, 184, 12, 15, 226, - 183, 12, 15, 226, 182, 12, 15, 226, 181, 12, 15, 226, 180, 12, 15, 226, - 179, 12, 15, 226, 178, 12, 15, 226, 177, 12, 15, 226, 176, 12, 15, 226, - 175, 12, 15, 226, 174, 12, 15, 226, 173, 12, 15, 226, 172, 12, 15, 226, - 171, 12, 15, 226, 170, 12, 15, 226, 169, 12, 15, 226, 168, 12, 15, 226, - 167, 12, 15, 226, 166, 12, 15, 226, 165, 12, 15, 226, 164, 12, 15, 226, - 163, 12, 15, 226, 162, 12, 15, 226, 161, 12, 15, 226, 160, 12, 15, 226, - 159, 12, 15, 226, 158, 12, 15, 226, 157, 12, 15, 226, 156, 12, 15, 226, - 155, 12, 15, 226, 154, 12, 15, 226, 153, 12, 15, 226, 152, 12, 15, 226, - 151, 12, 15, 226, 150, 12, 15, 226, 149, 12, 15, 226, 148, 12, 15, 226, - 147, 12, 15, 226, 146, 12, 15, 226, 145, 12, 15, 226, 144, 12, 15, 226, - 143, 12, 15, 226, 142, 12, 15, 226, 141, 12, 15, 226, 140, 12, 15, 226, - 139, 12, 15, 226, 138, 12, 15, 226, 137, 12, 15, 226, 136, 12, 15, 226, - 135, 12, 15, 226, 134, 12, 15, 226, 133, 12, 15, 226, 132, 12, 15, 226, - 131, 12, 15, 226, 130, 12, 15, 226, 129, 12, 15, 226, 128, 12, 15, 226, - 127, 12, 15, 226, 126, 12, 15, 226, 125, 12, 15, 226, 124, 12, 15, 226, - 123, 12, 15, 226, 122, 12, 15, 226, 121, 12, 15, 226, 120, 12, 15, 226, - 119, 12, 15, 226, 118, 12, 15, 226, 117, 12, 15, 226, 116, 12, 15, 226, - 115, 12, 15, 226, 114, 12, 15, 226, 113, 12, 15, 226, 112, 12, 15, 226, - 111, 12, 15, 226, 110, 12, 15, 226, 109, 12, 15, 226, 108, 12, 15, 226, - 107, 12, 15, 226, 106, 12, 15, 226, 105, 12, 15, 226, 104, 12, 15, 226, - 103, 12, 15, 226, 102, 12, 15, 226, 101, 12, 15, 226, 100, 12, 15, 226, - 99, 12, 15, 226, 98, 12, 15, 226, 97, 12, 15, 226, 96, 12, 15, 226, 95, - 12, 15, 226, 94, 12, 15, 226, 93, 12, 15, 226, 92, 12, 15, 226, 91, 12, - 15, 226, 90, 12, 15, 226, 89, 12, 15, 226, 88, 12, 15, 226, 87, 12, 15, - 226, 86, 12, 15, 226, 85, 12, 15, 226, 84, 12, 15, 226, 83, 12, 15, 226, - 82, 12, 15, 226, 81, 12, 15, 226, 80, 12, 15, 226, 79, 12, 15, 226, 78, - 12, 15, 226, 77, 12, 15, 226, 76, 12, 15, 226, 75, 12, 15, 226, 74, 12, - 15, 226, 73, 12, 15, 226, 72, 12, 15, 226, 71, 12, 15, 226, 70, 12, 15, - 226, 69, 12, 15, 226, 68, 12, 15, 226, 67, 12, 15, 226, 66, 12, 15, 226, - 65, 12, 15, 226, 64, 12, 15, 226, 63, 12, 15, 226, 62, 12, 15, 226, 61, - 12, 15, 226, 60, 12, 15, 226, 59, 12, 15, 226, 58, 12, 15, 226, 57, 12, - 15, 226, 56, 12, 15, 226, 55, 12, 15, 226, 54, 12, 15, 226, 53, 12, 15, - 226, 52, 12, 15, 226, 51, 12, 15, 226, 50, 12, 15, 226, 49, 12, 15, 226, - 48, 12, 15, 226, 47, 12, 15, 226, 46, 12, 15, 226, 45, 12, 15, 226, 44, - 12, 15, 226, 43, 12, 15, 226, 42, 12, 15, 226, 41, 12, 15, 226, 40, 12, - 15, 226, 39, 12, 15, 226, 38, 12, 15, 226, 37, 12, 15, 226, 36, 12, 15, - 226, 35, 12, 15, 226, 34, 12, 15, 226, 33, 12, 15, 226, 32, 12, 15, 226, - 31, 12, 15, 226, 30, 12, 15, 226, 29, 12, 15, 226, 28, 12, 15, 226, 27, - 12, 15, 226, 26, 12, 15, 226, 25, 12, 15, 226, 24, 12, 15, 226, 23, 12, - 15, 226, 22, 12, 15, 226, 21, 12, 15, 226, 20, 12, 15, 226, 19, 12, 15, - 226, 18, 12, 15, 226, 17, 12, 15, 226, 16, 12, 15, 226, 15, 12, 15, 226, - 14, 12, 15, 226, 13, 12, 15, 226, 12, 12, 15, 226, 11, 12, 15, 226, 10, - 12, 15, 226, 9, 12, 15, 226, 8, 12, 15, 226, 7, 12, 15, 226, 6, 12, 15, - 226, 5, 12, 15, 226, 4, 12, 15, 226, 3, 12, 15, 226, 2, 12, 15, 226, 1, - 12, 15, 226, 0, 12, 15, 225, 255, 12, 15, 225, 254, 12, 15, 225, 253, 12, - 15, 225, 252, 12, 15, 225, 251, 12, 15, 225, 250, 12, 15, 225, 249, 12, - 15, 225, 248, 12, 15, 225, 247, 12, 15, 225, 246, 12, 15, 225, 245, 12, - 15, 225, 244, 12, 15, 225, 243, 12, 15, 225, 242, 220, 22, 199, 223, 199, - 224, 201, 248, 199, 224, 233, 218, 77, 199, 224, 207, 254, 77, 199, 224, - 31, 56, 199, 224, 236, 157, 56, 199, 224, 210, 15, 56, 199, 224, 251, - 139, 199, 224, 251, 51, 199, 224, 45, 210, 115, 199, 224, 50, 210, 115, - 199, 224, 250, 195, 199, 224, 108, 56, 199, 224, 242, 76, 199, 224, 228, - 89, 199, 224, 232, 82, 201, 64, 199, 224, 202, 24, 199, 224, 17, 191, 77, - 199, 224, 17, 107, 199, 224, 17, 109, 199, 224, 17, 138, 199, 224, 17, - 134, 199, 224, 17, 150, 199, 224, 17, 169, 199, 224, 17, 175, 199, 224, - 17, 171, 199, 224, 17, 178, 199, 224, 242, 85, 199, 224, 204, 26, 199, - 224, 219, 182, 56, 199, 224, 234, 45, 56, 199, 224, 230, 206, 56, 199, - 224, 208, 15, 77, 199, 224, 242, 74, 250, 184, 199, 224, 8, 6, 1, 65, - 199, 224, 8, 6, 1, 250, 122, 199, 224, 8, 6, 1, 247, 195, 199, 224, 8, 6, - 1, 238, 129, 199, 224, 8, 6, 1, 71, 199, 224, 8, 6, 1, 233, 177, 199, - 224, 8, 6, 1, 232, 53, 199, 224, 8, 6, 1, 230, 118, 199, 224, 8, 6, 1, - 68, 199, 224, 8, 6, 1, 223, 37, 199, 224, 8, 6, 1, 222, 154, 199, 224, 8, - 6, 1, 172, 199, 224, 8, 6, 1, 218, 170, 199, 224, 8, 6, 1, 215, 63, 199, - 224, 8, 6, 1, 74, 199, 224, 8, 6, 1, 210, 238, 199, 224, 8, 6, 1, 208, - 106, 199, 224, 8, 6, 1, 146, 199, 224, 8, 6, 1, 206, 9, 199, 224, 8, 6, - 1, 200, 43, 199, 224, 8, 6, 1, 66, 199, 224, 8, 6, 1, 196, 12, 199, 224, - 8, 6, 1, 193, 224, 199, 224, 8, 6, 1, 192, 235, 199, 224, 8, 6, 1, 192, - 159, 199, 224, 8, 6, 1, 191, 166, 199, 224, 45, 51, 248, 55, 199, 224, - 207, 20, 202, 24, 199, 224, 50, 51, 248, 55, 199, 224, 243, 4, 252, 62, - 199, 224, 130, 219, 114, 199, 224, 230, 213, 252, 62, 199, 224, 8, 2, 1, - 65, 199, 224, 8, 2, 1, 250, 122, 199, 224, 8, 2, 1, 247, 195, 199, 224, - 8, 2, 1, 238, 129, 199, 224, 8, 2, 1, 71, 199, 224, 8, 2, 1, 233, 177, - 199, 224, 8, 2, 1, 232, 53, 199, 224, 8, 2, 1, 230, 118, 199, 224, 8, 2, - 1, 68, 199, 224, 8, 2, 1, 223, 37, 199, 224, 8, 2, 1, 222, 154, 199, 224, - 8, 2, 1, 172, 199, 224, 8, 2, 1, 218, 170, 199, 224, 8, 2, 1, 215, 63, - 199, 224, 8, 2, 1, 74, 199, 224, 8, 2, 1, 210, 238, 199, 224, 8, 2, 1, - 208, 106, 199, 224, 8, 2, 1, 146, 199, 224, 8, 2, 1, 206, 9, 199, 224, 8, - 2, 1, 200, 43, 199, 224, 8, 2, 1, 66, 199, 224, 8, 2, 1, 196, 12, 199, - 224, 8, 2, 1, 193, 224, 199, 224, 8, 2, 1, 192, 235, 199, 224, 8, 2, 1, - 192, 159, 199, 224, 8, 2, 1, 191, 166, 199, 224, 45, 238, 173, 248, 55, - 199, 224, 81, 219, 114, 199, 224, 50, 238, 173, 248, 55, 199, 224, 198, - 152, 247, 129, 199, 223, 67, 204, 212, 67, 204, 201, 67, 204, 190, 67, - 204, 178, 67, 204, 167, 67, 204, 156, 67, 204, 145, 67, 204, 134, 67, - 204, 123, 67, 204, 115, 67, 204, 114, 67, 204, 113, 67, 204, 112, 67, - 204, 110, 67, 204, 109, 67, 204, 108, 67, 204, 107, 67, 204, 106, 67, - 204, 105, 67, 204, 104, 67, 204, 103, 67, 204, 102, 67, 204, 101, 67, - 204, 99, 67, 204, 98, 67, 204, 97, 67, 204, 96, 67, 204, 95, 67, 204, 94, - 67, 204, 93, 67, 204, 92, 67, 204, 91, 67, 204, 90, 67, 204, 88, 67, 204, - 87, 67, 204, 86, 67, 204, 85, 67, 204, 84, 67, 204, 83, 67, 204, 82, 67, - 204, 81, 67, 204, 80, 67, 204, 79, 67, 204, 77, 67, 204, 76, 67, 204, 75, - 67, 204, 74, 67, 204, 73, 67, 204, 72, 67, 204, 71, 67, 204, 70, 67, 204, - 69, 67, 204, 68, 67, 204, 66, 67, 204, 65, 67, 204, 64, 67, 204, 63, 67, - 204, 62, 67, 204, 61, 67, 204, 60, 67, 204, 59, 67, 204, 58, 67, 204, 57, - 67, 204, 55, 67, 204, 54, 67, 204, 53, 67, 204, 52, 67, 204, 51, 67, 204, - 50, 67, 204, 49, 67, 204, 48, 67, 204, 47, 67, 204, 46, 67, 204, 44, 67, - 204, 43, 67, 204, 42, 67, 204, 41, 67, 204, 40, 67, 204, 39, 67, 204, 38, - 67, 204, 37, 67, 204, 36, 67, 204, 35, 67, 205, 32, 67, 205, 31, 67, 205, - 30, 67, 205, 29, 67, 205, 28, 67, 205, 27, 67, 205, 26, 67, 205, 25, 67, - 205, 24, 67, 205, 23, 67, 205, 21, 67, 205, 20, 67, 205, 19, 67, 205, 18, - 67, 205, 17, 67, 205, 16, 67, 205, 15, 67, 205, 14, 67, 205, 13, 67, 205, - 12, 67, 205, 10, 67, 205, 9, 67, 205, 8, 67, 205, 7, 67, 205, 6, 67, 205, - 5, 67, 205, 4, 67, 205, 3, 67, 205, 2, 67, 205, 1, 67, 204, 255, 67, 204, - 254, 67, 204, 253, 67, 204, 252, 67, 204, 251, 67, 204, 250, 67, 204, - 249, 67, 204, 248, 67, 204, 247, 67, 204, 246, 67, 204, 244, 67, 204, - 243, 67, 204, 242, 67, 204, 241, 67, 204, 240, 67, 204, 239, 67, 204, - 238, 67, 204, 237, 67, 204, 236, 67, 204, 235, 67, 204, 233, 67, 204, - 232, 67, 204, 231, 67, 204, 230, 67, 204, 229, 67, 204, 228, 67, 204, - 227, 67, 204, 226, 67, 204, 225, 67, 204, 224, 67, 204, 222, 67, 204, - 221, 67, 204, 220, 67, 204, 219, 67, 204, 218, 67, 204, 217, 67, 204, - 216, 67, 204, 215, 67, 204, 214, 67, 204, 213, 67, 204, 211, 67, 204, - 210, 67, 204, 209, 67, 204, 208, 67, 204, 207, 67, 204, 206, 67, 204, - 205, 67, 204, 204, 67, 204, 203, 67, 204, 202, 67, 204, 200, 67, 204, - 199, 67, 204, 198, 67, 204, 197, 67, 204, 196, 67, 204, 195, 67, 204, - 194, 67, 204, 193, 67, 204, 192, 67, 204, 191, 67, 204, 189, 67, 204, - 188, 67, 204, 187, 67, 204, 186, 67, 204, 185, 67, 204, 184, 67, 204, - 183, 67, 204, 182, 67, 204, 181, 67, 204, 180, 67, 204, 177, 67, 204, - 176, 67, 204, 175, 67, 204, 174, 67, 204, 173, 67, 204, 172, 67, 204, - 171, 67, 204, 170, 67, 204, 169, 67, 204, 168, 67, 204, 166, 67, 204, - 165, 67, 204, 164, 67, 204, 163, 67, 204, 162, 67, 204, 161, 67, 204, - 160, 67, 204, 159, 67, 204, 158, 67, 204, 157, 67, 204, 155, 67, 204, - 154, 67, 204, 153, 67, 204, 152, 67, 204, 151, 67, 204, 150, 67, 204, - 149, 67, 204, 148, 67, 204, 147, 67, 204, 146, 67, 204, 144, 67, 204, - 143, 67, 204, 142, 67, 204, 141, 67, 204, 140, 67, 204, 139, 67, 204, - 138, 67, 204, 137, 67, 204, 136, 67, 204, 135, 67, 204, 133, 67, 204, - 132, 67, 204, 131, 67, 204, 130, 67, 204, 129, 67, 204, 128, 67, 204, - 127, 67, 204, 126, 67, 204, 125, 67, 204, 124, 67, 204, 122, 67, 204, - 121, 67, 204, 120, 67, 204, 119, 67, 204, 118, 67, 204, 117, 67, 204, - 116, 212, 140, 212, 142, 201, 99, 79, 229, 234, 202, 28, 201, 99, 79, - 199, 53, 201, 7, 234, 97, 79, 199, 53, 233, 246, 234, 97, 79, 198, 11, - 234, 59, 234, 83, 234, 84, 252, 53, 252, 54, 251, 191, 248, 240, 249, - 142, 248, 18, 246, 242, 199, 230, 228, 243, 199, 230, 228, 167, 199, 236, - 219, 115, 233, 52, 214, 82, 219, 114, 234, 97, 79, 219, 114, 219, 163, - 213, 107, 234, 62, 219, 115, 199, 230, 81, 199, 230, 193, 251, 232, 148, - 233, 52, 233, 29, 247, 90, 207, 23, 238, 238, 203, 78, 211, 16, 219, 35, - 107, 202, 47, 203, 78, 223, 164, 219, 35, 191, 77, 202, 223, 237, 212, - 219, 105, 234, 16, 236, 187, 237, 77, 239, 24, 107, 237, 201, 237, 77, - 239, 24, 109, 237, 200, 237, 77, 239, 24, 138, 237, 199, 237, 77, 239, - 24, 134, 237, 198, 214, 108, 252, 53, 214, 235, 200, 69, 223, 230, 200, - 73, 234, 97, 79, 198, 12, 248, 131, 233, 254, 247, 128, 247, 130, 234, - 97, 79, 216, 214, 234, 60, 234, 116, 200, 227, 200, 246, 234, 16, 234, - 17, 223, 139, 204, 12, 134, 233, 9, 204, 11, 232, 92, 223, 139, 204, 12, - 138, 230, 189, 204, 11, 230, 186, 223, 139, 204, 12, 109, 207, 100, 204, - 11, 206, 75, 223, 139, 204, 12, 107, 196, 91, 204, 11, 196, 45, 201, 251, - 237, 118, 237, 120, 210, 210, 246, 241, 210, 212, 137, 211, 160, 208, - 222, 228, 246, 248, 44, 210, 3, 229, 194, 248, 60, 213, 46, 248, 44, 229, - 194, 214, 193, 223, 150, 223, 152, 214, 75, 219, 114, 214, 106, 201, 99, - 79, 205, 37, 251, 10, 201, 176, 234, 97, 79, 205, 37, 251, 10, 234, 19, - 246, 242, 199, 231, 203, 253, 228, 243, 199, 231, 203, 253, 228, 164, - 246, 242, 199, 231, 4, 222, 166, 228, 243, 199, 231, 4, 222, 166, 228, - 165, 219, 115, 199, 231, 203, 253, 81, 199, 231, 203, 253, 193, 250, 210, - 107, 219, 115, 232, 134, 210, 107, 219, 115, 235, 125, 209, 96, 210, 107, - 219, 115, 249, 141, 210, 107, 219, 115, 196, 77, 209, 90, 207, 20, 219, - 115, 233, 52, 207, 20, 223, 150, 207, 2, 202, 171, 203, 78, 109, 202, - 168, 201, 178, 202, 171, 203, 78, 138, 202, 167, 201, 177, 237, 77, 239, - 24, 201, 31, 237, 196, 208, 207, 196, 44, 107, 208, 207, 196, 42, 208, - 166, 208, 207, 196, 44, 109, 208, 207, 196, 41, 208, 165, 203, 254, 198, - 10, 201, 96, 201, 14, 247, 129, 246, 241, 247, 63, 216, 171, 193, 171, - 215, 83, 201, 99, 79, 230, 174, 251, 10, 201, 99, 79, 208, 184, 251, 10, - 201, 250, 234, 97, 79, 230, 174, 251, 10, 234, 97, 79, 208, 184, 251, 10, - 234, 57, 201, 99, 79, 201, 31, 202, 10, 202, 171, 230, 218, 246, 242, - 223, 98, 203, 171, 202, 171, 246, 242, 223, 98, 205, 86, 239, 24, 204, 8, - 223, 98, 238, 198, 201, 32, 199, 80, 201, 119, 211, 70, 200, 58, 242, 75, - 211, 36, 208, 208, 216, 170, 209, 78, 251, 47, 208, 200, 242, 75, 251, - 64, 214, 181, 202, 232, 8, 6, 1, 231, 93, 8, 2, 1, 231, 93, 247, 6, 9, 2, - 137, 34, 131, 4, 99, 249, 82, 251, 168, 200, 63, 200, 233, 242, 86, 202, - 111, 219, 226, 222, 83, 1, 219, 64, 220, 19, 1, 232, 178, 232, 168, 220, - 19, 1, 232, 178, 233, 64, 220, 19, 1, 206, 163, 220, 19, 1, 219, 45, 86, - 87, 248, 143, 203, 51, 231, 56, 216, 120, 207, 10, 30, 125, 192, 54, 30, - 125, 192, 50, 30, 125, 201, 154, 30, 125, 192, 55, 232, 68, 232, 67, 232, - 66, 215, 85, 232, 65, 200, 197, 1, 251, 16, 68, 190, 232, 190, 233, 190, - 235, 218, 231, 206, 171, 218, 233, 206, 173, 210, 68, 218, 230, 206, 170, - 213, 77, 216, 18, 193, 50, 218, 232, 206, 172, 232, 91, 210, 67, 193, - 111, 234, 121, 232, 78, 216, 94, 211, 107, 196, 46, 113, 216, 94, 237, - 218, 113, 118, 197, 240, 64, 4, 55, 81, 106, 96, 197, 240, 64, 4, 55, 81, - 106, 11, 5, 223, 53, 77, 80, 1, 221, 208, 219, 75, 194, 251, 194, 140, - 194, 72, 194, 61, 194, 50, 194, 39, 194, 28, 194, 17, 194, 6, 194, 250, - 194, 239, 194, 228, 194, 217, 194, 206, 194, 195, 194, 184, 208, 223, - 232, 148, 40, 81, 50, 63, 219, 189, 248, 55, 247, 200, 211, 53, 77, 248, - 102, 190, 234, 10, 3, 212, 150, 199, 84, 10, 3, 212, 150, 139, 212, 150, - 247, 233, 139, 247, 232, 216, 220, 6, 1, 230, 118, 216, 220, 6, 1, 214, - 72, 216, 220, 2, 1, 230, 118, 216, 220, 2, 1, 214, 72, 61, 1, 235, 16, - 73, 37, 16, 232, 90, 202, 107, 243, 54, 195, 164, 194, 173, 194, 162, - 194, 151, 194, 139, 194, 128, 194, 117, 194, 106, 194, 95, 194, 84, 194, - 76, 194, 75, 194, 74, 194, 73, 194, 71, 194, 70, 194, 69, 194, 68, 194, - 67, 194, 66, 194, 65, 194, 64, 194, 63, 194, 62, 194, 60, 194, 59, 194, - 58, 194, 57, 194, 56, 194, 55, 194, 54, 194, 53, 194, 52, 194, 51, 194, - 49, 194, 48, 194, 47, 194, 46, 194, 45, 194, 44, 194, 43, 194, 42, 194, - 41, 194, 40, 194, 38, 194, 37, 194, 36, 194, 35, 194, 34, 194, 33, 194, - 32, 194, 31, 194, 30, 194, 29, 194, 27, 194, 26, 194, 25, 194, 24, 194, - 23, 194, 22, 194, 21, 194, 20, 194, 19, 194, 18, 194, 16, 194, 15, 194, - 14, 194, 13, 194, 12, 194, 11, 194, 10, 194, 9, 194, 8, 194, 7, 194, 5, - 194, 4, 194, 3, 194, 2, 194, 1, 194, 0, 193, 255, 193, 254, 193, 253, - 193, 252, 194, 249, 194, 248, 194, 247, 194, 246, 194, 245, 194, 244, - 194, 243, 194, 242, 194, 241, 194, 240, 194, 238, 194, 237, 194, 236, - 194, 235, 194, 234, 194, 233, 194, 232, 194, 231, 194, 230, 194, 229, - 194, 227, 194, 226, 194, 225, 194, 224, 194, 223, 194, 222, 194, 221, - 194, 220, 194, 219, 194, 218, 194, 216, 194, 215, 194, 214, 194, 213, - 194, 212, 194, 211, 194, 210, 194, 209, 194, 208, 194, 207, 194, 205, - 194, 204, 194, 203, 194, 202, 194, 201, 194, 200, 194, 199, 194, 198, - 194, 197, 194, 196, 194, 194, 194, 193, 194, 192, 194, 191, 194, 190, - 194, 189, 194, 188, 194, 187, 194, 186, 194, 185, 194, 183, 194, 182, - 194, 181, 194, 180, 194, 179, 194, 178, 194, 177, 194, 176, 194, 175, - 194, 174, 194, 172, 194, 171, 194, 170, 194, 169, 194, 168, 194, 167, - 194, 166, 194, 165, 194, 164, 194, 163, 194, 161, 194, 160, 194, 159, - 194, 158, 194, 157, 194, 156, 194, 155, 194, 154, 194, 153, 194, 152, - 194, 150, 194, 149, 194, 148, 194, 147, 194, 146, 194, 145, 194, 144, - 194, 143, 194, 142, 194, 141, 194, 138, 194, 137, 194, 136, 194, 135, - 194, 134, 194, 133, 194, 132, 194, 131, 194, 130, 194, 129, 194, 127, - 194, 126, 194, 125, 194, 124, 194, 123, 194, 122, 194, 121, 194, 120, - 194, 119, 194, 118, 194, 116, 194, 115, 194, 114, 194, 113, 194, 112, - 194, 111, 194, 110, 194, 109, 194, 108, 194, 107, 194, 105, 194, 104, - 194, 103, 194, 102, 194, 101, 194, 100, 194, 99, 194, 98, 194, 97, 194, - 96, 194, 94, 194, 93, 194, 92, 194, 91, 194, 90, 194, 89, 194, 88, 194, - 87, 194, 86, 194, 85, 194, 83, 194, 82, 194, 81, 194, 80, 194, 79, 194, - 78, 194, 77, 221, 221, 31, 56, 221, 221, 250, 195, 221, 221, 17, 191, 77, - 221, 221, 17, 107, 221, 221, 17, 109, 221, 221, 17, 138, 221, 221, 17, - 134, 221, 221, 17, 150, 221, 221, 17, 169, 221, 221, 17, 175, 221, 221, - 17, 171, 221, 221, 17, 178, 8, 6, 1, 42, 4, 217, 149, 23, 230, 212, 8, 2, - 1, 42, 4, 217, 149, 23, 230, 212, 8, 6, 1, 228, 76, 4, 217, 149, 23, 230, - 212, 8, 2, 1, 228, 76, 4, 217, 149, 23, 230, 212, 8, 6, 1, 126, 4, 217, - 149, 23, 230, 212, 8, 2, 1, 126, 4, 217, 149, 23, 230, 212, 8, 6, 1, 235, - 17, 4, 81, 219, 115, 60, 8, 2, 1, 235, 17, 4, 81, 219, 115, 60, 8, 6, 1, - 235, 17, 4, 81, 219, 115, 248, 235, 23, 230, 212, 8, 2, 1, 235, 17, 4, - 81, 219, 115, 248, 235, 23, 230, 212, 8, 6, 1, 235, 17, 4, 81, 219, 115, - 248, 235, 23, 252, 48, 8, 2, 1, 235, 17, 4, 81, 219, 115, 248, 235, 23, - 252, 48, 8, 6, 1, 187, 4, 81, 219, 115, 60, 8, 2, 1, 187, 4, 81, 219, - 115, 60, 8, 6, 1, 187, 4, 81, 219, 115, 248, 235, 23, 230, 212, 8, 2, 1, - 187, 4, 81, 219, 115, 248, 235, 23, 230, 212, 8, 6, 1, 187, 4, 81, 219, - 115, 248, 235, 23, 252, 48, 8, 2, 1, 187, 4, 81, 219, 115, 248, 235, 23, - 252, 48, 8, 6, 1, 206, 10, 4, 81, 219, 115, 60, 8, 2, 1, 206, 10, 4, 81, - 219, 115, 60, 8, 6, 1, 235, 17, 4, 243, 4, 23, 217, 148, 8, 2, 1, 235, - 17, 4, 243, 4, 23, 217, 148, 8, 6, 1, 235, 17, 4, 243, 4, 23, 247, 94, 8, - 2, 1, 235, 17, 4, 243, 4, 23, 247, 94, 8, 2, 1, 228, 76, 4, 75, 93, 23, - 252, 48, 8, 2, 1, 214, 73, 4, 198, 153, 58, 8, 6, 1, 42, 4, 211, 141, 23, - 252, 48, 8, 2, 1, 42, 4, 211, 141, 23, 252, 48, 8, 6, 1, 42, 4, 211, 141, - 23, 198, 152, 8, 2, 1, 42, 4, 211, 141, 23, 198, 152, 8, 6, 1, 235, 17, - 4, 211, 141, 23, 252, 48, 8, 2, 1, 235, 17, 4, 211, 141, 23, 252, 48, 8, - 6, 1, 235, 17, 4, 211, 141, 23, 198, 152, 8, 2, 1, 235, 17, 4, 211, 141, - 23, 198, 152, 8, 6, 1, 235, 17, 4, 75, 93, 23, 252, 48, 8, 2, 1, 235, 17, - 4, 75, 93, 23, 252, 48, 8, 6, 1, 235, 17, 4, 75, 93, 23, 198, 152, 8, 2, - 1, 235, 17, 4, 75, 93, 23, 198, 152, 8, 2, 1, 228, 76, 4, 75, 93, 23, - 230, 212, 8, 2, 1, 228, 76, 4, 75, 93, 23, 198, 152, 8, 6, 1, 228, 76, 4, - 211, 141, 23, 252, 48, 8, 2, 1, 228, 76, 4, 211, 141, 23, 75, 93, 23, - 252, 48, 8, 6, 1, 228, 76, 4, 211, 141, 23, 198, 152, 8, 2, 1, 228, 76, - 4, 211, 141, 23, 75, 93, 23, 198, 152, 8, 6, 1, 223, 38, 4, 198, 152, 8, - 2, 1, 223, 38, 4, 75, 93, 23, 198, 152, 8, 6, 1, 220, 145, 4, 198, 152, - 8, 2, 1, 220, 145, 4, 198, 152, 8, 6, 1, 218, 171, 4, 198, 152, 8, 2, 1, - 218, 171, 4, 198, 152, 8, 6, 1, 207, 224, 4, 198, 152, 8, 2, 1, 207, 224, - 4, 198, 152, 8, 6, 1, 126, 4, 211, 141, 23, 252, 48, 8, 2, 1, 126, 4, - 211, 141, 23, 252, 48, 8, 6, 1, 126, 4, 211, 141, 23, 198, 152, 8, 2, 1, - 126, 4, 211, 141, 23, 198, 152, 8, 6, 1, 126, 4, 217, 149, 23, 252, 48, - 8, 2, 1, 126, 4, 217, 149, 23, 252, 48, 8, 6, 1, 126, 4, 217, 149, 23, - 198, 152, 8, 2, 1, 126, 4, 217, 149, 23, 198, 152, 8, 2, 1, 252, 28, 4, - 230, 212, 8, 2, 1, 211, 79, 187, 4, 230, 212, 8, 2, 1, 211, 79, 187, 4, - 252, 48, 8, 2, 1, 154, 196, 13, 4, 230, 212, 8, 2, 1, 154, 196, 13, 4, - 252, 48, 8, 2, 1, 205, 88, 4, 230, 212, 8, 2, 1, 205, 88, 4, 252, 48, 8, - 2, 1, 228, 252, 205, 88, 4, 230, 212, 8, 2, 1, 228, 252, 205, 88, 4, 252, - 48, 9, 204, 8, 99, 4, 230, 60, 93, 4, 251, 194, 9, 204, 8, 99, 4, 230, - 60, 93, 4, 193, 133, 9, 204, 8, 99, 4, 230, 60, 93, 4, 131, 217, 101, 9, - 204, 8, 99, 4, 230, 60, 93, 4, 211, 153, 9, 204, 8, 99, 4, 230, 60, 93, - 4, 66, 9, 204, 8, 99, 4, 230, 60, 93, 4, 191, 225, 9, 204, 8, 99, 4, 230, - 60, 93, 4, 71, 9, 204, 8, 99, 4, 230, 60, 93, 4, 252, 27, 9, 204, 8, 213, - 27, 4, 222, 6, 100, 204, 8, 40, 1, 208, 98, 100, 204, 8, 40, 1, 221, 195, - 100, 204, 8, 40, 1, 231, 68, 100, 204, 8, 40, 1, 191, 123, 100, 204, 8, - 40, 1, 237, 182, 100, 204, 8, 40, 1, 207, 7, 100, 204, 8, 40, 1, 233, - 111, 100, 204, 8, 40, 1, 191, 175, 248, 227, 204, 8, 40, 1, 206, 110, - 248, 227, 204, 8, 40, 1, 207, 7, 248, 227, 204, 8, 40, 1, 191, 175, 230, - 146, 204, 8, 40, 1, 219, 75, 230, 146, 204, 8, 40, 1, 203, 166, 230, 146, - 204, 8, 40, 1, 221, 195, 230, 146, 204, 8, 40, 1, 231, 68, 230, 146, 204, - 8, 40, 1, 191, 123, 230, 146, 204, 8, 40, 1, 233, 111, 211, 47, 204, 8, - 40, 1, 206, 110, 211, 47, 204, 8, 40, 1, 207, 7, 248, 227, 1, 221, 189, - 44, 120, 222, 154, 44, 120, 214, 72, 44, 120, 247, 195, 44, 120, 212, - 105, 44, 120, 197, 135, 44, 120, 213, 82, 44, 120, 200, 43, 44, 120, 215, - 63, 44, 120, 210, 238, 44, 120, 218, 170, 44, 120, 192, 159, 44, 120, - 146, 44, 120, 172, 44, 120, 196, 12, 44, 120, 219, 65, 44, 120, 219, 76, - 44, 120, 206, 111, 44, 120, 213, 64, 44, 120, 223, 37, 44, 120, 203, 168, - 44, 120, 201, 179, 44, 120, 206, 9, 44, 120, 230, 118, 44, 120, 220, 249, - 44, 5, 222, 129, 44, 5, 221, 168, 44, 5, 221, 147, 44, 5, 220, 234, 44, - 5, 220, 189, 44, 5, 222, 24, 44, 5, 222, 15, 44, 5, 222, 104, 44, 5, 221, - 69, 44, 5, 221, 43, 44, 5, 222, 44, 44, 5, 214, 69, 44, 5, 214, 18, 44, - 5, 214, 14, 44, 5, 213, 239, 44, 5, 213, 230, 44, 5, 214, 57, 44, 5, 214, - 55, 44, 5, 214, 66, 44, 5, 213, 251, 44, 5, 213, 246, 44, 5, 214, 59, 44, - 5, 247, 161, 44, 5, 243, 31, 44, 5, 243, 21, 44, 5, 238, 197, 44, 5, 238, - 155, 44, 5, 247, 44, 44, 5, 247, 36, 44, 5, 247, 150, 44, 5, 242, 101, - 44, 5, 239, 20, 44, 5, 247, 78, 44, 5, 212, 102, 44, 5, 212, 83, 44, 5, - 212, 77, 44, 5, 212, 60, 44, 5, 212, 52, 44, 5, 212, 92, 44, 5, 212, 91, - 44, 5, 212, 99, 44, 5, 212, 67, 44, 5, 212, 64, 44, 5, 212, 95, 44, 5, - 197, 131, 44, 5, 197, 111, 44, 5, 197, 110, 44, 5, 197, 99, 44, 5, 197, - 96, 44, 5, 197, 127, 44, 5, 197, 126, 44, 5, 197, 130, 44, 5, 197, 109, - 44, 5, 197, 108, 44, 5, 197, 129, 44, 5, 213, 80, 44, 5, 213, 66, 44, 5, - 213, 65, 44, 5, 213, 49, 44, 5, 213, 48, 44, 5, 213, 76, 44, 5, 213, 75, - 44, 5, 213, 79, 44, 5, 213, 51, 44, 5, 213, 50, 44, 5, 213, 78, 44, 5, - 199, 245, 44, 5, 198, 193, 44, 5, 198, 170, 44, 5, 197, 94, 44, 5, 197, - 49, 44, 5, 199, 145, 44, 5, 199, 121, 44, 5, 199, 217, 44, 5, 159, 44, 5, - 198, 59, 44, 5, 199, 166, 44, 5, 214, 252, 44, 5, 213, 221, 44, 5, 213, - 188, 44, 5, 212, 180, 44, 5, 212, 117, 44, 5, 214, 123, 44, 5, 214, 112, - 44, 5, 214, 238, 44, 5, 213, 45, 44, 5, 213, 28, 44, 5, 214, 207, 44, 5, - 210, 222, 44, 5, 209, 187, 44, 5, 209, 147, 44, 5, 208, 167, 44, 5, 208, - 130, 44, 5, 210, 65, 44, 5, 210, 51, 44, 5, 210, 200, 44, 5, 209, 75, 44, - 5, 209, 39, 44, 5, 210, 82, 44, 5, 217, 153, 44, 5, 216, 102, 44, 5, 216, - 63, 44, 5, 215, 157, 44, 5, 215, 95, 44, 5, 216, 234, 44, 5, 216, 213, - 44, 5, 217, 114, 44, 5, 216, 14, 44, 5, 215, 213, 44, 5, 217, 27, 44, 5, - 192, 140, 44, 5, 192, 33, 44, 5, 192, 23, 44, 5, 191, 225, 44, 5, 191, - 188, 44, 5, 192, 80, 44, 5, 192, 77, 44, 5, 192, 119, 44, 5, 192, 12, 44, - 5, 191, 246, 44, 5, 192, 91, 44, 5, 207, 180, 44, 5, 207, 2, 44, 5, 206, - 196, 44, 5, 206, 69, 44, 5, 206, 30, 44, 5, 207, 115, 44, 5, 207, 86, 44, - 5, 207, 158, 44, 5, 206, 163, 44, 5, 206, 135, 44, 5, 207, 127, 44, 5, - 220, 127, 44, 5, 219, 148, 44, 5, 219, 130, 44, 5, 218, 227, 44, 5, 218, - 196, 44, 5, 219, 240, 44, 5, 219, 230, 44, 5, 220, 98, 44, 5, 219, 45, - 44, 5, 219, 10, 44, 5, 220, 2, 44, 5, 195, 187, 44, 5, 195, 69, 44, 5, - 195, 51, 44, 5, 193, 249, 44, 5, 193, 241, 44, 5, 195, 153, 44, 5, 195, - 148, 44, 5, 195, 183, 44, 5, 195, 24, 44, 5, 195, 8, 44, 5, 195, 160, 44, - 5, 219, 63, 44, 5, 219, 58, 44, 5, 219, 57, 44, 5, 219, 54, 44, 5, 219, - 53, 44, 5, 219, 60, 44, 5, 219, 59, 44, 5, 219, 62, 44, 5, 219, 56, 44, - 5, 219, 55, 44, 5, 219, 61, 44, 5, 219, 74, 44, 5, 219, 67, 44, 5, 219, - 66, 44, 5, 219, 50, 44, 5, 219, 49, 44, 5, 219, 70, 44, 5, 219, 69, 44, - 5, 219, 73, 44, 5, 219, 52, 44, 5, 219, 51, 44, 5, 219, 71, 44, 5, 206, - 109, 44, 5, 206, 98, 44, 5, 206, 97, 44, 5, 206, 90, 44, 5, 206, 83, 44, - 5, 206, 105, 44, 5, 206, 104, 44, 5, 206, 108, 44, 5, 206, 96, 44, 5, - 206, 95, 44, 5, 206, 107, 44, 5, 213, 62, 44, 5, 213, 57, 44, 5, 213, 56, - 44, 5, 213, 53, 44, 5, 213, 52, 44, 5, 213, 59, 44, 5, 213, 58, 44, 5, - 213, 61, 44, 5, 213, 55, 44, 5, 213, 54, 44, 5, 213, 60, 44, 5, 223, 33, - 44, 5, 222, 246, 44, 5, 222, 238, 44, 5, 222, 184, 44, 5, 222, 164, 44, - 5, 223, 12, 44, 5, 223, 10, 44, 5, 223, 27, 44, 5, 222, 203, 44, 5, 222, - 193, 44, 5, 223, 19, 44, 5, 203, 161, 44, 5, 203, 82, 44, 5, 203, 77, 44, - 5, 203, 6, 44, 5, 202, 244, 44, 5, 203, 114, 44, 5, 203, 112, 44, 5, 203, - 149, 44, 5, 203, 57, 44, 5, 203, 49, 44, 5, 203, 123, 44, 5, 201, 175, - 44, 5, 201, 143, 44, 5, 201, 139, 44, 5, 201, 130, 44, 5, 201, 127, 44, - 5, 201, 149, 44, 5, 201, 148, 44, 5, 201, 174, 44, 5, 201, 135, 44, 5, - 201, 134, 44, 5, 201, 151, 44, 5, 205, 198, 44, 5, 202, 223, 44, 5, 202, - 194, 44, 5, 201, 5, 44, 5, 200, 160, 44, 5, 205, 69, 44, 5, 205, 51, 44, - 5, 205, 182, 44, 5, 202, 47, 44, 5, 202, 17, 44, 5, 205, 115, 44, 5, 230, - 93, 44, 5, 229, 160, 44, 5, 229, 132, 44, 5, 228, 161, 44, 5, 228, 130, - 44, 5, 229, 247, 44, 5, 229, 217, 44, 5, 230, 82, 44, 5, 229, 25, 44, 5, - 228, 254, 44, 5, 230, 4, 44, 5, 220, 248, 44, 5, 220, 247, 44, 5, 220, - 242, 44, 5, 220, 241, 44, 5, 220, 238, 44, 5, 220, 237, 44, 5, 220, 244, - 44, 5, 220, 243, 44, 5, 220, 246, 44, 5, 220, 240, 44, 5, 220, 239, 44, - 5, 220, 245, 44, 5, 203, 15, 166, 120, 3, 192, 105, 166, 120, 3, 207, - 146, 166, 120, 3, 207, 51, 101, 1, 196, 224, 95, 120, 3, 242, 93, 155, - 95, 120, 3, 242, 93, 221, 217, 95, 120, 3, 242, 93, 221, 69, 95, 120, 3, - 242, 93, 221, 185, 95, 120, 3, 242, 93, 213, 251, 95, 120, 3, 242, 93, - 247, 162, 95, 120, 3, 242, 93, 247, 3, 95, 120, 3, 242, 93, 242, 101, 95, - 120, 3, 242, 93, 243, 70, 95, 120, 3, 242, 93, 212, 67, 95, 120, 3, 242, - 93, 238, 34, 95, 120, 3, 242, 93, 197, 120, 95, 120, 3, 242, 93, 236, - 176, 95, 120, 3, 242, 93, 197, 115, 95, 120, 3, 242, 93, 181, 95, 120, 3, - 242, 93, 190, 190, 95, 120, 3, 242, 93, 199, 49, 95, 120, 3, 242, 93, - 159, 95, 120, 3, 242, 93, 198, 241, 95, 120, 3, 242, 93, 213, 45, 95, - 120, 3, 242, 93, 249, 155, 95, 120, 3, 242, 93, 209, 230, 95, 120, 3, - 242, 93, 209, 75, 95, 120, 3, 242, 93, 209, 201, 95, 120, 3, 242, 93, - 216, 14, 95, 120, 3, 242, 93, 192, 12, 95, 120, 3, 242, 93, 206, 163, 95, - 120, 3, 242, 93, 219, 45, 95, 120, 3, 242, 93, 195, 24, 95, 120, 3, 242, - 93, 203, 166, 95, 120, 3, 242, 93, 201, 176, 95, 120, 3, 242, 93, 188, - 95, 120, 3, 242, 93, 140, 95, 120, 3, 242, 93, 173, 95, 18, 3, 242, 93, - 208, 98, 95, 223, 151, 18, 3, 242, 93, 208, 35, 95, 223, 151, 18, 3, 242, - 93, 206, 18, 95, 223, 151, 18, 3, 242, 93, 206, 11, 95, 223, 151, 18, 3, - 242, 93, 208, 77, 95, 18, 3, 211, 115, 95, 18, 3, 252, 169, 229, 122, 1, - 248, 183, 214, 70, 229, 122, 1, 248, 183, 214, 18, 229, 122, 1, 248, 183, - 213, 239, 229, 122, 1, 248, 183, 214, 57, 229, 122, 1, 248, 183, 213, - 251, 72, 1, 248, 183, 214, 70, 72, 1, 248, 183, 214, 18, 72, 1, 248, 183, - 213, 239, 72, 1, 248, 183, 214, 57, 72, 1, 248, 183, 213, 251, 72, 1, - 251, 230, 247, 44, 72, 1, 251, 230, 197, 94, 72, 1, 251, 230, 159, 72, 1, - 251, 230, 210, 238, 52, 1, 233, 202, 233, 201, 239, 28, 163, 164, 52, 1, - 233, 201, 233, 202, 239, 28, 163, 164, +static const unsigned int dawg_pos_to_codepoint[] = { + 8448, 129687, 983053, 983052, 180, 10655, 8301, 8299, 9190, 125185, + 125197, 125214, 125208, 125204, 125187, 125201, 125216, 125213, 125200, + 125211, 125209, 125189, 125217, 125207, 125186, 125205, 125215, 125184, + 125191, 125188, 125198, 125202, 125199, 125210, 125190, 125194, 125206, + 125192, 125212, 125193, 125195, 125196, 125203, 125256, 125219, 125231, + 125248, 125242, 125238, 125221, 125235, 125250, 125247, 125234, 125245, + 125243, 125223, 125251, 125241, 125220, 125239, 125249, 125218, 125225, + 125222, 125232, 125236, 125233, 125244, 125224, 125228, 125240, 125226, + 125246, 125227, 125229, 125230, 125237, 125257, 125254, 125278, 125279, + 125259, 125258, 125255, 125252, 125253, 125269, 125268, 125271, 125270, + 125267, 125266, 125264, 125273, 125265, 125272, 11237, 127903, 129657, + 8449, 129489, 9772, 65852, 65853, 65854, 65855, 65848, 65851, 65849, + 65850, 65847, 65792, 65793, 65806, 65815, 65842, 65824, 65833, 65812, + 65839, 65803, 65821, 65830, 65811, 65838, 65802, 65820, 65829, 65807, + 65816, 65843, 65825, 65834, 65805, 65814, 65841, 65823, 65832, 65804, + 65813, 65840, 65822, 65831, 65810, 65837, 65801, 65819, 65828, 65809, + 65836, 65800, 65818, 65827, 65808, 65835, 65799, 65817, 65826, 65794, + 128673, 71455, 71453, 71454, 71442, 71450, 71446, 71429, 71492, 71491, + 71444, 71443, 71490, 71489, 71436, 71428, 71494, 71438, 71426, 71493, + 71440, 71427, 71448, 71432, 71435, 71488, 71447, 71445, 71449, 71434, + 71425, 71424, 71431, 71430, 71441, 71433, 71437, 71439, 71484, 71485, + 71467, 71486, 71487, 71456, 71457, 71465, 71466, 71463, 71460, 71461, + 71458, 71459, 71462, 71464, 71483, 71482, 71477, 71476, 71479, 71478, + 71475, 71474, 71472, 71481, 71473, 71480, 9992, 128747, 128748, 128874, + 128822, 128823, 128837, 128776, 128777, 128774, 128775, 128773, 128811, + 128859, 128829, 128826, 128855, 128769, 128875, 128876, 128830, 128834, + 128835, 128836, 128783, 128857, 128848, 128846, 128844, 128805, 128800, + 128803, 128804, 128798, 128869, 128870, 128871, 128872, 128873, 128787, + 128865, 128866, 128864, 128880, 128794, 128841, 128883, 128882, 128854, + 128878, 128796, 128797, 128801, 128810, 128851, 128784, 128785, 128786, + 128824, 128881, 128789, 128879, 128782, 128843, 128858, 128856, 128868, + 128867, 128863, 128768, 128833, 128818, 128816, 128817, 128799, 128819, + 128820, 128821, 128827, 128828, 128877, 128792, 128793, 128788, 128806, + 128813, 128825, 128850, 128860, 128861, 128814, 128807, 128812, 128802, + 128862, 128781, 128839, 128795, 128852, 128847, 128831, 128832, 128840, + 128809, 128849, 128845, 128778, 128815, 128779, 128780, 128790, 128791, + 128808, 128772, 128842, 128853, 128771, 128838, 128770, 9879, 8501, + 983054, 8780, 9006, 983203, 8776, 10863, 8778, 9095, 9941, 128126, 9200, + 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, 82965, 82966, + 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, 82996, 82987, + 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, 82998, 82999, + 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, 83014, 83015, + 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, 83044, 83045, + 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, 82976, 82977, + 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, 82955, 82956, + 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, 82945, 82946, + 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, 83002, 83003, + 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, 83025, 83026, + 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, 83035, 83036, + 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, 83064, 83065, + 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, 83056, 83057, + 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, 83077, 83078, + 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, 83087, 83088, + 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, 83097, 83098, + 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, 83107, 83108, + 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, 83117, 83118, + 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, 83127, 83128, + 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, 83137, 83138, + 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, 83147, 83148, + 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, 83157, 83158, + 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, 83167, 83168, + 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, 83184, 83171, + 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, 83192, 83193, + 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, 83274, 83275, + 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, 83278, 83279, + 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, 83265, 83266, + 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, 83202, 83203, + 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, 83242, 83243, + 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, 83213, 83214, + 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, 83223, 83224, + 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, 83233, 83234, + 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, 83254, 83255, + 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, 83285, 83286, + 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, 83305, 83306, + 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, 83340, 83341, + 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, 83324, 83325, + 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, 83326, 83327, + 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, 83353, 83356, + 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, 83382, 83373, + 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, 83387, 83388, + 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, 83362, 83365, + 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, 83294, 83295, + 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, 83330, 83331, + 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, 83407, 83408, + 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, 83447, 83448, + 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, 83457, 83458, + 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, 83396, 83397, + 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, 83417, 83418, + 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, 83427, 83428, + 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, 83437, 83438, + 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, 83459, 83460, + 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, 83469, 83470, + 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, 83479, 83480, + 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, 83526, 83497, + 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, 83506, 83507, + 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, 83516, 83517, + 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, 129728, 8736, + 10654, 10660, 128551, 8491, 128162, 128544, 128028, 11150, 11148, 11149, + 11151, 11119, 8630, 10560, 8755, 128260, 10226, 8634, 10769, 128246, + 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, 9022, 9066, 9058, 9067, + 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, 9050, 9075, 9080, 9014, + 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, 9109, 9020, 9056, 9017, + 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, 9031, 9072, 9071, 9016, + 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, 9076, 9070, 9023, 9059, + 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, 9053, 9039, 9065, 9064, + 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, 983195, 983196, 2277, + 2280, 2276, 2279, 2278, 2281, 1548, 1615, 65144, 65145, 2302, 1612, + 65138, 1549, 2206, 2299, 2300, 2274, 1643, 1771, 1770, 1757, 1565, 1614, + 1630, 2293, 2292, 65142, 65143, 1611, 65136, 1538, 1645, 1748, 2207, + 1621, 1620, 1616, 2294, 65146, 65147, 1613, 65140, 2258, 2255, 2254, + 2257, 2236, 2244, 2235, 2237, 1593, 1886, 2227, 1696, 1887, 1885, 65226, + 65227, 65225, 65228, 1575, 2165, 2176, 2173, 2161, 2174, 2171, 2167, + 2177, 2169, 2166, 2168, 2178, 2164, 2160, 2162, 2175, 2172, 1571, 65156, + 65155, 1573, 65160, 65159, 1651, 1650, 1908, 1907, 1570, 65154, 65153, + 2163, 2170, 1649, 64337, 64336, 1609, 65264, 65263, 65166, 65165, 1749, + 1576, 2230, 1878, 2208, 1874, 1875, 1872, 1876, 1877, 1873, 2209, 65168, + 65169, 65167, 65170, 1664, 64347, 64348, 64346, 64349, 1659, 64339, + 64340, 64338, 64341, 1583, 1674, 1675, 1882, 1774, 1679, 2222, 1881, + 1680, 1673, 65194, 65193, 1676, 64389, 64388, 1590, 1787, 65214, 65215, + 65213, 65216, 1677, 64387, 64386, 1672, 64393, 64392, 1646, 1697, 1647, + 1668, 64371, 64372, 64370, 64373, 1678, 64391, 64390, 1740, 1911, 1910, + 1909, 1599, 1598, 1597, 64509, 64510, 64508, 64511, 1601, 1699, 2212, + 1698, 1889, 1701, 1888, 65234, 65235, 65233, 65236, 1711, 1716, 1714, + 1712, 2224, 64403, 64404, 64402, 64405, 1594, 2243, 1788, 65230, 65231, + 65229, 65232, 1715, 64407, 64408, 64406, 64409, 2248, 1581, 1916, 2186, + 1903, 1906, 1902, 1880, 1669, 1666, 1879, 1665, 65186, 65187, 65185, + 65188, 1569, 65152, 1607, 1729, 1730, 64423, 64424, 64422, 64425, 1791, + 1728, 64421, 64420, 1726, 64427, 64428, 64426, 64429, 65258, 65259, + 65257, 65260, 1652, 1653, 1656, 1654, 1580, 2210, 2246, 2245, 65182, + 65183, 65181, 65184, 1688, 64395, 64394, 1603, 1919, 1710, 2228, 1708, + 1707, 65242, 65243, 65241, 65244, 1568, 1705, 1892, 1596, 1891, 2189, + 1595, 2242, 1890, 64399, 64400, 64398, 64401, 1733, 64481, 64480, 1737, + 64483, 64482, 1582, 65190, 65191, 65189, 65192, 1604, 2214, 1718, 2247, + 1717, 1720, 1719, 1898, 65246, 65247, 65245, 65248, 2221, 1605, 2215, + 1894, 1893, 65250, 65251, 65249, 65252, 1564, 1709, 1713, 64411, 64412, + 64410, 64413, 64468, 64469, 64467, 64470, 1667, 64375, 64376, 64374, + 64377, 1606, 1722, 64415, 64414, 1896, 1897, 1725, 1895, 2185, 1721, + 1724, 65254, 65255, 65253, 65256, 1662, 2231, 2238, 64343, 64344, 64342, + 64345, 1702, 64367, 64368, 64366, 64369, 1602, 2213, 2229, 1703, 1704, + 65238, 65239, 65237, 65240, 1585, 1905, 2233, 1682, 1685, 1883, 1899, + 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, 65198, 65197, 1681, + 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, 1589, 2223, 1694, + 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, 1904, 1691, 1692, + 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, 1786, 65206, 65207, + 65205, 65208, 1648, 2225, 1706, 1591, 2211, 2188, 1695, 2187, 65218, + 65219, 65217, 65220, 1670, 2241, 1727, 64379, 64380, 64378, 64381, 1671, + 64383, 64384, 64382, 64385, 1578, 1577, 65172, 65171, 1731, 65176, 2232, + 2239, 1661, 1660, 65174, 65175, 65173, 1663, 64355, 64356, 64354, 64357, + 1584, 65196, 65195, 1579, 65178, 65179, 65177, 65180, 2182, 1657, 2240, + 64359, 64360, 64358, 64361, 1658, 64351, 64352, 64350, 64353, 1735, 1655, + 64477, 64472, 64471, 64488, 64489, 1739, 1700, 64363, 64364, 64362, + 64365, 64479, 64478, 1608, 2219, 1743, 1913, 1912, 1572, 65158, 65157, + 1738, 1732, 65262, 65261, 1610, 1746, 1915, 1914, 1747, 64433, 64432, + 64431, 64430, 1574, 65162, 65163, 65161, 65164, 1742, 2216, 2234, 2217, + 1741, 1745, 65266, 65267, 65265, 65268, 1736, 64476, 64475, 1734, 64474, + 64473, 1592, 65222, 65223, 65221, 65224, 1586, 65200, 65199, 2226, 1744, + 64485, 64486, 64484, 64487, 2297, 2295, 64888, 64887, 64886, 64950, + 64699, 64554, 64964, 64885, 64698, 64553, 64787, 64759, 64788, 64760, + 64842, 64839, 64841, 64840, 64845, 65015, 64656, 64605, 64828, 64829, + 65010, 65011, 65023, 64962, 64669, 64518, 64672, 64738, 64926, 64670, + 64519, 64622, 64521, 64668, 64517, 64620, 64671, 64520, 64737, 64621, + 64619, 64618, 64623, 64522, 65021, 64878, 64939, 64693, 64547, 64880, + 64879, 64694, 64548, 64803, 64775, 64692, 64546, 64695, 64549, 64812, + 64784, 64804, 64776, 64893, 64892, 64704, 64559, 64636, 64561, 64702, + 64557, 64961, 64705, 64560, 64703, 64558, 64637, 64562, 64889, 64891, + 64890, 64701, 64556, 64789, 64761, 64700, 64555, 64790, 64762, 64859, + 64858, 64682, 64536, 64795, 64767, 64959, 64681, 64535, 64796, 64768, + 64915, 64916, 64728, 64594, 64595, 64596, 64727, 64593, 64729, 64934, + 64958, 64679, 64533, 64857, 64856, 64935, 64933, 64680, 64534, 64797, + 64769, 64798, 64770, 65019, 64643, 64573, 64640, 64567, 64708, 64568, + 64641, 64711, 64571, 64747, 64710, 64570, 64709, 64569, 64963, 64955, + 64951, 64642, 64712, 64572, 64748, 64644, 64574, 64538, 64799, 64771, + 64683, 64537, 64684, 64539, 64800, 64772, 65272, 65271, 65274, 65273, + 65270, 65269, 64646, 64579, 65276, 65275, 64898, 64949, 64896, 64897, + 64714, 64576, 64717, 64899, 64900, 64954, 64956, 64940, 64713, 64575, + 64902, 64901, 64715, 64577, 64904, 64903, 64941, 64645, 64716, 64578, + 64749, 64647, 64580, 64585, 64648, 64905, 64906, 64907, 64719, 64582, + 64910, 64911, 64953, 64720, 64583, 64909, 64914, 64908, 64960, 64718, + 64581, 64945, 64649, 64721, 64584, 64586, 65012, 64918, 64917, 64947, + 64723, 64588, 64726, 64751, 64952, 64957, 64921, 64920, 64919, 64967, + 64722, 64587, 64923, 64922, 64652, 64725, 64590, 64750, 64654, 64591, + 64653, 64651, 64724, 64589, 64650, 64655, 64592, 64895, 64948, 64894, + 64946, 64707, 64564, 64638, 64565, 64706, 64563, 64639, 64566, 65009, + 64843, 64833, 64835, 64836, 64837, 64834, 64832, 64847, 65014, 64604, + 64869, 64868, 64937, 64689, 64544, 64965, 64870, 64691, 64545, 64801, + 64773, 64690, 64811, 64783, 64802, 64774, 65013, 64975, 65008, 65017, + 65018, 64844, 64838, 64860, 64686, 64541, 64821, 64817, 64744, 64862, + 64861, 64685, 64540, 64820, 64936, 64966, 64687, 64542, 64822, 64864, + 64863, 64865, 64867, 64866, 64688, 64543, 64743, 64791, 64763, 64810, + 64782, 64792, 64764, 64606, 64609, 64755, 64607, 64610, 64756, 64611, + 64608, 64754, 64818, 64746, 64872, 64871, 64938, 64806, 64814, 64778, + 64824, 64873, 64805, 64813, 64777, 64823, 64875, 64874, 64877, 64876, + 64808, 64816, 64780, 64745, 64793, 64765, 64807, 64815, 64779, 64825, + 64809, 64781, 64794, 64766, 65022, 64846, 64882, 64881, 64883, 64884, + 64819, 64551, 64826, 64785, 64757, 64696, 64550, 64786, 64758, 64851, + 64850, 64849, 64674, 64524, 64677, 64740, 64930, 64852, 64929, 64675, + 64525, 64928, 64848, 64927, 64673, 64523, 64932, 64853, 64855, 64854, + 64931, 64626, 64676, 64526, 64739, 64628, 64527, 64627, 64625, 64624, + 64629, 64528, 64603, 64529, 64634, 64531, 64632, 64678, 64530, 64741, + 64633, 64631, 64630, 64635, 64532, 64742, 65016, 64661, 64601, 64616, + 64515, 64491, 64490, 64493, 64492, 64503, 64504, 64502, 64667, 64736, + 64664, 64513, 64663, 64512, 64665, 64614, 64666, 64514, 64735, 64615, + 64499, 64498, 64495, 64494, 64617, 64516, 64501, 64500, 64613, 64612, + 64497, 64496, 64942, 64731, 64598, 64734, 64753, 64660, 64658, 64943, + 64730, 64597, 64732, 64599, 64925, 64924, 64944, 64659, 64733, 64600, + 64752, 64657, 64662, 64602, 64506, 64507, 64505, 64697, 64552, 64827, + 2204, 1619, 1624, 2303, 126492, 126494, 126493, 126495, 126644, 126638, + 126641, 126647, 126648, 126646, 126632, 126645, 126630, 126650, 126626, + 126636, 126631, 126625, 126640, 126643, 126633, 126651, 126637, 126635, + 126649, 126627, 126642, 126639, 126629, 126489, 126467, 126518, 126517, + 126516, 126510, 126513, 126503, 126500, 126519, 126506, 126498, 126508, + 126497, 126512, 126505, 126523, 126509, 126507, 126514, 126511, 126521, + 126612, 126606, 126609, 126615, 126592, 126607, 126599, 126596, 126616, + 126614, 126600, 126613, 126598, 126618, 126594, 126604, 126593, 126608, + 126611, 126601, 126619, 126605, 126603, 126617, 126595, 126610, 126597, + 126475, 126704, 126705, 126588, 126590, 126585, 126582, 126568, 126581, + 126580, 126574, 126577, 126567, 126564, 126583, 126570, 126562, 126572, + 126586, 126561, 126576, 126569, 126587, 126573, 126578, 126575, 126484, + 126478, 126481, 126557, 126559, 126553, 126548, 126542, 126545, 126551, + 126530, 126535, 126537, 126555, 126541, 126539, 126546, 126543, 126472, + 126488, 126486, 126485, 126464, 126479, 126487, 126474, 126470, 126490, + 126466, 126476, 126471, 126465, 126480, 126483, 126473, 126491, 126477, + 126482, 126469, 1541, 1536, 2290, 2289, 2288, 1550, 2192, 1769, 2193, + 1642, 2184, 1544, 1629, 2296, 2301, 2298, 1772, 2183, 1623, 983633, + 983635, 983640, 983634, 983638, 983636, 983639, 983637, 983641, 1563, + 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, 1540, 1790, 1789, 1553, + 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, 983202, 1751, 1750, 1753, + 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, 1755, 1557, 2200, 2260, + 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, 1767, 2251, 1760, 1759, + 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, 69375, 2203, 69373, + 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, 1622, 1618, 65150, + 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, 64439, 64438, + 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, 64450, 64447, + 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, 2286, 2283, 2287, + 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, 1567, 1625, 1545, + 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, 1632, 1641, 1633, + 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, 1342, 1361, 1360, + 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, 1343, 1363, 1354, + 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, 1330, 1348, 1350, + 1338, 1334, 1344, 1332, 1365, 1373, 1371, 1372, 1395, 1401, 1390, 1409, + 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, 1412, 1391, 1411, + 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, 1397, 1416, 1410, + 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, 1392, 1380, 1413, + 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, 1423, 1417, 1374, + 129201, 10549, 10548, 129200, 10550, 10551, 129968, 128667, 127912, 9800, + 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, 9954, 8771, 8870, + 128095, 9883, 128663, 127975, 128762, 8371, 127814, 68352, 68353, 68357, + 68355, 68358, 68359, 68356, 68354, 68373, 68374, 68372, 68393, 68405, + 68388, 68387, 68386, 68391, 68390, 68389, 68371, 68370, 68369, 68403, + 68401, 68404, 68399, 68394, 68395, 68378, 68381, 68377, 68366, 68367, + 68362, 68363, 68364, 68365, 68385, 68384, 68380, 68379, 68402, 68400, + 68360, 68361, 68375, 68383, 68376, 68368, 68398, 68392, 68382, 68397, + 68396, 68409, 129361, 1547, 129518, 8525, 9810, 129683, 128118, 128036, + 127868, 128124, 128700, 128281, 128386, 11101, 11099, 983056, 10155, + 128043, 129363, 127992, 129441, 129366, 129391, 128708, 7005, 7007, 7006, + 6917, 6918, 6987, 6988, 6928, 6953, 6954, 6936, 6937, 6948, 6944, 6943, + 6949, 6984, 6927, 6933, 6934, 6919, 6920, 6929, 6930, 6921, 6922, 6938, + 6939, 6931, 6981, 6932, 6982, 6958, 6925, 6926, 6950, 6945, 6935, 6940, + 6951, 6952, 6957, 6923, 6924, 6962, 6960, 6961, 6946, 6942, 6941, 6947, + 6983, 6985, 6986, 6963, 6955, 6959, 6956, 7022, 7025, 7021, 7024, 7023, + 7026, 7020, 7019, 7027, 7012, 7013, 7018, 7015, 7017, 7016, 7010, 7014, + 7009, 7011, 7032, 7036, 7033, 7034, 7035, 7031, 7030, 7029, 7028, 7003, + 7038, 7008, 7002, 7037, 6915, 6913, 6912, 6916, 6914, 6964, 6972, 6973, + 6970, 6971, 6968, 6969, 6974, 6975, 6977, 6976, 6965, 6978, 6979, 6966, + 6967, 6980, 7004, 6997, 6996, 6999, 6998, 6995, 6994, 6992, 7001, 6993, + 7000, 127880, 10057, 9744, 128503, 128505, 128499, 128501, 11197, 9745, + 9746, 128502, 128500, 10007, 129526, 129648, 42737, 42736, 42741, 42740, + 42733, 42699, 42713, 42712, 42692, 42706, 42683, 42719, 42734, 42735, + 42682, 42729, 42657, 42725, 42659, 42670, 42718, 42666, 42716, 42701, + 42675, 42671, 42722, 42720, 42727, 42723, 42702, 42726, 42677, 42707, + 42708, 42709, 42686, 42694, 42674, 42731, 42695, 42685, 42684, 42691, + 42673, 42664, 42715, 42703, 92193, 92188, 92161, 92199, 92240, 92179, + 92211, 92219, 92213, 92243, 92207, 92216, 92183, 92184, 92238, 92171, + 92174, 92203, 92186, 92210, 92230, 92175, 92221, 92232, 92246, 92198, + 92214, 92190, 92176, 92197, 92196, 92164, 92245, 92212, 92201, 92160, + 92182, 92173, 92229, 92227, 92180, 92237, 92241, 92223, 92185, 92194, + 92178, 92239, 92225, 92233, 92167, 92191, 92231, 92244, 92204, 92226, + 92236, 92200, 92189, 92187, 92162, 92163, 92169, 92170, 92202, 92205, + 92222, 92168, 92165, 92215, 92224, 92208, 92220, 92235, 92177, 92181, + 92195, 92234, 92209, 92166, 92172, 92206, 92192, 92228, 92217, 92242, + 92218, 92262, 92271, 92272, 92270, 92294, 92253, 92301, 92256, 92273, + 92259, 92251, 92297, 92300, 92296, 92295, 92255, 92252, 92292, 92286, + 92268, 92290, 92281, 92267, 92284, 92287, 92282, 92283, 92277, 92298, + 92276, 92302, 92247, 92299, 92288, 92269, 92260, 92261, 92257, 92274, + 92289, 92263, 92279, 92265, 92266, 92250, 92249, 92285, 92248, 92264, + 92280, 92258, 92254, 92278, 92291, 92293, 92275, 92310, 92320, 92312, + 92394, 92393, 92319, 92384, 92321, 92386, 92361, 92373, 92366, 92328, + 92329, 92357, 92342, 92397, 92365, 92376, 92339, 92382, 92363, 92318, + 92387, 92383, 92315, 92385, 92354, 92311, 92381, 92343, 92326, 92364, + 92324, 92391, 92344, 92390, 92338, 92396, 92308, 92349, 92375, 92378, + 92317, 92331, 92362, 92336, 92341, 92370, 92347, 92307, 92303, 92309, + 92395, 92368, 92356, 92367, 92360, 92389, 92333, 92359, 92374, 92351, + 92325, 92371, 92350, 92345, 92372, 92314, 92340, 92323, 92304, 92313, + 92316, 92398, 92399, 92380, 92330, 92379, 92392, 92335, 92332, 92346, + 92400, 92358, 92334, 92353, 92337, 92306, 92369, 92388, 92322, 92305, + 92327, 92377, 92352, 92348, 92355, 92436, 92517, 92488, 92434, 92415, + 92431, 92454, 92460, 92420, 92489, 92422, 92474, 92479, 92449, 92502, + 92439, 92484, 92495, 92464, 92511, 92406, 92446, 92497, 92426, 92482, + 92448, 92515, 92401, 92478, 92427, 92496, 92458, 92424, 92445, 92404, + 92466, 92444, 92438, 92442, 92441, 92510, 92499, 92425, 92437, 92440, + 92471, 92465, 92409, 92483, 92475, 92467, 92485, 92457, 92432, 92476, + 92435, 92412, 92414, 92430, 92407, 92403, 92405, 92487, 92418, 92486, + 92408, 92447, 92459, 92480, 92472, 92505, 92514, 92450, 92463, 92410, + 92493, 92507, 92503, 92462, 92506, 92443, 92509, 92469, 92461, 92490, + 92512, 92494, 92455, 92417, 92501, 92413, 92508, 92500, 92504, 92473, + 92419, 92498, 92423, 92516, 92428, 92452, 92451, 92481, 92416, 92456, + 92433, 92477, 92492, 92491, 92513, 92411, 92402, 92421, 92453, 92468, + 92470, 92429, 92661, 92626, 92616, 92596, 92612, 92603, 92673, 92651, + 92627, 92597, 92585, 92578, 92615, 92565, 92552, 92602, 92674, 92570, + 92646, 92599, 92532, 92587, 92538, 92620, 92670, 92665, 92575, 92521, + 92633, 92595, 92523, 92556, 92539, 92664, 92653, 92667, 92600, 92542, + 92555, 92668, 92520, 92582, 92591, 92581, 92551, 92654, 92611, 92614, + 92666, 92671, 92617, 92637, 92562, 92518, 92592, 92558, 92528, 92607, + 92535, 92557, 92563, 92550, 92624, 92640, 92657, 92531, 92601, 92553, + 92543, 92619, 92598, 92541, 92544, 92579, 92658, 92554, 92628, 92648, + 92547, 92527, 92545, 92540, 92604, 92622, 92589, 92608, 92583, 92609, + 92662, 92613, 92584, 92625, 92634, 92524, 92536, 92605, 92647, 92548, + 92663, 92593, 92621, 92568, 92610, 92576, 92529, 92618, 92649, 92561, + 92656, 92526, 92655, 92546, 92534, 92560, 92580, 92659, 92660, 92638, + 92571, 92525, 92549, 92559, 92635, 92577, 92530, 92636, 92669, 92572, + 92672, 92537, 92519, 92630, 92586, 92569, 92566, 92573, 92652, 92522, + 92574, 92650, 92533, 92567, 92623, 92606, 92639, 92564, 92590, 92642, + 92643, 92594, 92641, 92645, 92644, 92588, 92629, 92632, 92631, 92710, + 92695, 92694, 92726, 92675, 92719, 92677, 92718, 92682, 92717, 92689, + 92720, 92724, 92685, 92722, 92723, 92711, 92712, 92698, 92688, 92697, + 92696, 92702, 92687, 92704, 92681, 92708, 92703, 92706, 92714, 92709, + 92679, 92721, 92684, 92683, 92707, 92691, 92713, 92700, 92693, 92727, + 92690, 92692, 92686, 92680, 92725, 92699, 92701, 92705, 92716, 92728, + 92678, 92715, 92676, 42693, 42698, 42711, 42696, 42667, 42717, 42704, + 42661, 42721, 42669, 42668, 42705, 42700, 42680, 42678, 42710, 42688, + 42681, 42732, 42676, 42679, 42730, 42728, 42672, 42662, 42724, 42687, + 42689, 42690, 42697, 42714, 42660, 42656, 42665, 42663, 42658, 42738, + 42742, 42739, 42743, 127974, 128183, 128180, 128182, 128181, 127820, + 129685, 128202, 129532, 128136, 129530, 127936, 92916, 92912, 92915, + 92913, 92914, 92887, 92894, 92908, 92880, 92907, 92893, 92886, 92888, + 92881, 92906, 92896, 92891, 92902, 92900, 92885, 92890, 92884, 92904, + 92905, 92899, 92889, 92897, 92892, 92895, 92882, 92898, 92883, 92901, + 92903, 92909, 92917, 9918, 129415, 7152, 7153, 7124, 7108, 7114, 7130, + 7139, 7127, 7138, 7133, 7136, 7113, 7111, 7117, 7119, 7107, 7135, 7125, + 7112, 7123, 7129, 7116, 7132, 7105, 7126, 7128, 7110, 7109, 7137, 7121, + 7118, 7106, 7120, 7134, 7122, 7115, 7131, 7104, 7140, 7141, 7154, 7155, + 7167, 7165, 7166, 7164, 7142, 7150, 7151, 7144, 7147, 7149, 7143, 7145, + 7146, 7148, 128704, 128705, 128267, 127900, 127901, 9835, 9836, 129492, + 128059, 127958, 128147, 129451, 129752, 127866, 129714, 983055, 128276, + 129745, 128277, 9086, 128718, 2519, 2557, 2432, 2548, 2552, 2551, 2550, + 2549, 2553, 2454, 2510, 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, + 2443, 2528, 2444, 2529, 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, + 2471, 2470, 2464, 2463, 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, + 2462, 2472, 2486, 2487, 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, + 2460, 2475, 2474, 2489, 2482, 2478, 2447, 2451, 2547, 2546, 983651, + 983650, 983652, 2558, 2493, 2434, 2433, 2492, 2509, 2435, 2554, 2494, + 2504, 2508, 2499, 2500, 2530, 2531, 2497, 2498, 2495, 2496, 2503, 2507, + 2539, 2538, 2541, 2540, 2537, 2536, 2534, 2543, 2535, 2542, 2555, 11102, + 127857, 9004, 9187, 8812, 8502, 8757, 129475, 128719, 72710, 72711, + 72712, 72746, 72704, 72705, 72715, 72717, 72731, 72730, 72736, 72735, + 72729, 72728, 72734, 72733, 72708, 72709, 72706, 72707, 72722, 72732, + 72727, 72737, 72747, 72748, 72749, 72741, 72740, 72724, 72723, 72721, + 72720, 72726, 72725, 72719, 72718, 72739, 72738, 72750, 72745, 72742, + 72744, 72743, 72714, 72716, 72801, 72810, 72806, 72797, 72807, 72798, + 72802, 72811, 72800, 72809, 72799, 72808, 72796, 72805, 72804, 72795, + 72803, 72794, 72768, 72765, 72764, 72767, 72766, 72756, 72757, 72758, + 72751, 72761, 72763, 72754, 72755, 72752, 72753, 72760, 72762, 72770, + 72769, 72789, 72788, 72791, 72790, 72787, 72786, 72784, 72793, 72785, + 72792, 72771, 72772, 72773, 72812, 128692, 128690, 10745, 10744, 129506, + 127921, 127874, 128038, 8383, 129766, 9763, 128089, 129452, 9679, 9864, + 10733, 9865, 9210, 11176, 11177, 11178, 11179, 11180, 11182, 11181, + 11183, 9960, 10028, 9821, 129554, 129596, 129609, 129612, 9818, 129551, + 129593, 9822, 129543, 129564, 129585, 129597, 129606, 129555, 129619, + 129617, 129618, 9823, 129556, 129598, 9819, 129552, 129594, 9820, 129553, + 129595, 129575, 129572, 129576, 129577, 129573, 129574, 9827, 9670, + 11201, 10070, 10730, 11230, 9830, 128419, 128899, 9196, 9662, 9660, + 11167, 127778, 9922, 9923, 10047, 9873, 10022, 128447, 128420, 9829, + 11042, 128426, 11052, 10711, 9194, 9198, 128896, 9668, 9666, 9664, 11164, + 8268, 9944, 128412, 9754, 9699, 9698, 10731, 11044, 11035, 9207, 11206, + 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, 128927, + 9726, 9724, 9912, 128392, 9648, 11039, 127986, 9193, 9197, 9654, 9199, + 128898, 11091, 9658, 9656, 10145, 10148, 11166, 8269, 128413, 9755, + 127990, 9644, 11049, 11050, 11089, 9642, 9787, 9632, 11200, 9209, 128306, + 9728, 128369, 9927, 9984, 128900, 128909, 9986, 9751, 9824, 9733, 128919, + 128925, 128908, 9951, 128383, 9742, 9942, 128418, 128897, 9195, 9652, + 9650, 9700, 9701, 11165, 9851, 11054, 9646, 128920, 128926, 11037, 11204, + 10707, 10002, 10067, 8493, 8460, 8465, 8476, 8488, 10164, 10166, 10165, + 9250, 129792, 129794, 129798, 129806, 129821, 129836, 129813, 129844, + 129829, 129802, 129817, 129848, 129832, 129810, 129840, 129825, 129796, + 129804, 129819, 129850, 129834, 129842, 129827, 129800, 129815, 129846, + 129831, 129808, 129838, 129823, 129793, 129797, 129805, 129820, 129851, + 129835, 129812, 129843, 129828, 129801, 129816, 129847, 129809, 129839, + 129824, 129795, 129803, 129818, 129849, 129833, 129811, 129841, 129826, + 129799, 129814, 129845, 129830, 129807, 129837, 129822, 127804, 128033, + 128216, 128153, 129744, 128939, 128951, 128957, 128945, 128902, 128912, + 128932, 128366, 128278, 128209, 128218, 129667, 12731, 12727, 12726, + 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, 12580, 12713, + 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, 12708, 12573, + 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, 12714, 12723, + 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, 12571, 12722, + 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, 12705, 12730, + 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, 12549, 12704, + 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, 12566, 12554, + 12586, 12562, 11211, 11867, 11868, 8993, 9141, 9142, 10555, 9183, 9181, + 8990, 8973, 11812, 8991, 8972, 11813, 9185, 127870, 128144, 127893, + 128335, 129379, 127923, 8904, 10705, 10706, 127993, 9574, 9559, 9556, + 9577, 9565, 9562, 9553, 9580, 9571, 9568, 9552, 9511, 9490, 9503, 9486, + 9537, 9520, 9513, 9489, 9505, 9485, 9543, 9519, 9558, 9555, 9573, 9557, + 9554, 9572, 9592, 9598, 9593, 9599, 9499, 9531, 9495, 9551, 9549, 9595, + 9523, 9491, 9487, 9483, 9481, 9479, 9477, 9475, 9547, 9515, 9507, 9473, + 9594, 9541, 9525, 9517, 9533, 9530, 9522, 9546, 9539, 9582, 9581, 9583, + 9584, 129959, 129964, 129954, 129958, 129965, 129955, 129952, 129956, + 129963, 129960, 129953, 129961, 129957, 129962, 9586, 9585, 9587, 129966, + 9550, 9548, 9591, 9516, 9488, 9484, 9588, 9596, 9482, 9480, 9478, 9476, + 9589, 9597, 9524, 9496, 9492, 9474, 9532, 9508, 9500, 9472, 129967, 9590, + 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, 9498, 9502, + 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, 9576, 9563, + 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, 9501, 9569, + 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, 129460, + 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, 69648, + 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, 69643, + 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, 69678, + 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, 69657, + 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, 69683, + 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, 69718, + 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, 69715, + 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, 69708, + 69744, 69632, 69635, 69634, 69636, 69633, 69700, 69747, 69748, 69689, + 69688, 69699, 69701, 69694, 69695, 69696, 69697, 69692, 69693, 69690, + 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, 69737, + 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, 10271, + 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, 10479, + 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, 10327, + 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, 10251, + 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, 10347, + 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, 10419, + 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, 10371, + 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, 10397, + 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, 10357, + 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, 10309, + 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, 10457, + 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, 10289, + 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, 10401, + 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, 10494, + 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, 10446, + 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, 10278, + 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, 10362, + 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, 10314, + 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, 10386, + 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, 10268, + 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, 10476, + 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, 10324, + 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, 10248, + 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, 10344, + 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, 10416, + 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, 10368, + 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, 128188, + 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, 129483, + 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, 6661, + 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, 6675, + 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, 5962, + 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, 5964, + 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, 128652, + 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, 118939, + 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, 118876, + 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, 118818, + 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, 119020, + 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, 118808, + 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, 118990, + 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, 118918, + 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, 118798, + 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, 118977, + 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, 118980, + 118972, 118983, 983272, 118982, 118974, 118988, 118981, 119000, 119001, + 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, 118853, + 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, 118839, + 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, 119027, + 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, 118858, + 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, 118922, + 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, 118948, + 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, 118950, + 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, 118856, + 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, 118822, + 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, 118841, + 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, 118889, + 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, 118910, + 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, 118792, + 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, 118905, + 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, 118894, + 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, 118908, + 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, 118867, + 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, 129305, + 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, 5148, + 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, 5564, + 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, 5439, + 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, 5673, + 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, 5576, + 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, 5195, + 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, 5663, + 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, 5633, + 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, 5621, + 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, 5720, + 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, 5613, + 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, 5687, + 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, 5738, + 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, 5599, + 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, 5681, + 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, 5732, + 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, 5581, + 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, 5649, + 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, 5268, + 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, 5469, + 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, 5160, + 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, 5459, + 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, 5244, + 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, 5227, + 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, 6333, + 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, 5331, + 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, 5302, + 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, 5475, + 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, 5280, + 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, 72369, + 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, 5523, + 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, 5748, + 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, 5316, + 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, 5312, + 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, 6361, + 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, 5165, + 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, 5176, + 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, 5443, + 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, 5448, + 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, 5569, + 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, 5311, + 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, 5395, + 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, 5254, + 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, 72383, + 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, 5549, + 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, 5477, + 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, 6375, + 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, 5197, + 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, 5490, + 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, 5343, + 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, 5301, + 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, 5440, + 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, 5455, + 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, 5377, + 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, 5271, + 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, 5214, + 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, 5133, + 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, 5534, + 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, 5173, + 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, 5422, + 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, 5423, + 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, 128367, + 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, 8248, 8257, + 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, 66252, + 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, 66243, + 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, 66235, + 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, 66237, + 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, 66228, + 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, + 128569, 128572, 66888, 66864, 66912, 66882, 66873, 66902, 66889, 66890, + 66911, 66891, 66895, 66901, 66881, 66867, 66870, 66868, 66904, 66866, + 66876, 66910, 66879, 66883, 66897, 66915, 66884, 66878, 66885, 66914, + 66903, 66877, 66896, 66909, 66906, 66908, 66899, 66872, 66913, 66874, + 66871, 66875, 66869, 66893, 66887, 66892, 66907, 66900, 66894, 66905, + 66880, 66898, 66865, 66886, 66927, 9761, 127797, 9963, 8373, 184, 65102, + 65098, 8452, 162, 128328, 9907, 9939, 129681, 69908, 69907, 69913, 69912, + 69899, 69909, 69904, 69914, 69906, 69905, 69911, 69910, 69920, 69921, + 69918, 69917, 69901, 69900, 69898, 69897, 69903, 69902, 69896, 69895, + 69956, 69923, 69916, 69915, 69926, 69919, 69922, 69925, 69959, 69924, + 69891, 69894, 69892, 69893, 69888, 69890, 69889, 69952, 69927, 69957, + 69933, 69935, 69930, 69931, 69932, 69958, 69928, 69929, 69934, 69936, + 69939, 69954, 69953, 69947, 69946, 69949, 69948, 69945, 69944, 69942, + 69951, 69943, 69950, 69955, 69940, 69938, 69937, 43587, 43597, 43596, + 43573, 43572, 43574, 43571, 43590, 43586, 43595, 43588, 43585, 43584, + 43594, 43591, 43593, 43589, 43592, 43530, 43531, 43536, 43538, 43537, + 43543, 43544, 43551, 43552, 43548, 43547, 43546, 43553, 43550, 43549, + 43545, 43542, 43541, 43558, 43559, 43520, 43524, 43533, 43532, 43529, + 43528, 43535, 43534, 43527, 43526, 43540, 43539, 43560, 43556, 43555, + 43557, 43554, 43523, 43521, 43525, 43522, 43612, 43614, 43613, 43615, + 43561, 43568, 43569, 43562, 43563, 43567, 43566, 43565, 43570, 43564, + 43605, 43604, 43607, 43606, 43603, 43602, 43600, 43609, 43601, 43608, + 983058, 983140, 983137, 8256, 128200, 128185, 128201, 128638, 129941, + 10003, 129472, 128227, 5084, 5075, 5077, 5079, 5081, 5082, 5083, 5055, + 5037, 5038, 5039, 5040, 5041, 5042, 5054, 5056, 5057, 5058, 5059, 5060, + 5061, 5069, 5068, 5070, 5071, 5072, 5073, 5074, 5085, 5086, 5087, 5088, + 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5076, 5078, 5080, 5030, + 5032, 5033, 5034, 5035, 5036, 5043, 5044, 5045, 5046, 5047, 5048, 5049, + 5050, 5051, 5052, 5053, 5109, 5062, 5063, 5064, 5065, 5066, 5067, 5097, + 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5031, + 5024, 5025, 5026, 5027, 5028, 5029, 43948, 43939, 43941, 43943, 43945, + 43946, 43947, 43919, 43901, 43902, 43903, 43904, 43905, 43906, 43918, + 43920, 43921, 43922, 43923, 43924, 43925, 43933, 43932, 43934, 43935, + 43936, 43937, 43938, 43949, 43950, 43951, 43952, 43953, 43954, 43955, + 43956, 43957, 43958, 43959, 43960, 43940, 43942, 43944, 43894, 43896, + 43897, 43898, 43899, 43900, 43907, 43908, 43909, 43910, 43911, 43912, + 43913, 43914, 43915, 43916, 43917, 5117, 43926, 43927, 43928, 43929, + 43930, 43931, 43961, 43962, 43963, 43964, 43965, 43966, 43967, 5112, + 5113, 5114, 5115, 5116, 43895, 43888, 43889, 43890, 43891, 43892, 43893, + 127800, 127826, 127792, 127937, 129490, 128696, 9767, 128063, 128020, + 9911, 69559, 69553, 69567, 69571, 69556, 69564, 69570, 69552, 69568, + 69555, 69560, 69562, 69557, 69561, 69563, 69554, 69566, 69572, 69558, + 69569, 69565, 69578, 69574, 69575, 69577, 69573, 69579, 69576, 129378, + 127851, 127876, 9962, 127910, 9680, 9682, 10690, 10683, 10691, 9684, + 9683, 9685, 9677, 9681, 10677, 10682, 10684, 127246, 8859, 11199, 10687, + 129282, 129280, 129281, 128320, 9938, 127342, 127341, 127277, 8856, + 10808, 9316, 9315, 9318, 9317, 9314, 9313, 9450, 9320, 9312, 9319, + 127247, 8857, 8861, 12905, 12919, 12904, 12918, 12903, 12917, 12926, + 12909, 12923, 12906, 12920, 12896, 12910, 12900, 12914, 12897, 12911, + 12908, 12922, 12901, 12915, 12899, 12913, 12902, 12916, 12907, 12921, + 12898, 12912, 9097, 10162, 127343, 12975, 127569, 12959, 127568, 12963, + 12951, 12962, 12965, 12973, 12943, 12957, 12935, 12950, 12939, 12932, + 12955, 12931, 12964, 12946, 12869, 12871, 12952, 12966, 12967, 12969, + 12942, 12954, 12938, 12976, 12936, 12948, 12868, 12974, 12961, 12970, + 12968, 12953, 12934, 12972, 12956, 12944, 12870, 12947, 12949, 12971, + 12945, 12933, 12930, 12937, 12929, 12941, 12940, 12960, 12958, 12928, + 127275, 127276, 128712, 13033, 13036, 13034, 13037, 13035, 13013, 13016, + 13014, 13017, 13015, 13038, 13041, 13039, 13042, 13040, 13028, 13031, + 13029, 13032, 13030, 13046, 13049, 13047, 13050, 13048, 13018, 13021, + 13019, 13022, 13020, 13023, 13026, 13024, 13027, 13025, 13051, 13053, + 13052, 13054, 13043, 13045, 13044, 13008, 13011, 13009, 13012, 13010, + 12925, 12924, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, + 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, + 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, + 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, + 9444, 9445, 9446, 9447, 9448, 9449, 10688, 10806, 8854, 12879, 9329, + 9322, 12991, 12876, 9326, 12981, 12875, 12982, 12986, 12985, 12988, + 12987, 12984, 12983, 12990, 12989, 9325, 12878, 9328, 12877, 9327, 9321, + 12872, 12890, 12874, 12891, 12895, 12894, 12978, 12977, 12893, 12892, + 12980, 12979, 9324, 9331, 12873, 12881, 12885, 12884, 12887, 12886, + 12883, 12882, 12889, 12888, 9323, 9330, 10050, 12342, 10679, 10681, 8853, + 8858, 10680, 128981, 9098, 8855, 10686, 10026, 127278, 127245, 8860, + 10678, 10689, 128983, 11198, 94, 10768, 127914, 127961, 127750, 195088, + 195089, 195090, 195091, 195092, 195093, 195094, 195095, 195096, 195097, + 195098, 195099, 195100, 195101, 195072, 195073, 195074, 195075, 195076, + 195077, 195078, 195079, 195080, 195081, 195082, 195083, 195084, 195085, + 195086, 195087, 194560, 194561, 194562, 194563, 194564, 194565, 194566, + 194567, 194568, 194569, 194570, 194571, 194572, 194573, 194574, 194575, + 194576, 194577, 194578, 194579, 194580, 194581, 194582, 194583, 194584, + 194585, 194586, 194587, 194588, 194589, 194590, 194591, 194592, 194593, + 194594, 194595, 194596, 194597, 194598, 194599, 194600, 194601, 194602, + 194603, 194604, 194605, 194606, 194607, 194608, 194609, 194610, 194611, + 194612, 194613, 194614, 194615, 194616, 194617, 194618, 194619, 194620, + 194621, 194622, 194623, 194624, 194625, 194626, 194627, 194628, 194629, + 194630, 194631, 194632, 194633, 194634, 194635, 194636, 194637, 194638, + 194639, 194640, 194641, 194642, 194643, 194644, 194645, 194646, 194647, + 194648, 194649, 194650, 194651, 194652, 194653, 194654, 194655, 194656, + 194657, 194658, 194659, 194660, 194661, 194662, 194663, 194664, 194665, + 194666, 194667, 194668, 194669, 194670, 194671, 194672, 194673, 194674, + 194675, 194676, 194677, 194678, 194679, 194680, 194681, 194682, 194683, + 194684, 194685, 194686, 194687, 194688, 194689, 194690, 194691, 194692, + 194693, 194694, 194695, 194696, 194697, 194698, 194699, 194700, 194701, + 194702, 194703, 194704, 194705, 194706, 194707, 194708, 194709, 194710, + 194711, 194712, 194713, 194714, 194715, 194716, 194717, 194718, 194719, + 194720, 194721, 194722, 194723, 194724, 194725, 194726, 194727, 194728, + 194729, 194730, 194731, 194732, 194733, 194734, 194735, 194736, 194737, + 194738, 194739, 194740, 194741, 194742, 194743, 194744, 194745, 194746, + 194747, 194748, 194749, 194750, 194751, 194752, 194753, 194754, 194755, + 194756, 194757, 194758, 194759, 194760, 194761, 194762, 194763, 194764, + 194765, 194766, 194767, 194768, 194769, 194770, 194771, 194772, 194773, + 194774, 194775, 194776, 194777, 194778, 194779, 194780, 194781, 194782, + 194783, 194784, 194785, 194786, 194787, 194788, 194789, 194790, 194791, + 194792, 194793, 194794, 194795, 194796, 194797, 194798, 194799, 194800, + 194801, 194802, 194803, 194804, 194805, 194806, 194807, 194808, 194809, + 194810, 194811, 194812, 194813, 194814, 194815, 194816, 194817, 194818, + 194819, 194820, 194821, 194822, 194823, 194824, 194825, 194826, 194827, + 194828, 194829, 194830, 194831, 194832, 194833, 194834, 194835, 194836, + 194837, 194838, 194839, 194840, 194841, 194842, 194843, 194844, 194845, + 194846, 194847, 194848, 194849, 194850, 194851, 194852, 194853, 194854, + 194855, 194856, 194857, 194858, 194859, 194860, 194861, 194862, 194863, + 194864, 194865, 194866, 194867, 194868, 194869, 194870, 194871, 194872, + 194873, 194874, 194875, 194876, 194877, 194878, 194879, 194880, 194881, + 194882, 194883, 194884, 194885, 194886, 194887, 194888, 194889, 194890, + 194891, 194892, 194893, 194894, 194895, 194896, 194897, 194898, 194899, + 194900, 194901, 194902, 194903, 194904, 194905, 194906, 194907, 194908, + 194909, 194910, 194911, 194912, 194913, 194914, 194915, 194916, 194917, + 194918, 194919, 194920, 194921, 194922, 194923, 194924, 194925, 194926, + 194927, 194928, 194929, 194930, 194931, 194932, 194933, 194934, 194935, + 194936, 194937, 194938, 194939, 194940, 194941, 194942, 194943, 194944, + 194945, 194946, 194947, 194948, 194949, 194950, 194951, 194952, 194953, + 194954, 194955, 194956, 194957, 194958, 194959, 194960, 194961, 194962, + 194963, 194964, 194965, 194966, 194967, 194968, 194969, 194970, 194971, + 194972, 194973, 194974, 194975, 194976, 194977, 194978, 194979, 194980, + 194981, 194982, 194983, 194984, 194985, 194986, 194987, 194988, 194989, + 194990, 194991, 194992, 194993, 194994, 194995, 194996, 194997, 194998, + 194999, 195000, 195001, 195002, 195003, 195004, 195005, 195006, 195007, + 195008, 195009, 195010, 195011, 195012, 195013, 195014, 195015, 195016, + 195017, 195018, 195019, 195020, 195021, 195022, 195023, 195024, 195025, + 195026, 195027, 195028, 195029, 195030, 195031, 195032, 195033, 195034, + 195035, 195036, 195037, 195038, 195039, 195040, 195041, 195042, 195043, + 195044, 195045, 195046, 195047, 195048, 195049, 195050, 195051, 195052, + 195053, 195054, 195055, 195056, 195057, 195058, 195059, 195060, 195061, + 195062, 195063, 195064, 195065, 195066, 195067, 195068, 195069, 195070, + 195071, 64096, 64097, 64098, 64099, 64100, 64101, 64102, 64103, 64104, + 64105, 64106, 64107, 64108, 64109, 64000, 64001, 64002, 64003, 64004, + 64005, 64006, 64007, 64008, 64009, 64010, 64011, 64012, 64013, 64014, + 64015, 64016, 64017, 64018, 64019, 64020, 64021, 64022, 64023, 64024, + 64025, 64026, 64027, 64028, 64029, 64030, 64031, 64032, 64033, 64034, + 64035, 64036, 64037, 64038, 64039, 64040, 64041, 64042, 64043, 64044, + 64045, 64046, 64047, 64048, 64049, 64050, 64051, 64052, 64053, 64054, + 64055, 64056, 64057, 64058, 64059, 64060, 64061, 64062, 64063, 64064, + 64065, 64066, 64067, 64068, 64069, 64070, 64071, 64072, 64073, 64074, + 64075, 64076, 64077, 64078, 64079, 64080, 64081, 64082, 64083, 64084, + 64085, 64086, 64087, 64088, 64089, 64090, 64091, 64092, 64093, 64094, + 64095, 64112, 64113, 64114, 64115, 64116, 64117, 64118, 64119, 64120, + 64121, 64122, 64123, 64124, 64125, 64126, 64127, 64128, 64129, 64130, + 64131, 64132, 64133, 64134, 64135, 64136, 64137, 64138, 64139, 64140, + 64141, 64142, 64143, 64144, 64145, 64146, 64147, 64148, 64149, 64150, + 64151, 64152, 64153, 64154, 64155, 64156, 64157, 64158, 64159, 64160, + 64161, 64162, 64163, 64164, 64165, 64166, 64167, 64168, 64169, 64170, + 64171, 64172, 64173, 64174, 64175, 64176, 64177, 64178, 64179, 64180, + 64181, 64182, 64183, 64184, 64185, 64186, 64187, 64188, 64189, 64190, + 64191, 64192, 64193, 64194, 64195, 64196, 64197, 64198, 64199, 64200, + 64201, 64202, 64203, 64204, 64205, 64206, 64207, 64208, 64209, 64210, + 64211, 64212, 64213, 64214, 64215, 64216, 64217, 63744, 63745, 63746, + 63747, 63748, 63749, 63750, 63751, 63752, 63753, 63754, 63755, 63756, + 63757, 63758, 63759, 63760, 63761, 63762, 63763, 63764, 63765, 63766, + 63767, 63768, 63769, 63770, 63771, 63772, 63773, 63774, 63775, 63776, + 63777, 63778, 63779, 63780, 63781, 63782, 63783, 63784, 63785, 63786, + 63787, 63788, 63789, 63790, 63791, 63792, 63793, 63794, 63795, 63796, + 63797, 63798, 63799, 63800, 63801, 63802, 63803, 63804, 63805, 63806, + 63807, 63808, 63809, 63810, 63811, 63812, 63813, 63814, 63815, 63816, + 63817, 63818, 63819, 63820, 63821, 63822, 63823, 63824, 63825, 63826, + 63827, 63828, 63829, 63830, 63831, 63832, 63833, 63834, 63835, 63836, + 63837, 63838, 63839, 63840, 63841, 63842, 63843, 63844, 63845, 63846, + 63847, 63848, 63849, 63850, 63851, 63852, 63853, 63854, 63855, 63856, + 63857, 63858, 63859, 63860, 63861, 63862, 63863, 63864, 63865, 63866, + 63867, 63868, 63869, 63870, 63871, 63872, 63873, 63874, 63875, 63876, + 63877, 63878, 63879, 63880, 63881, 63882, 63883, 63884, 63885, 63886, + 63887, 63888, 63889, 63890, 63891, 63892, 63893, 63894, 63895, 63896, + 63897, 63898, 63899, 63900, 63901, 63902, 63903, 63904, 63905, 63906, + 63907, 63908, 63909, 63910, 63911, 63912, 63913, 63914, 63915, 63916, + 63917, 63918, 63919, 63920, 63921, 63922, 63923, 63924, 63925, 63926, + 63927, 63928, 63929, 63930, 63931, 63932, 63933, 63934, 63935, 63936, + 63937, 63938, 63939, 63940, 63941, 63942, 63943, 63944, 63945, 63946, + 63947, 63948, 63949, 63950, 63951, 63952, 63953, 63954, 63955, 63956, + 63957, 63958, 63959, 63960, 63961, 63962, 63963, 63964, 63965, 63966, + 63967, 63968, 63969, 63970, 63971, 63972, 63973, 63974, 63975, 63976, + 63977, 63978, 63979, 63980, 63981, 63982, 63983, 63984, 63985, 63986, + 63987, 63988, 63989, 63990, 63991, 63992, 63993, 63994, 63995, 63996, + 63997, 63998, 63999, 11946, 12003, 11910, 11963, 11962, 11950, 11992, + 12012, 12000, 12005, 11996, 12010, 11984, 11988, 11994, 11987, 11952, + 12007, 11977, 11976, 11973, 12019, 11993, 12014, 11995, 12016, 12006, + 12002, 11979, 11936, 11983, 11905, 11970, 11943, 11931, 11914, 11934, + 11944, 11999, 11998, 11997, 11960, 11947, 11939, 11978, 11968, 11967, + 11966, 12004, 11927, 11926, 12001, 11975, 11928, 12018, 12013, 12015, + 12011, 11945, 11986, 11985, 11921, 11920, 11919, 11918, 11964, 11957, + 11990, 11989, 11935, 11965, 11933, 11941, 11940, 11909, 11991, 11959, + 11929, 11904, 11908, 11907, 11906, 11915, 11942, 11974, 11980, 12008, + 12009, 11951, 11925, 11924, 11922, 11949, 11948, 11917, 11916, 11958, + 11932, 12017, 11911, 11923, 11969, 11982, 11981, 11938, 11937, 11972, + 11971, 11956, 11955, 11954, 11953, 11913, 11912, 11961, 12752, 12743, + 12748, 12757, 12741, 12750, 12769, 12747, 12749, 12744, 12742, 12746, + 12768, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, 12760, + 12759, 12745, 12766, 12762, 12755, 12761, 12736, 12765, 12739, 12737, + 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, 129346, + 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, 128339, + 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, 128353, + 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, 10561, + 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, 8753, + 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, 128272, + 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, 129313, 9114, + 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, 769, 791, + 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, 810, 838, + 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, 42609, + 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, 11752, + 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, 11763, + 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, 11769, + 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, 11755, + 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, 1155, + 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, 43244, + 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, 43238, + 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, 775, + 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, 6840, + 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, 779, + 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, + 8415, 839, 850, 8412, 122892, 122884, 122891, 122890, 122921, 122919, + 122889, 122916, 122900, 122907, 122910, 122901, 122908, 122880, 122920, + 122881, 122909, 122903, 122922, 122883, 122895, 122894, 122896, 122898, + 122899, 122882, 122885, 122912, 122911, 122913, 122918, 122915, 122888, + 122886, 122904, 122902, 122897, 122893, 70508, 70507, 70506, 70505, + 70504, 70502, 70503, 70515, 70513, 70514, 70516, 70512, 768, 790, 832, + 7624, 7621, 847, 119363, 119362, 119364, 835, 834, 837, 836, 777, 843, + 795, 785, 815, 826, 811, 6855, 6834, 7632, 12442, 12441, 7671, 7670, + 7643, 7646, 7647, 7649, 7650, 867, 7666, 7655, 7636, 7637, 7638, 872, + 7639, 868, 7663, 7641, 7659, 7635, 869, 7640, 6860, 6861, 6862, 7645, + 7660, 7653, 870, 7667, 7661, 871, 7668, 7664, 876, 7651, 7626, 877, 6848, + 7652, 7658, 7656, 7657, 7665, 6847, 873, 7642, 874, 7644, 875, 7648, + 7662, 878, 879, 7654, 852, 7678, 8430, 8406, 841, 794, 6849, 6851, 796, + 849, 8400, 792, 845, 8417, 8429, 8426, 65056, 65063, 65057, 65064, 6841, + 8427, 824, 822, 8402, 818, 772, 65060, 65067, 65061, 65068, 817, 7622, + 7620, 7628, 800, 6854, 842, 66424, 66425, 66423, 66426, 66422, 6839, 808, + 7630, 773, 6846, 6845, 6843, 801, 799, 6856, 8421, 788, 802, 7679, 854, + 848, 853, 8431, 8407, 825, 855, 8401, 6850, 6852, 793, 8428, 8408, 805, + 778, 7631, 859, 823, 821, 8403, 6853, 827, 6842, 7619, 7618, 828, 8411, + 771, 65065, 65066, 820, 816, 6859, 8424, 6836, 786, 846, 797, 7669, 7633, + 7634, 809, 781, 830, 7673, 8425, 6838, 6837, 851, 829, 8274, 64, 44, + 128476, 9092, 8705, 129517, 9732, 127882, 128533, 128534, 128119, 128679, + 8715, 8883, 8885, 8954, 8955, 8957, 8750, 127899, 983187, 9089, 127978, + 9010, 9740, 10861, 127859, 127850, 127834, 11505, 11504, 11503, 11464, + 11392, 11506, 11499, 11501, 11446, 11452, 11458, 11466, 11442, 11448, + 11450, 11398, 1006, 1002, 11396, 1000, 11406, 998, 11436, 11412, 11420, + 996, 11434, 11472, 11414, 11422, 11478, 11468, 11470, 11476, 11474, + 11482, 11460, 11480, 11454, 11462, 11444, 11486, 11488, 11484, 11490, + 11440, 1004, 994, 11456, 11402, 11428, 11408, 11430, 11404, 11394, 11438, + 11424, 11410, 11426, 11400, 11416, 11418, 11432, 66272, 66298, 66289, + 66295, 66286, 66294, 66285, 66299, 66290, 66291, 66297, 66288, 66296, + 66287, 66293, 66284, 66292, 66283, 66282, 66277, 66276, 66279, 66278, + 66275, 66274, 66281, 66273, 66280, 11517, 11518, 11514, 11515, 11516, + 11513, 11465, 11393, 11507, 11500, 11502, 11447, 11453, 11459, 11467, + 11443, 11449, 11451, 11399, 1007, 1003, 11397, 1001, 11407, 999, 11437, + 11413, 11421, 997, 11435, 11473, 11415, 11423, 11479, 11469, 11471, + 11477, 11475, 11483, 11461, 11481, 11455, 11463, 11445, 11487, 11489, + 11485, 11491, 11441, 1005, 995, 11457, 11403, 11429, 11409, 11431, 11405, + 11395, 11439, 11425, 11411, 11427, 11401, 11417, 11419, 11433, 11497, + 11492, 11493, 11494, 11498, 11495, 11496, 11519, 127279, 169, 11855, + 8792, 129720, 9012, 9013, 119661, 119660, 119663, 119662, 119659, 119658, + 119665, 119657, 119664, 119652, 119651, 119654, 119653, 119650, 119649, + 119656, 119648, 119655, 128715, 128145, 128004, 128046, 9904, 129689, + 129509, 983074, 128179, 127769, 129431, 127951, 9769, 9768, 11857, 11856, + 128322, 128321, 10060, 127370, 127884, 9876, 9932, 128010, 129360, + 128081, 8354, 129660, 128575, 128546, 128302, 129408, 8731, 74794, 74771, + 74861, 74780, 74820, 74821, 74765, 74758, 74853, 74856, 74855, 74854, + 74791, 74801, 74844, 74836, 74837, 74809, 74755, 74829, 74777, 74786, + 74768, 74858, 74762, 74834, 74835, 74808, 74812, 74814, 74815, 74813, + 74754, 74828, 74776, 74785, 74790, 74800, 74767, 74857, 74761, 74839, + 74838, 74822, 74825, 74823, 74824, 74795, 74772, 74862, 74781, 74766, + 74759, 74849, 74850, 74840, 74847, 74848, 74851, 74831, 74804, 74773, + 74782, 74845, 74842, 74796, 74852, 74818, 74819, 74817, 74793, 74770, + 74860, 74779, 74764, 74757, 74802, 74803, 74792, 74756, 74830, 74769, + 74859, 74778, 74816, 74763, 74806, 74807, 74833, 74788, 74789, 74798, + 74799, 74810, 74811, 74753, 74827, 74775, 74784, 74760, 74752, 74826, + 74832, 74805, 74841, 74774, 74783, 74787, 74797, 74846, 74843, 74867, + 74868, 74866, 74865, 74864, 73728, 73734, 73731, 73733, 73735, 73736, + 73730, 73732, 73729, 73738, 73742, 73741, 73744, 73745, 74881, 73747, + 73748, 73740, 73739, 74608, 74880, 73746, 73743, 73749, 73750, 73753, + 73752, 73754, 73755, 73751, 74609, 73756, 74882, 73757, 73759, 73758, + 73760, 73765, 73766, 73762, 73763, 73768, 73761, 73767, 73764, 73770, + 73769, 73771, 74610, 73772, 73773, 73776, 73777, 73775, 73774, 73778, + 73780, 73781, 73782, 73784, 73788, 73789, 73787, 73785, 73786, 73791, + 73790, 73783, 73779, 73737, 73792, 73793, 74883, 73795, 74884, 74885, + 74886, 73796, 73797, 73798, 73799, 73800, 73794, 73801, 73804, 73803, + 73802, 73805, 74887, 73806, 73807, 73808, 73809, 73810, 73811, 73812, + 73813, 73814, 73815, 73816, 73817, 73818, 73819, 73820, 73821, 73822, + 73823, 73825, 73826, 73829, 73830, 73831, 73828, 73836, 74611, 73837, + 73833, 73835, 73827, 73832, 73834, 73824, 74889, 74612, 73839, 73840, + 73841, 74888, 73838, 73842, 73844, 74891, 74890, 73845, 73846, 74892, + 73847, 73848, 73849, 74613, 73843, 73850, 73852, 73853, 73851, 73854, + 73855, 74614, 73856, 73857, 74894, 74895, 74893, 74896, 74897, 74900, + 74901, 74902, 74899, 74908, 74909, 74907, 74906, 74911, 74912, 74910, + 74913, 74914, 74915, 74916, 74920, 74919, 74898, 74905, 74903, 74904, + 74917, 74918, 73858, 73860, 73861, 73862, 73863, 73864, 73865, 73859, + 73866, 73868, 73867, 73869, 73873, 73874, 73870, 73871, 74922, 74921, + 73872, 73875, 73879, 73883, 73884, 73880, 73881, 73882, 73885, 73887, + 74923, 73886, 73888, 74924, 73889, 74927, 74930, 74931, 74925, 74928, + 74929, 74932, 74926, 73890, 73891, 73892, 73893, 73895, 73896, 73900, + 73901, 73902, 73903, 73904, 73905, 73906, 74616, 74933, 73907, 73908, + 73899, 73897, 73898, 74615, 73894, 73877, 73876, 73878, 73909, 73911, + 73912, 73914, 73913, 73916, 74617, 73917, 74618, 73918, 73915, 74934, + 73920, 73919, 73921, 73922, 73924, 73925, 74935, 74936, 74937, 73926, + 73923, 73929, 73930, 73927, 73928, 74938, 74939, 73932, 74941, 74940, + 73931, 73933, 73934, 73935, 73936, 73937, 74942, 73938, 73939, 73941, + 73940, 73943, 73942, 73945, 73944, 73946, 73947, 74943, 73948, 73949, + 74944, 74945, 74946, 73950, 74947, 73951, 74948, 74949, 74950, 73952, + 73953, 73957, 73958, 73959, 74951, 73955, 73956, 73960, 73962, 73963, + 73964, 73961, 74952, 73954, 73965, 73966, 74953, 73967, 73968, 73969, + 73970, 73971, 73972, 73974, 73975, 73978, 73977, 73976, 73910, 73979, + 73980, 73981, 73973, 73982, 73983, 74954, 74619, 73984, 73985, 73986, + 73987, 73988, 73990, 73989, 73994, 73995, 73998, 73996, 73997, 73999, + 73992, 73993, 74001, 74955, 74004, 74003, 74005, 74002, 74000, 73991, + 74620, 74006, 74008, 74009, 74010, 74956, 74012, 74011, 74013, 74014, + 74957, 74015, 74016, 74017, 74019, 74020, 74021, 74024, 74023, 74022, + 74007, 74018, 74025, 74026, 74958, 74027, 74028, 74029, 74030, 74959, + 74031, 74033, 74036, 74035, 74032, 74034, 74037, 74038, 74039, 74040, + 74043, 74044, 74042, 74041, 74045, 74046, 74621, 74047, 74050, 74052, + 74051, 74053, 74054, 74058, 74057, 74055, 74056, 74059, 74060, 74061, + 74062, 74064, 74065, 74063, 74066, 74067, 74048, 74070, 74049, 74068, + 74069, 74071, 74072, 74073, 74074, 74622, 74075, 74623, 74077, 74076, + 74078, 74079, 74960, 74080, 74081, 74082, 74085, 74086, 74084, 74083, + 74087, 74624, 74090, 74089, 74088, 74091, 74092, 74625, 74093, 74094, + 74096, 74097, 74961, 74095, 74099, 74627, 74098, 74100, 74101, 74103, + 74102, 74104, 74105, 74115, 74629, 74114, 74112, 74113, 74110, 74111, + 74117, 74116, 74118, 74630, 74119, 74962, 74963, 74631, 74122, 74123, + 74120, 74121, 74626, 74107, 74106, 74628, 74108, 74109, 74124, 74125, + 74126, 74131, 74132, 74128, 74129, 74130, 74133, 74134, 74135, 74136, + 74137, 74138, 74139, 74140, 74141, 74142, 74607, 74127, 74144, 74146, + 74147, 74145, 74152, 74153, 74150, 74151, 74148, 74149, 74154, 74155, + 74157, 74158, 74163, 74164, 74165, 74160, 74161, 74156, 74159, 74162, + 74143, 74166, 74167, 74168, 74169, 74170, 74171, 74172, 74175, 74173, + 74174, 74176, 74177, 74182, 74183, 74180, 74181, 74632, 74186, 74184, + 74185, 74188, 74190, 74189, 74187, 74194, 74195, 74193, 74191, 74192, + 74196, 74197, 74198, 74199, 74200, 74201, 74202, 74205, 74206, 74207, + 74208, 74204, 74209, 74211, 74210, 74212, 74213, 74215, 74214, 74216, + 74218, 74217, 74964, 74178, 74179, 74203, 74219, 74220, 74223, 74224, + 74221, 74222, 74966, 74967, 74974, 74973, 74972, 74969, 74970, 74971, + 74975, 74968, 74965, 74977, 74976, 74980, 74981, 74982, 74984, 74985, + 74978, 74979, 74983, 74986, 74987, 74988, 74989, 74990, 74991, 74993, + 74997, 74996, 74998, 74995, 74994, 74992, 74999, 75000, 75003, 75004, + 75005, 75006, 75001, 75002, 75009, 75015, 75016, 75019, 75017, 75018, + 75013, 75012, 75010, 75011, 75014, 75021, 75030, 75029, 75027, 75024, + 75025, 75028, 75022, 75026, 75023, 75008, 75020, 75031, 75032, 75007, + 74226, 74227, 74228, 74229, 74230, 74225, 74231, 74233, 74234, 74232, + 74235, 74237, 74258, 74259, 75033, 74261, 74633, 74260, 74240, 74634, + 74241, 74243, 75035, 74246, 74247, 74245, 74248, 74249, 74250, 74251, + 75036, 75037, 74254, 74255, 74635, 74256, 75038, 74242, 74252, 74253, + 75034, 74238, 74239, 74244, 74257, 74263, 74265, 74264, 74266, 74269, + 74270, 74271, 74236, 74262, 74267, 74268, 74272, 74273, 74274, 74278, + 74279, 74275, 74276, 74277, 74280, 74281, 74636, 74282, 75039, 74283, + 74284, 74290, 74294, 74295, 75041, 75040, 74292, 74293, 74291, 74296, + 74297, 74298, 74299, 74300, 74637, 74301, 74286, 74287, 74285, 74289, + 74288, 74302, 74304, 74307, 74305, 74306, 74308, 74310, 74309, 74311, + 74303, 74638, 74312, 74314, 74313, 74315, 74316, 74319, 74321, 74320, + 74639, 74322, 74324, 74325, 74323, 74642, 75043, 74326, 75044, 75046, + 75047, 74327, 75048, 74329, 74328, 75049, 74330, 74332, 74333, 74331, + 75050, 75051, 74334, 75052, 74335, 75042, 74641, 75045, 74640, 74317, + 74336, 74318, 74337, 74338, 983266, 983265, 74643, 74339, 74347, 74348, + 74342, 74343, 74341, 74344, 74340, 74346, 74345, 74349, 74355, 74354, + 74350, 74352, 74358, 74359, 74353, 74357, 74351, 74356, 74360, 74361, + 74362, 74363, 74364, 74365, 74366, 74644, 74367, 74375, 74376, 74368, + 74369, 74373, 74374, 74370, 74371, 74372, 74377, 74378, 74379, 74380, + 74381, 74382, 74645, 74383, 74384, 74385, 74386, 74387, 74389, 74408, + 75053, 74388, 74646, 74395, 74394, 75055, 74400, 74399, 75056, 74401, + 74406, 74402, 74403, 74404, 74405, 74391, 74392, 74396, 74398, 75054, + 74397, 74393, 74390, 74407, 74409, 74410, 74411, 74412, 74413, 74414, + 74421, 74422, 74419, 74417, 74420, 74416, 74418, 74415, 74423, 75057, + 74424, 74425, 74426, 75058, 74428, 74429, 75059, 75060, 75061, 74427, + 74432, 74434, 74433, 74430, 74431, 74435, 74437, 74436, 74438, 74441, + 74440, 74444, 74445, 74446, 74448, 74447, 74443, 74449, 74442, 74439, + 74451, 74453, 74452, 74450, 74454, 74455, 74456, 74457, 75063, 75062, + 74458, 74459, 75064, 74460, 74461, 74462, 74463, 74465, 74464, 74466, + 74468, 74469, 74471, 74472, 74473, 74474, 74467, 74470, 74475, 74477, + 74478, 74479, 74476, 74480, 74481, 74482, 74483, 74488, 74486, 74487, + 74485, 74489, 74484, 74490, 75065, 74491, 74494, 74497, 74499, 74500, + 74498, 74495, 74647, 74496, 74501, 74504, 75066, 75067, 74505, 74506, + 74502, 74503, 74492, 74493, 74507, 74510, 74512, 74511, 74649, 74509, + 74508, 74515, 74516, 74522, 74523, 74519, 74520, 74517, 74518, 74521, + 74524, 74534, 74535, 74525, 74526, 74648, 74527, 74528, 74529, 74531, + 74532, 74533, 74530, 74536, 74538, 74537, 74539, 75068, 74540, 74541, + 74542, 74545, 74546, 74547, 75069, 74544, 74543, 74549, 74550, 74551, + 74552, 74553, 75070, 74555, 74556, 74558, 74557, 74559, 74560, 74562, + 74564, 74563, 75072, 74566, 75071, 74570, 74569, 74572, 74574, 74573, + 74554, 74567, 74571, 74565, 74561, 74568, 74575, 74576, 74548, 74577, + 74581, 74579, 74580, 74578, 74584, 74583, 74582, 74586, 74587, 74588, + 74585, 74513, 74514, 74589, 74591, 74590, 74593, 75073, 74592, 74595, + 74598, 74599, 74596, 74601, 74597, 74600, 74602, 75074, 74603, 75075, + 74604, 74605, 74606, 74594, 9982, 129380, 11232, 129473, 8911, 8910, + 10160, 9130, 129356, 128177, 164, 127835, 10081, 127854, 128707, 129362, + 129385, 67594, 67595, 67596, 67597, 67598, 67599, 67600, 67601, 67602, + 67603, 67604, 67605, 67606, 67607, 67608, 67609, 67610, 67611, 67612, + 67613, 67614, 67615, 67616, 67617, 67618, 67619, 67620, 67621, 67622, + 67623, 67624, 67625, 67626, 67627, 67628, 67629, 67630, 67631, 67632, + 67633, 67634, 67635, 67636, 67637, 67589, 67592, 67644, 67647, 67639, + 67640, 67584, 67585, 67586, 67587, 67588, 77712, 77713, 77714, 77715, + 77716, 77717, 77718, 77719, 77722, 77723, 77720, 77721, 77724, 77725, + 77726, 77727, 77728, 77729, 77730, 77731, 77732, 77733, 77734, 77735, + 77736, 77737, 77738, 77739, 77740, 77741, 77742, 77743, 77744, 77745, + 77746, 77747, 77748, 77749, 77750, 77751, 77752, 77753, 77754, 77755, + 77756, 77757, 77758, 77773, 77774, 77768, 77769, 77770, 77771, 77772, + 77775, 77776, 77777, 77788, 77789, 77790, 77791, 77792, 77793, 77794, + 77795, 77796, 77759, 77760, 77761, 77762, 77763, 77764, 77765, 77766, + 77767, 77778, 77779, 77780, 77781, 77782, 77783, 77784, 77785, 77786, + 77787, 77797, 77798, 77799, 77800, 77801, 77802, 77803, 77804, 77805, + 77806, 77807, 77808, 77809, 77810, 1126, 1033, 1300, 42574, 1034, 1055, + 1190, 1316, 1136, 1146, 42564, 42592, 42580, 1296, 1302, 1058, 42634, + 1196, 1035, 42640, 42638, 1062, 42642, 42636, 1059, 1266, 1264, 1262, + 1144, 1028, 1040, 1234, 1232, 1212, 1214, 1248, 1192, 1310, 1256, 1258, + 1184, 42602, 1130, 42572, 42586, 1030, 1041, 1063, 1268, 1206, 1208, + 42584, 42650, 42630, 1026, 42568, 42604, 42648, 1029, 42562, 1322, 42632, + 1039, 42626, 1324, 42624, 1044, 1069, 1051, 1312, 1326, 1221, 1298, 1053, + 1314, 1186, 1320, 1225, 1223, 1056, 1166, 1260, 1057, 1194, 1052, 1229, + 1060, 1043, 1172, 1168, 1270, 1170, 1274, 1027, 1061, 1202, 1276, 1278, + 1066, 42644, 1048, 1252, 1037, 1250, 1045, 1024, 1238, 1025, 42588, 1128, + 1132, 42578, 42582, 1124, 42566, 1140, 1142, 1050, 1178, 1219, 1180, + 1182, 1286, 1282, 1280, 1288, 1290, 1292, 1294, 1284, 1152, 1227, 1036, + 1134, 42600, 42570, 1054, 1120, 1148, 1254, 1150, 1240, 1242, 1049, 1162, + 1038, 1210, 1318, 1065, 42646, 1064, 42596, 42598, 1068, 42594, 1198, + 1200, 1164, 1071, 1304, 1122, 1067, 1272, 42576, 1031, 42590, 1070, 1047, + 1246, 1176, 42560, 1046, 1244, 1174, 1217, 42628, 1138, 1032, 1042, 1308, + 1306, 1236, 1204, 1188, 42622, 42606, 1216, 7467, 42623, 1072, 1235, + 1233, 1213, 1215, 1249, 1193, 1311, 1257, 1259, 1185, 42603, 1131, 42573, + 42587, 1110, 1073, 1095, 1269, 1207, 1209, 42585, 42651, 42631, 1106, + 42569, 42605, 42649, 1109, 42563, 1323, 42633, 1119, 42627, 1325, 42625, + 1076, 1101, 1083, 1313, 1327, 1222, 1299, 1085, 1315, 1187, 1321, 1226, + 1224, 1088, 1167, 1261, 1089, 1195, 1084, 1230, 1092, 1075, 1173, 1169, + 1271, 1171, 1275, 1107, 1093, 1203, 1277, 1279, 1098, 42645, 1080, 1253, + 1117, 1251, 1077, 1104, 1239, 1105, 42589, 1129, 1133, 42579, 42583, + 1125, 42567, 1141, 1143, 1082, 1179, 1220, 1181, 1183, 1287, 1283, 1281, + 1289, 1291, 1293, 1295, 1285, 1153, 1228, 1116, 1135, 1127, 7297, 1113, + 1301, 42601, 42571, 42575, 7298, 1114, 1086, 1121, 1149, 1255, 1151, + 1087, 1191, 1317, 1231, 1137, 42565, 42593, 42581, 1297, 1147, 7296, + 1303, 1241, 1243, 1081, 1163, 1118, 1211, 1319, 1097, 42647, 1096, 42597, + 42599, 1100, 42595, 1199, 1201, 1165, 7302, 7303, 7300, 1090, 42635, + 1197, 1115, 42641, 42639, 1094, 7301, 42643, 42637, 1091, 1267, 1265, + 1263, 1145, 1108, 7304, 7299, 1309, 1103, 1305, 1123, 1099, 1273, 42577, + 1111, 42591, 1102, 1079, 1247, 1177, 42561, 1078, 1245, 1175, 1218, + 42629, 1139, 1112, 1074, 1307, 1237, 1205, 1189, 122984, 122962, 122986, + 122985, 122965, 122976, 122971, 122974, 122964, 122983, 122977, 122981, + 122982, 122980, 122978, 122967, 122968, 122969, 122966, 122979, 122973, + 122963, 122970, 122961, 122972, 122975, 1154, 9005, 127744, 983201, + 983188, 983172, 8224, 11830, 11831, 128481, 128131, 127841, 128374, 9619, + 11843, 128168, 10143, 65101, 65097, 8504, 983081, 983084, 983086, 983088, + 983090, 983162, 127795, 9110, 9192, 128475, 8451, 176, 8457, 983120, + 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, 9160, 9154, 9157, + 9163, 9150, 9158, 9164, 9151, 9152, 9155, 127980, 9739, 66589, 66591, + 66596, 66597, 66587, 66585, 66594, 66595, 66593, 66599, 66562, 66563, + 66564, 66565, 66561, 66560, 66568, 66569, 66570, 66571, 66567, 66566, + 66598, 66573, 66588, 66579, 66592, 66590, 66581, 66578, 66580, 66582, + 66577, 66586, 66575, 66584, 66583, 66574, 66572, 66576, 66629, 66631, + 66636, 66637, 66627, 66625, 66634, 66635, 66633, 66639, 66602, 66603, + 66604, 66605, 66601, 66600, 66608, 66609, 66610, 66611, 66607, 66606, + 66638, 66613, 66628, 66619, 66632, 66630, 66621, 66618, 66620, 66622, + 66617, 66626, 66615, 66624, 66623, 66614, 66612, 66616, 127964, 127965, + 128421, 128468, 2388, 2416, 43258, 2387, 43257, 72448, 72449, 43259, + 2309, 2310, 2320, 2324, 2421, 43262, 2330, 2418, 2317, 2321, 2331, 2396, + 2430, 2338, 2337, 2343, 2342, 2429, 2394, 2328, 2427, 2327, 2426, 2361, + 2350, 2424, 2308, 2318, 2322, 2358, 2359, 2360, 2323, 2420, 2419, 2431, + 2349, 2348, 2333, 2428, 2332, 2393, 2326, 2325, 2397, 2353, 2352, 2399, + 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, 2344, 2356, 2355, + 2354, 2336, 2335, 2341, 2340, 2315, 2400, 2316, 2401, 2357, 2311, 2312, + 2347, 2346, 2425, 2395, 2398, 2392, 2319, 983644, 983643, 983646, 983647, + 983649, 983648, 983642, 983645, 72450, 72451, 72452, 72453, 2305, 43255, + 43251, 43254, 43253, 72455, 72454, 72456, 43256, 43250, 43260, 43252, + 2304, 2365, 2306, 72457, 2364, 2417, 2381, 2307, 2386, 2385, 2366, 2376, + 2380, 2383, 43263, 2389, 2373, 2377, 2379, 2363, 2362, 2382, 2369, 2370, + 2391, 2390, 2374, 2378, 2371, 2372, 2402, 2403, 2367, 2368, 2375, 2405, + 2404, 2411, 2410, 2413, 2412, 2409, 2408, 2406, 2415, 2407, 2414, 43261, + 2384, 983089, 983161, 983087, 983085, 983083, 127962, 129487, 129420, + 11033, 11032, 11030, 11031, 128924, 128160, 8900, 8960, 168, 9856, 9857, + 9858, 9859, 9860, 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, + 127242, 9360, 49, 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, + 127236, 9354, 50, 127235, 9353, 48, 127233, 127232, 56, 127241, 9359, + 119557, 119556, 119555, 9868, 9871, 9870, 9869, 119553, 119554, 10131, + 10126, 10125, 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, + 10111, 10106, 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, + 10121, 10116, 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, + 10119, 9107, 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, + 72020, 72023, 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, + 71958, 71963, 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, + 71940, 71941, 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, + 71978, 71979, 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, + 71948, 71969, 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, + 71945, 71997, 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, + 71986, 71989, 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, + 8725, 247, 8903, 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, + 9009, 128462, 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, + 8832, 8928, 8876, 8833, 8929, 8878, 128021, 71723, 71716, 71680, 71681, + 71687, 71689, 71703, 71702, 71708, 71707, 71701, 71700, 71706, 71705, + 71684, 71685, 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, + 71721, 71713, 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, + 71690, 71711, 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, + 71729, 71730, 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, + 71733, 71739, 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, + 127026, 127027, 127028, 127029, 127030, 127031, 127032, 127033, 127034, + 127035, 127036, 127037, 127038, 127039, 127040, 127041, 127042, 127043, + 127044, 127045, 127046, 127047, 127048, 127049, 127050, 127051, 127052, + 127053, 127054, 127055, 127056, 127057, 127058, 127059, 127060, 127061, + 127062, 127063, 127064, 127065, 127066, 127067, 127068, 127069, 127070, + 127071, 127072, 127073, 127024, 127075, 127076, 127077, 127078, 127079, + 127080, 127081, 127082, 127083, 127084, 127085, 127086, 127087, 127088, + 127089, 127090, 127091, 127092, 127093, 127094, 127095, 127096, 127097, + 127098, 127099, 127100, 127101, 127102, 127103, 127104, 127105, 127106, + 127107, 127108, 127109, 127110, 127111, 127112, 127113, 127114, 127115, + 127116, 127117, 127118, 127119, 127120, 127121, 127122, 127123, 127074, + 129743, 8363, 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, + 11795, 11784, 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, + 9469, 9461, 9468, 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, + 11842, 8222, 8215, 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, + 12318, 10746, 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, + 11005, 10987, 8915, 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, + 8252, 8263, 65100, 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, + 8511, 8450, 8461, 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, + 8509, 8508, 8512, 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, + 8945, 8964, 128315, 128317, 10728, 10729, 129288, 129290, 129289, 129291, + 8595, 8629, 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, + 8627, 10504, 8693, 129975, 8615, 10515, 11796, 11107, 11139, 11123, + 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, 11085, + 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, 11147, + 11015, 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, + 10581, 8642, 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, + 128687, 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, + 129656, 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, + 113783, 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, + 113794, 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, + 113814, 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, + 113777, 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, + 113746, 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, + 113709, 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, + 113670, 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, + 113690, 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, + 113732, 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, + 113675, 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, + 113728, 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, + 113706, 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, + 113752, 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, + 113738, 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, + 113721, 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, + 113671, 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, + 983082, 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, + 9178, 9841, 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, + 77825, 77826, 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, + 77836, 77837, 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, + 77872, 77873, 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, + 77881, 77882, 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, + 77866, 77867, 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, + 77854, 77855, 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, + 77889, 77890, 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, + 77899, 77900, 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, + 78864, 78865, 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, + 78874, 78875, 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, + 78884, 78885, 78886, 78887, 78888, 78889, 78890, 78891, 78903, 78910, + 78908, 77908, 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, + 77913, 77915, 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, + 77923, 77924, 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, + 77933, 77934, 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, + 77950, 77942, 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, + 77975, 77978, 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, + 77984, 77991, 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, + 77990, 77993, 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, + 78004, 78005, 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, + 78014, 78015, 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, + 78031, 78032, 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, + 77969, 77970, 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, + 77972, 77952, 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, + 77961, 78041, 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, + 78039, 78040, 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, + 78063, 78064, 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, + 78075, 78076, 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, + 78048, 78049, 78050, 78055, 78056, 78904, 78911, 78909, 78078, 78079, + 78080, 78081, 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, + 78088, 78089, 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, + 78112, 78118, 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, + 78117, 78122, 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, + 78125, 78126, 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, + 78141, 78142, 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, + 78107, 78108, 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, + 78144, 78145, 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, + 78159, 78160, 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, + 78187, 78178, 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, + 78194, 78196, 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, + 78201, 78166, 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, + 78175, 78176, 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, + 78207, 78208, 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, + 78220, 78221, 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, + 78224, 78227, 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, + 78901, 78899, 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, + 78243, 78246, 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, + 78257, 78258, 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, + 78269, 78270, 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, + 78279, 78280, 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, + 78286, 78289, 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, + 78294, 78295, 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, + 78303, 78307, 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, + 78316, 78317, 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, + 78929, 78925, 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, + 78337, 78338, 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, + 78339, 78354, 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, + 78352, 78353, 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, + 78343, 78344, 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, + 78320, 78321, 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, + 78368, 78369, 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, + 78378, 78379, 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, + 78399, 78400, 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, + 78407, 78408, 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, + 78393, 78394, 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, + 78427, 78428, 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, + 78430, 78431, 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, + 78438, 78439, 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, + 78453, 78454, 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, + 78458, 78469, 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, + 78463, 78464, 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, + 78490, 78477, 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, + 78486, 78902, 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, + 78499, 78500, 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, + 78511, 78517, 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, + 78531, 78525, 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, + 78536, 78537, 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, + 78547, 78551, 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, + 78560, 78561, 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, + 78564, 78567, 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, + 78573, 78574, 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, + 78584, 78585, 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, + 78596, 78597, 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, + 78611, 78599, 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, + 78612, 78615, 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, + 78638, 78639, 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, + 78627, 78628, 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, + 78650, 78643, 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, + 78668, 78674, 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, + 78673, 78678, 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, + 78685, 78686, 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, + 78659, 78660, 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, + 78692, 78693, 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, + 78702, 78703, 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, + 78895, 78716, 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, + 78724, 78725, 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, + 78734, 78735, 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, + 78750, 78739, 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, + 78754, 78756, 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, + 78764, 78765, 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, + 78771, 78772, 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, + 78788, 78789, 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, + 78797, 78792, 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, + 78806, 78807, 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, + 78812, 78813, 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, + 78823, 78824, 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, + 78820, 78833, 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, + 78847, 78848, 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, + 78857, 78858, 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, + 78236, 78237, 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, + 78507, 78508, 78509, 78510, 129370, 10037, 10039, 10036, 10049, 10058, + 10035, 9834, 66854, 66853, 66827, 66826, 66833, 66832, 66821, 66837, + 66836, 66835, 66842, 66841, 66824, 66823, 66819, 66818, 66822, 66820, + 66855, 66831, 66844, 66843, 66846, 66845, 66852, 66851, 66817, 66825, + 66828, 66830, 66834, 66839, 66840, 66848, 66849, 66816, 66829, 66838, + 66847, 66850, 128268, 128294, 128161, 8961, 9191, 8712, 8946, 8953, 8947, + 8952, 8950, 8949, 10969, 10194, 128024, 128727, 69608, 69621, 69603, + 69611, 69618, 69619, 69600, 69615, 69602, 69606, 69614, 69617, 69620, + 69609, 69604, 69607, 69610, 69601, 69613, 69605, 69616, 69612, 69622, + 129501, 983101, 129456, 129457, 129459, 129458, 127995, 127996, 127997, + 127998, 127999, 128453, 128454, 128455, 129721, 128460, 128461, 8709, + 10675, 10676, 10674, 10673, 128459, 9091, 8193, 8212, 8195, 8192, 8211, + 8194, 983179, 8718, 983178, 983135, 983099, 983048, 983095, 983046, + 983064, 128282, 983051, 983050, 9993, 128388, 128233, 9094, 983067, + 983100, 983049, 8926, 8927, 8925, 8924, 8797, 8917, 61, 10865, 10867, + 11072, 10609, 10723, 10724, 10926, 11257, 10871, 10854, 10862, 8789, + 10872, 8781, 8794, 10738, 10736, 10734, 10739, 10737, 10735, 11249, + 11248, 9003, 8998, 983105, 983104, 8494, 8793, 983136, 4957, 4959, 4958, + 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, 4982, 4981, 4986, 4984, + 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, 4997, 43816, 43819, + 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, 11653, 4709, 4708, + 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, 43809, 11704, + 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, 11693, 11692, + 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, 4911, 4906, + 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, 43789, + 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, 4860, + 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, 4851, + 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, 4936, + 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, 124924, + 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, 124919, + 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, 11670, + 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, 4771, 4774, + 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, 4882, 11736, + 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, 4879, 4877, + 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, 124901, 124900, + 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, 4630, 4625, 4608, + 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, 4805, 4804, 4802, + 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, 4789, 4788, 4786, + 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, 4779, 4782, 4783, + 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, 4635, 4638, 11649, + 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, 4765, 4764, 4767, + 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, 4754, 4753, 4912, + 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, 11665, 4917, 4916, + 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, 4950, 11666, 4949, + 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, 4691, 4693, 4692, + 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, 11715, 11717, + 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, 4676, 4674, + 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, 4649, 4661, + 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, 4668, 4671, + 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, 11680, 11683, + 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, 4663, 4658, + 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, 43781, + 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, 4932, + 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, 4723, + 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, 4869, + 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, 4618, + 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, 4846, + 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, 11731, + 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, 4740, + 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, 11699, + 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, 4828, + 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, 5009, + 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, 4975, + 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, 983047, + 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 128942, 128954, 128948, + 128905, 128915, 128935, 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, + 1785, 1777, 1784, 128529, 128065, 128083, 128064, 128231, 9167, 127794, + 983180, 128523, 128561, 129312, 128531, 129301, 128567, 129488, 128582, + 128558, 129326, 128560, 129762, 129320, 128581, 129395, 129763, 129402, + 128539, 128540, 128541, 128514, 129298, 129323, 128580, 128548, 129764, + 129396, 128566, 129318, 128134, 128536, 129401, 8507, 127981, 10540, + 10543, 9950, 127810, 129478, 128439, 128224, 128106, 9771, 127877, + 129498, 129718, 128552, 170, 9792, 127905, 9972, 129338, 8210, 8199, + 128193, 128452, 983107, 128253, 127902, 128293, 128658, 129519, 127879, + 127878, 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, + 127845, 128074, 8281, 11821, 127953, 10765, 129407, 129747, 9189, 9971, + 129449, 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, + 129712, 129359, 128760, 128389, 127787, 127745, 129709, 128448, 129462, + 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, 10021, + 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, + 129749, 129418, 8260, 8543, 128444, 128446, 128445, 127839, 8355, 129398, + 10156, 128037, 8994, 128550, 128056, 127844, 127773, 127765, 10199, 9608, + 46, 65342, 65312, 65292, 65306, 65504, 65371, 65375, 65288, 65339, 65308, + 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, + 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, + 65333, 65334, 65335, 65336, 65337, 65338, 65345, 65346, 65347, 65348, + 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, + 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, + 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65373, 65376, + 65289, 65341, 65340, 65307, 65295, 65509, 65507, 65287, 65286, 65290, + 65284, 65301, 65300, 65303, 65302, 65299, 65298, 65296, 65305, 65297, + 65304, 65309, 65281, 65344, 65310, 65293, 65282, 65311, 65510, 65374, + 65508, 65294, 65372, 8289, 9905, 9981, 9179, 983215, 983216, 983217, + 983219, 983108, 983236, 983072, 129476, 127922, 9881, 9965, 9966, 128142, + 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, 4281, + 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, 4280, + 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, 4279, + 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, 11552, + 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, 11522, + 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, 11526, + 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, 11525, + 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, 4349, + 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, 4350, + 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, 4318, + 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, 4316, + 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, 7337, + 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, 7349, + 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, 7353, + 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, 7315, + 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, 8503, + 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, 11276, + 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, 11307, + 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, 11283, + 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, 11292, + 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, 11288, + 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, 11319, + 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, 11358, + 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, 11356, + 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, 11317, + 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, 11325, + 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, 66352, + 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, 66370, + 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, 66371, + 66361, 66363, 66366, 66354, 66362, 129405, 129421, 128893, 129727, + 127948, 127891, 70495, 70494, 70411, 70496, 70412, 70497, 70453, 70405, + 70406, 70416, 70420, 70434, 70433, 70439, 70438, 70432, 70431, 70437, + 70436, 70409, 70410, 70407, 70408, 70451, 70450, 70425, 70435, 70430, + 70440, 70454, 70455, 70456, 70445, 70444, 70427, 70426, 70424, 70423, + 70429, 70428, 70422, 70421, 70443, 70442, 70419, 70415, 70457, 70446, + 70448, 70447, 70400, 70401, 70493, 70461, 70402, 70460, 70477, 70403, + 70462, 70472, 70476, 70467, 70468, 70498, 70499, 70465, 70466, 70463, + 70464, 70475, 70471, 70487, 70480, 96, 127815, 10896, 10894, 10900, + 10892, 10898, 10616, 10890, 10888, 10917, 8935, 8809, 10878, 10882, + 10884, 10880, 10886, 8819, 8805, 8823, 10916, 8807, 8923, 10919, 10921, + 10874, 10876, 8919, 62, 65860, 65863, 65878, 65866, 65873, 65859, 65861, + 65868, 65875, 65862, 65870, 65864, 65871, 65867, 65874, 65857, 65869, + 65876, 65856, 65858, 65865, 65877, 65872, 65879, 65903, 65885, 65904, + 65907, 65908, 65900, 65883, 65886, 65896, 65890, 65882, 65880, 65891, + 65902, 65906, 65897, 65899, 65893, 65892, 65884, 65881, 65898, 65905, + 65887, 65901, 65894, 65895, 65888, 65889, 903, 65927, 65926, 913, 7945, + 7949, 8077, 7947, 8075, 7951, 8079, 8073, 7944, 7948, 8076, 7946, 8074, + 7950, 8078, 8072, 8124, 8120, 8122, 8123, 902, 8121, 882, 919, 7977, + 7981, 8093, 7979, 8091, 7983, 8095, 8089, 7976, 7980, 8092, 7978, 8090, + 7982, 8094, 8088, 8140, 8139, 8138, 905, 917, 7961, 7965, 7963, 7960, + 7964, 7962, 8137, 8136, 904, 921, 7993, 7999, 7997, 7995, 938, 7992, + 7998, 7996, 7994, 8152, 8154, 8155, 906, 8153, 937, 8041, 8045, 8109, + 8043, 8107, 8047, 8111, 8105, 8040, 8044, 8108, 8042, 8106, 8046, 8110, + 8104, 8188, 8187, 8186, 911, 927, 8009, 8013, 8011, 8008, 8012, 8010, + 8185, 8184, 908, 929, 8172, 931, 1018, 1015, 933, 8025, 8031, 8029, 8027, + 939, 8168, 8170, 8171, 910, 8169, 886, 934, 936, 928, 920, 932, 916, 922, + 915, 935, 914, 880, 918, 923, 895, 924, 925, 926, 1017, 1023, 1021, 1022, + 975, 1012, 8129, 8174, 8173, 901, 65915, 8190, 8159, 8158, 8157, 65920, + 65919, 119325, 119331, 119332, 119333, 119334, 119335, 119336, 119337, + 119326, 119338, 119339, 119340, 119341, 119342, 119343, 119344, 119345, + 119346, 119347, 119348, 119349, 119327, 119350, 119351, 119352, 119353, + 119354, 119355, 119356, 119328, 119357, 119358, 119359, 119360, 119361, + 119329, 119330, 65933, 1008, 983, 8125, 65922, 7466, 7464, 7462, 43877, + 7465, 7463, 992, 986, 984, 990, 988, 1011, 885, 1010, 1013, 65923, 884, + 65921, 119365, 65925, 65909, 65910, 65931, 8189, 65924, 65916, 8126, + 8127, 8143, 8142, 8141, 8128, 981, 982, 1020, 1009, 1014, 945, 8112, + 8048, 8114, 7937, 7941, 8069, 7943, 8071, 7939, 8067, 8065, 7936, 7940, + 8068, 7942, 8070, 7938, 8066, 8064, 8118, 8119, 8049, 8116, 8115, 940, + 8113, 985, 883, 989, 948, 949, 7953, 7957, 7955, 7952, 7956, 7954, 8051, + 8050, 941, 951, 7969, 7973, 8085, 7975, 8087, 7971, 8083, 8081, 7968, + 7972, 8084, 7974, 8086, 7970, 8082, 8080, 8134, 8135, 8053, 8132, 8052, + 8130, 8131, 942, 953, 7985, 7991, 7989, 7987, 970, 8151, 8147, 8146, 912, + 7984, 7990, 7988, 7986, 8150, 8144, 8054, 8055, 943, 8145, 965, 8017, + 8023, 8021, 8019, 971, 8167, 8163, 8162, 944, 8016, 8022, 8020, 8018, + 8166, 8160, 8058, 8059, 973, 8161, 954, 991, 969, 8033, 8037, 8101, 8039, + 8103, 8035, 8099, 8097, 8032, 8036, 8100, 8038, 8102, 8034, 8098, 8096, + 8182, 8183, 8061, 8180, 8060, 8178, 8179, 974, 959, 8001, 8005, 8003, + 8000, 8004, 8002, 8057, 8056, 972, 887, 966, 968, 960, 961, 8165, 8164, + 993, 1019, 987, 963, 1016, 952, 964, 962, 947, 967, 946, 881, 950, 955, + 956, 957, 958, 893, 891, 892, 7527, 7530, 7529, 7528, 7526, 65952, 65932, + 65918, 65912, 977, 65929, 65917, 65911, 900, 65914, 979, 980, 978, 8175, + 119297, 119315, 119316, 119317, 119318, 119319, 119300, 119320, 119321, + 119322, 119323, 119324, 119296, 119305, 119306, 119307, 119308, 119309, + 119310, 119311, 119312, 119313, 119314, 119298, 119299, 119301, 119302, + 119303, 119304, 890, 65913, 976, 65928, 65930, 894, 127823, 128215, + 128154, 129367, 129654, 128512, 129322, 128513, 129321, 128568, 128556, + 983110, 11218, 128151, 8370, 128130, 127928, 129454, 2693, 2694, 2704, + 2708, 2722, 2721, 2727, 2726, 2720, 2719, 2725, 2724, 2699, 2784, 2700, + 2785, 2741, 2697, 2698, 2695, 2696, 2739, 2738, 2809, 2713, 2723, 2718, + 2728, 2742, 2743, 2744, 2733, 2732, 2715, 2714, 2712, 2711, 2717, 2716, + 2710, 2709, 2731, 2730, 2745, 2734, 2736, 2735, 2703, 2707, 2814, 2689, + 2812, 2815, 2813, 2811, 2810, 2749, 2690, 2748, 2765, 2691, 2757, 2761, + 2750, 2760, 2764, 2755, 2756, 2786, 2787, 2753, 2754, 2751, 2752, 2759, + 2763, 2701, 2705, 2800, 2801, 2795, 2794, 2797, 2796, 2793, 2792, 2790, + 2799, 2791, 2798, 2768, 73056, 73057, 73064, 73067, 73091, 73090, 73081, + 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73092, 73082, 73058, + 73059, 73087, 73077, 73071, 73070, 73084, 73083, 73079, 73078, 73089, + 73088, 73074, 73073, 73094, 73093, 73066, 73063, 73095, 73072, 73096, + 73097, 73069, 73068, 73110, 73109, 73098, 73105, 73108, 73101, 73102, + 73099, 73100, 73107, 73104, 73111, 73125, 73124, 73127, 73126, 73123, + 73122, 73120, 73129, 73121, 73128, 73112, 2678, 2673, 2650, 2584, 2583, + 2649, 2582, 2581, 2652, 2608, 2565, 2566, 2576, 2580, 2594, 2593, 2599, + 2598, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, + 2595, 2590, 2600, 2605, 2604, 2587, 2586, 2589, 2588, 2603, 2602, 2614, + 2616, 2579, 2575, 2654, 2617, 2606, 2613, 2607, 2651, 983656, 983655, + 983654, 983653, 983658, 983657, 2561, 2562, 2641, 2620, 2677, 2637, 2563, + 2622, 2632, 2636, 2625, 2626, 2623, 2624, 2635, 2631, 2672, 2674, 2676, + 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 128123, + 983111, 129710, 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, + 65469, 65458, 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, + 65453, 65465, 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, + 65477, 65499, 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, + 65463, 65457, 65468, 65447, 65493, 65492, 65485, 65486, 65494, 65470, + 65474, 65475, 65484, 65487, 65491, 65440, 65388, 65390, 65389, 65391, + 65383, 65386, 65384, 65387, 65385, 65403, 65406, 65404, 65407, 65405, + 65437, 65413, 65416, 65414, 65417, 65415, 65418, 65421, 65419, 65422, + 65420, 65398, 65401, 65399, 65402, 65400, 65423, 65426, 65424, 65427, + 65425, 65431, 65434, 65432, 65435, 65433, 65408, 65411, 65409, 65412, + 65410, 65428, 65430, 65429, 65436, 65382, 65393, 65396, 65394, 65397, + 65395, 65439, 65438, 65381, 65392, 65378, 65513, 65379, 65515, 65514, + 65512, 65380, 65377, 65517, 65518, 65516, 128296, 128736, 9773, 9874, + 128057, 129708, 127828, 129310, 129776, 129342, 128092, 129309, 4366, + 4434, 4435, 4431, 4413, 4412, 4436, 4430, 4433, 4415, 4414, 4437, 4432, + 4364, 4429, 4363, 4427, 4420, 4422, 43382, 4425, 4424, 4419, 4426, 4418, + 4417, 43383, 4421, 4379, 4439, 4395, 4396, 4381, 4352, 4442, 4367, 4358, + 43375, 4380, 43376, 43377, 4354, 4444, 4371, 4374, 4373, 4445, 4443, + 4369, 4438, 43386, 4359, 4385, 43378, 4387, 4390, 4386, 4388, 4389, 4394, + 43379, 4382, 4392, 4391, 4393, 4384, 4383, 43380, 4416, 4357, 43371, + 43374, 43364, 43365, 43370, 43367, 43372, 43368, 43373, 4376, 43369, + 43366, 4378, 4361, 4402, 4403, 4410, 4404, 4400, 4408, 4397, 4407, 4406, + 4401, 4409, 4399, 4398, 4411, 4405, 4365, 43384, 4362, 43381, 4377, + 43385, 4356, 43388, 4353, 4372, 4360, 4440, 4423, 4355, 4446, 43360, + 43363, 4375, 43361, 43362, 4368, 4428, 4441, 4370, 43387, 4447, 12335, + 12334, 4541, 55288, 55287, 4542, 4546, 4598, 4599, 4597, 4600, 4540, + 4591, 4588, 4589, 55261, 4596, 4582, 4578, 4520, 4605, 4547, 4522, 4548, + 4604, 4602, 4603, 4606, 4543, 4535, 4572, 55265, 4575, 55263, 4574, 4573, + 4571, 4576, 55266, 4570, 55262, 4577, 4523, 55243, 55244, 4524, 4553, + 4550, 4549, 4525, 4552, 4551, 4587, 55284, 55283, 4545, 55291, 4595, + 55290, 4536, 4579, 55268, 4537, 55271, 4580, 55273, 55272, 55269, 55267, + 4581, 4527, 4528, 55254, 4556, 4565, 4568, 4529, 4561, 55256, 4562, 4530, + 55258, 55257, 4564, 4563, 4533, 4567, 55253, 4566, 4531, 4558, 4559, + 4532, 4569, 55260, 55259, 4557, 4534, 4538, 55275, 4583, 4585, 55280, + 55279, 4586, 55278, 55274, 55281, 4584, 55282, 4560, 55255, 55245, 55246, + 983213, 4539, 55276, 55277, 55264, 55289, 4521, 4607, 55270, 4590, 4526, + 4555, 55248, 55249, 55252, 55251, 55250, 4554, 55247, 4544, 4592, 983214, + 983211, 983212, 4593, 55285, 55286, 4594, 4601, 4449, 4510, 55238, 4511, + 55237, 4513, 4512, 4515, 4470, 4471, 4450, 4454, 4453, 4476, 4474, 4475, + 4467, 55227, 55226, 4502, 55225, 55228, 4501, 4469, 4504, 4509, 4505, + 55229, 55230, 55232, 55231, 55234, 55235, 4506, 55233, 4508, 55236, 4507, + 4514, 4457, 4482, 55217, 4481, 55216, 4518, 4519, 4480, 4479, 4483, 4460, + 4462, 4492, 55221, 55222, 4491, 4489, 4490, 4493, 4451, 4473, 4472, 4516, + 4452, 4456, 4455, 4517, 4477, 4478, 4468, 4503, 4461, 4484, 4485, 4486, + 55218, 55219, 55220, 4488, 4487, 4466, 4498, 4497, 4496, 4495, 4494, + 55223, 4500, 55224, 4499, 4464, 4463, 4458, 4459, 4465, 4448, 12623, + 12685, 12686, 12624, 12618, 12616, 12628, 12627, 12641, 12643, 12615, + 12676, 12664, 12665, 12657, 12619, 12593, 12595, 12609, 12654, 12656, + 12655, 12596, 12597, 12646, 12598, 12648, 12647, 12610, 12612, 12660, + 12661, 12663, 12659, 12662, 12658, 12621, 12671, 12601, 12602, 12649, + 12603, 12607, 12604, 12651, 12652, 12606, 12650, 12653, 12608, 12605, + 12620, 12599, 12613, 12670, 12666, 12667, 12669, 12668, 12617, 12594, + 12645, 12611, 12600, 12677, 12614, 12672, 12638, 12637, 12632, 12633, + 12639, 12630, 12629, 12678, 12673, 12675, 12674, 12635, 12679, 12680, + 12681, 12640, 12683, 12682, 12684, 12625, 12626, 12642, 12622, 12631, + 12634, 12636, 12644, 12323, 12346, 12345, 12322, 12344, 12325, 12324, + 12327, 12326, 12329, 12321, 12328, 68875, 68874, 68887, 68889, 68872, + 68881, 68868, 68867, 68877, 68876, 68890, 68891, 68885, 68880, 68879, + 68865, 68870, 68873, 68882, 68871, 68869, 68883, 68884, 68866, 68892, + 68886, 68888, 68878, 68864, 68899, 68898, 68901, 68903, 68902, 68900, + 68893, 68896, 68894, 68897, 68895, 68917, 68916, 68919, 68918, 68915, + 68914, 68912, 68921, 68913, 68920, 5925, 5928, 5930, 5927, 5924, 5937, + 5923, 5934, 5931, 5929, 5933, 5936, 5926, 5935, 5932, 5920, 5921, 5922, + 5940, 5938, 5939, 128587, 128035, 67808, 67823, 67814, 67816, 67829, + 67819, 67826, 67811, 67810, 67822, 67825, 67828, 67817, 67812, 67815, + 67818, 67809, 67821, 67813, 67824, 67820, 67835, 67839, 67838, 67837, + 67836, 128891, 128436, 11233, 129702, 9980, 127911, 127892, 128157, + 128152, 128159, 129782, 128585, 129180, 129183, 129182, 129181, 128628, + 10033, 10008, 10149, 10150, 10084, 10167, 10169, 10168, 10054, 10004, + 11096, 9955, 11095, 11097, 10152, 10144, 10135, 129055, 129051, 10077, + 10078, 128178, 128977, 10040, 128975, 10059, 128958, 10071, 10082, + 129008, 10020, 128970, 128946, 129943, 10083, 11093, 128327, 10096, + 10094, 129052, 129048, 10080, 10079, 10157, 128627, 10006, 10134, 10012, + 11094, 10030, 10097, 10095, 10137, 129054, 129050, 10140, 128635, 128940, + 10075, 10076, 128972, 128952, 128607, 128615, 10136, 128605, 128613, + 128625, 10056, 128606, 128614, 10138, 128604, 128612, 10051, 10045, + 10171, 10142, 128980, 128979, 129053, 129049, 10158, 9947, 128903, + 128913, 10132, 10173, 128633, 10133, 10074, 10010, 1442, 1453, 1447, + 1436, 1437, 1438, 1445, 1446, 1443, 1444, 1433, 1441, 1425, 1439, 1448, + 1426, 1427, 1449, 1440, 1435, 1430, 1450, 1434, 1428, 1429, 1432, 1454, + 1431, 1451, 1452, 1488, 64304, 64302, 64303, 64288, 64297, 1506, 1489, + 64332, 64305, 1499, 64333, 64315, 1508, 64334, 64324, 1498, 64314, 1507, + 64323, 1509, 1503, 1501, 1511, 64327, 1492, 64308, 1495, 1504, 64320, + 1494, 64310, 1505, 64321, 1513, 64329, 64300, 64301, 64298, 64299, 1512, + 64328, 1496, 64312, 1514, 64330, 1510, 64326, 1491, 64307, 1490, 64306, + 1500, 64316, 1502, 64318, 1493, 64331, 64309, 64293, 64289, 64296, 64295, + 64290, 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, + 1521, 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, + 64286, 1464, 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 1469, 1524, + 1523, 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, + 110594, 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, + 110784, 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, + 110755, 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, + 110773, 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, + 110765, 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, + 110618, 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, + 110650, 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, + 110642, 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, + 110638, 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, + 110809, 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, + 110791, 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, + 110800, 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, + 110742, 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, + 110733, 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, + 110726, 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, + 110877, 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, + 110841, 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, + 110837, 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, + 110848, 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, + 110659, 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, + 110660, 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, + 110677, 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, + 110702, 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, + 110717, 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, + 110698, 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, + 110696, 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, + 110860, 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, + 110873, 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, + 110813, 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, + 110827, 110828, 110819, 110820, 110821, 110822, 983271, 110607, 110608, + 110609, 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, + 110613, 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, + 19966, 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, + 19956, 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, + 19931, 19929, 19925, 19911, 19964, 19928, 19945, 19934, 19918, 19930, + 19942, 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, + 19912, 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, + 19905, 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, + 19963, 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, + 128096, 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, + 12419, 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, + 12361, 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, + 983998, 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, + 12395, 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, + 12366, 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, + 12365, 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, + 12404, 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, + 12385, 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, + 12432, 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, + 12446, 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, + 128371, 128029, 127855, 11203, 11043, 8213, 9135, 129921, 129910, 129911, + 129912, 129913, 129914, 129915, 9146, 9147, 9148, 9149, 983059, 983141, + 983138, 11134, 128677, 8230, 9897, 128014, 127943, 128052, 127798, 9749, + 9832, 127789, 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, + 128298, 8763, 129693, 983124, 983060, 983142, 983139, 128559, 128175, + 129303, 128726, 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, + 8372, 127848, 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, + 12700, 12693, 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, + 12698, 12690, 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, + 12272, 12283, 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, + 12783, 12287, 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, + 119666, 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, + 13306, 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, + 13287, 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, + 13296, 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, + 13156, 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, + 13148, 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, + 12992, 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, + 12351, 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, + 128127, 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, + 67669, 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, + 67665, 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, + 67660, 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, + 126132, 126112, 126126, 126125, 126127, 126131, 126130, 126129, 126113, + 126114, 126110, 126111, 126072, 126081, 126108, 126090, 126099, 126078, + 126105, 126069, 126087, 126096, 126077, 126104, 126068, 126086, 126095, + 126073, 126082, 126109, 126091, 126100, 126071, 126080, 126107, 126089, + 126098, 126070, 126079, 126106, 126088, 126097, 126076, 126103, 126067, + 126085, 126094, 126075, 126102, 126066, 126084, 126093, 126074, 126101, + 126119, 126118, 126121, 126120, 126117, 126116, 126123, 126115, 126122, + 126065, 126083, 126092, 126128, 126124, 8377, 8734, 10718, 983106, + 983109, 983112, 983115, 8505, 128129, 8300, 8298, 128288, 128289, 128292, + 128291, 128290, 68456, 68466, 68448, 68451, 68459, 68460, 68453, 68450, + 68454, 68462, 68464, 68465, 68457, 68452, 68455, 68458, 68449, 68461, + 68463, 68477, 68473, 68474, 68476, 68472, 68478, 68479, 68475, 68424, + 68437, 68419, 68427, 68434, 68435, 68416, 68431, 68418, 68422, 68430, + 68433, 68436, 68425, 68420, 68423, 68426, 68417, 68429, 68421, 68432, + 68428, 68445, 68441, 68442, 68444, 68440, 68446, 68447, 68443, 9088, + 8747, 10767, 10773, 10776, 10780, 10778, 10775, 10777, 10779, 10766, + 9134, 65529, 65531, 65530, 9892, 8745, 10825, 10823, 10820, 10819, 10816, + 10827, 8253, 8890, 10812, 129942, 129969, 129972, 9689, 129936, 9688, + 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, 8290, 8292, 8291, + 128229, 127982, 129311, 127875, 127983, 127971, 12292, 9979, 127886, + 128122, 128121, 128304, 43453, 43454, 43455, 43457, 43421, 43422, 43426, + 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, 43402, 43403, + 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, 43410, 43411, + 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, 43424, 43425, + 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, 43433, 43438, + 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, 43465, 43461, + 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, 43456, 43469, + 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, 43450, 43445, + 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, 43475, 43474, + 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, 10781, + 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, 69763, + 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, 69765, + 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, 69797, + 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, 69795, + 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, 69760, + 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, 69808, + 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, 69837, + 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, 12191, + 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, 12088, + 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, 12058, + 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, 12229, + 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, 12111, + 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, 12042, + 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, 12222, + 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, 12065, + 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, 12221, + 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, 12046, + 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, 12131, + 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, 12192, + 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, 12080, + 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, 12097, + 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, 12104, + 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12083, 12203, + 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, 12076, + 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, 12211, + 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, 12166, + 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, 12181, + 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, 12195, + 12069, 12079, 12206, 12051, 12232, 12132, 129432, 983205, 3251, 3250, + 3240, 3293, 3225, 3235, 3230, 3249, 3248, 3205, 3206, 3216, 3220, 3234, + 3233, 3239, 3238, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, 3253, + 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, 3227, + 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, 3294, + 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3261, 3202, 3260, 3313, + 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3267, 3268, 3298, 3299, 3265, + 3266, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, 3305, + 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, 984010, + 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, 984006, + 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, 110585, + 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, 110581, + 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, 12494, + 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, 12799, + 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, 110948, + 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, 12449, + 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, 12480, + 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, 12495, + 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, 12497, + 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, 12479, + 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, 12470, + 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, 12520, + 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, 12448, + 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, 73503, + 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73482, 73483, 73484, + 73485, 73480, 73481, 73478, 73479, 73494, 73504, 73499, 73509, 73519, + 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, 73490, + 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, 73488, + 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, 73542, + 73547, 73474, 73537, 73472, 73475, 73473, 73525, 73524, 73535, 73530, + 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, 73557, 73556, + 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, 73538, 43283, + 43295, 43299, 43301, 43277, 43281, 43284, 43275, 43274, 43286, 43285, + 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, 43292, 43287, + 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, 43311, 43308, + 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, 43268, 43271, + 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, 119506, 119499, + 119503, 119493, 119492, 119502, 119497, 119507, 119495, 119505, 119494, + 119504, 119501, 119491, 119500, 119490, 119498, 119488, 119489, 128331, + 8490, 128273, 9000, 128422, 983552, 983553, 983559, 983558, 983561, + 983560, 983557, 983556, 983554, 983563, 983555, 983562, 128287, 68163, + 68162, 68161, 68160, 68113, 68146, 68112, 68147, 68148, 68123, 68122, + 68128, 68127, 68126, 68121, 68131, 68125, 68124, 68130, 68129, 68141, + 68142, 68143, 68135, 68134, 68118, 68117, 68115, 68114, 68133, 68132, + 68149, 68140, 68145, 68119, 68139, 68136, 68138, 68137, 68144, 68096, + 68165, 68164, 68166, 68167, 68179, 68178, 68183, 68176, 68182, 68181, + 68184, 68180, 68177, 68152, 68153, 68109, 68154, 68111, 68110, 68099, + 68101, 68097, 68102, 68098, 68108, 68159, 68168, 129711, 101120, 101121, + 101122, 101123, 101124, 101125, 101126, 101127, 101128, 101129, 101130, + 101131, 101132, 101133, 101134, 101135, 101136, 101137, 101138, 101139, + 101140, 101141, 101142, 101143, 101144, 101145, 101146, 101147, 101148, + 101149, 101150, 101151, 101152, 101153, 101154, 101155, 101156, 101157, + 101158, 101159, 101160, 101161, 101162, 101163, 101164, 101165, 101166, + 101167, 101168, 101169, 101170, 101171, 101172, 101173, 101174, 101175, + 101176, 101177, 101178, 101179, 101180, 101181, 101182, 101183, 101184, + 101185, 101186, 101187, 101188, 101189, 101190, 101191, 101192, 101193, + 101194, 101195, 101196, 101197, 101198, 101199, 101200, 101201, 101202, + 101203, 101204, 101205, 101206, 101207, 101208, 101209, 101210, 101211, + 101212, 101213, 101214, 101215, 101216, 101217, 101218, 101219, 101220, + 101221, 101222, 101223, 101224, 101225, 101226, 101227, 101228, 101229, + 101230, 101231, 101232, 101233, 101234, 101235, 101236, 101237, 101238, + 101239, 101240, 101241, 101242, 101243, 101244, 101245, 101246, 101247, + 101248, 101249, 101250, 101251, 101252, 101253, 101254, 101255, 101256, + 101257, 101258, 101259, 101260, 101261, 101262, 101263, 101264, 101265, + 101266, 101267, 101268, 101269, 101270, 101271, 101272, 101273, 101274, + 101275, 101276, 101277, 101278, 101279, 101280, 101281, 101282, 101283, + 101284, 101285, 101286, 101287, 101288, 101289, 101290, 101291, 101292, + 101293, 101294, 101295, 101296, 101297, 101298, 101299, 101300, 101301, + 101302, 101303, 101304, 101305, 101306, 101307, 101308, 101309, 101310, + 101311, 101312, 101313, 101314, 101315, 101316, 101317, 101318, 101319, + 101320, 101321, 101322, 101323, 101324, 101325, 101326, 101327, 101328, + 101329, 101330, 101331, 101332, 101333, 101334, 101335, 101336, 101337, + 101338, 101339, 101340, 101341, 101342, 101343, 101344, 101345, 101346, + 101347, 101348, 101349, 101350, 101351, 101352, 101353, 101354, 101355, + 101356, 101357, 101358, 101359, 101360, 101361, 101362, 101363, 101364, + 101365, 101366, 101367, 101368, 101369, 101370, 101371, 101372, 101373, + 101374, 101375, 101584, 101585, 101586, 101587, 101588, 101589, 101376, + 101377, 101378, 101379, 101380, 101381, 101382, 101383, 101384, 101385, + 101386, 101387, 101388, 101389, 101390, 101391, 101392, 101393, 101394, + 101395, 101396, 101397, 101398, 101399, 101400, 101401, 101402, 101403, + 101404, 101405, 101406, 101407, 101408, 101409, 101410, 101411, 101412, + 101413, 101414, 101415, 101416, 101417, 101418, 101419, 101420, 101421, + 101422, 101423, 101424, 101425, 101426, 101427, 101428, 101429, 101430, + 101431, 101432, 101433, 101434, 101435, 101436, 101437, 101438, 101439, + 101440, 101441, 101442, 101443, 101444, 101445, 101446, 101447, 101448, + 101449, 101450, 101451, 101452, 101453, 101454, 101455, 101456, 101457, + 101458, 101459, 101460, 101461, 101462, 101463, 101464, 101465, 101466, + 101467, 101468, 101469, 101470, 101471, 101472, 101473, 101474, 101475, + 101476, 101477, 101478, 101479, 101480, 101481, 101482, 101483, 101484, + 101485, 101486, 101487, 101488, 101489, 101490, 101491, 101492, 101493, + 101494, 101495, 101496, 101497, 101498, 101499, 101500, 101501, 101502, + 101503, 101504, 101505, 101506, 101507, 101508, 101509, 101510, 101511, + 101512, 101513, 101514, 101515, 101516, 101517, 101518, 101519, 101520, + 101521, 101522, 101523, 101524, 101525, 101526, 101527, 101528, 101529, + 101530, 101531, 101532, 101533, 101534, 101535, 101536, 101537, 101538, + 101539, 101540, 101541, 101542, 101543, 101544, 101545, 101546, 101547, + 101548, 101549, 101550, 101551, 101552, 101553, 101554, 101555, 101556, + 101557, 101558, 101559, 101560, 101561, 101562, 101563, 101564, 101565, + 101566, 101567, 101568, 101569, 101570, 101571, 101572, 101573, 101574, + 101575, 101576, 101577, 101578, 101579, 101580, 101581, 101582, 101583, + 94180, 983960, 983965, 983970, 983975, 983962, 983964, 983961, 983963, + 983957, 983959, 983956, 983958, 983977, 983979, 983978, 983972, 983974, + 983967, 983969, 983971, 983973, 983966, 983968, 983989, 983983, 983985, + 983986, 983987, 983980, 983982, 983984, 983981, 983976, 983988, 6107, + 6052, 6064, 6051, 6067, 6066, 6065, 6055, 6057, 6058, 6056, 6053, 6054, + 6063, 983994, 983991, 983992, 983993, 6061, 6062, 6059, 6060, 6022, 6024, + 6021, 6023, 6017, 6019, 6016, 6018, 6020, 6030, 6025, 6035, 6037, 6039, + 6038, 6046, 6045, 6047, 6032, 6034, 6027, 6029, 6031, 6033, 6026, 6028, + 6049, 6043, 6040, 6042, 6044, 6041, 6036, 6048, 6050, 6108, 6109, 6095, + 6096, 6099, 6091, 6101, 6104, 6102, 6098, 6094, 6100, 6106, 6092, 6087, + 6093, 6090, 6103, 6105, 6088, 6086, 6089, 6097, 6652, 6636, 6655, 6639, + 6653, 6637, 6654, 6638, 6651, 6635, 6650, 6634, 6133, 6137, 6136, 6134, + 6135, 6130, 6132, 6131, 6129, 6128, 6624, 6648, 6632, 6649, 6633, 6647, + 6631, 6646, 6630, 6645, 6629, 6642, 6626, 6640, 6643, 6627, 6644, 6628, + 6641, 6625, 6069, 6068, 6070, 983996, 6082, 6083, 6085, 6071, 6080, 6072, + 6078, 983995, 6084, 6073, 6079, 6074, 983990, 6075, 6077, 6076, 6081, + 6117, 6116, 6119, 6118, 6115, 6114, 6112, 6121, 6113, 6120, 70204, 70201, + 70200, 70208, 70185, 70178, 70179, 70177, 70155, 70156, 70154, 70172, + 70167, 70166, 70173, 70171, 70161, 70160, 70144, 70145, 70149, 70151, + 70165, 70164, 70170, 70169, 70187, 70183, 70157, 70168, 70163, 70174, + 70159, 70158, 70153, 70152, 70176, 70175, 70186, 70180, 70207, 70182, + 70184, 70181, 70148, 70146, 70150, 70147, 70199, 70206, 70197, 70198, + 70196, 70203, 70209, 70188, 70193, 70195, 70189, 70190, 70192, 70194, + 70191, 70202, 70205, 70357, 70358, 70356, 70333, 70334, 70332, 70340, + 70339, 70338, 70345, 70347, 70344, 70352, 70351, 70346, 70361, 70320, + 70321, 70327, 70329, 70343, 70342, 70350, 70349, 70324, 70325, 70322, + 70323, 70335, 70348, 70341, 70353, 70337, 70336, 70331, 70330, 70355, + 70354, 70364, 70365, 70366, 70362, 70359, 70363, 70360, 70326, 70328, + 70378, 70377, 70367, 70368, 70374, 70376, 70371, 70372, 70369, 70370, + 70373, 70375, 70389, 70388, 70391, 70390, 70387, 70386, 70384, 70393, + 70385, 70392, 128143, 128535, 128537, 128538, 128573, 128139, 129373, + 8365, 128088, 129665, 129486, 129698, 12927, 128040, 11235, 129404, + 127991, 129357, 128030, 129692, 3805, 3804, 983206, 983207, 3743, 3741, + 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, 3730, 3729, 3736, 3728, + 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, 3740, 3742, 3739, 3752, + 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, 3719, 3725, 3737, 3738, + 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, 3772, 3770, 3785, 3786, + 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, 3771, 3766, 3767, 3768, + 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, 3759, 3797, 3796, 3799, + 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, 3789, 128996, 129003, + 128309, 128311, 128998, 128994, 129001, 68413, 68415, 128992, 128310, + 128999, 68412, 68414, 11004, 10201, 10200, 10782, 128995, 129002, 128308, + 128997, 128993, 129000, 9711, 10923, 10925, 8382, 9790, 127772, 127767, + 65, 258, 7858, 7856, 7854, 7862, 7860, 550, 480, 7840, 512, 196, 478, + 197, 506, 7680, 256, 983564, 194, 7848, 7846, 7844, 7852, 7850, 461, + 7842, 260, 983590, 983592, 192, 193, 514, 195, 570, 198, 508, 482, 42946, + 11373, 393, 42808, 42810, 42802, 42804, 42806, 42812, 66, 386, 42902, + 7686, 7684, 7682, 385, 579, 42822, 42932, 67, 199, 7688, 264, 268, 262, + 42948, 391, 42898, 266, 571, 42960, 42796, 42798, 42862, 42931, 68, 498, + 453, 42951, 272, 7696, 7698, 270, 395, 7694, 7692, 7690, 394, 497, 452, + 69, 552, 7708, 202, 983586, 7874, 7872, 7870, 983584, 7878, 7876, 7704, + 282, 278, 983598, 983600, 7864, 516, 203, 7868, 7706, 274, 7700, 7702, + 983570, 983572, 983574, 7866, 280, 983594, 983596, 200, 201, 518, 276, + 582, 439, 494, 440, 42786, 42788, 42858, 208, 425, 330, 70, 401, 7710, + 42904, 71, 290, 284, 486, 500, 286, 7712, 42912, 403, 288, 484, 577, + 42938, 42940, 42942, 404, 983199, 72, 7722, 7720, 292, 542, 7718, 11367, + 7716, 7714, 42922, 294, 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, + 206, 463, 298, 983566, 296, 7724, 7880, 302, 983604, 983606, 204, 205, + 522, 300, 407, 42873, 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, + 308, 42930, 983608, 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, + 7728, 7732, 42914, 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, + 456, 321, 315, 7740, 317, 42824, 313, 7738, 983610, 455, 77, 7742, 7746, + 7744, 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, + 7752, 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, + 42828, 332, 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, + 554, 558, 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, + 983576, 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, + 211, 526, 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, + 42832, 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, + 7772, 7768, 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, + 42923, 42814, 398, 42844, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, + 7784, 7776, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, 7838, + 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, 430, + 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, 222, + 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, 473, + 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, 7918, + 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, 7800, + 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, 7804, + 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, 7814, + 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, 7922, + 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, 11371, + 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, 43005, + 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, 446, + 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, 7436, + 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, + 122626, 122640, 43002, 7450, 665, 7427, 610, 667, 7424, 7425, 7428, 7429, + 42800, 668, 7434, 7435, 7437, 628, 7448, 42927, 42801, 7452, 7456, 7457, + 655, 7458, 663, 122639, 42895, 447, 443, 448, 449, 660, 673, 661, 7460, + 422, 7547, 7550, 97, 259, 7859, 7857, 7855, 7863, 7861, 551, 481, 7841, + 513, 228, 479, 229, 507, 7681, 7834, 7567, 257, 983565, 226, 7849, 7847, + 7845, 7853, 7851, 462, 7843, 261, 983591, 983593, 224, 225, 515, 227, + 11365, 43825, 230, 983624, 509, 483, 42947, 593, 7568, 42809, 42811, + 42803, 42805, 42807, 42813, 98, 387, 42903, 7687, 7532, 7552, 7685, 7683, + 595, 384, 43824, 43827, 629, 43853, 43837, 43838, 43826, 42823, 7447, + 42933, 99, 231, 7689, 265, 597, 269, 263, 42900, 122653, 392, 42899, 267, + 572, 43859, 43860, 43861, 42961, 666, 631, 606, 42797, 42799, 42863, 100, + 42952, 273, 396, 598, 7697, 7699, 545, 271, 122661, 7533, 7695, 599, + 7569, 7553, 7693, 7691, 676, 122642, 122649, 7839, 567, 607, 644, 305, + 42965, 43848, 43850, 42963, 499, 454, 675, 677, 43878, 568, 42865, 101, + 553, 7709, 234, 983587, 7875, 7873, 7871, 983585, 7879, 7877, 7705, 283, + 279, 983599, 983601, 7865, 517, 235, 11384, 7869, 7707, 275, 7701, 7703, + 983571, 983573, 983575, 43828, 7867, 281, 983595, 983597, 232, 233, 519, + 277, 7570, 583, 42787, 42789, 331, 43836, 122644, 643, 122635, 122636, + 7563, 646, 7576, 658, 441, 659, 495, 122648, 7578, 442, 42859, 240, 102, + 7534, 7554, 402, 7711, 42905, 681, 122624, 103, 291, 285, 487, 501, 287, + 7713, 7555, 42913, 608, 289, 485, 578, 42939, 42941, 42943, 611, 983200, + 104, 7723, 7721, 293, 543, 7719, 11368, 7717, 7715, 7830, 42901, 614, + 295, 11382, 983631, 983632, 42791, 615, 405, 105, 238, 464, 239, 7727, + 983602, 983588, 983603, 7883, 521, 299, 983567, 303, 983605, 983607, 297, + 7725, 7881, 236, 237, 523, 301, 616, 122650, 7574, 42874, 42876, 7545, + 42883, 42885, 42887, 43876, 43840, 617, 7548, 43873, 42861, 106, 309, + 669, 496, 983609, 585, 107, 311, 489, 42819, 11370, 7731, 42817, 42821, + 7729, 7733, 7556, 42915, 409, 312, 108, 620, 122643, 410, 316, 7741, 564, + 318, 7735, 7737, 43832, 11361, 619, 43833, 320, 122662, 42825, 314, 7739, + 43831, 621, 42894, 122641, 7557, 983611, 322, 411, 622, 122629, 43829, + 383, 7836, 7835, 7837, 682, 683, 42866, 457, 109, 7743, 43834, 7535, + 7558, 7747, 7745, 983613, 625, 7931, 7933, 42967, 42867, 110, 326, 7755, + 43835, 565, 328, 414, 7753, 626, 122663, 7536, 505, 324, 42897, 7751, + 7749, 7559, 627, 42917, 241, 329, 983589, 42868, 460, 111, 244, 7893, + 7891, 7889, 7897, 7895, 466, 246, 555, 559, 561, 7885, 525, 337, 42827, + 11386, 42829, 333, 7761, 7763, 491, 493, 248, 511, 245, 7759, 7757, 557, + 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, 7887, 242, + 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, 43839, 43874, + 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, 42837, 7765, + 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, 569, 114, 343, + 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, 7775, 636, 637, + 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, 42843, 42998, + 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, 122627, 42869, + 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, 122654, 7779, 7785, + 7777, 42954, 575, 983583, 122665, 7540, 7562, 42921, 642, 42892, 43872, + 601, 983629, 983630, 7573, 602, 43851, 43852, 609, 43830, 223, 7441, + 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, 116, 355, 7793, 539, + 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, 7791, 427, 429, 122633, + 648, 359, 679, 122647, 122652, 7546, 254, 42853, 42855, 389, 424, 445, + 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, 43842, 43841, 43843, + 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, 635, 647, 122637, + 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, 122630, 43857, 477, + 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, 42793, 117, 649, + 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, 533, 369, 7909, + 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, 983569, 983621, + 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, 249, 43854, + 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, 43875, 118, + 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, 42909, 42911, + 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, 11379, 120, + 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, 7925, 7823, + 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, 122, 378, + 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, 656, 549, + 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, 8347, 8340, + 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, 8346, 7523, + 8348, 7524, 7525, 8339, 917505, 127811, 129388, 129897, 129916, 129899, + 129917, 129947, 10203, 10202, 128494, 12296, 10641, 10643, 11058, 11056, + 10576, 10571, 10570, 10574, 12304, 10647, 123, 9128, 9129, 9127, 12300, + 8968, 9948, 10714, 12298, 8220, 11816, 11780, 129287, 129284, 129285, + 129283, 129286, 9686, 11240, 9612, 129977, 129970, 129940, 129932, + 128379, 128709, 11804, 10204, 10197, 9614, 9615, 8596, 8700, 8697, 8622, + 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, 129112, 11108, 11788, + 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, 9121, 10639, 10637, + 8261, 10635, 11863, 11861, 9122, 11778, 10703, 129900, 11785, 129985, + 128492, 9613, 9610, 12308, 8867, 12302, 10627, 12310, 10629, 12314, + 12312, 10712, 128398, 9611, 10620, 8970, 8905, 40, 9117, 9115, 9116, + 11808, 9144, 10553, 10154, 1422, 4054, 4056, 129307, 9958, 10748, 9001, + 171, 128269, 8294, 8237, 8234, 8206, 8592, 10563, 11074, 11083, 11082, + 10611, 129973, 10618, 10615, 11070, 8676, 8633, 10525, 11064, 8698, + 129032, 8619, 10566, 129040, 129024, 8602, 11024, 11025, 8610, 11066, + 11065, 129028, 129176, 129044, 8617, 8695, 8612, 10527, 8646, 10521, + 129192, 129184, 11144, 11013, 10603, 10599, 10590, 10582, 8637, 10594, + 10602, 10598, 10586, 10578, 8636, 8651, 129778, 129088, 129092, 11104, + 11136, 11130, 983241, 11174, 11172, 129064, 129060, 129056, 129072, + 129068, 11120, 11114, 11140, 129168, 10510, 8666, 129186, 11067, 11069, + 11068, 11244, 11061, 11060, 11062, 11063, 8606, 8678, 129172, 8604, 8656, + 10498, 8653, 10502, 10523, 10508, 8672, 129194, 129076, 129188, 8701, + 8647, 129783, 129190, 128620, 8668, 129080, 129104, 129084, 11077, 9804, + 128006, 7213, 7221, 7216, 7220, 7215, 7214, 7217, 7218, 7219, 7170, 7169, + 7168, 7184, 7183, 7182, 7247, 7193, 7180, 7188, 7187, 7186, 7185, 7172, + 7171, 7198, 7197, 7190, 7189, 7173, 7177, 7181, 7192, 7191, 7246, 7245, + 7179, 7178, 7175, 7174, 7201, 7200, 7176, 7196, 7195, 7199, 7202, 7194, + 7203, 7227, 7231, 7230, 7228, 7229, 7223, 7222, 7205, 7204, 7210, 7211, + 7208, 7209, 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, + 7241, 7233, 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, + 8922, 8934, 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, + 8804, 8822, 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, + 128626, 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, + 9617, 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, + 128504, 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, + 6410, 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, + 6409, 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, + 6457, 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, + 6442, 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, + 6400, 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, + 6478, 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, + 67146, 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, + 67162, 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, + 67175, 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, + 67183, 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, + 67193, 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, + 67203, 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, + 67213, 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, + 67223, 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, + 67235, 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, + 67245, 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, + 67255, 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, + 67274, 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, + 67284, 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, + 67294, 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, + 67304, 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, + 67325, 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, + 67309, 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, + 67319, 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, + 67335, 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, + 67355, 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, + 67365, 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, + 67370, 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, + 67341, 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, + 67403, 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, + 67396, 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, + 67424, 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, + 67083, 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, + 67094, 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, + 67104, 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, + 67114, 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, + 67124, 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, + 67134, 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, + 67073, 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, + 67148, 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, + 65667, 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, + 65676, 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, + 65685, 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, + 65697, 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, + 65709, 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, + 65718, 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, + 65729, 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, + 65739, 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, + 65749, 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, + 65781, 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, + 65764, 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, + 65774, 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, + 65688, 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, + 65579, 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, + 65560, 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, + 65609, 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, + 65583, 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, + 65570, 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, + 65539, 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, + 65558, 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, + 65624, 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, + 65625, 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, + 42233, 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, + 42224, 42225, 42213, 42208, 42203, 42202, 42221, 42198, 42216, 42214, + 42200, 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, + 42222, 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, + 42215, 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, + 983079, 983076, 128271, 8743, 10848, 10846, 10833, 10844, 10842, 10847, + 8744, 10841, 10851, 10850, 10834, 10845, 10843, 10982, 10188, 129688, + 10234, 10231, 10206, 10229, 10235, 10232, 10237, 11059, 10230, 10236, + 10233, 10238, 10239, 10205, 129524, 128884, 129719, 128140, 127977, + 128261, 129707, 12319, 8270, 11847, 11848, 95, 9691, 129935, 9604, 9697, + 128394, 129852, 129853, 129870, 129872, 129868, 129856, 129871, 129869, + 129854, 129873, 129855, 128395, 128396, 128393, 10559, 129951, 9722, + 128397, 9695, 10558, 128318, 10065, 129950, 9727, 129863, 129865, 129867, + 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, 10195, + 10063, 9998, 9694, 9987, 9605, 9602, 9601, 9607, 9603, 9606, 129903, + 9674, 10208, 129438, 128557, 127853, 129523, 128886, 129729, 66190, + 66192, 66187, 66196, 66178, 66179, 66199, 66185, 66200, 66176, 66201, + 66177, 66202, 66191, 66193, 66181, 66180, 66203, 66182, 66186, 66189, + 66195, 66188, 66197, 66198, 66194, 66183, 66204, 66184, 67887, 67892, + 67881, 67895, 67891, 67886, 67872, 67893, 67876, 67894, 67883, 67896, + 67873, 67897, 67875, 67889, 67874, 67878, 67880, 67882, 67884, 67890, + 67885, 67888, 67877, 67879, 67903, 129317, 983226, 983234, 983224, + 983229, 8468, 129433, 983065, 129668, 129522, 129497, 69986, 69981, + 69991, 70002, 69997, 69985, 69984, 69990, 69989, 69983, 69982, 69988, + 69987, 69995, 69994, 69978, 69977, 69976, 69975, 69980, 69979, 69974, + 69973, 69993, 69992, 70001, 69998, 69996, 70000, 69999, 69968, 69971, + 69969, 69972, 69970, 70006, 70005, 70003, 70004, 127012, 127019, 127008, + 126990, 126999, 126976, 127005, 126987, 126996, 127004, 126986, 126995, + 126979, 127009, 126991, 127000, 127001, 126983, 126992, 127011, 127010, + 126977, 127007, 126989, 126998, 127006, 126988, 126997, 127015, 127014, + 127003, 126985, 126994, 127002, 126984, 126993, 126978, 126982, 127017, + 126981, 126980, 127016, 127018, 127013, 73464, 73442, 73451, 73448, + 73444, 73449, 73447, 73441, 73450, 73440, 73454, 73445, 73443, 73453, + 73456, 73446, 73455, 73452, 73457, 73463, 73461, 73459, 73462, 73460, + 73458, 128892, 3415, 3449, 3435, 3434, 3437, 3436, 3433, 3432, 3430, + 3439, 3431, 3438, 3419, 3420, 3446, 3417, 3422, 3416, 3447, 3444, 3443, + 3448, 3418, 3421, 3445, 3333, 3423, 3334, 3344, 3348, 3453, 3454, 3414, + 3451, 3450, 3452, 3455, 3412, 3413, 3355, 3354, 3406, 3362, 3361, 3367, + 3366, 3360, 3386, 3359, 3365, 3364, 3332, 3339, 3424, 3340, 3425, 3381, + 3369, 3363, 3353, 3358, 3368, 3380, 3379, 3378, 3377, 3376, 3337, 3338, + 3346, 3347, 3335, 3336, 3382, 3383, 3384, 3373, 3372, 3352, 3351, 3357, + 3356, 3350, 3349, 3371, 3370, 3342, 3343, 3385, 3374, 3375, 3441, 3442, + 3440, 3328, 3388, 3329, 3387, 3405, 3331, 3389, 3330, 3407, 3390, 3400, + 3404, 3395, 3396, 3426, 3427, 3393, 3394, 3402, 3403, 3391, 3392, 3398, + 3399, 9895, 9894, 9893, 9794, 10016, 129443, 128104, 128372, 129333, + 128114, 128115, 128378, 128107, 2122, 2121, 2133, 2120, 2126, 2132, 2129, + 2136, 2113, 2115, 2114, 2116, 2123, 2124, 2125, 2128, 2130, 2131, 2118, + 2134, 2117, 2112, 2127, 2119, 2135, 2139, 2138, 2142, 2137, 68288, 68314, + 68313, 68290, 68289, 68293, 68308, 68292, 68291, 68300, 68299, 68298, + 68297, 68306, 68304, 68320, 68318, 68323, 68312, 68317, 68322, 68309, + 68302, 68324, 68305, 68319, 68307, 68321, 68303, 68294, 68301, 68311, + 68295, 68316, 68315, 68310, 68331, 68335, 68334, 68333, 68332, 68340, + 68339, 68338, 68342, 68337, 68341, 68336, 68296, 68326, 68325, 128094, + 129469, 8380, 128368, 129389, 9967, 127809, 129671, 72835, 72834, 72827, + 72826, 72836, 72828, 72821, 72825, 72829, 72823, 72822, 72819, 72818, + 72831, 72830, 72844, 72845, 72838, 72839, 72840, 72832, 72820, 72846, + 72824, 72843, 72833, 72842, 72837, 72841, 72847, 72886, 72885, 72867, + 72866, 72859, 72858, 72868, 72860, 72853, 72857, 72861, 72855, 72854, + 72851, 72850, 72863, 72862, 72876, 72877, 72870, 72871, 72864, 72852, + 72878, 72856, 72875, 72865, 72874, 72869, 72873, 72879, 72880, 72883, + 72881, 72884, 72882, 72816, 72817, 9901, 129355, 73007, 72980, 72979, + 72983, 72982, 72988, 73008, 72987, 72960, 72961, 72968, 72971, 72985, + 72984, 72990, 72989, 72964, 72965, 72962, 72963, 73005, 72999, 73006, + 72973, 72972, 72976, 72986, 72981, 72991, 73001, 73002, 73003, 72995, + 72994, 72978, 72977, 72975, 72974, 72993, 72992, 73004, 72996, 72998, + 73000, 72997, 72966, 72969, 73031, 73030, 73027, 73028, 73026, 73025, + 73024, 73014, 73009, 73020, 73023, 73012, 73013, 73010, 73011, 73018, + 73021, 73029, 73045, 73044, 73047, 73046, 73043, 73042, 73040, 73049, + 73041, 73048, 186, 127405, 12348, 119811, 120778, 120491, 119827, 120495, + 120505, 120507, 119808, 120488, 119809, 120489, 119833, 120493, 119812, + 120492, 120494, 119814, 120490, 119816, 120496, 119818, 120497, 119819, + 120498, 119822, 120502, 120512, 119823, 120509, 120511, 120503, 119825, + 120504, 119826, 120506, 119828, 120508, 119810, 120510, 119820, 120499, + 119821, 120500, 119831, 120501, 119813, 119815, 119817, 119824, 119829, + 119830, 119832, 119837, 120779, 120517, 119834, 120514, 119835, 120515, + 119859, 120519, 119838, 120518, 120520, 119839, 120531, 119840, 120516, + 119842, 120522, 119844, 120523, 119845, 120524, 119848, 120528, 120538, + 119849, 120535, 120537, 120529, 119851, 120530, 119852, 120532, 119853, + 120521, 120533, 119854, 120534, 119836, 120536, 119846, 120525, 119847, + 120526, 119857, 120527, 119841, 119843, 119850, 119855, 119856, 119858, + 120016, 120017, 120018, 120019, 120020, 120021, 120022, 120023, 120024, + 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033, + 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, + 120043, 120044, 120045, 120046, 120047, 120048, 120049, 120050, 120051, + 120052, 120053, 120054, 120055, 120056, 120057, 120058, 120059, 120060, + 120061, 120062, 120063, 120064, 120065, 120066, 120067, 119931, 120611, + 120621, 120623, 119912, 120604, 119913, 120605, 119937, 120609, 119915, + 120607, 119916, 120608, 120610, 119918, 120606, 119920, 120612, 119922, + 120613, 119923, 120614, 119926, 120618, 120628, 119927, 120625, 120627, + 120619, 119929, 120620, 119930, 120622, 119932, 120624, 119914, 120626, + 119924, 120615, 119925, 120616, 119935, 120617, 119917, 119919, 119921, + 119928, 119933, 119934, 119936, 120656, 120658, 120629, 120659, 120655, + 120661, 120660, 119938, 120630, 119939, 120631, 119963, 120635, 119941, + 120633, 119942, 120634, 120636, 119943, 120647, 119944, 120632, 119946, + 120638, 119948, 120639, 119949, 120640, 119952, 120644, 120654, 119953, + 120651, 120653, 120645, 119955, 120646, 119956, 120648, 119957, 120637, + 120649, 119958, 120650, 119940, 120652, 119950, 120641, 119951, 120642, + 119961, 120643, 119945, 119947, 119954, 119959, 119960, 119962, 120657, + 120540, 120542, 120513, 120543, 120539, 120545, 120544, 120541, 120172, + 120173, 120174, 120175, 120176, 120177, 120178, 120179, 120180, 120181, + 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189, 120190, + 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, + 120200, 120201, 120202, 120203, 120204, 120205, 120206, 120207, 120208, + 120209, 120210, 120211, 120212, 120213, 120214, 120215, 120216, 120217, + 120218, 120219, 120220, 120221, 120222, 120223, 120787, 120786, 120789, + 120788, 120785, 120784, 120782, 120791, 120783, 120790, 120120, 120121, + 120123, 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, + 120134, 120138, 120139, 120140, 120141, 120142, 120143, 120144, 120146, + 120147, 120148, 120149, 120150, 120151, 120152, 120153, 120154, 120155, + 120156, 120157, 120158, 120159, 120160, 120161, 120162, 120163, 120164, + 120165, 120166, 120167, 120168, 120169, 120170, 120171, 120797, 120796, + 120799, 120798, 120795, 120794, 120792, 120801, 120793, 120800, 120068, + 120069, 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, + 120081, 120082, 120083, 120084, 120086, 120087, 120088, 120089, 120090, + 120091, 120092, 120094, 120095, 120096, 120097, 120098, 120099, 120100, + 120101, 120102, 120103, 120104, 120105, 120106, 120107, 120108, 120109, + 120110, 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, + 120119, 10189, 119889, 120484, 120485, 120575, 119886, 120572, 119887, + 120573, 119911, 120577, 119890, 120576, 120578, 119891, 120589, 119892, + 120574, 119894, 120580, 119896, 120581, 119897, 120582, 119900, 120586, + 120596, 119901, 120593, 120595, 120587, 119903, 120588, 119904, 120590, + 119905, 120579, 120591, 119906, 120592, 119888, 120594, 119898, 120583, + 119899, 120584, 119909, 120585, 119895, 119902, 119907, 119908, 119910, + 119879, 120553, 120563, 120565, 119860, 120546, 119861, 120547, 119885, + 120551, 119863, 120549, 119864, 120550, 120552, 119866, 120548, 119868, + 120554, 119870, 120555, 119871, 120556, 119874, 120560, 120570, 119875, + 120567, 120569, 120561, 119877, 120562, 119878, 120564, 119880, 120566, + 119862, 120568, 119872, 120557, 119873, 120558, 119883, 120559, 119865, + 119867, 119869, 119876, 119881, 119882, 119884, 120598, 120600, 120571, + 120601, 120597, 120603, 120602, 120599, 120432, 120433, 120434, 120435, + 120436, 120437, 120438, 120439, 120440, 120441, 120442, 120443, 120444, + 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453, + 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, + 120463, 120464, 120465, 120466, 120467, 120468, 120469, 120470, 120471, + 120472, 120473, 120474, 120475, 120476, 120477, 120478, 120479, 120480, + 120481, 120482, 120483, 120827, 120826, 120829, 120828, 120825, 120824, + 120822, 120831, 120823, 120830, 10215, 10221, 10219, 10217, 10223, 10187, + 10214, 10220, 10218, 10216, 10222, 120399, 120727, 120737, 120739, + 120380, 120720, 120381, 120721, 120405, 120725, 120383, 120723, 120384, + 120724, 120726, 120386, 120722, 120388, 120728, 120390, 120729, 120391, + 120730, 120394, 120734, 120744, 120395, 120741, 120743, 120735, 120397, + 120736, 120398, 120738, 120400, 120740, 120382, 120742, 120392, 120731, + 120393, 120732, 120403, 120733, 120385, 120387, 120389, 120396, 120401, + 120402, 120404, 120772, 120774, 120745, 120775, 120771, 120777, 120776, + 120406, 120746, 120407, 120747, 120431, 120751, 120409, 120749, 120410, + 120750, 120752, 120411, 120763, 120412, 120748, 120414, 120754, 120416, + 120755, 120417, 120756, 120420, 120760, 120770, 120421, 120767, 120769, + 120761, 120423, 120762, 120424, 120764, 120425, 120753, 120765, 120426, + 120766, 120408, 120768, 120418, 120757, 120419, 120758, 120429, 120759, + 120413, 120415, 120422, 120427, 120428, 120430, 120773, 120295, 120669, + 120679, 120681, 120276, 120662, 120277, 120663, 120301, 120667, 120279, + 120665, 120280, 120666, 120668, 120282, 120664, 120284, 120670, 120286, + 120671, 120287, 120672, 120290, 120676, 120686, 120291, 120683, 120685, + 120677, 120293, 120678, 120294, 120680, 120296, 120682, 120278, 120684, + 120288, 120673, 120289, 120674, 120299, 120675, 120281, 120283, 120285, + 120292, 120297, 120298, 120300, 120714, 120716, 120687, 120717, 120713, + 120719, 120718, 120302, 120688, 120303, 120689, 120327, 120693, 120305, + 120691, 120306, 120692, 120694, 120307, 120705, 120308, 120690, 120310, + 120696, 120312, 120697, 120313, 120698, 120316, 120702, 120712, 120317, + 120709, 120711, 120703, 120319, 120704, 120320, 120706, 120321, 120695, + 120707, 120322, 120708, 120304, 120710, 120314, 120699, 120315, 120700, + 120325, 120701, 120309, 120311, 120318, 120323, 120324, 120326, 120715, + 120817, 120816, 120819, 120818, 120815, 120814, 120812, 120821, 120813, + 120820, 120328, 120329, 120330, 120331, 120332, 120333, 120334, 120335, + 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, + 120345, 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, + 120354, 120355, 120356, 120357, 120358, 120359, 120360, 120361, 120362, + 120363, 120364, 120365, 120366, 120367, 120368, 120369, 120370, 120371, + 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120224, + 120225, 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, + 120234, 120235, 120236, 120237, 120238, 120239, 120240, 120241, 120242, + 120243, 120244, 120245, 120246, 120247, 120248, 120249, 120250, 120251, + 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, + 120261, 120262, 120263, 120264, 120265, 120266, 120267, 120268, 120269, + 120270, 120271, 120272, 120273, 120274, 120275, 120807, 120806, 120809, + 120808, 120805, 120804, 120802, 120811, 120803, 120810, 119964, 119966, + 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982, + 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, + 119992, 119993, 119995, 119997, 119998, 119999, 120000, 120001, 120002, + 120003, 120005, 120006, 120007, 120008, 120009, 120010, 120011, 120012, + 120013, 120014, 120015, 129481, 119528, 119538, 119531, 119535, 119525, + 119524, 119534, 119529, 119539, 119527, 119537, 119526, 119536, 119533, + 119523, 119532, 119522, 119530, 119520, 119521, 128470, 175, 8737, 10667, + 10666, 10671, 10669, 10670, 10668, 10665, 10664, 10651, 10653, 8798, + 127830, 129470, 129471, 93773, 93764, 93790, 93787, 983268, 93783, + 983267, 93782, 93772, 93766, 93791, 93779, 93789, 93786, 93776, 93777, + 93785, 93775, 93770, 93769, 93771, 93774, 93780, 93760, 93767, 93781, + 93788, 93761, 93768, 93778, 93762, 93763, 93784, 93765, 93847, 93827, + 93846, 93826, 93845, 93825, 93844, 93829, 93828, 93831, 93830, 93824, + 93833, 93832, 93837, 93836, 93834, 93842, 93835, 93838, 93839, 93843, + 93840, 93841, 93805, 93796, 93822, 93819, 983270, 93815, 983269, 93814, + 93804, 93798, 93823, 93811, 93821, 93818, 93808, 93809, 93817, 93807, + 93802, 93801, 93803, 93806, 93812, 93792, 93799, 93813, 93820, 93793, + 93800, 93810, 93794, 93795, 93816, 93797, 93849, 93850, 93848, 11859, + 11852, 11860, 128901, 9899, 10090, 10091, 128967, 128965, 128944, 10100, + 10088, 10092, 10101, 10089, 10093, 8287, 9900, 9618, 128971, 128950, + 128938, 128963, 128961, 10073, 128974, 128956, 9898, 128911, 128931, + 43761, 44013, 43762, 43760, 44011, 43994, 43989, 43974, 43746, 43993, + 43991, 43751, 43750, 43992, 43986, 43987, 43990, 43968, 43995, 43976, + 43973, 43999, 43977, 44001, 43752, 43747, 43972, 43998, 43984, 43969, + 43753, 43754, 43975, 44000, 43978, 43749, 43748, 43983, 44002, 43970, + 43996, 43971, 43997, 43981, 43988, 43979, 43985, 43980, 43982, 43744, + 43745, 44012, 43763, 43764, 44005, 43757, 43759, 43758, 44009, 44003, + 44007, 44006, 44004, 43755, 44008, 43756, 44010, 43765, 43766, 44021, + 44020, 44023, 44022, 44019, 44018, 44016, 44025, 44017, 44024, 129760, + 127816, 125140, 125137, 125136, 125139, 125141, 125138, 125142, 124928, + 124929, 124930, 124936, 124937, 124949, 124950, 124948, 124938, 124956, + 124990, 124992, 124974, 124955, 124964, 124963, 124991, 124957, 124962, + 124976, 124996, 124997, 124998, 124982, 124983, 124984, 125004, 124975, + 125003, 125005, 125011, 125018, 125028, 125029, 125012, 125019, 125020, + 125027, 125013, 125035, 125048, 124942, 124934, 125090, 125046, 125078, + 125033, 124946, 125037, 125070, 125000, 125095, 125121, 124951, 125044, + 125041, 125072, 124987, 125066, 125076, 125074, 125009, 125107, 124945, + 125108, 125068, 125124, 125002, 124931, 125089, 125022, 125119, 124980, + 124986, 125080, 125099, 125100, 124933, 125021, 125015, 125071, 124985, + 125117, 125056, 124993, 125039, 125049, 125043, 125024, 124932, 125047, + 125097, 124959, 125069, 125088, 124999, 125123, 124952, 125036, 125026, + 125001, 125057, 125085, 124960, 125073, 124966, 125098, 125014, 125091, + 124989, 125007, 124978, 124940, 125106, 125050, 125030, 125092, 124941, + 125060, 125077, 125102, 125094, 125053, 125103, 125040, 125055, 125104, + 124939, 125017, 125112, 124961, 125087, 124970, 124971, 124969, 125023, + 124979, 125042, 124947, 125075, 125051, 125111, 125086, 124968, 124944, + 125038, 125096, 125016, 125118, 125109, 124953, 125059, 125052, 125006, + 124958, 125093, 125115, 125054, 124988, 125008, 125084, 125061, 125064, + 125120, 125063, 124967, 124977, 124965, 125031, 125081, 125082, 125010, + 125067, 124973, 125032, 124935, 125116, 125122, 125101, 124994, 124995, + 125113, 125058, 125079, 125114, 125065, 125034, 125083, 124954, 125062, + 125105, 125110, 125045, 124943, 124972, 124981, 125025, 125131, 125130, + 125133, 125132, 125129, 125128, 125135, 125127, 125134, 128334, 128697, + 68028, 68093, 68090, 68089, 68086, 68029, 68092, 68091, 68095, 68088, + 68087, 68094, 68000, 68016, 68020, 68021, 68022, 68009, 68010, 68015, + 68017, 68014, 68013, 68018, 68006, 68023, 68012, 68008, 68007, 68019, + 68011, 68005, 68004, 68001, 68002, 68003, 68031, 68030, 68039, 68075, + 68057, 68084, 68066, 68036, 68054, 68081, 68063, 68045, 68072, 68035, + 68053, 68080, 68062, 68044, 68071, 68040, 68076, 68058, 68085, 68067, + 68038, 68056, 68083, 68065, 68047, 68074, 68037, 68055, 68082, 68064, + 68046, 68073, 68034, 68052, 68079, 68061, 68043, 68070, 68033, 68051, + 68078, 68060, 68042, 68069, 68041, 68068, 68032, 68050, 68077, 68059, + 67974, 67975, 67982, 67983, 67978, 67979, 67980, 67981, 67987, 67988, + 67989, 67992, 67993, 67994, 67995, 67996, 67986, 67985, 67990, 67997, + 67984, 67977, 67976, 67991, 67973, 67972, 67968, 67969, 67970, 67971, + 67998, 67999, 129500, 9791, 983173, 9170, 9172, 9177, 9176, 9175, 9173, + 9174, 9171, 9169, 128647, 128221, 94015, 93989, 93971, 93958, 94019, + 94021, 93953, 94023, 93999, 93995, 94008, 93981, 93979, 93967, 93963, + 93993, 93992, 93983, 93977, 93976, 93975, 93974, 93968, 94032, 93988, + 93987, 93973, 93972, 93997, 93996, 93969, 94106, 94107, 94108, 94109, + 94110, 94111, 94002, 94026, 94022, 94003, 94004, 94010, 93980, 93978, + 94099, 94100, 94101, 94102, 94103, 94104, 94105, 93998, 93994, 94007, + 94025, 93966, 93962, 94024, 93961, 93960, 94000, 94009, 93964, 93965, + 94001, 93970, 93984, 93954, 94017, 94014, 94016, 94013, 94006, 94012, + 94005, 94011, 93986, 93985, 93955, 93952, 94020, 93990, 93957, 93956, + 93959, 93982, 94018, 93991, 94035, 94034, 94033, 94031, 94098, 94096, + 94097, 94095, 94036, 94039, 94040, 94067, 94068, 94038, 94037, 94073, + 94075, 94045, 94071, 94069, 94046, 94047, 94085, 94074, 94049, 94050, + 94051, 94052, 94053, 94086, 94057, 94054, 94084, 94055, 94056, 94041, + 94082, 94048, 94081, 94042, 94076, 94058, 94059, 94060, 94061, 94063, + 94064, 94079, 94087, 94062, 94065, 94080, 94066, 94072, 94070, 94044, + 94043, 94077, 94078, 94083, 983239, 983240, 128300, 127908, 181, 129440, + 129986, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, + 8722, 10793, 10794, 10796, 10795, 10810, 8770, 8723, 10751, 129694, + 129705, 128241, 128242, 128244, 129339, 8871, 71232, 71229, 71236, 71231, + 71230, 71216, 71226, 71228, 71221, 71222, 71223, 71224, 71219, 71220, + 71217, 71218, 71225, 71227, 71234, 71233, 71253, 71252, 71255, 71254, + 71251, 71250, 71248, 71257, 71249, 71256, 71168, 71169, 71179, 71181, + 71195, 71194, 71200, 71199, 71193, 71192, 71198, 71197, 71174, 71175, + 71176, 71177, 71210, 71172, 71173, 71170, 71171, 71215, 71209, 71186, + 71196, 71191, 71201, 71211, 71212, 71213, 71205, 71204, 71188, 71187, + 71185, 71184, 71190, 71189, 71183, 71182, 71203, 71202, 71214, 71206, + 71208, 71207, 71178, 71180, 71235, 43867, 67512, 714, 700, 67509, 761, + 763, 7470, 7471, 7487, 7474, 7483, 7476, 43000, 7484, 7485, 7468, 7469, + 42994, 7472, 7473, 42995, 7475, 7477, 7478, 7479, 7480, 7481, 7482, 7486, + 42996, 7488, 7489, 11389, 7490, 723, 722, 42755, 42753, 42757, 42759, + 42754, 42752, 42756, 42758, 42652, 122956, 122958, 122929, 122954, + 122932, 122952, 122943, 122987, 122946, 122938, 122939, 122942, 122960, + 122941, 122959, 122989, 122955, 122950, 122948, 122944, 122951, 122988, + 122953, 122934, 122935, 122936, 122933, 122949, 122931, 122957, 122930, + 122947, 122937, 122928, 122940, 122945, 42653, 7544, 710, 735, 42889, + 67510, 42776, 42777, 42775, 750, 698, 725, 709, 984011, 42765, 42760, + 42770, 741, 984012, 42769, 42764, 42774, 745, 762, 764, 715, 704, 67507, + 4348, 42766, 42761, 42771, 742, 721, 67511, 42888, 42768, 42763, 751, + 767, 753, 42773, 717, 754, 755, 744, 759, 719, 718, 42783, 752, 716, + 42778, 703, 43882, 706, 713, 42767, 42762, 42772, 743, 758, 757, 756, + 727, 766, 726, 697, 42780, 42782, 42781, 42779, 760, 705, 67508, 701, + 67513, 702, 43883, 707, 734, 42890, 765, 7491, 7493, 7516, 67459, 7495, + 7601, 7509, 67461, 7517, 7580, 7590, 694, 7591, 67474, 67476, 7595, + 67484, 67491, 67456, 67460, 67478, 7600, 67498, 7608, 67506, 67471, + 67492, 7581, 7521, 7496, 67468, 67469, 67467, 67466, 7519, 7585, 67480, + 67463, 67465, 67464, 7611, 7613, 7612, 7497, 7604, 7582, 7614, 7505, + 7584, 67472, 7501, 7518, 7520, 67475, 736, 688, 689, 67477, 43868, 67479, + 7504, 7596, 7588, 7589, 690, 7592, 737, 43869, 43870, 7593, 67485, 7594, + 67483, 67481, 67482, 67486, 67487, 43001, 7599, 7598, 7506, 67490, 7499, + 7507, 7510, 7602, 691, 67497, 67496, 67473, 740, 7583, 67470, 738, 67514, + 7603, 7586, 7498, 7513, 7511, 7605, 67503, 67499, 67502, 7508, 67500, + 67501, 7492, 7579, 7494, 7514, 7597, 7500, 692, 67494, 67495, 693, 67488, + 67489, 7587, 7502, 7610, 43881, 7615, 7512, 43871, 7606, 7607, 7515, + 67504, 7609, 7503, 67493, 695, 739, 696, 42784, 42785, 67458, 67457, 720, + 699, 724, 708, 749, 42864, 748, 712, 747, 746, 10762, 128184, 128176, + 129297, 71266, 6165, 6164, 6167, 6166, 6163, 6162, 6160, 6169, 6161, + 6168, 6159, 6157, 6156, 6155, 6147, 6149, 71271, 71272, 6176, 6279, 6272, + 6295, 6273, 6289, 6274, 6313, 6286, 6311, 6310, 6280, 6276, 6275, 6282, + 6287, 6278, 6285, 6284, 6288, 6277, 6291, 6290, 6293, 6294, 6292, 6283, + 6281, 6185, 6196, 6264, 6210, 6190, 6303, 6305, 6307, 6300, 6302, 6304, + 6312, 6298, 6301, 6314, 6308, 6309, 6299, 6306, 6263, 6262, 6260, 6261, + 6259, 6254, 6248, 6244, 6252, 6245, 6253, 6238, 6239, 6257, 6247, 6256, + 6242, 6258, 6255, 6241, 6240, 6249, 6251, 6250, 6243, 6246, 6237, 6193, + 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6228, 6224, + 6222, 6232, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, + 6221, 6230, 6229, 6212, 6213, 6204, 6194, 6209, 6207, 6205, 6206, 6203, + 6202, 6208, 6191, 6177, 6183, 6179, 6181, 6180, 6182, 6186, 6195, 6201, + 6189, 6197, 6184, 6187, 6188, 6199, 6200, 6198, 6178, 71273, 71275, + 71274, 6151, 71265, 71270, 71269, 6144, 71268, 71264, 71267, 71276, 6150, + 6158, 6148, 6146, 6152, 6153, 6154, 6145, 9866, 9867, 119552, 9101, + 128669, 128018, 128053, 127889, 129390, 128496, 129742, 129439, 128332, + 129334, 128741, 128757, 129468, 128739, 9968, 128670, 128672, 128693, + 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, 92744, + 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, 92752, + 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, 92760, + 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, 92783, + 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, 92769, + 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, 70312, + 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, 70301, + 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, 70307, + 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, 127926, + 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, 8846, + 8888, 9838, 9837, 9839, 127929, 127896, 119161, 119159, 119155, 119157, + 119061, 119060, 119224, 119235, 119132, 119058, 119059, 119255, 119253, + 119130, 119131, 119163, 119169, 119149, 119167, 119168, 119210, 119178, + 119173, 119211, 119150, 119151, 119152, 119153, 119154, 119175, 119212, + 119213, 119166, 119164, 119141, 119142, 119176, 119179, 119143, 119144, + 119145, 119165, 119177, 119170, 119174, 119092, 119052, 119247, 119186, + 119073, 119109, 119093, 119050, 119220, 119221, 119044, 119049, 119187, + 119209, 119082, 119041, 119083, 119077, 119078, 119162, 119160, 119208, + 119156, 119158, 119136, 119102, 119056, 119057, 119146, 119147, 119148, + 119042, 119066, 119065, 119069, 119185, 119074, 119075, 119076, 119231, + 119232, 119085, 119084, 119070, 119071, 119072, 119218, 119217, 119189, + 119188, 119248, 119249, 119172, 119171, 119216, 119134, 119100, 119206, + 119262, 119270, 119271, 119263, 119268, 119269, 119272, 119264, 119267, + 119266, 119265, 119274, 119223, 119234, 119233, 119046, 119222, 119184, + 119227, 119237, 119228, 119122, 119123, 119081, 119098, 119207, 119129, + 119087, 119086, 119128, 119140, 119106, 119062, 119195, 119204, 119205, + 119196, 119197, 119198, 119199, 119200, 119201, 119202, 119203, 119094, + 119095, 119126, 119108, 119215, 119214, 119261, 119252, 119257, 119258, + 119183, 119090, 119091, 119135, 119101, 119096, 119097, 119053, 119054, + 119055, 119048, 119043, 119047, 119180, 119254, 119259, 119225, 119236, + 119226, 119229, 119238, 119230, 119051, 119045, 119089, 119088, 119040, + 119067, 119068, 119137, 119103, 119139, 119105, 119110, 119111, 119250, + 119181, 119273, 119243, 119244, 119245, 119246, 119242, 119240, 119239, + 119241, 119064, 119138, 119104, 119063, 119256, 119260, 119190, 119118, + 119119, 119120, 119121, 119112, 119113, 119116, 119117, 119114, 119115, + 119124, 119125, 119191, 119193, 119194, 119127, 119251, 119107, 119133, + 119099, 119219, 119192, 119182, 127932, 127925, 127812, 8811, 8810, 4158, + 4156, 4157, 4155, 4192, 4191, 4190, 4226, 4129, 43642, 4138, 4135, 4208, + 4207, 4206, 4159, 4099, 4098, 4097, 43625, 43624, 43626, 43623, 43622, + 43621, 43627, 43618, 43617, 43630, 43629, 43620, 43619, 983244, 43631, + 43616, 43635, 43628, 43633, 43634, 4096, 4188, 4189, 4187, 4186, 4136, + 4121, 4106, 4111, 4100, 4105, 4116, 4238, 4123, 4176, 43491, 4218, 4220, + 43490, 4221, 4224, 43492, 4223, 43489, 4216, 43488, 4215, 4214, 4213, + 4219, 4222, 4225, 4217, 4130, 43646, 43647, 4193, 4177, 4126, 4112, + 43503, 43495, 43502, 43501, 43516, 43515, 43518, 43517, 43498, 43497, + 43500, 43499, 43514, 43496, 4108, 4107, 4113, 4198, 4197, 4125, 4110, + 4109, 4115, 4114, 4178, 4179, 4180, 4181, 4133, 4134, 4131, 4132, 4128, + 4124, 4120, 4119, 4102, 4101, 4104, 4103, 4118, 4117, 4127, 4122, 4137, + 43636, 43637, 43638, 43632, 43494, 4154, 4150, 4250, 4251, 4237, 4235, + 4236, 4231, 4232, 4233, 4234, 43493, 4171, 43644, 43645, 4201, 4202, + 4203, 4204, 4205, 4151, 4239, 43643, 4170, 4153, 4152, 43639, 43641, + 43640, 4174, 4172, 4255, 4254, 4175, 4173, 4245, 4244, 4247, 4246, 4243, + 4242, 4240, 4249, 4241, 4248, 4195, 4196, 43509, 43508, 43511, 43510, + 43507, 43506, 43504, 43513, 43505, 43512, 4146, 4252, 4253, 4140, 4209, + 4212, 4210, 4211, 4147, 4148, 4194, 4228, 4229, 4230, 4227, 4145, 4149, + 4199, 4200, 4139, 4182, 4183, 4184, 4185, 4143, 4144, 4141, 4142, 4165, + 4164, 4167, 4166, 4163, 4162, 4160, 4169, 4161, 4168, 983218, 983232, + 983174, 10753, 10754, 10752, 8720, 10761, 10757, 10758, 8721, 8899, + 10756, 10755, 11007, 8896, 8897, 8898, 8719, 67712, 67728, 67740, 67724, + 67726, 67714, 67732, 67718, 67730, 67723, 67742, 67717, 67729, 67738, + 67739, 67713, 67735, 67716, 67721, 67734, 67737, 67741, 67725, 67719, + 67722, 67727, 67715, 67733, 67720, 67736, 67731, 67758, 67752, 67753, + 67757, 67751, 67759, 67756, 67754, 67755, 8711, 124117, 124120, 124119, + 124121, 124118, 124132, 124136, 124133, 124138, 124137, 124134, 124135, + 124122, 124124, 124126, 124123, 124125, 124112, 124116, 124114, 124113, + 124115, 124127, 124128, 124129, 124130, 124131, 124140, 124143, 124142, + 124139, 124141, 124149, 124148, 124151, 124150, 124147, 124146, 124144, + 124153, 124145, 124152, 128133, 8358, 8892, 72102, 72103, 72138, 72144, + 72136, 72096, 72097, 72107, 72109, 72123, 72122, 72128, 72127, 72121, + 72120, 72126, 72125, 72100, 72101, 72098, 72099, 72143, 72137, 72114, + 72124, 72119, 72129, 72139, 72140, 72141, 72133, 72132, 72116, 72115, + 72113, 72112, 72118, 72117, 72111, 72110, 72131, 72130, 72142, 72134, + 72135, 72106, 72108, 72162, 72161, 72158, 72160, 72159, 72164, 72150, + 72151, 72145, 72155, 72157, 72148, 72149, 72146, 72147, 72154, 72156, + 72163, 8302, 127966, 129314, 128219, 129535, 8239, 983092, 983197, + 983128, 9471, 9453, 9460, 9452, 9458, 9451, 9454, 9455, 9459, 9456, 9457, + 127312, 127313, 127314, 127315, 127316, 127317, 127318, 127319, 127320, + 127321, 127322, 127323, 127324, 127325, 127326, 127327, 127328, 127329, + 127330, 127331, 127332, 127333, 127334, 127335, 127336, 127337, 128982, + 128984, 129982, 129981, 129983, 127344, 127345, 127346, 127347, 127348, + 127349, 127350, 127351, 127352, 127353, 127354, 127355, 127356, 127357, + 127358, 127359, 127360, 127361, 127362, 127363, 127364, 127365, 127366, + 127367, 127368, 127369, 10062, 129988, 127374, 127371, 127375, 127372, + 127373, 983091, 8879, 8840, 8841, 8775, 8821, 8817, 8825, 8820, 8816, + 8824, 129670, 129722, 11228, 129540, 129544, 129565, 129586, 129603, + 129607, 129561, 129536, 129557, 129599, 129539, 129560, 129602, 129610, + 129613, 129541, 129562, 129604, 129537, 129558, 129600, 129538, 129559, + 129601, 129581, 129578, 129582, 129583, 129579, 129580, 128528, 9906, + 127770, 127761, 8362, 6595, 6594, 6599, 6598, 6597, 6596, 6593, 6566, + 6530, 6567, 6531, 6570, 6537, 6532, 6544, 6543, 6536, 6542, 6549, 6548, + 6562, 6561, 6554, 6560, 6556, 6550, 6528, 6555, 6538, 6568, 6533, 6569, + 6534, 6571, 6540, 6535, 6547, 6546, 6539, 6545, 6552, 6551, 6565, 6564, + 6557, 6563, 6559, 6553, 6529, 6558, 6541, 6622, 6623, 6600, 6601, 6618, + 6577, 6587, 6582, 6586, 6578, 6592, 6583, 6584, 6590, 6589, 6579, 6585, + 6591, 6580, 6588, 6576, 6581, 6613, 6612, 6615, 6614, 6611, 6610, 6608, + 6617, 6609, 6616, 983063, 70732, 70746, 70731, 70741, 70740, 70743, + 70742, 70739, 70738, 70736, 70745, 70737, 70744, 70734, 70675, 70674, + 70681, 70680, 70692, 70686, 70691, 70751, 70662, 70663, 70664, 70665, + 70656, 70657, 70667, 70669, 70685, 70684, 70690, 70689, 70683, 70682, + 70688, 70687, 70660, 70661, 70658, 70659, 70705, 70706, 70707, 70696, + 70695, 70677, 70676, 70673, 70672, 70679, 70678, 70671, 70670, 70703, + 70702, 70698, 70697, 70694, 70693, 70701, 70700, 70708, 70704, 70699, + 70666, 70668, 70727, 70724, 70723, 70726, 70752, 70728, 70753, 70722, + 70725, 70730, 70750, 70747, 70709, 70719, 70721, 70714, 70715, 70716, + 70717, 70712, 70713, 70710, 70711, 70718, 70720, 70735, 70749, 70733, + 70729, 11154, 11155, 128240, 9112, 983131, 9798, 11209, 128084, 129299, + 983132, 128985, 129399, 127747, 2031, 2032, 2033, 2030, 2034, 2027, 2028, + 2029, 2035, 2040, 2045, 2046, 1989, 1988, 1991, 1990, 1987, 1986, 1984, + 1993, 1985, 1992, 2042, 2008, 2001, 2025, 2024, 2026, 2006, 2002, 2019, + 2016, 2018, 2023, 2010, 2009, 2000, 1999, 2007, 1997, 1995, 2012, 2003, + 2013, 2020, 2014, 2015, 2017, 2004, 2011, 2005, 2021, 2022, 1994, 1996, + 1998, 2037, 2036, 2039, 2038, 2041, 2047, 983127, 128691, 9940, 128683, + 128695, 128370, 128286, 128245, 128685, 8303, 65934, 8209, 128689, 10973, + 8893, 8599, 10542, 10545, 10536, 10532, 10530, 128602, 128594, 128610, + 11111, 11127, 11016, 8663, 129109, 11008, 43063, 43062, 43065, 43064, + 43059, 43060, 43057, 43056, 43061, 43058, 10529, 8598, 8689, 8632, 10546, + 10535, 10531, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, + 11009, 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, + 8940, 8742, 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, + 9083, 128323, 10159, 128324, 10161, 128456, 128457, 128458, 128211, + 128212, 160, 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, + 9363, 9371, 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, + 110966, 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, + 110975, 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, + 110984, 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, + 110993, 110994, 110995, 110996, 110997, 110998, 110999, 111000, 111001, + 111002, 111003, 111004, 111005, 111006, 111007, 111008, 111009, 111010, + 111011, 111012, 111013, 111014, 111015, 111016, 111017, 111018, 111019, + 111020, 111021, 111022, 111023, 111024, 111025, 111026, 111027, 111028, + 111029, 111030, 111031, 111032, 111033, 111034, 111035, 111036, 111037, + 111038, 111039, 111040, 111041, 111042, 111043, 111044, 111045, 111046, + 111047, 111048, 111049, 111050, 111051, 111052, 111053, 111054, 111055, + 111056, 111057, 111058, 111059, 111060, 111061, 111062, 111063, 111064, + 111065, 111066, 111067, 111068, 111069, 111070, 111071, 111072, 111073, + 111074, 111075, 111076, 111077, 111078, 111079, 111080, 111081, 111082, + 111083, 111084, 111085, 111086, 111087, 111088, 111089, 111090, 111091, + 111092, 111093, 111094, 111095, 111096, 111097, 111098, 111099, 111100, + 111101, 111102, 111103, 111104, 111105, 111106, 111107, 111108, 111109, + 111110, 111111, 111112, 111113, 111114, 111115, 111116, 111117, 111118, + 111119, 111120, 111121, 111122, 111123, 111124, 111125, 111126, 111127, + 111128, 111129, 111130, 111131, 111132, 111133, 111134, 111135, 111136, + 111137, 111138, 111139, 111140, 111141, 111142, 111143, 111144, 111145, + 111146, 111147, 111148, 111149, 111150, 111151, 111152, 111153, 111154, + 111155, 111156, 111157, 111158, 111159, 111160, 111161, 111162, 111163, + 111164, 111165, 111166, 111167, 111168, 111169, 111170, 111171, 111172, + 111173, 111174, 111175, 111176, 111177, 111178, 111179, 111180, 111181, + 111182, 111183, 111184, 111185, 111186, 111187, 111188, 111189, 111190, + 111191, 111192, 111193, 111194, 111195, 111196, 111197, 111198, 111199, + 111200, 111201, 111202, 111203, 111204, 111205, 111206, 111207, 111208, + 111209, 111210, 111211, 111212, 111213, 111214, 111215, 111216, 111217, + 111218, 111219, 111220, 111221, 111222, 111223, 111224, 111225, 111226, + 111227, 111228, 111229, 111230, 111231, 111232, 111233, 111234, 111235, + 111236, 111237, 111238, 111239, 111240, 111241, 111242, 111243, 111244, + 111245, 111246, 111247, 111248, 111249, 111250, 111251, 111252, 111253, + 111254, 111255, 111256, 111257, 111258, 111259, 111260, 111261, 111262, + 111263, 111264, 111265, 111266, 111267, 111268, 111269, 111270, 111271, + 111272, 111273, 111274, 111275, 111276, 111277, 111278, 111279, 111280, + 111281, 111282, 111283, 111284, 111285, 111286, 111287, 111288, 111289, + 111290, 111291, 111292, 111293, 111294, 111295, 111296, 111297, 111298, + 111299, 111300, 111301, 111302, 111303, 111304, 111305, 111306, 111307, + 111308, 111309, 111310, 111311, 111312, 111313, 111314, 111315, 111316, + 111317, 111318, 111319, 111320, 111321, 111322, 111323, 111324, 111325, + 111326, 111327, 111328, 111329, 111330, 111331, 111332, 111333, 111334, + 111335, 111336, 111337, 111338, 111339, 111340, 111341, 111342, 111343, + 111344, 111345, 111346, 111347, 111348, 111349, 111350, 111351, 111352, + 111353, 111354, 111355, 94177, 128297, 983041, 983040, 123215, 123149, + 123155, 123138, 123166, 123164, 123148, 123143, 123161, 123153, 123152, + 123141, 123137, 123156, 123139, 123163, 123142, 123172, 123173, 123171, + 123158, 123140, 123167, 123176, 123177, 123165, 123151, 123168, 123136, + 123169, 123162, 123178, 123179, 123144, 123157, 123170, 123150, 123145, + 123159, 123146, 123154, 123160, 123147, 123174, 123175, 123180, 123214, + 123193, 123192, 123195, 123191, 123194, 123196, 123197, 123184, 123190, + 123189, 123186, 123185, 123188, 123187, 123205, 123204, 123207, 123206, + 123203, 123202, 123200, 123209, 123201, 123208, 983231, 983066, 10663, + 10662, 11869, 9215, 65532, 9287, 9286, 9284, 9285, 9289, 9281, 9290, + 9288, 9282, 9283, 9280, 128721, 128025, 128885, 127970, 5787, 5788, 5776, + 5761, 5786, 5770, 5769, 5781, 5779, 5785, 5763, 5772, 5780, 5784, 5762, + 5775, 5773, 5765, 5777, 5782, 5764, 5774, 5783, 5766, 5778, 5771, 5767, + 5768, 5760, 731, 128738, 7265, 7264, 7266, 7267, 7261, 7260, 7262, 7259, + 7280, 7282, 7281, 7279, 7271, 7270, 7272, 7269, 7258, 7263, 7278, 7268, + 7283, 7273, 7284, 7285, 7287, 7286, 7276, 7274, 7275, 7277, 7290, 7288, + 7289, 7295, 7294, 7292, 7291, 7253, 7252, 7255, 7254, 7251, 7250, 7248, + 7257, 7249, 7256, 7293, 94179, 94178, 68736, 68739, 68744, 68737, 68756, + 68745, 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, + 68740, 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, + 68784, 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, + 68782, 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, + 68786, 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, + 68820, 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, + 68837, 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, + 68847, 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, + 68834, 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, + 68821, 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, + 68862, 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, + 66330, 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, + 66350, 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, + 66305, 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, + 66338, 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, + 68249, 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, + 68232, 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, + 68238, 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, + 66393, 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, + 66402, 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, + 66389, 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, + 66412, 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, + 66506, 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, + 66474, 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, + 66467, 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, + 66498, 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, + 66466, 66512, 128435, 69414, 69395, 69376, 69394, 69391, 69392, 69398, + 69399, 69403, 69404, 69377, 69379, 69382, 69400, 69388, 69380, 69384, + 69393, 69397, 69401, 69386, 69381, 69385, 69387, 69378, 69390, 69402, + 69383, 69396, 69389, 69415, 69407, 69412, 69411, 69406, 69410, 69405, + 69413, 69409, 69408, 68209, 68210, 68217, 68211, 68213, 68214, 68212, + 68203, 68205, 68207, 68206, 68202, 68198, 68220, 68219, 68215, 68201, + 68216, 68193, 68196, 68199, 68218, 68192, 68194, 68200, 68204, 68197, + 68208, 68195, 68222, 68221, 68223, 68608, 68619, 68627, 68623, 68634, + 68640, 68644, 68668, 68670, 68677, 68632, 68669, 68671, 68617, 68625, + 68621, 68638, 68643, 68660, 68666, 68675, 68630, 68648, 68653, 68646, + 68650, 68641, 68673, 68658, 68642, 68655, 68628, 68611, 68657, 68662, + 68614, 68615, 68636, 68656, 68664, 68679, 68680, 68609, 68610, 68645, + 68654, 68620, 68624, 68635, 68678, 68633, 68672, 68652, 68618, 68626, + 68622, 68639, 68661, 68667, 68676, 68631, 68613, 68649, 68647, 68651, + 68674, 68659, 68629, 68612, 68663, 68616, 68637, 68665, 69509, 69508, + 69507, 69506, 69488, 69502, 69496, 69505, 69492, 69499, 69501, 69503, + 69494, 69493, 69490, 69495, 69489, 69498, 69504, 69491, 69500, 69497, + 69511, 69512, 69513, 69510, 128477, 128117, 129491, 128116, 129746, + 128283, 128664, 128753, 128660, 128662, 128653, 11819, 8228, 128431, + 129649, 129477, 128214, 9251, 10044, 10027, 10034, 10011, 128194, 128449, + 128080, 128237, 128236, 10180, 10179, 128275, 9103, 9104, 10174, 983191, + 8997, 128191, 128440, 9934, 9741, 128217, 129505, 129447, 8886, 2902, + 2903, 2933, 2934, 2931, 2930, 2935, 2932, 2928, 2909, 2908, 2864, 2911, + 2863, 2821, 2822, 2832, 2836, 2850, 2849, 2855, 2854, 2848, 2847, 2853, + 2852, 2827, 2912, 2828, 2913, 2869, 2825, 2826, 2823, 2824, 2867, 2866, + 2841, 2851, 2846, 2856, 2870, 2871, 2872, 2861, 2860, 2843, 2842, 2840, + 2839, 2845, 2844, 2838, 2837, 2859, 2858, 2873, 2862, 2929, 2831, 2835, + 983660, 983659, 2877, 2818, 2817, 2876, 2901, 2893, 2819, 2878, 2888, + 2892, 2883, 2884, 2914, 2915, 2881, 2882, 2879, 2880, 2887, 2891, 2923, + 2922, 2925, 2924, 2921, 2920, 2918, 2927, 2919, 2926, 64830, 64831, 9766, + 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, 66742, + 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, 66762, + 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, 66770, + 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, 66777, + 66778, 66779, 66783, 66803, 66801, 66782, 66789, 66797, 66784, 66808, + 66790, 66788, 66794, 66795, 66804, 66802, 66800, 66786, 66785, 66781, + 66805, 66809, 66799, 66798, 66811, 66810, 66780, 66791, 66792, 66793, + 66796, 66807, 66787, 66806, 66710, 66688, 66715, 66699, 66694, 66698, + 66703, 66693, 66697, 66696, 66705, 66702, 66713, 66717, 66704, 66706, + 66707, 66711, 66716, 66689, 66701, 66700, 66708, 66691, 66695, 66690, + 66692, 66709, 66712, 66714, 66725, 66724, 66727, 66726, 66723, 66722, + 66720, 66729, 66721, 66728, 983192, 126257, 126264, 126258, 126259, + 126265, 126260, 126263, 126267, 126255, 126266, 126256, 126262, 126261, + 126269, 126268, 126254, 126216, 126225, 126252, 126234, 126243, 126222, + 126249, 126213, 126231, 126240, 126221, 126248, 126212, 126230, 126239, + 126217, 126226, 126253, 126235, 126244, 126215, 126224, 126251, 126233, + 126242, 126214, 126223, 126250, 126232, 126241, 126220, 126247, 126211, + 126229, 126238, 126219, 126246, 126210, 126228, 126237, 126218, 126245, + 126209, 126227, 126236, 129446, 128228, 10015, 9885, 10029, 10009, 8485, + 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, 128076, 127842, + 128329, 129417, 128002, 983122, 983121, 128463, 128195, 128479, 128196, + 128223, 128464, 128724, 93059, 93065, 93064, 93058, 93070, 93056, 93055, + 93053, 93062, 93069, 93061, 93066, 93071, 93068, 93054, 93057, 93063, + 93067, 93060, 92967, 92975, 92965, 92969, 92959, 92968, 92971, 92957, + 92962, 92960, 92972, 92970, 92963, 92958, 92966, 92961, 92956, 92974, + 92964, 92973, 92979, 92978, 92980, 92977, 92976, 92982, 92981, 93023, + 93020, 93024, 93021, 93019, 93025, 93022, 93043, 92985, 93047, 93044, + 93045, 92997, 93046, 93042, 93032, 93029, 92995, 93038, 92993, 93041, + 93035, 93033, 93037, 93030, 93039, 92987, 92992, 92986, 92983, 92984, + 93027, 92994, 93034, 92988, 92990, 92989, 92991, 93028, 92996, 93031, + 93040, 93036, 92954, 92955, 92938, 92939, 92932, 92933, 92942, 92943, + 92950, 92951, 92928, 92929, 92936, 92937, 92948, 92949, 92930, 92931, + 92944, 92945, 92934, 92935, 92940, 92941, 92946, 92947, 92952, 92953, + 93013, 93012, 93015, 93014, 93011, 93010, 93008, 93017, 93009, 93016, + 9908, 11801, 127796, 129779, 129780, 67703, 67693, 67688, 67702, 67683, + 67691, 67699, 67700, 67680, 67696, 67682, 67686, 67695, 67698, 67701, + 67689, 67684, 67687, 67690, 67681, 67694, 67685, 67697, 67692, 67704, + 67711, 67706, 67707, 67710, 67709, 67708, 67705, 129330, 129374, 128060, + 11853, 8233, 11791, 10995, 10994, 8741, 129666, 12809, 12823, 12808, + 12822, 12828, 12813, 12827, 12810, 12824, 12800, 12814, 12804, 12818, + 12801, 12815, 12812, 12826, 12805, 12819, 12803, 12817, 12806, 12820, + 12811, 12825, 12802, 12816, 12807, 12821, 12863, 12855, 12858, 12861, + 12847, 12839, 12854, 12843, 12836, 12864, 12835, 12856, 12846, 12842, + 12840, 12852, 12862, 12865, 12857, 12867, 12838, 12866, 12851, 12853, + 12859, 12849, 12860, 12848, 12837, 12834, 12841, 12833, 12845, 12844, + 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, 9350, 9347, 9348, + 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, 9333, 9340, 9332, + 9339, 127248, 127249, 127250, 127251, 127252, 127253, 127254, 127255, + 127256, 127257, 127258, 127259, 127260, 127261, 127262, 127263, 127264, + 127265, 127266, 127267, 127268, 127269, 127270, 127271, 127272, 127273, + 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, 9381, 9382, 9383, + 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, + 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, 127881, 12880, + 12349, 129436, 11261, 128755, 9105, 9106, 128706, 72437, 72440, 72432, + 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, 72420, 72403, + 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, 72394, 72385, + 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, 72386, 72387, + 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, 72435, 72438, + 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, 72428, 72427, + 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, 983237, + 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, 9956, + 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129494, 129496, + 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, 129495, + 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, 129515, + 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, 43090, + 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, 43116, + 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, 43072, + 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, 43078, + 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, 43103, + 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, 66023, + 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, 66020, + 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, 66016, + 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, 66024, + 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, 66040, + 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, 67844, + 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, 67849, + 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, 67866, + 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, 128061, + 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, 10031, + 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, 129703, + 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, 127156, + 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, 127164, + 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, 127176, + 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, 127197, + 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, 127139, + 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, 127221, + 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, 127217, + 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, 127209, + 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, 127195, + 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, 10789, + 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, 11221, + 11220, 11219, 129696, 983148, 128659, 128680, 128110, 8297, 8236, 127871, + 128254, 11239, 128239, 12306, 12320, 128238, 8982, 128688, 129364, + 127858, 129716, 127831, 129751, 128574, 128545, 163, 128093, 9212, 9213, + 9214, 9211, 128041, 128425, 129328, 129732, 129731, 65043, 65040, 65045, + 65073, 65074, 65049, 65041, 65042, 65091, 65047, 65083, 65085, 65089, + 65079, 65087, 65077, 65095, 65081, 65075, 65084, 65086, 65092, 983261, + 65048, 65090, 65080, 65088, 65078, 65096, 65082, 65076, 65044, 65072, + 65046, 8478, 8826, 10937, 10933, 10927, 10929, 10935, 10931, 8936, 8830, + 8828, 8880, 9111, 129384, 9113, 128424, 128438, 129332, 128120, 983193, + 983166, 983163, 983164, 983167, 8242, 8965, 8759, 8733, 8522, 128711, + 129455, 11224, 128255, 68507, 68508, 68480, 68483, 68490, 68491, 68485, + 68482, 68486, 68493, 68495, 68496, 68488, 68484, 68487, 68489, 68481, + 68492, 68497, 68494, 68526, 68522, 68523, 68525, 68521, 68527, 68524, + 68505, 68506, 11854, 8200, 128156, 128091, 128686, 128204, 128226, + 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, + 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, 8799, 34, 9915, + 127949, 127950, 129437, 128251, 9762, 128280, 9143, 128740, 128643, 9926, + 127752, 11827, 11783, 11782, 9995, 128400, 128406, 127338, 127339, + 127340, 9994, 11828, 11787, 129306, 128000, 8758, 128007, 128048, 129682, + 128015, 129534, 9852, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, + 983113, 128665, 127822, 129511, 174, 127462, 127463, 127464, 127465, + 127466, 127467, 127468, 127469, 127470, 127471, 127472, 127473, 127474, + 127475, 127476, 127477, 127478, 127479, 127480, 127481, 127482, 127483, + 127484, 127485, 127486, 127487, 43344, 43343, 43346, 43345, 43330, 43320, + 43333, 43323, 43331, 43314, 43332, 43317, 43319, 43321, 43316, 43313, + 43329, 43322, 43312, 43326, 43318, 43325, 43324, 43315, 43328, 43327, + 43334, 43359, 43337, 43342, 43341, 43338, 43340, 43335, 43339, 43336, + 43347, 128524, 127895, 65533, 9952, 9953, 128699, 8479, 9166, 11152, + 11153, 128639, 128968, 983152, 92, 10184, 10741, 10743, 11073, 11079, + 983153, 10659, 10661, 8246, 12317, 10989, 8976, 11793, 8267, 8245, + 128401, 9753, 11262, 8515, 8271, 8765, 8909, 8247, 128402, 128403, + 128405, 11841, 11822, 128404, 10672, 128158, 8251, 129423, 983154, + 127872, 11190, 11188, 11191, 11189, 11186, 11187, 11184, 11185, 127832, + 127833, 129919, 129918, 128495, 8735, 12297, 10642, 11777, 11776, 9084, + 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, 9132, 9133, 9131, + 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, 129929, 10621, + 8971, 9687, 11241, 9616, 129978, 129971, 129933, 128381, 11805, 8906, + 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, 10182, 8908, + 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, 9126, 9124, + 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 129987, 128493, + 129928, 129930, 11786, 8895, 10702, 129902, 12309, 8866, 11809, 9145, + 12303, 10628, 12311, 10630, 12315, 12313, 10713, 1421, 4053, 4055, + 129308, 9957, 10749, 9002, 187, 128270, 10153, 10552, 8295, 8238, 8235, + 8207, 10813, 8594, 11080, 11084, 10562, 10613, 10612, 129974, 8614, + 10528, 11076, 11075, 10567, 10526, 8677, 8628, 10513, 8699, 129034, 8620, + 10565, 129042, 129026, 8603, 11022, 11023, 8611, 10517, 10516, 129030, + 129178, 129046, 8618, 8696, 8644, 10522, 129193, 129185, 11146, 11157, + 8658, 10499, 8655, 10503, 10524, 10509, 8674, 129195, 129078, 8652, + 10601, 10605, 10591, 10583, 8641, 10600, 10604, 10596, 10587, 10579, + 8640, 129777, 129090, 129094, 129191, 8702, 8649, 129784, 129189, 128622, + 8669, 129082, 129106, 129187, 11106, 11138, 11132, 983242, 11175, 11173, + 129066, 129062, 129058, 129074, 129070, 11122, 11116, 11142, 129170, + 10511, 8667, 10518, 10520, 10519, 11246, 10497, 10496, 10501, 10512, + 8608, 8605, 8680, 8688, 129174, 129086, 11078, 128141, 128735, 11824, + 8790, 8791, 8728, 730, 129680, 128365, 10539, 10544, 65020, 129350, + 983227, 983235, 983225, 983230, 129704, 128640, 128478, 127906, 128764, + 129531, 129315, 128019, 65947, 65942, 65945, 65940, 65943, 8556, 8583, + 8582, 8548, 8558, 8577, 8547, 8544, 8557, 8584, 8559, 8576, 8549, 8581, + 8550, 8553, 8578, 8555, 8545, 8546, 8579, 8554, 8551, 8552, 65938, 65944, + 65936, 65939, 65941, 65937, 65946, 127801, 127989, 10087, 10085, 11213, + 11215, 8506, 128205, 128907, 127588, 127586, 127589, 127584, 127585, + 127587, 128675, 127840, 129302, 8381, 127945, 10740, 69245, 69243, 69244, + 69246, 69241, 69232, 69238, 69229, 69237, 69228, 69242, 69233, 69234, + 69240, 69231, 69239, 69230, 69236, 69227, 69235, 69226, 69225, 69220, + 69219, 69222, 69221, 69218, 69217, 69224, 69216, 69223, 5872, 5869, 5803, + 5802, 5800, 5833, 5837, 5860, 5811, 5859, 5858, 5841, 5851, 5824, 5844, + 5854, 5826, 5846, 5856, 5799, 5814, 5880, 5879, 5877, 5876, 5878, 5792, + 5813, 5815, 5828, 5816, 5819, 5818, 5853, 5852, 5825, 5831, 5864, 5857, + 5827, 5873, 5812, 5810, 5850, 5829, 5820, 5848, 5804, 5862, 5806, 5801, + 5855, 5845, 5807, 5808, 5875, 5809, 5874, 5843, 5821, 5849, 5823, 5805, + 5836, 5840, 5830, 5863, 5835, 5834, 5861, 5842, 5822, 5798, 5839, 5797, + 5817, 5794, 5847, 5832, 5796, 5795, 5865, 5793, 5866, 5838, 5867, 5868, + 5870, 5871, 127933, 127939, 11254, 11252, 11253, 11251, 11255, 11256, + 8360, 983114, 10700, 129466, 129527, 9808, 129474, 9747, 129761, 2049, + 2053, 2051, 2063, 2055, 2052, 2058, 2059, 2062, 2068, 2069, 2056, 2065, + 2048, 2050, 2067, 2054, 2060, 2066, 2061, 2057, 2064, 2073, 2070, 2071, + 2075, 2093, 2072, 2074, 2084, 2088, 2097, 2110, 2098, 2100, 2108, 2099, + 2096, 2101, 2109, 2106, 2104, 2107, 2103, 2105, 2082, 2079, 2076, 2089, + 2086, 2091, 2081, 2078, 2085, 2092, 2083, 2080, 2077, 2090, 2087, 2102, + 128630, 128631, 128632, 128634, 129386, 128752, 128225, 9796, 43138, + 43139, 43150, 43153, 43167, 43166, 43172, 43171, 43165, 43164, 43170, + 43169, 43144, 43145, 43146, 43147, 43182, 43142, 43143, 43151, 43152, + 43140, 43141, 43187, 43181, 43158, 43168, 43163, 43173, 43183, 43184, + 43185, 43177, 43176, 43160, 43159, 43157, 43156, 43162, 43161, 43155, + 43154, 43175, 43174, 43148, 43149, 43186, 43178, 43180, 43179, 43205, + 43136, 43204, 43137, 43215, 43214, 43221, 43220, 43223, 43222, 43219, + 43218, 43216, 43225, 43217, 43224, 43188, 43189, 43200, 43203, 43194, + 43195, 43196, 43197, 43192, 43193, 43201, 43202, 43190, 43191, 43198, + 43199, 129429, 9973, 127927, 127862, 129403, 9878, 129507, 127979, + 127890, 129410, 9807, 128756, 129691, 128437, 8492, 8496, 8497, 8459, + 8464, 8466, 8499, 8472, 8475, 128624, 8495, 8458, 8467, 8500, 8456, + 128220, 983186, 129453, 128186, 167, 8980, 129352, 127793, 128584, 8979, + 130037, 130036, 130039, 130038, 130035, 130034, 130032, 130041, 130033, + 130040, 10802, 9914, 59, 8480, 129324, 9916, 65093, 983169, 8726, 129697, + 11259, 9913, 11250, 129331, 10014, 10061, 10032, 129368, 70086, 70085, + 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, 70097, 70104, + 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, 70045, 70051, + 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, 70028, 70062, + 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, 70042, 70052, + 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, 70035, 70041, + 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, 70058, 70029, + 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70081, 70017, 70016, + 70090, 70082, 70083, 70080, 70018, 70093, 70091, 70094, 70067, 70077, + 70079, 70072, 70073, 70074, 70075, 70070, 70071, 70068, 70069, 70076, + 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, 66682, 66665, + 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, 66663, 66659, + 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, 66678, 66668, + 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, 66654, 66641, + 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, 66652, 66650, + 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, 983078, 9961, + 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, 10976, 10985, + 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, 127856, + 129651, 9085, 129327, 128703, 128017, 129424, 129335, 10722, 983198, + 983080, 71107, 71106, 71113, 71040, 71131, 71041, 71051, 71053, 71128, + 71070, 71129, 71130, 71065, 71064, 71069, 71067, 71066, 71072, 71071, + 71046, 71047, 71048, 71049, 71082, 71044, 71045, 71042, 71043, 71058, + 71068, 71063, 71073, 71083, 71084, 71085, 71077, 71076, 71060, 71059, + 71057, 71056, 71062, 71061, 71055, 71054, 71075, 71074, 71086, 71081, + 71078, 71080, 71079, 71050, 71052, 71110, 71111, 71112, 71119, 71120, + 71127, 71126, 71125, 71123, 71124, 71117, 71118, 71116, 71121, 71115, + 71114, 71122, 71109, 71108, 71105, 71100, 71104, 71101, 71103, 71102, + 71132, 71133, 71087, 71097, 71099, 71092, 71093, 71090, 71091, 71088, + 71089, 71096, 71098, 128411, 128410, 128417, 128416, 128409, 128408, + 128415, 128414, 121399, 121397, 121400, 121398, 121402, 121401, 121104, + 121103, 121102, 121388, 121387, 121386, 121482, 121479, 121358, 121359, + 121357, 121360, 121341, 121335, 121339, 121340, 121336, 121334, 121333, + 121338, 121337, 121342, 121368, 121367, 121373, 121356, 121355, 121354, + 121380, 121382, 121381, 121384, 121383, 121385, 121377, 121379, 121378, + 121375, 121376, 121374, 121371, 121369, 121366, 121370, 121372, 121364, + 121365, 121452, 121392, 121352, 121353, 121351, 121499, 121500, 121501, + 121502, 121503, 121470, 121117, 121115, 121119, 121118, 121116, 121456, + 121363, 121362, 121361, 121480, 121455, 120965, 120837, 121000, 120969, + 121027, 121026, 121076, 121074, 121075, 120995, 120991, 120990, 120982, + 121005, 121011, 121044, 121042, 121043, 120950, 120833, 120847, 121002, + 120859, 120863, 120997, 120967, 120845, 121019, 120839, 120993, 120980, + 121038, 121045, 121031, 121009, 120934, 121046, 121047, 120905, 120936, + 120937, 120935, 121065, 120941, 120834, 121068, 121069, 121024, 120915, + 120916, 121040, 120940, 120944, 120946, 120942, 120945, 120947, 120943, + 120949, 121020, 121039, 120948, 121091, 120832, 120838, 120861, 120843, + 120852, 120870, 120844, 120846, 120848, 120865, 120853, 120856, 120885, + 120877, 120878, 120879, 120884, 120857, 120882, 120854, 120855, 120858, + 120883, 121001, 120871, 120850, 120867, 120966, 120972, 120971, 120849, + 120862, 120895, 120894, 120872, 120873, 120893, 120875, 120874, 120866, + 120864, 120996, 120842, 120891, 120890, 121067, 120892, 120887, 121064, + 121066, 121063, 121060, 121061, 121062, 120899, 120881, 121052, 121059, + 121057, 121054, 121055, 121056, 121058, 120851, 120869, 120868, 120841, + 121018, 120974, 120977, 120976, 120975, 120985, 120986, 120983, 120992, + 120988, 120984, 120978, 121032, 121037, 120897, 120896, 120898, 120889, + 120888, 120886, 120880, 121035, 121030, 121033, 121028, 121036, 120860, + 121003, 121012, 121014, 121013, 121007, 121016, 121008, 121015, 121006, + 121077, 121083, 120876, 121084, 121085, 121078, 121090, 121087, 121079, + 121080, 121081, 120840, 121086, 121088, 121089, 120979, 121082, 121092, + 120906, 120922, 120908, 120919, 120920, 120921, 120914, 120910, 120912, + 120900, 120903, 120904, 120902, 120901, 120924, 120909, 120911, 120913, + 120926, 120930, 120931, 120932, 120929, 120933, 120928, 120925, 120927, + 120923, 120957, 120917, 120907, 120836, 121051, 120968, 120973, 120998, + 121072, 121053, 121073, 121070, 121071, 121025, 120970, 120994, 120989, + 120987, 121029, 121041, 120955, 120961, 120956, 120959, 120999, 121004, + 120964, 120960, 120963, 120962, 120958, 120939, 120938, 121023, 121021, + 121022, 121050, 121048, 121049, 121034, 121017, 120951, 120981, 120953, + 121010, 120954, 120952, 120918, 120835, 121451, 121343, 121349, 121347, + 121348, 121346, 121345, 121350, 121344, 121462, 121463, 121464, 121465, + 121466, 121467, 121468, 121469, 121428, 121429, 121427, 121476, 121473, + 121477, 121475, 121474, 121478, 121472, 121471, 121404, 121405, 121403, + 121430, 121409, 121411, 121410, 121406, 121408, 121407, 121421, 121423, + 121422, 121415, 121416, 121417, 121418, 121419, 121420, 121414, 121413, + 121412, 121424, 121425, 121426, 121432, 121431, 121183, 121184, 121182, + 121181, 121187, 121188, 121186, 121185, 121175, 121176, 121174, 121173, + 121179, 121180, 121178, 121177, 121324, 121321, 121323, 121320, 121322, + 121319, 121210, 121209, 121208, 121203, 121207, 121305, 121272, 121271, + 121287, 121286, 121303, 121304, 121302, 121301, 121206, 121205, 121204, + 121198, 121332, 121331, 121129, 121276, 121274, 121275, 121273, 121288, + 121289, 121291, 121290, 121306, 121280, 121278, 121279, 121277, 121295, + 121293, 121294, 121292, 121307, 121314, 121191, 121192, 121190, 121189, + 121193, 121202, 121201, 121200, 121199, 121282, 121281, 121297, 121296, + 121308, 121309, 121310, 121328, 121327, 121194, 121196, 121197, 121195, + 121216, 121215, 121214, 121213, 121212, 121211, 121449, 121125, 121126, + 121121, 121122, 121123, 121124, 121127, 121318, 121316, 121317, 121315, + 121156, 121155, 121154, 121146, 121145, 121144, 121150, 121149, 121148, + 121147, 121230, 121231, 121229, 121228, 121261, 121254, 121247, 121233, + 121232, 121226, 121227, 121225, 121224, 121249, 121248, 121153, 121152, + 121151, 121139, 121135, 121137, 121138, 121136, 121330, 121329, 121128, + 121236, 121262, 121255, 121235, 121234, 121237, 121240, 121239, 121263, + 121256, 121238, 121162, 121161, 121160, 121132, 121133, 121131, 121130, + 121134, 121253, 121142, 121143, 121141, 121140, 121243, 121242, 121241, + 121246, 121245, 121244, 121270, 121269, 121268, 121264, 121257, 121326, + 121325, 121159, 121158, 121157, 121448, 121394, 121393, 121396, 121395, + 121450, 121513, 121514, 121515, 121516, 121517, 121518, 121519, 121505, + 121506, 121507, 121508, 121509, 121510, 121511, 121512, 121312, 121284, + 121299, 121311, 121283, 121298, 121313, 121285, 121300, 121267, 121260, + 121252, 121251, 121266, 121259, 121250, 121265, 121258, 121107, 121106, + 121105, 121454, 121453, 121457, 121120, 121112, 121110, 121114, 121113, + 121111, 121108, 121109, 121101, 121100, 121099, 121481, 121441, 121445, + 121446, 121443, 121444, 121442, 121447, 121390, 121389, 121391, 121440, + 121439, 121437, 121435, 121436, 121434, 121433, 121438, 121459, 121458, + 121460, 121095, 121094, 121093, 121219, 121218, 121217, 121222, 121221, + 121220, 121223, 121172, 121171, 121170, 121168, 121167, 121166, 121165, + 121164, 121163, 121169, 121098, 121097, 121096, 121461, 121483, 129304, + 10912, 10911, 10910, 10909, 10860, 983185, 983183, 8219, 8249, 8218, + 8250, 983158, 983155, 983156, 983159, 70117, 70116, 70119, 70118, 70115, + 70114, 70121, 70113, 70120, 70131, 70132, 70129, 70126, 70125, 70130, + 70128, 70127, 70124, 70123, 70122, 983953, 983954, 983952, 3464, 3463, + 3495, 3501, 3497, 3503, 3510, 3488, 3484, 3490, 3482, 3508, 3513, 3462, + 3475, 3478, 3461, 3474, 3473, 3472, 3471, 3470, 3469, 3466, 3465, 3468, + 3467, 3496, 3502, 3498, 3504, 3511, 3489, 3485, 3491, 3483, 3509, 3512, + 3525, 3499, 3522, 3517, 3505, 3523, 3477, 3476, 3500, 3507, 3487, 3494, + 3526, 3524, 3515, 3520, 3514, 3492, 3493, 3521, 3486, 3563, 3562, 3565, + 3564, 3561, 3560, 3558, 3567, 3559, 3566, 3530, 3458, 3457, 3459, 3570, + 3571, 3546, 3537, 3539, 3542, 3544, 3551, 3536, 3538, 3540, 3545, 3548, + 3549, 3550, 3547, 3535, 3572, 8767, 10046, 128973, 10038, 128303, 8198, + 10042, 128510, 127935, 9975, 128128, 9760, 129448, 128761, 10902, 10904, + 10901, 10903, 11098, 11100, 42611, 128716, 128164, 128564, 128554, + 128373, 128759, 127829, 128577, 128578, 10840, 10839, 9011, 127920, + 129445, 65120, 128745, 65121, 128313, 128312, 8717, 8956, 8958, 65131, + 65104, 65109, 65129, 8714, 8948, 8951, 65126, 65111, 65112, 65105, 65115, + 65113, 65117, 65124, 65116, 65114, 65118, 8570, 8567, 8564, 8574, 8572, + 8563, 8560, 8573, 8575, 8571, 8561, 8562, 8569, 8566, 8565, 8568, 65128, + 65108, 68411, 732, 10849, 65125, 65123, 65130, 65122, 65119, 65106, + 65110, 10922, 10924, 10803, 128570, 128571, 128525, 128520, 128519, + 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, 129394, + 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, 127938, + 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, 69453, + 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, 69450, + 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, 69439, + 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, 69428, + 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, 69462, + 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, 69846, + 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, 69855, + 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, 69862, + 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, 69873, + 69880, 8600, 10537, 10541, 8690, 10533, 128603, 128595, 128611, 11112, + 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, 128601, 128593, + 128609, 11113, 11129, 11019, 8665, 129111, 11011, 8471, 72328, 72329, + 72327, 72326, 72340, 72339, 72334, 72332, 72341, 72335, 72333, 72330, + 72331, 72338, 72336, 72337, 72344, 72352, 72351, 72350, 72297, 72296, + 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, 72293, 72303, + 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, 72299, 72307, + 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, 72304, 72312, + 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, 72272, 72349, + 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, 72353, 72354, + 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, 72276, 72273, + 72283, 8384, 983043, 983182, 983118, 983177, 127837, 128150, 10055, + 10024, 32, 128586, 128264, 128263, 128265, 128266, 128483, 128676, + 128172, 8375, 8738, 10656, 10657, 128375, 128376, 128467, 128466, 128026, + 128166, 129525, 129348, 128051, 127941, 129533, 13279, 13058, 13057, + 13059, 13056, 13250, 13171, 13278, 13101, 13172, 13108, 13105, 13118, + 13116, 13251, 13192, 8851, 13255, 13183, 13213, 13220, 13216, 13254, + 8852, 13252, 13253, 13170, 13092, 13175, 13177, 13176, 13093, 13094, + 13256, 127376, 13207, 13064, 13179, 13181, 13182, 13055, 13180, 13005, + 13063, 13006, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, + 13072, 13080, 13081, 13203, 13228, 13191, 13257, 13258, 13098, 13110, + 13113, 13121, 13122, 13119, 13107, 13106, 13109, 13259, 13169, 127488, + 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13067, + 13069, 13068, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, + 13218, 13086, 13085, 13082, 13083, 13084, 13201, 13193, 13248, 13226, + 13189, 13199, 13261, 13208, 13263, 13240, 13246, 8977, 13266, 10957, + 13264, 13265, 13267, 13223, 13224, 13249, 13221, 13217, 13187, 13123, + 13124, 13127, 13126, 13125, 13190, 13268, 13131, 13132, 13133, 13128, + 13129, 13130, 13269, 13212, 13219, 13215, 13186, 13196, 13197, 13205, + 13211, 13234, 13238, 13244, 13239, 13241, 13245, 13247, 13202, 13270, + 13227, 13198, 13206, 13235, 13185, 13096, 13097, 13195, 13210, 13233, + 13237, 13243, 13065, 8848, 8933, 8850, 13066, 13173, 13225, 13099, 13100, + 13184, 13115, 13112, 13114, 13111, 13103, 13104, 13102, 13117, 13120, + 11216, 13273, 13174, 13194, 13271, 13272, 13274, 13232, 13236, 13242, + 13229, 13230, 13231, 13142, 13141, 10958, 13137, 13138, 13140, 13139, + 8730, 13087, 13088, 13090, 13091, 13089, 13275, 13276, 128918, 13204, + 13095, 13143, 11027, 9641, 9638, 9636, 11029, 9706, 9703, 11026, 9705, + 9639, 11028, 9640, 9637, 9704, 10720, 13277, 13135, 13134, 13136, 13062, + 127529, 127530, 127512, 127508, 127533, 127545, 127520, 127516, 127534, + 127506, 127540, 127525, 127546, 127532, 127511, 127509, 127524, 127505, + 127517, 127518, 127527, 127537, 127504, 127528, 127535, 127519, 127515, + 127513, 127543, 127542, 127526, 127541, 127544, 127522, 127538, 127514, + 127521, 127539, 127510, 127536, 127523, 127547, 127531, 127378, 127377, + 8865, 10693, 11820, 127390, 127392, 127379, 10692, 127400, 127399, + 127398, 127306, 127489, 127507, 127490, 9919, 127397, 127280, 127281, + 127282, 127283, 127284, 127285, 127286, 127287, 127288, 127289, 127290, + 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298, 127299, + 127300, 127301, 127302, 127303, 127304, 127305, 10190, 10191, 127401, + 8863, 127307, 127381, 127382, 127396, 127383, 127310, 8862, 127388, + 127393, 127402, 127395, 9949, 10695, 10696, 127384, 127308, 127309, + 127387, 127394, 127389, 8864, 127391, 127385, 127403, 127404, 127386, + 10694, 127311, 127380, 10151, 129425, 983157, 983160, 983134, 983190, + 9877, 9882, 128387, 129485, 8795, 10017, 8902, 9770, 11242, 11243, + 983175, 983133, 983181, 983176, 983042, 983044, 128509, 128649, 127967, + 128642, 127836, 11836, 129658, 129989, 129990, 129993, 129991, 129992, + 128480, 9201, 128207, 9188, 127827, 10025, 983189, 8803, 127897, 129369, + 128723, 983170, 983045, 983103, 8332, 8328, 8330, 8331, 8333, 8334, 8325, + 8324, 8327, 8326, 8323, 8322, 8320, 8329, 8321, 10963, 10965, 10617, + 8834, 10953, 10949, 10951, 10955, 8842, 8838, 10947, 10943, 10945, 10941, + 983102, 8827, 10938, 10934, 10928, 10930, 10936, 10932, 8937, 8831, 8829, + 8881, 9139, 10763, 9138, 9737, 127774, 9925, 7098, 7073, 7074, 7075, + 7084, 7085, 7043, 983220, 7046, 7102, 7103, 7062, 7100, 7068, 7099, 7067, + 7087, 7070, 7048, 7049, 7053, 7057, 7060, 7101, 7064, 7086, 7050, 7054, + 7059, 7052, 7072, 7055, 7065, 7061, 7051, 7058, 7063, 7069, 7071, 7066, + 7056, 7044, 7047, 7045, 7367, 7366, 7365, 7364, 7363, 7361, 7362, 7360, + 7082, 7041, 7042, 7040, 7083, 7080, 7081, 7079, 7077, 7076, 7078, 7093, + 7092, 7095, 7094, 7091, 7090, 7088, 7097, 7089, 7096, 127749, 127748, + 127751, 127803, 8316, 8312, 8305, 8319, 8317, 8314, 8315, 8318, 8309, + 8308, 8311, 8310, 179, 178, 8304, 8313, 185, 10966, 10964, 10619, 10968, + 10967, 8835, 10954, 10950, 10952, 10956, 8843, 8839, 10948, 10944, 10946, + 10942, 10185, 129464, 129465, 8751, 127940, 128671, 127843, 128629, + 129442, 127946, 8275, 43027, 43026, 43031, 43030, 43040, 43038, 43025, + 43024, 43029, 43028, 43036, 43035, 43021, 43020, 43018, 43017, 43023, + 43022, 43016, 43015, 43034, 43033, 43042, 43039, 43037, 43032, 43041, + 43008, 43012, 43009, 43013, 43011, 43048, 43049, 43050, 43051, 43052, + 43019, 43014, 43010, 43047, 43043, 43046, 43044, 43045, 9223, 9224, 9229, + 9240, 9232, 9236, 9235, 9234, 9233, 9249, 9253, 9241, 9220, 9239, 9219, + 9221, 9243, 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, + 9246, 9245, 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, + 11159, 9007, 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, + 1848, 1792, 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, + 1813, 1815, 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, + 2154, 2145, 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, + 1870, 1871, 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, + 1811, 1812, 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, + 1841, 1840, 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, + 1794, 1795, 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, + 983184, 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, + 917568, 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, + 917554, 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, + 917549, 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, + 917577, 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, + 917586, 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, + 917601, 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, + 917610, 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, + 917619, 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, + 917627, 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, + 917545, 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, + 917628, 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, + 5897, 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, + 5989, 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, + 5996, 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, + 6512, 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, + 6504, 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, + 6484, 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, + 6749, 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, + 6792, 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, + 6689, 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, + 6706, 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, + 6709, 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, + 6701, 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, + 6703, 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, + 6819, 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, + 6817, 6773, 6774, 6821, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, + 6771, 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, + 43653, 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, + 43665, 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, + 43663, 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, + 43676, 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, + 43692, 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, + 43743, 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, + 43697, 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, + 43701, 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71338, 71332, + 71319, 71318, 71324, 71323, 71317, 71316, 71322, 71321, 71300, 71301, + 71298, 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, + 71309, 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, + 71337, 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, + 71340, 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, + 71365, 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, + 129377, 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, + 983677, 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, + 983683, 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, + 983670, 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, + 3052, 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, + 73679, 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, + 73675, 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, + 2979, 2969, 2974, 2984, 2975, 2980, 2996, 2995, 2994, 2993, 2992, 2949, + 2950, 2960, 2964, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, + 2958, 2959, 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, + 3058, 3056, 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, + 73693, 73692, 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, + 73713, 73699, 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, + 983950, 983943, 983944, 983948, 983949, 983941, 983942, 983945, 983946, + 983686, 983693, 983696, 983689, 983690, 983694, 983695, 983687, 983688, + 983691, 983692, 983840, 983847, 983850, 983843, 983844, 983848, 983849, + 983841, 983842, 983845, 983846, 983851, 983858, 983861, 983854, 983855, + 983859, 983860, 983852, 983853, 983856, 983857, 983818, 983825, 983828, + 983821, 983822, 983826, 983827, 983819, 983820, 983823, 983824, 983873, + 983880, 983883, 983876, 983877, 983881, 983882, 983874, 983875, 983878, + 983879, 983741, 983748, 983751, 983744, 983745, 983749, 983750, 983742, + 983743, 983746, 983747, 983697, 983704, 983707, 983700, 983701, 983705, + 983706, 983698, 983699, 983702, 983703, 983719, 983726, 983729, 983722, + 983723, 983727, 983728, 983720, 983721, 983724, 983725, 983763, 983770, + 983773, 983766, 983767, 983771, 983772, 983764, 983765, 983768, 983769, + 983862, 983869, 983872, 983865, 983866, 983870, 983871, 983863, 983864, + 983867, 983868, 983807, 983814, 983817, 983810, 983811, 983815, 983816, + 983808, 983809, 983812, 983813, 983895, 983902, 983905, 983898, 983899, + 983903, 983904, 983951, 983896, 983897, 983900, 983901, 983906, 983913, + 983916, 983909, 983910, 983914, 983915, 983907, 983908, 983911, 983912, + 983917, 983924, 983927, 983920, 983921, 983925, 983926, 983918, 983919, + 983922, 983923, 983730, 983737, 983740, 983733, 983734, 983738, 983739, + 983731, 983732, 983735, 983736, 983752, 983759, 983762, 983755, 983756, + 983760, 983761, 983753, 983754, 983757, 983758, 983708, 983715, 983718, + 983711, 983712, 983716, 983717, 983709, 983710, 983713, 983714, 983928, + 983935, 983938, 983931, 983932, 983936, 983937, 983929, 983930, 983933, + 983934, 983884, 983891, 983894, 983887, 983888, 983892, 983893, 983885, + 983886, 983889, 983890, 983785, 983792, 983795, 983788, 983789, 983793, + 983794, 983786, 983787, 983790, 983791, 983774, 983781, 983784, 983777, + 983778, 983782, 983783, 983775, 983776, 983779, 983780, 983829, 983836, + 983839, 983832, 983833, 983837, 983838, 983830, 983831, 983834, 983835, + 983796, 983803, 983806, 983799, 983800, 983804, 983805, 983797, 983798, + 983801, 983802, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, 3008, + 3014, 3015, 73709, 73704, 73705, 3061, 3060, 3024, 129748, 127883, + 127818, 92809, 92810, 92811, 92808, 92789, 92790, 92791, 92788, 92816, + 92859, 92856, 92847, 92845, 92817, 92846, 92843, 92829, 92830, 92831, + 92828, 92835, 92851, 92840, 92844, 92818, 92819, 92852, 92836, 92825, + 92826, 92827, 92824, 92813, 92814, 92815, 92812, 92820, 92822, 92823, + 92821, 92805, 92806, 92807, 92804, 92797, 92798, 92799, 92796, 92801, + 92802, 92803, 92800, 92785, 92786, 92787, 92784, 92793, 92794, 92795, + 92792, 92857, 92854, 92848, 92861, 92853, 92860, 92849, 92855, 92834, + 92833, 92832, 92841, 92839, 92842, 92850, 92838, 92858, 92837, 92862, + 92869, 92868, 92871, 92870, 92867, 92866, 92864, 92873, 92865, 92872, + 100352, 100353, 100354, 100355, 100356, 100357, 100358, 100359, 100360, + 100361, 100362, 100363, 100364, 100365, 100366, 100367, 100368, 100369, + 100370, 100371, 100372, 100373, 100374, 100375, 100376, 100377, 100378, + 100379, 100380, 100381, 100382, 100383, 100384, 100385, 100386, 100387, + 100388, 100389, 100390, 100391, 100392, 100393, 100394, 100395, 100396, + 100397, 100398, 100399, 100400, 100401, 100402, 100403, 100404, 100405, + 100406, 100407, 100408, 100409, 100410, 100411, 100412, 100413, 100414, + 100415, 100416, 100417, 100418, 100419, 100420, 100421, 100422, 100423, + 100424, 100425, 100426, 100427, 100428, 100429, 100430, 100431, 100432, + 100433, 100434, 100435, 100436, 100437, 100438, 100439, 100440, 100441, + 100442, 100443, 100444, 100445, 100446, 100447, 100448, 100449, 100450, + 100451, 100452, 100453, 100454, 100455, 100456, 100457, 100458, 100459, + 100460, 100461, 100462, 100463, 100464, 100465, 100466, 100467, 100468, + 100469, 100470, 100471, 100472, 100473, 100474, 100475, 100476, 100477, + 100478, 100479, 100480, 100481, 100482, 100483, 100484, 100485, 100486, + 100487, 100488, 100489, 100490, 100491, 100492, 100493, 100494, 100495, + 100496, 100497, 100498, 100499, 100500, 100501, 100502, 100503, 100504, + 100505, 100506, 100507, 100508, 100509, 100510, 100511, 100512, 100513, + 100514, 100515, 100516, 100517, 100518, 100519, 100520, 100521, 100522, + 100523, 100524, 100525, 100526, 100527, 100528, 100529, 100530, 100531, + 100532, 100533, 100534, 100535, 100536, 100537, 100538, 100539, 100540, + 100541, 100542, 100543, 100544, 100545, 100546, 100547, 100548, 100549, + 100550, 100551, 100552, 100553, 100554, 100555, 100556, 100557, 100558, + 100559, 100560, 100561, 100562, 100563, 100564, 100565, 100566, 100567, + 100568, 100569, 100570, 100571, 100572, 100573, 100574, 100575, 100576, + 100577, 100578, 100579, 100580, 100581, 100582, 100583, 100584, 100585, + 100586, 100587, 100588, 100589, 100590, 100591, 100592, 100593, 100594, + 100595, 100596, 100597, 100598, 100599, 100600, 100601, 100602, 100603, + 100604, 100605, 100606, 100607, 100608, 100609, 100610, 100611, 100612, + 100613, 100614, 100615, 100616, 100617, 100618, 100619, 100620, 100621, + 100622, 100623, 100624, 100625, 100626, 100627, 100628, 100629, 100630, + 100631, 100632, 100633, 100634, 100635, 100636, 100637, 100638, 100639, + 100640, 100641, 100642, 100643, 100644, 100645, 100646, 100647, 100648, + 100649, 100650, 100651, 100652, 100653, 100654, 100655, 100656, 100657, + 100658, 100659, 100660, 100661, 100662, 100663, 100664, 100665, 100666, + 100667, 100668, 100669, 100670, 100671, 100672, 100673, 100674, 100675, + 100676, 100677, 100678, 100679, 100680, 100681, 100682, 100683, 100684, + 100685, 100686, 100687, 100688, 100689, 100690, 100691, 100692, 100693, + 100694, 100695, 100696, 100697, 100698, 100699, 100700, 100701, 100702, + 100703, 100704, 100705, 100706, 100707, 100708, 100709, 100710, 100711, + 100712, 100713, 100714, 100715, 100716, 100717, 100718, 100719, 100720, + 100721, 100722, 100723, 100724, 100725, 100726, 100727, 100728, 100729, + 100730, 100731, 100732, 100733, 100734, 100735, 100736, 100737, 100738, + 100739, 100740, 100741, 100742, 100743, 100744, 100745, 100746, 100747, + 100748, 100749, 100750, 100751, 100752, 100753, 100754, 100755, 100756, + 100757, 100758, 100759, 100760, 100761, 100762, 100763, 100764, 100765, + 100766, 100767, 100768, 100769, 100770, 100771, 100772, 100773, 100774, + 100775, 100776, 100777, 100778, 100779, 100780, 100781, 100782, 100783, + 100784, 100785, 100786, 100787, 100788, 100789, 100790, 100791, 100792, + 100793, 100794, 100795, 100796, 100797, 100798, 100799, 100800, 100801, + 100802, 100803, 100804, 100805, 100806, 100807, 100808, 100809, 100810, + 100811, 100812, 100813, 100814, 100815, 100816, 100817, 100818, 100819, + 100820, 100821, 100822, 100823, 100824, 100825, 100826, 100827, 100828, + 100829, 100830, 100831, 100832, 100833, 100834, 100835, 100836, 100837, + 100838, 100839, 100840, 100841, 100842, 100843, 100844, 100845, 100846, + 100847, 100848, 100849, 100850, 100851, 100852, 100853, 100854, 100855, + 100856, 100857, 100858, 100859, 100860, 100861, 100862, 100863, 100864, + 100865, 100866, 100867, 100868, 100869, 100870, 100871, 100872, 100873, + 100874, 100875, 100876, 100877, 100878, 100879, 100880, 100881, 100882, + 100883, 100884, 100885, 100886, 100887, 100888, 100889, 100890, 100891, + 100892, 100893, 100894, 100895, 100896, 100897, 100898, 100899, 100900, + 100901, 100902, 100903, 100904, 100905, 100906, 100907, 100908, 100909, + 100910, 100911, 100912, 100913, 100914, 100915, 100916, 100917, 100918, + 100919, 100920, 100921, 100922, 100923, 100924, 100925, 100926, 100927, + 100928, 100929, 100930, 100931, 100932, 100933, 100934, 100935, 100936, + 100937, 100938, 100939, 100940, 100941, 100942, 100943, 100944, 100945, + 100946, 100947, 100948, 100949, 100950, 100951, 100952, 100953, 100954, + 100955, 100956, 100957, 100958, 100959, 100960, 100961, 100962, 100963, + 100964, 100965, 100966, 100967, 100968, 100969, 100970, 100971, 100972, + 100973, 100974, 100975, 100976, 100977, 100978, 100979, 100980, 100981, + 100982, 100983, 100984, 100985, 100986, 100987, 100988, 100989, 100990, + 100991, 100992, 100993, 100994, 100995, 100996, 100997, 100998, 100999, + 101000, 101001, 101002, 101003, 101004, 101005, 101006, 101007, 101008, + 101009, 101010, 101011, 101012, 101013, 101014, 101015, 101016, 101017, + 101018, 101019, 101020, 101021, 101022, 101023, 101024, 101025, 101026, + 101027, 101028, 101029, 101030, 101031, 101032, 101033, 101034, 101035, + 101036, 101037, 101038, 101039, 101040, 101041, 101042, 101043, 101044, + 101045, 101046, 101047, 101048, 101049, 101050, 101051, 101052, 101053, + 101054, 101055, 101056, 101057, 101058, 101059, 101060, 101061, 101062, + 101063, 101064, 101065, 101066, 101067, 101068, 101069, 101070, 101071, + 101072, 101073, 101074, 101075, 101076, 101077, 101078, 101079, 101080, + 101081, 101082, 101083, 101084, 101085, 101086, 101087, 101088, 101089, + 101090, 101091, 101092, 101093, 101094, 101095, 101096, 101097, 101098, + 101099, 101100, 101101, 101102, 101103, 101104, 101105, 101106, 101107, + 101108, 101109, 101110, 101111, 101112, 101113, 101114, 101115, 101116, + 101117, 101118, 101119, 94176, 128429, 9991, 9801, 127790, 128661, + 127861, 128198, 10043, 10170, 129750, 129528, 128222, 128380, 8981, 9990, + 8481, 128384, 128301, 128250, 3158, 3195, 3198, 3194, 3197, 3193, 3196, + 3192, 3106, 3105, 3111, 3161, 3110, 3124, 3123, 3122, 3112, 3165, 3097, + 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, 3160, 3108, 3077, 3078, + 3088, 3092, 3083, 3168, 3084, 3169, 3125, 3081, 3082, 3090, 3091, 3079, + 3080, 3126, 3127, 3128, 3117, 3116, 3099, 3098, 3096, 3095, 3101, 3100, + 3094, 3093, 3115, 3114, 3086, 3087, 3129, 3118, 3119, 3157, 3076, 3072, + 3073, 3191, 3199, 3133, 3074, 3132, 3149, 3075, 3134, 3144, 3148, 3139, + 3140, 3170, 3171, 3137, 3138, 3146, 3147, 3135, 3136, 3142, 3143, 3179, + 3178, 3181, 3180, 3177, 3176, 3174, 3183, 3175, 3182, 127934, 8376, 9978, + 129514, 119617, 119564, 119577, 119633, 119587, 119566, 119561, 119585, + 119613, 119590, 119631, 119634, 119630, 119608, 119582, 119563, 119573, + 119558, 119624, 119567, 119623, 119586, 119636, 119612, 119625, 119568, + 119584, 119619, 119618, 119583, 119603, 119600, 119626, 119610, 119580, + 119576, 119638, 119559, 119595, 119632, 119606, 119592, 119615, 119599, + 119602, 119614, 119629, 119574, 119569, 119570, 119622, 119562, 119591, + 119637, 119597, 119589, 119616, 119609, 119560, 119635, 119565, 119604, + 119588, 119571, 119594, 119578, 119596, 119579, 119598, 119572, 119605, + 119627, 119621, 119628, 119601, 119593, 119607, 119575, 119620, 119611, + 119581, 1959, 1958, 1964, 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, + 1956, 1934, 1935, 1955, 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, + 1942, 1941, 1922, 1969, 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, + 1952, 1928, 1957, 1938, 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, + 1960, 1962, 1968, 3610, 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, + 3663, 3630, 3627, 3588, 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, + 3653, 3622, 3659, 3633, 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, + 3661, 3603, 3609, 3631, 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, + 3652, 3651, 3634, 3649, 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, + 3650, 3625, 3624, 3626, 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, + 3605, 3599, 3623, 3662, 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, + 3667, 3666, 3664, 3673, 3665, 3672, 8708, 8707, 8756, 127777, 10727, + 128936, 8201, 128929, 129300, 129353, 128173, 129652, 10176, 8278, 9887, + 9886, 11057, 128485, 128484, 128486, 128487, 8694, 128433, 10870, 128491, + 128962, 128423, 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, + 128077, 128078, 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, + 3886, 3885, 3888, 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, + 3879, 3878, 3875, 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, + 3947, 3904, 3948, 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, + 3918, 3913, 3923, 3940, 3941, 3942, 3930, 3929, 3915, 3914, 3920, 3919, + 3927, 3926, 3910, 3909, 3907, 3906, 3925, 3924, 3934, 3935, 3936, 3943, + 3911, 3939, 3928, 3933, 3937, 3944, 3862, 3861, 983210, 3850, 4048, 3849, + 3892, 4049, 3894, 3859, 3846, 3896, 4052, 3845, 3847, 3842, 3843, 3841, + 3860, 3898, 3899, 3900, 3901, 3972, 4051, 3844, 3851, 3895, 3893, 4050, + 3856, 3854, 3973, 3858, 3857, 3848, 3853, 4058, 3897, 3855, 4057, 3852, + 3978, 3979, 3974, 3976, 3903, 3977, 3970, 3866, 3867, 3868, 3871, 3869, + 3870, 4047, 4046, 3966, 3967, 3975, 3902, 3980, 3971, 4028, 4026, 4027, + 3997, 3996, 4012, 4011, 4002, 4001, 4025, 3985, 3984, 3988, 3998, 3993, + 4003, 4020, 4021, 4022, 4010, 4009, 3995, 3994, 4000, 3999, 4007, 4006, + 3990, 3989, 3987, 3986, 4005, 4004, 4014, 4015, 4016, 4023, 3991, 4019, + 4008, 4018, 4013, 4017, 4024, 3983, 3982, 3981, 3840, 4036, 4041, 4044, + 4043, 4042, 4038, 4040, 4037, 4039, 3968, 3969, 3958, 3959, 3960, 3961, + 3956, 3957, 3964, 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, + 11608, 11595, 11585, 11573, 11620, 11607, 11600, 11582, 11590, 11596, + 11601, 11586, 11592, 11568, 11575, 11577, 11578, 11576, 11571, 11606, + 11572, 11581, 11589, 11583, 11569, 11570, 11584, 11587, 11609, 11611, + 11610, 11612, 11613, 11615, 11619, 11594, 11621, 11604, 11605, 11614, + 11588, 11580, 11574, 11597, 11598, 11599, 11602, 11591, 11616, 11617, + 11618, 11622, 11579, 11593, 11623, 11603, 11631, 11632, 128005, 128047, + 10053, 126, 8764, 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, + 10708, 10709, 10750, 68410, 70854, 70784, 70785, 70786, 70796, 70798, + 70812, 70811, 70817, 70816, 70810, 70809, 70815, 70814, 70791, 70792, + 70793, 70794, 70827, 70789, 70790, 70787, 70788, 70803, 70813, 70808, + 70818, 70828, 70829, 70830, 70822, 70821, 70805, 70804, 70802, 70801, + 70807, 70806, 70800, 70799, 70820, 70819, 70831, 70826, 70823, 70825, + 70824, 70795, 70797, 70852, 70848, 70847, 70851, 70850, 70849, 70832, + 70843, 70846, 70842, 70845, 70837, 70838, 70839, 70840, 70835, 70836, + 70833, 70834, 70841, 70844, 70869, 70868, 70871, 70870, 70867, 70866, + 70864, 70873, 70865, 70872, 70853, 70855, 11858, 8266, 128555, 127915, + 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, 11210, + 11865, 11866, 11833, 8992, 127913, 9180, 8988, 8975, 11810, 8989, 8974, + 11811, 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, + 127560, 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, + 123561, 123564, 123537, 123553, 123555, 123544, 123543, 123558, 123560, + 123546, 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, + 123542, 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, + 123566, 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, + 10971, 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, + 128305, 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, + 10857, 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, + 10999, 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, + 127930, 8366, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, + 11832, 8587, 8586, 128598, 128596, 8985, 128399, 8513, 8514, 8516, 11829, + 8526, 128599, 128597, 8489, 128034, 129347, 127799, 10041, 128256, 8273, + 128432, 10869, 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, + 10759, 128108, 128109, 128490, 11834, 66432, 66451, 66454, 66433, 66436, + 66447, 66457, 66434, 66437, 66440, 66443, 66435, 66445, 66455, 66453, + 66450, 66444, 66461, 66456, 66441, 66458, 66442, 66439, 66449, 66448, + 66452, 66438, 66446, 66459, 66460, 66463, 9730, 9969, 9748, 128530, + 11217, 8255, 9100, 8746, 10824, 10822, 10826, 10817, 10818, 10821, + 983116, 11258, 9842, 129412, 9903, 10685, 8963, 8996, 11193, 10577, + 10573, 10572, 10575, 8597, 8616, 11021, 8661, 129113, 8691, 11109, 10622, + 8944, 8869, 10207, 128742, 128744, 128316, 128743, 128314, 9709, 9710, + 129898, 129946, 129920, 129896, 9985, 129924, 9600, 129937, 129938, 9690, + 129934, 9696, 129885, 129887, 129889, 129886, 129883, 129888, 129881, + 129884, 129882, 129879, 129880, 10196, 129944, 129948, 9720, 9136, 9692, + 129922, 9620, 129874, 129875, 129892, 129894, 129890, 129878, 129893, + 129891, 129876, 129895, 129877, 10064, 9137, 10000, 9693, 128319, 10066, + 129945, 129949, 9721, 129926, 129923, 129925, 129901, 128579, 11797, + 8593, 129976, 8645, 8670, 129033, 129029, 129177, 129041, 129025, 129045, + 8624, 8625, 10505, 8613, 10514, 11145, 11014, 8657, 8673, 129077, 10606, + 10595, 10592, 10584, 8639, 10588, 10580, 8638, 129089, 129093, 129085, + 10224, 128621, 129105, 129081, 11105, 11137, 11121, 129065, 11170, 11171, + 129061, 129057, 129073, 129069, 11131, 11115, 11141, 129169, 10506, 8607, + 11245, 10569, 8648, 8679, 8682, 11192, 8683, 8684, 8685, 129173, 8686, + 8687, 9797, 983117, 42509, 42510, 42511, 42446, 42370, 42486, 42257, + 42333, 42294, 42407, 42445, 42369, 42485, 42256, 42332, 42293, 42406, + 42449, 42373, 42489, 42260, 42336, 42297, 42410, 42434, 42359, 42473, + 42246, 42321, 42283, 42396, 42435, 42360, 42474, 42247, 42322, 42284, + 42397, 42452, 42376, 42492, 42263, 42339, 42300, 42413, 42451, 42375, + 42491, 42262, 42338, 42299, 42412, 42444, 42368, 42484, 42255, 42331, + 42292, 42405, 42443, 42367, 42483, 42254, 42330, 42291, 42404, 42454, + 42378, 42494, 42265, 42341, 42302, 42415, 42453, 42377, 42493, 42264, + 42340, 42301, 42414, 42439, 42440, 42364, 42479, 42251, 42480, 42327, + 42288, 42401, 42502, 42272, 42503, 42461, 42385, 42349, 42309, 42422, + 42429, 42430, 42355, 42468, 42242, 42469, 42316, 42317, 42278, 42279, + 42391, 42392, 42476, 42249, 42477, 42437, 42362, 42324, 42325, 42286, + 42399, 42459, 42383, 42346, 42347, 42499, 42270, 42307, 42420, 42487, + 42508, 42258, 42447, 42371, 42334, 42295, 42408, 42436, 42361, 42475, + 42248, 42323, 42285, 42398, 42438, 42363, 42478, 42250, 42326, 42287, + 42400, 42462, 42386, 42504, 42273, 42350, 42310, 42423, 42450, 42514, + 42539, 42512, 42513, 42538, 42374, 42490, 42261, 42337, 42298, 42411, + 42507, 42500, 42271, 42501, 42460, 42384, 42348, 42308, 42421, 42315, + 42467, 42428, 42457, 42381, 42497, 42268, 42344, 42305, 42418, 42464, + 42388, 42506, 42275, 42352, 42312, 42425, 42463, 42387, 42505, 42274, + 42351, 42311, 42424, 42455, 42379, 42495, 42266, 42342, 42303, 42416, + 42441, 42365, 42481, 42252, 42328, 42289, 42402, 42456, 42380, 42496, + 42267, 42343, 42304, 42417, 42433, 42358, 42472, 42245, 42320, 42282, + 42395, 42448, 42372, 42488, 42259, 42335, 42296, 42409, 42442, 42366, + 42482, 42253, 42329, 42290, 42403, 42458, 42382, 42498, 42269, 42345, + 42306, 42419, 42470, 42243, 42244, 42471, 42431, 42356, 42357, 42432, + 42318, 42319, 42280, 42281, 42393, 42394, 42465, 42240, 42241, 42466, + 42426, 42353, 42354, 42427, 42313, 42314, 42276, 42277, 42389, 42390, + 42523, 42526, 42522, 42515, 42520, 42527, 42516, 42524, 42518, 42517, + 42525, 42521, 42519, 42533, 42532, 42535, 42534, 42531, 42530, 42528, + 42537, 42529, 42536, 65024, 65033, 917843, 917844, 917845, 917846, + 917847, 917848, 917849, 917850, 917851, 917852, 65034, 917853, 917854, + 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 65035, + 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, + 917872, 65036, 917873, 917874, 917875, 917876, 917877, 917878, 917879, + 917880, 917881, 917882, 65037, 917883, 917884, 917885, 917886, 917887, + 917888, 917889, 917890, 917891, 917892, 65038, 917893, 917894, 917895, + 917896, 917897, 917898, 917899, 917900, 917901, 917902, 65039, 917903, + 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, + 917760, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, + 917921, 917922, 917761, 917923, 917924, 917925, 917926, 917927, 917928, + 917929, 917930, 917931, 917932, 917762, 917933, 917934, 917935, 917936, + 917937, 917938, 917939, 917940, 917941, 917942, 65025, 917763, 917943, + 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, + 917764, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, + 917961, 917962, 917765, 917963, 917964, 917965, 917966, 917967, 917968, + 917969, 917970, 917971, 917972, 917766, 917973, 917974, 917975, 917976, + 917977, 917978, 917979, 917980, 917981, 917982, 917767, 917983, 917984, + 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917768, + 917993, 917994, 917995, 917996, 917997, 917998, 917999, 917769, 917770, + 917771, 917772, 65026, 917773, 917774, 917775, 917776, 917777, 917778, + 917779, 917780, 917781, 917782, 65027, 917783, 917784, 917785, 917786, + 917787, 917788, 917789, 917790, 917791, 917792, 65028, 917793, 917794, + 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 65029, + 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, + 917812, 65030, 917813, 917814, 917815, 917816, 917817, 917818, 917819, + 917820, 917821, 917822, 65031, 917823, 917824, 917825, 917826, 917827, + 917828, 917829, 917830, 917831, 917832, 65032, 917833, 917834, 917835, + 917836, 917837, 917838, 917839, 917840, 917841, 917842, 129499, 983245, + 983254, 983356, 983357, 983358, 983359, 983360, 983361, 983362, 983363, + 983364, 983365, 983255, 983366, 983367, 983368, 983369, 983370, 983371, + 983372, 983373, 983374, 983375, 983256, 983376, 983377, 983378, 983379, + 983380, 983381, 983382, 983383, 983384, 983385, 983257, 983386, 983387, + 983388, 983389, 983390, 983391, 983392, 983393, 983394, 983395, 983258, + 983396, 983397, 983398, 983399, 983400, 983401, 983402, 983403, 983404, + 983405, 983259, 983406, 983407, 983408, 983409, 983410, 983411, 983412, + 983413, 983414, 983415, 983260, 983416, 983417, 983418, 983419, 983420, + 983421, 983422, 983423, 983424, 983425, 983273, 983426, 983427, 983428, + 983429, 983430, 983431, 983432, 983433, 983434, 983435, 983274, 983436, + 983437, 983438, 983439, 983440, 983441, 983442, 983443, 983444, 983445, + 983275, 983446, 983447, 983448, 983449, 983450, 983451, 983452, 983453, + 983454, 983455, 983246, 983276, 983456, 983457, 983458, 983459, 983460, + 983461, 983462, 983463, 983464, 983465, 983277, 983466, 983467, 983468, + 983469, 983470, 983471, 983472, 983473, 983474, 983475, 983278, 983476, + 983477, 983478, 983479, 983480, 983481, 983482, 983483, 983484, 983485, + 983279, 983486, 983487, 983488, 983489, 983490, 983491, 983492, 983493, + 983494, 983495, 983280, 983496, 983497, 983498, 983499, 983500, 983501, + 983502, 983503, 983504, 983505, 983281, 983506, 983507, 983508, 983509, + 983510, 983511, 983512, 983282, 983283, 983284, 983285, 983247, 983286, + 983287, 983288, 983289, 983290, 983291, 983292, 983293, 983294, 983295, + 983248, 983296, 983297, 983298, 983299, 983300, 983301, 983302, 983303, + 983304, 983305, 983249, 983306, 983307, 983308, 983309, 983310, 983311, + 983312, 983313, 983314, 983315, 983250, 983316, 983317, 983318, 983319, + 983320, 983321, 983322, 983323, 983324, 983325, 983251, 983326, 983327, + 983328, 983329, 983330, 983331, 983332, 983333, 983334, 983335, 983252, + 983336, 983337, 983338, 983339, 983340, 983341, 983342, 983343, 983344, + 983345, 983253, 983346, 983347, 983348, 983349, 983350, 983351, 983352, + 983353, 983354, 983355, 7401, 7402, 7409, 7403, 7404, 7415, 7410, 7418, + 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, 7405, 7414, 7397, 7400, + 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, 7389, 7376, 7388, 7378, + 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, 7385, 7393, 7377, 8483, + 10704, 10980, 10978, 10186, 8942, 8286, 12347, 12337, 12339, 12341, + 12338, 12340, 124, 9168, 10992, 10991, 9087, 9896, 129904, 129905, + 129906, 129907, 129908, 129909, 11135, 983069, 983144, 11823, 128678, + 11837, 10650, 128976, 128959, 128947, 128637, 128941, 128953, 128636, + 128904, 128914, 128934, 8921, 8920, 128933, 10799, 9910, 128243, 9996, + 128249, 127918, 128252, 94193, 94192, 8983, 127931, 9805, 66929, 66930, + 66936, 66935, 66942, 66943, 66947, 66946, 66950, 66949, 66932, 66931, + 66934, 66933, 66957, 66956, 66959, 66958, 66941, 66940, 66937, 66944, + 66948, 66952, 66954, 66961, 66962, 66965, 66938, 66945, 66953, 66928, + 66951, 66960, 66964, 66968, 66969, 66975, 66974, 66981, 66982, 66986, + 66985, 66989, 66988, 66971, 66970, 66973, 66972, 66996, 66995, 66998, + 66997, 66980, 66979, 66976, 66983, 66987, 66991, 66993, 67000, 67001, + 67004, 66977, 66984, 66992, 66967, 66990, 66999, 67003, 129979, 127755, + 127952, 8752, 983070, 983145, 11238, 8541, 8538, 8536, 8539, 8533, 189, + 8529, 188, 8528, 8537, 8530, 8531, 8540, 190, 8535, 8534, 8532, 8542, + 8585, 123585, 123584, 123624, 123619, 123620, 123606, 123605, 123622, + 123618, 123611, 123623, 123612, 123621, 123615, 123592, 123613, 123625, + 123593, 123596, 123617, 123616, 123595, 123614, 123626, 123627, 123590, + 123609, 123604, 123591, 123594, 123599, 123598, 123586, 123587, 123588, + 123597, 123589, 123610, 123600, 123608, 123607, 123603, 123602, 123601, + 123647, 123630, 123631, 123628, 123629, 123637, 123636, 123639, 123638, + 123635, 123634, 123632, 123641, 123633, 123640, 127768, 127766, 127762, + 127764, 71841, 71850, 71862, 71861, 71848, 71856, 71853, 71867, 71866, + 71865, 71868, 71840, 71859, 71849, 71857, 71869, 71870, 71855, 71847, + 71843, 71854, 71844, 71845, 71858, 71871, 71863, 71864, 71852, 71846, + 71842, 71860, 71851, 71873, 71882, 71894, 71893, 71880, 71888, 71885, + 71899, 71898, 71897, 71900, 71872, 71891, 71881, 71889, 71901, 71902, + 71887, 71879, 71875, 71886, 71876, 71877, 71890, 71903, 71895, 71896, + 71884, 71878, 71874, 71892, 71883, 71909, 71908, 71911, 71910, 71907, + 71906, 71904, 71913, 71905, 71912, 71921, 71918, 71917, 71922, 71920, + 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, 129341, + 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, 65103, + 8967, 65099, 129479, 128465, 128576, 128553, 10172, 128146, 983238, + 127947, 9840, 128734, 9784, 9855, 129197, 129196, 9702, 9815, 129548, + 129590, 129608, 129611, 9812, 129545, 129587, 9816, 129542, 129563, + 129584, 129591, 129605, 129549, 129616, 129614, 129615, 9817, 129550, + 129592, 9813, 129546, 129588, 9814, 129547, 129589, 129569, 129566, + 129570, 129571, 129567, 129568, 9675, 128906, 9862, 10732, 9863, 9717, + 9718, 9716, 9719, 9831, 10209, 10210, 10211, 9671, 9672, 128922, 128923, + 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, 10069, + 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, 9825, + 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, 9756, + 9669, 9667, 9665, 128928, 11046, 11088, 9725, 9723, 11048, 11229, 10001, + 9649, 11040, 127985, 10068, 11092, 9659, 9657, 9655, 128073, 9758, 9645, + 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, 9633, 128307, 128916, + 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, 9715, 9707, 9093, + 127779, 127781, 127782, 9788, 127780, 9734, 128382, 9743, 9186, 10177, + 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11006, 11055, 9647, 11038, + 10163, 128011, 129144, 129152, 129120, 129136, 129128, 129146, 129154, + 129122, 129138, 129130, 129147, 129155, 129123, 129139, 129131, 129145, + 129153, 129121, 129137, 129129, 129149, 129157, 129125, 129141, 129133, + 129148, 129156, 129124, 129140, 129132, 129150, 129158, 129126, 129142, + 129134, 129151, 129159, 129127, 129143, 129135, 11838, 129344, 127888, + 127788, 129695, 127863, 128521, 129725, 128430, 128732, 128105, 128111, + 128098, 128090, 128082, 128097, 128698, 11825, 8288, 128506, 128543, + 129713, 8361, 129717, 128058, 127873, 8768, 129340, 128295, 9997, 983233, + 8999, 129659, 129644, 129643, 129641, 129639, 129642, 129645, 129640, + 129637, 129636, 129634, 129632, 129635, 129638, 129633, 8891, 129393, + 128155, 165, 69292, 69291, 69293, 69256, 69255, 69254, 69281, 69268, + 69259, 69248, 69271, 69289, 69286, 69287, 69257, 69278, 69277, 69279, + 69276, 69280, 69296, 69282, 69251, 69250, 69266, 69265, 69267, 69253, + 69252, 69269, 69274, 69275, 69284, 69285, 69272, 69258, 69288, 69297, + 69263, 69260, 69270, 69262, 69261, 69249, 69283, 69273, 69264, 42139, + 42149, 42173, 42172, 42132, 42147, 42179, 42174, 42148, 42169, 42150, + 42134, 42166, 42135, 42145, 42143, 42137, 42175, 42157, 42154, 42167, + 42165, 42163, 42130, 42182, 42129, 42171, 42138, 42140, 42136, 42131, + 42151, 42164, 42181, 42142, 42156, 42170, 42176, 42178, 42160, 42133, + 42144, 42152, 42159, 42161, 42158, 42141, 42146, 42177, 42180, 42155, + 42162, 42128, 42168, 42153, 41070, 41059, 41060, 41058, 41073, 41072, + 41071, 41068, 41069, 41066, 41067, 41065, 41048, 41052, 41053, 41050, + 41051, 41049, 41046, 41047, 41056, 41057, 41054, 41055, 41063, 41064, + 41061, 41062, 41076, 41077, 41074, 41075, 41006, 40995, 40996, 40994, + 41009, 41008, 41007, 41004, 41005, 41002, 41003, 41001, 40984, 40988, + 40989, 40986, 40987, 40985, 40982, 40983, 40992, 40993, 40990, 40991, + 40999, 41000, 40997, 40998, 41012, 41015, 41014, 41013, 41010, 41011, + 41842, 41831, 41832, 41829, 41830, 41845, 41844, 41843, 41841, 41827, + 41828, 41825, 41826, 41839, 41840, 41837, 41838, 41835, 41836, 41833, + 41834, 41848, 41851, 41850, 41849, 41846, 41847, 41670, 41659, 41660, + 41658, 41673, 41672, 41671, 41668, 41669, 41666, 41667, 41665, 41648, + 41652, 41653, 41650, 41651, 41649, 41646, 41647, 41656, 41657, 41654, + 41655, 41663, 41664, 41661, 41662, 41676, 41679, 41678, 41677, 41674, + 41675, 41293, 41282, 41283, 41281, 41296, 41295, 41294, 41291, 41292, + 41272, 41275, 41276, 41274, 41273, 41270, 41271, 41289, 41290, 41288, + 41279, 41280, 41277, 41278, 41286, 41287, 41284, 41285, 41238, 41228, + 41227, 41241, 41240, 41239, 41236, 41237, 41218, 41221, 41222, 41220, + 41219, 41216, 41217, 41234, 41235, 41233, 41225, 41226, 41223, 41224, + 41231, 41232, 41229, 41230, 41174, 41175, 41173, 41171, 41172, 41169, + 41170, 41167, 41168, 41165, 41166, 41184, 41185, 41182, 41183, 41178, + 41181, 41180, 41179, 41176, 41177, 41494, 41496, 41497, 41495, 41492, + 41493, 41516, 41504, 41505, 41502, 41503, 41519, 41518, 41517, 41514, + 41515, 41500, 41501, 41498, 41499, 41512, 41513, 41510, 41511, 41508, + 41509, 41506, 41507, 41460, 41448, 41449, 41446, 41447, 41463, 41462, + 41461, 41458, 41459, 41436, 41440, 41441, 41438, 41439, 41437, 41434, + 41435, 41444, 41445, 41442, 41443, 41456, 41457, 41454, 41455, 41452, + 41453, 41450, 41451, 41584, 41583, 41582, 41389, 41379, 41380, 41378, + 41392, 41391, 41390, 41387, 41388, 41369, 41372, 41373, 41371, 41370, + 41367, 41368, 41385, 41386, 41384, 41382, 41383, 41381, 41376, 41377, + 41374, 41375, 41395, 41398, 41397, 41396, 41393, 41394, 41125, 41117, + 41118, 41116, 41128, 41127, 41126, 41123, 41124, 41107, 41110, 41111, + 41109, 41108, 41105, 41106, 41114, 41115, 41112, 41113, 41121, 41122, + 41119, 41120, 41130, 41133, 41132, 41131, 41129, 41336, 41334, 41335, + 41333, 41332, 41340, 41338, 41339, 41337, 41322, 41326, 41327, 41324, + 41325, 41323, 41320, 41321, 41330, 41331, 41328, 41329, 41556, 41557, + 41554, 41555, 41563, 41564, 41562, 41544, 41548, 41549, 41546, 41547, + 41545, 41542, 41543, 41552, 41553, 41550, 41551, 41560, 41561, 41558, + 41559, 41591, 41592, 41589, 41590, 41598, 41599, 41597, 41587, 41588, + 41585, 41586, 41595, 41596, 41593, 41594, 40962, 40960, 983243, 40966, + 40967, 40964, 40965, 40963, 40961, 42025, 42017, 42018, 42016, 42028, + 42027, 42026, 42023, 42024, 42010, 42014, 42015, 42012, 42013, 42011, + 42008, 42009, 42021, 42022, 42019, 42020, 42031, 42032, 42029, 42030, + 41970, 41962, 41963, 41960, 41961, 41973, 41972, 41971, 41968, 41969, + 41954, 41958, 41959, 41956, 41957, 41955, 41952, 41953, 41966, 41967, + 41964, 41965, 41976, 41979, 41978, 41977, 41974, 41975, 41488, 41476, + 41477, 41475, 41491, 41490, 41489, 41486, 41487, 41466, 41469, 41470, + 41468, 41467, 41464, 41465, 41473, 41474, 41471, 41472, 41484, 41485, + 41482, 41483, 41480, 41481, 41478, 41479, 41424, 41413, 41414, 41411, + 41412, 41427, 41426, 41425, 41422, 41423, 41420, 41421, 41419, 41401, + 41405, 41406, 41403, 41404, 41402, 41399, 41400, 41409, 41410, 41407, + 41408, 41417, 41418, 41415, 41416, 41430, 41433, 41432, 41431, 41428, + 41429, 41538, 41527, 41528, 41526, 41541, 41540, 41539, 41536, 41537, + 41534, 41535, 41533, 41524, 41525, 41522, 41523, 41531, 41532, 41529, + 41530, 41521, 41520, 41157, 41147, 41148, 41145, 41146, 41160, 41159, + 41158, 41155, 41156, 41136, 41139, 41140, 41138, 41137, 41134, 41135, + 41143, 41144, 41141, 41142, 41151, 41152, 41149, 41150, 41163, 41164, + 41161, 41162, 41154, 41153, 41080, 41083, 41084, 41082, 41081, 41078, + 41079, 41087, 41088, 41085, 41086, 41091, 41092, 41089, 41090, 41095, + 41098, 41097, 41096, 41093, 41094, 41101, 41104, 41103, 41102, 41099, + 41100, 41299, 41302, 41301, 41300, 41297, 41298, 41312, 41313, 41311, + 41305, 41306, 41303, 41304, 41309, 41310, 41307, 41308, 41316, 41319, + 41318, 41317, 41314, 41315, 41574, 41572, 41573, 41580, 41581, 41579, + 41566, 41567, 41565, 41570, 41571, 41568, 41569, 41577, 41578, 41575, + 41576, 42048, 42042, 42041, 42051, 42050, 42049, 42047, 42035, 42039, + 42040, 42037, 42038, 42036, 42033, 42034, 42045, 42046, 42043, 42044, + 42054, 42057, 42056, 42055, 42052, 42053, 41881, 41882, 41880, 41878, + 41879, 41876, 41877, 41885, 41886, 41883, 41884, 41889, 41892, 41891, + 41890, 41887, 41888, 41895, 41898, 41897, 41896, 41893, 41894, 42075, + 42067, 42068, 42066, 42076, 42073, 42074, 42060, 42064, 42065, 42062, + 42063, 42061, 42058, 42059, 42071, 42072, 42069, 42070, 41727, 41721, + 41720, 41730, 41729, 41728, 41726, 41723, 41722, 41711, 41714, 41715, + 41713, 41712, 41709, 41710, 41718, 41719, 41716, 41717, 41733, 41736, + 41735, 41734, 41731, 41732, 41725, 41724, 41363, 41352, 41353, 41351, + 41366, 41365, 41364, 41361, 41362, 41343, 41346, 41347, 41345, 41344, + 41341, 41342, 41349, 41350, 41348, 41359, 41360, 41358, 41356, 41357, + 41354, 41355, 41036, 41028, 41029, 41027, 41039, 41038, 41037, 41034, + 41035, 41018, 41021, 41022, 41020, 41019, 41016, 41017, 41025, 41026, + 41023, 41024, 41032, 41033, 41030, 41031, 41042, 41045, 41044, 41043, + 41040, 41041, 41998, 41990, 41991, 41988, 41989, 42001, 42000, 41999, + 41996, 41997, 41982, 41986, 41987, 41984, 41985, 41983, 41980, 41981, + 41994, 41995, 41992, 41993, 42004, 42007, 42006, 42005, 42002, 42003, + 42115, 42107, 42108, 42105, 42106, 42118, 42117, 42116, 42113, 42114, + 42099, 42103, 42104, 42101, 42102, 42100, 42097, 42098, 42111, 42112, + 42109, 42110, 42121, 42124, 42123, 42122, 42119, 42120, 41866, 41855, + 41854, 41869, 41868, 41867, 41864, 41865, 41862, 41863, 41860, 41861, + 41858, 41859, 41856, 41857, 41872, 41875, 41874, 41873, 41870, 41871, + 41853, 41852, 41942, 41931, 41932, 41930, 41945, 41944, 41943, 41940, + 41941, 41938, 41939, 41937, 41928, 41929, 41926, 41927, 41935, 41936, + 41933, 41934, 41948, 41951, 41950, 41949, 41946, 41947, 41772, 41775, + 41776, 41774, 41773, 41770, 41771, 41786, 41787, 41785, 41779, 41780, + 41777, 41778, 41783, 41784, 41781, 41782, 41790, 41791, 41788, 41789, + 41794, 41797, 41796, 41795, 41792, 41793, 41916, 41904, 41905, 41903, + 41919, 41918, 41917, 41914, 41915, 41901, 41902, 41899, 41900, 41912, + 41913, 41910, 41911, 41908, 41909, 41906, 41907, 41922, 41925, 41924, + 41923, 41920, 41921, 41760, 41749, 41750, 41748, 41763, 41762, 41761, + 41758, 41759, 41739, 41742, 41743, 41741, 41740, 41737, 41738, 41756, + 41757, 41755, 41746, 41747, 41744, 41745, 41753, 41754, 41751, 41752, + 41766, 41769, 41768, 41767, 41764, 41765, 41266, 41255, 41256, 41253, + 41254, 41269, 41268, 41267, 41264, 41265, 41244, 41247, 41248, 41246, + 41245, 41242, 41243, 41262, 41263, 41261, 41251, 41252, 41249, 41250, + 41259, 41260, 41257, 41258, 41203, 41202, 41188, 41192, 41193, 41190, + 41191, 41189, 41186, 41187, 41196, 41197, 41194, 41195, 41200, 41201, + 41198, 41199, 41206, 41209, 41208, 41207, 41204, 41205, 41212, 41215, + 41214, 41213, 41210, 41211, 40981, 41605, 41606, 41604, 41611, 41612, + 41610, 41608, 41609, 41607, 41602, 41603, 41600, 41601, 42079, 42083, + 42084, 42081, 42082, 42080, 42077, 42078, 42089, 42090, 42087, 42088, + 42093, 42096, 42095, 42094, 42091, 42092, 42086, 42085, 41815, 41803, + 41804, 41802, 41818, 41817, 41816, 41813, 41814, 41800, 41801, 41798, + 41799, 41811, 41812, 41809, 41810, 41807, 41808, 41805, 41806, 41821, + 41824, 41823, 41822, 41819, 41820, 41636, 41625, 41626, 41624, 41639, + 41638, 41637, 41634, 41635, 41615, 41618, 41619, 41617, 41616, 41613, + 41614, 41696, 41697, 41695, 41693, 41694, 41692, 41682, 41686, 41687, + 41684, 41685, 41683, 41680, 41681, 41690, 41691, 41688, 41689, 41699, + 41702, 41701, 41700, 41698, 41705, 41708, 41707, 41706, 41703, 41704, + 41632, 41633, 41631, 41622, 41623, 41620, 41621, 41629, 41630, 41627, + 41628, 41642, 41645, 41644, 41643, 41640, 41641, 40973, 40974, 40972, + 40970, 40971, 40968, 40969, 40977, 40978, 40975, 40976, 40980, 40979, + 9775, 129664, 8959, 10853, 10632, 10634, 10814, 10852, 10631, 10633, + 10783, 10784, 10785, 10625, 10626, 72262, 72256, 72253, 72252, 72254, + 72251, 72250, 72261, 72255, 72243, 72215, 72214, 72230, 72229, 72220, + 72219, 72242, 72204, 72203, 72207, 72216, 72211, 72221, 72238, 72239, + 72240, 72228, 72227, 72213, 72212, 72218, 72217, 72225, 72224, 72209, + 72208, 72206, 72205, 72223, 72222, 72231, 72232, 72233, 72241, 72210, + 72236, 72226, 72235, 72237, 72234, 72192, 72259, 72258, 72260, 72257, + 72248, 72245, 72246, 72247, 72244, 72249, 72263, 72202, 72199, 72200, + 72198, 72197, 72195, 72194, 72201, 72196, 72193, 129427, 65279, 8204, + 8203, 8205, 11234, 129296, 118593, 118584, 118585, 118591, 118580, + 118589, 118528, 118540, 118531, 118543, 118559, 118529, 118541, 118532, + 118544, 118592, 118573, 118569, 118568, 118581, 118586, 118561, 118582, + 118583, 118566, 118538, 118550, 118555, 118556, 118535, 118547, 118537, + 118549, 118553, 118558, 118534, 118546, 118572, 118562, 118571, 118554, + 118533, 118545, 118587, 118588, 118530, 118542, 118552, 118563, 118539, + 118551, 118557, 118536, 118548, 118579, 118570, 118560, 118567, 118565, + 118564, 118590, 118576, 118577, 118578, 118619, 118639, 118637, 118645, + 118721, 118611, 118635, 118659, 118626, 118623, 118625, 118624, 118622, + 118638, 118612, 118660, 118608, 118609, 118657, 118719, 118695, 118699, + 118698, 118697, 118696, 118722, 118720, 118703, 118708, 118707, 118706, + 118705, 118704, 118610, 118620, 118723, 118616, 118617, 118640, 118672, + 118636, 118658, 118652, 118651, 118649, 118650, 118648, 118646, 118647, + 118644, 118643, 118641, 118642, 118653, 118656, 118654, 118655, 118662, + 118670, 118664, 118666, 118669, 118663, 118665, 118671, 118667, 118668, + 118673, 118614, 118615, 118618, 118685, 118687, 118684, 118683, 118686, + 118681, 118680, 118709, 118713, 118711, 118716, 118717, 118714, 118715, + 118712, 118718, 118710, 118676, 118679, 118690, 118688, 118693, 118694, + 118691, 118692, 118689, 118675, 118677, 118678, 118674, 118701, 118702, + 118700, 118682, 118632, 118631, 118634, 118633, 118628, 118627, 118630, + 118629, 118613, 118621, 118661, 118596, 118597, 118594, 118595, 118598, + 129503, 983264, 983222, 983221, 983223, }; -static const unsigned short phrasebook_offset1[] = { +#define DAWG_CODEPOINT_TO_POS_SHIFT 8 +#define DAWG_CODEPOINT_TO_POS_NOTFOUND 35762 +static const unsigned char dawg_codepoint_to_pos_index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 105, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, - 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 130, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 131, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 104, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 104, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 181, 104, 182, 183, 104, 184, 185, - 186, 187, 104, 188, 189, 190, 191, 192, 193, 194, 104, 195, 196, 197, - 198, 104, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 223, 224, 225, 226, 227, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 228, 229, 230, 231, 232, - 233, 234, 235, 104, 104, 104, 104, 236, 237, 238, 239, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 250, 251, - 252, 253, 254, 255, 256, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 257, 258, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 104, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 104, 104, 104, 104, 104, 104, 104, 104, 282, 104, 283, 284, 285, - 104, 104, 286, 104, 104, 104, 287, 104, 104, 104, 104, 104, 288, 289, - 290, 291, 104, 104, 104, 104, 104, 292, 293, 294, 104, 295, 296, 104, - 104, 297, 298, 299, 300, 301, 104, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 320, 321, 322, - 323, 324, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 325, 104, 326, 327, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 328, 329, 330, - 331, 332, 333, 334, 335, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 53, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 66, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 67, 68, 69, 70, 71, 72, + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 52, 52, 52, 52, 52, 52, 52, 52, 52, 112, 113, + 114, 115, 116, 117, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 118, 119, 120, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 121, 122, 123, 124, 52, 52, 125, 126, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 127, 128, 129, 130, 131, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 132, 133, 134, 135, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 136, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 52, 52, + 52, 52, 149, 150, 151, 152, 52, 153, 52, 52, 154, 155, 156, 52, 52, 157, + 158, 159, 52, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 172, 173, 174, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 175, + 176, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 177, 178, 179, 180, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, }; -static const unsigned int phrasebook_offset2[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 6, 9, 11, 14, 17, 19, 21, 24, 27, 29, 31, - 33, 35, 39, 41, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 69, 72, - 75, 78, 82, 86, 91, 96, 101, 105, 110, 115, 120, 124, 129, 134, 138, 143, - 148, 152, 157, 162, 166, 171, 176, 180, 185, 190, 195, 200, 205, 209, - 213, 217, 221, 224, 228, 232, 237, 242, 247, 251, 256, 261, 266, 270, - 275, 280, 284, 289, 294, 298, 303, 308, 312, 317, 322, 326, 331, 336, - 341, 346, 351, 356, 359, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 370, 375, - 378, 381, 384, 387, 390, 393, 395, 398, 404, 412, 415, 419, 422, 424, - 427, 430, 433, 436, 440, 443, 446, 449, 451, 454, 460, 468, 475, 482, - 489, 494, 501, 508, 515, 522, 529, 537, 542, 550, 558, 566, 574, 582, - 590, 598, 606, 614, 619, 627, 634, 641, 648, 655, 662, 665, 671, 678, - 685, 692, 699, 707, 712, 719, 726, 733, 740, 747, 754, 762, 767, 775, - 783, 791, 799, 807, 815, 823, 831, 839, 844, 852, 859, 866, 873, 880, - 887, 890, 896, 903, 910, 917, 924, 932, 937, 945, 952, 959, 966, 973, - 980, 987, 995, 1003, 1011, 1019, 1027, 1035, 1043, 1051, 1059, 1067, - 1074, 1081, 1089, 1097, 1105, 1113, 1121, 1129, 1137, 1145, 1153, 1161, - 1169, 1177, 1185, 1193, 1201, 1209, 1217, 1225, 1233, 1241, 1248, 1255, - 1263, 1271, 1279, 1287, 1295, 1303, 1311, 1319, 1327, 1333, 1338, 1343, - 1351, 1359, 1367, 1375, 1380, 1388, 1396, 1404, 1412, 1420, 1428, 1437, - 1446, 1453, 1460, 1468, 1476, 1484, 1492, 1500, 1508, 1519, 1524, 1529, - 1536, 1543, 1550, 1557, 1565, 1573, 1578, 1583, 1591, 1599, 1607, 1615, - 1623, 1631, 1639, 1647, 1655, 1663, 1671, 1679, 1687, 1695, 1703, 1711, - 1719, 1727, 1734, 1741, 1748, 1755, 1762, 1769, 1776, 1783, 1791, 1799, - 1807, 1815, 1822, 1829, 1837, 1845, 1853, 1861, 1869, 1877, 1885, 1893, - 1901, 1909, 1917, 1923, 1929, 1935, 1942, 1949, 1954, 1959, 1965, 1972, - 1979, 1986, 1993, 2001, 2009, 2015, 2021, 2026, 2032, 2039, 2046, 2053, - 2058, 2063, 2068, 2075, 2082, 2089, 2096, 2103, 2109, 2117, 2127, 2135, - 2142, 2149, 2154, 2159, 2166, 2173, 2177, 2182, 2187, 2192, 2200, 2209, - 2216, 2223, 2232, 2239, 2246, 2251, 2258, 2265, 2272, 2279, 2286, 2291, - 2298, 2305, 2313, 2318, 2323, 2328, 2338, 2342, 2348, 2354, 2360, 2366, - 2374, 2387, 2395, 2400, 2410, 2415, 2420, 2430, 2435, 2442, 2449, 2457, - 2465, 2472, 2479, 2486, 2493, 2503, 2513, 2523, 2533, 2543, 2553, 2563, - 2573, 2578, 2588, 2598, 2608, 2618, 2626, 2634, 2641, 2648, 2656, 2664, - 2672, 2680, 2687, 2694, 2704, 2714, 2722, 2730, 2738, 2743, 2753, 2758, - 2766, 2774, 2779, 2784, 2792, 2800, 2811, 2822, 2830, 2838, 2848, 2858, - 2866, 2874, 2883, 2892, 2901, 2910, 2920, 2930, 2939, 2948, 2958, 2968, - 2976, 2984, 2993, 3002, 3011, 3020, 3030, 3040, 3048, 3056, 3065, 3074, - 3083, 3092, 3101, 3110, 3115, 3120, 3128, 3136, 3146, 3154, 3159, 3164, - 3171, 3178, 3185, 3192, 3200, 3208, 3218, 3228, 3238, 3248, 3255, 3262, - 3272, 3282, 3290, 3298, 3306, 3314, 3322, 3329, 3336, 3343, 3349, 3356, - 3363, 3370, 3379, 3389, 3399, 3406, 3413, 3419, 3424, 3430, 3437, 3444, - 3451, 3458, 3469, 3479, 3486, 3493, 3500, 3507, 3512, 3517, 3523, 3529, - 3535, 3543, 3551, 3558, 3564, 3569, 3576, 3582, 3590, 3601, 3611, 3620, - 3627, 3633, 3639, 3644, 3651, 3657, 3664, 3671, 3678, 3683, 3688, 3697, - 3705, 3714, 3719, 3725, 3735, 3742, 3750, 3759, 3765, 3771, 3777, 3784, - 3789, 3794, 3804, 3812, 3821, 3829, 3837, 3847, 3852, 3859, 3866, 3871, - 3883, 3892, 3900, 3906, 3915, 3920, 3925, 3932, 3938, 3944, 3950, 3956, - 3965, 3973, 3978, 3986, 3992, 4000, 4008, 4014, 4020, 4026, 4034, 4042, - 4048, 4056, 4062, 4067, 4074, 4082, 4092, 4099, 4106, 4116, 4123, 4130, - 4140, 4147, 4154, 4161, 4167, 4173, 4182, 4194, 4199, 4206, 4211, 4215, - 4220, 4228, 4235, 4240, 4245, 4249, 4254, 4259, 4263, 4269, 4275, 4281, - 4287, 4295, 4300, 4305, 4310, 4315, 4321, 4323, 4328, 4332, 4338, 4344, - 4350, 4355, 4362, 4369, 4375, 4382, 4390, 4398, 4403, 4408, 4412, 4417, - 4419, 4421, 4424, 4426, 4429, 4434, 4439, 4445, 4450, 4454, 4459, 4464, - 4473, 4479, 4484, 4490, 4495, 4501, 4509, 4517, 4521, 4525, 4530, 4536, - 4542, 4548, 4554, 4559, 4566, 4574, 4582, 4587, 4593, 4600, 4607, 4614, - 4621, 4625, 4631, 4636, 4641, 4646, 4651, 4654, 4657, 4660, 4663, 4666, - 4669, 4673, 4677, 4683, 4686, 4691, 4697, 4703, 4706, 4711, 4716, 4720, - 4726, 4731, 4737, 4743, 4748, 4753, 4758, 4761, 4767, 4772, 4777, 4781, - 4786, 4792, 4798, 4801, 4805, 4809, 4813, 4816, 4819, 4824, 4828, 4835, - 4839, 4845, 4849, 4855, 4859, 4863, 4867, 4872, 4877, 4884, 4890, 4897, - 4903, 4909, 4915, 4918, 4922, 4926, 4930, 4934, 4939, 4944, 4948, 4952, - 4958, 4962, 4966, 4971, 4977, 4982, 4988, 4992, 4999, 5004, 5009, 5014, - 5019, 5025, 5028, 5032, 5037, 5042, 5051, 5057, 5061, 5065, 5070, 5074, - 5079, 5083, 5087, 5092, 5096, 5102, 5107, 5112, 5117, 5122, 5127, 5132, - 5138, 5144, 5150, 5156, 5161, 5167, 5173, 5179, 5184, 5189, 5196, 5203, - 5207, 5212, 5219, 0, 0, 5226, 5229, 5238, 5247, 5258, 5262, 0, 0, 0, 0, - 5267, 5270, 5275, 5283, 5288, 5296, 5304, 0, 5312, 0, 5320, 5328, 5336, - 5347, 5352, 5357, 5362, 5367, 5372, 5377, 5382, 5387, 5392, 5397, 5402, - 5407, 5412, 5417, 5422, 5427, 0, 5432, 5437, 5442, 5447, 5452, 5457, - 5462, 5467, 5475, 5483, 5491, 5499, 5507, 5515, 5526, 5531, 5536, 5541, - 5546, 5551, 5556, 5561, 5566, 5571, 5576, 5581, 5586, 5591, 5596, 5601, - 5606, 5611, 5617, 5622, 5627, 5632, 5637, 5642, 5647, 5652, 5660, 5668, - 5676, 5684, 5692, 5697, 5701, 5705, 5712, 5722, 5732, 5736, 5740, 5744, - 5750, 5757, 5761, 5766, 5770, 5775, 5779, 5784, 5788, 5793, 5798, 5803, - 5808, 5813, 5818, 5823, 5828, 5833, 5838, 5843, 5848, 5853, 5858, 5863, - 5867, 5871, 5877, 5881, 5886, 5892, 5900, 5905, 5910, 5917, 5922, 5927, - 5934, 5943, 5952, 5963, 5971, 5976, 5981, 5986, 5993, 5998, 6004, 6009, - 6014, 6019, 6024, 6029, 6034, 6042, 6048, 6053, 6057, 6062, 6067, 6072, - 6077, 6082, 6087, 6092, 6096, 6102, 6106, 6111, 6116, 6121, 6125, 6130, - 6135, 6140, 6145, 6149, 6154, 6158, 6163, 6168, 6173, 6178, 6184, 6189, - 6195, 6199, 6204, 6208, 6212, 6217, 6222, 6227, 6232, 6237, 6242, 6247, - 6251, 6257, 6261, 6266, 6271, 6276, 6280, 6285, 6290, 6295, 6300, 6304, - 6309, 6313, 6318, 6323, 6328, 6333, 6339, 6344, 6350, 6354, 6359, 6363, - 6371, 6376, 6381, 6386, 6393, 6398, 6404, 6409, 6414, 6419, 6424, 6429, - 6434, 6442, 6448, 6453, 6458, 6463, 6468, 6473, 6479, 6485, 6492, 6499, - 6508, 6517, 6524, 6531, 6540, 6549, 6554, 6559, 6564, 6569, 6574, 6579, - 6584, 6589, 6600, 6611, 6616, 6621, 6628, 6635, 6643, 6651, 6656, 6661, - 6666, 6671, 6675, 6679, 6683, 6689, 6695, 6699, 6706, 6711, 6721, 6731, - 6737, 6743, 6751, 6759, 6767, 6775, 6782, 6789, 6797, 6805, 6813, 6821, - 6829, 6837, 6845, 6853, 6861, 6869, 6876, 6883, 6889, 6895, 6903, 6911, - 6918, 6925, 6933, 6941, 6947, 6953, 6961, 6969, 6977, 6985, 6991, 6997, - 7005, 7013, 7021, 7029, 7036, 7043, 7051, 7059, 7067, 7075, 7080, 7085, - 7092, 7099, 7109, 7119, 7123, 7131, 7139, 7146, 7153, 7161, 7169, 7176, - 7183, 7191, 7199, 7206, 7213, 7221, 7229, 7234, 7241, 7248, 7255, 7262, - 7268, 7274, 7282, 7290, 7295, 7300, 7308, 7316, 7324, 7332, 7340, 7348, - 7355, 7362, 7370, 7378, 7386, 7394, 7401, 7408, 7414, 7420, 7429, 7438, - 7446, 7454, 7461, 7468, 7475, 7482, 7490, 7498, 7506, 7514, 7522, 7530, - 7538, 7546, 7556, 7566, 7573, 7580, 7587, 7594, 7601, 7608, 7615, 7622, - 7629, 7636, 7643, 7650, 7657, 7664, 7671, 7678, 7685, 7692, 7699, 7706, - 7713, 7720, 7727, 7734, 7739, 7744, 7749, 7754, 7759, 7764, 7769, 7774, - 7779, 7784, 7790, 7796, 7804, 7812, 7820, 7828, 7836, 7844, 7852, 7860, - 7868, 7876, 7881, 7886, 7891, 7896, 7904, 0, 7912, 7918, 7924, 7930, - 7936, 7942, 7948, 7954, 7960, 7965, 7971, 7977, 7983, 7989, 7995, 8001, - 8007, 8013, 8019, 8025, 8031, 8037, 8043, 8049, 8055, 8061, 8067, 8073, - 8078, 8084, 8090, 8096, 8102, 8108, 8114, 8120, 8126, 8132, 0, 0, 8138, - 8146, 8150, 8155, 8160, 8164, 8169, 8174, 8181, 8187, 8193, 8199, 8205, - 8211, 8217, 8223, 8229, 8234, 8240, 8246, 8252, 8258, 8264, 8270, 8276, - 8282, 8288, 8294, 8300, 8306, 8312, 8318, 8324, 8330, 8336, 8342, 8347, - 8353, 8359, 8365, 8371, 8377, 8383, 8389, 8395, 8401, 8407, 8415, 8422, - 8428, 0, 0, 8432, 8439, 8446, 0, 8451, 8456, 8461, 8466, 8473, 8480, - 8485, 8490, 8495, 8500, 8505, 8510, 8515, 8522, 8527, 8534, 8541, 8546, - 8553, 8558, 8563, 8568, 8575, 8580, 8585, 8592, 8601, 8606, 8611, 8616, - 8621, 8627, 8632, 8639, 8646, 8653, 8658, 8663, 8668, 8673, 8678, 8683, - 8693, 8698, 8707, 8712, 8717, 8722, 8727, 8734, 8741, 8748, 8753, 8758, - 8765, 0, 0, 0, 0, 0, 0, 0, 0, 8772, 8776, 8780, 8784, 8788, 8792, 8796, - 8800, 8804, 8808, 8812, 8817, 8821, 8825, 8830, 8834, 8839, 8843, 8847, - 8851, 8856, 8860, 8865, 8869, 8873, 8877, 8881, 0, 0, 0, 0, 8885, 8890, - 8897, 8905, 8912, 8917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8922, 8925, - 8929, 8934, 8938, 8942, 8946, 8952, 8958, 8961, 8968, 8977, 8980, 8983, - 8988, 8994, 8998, 9006, 9012, 9018, 9026, 9030, 9035, 9046, 9051, 9055, - 9059, 9063, 9066, 9069, 9076, 9083, 9087, 9093, 9097, 9104, 9111, 9119, - 9126, 9133, 9137, 9141, 9147, 9151, 9155, 9159, 9163, 9167, 9171, 9175, - 9179, 9183, 9187, 9191, 9195, 9199, 9203, 9207, 9211, 9215, 9223, 9231, - 9241, 9250, 9259, 9262, 9266, 9270, 9274, 9278, 9282, 9286, 9290, 9294, - 9299, 9303, 9306, 9309, 9312, 9315, 9318, 9321, 9324, 9327, 9331, 9335, - 9339, 9344, 9349, 9355, 9358, 9365, 9374, 9379, 9384, 9391, 9397, 9402, - 9406, 9410, 9414, 9418, 9422, 9426, 9430, 9434, 9438, 9442, 9447, 9452, - 9459, 9465, 9471, 9477, 9482, 9491, 9500, 9505, 9512, 9519, 9526, 9533, - 9537, 9541, 9545, 9552, 9562, 9566, 9570, 9574, 9581, 9589, 9593, 9597, - 9604, 9608, 9612, 9616, 9623, 9630, 9642, 9646, 9650, 9654, 9664, 9673, - 9677, 9685, 9692, 9699, 9708, 9719, 9727, 9731, 9740, 9751, 9759, 9772, - 9780, 9788, 9796, 9804, 9810, 9819, 9826, 9830, 9838, 9842, 9849, 9857, - 9861, 9867, 9874, 9881, 9885, 9893, 9897, 9904, 9908, 9916, 9920, 9928, - 9936, 9943, 9951, 9959, 9966, 9972, 9976, 9983, 9991, 9997, 10004, 10011, - 10017, 10027, 10035, 10042, 10048, 10052, 10055, 10059, 10065, 10073, - 10077, 10083, 10089, 10096, 10103, 10106, 10113, 10118, 10127, 10132, - 10136, 10149, 10162, 10168, 10175, 10180, 10186, 10191, 10197, 10207, - 10214, 10223, 10233, 10239, 10244, 10249, 10253, 10257, 10262, 10267, - 10273, 10281, 10289, 10300, 10305, 10314, 10323, 10330, 10336, 10342, - 10348, 10354, 10360, 10366, 10372, 10378, 10384, 10391, 10398, 10405, - 10411, 10419, 10428, 10435, 10443, 10451, 10457, 10463, 10469, 10477, - 10484, 10494, 10503, 10507, 10513, 10519, 0, 10525, 10530, 10535, 10542, - 10547, 10552, 10559, 10564, 10573, 10578, 10583, 10588, 10593, 10598, - 10605, 10610, 10617, 10622, 10627, 10632, 10637, 10642, 10648, 10652, - 10657, 10664, 10669, 10674, 10679, 10684, 10689, 10696, 10703, 10710, - 10715, 10720, 10726, 10731, 10736, 10742, 10747, 10752, 10760, 10768, - 10773, 10778, 10784, 10789, 10794, 10798, 10804, 10808, 10812, 10818, - 10824, 10829, 10834, 10841, 10848, 10852, 0, 0, 10856, 10863, 10870, - 10877, 10887, 10899, 10911, 10928, 10940, 10951, 10959, 10966, 10977, - 10992, 11003, 11009, 11018, 11026, 11038, 11048, 11056, 11068, 11075, - 11083, 11095, 11101, 11107, 11115, 11123, 11131, 11137, 11147, 11155, - 11165, 11175, 11188, 11202, 11216, 11226, 11237, 11248, 11261, 11274, - 11288, 11300, 11312, 11325, 11338, 11350, 11363, 11372, 11380, 11385, - 11390, 11395, 11400, 11405, 11410, 11415, 11420, 11425, 11430, 11435, - 11440, 11445, 11450, 11455, 11460, 11465, 11470, 11475, 11480, 11485, - 11490, 11495, 11500, 11505, 11510, 11515, 11520, 11525, 11530, 11535, - 11540, 11544, 11549, 11554, 11559, 11564, 11569, 11573, 11577, 11581, - 11585, 11589, 11593, 11597, 11601, 11605, 11609, 11613, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 11618, 11623, 11627, 11631, 11635, 11639, 11643, - 11647, 11651, 11655, 11659, 11663, 11668, 11672, 11676, 11680, 11685, - 11689, 11694, 11699, 11704, 11708, 11713, 11718, 11723, 11728, 11732, - 11737, 11741, 11746, 11751, 11755, 11759, 11766, 11770, 11775, 11779, - 11783, 11788, 11792, 11799, 11806, 11813, 11819, 11827, 11835, 11844, - 11852, 11859, 11866, 11874, 11880, 11886, 11892, 11898, 11905, 11910, - 11914, 11919, 0, 0, 11923, 11927, 11932, 11937, 11942, 11947, 11952, - 11957, 11962, 11967, 11972, 11977, 11982, 11987, 11992, 11997, 12002, - 12007, 12012, 12017, 12022, 12027, 12032, 12037, 12042, 12047, 12052, - 12057, 12062, 12067, 12075, 12082, 12088, 12093, 12101, 12108, 12114, - 12121, 12127, 12132, 12139, 12146, 12152, 12157, 12162, 12168, 12173, - 12178, 12184, 0, 0, 12189, 12195, 12201, 12207, 12213, 12219, 12225, - 12230, 12238, 12244, 12250, 12256, 12262, 12268, 12276, 0, 12282, 12287, - 12292, 12297, 12302, 12307, 12312, 12317, 12322, 12327, 12332, 12337, - 12342, 12347, 12352, 12357, 12362, 12367, 12372, 12377, 12382, 12387, - 12392, 12397, 12402, 12407, 12412, 12417, 0, 0, 12422, 0, 12426, 12432, - 12438, 12444, 12450, 12456, 12462, 12468, 12473, 12479, 12485, 0, 0, 0, - 0, 0, 12491, 12499, 12510, 12517, 12524, 12532, 12543, 12553, 12564, - 12575, 12585, 12591, 12605, 12616, 12630, 12645, 12657, 12672, 12681, - 12690, 12698, 12706, 12713, 12719, 12726, 12733, 12743, 12753, 12760, - 12768, 12778, 0, 12782, 12787, 0, 0, 0, 0, 0, 0, 12792, 12799, 12806, - 12813, 12820, 12825, 12831, 12836, 12843, 12852, 12860, 12868, 12876, - 12888, 12895, 12902, 12909, 12921, 12932, 12939, 12947, 12953, 12958, - 12966, 12974, 12982, 12988, 12998, 13006, 13013, 13026, 13034, 13042, - 13051, 13060, 13073, 13079, 13085, 13091, 13099, 13107, 13115, 13123, - 13131, 13139, 13149, 13157, 13165, 13176, 13180, 13186, 13193, 13203, - 13210, 13215, 13222, 13229, 13233, 13239, 13250, 13255, 13262, 13267, - 13272, 13277, 13285, 13293, 13300, 13307, 13314, 13321, 13328, 13335, - 13342, 13348, 13356, 13361, 13366, 13371, 13376, 13381, 13386, 13391, - 13396, 13401, 13406, 13411, 13416, 13421, 13426, 13431, 13436, 13441, - 13447, 13453, 13459, 13464, 13469, 13474, 13479, 13485, 13494, 13502, - 13508, 13516, 13522, 13526, 13530, 13534, 13539, 13542, 13546, 13549, - 13553, 13556, 13560, 13564, 13568, 13573, 13578, 13581, 13585, 13590, - 13595, 13598, 13602, 13605, 13609, 13613, 13617, 13621, 13625, 13629, - 13633, 13637, 13641, 13645, 13649, 13653, 13657, 13661, 13665, 13669, - 13673, 13677, 13680, 13684, 13687, 13691, 13695, 13699, 13702, 13705, - 13708, 13712, 13715, 13719, 13723, 13727, 13731, 13735, 13738, 13741, - 13746, 13751, 13755, 13759, 13764, 13768, 13773, 13777, 13782, 13787, - 13793, 13799, 13805, 13809, 13814, 13820, 13826, 13830, 13835, 13839, - 13845, 13850, 13853, 13859, 13865, 13870, 13875, 13882, 13887, 13892, - 13896, 13900, 13904, 13908, 13912, 13916, 13920, 13924, 13929, 13934, - 13939, 13945, 13948, 13952, 13956, 13959, 13962, 13965, 13968, 13971, - 13974, 13977, 13980, 13983, 13987, 13994, 13999, 14003, 14007, 14011, - 14015, 14019, 14025, 14029, 14033, 14037, 14041, 14047, 14051, 14055, - 14059, 14064, 14069, 0, 14074, 14078, 14083, 14087, 14092, 14096, 14101, - 14106, 0, 0, 14111, 14115, 0, 0, 14120, 14124, 14129, 14133, 14138, - 14143, 14148, 14153, 14158, 14163, 14168, 14173, 14178, 14183, 14188, - 14193, 14198, 14203, 14208, 14213, 14218, 14223, 0, 14227, 14231, 14236, - 14241, 14246, 14250, 14254, 0, 14258, 0, 0, 0, 14262, 14267, 14272, - 14276, 0, 0, 14280, 14285, 14290, 14296, 14301, 14307, 14312, 14318, - 14324, 0, 0, 14331, 14336, 0, 0, 14342, 14347, 14353, 14358, 0, 0, 0, 0, - 0, 0, 0, 0, 14365, 0, 0, 0, 0, 14372, 14377, 0, 14382, 14387, 14393, - 14399, 14405, 0, 0, 14412, 14417, 14421, 14425, 14429, 14433, 14437, - 14441, 14445, 14449, 14453, 14462, 14471, 14476, 14481, 14488, 14495, - 14502, 14509, 14524, 14532, 14536, 14541, 14548, 14553, 0, 0, 14558, - 14565, 14570, 0, 14575, 14579, 14584, 14588, 14593, 14597, 0, 0, 0, 0, - 14602, 14607, 0, 0, 14612, 14617, 14622, 14626, 14631, 14636, 14641, - 14646, 14651, 14656, 14661, 14666, 14671, 14676, 14681, 14686, 14691, - 14696, 14701, 14706, 14711, 14716, 0, 14720, 14724, 14729, 14734, 14739, - 14743, 14747, 0, 14751, 14755, 0, 14760, 14765, 0, 14770, 14774, 0, 0, - 14778, 0, 14783, 14789, 14794, 14800, 14805, 0, 0, 0, 0, 14811, 14817, 0, - 0, 14823, 14829, 14835, 0, 0, 0, 14840, 0, 0, 0, 0, 0, 0, 0, 14845, - 14850, 14855, 14860, 0, 14865, 0, 0, 0, 0, 0, 0, 0, 14870, 14875, 14879, - 14883, 14887, 14891, 14895, 14899, 14903, 14907, 14911, 14915, 14919, - 14923, 14927, 14933, 14938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14943, 14948, - 14953, 0, 14958, 14962, 14967, 14971, 14976, 14980, 14985, 14990, 14995, - 0, 15001, 15005, 15010, 0, 15016, 15020, 15025, 15029, 15034, 15039, - 15044, 15049, 15054, 15059, 15064, 15069, 15074, 15079, 15084, 15089, - 15094, 15099, 15104, 15109, 15114, 15119, 0, 15123, 15127, 15132, 15137, - 15142, 15146, 15150, 0, 15154, 15158, 0, 15163, 15168, 15173, 15178, - 15182, 0, 0, 15186, 15191, 15196, 15202, 15207, 15213, 15218, 15224, - 15230, 15237, 0, 15244, 15249, 15255, 0, 15262, 15267, 15273, 0, 0, - 15278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15282, 15288, 15294, - 15300, 0, 0, 15307, 15312, 15316, 15320, 15324, 15328, 15332, 15336, - 15340, 15344, 15348, 15353, 0, 0, 0, 0, 0, 0, 0, 15358, 15363, 15368, - 15373, 15378, 15386, 15394, 0, 15402, 15407, 15412, 0, 15417, 15421, - 15426, 15430, 15435, 15439, 15444, 15449, 0, 0, 15454, 15458, 0, 0, - 15463, 15467, 15472, 15476, 15481, 15486, 15491, 15496, 15501, 15506, - 15511, 15516, 15521, 15526, 15531, 15536, 15541, 15546, 15551, 15556, - 15561, 15566, 0, 15570, 15574, 15579, 15584, 15589, 15593, 15597, 0, - 15601, 15605, 0, 15610, 15615, 15620, 15625, 15629, 0, 0, 15633, 15638, - 15643, 15649, 15654, 15660, 15665, 15671, 15677, 0, 0, 15684, 15689, 0, - 0, 15695, 15700, 15706, 0, 0, 0, 0, 0, 0, 0, 15711, 15716, 15723, 0, 0, - 0, 0, 15730, 15735, 0, 15740, 15745, 15751, 15757, 15763, 0, 0, 15770, - 15775, 15779, 15783, 15787, 15791, 15795, 15799, 15803, 15807, 15811, - 15815, 15820, 15827, 15834, 15841, 15848, 15855, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 15862, 15866, 0, 15870, 15873, 15877, 15880, 15884, 15887, 0, 0, 0, - 15891, 15894, 15898, 0, 15902, 15905, 15909, 15913, 0, 0, 0, 15916, - 15920, 0, 15924, 0, 15928, 15932, 0, 0, 0, 15936, 15940, 0, 0, 0, 15944, - 15947, 15951, 0, 0, 0, 15954, 15957, 15960, 15963, 15967, 15970, 15974, - 15978, 15982, 15986, 15990, 15993, 0, 0, 0, 0, 15996, 16001, 16005, - 16010, 16014, 0, 0, 0, 16019, 16023, 16028, 0, 16033, 16037, 16042, - 16047, 0, 0, 16051, 0, 0, 0, 0, 0, 0, 16054, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16060, 16064, 16067, 16070, 16073, 16076, 16079, 16082, - 16085, 16088, 16091, 16095, 16100, 16104, 16108, 16112, 16116, 16120, - 16124, 16129, 16133, 0, 0, 0, 0, 0, 16136, 16143, 16148, 16153, 16158, - 16165, 16169, 16174, 16178, 16183, 16187, 16192, 16197, 0, 16202, 16206, - 16211, 0, 16216, 16220, 16225, 16230, 16234, 16239, 16244, 16249, 16254, - 16259, 16264, 16269, 16274, 16279, 16284, 16289, 16294, 16299, 16304, - 16309, 16314, 16319, 16324, 0, 16328, 16332, 16337, 16342, 16347, 16351, - 16355, 16359, 16364, 16368, 16373, 16378, 16383, 16388, 16393, 16397, 0, - 0, 16401, 16406, 16411, 16417, 16422, 16428, 16433, 16439, 16445, 0, - 16452, 16457, 16463, 0, 16469, 16474, 16480, 16486, 0, 0, 0, 0, 0, 0, 0, - 16491, 16496, 0, 16503, 16508, 16513, 0, 0, 16518, 0, 0, 16525, 16531, - 16537, 16543, 0, 0, 16550, 16555, 16559, 16563, 16567, 16571, 16575, - 16579, 16583, 16587, 0, 0, 0, 0, 0, 0, 0, 16591, 16596, 16610, 16623, - 16636, 16649, 16662, 16675, 16688, 16693, 16700, 16705, 16710, 16715, - 16720, 16724, 16729, 16733, 16738, 16742, 16747, 16752, 0, 16757, 16761, - 16766, 0, 16771, 16775, 16780, 16785, 16789, 16794, 16799, 16804, 16809, - 16814, 16819, 16824, 16829, 16834, 16839, 16844, 16849, 16854, 16859, - 16864, 16869, 16874, 16879, 0, 16883, 16887, 16892, 16897, 16902, 16906, - 16910, 16914, 16919, 16923, 0, 16928, 16933, 16938, 16943, 16947, 0, 0, - 16951, 16956, 16961, 16967, 16972, 16978, 16983, 16989, 16995, 0, 17002, - 17007, 17013, 0, 17019, 17024, 17030, 17036, 0, 0, 0, 0, 0, 0, 0, 17041, - 17046, 0, 0, 0, 0, 0, 0, 17053, 17060, 0, 17065, 17071, 17077, 17083, 0, - 0, 17090, 17095, 17099, 17103, 17107, 17111, 17115, 17119, 17123, 17127, - 0, 17131, 17136, 17141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17149, 17155, - 17159, 17163, 17167, 17173, 17176, 17180, 17183, 17187, 17190, 17194, - 17198, 0, 17202, 17205, 17209, 0, 17213, 17216, 17220, 17224, 17227, - 17231, 17235, 17239, 17243, 17247, 17251, 17255, 17259, 17263, 17267, - 17271, 17275, 17279, 17283, 17287, 17291, 17295, 17299, 17302, 17306, - 17309, 17313, 17317, 17321, 17324, 17327, 17330, 17334, 17337, 17341, - 17345, 17349, 17353, 17357, 17360, 17363, 17367, 17374, 17380, 17384, - 17389, 17393, 17398, 17402, 17407, 17412, 0, 17418, 17422, 17427, 0, - 17432, 17436, 17441, 17446, 17450, 17455, 0, 0, 0, 0, 17459, 17465, - 17471, 17477, 17483, 17489, 17495, 17501, 17507, 17513, 17519, 17525, - 17531, 17536, 17541, 17546, 0, 0, 17552, 17556, 17559, 17562, 17565, - 17568, 17571, 17574, 17577, 17580, 17583, 17587, 17592, 17596, 17602, - 17608, 17614, 17620, 17626, 17632, 17636, 17642, 17648, 17654, 17659, - 17665, 0, 17671, 17675, 17679, 0, 17683, 17687, 17691, 17695, 17699, - 17703, 17707, 17711, 17715, 17719, 17723, 17727, 17731, 17735, 17739, - 17743, 17747, 17751, 0, 0, 0, 17755, 17761, 17767, 17773, 17779, 17785, - 17791, 17797, 17803, 17809, 17815, 17821, 17829, 17835, 17841, 17847, - 17853, 17859, 17865, 17871, 17877, 17883, 17889, 17895, 0, 17901, 17907, - 17913, 17919, 17925, 17931, 17935, 17941, 17945, 0, 17949, 0, 0, 17955, - 17959, 17965, 17971, 17977, 17981, 17987, 0, 0, 0, 17991, 0, 0, 0, 0, - 17995, 18000, 18007, 18014, 18021, 18028, 0, 18035, 0, 18042, 18047, - 18052, 18059, 18066, 18075, 18086, 18095, 0, 0, 0, 0, 0, 0, 18100, 18106, - 18111, 18116, 18121, 18126, 18131, 18136, 18141, 18146, 0, 0, 18151, - 18158, 18165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18170, 18178, 18186, - 18194, 18202, 18210, 18218, 18226, 18234, 18242, 18250, 18258, 18266, - 18274, 18282, 18289, 18297, 18305, 18313, 18321, 18329, 18336, 18344, - 18352, 18360, 18368, 18376, 18384, 18392, 18400, 18408, 18416, 18424, - 18431, 18439, 18447, 18453, 18461, 18467, 18475, 18483, 18491, 18499, - 18507, 18515, 18522, 18530, 18536, 18543, 18551, 18559, 18567, 18574, - 18582, 18590, 18598, 18605, 18613, 0, 0, 0, 0, 18619, 18626, 18633, - 18641, 18648, 18658, 18668, 18674, 18680, 18686, 18694, 18702, 18710, - 18718, 18724, 18730, 18736, 18742, 18747, 18751, 18755, 18759, 18763, - 18767, 18771, 18775, 18779, 18783, 18789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 18795, 18800, 0, 18807, 0, 18814, 18821, 18826, 18831, 18838, 0, - 18845, 18852, 18857, 18864, 18871, 18878, 18885, 18892, 18899, 18904, - 18908, 18915, 18922, 18929, 18934, 18939, 18944, 18951, 18958, 18965, - 18972, 18979, 18984, 18989, 0, 18996, 0, 19003, 19008, 19015, 19022, - 19029, 19036, 19043, 19047, 19054, 19058, 19063, 19071, 19077, 19083, - 19088, 19094, 19100, 19106, 19111, 19117, 19124, 19132, 19139, 0, 0, - 19146, 19151, 19157, 19162, 19168, 0, 19174, 0, 19179, 19186, 19193, - 19200, 19207, 19212, 19216, 0, 19220, 19225, 19229, 19233, 19237, 19241, - 19245, 19249, 19253, 19257, 0, 0, 19261, 19267, 19273, 19280, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 19287, 19291, 19302, 19317, 19332, 19342, 19353, 19366, - 19377, 19383, 19391, 19401, 19407, 19415, 19419, 19425, 19431, 19439, - 19449, 19457, 19470, 19476, 19484, 19492, 19504, 19511, 19519, 19527, - 19535, 19543, 19551, 19559, 19569, 19573, 19576, 19579, 19582, 19585, - 19588, 19591, 19594, 19597, 19600, 19604, 19608, 19612, 19616, 19620, - 19624, 19628, 19632, 19636, 19641, 19647, 19657, 19671, 19681, 19687, - 19693, 19701, 19709, 19717, 19725, 19731, 19737, 19740, 19744, 19748, - 19752, 19756, 19760, 19764, 0, 19768, 19772, 19776, 19780, 19784, 19788, - 19792, 19796, 19800, 19804, 19808, 19811, 19814, 19818, 19822, 19826, - 19829, 19833, 19837, 19841, 19845, 19849, 19853, 19857, 19861, 19864, - 19867, 19870, 19874, 19878, 19881, 19884, 19887, 19891, 19896, 19900, 0, - 0, 0, 0, 19904, 19909, 19913, 19918, 19922, 19927, 19932, 19938, 19943, - 19949, 19953, 19958, 19962, 19967, 19977, 19983, 19989, 19996, 20006, - 20012, 20016, 20020, 20026, 20032, 20040, 20046, 20054, 20062, 20070, - 20080, 20088, 20098, 20103, 20109, 20115, 20121, 20127, 20133, 20139, 0, - 20145, 20151, 20157, 20163, 20169, 20175, 20181, 20187, 20193, 20199, - 20205, 20210, 20215, 20221, 20227, 20233, 20238, 20244, 20250, 20256, - 20262, 20268, 20274, 20280, 20286, 20291, 20296, 20301, 20307, 20313, - 20318, 20323, 20328, 20334, 20342, 20349, 0, 20356, 20363, 20376, 20383, - 20390, 20398, 20406, 20412, 20418, 20424, 20434, 20439, 20445, 20455, - 20465, 0, 20475, 20485, 20493, 20505, 20517, 20523, 20537, 20552, 20557, - 20562, 20570, 20578, 20586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20594, - 20597, 20601, 20605, 20609, 20613, 20617, 20621, 20625, 20629, 20633, - 20637, 20641, 20645, 20649, 20653, 20657, 20661, 20665, 20669, 20673, - 20676, 20679, 20683, 20687, 20691, 20694, 20697, 20700, 20703, 20707, - 20710, 20713, 20717, 20720, 20725, 20728, 20732, 20735, 20739, 20742, - 20747, 20750, 20754, 20761, 20766, 20770, 20775, 20779, 20784, 20788, - 20793, 20800, 20806, 20812, 20816, 20820, 20824, 20828, 20832, 20838, - 20844, 20851, 20857, 20862, 20866, 20869, 20872, 20875, 20878, 20881, - 20884, 20887, 20890, 20893, 20899, 20903, 20907, 20911, 20915, 20919, - 20923, 20927, 20931, 20936, 20940, 20945, 20950, 20956, 20961, 20967, - 20973, 20979, 20985, 20991, 20999, 21007, 21015, 21023, 21032, 21041, - 21052, 21062, 21072, 21083, 21094, 21104, 21114, 21124, 21134, 21144, - 21154, 21164, 21174, 21182, 21189, 21195, 21202, 21207, 21213, 21219, - 21225, 21231, 21237, 21243, 21248, 21254, 21260, 21266, 21272, 21277, - 21286, 21293, 21299, 21307, 21315, 21321, 21327, 21333, 21339, 21347, - 21355, 21365, 21373, 21381, 21387, 21392, 21397, 21402, 21407, 21412, - 21417, 21422, 21427, 21432, 21438, 21444, 21450, 21457, 21462, 21468, - 21473, 21478, 21483, 21488, 21493, 21498, 21503, 21508, 21513, 21518, - 21523, 21528, 21533, 21538, 21543, 21548, 21553, 21558, 21563, 21568, - 21573, 21578, 21583, 21588, 21593, 21598, 21603, 21608, 21613, 21618, - 21623, 21628, 21633, 21638, 21643, 21648, 21653, 0, 21658, 0, 0, 0, 0, 0, - 21663, 0, 0, 21668, 21672, 21676, 21680, 21684, 21688, 21692, 21696, - 21700, 21704, 21708, 21712, 21716, 21720, 21724, 21728, 21732, 21736, - 21740, 21744, 21748, 21752, 21756, 21760, 21764, 21768, 21772, 21776, - 21780, 21784, 21788, 21792, 21796, 21800, 21804, 21808, 21812, 21816, - 21820, 21824, 21828, 21832, 21837, 21841, 21846, 21851, 21855, 21860, - 21865, 21869, 21873, 21877, 21881, 21885, 21889, 21893, 21897, 21901, - 21905, 21909, 21913, 21917, 21921, 21925, 21929, 21933, 21937, 21941, - 21945, 21949, 21953, 21957, 21961, 21965, 21969, 21973, 21977, 21981, - 21985, 21989, 21993, 21997, 22001, 22005, 22009, 22013, 22017, 22021, - 22025, 22029, 22033, 22037, 22041, 22045, 22049, 22053, 22057, 22061, - 22065, 22069, 22073, 22077, 22081, 22085, 22089, 22093, 22097, 22101, - 22105, 22109, 22113, 22117, 22121, 22125, 22129, 22133, 22137, 22141, - 22145, 22149, 22153, 22157, 22161, 22165, 22169, 22173, 22177, 22181, - 22185, 22189, 22193, 22197, 22201, 22205, 22209, 22213, 22217, 22221, - 22225, 22229, 22233, 22237, 22241, 22245, 22249, 22254, 22258, 22263, - 22267, 22272, 22277, 22281, 22286, 22291, 22295, 22300, 22305, 22310, - 22315, 22319, 22324, 22329, 22334, 22339, 22344, 22349, 22353, 22358, - 22363, 22368, 22373, 22378, 22383, 22388, 22393, 22398, 22403, 22408, - 22413, 22418, 22423, 22428, 22433, 22438, 22443, 22448, 22453, 22458, - 22463, 22468, 22473, 22478, 22483, 22488, 22493, 22498, 22503, 22508, - 22513, 22518, 22523, 22528, 22533, 22538, 22543, 22548, 22553, 22558, - 22563, 22568, 22573, 22578, 22583, 22588, 22593, 22598, 22603, 22607, - 22611, 22615, 22619, 22623, 22627, 22631, 22635, 22639, 22643, 22647, - 22651, 22655, 22659, 22663, 22667, 22671, 22675, 22679, 22683, 22687, - 22691, 22695, 22699, 22703, 22707, 22711, 22715, 22719, 22723, 22727, - 22731, 22735, 22739, 22743, 22747, 22751, 22755, 22759, 22763, 22767, - 22771, 22775, 22779, 22783, 22787, 22791, 22795, 22799, 22803, 22807, - 22811, 22815, 22819, 22823, 22827, 22831, 22835, 22839, 22843, 22847, - 22851, 22855, 22859, 22863, 22867, 22871, 22875, 22879, 22883, 22887, - 22891, 22895, 22899, 22903, 22907, 22911, 22915, 22919, 22923, 22927, - 22931, 22935, 22939, 22943, 22947, 22951, 22955, 22958, 22962, 22966, - 22970, 22974, 22978, 22982, 22986, 22989, 22993, 22997, 23001, 23005, - 23009, 23013, 23017, 23021, 23025, 23029, 23033, 23037, 23041, 23045, - 23049, 23052, 23056, 23060, 23064, 23068, 23072, 23076, 23080, 23084, - 23088, 23092, 23096, 23100, 23104, 23108, 23112, 23115, 23119, 23123, - 23127, 23131, 23135, 23139, 23143, 23146, 23150, 23154, 23158, 23162, - 23166, 23170, 23174, 23178, 23182, 23186, 23190, 23194, 23198, 23202, - 23206, 23210, 23214, 23218, 23222, 23226, 23230, 23234, 23238, 0, 23242, - 23246, 23250, 23254, 0, 0, 23258, 23262, 23266, 23270, 23274, 23278, - 23282, 0, 23286, 0, 23290, 23294, 23298, 23302, 0, 0, 23306, 23310, - 23314, 23318, 23322, 23326, 23330, 23334, 23338, 23342, 23346, 23350, - 23354, 23358, 23362, 23366, 23370, 23374, 23378, 23382, 23386, 23390, - 23394, 23397, 23401, 23405, 23409, 23413, 23417, 23421, 23425, 23429, - 23433, 23437, 23441, 23445, 23449, 23453, 23457, 23461, 23465, 0, 23469, - 23473, 23477, 23481, 0, 0, 23485, 23488, 23492, 23496, 23500, 23504, - 23508, 23512, 23516, 23520, 23524, 23528, 23532, 23536, 23540, 23544, - 23548, 23553, 23558, 23563, 23569, 23575, 23580, 23585, 23591, 23594, - 23598, 23602, 23606, 23610, 23614, 23618, 23622, 0, 23626, 23630, 23634, - 23638, 0, 0, 23642, 23646, 23650, 23654, 23658, 23662, 23666, 0, 23670, - 0, 23674, 23678, 23682, 23686, 0, 0, 23690, 23694, 23698, 23702, 23706, - 23710, 23714, 23718, 23722, 23727, 23732, 23737, 23743, 23749, 23754, 0, - 23759, 23763, 23767, 23771, 23775, 23779, 23783, 23787, 23791, 23795, - 23799, 23803, 23807, 23811, 23815, 23819, 23823, 23826, 23830, 23834, - 23838, 23842, 23846, 23850, 23854, 23858, 23862, 23866, 23870, 23874, - 23878, 23882, 23886, 23890, 23894, 23898, 23902, 23906, 23910, 23914, - 23918, 23922, 23926, 23930, 23934, 23938, 23942, 23946, 23950, 23954, - 23958, 23962, 23966, 23970, 23974, 23978, 23982, 0, 23986, 23990, 23994, - 23998, 0, 0, 24002, 24006, 24010, 24014, 24018, 24022, 24026, 24030, - 24034, 24038, 24042, 24046, 24050, 24054, 24058, 24062, 24066, 24070, - 24074, 24078, 24082, 24086, 24090, 24094, 24098, 24102, 24106, 24110, - 24114, 24118, 24122, 24126, 24130, 24134, 24138, 24142, 24146, 24150, - 24154, 24158, 24162, 24166, 24170, 24174, 24178, 24182, 24186, 24190, - 24194, 24198, 24202, 24206, 24210, 24214, 24218, 24222, 24226, 24229, - 24233, 24237, 24241, 24245, 24249, 24253, 24257, 24261, 24265, 0, 0, - 24269, 24278, 24284, 24289, 24293, 24296, 24301, 24304, 24307, 24310, - 24315, 24319, 24324, 24327, 24330, 24333, 24336, 24339, 24342, 24345, - 24348, 24351, 24355, 24359, 24363, 24367, 24371, 24375, 24379, 24383, - 24387, 24391, 0, 0, 0, 24396, 24402, 24406, 24410, 24414, 24420, 24424, - 24428, 24432, 24438, 24442, 24446, 24450, 24456, 24460, 24464, 24468, - 24474, 24480, 24486, 24494, 24500, 24506, 24512, 24518, 24524, 0, 0, 0, - 0, 0, 0, 24530, 24533, 24536, 24539, 24542, 24545, 24549, 24553, 24556, - 24560, 24564, 24568, 24572, 24576, 24579, 24583, 24587, 24591, 24595, - 24599, 24602, 24606, 24610, 24614, 24618, 24622, 24625, 24629, 24633, - 24637, 24641, 24644, 24648, 24652, 24656, 24660, 24664, 24668, 24672, - 24676, 24680, 24684, 24688, 24692, 24696, 24699, 24703, 24707, 24711, - 24715, 24719, 24723, 24727, 24731, 24735, 24739, 24743, 24747, 24751, - 24755, 24759, 24763, 24767, 24771, 24775, 24779, 24783, 24787, 24791, - 24795, 24799, 24803, 24807, 24811, 24815, 24819, 24823, 24827, 24831, - 24835, 24838, 24842, 24846, 24850, 24854, 24858, 0, 0, 24862, 24867, - 24872, 24877, 24882, 24887, 0, 0, 24892, 24896, 24899, 24903, 24906, - 24910, 24913, 24917, 24923, 24928, 24932, 24935, 24939, 24943, 24949, - 24953, 24959, 24963, 24969, 24973, 24979, 24983, 24989, 24995, 24999, - 25005, 25009, 25015, 25021, 25025, 25031, 25037, 25042, 25047, 25055, - 25063, 25070, 25075, 25081, 25090, 25096, 25104, 25109, 25115, 25119, - 25123, 25127, 25131, 25135, 25139, 25143, 25147, 25151, 25155, 25161, - 25166, 25171, 25174, 25178, 25182, 25188, 25192, 25198, 25202, 25208, - 25212, 25218, 25222, 25228, 25232, 25238, 25242, 25248, 25254, 25258, - 25264, 25269, 25273, 25277, 25281, 25285, 25288, 25292, 25298, 25303, - 25308, 25312, 25316, 25320, 25326, 25330, 25336, 25340, 25346, 25349, - 25354, 25358, 25364, 25368, 25374, 25378, 25384, 25390, 25394, 25398, - 25402, 25406, 25410, 25414, 25418, 25422, 25426, 25430, 25434, 25440, - 25443, 25447, 25451, 25457, 25461, 25467, 25471, 25477, 25481, 25487, - 25491, 25497, 25501, 25507, 25511, 25517, 25523, 25527, 25531, 25537, - 25543, 25549, 25555, 25559, 25563, 25567, 25571, 25575, 25579, 25585, - 25589, 25593, 25597, 25603, 25607, 25613, 25617, 25623, 25627, 25633, - 25637, 25643, 25647, 25653, 25657, 25663, 25669, 25673, 25679, 25683, - 25687, 25691, 25695, 25699, 25703, 25709, 25712, 25716, 25720, 25726, - 25730, 25736, 25740, 25746, 25750, 25756, 25760, 25766, 25770, 25776, - 25780, 25786, 25792, 25796, 25802, 25806, 25812, 25818, 25822, 25826, - 25830, 25834, 25838, 25842, 25848, 25851, 25855, 25859, 25865, 25869, - 25875, 25879, 25885, 25891, 25895, 25900, 25904, 25908, 25912, 25916, - 25920, 25924, 25928, 25934, 25937, 25941, 25945, 25951, 25955, 25961, - 25965, 25971, 25975, 25981, 25985, 25991, 25995, 26001, 26005, 26011, - 26014, 26019, 26024, 26028, 26032, 26036, 26040, 26044, 26048, 26054, - 26057, 26061, 26065, 26071, 26075, 26081, 26085, 26091, 26095, 26101, - 26105, 26111, 26115, 26121, 26125, 26131, 26137, 26141, 26147, 26151, - 26157, 26163, 26169, 26175, 26181, 26187, 26193, 26199, 26203, 26207, - 26211, 26215, 26219, 26223, 26227, 26231, 26237, 26241, 26247, 26251, - 26257, 26261, 26267, 26271, 26277, 26281, 26287, 26291, 26297, 26301, - 26305, 26309, 26313, 26317, 26321, 26325, 26331, 26334, 26338, 26342, - 26348, 26352, 26358, 26362, 26368, 26372, 26378, 26382, 26388, 26392, - 26398, 26402, 26408, 26414, 26418, 26424, 26430, 26436, 26440, 26446, - 26452, 26456, 26460, 26464, 26468, 26472, 26478, 26481, 26485, 26490, - 26494, 26500, 26503, 26508, 26513, 26517, 26521, 26525, 26529, 26533, - 26537, 26541, 26545, 26549, 26555, 26559, 26563, 26569, 26573, 26579, - 26583, 26589, 26593, 26597, 26601, 26605, 26609, 26615, 26619, 26623, - 26627, 26631, 26635, 26639, 26643, 26647, 26651, 26655, 26661, 26667, - 26673, 26679, 26685, 26690, 26696, 26702, 26708, 26712, 26716, 26720, - 26724, 26728, 26732, 26736, 26740, 26744, 26748, 26752, 26756, 26760, - 26766, 26772, 26778, 26783, 26787, 26791, 26795, 26799, 26803, 26807, - 26811, 26815, 26819, 26825, 26831, 26837, 26843, 26849, 26855, 26861, - 26867, 26873, 26877, 26881, 26885, 26889, 26893, 26897, 26901, 26907, - 26913, 26919, 26925, 26931, 26937, 26943, 26949, 26955, 26960, 26965, - 26970, 26975, 26981, 26987, 26993, 26999, 27005, 27011, 27017, 27022, - 27028, 27034, 27040, 27045, 27051, 27057, 27063, 27068, 27073, 27078, - 27083, 27088, 27093, 27098, 27103, 27108, 27113, 27118, 27123, 27127, - 27132, 27137, 27142, 27147, 27152, 27157, 27162, 27167, 27172, 27177, - 27182, 27187, 27192, 27197, 27202, 27207, 27212, 27217, 27222, 27227, - 27232, 27237, 27242, 27247, 27252, 27257, 27262, 27267, 27272, 27276, - 27281, 27286, 27291, 27296, 27301, 27306, 27311, 27316, 27321, 27326, - 27331, 27336, 27341, 27346, 27351, 27356, 27361, 27366, 27371, 27376, - 27381, 27386, 27391, 27396, 27401, 27405, 27410, 27415, 27420, 27425, - 27430, 27434, 27439, 27444, 27449, 27454, 27459, 27463, 27468, 27474, - 27479, 27484, 27489, 27494, 27500, 27505, 27510, 27515, 27520, 27525, - 27530, 27535, 27540, 27545, 27550, 27555, 27560, 27564, 27569, 27574, - 27579, 27584, 27589, 27594, 27599, 27604, 27609, 27614, 27619, 27624, - 27629, 27634, 27639, 27644, 27649, 27654, 27659, 27664, 27669, 27674, - 27679, 27684, 27689, 27694, 27699, 27704, 27709, 27714, 27719, 27725, - 27730, 27735, 27740, 27745, 27750, 27755, 27760, 27765, 27770, 27775, - 27780, 27784, 27789, 27794, 27799, 27804, 27809, 27814, 27819, 27824, - 27829, 27834, 27839, 27844, 27849, 27854, 27859, 27864, 27869, 27874, - 27879, 27884, 27889, 27894, 27899, 27904, 27909, 27914, 27920, 27924, - 27928, 27932, 27936, 27940, 27944, 27948, 27952, 27958, 27964, 27970, - 27976, 27982, 27988, 27994, 28001, 28007, 28012, 28017, 28022, 28027, - 28032, 28037, 28042, 28047, 28052, 28057, 28062, 28067, 28072, 28077, - 28082, 28087, 28092, 28097, 28102, 28107, 28112, 28117, 28122, 28127, - 28132, 28137, 28142, 28147, 0, 0, 0, 28154, 28165, 28170, 28178, 28183, - 28188, 28193, 28202, 28207, 28213, 28219, 28225, 28230, 28236, 28242, - 28246, 28251, 28256, 28266, 28271, 28276, 28283, 28288, 28293, 28302, - 28307, 28316, 28323, 28330, 28337, 28344, 28355, 28362, 28367, 28377, - 28381, 28388, 28393, 28400, 28406, 28413, 28422, 28429, 28436, 28445, - 28452, 28457, 28462, 28473, 28480, 28485, 28496, 28503, 28508, 28513, - 28521, 28530, 28537, 28544, 28554, 28559, 28564, 28569, 28578, 28586, - 28591, 28596, 28601, 28606, 28611, 28616, 28621, 28626, 28631, 28636, - 28641, 28647, 28653, 28659, 28664, 28669, 28674, 28679, 28684, 28689, - 28698, 28707, 28716, 28725, 0, 0, 0, 0, 0, 0, 0, 28734, 28738, 28742, - 28746, 28750, 28755, 28760, 28765, 28770, 28774, 28778, 28783, 28787, - 28791, 28795, 28799, 28804, 28808, 28812, 28817, 28822, 28827, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 28832, 28838, 28842, 28846, 28850, 28854, 28859, 28864, - 28869, 28874, 28878, 28882, 28887, 28891, 28895, 28899, 28903, 28908, - 28912, 28916, 28921, 28926, 28931, 28937, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28942, 28946, 28950, 28954, 28958, 28963, 28968, 28973, 28978, 28982, - 28986, 28991, 28995, 28999, 29003, 29007, 29012, 29016, 29020, 29025, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29030, 29034, 29038, 29042, 29046, - 29051, 29056, 29061, 29066, 29070, 29074, 29079, 29083, 0, 29087, 29091, - 29096, 0, 29100, 29105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29110, 29113, - 29117, 29121, 29125, 29129, 29133, 29137, 29141, 29145, 29149, 29153, - 29157, 29161, 29165, 29169, 29173, 29177, 29180, 29184, 29188, 29192, - 29196, 29200, 29204, 29208, 29212, 29216, 29220, 29224, 29228, 29232, - 29235, 29238, 29241, 29245, 29251, 29257, 29263, 29269, 29275, 29281, - 29287, 29293, 29299, 29305, 29311, 29317, 29323, 29329, 29338, 29347, - 29353, 29359, 29365, 29370, 29374, 29379, 29384, 29389, 29393, 29398, - 29403, 29408, 29412, 29417, 29421, 29426, 29431, 29436, 29441, 29445, - 29449, 29453, 29457, 29461, 29465, 29469, 29473, 29477, 29481, 29487, - 29491, 29495, 29499, 29503, 29507, 29515, 29521, 29525, 29531, 29535, - 29541, 29545, 0, 0, 29549, 29553, 29556, 29559, 29562, 29565, 29568, - 29571, 29574, 29577, 0, 0, 0, 0, 0, 0, 29580, 29588, 29596, 29604, 29612, - 29620, 29628, 29636, 29644, 29652, 0, 0, 0, 0, 0, 0, 29660, 29663, 29666, - 29669, 29674, 29677, 29682, 29689, 29697, 29702, 29709, 29712, 29719, - 29726, 29733, 29737, 29744, 29748, 29751, 29754, 29757, 29760, 29763, - 29766, 29769, 29772, 0, 0, 0, 0, 0, 0, 29775, 29778, 29781, 29784, 29787, - 29790, 29794, 29798, 29802, 29805, 29809, 29813, 29816, 29820, 29824, - 29827, 29830, 29833, 29837, 29841, 29845, 29849, 29853, 29856, 29859, - 29863, 29867, 29870, 29874, 29878, 29882, 29886, 29890, 29894, 29898, - 29902, 29909, 29914, 29919, 29924, 29929, 29935, 29941, 29947, 29953, - 29958, 29964, 29970, 29975, 29981, 29987, 29993, 29999, 30005, 30010, - 30016, 30021, 30027, 30033, 30039, 30045, 30051, 30056, 30061, 30067, - 30073, 30078, 30084, 30089, 30095, 30100, 30105, 30111, 30117, 30123, - 30129, 30135, 30141, 30147, 30153, 30159, 30165, 30171, 30177, 30182, - 30187, 30192, 30198, 30204, 0, 0, 0, 0, 0, 0, 0, 30212, 30221, 30230, - 30238, 30246, 30256, 30264, 30273, 30280, 30287, 30294, 30302, 30310, - 30318, 30326, 30334, 30342, 30350, 30358, 30365, 30373, 30381, 30389, - 30397, 30405, 30415, 30425, 30435, 30445, 30455, 30465, 30475, 30485, - 30495, 30505, 30515, 30525, 30535, 30545, 30553, 30561, 30571, 30579, 0, - 0, 0, 0, 0, 30589, 30593, 30597, 30601, 30605, 30609, 30613, 30617, - 30621, 30625, 30629, 30633, 30637, 30641, 30645, 30649, 30653, 30657, - 30661, 30665, 30669, 30673, 30677, 30681, 30687, 30691, 30697, 30701, - 30707, 30711, 30717, 30721, 30725, 30729, 30733, 30737, 30741, 30747, - 30753, 30759, 30765, 30771, 30777, 30783, 30789, 30795, 30801, 30807, - 30814, 30820, 30826, 30832, 30836, 30840, 30844, 30848, 30852, 30856, - 30860, 30866, 30872, 30878, 30883, 30890, 30895, 30900, 30906, 30911, - 30918, 30925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30932, 30938, 30942, 30947, - 30952, 30957, 30962, 30967, 30972, 30977, 30982, 30987, 30992, 30997, - 31002, 31007, 31011, 31015, 31020, 31025, 31030, 31034, 31038, 31042, - 31046, 31051, 31056, 31061, 31065, 31069, 31074, 0, 31079, 31084, 31089, - 31094, 31100, 31106, 31112, 31118, 31123, 31128, 31134, 31140, 0, 0, 0, - 0, 31147, 31152, 31158, 31164, 31170, 31175, 31180, 31185, 31190, 31195, - 31200, 31205, 0, 0, 0, 0, 31210, 0, 0, 0, 31215, 31220, 31225, 31230, - 31234, 31238, 31242, 31246, 31250, 31254, 31258, 31262, 31266, 31271, - 31277, 31283, 31289, 31294, 31299, 31305, 31311, 31316, 31321, 31327, - 31332, 31338, 31344, 31349, 31355, 31361, 31367, 31372, 31377, 31382, - 31388, 31394, 31399, 31405, 31410, 31416, 31421, 31427, 0, 0, 31433, - 31439, 31445, 31451, 31457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31463, - 31472, 31481, 31489, 31498, 31507, 31515, 31524, 31533, 31542, 31550, - 31558, 31567, 31575, 31583, 31592, 31601, 31609, 31618, 31627, 31635, - 31643, 31652, 31660, 31668, 31677, 31685, 31694, 31703, 31711, 31720, - 31729, 31737, 31745, 31754, 31763, 31771, 31780, 31789, 31798, 31807, - 31816, 31825, 31834, 0, 0, 0, 0, 31843, 31853, 31862, 31871, 31879, - 31888, 31896, 31905, 31913, 31922, 31931, 31940, 31949, 31958, 31967, - 31976, 31985, 31994, 32003, 32012, 32021, 32030, 32039, 32048, 32057, - 32065, 0, 0, 0, 0, 0, 0, 32073, 32081, 32088, 32095, 32102, 32109, 32116, - 32123, 32130, 32137, 32144, 0, 0, 0, 32152, 32160, 32168, 32172, 32178, - 32184, 32190, 32196, 32202, 32208, 32214, 32220, 32226, 32232, 32238, - 32244, 32250, 32256, 32262, 32266, 32272, 32278, 32284, 32290, 32296, - 32302, 32308, 32314, 32320, 32326, 32332, 32338, 32344, 32350, 32356, - 32360, 32365, 32370, 32375, 32379, 32384, 32388, 32393, 32398, 32403, - 32407, 32412, 32417, 32422, 32427, 32432, 32436, 32440, 32444, 32449, - 32453, 32457, 32461, 32466, 32471, 32476, 32481, 0, 0, 32487, 32491, - 32498, 32503, 32509, 32515, 32520, 32526, 32532, 32537, 32543, 32549, - 32555, 32560, 32566, 32571, 32576, 32582, 32587, 32593, 32598, 32604, - 32610, 32616, 32622, 32626, 32631, 32636, 32642, 32648, 32653, 32659, - 32665, 32669, 32674, 32679, 32683, 32688, 32692, 32697, 32702, 32708, - 32714, 32719, 32724, 32729, 32733, 32738, 32742, 32747, 32751, 32756, - 32761, 32766, 32771, 32777, 32784, 32791, 32801, 32810, 32817, 32823, - 32834, 32839, 32845, 0, 32850, 32855, 32860, 32868, 32874, 32882, 32887, - 32893, 32899, 32905, 32910, 32916, 32921, 32928, 32934, 32939, 32945, - 32951, 32957, 32964, 32971, 32978, 32983, 32988, 32995, 33002, 33009, - 33016, 33023, 0, 0, 33030, 33037, 33044, 33050, 33056, 33062, 33068, - 33074, 33080, 33086, 33092, 0, 0, 0, 0, 0, 0, 33098, 33104, 33109, 33114, - 33119, 33124, 33129, 33134, 33139, 33144, 0, 0, 0, 0, 0, 0, 33149, 33154, - 33159, 33164, 33169, 33174, 33179, 33188, 33195, 33200, 33205, 33210, - 33215, 33220, 0, 0, 33225, 33232, 33235, 33238, 33242, 33247, 33251, - 33257, 33262, 33268, 33275, 33283, 33287, 33292, 33296, 33301, 33308, - 33316, 33323, 33329, 33337, 33344, 33349, 33353, 33360, 33364, 33369, - 33374, 33381, 33389, 33396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33404, 33410, 33416, 33420, 33424, 33428, - 33432, 33438, 33442, 33448, 33452, 33458, 33464, 33472, 33478, 33486, - 33490, 33494, 33498, 33504, 33507, 33513, 33517, 33523, 33527, 33531, - 33537, 33541, 33547, 33551, 33557, 33565, 33573, 33581, 33587, 33591, - 33597, 33601, 33607, 33610, 33613, 33619, 33623, 33629, 33632, 33635, - 33638, 33641, 33645, 33651, 33657, 33660, 33663, 33667, 33672, 33677, - 33684, 33689, 33696, 33703, 33712, 33719, 33728, 33733, 33740, 33747, - 33756, 33761, 33768, 33773, 33779, 33785, 33791, 33797, 33803, 33809, - 33815, 0, 0, 0, 33821, 33825, 33828, 33831, 33834, 33837, 33840, 33843, - 33846, 33849, 33852, 33855, 33858, 33861, 33866, 33871, 33876, 33879, - 33884, 33889, 33894, 33899, 33906, 33911, 33916, 33921, 33926, 33933, - 33939, 33945, 33951, 33957, 33963, 33972, 33981, 33987, 33993, 34002, - 34011, 34020, 34029, 34038, 34047, 34056, 34065, 34074, 34079, 0, 34084, - 34089, 34094, 34099, 34103, 34107, 34111, 34116, 34120, 34124, 34129, - 34133, 34138, 34143, 34148, 34153, 34158, 34163, 34168, 34173, 34178, - 34182, 34186, 34191, 34196, 34201, 34205, 34209, 34213, 34217, 34222, - 34226, 34231, 34235, 34241, 34247, 34253, 34259, 34265, 34271, 34277, - 34283, 34289, 34294, 34299, 34306, 34314, 34319, 34324, 34329, 34333, - 34337, 34341, 34345, 34349, 34353, 34357, 34361, 34365, 34369, 34374, - 34379, 34384, 34390, 34396, 34400, 34406, 34410, 34416, 34422, 34427, - 34434, 34438, 34444, 34448, 34454, 34459, 34466, 34473, 34478, 34485, - 34490, 34495, 34499, 34505, 34509, 34515, 34522, 34529, 34533, 34539, - 34545, 34549, 34555, 34560, 34564, 34570, 34575, 34580, 34585, 34590, - 34594, 34598, 34603, 34608, 34615, 34621, 34626, 34633, 34638, 34645, - 34650, 34659, 34665, 34671, 34675, 0, 0, 0, 0, 0, 0, 0, 0, 34679, 34688, - 34695, 34702, 34709, 34713, 34718, 34723, 34728, 34733, 34738, 34743, - 34748, 34753, 34758, 34763, 34768, 34773, 34777, 34781, 34786, 34791, - 34796, 34801, 34806, 34811, 34815, 34820, 34825, 34830, 34835, 34839, - 34843, 34847, 34851, 34856, 34861, 34865, 34870, 34875, 34879, 34885, - 34891, 34897, 34902, 34907, 34913, 34918, 34924, 34929, 34935, 34941, - 34946, 34952, 34958, 34963, 34969, 34975, 34981, 34986, 0, 0, 0, 34991, - 34997, 35007, 35013, 35021, 35027, 35032, 35036, 35040, 35044, 35048, - 35052, 35056, 35060, 35064, 0, 0, 0, 35068, 35073, 35078, 35083, 35090, - 35096, 35102, 35108, 35114, 35120, 35126, 35132, 35138, 35144, 35150, - 35157, 35164, 35171, 35178, 35185, 35192, 35199, 35206, 35213, 35220, - 35227, 35234, 35241, 35248, 35255, 35262, 35269, 35276, 35283, 35290, - 35297, 35304, 35311, 35318, 35325, 35332, 35339, 35346, 35353, 35361, - 35369, 35377, 35383, 35389, 35395, 35403, 35412, 35419, 35426, 35432, - 35439, 35446, 35453, 35461, 35468, 0, 0, 0, 0, 0, 0, 0, 35475, 35482, - 35489, 35496, 35503, 35510, 35517, 35524, 35531, 35538, 35545, 35552, - 35559, 35566, 35573, 35580, 35587, 35594, 35601, 35608, 35615, 35622, - 35629, 35636, 35643, 35650, 35657, 35664, 35671, 35678, 35685, 35692, - 35699, 35706, 35713, 35720, 35727, 35734, 35741, 35748, 35755, 35762, - 35770, 0, 0, 35777, 35784, 35792, 35800, 35808, 35816, 35824, 35832, - 35842, 35852, 35862, 0, 0, 0, 0, 0, 0, 0, 0, 35872, 35877, 35882, 35887, - 35892, 35901, 35912, 35921, 35932, 35938, 35951, 35957, 35964, 35971, - 35976, 35982, 35988, 35999, 36008, 36015, 36022, 36031, 36038, 36047, - 36057, 36067, 36074, 36081, 36088, 36098, 36103, 36111, 36117, 36125, - 36134, 36139, 36146, 36152, 36157, 36162, 36167, 36173, 36180, 0, 0, 0, - 0, 0, 36188, 36193, 36199, 36205, 36213, 36219, 36225, 36231, 36236, - 36243, 36248, 36254, 36260, 36268, 36274, 36282, 36287, 36294, 36300, - 36308, 36316, 36322, 36328, 36335, 36342, 36348, 36355, 36361, 36367, - 36372, 36378, 36386, 36394, 36400, 36406, 36412, 36418, 36426, 36430, - 36436, 36442, 36448, 36454, 36460, 36466, 36470, 36475, 36480, 36487, - 36492, 36496, 36502, 36507, 36512, 36516, 36521, 36526, 36530, 36535, - 36540, 36547, 36551, 36556, 36561, 36565, 36570, 36574, 36579, 36583, - 36588, 36593, 36599, 36604, 36609, 36613, 36618, 36624, 36631, 36636, - 36641, 36646, 36651, 36656, 36660, 36666, 36673, 36680, 36685, 36690, - 36694, 36700, 36706, 36711, 36716, 36721, 36727, 36732, 36738, 36743, - 36749, 36755, 36761, 36768, 36775, 36782, 36789, 36796, 36803, 36808, - 36816, 36825, 36834, 36843, 36852, 36861, 36870, 36882, 36891, 36900, - 36909, 36915, 36920, 36927, 36935, 36943, 36950, 36957, 36964, 36971, - 36979, 36988, 36997, 37006, 37015, 37024, 37033, 37042, 37051, 37060, - 37069, 37078, 37087, 37096, 37105, 37113, 37122, 37133, 37142, 37153, - 37166, 37175, 37184, 37194, 37203, 37211, 37220, 37226, 37231, 37239, - 37244, 37252, 37257, 37266, 37272, 37278, 37285, 37290, 37295, 37303, - 37311, 37320, 37329, 37334, 37341, 37351, 37359, 37368, 37374, 37380, - 37385, 37392, 37397, 37406, 37411, 37416, 37421, 37428, 37434, 37439, - 37448, 37456, 37461, 37466, 37473, 37480, 37484, 37488, 37491, 37494, - 37497, 37500, 37503, 37506, 37513, 37516, 37519, 37524, 37528, 37532, - 37536, 37540, 37544, 37554, 37560, 37566, 37572, 37580, 37588, 37594, - 37600, 37607, 37613, 37618, 37624, 37631, 37637, 37644, 37650, 37658, - 37664, 37671, 37677, 37683, 37689, 37695, 37701, 37707, 37718, 37728, - 37734, 37740, 37750, 37756, 37764, 37772, 37780, 37785, 37790, 37796, - 37801, 37809, 37815, 37819, 37826, 37833, 37838, 37847, 37855, 37863, - 37870, 37877, 37884, 37891, 37899, 37907, 37918, 37929, 37937, 37945, - 37953, 37961, 37970, 37979, 37987, 37995, 38004, 38013, 38024, 38035, - 38046, 38057, 38066, 38075, 38084, 38093, 38104, 38115, 38123, 38131, - 38139, 38147, 38155, 38163, 38171, 38179, 38187, 38195, 38203, 38211, - 38220, 38229, 38238, 38247, 38258, 38269, 38277, 38285, 38293, 38301, - 38310, 38319, 38327, 38335, 38347, 38359, 38368, 38377, 38386, 38395, - 38403, 38411, 38419, 38427, 38435, 38443, 38451, 38459, 38467, 38475, - 38484, 38493, 38502, 38511, 38521, 38531, 38541, 38551, 38561, 38571, - 38581, 38591, 38599, 38607, 38615, 38623, 38631, 38639, 38647, 38655, - 38667, 38679, 38688, 38697, 38705, 38713, 38721, 38729, 38740, 38751, - 38762, 38773, 38785, 38797, 38805, 38813, 38821, 38829, 38838, 38847, - 38856, 38865, 38873, 38881, 38889, 38897, 38905, 38913, 38923, 38933, - 38943, 38953, 38961, 38969, 38977, 38985, 38993, 39001, 39009, 39017, - 39025, 39033, 39041, 39049, 39057, 39065, 39073, 39081, 39089, 39097, - 39105, 39113, 39121, 39129, 39137, 39145, 39154, 39163, 39172, 39180, - 39189, 39198, 39207, 39216, 39226, 39235, 39242, 39247, 39254, 39261, - 39269, 39277, 39287, 39297, 39307, 39317, 39328, 39339, 39349, 39359, - 39369, 39379, 39389, 39399, 39409, 39419, 39430, 39441, 39451, 39461, - 39471, 39481, 39489, 39497, 39506, 39515, 39523, 39531, 39542, 39553, - 39564, 39575, 39587, 39599, 39610, 39621, 39632, 39643, 39652, 39661, - 39669, 39677, 39684, 39691, 39699, 39707, 39717, 39727, 39737, 39747, - 39758, 39769, 39779, 39789, 39799, 39809, 39819, 39829, 39839, 39849, - 39860, 39871, 39881, 39891, 39901, 39911, 39918, 39925, 39933, 39941, - 39951, 39961, 39971, 39981, 39992, 40003, 40013, 40023, 40033, 40043, - 40051, 40059, 40067, 40075, 40084, 40093, 40101, 40109, 40116, 40123, - 40130, 40137, 40145, 40153, 40161, 40169, 40180, 40191, 40202, 40213, - 40224, 40235, 40243, 40251, 40262, 40273, 40284, 40295, 40306, 40317, - 40325, 40333, 40344, 40355, 40366, 0, 0, 40377, 40385, 40393, 40404, - 40415, 40426, 0, 0, 40437, 40445, 40453, 40464, 40475, 40486, 40497, - 40508, 40519, 40527, 40535, 40546, 40557, 40568, 40579, 40590, 40601, - 40609, 40617, 40628, 40639, 40650, 40661, 40672, 40683, 40691, 40699, - 40710, 40721, 40732, 40743, 40754, 40765, 40773, 40781, 40792, 40803, - 40814, 0, 0, 40825, 40833, 40841, 40852, 40863, 40874, 0, 0, 40885, - 40893, 40901, 40912, 40923, 40934, 40945, 40956, 0, 40967, 0, 40975, 0, - 40986, 0, 40997, 41008, 41016, 41024, 41035, 41046, 41057, 41068, 41079, - 41090, 41098, 41106, 41117, 41128, 41139, 41150, 41161, 41172, 41180, - 41188, 41196, 41204, 41212, 41220, 41228, 41236, 41244, 41252, 41260, - 41268, 41276, 0, 0, 41284, 41295, 41306, 41320, 41334, 41348, 41362, - 41376, 41390, 41401, 41412, 41426, 41440, 41454, 41468, 41482, 41496, - 41507, 41518, 41532, 41546, 41560, 41574, 41588, 41602, 41613, 41624, - 41638, 41652, 41666, 41680, 41694, 41708, 41719, 41730, 41744, 41758, - 41772, 41786, 41800, 41814, 41825, 41836, 41850, 41864, 41878, 41892, - 41906, 41920, 41928, 41936, 41947, 41955, 0, 41966, 41974, 41985, 41993, - 42001, 42009, 42017, 42025, 42028, 42031, 42034, 42037, 42043, 42054, - 42062, 0, 42073, 42081, 42092, 42100, 42108, 42116, 42124, 42132, 42138, - 42144, 42150, 42158, 42166, 42177, 0, 0, 42188, 42196, 42207, 42215, - 42223, 42231, 0, 42239, 42245, 42251, 42257, 42265, 42273, 42284, 42295, - 42303, 42311, 42319, 42330, 42338, 42346, 42354, 42362, 42370, 42376, - 42382, 0, 0, 42385, 42396, 42404, 0, 42415, 42423, 42434, 42442, 42450, - 42458, 42466, 42474, 42477, 0, 42480, 42484, 42488, 42492, 42496, 42500, - 42504, 42508, 42512, 42516, 42520, 42524, 42530, 42536, 42542, 42545, - 42548, 42550, 42554, 42558, 42562, 42566, 42569, 42573, 42577, 42583, - 42589, 42596, 42603, 42608, 42613, 42619, 42625, 42627, 42630, 42632, - 42636, 42640, 42644, 42648, 42652, 42656, 42660, 42664, 42668, 42674, - 42678, 42682, 42688, 42693, 42700, 42702, 42705, 42709, 42713, 42718, - 42724, 42726, 42735, 42744, 42747, 42751, 42753, 42755, 42757, 42761, - 42767, 42769, 42773, 42777, 42784, 42791, 42795, 42800, 42805, 42810, - 42815, 42819, 42823, 42826, 42830, 42834, 42841, 42846, 42850, 42854, - 42859, 42863, 42867, 42872, 42877, 42881, 42885, 42889, 42891, 42896, - 42901, 42905, 42909, 42913, 42917, 0, 42921, 42925, 42929, 42935, 42941, - 42947, 42953, 42960, 42967, 42972, 42977, 42981, 0, 0, 42987, 42990, - 42993, 42996, 42999, 43002, 43005, 43009, 43013, 43018, 43023, 43028, - 43035, 43039, 43042, 43045, 43048, 43051, 43054, 43057, 43060, 43063, - 43066, 43070, 43074, 43079, 43084, 0, 43089, 43095, 43101, 43107, 43114, - 43121, 43128, 43135, 43141, 43148, 43155, 43162, 43169, 0, 0, 0, 43176, - 43179, 43182, 43185, 43190, 43193, 43196, 43199, 43202, 43205, 43208, - 43213, 43216, 43219, 43222, 43225, 43228, 43233, 43236, 43239, 43242, - 43245, 43248, 43253, 43256, 43259, 43264, 43269, 43273, 43276, 43279, - 43282, 43285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43288, 43293, - 43298, 43305, 43313, 43318, 43323, 43327, 43331, 43336, 43343, 43350, - 43354, 43359, 43364, 43369, 43374, 43381, 43386, 43391, 43396, 43405, - 43412, 43419, 43423, 43428, 43434, 43439, 43446, 43454, 43462, 43466, - 43470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43474, 43478, 43485, - 43490, 43494, 43499, 43503, 43507, 43511, 43513, 43517, 43521, 43525, - 43530, 43535, 43539, 43547, 43550, 43554, 43557, 43560, 43566, 43571, - 43574, 43580, 43584, 43589, 43594, 43597, 43601, 43605, 43609, 43611, - 43614, 43617, 43621, 43623, 43628, 43631, 43634, 43639, 43644, 43650, - 43653, 43656, 43660, 43665, 43668, 43671, 43674, 43678, 43682, 43686, - 43689, 43691, 43694, 43697, 43700, 43704, 43709, 43712, 43717, 43722, - 43727, 43732, 43738, 43743, 43747, 43752, 43757, 43763, 43769, 43774, - 43779, 43785, 43789, 43792, 43795, 43797, 43801, 43807, 43814, 43821, - 43828, 43835, 43842, 43849, 43856, 43863, 43871, 43878, 43886, 43893, - 43900, 43908, 43916, 43921, 43926, 43931, 43936, 43941, 43946, 43951, - 43956, 43961, 43966, 43972, 43978, 43984, 43990, 43997, 44005, 44011, - 44017, 44023, 44029, 44035, 44041, 44047, 44053, 44059, 44065, 44072, - 44079, 44086, 44093, 44101, 44110, 44117, 44128, 44135, 44142, 44151, - 44158, 44167, 44176, 44183, 44191, 44199, 44202, 0, 0, 0, 0, 44205, - 44207, 44210, 44212, 44215, 44218, 44221, 44225, 44229, 44234, 44239, - 44243, 44247, 44251, 44255, 44260, 44266, 44271, 44277, 44282, 44287, - 44292, 44298, 44303, 44309, 44315, 44319, 44323, 44328, 44333, 44338, - 44343, 44348, 44356, 44364, 44372, 44380, 44387, 44395, 44402, 44409, - 44416, 44426, 44433, 44440, 44447, 44454, 44462, 44470, 44477, 44484, - 44492, 44500, 44505, 44513, 44518, 44523, 44529, 44534, 44540, 44547, - 44554, 44559, 44565, 44570, 44573, 44577, 44580, 44584, 44588, 44592, - 44597, 44602, 44608, 44614, 44618, 44622, 44626, 44630, 44636, 44642, - 44646, 44651, 44655, 44660, 44664, 44668, 44671, 44675, 44678, 44682, - 44689, 44697, 44709, 44720, 44725, 44734, 44741, 44748, 44756, 44760, - 44766, 44774, 44778, 44783, 44788, 44794, 44800, 44806, 44813, 44817, - 44821, 44826, 44829, 44831, 44835, 44839, 44847, 44851, 44853, 44855, - 44859, 44867, 44872, 44878, 44888, 44895, 44900, 44904, 44908, 44912, - 44915, 44918, 44921, 44925, 44929, 44933, 44937, 44941, 44944, 44948, - 44952, 44955, 44957, 44960, 44962, 44966, 44970, 44972, 44978, 44981, - 44986, 44990, 44994, 44996, 44998, 45000, 45003, 45007, 45011, 45015, - 45019, 45023, 45029, 45035, 45037, 45039, 45041, 45043, 45046, 45048, - 45052, 45054, 45058, 45062, 45068, 45072, 45076, 45080, 45084, 45089, - 45096, 45101, 45112, 45123, 45128, 45135, 45144, 45148, 45153, 45156, - 45161, 45165, 45171, 45176, 45189, 45199, 45203, 45207, 45214, 45219, - 45222, 45224, 45227, 45231, 45236, 45243, 45247, 45252, 45257, 45260, - 45265, 45270, 45277, 45284, 45290, 45296, 45305, 45314, 45318, 45322, - 45324, 45329, 45333, 45337, 45346, 45355, 45362, 45369, 45378, 45387, - 45393, 45399, 45407, 45415, 45417, 45419, 45426, 45433, 45440, 45447, - 45453, 45459, 45463, 45467, 45474, 45481, 45489, 45497, 45508, 45519, - 45528, 45537, 45539, 45543, 45547, 45552, 45557, 45566, 45575, 45578, - 45581, 45584, 45587, 45590, 45595, 45599, 45604, 45609, 45612, 45615, - 45618, 45621, 45624, 45628, 45631, 45634, 45637, 45640, 45642, 45644, - 45646, 45648, 45656, 45664, 45670, 45674, 45680, 45690, 45696, 45702, - 45708, 45716, 45726, 45739, 45743, 45747, 45749, 45755, 45757, 45759, - 45761, 45763, 45769, 45772, 45778, 45784, 45788, 45792, 45796, 45799, - 45803, 45807, 45809, 45818, 45827, 45832, 45837, 45843, 45849, 45855, - 45858, 45861, 45864, 45867, 45869, 45875, 45880, 45885, 45891, 45897, - 45906, 45915, 45922, 45929, 45936, 45943, 45953, 45963, 45974, 45985, - 45996, 46007, 46016, 46025, 46034, 46043, 46051, 46063, 46075, 46091, - 46094, 46100, 46106, 46112, 46120, 46135, 46151, 46157, 46163, 46170, - 46176, 46185, 46192, 46206, 46221, 46226, 46232, 46240, 46243, 46246, - 46248, 46251, 46254, 46256, 46258, 46262, 46265, 46268, 46271, 46274, - 46279, 46284, 46289, 46294, 46299, 46302, 46304, 46306, 46308, 46312, - 46316, 46320, 46326, 46330, 46332, 46334, 46339, 46344, 46349, 46354, - 46359, 46364, 46366, 46368, 46378, 46382, 46388, 46397, 46399, 46405, - 46411, 46418, 46422, 46424, 46428, 46430, 46434, 46438, 46442, 46444, - 46446, 46448, 46455, 46464, 46473, 46482, 46491, 46500, 46509, 46518, - 46527, 46535, 46543, 46552, 46561, 46570, 46579, 46587, 46595, 46604, - 46613, 46622, 46632, 46641, 46651, 46660, 46670, 46679, 46689, 46699, - 46708, 46718, 46727, 46737, 46746, 46756, 46765, 46774, 46783, 46792, - 46801, 46811, 46820, 46829, 46838, 46848, 46857, 46866, 46875, 46884, - 46894, 46904, 46913, 46922, 46930, 46939, 46946, 46955, 46964, 46975, - 46984, 46994, 47004, 47011, 47018, 47025, 47034, 47043, 47052, 47061, - 47068, 47073, 47082, 47088, 47091, 47098, 47101, 47106, 47111, 47114, - 47117, 47125, 47128, 47133, 47136, 47144, 47149, 47157, 47160, 47163, - 47166, 47171, 47176, 47179, 47182, 47190, 47193, 47200, 47207, 47211, - 47215, 47220, 47225, 47230, 47235, 47240, 47245, 47250, 47255, 47262, - 47268, 47275, 47282, 47288, 47295, 47302, 47310, 47317, 47323, 47330, - 47338, 47345, 47349, 47355, 47367, 47379, 47383, 47387, 47392, 47397, - 47408, 47412, 47417, 47422, 47428, 47434, 47440, 47446, 47455, 47464, - 47472, 47483, 47494, 47502, 47513, 47524, 47532, 47543, 47554, 47562, - 47570, 47580, 47590, 47593, 47596, 47599, 47604, 47608, 47614, 47621, - 47628, 47636, 47643, 47647, 47651, 47655, 47659, 47661, 47665, 47669, - 47675, 47681, 47689, 47697, 47700, 47707, 47709, 47711, 47715, 47719, - 47724, 47730, 47736, 47742, 47748, 47757, 47766, 47775, 47779, 47781, - 47785, 47792, 47799, 47806, 47813, 47820, 47823, 47828, 47834, 47837, - 47842, 47847, 47852, 47857, 47861, 47868, 47875, 47882, 47889, 47893, - 47897, 47901, 47905, 47911, 47917, 47922, 47928, 47934, 47940, 47946, - 47954, 47961, 47968, 47975, 47982, 47988, 47994, 48003, 48007, 48014, - 48018, 48022, 48028, 48034, 48040, 48046, 48050, 48054, 48057, 48061, - 48065, 48072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 48079, 48082, 48086, 48090, 48096, 48102, 48108, 48116, - 48123, 48127, 48135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48140, 48143, 48146, 48149, 48152, 48155, 48158, 48161, - 48164, 48167, 48171, 48175, 48179, 48183, 48187, 48191, 48195, 48199, - 48203, 48207, 48211, 48214, 48217, 48220, 48223, 48226, 48229, 48232, - 48235, 48238, 48242, 48246, 48250, 48254, 48258, 48262, 48266, 48270, - 48274, 48278, 48282, 48288, 48294, 48300, 48307, 48314, 48321, 48328, - 48335, 48342, 48349, 48356, 48363, 48370, 48377, 48384, 48391, 48398, - 48405, 48412, 48419, 48424, 48430, 48436, 48442, 48447, 48453, 48459, - 48465, 48470, 48476, 48482, 48487, 48493, 48499, 48504, 48510, 48516, - 48521, 48527, 48533, 48538, 48544, 48550, 48556, 48562, 48568, 48573, - 48579, 48585, 48591, 48596, 48602, 48608, 48614, 48619, 48625, 48631, - 48636, 48642, 48648, 48653, 48659, 48665, 48670, 48676, 48682, 48687, - 48693, 48699, 48705, 48711, 48717, 48722, 48728, 48734, 48740, 48745, - 48751, 48757, 48763, 48768, 48774, 48780, 48785, 48791, 48797, 48802, - 48808, 48814, 48819, 48825, 48831, 48836, 48842, 48848, 48854, 48860, - 48866, 48870, 48876, 48882, 48888, 48894, 48900, 48906, 48912, 48918, - 48924, 48930, 48934, 48938, 48942, 48946, 48950, 48954, 48958, 48962, - 48966, 48971, 48977, 48982, 48987, 48992, 48997, 49006, 49015, 49024, - 49033, 49042, 49051, 49060, 49069, 49075, 49083, 49091, 49097, 49104, - 49112, 49120, 49127, 49133, 49141, 49149, 49155, 49162, 49170, 49178, - 49185, 49191, 49199, 49208, 49217, 49225, 49234, 49243, 49249, 49256, - 49264, 49273, 49282, 49290, 49299, 49308, 49315, 49322, 49331, 49340, - 49349, 49358, 49367, 49376, 49383, 49390, 49399, 49408, 49417, 49426, - 49435, 49444, 49451, 49458, 49467, 49476, 49485, 49495, 49505, 49514, - 49524, 49534, 49544, 49554, 49564, 49574, 49583, 49592, 49599, 49607, - 49615, 49623, 49631, 49636, 49641, 49650, 49658, 49664, 49673, 49681, - 49688, 49697, 49705, 49711, 49720, 49728, 49735, 49744, 49752, 49758, - 49767, 49775, 49782, 49792, 49801, 49808, 49818, 49827, 49834, 49844, - 49853, 49860, 49868, 49877, 49886, 49894, 49905, 49915, 49922, 49927, - 49932, 49936, 49941, 49946, 49951, 49955, 49960, 49967, 49975, 49982, - 49990, 49994, 50000, 50006, 50012, 50016, 50023, 50029, 50036, 50040, - 50047, 50053, 50060, 50064, 50070, 50076, 50082, 50086, 50089, 50093, - 50097, 50103, 50109, 50114, 50118, 50123, 50133, 50140, 50151, 50161, - 50165, 50173, 50183, 50186, 50189, 50196, 50204, 50210, 50215, 50223, - 50232, 50241, 50249, 50253, 50257, 50260, 50263, 50267, 50271, 50274, - 50277, 50282, 50287, 50293, 50299, 50304, 50309, 50315, 50321, 50326, - 50331, 50336, 50341, 50347, 50353, 50358, 50363, 50369, 50375, 50380, - 50385, 50388, 50391, 50400, 50402, 50404, 50407, 50411, 50417, 50419, - 50422, 50429, 50436, 50443, 50450, 50459, 50472, 50477, 50482, 50486, - 50491, 50498, 50505, 50513, 50521, 50529, 50537, 50541, 50545, 50550, - 50555, 50560, 50565, 50568, 50574, 50580, 50589, 50598, 50606, 50614, - 50623, 50632, 50636, 50643, 50650, 50657, 50664, 50672, 50680, 50688, - 50696, 50700, 50704, 50708, 50713, 50718, 50724, 50730, 50734, 50740, - 50742, 50744, 50746, 50748, 50751, 50754, 50756, 50758, 50760, 50764, - 50768, 50770, 50772, 50775, 50778, 50782, 50788, 50794, 50796, 50803, - 50807, 50812, 50817, 50819, 50829, 50835, 50841, 50847, 50853, 50859, - 50865, 50870, 50873, 50876, 50879, 50881, 50883, 50887, 50891, 50896, - 50901, 50906, 50909, 50913, 50918, 50921, 50925, 50930, 50935, 50940, - 50945, 50950, 50955, 50960, 50965, 50970, 50975, 50980, 50985, 50991, - 50997, 51003, 51005, 51008, 51010, 51013, 51015, 51017, 51019, 51021, - 51023, 51025, 51027, 51029, 51031, 51033, 51035, 51037, 51039, 51041, - 51043, 51045, 51047, 51052, 51057, 51062, 51067, 51072, 51077, 51082, - 51087, 51092, 51097, 51102, 51107, 51112, 51117, 51122, 51127, 51132, - 51137, 51142, 51147, 51151, 51155, 51159, 51165, 51171, 51176, 51181, - 51186, 51192, 51198, 51203, 51211, 51219, 51227, 51235, 51243, 51251, - 51259, 51267, 51273, 51278, 51283, 51288, 51291, 51295, 51299, 51303, - 51307, 51311, 51315, 51321, 51328, 51335, 51343, 51348, 51353, 51360, - 51367, 51374, 51381, 51384, 51387, 51392, 51394, 51398, 51403, 51405, - 51407, 51409, 51411, 51416, 51419, 51421, 51426, 51432, 51439, 51442, - 51446, 51451, 51456, 51464, 51470, 51476, 51488, 51495, 51503, 51508, - 51513, 51519, 51522, 51525, 51530, 51532, 51536, 51538, 51540, 51542, - 51544, 51546, 51548, 51553, 51555, 51557, 51559, 51561, 51565, 51567, - 51570, 51575, 51580, 51585, 51590, 51596, 51602, 51604, 51607, 51614, - 51620, 51626, 51633, 51637, 51641, 51643, 51645, 51649, 51655, 51660, - 51662, 51666, 51675, 51683, 51691, 51697, 51703, 51708, 51714, 51719, - 51722, 51736, 51739, 51744, 51749, 51755, 51765, 51767, 51773, 51779, - 51783, 51790, 51794, 51796, 51798, 51802, 51808, 51813, 51819, 51821, - 51827, 51829, 51835, 51837, 51839, 51844, 51846, 51850, 51855, 51857, - 51862, 51867, 51871, 51878, 51888, 51893, 51898, 51901, 51906, 51909, - 51914, 51919, 51923, 51925, 51927, 51931, 51935, 51939, 51943, 51947, - 51949, 51953, 51956, 51959, 51962, 51966, 51970, 51975, 51979, 51984, - 51989, 51993, 51999, 52006, 52009, 52015, 52020, 52024, 52029, 52035, - 52041, 52048, 52054, 52061, 52068, 52070, 52077, 52081, 52088, 52094, - 52099, 52105, 52109, 52114, 52117, 52123, 52129, 52136, 52144, 52151, - 52160, 52170, 52177, 52183, 52187, 52195, 52200, 52209, 52212, 52215, - 52224, 52235, 52242, 52244, 52250, 52255, 52257, 52260, 52264, 52272, - 52281, 52284, 52289, 52295, 52302, 52309, 52316, 52323, 52329, 52335, - 52341, 52349, 52354, 52357, 52361, 52364, 52375, 52385, 52395, 52404, - 52415, 52425, 52434, 52440, 52448, 52452, 52460, 52464, 52472, 52479, - 52486, 52495, 52504, 52514, 52524, 52534, 52544, 52553, 52562, 52572, - 52582, 52591, 52600, 52607, 52614, 52621, 52628, 52635, 52642, 52649, - 52656, 52663, 52671, 52677, 52683, 52689, 52695, 52701, 52707, 52713, - 52719, 52725, 52732, 52740, 52748, 52756, 52764, 52772, 52780, 52788, - 52796, 52804, 52813, 52818, 52821, 52825, 52829, 52835, 52838, 52843, - 52849, 52854, 52858, 52863, 52869, 52876, 52879, 52886, 52893, 52897, - 52906, 52915, 52920, 52926, 52931, 52936, 52943, 52950, 52957, 52964, - 52972, 52976, 52984, 52989, 52993, 53000, 53004, 53010, 53018, 53023, - 53030, 53034, 53039, 53043, 53048, 53052, 53057, 53062, 53071, 53073, - 53077, 53081, 53088, 53095, 53101, 53109, 53115, 53122, 53127, 53130, - 53135, 53140, 53145, 53153, 53157, 53164, 53171, 53178, 53183, 53188, - 53194, 53199, 53204, 53210, 53215, 53218, 53222, 53226, 53233, 53243, - 53248, 53257, 53266, 53272, 53278, 53284, 53290, 53296, 53302, 53309, - 53316, 53325, 53334, 53340, 53346, 53351, 53356, 53363, 53370, 53376, - 53379, 53382, 53386, 53390, 53394, 53399, 53405, 53411, 53418, 53425, - 53430, 53434, 53438, 53442, 53446, 53450, 53454, 53458, 53462, 53466, - 53470, 53474, 53478, 53482, 53486, 53490, 53494, 53498, 53502, 53506, - 53510, 53514, 53518, 53522, 53526, 53530, 53534, 53538, 53542, 53546, - 53550, 53554, 53558, 53562, 53566, 53570, 53574, 53578, 53582, 53586, - 53590, 53594, 53598, 53602, 53606, 53610, 53614, 53618, 53622, 53626, - 53630, 53634, 53638, 53642, 53646, 53650, 53654, 53658, 53662, 53666, - 53670, 53674, 53678, 53682, 53686, 53690, 53694, 53698, 53702, 53706, - 53710, 53714, 53718, 53722, 53726, 53730, 53734, 53738, 53742, 53746, - 53750, 53754, 53758, 53762, 53766, 53770, 53774, 53778, 53782, 53786, - 53790, 53794, 53798, 53802, 53806, 53810, 53814, 53818, 53822, 53826, - 53830, 53834, 53838, 53842, 53846, 53850, 53854, 53858, 53862, 53866, - 53870, 53874, 53878, 53882, 53886, 53890, 53894, 53898, 53902, 53906, - 53910, 53914, 53918, 53922, 53926, 53930, 53934, 53938, 53942, 53946, - 53950, 53954, 53958, 53962, 53966, 53970, 53974, 53978, 53982, 53986, - 53990, 53994, 53998, 54002, 54006, 54010, 54014, 54018, 54022, 54026, - 54030, 54034, 54038, 54042, 54046, 54050, 54054, 54058, 54062, 54066, - 54070, 54074, 54078, 54082, 54086, 54090, 54094, 54098, 54102, 54106, - 54110, 54114, 54118, 54122, 54126, 54130, 54134, 54138, 54142, 54146, - 54150, 54154, 54158, 54162, 54166, 54170, 54174, 54178, 54182, 54186, - 54190, 54194, 54198, 54202, 54206, 54210, 54214, 54218, 54222, 54226, - 54230, 54234, 54238, 54242, 54246, 54250, 54254, 54258, 54262, 54266, - 54270, 54274, 54278, 54282, 54286, 54290, 54294, 54298, 54302, 54306, - 54310, 54314, 54318, 54322, 54326, 54330, 54334, 54338, 54342, 54346, - 54350, 54354, 54358, 54362, 54366, 54370, 54374, 54378, 54382, 54386, - 54390, 54394, 54398, 54402, 54406, 54410, 54414, 54418, 54422, 54426, - 54430, 54434, 54438, 54442, 54446, 54450, 54454, 54461, 54469, 54475, - 54481, 54488, 54495, 54501, 54507, 54514, 54521, 54526, 54531, 54536, - 54541, 54547, 54553, 54561, 54568, 54573, 54578, 54586, 54595, 54602, - 54612, 54623, 54626, 54629, 54633, 54637, 54643, 54649, 54659, 54669, - 54678, 54687, 54693, 54699, 54706, 54713, 54722, 54732, 54743, 54753, - 54763, 54773, 54784, 54795, 54805, 54816, 54826, 54836, 54844, 54854, - 54864, 54875, 54886, 54893, 54900, 54907, 54914, 54924, 54934, 54941, - 54948, 54955, 54962, 54969, 54976, 54983, 54988, 54993, 54999, 55007, - 55017, 55025, 55033, 55041, 55049, 55057, 55065, 55073, 55081, 55089, - 55097, 55106, 55115, 55123, 55131, 55140, 55149, 55158, 55167, 55177, - 55187, 55196, 55205, 55215, 55225, 55239, 55255, 55269, 55285, 55299, - 55313, 55327, 55341, 55351, 55362, 55372, 55383, 55399, 55415, 55423, - 55429, 55436, 55443, 55450, 55458, 55463, 55469, 55474, 55479, 55485, - 55490, 55495, 55500, 55505, 55510, 55517, 55523, 55531, 55537, 55543, - 55547, 55551, 55560, 55569, 55578, 55587, 55594, 55601, 55614, 55627, - 55640, 55653, 55661, 55669, 55676, 55683, 55691, 55699, 55707, 55715, - 55719, 55724, 55732, 55740, 55748, 55755, 55759, 55767, 55775, 55778, - 55782, 55787, 55794, 55802, 55810, 55829, 55849, 55868, 55888, 55908, - 55928, 55948, 55968, 55974, 55981, 55990, 55998, 56006, 56012, 56015, - 56018, 56023, 56026, 56046, 56053, 56059, 56065, 56069, 56072, 56075, - 56078, 56088, 56100, 56107, 56114, 56117, 56121, 56124, 56129, 56134, - 56139, 56145, 56154, 56161, 56168, 56176, 56183, 56190, 56193, 56199, - 56205, 56208, 56211, 56216, 56221, 56227, 56233, 56237, 56242, 56249, - 56253, 56259, 56263, 56267, 56275, 56287, 56295, 56299, 56301, 56310, - 56319, 56325, 56328, 56334, 56340, 56345, 56350, 56355, 56360, 56365, - 56370, 56372, 56378, 56383, 56391, 56395, 56401, 56404, 56408, 56416, - 56424, 56426, 56428, 56434, 56440, 56446, 56455, 56464, 56471, 56478, - 56484, 56491, 56496, 56501, 56506, 56512, 56518, 56523, 56530, 56534, - 56538, 56551, 56564, 56576, 56585, 56591, 56598, 56603, 56608, 56613, - 56618, 56623, 56625, 56632, 56640, 56648, 56656, 56663, 56671, 56677, - 56682, 56688, 56694, 56700, 56707, 56713, 56721, 56729, 56737, 56745, - 56753, 56759, 56765, 56774, 56778, 56787, 56796, 56805, 56813, 56817, - 56823, 56830, 56837, 56841, 56847, 56855, 56861, 56866, 56872, 56877, - 56882, 56889, 56896, 56901, 56906, 56914, 56922, 56932, 56942, 56949, - 56956, 56960, 56964, 56976, 56982, 56989, 56994, 56999, 57006, 57013, - 57019, 57025, 57035, 57042, 57050, 57058, 57067, 57074, 57080, 57087, - 57093, 57101, 57109, 57117, 57125, 57131, 57136, 57146, 57157, 57164, - 57173, 57179, 57184, 57189, 57199, 57208, 57214, 57220, 57228, 57233, - 57240, 57247, 57258, 57265, 57272, 57279, 57286, 57293, 57302, 57311, - 57324, 57337, 57349, 57361, 57374, 57388, 57394, 57400, 57410, 57420, - 57427, 57434, 57444, 57454, 57463, 57472, 57480, 57488, 57498, 57508, - 57523, 57538, 57547, 57556, 57569, 57582, 57591, 57600, 57611, 57622, - 57628, 57634, 57643, 57652, 57657, 57662, 57670, 57676, 57682, 57690, - 57698, 57711, 57724, 57728, 57732, 57741, 57750, 57757, 57765, 57773, - 57783, 57793, 57799, 57805, 57813, 57821, 57829, 57837, 57847, 57857, - 57860, 57863, 57868, 57873, 57879, 57885, 57892, 57899, 57910, 57921, - 57928, 57935, 57943, 57951, 57960, 57969, 57978, 57987, 57994, 58001, - 58005, 58009, 58018, 58027, 58032, 58037, 58042, 58047, 58053, 58067, - 58074, 58081, 58085, 58087, 58089, 58094, 58099, 58104, 58109, 58117, - 58124, 58131, 58139, 58151, 58159, 58167, 58178, 58182, 58186, 58192, - 58200, 58213, 58220, 58227, 58234, 58240, 58247, 58256, 58265, 58271, - 58277, 58283, 58294, 58305, 58313, 58322, 58327, 58330, 58335, 58340, - 58345, 58351, 58357, 58361, 58364, 58368, 58372, 58377, 58382, 58388, - 58394, 58398, 58402, 58409, 58416, 58423, 58430, 58437, 58444, 58453, - 58462, 58469, 58476, 58484, 58492, 58496, 58501, 58506, 58512, 58518, - 58521, 58524, 58527, 58530, 58535, 58540, 58545, 58550, 58555, 58560, - 58564, 58568, 58572, 58577, 58582, 58586, 58590, 58596, 58600, 58606, - 58611, 58618, 58626, 58633, 58641, 58648, 58656, 58665, 58672, 58682, - 58693, 58699, 58708, 58714, 58723, 58733, 58739, 58745, 58749, 58753, - 58762, 58772, 58779, 58787, 58796, 58805, 58812, 58818, 58825, 58830, - 58834, 58838, 58843, 58848, 58853, 58861, 58869, 58872, 58876, 58885, - 58895, 58904, 58914, 58926, 58940, 58944, 58949, 58953, 58958, 58963, - 58968, 58974, 58980, 58987, 58994, 59000, 59007, 59013, 59020, 59029, - 59038, 59044, 59051, 59057, 0, 0, 59064, 59072, 59080, 59089, 59098, - 59107, 59117, 59126, 59136, 59142, 59147, 59156, 59168, 59177, 59189, - 59196, 59204, 59211, 59219, 59224, 59230, 59235, 59241, 59249, 59258, - 59266, 59275, 59279, 59282, 59286, 59289, 59299, 0, 59302, 59309, 59318, - 59328, 59337, 59347, 59353, 59360, 59366, 59373, 59384, 59395, 59406, - 59417, 59427, 59437, 59447, 59457, 59465, 59473, 59481, 59489, 59497, - 59505, 59513, 59521, 59527, 59532, 59538, 59543, 59549, 59555, 59561, - 59567, 59579, 59589, 59594, 59601, 59606, 59613, 59616, 59620, 59624, - 59629, 59633, 59638, 59641, 59650, 59659, 59668, 59677, 59682, 59688, - 59694, 59702, 59712, 59719, 59728, 59733, 59736, 59739, 59744, 59749, - 59754, 59759, 59761, 59763, 59765, 59767, 59769, 59771, 59776, 59783, - 59790, 59792, 59794, 59796, 59798, 59800, 59802, 59804, 59806, 59811, - 59816, 59823, 59830, 59839, 59849, 59858, 59868, 59873, 59878, 59880, - 59887, 59894, 59901, 59908, 59915, 59922, 59929, 59932, 59935, 59938, - 59941, 59946, 59951, 59956, 59961, 59966, 59971, 59976, 59981, 59986, - 59991, 59996, 60001, 60007, 60011, 60016, 60021, 60026, 60031, 60036, - 60041, 60046, 60051, 60056, 60061, 60066, 60071, 60076, 60081, 60086, - 60091, 60096, 60101, 60106, 60111, 60116, 60121, 60127, 60132, 60138, - 60147, 60152, 60160, 60167, 60176, 60181, 60186, 60191, 60197, 60204, - 60211, 60216, 60221, 60226, 60231, 60236, 60241, 60246, 60251, 60256, - 60261, 60267, 60271, 60276, 60281, 60286, 60291, 60296, 60301, 60306, - 60311, 60316, 60321, 60326, 60331, 60336, 60341, 60346, 60351, 60356, - 60361, 60366, 60371, 60376, 60381, 60387, 60392, 60398, 60407, 60412, - 60420, 60427, 60436, 60441, 60446, 60451, 60457, 60464, 60471, 60479, - 60487, 60496, 60503, 60511, 60517, 60526, 60534, 60542, 60550, 60558, - 60566, 60574, 60579, 60586, 60591, 60597, 60605, 60612, 60619, 60627, - 60633, 60639, 60646, 60654, 60663, 60673, 60679, 60686, 60691, 60701, - 60711, 60716, 60721, 60726, 60731, 60736, 60741, 60746, 60751, 60756, - 60761, 60766, 60771, 60776, 60781, 60786, 60791, 60796, 60801, 60806, - 60811, 60816, 60821, 60826, 60831, 60836, 60841, 60846, 60851, 60856, - 60861, 60865, 60869, 60874, 60879, 60884, 60889, 60894, 60899, 60904, - 60909, 60914, 60919, 60924, 60929, 60934, 60939, 60944, 60949, 60954, - 60959, 60966, 60973, 60980, 60987, 60994, 61001, 61008, 61015, 61022, - 61029, 61036, 61043, 61050, 61057, 61062, 61067, 61074, 61081, 61088, - 61095, 61102, 61109, 61116, 61123, 61130, 61137, 61144, 61151, 61157, - 61163, 61169, 61175, 61182, 61189, 61196, 61203, 61210, 61217, 61224, - 61231, 61238, 61245, 61253, 61261, 61269, 61277, 61285, 61293, 61301, - 61309, 61313, 61319, 61325, 61329, 61335, 61341, 61347, 61354, 61361, - 61368, 61375, 61380, 61386, 61392, 61399, 0, 0, 0, 0, 0, 61406, 61414, - 61423, 61432, 61440, 61446, 61451, 61456, 61461, 61466, 61471, 61476, - 61481, 61486, 61491, 61496, 61501, 61506, 61511, 61516, 61521, 61526, - 61531, 61536, 61541, 61546, 61551, 61556, 61561, 61566, 61571, 61576, - 61581, 61586, 61591, 61596, 61601, 61606, 61611, 61616, 61621, 61626, - 61631, 61636, 61641, 0, 61646, 0, 0, 0, 0, 0, 61651, 0, 0, 61656, 61660, - 61665, 61670, 61675, 61680, 61689, 61694, 61699, 61704, 61709, 61714, - 61719, 61724, 61729, 61736, 61741, 61746, 61755, 61762, 61767, 61772, - 61777, 61784, 61789, 61796, 61801, 61806, 61813, 61820, 61825, 61830, - 61835, 61842, 61849, 61854, 61859, 61864, 61869, 61874, 61881, 61888, - 61893, 61898, 61903, 61908, 61913, 61918, 61923, 61928, 61933, 61938, - 61943, 61950, 61955, 61960, 0, 0, 0, 0, 0, 0, 0, 61965, 61972, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61977, 61982, 61986, 61990, 61994, - 61998, 62002, 62006, 62010, 62014, 62018, 62022, 62028, 62032, 62036, - 62040, 62044, 62048, 62052, 62056, 62060, 62064, 62068, 62072, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 62076, 62080, 62084, 62088, 62092, 62096, 62100, 0, - 62104, 62108, 62112, 62116, 62120, 62124, 62128, 0, 62132, 62136, 62140, - 62144, 62148, 62152, 62156, 0, 62160, 62164, 62168, 62172, 62176, 62180, - 62184, 0, 62188, 62192, 62196, 62200, 62204, 62208, 62212, 0, 62216, - 62220, 62224, 62228, 62232, 62236, 62240, 0, 62244, 62248, 62252, 62256, - 62260, 62264, 62268, 0, 62272, 62276, 62280, 62284, 62288, 62292, 62296, - 0, 62300, 62305, 62310, 62315, 62320, 62325, 62330, 62334, 62339, 62344, - 62349, 62353, 62358, 62363, 62368, 62373, 62377, 62382, 62387, 62392, - 62397, 62402, 62407, 62411, 62416, 62421, 62428, 62433, 62438, 62444, - 62451, 62458, 62467, 62474, 62483, 62488, 62493, 62500, 62507, 62513, - 62521, 62527, 62532, 62537, 62541, 62548, 62555, 62559, 62561, 62565, - 62571, 62573, 62577, 62581, 62585, 62591, 62596, 62600, 62604, 62609, - 62615, 62621, 62627, 62632, 62637, 62644, 62651, 62657, 62663, 62669, - 62675, 62681, 62687, 62691, 62695, 62702, 62709, 62715, 62719, 62724, - 62727, 62731, 62738, 62741, 62745, 62749, 62752, 62758, 62764, 62767, - 62773, 62777, 62781, 62787, 62792, 62797, 62799, 62802, 62806, 62812, - 62818, 62822, 62827, 62836, 62839, 62845, 62850, 62854, 62858, 62862, - 62865, 62870, 62876, 62884, 62892, 62898, 62903, 62908, 62914, 62920, - 62927, 62934, 62940, 62946, 62952, 62958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 62962, 62966, 62970, 62975, 62980, 62985, 62989, 62993, 62997, 63002, - 63007, 63011, 63015, 63019, 63023, 63028, 63033, 63038, 63043, 63047, - 63051, 63056, 63061, 63066, 63071, 63075, 0, 63079, 63083, 63087, 63091, - 63095, 63099, 63103, 63108, 63113, 63117, 63122, 63127, 63136, 63140, - 63144, 63148, 63155, 63159, 63164, 63169, 63173, 63177, 63183, 63188, - 63193, 63198, 63203, 63207, 63211, 63215, 63219, 63223, 63228, 63233, - 63237, 63241, 63246, 63251, 63256, 63260, 63264, 63269, 63274, 63280, - 63286, 63290, 63296, 63302, 63306, 63312, 63318, 63323, 63328, 63332, - 63338, 63342, 63346, 63352, 63358, 63363, 63368, 63372, 63376, 63384, - 63390, 63396, 63402, 63407, 63412, 63417, 63423, 63427, 63433, 63437, - 63441, 63447, 63453, 63459, 63465, 63471, 63477, 63483, 63489, 63495, - 63501, 63507, 63513, 63517, 63523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 63529, 63532, 63536, 63540, 63544, 63548, 63551, 63554, 63558, 63562, - 63566, 63570, 63573, 63578, 63582, 63586, 63590, 63596, 63600, 63604, - 63608, 63612, 63619, 63625, 63629, 63633, 63637, 63641, 63645, 63649, - 63653, 63657, 63661, 63665, 63669, 63675, 63679, 63683, 63687, 63691, - 63695, 63699, 63703, 63707, 63711, 63715, 63719, 63723, 63727, 63731, - 63735, 63739, 63745, 63751, 63756, 63761, 63765, 63769, 63773, 63777, - 63781, 63785, 63789, 63793, 63797, 63801, 63805, 63809, 63813, 63817, - 63821, 63825, 63829, 63833, 63837, 63841, 63845, 63849, 63853, 63857, - 63863, 63867, 63871, 63875, 63879, 63883, 63887, 63891, 63895, 63900, - 63907, 63911, 63915, 63919, 63923, 63927, 63931, 63935, 63939, 63943, - 63947, 63951, 63955, 63962, 63966, 63972, 63976, 63980, 63984, 63988, - 63992, 63995, 63999, 64003, 64007, 64011, 64015, 64019, 64023, 64027, - 64031, 64035, 64039, 64043, 64047, 64051, 64055, 64059, 64063, 64067, - 64071, 64075, 64079, 64083, 64087, 64091, 64095, 64099, 64103, 64107, - 64111, 64115, 64119, 64123, 64129, 64133, 64137, 64141, 64145, 64149, - 64153, 64157, 64161, 64165, 64169, 64173, 64177, 64181, 64185, 64189, - 64193, 64197, 64201, 64205, 64209, 64213, 64217, 64221, 64225, 64229, - 64233, 64237, 64245, 64249, 64253, 64257, 64261, 64265, 64271, 64275, - 64279, 64283, 64287, 64291, 64295, 64299, 64303, 64307, 64311, 64315, - 64319, 64323, 64329, 64333, 64337, 64341, 64345, 64349, 64353, 64357, - 64361, 64365, 64369, 64373, 64377, 64381, 64385, 64389, 64393, 64397, - 64401, 64405, 64409, 64413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64417, 64425, 64433, 64443, 64453, - 64462, 64472, 64482, 64493, 64505, 64516, 64528, 64535, 64545, 64556, - 64565, 64572, 64575, 64578, 64583, 64586, 64593, 64597, 64601, 64605, - 64610, 64615, 64621, 64627, 64632, 64637, 64643, 64649, 64655, 64661, - 64664, 64667, 64674, 64681, 64687, 64693, 64701, 64709, 64714, 64719, - 64723, 64731, 64737, 64744, 64749, 64754, 64759, 64764, 64769, 64774, - 64779, 64784, 64789, 64794, 64799, 64804, 64809, 64814, 64820, 64825, - 64829, 64835, 64846, 64855, 64869, 64878, 64882, 64892, 64898, 64904, - 64910, 64915, 64918, 64923, 64927, 0, 64933, 64938, 64942, 64947, 64951, - 64956, 64960, 64965, 64969, 64974, 64978, 64982, 64987, 64992, 64997, - 65002, 65007, 65012, 65017, 65022, 65027, 65031, 65036, 65041, 65046, - 65051, 65056, 65061, 65066, 65071, 65076, 65081, 65086, 65091, 65096, - 65102, 65107, 65112, 65117, 65122, 65126, 65131, 65135, 65140, 65145, - 65150, 65155, 65159, 65164, 65168, 65173, 65178, 65183, 65188, 65193, - 65198, 65203, 65208, 65213, 65218, 65223, 65228, 65232, 65237, 65242, - 65247, 65252, 65257, 65261, 65267, 65272, 65278, 65283, 65287, 65292, - 65297, 65302, 65307, 65313, 65318, 65323, 65328, 65333, 65338, 65343, - 65348, 0, 0, 65354, 65362, 65370, 65377, 65384, 65389, 65396, 65402, - 65407, 65411, 65414, 65418, 65421, 65425, 65428, 65432, 65435, 65439, - 65442, 65445, 65449, 65453, 65457, 65461, 65465, 65469, 65473, 65477, - 65481, 65484, 65488, 65492, 65496, 65500, 65504, 65508, 65512, 65516, - 65520, 65524, 65528, 65532, 65536, 65541, 65545, 65549, 65553, 65557, - 65560, 65564, 65567, 65571, 65575, 65579, 65583, 65586, 65590, 65593, - 65597, 65601, 65605, 65609, 65613, 65617, 65621, 65625, 65629, 65633, - 65637, 65641, 65644, 65648, 65652, 65656, 65660, 65664, 65667, 65672, - 65676, 65681, 65685, 65688, 65692, 65696, 65700, 65704, 65709, 65713, - 65717, 65721, 65725, 65729, 65733, 65737, 65742, 65746, 65750, 65754, - 65758, 65762, 65769, 65773, 65779, 0, 0, 0, 0, 0, 65784, 65789, 65794, - 65799, 65804, 65809, 65814, 65819, 65823, 65828, 65833, 65838, 65843, - 65848, 65853, 65858, 65863, 65868, 65872, 65877, 65882, 65887, 65891, - 65895, 65899, 65904, 65909, 65914, 65919, 65924, 65929, 65934, 65939, - 65944, 65949, 65953, 65957, 65962, 65967, 65972, 65977, 65982, 65989, 0, - 65994, 65998, 66002, 66006, 66010, 66014, 66018, 66022, 66026, 66030, - 66034, 66038, 66042, 66046, 66050, 66054, 66058, 66062, 66066, 66070, - 66074, 66078, 66082, 66086, 66090, 66094, 66098, 66102, 66106, 66110, - 66114, 66117, 66121, 66124, 66128, 66132, 66135, 66139, 66143, 66146, - 66150, 66154, 66158, 66162, 66165, 66169, 66173, 66177, 66181, 66185, - 66189, 66192, 66195, 66199, 66203, 66207, 66211, 66215, 66219, 66223, - 66227, 66231, 66235, 66239, 66243, 66247, 66251, 66255, 66259, 66263, - 66267, 66271, 66275, 66279, 66283, 66287, 66291, 66295, 66299, 66303, - 66307, 66311, 66315, 66319, 66323, 66327, 66331, 66335, 66339, 66343, - 66347, 66351, 66355, 66359, 0, 66363, 66369, 66375, 66380, 66385, 66390, - 66396, 66402, 66407, 66413, 66419, 66425, 66431, 66437, 66443, 66449, - 66455, 66460, 66465, 66470, 66475, 66480, 66485, 66490, 66495, 66500, - 66505, 66510, 66515, 66520, 66525, 66530, 66535, 66540, 66545, 66550, - 66555, 66561, 66567, 66573, 66579, 66584, 66589, 66594, 66600, 66605, - 66610, 66615, 66620, 66625, 66630, 66635, 66640, 66645, 66650, 66655, - 66660, 66665, 66670, 66675, 66680, 66685, 66690, 66695, 66700, 66705, - 66710, 66715, 66720, 66725, 66730, 66735, 66740, 66745, 66750, 66755, - 66760, 66765, 66770, 66775, 66780, 66785, 66790, 66795, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 66800, 66807, 66812, 66817, 66822, 66826, 66831, 66835, - 66840, 66845, 66850, 66855, 66860, 66864, 66869, 66874, 66879, 66884, - 66888, 66892, 66896, 66900, 66904, 66908, 66912, 66916, 66920, 66924, - 66928, 66932, 66936, 66940, 66945, 66950, 66955, 66960, 66965, 66970, - 66975, 66980, 66985, 66990, 66995, 67000, 67005, 67010, 67015, 67022, 0, - 67030, 67034, 67038, 67042, 67046, 67050, 67054, 67058, 67062, 67066, - 67071, 67076, 67081, 67086, 67091, 67096, 67101, 67106, 67111, 67116, - 67121, 67126, 67131, 67136, 67141, 67146, 67151, 67156, 67161, 67166, - 67171, 67176, 67181, 67186, 67191, 67196, 67201, 67206, 67211, 67216, - 67221, 67230, 67239, 67248, 67257, 67266, 67275, 67284, 67293, 67296, - 67301, 67306, 67311, 67316, 67321, 67326, 67331, 67336, 67341, 67345, - 67350, 67355, 67360, 67365, 67370, 67374, 67378, 67382, 67386, 67390, - 67394, 67398, 67402, 67406, 67410, 67414, 67418, 67422, 67426, 67431, - 67436, 67441, 67446, 67451, 67456, 67461, 67466, 67471, 67476, 67481, - 67486, 67491, 67496, 67503, 67510, 67515, 67520, 67524, 67528, 67532, - 67536, 67540, 67544, 67548, 67552, 67556, 67561, 67566, 67571, 67576, - 67581, 67586, 67591, 67596, 67601, 67606, 67611, 67616, 67621, 67626, - 67631, 67636, 67641, 67646, 67651, 67656, 67661, 67666, 67671, 67676, - 67681, 67686, 67691, 67696, 67701, 67706, 67711, 67715, 67720, 67725, - 67730, 67735, 67740, 67745, 67750, 67755, 67760, 67765, 67770, 67775, - 67779, 67784, 67789, 67794, 67799, 67804, 67809, 67814, 67819, 67824, - 67828, 67835, 67842, 67849, 67856, 67863, 67870, 67877, 67884, 67891, - 67898, 67905, 67912, 67915, 67918, 67921, 67926, 67929, 67932, 67935, - 67938, 67941, 67944, 67948, 67952, 67956, 67960, 67963, 67967, 67971, - 67975, 67979, 67983, 67987, 67991, 67995, 67998, 68001, 68005, 68009, - 68013, 68017, 68020, 68024, 68028, 68032, 68036, 68039, 68043, 68047, - 68051, 68055, 68058, 68062, 68066, 68069, 68073, 68077, 68081, 68085, - 68089, 68093, 68097, 68101, 68108, 68111, 68114, 68117, 68120, 68123, - 68126, 68129, 68132, 68135, 68138, 68141, 68144, 68147, 68150, 68153, - 68156, 68159, 68162, 68165, 68168, 68171, 68174, 68177, 68180, 68183, - 68186, 68189, 68192, 68195, 68198, 68201, 68204, 68207, 68210, 68213, - 68216, 68219, 68222, 68225, 68228, 68231, 68234, 68237, 68240, 68243, - 68246, 68249, 68252, 68255, 68258, 68261, 68264, 68267, 68270, 68273, - 68276, 68279, 68282, 68285, 68288, 68291, 68294, 68297, 68300, 68303, - 68306, 68309, 68312, 68315, 68318, 68321, 68324, 68327, 68330, 68333, - 68336, 68339, 68342, 68345, 68348, 68351, 68354, 68357, 68360, 68363, - 68366, 68369, 68372, 68381, 68389, 68397, 68405, 68413, 68421, 68429, - 68437, 68445, 68453, 68462, 68471, 68480, 68489, 68498, 68507, 68516, - 68525, 68534, 68543, 68552, 68561, 68570, 68579, 68588, 68591, 68594, - 68597, 68599, 68602, 68605, 68608, 68613, 68618, 68621, 68628, 68635, - 68642, 68649, 68652, 68657, 68659, 68663, 68665, 68667, 68670, 68673, - 68676, 68679, 68682, 68685, 68688, 68693, 68698, 68701, 68704, 68707, - 68710, 68713, 68716, 68719, 68723, 68726, 68729, 68732, 68735, 68738, - 68743, 68746, 68749, 68752, 68757, 68762, 68767, 68772, 68777, 68782, - 68787, 68792, 68798, 68806, 68808, 68811, 68814, 68817, 68820, 68826, - 68834, 68837, 68840, 68845, 68848, 68851, 68854, 68859, 68862, 68865, - 68870, 68873, 68876, 68881, 68884, 68887, 68892, 68897, 68902, 68905, - 68908, 68911, 68914, 68920, 68923, 68926, 68929, 68931, 68934, 68937, - 68940, 68945, 68948, 68951, 68954, 68957, 68960, 68965, 68968, 68971, - 68974, 68977, 68980, 68983, 68986, 68989, 68992, 68998, 69003, 69011, - 69019, 69027, 69035, 69043, 69051, 69059, 69067, 69075, 69084, 69093, - 69102, 69111, 69120, 69129, 69138, 69147, 69156, 69165, 69174, 69183, - 69192, 69201, 69210, 69219, 69228, 69237, 69246, 69255, 69264, 69273, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69276, 69285, 69294, 69305, 69312, 69317, 69322, 69329, 69336, 69342, - 69347, 69352, 69357, 69362, 69369, 69374, 69379, 69384, 69395, 69400, - 69405, 69412, 69417, 69424, 69429, 69434, 69441, 69448, 69455, 69464, - 69473, 69478, 69483, 69488, 69495, 69500, 69510, 69517, 69522, 69527, - 69532, 69537, 69542, 69547, 69555, 69562, 69569, 69574, 69581, 69586, - 69593, 69602, 69613, 69618, 69627, 69632, 69639, 69648, 69657, 69662, - 69667, 69674, 69680, 69687, 69694, 69698, 69702, 69705, 69709, 69713, - 69717, 69721, 69725, 69729, 69733, 69736, 69740, 69744, 69748, 69752, - 69756, 69760, 69763, 69767, 69771, 69774, 69778, 69782, 69786, 69790, - 69794, 69798, 69802, 69806, 69810, 69814, 69818, 69822, 69826, 69830, - 69834, 69838, 69842, 69846, 69850, 69854, 69858, 69862, 69866, 69870, - 69874, 69878, 69882, 69886, 69890, 69894, 69898, 69902, 69906, 69910, - 69914, 69918, 69922, 69926, 69930, 69934, 69938, 69942, 69946, 69950, - 69953, 69957, 69961, 69965, 69969, 69973, 69977, 69981, 69985, 69989, - 69993, 69997, 70001, 70005, 70009, 70013, 70017, 70021, 70025, 70029, - 70033, 70037, 70041, 70045, 70049, 70053, 70057, 70061, 70065, 70069, - 70073, 70077, 70081, 70085, 70089, 70093, 70097, 70101, 70105, 70109, - 70113, 70117, 70121, 70125, 70129, 70133, 70137, 70141, 70145, 70149, - 70153, 70157, 70161, 70165, 70169, 70173, 70177, 70181, 70185, 70189, - 70193, 70197, 70201, 70205, 70209, 70213, 70217, 70221, 70225, 70229, - 70233, 70237, 70241, 70245, 70249, 70253, 70257, 70261, 70265, 70269, - 70273, 70277, 70281, 70285, 70289, 70293, 70297, 70301, 70305, 70309, - 70313, 70317, 70321, 70325, 70329, 70333, 70337, 70341, 70345, 70349, - 70353, 70357, 70361, 70365, 70369, 70373, 70377, 70381, 70385, 70389, - 70393, 70397, 70401, 70405, 70409, 70413, 70417, 70421, 70424, 70428, - 70432, 70436, 70440, 70444, 70448, 70452, 70456, 70460, 70464, 70468, - 70472, 70476, 70480, 70484, 70488, 70492, 70496, 70500, 70504, 70508, - 70512, 70516, 70520, 70524, 70528, 70532, 70536, 70540, 70544, 70548, - 70552, 70556, 70560, 70564, 70568, 70572, 70576, 70580, 70584, 70588, - 70592, 70596, 70600, 70604, 70608, 70612, 70616, 70620, 70624, 70628, - 70632, 70636, 70640, 70644, 70648, 70652, 70656, 70660, 70664, 70668, - 70672, 70676, 70680, 70684, 70688, 70692, 70696, 70700, 70704, 70708, - 70712, 70716, 70720, 70724, 70728, 70732, 70736, 70740, 70744, 70748, - 70752, 70756, 70760, 70764, 70768, 70772, 70776, 70780, 70784, 70788, - 70792, 70796, 70800, 70804, 70808, 70812, 70816, 70820, 70824, 70828, - 70832, 70836, 70840, 70844, 70848, 70852, 70856, 70860, 70864, 70868, - 70872, 70876, 70880, 70884, 70887, 70891, 70895, 70899, 70903, 70907, - 70911, 70915, 70919, 70923, 70927, 70931, 70935, 70939, 70943, 70947, - 70951, 70955, 70959, 70963, 70967, 70971, 70975, 70979, 70983, 70987, - 70991, 70995, 70999, 71003, 71007, 71011, 71015, 71019, 71023, 71027, - 71031, 71035, 71039, 71043, 71047, 71051, 71055, 71059, 71063, 71067, - 71071, 71075, 71079, 71083, 71087, 71091, 71095, 71099, 71103, 71107, - 71111, 71115, 71119, 71123, 71127, 71131, 71135, 71139, 71143, 71147, - 71151, 71155, 71159, 71163, 71167, 71171, 71175, 71179, 71183, 71187, - 71191, 71195, 71199, 71203, 71207, 71211, 71215, 71219, 71223, 71227, - 71231, 71235, 71239, 71243, 71246, 71250, 71254, 71258, 71262, 71266, - 71270, 71274, 71278, 71282, 71286, 71290, 71294, 71298, 71302, 71306, - 71310, 71314, 71318, 71322, 71326, 71330, 71334, 71338, 71342, 71346, - 71350, 71354, 71358, 71362, 71366, 71370, 71374, 71378, 71382, 71386, - 71390, 71394, 71398, 71402, 71406, 71410, 71414, 71418, 71422, 71426, - 71430, 71434, 71438, 71442, 71446, 71450, 71454, 71458, 71462, 71466, - 71470, 71474, 71478, 71482, 71485, 71489, 71493, 71497, 71501, 71505, - 71509, 71513, 71517, 71521, 71525, 71529, 71533, 71537, 71541, 71545, - 71549, 71553, 71557, 71561, 71565, 71569, 71573, 71577, 71581, 71585, - 71589, 71593, 71597, 71601, 71605, 71609, 71613, 71617, 71621, 71625, - 71629, 71633, 71637, 71641, 71645, 71649, 71653, 71657, 71661, 71665, - 71669, 71673, 71677, 71681, 71685, 71689, 71693, 71697, 71701, 71705, - 71709, 71713, 71717, 71721, 71725, 71729, 71733, 71737, 71740, 71744, - 71748, 71752, 71756, 71760, 71764, 71768, 71772, 71776, 71780, 71784, - 71788, 71792, 71796, 71800, 71804, 71808, 71812, 71816, 71820, 71824, - 71828, 71832, 71836, 71840, 71844, 71848, 71852, 71856, 71860, 71864, - 71868, 71872, 71876, 71880, 71884, 71888, 71892, 71896, 71900, 71904, - 71908, 71912, 71916, 71920, 71924, 71928, 71932, 71936, 71940, 71944, - 71948, 71952, 71956, 71960, 71964, 71968, 71972, 71976, 71980, 71984, - 71988, 71992, 71996, 72000, 72004, 72008, 72012, 72016, 72020, 72024, - 72028, 72032, 72036, 72040, 72044, 72048, 72052, 72056, 72060, 72064, - 72068, 72072, 72076, 72080, 72084, 72088, 72092, 72096, 72100, 72104, - 72108, 72112, 72116, 72120, 72124, 72128, 72132, 72136, 72140, 72144, - 72148, 72152, 72156, 72160, 72164, 72168, 72172, 72176, 72180, 72184, - 72188, 72192, 72195, 72199, 72203, 72207, 72211, 72215, 72219, 72223, - 72227, 72231, 72235, 72239, 72243, 72247, 72251, 72255, 72259, 72263, - 72267, 72271, 72275, 72279, 72283, 72287, 72291, 72295, 72299, 72303, - 72307, 72311, 72315, 72319, 72323, 72327, 72331, 72335, 72339, 72343, - 72347, 72351, 72355, 72359, 72363, 72367, 72371, 72375, 72379, 72383, - 72387, 72391, 72395, 72399, 72403, 72407, 72411, 72415, 72419, 72423, - 72427, 72431, 72435, 72439, 72443, 72447, 72451, 72455, 72459, 72463, - 72467, 72471, 72475, 72479, 72483, 72487, 72491, 72495, 72499, 72503, - 72507, 72511, 72515, 72519, 72523, 72527, 72531, 72535, 72539, 72543, - 72547, 72551, 72555, 72559, 72563, 72567, 72571, 72575, 72579, 72583, - 72587, 72591, 72595, 72599, 72603, 72607, 72611, 72615, 72619, 72623, - 72627, 72631, 72635, 72639, 72643, 72647, 72651, 72655, 72659, 72663, - 72667, 72671, 72675, 72679, 72683, 72687, 72691, 72695, 72699, 72703, - 72707, 72711, 72715, 72719, 72723, 72727, 72731, 72735, 72739, 72743, - 72747, 72751, 72755, 72759, 72763, 72767, 72771, 72775, 72779, 72783, - 72787, 72791, 72795, 72798, 72802, 72806, 72810, 72814, 72818, 72822, - 72826, 72829, 72833, 72837, 72841, 72845, 72849, 72853, 72857, 72861, - 72865, 72869, 72873, 72877, 72881, 72885, 72889, 72893, 72897, 72901, - 72905, 72909, 72913, 72917, 72921, 72925, 72929, 72933, 72937, 72941, - 72945, 72949, 72953, 72957, 72961, 72965, 72969, 72973, 72977, 72981, - 72985, 72989, 72993, 72997, 73001, 73005, 73009, 73013, 73017, 73021, - 73025, 73029, 73033, 73037, 73041, 73045, 73049, 73053, 73057, 73061, - 73065, 73069, 73073, 73077, 73081, 73085, 73089, 73093, 73097, 73101, - 73105, 73109, 73113, 73117, 73121, 73125, 73129, 73133, 73137, 73141, - 73145, 73149, 73153, 73157, 73161, 73165, 73169, 73173, 73177, 73181, - 73185, 73189, 73193, 73197, 73201, 73205, 73209, 73213, 73217, 73221, - 73225, 73229, 73233, 73237, 73241, 73245, 73249, 73253, 73257, 73261, - 73265, 73269, 73273, 73277, 73281, 73285, 73289, 73293, 73297, 73301, - 73305, 73309, 73313, 73317, 73321, 73325, 73329, 73333, 73337, 73341, - 73345, 73349, 73353, 73357, 73361, 73365, 73369, 73373, 73377, 73381, - 73385, 73389, 73393, 73397, 73401, 73405, 73409, 73413, 73417, 73421, - 73425, 73429, 73433, 73437, 73441, 73445, 73449, 73453, 73457, 73461, - 73465, 73469, 73473, 73477, 73481, 73485, 73489, 73493, 73497, 73501, - 73505, 73509, 73513, 73517, 73521, 73525, 73529, 73533, 73537, 73541, - 73545, 73549, 73553, 73556, 73560, 73564, 73568, 73572, 73576, 73580, - 73584, 73588, 73592, 73596, 73600, 73604, 73608, 73612, 73616, 73620, - 73624, 73628, 73632, 73636, 73640, 73644, 73648, 73652, 73656, 73660, - 73664, 73668, 73672, 73676, 73680, 73684, 73688, 73692, 73696, 73700, - 73704, 73708, 73712, 73716, 73720, 73724, 73728, 73732, 73736, 73740, - 73744, 73748, 73752, 73756, 73760, 73764, 73768, 73772, 73776, 73780, - 73784, 73788, 73792, 73796, 73800, 73804, 73808, 73812, 73816, 73820, - 73824, 73828, 73832, 73836, 73840, 73844, 73848, 73852, 73856, 73860, - 73864, 73868, 73872, 73876, 73880, 73884, 73888, 73892, 73896, 73900, - 73904, 73908, 73912, 73916, 73920, 73924, 73928, 73932, 73936, 73940, - 73944, 73948, 73952, 73956, 73960, 73964, 73968, 73972, 73976, 73980, - 73984, 73988, 73992, 73996, 74000, 74004, 74008, 74012, 74016, 74020, - 74024, 74028, 74032, 74036, 74040, 74044, 74048, 74052, 74056, 74060, - 74064, 74068, 74072, 74076, 74080, 74084, 74088, 74092, 74096, 74100, - 74104, 74108, 74112, 74116, 74120, 74124, 74128, 74132, 74136, 74140, - 74144, 74148, 74152, 74156, 74160, 74164, 74168, 74172, 74176, 74180, - 74184, 74188, 74192, 74196, 74200, 74204, 74208, 74212, 74216, 74220, - 74224, 74228, 74232, 74236, 74240, 74244, 74248, 74252, 74256, 74260, - 74264, 74268, 74272, 74276, 74280, 74284, 74288, 74292, 74296, 74300, - 74304, 74308, 74312, 74316, 74320, 74324, 74328, 74332, 74336, 0, 0, 0, - 74340, 74344, 74348, 74352, 74356, 74360, 74364, 74368, 74372, 74376, - 74380, 74384, 74388, 74392, 74396, 74400, 74404, 74408, 74412, 74416, - 74420, 74424, 74428, 74432, 74436, 74440, 74444, 74448, 74452, 74456, - 74460, 74464, 74468, 74472, 74476, 74480, 74484, 74488, 74492, 74496, - 74500, 74504, 74508, 74512, 74516, 74520, 74524, 74528, 74532, 74536, - 74540, 74544, 74548, 74552, 74556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74560, - 74565, 74569, 74574, 74579, 74584, 74589, 74594, 74598, 74603, 74608, - 74613, 74618, 74623, 74628, 74633, 74637, 74641, 74645, 74649, 74654, - 74659, 74664, 74668, 74673, 74678, 74683, 74688, 74693, 74697, 74702, - 74706, 74711, 74715, 74720, 74724, 74728, 74732, 74737, 74742, 74747, - 74755, 74763, 74771, 74779, 74786, 74794, 74800, 74808, 74812, 74816, - 74820, 74824, 74828, 74832, 74836, 74840, 74844, 74848, 74852, 74856, - 74860, 74864, 74868, 74872, 74876, 74880, 74884, 74888, 74892, 74896, - 74900, 74904, 74908, 74912, 74916, 74920, 74924, 74928, 74932, 74936, - 74940, 74944, 74948, 74952, 74955, 74959, 74963, 74967, 74971, 74975, - 74979, 74983, 74987, 74991, 74995, 74999, 75003, 75007, 75011, 75015, - 75019, 75023, 75027, 75031, 75035, 75039, 75043, 75047, 75051, 75055, - 75059, 75063, 75067, 75071, 75075, 75079, 75083, 75087, 75091, 75095, - 75099, 75102, 75106, 75110, 75113, 75117, 75121, 75125, 75128, 75132, - 75136, 75140, 75144, 75148, 75152, 75156, 75160, 75164, 75168, 75172, - 75176, 75180, 75183, 75186, 75190, 75194, 75197, 75201, 75205, 75209, - 75213, 75217, 75221, 75224, 75227, 75231, 75235, 75239, 75242, 75245, - 75249, 75253, 75257, 75261, 75265, 75269, 75273, 75277, 75281, 75285, - 75289, 75293, 75297, 75301, 75305, 75309, 75313, 75317, 75321, 75325, - 75329, 75333, 75337, 75341, 75345, 75349, 75353, 75357, 75361, 75365, - 75369, 75373, 75377, 75381, 75385, 75389, 75393, 75396, 75400, 75404, - 75408, 75412, 75416, 75420, 75424, 75428, 75432, 75436, 75440, 75444, - 75448, 75452, 75456, 75460, 75464, 75468, 75472, 75476, 75480, 75484, - 75488, 75492, 75496, 75500, 75504, 75508, 75512, 75516, 75520, 75524, - 75528, 75532, 75536, 75540, 75543, 75547, 75551, 75555, 75559, 75563, - 75567, 75571, 75575, 75579, 75583, 75587, 75591, 75595, 75599, 75603, - 75607, 75610, 75614, 75618, 75622, 75626, 75630, 75634, 75638, 75642, - 75646, 75650, 75654, 75658, 75662, 75666, 75670, 75674, 75678, 75682, - 75686, 75690, 75694, 75697, 75701, 75705, 75709, 75713, 75717, 75721, - 75725, 75729, 75733, 75737, 75741, 75745, 75749, 75753, 75757, 75761, - 75765, 75769, 75773, 75777, 75781, 75785, 75789, 75793, 75797, 75801, - 75805, 75809, 75813, 75817, 75821, 75825, 75829, 75833, 75837, 75841, - 75845, 75849, 75853, 75857, 75861, 75865, 75869, 75872, 75877, 75881, - 75887, 75892, 75898, 75902, 75906, 75910, 75914, 75918, 75922, 75926, - 75930, 75934, 75938, 75942, 75946, 75950, 75954, 75957, 75960, 75963, - 75966, 75969, 75972, 75975, 75978, 75981, 75986, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75992, 75997, 76002, 76007, 76012, - 76019, 76026, 76031, 76036, 76041, 76046, 76053, 76060, 76067, 76074, - 76081, 76088, 76098, 76108, 76115, 76122, 76129, 76136, 76142, 76148, - 76157, 76166, 76173, 76180, 76191, 76202, 76207, 76212, 76219, 76226, - 76233, 76240, 76247, 76254, 76261, 76268, 76274, 76280, 76286, 76292, - 76299, 76306, 76311, 76315, 76322, 76329, 76336, 76340, 76347, 76351, - 76356, 76360, 76366, 76371, 76377, 76382, 76386, 76390, 76393, 76396, - 76401, 76406, 76411, 76416, 76421, 76426, 76431, 76436, 76441, 76446, - 76454, 76462, 76467, 76472, 76477, 76482, 76487, 76492, 76497, 76502, - 76507, 76512, 76517, 76522, 76527, 76532, 76538, 76544, 76550, 76556, - 76561, 76567, 76570, 76573, 76576, 76580, 76584, 76588, 76592, 76595, - 76599, 76602, 76605, 76608, 76612, 76616, 76620, 76624, 76628, 76632, - 76636, 76640, 76644, 76648, 76652, 76656, 76660, 76664, 76668, 76672, - 76676, 76680, 76684, 76688, 76692, 76696, 76699, 76703, 76707, 76711, - 76715, 76719, 76723, 76727, 76731, 76735, 76739, 76743, 76747, 76751, - 76755, 76759, 76763, 76767, 76771, 76775, 76779, 76783, 76787, 76791, - 76795, 76798, 76802, 76806, 76810, 76814, 76818, 76822, 76826, 76829, - 76833, 76837, 76841, 76845, 76849, 76853, 76857, 76861, 76865, 76869, - 76873, 76877, 76882, 76887, 76890, 76895, 76898, 76901, 76904, 0, 0, 0, - 0, 0, 0, 0, 0, 76908, 76917, 76926, 76935, 76944, 76953, 76962, 76971, - 76980, 76988, 76995, 77003, 77010, 77018, 77028, 77037, 77047, 77056, - 77066, 77074, 77081, 77089, 77096, 77104, 77109, 77114, 77120, 77128, - 77134, 77140, 77147, 77156, 77164, 77172, 77180, 77187, 77194, 77201, - 77208, 77213, 77218, 77223, 77228, 77233, 77238, 77243, 77248, 77256, - 77264, 77270, 77276, 77281, 77286, 77291, 77296, 77301, 77306, 77311, - 77316, 77325, 77334, 77339, 77344, 77354, 77364, 77371, 77378, 77387, - 77396, 77408, 77420, 77426, 77432, 77440, 77448, 77458, 77468, 77475, - 77482, 77487, 77492, 77504, 77516, 77524, 77532, 77542, 77552, 77564, - 77576, 77585, 77594, 77601, 77608, 77615, 77622, 77631, 77640, 77645, - 77650, 77657, 77664, 77671, 77678, 77690, 77702, 77707, 77712, 77717, - 77722, 77727, 77732, 77737, 77742, 77746, 77751, 77756, 77761, 77766, - 77771, 77777, 77782, 77787, 77794, 77801, 77808, 77815, 77822, 77830, - 77838, 77843, 77848, 77854, 77860, 77867, 77874, 77881, 77888, 77895, - 77899, 77906, 77911, 77916, 77922, 77935, 77941, 77949, 77957, 77964, - 77971, 77980, 77989, 77996, 78003, 78010, 78017, 78024, 78031, 78038, - 78045, 78052, 78059, 78068, 78077, 78086, 78095, 78104, 78113, 78122, - 78131, 78140, 78149, 78156, 78164, 78170, 78178, 78184, 78190, 78196, - 78202, 78210, 78215, 78220, 78225, 78230, 78235, 78241, 78247, 78253, - 78259, 78265, 78271, 78277, 78283, 78290, 78297, 78304, 78311, 78320, - 78327, 78336, 78348, 78360, 78372, 0, 0, 0, 0, 0, 78384, 78393, 0, 78402, - 0, 78408, 78414, 78422, 78430, 78437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78444, 78449, 78454, 78459, 78467, - 78475, 78482, 78489, 78495, 78502, 78510, 78518, 78526, 78534, 78542, - 78548, 78554, 78561, 78567, 78573, 78579, 78586, 78593, 78600, 78607, - 78614, 78621, 78628, 78635, 78642, 78649, 78656, 78663, 78670, 78677, - 78683, 78690, 78697, 78704, 78711, 78718, 78725, 78732, 78739, 78746, - 78753, 78760, 78767, 78774, 78781, 78788, 78795, 78802, 78809, 78817, - 78825, 78833, 78841, 78849, 0, 0, 0, 78858, 78866, 78874, 78882, 78890, - 78898, 78906, 78912, 78918, 78924, 0, 0, 0, 0, 0, 0, 78930, 78934, 78939, - 78944, 78949, 78954, 78959, 78964, 78969, 78974, 78979, 78984, 78988, - 78992, 78997, 79002, 79006, 79011, 79016, 79021, 79026, 79031, 79036, - 79041, 79045, 79049, 79053, 79058, 79062, 79066, 79070, 79074, 79078, - 79082, 79086, 79091, 79096, 79101, 79106, 79111, 79118, 79124, 79129, - 79134, 79139, 79144, 79150, 79157, 79163, 79170, 79176, 79182, 79187, - 79194, 79200, 79205, 0, 0, 0, 0, 0, 0, 0, 0, 79211, 79216, 79221, 79225, - 79230, 79234, 79239, 79243, 79248, 79253, 79259, 79264, 79270, 79274, - 79279, 79284, 79288, 79293, 79298, 79302, 79307, 79312, 79317, 79322, - 79327, 79332, 79337, 79342, 79347, 79352, 79357, 79362, 79367, 79372, - 79377, 79382, 79387, 79392, 79396, 79400, 79405, 79410, 79415, 79419, - 79423, 79427, 79431, 79436, 79441, 79446, 79450, 79454, 79459, 79465, - 79471, 79476, 79482, 79487, 79493, 79499, 79506, 79512, 79519, 79524, - 79530, 79536, 79541, 79547, 79553, 79558, 0, 0, 0, 0, 0, 0, 0, 0, 79563, - 79567, 79572, 79577, 79581, 79585, 79589, 79593, 79597, 79601, 79605, - 79609, 0, 0, 0, 0, 0, 0, 79613, 79618, 79622, 79626, 79630, 79634, 79638, - 79642, 79646, 79650, 79654, 79658, 79662, 79666, 79670, 79674, 79678, - 79683, 79688, 79694, 79700, 79707, 79712, 79717, 79723, 79727, 79732, - 79735, 79738, 79742, 79747, 79751, 79756, 79763, 79769, 79775, 79781, - 79787, 79793, 79799, 79805, 79811, 79817, 79823, 79830, 79837, 79844, - 79850, 79857, 79864, 79871, 79878, 79885, 79891, 79897, 79904, 79910, - 79917, 79924, 79930, 79936, 79942, 79949, 79956, 79962, 79969, 79976, - 79982, 79989, 79995, 80002, 80009, 80015, 80021, 80028, 80034, 80041, - 80048, 80057, 80064, 80071, 80075, 80080, 80085, 80090, 80095, 80099, - 80103, 80108, 80112, 80117, 80122, 80127, 80131, 80135, 80139, 80143, - 80148, 80152, 80157, 80162, 80167, 80172, 80176, 80181, 80186, 80191, - 80197, 80202, 80208, 80214, 80220, 80226, 80232, 80237, 80243, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 80247, 80252, 80256, 80260, 80264, 80268, 80272, - 80276, 80280, 80284, 80288, 80292, 80296, 80300, 80304, 80308, 80312, - 80316, 80320, 80324, 80328, 80332, 80336, 80340, 80344, 80348, 80352, - 80356, 80360, 80364, 0, 0, 0, 80368, 80373, 80378, 80383, 80388, 80392, - 80399, 80403, 80408, 80412, 80419, 80426, 80435, 80439, 80444, 80448, - 80452, 80459, 80466, 80471, 80478, 80483, 80488, 80495, 80500, 80507, - 80514, 80519, 80524, 80531, 80536, 80543, 80550, 80555, 80562, 80567, - 80574, 80578, 80582, 80589, 80594, 80601, 80605, 80609, 80613, 80620, - 80624, 80629, 80636, 80643, 80647, 80651, 80658, 80664, 80670, 80676, - 80684, 80690, 80698, 80704, 80712, 80718, 80724, 80730, 80736, 80740, - 80745, 80750, 80756, 80762, 80768, 80774, 80780, 80786, 80792, 80798, - 80806, 80812, 0, 80819, 80823, 80828, 80832, 80836, 80840, 80844, 80848, - 80852, 80856, 80860, 0, 0, 0, 0, 80864, 80872, 80878, 80884, 80890, - 80896, 80902, 80908, 80914, 80921, 80928, 80935, 80942, 80949, 80956, - 80963, 80970, 80977, 80984, 80991, 80997, 81003, 81009, 81015, 81021, - 81027, 81033, 81039, 81045, 81052, 81059, 81066, 81073, 0, 81080, 81084, - 81088, 81092, 81096, 81101, 81105, 81109, 81114, 81119, 81124, 81129, - 81134, 81139, 81144, 81149, 81154, 81159, 81164, 81169, 81174, 81179, - 81184, 81189, 81194, 81198, 81203, 81207, 81212, 81217, 81222, 81227, - 81232, 81236, 81241, 81245, 81249, 81253, 81258, 81263, 81267, 81271, - 81277, 81282, 81288, 81294, 81299, 81305, 81310, 81316, 81322, 81328, - 81333, 81338, 81343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81349, 81355, 81361, - 81367, 81374, 81380, 81386, 81392, 81398, 81404, 81409, 81414, 81420, - 81427, 0, 0, 81434, 81439, 81443, 81447, 81451, 81455, 81459, 81463, - 81467, 81471, 0, 0, 81475, 81481, 81487, 81494, 81502, 81508, 81514, - 81520, 81526, 81532, 81538, 81544, 81550, 81556, 81562, 81568, 81573, - 81578, 81583, 81589, 81595, 81602, 81608, 81614, 81619, 81626, 81633, - 81640, 81646, 81651, 81656, 81661, 81669, 81676, 81683, 81691, 81699, - 81706, 81713, 81720, 81727, 81734, 81741, 81748, 81755, 81762, 81769, - 81776, 81783, 81790, 81797, 81804, 81811, 81818, 81825, 81832, 81839, - 81845, 81851, 81858, 81865, 81872, 81879, 81886, 81893, 81900, 81907, - 81914, 81921, 81928, 81935, 81942, 81949, 81956, 81963, 81970, 81977, - 81984, 81991, 81998, 82005, 82012, 82019, 82025, 82031, 82038, 82044, - 82049, 82055, 82060, 82065, 82070, 82077, 82083, 82089, 82095, 82101, - 82107, 82113, 82119, 82127, 82135, 82143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82151, 82157, 82163, 82169, - 82177, 82185, 82191, 82197, 82204, 82211, 82218, 82225, 82232, 82239, - 82246, 82253, 82260, 82268, 82276, 82284, 82292, 82300, 82306, 82314, - 82320, 82328, 82337, 82345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82351, 82355, - 82359, 82363, 82367, 82371, 0, 0, 82375, 82379, 82383, 82387, 82391, - 82395, 0, 0, 82399, 82403, 82407, 82411, 82415, 82419, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 82423, 82427, 82431, 82435, 82439, 82443, 82447, 0, 82451, - 82455, 82459, 82463, 82467, 82471, 82475, 0, 82479, 82486, 82492, 82498, - 82504, 82512, 82519, 82528, 82540, 82550, 82559, 82567, 82575, 82583, - 82589, 82597, 82605, 82612, 82620, 82630, 82637, 82646, 82652, 82662, - 82671, 82676, 82684, 82693, 82698, 82707, 82714, 82724, 82736, 82741, - 82747, 82754, 82759, 82769, 82779, 82789, 82799, 82814, 82827, 82838, - 82846, 82851, 82863, 82872, 82879, 82886, 82892, 82899, 82904, 82911, - 82917, 82928, 82939, 82949, 82955, 82960, 0, 0, 0, 0, 82965, 82969, - 82973, 82977, 82981, 82985, 82990, 82995, 82999, 83004, 83009, 83014, - 83019, 83024, 83028, 83033, 83038, 83043, 83048, 83053, 83057, 83062, - 83067, 83072, 83077, 83082, 83086, 83091, 83096, 83101, 83106, 83110, - 83115, 83120, 83125, 83130, 83135, 83140, 83145, 83150, 83155, 83160, - 83165, 83170, 83175, 83179, 83184, 83189, 83194, 83199, 83204, 83209, - 83214, 83219, 83224, 83229, 83234, 83239, 83244, 83249, 83254, 83259, - 83264, 83269, 83274, 83279, 83284, 83289, 83294, 83299, 83304, 83309, - 83314, 83319, 83324, 83329, 83334, 83339, 83344, 83349, 83353, 83360, - 83367, 83374, 83381, 83387, 83393, 83400, 83407, 83414, 83421, 83428, - 83435, 83442, 83449, 83456, 83462, 83469, 83476, 83483, 83490, 83497, - 83504, 83511, 83518, 83525, 83532, 83539, 83548, 83557, 83566, 83575, - 83584, 83593, 83602, 83611, 83619, 83627, 83635, 83643, 83651, 83659, - 83667, 83675, 83681, 83689, 0, 0, 83697, 83704, 83710, 83716, 83722, - 83728, 83734, 83740, 83746, 83752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83758, 83763, 83768, - 83773, 83778, 83783, 83788, 83793, 83798, 83803, 83808, 83813, 83818, - 83823, 83828, 83833, 83838, 83843, 83848, 83853, 83858, 83863, 83868, 0, - 0, 0, 0, 83873, 83877, 83881, 83885, 83889, 83893, 83897, 83901, 83905, - 83909, 83913, 83917, 83921, 83925, 83929, 83933, 83937, 83941, 83945, - 83949, 83953, 83957, 83961, 83965, 83969, 83973, 83977, 83981, 83985, - 83989, 83993, 83997, 84001, 84005, 84009, 84013, 84017, 84021, 84025, - 84029, 84033, 84037, 84041, 84045, 84049, 84053, 84057, 84061, 84065, 0, - 0, 0, 0, 84069, 84073, 84077, 84081, 84085, 84089, 84093, 84097, 84101, - 84105, 84109, 84113, 84117, 84121, 84125, 84129, 84133, 84137, 84141, - 84145, 84149, 84153, 84157, 84161, 84165, 84169, 84173, 84177, 84181, - 84185, 84189, 84193, 84197, 84201, 84205, 84209, 84213, 84217, 84221, - 84225, 84229, 84233, 84237, 84241, 84245, 84249, 84253, 84257, 84261, - 84265, 84269, 84273, 84277, 84281, 84285, 84289, 84293, 84297, 84301, - 84305, 84309, 84313, 84317, 84321, 84325, 84329, 84333, 84337, 84341, - 84345, 84349, 84353, 84357, 84361, 84365, 84369, 84373, 84377, 84381, - 84385, 84389, 84393, 84397, 84401, 84405, 84409, 84413, 84417, 84421, - 84425, 84429, 84433, 84437, 84441, 84445, 84449, 84453, 84457, 84461, - 84465, 84469, 84473, 84477, 84481, 84485, 84489, 84493, 84497, 84501, - 84505, 84509, 84513, 84517, 84521, 84525, 84529, 84533, 84537, 84541, - 84545, 84549, 84553, 84557, 84561, 84565, 84569, 84573, 84577, 84581, - 84585, 84589, 84593, 84597, 84601, 84605, 84609, 84613, 84617, 84621, - 84625, 84629, 84633, 84637, 84641, 84645, 84649, 84653, 84657, 84661, - 84665, 84669, 84673, 84677, 84681, 84685, 84689, 84693, 84697, 84701, - 84705, 84709, 84713, 84717, 84721, 84725, 84729, 84733, 84737, 84741, - 84745, 84749, 84753, 84757, 84761, 84765, 84769, 84773, 84777, 84781, - 84785, 84789, 84793, 84797, 84801, 84805, 84809, 84813, 84817, 84821, - 84825, 84829, 84833, 84837, 84841, 84845, 84849, 84853, 84857, 84861, - 84865, 84869, 84873, 84877, 84881, 84885, 84889, 84893, 84897, 84901, - 84905, 84909, 84913, 84917, 84921, 84925, 84929, 84933, 84937, 84941, - 84945, 84949, 84953, 84957, 84961, 84965, 84969, 84973, 84977, 84981, - 84985, 84989, 84993, 84997, 85001, 85005, 85009, 85013, 85017, 85021, - 85025, 85029, 85033, 85037, 85041, 85045, 85049, 85053, 85057, 85061, - 85065, 85069, 85073, 85077, 85081, 85085, 85089, 85093, 85097, 85101, - 85105, 85109, 85113, 85117, 85121, 85125, 85129, 85133, 85137, 85141, - 85145, 85149, 85153, 85157, 85161, 85165, 85169, 85173, 85177, 85181, - 85185, 85189, 85193, 85197, 85201, 85205, 85209, 85213, 85217, 85221, - 85225, 85229, 85233, 85237, 85241, 85245, 85249, 85253, 85257, 85261, - 85265, 85269, 85273, 85277, 85281, 85285, 85289, 85293, 85297, 85301, - 85305, 85309, 85313, 85317, 85321, 85325, 85329, 85333, 85337, 85341, - 85345, 85349, 85353, 85357, 85361, 85365, 85369, 85373, 85377, 85381, - 85385, 85389, 85393, 85397, 85401, 85405, 85409, 85413, 85417, 85421, - 85425, 85429, 85433, 85437, 85441, 85445, 85449, 85453, 85457, 85461, - 85465, 85469, 85473, 85477, 85481, 85485, 85489, 85493, 85497, 85501, - 85505, 85509, 85513, 85517, 85521, 85525, 85529, 0, 0, 85533, 85537, - 85541, 85545, 85549, 85553, 85557, 85561, 85565, 85569, 85573, 85577, - 85581, 85585, 85589, 85593, 85597, 85601, 85605, 85609, 85613, 85617, - 85621, 85625, 85629, 85633, 85637, 85641, 85645, 85649, 85653, 85657, - 85661, 85665, 85669, 85673, 85677, 85681, 85685, 85689, 85693, 85697, - 85701, 85705, 85709, 85713, 85717, 85721, 85725, 85729, 85733, 85737, - 85741, 85745, 85749, 85753, 85757, 85761, 85765, 85769, 85773, 85777, - 85781, 85785, 85789, 85793, 85797, 85801, 85805, 85809, 85813, 85817, - 85821, 85825, 85829, 85833, 85837, 85841, 85845, 85849, 85853, 85857, - 85861, 85865, 85869, 85873, 85877, 85881, 85885, 85889, 85893, 85897, - 85901, 85905, 85909, 85913, 85917, 85921, 85925, 85929, 85933, 85937, - 85941, 85945, 85949, 85953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 85957, 85962, 85967, 85972, 85977, 85982, 85990, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 85995, 86003, 86011, 86019, 86027, 0, 0, 0, 0, 0, 86035, - 86042, 86049, 86059, 86065, 86071, 86077, 86083, 86089, 86095, 86102, - 86108, 86114, 86120, 86129, 86138, 86150, 86162, 86168, 86174, 86180, - 86187, 86194, 86201, 86208, 86215, 0, 86222, 86229, 86236, 86244, 86251, - 0, 86258, 0, 86265, 86272, 0, 86279, 86287, 0, 86294, 86301, 86308, - 86315, 86322, 86329, 86336, 86343, 86350, 86357, 86362, 86369, 86376, - 86382, 86388, 86394, 86401, 86407, 86413, 86419, 86426, 86432, 86438, - 86444, 86451, 86457, 86463, 86469, 86476, 86482, 86488, 86494, 86501, - 86507, 86513, 86519, 86526, 86532, 86538, 86544, 86551, 86557, 86563, - 86569, 86576, 86582, 86588, 86594, 86601, 86607, 86613, 86619, 86626, - 86632, 86638, 86644, 86651, 86657, 86663, 86669, 86676, 86682, 86688, - 86694, 86700, 86706, 86712, 86718, 86724, 86730, 86736, 86742, 86748, - 86754, 86760, 86766, 86773, 86779, 86785, 86791, 86798, 86804, 86810, - 86816, 86823, 86829, 86835, 86841, 86848, 86856, 86864, 86870, 86876, - 86882, 86889, 86898, 86907, 86915, 86923, 86931, 86940, 86948, 86956, - 86964, 86973, 86980, 86987, 86998, 87009, 87013, 87017, 87022, 87027, - 87032, 87037, 87046, 87055, 87061, 87067, 87074, 87081, 87088, 87092, - 87098, 87104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87109, - 87115, 87121, 87127, 87134, 87139, 87144, 87150, 87156, 87162, 87168, - 87177, 87183, 87189, 87197, 87205, 87213, 87221, 87227, 87233, 87239, - 87246, 87259, 87273, 87284, 87295, 87307, 87319, 87331, 87343, 87354, - 87365, 87377, 87389, 87401, 87413, 87425, 87437, 87449, 87466, 87483, - 87500, 87507, 87514, 87521, 87529, 87541, 87552, 87563, 87576, 87587, - 87596, 87604, 87613, 87621, 87631, 87639, 87648, 87656, 87665, 87673, - 87683, 87691, 87700, 87708, 87718, 87726, 87734, 87742, 87750, 87757, - 87766, 87774, 87782, 87791, 87799, 87808, 87816, 87824, 87832, 87841, - 87849, 87858, 87866, 87874, 87882, 87890, 87899, 87907, 87916, 87924, - 87933, 87941, 87950, 87958, 87968, 87976, 87984, 87992, 88002, 88010, - 88018, 88027, 88035, 88044, 88053, 88061, 88071, 88079, 88088, 88096, - 88105, 88113, 88123, 88131, 88139, 88146, 88154, 88161, 88170, 88177, - 88186, 88194, 88203, 88211, 88221, 88229, 88238, 88246, 88256, 88264, - 88272, 88279, 88287, 88294, 88303, 88310, 88320, 88330, 88341, 88350, - 88359, 88368, 88377, 88386, 88396, 88408, 88420, 88431, 88443, 88456, - 88467, 88476, 88485, 88493, 88502, 88512, 88520, 88529, 88538, 88546, - 88555, 88565, 88573, 88582, 88591, 88599, 88608, 88618, 88626, 88636, - 88644, 88654, 88662, 88670, 88679, 88687, 88697, 88705, 88713, 88723, - 88731, 88738, 88745, 88754, 88763, 88771, 88780, 88790, 88798, 88809, - 88817, 88825, 88832, 88840, 88849, 88856, 88868, 88879, 88891, 88902, - 88914, 88923, 88931, 88940, 88948, 88957, 88966, 88974, 88983, 88991, - 89000, 89008, 89016, 89024, 89032, 89039, 89048, 89056, 89065, 89073, - 89082, 89090, 89098, 89107, 89115, 89124, 89132, 89141, 89149, 89157, - 89165, 89174, 89182, 89191, 89199, 89208, 89216, 89225, 89233, 89241, - 89249, 89258, 89266, 89275, 89284, 89292, 89301, 89309, 89318, 89326, - 89335, 89343, 89350, 89358, 89365, 89374, 89382, 89391, 89399, 89408, - 89417, 89425, 89435, 89443, 89450, 89458, 89465, 89473, 89485, 89498, - 89507, 89517, 89526, 89536, 89545, 89555, 89564, 89574, 89583, 89593, - 89603, 89612, 89621, 89630, 89640, 89648, 89657, 89667, 89677, 89687, - 89697, 89705, 89715, 89723, 89733, 89741, 89751, 89759, 89769, 89777, - 89786, 89793, 89803, 89811, 89821, 89829, 89839, 89847, 89857, 89865, - 89874, 89882, 89891, 89899, 89908, 89917, 89926, 89935, 89945, 89953, - 89963, 89971, 89981, 89989, 89999, 90007, 90017, 90025, 90034, 90041, - 90051, 90059, 90069, 90077, 90087, 90095, 90105, 90113, 90122, 90130, - 90139, 90147, 90156, 90165, 90174, 90183, 90192, 90200, 90209, 90217, - 90226, 90235, 90243, 90253, 90262, 90272, 90282, 90291, 90301, 90310, - 90319, 90327, 90335, 90340, 90345, 90351, 90359, 90367, 90375, 90383, - 90391, 90399, 90405, 90411, 90417, 90425, 90431, 90441, 90447, 90453, - 90459, 90470, 90481, 90492, 90502, 90513, 90524, 90534, 90545, 90555, - 90565, 90574, 90585, 90596, 90607, 90620, 90630, 90640, 90651, 90661, - 90671, 90681, 90691, 90701, 90711, 90721, 90732, 90743, 90754, 90764, - 90774, 90786, 90797, 90808, 90818, 90828, 90838, 90848, 90859, 90869, - 90879, 90891, 90901, 90911, 90923, 90934, 90945, 90955, 90965, 90975, - 90985, 90997, 91009, 91021, 91032, 91043, 91053, 91063, 91073, 91082, - 91091, 91101, 91111, 91122, 0, 0, 91132, 91143, 91154, 91164, 91174, - 91186, 91197, 91208, 91221, 91231, 91243, 91252, 91261, 91272, 91283, - 91296, 91307, 91320, 91330, 91342, 91352, 91364, 91376, 91389, 91399, - 91409, 91419, 91430, 91440, 91449, 91459, 91468, 91477, 91487, 91497, - 91507, 91517, 91527, 91537, 91548, 91558, 91569, 91579, 91590, 91601, - 91611, 91621, 91631, 91641, 91651, 91661, 91672, 91682, 91693, 0, 0, 0, - 0, 0, 0, 0, 91704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91710, 91725, 91740, 91746, - 91752, 91758, 91764, 91770, 91776, 91782, 91788, 91796, 91800, 91803, - 91811, 91819, 91827, 91830, 91833, 91836, 91839, 91842, 91845, 91848, - 91851, 91854, 91857, 91860, 91863, 91866, 91869, 91872, 91875, 91883, - 91892, 91903, 91911, 91919, 91928, 91937, 91949, 91961, 0, 0, 0, 0, 0, 0, - 91971, 91976, 91981, 91988, 91995, 92001, 92007, 92012, 92017, 92022, - 92028, 92034, 92040, 92046, 92052, 92059, 92066, 92076, 92086, 92096, - 92105, 92116, 92125, 92134, 92145, 92156, 92169, 92182, 92194, 92206, - 92218, 92230, 92241, 92252, 92263, 92274, 92286, 92298, 92302, 92307, - 92317, 92327, 92331, 92335, 92339, 92344, 92349, 92354, 92359, 92362, - 92366, 0, 92371, 92374, 92377, 92381, 92385, 92390, 92394, 92398, 92404, - 92410, 92418, 92426, 92429, 92432, 92435, 92438, 92441, 92445, 92449, 0, - 92453, 92458, 92462, 92466, 0, 0, 0, 0, 92471, 92476, 92483, 92488, - 92493, 0, 92498, 92503, 92509, 92514, 92520, 92525, 92531, 92536, 92542, - 92547, 92553, 92559, 92568, 92577, 92586, 92595, 92605, 92615, 92625, - 92635, 92644, 92653, 92662, 92672, 92677, 92682, 92688, 92694, 92700, - 92707, 92715, 92723, 92729, 92735, 92741, 92748, 92754, 92760, 92766, - 92773, 92779, 92785, 92791, 92798, 92803, 92808, 92813, 92819, 92825, - 92831, 92837, 92844, 92850, 92856, 92862, 92868, 92874, 92880, 92886, - 92892, 92898, 92904, 92910, 92917, 92923, 92929, 92935, 92942, 92948, - 92954, 92960, 92967, 92973, 92979, 92985, 92992, 92998, 93004, 93010, - 93017, 93023, 93029, 93035, 93042, 93048, 93054, 93060, 93067, 93073, - 93079, 93085, 93092, 93098, 93104, 93110, 93117, 93123, 93129, 93135, - 93142, 93148, 93154, 93160, 93167, 93173, 93179, 93185, 93192, 93197, - 93202, 93207, 93213, 93219, 93225, 93231, 93238, 93244, 93250, 93256, - 93263, 93269, 93275, 93282, 93289, 93294, 93299, 93304, 93310, 93322, - 93334, 93346, 93358, 93371, 93384, 93392, 0, 0, 93400, 0, 93408, 93413, - 93418, 93422, 93427, 93432, 93436, 93440, 93445, 93450, 93454, 93458, - 93462, 93466, 93472, 93476, 93481, 93485, 93489, 93493, 93497, 93501, - 93505, 93509, 93513, 93517, 93521, 93525, 93530, 93535, 93540, 93545, - 93551, 93557, 93564, 93571, 93578, 93584, 93591, 93598, 93605, 93611, - 93618, 93625, 93631, 93638, 93645, 93651, 93658, 93665, 93671, 93678, - 93685, 93691, 93698, 93705, 93712, 93719, 93726, 93732, 93738, 93744, - 93750, 93755, 93761, 93767, 93774, 93781, 93788, 93794, 93801, 93808, - 93815, 93821, 93828, 93835, 93841, 93848, 93855, 93861, 93868, 93875, - 93881, 93888, 93895, 93901, 93908, 93915, 93922, 93929, 93936, 93943, - 93948, 93955, 93959, 93965, 93971, 93977, 93983, 93989, 93993, 93998, - 94003, 94008, 94013, 94018, 94023, 94028, 94033, 94039, 94045, 94051, - 94059, 94063, 94067, 94071, 94075, 94079, 94083, 94088, 94093, 94098, - 94103, 94107, 94112, 94117, 94122, 94127, 94132, 94137, 94142, 94147, - 94151, 94155, 94160, 94165, 94170, 94175, 94179, 94184, 94189, 94194, - 94199, 94203, 94208, 94213, 94218, 94223, 94227, 94232, 94237, 94241, - 94246, 94251, 94256, 94261, 94266, 94271, 94278, 94285, 94289, 94294, - 94299, 94304, 94309, 94314, 94319, 94324, 94329, 94334, 94339, 94344, - 94349, 94354, 94359, 94364, 94369, 94374, 94379, 94384, 94389, 94394, - 94399, 94404, 94409, 94414, 94419, 94424, 94429, 94434, 0, 0, 0, 94439, - 94443, 94448, 94452, 94457, 94462, 0, 0, 94466, 94471, 94476, 94480, - 94485, 94490, 0, 0, 94495, 94500, 94504, 94509, 94514, 94519, 0, 0, - 94524, 94529, 94534, 0, 0, 0, 94538, 94543, 94548, 94553, 94557, 94562, - 94567, 0, 94572, 94578, 94581, 94585, 94588, 94592, 94596, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 94600, 94606, 94612, 94618, 94624, 0, 0, 94628, 94634, - 94640, 94646, 94652, 94658, 94665, 94672, 94679, 94686, 94693, 94700, 0, - 94707, 94714, 94721, 94727, 94734, 94741, 94748, 94755, 94761, 94768, - 94775, 94782, 94789, 94795, 94802, 94809, 94816, 94823, 94829, 94836, - 94843, 94850, 94857, 94864, 94871, 94878, 0, 94885, 94891, 94898, 94905, - 94912, 94919, 94925, 94932, 94939, 94946, 94953, 94960, 94967, 94974, - 94980, 94987, 94994, 95001, 95008, 0, 95015, 95022, 0, 95029, 95036, - 95043, 95050, 95057, 95064, 95071, 95078, 95085, 95092, 95099, 95106, - 95113, 95120, 95127, 0, 0, 95133, 95138, 95143, 95148, 95153, 95158, - 95163, 95168, 95173, 95178, 95183, 95188, 95193, 95198, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 95203, 95210, 95217, 95224, 95231, 95238, 95245, 95252, - 95259, 95266, 95273, 95280, 95287, 95294, 95301, 95308, 95315, 95322, - 95329, 95336, 95344, 95352, 95359, 95366, 95371, 95379, 95387, 95394, - 95401, 95406, 95413, 95418, 95423, 95430, 95435, 95440, 95445, 95453, - 95458, 95463, 95470, 95475, 95480, 95487, 95494, 95499, 95504, 95509, - 95514, 95519, 95524, 95529, 95534, 95539, 95546, 95551, 95558, 95563, - 95568, 95573, 95578, 95583, 95588, 95593, 95598, 95603, 95608, 95613, - 95620, 95627, 95634, 95641, 95647, 95652, 95659, 95664, 95669, 95678, - 95685, 95694, 95701, 95706, 95711, 95719, 95724, 95729, 95734, 95739, - 95744, 95751, 95756, 95761, 95766, 95771, 95776, 95783, 95790, 95797, - 95804, 95811, 95818, 95825, 95832, 95839, 95846, 95853, 95860, 95867, - 95874, 95881, 95888, 95895, 95902, 95909, 95916, 95923, 95930, 95937, - 95944, 95951, 95958, 95965, 95972, 0, 0, 0, 0, 0, 95979, 95987, 95995, 0, - 0, 0, 0, 96000, 96004, 96008, 96012, 96016, 96020, 96024, 96028, 96032, - 96036, 96041, 96046, 96051, 96056, 96061, 96066, 96071, 96076, 96081, - 96087, 96093, 96099, 96106, 96113, 96120, 96127, 96134, 96141, 96146, - 96151, 96156, 96162, 96168, 96174, 96180, 96186, 96192, 96198, 96204, - 96210, 96216, 96222, 96228, 96234, 96240, 0, 0, 0, 96246, 96254, 96262, - 96270, 96278, 96286, 96296, 96306, 96314, 96322, 96330, 96338, 96346, - 96352, 96359, 96368, 96376, 96384, 96393, 96402, 96411, 96421, 96432, - 96442, 96453, 96462, 96471, 96480, 96490, 96501, 96511, 96522, 96533, - 96542, 96550, 96556, 96562, 96568, 96574, 96582, 96590, 96596, 96603, - 96613, 96620, 96627, 96634, 96641, 96648, 96658, 96665, 96672, 96680, - 96688, 96697, 96706, 96715, 96724, 96733, 96740, 96748, 96757, 96766, - 96770, 96777, 96782, 96787, 96791, 96795, 96799, 96803, 96808, 96813, - 96819, 96825, 96829, 96835, 96839, 96843, 96847, 96851, 96855, 96859, - 96865, 96869, 96874, 96878, 96882, 0, 96885, 96890, 96895, 96900, 96905, - 96912, 96917, 96922, 96927, 96932, 96937, 96942, 96947, 0, 0, 0, 96950, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96956, 96963, 96972, 96981, 96988, 96995, 97002, 97009, 97016, 97023, - 97029, 97036, 97043, 97050, 97057, 97064, 97071, 97078, 97085, 97094, - 97101, 97108, 97115, 97122, 97129, 97136, 97143, 97150, 97159, 97166, - 97173, 97180, 97187, 97194, 97201, 97210, 97217, 97224, 97231, 97238, - 97247, 97254, 97261, 97268, 97276, 97285, 0, 0, 97294, 97298, 97302, - 97307, 97312, 97317, 97322, 97326, 97331, 97336, 97341, 97346, 97351, - 97356, 97360, 97365, 97370, 97375, 97380, 97384, 97389, 97394, 97398, - 97403, 97408, 97413, 97418, 97423, 97428, 0, 0, 0, 97433, 97437, 97442, - 97447, 97451, 97456, 97460, 97465, 97470, 97475, 97480, 97485, 97489, - 97494, 97499, 97504, 97509, 97514, 97519, 97523, 97528, 97533, 97538, - 97543, 97548, 97553, 97557, 97561, 97566, 97571, 97576, 97581, 97586, - 97591, 97596, 97601, 97606, 97611, 97616, 97621, 97626, 97631, 97636, - 97641, 97646, 97651, 97656, 97661, 97666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 97671, 97677, 97682, 97687, 97692, 97697, 97702, 97707, - 97712, 97717, 97722, 97728, 97734, 97740, 97746, 97752, 97758, 97764, - 97770, 97776, 97783, 97790, 97797, 97805, 97813, 97821, 97829, 97837, 0, - 0, 0, 0, 97845, 97849, 97854, 97859, 97864, 97868, 97873, 97878, 97883, - 97888, 97892, 97896, 97901, 97906, 97911, 97916, 97920, 97925, 97930, - 97935, 97940, 97945, 97950, 97954, 97959, 97964, 97969, 97974, 97979, - 97984, 97989, 97994, 97999, 98004, 98009, 98015, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 98021, 98026, 98033, 98040, 98045, 98050, 98055, 98060, 98065, 98070, - 98075, 98080, 98085, 98090, 98095, 98100, 98105, 98110, 98115, 98120, - 98125, 98130, 98135, 98140, 98145, 98150, 98155, 98160, 98165, 98170, 0, - 0, 0, 0, 0, 98177, 98183, 98189, 98195, 98201, 98206, 98212, 98218, - 98224, 98230, 98235, 98241, 98247, 98253, 98259, 98265, 98271, 98277, - 98283, 98289, 98294, 98300, 98306, 98312, 98318, 98324, 98329, 98335, - 98341, 98346, 98352, 98358, 98364, 98370, 98376, 98382, 98388, 98393, - 98399, 98406, 98413, 98420, 98427, 0, 0, 0, 0, 0, 98434, 98439, 98444, - 98449, 98454, 98459, 98464, 98469, 98474, 98479, 98484, 98489, 98494, - 98499, 98504, 98509, 98514, 98519, 98524, 98529, 98534, 98539, 98544, - 98549, 98554, 98559, 98564, 98568, 98572, 98576, 0, 98581, 98587, 98592, - 98597, 98602, 98607, 98613, 98619, 98625, 98631, 98637, 98643, 98649, - 98655, 98661, 98667, 98673, 98679, 98685, 98690, 98696, 98702, 98707, - 98713, 98718, 98724, 98730, 98735, 98741, 98747, 98752, 98758, 98763, - 98768, 98774, 98780, 98786, 0, 0, 0, 0, 98791, 98797, 98803, 98809, - 98815, 98821, 98827, 98833, 98839, 98846, 98851, 98856, 98862, 98868, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98874, 98880, 98886, - 98892, 98899, 98905, 98912, 98919, 98926, 98933, 98941, 98948, 98956, - 98962, 98968, 98974, 98980, 98986, 98992, 98998, 99004, 99010, 99016, - 99022, 99028, 99034, 99040, 99046, 99052, 99058, 99064, 99070, 99076, - 99082, 99088, 99094, 99100, 99106, 99112, 99118, 99124, 99130, 99136, - 99142, 99149, 99155, 99162, 99169, 99176, 99183, 99191, 99198, 99206, - 99212, 99218, 99224, 99230, 99236, 99242, 99248, 99254, 99260, 99266, - 99272, 99278, 99284, 99290, 99296, 99302, 99308, 99314, 99320, 99326, - 99332, 99338, 99344, 99350, 99356, 99362, 99368, 99374, 99379, 99384, - 99389, 99394, 99399, 99404, 99409, 99414, 99419, 99424, 99429, 99434, - 99439, 99444, 99449, 99454, 99459, 99464, 99469, 99474, 99479, 99484, - 99489, 99494, 99499, 99504, 99509, 99514, 99519, 99524, 99529, 99534, - 99539, 99544, 99549, 99554, 99559, 99564, 99569, 99574, 99579, 99584, - 99589, 99594, 99599, 99604, 99609, 99614, 99619, 99624, 99629, 99634, - 99639, 99644, 99649, 99653, 99657, 99662, 99667, 99672, 99677, 99682, - 99687, 99692, 99697, 99702, 99707, 99712, 99716, 99720, 99724, 99728, - 99732, 99736, 99740, 99745, 99750, 0, 0, 99755, 99760, 99764, 99768, - 99772, 99776, 99780, 99784, 99788, 99792, 0, 0, 0, 0, 0, 0, 99796, 99801, - 99807, 99813, 99819, 99825, 99831, 99837, 99842, 99848, 99853, 99859, - 99864, 99869, 99875, 99881, 99886, 99891, 99896, 99901, 99907, 99912, - 99918, 99923, 99929, 99935, 99941, 99947, 99953, 99959, 99965, 99970, - 99976, 99982, 99988, 99994, 0, 0, 0, 0, 100000, 100005, 100011, 100017, - 100023, 100029, 100035, 100041, 100046, 100052, 100057, 100063, 100068, - 100073, 100079, 100085, 100090, 100095, 100100, 100105, 100111, 100116, - 100122, 100127, 100133, 100139, 100145, 100151, 100157, 100163, 100169, - 100174, 100180, 100186, 100192, 100198, 0, 0, 0, 0, 100204, 100208, - 100213, 100218, 100223, 100228, 100233, 100238, 100243, 100247, 100252, - 100257, 100262, 100267, 100271, 100276, 100281, 100286, 100291, 100296, - 100301, 100305, 100310, 100314, 100319, 100324, 100329, 100334, 100339, - 100344, 100349, 100354, 100358, 100363, 100368, 100373, 100378, 100383, - 100388, 100393, 0, 0, 0, 0, 0, 0, 0, 0, 100398, 100405, 100412, 100419, - 100426, 100433, 100440, 100447, 100454, 100461, 100468, 100475, 100482, - 100489, 100496, 100503, 100510, 100517, 100524, 100531, 100538, 100545, - 100552, 100559, 100566, 100573, 100580, 100587, 100594, 100601, 100608, - 100615, 100622, 100629, 100636, 100643, 100650, 100657, 100664, 100671, - 100678, 100685, 100692, 100699, 100706, 100713, 100720, 100727, 100734, - 100741, 100748, 100755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100762, 100769, - 100774, 100780, 100786, 100792, 100798, 100804, 100810, 100816, 100821, - 100827, 0, 100833, 100838, 100844, 100849, 100855, 100861, 100866, - 100871, 100877, 100883, 100889, 100895, 100900, 100906, 100912, 0, - 100918, 100924, 100930, 100936, 100942, 100947, 100953, 0, 100959, - 100965, 0, 100971, 100976, 100982, 100988, 100994, 101000, 101006, - 101012, 101018, 101023, 101029, 0, 101035, 101040, 101046, 101051, - 101057, 101063, 101068, 101073, 101079, 101085, 101091, 101097, 101102, - 101108, 101114, 0, 101120, 101126, 101132, 101138, 101144, 101149, - 101155, 0, 101161, 101167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 101173, 101178, 101183, 101188, 101193, 101198, 101203, - 101208, 101213, 101218, 101223, 101228, 101233, 101238, 101243, 101248, - 101253, 101258, 101263, 101268, 101273, 101278, 101283, 101288, 101293, - 101298, 101303, 101308, 101313, 101318, 101323, 101328, 101333, 101338, - 101343, 101348, 101353, 101358, 101363, 101368, 101373, 101378, 101383, - 101388, 101393, 101398, 101403, 101408, 101413, 101418, 101423, 101428, - 101433, 101438, 101443, 101448, 101453, 101458, 101463, 101468, 101473, - 101478, 101483, 101488, 101493, 101498, 101503, 101508, 101513, 101518, - 101523, 101528, 101533, 101538, 101543, 101548, 101553, 101558, 101563, - 101568, 101573, 101578, 101583, 101588, 101593, 101598, 101603, 101608, - 101613, 101618, 101623, 101628, 101633, 101638, 101643, 101648, 101653, - 101658, 101663, 101668, 101673, 101678, 101683, 101688, 101693, 101698, - 101703, 101708, 101713, 101718, 101723, 101728, 101733, 101738, 101743, - 101748, 101753, 101758, 101763, 101768, 101773, 101778, 101783, 101788, - 101793, 101798, 101803, 101808, 101813, 101818, 101823, 101828, 101833, - 101838, 101843, 101848, 101853, 101858, 101863, 101868, 101873, 101878, - 101883, 101888, 101893, 101898, 101903, 101908, 101913, 101918, 101923, - 101928, 101933, 101938, 101943, 101948, 101953, 101958, 101963, 101968, - 101973, 101978, 101983, 101988, 101993, 101998, 102003, 102008, 102013, - 102018, 102023, 102028, 102033, 102038, 102043, 102048, 102053, 102058, - 102063, 102068, 102073, 102078, 102083, 102088, 102093, 102098, 102103, - 102108, 102113, 102118, 102123, 102128, 102133, 102138, 102143, 102148, - 102153, 102158, 102163, 102168, 102173, 102178, 102183, 102188, 102193, - 102198, 102203, 102208, 102213, 102218, 102223, 102228, 102233, 102238, - 102243, 102248, 102253, 102258, 102263, 102268, 102273, 102278, 102283, - 102288, 102293, 102298, 102303, 102308, 102313, 102318, 102323, 102328, - 102333, 102338, 102343, 102348, 102353, 102358, 102363, 102368, 102373, - 102378, 102383, 102388, 102393, 102398, 102403, 102408, 102413, 102418, - 102423, 102428, 102433, 102438, 102443, 102448, 102453, 102458, 102463, - 102468, 102473, 102478, 102483, 102488, 102493, 102498, 102503, 102508, - 102513, 102518, 102523, 102528, 102533, 102538, 102543, 102548, 102553, - 102558, 102563, 102568, 102573, 102578, 102583, 102588, 102593, 102598, - 102603, 102608, 102613, 102618, 102623, 102628, 102633, 102638, 102643, - 102648, 102653, 102658, 102663, 102668, 102673, 102678, 102683, 102688, - 102693, 102698, 102703, 102708, 102713, 102718, 102723, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 102728, 102734, 102741, 102748, 102754, 102761, 102768, 102775, - 102782, 102788, 102795, 102802, 102809, 102816, 102823, 102830, 102837, - 102844, 102851, 102858, 102865, 102872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 102879, 102884, 102889, 102894, 102899, 102904, 102909, 102914, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102919, - 102925, 102933, 102942, 102947, 102953, 0, 102959, 102966, 102977, - 102987, 102994, 103002, 103009, 103020, 103026, 103036, 103043, 103050, - 103056, 103063, 103071, 103078, 103084, 103091, 103103, 103110, 103117, - 103125, 103134, 103147, 103152, 103161, 103167, 103176, 103182, 103188, - 103195, 103200, 103210, 103224, 103232, 103240, 103245, 103255, 103262, - 103273, 103280, 103289, 0, 103297, 103303, 103311, 103321, 103327, - 103333, 103339, 103345, 103355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 103363, 103367, 103371, 103375, 103379, 103383, 0, - 0, 103388, 0, 103393, 103397, 103402, 103407, 103412, 103417, 103421, - 103426, 103431, 103436, 103441, 103445, 103450, 103455, 103460, 103465, - 103469, 103474, 103479, 103484, 103489, 103493, 103498, 103503, 103508, - 103513, 103517, 103522, 103527, 103532, 103537, 103541, 103546, 103551, - 103556, 103561, 103566, 103571, 103576, 103580, 103585, 103590, 103595, - 103600, 0, 103605, 103610, 0, 0, 0, 103615, 0, 0, 103620, 103625, 103632, - 103639, 103646, 103653, 103660, 103667, 103674, 103681, 103688, 103695, - 103702, 103709, 103716, 103723, 103730, 103737, 103744, 103751, 103758, - 103765, 103772, 0, 103779, 103786, 103792, 103798, 103804, 103811, - 103818, 103826, 103833, 103841, 103846, 103851, 103856, 103861, 103866, - 103871, 103876, 103881, 103886, 103891, 103896, 103901, 103906, 103912, - 103917, 103922, 103927, 103932, 103937, 103942, 103947, 103952, 103957, - 103963, 103969, 103973, 103977, 103981, 103985, 103989, 103994, 103999, - 104005, 104010, 104016, 104021, 104026, 104031, 104037, 104042, 104047, - 104052, 104057, 104062, 104068, 104073, 104079, 104084, 104090, 104095, - 104101, 104106, 104112, 104117, 104122, 104127, 104132, 104137, 104142, - 104147, 104153, 104158, 0, 0, 0, 0, 0, 0, 0, 0, 104163, 104167, 104171, - 104175, 104179, 104185, 104189, 104194, 104199, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104205, 104210, 104215, - 104220, 104225, 104230, 104235, 104240, 104245, 104250, 104255, 104260, - 104265, 104270, 104275, 104280, 104285, 104290, 104295, 0, 104300, - 104305, 0, 0, 0, 0, 0, 104310, 104314, 104318, 104323, 104328, 104334, - 104339, 104344, 104349, 104354, 104359, 104364, 104369, 104374, 104379, - 104384, 104389, 104394, 104399, 104404, 104409, 104414, 104419, 104424, - 104429, 104434, 104439, 104444, 104448, 104453, 104458, 104464, 104468, - 0, 0, 0, 104472, 104478, 104482, 104487, 104492, 104497, 104501, 104506, - 104510, 104515, 104520, 104524, 104529, 104534, 104538, 104542, 104547, - 104552, 104556, 104561, 104566, 104571, 104576, 104581, 104586, 104591, - 104596, 0, 0, 0, 0, 0, 104601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 104606, 104611, 104616, 104621, 104626, 104631, 104637, 104643, - 104649, 104654, 104659, 104664, 104670, 104676, 104682, 104687, 104693, - 104698, 104704, 104710, 104715, 104721, 104727, 104732, 104738, 104744, - 104750, 104756, 104762, 104767, 104773, 104779, 104785, 104790, 104795, - 104800, 104805, 104810, 104816, 104822, 104827, 104832, 104837, 104843, - 104848, 104853, 104859, 104865, 104870, 104877, 104883, 104888, 104894, - 104900, 104906, 104911, 0, 0, 0, 0, 104917, 104926, 104934, 104941, - 104948, 104953, 104958, 104963, 104968, 104973, 104978, 104983, 104988, - 104993, 104999, 105005, 105011, 105017, 105023, 105029, 0, 0, 105035, - 105042, 105049, 105056, 105064, 105072, 105080, 105088, 105096, 105104, - 105110, 105116, 105122, 105129, 105136, 105143, 105150, 105157, 105164, - 105171, 105178, 105185, 105192, 105199, 105206, 105213, 105220, 105227, - 105235, 105243, 105251, 105260, 105269, 105278, 105287, 105296, 105305, - 105313, 105321, 105329, 105338, 105347, 105356, 105365, 105374, 105383, - 105392, 105396, 105401, 105406, 0, 105412, 105417, 0, 0, 0, 0, 0, 105422, - 105428, 105435, 105440, 105445, 105449, 105454, 105459, 0, 105464, - 105469, 105474, 0, 105479, 105484, 105489, 105494, 105499, 105504, - 105509, 105514, 105519, 105524, 105529, 105533, 105537, 105542, 105547, - 105552, 105556, 105560, 105564, 105568, 105573, 105578, 105583, 105587, - 105592, 105596, 105601, 105606, 105611, 0, 0, 105616, 105622, 105627, 0, - 0, 0, 0, 105632, 105636, 105640, 105644, 105648, 105652, 105657, 105662, - 105668, 105673, 0, 0, 0, 0, 0, 0, 0, 105680, 105686, 105693, 105699, - 105706, 105712, 105718, 105724, 105731, 0, 0, 0, 0, 0, 0, 0, 105737, - 105745, 105753, 105761, 105769, 105777, 105785, 105793, 105801, 105809, - 105817, 105825, 105833, 105841, 105849, 105857, 105865, 105873, 105881, - 105889, 105897, 105905, 105913, 105921, 105929, 105937, 105945, 105953, - 105961, 105969, 105976, 105984, 105992, 105999, 106006, 106013, 106020, - 106027, 106034, 106041, 106048, 106055, 106062, 106069, 106076, 106083, - 106090, 106097, 106104, 106111, 106118, 106125, 106132, 106139, 106146, - 106153, 106160, 106167, 106174, 106181, 106188, 106195, 106201, 106208, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 106215, 106220, 106225, 106230, 106235, 106240, - 106245, 106250, 106255, 106260, 106265, 106270, 106275, 106280, 106285, - 106290, 106295, 106300, 106305, 106310, 106315, 106320, 106325, 106330, - 106335, 106340, 106345, 106350, 106355, 106360, 106365, 106370, 106375, - 106380, 106385, 106390, 106395, 106400, 106406, 0, 0, 0, 0, 106412, - 106416, 106420, 106425, 106430, 106436, 106442, 106448, 106458, 106467, - 106473, 106480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106488, 106492, 106497, - 106502, 106507, 106512, 106517, 106522, 106527, 106531, 106536, 106540, - 106545, 106549, 106554, 106558, 106563, 106568, 106573, 106578, 106583, - 106588, 106593, 106598, 106603, 106608, 106613, 106618, 106623, 106628, - 106633, 106638, 106643, 106648, 106653, 106658, 106663, 106668, 106673, - 106678, 106683, 106688, 106693, 106698, 106703, 106708, 106713, 106718, - 106723, 106728, 106733, 106738, 106743, 106748, 0, 0, 0, 106753, 106758, - 106767, 106775, 106784, 106793, 106804, 106815, 106822, 106829, 106836, - 106843, 106850, 106857, 106864, 106871, 106878, 106885, 106892, 106899, - 106906, 106913, 106920, 106927, 106934, 106941, 106948, 106955, 106962, - 0, 0, 106969, 106975, 106981, 106987, 106993, 107000, 107007, 107015, - 107022, 107029, 107036, 107043, 107050, 107057, 107064, 107071, 107078, - 107085, 107092, 107099, 107106, 107113, 107120, 107127, 107134, 107141, - 107148, 0, 0, 0, 0, 0, 107155, 107161, 107167, 107173, 107179, 107186, - 107193, 107201, 107208, 107215, 107222, 107229, 107236, 107243, 107250, - 107257, 107264, 107271, 107278, 107285, 107292, 107299, 107306, 107313, - 107320, 107327, 0, 0, 0, 0, 0, 0, 0, 107334, 107341, 107349, 107359, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107369, 107375, 107381, 107387, 107393, - 107400, 107407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107415, 107422, 107429, 107437, - 107444, 107451, 107458, 107465, 107473, 107481, 107489, 107497, 107505, - 107513, 107521, 107529, 107537, 107545, 107553, 107561, 107569, 107577, - 107585, 107593, 107601, 107609, 107617, 107625, 107633, 107641, 107649, - 107657, 107665, 107673, 107681, 107689, 107697, 107705, 107713, 107721, - 107729, 107737, 107745, 107753, 107761, 107769, 107777, 107785, 107793, - 107801, 107809, 107817, 107825, 107833, 107841, 107849, 107857, 107865, - 107873, 107881, 107889, 107897, 107905, 107913, 107921, 107929, 107937, - 107945, 107953, 107961, 107969, 107977, 107985, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 107993, 107998, 108004, 108010, 108016, 108022, 108028, 108034, 108040, - 108046, 108051, 108058, 108064, 108070, 108076, 108082, 108088, 108093, - 108099, 108105, 108111, 108117, 108123, 108129, 108135, 108141, 108147, - 108153, 108158, 108164, 108172, 108180, 108186, 108192, 108198, 108204, - 108212, 108218, 108224, 108230, 108236, 108242, 108248, 108253, 108259, - 108267, 108275, 108281, 108287, 108293, 108300, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 108306, 108311, 108317, 108323, 108329, 108335, 108341, - 108347, 108353, 108359, 108364, 108371, 108377, 108383, 108389, 108395, - 108401, 108406, 108412, 108418, 108424, 108430, 108436, 108442, 108448, - 108454, 108460, 108466, 108471, 108477, 108485, 108493, 108499, 108505, - 108511, 108517, 108525, 108531, 108537, 108543, 108549, 108555, 108561, - 108566, 108572, 108580, 108588, 108594, 108600, 108606, 108613, 0, 0, 0, - 0, 0, 0, 0, 108619, 108623, 108627, 108632, 108637, 108643, 108648, - 108654, 108661, 108667, 108674, 108681, 108688, 108695, 108701, 108708, - 108715, 108722, 108729, 108735, 108742, 108749, 108755, 108762, 108768, - 108775, 108781, 108787, 108793, 108800, 108809, 108815, 108823, 108830, - 108837, 108844, 108850, 108856, 108862, 108868, 108874, 108881, 108890, - 108897, 108904, 108911, 0, 0, 0, 0, 0, 0, 0, 0, 108918, 108925, 108931, - 108937, 108943, 108949, 108955, 108961, 108967, 108973, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108979, 108983, 108987, - 108991, 108995, 108999, 109003, 109007, 109011, 109015, 109020, 109025, - 109030, 109035, 109040, 109045, 109050, 109055, 109060, 109066, 109072, - 109078, 109085, 109092, 109099, 109106, 109113, 109120, 109127, 109134, - 109141, 0, 109148, 109153, 109158, 109163, 109168, 109173, 109178, - 109183, 109188, 109193, 109198, 109203, 109208, 109213, 109217, 109222, - 109227, 109232, 109237, 109242, 109247, 109252, 109257, 109262, 109267, - 109272, 109277, 109282, 109290, 109295, 109300, 109305, 109310, 109315, - 109320, 109325, 109330, 109335, 109340, 109345, 109350, 109355, 0, - 109360, 109366, 109372, 0, 0, 109377, 109385, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109394, 109401, - 109408, 109415, 109421, 109428, 109434, 109441, 109447, 109453, 109460, - 109466, 109472, 109478, 109484, 109490, 109496, 109502, 109508, 109515, - 109526, 109532, 109538, 109546, 109552, 109558, 109565, 109576, 109582, - 109588, 109594, 109601, 109612, 109617, 109622, 109627, 109632, 109637, - 109643, 109649, 109655, 109662, 109670, 0, 0, 0, 0, 0, 0, 0, 0, 109676, - 109681, 109686, 109691, 109696, 109701, 109706, 109711, 109716, 109721, - 109726, 109731, 109736, 109741, 109746, 109751, 109756, 109761, 109766, - 109771, 109776, 109781, 109787, 109792, 109798, 109803, 109809, 109815, - 109821, 109827, 109833, 109840, 109846, 109852, 109856, 109861, 109866, - 109872, 109880, 109891, 109900, 109910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109920, 109926, 109932, 109938, 109944, - 109950, 109957, 109963, 109969, 109975, 109981, 109987, 109993, 109999, - 110005, 110011, 110017, 110023, 110029, 110035, 110041, 110048, 110055, - 110061, 110069, 110077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110086, - 110091, 110097, 110102, 110107, 110112, 110117, 110122, 110129, 110134, - 110139, 110144, 110149, 110154, 110159, 110164, 110169, 110174, 110179, - 110184, 110189, 110194, 110198, 110202, 110206, 110210, 110215, 110220, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110226, - 110231, 110236, 110241, 110246, 110251, 110256, 110261, 110266, 110271, - 110276, 110281, 110286, 110291, 110296, 110301, 110306, 110311, 110316, - 110321, 110326, 110331, 110336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110341, - 110345, 110349, 110353, 110357, 110361, 110364, 110368, 110371, 110375, - 110378, 110382, 110386, 110391, 110395, 110400, 110403, 110407, 110410, - 110414, 110417, 110421, 110425, 110429, 110433, 110437, 110441, 110445, - 110449, 110453, 110457, 110461, 110465, 110469, 110473, 110477, 110481, - 110485, 110489, 110492, 110495, 110499, 110503, 110507, 110510, 110513, - 110516, 110519, 110523, 110527, 110531, 110534, 110537, 110541, 110547, - 110553, 110559, 110564, 110571, 110575, 110580, 110584, 110589, 110594, - 110600, 110605, 110611, 110615, 110620, 110624, 110629, 110632, 110635, - 110639, 110644, 110650, 110655, 110661, 0, 0, 0, 0, 110666, 110669, - 110672, 110675, 110678, 110681, 110684, 110687, 110690, 110693, 110697, - 110701, 110705, 110709, 110713, 110717, 110721, 110725, 110729, 110734, - 110738, 110742, 110745, 110748, 110751, 110754, 110757, 110760, 110763, - 110766, 110769, 110775, 110782, 110789, 110797, 110805, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 110811, 110815, 110820, 110825, 110830, 110834, 110839, 110843, - 110848, 110852, 110857, 110861, 110866, 110870, 110875, 110879, 110884, - 110889, 110894, 110899, 110904, 110909, 110914, 110919, 110924, 110929, - 110934, 110939, 110944, 110949, 110954, 110959, 110964, 110969, 110974, - 110979, 110983, 110987, 110992, 110997, 111002, 111006, 111010, 111014, - 111018, 111023, 111028, 111033, 111037, 111041, 111047, 111052, 111058, - 111063, 111069, 111074, 111080, 111085, 111091, 111096, 111101, 111106, - 111111, 111115, 111120, 111126, 111130, 111135, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 111141, 0, 0, 111146, 111153, 111160, 111167, 111174, 111181, - 111188, 111195, 111202, 111209, 111216, 111223, 111230, 111237, 111244, - 111251, 111258, 111265, 111272, 111279, 111286, 111293, 111300, 111307, - 111314, 0, 0, 0, 0, 0, 0, 0, 111321, 111328, 111334, 111340, 111346, - 111352, 111358, 111364, 111370, 111376, 0, 0, 0, 0, 0, 0, 111382, 111387, - 111392, 111397, 111402, 111406, 111410, 111414, 111419, 111424, 111429, - 111434, 111439, 111444, 111449, 111454, 111459, 111464, 111469, 111474, - 111479, 111484, 111489, 111494, 111499, 111504, 111509, 111514, 111519, - 111524, 111529, 111534, 111539, 111544, 111549, 111554, 111559, 111564, - 111569, 111574, 111579, 111584, 111590, 111595, 111601, 111606, 111612, - 111617, 111623, 111629, 111633, 111638, 111642, 0, 111646, 111651, - 111655, 111659, 111663, 111667, 111671, 111675, 111679, 111683, 111687, - 111692, 111696, 111701, 111706, 111711, 111717, 111723, 0, 0, 0, 0, 0, 0, - 0, 0, 111728, 111732, 111736, 111740, 111744, 111748, 111752, 111757, - 111762, 111767, 111772, 111777, 111782, 111787, 111792, 111797, 111802, - 111807, 111812, 111817, 111822, 111827, 111832, 111837, 111841, 111845, - 111850, 111855, 111860, 111864, 111868, 111872, 111877, 111881, 111885, - 111890, 111895, 111900, 111905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111910, - 111915, 111920, 111925, 111929, 111934, 111938, 111943, 111947, 111952, - 111957, 111963, 111968, 111974, 111978, 111983, 111987, 111992, 111996, - 112001, 112006, 112011, 112016, 112021, 112026, 112031, 112036, 112041, - 112046, 112051, 112056, 112061, 112066, 112071, 112076, 112081, 112086, - 112090, 112094, 112099, 112104, 112109, 112113, 112117, 112121, 112125, - 112130, 112135, 112140, 112145, 112149, 112153, 112159, 112164, 112170, - 112175, 112181, 112187, 112194, 112200, 112207, 112212, 112218, 112223, - 112229, 112234, 112239, 112244, 112249, 112253, 112257, 112262, 112267, - 112271, 112276, 112281, 112286, 112294, 112299, 112306, 112313, 112318, - 112322, 112326, 112330, 112334, 112338, 112342, 112346, 112350, 112354, - 112358, 112363, 112367, 112372, 112378, 0, 112384, 112389, 112394, - 112399, 112404, 112409, 112414, 112419, 112424, 112429, 112435, 112441, - 112447, 112453, 112459, 112465, 112471, 112477, 112483, 112490, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 112496, 112500, 112505, 112509, 112513, 112517, - 112522, 112526, 112531, 112535, 112540, 112545, 112550, 112555, 112560, - 112565, 112570, 112575, 0, 112580, 112585, 112590, 112595, 112600, - 112605, 112610, 112615, 112620, 112625, 112630, 112635, 112639, 112643, - 112648, 112653, 112658, 112663, 112667, 112671, 112675, 112679, 112684, - 112688, 112692, 112697, 112703, 112708, 112714, 112719, 112724, 112730, - 112735, 112741, 112746, 112751, 112756, 112761, 112765, 112770, 112776, - 112781, 112787, 112792, 112797, 112802, 112808, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 112814, 112818, 112822, 112826, 112830, 112834, 112839, - 0, 112844, 0, 112849, 112854, 112859, 112864, 0, 112869, 112874, 112879, - 112884, 112889, 112894, 112899, 112904, 112909, 112914, 112919, 112924, - 112928, 112932, 112937, 0, 112942, 112947, 112951, 112955, 112959, - 112963, 112968, 112972, 112976, 112981, 112986, 0, 0, 0, 0, 0, 0, 112991, - 112995, 113000, 113004, 113009, 113013, 113018, 113022, 113027, 113031, - 113036, 113040, 113045, 113050, 113055, 113060, 113065, 113070, 113075, - 113080, 113085, 113090, 113095, 113100, 113105, 113110, 113115, 113120, - 113125, 113130, 113135, 113140, 113145, 113150, 113154, 113158, 113163, - 113168, 113173, 113178, 113182, 113186, 113190, 113194, 113199, 113204, - 113208, 113212, 113217, 113223, 113228, 113234, 113239, 113245, 113250, - 113256, 113261, 113267, 113272, 0, 0, 0, 0, 0, 113277, 113282, 113286, - 113290, 113294, 113298, 113302, 113306, 113310, 113314, 0, 0, 0, 0, 0, 0, - 113318, 113325, 113330, 113335, 0, 113340, 113344, 113349, 113353, - 113358, 113362, 113367, 113372, 0, 0, 113377, 113382, 0, 0, 113387, - 113392, 113397, 113401, 113406, 113411, 113416, 113421, 113426, 113431, - 113436, 113441, 113446, 113451, 113456, 113461, 113466, 113471, 113476, - 113481, 113486, 113491, 0, 113495, 113499, 113504, 113509, 113514, - 113518, 113522, 0, 113526, 113530, 0, 113535, 113540, 113545, 113550, - 113554, 0, 113558, 113562, 113567, 113572, 113578, 113583, 113589, - 113594, 113600, 113606, 0, 0, 113613, 113619, 0, 0, 113625, 113631, - 113637, 0, 0, 113642, 0, 0, 0, 0, 0, 0, 113646, 0, 0, 0, 0, 0, 113653, - 113658, 113665, 113673, 113679, 113685, 113691, 0, 0, 113698, 113704, - 113709, 113714, 113719, 113724, 113729, 0, 0, 0, 113734, 113739, 113744, - 113749, 113755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113760, 113764, 113769, - 113773, 113778, 113782, 113787, 113792, 113798, 113803, 113809, 113813, - 113818, 113822, 113827, 113831, 113836, 113841, 113846, 113851, 113856, - 113861, 113866, 113871, 113876, 113881, 113886, 113891, 113896, 113901, - 113906, 113911, 113916, 113921, 113926, 113931, 113935, 113940, 113944, - 113949, 113954, 113959, 113963, 113968, 113972, 113976, 113981, 113985, - 113990, 113995, 114000, 114005, 114009, 114013, 114019, 114024, 114030, - 114035, 114041, 114047, 114054, 114060, 114067, 114072, 114078, 114083, - 114089, 114094, 114099, 114104, 114109, 114114, 114119, 114125, 114129, - 114133, 114137, 114142, 114146, 114152, 114157, 114162, 114166, 114170, - 114174, 114178, 114182, 114186, 114190, 114194, 114198, 114203, 0, - 114208, 114213, 114218, 114225, 114230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114235, 114239, - 114243, 114248, 114252, 114257, 114261, 114266, 114271, 114277, 114282, - 114288, 114292, 114297, 114301, 114306, 114310, 114315, 114320, 114325, - 114330, 114335, 114340, 114345, 114350, 114355, 114360, 114365, 114370, - 114375, 114380, 114385, 114390, 114395, 114400, 114404, 114408, 114413, - 114418, 114423, 114427, 114431, 114435, 114439, 114444, 114449, 114454, - 114458, 114462, 114468, 114473, 114479, 114484, 114490, 114496, 114503, - 114509, 114516, 114521, 114528, 114534, 114539, 114546, 114552, 114557, - 114562, 114567, 114572, 114577, 114582, 114586, 114591, 0, 0, 0, 0, 0, 0, - 0, 0, 114595, 114600, 114604, 114608, 114612, 114616, 114620, 114624, - 114628, 114632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114636, 114640, - 114645, 114649, 114654, 114658, 114663, 114668, 114674, 114679, 114685, - 114689, 114694, 114698, 114703, 114707, 114712, 114717, 114722, 114727, - 114732, 114737, 114742, 114747, 114752, 114757, 114762, 114767, 114772, - 114777, 114782, 114787, 114792, 114797, 114801, 114805, 114810, 114815, - 114820, 114824, 114828, 114832, 114836, 114841, 114846, 114851, 114855, - 114859, 114865, 114870, 114876, 114881, 114887, 114893, 0, 0, 114900, - 114905, 114911, 114916, 114922, 114927, 114932, 114937, 114942, 114947, - 114952, 114956, 114961, 114967, 114972, 114978, 114984, 114990, 114998, - 115011, 115024, 115037, 115051, 115066, 115074, 115085, 115094, 115104, - 115114, 115124, 115135, 115147, 115160, 115168, 115176, 115185, 115191, - 115198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115206, 115210, 115215, 115219, - 115224, 115228, 115233, 115238, 115244, 115249, 115255, 115259, 115264, - 115268, 115273, 115277, 115282, 115287, 115292, 115297, 115302, 115307, - 115312, 115317, 115322, 115327, 115332, 115337, 115342, 115347, 115352, - 115357, 115362, 115367, 115371, 115375, 115380, 115385, 115390, 115394, - 115398, 115402, 115406, 115411, 115416, 115421, 115425, 115429, 115434, - 115440, 115445, 115451, 115456, 115462, 115468, 115475, 115481, 115488, - 115493, 115499, 115504, 115510, 115515, 115520, 115525, 115530, 115534, - 115539, 115544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115549, 115554, 115558, - 115562, 115566, 115570, 115574, 115578, 115582, 115586, 0, 0, 0, 0, 0, 0, - 115590, 115596, 115601, 115608, 115616, 115623, 115631, 115640, 115645, - 115654, 115659, 115667, 115676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 115686, 115690, 115695, 115699, 115704, 115708, 115713, - 115717, 115722, 115726, 115731, 115735, 115740, 115745, 115750, 115755, - 115760, 115765, 115770, 115775, 115780, 115785, 115790, 115795, 115800, - 115805, 115810, 115815, 115820, 115825, 115829, 115833, 115838, 115843, - 115848, 115852, 115856, 115860, 115864, 115869, 115874, 115878, 115882, - 115887, 115892, 115897, 115903, 115908, 115914, 115919, 115925, 115930, - 115936, 115941, 115947, 115952, 115957, 115964, 0, 0, 0, 0, 0, 0, 115969, - 115974, 115978, 115982, 115986, 115990, 115994, 115998, 116002, 116006, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116010, 116014, 116019, 116024, 116028, 116033, 116040, - 116044, 116049, 116054, 116058, 116063, 116068, 116073, 116077, 116081, - 116085, 116090, 116094, 116098, 116103, 116108, 116113, 116120, 116125, - 116130, 116135, 0, 0, 116142, 116149, 116156, 116165, 116170, 116176, - 116181, 116187, 116192, 116198, 116203, 116209, 116214, 116220, 116226, - 0, 0, 0, 0, 116231, 116236, 116240, 116244, 116248, 116252, 116256, - 116260, 116264, 116268, 116272, 116277, 116282, 116288, 116293, 116298, - 116303, 116308, 116313, 116318, 116323, 116328, 116333, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 116338, 116342, 116347, 116351, 116356, 116360, 116365, 116369, - 116374, 116378, 116383, 116387, 116392, 116397, 116402, 116407, 116412, - 116417, 116422, 116427, 116432, 116437, 116442, 116447, 116452, 116457, - 116462, 116467, 116472, 116477, 116481, 116485, 116490, 116495, 116500, - 116504, 116508, 116512, 116516, 116521, 116526, 116531, 116535, 116539, - 116544, 116550, 116555, 116561, 116566, 116572, 116578, 116585, 116590, - 116596, 116601, 116607, 116612, 116617, 116622, 116627, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116632, - 116640, 116647, 116655, 116663, 116670, 116678, 116686, 116694, 116701, - 116708, 116716, 116724, 116732, 116740, 116748, 116756, 116764, 116772, - 116780, 116788, 116796, 116804, 116812, 116820, 116828, 116836, 116844, - 116852, 116860, 116868, 116876, 116884, 116892, 116899, 116907, 116915, - 116922, 116930, 116938, 116946, 116953, 116960, 116968, 116976, 116984, - 116992, 117000, 117008, 117016, 117024, 117032, 117040, 117048, 117056, - 117064, 117072, 117080, 117088, 117096, 117104, 117112, 117120, 117128, - 117136, 117143, 117149, 117155, 117161, 117167, 117173, 117179, 117185, - 117191, 117197, 117204, 117211, 117218, 117225, 117232, 117239, 117246, - 117253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117260, 117266, 117272, - 117279, 117285, 117292, 117298, 117305, 0, 0, 117311, 0, 0, 117317, - 117323, 117330, 117337, 117344, 117351, 117358, 117365, 0, 117372, - 117379, 0, 117386, 117393, 117400, 117407, 117414, 117421, 117428, - 117435, 117441, 117447, 117454, 117461, 117468, 117474, 117480, 117487, - 117493, 117499, 117506, 117513, 117520, 117526, 117532, 117539, 117546, - 117554, 117561, 117569, 117576, 117584, 0, 117591, 117599, 0, 0, 117606, - 117613, 117620, 117627, 117633, 117642, 117649, 117655, 117662, 117669, - 117676, 117684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117694, 117701, 117707, - 117713, 117719, 117725, 117731, 117737, 117743, 117749, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117755, 117759, 117764, - 117768, 117773, 117777, 117782, 117787, 0, 0, 117793, 117797, 117802, - 117806, 117811, 117815, 117820, 117825, 117830, 117835, 117840, 117845, - 117850, 117855, 117860, 117865, 117870, 117875, 117880, 117885, 117890, - 117895, 117900, 117905, 117909, 117913, 117918, 117923, 117928, 117932, - 117936, 117940, 117944, 117949, 117954, 117959, 117963, 117967, 117972, - 117977, 117983, 117988, 117994, 117999, 118005, 118011, 0, 0, 118018, - 118023, 118029, 118034, 118040, 118045, 118050, 118055, 118060, 118065, - 118069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 118076, 118081, 118087, 118094, 118100, 118106, 118113, - 118119, 118126, 118133, 118141, 118148, 118153, 118159, 118165, 118171, - 118177, 118183, 118189, 118195, 118201, 118207, 118213, 118219, 118225, - 118231, 118237, 118243, 118249, 118255, 118260, 118265, 118271, 118277, - 118283, 118288, 118294, 118300, 118306, 118312, 118318, 118324, 118330, - 118335, 118340, 118345, 118351, 118357, 118363, 118368, 118373, 118379, - 118385, 118391, 118397, 118406, 118415, 118421, 118427, 118434, 118441, - 118448, 118455, 118463, 118470, 118478, 118484, 118490, 118497, 118504, - 118513, 118523, 0, 0, 0, 0, 0, 0, 0, 0, 118528, 118532, 118537, 118543, - 118548, 118553, 118558, 118564, 118570, 118576, 118582, 118588, 118594, - 118598, 118603, 118608, 118613, 118618, 118623, 118628, 118633, 118638, - 118643, 118648, 118653, 118658, 118663, 118668, 118673, 118678, 118683, - 118688, 118692, 118696, 118701, 118706, 118711, 118715, 118720, 118725, - 118730, 118735, 118740, 118745, 118749, 118753, 118757, 118762, 118767, - 118772, 118776, 118780, 118785, 118790, 118795, 118801, 118807, 118814, - 118820, 118827, 118834, 118841, 118848, 118855, 118862, 118869, 118875, - 118881, 118888, 118895, 118902, 118907, 118912, 118917, 118921, 118926, - 118931, 118937, 118942, 118958, 118972, 118983, 118989, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 118995, 119001, 119007, 119013, 119019, 119024, - 119030, 119036, 119042, 119048, 119054, 119060, 119066, 119070, 119074, - 119078, 119082, 119090, 119098, 119106, 119114, 119123, 119132, 119141, - 119150, 119158, 119167, 119176, 119184, 119193, 119202, 119211, 119220, - 119228, 119237, 119245, 119254, 119263, 119271, 119279, 119287, 119295, - 119303, 119312, 119321, 119331, 119341, 119351, 119361, 119371, 119380, - 119390, 119400, 119410, 119421, 119431, 119443, 119455, 119466, 119480, - 119491, 119501, 119513, 119524, 119534, 119546, 119558, 119569, 119580, - 119590, 119600, 119612, 119623, 0, 0, 0, 0, 0, 0, 0, 119635, 119639, - 119646, 119650, 119656, 119662, 119670, 119678, 119686, 119694, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119698, 119702, - 119707, 119711, 119716, 119720, 119725, 119730, 119736, 0, 119741, - 119745, 119750, 119754, 119759, 119763, 119768, 119773, 119778, 119783, - 119788, 119793, 119798, 119803, 119808, 119813, 119818, 119823, 119828, - 119833, 119838, 119843, 119848, 119853, 119857, 119861, 119866, 119871, - 119876, 119880, 119884, 119888, 119892, 119897, 119902, 119907, 119911, - 119915, 119921, 119926, 119932, 119937, 119943, 119949, 119956, 0, - 119962, 119967, 119973, 119978, 119984, 119989, 119994, 119999, 120004, - 120009, 120013, 120018, 120024, 120030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120036, 120041, 120045, 120049, 120053, 120057, 120061, 120065, 120069, - 120073, 120077, 120081, 120085, 120089, 120093, 120097, 120101, 120105, - 120109, 120113, 120118, 120123, 120128, 120133, 120138, 120143, 120148, - 120153, 120158, 0, 0, 0, 120165, 120170, 120175, 120179, 120184, 120189, - 120194, 120199, 120204, 120209, 120214, 120219, 120224, 120229, 120233, - 120237, 120242, 120247, 120251, 120256, 120261, 120266, 120271, 120276, - 120281, 120286, 120290, 120294, 120298, 120303, 120307, 120311, 0, 0, - 120315, 120321, 120328, 120335, 120342, 120349, 120356, 120363, 120370, - 120377, 120384, 120391, 120397, 120403, 120410, 120417, 120423, 120430, - 120437, 120444, 120451, 120458, 0, 120465, 120471, 120477, 120483, - 120490, 120496, 120502, 120508, 120514, 120519, 120524, 120529, 120534, - 120539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120544, 120549, 120555, 120560, 120566, 120571, 120577, 0, - 120582, 120588, 0, 120593, 120599, 120604, 120610, 120616, 120622, - 120628, 120634, 120640, 120646, 120652, 120658, 120664, 120670, 120676, - 120682, 120688, 120694, 120700, 120706, 120712, 120717, 120722, 120728, - 120734, 120740, 120745, 120750, 120755, 120760, 120766, 120772, 120778, - 120783, 120788, 120794, 120800, 120806, 120812, 120819, 120825, 120832, - 120838, 120845, 0, 0, 0, 120852, 0, 120858, 120865, 0, 120871, 120878, - 120884, 120890, 120896, 120902, 120908, 120913, 120918, 0, 0, 0, 0, 0, 0, - 0, 0, 120923, 120929, 120934, 120939, 120944, 120949, 120954, 120959, - 120964, 120969, 0, 0, 0, 0, 0, 0, 120974, 120979, 120985, 120990, 120996, - 121001, 0, 121007, 121013, 0, 121019, 121025, 121031, 121036, 121042, - 121048, 121054, 121059, 121064, 121070, 121076, 121082, 121087, 121093, - 121099, 121105, 121111, 121116, 121122, 121128, 121134, 121140, 121146, - 121152, 121158, 121164, 121170, 121176, 121181, 121187, 121192, 121197, - 121202, 121209, 121215, 121222, 121228, 0, 121235, 121242, 0, 121249, - 121256, 121263, 121269, 121275, 121280, 0, 0, 0, 0, 0, 0, 0, 121285, - 121291, 121296, 121301, 121306, 121311, 121316, 121321, 121326, 121331, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121336, 121340, 121345, 121350, - 121354, 121359, 121363, 121368, 121373, 121377, 121382, 121387, 121392, - 121396, 121400, 121404, 121409, 121413, 121417, 121421, 121426, 121431, - 121436, 121441, 121445, 0, 0, 0, 0, 0, 0, 0, 121452, 121457, 121462, - 121467, 121472, 121476, 121481, 121485, 121490, 121494, 121499, 121504, - 121510, 121515, 121521, 121525, 121530, 0, 121534, 121538, 121543, - 121548, 121553, 121558, 121563, 121568, 121573, 121578, 121583, 121588, - 121593, 121598, 121603, 121608, 121613, 121618, 121623, 121628, 121632, - 121636, 121641, 121646, 121651, 121655, 121659, 121663, 121667, 121672, - 121677, 121682, 121686, 121690, 121695, 121701, 121709, 121714, 121720, - 121725, 121731, 0, 0, 0, 121737, 121742, 121748, 121754, 121759, 121763, - 121767, 121772, 121780, 121790, 121796, 121804, 121810, 121817, 121825, - 121831, 121839, 121845, 121853, 121858, 121862, 121866, 121870, 121874, - 121878, 121882, 121886, 121890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121899, 121905, - 121911, 121917, 121923, 121929, 121935, 121941, 121947, 121953, 121959, - 121965, 121971, 121977, 121983, 121989, 121995, 122001, 122007, 122013, - 122019, 122028, 122032, 122036, 122040, 122044, 122048, 122052, 122056, - 122060, 122064, 122068, 122072, 122076, 122080, 122084, 122088, 122094, - 122100, 122104, 122110, 122116, 122121, 122125, 122130, 122134, 122138, - 122144, 122150, 122154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122158, - 122166, 122169, 122174, 122180, 122188, 122193, 122199, 122207, 122213, - 122219, 122223, 122227, 122234, 122243, 122250, 122259, 122265, 122274, - 122281, 122288, 122295, 122305, 122311, 122315, 122322, 122331, 122341, - 122348, 122355, 122359, 122363, 122370, 122380, 122384, 122391, 122398, - 122405, 122411, 122418, 122425, 122432, 122439, 122443, 122447, 122451, - 122458, 122462, 122469, 122476, 122490, 122499, 122503, 122507, 122511, - 122518, 122522, 122526, 122530, 122538, 122546, 122565, 122575, 122595, - 122599, 122603, 122607, 122611, 122615, 122619, 122623, 122630, 122634, - 122637, 122641, 122645, 122651, 122658, 122667, 122671, 122680, 122689, - 122697, 122701, 122708, 122712, 122716, 122720, 122724, 122735, 122744, - 122753, 122762, 122771, 122783, 122792, 122801, 122810, 122818, 122827, - 122839, 122848, 122856, 122865, 122877, 122886, 122895, 122907, 122916, - 122925, 122937, 122946, 122950, 122954, 122958, 122962, 122966, 122970, - 122974, 122981, 122985, 122989, 123000, 123004, 123008, 123015, 123021, - 123027, 123031, 123038, 123042, 123046, 123050, 123054, 123058, 123062, - 123068, 123076, 123080, 123084, 123087, 123094, 123106, 123110, 123122, - 123129, 123136, 123143, 123150, 123156, 123160, 123164, 123168, 123172, - 123179, 123188, 123195, 123203, 123211, 123217, 123221, 123225, 123229, - 123233, 123239, 123248, 123260, 123267, 123274, 123283, 123294, 123300, - 123309, 123318, 123325, 123334, 123341, 123347, 123357, 123364, 123371, - 123378, 123385, 123389, 123395, 123399, 123410, 123418, 123427, 123439, - 123446, 123453, 123463, 123470, 123479, 123486, 123495, 123502, 123509, - 123519, 123526, 123533, 123542, 123549, 123561, 123570, 123577, 123584, - 123591, 123600, 123610, 123623, 123630, 123639, 123649, 123656, 123665, - 123678, 123685, 123692, 123699, 123709, 123719, 123725, 123735, 123742, - 123749, 123759, 123765, 123772, 123779, 123786, 123796, 123803, 123810, - 123817, 123823, 123830, 123840, 123847, 123851, 123859, 123863, 123875, - 123879, 123893, 123897, 123901, 123905, 123909, 123915, 123922, 123930, - 123934, 123938, 123942, 123946, 123953, 123957, 123963, 123969, 123977, - 123981, 123988, 123996, 124000, 124004, 124010, 124014, 124023, 124032, - 124039, 124049, 124055, 124059, 124063, 124071, 124078, 124085, 124091, - 124095, 124103, 124107, 124114, 124126, 124133, 124143, 124149, 124153, - 124162, 124169, 124178, 124182, 124186, 124193, 124197, 124201, 124205, - 124209, 124212, 124218, 124224, 124228, 124232, 124239, 124246, 124253, - 124260, 124267, 124274, 124281, 124288, 124294, 124298, 124302, 124309, - 124316, 124323, 124330, 124337, 124341, 124344, 124349, 124353, 124357, - 124366, 124375, 124379, 124383, 124389, 124395, 124412, 124418, 124422, - 124431, 124435, 124439, 124446, 124454, 124462, 124468, 124472, 124476, - 124480, 124484, 124487, 124493, 124500, 124510, 124517, 124524, 124531, - 124537, 124544, 124551, 124558, 124565, 124572, 124581, 124588, 124600, - 124607, 124614, 124624, 124635, 124642, 124649, 124656, 124663, 124670, - 124677, 124684, 124691, 124698, 124705, 124715, 124725, 124735, 124742, - 124752, 124759, 124766, 124773, 124780, 124786, 124793, 124800, 124807, - 124814, 124821, 124828, 124835, 124842, 124848, 124855, 124862, 124871, - 124878, 124885, 124889, 124897, 124901, 124905, 124909, 124913, 124917, - 124924, 124928, 124937, 124941, 124948, 124956, 124960, 124964, 124968, - 124981, 124997, 125001, 125005, 125012, 125018, 125025, 125029, 125033, - 125037, 125041, 125045, 125052, 125056, 125074, 125078, 125082, 125089, - 125093, 125097, 125103, 125107, 125111, 125119, 125123, 125127, 125130, - 125134, 125140, 125151, 125160, 125169, 125176, 125183, 125194, 125201, - 125208, 125215, 125222, 125229, 125236, 125243, 125253, 125259, 125266, - 125276, 125285, 125292, 125301, 125311, 125318, 125325, 125332, 125339, - 125351, 125358, 125365, 125372, 125379, 125386, 125396, 125403, 125410, - 125420, 125433, 125445, 125452, 125462, 125469, 125476, 125483, 125497, - 125503, 125511, 125521, 125531, 125538, 125545, 125551, 125555, 125562, - 125572, 125578, 125591, 125595, 125599, 125606, 125610, 125617, 125627, - 125631, 125635, 125639, 125643, 125647, 125654, 125658, 125665, 125672, - 125679, 125688, 125697, 125707, 125714, 125721, 125728, 125738, 125745, - 125755, 125762, 125772, 125779, 125786, 125796, 125806, 125813, 125819, - 125827, 125835, 125841, 125847, 125851, 125855, 125862, 125870, 125876, - 125880, 125884, 125888, 125895, 125907, 125910, 125917, 125923, 125927, - 125931, 125935, 125939, 125943, 125947, 125951, 125955, 125959, 125963, - 125970, 125974, 125980, 125984, 125988, 125992, 125998, 126005, 126012, - 126019, 126030, 126038, 126042, 126048, 126057, 126064, 126070, 126073, - 126077, 126081, 126087, 126096, 126104, 126108, 126114, 126118, 126122, - 126126, 126132, 126139, 126145, 126149, 126155, 126159, 126163, 126172, - 126184, 126188, 126195, 126202, 126212, 126219, 126231, 126238, 126245, - 126252, 126263, 126273, 126286, 126296, 126303, 126307, 126311, 126315, - 126319, 126328, 126337, 126346, 126363, 126372, 126378, 126385, 126393, - 126406, 126410, 126419, 126428, 126437, 126446, 126457, 126466, 126474, - 126483, 126492, 126501, 126510, 126520, 126523, 126527, 126531, 126535, - 126539, 126543, 126549, 126556, 126563, 126570, 126576, 126582, 126589, - 126595, 126602, 126610, 126614, 126621, 126628, 126635, 126643, 126646, - 126650, 126654, 126658, 126661, 126667, 126671, 126677, 126684, 126691, - 126697, 126704, 126711, 126718, 126725, 126732, 126739, 126746, 126753, - 126760, 126767, 126774, 126781, 126788, 126795, 126801, 126805, 126814, - 126818, 126822, 126826, 126830, 126836, 126843, 126850, 126857, 126864, - 126871, 126877, 126885, 126889, 126893, 126897, 126901, 126907, 126924, - 126941, 126945, 126949, 126953, 126957, 126961, 126965, 126971, 126978, - 126982, 126988, 126995, 127002, 127009, 127016, 127023, 127032, 127039, - 127046, 127053, 127060, 127064, 127068, 127074, 127086, 127090, 127094, - 127103, 127107, 127111, 127115, 127121, 127125, 127129, 127138, 127142, - 127146, 127150, 127157, 127161, 127165, 127169, 127173, 127177, 127181, - 127185, 127189, 127195, 127202, 127209, 127215, 127219, 127236, 127242, - 127246, 127253, 127260, 127267, 127274, 127281, 127288, 127292, 127296, - 127300, 127306, 127310, 127316, 127320, 127324, 127331, 127338, 127355, - 127359, 127363, 127367, 127371, 127375, 127387, 127390, 127395, 127400, - 127415, 127425, 127437, 127441, 127445, 127449, 127455, 127462, 127469, - 127479, 127491, 127497, 127503, 127512, 127516, 127520, 127527, 127537, - 127544, 127550, 127554, 127558, 127565, 127571, 127575, 127581, 127585, - 127593, 127599, 127603, 127611, 127619, 127626, 127632, 127639, 127646, - 127656, 127666, 127670, 127674, 127678, 127682, 127688, 127695, 127701, - 127708, 127715, 127722, 127731, 127738, 127745, 127751, 127758, 127765, - 127772, 127779, 127786, 127793, 127799, 127806, 127813, 127820, 127829, - 127836, 127843, 127847, 127853, 127857, 127863, 127870, 127877, 127884, - 127888, 127892, 127896, 127900, 127904, 127911, 127915, 127919, 127925, - 127933, 127937, 127941, 127945, 127949, 127956, 127960, 127964, 127972, - 127976, 127980, 127984, 127988, 127994, 127998, 128002, 128008, 128015, - 128021, 128028, 128040, 128044, 128051, 128058, 128065, 128072, 128084, - 128091, 128095, 128099, 128103, 128110, 128117, 128124, 128131, 128141, - 128148, 128154, 128161, 128168, 128175, 128182, 128191, 128201, 128208, - 128212, 128219, 128223, 128227, 128231, 128238, 128245, 128255, 128261, - 128265, 128274, 128278, 128285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128289, 128295, 128301, - 128308, 128315, 128322, 128329, 128336, 128343, 128349, 128356, 128363, - 128370, 128377, 128384, 128391, 128397, 128403, 128409, 128415, 128421, - 128427, 128433, 128439, 128445, 128452, 128459, 128466, 128473, 128480, - 128487, 128493, 128499, 128505, 128512, 128519, 128525, 128531, 128540, - 128547, 128554, 128561, 128568, 128575, 128582, 128588, 128594, 128600, - 128609, 128616, 128623, 128634, 128645, 128651, 128657, 128663, 128672, - 128679, 128686, 128696, 128706, 128717, 128728, 128740, 128753, 128764, - 128775, 128787, 128800, 128811, 128822, 128833, 128844, 128855, 128867, - 128875, 128883, 128892, 128901, 128910, 128916, 128922, 128928, 128935, - 128945, 128952, 128962, 128967, 128972, 128978, 128984, 128992, 129000, - 129009, 129020, 129031, 129039, 129047, 129056, 129065, 129073, 129080, - 129088, 129096, 129103, 129110, 129119, 129128, 129137, 129146, 129155, - 0, 129164, 129175, 129182, 129190, 129198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129206, 129215, 129222, 129229, 129238, 129245, 129252, 129259, - 129269, 129276, 129283, 129290, 129298, 129305, 129312, 129319, 129330, - 129337, 129344, 129351, 129358, 129365, 129374, 129381, 129387, 129394, - 129403, 129410, 129417, 129424, 129434, 129441, 129448, 129458, 129468, - 129475, 129482, 129489, 129496, 129503, 129510, 129519, 129526, 129533, - 129539, 129547, 129556, 129565, 129576, 129584, 129593, 129602, 129611, - 129620, 129627, 129634, 129643, 129655, 129665, 129672, 129679, 129689, - 129699, 129708, 129718, 129725, 129735, 129742, 129749, 129756, 129766, - 129776, 129783, 129790, 129800, 129806, 129817, 129826, 129836, 129844, - 129857, 129864, 129870, 129878, 129885, 129895, 129899, 129903, 129907, - 129911, 129915, 129919, 129923, 129932, 129936, 129943, 129947, 129951, - 129955, 129959, 129963, 129967, 129971, 129975, 129979, 129983, 129987, - 129991, 129995, 129999, 130003, 130007, 130011, 130015, 130019, 130026, - 130033, 130043, 130056, 130066, 130070, 130074, 130078, 130082, 130086, - 130090, 130094, 130098, 130102, 130106, 130110, 130117, 130124, 130135, - 130142, 130148, 130155, 130162, 130169, 130176, 130183, 130187, 130191, - 130198, 130205, 130212, 130221, 130228, 130241, 130251, 130258, 130265, - 130269, 130273, 130282, 130289, 130296, 130303, 130316, 130323, 130330, - 130340, 130350, 130359, 130366, 130373, 130380, 130387, 130394, 130401, - 130411, 130417, 130425, 130432, 130440, 130447, 130458, 130465, 130471, - 130478, 130485, 130492, 130499, 130509, 130519, 130526, 130533, 130542, - 130550, 130556, 130563, 130570, 130577, 130584, 130588, 130598, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 130608, 130613, 130618, 130623, 130628, 130633, 130638, 130643, - 130648, 130653, 130658, 130663, 130668, 130673, 130678, 130683, 130688, - 130693, 130698, 130703, 130708, 130713, 130718, 130723, 130728, 130733, - 130738, 130743, 130748, 130753, 130758, 130763, 130768, 130773, 130778, - 130783, 130788, 130793, 130798, 130803, 130808, 130813, 130818, 130823, - 130828, 130833, 130838, 130843, 130848, 130853, 130858, 130863, 130868, - 130873, 130878, 130883, 130888, 130893, 130898, 130903, 130908, 130913, - 130918, 130923, 130928, 130933, 130938, 130943, 130948, 130953, 130958, - 130963, 130968, 130973, 130978, 130983, 130988, 130993, 130998, 131003, - 131008, 131013, 131018, 131023, 131028, 131033, 131038, 131043, 131048, - 131053, 131058, 131063, 131068, 131073, 131078, 131083, 131088, 131093, - 131098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131103, 131107, 131111, - 131115, 131119, 131123, 131127, 131131, 131135, 131139, 131143, 131147, - 131151, 131155, 131159, 131163, 131167, 131171, 131175, 131179, 131183, - 131187, 131191, 131195, 131199, 131203, 131207, 131211, 131215, 131219, - 131223, 131227, 131231, 131235, 131239, 131243, 131247, 131251, 131255, - 131259, 131263, 131267, 131271, 131275, 131279, 131283, 131287, 131291, - 131295, 131299, 131303, 131307, 131311, 131315, 131319, 131323, 131327, - 131331, 131335, 131339, 131343, 131347, 131351, 131355, 131359, 131363, - 131367, 131371, 131375, 131379, 131383, 131387, 131391, 131395, 131399, - 131403, 131407, 131411, 131415, 131419, 131423, 131427, 131431, 131435, - 131439, 131443, 131447, 131451, 131455, 131459, 131463, 131467, 131471, - 131475, 131479, 131483, 131487, 131491, 131495, 131499, 131503, 131507, - 131511, 131515, 131519, 131523, 131527, 131531, 131535, 131539, 131543, - 131547, 131551, 131555, 131559, 131563, 131567, 131571, 131575, 131579, - 131583, 131587, 131591, 131595, 131599, 131603, 131607, 131611, 131615, - 131619, 131623, 131627, 131631, 131635, 131639, 131643, 131647, 131651, - 131655, 131659, 131663, 131667, 131671, 131675, 131679, 131683, 131687, - 131691, 131695, 131699, 131703, 131707, 131711, 131715, 131719, 131723, - 131727, 131731, 131735, 131739, 131743, 131747, 131751, 131755, 131759, - 131763, 131767, 131771, 131775, 131779, 131783, 131787, 131791, 131795, - 131799, 131803, 131807, 131811, 131815, 131819, 131823, 131827, 131831, - 131835, 131839, 131843, 131847, 131851, 131855, 131859, 131863, 131867, - 131871, 131875, 131879, 131883, 131887, 131891, 131895, 131899, 131903, - 131907, 131911, 131915, 131919, 131923, 131927, 131931, 131935, 131939, - 131943, 131947, 131951, 131955, 131959, 131963, 131967, 131971, 131975, - 131979, 131983, 131987, 131991, 131995, 131999, 132003, 132007, 132011, - 132015, 132019, 132023, 132027, 132031, 132035, 132039, 132043, 132047, - 132051, 132055, 132059, 132063, 132067, 132071, 132075, 132079, 132083, - 132087, 132091, 132095, 132099, 132103, 132107, 132111, 132115, 132119, - 132123, 132127, 132131, 132135, 132139, 132143, 132147, 132151, 132155, - 132159, 132163, 132167, 132171, 132175, 132179, 132183, 132187, 132191, - 132195, 132199, 132203, 132207, 132211, 132215, 132219, 132223, 132227, - 132231, 132235, 132239, 132243, 132247, 132251, 132255, 132259, 132263, - 132267, 132271, 132275, 132279, 132283, 132287, 132291, 132295, 132299, - 132303, 132307, 132311, 132315, 132319, 132323, 132327, 132331, 132335, - 132339, 132343, 132347, 132351, 132355, 132359, 132363, 132367, 132371, - 132375, 132379, 132383, 132387, 132391, 132395, 132399, 132403, 132407, - 132411, 132415, 132419, 132423, 132427, 132431, 132435, 132439, 132443, - 132447, 132451, 132455, 132459, 132463, 132467, 132471, 132475, 132479, - 132483, 132487, 132491, 132495, 132499, 132503, 132507, 132511, 132515, - 132519, 132523, 132527, 132531, 132535, 132539, 132543, 132547, 132551, - 132555, 132559, 132563, 132567, 132571, 132575, 132579, 132583, 132587, - 132591, 132595, 132599, 132603, 132607, 132611, 132615, 132619, 132623, - 132627, 132631, 132635, 132639, 132643, 132647, 132651, 132655, 132659, - 132663, 132667, 132671, 132675, 132679, 132683, 132687, 132691, 132695, - 132699, 132703, 132707, 132711, 132715, 132719, 132723, 132727, 132731, - 132735, 132739, 132743, 132747, 132751, 132755, 132759, 132763, 132767, - 132771, 132775, 132779, 132783, 132787, 132791, 132795, 132799, 132803, - 132807, 132811, 132815, 132819, 132823, 132827, 132831, 132835, 132839, - 132843, 132847, 132851, 132855, 132859, 132863, 132867, 132871, 132875, - 132879, 132883, 132887, 132891, 132895, 132899, 132903, 132907, 132911, - 132915, 132919, 132923, 132927, 132931, 132935, 132939, 132943, 132947, - 132951, 132955, 132959, 132963, 132967, 132971, 132975, 132979, 132983, - 132987, 132991, 132995, 132999, 133003, 133007, 133011, 133015, 133019, - 133023, 133027, 133031, 133035, 133039, 133043, 133047, 133051, 133055, - 133059, 133063, 133067, 133071, 133075, 133079, 133083, 133087, 133091, - 133095, 133099, 133103, 133107, 133111, 133115, 133119, 133123, 133127, - 133131, 133135, 133139, 133143, 133147, 133151, 133155, 133159, 133163, - 133167, 133171, 133175, 133179, 133183, 133187, 133191, 133195, 133199, - 133203, 133207, 133211, 133215, 133219, 133223, 133227, 133231, 133235, - 133239, 133243, 133247, 133251, 133255, 133259, 133263, 133267, 133271, - 133275, 133279, 133283, 133287, 133291, 133295, 133299, 133303, 133307, - 133311, 133315, 133319, 133323, 133327, 133331, 133335, 133339, 133343, - 133347, 133351, 133355, 133359, 133363, 133367, 133371, 133375, 133379, - 133383, 133387, 133391, 133395, 133399, 133403, 133407, 133411, 133415, - 133419, 133423, 133427, 133431, 133435, 133439, 133443, 133447, 133451, - 133455, 133459, 133463, 133467, 133471, 133475, 133479, 133483, 133487, - 133491, 133495, 133499, 133503, 133507, 133511, 133515, 133519, 133523, - 133527, 133531, 133535, 133539, 133543, 133547, 133551, 133555, 133559, - 133563, 133567, 133571, 133575, 133579, 133583, 133587, 133591, 133595, - 133599, 133603, 133607, 133611, 133615, 133619, 133623, 133627, 133631, - 133635, 133639, 133643, 133647, 133651, 133655, 133659, 133663, 133667, - 133671, 133675, 133679, 133683, 133687, 133691, 133695, 133699, 133703, - 133707, 133711, 133715, 133719, 133723, 133727, 133731, 133735, 133739, - 133743, 133747, 133751, 133755, 133759, 133763, 133767, 133771, 133775, - 133779, 133783, 133787, 133791, 133795, 133799, 133803, 133807, 133811, - 133815, 133819, 133823, 133827, 133831, 133835, 133839, 133843, 133847, - 133851, 133855, 133859, 133863, 133867, 133871, 133875, 133879, 133883, - 133887, 133891, 133895, 133899, 133903, 133907, 133911, 133915, 133919, - 133923, 133927, 133931, 133935, 133939, 133943, 133947, 133951, 133955, - 133959, 133963, 133967, 133971, 133975, 133979, 133983, 133987, 133991, - 133995, 133999, 134003, 134007, 134011, 134015, 134019, 134023, 134027, - 134031, 134035, 134039, 134043, 134047, 134051, 134055, 134059, 134063, - 134067, 134071, 134075, 134079, 134083, 134087, 134091, 134095, 134099, - 134103, 134107, 134111, 134115, 134119, 134123, 134127, 134131, 134135, - 134139, 134143, 134147, 134151, 134155, 134159, 134163, 134167, 134171, - 134175, 134179, 134183, 134187, 134191, 134195, 134199, 134203, 134207, - 134211, 134215, 134219, 134223, 134227, 134231, 134235, 134239, 134243, - 134247, 134251, 134255, 134259, 134263, 134267, 134271, 134275, 134279, - 134283, 134287, 134291, 134295, 134299, 134303, 134307, 134311, 134315, - 134319, 134323, 134327, 134331, 134335, 134339, 134343, 134347, 134351, - 134355, 134359, 134363, 134367, 134371, 134375, 134379, 134383, 134387, - 134391, 134395, 134399, 134403, 134407, 134411, 134415, 134419, 134423, - 134427, 134431, 134435, 134439, 134443, 134447, 134451, 134455, 134459, - 134463, 134467, 134471, 134475, 134479, 134483, 134487, 134491, 134495, - 134499, 134503, 134507, 134511, 134515, 134519, 134523, 134527, 134531, - 134535, 134539, 134543, 134547, 134551, 134555, 134559, 134563, 134567, - 134571, 134575, 134579, 134583, 134587, 134591, 134595, 134599, 134603, - 134607, 134611, 134615, 134619, 134623, 134627, 134631, 134635, 134639, - 134643, 134647, 134651, 134655, 134659, 134663, 134667, 134671, 134675, - 134679, 134683, 134687, 134691, 134695, 134699, 134703, 134707, 134711, - 134715, 134719, 134723, 134727, 134731, 134735, 134739, 134743, 134747, - 134751, 134755, 134759, 134763, 134767, 134771, 134775, 134779, 134783, - 134787, 134791, 134795, 134799, 134803, 134807, 134811, 134815, 134819, - 134823, 134827, 134831, 134835, 134839, 134843, 134847, 134851, 134855, - 134859, 134863, 134867, 134871, 134875, 134879, 134883, 134887, 134891, - 134895, 134899, 134903, 134907, 134911, 134915, 134919, 134923, 134927, - 134931, 134935, 134939, 134943, 134947, 134951, 134955, 134959, 134963, - 134967, 134971, 134975, 134979, 134983, 134987, 134991, 134995, 134999, - 135003, 135007, 135011, 135015, 135019, 135023, 135027, 135031, 135035, - 135039, 135043, 135047, 135051, 135055, 135059, 135063, 135067, 135071, - 135075, 135079, 135083, 135087, 135091, 135095, 135099, 135103, 135107, - 135111, 135115, 135119, 135123, 135127, 135131, 135135, 135139, 135143, - 135147, 135151, 135155, 135159, 135163, 135167, 135171, 135175, 135179, - 135183, 135187, 135191, 135195, 135199, 135203, 135207, 135211, 135215, - 135219, 135223, 135227, 135231, 135235, 135239, 135243, 135247, 135251, - 135255, 135259, 135263, 135267, 135271, 135275, 135279, 135283, 135287, - 135291, 135295, 135299, 135303, 135307, 135311, 135315, 135319, 135323, - 135327, 135331, 135335, 135339, 135343, 135347, 135351, 135355, 135359, - 135363, 135367, 135371, 135375, 135379, 135383, 135387, 135391, 135396, - 135402, 135412, 135422, 135432, 135442, 135448, 135454, 135460, 135468, - 135476, 135484, 135490, 135496, 135504, 135512, 135518, 135524, 135529, - 135534, 135540, 135547, 135554, 135565, 135576, 135585, 135596, 135605, - 135621, 135633, 135644, 135660, 135669, 135681, 135690, 135702, 135714, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135719, 135723, - 135727, 135731, 135735, 135739, 135743, 135747, 135751, 135755, 135759, - 135763, 135767, 135771, 135775, 135779, 135783, 135787, 135791, 135795, - 135799, 135803, 135807, 135811, 135815, 135819, 135823, 135827, 135831, - 135835, 135839, 135843, 135847, 135851, 135855, 135859, 135863, 135867, - 135871, 135875, 135879, 135883, 135887, 135891, 135895, 135899, 135903, - 135907, 135911, 135915, 135919, 135923, 135927, 135931, 135935, 135939, - 135943, 135947, 135951, 135955, 135959, 135963, 135967, 135971, 135975, - 135979, 135983, 135987, 135991, 135995, 135999, 136003, 136007, 136011, - 136015, 136019, 136023, 136027, 136031, 136035, 136039, 136043, 136047, - 136051, 136055, 136059, 136063, 136067, 136071, 136075, 136079, 136083, - 136087, 136091, 136095, 136099, 136103, 136107, 136111, 136115, 136119, - 136123, 136127, 136131, 136135, 136139, 136143, 136147, 136151, 136155, - 136159, 136163, 136167, 136171, 136175, 136179, 136183, 136187, 136191, - 136195, 136199, 136203, 136207, 136211, 136215, 136219, 136223, 136227, - 136231, 136235, 136239, 136243, 136247, 136251, 136255, 136259, 136263, - 136267, 136271, 136275, 136279, 136283, 136287, 136291, 136295, 136299, - 136303, 136307, 136311, 136315, 136319, 136323, 136327, 136331, 136335, - 136339, 136343, 136347, 136351, 136355, 136359, 136363, 136367, 136371, - 136375, 136379, 136383, 136387, 136391, 136395, 136399, 136403, 136407, - 136411, 136415, 136419, 136423, 136427, 136431, 136435, 136439, 136443, - 136447, 136451, 136455, 136459, 136463, 136467, 136471, 136475, 136479, - 136483, 136487, 136491, 136495, 136499, 136503, 136507, 136511, 136515, - 136519, 136523, 136527, 136531, 136535, 136539, 136543, 136547, 136551, - 136555, 136559, 136563, 136567, 136571, 136575, 136579, 136583, 136587, - 136591, 136595, 136599, 136603, 136607, 136611, 136615, 136619, 136623, - 136627, 136631, 136635, 136639, 136643, 136647, 136651, 136655, 136659, - 136663, 136667, 136671, 136675, 136679, 136683, 136687, 136691, 136695, - 136699, 136703, 136707, 136711, 136715, 136719, 136723, 136727, 136731, - 136735, 136739, 136743, 136747, 136751, 136755, 136759, 136763, 136767, - 136771, 136775, 136779, 136783, 136787, 136791, 136795, 136799, 136803, - 136807, 136811, 136815, 136819, 136823, 136827, 136831, 136835, 136839, - 136843, 136847, 136851, 136855, 136859, 136863, 136867, 136871, 136875, - 136879, 136883, 136887, 136891, 136895, 136899, 136903, 136907, 136911, - 136915, 136919, 136923, 136927, 136931, 136935, 136939, 136943, 136947, - 136951, 136955, 136959, 136963, 136967, 136971, 136975, 136979, 136983, - 136987, 136991, 136995, 136999, 137003, 137007, 137011, 137015, 137019, - 137023, 137027, 137031, 137035, 137039, 137043, 137047, 137051, 137055, - 137059, 137063, 137067, 137071, 137075, 137079, 137083, 137087, 137091, - 137095, 137099, 137103, 137107, 137111, 137115, 137119, 137123, 137127, - 137131, 137135, 137139, 137143, 137147, 137151, 137155, 137159, 137163, - 137167, 137171, 137175, 137179, 137183, 137187, 137191, 137195, 137199, - 137203, 137207, 137211, 137215, 137219, 137223, 137227, 137231, 137235, - 137239, 137243, 137247, 137251, 137255, 137259, 137263, 137267, 137271, - 137275, 137279, 137283, 137287, 137291, 137295, 137299, 137303, 137307, - 137311, 137315, 137319, 137323, 137327, 137331, 137335, 137339, 137343, - 137347, 137351, 137355, 137359, 137363, 137367, 137371, 137375, 137379, - 137383, 137387, 137391, 137395, 137399, 137403, 137407, 137411, 137415, - 137419, 137423, 137427, 137431, 137435, 137439, 137443, 137447, 137451, - 137461, 137465, 137469, 137473, 137477, 137481, 137485, 137489, 137493, - 137497, 137501, 137505, 137510, 137514, 137518, 137522, 137526, 137530, - 137534, 137538, 137542, 137546, 137550, 137554, 137558, 137562, 137566, - 137570, 137574, 137583, 137592, 137596, 137600, 137604, 137608, 137612, - 137616, 137620, 137624, 137628, 137632, 137636, 137640, 137644, 137648, - 137652, 137656, 137660, 137664, 137668, 137672, 137676, 137680, 137684, - 137688, 137692, 137696, 137700, 137704, 137708, 137712, 137716, 137720, - 137724, 137728, 137732, 137736, 137740, 137744, 137748, 137752, 137756, - 137760, 137764, 137768, 137772, 137776, 137780, 137784, 137788, 137792, - 137796, 137800, 137804, 137808, 137812, 137816, 137820, 137824, 137828, - 137832, 137836, 137840, 137844, 137848, 137852, 137856, 137860, 137864, - 137868, 137872, 137876, 137880, 137884, 137888, 137892, 137896, 137900, - 137904, 137908, 137912, 137916, 137920, 137924, 137928, 137932, 137936, - 137940, 137944, 137948, 137952, 137956, 137960, 137964, 137968, 137972, - 137976, 137980, 137984, 137988, 137992, 137996, 138000, 138004, 138008, - 138012, 138016, 138020, 138024, 138028, 138032, 138036, 138040, 138044, - 138048, 138052, 138056, 138060, 138064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138068, - 138076, 138084, 138094, 138104, 138112, 138118, 138126, 138134, 138144, - 138156, 138168, 138174, 138182, 138188, 138194, 138200, 138206, 138212, - 138218, 138224, 138230, 138236, 138242, 138248, 138256, 138264, 138270, - 138276, 138282, 138288, 138296, 138304, 138313, 138319, 138327, 138333, - 138339, 138345, 138351, 138357, 138365, 138373, 138379, 138385, 138391, - 138397, 138403, 138409, 138415, 138421, 138427, 138433, 138439, 138445, - 138451, 138457, 138463, 138469, 138475, 138481, 138487, 138495, 138501, - 138507, 138517, 138525, 138531, 138537, 138543, 138549, 138555, 138561, - 138567, 138573, 138579, 138585, 138591, 138597, 138603, 138609, 138615, - 138621, 138627, 138633, 138639, 138645, 138651, 138657, 138665, 138671, - 138679, 138687, 138695, 138701, 138707, 138713, 138719, 138725, 138733, - 138743, 138751, 138759, 138765, 138771, 138779, 138787, 138793, 138801, - 138809, 138817, 138823, 138829, 138835, 138841, 138847, 138853, 138861, - 138869, 138875, 138881, 138887, 138893, 138899, 138907, 138913, 138919, - 138925, 138931, 138937, 138943, 138951, 138957, 138963, 138969, 138975, - 138983, 138991, 138997, 139003, 139009, 139014, 139020, 139026, 139034, - 139040, 139046, 139052, 139058, 139064, 139070, 139076, 139082, 139088, - 139098, 139106, 139112, 139118, 139124, 139132, 139138, 139144, 139150, - 139158, 139164, 139170, 139176, 139182, 139188, 139194, 139200, 139206, - 139212, 139218, 139224, 139232, 139238, 139246, 139252, 139258, 139266, - 139272, 139278, 139284, 139290, 139296, 139302, 139308, 139314, 139320, - 139326, 139332, 139338, 139344, 139350, 139356, 139362, 139368, 139374, - 139380, 139388, 139394, 139400, 139406, 139412, 139418, 139424, 139430, - 139436, 139442, 139448, 139454, 139460, 139466, 139474, 139480, 139486, - 139494, 139500, 139506, 139512, 139518, 139524, 139530, 139536, 139542, - 139548, 139554, 139562, 139568, 139574, 139580, 139586, 139592, 139600, - 139608, 139614, 139620, 139626, 139632, 139638, 139644, 139649, 139654, - 139659, 139664, 139669, 139674, 139679, 139684, 139689, 139694, 139699, - 139704, 139709, 139714, 139719, 139724, 139729, 139734, 139739, 139744, - 139749, 139754, 139759, 139764, 139769, 139774, 139779, 139784, 139789, - 139794, 139801, 139806, 139811, 139816, 139821, 139826, 139831, 139836, - 139841, 139846, 139851, 139856, 139861, 139866, 139871, 139876, 139881, - 139886, 139891, 139896, 139901, 139906, 139911, 139916, 139921, 139926, - 139931, 139936, 139941, 139946, 139951, 139956, 139961, 139966, 139971, - 139976, 139981, 139986, 139991, 139996, 140001, 140006, 140011, 140016, - 140021, 140026, 140031, 140036, 140041, 140046, 140051, 140056, 140061, - 140066, 140071, 140076, 140081, 140086, 140091, 140098, 140103, 140108, - 140113, 140118, 140123, 140128, 140133, 140138, 140143, 140148, 140153, - 140158, 140163, 140168, 140173, 140178, 140183, 140188, 140193, 140198, - 140203, 140210, 140215, 140220, 140226, 140231, 140236, 140241, 140246, - 140251, 140256, 140261, 140266, 140271, 140276, 140281, 140286, 140291, - 140296, 140301, 140306, 140311, 140316, 140321, 140326, 140331, 140336, - 140341, 140346, 140351, 140356, 140361, 140366, 140371, 140376, 140381, - 140386, 140391, 140396, 140401, 140406, 140411, 140416, 140421, 140426, - 140431, 140436, 140441, 140448, 140453, 140458, 140465, 140472, 140477, - 140482, 140487, 140492, 140497, 140502, 140507, 140512, 140517, 140522, - 140527, 140532, 140537, 140542, 140547, 140552, 140557, 140562, 140567, - 140572, 140577, 140582, 140587, 140592, 140597, 140604, 140609, 140614, - 140619, 140624, 140629, 140634, 140639, 140644, 140649, 140654, 140659, - 140664, 140669, 140674, 140679, 140684, 140689, 140694, 140701, 140706, - 140711, 140716, 140721, 140726, 140731, 140736, 140742, 140747, 140752, - 140757, 140762, 140767, 140772, 140777, 140782, 140789, 140796, 140801, - 140806, 140810, 140815, 140819, 140823, 140828, 140835, 140840, 140845, - 140854, 140859, 140864, 140869, 140874, 140881, 140888, 140893, 140898, - 140903, 140908, 140915, 140920, 140925, 140930, 140935, 140940, 140945, - 140950, 140955, 140960, 140965, 140970, 140975, 140982, 140986, 140991, - 140996, 141001, 141006, 141010, 141015, 141020, 141025, 141030, 141035, - 141040, 141045, 141050, 141055, 141061, 141067, 141073, 141079, 141085, - 141090, 141096, 141102, 141108, 141114, 141120, 141126, 141132, 141138, - 141144, 141150, 141156, 141162, 141168, 141174, 141180, 141186, 141192, - 141198, 141203, 141209, 141215, 141221, 141227, 141233, 141239, 141245, - 141251, 141257, 141263, 141269, 141275, 141281, 141287, 141293, 141299, - 141305, 141311, 141317, 141323, 141328, 141334, 141340, 141346, 141352, - 141358, 0, 0, 0, 0, 0, 0, 0, 141364, 141369, 141374, 141379, 141384, - 141389, 141394, 141398, 141403, 141408, 141413, 141418, 141423, 141428, - 141433, 141438, 141443, 141447, 141452, 141456, 141461, 141466, 141471, - 141476, 141481, 141485, 141490, 141495, 141499, 141504, 141509, 0, - 141514, 141519, 141523, 141527, 141531, 141535, 141539, 141543, 141547, - 141551, 0, 0, 0, 0, 141555, 141559, 141564, 141569, 141574, 141579, - 141584, 141589, 141594, 141599, 141604, 141609, 141614, 141619, 141624, - 141629, 141634, 141639, 141644, 141649, 141654, 141659, 141664, 141669, - 141674, 141679, 141684, 141689, 141694, 141699, 141704, 141709, 141714, - 141719, 141724, 141730, 141736, 141743, 141750, 141755, 141760, 141765, - 141770, 141775, 141780, 141785, 141790, 141795, 141800, 141805, 141810, - 141814, 141819, 141824, 141829, 141833, 141837, 141842, 141846, 141851, - 141856, 141861, 141865, 141869, 141873, 141877, 141882, 141887, 141892, - 141896, 141901, 141906, 141911, 141916, 141921, 141926, 141931, 141936, - 141941, 141946, 141951, 0, 141956, 141961, 141965, 141969, 141973, - 141977, 141981, 141985, 141989, 141993, 0, 0, 0, 0, 0, 0, 141997, 142004, - 142010, 142017, 142024, 142031, 142038, 142045, 142052, 142059, 142066, - 142073, 142080, 142087, 142094, 142101, 142108, 142115, 142121, 142128, - 142135, 142142, 142148, 142155, 142161, 142167, 142174, 142180, 142187, - 142193, 0, 0, 142199, 142207, 142215, 142224, 142233, 142242, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 142250, 142255, 142260, 142265, 142270, 142275, 142280, - 142285, 142290, 142295, 142300, 142305, 142310, 142315, 142320, 142325, - 142330, 142335, 142340, 142345, 142350, 142355, 142360, 142365, 142370, - 142375, 142380, 142385, 142390, 142395, 142400, 142405, 142410, 142415, - 142420, 142425, 142430, 142435, 142440, 142445, 142450, 142455, 142460, - 142465, 142470, 142475, 142480, 142485, 142490, 142497, 142504, 142511, - 142518, 142525, 142532, 142539, 142546, 142555, 142562, 142569, 142576, - 142583, 142590, 142597, 142604, 142611, 142618, 142625, 142632, 142637, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142646, 142651, 142655, 142659, 142663, - 142667, 142671, 142675, 142679, 142683, 0, 142687, 142692, 142697, - 142704, 142709, 142716, 142723, 0, 142728, 142735, 142740, 142745, - 142752, 142759, 142764, 142769, 142774, 142779, 142784, 142791, 142798, - 142803, 142808, 142813, 142826, 142835, 142842, 142851, 142860, 0, 0, 0, - 0, 0, 142869, 142876, 142883, 142890, 142897, 142904, 142911, 142918, - 142925, 142932, 142939, 142946, 142953, 142960, 142967, 142974, 142981, - 142988, 142995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143002, 143008, 143014, 143020, - 143026, 143032, 143038, 143044, 143050, 143056, 143062, 143068, 143073, - 143079, 143084, 143090, 143095, 143101, 143107, 143112, 143118, 143123, - 143129, 143135, 143141, 143147, 143153, 143159, 143165, 143170, 143175, - 143181, 143187, 143193, 143199, 143205, 143211, 143217, 143223, 143229, - 143235, 143241, 143247, 143253, 143258, 143264, 143269, 143275, 143280, - 143286, 143292, 143297, 143303, 143308, 143314, 143320, 143326, 143332, - 143338, 143344, 143350, 143355, 143360, 143366, 143372, 143377, 143381, - 143385, 143389, 143393, 143397, 143401, 143405, 143409, 143413, 143418, - 143423, 143428, 143433, 143438, 143443, 143448, 143453, 143458, 143463, - 143470, 143477, 143484, 143488, 143494, 143499, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143505, - 143508, 143512, 143516, 143520, 143523, 143527, 143532, 143536, 143540, - 143544, 143548, 143552, 143557, 143562, 143566, 143570, 143573, 143577, - 143582, 143587, 143591, 143595, 143598, 143602, 143606, 143610, 143614, - 143618, 143622, 143626, 143629, 143633, 143637, 143641, 143645, 143649, - 143653, 143659, 143662, 143666, 143670, 143674, 143678, 143682, 143686, - 143690, 143694, 143698, 143703, 143708, 143714, 143718, 143722, 143726, - 143730, 143734, 143738, 143743, 143746, 143750, 143754, 143758, 143762, - 143768, 143772, 143776, 143780, 143784, 143788, 143792, 143796, 143800, - 143804, 143808, 0, 0, 0, 0, 143812, 143817, 143821, 143825, 143831, - 143837, 143841, 143846, 143851, 143856, 143861, 143865, 143870, 143875, - 143880, 143884, 143889, 143894, 143899, 143903, 143908, 143913, 143918, - 143923, 143928, 143933, 143938, 143943, 143947, 143952, 143957, 143962, - 143967, 143972, 143977, 143982, 143987, 143992, 143997, 144002, 144009, - 144014, 144021, 144026, 144031, 144036, 144041, 144046, 144051, 144056, - 144061, 144066, 144071, 144076, 144081, 144086, 144091, 0, 0, 0, 0, 0, 0, - 0, 144096, 144099, 144104, 144107, 144110, 144114, 144118, 144122, - 144126, 144130, 144134, 144138, 144144, 144150, 144156, 144162, 144168, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144174, 144178, 144182, - 144188, 144194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144199, 144208, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144217, 144220, 144223, 144226, 144229, - 144232, 144235, 144238, 144241, 144244, 144247, 144250, 144253, 144256, - 144259, 144262, 144265, 144268, 144271, 144274, 144277, 144280, 144283, - 144286, 144289, 144292, 144295, 144298, 144301, 144304, 144307, 144310, - 144313, 144316, 144319, 144322, 144325, 144328, 144331, 144334, 144337, - 144340, 144343, 144346, 144349, 144352, 144355, 144358, 144361, 144364, - 144367, 144370, 144373, 144376, 144379, 144382, 144385, 144388, 144391, - 144394, 144397, 144400, 144403, 144406, 144409, 144412, 144415, 144418, - 144421, 144424, 144427, 144430, 144433, 144436, 144439, 144442, 144445, - 144448, 144451, 144454, 144457, 144460, 144463, 144466, 144469, 144472, - 144475, 144478, 144481, 144484, 144487, 144490, 144493, 144496, 144499, - 144502, 144505, 144508, 144511, 144514, 144517, 144520, 144523, 144526, - 144529, 144532, 144535, 144538, 144541, 144544, 144547, 144550, 144553, - 144556, 144559, 144562, 144565, 144568, 144571, 144574, 144577, 144580, - 144583, 144586, 144589, 144592, 144595, 144598, 144601, 144604, 144607, - 144610, 144613, 144616, 144619, 144622, 144625, 144628, 144631, 144634, - 144637, 144640, 144643, 144646, 144649, 144652, 144655, 144658, 144661, - 144664, 144667, 144670, 144673, 144676, 144679, 144682, 144685, 144688, - 144691, 144694, 144697, 144700, 144703, 144706, 144709, 144712, 144715, - 144718, 144721, 144724, 144727, 144730, 144733, 144736, 144739, 144742, - 144745, 144748, 144751, 144754, 144757, 144760, 144763, 144766, 144769, - 144772, 144775, 144778, 144781, 144784, 144787, 144790, 144793, 144796, - 144799, 144802, 144805, 144808, 144811, 144814, 144817, 144820, 144823, - 144826, 144829, 144832, 144835, 144838, 144841, 144844, 144847, 144850, - 144853, 144856, 144859, 144862, 144865, 144868, 144871, 144874, 144877, - 144880, 144883, 144886, 144889, 144892, 144895, 144898, 144901, 144904, - 144907, 144910, 144913, 144916, 144919, 144922, 144925, 144928, 144931, - 144934, 144937, 144940, 144943, 144946, 144949, 144952, 144955, 144958, - 144961, 144964, 144967, 144970, 144973, 144976, 144979, 144982, 144985, - 144988, 144991, 144994, 144997, 145000, 145003, 145006, 145009, 145012, - 145015, 145018, 145021, 145024, 145027, 145030, 145033, 145036, 145039, - 145042, 145045, 145048, 145051, 145054, 145057, 145060, 145063, 145066, - 145069, 145072, 145075, 145078, 145081, 145084, 145087, 145090, 145093, - 145096, 145099, 145102, 145105, 145108, 145111, 145114, 145117, 145120, - 145123, 145126, 145129, 145132, 145135, 145138, 145141, 145144, 145147, - 145150, 145153, 145156, 145159, 145162, 145165, 145168, 145171, 145174, - 145177, 145180, 145183, 145186, 145189, 145192, 145195, 145198, 145201, - 145204, 145207, 145210, 145213, 145216, 145219, 145222, 145225, 145228, - 145231, 145234, 145237, 145240, 145243, 145246, 145249, 145252, 145255, - 145258, 145261, 145264, 145267, 145270, 145273, 145276, 145279, 145282, - 145285, 145288, 145291, 145294, 145297, 145300, 145303, 145306, 145309, - 145312, 145315, 145318, 145321, 145324, 145327, 145330, 145333, 145336, - 145339, 145342, 145345, 145348, 145351, 145354, 145357, 145360, 145363, - 145366, 145369, 145372, 145375, 145378, 145381, 145384, 145387, 145390, - 145393, 145396, 145399, 145402, 145405, 145408, 145411, 145414, 145417, - 145420, 145423, 145426, 145429, 145432, 145435, 145438, 145441, 145444, - 145447, 145450, 145453, 145456, 145459, 145462, 145465, 145468, 145471, - 145474, 145477, 145480, 145483, 145486, 145489, 145492, 145495, 145498, - 145501, 145504, 145507, 145510, 145513, 145516, 145519, 145522, 145525, - 145528, 145531, 145534, 145537, 145540, 145543, 145546, 145549, 145552, - 145555, 145558, 145561, 145564, 145567, 145570, 145573, 145576, 145579, - 145582, 145585, 145588, 145591, 145594, 145597, 145600, 145603, 145606, - 145609, 145612, 145615, 145618, 145621, 145624, 145627, 145630, 145633, - 145636, 145639, 145642, 145645, 145648, 145651, 145654, 145657, 145660, - 145663, 145666, 145669, 145672, 145675, 145678, 145681, 145684, 145687, - 145690, 145693, 145696, 145699, 145702, 145705, 145708, 145711, 145714, - 145717, 145720, 145723, 145726, 145729, 145732, 145735, 145738, 145741, - 145744, 145747, 145750, 145753, 145756, 145759, 145762, 145765, 145768, - 145771, 145774, 145777, 145780, 145783, 145786, 145789, 145792, 145795, - 145798, 145801, 145804, 145807, 145810, 145813, 145816, 145819, 145822, - 145825, 145828, 145831, 145834, 145837, 145840, 145843, 145846, 145849, - 145852, 145855, 145858, 145861, 145864, 145867, 145870, 145873, 145876, - 145879, 145882, 145885, 145888, 145891, 145894, 145897, 145900, 145903, - 145906, 145909, 145912, 145915, 145918, 145921, 145924, 145927, 145930, - 145933, 145936, 145939, 145942, 145945, 145948, 145951, 145954, 145957, - 145960, 145963, 145966, 145969, 145972, 145975, 145978, 145981, 145984, - 145987, 145990, 145993, 145996, 145999, 146002, 146005, 146008, 146011, - 146014, 146017, 146020, 146023, 146026, 146029, 146032, 146035, 146038, - 146041, 146044, 146047, 146050, 146053, 146056, 146059, 146062, 146065, - 146068, 146071, 146074, 146077, 146080, 146083, 146086, 146089, 146092, - 146095, 146098, 146101, 146104, 146107, 146110, 146113, 146116, 146119, - 146122, 146125, 146128, 146131, 146134, 146137, 146140, 146143, 146146, - 146149, 146152, 146155, 146158, 146161, 146164, 146167, 146170, 146173, - 146176, 146179, 146182, 146185, 146188, 146191, 146194, 146197, 146200, - 146203, 146206, 146209, 146212, 146215, 146218, 146221, 146224, 146227, - 146230, 146233, 146236, 146239, 146242, 146245, 146248, 146251, 146254, - 146257, 146260, 146263, 146266, 146269, 146272, 146275, 146278, 146281, - 146284, 146287, 146290, 146293, 146296, 146299, 146302, 146305, 146308, - 146311, 146314, 146317, 146320, 146323, 146326, 146329, 146332, 146335, - 146338, 146341, 146344, 146347, 146350, 146353, 146356, 146359, 146362, - 146365, 146368, 146371, 146374, 146377, 146380, 146383, 146386, 146389, - 146392, 146395, 146398, 146401, 146404, 146407, 146410, 146413, 146416, - 146419, 146422, 146425, 146428, 146431, 146434, 146437, 146440, 146443, - 146446, 146449, 146452, 146455, 146458, 146461, 146464, 146467, 146470, - 146473, 146476, 146479, 146482, 146485, 146488, 146491, 146494, 146497, - 146500, 146503, 146506, 146509, 146512, 146515, 146518, 146521, 146526, - 146531, 146536, 146541, 146546, 146551, 146556, 146561, 146566, 146571, - 146576, 146581, 146586, 146591, 146596, 146601, 146606, 146611, 146616, - 146621, 146626, 146631, 146636, 146641, 146646, 146651, 146656, 146661, - 146666, 146671, 146676, 146681, 146686, 146691, 146696, 146701, 146706, - 146711, 146716, 146721, 146726, 146731, 146736, 146741, 146746, 146751, - 146756, 146761, 146766, 146771, 146776, 146781, 146786, 146791, 146796, - 146801, 146806, 146811, 146816, 146821, 146826, 146831, 146836, 146841, - 146846, 146851, 146856, 146861, 146866, 146871, 146876, 146881, 146886, - 146891, 146896, 146901, 146906, 146911, 146916, 146921, 146926, 146931, - 146936, 146941, 146946, 146951, 146956, 146961, 146966, 146971, 146976, - 146981, 146986, 146991, 146996, 147001, 147006, 147011, 147016, 147021, - 147026, 147031, 147036, 147041, 147046, 147051, 147056, 147061, 147066, - 147071, 147076, 147081, 147086, 147091, 147096, 147101, 147106, 147111, - 147116, 147121, 147126, 147131, 147136, 147141, 147146, 147151, 147156, - 147161, 147166, 147171, 147176, 147181, 147186, 147191, 147196, 147201, - 147206, 147211, 147216, 147221, 147226, 147231, 147236, 147241, 147246, - 147251, 147256, 147261, 147266, 147271, 147276, 147281, 147286, 147291, - 147296, 147301, 147306, 147311, 147316, 147321, 147326, 147331, 147336, - 147341, 147346, 147351, 147356, 147361, 147366, 147371, 147376, 147381, - 147386, 147391, 147396, 147401, 147406, 147411, 147416, 147421, 147426, - 147431, 147436, 147441, 147446, 147451, 147456, 147461, 147466, 147471, - 147476, 147481, 147486, 147491, 147496, 147501, 147506, 147511, 147516, - 147521, 147526, 147531, 147536, 147541, 147546, 147551, 147556, 147561, - 147566, 147571, 147576, 147581, 147586, 147591, 147596, 147601, 147606, - 147611, 147616, 147621, 147626, 147631, 147636, 147641, 147646, 147651, - 147656, 147661, 147666, 147671, 147676, 147681, 147686, 147691, 147696, - 147701, 147706, 147711, 147716, 147721, 147726, 147731, 147736, 147741, - 147746, 147751, 147756, 147761, 147766, 147771, 147776, 147781, 147786, - 147791, 147796, 147801, 147806, 147811, 147816, 147821, 147826, 147831, - 147836, 147841, 147846, 147851, 147856, 147861, 147866, 147871, 147876, - 147881, 147886, 147891, 147896, 147901, 147906, 147911, 147916, 147921, - 147926, 147931, 147936, 147941, 147946, 147951, 147956, 147961, 147966, - 147971, 147976, 147981, 147986, 147991, 147996, 148001, 148006, 148011, - 148016, 148021, 148026, 148031, 148036, 148041, 148046, 148051, 148056, - 148061, 148066, 148071, 148076, 148081, 148086, 148091, 148096, 148101, - 148106, 148111, 148116, 148121, 148126, 148131, 148136, 148141, 148146, - 148151, 148156, 148161, 148166, 148171, 148176, 148181, 148186, 148191, - 148196, 148201, 148206, 148211, 148216, 148221, 148226, 148231, 148236, - 148241, 148246, 148251, 148256, 148261, 148266, 148271, 148276, 148281, - 148286, 148291, 148296, 148301, 148306, 148311, 148316, 148321, 148326, - 148331, 148336, 148341, 148346, 148351, 148356, 148361, 148366, 148371, - 148376, 148381, 148386, 148391, 148396, 148401, 148406, 148411, 148416, - 148421, 148426, 148431, 148436, 148441, 148446, 148451, 148456, 148461, - 148466, 148471, 148476, 148481, 148486, 148491, 148496, 148501, 148506, - 148511, 148516, 148521, 148526, 148531, 148536, 148541, 148546, 148551, - 148556, 148561, 148566, 148571, 148576, 148581, 148586, 148591, 148596, - 148601, 148606, 148611, 148616, 148621, 148626, 148631, 148636, 148641, - 148646, 148651, 148656, 148661, 148666, 148671, 148676, 148681, 148686, - 148691, 148696, 148701, 148706, 148711, 148716, 148721, 148726, 148731, - 148736, 148741, 148746, 148751, 148756, 148761, 148766, 148771, 148776, - 148781, 148786, 148791, 148796, 148801, 148806, 148811, 148816, 148821, - 148826, 148831, 148836, 148841, 148846, 148851, 148856, 148861, 148866, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148871, 148877, 148883, 148889, 0, 148895, - 148901, 148907, 148915, 148923, 148931, 148939, 0, 148947, 148955, 0, - 148963, 148968, 148975, 148979, 148983, 148987, 148991, 148995, 148999, - 149003, 149007, 149011, 149015, 149019, 149023, 149027, 149031, 149035, - 149039, 149043, 149047, 149051, 149055, 149059, 149063, 149067, 149071, - 149075, 149079, 149083, 149087, 149091, 149095, 149099, 149103, 149107, - 149111, 149115, 149119, 149123, 149127, 149131, 149135, 149139, 149143, - 149147, 149151, 149155, 149159, 149163, 149167, 149171, 149175, 149179, - 149183, 149187, 149191, 149195, 149199, 149203, 149207, 149211, 149215, - 149219, 149223, 149227, 149231, 149235, 149239, 149243, 149247, 149251, - 149255, 149259, 149263, 149267, 149271, 149275, 149279, 149283, 149287, - 149291, 149295, 149299, 149303, 149307, 149311, 149315, 149319, 149323, - 149327, 149331, 149335, 149339, 149343, 149347, 149351, 149355, 149359, - 149363, 149367, 149371, 149375, 149379, 149383, 149387, 149391, 149395, - 149399, 149403, 149407, 149411, 149415, 149419, 149423, 149427, 149431, - 149435, 149439, 149443, 149447, 149451, 149455, 149459, 149463, 149467, - 149471, 149475, 149479, 149483, 149487, 149491, 149495, 149499, 149503, - 149507, 149511, 149515, 149519, 149523, 149527, 149531, 149535, 149539, - 149543, 149547, 149551, 149555, 149559, 149563, 149567, 149571, 149575, - 149579, 149583, 149587, 149591, 149595, 149599, 149603, 149607, 149611, - 149615, 149619, 149623, 149627, 149631, 149635, 149639, 149643, 149647, - 149651, 149655, 149659, 149663, 149667, 149671, 149675, 149679, 149683, - 149687, 149691, 149695, 149699, 149703, 149707, 149711, 149715, 149719, - 149723, 149727, 149731, 149735, 149739, 149743, 149747, 149751, 149755, - 149759, 149763, 149767, 149771, 149775, 149779, 149783, 149787, 149791, - 149795, 149799, 149803, 149807, 149811, 149815, 149819, 149823, 149827, - 149831, 149835, 149839, 149843, 149847, 149851, 149855, 149859, 149863, - 149867, 149871, 149875, 149879, 149883, 149887, 149891, 149895, 149899, - 149903, 149907, 149911, 149915, 149919, 149923, 149927, 149931, 149935, - 149939, 149943, 149947, 149951, 149955, 149959, 149963, 149967, 149971, - 149975, 149979, 149983, 149987, 149991, 149995, 149999, 150003, 150007, - 150011, 150015, 150019, 150023, 150027, 150031, 150035, 150039, 150043, - 150047, 150051, 150055, 150059, 150063, 150067, 150071, 150075, 150079, - 150083, 150087, 150091, 150095, 150099, 150103, 150107, 150111, 150115, - 150122, 150128, 150134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 150140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 150146, 150152, 150158, 0, 0, 150164, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 150169, 150174, 150179, 150184, 0, 0, 0, 0, 0, - 0, 0, 0, 150189, 150192, 150195, 150198, 150201, 150204, 150207, 150210, - 150213, 150216, 150219, 150222, 150225, 150228, 150231, 150234, 150237, - 150240, 150243, 150246, 150249, 150252, 150255, 150258, 150261, 150264, - 150267, 150270, 150273, 150276, 150279, 150282, 150285, 150288, 150291, - 150294, 150297, 150300, 150303, 150306, 150309, 150312, 150315, 150318, - 150321, 150324, 150327, 150330, 150333, 150336, 150339, 150342, 150345, - 150348, 150351, 150354, 150357, 150360, 150363, 150366, 150369, 150372, - 150375, 150378, 150381, 150384, 150387, 150390, 150393, 150396, 150399, - 150402, 150405, 150408, 150411, 150414, 150417, 150420, 150423, 150426, - 150429, 150432, 150435, 150438, 150441, 150444, 150447, 150450, 150453, - 150456, 150459, 150462, 150465, 150468, 150471, 150474, 150477, 150480, - 150483, 150486, 150489, 150492, 150495, 150498, 150501, 150504, 150507, - 150510, 150513, 150516, 150519, 150522, 150525, 150528, 150531, 150534, - 150537, 150540, 150543, 150546, 150549, 150552, 150555, 150558, 150561, - 150564, 150567, 150570, 150573, 150576, 150579, 150582, 150585, 150588, - 150591, 150594, 150597, 150600, 150603, 150606, 150609, 150612, 150615, - 150618, 150621, 150624, 150627, 150630, 150633, 150636, 150639, 150642, - 150645, 150648, 150651, 150654, 150657, 150660, 150663, 150666, 150669, - 150672, 150675, 150678, 150681, 150684, 150687, 150690, 150693, 150696, - 150699, 150702, 150705, 150708, 150711, 150714, 150717, 150720, 150723, - 150726, 150729, 150732, 150735, 150738, 150741, 150744, 150747, 150750, - 150753, 150756, 150759, 150762, 150765, 150768, 150771, 150774, 150777, - 150780, 150783, 150786, 150789, 150792, 150795, 150798, 150801, 150804, - 150807, 150810, 150813, 150816, 150819, 150822, 150825, 150828, 150831, - 150834, 150837, 150840, 150843, 150846, 150849, 150852, 150855, 150858, - 150861, 150864, 150867, 150870, 150873, 150876, 150879, 150882, 150885, - 150888, 150891, 150894, 150897, 150900, 150903, 150906, 150909, 150912, - 150915, 150918, 150921, 150924, 150927, 150930, 150933, 150936, 150939, - 150942, 150945, 150948, 150951, 150954, 150957, 150960, 150963, 150966, - 150969, 150972, 150975, 150978, 150981, 150984, 150987, 150990, 150993, - 150996, 150999, 151002, 151005, 151008, 151011, 151014, 151017, 151020, - 151023, 151026, 151029, 151032, 151035, 151038, 151041, 151044, 151047, - 151050, 151053, 151056, 151059, 151062, 151065, 151068, 151071, 151074, - 151077, 151080, 151083, 151086, 151089, 151092, 151095, 151098, 151101, - 151104, 151107, 151110, 151113, 151116, 151119, 151122, 151125, 151128, - 151131, 151134, 151137, 151140, 151143, 151146, 151149, 151152, 151155, - 151158, 151161, 151164, 151167, 151170, 151173, 151176, 151179, 151182, - 151185, 151188, 151191, 151194, 151197, 151200, 151203, 151206, 151209, - 151212, 151215, 151218, 151221, 151224, 151227, 151230, 151233, 151236, - 151239, 151242, 151245, 151248, 151251, 151254, 151257, 151260, 151263, - 151266, 151269, 151272, 151275, 151278, 151281, 151284, 151287, 151290, - 151293, 151296, 151299, 151302, 151305, 151308, 151311, 151314, 151317, - 151320, 151323, 151326, 151329, 151332, 151335, 151338, 151341, 151344, - 151347, 151350, 151353, 151356, 151359, 151362, 151365, 151368, 151371, - 151374, 0, 0, 0, 0, 151377, 151381, 151385, 151389, 151393, 151397, - 151401, 151404, 151408, 151412, 151416, 151420, 151423, 151429, 151435, - 151441, 151447, 151453, 151457, 151463, 151467, 151471, 151477, 151481, - 151485, 151489, 151493, 151497, 151501, 151505, 151511, 151517, 151523, - 151529, 151536, 151543, 151550, 151560, 151567, 151574, 151580, 151586, - 151592, 151598, 151606, 151614, 151622, 151630, 151639, 151645, 151653, - 151659, 151666, 151672, 151679, 151685, 151693, 151697, 151701, 151706, - 151712, 151718, 151726, 151734, 151740, 151747, 151750, 151756, 151760, - 151763, 151767, 151770, 151773, 151777, 151782, 151786, 151790, 151796, - 151801, 151807, 151811, 151815, 151818, 151822, 151826, 151831, 151835, - 151840, 151844, 151849, 151853, 151857, 151861, 151865, 151869, 151873, - 151877, 151881, 151886, 151891, 151896, 151901, 151907, 151913, 151919, - 151925, 151931, 0, 0, 0, 0, 0, 151936, 151944, 151953, 151961, 151968, - 151976, 151983, 151990, 151999, 152006, 152013, 152021, 152029, 0, 0, 0, - 152037, 152043, 152051, 152057, 152064, 152070, 152076, 152082, 152088, - 0, 0, 0, 0, 0, 0, 0, 152094, 152100, 152108, 152114, 152121, 152127, - 152133, 152139, 152145, 152151, 0, 0, 152156, 152162, 152168, 152171, - 152180, 152187, 152195, 152202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 152209, 152224, 152237, 152246, 152257, 152266, 152275, - 152286, 152295, 152304, 152319, 152332, 152345, 152359, 152371, 152379, - 152389, 152397, 152405, 152415, 152423, 152431, 152445, 152457, 152469, - 152478, 152489, 152498, 152507, 152514, 152523, 152532, 152539, 152544, - 152549, 152554, 152559, 152564, 152569, 152574, 152579, 152584, 152589, - 152594, 152599, 152604, 0, 0, 152613, 152622, 152631, 152640, 152645, - 152652, 152657, 152662, 152670, 152675, 152682, 152687, 152694, 152699, - 152704, 152711, 152718, 152723, 152732, 152738, 152744, 152752, 152758, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 152764, 152768, 152774, 152778, 152786, - 152790, 152794, 152798, 152806, 152810, 152816, 152825, 152829, 152833, - 152837, 152843, 152849, 152855, 152861, 152867, 152873, 152879, 152885, - 152891, 152899, 152907, 152915, 152923, 152928, 152934, 152938, 152942, - 152946, 152950, 152956, 152962, 152968, 152974, 152980, 152988, 152994, - 153002, 153010, 153018, 153026, 153034, 153038, 153046, 153052, 153060, - 153064, 153068, 153072, 153076, 153080, 153084, 153092, 153100, 153112, - 153124, 153130, 153140, 153148, 153158, 153170, 153174, 153180, 153186, - 153192, 153198, 153204, 153210, 153216, 153222, 153228, 153236, 153242, - 153248, 153254, 153262, 153270, 153281, 153292, 153298, 153304, 153314, - 153320, 153328, 153332, 153338, 153344, 153350, 153356, 153362, 153368, - 153374, 153378, 153384, 153390, 153398, 153406, 153414, 153420, 153428, - 153441, 153454, 153462, 153470, 153482, 153490, 153500, 153508, 153512, - 153516, 153520, 153524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153528, - 153533, 153538, 153543, 153550, 153557, 153564, 153571, 153576, 153581, - 153586, 153591, 153598, 153603, 153610, 153617, 153622, 153627, 153632, - 153639, 153644, 153649, 153656, 153663, 153668, 153673, 153678, 153685, - 153692, 153699, 153704, 153709, 153716, 153723, 153730, 153737, 153742, - 153747, 153752, 153759, 153764, 153769, 153774, 153781, 153790, 153797, - 153802, 153807, 153812, 153817, 153822, 153827, 153836, 153843, 153848, - 153855, 153862, 153867, 153872, 153877, 153884, 153889, 153896, 153903, - 153908, 153913, 153918, 153925, 153932, 153937, 153942, 153949, 153956, - 153963, 153968, 153973, 153978, 153983, 153990, 153999, 154008, 154013, - 154020, 154029, 154034, 154039, 154044, 154049, 154056, 154063, 154070, - 154077, 154082, 154087, 154092, 154099, 154106, 154113, 154118, 154123, - 154130, 154135, 154142, 154147, 154154, 154159, 154166, 154173, 154178, - 154183, 154188, 154193, 154198, 154203, 154208, 154213, 154218, 154225, - 154232, 154239, 154246, 154253, 154262, 154267, 154272, 154279, 154286, - 154291, 154298, 154305, 154312, 154319, 154326, 154333, 154338, 154343, - 154348, 154353, 154358, 154367, 154376, 154385, 154394, 154403, 154412, - 154421, 154430, 154435, 154446, 154457, 154466, 154471, 154476, 154481, - 154486, 154495, 154502, 154509, 154516, 154523, 154530, 154537, 154546, - 154555, 154566, 154575, 154586, 154595, 154602, 154611, 154622, 154631, - 154640, 154649, 154658, 154665, 154672, 154679, 154688, 154697, 154708, - 154717, 154726, 154737, 154742, 154747, 154758, 154766, 154775, 154784, - 154793, 154804, 154813, 154822, 154833, 154844, 154855, 154866, 154877, - 154888, 154895, 154902, 154909, 154916, 154927, 154936, 154943, 154950, - 154957, 154968, 154979, 154990, 155001, 155012, 155023, 155034, 155045, - 155052, 155059, 155068, 155077, 155084, 155091, 155098, 155107, 155116, - 155125, 155132, 155141, 155150, 155159, 155166, 155173, 155178, 155184, - 155191, 155198, 155205, 155212, 155219, 155226, 155235, 155244, 155253, - 155262, 155269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155278, 155284, 155289, - 155294, 155301, 155307, 155313, 155319, 155325, 155331, 155337, 155343, - 155347, 155351, 155357, 155363, 155369, 155373, 155378, 155383, 155387, - 155391, 155395, 155401, 155407, 155413, 155419, 155425, 155431, 155437, - 155443, 155449, 155459, 155469, 155475, 155481, 155491, 155501, 155507, - 0, 0, 155513, 155521, 155526, 155531, 155537, 155543, 155549, 155555, - 155561, 155567, 155574, 155581, 155587, 155593, 155599, 155605, 155611, - 155617, 155623, 155629, 155634, 155640, 155646, 155652, 155658, 155664, - 155673, 155679, 155684, 155692, 155699, 155706, 155715, 155724, 155733, - 155742, 155751, 155760, 155769, 155778, 155788, 155798, 155806, 155814, - 155823, 155832, 155838, 155844, 155850, 155856, 155864, 155872, 155876, - 155882, 155887, 155893, 155899, 155905, 155911, 155917, 155926, 155931, - 155938, 155943, 155948, 155953, 155959, 155965, 155971, 155978, 155983, - 155988, 155993, 155998, 156003, 156009, 156015, 156021, 156027, 156033, - 156039, 156045, 156051, 156056, 156061, 156066, 156071, 156076, 156081, - 156086, 156091, 156097, 156103, 156108, 156113, 156118, 156123, 156128, - 156134, 156141, 156145, 156149, 156153, 156157, 156161, 156165, 156169, - 156173, 156181, 156191, 156195, 156199, 156205, 156211, 156217, 156223, - 156229, 156235, 156241, 156247, 156253, 156259, 156265, 156271, 156277, - 156283, 156287, 156291, 156298, 156304, 156310, 156316, 156321, 156328, - 156333, 156339, 156345, 156351, 156357, 156362, 156366, 156372, 156376, - 156380, 156384, 156390, 156396, 156400, 156406, 156412, 156418, 156424, - 156430, 156438, 156446, 156452, 156458, 156464, 156470, 156482, 156494, - 156508, 156520, 156532, 156546, 156560, 156574, 156578, 156586, 156594, - 156599, 156603, 156607, 156611, 156615, 156619, 156623, 156627, 156633, - 156639, 156645, 156651, 156659, 156668, 156675, 156682, 156690, 156697, - 156709, 156721, 156733, 156745, 156752, 156756, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156760, 156767, 156774, 156781, - 156788, 156795, 156802, 156809, 156816, 156823, 156830, 156837, 156844, - 156851, 156858, 156865, 156872, 156879, 156886, 156893, 156900, 156907, - 156914, 156921, 156928, 156935, 156942, 156949, 156956, 156963, 156970, - 156977, 156984, 156991, 156998, 157005, 157012, 157019, 157026, 157033, - 157040, 157047, 157054, 157061, 157068, 157075, 157082, 157089, 157096, - 157103, 157110, 157117, 157124, 157131, 157138, 157145, 157152, 157159, - 157166, 157173, 157180, 157187, 157194, 157201, 157208, 157215, 157222, - 157227, 157232, 157237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157241, 157247, 157252, 157257, 157262, - 157267, 157272, 157277, 157282, 157287, 157292, 157298, 157304, 157310, - 157316, 157322, 157328, 157334, 157340, 157346, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 157352, 157358, 157363, 157368, 157373, 157378, 157383, - 157388, 157393, 157398, 157403, 157409, 157415, 157421, 157427, 157433, - 157439, 157445, 157451, 157457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157463, 157468, 157475, 157482, 157489, 157496, 157501, 157506, 157513, - 157518, 157523, 157530, 157535, 157540, 157545, 157552, 157561, 157566, - 157571, 157576, 157581, 157586, 157591, 157598, 157603, 157608, 157613, - 157618, 157623, 157628, 157633, 157638, 157643, 157648, 157653, 157658, - 157664, 157669, 157674, 157679, 157684, 157689, 157694, 157699, 157704, - 157709, 157718, 157723, 157731, 157736, 157741, 157746, 157751, 157756, - 157761, 157766, 157775, 157780, 157785, 157790, 157795, 157800, 157807, - 157812, 157819, 157824, 157829, 157834, 157839, 157844, 157849, 157854, - 157859, 157864, 157869, 157874, 157879, 157884, 157889, 157894, 157899, - 157904, 157909, 157914, 157923, 157928, 157933, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 157938, 157946, 157954, 157962, 157970, 157978, 157986, 157994, - 158002, 158010, 158018, 158026, 158034, 158042, 158050, 158058, 158066, - 158074, 158082, 158087, 158092, 158097, 158102, 158107, 158111, 0, 0, 0, - 0, 0, 0, 0, 158115, 158119, 158124, 158129, 158134, 158138, 158143, - 158148, 158153, 158157, 158162, 158167, 158171, 158176, 158181, 158185, - 158190, 158195, 158199, 158204, 158209, 158213, 158218, 158223, 158228, - 158233, 158238, 158242, 158247, 158252, 158257, 158261, 158266, 158271, - 158276, 158280, 158285, 158290, 158294, 158299, 158304, 158308, 158313, - 158318, 158322, 158327, 158332, 158336, 158341, 158346, 158351, 158356, - 158361, 158365, 158370, 158375, 158380, 158384, 158389, 158394, 158399, - 158403, 158408, 158413, 158417, 158422, 158427, 158431, 158436, 158441, - 158445, 158450, 158455, 158459, 158464, 158469, 158474, 158479, 158484, - 158488, 158493, 158498, 158503, 158507, 158512, 0, 158517, 158521, - 158526, 158531, 158535, 158540, 158545, 158549, 158554, 158559, 158563, - 158568, 158573, 158577, 158582, 158587, 158592, 158597, 158602, 158607, - 158613, 158619, 158625, 158630, 158636, 158642, 158648, 158653, 158659, - 158665, 158670, 158676, 158682, 158687, 158693, 158699, 158704, 158710, - 158716, 158721, 158727, 158733, 158739, 158745, 158751, 158756, 158762, - 158768, 158774, 158779, 158785, 158791, 158797, 158802, 158808, 158814, - 158819, 158825, 158831, 158836, 158842, 158848, 158853, 158859, 158865, - 158870, 158876, 158882, 158888, 158894, 158900, 0, 158904, 158909, 0, 0, - 158914, 0, 0, 158919, 158924, 0, 0, 158929, 158934, 158938, 158943, 0, - 158948, 158953, 158958, 158962, 158967, 158972, 158977, 158982, 158987, - 158991, 158996, 159001, 0, 159006, 0, 159011, 159016, 159020, 159025, - 159030, 159034, 159039, 0, 159044, 159049, 159054, 159058, 159063, - 159068, 159072, 159077, 159082, 159087, 159092, 159097, 159102, 159108, - 159114, 159120, 159125, 159131, 159137, 159143, 159148, 159154, 159160, - 159165, 159171, 159177, 159182, 159188, 159194, 159199, 159205, 159211, - 159216, 159222, 159228, 159234, 159240, 159246, 159251, 159257, 159263, - 159269, 159274, 159280, 159286, 159292, 159297, 159303, 159309, 159314, - 159320, 159326, 159331, 159337, 159343, 159348, 159354, 159360, 159365, - 159371, 159377, 159383, 159389, 159395, 159400, 0, 159406, 159412, - 159417, 159423, 0, 0, 159429, 159435, 159441, 159446, 159452, 159458, - 159463, 159469, 0, 159475, 159481, 159487, 159492, 159498, 159504, - 159510, 0, 159516, 159521, 159527, 159533, 159539, 159544, 159550, - 159556, 159562, 159567, 159573, 159579, 159584, 159590, 159596, 159601, - 159607, 159613, 159618, 159624, 159630, 159635, 159641, 159647, 159653, - 159659, 159665, 159670, 0, 159676, 159682, 159687, 159693, 0, 159699, - 159704, 159710, 159716, 159721, 0, 159727, 0, 0, 0, 159732, 159738, - 159744, 159749, 159755, 159761, 159767, 0, 159773, 159778, 159784, - 159790, 159796, 159801, 159807, 159813, 159819, 159824, 159830, 159836, - 159841, 159847, 159853, 159858, 159864, 159870, 159875, 159881, 159887, - 159892, 159898, 159904, 159910, 159916, 159922, 159928, 159935, 159942, - 159949, 159955, 159962, 159969, 159976, 159982, 159989, 159996, 160002, - 160009, 160016, 160022, 160029, 160036, 160042, 160049, 160056, 160062, - 160069, 160076, 160083, 160090, 160097, 160103, 160110, 160117, 160124, - 160130, 160137, 160144, 160151, 160157, 160164, 160171, 160177, 160184, - 160191, 160197, 160204, 160211, 160217, 160224, 160231, 160237, 160244, - 160251, 160258, 160265, 160272, 160276, 160281, 160286, 160291, 160295, - 160300, 160305, 160310, 160314, 160319, 160324, 160328, 160333, 160338, - 160342, 160347, 160352, 160356, 160361, 160366, 160370, 160375, 160380, - 160385, 160390, 160395, 160399, 160404, 160409, 160414, 160418, 160423, - 160428, 160433, 160437, 160442, 160447, 160451, 160456, 160461, 160465, - 160470, 160475, 160479, 160484, 160489, 160493, 160498, 160503, 160508, - 160513, 160518, 160523, 160529, 160535, 160541, 160546, 160552, 160558, - 160564, 160569, 160575, 160581, 160586, 160592, 160598, 160603, 160609, - 160615, 160620, 160626, 160632, 160637, 160643, 160649, 160655, 160661, - 160667, 160672, 160678, 160684, 160690, 160695, 160701, 160707, 160713, - 160718, 160724, 160730, 160735, 160741, 160747, 160752, 160758, 160764, - 160769, 160775, 160781, 160786, 160792, 160798, 160804, 160810, 160816, - 160821, 160827, 160833, 160839, 160844, 160850, 160856, 160862, 160867, - 160873, 160879, 160884, 160890, 160896, 160901, 160907, 160913, 160918, - 160924, 160930, 160935, 160941, 160947, 160953, 160959, 160965, 160970, - 160976, 160982, 160988, 160993, 160999, 161005, 161011, 161016, 161022, - 161028, 161033, 161039, 161045, 161050, 161056, 161062, 161067, 161073, - 161079, 161084, 161090, 161096, 161102, 161108, 161114, 161120, 161127, - 161134, 161141, 161147, 161154, 161161, 161168, 161174, 161181, 161188, - 161194, 161201, 161208, 161214, 161221, 161228, 161234, 161241, 161248, - 161254, 161261, 161268, 161275, 161282, 161289, 161295, 161302, 161309, - 161316, 161322, 161329, 161336, 161343, 161349, 161356, 161363, 161369, - 161376, 161383, 161389, 161396, 161403, 161409, 161416, 161423, 161429, - 161436, 161443, 161450, 161457, 161464, 161469, 161475, 161481, 161487, - 161492, 161498, 161504, 161510, 161515, 161521, 161527, 161532, 161538, - 161544, 161549, 161555, 161561, 161566, 161572, 161578, 161583, 161589, - 161595, 161601, 161607, 161613, 161618, 161624, 161630, 161636, 161641, - 161647, 161653, 161659, 161664, 161670, 161676, 161681, 161687, 161693, - 161698, 161704, 161710, 161715, 161721, 161727, 161732, 161738, 161744, - 161750, 161756, 161762, 161768, 0, 0, 161775, 161780, 161785, 161790, - 161795, 161800, 161805, 161810, 161815, 161820, 161825, 161830, 161835, - 161840, 161845, 161850, 161855, 161860, 161866, 161871, 161876, 161881, - 161886, 161891, 161896, 161901, 161905, 161910, 161915, 161920, 161925, - 161930, 161935, 161940, 161945, 161950, 161955, 161960, 161965, 161970, - 161975, 161980, 161985, 161990, 161996, 162001, 162006, 162011, 162016, - 162021, 162026, 162031, 162037, 162042, 162047, 162052, 162057, 162062, - 162067, 162072, 162077, 162082, 162087, 162092, 162097, 162102, 162107, - 162112, 162117, 162122, 162127, 162132, 162137, 162142, 162147, 162152, - 162158, 162163, 162168, 162173, 162178, 162183, 162188, 162193, 162197, - 162202, 162207, 162212, 162217, 162222, 162227, 162232, 162237, 162242, - 162247, 162252, 162257, 162262, 162267, 162272, 162277, 162282, 162288, - 162293, 162298, 162303, 162308, 162313, 162318, 162323, 162329, 162334, - 162339, 162344, 162349, 162354, 162359, 162365, 162371, 162377, 162383, - 162389, 162395, 162401, 162407, 162413, 162419, 162425, 162431, 162437, - 162443, 162449, 162455, 162461, 162468, 162474, 162480, 162486, 162492, - 162498, 162504, 162510, 162515, 162521, 162527, 162533, 162539, 162545, - 162551, 162557, 162563, 162569, 162575, 162581, 162587, 162593, 162599, - 162605, 162611, 162617, 162624, 162630, 162636, 162642, 162648, 162654, - 162660, 162666, 162673, 162679, 162685, 162691, 162697, 162703, 162709, - 162715, 162721, 162727, 162733, 162739, 162745, 162751, 162757, 162763, - 162769, 162775, 162781, 162787, 162793, 162799, 162805, 162811, 162818, - 162824, 162830, 162836, 162842, 162848, 162854, 162860, 162865, 162871, - 162877, 162883, 162889, 162895, 162901, 162907, 162913, 162919, 162925, - 162931, 162937, 162943, 162949, 162955, 162961, 162967, 162974, 162980, - 162986, 162992, 162998, 163004, 163010, 163016, 163023, 163029, 163035, - 163041, 163047, 163053, 163059, 163066, 163073, 163080, 163087, 163094, - 163101, 163108, 163115, 163122, 163129, 163136, 163143, 163150, 163157, - 163164, 163171, 163178, 163186, 163193, 163200, 163207, 163214, 163221, - 163228, 163235, 163241, 163248, 163255, 163262, 163269, 163276, 163283, - 163290, 163297, 163304, 163311, 163318, 163325, 163332, 163339, 163346, - 163353, 163360, 163368, 163375, 163382, 163389, 163396, 163403, 163410, - 163417, 163425, 163432, 163439, 163446, 163453, 163460, 163467, 163472, - 0, 0, 163477, 163482, 163486, 163490, 163494, 163498, 163502, 163506, - 163510, 163514, 163518, 163524, 163529, 163534, 163539, 163544, 163549, - 163554, 163559, 163564, 163569, 163574, 163578, 163582, 163586, 163590, - 163594, 163598, 163602, 163606, 163610, 163616, 163621, 163626, 163631, - 163636, 163641, 163646, 163651, 163656, 163661, 163667, 163672, 163677, - 163682, 163687, 163692, 163697, 163702, 163707, 163712, 163716, 163721, - 163726, 163731, 163736, 163741, 163746, 163752, 163760, 163767, 163772, - 163777, 163784, 163790, 163795, 163801, 163807, 163815, 163821, 163828, - 163836, 163842, 163851, 163860, 163868, 163876, 163882, 163889, 163897, - 163905, 163911, 163918, 163927, 163936, 163943, 163954, 163964, 163974, - 163984, 163994, 164001, 164008, 164015, 164022, 164031, 164040, 164051, - 164062, 164071, 164080, 164091, 164100, 164109, 164120, 164129, 164138, - 164146, 164154, 164165, 164176, 164184, 164193, 164202, 164209, 164220, - 164231, 164240, 164249, 164256, 164265, 164274, 164283, 164294, 164303, - 164313, 164322, 164331, 164342, 164355, 164370, 164381, 164394, 164406, - 164415, 164426, 164437, 164446, 164457, 164471, 164486, 164489, 164498, - 164503, 164509, 164517, 164523, 164529, 164538, 164545, 164555, 164567, - 164574, 164577, 164583, 164590, 164596, 164601, 164604, 164609, 164612, - 164620, 164626, 164635, 164642, 164650, 164656, 164661, 164664, 164667, - 164670, 164676, 164683, 164689, 164694, 164702, 164705, 164710, 164718, - 164724, 164733, 164740, 164750, 164759, 164762, 164768, 164775, 164782, - 164789, 164794, 164802, 164810, 164819, 164825, 164834, 164843, 164852, - 164858, 164867, 164874, 164881, 164888, 164896, 164902, 164910, 164916, - 164923, 164930, 164938, 164949, 164959, 164965, 164972, 164979, 164986, - 164992, 164999, 165006, 165011, 165018, 165026, 165035, 165041, 165053, - 165064, 165070, 165078, 165084, 165091, 165098, 165105, 165111, 165118, - 165127, 165133, 165139, 165146, 165153, 165161, 165171, 165181, 165191, - 165201, 165209, 165217, 165227, 165235, 165240, 165245, 165250, 165256, - 165263, 165270, 165276, 165282, 165287, 165294, 165302, 165312, 165320, - 165328, 165338, 165348, 165356, 165366, 165376, 165388, 165400, 165412, - 165422, 165428, 165434, 165441, 165450, 165459, 165468, 165477, 165487, - 165496, 165505, 165514, 165519, 165525, 165534, 165544, 165553, 165559, - 165565, 165572, 165579, 165586, 165592, 165599, 165606, 165613, 165619, - 165623, 165628, 165635, 165642, 165649, 165654, 165662, 165670, 165679, - 165687, 165694, 165702, 165711, 165721, 165724, 165728, 165733, 165738, - 165743, 165748, 165753, 165758, 165763, 165768, 165773, 165778, 165783, - 165788, 165793, 165798, 165803, 165808, 165813, 165820, 165826, 165833, - 165839, 165844, 165851, 165857, 165864, 165870, 165875, 165882, 165889, - 165896, 165902, 165908, 165917, 165926, 165936, 165943, 165950, 165959, - 165968, 165977, 165986, 165995, 166001, 166009, 166015, 166025, 166030, - 166039, 166048, 166055, 166066, 166073, 166080, 166087, 166094, 166101, - 166108, 166115, 166122, 166129, 166136, 166142, 166148, 166154, 166161, - 166168, 166175, 166182, 166189, 166196, 166203, 166210, 166217, 166224, - 166231, 166238, 166243, 166252, 166261, 166270, 166277, 166284, 166291, - 166298, 166305, 166312, 166319, 166326, 166335, 166344, 166353, 166362, - 166371, 166380, 166389, 166398, 166407, 166416, 166425, 166434, 166443, - 166449, 166457, 166463, 166473, 166478, 166487, 166496, 166505, 166516, - 166521, 166528, 166535, 166542, 166547, 166553, 166559, 166565, 166572, - 166579, 166586, 166593, 166600, 166607, 166614, 166621, 166628, 166635, - 166642, 166649, 166654, 166663, 166672, 166681, 166690, 166699, 166708, - 166717, 166726, 166737, 166748, 166755, 166762, 166769, 166776, 166783, - 166790, 166798, 166808, 166818, 166828, 166839, 166850, 166861, 166870, - 166879, 166888, 166893, 166898, 166903, 166908, 166919, 166930, 166941, - 166952, 166963, 166973, 166984, 166993, 167002, 167011, 167020, 167029, - 167037, 167046, 167057, 167068, 167079, 167090, 167101, 167113, 167126, - 167138, 167151, 167163, 167176, 167188, 167201, 167212, 167223, 167232, - 167240, 167249, 167260, 167271, 167283, 167296, 167310, 167325, 167337, - 167350, 167362, 167375, 167386, 167397, 167406, 167414, 167423, 167430, - 167437, 167444, 167451, 167458, 167465, 167472, 167479, 167486, 167493, - 167498, 167503, 167508, 167515, 167525, 167536, 167546, 167557, 167571, - 167586, 167601, 167615, 167630, 167645, 167656, 167667, 167680, 167693, - 167702, 167711, 167724, 167737, 167744, 167751, 167756, 167761, 167766, - 167771, 167776, 167783, 167792, 167797, 167800, 167805, 167812, 167819, - 167826, 167833, 167840, 167847, 167860, 167874, 167889, 167896, 167903, - 167910, 167919, 167927, 167935, 167944, 167949, 167954, 167959, 167964, - 167969, 167974, 167981, 167988, 167994, 168001, 168007, 168014, 168019, - 168024, 168029, 168034, 168039, 168046, 168053, 168058, 168065, 168072, - 168077, 168082, 168087, 168092, 168097, 168102, 168109, 168116, 168123, - 168126, 168131, 168136, 168141, 168146, 168153, 168160, 168168, 168176, - 168181, 168186, 168193, 168200, 168207, 168212, 168219, 168226, 168231, - 168238, 168245, 168252, 168259, 168266, 168273, 168282, 168291, 168298, - 168307, 168316, 168321, 168328, 168335, 168340, 168347, 168354, 168361, - 168368, 168375, 168380, 168387, 168394, 168403, 168410, 168419, 168430, - 168439, 168448, 168457, 168466, 168469, 168474, 168481, 168490, 168497, - 168506, 168513, 168518, 168523, 168526, 168529, 168532, 168539, 168546, - 168555, 168564, 168573, 168580, 168587, 168592, 168604, 168609, 168614, - 168619, 168624, 168629, 168634, 168639, 168644, 168647, 168652, 168657, - 168662, 168667, 168672, 168679, 168684, 168691, 168694, 168699, 168702, - 168705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168708, 168713, - 168718, 168723, 168728, 0, 168733, 168738, 168743, 168748, 168753, - 168758, 168763, 168768, 168773, 168778, 168783, 168788, 168793, 168798, - 168803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168808, 168818, 168826, 168833, 168840, - 168849, 168858, 168867, 168874, 168888, 168900, 168910, 168918, 168930, - 168939, 168950, 168959, 168966, 168974, 168985, 168997, 169006, 169016, - 169028, 169039, 169048, 169059, 169071, 169079, 169090, 169099, 0, 0, 0, - 0, 0, 0, 169107, 169117, 169127, 169137, 169147, 169157, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 169167, 169172, 169177, 169182, 169187, 169192, - 169197, 0, 169202, 169207, 169212, 169218, 169222, 169227, 169232, - 169237, 169242, 169247, 169252, 169257, 169262, 169267, 169272, 169277, - 169282, 0, 0, 169287, 169292, 169297, 169302, 169307, 169312, 169317, 0, - 169322, 169327, 0, 169333, 169338, 169346, 169353, 169362, 0, 0, 0, 0, 0, - 169367, 169372, 169378, 169384, 169390, 169396, 169402, 169408, 169414, - 169419, 169424, 169430, 169436, 169441, 169447, 169453, 169459, 169465, - 169470, 169476, 169481, 169487, 169493, 169499, 169505, 169510, 169516, - 169522, 169528, 169535, 169541, 169548, 169555, 169561, 169567, 169574, - 169581, 169588, 169595, 169602, 169609, 169616, 169622, 169628, 169635, - 169641, 169648, 169655, 169661, 169668, 169674, 169681, 169688, 169695, - 169703, 169710, 169720, 169728, 169735, 169742, 169751, 169762, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 169771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169778, 169785, 169793, 169801, 169809, 169816, 169823, 169831, 169839, - 169847, 169854, 169861, 169869, 169877, 169885, 169892, 169900, 169908, - 169916, 169924, 169932, 169940, 169948, 169955, 169963, 169970, 169978, - 169985, 169993, 170001, 170009, 170017, 170025, 170033, 170041, 170049, - 170057, 170064, 170072, 170079, 170086, 170093, 170101, 170108, 170116, - 0, 0, 0, 170124, 170131, 170138, 170145, 170152, 170159, 170166, 170173, - 170182, 170191, 170200, 170209, 170218, 170228, 0, 0, 170236, 170244, - 170251, 170258, 170265, 170272, 170279, 170286, 170293, 170300, 0, 0, 0, - 0, 170307, 170316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170324, - 170328, 170333, 170338, 170343, 170347, 170352, 170356, 170360, 170365, - 170369, 170374, 170378, 170383, 170388, 170392, 170396, 170400, 170404, - 170410, 170415, 170422, 170426, 170430, 170436, 170441, 170448, 170452, - 170457, 170464, 170468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 170475, 170480, 170484, 170489, 170494, 170499, 170504, 170508, - 170513, 170517, 170521, 170525, 170530, 170535, 170540, 170544, 170549, - 170554, 170559, 170564, 170569, 170573, 170577, 170582, 170586, 170590, - 170595, 170599, 170603, 170607, 170612, 170616, 170621, 170626, 170631, - 170636, 170641, 170646, 170651, 170656, 170661, 170666, 170671, 170676, - 170681, 170686, 170691, 170696, 170701, 170706, 170710, 170714, 170718, - 170722, 170726, 170730, 170734, 170738, 0, 0, 0, 0, 0, 170742, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 170747, 170753, 170760, 170767, 170774, 170781, 170787, - 170794, 170801, 170808, 170815, 170821, 170828, 170835, 170842, 170849, - 170855, 170862, 170869, 170876, 170883, 170889, 170896, 170903, 170910, - 170917, 170924, 170931, 170938, 170945, 170952, 170959, 170966, 170973, - 170979, 170985, 170991, 170997, 171003, 171009, 171015, 171021, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 171027, 171031, 171035, 171039, 171043, 171047, 171051, 0, - 171055, 171061, 171065, 171069, 0, 171073, 171079, 0, 171085, 171091, - 171097, 171103, 171109, 171115, 171121, 171127, 171133, 171139, 171145, - 171151, 171157, 171163, 171169, 0, 171175, 171182, 171188, 171195, - 171202, 171209, 171216, 171223, 171230, 171237, 171244, 171251, 171258, - 171265, 171272, 171279, 171286, 171293, 171300, 171307, 171314, 171321, - 171328, 171335, 171342, 171349, 171356, 171363, 171370, 171377, 171384, - 171391, 171398, 171405, 171412, 171418, 171424, 171430, 171437, 171443, - 171450, 171456, 171463, 171470, 171477, 171484, 171491, 171498, 171504, - 171511, 171518, 171525, 171532, 171539, 171546, 171553, 171559, 171566, - 171573, 171580, 171587, 171594, 171602, 171609, 171616, 171623, 171630, - 171637, 171644, 171651, 171658, 171665, 171672, 171679, 171686, 171692, - 171699, 171706, 171713, 171720, 171727, 171734, 171741, 171749, 171756, - 171762, 171769, 171776, 171783, 171790, 171797, 171804, 171811, 171818, - 171825, 171832, 171839, 171846, 171853, 171860, 171867, 171874, 171881, - 171888, 171895, 171902, 171908, 171915, 171922, 171929, 171936, 171943, - 171950, 171957, 171964, 171971, 171978, 171985, 171992, 171999, 172006, - 172013, 172020, 172027, 172034, 172041, 172048, 172055, 172062, 172070, - 172078, 172086, 172093, 172100, 172107, 172114, 172121, 172128, 172135, - 172142, 172149, 172156, 172162, 172169, 172176, 172183, 172190, 172197, - 172204, 172211, 172218, 172225, 172232, 172239, 172246, 172253, 172260, - 172268, 172276, 172284, 172291, 172298, 172305, 172312, 172319, 172326, - 172333, 172340, 172347, 172354, 172361, 172368, 172375, 172382, 172388, - 172395, 172402, 172409, 172416, 172423, 172430, 172437, 172444, 172451, - 172458, 172465, 172472, 172479, 172486, 172493, 172500, 172507, 172514, - 172521, 172528, 172535, 172542, 0, 0, 172549, 172553, 172557, 172561, - 172565, 172569, 172573, 172577, 172581, 172585, 172591, 172597, 172603, - 172609, 172617, 172625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 172631, 172637, 172643, 172649, 172655, 172661, 172667, 172673, - 172679, 172684, 172689, 172695, 172700, 172705, 172711, 172717, 172723, - 172729, 172735, 172740, 172745, 172751, 172757, 172762, 172768, 172774, - 172780, 172786, 172792, 172798, 172804, 172810, 172816, 172822, 172828, - 172834, 172840, 172846, 172852, 172858, 172864, 172870, 172876, 172881, - 172886, 172892, 172897, 172902, 172908, 172914, 172920, 172926, 172932, - 172937, 172942, 172948, 172954, 172959, 172965, 172971, 172977, 172983, - 172989, 172995, 173001, 173007, 173013, 173019, 173025, 173031, 173036, - 173041, 173045, 173050, 173057, 173061, 0, 0, 0, 0, 173066, 173071, - 173075, 173079, 173083, 173087, 173091, 173095, 173099, 173103, 0, 0, 0, - 0, 173107, 173113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 173119, 173124, 173129, 173134, 173139, 173144, - 173149, 173154, 173159, 173164, 173170, 173176, 173182, 173188, 173194, - 173200, 173206, 173212, 173218, 173225, 173232, 173239, 173247, 173255, - 173263, 173271, 173279, 173287, 173293, 173299, 173305, 173312, 173319, - 173326, 173333, 173340, 173347, 173354, 173361, 173368, 173375, 173382, - 173389, 173396, 173403, 173410, 173416, 173422, 173428, 173434, 173440, - 173447, 173454, 173461, 173468, 173475, 173482, 173489, 173496, 173503, - 173508, 173516, 173524, 173532, 173538, 173545, 173552, 173561, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 173569, 173574, 173579, 173584, 173589, 173594, 173599, 173604, - 173609, 173614, 173620, 173626, 173632, 173638, 173644, 173650, 173656, - 173662, 173668, 173675, 173682, 173689, 173697, 173705, 173713, 173721, - 173729, 173737, 173743, 173749, 173755, 173762, 173769, 173776, 173783, - 173790, 173797, 173804, 173811, 173818, 173825, 173832, 173839, 173846, - 173853, 173860, 173865, 173872, 173879, 173886, 173893, 173900, 173907, - 173914, 173921, 173929, 173939, 173949, 173957, 173966, 173974, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173982, 173986, 173990, - 173994, 0, 173998, 174002, 174006, 174010, 174014, 174018, 174022, - 174026, 174030, 174034, 174038, 174042, 174046, 174050, 174054, 174058, - 174062, 174066, 174070, 174074, 174078, 174082, 174086, 174090, 174096, - 174102, 174108, 0, 174114, 174119, 0, 174124, 0, 0, 174129, 0, 174134, - 174139, 174144, 174149, 174154, 174159, 174164, 174169, 174174, 174179, - 0, 174184, 174189, 174194, 174199, 0, 174204, 0, 174209, 0, 0, 0, 0, 0, - 0, 174214, 0, 0, 0, 0, 174220, 0, 174226, 0, 174232, 0, 174238, 174244, - 174250, 0, 174256, 174262, 0, 174268, 0, 0, 174274, 0, 174280, 0, 174286, - 0, 174292, 0, 174300, 0, 174308, 174314, 0, 174320, 0, 0, 174326, 174332, - 174338, 174344, 0, 174350, 174356, 174362, 174368, 174374, 174380, - 174386, 0, 174392, 174398, 174404, 174410, 0, 174416, 174422, 174428, - 174434, 0, 174442, 0, 174450, 174456, 174462, 174468, 174474, 174480, - 174486, 174492, 174498, 174504, 0, 174510, 174516, 174522, 174528, - 174534, 174540, 174546, 174552, 174558, 174564, 174570, 174576, 174582, - 174588, 174594, 174600, 174606, 0, 0, 0, 0, 0, 174612, 174618, 174624, 0, - 174630, 174636, 174642, 174648, 174654, 0, 174660, 174666, 174672, - 174678, 174684, 174690, 174696, 174702, 174708, 174714, 174720, 174726, - 174732, 174738, 174744, 174750, 174756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174762, 174772, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174780, 174787, 174794, 174801, - 174807, 174814, 174821, 174827, 174834, 174841, 174848, 174856, 174864, - 174872, 174880, 174888, 174896, 174903, 174910, 174917, 174925, 174933, - 174941, 174949, 174957, 174965, 174972, 174979, 174986, 174994, 175002, - 175010, 175018, 175026, 175034, 175039, 175044, 175049, 175054, 175059, - 175064, 175069, 175074, 175079, 0, 0, 0, 0, 175084, 175091, 175096, - 175101, 175106, 175111, 175116, 175121, 175126, 175131, 175136, 175141, - 175146, 175151, 175156, 175161, 175166, 175171, 175176, 175181, 175186, - 175191, 175196, 175201, 175206, 175211, 175216, 175221, 175226, 175231, - 175236, 175241, 175246, 175251, 175256, 175261, 175266, 175271, 175276, - 175281, 175286, 175291, 175296, 175301, 175306, 175311, 175316, 175321, - 175326, 175331, 175336, 175342, 175347, 175352, 175357, 175362, 175367, - 175372, 175377, 175382, 175387, 175392, 175397, 175402, 175407, 175412, - 175417, 175422, 175427, 175432, 175437, 175442, 175447, 175452, 175457, - 175462, 175467, 175472, 175477, 175482, 175487, 175492, 175497, 175502, - 175507, 175512, 175517, 175522, 175527, 175532, 175537, 175542, 175547, - 175552, 175557, 175562, 175567, 175572, 175577, 175582, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 175587, 175593, 175602, 175610, 175618, 175627, 175636, - 175645, 175654, 175663, 175672, 175681, 175690, 175699, 175708, 0, 0, - 175717, 175726, 175734, 175742, 175751, 175760, 175769, 175778, 175787, - 175796, 175805, 175814, 175823, 175832, 175841, 0, 175849, 175858, - 175866, 175874, 175883, 175892, 175901, 175910, 175919, 175928, 175937, - 175946, 175955, 175964, 175973, 0, 175980, 175989, 175997, 176005, - 176014, 176023, 176032, 176041, 176050, 176059, 176068, 176077, 176086, - 176095, 176104, 176111, 176117, 176123, 176129, 176135, 176141, 176147, - 176153, 176159, 176165, 176171, 176177, 176183, 176189, 176195, 176201, - 176207, 176213, 176219, 176225, 176231, 176237, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 176243, 176250, 176255, 176259, 176263, 176267, 176272, 176277, - 176282, 176287, 176292, 176297, 176304, 176313, 176319, 176323, 176332, - 176337, 176343, 176349, 176355, 176360, 176366, 176372, 176378, 176383, - 176389, 176395, 176400, 176406, 176412, 176417, 176423, 176429, 176434, - 176440, 176446, 176451, 176457, 176463, 176469, 176475, 176481, 176492, - 176499, 176505, 176508, 176511, 176514, 176519, 176525, 176531, 176537, - 176542, 176548, 176554, 176560, 176565, 176571, 176577, 176582, 176588, - 176594, 176599, 176605, 176611, 176616, 176622, 176628, 176633, 176639, - 176645, 176651, 176657, 176663, 176666, 176669, 176672, 176675, 176678, - 176681, 176688, 176696, 176704, 176712, 176719, 176727, 176735, 176743, - 176750, 176758, 176766, 176773, 176781, 176789, 176796, 176804, 176812, - 176819, 176827, 176835, 176842, 176850, 176858, 176866, 176874, 176882, - 176887, 176892, 176897, 176900, 176908, 176913, 176920, 176928, 176936, - 176944, 176951, 176959, 176967, 176975, 176982, 176990, 176998, 177005, - 177013, 177021, 177028, 177036, 177044, 177051, 177059, 177067, 177074, - 177082, 177090, 177098, 177106, 177114, 177124, 177129, 177133, 177137, - 177142, 177147, 177150, 177153, 177156, 177159, 177162, 177165, 177168, - 177171, 177174, 177180, 177183, 177187, 177192, 177196, 177201, 177206, - 177212, 177218, 177224, 177229, 177237, 177243, 177246, 177249, 177252, - 177255, 177258, 177261, 177264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177269, 177276, - 177284, 177292, 177300, 177307, 177315, 177323, 177331, 177338, 177346, - 177354, 177361, 177369, 177377, 177384, 177392, 177400, 177407, 177415, - 177423, 177430, 177438, 177446, 177454, 177462, 177470, 177475, 177479, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177482, 177488, 177494, 177500, - 177504, 177510, 177516, 177522, 177528, 177534, 177540, 177546, 177552, - 177558, 177564, 177570, 177576, 177582, 177588, 177594, 177600, 177606, - 177612, 177618, 177624, 177630, 177636, 177642, 177648, 177654, 177660, - 177666, 177672, 177678, 177684, 177690, 177696, 177702, 177708, 177714, - 177720, 177726, 177732, 177738, 0, 0, 0, 0, 177744, 177755, 177766, - 177777, 177788, 177799, 177810, 177821, 177832, 0, 0, 0, 0, 0, 0, 0, - 177843, 177848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177853, 177859, - 177865, 177871, 177877, 177883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177889, 177891, 177893, 177897, - 177902, 177907, 177909, 177915, 177920, 177922, 177928, 177932, 177934, - 177938, 177944, 177950, 177956, 177961, 177966, 177973, 177980, 177987, - 177992, 177999, 178006, 178013, 178017, 178024, 178033, 178042, 178049, - 178054, 178058, 178062, 178064, 178067, 178070, 178077, 178084, 178094, - 178099, 178104, 178109, 178114, 178116, 178122, 178126, 178128, 178130, - 178132, 178134, 178138, 178142, 178146, 178148, 178152, 178154, 178158, - 178160, 178162, 178164, 178166, 178171, 178176, 178178, 178184, 178188, - 178192, 178200, 178202, 178204, 178206, 178208, 178210, 178212, 178214, - 178216, 178218, 178220, 178224, 178228, 178230, 178232, 178234, 178236, - 178238, 178243, 178249, 178253, 178257, 178261, 178265, 178270, 178274, - 178276, 178278, 178282, 178288, 178290, 178292, 178294, 178298, 178307, - 178313, 178317, 178321, 178323, 178325, 178328, 178330, 178332, 178334, - 178338, 178340, 178344, 178349, 178351, 178356, 178362, 178369, 178373, - 178377, 178381, 178385, 178391, 178395, 178403, 178410, 178412, 178414, - 178418, 178422, 178424, 178428, 178432, 178434, 178438, 178440, 178444, - 178448, 178452, 178456, 178460, 178464, 178468, 178472, 178478, 178482, - 178486, 178497, 178502, 178506, 178510, 178516, 178520, 178524, 178528, - 178535, 178542, 178546, 178550, 178554, 178558, 178562, 178569, 178571, - 178575, 178577, 178579, 178583, 178587, 178591, 178593, 178597, 178601, - 178605, 178609, 178613, 178615, 178619, 178621, 178627, 178630, 178635, - 178637, 178639, 178642, 178644, 178646, 178649, 178656, 178663, 178670, - 178675, 178679, 178681, 178683, 178685, 178689, 178691, 178695, 178699, - 178703, 178705, 178709, 178711, 178715, 178719, 178726, 178728, 178737, - 178746, 178755, 178761, 178763, 178768, 178772, 178776, 178778, 178784, - 178788, 178790, 178794, 178798, 178800, 178804, 178809, 178813, 178819, - 178825, 178827, 178829, 178835, 178837, 178841, 178845, 178847, 178851, - 178853, 178857, 178861, 178865, 178868, 178871, 178876, 178881, 178883, - 178886, 178888, 178895, 178899, 178901, 178908, 178915, 178922, 178929, - 178936, 178938, 178940, 178942, 178946, 178948, 178950, 178952, 178954, - 178956, 178958, 178960, 178962, 178964, 178966, 178968, 178970, 178972, - 178974, 178976, 178978, 178980, 178982, 178984, 178986, 178988, 178990, - 178994, 178996, 178998, 179000, 179004, 179006, 179010, 179012, 179014, - 179018, 179022, 179028, 179030, 179032, 179034, 179036, 179040, 179044, - 179046, 179050, 179054, 179058, 179062, 179066, 179070, 179074, 179078, - 179082, 179086, 179090, 179094, 179098, 179102, 179106, 179110, 179114, - 179118, 179120, 179122, 179124, 179126, 179128, 179130, 179132, 179140, - 179148, 179156, 179164, 179169, 179174, 179179, 179183, 179187, 179192, - 179197, 179199, 179203, 179205, 179207, 179209, 179211, 179213, 179215, - 179217, 179221, 179223, 179225, 179227, 179231, 179235, 179239, 179243, - 179247, 179249, 179255, 179261, 179263, 179265, 179267, 179269, 179271, - 179280, 179287, 179294, 179298, 179305, 179310, 179317, 179326, 179331, - 179335, 179339, 179341, 179345, 179347, 179351, 179355, 179357, 179361, - 179365, 179369, 179371, 179373, 179379, 179381, 179383, 179385, 179389, - 179393, 179395, 179399, 179401, 179403, 179406, 179410, 179412, 179416, - 179418, 179420, 179425, 179427, 179431, 179435, 179438, 179442, 179446, - 179450, 179454, 179458, 179462, 179466, 179471, 179475, 179479, 179488, - 179493, 179496, 179498, 179501, 179504, 179509, 179511, 179514, 179519, - 179523, 179526, 179530, 179534, 179537, 179542, 179546, 179550, 179554, - 179558, 179564, 179570, 179576, 179582, 179587, 179598, 179600, 179604, - 179606, 179608, 179612, 179616, 179618, 179622, 179628, 179633, 179639, - 179641, 179645, 179649, 179656, 179663, 179667, 179669, 179671, 179675, - 179677, 179681, 179685, 179689, 179691, 179693, 179700, 179704, 179708, - 179712, 179716, 179720, 179722, 179726, 179728, 179730, 179734, 179736, - 179740, 179744, 179750, 179754, 179758, 179762, 179764, 179767, 179771, - 179778, 179787, 179796, 179805, 179814, 179816, 179820, 179822, 179826, - 179837, 179841, 179847, 179853, 179858, 179860, 179865, 179869, 179871, - 179873, 179875, 179879, 179883, 179887, 179892, 179903, 179919, 179932, - 179945, 179949, 179953, 179959, 179961, 179969, 179977, 179979, 179983, - 179989, 179995, 180002, 180009, 180011, 180013, 180017, 180019, 180025, - 180027, 180030, 180034, 180040, 180046, 180057, 180063, 180070, 180078, - 180082, 180090, 180098, 180104, 180110, 180117, 180119, 180123, 180125, - 180127, 180132, 180134, 180136, 180138, 180140, 180144, 180154, 180160, - 180164, 180168, 180172, 180178, 180184, 180190, 180196, 180201, 180206, - 180212, 180218, 180225, 180232, 180239, 180246, 180251, 180259, 180263, - 180272, 180281, 180287, 180291, 180295, 180299, 180302, 180307, 180309, - 180311, 180313, 180320, 180325, 180332, 180339, 180346, 180354, 180362, - 180370, 180378, 180386, 180394, 180402, 180410, 180418, 180424, 180430, - 180436, 180442, 180448, 180454, 180460, 180466, 180472, 180478, 180484, - 180490, 180493, 180502, 180511, 180513, 180520, 180524, 180526, 180528, - 180532, 180538, 180542, 180544, 180554, 180560, 180564, 180566, 180570, - 180572, 180576, 180583, 180590, 180597, 180602, 180607, 180616, 180622, - 180627, 180631, 180636, 180640, 180647, 180651, 180654, 180658, 180664, - 180670, 180674, 180678, 180683, 180689, 180698, 180709, 180715, 180721, - 180727, 180737, 180752, 180761, 180769, 180777, 180785, 180793, 180801, - 180809, 180817, 180825, 180833, 180841, 180849, 180857, 180860, 180864, - 180869, 180874, 180876, 180880, 180889, 180898, 180906, 180910, 180914, - 180919, 180924, 180929, 180931, 180936, 180940, 180942, 180946, 180950, - 180956, 180961, 180969, 180974, 180979, 180984, 180991, 180994, 180996, - 181000, 181005, 181011, 181015, 181019, 181025, 181031, 181033, 181037, - 181041, 181045, 181049, 181053, 181055, 181057, 181059, 181061, 181067, - 181073, 181077, 181079, 181081, 181083, 181092, 181096, 181103, 181110, - 181112, 181115, 181119, 181125, 181129, 181133, 181135, 181143, 181147, - 181151, 181156, 181160, 181165, 181170, 181175, 181180, 181185, 181190, - 181195, 181200, 181204, 181210, 181214, 181220, 181225, 181232, 181238, - 181246, 181250, 181257, 181261, 181265, 181269, 181274, 181279, 181281, - 181285, 181294, 181302, 181311, 181325, 181339, 181353, 181360, 181367, - 181371, 181380, 181388, 181392, 181401, 181408, 181412, 181416, 181420, - 181424, 181431, 181435, 181439, 181443, 181447, 181454, 181463, 181472, - 181479, 181491, 181503, 181507, 181511, 181515, 181519, 181523, 181527, - 181535, 181543, 181552, 181556, 181560, 181564, 181568, 181572, 181576, - 181582, 181589, 181593, 181605, 181613, 181617, 181621, 181625, 181629, - 181635, 181642, 181653, 181663, 181674, 181685, 181694, 181705, 181711, - 181717, 181723, 181729, 181735, 181739, 181746, 181755, 181762, 181768, - 181772, 181776, 181780, 181789, 181801, 181805, 181812, 181819, 181826, - 181834, 181841, 181849, 181857, 181866, 181874, 181883, 181892, 181902, - 181911, 181921, 181931, 181942, 181952, 181963, 181970, 181978, 181985, - 181993, 182001, 182010, 182018, 182027, 182034, 182046, 182053, 182065, - 182068, 182072, 182075, 182079, 182085, 182092, 182099, 182107, 182112, - 182118, 182129, 182139, 182150, 182155, 182160, 182166, 182171, 182178, - 182182, 182188, 182190, 182192, 182196, 182200, 182204, 182213, 182215, - 182217, 182220, 182222, 182224, 182228, 182230, 182234, 182236, 182240, - 182242, 182244, 182248, 182252, 182258, 182260, 182264, 182266, 182270, - 182274, 182278, 182282, 182284, 182286, 182290, 182294, 182298, 182302, - 182304, 182306, 182308, 182314, 182319, 182322, 182330, 182338, 182340, - 182345, 182348, 182353, 182364, 182371, 182376, 182381, 182383, 182387, - 182389, 182393, 182395, 182399, 182403, 182406, 182409, 182411, 182414, - 182416, 182420, 182422, 182424, 182426, 182430, 182432, 182436, 182439, - 182446, 182449, 182454, 182457, 182460, 182465, 182469, 182473, 182477, - 182479, 182484, 182487, 182491, 182493, 182495, 182499, 182501, 0, 0, 0, - 0, 182503, 182505, 182509, 182511, 182515, 182520, 182522, 182526, - 182528, 182532, 182536, 182542, 182546, 182551, 182554, 182558, 182562, - 0, 0, 0, 182566, 182568, 182574, 182578, 182582, 182584, 182588, 182590, - 182592, 182596, 182598, 182602, 182606, 0, 0, 0, 182610, 182615, 182620, - 182625, 182630, 182635, 182640, 182647, 182654, 182661, 182668, 182673, - 182678, 182683, 182688, 182695, 182701, 182708, 182715, 182722, 182727, - 182732, 182737, 182742, 182747, 182754, 182761, 182766, 182771, 182778, - 182785, 182793, 182801, 182808, 182815, 182823, 182831, 182839, 182846, - 182856, 182867, 182872, 182879, 182886, 182893, 182901, 182909, 182920, - 182928, 182936, 182944, 182949, 182954, 182959, 182964, 182969, 182974, - 182979, 182984, 182989, 182994, 182999, 183004, 183011, 183016, 183021, - 183028, 183033, 183038, 183043, 183048, 183053, 183058, 183063, 183068, - 183073, 183078, 183083, 183088, 183095, 183103, 183108, 183113, 183120, - 183125, 183130, 183135, 183142, 183147, 183154, 183159, 183166, 183171, - 183180, 183189, 183194, 183199, 183204, 183209, 183214, 183219, 183224, - 183229, 183234, 183239, 183244, 183249, 183254, 183262, 183270, 183275, - 183280, 183285, 183290, 183295, 183301, 183307, 183312, 183314, 0, 0, 0, - 0, 183318, 183320, 183322, 183324, 183326, 183328, 183336, 183344, - 183352, 183360, 183366, 183372, 183376, 183380, 183386, 183392, 183401, - 183405, 183410, 183416, 183420, 183425, 183429, 183433, 183439, 183445, - 183455, 183464, 183467, 183472, 183478, 183484, 183495, 183505, 183509, - 183514, 183520, 183526, 183535, 183540, 183544, 183549, 183553, 183559, - 183565, 183571, 183575, 183578, 183582, 183585, 183588, 183593, 183598, - 183605, 183613, 183620, 183627, 183636, 183645, 183652, 183660, 183667, - 183674, 183683, 183692, 183699, 183707, 183714, 183721, 183730, 183737, - 183745, 183751, 183760, 183768, 183777, 183784, 183794, 183805, 183813, - 183821, 183830, 183838, 183846, 183855, 183863, 183873, 183882, 183890, - 183898, 183907, 183910, 183915, 183918, 183923, 0, 0, 0, 0, 0, 0, 183930, - 183936, 183942, 183948, 183954, 183960, 183966, 183972, 183978, 183984, - 183990, 183996, 0, 0, 0, 0, 184002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 184006, 184014, 184023, 184031, 184040, 184049, 184059, 184068, - 184078, 184087, 184097, 184106, 0, 0, 0, 0, 184116, 184124, 184133, - 184141, 184150, 184157, 184165, 184172, 184180, 184188, 184197, 184205, - 184214, 184224, 184235, 184245, 184256, 184265, 184275, 184284, 184294, - 184303, 184313, 184322, 184332, 184340, 184349, 184357, 184366, 184374, - 184383, 184391, 184400, 184410, 184421, 184431, 184442, 184446, 184451, - 184455, 184460, 184463, 184467, 184470, 184474, 184478, 184483, 184487, - 184492, 184497, 184503, 184508, 184514, 184517, 184521, 184524, 0, 0, 0, - 0, 0, 0, 0, 0, 184528, 184531, 184535, 184538, 184542, 184547, 184552, - 184558, 184564, 184568, 0, 0, 0, 0, 0, 0, 184572, 184578, 184585, 184591, - 184598, 184606, 184614, 184623, 184632, 184637, 184643, 184648, 184654, - 184661, 184668, 184676, 184684, 184691, 184699, 184706, 184714, 184723, - 184732, 184742, 184752, 184758, 184765, 184771, 184778, 184786, 184794, - 184803, 184812, 184820, 184829, 184837, 184846, 184856, 184866, 184877, - 0, 0, 0, 0, 0, 0, 0, 0, 184888, 184893, 184899, 184904, 184910, 184919, - 184929, 184938, 184948, 184955, 184963, 184970, 184978, 184985, 184994, - 185003, 185012, 185017, 185024, 185031, 185038, 185043, 185048, 185053, - 185058, 185065, 185072, 185079, 185086, 185093, 0, 0, 185102, 185112, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 185124, 185134, 185143, 185148, 185157, 185165, 185173, - 185180, 185184, 185189, 185196, 185205, 185216, 185220, 185223, 185227, - 185231, 185235, 185239, 185244, 185248, 185252, 185257, 185261, 185265, - 185271, 185277, 185284, 185288, 185292, 185294, 185304, 185313, 185320, - 185324, 185328, 185338, 185342, 185346, 185350, 185354, 185362, 185371, - 185384, 185395, 185406, 185422, 185431, 185440, 185444, 185446, 185451, - 185453, 185455, 185461, 185465, 185467, 185473, 185475, 185477, 185481, - 185483, 185487, 185489, 185493, 185497, 185502, 185506, 185510, 185512, - 185516, 185518, 185524, 185530, 185536, 185540, 185546, 185550, 185557, - 185559, 185563, 185565, 185567, 185569, 185571, 185573, 185575, 185579, - 185583, 185590, 185594, 185596, 185601, 185603, 185605, 185607, 185609, - 185613, 185617, 185619, 185624, 185629, 185631, 185633, 185635, 185637, - 185642, 185644, 185648, 185652, 185654, 185658, 185660, 185673, 185677, - 185684, 185696, 185708, 185712, 185716, 185718, 185722, 185730, 185737, - 185739, 185743, 185745, 185749, 185753, 185755, 185759, 185761, 185763, - 185767, 185769, 185771, 185773, 185775, 185777, 185781, 185783, 185785, - 185787, 185789, 185791, 185793, 185795, 185799, 185803, 185805, 185807, - 185809, 185811, 185813, 185815, 185817, 185819, 185821, 185823, 185825, - 185827, 185829, 185831, 185833, 185835, 185837, 185839, 185841, 185843, - 185845, 185847, 185849, 185851, 185853, 185855, 185859, 185863, 185871, - 185879, 185885, 185892, 185894, 185896, 185898, 185900, 185902, 185904, - 185908, 185915, 185919, 185923, 185927, 185931, 185935, 185937, 185941, - 185945, 185947, 185949, 185951, 185953, 185955, 185959, 185963, 185967, - 185969, 185973, 185977, 185981, 185986, 185988, 185990, 185994, 185998, - 186003, 186011, 186015, 186023, 186025, 186027, 186029, 186031, 186033, - 186035, 186037, 186039, 186043, 186047, 186049, 186051, 186053, 186055, - 186061, 186063, 186069, 186073, 186077, 186082, 186084, 186086, 186090, - 186092, 186094, 186096, 186098, 186102, 186107, 186112, 186116, 186120, - 186122, 186124, 186129, 186134, 186136, 186138, 186142, 186148, 186154, - 186160, 186166, 186172, 186178, 186189, 186200, 186212, 186223, 186234, - 186245, 186256, 186267, 186278, 186289, 186300, 186311, 186322, 186333, - 186344, 186356, 186368, 186380, 186392, 186404, 186416, 186430, 186444, - 186459, 186465, 186471, 186477, 186483, 186489, 186495, 186501, 186507, - 186513, 186519, 186525, 186531, 186538, 186545, 186552, 186559, 186566, - 186573, 186587, 186601, 186616, 186630, 186644, 186658, 186672, 186686, - 186700, 186714, 186728, 186742, 186756, 186770, 186784, 186799, 186814, - 186829, 186844, 186859, 186874, 186888, 186902, 186917, 186922, 186927, - 186933, 186944, 186955, 186967, 186972, 186977, 186982, 186987, 186992, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186997, 187003, 187009, 187015, - 187021, 187027, 187033, 187039, 187044, 187049, 187054, 187059, 187064, - 187069, 0, 0, 187074, 187078, 187082, 187084, 187086, 187090, 187095, - 187099, 187103, 187108, 187112, 187114, 187116, 0, 0, 0, 187118, 187120, - 187122, 187124, 187126, 187130, 187132, 187136, 187138, 0, 0, 0, 0, 0, 0, - 0, 187140, 187144, 187146, 187148, 187150, 187154, 187156, 187160, - 187162, 187165, 187167, 187171, 187173, 187175, 187176, 187178, 187180, - 187182, 187186, 187188, 187190, 187194, 187196, 187198, 187200, 187202, - 187206, 187210, 187213, 187215, 187221, 187225, 187227, 187229, 187231, - 187233, 187235, 187239, 187241, 187243, 187245, 187247, 187251, 187256, - 187258, 187260, 0, 187262, 187264, 187268, 187270, 187274, 187278, - 187282, 0, 0, 0, 0, 0, 0, 0, 0, 187287, 187289, 187291, 187293, 187297, - 187299, 187301, 187303, 187305, 187307, 187311, 187313, 187315, 187319, - 0, 0, 0, 0, 187323, 187327, 187331, 187344, 187351, 187358, 187364, - 187368, 187370, 0, 0, 0, 0, 0, 0, 0, 187374, 187384, 187387, 187390, - 187395, 187400, 187409, 187413, 187418, 0, 0, 0, 0, 0, 0, 0, 187423, - 187426, 187429, 187432, 187435, 187438, 187441, 187444, 187447, 187450, - 187453, 187456, 187459, 187462, 187465, 187468, 187471, 187474, 187477, - 187480, 187483, 187486, 187489, 187492, 187495, 187498, 187501, 187504, - 187507, 187510, 187513, 187516, 187519, 187522, 187525, 187528, 187531, - 187534, 187537, 187540, 187543, 187546, 187549, 187552, 187555, 187558, - 187561, 187564, 187567, 187570, 187573, 187576, 187579, 187582, 187585, - 187588, 187591, 187594, 187597, 187600, 187603, 187615, 187626, 187638, - 187649, 187660, 187672, 187683, 187695, 187706, 187717, 187729, 187741, - 187752, 187764, 187775, 187786, 187798, 187809, 187821, 187832, 187843, - 187855, 187867, 187878, 187890, 187901, 187912, 187924, 187935, 187947, - 187958, 187969, 187981, 187993, 188004, 188016, 188027, 188038, 188050, - 188061, 188073, 188084, 188095, 188107, 188119, 188131, 188143, 188155, - 188163, 188171, 188179, 188187, 188193, 188199, 188205, 188211, 188217, - 188223, 188230, 188237, 188244, 188251, 188258, 188265, 188273, 188281, - 188289, 188297, 188305, 188312, 188318, 188324, 188331, 188337, 188344, - 188350, 188356, 188363, 188369, 188376, 188382, 188388, 188394, 188400, - 188406, 188418, 0, 188431, 188444, 188450, 188458, 188463, 188470, - 188477, 188485, 188493, 188501, 188509, 188517, 188525, 188537, 188548, - 188559, 188570, 188585, 188600, 188614, 188628, 188646, 188664, 188683, - 188701, 188719, 188737, 188744, 188752, 188756, 188761, 188767, 188773, - 188783, 188794, 188805, 188815, 188825, 188829, 188833, 188838, 188844, - 188850, 188860, 188866, 188875, 188884, 188893, 188902, 188908, 188912, - 188921, 188929, 188936, 188943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188948, 188953, 188957, 188961, 188965, 188969, 188973, 188977, 188981, - 188985, 0, 0, 0, 0, 0, 0, 188989, 188993, 188997, 189001, 189005, 189009, - 189013, 189017, 189021, 189025, 189029, 189033, 189037, 189041, 189045, - 189049, 189053, 189057, 189061, 189065, 189069, 189073, 189077, 189081, - 189085, 189089, 189093, 189097, 189101, 189105, 189109, 189113, 189117, - 189121, 189125, 189129, 189133, 189137, 189141, 189145, 189149, 189153, - 189157, 189161, 189165, 189169, 189173, 189177, 189181, 189185, 189189, - 189193, 189197, 189201, 189205, 189209, 189213, 189217, 189221, 189225, - 189229, 189233, 189237, 189241, 189245, 189249, 189253, 189257, 189261, - 189265, 189269, 189273, 189277, 189281, 189285, 189289, 189293, 189297, - 189301, 189305, 189309, 189313, 189317, 189321, 189325, 189329, 189333, - 189337, 189341, 189345, 189349, 189353, 189357, 189361, 189365, 189369, - 189373, 189377, 189381, 189385, 189389, 189393, 189397, 189401, 189405, - 189409, 189413, 189417, 189421, 189425, 189429, 189433, 189437, 189441, - 189445, 189449, 189453, 189457, 189461, 189465, 189469, 189473, 189477, - 189481, 189485, 189489, 189493, 189497, 189501, 189505, 189509, 189513, - 189517, 189521, 189525, 189529, 189533, 189537, 189541, 189545, 189549, - 189553, 189557, 189561, 189565, 189569, 189573, 189577, 189581, 189585, - 189589, 189593, 189597, 189601, 189605, 189609, 189613, 189617, 189621, - 189625, 189629, 189633, 189637, 189641, 189645, 189649, 189653, 189657, - 189661, 189665, 189669, 189673, 189677, 189681, 189685, 189689, 189693, - 189697, 189701, 189705, 189709, 189713, 189717, 189721, 189725, 189729, - 189733, 189737, 189741, 189745, 189749, 189753, 189757, 189761, 189765, - 189769, 189773, 189777, 189781, 189785, 189789, 189793, 189797, 189801, - 189805, 189809, 189813, 189817, 189821, 189825, 189829, 189833, 189837, - 189841, 189845, 189849, 189853, 189857, 189861, 189865, 189869, 189873, - 189877, 189881, 189885, 189889, 189893, 189897, 189901, 189905, 189909, - 189913, 189917, 189921, 189925, 189929, 189933, 189937, 189941, 189945, - 189949, 189953, 189957, 189961, 189965, 189969, 189973, 189977, 189981, - 189985, 189989, 189993, 189997, 190001, 190005, 190009, 190013, 190017, - 190021, 190025, 190029, 190033, 190037, 190041, 190045, 190049, 190053, - 190057, 190061, 190065, 190069, 190073, 190077, 190081, 190085, 190089, - 190093, 190097, 190101, 190105, 190109, 190113, 190117, 190121, 190125, - 190129, 190133, 190137, 190141, 190145, 190149, 190153, 190157, 190161, - 190165, 190169, 190173, 190177, 190181, 190185, 190189, 190193, 190197, - 190201, 190205, 190209, 190213, 190217, 190221, 190225, 190229, 190233, - 190237, 190241, 190245, 190249, 190253, 190257, 190261, 190265, 190269, - 190273, 190277, 190281, 190285, 190289, 190293, 190297, 190301, 190305, - 190309, 190313, 190317, 190321, 190325, 190329, 190333, 190337, 190341, - 190345, 190349, 190353, 190357, 190361, 190365, 190369, 190373, 190377, - 190381, 190385, 190389, 190393, 190397, 190401, 190405, 190409, 190413, - 190417, 190421, 190425, 190429, 190433, 190437, 190441, 190445, 190449, - 190453, 190457, 190461, 190465, 190469, 190473, 190477, 190481, 190485, - 190489, 190493, 190497, 190501, 190505, 190509, 190513, 190517, 190521, - 190525, 190529, 190533, 190537, 190541, 190545, 190549, 190553, 190557, - 190561, 190565, 190569, 190573, 190577, 190581, 190585, 190589, 190593, - 190597, 190601, 190605, 190609, 190613, 190617, 190621, 190625, 190629, - 190633, 190637, 190641, 190645, 190649, 190653, 190657, 190661, 190665, - 190669, 190673, 190677, 190681, 190685, 190689, 190693, 190697, 190701, - 190705, 190709, 190713, 190717, 190721, 190725, 190729, 190733, 190737, - 190741, 190745, 190749, 190753, 190757, 190761, 190765, 190769, 190773, - 190777, 190781, 190785, 190789, 190793, 190797, 190801, 190805, 190809, - 190813, 190817, 190821, 190825, 190829, 190833, 190837, 190841, 190845, - 190849, 190853, 190857, 190861, 190865, 190869, 190873, 190877, 190881, - 190885, 190889, 190893, 190897, 190901, 190905, 190909, 190913, 190917, - 190921, 190925, 190929, 190933, 190937, 190941, 190945, 190949, 190953, - 190957, 190961, 190965, 190969, 190973, 190977, 190981, 190985, 190989, - 190993, 190997, 191001, 191005, 191009, 191013, 191017, 191021, 191025, - 191029, 191033, 191037, 191041, 191045, 191049, 191053, 191057, 191061, - 191065, 191069, 191073, 191077, 191081, 191085, 191089, 191093, 191097, - 191101, 191105, 191109, 191113, 191117, 191121, 191125, 191129, 191133, - 191137, 191141, 191145, 191149, 191153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191157, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 191161, 191165, 191170, 191175, 191179, 191184, 191189, 191193, 191197, - 191202, 191207, 191211, 191215, 191219, 191223, 191229, 191233, 191238, - 191242, 191246, 191250, 191254, 191258, 191262, 191266, 191270, 191274, - 191278, 191282, 191287, 191292, 191297, 191302, 191308, 191314, 191321, - 191328, 191335, 191341, 191348, 191355, 191362, 191368, 191375, 191382, - 191388, 191395, 191402, 191408, 191415, 191422, 191428, 191435, 191442, - 191448, 191455, 191462, 191469, 191476, 191483, 191489, 191495, 191501, - 191507, 191512, 191518, 191524, 191531, 191538, 191545, 191551, 191558, - 191565, 191572, 191578, 191585, 191592, 191598, 191605, 191612, 191618, - 191625, 191632, 191638, 191645, 191652, 191658, 191665, 191672, 191679, - 191686, 191693, 191700, 191705, 191712, 191716, 191720, 191723, 191726, - 191729, 191732, 191735, 191738, 191741, 191744, 191747, 191750, 191753, - 191756, 191759, 191762, 191765, 191768, 191771, 191774, 191777, 191780, - 191783, 191786, 191789, 191792, 191795, 191798, 191801, 191804, 191807, - 191810, 191813, 191816, 191819, 191822, 191825, 191828, 191831, 191834, - 191837, 191840, 191843, 191846, 191849, 191852, 191855, 191858, 191861, - 191864, 191867, 191870, 191873, 191876, 191879, 191882, 191885, 191888, - 191891, 191894, 191897, 191900, 191903, 191906, 191909, 191912, 191915, - 191918, 191921, 191924, 191927, 191930, 191933, 191936, 191939, 191942, - 191945, 191948, 191951, 191954, 191957, 191960, 191963, 191966, 191969, - 191972, 191975, 191978, 191981, 191984, 191987, 191990, 191993, 191996, - 191999, 192002, 192005, 192008, 192011, 192014, 192017, 192020, 192023, - 192026, 192029, 192032, 192035, 192038, 192041, 192044, 192047, 192050, - 192053, 192056, 192059, 192062, 192065, 192068, 192071, 192074, 192077, - 192080, 192083, 192086, 192089, 192092, 192095, 192098, 192101, 192104, - 192107, 192110, 192113, 192116, 192119, 192122, 192125, 192128, 192131, - 192134, 192137, 192140, 192143, 192146, 192149, 192152, 192155, 192158, - 192161, 192164, 192167, 192170, 192173, 192176, 192179, 192182, 192185, - 192188, 192191, 192194, 192197, 192200, 192203, 192206, 192209, 192212, - 192215, 192218, 192221, 192224, 192227, 192230, 192233, 192236, 192239, - 192242, 192245, 192248, 192251, 192254, 192257, 192260, 192263, 192266, - 192269, 192272, 192275, 192278, 192281, 192284, 192287, 192290, 192293, - 192296, 192299, 192302, 192305, 192308, 192311, 192314, 192317, 192320, - 192323, 192326, 192329, 192332, 192335, 192338, 192341, 192344, 192347, - 192350, 192353, 192356, 192359, 192362, 192365, 192368, 192371, 192374, - 192377, 192380, 192383, 192386, 192389, 192392, 192395, 192398, 192401, - 192404, 192407, 192410, 192413, 192416, 192419, 192422, 192425, 192428, - 192431, 192434, 192437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 192440, 192442, 192444, 192449, 192451, 192456, 192458, 192463, 192465, - 192470, 192472, 192474, 192476, 192478, 192480, 192482, 192484, 192486, - 192488, 192492, 192496, 192498, 192500, 192504, 192508, 192513, 192515, - 192517, 192519, 192523, 192526, 192528, 192532, 192534, 192538, 192540, - 192544, 192547, 192549, 192553, 192557, 192559, 192565, 192567, 192572, - 192574, 192579, 192581, 192586, 192588, 192593, 192595, 192599, 192601, - 192605, 192607, 192614, 192616, 192618, 192620, 192625, 192627, 192629, - 192631, 192633, 192635, 192637, 192642, 192646, 192648, 192653, 192657, - 192659, 192664, 192668, 192670, 192675, 192679, 192681, 192683, 192685, - 192687, 192691, 192693, 192698, 192700, 192706, 192708, 192714, 192716, - 192718, 192720, 192724, 192726, 192733, 192735, 192742, 192744, 192750, - 192756, 192758, 192765, 192772, 192774, 192780, 192785, 192787, 192793, - 192799, 192801, 192807, 192813, 192815, 192821, 192825, 192827, 192832, - 192834, 192836, 192841, 192843, 192845, 192851, 192853, 192858, 192862, - 192864, 192869, 192873, 192875, 192881, 192883, 192887, 192889, 192893, - 192895, 192902, 192909, 192911, 192918, 192925, 192927, 192932, 192934, - 192942, 192944, 192950, 192952, 192958, 192960, 192964, 192966, 192972, - 192974, 192978, 192980, 192986, 192988, 192990, 192992, 192997, 193002, - 193004, 193013, 193015, 193025, 193030, 193037, 193044, 193049, 193054, - 193066, 193070, 193074, 193078, 193082, 193084, 193086, 193088, 193090, - 193092, 193098, 193100, 193102, 193104, 193106, 193108, 193110, 193112, - 193114, 193116, 193118, 193120, 193122, 193124, 193126, 193128, 193130, - 193132, 193138, 193145, 193150, 193158, 193166, 193171, 193177, 193179, - 193181, 193183, 193185, 193187, 193189, 193191, 193193, 193195, 193197, - 193199, 193201, 193203, 193205, 193207, 193209, 193221, 193226, 193228, - 193230, 193236, 193248, 193254, 193260, 193266, 193272, 193276, 193287, - 193289, 193291, 193293, 193295, 193297, 193299, 193301, 193303, 193305, - 193307, 193309, 193311, 193313, 193315, 193317, 193319, 193321, 193323, - 193325, 193327, 193329, 193331, 193333, 193335, 193337, 193339, 193341, - 193343, 193345, 193347, 193349, 193351, 193353, 193355, 193357, 193359, - 193361, 193363, 193365, 193367, 193369, 193371, 193373, 193375, 193377, - 193379, 193381, 193383, 193385, 193387, 193389, 193391, 193393, 193395, - 193397, 193399, 193401, 193403, 193405, 193407, 193409, 193411, 193413, - 193415, 193417, 193419, 193421, 193423, 193425, 193427, 193429, 193431, - 193433, 193435, 193437, 193439, 193441, 193443, 193445, 193447, 193449, - 193451, 193453, 193455, 193457, 193459, 193461, 193463, 193465, 193467, - 193469, 193471, 193473, 193475, 193477, 193479, 193481, 193483, 193485, - 193487, 193489, 193491, 193493, 193495, 193497, 193499, 193501, 193503, - 193505, 193507, 193509, 193511, 193513, 193515, 193517, 193519, 193521, - 193523, 193525, 193527, 193529, 193531, 193533, 193535, 193537, 193539, - 193541, 193543, 193545, 193547, 193549, 193551, 193553, 193555, 193557, - 193559, 193561, 193563, 193565, 193567, 193569, 193571, 193573, 193575, - 193577, 193579, 193581, 193583, 193585, 193587, 193589, 193591, 193593, - 193595, 193597, 193599, 193601, 193603, 193605, 193607, 193609, 193611, - 193613, 193615, 193617, 193619, 193621, 193623, 193625, 193627, 193629, - 193631, 193633, 193635, 193637, 193639, 193641, 193643, 193645, 193647, - 193649, 193651, 193653, 193655, 193657, 193659, 193661, 193663, 193665, - 193667, 193669, 193671, 193673, 193675, 193677, 193679, 193681, 193683, - 193685, 193687, 193689, 193691, 193693, 193695, 193697, 193699, 193701, - 193703, 193705, 193707, 193709, 193711, 193713, 193715, 193717, 193719, - 193721, 193723, 193725, 193727, 193729, 193731, 193733, 193735, 193737, - 193739, 193741, 193743, 193745, 193747, 193749, 193751, 193753, 193755, - 193757, 193759, 193761, 193763, 193765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 193767, 193771, 193775, 193780, 193784, 193788, 193792, - 193796, 193800, 193804, 193808, 193812, 193816, 193826, 193836, 193847, - 193858, 193868, 193878, 193888, 193898, 193912, 193926, 193940, 193954, - 193963, 193972, 193985, 193998, 194011, 194024, 194034, 194044, 194055, - 194066, 194077, 194088, 194099, 194108, 194118, 194128, 194138, 194148, - 194159, 194170, 194181, 194192, 194203, 194214, 194225, 194236, 194247, - 194258, 194269, 194283, 194294, 194308, 194316, 194327, 194335, 194343, - 194351, 194359, 194367, 194375, 194385, 194395, 194405, 194415, 194425, - 194435, 194445, 194455, 194463, 194472, 194481, 194490, 194499, 194507, - 194515, 194525, 194535, 194546, 194557, 194569, 194580, 194590, 194601, - 194611, 194622, 194630, 194637, 194644, 194651, 194658, 194665, 194672, - 194679, 194686, 194694, 194702, 194710, 194718, 194726, 194734, 194742, - 194750, 194758, 194766, 194774, 194779, 194783, 194787, 194791, 194795, - 194799, 194803, 194807, 194811, 194815, 194819, 194823, 194826, 194829, - 194833, 194837, 194841, 194845, 194849, 194853, 194857, 194861, 194865, - 194869, 194873, 194877, 194881, 194885, 194889, 194893, 194897, 194901, - 194905, 194909, 194913, 194917, 194921, 194925, 194929, 194933, 194937, - 194941, 194945, 194949, 194953, 194957, 194961, 194965, 194969, 194973, - 194977, 194981, 194985, 194989, 194993, 194997, 195001, 195005, 195009, - 195013, 195017, 195021, 195025, 195029, 195033, 195037, 195041, 195045, - 195049, 195053, 195057, 195061, 195065, 195069, 195073, 195077, 195081, - 195085, 195089, 195093, 195097, 195101, 195105, 195109, 195113, 195117, - 195121, 195125, 195129, 195133, 195137, 195141, 195145, 195149, 195153, - 195157, 195161, 195165, 195169, 195172, 195176, 195180, 195184, 195188, - 195192, 195196, 195200, 195204, 195208, 195212, 195216, 195220, 195224, - 195228, 195232, 195236, 195240, 195244, 195248, 195252, 195256, 195260, - 195264, 195268, 195272, 195276, 195280, 195284, 195288, 195292, 195296, - 195300, 195304, 195308, 195312, 195316, 195320, 195324, 195328, 195332, - 195336, 195340, 195344, 195348, 195352, 195356, 195360, 195364, 195368, - 195372, 195376, 195380, 195384, 195388, 195392, 195396, 195400, 195404, - 195408, 195412, 195416, 195420, 195424, 195428, 195432, 195436, 195440, - 195444, 195448, 195452, 195456, 195460, 195464, 195468, 195472, 195476, - 195480, 195484, 195488, 195492, 195496, 195500, 195504, 195508, 195512, - 195516, 195520, 195524, 195528, 195532, 195536, 195540, 195544, 195548, - 195552, 195556, 195560, 195564, 195568, 195572, 195576, 195580, 195584, - 195588, 195592, 195596, 195600, 195604, 195608, 195612, 195616, 195620, - 195624, 195628, 195632, 195636, 195640, 195644, 195648, 195652, 195656, - 195660, 195664, 195668, 195672, 195676, 195680, 195684, 195688, 195692, - 195696, 195700, 195704, 195708, 195712, 195716, 195720, 195724, 195728, - 195732, 195736, 195740, 195744, 195748, 195752, 195756, 195760, 195764, - 195768, 195772, 195776, 195780, 195784, 195788, 195792, 195796, 195800, - 195804, 195808, 195812, 195816, 195820, 195824, 195828, 195832, 195836, - 195840, 195844, 195848, 195852, 195856, 195860, 195864, 195868, 195872, - 195876, 195880, 195884, 195888, 195892, 195896, 195900, 195904, 195908, - 195912, 195916, 195920, 195924, 195928, 195932, 195936, 195941, 195946, - 195951, 195955, 195961, 195968, 195975, 195982, 195989, 195996, 196003, - 196010, 196017, 196024, 196031, 196038, 196045, 196052, 196058, 196065, - 196072, 196078, 196085, 196092, 196099, 196106, 196113, 196120, 196127, - 196134, 196141, 196148, 196155, 196162, 196169, 196175, 196181, 196187, - 196194, 196203, 196212, 196221, 196230, 196235, 196240, 196247, 196254, - 196261, 196268, 196275, 196281, 196287, 196293, 196299, 196305, 196311, - 196317, 196322, 196328, 196338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -/* name->code dictionary */ -static const unsigned int code_hash[] = { - 74224, 4851, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 127931, 0, 42833, 983091, - 12064, 110752, 129548, 194597, 69850, 65842, 0, 0, 0, 78159, 68476, - 72392, 1373, 0, 0, 5816, 0, 0, 4231, 0, 0, 4233, 4234, 4232, 68885, - 70351, 0, 7404, 72393, 0, 0, 0, 0, 0, 41601, 8874, 0, 0, 0, 128498, 0, 0, - 41603, 9784, 0, 9188, 41600, 0, 0, 0, 0, 3535, 0, 0, 0, 66797, 0, 74491, - 0, 3404, 100419, 0, 72411, 1759, 100417, 0, 100418, 69972, 11240, 121038, - 100416, 127764, 0, 0, 0, 0, 0, 69970, 0, 0, 9834, 43249, 2234, 983891, 0, - 0, 0, 0, 92417, 0, 74398, 12035, 0, 983074, 43548, 0, 0, 0, 0, 0, 64318, - 917549, 0, 3390, 74483, 43265, 0, 983884, 0, 0, 0, 3400, 0, 0, 11647, 0, - 0, 0, 0, 2121, 128741, 4043, 8712, 0, 983814, 0, 121172, 0, 129456, 0, 0, - 93042, 0, 0, 983875, 0, 0, 0, 11851, 0, 3181, 66002, 0, 69601, 0, 66021, - 0, 194588, 5457, 5440, 0, 93981, 65282, 2843, 5355, 0, 129333, 69971, - 5194, 11657, 128353, 0, 0, 0, 0, 0, 120766, 100525, 0, 0, 74350, 0, - 10682, 110820, 10602, 800, 70044, 118883, 0, 0, 64930, 118940, 67853, - 72001, 0, 762, 120485, 0, 0, 0, 10906, 1353, 6960, 0, 0, 5828, 8724, 0, - 0, 118548, 0, 0, 7080, 0, 128806, 122979, 0, 72388, 0, 11859, 0, 0, - 68878, 0, 0, 0, 7240, 0, 556, 0, 118544, 0, 0, 0, 72397, 0, 0, 0, 0, 0, - 0, 0, 0, 72986, 0, 0, 43931, 0, 11093, 0, 0, 122960, 7341, 66801, 68527, - 0, 1874, 0, 0, 129314, 0, 0, 0, 0, 0, 0, 7688, 0, 0, 9036, 0, 0, 66389, - 0, 121347, 0, 0, 10100, 0, 2725, 0, 0, 43981, 42128, 0, 0, 68146, 0, 0, - 0, 0, 71349, 7859, 1945, 0, 0, 0, 65918, 7188, 9992, 0, 7389, 127008, - 71341, 0, 0, 0, 528, 129681, 44017, 11429, 71347, 0, 0, 120864, 0, 0, 0, - 11530, 73102, 6188, 0, 0, 68208, 1823, 0, 0, 92928, 0, 64843, 7233, - 92929, 0, 0, 6639, 0, 0, 123149, 0, 1176, 0, 0, 8276, 128667, 0, 0, - 68892, 42931, 0, 0, 0, 0, 0, 0, 0, 5388, 0, 0, 0, 11310, 0, 123607, 0, - 68888, 4199, 119264, 0, 119020, 0, 0, 9560, 0, 0, 43869, 0, 0, 0, 83172, - 0, 0, 0, 83173, 101559, 128875, 0, 0, 74327, 0, 0, 0, 0, 0, 123623, - 68886, 0, 0, 0, 8408, 64704, 0, 0, 0, 0, 118711, 67999, 0, 0, 0, 0, - 43049, 0, 43050, 73028, 0, 0, 0, 0, 0, 127396, 0, 69847, 9322, 0, 0, - 129321, 68192, 120507, 983634, 0, 0, 0, 6199, 67249, 0, 0, 0, 0, 11329, - 66285, 0, 983086, 0, 0, 0, 0, 41335, 118866, 43401, 0, 41334, 0, 0, 0, - 983484, 71997, 983483, 128114, 0, 42627, 0, 32, 6187, 0, 123619, 983480, - 3665, 121083, 42871, 983119, 41336, 0, 0, 983476, 0, 0, 0, 4412, 0, 0, 0, - 0, 119533, 0, 4181, 0, 0, 127589, 0, 0, 71453, 6181, 74755, 917895, 0, 0, - 0, 73557, 121107, 0, 0, 10073, 0, 100738, 127186, 0, 42844, 7498, 1098, - 92565, 119530, 0, 0, 10207, 0, 983233, 0, 983555, 0, 9234, 0, 6182, 0, - 92552, 0, 0, 0, 0, 5471, 9461, 6697, 0, 5473, 0, 0, 0, 0, 0, 0, 70073, 0, - 0, 7767, 8304, 41339, 0, 983494, 69450, 0, 0, 983492, 43855, 41337, 0, 0, - 0, 129706, 0, 0, 0, 72396, 0, 0, 0, 42633, 0, 0, 0, 0, 0, 0, 0, 70005, - 129506, 0, 0, 0, 129580, 69817, 128299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, - 41617, 0, 0, 0, 128853, 0, 0, 0, 0, 0, 128529, 12113, 0, 42772, 0, 0, - 7693, 10749, 67485, 65210, 5773, 978, 128134, 0, 41619, 10239, 0, 0, 0, - 74328, 0, 9748, 0, 0, 0, 0, 0, 0, 0, 70681, 0, 72811, 0, 67464, 0, 92776, - 0, 0, 2379, 11325, 0, 0, 67854, 0, 78547, 42209, 0, 120392, 2369, 0, - 984003, 984004, 0, 0, 73936, 7008, 69415, 122919, 0, 43841, 2367, 127827, - 983888, 0, 2375, 8060, 6194, 0, 0, 119084, 0, 0, 0, 0, 6961, 0, 0, 0, - 68426, 0, 42862, 0, 0, 6192, 127900, 42771, 0, 0, 11435, 128445, 118797, - 120800, 0, 12892, 0, 128621, 67149, 0, 0, 0, 0, 120707, 0, 0, 19954, 0, - 121164, 8983, 0, 0, 0, 0, 0, 6198, 121344, 0, 196, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 983512, 41323, 0, 0, 92289, 0, 0, 0, 983508, 41321, - 12907, 3048, 7752, 41320, 0, 0, 12819, 111247, 72127, 0, 0, 0, 0, 0, - 72971, 0, 0, 0, 0, 78650, 78649, 0, 41326, 0, 11806, 43167, 0, 1245, 0, - 66463, 0, 0, 0, 0, 0, 194619, 0, 194618, 0, 0, 194620, 0, 70403, 325, - 12874, 128454, 74178, 0, 0, 119110, 0, 0, 0, 0, 0, 0, 983563, 92175, 0, - 125016, 0, 121049, 0, 0, 0, 0, 0, 0, 110844, 11776, 0, 19908, 0, 0, 0, - 8753, 69278, 0, 0, 9511, 43493, 0, 93032, 6205, 0, 0, 0, 0, 0, 78928, 0, - 0, 120269, 0, 41607, 0, 0, 120617, 0, 0, 0, 7005, 41609, 9580, 0, 401, 0, - 43779, 0, 127962, 0, 65486, 0, 12857, 0, 11983, 0, 0, 0, 121371, 0, - 194971, 74258, 983647, 0, 0, 0, 0, 0, 8295, 6200, 0, 127864, 0, 0, 71435, - 0, 92523, 0, 128631, 0, 0, 125197, 0, 100426, 0, 127556, 0, 0, 0, 64775, - 0, 68862, 120590, 0, 0, 129959, 8074, 8199, 126641, 1907, 127269, 4432, - 127271, 10808, 120668, 127272, 127259, 3888, 127261, 72724, 127263, - 127262, 127265, 123169, 121195, 127250, 66879, 127252, 100422, 66023, - 67363, 7663, 0, 0, 0, 0, 66321, 0, 12814, 127248, 127169, 0, 0, 194603, - 7641, 92694, 0, 0, 0, 0, 74320, 120818, 120268, 0, 128475, 0, 110627, 0, - 9622, 128972, 120264, 0, 0, 0, 0, 68319, 0, 0, 71484, 118613, 0, 0, - 69906, 0, 0, 947, 0, 194586, 129059, 10969, 119935, 7613, 119937, 119936, - 4795, 119930, 119933, 7376, 0, 0, 0, 72343, 69373, 0, 0, 0, 119919, 7216, - 119921, 7217, 119915, 7218, 119917, 7219, 119927, 119926, 119929, 119928, - 7213, 119922, 7214, 7215, 128622, 0, 8880, 7685, 128849, 0, 0, 119618, - 119853, 8187, 119913, 12815, 7236, 7915, 71906, 0, 121284, 0, 0, 0, 0, 0, - 0, 0, 122969, 0, 0, 0, 0, 0, 10468, 0, 0, 0, 0, 0, 0, 0, 0, 917909, 0, - 110633, 1616, 3795, 67732, 11529, 0, 126225, 0, 0, 1138, 194577, 12677, - 0, 0, 3239, 0, 0, 194809, 194583, 0, 42164, 0, 11778, 67473, 43259, - 118543, 119073, 122975, 0, 0, 67094, 129638, 0, 78421, 128123, 78418, 0, - 0, 0, 0, 43959, 43960, 0, 72257, 0, 9359, 78416, 0, 0, 0, 6662, 0, 0, - 3863, 0, 41329, 55266, 0, 127822, 41328, 75026, 194569, 129516, 0, 0, - 2178, 119595, 569, 0, 0, 0, 119085, 110669, 0, 0, 11610, 11368, 0, - 194570, 41331, 1006, 127747, 120883, 1550, 8201, 0, 194811, 5499, 43956, - 77908, 77910, 77906, 43957, 77904, 77905, 128410, 0, 0, 129581, 100447, - 43955, 77913, 122989, 0, 5511, 0, 983721, 0, 69241, 8255, 5512, 128560, - 119560, 127858, 64313, 127928, 5906, 1119, 128180, 67088, 983367, 0, - 113798, 0, 66423, 0, 0, 0, 67089, 0, 0, 0, 0, 128177, 983728, 0, 0, 0, - 5821, 6186, 129960, 128034, 19961, 0, 983719, 0, 65138, 302, 41113, - 41115, 0, 6637, 5907, 128789, 0, 43642, 0, 128625, 0, 70345, 5513, 6666, - 100567, 78442, 5510, 0, 0, 0, 983725, 78437, 0, 0, 0, 110838, 0, 0, 0, - 92710, 0, 0, 0, 0, 0, 74497, 92395, 120511, 6929, 69412, 0, 110835, - 64442, 0, 0, 74496, 0, 6674, 43397, 0, 1476, 0, 0, 72276, 3233, 0, 0, - 10164, 118555, 0, 3530, 67243, 0, 111219, 6656, 0, 0, 74647, 8512, 72275, - 74261, 8967, 0, 0, 0, 72277, 7986, 73782, 120556, 9006, 983562, 72273, 0, - 7853, 0, 983360, 0, 0, 0, 0, 983971, 0, 0, 0, 0, 0, 0, 0, 0, 127971, - 67983, 13296, 517, 0, 0, 0, 41528, 19923, 65454, 73518, 0, 0, 10531, - 7784, 41526, 71727, 0, 8057, 1126, 73895, 0, 0, 130040, 119186, 4251, - 8235, 43142, 0, 489, 71733, 4250, 71731, 110721, 43151, 94177, 71725, 0, - 121238, 0, 0, 0, 110726, 0, 8711, 6183, 110722, 110723, 0, 0, 7623, 0, 0, - 9235, 12760, 74176, 0, 0, 0, 0, 3743, 11514, 11078, 74582, 0, 0, 126597, - 0, 0, 0, 0, 983907, 267, 3393, 127504, 2364, 0, 69233, 6958, 0, 6201, 0, - 42360, 0, 10652, 41612, 917802, 3402, 917801, 3398, 0, 0, 0, 3391, 70683, - 0, 92541, 128017, 126087, 126590, 0, 12767, 0, 983380, 64261, 0, 127537, - 70852, 70347, 0, 6673, 0, 0, 129346, 12438, 0, 0, 0, 71128, 0, 9053, - 43954, 74523, 0, 0, 0, 6195, 0, 6660, 0, 917760, 917793, 0, 12629, 0, 0, - 0, 0, 0, 127940, 0, 0, 0, 65448, 0, 0, 121084, 129688, 43949, 0, 78099, - 0, 983385, 0, 0, 0, 5741, 1131, 0, 0, 74862, 0, 43952, 42533, 119598, - 78107, 0, 0, 43950, 121297, 118990, 7691, 43951, 578, 0, 0, 0, 42514, - 74547, 74196, 120608, 74561, 0, 983976, 0, 0, 0, 0, 0, 0, 0, 0, 7241, 0, - 93846, 119167, 0, 12811, 78082, 3946, 0, 10998, 66807, 673, 0, 0, 0, 0, - 119301, 0, 68890, 0, 0, 78085, 10267, 0, 74560, 78083, 0, 8729, 0, 0, 0, - 0, 0, 0, 0, 119296, 0, 0, 0, 2181, 983463, 731, 0, 71904, 128316, 0, - 73539, 0, 1175, 0, 68167, 0, 0, 10793, 0, 67644, 7723, 983458, 0, 0, 0, - 0, 5273, 0, 5269, 0, 69607, 2404, 5267, 124967, 124913, 0, 5277, 0, 0, - 6189, 65469, 1314, 0, 0, 118873, 8785, 0, 0, 127527, 68414, 43535, 9204, - 0, 3879, 0, 71696, 6197, 9497, 0, 7567, 64484, 78128, 41390, 41379, - 41882, 67647, 67279, 70085, 0, 121413, 41388, 64446, 41392, 64288, 41387, - 0, 8706, 10675, 0, 700, 0, 5775, 0, 7088, 74756, 7499, 0, 78120, 78111, - 67251, 126557, 0, 0, 128945, 10311, 78115, 6665, 11115, 0, 7618, 10821, - 11455, 0, 64632, 64447, 0, 0, 78093, 78091, 0, 0, 65033, 0, 6668, 0, 0, - 0, 656, 69686, 65037, 0, 0, 127024, 0, 0, 0, 0, 73014, 0, 0, 917774, - 9702, 0, 92273, 66580, 118895, 66683, 43640, 3417, 0, 6832, 0, 917768, 0, - 917767, 118583, 4935, 11906, 0, 0, 67296, 92896, 3651, 0, 67294, 70848, - 0, 67292, 0, 12983, 0, 55272, 0, 0, 1439, 0, 74897, 0, 0, 0, 78373, 0, - 42087, 3063, 0, 0, 7838, 0, 129282, 0, 0, 67968, 0, 128582, 9078, 92446, - 0, 0, 0, 0, 0, 0, 119586, 0, 7750, 128422, 68237, 6190, 0, 0, 0, 72340, - 9857, 7014, 9856, 0, 92620, 120547, 0, 8481, 0, 6202, 0, 10920, 67970, 0, - 0, 983297, 0, 7843, 65818, 66824, 0, 73481, 0, 0, 0, 0, 0, 6657, 207, 0, - 69728, 74819, 0, 0, 0, 0, 0, 0, 0, 0, 41368, 43974, 488, 0, 0, 71339, - 10157, 118700, 43034, 11982, 0, 0, 0, 0, 0, 41372, 6669, 8504, 72103, 0, - 41367, 129328, 119272, 0, 11726, 8261, 129793, 304, 129799, 129795, - 129822, 129807, 113683, 983239, 238, 74522, 0, 0, 19905, 120577, 122968, - 129200, 41044, 67640, 67302, 64814, 9912, 65939, 983470, 0, 0, 0, 917925, - 0, 0, 309, 6622, 0, 10858, 0, 67636, 0, 72749, 0, 0, 0, 67637, 123138, - 9712, 68680, 43970, 0, 65165, 93047, 983831, 0, 0, 0, 0, 0, 6191, 12944, - 0, 0, 67634, 43763, 0, 0, 67635, 9370, 41381, 0, 0, 123148, 118817, 0, - 3222, 121439, 0, 0, 66663, 0, 0, 0, 0, 0, 65732, 121144, 0, 983219, 0, 0, - 67309, 72192, 41383, 64568, 0, 0, 0, 0, 984009, 66725, 0, 0, 0, 0, 73766, - 67306, 3632, 128246, 0, 8376, 3648, 0, 74844, 67639, 3636, 0, 3650, 8837, - 0, 0, 0, 43250, 41562, 0, 0, 68839, 3640, 127190, 0, 11781, 0, 0, 0, 0, - 129659, 0, 126649, 0, 42080, 2529, 0, 78004, 0, 42083, 0, 0, 120531, - 67619, 0, 0, 9634, 0, 0, 0, 0, 0, 0, 0, 68841, 0, 92545, 68874, 127399, - 0, 0, 41987, 119667, 67623, 983779, 0, 925, 127156, 0, 41985, 64441, - 9586, 120988, 41984, 9217, 128372, 0, 0, 9186, 67620, 4016, 983834, 0, - 381, 983700, 0, 42077, 0, 128777, 67622, 42078, 0, 10810, 0, 4585, 19943, - 5860, 67633, 0, 0, 812, 0, 0, 0, 92518, 0, 0, 0, 0, 67629, 0, 10692, 0, - 67630, 0, 924, 0, 67631, 42616, 0, 0, 0, 67317, 67632, 0, 12771, 12736, - 12753, 0, 983753, 67626, 67722, 0, 0, 0, 0, 12751, 74906, 8542, 0, 0, - 3626, 66706, 0, 0, 3883, 64388, 0, 0, 0, 0, 0, 0, 126268, 67624, 0, - 10932, 0, 65585, 64338, 806, 0, 41884, 110845, 1318, 128828, 0, 0, 0, - 983808, 3465, 2405, 983395, 0, 12756, 65259, 69381, 983812, 12752, 5833, - 1432, 110843, 41883, 110841, 9799, 0, 41886, 0, 0, 2062, 0, 0, 0, 0, - 129376, 0, 124969, 983392, 0, 120971, 0, 118832, 0, 983286, 0, 68005, - 10622, 0, 0, 0, 6566, 71195, 0, 73780, 0, 68865, 0, 0, 0, 8284, 0, 0, 0, - 0, 0, 43023, 0, 983290, 6642, 3977, 72743, 64729, 836, 983386, 92947, 0, - 0, 0, 0, 0, 0, 125239, 917923, 0, 0, 0, 0, 0, 0, 1374, 65149, 119014, - 67720, 0, 2273, 0, 0, 0, 11234, 0, 0, 9630, 12597, 0, 0, 0, 6661, 0, - 113751, 120551, 125015, 0, 0, 72151, 0, 73674, 7718, 113755, 0, 69570, 0, - 0, 983777, 0, 0, 0, 127841, 6365, 1887, 983414, 0, 8080, 113681, 0, 0, 0, - 129855, 1544, 0, 0, 64677, 0, 0, 0, 0, 73561, 0, 0, 12812, 7342, 0, - 73784, 66947, 7904, 0, 0, 120910, 0, 0, 0, 0, 9724, 0, 983804, 9524, 0, - 0, 0, 0, 0, 129344, 0, 471, 0, 0, 128302, 72450, 0, 0, 983769, 0, 0, - 6918, 118685, 0, 5156, 0, 128683, 10232, 10615, 10213, 0, 0, 42528, 0, 0, - 0, 0, 65311, 74935, 0, 13306, 10533, 7870, 0, 7625, 0, 120544, 0, 0, - 128816, 126098, 118870, 0, 92819, 0, 0, 92341, 0, 12978, 128533, 0, 0, - 43836, 42675, 0, 12845, 0, 19942, 0, 0, 0, 0, 0, 120000, 120008, 120001, - 0, 194894, 983746, 0, 0, 0, 7186, 73107, 0, 70093, 445, 119028, 0, 0, 0, - 73047, 0, 0, 128442, 0, 0, 0, 3902, 68913, 129916, 0, 0, 1560, 43958, 0, - 4584, 0, 67862, 0, 10866, 92905, 1118, 92209, 74888, 0, 1081, 7436, - 11147, 7252, 0, 121188, 0, 0, 0, 41386, 5162, 129823, 1330, 0, 121270, 0, - 12047, 7675, 0, 0, 1848, 74528, 983148, 64708, 0, 0, 194880, 0, 0, 0, - 983772, 12715, 128349, 0, 101402, 0, 66672, 73710, 66685, 0, 0, 92464, 0, - 68884, 0, 72835, 123546, 70800, 70101, 120725, 0, 194893, 9214, 43494, 0, - 0, 120841, 0, 0, 6313, 65513, 119355, 0, 0, 0, 2345, 72975, 0, 0, 129937, - 0, 3117, 0, 71882, 0, 73100, 0, 0, 0, 0, 78415, 983236, 100907, 0, 13248, - 0, 120241, 129416, 128415, 0, 94193, 12382, 71120, 0, 0, 0, 0, 1471, 0, - 113747, 0, 12378, 0, 69664, 0, 12374, 121357, 0, 0, 0, 0, 0, 0, 12376, 0, - 0, 0, 12380, 10557, 0, 12520, 11122, 2024, 127180, 0, 0, 74588, 0, 0, - 70120, 3853, 0, 0, 0, 983763, 0, 0, 12090, 0, 12474, 92579, 9503, 0, 0, - 73505, 68318, 0, 110834, 0, 0, 0, 12470, 0, 74189, 2742, 12476, 66370, - 10946, 0, 12472, 0, 0, 0, 0, 8213, 43824, 7771, 6161, 983280, 68010, 0, - 0, 0, 68235, 0, 0, 0, 120985, 0, 0, 0, 129814, 73791, 129830, 68871, 0, - 0, 0, 0, 0, 73704, 12015, 128561, 8275, 0, 43459, 120927, 127555, 0, 0, - 0, 68881, 71215, 983642, 118841, 0, 12516, 4444, 0, 119017, 120506, - 10892, 118828, 0, 6473, 0, 0, 71735, 3591, 0, 0, 0, 0, 72345, 0, 0, 0, - 127547, 0, 0, 0, 0, 128253, 0, 0, 0, 0, 94060, 687, 0, 0, 983404, 0, 0, - 43882, 0, 128526, 285, 0, 0, 0, 4459, 0, 0, 74917, 0, 0, 126255, 0, - 119248, 0, 9743, 0, 0, 126535, 0, 0, 73104, 0, 69659, 0, 0, 3081, 74577, - 42921, 0, 0, 0, 0, 0, 0, 0, 9125, 119023, 0, 120820, 0, 65221, 0, 0, - 64852, 0, 0, 0, 0, 66578, 5001, 41879, 0, 0, 5003, 884, 0, 0, 4943, 5150, - 73889, 74182, 0, 41876, 0, 78915, 42448, 42299, 72804, 0, 0, 0, 0, 8491, - 0, 0, 983635, 4530, 42409, 7126, 119526, 66200, 0, 118559, 19929, 0, 0, - 0, 4242, 0, 0, 0, 0, 66034, 65941, 124929, 64522, 10740, 8958, 128257, - 9754, 119102, 983251, 74222, 983249, 983248, 119064, 983246, 983245, 0, - 0, 0, 74518, 66026, 4306, 41468, 68432, 0, 0, 66667, 0, 0, 983499, 42200, - 0, 0, 0, 120236, 6948, 0, 8524, 0, 0, 12385, 0, 74926, 0, 1386, 73996, 0, - 0, 0, 121184, 12392, 0, 8064, 0, 0, 78216, 119004, 2080, 710, 128491, - 12390, 1666, 42091, 0, 12383, 92968, 42092, 68418, 0, 128106, 0, 0, - 42096, 0, 3362, 12377, 127878, 0, 0, 0, 0, 1244, 4401, 73786, 12683, - 10662, 0, 8112, 129837, 119021, 121017, 12379, 73108, 120534, 0, 42208, - 0, 12381, 0, 0, 0, 4327, 0, 0, 128350, 0, 78232, 0, 584, 12933, 0, 12373, - 73105, 13000, 0, 2935, 129113, 12665, 0, 43081, 73098, 120505, 12427, 0, - 983625, 78227, 0, 0, 0, 0, 128760, 74551, 0, 0, 12426, 0, 73497, 0, - 12428, 0, 0, 0, 0, 0, 12429, 6727, 0, 0, 0, 3387, 0, 0, 0, 0, 0, 0, - 73517, 0, 3536, 120589, 9752, 92397, 6162, 0, 0, 10113, 0, 0, 0, 12422, - 0, 439, 3072, 0, 42207, 74549, 120830, 0, 0, 0, 0, 8308, 0, 70807, 0, 0, - 0, 13218, 0, 0, 8082, 12424, 0, 6819, 3539, 93838, 0, 0, 74539, 0, 68181, - 0, 72964, 0, 72969, 12420, 11371, 0, 4600, 0, 127810, 0, 0, 0, 72962, - 128552, 6704, 4591, 72966, 0, 0, 0, 72960, 120623, 561, 12159, 78223, 0, - 78224, 0, 71068, 11932, 7172, 42687, 8368, 0, 0, 93068, 0, 0, 75010, 0, - 0, 0, 0, 42463, 0, 2924, 67183, 0, 129947, 0, 128958, 0, 0, 42330, 73079, - 3969, 0, 129973, 7169, 1992, 9652, 0, 0, 42086, 0, 100865, 0, 0, 0, 0, 0, - 327, 0, 0, 0, 0, 73509, 12433, 0, 0, 118570, 12431, 0, 12434, 983439, 0, - 73544, 0, 7712, 12432, 0, 69377, 129147, 100867, 0, 8212, 0, 128014, 0, - 119066, 7333, 0, 0, 0, 67407, 70006, 128461, 0, 12436, 0, 43160, 0, - 74896, 92757, 71360, 42350, 0, 0, 0, 100566, 0, 11348, 0, 0, 9194, - 983185, 0, 55250, 0, 100569, 0, 0, 0, 0, 0, 64746, 66012, 100565, 3444, - 75029, 64651, 0, 41503, 0, 0, 0, 0, 0, 0, 0, 120876, 0, 0, 129408, 65309, - 12416, 0, 0, 0, 0, 93024, 12418, 74111, 121046, 0, 0, 0, 119361, 0, 4596, - 66339, 12417, 66001, 0, 126491, 12414, 8287, 0, 69499, 0, 1143, 0, 0, - 12415, 0, 0, 983247, 0, 9021, 120783, 0, 11724, 0, 0, 0, 194794, 0, 0, - 8027, 194796, 74257, 127375, 11400, 74197, 194799, 66833, 194798, 0, 0, - 983252, 0, 0, 1324, 0, 0, 0, 194878, 7715, 0, 0, 194777, 194780, 0, 0, 0, - 194787, 0, 0, 0, 0, 0, 66289, 127109, 3889, 129561, 194800, 0, 0, 0, 0, - 121226, 12999, 0, 120902, 0, 0, 0, 0, 0, 64802, 42210, 4597, 983134, 0, - 0, 12371, 67164, 0, 67163, 10805, 0, 0, 0, 0, 118662, 12367, 0, 0, 92557, - 12363, 0, 0, 128611, 983645, 0, 0, 8005, 12365, 0, 0, 3756, 12369, 10649, - 0, 70095, 0, 0, 0, 42923, 0, 0, 0, 0, 0, 0, 66659, 0, 0, 0, 0, 5268, - 4954, 0, 0, 5266, 126980, 5272, 92294, 0, 42230, 983980, 0, 9128, 0, 0, - 0, 0, 6928, 9803, 42282, 9110, 1505, 0, 0, 5276, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 8722, 120805, 0, 0, 66695, 0, 0, 4383, 8900, 0, 0, 74930, 64297, 0, 0, - 0, 0, 3419, 42229, 0, 0, 8911, 0, 42353, 0, 0, 0, 0, 0, 0, 0, 100629, - 41576, 42215, 122888, 0, 0, 8578, 68178, 7573, 41575, 74789, 92310, 0, - 73863, 0, 2670, 0, 0, 11723, 0, 0, 0, 0, 0, 43414, 0, 0, 65675, 0, 67179, - 67168, 12413, 129746, 67177, 0, 0, 0, 0, 12302, 0, 5250, 12407, 12245, - 4404, 9189, 12401, 42007, 0, 42005, 65806, 43997, 122922, 42002, 12404, - 0, 74928, 4940, 12410, 0, 128761, 0, 64567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 11956, 0, 0, 122882, 0, 6631, 128923, 120704, 74583, 42218, 0, 0, 70094, - 0, 0, 0, 71058, 0, 0, 0, 127341, 0, 0, 0, 0, 0, 43370, 0, 5016, 121052, - 0, 0, 9491, 0, 0, 0, 0, 64922, 0, 0, 0, 0, 92198, 0, 118622, 0, 74619, 0, - 0, 70422, 983688, 10565, 0, 12177, 0, 0, 0, 0, 0, 12395, 127874, 12878, - 92630, 12396, 0, 0, 92537, 0, 43113, 0, 0, 0, 9781, 0, 4927, 0, 0, 0, 0, - 12397, 129089, 128910, 0, 12394, 0, 0, 0, 0, 0, 72789, 10781, 1546, 0, - 5010, 0, 10507, 127891, 128291, 0, 0, 0, 0, 7267, 0, 0, 0, 0, 2819, 0, 0, - 71063, 0, 7266, 128553, 7264, 7265, 0, 1363, 0, 119581, 65080, 0, 0, 0, - 0, 43336, 67004, 0, 126263, 73776, 0, 43339, 0, 9836, 0, 0, 0, 43335, - 41276, 0, 73795, 43337, 817, 11211, 2241, 128841, 41274, 11340, 42408, - 42447, 74932, 0, 0, 12386, 0, 0, 0, 12389, 128398, 0, 41996, 41686, 0, - 8269, 1147, 43849, 120896, 1987, 128540, 43195, 42001, 41990, 41999, - 12391, 0, 0, 4939, 12384, 0, 0, 43243, 0, 0, 70746, 0, 0, 0, 0, 0, 8247, - 0, 0, 7545, 0, 43643, 121445, 0, 10036, 0, 119813, 10178, 119816, 0, - 119815, 11762, 119818, 0, 92282, 120597, 0, 0, 119819, 0, 0, 7719, 0, - 2486, 0, 119808, 1507, 0, 129185, 70301, 9687, 119825, 0, 119811, 66196, - 0, 5262, 0, 74642, 12681, 0, 0, 12406, 12219, 0, 127528, 42810, 110991, - 0, 983692, 128144, 121027, 126096, 120753, 12403, 2500, 118672, 0, 12409, - 0, 0, 0, 74113, 2343, 12412, 19946, 74112, 125042, 13112, 0, 120603, - 67866, 110634, 0, 66369, 5861, 110632, 11999, 12400, 0, 0, 12645, 0, - 11320, 68410, 6748, 65040, 0, 64184, 12974, 66927, 67613, 120645, 0, 0, - 0, 0, 0, 1928, 0, 67649, 0, 0, 67609, 11235, 0, 0, 67610, 8241, 0, 0, - 4206, 0, 0, 0, 128298, 110980, 0, 67238, 0, 0, 0, 1422, 8357, 0, 7187, 0, - 120641, 0, 0, 0, 0, 125022, 111064, 92539, 10120, 12405, 0, 72997, 0, - 13278, 0, 6366, 0, 7945, 0, 4402, 0, 12402, 129372, 0, 74754, 12408, 0, - 44007, 0, 0, 0, 12411, 0, 120824, 128306, 121092, 0, 1575, 0, 0, 0, - 73003, 119622, 0, 0, 12399, 0, 6833, 0, 0, 0, 71878, 9692, 0, 0, 100615, - 6750, 66855, 0, 0, 0, 0, 43527, 0, 727, 0, 0, 0, 0, 6726, 127387, 0, - 12370, 44023, 0, 126592, 2280, 0, 12372, 120642, 0, 0, 0, 0, 12366, - 10963, 6066, 1329, 0, 3052, 72987, 0, 66029, 0, 10803, 0, 983648, 0, - 92473, 0, 0, 0, 0, 1499, 0, 0, 42740, 0, 0, 0, 0, 12056, 126484, 0, 3660, - 69404, 42192, 74253, 0, 42223, 67617, 125254, 0, 0, 0, 0, 9941, 0, 0, - 1933, 0, 0, 0, 0, 73866, 0, 0, 2487, 67614, 7361, 1804, 0, 67615, 0, 0, - 12220, 67616, 0, 0, 0, 68200, 6675, 0, 0, 67592, 126582, 0, 64771, 0, - 9132, 0, 111004, 510, 0, 0, 0, 4561, 7711, 92769, 92944, 111007, 0, - 41569, 121282, 0, 8167, 66885, 123197, 0, 0, 69992, 66403, 6967, 0, 0, 0, - 0, 333, 0, 0, 10566, 66409, 0, 121373, 0, 72965, 110999, 66388, 6678, 0, - 0, 12621, 0, 128775, 10227, 4764, 0, 9981, 0, 70278, 11589, 0, 0, 42202, - 12754, 0, 0, 69576, 0, 67594, 2048, 0, 4050, 67595, 0, 0, 43221, 11184, - 72709, 0, 0, 64175, 0, 72746, 0, 0, 129966, 65461, 9798, 0, 71210, 0, - 69841, 0, 952, 128235, 125107, 0, 70296, 6449, 72102, 0, 0, 43098, 64171, - 8142, 64160, 0, 0, 0, 0, 0, 0, 0, 0, 67597, 6676, 3930, 42615, 73124, - 69991, 67598, 0, 0, 0, 65591, 41581, 128056, 1453, 0, 0, 0, 8500, 42222, - 0, 119270, 72992, 69996, 0, 0, 64676, 0, 0, 67606, 66385, 0, 42217, - 13102, 0, 67607, 6672, 0, 0, 0, 0, 67608, 0, 9001, 0, 11274, 67601, 0, - 64210, 6664, 0, 42056, 67602, 0, 0, 0, 0, 1469, 67603, 65381, 69921, - 4988, 42372, 0, 9598, 904, 352, 42225, 0, 8061, 10673, 0, 0, 128276, - 67600, 67477, 0, 127293, 8575, 127295, 127296, 127289, 127290, 127291, - 127292, 127285, 127286, 127287, 118877, 127281, 127282, 9460, 823, 11587, - 0, 0, 0, 127305, 12387, 0, 0, 127301, 126979, 42783, 69998, 64208, - 127298, 127299, 66031, 0, 11606, 64784, 0, 69973, 0, 124149, 0, 5152, - 11048, 0, 120121, 67605, 0, 69604, 0, 70276, 194847, 0, 127052, 42587, - 42214, 41394, 0, 4763, 0, 118935, 0, 5260, 0, 94038, 326, 120131, 74119, - 0, 10771, 42198, 194920, 194835, 194925, 41398, 127079, 41393, 127077, - 127076, 453, 41396, 0, 13159, 11227, 9572, 0, 0, 194576, 128835, 127081, - 0, 126617, 43144, 0, 72972, 194887, 0, 0, 0, 0, 0, 64061, 0, 0, 64056, - 70310, 0, 0, 0, 66971, 0, 111084, 64301, 72998, 10464, 0, 128393, 72847, - 0, 11528, 64024, 128072, 679, 0, 0, 5850, 758, 7536, 0, 0, 43712, 0, - 64006, 983589, 64005, 70298, 0, 126487, 0, 0, 0, 0, 0, 72999, 0, 64027, - 64029, 0, 0, 64000, 0, 194874, 0, 42201, 12421, 194875, 0, 1852, 0, 0, - 73744, 0, 64041, 129127, 0, 0, 0, 92322, 12423, 12854, 0, 3496, 0, - 110966, 0, 194823, 0, 0, 6158, 8327, 74553, 0, 12419, 0, 11570, 2169, 0, - 123618, 0, 7844, 983820, 194909, 0, 1682, 93039, 194911, 42756, 6765, - 128178, 0, 0, 0, 11412, 6768, 0, 194830, 71316, 0, 0, 0, 11577, 0, - 194829, 1833, 11576, 74334, 0, 0, 42854, 69438, 0, 70307, 0, 194856, - 8085, 0, 194850, 0, 72996, 128778, 1949, 11614, 7847, 120489, 120997, - 64483, 0, 0, 0, 122639, 0, 0, 0, 126651, 42864, 0, 64667, 74624, 0, 0, - 43261, 11484, 127535, 67840, 0, 0, 128965, 0, 72974, 0, 72456, 8995, - 3455, 0, 0, 9879, 0, 0, 4158, 128050, 0, 0, 110929, 0, 0, 0, 332, 118808, - 0, 0, 2407, 0, 42199, 92386, 110865, 0, 77921, 55217, 123161, 125199, - 70043, 0, 0, 0, 121093, 1834, 0, 0, 71315, 0, 65249, 0, 8662, 0, 0, - 123153, 0, 11539, 10784, 0, 67674, 0, 92233, 0, 0, 118858, 0, 0, 0, 0, 0, - 0, 12499, 6280, 0, 0, 0, 0, 0, 0, 43851, 6279, 12508, 0, 12502, 9161, 0, - 1620, 0, 3601, 0, 0, 67246, 609, 11555, 0, 12496, 0, 74181, 120492, - 12505, 0, 194902, 0, 43567, 239, 0, 127085, 0, 0, 42671, 0, 0, 83095, - 43565, 127082, 983955, 12696, 127753, 0, 94062, 12929, 0, 712, 0, 4197, - 0, 42818, 0, 70306, 0, 0, 983824, 0, 43562, 0, 119506, 68076, 0, 111074, - 64628, 0, 0, 0, 0, 7494, 0, 4924, 0, 0, 0, 0, 72368, 0, 127087, 69987, - 64796, 0, 0, 12033, 119492, 0, 72370, 0, 0, 0, 0, 70299, 0, 0, 68324, - 72420, 0, 0, 0, 0, 70309, 127000, 0, 0, 0, 72418, 72963, 0, 5699, 0, - 983898, 9488, 74410, 119112, 70477, 11170, 0, 0, 72312, 0, 5265, 0, 0, 0, - 0, 12464, 0, 43264, 72977, 0, 43345, 120853, 0, 120592, 6807, 0, 9829, - 69997, 0, 0, 43346, 11393, 795, 0, 72412, 12462, 72416, 72415, 0, 0, - 64362, 0, 0, 120811, 0, 12468, 8607, 1008, 0, 120670, 0, 0, 67855, - 125018, 72372, 6758, 0, 0, 1820, 41112, 0, 11202, 129451, 0, 13223, 0, - 64595, 0, 0, 0, 983649, 12616, 0, 127088, 0, 74467, 0, 0, 0, 0, 0, 0, - 67233, 119060, 0, 83448, 19920, 69897, 0, 129057, 0, 1130, 0, 0, 0, - 11823, 0, 0, 118896, 0, 0, 13280, 0, 10747, 118925, 0, 43509, 0, 0, 8959, - 0, 6747, 0, 0, 8568, 0, 120870, 0, 120803, 83060, 42670, 0, 11621, 12460, - 0, 0, 0, 0, 111188, 0, 66570, 72989, 121305, 126476, 120582, 0, 0, 0, - 111191, 70308, 11594, 0, 68333, 69427, 10491, 0, 0, 0, 0, 0, 127506, 0, - 194910, 4923, 65086, 8981, 0, 42133, 0, 72244, 0, 70294, 0, 0, 12485, 0, - 8642, 0, 42766, 0, 2210, 11109, 0, 0, 0, 0, 0, 7398, 0, 0, 0, 8041, 1461, - 0, 119133, 0, 6749, 0, 0, 0, 71705, 0, 0, 68071, 0, 67668, 0, 0, 9193, 0, - 0, 0, 0, 73810, 0, 0, 64305, 0, 0, 623, 781, 670, 10660, 5769, 613, 7543, - 0, 477, 92633, 92521, 0, 592, 0, 12459, 0, 0, 0, 12465, 119578, 654, - 11345, 653, 652, 111250, 647, 0, 633, 120744, 0, 111262, 12480, 74354, 0, - 39, 12487, 0, 0, 74803, 12482, 0, 12489, 0, 128962, 5550, 129175, 0, 0, - 0, 0, 1813, 0, 41311, 111205, 0, 11229, 0, 70496, 1675, 69840, 129435, 0, - 119078, 10070, 10595, 111207, 119077, 111206, 121162, 0, 0, 0, 11222, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 71716, 917841, 0, 0, 270, 0, 0, 0, 0, 0, - 120561, 0, 69741, 0, 0, 68251, 0, 71721, 364, 9595, 0, 0, 0, 707, 110603, - 0, 9282, 0, 224, 0, 68670, 9332, 65581, 68677, 0, 68644, 0, 11764, 68634, - 0, 10732, 68640, 850, 0, 0, 71123, 0, 68619, 44008, 68627, 0, 0, 0, 0, - 66969, 0, 0, 0, 12507, 0, 0, 128311, 0, 120529, 4375, 0, 0, 0, 12198, 0, - 67339, 0, 0, 72994, 74293, 128434, 0, 0, 64546, 0, 71208, 0, 0, 78916, - 42334, 42502, 0, 120887, 72961, 0, 917838, 5767, 0, 0, 71710, 8353, 0, 0, - 0, 121233, 0, 0, 0, 0, 119920, 0, 0, 121186, 0, 0, 0, 72719, 64604, 0, - 6096, 122632, 10063, 0, 0, 119630, 3485, 12987, 0, 127522, 0, 0, 0, 0, 0, - 0, 0, 0, 127173, 0, 0, 68249, 0, 0, 118923, 0, 64574, 128794, 0, 1640, - 12495, 66691, 0, 3138, 12504, 11171, 1922, 0, 12498, 128733, 0, 69939, 0, - 65543, 0, 0, 0, 66643, 0, 120734, 0, 4228, 0, 10303, 128884, 0, 0, 10335, - 3520, 0, 12490, 0, 0, 0, 12493, 121452, 64636, 1002, 12491, 0, 0, 92615, - 2096, 0, 0, 0, 0, 11611, 66228, 0, 11241, 66224, 66221, 66226, 66229, - 66219, 66231, 66216, 0, 66236, 66211, 66218, 0, 66240, 78041, 66233, - 66217, 0, 7909, 66234, 11605, 0, 0, 66208, 0, 0, 128282, 73875, 0, 12898, - 12494, 120939, 12492, 0, 128774, 0, 74153, 0, 127391, 127489, 4882, - 13040, 0, 120762, 4885, 194732, 0, 13042, 4880, 128834, 2429, 118674, - 8647, 0, 0, 0, 0, 0, 0, 68896, 0, 119101, 66693, 0, 1870, 78040, 470, - 68893, 78035, 78036, 983577, 78034, 110607, 110608, 0, 12511, 74453, - 12514, 0, 128609, 7239, 7001, 11974, 121214, 0, 0, 7378, 12512, 11615, - 13041, 0, 0, 128057, 13038, 0, 0, 71717, 70195, 120836, 12510, 127070, - 13039, 75019, 12513, 71969, 12471, 110761, 0, 121385, 70193, 0, 0, 0, - 71714, 0, 12477, 0, 12473, 7666, 67362, 237, 6281, 0, 0, 0, 0, 1312, 0, - 0, 12469, 0, 0, 64335, 12475, 0, 69382, 0, 11524, 10367, 10431, 74368, - 13017, 3388, 129547, 69573, 0, 0, 128725, 4932, 0, 0, 13015, 0, 0, 65451, - 8185, 0, 0, 2189, 129362, 74375, 10129, 0, 7948, 9236, 0, 0, 69512, - 92726, 43473, 6289, 10484, 0, 0, 0, 12082, 12521, 3147, 110643, 110644, - 12524, 110642, 2310, 0, 0, 0, 0, 13013, 0, 8596, 983871, 10804, 70497, 0, - 0, 13014, 12444, 0, 43088, 13016, 0, 0, 0, 0, 12331, 0, 0, 8744, 726, - 121090, 983868, 4155, 0, 0, 0, 71690, 12522, 73128, 0, 0, 127805, 0, - 110647, 0, 0, 983872, 12525, 0, 12523, 2152, 11969, 120596, 403, 0, - 11021, 0, 0, 11030, 8610, 92567, 0, 0, 63998, 0, 0, 0, 0, 0, 0, 0, 12506, - 0, 11146, 71477, 12500, 0, 12509, 0, 0, 0, 0, 6608, 0, 0, 0, 0, 69288, - 77995, 0, 3608, 0, 0, 1107, 0, 129658, 0, 0, 0, 0, 983956, 43217, 66571, - 13222, 118963, 0, 126514, 10463, 11553, 0, 63995, 9043, 128634, 71722, 0, - 0, 127751, 92974, 12529, 8042, 0, 2344, 12528, 0, 0, 0, 69719, 120956, 0, - 0, 66512, 0, 12530, 0, 0, 68917, 12658, 0, 71683, 0, 983241, 0, 127526, - 469, 0, 4363, 3313, 0, 0, 2023, 0, 72251, 78225, 65706, 10051, 78219, - 78220, 0, 9920, 12215, 0, 4931, 1951, 12497, 119363, 0, 0, 119336, 0, 0, - 0, 0, 0, 1491, 128578, 129169, 0, 0, 0, 0, 78898, 94086, 41993, 0, 67379, - 0, 0, 0, 0, 9738, 41995, 1075, 0, 12535, 41992, 0, 2187, 0, 0, 128117, 0, - 9940, 0, 7692, 0, 9727, 41131, 330, 8566, 0, 41133, 41117, 128482, 12532, - 78550, 78546, 43177, 0, 43235, 0, 917542, 78229, 78231, 13031, 12910, - 67710, 78555, 13028, 78553, 12537, 0, 0, 71692, 12536, 2350, 13029, - 78233, 0, 0, 13030, 0, 4527, 71250, 12538, 0, 0, 0, 0, 0, 0, 0, 12484, - 4032, 71459, 194728, 0, 64344, 0, 66700, 66000, 8412, 0, 43466, 1296, - 2325, 0, 121020, 10149, 74118, 0, 0, 12481, 121280, 12488, 0, 0, 0, - 67972, 0, 2354, 42619, 0, 73027, 6295, 901, 0, 0, 0, 0, 0, 128653, 11927, - 66584, 78559, 78560, 78557, 78558, 0, 74649, 0, 126241, 67220, 194726, - 78568, 67226, 78565, 70190, 78563, 78564, 2352, 67219, 78569, 71945, - 11289, 1407, 67973, 0, 13026, 6762, 10399, 70192, 13023, 78578, 9777, - 67208, 1871, 0, 0, 0, 13024, 71936, 0, 9325, 6818, 6283, 11738, 0, 0, - 71938, 11741, 0, 0, 9216, 8263, 11279, 0, 983856, 0, 13021, 71922, 3136, - 0, 983859, 0, 13022, 129143, 9956, 0, 0, 0, 42580, 0, 0, 0, 13020, 10024, - 0, 94013, 0, 0, 0, 43001, 8029, 0, 0, 0, 3335, 127924, 9209, 13048, - 73126, 0, 0, 0, 3333, 119100, 0, 0, 3342, 78582, 78583, 73056, 78581, - 4156, 0, 0, 0, 78591, 1611, 73058, 13018, 78586, 78588, 78584, 3337, - 4537, 78593, 11736, 0, 0, 0, 4214, 73790, 0, 0, 13046, 194844, 425, - 74763, 42066, 78595, 0, 2392, 13047, 0, 0, 12425, 13049, 0, 92243, 0, - 72715, 73944, 13050, 0, 0, 0, 0, 983506, 0, 0, 8929, 6849, 0, 0, 0, - 983990, 0, 13045, 0, 0, 7751, 0, 9726, 0, 3997, 0, 8768, 13044, 0, 0, - 4024, 0, 0, 2419, 9757, 69736, 0, 0, 0, 129500, 0, 0, 0, 72735, 0, 0, 0, - 0, 0, 11911, 124990, 0, 2346, 194691, 69931, 0, 9646, 3773, 43557, 68154, - 42536, 0, 70108, 13043, 92686, 92494, 0, 208, 0, 43766, 0, 0, 0, 10699, - 0, 0, 7825, 7110, 111275, 0, 111274, 41109, 2398, 111271, 0, 0, 0, 0, 0, - 0, 72723, 8294, 42912, 129343, 0, 0, 4876, 111316, 0, 111326, 111282, 0, - 0, 0, 73950, 13053, 9944, 0, 2811, 13051, 111313, 3143, 111246, 66374, - 110759, 0, 0, 13052, 0, 0, 63972, 119071, 7025, 0, 0, 100464, 74161, - 4154, 9863, 129686, 0, 0, 63970, 1564, 0, 0, 0, 0, 0, 9942, 0, 0, 111227, - 0, 128471, 0, 63957, 0, 1626, 0, 63983, 2161, 111232, 0, 0, 121275, - 111292, 6254, 4910, 69453, 0, 64753, 100458, 111303, 111204, 127404, - 111297, 3229, 111306, 42774, 0, 0, 111218, 111286, 2331, 0, 7085, 6137, - 0, 70411, 0, 126070, 0, 128438, 0, 0, 65043, 0, 127588, 70412, 128921, - 64721, 0, 0, 0, 0, 0, 983789, 0, 0, 5311, 0, 965, 0, 11993, 78055, 11278, - 128787, 0, 0, 0, 121076, 120705, 0, 6294, 3144, 0, 0, 65019, 0, 0, 0, 0, - 118721, 63966, 2330, 535, 3148, 12375, 110774, 0, 10556, 2475, 12388, - 4889, 0, 67863, 120404, 0, 12287, 2342, 0, 0, 0, 4894, 0, 4890, 0, 0, 0, - 4893, 128426, 6571, 118581, 4888, 4157, 78048, 78049, 78046, 11263, 0, - 78045, 64895, 121437, 0, 0, 0, 0, 0, 119041, 2332, 78063, 78060, 78061, - 64932, 78059, 65125, 121098, 0, 0, 129991, 73941, 78066, 12203, 78064, - 78065, 8913, 120390, 4875, 73678, 120396, 120389, 71854, 0, 120394, - 120386, 120395, 13104, 78076, 78077, 120393, 78075, 0, 3134, 83096, - 65696, 72432, 0, 0, 0, 8334, 0, 83207, 3449, 0, 0, 83215, 0, 0, 0, 83204, - 0, 0, 0, 69707, 0, 0, 10734, 0, 83198, 83108, 7804, 121401, 83166, 8457, - 83212, 0, 11367, 0, 78054, 0, 72762, 0, 64285, 0, 5464, 0, 83100, 2361, - 7971, 78072, 78073, 78070, 78071, 0, 8086, 0, 6707, 0, 2312, 40977, 0, - 40962, 0, 0, 74962, 40980, 0, 0, 0, 40970, 92895, 110823, 0, 42438, - 72752, 6288, 0, 127946, 5653, 42400, 10891, 73946, 5658, 70401, 0, 0, 0, - 0, 71060, 0, 0, 42326, 100482, 92191, 92685, 42478, 2327, 0, 12959, - 42287, 92883, 0, 83081, 917550, 0, 0, 2867, 128562, 66312, 698, 0, 0, 0, - 70017, 0, 8000, 12641, 83140, 0, 0, 129064, 0, 72979, 83133, 0, 83134, 0, - 0, 111011, 92960, 74356, 0, 74562, 0, 72745, 0, 0, 120568, 0, 0, 0, 0, 0, - 8703, 5462, 83195, 0, 10101, 0, 70049, 0, 0, 128793, 0, 0, 66254, 120821, - 0, 1565, 123621, 0, 119194, 0, 42651, 0, 0, 917847, 83227, 83218, 0, - 75011, 0, 129724, 0, 64399, 0, 12899, 74564, 0, 42206, 0, 72718, 71715, - 83149, 983794, 83146, 12192, 917826, 0, 0, 0, 0, 68056, 0, 67426, 128687, - 0, 0, 0, 0, 0, 0, 67431, 71718, 74357, 0, 121176, 43596, 6090, 0, 7812, - 10534, 0, 0, 0, 0, 129763, 0, 0, 0, 0, 0, 0, 43306, 0, 0, 0, 7930, 0, - 2292, 0, 0, 72737, 0, 6130, 0, 0, 0, 0, 0, 70463, 968, 0, 0, 0, 43304, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 11838, 0, 0, 0, 42682, 0, 0, 0, 41227, 0, - 71475, 0, 64848, 0, 78574, 0, 113792, 0, 0, 129133, 0, 66015, 74614, 959, - 8885, 0, 0, 0, 9469, 9632, 128211, 74761, 64323, 100478, 0, 2266, 78575, - 310, 0, 0, 68403, 100480, 72738, 125279, 0, 0, 6497, 127320, 0, 0, 19958, - 0, 128691, 74953, 0, 118998, 67332, 374, 0, 41933, 120975, 0, 0, 41934, - 7465, 0, 128168, 70666, 11151, 6101, 0, 41936, 100476, 4879, 0, 65446, 0, - 0, 983695, 0, 5374, 0, 128059, 127390, 0, 126618, 983575, 129146, 0, 0, - 1929, 0, 12142, 0, 0, 0, 121472, 0, 12982, 0, 5378, 0, 128679, 0, 0, - 127869, 0, 127343, 0, 0, 0, 78832, 74481, 0, 43262, 100511, 2421, 0, - 2324, 828, 3611, 121055, 0, 64314, 0, 0, 0, 0, 0, 0, 7999, 0, 11217, - 983266, 10634, 10942, 0, 2348, 0, 0, 0, 0, 118587, 9982, 64324, 41240, 0, - 100470, 78462, 1810, 0, 92566, 71299, 0, 0, 917848, 0, 0, 100515, 0, 0, - 0, 43912, 128385, 0, 0, 0, 917850, 0, 7485, 0, 129382, 74576, 44019, - 128171, 917851, 3967, 129335, 0, 0, 0, 0, 119096, 0, 0, 8699, 723, 83084, - 966, 0, 0, 0, 128428, 78778, 2320, 0, 65740, 4968, 0, 0, 8075, 55276, - 123589, 8047, 983787, 78827, 12634, 0, 78781, 71322, 0, 12174, 42610, 0, - 0, 0, 1584, 0, 6045, 0, 0, 65218, 11559, 0, 0, 0, 124991, 0, 2257, 64418, - 0, 0, 0, 0, 0, 0, 67821, 0, 13092, 0, 128365, 0, 0, 0, 0, 0, 11414, 0, - 2531, 13034, 0, 0, 0, 13036, 0, 70866, 70198, 10394, 129979, 13037, 0, - 129956, 0, 0, 100496, 120640, 41129, 0, 42850, 13035, 0, 0, 5466, 0, 0, - 0, 129439, 4535, 0, 4271, 0, 0, 6769, 0, 0, 67350, 6767, 0, 66273, 0, - 6755, 73827, 9046, 67355, 0, 0, 0, 0, 0, 0, 0, 0, 92221, 83235, 2563, - 13033, 247, 83229, 0, 12338, 0, 83231, 11270, 0, 0, 0, 0, 70107, 0, 0, 0, - 0, 3752, 83243, 68895, 66973, 68897, 0, 0, 0, 0, 5009, 0, 0, 0, 0, - 119521, 78823, 78824, 70353, 68399, 3877, 0, 78825, 10145, 43566, 0, 0, - 10236, 0, 43782, 0, 127329, 0, 69652, 2247, 120612, 128058, 0, 43200, - 43777, 71253, 983644, 69558, 0, 71866, 43203, 0, 68894, 0, 127326, 0, - 43778, 119538, 0, 0, 43781, 11303, 65547, 0, 7031, 0, 0, 67343, 83237, - 83267, 0, 67341, 120522, 8535, 0, 0, 0, 66032, 0, 0, 120786, 42233, 0, - 9946, 7667, 0, 11822, 0, 43189, 120673, 100507, 2979, 1579, 0, 0, 0, 0, - 0, 12635, 71337, 0, 94055, 0, 1285, 64882, 0, 0, 83113, 12640, 83112, - 7401, 92869, 12625, 0, 71296, 72744, 0, 74286, 55260, 3396, 12642, 0, - 110719, 0, 12630, 0, 0, 10153, 0, 6166, 120516, 0, 110680, 0, 0, 119499, - 9285, 913, 42259, 83017, 0, 2142, 127889, 0, 94012, 7878, 0, 72733, 0, 0, - 0, 0, 92868, 0, 0, 0, 0, 128918, 5263, 74782, 0, 41939, 43702, 0, 917856, - 0, 10139, 980, 43698, 0, 2208, 0, 43701, 0, 125132, 0, 100528, 0, 10085, - 0, 0, 119989, 100529, 0, 71699, 0, 8072, 0, 43700, 0, 7304, 7783, 66894, - 12398, 0, 0, 0, 0, 0, 0, 120565, 0, 2217, 0, 94015, 6367, 0, 66688, 0, 0, - 0, 0, 0, 92199, 7808, 1829, 0, 41937, 0, 43272, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 92467, 6627, 0, 6258, 10683, 0, 0, 0, 5649, 0, 0, 0, 1643, 127898, - 0, 127846, 67244, 0, 42452, 0, 0, 0, 0, 64291, 0, 0, 0, 6576, 74773, 0, - 0, 66309, 0, 9886, 55225, 11292, 0, 72867, 55227, 0, 12632, 0, 194817, 0, - 7680, 0, 92745, 120714, 12639, 3380, 8123, 0, 12638, 42262, 4501, 0, 0, - 0, 0, 125131, 1494, 983147, 0, 0, 0, 0, 10494, 0, 65872, 0, 0, 0, 0, 0, - 0, 983587, 0, 0, 0, 0, 0, 0, 0, 71077, 0, 127335, 121128, 0, 5570, 1881, - 7210, 0, 1012, 66630, 0, 128982, 7208, 66442, 5569, 113723, 42339, 92655, - 0, 0, 0, 0, 92378, 65602, 0, 92375, 64727, 9160, 0, 0, 0, 124928, 10503, - 0, 3423, 3870, 8483, 10162, 0, 4319, 0, 0, 0, 0, 0, 983117, 0, 69562, 0, - 0, 0, 0, 0, 0, 5571, 7630, 9740, 9121, 5568, 0, 0, 42085, 0, 0, 65056, 0, - 589, 0, 0, 0, 10233, 66252, 66251, 78734, 66253, 0, 0, 42645, 0, 128424, - 8583, 0, 0, 0, 129932, 0, 0, 0, 0, 0, 12204, 92436, 120453, 0, 0, 0, - 983262, 0, 0, 70311, 0, 0, 128012, 41063, 0, 10664, 0, 983660, 0, 4551, - 129090, 74759, 0, 983270, 0, 0, 72806, 0, 0, 12517, 7806, 0, 12034, 0, - 6355, 12519, 41004, 0, 0, 93849, 0, 71707, 0, 121231, 7332, 129075, - 12111, 3927, 0, 12515, 1474, 68768, 0, 6923, 69509, 0, 127802, 0, 43990, - 74639, 126229, 121007, 0, 92706, 0, 0, 0, 0, 0, 9645, 0, 121026, 5853, 0, - 10363, 120729, 12956, 0, 0, 0, 0, 127888, 0, 0, 0, 0, 0, 10514, 65517, 0, - 0, 71101, 0, 0, 0, 43570, 2969, 43420, 129944, 0, 0, 92366, 70809, 0, 0, - 0, 0, 0, 118714, 12125, 41124, 0, 1164, 128817, 0, 120466, 0, 0, 65014, - 66009, 74451, 125075, 983129, 7469, 0, 0, 0, 69988, 120671, 83171, 41123, - 11176, 0, 0, 41126, 9991, 41128, 0, 0, 110949, 0, 0, 42877, 7994, 0, - 6104, 983612, 0, 129869, 0, 0, 0, 0, 74438, 128272, 121409, 41981, 0, - 69296, 42904, 0, 0, 74435, 126640, 0, 0, 0, 127968, 92442, 12703, 9661, - 67360, 67359, 7455, 70732, 11473, 119217, 128512, 0, 92323, 0, 0, 129632, - 67358, 0, 0, 0, 0, 174, 121131, 883, 4161, 128033, 42603, 0, 0, 72256, 0, - 0, 128356, 0, 0, 0, 0, 3846, 8070, 6150, 128109, 4370, 118617, 0, 0, - 74587, 0, 0, 0, 0, 4986, 12189, 917553, 67648, 120499, 0, 4257, 71695, - 123620, 6220, 0, 65561, 0, 0, 0, 0, 122652, 0, 0, 0, 69684, 0, 0, 128452, - 120873, 0, 0, 74922, 0, 71897, 0, 0, 67368, 67367, 8871, 67366, 0, 0, 0, - 0, 0, 67361, 0, 0, 67365, 67364, 3427, 4240, 67376, 67375, 67374, 67373, - 0, 0, 0, 67377, 0, 71689, 0, 0, 67372, 67371, 67370, 67369, 0, 0, 0, - 124962, 0, 0, 0, 0, 65898, 0, 65312, 0, 0, 0, 0, 4010, 121208, 41106, 0, - 0, 0, 41105, 0, 64803, 83456, 0, 0, 0, 0, 0, 0, 0, 11008, 0, 0, 71351, - 41110, 71681, 64892, 9113, 1954, 41108, 0, 42878, 0, 67405, 0, 0, 0, 0, - 0, 119539, 69435, 73463, 0, 4586, 129342, 0, 0, 0, 0, 0, 125233, 92307, - 0, 0, 0, 67382, 0, 9500, 0, 4957, 0, 2422, 2212, 0, 67381, 67380, 11045, - 67378, 0, 0, 3890, 12168, 121328, 0, 0, 0, 41947, 0, 120828, 74946, - 917901, 0, 1571, 66461, 41949, 42805, 8270, 943, 41946, 0, 2073, 0, - 41980, 0, 0, 0, 0, 4429, 6272, 0, 1460, 6954, 128572, 41120, 0, 65733, 0, - 41119, 0, 127006, 0, 0, 0, 129168, 12895, 0, 0, 0, 69440, 0, 1985, 6296, - 0, 0, 0, 0, 0, 41122, 0, 2457, 0, 0, 0, 0, 0, 0, 8840, 8035, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 8681, 0, 121505, 128747, 0, 0, 70102, 0, 124976, 9605, - 0, 13220, 0, 67354, 11312, 0, 9246, 67349, 0, 0, 0, 0, 10012, 12123, 0, - 0, 0, 0, 983846, 0, 0, 0, 67817, 0, 1272, 0, 0, 0, 983578, 0, 1467, - 119501, 917806, 0, 0, 0, 70312, 73537, 124955, 0, 70400, 0, 0, 72817, 0, - 19935, 0, 92162, 0, 0, 0, 128406, 5275, 0, 0, 44006, 129082, 0, 3789, - 128205, 0, 0, 0, 11474, 0, 0, 0, 129050, 0, 92194, 129503, 9537, 4496, 0, - 120443, 2605, 4500, 0, 55224, 8600, 0, 0, 41646, 11667, 69569, 0, 0, - 917905, 4499, 41649, 0, 0, 0, 69254, 0, 0, 0, 65804, 0, 70034, 41866, 0, - 0, 0, 11174, 0, 0, 0, 9559, 128773, 41940, 8299, 41945, 0, 41941, 5455, - 7190, 0, 0, 917810, 65266, 0, 41943, 10762, 0, 41931, 0, 0, 8106, 4128, - 0, 0, 4494, 0, 0, 72405, 0, 119567, 42068, 917808, 0, 11004, 12794, - 65072, 5271, 7317, 0, 0, 0, 0, 0, 0, 92281, 0, 0, 0, 0, 71880, 3868, - 71881, 983573, 128431, 7703, 0, 64390, 0, 7406, 120358, 93850, 0, 3985, - 66425, 0, 66615, 10177, 0, 41853, 71873, 12809, 0, 12193, 0, 10879, - 122945, 0, 9055, 0, 3851, 8132, 0, 0, 119263, 917908, 0, 0, 0, 0, 122940, - 42657, 122952, 7643, 0, 0, 122936, 43568, 0, 11949, 7650, 43569, 64951, - 7647, 7649, 0, 7646, 0, 0, 9651, 125005, 3891, 0, 0, 2337, 77831, 77832, - 67860, 129288, 0, 0, 43561, 67706, 119669, 0, 1860, 0, 68835, 5812, - 12784, 0, 0, 0, 0, 69260, 7727, 0, 69292, 69818, 66444, 128665, 42719, 0, - 1569, 0, 12534, 12124, 7690, 194871, 12533, 0, 68383, 67997, 0, 6969, 0, - 0, 0, 67974, 63895, 128650, 0, 0, 0, 42144, 0, 0, 0, 0, 92211, 119043, 0, - 0, 917545, 0, 0, 12791, 0, 0, 0, 4447, 71065, 12793, 0, 0, 43385, 0, 0, - 12790, 120256, 0, 983840, 12792, 120254, 0, 0, 12789, 128489, 12317, - 74934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127840, 41652, 2974, 78689, 11476, - 0, 0, 0, 0, 43871, 0, 10894, 119176, 74557, 65686, 0, 0, 3724, 67335, - 67334, 67333, 67338, 67337, 0, 67336, 0, 65306, 0, 128421, 0, 8646, - 129593, 77829, 0, 0, 74852, 0, 0, 0, 0, 0, 220, 120252, 43551, 0, 10044, - 0, 0, 983847, 68659, 110825, 5707, 71362, 0, 0, 0, 0, 0, 0, 10297, 0, - 41308, 67331, 0, 0, 0, 0, 2467, 0, 6003, 0, 0, 8040, 0, 0, 4182, 0, - 11135, 120501, 0, 0, 2510, 0, 10208, 0, 78302, 70829, 0, 0, 6837, 0, 0, - 67348, 0, 0, 0, 0, 1559, 67342, 11104, 67340, 67347, 67346, 67345, 67344, - 0, 0, 67357, 67356, 0, 0, 0, 0, 67352, 67351, 5516, 2845, 7717, 8036, - 65161, 67353, 5514, 12045, 6278, 0, 5515, 0, 0, 0, 0, 0, 65194, 100387, - 5517, 70116, 92774, 0, 67884, 0, 67890, 42094, 67880, 67881, 67882, - 67883, 0, 0, 67879, 120411, 1902, 67887, 67888, 12976, 126546, 12483, - 12368, 41769, 42726, 41765, 69557, 12787, 67874, 7556, 67878, 74351, - 67897, 989, 42677, 67889, 0, 6060, 0, 4326, 11000, 64601, 68478, 0, 0, - 6917, 0, 120837, 0, 0, 0, 6148, 8605, 74205, 0, 0, 0, 42715, 0, 101047, - 0, 68663, 0, 41796, 1269, 42703, 64754, 101049, 101042, 5144, 12221, - 42716, 71048, 5133, 4331, 0, 128675, 0, 5279, 121362, 71046, 0, 0, 42701, - 0, 0, 0, 121470, 0, 0, 0, 983311, 0, 72455, 121259, 42666, 12207, 1067, - 255, 12131, 0, 0, 0, 0, 0, 0, 0, 70728, 43460, 0, 42723, 125216, 0, - 70427, 0, 12797, 0, 0, 983722, 0, 67977, 12799, 0, 92504, 9746, 5135, 0, - 12796, 0, 0, 0, 5139, 346, 74303, 121134, 12795, 125109, 5168, 0, 43845, - 983727, 0, 8253, 8817, 1136, 983735, 43563, 127774, 129542, 0, 0, 0, 0, - 0, 0, 983619, 0, 0, 4041, 0, 2357, 43240, 12786, 0, 0, 0, 44004, 7142, 0, - 67984, 0, 0, 0, 0, 12785, 0, 0, 7770, 10712, 64853, 42679, 118916, 42375, - 0, 983124, 94074, 12119, 0, 11059, 10791, 111092, 450, 0, 0, 0, 0, 5450, - 64691, 0, 0, 44009, 0, 0, 111097, 94085, 1839, 94004, 0, 10927, 1701, 0, - 129610, 41749, 41761, 5453, 8361, 66045, 41758, 5444, 41763, 0, 0, 0, - 66349, 983138, 121274, 0, 0, 8801, 0, 4340, 0, 0, 0, 0, 70001, 41824, 0, - 0, 0, 0, 42700, 0, 127980, 0, 0, 0, 0, 0, 0, 4493, 4336, 129171, 2314, - 983061, 41808, 0, 0, 0, 64638, 0, 65937, 4489, 71331, 0, 0, 5358, 42717, - 0, 71236, 0, 0, 0, 127042, 41813, 2712, 0, 127044, 1410, 0, 0, 0, 0, 0, - 0, 0, 0, 128587, 0, 0, 0, 4892, 0, 0, 0, 0, 122966, 5777, 0, 759, 0, - 2079, 65248, 12788, 0, 64552, 0, 41803, 68043, 0, 0, 0, 0, 128785, 0, - 68492, 67991, 75071, 2340, 0, 120638, 0, 983902, 0, 0, 917865, 64749, 0, - 2321, 3587, 0, 67236, 9953, 9952, 0, 0, 42714, 9951, 0, 0, 127902, 74150, - 0, 0, 74757, 127554, 0, 983826, 2395, 0, 9976, 0, 125128, 0, 0, 0, 42809, - 42807, 0, 66290, 70854, 4150, 64424, 8318, 41790, 67976, 65559, 2360, - 41794, 0, 0, 120987, 0, 0, 2418, 0, 2411, 0, 41783, 0, 41786, 65108, 0, - 0, 41772, 42813, 2317, 0, 118980, 0, 0, 0, 0, 0, 0, 78682, 7753, 2351, - 6655, 64489, 0, 0, 0, 4443, 41697, 230, 65793, 0, 65943, 42803, 0, 0, - 5441, 0, 0, 127053, 0, 855, 0, 6109, 101021, 0, 119116, 69989, 0, 0, - 72146, 0, 101023, 0, 72148, 124918, 19915, 41892, 0, 0, 128901, 41887, 0, - 67980, 9735, 0, 0, 120591, 13082, 101026, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, - 64504, 0, 69489, 120514, 0, 92962, 0, 42724, 69977, 0, 0, 0, 0, 67994, 0, - 0, 983823, 3565, 0, 0, 127553, 43035, 69898, 0, 0, 0, 0, 4891, 0, 0, - 4602, 0, 121065, 0, 0, 121157, 0, 43978, 8988, 0, 0, 0, 0, 0, 119184, - 121436, 73902, 69740, 0, 0, 72976, 0, 0, 8771, 0, 0, 0, 119209, 74974, - 71737, 0, 0, 67987, 0, 0, 0, 67989, 0, 10065, 8207, 0, 983588, 0, 0, 662, - 0, 41927, 0, 0, 0, 0, 0, 0, 0, 41929, 0, 0, 0, 41926, 69994, 0, 0, 0, - 126230, 68013, 1433, 64648, 6475, 0, 120983, 0, 73876, 0, 0, 0, 67992, - 78052, 0, 3978, 0, 0, 0, 0, 120761, 12281, 0, 0, 13241, 0, 0, 0, 0, - 11765, 42577, 0, 0, 2641, 7192, 0, 0, 118809, 101015, 0, 101016, 128948, - 101013, 6479, 64294, 118683, 0, 0, 0, 64334, 0, 0, 0, 92266, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9478, 127339, 124964, 0, 202, 0, 0, 1242, 0, 121170, 0, - 63940, 0, 0, 0, 63939, 11990, 92430, 67982, 0, 65440, 70068, 0, 0, 64829, - 0, 0, 0, 0, 0, 2858, 0, 63989, 0, 69239, 0, 121152, 0, 77841, 0, 70078, - 92574, 129519, 0, 0, 0, 128974, 0, 12922, 92498, 0, 66424, 71124, 0, 0, - 0, 2856, 0, 47, 0, 126986, 65858, 0, 0, 0, 0, 119161, 8417, 65903, 0, 0, - 0, 4033, 128164, 0, 0, 0, 129961, 64600, 1903, 12320, 0, 120894, 0, 0, - 8915, 0, 945, 0, 0, 0, 0, 111068, 0, 74828, 0, 69560, 9531, 0, 8505, 0, - 119238, 0, 0, 65538, 0, 0, 0, 0, 0, 0, 63935, 0, 0, 0, 0, 0, 64787, - 111060, 0, 0, 110828, 0, 2230, 0, 0, 71886, 9843, 0, 92419, 111062, - 67488, 92715, 0, 1320, 0, 1673, 0, 92383, 129902, 9338, 128355, 0, 0, 0, - 0, 11997, 0, 0, 0, 0, 0, 0, 43308, 0, 0, 0, 0, 0, 0, 0, 63920, 0, 0, 0, - 0, 0, 0, 3514, 78723, 0, 7492, 0, 0, 0, 7514, 0, 63924, 0, 7502, 7587, 0, - 0, 0, 118689, 43881, 7610, 0, 0, 118710, 692, 43588, 0, 0, 75056, 9688, - 0, 9535, 0, 0, 0, 64530, 0, 125251, 194861, 0, 72209, 7453, 0, 8013, - 66396, 0, 0, 8895, 5356, 0, 5458, 0, 2866, 0, 127860, 71732, 71724, 6700, - 0, 111081, 120583, 0, 110614, 0, 9641, 63830, 65294, 0, 0, 67969, 0, - 7441, 0, 63826, 0, 0, 0, 0, 2844, 983972, 0, 63824, 12139, 67971, 0, 0, - 3358, 65295, 0, 3104, 0, 0, 0, 0, 65772, 0, 0, 0, 0, 2862, 11326, 0, 0, - 94001, 3268, 66591, 0, 6552, 42367, 7035, 120558, 0, 0, 1814, 195092, - 10240, 195093, 0, 0, 0, 0, 0, 66960, 0, 0, 2837, 4341, 0, 0, 129982, - 125064, 195094, 0, 0, 66964, 0, 72721, 863, 66936, 0, 0, 43323, 66928, 0, - 0, 68054, 0, 3654, 66951, 0, 66942, 0, 0, 7653, 0, 0, 66587, 0, 0, 92401, - 0, 0, 12927, 0, 0, 129697, 13056, 0, 0, 3056, 0, 0, 195101, 0, 0, 74506, - 73770, 0, 0, 0, 0, 0, 0, 0, 0, 72233, 0, 5811, 0, 0, 0, 66817, 983855, 0, - 0, 128636, 129311, 0, 128041, 0, 67739, 120965, 0, 0, 67507, 0, 68375, 0, - 0, 70300, 0, 0, 0, 983698, 111078, 0, 11991, 128079, 0, 92943, 1502, - 74117, 127988, 0, 129478, 121253, 0, 67661, 0, 0, 125084, 68667, 0, - 74057, 68639, 0, 42898, 120742, 0, 74388, 74838, 120822, 0, 0, 0, 0, - 69452, 43214, 5893, 0, 0, 92496, 0, 0, 119907, 119900, 0, 0, 0, 0, 41950, - 0, 0, 68610, 0, 68626, 894, 0, 0, 12306, 73846, 0, 0, 0, 8636, 0, 121028, - 42503, 0, 92942, 0, 121468, 119241, 0, 126569, 5096, 5095, 2863, 127505, - 0, 10454, 42530, 5094, 0, 0, 13156, 0, 111035, 5093, 111024, 983419, 0, - 5092, 10708, 11327, 0, 5091, 0, 0, 9153, 4104, 78599, 78601, 2929, 42712, - 75067, 12272, 9832, 0, 0, 111105, 0, 0, 0, 0, 0, 0, 13106, 0, 0, 129111, - 0, 0, 0, 0, 9074, 111111, 0, 111110, 0, 8113, 11168, 92563, 1786, 111109, - 0, 111108, 0, 74423, 0, 586, 74414, 64359, 1267, 0, 127531, 0, 65731, 0, - 0, 0, 92932, 0, 0, 0, 0, 0, 0, 1228, 0, 42846, 0, 0, 70343, 1714, 74406, - 0, 0, 0, 127389, 66225, 0, 0, 42660, 0, 0, 3804, 0, 0, 129859, 0, 2826, - 0, 0, 0, 128396, 0, 0, 0, 0, 0, 0, 12206, 5839, 0, 68524, 74065, 73521, - 0, 0, 126240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67241, 917821, 7030, 0, - 10479, 64959, 2852, 0, 121225, 0, 0, 128586, 0, 6963, 0, 0, 0, 74786, 0, - 0, 0, 0, 121281, 0, 0, 0, 0, 113815, 121360, 0, 9994, 118680, 2864, - 64719, 1148, 0, 41677, 0, 0, 2765, 0, 128181, 0, 0, 0, 92516, 74777, 0, - 0, 65206, 0, 0, 0, 0, 69391, 0, 0, 983770, 0, 41839, 129616, 983773, 0, - 0, 6931, 0, 0, 7177, 125137, 0, 0, 0, 93020, 0, 10722, 0, 0, 128186, - 121050, 0, 0, 127207, 0, 750, 0, 129453, 63912, 0, 0, 7032, 0, 0, 4314, - 128600, 0, 128409, 730, 0, 127866, 0, 0, 41380, 0, 0, 0, 69697, 8240, - 92939, 0, 41378, 0, 6938, 70026, 0, 0, 66246, 0, 0, 0, 0, 0, 0, 983094, - 0, 92754, 41470, 64805, 0, 0, 0, 0, 0, 0, 0, 0, 92938, 68370, 0, 0, - 73831, 0, 0, 0, 2872, 0, 0, 0, 0, 604, 41097, 0, 0, 0, 0, 0, 127488, 0, - 2836, 0, 0, 9707, 0, 43202, 0, 0, 69374, 0, 0, 120916, 2832, 92702, 9670, - 12937, 0, 0, 0, 0, 2822, 0, 0, 92519, 0, 73752, 0, 0, 0, 1331, 92603, 0, - 11856, 73510, 129432, 5090, 5089, 0, 3200, 0, 0, 0, 5088, 0, 0, 9477, 0, - 0, 5087, 92325, 0, 96, 5086, 0, 0, 0, 5085, 64286, 0, 0, 43820, 0, - 129710, 0, 0, 119042, 0, 129660, 0, 0, 0, 0, 0, 127241, 120891, 7601, 0, - 591, 0, 118953, 0, 0, 0, 0, 0, 10939, 7246, 6933, 67142, 67141, 0, 74600, - 120695, 0, 67138, 65574, 0, 78058, 67140, 73851, 74598, 67139, 128094, 0, - 6372, 0, 73514, 7963, 6371, 0, 0, 125040, 0, 0, 0, 0, 0, 0, 0, 8258, - 123591, 0, 0, 65148, 118919, 42, 0, 0, 0, 0, 0, 0, 0, 0, 67135, 67134, - 67133, 0, 0, 0, 0, 67136, 67130, 74597, 11550, 0, 67132, 65868, 0, 12826, - 127872, 124116, 126235, 9737, 92448, 0, 0, 0, 8878, 0, 0, 0, 0, 0, 72220, - 9086, 0, 100952, 0, 7437, 7454, 0, 0, 0, 0, 9042, 0, 0, 0, 0, 3805, 0, - 67128, 44001, 67126, 0, 44022, 19949, 12200, 43522, 983045, 43525, 0, 0, - 0, 64422, 67125, 67124, 7602, 0, 0, 43521, 0, 0, 43711, 43523, 41447, - 8424, 68483, 8704, 2397, 0, 0, 0, 0, 0, 10916, 0, 129290, 93998, 0, 0, 0, - 127800, 67686, 9961, 123203, 0, 68842, 10792, 8889, 121402, 6951, 0, - 68827, 917835, 74342, 0, 0, 0, 68816, 129152, 0, 42909, 66597, 70092, 0, - 0, 10481, 4559, 0, 1956, 43138, 0, 0, 43490, 43148, 0, 0, 0, 43140, 0, 0, - 0, 0, 69268, 8533, 0, 0, 0, 0, 0, 4357, 0, 70289, 983157, 0, 42911, 0, 0, - 0, 10941, 0, 6962, 0, 0, 113808, 0, 11014, 0, 8942, 12000, 0, 0, 73515, - 0, 0, 0, 42650, 0, 75016, 63975, 0, 66210, 0, 0, 129150, 0, 11193, 0, 0, - 0, 0, 0, 0, 0, 43476, 0, 11024, 74811, 72787, 10563, 92954, 0, 0, 2462, - 92955, 0, 0, 66213, 6957, 0, 120559, 0, 0, 0, 74594, 983424, 92347, 0, - 110702, 110708, 110707, 127119, 3109, 127117, 119909, 0, 121434, 0, 0, - 4042, 0, 0, 0, 127123, 127122, 127121, 0, 127999, 0, 3503, 74444, 68300, - 6694, 127997, 0, 0, 74306, 0, 983757, 7736, 0, 0, 0, 10521, 0, 42173, - 9705, 0, 129719, 6955, 71467, 0, 6149, 3887, 19956, 1411, 2824, 0, 0, 0, - 1403, 0, 1347, 66282, 127996, 0, 0, 0, 0, 8640, 0, 1178, 1654, 0, 0, - 129529, 43314, 0, 0, 0, 0, 2873, 67461, 0, 0, 67085, 10861, 0, 0, 70377, - 0, 67082, 11159, 41391, 67084, 0, 376, 6987, 983182, 119904, 0, 8823, 0, - 12943, 65185, 100988, 42099, 0, 0, 100990, 0, 8301, 0, 0, 1684, 0, 0, 0, - 120620, 0, 0, 0, 42121, 0, 66781, 78067, 42115, 0, 127998, 0, 67080, - 1493, 42111, 67077, 4097, 0, 983767, 0, 65808, 41642, 0, 118568, 67076, - 41636, 67074, 65095, 110660, 72254, 121240, 41629, 12154, 75073, 0, - 128179, 74084, 64380, 0, 0, 0, 0, 0, 71193, 65371, 7078, 121218, 0, 0, - 74592, 0, 0, 43275, 0, 41434, 6062, 0, 0, 19916, 0, 6950, 9606, 9842, 0, - 65744, 0, 0, 128659, 0, 41615, 10105, 0, 0, 41632, 7493, 0, 0, 41622, 0, - 0, 0, 0, 7632, 983217, 983216, 9805, 5990, 900, 0, 122955, 0, 120869, - 3612, 0, 64376, 0, 5389, 129469, 73495, 0, 2839, 9621, 582, 0, 0, 3749, - 0, 7569, 0, 0, 92865, 6956, 4403, 0, 0, 3299, 0, 0, 119127, 65676, 0, - 74372, 0, 983497, 7598, 69819, 42469, 42242, 1918, 9542, 480, 7716, 0, 0, - 0, 0, 0, 69918, 0, 8328, 0, 118894, 0, 0, 0, 0, 11132, 983502, 66743, - 74185, 100531, 2854, 66747, 0, 65755, 0, 67120, 67119, 65835, 67117, - 66736, 67123, 67122, 67121, 9881, 100481, 65757, 100538, 100459, 67116, - 8648, 128377, 6741, 43047, 0, 13180, 0, 100487, 66754, 73507, 73487, 0, - 0, 41752, 0, 8641, 100490, 125185, 73477, 100462, 100541, 6942, 69501, - 1024, 42849, 41751, 0, 8941, 101034, 11121, 0, 9023, 40973, 121476, 9928, - 67109, 66865, 0, 67114, 67113, 67112, 67111, 0, 41206, 120724, 9049, - 67108, 43166, 0, 41200, 128201, 125142, 126537, 0, 0, 41188, 119553, 0, - 101007, 917548, 74585, 78626, 0, 0, 11466, 0, 120797, 0, 125067, 2261, 0, - 2860, 0, 0, 70828, 127925, 92357, 67106, 12065, 42872, 0, 43875, 67103, - 43856, 0, 67102, 67105, 7531, 40981, 2413, 100522, 67404, 100521, 0, - 67101, 41196, 100523, 0, 129723, 73512, 43117, 100495, 0, 0, 0, 0, 69876, - 0, 7173, 496, 0, 4313, 64607, 0, 0, 983202, 2065, 42793, 2842, 0, 83152, - 13132, 798, 0, 12801, 67098, 10686, 118528, 128143, 0, 8054, 9174, 67087, - 67086, 67097, 67096, 41611, 67095, 74504, 78854, 42512, 0, 78857, 42089, - 74613, 78856, 0, 101029, 100468, 42079, 100467, 0, 66961, 100474, 0, 0, - 0, 68338, 69958, 0, 0, 0, 0, 0, 78859, 42093, 128951, 100504, 0, 0, 0, - 4580, 0, 0, 0, 92167, 0, 3021, 42004, 0, 0, 42317, 41998, 0, 6946, 77920, - 0, 123610, 0, 0, 0, 121442, 42690, 9880, 0, 0, 64589, 0, 0, 127880, - 68035, 0, 11360, 0, 0, 72242, 0, 0, 0, 0, 0, 64941, 0, 0, 0, 6856, 65671, - 11244, 73706, 6959, 41994, 42907, 0, 0, 122902, 8617, 41982, 8860, 0, 0, - 121256, 0, 0, 9597, 0, 43172, 0, 10117, 0, 92297, 65865, 73549, 0, - 128077, 0, 126065, 0, 187, 0, 65669, 0, 4963, 0, 0, 0, 8964, 0, 7775, 0, - 41948, 0, 0, 101010, 41942, 65449, 3160, 65922, 13226, 42665, 0, 42663, - 128210, 41766, 983503, 78848, 78849, 41760, 1189, 905, 110620, 42658, - 78851, 67859, 9629, 6742, 0, 43625, 12952, 7888, 0, 3980, 0, 42656, 0, - 42055, 0, 0, 0, 64540, 0, 7867, 69218, 6236, 0, 73490, 10505, 0, 12851, - 118948, 0, 5474, 128843, 3103, 0, 41753, 41733, 78051, 983477, 78844, - 78845, 41739, 78843, 70744, 10931, 41756, 43347, 68098, 122909, 41746, - 119147, 92591, 41259, 66954, 69930, 2691, 121338, 11231, 41244, 0, 69800, - 66364, 41262, 67503, 0, 0, 41251, 0, 0, 11805, 0, 0, 68331, 94045, 0, 0, - 0, 74633, 41266, 126642, 0, 0, 0, 65741, 41737, 2275, 2666, 121232, - 41738, 4967, 419, 13126, 0, 0, 42822, 0, 6434, 74913, 0, 0, 6432, 0, - 69932, 128862, 769, 41742, 69927, 74805, 6433, 0, 547, 1943, 6439, 0, - 4994, 487, 0, 0, 3754, 0, 0, 0, 0, 74780, 0, 0, 1595, 92777, 74431, 0, 0, - 74860, 43267, 0, 0, 129083, 12185, 69406, 0, 73479, 100984, 0, 42856, 0, - 0, 983765, 128319, 75057, 0, 0, 0, 65612, 0, 669, 0, 0, 0, 0, 0, 70445, - 100404, 69929, 0, 0, 460, 121513, 0, 0, 0, 120747, 0, 121519, 121518, 0, - 0, 121515, 71491, 65187, 9044, 78497, 11760, 78494, 7577, 78491, 41912, - 100412, 0, 100411, 0, 0, 100394, 78501, 0, 2933, 78500, 0, 66441, 100392, - 100397, 100391, 1549, 0, 100415, 0, 41755, 6206, 8670, 120587, 0, 69935, - 0, 0, 69768, 73492, 0, 66958, 0, 0, 10552, 64342, 41922, 0, 917858, 0, - 917857, 2717, 0, 0, 0, 73664, 41908, 100722, 41916, 0, 0, 0, 92506, - 100723, 66664, 69803, 0, 100725, 0, 0, 43373, 0, 0, 8468, 100729, 121173, - 128297, 119210, 118952, 0, 0, 0, 100686, 0, 0, 0, 128703, 100670, 457, - 78502, 78503, 123180, 43006, 0, 8802, 113777, 0, 0, 0, 0, 126632, 0, - 41757, 0, 100657, 44000, 0, 0, 43534, 0, 0, 11961, 121316, 0, 0, 0, - 128736, 0, 0, 9499, 73522, 128330, 0, 0, 92260, 68184, 0, 0, 7256, 66993, - 983180, 0, 42161, 0, 119126, 128022, 65880, 0, 10802, 64861, 0, 0, 0, 0, - 0, 0, 73109, 0, 955, 0, 0, 5350, 64339, 0, 100705, 10875, 0, 5477, 73121, - 0, 0, 0, 67693, 69790, 0, 0, 3874, 0, 983741, 0, 0, 83272, 100674, - 127397, 0, 100989, 0, 41038, 67502, 9207, 42239, 0, 0, 0, 0, 74432, 0, 0, - 1455, 129680, 0, 11753, 119233, 0, 118594, 127854, 100716, 69801, 0, 0, - 43520, 0, 119556, 0, 0, 0, 0, 100733, 10788, 6088, 0, 129587, 190, - 983346, 12593, 100737, 129308, 64408, 0, 4417, 128615, 74359, 41744, 0, - 0, 100435, 6965, 0, 0, 13201, 100430, 69896, 78868, 74382, 11841, 7918, - 92721, 0, 0, 0, 1728, 0, 0, 0, 983350, 92679, 0, 0, 92711, 0, 0, 119536, - 73491, 66679, 8382, 0, 0, 100381, 0, 917889, 42254, 68371, 100383, 0, 0, - 0, 9923, 0, 0, 11763, 100386, 120688, 0, 78187, 0, 0, 0, 0, 8333, 0, 0, - 0, 917805, 74464, 0, 92320, 74080, 0, 69911, 11910, 0, 74141, 8963, 0, 0, - 0, 121396, 0, 41747, 0, 0, 8968, 0, 0, 129110, 110590, 0, 8836, 12315, 0, - 8300, 0, 0, 0, 8856, 0, 0, 69891, 0, 66965, 120405, 120402, 120403, - 120400, 120401, 12853, 43269, 7263, 120244, 6536, 120238, 120239, 65516, - 12321, 120391, 120388, 55287, 2237, 120246, 9588, 120248, 120382, 120383, - 120380, 120381, 0, 0, 3561, 0, 0, 10613, 0, 110583, 0, 0, 0, 128689, - 5006, 64328, 68219, 917894, 0, 8825, 122972, 0, 0, 0, 128616, 0, 119177, - 0, 0, 128641, 120225, 71366, 120227, 120228, 438, 4510, 41707, 8721, - 120233, 120234, 120235, 12840, 120229, 10845, 120231, 8096, 0, 120935, 0, - 0, 65589, 8733, 0, 0, 0, 0, 0, 0, 93984, 11262, 73747, 128522, 917902, - 64591, 42405, 0, 0, 1632, 127982, 128326, 0, 0, 121327, 121477, 42444, 0, - 0, 215, 41258, 128494, 64494, 1953, 10185, 0, 1256, 3910, 41260, 917903, - 0, 0, 41257, 0, 8675, 10700, 0, 124951, 0, 9333, 0, 121471, 0, 0, 0, 0, - 0, 499, 0, 70729, 42915, 0, 101000, 0, 100999, 0, 0, 73111, 128893, - 122897, 0, 125006, 0, 11118, 0, 128009, 0, 0, 118633, 9180, 0, 0, 0, - 100986, 43438, 118588, 0, 0, 0, 0, 120669, 64782, 0, 0, 73969, 565, - 42484, 118913, 201, 0, 42292, 69610, 0, 0, 119625, 43518, 0, 0, 1022, - 113788, 3880, 74247, 0, 0, 0, 0, 0, 0, 0, 0, 72272, 100997, 0, 0, 66937, - 74255, 0, 0, 92598, 0, 9903, 118993, 0, 68226, 0, 0, 0, 127788, 100955, - 83280, 7892, 0, 10777, 0, 0, 65562, 0, 101002, 0, 8039, 3363, 101009, 0, - 0, 66940, 12596, 70812, 0, 0, 0, 0, 42944, 92425, 74992, 64541, 0, 0, - 10520, 12802, 0, 12998, 0, 83270, 42861, 83273, 11415, 0, 7541, 125068, - 65878, 822, 0, 0, 5774, 194746, 43252, 0, 92619, 7672, 129281, 0, 0, - 7463, 0, 0, 0, 0, 0, 0, 121411, 0, 0, 0, 66938, 0, 475, 0, 120586, 7329, - 0, 0, 195088, 66291, 10645, 0, 6543, 100966, 0, 0, 119065, 0, 0, 0, - 983237, 195095, 0, 8923, 1645, 0, 0, 0, 3196, 72404, 0, 0, 43595, 0, 0, - 0, 0, 0, 195076, 0, 0, 5258, 4328, 0, 0, 0, 405, 11454, 0, 0, 0, 0, - 75052, 41245, 0, 195078, 4523, 11369, 0, 0, 0, 195079, 0, 0, 983510, 0, - 100961, 10480, 74610, 0, 0, 0, 12610, 0, 41247, 0, 7609, 118837, 0, 0, - 92253, 0, 984, 0, 92621, 0, 0, 129885, 73982, 0, 0, 0, 43369, 0, 0, 0, - 983507, 6634, 0, 71952, 0, 66930, 74214, 0, 67709, 0, 0, 0, 71114, 9552, - 0, 0, 0, 12997, 0, 0, 0, 0, 129109, 12883, 10994, 10529, 55283, 0, 74618, - 0, 67736, 10661, 19951, 9614, 2428, 0, 121023, 92837, 126224, 66933, - 71127, 0, 124996, 119162, 1952, 92181, 8455, 100958, 118654, 93033, - 119566, 100960, 0, 12183, 100951, 0, 64929, 0, 0, 0, 128290, 42509, - 73087, 3922, 9187, 983626, 0, 0, 119057, 0, 3353, 9358, 0, 0, 66680, 0, - 73975, 12879, 0, 9795, 68380, 0, 0, 119488, 0, 0, 41027, 0, 66931, 0, - 983631, 0, 70378, 0, 11751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129356, 0, 0, - 0, 0, 41029, 0, 126513, 0, 0, 0, 11294, 0, 66665, 0, 0, 127750, 0, 0, - 70105, 0, 983643, 0, 67843, 0, 0, 121167, 983895, 0, 8088, 129412, 0, 0, - 0, 983992, 6926, 72423, 0, 129569, 42369, 4350, 0, 65145, 9041, 43559, 0, - 0, 0, 41263, 0, 0, 0, 65825, 9577, 68199, 0, 0, 983122, 0, 6793, 0, - 70409, 0, 0, 0, 0, 64669, 0, 0, 0, 11200, 72725, 2995, 0, 0, 0, 7868, - 72720, 72020, 11386, 1009, 70405, 66871, 2333, 0, 0, 0, 0, 0, 70407, - 128121, 0, 0, 0, 0, 983657, 66949, 0, 74968, 0, 0, 110601, 0, 0, 41261, - 0, 0, 0, 0, 118989, 6736, 917883, 124132, 43010, 66952, 0, 69635, 73011, - 983716, 72750, 0, 7293, 0, 0, 0, 0, 111332, 0, 128245, 69928, 127071, 0, - 127072, 64445, 111336, 6635, 0, 0, 72707, 74936, 0, 0, 917876, 0, 93025, - 66948, 0, 111329, 0, 129887, 128045, 65219, 11925, 0, 92434, 0, 0, 9845, - 101317, 7546, 0, 0, 11230, 4985, 13288, 672, 8098, 0, 0, 0, 128126, - 42655, 0, 0, 1577, 11772, 78327, 0, 66673, 0, 65911, 118705, 0, 0, - 101303, 92180, 0, 0, 120566, 125140, 127177, 0, 0, 119593, 1539, 0, - 74969, 42731, 0, 74970, 71066, 0, 3051, 0, 73783, 0, 0, 0, 0, 78777, 0, - 983161, 0, 0, 101310, 0, 0, 0, 0, 0, 0, 3505, 8707, 0, 6725, 128013, 0, - 92314, 0, 66391, 5479, 0, 6686, 0, 0, 983318, 42754, 0, 0, 0, 0, 0, 0, - 128523, 0, 0, 4433, 41156, 0, 74971, 1443, 9339, 0, 92871, 10926, 0, - 43511, 0, 0, 983324, 0, 126086, 72236, 10021, 0, 101329, 0, 65914, 0, - 66749, 0, 6721, 217, 12466, 0, 0, 10443, 0, 68654, 0, 0, 0, 78334, 0, - 41250, 0, 129532, 128375, 0, 0, 69232, 0, 41252, 66682, 0, 119637, 41249, - 1366, 0, 0, 101326, 0, 0, 4397, 101324, 0, 66946, 9545, 101323, 0, 0, 0, - 3511, 0, 92190, 0, 0, 126244, 760, 0, 12088, 0, 0, 42256, 0, 0, 417, 0, - 111347, 41565, 74965, 0, 111355, 0, 0, 0, 2284, 0, 0, 11311, 0, 0, 0, 0, - 0, 0, 42273, 0, 69430, 121041, 0, 126643, 0, 65910, 0, 10246, 0, 68224, - 12169, 128858, 4552, 0, 0, 0, 1375, 66705, 128412, 0, 3329, 0, 42811, - 74251, 74192, 120794, 7840, 0, 0, 65374, 0, 0, 71072, 0, 4396, 0, 126608, - 0, 10331, 125224, 0, 11543, 0, 8944, 0, 0, 0, 0, 0, 19965, 43025, 10299, - 128436, 68845, 0, 69724, 67412, 92952, 0, 43811, 0, 128924, 0, 11062, - 128748, 0, 0, 0, 69276, 2901, 7865, 66945, 78354, 0, 78347, 0, 126123, 0, - 66363, 0, 0, 0, 74967, 7414, 0, 0, 92691, 0, 128507, 885, 64772, 65180, - 0, 71267, 852, 0, 0, 0, 78614, 121174, 129092, 67809, 9609, 12156, 0, - 122930, 43586, 11035, 10411, 0, 13268, 6710, 0, 0, 0, 43853, 77949, 4315, - 0, 111104, 0, 43639, 43343, 0, 0, 0, 73074, 0, 65812, 43431, 0, 0, 0, 0, - 0, 129890, 0, 0, 0, 0, 994, 125222, 127104, 127103, 73966, 66890, 0, - 65291, 70753, 0, 0, 0, 0, 66873, 4186, 92531, 127106, 127105, 6718, 7330, - 4406, 122946, 8480, 7319, 64373, 128699, 4413, 0, 0, 3198, 0, 0, 92469, - 111126, 0, 128591, 128681, 0, 0, 0, 101321, 73023, 742, 0, 2893, 78738, - 0, 0, 0, 2553, 42294, 6756, 0, 73020, 8363, 0, 2993, 128381, 3916, 4301, - 0, 1141, 42407, 0, 0, 7572, 973, 0, 125077, 0, 2415, 0, 0, 9640, 42333, - 0, 0, 129546, 42486, 43381, 65390, 0, 69434, 1202, 0, 0, 0, 0, 68484, 0, - 0, 64542, 3260, 0, 65388, 43502, 69904, 0, 6738, 0, 0, 74193, 0, 0, 0, - 74641, 6312, 0, 74556, 12446, 0, 0, 128076, 8229, 1235, 0, 11472, 83064, - 0, 0, 101366, 0, 0, 1740, 12872, 0, 985, 0, 0, 0, 12068, 983658, 0, - 101363, 0, 0, 0, 13133, 65071, 110780, 12655, 12134, 0, 92934, 0, 66915, - 120349, 119572, 0, 93030, 41572, 0, 0, 0, 41573, 0, 3931, 0, 74143, 2251, - 127034, 0, 0, 0, 0, 83067, 0, 129303, 0, 0, 0, 0, 0, 0, 83068, 0, 72740, - 128637, 72717, 92935, 120291, 0, 1780, 6936, 0, 0, 819, 0, 9694, 125228, - 0, 0, 0, 0, 8343, 8342, 8345, 8344, 8346, 8338, 7523, 6922, 8348, 8347, - 7525, 3346, 8339, 125004, 72705, 69462, 268, 0, 0, 5754, 94019, 0, - 110684, 8336, 0, 0, 0, 8337, 8341, 0, 11388, 7522, 0, 0, 0, 11090, 6953, - 125240, 0, 74973, 120708, 0, 0, 0, 0, 0, 110782, 0, 9038, 7887, 0, 0, - 42534, 64347, 0, 0, 67660, 120341, 0, 122933, 0, 120878, 0, 0, 73999, 0, - 64580, 0, 0, 64643, 0, 0, 74975, 0, 92227, 129052, 0, 83071, 83072, - 83073, 119154, 0, 119153, 0, 0, 5349, 72440, 2160, 917554, 7411, 0, - 983224, 0, 0, 0, 42736, 70747, 5756, 983229, 92946, 0, 42764, 0, 0, - 119529, 5752, 120600, 0, 0, 0, 0, 0, 78893, 0, 0, 0, 125242, 0, 0, - 120331, 122957, 0, 0, 67501, 0, 10080, 83056, 12647, 0, 0, 69252, 66882, - 0, 0, 0, 0, 0, 72005, 72845, 0, 0, 0, 0, 0, 74213, 0, 0, 0, 0, 0, 6302, - 0, 0, 0, 0, 1417, 983226, 0, 9452, 0, 74393, 0, 0, 110850, 0, 65391, - 63789, 69251, 78659, 78660, 41264, 78658, 6426, 42398, 9179, 78654, - 64906, 41255, 42036, 0, 41269, 0, 41267, 42436, 67759, 42323, 42034, 0, - 0, 42475, 42033, 0, 0, 68916, 43948, 0, 78673, 78674, 1659, 919, 42784, - 1671, 0, 6069, 9219, 0, 1661, 71489, 0, 92690, 10140, 9713, 78400, - 119143, 125236, 0, 2306, 0, 0, 6068, 10612, 0, 0, 121314, 92561, 41462, - 0, 0, 0, 0, 0, 0, 0, 128204, 10635, 0, 983225, 0, 0, 0, 983235, 92251, 0, - 121029, 983227, 0, 8100, 0, 78669, 78670, 13301, 78667, 9667, 78665, 0, - 0, 11003, 9904, 0, 0, 0, 0, 0, 0, 78680, 78681, 78677, 78678, 0, 10313, - 0, 0, 64320, 10265, 78686, 129404, 78684, 78685, 8945, 78683, 70750, 41, - 0, 0, 0, 0, 8655, 0, 0, 71333, 0, 0, 0, 0, 2585, 0, 65254, 3126, 0, - 74136, 10957, 0, 11160, 0, 0, 0, 0, 0, 11408, 0, 7443, 0, 0, 6997, 68004, - 0, 0, 8739, 11075, 0, 65216, 0, 69795, 2593, 0, 0, 0, 0, 125062, 0, 0, 0, - 4411, 0, 72837, 0, 43442, 78799, 0, 0, 0, 66351, 0, 0, 13061, 0, 78687, - 78688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119925, 0, 983637, 0, 0, 0, - 71494, 83399, 127541, 83398, 8022, 78808, 0, 73794, 0, 0, 83414, 119916, - 0, 0, 0, 0, 0, 0, 63799, 78427, 12063, 78425, 78424, 0, 0, 0, 75025, 0, - 297, 0, 0, 68326, 0, 78429, 78428, 7077, 2497, 128651, 0, 983112, 0, 0, - 0, 4292, 0, 74815, 10512, 0, 74814, 119931, 0, 72841, 2503, 65070, 1762, - 69794, 2495, 0, 71230, 94069, 77984, 0, 12654, 0, 1899, 0, 2507, 0, 8726, - 0, 65594, 0, 71272, 8892, 0, 0, 0, 0, 0, 420, 0, 0, 125130, 10797, 74637, - 0, 0, 0, 0, 63796, 0, 66581, 0, 119205, 0, 0, 194840, 0, 120788, 92817, - 0, 92956, 0, 0, 0, 0, 119230, 0, 0, 83496, 7399, 0, 66434, 42779, 0, - 92220, 42197, 92543, 5380, 0, 0, 1155, 11365, 43126, 0, 77971, 65684, 0, - 5601, 0, 42765, 78258, 0, 7987, 72843, 0, 69799, 6857, 0, 78735, 119165, - 0, 0, 4473, 0, 72426, 0, 65347, 65346, 65345, 0, 127384, 0, 69802, 0, - 73868, 0, 64826, 0, 0, 0, 0, 6218, 78422, 69676, 4555, 0, 83459, 70071, - 128670, 65190, 0, 0, 65244, 0, 0, 42519, 74472, 0, 0, 83466, 0, 0, 0, - 83468, 92581, 0, 0, 65370, 65369, 65368, 65367, 65366, 65365, 41086, - 65363, 65362, 65361, 65360, 43410, 11323, 65357, 65356, 65355, 5345, - 65353, 65352, 65351, 761, 65349, 19959, 69718, 0, 0, 0, 0, 64647, 0, 0, - 4699, 126077, 0, 129940, 0, 0, 0, 68074, 0, 0, 0, 128347, 0, 72829, 0, - 69773, 121438, 0, 0, 0, 73980, 78255, 78254, 83453, 43157, 0, 0, 0, 7946, - 12541, 0, 74615, 69780, 0, 0, 0, 0, 9005, 1225, 0, 0, 0, 0, 68011, 8847, - 0, 0, 0, 8329, 74590, 43878, 0, 0, 0, 3127, 2595, 71040, 69766, 129188, - 0, 41089, 0, 0, 70292, 983613, 12112, 0, 74200, 0, 8764, 0, 0, 0, 67273, - 67272, 67271, 71044, 0, 0, 0, 71042, 67266, 67265, 0, 67270, 67269, - 67268, 67267, 67282, 67281, 67280, 3572, 10023, 4959, 0, 0, 67275, 9729, - 125110, 0, 67278, 67277, 0, 67276, 0, 7996, 9907, 0, 13304, 83392, 0, - 72830, 0, 11095, 11100, 0, 83358, 83387, 10142, 0, 0, 0, 0, 68022, 0, - 83363, 128292, 19955, 0, 83366, 69807, 125246, 70124, 0, 72230, 83373, - 83385, 0, 0, 0, 0, 68020, 0, 2239, 261, 8406, 7791, 0, 7362, 0, 10696, 0, - 0, 9838, 118920, 0, 83477, 0, 0, 0, 6437, 68830, 83476, 0, 0, 74177, 0, - 0, 67288, 67287, 0, 67286, 0, 83470, 0, 67289, 67283, 83471, 70002, 0, 0, - 0, 67285, 11499, 67297, 7816, 67295, 55247, 68015, 10929, 67298, 122640, - 68017, 9642, 10912, 0, 67293, 11387, 67291, 67290, 70792, 0, 67715, 0, 0, - 68099, 13287, 74430, 10836, 0, 75053, 69775, 0, 128746, 7450, 0, 0, - 119648, 9697, 3606, 0, 0, 0, 0, 125029, 0, 0, 121262, 0, 128873, 1389, - 128871, 0, 0, 0, 12941, 0, 83438, 121062, 0, 12301, 83440, 0, 41102, - 66604, 72025, 0, 0, 0, 66600, 523, 92642, 71100, 74436, 0, 0, 0, 8608, - 83435, 72828, 128704, 0, 127402, 11307, 66707, 67301, 67300, 67299, 0, - 67304, 67303, 0, 0, 0, 0, 122654, 5908, 0, 0, 6744, 67310, 1699, 67308, - 67307, 67314, 67313, 6306, 67311, 983209, 72150, 69862, 3766, 2389, - 67305, 74569, 6611, 65700, 0, 0, 0, 42386, 0, 0, 2599, 917972, 119131, - 119049, 65717, 0, 0, 119654, 0, 0, 73538, 74203, 3760, 1718, 68160, 0, - 3776, 7335, 0, 0, 67324, 69861, 0, 69792, 0, 0, 3778, 0, 9462, 7824, 0, - 78896, 3768, 68142, 765, 72822, 3764, 0, 0, 113822, 129667, 12947, 0, 0, - 0, 118806, 73753, 0, 0, 0, 6829, 5225, 66901, 0, 0, 0, 0, 67319, 67318, - 3162, 67316, 67323, 67322, 67321, 67320, 0, 5353, 128190, 74179, 67315, - 0, 1010, 6851, 0, 67326, 67325, 127870, 6952, 67329, 67328, 67327, 2590, - 120036, 65552, 120034, 120039, 7183, 120037, 120038, 120027, 120028, - 118536, 120026, 120031, 970, 120029, 74611, 120019, 120020, 120017, - 67330, 120023, 120024, 120021, 10961, 113693, 11148, 67486, 0, 0, 128448, - 0, 113703, 64378, 0, 0, 0, 68821, 119649, 11358, 71172, 69797, 0, 11065, - 126464, 0, 68864, 0, 5694, 120839, 66784, 0, 4325, 3047, 0, 43652, - 120962, 93029, 69764, 0, 0, 0, 0, 5431, 6652, 0, 67753, 71460, 0, 0, 0, - 1129, 65016, 0, 65900, 1986, 7846, 0, 8661, 75058, 0, 0, 3845, 0, 0, 0, - 74400, 1456, 7530, 121382, 118631, 0, 0, 0, 120016, 0, 0, 0, 917863, - 127772, 119966, 0, 11002, 7026, 8145, 68216, 0, 12138, 71464, 0, 0, 0, - 12323, 130033, 917869, 0, 0, 0, 92316, 68494, 0, 0, 129384, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 42205, 0, 236, 0, 78867, 0, 0, 113784, 0, 0, 984001, 0, - 0, 119985, 8097, 0, 0, 68012, 72820, 11194, 0, 72824, 0, 127974, 0, 0, - 110602, 129948, 10416, 68070, 3872, 127508, 0, 0, 0, 0, 2838, 917867, 0, - 917866, 119589, 0, 0, 0, 0, 11096, 83019, 10553, 83421, 0, 0, 0, 0, 0, 0, - 73742, 6436, 10096, 0, 0, 0, 113687, 0, 4463, 68018, 0, 78074, 0, 983591, - 7184, 0, 0, 0, 0, 0, 0, 93825, 12818, 12032, 0, 0, 0, 0, 10357, 121418, - 8170, 0, 8556, 0, 9659, 0, 0, 0, 9556, 0, 4503, 92700, 9647, 64004, - 78185, 0, 0, 64002, 78889, 0, 0, 118910, 0, 6438, 0, 9109, 78884, 72382, - 64599, 0, 68009, 0, 0, 2447, 129863, 72381, 0, 126545, 0, 119002, 0, 0, - 0, 19937, 0, 1322, 0, 119204, 254, 72373, 0, 69392, 42425, 0, 0, 65204, - 42312, 0, 118578, 0, 42826, 129731, 42464, 120567, 0, 67155, 74796, - 64400, 64693, 126212, 77861, 0, 0, 67154, 0, 0, 0, 68008, 11785, 0, - 119142, 41978, 0, 0, 43244, 10536, 0, 9901, 7103, 0, 7102, 71428, 120748, - 3140, 0, 0, 68007, 0, 67258, 10909, 0, 1428, 0, 67254, 67253, 7699, - 12393, 67257, 0, 67256, 67255, 0, 0, 69389, 0, 0, 0, 0, 0, 67153, 0, 0, - 127383, 69376, 64554, 0, 3878, 0, 42352, 1752, 0, 129702, 42506, 0, - 10199, 0, 983468, 125231, 0, 0, 0, 720, 0, 0, 0, 68831, 0, 1464, 128339, - 0, 7974, 0, 125017, 68082, 0, 0, 0, 0, 74787, 0, 78864, 92258, 0, 0, - 78863, 0, 1302, 66288, 0, 0, 0, 67152, 0, 983611, 983618, 0, 0, 3995, 0, - 65608, 3714, 0, 0, 67262, 67261, 67260, 67259, 43251, 67264, 67263, 0, - 120557, 92346, 8672, 68006, 11964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 92610, 0, 468, 0, 0, 0, 983475, 0, 0, 128544, 129397, 65907, 983164, 0, - 0, 0, 0, 0, 983473, 41743, 0, 0, 0, 74880, 0, 121001, 820, 41741, 0, - 120667, 0, 64684, 126992, 128604, 126082, 69934, 65177, 6226, 353, 43645, - 0, 119612, 120738, 67700, 0, 0, 0, 0, 42457, 120276, 0, 120277, 1884, - 129637, 42418, 113678, 41157, 0, 42305, 120279, 0, 0, 41151, 0, 71430, 0, - 42344, 0, 194864, 0, 42497, 0, 194870, 72754, 69933, 73703, 0, 42521, - 8539, 128606, 0, 123609, 69957, 4788, 0, 68023, 0, 0, 983053, 0, 0, 0, 0, - 41590, 118717, 113754, 0, 0, 118901, 68637, 41136, 64351, 0, 128453, - 41154, 113731, 127038, 4038, 41143, 68232, 64859, 0, 129910, 0, 5435, - 129911, 6734, 41343, 127035, 0, 0, 41359, 66761, 0, 119835, 41349, 0, - 129915, 10374, 10310, 0, 0, 10254, 119836, 10278, 10262, 69858, 41363, 0, - 0, 0, 119840, 0, 41356, 10314, 10282, 0, 10378, 0, 40976, 10266, 0, - 119848, 40975, 123547, 129554, 0, 40978, 119851, 92945, 0, 0, 0, 119098, - 119083, 0, 71437, 119854, 69936, 0, 0, 3525, 6824, 0, 0, 119858, 128451, - 0, 72239, 113738, 0, 71424, 0, 0, 0, 0, 0, 10727, 7212, 129071, 71957, 0, - 0, 0, 67156, 808, 7207, 42387, 0, 0, 0, 0, 0, 0, 0, 0, 9225, 121149, 0, - 9145, 128060, 41018, 67841, 983159, 42300, 0, 3084, 983156, 125014, - 41025, 6037, 0, 194885, 0, 10290, 0, 3083, 10322, 111017, 129030, 0, - 41036, 0, 0, 43321, 65606, 0, 41032, 42388, 0, 64700, 0, 1445, 40961, 0, - 0, 0, 40960, 0, 67727, 0, 2223, 64952, 10402, 0, 0, 0, 10603, 0, 118577, - 71438, 0, 0, 0, 128469, 0, 0, 0, 0, 0, 0, 42585, 65032, 10704, 65030, - 4787, 0, 917556, 0, 127015, 0, 128118, 0, 0, 9525, 0, 0, 68773, 0, 0, 0, - 0, 40966, 0, 0, 3998, 0, 0, 65919, 71433, 11792, 2690, 0, 42836, 127150, - 41954, 194921, 194923, 6737, 0, 64933, 126260, 3487, 194873, 71427, - 72758, 65426, 72756, 66757, 0, 0, 41976, 9720, 74964, 11179, 41970, 0, - 12116, 65024, 0, 127912, 9048, 65028, 65027, 65026, 65025, 64757, 0, - 41488, 0, 8527, 0, 126480, 0, 41480, 41053, 3266, 0, 194876, 12093, - 41466, 122881, 78642, 1519, 983925, 3638, 65887, 65429, 0, 0, 0, 0, 8633, - 0, 0, 0, 125118, 0, 70375, 0, 0, 6368, 128124, 0, 70369, 8078, 0, 0, - 70373, 72876, 0, 7002, 121003, 41430, 0, 41051, 41484, 0, 0, 41050, 8872, - 0, 13099, 71445, 70371, 0, 6435, 72154, 11362, 0, 0, 0, 0, 41420, 0, - 3625, 74915, 41409, 71441, 0, 0, 0, 9672, 0, 0, 43317, 0, 0, 0, 41424, - 917598, 0, 0, 0, 0, 41417, 1261, 0, 0, 12102, 119662, 41401, 0, 127538, - 129518, 0, 124943, 72765, 3275, 92472, 0, 0, 0, 118686, 0, 128946, 0, 0, - 125129, 983141, 10598, 0, 128633, 6711, 0, 2920, 0, 0, 0, 0, 19928, 0, 0, - 3917, 0, 113756, 0, 0, 66588, 128078, 0, 0, 113721, 113758, 983081, 0, 0, - 41184, 0, 232, 0, 0, 74170, 0, 0, 0, 0, 9094, 0, 0, 92585, 0, 1064, 0, 0, - 10115, 0, 0, 0, 7862, 0, 13224, 0, 0, 66650, 0, 0, 72877, 1878, 0, 71434, - 2911, 0, 41178, 5427, 0, 0, 0, 12617, 41174, 0, 67148, 67147, 0, 42413, - 41167, 2406, 0, 0, 0, 0, 0, 9618, 128668, 0, 0, 0, 0, 41436, 9337, - 126067, 0, 41456, 0, 119086, 11333, 0, 6703, 0, 125071, 1613, 0, 0, 0, - 983192, 0, 0, 74500, 41460, 78197, 0, 0, 194899, 67144, 65841, 0, 121109, - 74064, 111146, 111144, 120375, 0, 111122, 0, 111121, 64687, 111120, - 42592, 3871, 0, 128305, 9111, 111163, 0, 111156, 120366, 121462, 11150, - 111154, 71488, 111179, 0, 111168, 0, 120362, 41587, 70391, 0, 74322, - 110589, 194908, 111166, 111133, 0, 71443, 194842, 0, 111151, 0, 0, 7928, - 111127, 111140, 41595, 0, 0, 65801, 110587, 110586, 110585, 110584, - 73712, 0, 41598, 3993, 121269, 1545, 40971, 121286, 72874, 0, 0, 0, - 120767, 65286, 0, 0, 0, 0, 2201, 0, 0, 5402, 0, 0, 74462, 73457, 0, 0, - 78194, 64326, 40969, 0, 128110, 983703, 40968, 0, 121139, 0, 128891, 0, - 0, 128513, 8020, 0, 41012, 0, 0, 65805, 41006, 0, 0, 74605, 0, 118942, - 43432, 0, 0, 92900, 0, 0, 68671, 120687, 0, 92958, 0, 0, 68332, 0, 40992, - 0, 0, 0, 0, 0, 42235, 0, 1741, 42370, 0, 0, 0, 11413, 126583, 0, 0, - 128769, 6470, 0, 74517, 0, 0, 120651, 40984, 0, 42742, 0, 12916, 6284, 0, - 41663, 0, 0, 68313, 72840, 70164, 41648, 0, 0, 2299, 41666, 0, 0, 2056, - 41656, 0, 0, 71917, 42219, 0, 0, 78112, 41676, 0, 0, 0, 41670, 0, 92590, - 2796, 0, 0, 9902, 0, 67988, 64785, 82995, 128822, 42631, 983040, 71890, - 0, 74164, 41238, 10049, 11405, 0, 64368, 0, 120925, 0, 397, 12299, 42139, - 0, 9590, 0, 0, 43661, 43819, 0, 6651, 3544, 0, 0, 9620, 0, 0, 0, 92229, - 1333, 7104, 0, 6425, 0, 0, 123561, 0, 0, 129725, 11976, 8554, 13055, 0, - 110733, 0, 110731, 41218, 0, 0, 128673, 1883, 0, 0, 70443, 41225, 70788, - 42419, 983707, 129450, 0, 127896, 0, 65809, 11837, 0, 129104, 7141, 0, 0, - 0, 0, 0, 42363, 0, 0, 0, 0, 69949, 119157, 64732, 0, 0, 126983, 0, 0, - 983697, 7140, 42051, 0, 4164, 118799, 0, 120569, 42049, 42042, 0, 0, 0, - 120637, 69938, 0, 42047, 0, 0, 8470, 11807, 128935, 0, 0, 194825, 74300, - 126267, 0, 120517, 68128, 0, 0, 0, 8736, 0, 42643, 72753, 129925, 0, 0, - 71432, 0, 93023, 110730, 72869, 110728, 0, 0, 0, 0, 68445, 0, 0, 2106, 0, - 11273, 120986, 43004, 0, 82988, 0, 961, 64307, 0, 0, 129752, 67711, - 110615, 0, 1696, 0, 9762, 12105, 0, 110622, 110623, 3264, 110621, 110618, - 43003, 110616, 110617, 0, 120359, 0, 128660, 0, 2322, 0, 70831, 11449, - 128187, 42868, 0, 0, 0, 0, 113746, 983238, 0, 129583, 66398, 0, 0, 0, 0, - 0, 69494, 119224, 0, 0, 64421, 0, 113739, 0, 65823, 0, 11182, 0, 0, 0, - 7766, 55268, 0, 4598, 0, 65839, 0, 0, 3315, 10851, 0, 6179, 92602, 6180, - 129524, 11952, 0, 78648, 78651, 78646, 78647, 78644, 78645, 3801, 78643, - 6176, 120580, 0, 0, 6177, 0, 78652, 78653, 6178, 0, 0, 0, 0, 2214, 8754, - 0, 0, 2137, 0, 0, 0, 0, 66889, 0, 0, 0, 8974, 2308, 0, 74579, 0, 2318, - 122920, 0, 8198, 0, 0, 0, 74307, 0, 119524, 0, 0, 6970, 0, 0, 0, 41159, - 0, 120363, 6385, 0, 128403, 0, 0, 126258, 0, 72785, 42053, 2075, 42057, - 0, 42052, 0, 0, 67651, 0, 9665, 0, 0, 13181, 0, 0, 69379, 0, 0, 0, 0, - 73010, 0, 0, 0, 41145, 0, 0, 0, 41148, 0, 7594, 113686, 75033, 119090, - 10869, 43458, 41146, 0, 0, 121456, 917630, 0, 0, 0, 0, 0, 65184, 11780, - 0, 42796, 0, 69742, 0, 65146, 66803, 0, 0, 0, 7358, 78241, 0, 7988, - 101371, 101370, 3271, 101372, 0, 69281, 0, 0, 0, 101369, 13070, 113736, - 42044, 101365, 1095, 101367, 3599, 101361, 101364, 101157, 129087, 66390, - 101360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42043, 43232, 67656, 121014, 42046, - 64355, 4036, 123601, 0, 0, 983062, 0, 11954, 0, 41191, 12986, 0, 194854, - 65441, 0, 72202, 0, 129338, 0, 101353, 101352, 12834, 0, 101349, 101348, - 101351, 72858, 101345, 101344, 41190, 101346, 129693, 4575, 41193, 0, - 429, 119174, 124931, 194859, 0, 65792, 128754, 78509, 0, 128866, 0, 0, 0, - 66786, 0, 194862, 10590, 0, 0, 0, 0, 0, 0, 6247, 10214, 65126, 68253, 0, - 0, 0, 983699, 1617, 8050, 0, 0, 0, 101358, 101357, 0, 101359, 101354, 0, - 101356, 6352, 0, 0, 0, 0, 0, 0, 0, 0, 92335, 0, 0, 42926, 0, 0, 0, 8689, - 78750, 70067, 42896, 74147, 3559, 101327, 0, 74526, 65850, 12327, 72763, - 101325, 0, 0, 72761, 0, 78903, 0, 0, 0, 0, 70079, 72751, 0, 12153, - 101340, 101343, 101342, 71940, 0, 101339, 101338, 0, 0, 681, 129406, 703, - 101335, 3272, 101337, 101332, 70077, 101334, 101333, 70514, 78902, 92532, - 71436, 42238, 124930, 3276, 0, 101311, 65928, 0, 0, 101307, 101306, - 101309, 101308, 78813, 78814, 3262, 78811, 42711, 101305, 0, 0, 101302, - 92746, 9995, 1655, 70131, 78818, 78815, 12479, 0, 0, 101296, 70513, - 42797, 0, 0, 128371, 43112, 101318, 43488, 101320, 101315, 101314, 42684, - 101316, 0, 0, 101313, 101312, 128308, 0, 0, 0, 0, 0, 0, 11031, 0, 0, 0, - 70386, 10348, 10412, 0, 0, 74329, 0, 0, 122647, 101285, 101284, 101287, - 101286, 8810, 101280, 686, 101282, 0, 0, 0, 0, 110709, 0, 0, 12040, 0, 0, - 65118, 110704, 0, 118891, 110599, 0, 110598, 0, 120543, 983679, 0, 65455, - 74413, 94097, 0, 119129, 0, 0, 0, 78776, 0, 64467, 10300, 10161, 10396, - 0, 0, 0, 0, 78773, 101294, 101293, 0, 1458, 101290, 0, 72429, 65120, - 11479, 0, 0, 6350, 101289, 101288, 71473, 1061, 69787, 9115, 43111, 0, 0, - 0, 0, 983979, 0, 120907, 1045, 0, 73913, 983564, 0, 0, 0, 0, 0, 0, 8486, - 0, 0, 0, 4362, 0, 0, 93054, 1025, 0, 0, 0, 0, 0, 92328, 128206, 0, 1774, - 0, 122913, 0, 0, 0, 11207, 0, 0, 3988, 0, 0, 983048, 0, 0, 8564, 983977, - 0, 0, 0, 0, 0, 0, 66513, 6256, 0, 579, 55218, 0, 0, 0, 127337, 0, 11814, - 0, 4488, 128716, 127336, 0, 10444, 118846, 78238, 0, 0, 127331, 4487, - 127849, 42832, 1032, 0, 43450, 0, 70155, 0, 614, 0, 127325, 0, 0, 128466, - 0, 127321, 0, 127322, 0, 0, 0, 1050, 7549, 127319, 0, 9314, 0, 0, 0, 0, - 0, 70434, 127314, 12527, 66504, 0, 0, 0, 0, 64333, 127312, 128547, 92594, - 0, 0, 0, 129316, 0, 124960, 10360, 6746, 0, 0, 0, 0, 13085, 9233, 0, 0, - 0, 0, 983474, 0, 92766, 0, 121114, 983944, 74212, 42819, 10910, 118627, - 68044, 9896, 0, 0, 120915, 0, 0, 7970, 0, 0, 0, 0, 113699, 9849, 0, - 122910, 0, 0, 10487, 69714, 0, 10103, 0, 4769, 0, 129967, 0, 2283, 0, 0, - 74785, 0, 0, 0, 110595, 110596, 0, 110594, 64565, 4773, 0, 0, 0, 4770, 0, - 0, 0, 65457, 69441, 0, 0, 127338, 983593, 4774, 0, 68497, 2259, 0, 0, - 10215, 0, 0, 0, 0, 0, 74776, 92160, 4768, 0, 0, 4099, 0, 110699, 110700, - 110697, 2225, 0, 0, 0, 41183, 125217, 11255, 42814, 880, 0, 0, 0, 0, 0, - 67756, 65246, 0, 0, 129463, 7095, 0, 0, 0, 0, 0, 0, 2427, 0, 7093, 0, - 11585, 0, 9962, 0, 12223, 0, 78211, 1434, 42939, 0, 11573, 0, 0, 0, - 121257, 0, 0, 0, 0, 74437, 0, 113711, 917596, 0, 8740, 0, 3782, 64331, 0, - 65167, 1014, 0, 0, 0, 10835, 129987, 0, 0, 0, 0, 0, 118824, 7302, 0, - 67707, 0, 1150, 10547, 0, 0, 68427, 0, 0, 0, 0, 118788, 0, 0, 0, 42257, - 8010, 0, 0, 0, 9643, 0, 0, 12864, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 68217, 0, - 68447, 129971, 0, 0, 0, 73701, 0, 0, 0, 65383, 0, 0, 0, 0, 0, 0, 43196, - 43194, 92549, 10744, 0, 990, 93772, 0, 0, 0, 0, 0, 66470, 0, 0, 0, 3945, - 0, 0, 0, 130039, 0, 127546, 127746, 1020, 73763, 92257, 118669, 0, 64748, - 0, 0, 10205, 0, 0, 10016, 0, 74051, 0, 43242, 125096, 2667, 0, 125037, 0, - 9911, 0, 0, 10097, 0, 0, 0, 118836, 0, 0, 0, 0, 68889, 10159, 113759, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 983343, 92291, 0, 127973, 72882, 0, 1041, 127182, - 6354, 0, 65364, 0, 0, 0, 72884, 0, 128477, 0, 65906, 127819, 72883, 0, - 128470, 5375, 72881, 0, 8215, 0, 10074, 0, 0, 0, 69899, 0, 0, 121426, - 41382, 0, 0, 5173, 65348, 527, 0, 0, 0, 128250, 0, 0, 0, 0, 0, 0, 42695, - 0, 42250, 0, 11187, 113695, 0, 1568, 66806, 0, 0, 113705, 0, 0, 129487, - 0, 0, 128839, 9069, 6144, 0, 0, 0, 0, 66783, 0, 74027, 118934, 66787, - 74580, 0, 110790, 6364, 0, 66794, 43508, 0, 92612, 0, 0, 0, 0, 128405, - 66449, 0, 0, 0, 0, 70714, 0, 70716, 0, 1044, 42411, 0, 0, 0, 0, 43239, 0, - 0, 0, 118572, 42450, 0, 0, 68479, 119237, 0, 0, 0, 0, 0, 69956, 11537, 0, - 121206, 0, 0, 0, 0, 1057, 566, 0, 0, 10907, 42274, 43464, 0, 118698, 0, - 78472, 71207, 42636, 0, 123603, 0, 0, 121171, 64659, 0, 127749, 0, 6357, - 6362, 0, 0, 2216, 9090, 0, 0, 0, 0, 68227, 0, 0, 0, 0, 1053, 12830, 0, 0, - 0, 1052, 1051, 459, 1060, 0, 66479, 0, 0, 0, 128061, 42490, 689, 6508, - 4163, 42298, 8639, 983338, 4246, 0, 43514, 42362, 0, 42337, 64596, 0, 0, - 0, 0, 0, 6359, 0, 43471, 0, 0, 0, 127274, 0, 6358, 6361, 1926, 6356, 0, - 7898, 0, 10935, 0, 127972, 121285, 0, 43685, 0, 0, 42910, 0, 8693, 0, 0, - 44010, 0, 120991, 121454, 0, 0, 0, 0, 129514, 0, 0, 0, 0, 73947, 0, - 129361, 92412, 0, 66477, 0, 0, 0, 43854, 71913, 0, 0, 0, 0, 72227, 65899, - 92275, 0, 0, 0, 68887, 0, 71057, 0, 0, 0, 0, 119183, 2923, 10853, 0, 0, - 0, 0, 72864, 0, 72773, 72772, 0, 120801, 65251, 122624, 68228, 0, 128548, - 0, 0, 5370, 70465, 2931, 73848, 0, 10188, 0, 118848, 0, 983942, 0, 0, - 120584, 72212, 0, 10844, 121016, 128195, 92424, 0, 0, 0, 286, 0, 1062, 0, - 0, 124127, 7395, 0, 1070, 128993, 0, 6095, 0, 0, 0, 127796, 126465, - 64497, 0, 0, 0, 0, 70054, 8189, 78272, 0, 0, 0, 0, 0, 113783, 42102, - 78276, 0, 0, 42101, 0, 78402, 67427, 33, 67425, 67424, 10824, 67430, - 67429, 67428, 427, 64723, 0, 0, 0, 0, 1031, 0, 0, 42104, 0, 0, 2328, 0, - 1071, 42899, 128486, 0, 7673, 0, 0, 1047, 194837, 0, 42908, 0, 0, 10651, - 0, 0, 0, 72433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13216, 0, 69716, 0, - 0, 0, 0, 0, 92411, 69654, 0, 0, 129904, 2761, 129909, 0, 0, 0, 0, 8643, - 0, 0, 94021, 2757, 11067, 0, 74498, 8910, 10689, 0, 0, 0, 71173, 0, 9196, - 71214, 0, 0, 0, 0, 118911, 0, 0, 0, 0, 0, 0, 0, 0, 68130, 119616, 0, 0, - 42477, 67482, 0, 4495, 0, 0, 0, 0, 70080, 10992, 0, 0, 0, 0, 9318, 0, - 6002, 0, 73808, 0, 92601, 42249, 7639, 43995, 0, 0, 5454, 0, 0, 0, 0, 0, - 0, 0, 121189, 0, 119173, 0, 9704, 120686, 0, 78436, 78435, 11204, 0, 0, - 1731, 0, 92937, 0, 67990, 0, 0, 0, 126576, 127018, 71951, 55265, 0, 0, 0, - 0, 127257, 73826, 0, 3840, 0, 41432, 0, 0, 68430, 0, 43253, 128284, 0, - 3371, 92936, 0, 0, 1479, 69282, 0, 1109, 77997, 0, 129154, 0, 92782, 0, - 0, 8868, 399, 67978, 74842, 0, 0, 194839, 73498, 551, 0, 10156, 0, 92572, - 0, 2544, 65074, 0, 0, 0, 0, 0, 0, 0, 128713, 0, 0, 74268, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 68045, 0, 0, 0, 3447, 0, 0, 121414, 2549, 110818, 0, 0, - 43564, 8946, 0, 74411, 66864, 0, 70480, 7980, 0, 113698, 0, 119653, - 66489, 0, 64695, 128063, 0, 0, 0, 0, 0, 0, 43452, 0, 92993, 0, 10919, 0, - 67810, 0, 0, 0, 0, 6450, 10055, 0, 0, 0, 0, 42720, 0, 9626, 0, 128055, - 74447, 0, 125127, 92573, 0, 0, 0, 119075, 0, 0, 66486, 0, 0, 0, 0, 0, 0, - 75028, 983883, 74839, 0, 0, 0, 0, 0, 55286, 0, 1055, 917628, 0, 0, 0, - 70516, 12146, 118623, 73956, 66488, 0, 0, 0, 0, 0, 0, 42518, 0, 0, 0, - 7407, 74978, 0, 0, 0, 0, 0, 0, 0, 10231, 0, 66626, 0, 0, 92951, 0, 65927, - 0, 0, 69696, 0, 92389, 0, 0, 0, 68095, 92950, 0, 10555, 0, 0, 9091, - 10798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43222, 0, 74982, 0, 0, 120952, 0, 0, - 2992, 7826, 74321, 110879, 125103, 74981, 92628, 0, 129903, 128289, - 128203, 4361, 129597, 1306, 78770, 1497, 983628, 0, 0, 0, 8248, 0, - 127253, 7973, 128706, 0, 0, 73122, 983949, 0, 0, 2963, 120653, 0, 128554, - 0, 0, 64258, 0, 0, 69677, 74983, 65103, 0, 125008, 42625, 0, 72022, 0, 0, - 64905, 0, 9512, 0, 119076, 6443, 983267, 0, 9135, 0, 0, 123202, 0, 0, - 983882, 93788, 0, 0, 0, 93767, 64256, 0, 11669, 0, 0, 4524, 0, 129182, - 128390, 0, 74266, 0, 0, 0, 70119, 78410, 69809, 121031, 55219, 69815, - 93765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2986, 0, 93763, 3437, 0, 6203, - 4247, 0, 11920, 8274, 68240, 129694, 1657, 0, 121276, 122951, 0, 2954, - 43506, 42837, 0, 0, 71179, 0, 0, 0, 66476, 68450, 0, 0, 0, 43362, 983135, - 129596, 11705, 0, 0, 0, 127354, 0, 11710, 0, 0, 0, 119507, 74429, 0, 0, - 1058, 129555, 0, 0, 5484, 1144, 0, 0, 0, 0, 0, 118972, 0, 65322, 0, 6441, - 0, 0, 2547, 66484, 43634, 0, 5871, 0, 0, 0, 0, 0, 0, 71204, 0, 0, 1865, - 0, 0, 69950, 0, 93021, 73713, 0, 71199, 65826, 2069, 0, 119092, 43999, - 2997, 0, 126588, 0, 65319, 0, 12316, 0, 0, 123630, 8776, 0, 0, 66294, - 13130, 0, 71191, 126625, 0, 10030, 11709, 12364, 983853, 0, 11704, 0, - 118641, 68672, 0, 0, 0, 0, 11706, 9710, 0, 82985, 0, 413, 65623, 0, 0, - 93980, 74446, 0, 1042, 0, 128378, 12171, 119240, 0, 69384, 4984, 0, 708, - 11391, 0, 0, 0, 983930, 1308, 0, 3673, 810, 0, 120933, 118567, 0, 0, - 1917, 3000, 0, 0, 0, 65628, 66387, 74470, 0, 0, 0, 10027, 0, 0, 0, 0, - 128831, 983168, 2980, 755, 0, 0, 65622, 0, 121012, 7277, 121022, 0, 0, 0, - 0, 8730, 0, 0, 0, 7274, 119250, 0, 7275, 0, 935, 0, 0, 377, 42325, - 121103, 0, 101133, 101132, 101135, 101134, 0, 74911, 2417, 101130, 0, - 19912, 0, 0, 101128, 101127, 0, 101129, 101124, 7248, 101126, 101125, - 1781, 5496, 3627, 62, 1649, 0, 964, 0, 0, 0, 0, 92897, 0, 0, 127364, 0, - 43689, 127911, 66287, 78812, 64389, 66575, 0, 73041, 0, 129687, 0, 7677, - 2991, 3293, 0, 0, 0, 72201, 0, 11341, 127049, 0, 65625, 9714, 11692, 0, - 0, 120850, 6478, 10195, 43673, 65237, 6241, 0, 0, 0, 6238, 0, 129889, 0, - 4409, 0, 0, 67170, 0, 0, 0, 94047, 6237, 5461, 66851, 9176, 92882, - 121341, 65231, 0, 0, 121182, 110581, 0, 44018, 0, 64765, 0, 0, 0, 5685, - 0, 2461, 0, 7091, 0, 0, 0, 68163, 0, 73030, 0, 0, 73928, 0, 0, 0, 0, 0, - 0, 110582, 0, 0, 68506, 0, 0, 0, 0, 0, 2542, 0, 0, 0, 128176, 5776, 0, 0, - 0, 0, 0, 11987, 0, 0, 75036, 68744, 0, 0, 10039, 42828, 0, 0, 0, 0, 0, - 10721, 67664, 43433, 0, 0, 41875, 0, 41870, 266, 129066, 0, 41873, 71271, - 0, 0, 0, 0, 0, 0, 41871, 66186, 3734, 7734, 43683, 8750, 110600, 66011, - 92899, 0, 127937, 0, 0, 10572, 0, 42906, 0, 64349, 7287, 0, 0, 0, 0, - 11167, 69220, 0, 43429, 0, 1697, 0, 0, 68633, 7286, 0, 128738, 10031, - 78754, 0, 68645, 8620, 0, 42162, 0, 0, 7285, 0, 119577, 0, 66842, 43677, - 41583, 0, 65799, 129332, 0, 0, 0, 0, 110806, 0, 3609, 0, 129448, 119074, - 125116, 126254, 128108, 73948, 0, 0, 0, 0, 129189, 42732, 92699, 74984, - 68620, 11691, 74985, 0, 0, 0, 0, 0, 6348, 243, 74075, 0, 0, 92309, - 123585, 0, 0, 10648, 8538, 43687, 0, 118723, 0, 70515, 0, 118954, 92886, - 13307, 129573, 92891, 0, 120770, 983850, 0, 0, 0, 0, 214, 0, 0, 0, 65893, - 0, 120488, 128386, 0, 92893, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 43, 0, 0, 1016, 0, 0, 0, 3885, 92, 65456, 64608, 0, 0, 0, 70656, 113742, - 0, 0, 0, 128128, 983857, 0, 0, 6791, 983861, 127960, 0, 0, 0, 118976, 0, - 7328, 92358, 0, 7995, 8759, 43421, 0, 68029, 92860, 0, 125272, 0, 3197, - 0, 0, 0, 983151, 0, 11595, 0, 0, 43435, 0, 0, 0, 0, 0, 70660, 0, 741, - 83291, 5494, 0, 70668, 1990, 11107, 4498, 0, 0, 70658, 0, 0, 2960, 73779, - 0, 8969, 101256, 43424, 0, 101257, 2950, 101251, 101254, 101253, 370, 0, - 101250, 101249, 0, 0, 0, 122967, 0, 0, 0, 122900, 0, 0, 983256, 0, 2964, - 43663, 0, 6344, 0, 0, 10144, 0, 8252, 729, 66016, 78446, 0, 0, 0, 78740, - 43669, 9032, 0, 0, 0, 0, 0, 0, 0, 0, 74612, 3761, 101261, 101260, 101263, - 101262, 0, 0, 3850, 101258, 0, 128389, 0, 0, 0, 0, 8611, 0, 0, 0, 43691, - 125032, 0, 41802, 120540, 0, 0, 0, 0, 0, 3848, 101230, 113800, 127536, - 101227, 101226, 101229, 101228, 663, 0, 0, 0, 0, 0, 0, 0, 0, 13221, 0, 0, - 101244, 101243, 101246, 101245, 0, 65579, 12980, 68046, 12143, 101069, - 128067, 0, 43441, 41804, 101241, 101240, 101235, 101234, 101237, 101236, - 66329, 0, 72324, 101232, 0, 125038, 0, 129383, 101214, 101213, 0, 101215, - 101210, 0, 101212, 101211, 0, 1097, 129033, 0, 101209, 101208, 93828, 0, - 101205, 101204, 101207, 101206, 101201, 101200, 101203, 101202, 0, 13110, - 0, 983886, 68229, 1000, 0, 0, 101222, 1209, 101224, 101223, 92354, 1073, - 6321, 77878, 92818, 0, 68213, 0, 12167, 0, 0, 0, 0, 73673, 121500, 0, - 121501, 0, 6587, 0, 0, 0, 9231, 0, 2959, 101191, 0, 101193, 101188, - 101187, 101190, 101189, 101184, 0, 101186, 42941, 0, 0, 68434, 0, 70742, - 0, 0, 12290, 0, 0, 110801, 0, 77873, 8205, 110803, 5131, 118542, 0, 0, 0, - 0, 0, 1944, 78453, 0, 0, 119990, 119991, 12701, 78492, 11308, 119995, 0, - 113702, 66836, 119999, 74263, 92382, 120002, 120003, 7075, 101196, - 101199, 101198, 41817, 73934, 42275, 101194, 120012, 120013, 120014, - 42943, 6041, 0, 41899, 0, 8002, 0, 41902, 0, 0, 64332, 0, 7813, 119117, - 0, 41900, 120633, 101167, 7281, 78455, 7279, 12041, 93027, 101165, 12673, - 0, 129123, 9660, 0, 72984, 101161, 0, 0, 0, 92901, 2970, 0, 101180, - 101179, 77870, 101181, 0, 0, 101178, 0, 0, 0, 0, 0, 3486, 101174, 69498, - 101176, 101171, 101170, 101173, 101172, 0, 69920, 101169, 66834, 0, - 984006, 0, 68312, 101150, 65673, 1019, 78495, 4148, 0, 12289, 101147, - 4316, 0, 13119, 983932, 101145, 101144, 0, 0, 101141, 101140, 43434, - 41865, 101137, 9163, 8659, 9072, 5867, 13302, 7622, 7120, 0, 0, 0, 0, - 7400, 5416, 101160, 101159, 10817, 101153, 101156, 101155, 0, 68162, - 41855, 41867, 0, 983228, 0, 11536, 71988, 0, 7115, 0, 0, 5498, 7337, - 41536, 0, 0, 92587, 7221, 8997, 0, 0, 0, 71949, 0, 0, 127814, 0, 0, 0, 0, - 0, 295, 0, 0, 0, 0, 121292, 0, 43454, 63903, 63902, 63901, 122644, 3971, - 0, 0, 2952, 0, 11038, 10901, 63900, 63899, 63898, 5198, 667, 43273, - 63887, 63886, 128458, 78521, 66830, 0, 92714, 4159, 0, 0, 63885, 63884, - 63883, 63882, 63880, 8555, 63878, 63877, 93057, 0, 0, 63881, 10746, 0, - 118983, 0, 63876, 63875, 63874, 63873, 7432, 1913, 41913, 43686, 0, - 128971, 0, 983894, 0, 446, 41911, 0, 63851, 63850, 41910, 0, 63846, 2972, - 63844, 7262, 0, 63849, 63848, 63847, 72990, 6570, 0, 7259, 63842, 4178, - 63840, 121321, 41521, 63894, 63893, 63892, 0, 0, 1105, 4180, 0, 7418, 0, - 129714, 63891, 63890, 63889, 63888, 0, 0, 0, 0, 1678, 0, 66909, 0, 0, 0, - 0, 11192, 128360, 128404, 9159, 70089, 63861, 63860, 63859, 63858, 63865, - 1615, 63863, 63862, 0, 0, 0, 0, 63857, 63856, 71902, 0, 1077, 0, 65099, - 0, 0, 0, 0, 0, 0, 42773, 121331, 0, 0, 119220, 120912, 129564, 0, 1112, - 119122, 8686, 120654, 0, 65081, 0, 0, 0, 11077, 0, 7260, 0, 5327, 0, - 63870, 63869, 3847, 63867, 0, 2903, 0, 3001, 66762, 0, 43746, 0, 63866, - 0, 0, 0, 0, 0, 127785, 68420, 2990, 0, 128254, 66957, 0, 0, 0, 1117, - 118987, 12212, 129003, 129151, 63836, 63835, 63834, 0, 0, 63839, 63838, - 63837, 0, 125095, 63833, 6042, 66360, 0, 74808, 0, 63821, 63820, 63819, - 63818, 0, 0, 9047, 63822, 128328, 6091, 0, 10691, 0, 74344, 8226, 0, - 63812, 63811, 63810, 63809, 2289, 63815, 63814, 63813, 6047, 0, 0, 780, - 63808, 77925, 77922, 65147, 63931, 63930, 2076, 1093, 9882, 63934, 2082, - 63932, 75050, 63929, 63928, 63927, 77934, 9806, 65566, 77933, 63922, - 63921, 2086, 0, 63926, 2984, 5968, 63923, 0, 0, 129458, 11137, 13169, - 5290, 2089, 0, 63827, 1088, 63825, 7268, 1084, 1085, 63829, 1083, 10131, - 7283, 0, 0, 0, 1092, 0, 7273, 983277, 44016, 43627, 0, 0, 0, 11809, 0, 0, - 0, 2965, 7258, 8808, 0, 1089, 7278, 63937, 63936, 43405, 11106, 940, - 5787, 10099, 63938, 101269, 63897, 101271, 2994, 101265, 101264, 101267, - 101266, 77939, 77940, 77937, 77938, 74343, 93043, 72704, 660, 10127, 666, - 0, 5532, 43667, 5533, 77941, 0, 0, 0, 979, 0, 0, 72706, 92652, 9108, 0, - 72716, 129403, 63951, 71685, 0, 0, 128782, 63946, 1707, 983843, 128612, - 63950, 63949, 63948, 63947, 63945, 6038, 63943, 63942, 101274, 0, 101276, - 101275, 0, 0, 0, 0, 0, 0, 73884, 0, 1690, 63919, 63918, 63917, 70865, - 43659, 0, 983848, 0, 2054, 0, 78515, 63916, 9184, 63914, 69737, 63911, - 63910, 63909, 63908, 0, 0, 63913, 6044, 0, 64838, 9061, 5534, 10672, - 11653, 124932, 5531, 101501, 101500, 101503, 101502, 0, 0, 11957, 101498, - 68668, 124922, 0, 0, 10474, 43426, 0, 42354, 101492, 101491, 101494, - 101493, 101488, 8413, 66841, 101489, 7269, 7272, 0, 0, 101471, 101470, - 78460, 0, 101467, 101466, 101469, 101468, 0, 0, 0, 66840, 0, 101465, - 128441, 0, 101462, 101461, 92187, 7270, 101458, 101457, 6628, 1076, - 128700, 0, 101456, 0, 0, 0, 0, 12807, 43413, 63906, 4548, 63904, 71187, - 70393, 41729, 44005, 1307, 0, 101473, 101472, 0, 0, 128268, 0, 8180, 0, - 127778, 0, 0, 5413, 43681, 123205, 3493, 0, 0, 0, 92544, 73937, 10517, 0, - 4518, 10990, 101447, 5167, 4481, 3771, 101443, 2710, 0, 66277, 0, 0, - 43073, 118579, 0, 0, 0, 121071, 0, 119659, 1628, 0, 0, 0, 65262, 66809, - 10783, 11172, 0, 0, 70840, 113679, 0, 119029, 0, 0, 41530, 66843, 4457, - 0, 0, 0, 0, 0, 41529, 0, 121210, 6031, 65807, 70814, 0, 101455, 71984, - 69705, 101452, 101451, 11926, 6033, 9656, 0, 0, 0, 68869, 0, 128930, 0, - 128100, 0, 42612, 43655, 0, 0, 0, 66468, 0, 0, 68623, 101423, 0, 0, - 101420, 101419, 101422, 101421, 0, 1151, 101418, 73709, 127544, 0, 71106, - 118722, 0, 0, 0, 0, 101437, 101436, 11527, 101438, 0, 0, 11538, 101434, - 0, 11020, 0, 66467, 101432, 8087, 71700, 101433, 9894, 101427, 73485, - 70824, 101424, 0, 78513, 8053, 0, 0, 0, 0, 101407, 101406, 0, 63845, - 101403, 78912, 78602, 101404, 13084, 42966, 8741, 0, 0, 101401, 0, 64605, - 83051, 101397, 473, 43415, 101394, 101393, 101396, 1087, 124966, 71275, - 101392, 0, 66439, 43218, 0, 0, 7237, 101414, 101417, 101416, 71996, - 101410, 92261, 101412, 121036, 4384, 74220, 101408, 2058, 917561, 0, - 129462, 0, 0, 0, 3857, 0, 0, 0, 64630, 0, 0, 74168, 127113, 125088, 4421, - 0, 0, 101381, 66400, 101383, 68431, 101377, 101376, 101379, 83053, 0, 0, - 69640, 127861, 0, 437, 73483, 0, 0, 0, 65236, 13290, 119180, 4997, 64306, - 0, 0, 4999, 0, 0, 0, 4711, 120769, 0, 2739, 0, 92915, 74834, 0, 127175, - 0, 0, 0, 0, 0, 1779, 6600, 6601, 0, 5325, 101390, 101389, 13058, 101391, - 101386, 0, 92186, 101387, 71845, 10575, 43399, 0, 101385, 101384, 1104, - 0, 0, 10655, 0, 0, 69497, 0, 1082, 110878, 0, 67401, 0, 0, 0, 0, 6783, 0, - 0, 42867, 69655, 44021, 6458, 0, 0, 0, 0, 0, 0, 1273, 43407, 0, 0, 0, 0, - 1313, 6322, 41720, 128627, 66433, 0, 0, 0, 11216, 0, 0, 0, 43437, 93833, - 0, 0, 0, 5122, 0, 72728, 129520, 70161, 0, 0, 0, 0, 0, 8303, 0, 128926, - 0, 10003, 0, 0, 0, 1686, 0, 0, 42834, 3664, 0, 126088, 121346, 0, 0, - 4324, 126, 0, 0, 0, 0, 0, 65166, 0, 0, 0, 0, 43817, 0, 43822, 0, 0, - 65600, 13002, 0, 0, 0, 1103, 0, 119575, 129452, 0, 13078, 0, 8116, 0, - 2050, 0, 0, 1102, 0, 6555, 0, 0, 74003, 74794, 0, 0, 42591, 127278, 0, - 1111, 0, 75047, 4707, 0, 0, 0, 0, 43468, 4522, 8645, 0, 74857, 0, 11352, - 0, 92787, 0, 2293, 0, 0, 0, 128265, 71709, 0, 121194, 0, 93827, 0, 0, 0, - 128488, 0, 160, 2677, 0, 0, 120141, 0, 983646, 70790, 0, 42770, 0, 71986, - 0, 43821, 113769, 0, 0, 43816, 0, 0, 1079, 3867, 64817, 0, 118549, 0, 0, - 64768, 0, 0, 4005, 983213, 0, 10991, 0, 92957, 917578, 92850, 917580, - 917575, 128314, 917577, 917576, 917571, 78534, 917573, 917572, 0, 0, - 128359, 73458, 0, 3339, 11448, 1106, 917591, 917590, 129192, 3340, - 917587, 917586, 917589, 917588, 917583, 10605, 1309, 74996, 120743, - 92650, 0, 0, 9485, 0, 129781, 0, 0, 0, 125002, 92533, 128487, 0, 129285, - 4338, 11238, 0, 66825, 0, 0, 0, 0, 122939, 0, 74128, 0, 0, 73680, 0, - 129438, 9553, 1590, 63777, 63776, 128677, 63782, 63781, 63780, 63779, - 1583, 101525, 101528, 101527, 101522, 101521, 101524, 101523, 41522, 0, - 92168, 983803, 66759, 0, 983580, 0, 0, 0, 0, 11394, 0, 983071, 0, 66823, - 1334, 0, 4479, 0, 0, 120663, 0, 122883, 10497, 0, 0, 983796, 66828, 0, 0, - 0, 6809, 63786, 0, 0, 63791, 63790, 1145, 63788, 101535, 63785, 63784, - 63783, 10192, 65267, 101533, 101532, 8928, 0, 0, 0, 0, 0, 74216, 66805, - 0, 0, 63759, 63758, 3523, 1074, 0, 121340, 74077, 92832, 0, 0, 63757, - 43145, 63755, 63754, 63752, 1349, 63750, 63749, 0, 0, 0, 63753, 63802, - 41084, 72784, 0, 41930, 63805, 63804, 11140, 63801, 41082, 43843, 42787, - 101514, 0, 101516, 101515, 63793, 63792, 0, 128241, 10201, 12238, 63795, - 42358, 92394, 43862, 101511, 101510, 41932, 66826, 101507, 101506, 92809, - 121136, 0, 7950, 63772, 63771, 63770, 0, 63767, 63766, 2793, 63764, 0, - 128501, 63769, 9530, 0, 92398, 0, 128642, 63763, 63762, 4595, 63760, 792, - 92808, 0, 0, 8742, 0, 0, 0, 63744, 0, 0, 120815, 63748, 63747, 63746, - 63745, 5055, 0, 0, 1090, 0, 125268, 11665, 92830, 4558, 78919, 72211, 0, - 0, 0, 11513, 983978, 6157, 63775, 63774, 63773, 0, 12170, 9067, 92843, 0, - 10872, 129643, 43891, 43893, 43892, 129747, 43933, 0, 128231, 0, 0, 0, 0, - 0, 11063, 0, 43888, 0, 0, 128368, 43889, 0, 73807, 983105, 7386, 0, 0, - 70295, 0, 0, 0, 71201, 128460, 0, 0, 0, 0, 69915, 2918, 66820, 65300, 0, - 124898, 64726, 2790, 0, 3793, 42065, 127829, 0, 124901, 0, 0, 0, 0, 0, - 92712, 0, 12923, 5270, 2166, 0, 0, 65813, 0, 128499, 0, 75012, 0, 10888, - 0, 93997, 94180, 3330, 129417, 0, 0, 0, 0, 0, 8220, 0, 0, 101581, 72457, - 1627, 101582, 0, 0, 5371, 101578, 0, 1826, 118794, 0, 0, 70023, 0, 0, 0, - 71108, 0, 0, 124907, 0, 92207, 68125, 74898, 71353, 0, 72006, 71098, - 70029, 0, 43116, 10190, 70019, 64346, 0, 101585, 66818, 101587, 70031, 0, - 12666, 120413, 120420, 120414, 101567, 120428, 0, 101564, 101563, 65509, - 101565, 7449, 0, 101562, 0, 7438, 0, 0, 9054, 971, 101558, 92803, 101560, - 65195, 64767, 101557, 101556, 0, 0, 101553, 101552, 0, 0, 0, 64303, - 101576, 2303, 0, 101577, 101572, 101571, 65833, 101573, 7271, 0, 101570, - 92802, 0, 12229, 0, 0, 43411, 73751, 126577, 64813, 0, 0, 10476, 0, 0, - 3932, 64958, 0, 0, 73989, 0, 0, 101542, 101541, 101544, 101543, 101538, - 101537, 101540, 101539, 92645, 65474, 4796, 118892, 129357, 65479, 0, - 42895, 11858, 65500, 983600, 9899, 92608, 2162, 404, 65484, 120639, 0, - 5788, 127852, 0, 65491, 1831, 66020, 0, 984012, 92588, 0, 1343, 120784, - 0, 0, 12018, 0, 0, 0, 0, 0, 4422, 4708, 3799, 101550, 119357, 0, 101547, - 101546, 101549, 101548, 983095, 0, 1364, 0, 8038, 101545, 0, 12868, - 129560, 70425, 55223, 0, 64414, 110689, 122978, 0, 0, 0, 0, 0, 118802, - 118644, 42855, 118856, 42866, 73525, 0, 0, 0, 66438, 0, 983996, 119356, - 92853, 119354, 0, 123556, 0, 73013, 67685, 128062, 119350, 0, 11864, - 10404, 10340, 119352, 1556, 5274, 0, 127821, 10017, 9733, 0, 69488, 0, - 41373, 0, 0, 0, 0, 0, 349, 4863, 41371, 0, 0, 0, 0, 72295, 4398, 8543, - 65618, 128018, 129784, 0, 0, 0, 12441, 0, 119348, 119347, 4318, 10452, 0, - 8032, 0, 119349, 119344, 0, 127844, 121156, 0, 110729, 119345, 8597, 0, - 110727, 9864, 0, 92796, 0, 92799, 0, 0, 0, 7722, 0, 0, 92797, 0, 0, - 66590, 0, 0, 129850, 0, 0, 0, 4965, 0, 917536, 0, 123196, 0, 0, 0, 10436, - 119342, 43147, 119340, 10356, 10420, 982, 2756, 0, 983997, 0, 0, 11162, - 119338, 0, 92914, 0, 65110, 0, 0, 983800, 78543, 0, 118793, 0, 128112, - 119179, 64476, 1694, 8216, 0, 0, 78539, 0, 65620, 0, 78537, 0, 0, 42158, - 65621, 69955, 120324, 120327, 120326, 120321, 120320, 120323, 120322, - 12314, 65616, 55221, 43825, 983553, 119337, 68060, 119335, 0, 71874, - 123628, 128537, 119332, 73089, 0, 41347, 0, 0, 8842, 0, 0, 4379, 127393, - 12692, 0, 0, 66353, 71875, 0, 0, 92907, 0, 0, 71877, 120303, 65619, 9872, - 0, 0, 1846, 120309, 120308, 119256, 71192, 120305, 120304, 120307, 6442, - 120317, 120316, 5379, 120318, 110717, 120312, 120315, 71876, 0, 65934, - 66497, 0, 66986, 0, 0, 0, 0, 0, 0, 0, 0, 72002, 0, 6151, 12110, 0, - 129761, 0, 66959, 0, 0, 0, 0, 68335, 129655, 0, 0, 0, 0, 0, 66041, 9676, - 10202, 0, 0, 0, 64575, 78929, 11965, 0, 124936, 0, 0, 0, 0, 0, 9698, - 66293, 0, 119651, 0, 0, 41921, 0, 0, 0, 119258, 0, 0, 0, 0, 0, 8012, - 12355, 12353, 0, 0, 74107, 0, 0, 41925, 0, 41920, 65444, 0, 0, 41923, - 12694, 0, 10112, 1294, 0, 120091, 0, 120092, 0, 0, 128474, 121400, 0, 0, - 0, 8718, 0, 10284, 10268, 10380, 10316, 92593, 0, 71850, 0, 0, 92889, 0, - 0, 0, 0, 9342, 12829, 0, 0, 101239, 127978, 0, 0, 69428, 0, 73767, 72347, - 0, 7956, 598, 0, 72329, 93837, 0, 0, 128860, 0, 120041, 0, 0, 101242, 0, - 0, 847, 0, 9529, 0, 0, 0, 101247, 120035, 0, 0, 0, 67411, 0, 0, 119497, - 120040, 0, 128580, 0, 9624, 0, 0, 0, 65463, 1554, 0, 0, 0, 0, 71879, 0, - 0, 0, 121161, 19963, 123566, 0, 72326, 92933, 71887, 10324, 10292, 65546, - 0, 68141, 8372, 0, 0, 83018, 120022, 10175, 10388, 42799, 0, 983181, - 10568, 0, 127400, 0, 0, 0, 983762, 0, 4366, 0, 983805, 0, 0, 42608, 0, - 9884, 0, 0, 0, 0, 129180, 0, 128964, 0, 0, 1609, 0, 92773, 73448, 0, - 11661, 0, 5818, 0, 0, 0, 9540, 0, 2554, 5158, 0, 2213, 0, 0, 78522, - 43079, 0, 0, 8264, 11175, 64553, 120863, 42155, 0, 0, 0, 0, 0, 69552, - 8676, 0, 129927, 0, 451, 0, 0, 0, 0, 0, 0, 123167, 43609, 0, 0, 1440, 0, - 0, 0, 127061, 11005, 0, 66656, 127063, 0, 129936, 0, 127065, 43393, 0, - 120643, 0, 0, 0, 0, 120798, 0, 0, 0, 0, 0, 0, 70435, 64356, 0, 0, 0, 383, - 7154, 127815, 43495, 128809, 121448, 0, 0, 0, 11286, 0, 0, 0, 0, 0, 0, 0, - 42644, 73555, 129797, 129801, 8292, 0, 4980, 113726, 92674, 70130, 0, 0, - 0, 0, 74912, 0, 10631, 83330, 100488, 68042, 0, 0, 7900, 101252, 0, - 78779, 4198, 128555, 0, 0, 0, 123159, 0, 0, 12931, 0, 0, 0, 2088, 0, - 72164, 129284, 0, 0, 69265, 0, 0, 0, 69694, 92838, 129794, 8593, 0, 0, 0, - 0, 0, 0, 11798, 0, 100483, 0, 0, 0, 64211, 128865, 120494, 0, 0, 0, - 121228, 68901, 128788, 0, 0, 65162, 0, 0, 0, 0, 0, 128130, 0, 92264, - 127153, 0, 128818, 0, 0, 61, 0, 74373, 92182, 119554, 92862, 0, 12089, 0, - 65834, 83281, 119671, 128701, 0, 0, 42566, 42743, 0, 69824, 0, 92653, 0, - 0, 42621, 0, 64833, 0, 0, 0, 43266, 0, 0, 0, 74843, 0, 0, 119103, 64417, - 0, 0, 64737, 0, 0, 8930, 0, 0, 66900, 10056, 1800, 0, 0, 0, 129337, - 121175, 7743, 0, 0, 119528, 92640, 92453, 9034, 6039, 129139, 10075, 0, - 0, 0, 10748, 0, 0, 0, 0, 0, 92984, 0, 0, 128183, 129421, 0, 43064, - 127558, 0, 7539, 0, 0, 0, 0, 0, 0, 0, 92898, 42567, 0, 0, 73886, 0, - 129988, 12326, 0, 92848, 0, 0, 11355, 0, 0, 0, 0, 69437, 128222, 129803, - 129811, 119537, 72327, 43005, 65342, 118902, 0, 0, 8644, 0, 0, 11186, - 74296, 41909, 0, 128682, 2791, 127472, 1891, 0, 0, 41907, 66647, 0, 0, - 41906, 0, 129672, 10773, 70206, 0, 0, 0, 6412, 2061, 8520, 13146, 0, - 92836, 83275, 65902, 2882, 0, 126232, 65852, 0, 92795, 0, 123627, 0, 0, - 92794, 0, 0, 128098, 0, 0, 0, 70871, 0, 92792, 0, 120087, 0, 0, 92793, - 93971, 0, 3844, 6842, 0, 0, 6612, 0, 0, 0, 0, 0, 783, 0, 0, 0, 983064, - 68032, 119225, 0, 0, 68378, 4556, 67839, 68480, 78663, 120069, 120074, - 67657, 10510, 4382, 74218, 42194, 0, 92806, 9177, 8902, 93958, 9839, - 92804, 120700, 92807, 0, 63999, 41904, 41917, 9788, 120973, 92805, 1862, - 0, 0, 0, 41915, 0, 41919, 63994, 41914, 7981, 0, 0, 0, 0, 0, 0, 0, - 120834, 0, 0, 0, 6784, 78788, 0, 0, 0, 0, 127534, 127484, 127476, 983874, - 0, 983960, 64289, 65289, 0, 129539, 129575, 64509, 0, 0, 126505, 11051, - 0, 66635, 55259, 65885, 0, 128310, 0, 0, 0, 0, 7500, 4506, 0, 0, 0, 0, 0, - 126609, 4040, 128680, 6167, 0, 42945, 0, 0, 0, 0, 7830, 43036, 0, 0, - 63990, 19947, 63988, 63987, 0, 63993, 10440, 9611, 2244, 71883, 0, 65260, - 63986, 11446, 63984, 92641, 3435, 119652, 0, 119108, 0, 128632, 0, 0, - 12748, 0, 0, 92705, 0, 78790, 0, 0, 63956, 42458, 63954, 63953, 63960, - 63959, 63958, 11596, 0, 11469, 69267, 42306, 2723, 0, 0, 70027, 0, 0, 0, - 128093, 2880, 0, 0, 0, 0, 128506, 3498, 4378, 0, 129825, 0, 65551, - 118928, 0, 43387, 0, 64415, 128898, 0, 0, 0, 0, 8161, 393, 12013, 0, - 92216, 126479, 63965, 63964, 63963, 42345, 0, 2174, 63967, 42498, 0, - 2927, 0, 63961, 0, 0, 983946, 0, 69699, 0, 42340, 0, 0, 0, 10730, 0, - 69688, 0, 64187, 118535, 0, 12437, 9813, 0, 42453, 1604, 9565, 0, 69701, - 69235, 42414, 110724, 129196, 0, 42301, 11372, 0, 917973, 0, 0, 63980, - 63979, 63978, 0, 128207, 12017, 63982, 63981, 73687, 0, 63977, 63976, - 72794, 0, 0, 0, 63971, 4347, 4416, 63968, 11009, 63974, 63973, 402, - 69390, 13147, 0, 0, 64646, 13228, 0, 0, 3515, 74252, 65261, 0, 0, 6259, - 0, 0, 0, 0, 0, 0, 74813, 74425, 0, 126998, 126114, 0, 0, 0, 129933, - 983717, 0, 0, 74301, 0, 122633, 0, 0, 74060, 69508, 0, 66235, 5145, 0, 0, - 128394, 0, 73120, 0, 7402, 0, 0, 0, 7952, 7832, 43382, 66616, 0, 983950, - 120852, 0, 127875, 64866, 0, 0, 0, 78784, 74248, 0, 0, 983197, 0, 0, 0, - 78656, 42390, 0, 0, 983940, 0, 0, 0, 92839, 9508, 0, 9544, 11520, 0, - 110898, 3377, 0, 129562, 0, 0, 0, 0, 66989, 66280, 0, 127198, 0, 0, 0, - 1955, 119565, 0, 0, 3076, 0, 42168, 73049, 66304, 0, 0, 8917, 42403, 301, - 0, 111175, 0, 0, 0, 0, 0, 0, 67819, 92987, 0, 0, 0, 983206, 0, 69403, - 3182, 0, 0, 0, 0, 0, 42169, 123162, 74244, 0, 42329, 0, 66326, 6841, 0, - 128913, 0, 1219, 3934, 71276, 11483, 74510, 101122, 121110, 42442, 65470, - 69565, 0, 64622, 7759, 42482, 485, 0, 0, 42290, 0, 0, 42280, 0, 0, 11655, - 64379, 127913, 42431, 10126, 42318, 0, 119631, 74397, 42470, 0, 68315, 0, - 110829, 74041, 0, 0, 0, 5411, 0, 0, 0, 64205, 0, 64206, 42393, 64478, - 1310, 125007, 0, 12052, 10643, 55271, 72727, 0, 121045, 0, 0, 118852, 0, - 0, 0, 0, 113826, 0, 0, 64385, 0, 0, 0, 0, 0, 0, 93848, 92560, 2713, 0, - 9650, 0, 0, 120602, 1406, 983650, 78174, 92659, 0, 68223, 0, 0, 0, 0, - 43475, 0, 65287, 1508, 127938, 8779, 10569, 75034, 0, 0, 0, 0, 0, 0, 0, - 70786, 0, 0, 128344, 9185, 0, 42932, 43403, 0, 0, 0, 0, 0, 0, 0, 0, - 12955, 0, 2888, 0, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 0, 0, 0, 0, 129028, - 13203, 129722, 10429, 10365, 0, 0, 127165, 7503, 0, 113676, 68381, - 119658, 0, 8986, 0, 10632, 11934, 11452, 1332, 0, 0, 0, 0, 73741, 1791, - 8850, 9288, 0, 2892, 0, 43394, 555, 0, 0, 0, 0, 64172, 118899, 0, 0, 0, - 0, 8854, 0, 5858, 73101, 10582, 0, 0, 1361, 0, 0, 7905, 0, 65256, 0, - 41210, 0, 0, 71884, 0, 0, 0, 6828, 0, 92302, 0, 1342, 68440, 0, 64161, - 10903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64381, 0, 0, 0, 42245, 126467, - 41972, 0, 0, 0, 9127, 0, 66619, 126489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11620, - 0, 1149, 68316, 0, 0, 0, 0, 0, 92492, 0, 118784, 0, 0, 0, 12838, 0, - 118819, 0, 0, 0, 0, 41087, 0, 0, 0, 0, 12036, 0, 124143, 0, 0, 0, 64428, - 12227, 0, 0, 0, 0, 125248, 120964, 0, 0, 0, 0, 0, 69566, 0, 0, 0, 0, 0, - 1743, 0, 0, 0, 65186, 122626, 0, 0, 0, 0, 64439, 0, 68062, 0, 111259, - 111258, 43866, 0, 111263, 3395, 9362, 111260, 0, 111257, 111256, 111255, - 0, 0, 41091, 3426, 1344, 111249, 111248, 126215, 4735, 11111, 6119, - 111251, 42699, 0, 0, 74818, 1423, 0, 0, 0, 0, 12039, 10559, 0, 0, 0, - 9472, 67734, 11929, 0, 0, 0, 0, 128826, 0, 11579, 0, 0, 128364, 0, 92185, - 0, 0, 1004, 92584, 0, 0, 0, 129755, 0, 2556, 0, 0, 72790, 0, 0, 9686, 0, - 0, 0, 70109, 111102, 0, 10718, 13154, 111100, 9139, 0, 0, 0, 0, 0, 0, 0, - 0, 92831, 92810, 41708, 12860, 41703, 0, 42090, 5403, 10352, 73917, - 129144, 111096, 111088, 5140, 3753, 118785, 41704, 0, 43078, 127789, - 2207, 129360, 0, 983207, 92362, 0, 0, 2410, 92525, 0, 0, 0, 0, 0, 0, 0, - 0, 119253, 0, 126601, 0, 2066, 74199, 0, 43463, 10659, 119623, 68863, 0, - 1336, 0, 0, 69463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126639, 0, 272, 0, 0, 0, 0, - 983965, 128133, 0, 0, 124940, 0, 1190, 42146, 1335, 42177, 43867, 0, 0, - 10448, 0, 125041, 0, 0, 2099, 5120, 2409, 7799, 0, 74424, 0, 126581, - 4731, 0, 111199, 111198, 111197, 111196, 11689, 0, 74977, 9913, 129430, - 0, 0, 0, 111195, 111194, 11694, 0, 11690, 111189, 92842, 111187, 11693, - 111193, 111192, 43097, 11688, 0, 78797, 194, 111186, 111185, 111184, 0, - 0, 0, 11226, 4519, 70337, 10898, 43072, 70205, 0, 0, 0, 73094, 10695, 0, - 7540, 0, 110984, 41859, 6067, 92790, 110982, 0, 110981, 13311, 92788, - 41857, 92791, 8359, 121224, 12689, 0, 983132, 64577, 92789, 111203, - 68183, 111209, 111208, 6064, 110988, 0, 110979, 74142, 0, 111201, 111200, - 6051, 123613, 0, 0, 983374, 0, 983651, 0, 0, 0, 110864, 10537, 110862, - 1276, 0, 6549, 6052, 0, 0, 0, 0, 118687, 0, 0, 0, 0, 1960, 0, 71232, - 66297, 0, 129313, 0, 0, 1345, 111213, 111212, 111211, 8956, 43083, 0, - 111215, 64682, 0, 6430, 69563, 111210, 119814, 0, 0, 0, 119817, 0, 492, - 43087, 0, 0, 0, 0, 0, 2582, 0, 0, 7444, 72863, 0, 2297, 111243, 73837, 0, - 0, 65096, 197, 74183, 0, 69571, 111241, 111240, 129760, 66515, 43550, - 119829, 111229, 111228, 93764, 111226, 0, 0, 111231, 111230, 71686, 1799, - 0, 42148, 74336, 0, 0, 65340, 111220, 110974, 2262, 111217, 111224, - 74931, 111222, 10896, 0, 0, 0, 0, 6338, 111003, 110997, 110994, 111006, - 111002, 111005, 0, 111279, 111278, 111277, 72133, 0, 111273, 111272, - 110961, 3171, 6623, 4961, 0, 886, 55216, 8654, 110965, 111270, 74390, - 64603, 111267, 129283, 68122, 0, 43084, 0, 0, 0, 0, 69693, 8994, 10944, - 65938, 111239, 111238, 111237, 111236, 66279, 92890, 42510, 0, 0, 6804, - 0, 1947, 0, 0, 0, 42759, 0, 1705, 122631, 0, 0, 0, 0, 72722, 74036, 0, 0, - 66720, 111281, 111280, 0, 4909, 111285, 111284, 111283, 4904, 0, 43503, - 1365, 9253, 42757, 0, 7462, 0, 11860, 0, 0, 119587, 0, 917579, 92526, 0, - 125035, 0, 111311, 111310, 0, 0, 0, 0, 93977, 0, 0, 0, 0, 3629, 0, 13005, - 0, 3628, 0, 111295, 0, 0, 0, 0, 111290, 64809, 2928, 4905, 111083, 851, - 55233, 111291, 111059, 43086, 9114, 43870, 42583, 9315, 4822, 4906, - 121097, 2847, 111028, 10330, 0, 1251, 7777, 41852, 125059, 111327, - 111032, 111325, 12646, 0, 10259, 0, 65821, 75046, 6018, 0, 111324, - 111323, 111322, 68372, 111319, 111318, 71893, 2558, 0, 64584, 111321, - 111320, 0, 0, 0, 0, 78911, 111308, 111307, 69500, 73987, 74599, 71895, - 93012, 0, 128715, 0, 12867, 111296, 0, 0, 11044, 111300, 111299, 8904, - 11824, 65857, 0, 128674, 129027, 4387, 0, 0, 124920, 0, 0, 0, 0, 11842, - 0, 0, 0, 5136, 1968, 983041, 126627, 1337, 0, 0, 0, 0, 66506, 0, 0, 0, 0, - 42314, 121384, 0, 0, 6120, 0, 65670, 67457, 0, 43082, 6016, 0, 42284, - 71894, 4276, 111314, 3619, 41638, 69691, 0, 42322, 8853, 111043, 0, 490, - 0, 13231, 68384, 72310, 65350, 0, 0, 0, 68245, 42435, 6154, 0, 65354, 0, - 0, 42397, 334, 72732, 42416, 65359, 65273, 74634, 128227, 4442, 10364, 0, - 778, 41626, 42455, 7989, 0, 3227, 69907, 111053, 0, 2915, 11502, 983214, - 41702, 10309, 0, 0, 0, 0, 0, 0, 0, 127268, 127258, 127267, 65215, 64410, - 127260, 71175, 0, 0, 0, 0, 0, 0, 41700, 110651, 69266, 126488, 0, 0, - 42495, 0, 0, 0, 10460, 43364, 0, 1356, 3728, 42713, 0, 0, 42342, 10914, - 0, 42489, 64310, 66896, 41861, 42297, 0, 0, 41860, 64862, 0, 0, 5289, - 42336, 128658, 0, 92529, 42410, 71129, 120624, 0, 2649, 74493, 0, 126635, - 0, 3382, 42449, 9081, 1658, 11936, 93019, 113814, 11269, 0, 0, 43100, - 69888, 65508, 0, 0, 121451, 0, 0, 0, 129780, 69272, 4732, 128283, 0, 0, - 0, 121113, 2236, 126551, 0, 6048, 0, 0, 73965, 0, 0, 0, 0, 10151, 9681, - 4475, 0, 41142, 2100, 0, 0, 6035, 0, 123599, 10296, 0, 0, 0, 0, 0, 0, 0, - 983312, 68488, 10392, 10328, 0, 43462, 0, 0, 0, 8979, 0, 0, 983309, 0, 0, - 0, 10977, 0, 10344, 0, 65299, 10408, 0, 0, 121187, 66505, 0, 0, 0, 0, 0, - 122648, 43074, 73799, 0, 0, 122944, 0, 3446, 0, 129891, 128692, 0, 0, - 119582, 4474, 0, 43093, 6282, 0, 0, 127372, 0, 0, 0, 129881, 0, 0, 0, 0, - 66910, 67811, 92277, 0, 64948, 0, 74347, 0, 0, 0, 983981, 8194, 0, - 121165, 11010, 0, 8893, 0, 983988, 0, 0, 0, 983322, 7925, 0, 0, 113825, - 0, 1352, 11069, 7707, 0, 126486, 0, 0, 0, 0, 65605, 6040, 0, 10071, 0, - 128156, 43750, 0, 8899, 69873, 0, 0, 983316, 128208, 7820, 69615, 0, 0, - 7746, 1492, 0, 0, 0, 66866, 0, 11788, 65913, 0, 0, 43095, 0, 0, 92265, - 2999, 0, 120720, 0, 371, 120759, 6023, 0, 0, 11708, 0, 0, 6323, 0, 0, 0, - 8938, 6043, 65866, 0, 78910, 0, 72419, 0, 129480, 2589, 74332, 1689, - 7802, 0, 0, 0, 0, 66704, 0, 129992, 0, 0, 128127, 6049, 0, 4027, 0, 0, - 111334, 111333, 1503, 111331, 0, 111337, 11951, 111335, 2387, 0, 0, 8289, - 111330, 7326, 66514, 65514, 0, 64865, 0, 9668, 0, 0, 0, 0, 93060, 6036, - 92768, 4026, 74089, 127091, 0, 0, 75044, 110821, 0, 110819, 0, 0, 0, 0, - 6021, 0, 128288, 0, 43155, 0, 110822, 124152, 111343, 42691, 111341, - 111340, 2246, 166, 0, 0, 0, 10623, 408, 0, 111339, 13298, 0, 7426, 43694, - 0, 0, 8811, 0, 0, 129753, 0, 0, 74134, 983054, 0, 127811, 0, 0, 0, 6645, - 646, 128813, 0, 42129, 0, 120880, 0, 8697, 0, 120936, 122953, 0, 0, 0, - 5809, 1950, 0, 92432, 68339, 0, 42136, 0, 0, 0, 0, 0, 0, 111354, 983984, - 0, 0, 111349, 111348, 43330, 111346, 111353, 111352, 41567, 111350, 0, 0, - 0, 0, 111345, 111344, 8285, 0, 4509, 0, 128361, 0, 77774, 129851, 0, 0, - 41727, 0, 0, 0, 0, 0, 71188, 0, 74512, 7027, 3886, 0, 74023, 92888, 0, 0, - 126092, 94058, 119855, 0, 121455, 11707, 119852, 0, 7939, 10342, 92460, - 72747, 121408, 917569, 0, 71198, 94077, 119847, 0, 0, 7201, 0, 123554, - 120866, 983987, 1540, 0, 0, 124923, 0, 119856, 41718, 71177, 0, 0, - 128001, 118699, 0, 119040, 0, 9619, 120840, 0, 0, 0, 0, 3560, 0, 6070, - 129000, 0, 2922, 6082, 70147, 65009, 983973, 0, 0, 0, 0, 0, 0, 3607, - 65863, 0, 92487, 42153, 121042, 0, 983862, 2032, 0, 0, 0, 0, 129985, 0, - 43085, 6057, 0, 0, 0, 0, 0, 0, 0, 0, 638, 6083, 126976, 0, 0, 2305, 0, 0, - 118658, 6056, 10878, 0, 0, 6085, 119351, 0, 3915, 0, 0, 0, 0, 0, 0, 4028, - 1787, 0, 43096, 0, 0, 1768, 0, 0, 0, 128125, 0, 0, 583, 129137, 0, 0, - 66004, 0, 0, 0, 92859, 0, 55267, 120810, 128995, 43075, 65049, 0, 74531, - 0, 93009, 70694, 0, 0, 129375, 9869, 128815, 1771, 0, 0, 0, 0, 0, 0, - 119115, 113708, 0, 0, 74101, 0, 0, 0, 0, 83367, 0, 0, 0, 12539, 123631, - 0, 0, 129846, 73862, 69842, 9897, 0, 100561, 0, 124142, 0, 0, 0, 8931, 0, - 1415, 8866, 74552, 0, 128312, 0, 983566, 43106, 127275, 71089, 1580, - 92278, 68424, 0, 0, 7658, 3440, 78215, 1562, 0, 0, 129031, 0, 0, 0, 0, 0, - 0, 6028, 68900, 42892, 0, 111016, 0, 0, 0, 0, 0, 128269, 0, 66776, 42946, - 127276, 92849, 0, 72448, 120510, 11599, 0, 11602, 11591, 11574, 11581, - 11597, 11598, 6253, 11571, 11584, 70273, 11569, 122937, 8906, 0, 5755, - 2636, 0, 10815, 11619, 129094, 0, 7815, 11616, 11617, 70064, 11618, - 11604, 7869, 11612, 0, 42152, 0, 122941, 0, 92586, 126247, 0, 92173, 0, - 0, 6616, 0, 0, 120875, 391, 0, 0, 0, 42296, 11588, 0, 0, 0, 68397, 0, 0, - 42335, 983189, 0, 0, 7538, 94040, 0, 42491, 0, 0, 128088, 4576, 0, 0, - 43809, 4277, 0, 3563, 0, 42338, 368, 0, 0, 42412, 0, 78209, 119144, 0, - 43814, 983616, 1849, 0, 9921, 42451, 4253, 0, 0, 118688, 42404, 64657, - 73919, 3618, 78338, 0, 0, 0, 0, 0, 929, 6827, 42035, 0, 0, 0, 67847, 0, - 0, 0, 0, 0, 0, 0, 0, 4578, 64513, 0, 0, 0, 71049, 68090, 127086, 43305, - 0, 73462, 118530, 0, 42048, 10166, 0, 127095, 113810, 983128, 0, 983991, - 0, 0, 42483, 0, 0, 0, 42291, 0, 71047, 0, 6641, 525, 66404, 0, 8763, - 125091, 0, 0, 0, 0, 0, 42504, 42581, 74280, 6915, 42310, 0, 8559, 0, - 983994, 125100, 0, 0, 11666, 8679, 0, 1576, 42423, 0, 0, 73840, 983092, - 11374, 0, 10889, 129076, 0, 42462, 0, 77982, 0, 2718, 42424, 0, 0, - 127166, 0, 1179, 0, 0, 0, 363, 11015, 72229, 0, 43857, 0, 66692, 0, 0, 0, - 11041, 72018, 0, 0, 0, 0, 125184, 0, 92520, 0, 9492, 66709, 9212, 12833, - 0, 0, 1297, 122932, 0, 0, 0, 0, 0, 12924, 0, 0, 10090, 125249, 0, 42505, - 0, 42507, 0, 42311, 92940, 120919, 68401, 10759, 0, 0, 120924, 42351, - 42919, 9398, 66292, 0, 9422, 122942, 122943, 0, 0, 0, 129440, 92575, - 1603, 0, 0, 0, 0, 0, 69703, 11250, 0, 0, 10546, 0, 0, 11600, 0, 2797, - 73821, 42427, 306, 714, 3058, 120154, 0, 0, 0, 42395, 0, 11607, 0, 11198, - 127512, 0, 72232, 129067, 0, 42433, 0, 7603, 74063, 0, 42141, 0, 0, 0, - 129085, 8244, 362, 125069, 0, 8037, 0, 0, 0, 69510, 41606, 66696, 77912, - 0, 2093, 0, 120676, 122929, 41604, 0, 0, 0, 0, 10523, 1446, 42320, 0, - 120247, 64773, 42472, 0, 0, 1722, 5581, 0, 64496, 0, 0, 64914, 0, 42620, - 128603, 124988, 0, 0, 10549, 130035, 71190, 0, 0, 0, 0, 0, 71712, 0, 0, - 0, 0, 0, 0, 0, 7684, 66338, 0, 1174, 0, 0, 983621, 0, 0, 0, 42277, 0, - 42456, 65667, 0, 0, 0, 0, 42417, 0, 0, 120812, 42304, 0, 0, 0, 74443, - 127894, 0, 8313, 0, 0, 1316, 66690, 0, 0, 0, 0, 0, 0, 66844, 983715, 0, - 0, 0, 65200, 3383, 0, 0, 70063, 122947, 0, 0, 42420, 119185, 0, 0, - 983917, 0, 121079, 72369, 0, 42343, 124980, 42706, 1751, 42496, 65742, - 13166, 0, 0, 0, 0, 0, 42683, 12697, 0, 0, 0, 125047, 0, 42346, 0, 0, - 3757, 0, 0, 121075, 65869, 0, 9247, 74976, 3193, 0, 0, 42459, 7596, 7921, - 0, 74095, 0, 42499, 11590, 66006, 0, 42307, 0, 43953, 118591, 0, 1023, - 474, 0, 0, 0, 0, 42487, 0, 0, 0, 42295, 0, 121474, 72237, 0, 9835, 0, - 127782, 0, 12275, 0, 0, 8595, 0, 0, 0, 0, 0, 10118, 0, 129156, 0, 0, 0, - 0, 0, 0, 699, 0, 120923, 11601, 0, 92941, 0, 7581, 0, 92530, 0, 0, 0, - 7765, 65583, 0, 0, 64597, 43444, 0, 92197, 0, 64279, 7036, 5823, 1937, 0, - 917854, 65415, 13308, 65417, 0, 65217, 0, 0, 11017, 0, 0, 7294, 0, 0, 0, - 0, 42466, 65416, 68858, 0, 71350, 65413, 92381, 126498, 12964, 42240, - 1941, 0, 0, 1713, 118679, 0, 0, 11407, 42441, 128262, 6297, 0, 0, 0, - 42481, 0, 0, 7179, 42289, 0, 120921, 969, 0, 0, 0, 6165, 0, 0, 0, 0, - 42402, 0, 0, 0, 129511, 0, 72234, 0, 0, 64876, 92635, 6046, 0, 6208, - 128870, 129309, 73749, 0, 0, 42422, 0, 0, 128155, 73775, 338, 0, 121369, - 0, 42328, 10767, 0, 8115, 0, 0, 0, 0, 92687, 0, 0, 0, 0, 73029, 0, 0, 0, - 71687, 4486, 128082, 2171, 0, 10925, 0, 0, 0, 0, 42309, 10257, 0, 10273, - 7668, 10305, 42461, 74882, 42349, 8832, 0, 0, 10644, 0, 129531, 42278, 0, - 0, 69874, 0, 129949, 42429, 0, 42316, 11223, 0, 0, 42468, 0, 0, 0, 65402, - 0, 0, 72235, 0, 0, 41963, 120990, 0, 0, 125013, 6823, 42391, 1588, 65400, - 0, 0, 0, 65398, 787, 0, 0, 0, 0, 2078, 127239, 65399, 0, 0, 0, 65401, 0, - 121196, 0, 113816, 644, 0, 71335, 0, 3659, 0, 0, 0, 13107, 92669, 0, - 10502, 74457, 0, 11221, 41554, 0, 0, 0, 41557, 11209, 0, 11070, 119221, - 0, 0, 73858, 41555, 9514, 0, 66771, 64641, 92447, 0, 7520, 73888, 77955, - 0, 0, 0, 0, 0, 64527, 0, 118707, 12723, 0, 68776, 0, 0, 0, 78835, 4055, - 78826, 77960, 65212, 0, 127353, 12319, 0, 0, 983218, 7964, 65427, 0, - 65424, 72217, 120966, 0, 65425, 74890, 128251, 0, 0, 0, 3448, 10827, 0, - 9866, 74527, 0, 0, 8625, 69783, 92304, 10477, 0, 0, 0, 65423, 0, 0, 0, 0, - 6152, 0, 0, 6629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11046, 11490, 0, 4485, - 71126, 0, 0, 0, 0, 0, 5869, 118533, 119633, 0, 7040, 3588, 0, 12825, 0, - 0, 128569, 0, 0, 0, 0, 0, 126637, 0, 0, 128449, 64499, 65245, 127367, - 1171, 127368, 69717, 127365, 1805, 8772, 0, 127363, 9930, 65247, 0, 0, - 2338, 127362, 92695, 0, 0, 0, 69219, 0, 120104, 0, 120103, 72221, 120102, - 129924, 118814, 8734, 4212, 0, 0, 66701, 0, 65862, 0, 120095, 42903, 0, - 0, 0, 126117, 426, 0, 120098, 8251, 0, 65436, 0, 2120, 43302, 1224, 0, - 65576, 0, 66876, 1764, 6074, 0, 12858, 0, 0, 65439, 6378, 74566, 128885, - 41960, 0, 41644, 0, 2129, 0, 9222, 0, 0, 4259, 9092, 0, 41961, 0, 0, - 66357, 42331, 64935, 0, 0, 1293, 0, 2132, 0, 983569, 0, 2454, 0, 3613, - 128837, 71117, 0, 0, 69681, 10978, 10840, 0, 10668, 72826, 127197, 9118, - 120164, 0, 0, 0, 1157, 64903, 8638, 0, 101295, 0, 0, 0, 0, 67466, 128981, - 10086, 0, 11128, 0, 0, 65430, 74013, 6079, 0, 10764, 127910, 64435, - 128051, 1339, 118643, 65428, 1317, 8822, 0, 0, 0, 127143, 0, 0, 0, 43110, - 0, 10428, 129848, 0, 0, 5742, 43076, 4692, 0, 0, 4007, 5004, 128781, 0, - 751, 6595, 6596, 0, 66373, 0, 0, 64908, 0, 6593, 72349, 12004, 119192, - 74097, 43108, 0, 0, 119333, 92188, 6598, 0, 6599, 0, 93031, 74194, 0, - 121483, 66674, 6597, 0, 73921, 0, 64745, 2281, 0, 0, 128996, 43790, 0, - 2430, 41678, 71492, 0, 43785, 113716, 0, 121263, 0, 0, 1921, 0, 19927, - 70390, 65406, 0, 43786, 4284, 128346, 72210, 43789, 12841, 9229, 0, - 42285, 0, 0, 0, 0, 3521, 0, 118690, 8325, 0, 65403, 0, 1854, 0, 0, 0, 0, - 0, 0, 0, 0, 4344, 0, 65433, 6076, 0, 0, 74764, 12074, 0, 0, 129148, 0, - 12934, 119555, 65432, 128877, 0, 6071, 65434, 0, 65435, 4053, 128623, 0, - 0, 0, 917934, 69823, 127463, 0, 121403, 127473, 8421, 73836, 0, 43705, - 502, 0, 65431, 0, 0, 0, 1303, 316, 7364, 0, 2136, 0, 120796, 64365, - 43480, 92639, 4860, 0, 127877, 0, 129728, 9583, 0, 5546, 0, 118565, 0, 0, - 0, 5544, 127475, 0, 70352, 5543, 128917, 72821, 12137, 5548, 0, 0, 10007, - 0, 127523, 6077, 0, 65452, 0, 119341, 11214, 65952, 0, 72226, 0, 0, 1319, - 74210, 65410, 67399, 92606, 0, 0, 118660, 0, 66716, 83513, 4691, 128619, - 9345, 621, 92872, 0, 122889, 65411, 0, 74575, 121246, 65408, 73899, 0, - 9474, 2812, 119118, 65412, 3786, 65409, 8894, 83246, 119611, 7923, 3716, - 92798, 0, 0, 0, 7012, 124122, 128439, 9566, 0, 94176, 0, 65012, 126242, - 545, 9575, 0, 10050, 12718, 0, 8859, 6820, 124915, 129941, 120740, 0, 0, - 9119, 2787, 0, 984000, 8507, 2012, 7985, 0, 0, 0, 0, 194634, 0, 410, 0, - 0, 120789, 120609, 0, 120378, 120379, 0, 0, 120374, 72742, 120376, - 120377, 120370, 120371, 120372, 120373, 3860, 120367, 72205, 74031, - 111131, 73685, 11748, 120365, 7941, 111134, 8749, 111132, 12698, 111129, - 361, 110793, 845, 67509, 0, 0, 4562, 72241, 2926, 0, 4569, 0, 110797, - 43487, 0, 0, 0, 74287, 122885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6291, 0, 0, 0, - 9734, 0, 0, 0, 0, 127754, 7359, 83523, 43863, 0, 111150, 8769, 111148, - 111147, 111145, 4859, 111143, 111142, 0, 0, 0, 0, 12172, 111136, 0, - 127899, 111141, 64764, 4210, 111138, 0, 804, 0, 83520, 0, 70344, 0, 0, - 67202, 10091, 67200, 119257, 67206, 67205, 67204, 67203, 72302, 0, 0, 0, - 128959, 0, 1425, 92259, 119229, 11049, 0, 71480, 42649, 8482, 0, 0, - 66715, 67209, 11940, 67207, 664, 0, 0, 0, 70200, 127525, 0, 70194, 93061, - 111155, 68474, 111153, 6032, 67218, 67217, 7430, 194670, 70191, 0, 0, 0, - 0, 0, 41161, 0, 9765, 10993, 41162, 0, 70189, 1169, 111181, 0, 1905, - 6034, 41164, 64744, 43236, 0, 128800, 73110, 0, 0, 788, 0, 0, 111167, - 111128, 1663, 128976, 42901, 127237, 67211, 67210, 0, 0, 67215, 67214, - 67213, 67212, 111160, 111159, 111158, 111157, 0, 69492, 0, 111161, 43612, - 0, 0, 0, 10855, 67223, 9355, 67221, 65198, 120355, 0, 221, 0, 0, 0, - 121141, 7191, 118930, 72208, 125212, 0, 0, 0, 0, 67228, 67227, 43333, - 67225, 0, 0, 0, 67229, 0, 7245, 0, 74405, 69922, 72219, 111178, 3873, - 8367, 111174, 111173, 111172, 43649, 0, 111177, 111176, 0, 11164, 0, - 74403, 111171, 111170, 111169, 7682, 74404, 1462, 10235, 0, 0, 0, 0, 0, - 111130, 0, 0, 74402, 0, 92299, 0, 0, 74052, 0, 126127, 120549, 0, 64295, - 0, 0, 0, 0, 0, 120662, 0, 0, 67231, 67230, 10755, 55257, 11155, 128568, - 983137, 9470, 0, 127540, 0, 69680, 64384, 0, 128607, 0, 0, 0, 122987, - 73764, 8204, 0, 0, 0, 0, 0, 8728, 0, 10904, 73446, 19936, 7833, 0, 0, 0, - 0, 92546, 0, 0, 0, 8537, 0, 0, 0, 121244, 0, 0, 2254, 128193, 0, 0, 0, 0, - 3062, 0, 0, 0, 0, 0, 41160, 41147, 41158, 0, 120777, 0, 41155, 111116, - 111115, 111114, 0, 121332, 111119, 111118, 111117, 129878, 0, 129091, 0, - 0, 0, 64594, 2456, 66867, 0, 0, 0, 0, 3721, 0, 0, 1230, 2678, 0, 3597, - 917795, 0, 0, 92215, 0, 67737, 8352, 0, 0, 0, 64515, 121378, 0, 129128, - 67846, 0, 129767, 92466, 0, 0, 71338, 0, 8660, 0, 0, 0, 0, 0, 4483, 0, 0, - 0, 6080, 0, 0, 1746, 1315, 0, 70201, 0, 13140, 74508, 0, 0, 4480, 0, - 111113, 111112, 0, 67979, 0, 6360, 10897, 111106, 605, 68302, 110737, - 69875, 110735, 110736, 66681, 0, 0, 0, 0, 0, 0, 0, 10877, 118868, 64885, - 0, 0, 0, 0, 0, 0, 345, 0, 0, 64606, 9917, 0, 0, 92196, 0, 1776, 8422, - 43992, 0, 0, 0, 126543, 43328, 0, 0, 1295, 0, 42869, 0, 0, 0, 0, 128772, - 65123, 125210, 11293, 11288, 0, 0, 65666, 0, 92369, 65420, 0, 0, 4252, 0, - 0, 0, 706, 72800, 0, 0, 129931, 65419, 92177, 0, 8419, 65421, 0, 66702, - 0, 12670, 118608, 0, 0, 0, 72825, 65422, 83008, 0, 0, 0, 0, 0, 124153, - 9736, 4184, 65418, 0, 0, 74035, 0, 129955, 0, 0, 0, 0, 129447, 0, 7962, - 12211, 9837, 83505, 0, 0, 5719, 0, 129720, 119068, 73777, 1857, 0, 9927, - 0, 983959, 0, 10037, 0, 73695, 78322, 78319, 7818, 0, 0, 127769, 0, 0, 0, - 65077, 0, 78325, 78326, 78323, 43327, 43989, 0, 65828, 0, 0, 83499, 0, - 68390, 0, 110687, 78336, 78339, 9543, 78335, 78332, 78333, 0, 127964, 0, - 129552, 983914, 0, 69448, 0, 71429, 0, 0, 0, 11914, 69431, 0, 0, 0, 9949, - 0, 0, 119215, 0, 12073, 73519, 0, 0, 0, 101218, 2260, 0, 0, 0, 0, 0, 0, - 1939, 0, 0, 0, 69903, 0, 0, 0, 0, 6643, 92477, 128485, 0, 78330, 78331, - 78328, 78329, 0, 92551, 0, 0, 0, 0, 124124, 72417, 0, 0, 0, 0, 78341, - 78342, 120944, 78340, 129513, 127529, 92350, 3784, 78350, 0, 78348, - 78349, 78345, 43324, 78343, 78344, 2231, 0, 0, 0, 42467, 0, 0, 42894, - 78363, 13281, 78360, 78361, 78356, 78358, 78353, 64899, 0, 41149, 0, - 43162, 68096, 41150, 0, 10571, 67162, 67161, 67160, 67159, 6947, 41152, - 887, 9249, 6565, 64806, 74366, 0, 67158, 67157, 0, 10831, 67175, 67174, - 120232, 65827, 43325, 67178, 10168, 67176, 0, 0, 9190, 128497, 9666, - 41997, 0, 0, 0, 0, 0, 0, 129411, 0, 78508, 0, 78351, 78352, 0, 75063, - 72839, 983749, 0, 126604, 0, 0, 0, 983422, 0, 2270, 0, 129957, 0, 78365, - 0, 67189, 72818, 0, 0, 0, 0, 0, 0, 0, 72833, 101119, 78366, 78367, 0, 0, - 0, 0, 10137, 6121, 10995, 0, 71050, 8119, 0, 71052, 0, 0, 0, 0, 0, 0, 0, - 1394, 0, 0, 128960, 0, 67184, 2998, 67182, 67181, 67188, 67187, 67186, - 67185, 0, 101185, 0, 0, 67180, 42003, 0, 0, 67193, 67192, 67191, 67190, - 67197, 67196, 67195, 67194, 0, 72770, 43315, 71051, 0, 1593, 0, 125120, - 619, 4635, 0, 72875, 0, 128859, 118657, 0, 0, 0, 67199, 67198, 0, 42790, - 42006, 0, 0, 0, 128998, 10757, 9347, 127767, 0, 0, 74227, 78904, 0, - 74116, 128423, 121073, 120860, 0, 92427, 0, 0, 0, 0, 64590, 0, 4371, 0, - 0, 92478, 0, 0, 73977, 0, 0, 127847, 0, 120862, 0, 64550, 73745, 70451, - 0, 121013, 0, 0, 0, 129286, 0, 0, 0, 0, 9131, 118648, 125214, 983223, 0, - 0, 64260, 0, 12606, 0, 0, 0, 0, 562, 983614, 0, 129648, 66455, 127533, - 3219, 0, 0, 0, 1037, 0, 64491, 0, 78579, 78572, 78580, 4568, 549, 0, 0, - 0, 0, 0, 128095, 70851, 2205, 0, 0, 0, 0, 129716, 0, 10825, 8079, 118962, - 12285, 0, 0, 128855, 0, 13071, 0, 0, 41049, 42840, 43614, 129341, 74881, - 74596, 127191, 5212, 0, 66402, 119191, 0, 9747, 0, 0, 129778, 984008, - 41047, 1668, 0, 0, 0, 1187, 0, 74416, 0, 0, 0, 0, 3240, 128518, 9213, 0, - 0, 0, 127174, 69822, 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 11272, 0, 73914, - 65048, 1909, 42172, 0, 0, 10736, 11580, 72228, 7615, 0, 0, 4237, 66576, - 0, 65815, 68083, 0, 0, 0, 3489, 0, 0, 0, 0, 0, 0, 127146, 3796, 6800, 0, - 65582, 0, 129521, 0, 0, 68036, 0, 0, 64857, 121213, 126493, 0, 66308, 0, - 0, 64634, 127817, 0, 0, 0, 0, 3246, 0, 43972, 128643, 0, 0, 0, 0, 120751, - 0, 0, 0, 0, 1496, 42827, 0, 942, 2378, 119213, 0, 0, 0, 0, 9510, 1232, - 8139, 0, 0, 0, 11409, 0, 6382, 0, 66319, 121237, 0, 0, 0, 127887, 2374, - 0, 8475, 120844, 66313, 0, 0, 64879, 119298, 0, 0, 70869, 0, 0, 129025, - 0, 7705, 11942, 0, 0, 3309, 0, 119302, 0, 83345, 983866, 0, 0, 1280, - 6998, 128104, 0, 0, 0, 129945, 0, 0, 0, 0, 0, 0, 0, 74239, 983073, 0, 0, - 0, 6078, 121354, 0, 1475, 0, 9938, 6084, 0, 983995, 0, 118571, 0, 3256, - 0, 43973, 0, 0, 0, 8727, 0, 0, 0, 110831, 110832, 10562, 110830, 0, 0, 0, - 3248, 0, 0, 9015, 0, 0, 3635, 64337, 0, 0, 43852, 7195, 0, 2007, 64431, - 0, 0, 0, 0, 0, 0, 0, 65613, 77909, 0, 0, 0, 0, 119218, 7984, 11670, - 74434, 127770, 4176, 69248, 2034, 69442, 11154, 65891, 0, 0, 318, 2038, - 0, 0, 0, 3649, 13149, 42145, 42798, 3634, 0, 0, 128483, 122928, 124113, - 0, 11402, 120954, 94032, 74238, 0, 43313, 0, 0, 7938, 0, 1761, 0, 65379, - 68386, 128185, 1159, 71183, 0, 0, 0, 66687, 120851, 0, 41680, 0, 0, 0, - 1514, 11668, 67891, 9313, 0, 128490, 67877, 0, 41681, 0, 0, 12848, 69982, - 67873, 0, 74278, 0, 0, 12649, 0, 0, 1194, 3242, 9761, 9555, 8598, 0, - 120524, 0, 1551, 65447, 129414, 126211, 0, 0, 0, 67875, 0, 3495, 66648, - 125079, 0, 73024, 983232, 0, 126130, 10641, 0, 0, 0, 77845, 0, 0, 0, 0, - 0, 11131, 0, 0, 0, 0, 0, 42685, 72017, 193, 0, 0, 0, 42667, 0, 0, 92318, - 71958, 0, 1362, 9558, 0, 0, 0, 7351, 73789, 0, 0, 4426, 0, 0, 0, 0, 7276, - 42163, 5220, 0, 0, 67822, 0, 0, 0, 0, 41692, 0, 72283, 0, 0, 3223, 65492, - 0, 0, 4549, 983706, 0, 0, 101162, 10807, 0, 0, 0, 42182, 8688, 12866, 0, - 3294, 0, 0, 128101, 0, 64514, 0, 43329, 129989, 0, 0, 0, 119061, 0, - 43422, 0, 0, 128618, 0, 42729, 0, 3215, 120982, 68880, 917564, 0, 0, 0, - 65682, 0, 0, 65924, 0, 73506, 0, 1501, 0, 118807, 0, 0, 9607, 0, 65794, - 72243, 983046, 10989, 0, 74399, 0, 0, 7152, 0, 0, 129530, 7483, 125083, - 0, 8104, 70128, 7474, 0, 5189, 0, 0, 0, 8141, 0, 42537, 69612, 0, 0, 0, - 0, 0, 127307, 42934, 0, 0, 0, 0, 0, 0, 64517, 0, 0, 1650, 0, 0, 128502, - 7901, 3238, 0, 65556, 0, 0, 65158, 43416, 74959, 0, 7527, 0, 43319, 0, 0, - 45, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 13129, 0, 9084, 0, 8737, 0, 0, 0, - 66808, 9639, 7912, 2620, 129653, 3564, 0, 0, 0, 0, 75049, 0, 2853, 0, 0, - 0, 0, 0, 2850, 8084, 0, 0, 71446, 92284, 43122, 0, 0, 0, 126503, 72214, - 0, 74767, 0, 7331, 110646, 0, 8245, 0, 3158, 92396, 3983, 0, 923, 0, - 69397, 292, 0, 126548, 0, 3221, 1763, 0, 0, 0, 0, 7253, 194636, 68391, - 75002, 0, 3637, 12996, 0, 70461, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 120833, - 118939, 0, 7696, 78589, 0, 0, 0, 43316, 4177, 0, 9089, 0, 128805, 72116, - 64500, 68133, 0, 0, 1856, 100572, 0, 6379, 0, 118999, 0, 3208, 0, 0, 0, - 0, 0, 0, 129402, 0, 0, 0, 2033, 0, 0, 0, 55254, 7740, 0, 0, 0, 128197, 0, - 93988, 0, 67612, 0, 0, 41689, 129380, 0, 0, 6646, 0, 0, 0, 983964, 0, 0, - 4573, 0, 0, 0, 0, 0, 92961, 0, 118620, 41688, 0, 0, 0, 8314, 0, 0, 0, 0, - 0, 66721, 0, 0, 121033, 0, 128226, 0, 0, 0, 13164, 0, 66237, 983982, 0, - 0, 0, 3257, 0, 0, 1845, 0, 0, 0, 0, 128783, 0, 0, 0, 0, 3499, 8609, 0, - 7145, 0, 0, 0, 0, 74829, 984007, 983296, 0, 0, 0, 7591, 0, 0, 0, 73778, - 70132, 128167, 0, 0, 0, 0, 119261, 0, 0, 118561, 13083, 0, 0, 0, 0, - 66177, 983274, 5429, 0, 0, 68168, 66181, 0, 0, 983258, 0, 0, 5433, 67659, - 0, 42776, 1547, 66176, 92428, 0, 5425, 4977, 9999, 0, 5423, 64560, - 125094, 0, 0, 0, 74122, 0, 0, 0, 128003, 4418, 66199, 0, 92300, 0, 0, 0, - 11863, 124995, 0, 11908, 0, 9360, 125101, 983204, 0, 66187, 12837, - 983293, 0, 11112, 0, 92321, 43318, 0, 0, 0, 0, 126518, 120604, 0, 983291, - 0, 129595, 0, 983801, 0, 9958, 0, 125108, 0, 0, 0, 2433, 128602, 0, 3352, - 0, 0, 0, 0, 0, 0, 305, 567, 67662, 0, 69979, 65242, 0, 41695, 0, 0, 0, - 7837, 92873, 129002, 5337, 917622, 7325, 43312, 917619, 68742, 917617, - 74086, 68777, 917614, 917613, 10973, 917611, 1372, 128768, 917608, - 917607, 1254, 917605, 917604, 93967, 917602, 65228, 113753, 129367, - 67723, 8068, 0, 0, 983970, 0, 3245, 64393, 119069, 118681, 0, 0, 0, 0, 0, - 0, 983284, 0, 119563, 129935, 78865, 0, 126638, 0, 0, 43322, 0, 0, 0, 0, - 92698, 3226, 67695, 0, 0, 983958, 10200, 0, 128779, 101143, 0, 65610, 0, - 0, 0, 3585, 250, 101142, 43320, 0, 0, 0, 0, 1152, 129849, 1688, 0, 0, 0, - 0, 0, 121040, 128340, 0, 0, 0, 2107, 0, 129048, 0, 0, 0, 43868, 129832, - 129817, 0, 128239, 0, 0, 127777, 0, 6927, 42267, 42261, 11464, 3365, 0, - 0, 0, 0, 0, 41869, 0, 0, 0, 43326, 0, 11519, 0, 5530, 5210, 0, 983989, 0, - 5208, 0, 128842, 0, 2424, 7976, 0, 0, 3244, 5529, 0, 73894, 128852, 5432, - 0, 5527, 0, 78484, 0, 5528, 0, 0, 120281, 0, 0, 43545, 120282, 0, 0, - 73686, 42565, 0, 0, 3206, 120278, 73985, 0, 101149, 0, 0, 211, 3216, - 83407, 0, 120998, 3220, 68750, 0, 118586, 8951, 5214, 0, 8118, 0, 10768, - 8735, 0, 5852, 124952, 0, 0, 67513, 0, 0, 2623, 127859, 0, 0, 127388, - 4698, 66509, 0, 0, 4701, 0, 120289, 74225, 120284, 8267, 0, 1421, 66426, - 0, 0, 2625, 92724, 0, 74309, 0, 0, 0, 7850, 120296, 69639, 127032, 0, 0, - 43384, 12660, 110663, 0, 0, 110706, 110661, 0, 92380, 0, 0, 69649, 0, - 713, 41073, 0, 3990, 0, 0, 0, 5017, 128313, 120352, 0, 0, 1030, 0, - 983121, 9513, 0, 0, 0, 4668, 0, 120350, 0, 6339, 0, 0, 0, 64650, 0, 0, - 74766, 983869, 8908, 0, 0, 0, 0, 10752, 13003, 68769, 0, 41307, 8732, - 120336, 0, 41310, 0, 4696, 0, 983953, 0, 120334, 3641, 5419, 124119, 0, - 0, 0, 120344, 128129, 0, 7320, 65230, 11808, 0, 93970, 936, 13289, 0, - 69892, 65774, 0, 65243, 0, 19953, 0, 126469, 121375, 127256, 12913, - 70722, 68759, 0, 0, 70203, 0, 4113, 0, 2372, 1819, 0, 128053, 12152, 0, - 682, 7655, 120330, 129921, 0, 10593, 1703, 0, 0, 8033, 69953, 0, 9810, 0, - 0, 127949, 0, 119159, 10109, 0, 73898, 0, 71730, 126704, 0, 0, 917620, - 1965, 917621, 0, 0, 73887, 0, 0, 0, 6314, 0, 8501, 0, 0, 0, 41317, 0, - 5417, 983582, 0, 0, 9353, 68148, 41315, 0, 11161, 0, 41314, 194892, 0, - 126562, 119236, 634, 0, 0, 0, 69779, 4355, 12016, 0, 9654, 12856, 6924, - 7660, 0, 0, 0, 0, 0, 42692, 0, 74604, 0, 0, 0, 680, 6274, 0, 1181, 0, - 3174, 67248, 0, 0, 0, 0, 113776, 10650, 917603, 92295, 70672, 118965, 0, - 64644, 126981, 0, 0, 0, 0, 983961, 0, 65302, 40989, 68239, 68230, 68234, - 0, 0, 124989, 0, 40987, 4667, 0, 983963, 8828, 0, 0, 0, 4746, 0, 129840, - 2269, 4749, 0, 100598, 65192, 4744, 7345, 0, 242, 100595, 0, 8217, 0, - 68919, 0, 2245, 0, 0, 66790, 10850, 0, 0, 0, 983391, 0, 129853, 64680, 0, - 0, 120562, 0, 127324, 0, 100551, 128721, 0, 7316, 0, 983610, 100552, - 74157, 1646, 0, 0, 73995, 120857, 73500, 0, 7350, 0, 0, 0, 9099, 4107, - 3441, 0, 2975, 194701, 0, 983966, 55220, 10084, 73943, 120845, 118649, 0, - 0, 3399, 0, 0, 11909, 0, 0, 7687, 0, 6789, 0, 0, 72739, 71367, 0, 0, - 92589, 9151, 1137, 0, 749, 7505, 125076, 5385, 0, 69387, 0, 0, 41298, 0, - 69461, 0, 0, 0, 0, 0, 0, 128455, 0, 519, 0, 64547, 5766, 0, 0, 0, 8848, - 0, 41297, 0, 0, 0, 41300, 74468, 65160, 0, 129839, 127511, 0, 0, 6558, 0, - 0, 128686, 92775, 0, 71450, 41302, 127927, 0, 0, 128646, 68762, 11729, - 8719, 9060, 0, 128796, 0, 0, 118573, 129682, 0, 11734, 93011, 11730, - 73450, 9593, 5757, 2403, 0, 55275, 0, 11728, 65894, 0, 0, 0, 68741, 0, 0, - 0, 43489, 4282, 983864, 0, 83497, 70328, 128103, 70324, 0, 69490, 127509, - 0, 8456, 0, 0, 74783, 0, 78250, 0, 70320, 120722, 9792, 0, 70326, 0, 0, - 83500, 70322, 10019, 71701, 123617, 6568, 4365, 129399, 0, 3647, 0, - 41134, 128341, 0, 125043, 41135, 0, 0, 0, 129938, 0, 123616, 0, 41137, - 41139, 0, 6545, 0, 125139, 7597, 10528, 75054, 0, 3732, 73910, 0, 0, 0, - 7312, 983639, 9062, 93840, 11853, 0, 0, 128324, 41538, 0, 0, 118702, 0, - 194706, 41531, 1263, 3720, 0, 68028, 0, 41524, 64692, 119635, 0, 41534, - 0, 92193, 0, 41168, 0, 67398, 127347, 3524, 0, 8831, 127349, 127357, 0, - 127360, 127352, 129816, 0, 0, 0, 0, 0, 5845, 0, 0, 0, 71909, 8200, 0, - 68460, 0, 43283, 5551, 0, 0, 0, 6340, 983552, 100602, 0, 0, 0, 0, 0, - 5422, 0, 0, 0, 2471, 0, 0, 2749, 0, 73774, 10913, 72122, 0, 8666, 675, - 74093, 0, 194986, 0, 69262, 0, 0, 0, 10928, 0, 41153, 0, 0, 0, 3738, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42347, 12092, 9615, 7234, 74047, - 129782, 0, 0, 123639, 0, 0, 2934, 0, 0, 0, 0, 74507, 0, 74461, 0, 0, - 74290, 0, 64562, 129975, 64473, 0, 0, 73728, 0, 11212, 0, 12128, 6534, 0, - 0, 1901, 0, 0, 0, 0, 0, 127520, 0, 0, 0, 0, 69940, 65459, 68293, 92290, - 128808, 3770, 0, 0, 0, 64579, 128511, 0, 0, 983337, 983345, 0, 0, 0, - 5941, 0, 0, 65079, 0, 0, 0, 73961, 983339, 0, 0, 0, 0, 0, 0, 10638, 0, 0, - 0, 71486, 0, 0, 983354, 0, 43840, 129495, 0, 5233, 983351, 64792, 71233, - 0, 983329, 0, 73553, 9847, 0, 1685, 595, 0, 73971, 1292, 8940, 0, 11088, - 0, 10004, 0, 0, 6541, 0, 0, 0, 5603, 9014, 5606, 0, 538, 128705, 5602, - 8467, 74391, 6547, 0, 0, 0, 0, 8458, 129534, 8495, 0, 0, 917552, 10981, - 78314, 125057, 2465, 0, 0, 0, 9730, 9280, 0, 0, 74155, 72766, 113690, 0, - 504, 0, 120715, 0, 983606, 0, 0, 0, 123141, 125024, 0, 0, 732, 3737, 0, - 1548, 0, 0, 1832, 5604, 0, 41141, 0, 5607, 72854, 2176, 3745, 0, 0, - 128137, 0, 0, 3869, 11937, 5725, 0, 66566, 7416, 5728, 0, 0, 0, 11918, - 66567, 5724, 118829, 5727, 0, 0, 0, 5723, 118585, 128116, 71999, 0, 0, 0, - 42532, 0, 12303, 0, 11423, 0, 983116, 68303, 74074, 0, 128267, 6559, - 64557, 71348, 0, 66763, 43019, 0, 10238, 0, 0, 43377, 0, 71346, 124937, - 9783, 42704, 0, 71719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41144, 129465, 0, - 0, 0, 72793, 92176, 0, 70682, 0, 8820, 0, 0, 0, 11515, 526, 0, 0, 0, 0, - 0, 0, 8635, 0, 0, 8288, 11815, 0, 0, 0, 1543, 3713, 0, 0, 0, 68041, - 127816, 0, 0, 64357, 0, 42082, 0, 0, 8987, 42081, 0, 0, 0, 0, 0, 0, 6553, - 0, 0, 11253, 0, 0, 5475, 0, 0, 0, 119334, 12990, 1160, 42084, 0, 123152, - 0, 0, 360, 0, 0, 128274, 5863, 3137, 0, 983320, 0, 0, 10959, 3146, 0, - 127374, 0, 68341, 13076, 3135, 983303, 0, 0, 3142, 0, 94068, 10819, - 128479, 0, 74635, 12877, 119867, 73967, 0, 70808, 0, 0, 0, 0, 6163, - 129745, 113728, 0, 0, 0, 8603, 0, 0, 3306, 0, 43392, 0, 917565, 5751, 0, - 0, 0, 0, 0, 7403, 0, 118933, 0, 122628, 64783, 92658, 0, 0, 129592, 0, 0, - 65569, 7021, 0, 0, 119864, 0, 0, 6540, 6974, 0, 0, 0, 0, 0, 0, 0, 983655, - 0, 43585, 0, 6551, 983993, 0, 0, 0, 0, 0, 72216, 8977, 602, 120814, 0, 0, - 0, 72119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983624, 74812, 0, 0, 0, 9475, 0, - 65105, 0, 118556, 0, 43592, 7831, 66751, 0, 0, 73915, 0, 43593, 0, 43591, - 43061, 0, 0, 43589, 43584, 0, 13113, 0, 0, 43590, 8766, 9087, 0, 0, - 41574, 78337, 0, 42900, 6376, 0, 0, 0, 0, 9854, 0, 0, 0, 0, 0, 0, 0, - 2909, 110928, 0, 0, 6529, 110930, 75004, 3751, 0, 0, 0, 1798, 0, 0, 1354, - 0, 13152, 6557, 12430, 0, 94098, 0, 0, 0, 68123, 128097, 0, 0, 0, 71264, - 123559, 11082, 0, 65677, 8682, 42054, 92595, 42045, 9804, 0, 0, 3595, 0, - 0, 119498, 0, 42399, 0, 0, 0, 65541, 0, 7324, 0, 0, 0, 8797, 77895, 0, - 64888, 7167, 2356, 95, 110810, 0, 0, 42286, 0, 0, 69999, 0, 120877, 0, 0, - 42324, 129359, 0, 0, 43492, 0, 43406, 0, 0, 0, 0, 0, 43400, 0, 0, 71720, - 0, 66435, 0, 0, 3201, 514, 74502, 0, 43396, 0, 64493, 0, 43404, 11218, 0, - 0, 43398, 0, 0, 41341, 129485, 6564, 1463, 41342, 0, 5293, 0, 0, 3733, 0, - 0, 41344, 0, 0, 0, 0, 41346, 0, 69747, 0, 0, 0, 0, 0, 0, 0, 983764, 0, 0, - 0, 65272, 0, 0, 1270, 1132, 0, 0, 0, 66655, 0, 0, 74314, 64761, 0, - 110853, 8510, 0, 129600, 0, 0, 0, 0, 0, 0, 69692, 0, 0, 42383, 69690, 0, - 69700, 13141, 0, 92465, 0, 0, 0, 41566, 0, 0, 129334, 127171, 0, 0, 0, 0, - 0, 0, 0, 6308, 0, 0, 2611, 0, 66881, 0, 65063, 0, 0, 0, 0, 4484, 8747, - 110597, 128369, 0, 0, 0, 0, 0, 0, 12902, 0, 0, 7299, 0, 0, 12107, 7100, - 10905, 65010, 0, 125135, 66018, 9284, 0, 0, 0, 0, 0, 0, 0, 12010, 0, - 126093, 120949, 121032, 0, 0, 0, 0, 0, 0, 0, 0, 6618, 3562, 66365, 0, - 42234, 12648, 128039, 0, 0, 0, 41309, 9764, 41316, 0, 0, 13230, 41299, 0, - 0, 68365, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 128047, 0, 0, 42889, 0, 129322, - 41578, 0, 41577, 0, 68092, 0, 6533, 0, 41570, 0, 72414, 0, 41580, 74628, - 0, 12901, 0, 0, 0, 0, 71461, 41360, 0, 0, 4743, 0, 0, 0, 0, 68398, - 110781, 5890, 110779, 111103, 3739, 8695, 92514, 0, 3964, 8984, 111095, - 68288, 0, 0, 70000, 111090, 111089, 67504, 3956, 82952, 111093, 6563, - 111091, 41305, 0, 0, 12067, 41312, 0, 0, 0, 129708, 0, 8175, 0, 3600, 0, - 934, 0, 0, 173, 129844, 0, 110784, 110785, 1750, 110783, 41358, 68368, - 1807, 0, 92298, 0, 5889, 0, 0, 0, 67127, 129472, 0, 121395, 6982, 1721, - 0, 7891, 0, 42160, 67129, 4512, 983790, 69460, 0, 0, 0, 122636, 0, - 120716, 0, 0, 0, 0, 0, 119140, 3975, 72253, 74087, 0, 12672, 0, 129821, - 0, 0, 129836, 0, 121100, 0, 0, 41095, 3962, 68242, 2932, 41101, 3954, - 6457, 4513, 0, 0, 0, 0, 1468, 0, 0, 55237, 128230, 0, 127244, 55238, - 41080, 0, 0, 4320, 74104, 0, 0, 83315, 0, 77918, 118563, 128384, 8256, 0, - 72413, 0, 8879, 0, 0, 8770, 0, 0, 92214, 0, 0, 128786, 4283, 129689, 0, - 68361, 0, 74826, 0, 0, 0, 0, 127954, 65106, 42761, 121516, 4581, 8411, 0, - 0, 72259, 0, 93037, 0, 0, 0, 92452, 4392, 0, 10786, 69661, 0, 8184, 0, 0, - 7396, 0, 0, 69788, 0, 43512, 7965, 111039, 111038, 111037, 111036, 41350, - 0, 0, 0, 2294, 64501, 68034, 0, 68405, 111034, 0, 0, 111030, 111029, - 71105, 111027, 0, 111033, 92200, 111031, 0, 6764, 0, 0, 111026, 111025, - 72454, 65203, 128010, 0, 0, 0, 3210, 0, 129978, 0, 0, 82958, 127970, - 82957, 0, 68875, 10043, 71979, 1186, 41571, 0, 5209, 9464, 82960, 66657, - 5207, 65062, 5213, 0, 0, 41348, 41568, 128803, 3253, 111045, 111044, - 74067, 111042, 111049, 5596, 111047, 111046, 0, 64887, 0, 5217, 111041, - 72252, 0, 0, 0, 0, 2635, 92760, 0, 0, 0, 92742, 0, 113672, 0, 0, 0, 2258, - 67081, 0, 67083, 0, 0, 0, 5784, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 4254, 0, - 111054, 5600, 111052, 111051, 10447, 5598, 1207, 111055, 0, 3501, 42582, - 0, 111050, 0, 1124, 5597, 983501, 78908, 9321, 129464, 75040, 983498, 0, - 1719, 68356, 68354, 9671, 1125, 2721, 0, 129876, 983504, 7631, 5488, - 111082, 0, 0, 5491, 111086, 8937, 0, 3236, 74187, 5490, 0, 5489, 8522, - 68358, 111069, 6300, 111067, 111066, 0, 0, 111071, 111070, 0, 9875, 7593, - 111065, 0, 0, 43182, 0, 68379, 3311, 111058, 111057, 3746, 11016, 65752, - 111061, 0, 43423, 68775, 0, 111056, 72225, 0, 0, 127120, 0, 2232, 0, 0, - 0, 0, 0, 126555, 0, 0, 8656, 0, 128358, 0, 0, 983490, 983491, 917563, - 983489, 983486, 983487, 0, 0, 0, 129669, 0, 111183, 128043, 983495, 1036, - 983493, 111075, 1723, 111073, 111072, 111079, 41579, 111077, 111076, - 10705, 0, 983485, 74486, 71693, 740, 983481, 983482, 129645, 0, 0, 74846, - 92255, 0, 0, 0, 0, 0, 10438, 74487, 73798, 13285, 0, 0, 0, 5690, 0, - 93992, 0, 0, 13095, 0, 127857, 121419, 7321, 121203, 13254, 70176, 75070, - 0, 0, 0, 0, 127845, 3247, 317, 0, 0, 0, 0, 917543, 0, 10173, 0, 0, 0, 0, - 0, 5223, 0, 0, 119564, 5226, 0, 94044, 5880, 94065, 7758, 0, 0, 5224, - 5487, 94041, 5692, 41725, 983467, 0, 5695, 41711, 0, 43171, 0, 94049, - 5691, 983472, 866, 1488, 983471, 983457, 65665, 94036, 983456, 74797, 0, - 0, 11039, 983465, 11145, 71211, 983464, 983461, 983462, 983459, 983460, - 42492, 43402, 125208, 3302, 0, 72842, 43153, 0, 0, 120885, 121300, 0, - 7856, 8690, 0, 73076, 110880, 0, 0, 73091, 0, 69925, 120635, 65153, 0, 0, - 0, 0, 0, 0, 4540, 0, 0, 0, 0, 11844, 121209, 8863, 0, 75061, 71978, 6389, - 0, 42371, 83205, 8790, 120911, 0, 111125, 71168, 8869, 0, 0, 42060, 0, - 9648, 111123, 71170, 10270, 10286, 10318, 10382, 43529, 0, 0, 0, 0, 0, - 70110, 43835, 119520, 70111, 119360, 118815, 127084, 127083, 8767, 0, - 128437, 41281, 0, 5201, 0, 6215, 67072, 6214, 13101, 0, 0, 65268, 67073, - 0, 0, 127976, 72995, 127073, 10511, 42075, 0, 73475, 129509, 0, 67115, - 127069, 111293, 127068, 0, 127067, 0, 74845, 0, 42071, 43156, 0, 0, 0, 0, - 7954, 0, 0, 0, 8485, 4671, 0, 69513, 4740, 0, 0, 42618, 78294, 3064, - 6212, 0, 0, 0, 9554, 0, 83044, 0, 126598, 0, 78291, 6159, 6213, 12885, 0, - 129086, 64720, 0, 983926, 0, 0, 0, 11430, 0, 7518, 9317, 0, 3729, 10406, - 0, 119259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73825, 0, 0, 129599, 8786, 10390, - 0, 0, 917601, 93034, 0, 7924, 0, 43307, 0, 0, 0, 0, 0, 0, 118843, 9623, - 435, 0, 0, 12893, 8093, 9079, 0, 0, 0, 0, 0, 64430, 0, 10294, 10326, 0, - 0, 0, 0, 0, 0, 3623, 125188, 83378, 0, 43197, 0, 111149, 0, 78296, 0, 0, - 0, 7914, 0, 69749, 0, 2624, 0, 0, 0, 120859, 67110, 11058, 0, 67107, 0, - 0, 0, 0, 120793, 0, 0, 6717, 10619, 0, 0, 0, 11832, 128664, 0, 0, 0, - 70202, 0, 0, 0, 3232, 73824, 74581, 0, 0, 0, 41889, 0, 0, 1161, 41895, - 74103, 9701, 0, 0, 129385, 73819, 120588, 5012, 0, 41362, 0, 68507, 0, 0, - 0, 0, 0, 41364, 0, 0, 41352, 41361, 0, 41366, 0, 70129, 129065, 917, 0, - 119934, 119923, 92421, 119912, 0, 119924, 64841, 118582, 71482, 0, 0, 0, - 125136, 128583, 0, 7022, 0, 4739, 0, 5802, 9816, 8615, 0, 0, 491, 65837, - 0, 0, 128644, 0, 8426, 11092, 9891, 0, 0, 0, 41881, 118823, 3736, 7394, - 42648, 0, 68448, 9095, 7741, 12684, 41885, 0, 0, 0, 0, 5815, 0, 0, 0, - 127392, 0, 0, 41878, 0, 0, 0, 0, 0, 0, 0, 0, 119503, 0, 120804, 0, 0, - 2267, 0, 78289, 78359, 78288, 0, 0, 78318, 65920, 0, 194819, 7057, 9408, - 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, - 9421, 5897, 9423, 917933, 127107, 0, 127108, 917937, 127963, 8955, 9399, - 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 0, 128626, 42669, 73832, - 78261, 67683, 2631, 119308, 78259, 0, 78260, 3996, 0, 119307, 0, 0, 0, 0, - 0, 0, 64825, 917916, 917913, 917914, 917919, 5899, 917917, 129990, 12085, - 0, 574, 69734, 77825, 73828, 9473, 77824, 118918, 73900, 41735, 42211, 0, - 4190, 77834, 77835, 77830, 77833, 3616, 77828, 77837, 77838, 7708, 77836, - 2228, 113765, 0, 0, 4191, 42968, 77844, 73800, 77842, 77843, 77839, - 77840, 0, 78311, 83375, 0, 0, 10415, 74102, 0, 5896, 0, 10351, 67151, 0, - 73829, 0, 127159, 0, 73998, 41355, 42883, 70736, 71212, 8021, 0, 119150, - 983732, 41357, 8011, 42885, 42887, 41354, 0, 0, 10026, 5472, 120554, - 1191, 101217, 5470, 128784, 5476, 101216, 0, 0, 0, 42874, 78281, 42876, - 6304, 78283, 0, 2675, 120690, 0, 0, 128954, 0, 0, 5478, 5904, 0, 0, 0, - 7291, 77848, 43761, 13067, 0, 0, 119271, 120360, 69731, 77856, 77857, - 77854, 77855, 77852, 77853, 77850, 10750, 43714, 77858, 0, 0, 0, 12887, - 120364, 127745, 77866, 77867, 77864, 77865, 9929, 5199, 77859, 1120, 0, - 0, 0, 9486, 7554, 0, 77868, 72832, 0, 0, 5894, 70069, 0, 0, 92511, 70358, - 1323, 13162, 120937, 0, 0, 0, 77881, 66022, 0, 72857, 0, 0, 0, 0, 0, - 1142, 0, 8271, 0, 0, 126645, 12903, 43622, 4002, 0, 10442, 10676, 120368, - 0, 120369, 0, 0, 0, 0, 66642, 1277, 0, 7871, 0, 0, 78853, 0, 119015, 0, - 0, 11784, 0, 78012, 4700, 0, 78858, 0, 78855, 0, 0, 92400, 77879, 19932, - 77876, 77877, 74804, 77874, 77869, 77871, 0, 71487, 43118, 0, 0, 6774, - 6773, 0, 194684, 10346, 10410, 78860, 118974, 0, 101197, 6108, 0, 110612, - 0, 0, 5901, 121309, 74166, 124973, 0, 0, 0, 69407, 0, 70357, 0, 0, 74217, - 0, 64698, 4192, 9289, 0, 0, 128847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11603, 0, - 0, 101166, 82976, 0, 0, 67812, 0, 101163, 0, 42572, 0, 128300, 119146, - 1963, 11622, 0, 43237, 82981, 7550, 67100, 5903, 82984, 78009, 129750, - 9662, 0, 128391, 0, 0, 0, 0, 11013, 0, 0, 0, 128433, 67090, 0, 0, 0, 0, - 0, 11568, 983704, 43367, 0, 0, 7852, 119496, 0, 0, 0, 0, 194676, 0, - 194675, 0, 0, 416, 129668, 0, 73834, 0, 68921, 10984, 0, 0, 101175, - 129838, 101182, 0, 127013, 92423, 0, 983261, 121199, 0, 0, 12540, 0, - 983220, 0, 0, 11445, 101168, 2112, 0, 0, 0, 1021, 0, 9507, 10210, 78005, - 8023, 93963, 78006, 78001, 43181, 78003, 9532, 119094, 0, 0, 0, 0, 0, - 1885, 43268, 129802, 129798, 120542, 121153, 392, 7894, 4391, 129810, - 8221, 119597, 77999, 77998, 0, 0, 0, 92967, 0, 3558, 0, 3913, 70429, - 121376, 0, 0, 1265, 0, 6309, 0, 12969, 0, 101151, 0, 101146, 0, 101139, - 0, 41864, 0, 0, 74294, 0, 167, 0, 917584, 0, 93983, 72354, 68477, 0, 0, - 917594, 0, 2493, 129827, 0, 129804, 0, 917570, 917593, 0, 0, 406, 917592, - 0, 0, 0, 0, 0, 0, 0, 127161, 0, 128597, 0, 0, 0, 3421, 10561, 0, 8365, - 917581, 0, 127569, 120787, 128669, 0, 0, 0, 0, 7834, 0, 0, 101154, 10298, - 6624, 4908, 0, 1639, 120842, 0, 0, 6327, 6724, 0, 0, 0, 69910, 4817, 0, - 0, 0, 68059, 0, 11022, 0, 0, 0, 118888, 0, 0, 7548, 64794, 0, 12291, - 983166, 0, 0, 0, 0, 0, 0, 1134, 1838, 0, 2057, 0, 0, 0, 0, 0, 0, 5206, 0, - 0, 42523, 0, 0, 0, 0, 65550, 8570, 4816, 0, 127926, 0, 4821, 0, 0, 0, - 4818, 125257, 119974, 119977, 0, 0, 119970, 119973, 0, 119983, 119982, - 67470, 119984, 119979, 119978, 0, 119980, 119670, 129297, 0, 11284, - 119987, 70097, 65155, 119988, 0, 9363, 0, 0, 0, 5900, 93990, 7889, 2722, - 128770, 0, 0, 0, 0, 2282, 0, 0, 0, 68093, 0, 0, 0, 0, 0, 70150, 118628, - 0, 0, 0, 129651, 70146, 983079, 119967, 71330, 70148, 0, 0, 94006, 70144, - 119964, 110677, 110678, 110675, 110676, 0, 110674, 4226, 0, 123165, 5732, - 71327, 0, 0, 65119, 0, 0, 92971, 64770, 0, 0, 6093, 0, 0, 1395, 0, 0, 0, - 121179, 786, 0, 43174, 64340, 0, 125269, 0, 983662, 125138, 10132, 0, 0, - 0, 0, 0, 93956, 0, 68444, 0, 92437, 123143, 0, 0, 92656, 0, 0, 0, 1399, - 121463, 0, 121465, 121464, 120808, 241, 121469, 4907, 0, 0, 0, 0, 0, 0, - 0, 0, 127904, 0, 0, 42780, 0, 0, 0, 4217, 0, 0, 0, 0, 72158, 0, 0, 43099, - 3965, 0, 0, 0, 13300, 0, 0, 43057, 0, 0, 0, 0, 0, 65372, 0, 6410, 126073, - 125252, 70468, 0, 0, 0, 119558, 0, 0, 0, 0, 0, 0, 43188, 2626, 7762, 0, - 0, 0, 127183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67726, 0, 126993, 1542, 0, 0, - 92550, 0, 0, 74311, 0, 0, 10181, 2150, 0, 0, 0, 0, 124921, 68053, 6029, - 72852, 0, 0, 0, 0, 8993, 0, 0, 0, 93968, 606, 118664, 0, 0, 0, 4311, 0, - 6027, 126615, 4322, 0, 65207, 0, 2184, 983920, 0, 0, 2735, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 70806, 0, 73547, 0, 92783, 92844, 0, 65817, 55288, - 127934, 66564, 8530, 0, 7709, 0, 121202, 66560, 128528, 917595, 12876, - 66561, 0, 121430, 983957, 7789, 5855, 809, 0, 0, 72853, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 64386, 0, 74909, 64845, 120607, 66416, 83360, 6532, 0, 0, - 0, 0, 128224, 0, 0, 0, 0, 43091, 92287, 0, 0, 129312, 0, 0, 0, 11361, 0, - 0, 8153, 128105, 0, 10741, 0, 0, 0, 0, 0, 64706, 917922, 0, 69505, 78870, - 9466, 78866, 9824, 0, 0, 0, 120977, 915, 0, 0, 43865, 0, 0, 0, 67131, - 70096, 67137, 0, 129614, 73648, 6730, 78862, 68161, 0, 78861, 126542, 0, - 0, 94010, 118655, 0, 0, 66043, 0, 0, 43107, 0, 0, 92343, 0, 73879, 0, 0, - 0, 6103, 0, 0, 92470, 0, 12889, 0, 127137, 0, 0, 0, 0, 0, 0, 119262, - 83028, 0, 0, 0, 69279, 0, 0, 0, 13118, 7700, 917537, 9690, 0, 0, 68080, - 512, 0, 72792, 0, 0, 77892, 632, 77890, 77891, 42529, 0, 0, 0, 0, 0, 0, - 0, 128273, 0, 0, 7379, 64581, 5386, 0, 0, 10633, 72316, 64488, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 124956, 71307, 0, 0, 0, 0, 0, 92370, 0, 0, 0, 0, 0, - 71314, 1801, 0, 0, 120867, 0, 0, 77888, 2085, 702, 77887, 77884, 77885, - 13074, 77883, 66299, 0, 0, 12106, 78905, 0, 1755, 0, 77897, 77898, 1163, - 3102, 77893, 77894, 0, 0, 0, 0, 69227, 0, 77901, 77902, 77899, 77900, - 65171, 0, 0, 0, 70157, 0, 0, 0, 64846, 2908, 0, 11177, 64902, 64950, 0, - 128740, 66906, 124959, 70499, 0, 0, 0, 64352, 0, 125031, 1007, 0, 9199, - 0, 127371, 118992, 41890, 0, 2730, 119072, 0, 5428, 0, 73771, 129993, 0, - 0, 0, 71458, 0, 0, 0, 68089, 0, 44012, 0, 71456, 0, 9158, 66878, 69905, - 92440, 0, 101272, 0, 484, 0, 0, 0, 194742, 0, 0, 0, 0, 572, 7041, 2736, - 0, 0, 93962, 0, 68628, 0, 128727, 5438, 5222, 5381, 43114, 0, 5193, 5125, - 5456, 5509, 0, 120664, 113700, 0, 0, 0, 3430, 0, 42905, 0, 74929, 6050, - 0, 0, 129197, 0, 0, 10908, 0, 0, 127809, 64617, 0, 0, 3957, 0, 0, 0, 674, - 0, 0, 2946, 5354, 5251, 5328, 5307, 3759, 72318, 8364, 5123, 0, 5281, - 5469, 5121, 0, 0, 0, 5130, 0, 129608, 0, 0, 0, 1221, 2733, 0, 0, 0, - 72321, 127245, 0, 0, 0, 101277, 0, 5939, 0, 0, 0, 71867, 68400, 128216, - 10321, 10289, 0, 10385, 123164, 0, 0, 0, 0, 118943, 0, 11411, 0, 5938, 0, - 120865, 0, 0, 10401, 10337, 0, 0, 0, 0, 0, 0, 0, 71942, 0, 0, 12165, 0, - 0, 9885, 0, 8077, 0, 127908, 0, 0, 0, 0, 129138, 4220, 10725, 10433, 0, - 68395, 4987, 64519, 0, 0, 0, 123626, 120356, 0, 11733, 0, 120792, 0, - 127233, 0, 0, 0, 92345, 68254, 983661, 77991, 0, 2724, 0, 0, 12313, - 110619, 515, 119947, 119944, 119945, 119942, 119943, 119940, 119941, - 119938, 8606, 4046, 4589, 4521, 0, 9141, 0, 0, 2741, 0, 0, 1370, 0, 0, 0, - 0, 0, 0, 66880, 0, 66003, 0, 64440, 0, 129943, 69458, 0, 11593, 68669, - 68666, 68660, 0, 0, 2744, 72285, 68638, 0, 814, 0, 119962, 119963, - 119960, 119961, 101106, 43029, 119956, 11623, 119954, 11955, 119952, - 119953, 41986, 119951, 0, 120497, 4847, 110975, 0, 0, 0, 0, 1581, 64920, - 93830, 12954, 963, 110973, 110972, 110971, 110969, 5278, 110967, 68621, - 92222, 983454, 68625, 983452, 68617, 110960, 0, 101459, 101487, 110964, - 110963, 110962, 0, 0, 101464, 101483, 101463, 983443, 983440, 92648, - 127379, 0, 65137, 6483, 65392, 0, 4213, 129649, 41303, 0, 0, 0, 41306, - 129751, 2698, 0, 0, 0, 68396, 0, 41304, 824, 0, 78011, 72315, 78894, - 74827, 78892, 64804, 9820, 119820, 110985, 110976, 0, 6739, 0, 5481, - 3490, 110978, 110977, 71706, 69947, 67702, 9124, 12688, 119833, 101496, - 0, 101495, 119821, 119824, 67480, 42575, 101474, 101478, 119827, 101481, - 101476, 71087, 68658, 119946, 8025, 68630, 101490, 68675, 92445, 71097, - 69613, 0, 0, 0, 0, 983435, 2745, 11797, 110990, 983431, 9202, 983429, - 983430, 0, 0, 0, 10525, 5436, 74584, 110987, 110986, 121506, 43080, - 121508, 121507, 983420, 6246, 119958, 10921, 9723, 6777, 6776, 6775, 0, - 0, 70287, 92384, 0, 8669, 0, 0, 65093, 0, 78881, 2716, 0, 0, 11252, - 101475, 68369, 0, 11060, 12985, 2711, 78872, 78027, 78026, 7992, 0, 0, - 42938, 78033, 78032, 78877, 70724, 78029, 78028, 78031, 78030, 64535, - 110998, 10130, 110996, 0, 0, 111001, 111000, 127914, 983417, 78014, 5713, - 110995, 7570, 110993, 110992, 0, 11190, 129700, 9026, 0, 74864, 7547, - 78891, 0, 10008, 10222, 0, 129543, 9744, 0, 68809, 983413, 119656, - 983411, 94070, 983409, 983410, 983407, 9045, 78888, 4225, 78886, 78887, - 68757, 78885, 78882, 78883, 983402, 983403, 8405, 983401, 10423, 10359, - 983396, 983397, 0, 129149, 4215, 9789, 0, 4321, 12309, 983405, 41313, 0, - 5368, 66886, 0, 0, 5366, 0, 5372, 101482, 67512, 0, 7720, 7390, 2696, 0, - 0, 8268, 0, 1790, 0, 0, 118977, 0, 0, 0, 5376, 1835, 72313, 78704, - 128089, 0, 0, 68655, 1180, 0, 0, 0, 0, 119274, 0, 0, 9122, 118584, 11928, - 0, 65283, 0, 101449, 5971, 101448, 43500, 1268, 65097, 983222, 0, 101445, - 0, 1427, 128440, 0, 5970, 3431, 72299, 101439, 101435, 983389, 983390, - 983387, 2738, 125066, 10455, 0, 74026, 0, 4222, 6240, 0, 119013, 983394, - 68377, 6248, 983378, 67815, 983376, 917907, 92582, 0, 101453, 125215, 0, - 2728, 65549, 64563, 101428, 101425, 101429, 128145, 0, 10713, 7166, - 119559, 2622, 101450, 0, 0, 0, 8954, 0, 94008, 2632, 42617, 10108, 1011, - 42852, 12080, 2709, 0, 5716, 0, 0, 0, 0, 127100, 69378, 0, 9515, 127098, - 66465, 6451, 0, 127097, 8918, 983556, 0, 0, 19950, 0, 0, 0, 44003, 0, - 64735, 0, 0, 0, 0, 983500, 74022, 0, 128795, 68643, 67410, 0, 5721, 0, 0, - 0, 121074, 11267, 983369, 66464, 5720, 983368, 0, 4219, 5718, 8696, 5717, - 122651, 983375, 983897, 983373, 541, 983371, 983372, 119948, 119089, - 68389, 983357, 119949, 56, 4216, 10577, 0, 0, 77849, 69620, 983362, - 983363, 66899, 983361, 0, 0, 67628, 0, 0, 7086, 0, 67998, 67621, 0, 2734, - 69616, 0, 67627, 118937, 0, 67625, 0, 0, 0, 42593, 0, 128217, 0, 0, - 119939, 0, 68180, 0, 0, 71104, 7442, 43665, 359, 41253, 68392, 6239, - 120599, 41256, 0, 67740, 111023, 111022, 111021, 9346, 69660, 41254, 0, - 43291, 78002, 0, 67491, 124993, 93841, 0, 0, 0, 4368, 983505, 0, 68137, - 0, 0, 41024, 0, 0, 121359, 121420, 0, 0, 0, 4223, 0, 8574, 83502, 0, 0, - 0, 118576, 0, 92718, 983636, 70432, 128323, 68382, 0, 0, 0, 0, 0, 4144, - 0, 83193, 6245, 0, 2732, 92644, 0, 0, 64558, 83501, 0, 0, 0, 128005, 0, - 0, 129652, 983149, 3097, 0, 0, 77996, 0, 0, 10863, 111020, 111019, - 111018, 0, 111015, 111014, 111013, 111012, 118964, 0, 10216, 64293, 0, 0, - 69393, 128331, 12325, 111010, 8717, 111008, 101413, 0, 101380, 0, 8700, - 0, 101382, 68363, 10426, 0, 71091, 10362, 0, 1715, 101378, 0, 64918, - 101409, 43278, 42635, 0, 0, 65275, 0, 0, 101319, 0, 69746, 1607, 466, - 118949, 0, 0, 127918, 6243, 983901, 1350, 74195, 64420, 1993, 5362, - 10666, 2708, 92471, 0, 13143, 234, 3199, 0, 41268, 6334, 2173, 0, 0, - 73750, 0, 73762, 10458, 0, 8576, 127136, 0, 2704, 64953, 0, 64832, 8322, - 0, 3132, 0, 2694, 0, 0, 2439, 65104, 69804, 0, 303, 74625, 92622, 0, - 2437, 0, 9817, 4844, 0, 0, 0, 0, 0, 121120, 43292, 0, 2441, 0, 0, 0, 0, - 0, 2451, 2714, 0, 0, 43379, 127984, 74541, 753, 5849, 0, 43089, 0, 0, - 119534, 72380, 0, 0, 0, 2726, 3107, 0, 0, 64937, 0, 78841, 1408, 0, 4607, - 101299, 181, 0, 67728, 9539, 0, 0, 65201, 121121, 92973, 64185, 4142, - 64183, 0, 0, 0, 9706, 64178, 64177, 64176, 0, 64182, 64181, 64180, 64179, - 11401, 125124, 0, 1822, 0, 128581, 68055, 3865, 122918, 0, 10500, 129602, - 119024, 0, 110732, 9830, 0, 0, 0, 65131, 0, 0, 0, 0, 74608, 7038, 0, - 9599, 8748, 0, 0, 9557, 0, 0, 0, 11494, 0, 0, 10865, 0, 43279, 64186, - 68521, 0, 64191, 64190, 8898, 64188, 129153, 41030, 78836, 0, 0, 78820, - 126100, 0, 78805, 78806, 78801, 78802, 6745, 78800, 0, 0, 0, 110866, 0, - 0, 73679, 67838, 41039, 78809, 128162, 0, 129893, 0, 110869, 127045, - 110867, 110868, 127039, 4400, 0, 64207, 10275, 8925, 10371, 10307, 64202, - 4248, 0, 72802, 4541, 6299, 64204, 64203, 64201, 64200, 64199, 64198, - 126471, 0, 0, 0, 64193, 64192, 0, 9943, 64197, 64196, 64195, 64194, - 13282, 42652, 64174, 64173, 83495, 846, 72337, 9965, 74495, 72330, 83493, - 83494, 2543, 12163, 64170, 83490, 64167, 64166, 64165, 64164, 72333, 0, - 64169, 64168, 64949, 0, 10251, 10247, 64163, 64162, 2295, 43299, 43301, - 129363, 0, 70791, 0, 0, 550, 9910, 0, 0, 66579, 0, 0, 0, 9504, 0, 0, - 10373, 0, 0, 10261, 10253, 7821, 10277, 0, 74823, 1552, 0, 0, 129389, 0, - 121435, 19910, 0, 0, 118849, 121150, 0, 43985, 68051, 0, 69890, 121329, - 78355, 983776, 0, 66405, 2431, 3744, 66852, 1809, 0, 0, 0, 73759, 1264, - 0, 78676, 11697, 121278, 9785, 64716, 0, 0, 0, 0, 121307, 0, 0, 42609, - 128388, 0, 66912, 127016, 0, 983904, 74229, 118590, 6487, 93798, 70743, - 0, 0, 0, 83484, 83485, 83486, 83487, 83480, 8355, 7854, 83483, 954, - 64927, 0, 41045, 0, 41438, 0, 0, 10711, 0, 0, 0, 0, 64774, 13309, 10947, - 66727, 101426, 0, 0, 66795, 0, 0, 0, 0, 0, 0, 0, 120634, 69228, 0, 0, 0, - 0, 101430, 0, 3060, 83478, 9986, 0, 83473, 83474, 11698, 77880, 83469, - 9916, 11701, 83472, 42586, 0, 8320, 0, 119095, 0, 0, 1477, 43289, 0, - 74358, 10884, 69446, 9908, 0, 0, 0, 3414, 74304, 0, 0, 0, 0, 2110, 0, - 68306, 0, 74532, 0, 129865, 0, 0, 7164, 0, 0, 0, 11950, 5392, 42248, - 65129, 68656, 5397, 129579, 0, 68136, 0, 0, 5395, 72870, 5393, 354, - 68615, 0, 0, 0, 0, 0, 126236, 0, 0, 626, 0, 5895, 0, 0, 5780, 0, 66407, - 10220, 0, 71121, 43297, 0, 0, 11468, 64436, 0, 0, 0, 73818, 3918, 0, - 3797, 72786, 122961, 0, 4140, 0, 71254, 0, 9030, 813, 0, 68131, 4146, - 119957, 5360, 0, 129498, 0, 0, 6249, 0, 0, 0, 0, 0, 73092, 0, 4911, 988, - 0, 73125, 0, 42948, 0, 0, 0, 0, 74972, 0, 0, 0, 9825, 0, 0, 12803, - 126977, 11032, 67654, 6244, 0, 0, 68662, 0, 129351, 0, 72131, 4169, 0, 0, - 0, 129986, 121410, 120657, 0, 0, 68657, 128943, 78496, 0, 0, 5898, 74540, - 0, 41856, 93056, 194926, 118538, 127373, 83424, 83425, 83426, 73736, - 83420, 68870, 6448, 6835, 0, 4831, 83418, 83419, 67731, 0, 0, 0, 0, 0, 0, - 0, 78499, 0, 0, 0, 43288, 122931, 0, 0, 0, 0, 43418, 0, 0, 0, 7876, - 68132, 917872, 0, 917870, 43378, 0, 0, 120890, 5892, 43605, 0, 0, 0, - 129058, 0, 0, 6251, 83409, 83410, 83411, 83412, 126512, 0, 71092, 83408, - 10114, 0, 0, 5387, 0, 0, 0, 0, 65553, 78346, 1747, 917849, 65109, 69240, - 917852, 126509, 0, 0, 0, 0, 125065, 0, 9850, 0, 367, 1472, 917859, 6687, - 0, 0, 5905, 12339, 8919, 73953, 65680, 0, 2204, 78664, 0, 9134, 118589, - 78666, 43011, 0, 126626, 0, 0, 0, 43013, 10614, 0, 0, 83413, 66646, - 83415, 83416, 0, 73881, 43012, 121127, 83293, 54, 43009, 73885, 0, 6211, - 0, 0, 83295, 68119, 43008, 10758, 0, 0, 0, 0, 0, 70018, 0, 0, 0, 0, - 12765, 0, 0, 0, 0, 126580, 0, 0, 43657, 0, 0, 0, 983737, 0, 83405, - 917843, 0, 0, 83401, 83402, 83403, 83404, 83397, 11363, 12057, 83400, - 1567, 0, 0, 83396, 0, 8957, 4139, 0, 0, 129336, 0, 0, 12740, 0, 92195, - 12761, 127793, 12759, 0, 72304, 67169, 83467, 44002, 0, 83462, 83463, - 83464, 12755, 12762, 41022, 67690, 64217, 476, 0, 983734, 0, 64212, - 41020, 1382, 64209, 64216, 64215, 64214, 64213, 0, 0, 0, 67584, 8720, - 3908, 0, 0, 0, 0, 101529, 129576, 0, 0, 3849, 92324, 94026, 9778, 917906, - 5891, 917912, 55, 917910, 917911, 0, 0, 7935, 67586, 0, 1114, 92599, - 67585, 78675, 0, 83447, 83449, 0, 0, 0, 64717, 0, 0, 0, 66884, 6292, - 65303, 0, 6452, 917886, 917887, 66249, 917885, 917890, 917891, 917888, - 719, 101446, 0, 917892, 0, 0, 0, 94083, 10868, 121333, 2349, 5902, - 917896, 6335, 101350, 917899, 917900, 0, 64369, 0, 0, 0, 69245, 0, - 126564, 0, 0, 128565, 0, 0, 0, 0, 0, 6454, 1229, 83457, 83458, 83450, - 83451, 83452, 65100, 120508, 8224, 917873, 917874, 917879, 917880, - 917877, 917878, 128929, 0, 917881, 917882, 5365, 67836, 8901, 0, 0, - 129951, 0, 69257, 5925, 83436, 64330, 128400, 83431, 83432, 83433, 83434, - 83427, 83428, 83429, 83430, 64928, 10543, 0, 0, 83446, 414, 0, 0, 83442, - 6456, 71490, 83445, 11905, 83439, 66284, 83441, 0, 68337, 0, 83437, - 43832, 983140, 9751, 0, 128085, 11770, 0, 0, 69600, 65061, 0, 0, 0, 0, 0, - 0, 121087, 0, 0, 69924, 0, 0, 0, 69913, 0, 121387, 101513, 101504, - 101512, 42038, 387, 0, 12737, 0, 0, 43368, 0, 0, 0, 0, 129713, 129449, - 121295, 0, 69400, 127309, 0, 375, 0, 0, 0, 983905, 0, 0, 119202, 119203, - 124117, 43120, 0, 0, 119196, 119197, 0, 4529, 119200, 119201, 119198, - 119199, 0, 0, 69698, 13150, 64492, 0, 0, 0, 0, 0, 42891, 66327, 74298, 0, - 0, 0, 2587, 42193, 0, 6455, 0, 4241, 0, 0, 0, 0, 0, 0, 0, 118821, 0, 0, - 0, 125030, 0, 128684, 129390, 6988, 5373, 0, 0, 119232, 10015, 0, 0, 0, - 68642, 0, 120855, 42040, 128827, 5779, 129841, 42037, 83282, 0, 0, 93040, - 83283, 101116, 0, 101117, 6983, 0, 0, 101115, 0, 0, 0, 127323, 101111, 0, - 119588, 0, 92495, 74558, 0, 68138, 70163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 11144, 73484, 2551, 73482, 6453, 0, 6235, 0, 0, 129081, 72886, 44020, - 11826, 0, 7780, 5369, 118958, 0, 0, 5367, 66870, 0, 0, 5377, 0, 68143, - 128624, 78245, 5218, 0, 127333, 0, 0, 129717, 0, 0, 1300, 0, 127334, - 64505, 0, 0, 119624, 1465, 0, 127316, 0, 0, 0, 101109, 0, 113694, 10729, - 0, 0, 8839, 119243, 0, 7785, 126530, 0, 0, 0, 0, 118638, 0, 0, 0, 3897, - 0, 92331, 74417, 113704, 0, 68127, 71425, 70688, 0, 0, 0, 0, 69287, 3542, - 0, 120685, 7951, 68152, 118857, 0, 92972, 0, 0, 127311, 73683, 0, 65150, - 68031, 0, 0, 0, 0, 9985, 0, 127328, 0, 0, 0, 0, 10830, 0, 615, 64490, - 7574, 0, 0, 0, 12909, 73698, 64559, 127332, 73951, 0, 67996, 2020, 0, 0, - 0, 120701, 0, 983640, 0, 0, 0, 92991, 0, 0, 9070, 0, 68411, 11281, 42829, - 0, 1033, 0, 78918, 0, 118610, 0, 65226, 0, 0, 0, 0, 0, 3450, 0, 7397, 0, - 0, 42778, 10000, 41088, 449, 0, 0, 68458, 113725, 0, 0, 10738, 69634, 0, - 0, 41085, 0, 0, 0, 12764, 0, 93058, 3596, 7322, 0, 0, 0, 0, 0, 0, 0, 0, - 2092, 0, 0, 0, 121350, 10820, 0, 0, 126567, 1853, 0, 0, 93014, 0, 12770, - 0, 0, 124997, 0, 0, 0, 0, 0, 129053, 4828, 1258, 0, 2006, 0, 0, 74285, - 127987, 0, 120683, 122880, 983900, 983903, 8846, 128255, 0, 128091, 2650, - 9182, 1961, 121399, 11525, 0, 1959, 0, 55228, 11774, 41016, 0, 0, 128054, - 41017, 13109, 0, 10519, 66331, 3454, 19930, 0, 41019, 92894, 0, 0, 78362, - 41021, 101566, 0, 0, 0, 0, 65531, 0, 0, 0, 0, 0, 0, 8865, 6402, 113827, - 77923, 0, 101536, 0, 7733, 0, 4998, 68493, 0, 78930, 0, 4268, 101368, 0, - 0, 101555, 101579, 10881, 0, 0, 0, 0, 2014, 0, 71901, 0, 0, 195057, 0, 0, - 78357, 65281, 0, 0, 0, 0, 0, 2015, 0, 0, 71840, 66318, 74824, 101575, 0, - 101574, 101569, 0, 70061, 8094, 10135, 101551, 0, 794, 0, 0, 66335, 0, - 121303, 4343, 0, 4833, 0, 0, 0, 0, 189, 12611, 0, 72215, 0, 4838, 126214, - 4834, 65078, 2183, 126104, 4837, 118853, 0, 121230, 4832, 128271, 0, - 101584, 127838, 0, 0, 0, 0, 0, 118697, 0, 3976, 118995, 128937, 0, 0, 0, - 0, 0, 119010, 0, 121015, 0, 0, 0, 0, 2871, 0, 0, 999, 0, 68177, 0, 0, - 2017, 0, 67824, 0, 0, 0, 0, 0, 0, 4775, 12555, 12571, 12550, 12583, - 12560, 2019, 12556, 12584, 12586, 0, 12562, 12561, 12566, 12569, 12554, - 0, 83344, 0, 68882, 0, 12567, 1402, 0, 0, 83348, 125072, 83347, 0, 83346, - 0, 0, 0, 0, 64391, 0, 83341, 69602, 0, 1999, 0, 128141, 0, 0, 0, 0, 0, 0, - 0, 68873, 0, 0, 66913, 2377, 101090, 0, 12572, 11318, 12557, 12559, 9192, - 12549, 12568, 2373, 9446, 9447, 9448, 9449, 0, 9480, 481, 0, 9438, 9439, - 9440, 9441, 9442, 9443, 9444, 9445, 9430, 9431, 9432, 9433, 9434, 9435, - 9436, 9437, 983097, 0, 9424, 9425, 9426, 9427, 9428, 7481, 0, 2362, 9655, - 0, 2004, 0, 9782, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 92461, 0, 128764, 0, - 64781, 0, 0, 0, 121126, 0, 1392, 0, 0, 917557, 0, 8065, 42994, 128739, 0, - 0, 0, 121068, 92418, 0, 0, 0, 43280, 0, 70718, 1812, 0, 73046, 0, 0, 0, - 0, 0, 6054, 10697, 3169, 0, 0, 70720, 11487, 70712, 0, 0, 0, 194716, 0, - 0, 41863, 0, 0, 2304, 0, 92326, 0, 42951, 0, 0, 64760, 11766, 0, 0, 0, 0, - 69236, 119171, 0, 8773, 10733, 36, 0, 0, 0, 0, 0, 11074, 0, 64910, - 983131, 2009, 0, 0, 128036, 68114, 128906, 0, 0, 0, 983998, 12852, 3031, - 0, 0, 129088, 0, 66414, 0, 0, 119950, 42613, 65933, 366, 0, 9892, 0, - 11754, 101107, 83329, 65301, 44013, 83058, 67245, 10102, 0, 7739, 41026, - 0, 0, 0, 0, 0, 0, 0, 0, 78386, 129475, 71868, 113811, 13081, 10923, - 129330, 0, 68145, 0, 65861, 74083, 0, 0, 128392, 83063, 83065, 0, 70706, - 0, 0, 0, 70168, 66586, 4183, 64967, 66250, 0, 92547, 0, 0, 113685, 0, - 3792, 2011, 0, 0, 77748, 83332, 77749, 120595, 0, 68489, 41023, 77747, 0, - 11659, 7922, 12614, 2005, 8523, 129880, 0, 7513, 1863, 129436, 83337, - 128969, 0, 120274, 120033, 0, 8144, 0, 73031, 77767, 127524, 120270, - 42241, 8783, 77764, 77765, 77766, 77760, 77761, 77762, 77763, 0, 10680, - 0, 43293, 68771, 0, 119164, 83320, 72003, 10187, 77742, 77743, 0, 77737, - 77738, 77739, 0, 10968, 43296, 119044, 0, 0, 101400, 0, 1005, 43826, - 120030, 0, 2870, 0, 101399, 0, 0, 983798, 0, 235, 1384, 77758, 74887, - 70494, 77754, 77755, 9796, 69895, 77750, 77751, 77752, 13186, 120407, - 120250, 0, 0, 0, 42527, 12911, 43427, 1383, 983581, 0, 0, 0, 6156, 68117, - 0, 7993, 4288, 0, 0, 13238, 13244, 0, 0, 120426, 13234, 120427, 0, - 118904, 0, 11364, 0, 1380, 65617, 120253, 120261, 13196, 13197, 120311, - 120419, 9495, 0, 0, 120418, 0, 73976, 128160, 0, 6941, 0, 13205, 13211, - 5801, 0, 74271, 120319, 0, 120302, 7670, 0, 68075, 983583, 0, 19957, - 72314, 2021, 93811, 43877, 0, 0, 0, 0, 3875, 120431, 64341, 0, 9814, - 43457, 13066, 3314, 7787, 0, 0, 0, 0, 0, 0, 64531, 129860, 0, 0, 0, 0, 0, - 127138, 0, 0, 9742, 0, 0, 10800, 77718, 8404, 0, 92592, 77714, 7089, - 77716, 78545, 0, 77712, 77713, 0, 0, 4772, 5771, 101405, 0, 9841, 8843, - 0, 0, 0, 129862, 120816, 0, 123137, 0, 77735, 0, 0, 0, 77731, 8849, - 77733, 77734, 65112, 1796, 77729, 77730, 69665, 8164, 41301, 3502, 0, - 122884, 128387, 0, 983835, 5825, 0, 0, 0, 0, 121322, 10983, 10354, 10418, - 0, 2022, 0, 1409, 100789, 0, 0, 0, 0, 1390, 0, 0, 10471, 65904, 5846, - 126472, 0, 0, 0, 0, 0, 0, 66035, 77725, 0, 77726, 77720, 77721, 67458, - 3168, 67733, 0, 0, 2370, 0, 126243, 0, 195049, 0, 0, 1836, 0, 121207, - 119137, 118959, 125232, 0, 0, 0, 2390, 3944, 0, 0, 0, 0, 69908, 125011, - 0, 0, 123200, 0, 0, 8975, 64739, 0, 0, 0, 0, 64409, 0, 0, 0, 0, 128564, - 0, 0, 0, 0, 6204, 0, 0, 0, 10911, 64954, 119003, 74809, 118903, 4267, 0, - 0, 0, 0, 0, 0, 72023, 0, 0, 119504, 92887, 0, 0, 0, 0, 121125, 0, 128337, - 5842, 0, 41439, 0, 0, 0, 9328, 0, 120980, 120917, 0, 0, 2285, 0, 0, 0, 0, - 118634, 64555, 0, 0, 72162, 9541, 0, 0, 0, 41441, 0, 0, 0, 41040, 2459, - 0, 0, 41041, 0, 0, 0, 0, 0, 10450, 0, 41043, 0, 0, 43125, 0, 0, 0, 0, 0, - 121008, 68436, 128040, 0, 120649, 0, 0, 4312, 43927, 0, 0, 11923, 42227, - 0, 5763, 0, 4827, 74559, 42228, 64406, 0, 0, 129703, 433, 119620, 0, - 2499, 67167, 67166, 0, 11973, 0, 4293, 42271, 42224, 0, 0, 66322, 42226, - 0, 0, 0, 74180, 0, 55277, 0, 0, 0, 119317, 0, 74632, 0, 0, 71103, 0, 0, - 0, 585, 2383, 0, 43263, 0, 4290, 64842, 0, 68920, 0, 8511, 0, 0, 0, - 119048, 2380, 126119, 0, 71704, 2376, 0, 0, 0, 5197, 127046, 127047, - 127048, 2366, 127050, 127051, 73442, 0, 0, 0, 93835, 0, 93818, 0, 0, - 74188, 113813, 0, 0, 0, 983838, 0, 0, 0, 0, 1847, 0, 72771, 0, 42384, 0, - 4227, 74158, 0, 92501, 0, 0, 42365, 0, 128902, 0, 92821, 0, 0, 0, 0, 0, - 0, 0, 0, 122934, 128563, 0, 983509, 127560, 2754, 0, 0, 128900, 0, - 127867, 119638, 0, 1711, 12984, 92365, 77776, 6255, 77770, 77771, 77772, - 77773, 0, 42063, 74184, 0, 0, 0, 0, 0, 77785, 77786, 41035, 43274, 77781, - 11256, 77783, 77784, 520, 77778, 41037, 77780, 0, 0, 41034, 0, 983829, - 64815, 0, 0, 321, 41028, 0, 0, 0, 0, 0, 0, 0, 74191, 0, 0, 72767, 1861, - 118938, 129666, 0, 0, 100770, 0, 0, 128530, 3859, 0, 41660, 0, 70793, 0, - 983756, 75014, 0, 127514, 41658, 0, 0, 0, 0, 0, 4414, 77769, 0, 42632, 0, - 0, 0, 0, 0, 1405, 0, 43220, 43341, 0, 0, 0, 0, 0, 983733, 11199, 0, 3513, - 0, 70341, 43342, 0, 65529, 0, 0, 0, 6485, 1397, 0, 0, 92678, 118566, - 124137, 0, 82961, 0, 82962, 0, 74270, 43287, 983731, 0, 0, 983738, 0, - 71914, 4317, 10490, 0, 0, 194867, 74463, 128952, 464, 41624, 0, 0, 0, - 1346, 128240, 69271, 64724, 128566, 423, 0, 0, 113748, 0, 128161, 0, 0, - 120563, 64960, 0, 0, 0, 0, 9584, 77795, 77796, 78927, 0, 9718, 77792, - 42642, 77794, 64750, 77789, 77790, 0, 0, 128333, 0, 3204, 64666, 0, - 43530, 2752, 0, 0, 119594, 0, 0, 0, 0, 92371, 0, 41983, 0, 7010, 0, 0, - 41495, 92379, 5877, 42252, 93070, 8009, 3305, 0, 0, 0, 0, 92293, 0, 0, 0, - 100836, 0, 65915, 1400, 75018, 10685, 75017, 2103, 122908, 0, 43276, 0, - 11169, 0, 6481, 0, 0, 0, 100837, 72249, 100838, 74198, 0, 9116, 0, 0, 0, - 0, 0, 0, 8129, 92994, 0, 124992, 0, 11658, 0, 0, 3452, 41031, 0, 1385, 0, - 100754, 0, 43340, 11123, 41033, 6493, 12758, 0, 0, 11426, 0, 1681, - 100755, 1204, 11960, 69902, 0, 69457, 0, 119322, 129483, 7415, 43338, 0, - 0, 67717, 64915, 0, 100759, 72021, 41497, 65044, 0, 19960, 65358, 983601, - 0, 0, 0, 73670, 0, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64728, 0, 0, 0, - 6506, 64312, 0, 2368, 0, 0, 0, 0, 3439, 1825, 1192, 0, 73739, 10639, 0, - 7790, 5430, 0, 0, 2848, 92981, 0, 0, 7607, 0, 0, 0, 120658, 0, 0, 8883, - 0, 728, 0, 0, 0, 0, 92931, 0, 121372, 128348, 0, 68078, 8091, 11447, 0, - 0, 126261, 983729, 0, 70003, 0, 0, 74419, 12335, 0, 0, 3443, 0, 0, 0, - 127145, 0, 0, 0, 0, 11843, 0, 9205, 8624, 128543, 92930, 43295, 0, 65445, - 0, 6277, 41672, 0, 10010, 70186, 983052, 0, 835, 71340, 0, 0, 0, 0, 0, - 5426, 4258, 0, 64891, 5424, 0, 8283, 0, 5434, 0, 0, 0, 0, 0, 11947, 0, - 1404, 0, 11432, 0, 3464, 6486, 4819, 0, 0, 570, 8095, 0, 0, 1498, 0, 0, - 0, 431, 67820, 0, 120574, 128096, 0, 0, 13096, 0, 0, 43408, 0, 128538, - 8835, 77875, 0, 0, 122973, 0, 0, 0, 0, 0, 3477, 227, 10488, 0, 382, - 11418, 0, 5878, 0, 0, 0, 0, 6484, 92355, 66039, 0, 0, 0, 78717, 0, 92662, - 119665, 0, 0, 43290, 0, 0, 0, 0, 8782, 0, 0, 4323, 128649, 0, 120903, - 12094, 67499, 0, 0, 0, 92953, 3856, 120970, 124138, 5872, 6495, 72306, 0, - 0, 0, 67173, 67172, 67171, 3953, 0, 0, 93063, 11994, 4339, 0, 92654, 0, - 0, 0, 0, 128804, 0, 5228, 0, 9766, 0, 92741, 0, 0, 0, 0, 68860, 0, 1162, - 0, 2671, 0, 73666, 92632, 92631, 72117, 0, 73811, 0, 194895, 0, 68085, 0, - 74331, 11424, 0, 10466, 121239, 0, 194890, 0, 4820, 0, 0, 0, 194891, 0, - 119212, 4896, 0, 4897, 42821, 64611, 0, 4438, 0, 0, 1753, 11331, 6147, 0, - 43282, 8833, 0, 0, 6504, 0, 0, 0, 118670, 0, 1413, 0, 0, 64353, 12141, - 121138, 0, 0, 43163, 0, 72880, 64789, 127094, 838, 127092, 120697, - 127090, 5014, 0, 256, 0, 0, 42443, 42739, 0, 7542, 0, 70389, 0, 6489, - 10048, 74326, 0, 66573, 0, 125271, 78712, 11761, 126078, 129603, 41094, - 0, 0, 0, 0, 92689, 8453, 0, 0, 120942, 128184, 0, 11816, 0, 0, 2930, - 93845, 0, 41098, 92771, 41093, 0, 0, 6498, 41096, 0, 0, 1238, 200, 0, - 1660, 74476, 0, 0, 74362, 0, 0, 72301, 9224, 0, 0, 0, 0, 0, 0, 0, 0, - 72729, 43284, 0, 72110, 67459, 13183, 0, 0, 0, 1669, 10776, 0, 0, 0, 0, - 0, 1732, 4030, 0, 3963, 0, 0, 0, 6491, 0, 0, 914, 121394, 0, 0, 0, 78713, - 0, 92441, 74367, 42960, 0, 0, 0, 0, 0, 0, 0, 65537, 0, 0, 43430, 5301, 0, - 92618, 0, 43285, 0, 0, 125186, 0, 0, 5876, 0, 69555, 0, 0, 110713, 0, 0, - 0, 0, 0, 0, 11114, 74536, 0, 0, 0, 0, 983130, 0, 0, 0, 0, 10915, 983069, - 12007, 0, 0, 0, 0, 67655, 92604, 0, 8629, 0, 43168, 41872, 0, 0, 0, - 42488, 0, 0, 0, 0, 0, 64730, 70041, 0, 122895, 0, 0, 0, 92306, 11416, - 4280, 128516, 8765, 73451, 0, 1393, 0, 11157, 74386, 0, 0, 0, 0, 6683, 0, - 93832, 12144, 0, 74513, 13019, 74994, 0, 0, 0, 983272, 0, 6488, 357, 0, - 41100, 0, 41104, 0, 41099, 0, 71320, 0, 0, 0, 4434, 0, 0, 0, 74231, - 83107, 0, 194914, 0, 0, 72286, 68305, 0, 41759, 12757, 0, 0, 72769, 9790, - 7674, 0, 121095, 68209, 0, 41764, 0, 0, 72322, 2268, 0, 129845, 983608, - 12743, 0, 6480, 0, 41779, 0, 66601, 0, 74490, 10986, 66602, 0, 64807, 0, - 0, 41767, 119629, 0, 0, 0, 3955, 64571, 194918, 127089, 0, 70187, 69975, - 9770, 12305, 12230, 0, 73551, 0, 0, 74752, 0, 0, 123168, 128263, 74449, - 0, 65948, 69611, 0, 0, 71131, 129505, 78573, 0, 0, 11116, 0, 5747, 0, - 110667, 9802, 41092, 120731, 0, 0, 0, 0, 0, 120733, 41090, 0, 0, 0, - 11271, 57, 0, 0, 983244, 0, 71268, 121290, 43137, 0, 0, 110672, 126221, - 0, 0, 0, 0, 0, 277, 74385, 0, 0, 0, 72155, 0, 13025, 8757, 0, 0, 1574, 0, - 126124, 100800, 0, 5749, 129923, 0, 42824, 0, 1039, 9801, 0, 5745, 0, - 41858, 0, 0, 120655, 0, 41862, 0, 0, 78822, 436, 4771, 118635, 42501, 0, - 10573, 0, 77931, 118560, 917986, 9644, 0, 0, 0, 0, 69837, 0, 0, 0, 0, - 67409, 0, 0, 0, 125204, 11939, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 92854, - 126209, 0, 10226, 65558, 0, 3594, 0, 0, 40, 0, 0, 0, 0, 0, 74312, 72138, - 74337, 0, 69577, 0, 0, 0, 70476, 0, 121143, 72317, 0, 0, 4304, 0, 0, - 78707, 0, 0, 0, 78597, 1348, 78596, 0, 0, 0, 70406, 92392, 0, 7599, 0, 0, - 13269, 0, 129729, 0, 100804, 0, 74494, 6097, 7568, 43980, 4982, 78592, 0, - 0, 0, 0, 13270, 0, 128090, 13138, 0, 9484, 0, 0, 71364, 0, 0, 0, 9487, 0, - 92913, 0, 71911, 78668, 73963, 6193, 0, 0, 0, 194848, 7228, 10011, - 194849, 194852, 194851, 11654, 194853, 126218, 194855, 0, 194857, 3604, - 0, 0, 0, 0, 0, 94110, 43740, 94109, 194860, 194863, 66750, 121021, 0, - 94111, 6995, 74173, 5437, 74174, 0, 8702, 7339, 129981, 0, 199, 194843, - 194846, 194845, 0, 126069, 0, 67818, 0, 7560, 0, 0, 0, 0, 6472, 65814, 0, - 128983, 70845, 0, 0, 9191, 0, 0, 0, 0, 124904, 10196, 0, 72452, 6585, 0, - 120750, 0, 0, 71872, 129129, 0, 0, 78590, 72308, 11382, 129499, 0, - 983670, 0, 194833, 194832, 2165, 129540, 94020, 194836, 42727, 194838, - 128252, 78585, 43874, 119610, 0, 0, 43248, 0, 194816, 0, 194818, 128845, - 194820, 127879, 5297, 194821, 13284, 6112, 93964, 93010, 73927, 42947, 0, - 65746, 0, 0, 194827, 194826, 4342, 42839, 194831, 1677, 0, 72135, 0, 0, - 0, 11011, 66399, 0, 0, 0, 10160, 0, 0, 0, 0, 2052, 4308, 92174, 43000, - 118659, 543, 64916, 122964, 0, 0, 119170, 0, 118922, 2064, 0, 43158, 0, - 0, 69984, 0, 0, 129187, 0, 0, 0, 0, 41631, 92728, 0, 0, 6228, 0, 0, 0, 0, - 0, 0, 506, 0, 0, 65735, 2055, 43255, 121407, 0, 0, 0, 0, 0, 0, 194666, - 2063, 0, 0, 0, 0, 72136, 0, 74333, 194912, 11827, 74308, 194913, 194916, - 194915, 64564, 194917, 67986, 194919, 0, 11037, 0, 121102, 0, 0, 10560, - 0, 120756, 194922, 113737, 194924, 194927, 120495, 1931, 0, 0, 0, 128228, - 0, 12643, 8751, 123629, 0, 12294, 0, 78834, 9138, 78831, 78833, 12631, - 78829, 11080, 78821, 0, 0, 1239, 0, 121067, 0, 12636, 0, 0, 0, 0, 0, 0, - 8998, 0, 0, 9152, 0, 0, 0, 67589, 0, 64290, 0, 92393, 12615, 0, 129141, - 6914, 93013, 0, 119569, 0, 65188, 0, 67611, 4337, 0, 194897, 194896, - 78516, 194898, 7681, 194900, 194903, 67596, 194905, 194904, 2477, 93974, - 0, 0, 0, 67604, 70705, 0, 194882, 194881, 194884, 194883, 194886, 128914, - 194888, 67599, 0, 194889, 0, 0, 0, 0, 3357, 0, 78852, 4207, 1288, 78842, - 78839, 78840, 78837, 78838, 66354, 194872, 0, 128432, 0, 67618, 92664, 0, - 42788, 0, 64612, 129897, 10774, 194877, 0, 194879, 129125, 0, 0, 997, - 194901, 0, 92577, 0, 11440, 11379, 42000, 13139, 0, 0, 74030, 72293, - 73796, 0, 0, 0, 0, 2818, 0, 0, 73793, 0, 4172, 93028, 126523, 124981, 0, - 129896, 0, 0, 129522, 69706, 0, 6834, 0, 0, 194865, 126982, 121211, - 194866, 194869, 194868, 766, 1257, 0, 0, 0, 3265, 66617, 3274, 0, 0, - 94042, 0, 8373, 41989, 69507, 73460, 3418, 3263, 0, 0, 0, 3270, 64539, - 11489, 0, 118945, 126220, 0, 127795, 0, 94031, 0, 0, 0, 0, 0, 70512, - 983983, 186, 0, 119156, 5770, 13179, 0, 12612, 12949, 64856, 12800, 0, 0, - 983152, 11507, 0, 0, 118929, 0, 0, 72141, 0, 73459, 0, 0, 0, 73461, 9254, - 66877, 194907, 0, 92338, 5624, 126253, 0, 0, 0, 120472, 120464, 0, 0, - 122915, 120462, 0, 1872, 66508, 120467, 41079, 0, 5502, 119330, 41078, - 194906, 0, 0, 4511, 68449, 0, 0, 0, 0, 43245, 41083, 68861, 0, 0, 9003, - 119959, 0, 5305, 9653, 41081, 43146, 9546, 0, 0, 120478, 0, 65205, 71713, - 64063, 120459, 0, 0, 0, 64058, 43101, 43102, 0, 64062, 1028, 64060, - 64059, 0, 10567, 110816, 110817, 92857, 110815, 0, 2902, 64043, 64042, - 43749, 10756, 64047, 64046, 64045, 64044, 0, 10076, 64040, 64039, 0, - 1034, 0, 0, 64034, 64033, 64032, 42735, 64038, 64037, 64036, 64035, 4291, - 67497, 64015, 64014, 83393, 83394, 83395, 983784, 0, 43090, 83391, 3476, - 64013, 64012, 64011, 64010, 64008, 64007, 2003, 7706, 0, 0, 119050, - 64009, 204, 0, 0, 4430, 8239, 64003, 10626, 64001, 64057, 13079, 64055, - 64054, 0, 0, 43246, 9343, 64049, 64048, 0, 1133, 64053, 64052, 64051, - 64050, 0, 0, 0, 66415, 12329, 0, 0, 129698, 1942, 0, 0, 0, 128249, 0, - 68291, 10760, 64023, 64022, 64021, 64020, 43670, 77924, 64025, 41412, - 78243, 78244, 0, 0, 64019, 64018, 64017, 64016, 0, 0, 78251, 78252, - 78248, 78249, 77914, 78247, 0, 917560, 77919, 6788, 13094, 0, 7532, - 41414, 0, 3179, 70745, 64769, 0, 0, 71967, 0, 10751, 0, 0, 0, 0, 0, 0, 0, - 2008, 64031, 64030, 294, 41874, 83383, 83384, 64844, 83376, 129063, - 83379, 83380, 64028, 11396, 64026, 83374, 0, 0, 118795, 71739, 43247, 0, - 70153, 70846, 0, 0, 0, 0, 0, 0, 0, 7801, 83359, 83361, 128931, 0, 3297, - 83356, 83357, 1135, 83350, 83351, 73696, 1995, 7927, 71738, 110742, 2552, - 83372, 60, 0, 8649, 83368, 83369, 83370, 83371, 10541, 83365, 78679, - 43833, 0, 0, 2013, 83362, 0, 110636, 0, 0, 12832, 110638, 8081, 8362, - 120188, 0, 9137, 0, 0, 0, 0, 3466, 0, 0, 1996, 0, 3453, 3412, 0, 2002, - 2000, 120176, 0, 0, 0, 0, 1998, 0, 1842, 7037, 0, 9628, 68446, 0, 9826, - 64502, 1767, 3413, 0, 0, 0, 0, 0, 0, 13108, 44024, 120204, 0, 92693, 0, - 0, 0, 70291, 12650, 983210, 0, 68061, 0, 3592, 0, 0, 0, 0, 983975, 0, - 66417, 128792, 10742, 0, 0, 1994, 9281, 3296, 64475, 1997, 1895, 128936, - 43024, 0, 0, 123184, 72391, 0, 8999, 0, 983633, 0, 66480, 0, 0, 0, - 983083, 0, 596, 0, 0, 120216, 8651, 120217, 0, 0, 12995, 0, 0, 70740, 0, - 42930, 119955, 64810, 917834, 6825, 0, 917839, 120208, 64275, 120889, - 128069, 120210, 6384, 917840, 126477, 0, 67698, 0, 0, 0, 120496, 0, - 43412, 0, 0, 0, 0, 0, 120172, 0, 120763, 0, 0, 0, 128343, 1457, 0, 0, - 6381, 2815, 0, 65240, 129664, 0, 0, 119522, 70487, 0, 0, 0, 0, 0, 120572, - 0, 0, 0, 0, 0, 125253, 0, 0, 0, 11862, 0, 0, 0, 3055, 9852, 0, 65288, 0, - 11398, 0, 0, 93016, 123550, 0, 603, 128557, 0, 0, 0, 129366, 3350, 0, 0, - 917828, 917827, 68428, 917825, 73045, 917831, 917830, 917829, 0, 1919, 0, - 110767, 83296, 83297, 83298, 66446, 64141, 8562, 64139, 64138, 64136, - 64135, 64134, 64133, 11297, 0, 0, 11966, 64128, 66286, 0, 123541, 64132, - 10867, 64130, 64129, 0, 43374, 9779, 2764, 0, 0, 9471, 0, 0, 0, 0, 66010, - 0, 8857, 128771, 121423, 0, 69223, 0, 194660, 983876, 0, 0, 43984, 0, 0, - 0, 0, 0, 0, 10717, 64570, 5630, 0, 64143, 64142, 83300, 67758, 83302, 0, - 77930, 0, 0, 0, 11631, 64146, 64145, 64144, 2384, 72801, 127380, 0, 0, 0, - 0, 118547, 0, 0, 0, 0, 122916, 8933, 1601, 917803, 858, 118637, 64109, - 64108, 8090, 0, 72387, 917811, 587, 0, 82971, 0, 0, 0, 78214, 2750, 0, - 9983, 64158, 64157, 83288, 83289, 83290, 2167, 83284, 83285, 83286, - 83287, 64156, 64155, 64154, 69495, 64151, 64150, 12679, 10053, 10421, 0, - 64153, 64152, 0, 0, 4839, 0, 0, 4435, 119016, 0, 64126, 64125, 64124, - 64123, 129287, 0, 67478, 7007, 0, 65443, 0, 0, 64122, 0, 0, 93834, 64117, - 64116, 6287, 64114, 64121, 64120, 64119, 64118, 110659, 127842, 1177, - 65601, 12322, 64106, 92169, 110654, 64102, 64101, 64100, 64099, 0, 10453, - 64104, 64103, 7997, 0, 92534, 0, 8705, 64097, 64096, 9571, 0, 110652, - 127398, 12132, 0, 0, 0, 110624, 73841, 83339, 83340, 9056, 0, 129970, 0, - 6155, 64068, 64067, 64066, 64065, 64072, 64071, 63, 64069, 127382, 0, - 93822, 7257, 64064, 0, 0, 0, 0, 0, 0, 78748, 0, 0, 0, 120519, 0, 66242, - 66232, 4333, 9855, 64112, 0, 0, 118531, 0, 0, 0, 0, 66222, 0, 0, 0, 0, - 69816, 0, 118796, 0, 8708, 0, 64077, 64076, 8996, 4992, 4471, 83343, - 64079, 64078, 92179, 0, 0, 123540, 64615, 0, 0, 12075, 42041, 0, 0, 0, 0, - 127557, 3123, 0, 983754, 0, 0, 0, 83328, 0, 9223, 0, 83321, 83322, 73797, - 83327, 1116, 0, 83319, 7136, 73550, 0, 0, 0, 75031, 0, 0, 0, 64092, - 43675, 10104, 83338, 83331, 64095, 64094, 8111, 66247, 0, 64089, 64088, - 0, 70106, 42236, 11434, 64083, 64082, 43216, 7737, 64087, 64086, 64085, - 64084, 0, 0, 0, 4118, 1797, 83312, 0, 0, 46, 83308, 83309, 298, 83303, - 72402, 83305, 83306, 0, 0, 0, 128905, 11495, 0, 67490, 0, 127377, 194828, - 127370, 0, 0, 0, 66239, 74945, 64403, 0, 0, 83314, 0, 0, 65758, 43536, 0, - 8544, 0, 0, 0, 0, 194824, 0, 0, 0, 0, 0, 3639, 11242, 194822, 0, 0, 0, 0, - 0, 0, 68409, 0, 0, 0, 101121, 0, 0, 0, 128654, 8789, 126248, 0, 0, - 128325, 0, 0, 0, 0, 65058, 0, 78234, 68064, 0, 66227, 71694, 5573, - 118936, 0, 44, 0, 66244, 118907, 0, 66238, 12844, 0, 1622, 129190, 1900, - 0, 11458, 0, 0, 6581, 5576, 128303, 0, 126122, 0, 113680, 8947, 0, - 113812, 0, 69744, 0, 7908, 0, 917998, 6579, 0, 0, 0, 0, 2138, 6583, 7761, - 0, 0, 0, 66802, 5058, 0, 0, 0, 5057, 125256, 0, 74538, 5054, 0, 0, 0, 0, - 0, 0, 658, 3497, 128509, 0, 5061, 5060, 4235, 0, 0, 0, 127757, 4236, - 4727, 0, 0, 0, 128791, 0, 7488, 128693, 7476, 0, 125259, 120646, 0, 0, 0, - 66209, 0, 0, 0, 78931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128903, 0, 9341, - 119596, 0, 0, 0, 64668, 0, 8125, 0, 6743, 119175, 0, 129441, 83406, 0, - 127966, 119235, 74092, 0, 0, 43660, 71125, 0, 127901, 0, 0, 0, 264, 0, - 74954, 0, 0, 0, 0, 0, 6019, 0, 0, 129121, 0, 0, 0, 8800, 0, 66376, 0, - 120948, 0, 100744, 0, 0, 92333, 725, 68014, 110579, 110578, 72099, 0, 0, - 0, 0, 74899, 0, 0, 0, 110804, 0, 72142, 5074, 5073, 0, 0, 128726, 0, - 70723, 5072, 128576, 13098, 72403, 0, 11040, 0, 0, 0, 4929, 0, 0, 0, 0, - 0, 0, 0, 0, 67754, 4934, 0, 0, 9758, 0, 0, 70181, 42584, 0, 4329, 0, - 4979, 8663, 74521, 0, 983042, 74418, 983653, 0, 5071, 0, 3642, 0, 5070, - 10042, 0, 3987, 5068, 120209, 8909, 0, 0, 69917, 0, 73981, 983142, 70749, - 4531, 120212, 9105, 0, 4921, 121059, 4926, 65544, 113786, 69621, 0, 0, 0, - 83269, 0, 120790, 4922, 0, 992, 119568, 4925, 0, 0, 9526, 4920, 128617, - 948, 0, 0, 4930, 0, 0, 0, 4933, 0, 0, 0, 4928, 0, 0, 0, 0, 128379, 722, - 118696, 127483, 127482, 127485, 82997, 127487, 1509, 0, 5468, 66214, - 127474, 127477, 1672, 127479, 10864, 127481, 72132, 127467, 72159, - 127469, 127468, 127471, 127470, 68336, 82999, 120115, 1679, 120116, 0, - 120113, 127462, 127465, 127464, 127110, 120119, 120112, 0, 120109, 6968, - 5761, 342, 8553, 0, 8143, 127115, 127114, 2253, 624, 127111, 4057, 0, - 5078, 0, 0, 0, 5076, 0, 0, 0, 120097, 685, 9025, 1524, 8003, 0, 5539, - 113727, 113795, 118673, 7138, 120552, 0, 0, 0, 113724, 0, 8058, 9732, 0, - 5080, 0, 5036, 5035, 0, 42604, 72118, 0, 0, 275, 13291, 69995, 0, 0, - 983927, 5033, 0, 0, 4836, 70184, 73792, 0, 0, 0, 120681, 43704, 0, 2274, - 119000, 124983, 0, 8858, 6409, 0, 119585, 0, 0, 0, 0, 0, 68442, 0, 3432, - 10218, 0, 6094, 11232, 0, 0, 0, 0, 1676, 129157, 0, 0, 5030, 0, 118810, - 0, 73869, 0, 0, 69944, 6787, 0, 0, 0, 983595, 10544, 12919, 69425, 92218, - 0, 0, 0, 129172, 118715, 67703, 0, 0, 0, 0, 0, 72290, 0, 0, 0, 0, 7018, - 66241, 0, 0, 0, 0, 0, 74056, 0, 11833, 0, 67975, 65232, 40964, 251, - 12686, 7895, 4395, 43538, 0, 0, 0, 78042, 0, 0, 40967, 5879, 0, 0, 0, 0, - 0, 65540, 128590, 625, 0, 120194, 1113, 120195, 13103, 3630, 67224, 8179, - 74264, 67886, 9316, 10980, 2489, 120958, 8150, 1359, 121353, 70464, - 127330, 127327, 5042, 5041, 42769, 12084, 11196, 42961, 92279, 72398, - 120535, 127317, 127318, 127315, 12283, 127313, 11453, 70207, 8795, 66245, - 0, 5919, 0, 5037, 118864, 0, 0, 67724, 0, 66893, 74006, 129535, 8431, 0, - 0, 0, 0, 12620, 6826, 73773, 70169, 5040, 0, 0, 0, 0, 0, 5039, 0, 0, 0, - 5038, 0, 0, 0, 0, 0, 65908, 0, 0, 0, 0, 0, 65157, 0, 0, 70182, 0, 73909, - 4835, 0, 0, 0, 4309, 7127, 0, 0, 0, 1301, 0, 0, 12222, 0, 73813, 711, - 92439, 7133, 0, 0, 0, 0, 0, 0, 0, 7661, 72263, 129541, 0, 0, 70453, 7627, - 0, 5031, 92340, 42738, 65784, 0, 65782, 3758, 0, 65781, 67865, 0, 2440, - 65780, 70795, 8449, 121393, 121479, 0, 2118, 0, 12121, 0, 0, 129510, - 2128, 2130, 2131, 2126, 2133, 0, 121250, 2114, 2116, 2455, 0, 2122, 2123, - 2124, 2125, 983806, 8714, 0, 2113, 0, 2115, 0, 127907, 43713, 5052, - 66220, 66653, 65777, 65778, 65775, 5051, 65773, 1429, 42647, 5050, 65769, - 388, 70685, 735, 0, 129899, 128035, 0, 12726, 0, 0, 0, 0, 0, 5109, 5053, - 0, 120854, 0, 0, 0, 2470, 0, 0, 1925, 71251, 0, 10971, 113770, 5048, - 5047, 0, 0, 194946, 92313, 129972, 0, 0, 8089, 128468, 639, 0, 68179, 0, - 70180, 0, 4599, 0, 0, 0, 0, 983817, 648, 194948, 65819, 0, 0, 0, 129968, - 94017, 0, 11777, 9750, 983123, 0, 0, 92367, 70175, 5046, 66255, 0, 0, - 65253, 0, 5045, 0, 1916, 74069, 5044, 92348, 0, 0, 5043, 0, 0, 0, 74004, - 9669, 12341, 0, 8402, 0, 0, 70174, 0, 3586, 64508, 92456, 0, 0, 119606, - 0, 42628, 10069, 0, 0, 0, 0, 123, 120703, 0, 121326, 0, 10719, 129409, - 120444, 10829, 120593, 0, 12130, 0, 0, 0, 0, 3925, 0, 0, 75065, 71112, - 92372, 71110, 71111, 0, 120441, 120452, 983179, 0, 0, 0, 0, 0, 0, 0, 0, - 69879, 8509, 120449, 0, 0, 0, 120448, 0, 118889, 194858, 0, 0, 0, 66445, - 0, 71109, 0, 0, 72425, 0, 12136, 0, 983629, 0, 0, 0, 0, 19922, 41768, - 74002, 0, 0, 0, 0, 2458, 0, 0, 0, 41074, 4266, 64834, 0, 41077, 0, 9050, - 0, 0, 73693, 0, 0, 41075, 2476, 0, 0, 0, 69761, 0, 0, 74202, 78745, 0, - 121324, 70152, 66033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83106, 0, 0, 0, 43693, - 78753, 0, 12194, 66215, 71987, 121273, 67216, 121499, 0, 121118, 0, - 78756, 0, 0, 55256, 0, 0, 0, 0, 43876, 0, 0, 0, 12948, 195003, 195002, - 195005, 195004, 195007, 195006, 0, 128320, 4287, 70183, 4902, 74020, 0, - 917961, 0, 1816, 0, 0, 168, 0, 4898, 64298, 0, 78450, 4901, 1821, 0, - 43294, 3653, 120899, 791, 9162, 6977, 121183, 0, 70160, 0, 73731, 8354, - 0, 0, 0, 7557, 0, 0, 8234, 194992, 78456, 194994, 194993, 194996, 101519, - 65925, 194997, 195000, 194999, 0, 66996, 0, 64397, 0, 0, 0, 71310, - 194977, 194976, 2448, 194978, 194981, 194980, 2452, 194982, 194985, - 194984, 78694, 72292, 7845, 0, 78692, 4408, 4122, 6772, 194988, 8723, - 72147, 194989, 73472, 11857, 119304, 119303, 2438, 119297, 119300, - 119299, 41953, 0, 42135, 373, 119172, 2119, 11457, 129618, 41955, 0, 0, - 0, 41952, 0, 0, 2127, 0, 128496, 5202, 0, 78765, 42823, 11291, 0, 0, - 12963, 0, 0, 4125, 41958, 12133, 0, 125099, 1271, 129427, 0, 66024, 0, - 3864, 127825, 0, 0, 0, 0, 4166, 0, 0, 129917, 7459, 0, 119914, 5384, 0, - 0, 70154, 5759, 0, 0, 0, 0, 66744, 0, 120571, 0, 75066, 5552, 0, 0, - 127192, 5553, 0, 0, 0, 12906, 0, 0, 110787, 110792, 110788, 5554, 0, - 12344, 110786, 101508, 0, 0, 0, 0, 8517, 101509, 0, 0, 66017, 5555, - 92317, 0, 983672, 0, 0, 0, 9143, 0, 195067, 67995, 195069, 127162, - 195071, 195070, 4577, 64624, 0, 0, 125105, 983680, 4269, 983674, 983671, - 983669, 0, 950, 0, 983673, 983683, 983668, 0, 983675, 0, 119121, 0, 5098, - 0, 0, 119099, 5097, 0, 9848, 0, 10293, 983664, 72798, 0, 0, 70303, - 983684, 5102, 5101, 128370, 0, 8138, 4517, 1932, 5100, 195060, 65022, - 1247, 10034, 195064, 5099, 0, 1441, 0, 4724, 650, 0, 73954, 983271, - 129348, 195040, 195043, 9031, 195045, 195044, 195047, 8545, 66356, - 195048, 0, 9154, 127243, 0, 0, 2676, 2277, 0, 73812, 195051, 8599, - 195053, 917918, 195055, 65462, 0, 92524, 195033, 71903, 0, 0, 41199, 0, - 11399, 195035, 195034, 195037, 195036, 195039, 195038, 5108, 5107, 0, - 66019, 0, 0, 5541, 0, 0, 12613, 5284, 0, 0, 118537, 4275, 74865, 854, - 68147, 74381, 120918, 0, 5103, 124986, 64348, 0, 0, 5221, 69811, 0, - 71493, 121163, 0, 0, 11438, 0, 0, 70158, 0, 0, 5106, 195024, 110749, - 65154, 69813, 195028, 5105, 195030, 69720, 195032, 5104, 983780, 0, 3176, - 127342, 70149, 932, 0, 6567, 195009, 195008, 195011, 195010, 70145, - 43850, 195015, 195014, 195017, 195016, 0, 0, 0, 69511, 10670, 0, 13273, - 0, 195020, 121370, 8803, 195021, 72431, 8151, 67145, 72436, 0, 12553, 0, - 0, 0, 0, 13065, 12570, 0, 0, 0, 983199, 124985, 0, 0, 66466, 0, 0, - 194595, 0, 194596, 11351, 43256, 0, 0, 0, 0, 41754, 0, 0, 2720, 194975, - 68462, 8232, 120758, 0, 0, 0, 0, 122959, 0, 0, 93067, 10834, 0, 0, - 119266, 0, 0, 125025, 67679, 0, 75064, 7781, 0, 0, 126076, 0, 12077, 0, - 64586, 127164, 42396, 0, 3475, 0, 2479, 0, 0, 0, 120728, 0, 42434, - 129709, 194963, 194962, 110611, 67894, 42473, 194966, 110609, 1843, - 42283, 0, 0, 0, 0, 0, 194970, 0, 42321, 7284, 194974, 194973, 194950, - 194949, 194952, 194951, 0, 194953, 123614, 128645, 0, 0, 0, 0, 74952, - 194954, 194957, 194956, 66367, 194958, 41069, 67689, 9988, 0, 41068, 0, - 4295, 0, 0, 41951, 67835, 0, 785, 8236, 128647, 9027, 0, 194943, 0, - 122986, 0, 0, 0, 0, 41071, 41059, 0, 92458, 129442, 0, 0, 0, 123612, - 2067, 4310, 0, 123611, 5180, 123605, 0, 73872, 0, 69880, 5184, 42385, - 194947, 983774, 128531, 0, 0, 119149, 73503, 121334, 0, 983781, 0, 0, - 5178, 194929, 120548, 194931, 5188, 194933, 194932, 72245, 194934, 1166, - 64429, 42639, 0, 0, 0, 0, 128071, 2442, 10703, 194940, 194939, 194635, - 42439, 0, 0, 0, 73933, 983242, 42401, 0, 0, 0, 42288, 0, 0, 0, 13145, 0, - 2468, 0, 42327, 0, 0, 0, 42479, 128698, 0, 0, 92580, 0, 74939, 120678, 0, - 73733, 0, 0, 2715, 0, 71257, 0, 74114, 0, 0, 0, 0, 0, 66325, 69603, 0, - 9240, 0, 0, 129142, 0, 0, 0, 9815, 0, 11246, 0, 73912, 42733, 0, 0, 2480, - 0, 0, 0, 6494, 5537, 0, 0, 0, 0, 1211, 0, 121379, 0, 0, 12318, 0, 113796, - 0, 0, 0, 0, 0, 64642, 0, 0, 0, 0, 64864, 0, 0, 0, 121212, 0, 0, 3589, - 92719, 4035, 6492, 92236, 4265, 6843, 0, 74186, 41778, 113764, 119216, - 2488, 0, 4582, 0, 71426, 41777, 12926, 72708, 7528, 10550, 113761, 0, 0, - 11439, 0, 0, 64878, 0, 0, 0, 0, 2286, 0, 0, 126646, 127909, 5909, 400, - 126500, 0, 0, 0, 0, 0, 64827, 0, 74948, 390, 0, 71301, 0, 3473, 0, 0, - 66742, 0, 55285, 0, 0, 0, 92206, 194964, 0, 8004, 0, 6763, 0, 0, 7006, 0, - 0, 6757, 73707, 126648, 0, 6766, 0, 0, 0, 6146, 0, 771, 0, 0, 41318, 0, - 42272, 0, 120211, 69559, 0, 953, 12917, 72287, 12300, 64837, 11491, - 68612, 0, 0, 71321, 7490, 11389, 7489, 3379, 0, 7487, 42996, 7486, 7484, - 7482, 6753, 7480, 7479, 7478, 7477, 6501, 7475, 42995, 7473, 7472, 2474, - 7470, 7468, 124977, 0, 0, 0, 0, 71871, 11834, 128376, 0, 6017, 0, 128763, - 0, 0, 0, 119365, 73949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 69945, - 120699, 121133, 2139, 4256, 120776, 74380, 0, 73847, 73844, 0, 0, 101375, - 0, 101374, 0, 0, 101347, 7083, 0, 8066, 7678, 0, 121124, 101341, 101373, - 101336, 0, 101331, 0, 101304, 0, 101301, 0, 0, 0, 8330, 0, 101298, - 101322, 101297, 0, 0, 19934, 0, 1770, 67091, 0, 128671, 129617, 110605, - 101355, 73843, 110604, 0, 101362, 67092, 0, 71334, 0, 0, 0, 0, 0, 8162, - 0, 5996, 129644, 4903, 0, 0, 43063, 0, 5172, 0, 7139, 0, 127385, 0, - 118667, 0, 0, 4334, 6324, 41975, 12186, 10674, 12308, 0, 0, 0, 72807, - 41977, 68002, 0, 126630, 2018, 121388, 41979, 68003, 0, 68000, 0, 0, - 126984, 68001, 9334, 118609, 71440, 0, 7975, 0, 0, 0, 66621, 4884, 70367, - 983759, 0, 121010, 0, 0, 0, 0, 127799, 0, 0, 0, 463, 0, 194584, 69617, - 6509, 5460, 0, 0, 0, 0, 42279, 0, 0, 0, 0, 0, 0, 0, 125027, 0, 121119, 0, - 0, 0, 5663, 0, 0, 0, 0, 2482, 66202, 0, 0, 42247, 65174, 73925, 0, - 100940, 0, 0, 126573, 0, 0, 2460, 0, 11944, 0, 0, 64679, 120835, 127310, - 0, 0, 0, 5870, 0, 0, 0, 100931, 539, 100933, 100932, 100935, 9064, - 100937, 100936, 100939, 100938, 0, 0, 0, 0, 0, 0, 41295, 100941, 2478, - 100943, 4162, 100945, 4260, 12953, 100950, 100949, 129800, 0, 0, 0, 0, 0, - 0, 0, 5000, 0, 0, 0, 69672, 71439, 0, 74017, 0, 0, 6709, 0, 0, 983739, 0, - 0, 100922, 100921, 10301, 10333, 10397, 100925, 100928, 100927, 0, 0, 0, - 127830, 0, 4014, 12842, 0, 67413, 0, 0, 3893, 0, 0, 12210, 0, 42147, 0, - 983622, 74465, 0, 0, 0, 0, 0, 0, 0, 0, 110805, 8231, 0, 69946, 41968, - 100929, 41973, 12935, 41969, 0, 2453, 0, 0, 78807, 122893, 0, 10349, - 10413, 122956, 41962, 3202, 119097, 0, 8316, 129174, 0, 7314, 0, 0, 0, 0, - 1840, 0, 0, 0, 4883, 100908, 4723, 70099, 100909, 0, 0, 0, 0, 11089, 240, - 19906, 0, 0, 0, 43600, 121004, 13134, 93065, 0, 65931, 110649, 110650, - 42634, 110648, 0, 121005, 11463, 0, 0, 129861, 10445, 0, 92969, 0, 2614, - 0, 129954, 1729, 0, 0, 100911, 0, 43334, 100912, 100915, 100914, 66201, - 100916, 69662, 100896, 100899, 100898, 4121, 100900, 70272, 82954, 63879, - 0, 70872, 0, 0, 4039, 643, 7726, 120082, 0, 120068, 58, 0, 0, 0, 63872, - 0, 0, 100891, 0, 10625, 100892, 100895, 100894, 1416, 120073, 917761, - 67393, 0, 0, 0, 6996, 4264, 0, 100902, 66179, 66768, 100903, 13114, - 72311, 67510, 3094, 0, 0, 127074, 4437, 0, 0, 0, 55280, 42174, 0, 42430, - 129796, 72246, 42355, 0, 0, 0, 0, 121251, 127401, 0, 0, 0, 0, 0, 0, - 100882, 100881, 74037, 100883, 0, 127099, 0, 0, 0, 0, 0, 69646, 65035, - 65034, 11480, 6116, 65039, 65038, 41180, 65036, 194565, 0, 12101, 5822, - 0, 0, 0, 0, 11663, 127873, 63854, 119657, 63853, 0, 63852, 65810, 4289, - 100885, 63896, 100887, 100890, 43621, 0, 0, 0, 129613, 194560, 7461, - 73901, 0, 331, 0, 0, 0, 128029, 0, 0, 0, 74629, 0, 0, 0, 41964, 0, 63843, - 2084, 41965, 0, 100864, 100863, 100866, 63841, 78549, 41220, 13032, - 100869, 8383, 0, 78548, 126102, 0, 0, 1351, 983865, 8698, 100874, 100877, - 1930, 100879, 78554, 74360, 100880, 69859, 78551, 0, 0, 129433, 3657, 0, - 65202, 6000, 119206, 41901, 0, 0, 41740, 0, 41283, 73543, 119267, 0, 0, - 100871, 9695, 100873, 7562, 100853, 5170, 100855, 100854, 676, 100856, - 100859, 100858, 9978, 100860, 0, 0, 64934, 0, 0, 0, 113714, 113706, - 41829, 65886, 5159, 0, 41832, 704, 43077, 0, 120532, 0, 68496, 65065, - 41830, 0, 917799, 917798, 917797, 917796, 0, 67864, 113696, 917800, - 12336, 4135, 69805, 341, 2727, 4129, 100862, 100861, 0, 64503, 7913, 0, - 0, 4131, 63868, 0, 63871, 4133, 63864, 210, 0, 0, 0, 4137, 78505, 78506, - 0, 78504, 78830, 0, 0, 43873, 0, 0, 0, 0, 11988, 78510, 195, 68321, - 41501, 0, 42031, 0, 13135, 0, 0, 0, 41499, 0, 0, 9680, 41498, 917794, - 42025, 78567, 78556, 0, 0, 0, 0, 0, 0, 101074, 120502, 92597, 0, 0, - 917784, 7864, 129001, 129704, 917788, 121106, 917786, 917785, 5753, - 67816, 72371, 2219, 0, 0, 0, 0, 0, 0, 121277, 0, 917777, 917776, 917775, - 69644, 917781, 917780, 917779, 917778, 8668, 0, 121383, 917782, 5999, 0, - 0, 129195, 128243, 43653, 1726, 1015, 0, 127247, 0, 0, 64919, 0, 0, 0, - 128478, 0, 69791, 927, 0, 0, 42010, 0, 42021, 0, 0, 1299, 12240, 64537, - 0, 0, 0, 0, 0, 0, 69454, 0, 0, 0, 122903, 19914, 12179, 0, 2296, 0, 0, - 63832, 917773, 0, 63816, 2594, 63823, 63817, 11178, 0, 0, 0, 11265, - 68295, 0, 0, 0, 10554, 3972, 0, 121198, 0, 917766, 10816, 917764, 119608, - 74374, 917769, 11210, 93069, 8586, 3882, 8532, 120183, 1573, 128648, 0, - 69916, 0, 101051, 67719, 0, 0, 0, 0, 0, 0, 0, 128821, 119169, 0, 0, 6626, - 42763, 130034, 118884, 128613, 0, 83128, 0, 0, 0, 0, 0, 983561, 0, 0, 0, - 9171, 0, 0, 71305, 983919, 121146, 0, 101095, 128881, 119604, 126596, 0, - 0, 0, 128214, 42368, 0, 983106, 2271, 41487, 12118, 74124, 68651, 110836, - 110833, 3009, 41476, 41489, 69825, 3007, 1448, 3018, 0, 41491, 8521, - 5083, 5082, 0, 0, 8519, 0, 3014, 5081, 73926, 0, 128549, 0, 69951, 5079, - 129963, 2557, 128086, 65532, 11828, 0, 71297, 11105, 0, 0, 0, 8518, - 10779, 0, 71303, 0, 0, 42170, 110769, 0, 629, 1924, 0, 12037, 0, 5987, - 8462, 127744, 0, 63933, 69735, 110770, 128295, 63941, 67981, 5077, 0, - 10880, 64849, 5075, 0, 128152, 65075, 0, 11007, 983736, 0, 0, 0, 66684, - 72331, 3434, 72338, 1904, 0, 0, 72730, 0, 10499, 4507, 9578, 63925, 0, - 7979, 0, 9831, 66689, 0, 461, 194834, 0, 4504, 0, 0, 6325, 0, 43021, 0, - 0, 55236, 0, 0, 5177, 41324, 12055, 63831, 0, 41327, 12591, 0, 4114, 409, - 0, 0, 8948, 41325, 0, 721, 10182, 0, 71311, 0, 0, 94052, 74963, 83503, - 5998, 0, 0, 74825, 0, 12587, 0, 78571, 74889, 71328, 128955, 0, 74121, - 78570, 73499, 0, 0, 5995, 0, 42568, 0, 0, 63944, 73860, 126586, 0, 4167, - 0, 43175, 0, 74120, 0, 65076, 938, 73857, 73854, 11737, 9721, 0, 0, 0, - 11742, 0, 0, 11493, 12334, 128762, 0, 66623, 0, 9173, 0, 11978, 0, 12734, - 113750, 113741, 0, 6759, 0, 0, 0, 126222, 0, 70388, 129093, 13027, 42777, - 7683, 1167, 0, 4983, 0, 861, 0, 0, 68297, 0, 43757, 92978, 129298, - 122630, 127804, 0, 73546, 70815, 9616, 0, 0, 12816, 43759, 0, 12710, - 68674, 12721, 4101, 66185, 0, 5992, 7616, 0, 0, 12577, 0, 0, 853, 42693, - 0, 121088, 0, 0, 917915, 0, 42835, 0, 0, 0, 0, 0, 12712, 7105, 127807, - 65060, 66875, 9900, 0, 0, 0, 121482, 119265, 0, 64778, 12585, 0, 0, 0, 0, - 0, 0, 77826, 0, 4900, 125245, 0, 0, 0, 4119, 74768, 8971, 0, 0, 0, 78594, - 41132, 9245, 73060, 0, 4138, 194841, 0, 0, 0, 77827, 0, 13054, 0, 0, - 128416, 110760, 0, 0, 3948, 128878, 0, 0, 0, 1680, 0, 11861, 0, 0, - 120032, 0, 0, 0, 0, 74833, 74190, 5993, 42709, 0, 12706, 77846, 1893, 0, - 63915, 0, 0, 110744, 129826, 0, 63997, 120018, 63996, 3077, 0, 0, 1512, - 0, 12589, 41479, 0, 0, 0, 0, 11831, 120727, 122949, 41481, 0, 118912, 0, - 3090, 0, 3086, 1664, 1850, 0, 3079, 0, 0, 94080, 127140, 0, 0, 74401, 0, - 917555, 0, 0, 0, 0, 0, 11526, 63985, 5864, 0, 63992, 0, 63991, 0, 5480, - 7858, 0, 4116, 78149, 0, 0, 0, 63907, 0, 0, 126131, 63905, 119601, 0, - 983191, 0, 119666, 0, 0, 7534, 507, 91, 2042, 120775, 118596, 0, 66028, - 118811, 41844, 70680, 774, 0, 0, 0, 5994, 0, 12733, 0, 0, 0, 72297, 0, 0, - 0, 0, 6026, 0, 0, 0, 162, 0, 125247, 78151, 78152, 983590, 92709, 0, - 68304, 0, 0, 0, 66658, 0, 0, 0, 0, 121511, 2226, 121512, 129349, 10492, - 0, 121510, 0, 43119, 0, 0, 0, 66192, 0, 0, 4899, 12729, 0, 0, 0, 0, 4103, - 0, 129842, 77851, 69429, 129046, 0, 12859, 70087, 0, 101580, 0, 0, 0, 0, - 0, 0, 65264, 5146, 0, 194694, 71684, 0, 0, 983652, 983863, 78924, 71688, - 78463, 5147, 125019, 0, 74524, 71682, 128435, 0, 194692, 5991, 3445, 0, - 4976, 66193, 0, 0, 0, 0, 128309, 128594, 129819, 69579, 0, 63855, 0, - 10138, 0, 0, 8897, 0, 75027, 0, 120931, 77862, 65836, 0, 0, 77860, 0, 0, - 1123, 4124, 41553, 77903, 0, 71680, 121386, 398, 0, 129035, 41551, 0, 0, - 0, 41550, 9970, 0, 93062, 42392, 1305, 78901, 0, 129292, 0, 7346, 41464, - 0, 0, 0, 41465, 983567, 8528, 9149, 0, 63955, 165, 3024, 11852, 119163, - 0, 9093, 0, 9147, 0, 0, 110989, 9148, 0, 4096, 53, 8296, 0, 71352, 0, - 9594, 0, 0, 63952, 0, 10997, 0, 0, 5805, 0, 0, 129777, 42176, 71455, - 74601, 129604, 10591, 0, 92852, 0, 0, 0, 0, 0, 0, 92475, 0, 0, 42379, 0, - 0, 9220, 0, 121425, 0, 0, 4132, 0, 0, 11239, 0, 0, 74837, 0, 66408, 0, - 8055, 0, 0, 0, 63962, 74042, 8924, 43123, 5988, 0, 63969, 0, 42718, 8788, - 1357, 77872, 65743, 0, 8774, 0, 0, 0, 0, 92748, 120598, 128234, 9564, 0, - 0, 119124, 0, 121241, 110983, 92975, 3121, 0, 0, 0, 70081, 0, 0, 0, 0, 0, - 64851, 0, 0, 73085, 119532, 0, 0, 0, 0, 1198, 69293, 66708, 64619, 0, - 64663, 93991, 0, 0, 2101, 1398, 0, 92554, 0, 0, 92684, 11406, 101588, - 12127, 66998, 840, 0, 0, 7101, 120938, 0, 0, 12880, 0, 43104, 0, 0, 0, - 2117, 0, 0, 0, 0, 123023, 0, 0, 7769, 129867, 92413, 0, 0, 100695, 0, - 40986, 83117, 0, 0, 4127, 0, 0, 129034, 0, 0, 0, 70738, 0, 129466, 0, 0, - 0, 0, 119081, 0, 10581, 0, 4533, 0, 128941, 6490, 0, 12038, 0, 0, 68225, - 0, 0, 69704, 0, 1948, 119007, 129607, 101586, 0, 0, 0, 120802, 0, 9494, - 0, 0, 0, 4843, 0, 74772, 4098, 0, 0, 0, 3436, 0, 127279, 12817, 0, - 126607, 118678, 0, 0, 0, 74433, 0, 0, 71962, 0, 121296, 65916, 0, 0, - 121458, 0, 129107, 93815, 0, 73743, 0, 0, 983133, 67676, 0, 0, 74627, - 128928, 0, 127892, 0, 71326, 67222, 0, 75013, 92435, 12284, 128500, 0, 0, - 9613, 43425, 4526, 121415, 0, 64520, 71336, 0, 0, 55278, 10228, 64957, 0, - 0, 3807, 2081, 66640, 0, 0, 0, 0, 119269, 0, 128688, 0, 128142, 1451, 0, - 0, 4134, 0, 74847, 0, 74793, 0, 78913, 74295, 9960, 1201, 0, 12846, - 121271, 0, 11919, 64962, 0, 43739, 0, 66358, 0, 0, 0, 43679, 72284, - 72289, 0, 129523, 1253, 983870, 65766, 500, 65764, 65765, 65762, 65763, - 65760, 65761, 70334, 983867, 9821, 11702, 110630, 110631, 110628, 110629, - 128481, 0, 7533, 66717, 92500, 92305, 0, 0, 69277, 127758, 71332, 0, 0, - 0, 0, 11188, 0, 4112, 0, 0, 12890, 0, 0, 9915, 0, 68423, 0, 0, 2876, 0, - 0, 0, 0, 7382, 92415, 0, 128132, 0, 0, 0, 0, 69561, 127915, 0, 7003, 0, - 0, 7704, 0, 0, 0, 4123, 0, 0, 9977, 0, 0, 65759, 0, 0, 128266, 9808, 0, - 92611, 4126, 0, 9521, 9589, 64755, 0, 0, 0, 69948, 0, 92368, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 93814, 0, 0, 92234, 0, 10693, 0, 0, 65897, 4058, 0, 0, - 64660, 0, 0, 0, 983730, 1139, 43298, 0, 65929, 8970, 0, 9934, 0, 11023, - 128020, 42522, 0, 0, 0, 78899, 3057, 128113, 7349, 69959, 128722, 68065, - 110813, 0, 92826, 67201, 0, 0, 0, 9528, 0, 0, 0, 9102, 627, 92827, 6273, - 129496, 0, 0, 983212, 92966, 43300, 0, 983740, 11696, 92825, 1018, 65554, - 0, 74338, 0, 7645, 0, 128321, 0, 0, 0, 0, 73814, 11544, 12563, 10728, 0, - 0, 127340, 43311, 64966, 92841, 0, 0, 118946, 0, 0, 74779, 0, 185, 65085, - 74533, 0, 0, 7535, 0, 42525, 0, 9749, 41701, 6131, 0, 4117, 129062, - 126988, 0, 92429, 65693, 0, 73445, 0, 69695, 0, 0, 0, 0, 0, 0, 0, 1184, - 0, 815, 0, 0, 0, 0, 0, 71325, 0, 0, 64683, 983816, 0, 127959, 0, 0, 0, 0, - 0, 0, 0, 68166, 0, 0, 0, 0, 66799, 0, 128912, 0, 5142, 0, 69643, 0, 0, - 68367, 93975, 0, 0, 0, 123209, 124133, 0, 0, 74855, 121330, 0, 0, 0, 0, - 10940, 66030, 0, 70385, 73494, 0, 2652, 120527, 0, 129946, 0, 126508, 0, - 0, 0, 0, 0, 0, 1828, 0, 128357, 0, 8531, 0, 74799, 12324, 72434, 65238, - 68374, 0, 65573, 0, 68308, 68679, 12904, 43445, 0, 0, 0, 11247, 0, 0, - 41426, 0, 0, 0, 0, 0, 67250, 69451, 83354, 11869, 0, 0, 0, 0, 0, 0, 637, - 0, 0, 0, 121178, 0, 0, 74474, 71306, 0, 7298, 128256, 0, 0, 0, 0, 8210, - 0, 0, 0, 2046, 0, 0, 0, 70333, 0, 1506, 69926, 0, 83353, 0, 12651, 0, 0, - 11867, 12058, 120626, 72111, 7803, 0, 0, 65592, 118844, 0, 0, 355, 9719, - 0, 118961, 0, 121077, 127246, 0, 42178, 0, 69760, 42571, 0, 0, 0, 0, 0, - 0, 127176, 3178, 0, 0, 92704, 83381, 9080, 120943, 67697, 0, 121342, - 129875, 0, 71485, 0, 917837, 0, 0, 78157, 0, 0, 0, 0, 0, 71313, 0, 70710, - 128212, 0, 72238, 67858, 0, 0, 0, 0, 0, 0, 0, 10770, 118994, 0, 465, 0, - 983656, 74348, 0, 0, 0, 0, 0, 0, 0, 10930, 0, 0, 0, 119091, 69388, - 122637, 129918, 0, 0, 0, 0, 0, 10092, 0, 0, 0, 0, 119019, 1766, 11282, - 11996, 66644, 4547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120906, 4345, 0, 0, - 128947, 0, 0, 0, 0, 0, 5382, 0, 0, 118552, 0, 0, 5406, 43127, 120007, 0, - 3590, 129874, 0, 0, 0, 42016, 0, 0, 121002, 0, 7742, 0, 66562, 71323, 0, - 0, 5310, 0, 123625, 0, 43594, 0, 128260, 66723, 0, 73816, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1326, 128723, 0, 0, 74519, 0, 0, 0, 0, 71308, 0, 5410, 5783, - 0, 8403, 5400, 120526, 0, 128863, 0, 0, 0, 64412, 0, 0, 5587, 42865, - 71858, 0, 0, 129854, 0, 113785, 0, 120755, 0, 69738, 0, 74867, 10461, - 12103, 0, 0, 70701, 0, 0, 0, 0, 0, 94009, 0, 2760, 0, 8816, 41515, 0, - 11802, 0, 7585, 910, 0, 0, 0, 3658, 83386, 120525, 0, 7617, 0, 12888, 0, - 0, 64631, 0, 41514, 11097, 5703, 0, 41517, 41504, 41519, 0, 70104, 0, - 65864, 0, 120533, 0, 121037, 0, 0, 43553, 120774, 0, 0, 0, 0, 0, 1578, 0, - 43449, 0, 0, 8225, 121191, 94024, 72799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 110655, 0, 110656, 121247, 72213, 0, 110658, 0, 74997, 0, 3195, 10999, - 983570, 7897, 0, 1203, 74396, 0, 64544, 0, 0, 0, 2877, 0, 0, 0, 121112, - 0, 0, 128977, 119607, 0, 0, 0, 0, 983623, 0, 0, 0, 0, 0, 0, 0, 0, 983078, - 0, 0, 0, 9939, 0, 0, 0, 0, 0, 0, 0, 10714, 0, 0, 0, 0, 0, 67738, 0, - 74038, 0, 42897, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 11163, 0, 0, 0, 113701, - 4966, 128802, 70674, 129468, 123207, 3841, 0, 0, 983231, 77886, 0, 4972, - 0, 64699, 0, 0, 0, 0, 0, 12705, 10203, 9608, 0, 0, 11962, 121397, 0, - 1196, 67684, 0, 777, 0, 0, 65271, 0, 0, 0, 0, 64824, 983195, 0, 9454, - 63778, 8658, 0, 0, 2705, 0, 64894, 0, 0, 11986, 92636, 0, 8280, 0, 2701, - 0, 0, 0, 0, 0, 9809, 0, 0, 0, 0, 0, 63761, 1748, 0, 65719, 121078, 0, 0, - 0, 55244, 3061, 0, 63765, 63787, 0, 41520, 0, 7694, 0, 8896, 63768, - 55282, 0, 127781, 0, 0, 63807, 1591, 0, 6386, 118554, 0, 0, 0, 983200, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 68289, 0, 0, 7624, 67487, 10996, 92247, 10609, 0, - 127181, 10987, 0, 70370, 3894, 0, 0, 0, 0, 493, 0, 0, 1717, 12228, 479, - 917941, 129347, 129473, 917935, 917939, 917924, 917932, 92303, 64315, - 92170, 0, 83522, 6233, 42681, 83525, 83518, 83519, 64911, 83521, 0, 0, - 83516, 83517, 129843, 8378, 11632, 0, 0, 7323, 0, 120771, 0, 0, 0, 0, - 120904, 83526, 0, 128710, 92672, 0, 0, 0, 0, 0, 0, 0, 63806, 63800, 0, 0, - 0, 63798, 63803, 244, 11542, 0, 0, 73761, 0, 12669, 120310, 0, 0, 0, 0, - 120680, 71908, 0, 0, 8612, 0, 0, 0, 0, 0, 64662, 125056, 1360, 248, 0, - 63797, 0, 63794, 0, 7292, 983685, 63756, 42786, 74957, 0, 12663, 0, 0, 0, - 0, 0, 0, 0, 4579, 0, 0, 0, 0, 0, 0, 71130, 65545, 9602, 8623, 0, 128052, - 0, 0, 0, 0, 0, 0, 0, 659, 6098, 0, 12234, 83511, 83512, 8311, 83514, - 7669, 83508, 83509, 83510, 0, 0, 0, 0, 983951, 0, 0, 2323, 0, 2319, - 77917, 120900, 77916, 2311, 83077, 4415, 1586, 68050, 0, 128724, 83020, - 2309, 83022, 8173, 83013, 83014, 83015, 83016, 0, 83010, 69275, 83012, - 9397, 0, 9395, 9396, 9393, 9394, 9391, 9392, 9389, 6209, 9387, 9388, - 9385, 9386, 9383, 9384, 0, 0, 0, 0, 0, 11259, 0, 0, 0, 2313, 0, 119661, - 0, 0, 0, 0, 10570, 65776, 110968, 0, 83006, 83007, 11998, 83009, 83002, - 83003, 83004, 66406, 0, 128780, 83000, 11818, 9381, 9382, 9379, 9380, - 9377, 9378, 9375, 9376, 1683, 9374, 0, 9372, 0, 0, 0, 0, 127801, 0, - 42029, 11079, 0, 43451, 42032, 0, 0, 118666, 0, 5005, 0, 0, 42030, 5007, - 78828, 126210, 0, 4951, 110776, 0, 110775, 0, 43309, 121222, 92172, 0, - 92334, 0, 9548, 0, 119138, 71896, 0, 0, 0, 0, 0, 0, 65691, 65580, 64361, - 10496, 0, 0, 0, 917975, 0, 0, 41046, 0, 0, 0, 13177, 0, 64703, 0, 43499, - 3389, 10589, 0, 11208, 120719, 78395, 73964, 78393, 78392, 78391, 11314, - 8281, 113732, 113667, 113745, 9076, 8862, 69743, 41052, 78397, 64766, - 69821, 0, 0, 0, 82992, 82994, 10671, 82998, 82987, 82989, 82990, 6303, - 113664, 498, 64471, 82986, 129901, 0, 9349, 0, 0, 119343, 8031, 2249, 0, - 128999, 3231, 0, 6422, 0, 0, 119339, 2537, 78405, 41429, 78403, 78401, - 78399, 0, 0, 41433, 4719, 41431, 0, 78411, 5211, 41428, 78407, 82983, - 1772, 0, 0, 82979, 66850, 64812, 82982, 82975, 68767, 82977, 82978, 0, 0, - 0, 0, 41064, 70368, 9663, 66838, 129381, 12304, 125113, 0, 41062, 66847, - 0, 0, 41061, 70454, 0, 127187, 83049, 83050, 41509, 83054, 83045, 83046, - 83047, 83048, 0, 43184, 41507, 1958, 0, 66816, 41506, 0, 0, 0, 120717, 0, - 0, 0, 74349, 72113, 8008, 0, 0, 0, 65083, 6839, 0, 126517, 73803, 127055, - 127056, 3508, 127058, 127059, 78038, 0, 120932, 0, 6411, 128115, 0, 0, - 128832, 100930, 0, 0, 0, 0, 0, 129776, 128546, 0, 0, 120914, 0, 0, 0, 0, - 917822, 128810, 983676, 65599, 0, 9966, 12607, 4948, 128070, 0, 128149, - 0, 0, 6207, 0, 6117, 73916, 0, 0, 0, 0, 68244, 41511, 0, 129489, 127304, - 0, 121289, 0, 118618, 83031, 83032, 0, 41556, 0, 0, 0, 128571, 73504, 0, - 0, 118645, 41510, 7953, 0, 0, 41513, 0, 0, 0, 83038, 83039, 83040, 83041, - 83034, 83035, 848, 9868, 983150, 6424, 118625, 83033, 0, 0, 0, 0, 118539, - 0, 893, 64576, 13299, 0, 0, 71998, 71447, 0, 0, 0, 0, 8903, 0, 0, 0, - 8099, 0, 0, 0, 0, 0, 0, 0, 0, 113713, 0, 0, 0, 0, 0, 83027, 41483, 83029, - 83030, 83023, 83024, 69436, 64836, 194756, 41485, 194758, 194757, 194760, - 41482, 42737, 64588, 0, 127787, 0, 10014, 0, 0, 194763, 194762, 68785, - 194764, 194767, 194766, 0, 0, 0, 11377, 122634, 0, 983811, 0, 0, 0, 9776, - 0, 93824, 5215, 194750, 13227, 8758, 194751, 128744, 0, 0, 5363, 12957, - 0, 0, 129051, 129526, 6421, 0, 0, 121304, 0, 0, 0, 0, 92625, 119070, - 67895, 983962, 0, 68608, 6482, 0, 0, 11945, 0, 0, 8838, 0, 4025, 10709, - 0, 2108, 0, 73929, 0, 0, 10617, 194737, 128031, 194739, 194738, 68614, - 194740, 68611, 9924, 129952, 194744, 0, 0, 0, 3277, 0, 4947, 41055, 0, - 194722, 129930, 194724, 194723, 64626, 194725, 42266, 194727, 8371, - 194729, 127028, 12806, 41492, 0, 0, 73930, 194731, 124140, 41054, 1078, - 194735, 194734, 41057, 0, 0, 0, 0, 0, 92210, 73009, 0, 41496, 0, 9165, - 1572, 0, 129712, 0, 128635, 9215, 9330, 129809, 10032, 41745, 43183, - 6401, 5831, 0, 0, 0, 8056, 0, 65681, 92377, 0, 0, 0, 121048, 0, 118887, - 6408, 0, 0, 5661, 82972, 82973, 3603, 0, 82967, 3548, 82969, 82970, 0, - 82964, 82965, 9918, 118787, 11321, 0, 0, 0, 128992, 0, 0, 0, 0, 0, 0, - 41558, 41471, 0, 8158, 41561, 41472, 0, 0, 194672, 43762, 77927, 6701, - 41559, 1896, 66256, 66248, 194680, 5665, 0, 194681, 0, 0, 0, 74352, 0, - 5664, 127895, 194682, 12310, 5662, 194687, 194686, 73924, 1121, 82953, - 82955, 0, 74378, 0, 0, 74966, 0, 71892, 0, 69413, 194667, 8627, 194669, - 10110, 194671, 42024, 6420, 42028, 0, 10509, 2795, 73923, 0, 69231, 0, - 6275, 93957, 917927, 124972, 194655, 127786, 6423, 129733, 0, 0, 68526, - 12823, 0, 0, 42026, 42017, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12691, - 68072, 42722, 69877, 82956, 78655, 78661, 82959, 78662, 41265, 41065, - 1795, 917809, 118791, 10587, 0, 917807, 0, 194640, 0, 12946, 194641, - 71921, 194643, 9169, 70372, 194648, 194647, 68202, 194649, 73990, 65111, - 0, 748, 41067, 6234, 194651, 9990, 72795, 194652, 194629, 194628, 194631, - 194630, 67896, 194632, 917812, 3593, 82948, 82949, 82950, 82951, 82944, - 69729, 82946, 82947, 194638, 194637, 0, 581, 0, 42929, 7944, 0, 0, 0, 0, - 0, 0, 72143, 0, 10119, 6415, 42893, 0, 69702, 0, 0, 11375, 0, 0, 0, 412, - 92765, 42928, 42880, 43587, 0, 0, 0, 0, 0, 0, 122638, 0, 0, 0, 65854, - 92508, 65811, 75024, 194624, 194627, 9344, 8826, 92916, 0, 125090, 74781, - 0, 0, 129582, 0, 0, 0, 127783, 0, 0, 0, 0, 10133, 92755, 0, 0, 0, 0, - 78414, 78413, 118950, 74011, 0, 0, 121080, 0, 1908, 127378, 4918, 0, 0, - 70709, 67825, 6250, 0, 10811, 78412, 11339, 4914, 0, 0, 118971, 4917, - 70686, 0, 0, 4912, 69722, 73845, 0, 0, 129527, 0, 0, 0, 118986, 0, 0, - 74317, 0, 8319, 194714, 194717, 10960, 72196, 8305, 12573, 983620, 72193, - 0, 13202, 0, 12582, 0, 72198, 69856, 0, 0, 78598, 0, 72195, 0, 65802, - 74822, 7698, 12708, 74045, 0, 0, 70460, 4913, 127990, 0, 123539, 0, 0, - 12728, 129980, 128895, 0, 101281, 0, 130038, 0, 101283, 0, 12588, 8821, - 6153, 194705, 78900, 194707, 194710, 194709, 194712, 194711, 118854, - 194713, 651, 0, 0, 0, 0, 0, 78468, 78469, 69433, 78467, 69614, 74905, - 194695, 78461, 194697, 194696, 0, 4716, 43277, 0, 2185, 78475, 128592, - 120928, 194700, 55264, 194702, 12732, 0, 12707, 0, 0, 0, 0, 121417, 8479, - 4151, 0, 0, 0, 0, 0, 0, 0, 0, 113799, 0, 74050, 0, 0, 0, 0, 0, 129467, - 12278, 0, 129507, 0, 2700, 12576, 7842, 0, 67471, 0, 2699, 0, 0, 2985, 0, - 126475, 0, 129873, 119314, 0, 119312, 9827, 101292, 119311, 101291, - 119309, 119306, 11481, 118718, 119305, 0, 35, 78481, 78482, 66694, 78480, - 78477, 78478, 0, 0, 64257, 0, 0, 0, 78485, 78486, 78483, 4272, 0, 0, - 40965, 0, 12704, 78487, 983568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 4189, - 94108, 0, 127948, 4188, 1879, 0, 0, 0, 43743, 71974, 8873, 2279, 0, 0, 0, - 12574, 12735, 92749, 92753, 983921, 0, 0, 75001, 0, 0, 0, 12578, 12720, - 128628, 101088, 0, 12346, 128596, 101089, 0, 0, 7251, 0, 0, 118850, - 73025, 0, 0, 0, 0, 0, 12564, 66457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101131, - 0, 41564, 10976, 0, 121223, 0, 0, 10054, 9197, 120618, 0, 9012, 65737, - 74420, 0, 13215, 12730, 0, 0, 0, 0, 816, 0, 101123, 122664, 83191, 0, 0, - 92752, 101120, 4715, 94107, 94106, 71075, 0, 0, 0, 67729, 0, 307, 0, - 9585, 0, 0, 0, 101255, 0, 125267, 0, 70727, 65567, 101238, 75006, 101231, - 983909, 0, 12236, 41419, 101259, 194621, 101248, 75003, 194622, 73675, - 120957, 41421, 75005, 4462, 118569, 126599, 983911, 821, 0, 2498, 5800, - 100834, 100833, 1760, 67483, 4469, 64377, 100840, 100839, 0, 757, 1185, - 118558, 100841, 0, 10628, 100842, 68849, 100844, 43971, 100846, 100849, - 64763, 0, 7713, 0, 0, 0, 4380, 194608, 128073, 194610, 194609, 194612, - 862, 65626, 194613, 65627, 65629, 5137, 194617, 0, 0, 0, 65069, 7566, - 64688, 67143, 118592, 100823, 100822, 100825, 4748, 92228, 100826, - 100829, 42260, 129494, 64107, 0, 0, 0, 0, 128189, 0, 194604, 13137, 8775, - 127945, 123633, 194607, 0, 8410, 4454, 194585, 0, 92542, 4449, 92330, - 127064, 75022, 92761, 70664, 194589, 339, 194591, 194590, 0, 70662, 0, - 100830, 41543, 0, 0, 0, 41542, 127066, 8916, 6705, 0, 129296, 0, 0, 0, 0, - 0, 41548, 6729, 119329, 0, 7348, 0, 0, 7537, 0, 11819, 0, 0, 123624, - 71269, 0, 7344, 100808, 129073, 9780, 0, 11117, 74993, 0, 194578, 10483, - 194580, 194579, 194582, 194581, 68781, 125114, 100820, 100819, 0, 4211, - 1259, 7517, 0, 0, 194561, 70827, 194563, 194562, 641, 5219, 94034, - 194566, 11064, 194568, 0, 129820, 0, 0, 0, 0, 100812, 100811, 100814, - 100813, 100816, 100815, 100818, 100817, 100798, 100797, 41410, 100799, - 64262, 0, 41407, 75000, 0, 0, 93812, 0, 0, 72803, 74999, 78897, 0, 0, - 67675, 0, 0, 0, 0, 43647, 0, 0, 100792, 100791, 100794, 100793, 100796, - 100795, 983276, 74630, 11933, 0, 0, 41903, 67892, 11001, 100801, 42255, - 100803, 100802, 100805, 41905, 100807, 100806, 10775, 9793, 0, 0, 74452, - 0, 983063, 42535, 0, 64529, 41408, 42853, 0, 0, 42674, 118915, 0, 0, - 983807, 0, 70838, 0, 0, 0, 64506, 0, 66738, 4747, 100783, 69844, 100785, - 5832, 0, 0, 5141, 42600, 124147, 0, 0, 0, 0, 0, 93790, 0, 7657, 0, 71132, - 74137, 0, 128362, 73682, 73681, 859, 0, 0, 0, 6059, 126985, 55235, 0, 0, - 0, 0, 0, 100787, 11488, 72838, 100788, 0, 100790, 10558, 0, 124144, - 118646, 126090, 71069, 0, 0, 1788, 0, 0, 0, 0, 119571, 92822, 9028, 0, - 69234, 73665, 0, 9905, 73556, 41242, 70086, 0, 74109, 100765, 100764, - 100767, 100766, 70830, 83184, 70082, 3940, 0, 43754, 0, 128188, 8665, 0, - 0, 0, 1653, 100775, 42406, 100777, 100780, 70825, 120523, 0, 8815, 0, - 65046, 0, 42445, 0, 11180, 119318, 119315, 68454, 42485, 0, 0, 8211, - 42293, 983602, 0, 0, 0, 0, 65385, 100771, 42332, 100773, 78431, 78432, - 78423, 78430, 78420, 10022, 65387, 78419, 65384, 0, 0, 0, 65386, 0, - 11248, 0, 43198, 64751, 0, 0, 0, 0, 0, 0, 101102, 7363, 0, 0, 119323, - 119324, 100752, 100751, 0, 119320, 0, 983632, 0, 8237, 0, 0, 0, 0, 0, 0, - 9914, 0, 100763, 100762, 120009, 6351, 119993, 92740, 68766, 0, 120010, - 41243, 0, 74108, 11467, 120165, 119998, 4358, 0, 6353, 0, 0, 0, 93045, - 1710, 0, 0, 92237, 0, 49, 73871, 120005, 78671, 0, 78672, 9741, 78443, - 78444, 78441, 43443, 78439, 78440, 69244, 78438, 3470, 0, 0, 92814, 0, 0, - 78445, 0, 1072, 78457, 78452, 78454, 74230, 78451, 78447, 78449, 1080, 0, - 74100, 0, 1101, 68404, 78458, 78459, 71082, 0, 1086, 1869, 0, 0, 0, - 65458, 0, 0, 41988, 0, 1091, 0, 7977, 0, 66992, 0, 0, 0, 92758, 0, 0, 0, - 0, 0, 71255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64582, 0, 0, 70794, 0, 120989, - 128495, 74106, 0, 66883, 0, 0, 0, 0, 0, 0, 0, 92553, 43752, 110592, 0, - 71249, 120886, 0, 0, 0, 0, 6063, 100857, 101221, 917995, 6053, 74096, 0, - 0, 74169, 13100, 0, 917999, 0, 71081, 0, 70387, 6055, 7800, 4279, 8490, - 120114, 120111, 64786, 8602, 120110, 83389, 92204, 0, 0, 74961, 0, - 120117, 120118, 120099, 120100, 65087, 64402, 3674, 120096, 0, 120094, - 120107, 118624, 120105, 10107, 42159, 42870, 120101, 69632, 0, 0, 43281, - 127078, 0, 74098, 0, 0, 126497, 74099, 129056, 0, 0, 0, 121123, 5847, - 125258, 0, 0, 0, 0, 0, 66592, 64469, 71698, 19966, 0, 42561, 0, 129170, - 66854, 8120, 75042, 0, 0, 0, 0, 0, 0, 126068, 8369, 0, 0, 122912, 3369, - 0, 121094, 0, 0, 69238, 10495, 121365, 0, 557, 9457, 0, 0, 121054, 73880, - 127220, 0, 74937, 74094, 0, 0, 119001, 92171, 127219, 128175, 127939, - 120424, 0, 127214, 2109, 67893, 127211, 69656, 127217, 10604, 127215, 0, - 0, 0, 129727, 126561, 0, 0, 0, 0, 1618, 0, 0, 83175, 10430, 0, 0, 13063, - 917585, 0, 92982, 113666, 0, 78390, 83489, 12060, 0, 113669, 0, 6329, 0, - 0, 0, 74395, 2707, 8309, 0, 127054, 78398, 0, 2697, 0, 78396, 127057, - 2695, 0, 0, 68334, 0, 0, 0, 72325, 2693, 74091, 0, 0, 2703, 113729, - 70283, 41918, 983169, 127542, 8687, 127543, 12178, 43361, 92540, 64075, - 110705, 5248, 110703, 120538, 6427, 0, 0, 0, 0, 110710, 0, 74990, 74989, - 70703, 127031, 0, 9873, 0, 0, 0, 64762, 2053, 0, 6591, 9340, 0, 1589, 0, - 296, 67712, 128315, 12766, 118931, 74370, 120417, 2414, 128068, 43829, - 111202, 74836, 0, 12579, 0, 12575, 6416, 5656, 0, 13262, 65590, 5299, - 983702, 0, 5449, 1252, 0, 78404, 69748, 74369, 65373, 5295, 0, 121066, - 1223, 1642, 78408, 0, 12158, 5303, 0, 120546, 41413, 3212, 127025, 3211, - 74810, 41425, 127029, 0, 74450, 9728, 0, 10924, 74778, 6636, 73552, - 129884, 0, 0, 129882, 9519, 0, 0, 129106, 101110, 68780, 0, 0, 0, 119182, - 0, 12104, 77942, 77951, 9004, 0, 74249, 10230, 0, 0, 0, 77947, 0, 69679, - 121475, 9890, 125049, 12971, 0, 92556, 0, 67903, 70051, 983924, 0, 0, - 9635, 12600, 0, 0, 0, 118900, 6469, 0, 101113, 65304, 4679, 101114, - 64300, 64867, 6531, 101118, 101099, 101098, 92813, 101100, 42916, 0, 0, - 0, 0, 0, 0, 4445, 72296, 0, 11533, 0, 3416, 124112, 0, 0, 0, 78566, 0, 0, - 101091, 92815, 101093, 5447, 72140, 70752, 101097, 101096, 0, 0, 0, - 64448, 0, 43920, 70677, 0, 6232, 101101, 101104, 101103, 43608, 101105, - 101108, 6538, 4335, 0, 3941, 74986, 11061, 0, 74988, 74987, 0, 12155, - 128278, 0, 0, 0, 0, 74578, 0, 65832, 0, 129459, 70789, 0, 125050, 0, 0, - 350, 10951, 101081, 509, 101083, 101086, 101085, 0, 0, 0, 917540, 0, - 100905, 110970, 12162, 64741, 0, 9354, 0, 70802, 100901, 2496, 11516, - 944, 128238, 0, 0, 1438, 0, 0, 120185, 70785, 1220, 917952, 93844, 0, 0, - 5008, 42630, 70787, 101087, 2229, 68206, 564, 0, 312, 0, 0, 0, 70797, - 8877, 269, 0, 128065, 9617, 0, 0, 100910, 0, 0, 10862, 0, 0, 41416, 0, - 4173, 0, 0, 0, 1906, 983854, 41418, 74073, 101068, 101067, 41415, 69622, - 9582, 0, 64287, 0, 0, 11428, 1730, 0, 0, 19918, 10469, 101076, 101079, - 68088, 0, 101080, 72342, 0, 129692, 0, 6129, 0, 0, 0, 0, 7874, 0, 0, - 11206, 13136, 118529, 129305, 0, 64374, 74925, 0, 73892, 0, 101073, - 101072, 101075, 74960, 9228, 101054, 101057, 101056, 5240, 9811, 0, - 101060, 129718, 0, 0, 74079, 65873, 0, 0, 0, 9501, 0, 68081, 72808, - 65465, 64654, 7467, 0, 0, 83460, 10040, 0, 3096, 0, 101053, 101052, - 68820, 83461, 0, 0, 0, 0, 0, 0, 83377, 0, 68801, 0, 101062, 101061, - 101064, 101063, 0, 8637, 70741, 0, 77983, 77969, 11471, 43554, 0, 77968, - 0, 0, 0, 2426, 12042, 0, 0, 0, 3961, 12115, 129633, 0, 77972, 64561, 0, - 4981, 74644, 129558, 0, 0, 42686, 77976, 128776, 64686, 0, 77958, 7589, - 0, 0, 3237, 0, 68215, 0, 8541, 127157, 71067, 120174, 0, 0, 0, 0, 0, - 43555, 0, 0, 10060, 111261, 100917, 0, 0, 0, 64877, 0, 0, 8614, 65220, - 41493, 0, 0, 0, 43780, 0, 0, 70689, 0, 0, 0, 0, 0, 0, 4012, 10395, 0, 0, - 111253, 126511, 111254, 125051, 695, 739, 696, 7611, 0, 42755, 68421, - 9227, 7506, 7510, 67493, 691, 738, 7511, 7512, 7515, 7501, 688, 41847, - 690, 2548, 737, 974, 43386, 0, 0, 0, 0, 0, 0, 65860, 0, 7051, 69777, - 4682, 0, 983096, 6406, 4685, 0, 0, 10347, 4680, 6341, 0, 0, 92607, 74325, - 0, 123555, 0, 0, 0, 0, 0, 0, 43505, 92468, 11718, 42373, 11714, 0, 0, - 129567, 11717, 0, 10594, 129732, 11712, 122962, 0, 10967, 0, 0, 0, 66632, - 118647, 0, 0, 0, 1735, 0, 11134, 2363, 983136, 0, 0, 70695, 128032, 0, - 7491, 7495, 7580, 7496, 7497, 7584, 121478, 127853, 0, 0, 70025, 0, 8498, - 0, 8949, 3065, 0, 0, 0, 0, 0, 0, 11713, 0, 64939, 0, 6418, 4543, 0, 0, 0, - 74800, 0, 0, 0, 0, 0, 0, 0, 12282, 3165, 0, 0, 64556, 0, 9238, 0, 68063, - 0, 0, 0, 65438, 0, 128525, 0, 119268, 0, 0, 12900, 67489, 10950, 0, 0, 0, - 41400, 126636, 119664, 0, 42232, 0, 1744, 0, 41402, 0, 0, 0, 41399, 0, - 125028, 0, 0, 12690, 0, 0, 43672, 0, 0, 0, 100870, 11315, 0, 278, 121204, - 41405, 129345, 0, 10077, 129650, 70667, 0, 0, 0, 68210, 0, 0, 11189, - 70657, 0, 0, 0, 7934, 0, 93829, 120940, 0, 0, 122971, 0, 0, 0, 6413, - 6550, 0, 1940, 2809, 43637, 70045, 0, 0, 10678, 0, 0, 0, 129701, 78804, - 6403, 6556, 78803, 0, 0, 123557, 0, 0, 0, 123553, 0, 3742, 74408, 3959, - 0, 0, 917969, 123565, 0, 128024, 0, 123558, 127956, 0, 0, 0, 6855, 4676, - 983049, 9210, 0, 78143, 983922, 0, 78168, 983100, 11540, 43546, 6692, 0, - 0, 0, 0, 9083, 0, 0, 78144, 128515, 0, 9677, 0, 70867, 74175, 0, 74070, - 0, 0, 365, 0, 43027, 0, 0, 128236, 0, 119574, 70284, 13151, 0, 0, 127935, - 127950, 544, 13249, 119018, 0, 120846, 0, 0, 73671, 65339, 73000, 2211, - 0, 0, 0, 0, 0, 0, 0, 0, 128037, 0, 0, 0, 0, 0, 0, 0, 127188, 92977, - 69708, 9638, 0, 100878, 0, 0, 0, 74545, 128820, 128819, 75062, 128963, 0, - 0, 0, 11264, 43994, 0, 0, 0, 1311, 0, 0, 0, 0, 13068, 0, 0, 78164, 78155, - 0, 949, 0, 0, 0, 78176, 69709, 78177, 63828, 0, 0, 118629, 70282, 0, 0, - 0, 64822, 0, 6530, 983275, 0, 70493, 0, 129325, 0, 0, 4431, 118839, - 127490, 983760, 73667, 127986, 0, 10336, 10400, 0, 0, 92959, 0, 0, 0, - 42270, 128880, 6428, 0, 0, 0, 0, 43455, 0, 43526, 100888, 12835, 129501, - 9493, 0, 0, 11793, 0, 127897, 74394, 0, 10653, 0, 0, 0, 0, 6560, 7016, - 74274, 983627, 43556, 3929, 123615, 6614, 2768, 0, 65609, 0, 11811, - 129696, 0, 118615, 127513, 0, 6554, 0, 6305, 66283, 4675, 118826, 78552, - 0, 0, 74361, 0, 0, 68108, 0, 0, 92232, 0, 93022, 7392, 8230, 9365, - 983742, 0, 0, 0, 0, 42925, 0, 0, 122965, 0, 229, 43834, 119884, 0, 43552, - 119881, 119880, 119883, 119882, 119877, 119876, 119879, 119878, 119873, - 119872, 119875, 119874, 0, 0, 0, 0, 0, 66352, 0, 0, 0, 128663, 0, 12239, - 0, 0, 10432, 12097, 0, 194815, 1233, 78179, 0, 127200, 0, 66395, 0, 0, - 129504, 0, 0, 92342, 0, 2388, 92555, 119868, 119871, 119870, 119865, 895, - 92668, 119866, 64889, 7143, 119863, 119862, 0, 0, 69983, 0, 74376, 3053, - 2168, 0, 2047, 0, 0, 0, 121279, 67985, 194801, 92600, 194803, 194802, - 194805, 194804, 194807, 194806, 129134, 194808, 0, 0, 0, 10473, 129331, - 0, 194810, 129806, 194812, 129813, 194814, 194813, 123195, 43528, 69673, - 194791, 0, 194793, 1912, 120779, 10306, 10370, 0, 0, 8867, 10250, 10258, - 10274, 1635, 120152, 0, 0, 0, 129379, 0, 0, 9919, 120148, 559, 128157, - 41825, 127975, 92989, 0, 74016, 194781, 6542, 41957, 7318, 124126, 0, - 41956, 65749, 65750, 65751, 121323, 64487, 0, 0, 10223, 42062, 100640, - 101195, 125044, 3668, 65754, 43560, 12226, 0, 93973, 194784, 41959, - 194786, 194785, 194788, 43618, 65747, 10937, 2962, 0, 2953, 10062, 65745, - 71457, 8921, 66013, 129370, 0, 194769, 194768, 43409, 194770, 2949, - 194772, 194775, 194774, 2958, 194776, 74868, 2300, 2951, 120061, 0, - 120043, 194778, 0, 120051, 194779, 120056, 120065, 70798, 120048, 0, - 120062, 120055, 71989, 100668, 0, 0, 71985, 0, 71992, 70796, 127818, 0, - 0, 64890, 0, 43630, 11336, 799, 0, 10276, 10308, 10372, 917541, 0, 0, - 10252, 10260, 68220, 55284, 125225, 0, 10384, 0, 0, 0, 64523, 129744, 0, - 65736, 0, 0, 0, 0, 0, 0, 0, 124912, 43549, 65738, 42150, 65739, 0, 78195, - 10288, 10320, 0, 10596, 129829, 67673, 65045, 121283, 78198, 2049, 10098, - 0, 122904, 127943, 10264, 10280, 10312, 10376, 7013, 0, 69504, 0, 0, - 66375, 0, 4862, 0, 6537, 0, 128335, 3914, 92178, 93976, 9065, 64816, 0, - 72218, 73026, 0, 0, 72139, 4694, 11420, 4690, 0, 0, 983211, 4693, 0, 0, - 0, 4688, 0, 0, 128892, 0, 8238, 3110, 0, 983939, 0, 6528, 0, 0, 0, 218, - 0, 1520, 129577, 70039, 0, 983594, 0, 120167, 78167, 10088, 6548, 100786, - 0, 0, 0, 8888, 0, 124954, 0, 0, 126593, 68876, 0, 0, 0, 0, 0, 0, 0, 4689, - 43541, 77954, 120157, 0, 120156, 78810, 120163, 0, 0, 0, 0, 78121, 0, 0, - 11450, 0, 71900, 92613, 0, 121317, 74622, 128720, 9244, 0, 0, 127763, 0, - 0, 0, 0, 0, 0, 71084, 0, 0, 0, 0, 10513, 0, 0, 0, 52, 119178, 0, 0, - 93961, 0, 0, 4812, 0, 0, 0, 0, 0, 0, 128425, 0, 6850, 0, 77959, 10170, - 120450, 6544, 0, 0, 69782, 121517, 0, 0, 65258, 10369, 0, 1585, 74014, - 10249, 422, 1500, 2036, 986, 0, 64394, 69502, 5599, 917981, 2494, 0, 0, - 74021, 983896, 78203, 127808, 0, 72871, 65102, 8961, 74305, 10243, 10245, - 128170, 0, 0, 0, 0, 0, 2508, 129591, 120440, 0, 120439, 0, 0, 0, 0, 0, 0, - 64533, 983187, 0, 0, 74008, 0, 0, 43375, 0, 2504, 0, 121313, 0, 983941, - 6943, 0, 5859, 100677, 0, 0, 72873, 983945, 0, 0, 983923, 92390, 2753, - 1936, 2153, 67701, 2751, 12662, 2763, 8953, 0, 10731, 0, 7052, 0, 0, 0, - 0, 119899, 0, 66675, 0, 119897, 0, 71053, 0, 119903, 0, 67829, 7899, - 119901, 71119, 43798, 7072, 119902, 122898, 11260, 0, 71059, 0, 0, 212, - 0, 12350, 0, 0, 0, 0, 0, 128402, 2759, 0, 0, 93064, 0, 0, 0, 1291, 0, - 195065, 121318, 119911, 0, 119910, 0, 12062, 0, 121216, 0, 129124, - 121044, 120611, 8246, 128874, 0, 0, 0, 0, 0, 73962, 0, 0, 43524, 0, - 64426, 0, 0, 0, 0, 65664, 6693, 0, 0, 8674, 0, 128812, 0, 11846, 70690, - 121461, 69395, 4811, 0, 5986, 0, 3046, 74480, 5985, 0, 0, 0, 0, 12187, - 83148, 71041, 5984, 0, 93817, 4393, 126264, 120206, 917599, 0, 0, 0, - 93806, 93805, 0, 3491, 0, 67146, 0, 93819, 0, 72428, 0, 0, 0, 124968, - 41284, 126228, 0, 0, 41287, 0, 100689, 0, 0, 92189, 0, 0, 219, 120874, 0, - 0, 0, 68485, 119672, 43241, 0, 7147, 73554, 0, 0, 0, 0, 0, 64610, 11804, - 0, 7149, 64808, 0, 0, 0, 92301, 73690, 0, 5253, 0, 0, 0, 0, 129045, - 983596, 11098, 68433, 0, 120484, 111009, 0, 0, 0, 0, 0, 70801, 100779, 0, - 128198, 9604, 0, 130036, 0, 0, 118941, 64392, 0, 118684, 0, 0, 41974, - 126262, 0, 0, 0, 129818, 0, 129833, 0, 0, 0, 0, 0, 983243, 5308, 0, 290, - 0, 125278, 128382, 2792, 0, 0, 120521, 0, 126237, 0, 126099, 0, 0, 0, 0, - 128503, 0, 0, 72816, 0, 0, 0, 92671, 0, 195061, 42646, 7606, 2591, 73896, - 0, 43513, 64482, 0, 0, 65270, 0, 0, 983701, 9112, 0, 113763, 9490, 0, 0, - 0, 0, 0, 9071, 0, 0, 0, 0, 74607, 0, 2535, 65504, 43602, 0, 0, 71256, - 2248, 0, 123147, 11845, 11006, 92315, 7807, 8073, 0, 10629, 0, 74088, 0, - 10823, 0, 113762, 8762, 0, 69689, 123536, 43969, 65047, 10737, 3463, - 67467, 129585, 66645, 0, 4815, 0, 0, 12345, 983761, 0, 5195, 129808, 0, - 66639, 0, 0, 66941, 0, 92759, 92385, 1262, 0, 6561, 19939, 0, 0, 100772, - 123160, 69269, 0, 100774, 0, 0, 0, 0, 0, 0, 67511, 0, 0, 0, 0, 0, 0, - 5702, 3655, 0, 8430, 0, 68807, 0, 0, 121137, 0, 0, 5254, 0, 0, 124917, 0, - 119107, 5129, 0, 70816, 0, 92280, 5614, 0, 0, 11720, 0, 11721, 70804, - 4798, 0, 120541, 66038, 4793, 67851, 7352, 0, 0, 0, 0, 917600, 0, 300, 0, - 0, 128575, 92660, 0, 0, 2562, 70156, 120856, 0, 0, 92738, 0, 0, 127820, - 71093, 0, 127969, 128221, 0, 3424, 93843, 0, 0, 7074, 70873, 128519, 0, - 0, 10832, 0, 0, 69852, 72430, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1215, 0, - 5744, 0, 66440, 0, 0, 0, 42881, 0, 8980, 118988, 67861, 8844, 7433, 0, 0, - 4278, 124925, 0, 0, 70821, 9312, 4348, 0, 128401, 65946, 0, 7087, 5255, - 0, 661, 0, 0, 0, 0, 0, 0, 0, 121009, 73694, 0, 123154, 0, 73688, 0, - 127179, 3621, 83325, 66666, 72968, 0, 6562, 12928, 0, 73991, 0, 0, 11383, - 0, 0, 65588, 120739, 0, 0, 0, 0, 0, 0, 0, 0, 11436, 2070, 64, 110824, 0, - 10291, 10323, 10387, 0, 0, 0, 42008, 9708, 42710, 0, 42011, 0, 92164, 0, - 0, 1702, 1240, 128383, 6286, 9689, 111080, 0, 0, 0, 1765, 0, 0, 92373, 0, - 0, 0, 8401, 72991, 42014, 0, 67237, 0, 0, 0, 0, 0, 0, 0, 70819, 0, 0, 0, - 0, 12667, 0, 0, 10147, 0, 127568, 126483, 72812, 0, 0, 0, 0, 123139, - 128968, 0, 64947, 0, 0, 0, 0, 10435, 11462, 0, 7084, 0, 0, 0, 0, 0, - 126084, 0, 66662, 0, 0, 0, 0, 125134, 0, 0, 77990, 263, 983747, 41288, - 127953, 0, 78387, 74340, 70313, 129140, 0, 0, 0, 42022, 71265, 0, 0, 0, - 0, 0, 0, 42020, 123146, 0, 6992, 42019, 0, 41290, 0, 12295, 126233, - 71304, 0, 120984, 71300, 120631, 5954, 64931, 69385, 100699, 198, 68453, - 78129, 0, 121351, 0, 70818, 13165, 7107, 0, 42804, 678, 72850, 118960, 0, - 72985, 42806, 42808, 0, 0, 2097, 0, 120560, 70823, 0, 0, 3892, 68632, 0, - 6712, 917959, 0, 0, 0, 0, 123158, 69954, 0, 497, 12100, 5953, 92667, - 7796, 0, 43254, 0, 0, 11072, 5952, 1281, 43747, 0, 69380, 10677, 0, 0, 0, - 1859, 0, 72856, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65199, 1738, 0, - 122911, 0, 0, 0, 11101, 0, 0, 0, 0, 127002, 69651, 4436, 194683, 73984, - 6860, 70305, 64872, 128296, 0, 0, 0, 121377, 0, 6862, 0, 6861, 983109, 0, - 119109, 0, 70826, 319, 0, 43479, 73001, 0, 0, 12849, 0, 7640, 71083, - 9673, 0, 0, 0, 92670, 0, 92665, 113717, 41422, 0, 100708, 74941, 3772, 0, - 120660, 5011, 0, 0, 126587, 111315, 0, 0, 6677, 111312, 0, 41427, 64419, - 129445, 92262, 0, 70799, 0, 0, 0, 6106, 0, 41271, 6760, 983758, 4534, - 41270, 128876, 0, 0, 119561, 0, 0, 3671, 8976, 123177, 0, 41275, 0, - 128084, 55261, 0, 42013, 0, 568, 0, 41273, 0, 0, 6728, 0, 9715, 0, 0, - 121058, 74820, 0, 92268, 0, 194564, 11191, 43688, 128023, 0, 0, 0, - 126266, 0, 0, 0, 11958, 11165, 0, 125087, 0, 0, 66336, 127944, 0, 0, 0, - 0, 42858, 11789, 72878, 5557, 0, 69444, 7300, 0, 9467, 5558, 64486, - 43844, 0, 0, 6706, 10146, 0, 127185, 64566, 0, 0, 0, 0, 0, 0, 0, 4546, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 64528, 123136, 6307, 128966, 0, 7544, 0, 43469, - 111317, 0, 10152, 0, 65091, 0, 129047, 0, 0, 66652, 0, 0, 0, 0, 64823, - 5559, 0, 70711, 6702, 5556, 0, 0, 0, 0, 0, 11166, 0, 0, 5506, 0, 1911, - 73021, 0, 12598, 8845, 66698, 0, 73012, 123145, 73496, 2098, 0, 0, 0, - 66622, 194678, 0, 0, 0, 9898, 0, 0, 7552, 0, 0, 0, 7223, 65723, 0, 0, 0, - 7024, 65728, 127155, 1210, 0, 65175, 10184, 65726, 43654, 0, 0, 0, 38, - 65729, 66669, 0, 917948, 0, 0, 0, 0, 119837, 0, 74233, 73018, 119843, - 42860, 111301, 92576, 65721, 65722, 0, 0, 0, 0, 68843, 0, 68850, 0, - 92388, 92267, 128536, 65577, 42967, 0, 127518, 11650, 5013, 92663, 68810, - 92568, 118914, 6613, 74371, 0, 0, 122985, 0, 64714, 71479, 0, 983797, - 12120, 0, 0, 43124, 0, 0, 78037, 69263, 0, 126219, 0, 0, 1837, 125086, 0, - 0, 0, 127210, 4952, 65718, 64405, 5504, 65720, 65714, 65715, 65716, - 10403, 127005, 0, 41449, 0, 74028, 72019, 0, 119234, 1127, 455, 0, 0, - 72860, 3483, 0, 1989, 0, 69678, 9104, 0, 65375, 0, 0, 0, 1864, 0, 72810, - 8107, 2540, 0, 0, 11257, 128807, 119576, 0, 120999, 0, 73501, 8604, 0, 0, - 0, 0, 128270, 0, 0, 3115, 0, 10106, 120498, 118842, 101136, 0, 9631, 0, - 0, 0, 0, 0, 0, 0, 258, 129079, 0, 0, 0, 92292, 0, 70699, 0, 11478, 0, - 129640, 11522, 0, 8549, 0, 128430, 0, 0, 0, 0, 0, 0, 123140, 0, 0, 0, - 9221, 12590, 73048, 0, 0, 0, 67741, 111294, 12619, 0, 10154, 111266, - 74439, 2039, 0, 7446, 0, 111276, 10974, 458, 72831, 0, 0, 0, 11916, 0, 0, - 69671, 0, 121057, 12288, 0, 111288, 0, 111289, 983177, 0, 128199, 13080, - 0, 67828, 6610, 6030, 8059, 7508, 123170, 0, 0, 0, 0, 41278, 129393, - 118691, 128192, 41277, 64658, 984002, 101278, 6625, 983160, 19904, 0, 0, - 0, 0, 0, 0, 833, 0, 6369, 0, 0, 42664, 0, 0, 0, 0, 129765, 0, 6913, 933, - 1341, 68828, 6720, 0, 0, 983604, 0, 0, 7405, 128025, 0, 0, 0, 0, 0, 0, 0, - 70704, 0, 0, 0, 0, 9716, 0, 0, 0, 70719, 0, 0, 0, 0, 72862, 70687, 0, - 93987, 0, 0, 0, 70721, 9573, 0, 0, 111245, 83225, 83226, 6949, 126482, - 74061, 83222, 83223, 83224, 0, 19962, 83219, 83220, 0, 111233, 0, 42830, - 0, 111234, 74236, 66276, 0, 546, 72861, 0, 70661, 0, 472, 11083, 10319, - 10383, 917971, 0, 83202, 83203, 3602, 83206, 41182, 83199, 83200, 69796, - 3790, 0, 10271, 10287, 684, 0, 0, 0, 83214, 4592, 83216, 83217, 83210, - 11963, 43620, 83213, 0, 0, 83208, 83209, 0, 92623, 128559, 3415, 0, - 121267, 0, 0, 123151, 43447, 0, 92212, 0, 418, 0, 0, 10295, 10327, 10391, - 0, 83189, 83190, 83192, 83194, 83185, 83186, 83187, 83188, 120879, 0, - 41446, 70700, 118652, 0, 120809, 10599, 66892, 0, 0, 0, 0, 0, 129184, - 11437, 0, 0, 0, 0, 0, 0, 12624, 0, 41185, 72865, 69439, 8159, 0, 11686, - 71478, 65224, 0, 4655, 0, 0, 92183, 0, 10343, 10407, 0, 0, 0, 111221, 0, - 0, 0, 94057, 68201, 129574, 0, 983572, 72156, 42792, 5743, 10424, 0, 0, - 0, 0, 0, 8875, 111225, 0, 917991, 13117, 12847, 4651, 118917, 0, 962, 0, - 0, 2242, 42564, 0, 1582, 0, 5508, 0, 0, 0, 10801, 123602, 118798, 73705, - 0, 66911, 10439, 66891, 0, 0, 7860, 0, 906, 917985, 0, 6405, 64722, 0, - 83266, 64694, 83268, 917990, 1153, 83263, 64788, 83265, 0, 12626, 83260, - 83261, 9964, 0, 0, 4642, 66574, 127886, 0, 0, 0, 0, 0, 9008, 100847, 0, - 0, 0, 83248, 917976, 917993, 123173, 42842, 83244, 83245, 83247, 83239, - 83240, 83241, 83242, 0, 11335, 92661, 83238, 3920, 0, 0, 0, 83255, 83256, - 41967, 83258, 83251, 83252, 83253, 8920, 0, 0, 83249, 83250, 0, 0, 43919, - 0, 0, 0, 0, 128021, 0, 68113, 65196, 0, 0, 128472, 0, 10111, 64875, 0, - 83491, 43998, 83232, 83233, 83234, 70691, 83228, 42149, 83230, 68508, 0, - 0, 0, 0, 0, 0, 0, 4110, 66005, 74034, 0, 0, 0, 66703, 0, 0, 983158, 6025, - 69242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70733, 0, 983043, 0, 73513, 0, 68817, - 0, 0, 0, 0, 0, 0, 43286, 0, 68765, 0, 0, 0, 0, 129871, 65144, 0, 0, - 83236, 65840, 0, 0, 10081, 0, 0, 983912, 0, 0, 0, 127394, 65882, 0, - 128758, 0, 0, 3605, 10985, 0, 0, 128872, 93972, 1745, 0, 73835, 0, 0, 0, - 0, 0, 0, 8806, 7023, 0, 0, 0, 70702, 70304, 0, 0, 0, 0, 0, 0, 0, 0, 348, - 10089, 0, 9017, 0, 0, 0, 0, 0, 0, 0, 67465, 0, 42515, 0, 0, 0, 0, 5391, - 983240, 110576, 0, 0, 5561, 0, 9429, 0, 67150, 7933, 5562, 0, 0, 0, 0, - 78039, 0, 0, 0, 0, 3979, 71248, 0, 0, 0, 68847, 0, 0, 118847, 65847, - 68836, 68838, 0, 10585, 0, 92676, 7334, 0, 0, 0, 831, 0, 0, 10716, 0, - 121325, 0, 12218, 0, 6939, 70697, 65042, 0, 0, 916, 0, 0, 11968, 0, - 122641, 5563, 0, 0, 128830, 5560, 41212, 41774, 0, 4497, 0, 0, 0, 9039, - 70678, 41776, 0, 8716, 3567, 119252, 0, 0, 74260, 0, 93954, 0, 0, 100827, - 0, 128879, 70072, 68355, 68357, 0, 0, 8634, 0, 0, 4209, 120702, 68832, - 65879, 68825, 68819, 68822, 0, 5679, 68813, 68815, 68811, 68812, 64697, - 5678, 11821, 68802, 93969, 0, 0, 0, 0, 70114, 0, 0, 0, 0, 0, 0, 0, 0, - 7782, 0, 0, 0, 0, 129977, 65711, 65712, 1216, 0, 69409, 5792, 0, 0, 0, 0, - 0, 12244, 0, 5683, 0, 120895, 121336, 43448, 70670, 0, 0, 5682, 10242, - 75043, 74520, 5680, 917568, 10001, 0, 0, 1449, 10241, 0, 70708, 0, 0, - 83180, 83182, 83183, 8584, 83176, 5567, 83178, 83179, 0, 5564, 42886, - 42884, 42882, 5565, 119022, 120881, 0, 65708, 65709, 5566, 0, 65704, - 65705, 11904, 42875, 0, 42873, 5942, 0, 0, 10361, 10425, 65697, 65698, - 65699, 0, 66598, 0, 64664, 10647, 78702, 78703, 78690, 78700, 0, 65701, - 1934, 0, 0, 0, 78710, 0, 78706, 78709, 6087, 78705, 78716, 78719, 78711, - 8043, 8950, 65694, 64485, 0, 10457, 0, 78724, 78725, 78722, 72332, 78720, - 78721, 0, 65515, 0, 10035, 13069, 0, 0, 127773, 0, 0, 0, 125207, 0, 0, - 1667, 0, 0, 42428, 110950, 0, 0, 41750, 0, 0, 93999, 0, 8101, 3610, - 113670, 41748, 110948, 0, 78394, 119208, 0, 0, 113691, 64549, 68359, 0, - 0, 65692, 92701, 0, 917960, 12896, 10456, 68298, 0, 0, 0, 0, 917962, 0, - 0, 113665, 70502, 0, 65687, 0, 0, 74009, 0, 113673, 8536, 70671, 0, - 78726, 0, 724, 0, 113675, 78749, 9975, 78746, 78747, 78744, 4175, 78741, - 78743, 78751, 939, 0, 128799, 983120, 0, 0, 0, 78763, 78764, 78760, - 78761, 78758, 78759, 78755, 8425, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 6370, 0, - 7827, 68441, 75008, 0, 917943, 0, 118863, 0, 0, 0, 0, 121243, 73988, - 12286, 113668, 0, 11012, 0, 43764, 178, 12972, 74620, 113671, 0, 113735, - 0, 66764, 0, 0, 65690, 72339, 0, 0, 917950, 9252, 0, 4652, 74259, 0, - 917947, 0, 0, 0, 10806, 0, 0, 70016, 0, 6723, 0, 0, 6993, 0, 0, 12855, 0, - 0, 11390, 0, 0, 0, 92503, 0, 0, 983162, 125270, 92627, 8278, 0, 4034, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12750, 9350, 66037, 0, 0, 73700, 12747, 0, 0, - 128064, 8922, 74640, 0, 0, 43150, 0, 983090, 983088, 66779, 66777, 10813, - 2592, 43139, 0, 0, 118612, 0, 0, 71891, 0, 0, 0, 0, 0, 0, 71697, 0, - 128825, 1596, 0, 0, 0, 0, 6838, 66572, 0, 126574, 120627, 8092, 12805, - 41928, 0, 78406, 78409, 0, 0, 0, 9931, 0, 0, 0, 0, 0, 983778, 6107, 0, 0, - 0, 0, 128745, 0, 335, 127003, 64689, 0, 0, 5765, 0, 0, 119227, 6092, - 118851, 0, 8876, 83465, 74947, 83455, 129186, 83454, 70713, 0, 0, 126606, - 70121, 41602, 0, 92308, 74831, 0, 11783, 68482, 0, 0, 0, 0, 0, 0, 843, 0, - 71099, 0, 0, 41935, 0, 0, 0, 0, 1371, 0, 43818, 43159, 8069, 9579, 41938, - 41608, 0, 92444, 6242, 0, 0, 128595, 128244, 0, 92499, 8805, 1742, - 113722, 0, 8202, 72399, 0, 983198, 0, 0, 73882, 100809, 0, 43467, 123636, - 55290, 0, 1712, 5932, 0, 41762, 71982, 0, 11967, 1775, 0, 75009, 0, - 11868, 120387, 9458, 0, 126614, 0, 0, 43176, 101032, 101031, 42782, - 101033, 101036, 101035, 101038, 101037, 101040, 101039, 0, 0, 0, 0, - 101041, 5794, 92274, 2662, 101045, 101044, 8254, 101046, 10975, 101048, - 120625, 101050, 917977, 4108, 8478, 917982, 194790, 0, 92263, 917980, - 7507, 0, 43149, 0, 65031, 7961, 1636, 0, 65029, 0, 129665, 70188, 9674, - 0, 99, 98, 97, 101022, 92203, 4049, 101027, 43880, 7090, 101028, 0, - 101030, 66589, 0, 65310, 66593, 66599, 129805, 0, 0, 7447, 66594, 0, 0, - 0, 73920, 66595, 66596, 42570, 5593, 0, 0, 0, 0, 6061, 64854, 119, 118, - 117, 116, 0, 122, 121, 120, 111, 110, 109, 108, 115, 114, 113, 112, 103, - 102, 101, 100, 107, 106, 105, 104, 128504, 73974, 534, 0, 67713, 1536, - 73973, 73970, 0, 129671, 0, 6020, 12716, 0, 12744, 65143, 0, 13266, - 127813, 0, 0, 0, 127116, 0, 1212, 65560, 0, 8134, 42935, 12129, 73870, 0, - 1866, 0, 122948, 0, 0, 65073, 12059, 66585, 121391, 0, 0, 0, 5935, 1250, - 0, 8174, 9787, 6733, 9859, 9858, 9861, 9860, 101012, 1882, 1892, 6731, - 10882, 10795, 101018, 73911, 101020, 101019, 41169, 8939, 0, 120713, - 41170, 1454, 0, 65130, 69732, 0, 0, 129611, 41172, 7855, 0, 71472, 0, 0, - 0, 71691, 65901, 0, 0, 645, 100992, 100991, 100994, 100993, 100996, - 100995, 100998, 65587, 0, 10688, 0, 0, 7729, 0, 101001, 120518, 101003, - 66722, 101005, 101004, 68415, 101006, 4538, 101008, 43141, 0, 0, 73699, - 0, 0, 0, 71918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71454, 0, 2381, 983752, 0, 0, - 69857, 100981, 0, 100983, 100982, 100985, 10856, 100987, 55255, 41478, - 8582, 10064, 0, 0, 0, 0, 64896, 0, 74609, 0, 128048, 10082, 11575, 0, 0, - 0, 917505, 0, 6145, 75020, 0, 92433, 71916, 83279, 43186, 0, 0, 83274, - 83276, 83277, 83278, 10191, 83271, 69633, 72353, 0, 0, 0, 0, 120090, - 120089, 7931, 8558, 917946, 0, 0, 0, 119145, 120081, 120084, 120083, - 120086, 71449, 120088, 7366, 7019, 75021, 0, 917951, 120078, 120077, - 120080, 8657, 100967, 8594, 100969, 100968, 0, 100970, 120072, 120071, 0, - 0, 43154, 0, 0, 11332, 0, 7728, 100978, 100977, 100980, 100979, 7851, 0, - 8375, 128662, 0, 0, 126095, 9085, 0, 0, 9327, 6160, 0, 0, 0, 0, 70698, - 74012, 0, 0, 4439, 121151, 100972, 100971, 100974, 100973, 100976, - 100975, 100956, 42524, 71220, 100957, 10826, 100959, 11296, 0, 0, 0, - 7504, 43161, 127868, 0, 64670, 0, 78056, 0, 11295, 0, 78053, 0, 0, 0, - 10902, 0, 0, 122650, 78068, 10472, 100954, 100953, 120215, 78062, 2371, - 78069, 118893, 259, 0, 0, 2402, 12157, 6440, 0, 100963, 100962, 100965, - 100964, 65380, 9103, 2278, 0, 0, 7301, 0, 10219, 0, 0, 0, 67718, 43178, - 0, 120214, 119362, 917974, 8613, 0, 126121, 917978, 917979, 121449, - 12005, 7353, 0, 1890, 129130, 0, 0, 0, 42815, 7991, 0, 10578, 0, 0, 0, 0, - 0, 0, 0, 111190, 120601, 42668, 9348, 0, 6164, 0, 0, 0, 7676, 0, 0, 0, 0, - 128732, 129422, 83443, 71096, 83444, 9175, 0, 78047, 9088, 73689, 0, - 1396, 0, 0, 11461, 71088, 127835, 92252, 0, 71090, 121185, 69872, 0, 0, - 0, 0, 74043, 119632, 0, 0, 0, 5928, 4525, 10658, 0, 1266, 10180, 64472, - 0, 12622, 0, 0, 0, 0, 127139, 13310, 773, 19933, 0, 0, 0, 0, 92205, 0, 0, - 0, 0, 5862, 7823, 0, 0, 0, 3250, 43991, 69687, 66649, 0, 0, 0, 0, 0, - 64673, 917963, 917964, 0, 0, 917967, 917968, 917965, 917966, 127791, - 75041, 3471, 917970, 64573, 882, 0, 119584, 0, 120772, 0, 0, 0, 92696, 0, - 0, 72988, 0, 3225, 0, 73729, 0, 0, 43173, 11752, 4381, 0, 0, 917945, - 11756, 11757, 917944, 917949, 42654, 127848, 118663, 0, 0, 5160, 1387, 0, - 917953, 0, 128933, 917956, 917957, 917954, 917955, 118595, 121082, - 917958, 10789, 68314, 0, 126521, 11143, 0, 0, 70669, 128904, 42179, 0, - 5931, 11744, 11215, 70676, 119245, 0, 0, 0, 77915, 10217, 64635, 128661, - 83292, 0, 0, 0, 0, 0, 41296, 11747, 41291, 0, 0, 0, 41294, 41282, 5923, - 120610, 0, 0, 0, 0, 66800, 5786, 68252, 42539, 119869, 119860, 0, 41474, - 0, 0, 0, 5934, 74572, 66583, 119231, 0, 94072, 64481, 0, 0, 0, 0, 67240, - 0, 0, 123201, 0, 5819, 0, 0, 0, 0, 0, 129387, 0, 0, 0, 67993, 1237, - 194749, 0, 0, 983557, 0, 0, 0, 0, 0, 0, 0, 69789, 11266, 69845, 0, 10506, - 194747, 0, 0, 0, 0, 43185, 194748, 100533, 100532, 100535, 10769, 100537, - 100536, 100539, 9753, 121035, 100540, 0, 0, 121433, 0, 100542, 6072, - 100544, 100543, 100546, 100545, 100548, 100547, 100550, 100549, 0, - 113744, 0, 0, 7222, 10283, 10315, 10379, 4996, 0, 129294, 66517, 0, - 10087, 127833, 74938, 0, 0, 83492, 7565, 42890, 0, 73520, 43180, 77928, - 74891, 77929, 43982, 100526, 622, 77926, 100527, 100530, 1602, 0, 0, 0, - 129559, 12160, 0, 10212, 77936, 194605, 12071, 43143, 77935, 917983, - 917984, 917989, 77932, 917987, 917988, 10255, 10263, 10279, 4194, 10375, - 93035, 0, 0, 12644, 127516, 917994, 75007, 110791, 67408, 110789, 11501, - 41177, 0, 0, 71912, 0, 0, 8715, 0, 41179, 0, 0, 0, 41176, 0, 41181, 0, - 8452, 121006, 13161, 0, 70503, 5921, 0, 2597, 0, 5922, 72128, 0, 74242, - 128374, 0, 0, 0, 0, 0, 0, 0, 127906, 0, 64944, 0, 0, 0, 0, 5924, 5920, - 129508, 6921, 78081, 74007, 78078, 8418, 11681, 43169, 10176, 0, 0, 0, - 78087, 10772, 65276, 5937, 1914, 78084, 11682, 0, 0, 0, 11685, 0, 100513, - 7772, 11680, 100514, 100517, 100516, 100519, 7417, 718, 100520, 70083, - 100500, 120718, 3235, 0, 43164, 0, 8018, 0, 0, 128708, 6937, 67672, - 128508, 0, 10067, 120849, 0, 0, 0, 118693, 0, 100491, 0, 100493, 100492, - 13116, 100494, 100497, 9945, 100499, 100498, 0, 0, 0, 0, 2059, 0, 100502, - 100501, 1431, 100503, 66565, 100505, 100508, 12804, 100510, 100509, - 78090, 3307, 78088, 78089, 0, 4544, 71228, 0, 0, 0, 78097, 11110, 66810, - 12882, 64511, 78094, 78100, 78102, 71226, 10141, 0, 78280, 65298, 4476, - 78109, 94005, 71216, 8907, 78105, 78106, 78103, 78104, 120898, 0, 10665, - 64616, 128944, 0, 127545, 69605, 83159, 83160, 4554, 0, 83155, 83156, - 83157, 83158, 0, 125123, 0, 72258, 129831, 0, 129815, 0, 43179, 0, 0, 0, - 717, 10754, 83168, 83169, 83162, 83163, 83164, 83165, 78282, 0, 0, 83161, - 68848, 10611, 72859, 126978, 71474, 129426, 127871, 0, 0, 0, 12820, - 110882, 0, 7009, 70103, 0, 0, 67848, 41173, 4574, 0, 0, 128338, 575, - 78110, 43456, 8563, 100469, 0, 0, 65565, 123598, 5936, 7290, 78117, - 78118, 74919, 308, 78113, 78114, 83151, 78123, 83153, 83154, 0, 0, 0, 0, - 67496, 5926, 68250, 78130, 78126, 78127, 78124, 78125, 42513, 0, 129026, - 0, 11651, 13093, 78135, 0, 100471, 0, 100473, 100472, 100475, 74048, - 100477, 71995, 100457, 100456, 43703, 13097, 0, 100460, 13283, 0, 0, - 125073, 3488, 5933, 10033, 983947, 0, 65570, 0, 12297, 0, 0, 0, 128517, - 42538, 0, 129293, 0, 100451, 0, 100453, 100452, 100455, 100454, 121221, - 0, 0, 7638, 0, 129193, 0, 43109, 7637, 0, 11213, 100461, 83355, 100463, - 100466, 100465, 0, 0, 7636, 0, 0, 0, 128848, 983087, 291, 0, 0, 2027, - 78141, 78142, 78136, 78137, 83481, 4640, 64713, 10224, 120429, 11183, - 83482, 120430, 0, 0, 0, 127148, 83479, 0, 0, 83488, 0, 0, 0, 0, 68837, - 5778, 0, 0, 0, 12680, 119130, 0, 67242, 93041, 0, 0, 0, 11552, 0, 127855, - 0, 70091, 0, 10172, 65453, 120408, 66014, 120410, 0, 4641, 11556, 64819, - 78269, 120416, 72341, 41469, 41467, 120412, 120415, 4646, 120425, 865, - 78275, 78274, 78273, 4645, 78271, 78270, 0, 983173, 7338, 0, 68840, 0, - 12565, 0, 0, 0, 195089, 119655, 195091, 195090, 2913, 13120, 128956, - 69493, 195097, 195096, 128019, 0, 71462, 0, 7916, 10485, 195098, 0, - 195100, 195099, 0, 67705, 128351, 195077, 195080, 129636, 129549, 195081, - 0, 0, 0, 10229, 10687, 826, 128081, 195082, 195085, 195084, 195087, - 195086, 0, 1808, 7848, 0, 0, 0, 0, 0, 0, 128897, 69255, 42942, 67704, 0, - 0, 0, 0, 42940, 0, 9144, 0, 0, 92992, 9840, 0, 0, 0, 0, 0, 0, 74448, - 83475, 0, 10962, 66904, 113718, 983188, 0, 0, 74537, 195072, 1792, - 195074, 195073, 78266, 195075, 0, 0, 12066, 0, 385, 4152, 0, 0, 0, 67397, - 0, 0, 0, 0, 43258, 0, 0, 13157, 0, 0, 3570, 0, 0, 0, 67252, 0, 71218, - 126631, 7879, 68247, 128579, 78914, 0, 70196, 0, 0, 8463, 7810, 917862, - 7839, 983878, 127768, 917860, 9691, 0, 129323, 0, 120385, 0, 917844, 0, - 10066, 0, 2175, 0, 0, 0, 8016, 0, 983072, 64831, 0, 126103, 0, 73493, - 1634, 68115, 94192, 11056, 0, 0, 0, 41165, 11328, 12450, 0, 41166, 0, - 12456, 0, 171, 67508, 12452, 917544, 12458, 12531, 0, 917853, 0, 74162, - 0, 0, 9969, 0, 12454, 74160, 42132, 110755, 78878, 110753, 3230, 73711, - 0, 0, 8932, 4399, 5810, 64534, 8415, 0, 110756, 110757, 74159, 0, 0, 960, - 74156, 6981, 92374, 12938, 9201, 0, 118713, 74904, 0, 72866, 92270, 0, 0, - 0, 129792, 5851, 73833, 5824, 0, 5844, 110848, 110849, 110846, 110847, - 4663, 0, 0, 0, 0, 0, 74085, 0, 0, 0, 0, 0, 92339, 0, 0, 5782, 67495, 0, - 0, 43796, 129639, 0, 195083, 125223, 128004, 0, 43861, 0, 0, 0, 92976, 0, - 0, 0, 4659, 0, 128894, 0, 0, 129386, 0, 11129, 2238, 329, 0, 92707, - 121416, 0, 0, 0, 69943, 67692, 42167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69618, 43671, 0, 64701, 0, 0, 0, 93055, 1172, 125089, 6786, 43601, 0, - 74126, 0, 0, 0, 0, 0, 118695, 0, 0, 118804, 0, 66741, 5347, 125026, - 983663, 0, 0, 10588, 0, 0, 0, 0, 5343, 0, 0, 0, 5341, 0, 0, 74916, 5351, - 0, 0, 917884, 0, 92692, 0, 121148, 128916, 0, 0, 66785, 126256, 6638, 0, - 0, 271, 0, 917904, 0, 0, 12653, 67588, 0, 0, 0, 0, 128838, 11912, 128301, - 983665, 0, 11800, 0, 0, 11103, 0, 7340, 0, 110695, 0, 0, 70170, 0, 2423, - 0, 0, 0, 128136, 42705, 0, 0, 0, 11854, 0, 0, 0, 0, 4916, 0, 380, 10958, - 66563, 127790, 78284, 67587, 0, 12918, 0, 917897, 0, 917898, 917893, - 10684, 0, 125063, 92906, 0, 0, 8182, 0, 0, 129434, 0, 0, 0, 6859, 0, - 6630, 100405, 0, 123191, 0, 0, 0, 65876, 5535, 129892, 0, 0, 92609, 0, - 983348, 6477, 43795, 92217, 129571, 72163, 69496, 43848, 0, 0, 74256, - 2665, 11304, 43751, 0, 4970, 74353, 0, 8934, 0, 93996, 4492, 92908, - 65011, 0, 0, 92909, 1188, 7254, 1100, 0, 0, 0, 2912, 11749, 92643, 0, 0, - 65057, 0, 12343, 0, 78879, 0, 78880, 0, 0, 0, 70355, 0, 0, 11803, 0, 0, - 41450, 0, 100897, 0, 41451, 0, 0, 8273, 0, 3451, 0, 972, 41453, 68164, - 78876, 0, 92408, 73945, 43504, 2288, 78873, 9538, 78874, 128685, 0, - 129095, 0, 0, 0, 0, 11019, 0, 0, 121205, 0, 73007, 71365, 92716, 5927, 0, - 0, 0, 0, 128484, 0, 6073, 0, 0, 0, 6075, 93995, 282, 126510, 0, 74078, - 121459, 2206, 0, 0, 66791, 0, 3474, 0, 0, 0, 6081, 0, 127843, 74076, 0, - 0, 0, 128908, 0, 0, 0, 12623, 120273, 9120, 120275, 4665, 12628, 4670, - 120271, 120272, 0, 0, 121480, 958, 0, 0, 0, 4666, 0, 4915, 0, 4669, 0, 0, - 0, 4664, 0, 120550, 0, 0, 0, 0, 94023, 0, 917875, 8664, 11664, 0, 129327, - 11224, 0, 0, 1063, 119088, 120251, 9772, 7255, 8886, 0, 127932, 120257, - 120258, 120259, 120260, 42661, 71345, 120255, 119125, 120265, 120266, - 120267, 42721, 92407, 120262, 120263, 66788, 1017, 0, 118580, 505, 1447, - 0, 0, 70340, 66793, 65115, 42789, 128443, 0, 0, 123634, 0, 119195, 0, 0, - 11745, 7919, 0, 1641, 0, 0, 8966, 0, 0, 8743, 71870, 0, 67813, 0, 0, 0, - 123206, 0, 0, 128505, 10169, 71324, 0, 10068, 0, 120457, 120456, 120455, - 120454, 257, 43170, 13153, 0, 0, 0, 0, 0, 0, 6496, 19917, 5930, 128354, - 11033, 0, 0, 5622, 120436, 8477, 8474, 120433, 120432, 0, 0, 0, 41435, - 4352, 0, 2435, 0, 5621, 0, 4201, 8450, 4203, 4202, 4205, 4204, 120447, - 120446, 120445, 66792, 41440, 120442, 8473, 6373, 8469, 120438, 0, 4564, - 125206, 0, 0, 0, 8374, 73669, 0, 0, 66796, 0, 0, 0, 0, 0, 69297, 129762, - 5626, 43507, 11771, 0, 0, 0, 42614, 0, 5625, 0, 0, 0, 5623, 0, 0, 42623, - 64277, 69942, 0, 0, 120752, 0, 5817, 5629, 0, 7551, 10325, 5632, 69674, - 0, 0, 124946, 125194, 5628, 129766, 5631, 0, 0, 2400, 5627, 0, 0, 118786, - 74792, 0, 0, 0, 203, 129084, 74365, 0, 0, 0, 0, 83382, 83422, 0, 0, 554, - 0, 0, 0, 12182, 0, 64569, 110840, 73891, 0, 0, 0, 7689, 69798, 9323, - 10269, 10285, 10317, 175, 0, 0, 0, 0, 0, 1243, 42154, 0, 92387, 0, 0, - 43651, 0, 125021, 0, 9075, 118597, 0, 64777, 128570, 0, 0, 0, 0, 65255, - 0, 121142, 4490, 0, 6649, 120698, 12181, 0, 11977, 7249, 8366, 0, 7756, - 12342, 0, 51, 41516, 69432, 0, 9568, 71318, 456, 0, 10437, 1168, 9251, - 9082, 0, 0, 42781, 3866, 0, 41512, 0, 0, 68121, 41494, 0, 4660, 0, 10405, - 0, 0, 0, 0, 0, 73918, 119627, 110686, 41454, 12605, 0, 126611, 41455, - 917996, 983605, 0, 8214, 0, 100413, 129320, 41457, 983077, 0, 1969, - 127771, 0, 69554, 7413, 0, 69426, 10341, 43864, 78079, 5854, 0, 0, 0, - 129684, 72819, 0, 0, 73548, 0, 0, 8429, 0, 72328, 0, 6429, 0, 0, 0, 0, - 110688, 83417, 0, 917864, 120813, 83423, 1662, 125000, 0, 0, 917871, - 917868, 0, 0, 66, 65, 68, 67, 70, 69, 72, 71, 74, 73, 76, 75, 78, 77, 80, - 79, 82, 81, 84, 83, 86, 85, 88, 87, 90, 89, 0, 0, 7385, 70508, 1704, - 12993, 0, 0, 0, 0, 0, 0, 0, 0, 11353, 72207, 0, 0, 0, 0, 118831, 0, 0, 0, - 0, 0, 118719, 83364, 0, 0, 1289, 0, 0, 119583, 0, 65507, 0, 0, 0, 128042, - 0, 74409, 0, 0, 0, 0, 64793, 0, 0, 100843, 5675, 119239, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 6972, 70735, 0, 121108, 126217, 0, 0, 0, 0, 0, 110640, - 67687, 0, 0, 119634, 0, 43977, 111252, 129105, 0, 7412, 64671, 0, 1412, - 4594, 1391, 0, 8067, 12478, 110639, 78375, 110637, 10281, 110635, 0, 0, - 7960, 43271, 0, 12518, 69846, 0, 3566, 0, 0, 69864, 0, 0, 68021, 0, 0, 0, - 8223, 0, 4261, 121460, 68918, 0, 0, 121294, 113712, 0, 128046, 43419, - 72748, 92866, 10574, 0, 67691, 0, 0, 73785, 0, 78875, 128541, 0, 127366, - 0, 0, 0, 0, 6695, 65113, 324, 0, 128373, 40985, 0, 0, 0, 0, 0, 72307, - 43474, 0, 121190, 0, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110871, 9574, - 120684, 110870, 110814, 5204, 74774, 0, 11835, 0, 0, 983186, 0, 0, 0, 0, - 0, 0, 11750, 68898, 127004, 0, 0, 0, 0, 8130, 0, 0, 0, 121268, 0, 129443, - 0, 68455, 42863, 73839, 0, 0, 0, 92288, 0, 0, 0, 612, 110875, 110876, - 72231, 10538, 0, 1674, 0, 0, 0, 12280, 0, 540, 74550, 0, 66422, 8432, 0, - 11073, 0, 64316, 129894, 0, 7388, 0, 0, 0, 0, 126107, 0, 3359, 0, 0, - 67284, 0, 0, 65482, 129589, 0, 64742, 129304, 0, 124141, 74273, 0, 19941, - 0, 0, 0, 0, 9481, 65555, 0, 66628, 129126, 1195, 64898, 0, 0, 0, 2010, 0, - 0, 0, 0, 0, 0, 4360, 127009, 9739, 0, 72885, 0, 0, 0, 126265, 72200, 0, - 0, 120025, 72199, 0, 0, 65734, 0, 0, 129690, 13075, 0, 94063, 0, 43532, - 10837, 2492, 74516, 983075, 120882, 0, 0, 11813, 9649, 0, 119617, 5128, - 7377, 0, 65604, 0, 0, 6771, 1648, 7819, 0, 0, 0, 125192, 128131, 12709, - 6986, 0, 0, 0, 0, 0, 12581, 0, 5175, 0, 73806, 0, 128420, 0, 0, 77950, 0, - 0, 607, 0, 0, 128846, 119605, 67475, 129528, 65477, 0, 121130, 0, 8265, - 0, 0, 0, 5840, 42838, 0, 0, 68366, 0, 119255, 0, 0, 0, 127929, 0, 2550, - 121011, 6779, 70059, 0, 0, 0, 0, 0, 0, 5619, 65822, 0, 0, 0, 129392, - 5616, 11486, 0, 0, 0, 0, 5615, 0, 121319, 42380, 127958, 0, 66451, 74407, - 0, 11347, 0, 1026, 5620, 0, 0, 11350, 5617, 0, 0, 64639, 0, 0, 0, 1338, - 0, 0, 0, 4603, 0, 70715, 92484, 0, 9002, 0, 3974, 78213, 0, 0, 0, 0, 0, - 0, 75038, 66040, 70455, 0, 0, 0, 72982, 0, 0, 0, 0, 0, 118661, 0, 0, - 119105, 0, 0, 0, 0, 0, 128883, 0, 66897, 0, 0, 0, 42594, 0, 0, 0, 0, - 6714, 10083, 0, 121019, 0, 69976, 0, 0, 9073, 0, 64302, 0, 128286, 9725, - 0, 0, 121288, 73769, 121306, 0, 9570, 0, 11500, 2689, 917626, 0, 983813, - 66740, 0, 0, 0, 917623, 13286, 5500, 42598, 42596, 503, 0, 0, 917618, 0, - 0, 0, 0, 917615, 1652, 772, 6688, 8310, 0, 0, 72124, 0, 10194, 43542, 0, - 125054, 0, 6468, 68110, 0, 917606, 11767, 0, 0, 5836, 12358, 0, 0, 65624, - 12180, 0, 127994, 0, 43699, 0, 0, 72114, 43706, 0, 12362, 12435, 12360, - 0, 9020, 0, 12356, 8616, 0, 42924, 2227, 0, 0, 7315, 12354, 83097, 83098, - 83099, 2358, 83092, 83093, 83094, 0, 0, 83089, 83090, 0, 11759, 71723, 0, - 72834, 83109, 41423, 0, 83103, 83104, 83105, 42237, 110653, 70717, 72260, - 83102, 0, 67856, 0, 128534, 110657, 129354, 129194, 0, 64395, 0, 73008, - 120897, 74816, 0, 0, 0, 83088, 0, 0, 94064, 83083, 83085, 83086, 83087, - 83079, 83080, 2041, 9178, 0, 64870, 0, 83076, 74924, 0, 0, 0, 0, 0, - 78739, 0, 0, 0, 0, 0, 0, 3726, 0, 0, 0, 0, 0, 121432, 129457, 0, 0, 0, 0, - 0, 74901, 0, 0, 0, 0, 0, 124944, 113781, 0, 7410, 2669, 903, 0, 0, 0, - 127232, 74603, 0, 128264, 0, 128411, 0, 0, 11732, 0, 72797, 41448, 41461, - 124934, 0, 917558, 0, 8819, 0, 0, 74606, 92847, 121412, 74835, 0, 9168, - 65786, 0, 73691, 0, 67665, 0, 11758, 68425, 0, 0, 0, 128044, 0, 19924, - 67312, 0, 128755, 64551, 0, 8516, 0, 0, 7561, 983999, 74018, 0, 0, 0, 0, - 83074, 83075, 0, 11233, 83062, 83066, 3787, 83070, 83055, 41458, 83059, - 41463, 65308, 41459, 8683, 775, 0, 65584, 69923, 0, 110798, 110799, - 110796, 43440, 0, 0, 0, 3656, 0, 0, 0, 67694, 1599, 83138, 83139, 8514, - 8513, 83036, 83135, 83136, 110794, 110795, 83131, 83132, 0, 0, 0, 11684, - 10542, 9937, 83150, 0, 75037, 83145, 65730, 83147, 0, 8427, 83142, 55246, - 0, 0, 11497, 0, 0, 0, 119222, 0, 983598, 0, 10621, 0, 0, 129295, 119111, - 120745, 0, 0, 0, 11648, 83126, 83127, 42118, 83129, 83122, 65512, 83124, - 83125, 0, 0, 0, 83121, 74530, 128456, 0, 0, 0, 65724, 0, 0, 0, 65727, 0, - 0, 64963, 73830, 66042, 0, 0, 7875, 0, 0, 0, 129476, 0, 0, 536, 0, 0, 0, - 0, 65173, 129122, 0, 70331, 0, 0, 118598, 0, 129419, 0, 0, 0, 1687, 0, 0, - 0, 0, 0, 0, 10526, 0, 8323, 0, 83301, 11731, 73530, 0, 65460, 12242, 0, - 0, 10843, 11554, 0, 0, 8266, 0, 121101, 0, 0, 0, 0, 67667, 118694, - 119155, 0, 0, 119636, 67857, 0, 0, 0, 11755, 66305, 0, 0, 10917, 93979, - 113688, 0, 2040, 92596, 0, 0, 0, 0, 1227, 83119, 83120, 0, 0, 83115, - 74427, 11149, 4978, 83111, 1984, 11830, 83114, 128934, 74548, 118545, - 9373, 0, 0, 0, 0, 0, 0, 0, 0, 9237, 9390, 0, 0, 0, 0, 0, 1830, 0, 0, 0, - 0, 0, 128577, 983839, 68086, 0, 0, 0, 983059, 0, 983145, 0, 0, 0, 72197, - 55291, 11683, 0, 983659, 0, 11451, 0, 72714, 3731, 2359, 0, 67844, 0, - 121503, 548, 121502, 983250, 121405, 983253, 0, 66272, 0, 64678, 0, 9547, - 0, 0, 1614, 0, 0, 66307, 128092, 1358, 120871, 428, 0, 1466, 0, 10982, 0, - 0, 0, 407, 0, 0, 0, 0, 0, 0, 5804, 73464, 0, 0, 0, 70167, 9057, 42446, 0, - 125097, 0, 0, 8250, 10952, 8048, 0, 129155, 0, 118955, 0, 0, 118593, - 4407, 74648, 0, 0, 0, 8448, 92491, 0, 0, 12675, 12659, 0, 0, 983285, - 68077, 55273, 10766, 12012, 2386, 0, 9170, 0, 9123, 128194, 0, 0, 0, 0, - 129942, 0, 0, 0, 0, 0, 0, 8709, 0, 72383, 0, 0, 0, 0, 0, 0, 0, 128342, 0, - 577, 128610, 0, 0, 124999, 68087, 74840, 126474, 127036, 0, 0, 0, 1414, - 124963, 9683, 43486, 92231, 0, 2536, 0, 66330, 0, 0, 0, 0, 0, 0, 0, - 66317, 0, 66315, 66316, 0, 0, 0, 0, 0, 0, 0, 0, 66323, 66324, 0, 0, 3106, - 65917, 0, 2182, 0, 891, 0, 0, 42624, 0, 0, 8824, 65089, 128734, 10936, 0, - 0, 0, 0, 92688, 0, 0, 0, 0, 12745, 0, 0, 41285, 3547, 0, 0, 129877, 0, - 118701, 6089, 0, 68490, 120578, 4170, 1029, 127761, 0, 0, 42374, 917625, - 744, 917624, 0, 0, 0, 93046, 0, 3551, 0, 0, 4623, 0, 0, 12340, 0, 65136, - 0, 0, 0, 0, 0, 0, 0, 72291, 0, 0, 120778, 0, 11972, 0, 78757, 0, 122886, - 177, 122894, 0, 0, 0, 0, 55243, 0, 0, 0, 70172, 120249, 120242, 128027, - 120243, 0, 0, 0, 120237, 120245, 94079, 0, 0, 9136, 120240, 120614, - 41280, 0, 0, 0, 0, 74149, 128327, 0, 0, 66361, 12601, 72194, 64360, - 65163, 125241, 0, 0, 0, 0, 0, 5404, 43332, 3667, 7936, 12925, 0, 0, 0, 0, - 0, 10874, 65505, 0, 0, 0, 0, 128920, 983681, 0, 0, 0, 0, 0, 0, 0, 0, - 66677, 0, 0, 0, 70088, 74148, 0, 0, 72868, 120230, 120224, 74172, 0, 0, - 94096, 0, 128414, 120636, 0, 127519, 917609, 917616, 0, 128652, 0, 0, - 11441, 0, 3512, 0, 0, 43597, 0, 0, 72734, 68153, 41563, 0, 0, 129352, - 41544, 0, 0, 74927, 0, 129177, 0, 0, 0, 118908, 0, 78108, 67396, 73804, - 64711, 0, 0, 917610, 0, 0, 0, 11557, 127776, 0, 12079, 0, 0, 0, 0, - 128861, 0, 0, 0, 0, 0, 983201, 8103, 72303, 128174, 92486, 110698, 0, - 64587, 0, 0, 124961, 0, 0, 0, 126481, 0, 0, 0, 0, 0, 70348, 1450, 0, - 1340, 0, 0, 128970, 0, 0, 125117, 0, 0, 0, 0, 6539, 92948, 0, 128213, - 125060, 0, 0, 0, 3973, 0, 70504, 121193, 7982, 0, 0, 127194, 0, 0, 0, - 128408, 118968, 6417, 120619, 129748, 0, 0, 0, 129455, 4919, 65121, - 110872, 7755, 0, 0, 64548, 0, 1621, 0, 0, 0, 0, 0, 12188, 0, 0, 0, 0, - 5015, 0, 0, 42590, 70354, 1756, 0, 0, 0, 120694, 0, 0, 7555, 73874, 5408, - 2817, 1214, 69919, 0, 983126, 0, 0, 125055, 127195, 7957, 0, 0, 1056, - 74944, 0, 0, 0, 0, 7073, 74979, 0, 70853, 0, 110874, 0, 0, 2341, 126644, - 8484, 0, 0, 68322, 0, 8461, 67721, 42269, 0, 0, 43709, 43708, 9451, 7571, - 13073, 43847, 126647, 0, 983263, 0, 0, 0, 8781, 12894, 78134, 0, 78132, - 0, 0, 78184, 0, 11338, 120768, 0, 0, 0, 0, 0, 121367, 65021, 64795, - 74574, 0, 10047, 0, 0, 0, 0, 0, 0, 119181, 163, 576, 9895, 0, 0, 74591, - 0, 0, 66888, 0, 0, 0, 0, 0, 0, 7017, 128111, 0, 0, 129922, 0, 41591, - 11036, 65252, 120795, 129488, 0, 0, 0, 0, 0, 0, 8887, 0, 7295, 71203, 0, - 127221, 0, 0, 0, 0, 8755, 0, 0, 8147, 73127, 0, 0, 121348, 0, 129377, 0, - 74499, 0, 0, 0, 4619, 0, 6654, 123192, 0, 0, 0, 65689, 10128, 0, 129612, - 0, 0, 92651, 0, 2401, 0, 8792, 118546, 0, 74980, 0, 92246, 0, 0, 0, - 12886, 0, 66624, 0, 0, 74133, 65170, 0, 74135, 0, 0, 9984, 73867, 3010, - 0, 70349, 10698, 41475, 0, 119151, 0, 119152, 0, 0, 9100, 0, 0, 0, 78116, - 64780, 2001, 0, 55230, 0, 4052, 92856, 7626, 78080, 0, 0, 0, 41477, 0, 0, - 0, 43707, 74127, 0, 0, 0, 78086, 73758, 2335, 10663, 0, 0, 129872, - 119602, 0, 0, 70325, 0, 41443, 0, 0, 0, 9711, 1523, 0, 0, 41445, 0, 0, - 8567, 41442, 12821, 0, 0, 118978, 0, 65274, 0, 94082, 0, 127515, 0, 0, - 43446, 0, 0, 0, 0, 127985, 0, 10206, 127167, 6375, 2673, 0, 0, 0, 43219, - 129355, 0, 0, 0, 0, 129400, 11799, 101225, 68466, 0, 0, 0, 0, 0, 120736, - 0, 7203, 0, 0, 70361, 127213, 120615, 127216, 0, 0, 0, 0, 43121, 0, - 128366, 72161, 0, 129868, 0, 121260, 73781, 70365, 0, 68039, 70446, - 10057, 0, 0, 0, 101219, 120963, 101220, 2307, 0, 0, 0, 0, 73873, 0, - 94035, 0, 0, 67469, 0, 129983, 7327, 0, 0, 440, 0, 0, 68613, 75059, 0, 0, - 9957, 0, 0, 8046, 0, 119158, 0, 0, 68609, 0, 129405, 1521, 129460, 92256, - 65344, 0, 11850, 68737, 0, 0, 68914, 7303, 65770, 5243, 0, 5239, 65771, - 121429, 0, 5237, 0, 68756, 0, 5247, 0, 0, 0, 12873, 5764, 0, 0, 3008, - 118981, 128102, 0, 0, 55231, 41103, 0, 92756, 0, 0, 92717, 70074, 7872, - 74886, 917567, 8731, 65378, 118564, 0, 11316, 128163, 126600, 70360, - 3019, 9997, 0, 0, 9456, 129545, 0, 0, 101192, 0, 0, 92682, 4281, 0, 0, 0, - 118982, 0, 69993, 78096, 0, 78095, 0, 78098, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2134, 0, 10116, 9877, 70679, 0, 0, 92723, 8379, 0, 6778, 0, 0, 8243, 0, - 0, 0, 0, 128008, 0, 0, 0, 983630, 119668, 129962, 92722, 983098, 5637, - 125115, 0, 0, 120479, 0, 113730, 0, 0, 194990, 64432, 0, 70363, 121368, - 1156, 68052, 0, 0, 120482, 0, 68030, 0, 0, 0, 7634, 0, 0, 65536, 0, 0, 0, - 7702, 0, 78890, 0, 65779, 65783, 195066, 120961, 5700, 0, 0, 92161, 2339, - 92476, 5697, 0, 0, 0, 74923, 0, 5696, 92677, 0, 3862, 0, 0, 0, 983055, 0, - 0, 0, 0, 5701, 9722, 41490, 41370, 5698, 0, 0, 0, 42204, 55270, 8571, 0, - 0, 43859, 0, 78731, 0, 12184, 0, 0, 0, 0, 0, 5650, 0, 64712, 120474, 0, - 120458, 5647, 120473, 7387, 0, 92675, 11477, 5646, 0, 11018, 0, 0, 0, 0, - 0, 0, 69280, 128459, 126128, 5651, 0, 0, 0, 5648, 0, 120920, 0, 127517, - 3545, 0, 6984, 0, 0, 0, 69414, 126613, 0, 10123, 0, 69274, 0, 0, 65020, - 74885, 119166, 0, 0, 0, 0, 0, 1140, 78426, 0, 0, 0, 122665, 8128, 9889, - 0, 0, 1815, 0, 890, 0, 3267, 0, 0, 0, 983686, 4410, 125081, 10576, 8102, - 0, 580, 74232, 0, 0, 0, 0, 0, 19938, 0, 0, 0, 0, 3298, 6546, 0, 0, 0, 0, - 6134, 41246, 0, 0, 0, 917770, 0, 6264, 0, 0, 0, 0, 0, 0, 69445, 0, 0, 0, - 92697, 11915, 10377, 0, 10072, 0, 0, 2329, 0, 0, 0, 0, 0, 0, 0, 67498, 0, - 101164, 0, 11201, 92708, 74769, 0, 13263, 0, 0, 92404, 126066, 69491, 0, - 0, 64917, 0, 0, 494, 128026, 0, 65098, 0, 956, 125265, 129556, 0, 73740, - 0, 0, 0, 74281, 128638, 0, 0, 69217, 120930, 0, 0, 0, 0, 0, 9922, 0, - 126269, 100948, 0, 65229, 0, 118671, 0, 0, 0, 0, 3907, 118833, 64526, - 11829, 68197, 0, 0, 11475, 70329, 3020, 42264, 0, 0, 0, 7098, 0, 0, - 127967, 957, 42696, 0, 3016, 0, 0, 0, 0, 0, 121248, 92510, 3006, 4620, 0, - 0, 0, 0, 129369, 129425, 0, 0, 0, 126246, 8626, 0, 128824, 0, 65377, 0, - 983103, 42920, 1698, 0, 64477, 0, 0, 43813, 100432, 100431, 100434, - 100433, 100436, 70321, 100438, 100437, 100440, 100439, 0, 121024, 101177, - 70327, 100441, 55252, 100443, 100442, 100445, 100444, 66641, 100446, - 100449, 100448, 0, 100450, 113820, 74866, 64375, 0, 127850, 129477, 0, 0, - 0, 0, 983799, 0, 0, 120827, 0, 0, 123637, 0, 0, 0, 101183, 8110, 100421, - 0, 100423, 5830, 100425, 100424, 100427, 73540, 100429, 100428, 42389, - 78611, 121398, 0, 0, 0, 0, 0, 0, 0, 83342, 983954, 0, 127147, 119187, - 2135, 11836, 0, 0, 78869, 42313, 5579, 0, 70384, 983082, 94002, 0, 5578, - 11840, 73006, 42023, 69849, 5669, 92559, 0, 0, 68833, 917845, 128275, - 5583, 0, 0, 42426, 5580, 42276, 0, 892, 2220, 42465, 74313, 73440, 5795, - 194991, 68774, 65702, 68770, 0, 65695, 0, 65710, 128399, 0, 0, 68783, 0, - 0, 0, 1638, 10966, 0, 917547, 0, 118921, 0, 0, 0, 8172, 120760, 0, 0, 0, - 0, 0, 6374, 0, 0, 120972, 0, 0, 0, 0, 0, 0, 0, 72204, 64900, 7153, 65785, - 68826, 0, 3015, 68743, 68740, 68738, 68805, 6400, 68749, 68748, 68760, - 68758, 11276, 68754, 100420, 372, 101138, 68761, 118874, 0, 41585, - 128202, 0, 74228, 276, 129895, 74234, 0, 74226, 0, 9007, 0, 41588, - 125001, 119189, 10763, 0, 0, 983560, 126097, 68525, 6257, 73112, 100393, - 100396, 100395, 100398, 92409, 100400, 100399, 101148, 74848, 120006, - 983592, 100401, 66498, 100403, 100402, 64790, 73454, 100407, 100406, - 70356, 100408, 0, 100410, 66829, 70817, 5711, 41633, 12098, 65571, 9166, - 0, 5710, 0, 6790, 65213, 0, 0, 0, 69726, 0, 73817, 0, 0, 5715, 0, 70408, - 0, 5712, 100382, 41620, 100384, 3074, 5722, 100389, 100388, 73768, 0, - 118906, 0, 0, 0, 66419, 71972, 0, 0, 0, 0, 64839, 78607, 0, 129074, 0, 0, - 0, 0, 0, 0, 113682, 0, 11261, 0, 0, 0, 8701, 0, 11236, 0, 129490, 100390, - 0, 0, 0, 78293, 0, 0, 0, 64946, 0, 0, 0, 70336, 0, 0, 93986, 68814, - 42902, 0, 0, 0, 0, 92344, 0, 67845, 42641, 71444, 0, 0, 70366, 101152, - 100369, 100368, 5084, 100370, 101158, 118861, 0, 733, 74646, 0, 0, 0, - 125085, 0, 9218, 0, 100380, 100379, 71070, 0, 0, 0, 0, 70323, 0, 0, 5155, - 0, 0, 983775, 0, 0, 72351, 0, 0, 0, 122891, 0, 0, 0, 100372, 100371, - 100374, 100373, 100376, 100375, 100378, 100377, 4974, 100357, 100360, - 100359, 0, 0, 0, 12205, 0, 0, 64507, 0, 0, 0, 0, 0, 0, 12149, 13088, - 78290, 0, 12241, 0, 0, 0, 6932, 100352, 73676, 100354, 100353, 100356, - 351, 68764, 0, 0, 0, 0, 73443, 0, 0, 100361, 42377, 100363, 100362, - 100365, 100364, 100367, 9013, 4054, 0, 0, 71939, 0, 120782, 5585, 65881, - 0, 0, 118541, 0, 5584, 8358, 128975, 121177, 0, 0, 0, 41616, 0, 983815, - 2218, 0, 5589, 0, 2664, 41613, 5586, 118890, 0, 11356, 0, 0, 0, 78609, 0, - 0, 0, 0, 0, 129870, 0, 0, 8135, 129685, 0, 983810, 0, 0, 0, 5657, 0, - 12915, 121453, 0, 10179, 5654, 12939, 0, 120799, 0, 0, 5652, 10945, - 118626, 0, 0, 113710, 0, 73449, 68069, 0, 70332, 0, 5659, 0, 0, 66729, - 5655, 0, 0, 0, 68806, 0, 128225, 66310, 73444, 0, 0, 70362, 0, 11609, 0, - 126990, 92949, 10272, 10304, 10368, 74511, 594, 10244, 10248, 10256, - 983918, 122974, 0, 3467, 41010, 0, 3331, 946, 0, 1495, 13184, 74330, - 128242, 9562, 0, 123175, 0, 70036, 122976, 0, 0, 123176, 0, 0, 0, 5666, - 65227, 123174, 68419, 0, 11796, 123178, 0, 0, 10186, 123172, 7732, - 983755, 0, 0, 0, 5668, 83334, 0, 74645, 5670, 0, 0, 12741, 126619, - 123638, 5667, 19952, 120807, 113766, 12749, 0, 67757, 2263, 0, 0, 119260, - 129131, 9286, 83335, 128457, 83336, 70359, 0, 3571, 13247, 5874, 78279, - 73447, 68435, 78278, 78267, 78268, 0, 78265, 553, 113768, 0, 93053, 5829, - 0, 4587, 78285, 78299, 129699, 12746, 0, 70338, 0, 5633, 0, 94101, 94102, - 94099, 94100, 94105, 74856, 94103, 12742, 0, 983837, 0, 0, 0, 70330, 0, - 983830, 0, 0, 0, 12148, 0, 0, 0, 0, 0, 64938, 67234, 5634, 0, 0, 2146, 0, - 118880, 2425, 65182, 983832, 43636, 0, 83326, 328, 0, 68736, 0, 5636, - 123163, 5329, 0, 5638, 0, 7940, 0, 43223, 43760, 5635, 3373, 72424, - 78292, 74223, 73441, 68763, 78287, 9833, 0, 74208, 41635, 0, 77775, - 43040, 78297, 68778, 78295, 5639, 65603, 5660, 5640, 78303, 0, 78300, 0, - 68301, 0, 0, 78312, 0, 78310, 41625, 78308, 78309, 100731, 41780, 5642, - 100732, 100735, 100734, 4356, 100736, 100739, 12051, 70166, 100740, 5641, - 8259, 0, 0, 0, 119570, 0, 0, 121264, 983558, 0, 0, 0, 73890, 0, 0, 2800, - 11220, 5645, 64964, 8652, 83323, 0, 0, 121356, 5608, 128281, 119932, - 118562, 0, 0, 9000, 0, 83324, 92673, 129176, 0, 5613, 74267, 100721, - 100724, 5610, 100726, 92965, 100728, 5612, 100730, 10787, 0, 3615, - 123647, 5609, 78316, 78317, 78313, 78315, 5875, 5808, 0, 8186, 0, 74269, - 122977, 70004, 65874, 72422, 5807, 0, 66320, 5306, 12936, 0, 92970, - 127961, 0, 92583, 10211, 0, 0, 78871, 121063, 0, 129512, 0, 0, 0, 0, 0, - 74237, 0, 9133, 74262, 0, 92840, 0, 64779, 4672, 73529, 6185, 64776, 0, - 121266, 6499, 0, 0, 0, 92720, 0, 67494, 93791, 2534, 0, 93768, 93778, - 93762, 71849, 71869, 93781, 64583, 93761, 93780, 78922, 93787, 92443, - 128714, 71848, 93774, 66411, 93785, 71841, 93770, 93769, 0, 0, 0, 121168, - 68443, 69774, 931, 0, 125052, 6363, 2748, 0, 0, 0, 983603, 44011, 0, 0, - 100711, 119009, 100713, 100712, 100715, 65896, 100717, 78298, 100719, - 100718, 128836, 100720, 11649, 0, 0, 0, 0, 0, 42341, 65284, 0, 0, 12884, - 0, 7907, 127255, 0, 0, 0, 0, 68779, 0, 68786, 0, 100691, 0, 100693, - 100692, 42851, 100694, 100697, 100696, 92276, 78226, 66393, 100700, 0, - 93773, 93776, 93777, 100702, 78301, 100704, 100703, 42415, 78307, 4542, - 69909, 94022, 100709, 0, 0, 0, 0, 42454, 11565, 7949, 124939, 0, 0, - 42494, 3073, 0, 0, 42302, 0, 126553, 70810, 0, 72401, 0, 0, 0, 129319, - 4877, 100681, 100684, 100683, 10548, 100685, 100688, 100687, 100690, - 64798, 70805, 5346, 0, 126570, 124135, 4874, 124136, 0, 0, 0, 0, 65884, - 0, 0, 0, 11378, 0, 42785, 0, 3251, 11203, 0, 0, 0, 69568, 11052, 0, 5342, - 8317, 0, 0, 5340, 0, 122970, 128599, 0, 129538, 0, 128395, 0, 128510, 0, - 0, 9142, 0, 0, 0, 10938, 0, 0, 1182, 127381, 4829, 0, 0, 72438, 529, 0, - 0, 0, 10586, 10790, 10839, 121427, 41593, 100669, 0, 118532, 41594, 225, - 66418, 0, 0, 983969, 11376, 0, 41596, 0, 64975, 0, 0, 11084, 3194, 0, - 78306, 78305, 0, 0, 0, 11324, 0, 0, 8420, 127756, 128844, 0, 41338, - 129683, 11485, 0, 41322, 66605, 100671, 0, 100673, 100672, 100675, 5161, - 41330, 100676, 100679, 100678, 100659, 100658, 0, 100660, 0, 100485, - 12361, 0, 12359, 983559, 41369, 66412, 12191, 0, 0, 0, 0, 78221, 41376, - 0, 9870, 0, 41385, 65824, 100651, 11938, 100653, 100652, 100655, 100654, - 42678, 100656, 0, 64649, 0, 0, 0, 0, 0, 983967, 100662, 100661, 100664, - 66334, 100666, 70280, 832, 100667, 2240, 78473, 66007, 78471, 65703, 0, - 0, 0, 12357, 0, 41395, 0, 0, 0, 0, 0, 0, 983466, 0, 41114, 65466, 0, - 983844, 6024, 0, 9979, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 4230, 0, 7367, 0, - 92353, 7563, 42376, 0, 128532, 0, 0, 0, 0, 67500, 0, 78466, 0, 12208, - 128138, 0, 66311, 71309, 0, 41130, 78286, 0, 0, 70047, 0, 6022, 0, 0, 0, - 0, 0, 41125, 0, 66453, 0, 41107, 0, 41121, 5300, 129588, 0, 0, 128759, - 74801, 70855, 2074, 73456, 0, 0, 12453, 0, 0, 0, 0, 68159, 12457, 0, 0, - 66278, 0, 0, 0, 0, 0, 66637, 12455, 0, 128473, 0, 12449, 0, 71224, 0, 0, - 66908, 0, 10165, 0, 119249, 113715, 0, 128223, 0, 0, 0, 0, 4993, 0, 6168, - 74033, 4995, 0, 69459, 77756, 4639, 0, 72223, 0, 73478, 0, 0, 0, 73486, - 66991, 0, 0, 0, 0, 0, 0, 83310, 0, 0, 0, 0, 0, 0, 0, 0, 129594, 4953, 0, - 0, 0, 0, 83311, 0, 73453, 65688, 0, 10125, 3517, 0, 0, 0, 65094, 74791, - 78262, 10627, 66333, 78256, 78257, 83304, 78253, 0, 71317, 64923, 0, - 65208, 10608, 78263, 78264, 0, 0, 0, 65883, 0, 0, 74914, 0, 0, 6853, 0, - 0, 12912, 119012, 0, 128191, 0, 0, 129586, 0, 1290, 0, 0, 0, 0, 113719, - 71442, 0, 0, 8978, 0, 119135, 120979, 10527, 71079, 0, 0, 0, 0, 0, 0, - 5336, 0, 0, 6934, 0, 10780, 0, 0, 78767, 0, 0, 0, 347, 0, 0, 78775, - 64675, 41582, 78774, 78771, 68094, 74903, 78769, 69221, 69657, 0, 0, - 11153, 120981, 78526, 0, 0, 0, 0, 41584, 0, 69464, 0, 0, 0, 0, 43510, - 66661, 0, 66306, 78791, 66384, 0, 6609, 0, 0, 11319, 0, 66984, 0, 41730, - 0, 0, 127920, 0, 65172, 41728, 41721, 0, 0, 0, 41203, 0, 0, 41726, 0, 0, - 5758, 0, 0, 41140, 2028, 78092, 0, 0, 0, 92739, 983196, 41138, 0, 0, 0, - 125082, 1115, 127060, 9794, 127062, 67671, 92238, 12237, 78787, 66314, - 78785, 9290, 73668, 78783, 78780, 66985, 127144, 7926, 0, 0, 0, 64398, - 100924, 71274, 12311, 101268, 78796, 78798, 78794, 78795, 78792, 78793, - 0, 101270, 0, 73455, 0, 0, 0, 42142, 9968, 11583, 0, 7092, 129835, 9627, - 78536, 73677, 78535, 0, 0, 1248, 10148, 127755, 0, 0, 101273, 0, 66447, - 0, 0, 0, 0, 65305, 0, 4031, 42794, 119986, 0, 8154, 0, 0, 128028, 126259, - 129926, 125220, 73452, 0, 0, 0, 6696, 0, 119599, 0, 0, 0, 4364, 0, 0, 0, - 120976, 0, 120922, 0, 10124, 7526, 8601, 0, 68246, 0, 129318, 1418, - 10885, 0, 0, 0, 0, 0, 0, 4571, 0, 0, 0, 12078, 41597, 0, 10933, 0, 72129, - 67479, 0, 0, 41599, 0, 0, 0, 12950, 119190, 10498, 0, 66782, 4239, 0, 0, - 66511, 68066, 2637, 110685, 8460, 110683, 8476, 110681, 0, 110679, 0, - 127919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 66243, 9935, 122892, 0, - 73864, 41734, 8206, 74081, 0, 3286, 120730, 0, 0, 41732, 0, 41736, - 983203, 41731, 0, 0, 70842, 0, 0, 0, 0, 129329, 0, 66853, 0, 0, 78742, - 72755, 11277, 65892, 0, 10620, 92272, 68165, 0, 0, 0, 73942, 67001, - 100479, 0, 119093, 3459, 0, 129398, 0, 0, 72130, 92512, 0, 66377, 69781, - 128718, 0, 111304, 3161, 69981, 0, 0, 0, 0, 0, 9016, 78153, 0, 0, 43641, - 0, 121018, 0, 101279, 0, 0, 0, 0, 0, 68342, 120950, 94043, 0, 12332, - 121310, 6086, 41722, 0, 120709, 0, 0, 111305, 118677, 0, 128307, 74288, - 0, 74546, 124123, 129178, 124125, 92224, 42460, 0, 0, 0, 0, 120941, - 42421, 0, 41723, 110606, 64358, 11460, 983511, 0, 64718, 120838, 66869, - 0, 42348, 0, 6752, 452, 42500, 0, 128258, 0, 42308, 0, 0, 0, 12932, 0, - 69968, 42950, 66827, 917582, 0, 0, 8302, 0, 66929, 0, 0, 7250, 13214, - 10041, 8105, 65568, 127780, 69969, 127759, 0, 0, 121467, 0, 121466, - 41384, 0, 69878, 0, 5538, 9987, 111298, 118932, 129307, 0, 552, 0, 7357, - 10785, 66995, 0, 4557, 0, 0, 10171, 68320, 0, 5540, 0, 0, 281, 0, 0, - 42622, 0, 5536, 0, 0, 1388, 0, 0, 10504, 0, 0, 11531, 74324, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3663, 0, 121081, 70335, 74859, 0, 5334, 0, 110738, - 72319, 0, 11305, 66997, 68456, 0, 66611, 0, 19907, 64363, 3478, 7583, - 7679, 74154, 0, 0, 1158, 0, 983890, 73748, 0, 0, 1915, 4846, 0, 120132, - 118984, 120134, 120129, 120128, 805, 120130, 64438, 120124, 8760, 120126, - 72137, 120120, 120123, 94003, 0, 0, 0, 0, 0, 12225, 0, 0, 0, 70173, - 75045, 0, 129515, 8083, 0, 0, 0, 111094, 92626, 0, 0, 0, 0, 0, 0, 110837, - 0, 67699, 560, 5643, 0, 0, 0, 0, 0, 0, 0, 120144, 0, 120661, 78304, 1597, - 120143, 120142, 206, 70126, 120139, 120138, 8168, 0, 73086, 0, 0, 0, - 118650, 125036, 0, 0, 3546, 42573, 66811, 67000, 0, 128397, 8400, 0, 0, - 0, 0, 0, 7903, 9287, 72791, 0, 0, 0, 0, 72134, 66603, 1695, 917861, - 124150, 0, 111101, 0, 0, 0, 0, 0, 0, 0, 111099, 0, 111098, 4754, 0, - 69222, 128229, 0, 0, 7354, 7408, 0, 0, 121181, 0, 0, 0, 12739, 0, 1278, - 4187, 0, 42119, 42120, 0, 121158, 0, 12467, 0, 68902, 0, 12463, 0, 0, - 118827, 0, 9664, 70834, 74475, 0, 0, 0, 0, 0, 3661, 0, 0, 9022, 127955, - 0, 101460, 126257, 0, 6118, 222, 126250, 3884, 0, 74151, 0, 6502, 0, - 11085, 121261, 0, 0, 0, 0, 0, 0, 0, 0, 12461, 0, 0, 0, 94059, 11254, - 10860, 64880, 0, 64685, 0, 0, 94087, 7776, 11219, 0, 0, 121339, 69730, - 801, 43165, 0, 78212, 0, 0, 13277, 0, 12951, 0, 9906, 5486, 2334, 128672, - 67680, 5483, 73732, 120884, 119128, 2256, 0, 127876, 2539, 0, 78507, - 5485, 69826, 42697, 0, 0, 113689, 4502, 68057, 253, 73672, 0, 0, 9203, 0, - 0, 0, 0, 0, 121242, 11127, 0, 0, 0, 13257, 0, 0, 0, 69645, 0, 0, 0, - 70431, 0, 5693, 64470, 0, 66610, 67678, 0, 983678, 0, 0, 0, 0, 0, 0, 0, - 94078, 0, 0, 66608, 3111, 0, 8804, 66607, 0, 0, 0, 66606, 0, 0, 0, 1436, - 0, 55226, 0, 111287, 7393, 41592, 0, 0, 1598, 78101, 0, 0, 65193, 4423, - 0, 113692, 10515, 41589, 0, 0, 0, 101485, 1430, 101486, 0, 120606, 0, - 66223, 7619, 3255, 128280, 74032, 11549, 10735, 93038, 100741, 6801, - 100743, 100746, 2148, 100748, 100747, 100750, 100749, 0, 121229, 101479, - 69243, 41724, 67716, 69669, 41690, 111269, 983666, 8380, 100355, 983849, - 0, 101480, 0, 0, 0, 0, 6333, 111264, 42315, 0, 129502, 111265, 0, 0, - 5339, 74323, 0, 13004, 0, 0, 0, 0, 0, 0, 5684, 0, 0, 0, 5689, 0, 0, - 68464, 12633, 12870, 0, 65183, 5688, 0, 0, 6310, 5686, 0, 0, 0, 120647, - 70046, 50, 94095, 9871, 0, 0, 121446, 0, 0, 0, 66905, 0, 4448, 0, 121406, - 113734, 72125, 1321, 0, 10640, 0, 0, 101497, 0, 0, 73542, 0, 0, 0, 0, 0, - 12501, 0, 0, 0, 0, 8812, 0, 69986, 8673, 0, 129024, 0, 0, 2105, 72101, - 72712, 0, 129929, 0, 0, 0, 4636, 55262, 77745, 4515, 2382, 0, 0, 7313, - 101477, 0, 0, 194626, 0, 0, 0, 0, 0, 0, 0, 10197, 194719, 0, 0, 0, - 194718, 0, 0, 0, 64189, 0, 1873, 0, 0, 0, 0, 0, 983682, 0, 0, 101499, - 72282, 126991, 71113, 0, 0, 129340, 9489, 0, 70843, 0, 0, 0, 0, 128030, - 13295, 43191, 0, 0, 1154, 0, 1205, 0, 0, 0, 12958, 0, 0, 0, 66968, 0, - 10592, 0, 495, 0, 41712, 7983, 0, 0, 0, 6347, 69465, 7654, 41710, 4196, - 0, 0, 41709, 73772, 70832, 0, 9465, 983783, 0, 0, 917612, 0, 72374, - 41714, 0, 0, 0, 6343, 72376, 0, 43996, 0, 8044, 66979, 0, 41789, 0, - 10809, 71953, 0, 0, 0, 8146, 11025, 0, 120513, 642, 0, 0, 0, 12875, 0, 0, - 13229, 71950, 41788, 0, 92835, 0, 41791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8428, 6569, 92851, 0, 0, 0, 10167, 0, 68248, 8049, 0, 0, 0, 0, 128882, - 4761, 0, 4766, 64623, 0, 121180, 194653, 118876, 0, 6912, 9232, 7033, 0, - 0, 41545, 0, 71970, 72160, 72107, 0, 0, 0, 3484, 0, 0, 0, 8503, 41539, - 41527, 0, 0, 983842, 0, 0, 66983, 41537, 0, 41541, 8282, 11817, 129965, - 128219, 0, 0, 126132, 0, 0, 70115, 66609, 111235, 65921, 0, 0, 194664, 0, - 129326, 77970, 42246, 75030, 120605, 0, 65926, 7744, 68859, 94056, 74277, - 126108, 0, 6966, 194633, 8136, 0, 0, 0, 0, 0, 4762, 0, 0, 0, 4765, 69443, - 983585, 66970, 4760, 0, 0, 10871, 43199, 194645, 0, 93955, 0, 0, 11546, - 0, 337, 0, 0, 0, 12279, 7768, 0, 128352, 0, 69812, 10143, 7883, 121444, - 7880, 64618, 13012, 5704, 13010, 0, 0, 119531, 0, 0, 0, 0, 66654, 0, 0, - 0, 13008, 0, 4385, 0, 13011, 0, 92569, 66972, 13009, 74771, 70159, 0, 0, - 41793, 64450, 74221, 120996, 41792, 111242, 94054, 126094, 0, 111244, - 5709, 120689, 71076, 0, 0, 0, 0, 0, 5708, 0, 0, 0, 5706, 66362, 5705, - 8791, 41797, 0, 10237, 66436, 0, 66974, 0, 0, 128083, 13170, 0, 127075, - 0, 0, 41377, 0, 0, 10058, 120735, 101431, 0, 0, 0, 0, 0, 0, 129641, - 119525, 0, 0, 72350, 0, 983584, 2144, 0, 120765, 0, 0, 1754, 92226, - 13246, 864, 0, 118926, 8972, 0, 7849, 0, 0, 13240, 0, 5192, 0, 0, 10948, - 0, 13199, 0, 1236, 13208, 13261, 13189, 13188, 93993, 0, 7440, 66976, 0, - 0, 1844, 125229, 0, 13178, 0, 0, 0, 125230, 0, 0, 13260, 4550, 121249, - 125227, 0, 71071, 0, 0, 68523, 0, 0, 11354, 94071, 0, 42795, 129317, 0, - 0, 0, 125237, 0, 13194, 13274, 0, 0, 129533, 65586, 68311, 0, 119193, - 4601, 194661, 101454, 194658, 0, 194659, 0, 121422, 128790, 194657, - 41717, 67402, 101444, 121129, 41716, 127376, 7910, 0, 0, 754, 41944, 0, - 8183, 120741, 2037, 101440, 0, 101441, 125, 0, 0, 0, 983125, 101442, - 41719, 0, 7990, 12637, 13258, 9536, 71056, 0, 4427, 0, 71200, 0, 12217, - 0, 41532, 129315, 0, 0, 0, 0, 111063, 83349, 0, 0, 120622, 0, 0, 0, 0, - 43632, 0, 0, 8140, 0, 6260, 0, 0, 66765, 129657, 0, 3898, 0, 0, 13200, 0, - 0, 66582, 0, 0, 0, 0, 1068, 71178, 13259, 12945, 0, 42203, 0, 3124, - 69411, 0, 4386, 12224, 6973, 129563, 0, 0, 119535, 0, 121312, 0, 12232, - 0, 0, 5681, 64578, 75023, 72016, 13209, 0, 0, 0, 0, 0, 11053, 0, 74902, - 128107, 128942, 7588, 0, 1693, 74942, 43204, 65831, 124120, 0, 0, 68803, - 111216, 111223, 0, 0, 65685, 9523, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 13191, - 0, 3500, 3139, 100643, 3170, 100645, 100644, 66934, 100646, 13006, 64433, - 0, 100650, 941, 0, 0, 120967, 3727, 0, 0, 0, 72378, 0, 0, 118611, 94039, - 129299, 92455, 0, 0, 64444, 0, 0, 43603, 94075, 65397, 288, 0, 0, 0, - 10025, 73692, 0, 0, 68182, 0, 0, 0, 92438, 65395, 0, 0, 0, 65393, 83078, - 121111, 0, 122666, 0, 0, 0, 65394, 11548, 72305, 0, 65396, 0, 0, 13256, - 1282, 0, 0, 0, 111085, 0, 0, 0, 111087, 72115, 0, 0, 0, 0, 0, 3304, 0, 0, - 0, 126595, 72437, 68353, 0, 0, 42113, 0, 0, 0, 0, 0, 43094, 0, 0, 94037, - 68317, 9035, 0, 0, 0, 0, 0, 70822, 128467, 164, 68309, 94067, 94000, - 100631, 100634, 100633, 100636, 100635, 100638, 100637, 68808, 100639, - 110665, 73893, 11099, 110664, 13175, 13207, 0, 127552, 0, 74643, 5929, 0, - 0, 119502, 983654, 11306, 0, 119059, 3180, 125102, 0, 0, 0, 13062, 0, - 129551, 128707, 0, 0, 74428, 0, 128000, 0, 11251, 70204, 0, 10045, 0, - 13275, 0, 11057, 0, 13276, 125133, 41525, 983084, 128015, 11444, 0, - 129158, 0, 122642, 41523, 127765, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 119573, - 0, 0, 0, 0, 0, 0, 101014, 369, 74908, 41784, 0, 120994, 0, 71180, 0, 0, - 13210, 41782, 0, 73536, 101388, 41781, 10486, 74058, 43002, 0, 0, 0, 0, - 0, 3741, 0, 0, 0, 118540, 41222, 0, 128317, 3982, 0, 4388, 126105, 746, - 0, 0, 0, 13131, 0, 0, 0, 0, 0, 10434, 8794, 122963, 0, 0, 0, 0, 0, 11700, - 4374, 129413, 0, 0, 0, 0, 0, 917597, 0, 69814, 0, 6735, 73979, 13174, - 73968, 13225, 0, 69808, 0, 0, 2365, 7841, 71476, 0, 120934, 66510, - 128099, 0, 0, 0, 41785, 41171, 0, 13173, 4372, 6854, 0, 0, 0, 128939, 0, - 0, 12965, 384, 0, 0, 12685, 41473, 0, 13242, 13236, 0, 0, 0, 41787, 0, - 70684, 0, 68486, 13272, 0, 13232, 13233, 65838, 0, 0, 11656, 0, 126110, - 119885, 12861, 0, 13271, 0, 92737, 1096, 0, 0, 0, 0, 0, 0, 2255, 5203, 0, - 92902, 0, 13243, 13237, 12719, 0, 0, 0, 64884, 78043, 43052, 0, 0, 0, - 12014, 0, 101415, 0, 0, 13195, 41452, 64961, 41535, 42969, 10459, 0, - 124949, 0, 0, 0, 41533, 66337, 0, 92184, 0, 126091, 0, 0, 73849, 0, - 43638, 0, 101398, 6261, 0, 129568, 0, 1957, 0, 0, 0, 13292, 13206, 0, 0, - 2925, 73809, 42576, 101395, 13212, 43238, 0, 13190, 13187, 0, 13198, 0, - 0, 5242, 0, 0, 128146, 0, 73535, 6770, 43331, 127539, 0, 0, 71074, - 126466, 73524, 41444, 0, 0, 64799, 5246, 119106, 13185, 9709, 0, 0, - 92751, 0, 5238, 0, 71085, 0, 5236, 40979, 0, 74201, 8286, 0, 3936, 92833, - 11699, 0, 127249, 13235, 69578, 41248, 127264, 13245, 13239, 0, 7969, - 127266, 74832, 127251, 0, 120509, 0, 983893, 734, 127270, 0, 127254, - 70297, 127273, 64921, 120969, 66631, 41771, 120490, 0, 983172, 41770, - 1670, 42560, 0, 121349, 129634, 0, 41163, 0, 11136, 0, 11506, 0, 42841, - 13267, 126109, 0, 41775, 0, 7130, 41773, 0, 0, 0, 0, 0, 0, 0, 42673, - 65572, 0, 65250, 13265, 13264, 64518, 66798, 6100, 0, 0, 6740, 71080, - 67814, 12967, 70028, 68101, 4583, 0, 0, 68097, 0, 0, 0, 0, 119211, 0, 0, - 42653, 83181, 68102, 0, 7814, 71045, 0, 73702, 0, 0, 0, 9756, 6985, 0, 0, - 74219, 0, 0, 129069, 124987, 5674, 0, 66421, 0, 5677, 5588, 0, 0, 0, 0, - 5673, 73488, 5676, 0, 94048, 0, 5672, 6476, 0, 128798, 110951, 42511, - 1727, 0, 0, 0, 0, 0, 0, 0, 3550, 736, 0, 4505, 5873, 74090, 5826, 55232, - 5813, 0, 120712, 5841, 5837, 55234, 0, 3105, 64370, 5838, 5796, 0, - 119592, 5793, 0, 5866, 5797, 41011, 5865, 0, 0, 71899, 0, 71235, 5806, - 73528, 0, 9037, 5671, 0, 0, 0, 0, 71266, 126616, 7296, 0, 0, 0, 0, 6980, - 0, 72108, 0, 0, 0, 0, 0, 64613, 983910, 0, 129969, 0, 78277, 7114, 0, - 72100, 43190, 93842, 128666, 72096, 42611, 42563, 0, 125080, 0, 6792, - 43201, 72098, 0, 128719, 0, 72106, 73534, 0, 5644, 0, 66627, 69727, 0, 0, - 0, 65116, 0, 0, 73526, 0, 66410, 94104, 41013, 0, 0, 0, 2869, 0, 41015, - 0, 2785, 120616, 0, 73907, 194689, 0, 0, 0, 194688, 4759, 0, 0, 43192, - 129913, 1170, 43365, 69810, 73908, 0, 902, 0, 0, 0, 0, 8122, 66420, - 129642, 0, 3861, 0, 11028, 0, 73820, 5714, 0, 0, 0, 807, 127001, 78474, - 0, 976, 113782, 0, 0, 0, 0, 0, 128657, 118801, 71043, 0, 127017, 0, 0, - 5582, 0, 0, 5798, 0, 0, 0, 128521, 0, 0, 68058, 120553, 983184, 0, 0, - 74933, 74283, 0, 0, 194698, 66044, 0, 0, 0, 0, 0, 10094, 0, 0, 10857, - 69225, 0, 0, 93, 0, 10954, 0, 0, 0, 8171, 0, 0, 82996, 0, 0, 0, 73527, - 92634, 0, 0, 5187, 120711, 71086, 118704, 0, 0, 0, 5232, 0, 41009, 0, - 41005, 0, 43205, 0, 0, 0, 194708, 0, 71054, 10028, 66478, 7076, 13182, - 100385, 0, 0, 0, 78782, 7972, 78786, 0, 0, 0, 78789, 11309, 3806, 71252, - 0, 0, 0, 78819, 0, 125218, 0, 127532, 0, 0, 0, 78817, 0, 64366, 65156, - 8814, 0, 0, 0, 0, 12836, 42725, 120079, 0, 0, 0, 0, 69258, 13255, 0, 0, - 7464, 0, 93831, 0, 0, 0, 0, 13213, 118557, 0, 64516, 0, 0, 0, 41007, - 983929, 0, 40995, 12209, 983933, 119136, 123635, 0, 0, 0, 0, 0, 69283, - 43558, 5522, 0, 71061, 0, 74105, 3633, 983931, 119364, 41234, 41231, 0, - 9771, 983936, 13251, 0, 0, 6262, 2784, 0, 71078, 8126, 66483, 0, 0, 441, - 0, 0, 0, 41002, 40999, 0, 129394, 7108, 0, 10890, 0, 74445, 8324, 0, 0, - 74817, 2813, 119056, 74853, 983690, 0, 0, 0, 1193, 10462, 65197, 13253, - 13252, 7829, 120992, 130032, 0, 0, 0, 77911, 0, 77907, 0, 10386, 0, - 41042, 0, 65944, 65683, 10338, 66469, 0, 0, 0, 0, 0, 41966, 0, 0, 0, - 68915, 0, 0, 911, 983889, 128932, 40963, 0, 65159, 0, 122950, 0, 5520, 0, - 0, 0, 0, 0, 0, 0, 42965, 0, 0, 0, 0, 0, 983892, 0, 0, 66839, 0, 0, 0, - 68647, 0, 5857, 68135, 92727, 119120, 983694, 13171, 0, 0, 0, 120338, 0, - 0, 0, 13250, 69663, 0, 92201, 66397, 0, 0, 0, 8761, 12942, 5748, 92713, - 92414, 0, 83174, 8796, 0, 0, 0, 43633, 0, 72805, 71073, 0, 0, 0, 0, 0, - 12843, 4520, 0, 0, 73004, 983691, 0, 0, 194935, 110754, 64345, 0, 983677, - 3457, 0, 0, 0, 110750, 110758, 110751, 0, 0, 10427, 0, 73859, 0, 9755, - 1110, 65239, 0, 0, 0, 0, 0, 0, 0, 194936, 0, 983821, 0, 70437, 3620, 0, - 0, 72855, 0, 0, 0, 74250, 0, 0, 11980, 0, 66482, 67823, 0, 128345, - 110768, 0, 0, 0, 0, 12891, 983786, 983667, 0, 2016, 0, 65668, 92311, - 67696, 10366, 70117, 9155, 120652, 9786, 65082, 0, 8579, 0, 0, 0, 0, - 4508, 64883, 0, 92522, 129847, 0, 64592, 74276, 67688, 0, 69270, 0, - 69456, 0, 113821, 0, 12147, 9024, 66378, 66472, 0, 0, 0, 0, 0, 71935, 0, - 0, 113697, 0, 0, 69285, 0, 74275, 0, 122896, 127941, 41214, 0, 67476, 0, - 0, 0, 7773, 0, 0, 9963, 68649, 0, 73734, 0, 0, 0, 0, 6594, 983771, 0, 0, - 3624, 70342, 0, 64655, 121481, 0, 0, 0, 0, 0, 65932, 0, 983809, 6803, - 120968, 7738, 0, 0, 120628, 129721, 66614, 122921, 0, 43810, 7029, 0, - 41292, 118898, 0, 43115, 9517, 11518, 0, 0, 0, 0, 64423, 0, 0, 0, 12503, - 9591, 4516, 0, 118845, 0, 0, 129479, 43650, 983193, 69250, 0, 0, 68079, - 0, 11397, 2884, 0, 0, 12678, 0, 0, 41014, 73730, 917539, 4270, 92254, - 127836, 68205, 6633, 118947, 0, 5230, 101055, 0, 0, 983234, 121392, 0, - 92985, 0, 0, 0, 0, 415, 0, 0, 0, 0, 5183, 1877, 0, 0, 0, 0, 0, 4472, 0, - 0, 0, 128285, 110682, 78230, 4756, 0, 7081, 0, 0, 0, 78606, 0, 42922, - 42103, 8628, 74861, 0, 0, 0, 43059, 10539, 0, 0, 0, 0, 0, 0, 0, 0, 64873, - 11992, 129444, 0, 0, 11801, 3622, 0, 0, 983215, 0, 0, 11521, 0, 1966, - 43628, 111048, 0, 0, 0, 0, 0, 0, 42098, 66671, 10694, 128520, 0, 0, 0, 0, - 42100, 0, 111040, 0, 42097, 0, 0, 0, 0, 11302, 118640, 129145, 43395, - 83259, 0, 0, 92351, 0, 0, 11299, 1561, 0, 92359, 92725, 69253, 0, 194733, - 0, 194730, 0, 127893, 11280, 0, 0, 983802, 0, 0, 72760, 0, 12486, 65018, - 66516, 5409, 0, 0, 194720, 5399, 9685, 0, 983713, 5401, 0, 0, 66832, 0, - 0, 5405, 0, 0, 0, 0, 0, 2235, 0, 11330, 983711, 64690, 3254, 0, 129974, - 0, 0, 43678, 0, 0, 983146, 0, 6388, 3355, 0, 9867, 0, 55258, 5611, 0, - 128527, 0, 0, 129181, 0, 78228, 0, 0, 119119, 0, 0, 194959, 0, 0, 1379, - 246, 0, 0, 64736, 0, 0, 0, 121227, 0, 0, 0, 0, 0, 0, 11855, 0, 0, 0, - 71961, 10656, 0, 65214, 119242, 0, 0, 13163, 0, 120831, 0, 0, 101484, 0, - 0, 0, 0, 0, 4755, 0, 122627, 11443, 0, 0, 0, 608, 600, 0, 8580, 128712, - 0, 43635, 0, 129695, 74485, 43808, 0, 0, 0, 13160, 0, 129418, 42268, - 128006, 70505, 9828, 0, 69261, 0, 0, 9351, 7778, 0, 0, 0, 6916, 1208, 0, - 0, 194754, 0, 0, 0, 0, 0, 83318, 83317, 0, 43539, 0, 72024, 0, 0, 0, - 9150, 66831, 0, 128322, 0, 66848, 0, 0, 12166, 128492, 194685, 0, 2546, - 0, 213, 0, 65611, 83316, 0, 0, 74310, 70836, 0, 65285, 5452, 0, 983938, - 92772, 0, 0, 0, 0, 65518, 129029, 12609, 194679, 125255, 123193, 0, 0, 0, - 74638, 194677, 125190, 4143, 110854, 110855, 65748, 4141, 9682, 110851, - 118790, 194674, 0, 0, 8725, 0, 66638, 0, 42263, 4145, 6380, 0, 66613, 0, - 119207, 0, 0, 9550, 100621, 0, 100623, 100622, 78050, 100624, 65753, - 100626, 65756, 72731, 0, 100630, 0, 0, 0, 0, 9657, 9019, 121154, 0, 0, - 5390, 0, 0, 194965, 72144, 69937, 69286, 6328, 0, 0, 0, 0, 0, 983047, 0, - 5235, 803, 69289, 0, 0, 127979, 43838, 0, 119562, 43544, 0, 0, 0, 0, - 194960, 70426, 9107, 5191, 119113, 0, 0, 0, 121099, 0, 0, 0, 0, 0, - 128150, 983067, 0, 7289, 74055, 0, 0, 0, 0, 0, 0, 0, 1784, 124947, 0, 0, - 0, 0, 64868, 0, 13158, 0, 7211, 0, 9371, 129378, 0, 0, 1625, 7664, 0, 0, - 0, 0, 0, 0, 69273, 0, 0, 0, 0, 4482, 118886, 0, 0, 0, 0, 0, 0, 0, 100612, - 66849, 100614, 100613, 100616, 444, 100618, 100617, 100620, 100619, 0, - 129401, 0, 11349, 40991, 0, 0, 129324, 0, 0, 1197, 0, 40993, 0, 0, 0, - 40990, 43765, 0, 3492, 0, 127942, 0, 0, 100592, 100591, 100594, 19948, - 100596, 3099, 92239, 100597, 100600, 100599, 0, 129042, 0, 0, 100601, - 194969, 100603, 8152, 100605, 100604, 100607, 100606, 100609, 12828, 0, - 75015, 0, 0, 129950, 0, 0, 75068, 127507, 0, 92680, 0, 0, 129928, 129920, - 0, 130037, 0, 118820, 0, 0, 0, 0, 0, 100581, 0, 100583, 100582, 100585, - 100584, 100587, 100586, 100589, 7576, 11995, 100590, 43260, 0, 0, 64830, - 0, 125046, 101526, 0, 43979, 8870, 0, 0, 42357, 0, 0, 12822, 0, 0, 0, - 118944, 0, 0, 42637, 0, 0, 70725, 0, 129934, 0, 71344, 0, 0, 72449, - 194745, 7170, 9596, 8277, 194743, 43629, 110610, 0, 0, 983571, 123545, 0, - 66699, 42952, 0, 0, 0, 43234, 66008, 12627, 0, 0, 0, 43619, 43303, 11300, - 0, 0, 8745, 0, 7558, 71342, 100570, 0, 0, 127881, 3461, 121258, 129471, - 69264, 0, 0, 0, 73877, 74335, 124982, 0, 0, 0, 64620, 74762, 12069, - 10838, 92548, 43616, 0, 10061, 0, 64840, 10508, 209, 0, 43193, 120581, 0, - 0, 128049, 0, 10899, 69855, 100571, 100574, 100573, 100576, 993, 100578, - 100577, 100580, 100579, 100560, 100559, 7232, 0, 0, 0, 0, 0, 0, 10489, - 42166, 0, 128588, 0, 0, 4224, 7671, 41518, 121311, 0, 0, 0, 0, 64820, - 92538, 12966, 100554, 100553, 100556, 100555, 100558, 100557, 4263, 8793, - 0, 0, 41502, 0, 983, 0, 100563, 100562, 13086, 4109, 4274, 841, 5888, - 100568, 68522, 0, 43481, 0, 120926, 0, 7209, 0, 41505, 0, 78698, 127012, - 0, 2147, 0, 0, 66629, 0, 0, 1255, 4149, 0, 0, 66633, 0, 129391, 92352, 0, - 65101, 0, 0, 0, 0, 5835, 128797, 66625, 10842, 0, 42123, 0, 0, 66634, - 1094, 66636, 0, 0, 0, 0, 0, 9972, 73865, 129289, 6114, 0, 0, 0, 0, 93960, - 0, 0, 0, 0, 12070, 0, 881, 7857, 0, 65164, 0, 0, 118703, 124151, 0, - 64404, 64321, 0, 125187, 0, 0, 11245, 129395, 69506, 71859, 128886, 0, 0, - 1287, 121509, 0, 0, 0, 125264, 74152, 120504, 64545, 0, 69668, 8985, 0, - 0, 0, 0, 0, 0, 3652, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 1489, 125189, 0, 0, - 3899, 0, 42124, 43828, 42122, 0, 0, 0, 11985, 73755, 78600, 0, 0, 10988, - 0, 0, 42138, 78610, 0, 65768, 78608, 78604, 78605, 6285, 78603, 78612, - 78613, 74339, 65767, 8685, 0, 0, 0, 78622, 78623, 68475, 11470, 64538, - 78618, 78615, 78616, 0, 0, 0, 101534, 2527, 0, 128209, 2799, 0, 0, 0, - 9933, 0, 0, 767, 5524, 7028, 0, 101520, 0, 0, 0, 78633, 67481, 0, 94011, - 0, 6971, 0, 70731, 0, 0, 118979, 126075, 2434, 94018, 0, 120579, 0, 4631, - 0, 0, 6407, 0, 19931, 0, 0, 124905, 0, 3192, 0, 8414, 0, 0, 0, 124902, 0, - 9164, 66612, 93959, 8228, 124897, 0, 0, 0, 78624, 0, 0, 9993, 0, 0, - 129350, 78631, 78632, 78629, 78630, 78627, 78628, 78625, 2399, 0, 92399, - 71202, 41208, 0, 0, 8178, 2149, 3367, 0, 78640, 78641, 78636, 78638, - 78634, 6337, 0, 78909, 0, 0, 11068, 0, 9331, 0, 74798, 9181, 0, 0, 8017, - 0, 0, 0, 0, 0, 0, 0, 12126, 119494, 129306, 0, 0, 69650, 0, 0, 0, 43436, - 983744, 0, 0, 0, 0, 66845, 69249, 0, 0, 5398, 0, 127386, 93953, 0, 0, 0, - 0, 0, 9476, 68899, 0, 12763, 126603, 74788, 0, 42114, 11181, 92502, 0, 0, - 0, 3469, 42107, 42116, 0, 0, 119493, 0, 9853, 69648, 9040, 101518, 64665, - 119557, 0, 0, 0, 69638, 12602, 983068, 3852, 0, 67872, 12231, 11317, 0, - 119812, 0, 11410, 10964, 12274, 122890, 100524, 0, 119810, 9865, 195019, - 0, 0, 0, 0, 12276, 0, 124919, 0, 0, 119613, 0, 111214, 10467, 0, 2443, - 10918, 0, 0, 1001, 9241, 1927, 0, 0, 0, 127885, 195022, 0, 113752, - 119830, 65678, 0, 0, 8260, 0, 7519, 11505, 101505, 0, 518, 0, 119832, 0, - 13204, 0, 857, 121252, 0, 0, 92336, 83177, 0, 0, 0, 0, 0, 0, 92762, 0, 0, - 120613, 67247, 1629, 124926, 796, 0, 0, 74123, 72334, 127587, 72336, - 43388, 0, 43944, 72335, 478, 65151, 0, 128147, 0, 0, 0, 0, 0, 42933, - 1206, 71209, 43837, 0, 3843, 12011, 0, 3361, 0, 8121, 10715, 7578, 0, 0, - 0, 10530, 12348, 8653, 0, 73545, 0, 9551, 0, 0, 784, 0, 0, 0, 0, 0, 0, - 43937, 0, 0, 43938, 43935, 73765, 66230, 0, 0, 0, 43936, 0, 43932, 11102, - 0, 0, 42753, 67165, 0, 78324, 0, 0, 6975, 917928, 5415, 12176, 0, 0, - 3462, 43940, 42629, 78691, 128016, 43942, 0, 9759, 0, 0, 78320, 8114, - 78321, 78697, 78696, 78695, 8710, 118812, 118956, 0, 4051, 92657, 0, - 71206, 0, 0, 0, 128857, 0, 1619, 9703, 77986, 0, 42112, 0, 1875, 0, - 42109, 0, 0, 71189, 121160, 64907, 5396, 13144, 0, 0, 5575, 9675, 0, - 5940, 226, 0, 6336, 0, 0, 0, 5116, 64521, 0, 0, 0, 121390, 125048, 74138, - 0, 74139, 128447, 92249, 0, 0, 0, 0, 8935, 0, 0, 0, 0, 616, 78131, 65178, - 4684, 78701, 983899, 74631, 0, 0, 0, 74460, 42110, 0, 10870, 8557, 11054, - 68664, 0, 0, 0, 122629, 0, 0, 0, 0, 65597, 0, 7651, 6846, 0, 0, 68868, 0, - 0, 118966, 129302, 40997, 127218, 0, 0, 40998, 0, 74488, 71182, 9800, 0, - 0, 0, 41000, 0, 5114, 55263, 3386, 70730, 42574, 0, 5115, 5394, 0, - 128756, 5113, 0, 64855, 0, 4425, 0, 0, 0, 43967, 0, 0, 0, 5112, 12173, - 127037, 0, 0, 74998, 118668, 0, 0, 0, 0, 64874, 43964, 1587, 0, 0, 0, 0, - 1369, 917931, 9959, 0, 43963, 4560, 0, 0, 0, 0, 124896, 0, 43961, 42601, - 4514, 72149, 0, 0, 0, 65041, 10965, 120905, 0, 0, 12542, 0, 65341, 0, - 65829, 0, 0, 10475, 0, 0, 0, 0, 11795, 0, 0, 2164, 127102, 127101, 74956, - 7099, 11275, 67681, 127096, 0, 9336, 0, 42626, 43966, 7798, 64474, 64259, - 0, 5730, 119809, 43018, 983175, 93796, 0, 0, 0, 69401, 0, 0, 5127, 11285, - 0, 5495, 4273, 0, 74765, 10849, 6346, 5493, 6342, 68636, 74319, 5492, 0, - 0, 169, 5497, 125053, 0, 0, 68198, 0, 0, 128417, 0, 0, 12738, 0, 983076, - 5321, 0, 0, 0, 5323, 120732, 9773, 125209, 4683, 74318, 0, 68823, 0, 0, - 0, 0, 129553, 0, 123562, 0, 0, 834, 0, 1803, 0, 5733, 0, 0, 71312, 5731, - 1381, 2891, 128639, 0, 127212, 64525, 0, 2881, 92996, 93847, 9601, 2879, - 0, 0, 73129, 5729, 0, 0, 0, 64881, 127905, 9361, 0, 2887, 0, 3526, 6298, - 0, 121219, 0, 0, 0, 8572, 127863, 77896, 0, 71174, 0, 0, 71197, 0, 12096, - 0, 0, 0, 110745, 71176, 110746, 65279, 0, 121236, 5734, 0, 0, 0, 0, 0, - 41641, 12717, 0, 12552, 983615, 66713, 0, 0, 41643, 110747, 0, 8713, - 41640, 78657, 41645, 66712, 125196, 0, 66726, 66711, 0, 93994, 0, 3472, - 64863, 0, 121424, 0, 0, 0, 125203, 67837, 0, 0, 0, 0, 0, 0, 121440, 0, 0, - 129461, 119008, 92402, 65017, 0, 0, 66668, 0, 0, 0, 0, 0, 119822, 0, 0, - 124148, 0, 0, 0, 0, 0, 0, 0, 0, 121043, 66471, 12216, 0, 40988, 0, 0, 0, - 0, 0, 2396, 129078, 0, 0, 0, 64940, 0, 8321, 119823, 128165, 100409, - 83299, 996, 0, 0, 4249, 0, 83294, 92535, 8222, 0, 118875, 71213, 0, 0, 0, - 0, 8534, 72844, 40983, 0, 125195, 0, 12551, 73960, 125193, 74469, 12558, - 121039, 0, 10052, 40982, 129371, 0, 0, 0, 127403, 0, 917559, 0, 78364, - 1563, 0, 0, 19911, 0, 0, 0, 71363, 0, 7797, 78708, 10006, 0, 3308, - 119134, 74940, 0, 0, 78488, 0, 0, 0, 0, 0, 128462, 9200, 10046, 9612, 0, - 8218, 66496, 0, 43742, 78489, 0, 0, 0, 0, 67826, 0, 70056, 508, 128585, - 0, 126539, 0, 0, 0, 0, 0, 0, 0, 124950, 0, 194601, 0, 0, 0, 0, 6659, 0, - 0, 0, 0, 0, 0, 41634, 0, 41639, 71169, 11941, 0, 0, 0, 42180, 68505, - 43753, 3249, 41637, 93982, 12328, 501, 93985, 10601, 129783, 6503, 0, - 92192, 0, 71181, 0, 6505, 74010, 0, 13064, 126112, 121105, 6500, 5526, 0, - 128949, 0, 0, 92376, 0, 9678, 120832, 0, 41706, 0, 0, 0, 8936, 92964, - 119123, 4208, 0, 0, 0, 67742, 0, 74379, 128605, 0, 0, 92422, 983110, 0, - 66475, 0, 5027, 0, 0, 0, 5069, 0, 5028, 0, 0, 0, 5026, 0, 0, 6331, 0, 0, - 0, 0, 41076, 0, 74790, 0, 0, 0, 0, 5029, 0, 5317, 3598, 0, 41070, 92166, - 11185, 6663, 0, 6507, 0, 126079, 0, 1716, 983710, 0, 917824, 620, 41001, - 0, 917823, 43758, 0, 71116, 5024, 0, 41003, 0, 5025, 7297, 122988, 75039, - 69745, 119328, 65557, 0, 0, 983599, 0, 0, 0, 0, 43947, 43946, 0, 0, - 128363, 6105, 0, 119325, 983230, 0, 68203, 43945, 66491, 43939, 0, 68144, - 78718, 2301, 0, 0, 66490, 6979, 101561, 7721, 0, 0, 1592, 0, 0, 121096, - 41048, 129358, 829, 0, 92406, 0, 73541, 0, 41056, 0, 118665, 10953, - 41066, 0, 917813, 482, 101554, 0, 0, 43606, 71185, 0, 917926, 0, 72262, - 110863, 72421, 12050, 0, 5315, 917817, 0, 0, 42061, 917816, 0, 0, 68417, - 917815, 0, 0, 42059, 0, 0, 120723, 42058, 3960, 11043, 11337, 121358, 0, - 92824, 3958, 101568, 0, 917818, 0, 917819, 0, 0, 42064, 11959, 983714, 0, - 0, 0, 0, 73511, 64336, 10478, 92629, 70350, 118692, 0, 0, 42437, 1555, 0, - 8691, 129656, 2215, 41662, 119046, 0, 0, 0, 93952, 0, 66481, 41664, 0, - 42578, 0, 41661, 78715, 78714, 9356, 0, 129544, 0, 1286, 110701, 0, 0, - 983208, 128925, 42476, 0, 11156, 78895, 0, 0, 101583, 72123, 0, 10020, - 43359, 72827, 0, 120946, 41627, 0, 11979, 0, 41628, 533, 11931, 65225, 0, - 125122, 129994, 0, 68118, 0, 4377, 0, 0, 8587, 72097, 13193, 64350, - 68233, 0, 41924, 0, 7735, 0, 127585, 120843, 0, 65820, 0, 0, 43461, 7757, - 0, 0, 43787, 66493, 77943, 4168, 43904, 73952, 0, 0, 121072, 4440, 43902, - 77948, 66837, 77946, 43903, 77944, 77945, 0, 120909, 120826, 120226, - 66492, 43901, 64625, 0, 0, 0, 0, 10013, 64434, 0, 983113, 0, 11782, - 64382, 0, 0, 0, 0, 41630, 630, 120960, 0, 0, 70165, 1043, 93017, 0, 0, 0, - 124945, 313, 129590, 0, 0, 65593, 7445, 43906, 5750, 42258, 0, 55222, - 68222, 11268, 11225, 0, 8526, 0, 0, 43894, 66495, 69990, 0, 92990, 0, - 10707, 7863, 0, 0, 70692, 631, 77952, 77953, 66443, 71171, 83313, 0, 0, - 0, 13305, 77961, 43925, 43924, 77956, 77957, 66903, 66328, 42381, 77962, - 0, 0, 0, 0, 0, 0, 43899, 66821, 77967, 9157, 77965, 77966, 77963, 77964, - 0, 0, 180, 73904, 0, 0, 66494, 12674, 43896, 0, 0, 43890, 43897, 0, - 11535, 0, 66769, 5185, 7165, 5521, 10334, 5519, 71329, 10302, 12351, - 83333, 1027, 5181, 0, 5117, 2186, 5179, 73955, 6845, 991, 3332, 43676, - 41647, 0, 73883, 92571, 77979, 3405, 69572, 0, 5523, 43915, 66487, 92459, - 74943, 9549, 0, 125093, 43923, 0, 43682, 74884, 120537, 0, 43921, 0, - 71184, 0, 43922, 128709, 0, 10414, 9846, 0, 10350, 0, 43918, 77981, - 75075, 77978, 77980, 66485, 77977, 77973, 77974, 78057, 43909, 73983, - 12330, 0, 0, 0, 43910, 69291, 3407, 6293, 0, 68149, 43908, 129060, 0, - 10209, 0, 4195, 0, 9010, 983705, 75072, 6332, 0, 0, 65871, 0, 1736, 0, - 3901, 0, 0, 65890, 128801, 10446, 0, 693, 9130, 314, 78119, 64149, 0, 0, - 0, 11026, 0, 5332, 6940, 0, 0, 127007, 119831, 0, 273, 8165, 118551, - 83307, 0, 0, 12824, 43911, 4528, 5320, 6301, 43662, 6133, 0, 9463, 73738, - 127141, 10922, 121069, 0, 0, 0, 0, 0, 2569, 0, 2326, 0, 2565, 0, 66401, - 0, 0, 0, 0, 41848, 2567, 78620, 121145, 4044, 92646, 0, 12233, 0, 9509, - 0, 0, 127158, 7336, 0, 0, 0, 129598, 0, 67235, 0, 0, 0, 0, 2222, 66499, - 0, 127170, 0, 10895, 118682, 274, 983782, 1858, 0, 67849, 55251, 2172, - 3133, 0, 71857, 0, 9610, 0, 8197, 0, 0, 0, 41665, 5868, 0, 0, 72120, 0, - 19940, 43668, 41667, 0, 0, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 6464, 92750, - 2996, 125221, 0, 68481, 41835, 4047, 41842, 0, 0, 129601, 0, 0, 0, 0, - 293, 0, 0, 64791, 41827, 0, 0, 10579, 8560, 0, 0, 118835, 4803, 73805, - 1739, 0, 3900, 128967, 73737, 0, 72451, 73957, 0, 66474, 41971, 0, 0, 0, - 0, 0, 11716, 66473, 0, 92647, 0, 78920, 0, 0, 0, 0, 0, 0, 0, 6632, 73861, - 0, 74770, 0, 0, 8914, 0, 0, 3183, 1435, 0, 0, 0, 0, 0, 0, 5746, 67392, 0, - 0, 0, 83506, 0, 7082, 71481, 12618, 5059, 983597, 83524, 43604, 0, 0, 0, - 0, 0, 0, 8227, 0, 1218, 0, 64416, 65848, 92884, 0, 0, 0, 126987, 0, 0, 0, - 0, 0, 0, 83515, 83507, 0, 0, 42672, 71194, 43224, 0, 0, 0, 0, 0, 0, 0, - 65905, 0, 42662, 0, 121159, 0, 129536, 0, 7794, 0, 42953, 6377, 0, - 126080, 3669, 3968, 0, 71319, 69658, 129550, 0, 66296, 118616, 0, 0, 0, - 124998, 6699, 126120, 0, 0, 66678, 0, 0, 0, 8409, 119527, 19967, 0, 0, - 9502, 0, 0, 6115, 0, 41654, 0, 0, 0, 41655, 113779, 43975, 72427, 128080, - 0, 0, 0, 41657, 10778, 0, 9533, 184, 1553, 128868, 69574, 0, 0, 0, - 129420, 0, 101589, 983576, 73697, 0, 92480, 0, 128938, 74292, 0, 5157, - 4020, 0, 128154, 43788, 64818, 0, 0, 0, 92979, 0, 0, 74377, 11029, 66651, - 0, 0, 125202, 0, 0, 7877, 121070, 101411, 0, 119828, 2810, 9955, 0, - 69375, 42817, 0, 65122, 11715, 0, 0, 0, 71270, 0, 0, 0, 0, 0, 70199, 0, - 0, 0, 0, 0, 0, 127862, 0, 0, 0, 78222, 127981, 0, 0, 0, 0, 0, 11290, 0, - 0, 0, 0, 8315, 0, 0, 0, 74595, 0, 0, 0, 42531, 0, 0, 0, 74589, 43993, 0, - 0, 0, 0, 43690, 0, 119139, 42730, 0, 0, 0, 64926, 0, 0, 43830, 65257, 0, - 42728, 0, 128697, 123150, 0, 43540, 0, 0, 12725, 72993, 78635, 127826, - 223, 0, 69675, 0, 0, 0, 0, 0, 0, 42605, 0, 0, 0, 0, 0, 0, 0, 0, 78621, 0, - 78619, 119062, 0, 0, 0, 42676, 129353, 64800, 78617, 83504, 68126, 1213, - 0, 0, 797, 0, 0, 83021, 83005, 64387, 4115, 0, 0, 0, 129857, 10679, - 83001, 121091, 0, 64276, 83498, 13168, 83011, 0, 10136, 0, 0, 65088, 0, - 4262, 129866, 0, 0, 10701, 0, 3101, 0, 123204, 0, 0, 11373, 0, 0, 12731, - 9117, 0, 0, 4539, 0, 0, 12727, 0, 0, 0, 43684, 74567, 68877, 983726, - 12724, 73940, 0, 0, 0, 0, 0, 7947, 12003, 0, 74593, 121140, 69653, 74807, - 42018, 0, 0, 0, 65888, 0, 0, 69683, 0, 120306, 0, 0, 12595, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 69848, 68307, 0, 4405, 0, 128336, 129032, 69216, 0, - 128011, 118656, 0, 6817, 67400, 120314, 0, 0, 998, 0, 13105, 120313, - 64327, 1558, 0, 1991, 7882, 0, 0, 0, 530, 0, 0, 0, 12002, 0, 68422, 0, - 10979, 0, 41823, 70696, 0, 0, 7896, 0, 66676, 0, 120325, 0, 0, 129407, - 94033, 0, 6311, 110725, 41698, 0, 12049, 78133, 0, 125020, 41705, 0, 0, - 121298, 0, 66822, 0, 65389, 0, 66027, 0, 0, 41699, 8340, 0, 69776, 0, - 78921, 0, 1988, 5407, 69978, 0, 65912, 93059, 0, 2336, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 126238, 0, 19913, 0, 113733, 0, 0, 74279, 0, 10956, 0, 41674, - 19964, 41679, 65084, 41675, 195031, 0, 0, 0, 0, 983089, 0, 10794, 128961, - 13217, 0, 0, 0, 5280, 0, 0, 12905, 41610, 11532, 0, 0, 768, 120545, 442, - 0, 0, 0, 64081, 41682, 0, 41693, 0, 77993, 77994, 0, 4804, 6994, 983115, - 0, 0, 41696, 467, 983934, 0, 0, 0, 0, 8678, 0, 69682, 64801, 0, 0, 0, - 2193, 64093, 12043, 0, 69666, 0, 2029, 65191, 119246, 42847, 0, 0, 0, 0, - 0, 0, 0, 70339, 126116, 0, 0, 8019, 73856, 0, 0, 0, 118709, 2355, 12150, - 65725, 77988, 77989, 68033, 77987, 0, 77985, 0, 0, 68388, 0, 74171, 0, 0, - 0, 11301, 78013, 78008, 78010, 9874, 78007, 983331, 71064, 3050, 0, 0, 0, - 78016, 78017, 71852, 78015, 0, 0, 0, 92242, 0, 69642, 0, 0, 43883, 0, 0, - 0, 78025, 0, 78023, 78024, 11847, 10545, 0, 10887, 0, 123179, 0, 0, 0, - 83352, 64942, 92363, 9996, 8508, 0, 0, 8195, 0, 42171, 0, 3722, 0, 63751, - 0, 0, 92637, 69670, 0, 41552, 69854, 0, 78639, 0, 0, 129374, 128978, 0, - 0, 0, 7920, 70285, 4021, 0, 0, 0, 119663, 0, 0, 78021, 78022, 78019, - 78020, 1802, 78018, 0, 74895, 41659, 41671, 1827, 0, 64396, 41668, - 128524, 41673, 0, 11422, 71846, 0, 11370, 0, 68412, 41345, 0, 0, 0, 0, 0, - 0, 65114, 0, 2104, 64858, 0, 0, 7553, 0, 41560, 11970, 0, 917920, 0, - 68495, 74131, 74130, 0, 0, 0, 611, 74129, 64871, 129958, 0, 0, 0, 74854, - 0, 70466, 0, 0, 0, 121147, 0, 68487, 41669, 7094, 917921, 0, 123144, - 74054, 0, 0, 0, 839, 0, 7695, 0, 0, 0, 92202, 0, 121053, 123157, 67885, - 0, 7206, 0, 6647, 43986, 129743, 0, 0, 122646, 0, 0, 127936, 43748, - 66746, 0, 12298, 110802, 984011, 110800, 64924, 0, 73931, 9468, 74245, 0, - 0, 74246, 0, 0, 118830, 0, 71851, 1279, 0, 6224, 0, 92405, 128601, - 129886, 128997, 0, 0, 0, 5032, 0, 0, 0, 0, 0, 5034, 0, 0, 72846, 42702, - 0, 0, 13294, 0, 64869, 0, 67808, 9129, 123632, 0, 0, 120819, 68387, - 120168, 120169, 120170, 120171, 5518, 4174, 120166, 66932, 120160, - 120161, 120162, 434, 41437, 66212, 120158, 120159, 0, 0, 118867, 0, 524, - 0, 74029, 0, 126559, 0, 0, 0, 10355, 10419, 74025, 77847, 0, 69725, 0, - 120656, 0, 67876, 0, 0, 0, 74145, 74039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5445, 0, 93779, 71855, 7391, 8989, 0, 74068, 0, 0, 0, 0, 4962, 120409, - 8855, 0, 70820, 0, 0, 0, 0, 71847, 0, 120406, 0, 10451, 0, 67653, 120153, - 12443, 120155, 9947, 120149, 120150, 120151, 13128, 0, 120146, 120147, 0, - 0, 0, 0, 0, 129715, 74059, 74062, 6217, 74053, 43846, 0, 74049, 0, 0, 0, - 0, 0, 0, 0, 0, 42595, 0, 68112, 118860, 0, 0, 92497, 74949, 128953, - 126245, 0, 0, 0, 42997, 122984, 119251, 0, 0, 0, 0, 0, 6216, 0, 0, 9455, - 127027, 8124, 128851, 0, 6944, 0, 0, 0, 2828, 128550, 531, 42638, 0, 0, - 129888, 43428, 0, 3614, 2827, 9696, 0, 129711, 0, 4354, 0, 78562, 78561, - 0, 118553, 0, 42599, 42597, 0, 68829, 125012, 0, 127277, 0, 120421, 0, - 983165, 0, 0, 10121, 120422, 74950, 123142, 69715, 0, 0, 120423, 120630, - 12608, 125244, 0, 74144, 9700, 12580, 0, 128911, 0, 71864, 0, 74071, 0, - 0, 12713, 0, 70402, 0, 0, 0, 1734, 0, 0, 0, 119491, 118951, 231, 0, - 74167, 542, 0, 0, 0, 0, 128074, 0, 121343, 0, 4446, 10584, 74235, 0, - 4037, 0, 0, 0, 5687, 0, 0, 0, 0, 0, 0, 78434, 92816, 0, 113709, 74284, 0, - 0, 0, 126495, 0, 0, 0, 74482, 93978, 1709, 69721, 9909, 92286, 0, 0, 0, - 55229, 8667, 0, 0, 0, 0, 0, 0, 0, 0, 127586, 1226, 6930, 124146, 71736, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 41500, 0, 311, 74282, 6221, 92988, 0, 67682, - 0, 120528, 122901, 74272, 0, 0, 0, 0, 69667, 0, 124933, 74456, 74302, - 42589, 0, 0, 0, 0, 64847, 0, 66987, 0, 41508, 0, 323, 125211, 0, 42698, - 8131, 0, 4625, 0, 4630, 0, 0, 0, 74316, 78417, 2668, 92483, 0, 42640, 0, - 2519, 0, 92474, 92479, 0, 983085, 5049, 42659, 119011, 64705, 7754, - 10854, 8738, 74623, 0, 0, 0, 649, 0, 0, 73480, 0, 0, 1013, 70707, 68212, - 705, 0, 0, 127803, 1183, 126519, 9320, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, - 11913, 0, 42848, 0, 64925, 0, 0, 70693, 0, 0, 2051, 0, 0, 0, 66988, 0, 0, - 0, 8466, 0, 4626, 8464, 8472, 68844, 4629, 8499, 0, 0, 4624, 194623, 0, - 94025, 0, 7805, 0, 94007, 6935, 0, 0, 0, 0, 0, 0, 0, 8492, 0, 8459, 0, - 8497, 8496, 0, 129864, 0, 0, 129834, 69553, 73476, 0, 65849, 0, 0, 0, - 12451, 3328, 8684, 0, 6102, 0, 5298, 110881, 5294, 0, 129615, 0, 0, 0, 0, - 43617, 0, 0, 0, 0, 0, 77863, 128695, 0, 0, 0, 0, 0, 5292, 0, 0, 42688, - 5302, 3970, 73516, 0, 1793, 0, 0, 0, 0, 0, 65263, 0, 0, 0, 0, 0, 0, - 13219, 9569, 69567, 74383, 0, 0, 72157, 0, 42949, 0, 0, 0, 5322, 0, 0, - 43631, 5324, 0, 128694, 41614, 65269, 6230, 0, 0, 0, 3360, 0, 11523, - 72726, 92488, 9926, 7197, 0, 68429, 126575, 41821, 1249, 0, 127951, 0, - 123641, 0, 0, 0, 74459, 41807, 0, 41815, 0, 0, 0, 119918, 0, 128248, 0, - 66835, 0, 0, 72145, 41800, 0, 0, 0, 41811, 74466, 93966, 6670, 77882, 0, - 0, 43092, 0, 0, 0, 0, 0, 128655, 0, 0, 0, 0, 74501, 74005, 0, 74387, - 69860, 315, 12813, 128556, 72409, 0, 72408, 0, 0, 73061, 0, 0, 1378, 0, - 0, 0, 72407, 3066, 0, 0, 72406, 0, 0, 0, 8787, 194615, 0, 41618, 0, 0, 0, - 194614, 64652, 194611, 42088, 125226, 0, 0, 0, 0, 7176, 43756, 0, 122649, - 74492, 0, 74534, 0, 0, 0, 127199, 0, 128630, 74525, 0, 194594, 12930, - 7168, 74514, 0, 74515, 0, 128919, 43962, 9527, 120659, 70123, 12977, - 69723, 0, 93783, 194598, 41236, 92235, 65168, 118838, 41237, 5848, 0, - 194600, 3670, 129905, 129906, 129907, 129908, 7890, 0, 11298, 0, 0, 6229, - 0, 0, 0, 194593, 128907, 0, 0, 194592, 4120, 65337, 65336, 0, 0, 0, 0, - 9366, 0, 0, 0, 65327, 65326, 65325, 65324, 65323, 42216, 65321, 65320, - 65335, 65334, 65333, 65332, 65331, 65330, 65329, 42689, 0, 43943, 118885, - 42073, 6785, 68491, 0, 42076, 7196, 65318, 2035, 65316, 4106, 65314, - 65313, 42074, 0, 41228, 0, 0, 41241, 93786, 41239, 43533, 0, 7189, - 194602, 0, 43941, 0, 42802, 0, 8487, 0, 0, 4615, 12695, 0, 0, 12175, - 100414, 0, 0, 7809, 0, 0, 0, 0, 6590, 69762, 0, 64738, 0, 0, 0, 0, 0, 0, - 2025, 0, 0, 0, 10637, 71860, 0, 1570, 43839, 2835, 83052, 10624, 43623, - 194587, 0, 78433, 0, 42812, 0, 2825, 0, 128287, 0, 2821, 0, 92327, 7365, - 83043, 0, 68296, 0, 2823, 0, 0, 0, 2831, 0, 0, 11465, 0, 0, 0, 0, 0, - 7181, 92855, 41332, 0, 12333, 0, 0, 0, 124914, 0, 9883, 127294, 73906, - 70751, 0, 71863, 0, 0, 0, 0, 0, 0, 43741, 0, 8166, 70739, 0, 0, 74535, 0, - 65297, 68294, 571, 0, 8752, 0, 5288, 118822, 1541, 0, 127284, 8864, 0, - 73559, 0, 0, 0, 113778, 12151, 0, 66874, 0, 1035, 0, 0, 7881, 701, 65936, - 128493, 0, 70462, 0, 11403, 0, 0, 82991, 0, 983143, 70472, 3994, 11421, - 121217, 127297, 127242, 127300, 70659, 127303, 0, 125205, 2855, 127828, - 0, 41621, 68214, 0, 0, 10654, 82945, 119226, 12164, 41623, 7906, 0, - 74297, 7182, 0, 83069, 0, 0, 0, 0, 121115, 0, 0, 747, 0, 92463, 12019, - 43136, 0, 110861, 0, 0, 8001, 0, 0, 69394, 0, 0, 0, 68373, 0, 0, 0, - 128279, 0, 71915, 0, 129742, 7282, 94066, 0, 0, 0, 0, 0, 5286, 83061, 0, - 3718, 0, 83057, 78933, 124906, 71905, 0, 128480, 0, 0, 0, 0, 9206, 82980, - 113824, 6802, 0, 41653, 0, 1241, 0, 0, 0, 0, 68124, 41651, 42937, 0, - 83042, 41650, 0, 83037, 0, 12914, 2814, 0, 119552, 120691, 0, 0, 71968, - 0, 0, 0, 917546, 71862, 0, 0, 0, 3494, 10189, 69784, 0, 0, 71861, 0, 0, - 65875, 0, 0, 127762, 0, 74215, 43065, 0, 0, 7200, 0, 3261, 0, 0, 0, - 65889, 71888, 71975, 0, 0, 0, 0, 0, 77793, 0, 0, 129424, 77791, 635, 0, - 0, 74753, 0, 92420, 73997, 0, 0, 43905, 0, 118834, 126125, 0, 6667, 0, - 983268, 0, 0, 125200, 0, 0, 0, 0, 83137, 0, 0, 0, 0, 0, 121104, 127856, - 125112, 71885, 0, 120125, 7866, 194573, 92770, 194574, 0, 120140, 126074, - 2849, 0, 0, 42157, 12960, 0, 11812, 0, 74509, 0, 69881, 0, 0, 0, 123156, - 7178, 0, 0, 0, 0, 129041, 11534, 1967, 0, 0, 71361, 7015, 120298, 72757, - 0, 12989, 0, 9368, 983638, 1624, 43270, 0, 0, 10818, 0, 83091, 0, 120908, - 0, 0, 0, 0, 0, 0, 6169, 12871, 0, 2798, 65176, 4958, 42752, 119025, 0, 0, - 0, 70346, 66448, 0, 113780, 68364, 0, 0, 0, 68360, 0, 73746, 120945, - 68352, 0, 73787, 83110, 2154, 7199, 64955, 0, 0, 0, 0, 71980, 66507, 0, - 69853, 0, 0, 0, 0, 0, 0, 0, 92517, 118882, 120301, 13297, 0, 129446, - 71963, 0, 0, 0, 6658, 8045, 0, 0, 983873, 92319, 83101, 0, 72126, 0, 0, - 0, 2416, 3310, 0, 0, 379, 0, 43755, 0, 0, 0, 68362, 1284, 0, 73756, 0, 0, - 83141, 70784, 71977, 0, 0, 0, 8515, 83144, 83143, 0, 0, 0, 8529, 93782, - 0, 7564, 0, 0, 0, 0, 73757, 73760, 42359, 0, 2031, 0, 7202, 129984, - 12676, 0, 0, 128418, 0, 7710, 1610, 73801, 0, 0, 118706, 983607, 43917, - 0, 9974, 228, 0, 10398, 0, 0, 0, 92241, 70062, 118927, 42999, 1725, - 65533, 8196, 9352, 0, 0, 66868, 0, 8502, 5762, 0, 0, 43898, 0, 0, 0, 0, - 43914, 0, 126507, 64598, 13001, 9326, 83082, 43916, 1557, 0, 983879, - 6330, 6805, 8631, 2545, 70052, 0, 0, 0, 42998, 70410, 0, 42762, 71941, - 42914, 126516, 262, 1637, 0, 83025, 129491, 83026, 128757, 0, 0, 0, - 128922, 0, 43658, 0, 0, 129183, 6419, 0, 0, 0, 0, 93989, 0, 128173, 7194, - 5291, 67395, 43666, 0, 0, 0, 0, 128293, 0, 12881, 123596, 0, 73842, 0, - 9011, 0, 0, 0, 70436, 179, 43644, 0, 0, 64747, 0, 118813, 0, 0, 121389, - 92649, 126629, 0, 73850, 2801, 119495, 42069, 119839, 119838, 119841, - 42072, 92736, 119842, 0, 0, 0, 8377, 0, 42070, 119313, 119834, 119310, - 4389, 43656, 1633, 119857, 118632, 119859, 11119, 119845, 119844, 9967, - 119846, 119849, 4612, 92867, 119850, 42913, 70456, 0, 71983, 10782, - 66898, 0, 119141, 0, 0, 0, 11541, 69636, 0, 0, 119614, 2731, 0, 0, 0, - 4102, 0, 73878, 0, 0, 0, 0, 0, 11283, 0, 0, 0, 0, 0, 43674, 0, 0, 126705, - 0, 0, 0, 0, 11142, 128304, 0, 12975, 0, 123208, 0, 0, 74072, 0, 55269, 0, - 0, 0, 78577, 78576, 0, 0, 82966, 82974, 70448, 0, 0, 82968, 0, 0, 0, 0, - 0, 113809, 0, 69399, 64909, 0, 11790, 74019, 0, 128066, 0, 8561, 94076, - 129481, 125045, 69259, 65674, 7230, 0, 0, 8778, 0, 0, 67725, 2071, 0, - 6459, 68325, 7628, 65092, 73903, 0, 11342, 129388, 0, 0, 93965, 94081, 0, - 11810, 70057, 10723, 967, 0, 71973, 73905, 0, 6387, 0, 12307, 43913, - 121089, 0, 127584, 0, 1886, 0, 43895, 870, 7648, 0, 7662, 7652, 876, 871, - 877, 7665, 878, 42015, 879, 43692, 4563, 0, 0, 0, 73072, 867, 9520, 872, - 7656, 868, 873, 7642, 7659, 869, 874, 7644, 0, 875, 790, 0, 0, 0, 0, 0, - 124899, 0, 0, 0, 0, 0, 68452, 0, 0, 42067, 0, 0, 0, 12292, 0, 0, 0, - 42012, 0, 0, 83388, 0, 0, 8494, 4611, 0, 72344, 0, 9679, 0, 0, 0, 0, - 93015, 0, 74364, 4628, 4245, 0, 0, 0, 1851, 0, 127189, 0, 0, 0, 118897, - 0, 64674, 124971, 983887, 8829, 983693, 128864, 0, 0, 0, 0, 8809, 983696, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 7427, 0, 4588, 43680, 72300, 74484, 0, 0, 0, - 0, 113787, 74363, 129043, 0, 793, 0, 11197, 0, 0, 0, 842, 0, 8208, 70833, - 0, 1647, 0, 70841, 0, 73508, 818, 0, 0, 0, 0, 0, 0, 120594, 0, 0, 70179, - 0, 13167, 66359, 0, 127172, 0, 4969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 0, - 0, 0, 66887, 65877, 9068, 0, 68194, 0, 0, 12991, 0, 2651, 68016, 983915, - 0, 983264, 70835, 0, 70844, 43648, 0, 0, 0, 0, 0, 0, 64372, 121064, 7458, - 655, 752, 7457, 7456, 7452, 3285, 74894, 11152, 73099, 0, 2391, 93766, - 92271, 671, 7435, 7434, 618, 668, 610, 42800, 7431, 7451, 42801, 640, - 42927, 7448, 7439, 628, 3905, 100742, 0, 0, 0, 67850, 0, 0, 0, 4605, 0, - 100745, 43372, 65945, 72710, 0, 119590, 0, 0, 70495, 987, 71229, 11572, - 0, 0, 10002, 9971, 70673, 0, 0, 0, 0, 0, 0, 11334, 0, 129493, 42364, - 11503, 0, 0, 0, 4627, 70090, 127784, 73473, 0, 74046, 68872, 92562, 0, 0, - 129900, 0, 129812, 0, 0, 42569, 64965, 0, 0, 10516, 129828, 12190, 0, - 42140, 0, 0, 0, 0, 9887, 0, 4000, 7429, 7428, 665, 7424, 0, 0, 7884, 0, - 0, 0, 0, 0, 2509, 0, 120573, 0, 0, 92449, 0, 10690, 0, 119114, 126226, 0, - 0, 73080, 4590, 0, 74440, 0, 0, 0, 1708, 0, 0, 983609, 0, 0, 69226, - 69974, 8813, 0, 1066, 0, 0, 71965, 127921, 70447, 0, 0, 0, 2202, 0, 7516, - 0, 0, 0, 8034, 0, 0, 3631, 110696, 0, 0, 8416, 110694, 71937, 0, 0, - 110692, 74621, 0, 70185, 0, 74850, 0, 0, 12099, 70475, 0, 6252, 0, 0, 0, - 0, 0, 0, 66368, 0, 64956, 7071, 129070, 70457, 128159, 118800, 0, 77757, - 0, 9357, 0, 1773, 0, 125092, 0, 68451, 7745, 9844, 0, 0, 94, 1880, - 120929, 0, 0, 0, 0, 0, 0, 0, 0, 11237, 0, 129173, 0, 0, 0, 1757, 6964, - 42480, 72823, 0, 120806, 0, 0, 7731, 0, 0, 127883, 0, 77777, 43988, - 70423, 74758, 0, 7592, 856, 74299, 0, 0, 0, 78138, 1459, 0, 0, 0, 0, 0, - 1504, 0, 0, 0, 0, 7529, 0, 0, 0, 0, 12594, 0, 0, 336, 0, 7509, 0, 0, 0, - 0, 127882, 0, 0, 0, 65859, 0, 983986, 43062, 124948, 0, 0, 0, 0, 12970, - 0, 0, 0, 0, 0, 0, 0, 119247, 0, 65068, 74291, 122938, 7069, 0, 0, 66977, - 11130, 2087, 0, 0, 0, 0, 126249, 0, 92747, 0, 92614, 2091, 0, 2090, 0, 0, - 7117, 2077, 72281, 0, 77889, 2083, 0, 71196, 0, 0, 71981, 0, 0, 0, 0, - 4165, 8746, 0, 0, 0, 0, 129572, 7066, 77779, 70415, 128135, 0, 0, 7786, - 127766, 2233, 0, 124965, 121122, 2302, 0, 0, 7056, 0, 0, 0, 0, 118639, 0, - 126506, 6920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983099, 70438, 2613, 0, 0, - 110734, 0, 74571, 42760, 0, 0, 0, 0, 0, 0, 71843, 0, 0, 70506, 1246, - 74243, 0, 0, 41008, 0, 0, 0, 921, 70048, 0, 12702, 119500, 0, 1566, 8407, - 0, 64653, 0, 74617, 0, 0, 72711, 5313, 951, 0, 0, 0, 0, 77807, 4009, - 70277, 71844, 0, 83123, 0, 72250, 0, 119898, 113760, 0, 0, 0, 0, 70024, - 0, 0, 119892, 0, 0, 0, 119890, 2579, 119906, 3177, 11357, 69224, 0, 0, - 83130, 64734, 0, 9822, 110670, 70471, 110668, 66990, 110666, 66967, 0, 0, - 0, 9851, 983748, 110673, 9059, 110671, 77736, 0, 41687, 129054, 0, 71842, - 70178, 0, 66975, 1777, 67003, 10158, 69767, 122982, 42366, 70444, 0, 0, - 0, 70127, 71955, 5989, 110716, 74636, 126999, 0, 41685, 0, 0, 9769, - 41684, 0, 6225, 111328, 11740, 0, 118840, 0, 2600, 0, 70416, 0, 118720, - 3666, 70420, 127193, 71976, 0, 0, 74542, 69771, 0, 0, 0, 0, 0, 69765, - 77804, 252, 0, 69769, 0, 194616, 0, 69763, 0, 0, 0, 0, 0, 0, 0, 120947, - 0, 129410, 0, 118792, 0, 68323, 125219, 0, 119188, 0, 2177, 121335, 0, 0, - 0, 0, 0, 7764, 983745, 11094, 120825, 119490, 0, 92505, 8298, 0, 0, 0, 0, - 0, 64449, 0, 126650, 0, 0, 0, 70442, 0, 0, 0, 0, 7774, 10607, 0, 0, 0, 0, - 0, 120764, 0, 0, 77746, 0, 3458, 0, 70053, 0, 120995, 0, 2602, 0, 0, 0, - 74907, 0, 0, 0, 0, 172, 0, 4971, 70419, 1889, 7238, 0, 0, 0, 8257, 0, 0, - 78917, 129570, 0, 111342, 71948, 0, 43366, 43363, 9807, 0, 0, 0, 72247, - 64479, 0, 0, 0, 113707, 0, 10900, 121355, 0, 0, 12048, 0, 64292, 0, 0, 0, - 6099, 94084, 129486, 0, 0, 299, 0, 8525, 92356, 0, 0, 111338, 0, 92564, - 3075, 0, 94053, 0, 94050, 0, 0, 70440, 0, 123590, 0, 0, 0, 2581, 11395, - 0, 0, 0, 0, 128584, 0, 0, 129423, 101092, 118855, 0, 0, 0, 7204, 70065, - 2588, 2914, 7011, 55281, 0, 7466, 0, 2883, 42253, 83118, 0, 0, 0, 83116, - 0, 41230, 68299, 0, 43571, 0, 6219, 0, 9980, 41232, 92245, 0, 66036, - 41229, 118967, 0, 120666, 94016, 0, 12711, 0, 0, 74289, 68472, 42857, - 66950, 0, 0, 0, 127306, 119006, 0, 11380, 72348, 0, 0, 0, 0, 0, 0, 0, - 983579, 12722, 0, 922, 0, 0, 983127, 74958, 3218, 120471, 120470, 120469, - 120476, 120475, 8569, 11404, 70450, 120463, 3214, 120461, 120468, 74910, - 3207, 120465, 78729, 78728, 78727, 0, 120460, 7425, 3205, 0, 78737, - 78736, 71729, 43383, 78733, 78732, 2606, 78730, 73897, 0, 11496, 1173, 0, - 0, 129135, 0, 0, 0, 120737, 120953, 120872, 120629, 378, 2610, 0, 0, 0, - 0, 0, 37, 7068, 0, 120480, 70421, 3209, 120477, 0, 120483, 9768, 120481, - 0, 0, 0, 0, 0, 0, 65510, 0, 100625, 0, 0, 0, 100627, 0, 126633, 0, 7060, - 100628, 0, 127752, 0, 69284, 70428, 71463, 0, 7380, 0, 0, 100593, 126997, - 0, 124900, 0, 71465, 121030, 3243, 0, 0, 0, 7050, 0, 70050, 0, 0, 122983, - 71466, 8203, 71102, 68241, 0, 65211, 194599, 983406, 118636, 0, 779, - 125061, 64367, 100906, 69901, 8193, 55279, 0, 0, 0, 7065, 0, 4346, 0, 0, - 908, 0, 0, 8982, 0, 0, 0, 782, 0, 10883, 0, 0, 129396, 65542, 121302, 0, - 68650, 100575, 92244, 0, 0, 111351, 0, 4376, 0, 11787, 12961, 0, 0, - 42888, 0, 100610, 6231, 0, 65713, 100608, 1783, 0, 68238, 0, 0, 0, - 194945, 0, 0, 0, 68653, 0, 983051, 0, 764, 0, 0, 43531, 0, 9033, 0, 0, - 6223, 11042, 0, 0, 0, 0, 0, 917792, 0, 0, 0, 0, 0, 0, 120648, 0, 0, 0, 0, - 0, 0, 71971, 0, 1478, 78923, 11825, 2607, 0, 0, 0, 74543, 0, 0, 100588, - 6132, 0, 0, 0, 70058, 0, 0, 0, 43537, 6761, 10093, 4369, 0, 0, 73735, - 100564, 3947, 110778, 0, 0, 0, 0, 100942, 0, 0, 0, 0, 0, 0, 7686, 0, 0, - 0, 100934, 0, 100944, 66577, 41221, 0, 42281, 0, 74024, 12293, 0, 94014, - 11794, 0, 120893, 1737, 0, 0, 0, 7205, 0, 9335, 12850, 77810, 2272, 7055, - 0, 0, 0, 67751, 0, 124910, 6780, 65067, 0, 1327, 68393, 983574, 0, 41217, - 0, 10018, 0, 0, 0, 100611, 68176, 41219, 0, 4147, 983171, 41216, 983712, - 2616, 70197, 68461, 65234, 0, 0, 0, 0, 119660, 0, 0, 0, 0, 127930, - 119580, 70675, 64943, 2608, 1470, 0, 0, 6227, 0, 0, 74775, 0, 0, 72320, - 101024, 0, 73822, 67456, 0, 0, 0, 0, 10876, 92482, 0, 0, 5834, 0, 6222, - 0, 0, 12086, 0, 1600, 64309, 0, 0, 68883, 127957, 93836, 0, 8882, 0, - 129415, 2570, 0, 0, 194606, 0, 0, 1234, 0, 13115, 110743, 110740, 100923, - 5002, 110739, 41286, 100926, 127019, 0, 0, 0, 0, 0, 0, 0, 41289, 0, 0, - 75051, 41272, 0, 0, 0, 0, 0, 124978, 0, 41279, 0, 0, 0, 11081, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9637, 7112, 77975, 128984, 0, 10886, 0, 8548, 983860, - 0, 0, 0, 8076, 43048, 8290, 8291, 43051, 92570, 0, 2596, 0, 0, 41293, 0, - 0, 2393, 7058, 66432, 0, 68673, 0, 0, 0, 0, 0, 128558, 0, 0, 0, 0, 0, - 64696, 0, 0, 121086, 74165, 0, 0, 0, 0, 0, 0, 7063, 983183, 64893, 73096, - 0, 68038, 113757, 709, 0, 0, 1876, 0, 0, 120868, 8137, 110662, 67752, - 70850, 100832, 245, 100831, 11456, 41233, 7070, 0, 94046, 6136, 100835, - 0, 100781, 41235, 73474, 0, 100782, 100642, 432, 0, 100784, 65437, 0, - 100647, 128909, 0, 100641, 100649, 0, 100648, 0, 43215, 0, 0, 0, 0, 9052, - 0, 0, 110826, 110827, 74784, 10580, 0, 100845, 0, 64640, 983176, 74455, - 0, 129670, 70035, 0, 12652, 12199, 127030, 0, 2566, 11971, 0, 0, 1065, 0, - 0, 0, 2576, 0, 66819, 0, 984005, 129852, 0, 0, 983050, 983845, 0, 2921, - 119104, 0, 5772, 12968, 70055, 0, 0, 0, 2580, 983841, 0, 0, 70032, 0, 0, - 0, 128148, 0, 0, 121308, 11346, 0, 12054, 100824, 92426, 101112, 0, - 13091, 0, 0, 100821, 100828, 0, 127026, 128334, 74821, 0, 66295, 68037, - 68047, 127865, 13090, 0, 67492, 0, 118985, 0, 0, 0, 0, 0, 127824, 0, 0, - 100776, 119319, 42356, 42432, 100778, 92823, 0, 0, 0, 78752, 70030, - 66914, 0, 0, 7061, 0, 3854, 0, 70020, 68413, 0, 42319, 0, 0, 7067, 0, 0, - 0, 0, 0, 0, 127797, 9029, 43543, 92820, 2353, 119316, 0, 100769, 0, - 100768, 983178, 0, 0, 43664, 0, 0, 0, 12277, 0, 78122, 11066, 65233, 0, - 41224, 0, 0, 3747, 10522, 0, 77722, 1691, 41226, 0, 77724, 0, 41223, - 121135, 121299, 697, 0, 121051, 4244, 0, 0, 0, 13121, 128573, 0, 0, 0, 0, - 0, 0, 129879, 0, 65816, 68111, 0, 127933, 0, 0, 0, 0, 0, 0, 66895, 74602, - 0, 7123, 70038, 5785, 9198, 0, 100810, 0, 7383, 64656, 0, 0, 0, 0, 0, 0, - 0, 0, 13122, 0, 191, 70060, 8585, 126610, 64411, 0, 0, 64850, 41072, - 118996, 0, 0, 0, 0, 78907, 127010, 100753, 0, 100756, 683, 396, 0, - 100758, 0, 100757, 43058, 100760, 343, 7129, 42680, 0, 0, 0, 0, 0, - 100761, 0, 74040, 0, 1724, 0, 119321, 0, 0, 2203, 0, 0, 0, 6592, 0, - 983044, 0, 0, 0, 0, 3730, 1778, 0, 0, 128854, 121254, 0, 9018, 0, 0, 0, - 0, 92763, 5547, 0, 0, 128950, 0, 0, 284, 8108, 0, 0, 74001, 0, 66460, - 7174, 92703, 126072, 0, 0, 4394, 127480, 0, 0, 0, 101082, 66459, 0, 7180, - 101084, 0, 92812, 68800, 42471, 0, 0, 67232, 64304, 42243, 101094, 2583, - 0, 77728, 0, 0, 0, 71702, 3855, 0, 0, 0, 0, 0, 0, 0, 92416, 7132, 0, - 92743, 0, 64756, 3798, 6578, 0, 0, 92481, 9774, 1275, 0, 119273, 983056, - 0, 120515, 7873, 77719, 129754, 0, 0, 77717, 0, 73994, 73992, 0, 0, 0, - 41851, 0, 41846, 126485, 92337, 7633, 41849, 68385, 70726, 3224, 0, - 69806, 0, 0, 0, 1510, 68129, 0, 0, 0, 0, 12109, 0, 0, 0, 0, 0, 78377, - 1910, 8671, 78374, 127118, 70290, 0, 0, 0, 2654, 7893, 0, 0, 0, 72394, 0, - 67394, 0, 118970, 70066, 78372, 78371, 78370, 78369, 78368, 0, 0, 0, - 1733, 0, 2568, 0, 0, 0, 0, 41486, 0, 127839, 7116, 0, 0, 0, 7185, 0, 0, - 0, 0, 0, 120575, 120829, 0, 0, 0, 0, 92489, 0, 0, 0, 70022, 7171, 0, 340, - 0, 0, 72980, 0, 128535, 0, 124979, 94073, 0, 0, 0, 11392, 92509, 0, 0, 0, - 0, 0, 0, 0, 100632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11948, 0, 6999, 617, - 983825, 0, 3675, 10600, 0, 0, 74616, 2617, 0, 0, 0, 128446, 0, 0, 8630, - 194771, 7288, 983828, 5545, 983818, 2586, 0, 0, 73123, 983851, 0, 0, 0, - 70847, 0, 0, 0, 0, 11195, 71708, 0, 7835, 70040, 0, 0, 92285, 0, 0, - 72973, 0, 0, 100852, 71118, 10029, 983167, 0, 0, 70033, 11359, 0, 0, - 194782, 0, 0, 118975, 0, 0, 3903, 100893, 983858, 0, 120555, 0, 93036, - 110645, 0, 983565, 0, 0, 194773, 0, 0, 0, 127238, 983822, 100919, 0, - 100918, 64752, 0, 983139, 100920, 118642, 43045, 100904, 0, 0, 0, 66394, - 7128, 0, 0, 0, 0, 0, 43044, 2604, 0, 100851, 43046, 121421, 69985, 11768, - 43043, 10470, 0, 7122, 194789, 4390, 454, 41397, 194792, 0, 78762, 0, 0, - 120576, 64572, 0, 68091, 2394, 2575, 113749, 0, 0, 74802, 100913, 129280, - 0, 0, 11989, 0, 0, 128856, 0, 0, 8249, 128172, 0, 0, 6640, 74806, 2598, - 513, 0, 6586, 127521, 129301, 120710, 65008, 0, 0, 92515, 0, 194795, - 66755, 0, 126585, 0, 43152, 78637, 0, 194797, 0, 69893, 6582, 0, 0, - 12839, 0, 78906, 983221, 0, 2444, 119489, 66620, 0, 0, 0, 0, 69894, 0, 0, - 0, 0, 4238, 11071, 9459, 68437, 78140, 78139, 0, 10079, 128985, 0, 0, 0, - 0, 11907, 43928, 0, 0, 0, 0, 92490, 43929, 0, 43926, 64498, 0, 9506, - 6978, 126234, 0, 0, 0, 0, 43934, 0, 1122, 65564, 0, 71055, 0, 0, 1920, 0, - 43930, 827, 0, 0, 0, 0, 6577, 1304, 64733, 0, 10606, 0, 0, 69503, 9329, - 92997, 9239, 74422, 0, 129373, 1222, 11076, 0, 69229, 43615, 8262, 72280, - 64627, 19909, 983554, 72279, 0, 287, 0, 233, 0, 0, 42816, 0, 0, 65140, - 128158, 8830, 0, 0, 10524, 41175, 125033, 72294, 0, 5296, 0, 127559, 0, - 0, 0, 127154, 74858, 6516, 6515, 6514, 6513, 6512, 0, 70870, 0, 0, 0, - 12122, 92462, 100868, 43976, 1785, 92507, 0, 0, 917771, 5138, 0, 0, 0, - 100884, 0, 0, 0, 123564, 0, 5134, 69980, 322, 4643, 5132, 0, 194942, 0, - 5143, 0, 72309, 119628, 0, 0, 72112, 0, 129964, 0, 0, 0, 0, 0, 0, 73097, - 0, 0, 0, 127923, 0, 0, 0, 0, 0, 3234, 0, 100886, 0, 100889, 118924, 0, 0, - 100875, 68231, 74489, 100872, 120746, 12783, 100876, 0, 12714, 0, 64585, - 93775, 0, 0, 0, 129428, 0, 11027, 0, 10059, 0, 64524, 9767, 789, 1749, 0, - 66766, 984010, 320, 0, 0, 0, 3049, 0, 6471, 0, 74479, 9925, 127356, - 127355, 127358, 4960, 5549, 127359, 127346, 127345, 127348, 5418, 127350, - 3351, 120892, 127351, 10610, 5414, 93789, 0, 4286, 5421, 127344, 67867, - 0, 127794, 0, 6653, 122958, 0, 64510, 0, 41868, 0, 128823, 0, 0, 11613, - 70737, 12603, 7131, 11108, 4566, 0, 0, 0, 0, 0, 124938, 127369, 0, 0, - 5200, 0, 129484, 0, 9183, 127361, 74458, 73075, 395, 5482, 1376, 4349, 0, - 0, 5196, 0, 6113, 42009, 5205, 0, 120530, 0, 118973, 70467, 0, 0, 129691, - 0, 9126, 70498, 0, 0, 0, 0, 0, 3203, 192, 0, 3385, 120785, 128620, 5383, - 0, 0, 0, 5738, 69449, 3336, 0, 5361, 9633, 0, 0, 0, 0, 8581, 0, 1260, - 3149, 5359, 12962, 74955, 10441, 5357, 0, 0, 0, 5364, 0, 11431, 0, 9101, - 0, 0, 0, 0, 78378, 121155, 42917, 0, 129179, 0, 0, 0, 43360, 78385, - 78384, 78383, 78382, 78381, 78380, 78379, 9319, 7097, 0, 127748, 0, 0, 0, - 120632, 0, 71205, 0, 0, 0, 1720, 0, 0, 0, 8622, 0, 70430, 68772, 0, 0, 0, - 73084, 0, 0, 11921, 0, 11769, 68782, 0, 0, 0, 0, 194571, 41586, 0, 0, 0, - 3356, 194572, 64709, 194575, 0, 7134, 0, 78389, 0, 677, 0, 0, 0, 129474, - 68747, 0, 68751, 3349, 74125, 0, 8927, 0, 0, 0, 0, 0, 0, 0, 6806, 0, - 8384, 68755, 0, 0, 0, 0, 0, 124924, 0, 7113, 7586, 0, 10852, 0, 0, 4606, - 0, 0, 70084, 0, 0, 1046, 7124, 121192, 68753, 0, 5171, 65539, 0, 0, 0, - 42394, 0, 74849, 127823, 0, 5169, 11935, 0, 0, 3175, 0, 1537, 0, 5176, - 8905, 4136, 4871, 78388, 0, 0, 122661, 0, 1128, 0, 0, 0, 74066, 0, 73069, - 0, 0, 3662, 113767, 3378, 0, 71298, 0, 127995, 6320, 71302, 983163, - 10163, 0, 5165, 5126, 0, 66902, 41389, 0, 71368, 3374, 113740, 0, 7119, - 0, 0, 3507, 0, 7629, 6848, 19925, 0, 68463, 183, 127208, 127209, 70811, - 10636, 0, 128465, 2250, 0, 78772, 0, 0, 0, 78768, 6580, 4332, 123584, 0, - 10726, 66686, 127203, 127204, 127205, 127206, 0, 70813, 127201, 127202, - 0, 0, 5448, 41058, 5446, 0, 0, 71369, 5442, 7135, 0, 0, 5451, 0, 78470, - 0, 0, 0, 0, 11243, 10859, 65867, 10345, 10409, 123606, 0, 0, 129077, - 42181, 0, 0, 2060, 0, 7111, 0, 0, 0, 0, 72741, 0, 205, 93784, 72346, - 93771, 0, 9862, 6588, 43257, 0, 0, 0, 5505, 93760, 5503, 65376, 0, 7125, - 9819, 0, 0, 0, 5507, 12044, 194567, 0, 0, 0, 7109, 0, 0, 7911, 10329, - 10393, 8991, 125104, 69778, 11133, 129619, 8550, 0, 5592, 2919, 0, 0, - 5595, 0, 0, 4367, 0, 0, 5591, 41060, 5594, 0, 0, 13142, 5590, 0, 72274, - 118909, 75069, 123586, 9731, 71225, 64633, 0, 0, 71217, 121361, 71227, 0, - 0, 0, 0, 7137, 0, 0, 0, 10551, 10710, 0, 0, 0, 120570, 0, 92364, 9936, - 3348, 0, 0, 1444, 119058, 0, 74206, 983107, 0, 1442, 129080, 0, 120959, - 0, 0, 0, 0, 0, 0, 0, 3334, 73068, 118803, 0, 0, 71219, 69770, 1651, 0, - 8861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43626, 0, 0, 3344, 0, 0, 12920, 0, - 0, 0, 71853, 3438, 128711, 0, 0, 0, 0, 129068, 0, 0, 65117, 0, 0, 0, 0, - 66366, 128915, 0, 69772, 0, 0, 0, 0, 4973, 8784, 0, 0, 0, 0, 0, 0, 0, - 125198, 983288, 0, 0, 66413, 0, 0, 0, 0, 122663, 9243, 2464, 0, 0, 3372, - 0, 0, 0, 70364, 7121, 0, 0, 0, 92163, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, - 983104, 101233, 0, 3876, 0, 127983, 6858, 43696, 43380, 0, 74240, 0, 0, - 0, 983985, 75074, 6589, 0, 0, 120993, 0, 0, 69609, 0, 66962, 0, 10630, - 71960, 0, 121293, 0, 0, 121287, 917942, 121337, 121215, 0, 0, 0, 0, 0, - 917940, 3366, 0, 917938, 0, 0, 0, 71062, 0, 121197, 0, 6925, 71856, 0, - 917929, 66780, 66274, 0, 72768, 0, 917930, 129482, 11138, 0, 6754, 7118, - 0, 64672, 65296, 0, 118957, 0, 0, 12296, 68457, 121320, 0, 5282, 0, - 72278, 0, 0, 0, 0, 0, 0, 66355, 0, 0, 68073, 64343, 0, 92744, 195058, - 195029, 0, 0, 195056, 195027, 0, 0, 128814, 195025, 6584, 195026, 10657, - 0, 74544, 0, 1200, 12243, 92269, 195062, 0, 129300, 11545, 0, 120493, - 3343, 4424, 11047, 0, 69863, 3896, 0, 0, 2947, 0, 0, 42221, 0, 68139, - 13059, 7942, 0, 3381, 0, 0, 0, 0, 0, 0, 78235, 0, 0, 0, 7044, 65800, - 78236, 0, 7045, 7175, 7047, 127884, 11791, 0, 0, 3881, 0, 0, 127395, 0, - 0, 67075, 7106, 72000, 0, 0, 74211, 41897, 92513, 0, 73040, 66745, 0, 0, - 0, 0, 121245, 0, 64354, 73083, 8777, 0, 129108, 8884, 2385, 73067, 92450, - 0, 0, 0, 42027, 12114, 0, 0, 64936, 0, 0, 0, 0, 0, 126605, 0, 0, 0, 0, - 73064, 0, 0, 0, 0, 0, 0, 0, 73057, 0, 123587, 0, 0, 0, 0, 0, 70803, 0, 0, - 124953, 0, 0, 0, 7048, 11087, 123600, 92536, 7043, 9600, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 42050, 0, 55289, 0, 0, 657, 0, 195054, 4461, 92903, 0, 92904, - 126490, 0, 4468, 0, 0, 0, 4456, 73070, 10720, 123588, 0, 123544, 0, 0, 0, - 195046, 260, 7714, 74163, 2045, 0, 65064, 4466, 0, 0, 128087, 129768, - 41403, 0, 0, 0, 41406, 120692, 0, 0, 73939, 0, 0, 0, 41404, 1165, 0, - 4451, 13087, 0, 11258, 0, 73855, 0, 43014, 5439, 12061, 74586, 3375, - 128869, 0, 0, 0, 0, 0, 0, 0, 113823, 67078, 0, 67079, 0, 0, 0, 0, 68459, - 0, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 4868, 8297, 0, 0, 42791, 0, 66737, - 66739, 0, 0, 5182, 0, 0, 72764, 0, 4465, 0, 12135, 0, 4464, 0, 0, 977, - 4458, 43827, 0, 0, 120888, 0, 344, 67463, 0, 0, 0, 0, 92240, 0, 64443, - 126995, 73078, 129525, 0, 0, 0, 43026, 7612, 119591, 64413, 0, 0, 0, 0, - 0, 0, 0, 0, 123622, 0, 119160, 10204, 127947, 73063, 0, 0, 127236, 0, - 68746, 0, 8852, 0, 0, 0, 0, 128427, 123597, 7932, 92858, 128463, 0, 0, - 72453, 0, 0, 0, 0, 74893, 9567, 0, 73095, 0, 8650, 0, 0, 0, 69900, - 118872, 0, 70868, 0, 6719, 0, 0, 0, 72836, 0, 0, 118991, 0, 123594, - 73815, 4420, 0, 10583, 7760, 0, 0, 128752, 71711, 0, 128407, 0, 0, 77809, - 9066, 0, 74795, 0, 0, 0, 0, 0, 0, 0, 42825, 41854, 5304, 0, 124942, 6919, - 8619, 0, 10038, 66454, 9592, 129049, 0, 0, 110771, 110777, 110772, 0, 0, - 0, 0, 0, 78498, 110773, 43624, 0, 7779, 0, 0, 9479, 78493, 0, 66956, - 2224, 0, 0, 0, 0, 0, 42378, 3368, 0, 66804, 7697, 69237, 0, 2030, 0, - 68236, 8370, 0, 66953, 0, 0, 983355, 127903, 983353, 983352, 5174, 42831, - 983349, 70439, 983347, 8881, 119047, 0, 70433, 0, 0, 0, 0, 0, 0, 9576, 0, - 3347, 4160, 5154, 0, 3794, 0, 0, 0, 0, 0, 127916, 73073, 8381, 4572, - 69564, 126101, 0, 0, 0, 0, 0, 0, 0, 92283, 0, 0, 5799, 983344, 70100, - 983342, 983341, 983340, 43031, 64425, 65128, 983336, 0, 73059, 0, 68616, - 0, 0, 0, 0, 119826, 0, 0, 123604, 0, 0, 283, 68665, 0, 532, 0, 0, 983827, - 0, 0, 3370, 73077, 119132, 5443, 71431, 0, 118630, 0, 0, 0, 2298, 0, 0, - 0, 983335, 983334, 983333, 983332, 7144, 983330, 119600, 983328, 983327, - 983326, 0, 78816, 128833, 0, 0, 0, 0, 0, 0, 0, 0, 73088, 0, 123592, - 983952, 0, 0, 0, 0, 5186, 7360, 127837, 0, 12108, 0, 65124, 0, 0, 0, - 6326, 43344, 0, 0, 42562, 0, 0, 0, 983325, 65495, 983323, 101066, 983321, - 101065, 983319, 65490, 983317, 125034, 0, 101070, 127178, 55245, 128927, - 1630, 128232, 65483, 0, 0, 0, 65476, 0, 0, 119214, 9283, 10183, 0, 0, - 65499, 0, 64593, 66758, 3376, 0, 0, 0, 101077, 43872, 12940, 0, 0, 78587, - 101078, 5957, 0, 8926, 983315, 983314, 983313, 10745, 10174, 983310, - 113793, 983308, 983307, 983306, 0, 123593, 5056, 0, 0, 0, 120773, 0, - 9812, 0, 4460, 127792, 73066, 0, 128038, 0, 123608, 0, 64278, 0, 0, 0, - 66760, 0, 0, 70122, 0, 0, 917627, 0, 73823, 101071, 127922, 2276, 0, - 42579, 0, 983305, 983304, 127831, 983302, 983301, 983300, 983299, 983298, - 74207, 121255, 10482, 12863, 73002, 2412, 0, 9522, 0, 983906, 120674, - 101059, 3384, 101058, 10702, 830, 0, 128166, 0, 8451, 0, 0, 121380, - 69739, 128957, 0, 0, 0, 0, 0, 0, 0, 4243, 92454, 73093, 0, 129705, 4441, - 0, 983295, 983294, 66618, 983292, 125141, 411, 983289, 68068, 983287, - 4056, 983913, 0, 92666, 0, 983916, 983968, 0, 0, 3364, 42265, 64437, - 129635, 118816, 0, 9684, 216, 0, 1401, 0, 0, 0, 122643, 0, 0, 0, 11126, - 5768, 3191, 0, 0, 0, 0, 0, 0, 65895, 0, 0, 3338, 73935, 983283, 983282, - 983281, 129605, 983279, 983278, 2794, 8807, 0, 0, 110720, 0, 8312, 0, - 110718, 11953, 11662, 0, 0, 0, 0, 9534, 66767, 129040, 0, 11113, 0, 0, - 73082, 0, 981, 0, 4330, 119244, 120536, 1824, 0, 0, 7034, 41683, 123166, - 0, 73754, 0, 0, 74478, 128259, 983273, 983260, 983259, 43831, 983257, - 66752, 983255, 983254, 0, 70288, 65343, 0, 0, 43225, 0, 0, 0, 0, 126129, - 0, 128608, 0, 0, 0, 120726, 0, 983852, 11746, 0, 5216, 0, 0, 0, 0, 3468, - 127149, 9230, 65942, 0, 0, 5803, 120677, 0, 0, 13124, 0, 0, 0, 42843, 0, - 0, 0, 66753, 11739, 128318, 0, 128444, 0, 0, 0, 12448, 0, 121441, 13057, - 73852, 124994, 0, 0, 0, 0, 0, 0, 126612, 0, 68903, 0, 129470, 0, 917992, - 0, 0, 0, 0, 0, 0, 0, 92457, 0, 0, 0, 0, 0, 0, 0, 0, 125078, 0, 0, 0, - 10970, 92208, 0, 0, 0, 19944, 0, 9009, 8551, 0, 0, 0, 7575, 67484, 0, - 128899, 0, 129609, 78847, 0, 78846, 73502, 0, 69256, 0, 0, 0, 0, 9775, - 100682, 129191, 119052, 68629, 194703, 0, 0, 78850, 92880, 0, 0, 0, 0, 0, - 0, 0, 71273, 6184, 41540, 3303, 66182, 11786, 66180, 66203, 3422, 0, - 68290, 43007, 4478, 66178, 0, 0, 126216, 0, 4477, 0, 69608, 66184, 66183, - 66204, 66194, 0, 66198, 41880, 66188, 66197, 78148, 66195, 66190, 66191, - 41111, 66189, 73788, 7788, 0, 0, 0, 0, 0, 2221, 78163, 6535, 78161, - 78162, 430, 78160, 78156, 78158, 0, 0, 4945, 0, 4950, 0, 78165, 0, 67118, - 0, 5964, 12908, 0, 0, 0, 74477, 83390, 0, 4949, 0, 443, 0, 4944, 5467, - 119603, 983265, 0, 9364, 0, 119148, 4946, 0, 3788, 126106, 983718, 0, - 120847, 129858, 74441, 0, 0, 12072, 92248, 0, 983708, 0, 128676, 12091, - 0, 0, 0, 4673, 0, 4678, 0, 0, 65059, 43860, 0, 0, 0, 128151, 1199, 0, - 8356, 0, 0, 4677, 0, 0, 0, 2192, 78173, 78175, 78171, 78172, 72255, - 78170, 78166, 4674, 128450, 194944, 0, 124970, 0, 119579, 0, 129919, - 1855, 0, 0, 127806, 0, 0, 68912, 72323, 0, 12988, 121000, 0, 0, 0, 4654, - 6840, 983432, 0, 73993, 0, 4649, 65209, 983908, 93839, 4648, 122635, - 121169, 983436, 126231, 983427, 66846, 7828, 4650, 983426, 72879, 0, - 4653, 7822, 0, 0, 43187, 0, 983586, 6821, 0, 0, 0, 0, 0, 0, 66756, - 983433, 0, 0, 0, 8547, 0, 42165, 0, 119228, 6836, 0, 0, 4662, 0, 0, 0, - 9146, 599, 4657, 0, 120754, 0, 4656, 0, 0, 7811, 40994, 0, 6414, 5967, - 4658, 3725, 0, 5814, 4661, 127760, 194961, 0, 0, 64904, 0, 10833, 0, 0, - 4867, 128717, 0, 11459, 3054, 0, 40996, 0, 7605, 4622, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19926, 0, 0, 65307, 4617, 0, 0, 0, 4616, 10518, 0, 127160, 0, - 5958, 0, 983449, 4618, 0, 983442, 120675, 4621, 0, 983444, 522, 125213, - 11139, 65803, 194972, 0, 12201, 6135, 121060, 983425, 0, 983093, 0, - 983423, 983416, 983437, 4638, 983421, 0, 78242, 5965, 78240, 66569, - 68646, 0, 983455, 74392, 5335, 0, 0, 4633, 0, 119045, 983451, 4632, 0, - 5542, 5333, 0, 983428, 68648, 5331, 4634, 0, 92870, 5338, 4637, 0, 0, - 43477, 0, 42493, 0, 42361, 0, 0, 73853, 0, 0, 0, 74204, 11343, 0, 10358, - 10422, 4758, 0, 1608, 5252, 0, 0, 4753, 78239, 11344, 78237, 0, 5231, - 74384, 0, 0, 118676, 0, 0, 0, 0, 71991, 5229, 4757, 0, 0, 5227, 4752, 0, - 65235, 5234, 73044, 0, 0, 0, 0, 0, 0, 7460, 0, 917936, 0, 0, 74760, - 65189, 0, 92230, 0, 0, 5574, 128980, 0, 65139, 5577, 0, 0, 118871, 68641, - 8965, 7635, 0, 5316, 70021, 5314, 74555, 5572, 0, 5312, 0, 5525, 5330, - 5319, 68292, 0, 65066, 0, 0, 983496, 0, 0, 127851, 0, 74851, 0, 0, 64609, - 0, 0, 128593, 0, 129339, 0, 8632, 0, 0, 0, 195012, 5735, 195013, 1692, - 70151, 4610, 122653, 4305, 0, 4609, 43478, 4614, 77753, 118534, 5287, - 5309, 5285, 0, 5961, 4647, 5283, 10743, 0, 71889, 601, 4613, 77759, 0, - 9208, 4608, 74044, 71107, 5190, 0, 0, 92410, 43965, 2265, 0, 0, 0, 0, 0, - 0, 0, 129953, 0, 0, 5960, 0, 8992, 65293, 0, 1782, 0, 0, 0, 0, 0, 5501, - 0, 42508, 69759, 120749, 129120, 0, 195023, 77740, 43900, 77741, 0, - 68134, 111180, 74209, 0, 64740, 0, 0, 0, 983935, 3767, 5737, 0, 4865, 0, - 5740, 0, 5736, 7724, 0, 7193, 0, 0, 5739, 77744, 4866, 0, 0, 0, 4869, - 67093, 0, 0, 128514, 6650, 983488, 0, 983479, 78376, 4870, 0, 68661, - 6716, 983478, 2190, 69786, 68676, 0, 10122, 4864, 66568, 0, 0, 0, 9603, - 68652, 126213, 42734, 745, 0, 124131, 124130, 4777, 0, 77788, 68631, - 42775, 68196, 0, 124128, 124129, 0, 5966, 0, 4778, 127890, 0, 0, 4781, - 127196, 64407, 0, 74132, 8577, 71221, 0, 71223, 0, 4782, 0, 0, 120757, - 68618, 43472, 43056, 68622, 0, 92986, 4776, 0, 11492, 0, 0, 13176, 0, 0, - 0, 73558, 0, 0, 0, 4849, 8242, 9561, 73922, 0, 0, 0, 0, 5963, 0, 125201, - 0, 4850, 72121, 0, 590, 4853, 0, 4854, 0, 5164, 0, 1605, 5124, 0, 111165, - 0, 8471, 0, 111164, 12445, 3785, 0, 111162, 0, 0, 4848, 2530, 0, 2068, - 1964, 0, 0, 10796, 0, 0, 0, 0, 0, 4794, 0, 0, 0, 4797, 68040, 111152, - 43465, 4792, 0, 0, 0, 0, 0, 110842, 983102, 92963, 0, 0, 0, 4221, 92360, - 118869, 0, 0, 0, 70042, 0, 0, 0, 0, 10739, 65090, 0, 119327, 126541, 0, - 0, 119326, 0, 0, 4937, 43376, 0, 0, 10597, 983445, 11722, 9248, 129566, - 42879, 11725, 0, 0, 7579, 11141, 73958, 4941, 0, 917538, 9140, 4936, - 5261, 0, 0, 72298, 0, 4942, 0, 4938, 0, 0, 5259, 9369, 983434, 111182, - 5257, 78932, 6844, 4964, 5264, 0, 0, 0, 41411, 0, 121473, 73684, 128233, - 9482, 4873, 41991, 64707, 42526, 127989, 64480, 64725, 983447, 0, 0, 0, - 0, 0, 0, 73043, 0, 389, 10893, 7521, 0, 4872, 5463, 0, 3125, 111124, 0, - 4878, 5459, 4604, 0, 0, 5465, 0, 0, 0, 0, 9563, 0, 0, 128419, 125273, - 82963, 0, 0, 0, 67735, 0, 0, 0, 0, 0, 73560, 0, 129707, 0, 917833, 0, - 917836, 0, 0, 3082, 0, 0, 0, 0, 118621, 7079, 5856, 917842, 5163, 0, 0, - 1817, 66724, 0, 0, 10564, 7763, 13077, 124115, 0, 68140, 111137, 0, - 77782, 0, 111139, 123548, 77787, 121457, 0, 0, 0, 983190, 73081, 0, 0, - 983118, 124114, 0, 42156, 0, 0, 0, 983080, 0, 0, 0, 119254, 120693, 0, - 69386, 0, 118881, 0, 78189, 0, 78186, 78188, 129654, 0, 0, 0, 110877, 0, - 3108, 9745, 0, 0, 0, 118825, 92785, 0, 122954, 0, 0, 10972, 92786, 0, - 42768, 715, 983114, 121117, 9453, 5348, 10943, 0, 983170, 92784, 0, 0, - 983154, 0, 0, 11551, 128464, 0, 0, 9051, 0, 71728, 0, 120791, 119523, 0, - 6404, 66458, 68376, 11984, 9156, 65222, 74454, 78180, 0, 3128, 4789, - 5067, 5066, 0, 4784, 0, 8827, 1146, 5065, 78196, 78192, 78193, 78190, - 78191, 5064, 5326, 0, 9450, 5063, 120361, 78200, 78201, 5062, 69733, - 74146, 0, 0, 0, 0, 77992, 0, 3933, 77768, 0, 12337, 0, 125023, 0, 0, 0, - 194759, 0, 0, 82993, 42130, 0, 5151, 917832, 120357, 0, 73523, 0, 7620, - 3800, 0, 0, 0, 127952, 0, 0, 4786, 127991, 4185, 0, 128742, 0, 983194, - 73978, 0, 4593, 77715, 77727, 124909, 0, 110715, 10532, 77732, 110714, - 110711, 110712, 64759, 1325, 5166, 9888, 0, 5148, 0, 0, 78205, 78206, - 64140, 78204, 64131, 3119, 917814, 0, 983438, 917820, 12095, 0, 0, 636, - 128002, 0, 983469, 0, 78531, 7836, 42741, 64137, 0, 118969, 0, 92431, 0, - 0, 0, 0, 0, 8618, 0, 11865, 0, 0, 0, 3937, 12312, 128261, 0, 0, 0, 912, - 6349, 4536, 71964, 0, 126594, 0, 0, 0, 3935, 120665, 0, 0, 0, 0, 118859, - 0, 121116, 0, 0, 12046, 12599, 0, 0, 0, 0, 7227, 0, 0, 0, 983066, 0, 0, - 0, 113817, 2179, 78246, 0, 0, 0, 0, 0, 127405, 101531, 0, 101530, 43907, - 0, 0, 0, 0, 4644, 8818, 0, 0, 0, 0, 93066, 66452, 126081, 1644, 101043, - 9658, 43744, 11385, 65947, 983174, 43983, 0, 0, 0, 8962, 0, 0, 2466, - 42039, 67669, 0, 0, 42117, 100698, 0, 0, 0, 0, 43745, 5318, 0, 77723, 0, - 0, 0, 7054, 64147, 0, 917804, 68195, 6698, 0, 0, 0, 70849, 11981, 12202, - 0, 121364, 0, 7059, 11608, 975, 0, 65843, 170, 0, 67239, 42708, 0, 0, - 6058, 0, 0, 0, 70507, 0, 0, 9818, 0, 0, 42106, 0, 983065, 4738, 42105, - 7062, 0, 4737, 11779, 4742, 120564, 92391, 0, 41374, 41375, 983381, 6715, - 12700, 7049, 983379, 0, 0, 0, 4741, 42108, 983370, 64159, 4736, 64148, 0, - 849, 0, 128247, 983366, 0, 120913, 917997, 0, 983384, 9496, 66371, - 983408, 983382, 11322, 0, 93008, 3928, 983153, 0, 10706, 7198, 0, 4842, - 12053, 0, 0, 4841, 0, 4171, 12008, 68416, 3923, 1490, 0, 0, 983398, - 40972, 5245, 72288, 983400, 126578, 0, 4845, 8332, 40974, 0, 4840, 9077, - 2252, 2408, 72851, 4825, 0, 917574, 0, 0, 126251, 0, 0, 983358, 0, - 983359, 0, 4826, 42440, 0, 0, 1274, 0, 74315, 0, 120384, 118614, 121200, - 0, 0, 0, 4830, 983393, 129044, 0, 0, 119082, 0, 64105, 0, 0, 4824, - 120397, 0, 0, 1888, 64127, 7861, 125111, 78524, 41836, 110613, 10873, - 72439, 0, 64098, 12214, 124134, 41834, 0, 358, 128120, 41833, 11442, 0, - 0, 0, 0, 64115, 0, 0, 0, 120721, 119053, 0, 119055, 119054, 0, 0, 0, 0, - 4017, 12827, 5241, 0, 73042, 41118, 3924, 0, 11366, 0, 0, 0, 0, 41116, - 69455, 0, 0, 0, 0, 11917, 0, 74000, 4721, 123551, 983937, 0, 0, 0, 0, 0, - 0, 122907, 0, 128702, 4722, 6816, 124974, 0, 4725, 67099, 4726, 0, - 129856, 123171, 0, 123194, 0, 0, 0, 4015, 0, 8052, 78766, 123538, 0, - 128294, 0, 0, 4720, 73090, 125003, 0, 0, 1656, 41831, 0, 0, 41843, 92846, - 0, 1452, 13111, 0, 0, 0, 8552, 64113, 41845, 64073, 120354, 0, 0, 120066, - 120067, 7064, 64070, 9948, 0, 0, 0, 92828, 2420, 92811, 0, 0, 0, 120052, - 120053, 120050, 74920, 3938, 120057, 120054, 92829, 120060, 71920, - 120058, 120059, 120064, 72203, 7955, 64074, 4713, 128196, 983108, 0, 0, - 0, 65152, 10198, 120044, 120045, 120042, 6713, 4532, 120049, 120046, - 120047, 4717, 7046, 0, 66450, 4712, 75055, 0, 121085, 0, 8155, 4718, - 3942, 4714, 9625, 0, 6383, 0, 12006, 0, 0, 0, 0, 0, 65414, 0, 0, 129061, - 66437, 66025, 74115, 0, 0, 11228, 4809, 0, 68211, 72352, 0, 0, 983101, - 65405, 129912, 0, 0, 2163, 4545, 0, 917566, 0, 4813, 78699, 0, 0, 4808, - 0, 0, 65475, 0, 0, 4814, 72240, 4810, 0, 0, 68784, 10761, 67514, 3522, 0, - 78693, 65404, 0, 0, 0, 0, 0, 6691, 70125, 0, 126223, 0, 0, 0, 43858, - 129914, 0, 12992, 65407, 0, 0, 3919, 72379, 0, 92845, 0, 0, 0, 12235, - 110748, 0, 0, 64091, 68739, 64080, 0, 64090, 0, 0, 0, 0, 0, 8454, 0, 0, - 983877, 0, 0, 120398, 4780, 0, 0, 92764, 64621, 6732, 0, 0, 0, 0, 121363, - 0, 0, 120817, 6976, 0, 119005, 0, 93809, 0, 93808, 0, 12526, 120399, - 2315, 0, 1938, 0, 0, 0, 0, 0, 0, 0, 111135, 93794, 0, 0, 0, 93810, 0, - 2291, 0, 0, 0, 0, 129429, 0, 10799, 0, 0, 66372, 0, 4193, 0, 0, 983057, - 7998, 0, 0, 0, 0, 2316, 0, 0, 0, 0, 120106, 0, 0, 74140, 0, 0, 0, 0, - 3762, 93813, 120672, 93820, 0, 0, 0, 70098, 3780, 12808, 8163, 983155, 0, - 0, 3906, 12349, 0, 8326, 0, 65498, 3763, 0, 5618, 0, 3779, 0, 43613, 0, - 128007, 0, 0, 0, 0, 280, 0, 126252, 983453, 13072, 1894, 0, 0, 65478, - 43310, 7231, 0, 11773, 0, 0, 0, 101517, 122662, 0, 7559, 11652, 10009, - 110765, 110766, 110763, 110764, 4470, 110762, 0, 0, 983446, 0, 5249, 0, - 0, 8756, 0, 0, 41694, 120585, 92349, 0, 0, 0, 69685, 123549, 983450, - 113794, 0, 6808, 41319, 13125, 66332, 127977, 0, 2290, 0, 983418, 0, 0, - 3943, 0, 41205, 0, 0, 0, 0, 5352, 0, 0, 41207, 0, 7384, 69647, 41204, - 123552, 41209, 69637, 0, 43607, 0, 0, 5420, 0, 10134, 0, 0, 4018, 7150, - 0, 0, 0, 0, 0, 129606, 2561, 65023, 0, 7148, 12076, 0, 0, 129201, 0, - 6276, 1706, 0, 0, 7146, 0, 128277, 41819, 74991, 0, 10847, 41822, 72248, - 860, 0, 0, 0, 69641, 10753, 41820, 126118, 0, 71898, 0, 92617, 128567, 0, - 121514, 43016, 0, 0, 92225, 0, 0, 0, 0, 4022, 0, 0, 110807, 0, 41691, 0, - 75060, 11866, 0, 65292, 0, 110812, 0, 3911, 110811, 110808, 110809, 0, - 125191, 7000, 3904, 118997, 72261, 0, 0, 0, 13123, 10846, 0, 0, 0, 0, 0, - 74082, 0, 123542, 0, 0, 3777, 128329, 0, 9636, 71726, 0, 0, 9367, 593, 0, - 3999, 0, 41713, 0, 0, 67677, 0, 0, 0, 9763, 120280, 120283, 12347, 124, - 12981, 41127, 92527, 0, 0, 0, 0, 0, 43987, 0, 0, 1769, 41715, 2463, 2151, - 0, 0, 71222, 1538, 93044, 0, 0, 123543, 7795, 120300, 0, 92493, 10955, 0, - 0, 72375, 78208, 9498, 78207, 127033, 78210, 120288, 3939, 120290, - 120285, 8943, 120287, 120286, 120297, 4491, 120299, 42602, 120293, - 120292, 120295, 120294, 0, 0, 0, 0, 0, 0, 1511, 9324, 0, 0, 0, 0, 0, - 64536, 0, 0, 0, 124935, 6822, 12862, 0, 0, 42143, 41828, 0, 917629, - 70864, 118879, 0, 0, 0, 41826, 128413, 0, 0, 13279, 7917, 0, 0, 0, 0, 0, - 92800, 92332, 0, 0, 43515, 0, 0, 0, 4013, 0, 66980, 0, 72224, 125266, 0, - 68243, 2432, 92834, 0, 0, 0, 0, 69952, 0, 0, 0, 10949, 0, 0, 0, 0, 0, 0, - 0, 128574, 43233, 0, 42517, 0, 0, 0, 0, 0, 64468, 119359, 6474, 119358, - 43497, 12656, 128122, 119353, 0, 1665, 0, 0, 0, 64512, 0, 0, 5256, 0, 0, - 0, 2859, 123563, 0, 0, 0, 0, 92801, 128220, 0, 770, 0, 811, 0, 0, 917551, - 42244, 64427, 0, 72222, 0, 3895, 0, 74341, 12087, 0, 42859, 10193, 3116, - 7747, 0, 0, 43496, 0, 0, 0, 0, 41877, 0, 65382, 64614, 0, 64296, 0, 6345, - 0, 2663, 0, 121234, 0, 0, 10150, 0, 64308, 1522, 597, 0, 0, 41201, 64731, - 0, 0, 41198, 123537, 71483, 3092, 0, 0, 4783, 71448, 92213, 0, 0, 10812, - 0, 0, 0, 3078, 0, 0, 0, 0, 0, 71703, 394, 3088, 0, 0, 0, 3991, 0, 129072, - 0, 424, 67652, 72377, 0, 0, 0, 0, 0, 0, 42231, 2209, 128215, 72983, 0, - 41840, 129136, 5344, 1298, 0, 13155, 0, 128973, 41838, 0, 8488, 1003, - 41837, 0, 0, 0, 48, 0, 0, 8493, 0, 0, 0, 65487, 118550, 8465, 10332, - 13172, 0, 0, 10449, 126989, 127014, 69606, 69447, 3984, 129159, 0, 0, 0, - 0, 0, 0, 0, 0, 64758, 0, 100947, 0, 0, 9096, 0, 0, 9172, 128545, 0, 0, - 5955, 67666, 0, 0, 0, 0, 0, 74426, 3926, 71734, 0, 8798, 100946, 92165, - 0, 0, 120696, 0, 0, 0, 118805, 10353, 10417, 0, 123560, 0, 128629, 4019, - 0, 0, 0, 8219, 68402, 0, 0, 121301, 128218, 0, 0, 0, 0, 0, 0, 0, 110625, - 42474, 10642, 3909, 9950, 0, 128139, 69619, 68678, 92917, 0, 1049, 43517, - 65707, 11943, 41806, 0, 68635, 3921, 0, 11775, 121352, 69820, 1038, - 42303, 9823, 0, 2145, 4008, 68624, 0, 121025, 0, 0, 5153, 41805, 0, 0, - 763, 9211, 0, 0, 0, 0, 2188, 127142, 0, 0, 65179, 0, 8621, 0, 118878, 0, - 0, 0, 0, 182, 0, 0, 0, 0, 72978, 9058, 8489, 0, 66935, 5969, 65909, - 10848, 4570, 0, 128614, 4255, 0, 0, 41189, 4003, 69785, 68109, 13293, - 41192, 0, 0, 42251, 0, 0, 126085, 11287, 6128, 121315, 11034, 0, 68207, - 0, 65506, 42382, 0, 0, 66872, 9932, 43516, 0, 125098, 0, 41814, 0, 71234, - 64835, 12117, 127040, 127041, 10540, 127043, 9063, 78000, 0, 0, 0, 12897, - 0, 0, 0, 6065, 0, 0, 0, 8692, 41186, 41816, 0, 41818, 41187, 0, 42196, 0, - 110690, 110691, 126115, 0, 0, 125235, 4710, 0, 5956, 7621, 110641, 92624, - 4705, 716, 74918, 110693, 4704, 124916, 0, 127112, 161, 67468, 0, 0, - 4706, 0, 0, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 119223, 0, 0, - 128119, 4004, 0, 73071, 69383, 69914, 8506, 128237, 0, 126996, 2538, 937, - 0, 4734, 0, 0, 0, 0, 0, 4729, 0, 0, 0, 4728, 0, 72809, 120644, 0, 8109, - 43105, 11249, 4730, 447, 0, 1513, 4733, 0, 0, 0, 0, 0, 0, 0, 8565, 2469, - 0, 6690, 0, 0, 43439, 78218, 43103, 78217, 2674, 0, 11922, 0, 0, 3510, 0, - 129368, 0, 5605, 42095, 126572, 0, 9098, 120512, 0, 121272, 68891, 74570, - 0, 67708, 0, 119346, 0, 5959, 0, 0, 66275, 43371, 0, 0, 0, 0, 0, 12769, - 69793, 0, 1283, 0, 4779, 0, 3719, 4006, 0, 0, 71186, 68204, 124957, 0, - 119331, 43028, 65493, 0, 125058, 5962, 65485, 92616, 0, 43501, 5827, 0, - 120951, 0, 65494, 0, 129365, 0, 0, 43879, 0, 0, 0, 0, 983205, 65467, 0, - 0, 0, 0, 521, 0, 0, 983928, 0, 0, 483, 7096, 0, 0, 928, 0, 0, 0, 0, - 92983, 3989, 73972, 122980, 0, 0, 0, 12145, 0, 73932, 0, 0, 3769, 67460, - 0, 0, 0, 0, 0, 65290, 92223, 0, 65855, 0, 0, 0, 0, 128811, 0, 0, 0, 0, 0, - 0, 73838, 0, 0, 13007, 67506, 0, 0, 12661, 7608, 75032, 12213, 0, 0, 0, - 0, 12195, 4001, 3112, 67474, 0, 7590, 0, 0, 421, 0, 0, 0, 4130, 127775, - 7595, 42588, 7600, 0, 0, 0, 0, 65851, 42607, 0, 92403, 8680, 0, 42134, 0, - 0, 2846, 92605, 0, 0, 0, 0, 12979, 0, 0, 92558, 3740, 69843, 120437, 0, - 120451, 65923, 120435, 0, 120434, 0, 93800, 3118, 74265, 93795, 93816, - 93823, 93797, 8127, 92912, 93792, 7943, 93821, 93799, 10618, 2584, 93793, - 0, 0, 9998, 0, 0, 0, 66350, 0, 0, 0, 121374, 8279, 128169, 0, 4975, - 70075, 0, 118675, 1631, 0, 0, 0, 6290, 128994, 66386, 0, 64645, 0, 0, 0, - 0, 0, 9242, 93807, 93802, 93801, 983269, 93803, 3122, 93804, 7793, 0, 0, - 0, 0, 12604, 92885, 6615, 67650, 0, 3986, 44025, 0, 8912, 0, 7409, 0, 0, - 0, 0, 0, 0, 8540, 11498, 0, 0, 0, 0, 0, 13060, 120682, 0, 0, 0, 0, 0, - 121345, 0, 0, 7020, 120353, 3765, 92881, 0, 1606, 120348, 120351, 3093, - 110593, 0, 0, 0, 0, 0, 0, 92892, 120337, 69402, 120339, 4023, 120333, - 120332, 120335, 92250, 120345, 12810, 120347, 120346, 4455, 120340, - 120343, 120342, 66660, 0, 0, 0, 0, 113720, 13089, 74355, 120329, 120328, - 42758, 12196, 128429, 0, 0, 0, 0, 128867, 94179, 0, 3120, 9797, 0, 0, - 11086, 10389, 0, 101025, 4895, 128153, 124941, 4359, 0, 0, 3509, 70037, - 486, 0, 0, 0, 0, 0, 7004, 0, 0, 0, 0, 4855, 128200, 0, 0, 0, 0, 0, 0, - 10381, 70839, 0, 0, 0, 0, 125121, 70837, 125070, 129431, 983377, 983365, - 0, 983364, 0, 120063, 0, 0, 0, 75048, 0, 74900, 0, 0, 120978, 12161, - 983356, 0, 10339, 0, 77808, 0, 0, 917846, 77806, 0, 43032, 125010, 0, - 983383, 12671, 11384, 0, 0, 120901, 64797, 0, 5820, 0, 0, 0, 0, 0, - 120650, 42137, 9893, 8851, 12664, 0, 0, 13192, 0, 41799, 65530, 0, 0, - 43039, 3114, 0, 0, 0, 0, 0, 926, 77803, 72004, 77805, 77799, 77800, - 77801, 77802, 43037, 41798, 77797, 77798, 123214, 41801, 0, 0, 0, 4200, - 12699, 8331, 70118, 3091, 92980, 66298, 70293, 8360, 0, 78044, 0, 4229, - 64543, 126227, 65563, 0, 129310, 2861, 43793, 10095, 121428, 9195, - 121381, 121132, 0, 129578, 0, 0, 43041, 0, 43794, 0, 83167, 0, 43797, - 8209, 0, 129132, 12973, 0, 0, 0, 0, 0, 121235, 5760, 0, 743, 0, 0, 0, - 118712, 0, 0, 83170, 128589, 129537, 0, 119063, 0, 0, 0, 19919, 0, 64532, - 0, 43710, 0, 0, 9483, 71115, 0, 43697, 0, 0, 83211, 0, 0, 0, 7247, 0, 0, - 0, 0, 0, 113674, 0, 7471, 120823, 128743, 12682, 0, 0, 65679, 983144, 0, - 0, 83201, 1099, 74241, 0, 10501, 0, 0, 113743, 0, 64743, 128476, 67663, - 0, 0, 92219, 0, 83197, 64897, 9973, 1818, 0, 0, 8272, 127812, 0, 4218, - 3087, 0, 127234, 122935, 101300, 65181, 9954, 10465, 0, 0, 0, 9106, 0, - 67406, 0, 0, 0, 0, 43038, 0, 0, 265, 70208, 0, 0, 0, 0, 0, 69405, 0, 59, - 0, 0, 0, 0, 126239, 41810, 0, 126492, 0, 41809, 41888, 0, 41795, 0, - 42213, 0, 0, 43033, 511, 42963, 0, 13127, 0, 0, 0, 0, 111107, 100489, - 4467, 41812, 41215, 0, 41211, 917783, 4453, 69575, 0, 129883, 0, 983412, - 41213, 92864, 118716, 0, 0, 129730, 41841, 6617, 130041, 0, 92995, 462, - 0, 10493, 0, 55248, 0, 0, 74471, 6644, 0, 0, 0, 983388, 100484, 9581, - 67104, 3098, 0, 0, 983415, 125250, 0, 120621, 0, 0, 0, 129584, 101011, 0, - 118789, 74473, 3755, 64661, 7748, 7235, 3966, 0, 0, 127510, 0, 0, 0, - 5726, 66456, 42175, 100486, 0, 42212, 92681, 121443, 2851, 43017, 120108, - 121056, 4373, 0, 0, 9587, 0, 6671, 128840, 3100, 0, 917790, 0, 0, 0, - 917789, 70209, 8190, 12083, 917791, 0, 6689, 64629, 0, 0, 0, 4419, - 917787, 101017, 0, 69851, 0, 0, 8891, 3080, 0, 2347, 0, 0, 8990, 0, - 121201, 0, 92528, 249, 129008, 0, 69424, 0, 0, 0, 55253, 0, 0, 11173, - 995, 0, 121047, 119861, 0, 73708, 0, 0, 19945, 0, 558, 983399, 12273, 0, - 983881, 0, 69912, 120861, 129492, 67274, 94178, 0, 68019, 43030, 3129, 0, - 2102, 0, 0, 121450, 0, 7725, 0, 11120, 0, 126111, 69246, 0, 0, 0, 41894, - 0, 41898, 0, 41893, 74921, 128678, 3540, 11848, 0, 73005, 120848, 0, 0, - 126113, 73959, 0, 0, 128735, 120858, 0, 0, 9699, 128656, 41896, 0, 83196, - 69230, 74951, 0, 72736, 0, 0, 3095, 983689, 11946, 983885, 0, 0, 0, 0, 0, - 113677, 3672, 111309, 0, 0, 0, 128539, 8890, 93826, 0, 128182, 0, 0, 0, - 126568, 0, 0, 983617, 9516, 983441, 72109, 0, 42220, 0, 4450, 0, 11547, - 43417, 128542, 356, 0, 0, 0, 0, 64901, 0, 0, 0, 0, 0, 0, 111302, 65940, - 2541, 71231, 0, 123215, 126470, 3549, 0, 0, 0, 2743, 0, 0, 0, 9097, - 128896, 43015, 0, 0, 776, 2524, 0, 8573, 100665, 126494, 0, 0, 42694, - 71122, 8952, 10814, 118818, 0, 43646, 128598, 66944, 0, 0, 128380, - 100663, 0, 65853, 42707, 1897, 93071, 0, 0, 71907, 69410, 0, 125106, 0, - 0, 0, 68473, 66778, 43573, 92638, 0, 0, 0, 120955, 73986, 0, 0, 43022, 0, - 74841, 0, 67714, 0, 0, 0, 0, 0, 4553, 0, 0, 0, 0, 0, 19921, 0, 0, 983687, - 4567, 41891, 0, 983819, 55249, 194663, 0, 194662, 0, 194665, 43042, - 121291, 1377, 12869, 0, 0, 9250, 0, 0, 0, 129779, 125039, 194642, 0, - 74995, 0, 194644, 0, 0, 101328, 194668, 121166, 0, 70275, 1898, 69556, 0, - 0, 802, 0, 0, 0, 6648, 0, 2528, 0, 0, 194646, 194625, 101330, 68804, 844, - 0, 68824, 0, 68818, 194650, 0, 0, 0, 983743, 65464, 0, 0, 0, 0, 83221, 0, - 0, 100680, 42954, 0, 64371, 70665, 0, 194654, 0, 0, 0, 0, 0, 6196, 6945, - 0, 0, 0, 120491, 0, 68846, 6210, 0, 70274, 0, 0, 0, 68067, 68834, 194715, - 588, 9760, 129112, 0, 983723, 119505, 0, 127992, 0, 0, 118905, 0, 0, - 92485, 110839, 69396, 0, 3394, 70734, 194639, 0, 0, 0, 0, 0, 0, 194656, - 7817, 1841, 11055, 195001, 194979, 194983, 127011, 67403, 194987, 7701, - 194998, 0, 194995, 1946, 121404, 0, 0, 917631, 0, 0, 10934, 0, 70376, 0, - 0, 8071, 3538, 0, 2287, 65328, 0, 0, 7614, 0, 0, 0, 12009, 43968, 0, - 67852, 0, 0, 10841, 123640, 0, 0, 0, 0, 8960, 0, 0, 65317, 128829, 0, 0, - 70374, 0, 0, 0, 65315, 0, 0, 0, 0, 0, 119621, 0, 11849, 12447, 0, 0, - 110741, 0, 0, 0, 129976, 42767, 0, 0, 0, 43695, 120520, 11975, 194941, - 983448, 0, 2555, 0, 128640, 70070, 42936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 66714, 0, 0, 70076, 65596, 121034, 66710, 67658, 0, 126994, 65338, - 7792, 0, 0, 67871, 119027, 0, 8233, 43572, 0, 0, 0, 3442, 110933, 2841, - 12543, 0, 1473, 42820, 64329, 127832, 917772, 126126, 7937, 0, 1048, 0, - 0, 983943, 0, 3406, 1054, 100701, 1040, 65450, 0, 92329, 1069, 917763, - 128367, 128940, 0, 917765, 0, 983724, 9693, 110873, 0, 0, 0, 983948, - 4353, 118653, 1059, 127530, 0, 0, 0, 127093, 118862, 120500, 10646, - 118708, 100710, 917762, 70424, 74830, 0, 0, 983720, 10221, 100706, 68255, - 0, 0, 74346, 119619, 100707, 64945, 12921, 0, 0, 0, 0, 0, 983795, 43020, - 0, 0, 74254, 0, 983785, 0, 0, 983792, 0, 71954, 0, 0, 0, 0, 122625, 0, - 120503, 70663, 0, 2755, 0, 0, 0, 4857, 0, 4428, 0, 0, 983791, 0, 0, 0, - 43842, 0, 122899, 0, 7978, 0, 70392, 127080, 11924, 43812, 0, 65015, - 67472, 563, 68340, 0, 12798, 0, 100727, 0, 0, 0, 74110, 0, 94051, 0, 694, - 0, 9876, 0, 119168, 0, 0, 0, 92361, 0, 0, 7229, 0, 0, 0, 0, 64811, 0, - 119087, 126478, 0, 7381, 0, 2525, 4852, 11586, 68465, 41605, 126089, 0, - 11582, 7151, 10155, 92578, 188, 0, 11592, 0, 74015, 0, 0, 4858, 122645, - 0, 0, 4861, 0, 2786, 121431, 4856, 8051, 0, 119609, 0, 113797, 71133, 0, - 78448, 0, 0, 67842, 68084, 0, 0, 0, 0, 0, 10234, 5843, 0, 71865, 66728, - 0, 3157, 0, 0, 75035, 72788, 983750, 0, 10822, 5149, 129517, 0, 65142, - 129454, 4565, 0, 0, 0, 12657, 0, 0, 386, 0, 8834, 120974, 0, 43574, 0, 0, - 0, 70113, 7220, 11839, 124984, 74883, 194752, 0, 65241, 74503, 8160, 0, - 194753, 0, 0, 0, 0, 0, 121265, 6847, 13303, 0, 0, 194755, 0, 118865, 0, - 194761, 0, 0, 74505, 0, 0, 0, 100518, 194721, 8780, 100512, 0, 68745, - 110626, 66697, 0, 2672, 3735, 983641, 0, 68752, 11205, 10724, 41202, 0, - 100714, 0, 0, 0, 0, 194765, 3842, 0, 78183, 12442, 78182, 9791, 78181, 0, - 42516, 67730, 64821, 195059, 78178, 0, 78464, 119219, 78465, 127466, - 194690, 195063, 0, 0, 0, 0, 78540, 78541, 78538, 1962, 78490, 78476, - 65930, 11660, 0, 2072, 0, 0, 78544, 194704, 78542, 10669, 110859, 110860, - 110857, 110858, 129749, 110856, 4105, 0, 194699, 0, 0, 0, 13148, 195068, - 78479, 9226, 0, 0, 10765, 127486, 71919, 6263, 195050, 0, 195041, 0, 0, - 0, 0, 0, 0, 92312, 7886, 0, 6682, 0, 6680, 195042, 126473, 195052, 6679, - 74412, 0, 72206, 74421, 66281, 0, 0, 127478, 0, 0, 92861, 6681, 0, 12693, - 0, 0, 0, 0, 0, 65442, 129055, 0, 9989, 74415, 194673, 0, 0, 983788, 0, 0, - 0, 0, 7042, 127240, 119026, 7968, 0, 983768, 194741, 194736, 983793, 0, - 69889, 74389, 128696, 0, 0, 128979, 5781, 0, 78199, 0, 124145, 11091, 0, - 2719, 0, 0, 0, 64495, 0, 0, 0, 65169, 42845, 0, 128551, 983766, 2200, - 72435, 0, 0, 0, 917855, 66670, 0, 983709, 0, 0, 0, 7902, 0, 65265, 0, 0, - 0, 0, 0, 0, 0, 12994, 0, 10828, 983974, 0, 4307, 3482, 0, 0, 72389, 0, - 64299, 74573, 41194, 7343, 0, 0, 41195, 0, 8169, 0, 8841, 66770, 516, - 72981, 41197, 119051, 34, 128850, 120186, 11504, 1612, 120187, 120182, - 120181, 120184, 12001, 120178, 120177, 120180, 120179, 71966, 120173, - 7749, 120175, 0, 1758, 0, 10667, 0, 120197, 0, 1935, 11517, 120193, - 120196, 78925, 120190, 120189, 120192, 120191, 1217, 64702, 128075, 825, - 0, 129824, 0, 0, 66748, 0, 11050, 0, 123187, 0, 0, 74554, 110577, 0, - 8677, 123188, 11313, 123185, 3403, 0, 123186, 64364, 92683, 0, 0, 0, 0, - 123189, 0, 0, 983880, 0, 69408, 41850, 0, 3433, 127965, 0, 1594, 65607, - 0, 66392, 0, 129291, 74565, 41353, 125119, 78926, 0, 0, 0, 918, 127280, - 41351, 0, 0, 12140, 0, 12668, 72395, 0, 128753, 0, 127302, 0, 127288, - 129497, 127235, 573, 0, 0, 11417, 0, 127283, 0, 0, 0, 72410, 0, 11482, 0, - 3981, 74345, 0, 0, 0, 0, 0, 0, 125238, 0, 0, 42195, 0, 123190, 129764, - 64602, 0, 0, 121366, 0, 121061, 128690, 0, 8423, 0, 448, 66907, 9717, 0, - 0, 0, 0, 0, 0, 0, 71910, 129898, 0, 0, 120679, 65013, 78169, 0, 72390, 0, - 0, 127917, 0, 74892, 0, 0, 127798, 0, 0, 66982, 0, 0, 0, 12197, 125074, - 0, 121447, 0, 0, 0, 0, 0, 0, 0, 74563, 64828, 11419, 0, 8592, 0, 0, 0, - 11381, 0, 0, 74529, 0, 0, 83254, 0, 72796, 0, 83257, 0, 0, 0, 129437, - 65672, 0, 0, 0, 0, 0, 0, 0, 0, 9505, 0, 0, 756, 0, 125243, 100358, - 110852, 7261, 0, 0, 0, 0, 0, 64401, 65830, 41365, 0, 0, 0, 127834, 0, 0, - 0, 0, 0, 74626, 123155, 11578, 0, 2170, 0, 0, 0, 0, 74568, 0, 113684, - 1794, 68310, 120218, 120219, 120220, 120221, 120222, 120223, 3617, - 120011, 64886, 94061, 78202, 120213, 66999, 10225, 983060, 0, 65223, - 983058, 0, 0, 4452, 127779, 0, 0, 66981, 0, 0, 0, 11425, 0, 0, 1231, 0, - 0, 0, 124121, 8192, 124118, 0, 0, 10616, 8694, 0, 68867, 128332, 123595, - 120200, 120201, 120202, 120203, 9878, 120205, 119626, 120207, 0, 8799, - 42131, 0, 127163, 0, 120198, 120199, 837, 120015, 72384, 0, 983836, 2180, - 11427, 0, 78154, 0, 70171, 0, 78150, 42606, 0, 119615, 78147, 64637, - 78146, 43060, 78145, 125009, 3392, 0, 194783, 119067, 119650, 65468, - 43498, 126083, 0, 0, 0, 194928, 194937, 194938, 64681, 194930, 83264, - 92451, 0, 194955, 83262, 983751, 8973, 0, 194967, 70177, 194968, 0, 4800, - 195018, 0, 0, 11820, 6852, 122981, 0, 4802, 4111, 111268, 0, 4805, - 127308, 68193, 7885, 121220, 0, 0, 0, 4767, 0, 0, 0, 0, 0, 125234, - 100366, 43453, 0, 41340, 0, 0, 10005, 65856, 41333, 0, 9518, 0, 0, 0, - 42520, 100850, 0, 0, 917562, 100506, 0, 0, 0, 0, 0, 0, 9167, 42151, - 124958, 0, 2026, 100848, 124139, 0, 100534, 12768, 0, 7582, 0, 0, 0, 0, - 129557, 0, 120539, 68879, 0, 43547, 119992, 8546, 126071, 78520, 7604, - 78518, 78519, 78514, 78517, 78511, 78512, 73802, 128140, 0, 6708, 10535, - 0, 68218, 55274, 68221, 92296, 0, 0, 0, 0, 0, 72385, 0, 0, 0, 73727, 0, - 120706, 74442, 66943, 0, 0, 4351, 0, 119887, 119888, 0, 119886, 119891, - 68866, 119889, 11433, 119895, 119896, 0, 119894, 65578, 194693, 0, 0, - 983070, 10681, 0, 0, 128737, 0, 983111, 0, 6722, 129364, 0, 119997, - 41546, 64860, 68394, 0, 41549, 118619, 72386, 0, 0, 0, 0, 64710, 41547, - 0, 0, 0, 78530, 78532, 78528, 78529, 71343, 78527, 78523, 78525, 3537, - 119908, 119905, 7155, 2264, 0, 78533, 67755, 0, 0, 0, 0, 0, 0, 0, 64715, - 0, 0, 537, 0, 4179, 0, 0, 0, 0, 0, 0, 0, 0, 12081, 0, 0, 4048, 7053, 0, - 0, 70459, 0, 124975, 0, 3059, 0, 0, 43491, 983833, 0, 0, 127993, 4100, - 920, 1811, 1355, 0, 0, 64383, 10078, 69398, 0, 118651, 0, 65870, 0, - 129565, 0, 72400, 42918, 0, 66789, 0, 12865, 0, 73938, +static const unsigned short dawg_codepoint_to_pos_index2[] = { + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 29174, 13037, 26950, 24736, 10779, 26523, 358, 1041, 19550, + 27163, 2511, 26774, 8150, 15996, 13206, 29017, 10563, 10548, 10560, + 10557, 10542, 10539, 10554, 10551, 10566, 10545, 7753, 27749, 19776, + 12475, 13751, 26948, 8149, 18159, 18205, 18215, 18231, 18246, 18291, + 18295, 18312, 18326, 18355, 18360, 18372, 18390, 18399, 18415, 18465, + 18473, 18476, 18496, 18519, 18548, 18587, 18598, 18607, 18610, 18624, + 19518, 27073, 27177, 6518, 20535, 13722, 18716, 18766, 18786, 18809, + 18845, 18904, 18912, 18930, 18949, 18986, 18992, 19006, 19044, 19057, + 19081, 19138, 19149, 19155, 19193, 19234, 19307, 19355, 19369, 19378, + 19386, 19402, 19466, 33652, 27135, 32321, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24729, + 16349, 5830, 26815, 9647, 34209, 4636, 27731, 10531, 8344, 13111, 19565, + 24711, 28968, 26996, 22169, 10248, 26786, 29740, 29739, 4, 22895, 26671, + 22898, 5826, 29743, 21136, 27214, 33780, 33778, 33786, 16352, 18188, + 18189, 18177, 18191, 18170, 18172, 18193, 18216, 18277, 18278, 18249, + 18264, 18342, 18343, 18332, 18330, 18288, 18413, 18454, 18455, 18422, + 18440, 18429, 23603, 18438, 18579, 18580, 18549, 18552, 18619, 18540, + 19224, 18747, 18748, 18736, 18750, 18727, 18729, 18753, 18787, 18878, + 18879, 18848, 18863, 18967, 18968, 18950, 18952, 18903, 19076, 19120, + 19121, 19082, 19106, 19089, 10690, 19104, 19342, 19345, 19310, 19313, + 19397, 19256, 19388, 18175, 18734, 18160, 18717, 18185, 18744, 18220, + 18792, 18218, 18789, 18224, 18797, 18219, 18791, 18238, 18817, 18235, + 18811, 18267, 18867, 18280, 18881, 18259, 18858, 18274, 18875, 18258, + 18857, 18297, 18914, 18300, 18917, 18304, 18922, 18296, 18913, 18315, + 18933, 18322, 18942, 18336, 18964, 18334, 18959, 18345, 18970, 18339, + 18961, 18329, 18833, 18636, 19422, 18356, 18987, 18361, 18993, 19005, + 18386, 19023, 18382, 19010, 18384, 19013, 18379, 19020, 18381, 19031, + 18408, 19069, 18400, 19058, 18402, 19062, 19077, 18290, 18886, 18418, + 19099, 18457, 19123, 18435, 19095, 18637, 19425, 18485, 19172, 18477, + 19156, 18478, 19158, 18502, 19194, 18501, 19200, 18499, 19198, 18497, + 19196, 18520, 19235, 18523, 19239, 18530, 19251, 18576, 19339, 18568, + 19329, 18582, 19347, 18583, 19338, 18559, 19320, 18573, 19334, 18599, + 19370, 18611, 19387, 18612, 18627, 19403, 18630, 19409, 18626, 19406, + 19036, 18775, 18211, 18206, 18767, 18543, 19259, 18460, 18222, 18795, + 18198, 18243, 18239, 18812, 19264, 18494, 18518, 18459, 18292, 18907, + 18303, 18310, 18948, 18353, 18346, 18371, 19004, 19009, 19032, 18537, + 18406, 19063, 18421, 18447, 19113, 18462, 19135, 18471, 19146, 18713, + 18544, 19260, 18289, 18653, 19247, 18529, 19248, 18528, 18561, 19322, + 18586, 18591, 18616, 19392, 18633, 19412, 18282, 18284, 18896, 18901, + 18706, 18545, 19261, 18652, 18705, 18707, 18708, 18645, 18655, 18245, + 18233, 18839, 18389, 18380, 19043, 18414, 18403, 19080, 18183, 18742, + 18333, 18951, 18428, 19088, 18551, 19312, 18556, 19317, 18555, 19316, + 18553, 19314, 18554, 19315, 19296, 18171, 18728, 18167, 18724, 18195, + 18756, 18305, 18923, 18298, 18915, 18362, 18994, 18436, 19102, 18437, + 19103, 18283, 18898, 18989, 18244, 18232, 18838, 18299, 18916, 18324, + 18606, 18407, 19068, 18173, 18730, 18194, 18755, 18439, 19105, 18169, + 18726, 18190, 18749, 18263, 18862, 18279, 18880, 18327, 18958, 18344, + 18969, 18434, 19094, 18456, 19122, 18482, 19162, 18487, 19173, 18558, + 19319, 18581, 19346, 18500, 19199, 18522, 19237, 18623, 19401, 18316, + 18934, 18404, 18816, 18464, 19137, 18635, 19416, 18166, 18723, 18247, + 18846, 18430, 19090, 18443, 19109, 18431, 19091, 18432, 19092, 18620, + 19398, 19012, 19061, 19238, 18830, 18843, 19154, 18192, 18225, 18798, + 18374, 18524, 19206, 19411, 18306, 18924, 18212, 18585, 18539, 18281, + 18883, 18359, 18991, 18512, 19152, 18489, 19176, 18622, 19396, 19288, + 18758, 19289, 18774, 19126, 18790, 18813, 18821, 19185, 19215, 19219, + 19132, 19180, 19182, 18805, 18831, 18921, 19222, 18682, 18928, 19192, + 19265, 18941, 18947, 18971, 18982, 18660, 19018, 19007, 19026, 19033, + 19291, 19292, 19052, 19065, 19074, 18693, 18778, 18667, 18804, 19148, + 19276, 19279, 19282, 19167, 19168, 19163, 19183, 18669, 18661, 19212, + 18889, 18832, 19232, 18893, 19283, 19250, 19308, 19349, 19362, 19285, + 19300, 19293, 18700, 19415, 19405, 18895, 18897, 18709, 18711, 18650, + 18702, 18648, 18680, 18803, 18683, 18689, 18988, 19298, 18662, 19151, + 18710, 18654, 18840, 18826, 18841, 19303, 19252, 19302, 18910, 19040, + 19041, 18647, 18649, 19266, 19267, 23222, 23223, 23231, 23253, 23280, + 23283, 23178, 23300, 23302, 23151, 23094, 23308, 23005, 23159, 23161, + 23137, 23110, 23157, 23139, 23163, 23310, 23096, 23086, 5759, 23314, + 23140, 23004, 23109, 23135, 23126, 23132, 23131, 23307, 23117, 23038, + 23037, 23309, 23095, 23150, 23148, 4629, 10885, 27334, 25266, 28928, + 10944, 23164, 23087, 23221, 23233, 23260, 23301, 23257, 23101, 23116, + 23144, 23129, 23106, 23316, 23315, 23313, 23311, 23093, 23122, 23134, + 23124, 23127, 23128, 23147, 23146, 23145, 23130, 23156, 23007, 23107, + 23008, 23108, 23166, 23149, 23123, 7961, 7759, 7843, 8125, 8067, 8087, + 7769, 7868, 7865, 7974, 8112, 7894, 7775, 8140, 7893, 7895, 7777, 7977, + 8133, 7780, 8095, 7781, 7962, 7760, 8052, 8108, 8046, 7976, 8049, 8135, + 7899, 8092, 8076, 8091, 8096, 7871, 7867, 8111, 7782, 7845, 8085, 8139, + 7772, 7980, 7776, 7844, 7771, 7978, 8129, 8072, 8066, 7896, 8128, 8116, + 8064, 8115, 8063, 8103, 7979, 8119, 8123, 8147, 8141, 7882, 7963, 7761, + 7971, 7970, 7973, 7972, 7773, 7908, 7892, 8045, 8078, 7975, 7768, 8053, + 8134, 7966, 8099, 8050, 7909, 8146, 8041, 8100, 8098, 8104, 7870, 7765, + 7887, 8114, 7876, 7875, 7879, 7880, 7888, 7877, 7886, 7993, 8001, 8006, + 8014, 8017, 7999, 8031, 8033, 8035, 8020, 8023, 8038, 8039, 13936, 14200, + 13831, 14067, 14018, 14014, 13925, 14182, 35762, 35762, 14257, 14207, + 14208, 14206, 14262, 13939, 35762, 35762, 35762, 35762, 14222, 13952, + 13829, 13805, 13862, 13852, 13876, 35762, 13908, 35762, 13923, 13898, + 14114, 13808, 13935, 13933, 13931, 13853, 13937, 13832, 13929, 13863, + 13932, 13938, 13940, 13941, 13942, 13899, 13928, 13909, 35762, 13911, + 13930, 13914, 13926, 13934, 13927, 13878, 13868, 13919, 14064, 14079, + 14104, 14123, 14134, 14039, 14199, 14197, 14069, 14070, 14201, 14080, + 14194, 14105, 14145, 14202, 14203, 14204, 14205, 14172, 14185, 14186, + 14196, 14192, 14195, 14125, 14183, 14198, 14184, 14147, 14110, 14130, + 14181, 14143, 14171, 13947, 14259, 14218, 14226, 14224, 14225, 14034, + 14035, 13999, 14010, 14066, 14009, 14191, 14012, 14068, 14011, 14146, + 14008, 14189, 8225, 8319, 8203, 8297, 8199, 8293, 8197, 8291, 8195, 8289, + 8224, 8318, 8194, 8288, 13998, 14037, 14015, 14013, 13948, 14016, 14038, + 13913, 14193, 13943, 13912, 14190, 14036, 13945, 13946, 13944, 9913, + 9915, 9862, 9901, 9837, 9866, 9853, 9972, 9985, 9809, 9812, 9826, 9940, + 9910, 9953, 9870, 9838, 9854, 9986, 9895, 9874, 9912, 9979, 9975, 9908, + 9951, 9925, 9876, 9892, 9881, 9944, 9813, 9887, 9890, 9823, 9832, 9894, + 9902, 9829, 9855, 9958, 9956, 9906, 9969, 9961, 9875, 9974, 9966, 9997, + 10013, 10186, 10054, 10033, 10071, 10179, 10175, 10067, 10129, 10084, + 10035, 10051, 10040, 10110, 10115, 10046, 10049, 10147, 10157, 10053, + 10061, 10153, 10014, 10136, 10134, 10065, 10169, 10139, 10034, 10174, + 10166, 10072, 10074, 10021, 10060, 10162, 10025, 10012, 10172, 10185, + 10103, 10109, 10150, 10099, 10069, 10131, 10029, 9945, 10111, 9968, + 10168, 9921, 10080, 9808, 10101, 9917, 10076, 9850, 10009, 9918, 10077, + 9941, 10100, 9816, 10119, 9984, 10184, 9923, 10082, 9924, 10083, 9836, + 10161, 9817, 10124, 9946, 10112, 9948, 10114, 9938, 10097, 10217, 7836, + 7830, 7833, 7831, 7832, 7786, 7839, 9952, 10130, 9965, 10143, 9888, + 10047, 9897, 10056, 9899, 10058, 9896, 10055, 9981, 10181, 9977, 10177, + 9926, 10085, 9928, 10087, 9929, 10088, 9848, 10007, 9883, 10042, 9991, + 10190, 9814, 10116, 9844, 10003, 9891, 10050, 9825, 10149, 9963, 10141, + 9964, 10142, 9903, 10062, 9990, 10189, 9857, 10016, 9858, 10017, 9954, + 10132, 9841, 10000, 9842, 10001, 9994, 9982, 10182, 9927, 10086, 9879, + 10038, 9886, 10045, 9885, 10044, 9939, 10098, 9893, 10052, 10118, 9840, + 9999, 9839, 9998, 9989, 10188, 9914, 10073, 9949, 10127, 9950, 10128, + 9980, 10180, 9976, 10176, 9843, 10002, 9911, 10070, 9909, 10068, 9947, + 10113, 9846, 10005, 9847, 10006, 9889, 10048, 9835, 10160, 9834, 10159, + 9833, 10158, 9856, 10015, 9898, 10057, 9970, 10170, 9900, 10059, 9904, + 10063, 9905, 10064, 9932, 10091, 9931, 10090, 9937, 10096, 9930, 10089, + 9933, 10092, 9934, 10093, 9935, 10094, 9936, 10095, 9821, 10123, 9880, + 10039, 9810, 10104, 9822, 10126, 9967, 10167, 9988, 10187, 9987, 10165, + 9845, 10004, 9877, 10036, 9882, 10041, 9815, 10117, 9955, 10133, 9884, + 10043, 9868, 10027, 9872, 10031, 9878, 10037, 35762, 2429, 2436, 2420, + 2442, 2416, 2440, 2417, 2418, 2407, 2439, 2435, 2432, 2434, 2412, 2424, + 2441, 2422, 2419, 2410, 2437, 2408, 2438, 2427, 2431, 2411, 2426, 2421, + 2415, 2428, 2430, 2406, 2414, 2413, 2409, 2425, 2423, 2443, 2433, 35762, + 35762, 2493, 2405, 2445, 2446, 2444, 2497, 2404, 2466, 2469, 2479, 2457, + 2485, 2453, 2483, 2454, 2455, 2468, 2482, 2478, 2475, 2477, 2449, 2461, + 2484, 2459, 2456, 2447, 2480, 2472, 2481, 2464, 2471, 2448, 2463, 2458, + 2452, 2465, 2470, 2467, 2451, 2450, 2474, 2462, 2460, 2486, 2476, 2487, + 2473, 2496, 2494, 35762, 35762, 27207, 19558, 2495, 35762, 15349, 15352, + 15353, 15360, 15361, 15357, 15364, 15362, 15347, 15359, 15356, 15340, + 15341, 15342, 15350, 15355, 15348, 15337, 15345, 15346, 15343, 15344, + 15339, 15351, 15354, 15358, 15365, 15366, 15338, 15363, 15444, 15460, + 15448, 15446, 15447, 15449, 15462, 15458, 15452, 15454, 15450, 15451, + 15456, 15445, 15463, 15469, 15457, 15466, 15459, 15461, 15467, 15443, + 15442, 15468, 15455, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 15367, 15374, 15417, 15415, 15392, 15423, 15397, 15394, 15409, + 15434, 15383, 15377, 15419, 15389, 15421, 15388, 15395, 15399, 15373, + 15385, 15380, 15387, 15413, 15390, 15407, 15401, 15411, 35762, 35762, + 35762, 35762, 15470, 15438, 15441, 15439, 15465, 15464, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2249, + 2283, 1081, 2284, 2285, 2248, 2392, 2393, 2259, 2390, 2391, 2582, 1056, + 1063, 2253, 2289, 2282, 2288, 2280, 2281, 2290, 2310, 2296, 2325, 2292, + 2341, 2340, 2276, 1367, 1072, 2380, 2388, 1322, 1275, 1139, 1129, 1554, + 1132, 1570, 1111, 1153, 1497, 1496, 1517, 1301, 1260, 1341, 1178, 1514, + 1419, 1596, 1453, 1466, 1445, 1193, 1475, 1591, 1101, 1247, 1328, 1325, + 1222, 1221, 1220, 2368, 1227, 1410, 1312, 1346, 1359, 1383, 1277, 1549, + 1147, 1561, 1079, 1061, 1091, 1073, 1057, 1087, 2277, 2345, 2102, 1086, + 1085, 2344, 2266, 2103, 2389, 2384, 2383, 2385, 2260, 1074, 2387, 2400, + 2402, 2399, 2398, 2395, 2394, 2397, 2396, 2403, 2401, 2257, 1068, 2382, + 1082, 1205, 1207, 1472, 1144, 1136, 1135, 1297, 1298, 1300, 1535, 1299, + 1523, 1529, 1173, 1505, 1504, 1398, 1509, 1168, 1270, 1268, 1378, 1208, + 1267, 1484, 1491, 1202, 1187, 1179, 1180, 1190, 1199, 1213, 1183, 1186, + 1436, 1422, 1433, 1430, 1423, 1431, 1426, 1309, 1429, 1454, 1457, 1458, + 1448, 1447, 1478, 1104, 1206, 1230, 1228, 1542, 1232, 1405, 1413, 1414, + 1323, 1474, 1317, 1316, 1368, 1314, 1238, 1241, 1369, 1240, 1254, 1239, + 1350, 1348, 1352, 1351, 1392, 1384, 1439, 1393, 1389, 1288, 1486, 1285, + 1278, 1279, 1500, 1558, 1335, 1588, 1534, 1585, 1338, 1557, 1541, 1216, + 1579, 1575, 1551, 1600, 1580, 1562, 1565, 1083, 1152, 2299, 2298, 2301, + 2300, 2327, 2309, 2307, 1071, 2367, 2324, 2323, 2293, 2302, 2339, 2303, + 2343, 2342, 2321, 2304, 2255, 1070, 1069, 2264, 2338, 1182, 1427, 13052, + 13054, 13051, 13050, 13047, 13046, 13049, 13048, 13055, 13053, 1467, + 1194, 1249, 2287, 2286, 1284, 29867, 29941, 29938, 29939, 29935, 29876, + 29863, 29864, 29940, 29937, 29862, 29872, 29871, 29870, 35762, 29860, + 29908, 29904, 29918, 29914, 29915, 29878, 29877, 29879, 29921, 29919, + 29880, 29911, 29912, 29916, 29917, 29909, 29881, 29893, 29920, 29900, + 29907, 29922, 29894, 29898, 29906, 29910, 29899, 29905, 29913, 29895, + 29897, 29896, 29927, 29926, 29925, 29930, 29929, 29928, 29932, 29931, + 29866, 29865, 29875, 29874, 29873, 29869, 29868, 29933, 29947, 29948, + 29934, 29945, 29944, 29943, 29942, 29924, 29923, 29946, 29861, 35762, + 35762, 29901, 29902, 29903, 1159, 1162, 1157, 1158, 1160, 1161, 1155, + 1269, 1266, 1185, 1181, 1424, 1460, 1106, 1102, 1105, 1233, 1231, 1330, + 1326, 1324, 1362, 1361, 1390, 1387, 1388, 1353, 1425, 1428, 1459, 1265, + 1263, 1456, 1420, 1264, 1138, 1137, 1219, 1218, 1217, 1553, 1552, 1564, + 1563, 1261, 1461, 1455, 1313, 31890, 31903, 31899, 31916, 31915, 31894, + 31891, 31879, 31910, 31895, 31884, 31883, 31906, 31893, 31886, 31887, + 31904, 31882, 31912, 31905, 31917, 31898, 31897, 31896, 31908, 31889, + 31892, 31907, 31913, 31902, 31901, 31881, 31909, 31914, 31880, 31888, + 31885, 31911, 31875, 31874, 31921, 31877, 31922, 31920, 31876, 31878, + 31919, 31918, 31923, 31900, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24597, 24599, + 24596, 24595, 24592, 24591, 24594, 24593, 24600, 24598, 24632, 24619, + 24633, 24618, 24634, 24616, 24615, 24603, 24608, 24621, 24627, 24629, + 24607, 24617, 24602, 24614, 24613, 24628, 24620, 24622, 24624, 24625, + 24610, 24626, 24611, 24609, 24623, 24630, 24631, 24612, 24605, 24604, + 24606, 24584, 24585, 24586, 24582, 24579, 24580, 24581, 24583, 24587, + 24636, 24635, 24638, 24637, 24588, 24639, 24601, 35762, 35762, 24589, + 24590, 24640, 27560, 27547, 27561, 27549, 27552, 27548, 27563, 27551, + 27558, 27567, 27553, 27554, 27564, 27566, 27555, 27550, 27568, 27559, + 27565, 27562, 27556, 27557, 27570, 27571, 27574, 27569, 27575, 27572, + 27594, 27604, 27599, 27593, 27603, 27598, 27592, 27602, 27576, 27600, + 27596, 27606, 27577, 27595, 27605, 27597, 27601, 27573, 35762, 35762, + 27584, 27578, 27580, 27583, 27581, 27585, 27607, 27590, 27588, 27591, + 27587, 27589, 27582, 27586, 27579, 35762, 20924, 20911, 20913, 20912, + 20914, 20923, 20921, 20926, 20906, 20904, 20903, 20915, 20916, 20917, + 20907, 20925, 20918, 20909, 20919, 20920, 20908, 20905, 20922, 20927, + 20910, 20931, 20929, 20928, 35762, 35762, 20930, 35762, 29886, 29891, + 29887, 29889, 29885, 29884, 29888, 29892, 29883, 29882, 29890, 35762, + 35762, 35762, 35762, 35762, 1125, 1115, 1126, 1142, 1124, 1112, 1121, + 1118, 1122, 1120, 1143, 1117, 1128, 1114, 1116, 1127, 1113, 1119, 1123, + 2369, 2370, 2372, 1522, 2265, 2258, 1391, 1262, 1479, 1477, 1327, 2386, + 35762, 2254, 2256, 35762, 35762, 35762, 35762, 35762, 35762, 2311, 2332, + 2331, 2334, 2101, 2349, 1064, 1084, 1156, 1163, 1302, 1476, 1229, 1411, + 1347, 1360, 1576, 1578, 1432, 1550, 1444, 1358, 1184, 1446, 1242, 1473, + 1599, 1103, 1315, 1412, 1154, 1399, 1502, 1421, 1577, 1099, 1097, 1100, + 1400, 1503, 1524, 1485, 1329, 1248, 1098, 1304, 1303, 1349, 1259, 2291, + 2294, 2322, 2317, 2326, 1095, 1094, 2348, 1096, 1093, 2337, 2312, 2308, + 2329, 2328, 2305, 2330, 2313, 2315, 2314, 2316, 2319, 2318, 2295, 2306, + 1067, 2381, 1052, 1050, 1054, 1053, 1051, 1055, 2375, 2377, 2379, 2374, + 2376, 2378, 2252, 2251, 2250, 2320, 1076, 1075, 1088, 1606, 2261, 1605, + 2263, 1065, 1066, 2262, 1060, 2104, 10466, 10454, 10468, 10473, 10389, + 10363, 10364, 10433, 10434, 10412, 10413, 10428, 10430, 10371, 10390, + 10441, 10365, 10372, 10391, 10395, 10366, 10406, 10405, 10384, 10382, + 10418, 10369, 10373, 10403, 10401, 10419, 10425, 10424, 10377, 10376, + 10417, 10427, 10426, 10379, 10378, 10420, 10416, 10436, 10435, 10400, + 10399, 10387, 10411, 10409, 10408, 10423, 10422, 10421, 10432, 10392, + 10393, 10394, 10386, 10486, 10485, 10470, 10467, 10476, 10498, 10499, + 10488, 10489, 10494, 10495, 10482, 10492, 10500, 10477, 10483, 10493, + 10484, 10478, 10472, 10487, 10479, 10514, 10475, 10474, 10358, 10355, + 10481, 10491, 10490, 10440, 10404, 10381, 10438, 10374, 10407, 10439, + 10410, 10429, 10431, 10496, 10497, 10502, 10501, 10509, 10511, 10508, + 10507, 10504, 10503, 10506, 10505, 10512, 10510, 10356, 10471, 10370, + 10397, 10396, 10367, 10415, 10414, 10388, 10437, 10385, 10383, 10402, + 10380, 10375, 10398, 3530, 3599, 3598, 3602, 35762, 3553, 3554, 3567, + 3568, 3565, 3566, 3547, 3549, 35762, 35762, 3589, 3555, 35762, 35762, + 3590, 3556, 3540, 3537, 3581, 3580, 3569, 3579, 3578, 3583, 3582, 3571, + 3562, 3561, 3558, 3557, 3570, 3564, 3563, 3560, 3559, 3572, 35762, 3585, + 3584, 3577, 3576, 3588, 3552, 3541, 35762, 3587, 35762, 35762, 35762, + 3573, 3574, 3575, 3586, 35762, 35762, 3600, 3597, 3604, 3613, 3614, 3611, + 3612, 3607, 3608, 35762, 35762, 3615, 3605, 35762, 35762, 3616, 3606, + 3601, 3538, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3528, + 35762, 35762, 35762, 35762, 3545, 3544, 35762, 3551, 3548, 3550, 3609, + 3610, 35762, 35762, 3623, 3625, 3622, 3621, 3618, 3617, 3620, 3619, 3626, + 3624, 3543, 3542, 3592, 3591, 3531, 3535, 3534, 3533, 3532, 3536, 3603, + 3627, 3546, 3529, 3596, 35762, 35762, 14491, 14492, 14497, 35762, 14445, + 14446, 14459, 14460, 14457, 14458, 35762, 35762, 35762, 35762, 14478, + 14447, 35762, 35762, 14477, 14448, 14442, 14441, 14439, 14438, 14463, + 14470, 14469, 14472, 14471, 14465, 14454, 14453, 14450, 14449, 14464, + 14456, 14455, 14452, 14451, 14466, 35762, 14474, 14473, 14468, 14467, + 14481, 14483, 14444, 35762, 14462, 14461, 35762, 14482, 14475, 35762, + 14476, 14480, 35762, 35762, 14494, 35762, 14498, 14503, 14504, 14501, + 14502, 35762, 35762, 35762, 35762, 14506, 14499, 35762, 35762, 14505, + 14500, 14496, 35762, 35762, 35762, 14493, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 14440, 14437, 14484, 14443, 35762, 14479, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 14516, 14518, 14515, 14514, + 14511, 14510, 14513, 14512, 14519, 14517, 14507, 14436, 14508, 14520, + 14509, 14495, 14435, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 14331, 14338, 14341, 35762, 14281, 14282, 14300, + 14301, 14298, 14299, 14293, 14295, 14357, 35762, 14328, 14283, 14358, + 35762, 14329, 14284, 14321, 14320, 14317, 14316, 14305, 14315, 14314, + 14319, 14318, 14307, 14290, 14289, 14286, 14285, 14306, 14292, 14291, + 14288, 14287, 14308, 35762, 14323, 14322, 14313, 14312, 14325, 14327, + 14326, 35762, 14303, 14302, 35762, 14297, 14309, 14310, 14311, 14324, + 35762, 35762, 14339, 14337, 14344, 14353, 14354, 14351, 14352, 14347, + 14348, 14342, 35762, 14355, 14345, 14343, 35762, 14356, 14346, 14340, + 35762, 35762, 14371, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14294, 14296, + 14349, 14350, 35762, 35762, 14367, 14369, 14366, 14365, 14362, 14361, + 14364, 14363, 14370, 14368, 14359, 14360, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 14304, 14336, 14335, 14332, 14334, 14330, 14333, + 35762, 25865, 25864, 25869, 35762, 25814, 25815, 25833, 25834, 25831, + 25832, 25826, 25828, 35762, 35762, 25859, 25816, 35762, 35762, 25860, + 25817, 25853, 25852, 25849, 25848, 25837, 25847, 25846, 25851, 25850, + 25839, 25823, 25822, 25819, 25818, 25838, 25825, 25824, 25821, 25820, + 25840, 35762, 25855, 25854, 25845, 25844, 25857, 25813, 25811, 35762, + 25836, 25835, 35762, 25830, 25841, 25842, 25843, 25856, 35762, 35762, + 25866, 25863, 25870, 25879, 25880, 25877, 25878, 25873, 25874, 35762, + 35762, 25881, 25871, 35762, 35762, 25882, 25872, 25868, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 25867, 25800, 25801, 35762, 35762, + 35762, 35762, 25810, 25809, 35762, 25812, 25827, 25829, 25875, 25876, + 35762, 35762, 25889, 25891, 25888, 25887, 25884, 25883, 25886, 25885, + 25892, 25890, 25808, 25858, 25805, 25804, 25807, 25802, 25803, 25806, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30505, 30522, 35762, 30473, 30474, 30481, 30482, 30477, 30478, 35762, + 35762, 35762, 30486, 30487, 30475, 35762, 30479, 30480, 30476, 30491, + 35762, 35762, 35762, 30463, 30488, 35762, 30490, 35762, 30464, 30466, + 35762, 35762, 35762, 30462, 30467, 35762, 35762, 35762, 30465, 30461, + 30493, 35762, 35762, 35762, 30492, 30495, 30472, 30471, 30470, 30469, + 30468, 30494, 30483, 30484, 30485, 30489, 35762, 35762, 35762, 35762, + 30792, 30799, 30800, 30795, 30796, 35762, 35762, 35762, 30801, 30802, + 30793, 35762, 30797, 30798, 30794, 30521, 35762, 35762, 30808, 35762, + 35762, 35762, 35762, 35762, 35762, 30397, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30434, 30436, 30433, 30432, 30429, 30428, 30431, 30430, 30437, 30435, + 30499, 30497, 30498, 30427, 30807, 30806, 30426, 30424, 30396, 30502, + 30500, 35762, 35762, 35762, 35762, 35762, 31755, 31756, 31760, 31763, + 31754, 31718, 31719, 31731, 31732, 31727, 31728, 31722, 31724, 35762, + 31748, 31749, 31720, 35762, 31729, 31730, 31721, 31745, 31744, 31741, + 31740, 31707, 31739, 31738, 31743, 31742, 31709, 31714, 31713, 31698, + 31697, 31708, 31717, 31715, 31701, 31699, 31705, 35762, 31747, 31746, + 31737, 31736, 31751, 31752, 31712, 31711, 31704, 31703, 31702, 31726, + 31733, 31734, 31735, 31750, 35762, 35762, 31761, 31759, 31764, 31775, + 31776, 31771, 31772, 31767, 31768, 35762, 31777, 31778, 31765, 35762, + 31773, 31774, 31766, 31762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 31753, 31689, 35762, 31716, 31700, 31710, 35762, 35762, 31706, + 35762, 35762, 31723, 31725, 31769, 31770, 35762, 35762, 31785, 31787, + 31784, 31783, 31780, 31779, 31782, 31781, 31788, 31786, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 31757, 31696, 31694, 31692, 31690, + 31695, 31693, 31691, 31758, 16808, 16807, 16811, 16816, 16809, 16761, + 16762, 16782, 16783, 16778, 16779, 16773, 16775, 35762, 16799, 16800, + 16763, 35762, 16780, 16781, 16764, 16796, 16795, 16792, 16791, 16756, + 16790, 16789, 16794, 16793, 16758, 16770, 16769, 16766, 16765, 16757, + 16772, 16771, 16768, 16767, 16754, 35762, 16798, 16797, 16788, 16787, + 16803, 16804, 16760, 16759, 16753, 16752, 35762, 16777, 16784, 16785, + 16786, 16802, 35762, 35762, 16812, 16810, 16818, 16829, 16830, 16825, + 16826, 16821, 16822, 35762, 16831, 16832, 16819, 35762, 16827, 16828, + 16820, 16815, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16805, + 16817, 35762, 35762, 35762, 35762, 35762, 35762, 16755, 16801, 35762, + 16774, 16776, 16823, 16824, 35762, 35762, 16839, 16841, 16838, 16837, + 16834, 16833, 16836, 16835, 16842, 16840, 35762, 16813, 16814, 16806, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 20866, 20868, 20873, 20871, 20823, 20797, 20799, 20843, + 20844, 20839, 20840, 20824, 20826, 35762, 20858, 20859, 20800, 35762, + 20841, 20842, 20801, 20855, 20854, 20851, 20850, 20831, 20812, 20811, + 20853, 20852, 20832, 20820, 20818, 20815, 20814, 20830, 20822, 20821, + 20817, 20816, 20833, 20829, 20857, 20856, 20849, 20848, 20861, 20862, + 20838, 20837, 20836, 20835, 20834, 20828, 20845, 20846, 20847, 20860, + 20819, 20869, 20867, 20872, 20875, 20886, 20887, 20882, 20883, 20878, + 20879, 35762, 20888, 20889, 20876, 35762, 20884, 20885, 20877, 20870, + 20813, 20874, 35762, 35762, 35762, 35762, 20809, 20810, 20804, 20772, + 20789, 20787, 20794, 20784, 20785, 20795, 20788, 20798, 20825, 20827, + 20880, 20881, 35762, 35762, 20780, 20782, 20779, 20778, 20775, 20774, + 20777, 20776, 20783, 20781, 20865, 20863, 20864, 20792, 20791, 20796, + 20786, 20790, 20793, 20773, 20806, 20805, 20807, 20802, 20803, 20808, + 35762, 28828, 28827, 28829, 35762, 28773, 28770, 28758, 28757, 28781, + 28780, 28783, 28782, 28779, 28778, 28777, 28776, 28775, 28774, 28771, + 28802, 28801, 28772, 35762, 35762, 35762, 28767, 28792, 28765, 28790, + 28815, 28805, 28764, 28789, 28766, 28791, 28812, 28813, 28806, 28759, + 28784, 28761, 28786, 28796, 28803, 28760, 28785, 28762, 28787, 28799, + 35762, 28804, 28768, 28793, 28763, 28788, 28794, 28769, 28811, 28809, + 35762, 28798, 35762, 35762, 28810, 28814, 28797, 28800, 28808, 28795, + 28807, 35762, 35762, 35762, 28826, 35762, 35762, 35762, 35762, 28846, + 28838, 28833, 28839, 28834, 28840, 35762, 28835, 35762, 28836, 28841, + 28832, 28845, 28842, 28843, 28844, 28837, 35762, 35762, 35762, 35762, + 35762, 35762, 28822, 28824, 28821, 28820, 28817, 28816, 28819, 28818, + 28825, 28823, 35762, 35762, 28830, 28831, 28847, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31942, + 31939, 31937, 31936, 31938, 31940, 31956, 31925, 31927, 31926, 31985, + 31928, 31997, 31929, 31994, 31990, 31987, 31988, 31958, 31930, 31993, + 31992, 31989, 31991, 31959, 31924, 31965, 31962, 31931, 31963, 31932, + 31964, 31954, 31998, 31966, 31967, 31944, 31946, 31995, 31983, 31982, + 31984, 31935, 31943, 31999, 31934, 31960, 31968, 31948, 31971, 31973, + 31978, 31979, 31975, 31976, 31974, 31977, 31961, 35762, 35762, 35762, + 35762, 32000, 31980, 31972, 31981, 31970, 31969, 31945, 31953, 31952, + 31951, 31949, 31950, 31947, 31986, 31957, 31996, 31933, 32007, 32009, + 32006, 32005, 32002, 32001, 32004, 32003, 32010, 32008, 31955, 31941, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 18051, 18049, 35762, + 18050, 35762, 18064, 18079, 18083, 18063, 18073, 35762, 18065, 18080, + 18061, 18059, 18058, 18056, 18055, 18060, 18084, 18076, 18074, 18075, + 18057, 18081, 18082, 18069, 18067, 18046, 18068, 18045, 18062, 18085, + 18088, 18053, 35762, 18054, 35762, 18087, 18070, 18071, 18072, 18077, + 18066, 18089, 18078, 18115, 18097, 18102, 18098, 18100, 18110, 18111, + 18104, 18105, 18106, 18107, 18092, 18103, 18091, 18090, 35762, 35762, + 18108, 18109, 18112, 18101, 18099, 35762, 18113, 35762, 18096, 18093, + 18094, 18095, 18126, 18127, 18114, 35762, 18122, 18124, 18121, 18120, + 18117, 18116, 18119, 18118, 18125, 18123, 35762, 35762, 18042, 18041, + 18048, 18047, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 32233, 32140, 32138, 32139, 32148, 32136, + 32133, 32137, 32158, 32128, 32126, 32149, 32164, 32159, 32154, 32162, + 32153, 32157, 32156, 32132, 32141, 32124, 32123, 32051, 32052, 32050, + 32172, 32173, 32174, 32176, 32177, 32175, 32073, 32075, 32072, 32071, + 32068, 32067, 32070, 32069, 32076, 32074, 32065, 32062, 32061, 32058, + 32057, 32060, 32059, 32066, 32064, 32063, 32129, 32151, 32131, 32150, + 32134, 32161, 32142, 32143, 32144, 32145, 32183, 32169, 32082, 32080, + 32110, 32109, 32092, 32108, 32107, 32117, 35762, 32094, 32102, 32101, + 32087, 32086, 32093, 32104, 32103, 32091, 32090, 32095, 32112, 32111, + 32106, 32105, 32119, 32100, 32099, 32089, 32088, 32120, 32113, 32114, + 32115, 32121, 32084, 32118, 32096, 32097, 32098, 32116, 32122, 32079, + 32085, 32081, 32083, 35762, 35762, 35762, 35762, 32257, 32253, 32254, + 32249, 32250, 32245, 32246, 32247, 32248, 32255, 32256, 32251, 32252, + 32180, 32181, 32243, 32244, 32171, 32185, 32146, 32155, 32167, 32182, + 32168, 32170, 32165, 32166, 32184, 32232, 32231, 32230, 32197, 32196, + 32216, 32215, 32198, 32214, 32213, 32223, 35762, 32200, 32208, 32207, + 32190, 32189, 32199, 32210, 32209, 32194, 32193, 32201, 32218, 32217, + 32212, 32211, 32225, 32206, 32205, 32192, 32191, 32227, 32219, 32220, + 32221, 32228, 32226, 32224, 32202, 32203, 32204, 32222, 32229, 32195, + 32187, 32188, 32186, 35762, 32077, 32078, 32053, 32054, 32056, 32055, + 32234, 32241, 32239, 32242, 32240, 32235, 32238, 32237, 32236, 35762, + 32179, 32178, 32127, 32130, 32152, 32147, 32135, 27208, 19559, 27209, + 19560, 32163, 32160, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 23897, 23876, 23875, 23874, 23906, 23974, 23973, 23976, 23975, 23907, + 23904, 23952, 23951, 23958, 23957, 23905, 23936, 23953, 23960, 23959, + 23908, 23978, 23977, 23972, 23971, 23903, 23980, 23910, 23970, 23956, + 23935, 23979, 23969, 23866, 23930, 23967, 23968, 23965, 23966, 23869, + 23902, 23981, 23868, 24063, 24047, 24070, 24071, 24068, 24069, 24059, + 24044, 24052, 24053, 24060, 23988, 24007, 24012, 24011, 23987, 23861, + 23859, 23860, 23858, 23873, 24078, 24080, 24077, 24076, 24073, 24072, + 24075, 24074, 24081, 24079, 24010, 23999, 24017, 24021, 24016, 24020, + 23911, 23934, 23961, 23962, 23963, 23964, 24064, 24065, 24066, 24067, + 23901, 23900, 23898, 23899, 23864, 23863, 23862, 23933, 24054, 24032, + 24033, 23955, 23954, 24061, 24062, 24002, 24003, 24004, 24005, 24006, + 23872, 23871, 23870, 24048, 24050, 24051, 24049, 23925, 23924, 23923, + 23921, 23929, 23913, 23926, 23914, 23916, 23927, 23919, 23917, 23928, + 23865, 24058, 24055, 24056, 24057, 23994, 23995, 23996, 23997, 23992, + 23993, 23991, 23909, 24008, 24028, 24030, 24027, 24026, 24023, 24022, + 24025, 24024, 24031, 24029, 23989, 23990, 24045, 24046, 24019, 24018, + 13332, 13358, 13343, 13364, 13365, 13363, 13353, 13354, 13366, 13347, + 13356, 13359, 13361, 13367, 13349, 13352, 13357, 13351, 13355, 13368, + 13348, 13346, 13342, 13362, 13350, 13338, 13341, 13345, 13340, 13339, + 13360, 13344, 13333, 13337, 13335, 13370, 13334, 13336, 35762, 13369, + 35762, 35762, 35762, 35762, 35762, 13331, 35762, 35762, 13415, 13446, + 13423, 13452, 13421, 13451, 13444, 13441, 13453, 13433, 13435, 13447, + 13449, 13454, 13437, 13443, 13445, 13439, 13442, 13412, 13436, 13432, + 13422, 13450, 13438, 13416, 13419, 13431, 13418, 13417, 13448, 13430, + 13426, 13429, 13427, 13456, 13424, 13428, 13457, 13455, 13420, 13440, + 13414, 13504, 23112, 13413, 13425, 13434, 14693, 14767, 14701, 14772, + 14765, 14729, 14696, 14711, 14769, 14743, 14761, 14675, 14673, 14759, + 14660, 14695, 14779, 14708, 14782, 14703, 14768, 14705, 14704, 14776, + 14739, 14763, 14742, 14688, 14698, 14692, 14721, 14726, 14725, 14712, + 14716, 14714, 14717, 14718, 14715, 14723, 14722, 14724, 14719, 14690, + 14691, 14750, 14756, 14755, 14748, 14753, 14744, 14745, 14747, 14758, + 14752, 14751, 14749, 14754, 14746, 14757, 14665, 14664, 14670, 14669, + 14728, 14685, 14684, 14682, 14677, 14687, 14678, 14771, 14681, 14680, + 14683, 14676, 14780, 14674, 14667, 14663, 14672, 14668, 14661, 14662, + 14666, 14671, 14709, 14689, 14770, 14781, 14694, 14707, 14702, 14706, + 14773, 14784, 15022, 14928, 14938, 14986, 14990, 14940, 14939, 14992, + 14991, 14967, 15019, 15020, 14977, 14998, 14978, 15018, 15017, 15021, + 15007, 14944, 14996, 14951, 14936, 14937, 14988, 14987, 14942, 14943, + 14941, 14994, 14995, 14975, 14974, 14970, 14968, 14976, 14999, 15000, + 15001, 15006, 15005, 14983, 14984, 14982, 14979, 14985, 15012, 15011, + 15010, 15009, 15008, 15016, 15014, 14950, 14947, 14997, 14952, 14954, + 14961, 14965, 14963, 14953, 14929, 14931, 14934, 14933, 14966, 14935, + 14989, 14993, 14972, 14973, 14804, 14905, 14807, 14827, 14830, 14834, + 14909, 14855, 14856, 14861, 14865, 14874, 14877, 14870, 14882, 14814, + 14844, 14847, 14883, 14900, 14796, 14787, 14790, 14813, 14918, 14840, + 14791, 14806, 14808, 14833, 14832, 14836, 14835, 14831, 14916, 14910, + 14858, 14881, 14875, 14876, 14895, 14862, 14864, 14869, 14868, 14859, + 14873, 14871, 14860, 14878, 14824, 14821, 14815, 14820, 14819, 14817, + 14822, 14826, 14803, 14845, 14849, 14854, 14802, 14885, 14893, 14886, + 14889, 14837, 14798, 14799, 14908, 14797, 14919, 14923, 14926, 14842, + 14801, 14794, 14792, 14793, 14795, 14927, 14810, 14811, 14809, 14805, + 14812, 14906, 12703, 12710, 12709, 12704, 12708, 12707, 12705, 12706, + 12930, 12938, 12937, 12931, 12935, 12934, 12932, 12936, 12696, 12702, + 12700, 12697, 12699, 12698, 12701, 12687, 12747, 12755, 12754, 12748, + 12752, 12751, 12749, 12745, 12859, 12866, 12864, 12860, 12862, 12861, + 12865, 12863, 12834, 12843, 12842, 12835, 12839, 12838, 12836, 12840, + 12874, 12880, 12879, 12875, 12849, 12844, 12876, 12878, 12850, 12858, + 12857, 12851, 12855, 12854, 12852, 12856, 12826, 12833, 12832, 12827, + 12831, 12830, 12828, 12829, 12814, 35762, 12818, 12815, 12817, 12816, + 35762, 35762, 12807, 12813, 12811, 12808, 12810, 12809, 12812, 35762, + 12802, 35762, 12806, 12803, 12805, 12804, 35762, 35762, 12537, 12544, + 12543, 12538, 12542, 12541, 12539, 12528, 12999, 13006, 13004, 13000, + 13002, 13001, 13005, 13003, 12912, 12920, 12919, 12913, 12917, 12916, + 12914, 12918, 12575, 12583, 12582, 12576, 12580, 12579, 12577, 12581, + 12967, 12974, 12973, 12968, 12972, 12971, 12969, 12970, 12955, 35762, + 12959, 12956, 12958, 12957, 35762, 35762, 12765, 12773, 12772, 12766, + 12770, 12769, 12767, 12771, 12756, 12764, 12763, 12757, 12761, 12760, + 12758, 12762, 12657, 12665, 12664, 12658, 12662, 12661, 12659, 12663, + 12735, 12742, 12741, 12736, 12740, 12739, 12737, 12738, 12723, 35762, + 12727, 12724, 12726, 12725, 35762, 35762, 12716, 12722, 12720, 12717, + 12719, 12718, 12721, 35762, 12711, 35762, 12715, 12712, 12714, 12713, + 35762, 35762, 12939, 12946, 12945, 12940, 12944, 12943, 12941, 12942, + 12775, 12781, 12779, 12776, 12778, 12777, 12780, 35762, 12990, 12998, + 12997, 12991, 12995, 12994, 12992, 12996, 12975, 12982, 12980, 12976, + 12978, 12977, 12981, 12979, 12947, 12954, 12953, 12948, 12952, 12951, + 12949, 12950, 12605, 12613, 12612, 12606, 12610, 12609, 12607, 12611, + 12590, 12598, 12597, 12591, 12595, 12594, 12592, 12596, 12921, 12929, + 12928, 12922, 12926, 12925, 12923, 12927, 12678, 12626, 12684, 12679, + 12683, 12682, 12680, 12681, 12666, 35762, 12670, 12667, 12669, 12668, + 35762, 35762, 12650, 12656, 12654, 12651, 12653, 12652, 12655, 12646, + 12881, 12889, 12888, 12882, 12886, 12885, 12883, 12887, 12566, 12574, + 12573, 12567, 12571, 12570, 12568, 12572, 12774, 12789, 12788, 12782, + 12786, 12785, 12783, 12787, 12904, 12911, 12909, 12905, 12907, 12906, + 12910, 12908, 12896, 12903, 12902, 12897, 12901, 12900, 12898, 12899, + 12618, 12625, 12623, 12619, 12621, 12620, 12624, 12616, 12794, 12801, + 12800, 12795, 12799, 12798, 12796, 12792, 12841, 12753, 12622, 35762, + 35762, 12506, 12508, 12507, 12525, 13028, 13026, 12509, 12524, 12510, + 12523, 13027, 12522, 13024, 13022, 13021, 13018, 13017, 13020, 13019, + 13025, 13023, 12511, 12514, 12513, 12518, 12517, 12521, 12520, 12516, + 12519, 12515, 12512, 35762, 35762, 35762, 12847, 12746, 12744, 12743, + 12845, 12529, 12527, 12526, 12846, 12617, 12615, 12614, 12848, 12793, + 12791, 12790, 13016, 13007, 13013, 13014, 13009, 13011, 13015, 13010, + 13008, 13012, 35762, 35762, 35762, 35762, 35762, 35762, 6081, 6082, 6083, + 6084, 6085, 6086, 6044, 6080, 6045, 6046, 6047, 6048, 6049, 6009, 6010, + 6011, 6012, 6013, 6014, 6050, 6051, 6052, 6053, 6054, 6055, 6056, 6057, + 6058, 6059, 6060, 6015, 6008, 6016, 6017, 6018, 6019, 6020, 6021, 6062, + 6063, 6064, 6065, 6066, 6067, 6023, 6022, 6024, 6025, 6026, 6027, 6028, + 6002, 6041, 6003, 6042, 6004, 6043, 6005, 6006, 6007, 6001, 6029, 6030, + 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6068, 6069, + 6070, 6071, 6072, 6073, 6074, 6075, 6076, 6077, 6078, 6079, 6061, 35762, + 35762, 6161, 6162, 6163, 6164, 6165, 6147, 35762, 35762, 5223, 5195, + 4968, 5225, 5226, 5375, 5389, 5672, 5179, 5039, 4966, 4967, 5549, 5639, + 5659, 5637, 5660, 5638, 5641, 5635, 5642, 5636, 5304, 5656, 5633, 5657, + 5634, 5305, 4970, 5673, 5691, 5212, 5211, 5213, 5214, 5206, 5207, 5204, + 5203, 5216, 5210, 5215, 5205, 5197, 5227, 5388, 4974, 5409, 5402, 5407, + 5408, 5404, 5405, 5663, 5037, 5038, 5400, 5401, 5399, 5578, 5397, 5576, + 5398, 5577, 5392, 5574, 5393, 5575, 5395, 5572, 5396, 5573, 5662, 5391, + 5571, 5030, 5548, 5531, 5546, 5547, 5544, 5545, 5670, 5001, 5014, 5529, + 5530, 5539, 5631, 5537, 5629, 5538, 5630, 5535, 5627, 5536, 5628, 5533, + 5625, 5534, 5626, 5310, 5491, 5526, 5527, 5528, 5525, 5246, 5240, 5244, + 5245, 5242, 5243, 5665, 5238, 5239, 5237, 5623, 5235, 5621, 5236, 5622, + 5233, 5619, 5234, 5620, 5230, 5617, 5231, 5618, 5307, 5228, 5229, 5471, + 5472, 5473, 5470, 5194, 5181, 5192, 5193, 5190, 5191, 5664, 4998, 5180, + 5188, 5616, 5186, 5614, 5187, 5615, 5184, 5612, 5185, 5613, 5182, 5610, + 5183, 5611, 5306, 4997, 5447, 5272, 5280, 5289, 5290, 5275, 5276, 5667, + 5278, 5279, 5288, 5570, 5286, 5568, 5287, 5569, 5284, 5566, 5285, 5567, + 5282, 5564, 5283, 5565, 5308, 5271, 5563, 5291, 4972, 5448, 5364, 5325, + 5362, 5363, 5352, 5353, 5668, 5296, 5324, 5361, 5589, 5355, 5587, 5356, + 5588, 5309, 5292, 5070, 5365, 5270, 5257, 5268, 5269, 5266, 5267, 5666, + 5255, 5256, 5265, 5557, 5263, 5555, 5264, 5556, 5261, 5553, 5262, 5554, + 5259, 5551, 5260, 5552, 5247, 5550, 5273, 5490, 5450, 5488, 5489, 5469, + 5474, 5669, 5430, 5449, 5483, 5609, 5481, 5607, 5482, 5608, 5479, 5605, + 5480, 5606, 5477, 5603, 5478, 5604, 5302, 5429, 4973, 5476, 4993, 5277, + 5297, 5303, 5300, 5301, 5298, 5299, 5468, 5466, 5467, 5460, 5461, 5463, + 5464, 5459, 5602, 5457, 5600, 5458, 5601, 5452, 5598, 5453, 5599, 5455, + 5596, 5456, 5597, 5451, 5690, 5676, 5688, 5689, 5678, 5679, 5671, 5674, + 5675, 5687, 5586, 5685, 5584, 5686, 5585, 5683, 5582, 5684, 5583, 5681, + 5580, 5682, 5581, 5311, 5661, 4994, 5579, 5446, 5428, 5412, 5562, 5422, + 5426, 5427, 5424, 5425, 5560, 5420, 5421, 5558, 5414, 5591, 5410, 5590, + 5274, 5222, 5201, 5202, 5217, 5219, 5220, 5199, 5200, 5221, 5632, 5198, + 5510, 5295, 5508, 5293, 5509, 5294, 5506, 5507, 5504, 5505, 5502, 5624, + 5492, 5523, 5524, 5520, 5518, 5517, 5541, 5542, 5543, 5540, 5350, 5348, + 5349, 5346, 5347, 5344, 5345, 5343, 5351, 5224, 5369, 5373, 5374, 5371, + 5372, 5367, 5368, 5366, 5515, 5516, 5511, 5514, 5593, 5594, 5595, 5592, + 5330, 5334, 5335, 5332, 5333, 5328, 5329, 5327, 5336, 5444, 5445, 5440, + 5443, 5652, 5653, 5654, 5651, 5643, 5253, 5254, 5251, 5252, 5249, 5250, + 5248, 5500, 5498, 5499, 5496, 5497, 5494, 5495, 5493, 4971, 4990, 4991, + 4992, 4989, 4978, 4979, 4980, 4977, 4986, 4987, 4988, 4985, 4982, 4983, + 4984, 4981, 5435, 5436, 5432, 5434, 5020, 5019, 5015, 5016, 5018, 5017, + 5166, 5165, 5161, 5162, 5164, 5163, 5172, 5171, 5167, 5168, 5170, 5169, + 5036, 5035, 5031, 5032, 5034, 5033, 5130, 5129, 5125, 5126, 5128, 5127, + 5124, 5123, 5119, 5120, 5122, 5121, 5093, 5092, 5088, 5089, 5091, 5090, + 5087, 5029, 5028, 5025, 5026, 5027, 5023, 5066, 5065, 5061, 5062, 5064, + 5063, 5060, 5059, 5055, 5056, 5058, 5057, 5054, 5073, 5072, 5067, 5068, + 5071, 5069, 5160, 5159, 5155, 5156, 5158, 5157, 5178, 5177, 5173, 5174, + 5176, 5175, 5053, 5437, 5052, 5047, 5048, 5051, 5439, 5050, 5046, 5045, + 5041, 5042, 5044, 5043, 5148, 5147, 5143, 5144, 5146, 5145, 5007, 5006, + 5002, 5003, 5005, 5004, 5142, 5141, 5137, 5138, 5140, 5139, 5106, 5105, + 5101, 5102, 5104, 5103, 5112, 5111, 5107, 5108, 5110, 5109, 5100, 5099, + 5095, 5096, 5098, 5097, 5094, 5040, 5013, 5012, 5008, 5009, 5011, 5010, + 5086, 5085, 5081, 5082, 5084, 5083, 5080, 5079, 5075, 5076, 5078, 5077, + 5074, 5136, 5135, 5131, 5132, 5134, 5133, 5154, 5153, 5149, 5150, 5152, + 5151, 5118, 5117, 5113, 5114, 5116, 5115, 5189, 5218, 5370, 5331, 5341, + 5342, 5339, 5340, 5337, 5338, 5650, 5648, 5649, 5646, 5647, 5644, 5645, + 5655, 4976, 25265, 25240, 25251, 25247, 25257, 25254, 25260, 25263, + 25264, 25243, 25242, 25262, 25248, 25253, 25258, 25252, 25239, 25255, + 25261, 25245, 25249, 25244, 25256, 25259, 25250, 25246, 25241, 25237, + 25238, 35762, 35762, 35762, 27467, 27523, 27517, 27521, 27520, 27515, + 27513, 27460, 27445, 27491, 27444, 27443, 27488, 27503, 27490, 27494, + 27495, 27497, 27483, 27449, 27482, 27468, 27461, 27469, 27471, 27516, + 27473, 27472, 27486, 27500, 27512, 27502, 27454, 27476, 27457, 27480, + 27470, 27485, 27506, 27477, 27519, 27446, 27509, 27508, 27504, 27447, + 27525, 27514, 27505, 27452, 27511, 27499, 27455, 27493, 27458, 27518, + 27487, 27501, 27484, 27453, 27475, 27474, 27456, 27492, 27459, 27479, + 27451, 27450, 27448, 27510, 27489, 27507, 27478, 27522, 27524, 27526, + 27527, 27442, 27528, 27529, 27441, 27481, 27498, 27496, 27465, 27464, + 27466, 27463, 27462, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30050, 30067, 30068, 30058, 30056, 30052, 30064, 30055, 30053, 30061, + 30054, 30060, 30066, 30062, 30059, 30065, 30063, 30057, 30071, 30072, + 30070, 30069, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 30051, 15194, 15195, 15196, 15185, 15183, 15179, 15191, 15182, + 15180, 15188, 15181, 15187, 15193, 15189, 15186, 15192, 15190, 15184, + 15198, 15199, 15197, 26633, 26634, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 4689, 4690, 4691, 4680, 4678, 4674, 4686, + 4677, 4675, 4683, 4676, 4682, 4688, 4684, 4681, 4687, 4685, 4679, 4692, + 4693, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 30086, 30087, 30088, 30078, 30077, 30073, 30083, + 30076, 30074, 30081, 30075, 30080, 30085, 35762, 30079, 30084, 30082, + 35762, 30089, 30090, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 17762, 17760, 17763, 17761, 17764, + 17758, 17756, 17759, 17757, 17766, 17780, 17776, 17781, 17777, 17765, + 17778, 17774, 17779, 17775, 17767, 17788, 17768, 17770, 17769, 17784, + 17787, 17785, 17783, 17786, 17772, 17771, 17773, 17789, 17782, 17790, + 17737, 17735, 17745, 17746, 17741, 17744, 17742, 17743, 17754, 17755, + 17752, 17753, 17747, 17736, 17740, 17739, 17738, 17857, 17856, 17858, + 17863, 17865, 17869, 17871, 17873, 17875, 17874, 17866, 17870, 17864, + 17876, 17860, 17861, 17868, 17862, 17811, 17805, 17810, 17812, 17807, + 17796, 17804, 17806, 17801, 17793, 17794, 17813, 17800, 17795, 17802, + 17797, 17799, 17808, 17798, 17809, 17803, 17734, 17791, 17792, 35762, + 35762, 17883, 17885, 17882, 17881, 17878, 17877, 17880, 17879, 17886, + 17884, 35762, 35762, 35762, 35762, 35762, 35762, 17835, 17834, 17831, + 17833, 17832, 17826, 17829, 17830, 17828, 17827, 35762, 35762, 35762, + 35762, 35762, 35762, 23479, 23491, 23487, 23336, 23486, 23337, 23484, + 23475, 23488, 23489, 23490, 23335, 23334, 23333, 23485, 23332, 23328, + 23330, 23327, 23326, 23323, 23322, 23325, 23324, 23331, 23329, 35762, + 35762, 35762, 35762, 35762, 35762, 23340, 23454, 23471, 23456, 23458, + 23457, 23459, 23455, 23465, 23368, 23460, 23466, 23467, 23463, 23372, + 23453, 23415, 23414, 23445, 23461, 23369, 23464, 23470, 23468, 23469, + 23462, 23451, 23450, 23444, 23448, 23449, 23447, 23452, 23446, 23371, + 23424, 23442, 23443, 23431, 23433, 23432, 23434, 23418, 23435, 23438, + 23439, 23427, 23437, 23426, 23420, 23429, 23422, 23425, 23441, 23440, + 23436, 23428, 23430, 23421, 23423, 23419, 23413, 23398, 23399, 23407, + 23406, 23403, 23411, 23394, 23396, 23412, 23401, 23393, 23408, 23410, + 23409, 23395, 23397, 23392, 23405, 23402, 23400, 23404, 23391, 23389, + 23390, 23388, 23387, 23370, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 23342, 23344, 23346, 23353, 23352, 23360, 23356, 23341, 23351, + 23367, 23354, 23366, 23358, 23357, 23348, 23355, 23359, 23345, 23362, + 23361, 23365, 23363, 23364, 23343, 23417, 23416, 23380, 23385, 23376, + 23381, 23377, 23373, 23378, 23374, 23386, 23375, 23383, 23384, 23350, + 23349, 23379, 23347, 23382, 35762, 35762, 35762, 35762, 35762, 5390, + 4975, 4969, 5658, 5406, 5403, 5394, 5532, 5241, 5232, 5281, 5354, 5326, + 5258, 5475, 5431, 5462, 5465, 5454, 5680, 5677, 5423, 5359, 5379, 5360, + 5380, 5357, 5377, 5358, 5378, 5419, 5417, 5418, 5415, 5416, 5413, 5386, + 5387, 5384, 5383, 5385, 5376, 5381, 5382, 5196, 5640, 5209, 5208, 5411, + 5561, 5559, 5503, 5501, 5522, 5521, 5519, 5513, 5512, 5442, 5441, 5433, + 5022, 4999, 5024, 5021, 5438, 5049, 4995, 4996, 5000, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19858, 19825, + 19824, 19805, 19804, 19811, 19819, 19818, 19823, 19822, 19810, 19808, + 19806, 19821, 19820, 19812, 19827, 19826, 19817, 19816, 19830, 19809, + 19831, 19829, 19832, 19813, 19814, 19815, 19828, 19803, 19807, 35762, + 19849, 19856, 19857, 19855, 19850, 19853, 19851, 19854, 19852, 19848, + 19846, 19847, 35762, 35762, 35762, 35762, 19840, 19837, 19839, 19845, + 19838, 19843, 19842, 19844, 19841, 19834, 19833, 19835, 35762, 35762, + 35762, 35762, 19836, 35762, 35762, 35762, 19859, 19860, 19867, 19869, + 19866, 19865, 19862, 19861, 19864, 19863, 19870, 19868, 30111, 30123, + 30106, 30103, 30121, 30124, 30105, 30104, 30118, 30113, 30112, 30119, + 30116, 30122, 30117, 30120, 30110, 30102, 30107, 30091, 30125, 30095, + 30096, 30114, 30109, 30108, 30115, 30094, 30092, 30093, 35762, 35762, + 30097, 30098, 30099, 30100, 30101, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 24411, 24433, 24393, 24395, + 24398, 24415, 24417, 24420, 24401, 24397, 24413, 24423, 24419, 24435, + 24402, 24400, 24399, 24424, 24422, 24421, 24404, 24403, 24410, 24426, + 24425, 24432, 24407, 24412, 24409, 24429, 24434, 24431, 24408, 24406, + 24405, 24430, 24428, 24427, 24392, 24394, 24414, 24416, 24396, 24418, + 35762, 35762, 35762, 35762, 24456, 24441, 24445, 24451, 24454, 24457, + 24443, 24447, 24448, 24452, 24444, 24442, 24455, 24450, 24449, 24453, + 24446, 24391, 24386, 24385, 24390, 24389, 24388, 24387, 24438, 24439, + 35762, 35762, 35762, 35762, 35762, 35762, 24464, 24466, 24463, 24462, + 24459, 24458, 24461, 24460, 24467, 24465, 24440, 35762, 35762, 35762, + 24436, 24437, 17836, 17855, 17848, 17851, 17853, 17846, 17844, 17842, + 17838, 17840, 17825, 17823, 17815, 17819, 17821, 17817, 17849, 17854, + 17847, 17850, 17852, 17845, 17843, 17841, 17837, 17839, 17824, 17822, + 17814, 17818, 17820, 17816, 4658, 4655, 4647, 4646, 4660, 4652, 4645, + 4644, 4663, 4654, 4651, 4650, 4653, 4657, 4649, 4648, 4665, 4661, 4659, + 4664, 4662, 4666, 4656, 4669, 4671, 4668, 4670, 4667, 35762, 35762, 4673, + 4672, 30159, 30157, 30158, 30175, 30174, 30173, 30199, 30165, 30164, + 30178, 30185, 30177, 30200, 30193, 30160, 30205, 30176, 30192, 30169, + 30168, 30182, 30181, 30201, 30204, 30167, 30166, 30170, 30180, 30183, + 30179, 30206, 30186, 30172, 30191, 30194, 30187, 30189, 30207, 30161, + 30162, 30163, 30171, 30190, 30208, 30184, 30197, 30198, 30195, 30196, + 30203, 30202, 30188, 30156, 30131, 30130, 30128, 30219, 30126, 30127, + 30129, 30132, 30133, 30134, 35762, 30227, 30234, 30238, 30235, 30244, + 30250, 30251, 30249, 30248, 30246, 30247, 30239, 30240, 30243, 30252, + 30236, 30242, 30237, 30245, 30241, 30218, 30230, 30231, 30211, 30212, + 30213, 30222, 30221, 30214, 35762, 35762, 30135, 30142, 30144, 30141, + 30140, 30137, 30136, 30139, 30138, 30145, 30143, 35762, 35762, 35762, + 35762, 35762, 35762, 30152, 30154, 30151, 30150, 30147, 30146, 30149, + 30148, 30155, 30153, 35762, 35762, 35762, 35762, 35762, 35762, 30228, + 30229, 30226, 30217, 30210, 30232, 30223, 30220, 30215, 30216, 30224, + 30225, 30209, 30233, 35762, 35762, 7898, 7866, 7982, 7900, 8132, 8145, + 8144, 8084, 7881, 8061, 8120, 8090, 7885, 8089, 8088, 8030, 8024, 8047, + 8106, 8048, 8107, 8118, 8077, 7981, 8093, 7884, 7883, 8130, 8008, 8009, + 8010, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 2698, 2697, 2700, 2696, 2699, 2608, 2609, 2625, 2626, 2629, 2630, 2647, + 2648, 2638, 2639, 2622, 2612, 2627, 2628, 2633, 2635, 2623, 2624, 2642, + 2615, 2616, 2631, 2632, 2643, 2654, 2653, 2619, 2618, 2641, 2652, 2655, + 2617, 2620, 2640, 2644, 2645, 2613, 2614, 2660, 2662, 2646, 2637, 2661, + 2650, 2651, 2649, 2659, 2701, 2712, 2715, 2716, 2706, 2707, 2704, 2705, + 2702, 2703, 2708, 2709, 2711, 2710, 2713, 2714, 2717, 2634, 2636, 2656, + 2621, 2657, 2658, 2610, 2611, 35762, 35762, 35762, 2725, 2727, 2724, + 2723, 2720, 2719, 2722, 2721, 2728, 2726, 2694, 2691, 2718, 2605, 2607, + 2606, 2693, 2680, 2678, 2681, 2672, 2673, 2679, 2675, 2677, 2676, 2674, + 2670, 2669, 2665, 2663, 2667, 2666, 2664, 2668, 2671, 2690, 2689, 2688, + 2687, 2682, 2684, 2685, 2686, 2683, 2695, 2692, 35762, 29705, 29703, + 29704, 29656, 29691, 29693, 29658, 29692, 29668, 29669, 29676, 29684, + 29679, 29670, 29677, 29681, 29690, 29671, 29685, 29678, 29672, 29683, + 29661, 29686, 29674, 29682, 29689, 29665, 29663, 29687, 29667, 29688, + 29680, 29651, 29652, 29653, 29711, 29710, 29712, 29709, 29707, 29708, + 29702, 29706, 29654, 29655, 29675, 29666, 29719, 29721, 29718, 29717, + 29714, 29713, 29716, 29715, 29722, 29720, 29650, 29664, 29662, 29673, + 29659, 29660, 3488, 3474, 3482, 3466, 3454, 3478, 3477, 3463, 3469, 3462, + 3455, 3486, 3472, 3464, 3481, 3465, 3483, 3480, 3485, 3470, 3453, 3468, + 3475, 3458, 3476, 3471, 3456, 3487, 3473, 3460, 3484, 3467, 3461, 3479, + 3459, 3457, 3489, 3490, 3497, 3503, 3500, 3504, 3505, 3501, 3506, 3502, + 3498, 3499, 3451, 3452, 3491, 3492, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 3496, 3494, 3495, 3493, 19688, 19687, 19686, 19700, + 19699, 19705, 19715, 19714, 19718, 19706, 19713, 19712, 19694, 19707, + 19691, 19690, 19689, 19698, 19697, 19696, 19695, 19704, 19703, 19709, + 19708, 19693, 19723, 19720, 19719, 19702, 19701, 19721, 19717, 19716, + 19722, 19724, 19733, 19732, 19738, 19740, 19736, 19737, 19734, 19735, + 19739, 19677, 19682, 19681, 19679, 19683, 19684, 19685, 19680, 19678, + 19731, 19730, 35762, 35762, 35762, 19725, 19728, 19729, 19727, 19726, + 19747, 19749, 19746, 19745, 19742, 19741, 19744, 19743, 19750, 19748, + 35762, 35762, 35762, 19711, 19710, 19692, 25311, 25313, 25310, 25309, + 25306, 25305, 25308, 25307, 25314, 25312, 25284, 25275, 25273, 25272, + 25274, 25285, 25269, 25268, 25270, 25271, 25287, 25283, 25281, 25280, + 25282, 25289, 25295, 25296, 25294, 25297, 25286, 25279, 25276, 25278, + 25277, 25288, 25290, 25291, 25293, 25292, 25299, 25300, 25298, 25304, + 25303, 25315, 25302, 25301, 10125, 10102, 10108, 10164, 10146, 10154, + 10144, 10145, 10163, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 13460, 13491, 13468, 13497, 13466, 13496, 13489, 13486, 13498, 13478, + 13480, 13492, 13494, 13499, 13482, 13488, 13490, 13484, 13487, 13500, + 13481, 13477, 13467, 13495, 13483, 13461, 13464, 13476, 13463, 13462, + 13493, 13475, 13471, 13474, 13472, 13502, 13469, 13473, 13503, 13501, + 13465, 13485, 13459, 35762, 35762, 13458, 13470, 13479, 29701, 29699, + 29700, 29698, 29697, 29696, 29695, 29694, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 33625, 33638, 33627, 33605, 33619, 33633, + 33634, 33635, 33620, 33636, 33623, 33631, 33626, 33624, 33632, 33630, + 33629, 33637, 33618, 33616, 33607, 33614, 33606, 33617, 33615, 33596, + 33597, 33599, 33600, 33612, 33610, 33611, 33609, 33598, 33602, 33608, + 33621, 33604, 33613, 33601, 33628, 33622, 33603, 35762, 35762, 35762, + 35762, 35762, 18684, 18685, 19290, 18681, 18686, 18687, 18658, 18657, + 19275, 19268, 18690, 18691, 18664, 18692, 18670, 18665, 18666, 19225, + 19226, 19227, 19270, 18668, 19262, 18784, 18694, 18671, 18679, 18674, + 18697, 19230, 19229, 19228, 18698, 18699, 18701, 18659, 18712, 18646, + 14004, 14007, 14003, 14006, 14002, 9995, 23018, 23019, 23009, 23010, + 23021, 23022, 23012, 23024, 23014, 23025, 23026, 23027, 23028, 23029, + 23030, 23013, 23016, 23017, 23031, 23011, 23033, 23034, 23036, 23167, + 23274, 23168, 23276, 23171, 23196, 23210, 23264, 23249, 23279, 23217, + 23287, 23298, 23227, 23214, 23247, 23250, 23271, 23173, 23251, 23266, + 23291, 23265, 23277, 23295, 23169, 23175, 23218, 23201, 23219, 23195, + 19431, 19439, 19441, 19442, 14213, 14209, 14212, 14211, 14210, 19351, + 18770, 18819, 18905, 19047, 19067, 19144, 19171, 19164, 19209, 19245, + 19413, 19297, 23085, 18976, 19255, 18714, 18983, 19140, 18715, 19350, + 18771, 18823, 18906, 18919, 19002, 19029, 19048, 19073, 19145, 19174, + 19210, 18892, 19358, 19385, 19414, 18733, 18759, 18822, 18882, 19133, + 19181, 19218, 18973, 19129, 18894, 19337, 18900, 23275, 23176, 23194, + 23212, 23258, 23215, 23202, 23263, 23286, 23229, 23230, 23177, 23179, + 23232, 23236, 23238, 23182, 23228, 23278, 23246, 23245, 23188, 23172, + 23252, 23262, 23211, 23267, 23293, 23294, 23190, 23297, 23288, 23207, + 23209, 23208, 23213, 23290, 7874, 7873, 8122, 8121, 8074, 7965, 8073, + 7758, 7964, 7757, 8022, 7770, 8075, 7897, 8086, 8113, 7983, 8137, 8138, + 8005, 7996, 7997, 7998, 8000, 8007, 8003, 8032, 7988, 8034, 8011, 7989, + 7990, 8036, 7991, 7992, 8021, 8025, 8013, 8040, 7995, 8027, 8028, 8026, + 8004, 8012, 8016, 8037, 8002, 8019, 8029, 7994, 8015, 8018, 8136, 7987, + 7986, 7869, 8142, 7872, 7864, 7878, 7767, 8042, 8097, 18174, 18731, + 18210, 18773, 18209, 18772, 18208, 18769, 18217, 18788, 18242, 18825, + 18241, 18824, 18240, 18820, 18236, 18814, 18237, 18815, 18268, 18868, + 18269, 18869, 18257, 18856, 18266, 18866, 18248, 18847, 18293, 18908, + 18301, 18918, 18320, 18938, 18319, 18937, 18317, 18935, 18314, 18932, + 18313, 18931, 18337, 18965, 18331, 18953, 18368, 19000, 18365, 18997, + 18369, 19001, 18376, 19014, 18377, 19015, 18387, 19024, 18383, 19011, + 18391, 19045, 18393, 19050, 18392, 19049, 18411, 19072, 18410, 19071, + 18405, 19064, 18401, 19059, 18442, 19108, 18441, 19107, 18419, 19100, + 18420, 19101, 18470, 19143, 18472, 19147, 18481, 19161, 18479, 19159, + 18480, 19160, 18486, 19166, 18506, 19204, 18504, 19202, 18503, 19195, + 18498, 19197, 18505, 19203, 18526, 19243, 18525, 19242, 18527, 19246, + 18521, 19236, 18557, 19318, 18578, 19341, 18550, 19311, 18577, 19340, + 18569, 19330, 18590, 19361, 18589, 19357, 18603, 19374, 18604, 19375, + 18600, 19371, 18602, 19373, 18601, 19372, 18609, 19380, 18608, 19379, + 18614, 19390, 18625, 19404, 18629, 19408, 18631, 19410, 18939, 19241, + 19376, 19400, 18732, 19038, 19037, 19039, 18515, 18829, 18168, 18725, + 18184, 18743, 18180, 18739, 18179, 18738, 18178, 18737, 18182, 18741, + 18181, 18740, 18163, 18720, 18162, 18719, 18161, 18718, 18165, 18722, + 18164, 18721, 18262, 18861, 18273, 18874, 18265, 18865, 18253, 18852, + 18252, 18851, 18251, 18850, 18256, 18855, 18255, 18854, 18338, 18966, + 18328, 18957, 18433, 19093, 18453, 19119, 18425, 19085, 18424, 19084, + 18423, 19083, 18427, 19087, 18426, 19086, 18450, 19116, 18449, 19115, + 18448, 19114, 18452, 19118, 18451, 19117, 18560, 19321, 18567, 19328, + 18564, 19325, 18563, 19324, 18562, 19323, 18566, 19327, 18565, 19326, + 18615, 19391, 18613, 19389, 18617, 19393, 18621, 19399, 18396, 19053, + 18397, 19054, 18618, 19394, 14051, 14043, 14056, 14048, 14052, 14044, + 14054, 14046, 13817, 13809, 13820, 13812, 13818, 13810, 13822, 13814, + 14074, 14071, 14076, 14073, 14075, 14072, 35762, 35762, 13857, 13854, + 13859, 13856, 13858, 13855, 35762, 35762, 14089, 14081, 14094, 14086, + 14090, 14082, 14092, 14084, 13841, 13833, 13844, 13836, 13842, 13834, + 13846, 13838, 14115, 14106, 14118, 14109, 14117, 14108, 14116, 14107, + 13869, 13864, 13872, 13867, 13871, 13866, 13870, 13865, 14176, 14173, + 14178, 14175, 14177, 14174, 35762, 35762, 13903, 13900, 13905, 13902, + 13904, 13901, 35762, 35762, 14135, 14126, 14138, 14129, 14137, 14128, + 14136, 14127, 35762, 13915, 35762, 13918, 35762, 13917, 35762, 13916, + 14156, 14148, 14161, 14153, 14157, 14149, 14159, 14151, 13887, 13879, + 13890, 13882, 13888, 13880, 13892, 13884, 14041, 14061, 14078, 14077, + 14101, 14099, 14121, 14122, 14180, 14179, 14141, 14142, 14168, 14166, + 35762, 35762, 14058, 14050, 14057, 14049, 14053, 14045, 14055, 14047, + 13824, 13816, 13821, 13813, 13819, 13811, 13823, 13815, 14096, 14088, + 14095, 14087, 14091, 14083, 14093, 14085, 13848, 13840, 13845, 13837, + 13843, 13835, 13847, 13839, 14163, 14155, 14162, 14154, 14158, 14150, + 14160, 14152, 13894, 13886, 13891, 13883, 13889, 13881, 13893, 13885, + 14040, 14065, 14042, 14063, 14062, 35762, 14059, 14060, 13826, 13830, + 13827, 13828, 13825, 14000, 14028, 14029, 14033, 13949, 14102, 14103, + 14100, 35762, 14097, 14098, 13861, 13860, 13851, 13850, 13849, 14032, + 14031, 14030, 14120, 14124, 14113, 14112, 35762, 35762, 14119, 14111, + 13873, 13877, 13874, 13875, 35762, 13957, 13956, 13955, 14140, 14144, + 14133, 14132, 14188, 14187, 14139, 14131, 13920, 13924, 13921, 13922, + 13910, 13951, 13950, 14227, 35762, 35762, 14169, 14170, 14167, 35762, + 14164, 14165, 13907, 13906, 13897, 13896, 13895, 14025, 13954, 35762, + 12447, 12444, 12449, 12446, 32044, 13184, 28853, 13117, 26923, 32017, + 14524, 35568, 35567, 35569, 19570, 27221, 15992, 24652, 13116, 12448, + 12445, 15946, 10941, 10915, 19515, 27171, 28728, 28726, 19475, 27145, + 10914, 10909, 10223, 10908, 4694, 32505, 25769, 32572, 15964, 15995, + 19876, 26269, 19569, 27220, 26798, 19568, 27219, 24257, 26502, 26503, + 26884, 10923, 32519, 27088, 27082, 27096, 5706, 28727, 28729, 27105, + 10945, 16335, 26084, 32621, 5992, 5707, 2513, 15994, 13188, 19523, 27182, + 10946, 26947, 13036, 32418, 27087, 3828, 3867, 20532, 27093, 7742, 32569, + 8148, 29770, 16351, 13152, 32024, 26943, 13177, 13139, 32573, 13178, + 10887, 32530, 33645, 22296, 34177, 13310, 16353, 16355, 16354, 35762, + 19567, 27218, 13132, 26797, 16249, 7, 16248, 6, 24252, 24650, 29741, + 29729, 35762, 35762, 29736, 29735, 29738, 29737, 29728, 29742, 29732, + 29733, 29727, 29731, 29734, 29730, 29615, 29617, 29614, 29613, 29610, + 29609, 29612, 29611, 29604, 29616, 29605, 29606, 29603, 29607, 29608, + 35762, 19428, 19429, 19437, 19443, 19427, 19430, 19433, 19434, 19435, + 19436, 19438, 19426, 19440, 35762, 35762, 35762, 13033, 7755, 8394, + 13194, 20484, 22903, 24185, 26526, 27538, 34181, 24384, 10881, 13034, + 18028, 32539, 11052, 13505, 26527, 14277, 2524, 16001, 5825, 20485, + 29183, 31790, 16239, 32540, 24700, 20985, 27407, 18155, 3742, 29165, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 8051, 8105, 8065, 8117, 7763, 7779, + 8044, 8102, 8110, 7778, 7762, 8124, 7910, 7901, 7904, 7907, 7902, 8054, + 7903, 7905, 7906, 8094, 7891, 7764, 8131, 8143, 8056, 8062, 8109, 8055, + 8043, 8101, 7766, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 0, 100, 10957, 10247, + 5829, 5708, 4959, 13035, 27726, 10249, 27723, 27715, 3917, 10958, 26688, + 26689, 27716, 3918, 27717, 27724, 20657, 10959, 24743, 29081, 27719, + 10955, 10960, 27720, 3919, 10961, 26860, 27066, 27750, 31685, 32492, + 33639, 10962, 26078, 26086, 16350, 3920, 32563, 17143, 951, 27712, 3916, + 12503, 27722, 27713, 27714, 32547, 27718, 27725, 346, 3633, 13508, 10236, + 16246, 27395, 13096, 10969, 10968, 10954, 10956, 10970, 32556, 32557, + 27092, 32558, 10967, 10963, 10964, 10965, 10966, 26888, 32542, 26504, + 2584, 32560, 29846, 33781, 33779, 33783, 33784, 33789, 33777, 33788, + 33787, 33775, 33782, 33774, 33776, 33785, 33773, 33790, 13189, 27365, + 27376, 27377, 27364, 27361, 27370, 27372, 27380, 27381, 27373, 27379, + 27375, 27358, 27366, 27362, 27368, 28915, 28919, 28920, 28914, 28911, + 28923, 28922, 28910, 28924, 28921, 28909, 28918, 28913, 28916, 28912, + 28917, 27369, 27363, 27374, 27378, 19187, 27371, 27360, 27359, 27367, + 33791, 32551, 32550, 35762, 35762, 35762, 35762, 19571, 32718, 27223, + 10989, 19497, 32643, 24682, 24656, 29055, 29069, 19591, 27245, 19654, + 27322, 19651, 32768, 27321, 11022, 19594, 27248, 19602, 32731, 27230, + 11003, 32644, 19600, 27254, 19587, 27241, 19505, 19500, 11026, 32728, + 32729, 10998, 10999, 27237, 10990, 960, 7726, 24684, 19582, 965, 7728, + 19620, 19614, 32745, 32742, 27282, 27276, 11039, 11036, 27256, 32720, + 19604, 19666, 32771, 27288, 11047, 19621, 27271, 19657, 19504, 27264, + 19655, 32735, 27262, 11029, 19502, 32646, 24694, 24668, 29066, 29078, + 19641, 27312, 19670, 27292, 32721, 10991, 19661, 32736, 27268, 11030, + 19581, 27236, 19652, 32772, 27323, 11024, 32773, 32775, 32776, 32777, + 32779, 32780, 27324, 24683, 29058, 32648, 27132, 11001, 32032, 19601, + 27255, 19499, 19585, 27239, 19498, 19665, 27287, 19507, 13172, 8153, + 26414, 32012, 32011, 12437, 16167, 24141, 12386, 24703, 28895, 8161, + 10707, 28888, 12451, 24100, 24088, 24092, 22907, 22914, 10882, 10689, + 27755, 2512, 27333, 4695, 29403, 8400, 13183, 26887, 16240, 27122, 947, + 22170, 29184, 10693, 10708, 26273, 24708, 20493, 20500, 16328, 32623, + 16313, 10912, 32529, 8167, 29763, 33769, 7729, 7720, 962, 32013, 3634, + 26977, 26886, 10883, 13038, 13330, 15982, 32322, 27094, 16347, 28848, + 34185, 24713, 22913, 2517, 24704, 1043, 1046, 24340, 351, 24705, 353, + 32520, 348, 12489, 13328, 10699, 1047, 13329, 1044, 16130, 7754, 12487, + 27331, 27332, 8346, 12504, 12490, 29565, 10252, 12473, 22181, 26949, + 24715, 16006, 24716, 29596, 19770, 13741, 19772, 13744, 19761, 13734, + 23857, 23856, 3632, 24714, 24718, 24717, 24345, 24342, 19769, 13740, + 24344, 24341, 19771, 13742, 24346, 24343, 26861, 29633, 26870, 29642, + 26869, 29641, 10710, 10713, 29621, 29749, 24701, 24702, 29627, 29755, + 24338, 24339, 29626, 29754, 23611, 23612, 23613, 29276, 29365, 29278, + 29367, 29214, 29221, 6503, 6449, 6508, 6241, 6254, 6504, 6230, 6513, + 6255, 29532, 29525, 29546, 29480, 27197, 19537, 10976, 32652, 2518, + 22922, 32536, 13173, 32523, 10939, 10712, 24712, 10715, 24337, 26871, + 29643, 24698, 8162, 24699, 8163, 25799, 16129, 23614, 15761, 16336, + 34206, 24186, 24655, 27128, 27193, 24097, 24098, 24099, 24093, 10529, + 10884, 29567, 10691, 4099, 19549, 27158, 19513, 27169, 27095, 9642, 9641, + 10933, 10932, 10911, 10936, 26682, 12474, 19775, 13750, 33682, 33681, + 19759, 13745, 12472, 12471, 12469, 12470, 10711, 10714, 24709, 24710, + 29277, 29366, 19760, 13733, 26868, 29640, 24706, 10705, 24707, 10706, + 33644, 22899, 32651, 10979, 12387, 12389, 28896, 12392, 12391, 28897, + 12390, 12388, 8164, 8165, 28889, 8166, 28890, 35480, 10530, 12384, 15976, + 32636, 10980, 26885, 26521, 33954, 19471, 27140, 19548, 27149, 4090, + 4087, 32441, 32438, 27085, 29309, 2508, 27736, 27732, 31683, 26806, + 33693, 26685, 32554, 33945, 15974, 32437, 32440, 4086, 4089, 32434, 4080, + 13198, 28954, 32637, 25791, 12500, 34190, 17145, 19564, 27213, 12499, + 3630, 10218, 349, 29856, 32456, 10700, 8172, 28880, 8348, 8349, 991, + 1029, 1015, 1003, 1004, 1020, 1001, 971, 976, 1026, 1031, 1017, 1016, + 1011, 1018, 999, 1023, 1012, 1019, 974, 983, 982, 1005, 1008, 984, 1037, + 1010, 1034, 980, 1009, 1007, 1035, 987, 1006, 1022, 981, 988, 997, 975, + 1036, 1021, 972, 1002, 1033, 978, 1027, 996, 973, 985, 998, 1039, 1038, + 977, 979, 1040, 1028, 1025, 1014, 1013, 986, 1032, 989, 1024, 994, 993, + 1030, 990, 995, 992, 24719, 27127, 27937, 3526, 33656, 16312, 8170, + 10615, 12443, 8152, 34095, 12465, 354, 15472, 6285, 6507, 4635, 32622, + 23495, 15998, 25787, 25788, 26426, 26427, 10610, 28971, 1000, 10244, + 26872, 24569, 26874, 7749, 19552, 19553, 19551, 27165, 27166, 27164, + 19520, 19527, 19519, 27179, 27186, 27178, 19469, 19467, 19468, 9644, + 27138, 27136, 27137, 16323, 15947, 32688, 32704, 29646, 29644, 32443, + 4081, 4082, 26958, 19555, 27199, 15955, 15956, 15957, 15958, 10263, + 10266, 10267, 10256, 10260, 10268, 10257, 10261, 10264, 10255, 10259, + 10254, 10258, 10262, 10265, 29244, 27067, 13061, 33653, 22739, 22731, + 22738, 22732, 22736, 22737, 22735, 22734, 22733, 11219, 13313, 32436, + 4085, 32429, 4084, 32444, 4092, 34104, 3631, 29592, 13145, 8, 12385, + 10245, 3856, 3821, 3900, 3804, 3857, 3822, 3859, 357, 29590, 32330, + 15975, 3840, 3843, 3845, 3837, 10937, 3879, 3751, 26820, 26817, 26818, + 26819, 25220, 29841, 29849, 29850, 29830, 29828, 29831, 29842, 29816, + 29817, 29835, 29837, 29836, 29834, 29818, 29847, 29848, 29820, 29824, + 29823, 29822, 29821, 29839, 29853, 29829, 29819, 29827, 29851, 29832, + 29833, 29844, 29843, 29845, 29854, 29825, 3924, 25774, 29840, 29826, + 29852, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 25232, 25227, 25230, 25231, + 25224, 25225, 25223, 25222, 25229, 25226, 25228, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 6251, 6248, 6247, + 6244, 6243, 6246, 6245, 6252, 6250, 6472, 6452, 6497, 6485, 6467, 6455, + 6471, 6469, 6451, 6498, 6486, 26360, 26358, 26357, 26354, 26353, 26356, + 26355, 26361, 26359, 26352, 26343, 26351, 26349, 26344, 26345, 26347, + 26348, 26342, 26346, 26350, 10550, 10562, 10559, 10544, 10541, 10556, + 10553, 10568, 10547, 24739, 24732, 24740, 24738, 24733, 24734, 24735, + 24737, 24731, 24742, 24741, 26388, 26389, 26390, 26391, 26392, 26393, + 26394, 26395, 26396, 26397, 26398, 26399, 26400, 26401, 26402, 26403, + 26404, 26405, 26406, 26407, 26408, 26409, 26410, 26411, 26412, 26413, + 6395, 6396, 6397, 6398, 6399, 6400, 6401, 6402, 6403, 6404, 6405, 6406, + 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, + 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6428, 6429, 6430, + 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, + 6443, 6444, 6445, 6446, 6249, 24266, 24264, 24262, 24267, 24268, 24270, + 24271, 24265, 24269, 24263, 10903, 10901, 10900, 10897, 10896, 10899, + 10898, 10904, 10902, 10895, 24261, 4206, 4153, 4202, 4149, 4196, 4148, + 4195, 4147, 4194, 4146, 4193, 4145, 4190, 4123, 4117, 4144, 4189, 4121, + 4115, 4143, 4201, 4227, 4221, 4138, 4200, 4225, 4219, 4136, 4205, 4243, + 4220, 4116, 4240, 4122, 4226, 4152, 4204, 4242, 4218, 4114, 4239, 4120, + 4224, 4151, 4188, 4157, 4211, 4125, 4119, 4214, 4160, 4142, 4199, 4156, + 4210, 4228, 4222, 4213, 4159, 4137, 4203, 4158, 4212, 4241, 4217, 4118, + 4238, 4162, 4216, 4155, 4209, 4124, 4223, 4215, 4161, 4150, 4186, 4140, + 4185, 4139, 4113, 4109, 4130, 4127, 4105, 4129, 4126, 4104, 4233, 4230, + 4108, 4232, 4229, 4107, 4245, 4236, 4112, 4244, 4235, 4111, 4131, 4128, + 4103, 4234, 4231, 4106, 4246, 4237, 4110, 4164, 4163, 4165, 4166, 4182, + 4181, 4183, 4191, 4197, 4208, 4187, 4132, 4134, 4154, 4141, 4192, 4198, + 4133, 4135, 32667, 20583, 20582, 20585, 20538, 20581, 20586, 20584, + 13205, 19514, 19535, 19546, 19485, 19534, 19495, 19496, 27152, 19790, + 22298, 10230, 32691, 27161, 26940, 26941, 26932, 26933, 26934, 26935, + 26936, 26937, 26938, 26939, 3877, 34082, 34091, 34085, 29418, 29427, + 29417, 29424, 29426, 29416, 3875, 34079, 3871, 34073, 3908, 34115, 3853, + 34063, 3902, 34111, 3901, 34110, 3858, 34070, 3863, 34069, 3862, 34068, + 3806, 34028, 3805, 34027, 3826, 34054, 3825, 34053, 3824, 34052, 3796, + 34017, 34018, 13136, 20588, 34004, 10886, 6224, 4698, 3747, 6216, 6225, + 6217, 6222, 6221, 6223, 19483, 27150, 16343, 16341, 32670, 20536, 32689, + 32706, 20579, 20559, 32672, 20539, 3833, 3832, 3903, 3904, 33970, 29421, + 29428, 29423, 29420, 34094, 34112, 32659, 32660, 18152, 34092, 34088, + 34089, 34093, 34011, 34009, 34010, 34012, 32687, 32711, 20557, 34059, + 3850, 34058, 3849, 20564, 3881, 7743, 32616, 28961, 8155, 3890, 34101, + 19800, 32049, 29647, 2509, 10270, 8173, 25795, 3896, 34103, 2731, 2737, + 2738, 27545, 32618, 15970, 34076, 3888, 27911, 27090, 3831, 3869, 34051, + 34108, 34072, 34026, 28859, 5822, 26956, 3744, 4958, 970, 25895, 6179, + 8381, 8380, 29568, 13106, 102, 14650, 26492, 35478, 32508, 32509, 32514, + 32511, 32513, 32512, 32510, 32507, 33966, 34038, 34080, 3876, 34099, + 13129, 18156, 22729, 13112, 11215, 20893, 16465, 27615, 32781, 24571, + 26787, 2507, 31672, 13327, 5696, 19675, 33695, 20488, 27708, 27543, 5702, + 2585, 26680, 33976, 33992, 33995, 33971, 33979, 33989, 3767, 3783, 3786, + 3762, 3770, 3780, 3889, 34041, 34022, 3795, 34081, 3816, 3801, 34013, + 15971, 26945, 12340, 3512, 3513, 23616, 23615, 23617, 33964, 11220, + 32632, 26984, 26985, 26986, 26987, 26988, 26989, 26990, 26991, 3906, + 26983, 26419, 26524, 33967, 10532, 10533, 10534, 10535, 10536, 10537, + 34006, 34008, 3748, 3750, 23492, 23493, 10572, 10575, 10574, 10573, + 34032, 3812, 14651, 968, 8389, 29561, 27703, 345, 13151, 13323, 29562, + 2520, 13149, 26075, 32026, 32025, 33939, 15829, 10972, 10973, 16327, + 20892, 20891, 20890, 33657, 15965, 22307, 22284, 22297, 21059, 10694, + 32634, 8372, 13311, 24381, 5832, 26228, 16466, 33685, 6182, 3851, 27758, + 27748, 26951, 27752, 28966, 3449, 29494, 34029, 34030, 3809, 3810, 28962, + 29649, 26961, 3883, 32048, 32546, 32545, 34023, 8390, 10614, 25794, + 26665, 5764, 15471, 6237, 5833, 24643, 355, 3897, 34106, 3829, 34049, + 11061, 15328, 19472, 29537, 13100, 3894, 27063, 27064, 2516, 15256, + 26499, 27211, 19562, 16348, 3760, 27916, 6214, 5824, 15934, 13324, 13325, + 20988, 23510, 32617, 13185, 13146, 13114, 27699, 29243, 28857, 16005, + 26511, 31791, 16363, 15232, 13312, 9637, 34033, 3884, 32665, 3887, 20580, + 34074, 34042, 31684, 31671, 226, 12462, 26972, 26966, 33687, 34188, + 20578, 26501, 32705, 34062, 3914, 5998, 15254, 23610, 15290, 2741, 15246, + 26077, 15336, 25778, 15292, 18638, 27761, 26074, 20894, 29566, 13181, + 13179, 15275, 13175, 3813, 34037, 29173, 29594, 6510, 25776, 3761, 26076, + 15294, 26677, 27763, 15245, 25777, 12339, 12336, 12334, 28851, 12335, + 15268, 32567, 28854, 31677, 25775, 15320, 28849, 3811, 34034, 12337, + 6499, 15319, 28964, 32320, 15253, 29172, 15313, 2730, 12338, 15270, 8386, + 27762, 24329, 20577, 32703, 20562, 32708, 3915, 34066, 34031, 3798, + 15272, 19797, 22304, 15335, 15303, 15304, 15264, 15265, 15287, 15286, + 9649, 15273, 15279, 15249, 27392, 13150, 27391, 22291, 22294, 22285, + 22286, 22292, 22295, 15283, 15296, 15282, 15295, 19789, 19787, 22290, + 22293, 10597, 10595, 10594, 10591, 10590, 10593, 10592, 10598, 10596, + 10589, 10608, 10605, 10604, 10601, 10600, 10603, 10602, 10609, 10607, + 10599, 10587, 10584, 10583, 10580, 10579, 10582, 10581, 10588, 10586, + 10578, 15331, 15334, 15291, 15261, 15309, 15297, 15316, 11053, 15300, + 32502, 15322, 10233, 15260, 3864, 32040, 32043, 3865, 15247, 15248, + 29555, 15259, 27216, 19557, 2597, 13196, 15288, 15327, 24721, 9643, + 24723, 6286, 34117, 3921, 3923, 3922, 15250, 15252, 15251, 31678, 15321, + 33960, 15332, 25789, 10905, 32023, 34105, 26505, 25785, 25784, 19512, + 27168, 25896, 27074, 29760, 33643, 21756, 20508, 21577, 29522, 29523, + 34021, 969, 12394, 20576, 32684, 19494, 27159, 13204, 18144, 18143, + 19453, 19452, 19493, 20523, 20512, 32653, 20589, 34014, 34015, 34016, + 34087, 34090, 21757, 21751, 21760, 21754, 21759, 21753, 21758, 21752, + 21761, 21755, 32749, 11043, 964, 7722, 27131, 20513, 20518, 20511, 20515, + 20520, 20510, 20514, 20519, 20516, 20521, 20522, 4624, 4369, 4497, 4370, + 4561, 4434, 4498, 4371, 4593, 4466, 4530, 4403, 4562, 4435, 4499, 4372, + 4609, 4482, 4546, 4419, 4578, 4451, 4515, 4388, 4594, 4467, 4531, 4404, + 4563, 4436, 4500, 4373, 4617, 4490, 4554, 4427, 4586, 4459, 4523, 4396, + 4602, 4475, 4539, 4412, 4571, 4444, 4508, 4381, 4610, 4483, 4547, 4420, + 4579, 4452, 4516, 4389, 4595, 4468, 4532, 4405, 4564, 4437, 4501, 4374, + 4621, 4494, 4558, 4431, 4590, 4463, 4527, 4400, 4606, 4479, 4543, 4416, + 4575, 4448, 4512, 4385, 4614, 4487, 4551, 4424, 4583, 4456, 4520, 4393, + 4599, 4472, 4536, 4409, 4568, 4441, 4505, 4378, 4618, 4491, 4555, 4428, + 4587, 4460, 4524, 4397, 4603, 4476, 4540, 4413, 4572, 4445, 4509, 4382, + 4611, 4484, 4548, 4421, 4580, 4453, 4517, 4390, 4596, 4469, 4533, 4406, + 4565, 4438, 4502, 4375, 4623, 4496, 4560, 4433, 4592, 4465, 4529, 4402, + 4608, 4481, 4545, 4418, 4577, 4450, 4514, 4387, 4616, 4489, 4553, 4426, + 4585, 4458, 4522, 4395, 4601, 4474, 4538, 4411, 4570, 4443, 4507, 4380, + 4620, 4493, 4557, 4430, 4589, 4462, 4526, 4399, 4605, 4478, 4542, 4415, + 4574, 4447, 4511, 4384, 4613, 4486, 4550, 4423, 4582, 4455, 4519, 4392, + 4598, 4471, 4535, 4408, 4567, 4440, 4504, 4377, 4622, 4495, 4559, 4432, + 4591, 4464, 4528, 4401, 4607, 4480, 4544, 4417, 4576, 4449, 4513, 4386, + 4615, 4488, 4552, 4425, 4584, 4457, 4521, 4394, 4600, 4473, 4537, 4410, + 4569, 4442, 4506, 4379, 4619, 4492, 4556, 4429, 4588, 4461, 4525, 4398, + 4604, 4477, 4541, 4414, 4573, 4446, 4510, 4383, 4612, 4485, 4549, 4422, + 4581, 4454, 4518, 4391, 4597, 4470, 4534, 4407, 4566, 4439, 4503, 4376, + 27318, 27317, 19656, 27263, 19503, 27319, 19658, 27265, 11000, 32730, + 32767, 11021, 19660, 27267, 19640, 27311, 27320, 27238, 32732, 11004, + 27250, 27249, 27313, 27315, 27314, 19605, 27257, 19659, 27266, 19583, + 27235, 19603, 27231, 24681, 24661, 24687, 24660, 29059, 29071, 24686, + 24659, 29056, 29070, 27337, 13098, 29057, 24657, 13099, 27338, 24658, + 24685, 33948, 2500, 2499, 2502, 2503, 27217, 19556, 32426, 4083, 32428, + 32427, 20560, 20555, 961, 7719, 27226, 19572, 27924, 27242, 19588, 27234, + 19501, 32770, 19462, 19461, 32641, 32640, 19463, 32642, 19460, 32639, + 19619, 27281, 32744, 11038, 19613, 27275, 32741, 11035, 19618, 27280, + 32743, 11037, 19612, 27274, 32740, 11034, 19615, 32739, 27279, 11032, + 19617, 19611, 27277, 27272, 19616, 19610, 27278, 27273, 32738, 11033, + 27143, 12479, 32324, 19576, 27228, 27227, 19756, 19579, 13729, 29620, + 19578, 29746, 19547, 27148, 32650, 10978, 32522, 35491, 35492, 19539, + 27201, 19541, 27203, 35486, 35482, 35487, 35483, 19524, 27183, 19522, + 27180, 19521, 27181, 19456, 27124, 19457, 27130, 10918, 10943, 19465, + 27134, 10893, 33670, 22179, 27129, 22180, 948, 5, 29185, 29186, 32543, + 27080, 949, 27081, 25218, 25217, 22178, 22177, 22172, 22171, 22176, + 22174, 22175, 22173, 27103, 12441, 12440, 12438, 12439, 6226, 6514, 6501, + 6505, 6502, 6227, 6219, 6228, 32635, 6509, 6232, 6447, 6515, 6218, 6220, + 29486, 29481, 29552, 29538, 29539, 32578, 32501, 32499, 27540, 32498, + 27194, 19529, 33640, 4100, 4101, 3913, 32331, 32332, 34046, 3820, 19544, + 27206, 19473, 27144, 16164, 32258, 16241, 10971, 29429, 16166, 27943, + 12480, 12481, 16007, 13613, 32015, 10983, 10984, 3799, 3834, 34007, 3749, + 12493, 12496, 12492, 12495, 12491, 12494, 27409, 27075, 29018, 27076, + 3737, 3736, 10925, 32518, 19563, 27212, 32333, 22915, 24087, 24085, + 24086, 24095, 24094, 24090, 24091, 32580, 32579, 24089, 23317, 29645, + 26942, 13142, 16322, 16314, 6519, 966, 19872, 19873, 19874, 16315, 26944, + 16319, 16316, 16320, 16318, 16321, 16317, 16463, 18145, 35488, 35489, + 35490, 26779, 26784, 26782, 26778, 26781, 26780, 26783, 22908, 22909, + 22911, 22910, 26775, 26776, 33684, 23609, 23608, 27747, 28939, 23604, + 23605, 6448, 23606, 6242, 26777, 22912, 23607, 16337, 27222, 35484, 362, + 16333, 32627, 32628, 16332, 16331, 32629, 32625, 16330, 32624, 16329, + 32626, 16334, 7735, 7741, 10926, 10927, 7736, 20496, 20504, 10916, 10917, + 32576, 32577, 28879, 28878, 20501, 20498, 20506, 20497, 20505, 20495, + 20499, 20494, 28929, 20503, 20502, 35485, 35481, 12485, 16008, 32516, + 32517, 32326, 32325, 28723, 8174, 12486, 352, 1045, 12476, 26785, 12477, + 10906, 32571, 32034, 12484, 12488, 19773, 13748, 19774, 13749, 19765, + 13735, 19768, 13738, 19766, 13736, 19767, 13737, 19764, 13739, 19758, + 13731, 19757, 13730, 19755, 13727, 19753, 13725, 19752, 13724, 19751, + 13728, 19754, 13726, 28864, 28862, 28865, 28863, 10953, 10952, 10949, + 10948, 28722, 28721, 28720, 28719, 10919, 10921, 10920, 13743, 13732, + 19762, 13746, 19763, 13747, 28937, 18153, 28938, 18154, 12482, 26864, + 29636, 26865, 29637, 26867, 29639, 26863, 29635, 26866, 29638, 26862, + 29634, 10922, 10931, 29631, 29759, 29629, 29757, 29630, 29758, 29628, + 29756, 29623, 29751, 29624, 29752, 29622, 29750, 29625, 29753, 29311, + 29398, 7730, 7732, 7731, 7733, 29618, 29745, 29619, 29744, 29748, 29747, + 12393, 26683, 32496, 13170, 24654, 27928, 27929, 27925, 26506, 33642, + 10940, 33641, 10938, 20507, 27930, 27927, 27926, 10907, 10935, 10929, + 27084, 10709, 33655, 33654, 10977, 26272, 26271, 32521, 32524, 32515, + 32528, 32527, 10951, 10950, 32525, 18142, 10934, 34113, 24096, 24670, + 24696, 29068, 29080, 19506, 19609, 32734, 11028, 24667, 24693, 29065, + 29077, 19508, 32645, 27246, 27247, 19592, 19593, 29422, 29415, 29425, + 29419, 10525, 10526, 10524, 10523, 10889, 3836, 34047, 3911, 34116, 3854, + 34064, 34044, 3817, 15945, 3835, 3839, 34056, 3842, 34060, 3872, 3873, + 34077, 3819, 34045, 3907, 34114, 19459, 32027, 19458, 20517, 19648, + 19647, 19649, 19650, 19584, 19596, 19595, 19643, 19645, 19644, 19580, + 33947, 12478, 27077, 19573, 27233, 27232, 19674, 27327, 27078, 27224, + 32323, 19575, 19574, 27225, 11017, 27923, 27921, 34057, 3874, 34078, + 3861, 34067, 15280, 15293, 15257, 15255, 15258, 28866, 2595, 28867, 2594, + 3628, 27922, 19625, 32753, 27296, 11006, 19510, 32649, 24691, 24665, + 29063, 29075, 19637, 32764, 27308, 11018, 7727, 959, 19636, 32755, 27307, + 11008, 35762, 35762, 24692, 24666, 29064, 29076, 19627, 32763, 27298, + 11016, 15962, 33664, 19626, 32754, 27297, 11007, 19638, 32765, 27309, + 11019, 19608, 32733, 27260, 11027, 956, 957, 955, 958, 27068, 27069, + 24566, 24567, 13176, 27261, 35762, 29855, 32038, 32042, 32039, 32041, + 3827, 3905, 3866, 3807, 11010, 11011, 32757, 32758, 19630, 27301, 19629, + 27300, 3752, 3753, 3754, 3755, 3756, 3758, 3757, 3759, 27115, 27116, + 27113, 27114, 27110, 27112, 27109, 27111, 32774, 32638, 26082, 26081, + 26083, 2736, 6517, 6231, 3878, 3797, 32544, 15944, 3912, 3846, 3838, + 3841, 3844, 24572, 32430, 4077, 19785, 27393, 34036, 27394, 29383, 32620, + 14275, 26791, 26790, 26789, 26788, 32495, 26891, 2515, 15999, 26664, + 24349, 34061, 3800, 32537, 9639, 15230, 35570, 18035, 1042, 97, 33772, + 26801, 19484, 27151, 29569, 29570, 19646, 32769, 27316, 11023, 12498, + 12497, 27759, 27535, 27533, 27534, 27532, 27536, 27537, 12483, 32631, + 27757, 10974, 26424, 27091, 15473, 13513, 13515, 13549, 13523, 13519, + 13550, 13557, 13520, 13556, 13529, 13525, 13524, 13518, 13560, 13531, + 13532, 13533, 13534, 13536, 13538, 13542, 13546, 13559, 13521, 13558, + 13535, 13537, 13539, 13548, 13517, 13541, 13552, 13551, 13553, 13543, + 13555, 13544, 13545, 13554, 13527, 13514, 13526, 13522, 13528, 13540, + 13547, 13530, 13516, 13561, 13563, 13597, 13571, 13567, 13598, 13605, + 13568, 13604, 13577, 13573, 13572, 13566, 13608, 13579, 13580, 13581, + 13582, 13584, 13586, 13590, 13594, 13607, 13569, 13606, 13583, 13585, + 13587, 13596, 13565, 13589, 13600, 13599, 13601, 13591, 13603, 13592, + 13593, 13602, 13575, 13562, 13574, 13570, 13576, 13588, 13595, 13578, + 13564, 18375, 19017, 18378, 18467, 18483, 18751, 19240, 18318, 18936, + 18364, 18996, 18628, 19407, 18197, 18395, 18532, 18533, 19360, 18605, + 19377, 19359, 18323, 18943, 19305, 18864, 19281, 19097, 18675, 19432, + 23035, 18508, 18632, 8182, 8276, 8232, 8326, 8196, 8290, 8193, 8287, + 8237, 8331, 8227, 8321, 8231, 8325, 8198, 8292, 8229, 8323, 8235, 8329, + 8201, 8295, 8206, 8300, 8238, 8332, 8239, 8333, 8202, 8296, 8207, 8301, + 8234, 8328, 8236, 8330, 8228, 8322, 8230, 8324, 8240, 8334, 8204, 8298, + 8200, 8294, 8233, 8327, 8223, 8317, 8190, 8284, 8218, 8312, 8186, 8280, + 8191, 8285, 8192, 8286, 8187, 8281, 8216, 8310, 8226, 8320, 8188, 8282, + 8214, 8308, 8217, 8311, 8181, 8275, 8189, 8283, 8209, 8303, 8210, 8304, + 8205, 8299, 8212, 8306, 8211, 8305, 8208, 8302, 8215, 8309, 8213, 8307, + 8221, 8315, 8219, 8313, 8220, 8314, 8222, 8316, 8336, 8337, 8338, 8340, + 8341, 8335, 8339, 8184, 8278, 8185, 8279, 8180, 8179, 8178, 8183, 8277, + 35762, 35762, 35762, 35762, 35762, 8274, 8271, 8272, 8273, 8269, 8270, + 8342, 13372, 13398, 13383, 13404, 13405, 13403, 13393, 13394, 13406, + 13387, 13396, 13399, 13401, 13407, 13389, 13392, 13397, 13391, 13395, + 13408, 13388, 13386, 13382, 13402, 13390, 13378, 13381, 13385, 13380, + 13379, 13400, 13384, 13373, 13377, 13375, 13410, 13374, 13376, 35762, + 13409, 35762, 35762, 35762, 35762, 35762, 13371, 35762, 35762, 32273, + 32284, 32285, 32278, 32280, 32263, 32302, 32274, 32277, 32275, 32276, + 32312, 32301, 32281, 32267, 32283, 32286, 32262, 32271, 32287, 32300, + 32282, 32268, 32307, 32272, 32313, 32295, 32261, 32269, 32303, 32304, + 32305, 32266, 32270, 32306, 32315, 32297, 32298, 32279, 32265, 32260, + 32288, 32290, 32289, 32291, 32292, 32299, 32293, 32308, 32309, 32310, + 32294, 32264, 32296, 32311, 32314, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 32316, 32317, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32259, 12933, + 12750, 12837, 12877, 12853, 12540, 12915, 12578, 12768, 12759, 12660, + 12993, 12608, 12593, 12924, 12884, 12569, 12784, 12797, 12645, 12649, + 12648, 12647, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 12867, 12873, 12871, 12868, 12870, 12869, 12872, 35762, 12559, + 12565, 12563, 12560, 12562, 12561, 12564, 35762, 12983, 12989, 12987, + 12984, 12986, 12985, 12988, 35762, 12552, 12558, 12556, 12553, 12555, + 12554, 12557, 35762, 12819, 12825, 12823, 12820, 12822, 12821, 12824, + 35762, 12728, 12734, 12732, 12729, 12731, 12730, 12733, 35762, 12960, + 12966, 12964, 12961, 12963, 12962, 12965, 35762, 12671, 12677, 12675, + 12672, 12674, 12673, 12676, 35762, 7788, 7826, 7823, 7790, 7820, 7821, + 7827, 7794, 7795, 7796, 7803, 7825, 7797, 7791, 7819, 7816, 7818, 7822, + 7806, 7805, 7824, 7792, 7828, 7802, 7789, 7815, 7811, 7813, 7800, 7814, + 7787, 7799, 27126, 27125, 19528, 27187, 19477, 27141, 26965, 26964, + 10892, 19531, 27192, 26974, 19511, 27167, 11222, 26270, 13169, 27086, + 15997, 10891, 11005, 32717, 10894, 10942, 16344, 26229, 15993, 32329, + 19492, 27157, 32328, 32327, 19554, 27198, 32439, 32442, 4088, 4091, + 19516, 27172, 19476, 27146, 32574, 25768, 29482, 13140, 27101, 33667, + 27330, 34176, 32548, 26963, 26973, 32559, 10224, 10225, 32549, 32433, + 32584, 32045, 29582, 33669, 34159, 5701, 10910, 27100, 10913, 10231, + 10930, 16345, 16346, 20533, 20534, 10928, 10888, 32526, 22281, 26268, + 26922, 8345, 8383, 8382, 32417, 22280, 22282, 19526, 27185, 19525, 27184, + 32431, 32432, 4078, 4079, 25219, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7615, + 7568, 7618, 7617, 7616, 7611, 7539, 7636, 7650, 7649, 7572, 7619, 7632, + 7631, 7601, 7600, 7599, 7598, 7628, 7637, 7627, 7626, 7587, 7586, 7590, + 7614, 35762, 7571, 7634, 7608, 7573, 7606, 7566, 7642, 7641, 7580, 7610, + 7609, 7620, 7570, 7574, 7595, 7537, 7579, 7630, 7629, 7542, 7625, 7553, + 7648, 7647, 7646, 7645, 7603, 7633, 7613, 7578, 7651, 7541, 7540, 7602, + 7607, 7584, 7583, 7582, 7638, 7569, 7644, 7643, 7557, 7621, 7589, 7556, + 7555, 7581, 7565, 7622, 7640, 7639, 7567, 7549, 7597, 7596, 7552, 7550, + 7605, 7604, 7612, 7543, 7559, 7551, 7561, 7547, 7577, 7576, 7575, 7545, + 7588, 7564, 7538, 7585, 7546, 7563, 7554, 7623, 7624, 7548, 7594, 7544, + 7592, 7560, 7593, 7562, 7635, 7591, 7558, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16662, 16644, + 16578, 16698, 16686, 16630, 16730, 16645, 16658, 16641, 16593, 16597, + 16582, 16567, 16633, 16719, 16665, 16636, 16670, 16747, 16704, 16675, + 16627, 16732, 16576, 16687, 16563, 16667, 16538, 16656, 16592, 16590, + 16685, 16613, 16615, 16595, 16545, 16744, 16570, 16678, 16632, 16714, + 16637, 16565, 16703, 16654, 16676, 16745, 16663, 16728, 16588, 16692, + 16579, 16647, 16731, 16694, 16553, 16712, 16554, 16706, 16624, 16621, + 16584, 16622, 16555, 16673, 16684, 16577, 16540, 16715, 16660, 16717, + 16683, 16657, 16727, 16638, 16707, 16572, 16738, 16583, 16566, 16612, + 16561, 16705, 16737, 16604, 16562, 16599, 16581, 16620, 16699, 16601, + 16569, 16585, 16668, 16634, 16648, 16722, 16711, 16643, 16749, 16602, + 16549, 16695, 16580, 16740, 16716, 16575, 16598, 16700, 16536, 16709, + 16702, 16726, 16616, 16560, 16710, 16541, 16677, 16696, 16635, 16661, + 16691, 16610, 16666, 16539, 16669, 16589, 16556, 16649, 16650, 16688, + 16537, 16652, 16723, 16664, 16550, 16708, 16568, 16617, 16721, 16631, + 16546, 16736, 16564, 16739, 16689, 16629, 16701, 16733, 16557, 16671, + 16542, 16690, 16680, 16679, 16611, 16552, 16559, 16543, 16653, 16735, + 16571, 16743, 16574, 16734, 16614, 16646, 16619, 16655, 16697, 16693, + 16672, 16548, 16746, 16600, 16639, 16718, 16642, 16713, 16640, 16742, + 16607, 16591, 16625, 16608, 16628, 16551, 16720, 16623, 16603, 16681, + 16558, 16618, 16605, 16544, 16682, 16573, 16741, 16626, 16748, 16651, + 16547, 16596, 16609, 16725, 16587, 16674, 16659, 16594, 16724, 16586, + 16729, 16606, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16031, 16029, + 16030, 16028, 16043, 16039, 16038, 16035, 16036, 16037, 16033, 16032, + 16040, 16034, 16044, 16042, 16128, 16027, 16126, 10698, 16362, 16127, + 16026, 16046, 19455, 27123, 19474, 27142, 19470, 27139, 19538, 27200, + 19464, 27133, 26803, 13506, 19536, 27196, 19540, 27202, 19543, 27205, + 19542, 27204, 33946, 27083, 10924, 20531, 26804, 15127, 15120, 15117, + 15123, 15122, 15125, 15124, 15128, 15126, 16124, 16123, 16045, 16122, + 14786, 14785, 33952, 33647, 33650, 33648, 33651, 33649, 6500, 16120, + 15121, 15119, 15118, 33646, 21138, 26422, 16125, 16121, 35762, 15849, + 15835, 15851, 15929, 15853, 15931, 15850, 15928, 15852, 15930, 15890, + 15880, 15892, 15882, 15894, 15884, 15891, 15881, 15893, 15883, 15854, + 15915, 15856, 15917, 15858, 15919, 15855, 15916, 15857, 15918, 15910, + 15875, 15912, 15877, 15848, 15914, 15879, 15911, 15876, 15913, 15878, + 15870, 15872, 15874, 15871, 15873, 15885, 15865, 15900, 15887, 15859, + 15902, 15889, 15868, 15904, 15886, 15866, 15901, 15888, 15867, 15903, + 15895, 15897, 15899, 15896, 15898, 15842, 15924, 15844, 15926, 15843, + 15925, 15905, 15907, 15909, 15906, 15908, 15838, 15920, 15922, 15921, + 15923, 15869, 15927, 15845, 15846, 35762, 35762, 7985, 7984, 16987, + 16986, 15933, 15932, 15834, 16984, 16914, 16843, 16916, 16977, 16918, + 16979, 16915, 16976, 16917, 16978, 16939, 16929, 16941, 16931, 16943, + 16933, 16940, 16930, 16942, 16932, 16919, 16964, 16921, 16966, 16923, + 16968, 16920, 16965, 16922, 16967, 16954, 16924, 16956, 16926, 16901, + 16958, 16928, 16955, 16925, 16957, 16927, 16881, 16883, 16885, 16882, + 16884, 16934, 16858, 16944, 16936, 16852, 16946, 16938, 16861, 16948, + 16935, 16859, 16945, 16937, 16860, 16947, 16876, 16862, 16879, 16877, + 16878, 16906, 16973, 16908, 16975, 16907, 16974, 16949, 16951, 16953, + 16950, 16952, 16902, 16969, 16971, 16970, 16972, 16880, 16963, 16896, + 16897, 16959, 16961, 16960, 16962, 16983, 16985, 16982, 16980, 16981, + 35762, 35762, 35762, 35762, 35762, 4063, 4071, 4070, 4068, 4067, 4074, + 4039, 4059, 4027, 4055, 4069, 4065, 4072, 4076, 4052, 4058, 4062, 4073, + 4051, 4057, 4061, 4007, 4043, 4019, 4024, 4008, 4025, 4010, 4050, 4012, + 4020, 4013, 4021, 4026, 4032, 4017, 4038, 4075, 4040, 4029, 4035, 4046, + 4042, 35762, 15039, 15083, 15040, 15045, 15046, 15048, 15075, 15086, + 15061, 15062, 15064, 15066, 15073, 15069, 15065, 15072, 15041, 15051, + 15085, 15052, 15076, 15088, 15033, 15028, 15082, 15027, 15038, 15074, + 15059, 15112, 15023, 15026, 15109, 15110, 15030, 15029, 15096, 15095, + 15113, 15092, 15093, 15114, 15101, 15115, 15091, 15090, 15094, 15105, + 15031, 15111, 15032, 15116, 15084, 15047, 15050, 15049, 15063, 15070, + 15067, 15068, 15071, 15042, 15044, 15043, 15037, 15058, 15056, 15053, + 15054, 15057, 15055, 15035, 15036, 15078, 15079, 15081, 15080, 15077, + 15060, 15089, 15098, 15100, 15099, 15034, 15087, 15097, 15102, 15103, + 15104, 15107, 15106, 15108, 15024, 15025, 35762, 16020, 16023, 16022, + 16018, 16016, 16012, 16019, 16014, 16010, 16013, 16021, 16017, 16011, + 16024, 16025, 16015, 4064, 4053, 4066, 4030, 4023, 4022, 4049, 4045, + 4037, 4014, 4033, 4018, 4036, 4041, 4009, 4011, 4016, 4048, 4044, 4034, + 4005, 4006, 4004, 4003, 4028, 4060, 4054, 4002, 4031, 4056, 4047, 4015, + 7680, 7683, 7684, 7682, 7671, 7656, 7662, 7653, 7661, 7675, 7663, 7659, + 7654, 7660, 7657, 7686, 7652, 7670, 7666, 7678, 7685, 7655, 7665, 7674, + 7673, 7679, 7677, 7667, 7669, 7681, 7676, 7672, 7664, 7658, 7668, 7687, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 16041, 16899, 16911, 16912, 16900, 16910, 16886, 16888, 16890, + 16887, 16889, 16913, 16891, 16893, 16895, 16892, 16894, 26284, 26288, + 26300, 26294, 26286, 26292, 26296, 26302, 26277, 26275, 26282, 26298, + 26290, 26280, 26285, 26289, 26301, 26295, 26287, 26293, 26297, 26303, + 26278, 26276, 26283, 26299, 26291, 26281, 26279, 26341, 26340, 35762, + 26339, 26335, 26333, 26314, 26312, 26332, 26324, 26309, 26318, 26334, + 26317, 26311, 26337, 26336, 26316, 26308, 26331, 26329, 26338, 26326, + 26319, 26327, 26310, 26305, 26315, 26322, 26306, 26328, 26330, 26307, + 26320, 26304, 26313, 26321, 26325, 26323, 6319, 6307, 6329, 6308, 6473, + 6487, 6475, 6457, 6454, 6470, 6468, 6450, 26421, 6488, 6494, 6493, 6490, + 6489, 6492, 6491, 6496, 6495, 6474, 6476, 6482, 6481, 6478, 6477, 6267, + 6271, 6283, 6277, 6269, 6275, 6279, 6260, 6258, 6256, 6265, 6281, 6273, + 6263, 6268, 6272, 6284, 6278, 6270, 6276, 6280, 6261, 6259, 6257, 6266, + 6282, 6274, 6264, 6394, 6393, 6262, 18033, 6342, 6337, 6335, 6304, 6302, + 6334, 6325, 6299, 6317, 6336, 6315, 6301, 6339, 6338, 6313, 6297, 6328, + 6333, 6306, 6330, 6318, 6331, 6300, 6293, 6309, 6324, 6314, 6303, 6327, + 6298, 6341, 6290, 6340, 6321, 6294, 6292, 6305, 6295, 6310, 6311, 6323, + 6312, 6322, 6332, 6326, 6296, 6320, 6288, 6316, 6480, 6479, 6484, 6483, + 6456, 6458, 6464, 6463, 6460, 6459, 6462, 6461, 6466, 6465, 6453, 16111, + 16114, 16115, 16053, 16116, 16112, 16113, 16052, 16118, 16119, 16117, + 16085, 29271, 29240, 29242, 19871, 6388, 6390, 6392, 6389, 6391, 6351, + 6353, 6355, 6352, 6354, 6371, 6373, 6375, 6372, 6374, 6376, 6378, 6380, + 6377, 6379, 6361, 6363, 6365, 6362, 6364, 6346, 6348, 6350, 6347, 6349, + 6356, 6358, 6360, 6357, 6359, 6385, 6387, 6386, 6366, 6368, 6370, 6367, + 6369, 6381, 6383, 6382, 6384, 29238, 29202, 29200, 29199, 29201, 29274, + 29275, 29434, 29241, 29234, 29364, 29368, 29281, 29283, 29282, 29246, + 29247, 29251, 29250, 29284, 29249, 29285, 29288, 29286, 29287, 29252, + 29253, 29295, 29296, 29297, 29294, 29293, 29404, 29405, 29408, 29406, + 29407, 29225, 29229, 29230, 29413, 29357, 29358, 29259, 29371, 29372, + 29206, 29380, 29378, 29379, 29209, 29266, 29265, 29208, 29267, 29260, + 29377, 29375, 29261, 29376, 29374, 29211, 29381, 29210, 29264, 29382, + 29262, 29263, 29321, 29322, 29325, 29324, 29323, 29331, 29332, 29333, + 29328, 29329, 29330, 29432, 29431, 29433, 29399, 29400, 29402, 29401, + 29397, 29396, 29414, 16109, 16110, 16092, 16094, 16101, 16100, 16107, + 16105, 16096, 16103, 16095, 16098, 16091, 16093, 16102, 16099, 16108, + 16106, 16097, 16104, 16086, 16090, 16089, 16088, 16087, 29269, 29224, + 29204, 29207, 29369, 29385, 29226, 29228, 29227, 29279, 29235, 29239, + 29236, 29237, 29216, 29373, 29356, 29338, 29320, 29280, 29302, 29326, + 29256, 29213, 29299, 29386, 29359, 29339, 29340, 29353, 29303, 29272, + 29298, 29350, 29254, 29412, 29341, 29354, 29233, 29305, 29245, 29360, + 29342, 29335, 29217, 29289, 29337, 29219, 29319, 29292, 29336, 29218, + 29318, 29291, 29315, 29316, 29370, 29301, 29352, 29255, 29393, 29394, + 29395, 29390, 29361, 29343, 29355, 29391, 29362, 29344, 29346, 29307, + 29347, 29392, 29363, 29345, 29348, 29308, 29349, 29300, 29317, 29203, + 29212, 29222, 29223, 29220, 29215, 29231, 29257, 29258, 29268, 29273, + 29304, 29290, 29306, 29312, 29313, 29310, 29314, 29327, 29334, 29351, + 29387, 29388, 29384, 29389, 29409, 29410, 29430, 29205, 29198, 16084, + 16069, 16057, 16076, 16075, 16082, 16080, 16071, 16078, 16070, 16073, + 16068, 16056, 16077, 16074, 16083, 16081, 16072, 16079, 16058, 16066, + 16064, 16063, 16060, 16059, 16062, 16061, 16067, 16065, 16054, 16055, + 29248, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 15811, + 15814, 15775, 15825, 15822, 15770, 15808, 15787, 15804, 15821, 15799, + 15805, 15780, 15782, 15792, 15826, 15779, 15823, 15763, 15769, 15767, + 15786, 15806, 15800, 15789, 15785, 15793, 15784, 15809, 15812, 15791, + 15777, 15801, 15783, 15798, 15778, 15813, 15796, 15794, 15773, 15772, + 15790, 15768, 15771, 15781, 15797, 15795, 15815, 15802, 15810, 15807, + 15819, 15774, 15817, 15765, 15816, 15818, 15820, 15776, 15824, 15788, + 15803, 15764, 15766, 34695, 34702, 34694, 34701, 34699, 34700, 34697, + 34698, 35470, 35471, 35468, 35469, 35467, 35465, 35466, 35474, 35475, + 35472, 35473, 35477, 35476, 35342, 34362, 34363, 34356, 34361, 34359, + 34360, 34357, 34358, 34366, 34367, 34364, 34365, 34347, 34345, 34346, + 34370, 34371, 34368, 34369, 34355, 34353, 34354, 34351, 34352, 34344, + 34350, 34349, 34348, 34376, 34377, 34372, 34375, 34374, 34373, 35074, + 35075, 35069, 35073, 35072, 35070, 35071, 35078, 35079, 35076, 35077, + 35063, 35061, 35062, 35082, 35083, 35080, 35081, 35067, 35068, 35060, + 35066, 35065, 35064, 35088, 35089, 35084, 35087, 35086, 35085, 34330, + 34331, 34324, 34329, 34327, 34328, 34325, 34326, 34334, 34335, 34332, + 34333, 34315, 34313, 34314, 34338, 34339, 34336, 34337, 34323, 34321, + 34322, 34319, 34320, 34312, 34318, 34317, 34316, 34342, 34343, 34340, + 34341, 34877, 34878, 34872, 34876, 34875, 34873, 34874, 34881, 34882, + 34879, 34880, 34885, 34886, 34883, 34884, 34891, 34892, 34887, 34890, + 34889, 34888, 34897, 34898, 34893, 34896, 34895, 34894, 34620, 34621, + 34615, 34619, 34618, 34616, 34617, 34624, 34625, 34622, 34623, 34609, + 34607, 34608, 34628, 34629, 34626, 34627, 34613, 34614, 34606, 34612, + 34611, 34610, 34634, 34630, 34633, 34632, 34631, 34856, 34857, 34851, + 34855, 34854, 34852, 34853, 34860, 34861, 34858, 34859, 34844, 34845, + 34842, 34843, 34864, 34865, 34862, 34863, 34871, 34870, 34849, 34850, + 34841, 34848, 34847, 34846, 34868, 34869, 34866, 34867, 34501, 34502, + 34499, 34500, 34497, 34498, 34495, 34496, 34494, 34492, 34493, 34511, + 34512, 34507, 34510, 34509, 34508, 34505, 34506, 34503, 34504, 35320, + 35321, 35314, 35319, 35317, 35318, 35315, 35316, 35324, 35325, 35322, + 35323, 35328, 35329, 35326, 35327, 35313, 35312, 35334, 35335, 35330, + 35333, 35332, 35331, 35340, 35341, 35336, 35339, 35338, 35337, 34479, + 34480, 34474, 34478, 34477, 34475, 34476, 34486, 34487, 34484, 34485, + 34468, 34467, 34490, 34491, 34488, 34489, 34483, 34481, 34482, 34472, + 34473, 34466, 34471, 34470, 34469, 35299, 35300, 35294, 35298, 35297, + 35295, 35296, 35306, 35307, 35304, 35305, 35287, 35288, 35285, 35286, + 35310, 35311, 35308, 35309, 35303, 35301, 35302, 35292, 35293, 35284, + 35291, 35290, 35289, 34453, 34454, 34448, 34452, 34451, 34449, 34450, + 34460, 34461, 34458, 34459, 34442, 34440, 34441, 34464, 34465, 34462, + 34463, 34457, 34455, 34456, 34446, 34447, 34439, 34445, 34444, 34443, + 34903, 34904, 34899, 34902, 34901, 34900, 34910, 34911, 34908, 34909, + 34914, 34915, 34912, 34913, 34907, 34905, 34906, 34920, 34921, 34916, + 34919, 34918, 34917, 34650, 34651, 34644, 34649, 34647, 34648, 34645, + 34646, 34654, 34655, 34652, 34653, 34639, 34638, 34636, 34637, 34635, + 34643, 34641, 34642, 34640, 35048, 35049, 35043, 35047, 35046, 35044, + 35045, 35052, 35050, 35051, 35037, 35035, 35036, 35058, 35059, 35056, + 35057, 35055, 35053, 35054, 35041, 35042, 35034, 35040, 35039, 35038, + 34588, 34589, 34583, 34587, 34586, 34584, 34585, 34598, 34599, 34596, + 34597, 34577, 34575, 34576, 34595, 34593, 34594, 34592, 34590, 34591, + 34581, 34582, 34574, 34580, 34579, 34578, 34604, 34605, 34600, 34603, + 34602, 34601, 34803, 34804, 34797, 34802, 34800, 34801, 34798, 34799, + 34807, 34808, 34805, 34806, 34787, 34788, 34785, 34786, 34811, 34812, + 34809, 34810, 34796, 34794, 34795, 34792, 34793, 34784, 34791, 34790, + 34789, 34817, 34818, 34813, 34816, 34815, 34814, 34557, 34558, 34551, + 34556, 34554, 34555, 34552, 34553, 34561, 34562, 34559, 34560, 34544, + 34545, 34542, 34543, 34569, 34570, 34567, 34568, 34565, 34566, 34563, + 34564, 34549, 34550, 34541, 34548, 34547, 34546, 34770, 34771, 34765, + 34769, 34768, 34766, 34767, 34774, 34775, 34772, 34773, 34759, 34757, + 34758, 34782, 34783, 34780, 34781, 34778, 34779, 34776, 34777, 34763, + 34764, 34756, 34762, 34761, 34760, 34517, 34518, 34513, 34516, 34514, + 34515, 34531, 34532, 34529, 34530, 34522, 34523, 34520, 34521, 34539, + 34540, 34537, 34538, 34535, 34536, 34533, 34534, 34527, 34528, 34519, + 34526, 34525, 34524, 34840, 34839, 34833, 34834, 34831, 34832, 34822, + 34820, 34821, 34837, 34838, 34835, 34836, 34830, 34828, 34829, 34826, + 34827, 34819, 34825, 34824, 34823, 34669, 34670, 34663, 34668, 34666, + 34667, 34664, 34665, 34673, 34674, 34671, 34672, 34658, 34659, 34656, + 34657, 34677, 34678, 34675, 34676, 34662, 34660, 34661, 34930, 34928, + 34929, 34933, 34934, 34931, 34932, 34923, 34924, 34922, 34937, 34938, + 34935, 34936, 34927, 34925, 34926, 34573, 34572, 34571, 34688, 34689, + 34686, 34687, 34681, 34682, 34679, 34680, 34692, 34693, 34690, 34691, + 34685, 34683, 34684, 35354, 35355, 35352, 35353, 35345, 35343, 35344, + 35351, 35349, 35350, 35348, 35346, 35347, 35417, 35418, 35412, 35416, + 35415, 35413, 35414, 35453, 35454, 35451, 35452, 35406, 35404, 35405, + 35457, 35458, 35455, 35456, 35450, 35448, 35449, 35410, 35411, 35403, + 35409, 35408, 35407, 35463, 35464, 35459, 35462, 35461, 35460, 34423, + 34424, 34417, 34422, 34420, 34421, 34418, 34419, 34427, 34428, 34425, + 34426, 34408, 34406, 34407, 34431, 34432, 34429, 34430, 34416, 34414, + 34415, 34412, 34413, 34405, 34411, 34410, 34409, 34437, 34438, 34433, + 34436, 34435, 34434, 35431, 35432, 35425, 35430, 35428, 35429, 35426, + 35427, 35435, 35436, 35433, 35434, 35424, 35422, 35423, 35421, 35419, + 35420, 35441, 35437, 35440, 35439, 35438, 35446, 35447, 35442, 35445, + 35444, 35443, 35020, 35021, 35015, 35019, 35018, 35016, 35017, 35024, + 35025, 35022, 35023, 35008, 35007, 35014, 35013, 35033, 35032, 35012, + 35006, 35011, 35010, 35009, 35030, 35031, 35026, 35029, 35028, 35027, + 35265, 35266, 35260, 35264, 35263, 35261, 35262, 35272, 35273, 35270, + 35271, 35254, 35252, 35253, 35276, 35277, 35274, 35275, 35269, 35267, + 35268, 35258, 35259, 35251, 35257, 35256, 35255, 35282, 35283, 35278, + 35281, 35280, 35279, 35201, 35202, 35196, 35200, 35199, 35197, 35198, + 35208, 35209, 35206, 35207, 35212, 35213, 35210, 35211, 35205, 35203, + 35204, 35216, 35217, 35214, 35215, 35222, 35223, 35218, 35221, 35220, + 35219, 35387, 35388, 35385, 35386, 35379, 35377, 35378, 35395, 35396, + 35393, 35394, 35391, 35392, 35389, 35390, 35383, 35384, 35376, 35382, + 35381, 35380, 35401, 35402, 35397, 35400, 35399, 35398, 34389, 34390, + 34387, 34388, 34381, 34382, 34379, 34380, 34397, 34398, 34395, 34396, + 34393, 34394, 34391, 34392, 34386, 34378, 34385, 34384, 34383, 34403, + 34404, 34399, 34402, 34401, 34400, 35169, 35168, 35148, 35147, 35160, + 35161, 35158, 35159, 35156, 35157, 35154, 35155, 35152, 35153, 35146, + 35151, 35150, 35149, 35166, 35167, 35162, 35165, 35164, 35163, 34969, + 34970, 34967, 34968, 34966, 34964, 34965, 34973, 34974, 34971, 34972, + 34979, 34980, 34975, 34978, 34977, 34976, 34985, 34986, 34981, 34984, + 34983, 34982, 35235, 35236, 35233, 35234, 35227, 35225, 35226, 35243, + 35244, 35241, 35242, 35239, 35240, 35237, 35238, 35231, 35232, 35224, + 35230, 35229, 35228, 35249, 35250, 35245, 35248, 35247, 35246, 35184, + 35185, 35182, 35183, 35173, 35171, 35172, 35188, 35189, 35186, 35187, + 35181, 35179, 35180, 35177, 35178, 35170, 35176, 35175, 35174, 35194, + 35195, 35190, 35193, 35192, 35191, 34744, 34745, 34738, 34743, 34741, + 34742, 34739, 34740, 34731, 34732, 34729, 34730, 34748, 34749, 34746, + 34747, 34736, 34737, 34728, 34735, 34734, 34733, 34754, 34755, 34750, + 34753, 34752, 34751, 35106, 35107, 35100, 35105, 35103, 35104, 35101, + 35102, 35093, 35094, 35091, 35092, 35110, 35111, 35108, 35109, 35098, + 35099, 35090, 35097, 35096, 35095, 35116, 35117, 35112, 35115, 35114, + 35113, 34718, 34719, 34712, 34717, 34715, 34716, 34713, 34714, 34706, + 34704, 34705, 34722, 34723, 34720, 34721, 34710, 34711, 34703, 34709, + 34708, 34707, 34726, 34727, 34724, 34725, 34952, 34953, 34946, 34951, + 34949, 34950, 34947, 34948, 34941, 34940, 34956, 34957, 34954, 34955, + 34945, 34939, 34944, 34943, 34942, 34962, 34963, 34958, 34961, 34960, + 34959, 35000, 35001, 34994, 34999, 34997, 34998, 34995, 34996, 34990, + 34988, 34989, 35004, 35005, 35002, 35003, 34992, 34993, 34987, 34991, + 35362, 35363, 35356, 35361, 35359, 35360, 35357, 35358, 35375, 35374, + 35366, 35367, 35364, 35365, 35372, 35373, 35368, 35371, 35370, 35369, + 35134, 35135, 35128, 35133, 35131, 35132, 35129, 35130, 35121, 35122, + 35119, 35120, 35138, 35139, 35136, 35137, 35126, 35127, 35118, 35125, + 35124, 35123, 35144, 35145, 35140, 35143, 35142, 35141, 35762, 35762, + 35762, 34309, 34282, 34280, 34287, 34261, 34297, 34268, 34270, 34286, + 34273, 34284, 34257, 34285, 34303, 34291, 34272, 34298, 34271, 34304, + 34262, 34265, 34258, 34267, 34288, 34299, 34311, 34276, 34307, 34292, + 34275, 34302, 34300, 34296, 34301, 34308, 34279, 34289, 34278, 34269, + 34277, 34310, 34266, 34293, 34283, 34260, 34259, 34264, 34274, 34294, + 34305, 34295, 34263, 34306, 34290, 34281, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 20474, 20463, 20462, 20446, 20444, + 20443, 20457, 20461, 20460, 20476, 20455, 20454, 20445, 20442, 20441, + 20478, 20453, 20477, 20465, 20468, 20469, 20452, 20459, 20480, 20458, + 20475, 20479, 20464, 20467, 20456, 20470, 20471, 20450, 20451, 20481, + 20472, 20447, 20448, 20449, 20473, 20437, 20440, 20438, 20436, 20439, + 20435, 20482, 20483, 33047, 33048, 32884, 33033, 33034, 33007, 32810, + 32817, 32920, 32893, 32927, 32867, 32993, 33021, 32845, 32838, 32796, + 32789, 32911, 33014, 32803, 32946, 32831, 32824, 32859, 32852, 32986, + 33000, 32965, 33028, 32906, 32952, 32873, 32934, 32979, 32972, 33056, + 33057, 32888, 32889, 33042, 33043, 33009, 32812, 32819, 32922, 32899, + 32929, 32870, 32995, 33023, 32847, 32840, 32798, 32791, 32915, 33016, + 32805, 32948, 32833, 32826, 32861, 32854, 32988, 33002, 32967, 33030, + 32907, 32957, 32878, 32936, 32981, 32974, 33054, 33055, 32959, 32886, + 32887, 33040, 33041, 33008, 32811, 32818, 32921, 32897, 32898, 32928, + 32869, 32994, 33022, 32846, 32839, 32797, 32790, 32914, 33015, 32804, + 32947, 32832, 32825, 32860, 32853, 32987, 33001, 32966, 33029, 32903, + 32904, 32956, 32877, 32935, 32980, 32973, 33051, 33052, 32882, 33037, + 33038, 33005, 32808, 32815, 32918, 32896, 32925, 32865, 32991, 33019, + 32843, 32836, 32794, 32787, 32913, 33012, 32801, 32944, 32829, 32822, + 32857, 32850, 32984, 32998, 32963, 33026, 32902, 32955, 32876, 32932, + 32977, 32970, 33058, 33059, 32890, 32891, 33044, 33045, 33010, 32813, + 32820, 32923, 32900, 32930, 32871, 32996, 33024, 32848, 32841, 32799, + 32792, 32916, 33017, 32806, 32949, 32834, 32827, 32862, 32855, 32989, + 33003, 32968, 33031, 32908, 32958, 32879, 32937, 32982, 32975, 33050, + 33053, 32961, 32880, 32881, 33036, 33039, 33004, 32807, 32814, 32917, + 32895, 32924, 32863, 32864, 32990, 33018, 32842, 32835, 32793, 32786, + 32912, 33011, 32800, 32938, 32828, 32821, 32856, 32849, 32983, 32997, + 32962, 33025, 32901, 32954, 32875, 32931, 32976, 32969, 33046, 33049, + 32960, 32883, 32885, 33032, 33035, 33006, 32809, 32816, 32919, 32892, + 32894, 32926, 32866, 32868, 32992, 33020, 32844, 32837, 32795, 32788, + 32909, 33013, 32802, 32945, 32830, 32823, 32858, 32851, 32985, 32999, + 32964, 33027, 32905, 32951, 32953, 32872, 32874, 32933, 32978, 32971, + 32950, 32910, 32783, 32784, 32785, 32941, 32942, 32939, 33063, 33066, + 33069, 33068, 33072, 33064, 33071, 33062, 33060, 33067, 33070, 33061, + 33065, 33079, 33081, 33078, 33077, 33074, 33073, 33076, 33075, 33082, + 33080, 32943, 32940, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 9978, 10178, 9867, 10026, 9818, 10120, 9922, 10081, + 9863, 10022, 9943, 10106, 9851, 10010, 9811, 10107, 9971, 10171, 9919, + 10078, 9820, 10122, 9920, 10079, 9859, 10018, 9852, 10011, 9916, 10075, + 9973, 10173, 9819, 10121, 9962, 10140, 9959, 10137, 9960, 10138, 9942, + 10105, 9849, 10008, 9864, 10023, 9993, 7842, 7834, 7785, 7835, 28868, + 7809, 7798, 7812, 7808, 7817, 7810, 7807, 7804, 7840, 7829, 9992, 9996, + 9873, 10032, 9871, 10030, 9983, 10183, 9861, 10020, 9869, 10028, 9824, + 10148, 9831, 10156, 9828, 10152, 9827, 10151, 9830, 10155, 9907, 10066, + 9957, 10135, 9865, 10024, 9860, 10019, 23047, 23084, 7793, 7801, 3393, + 2760, 3396, 2762, 3392, 3368, 3385, 3395, 2789, 3394, 2765, 3365, 3371, + 3370, 2763, 2769, 3384, 2788, 2782, 2768, 3380, 2776, 3375, 3381, 3374, + 3378, 2758, 2754, 2786, 2785, 2780, 3387, 3377, 3388, 3389, 2787, 2752, + 3361, 2781, 2784, 3364, 3390, 3362, 2749, 3373, 2767, 2774, 2791, 3367, + 3372, 2753, 2777, 2778, 2779, 3376, 3363, 2751, 2750, 3391, 2790, 2766, + 3366, 2764, 2755, 2771, 3369, 2770, 2773, 3386, 2761, 2775, 2772, 3383, + 2759, 3382, 2783, 3379, 2748, 2756, 2757, 2745, 2744, 3397, 3399, 2747, + 2746, 3398, 3400, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 23044, 23040, 23043, 23039, 23045, 23041, 23046, 23042, 23099, 23114, + 23142, 23121, 23104, 23098, 23113, 23141, 23120, 23103, 23100, 23115, + 23143, 23125, 23105, 23092, 23090, 23091, 23136, 23155, 23152, 23154, + 23153, 23133, 23303, 23304, 18285, 18884, 18286, 18885, 18325, 18946, + 18547, 19306, 18546, 19263, 18227, 18806, 18228, 18807, 18688, 18696, + 18201, 18762, 18202, 18763, 18203, 18764, 18199, 18760, 18200, 18761, + 18204, 18765, 18493, 19188, 18366, 18998, 18363, 18995, 18367, 18999, + 18213, 18783, 18385, 19022, 18416, 19096, 18417, 19098, 18463, 19136, + 18468, 19141, 18466, 19139, 18469, 19142, 18475, 19153, 18474, 19150, + 18490, 19178, 18495, 19191, 18588, 19356, 18597, 19368, 18592, 19363, + 18541, 19257, 18542, 19258, 18596, 19367, 18287, 18902, 18354, 18985, + 18229, 18808, 23312, 18844, 19042, 19056, 19079, 19190, 18673, 19301, + 19353, 18347, 18974, 18348, 18975, 18349, 18531, 19269, 18536, 19299, + 18350, 18977, 18351, 18978, 18352, 18979, 23119, 23088, 23165, 18514, + 19213, 18534, 19027, 18704, 18409, 19070, 18223, 18796, 18793, 18940, + 18207, 18768, 18294, 18909, 18593, 19364, 18594, 19365, 18595, 19366, + 18302, 18920, 18370, 19003, 18412, 19075, 18488, 19175, 18510, 19211, + 18321, 18492, 18517, 18373, 18513, 18695, 18535, 18538, 18357, 18230, + 18214, 18785, 18461, 19134, 18584, 19344, 18307, 18925, 18308, 18926, + 18309, 18927, 18458, 19125, 18196, 18757, 18221, 18511, 18634, 18234, + 18810, 18507, 19205, 35762, 35762, 35762, 35762, 35762, 18226, 18802, + 35762, 18837, 35762, 18834, 18398, 19055, 18516, 19231, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 23020, 23023, 23032, 18491, 19179, 18644, 23015, 23244, + 18678, 18642, 18643, 18640, 18641, 18639, 29798, 29800, 29810, 29802, + 29799, 29801, 29809, 29790, 29789, 29786, 29785, 29808, 29784, 29783, + 29788, 29787, 29778, 29777, 29772, 29771, 29780, 29779, 29774, 29773, + 29796, 29792, 29791, 29782, 29781, 29795, 29776, 29794, 29775, 29797, + 29793, 29812, 29814, 29815, 29813, 29811, 29803, 29804, 29805, 29806, + 29807, 35762, 35762, 35762, 24678, 24677, 24680, 24675, 24676, 24679, + 24672, 24671, 24674, 24673, 35762, 35762, 35762, 35762, 35762, 35762, + 26558, 26557, 26546, 26547, 26534, 26536, 26568, 26549, 26556, 26555, + 26539, 26550, 26560, 26559, 26565, 26570, 26552, 26551, 26538, 26573, + 26561, 26562, 26540, 26575, 26572, 26569, 26541, 26542, 26567, 26531, + 26576, 26578, 26563, 26577, 26571, 26574, 26566, 26545, 26564, 26583, + 26584, 26554, 26553, 26537, 26548, 26532, 26544, 26543, 26533, 26582, + 26585, 26535, 26581, 26586, 26580, 26579, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 27667, 27669, 27616, 27617, 27637, 27638, + 27633, 27634, 27628, 27629, 27630, 27631, 27660, 27661, 27618, 27635, + 27636, 27619, 27657, 27656, 27653, 27652, 27641, 27651, 27650, 27655, + 27654, 27643, 27625, 27624, 27621, 27620, 27642, 27627, 27626, 27623, + 27622, 27644, 27659, 27658, 27649, 27648, 27663, 27665, 27664, 27640, + 27632, 27645, 27646, 27647, 27662, 27639, 27682, 27683, 27694, 27695, + 27690, 27691, 27686, 27687, 27688, 27689, 27696, 27697, 27684, 27692, + 27693, 27685, 27668, 27666, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 27671, 27670, 27678, 27680, 27677, 27676, 27673, 27672, + 27675, 27674, 27681, 27679, 35762, 35762, 35762, 35762, 35762, 35762, + 7860, 7862, 7859, 7858, 7855, 7854, 7857, 7856, 7863, 7861, 7851, 7852, + 7847, 7848, 7849, 7850, 7846, 7853, 10463, 10456, 10465, 10458, 10457, + 10455, 10462, 10359, 10357, 10362, 10464, 10513, 10368, 10480, 17118, + 17120, 17117, 17116, 17113, 17112, 17115, 17114, 17121, 17119, 17082, + 17081, 17092, 17078, 17086, 17085, 17099, 17079, 17088, 17074, 17080, + 17084, 17083, 17094, 17091, 17089, 17095, 17098, 17093, 17097, 17087, + 17075, 17096, 17090, 17100, 17076, 17101, 17077, 17110, 17107, 17109, + 17108, 17111, 17106, 17104, 17105, 17102, 17103, 27041, 27038, 27032, + 27046, 27037, 27034, 27043, 27035, 27028, 27036, 27040, 27030, 27045, + 27044, 27042, 27048, 27047, 27039, 27027, 27031, 27033, 27029, 27049, + 27056, 27058, 27051, 27054, 27057, 27055, 27053, 27052, 27024, 27023, + 27026, 27025, 27059, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 27050, 14774, 14777, 14778, 14775, 14732, + 14733, 14741, 14735, 14737, 14740, 14734, 14730, 14736, 14738, 14731, + 14697, 14699, 14700, 14713, 14720, 14727, 14762, 14679, 14686, 14760, + 14764, 14710, 14783, 14766, 35762, 35762, 35762, 16437, 16435, 16439, + 16438, 16409, 16377, 16376, 16378, 16418, 16397, 16383, 16384, 16416, + 16410, 16417, 16379, 16380, 16381, 16393, 16394, 16382, 16391, 16392, + 16407, 16386, 16408, 16385, 16405, 16406, 16372, 16373, 16388, 16403, + 16404, 16374, 16375, 16387, 16395, 16396, 16389, 16390, 16413, 16415, + 16398, 16399, 16412, 16414, 16401, 16402, 16400, 16411, 16436, 16442, + 16444, 16445, 16446, 16440, 16441, 16443, 16448, 16447, 16368, 16369, + 16370, 16433, 16371, 16419, 16422, 16431, 16424, 16429, 16427, 16425, + 16423, 16420, 16421, 16426, 16434, 35762, 16432, 16455, 16457, 16454, + 16453, 16450, 16449, 16452, 16451, 16458, 16456, 35762, 35762, 35762, + 35762, 16428, 16430, 23922, 23920, 23915, 23912, 23918, 23998, 23986, + 23938, 23950, 23946, 23945, 23948, 23947, 23940, 23939, 23937, 24040, + 24042, 24039, 24038, 24035, 24034, 24037, 24036, 24043, 24041, 23949, + 23942, 23941, 23944, 23943, 35762, 5944, 5962, 5964, 5961, 5945, 5963, + 5953, 5952, 5949, 5948, 5924, 5925, 5947, 5946, 5951, 5950, 5926, 5928, + 5927, 5955, 5954, 5941, 5940, 5929, 5930, 5939, 5935, 5934, 5933, 5938, + 5937, 5931, 5932, 5936, 5960, 5958, 5957, 5959, 5942, 5943, 5956, 5969, + 5972, 5973, 5978, 5976, 5975, 5974, 5970, 5971, 5977, 5912, 5910, 5909, + 5911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 5918, 5917, 5914, 5906, 5916, 5922, 5913, 5920, 5923, 5921, 5919, 5915, + 5908, 5907, 35762, 35762, 5985, 5987, 5984, 5983, 5980, 5979, 5982, 5981, + 5988, 5986, 35762, 35762, 5965, 5967, 5966, 5968, 23892, 23885, 23884, + 23889, 23888, 23882, 23881, 23880, 23878, 23877, 23879, 23883, 23894, + 23887, 23886, 23891, 23985, 23895, 23896, 23893, 23982, 23983, 23984, + 24013, 24015, 24014, 23867, 24009, 24000, 24001, 23931, 23932, 30279, + 30255, 30278, 30254, 30277, 30253, 30292, 30268, 30286, 30262, 30281, + 30257, 30280, 30256, 30297, 30273, 30287, 30263, 30290, 30266, 30285, + 30261, 30284, 30260, 30288, 30264, 30289, 30265, 30283, 30259, 30282, + 30258, 30291, 30267, 30295, 30271, 30299, 30275, 30296, 30272, 30294, + 30270, 30298, 30274, 30293, 30269, 30300, 30276, 30301, 30313, 30321, + 30318, 30317, 30323, 30324, 30302, 30322, 30319, 30320, 30312, 30316, + 30315, 30314, 30311, 30308, 30309, 30310, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30304, 30305, 30307, 30306, 30303, 22359, 22360, 22318, 22335, 22346, + 22345, 22322, 22321, 22334, 22340, 22341, 22373, 22375, 22365, 22367, + 22366, 22313, 22310, 22312, 22362, 22363, 22377, 22378, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12895, 12893, + 12892, 12891, 12890, 12894, 35762, 35762, 12589, 12587, 12586, 12585, + 12584, 12588, 35762, 35762, 12604, 12602, 12601, 12600, 12599, 12603, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12545, + 12551, 12549, 12546, 12548, 12547, 12550, 35762, 12530, 12536, 12534, + 12531, 12533, 12532, 12535, 35762, 18776, 18752, 18782, 18777, 18873, + 19035, 19223, 19025, 19016, 19019, 19046, 19060, 18887, 18780, 18781, + 19130, 18981, 19272, 19271, 19273, 19274, 19233, 18672, 19177, 18835, + 19157, 18836, 19220, 19221, 18779, 19343, 19309, 19352, 19295, 19348, + 18799, 18800, 18801, 19384, 19381, 19382, 19383, 19395, 23002, 23225, + 23234, 23235, 23292, 19214, 18984, 19131, 19354, 18980, 14005, 18842, + 19304, 19277, 23289, 23138, 23162, 35762, 35762, 35762, 35762, 6167, + 6168, 6169, 6170, 6171, 6172, 6130, 6166, 6131, 6132, 6133, 6134, 6135, + 6095, 6096, 6097, 6098, 6099, 6100, 6136, 6137, 6138, 6139, 6140, 6141, + 6142, 6143, 6144, 6145, 6146, 6101, 6094, 6102, 6103, 6104, 6105, 6106, + 6107, 6148, 6149, 6150, 6151, 6152, 6153, 6109, 6108, 6110, 6111, 6112, + 6113, 6114, 6088, 6127, 6089, 6128, 6090, 6129, 6091, 6092, 6093, 6087, + 6115, 6116, 6117, 6118, 6119, 6120, 6121, 6122, 6123, 6124, 6125, 6126, + 6154, 6155, 6156, 6157, 6158, 6159, 6160, 22327, 22339, 22349, 22351, + 22336, 22330, 22317, 22342, 22329, 22332, 22344, 22355, 22357, 22353, + 22358, 22347, 22338, 22356, 22324, 22325, 22354, 22316, 22326, 22320, + 22323, 22319, 22315, 22328, 22350, 22352, 22337, 22331, 22343, 22333, + 22348, 22369, 22372, 22364, 22371, 22370, 22374, 22368, 22376, 22314, + 22361, 22311, 35762, 35762, 22385, 22387, 22384, 22383, 22380, 22379, + 22382, 22381, 22388, 22386, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 14971, 14969, 15002, 15003, + 15004, 14980, 14981, 15013, 15015, 14948, 14946, 14945, 14949, 14955, + 14956, 14958, 14957, 14962, 14959, 14960, 14964, 14932, 14930, 35762, + 35762, 35762, 35762, 14828, 14829, 14897, 14898, 14917, 14911, 14912, + 14915, 14914, 14913, 14872, 14857, 14896, 14863, 14867, 14866, 14880, + 14879, 14800, 14825, 14818, 14903, 14816, 14823, 14853, 14846, 14852, + 14907, 14848, 14851, 14850, 14891, 14884, 14901, 14902, 14890, 14888, + 14887, 14892, 14894, 14839, 14838, 14924, 14925, 14789, 14788, 14904, + 14843, 14841, 35762, 35762, 35762, 35762, 7281, 7282, 7283, 7284, 7285, + 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, 7296, 7297, + 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, 7308, 7309, + 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, + 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, + 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, 7345, + 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, 7356, 7357, + 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 7366, 7367, 7368, 7369, + 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, + 7382, 7383, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, + 7394, 7395, 7396, 7397, 7398, 7399, 7400, 7401, 7402, 7403, 7404, 7405, + 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413, 7414, 7415, 7416, 7417, + 7418, 7419, 7420, 7421, 7422, 7423, 7424, 7425, 7426, 7427, 7428, 7429, + 7430, 7431, 7432, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, + 7442, 7443, 7444, 7445, 7446, 7447, 7448, 7449, 7450, 7451, 7452, 7453, + 7454, 7455, 7456, 7457, 7458, 7459, 7460, 7461, 7462, 7463, 7464, 7465, + 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7474, 7475, 7476, 7477, + 7478, 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, + 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, + 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, + 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, + 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7079, + 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, + 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, + 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, + 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, + 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, + 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, + 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, + 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7065, + 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, + 7078, 35762, 35762, 7175, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, + 7184, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, + 7196, 7197, 7198, 7199, 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, + 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, + 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, + 7232, 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, + 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, + 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, + 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, + 7280, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19417, + 19420, 19421, 19418, 19419, 19423, 19424, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2490, 2491, 2489, + 2492, 2488, 35762, 35762, 35762, 35762, 35762, 15435, 15453, 15440, + 15371, 15427, 15430, 15432, 15431, 15426, 15433, 15429, 15428, 15372, + 15405, 15406, 15403, 15404, 15369, 15370, 15368, 15376, 15418, 15416, + 15393, 15425, 15398, 35762, 15410, 15436, 15384, 15379, 15420, 35762, + 15422, 35762, 15396, 15400, 35762, 15386, 15382, 35762, 15414, 15391, + 15408, 15402, 15412, 15424, 15375, 15378, 15381, 15437, 1146, 1145, 1176, + 1174, 1175, 1177, 1403, 1401, 1402, 1404, 1171, 1169, 1170, 1172, 1532, + 1530, 1531, 1533, 1512, 1510, 1511, 1513, 1527, 1525, 1526, 1528, 1545, + 1543, 1544, 1546, 1408, 1406, 1407, 1409, 1211, 1209, 1210, 1212, 1381, + 1379, 1380, 1382, 1489, 1487, 1488, 1490, 1494, 1492, 1493, 1495, 1201, + 1200, 1192, 1191, 1215, 1214, 1204, 1203, 1311, 1310, 1438, 1437, 1333, + 1331, 1332, 1334, 1245, 1243, 1244, 1246, 1257, 1255, 1256, 1258, 1372, + 1370, 1371, 1373, 1386, 1385, 1442, 1440, 1441, 1443, 1287, 1286, 1282, + 1280, 1281, 1283, 1291, 1289, 1290, 1292, 1569, 1568, 1567, 1566, 2352, + 2351, 2360, 2359, 2356, 2355, 2354, 2353, 2364, 2363, 2350, 2358, 2357, + 2366, 2362, 2361, 2365, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1376, + 1374, 1375, 1377, 1538, 1537, 1590, 1589, 1587, 1586, 1536, 1548, 1547, + 1337, 1336, 1340, 1339, 1603, 1601, 1602, 1604, 1539, 1540, 2043, 2042, + 2045, 2044, 2064, 2063, 2072, 2071, 2062, 2061, 2068, 2067, 2048, 2046, + 2047, 2097, 2095, 2096, 1225, 1223, 1224, 1226, 2054, 2052, 2058, 2041, + 2066, 1645, 1636, 1641, 1648, 1643, 1654, 2006, 1994, 2001, 2014, 2017, + 2022, 2024, 2029, 2026, 2035, 1723, 1729, 1708, 1703, 1762, 1758, 1764, + 1913, 1906, 1918, 1926, 1885, 1889, 1667, 1659, 1663, 1669, 1987, 1982, + 2099, 1616, 1612, 1697, 1693, 1681, 1686, 1677, 1684, 1679, 1688, 1867, + 1863, 1865, 1869, 1738, 1740, 1748, 1746, 1743, 1754, 1736, 1757, 1790, + 1782, 1794, 1800, 1774, 1803, 1821, 1810, 1815, 1825, 1804, 1826, 1842, + 1832, 1854, 1847, 1850, 1857, 1718, 1714, 1715, 1716, 2082, 2075, 2084, + 2090, 2039, 2094, 2023, 1880, 1628, 1934, 1937, 1941, 1935, 1938, 1940, + 2070, 2069, 2056, 2060, 2040, 2065, 1652, 1651, 1646, 1650, 1642, 1653, + 2020, 2019, 2012, 2018, 2016, 2021, 2033, 2032, 2027, 2031, 2025, 2034, + 1678, 1687, 1864, 1868, 1737, 1741, 1752, 1735, 1756, 1798, 1773, 1802, + 1805, 1823, 1855, 1852, 1845, 1851, 1849, 1856, 1627, 2092, 2079, 2088, + 2078, 2038, 2093, 2053, 2051, 2055, 2057, 2049, 1644, 1635, 1640, 1647, + 1637, 2005, 1993, 2000, 2013, 1995, 2028, 1722, 1728, 1707, 1702, 1761, + 1763, 1912, 1905, 1917, 1925, 1884, 1892, 1888, 1666, 1658, 1662, 1668, + 1986, 2098, 1615, 1611, 1696, 1692, 1680, 1685, 1676, 1683, 1866, 1862, + 1739, 1747, 1745, 1742, 1753, 1789, 1781, 1793, 1799, 1783, 1820, 1809, + 1814, 1824, 1841, 1831, 1853, 1846, 1833, 1717, 1713, 1719, 2081, 2074, + 2083, 2089, 2076, 2059, 2050, 1649, 1638, 2015, 1996, 2030, 2036, 1927, + 1909, 1964, 1944, 1744, 1755, 1801, 1848, 1834, 2091, 2077, 1942, 1936, + 1939, 1985, 1989, 1618, 1620, 1695, 1699, 1929, 1933, 1966, 1974, 1705, + 1710, 1731, 1733, 1760, 1766, 1891, 1896, 1665, 1673, 1955, 1950, 1969, + 1963, 1972, 1931, 1894, 1671, 1984, 1988, 1617, 1619, 1694, 1698, 1928, + 1932, 1965, 1973, 1704, 1709, 1730, 1732, 1759, 1765, 1890, 1895, 1664, + 1672, 1953, 1948, 1967, 1961, 1971, 1930, 1893, 1670, 1954, 1949, 1968, + 1962, 1908, 1943, 1981, 1914, 1907, 1919, 1956, 1951, 1970, 1983, 2100, + 1629, 1630, 25893, 25894, 1877, 1872, 1876, 1873, 1874, 1875, 1903, 1622, + 1624, 1623, 1621, 1871, 1902, 1625, 1976, 1878, 2003, 1992, 1991, 1990, + 1998, 2008, 2010, 2009, 1725, 1724, 1701, 1700, 1904, 1911, 1910, 1921, + 1920, 1922, 1924, 1923, 1882, 1881, 1887, 1946, 1945, 1952, 1958, 1957, + 1960, 1959, 1656, 1661, 1660, 1978, 1977, 1979, 1980, 1614, 1609, 1608, + 1607, 1689, 1691, 1690, 1675, 1674, 1860, 1858, 1779, 1780, 1777, 1784, + 1785, 1792, 1791, 1796, 1795, 1806, 1807, 1808, 1818, 1816, 1811, 1812, + 35762, 35762, 1817, 1711, 1712, 1829, 1828, 1839, 1838, 1837, 1844, 1843, + 2086, 2085, 1639, 2004, 2002, 1999, 1997, 2011, 2007, 1727, 1720, 1726, + 1915, 1883, 1947, 1657, 1788, 1797, 2073, 2080, 2087, 1822, 1861, 1830, + 1859, 1778, 1610, 1751, 1835, 1813, 1786, 1750, 1787, 1836, 1721, 1706, + 1819, 1682, 1634, 1749, 1613, 1886, 1916, 1840, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 1898, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1899, 1870, 1631, 1632, + 1827, 1897, 1879, 1626, 2037, 1900, 1901, 1734, 27339, 1655, 1975, 1633, + 33083, 33194, 33262, 33273, 33284, 33295, 33306, 33317, 33328, 33084, + 33095, 33106, 33117, 33128, 33139, 33150, 26827, 26832, 26833, 26826, + 26857, 26828, 26859, 26835, 26849, 26831, 35762, 35762, 35762, 35762, + 35762, 35762, 8057, 8059, 7889, 7890, 8068, 8070, 7783, 8058, 8060, 8126, + 8127, 8069, 8071, 7784, 7837, 7838, 26858, 26829, 26830, 26844, 26856, + 26841, 26853, 26839, 26851, 26843, 26855, 26836, 26845, 26837, 26846, + 26840, 26852, 26838, 26850, 26834, 26847, 27753, 34075, 26842, 26854, + 10235, 5828, 33955, 10947, 10234, 5827, 33953, 28892, 28901, 28935, + 35762, 28926, 28893, 28936, 28899, 28900, 28903, 28907, 28902, 28906, + 28904, 28908, 28934, 28883, 28885, 28933, 28931, 28905, 28930, 28898, + 35762, 28925, 28894, 28932, 28891, 35762, 35762, 35762, 35762, 1080, + 2371, 1062, 2373, 1092, 35762, 1077, 1078, 1058, 1059, 1089, 1090, 2278, + 2279, 2346, 2347, 1276, 1141, 1140, 1131, 1130, 1556, 1555, 1134, 1133, + 1573, 1571, 1572, 1574, 1151, 1150, 1166, 1164, 1165, 1167, 1499, 1498, + 1508, 1506, 1507, 1501, 1520, 1518, 1519, 1521, 1307, 1305, 1306, 1308, + 1273, 1271, 1272, 1274, 1344, 1342, 1343, 1345, 1189, 1188, 1516, 1515, + 1435, 1434, 1598, 1597, 1464, 1462, 1463, 1465, 1470, 1468, 1469, 1471, + 1451, 1449, 1450, 1452, 1197, 1195, 1196, 1198, 1482, 1480, 1481, 1483, + 1594, 1592, 1593, 1595, 1109, 1107, 1108, 1110, 1252, 1250, 1251, 1253, + 1236, 1234, 1235, 1237, 1417, 1415, 1416, 1418, 1320, 1318, 1319, 1321, + 1356, 1354, 1355, 1357, 1365, 1363, 1364, 1366, 1396, 1394, 1395, 1397, + 1295, 1293, 1294, 1296, 1560, 1559, 1149, 1148, 1583, 1581, 1582, 1584, + 1772, 1771, 1768, 1767, 1770, 1769, 1776, 1775, 35762, 35762, 35566, + 35762, 13299, 13303, 13271, 13287, 13273, 13285, 13284, 13214, 13277, + 13286, 13274, 13209, 13302, 13308, 13281, 13294, 13296, 13293, 13292, + 13289, 13288, 13291, 13290, 13297, 13295, 13210, 13280, 13216, 13298, + 13301, 13304, 13208, 13217, 13218, 13219, 13220, 13221, 13222, 13223, + 13224, 13225, 13226, 13227, 13228, 13229, 13230, 13231, 13232, 13233, + 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13215, + 13279, 13278, 13207, 13269, 13300, 13243, 13244, 13245, 13246, 13247, + 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, + 13258, 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, 13267, + 13268, 13212, 13309, 13275, 13306, 13213, 13276, 14644, 14637, 14639, + 14643, 14635, 14627, 14582, 14584, 14586, 14583, 14585, 14578, 14580, + 14579, 14581, 14636, 14628, 14630, 14632, 14629, 14631, 14603, 14605, + 14607, 14604, 14606, 14587, 14589, 14591, 14588, 14590, 14618, 14620, + 14622, 14619, 14621, 14593, 14595, 14597, 14594, 14596, 14598, 14600, + 14602, 14599, 14601, 14608, 14610, 14612, 14609, 14611, 14623, 14625, + 14624, 14613, 14615, 14617, 14614, 14616, 14626, 14592, 14634, 14633, + 14577, 14527, 14544, 14528, 14529, 14530, 14531, 14565, 14546, 14535, + 14540, 14539, 14538, 14542, 14536, 14537, 14541, 14563, 14533, 14545, + 14534, 14548, 14547, 14562, 14557, 14543, 14556, 14526, 14564, 14532, + 14571, 35762, 35762, 35762, 14572, 14573, 14551, 14552, 14559, 14558, + 35762, 35762, 14550, 14549, 14574, 14568, 14569, 14575, 35762, 35762, + 14554, 14576, 14567, 14566, 14570, 14555, 35762, 35762, 14560, 14553, + 14561, 35762, 35762, 35762, 13211, 13272, 13270, 13283, 13307, 13282, + 13305, 35762, 14642, 14638, 14641, 14640, 14647, 14645, 14646, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16324, + 16326, 16325, 25221, 27062, 35762, 35762, 20348, 20374, 20367, 20400, + 20358, 20349, 20378, 20344, 20356, 20386, 20389, 20383, 35762, 20372, + 20399, 20403, 20382, 20397, 20404, 20411, 20414, 20359, 20410, 20357, + 20360, 20343, 20366, 20369, 20392, 20393, 20351, 20408, 20373, 20354, + 20390, 20352, 20409, 20368, 20376, 35762, 20401, 20365, 20385, 20350, + 20361, 20375, 20346, 20380, 20355, 20387, 20388, 20345, 20371, 20347, + 20398, 20391, 20405, 20379, 20381, 35762, 20353, 20407, 35762, 20364, + 20363, 20377, 20413, 20406, 20416, 20384, 20362, 20394, 20402, 20370, + 20395, 20396, 20412, 20415, 35762, 35762, 20426, 20427, 20429, 20428, + 20417, 20418, 20425, 20419, 20420, 20430, 20421, 20422, 20423, 20424, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 20232, 20231, 20233, 20220, 20221, 20222, + 20223, 20224, 20225, 20226, 20228, 20227, 20230, 20229, 20238, 20235, + 20236, 20234, 20237, 20337, 20338, 20240, 20239, 20241, 20340, 20339, + 20243, 20244, 20245, 20242, 20246, 20249, 20248, 20250, 20251, 20252, + 20341, 20253, 20254, 20247, 20257, 20258, 20256, 20255, 20259, 20260, + 20261, 20262, 20263, 20264, 20267, 20268, 20269, 20266, 20270, 20265, + 20271, 20272, 20273, 20274, 20275, 20276, 20277, 20278, 20279, 20280, + 20282, 20281, 20283, 20284, 20286, 20287, 20288, 20285, 20289, 20290, + 20291, 20292, 20294, 20293, 20295, 20296, 20342, 20297, 20298, 20300, + 20301, 20302, 20299, 20303, 20304, 20305, 20306, 20307, 20335, 20315, + 20316, 20317, 20318, 20319, 20320, 20321, 20322, 20323, 20324, 20325, + 20326, 20327, 20328, 20329, 20330, 20331, 20332, 20333, 20334, 20308, + 20309, 20310, 20311, 20312, 20313, 20314, 20336, 35762, 35762, 35762, + 35762, 35762, 112, 113, 159, 35762, 35762, 35762, 35762, 156, 151, 146, + 126, 121, 139, 134, 114, 129, 154, 149, 144, 124, 119, 140, 135, 115, + 130, 157, 152, 147, 127, 122, 142, 137, 117, 132, 158, 153, 148, 128, + 123, 143, 138, 118, 133, 155, 150, 145, 125, 120, 141, 136, 116, 131, + 35762, 35762, 35762, 111, 107, 109, 110, 108, 103, 104, 105, 106, 13770, + 13767, 13771, 13757, 13752, 13758, 13761, 13753, 13763, 13772, 13755, + 13765, 13759, 13768, 13762, 13764, 13774, 13756, 13766, 13760, 13769, + 13773, 13754, 13775, 13787, 13796, 13786, 13782, 13795, 13777, 13783, + 13799, 13803, 13804, 13785, 13788, 13794, 13793, 13801, 13802, 13784, + 13791, 13797, 13792, 13781, 13800, 13789, 13776, 13778, 13798, 13790, + 13779, 13780, 14022, 14023, 14221, 14217, 14258, 14223, 13953, 14027, + 14220, 14216, 13959, 13958, 14019, 14001, 14017, 14026, 14021, 13807, + 13806, 14260, 14219, 14261, 14024, 14215, 13997, 24651, 35762, 27384, + 27387, 27382, 27385, 27356, 27386, 27354, 27357, 27383, 27355, 27388, + 27353, 2510, 35762, 35762, 35762, 14214, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 26611, 26612, 26623, 26592, 26595, 26626, 26604, + 26603, 26624, 26631, 26590, 26617, 26596, 26609, 26610, 26619, 26608, + 26589, 26593, 26600, 26598, 26620, 26597, 26588, 26618, 26605, 26606, + 26591, 26594, 26616, 26630, 26601, 26625, 26587, 26613, 26632, 26614, + 26615, 26607, 26629, 26628, 26602, 26621, 26622, 26627, 26599, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 20605, + 20607, 20600, 20601, 20612, 20611, 20614, 20622, 20624, 20603, 20615, + 20598, 20618, 20616, 20596, 20609, 20597, 20610, 20621, 20617, 20599, + 20619, 20620, 20602, 20604, 20606, 20608, 20613, 20623, 35762, 35762, + 35762, 5738, 5749, 5740, 5711, 5734, 5750, 5712, 5739, 5756, 5754, 5714, + 5755, 5741, 5729, 5724, 5725, 5723, 5709, 5732, 5722, 5757, 5719, 5731, + 5748, 5728, 5752, 5742, 5737, 5746, 5747, 5720, 5733, 5744, 5745, 5726, + 5727, 5721, 5753, 5710, 5730, 5735, 5751, 5715, 5716, 5717, 5718, 5713, + 5743, 5736, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8241, 8267, 8265, 8264, + 8261, 8260, 8263, 8262, 8268, 8266, 8259, 8258, 8256, 8247, 8245, 8254, + 8252, 8243, 8249, 8250, 8257, 8255, 8246, 8244, 8253, 8251, 8242, 8248, + 35762, 35762, 35762, 35762, 25459, 25453, 25439, 25454, 25426, 25456, + 25458, 25455, 25450, 25446, 25438, 25434, 25435, 25436, 25428, 25460, + 25449, 25442, 25440, 25430, 25427, 25451, 25444, 25432, 25448, 25437, + 25433, 25431, 25452, 25447, 25445, 25429, 25464, 25462, 25463, 25461, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25457, + 25443, 25441, 13617, 13633, 13641, 13635, 13616, 13626, 13620, 13619, + 13628, 13638, 13642, 13639, 13636, 13624, 13640, 13631, 13625, 13623, + 13627, 13637, 13629, 13630, 13632, 13621, 13618, 13634, 13622, 35762, + 35762, 35762, 35762, 35762, 25531, 25530, 25527, 25500, 25501, 25523, + 25498, 25524, 25499, 25503, 25532, 25525, 25506, 25507, 25514, 25508, + 25526, 25511, 25513, 25534, 25497, 25510, 25509, 25521, 25518, 25528, + 25529, 25502, 25533, 25512, 25515, 25516, 25517, 25520, 25505, 25522, + 25519, 25504, 8083, 8081, 8079, 8080, 8082, 35762, 35762, 35762, 35762, + 35762, 32585, 32588, 32592, 32596, 32589, 32593, 32611, 32607, 32594, + 32604, 32606, 32595, 32601, 32597, 32612, 32590, 32609, 32608, 32600, + 32586, 32610, 32599, 32587, 32598, 32603, 32591, 32605, 32613, 32614, + 32602, 35762, 32615, 25540, 25582, 25583, 25563, 25564, 25561, 25562, + 25560, 25575, 25552, 25553, 25557, 25558, 25547, 25550, 25551, 25556, + 25579, 25544, 25576, 25565, 25566, 25569, 25570, 25571, 25580, 25554, + 25555, 25567, 25568, 25578, 25574, 25581, 25572, 25573, 25577, 35762, + 35762, 35762, 35762, 25541, 25542, 25543, 25559, 25548, 25549, 25545, + 25546, 25584, 25539, 25536, 25537, 25535, 25538, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10286, + 10285, 10281, 10282, 10283, 10284, 10292, 10291, 10287, 10288, 10289, + 10290, 10309, 10294, 10308, 10305, 10310, 10303, 10300, 10296, 10301, + 10299, 10302, 10307, 10306, 10276, 10304, 10275, 10295, 10271, 10298, + 10272, 10297, 10279, 10277, 10278, 10273, 10274, 10293, 10280, 10326, + 10325, 10321, 10322, 10323, 10324, 10332, 10331, 10327, 10328, 10329, + 10330, 10349, 10334, 10348, 10345, 10350, 10343, 10340, 10336, 10341, + 10339, 10342, 10347, 10346, 10316, 10344, 10315, 10335, 10311, 10338, + 10312, 10337, 10319, 10317, 10318, 10313, 10314, 10333, 10320, 27890, + 27896, 27907, 27904, 27894, 27893, 27892, 27871, 27899, 27877, 27906, + 27902, 27905, 27908, 27895, 27903, 27882, 27901, 27898, 27876, 27881, + 27883, 27880, 27875, 27869, 27866, 27888, 27897, 27886, 27870, 27891, + 27909, 27873, 27867, 27879, 27910, 27887, 27884, 27885, 27868, 27864, + 27889, 27865, 27874, 27863, 27872, 27878, 27900, 25971, 25989, 25995, + 25993, 25996, 25977, 25974, 25994, 25979, 25978, 25975, 25973, 25991, + 25990, 25981, 25976, 25984, 25980, 25985, 25986, 25992, 25997, 25970, + 25987, 25998, 25982, 25999, 25972, 25988, 25983, 35762, 35762, 26006, + 26008, 26005, 26004, 26001, 26000, 26003, 26002, 26009, 26007, 35762, + 35762, 35762, 35762, 35762, 35762, 25898, 25899, 25900, 25901, 25926, + 25919, 25905, 25902, 25908, 25918, 25917, 25932, 25911, 25906, 25910, + 25927, 25928, 25929, 25912, 25913, 25930, 25907, 25923, 25922, 25916, + 25904, 25915, 25903, 25914, 25920, 25933, 25931, 25909, 25921, 25925, + 25924, 35762, 35762, 35762, 35762, 25934, 25935, 25936, 25937, 25962, + 25955, 25941, 25938, 25944, 25954, 25953, 25968, 25947, 25942, 25946, + 25963, 25964, 25965, 25948, 25949, 25966, 25943, 25959, 25958, 25952, + 25940, 25951, 25939, 25950, 25956, 25969, 25967, 25945, 25957, 25961, + 25960, 35762, 35762, 35762, 35762, 12376, 12367, 12356, 12355, 12358, + 12347, 12357, 12354, 12353, 12368, 12344, 12343, 12369, 12377, 12370, + 12360, 12346, 12345, 12371, 12350, 12349, 12348, 12378, 12372, 12373, + 12352, 12351, 12362, 12361, 12364, 12363, 12379, 12374, 12375, 12380, + 12366, 12365, 12342, 12341, 12359, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 5770, 5819, 5786, 5782, 5784, 5809, 5783, 5807, + 5804, 5773, 5806, 5808, 5787, 5798, 5794, 5789, 5817, 5781, 5772, 5790, + 5793, 5795, 5820, 5811, 5769, 5775, 5776, 5778, 5812, 5810, 5815, 5779, + 5799, 5791, 5818, 5803, 5814, 5780, 5774, 5797, 5785, 5816, 5801, 5813, + 5802, 5800, 5788, 5777, 5771, 5805, 5796, 5792, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 5821, 33727, + 33696, 33697, 33707, 33706, 33709, 33708, 33699, 33698, 33716, 33724, + 35762, 33715, 33714, 33700, 33701, 33717, 33725, 33703, 33702, 33718, + 33705, 33704, 33728, 33719, 33726, 33720, 35762, 33711, 33710, 33713, + 33712, 33729, 33721, 33722, 35762, 33730, 33723, 35762, 33762, 33731, + 33732, 33742, 33741, 33744, 33743, 33734, 33733, 33751, 33759, 35762, + 33750, 33749, 33735, 33736, 33752, 33760, 33738, 33737, 33753, 33740, + 33739, 33763, 33754, 33761, 33755, 35762, 33746, 33745, 33748, 33747, + 33764, 33756, 33757, 35762, 33765, 33758, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 20199, 20200, 20201, 20202, 20203, 20204, 20205, + 20206, 20207, 20138, 20139, 20140, 20141, 20142, 20151, 20143, 20144, + 20145, 20146, 20147, 20148, 20149, 20150, 20152, 20153, 20154, 20155, + 20219, 20156, 20157, 20158, 20159, 20160, 20161, 20162, 20163, 20164, + 20165, 20166, 20167, 20168, 20169, 20170, 20171, 20172, 20173, 20174, + 20175, 20176, 20177, 20178, 20179, 20180, 20181, 20182, 20183, 20184, + 20185, 20186, 20187, 20188, 20189, 20190, 20191, 20192, 20193, 20194, + 20195, 20196, 20197, 20198, 19879, 20215, 20208, 19880, 20209, 20210, + 20211, 20212, 19881, 20216, 20217, 20213, 20214, 20218, 19885, 19886, + 19887, 19888, 19889, 19890, 19891, 19892, 19882, 19883, 19884, 19896, + 19897, 19898, 19893, 19894, 19895, 19899, 19900, 19901, 19902, 19903, + 19904, 19907, 19908, 19909, 19910, 19911, 19912, 19913, 19914, 19915, + 19916, 19917, 19918, 19919, 19920, 19921, 19922, 19923, 19924, 19925, + 19926, 19927, 19928, 19929, 19930, 19931, 19932, 19933, 19934, 19935, + 19936, 19937, 19938, 19939, 19940, 19941, 19942, 19943, 19944, 19945, + 19946, 19947, 19948, 19949, 19950, 19951, 19952, 19953, 19954, 19955, + 19956, 19905, 19906, 19957, 19958, 19959, 19960, 19961, 19962, 19963, + 19964, 19965, 19966, 19967, 19968, 19969, 19970, 19971, 19972, 19973, + 19974, 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19983, + 19984, 19985, 19986, 19987, 19988, 19989, 20021, 20022, 20023, 20024, + 20025, 20026, 20027, 20028, 20029, 19990, 19991, 19992, 19993, 19994, + 19995, 19996, 19997, 19998, 19999, 20000, 20001, 20002, 20003, 20004, + 20005, 20006, 20007, 20008, 20009, 20010, 20011, 20012, 20013, 20014, + 20015, 20016, 20017, 20018, 20019, 20020, 20036, 20037, 20038, 20039, + 20040, 20041, 20042, 20043, 20044, 20045, 20046, 20047, 20048, 20049, + 20050, 20051, 20052, 20053, 20054, 20055, 20030, 20031, 20032, 20033, + 20034, 20035, 20056, 20057, 20058, 20059, 20060, 20061, 20062, 20063, + 20098, 20099, 20100, 20101, 20102, 20103, 20104, 20105, 20106, 20107, + 20064, 20065, 20066, 20067, 20068, 20069, 20070, 20071, 20072, 20073, + 20074, 20075, 20076, 20077, 20078, 20079, 20080, 20081, 20082, 20083, + 20089, 20090, 20091, 20092, 20093, 20094, 20095, 20096, 20097, 20084, + 20085, 20086, 20087, 20088, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 20119, 20114, 20117, 20116, 20120, 20115, 20113, + 20118, 20112, 20108, 20111, 20110, 20109, 20126, 20121, 20125, 20122, + 20123, 20124, 20127, 20129, 20128, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 20130, 20131, 20132, 20133, 20134, + 20135, 20136, 20137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23185, 23306, 23305, + 23170, 23186, 23174, 35762, 23204, 23206, 23205, 23200, 23199, 23197, + 23198, 23259, 23192, 23216, 23256, 23180, 23220, 23181, 23224, 23187, + 23226, 23203, 23240, 23241, 23239, 23183, 23237, 23242, 23243, 23284, + 23285, 23248, 23184, 23193, 23299, 23281, 23282, 23255, 23254, 23189, + 23269, 23272, 23273, 23270, 23268, 23296, 35762, 23191, 23111, 23158, + 23006, 23089, 23118, 23003, 23160, 23261, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 9704, 9705, 9706, 9707, 9708, 9698, + 35762, 35762, 9699, 35762, 9654, 9655, 9656, 9657, 9658, 9659, 9660, + 9661, 9662, 9663, 9664, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9672, + 9673, 9674, 9675, 9676, 9677, 9678, 9679, 9680, 9681, 9682, 9683, 9684, + 9685, 9686, 9687, 9688, 9689, 9690, 9691, 9692, 9693, 9694, 9695, 9696, + 9697, 35762, 9702, 9703, 35762, 35762, 35762, 9700, 35762, 35762, 9701, + 16146, 16157, 16148, 16142, 16154, 16159, 16149, 16155, 16140, 16153, + 16156, 16143, 16161, 16158, 16150, 16147, 16160, 16151, 16144, 16145, + 16152, 16141, 35762, 16162, 16137, 16133, 16136, 16134, 16132, 16138, + 16139, 16135, 26241, 26252, 26243, 26237, 26249, 26254, 26244, 26250, + 26235, 26248, 26251, 26238, 26256, 26234, 26253, 26245, 26242, 26255, + 26246, 26239, 26240, 26247, 26236, 26233, 26257, 26264, 26259, 26260, + 26263, 26262, 26261, 26258, 24101, 24116, 24106, 24127, 24118, 24112, + 24108, 24124, 24129, 24119, 24125, 24110, 24104, 24123, 24105, 24126, + 24102, 24113, 24109, 24131, 24107, 24128, 24120, 24117, 24130, 24121, + 24114, 24115, 24103, 24122, 24111, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 24136, 24133, 24134, 24139, 24140, 24138, 24135, + 24132, 24137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 15202, 15218, 15210, 15209, 15215, 15220, 15204, 15216, 15205, 15214, + 15217, 15207, 15222, 15219, 15211, 15203, 15221, 15212, 15208, 35762, + 15213, 15206, 35762, 35762, 35762, 35762, 35762, 15223, 15227, 15226, + 15225, 15224, 26635, 26654, 26650, 26637, 26638, 26646, 26647, 26639, + 26645, 26648, 26651, 26653, 26656, 26652, 26643, 26636, 26655, 26641, + 26640, 26649, 26642, 26644, 26661, 26660, 26657, 26662, 26658, 26659, + 35762, 35762, 35762, 26663, 20631, 20637, 20641, 20639, 20633, 20649, + 20642, 20650, 20643, 20627, 20644, 20635, 20645, 20647, 20630, 20625, + 20648, 20640, 20646, 20629, 20626, 20632, 20634, 20628, 20636, 20638, + 35762, 35762, 35762, 35762, 35762, 20651, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 22722, 22723, 22724, 22725, 22721, 22720, 22696, 22697, 22718, 22717, + 22700, 22701, 22702, 22703, 22698, 22699, 22716, 22713, 22712, 22704, + 22705, 22706, 22714, 22719, 22707, 22708, 22709, 22710, 22711, 22715, + 22726, 22727, 22618, 22639, 22640, 22641, 22638, 22637, 22630, 22634, + 22633, 22623, 22624, 22636, 22632, 22628, 22627, 22625, 22619, 22626, + 22629, 22635, 22620, 22621, 22622, 22631, 35762, 35762, 35762, 35762, + 22606, 22611, 22643, 22642, 22692, 22684, 22678, 22655, 22649, 22672, + 22666, 22644, 22661, 22690, 22688, 22682, 22659, 22653, 22676, 22670, + 35762, 35762, 22693, 22685, 22679, 22656, 22650, 22673, 22667, 22646, + 22663, 22695, 22687, 22681, 22658, 22652, 22675, 22669, 22648, 22665, + 22691, 22689, 22683, 22660, 22654, 22677, 22671, 22645, 22662, 22694, + 22686, 22680, 22657, 22651, 22674, 22668, 22647, 22664, 22610, 22616, + 22615, 22609, 22608, 22613, 22612, 22607, 22617, 22614, 17200, 17222, + 17224, 17220, 35762, 17221, 17223, 35762, 35762, 35762, 35762, 35762, + 17225, 17216, 17219, 17218, 17166, 17164, 17188, 17187, 35762, 17186, + 17185, 17194, 35762, 17174, 17170, 17169, 17177, 17176, 17173, 17172, + 17171, 17179, 17178, 17175, 17190, 17189, 17184, 17183, 17196, 17198, + 17197, 17195, 17192, 17180, 17181, 17182, 17199, 17193, 17165, 17167, + 17168, 17191, 35762, 35762, 17214, 17215, 17217, 35762, 35762, 35762, + 35762, 17226, 17163, 17162, 17161, 17160, 17202, 17201, 17203, 17204, + 17227, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17208, 17213, + 17206, 17205, 17212, 17210, 17209, 17207, 17211, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 25648, 25644, 25649, 25654, 25645, 25652, + 25638, 25646, 25650, 25642, 25637, 25633, 25651, 25634, 25636, 25635, + 25653, 25626, 25627, 25629, 25632, 25630, 25631, 25641, 25643, 25628, + 25647, 25640, 25639, 25656, 25655, 25657, 25470, 25488, 25469, 25479, + 25491, 25492, 25481, 25485, 25483, 25476, 25480, 25472, 25487, 25471, + 25493, 25482, 25484, 25465, 25466, 25489, 25468, 25490, 25467, 25475, + 25477, 25473, 25486, 25474, 25478, 25496, 25495, 25494, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 20932, 20936, 20935, 20940, 20939, 20937, 20961, 20964, 20980, 20944, + 20943, 20942, 20941, 20962, 20954, 20960, 20946, 20956, 20945, 20958, + 20938, 20953, 20967, 20963, 20950, 20934, 20933, 20966, 20965, 20951, + 20948, 20957, 20947, 20959, 20952, 20949, 20955, 20982, 20981, 35762, + 35762, 35762, 35762, 20968, 20972, 20971, 20970, 20969, 20979, 20977, + 20975, 20974, 20973, 20978, 20976, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 2526, 2527, 2533, 2529, 2532, 2528, 2530, + 2531, 2569, 2570, 2559, 2560, 2561, 2562, 2557, 2558, 2574, 2547, 2546, + 2545, 2536, 2534, 2535, 2571, 2573, 2556, 2554, 2566, 2565, 2555, 2577, + 2572, 2564, 2563, 2541, 2540, 2539, 2544, 2543, 2542, 2576, 2537, 2552, + 2553, 2579, 2578, 2575, 2551, 2568, 2549, 2567, 2548, 2550, 2538, 35762, + 35762, 35762, 2580, 32334, 28927, 18140, 18135, 18141, 18136, 16288, + 16299, 16290, 16284, 16296, 16301, 16291, 16297, 16282, 16295, 16298, + 16285, 16303, 16300, 16292, 16289, 16302, 16293, 16286, 16287, 16294, + 16283, 35762, 35762, 16308, 16305, 16306, 16311, 16307, 16304, 16309, + 16310, 16257, 16271, 16262, 16258, 16268, 16261, 16263, 16269, 16255, + 16267, 16270, 16259, 16260, 16272, 16264, 16273, 16265, 16266, 16256, + 35762, 35762, 35762, 35762, 35762, 16278, 16275, 16276, 16281, 16277, + 16274, 16279, 16280, 26895, 26909, 26900, 26896, 26906, 26899, 26901, + 26907, 26905, 26908, 26897, 26898, 26910, 26902, 26912, 26903, 26904, + 26911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 26920, 26921, + 26893, 26894, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 26917, 26914, 26915, 26919, 26916, 26913, + 26918, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 25658, 25700, 25701, 25690, 25726, 25719, 25693, 25694, 25728, + 25671, 25711, 25659, 25704, 25673, 25713, 25661, 25705, 25672, 25712, + 25660, 25689, 25725, 25679, 25718, 25668, 25708, 25662, 25706, 25695, + 25729, 25674, 25714, 25663, 25684, 25687, 25675, 25664, 25702, 25682, + 25721, 25680, 25720, 25683, 25722, 25710, 25681, 25703, 25688, 25696, + 25691, 25686, 25724, 25676, 25715, 25692, 25727, 25697, 25730, 25677, + 25716, 25665, 25669, 25666, 25670, 25709, 25685, 25723, 25678, 25717, + 25667, 25707, 25698, 25699, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25318, + 25321, 25344, 25319, 25333, 25329, 25335, 25345, 25320, 25323, 25365, + 25346, 25347, 25336, 25337, 25348, 25366, 25367, 25349, 25350, 25322, + 25362, 25338, 25339, 25324, 25326, 25330, 25358, 25360, 25354, 25356, + 25359, 25351, 25325, 25352, 25368, 25331, 25332, 25340, 25327, 25341, + 25334, 25361, 25364, 25355, 25357, 25353, 25342, 25343, 25328, 25363, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 25369, 25372, 25395, 25370, 25384, 25380, 25386, + 25396, 25371, 25374, 25416, 25397, 25398, 25387, 25388, 25399, 25417, + 25418, 25400, 25401, 25373, 25413, 25389, 25390, 25375, 25377, 25381, + 25409, 25411, 25405, 25407, 25410, 25402, 25376, 25403, 25419, 25382, + 25383, 25391, 25378, 25392, 25385, 25412, 25415, 25406, 25408, 25404, + 25393, 25394, 25379, 25414, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 25422, 25421, 25425, 25420, 25423, 25424, 15157, 15144, 15152, + 15136, 15135, 15149, 15145, 15148, 15133, 15146, 15130, 15129, 15138, + 15137, 15156, 15143, 15142, 15134, 15147, 15150, 15151, 15141, 15154, + 15131, 15155, 15132, 15139, 15140, 15153, 15164, 15166, 15168, 15165, + 15167, 15159, 15158, 15163, 15160, 15162, 15161, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 15175, 15177, 15174, 15173, 15170, + 15169, 15172, 15171, 15178, 15176, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27439, + 27437, 27436, 27433, 27432, 27435, 27434, 27440, 27438, 27431, 27430, + 27428, 27419, 27417, 27426, 27424, 27415, 27421, 27422, 27429, 27427, + 27418, 27416, 27425, 27423, 27414, 27420, 27411, 27412, 27410, 27413, + 35762, 34219, 34253, 34233, 34232, 34238, 34237, 34215, 34214, 34213, + 34224, 34245, 34218, 34249, 34252, 34251, 34248, 34256, 34235, 34234, + 34236, 34217, 34239, 34250, 34220, 34244, 34255, 34240, 34241, 34228, + 34226, 34225, 34227, 34229, 34216, 34231, 34254, 34242, 34243, 34222, + 34223, 34246, 34221, 35762, 34211, 34210, 34212, 35762, 35762, 34230, + 34247, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 2335, 2336, 2333, 25588, 25596, + 25610, 25597, 25601, 25607, 25598, 25613, 25602, 25608, 25606, 25609, + 25600, 25615, 25611, 25590, 25591, 25603, 25589, 25587, 25614, 25604, + 25592, 25593, 25599, 25605, 25612, 25594, 25595, 25622, 25620, 25617, + 25625, 25624, 25621, 25619, 25618, 25623, 25586, 25616, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 28985, 28999, 28987, 28996, + 29003, 28991, 28997, 28995, 28998, 28988, 29005, 29001, 28992, 28986, + 29004, 28993, 28990, 28994, 29002, 29000, 28989, 28984, 28979, 28977, + 28980, 28978, 28983, 28982, 28974, 28973, 28975, 28981, 28976, 29006, + 29009, 29008, 29007, 29012, 29013, 29010, 29014, 29011, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 25735, 25747, 25745, 25750, 25739, 25744, 25743, 25746, 25737, 25752, + 25748, 25740, 25751, 25741, 25736, 25742, 25749, 25738, 25734, 25733, + 25732, 25731, 25756, 25753, 25754, 25755, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 6190, 6184, 6198, 6192, 6187, 6195, 6201, + 6183, 6193, 6196, 6194, 6197, 6188, 6203, 6199, 6185, 6191, 6202, 6189, + 6186, 6200, 6208, 6205, 6206, 6210, 6207, 6204, 6209, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12403, 12414, + 12405, 12399, 12411, 12416, 12406, 12412, 12397, 12410, 12413, 12400, + 12418, 12415, 12407, 12404, 12417, 12408, 12401, 12402, 12409, 12398, + 12419, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 4335, 4339, 4337, 4336, 4338, 4261, 4262, 4280, 4281, 4278, 4279, 4273, + 4274, 4275, 4276, 4307, 4263, 4254, 4264, 4300, 4299, 4296, 4295, 4284, + 4294, 4293, 4298, 4297, 4286, 4270, 4269, 4266, 4265, 4285, 4272, 4271, + 4268, 4267, 4287, 4302, 4301, 4292, 4291, 4304, 4306, 4305, 4283, 4277, + 4288, 4289, 4290, 4303, 4282, 4255, 4260, 4259, 4344, 4343, 4353, 4354, + 4351, 4352, 4347, 4348, 4349, 4350, 4355, 4345, 4340, 4346, 4356, 4358, + 4357, 4332, 4331, 4330, 4333, 4329, 35762, 35762, 35762, 35762, 4325, + 4323, 4320, 4311, 4313, 4318, 4316, 4308, 4314, 4324, 4322, 4321, 4310, + 4312, 4319, 4317, 4309, 4315, 4326, 4327, 4365, 4367, 4364, 4363, 4360, + 4359, 4362, 4361, 4368, 4366, 4334, 4257, 4258, 4341, 4342, 4256, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 4328, 16513, + 16515, 16517, 16473, 16474, 16483, 16484, 16481, 16482, 16511, 16475, + 16512, 16476, 16501, 16500, 16497, 16496, 16485, 16495, 16494, 16499, + 16498, 16487, 16478, 16477, 16470, 16468, 16469, 16504, 16486, 16480, + 16479, 16472, 16471, 16488, 16503, 16502, 16493, 16492, 16508, 16510, + 16505, 16507, 16509, 16489, 16490, 16491, 16506, 16523, 16528, 16529, + 16526, 16527, 16530, 16524, 16531, 16525, 16516, 16514, 16534, 16535, + 16532, 16518, 16519, 16521, 16520, 16522, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 16533, 35762, 35762, 29037, + 29038, 29027, 29028, 29029, 29030, 29023, 29024, 29034, 29026, 29039, + 29035, 29040, 29036, 29031, 29033, 29032, 29025, 29041, 29020, 29042, + 29044, 29043, 29021, 29022, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 29051, 29053, 29050, 29049, 29046, 29045, 29048, 29047, 29054, + 29052, 35762, 35762, 35762, 35762, 35762, 35762, 5873, 5875, 5874, 5869, + 5871, 5872, 5870, 5858, 5857, 5854, 5853, 5839, 5852, 5851, 5856, 5855, + 5841, 5844, 5843, 5836, 5835, 5840, 5846, 5845, 5838, 5837, 5842, 5862, + 5861, 5850, 5849, 5864, 5847, 5848, 5865, 5860, 5868, 5866, 5863, 5877, + 5885, 5886, 5881, 5882, 5883, 5879, 5887, 5880, 5888, 5905, 5904, 5889, + 5903, 35762, 5898, 5900, 5897, 5896, 5893, 5892, 5895, 5894, 5901, 5899, + 5876, 5891, 5890, 5902, 5859, 5878, 5884, 5867, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 20693, 20695, 20697, 20694, 20696, + 20685, 20684, 20681, 20680, 20679, 20678, 20683, 20682, 20664, 20673, + 20672, 20669, 20668, 20663, 20675, 20674, 20671, 20670, 20665, 20687, + 20686, 20677, 20676, 20690, 20667, 20689, 20692, 20691, 20688, 20666, + 20700, 20701, 20699, 20698, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 27836, 27835, 27841, 27780, 27781, 27799, 27800, + 27797, 27798, 27792, 27793, 27794, 27795, 27826, 27782, 27827, 27783, + 27819, 27818, 27815, 27814, 27803, 27813, 27812, 27817, 27816, 27805, + 27789, 27788, 27785, 27784, 27804, 27791, 27790, 27787, 27786, 27806, + 27821, 27820, 27811, 27810, 27823, 27825, 27824, 27802, 27801, 27796, + 27807, 27808, 27809, 27822, 27845, 27854, 27855, 27852, 27853, 27848, + 27849, 27850, 27851, 27856, 27846, 27857, 27847, 27840, 27834, 27838, + 27839, 27860, 27766, 27765, 27858, 27831, 27828, 27837, 27843, 27777, + 27842, 27844, 27832, 27773, 27775, 27772, 27771, 27768, 27767, 27770, + 27769, 27776, 27774, 27778, 27833, 27779, 27859, 27829, 27830, 35762, + 28741, 28739, 28738, 28735, 28734, 28737, 28736, 28742, 28740, 28753, + 28752, 28751, 28747, 28746, 28750, 28749, 28745, 28748, 28743, 28744, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 17905, 17906, 17932, 17934, 17931, 17907, 17933, 17908, 17922, + 17921, 17897, 17895, 17896, 17915, 17920, 17919, 17904, 17903, 35762, + 17917, 17910, 17909, 17900, 17899, 17916, 17912, 17911, 17902, 17898, + 17901, 17918, 17924, 17923, 17894, 17892, 17893, 17926, 17930, 17928, + 17914, 17929, 17891, 17925, 17913, 17942, 17945, 17946, 17949, 17947, + 17943, 17948, 17944, 17939, 17937, 17938, 17935, 17889, 17888, 17950, + 17940, 17887, 17951, 17936, 17927, 17890, 17941, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23597, + 23599, 23600, 23598, 23588, 23587, 23586, 35762, 23585, 35762, 23584, + 23583, 23576, 23575, 35762, 23570, 23578, 23577, 23566, 23564, 23565, + 23569, 23580, 23579, 23568, 23567, 23571, 23590, 23589, 23582, 35762, + 23581, 23593, 23596, 23574, 23592, 23595, 23594, 23591, 23573, 23572, + 23601, 35762, 35762, 35762, 35762, 35762, 35762, 17968, 17969, 17978, + 17979, 17976, 17977, 17997, 17970, 17998, 17971, 17987, 17986, 17957, + 17955, 17956, 17980, 17985, 17984, 17960, 17959, 17958, 17982, 17973, + 17972, 17963, 17961, 17966, 17962, 17981, 17975, 17974, 17965, 17964, + 17983, 17989, 17988, 17954, 17952, 17953, 17994, 17996, 17967, 17993, + 17995, 17990, 17991, 17992, 18001, 18002, 18007, 18008, 18005, 18006, + 18009, 18003, 18010, 18004, 18000, 17999, 35762, 35762, 35762, 35762, + 35762, 18017, 18019, 18016, 18015, 18012, 18011, 18014, 18013, 18020, + 18018, 35762, 35762, 35762, 35762, 35762, 35762, 13699, 13700, 13703, + 13706, 35762, 13656, 13657, 13670, 13671, 13668, 13669, 13651, 13653, + 35762, 35762, 13694, 13658, 35762, 35762, 13693, 13659, 13690, 13689, + 13686, 13685, 13674, 13684, 13683, 13688, 13687, 13676, 13665, 13664, + 13661, 13660, 13675, 13667, 13666, 13663, 13662, 13677, 35762, 13692, + 13691, 13682, 13681, 13696, 13698, 13697, 35762, 13673, 13672, 35762, + 13655, 13678, 13679, 13680, 13695, 35762, 7774, 13704, 13702, 13707, + 13716, 13717, 13714, 13715, 13710, 13711, 35762, 35762, 13719, 13708, + 35762, 35762, 13718, 13709, 13705, 35762, 35762, 13721, 35762, 35762, + 35762, 35762, 35762, 35762, 13720, 35762, 35762, 35762, 35762, 35762, + 13701, 13650, 13649, 13652, 13654, 13712, 13713, 35762, 35762, 7954, + 7955, 7953, 7952, 7951, 7950, 7949, 35762, 35762, 35762, 7960, 7957, + 7958, 7956, 7959, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 24495, 24496, 24509, 24510, 24507, 24508, 24491, 24492, 24493, + 24494, 24535, 24497, 24536, 24498, 24523, 24522, 24519, 24518, 24484, + 24483, 24517, 24516, 24521, 24520, 24486, 24485, 24504, 24503, 24500, + 24499, 24488, 24506, 24505, 24502, 24501, 24489, 24487, 24529, 24528, + 24515, 24514, 24527, 24526, 24534, 24531, 24530, 24525, 24524, 24533, + 24511, 24512, 24513, 24532, 24549, 24558, 24559, 24556, 24557, 24552, + 24553, 24554, 24555, 24560, 24550, 24561, 24551, 24544, 24539, 24538, + 24545, 24540, 24537, 24542, 24565, 24546, 24471, 24469, 24564, 24482, + 24562, 24478, 24480, 24477, 24476, 24473, 24472, 24475, 24474, 24481, + 24479, 24470, 24548, 35762, 24563, 24547, 24490, 24541, 24543, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32336, + 32337, 32338, 32356, 32357, 32354, 32355, 32349, 32350, 32351, 32352, + 32382, 32339, 32383, 32340, 32374, 32373, 32370, 32369, 32358, 32368, + 32367, 32372, 32371, 32360, 32346, 32345, 32342, 32341, 32359, 32348, + 32347, 32344, 32343, 32361, 32376, 32375, 32366, 32365, 32379, 32381, + 32380, 32378, 32353, 32362, 32363, 32364, 32377, 32390, 32401, 32402, + 32399, 32400, 32395, 32396, 32397, 32398, 32403, 32393, 32391, 32404, + 32394, 32392, 32386, 32385, 32389, 32388, 32387, 32384, 32415, 32335, + 32416, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32411, + 32413, 32410, 32409, 32406, 32405, 32408, 32407, 32414, 32412, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 27949, 27951, 27972, 27973, 27970, + 27971, 27965, 27966, 27967, 27968, 27998, 27952, 27999, 27953, 27990, + 27989, 27986, 27985, 27974, 27984, 27983, 27988, 27987, 27976, 27959, + 27958, 27962, 27961, 27975, 27960, 27955, 27964, 27963, 27977, 27992, + 27991, 27982, 27981, 27995, 27997, 27996, 27994, 27969, 27978, 27979, + 27980, 27993, 28027, 28034, 28035, 28032, 28033, 28030, 28031, 35762, + 35762, 28036, 28028, 28037, 28029, 28020, 28022, 28024, 28023, 28021, + 28019, 27947, 27946, 28018, 28017, 28000, 28001, 28002, 27948, 28015, + 28014, 28012, 28010, 28011, 28003, 28004, 28013, 28016, 28008, 28009, + 28007, 28006, 28005, 27954, 27956, 27957, 27950, 28025, 28026, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 22953, 22954, 22972, 22973, 22970, 22971, 22965, + 22966, 22967, 22968, 22999, 22955, 23000, 22956, 22992, 22991, 22988, + 22987, 22976, 22986, 22985, 22990, 22989, 22978, 22962, 22961, 22958, + 22957, 22977, 22964, 22963, 22960, 22959, 22979, 22994, 22993, 22984, + 22983, 22996, 22998, 22997, 22975, 22969, 22980, 22981, 22982, 22995, + 22974, 22928, 22937, 22938, 22935, 22936, 22931, 22932, 22933, 22934, + 22939, 22929, 22940, 22930, 22924, 22927, 22926, 22923, 22942, 22941, + 23001, 22925, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 22949, 22951, 22948, 22947, 22944, 22943, 22946, + 22945, 22952, 22950, 35762, 35762, 35762, 35762, 35762, 35762, 23481, + 23476, 23321, 23482, 23480, 23478, 23477, 23338, 23339, 23472, 23474, + 23473, 23483, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 30326, 30328, 30343, 30344, 30341, 30342, 30368, 30329, 30369, + 30330, 30358, 30357, 30354, 30353, 30345, 30352, 30351, 30356, 30355, + 30347, 30338, 30337, 30334, 30333, 30346, 30340, 30339, 30336, 30335, + 30348, 30360, 30359, 30350, 30349, 30365, 30367, 30332, 30364, 30366, + 30361, 30362, 30363, 30331, 30371, 30373, 30374, 30379, 30380, 30377, + 30378, 30381, 30375, 30382, 30376, 30372, 30370, 30327, 30325, 35762, + 35762, 35762, 35762, 35762, 35762, 30389, 30391, 30388, 30387, 30384, + 30383, 30386, 30385, 30392, 30390, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 191, 190, + 178, 181, 175, 167, 193, 192, 183, 195, 189, 184, 174, 196, 177, 197, + 180, 194, 164, 171, 170, 187, 166, 186, 182, 188, 165, 35762, 35762, 162, + 163, 161, 203, 204, 210, 211, 208, 209, 212, 207, 213, 205, 206, 200, + 35762, 35762, 35762, 35762, 222, 224, 221, 220, 217, 216, 219, 218, 225, + 223, 215, 214, 198, 199, 201, 202, 185, 173, 172, 169, 168, 179, 176, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 10719, 10720, 10733, 10734, 10731, + 10732, 10759, 10721, 10760, 10722, 10751, 10750, 10747, 10746, 10735, + 10745, 10744, 10749, 10748, 10737, 10728, 10727, 10724, 10723, 10736, + 10730, 10729, 10726, 10725, 10738, 10753, 10752, 10743, 10742, 10756, + 10758, 10718, 10755, 10757, 10739, 10740, 10741, 10754, 10717, 10763, + 10768, 10769, 10766, 10767, 10761, 10762, 10770, 10764, 10771, 10765, + 10774, 10776, 10775, 10773, 10772, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 33866, 33855, 33884, 33874, 33876, + 33877, 33883, 33873, 33859, 33868, 33856, 33886, 33882, 33861, 33875, + 33872, 33860, 33869, 33878, 33867, 33885, 33858, 33857, 33880, 33881, + 33864, 33863, 33862, 33865, 33870, 33871, 33879, 33898, 33887, 33916, + 33906, 33908, 33909, 33915, 33905, 33891, 33900, 33888, 33918, 33914, + 33893, 33907, 33904, 33892, 33901, 33910, 33899, 33917, 33890, 33889, + 33912, 33913, 33896, 33895, 33894, 33897, 33902, 33903, 33911, 33925, + 33927, 33924, 33923, 33920, 33919, 33922, 33921, 33928, 33926, 33937, + 33936, 33935, 33931, 33930, 33934, 33933, 33929, 33932, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 33938, 10633, 10634, 10641, 10642, 10639, 10640, 10668, 35762, 35762, + 10669, 35762, 35762, 10659, 10658, 10657, 10656, 10645, 10655, 10654, + 10663, 35762, 10647, 10629, 35762, 10636, 10635, 10646, 10630, 10628, + 10638, 10637, 10648, 10661, 10660, 10653, 10652, 10664, 10632, 10631, + 10665, 10644, 10666, 10649, 10650, 10651, 10662, 10643, 10667, 10674, + 10678, 10679, 10676, 10677, 10680, 35762, 10675, 10681, 35762, 35762, + 10673, 10671, 10670, 10682, 10687, 10684, 10688, 10683, 10672, 10617, + 10685, 10686, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 10624, 10626, 10623, 10622, 10619, 10618, 10621, 10620, 10627, + 10625, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 24192, 24193, 24206, 24207, 24204, 24205, 24187, 24188, 35762, + 35762, 24232, 24194, 24233, 24195, 24226, 24225, 24222, 24221, 24210, + 24220, 24219, 24224, 24223, 24212, 24201, 24200, 24197, 24196, 24211, + 24203, 24202, 24199, 24198, 24213, 24228, 24227, 24218, 24217, 24230, + 24231, 24191, 24209, 24189, 24214, 24215, 24216, 24229, 24208, 24190, + 24242, 24247, 24248, 24245, 24246, 24240, 24241, 35762, 35762, 24249, + 24243, 24250, 24244, 24236, 24238, 24237, 24235, 24234, 24251, 24239, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35543, 35564, 35561, + 35560, 35563, 35559, 35558, 35556, 35557, 35562, 35555, 35511, 35510, + 35530, 35529, 35512, 35528, 35527, 35537, 35514, 35522, 35521, 35504, + 35503, 35513, 35524, 35523, 35508, 35507, 35515, 35532, 35531, 35526, + 35525, 35539, 35520, 35519, 35506, 35505, 35533, 35534, 35535, 35542, + 35540, 35538, 35541, 35516, 35517, 35518, 35536, 35509, 35502, 35552, + 35549, 35550, 35551, 35548, 35553, 35499, 35498, 35496, 35495, 35497, + 35501, 35494, 35547, 35545, 35544, 35546, 35500, 35493, 35554, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29142, 29163, 29161, + 29160, 29162, 29158, 29159, 29156, 29157, 29155, 29154, 29164, 29109, + 29108, 29128, 29127, 29110, 29126, 29125, 29130, 29129, 29112, 29120, + 29119, 29103, 29102, 29111, 29122, 29121, 29106, 29104, 29113, 29132, + 29131, 29124, 29123, 29138, 29118, 29117, 29105, 29133, 29134, 29135, + 29141, 29139, 29137, 29140, 29114, 29115, 29116, 29136, 29107, 29147, + 29149, 29085, 29084, 29082, 29083, 29093, 29094, 29089, 29092, 29088, + 29091, 29096, 29097, 29095, 29087, 29086, 29090, 29150, 29148, 29098, + 29151, 29146, 29145, 29144, 29143, 29101, 29100, 29099, 29152, 29153, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 5316, 5317, 5314, 5315, 5312, 5313, 5322, 5323, + 5320, 5321, 5318, 5319, 5485, 5486, 5487, 5484, 26453, 26451, 26460, + 26461, 26457, 26465, 26464, 26446, 26459, 26458, 26450, 26463, 26456, + 26449, 26455, 26454, 26447, 26452, 26462, 26441, 26448, 26466, 26467, + 26442, 26468, 26444, 26445, 26443, 26437, 26434, 26438, 26436, 26432, + 26435, 26439, 26433, 26440, 26474, 26473, 26484, 26475, 26476, 26485, + 26481, 26480, 26482, 26483, 26477, 26431, 26478, 26479, 26470, 26469, + 26429, 26471, 26472, 26430, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 10360, 10361, 10450, 10451, 10452, 10453, 10460, 10459, 10461, + 10469, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3641, 3642, 3655, 3656, + 3653, 3654, 3637, 3638, 3639, 35762, 3681, 3643, 3682, 3644, 3673, 3672, + 3669, 3668, 3657, 3667, 3666, 3671, 3670, 3659, 3650, 3649, 3646, 3645, + 3658, 3652, 3651, 3648, 3647, 3660, 3675, 3674, 3665, 3664, 3678, 3680, + 3679, 3677, 3640, 3661, 3662, 3663, 3676, 3709, 3714, 3715, 3712, 3713, + 3706, 3707, 3708, 35762, 3716, 3710, 3717, 3711, 3703, 3702, 3705, 3704, + 3701, 3719, 3718, 3730, 3731, 3732, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 3726, 3728, 3725, 3724, 3721, 3720, + 3723, 3722, 3729, 3727, 3700, 3698, 3695, 3686, 3688, 3693, 3691, 3683, + 3689, 3699, 3697, 3696, 3685, 3687, 3694, 3692, 3684, 3690, 3733, 35762, + 35762, 35762, 21057, 21058, 21003, 21002, 21012, 20997, 21001, 21000, + 21014, 20998, 20994, 20993, 20996, 20999, 21005, 21004, 21011, 21016, + 20992, 20991, 20995, 21018, 21008, 21009, 21010, 21019, 21017, 21015, + 21006, 21007, 21013, 21020, 35762, 35762, 21035, 21034, 21043, 21029, + 21033, 21032, 21045, 21030, 21026, 21025, 21028, 21031, 21037, 21036, + 21042, 21047, 21024, 21023, 21027, 21049, 21040, 21041, 35762, 21050, + 21048, 21046, 21038, 21039, 21044, 21051, 21052, 21054, 21056, 21053, + 21055, 21022, 21021, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 21069, 21070, 21079, 21080, + 21077, 21078, 21106, 35762, 21071, 21107, 35762, 21072, 21085, 21084, + 21098, 21097, 21086, 21096, 21095, 21063, 21062, 21088, 21065, 21064, + 21074, 21073, 21087, 21068, 21066, 21076, 21075, 21089, 21100, 21099, + 21094, 21093, 21102, 21105, 21103, 21082, 21104, 21090, 21091, 21092, + 21101, 21081, 21083, 21061, 21067, 21116, 21121, 21122, 21119, 21120, + 21115, 35762, 35762, 35762, 21123, 35762, 21117, 21124, 35762, 21118, + 21114, 21113, 21112, 21110, 21111, 21125, 21109, 21108, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 21132, 21134, 21131, 21130, + 21127, 21126, 21129, 21128, 21135, 21133, 35762, 35762, 35762, 35762, + 35762, 35762, 14372, 14373, 14388, 14389, 14384, 14385, 35762, 14405, + 14374, 35762, 14404, 14375, 14411, 14410, 14393, 14392, 14407, 14401, + 14400, 14383, 14382, 14391, 14397, 14396, 14379, 14378, 14387, 14395, + 14394, 14381, 14380, 14390, 14399, 14398, 14377, 14376, 14386, 14403, + 14402, 14406, 14408, 14409, 14414, 14419, 14420, 14417, 14418, 35762, + 14422, 14415, 35762, 14421, 14416, 14413, 14412, 14423, 14434, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 14430, 14432, 14429, 14428, + 14425, 14424, 14427, 14426, 14433, 14431, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 20755, 20753, 20747, 20758, + 20750, 20757, 20761, 20752, 20749, 20751, 20754, 20748, 20763, 20759, + 20756, 20762, 20760, 20764, 20770, 20767, 20769, 20766, 20768, 20765, + 20746, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17048, 17050, + 17046, 17049, 16988, 16989, 17008, 17009, 17006, 17007, 17002, 17003, + 17004, 17005, 17033, 16990, 17034, 35762, 17024, 17023, 17022, 17021, + 17010, 17020, 17019, 16993, 16992, 17012, 16999, 16998, 16995, 16994, + 17011, 17001, 17000, 16997, 16996, 17013, 17026, 17025, 17018, 17017, + 17029, 17032, 17030, 17028, 17031, 17014, 17015, 17016, 17027, 16991, + 17052, 17051, 17059, 17060, 17057, 17058, 17054, 35762, 35762, 35762, + 17055, 17053, 17056, 17047, 17073, 17062, 17061, 17041, 17044, 17040, + 17042, 17038, 17037, 17045, 17036, 17039, 17043, 17035, 17069, 17071, + 17068, 17067, 17064, 17063, 17066, 17065, 17072, 17070, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 20466, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30449, 30445, 30439, 30448, 30441, 30450, 30454, 30456, 30451, 30446, + 30447, 30452, 30440, 30457, 30455, 30442, 30453, 30443, 30444, 30458, + 30459, 30523, 30506, 30504, 30513, 30512, 30508, 30514, 30510, 30509, + 30516, 30517, 30518, 30515, 30507, 30520, 30438, 30425, 30496, 30503, + 30804, 30805, 30423, 30398, 30524, 30803, 30460, 30525, 30511, 30519, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 30501, 8517, 8525, 8523, 8519, 8524, 8520, 8518, + 8521, 8522, 8586, 8526, 8535, 8534, 8528, 8527, 8539, 8529, 8530, 8538, + 8532, 8533, 8540, 8541, 8546, 8543, 8542, 8544, 8545, 8548, 8550, 8552, + 8551, 8553, 8559, 8556, 8557, 8561, 8554, 8555, 8560, 8558, 8563, 8562, + 8564, 8566, 8567, 8571, 8570, 8568, 8569, 8572, 8585, 8573, 8574, 8575, + 8584, 8576, 8580, 8581, 8579, 8577, 8578, 8583, 8582, 8587, 8588, 8599, + 8590, 8594, 8595, 8596, 8597, 8598, 8600, 8603, 8602, 8601, 8604, 8606, + 8607, 8608, 8609, 8610, 8611, 8612, 8613, 8614, 8615, 8616, 8617, 8618, + 8619, 8620, 8621, 8622, 8623, 8638, 8624, 8625, 8635, 8629, 8626, 8627, + 8628, 8636, 8633, 8637, 8634, 8630, 8632, 8645, 8641, 8642, 8643, 8646, + 8657, 8647, 8650, 8651, 8653, 8654, 8655, 8658, 8661, 8659, 8660, 8662, + 8663, 8665, 8666, 8695, 8702, 8696, 8697, 8698, 8699, 8700, 8701, 8703, + 8705, 8704, 8706, 8709, 8710, 8713, 8707, 8708, 8714, 8759, 8758, 8760, + 8715, 8718, 8719, 8720, 8716, 8717, 8721, 8724, 8722, 8725, 8727, 8736, + 8737, 8738, 8739, 8757, 8740, 8741, 8754, 8755, 8753, 8742, 8743, 8744, + 8745, 8746, 8747, 8748, 8751, 8752, 8761, 8851, 8762, 8763, 8765, 8764, + 8771, 8766, 8768, 8770, 8774, 8773, 8775, 8776, 8783, 8777, 8778, 8782, + 8786, 8787, 8784, 8785, 8793, 8790, 8794, 8795, 8796, 8797, 8798, 8800, + 8801, 8803, 8802, 8805, 8804, 8807, 8806, 8808, 8809, 8811, 8812, 8816, + 8818, 8822, 8823, 8836, 8828, 8829, 8824, 8825, 8826, 8830, 8834, 8831, + 8832, 8833, 8837, 8838, 8840, 8841, 8842, 8843, 8844, 8845, 8855, 8846, + 8847, 8850, 8849, 8848, 8852, 8853, 8854, 8856, 8857, 8860, 8861, 8862, + 8863, 8864, 8866, 8865, 8882, 8873, 8874, 8867, 8868, 8870, 8871, 8869, + 8872, 8881, 8875, 8880, 8878, 8877, 8879, 8884, 8903, 8885, 8886, 8887, + 8890, 8889, 8891, 8892, 8894, 8895, 8896, 8904, 8897, 8898, 8899, 8902, + 8901, 8900, 8905, 8906, 8908, 8909, 8910, 8911, 8913, 8917, 8914, 8918, + 8916, 8915, 8919, 8920, 8921, 8922, 8926, 8925, 8923, 8924, 8927, 8928, + 8930, 8949, 8951, 8931, 8933, 8932, 8934, 8935, 8938, 8939, 8937, 8936, + 8940, 8941, 8942, 8943, 8946, 8944, 8945, 8947, 8948, 8952, 8953, 8950, + 8954, 8955, 8956, 8957, 8959, 8962, 8961, 8963, 8964, 8966, 8967, 8968, + 8972, 8971, 8969, 8970, 8973, 8977, 8976, 8975, 8978, 8979, 8981, 8982, + 8986, 8983, 8984, 8989, 8987, 8990, 8991, 8993, 8992, 8994, 8995, 9017, + 9016, 9019, 9020, 9001, 9002, 8999, 9000, 8998, 8996, 9004, 9003, 9005, + 9007, 9013, 9014, 9011, 9012, 9021, 9022, 9023, 9040, 9026, 9027, 9028, + 9024, 9025, 9029, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9037, 9038, + 9063, 9041, 9044, 9042, 9043, 9049, 9050, 9047, 9048, 9045, 9046, 9051, + 9052, 9060, 9053, 9054, 9061, 9058, 9059, 9062, 9055, 9056, 9057, 9064, + 9065, 9066, 9067, 9068, 9069, 9070, 9072, 9073, 9071, 9074, 9075, 9116, + 9117, 9078, 9079, 9076, 9077, 9082, 9083, 9081, 9087, 9084, 9086, 9085, + 9091, 9092, 9090, 9088, 9089, 9093, 9094, 9095, 9096, 9097, 9098, 9099, + 9118, 9104, 9100, 9101, 9102, 9103, 9105, 9107, 9106, 9108, 9109, 9111, + 9110, 9112, 9114, 9113, 9119, 9120, 9123, 9124, 9121, 9122, 9198, 9193, + 9194, 9195, 9196, 9197, 9199, 9202, 9200, 9201, 9203, 9245, 9204, 9234, + 9235, 9211, 9213, 9230, 9214, 9236, 9218, 9216, 9217, 9219, 9220, 9221, + 9222, 9231, 9232, 9225, 9226, 9228, 9237, 9205, 9206, 9210, 9208, 9246, + 9238, 9240, 9239, 9241, 9247, 9248, 9242, 9243, 9244, 9249, 9250, 9251, + 9254, 9255, 9256, 9252, 9253, 9257, 9258, 9260, 9262, 9263, 9281, 9279, + 9280, 9283, 9282, 9264, 9271, 9269, 9270, 9265, 9266, 9272, 9273, 9274, + 9275, 9276, 9278, 9284, 9293, 9285, 9287, 9288, 9286, 9289, 9291, 9290, + 9292, 9295, 9297, 9296, 9298, 9299, 9332, 9334, 9300, 9302, 9301, 9304, + 9307, 9305, 9306, 9310, 9314, 9317, 9316, 9319, 9322, 9320, 9321, 9325, + 9327, 9333, 9335, 9336, 9340, 9347, 9345, 9343, 9344, 9346, 9349, 9348, + 9341, 9342, 9350, 9353, 9359, 9354, 9357, 9352, 9351, 9360, 9358, 9355, + 9356, 9361, 9362, 9363, 9364, 9365, 9366, 9367, 9369, 9372, 9373, 9376, + 9377, 9378, 9374, 9375, 9370, 9371, 9379, 9380, 9381, 9382, 9383, 9384, + 9386, 9387, 9388, 9389, 9390, 9394, 9391, 9415, 9408, 9409, 9414, 9397, + 9396, 9410, 9413, 9411, 9400, 9399, 9402, 9404, 9405, 9406, 9407, 9403, + 9416, 9392, 9417, 9418, 9419, 9420, 9421, 9422, 9430, 9428, 9426, 9429, + 9425, 9427, 9423, 9424, 9431, 9433, 9434, 9435, 9442, 9437, 9438, 9446, + 9447, 9443, 9445, 9444, 9448, 9450, 9449, 9451, 9462, 9453, 9452, 9461, + 9459, 9454, 9455, 9456, 9458, 9457, 9460, 9466, 9463, 9465, 9464, 9467, + 9468, 9469, 9470, 9473, 9474, 9476, 9477, 9478, 9479, 9481, 9480, 9482, + 9489, 9483, 9484, 9490, 9485, 9486, 9487, 9488, 9491, 9495, 9492, 9493, + 9494, 9496, 9497, 9498, 9499, 9505, 9503, 9501, 9502, 9500, 9504, 9506, + 9508, 9525, 9526, 9509, 9514, 9516, 9510, 9513, 9511, 9512, 9517, 9523, + 9524, 9518, 9521, 9522, 9527, 9533, 9532, 9528, 9530, 9529, 9614, 9615, + 9534, 9535, 9540, 9541, 9538, 9539, 9542, 9536, 9537, 9543, 9546, 9547, + 9549, 9550, 9551, 9555, 9552, 9553, 9554, 9544, 9545, 9556, 9558, 9557, + 9559, 9561, 9562, 9563, 9569, 9568, 9564, 9565, 9566, 9601, 9570, 9571, + 9572, 9573, 9574, 9593, 9576, 9577, 9579, 9578, 9580, 9581, 9597, 9582, + 9584, 9583, 9596, 9586, 9594, 9598, 9589, 9588, 9595, 9590, 9592, 9591, + 9599, 9600, 9602, 9606, 9604, 9605, 9603, 9609, 9608, 9607, 9613, 9610, + 9611, 9612, 9616, 9618, 9617, 9621, 9619, 9636, 9622, 9625, 9627, 9623, + 9624, 9628, 9626, 9629, 9631, 9633, 9634, 9635, 9039, 8536, 8547, 8565, + 8631, 8640, 8656, 8664, 8756, 8749, 8767, 8769, 8859, 8883, 8929, 8958, + 8960, 8974, 8980, 9015, 8988, 9018, 8997, 9006, 9010, 9080, 9209, 9212, + 9227, 9259, 9277, 9294, 9303, 9331, 9329, 9308, 9339, 9368, 9385, 9395, + 9515, 9548, 9531, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 8501, 8496, 8433, 8419, 8480, 8476, 8408, + 8453, 8500, 8441, 8425, 8486, 8475, 8407, 8452, 8439, 8423, 8482, 8472, + 8402, 8449, 8462, 8506, 8498, 8435, 8421, 8484, 8474, 8404, 8451, 8463, + 8507, 8499, 8436, 8422, 8508, 8490, 8491, 8437, 8413, 8479, 8471, 8401, + 8448, 8466, 8509, 8492, 8493, 8438, 8414, 8477, 8478, 8461, 8504, 8487, + 8488, 8428, 8418, 8494, 8495, 8429, 8432, 8430, 8431, 8485, 8470, 8468, + 8469, 8405, 8406, 8444, 8446, 8447, 8445, 8502, 8497, 8434, 8420, 8481, + 8460, 8503, 8489, 8426, 8427, 8416, 8417, 8443, 8442, 8456, 8505, 8465, + 8511, 8415, 8464, 8510, 8457, 8458, 8454, 8455, 8459, 8467, 8409, 8412, + 8411, 8410, 8440, 8424, 8483, 8473, 8403, 8450, 35762, 8516, 8515, 8514, + 8512, 8513, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 8537, 8531, 8549, 8589, 8591, 8592, 8593, 8605, + 8644, 8639, 8649, 8648, 8652, 8669, 8667, 8668, 8670, 8671, 8689, 8675, + 8672, 8673, 8674, 8691, 8692, 8690, 8679, 8678, 8676, 8677, 8682, 8680, + 8681, 8683, 8684, 8685, 8686, 8693, 8694, 8688, 8687, 8712, 8711, 8723, + 8726, 8731, 8735, 8728, 8732, 8733, 8729, 8730, 8734, 8750, 8772, 8779, + 8780, 8781, 8788, 8789, 8792, 8791, 8799, 8810, 8813, 8814, 8815, 8817, + 8819, 8820, 8821, 8827, 8835, 8839, 8858, 8876, 8888, 8893, 8907, 8912, + 8965, 8985, 9008, 9009, 9115, 9135, 9125, 9126, 9134, 9130, 9131, 9132, + 9129, 9128, 9127, 9133, 9137, 9136, 9143, 9144, 9138, 9139, 9140, 9145, + 9141, 9142, 9146, 9147, 9148, 9149, 9150, 9151, 9158, 9152, 9157, 9156, + 9154, 9153, 9155, 9159, 9160, 9165, 9166, 9161, 9162, 9163, 9164, 9192, + 9188, 9167, 9175, 9176, 9174, 9173, 9177, 9168, 9169, 9171, 9172, 9170, + 9189, 9178, 9185, 9187, 9182, 9183, 9186, 9181, 9184, 9180, 9179, 9190, + 9191, 9207, 9233, 9215, 9223, 9224, 9229, 9261, 9268, 9267, 9328, 9309, + 9311, 9330, 9312, 9313, 9315, 9318, 9323, 9324, 9326, 9393, 9412, 9398, + 9401, 9432, 9436, 9439, 9440, 9441, 9472, 9471, 9475, 9507, 9519, 9520, + 9560, 9567, 9575, 9587, 9585, 9620, 9630, 9632, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 9709, + 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9719, 9720, 9717, 9718, 9721, + 9722, 9723, 9724, 9725, 9726, 9727, 9728, 9729, 9730, 9731, 9732, 9733, + 9734, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9744, 9745, + 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9755, 9775, 9776, + 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9758, 9759, 9760, 9761, 9762, + 9756, 9757, 9763, 9764, 9765, 9784, 9785, 9786, 9787, 9788, 9789, 9790, + 9791, 9792, 9793, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, + 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, + 9806, 9807, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 11228, 11229, 11230, 11231, 11226, + 11227, 11223, 11224, 11225, 11232, 11233, 11234, 11239, 11240, 11241, + 11242, 11235, 11236, 11243, 11244, 11237, 11238, 11245, 11246, 11273, + 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11263, + 11264, 11261, 11262, 11265, 11266, 11267, 11268, 11269, 11270, 11271, + 11247, 11248, 11255, 11249, 11250, 11251, 11252, 11256, 11253, 11254, + 11257, 11258, 11259, 11260, 11283, 11284, 11285, 11286, 11287, 11288, + 11289, 11290, 11291, 11292, 11293, 11294, 11295, 11296, 11297, 11298, + 11299, 11300, 11301, 11302, 11272, 11342, 11343, 11344, 11345, 11340, + 11341, 11346, 11347, 11348, 11349, 11354, 11350, 11351, 11352, 11353, + 11355, 11356, 11357, 11358, 11359, 11360, 11361, 11362, 11363, 11364, + 11365, 11366, 11367, 11368, 11369, 11370, 11371, 11372, 11373, 11374, + 11375, 11376, 11377, 11380, 11381, 11382, 11383, 11384, 11385, 11386, + 11378, 11379, 11387, 11460, 11461, 11462, 11463, 11464, 11465, 11466, + 11467, 11468, 11469, 11451, 11452, 11453, 11454, 11455, 11456, 11457, + 11449, 11450, 11458, 11459, 11392, 11388, 11389, 11393, 11394, 11390, + 11391, 11395, 11396, 11397, 11398, 11399, 11404, 11405, 11406, 11407, + 11408, 11409, 11400, 11401, 11410, 11402, 11403, 11411, 11412, 11413, + 11414, 11415, 11416, 11417, 11418, 11419, 11420, 11421, 11426, 11422, + 11423, 11427, 11424, 11425, 11428, 11429, 11430, 11431, 11432, 11442, + 11443, 11444, 11445, 11446, 11447, 11448, 11433, 11434, 11435, 11436, + 11437, 11438, 11439, 11440, 11441, 11474, 11475, 11476, 11477, 11478, + 11479, 11480, 11470, 11471, 11472, 11473, 11506, 11507, 11508, 11509, + 11510, 11511, 11502, 11503, 11504, 11505, 11512, 11513, 11481, 11482, + 11485, 11486, 11487, 11488, 11489, 11490, 11491, 11483, 11484, 11492, + 11495, 11496, 11497, 11498, 11493, 11494, 11499, 11500, 11501, 11517, + 11518, 11519, 11520, 11521, 11522, 11523, 11524, 11525, 11526, 11529, + 11530, 11531, 11527, 11528, 11532, 11533, 11534, 11535, 11536, 11537, + 11573, 11571, 11572, 11574, 11575, 11576, 11577, 11578, 11579, 11580, + 11581, 11544, 11538, 11539, 11545, 11546, 11547, 11548, 11549, 11540, + 11541, 11542, 11543, 11550, 11557, 11558, 11559, 11560, 11561, 11551, + 11552, 11553, 11554, 11555, 11556, 11562, 11563, 11568, 11564, 11565, + 11566, 11567, 11569, 11570, 11588, 11589, 11590, 11591, 11592, 11586, + 11587, 11583, 11584, 11585, 11593, 11594, 11597, 11595, 11596, 11598, + 11599, 11600, 11601, 11602, 11603, 11604, 11605, 11630, 11631, 11634, + 11635, 11636, 11637, 11638, 11632, 11633, 11639, 11640, 11641, 11610, + 11611, 11612, 11613, 11614, 11615, 11606, 11607, 11608, 11609, 11616, + 11617, 11622, 11623, 11624, 11618, 11619, 11625, 11620, 11621, 11626, + 11627, 11628, 11629, 11642, 11643, 11644, 11645, 11646, 11649, 11650, + 11651, 11652, 11653, 11647, 11648, 11654, 11655, 11663, 11664, 11665, + 11666, 11659, 11660, 11667, 11668, 11669, 11661, 11662, 11670, 11671, + 11672, 11673, 11674, 11675, 11676, 11677, 12318, 12319, 12320, 12321, + 12322, 12323, 12324, 12325, 11689, 11685, 11686, 11690, 11691, 11692, + 11687, 11688, 11693, 11694, 11696, 11697, 11698, 11701, 11699, 11700, + 11702, 11703, 11704, 11705, 11706, 11707, 11717, 11718, 11725, 11708, + 11709, 11710, 11711, 11712, 11713, 11714, 11715, 11716, 11726, 11727, + 11719, 11720, 11721, 11722, 11723, 11724, 11728, 11729, 11736, 11737, + 11730, 11731, 11738, 11732, 11733, 11739, 11740, 11741, 11734, 11735, + 11742, 11748, 11746, 11747, 11749, 11743, 11744, 11745, 11750, 11751, + 11752, 11753, 11754, 11755, 11756, 11757, 11758, 11759, 11760, 11761, + 11818, 11819, 11820, 11821, 11822, 11823, 11824, 11825, 11826, 11781, + 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11778, 11779, 11780, + 11789, 11806, 11807, 11808, 11809, 11810, 11804, 11805, 11811, 11812, + 11813, 11814, 11798, 11799, 11800, 11790, 11791, 11792, 11793, 11794, + 11795, 11801, 11796, 11797, 11802, 11803, 11815, 11816, 11817, 11829, + 11830, 11831, 11832, 11827, 11828, 11833, 11834, 11835, 11836, 11839, + 11840, 11841, 11842, 11843, 11844, 11845, 11837, 11838, 11846, 11847, + 11848, 11866, 11867, 11868, 11869, 11870, 11871, 11872, 11873, 11874, + 11849, 11850, 11851, 11852, 11855, 11856, 11857, 11858, 11859, 11860, + 11853, 11854, 11861, 11864, 11865, 11862, 11863, 11882, 11883, 11886, + 11887, 11888, 11884, 11885, 11875, 11876, 11877, 11878, 11879, 11880, + 11881, 11889, 11890, 11891, 11892, 11893, 11894, 11895, 11898, 11899, + 11900, 11901, 11902, 11903, 11904, 11905, 11896, 11897, 11906, 11907, + 11914, 11915, 11916, 11908, 11909, 11910, 11911, 11917, 11918, 11919, + 11912, 11913, 11925, 11926, 11929, 11930, 11927, 11928, 11931, 11932, + 11920, 11921, 11922, 11923, 11924, 11933, 11934, 11935, 11940, 11941, + 11942, 11943, 11944, 11945, 11946, 11947, 11948, 11949, 11936, 11937, + 11938, 11939, 11951, 11952, 11955, 11953, 11954, 11956, 11957, 11958, + 11959, 11960, 11961, 11962, 11963, 12326, 12327, 12328, 12329, 12330, + 12331, 12332, 11969, 11967, 11968, 11964, 11965, 11966, 11970, 11971, + 11972, 11973, 11974, 11975, 11976, 11977, 11980, 11981, 11982, 11983, + 11984, 11978, 11979, 11985, 11986, 11987, 11988, 11989, 11990, 11991, + 11992, 11993, 11994, 11995, 11996, 11997, 12002, 11998, 11999, 12003, + 12004, 12005, 12000, 12001, 12006, 12007, 12008, 12014, 12015, 12016, + 12017, 12009, 12010, 12011, 12018, 12019, 12012, 12013, 12020, 12021, + 12025, 12026, 12027, 12028, 12029, 12030, 12022, 12023, 12024, 12031, + 12032, 12033, 12036, 12037, 12038, 12039, 12040, 12034, 12035, 12041, + 12042, 12043, 12044, 12045, 12046, 12047, 12048, 12049, 12050, 12051, + 12060, 12061, 12052, 12053, 12062, 12063, 12064, 12054, 12055, 12056, + 12057, 12058, 12059, 12069, 12065, 12066, 12070, 12071, 12072, 12073, + 12067, 12068, 12074, 12075, 12076, 12086, 12087, 12088, 12089, 12090, + 12091, 12092, 12093, 12094, 12095, 12081, 12082, 12077, 12078, 12079, + 12080, 12083, 12084, 12085, 12100, 12101, 12102, 12103, 12104, 12097, + 12098, 12099, 12105, 12106, 12107, 12134, 12135, 12136, 12137, 12138, + 12139, 12140, 12141, 12142, 12143, 12112, 12113, 12114, 12108, 12109, + 12115, 12116, 12117, 12118, 12119, 12110, 12111, 12122, 12123, 12120, + 12121, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, + 12133, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155, + 12156, 12157, 12158, 12159, 12160, 12161, 12162, 12144, 12145, 12146, + 12163, 12164, 12173, 12165, 12166, 12167, 12168, 12170, 12171, 12172, + 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, + 12184, 12185, 12186, 12187, 12188, 12189, 12190, 12191, 12192, 12193, + 12200, 12201, 12194, 12195, 12202, 12203, 12204, 12205, 12196, 12197, + 12198, 12199, 12206, 12207, 12208, 12209, 12214, 12210, 12211, 12215, + 12216, 12217, 12212, 12213, 12218, 12219, 12220, 12221, 12227, 12228, + 12223, 12224, 12229, 12230, 12231, 12232, 12233, 12225, 12226, 12234, + 12235, 12242, 12243, 12244, 12236, 12237, 12245, 12246, 12238, 12239, + 12240, 12241, 12247, 12250, 12251, 12252, 12253, 12248, 12249, 12254, + 12263, 12264, 12265, 12256, 12257, 12258, 12266, 12259, 12260, 12267, + 12261, 12262, 12268, 12269, 12270, 12271, 12272, 12273, 12274, 12275, + 12276, 12289, 12277, 12278, 12279, 12280, 12281, 12282, 12283, 12284, + 12285, 12286, 12287, 12288, 12290, 12291, 12292, 12293, 12313, 12314, + 12315, 12316, 12317, 12294, 12295, 12296, 12297, 12298, 12299, 12300, + 12301, 12302, 12303, 12304, 12305, 12306, 12307, 12308, 12309, 12310, + 12311, 12312, 11306, 11307, 11308, 11309, 11310, 11311, 11303, 11304, + 11305, 11312, 11313, 11317, 11318, 11319, 11320, 11321, 11322, 11323, + 11324, 11325, 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, + 11334, 11335, 11336, 11314, 11315, 11316, 12169, 12222, 11658, 11683, + 11680, 11682, 11679, 11950, 11337, 11514, 11684, 11681, 11678, 11339, + 11516, 11338, 11515, 11777, 11582, 11656, 11695, 11657, 12096, 12255, + 11773, 11764, 11768, 11775, 11771, 11765, 11770, 11767, 11774, 11763, + 11769, 11776, 11772, 11766, 11762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 365, + 366, 367, 368, 369, 370, 363, 364, 371, 372, 373, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 413, 414, 381, 377, 378, 382, 383, 384, 379, + 380, 374, 375, 376, 385, 386, 387, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 392, 393, 394, 395, 396, 397, 388, 389, 390, 391, 398, + 399, 400, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, + 466, 467, 468, 469, 470, 471, 472, 473, 405, 406, 407, 408, 409, 410, + 411, 401, 402, 403, 404, 412, 485, 486, 487, 488, 489, 490, 491, 474, + 475, 476, 477, 482, 483, 484, 492, 478, 479, 480, 481, 493, 494, 495, + 496, 497, 500, 501, 502, 503, 498, 499, 504, 505, 506, 507, 510, 511, + 512, 513, 514, 508, 509, 515, 516, 517, 518, 521, 522, 523, 524, 525, + 519, 520, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, + 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, + 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, + 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, + 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 597, 598, 590, 591, + 592, 599, 600, 601, 602, 593, 594, 603, 595, 596, 608, 609, 610, 611, + 612, 604, 605, 606, 607, 613, 614, 615, 641, 642, 643, 644, 645, 646, + 647, 639, 640, 648, 649, 661, 662, 663, 664, 665, 666, 667, 668, 669, + 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 686, 687, 688, 689, 690, 652, 653, 654, 655, 656, 657, 658, + 650, 651, 659, 660, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, + 630, 631, 632, 633, 634, 635, 636, 637, 638, 628, 629, 620, 621, 622, + 623, 616, 617, 624, 625, 626, 627, 618, 619, 703, 704, 705, 706, 707, + 708, 709, 710, 711, 701, 702, 795, 796, 797, 798, 799, 800, 801, 802, + 803, 804, 714, 715, 716, 717, 718, 719, 720, 721, 722, 712, 713, 741, + 742, 738, 739, 740, 743, 744, 745, 734, 735, 736, 737, 746, 747, 748, + 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 725, 726, 727, 728, + 729, 730, 731, 732, 733, 723, 724, 753, 754, 755, 756, 749, 750, 757, + 758, 759, 751, 752, 760, 786, 784, 785, 787, 788, 789, 790, 791, 792, + 793, 794, 767, 763, 764, 768, 761, 762, 769, 770, 765, 766, 771, 772, + 773, 775, 776, 777, 774, 778, 779, 780, 781, 782, 783, 846, 847, 848, + 849, 850, 851, 852, 853, 854, 855, 815, 816, 817, 818, 819, 820, 821, + 822, 823, 824, 825, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, + 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, + 880, 881, 882, 883, 884, 885, 826, 827, 830, 831, 832, 833, 834, 835, + 828, 829, 836, 837, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, + 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, + 910, 911, 912, 913, 914, 915, 838, 839, 840, 841, 842, 843, 844, 845, + 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, + 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, + 945, 916, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 2827, 2794, 2852, 2853, 2823, + 2860, 2871, 2842, 2859, 2854, 2855, 2807, 2872, 2829, 2808, 2813, 2820, + 2866, 2838, 2797, 2832, 2867, 2828, 2804, 2805, 2836, 2810, 2851, 2793, + 2850, 2819, 2843, 2874, 2792, 2837, 2868, 2822, 2821, 2817, 2795, 2849, + 2826, 2856, 2809, 2846, 2857, 2873, 2802, 2863, 2870, 2811, 2798, 2825, + 2800, 2818, 2861, 2803, 2876, 2878, 2799, 2864, 2814, 2858, 2835, 2862, + 2840, 2847, 2831, 2875, 2830, 2812, 2844, 2815, 2841, 2869, 2865, 2848, + 2833, 2806, 2839, 2796, 2834, 2877, 2801, 2845, 2824, 2816, 2910, 2926, + 2924, 2923, 2889, 2895, 2884, 2930, 2894, 2886, 2916, 2929, 2888, 2914, + 2915, 2879, 2919, 2927, 2921, 2922, 2901, 2898, 2913, 2882, 2880, 2881, + 2887, 2917, 2934, 2908, 2906, 2931, 2920, 2928, 2900, 2904, 2905, 2902, + 2925, 2897, 2903, 2912, 2918, 2899, 2932, 2896, 2933, 2883, 2893, 2892, + 2890, 2907, 2911, 2891, 2885, 2909, 2986, 3006, 3027, 3023, 2985, 2974, + 2987, 2935, 2963, 2937, 3007, 3003, 2960, 3008, 2978, 2957, 2940, 2936, + 2942, 3026, 3005, 2968, 2998, 2966, 3028, 2947, 2948, 3012, 2979, 3016, + 2994, 3020, 3015, 2981, 3022, 2972, 2954, 3004, 2982, 2950, 2965, 2970, + 3001, 3017, 2984, 3031, 2975, 3000, 2997, 3030, 3021, 2962, 3032, 2990, + 2949, 3019, 2995, 2992, 2944, 2980, 2956, 2967, 2952, 2946, 2991, 2989, + 3024, 2983, 2999, 3002, 2945, 2996, 2976, 2953, 3029, 2977, 3013, 3011, + 2964, 2955, 2959, 2941, 2961, 2943, 2958, 3025, 2993, 2971, 2969, 3014, + 2939, 2938, 2988, 2973, 2951, 3009, 3010, 3018, 3060, 3144, 3093, 3067, + 3094, 3053, 3092, 3098, 3080, 3107, 3143, 3089, 3123, 3090, 3037, 3136, + 3121, 3096, 3128, 3041, 3145, 3043, 3130, 3065, 3075, 3056, 3062, 3132, + 3149, 3091, 3038, 3086, 3138, 3036, 3088, 3033, 3076, 3070, 3048, 3077, + 3072, 3071, 3113, 3069, 3066, 3054, 3099, 3058, 3046, 3105, 3134, 3133, + 3146, 3039, 3120, 3137, 3085, 3064, 3100, 3040, 3116, 3111, 3106, 3051, + 3079, 3068, 3083, 3147, 3115, 3148, 3078, 3102, 3127, 3044, 3082, 3087, + 3139, 3061, 3045, 3101, 3135, 3057, 3081, 3049, 3084, 3097, 3095, 3035, + 3042, 3117, 3141, 3140, 3108, 3119, 3050, 3063, 3055, 3129, 3074, 3125, + 3122, 3047, 3110, 3126, 3103, 3112, 3109, 3124, 3114, 3073, 3052, 3118, + 3142, 3104, 3059, 3131, 3034, 3203, 3280, 3190, 3177, 3287, 3180, 3243, + 3269, 3259, 3229, 3206, 3254, 3274, 3215, 3170, 3290, 3262, 3208, 3244, + 3279, 3172, 3182, 3231, 3221, 3187, 3218, 3222, 3230, 3261, 3228, 3247, + 3270, 3211, 3194, 3164, 3217, 3225, 3188, 3181, 3209, 3205, 3271, 3263, + 3257, 3202, 3210, 3295, 3163, 3284, 3291, 3251, 3283, 3167, 3268, 3277, + 3285, 3288, 3176, 3253, 3273, 3161, 3223, 3264, 3193, 3191, 3236, 3240, + 3160, 3282, 3171, 3303, 3234, 3296, 3192, 3204, 3249, 3299, 3179, 3153, + 3159, 3220, 3169, 3186, 3216, 3165, 3155, 3232, 3245, 3293, 3207, 3235, + 3237, 3252, 3196, 3154, 3239, 3197, 3162, 3152, 3200, 3255, 3219, 3173, + 3250, 3233, 3292, 3212, 3241, 3151, 3158, 3226, 3304, 3281, 3306, 3305, + 3178, 3242, 3272, 3275, 3201, 3267, 3294, 3213, 3300, 3297, 3298, 3302, + 3301, 3168, 3246, 3227, 3256, 3289, 3157, 3286, 3184, 3195, 3260, 3258, + 3214, 3224, 3265, 3266, 3150, 3238, 3248, 3183, 3175, 3198, 3185, 3189, + 3276, 3174, 3199, 3278, 3156, 3166, 3311, 3360, 3313, 3358, 3338, 3351, + 3332, 3315, 3341, 3340, 3320, 3350, 3330, 3326, 3317, 3348, 3343, 3349, + 3346, 3309, 3308, 3328, 3327, 3325, 3353, 3345, 3354, 3329, 3334, 3331, + 3355, 3335, 3342, 3333, 3337, 3307, 3323, 3324, 3344, 3336, 3359, 3356, + 3316, 3314, 3312, 3318, 3339, 3321, 3322, 3319, 3352, 3310, 3347, 3357, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23535, 23527, 23548, + 23525, 23549, 23536, 23551, 23531, 23522, 23539, 23537, 23547, 23521, + 23529, 23524, 23526, 23532, 23530, 23528, 23541, 23544, 23533, 23543, + 23550, 23542, 23523, 23546, 23545, 23538, 23540, 23534, 35762, 23560, + 23562, 23559, 23558, 23555, 23554, 23557, 23556, 23563, 23561, 35762, + 35762, 35762, 35762, 23553, 23552, 30867, 30864, 30865, 30866, 30819, + 30816, 30817, 30818, 30871, 30868, 30869, 30870, 30859, 30856, 30857, + 30858, 30863, 30860, 30861, 30862, 30855, 30852, 30853, 30854, 30815, + 30812, 30813, 30814, 30847, 30844, 30845, 30846, 30820, 30825, 30836, + 30837, 30848, 30851, 30849, 30850, 30843, 30840, 30841, 30842, 30831, + 30828, 30829, 30830, 30882, 30881, 30880, 30832, 30839, 30889, 30887, + 30884, 30834, 30883, 30885, 30827, 30835, 30824, 30826, 30823, 30874, + 30878, 30886, 30833, 30838, 30876, 30873, 30879, 30822, 30872, 30888, + 30821, 30877, 30875, 30890, 35762, 30897, 30899, 30896, 30895, 30892, + 30891, 30894, 30893, 30900, 30898, 35762, 35762, 35762, 35762, 35762, + 35762, 3421, 3426, 3442, 3444, 3434, 3432, 3424, 3418, 3425, 3438, 3433, + 3429, 3440, 3423, 3419, 3441, 3428, 3439, 3443, 3437, 3431, 3445, 3430, + 3446, 3435, 3436, 3427, 3422, 3420, 3447, 35762, 35762, 3414, 3416, 3417, + 3415, 3413, 3448, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 26200, 26201, 26206, 26207, 26194, 26195, 26210, 26211, + 26202, 26203, 26192, 26193, 26212, 26213, 26196, 26197, 26208, 26209, + 26214, 26215, 26204, 26205, 26198, 26199, 26216, 26217, 26190, 26191, + 26136, 26127, 26133, 26124, 26129, 26135, 26128, 26132, 26138, 26122, + 26134, 26120, 26125, 26123, 26131, 26126, 26130, 26139, 26137, 26121, + 26144, 26143, 26141, 26140, 26142, 26146, 26145, 26176, 26177, 26155, + 26175, 26173, 26181, 26183, 26182, 26184, 26174, 26166, 26179, 26164, + 26186, 26159, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 26224, 26226, 26223, 26222, 26219, 26218, 26221, 26220, + 26227, 26225, 35762, 26151, 26148, 26150, 26153, 26147, 26149, 26152, + 35762, 26178, 26185, 26163, 26171, 26187, 26162, 26169, 26180, 26168, + 26189, 26170, 26165, 26172, 26188, 26167, 26161, 26154, 26157, 26158, + 26160, 26156, 35762, 35762, 35762, 35762, 35762, 26108, 26115, 26107, + 26106, 26116, 26104, 26101, 26119, 26111, 26109, 26117, 26103, 26102, + 26112, 26118, 26114, 26110, 26105, 26113, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 22208, 22212, 22215, 22216, 22186, 22218, 22194, 22209, + 22213, 22204, 22203, 22205, 22193, 22185, 22206, 22202, 22199, 22200, + 22214, 22196, 22207, 22210, 22192, 22190, 22217, 22201, 22198, 22188, + 22211, 22197, 22187, 22195, 22266, 22270, 22273, 22274, 22244, 22276, + 22252, 22267, 22271, 22262, 22261, 22263, 22251, 22243, 22264, 22260, + 22257, 22258, 22272, 22254, 22265, 22268, 22250, 22248, 22275, 22259, + 22256, 22246, 22269, 22255, 22245, 22253, 22230, 22224, 22222, 22220, + 22227, 22226, 22229, 22228, 22232, 22231, 22235, 22237, 22234, 22233, + 22238, 22239, 22241, 22242, 22236, 22240, 22225, 22223, 22221, 22219, + 22279, 22277, 22278, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 22822, 22748, 22810, 22821, 22826, 22825, + 22745, 22827, 22802, 22801, 22799, 22756, 22805, 22806, 22798, 22755, + 22764, 22772, 22808, 22744, 22769, 22768, 22763, 22762, 22761, 22760, + 22786, 22754, 22785, 22753, 22828, 22759, 22809, 22820, 22819, 22767, + 22766, 22743, 22824, 22830, 22758, 22757, 22795, 22751, 22771, 22770, + 22794, 22750, 22803, 22807, 22779, 22782, 22783, 22817, 22815, 22796, + 22752, 22804, 22784, 22818, 22816, 22814, 22812, 22742, 22813, 22811, + 22829, 22746, 22823, 22747, 22781, 22749, 22800, 22797, 22780, 35762, + 35762, 35762, 35762, 22834, 22765, 22833, 22832, 22831, 22839, 22845, + 22844, 22840, 22841, 22866, 22870, 22887, 22886, 22848, 22851, 22852, + 22868, 22855, 22856, 22857, 22858, 22859, 22862, 22864, 22865, 22861, + 22872, 22873, 22874, 22875, 22880, 22876, 22877, 22881, 22883, 22842, + 22843, 22850, 22885, 22849, 22884, 22846, 22854, 22847, 22871, 22888, + 22889, 22878, 22882, 22869, 22867, 22890, 22863, 22853, 22860, 22879, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 22838, 22836, 22837, + 22835, 22787, 22788, 22789, 22790, 22791, 22792, 22793, 22773, 22774, + 22775, 22776, 22777, 22778, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31669, 25140, + 25317, 25316, 17699, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 33692, 33691, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 30901, 30902, 30903, 30904, 30905, 30906, 30907, 30908, 30909, 30910, + 30911, 30912, 30913, 30914, 30915, 30916, 30917, 30918, 30919, 30920, + 30921, 30922, 30923, 30924, 30925, 30926, 30927, 30928, 30929, 30930, + 30931, 30932, 30933, 30934, 30935, 30936, 30937, 30938, 30939, 30940, + 30941, 30942, 30943, 30944, 30945, 30946, 30947, 30948, 30949, 30950, + 30951, 30952, 30953, 30954, 30955, 30956, 30957, 30958, 30959, 30960, + 30961, 30962, 30963, 30964, 30965, 30966, 30967, 30968, 30969, 30970, + 30971, 30972, 30973, 30974, 30975, 30976, 30977, 30978, 30979, 30980, + 30981, 30982, 30983, 30984, 30985, 30986, 30987, 30988, 30989, 30990, + 30991, 30992, 30993, 30994, 30995, 30996, 30997, 30998, 30999, 31000, + 31001, 31002, 31003, 31004, 31005, 31006, 31007, 31008, 31009, 31010, + 31011, 31012, 31013, 31014, 31015, 31016, 31017, 31018, 31019, 31020, + 31021, 31022, 31023, 31024, 31025, 31026, 31027, 31028, 31029, 31030, + 31031, 31032, 31033, 31034, 31035, 31036, 31037, 31038, 31039, 31040, + 31041, 31042, 31043, 31044, 31045, 31046, 31047, 31048, 31049, 31050, + 31051, 31052, 31053, 31054, 31055, 31056, 31057, 31058, 31059, 31060, + 31061, 31062, 31063, 31064, 31065, 31066, 31067, 31068, 31069, 31070, + 31071, 31072, 31073, 31074, 31075, 31076, 31077, 31078, 31079, 31080, + 31081, 31082, 31083, 31084, 31085, 31086, 31087, 31088, 31089, 31090, + 31091, 31092, 31093, 31094, 31095, 31096, 31097, 31098, 31099, 31100, + 31101, 31102, 31103, 31104, 31105, 31106, 31107, 31108, 31109, 31110, + 31111, 31112, 31113, 31114, 31115, 31116, 31117, 31118, 31119, 31120, + 31121, 31122, 31123, 31124, 31125, 31126, 31127, 31128, 31129, 31130, + 31131, 31132, 31133, 31134, 31135, 31136, 31137, 31138, 31139, 31140, + 31141, 31142, 31143, 31144, 31145, 31146, 31147, 31148, 31149, 31150, + 31151, 31152, 31153, 31154, 31155, 31156, 31157, 31158, 31159, 31160, + 31161, 31162, 31163, 31164, 31165, 31166, 31167, 31168, 31169, 31170, + 31171, 31172, 31173, 31174, 31175, 31176, 31177, 31178, 31179, 31180, + 31181, 31182, 31183, 31184, 31185, 31186, 31187, 31188, 31189, 31190, + 31191, 31192, 31193, 31194, 31195, 31196, 31197, 31198, 31199, 31200, + 31201, 31202, 31203, 31204, 31205, 31206, 31207, 31208, 31209, 31210, + 31211, 31212, 31213, 31214, 31215, 31216, 31217, 31218, 31219, 31220, + 31221, 31222, 31223, 31224, 31225, 31226, 31227, 31228, 31229, 31230, + 31231, 31232, 31233, 31234, 31235, 31236, 31237, 31238, 31239, 31240, + 31241, 31242, 31243, 31244, 31245, 31246, 31247, 31248, 31249, 31250, + 31251, 31252, 31253, 31254, 31255, 31256, 31257, 31258, 31259, 31260, + 31261, 31262, 31263, 31264, 31265, 31266, 31267, 31268, 31269, 31270, + 31271, 31272, 31273, 31274, 31275, 31276, 31277, 31278, 31279, 31280, + 31281, 31282, 31283, 31284, 31285, 31286, 31287, 31288, 31289, 31290, + 31291, 31292, 31293, 31294, 31295, 31296, 31297, 31298, 31299, 31300, + 31301, 31302, 31303, 31304, 31305, 31306, 31307, 31308, 31309, 31310, + 31311, 31312, 31313, 31314, 31315, 31316, 31317, 31318, 31319, 31320, + 31321, 31322, 31323, 31324, 31325, 31326, 31327, 31328, 31329, 31330, + 31331, 31332, 31333, 31334, 31335, 31336, 31337, 31338, 31339, 31340, + 31341, 31342, 31343, 31344, 31345, 31346, 31347, 31348, 31349, 31350, + 31351, 31352, 31353, 31354, 31355, 31356, 31357, 31358, 31359, 31360, + 31361, 31362, 31363, 31364, 31365, 31366, 31367, 31368, 31369, 31370, + 31371, 31372, 31373, 31374, 31375, 31376, 31377, 31378, 31379, 31380, + 31381, 31382, 31383, 31384, 31385, 31386, 31387, 31388, 31389, 31390, + 31391, 31392, 31393, 31394, 31395, 31396, 31397, 31398, 31399, 31400, + 31401, 31402, 31403, 31404, 31405, 31406, 31407, 31408, 31409, 31410, + 31411, 31412, 31413, 31414, 31415, 31416, 31417, 31418, 31419, 31420, + 31421, 31422, 31423, 31424, 31425, 31426, 31427, 31428, 31429, 31430, + 31431, 31432, 31433, 31434, 31435, 31436, 31437, 31438, 31439, 31440, + 31441, 31442, 31443, 31444, 31445, 31446, 31447, 31448, 31449, 31450, + 31451, 31452, 31453, 31454, 31455, 31456, 31457, 31458, 31459, 31460, + 31461, 31462, 31463, 31464, 31465, 31466, 31467, 31468, 31469, 31470, + 31471, 31472, 31473, 31474, 31475, 31476, 31477, 31478, 31479, 31480, + 31481, 31482, 31483, 31484, 31485, 31486, 31487, 31488, 31489, 31490, + 31491, 31492, 31493, 31494, 31495, 31496, 31497, 31498, 31499, 31500, + 31501, 31502, 31503, 31504, 31505, 31506, 31507, 31508, 31509, 31510, + 31511, 31512, 31513, 31514, 31515, 31516, 31517, 31518, 31519, 31520, + 31521, 31522, 31523, 31524, 31525, 31526, 31527, 31528, 31529, 31530, + 31531, 31532, 31533, 31534, 31535, 31536, 31537, 31538, 31539, 31540, + 31541, 31542, 31543, 31544, 31545, 31546, 31547, 31548, 31549, 31550, + 31551, 31552, 31553, 31554, 31555, 31556, 31557, 31558, 31559, 31560, + 31561, 31562, 31563, 31564, 31565, 31566, 31567, 31568, 31569, 31570, + 31571, 31572, 31573, 31574, 31575, 31576, 31577, 31578, 31579, 31580, + 31581, 31582, 31583, 31584, 31585, 31586, 31587, 31588, 31589, 31590, + 31591, 31592, 31593, 31594, 31595, 31596, 31597, 31598, 31599, 31600, + 31601, 31602, 31603, 31604, 31605, 31606, 31607, 31608, 31609, 31610, + 31611, 31612, 31613, 31614, 31615, 31616, 31617, 31618, 31619, 31620, + 31621, 31622, 31623, 31624, 31625, 31626, 31627, 31628, 31629, 31630, + 31631, 31632, 31633, 31634, 31635, 31636, 31637, 31638, 31639, 31640, + 31641, 31642, 31643, 31644, 31645, 31646, 31647, 31648, 31649, 31650, + 31651, 31652, 31653, 31654, 31655, 31656, 31657, 31658, 31659, 31660, + 31661, 31662, 31663, 31664, 31665, 31666, 31667, 31668, 17229, 17230, + 17231, 17232, 17233, 17234, 17235, 17236, 17237, 17238, 17239, 17240, + 17241, 17242, 17243, 17244, 17245, 17246, 17247, 17248, 17249, 17250, + 17251, 17252, 17253, 17254, 17255, 17256, 17257, 17258, 17259, 17260, + 17261, 17262, 17263, 17264, 17265, 17266, 17267, 17268, 17269, 17270, + 17271, 17272, 17273, 17274, 17275, 17276, 17277, 17278, 17279, 17280, + 17281, 17282, 17283, 17284, 17285, 17286, 17287, 17288, 17289, 17290, + 17291, 17292, 17293, 17294, 17295, 17296, 17297, 17298, 17299, 17300, + 17301, 17302, 17303, 17304, 17305, 17306, 17307, 17308, 17309, 17310, + 17311, 17312, 17313, 17314, 17315, 17316, 17317, 17318, 17319, 17320, + 17321, 17322, 17323, 17324, 17325, 17326, 17327, 17328, 17329, 17330, + 17331, 17332, 17333, 17334, 17335, 17336, 17337, 17338, 17339, 17340, + 17341, 17342, 17343, 17344, 17345, 17346, 17347, 17348, 17349, 17350, + 17351, 17352, 17353, 17354, 17355, 17356, 17357, 17358, 17359, 17360, + 17361, 17362, 17363, 17364, 17365, 17366, 17367, 17368, 17369, 17370, + 17371, 17372, 17373, 17374, 17375, 17376, 17377, 17378, 17379, 17380, + 17381, 17382, 17383, 17384, 17385, 17386, 17387, 17388, 17389, 17390, + 17391, 17392, 17393, 17394, 17395, 17396, 17397, 17398, 17399, 17400, + 17401, 17402, 17403, 17404, 17405, 17406, 17407, 17408, 17409, 17410, + 17411, 17412, 17413, 17414, 17415, 17416, 17417, 17418, 17419, 17420, + 17421, 17422, 17423, 17424, 17425, 17426, 17427, 17428, 17429, 17430, + 17431, 17432, 17433, 17434, 17435, 17436, 17437, 17438, 17439, 17440, + 17441, 17442, 17443, 17444, 17445, 17446, 17447, 17448, 17449, 17450, + 17451, 17452, 17453, 17454, 17455, 17456, 17457, 17458, 17459, 17460, + 17461, 17462, 17463, 17464, 17465, 17466, 17467, 17468, 17469, 17470, + 17471, 17472, 17473, 17474, 17475, 17476, 17477, 17478, 17479, 17480, + 17481, 17482, 17483, 17484, 17491, 17492, 17493, 17494, 17495, 17496, + 17497, 17498, 17499, 17500, 17501, 17502, 17503, 17504, 17505, 17506, + 17507, 17508, 17509, 17510, 17511, 17512, 17513, 17514, 17515, 17516, + 17517, 17518, 17519, 17520, 17521, 17522, 17523, 17524, 17525, 17526, + 17527, 17528, 17529, 17530, 17531, 17532, 17533, 17534, 17535, 17536, + 17537, 17538, 17539, 17540, 17541, 17542, 17543, 17544, 17545, 17546, + 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, 17555, 17556, + 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, 17565, 17566, + 17567, 17568, 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, + 17577, 17578, 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, + 17587, 17588, 17589, 17590, 17591, 17592, 17593, 17594, 17595, 17596, + 17597, 17598, 17599, 17600, 17601, 17602, 17603, 17604, 17605, 17606, + 17607, 17608, 17609, 17610, 17611, 17612, 17613, 17614, 17615, 17616, + 17617, 17618, 17619, 17620, 17621, 17622, 17623, 17624, 17625, 17626, + 17627, 17628, 17629, 17630, 17631, 17632, 17633, 17634, 17635, 17636, + 17637, 17638, 17639, 17640, 17641, 17642, 17643, 17644, 17645, 17646, + 17647, 17648, 17649, 17650, 17651, 17652, 17653, 17654, 17655, 17656, + 17657, 17658, 17659, 17660, 17661, 17662, 17663, 17664, 17665, 17666, + 17667, 17668, 17669, 17670, 17671, 17672, 17673, 17674, 17675, 17676, + 17677, 17678, 17679, 17680, 17681, 17682, 17683, 17684, 17685, 17686, + 17687, 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17696, + 17697, 17698, 17485, 17486, 17487, 17488, 17489, 17490, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 16870, 16871, 16872, 16873, 35762, 16874, 16875, 16863, 16864, 16865, + 16866, 16867, 35762, 16868, 16869, 35762, 16851, 15837, 15476, 15477, + 15478, 15475, 15757, 15758, 15759, 15760, 15749, 15750, 15751, 15752, + 15753, 15744, 15745, 15746, 15747, 15748, 15754, 15755, 15756, 15515, + 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, 15516, 15517, + 15518, 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15545, + 15546, 15547, 15548, 15549, 15550, 15551, 15539, 15540, 15541, 15542, + 15543, 15544, 15528, 15529, 15530, 15527, 15640, 15641, 15642, 15643, + 15644, 15645, 15646, 15647, 15656, 15657, 15658, 15659, 15660, 15661, + 15648, 15649, 15650, 15651, 15652, 15653, 15654, 15655, 15662, 15663, + 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, + 15702, 15703, 15704, 15705, 15695, 15696, 15697, 15698, 15699, 15700, + 15701, 15691, 15692, 15693, 15694, 15690, 15674, 15675, 15676, 15677, + 15678, 15679, 15680, 15681, 15682, 15684, 15685, 15686, 15687, 15688, + 15689, 15683, 15594, 15595, 15596, 15597, 15598, 15599, 15600, 15601, + 15602, 15587, 15588, 15589, 15590, 15591, 15592, 15593, 15586, 15608, + 15609, 15610, 15580, 15581, 15582, 15583, 15584, 15585, 15579, 15603, + 15604, 15605, 15606, 15607, 15487, 15490, 15491, 15492, 15493, 15494, + 15495, 15496, 15497, 15488, 15489, 15508, 15509, 15510, 15511, 15512, + 15513, 15514, 15498, 15499, 15500, 15501, 15502, 15503, 15504, 15505, + 15506, 15507, 15479, 15480, 15481, 15482, 15483, 15484, 15485, 15486, + 15561, 15562, 15563, 15564, 15565, 15566, 15567, 15568, 15569, 15570, + 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15553, 15554, + 15552, 15555, 15556, 15557, 15558, 15559, 15560, 15728, 15729, 15730, + 15731, 15732, 15727, 15739, 15740, 15741, 15742, 15733, 15734, 15735, + 15736, 15737, 15738, 15632, 15633, 15634, 15635, 15625, 15626, 15627, + 15628, 15629, 15630, 15631, 15619, 15620, 15621, 15622, 15623, 15624, + 15636, 15637, 15638, 15639, 15613, 15614, 15615, 15616, 15617, 15618, + 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, + 15723, 15724, 15725, 15726, 15716, 15717, 15718, 15719, 15720, 15721, + 15722, 15611, 15612, 15836, 16849, 16848, 16850, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 15847, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 15840, 15839, 15841, 35762, 35762, 16898, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 16904, 16903, 16905, 16909, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 24744, 24745, 24746, 24747, 24748, 24749, + 24750, 24751, 24752, 24753, 24754, 24755, 24756, 24757, 24758, 24759, + 24760, 24761, 24762, 24763, 24764, 24765, 24766, 24767, 24768, 24769, + 24770, 24771, 24772, 24773, 24774, 24775, 24776, 24777, 24778, 24779, + 24780, 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, + 24790, 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, + 24800, 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, + 24810, 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24819, + 24820, 24821, 24822, 24823, 24824, 24825, 24826, 24827, 24828, 24829, + 24830, 24831, 24832, 24833, 24834, 24835, 24836, 24837, 24838, 24839, + 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24847, 24848, 24849, + 24850, 24851, 24852, 24853, 24854, 24855, 24856, 24857, 24858, 24859, + 24860, 24861, 24862, 24863, 24864, 24865, 24866, 24867, 24868, 24869, + 24870, 24871, 24872, 24873, 24874, 24875, 24876, 24877, 24878, 24879, + 24880, 24881, 24882, 24883, 24884, 24885, 24886, 24887, 24888, 24889, + 24890, 24891, 24892, 24893, 24894, 24895, 24896, 24897, 24898, 24899, + 24900, 24901, 24902, 24903, 24904, 24905, 24906, 24907, 24908, 24909, + 24910, 24911, 24912, 24913, 24914, 24915, 24916, 24917, 24918, 24919, + 24920, 24921, 24922, 24923, 24924, 24925, 24926, 24927, 24928, 24929, + 24930, 24931, 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24939, + 24940, 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, + 24950, 24951, 24952, 24953, 24954, 24955, 24956, 24957, 24958, 24959, + 24960, 24961, 24962, 24963, 24964, 24965, 24966, 24967, 24968, 24969, + 24970, 24971, 24972, 24973, 24974, 24975, 24976, 24977, 24978, 24979, + 24980, 24981, 24982, 24983, 24984, 24985, 24986, 24987, 24988, 24989, + 24990, 24991, 24992, 24993, 24994, 24995, 24996, 24997, 24998, 24999, + 25000, 25001, 25002, 25003, 25004, 25005, 25006, 25007, 25008, 25009, + 25010, 25011, 25012, 25013, 25014, 25015, 25016, 25017, 25018, 25019, + 25020, 25021, 25022, 25023, 25024, 25025, 25026, 25027, 25028, 25029, + 25030, 25031, 25032, 25033, 25034, 25035, 25036, 25037, 25038, 25039, + 25040, 25041, 25042, 25043, 25044, 25045, 25046, 25047, 25048, 25049, + 25050, 25051, 25052, 25053, 25054, 25055, 25056, 25057, 25058, 25059, + 25060, 25061, 25062, 25063, 25064, 25065, 25066, 25067, 25068, 25069, + 25070, 25071, 25072, 25073, 25074, 25075, 25076, 25077, 25078, 25079, + 25080, 25081, 25082, 25083, 25084, 25085, 25086, 25087, 25088, 25089, + 25090, 25091, 25092, 25093, 25094, 25095, 25096, 25097, 25098, 25099, + 25100, 25101, 25102, 25103, 25104, 25105, 25106, 25107, 25108, 25109, + 25110, 25111, 25112, 25113, 25114, 25115, 25116, 25117, 25118, 25119, + 25120, 25121, 25122, 25123, 25124, 25125, 25126, 25127, 25128, 25129, + 25130, 25131, 25132, 25133, 25134, 25135, 25136, 25137, 25138, 25139, + 35762, 35762, 35762, 35762, 11196, 11194, 11143, 11176, 11103, 11116, + 11120, 11201, 11097, 11184, 11105, 11147, 11146, 11098, 11104, 11118, + 11150, 11179, 11172, 11099, 11119, 11173, 11197, 11123, 11151, 11124, + 11129, 11107, 11152, 11125, 11130, 11110, 11153, 11127, 11132, 11108, + 11109, 11161, 11162, 11128, 11133, 11114, 11165, 11126, 11131, 11111, + 11154, 11115, 11112, 11113, 11159, 11160, 11157, 11158, 11178, 11177, + 11186, 11192, 11189, 11164, 11163, 11117, 11106, 11155, 11156, 11095, + 11170, 11140, 11138, 11096, 11198, 11100, 11199, 11175, 11183, 11101, + 11167, 11148, 11166, 11121, 11200, 11180, 11102, 11195, 11181, 11122, + 11149, 11182, 11174, 11139, 11142, 11141, 11191, 11187, 11193, 11190, + 11188, 11137, 11136, 11135, 11134, 11145, 11144, 11168, 11171, 11169, + 11185, 35762, 35762, 35762, 35762, 35762, 11080, 11093, 11092, 11087, + 11094, 11074, 11067, 11066, 11063, 11065, 11068, 11069, 11064, 35762, + 35762, 35762, 11076, 11072, 11075, 11070, 11079, 11078, 11071, 11077, + 11073, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 11081, 11085, + 11088, 11083, 11091, 11090, 11084, 11089, 11086, 11082, 35762, 35762, + 11205, 11202, 11203, 11204, 27932, 27931, 27933, 27934, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35578, 35583, 35617, 35580, 35585, 35613, 35607, 35601, 35624, 35603, + 35597, 35621, 35579, 35584, 35618, 35581, 35586, 35614, 35608, 35602, + 35625, 35604, 35598, 35622, 35619, 35605, 35612, 35599, 35600, 35623, + 35606, 35582, 35628, 35593, 35610, 35620, 35631, 35630, 35596, 35629, + 35590, 35589, 35627, 35611, 35609, 35588, 35762, 35762, 35633, 35634, + 35635, 35626, 35576, 35591, 35594, 35595, 35573, 35574, 35592, 35615, + 35616, 35577, 35632, 35575, 35587, 35572, 35754, 35755, 35752, 35753, + 35756, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35652, 35653, 35669, 35641, 35650, 35749, 35704, 35705, 35672, 35673, + 35706, 35636, 35670, 35750, 35648, 35645, 35647, 35646, 35644, 35746, + 35745, 35748, 35747, 35742, 35741, 35744, 35743, 35642, 35676, 35638, + 35649, 35637, 35674, 35687, 35688, 35686, 35685, 35639, 35683, 35684, + 35682, 35680, 35681, 35679, 35678, 35689, 35691, 35692, 35690, 35654, + 35677, 35643, 35651, 35751, 35693, 35698, 35695, 35699, 35696, 35701, + 35702, 35697, 35694, 35700, 35675, 35703, 35736, 35733, 35724, 35734, + 35735, 35725, 35713, 35712, 35740, 35710, 35709, 35707, 35711, 35708, + 35727, 35732, 35726, 35730, 35731, 35728, 35729, 35656, 35660, 35659, + 35658, 35657, 35739, 35737, 35738, 35663, 35668, 35667, 35666, 35665, + 35664, 35714, 35723, 35716, 35721, 35715, 35719, 35720, 35717, 35718, + 35722, 35655, 35662, 35640, 35661, 35671, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 4893, 4765, 4886, 4868, 4869, + 4937, 4938, 4818, 4915, 4875, 4949, 4950, 4841, 4720, 4771, 4918, 4819, + 4723, 4724, 4913, 4925, 4864, 4801, 4892, 4744, 4941, 4813, 4827, 4823, + 4916, 4878, 4907, 4871, 4940, 4726, 4728, 4828, 4894, 4879, 4936, 4718, + 4896, 4910, 4912, 4866, 4920, 4848, 4766, 4928, 4919, 4838, 4719, 4778, + 4809, 4930, 4816, 4884, 4888, 4831, 4742, 4895, 4873, 4876, 4815, 4953, + 4883, 4832, 4931, 4908, 4807, 4814, 4865, 4870, 4882, 4834, 4881, 4839, + 4885, 4822, 4826, 4952, 4725, 4721, 4951, 4840, 4774, 4745, 4863, 4939, + 4880, 4889, 4872, 4717, 4849, 4877, 4874, 4770, 4842, 4716, 4932, 4773, + 4911, 4914, 4743, 4772, 4897, 4954, 4934, 4890, 4929, 4933, 4887, 4935, + 4891, 4804, 4730, 4769, 4867, 4922, 4923, 4921, 4924, 4817, 4767, 4942, + 4943, 4906, 4830, 4763, 4835, 4836, 4837, 4727, 4729, 4762, 4927, 4917, + 4833, 4843, 4847, 4846, 4845, 4844, 4803, 4800, 4799, 4757, 4759, 4760, + 4758, 4926, 4731, 4811, 4750, 4714, 4708, 4709, 4712, 4713, 4711, 4710, + 4715, 4856, 4851, 4852, 4850, 4861, 4860, 4859, 4858, 4862, 4854, 4812, + 4722, 4777, 4776, 4775, 4857, 4855, 4853, 4805, 4806, 4768, 4810, 4808, + 4779, 4786, 4782, 4790, 4785, 4794, 4784, 4783, 4780, 4781, 4788, 4789, + 4796, 4793, 4791, 4740, 4741, 4739, 4787, 4795, 4948, 4753, 4751, 4754, + 4756, 4755, 4752, 4944, 4946, 4945, 4947, 4797, 4798, 4747, 4746, 4748, + 4749, 4902, 4905, 4903, 4904, 4898, 4901, 4899, 4900, 4761, 4764, 4909, + 4738, 4732, 4737, 4735, 4734, 4733, 4736, 4820, 4824, 4821, 4825, 4829, + 4802, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 23803, 23680, 23696, 23788, 23675, 23800, 23738, 23789, 23787, + 23676, 23672, 23799, 23666, 23784, 23785, 23786, 23691, 23692, 23629, + 23630, 23625, 23624, 23755, 23826, 23823, 23698, 23697, 23804, 23805, + 23699, 23708, 23709, 23710, 23669, 23701, 23702, 23703, 23682, 23683, + 35762, 35762, 23746, 23679, 23681, 23707, 23706, 23751, 23750, 23802, + 23801, 23778, 23779, 23665, 23671, 23767, 23768, 23782, 23783, 23747, + 23849, 23721, 23781, 23690, 23807, 23825, 23809, 23754, 23847, 23770, + 23670, 23810, 23811, 23834, 23835, 23838, 23839, 23836, 23837, 23830, + 23831, 23832, 23833, 23744, 23745, 23840, 23841, 23769, 23845, 23752, + 23749, 23633, 23634, 23628, 23848, 23720, 23780, 23689, 23806, 23824, + 23808, 23753, 23654, 23655, 23658, 23659, 23660, 23693, 23694, 23695, + 23637, 23644, 23645, 23646, 23647, 23648, 23622, 23687, 23623, 23688, + 23621, 23685, 23620, 23684, 23635, 23653, 23661, 23652, 23638, 23639, + 23636, 23663, 23718, 23717, 23642, 23664, 23649, 23656, 23662, 23641, + 23657, 23790, 23813, 23852, 23777, 23740, 23700, 23668, 23677, 23714, + 23713, 23829, 23842, 23851, 23843, 23844, 23756, 23759, 23760, 23761, + 23762, 23763, 23764, 23765, 23766, 23757, 23758, 23722, 23748, 23686, + 23678, 23640, 23643, 23650, 23651, 23772, 23771, 23719, 23712, 23711, + 23850, 23673, 23674, 23739, 23735, 23626, 23793, 23795, 23741, 23743, + 23796, 23798, 23704, 23705, 23737, 23736, 23627, 23794, 23742, 23797, + 23821, 23820, 23822, 23819, 23815, 23816, 23817, 23818, 23667, 23715, + 23716, 23812, 23846, 23774, 23632, 23791, 23631, 23827, 23775, 23776, + 23792, 23828, 23773, 23723, 23726, 23730, 23733, 23732, 23731, 23727, + 23728, 23724, 23725, 23729, 23814, 23734, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14240, 14228, 14251, + 14252, 14234, 14253, 14254, 14255, 14256, 14241, 14242, 14243, 14244, + 14245, 14246, 14247, 14248, 14249, 14250, 14229, 14230, 14231, 14232, + 14233, 14235, 14236, 14237, 14238, 14239, 13960, 13968, 13981, 13989, + 13995, 13996, 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13969, + 13970, 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, + 13980, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13990, 13991, + 13992, 13993, 13994, 7968, 7967, 7969, 14020, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17140, 17141, + 17138, 17136, 17127, 17126, 17133, 17131, 17122, 17129, 17139, 17124, + 17137, 17135, 17128, 17125, 17134, 17132, 17123, 17130, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 22166, 22167, 22164, 22162, 22153, 22152, 22159, 22157, 22148, 22155, + 22165, 22150, 22163, 22161, 22154, 22151, 22160, 22158, 22149, 22156, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 23494, 10576, 10577, 10571, 10570, 10569, 31810, 31830, + 31851, 31799, 31844, 31808, 31794, 31853, 31798, 31812, 31818, 31841, + 31842, 31856, 31862, 31809, 31840, 31870, 31828, 31795, 31858, 31860, + 31827, 31873, 31807, 31822, 31819, 31800, 31814, 31797, 31855, 31848, + 31802, 31845, 31834, 31868, 31857, 31831, 31859, 31847, 31861, 31836, + 31824, 31867, 31837, 31823, 31854, 31863, 31833, 31869, 31806, 31850, + 31826, 31872, 31816, 31801, 31838, 31835, 31849, 31793, 31821, 31820, + 31871, 31865, 31843, 31813, 31811, 31817, 31825, 31864, 31866, 31839, + 31805, 31803, 31832, 31796, 31804, 31852, 31815, 31846, 31829, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8366, 8364, 8363, + 8360, 8359, 8362, 8361, 8367, 8365, 8357, 8355, 8354, 8351, 8350, 8353, + 8352, 8358, 8356, 16051, 16050, 16049, 16048, 16047, 30395, 30394, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 21146, 21148, 21176, 21139, 21152, 21184, + 21155, 21185, 21157, 21186, 21159, 21161, 21178, 21180, 21163, 21166, + 21187, 21170, 21172, 21142, 21174, 21188, 21189, 21182, 21190, 21150, + 21194, 21196, 21229, 21191, 21200, 21203, 21205, 21237, 21207, 21238, + 21209, 21211, 21231, 21233, 21213, 21216, 21239, 21220, 21222, 21224, + 21227, 21240, 21241, 21235, 21242, 21198, 21634, 21636, 21666, 21640, + 21642, 21674, 21645, 21675, 21647, 21676, 21649, 21651, 21668, 21670, + 21653, 21656, 21677, 21660, 21662, 21630, 21664, 21678, 21679, 21672, + 21680, 21638, 21582, 21584, 21617, 21578, 21588, 21591, 21593, 35762, + 21595, 21625, 21597, 21599, 21619, 21621, 21601, 21604, 21626, 21608, + 21610, 21612, 21615, 21627, 21628, 21623, 21629, 21586, 21299, 21301, + 21331, 21305, 21307, 21339, 21310, 21340, 21312, 21341, 21314, 21316, + 21333, 21335, 21318, 21321, 21342, 21325, 21327, 21295, 21329, 21343, + 21344, 21337, 21345, 21303, 21353, 21355, 21390, 21359, 21361, 21364, + 21366, 21398, 21368, 21399, 21370, 21372, 21392, 21394, 21374, 21377, + 21400, 21381, 21383, 21385, 21388, 21401, 21402, 21396, 21403, 21357, + 22106, 35762, 22107, 22108, 35762, 35762, 22109, 35762, 35762, 22110, + 22111, 35762, 35762, 22112, 22113, 22114, 22115, 35762, 22116, 22117, + 22118, 22119, 22120, 22121, 22122, 22123, 22124, 22125, 22126, 22127, + 35762, 22128, 35762, 22129, 22130, 22131, 22132, 22133, 22134, 22135, + 35762, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, + 22145, 22146, 21243, 21244, 21245, 21246, 21247, 21248, 21249, 21250, + 21251, 21252, 21253, 21254, 21255, 21256, 21257, 21258, 21259, 21260, + 21261, 21262, 21263, 21264, 21265, 21266, 21267, 21268, 21269, 21270, + 21271, 21272, 21273, 21274, 21275, 21276, 21277, 21278, 21279, 21280, + 21281, 21282, 21283, 21284, 21285, 21286, 21287, 21288, 21289, 21290, + 21291, 21292, 21293, 21294, 21530, 21531, 35762, 21532, 21533, 21534, + 21535, 35762, 35762, 21536, 21537, 21538, 21539, 21540, 21541, 21542, + 21543, 35762, 21544, 21545, 21546, 21547, 21548, 21549, 21550, 35762, + 21551, 21552, 21553, 21554, 21555, 21556, 21557, 21558, 21559, 21560, + 21561, 21562, 21563, 21564, 21565, 21566, 21567, 21568, 21569, 21570, + 21571, 21572, 21573, 21574, 21575, 21576, 21475, 21476, 35762, 21477, + 21478, 21479, 21480, 35762, 21481, 21482, 21483, 21484, 21485, 35762, + 21486, 35762, 35762, 35762, 21487, 21488, 21489, 21490, 21491, 21492, + 21493, 35762, 21494, 21495, 21496, 21497, 21498, 21499, 21500, 21501, + 21502, 21503, 21504, 21505, 21506, 21507, 21508, 21509, 21510, 21511, + 21512, 21513, 21514, 21515, 21516, 21517, 21518, 21519, 21413, 21414, + 21415, 21416, 21417, 21418, 21419, 21420, 21421, 21422, 21423, 21424, + 21425, 21426, 21427, 21428, 21429, 21430, 21431, 21432, 21433, 21434, + 21435, 21436, 21437, 21438, 21439, 21440, 21441, 21442, 21443, 21444, + 21445, 21446, 21447, 21448, 21449, 21450, 21451, 21452, 21453, 21454, + 21455, 21456, 21457, 21458, 21459, 21460, 21461, 21462, 21463, 21464, + 22044, 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, + 22054, 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, + 22064, 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, + 22074, 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, + 22084, 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, + 22094, 22095, 21876, 21878, 21908, 21882, 21884, 21916, 21887, 21917, + 21889, 21918, 21891, 21893, 21910, 21912, 21895, 21898, 21919, 21902, + 21904, 21872, 21906, 21920, 21921, 21914, 21922, 21880, 21930, 21932, + 21967, 21936, 21938, 21941, 21943, 21975, 21945, 21976, 21947, 21949, + 21969, 21971, 21951, 21954, 21977, 21958, 21960, 21962, 21965, 21978, + 21979, 21973, 21980, 21934, 21992, 21993, 21994, 21995, 21996, 21997, + 21998, 21999, 22000, 22001, 22002, 22003, 22004, 22005, 22006, 22007, + 22008, 22009, 22010, 22011, 22012, 22013, 22014, 22015, 22016, 22017, + 22018, 22019, 22020, 22021, 22022, 22023, 22024, 22025, 22026, 22027, + 22028, 22029, 22030, 22031, 22032, 22033, 22034, 22035, 22036, 22037, + 22038, 22039, 22040, 22041, 22042, 22043, 21766, 21768, 21798, 21772, + 21774, 21806, 21777, 21807, 21779, 21808, 21781, 21783, 21800, 21802, + 21785, 21788, 21809, 21792, 21794, 21762, 21796, 21810, 21811, 21804, + 21812, 21770, 21820, 21822, 21857, 21826, 21828, 21831, 21833, 21865, + 21835, 21866, 21837, 21839, 21859, 21861, 21841, 21844, 21867, 21848, + 21850, 21852, 21855, 21868, 21869, 21863, 21870, 21824, 21689, 21690, + 21691, 21692, 21693, 21694, 21695, 21696, 21697, 21698, 21699, 21700, + 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, 21709, 21710, + 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, 21719, 21720, + 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, 21729, 21730, + 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, 21739, 21740, + 21579, 21580, 35762, 35762, 21147, 21149, 21156, 21141, 21153, 21151, + 21154, 21143, 21158, 21160, 21162, 21179, 21181, 21183, 21164, 21169, + 21171, 21144, 21173, 21145, 21175, 21167, 21177, 21168, 21165, 21407, + 21195, 21197, 21206, 21193, 21201, 21199, 21202, 21225, 21208, 21210, + 21212, 21232, 21234, 21236, 21214, 21219, 21221, 21204, 21223, 21226, + 21228, 21217, 21230, 21218, 21215, 21409, 21405, 21412, 21406, 21408, + 21411, 21410, 21635, 21637, 21646, 21641, 21643, 21639, 21644, 21631, + 21648, 21650, 21652, 21669, 21671, 21673, 21654, 21659, 21661, 21632, + 21663, 21633, 21665, 21657, 21667, 21658, 21655, 21683, 21583, 21585, + 21594, 21581, 21589, 21587, 21590, 21613, 21596, 21598, 21600, 21620, + 21622, 21624, 21602, 21607, 21609, 21592, 21611, 21614, 21616, 21605, + 21618, 21606, 21603, 21685, 21681, 21688, 21682, 21684, 21687, 21686, + 21300, 21302, 21311, 21306, 21308, 21304, 21309, 21296, 21313, 21315, + 21317, 21334, 21336, 21338, 21319, 21324, 21326, 21297, 21328, 21298, + 21330, 21322, 21332, 21323, 21320, 21348, 21354, 21356, 21367, 21360, + 21362, 21358, 21363, 21386, 21369, 21371, 21373, 21393, 21395, 21397, + 21375, 21380, 21382, 21365, 21384, 21387, 21389, 21378, 21391, 21379, + 21376, 21350, 21346, 21404, 21347, 21349, 21352, 21351, 21877, 21879, + 21888, 21883, 21885, 21881, 21886, 21873, 21890, 21892, 21894, 21911, + 21913, 21915, 21896, 21901, 21903, 21874, 21905, 21875, 21907, 21899, + 21909, 21900, 21897, 21925, 21931, 21933, 21944, 21937, 21939, 21935, + 21940, 21963, 21946, 21948, 21950, 21970, 21972, 21974, 21952, 21957, + 21959, 21942, 21961, 21964, 21966, 21955, 21968, 21956, 21953, 21927, + 21923, 21981, 21924, 21926, 21929, 21928, 21767, 21769, 21778, 21773, + 21775, 21771, 21776, 21763, 21780, 21782, 21784, 21801, 21803, 21805, + 21786, 21791, 21793, 21764, 21795, 21765, 21797, 21789, 21799, 21790, + 21787, 21815, 21821, 21823, 21834, 21827, 21829, 21825, 21830, 21853, + 21836, 21838, 21840, 21860, 21862, 21864, 21842, 21847, 21849, 21832, + 21851, 21854, 21856, 21845, 21858, 21846, 21843, 21817, 21813, 21871, + 21814, 21816, 21819, 21818, 21140, 21192, 35762, 35762, 21471, 21473, + 21470, 21469, 21466, 21465, 21468, 21467, 21474, 21472, 21526, 21528, + 21525, 21524, 21521, 21520, 21523, 21522, 21529, 21527, 22102, 22104, + 22101, 22100, 22097, 22096, 22099, 22098, 22105, 22103, 21988, 21990, + 21987, 21986, 21983, 21982, 21985, 21984, 21991, 21989, 21747, 21749, + 21746, 21745, 21742, 21741, 21744, 21743, 21750, 21748, 28184, 28140, + 28165, 28381, 28336, 28122, 28185, 28149, 28298, 28250, 28226, 28187, + 28190, 28147, 28191, 28141, 28192, 28214, 28209, 28247, 28188, 28194, + 28203, 28204, 28195, 28201, 28205, 28143, 28277, 28186, 28215, 28144, + 28224, 28193, 28223, 28210, 28249, 28248, 28189, 28208, 28218, 28219, + 28222, 28221, 28289, 28197, 28198, 28199, 28271, 28239, 28202, 28206, + 28200, 28196, 28270, 28231, 28269, 28268, 28228, 28227, 28230, 28220, + 28217, 28216, 28266, 28265, 28267, 28238, 28314, 28318, 28317, 28315, + 28316, 28159, 28305, 28335, 28307, 28320, 28312, 28321, 28313, 28322, + 28311, 28169, 28170, 28334, 28380, 28308, 28309, 28310, 28306, 28332, + 28319, 28330, 28323, 28331, 28329, 28327, 28324, 28325, 28326, 28328, + 28156, 28162, 28160, 28161, 28365, 28364, 28172, 28164, 28175, 28178, + 28173, 28176, 28174, 28177, 28182, 28179, 28139, 28374, 28379, 28376, + 28378, 28353, 28355, 28333, 28363, 28356, 28360, 28354, 28362, 28361, + 28359, 28121, 28211, 28146, 28338, 28124, 28347, 28213, 28212, 28339, + 28252, 28255, 28254, 28253, 28262, 28302, 28151, 28375, 28133, 28258, + 28261, 28256, 28257, 28350, 28260, 28349, 28132, 28131, 28259, 28150, + 28348, 28130, 28225, 28145, 28340, 28357, 28123, 28207, 28142, 28278, + 28358, 28134, 28286, 28282, 28284, 28155, 28377, 28135, 28279, 28281, + 28280, 28285, 28283, 28373, 28251, 28148, 28180, 28367, 28368, 28366, + 28168, 28346, 28126, 28125, 28275, 28351, 28273, 28154, 28263, 28274, + 28372, 28272, 28276, 28264, 28152, 28181, 28171, 28352, 28137, 28138, + 28136, 28153, 28157, 28158, 28370, 28371, 28369, 28337, 28240, 28342, + 28243, 28244, 28245, 28242, 28246, 28241, 28235, 28236, 28237, 28234, + 28232, 28163, 28233, 28229, 28166, 28167, 28344, 28345, 28341, 28343, + 28128, 28129, 28127, 28287, 28292, 28295, 28296, 28297, 28303, 28288, + 28290, 28291, 28299, 28294, 28300, 28301, 28293, 28183, 28304, 28695, + 28694, 28693, 28715, 28714, 28713, 28670, 28669, 28668, 28054, 28053, + 28052, 28656, 28655, 28654, 28666, 28667, 28662, 28665, 28661, 28664, + 28663, 28111, 28114, 28110, 28113, 28112, 28660, 28530, 28531, 28532, + 28533, 28528, 28529, 28534, 28574, 28479, 28592, 28591, 28589, 28590, + 28593, 28568, 28571, 28569, 28570, 28567, 28598, 28597, 28595, 28596, + 28544, 28543, 28542, 28548, 28547, 28546, 28545, 28566, 28565, 28564, + 28541, 28540, 28539, 28614, 28613, 28612, 28588, 28587, 28586, 28711, + 28710, 28709, 28708, 28707, 28706, 28712, 28705, 28704, 28703, 28448, + 28447, 28445, 28446, 28452, 28451, 28449, 28450, 28440, 28439, 28437, + 28438, 28444, 28443, 28441, 28442, 28502, 28501, 28499, 28500, 28503, + 28517, 28520, 28518, 28519, 28476, 28507, 28506, 28505, 28504, 28462, + 28475, 28474, 28473, 28463, 28461, 28460, 28459, 28526, 28525, 28524, + 28523, 28522, 28521, 28698, 28697, 28696, 28701, 28700, 28699, 28702, + 28561, 28560, 28558, 28559, 28552, 28551, 28549, 28550, 28557, 28556, + 28579, 28578, 28575, 28580, 28585, 28582, 28581, 28601, 28600, 28599, + 28604, 28603, 28602, 28555, 28563, 28562, 28651, 28648, 28647, 28594, + 28554, 28577, 28584, 28609, 28653, 28650, 28646, 28553, 28576, 28583, + 28608, 28652, 28649, 28645, 28607, 28606, 28605, 28466, 28465, 28483, + 28481, 28482, 28480, 28492, 28490, 28491, 28489, 28509, 28508, 28640, + 28637, 28643, 28468, 28467, 28484, 28485, 28487, 28486, 28496, 28494, + 28495, 28493, 28511, 28510, 28641, 28638, 28644, 28472, 28471, 28469, + 28470, 28464, 28488, 28497, 28512, 28513, 28514, 28639, 28636, 28642, + 28498, 28538, 28536, 28537, 28535, 28458, 28456, 28454, 28457, 28455, + 28453, 28611, 28610, 28516, 28515, 28573, 28572, 28478, 28477, 28070, + 28069, 28065, 28068, 28072, 28071, 28066, 28067, 28064, 28073, 28383, + 28390, 28388, 28387, 28385, 28386, 28384, 28389, 28103, 28101, 28102, + 28079, 28078, 28077, 28062, 28060, 28061, 28063, 28118, 28117, 28116, + 28097, 28098, 28094, 28075, 28074, 28093, 28095, 28092, 28096, 28076, + 28091, 28089, 28090, 28086, 28088, 28087, 28080, 28082, 28081, 28084, + 28083, 28085, 28057, 28056, 28055, 28680, 28679, 28681, 28100, 28617, + 28616, 28619, 28618, 28047, 28049, 28046, 28048, 28051, 28050, 28412, + 28410, 28411, 28417, 28419, 28418, 28414, 28416, 28415, 28431, 28430, + 28429, 28423, 28424, 28425, 28426, 28427, 28428, 28420, 28422, 28421, + 28432, 28433, 28434, 28401, 28399, 28400, 28413, 28436, 28435, 28688, + 28687, 28685, 28686, 28684, 28689, 28683, 28682, 28672, 28677, 28675, + 28676, 28673, 28674, 28678, 28615, 28527, 28620, 28382, 28099, 28658, + 28657, 28120, 28115, 28659, 28691, 28690, 28692, 28716, 28391, 28392, + 28393, 28394, 28395, 28396, 28397, 28398, 28109, 28409, 28408, 28403, + 28406, 28405, 28402, 28404, 28407, 28059, 28119, 28671, 28058, 28717, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 28104, 28105, 28106, 28107, 28108, + 35762, 28628, 28629, 28630, 28631, 28632, 28633, 28634, 28635, 28621, + 28622, 28623, 28624, 28625, 28626, 28627, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 18911, 19184, 18676, 19189, + 18663, 19034, 19294, 19186, 19280, 19249, 18656, 18890, 18891, 19284, + 18651, 18703, 18677, 19028, 18827, 19008, 18888, 19278, 19165, 19253, + 18899, 18828, 18972, 19124, 19254, 18794, 19201, 35762, 35762, 35762, + 35762, 35762, 35762, 18818, 19021, 19066, 19170, 19208, 19244, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 7924, 7926, 7936, 7930, 7912, 7937, 7944, 35762, 7943, + 7917, 7914, 7913, 7911, 7948, 7932, 7931, 7933, 7947, 7934, 7935, 7919, + 7922, 7946, 7928, 7945, 35762, 35762, 7920, 7923, 7927, 7921, 7939, 7938, + 7940, 35762, 7942, 7918, 35762, 7941, 7916, 7925, 7915, 7929, 35762, + 35762, 35762, 35762, 35762, 23081, 23050, 23078, 23076, 23052, 23074, + 23071, 23072, 23073, 23080, 23057, 23058, 23082, 23061, 23059, 23054, + 23067, 23083, 23056, 23079, 23066, 23075, 23065, 23068, 23053, 23070, + 23051, 23064, 23048, 23077, 23049, 23062, 23060, 10214, 10192, 10212, + 10199, 10195, 10209, 10206, 10207, 10208, 10213, 10197, 10215, 10211, + 10198, 10216, 10196, 10201, 10205, 10210, 10204, 10202, 10203, 10200, + 10191, 10194, 10193, 23055, 23069, 23063, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7841, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 25172, 25156, 25147, 25158, 25165, 25155, 25160, 25151, + 25177, 25181, 25183, 25186, 25150, 25145, 25180, 25170, 25154, 25153, + 25184, 25146, 25157, 25178, 25164, 25182, 25185, 25152, 25174, 25159, + 25149, 25169, 25148, 25166, 25171, 25173, 25179, 25163, 25161, 25162, + 25187, 25188, 25167, 25168, 25175, 25176, 25189, 35762, 35762, 35762, + 25198, 25202, 25201, 25204, 25203, 25200, 25199, 25194, 25192, 25191, + 25195, 25193, 25196, 25197, 35762, 35762, 25211, 25213, 25210, 25209, + 25206, 25205, 25208, 25207, 25214, 25212, 35762, 35762, 35762, 35762, + 25190, 25144, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 32479, 32462, 32482, 32472, 32476, 32473, 32478, 32466, + 32465, 32481, 32469, 32484, 32483, 32475, 32474, 32480, 32477, 32463, + 32457, 32464, 32458, 32486, 32467, 32459, 32468, 32460, 32485, 32471, + 32461, 32470, 32487, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 33793, 33792, 33824, 33825, 33826, 33828, 33817, 33820, 33806, 33809, + 33821, 33813, 33810, 33827, 33823, 33822, 33830, 33835, 33834, 33833, + 33819, 33798, 33797, 33832, 33831, 33818, 33829, 33801, 33803, 33807, + 33814, 33805, 33812, 33811, 33800, 33795, 33796, 33804, 33799, 33802, + 33794, 33808, 33815, 33816, 33839, 33840, 33837, 33838, 33847, 33849, + 33846, 33845, 33842, 33841, 33844, 33843, 33850, 33848, 35762, 35762, + 35762, 35762, 35762, 33836, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 24159, 24162, 24161, 24163, 24160, 24142, 24146, 24144, + 24143, 24145, 24154, 24157, 24155, 24158, 24156, 24164, 24165, 24166, + 24167, 24168, 24147, 24149, 24152, 24153, 24148, 24151, 24150, 24172, + 24169, 24173, 24171, 24170, 24180, 24182, 24179, 24178, 24175, 24174, + 24177, 24176, 24183, 24181, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 12689, 12695, 12693, 12690, 12692, 12691, + 12694, 35762, 12644, 12688, 12686, 12685, 35762, 12632, 12631, 35762, + 12643, 12642, 12641, 12628, 12627, 12640, 12639, 12638, 12637, 12636, + 12635, 12630, 12629, 12634, 12633, 35762, 22398, 22399, 22400, 22466, + 22487, 22475, 22440, 22573, 22401, 22402, 22406, 22525, 22510, 22515, + 22439, 22591, 22542, 22461, 22445, 22536, 22405, 22403, 22404, 22451, + 22495, 22548, 22586, 22411, 22407, 22415, 22552, 22490, 22501, 22528, + 22416, 22413, 22412, 22565, 22503, 22563, 22541, 22532, 22530, 22531, + 22592, 22571, 22410, 22425, 22417, 22564, 22509, 22534, 22470, 22593, + 22421, 22422, 22423, 22479, 22471, 22455, 22556, 22507, 22408, 22414, + 22409, 22482, 22577, 22578, 22418, 22419, 22420, 22493, 22448, 22498, + 22465, 22426, 22424, 22427, 22551, 22508, 22557, 22459, 22569, 22428, + 22432, 22436, 22505, 22477, 22545, 22526, 22429, 22433, 22434, 22476, + 22468, 22533, 22486, 22594, 22497, 22435, 22430, 22431, 22513, 22566, + 22572, 22444, 22584, 22437, 22496, 22446, 22543, 22483, 22522, 22453, + 22535, 22485, 22452, 22590, 22442, 22488, 22438, 22484, 22512, 22538, + 22550, 22520, 22555, 22523, 22481, 22499, 22580, 22549, 22516, 22559, + 22587, 22562, 22560, 22583, 22456, 22570, 22463, 22491, 22447, 22478, + 22454, 22502, 22458, 22537, 22457, 22517, 22443, 22581, 22472, 22567, + 22568, 22585, 22558, 22500, 22540, 22529, 22492, 22467, 22441, 22506, + 22514, 22553, 22519, 22449, 22544, 22489, 22504, 22473, 22474, 22576, + 22518, 22521, 22524, 22588, 22511, 22460, 22462, 22547, 22589, 22539, + 22527, 22579, 22582, 22554, 22574, 22480, 22546, 22469, 22561, 22450, + 22575, 22494, 22464, 35762, 35762, 22602, 22600, 22599, 22596, 22595, + 22598, 22597, 22603, 22601, 22393, 22392, 22396, 22394, 22391, 22395, + 22397, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 27, 9, 24, 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, + 32, 18, 15, 31, 42, 13, 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, + 22, 62, 44, 59, 49, 64, 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, + 50, 66, 77, 48, 60, 71, 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, + 86, 79, 84, 43, 78, 83, 82, 35762, 35762, 35762, 35762, 93, 95, 92, 91, + 88, 87, 90, 89, 96, 94, 35762, 35762, 35762, 35762, 80, 81, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 16234, 16220, 16215, 16195, 16190, 16208, 16203, 16183, 16198, + 16223, 16218, 16213, 16193, 16188, 16209, 16204, 16184, 16199, 16235, + 16221, 16216, 16196, 16191, 16211, 16206, 16186, 16201, 16236, 16222, + 16217, 16197, 16192, 16212, 16207, 16187, 16202, 16224, 16219, 16214, + 16194, 16189, 16210, 16205, 16185, 16200, 16181, 16182, 16172, 16179, + 16180, 16232, 16230, 16229, 16226, 16225, 16228, 16227, 16233, 16231, + 16238, 16174, 16173, 16175, 16237, 16178, 16177, 16176, 16171, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 26069, 26064, 26059, 26039, 26034, + 26052, 26047, 26027, 26042, 26067, 26062, 26057, 26037, 26032, 26053, + 26048, 26028, 26043, 26070, 26065, 26060, 26040, 26035, 26055, 26050, + 26030, 26045, 26071, 26066, 26061, 26041, 26036, 26056, 26051, 26031, + 26046, 26068, 26063, 26058, 26038, 26033, 26054, 26049, 26029, 26044, + 26026, 26019, 26021, 26011, 26013, 26014, 26016, 26023, 26022, 26017, + 26012, 26015, 26020, 26018, 26025, 26024, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 2231, 2240, 2237, 2135, 35762, 2247, 2235, 2239, 2227, 2243, 2234, 2183, + 2238, 2245, 2210, 2232, 2241, 2211, 2246, 2242, 2209, 2230, 2229, 2233, + 2228, 2134, 2236, 2244, 2105, 2107, 2106, 2108, 35762, 2147, 2145, 35762, + 2142, 35762, 35762, 2141, 35762, 2149, 2144, 2152, 2146, 2151, 2139, + 2154, 2148, 2140, 2153, 35762, 2138, 2137, 2136, 2143, 35762, 2155, + 35762, 2150, 35762, 35762, 35762, 35762, 35762, 35762, 2219, 35762, + 35762, 35762, 35762, 2220, 35762, 2221, 35762, 2224, 35762, 2223, 2216, + 2226, 35762, 2217, 2225, 35762, 2215, 35762, 35762, 2218, 35762, 2214, + 35762, 2222, 35762, 2212, 35762, 2213, 35762, 2202, 2199, 35762, 2196, + 35762, 35762, 2195, 2190, 2204, 2198, 35762, 2200, 2206, 2193, 2208, + 2203, 2194, 2207, 35762, 2192, 2191, 2189, 2197, 35762, 2188, 2201, 2205, + 2186, 35762, 2187, 35762, 2160, 2172, 2170, 2180, 2163, 2182, 2168, 2162, + 2166, 2175, 35762, 2178, 2171, 2177, 2157, 2161, 2173, 2158, 2181, 2174, + 2156, 2167, 2165, 2159, 2164, 2179, 2169, 2176, 35762, 35762, 35762, + 35762, 35762, 2122, 2119, 2130, 35762, 2133, 2117, 2121, 2115, 2125, + 35762, 2128, 2120, 2127, 2110, 2132, 2123, 2111, 2131, 2124, 2109, 2116, + 2114, 2112, 2113, 2129, 2118, 2126, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2184, 2185, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 20707, 20723, 20738, 20714, 20742, 20741, 20739, + 20719, 20736, 20733, 20712, 20709, 20728, 20725, 20705, 20716, 20720, + 20737, 20734, 20713, 20710, 20729, 20726, 20706, 20717, 20718, 20735, + 20732, 20711, 20708, 20727, 20724, 20704, 20715, 20722, 20721, 20702, + 20745, 20731, 20730, 20743, 20740, 20744, 20703, 35762, 35762, 35762, + 35762, 10829, 10780, 10781, 10782, 10783, 10784, 10785, 10786, 10787, + 10788, 10789, 10790, 10791, 10792, 10793, 10794, 10795, 10796, 10797, + 10798, 10799, 10800, 10801, 10802, 10803, 10804, 10805, 10806, 10807, + 10808, 10809, 10810, 10811, 10812, 10813, 10814, 10815, 10816, 10817, + 10818, 10819, 10820, 10821, 10822, 10823, 10824, 10825, 10826, 10827, + 10828, 10879, 10830, 10831, 10832, 10833, 10834, 10835, 10836, 10837, + 10838, 10839, 10840, 10841, 10842, 10843, 10844, 10845, 10846, 10847, + 10848, 10849, 10850, 10851, 10852, 10853, 10854, 10855, 10856, 10857, + 10858, 10859, 10860, 10861, 10862, 10863, 10864, 10865, 10866, 10867, + 10868, 10869, 10870, 10871, 10872, 10873, 10874, 10875, 10876, 10877, + 10878, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 26691, 26761, 26736, 26732, 26695, 26700, 26720, + 26716, 26712, 26765, 26728, 26769, 26704, 26724, 26708, 35762, 35762, + 26762, 26737, 26733, 26696, 26701, 26721, 26717, 26713, 26766, 26729, + 26770, 26705, 26725, 26709, 26692, 35762, 26763, 26738, 26734, 26697, + 26702, 26722, 26718, 26714, 26767, 26730, 26771, 26706, 26726, 26710, + 26690, 35762, 26760, 26735, 26731, 26694, 26699, 26719, 26715, 26711, + 26764, 26727, 26768, 26703, 26723, 26707, 26693, 26698, 26742, 26739, + 26753, 26754, 26755, 26756, 26757, 26758, 26759, 26743, 26744, 26745, + 26746, 26747, 26748, 26749, 26750, 26751, 26752, 26740, 26741, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10565, + 10564, 10549, 10561, 10558, 10543, 10540, 10555, 10552, 10567, 10546, + 10606, 10585, 6512, 6229, 6253, 26362, 26363, 26364, 26365, 26366, 26367, + 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, 26377, + 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, 26387, + 32455, 6343, 6344, 6240, 6511, 8343, 29496, 29497, 29498, 29499, 29500, + 29501, 29502, 29503, 29504, 29505, 29506, 29507, 29508, 29509, 29510, + 29511, 29512, 29513, 29514, 29515, 29516, 29517, 29518, 29519, 29520, + 29521, 29490, 29526, 29541, 29542, 29531, 29553, 24272, 24273, 24274, + 24275, 24276, 24277, 24278, 24279, 24280, 24281, 24282, 24283, 24284, + 24285, 24286, 24287, 24288, 24289, 24290, 24291, 24292, 24293, 24294, + 24295, 24296, 24297, 26969, 26970, 26971, 6239, 6238, 6287, 24303, 24304, + 24305, 24306, 24307, 24308, 24309, 24310, 24311, 24312, 24313, 24314, + 24315, 24316, 24317, 24318, 24319, 24320, 24321, 24322, 24323, 24324, + 24325, 24326, 24327, 24328, 8387, 24332, 24334, 24335, 24331, 24333, + 29232, 29479, 29478, 29485, 29554, 29527, 29528, 29530, 29540, 29548, + 29551, 29543, 29533, 29545, 29483, 29547, 29484, 29534, 29544, 29536, + 29529, 29495, 29489, 29488, 29487, 29524, 29535, 29549, 29550, 21137, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 26997, 26998, 26999, 27000, + 27001, 27002, 27003, 27004, 27005, 27006, 27007, 27008, 27009, 27010, + 27011, 27012, 27013, 27014, 27015, 27016, 27017, 27018, 27019, 27020, + 27021, 27022, 29270, 29491, 29493, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29457, 29452, + 29444, 29492, 29438, 29450, 29473, 29449, 29437, 29462, 29470, 29461, + 29442, 29453, 29454, 29460, 29441, 29471, 29468, 29475, 29451, 29446, + 29465, 29455, 29458, 29435, 29436, 29477, 29448, 29439, 29443, 29459, + 29474, 29456, 29469, 29472, 29445, 29466, 29464, 29463, 29467, 29440, + 29447, 29476, 35762, 35762, 35762, 35762, 32452, 32446, 32447, 32449, + 32453, 32450, 32454, 32448, 32451, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 6291, 6289, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27401, 27402, + 27399, 27403, 27398, 27400, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10219, 13162, + 7734, 24578, 29724, 29723, 6522, 29725, 26962, 4630, 33941, 33767, 22902, + 11218, 11216, 11217, 13610, 24383, 33853, 13131, 33854, 13203, 33852, + 18158, 33851, 8377, 24382, 13130, 18157, 13202, 29648, 13611, 27918, + 32014, 3808, 34096, 34100, 34097, 34098, 7747, 7746, 7745, 7744, 13161, + 34162, 15972, 31673, 4706, 6175, 27734, 13062, 10243, 26230, 5823, 15969, + 32566, 6173, 27389, 15937, 29726, 3985, 11212, 11213, 15762, 13180, + 20989, 13101, 19445, 23855, 32421, 2525, 13723, 22390, 33944, 30811, + 19779, 3406, 26676, 26994, 14263, 26496, 26493, 6174, 29593, 14654, + 28875, 22182, 26811, 27117, 27118, 8177, 9648, 29581, 29170, 4628, 13193, + 27405, 10228, 26088, 29766, 13201, 13137, 28969, 27862, 16002, 10975, + 8176, 6212, 5698, 20592, 9650, 15943, 27935, 3629, 26809, 8175, 13167, + 31675, 27701, 34164, 7751, 32533, 3520, 7693, 2589, 13168, 4093, 26799, + 27108, 34184, 3740, 16359, 6213, 13107, 13127, 13126, 2729, 26420, 8156, + 30810, 8388, 26675, 16364, 5760, 34161, 23499, 27706, 13648, 15234, 4095, + 22901, 27061, 23619, 29597, 19778, 8168, 3510, 3511, 13122, 98, 5758, + 13113, 27348, 13135, 22894, 23520, 6215, 15233, 2506, 32435, 6520, 32420, + 7689, 26525, 33689, 10611, 28881, 3739, 13322, 4098, 13153, 23854, 23602, + 27700, 14279, 23618, 32538, 33694, 23853, 27530, 31789, 28856, 3412, + 6176, 28963, 27531, 29764, 29196, 32535, 15967, 360, 27408, 29769, 33963, + 13647, 26952, 26953, 8379, 33768, 13141, 16004, 29954, 28960, 4964, 3516, + 4705, 15979, 6521, 10520, 7690, 10351, 10352, 24253, 29579, 15978, 15977, + 25236, 16361, 13032, 15980, 3401, 2522, 15973, 20528, 8171, 27705, 10269, + 13097, 16357, 16360, 13031, 34065, 3855, 33950, 33949, 27390, 3870, + 18037, 2600, 4102, 359, 12426, 12427, 12428, 12429, 12430, 26976, 23515, + 26091, 33942, 8370, 32318, 19676, 26978, 5765, 11054, 8391, 34118, 28957, + 28958, 15966, 26981, 13615, 27940, 23497, 27352, 6181, 10716, 26667, + 4252, 12395, 25234, 29191, 4643, 954, 15942, 18039, 13134, 32534, 3986, + 32564, 15201, 2588, 13197, 3741, 26497, 18034, 26821, 11056, 2598, 10778, + 23517, 8371, 32319, 26979, 5766, 11055, 29195, 15968, 23498, 10777, + 26669, 13200, 14652, 34183, 3515, 26267, 26668, 26486, 6180, 13059, + 13057, 11211, 24730, 23518, 32422, 34107, 34024, 34050, 34071, 13138, + 33951, 26087, 32046, 32047, 7688, 25781, 8393, 34173, 13058, 24573, + 29951, 16461, 11062, 18029, 3745, 34172, 26925, 14658, 26816, 20983, + 2519, 15833, 34174, 34171, 13166, 4701, 4700, 4250, 13510, 20896, 34169, + 13105, 20902, 32581, 32582, 26796, 34170, 4631, 26510, 20899, 20900, + 25760, 25758, 2587, 8159, 26878, 16366, 16365, 14521, 2590, 13045, 356, + 16131, 28858, 16247, 14278, 10227, 20434, 24184, 13093, 14525, 3410, + 29949, 26672, 18026, 20527, 27328, 13326, 18021, 4094, 8369, 33961, 3517, + 4637, 32575, 29171, 14276, 15236, 3988, 14265, 34208, 26924, 15235, + 27104, 15237, 10528, 12383, 952, 4249, 28870, 7756, 29192, 11058, 10232, + 26670, 13148, 10696, 29182, 32021, 34035, 15989, 23319, 9646, 15266, + 8376, 3403, 3405, 3404, 3402, 23318, 5994, 27730, 26520, 4632, 22904, + 13154, 25792, 11209, 13118, 25779, 26095, 26097, 4961, 31676, 5703, 5993, + 5995, 3408, 7694, 26927, 27396, 26488, 29591, 32504, 3999, 19777, 24727, + 24728, 7740, 25773, 14264, 3987, 25796, 4000, 24255, 27727, 22741, 31681, + 26098, 13104, 27614, 26928, 6000, 26073, 16356, 26487, 13060, 16163, + 12464, 7737, 7738, 25783, 25782, 26805, 26802, 24568, 22918, 22919, + 33686, 22920, 24648, 967, 4962, 4963, 33688, 31688, 26955, 33690, 13121, + 26800, 26892, 32568, 7724, 7725, 7721, 963, 20529, 15828, 29177, 29176, + 29178, 29179, 3509, 12381, 19566, 27215, 20492, 7739, 17144, 20489, + 25786, 3523, 3525, 3998, 20432, 26957, 2592, 12459, 25762, 29019, 32445, + 24647, 17159, 16250, 16251, 16254, 16253, 16252, 13123, 12382, 34187, + 14648, 25141, 15981, 26681, 22893, 31687, 8398, 28852, 16367, 32506, + 3880, 34083, 18148, 18130, 18138, 18131, 28887, 28886, 32658, 10981, + 32656, 10982, 20561, 32707, 6236, 8385, 8384, 24720, 24722, 29838, 34048, + 15281, 5831, 26089, 11049, 17142, 23504, 29859, 22604, 4096, 7705, 7715, + 7717, 7701, 7699, 7709, 7707, 7697, 7703, 7711, 7695, 7713, 7706, 7716, + 7718, 7702, 7700, 7710, 7708, 7698, 7704, 7712, 7696, 7714, 27173, 27174, + 27175, 4696, 4697, 27336, 3997, 5697, 20986, 3882, 24646, 15941, 20897, + 28873, 10229, 29187, 29188, 16462, 20901, 19490, 31682, 27156, 34102, + 3895, 31686, 7691, 2593, 29563, 12463, 13160, 26500, 20431, 3852, 20554, + 20540, 20552, 20553, 20558, 19545, 32555, 26967, 27089, 27097, 27098, + 27102, 27099, 26968, 34025, 28043, 28042, 28039, 28038, 3830, 3868, + 28045, 28044, 28041, 28040, 3898, 3802, 3815, 10353, 17146, 32037, 26875, + 26822, 3818, 34039, 28970, 31670, 34167, 25770, 32570, 32033, 32490, + 25585, 15229, 27711, 26876, 13103, 25793, 10702, 10703, 10704, 13190, + 13192, 13191, 3814, 13164, 25780, 5704, 5705, 13119, 12431, 12432, 12433, + 24724, 24725, 24726, 12442, 12435, 12436, 10701, 26094, 26099, 33957, + 29190, 29189, 10354, 22905, 22168, 26080, 7723, 5695, 16165, 10246, 8151, + 25757, 27347, 26096, 29589, 10226, 20433, 29180, 32029, 32028, 32030, + 32031, 19517, 27176, 32583, 32035, 19533, 27189, 19454, 27121, 23501, + 19802, 19801, 2734, 2740, 2735, 2739, 2732, 19799, 2733, 34178, 23514, + 32489, 29577, 28855, 23519, 14268, 14270, 13084, 28945, 28947, 28946, + 28948, 28944, 28943, 34165, 28949, 13064, 27060, 28942, 28952, 28955, + 24380, 13056, 32619, 13067, 26498, 8157, 8158, 18022, 13094, 18023, + 18024, 13081, 13082, 13083, 10613, 34179, 953, 26814, 8397, 26522, 13088, + 10612, 13199, 950, 13110, 33959, 28872, 32419, 14273, 20591, 13072, + 15988, 13074, 13065, 2514, 13155, 28871, 10697, 13091, 13069, 14272, + 5767, 28940, 28941, 5768, 18025, 26813, 8396, 33958, 28876, 28877, 32716, + 13087, 13077, 13071, 26519, 27735, 15239, 29175, 15200, 26517, 26516, + 26512, 26514, 24689, 29073, 24663, 29061, 32553, 32562, 32552, 32561, + 24688, 29072, 24662, 29060, 15317, 15310, 15314, 15307, 24690, 29074, + 24664, 29062, 15318, 15311, 15315, 15308, 15939, 15940, 29015, 29016, + 19669, 32750, 27291, 11044, 27721, 15312, 19781, 15289, 15244, 29767, + 27608, 27609, 27610, 15333, 27611, 15301, 33677, 33674, 5996, 27070, + 27346, 15474, 29580, 26960, 15831, 15832, 32497, 22740, 19788, 29578, + 32493, 32494, 4699, 25767, 32532, 4702, 22906, 361, 13124, 26794, 25765, + 31674, 25766, 2521, 25763, 26993, 10251, 2505, 32491, 23496, 23511, + 29765, 23512, 160, 27917, 27404, 29181, 15963, 33668, 8160, 26795, 32503, + 11050, 24644, 28956, 24649, 26926, 11048, 26807, 24653, 3735, 24642, + 3734, 23513, 26529, 24645, 6178, 22605, 34175, 27065, 2591, 32488, 33940, + 27939, 3507, 3508, 26428, 9651, 2604, 19491, 32500, 26889, 6345, 4251, + 13511, 8368, 28869, 27920, 3527, 3636, 26686, 25233, 27919, 29599, 26100, + 15935, 15991, 12396, 35762, 35762, 35762, 35762, 34168, 26772, 33965, + 27329, 14649, 27913, 25267, 23509, 26959, 23506, 32654, 32657, 32655, + 28884, 24697, 227, 228, 35762, 35762, 35762, 27613, 25764, 10538, 26425, + 27709, 23507, 5700, 28874, 13159, 28861, 2523, 26666, 27349, 35762, + 35762, 35762, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, + 335, 321, 288, 250, 281, 282, 283, 265, 309, 286, 336, 337, 307, 308, + 270, 323, 276, 277, 259, 300, 256, 278, 319, 257, 258, 255, 310, 317, + 338, 329, 279, 238, 318, 311, 316, 333, 298, 299, 297, 301, 302, 303, + 230, 231, 284, 312, 241, 304, 305, 240, 246, 326, 327, 296, 247, 248, + 249, 232, 343, 322, 328, 271, 340, 289, 254, 331, 253, 325, 252, 330, + 313, 280, 324, 341, 274, 242, 291, 251, 290, 239, 314, 315, 320, 294, + 268, 266, 267, 293, 292, 260, 261, 262, 263, 264, 229, 244, 245, 306, + 275, 287, 269, 285, 273, 272, 20525, 25235, 20594, 35762, 35762, 35762, + 35762, 15228, 20771, 13645, 26946, 25897, 3823, 3899, 3860, 3803, 3885, + 22283, 3994, 15329, 33678, 13042, 34005, 27397, 3893, 3886, 19796, 22308, + 3995, 15330, 33679, 13043, 34084, 34086, 29411, 3891, 3909, 3847, 34019, + 34020, 10527, 3892, 3910, 3848, 34055, 32018, 19798, 22309, 3996, 33683, + 33680, 13044, 32016, 19791, 22301, 3990, 15302, 33675, 13039, 19784, + 22289, 3993, 15277, 33673, 13041, 19792, 22300, 3991, 15306, 33676, + 13040, 19782, 22306, 3992, 15271, 33672, 19794, 22303, 32036, 22302, + 19786, 22288, 13182, 22287, 27071, 19783, 15276, 22299, 15305, 28850, + 22305, 15269, 33671, 15267, 19793, 15324, 15323, 6506, 24298, 6516, + 24299, 24576, 35762, 35762, 35762, 35762, 35762, 35762, 18137, 18150, + 18133, 18146, 18128, 18149, 18132, 18139, 18151, 18134, 18147, 18129, + 35762, 35762, 35762, 35762, 15274, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 19590, 32726, 27244, 10996, 19597, 32723, 27251, 10993, 19586, 32722, + 27240, 10992, 35762, 35762, 35762, 35762, 19589, 32725, 27243, 10995, + 19599, 32727, 27253, 10997, 15285, 15326, 15299, 15263, 15284, 15325, + 15298, 15262, 19633, 32760, 27304, 11013, 19632, 32759, 27303, 11012, + 19631, 32756, 27302, 11009, 19635, 32762, 27306, 11015, 19634, 32761, + 27305, 11014, 19663, 32737, 27270, 11031, 19671, 32752, 27293, 11046, + 19673, 32748, 27326, 11042, 19623, 32746, 27284, 11040, 19624, 32747, + 27285, 11041, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 19672, 32751, 27294, 11045, 24695, 24669, 29067, 29079, 19509, 32647, + 35762, 35762, 35762, 35762, 35762, 35762, 34121, 34136, 34126, 34131, + 34146, 34141, 34151, 34156, 34123, 34138, 34128, 34133, 34148, 34143, + 34153, 34158, 34122, 34137, 34127, 34132, 34147, 34142, 34152, 34157, + 34119, 34134, 34124, 34129, 34144, 34139, 34149, 34154, 34120, 34135, + 34125, 34130, 34145, 34140, 34150, 34155, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 19639, 32766, 27310, 11020, 19653, 32778, + 27325, 11025, 19598, 32724, 27252, 10994, 15240, 15243, 15242, 15241, + 19607, 27259, 19642, 27295, 19664, 27290, 19668, 27286, 19606, 27258, + 19662, 27269, 33969, 33968, 35762, 35762, 2501, 2498, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 6234, 6235, 6233, 19481, 19479, + 19480, 19482, 19478, 10985, 10987, 10986, 10988, 26673, 34040, 4639, + 26674, 35571, 23320, 13085, 24574, 32019, 13068, 27406, 15990, 28718, + 4960, 26975, 19561, 27210, 14659, 14655, 16358, 13066, 7748, 24254, + 27351, 11059, 20652, 13092, 28959, 13076, 14271, 14269, 13086, 27751, + 28950, 13073, 27938, 26823, 4627, 26265, 27760, 26877, 20898, 23505, + 27942, 26507, 16464, 13115, 22921, 34186, 33943, 14657, 10692, 34160, + 11060, 7692, 32565, 29194, 13614, 27340, 13133, 27733, 32020, 4247, + 21060, 9645, 18038, 28972, 13158, 8392, 2581, 9652, 2599, 26808, 5763, + 2602, 14266, 27764, 29598, 12333, 13609, 26494, 18027, 26266, 11206, + 13174, 30393, 6211, 4097, 9638, 7752, 4638, 26684, 26873, 9653, 27612, + 5699, 19446, 20987, 23500, 2603, 28951, 34207, 28953, 13078, 13090, + 26079, 13195, 24577, 10616, 13095, 13080, 27702, 18036, 13643, 15936, + 13143, 8399, 20486, 27707, 32541, 32633, 11221, 11207, 3450, 27861, + 26090, 13187, 4704, 10522, 13644, 20487, 27106, 27941, 29556, 13512, + 35565, 15827, 27698, 29952, 8378, 16750, 20658, 26491, 15938, 26423, + 26954, 20590, 23503, 22896, 2601, 29768, 20895, 11051, 28882, 26072, + 25798, 28860, 13147, 26085, 3518, 3746, 27729, 14280, 26890, 12422, + 12423, 12425, 12424, 4253, 19780, 13165, 32423, 29761, 29762, 27541, + 11214, 23508, 20984, 22183, 22184, 5999, 9640, 27544, 3635, 13321, 25772, + 13102, 33956, 4703, 22147, 16003, 4641, 32531, 29564, 18031, 10521, + 13070, 101, 6177, 25759, 3514, 26515, 26508, 26518, 26509, 20662, 13108, + 33339, 22728, 12420, 13507, 35757, 4625, 25797, 3738, 27704, 13612, 8374, + 28967, 26995, 13128, 16467, 31792, 26528, 11208, 8154, 2583, 13125, + 32425, 4634, 20661, 20593, 20524, 29193, 2742, 27542, 31680, 4640, 3411, + 27350, 3409, 29197, 26982, 24256, 24357, 24368, 24371, 24360, 24350, + 24365, 33980, 3771, 24351, 33977, 33993, 33996, 33972, 33985, 33990, + 3768, 3784, 3787, 3763, 3776, 3781, 24358, 24369, 24372, 24361, 24356, + 24366, 33981, 3772, 24352, 33999, 34002, 34003, 33998, 34000, 34001, + 3790, 3793, 3794, 3789, 3791, 3792, 24375, 24378, 24379, 24374, 24376, + 24377, 33982, 3773, 24353, 33978, 33994, 33997, 33973, 33983, 33991, + 3769, 3785, 3788, 3764, 3774, 3782, 24359, 24370, 24373, 24362, 24354, + 24367, 33984, 3775, 24355, 33974, 3765, 24363, 33975, 3766, 24364, 33987, + 33988, 33986, 3778, 3779, 3777, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 34202, 34205, 34201, 34203, + 34200, 34199, 34204, 34195, 34198, 34194, 34196, 34193, 34192, 34197, + 35762, 35762, 2743, 25771, 4633, 27936, 32022, 19795, 14267, 26678, + 11057, 99, 29583, 34191, 8395, 35762, 35762, 35762, 35479, 18030, 26274, + 4001, 20660, 26679, 24347, 20990, 13156, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 27335, 5834, 26980, 2586, 10695, 3407, 22900, 1, + 20509, 8373, 5761, 27710, 18040, 15983, 22916, 34163, 26792, 27756, + 18032, 4707, 23516, 32424, 15231, 26687, 27345, 22917, 16009, 20530, + 14653, 13163, 14523, 17228, 13157, 34180, 3521, 7750, 26810, 34182, + 13109, 20526, 8347, 12434, 24348, 16000, 16460, 34166, 35762, 13646, 946, + 20595, 26530, 26825, 26824, 26513, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 23502, 10880, 3989, 3524, 25761, 13144, 30809, + 13186, 31679, 26812, 3519, 16459, 13509, 26495, 35762, 35762, 35762, + 35762, 22389, 27546, 13075, 13079, 13089, 10890, 3743, 4642, 27912, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14656, 27283, 19622, + 26231, 26232, 16170, 15238, 19667, 27289, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 3925, 3955, 3926, 3970, 3941, 3956, 3927, 3978, + 3948, 3964, 3934, 3971, 3942, 3957, 3928, 3982, 3952, 3967, 3938, 3975, + 3961, 3931, 3979, 3949, 3965, 3935, 3972, 3943, 3958, 3929, 3984, 3954, + 3969, 3940, 3977, 3947, 3963, 3933, 3981, 3951, 3937, 3974, 3945, 3960, + 3930, 3983, 3953, 3968, 3939, 3976, 3946, 3962, 3932, 3980, 3950, 3966, + 3936, 3973, 3944, 3959, 20541, 20542, 20549, 20551, 20546, 20574, 20575, + 20571, 20573, 20569, 20572, 20565, 20568, 20566, 20570, 20567, 20545, + 20548, 20543, 20547, 20544, 20550, 32692, 32693, 32700, 32702, 32697, + 32682, 32683, 32679, 32681, 32677, 32680, 32673, 32676, 32674, 32678, + 32675, 32696, 32699, 32694, 32698, 32695, 32701, 32664, 19447, 32661, + 19449, 19530, 32715, 27195, 20587, 33658, 33659, 33660, 33661, 33662, + 33663, 15949, 15950, 15951, 15952, 15953, 15954, 19448, 19450, 27120, + 27119, 32663, 15948, 32690, 32713, 32666, 32714, 32712, 27160, 27190, + 27147, 27191, 27170, 19489, 27155, 32671, 20537, 16342, 32668, 32669, + 35762, 19488, 5997, 16338, 15278, 32685, 32709, 32662, 19451, 32686, + 32710, 20563, 20556, 4173, 4177, 4169, 4172, 4174, 4179, 4170, 4167, + 4176, 4178, 4180, 4175, 4168, 4171, 4184, 4207, 2504, 16339, 19487, + 27154, 16340, 19577, 27229, 11002, 32719, 19486, 27153, 33766, 27162, + 24301, 24300, 24302, 34043, 19532, 22897, 27188, 24330, 29584, 29585, + 29587, 29588, 29586, 34109, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 27743, 27745, 27742, 27741, 27738, 27737, 27740, 27739, 27746, + 27744, 35762, 35762, 35762, 35762, 35762, 35762, 6553, 6554, 6555, 6556, + 6557, 6558, 6559, 6560, 6561, 6562, 6563, 6564, 6565, 6566, 6567, 6568, + 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6577, 6578, 6579, 6580, + 6581, 6582, 6583, 6584, 6585, 6586, 6587, 6588, 6589, 6590, 6591, 6592, + 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6602, 6603, 6604, + 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6616, + 6617, 6618, 6619, 6620, 6621, 6622, 6623, 6624, 6625, 6626, 6627, 6628, + 6629, 6630, 6631, 6632, 6633, 6634, 6635, 6636, 6637, 6638, 6639, 6640, + 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6648, 6649, 6650, 6651, 6652, + 6653, 6654, 6655, 6656, 6657, 6658, 6659, 6660, 6661, 6662, 6663, 6664, + 6665, 6666, 6667, 6668, 6669, 6670, 6671, 6672, 6673, 6674, 6675, 6676, + 6677, 6678, 6679, 6680, 6681, 6682, 6683, 6684, 6685, 6686, 6687, 6688, + 6689, 6690, 6691, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, + 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, + 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, + 6725, 6726, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, + 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, + 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, + 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, + 6773, 6774, 6775, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 6784, + 6785, 6786, 6787, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, + 6797, 6798, 6799, 6800, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, + 6809, 6810, 6811, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, + 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, + 6833, 6834, 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, + 6845, 6846, 6847, 6848, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, + 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6865, 6866, 6867, 6868, + 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, + 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, + 6893, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, + 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, + 6917, 6918, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, + 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, + 6941, 6942, 6943, 6944, 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, + 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, + 6965, 6966, 6967, 6968, 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, + 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, + 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, + 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, + 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, + 7025, 7026, 7027, 7028, 7029, 7030, 7031, 7032, 7033, 7034, 7035, 7036, + 7037, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, + 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, + 7061, 7062, 7063, 7064, 6537, 6538, 6539, 6540, 6541, 6542, 6543, 6544, + 6545, 6546, 6547, 6548, 6549, 6550, 6551, 6552, 6523, 6524, 6525, 6526, + 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 19444, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 30045, 29974, 30037, 30046, + 29962, 30035, 29956, 29955, 30032, 30040, 29957, 30036, 29960, 29977, + 30048, 30044, 29969, 29971, 29968, 29967, 29964, 29963, 29966, 29965, + 29972, 29970, 29961, 30043, 30030, 29973, 29976, 30038, 29959, 29978, + 29979, 29980, 29981, 29982, 29983, 29984, 29985, 29986, 29987, 29988, + 29989, 29990, 29991, 29992, 29993, 29994, 29995, 29996, 29997, 29998, + 29999, 30000, 30001, 30002, 30003, 30033, 30042, 30041, 29958, 30034, + 29975, 30004, 30005, 30006, 30007, 30008, 30009, 30010, 30011, 30012, + 30013, 30014, 30015, 30016, 30017, 30018, 30019, 30020, 30021, 30022, + 30023, 30024, 30025, 30026, 30027, 30028, 30029, 30031, 30049, 30039, + 30047, 5694, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 33161, 33172, 33183, 33195, 33206, 33217, 33228, 33239, 33250, 33258, + 33259, 33260, 33261, 33263, 33264, 33265, 33266, 33267, 33268, 33269, + 33270, 33271, 33272, 33274, 33275, 33276, 33277, 33278, 33279, 33280, + 33281, 33282, 33283, 33285, 33286, 33287, 33288, 33289, 33290, 33291, + 33292, 33293, 33294, 33296, 33297, 33298, 33299, 33300, 33301, 33302, + 33303, 33304, 33305, 33307, 33308, 33309, 33310, 33311, 33312, 33313, + 33314, 33315, 33316, 33318, 33319, 33320, 33321, 33322, 33323, 33324, + 33325, 33326, 33327, 33329, 33330, 33331, 33332, 33333, 33334, 33335, + 33336, 33337, 33338, 33085, 33086, 33087, 33088, 33089, 33090, 33091, + 33092, 33093, 33094, 33096, 33097, 33098, 33099, 33100, 33101, 33102, + 33103, 33104, 33105, 33107, 33108, 33109, 33110, 33111, 33112, 33113, + 33114, 33115, 33116, 33118, 33119, 33120, 33121, 33122, 33123, 33124, + 33125, 33126, 33127, 33129, 33130, 33131, 33132, 33133, 33134, 33135, + 33136, 33137, 33138, 33140, 33141, 33142, 33143, 33144, 33145, 33146, + 33147, 33148, 33149, 33151, 33152, 33153, 33154, 33155, 33156, 33157, + 33158, 33159, 33160, 33162, 33163, 33164, 33165, 33166, 33167, 33168, + 33169, 33170, 33171, 33173, 33174, 33175, 33176, 33177, 33178, 33179, + 33180, 33181, 33182, 33184, 33185, 33186, 33187, 33188, 33189, 33190, + 33191, 33192, 33193, 33196, 33197, 33198, 33199, 33200, 33201, 33202, + 33203, 33204, 33205, 33207, 33208, 33209, 33210, 33211, 33212, 33213, + 33214, 33215, 33216, 33218, 33219, 33220, 33221, 33222, 33223, 33224, + 33225, 33226, 33227, 33229, 33230, 33231, 33232, 33233, 33234, 33235, + 33236, 33237, 33238, 33240, 33241, 33242, 33243, 33244, 33245, 33246, + 33247, 33248, 33249, 33251, 33252, 33253, 33254, 33255, 33256, 33257, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 25143, 25142, 29575, 29166, + 29576, 29601, 12457, 13030, 12455, 12468, 12461, 12460, 3, 2, 347, 3522, + 2596, 4957, 5989, 15959, 15985, 29953, 19875, 24468, 12458, 20659, 25216, + 12466, 19877, 33665, 33770, 13171, 13320, 5762, 8375, 27914, 20491, + 28965, 27915, 20490, 27945, 10237, 11210, 10519, 10238, 10518, 10239, + 10517, 10240, 10515, 10241, 24336, 24258, 29858, 29857, 12456, 13029, + 5692, 4965, 12454, 12467, 12421, 29632, 29602, 12502, 12501, 16242, + 13120, 13318, 16243, 14274, 14522, 16244, 26992, 27539, 16245, 32630, + 32782, 29168, 10253, 10250, 26093, 26092, 15830, 15984, 4626, 4956, + 24641, 24260, 16169, 16168, 24570, 24575, 29572, 29559, 12453, 12505, + 5991, 15961, 15987, 5990, 15960, 15986, 19878, 33666, 33771, 26416, + 26415, 26793, 26417, 26418, 26773, 27072, 27079, 27107, 28731, 28732, + 29557, 28730, 28733, 29558, 10516, 10242, 26881, 26882, 26929, 26880, + 26883, 26930, 27754, 29600, 5693, 10222, 22730, 24084, 29571, 29574, + 29169, 12452, 12450, 13063, 29573, 29167, 28725, 29950, 28724, 27728, + 8169, 10221, 29595, 29560, 25790, 26010, 26879, 26931, 1048, 1049, 24259, + 27944, 18311, 18929, 10220, 2297, 350, 29936, 16751, 18043, 18044, 18086, + 18052, 32125, 14921, 14922, 14899, 14920, 13314, 13315, 13316, 24082, + 13317, 29657, 35760, 35759, 35761, 20655, 27343, 20653, 27341, 26489, + 20656, 27344, 25215, 24083, 34189, 20654, 27342, 13319, 26490, 33962, + 22891, 22892, 19628, 27299, 34696, 23890, 33340, 33451, 33519, 33530, + 33541, 33552, 33563, 33574, 33585, 33341, 33352, 33363, 33374, 33385, + 33396, 33407, 26848, 4955, 4248, 35758, 9338, 9337, 22191, 22189, 22249, + 22247, 15743, 4792, 33418, 33429, 33440, 33452, 33463, 33474, 33485, + 33496, 33507, 33515, 33516, 33517, 33518, 33520, 33521, 33522, 33523, + 33524, 33525, 33526, 33527, 33528, 33529, 33531, 33532, 33533, 33534, + 33535, 33536, 33537, 33538, 33539, 33540, 33542, 33543, 33544, 33545, + 33546, 33547, 33548, 33549, 33550, 33551, 33553, 33554, 33555, 33556, + 33557, 33558, 33559, 33560, 33561, 33562, 33564, 33565, 33566, 33567, + 33568, 33569, 33570, 33571, 33572, 33573, 33575, 33576, 33577, 33578, + 33579, 33580, 33581, 33582, 33583, 33584, 33586, 33587, 33588, 33589, + 33590, 33591, 33592, 33593, 33594, 33595, 33342, 33343, 33344, 33345, + 33346, 33347, 33348, 33349, 33350, 33351, 33353, 33354, 33355, 33356, + 33357, 33358, 33359, 33360, 33361, 33362, 33364, 33365, 33366, 33367, + 33368, 33369, 33370, 33371, 33372, 33373, 33375, 33376, 33377, 33378, + 33379, 33380, 33381, 33382, 33383, 33384, 33386, 33387, 33388, 33389, + 33390, 33391, 33392, 33393, 33394, 33395, 33397, 33398, 33399, 33400, + 33401, 33402, 33403, 33404, 33405, 33406, 33408, 33409, 33410, 33411, + 33412, 33413, 33414, 33415, 33416, 33417, 33419, 33420, 33421, 33422, + 33423, 33424, 33425, 33426, 33427, 33428, 33430, 33431, 33432, 33433, + 33434, 33435, 33436, 33437, 33438, 33439, 33441, 33442, 33443, 33444, + 33445, 33446, 33447, 33448, 33449, 33450, 33453, 33454, 33455, 33456, + 33457, 33458, 33459, 33460, 33461, 33462, 33464, 33465, 33466, 33467, + 33468, 33469, 33470, 33471, 33472, 33473, 33475, 33476, 33477, 33478, + 33479, 33480, 33481, 33482, 33483, 33484, 33486, 33487, 33488, 33489, + 33490, 33491, 33492, 33493, 33494, 33495, 33497, 33498, 33499, 33500, + 33501, 33502, 33503, 33504, 33505, 33506, 33508, 33509, 33510, 33511, + 33512, 33513, 33514, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 17147, 17148, 17155, 17157, 17154, 17153, 17150, 17149, + 17152, 17151, 17158, 17156, 18176, 18735, 18335, 18960, 18570, 19331, + 18270, 18870, 18271, 18871, 18272, 18872, 18444, 19110, 18445, 19111, + 18446, 19112, 18509, 19207, 18254, 18853, 18250, 18849, 18955, 19078, + 18186, 18745, 18187, 18746, 18275, 18876, 18276, 18877, 18260, 18859, + 18261, 18860, 18954, 18956, 18340, 18962, 18341, 18963, 18358, 18990, + 18388, 19030, 18394, 19051, 18484, 19169, 18574, 19335, 18575, 19336, + 18571, 19332, 18572, 19333, 18754, 19127, 19128, 19286, 19287, 19216, + 19217, 18944, 18945, 2267, 2270, 2268, 2272, 2274, 2271, 2273, 2269, + 2275, 10448, 10443, 10442, 10449, 10444, 10445, 10447, 10446, 3594, 3593, + 3595, 14488, 14487, 14486, 14485, 14490, 14489, 25862, 25861, 3539, + 30399, 30407, 30416, 30408, 30410, 30405, 30409, 30404, 30420, 30419, + 30422, 30414, 30401, 30421, 30403, 30402, 30415, 30406, 30418, 30412, + 30413, 30411, 30417, 30400, 30538, 30545, 30546, 30541, 30542, 30547, + 30548, 30539, 30543, 30544, 30540, 30604, 30611, 30612, 30607, 30608, + 30613, 30614, 30605, 30609, 30610, 30606, 30715, 30722, 30723, 30718, + 30719, 30724, 30725, 30716, 30720, 30721, 30717, 30615, 30622, 30623, + 30618, 30619, 30624, 30625, 30616, 30620, 30621, 30617, 30693, 30700, + 30701, 30696, 30697, 30702, 30703, 30694, 30698, 30699, 30695, 30593, + 30600, 30601, 30596, 30597, 30602, 30603, 30594, 30598, 30599, 30595, + 30704, 30711, 30712, 30707, 30708, 30713, 30714, 30705, 30709, 30710, + 30706, 30626, 30633, 30634, 30629, 30630, 30635, 30636, 30627, 30631, + 30632, 30628, 30759, 30766, 30767, 30762, 30763, 30768, 30769, 30760, + 30764, 30765, 30761, 30748, 30755, 30756, 30751, 30752, 30757, 30758, + 30749, 30753, 30754, 30750, 30781, 30788, 30789, 30784, 30785, 30790, + 30791, 30782, 30786, 30787, 30783, 30648, 30655, 30656, 30651, 30652, + 30657, 30658, 30649, 30653, 30654, 30650, 30571, 30578, 30579, 30574, + 30575, 30580, 30581, 30572, 30576, 30577, 30573, 30770, 30777, 30778, + 30773, 30774, 30779, 30780, 30771, 30775, 30776, 30772, 30549, 30556, + 30557, 30552, 30553, 30558, 30559, 30550, 30554, 30555, 30551, 30560, + 30567, 30568, 30563, 30564, 30569, 30570, 30561, 30565, 30566, 30562, + 30637, 30644, 30645, 30640, 30641, 30646, 30647, 30638, 30642, 30643, + 30639, 30582, 30589, 30590, 30585, 30586, 30591, 30592, 30583, 30587, + 30588, 30584, 30737, 30744, 30745, 30740, 30741, 30746, 30747, 30738, + 30742, 30743, 30739, 30659, 30667, 30668, 30662, 30663, 30669, 30670, + 30660, 30664, 30665, 30661, 30671, 30678, 30679, 30674, 30675, 30680, + 30681, 30672, 30676, 30677, 30673, 30682, 30689, 30690, 30685, 30686, + 30691, 30692, 30683, 30687, 30688, 30684, 30726, 30733, 30734, 30729, + 30730, 30735, 30736, 30727, 30731, 30732, 30728, 30526, 30527, 30534, + 30535, 30530, 30531, 30536, 30537, 30528, 30532, 30533, 30529, 30666, + 28756, 28754, 28755, 13411, 17710, 17708, 17711, 17709, 17700, 17706, + 17704, 17707, 17705, 17701, 17721, 17717, 17722, 17718, 17702, 17719, + 17715, 17720, 17716, 17703, 17732, 17712, 17714, 17713, 17728, 17731, + 17729, 17724, 17730, 17725, 17726, 17727, 17733, 17723, 17872, 17749, + 17750, 17751, 17748, 17867, 17859, 15860, 15862, 15864, 15861, 15863, + 16853, 16855, 16857, 16854, 16856, 16846, 16845, 16844, 16847, 23097, + 23102, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, + 35762, 35762, }; -#define code_magic 47 -#define code_size 65536 -#define code_poly 65581 static const unsigned int aliases_start = 0xf0000; static const unsigned int aliases_end = 0xf01d9; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 3935c00fc26530..d86741e1dfc18c 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,7 +62,11 @@ pass */ -#define Py_LIMITED_API 0x030b0000 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif #include "Python.h" #include @@ -391,6 +395,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 1ff3ef1cb6f296..1063e54217b746 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,7 +5,11 @@ * See the xxlimited module for an extension module template. */ -#define Py_LIMITED_API 0x03050000 +// Test the limited C API version 3.5 +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x03050000 +#endif #include "Python.h" @@ -293,7 +297,10 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, +#ifdef Py_GIL_DISABLED + // These definitions are in the limited API, but not until 3.13. + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3..a46bf8f0e64ee2 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -384,6 +384,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 560f43e5b3a643..9c548f44558d41 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -288,6 +288,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 9b76afa0e56f76..c5aaf22eeb2948 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -489,8 +489,8 @@ zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; case Z_MEM_ERROR: @@ -915,8 +915,8 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1293,8 +1293,8 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1495,8 +1495,8 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length) err = inflate(&self->zst, Z_SYNC_FLUSH); Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1896,12 +1896,20 @@ zlib_crc32_impl(PyObject *module, Py_buffer *data, unsigned int value) Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. crc32() takes - length as an unsigned int, which may be narrower than Py_ssize_t. */ - while ((size_t)len > UINT_MAX) { - value = crc32(value, buf, UINT_MAX); - buf += (size_t) UINT_MAX; - len -= (size_t) UINT_MAX; + length as an unsigned int, which may be narrower than Py_ssize_t. + We further limit size due to bugs in Apple's macOS zlib. + See https://github.com/python/cpython/issues/105967. + */ +#define ZLIB_CRC_CHUNK_SIZE 0x40000000 +#if ZLIB_CRC_CHUNK_SIZE > INT_MAX +# error "unsupported less than 32-bit platform?" +#endif + while ((size_t)len > ZLIB_CRC_CHUNK_SIZE) { + value = crc32(value, buf, ZLIB_CRC_CHUNK_SIZE); + buf += (size_t) ZLIB_CRC_CHUNK_SIZE; + len -= (size_t) ZLIB_CRC_CHUNK_SIZE; } +#undef ZLIB_CRC_CHUNK_SIZE value = crc32(value, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { @@ -2098,6 +2106,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 806ca6584bda95..afb068718bb010 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -5,6 +5,7 @@ #include "pycore_pybuffer.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() +#include "pycore_crossinterp.h" // _Py_CallInInterpreter() #include "pycore_object.h" // _Py_CheckSlotResult() #include "pycore_long.h" // _Py_IsNegative #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -203,12 +204,7 @@ int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) { if (PyDict_CheckExact(obj)) { - *result = PyDict_GetItemWithError(obj, key); /* borrowed */ - if (*result) { - Py_INCREF(*result); - return 1; - } - return PyErr_Occurred() ? -1 : 0; + return PyDict_GetItemRef(obj, key, result); } *result = PyObject_GetItem(obj, key); @@ -429,6 +425,12 @@ PyObject_AsWriteBuffer(PyObject *obj, int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + } PyBufferProcs *pb = Py_TYPE(obj)->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { @@ -765,11 +767,17 @@ PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, return -1; } - if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && - (readonly == 1)) { - PyErr_SetString(PyExc_BufferError, - "Object is not writable."); - return -1; + if (flags != PyBUF_SIMPLE) { /* fast path */ + if (flags == PyBUF_READ || flags == PyBUF_WRITE) { + PyErr_BadInternalCall(); + return -1; + } + if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && + (readonly == 1)) { + PyErr_SetString(PyExc_BufferError, + "Object is not writable."); + return -1; + } } view->obj = Py_XNewRef(obj); @@ -1184,29 +1192,10 @@ PyNumber_Multiply(PyObject *v, PyObject *w) return result; } -PyObject * -PyNumber_MatrixMultiply(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_matrix_multiply), "@"); -} - -PyObject * -PyNumber_FloorDivide(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_floor_divide), "//"); -} - -PyObject * -PyNumber_TrueDivide(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_true_divide), "/"); -} - -PyObject * -PyNumber_Remainder(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_remainder), "%"); -} +BINARY_FUNC(PyNumber_MatrixMultiply, nb_matrix_multiply, "@") +BINARY_FUNC(PyNumber_FloorDivide, nb_floor_divide, "//") +BINARY_FUNC(PyNumber_TrueDivide, nb_true_divide, "/") +BINARY_FUNC(PyNumber_Remainder, nb_remainder, "%") PyObject * PyNumber_Power(PyObject *v, PyObject *w, PyObject *z) @@ -1383,73 +1372,27 @@ _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs) /* Unary operators and functions */ -PyObject * -PyNumber_Negative(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_negative) { - PyObject *res = (*m->nb_negative)(o); - assert(_Py_CheckSlotResult(o, "__neg__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary -: '%.200s'", o); -} - -PyObject * -PyNumber_Positive(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_positive) { - PyObject *res = (*m->nb_positive)(o); - assert(_Py_CheckSlotResult(o, "__pos__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary +: '%.200s'", o); -} - -PyObject * -PyNumber_Invert(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_invert) { - PyObject *res = (*m->nb_invert)(o); - assert(_Py_CheckSlotResult(o, "__invert__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary ~: '%.200s'", o); -} - -PyObject * -PyNumber_Absolute(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_absolute) { - PyObject *res = m->nb_absolute(o); - assert(_Py_CheckSlotResult(o, "__abs__", res != NULL)); - return res; - } - - return type_error("bad operand type for abs(): '%.200s'", o); -} +#define UNARY_FUNC(func, op, meth_name, descr) \ + PyObject * \ + func(PyObject *o) { \ + if (o == NULL) { \ + return null_error(); \ + } \ + \ + PyNumberMethods *m = Py_TYPE(o)->tp_as_number; \ + if (m && m->op) { \ + PyObject *res = (*m->op)(o); \ + assert(_Py_CheckSlotResult(o, #meth_name, res != NULL)); \ + return res; \ + } \ + \ + return type_error("bad operand type for "descr": '%.200s'", o); \ + } + +UNARY_FUNC(PyNumber_Negative, nb_negative, __neg__, "unary -") +UNARY_FUNC(PyNumber_Positive, nb_positive, __pos__, "unary +") +UNARY_FUNC(PyNumber_Invert, nb_invert, __invert__, "unary ~") +UNARY_FUNC(PyNumber_Absolute, nb_absolute, __abs__, "abs()") int @@ -1578,7 +1521,6 @@ PyNumber_Long(PyObject *o) { PyObject *result; PyNumberMethods *m; - PyObject *trunc_func; Py_buffer view; if (o == NULL) { @@ -1620,37 +1562,6 @@ PyNumber_Long(PyObject *o) if (m && m->nb_index) { return PyNumber_Index(o); } - trunc_func = _PyObject_LookupSpecial(o, &_Py_ID(__trunc__)); - if (trunc_func) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The delegation of int() to __trunc__ is deprecated.", 1)) { - Py_DECREF(trunc_func); - return NULL; - } - result = _PyObject_CallNoArgs(trunc_func); - Py_DECREF(trunc_func); - if (result == NULL || PyLong_CheckExact(result)) { - return result; - } - if (PyLong_Check(result)) { - Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); - return result; - } - /* __trunc__ is specified to return an Integral type, - but int() needs to return an int. */ - if (!PyIndex_Check(result)) { - PyErr_Format( - PyExc_TypeError, - "__trunc__ returned non-Integral (type %.200s)", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - return NULL; - } - Py_SETREF(result, PyNumber_Index(result)); - return result; - } - if (PyErr_Occurred()) - return NULL; if (PyUnicode_Check(o)) /* The below check is done in PyLong_FromUnicodeObject(). */ @@ -2230,7 +2141,7 @@ PySequence_Fast(PyObject *v, const char *m) PY_ITERSEARCH_COUNT: -1 if error, else # of times obj appears in seq. PY_ITERSEARCH_INDEX: 0-based index of first occurrence of obj in seq; set ValueError and return -1 if none found; also return -1 on error. - Py_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. + PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. */ Py_ssize_t _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) @@ -2247,7 +2158,15 @@ _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) it = PyObject_GetIter(seq); if (it == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - type_error("argument of type '%.200s' is not iterable", seq); + if (operation == PY_ITERSEARCH_CONTAINS) { + type_error( + "argument of type '%.200s' is not a container or iterable", + seq + ); + } + else { + type_error("argument of type '%.200s' is not iterable", seq); + } } return -1; } @@ -2467,31 +2386,53 @@ PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) } int -PyMapping_HasKeyString(PyObject *o, const char *key) +PyMapping_HasKeyString(PyObject *obj, const char *key) { - PyObject *v; - - v = PyMapping_GetItemString(o, key); - if (v) { - Py_DECREF(v); - return 1; + PyObject *value; + int rc; + if (obj == NULL) { + // For backward compatibility. + // PyMapping_GetOptionalItemString() crashes if obj is NULL. + null_error(); + rc = -1; } - PyErr_Clear(); - return 0; + else { + rc = PyMapping_GetOptionalItemString(obj, key, &value); + } + if (rc < 0) { + PyErr_FormatUnraisable( + "Exception ignored in PyMapping_HasKeyString(); consider using " + "PyMapping_HasKeyStringWithError(), " + "PyMapping_GetOptionalItemString() or PyMapping_GetItemString()"); + return 0; + } + Py_XDECREF(value); + return rc; } int -PyMapping_HasKey(PyObject *o, PyObject *key) +PyMapping_HasKey(PyObject *obj, PyObject *key) { - PyObject *v; - - v = PyObject_GetItem(o, key); - if (v) { - Py_DECREF(v); - return 1; + PyObject *value; + int rc; + if (obj == NULL || key == NULL) { + // For backward compatibility. + // PyMapping_GetOptionalItem() crashes if any of them is NULL. + null_error(); + rc = -1; } - PyErr_Clear(); - return 0; + else { + rc = PyMapping_GetOptionalItem(obj, key, &value); + } + if (rc < 0) { + PyErr_FormatUnraisable( + "Exception ignored in PyMapping_HasKey(); consider using " + "PyMapping_HasKeyWithError(), " + "PyMapping_GetOptionalItem() or PyObject_GetItem()"); + return 0; + } + Py_XDECREF(value); + return rc; } /* This function is quite similar to PySequence_Fast(), but specialized to be diff --git a/Objects/boolobject.c b/Objects/boolobject.c index e2e359437f0edf..fb48dcbeca7850 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -110,9 +110,10 @@ bool_xor(PyObject *a, PyObject *b) /* Doc string */ PyDoc_STRVAR(bool_doc, -"bool(x) -> bool\n\ +"bool(object=False, /)\n\ +--\n\ \n\ -Returns True when the argument x is true, False otherwise.\n\ +Returns True when the argument is true, False otherwise.\n\ The builtins True and False are the only two instances of the class bool.\n\ The class bool is a subclass of the class int, and cannot be subclassed."); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 67073190cc889d..80679f93cd4c13 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -132,7 +132,7 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) } else { alloc = size + 1; - new->ob_bytes = PyObject_Malloc(alloc); + new->ob_bytes = PyMem_Malloc(alloc); if (new->ob_bytes == NULL) { Py_DECREF(new); return PyErr_NoMemory(); @@ -221,17 +221,17 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) } if (logical_offset > 0) { - sval = PyObject_Malloc(alloc); + sval = PyMem_Malloc(alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; } memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyObject_Free(obj->ob_bytes); + PyMem_Free(obj->ob_bytes); } else { - sval = PyObject_Realloc(obj->ob_bytes, alloc); + sval = PyMem_Realloc(obj->ob_bytes, alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; @@ -951,7 +951,7 @@ bytearray_repr(PyByteArrayObject *self) } newsize += 6 + length * 4; - buffer = PyObject_Malloc(newsize); + buffer = PyMem_Malloc(newsize); if (buffer == NULL) { PyErr_NoMemory(); return NULL; @@ -1008,7 +1008,7 @@ bytearray_repr(PyByteArrayObject *self) } v = PyUnicode_FromStringAndSize(buffer, p - buffer); - PyObject_Free(buffer); + PyMem_Free(buffer); return v; } @@ -1088,7 +1088,7 @@ bytearray_dealloc(PyByteArrayObject *self) PyErr_Print(); } if (self->ob_bytes != 0) { - PyObject_Free(self->ob_bytes); + PyMem_Free(self->ob_bytes); } Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1121,16 +1121,44 @@ bytearray_dealloc(PyByteArrayObject *self) #include "stringlib/transmogrify.h" +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytearray.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_find(PyByteArrayObject *self, PyObject *args) +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=413e1cab2ae87da0 input=793dfad803e2952f]*/ { - return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.count = bytearray.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytearray_count(PyByteArrayObject *self, PyObject *args) +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a21ee2692e4f1233 input=4deb529db38deda8]*/ { - return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } /*[clinic input] @@ -1162,22 +1190,55 @@ bytearray_copy_impl(PyByteArrayObject *self) PyByteArray_GET_SIZE(self)); } +/*[clinic input] +bytearray.index = bytearray.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_index(PyByteArrayObject *self, PyObject *args) +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=067a1e78efc672a7 input=8cbaf6836dbd2a9a]*/ { - return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rfind = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_rfind(PyByteArrayObject *self, PyObject *args) +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=51bf886f932b283c input=eaa107468a158423]*/ { - return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rindex = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_rindex(PyByteArrayObject *self, PyObject *args) +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=38e1cf66bafb08b9 input=81cf49d0af4d5bd0]*/ { - return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } static int @@ -1186,16 +1247,52 @@ bytearray_contains(PyObject *self, PyObject *arg) return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytearray.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_startswith(PyByteArrayObject *self, PyObject *args) +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a3d9b6d44d3662a6 input=76385e0b376b45c1]*/ { - return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytearray.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_endswith(PyByteArrayObject *self, PyObject *args) +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=e75ea8c227954caa input=9b8baa879aa3d74b]*/ { - return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } /*[clinic input] @@ -1729,6 +1826,10 @@ bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) while ((item = PyIter_Next(it)) != NULL) { if (! _getbytevalue(item, &value)) { + if (PyErr_ExceptionMatches(PyExc_TypeError) && PyUnicode_Check(iterable_of_ints)) { + PyErr_Format(PyExc_TypeError, + "expected iterable of integers; got: 'str'"); + } Py_DECREF(item); Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2007,7 +2108,10 @@ static PyObject * bytearray_join(PyByteArrayObject *self, PyObject *iterable_of_bytes) /*[clinic end generated code: output=a8516370bf68ae08 input=aba6b1f9b30fcb8e]*/ { - return stringlib_bytes_join((PyObject*)self, iterable_of_bytes); + self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator + PyObject* ret = stringlib_bytes_join((PyObject*)self, iterable_of_bytes); + self->ob_exports--; // unexport `self` + return ret; } /*[clinic input] @@ -2193,18 +2297,15 @@ bytearray_methods[] = { STRINGLIB_CENTER_METHODDEF BYTEARRAY_CLEAR_METHODDEF BYTEARRAY_COPY_METHODDEF - {"count", (PyCFunction)bytearray_count, METH_VARARGS, - _Py_count__doc__}, + BYTEARRAY_COUNT_METHODDEF BYTEARRAY_DECODE_METHODDEF - {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTEARRAY_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF BYTEARRAY_EXTEND_METHODDEF - {"find", (PyCFunction)bytearray_find, METH_VARARGS, - _Py_find__doc__}, + BYTEARRAY_FIND_METHODDEF BYTEARRAY_FROMHEX_METHODDEF BYTEARRAY_HEX_METHODDEF - {"index", (PyCFunction)bytearray_index, METH_VARARGS, _Py_index__doc__}, + BYTEARRAY_INDEX_METHODDEF BYTEARRAY_INSERT_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, @@ -2234,16 +2335,15 @@ bytearray_methods[] = { BYTEARRAY_REMOVEPREFIX_METHODDEF BYTEARRAY_REMOVESUFFIX_METHODDEF BYTEARRAY_REVERSE_METHODDEF - {"rfind", (PyCFunction)bytearray_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTEARRAY_RFIND_METHODDEF + BYTEARRAY_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTEARRAY_RPARTITION_METHODDEF BYTEARRAY_RSPLIT_METHODDEF BYTEARRAY_RSTRIP_METHODDEF BYTEARRAY_SPLIT_METHODDEF BYTEARRAY_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , - _Py_startswith__doc__}, + BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c1bc6383df30ce..c239ae18a593e3 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -92,57 +92,6 @@ _Py_bytes_isalnum(const char *cptr, Py_ssize_t len) } -PyDoc_STRVAR_shared(_Py_isascii__doc__, -"B.isascii() -> bool\n\ -\n\ -Return True if B is empty or all characters in B are ASCII,\n\ -False otherwise."); - -// Optimization is copied from ascii_decode in unicodeobject.c -/* Mask to quickly check whether a C 'size_t' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) -# define ASCII_CHAR_MASK 0x8080808080808080ULL -#elif (SIZEOF_SIZE_T == 4) -# define ASCII_CHAR_MASK 0x80808080U -#else -# error C 'size_t' size should be either 4 or 8! -#endif - -PyObject* -_Py_bytes_isascii(const char *cptr, Py_ssize_t len) -{ - const char *p = cptr; - const char *end = p + len; - - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) { - Py_RETURN_FALSE; - } - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) { - Py_RETURN_FALSE; - } - p++; - } - Py_RETURN_TRUE; -} - -#undef ASCII_CHAR_MASK - - PyDoc_STRVAR_shared(_Py_isdigit__doc__, "B.isdigit() -> bool\n\ \n\ @@ -438,6 +387,7 @@ _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) #include "stringlib/fastsearch.h" #include "stringlib/count.h" #include "stringlib/find.h" +#include "stringlib/find_max_char.h" /* Wraps stringlib_parse_args_finds() and additionally checks the first @@ -453,31 +403,21 @@ stringlib_parse_args_finds(). */ Py_LOCAL_INLINE(int) -parse_args_finds_byte(const char *function_name, PyObject *args, - PyObject **subobj, char *byte, - Py_ssize_t *start, Py_ssize_t *end) +parse_args_finds_byte(const char *function_name, PyObject **subobj, char *byte) { - PyObject *tmp_subobj; - Py_ssize_t ival; - - if(!stringlib_parse_args_finds(function_name, args, &tmp_subobj, - start, end)) - return 0; - - if (PyObject_CheckBuffer(tmp_subobj)) { - *subobj = tmp_subobj; + if (PyObject_CheckBuffer(*subobj)) { return 1; } - if (!_PyIndex_Check(tmp_subobj)) { + if (!_PyIndex_Check(*subobj)) { PyErr_Format(PyExc_TypeError, "argument should be integer or bytes-like object, " "not '%.200s'", - Py_TYPE(tmp_subobj)->tp_name); + Py_TYPE(*subobj)->tp_name); return 0; } - ival = PyNumber_AsSsize_t(tmp_subobj, NULL); + Py_ssize_t ival = PyNumber_AsSsize_t(*subobj, NULL); if (ival == -1 && PyErr_Occurred()) { return 0; } @@ -492,35 +432,40 @@ parse_args_finds_byte(const char *function_name, PyObject *args, } /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) Py_LOCAL_INLINE(Py_ssize_t) find_internal(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, int dir) + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, + int dir) { - PyObject *subobj; char byte; Py_buffer subbuf; const char *sub; Py_ssize_t sub_len; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_ssize_t res; - if (!parse_args_finds_byte(function_name, args, - &subobj, &byte, &start, &end)) + if (!parse_args_finds_byte(function_name, &subobj, &byte)) { return -2; + } if (subobj) { if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) @@ -566,37 +511,21 @@ find_internal(const char *str, Py_ssize_t len, return res; } -PyDoc_STRVAR_shared(_Py_find__doc__, -"B.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "find", args, +1); + Py_ssize_t result = find_internal(str, len, "find", sub, start, end, +1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_index__doc__, -"B.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the subsection is not found."); - PyObject * -_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "index", args, +1); + Py_ssize_t result = find_internal(str, len, "index", sub, start, end, +1); if (result == -2) return NULL; if (result == -1) { @@ -607,37 +536,21 @@ _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rfind__doc__, -"B.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rfind", args, -1); + Py_ssize_t result = find_internal(str, len, "rfind", sub, start, end, -1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rindex__doc__, -"B.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raise ValueError when the subsection is not found."); - PyObject * -_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rindex", args, -1); + Py_ssize_t result = find_internal(str, len, "rindex", sub, start, end, -1); if (result == -2) return NULL; if (result == -1) { @@ -648,28 +561,20 @@ _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_count__doc__, -"B.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of subsection sub in\n\ -bytes B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - PyObject * -_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub_obj, + Py_ssize_t start, Py_ssize_t end) { - PyObject *sub_obj; const char *sub; Py_ssize_t sub_len; char byte; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_buffer vsub; PyObject *count_obj; - if (!parse_args_finds_byte("count", args, - &sub_obj, &byte, &start, &end)) + if (!parse_args_finds_byte("count", &sub_obj, &byte)) { return NULL; + } if (sub_obj) { if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) @@ -771,66 +676,65 @@ tailmatch(const char *str, Py_ssize_t len, PyObject *substr, static PyObject * _Py_bytes_tailmatch(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj = NULL; - int result; - - if (!stringlib_parse_args_finds(function_name, args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = tailmatch(str, len, PyTuple_GET_ITEM(subobj, i), - start, end, direction); - if (result == -1) + PyObject *item = PyTuple_GET_ITEM(subobj, i); + int result = tailmatch(str, len, item, start, end, direction); + if (result < 0) { return NULL; + } else if (result) { Py_RETURN_TRUE; } } Py_RETURN_FALSE; } - result = tailmatch(str, len, subobj, start, end, direction); + int result = tailmatch(str, len, subobj, start, end, direction); if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) + if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Format(PyExc_TypeError, "%s first arg must be bytes or a tuple of bytes, " "not %s", function_name, Py_TYPE(subobj)->tp_name); + } return NULL; } - else - return PyBool_FromLong(result); + return PyBool_FromLong(result); } -PyDoc_STRVAR_shared(_Py_startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); +PyObject * +_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +{ + return _Py_bytes_tailmatch(str, len, "startswith", subobj, start, end, -1); +} PyObject * -_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) { - return _Py_bytes_tailmatch(str, len, "startswith", args, -1); + return _Py_bytes_tailmatch(str, len, "endswith", subobj, start, end, +1); } -PyDoc_STRVAR_shared(_Py_endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ +PyDoc_STRVAR_shared(_Py_isascii__doc__, +"B.isascii() -> bool\n\ \n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); +Return True if B is empty or all characters in B are ASCII,\n\ +False otherwise."); -PyObject * -_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args) +PyObject* +_Py_bytes_isascii(const char *cptr, Py_ssize_t len) { - return _Py_bytes_tailmatch(str, len, "endswith", args, +1); + const char *p = cptr; + const char *end = p + len; + Py_ssize_t max_char = stringlib_find_max_char(cptr, end); + if (max_char > 127) { + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 26227dd251122d..459df6ceacf3a8 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -52,25 +52,25 @@ static inline PyObject* bytes_get_empty(void) /* - For PyBytes_FromString(), the parameter `str' points to a null-terminated - string containing exactly `size' bytes. + For PyBytes_FromString(), the parameter 'str' points to a null-terminated + string containing exactly 'size' bytes. - For PyBytes_FromStringAndSize(), the parameter `str' is - either NULL or else points to a string containing at least `size' bytes. - For PyBytes_FromStringAndSize(), the string in the `str' parameter does + For PyBytes_FromStringAndSize(), the parameter 'str' is + either NULL or else points to a string containing at least 'size' bytes. + For PyBytes_FromStringAndSize(), the string in the 'str' parameter does not have to be null-terminated. (Therefore it is safe to construct a - substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) - If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' + substring by calling 'PyBytes_FromStringAndSize(origstring, substrlen)'.) + If 'str' is NULL then PyBytes_FromStringAndSize() will allocate 'size+1' bytes (setting the last byte to the null terminating character) and you can - fill in the data yourself. If `str' is non-NULL then the resulting + fill in the data yourself. If 'str' is non-NULL then the resulting PyBytes object must be treated as immutable and you must not fill in nor alter the data yourself, since the strings may be shared. - The PyObject member `op->ob_size', which denotes the number of "extra + The PyObject member 'op->ob_size', which denotes the number of "extra items" in a variable-size object, will contain the number of bytes allocated for string data, not counting the null terminating character. - It is therefore equal to the `size' parameter (for - PyBytes_FromStringAndSize()) or the length of the string in the `str' + It is therefore equal to the 'size' parameter (for + PyBytes_FromStringAndSize()) or the length of the string in the 'str' parameter (for PyBytes_FromString()). */ static PyObject * @@ -477,21 +477,32 @@ formatlong(PyObject *v, int flags, int prec, int type) static int byte_converter(PyObject *arg, char *p) { - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + return 0; + } *p = PyBytes_AS_STRING(arg)[0]; return 1; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + return 0; + } *p = PyByteArray_AS_STRING(arg)[0]; return 1; } - else { + else if (PyIndex_Check(arg)) { int overflow; long ival = PyLong_AsLongAndOverflow(arg, &overflow); if (ival == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; - } return 0; } if (!(0 <= ival && ival <= 255)) { @@ -503,9 +514,9 @@ byte_converter(PyObject *arg, char *p) *p = (char)ival; return 1; } - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires an integer in range(256) or a single byte"); + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or a single byte, not %T", + arg); return 0; } @@ -1863,30 +1874,80 @@ _PyBytes_Join(PyObject *sep, PyObject *x) return bytes_join((PyBytesObject*)sep, x); } +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytes.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytes_find(PyBytesObject *self, PyObject *args) +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=d5961a1c77b472a1 input=3171e62a8ae7f240]*/ { - return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.index = bytes.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytes_index(PyBytesObject *self, PyObject *args) +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=0da25cc74683ba42 input=aa34ad71ba0bafe3]*/ { - return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rfind = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ static PyObject * -bytes_rfind(PyBytesObject *self, PyObject *args) +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51b60fa4ad011c09 input=864c3e7f3010b33c]*/ { - return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rindex = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ static PyObject * -bytes_rindex(PyBytesObject *self, PyObject *args) +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=42bf674e0a0aabf6 input=21051fc5cfeacf2c]*/ { - return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2023,10 +2084,19 @@ bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) } +/*[clinic input] +bytes.count = bytes.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytes_count(PyBytesObject *self, PyObject *args) +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=9848140b9be17d0f input=b6e4a5ed515e1e59]*/ { - return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2285,16 +2355,52 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) return PyBytes_FromStringAndSize(self_start, self_len); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytes.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_startswith(PyBytesObject *self, PyObject *args) +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=b1e8da1cbd528e8c input=8a4165df8adfa6c9]*/ { - return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytes.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_endswith(PyBytesObject *self, PyObject *args) +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=038b633111f3629d input=b5c3407a2a5c9aac]*/ { - return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } @@ -2488,17 +2594,14 @@ bytes_methods[] = { {"capitalize", stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, STRINGLIB_CENTER_METHODDEF - {"count", (PyCFunction)bytes_count, METH_VARARGS, - _Py_count__doc__}, + BYTES_COUNT_METHODDEF BYTES_DECODE_METHODDEF - {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTES_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF - {"find", (PyCFunction)bytes_find, METH_VARARGS, - _Py_find__doc__}, + BYTES_FIND_METHODDEF BYTES_FROMHEX_METHODDEF BYTES_HEX_METHODDEF - {"index", (PyCFunction)bytes_index, METH_VARARGS, _Py_index__doc__}, + BYTES_INDEX_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, {"isalpha", stringlib_isalpha, METH_NOARGS, @@ -2524,16 +2627,15 @@ bytes_methods[] = { BYTES_REPLACE_METHODDEF BYTES_REMOVEPREFIX_METHODDEF BYTES_REMOVESUFFIX_METHODDEF - {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTES_RFIND_METHODDEF + BYTES_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTES_RPARTITION_METHODDEF BYTES_RSPLIT_METHODDEF BYTES_RSTRIP_METHODDEF BYTES_SPLIT_METHODDEF BYTES_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, - _Py_startswith__doc__}, + BYTES_STARTSWITH_METHODDEF BYTES_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, @@ -3025,11 +3127,9 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) /* The following function breaks the notion that bytes are immutable: - it changes the size of a bytes object. We get away with this only if there - is only one module referencing the object. You can also think of it + it changes the size of a bytes object. You can think of it as creating a new bytes object and destroying the old one, only - more efficiently. In any case, don't use this if the bytes object may - already be known to some other part of the code... + more efficiently. Note that if there's not enough memory to resize the bytes object, the original bytes object at *pv is deallocated, *pv is set to NULL, an "out of memory" exception is set, and -1 is returned. Else (on success) 0 is @@ -3045,28 +3145,40 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyBytesObject *sv; v = *pv; if (!PyBytes_Check(v) || newsize < 0) { - goto error; + *pv = 0; + Py_DECREF(v); + PyErr_BadInternalCall(); + return -1; } - if (Py_SIZE(v) == newsize) { + Py_ssize_t oldsize = PyBytes_GET_SIZE(v); + if (oldsize == newsize) { /* return early if newsize equals to v->ob_size */ return 0; } - if (Py_SIZE(v) == 0) { - if (newsize == 0) { - return 0; - } + if (oldsize == 0) { *pv = _PyBytes_FromSize(newsize, 0); Py_DECREF(v); return (*pv == NULL) ? -1 : 0; } - if (Py_REFCNT(v) != 1) { - goto error; - } if (newsize == 0) { *pv = bytes_get_empty(); Py_DECREF(v); return 0; } + if (Py_REFCNT(v) != 1) { + if (oldsize < newsize) { + *pv = _PyBytes_FromSize(newsize, 0); + if (*pv) { + memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), oldsize); + } + } + else { + *pv = PyBytes_FromStringAndSize(PyBytes_AS_STRING(v), newsize); + } + Py_DECREF(v); + return (*pv == NULL) ? -1 : 0; + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif @@ -3074,7 +3186,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_Free(v); PyErr_NoMemory(); @@ -3089,11 +3201,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS sv->ob_shash = -1; /* invalidate cached hash value */ _Py_COMP_DIAG_POP return 0; -error: - *pv = 0; - Py_DECREF(v); - PyErr_BadInternalCall(); - return -1; } diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f516707f6f8086..b1154e4ca4ace6 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -1,6 +1,8 @@ /* Cell object implementation */ #include "Python.h" +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" PyObject * @@ -55,8 +57,7 @@ PyCell_Get(PyObject *op) PyErr_BadInternalCall(); return NULL; } - PyObject *value = PyCell_GET(op); - return Py_XNewRef(value); + return PyCell_GetRef((PyCellObject *)op); } int @@ -66,9 +67,7 @@ PyCell_Set(PyObject *op, PyObject *value) PyErr_BadInternalCall(); return -1; } - PyObject *old_value = PyCell_GET(op); - PyCell_SET(op, Py_XNewRef(value)); - Py_XDECREF(old_value); + PyCell_SetTakeRef((PyCellObject *)op, Py_XNewRef(value)); return 0; } diff --git a/Objects/classobject.c b/Objects/classobject.c index 618d88894debbe..69a7d5f046e30d 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -188,15 +188,18 @@ method_getattro(PyObject *obj, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); } if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, obj, (PyObject *)Py_TYPE(obj)); + if (f != NULL) { + PyObject *res = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -301,7 +304,7 @@ static Py_hash_t method_hash(PyMethodObject *a) { Py_hash_t x, y; - x = _Py_HashPointer(a->im_self); + x = PyObject_GenericHash(a->im_self); y = PyObject_Hash(a->im_func); if (y == -1) return -1; @@ -319,6 +322,13 @@ method_traverse(PyMethodObject *im, visitproc visit, void *arg) return 0; } +static PyObject * +method_descr_get(PyObject *meth, PyObject *obj, PyObject *cls) +{ + Py_INCREF(meth); + return meth; +} + PyTypeObject PyMethod_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "method", @@ -339,6 +349,7 @@ PyTypeObject PyMethod_Type = { .tp_methods = method_methods, .tp_members = method_memberlist, .tp_getset = method_getset, + .tp_descr_get = method_descr_get, .tp_new = method_new, }; @@ -402,14 +413,17 @@ instancemethod_getattro(PyObject *self, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, self, (PyObject *)Py_TYPE(self)); + if (f != NULL) { + PyObject *res = f(descr, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 7039db09721998..c748c53e1c0a75 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static int bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, @@ -100,6 +101,106 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(bytearray_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytearray_find), METH_FASTCALL, bytearray_find__doc__}, + +static PyObject * +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytearray_find(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_find_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTEARRAY_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytearray_count), METH_FASTCALL, bytearray_count__doc__}, + +static PyObject * +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_count(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_count_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_clear__doc__, "clear($self, /)\n" "--\n" @@ -136,6 +237,261 @@ bytearray_copy(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) return bytearray_copy_impl(self); } +PyDoc_STRVAR(bytearray_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytearray_index), METH_FASTCALL, bytearray_index__doc__}, + +static PyObject * +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_index(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_index_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytearray_rfind), METH_FASTCALL, bytearray_rfind__doc__}, + +static PyObject * +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rfind(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rfind_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytearray_rindex), METH_FASTCALL, bytearray_rindex__doc__}, + +static PyObject * +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rindex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rindex_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytearray_startswith), METH_FASTCALL, bytearray_startswith__doc__}, + +static PyObject * +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_startswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytearray_endswith), METH_FASTCALL, bytearray_endswith__doc__}, + +static PyObject * +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_endswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_removeprefix__doc__, "removeprefix($self, prefix, /)\n" "--\n" @@ -161,10 +517,6 @@ bytearray_removeprefix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&prefix, 'C')) { - _PyArg_BadArgument("removeprefix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytearray_removeprefix_impl(self, &prefix); exit: @@ -201,10 +553,6 @@ bytearray_removesuffix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&suffix, 'C')) { - _PyArg_BadArgument("removesuffix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytearray_removesuffix_impl(self, &suffix); exit: @@ -315,17 +663,9 @@ bytearray_maketrans(void *null, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &frm, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&frm, 'C')) { - _PyArg_BadArgument("maketrans", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &to, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&to, 'C')) { - _PyArg_BadArgument("maketrans", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = bytearray_maketrans_impl(&frm, &to); exit: @@ -375,17 +715,9 @@ bytearray_replace(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&old, 'C')) { - _PyArg_BadArgument("replace", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&new, 'C')) { - _PyArg_BadArgument("replace", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -1284,4 +1616,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=94b9b5f492b5fed6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f861b02e3fa278b input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index e2f9ba6c1e55bd..0b4b37501735c1 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(bytes___bytes____doc__, "__bytes__($self, /)\n" @@ -140,10 +141,6 @@ bytes_partition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&sep, 'C')) { - _PyArg_BadArgument("partition", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_partition_impl(self, &sep); exit: @@ -183,10 +180,6 @@ bytes_rpartition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&sep, 'C')) { - _PyArg_BadArgument("rpartition", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_rpartition_impl(self, &sep); exit: @@ -301,6 +294,210 @@ PyDoc_STRVAR(bytes_join__doc__, #define BYTES_JOIN_METHODDEF \ {"join", (PyCFunction)bytes_join, METH_O, bytes_join__doc__}, +PyDoc_STRVAR(bytes_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytes_find), METH_FASTCALL, bytes_find__doc__}, + +static PyObject * +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_find(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_find_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytes_index), METH_FASTCALL, bytes_index__doc__}, + +static PyObject * +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_index(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_index_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytes_rfind), METH_FASTCALL, bytes_rfind__doc__}, + +static PyObject * +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rfind(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rfind_impl(self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytes_rindex), METH_FASTCALL, bytes_rindex__doc__}, + +static PyObject * +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rindex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rindex_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_strip__doc__, "strip($self, bytes=None, /)\n" "--\n" @@ -403,6 +600,55 @@ bytes_rstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(bytes_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytes_count), METH_FASTCALL, bytes_count__doc__}, + +static PyObject * +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_count(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_count_impl(self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -502,17 +748,9 @@ bytes_maketrans(void *null, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &frm, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&frm, 'C')) { - _PyArg_BadArgument("maketrans", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &to, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&to, 'C')) { - _PyArg_BadArgument("maketrans", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = bytes_maketrans_impl(&frm, &to); exit: @@ -562,17 +800,9 @@ bytes_replace(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&old, 'C')) { - _PyArg_BadArgument("replace", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&new, 'C')) { - _PyArg_BadArgument("replace", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -628,10 +858,6 @@ bytes_removeprefix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&prefix, 'C')) { - _PyArg_BadArgument("removeprefix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_removeprefix_impl(self, &prefix); exit: @@ -668,10 +894,6 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&suffix, 'C')) { - _PyArg_BadArgument("removesuffix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_removesuffix_impl(self, &suffix); exit: @@ -683,6 +905,108 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytes_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytes_startswith), METH_FASTCALL, bytes_startswith__doc__}, + +static PyObject * +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytes_startswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytes_endswith), METH_FASTCALL, bytes_endswith__doc__}, + +static PyObject * +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_endswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_decode__doc__, "decode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -1060,4 +1384,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8a9f5c28cbfe7592 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d6801c6001e57f91 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/classobject.c.h b/Objects/clinic/classobject.c.h index 48cfd6c7b78ca3..3e149c97324a6a 100644 --- a/Objects/clinic/classobject.c.h +++ b/Objects/clinic/classobject.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(method___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -80,4 +82,4 @@ instancemethod_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a0d17bad3b0734d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a5e3f2d0726f189 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index b20b066f494901..68e2d7f528c061 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(code_new__doc__, "code(argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize,\n" @@ -463,4 +464,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=b9ccfbfabe1a5f46 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d604263a3ca72a0f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index fa871b7d34c2d7..58fd4e26871b4d 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_BadArgument() PyDoc_STRVAR(complex_conjugate__doc__, "conjugate($self, /)\n" @@ -93,9 +94,12 @@ PyDoc_STRVAR(complex_new__doc__, "complex(real=0, imag=0)\n" "--\n" "\n" -"Create a complex number from a real part and an optional imaginary part.\n" +"Create a complex number from a string or numbers.\n" "\n" -"This is equivalent to (real + imag*1j) where imag defaults to 0."); +"If a string is given, parse it as a complex number.\n" +"If a single number is given, convert it to a complex number.\n" +"If the \'real\' or \'imag\' arguments are given, create a complex number\n" +"with the specified real and imaginary components."); static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i); @@ -165,4 +169,4 @@ PyDoc_STRVAR(complex_from_number__doc__, #define COMPLEX_FROM_NUMBER_METHODDEF \ {"from_number", (PyCFunction)complex_from_number, METH_O|METH_CLASS, complex_from_number__doc__}, -/*[clinic end generated code: output=6a99688764c101d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=188438cc9ae167f7 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/descrobject.c.h b/Objects/clinic/descrobject.c.h index 4f18fd77a2c7a0..d79be80d3ec165 100644 --- a/Objects/clinic/descrobject.c.h +++ b/Objects/clinic/descrobject.c.h @@ -6,6 +6,13 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(mappingproxy_new__doc__, +"mappingproxy(mapping)\n" +"--\n" +"\n" +"Read-only proxy of a mapping."); static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping); @@ -166,4 +173,4 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a97dc44d12f9f9b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=050e331316a04207 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index eda86c31fcc578..fb46c4c64334f9 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(dict_fromkeys__doc__, "fromkeys($type, iterable, value=None, /)\n" "--\n" @@ -36,6 +39,24 @@ dict_fromkeys(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(dict_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of the dict."); + +#define DICT_COPY_METHODDEF \ + {"copy", (PyCFunction)dict_copy, METH_NOARGS, dict_copy__doc__}, + +static PyObject * +dict_copy_impl(PyDictObject *self); + +static PyObject * +dict_copy(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_copy_impl(self); +} + PyDoc_STRVAR(dict___contains____doc__, "__contains__($self, key, /)\n" "--\n" @@ -73,7 +94,9 @@ dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_get_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -110,12 +133,32 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = dict_setdefault_impl(self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(dict_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the dict."); + +#define DICT_CLEAR_METHODDEF \ + {"clear", (PyCFunction)dict_clear, METH_NOARGS, dict_clear__doc__}, + +static PyObject * +dict_clear_impl(PyDictObject *self); + +static PyObject * +dict_clear(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_clear_impl(self); +} + PyDoc_STRVAR(dict_pop__doc__, "pop($self, key, default=, /)\n" "--\n" @@ -171,7 +214,31 @@ dict_popitem_impl(PyDictObject *self); static PyObject * dict_popitem(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_popitem_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = dict_popitem_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(dict___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return the size of the dict in memory, in bytes."); + +#define DICT___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)dict___sizeof__, METH_NOARGS, dict___sizeof____doc__}, + +static PyObject * +dict___sizeof___impl(PyDictObject *self); + +static PyObject * +dict___sizeof__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict___sizeof___impl(self); } PyDoc_STRVAR(dict___reversed____doc__, @@ -191,4 +258,58 @@ dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict___reversed___impl(self); } -/*[clinic end generated code: output=582766ac0154c8bf input=a9049054013a1b77]*/ + +PyDoc_STRVAR(dict_keys__doc__, +"keys($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s keys."); + +#define DICT_KEYS_METHODDEF \ + {"keys", (PyCFunction)dict_keys, METH_NOARGS, dict_keys__doc__}, + +static PyObject * +dict_keys_impl(PyDictObject *self); + +static PyObject * +dict_keys(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_keys_impl(self); +} + +PyDoc_STRVAR(dict_items__doc__, +"items($self, /)\n" +"--\n" +"\n" +"Return a set-like object providing a view on the dict\'s items."); + +#define DICT_ITEMS_METHODDEF \ + {"items", (PyCFunction)dict_items, METH_NOARGS, dict_items__doc__}, + +static PyObject * +dict_items_impl(PyDictObject *self); + +static PyObject * +dict_items(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_items_impl(self); +} + +PyDoc_STRVAR(dict_values__doc__, +"values($self, /)\n" +"--\n" +"\n" +"Return an object providing a view on the dict\'s values."); + +#define DICT_VALUES_METHODDEF \ + {"values", (PyCFunction)dict_values, METH_NOARGS, dict_values__doc__}, + +static PyObject * +dict_values_impl(PyDictObject *self); + +static PyObject * +dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +{ + return dict_values_impl(self); +} +/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index 97f20c63cd4adf..09774a73c8145c 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(enum_new__doc__, "enumerate(iterable, start=0)\n" @@ -106,4 +107,4 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=661b29708f501d19 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c48a9a482a52e91 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index caaf9730389a73..d20b314514a6f2 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(float_is_integer__doc__, "is_integer($self, /)\n" "--\n" @@ -325,4 +327,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=77409227fa9caf8d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b9c8a1b6759ca073 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/funcobject.c.h b/Objects/clinic/funcobject.c.h index 1be45674286905..8f20bda26438cf 100644 --- a/Objects/clinic/funcobject.c.h +++ b/Objects/clinic/funcobject.c.h @@ -6,9 +6,11 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(func_new__doc__, -"function(code, globals, name=None, argdefs=None, closure=None)\n" +"function(code, globals, name=None, argdefs=None, closure=None,\n" +" kwdefaults=None)\n" "--\n" "\n" "Create a function object.\n" @@ -22,11 +24,14 @@ PyDoc_STRVAR(func_new__doc__, " argdefs\n" " a tuple that specifies the default argument values\n" " closure\n" -" a tuple that supplies the bindings for free variables"); +" a tuple that supplies the bindings for free variables\n" +" kwdefaults\n" +" a dictionary that specifies the default keyword argument values"); static PyObject * func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, - PyObject *name, PyObject *defaults, PyObject *closure); + PyObject *name, PyObject *defaults, PyObject *closure, + PyObject *kwdefaults); static PyObject * func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -34,14 +39,14 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(code), &_Py_ID(globals), &_Py_ID(name), &_Py_ID(argdefs), &_Py_ID(closure), }, + .ob_item = { &_Py_ID(code), &_Py_ID(globals), &_Py_ID(name), &_Py_ID(argdefs), &_Py_ID(closure), &_Py_ID(kwdefaults), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -50,14 +55,14 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"code", "globals", "name", "argdefs", "closure", NULL}; + static const char * const _keywords[] = {"code", "globals", "name", "argdefs", "closure", "kwdefaults", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "function", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; @@ -66,8 +71,9 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *name = Py_None; PyObject *defaults = Py_None; PyObject *closure = Py_None; + PyObject *kwdefaults = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 6, 0, argsbuf); if (!fastargs) { goto exit; } @@ -96,11 +102,17 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - closure = fastargs[4]; + if (fastargs[4]) { + closure = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + kwdefaults = fastargs[5]; skip_optional_pos: - return_value = func_new_impl(type, code, globals, name, defaults, closure); + return_value = func_new_impl(type, code, globals, name, defaults, closure, kwdefaults); exit: return return_value; } -/*[clinic end generated code: output=b2d676ff51c992d0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=10947342188f38a9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index b6f0c5c0c42dd4..588e021fb71fd3 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -7,6 +7,8 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(list_insert__doc__, "insert($self, index, object, /)\n" @@ -43,28 +45,36 @@ list_insert(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) index = ival; } object = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_insert_impl(self, index, object); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -PyDoc_STRVAR(list_clear__doc__, +PyDoc_STRVAR(py_list_clear__doc__, "clear($self, /)\n" "--\n" "\n" "Remove all items from list."); -#define LIST_CLEAR_METHODDEF \ - {"clear", (PyCFunction)list_clear, METH_NOARGS, list_clear__doc__}, +#define PY_LIST_CLEAR_METHODDEF \ + {"clear", (PyCFunction)py_list_clear, METH_NOARGS, py_list_clear__doc__}, static PyObject * -list_clear_impl(PyListObject *self); +py_list_clear_impl(PyListObject *self); static PyObject * -list_clear(PyListObject *self, PyObject *Py_UNUSED(ignored)) +py_list_clear(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return list_clear_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = py_list_clear_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_copy__doc__, @@ -82,7 +92,13 @@ list_copy_impl(PyListObject *self); static PyObject * list_copy(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return list_copy_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_copy_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_append__doc__, @@ -94,6 +110,21 @@ PyDoc_STRVAR(list_append__doc__, #define LIST_APPEND_METHODDEF \ {"append", (PyCFunction)list_append, METH_O, list_append__doc__}, +static PyObject * +list_append_impl(PyListObject *self, PyObject *object); + +static PyObject * +list_append(PyListObject *self, PyObject *object) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_append_impl(self, object); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(list_extend__doc__, "extend($self, iterable, /)\n" "--\n" @@ -142,7 +173,9 @@ list_pop(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) index = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_pop_impl(self, index); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -220,7 +253,9 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_sort_impl(self, keyfunc, reverse); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -241,7 +276,13 @@ list_reverse_impl(PyListObject *self); static PyObject * list_reverse(PyListObject *self, PyObject *Py_UNUSED(ignored)) { - return list_reverse_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_reverse_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(list_index__doc__, @@ -310,6 +351,21 @@ PyDoc_STRVAR(list_remove__doc__, #define LIST_REMOVE_METHODDEF \ {"remove", (PyCFunction)list_remove, METH_O, list_remove__doc__}, +static PyObject * +list_remove_impl(PyListObject *self, PyObject *value); + +static PyObject * +list_remove(PyListObject *self, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_remove_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(list___init____doc__, "list(iterable=(), /)\n" "--\n" @@ -383,4 +439,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=e2d9f4092498a5ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=854957a1d4a89bbd input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 9288648fe3b7c2..90375b9a082cca 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * long_new_impl(PyTypeObject *type, PyObject *x, PyObject *obase); @@ -115,7 +116,7 @@ int___format__(PyObject *self, PyObject *arg) } PyDoc_STRVAR(int___round____doc__, -"__round__($self, ndigits=, /)\n" +"__round__($self, ndigits=None, /)\n" "--\n" "\n" "Rounding an Integral returns itself.\n" @@ -132,7 +133,7 @@ static PyObject * int___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *o_ndigits = NULL; + PyObject *o_ndigits = Py_None; if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) { goto exit; @@ -266,7 +267,7 @@ PyDoc_STRVAR(int_to_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Determines whether two\'s complement is used to represent the integer.\n" " If signed is False and a negative integer is given, an OverflowError\n" @@ -379,7 +380,7 @@ PyDoc_STRVAR(int_from_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Indicates whether two\'s complement is used to represent the integer."); @@ -475,4 +476,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=009a537ab558763c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a53f5ba9a6c16737 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 74c749e0b6ccc8..f199434dacb9e8 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(memoryview__doc__, "memoryview(object)\n" @@ -412,4 +413,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=7ebdadda3b0fcd35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7e76a09106921ba2 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/moduleobject.c.h b/Objects/clinic/moduleobject.c.h index ce21e4728dcb9e..3c0bbe22d5ebab 100644 --- a/Objects/clinic/moduleobject.c.h +++ b/Objects/clinic/moduleobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(module___init____doc__, "module(name, doc=None)\n" @@ -73,4 +74,4 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=9d3d7854d17a033c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e8a71bfbed774c15 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 643f504535d257..5ef5380656905a 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(OrderedDict_fromkeys__doc__, "fromkeys($type, /, iterable, value=None)\n" @@ -331,4 +332,4 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=6d7ae9fb552c6108 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eff78d2a3f9379bd input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h new file mode 100644 index 00000000000000..3853ce3bce685b --- /dev/null +++ b/Objects/clinic/setobject.c.h @@ -0,0 +1,571 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(set_pop__doc__, +"pop($self, /)\n" +"--\n" +"\n" +"Remove and return an arbitrary set element.\n" +"\n" +"Raises KeyError if the set is empty."); + +#define SET_POP_METHODDEF \ + {"pop", (PyCFunction)set_pop, METH_NOARGS, set_pop__doc__}, + +static PyObject * +set_pop_impl(PySetObject *so); + +static PyObject * +set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_pop_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set_update__doc__, +"update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, adding elements from all others."); + +#define SET_UPDATE_METHODDEF \ + {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, + +static PyObject * +set_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define SET_COPY_METHODDEF \ + {"copy", (PyCFunction)set_copy, METH_NOARGS, set_copy__doc__}, + +static PyObject * +set_copy_impl(PySetObject *so); + +static PyObject * +set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(frozenset_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a shallow copy of a set."); + +#define FROZENSET_COPY_METHODDEF \ + {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, frozenset_copy__doc__}, + +static PyObject * +frozenset_copy_impl(PySetObject *so); + +static PyObject * +frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = frozenset_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all elements from this set."); + +#define SET_CLEAR_METHODDEF \ + {"clear", (PyCFunction)set_clear, METH_NOARGS, set_clear__doc__}, + +static PyObject * +set_clear_impl(PySetObject *so); + +static PyObject * +set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_clear_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set_union__doc__, +"union($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements from the set and all others."); + +#define SET_UNION_METHODDEF \ + {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, + +static PyObject * +set_union_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_union_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_multi__doc__, +"intersection($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements common to the set and all others."); + +#define SET_INTERSECTION_MULTI_METHODDEF \ + {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, + +static PyObject * +set_intersection_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_intersection_update_multi__doc__, +"intersection_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in it and all others."); + +#define SET_INTERSECTION_UPDATE_MULTI_METHODDEF \ + {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, + +static PyObject * +set_intersection_update_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_intersection_update_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_isdisjoint__doc__, +"isdisjoint($self, other, /)\n" +"--\n" +"\n" +"Return True if two sets have a null intersection."); + +#define SET_ISDISJOINT_METHODDEF \ + {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, + +static PyObject * +set_isdisjoint_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_isdisjoint(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_isdisjoint_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + +PyDoc_STRVAR(set_difference_update__doc__, +"difference_update($self, /, *others)\n" +"--\n" +"\n" +"Update the set, removing elements found in others."); + +#define SET_DIFFERENCE_UPDATE_METHODDEF \ + {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, + +static PyObject * +set_difference_update_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_update_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_difference_multi__doc__, +"difference($self, /, *others)\n" +"--\n" +"\n" +"Return a new set with elements in the set that are not in the others."); + +#define SET_DIFFERENCE_MULTI_METHODDEF \ + {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, + +static PyObject * +set_difference_multi_impl(PySetObject *so, PyObject *args); + +static PyObject * +set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_New(nargs - 0); + if (!__clinic_args) { + goto exit; + } + for (Py_ssize_t i = 0; i < nargs - 0; ++i) { + PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + } + return_value = set_difference_multi_impl(so, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(set_symmetric_difference_update__doc__, +"symmetric_difference_update($self, other, /)\n" +"--\n" +"\n" +"Update the set, keeping only elements found in either set, but not in both."); + +#define SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF \ + {"symmetric_difference_update", (PyCFunction)set_symmetric_difference_update, METH_O, set_symmetric_difference_update__doc__}, + +PyDoc_STRVAR(set_symmetric_difference__doc__, +"symmetric_difference($self, other, /)\n" +"--\n" +"\n" +"Return a new set with elements in either the set or other but not both."); + +#define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ + {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, + +static PyObject * +set_symmetric_difference_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_symmetric_difference(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_symmetric_difference_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + +PyDoc_STRVAR(set_issubset__doc__, +"issubset($self, other, /)\n" +"--\n" +"\n" +"Report whether another set contains this set."); + +#define SET_ISSUBSET_METHODDEF \ + {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, + +static PyObject * +set_issubset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issubset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issubset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + +PyDoc_STRVAR(set_issuperset__doc__, +"issuperset($self, other, /)\n" +"--\n" +"\n" +"Report whether this set contains another set."); + +#define SET_ISSUPERSET_METHODDEF \ + {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, + +static PyObject * +set_issuperset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issuperset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issuperset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + +PyDoc_STRVAR(set_add__doc__, +"add($self, object, /)\n" +"--\n" +"\n" +"Add an element to a set.\n" +"\n" +"This has no effect if the element is already present."); + +#define SET_ADD_METHODDEF \ + {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, + +static PyObject * +set_add_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_add(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_add_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set___contains____doc__, +"__contains__($self, object, /)\n" +"--\n" +"\n" +"x.__contains__(y) <==> y in x."); + +#define SET___CONTAINS___METHODDEF \ + {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, + +static PyObject * +set___contains___impl(PySetObject *so, PyObject *key); + +static PyObject * +set___contains__(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___contains___impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set_remove__doc__, +"remove($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set; it must be a member.\n" +"\n" +"If the element is not a member, raise a KeyError."); + +#define SET_REMOVE_METHODDEF \ + {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, + +static PyObject * +set_remove_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_remove(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_remove_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set_discard__doc__, +"discard($self, object, /)\n" +"--\n" +"\n" +"Remove an element from a set if it is a member.\n" +"\n" +"Unlike set.remove(), the discard() method does not raise\n" +"an exception when an element is missing from the set."); + +#define SET_DISCARD_METHODDEF \ + {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, + +static PyObject * +set_discard_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_discard(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_discard_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define SET___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)set___reduce__, METH_NOARGS, set___reduce____doc__}, + +static PyObject * +set___reduce___impl(PySetObject *so); + +static PyObject * +set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___reduce___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(set___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"S.__sizeof__() -> size of S in memory, in bytes."); + +#define SET___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)set___sizeof__, METH_NOARGS, set___sizeof____doc__}, + +static PyObject * +set___sizeof___impl(PySetObject *so); + +static PyObject * +set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___sizeof___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=de4ee725bd29f758 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/structseq.c.h b/Objects/clinic/structseq.c.h index 2571888454cb79..cec49ebccccb24 100644 --- a/Objects/clinic/structseq.c.h +++ b/Objects/clinic/structseq.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict); @@ -61,4 +62,4 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=2f88fe2a6f5c13a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5bf39b3f06a34ce4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/tupleobject.c.h b/Objects/clinic/tupleobject.c.h index f21712746a5ad1..5d6a2c481a5f2a 100644 --- a/Objects/clinic/tupleobject.c.h +++ b/Objects/clinic/tupleobject.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(tuple_index__doc__, "index($self, value, start=0, stop=sys.maxsize, /)\n" "--\n" @@ -112,4 +114,4 @@ tuple___getnewargs__(PyTupleObject *self, PyObject *Py_UNUSED(ignored)) { return tuple___getnewargs___impl(self); } -/*[clinic end generated code: output=7c5d9d12e0cf6a83 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a6a9abba5d121f4c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h index eb0b22a943ae8f..1fa153598db213 100644 --- a/Objects/clinic/typeobject.c.h +++ b/Objects/clinic/typeobject.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_BadArgument() + PyDoc_STRVAR(type___instancecheck____doc__, "__instancecheck__($self, instance, /)\n" "--\n" @@ -260,4 +262,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return object___dir___impl(self); } -/*[clinic end generated code: output=943f639f264362d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b56c87f9cace1921 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 4762cd63545b60..0ba4ff48bc8804 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -6,18 +6,19 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() PyDoc_STRVAR(typevar_new__doc__, -"typevar(name, *constraints, *, bound=None, covariant=False,\n" -" contravariant=False, infer_variance=False)\n" +"typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a TypeVar."); static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance); + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance); static PyObject * typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -25,14 +26,14 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -41,20 +42,21 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevar", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *constraints = NULL; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -79,7 +81,13 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[3]) { - covariant = PyObject_IsTrue(fastargs[3]); + default_value = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + covariant = PyObject_IsTrue(fastargs[4]); if (covariant < 0) { goto exit; } @@ -87,8 +95,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[4]) { - contravariant = PyObject_IsTrue(fastargs[4]); + if (fastargs[5]) { + contravariant = PyObject_IsTrue(fastargs[5]); if (contravariant < 0) { goto exit; } @@ -96,12 +104,12 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[5]); + infer_variance = PyObject_IsTrue(fastargs[6]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = typevar_new_impl(type, name, constraints, bound, covariant, contravariant, infer_variance); + return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: Py_XDECREF(constraints); @@ -116,6 +124,36 @@ PyDoc_STRVAR(typevar_typing_subst__doc__, #define TYPEVAR_TYPING_SUBST_METHODDEF \ {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, +PyDoc_STRVAR(typevar_typing_prepare_subst__doc__, +"__typing_prepare_subst__($self, alias, args, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF \ + {"__typing_prepare_subst__", _PyCFunction_CAST(typevar_typing_prepare_subst), METH_FASTCALL, typevar_typing_prepare_subst__doc__}, + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args); + +static PyObject * +typevar_typing_prepare_subst(typevarobject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *alias; + PyObject *__clinic_args; + + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { + goto exit; + } + alias = args[0]; + __clinic_args = args[1]; + return_value = typevar_typing_prepare_subst_impl(self, alias, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -133,6 +171,23 @@ typevar_reduce(typevarobject *self, PyObject *Py_UNUSED(ignored)) return typevar_reduce_impl(self); } +PyDoc_STRVAR(typevar_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevar_has_default, METH_NOARGS, typevar_has_default__doc__}, + +static PyObject * +typevar_has_default_impl(typevarobject *self); + +static PyObject * +typevar_has_default(typevarobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevar_has_default_impl(self); +} + PyDoc_STRVAR(paramspecargs_new__doc__, "paramspecargs(origin)\n" "--\n" @@ -242,15 +297,16 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_new__doc__, -"paramspec(name, *, bound=None, covariant=False, contravariant=False,\n" -" infer_variance=False)\n" +"paramspec(name, *, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a ParamSpec object."); static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance); + PyObject *default_value, int covariant, int contravariant, + int infer_variance); static PyObject * paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -258,14 +314,14 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -274,19 +330,20 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "paramspec", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -310,7 +367,13 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - covariant = PyObject_IsTrue(fastargs[2]); + default_value = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -318,8 +381,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[3]) { - contravariant = PyObject_IsTrue(fastargs[3]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -327,12 +390,12 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[4]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = paramspec_new_impl(type, name, bound, covariant, contravariant, infer_variance); + return_value = paramspec_new_impl(type, name, bound, default_value, covariant, contravariant, infer_variance); exit: return return_value; @@ -393,14 +456,32 @@ paramspec_reduce(paramspecobject *self, PyObject *Py_UNUSED(ignored)) return paramspec_reduce_impl(self); } +PyDoc_STRVAR(paramspec_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define PARAMSPEC_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)paramspec_has_default, METH_NOARGS, paramspec_has_default__doc__}, + +static PyObject * +paramspec_has_default_impl(paramspecobject *self); + +static PyObject * +paramspec_has_default(paramspecobject *self, PyObject *Py_UNUSED(ignored)) +{ + return paramspec_has_default_impl(self); +} + PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name)\n" +"typevartuple(name, *, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name); +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value); static PyObject * typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -408,14 +489,14 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, + .ob_item = { &_Py_ID(name), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -424,17 +505,19 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", NULL}; + static const char * const _keywords[] = {"name", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -445,7 +528,12 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - return_value = typevartuple_impl(type, name); + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = fastargs[1]; +skip_optional_kwonly: + return_value = typevartuple_impl(type, name, default_value); exit: return return_value; @@ -506,6 +594,23 @@ typevartuple_reduce(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) return typevartuple_reduce_impl(self); } +PyDoc_STRVAR(typevartuple_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVARTUPLE_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevartuple_has_default, METH_NOARGS, typevartuple_has_default__doc__}, + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self); + +static PyObject * +typevartuple_has_default(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevartuple_has_default_impl(self); +} + PyDoc_STRVAR(typealias_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -590,4 +695,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=db0b327ebbb1488f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b39e550e4e336c input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 42b8a3c69cad8a..78e14b0021d006 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(EncodingMap_size__doc__, "size($self, /)\n" @@ -135,6 +136,61 @@ unicode_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(unicode_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation."); + +#define UNICODE_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_count(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("count", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_count_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_encode__doc__, "encode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -300,6 +356,118 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(unicode_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, + +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_find(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("find", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_find_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, + +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_index(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("index", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_index_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_isascii__doc__, "isascii($self, /)\n" "--\n" @@ -891,6 +1059,118 @@ unicode_removesuffix(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, + +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rfind(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rfind", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rfind_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, + +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rindex(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rindex", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rindex_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_rjust__doc__, "rjust($self, width, fillchar=\' \', /)\n" "--\n" @@ -953,9 +1233,11 @@ PyDoc_STRVAR(unicode_split__doc__, " character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" -" Maximum number of splits (starting from the left).\n" +" Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" +"Splitting starts at the front of the string and works to the end.\n" +"\n" "Note, str.split() is mainly useful for data that has been intentionally\n" "delimited. With natural text that includes punctuation, consider using\n" "the regular expression module."); @@ -1077,7 +1359,7 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" -" Maximum number of splits (starting from the left).\n" +" Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the end of the string and works to the front."); @@ -1366,6 +1648,108 @@ unicode_zfill(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(unicode_startswith), METH_FASTCALL, unicode_startswith__doc__}, + +static PyObject * +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(unicode_endswith), METH_FASTCALL, unicode_endswith__doc__}, + +static PyObject * +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode___format____doc__, "__format__($self, format_spec, /)\n" "--\n" @@ -1504,4 +1888,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4acdcfdc93f2a0f6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fee62bd337f809b input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f662b8e354bb1e..7493280c898750 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -5,15 +5,17 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs +#include "pycore_object.h" // _PyObject_SetDeferredRefcount #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches +#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "clinic/codeobject.c.h" -static PyObject* code_repr(PyCodeObject *co); - static const char * code_event_name(PyCodeEvent event) { switch (event) { @@ -41,21 +43,9 @@ notify_code_watchers(PyCodeEvent event, PyCodeObject *co) // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, co) < 0) { - // Don't risk resurrecting the object if an unraisablehook keeps - // a reference; pass a string as context. - PyObject *context = NULL; - PyObject *repr = code_repr(co); - if (repr) { - context = PyUnicode_FromFormat( - "%s watcher callback for %U", - code_event_name(event), repr); - Py_DECREF(repr); - } - if (context == NULL) { - context = Py_NewRef(Py_None); - } - PyErr_WriteUnraisable(context); - Py_DECREF(context); + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for %R", + code_event_name(event), co); } } i++; @@ -112,10 +102,20 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + // unless we've disabled immortalizing objects that use deferred reference + // counting. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_atomic_load_int(&interp->gc.immortalize) < 0) { + return 1; + } +#endif + + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -130,9 +130,14 @@ all_name_chars(PyObject *o) return 1; } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); +#endif + static int intern_strings(PyObject *tuple) { + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t i; for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { @@ -142,21 +147,24 @@ intern_strings(PyObject *tuple) "non-string found in code slot"); return -1; } - PyUnicode_InternInPlace(&_PyTuple_ITEMS(tuple)[i]); + _PyUnicode_InternMortal(interp, &_PyTuple_ITEMS(tuple)[i]); } return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; - PyUnicode_InternInPlace(&v); + _PyUnicode_InternMortal(interp, &v); if (w != v) { PyTuple_SET_ITEM(tuple, i, v); if (modified) { @@ -166,7 +174,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -177,7 +185,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -196,6 +204,59 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string constants in the free-threaded build, but only if + // we are also immortalizing objects that use deferred reference + // counting. + PyThreadState *tstate = PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyCode_Check(v) && + !PyUnicode_CheckExact(v) && + _Py_atomic_load_int(&tstate->interp->gc.immortalize) >= 0) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -399,9 +460,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) con->stacksize = 1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); + _PyUnicode_InternMortal(interp, &co->co_filename); + _PyUnicode_InternMortal(interp, &co->co_name); + _PyUnicode_InternMortal(interp, &co->co_qualname); co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; @@ -427,11 +492,16 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - PyInterpreterState *interp = _PyInterpreterState_GET(); - co->co_version = interp->next_func_version; - if (interp->next_func_version != 0) { - interp->next_func_version++; - } +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -503,7 +573,7 @@ get_line_delta(const uint8_t *ptr) static PyObject * remove_column_info(PyObject *locations) { - int offset = 0; + Py_ssize_t offset = 0; const uint8_t *data = (const uint8_t *)PyBytes_AS_STRING(locations); PyObject *res = PyBytes_FromStringAndSize(NULL, 32); if (res == NULL) { @@ -543,18 +613,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -570,13 +663,22 @@ _PyCode_New(struct _PyCodeConstructor *con) } Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); - PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); + PyCodeObject *co; +#ifdef Py_GIL_DISABLED + co = PyObject_GC_NewVar(PyCodeObject, &PyCode_Type, size); +#else + co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); +#endif if (co == NULL) { Py_XDECREF(replacement_locations); PyErr_NoMemory(); return NULL; } init_code(co, con); +#ifdef Py_GIL_DISABLED + _PyObject_SetDeferredRefcount((PyObject *)co); + _PyObject_GC_TRACK(co); +#endif Py_XDECREF(replacement_locations); return co; } @@ -656,6 +758,35 @@ PyUnstable_Code_NewWithPosOnlyArgs( _Py_set_localsplus_info(offset, name, CO_FAST_FREE, localsplusnames, localspluskinds); } + + // gh-110543: Make sure the CO_FAST_HIDDEN flag is set correctly. + if (!(flags & CO_OPTIMIZED)) { + Py_ssize_t code_len = PyBytes_GET_SIZE(code); + _Py_CODEUNIT *code_data = (_Py_CODEUNIT *)PyBytes_AS_STRING(code); + Py_ssize_t num_code_units = code_len / sizeof(_Py_CODEUNIT); + int extended_arg = 0; + for (int i = 0; i < num_code_units; i += 1 + _PyOpcode_Caches[code_data[i].op.code]) { + _Py_CODEUNIT *instr = &code_data[i]; + uint8_t opcode = instr->op.code; + if (opcode == EXTENDED_ARG) { + extended_arg = extended_arg << 8 | instr->op.arg; + continue; + } + if (opcode == LOAD_FAST_AND_CLEAR) { + int oparg = extended_arg << 8 | instr->op.arg; + if (oparg >= nlocalsplus) { + PyErr_Format(PyExc_ValueError, + "code: LOAD_FAST_AND_CLEAR oparg %d out of range", + oparg); + goto error; + } + _PyLocals_Kind kind = _PyLocals_GetKind(localspluskinds, oparg); + _PyLocals_SetKind(localspluskinds, oparg, kind | CO_FAST_HIDDEN); + } + extended_arg = 0; + } + } + // If any cells were args then nlocalsplus will have shrunk. if (nlocalsplus != PyTuple_GET_SIZE(localsplusnames)) { if (_PyTuple_Resize(&localsplusnames, nlocalsplus) < 0 @@ -733,8 +864,8 @@ PyUnstable_Code_New(int argcount, int kwonlyargcount, // test.test_code.CodeLocationTest.test_code_new_empty to keep it in sync! static const uint8_t assert0[6] = { - RESUME, 0, - LOAD_ASSERTION_ERROR, 0, + RESUME, RESUME_AT_FUNC_START, + LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR, RAISE_VARARGS, 1 }; @@ -1470,33 +1601,30 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + static void clear_executors(PyCodeObject *co) { + assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { - Py_CLEAR(co->co_executors->executors[i]); + if (co->co_executors->executors[i]) { + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); + } } PyMem_Free(co->co_executors); co->co_executors = NULL; } void -_PyCode_Clear_Executors(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - uint8_t opcode = instr->op.code; - uint8_t oparg = instr->op.arg; - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[oparg]; - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - instr->op.code = exec->vm_data.opcode; - instr->op.arg = exec->vm_data.oparg; - } - } +_PyCode_Clear_Executors(PyCodeObject *code) +{ clear_executors(code); } +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { @@ -1702,6 +1830,11 @@ code_dealloc(PyCodeObject *co) } Py_SET_REFCNT(co, 0); +#ifdef Py_GIL_DISABLED + PyObject_GC_UnTrack(co); +#endif + + _PyFunction_ClearCodeByVersion(co->co_version); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; @@ -1716,9 +1849,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 if (co->co_executors != NULL) { clear_executors(co); } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); @@ -1743,6 +1878,15 @@ code_dealloc(PyCodeObject *co) PyObject_Free(co); } +#ifdef Py_GIL_DISABLED +static int +code_traverse(PyCodeObject *co, visitproc visit, void *arg) +{ + Py_VISIT(co->co_consts); + return 0; +} +#endif + static PyObject * code_repr(PyCodeObject *co) { @@ -1878,7 +2022,7 @@ code_hash(PyCodeObject *co) Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ - uhash *= _PyHASH_MULTIPLIER; \ + uhash *= PyHASH_MULTIPLIER; \ } while (0) #define SCRAMBLE_IN_HASH(EXPR) do { \ Py_hash_t h = PyObject_Hash(EXPR); \ @@ -2161,7 +2305,8 @@ static struct PyMethodDef code_methods[] = { {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF - {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {NULL, NULL} /* sentinel */ }; @@ -2186,9 +2331,17 @@ PyTypeObject PyCode_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ +#ifdef Py_GIL_DISABLED + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ +#else Py_TPFLAGS_DEFAULT, /* tp_flags */ +#endif code_new__doc__, /* tp_doc */ +#ifdef Py_GIL_DISABLED + (traverseproc)code_traverse, /* tp_traverse */ +#else 0, /* tp_traverse */ +#endif 0, /* tp_clear */ code_richcompare, /* tp_richcompare */ offsetof(PyCodeObject, co_weakreflist), /* tp_weaklistoffset */ @@ -2341,48 +2494,182 @@ _PyCode_ConstantKey(PyObject *op) return key; } -void -_PyStaticCode_Fini(PyCodeObject *co) +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) { - deopt_code(co, _PyCode_CODE(co)); - if (co->co_executors != NULL) { - clear_executors(co); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; } - PyMem_Free(co->co_extra); - if (co->_co_cached != NULL) { - Py_CLEAR(co->_co_cached->_co_code); - Py_CLEAR(co->_co_cached->_co_cellvars); - Py_CLEAR(co->_co_cached->_co_freevars); - Py_CLEAR(co->_co_cached->_co_varnames); - PyMem_Free(co->_co_cached); - co->_co_cached = NULL; + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; +} + +static int +compare_constants(const void *key1, const void *key2) { + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; } - co->co_extra = NULL; - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)co); - co->co_weakreflist = NULL; + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; } - free_monitoring_data(co->_co_monitoring); - co->_co_monitoring = NULL; + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + _Py_FatalErrorFormat("unexpected type in compare_constants: %s", + Py_TYPE(op1)->tp_name); + return 0; } -int -_PyStaticCode_Init(PyCodeObject *co) +static Py_uhash_t +hash_const(const void *key) { - int res = intern_strings(co->co_names); - if (res < 0) { - return -1; + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return _Py_HashBytes(&data, sizeof(data)); } - res = intern_string_constants(co->co_consts, NULL); - if (res < 0) { - return -1; + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return _Py_HashBytes(data, sizeof(PyObject *) * size); } - res = intern_strings(co->co_localsplusnames); - if (res < 0) { - return -1; + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // This should never happen: all the constants we support have + // infallible hash functions. + Py_FatalError("code: hash failed"); + } + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } + } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); + } + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); } - _PyCode_Quicken(co); return 0; } -#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif + +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} + +void +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } +#endif +} diff --git a/Objects/complexobject.c b/Objects/complexobject.c index b5598b9f355eac..7c8a6bd9dfcd3f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -88,8 +88,7 @@ _Py_c_quot(Py_complex a, Py_complex b) * numerators and denominator by whichever of {b.real, b.imag} has * larger magnitude. The earliest reference I found was to CACM * Algorithm 116 (Complex Division, Robert L. Smith, Stanford - * University). As usual, though, we're still ignoring all IEEE - * endcases. + * University). */ Py_complex r; /* the result */ const double abs_breal = b.real < 0 ? -b.real : b.real; @@ -120,6 +119,28 @@ _Py_c_quot(Py_complex a, Py_complex b) /* At least one of b.real or b.imag is a NaN */ r.real = r.imag = Py_NAN; } + + /* Recover infinities and zeros that computed as nan+nanj. See e.g. + the C11, Annex G.5.2, routine _Cdivd(). */ + if (isnan(r.real) && isnan(r.imag)) { + if ((isinf(a.real) || isinf(a.imag)) + && isfinite(b.real) && isfinite(b.imag)) + { + const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); + const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); + r.real = Py_INFINITY * (x*b.real + y*b.imag); + r.imag = Py_INFINITY * (y*b.real - x*b.imag); + } + else if ((isinf(abs_breal) || isinf(abs_bimag)) + && isfinite(a.real) && isfinite(a.imag)) + { + const double x = copysign(isinf(b.real) ? 1.0 : 0.0, b.real); + const double y = copysign(isinf(b.imag) ? 1.0 : 0.0, b.imag); + r.real = 0.0 * (a.real*x + a.imag*y); + r.imag = 0.0 * (a.imag*x - a.real*y); + } + } + return r; } #ifdef _M_ARM64 @@ -188,16 +209,16 @@ _Py_c_abs(Py_complex z) /* sets errno = ERANGE on overflow; otherwise errno = 0 */ double result; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { + if (!isfinite(z.real) || !isfinite(z.imag)) { /* C99 rules: if either the real or the imaginary part is an infinity, return infinity, even if the other part is a NaN. */ - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.real)) { result = fabs(z.real); errno = 0; return result; } - if (Py_IS_INFINITY(z.imag)) { + if (isinf(z.imag)) { result = fabs(z.imag); errno = 0; return result; @@ -207,7 +228,7 @@ _Py_c_abs(Py_complex z) return Py_NAN; } result = hypot(z.real, z.imag); - if (!Py_IS_FINITE(result)) + if (!isfinite(result)) errno = ERANGE; else errno = 0; @@ -256,26 +277,51 @@ PyComplex_FromDoubles(double real, double imag) return PyComplex_FromCComplex(c); } +static PyObject * try_complex_special_method(PyObject *); + double PyComplex_RealAsDouble(PyObject *op) { + double real = -1.0; + if (PyComplex_Check(op)) { - return ((PyComplexObject *)op)->cval.real; + real = ((PyComplexObject *)op)->cval.real; } else { - return PyFloat_AsDouble(op); + PyObject* newop = try_complex_special_method(op); + if (newop) { + real = ((PyComplexObject *)newop)->cval.real; + Py_DECREF(newop); + } else if (!PyErr_Occurred()) { + real = PyFloat_AsDouble(op); + } } + + return real; } double PyComplex_ImagAsDouble(PyObject *op) { + double imag = -1.0; + if (PyComplex_Check(op)) { - return ((PyComplexObject *)op)->cval.imag; + imag = ((PyComplexObject *)op)->cval.imag; } else { - return 0.0; + PyObject* newop = try_complex_special_method(op); + if (newop) { + imag = ((PyComplexObject *)newop)->cval.imag; + Py_DECREF(newop); + } else if (!PyErr_Occurred()) { + PyFloat_AsDouble(op); + if (!PyErr_Occurred()) { + imag = 0.0; + } + } } + + return imag; } static PyObject * @@ -498,7 +544,7 @@ complex_div(PyObject *v, PyObject *w) errno = 0; quot = _Py_c_quot(a, b); if (errno == EDOM) { - PyErr_SetString(PyExc_ZeroDivisionError, "complex division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } return PyComplex_FromCComplex(quot); @@ -529,7 +575,7 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 to a negative or complex power"); + "zero to a negative or complex power"); return NULL; } else if (errno == ERANGE) { @@ -853,8 +899,8 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) } else { PyErr_Format(PyExc_TypeError, - "complex() argument must be a string or a number, not '%.200s'", - Py_TYPE(v)->tp_name); + "complex() argument must be a string or a number, not %T", + v); return NULL; } @@ -864,20 +910,94 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) return result; } +/* The constructor should only accept a string as a positional argument, + * not as by the 'real' keyword. But Argument Clinic does not allow + * to distinguish between argument passed positionally and by keyword. + * So the constructor must be split into two parts: actual_complex_new() + * handles the case of no arguments and one positional argument, and calls + * complex_new(), implemented with Argument Clinic, to handle the remaining + * cases: 'real' and 'imag' arguments. This separation is well suited + * for different constructor roles: converting a string or number to a complex + * number and constructing a complex number from real and imaginary parts. + */ +static PyObject * +actual_complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res = NULL; + PyNumberMethods *nbr; + + if (PyTuple_GET_SIZE(args) > 1 || (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { + return complex_new(type, args, kwargs); + } + if (!PyTuple_GET_SIZE(args)) { + return complex_subtype_from_doubles(type, 0, 0); + } + + PyObject *arg = PyTuple_GET_ITEM(args, 0); + /* Special-case for a single argument when type(arg) is complex. */ + if (PyComplex_CheckExact(arg) && type == &PyComplex_Type) { + /* Note that we can't know whether it's safe to return + a complex *subclass* instance as-is, hence the restriction + to exact complexes here. If either the input or the + output is a complex subclass, it will be handled below + as a non-orthogonal vector. */ + return Py_NewRef(arg); + } + if (PyUnicode_Check(arg)) { + return complex_subtype_from_string(type, arg); + } + PyObject *tmp = try_complex_special_method(arg); + if (tmp) { + Py_complex c = ((PyComplexObject*)tmp)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + Py_DECREF(tmp); + } + else if (PyErr_Occurred()) { + return NULL; + } + else if (PyComplex_Check(arg)) { + /* Note that if arg is of a complex subtype, we're only + retaining its real & imag parts here, and the return + value is (properly) of the builtin complex type. */ + Py_complex c = ((PyComplexObject*)arg)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + } + else if ((nbr = Py_TYPE(arg)->tp_as_number) != NULL && + (nbr->nb_float != NULL || nbr->nb_index != NULL)) + { + /* The argument really is entirely real, and contributes + nothing in the imaginary direction. + Just treat it as a double. */ + double r = PyFloat_AsDouble(arg); + if (r != -1.0 || !PyErr_Occurred()) { + res = complex_subtype_from_doubles(type, r, 0); + } + } + else { + PyErr_Format(PyExc_TypeError, + "complex() argument must be a string or a number, not %T", + arg); + } + return res; +} + /*[clinic input] @classmethod complex.__new__ as complex_new real as r: object(c_default="NULL") = 0 imag as i: object(c_default="NULL") = 0 -Create a complex number from a real part and an optional imaginary part. +Create a complex number from a string or numbers. -This is equivalent to (real + imag*1j) where imag defaults to 0. +If a string is given, parse it as a complex number. +If a single number is given, convert it to a complex number. +If the 'real' or 'imag' arguments are given, create a complex number +with the specified real and imaginary components. [clinic start generated code]*/ static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) -/*[clinic end generated code: output=b6c7dd577b537dc1 input=f4c667f2596d4fd1]*/ +/*[clinic end generated code: output=b6c7dd577b537dc1 input=ff4268dc540958a4]*/ { PyObject *tmp; PyNumberMethods *nbr, *nbi = NULL; @@ -889,32 +1009,10 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) if (r == NULL) { r = _PyLong_GetZero(); } + PyObject *orig_r = r; - /* Special-case for a single argument when type(arg) is complex. */ - if (PyComplex_CheckExact(r) && i == NULL && - type == &PyComplex_Type) { - /* Note that we can't know whether it's safe to return - a complex *subclass* instance as-is, hence the restriction - to exact complexes here. If either the input or the - output is a complex subclass, it will be handled below - as a non-orthogonal vector. */ - return Py_NewRef(r); - } - if (PyUnicode_Check(r)) { - if (i != NULL) { - PyErr_SetString(PyExc_TypeError, - "complex() can't take second arg" - " if first is a string"); - return NULL; - } - return complex_subtype_from_string(type, r); - } - if (i != NULL && PyUnicode_Check(i)) { - PyErr_SetString(PyExc_TypeError, - "complex() second arg can't be a string"); - return NULL; - } - + /* DEPRECATED: The call of try_complex_special_method() for the "real" + * part will be dropped after the end of the deprecation period. */ tmp = try_complex_special_method(r); if (tmp) { r = tmp; @@ -929,9 +1027,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r))) { PyErr_Format(PyExc_TypeError, - "complex() first argument must be a string or a number, " - "not '%.200s'", - Py_TYPE(r)->tp_name); + "complex() argument 'real' must be a real number, not %T", + r); if (own_r) { Py_DECREF(r); } @@ -943,9 +1040,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i))) { PyErr_Format(PyExc_TypeError, - "complex() second argument must be a number, " - "not '%.200s'", - Py_TYPE(i)->tp_name); + "complex() argument 'imag' must be a real number, not %T", + i); if (own_r) { Py_DECREF(r); } @@ -957,6 +1053,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) both be treated as numbers, and the constructor should return a complex number equal to (real + imag*1j). + The following is DEPRECATED: Note that we do NOT assume the input to already be in canonical form; the "real" and "imag" parts might themselves be complex numbers, which slightly complicates the code below. */ @@ -967,19 +1064,27 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) cr = ((PyComplexObject*)r)->cval; cr_is_complex = 1; if (own_r) { + /* r was a newly created complex number, rather + than the original "real" argument. */ Py_DECREF(r); } + nbr = Py_TYPE(orig_r)->tp_as_number; + if (nbr == NULL || + (nbr->nb_float == NULL && nbr->nb_index == NULL)) + { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'real' must be a real number, not %T", + orig_r)) { + return NULL; + } + } } else { /* The "real" part really is entirely real, and contributes nothing in the imaginary direction. Just treat it as a double. */ tmp = PyNumber_Float(r); - if (own_r) { - /* r was a newly created complex number, rather - than the original "real" argument. */ - Py_DECREF(r); - } + assert(!own_r); if (tmp == NULL) return NULL; assert(PyFloat_Check(tmp)); @@ -991,6 +1096,11 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) ci.real = cr.imag; } else if (PyComplex_Check(i)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'imag' must be a real number, not %T", + i)) { + return NULL; + } ci = ((PyComplexObject*)i)->cval; ci_is_complex = 1; } else { @@ -1136,6 +1246,6 @@ PyTypeObject PyComplex_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - complex_new, /* tp_new */ + actual_complex_new, /* tp_new */ PyObject_Del, /* tp_free */ }; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 56ce34f80ca8e9..4eccd1704eb95a 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -6,6 +6,7 @@ #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_emscripten_trampoline.h" // descr_set_trampoline_call(), descr_get_trampoline_call() #include "pycore_descrobject.h" // _PyMethodWrapper_Type +#include "pycore_modsupport.h" // _PyArg_UnpackStack() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -18,8 +19,9 @@ class property "propertyobject *" "&PyProperty_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=556352653fd4c02e]*/ static void -descr_dealloc(PyDescrObject *descr) +descr_dealloc(PyObject *self) { + PyDescrObject *descr = (PyDescrObject *)self; _PyObject_GC_UNTRACK(descr); Py_XDECREF(descr->d_type); Py_XDECREF(descr->d_name); @@ -46,28 +48,28 @@ descr_repr(PyDescrObject *descr, const char *format) } static PyObject * -method_repr(PyMethodDescrObject *descr) +method_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -member_repr(PyMemberDescrObject *descr) +member_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -getset_repr(PyGetSetDescrObject *descr) +getset_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -wrapperdescr_repr(PyWrapperDescrObject *descr) +wrapperdescr_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); @@ -89,8 +91,9 @@ descr_check(PyDescrObject *descr, PyObject *obj) } static PyObject * -classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +classmethod_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; /* Ensure a valid type. Class methods ignore obj. */ if (type == NULL) { if (obj != NULL) @@ -131,8 +134,9 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +method_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -155,8 +159,9 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) +member_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -175,8 +180,9 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) +getset_get(PyObject *self, PyObject *obj, PyObject *type) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -194,8 +200,9 @@ getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -wrapperdescr_get(PyWrapperDescrObject *descr, PyObject *obj, PyObject *type) +wrapperdescr_get(PyObject *self, PyObject *obj, PyObject *type) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -222,8 +229,9 @@ descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value) } static int -member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) +member_set(PyObject *self, PyObject *obj, PyObject *value) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -231,8 +239,9 @@ member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) } static int -getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) +getset_set(PyObject *self, PyObject *obj, PyObject *value) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -384,7 +393,7 @@ method_vectorcall_FASTCALL( if (method_check_args(func, args, nargs, kwnames)) { return NULL; } - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -403,7 +412,7 @@ method_vectorcall_FASTCALL_KEYWORDS( if (method_check_args(func, args, nargs, NULL)) { return NULL; } - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) method_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -478,9 +487,10 @@ method_vectorcall_O( we implement this simply by calling __get__ and then calling the result. */ static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, +classmethoddescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; Py_ssize_t argc = PyTuple_GET_SIZE(args); if (argc < 1) { PyErr_Format(PyExc_TypeError, @@ -491,7 +501,7 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, return NULL; } PyObject *self = PyTuple_GET_ITEM(args, 0); - PyObject *bound = classmethod_get(descr, NULL, self); + PyObject *bound = classmethod_get((PyObject *)descr, NULL, self); if (bound == NULL) { return NULL; } @@ -522,8 +532,9 @@ wrapperdescr_raw_call(PyWrapperDescrObject *descr, PyObject *self, } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)_descr; Py_ssize_t argc; PyObject *self, *result; @@ -562,14 +573,16 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) static PyObject * -method_get_doc(PyMethodDescrObject *descr, void *closure) +method_get_doc(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetDocFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); } static PyObject * -method_get_text_signature(PyMethodDescrObject *descr, void *closure) +method_get_text_signature(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc, descr->d_method->ml_flags); @@ -604,22 +617,24 @@ calculate_qualname(PyDescrObject *descr) } static PyObject * -descr_get_qualname(PyDescrObject *descr, void *Py_UNUSED(ignored)) +descr_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; if (descr->d_qualname == NULL) descr->d_qualname = calculate_qualname(descr); return Py_XNewRef(descr->d_qualname); } static PyObject * -descr_reduce(PyDescrObject *descr, PyObject *Py_UNUSED(ignored)) +descr_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), PyDescr_TYPE(descr), PyDescr_NAME(descr)); } static PyMethodDef descr_methods[] = { - {"__reduce__", (PyCFunction)descr_reduce, METH_NOARGS, NULL}, + {"__reduce__", descr_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -630,15 +645,16 @@ static PyMemberDef descr_members[] = { }; static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)method_get_text_signature}, + {"__doc__", method_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", method_get_text_signature}, {0} }; static PyObject * -member_get_doc(PyMemberDescrObject *descr, void *closure) +member_get_doc(PyObject *_descr, void *closure) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)_descr; if (descr->d_member->doc == NULL) { Py_RETURN_NONE; } @@ -646,14 +662,15 @@ member_get_doc(PyMemberDescrObject *descr, void *closure) } static PyGetSetDef member_getset[] = { - {"__doc__", (getter)member_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", member_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -getset_get_doc(PyGetSetDescrObject *descr, void *closure) +getset_get_doc(PyObject *self, void *closure) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr->d_getset->doc == NULL) { Py_RETURN_NONE; } @@ -661,28 +678,30 @@ getset_get_doc(PyGetSetDescrObject *descr, void *closure) } static PyGetSetDef getset_getset[] = { - {"__doc__", (getter)getset_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", getset_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_doc(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetDocFromInternalDoc(descr->d_base->name, descr->d_base->doc); } static PyObject * -wrapperdescr_get_text_signature(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_text_signature(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, descr->d_base->doc, 0); } static PyGetSetDef wrapperdescr_getset[] = { - {"__doc__", (getter)wrapperdescr_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)wrapperdescr_get_text_signature}, + {"__doc__", wrapperdescr_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", wrapperdescr_get_text_signature}, {0} }; @@ -699,12 +718,12 @@ PyTypeObject PyMethodDescr_Type = { "method_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ offsetof(PyMethodDescrObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -729,7 +748,7 @@ PyTypeObject PyMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)method_get, /* tp_descr_get */ + method_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -739,17 +758,17 @@ PyTypeObject PyClassMethodDescr_Type = { "classmethod_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + classmethoddescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -767,7 +786,7 @@ PyTypeObject PyClassMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)classmethod_get, /* tp_descr_get */ + classmethod_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -776,12 +795,12 @@ PyTypeObject PyMemberDescr_Type = { "member_descriptor", sizeof(PyMemberDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)member_repr, /* tp_repr */ + member_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -804,8 +823,8 @@ PyTypeObject PyMemberDescr_Type = { member_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)member_get, /* tp_descr_get */ - (descrsetfunc)member_set, /* tp_descr_set */ + member_get, /* tp_descr_get */ + member_set, /* tp_descr_set */ }; PyTypeObject PyGetSetDescr_Type = { @@ -813,12 +832,12 @@ PyTypeObject PyGetSetDescr_Type = { "getset_descriptor", sizeof(PyGetSetDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)getset_repr, /* tp_repr */ + getset_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -841,8 +860,8 @@ PyTypeObject PyGetSetDescr_Type = { getset_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)getset_get, /* tp_descr_get */ - (descrsetfunc)getset_set, /* tp_descr_set */ + getset_get, /* tp_descr_get */ + getset_set, /* tp_descr_set */ }; PyTypeObject PyWrapperDescr_Type = { @@ -850,17 +869,17 @@ PyTypeObject PyWrapperDescr_Type = { "wrapper_descriptor", sizeof(PyWrapperDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapperdescr_repr, /* tp_repr */ + wrapperdescr_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + wrapperdescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -879,7 +898,7 @@ PyTypeObject PyWrapperDescr_Type = { wrapperdescr_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ + wrapperdescr_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -890,6 +909,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name) descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0); if (descr != NULL) { + _PyObject_SetDeferredRefcount((PyObject *)descr); descr->d_type = (PyTypeObject*)Py_XNewRef(type); descr->d_name = PyUnicode_InternFromString(name); if (descr->d_name == NULL) { @@ -1021,20 +1041,22 @@ typedef struct { } mappingproxyobject; static Py_ssize_t -mappingproxy_len(mappingproxyobject *pp) +mappingproxy_len(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Size(pp->mapping); } static PyObject * -mappingproxy_getitem(mappingproxyobject *pp, PyObject *key) +mappingproxy_getitem(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetItem(pp->mapping, key); } static PyMappingMethods mappingproxy_as_mapping = { - (lenfunc)mappingproxy_len, /* mp_length */ - (binaryfunc)mappingproxy_getitem, /* mp_subscript */ + mappingproxy_len, /* mp_length */ + mappingproxy_getitem, /* mp_subscript */ 0, /* mp_ass_subscript */ }; @@ -1063,8 +1085,9 @@ static PyNumberMethods mappingproxy_as_number = { }; static int -mappingproxy_contains(mappingproxyobject *pp, PyObject *key) +mappingproxy_contains(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; if (PyDict_CheckExact(pp->mapping)) return PyDict_Contains(pp->mapping, key); else @@ -1079,14 +1102,15 @@ static PySequenceMethods mappingproxy_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)mappingproxy_contains, /* sq_contains */ + mappingproxy_contains, /* sq_contains */ 0, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ }; static PyObject * -mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs) +mappingproxy_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + mappingproxyobject *pp = (mappingproxyobject *)self; /* newargs: mapping, key, default=None */ PyObject *newargs[3]; newargs[0] = pp->mapping; @@ -1103,32 +1127,37 @@ mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs } static PyObject * -mappingproxy_keys(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(keys)); } static PyObject * -mappingproxy_values(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(values)); } static PyObject * -mappingproxy_items(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(items)); } static PyObject * -mappingproxy_copy(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(copy)); } static PyObject * -mappingproxy_reversed(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(__reversed__)); } @@ -1137,52 +1166,57 @@ mappingproxy_reversed(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) static PyMethodDef mappingproxy_methods[] = { {"get", _PyCFunction_CAST(mappingproxy_get), METH_FASTCALL, - PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." - " d defaults to None.")}, - {"keys", (PyCFunction)mappingproxy_keys, METH_NOARGS, + PyDoc_STR("get($self, key, default=None, /)\n--\n\n" + "Return the value for key if key is in the mapping, else default.")}, + {"keys", mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, - {"values", (PyCFunction)mappingproxy_values, METH_NOARGS, + {"values", mappingproxy_values, METH_NOARGS, PyDoc_STR("D.values() -> an object providing a view on D's values")}, - {"items", (PyCFunction)mappingproxy_items, METH_NOARGS, + {"items", mappingproxy_items, METH_NOARGS, PyDoc_STR("D.items() -> a set-like object providing a view on D's items")}, - {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, + {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, - {"__reversed__", (PyCFunction)mappingproxy_reversed, METH_NOARGS, + {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} }; static void -mappingproxy_dealloc(mappingproxyobject *pp) +mappingproxy_dealloc(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; _PyObject_GC_UNTRACK(pp); Py_DECREF(pp->mapping); PyObject_GC_Del(pp); } static PyObject * -mappingproxy_getiter(mappingproxyobject *pp) +mappingproxy_getiter(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetIter(pp->mapping); } static Py_hash_t -mappingproxy_hash(mappingproxyobject *pp) +mappingproxy_hash(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Hash(pp->mapping); } static PyObject * -mappingproxy_str(mappingproxyobject *pp) +mappingproxy_str(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Str(pp->mapping); } static PyObject * -mappingproxy_repr(mappingproxyobject *pp) +mappingproxy_repr(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyUnicode_FromFormat("mappingproxy(%R)", pp->mapping); } @@ -1195,8 +1229,9 @@ mappingproxy_traverse(PyObject *self, visitproc visit, void *arg) } static PyObject * -mappingproxy_richcompare(mappingproxyobject *v, PyObject *w, int op) +mappingproxy_richcompare(PyObject *self, PyObject *w, int op) { + mappingproxyobject *v = (mappingproxyobject *)self; return PyObject_RichCompare(v->mapping, w, op); } @@ -1220,11 +1255,12 @@ mappingproxy.__new__ as mappingproxy_new mapping: object +Read-only proxy of a mapping. [clinic start generated code]*/ static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping) -/*[clinic end generated code: output=65f27f02d5b68fa7 input=d2d620d4f598d4f8]*/ +/*[clinic end generated code: output=65f27f02d5b68fa7 input=c156df096ef7590c]*/ { mappingproxyobject *mappingproxy; @@ -1270,8 +1306,9 @@ typedef struct { #define Wrapper_Check(v) Py_IS_TYPE(v, &_PyMethodWrapper_Type) static void -wrapper_dealloc(wrapperobject *wp) +wrapper_dealloc(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; PyObject_GC_UnTrack(wp); Py_TRASHCAN_BEGIN(wp, wrapper_dealloc) Py_XDECREF(wp->descr); @@ -1307,10 +1344,11 @@ wrapper_richcompare(PyObject *a, PyObject *b, int op) } static Py_hash_t -wrapper_hash(wrapperobject *wp) +wrapper_hash(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; Py_hash_t x, y; - x = _Py_HashPointer(wp->self); + x = PyObject_GenericHash(wp->self); y = _Py_HashPointer(wp->descr); x = x ^ y; if (x == -1) @@ -1319,8 +1357,9 @@ wrapper_hash(wrapperobject *wp) } static PyObject * -wrapper_repr(wrapperobject *wp) +wrapper_repr(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; return PyUnicode_FromFormat("", wp->descr->d_base->name, Py_TYPE(wp->self)->tp_name, @@ -1328,14 +1367,15 @@ wrapper_repr(wrapperobject *wp) } static PyObject * -wrapper_reduce(wrapperobject *wp, PyObject *Py_UNUSED(ignored)) +wrapper_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), wp->self, PyDescr_NAME(wp->descr)); } static PyMethodDef wrapper_methods[] = { - {"__reduce__", (PyCFunction)wrapper_reduce, METH_NOARGS, NULL}, + {"__reduce__", wrapper_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -1345,52 +1385,56 @@ static PyMemberDef wrapper_members[] = { }; static PyObject * -wrapper_objclass(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_objclass(PyObject *wp, void *Py_UNUSED(ignored)) { - PyObject *c = (PyObject *)PyDescr_TYPE(wp->descr); + PyObject *c = (PyObject *)PyDescr_TYPE(((wrapperobject *)wp)->descr); return Py_NewRef(c); } static PyObject * -wrapper_name(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_name(PyObject *wp, void *Py_UNUSED(ignored)) { - const char *s = wp->descr->d_base->name; + const char *s = ((wrapperobject *)wp)->descr->d_base->name; return PyUnicode_FromString(s); } static PyObject * -wrapper_doc(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_doc(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetDocFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc); } static PyObject * -wrapper_text_signature(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_text_signature(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc, 0); } static PyObject * -wrapper_qualname(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_qualname(PyObject *self, void *Py_UNUSED(ignored)) { - return descr_get_qualname((PyDescrObject *)wp->descr, NULL); + wrapperobject *wp = (wrapperobject *)self; + return descr_get_qualname((PyObject *)wp->descr, NULL); } static PyGetSetDef wrapper_getsets[] = { - {"__objclass__", (getter)wrapper_objclass}, - {"__name__", (getter)wrapper_name}, - {"__qualname__", (getter)wrapper_qualname}, - {"__doc__", (getter)wrapper_doc}, - {"__text_signature__", (getter)wrapper_text_signature}, + {"__objclass__", wrapper_objclass}, + {"__name__", wrapper_name}, + {"__qualname__", wrapper_qualname}, + {"__doc__", wrapper_doc}, + {"__text_signature__", wrapper_text_signature}, {0} }; static PyObject * -wrapper_call(wrapperobject *wp, PyObject *args, PyObject *kwds) +wrapper_call(PyObject *self, PyObject *args, PyObject *kwds) { + wrapperobject *wp = (wrapperobject *)self; return wrapperdescr_raw_call(wp->descr, wp->self, args, kwds); } @@ -1409,17 +1453,17 @@ PyTypeObject _PyMethodWrapper_Type = { sizeof(wrapperobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)wrapper_dealloc, /* tp_dealloc */ + wrapper_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapper_repr, /* tp_repr */ + wrapper_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)wrapper_hash, /* tp_hash */ - (ternaryfunc)wrapper_call, /* tp_call */ + wrapper_hash, /* tp_hash */ + wrapper_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1477,22 +1521,34 @@ class property(object): self.__doc__ = doc except AttributeError: # read-only or dict-less class pass + self.__name = None + + def __set_name__(self, owner, name): + self.__name = name + + @property + def __name__(self): + return self.__name if self.__name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self.__name = value def __get__(self, inst, type=None): if inst is None: return self if self.__get is None: - raise AttributeError, "property has no getter" + raise AttributeError("property has no getter") return self.__get(inst) def __set__(self, inst, value): if self.__set is None: - raise AttributeError, "property has no setter" + raise AttributeError("property has no setter") return self.__set(inst, value) def __delete__(self, inst): if self.__del is None: - raise AttributeError, "property has no deleter" + raise AttributeError("property has no deleter") return self.__del(inst) */ @@ -1540,6 +1596,9 @@ property_deleter(PyObject *self, PyObject *deleter) PyDoc_STRVAR(set_name_doc, + "__set_name__($self, owner, name, /)\n" + "--\n" + "\n" "Method to set name of a property."); static PyObject * @@ -1583,6 +1642,20 @@ property_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } +static int +property_name(propertyobject *prop, PyObject **name) +{ + if (prop->prop_name != NULL) { + *name = Py_NewRef(prop->prop_name); + return 1; + } + if (prop->prop_get == NULL) { + *name = NULL; + return 0; + } + return PyObject_GetOptionalAttr(prop->prop_get, &_Py_ID(__name__), name); +} + static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { @@ -1592,11 +1665,15 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) propertyobject *gs = (propertyobject *)self; if (gs->prop_get == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return NULL; + } PyObject *qualname = PyType_GetQualName(Py_TYPE(obj)); - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, "property %R of %R object has no getter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1607,6 +1684,7 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyErr_SetString(PyExc_AttributeError, "property has no getter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return NULL; } @@ -1628,16 +1706,20 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) } if (func == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return -1; + } PyObject *qualname = NULL; if (obj != NULL) { qualname = PyType_GetQualName(Py_TYPE(obj)); } - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, value == NULL ? "property %R of %R object has no deleter" : "property %R of %R object has no setter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1653,6 +1735,7 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) "property has no deleter" : "property has no setter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return -1; } @@ -1685,15 +1768,12 @@ property_copy(PyObject *old, PyObject *get, PyObject *set, PyObject *del) return NULL; if (get == NULL || get == Py_None) { - Py_XDECREF(get); get = pold->prop_get ? pold->prop_get : Py_None; } if (set == NULL || set == Py_None) { - Py_XDECREF(set); set = pold->prop_set ? pold->prop_set : Py_None; } if (del == NULL || del == Py_None) { - Py_XDECREF(del); del = pold->prop_del ? pold->prop_del : Py_None; } if (pold->getter_doc && get != Py_None) { @@ -1779,22 +1859,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, /* if no docstring given and the getter has one, use that one */ else if (fget != NULL) { int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc); - if (rc <= 0) { + if (rc < 0) { return rc; } - if (!Py_IS_TYPE(self, &PyProperty_Type) && - prop_doc != NULL && prop_doc != Py_None) { - // This oddity preserves the long existing behavior of surfacing - // an AttributeError when using a dict-less (__slots__) property - // subclass as a decorator on a getter method with a docstring. - // See PropertySubclassTest.test_slots_docstring_copy_exception. - int err = PyObject_SetAttr( - (PyObject *)self, &_Py_ID(__doc__), prop_doc); - if (err < 0) { - Py_DECREF(prop_doc); // release our new reference. - return -1; - } - } if (prop_doc == Py_None) { prop_doc = NULL; Py_DECREF(Py_None); @@ -1822,7 +1889,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, Py_DECREF(prop_doc); if (err < 0) { assert(PyErr_Occurred()); - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!self->getter_doc && + PyErr_ExceptionMatches(PyExc_AttributeError)) + { PyErr_Clear(); // https://github.com/python/cpython/issues/98963#issuecomment-1574413319 // Python silently dropped this doc assignment through 3.11. @@ -1841,6 +1910,28 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, return 0; } +static PyObject * +property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +{ + PyObject *name; + if (property_name(prop, &name) < 0) { + return NULL; + } + if (name == NULL) { + PyErr_SetString(PyExc_AttributeError, + "'property' object has no attribute '__name__'"); + } + return name; +} + +static int +property_set__name__(propertyobject *prop, PyObject *value, + void *Py_UNUSED(ignored)) +{ + Py_XSETREF(prop->prop_name, Py_XNewRef(value)); + return 0; +} + static PyObject * property_get___isabstractmethod__(propertyobject *prop, void *closure) { @@ -1871,6 +1962,7 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { + {"__name__", (getter)property_get__name__, (setter)property_set__name__}, {"__isabstractmethod__", (getter)property_get___isabstractmethod__, NULL, NULL, @@ -1906,29 +1998,29 @@ PyTypeObject PyDictProxy_Type = { sizeof(mappingproxyobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)mappingproxy_dealloc, /* tp_dealloc */ + mappingproxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)mappingproxy_repr, /* tp_repr */ + mappingproxy_repr, /* tp_repr */ &mappingproxy_as_number, /* tp_as_number */ &mappingproxy_as_sequence, /* tp_as_sequence */ &mappingproxy_as_mapping, /* tp_as_mapping */ - (hashfunc)mappingproxy_hash, /* tp_hash */ + mappingproxy_hash, /* tp_hash */ 0, /* tp_call */ - (reprfunc)mappingproxy_str, /* tp_str */ + mappingproxy_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, /* tp_flags */ - 0, /* tp_doc */ + mappingproxy_new__doc__, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)mappingproxy_richcompare, /* tp_richcompare */ + mappingproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)mappingproxy_getiter, /* tp_iter */ + mappingproxy_getiter, /* tp_iter */ 0, /* tp_iternext */ mappingproxy_methods, /* tp_methods */ 0, /* tp_members */ @@ -1968,7 +2060,7 @@ PyTypeObject PyProperty_Type = { Py_TPFLAGS_BASETYPE, /* tp_flags */ property_init__doc__, /* tp_doc */ property_traverse, /* tp_traverse */ - (inquiry)property_clear, /* tp_clear */ + property_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 361f8e93064b25..149e552af3a729 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -21,6 +21,7 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: | dk_log2_size | | dk_log2_index_bytes | | dk_kind | +| dk_version | | dk_usable | | dk_nentries | +---------------------+ @@ -55,7 +56,7 @@ The DictObject can be in one of two forms. Either: A combined table: ma_values == NULL, dk_refcnt == 1. - Values are stored in the me_value field of the PyDictKeysObject. + Values are stored in the me_value field of the PyDictKeyEntry. Or: A split table: ma_values != NULL, dk_refcnt >= 1 @@ -80,6 +81,8 @@ DK_ENTRIES(keys)[index] if index >= 0): Active upon key insertion. Dummy slots cannot be made Unused again else the probe sequence in case of collision would have no way to know they were once active. + In free-threaded builds dummy slots are not re-used to allow lock-free + lookups to proceed safely. 4. Pending. index >= 0, key != NULL, and value == NULL (split only) Not yet inserted in split-table. @@ -113,17 +116,20 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_dict.h" // PyDictKeysObject -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include @@ -140,6 +146,143 @@ To avoid slowing down lookups on a near-full table, we resize the table when it's USABLE_FRACTION (currently two-thirds) full. */ +#ifdef Py_GIL_DISABLED + +static inline void +ASSERT_DICT_LOCKED(PyObject *op) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +} +#define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + ASSERT_DICT_LOCKED(op); \ + } +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); \ + } + +#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) +#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); +#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); +#define ASSERT_OWNED_OR_SHARED(mp) \ + assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); + +#define LOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + LOCK_KEYS(keys); \ + } + +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + UNLOCK_KEYS(keys); \ + } + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + PyDictKeysObject *keys = _Py_atomic_load_ptr(&mp->ma_keys); + return _Py_atomic_load_ssize(&keys->dk_nentries); +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_keys, keys); +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_values, values); +} + +#define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) + +#define ASSERT_KEYS_LOCKED(keys) assert(PyMutex_IsLocked(&keys->dk_mutex)) +#define LOAD_SHARED_KEY(key) _Py_atomic_load_ptr_acquire(&key) +#define STORE_SHARED_KEY(key, value) _Py_atomic_store_ptr_release(&key, value) +// Inc refs the keys object, giving the previous value +#define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) +// Dec refs the keys object, giving the previous value +#define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) +#define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) + +#define INCREF_KEYS_FT(dk) dictkeys_incref(dk) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + ASSERT_KEYS_LOCKED(keys); + + // We increase before we decrease so we never get too small of a value + // when we're racing with reads + _Py_atomic_store_ssize_relaxed(&keys->dk_nentries, keys->dk_nentries + 1); + _Py_atomic_store_ssize_release(&keys->dk_usable, keys->dk_usable - 1); +} + +#else /* Py_GIL_DISABLED */ + +#define ASSERT_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) +#define LOCK_KEYS(keys) +#define UNLOCK_KEYS(keys) +#define ASSERT_KEYS_LOCKED(keys) +#define LOAD_SHARED_KEY(key) key +#define STORE_SHARED_KEY(key, value) key = value +#define INCREF_KEYS(dk) dk->dk_refcnt++ +#define DECREF_KEYS(dk) dk->dk_refcnt-- +#define LOAD_KEYS_NENTRIES(keys) keys->dk_nentries +#define INCREF_KEYS_FT(dk) +#define DECREF_KEYS_FT(dk, shared) +#define LOCK_KEYS_IF_SPLIT(keys, kind) +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) +#define IS_DICT_SHARED(mp) (false) +#define SET_DICT_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] +#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + keys->dk_usable--; + keys->dk_nentries++; +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + mp->ma_keys = keys; +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + mp->ma_values = values; +} + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + return mp->ma_keys->dk_nentries; +} + + +#endif + +#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_key, key) +#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, value) +#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value) +#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED(ep->me_hash, hash) +#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable) +#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries) +#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used) + #define PERTURB_SHIFT 5 /* @@ -237,45 +380,56 @@ equally good collision statistics, needed less code & used less memory. static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log_newsize, int unicode); -static PyObject* dict_iter(PyDictObject *dict); +static PyObject* dict_iter(PyObject *dict); + +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result); + +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); +#endif #include "clinic/dictobject.c.h" -#if PyDict_MAXFREELIST > 0 -static struct _Py_dict_state * -get_dict_state(PyInterpreterState *interp) +#ifdef WITH_FREELISTS +static struct _Py_dict_freelist * +get_dict_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->dicts; +} + +static struct _Py_dictkeys_freelist * +get_dictkeys_freelist(void) { - return &interp->dict_state; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->dictkeys; } #endif void -_PyDict_ClearFreeList(PyInterpreterState *interp) +_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - while (state->numfree) { - PyDictObject *op = state->free_list[--state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = &freelists->dicts; + while (freelist->numfree > 0) { + PyDictObject *op = freelist->items[--freelist->numfree]; assert(PyDict_CheckExact(op)); PyObject_GC_Del(op); } - while (state->keys_numfree) { - PyObject_Free(state->keys_free_list[--state->keys_numfree]); + struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; + while (keys_freelist->numfree > 0) { + PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); + } + if (is_finalization) { + freelist->numfree = -1; + keys_freelist->numfree = -1; } -#endif -} - - -void -_PyDict_Fini(PyInterpreterState *interp) -{ - _PyDict_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = &interp->dict_state; - state->numfree = -1; - state->keys_numfree = -1; #endif } @@ -283,24 +437,26 @@ static inline Py_hash_t unicode_get_hash(PyObject *o) { assert(PyUnicode_CheckExact(o)); - return _PyASCIIObject_CAST(o)->hash; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); } /* Print summary info about the state of the optimized allocator */ void _PyDict_DebugMallocStats(FILE *out) { -#if PyDict_MAXFREELIST > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_dict_state *state = get_dict_state(interp); +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", - state->numfree, sizeof(PyDictObject)); + dict_freelist->numfree, sizeof(PyDictObject)); + struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _PyDebugAllocatorStats(out, "free PyDictKeysObject", + dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); #endif } #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -312,27 +468,43 @@ static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) static inline void dictkeys_incref(PyDictKeysObject *dk) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif - dk->dk_refcnt++; + INCREF_KEYS(dk); } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } - assert(dk->dk_refcnt > 0); + assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif - if (--dk->dk_refcnt == 0) { - free_keys_object(interp, dk); + if (DECREF_KEYS(dk) == 1) { + if (DK_IS_UNICODE(dk)) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + else { + PyDictKeyEntry *entries = DK_ENTRIES(dk); + Py_ssize_t i, n; + for (i = 0, n = dk->dk_nentries; i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); + } + } + free_keys_object(dk, use_qsbr); } } @@ -344,22 +516,18 @@ dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i) Py_ssize_t ix; if (log2size < 8) { - const int8_t *indices = (const int8_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 8, i); } else if (log2size < 16) { - const int16_t *indices = (const int16_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 16, i); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - const int64_t *indices = (const int64_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 64, i); } #endif else { - const int32_t *indices = (const int32_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 32, i); } assert(ix >= DKIX_DUMMY); return ix; @@ -375,25 +543,21 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) assert(keys->dk_version == 0); if (log2size < 8) { - int8_t *indices = (int8_t*)(keys->dk_indices); assert(ix <= 0x7f); - indices[i] = (char)ix; + STORE_INDEX(keys, 8, i, ix); } else if (log2size < 16) { - int16_t *indices = (int16_t*)(keys->dk_indices); assert(ix <= 0x7fff); - indices[i] = (int16_t)ix; + STORE_INDEX(keys, 16, i, ix); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - int64_t *indices = (int64_t*)(keys->dk_indices); - indices[i] = ix; + STORE_INDEX(keys, 64, i, ix); } #endif else { - int32_t *indices = (int32_t*)(keys->dk_indices); assert(ix <= 0x7fffffff); - indices[i] = (int32_t)ix; + STORE_INDEX(keys, 32, i, ix); } } @@ -466,6 +630,9 @@ static PyDictKeysObject empty_keys_struct = { 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ +#ifdef Py_GIL_DISABLED + {0}, /* dk_mutex */ +#endif 1, /* dk_version */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ @@ -488,8 +655,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -512,6 +680,8 @@ dump_entries(PyDictKeysObject *dk) int _PyDict_CheckConsistency(PyObject *op, int check_content) { + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) @@ -523,10 +693,15 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) int splitted = _PyDict_HasSplitTable(mp); Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); + // In the free-threaded build, shared keys may be concurrently modified, + // so use atomic loads. + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); - CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); - CHECK(keys->dk_usable + keys->dk_nentries <= usable); + CHECK(0 <= dk_usable && dk_usable <= usable); + CHECK(0 <= dk_nentries && dk_nentries <= usable); + CHECK(dk_usable + dk_nentries <= usable); if (!splitted) { /* combined table */ @@ -536,9 +711,14 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { + LOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); @@ -594,6 +774,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) CHECK(mp->ma_values->values[index] != NULL); } } + UNLOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); } return 1; @@ -627,34 +808,33 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) { - dk = state->keys_free_list[--state->keys_numfree]; +#ifdef WITH_FREELISTS + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); + if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { + dk = freelist->items[--freelist->numfree]; OBJECT_STAT_INC(from_freelist); } else #endif { - dk = PyObject_Malloc(sizeof(PyDictKeysObject) - + ((size_t)1 << log2_bytes) - + entry_size * usable); + dk = PyMem_Malloc(sizeof(PyDictKeysObject) + + ((size_t)1 << log2_bytes) + + entry_size * usable); if (dk == NULL) { PyErr_NoMemory(); return NULL; } } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; dk->dk_log2_index_bytes = log2_bytes; dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL; +#ifdef Py_GIL_DISABLED + dk->dk_mutex = (PyMutex){0}; +#endif dk->dk_nentries = 0; dk->dk_usable = usable; dk->dk_version = 0; @@ -664,63 +844,66 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { - assert(keys != Py_EMPTY_KEYS); - if (DK_IS_UNICODE(keys)) { - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } - } - else { - PyDictKeyEntry *entries = DK_ENTRIES(keys); - Py_ssize_t i, n; - for (i = 0, n = keys->dk_nentries; i < n; i++) { - Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); - } +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeDelayed(keys); + return; } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // free_keys_object() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); #endif +#ifdef WITH_FREELISTS + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && state->keys_numfree < PyDict_MAXFREELIST + && freelist->numfree < PyDict_MAXFREELIST + && freelist->numfree >= 0 && DK_IS_UNICODE(keys)) { - state->keys_free_list[state->keys_numfree++] = keys; + freelist->items[freelist->numfree++] = keys; OBJECT_STAT_INC(to_freelist); return; } #endif - PyObject_Free(keys); + PyMem_Free(keys); +} + +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; } +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void -free_values(PyDictValues *values) +free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = ((uint8_t *)values)[-1]; - PyMem_Free(((char *)values)-prefix_size); + assert(values->embedded == 0); +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeDelayed(values); + return; + } +#endif + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -731,14 +914,10 @@ new_dict(PyInterpreterState *interp, { PyDictObject *mp; assert(keys != NULL); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { - mp = state->free_list[--state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree > 0) { + mp = freelist->items[--freelist->numfree]; assert (mp != NULL); assert (Py_IS_TYPE(mp, &PyDict_Type)); OBJECT_STAT_INC(from_freelist); @@ -749,9 +928,9 @@ new_dict(PyInterpreterState *interp, { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); if (free_values_on_failure) { - free_values(values); + free_values(values, false); } return NULL; } @@ -764,23 +943,15 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -} - -/* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; + dictkeys_incref(keys); for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -792,13 +963,15 @@ static PyDictKeysObject * clone_combined_dict_keys(PyDictObject *orig) { assert(PyDict_Check(orig)); - assert(Py_TYPE(orig)->tp_iter == (getiterfunc)dict_iter); + assert(Py_TYPE(orig)->tp_iter == dict_iter); assert(orig->ma_values == NULL); assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); + ASSERT_DICT_LOCKED(orig); + size_t keys_size = _PyDict_KeysSize(orig->ma_keys); - PyDictKeysObject *keys = PyObject_Malloc(keys_size); + PyDictKeysObject *keys = PyMem_Malloc(keys_size); if (keys == NULL) { PyErr_NoMemory(); return NULL; @@ -840,7 +1013,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return keys; } @@ -875,11 +1048,11 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) Py_UNREACHABLE(); } -// Search non-Unicode key from Unicode table -static Py_ssize_t -unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +static inline Py_ALWAYS_INLINE Py_ssize_t +do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, + Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); + void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); size_t perturb = hash; size_t i = (size_t)hash & mask; @@ -887,73 +1060,26 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } - if (unicode_get_hash(ep->me_key) == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } } else if (ix == DKIX_EMPTY) { return DKIX_EMPTY; } perturb >>= PERTURB_SHIFT; i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} -// Search Unicode key from Unicode table. -static Py_ssize_t _Py_HOT_FUNCTION -unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) -{ - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { - return ix; - } - } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - PyDictUnicodeEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + if (cmp < 0) { + return cmp; + } else if (cmp) { return ix; } } @@ -966,49 +1092,94 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_UNREACHABLE(); } -// Search key from Generic table. +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + + if (unicode_get_hash(ep->me_key) == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +// Search non-Unicode key from Unicode table static Py_ssize_t -dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) { - PyDictKeyEntry *ep0 = DK_ENTRIES(dk); - size_t mask = DK_MASK(dk); - size_t perturb = hash; - size_t i = (size_t)hash & mask; - Py_ssize_t ix; - for (;;) { - ix = dictkeys_get_index(dk, i); - if (ix >= 0) { - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - if (ep->me_key == key) { - return ix; - } - if (ep->me_hash == hash) { - PyObject *startkey = ep->me_key; - Py_INCREF(startkey); - int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) { - return DKIX_ERROR; - } - if (dk == mp->ma_keys && ep->me_key == startkey) { - if (cmp > 0) { - return ix; - } - } - else { - /* The dict was mutated, restart */ - return DKIX_KEY_CHANGED; - } - } + return do_lookup(mp, dk, key, hash, compare_unicode_generic); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + assert(ep_key != NULL); + assert(PyUnicode_CheckExact(ep_key)); + if (ep_key == key || + (unicode_get_hash(ep_key) == hash && unicode_eq(ep_key, key))) { + return 1; + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_generic(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + assert(ep->me_key != NULL); + if (ep->me_key == key) { + return 1; + } + if (ep->me_hash == hash) { + PyObject *startkey = ep->me_key; + Py_INCREF(startkey); + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; } - else if (ix == DKIX_EMPTY) { - return DKIX_EMPTY; + if (dk == mp->ma_keys && ep->me_key == startkey) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); } - Py_UNREACHABLE(); + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic); } /* Lookup a string in a (all unicode) dict keys. @@ -1035,6 +1206,14 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) return unicodekeys_lookup_unicode(dk, key, hash); } +#ifdef Py_GIL_DISABLED + +static Py_ssize_t +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, + Py_hash_t hash); + +#endif + /* The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -1057,16 +1236,40 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu DictKeysKind kind; Py_ssize_t ix; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); start: dk = mp->ma_keys; kind = dk->dk_kind; if (kind != DICT_KEYS_GENERAL) { if (PyUnicode_CheckExact(key)) { +#ifdef Py_GIL_DISABLED + if (kind == DICT_KEYS_SPLIT) { + // A split dictionaries keys can be mutated by other + // dictionaries but if we have a unicode key we can avoid + // locking the shared keys. + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + UNLOCK_KEYS(dk); + } + } + else { + ix = unicodekeys_lookup_unicode(dk, key, hash); + } +#else ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif } else { + INCREF_KEYS_FT(dk); + LOCK_KEYS_IF_SPLIT(dk, kind); + ix = unicodekeys_lookup_generic(mp, dk, key, hash); + + UNLOCK_KEYS_IF_SPLIT(dk, kind); + DECREF_KEYS_FT(dk, IS_DICT_SHARED(mp)); if (ix == DKIX_KEY_CHANGED) { goto start; } @@ -1100,44 +1303,311 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } -int -_PyDict_HasOnlyStringKeys(PyObject *dict) +#ifdef Py_GIL_DISABLED +static inline void +ensure_shared_on_read(PyDictObject *mp) { - Py_ssize_t pos = 0; - PyObject *key, *value; - assert(PyDict_Check(dict)); - /* Shortcut */ - if (((PyDictObject *)dict)->ma_keys->dk_kind != DICT_KEYS_GENERAL) - return 1; - while (PyDict_Next(dict, &pos, &key, &value)) - if (!PyUnicode_Check(key)) - return 0; - return 1; + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // The first time we access a dict from a non-owning thread we mark it + // as shared. This ensures that a concurrent resize operation will + // delay freeing the old keys or values using QSBR, which is necessary + // to safely allow concurrent reads without locking... + Py_BEGIN_CRITICAL_SECTION(mp); + if (!IS_DICT_SHARED(mp)) { + SET_DICT_SHARED(mp); + } + Py_END_CRITICAL_SECTION(); + } } +#endif -#define MAINTAIN_TRACKING(mp, key, value) \ - do { \ - if (!_PyObject_GC_IS_TRACKED(mp)) { \ - if (_PyObject_GC_MAY_BE_TRACKED(key) || \ - _PyObject_GC_MAY_BE_TRACKED(value)) { \ - _PyObject_GC_TRACK(mp); \ - } \ - } \ - } while(0) +static inline void +ensure_shared_on_resize(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are writing to the dict from another thread that owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + // Technically this is a little heavy handed, we could just + // free the individual old keys / old-values using qsbr + SET_DICT_SHARED(mp); + } +#endif +} -void -_PyDict_MaybeUntrack(PyObject *op) +#ifdef Py_GIL_DISABLED + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { - PyDictObject *mp; - PyObject *value; - Py_ssize_t i, numentries; + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); - if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) - return; + if (startkey != NULL) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } - mp = (PyDictObject *) op; - numentries = mp->ma_keys->dk_nentries; - if (_PyDict_HasSplitTable(mp)) { + if (unicode_get_hash(startkey) == hash) { + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + else { + Py_DECREF(startkey); + } + } + return 0; +} + +// Search non-Unicode key from Unicode table +static Py_ssize_t +unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(startkey)); + if (startkey == key) { + return 1; + } + if (startkey != NULL) { + if (_Py_IsImmortal(startkey)) { + return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); + } + else { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { + Py_DECREF(startkey); + return 1; + } + Py_DECREF(startkey); + } + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); +} + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + if (startkey == key) { + return 1; + } + Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); + if (ep_hash == hash) { + if (startkey == NULL || !_Py_TryIncrefCompare(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); +} + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyObject *value; + + ensure_shared_on_read(mp); + + dk = _Py_atomic_load_ptr(&mp->ma_keys); + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic_threadsafe(mp, dk, key, hash); + } + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + + if (ix >= 0) { + if (kind == DICT_KEYS_SPLIT) { + PyDictValues *values = _Py_atomic_load_ptr(&mp->ma_values); + if (values == NULL) + goto read_failed; + + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); + if (ix >= (Py_ssize_t)capacity) + goto read_failed; + + value = _Py_TryXGetRef(&values->values[ix]); + if (value == NULL) + goto read_failed; + + if (values != _Py_atomic_load_ptr(&mp->ma_values)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = _Py_TryXGetRef(&DK_UNICODE_ENTRIES(dk)[ix].me_value); + if (value == NULL) { + goto read_failed; + } + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + } + else { + value = NULL; + } + } + else { + ix = dictkeys_generic_lookup_threadsafe(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + if (ix >= 0) { + value = _Py_TryXGetRef(&DK_ENTRIES(dk)[ix].me_value); + if (value == NULL) + goto read_failed; + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = NULL; + } + } + + *value_addr = value; + return ix; + +read_failed: + // In addition to the normal races of the dict being modified the _Py_TryXGetRef + // can all fail if they don't yet have a shared ref count. That can happen here + // or in the *_lookup_* helper. In that case we need to take the lock to avoid + // mutation and do a normal incref which will make them shared. + Py_BEGIN_CRITICAL_SECTION(mp); + ix = _Py_dict_lookup(mp, key, hash, &value); + *value_addr = value; + if (value != NULL) { + assert(ix >= 0); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + return ix; +} + +#else // Py_GIL_DISABLED + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, value_addr); + Py_XNewRef(*value_addr); + return ix; +} + +#endif + +int +_PyDict_HasOnlyStringKeys(PyObject *dict) +{ + Py_ssize_t pos = 0; + PyObject *key, *value; + assert(PyDict_Check(dict)); + /* Shortcut */ + if (((PyDictObject *)dict)->ma_keys->dk_kind != DICT_KEYS_GENERAL) + return 1; + while (PyDict_Next(dict, &pos, &key, &value)) + if (!PyUnicode_Check(key)) + return 0; + return 1; +} + +#define MAINTAIN_TRACKING(mp, key, value) \ + do { \ + if (!_PyObject_GC_IS_TRACKED(mp)) { \ + if (_PyObject_GC_MAY_BE_TRACKED(key) || \ + _PyObject_GC_MAY_BE_TRACKED(value)) { \ + _PyObject_GC_TRACK(mp); \ + } \ + } \ + } while(0) + +void +_PyDict_MaybeUntrack(PyObject *op) +{ + PyDictObject *mp; + PyObject *value; + Py_ssize_t i, numentries; + + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + + if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) + return; + + mp = (PyDictObject *) op; + ASSERT_CONSISTENT(mp); + numentries = mp->ma_keys->dk_nentries; + if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < numentries; i++) { if ((value = mp->ma_values->values[i]) == NULL) continue; @@ -1170,10 +1640,19 @@ _PyDict_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +static inline int +is_unusable_slot(Py_ssize_t ix) +{ +#ifdef Py_GIL_DISABLED + return ix >= 0 || ix == DKIX_DUMMY; +#else + return ix >= 0; +#endif +} + /* Internal function to find slot for an item from its hash when it is known that the key is not present in the dict. - - The dict must be combined. */ + */ static Py_ssize_t find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) { @@ -1182,7 +1661,7 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) const size_t mask = DK_MASK(keys); size_t i = hash & mask; Py_ssize_t ix = dictkeys_get_index(keys, i); - for (size_t perturb = hash; ix >= 0;) { + for (size_t perturb = hash; is_unusable_slot(ix);) { perturb >>= PERTURB_SHIFT; i = (i*5 + perturb + 1) & mask; ix = dictkeys_get_index(keys, i); @@ -1196,38 +1675,93 @@ insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } -static Py_ssize_t -insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) +static inline int +insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) { - assert(PyUnicode_CheckExact(name)); - Py_hash_t hash = unicode_get_hash(name); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(name); - if (hash == -1) { - PyErr_Clear(); - return DKIX_EMPTY; + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(interp, mp, 1) < 0) { + return -1; } } - Py_ssize_t ix = unicodekeys_lookup_unicode(keys, name, hash); - if (ix == DKIX_EMPTY) { - if (keys->dk_usable <= 0) { - return DKIX_EMPTY; - } - /* Insert into new slot. */ + + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); + dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); + + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + } + else { + PyDictKeyEntry *ep; + ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + STORE_HASH(ep, hash); + } + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); + STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); + assert(mp->ma_keys->dk_usable >= 0); + return 0; +} + +static Py_ssize_t +insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) +{ + assert(PyUnicode_CheckExact(key)); + Py_ssize_t ix; + + +#ifdef Py_GIL_DISABLED + ix = unicodekeys_lookup_unicode_threadsafe(keys, key, hash); + if (ix >= 0) { + return ix; + } +#endif + + LOCK_KEYS(keys); + ix = unicodekeys_lookup_unicode(keys, key, hash); + if (ix == DKIX_EMPTY && keys->dk_usable > 0) { + // Insert into new slot keys->dk_version = 0; Py_ssize_t hashpos = find_empty_slot(keys, hash); ix = keys->dk_nentries; - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; dictkeys_set_index(keys, hashpos, ix); - assert(ep->me_key == NULL); - ep->me_key = Py_NewRef(name); - keys->dk_usable--; - keys->dk_nentries++; + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; + STORE_SHARED_KEY(ep->me_key, Py_NewRef(key)); + split_keys_entry_added(keys); } assert (ix < SHARED_KEYS_MAX_SIZE); + UNLOCK_KEYS(keys); return ix; } +static void +insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) +{ + assert(PyUnicode_CheckExact(key)); + ASSERT_DICT_LOCKED(mp); + MAINTAIN_TRACKING(mp, key, value); + PyObject *old_value = mp->ma_values->values[ix]; + if (old_value == NULL) { + uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); + STORE_USED(mp, mp->ma_used + 1); + mp->ma_version_tag = new_version; + } + else { + uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + mp->ma_version_tag = new_version; + Py_DECREF(old_value); + } + ASSERT_CONSISTENT(mp); +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -1240,12 +1774,29 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, { PyObject *old_value; + ASSERT_DICT_LOCKED(mp); + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(interp, mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + insert_split_value(interp, mp, key, value, ix); + Py_DECREF(key); + Py_DECREF(value); + return 0; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto Fail; + } + } + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) goto Fail; @@ -1253,46 +1804,17 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { + assert(!_PyDict_HasSplitTable(mp)); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); - if (mp->ma_keys->dk_usable <= 0) { - /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) - goto Fail; - } - - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - if (mp->ma_values) { - Py_ssize_t index = mp->ma_keys->dk_nentries; - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - assert (mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = value; - } - else { - ep->me_value = value; - } - } - else { - PyDictKeyEntry *ep; - ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; + if (insert_combined_dict(interp, mp, hash, key, value) < 0) { + goto Fail; } - mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; - assert(mp->ma_keys->dk_usable >= 0); + STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; } @@ -1300,21 +1822,15 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (old_value != value) { uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_MODIFIED, mp, key, value); - if (_PyDict_HasSplitTable(mp)) { - mp->ma_values->values[ix] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - } + assert(old_value != NULL); + assert(!_PyDict_HasSplitTable(mp)); + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } else { - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; - } - else { - DK_ENTRIES(mp->ma_keys)[ix].me_value = value; - } + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } mp->ma_version_tag = new_version; } @@ -1329,13 +1845,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, return -1; } -// Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. +// Same as insertdict but specialized for ma_keys == Py_EMPTY_KEYS. // Consumes key and value references. static int insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); + ASSERT_DICT_LOCKED(mp); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1349,28 +1866,33 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, return -1; } /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ - mp->ma_keys = newkeys; - mp->ma_values = NULL; + assert(mp->ma_values == NULL); MAINTAIN_TRACKING(mp, key, value); size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); - dictkeys_set_index(mp->ma_keys, hashpos, 0); + dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; - ep->me_value = value; + STORE_VALUE(ep, value); } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; + STORE_VALUE(ep, value); } - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + newkeys->dk_usable--; + newkeys->dk_nentries++; + // We store the keys last so no one can see them in a partially inconsistent + // state so that we don't need to switch the keys to being shared yet for + // the case where we're inserting from the non-owner thread. We don't use + // set_keys here because the transition from empty to non-empty is safe + // as the empty keys will never be freed. + FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys); return 0; } @@ -1415,7 +1937,7 @@ actually be smaller than the old one. If a table is split (its keys and hashes are shared, its values are not), then the values are temporarily copied into the table, it is resized as a combined table, then the me_value slots in the old table are NULLed out. -After resizing a table is always combined. +After resizing, a table is always combined. This function supports: - Unicode split -> Unicode combined or Generic @@ -1426,9 +1948,11 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log2_newsize, int unicode) { - PyDictKeysObject *oldkeys; + PyDictKeysObject *oldkeys, *newkeys; PyDictValues *oldvalues; + ASSERT_DICT_LOCKED(mp); + if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; @@ -1442,30 +1966,31 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, unicode = 0; } + ensure_shared_on_resize(mp); /* NOTE: Current odict checks mp->ma_keys to detect resize happen. * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. * TODO: Try reusing oldkeys when reimplement odict. */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); - if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; + newkeys = new_keys_object(interp, log2_newsize, unicode); + if (newkeys == NULL) { return -1; } // New table must be large enough. - assert(mp->ma_keys->dk_usable >= mp->ma_used); + assert(newkeys->dk_usable >= mp->ma_used); Py_ssize_t numentries = mp->ma_used; if (oldvalues != NULL) { - PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); + LOCK_KEYS(oldkeys); + PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); /* Convert split table into new combined table. * We must incref keys; we can transfer values. */ - if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { + if (newkeys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1475,10 +2000,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_hash = unicode_get_hash(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // split -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1487,18 +2012,27 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_key = Py_NewRef(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); + } + UNLOCK_KEYS(oldkeys); + set_keys(mp, newkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + set_values(mp, NULL); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0); + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); } - dictkeys_decref(interp, oldkeys); - mp->ma_values = NULL; - free_values(oldvalues); } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { // generic -> generic - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + assert(newkeys->dk_kind == DICT_KEYS_GENERAL); PyDictKeyEntry *oldentries = DK_ENTRIES(oldkeys); - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries) { memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); } @@ -1510,12 +2044,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // oldkeys is combined unicode PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); if (unicode) { // combined unicode -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries && mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { memcpy(newentries, oldentries, numentries * sizeof(PyDictUnicodeEntry)); } @@ -1527,10 +2061,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } else { // combined unicode -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { while (ep->me_value == NULL) @@ -1540,41 +2074,24 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_value = ep->me_value; ep++; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } } - // We can not use free_keys_object here because key's reference - // are moved already. + set_keys(mp, newkeys); + if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // dictresize() must not be called after _PyDict_Fini() - assert(state->keys_numfree != -1); -#endif - if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE && - DK_IS_UNICODE(oldkeys) && - state->keys_numfree < PyDict_MAXFREELIST) - { - state->keys_free_list[state->keys_numfree++] = oldkeys; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_Free(oldkeys); - } + free_keys_object(oldkeys, IS_DICT_SHARED(mp)); } } - mp->ma_keys->dk_usable -= numentries; - mp->ma_keys->dk_nentries = numentries; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries); + STORE_KEYS_NENTRIES(mp->ma_keys, numentries); ASSERT_CONSISTENT(mp); return 0; } @@ -1642,7 +2159,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, for (Py_ssize_t i = 0; i < length; i++) { PyObject *key = *ks; PyObject *value = *vs; - if (PyDict_SetItem(dict, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) { Py_DECREF(dict); return NULL; } @@ -1663,21 +2180,18 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, * function hits a stack-depth error, which can cause this to return NULL * even if the key is present. */ -PyObject * -PyDict_GetItem(PyObject *op, PyObject *key) +static PyObject * +dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) { if (!PyDict_Check(op)) { return NULL; } PyDictObject *mp = (PyDictObject *)op; - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - PyErr_Clear(); - return NULL; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + PyErr_FormatUnraisable(warnmsg); + return NULL; } PyThreadState *tstate = _PyThreadState_GET(); @@ -1691,30 +2205,44 @@ PyDict_GetItem(PyObject *op, PyObject *key) PyObject *value; Py_ssize_t ix; (void)ix; - PyObject *exc = _PyErr_GetRaisedException(tstate); +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif /* Ignore any exception raised by the lookup */ + PyObject *exc2 = _PyErr_Occurred(tstate); + if (exc2 && !PyErr_GivenExceptionMatches(exc2, PyExc_KeyError)) { + PyErr_FormatUnraisable(warnmsg); + } _PyErr_SetRaisedException(tstate, exc); assert(ix >= 0 || value == NULL); return value; // borrowed reference } +PyObject * +PyDict_GetItem(PyObject *op, PyObject *key) +{ + return dict_getitem(op, key, + "Exception ignored in PyDict_GetItem(); consider using " + "PyDict_GetItemRef() or PyDict_GetItemWithError()"); +} + Py_ssize_t _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) { + // TODO: Thread safety PyObject *value; assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); - Py_hash_t hash = unicode_get_hash(key); + Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - return -1; - } + return -1; } return _Py_dict_lookup(mp, key, hash, &value); @@ -1736,11 +2264,45 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ + PyObject *value; +#ifdef Py_GIL_DISABLED + Py_ssize_t ix = _Py_dict_lookup_threadsafe(op, key, hash, &value); +#else + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); +#endif + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } +#ifdef Py_GIL_DISABLED + *result = value; +#else + *result = Py_NewRef(value); +#endif + return 1; // key is present +} int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) @@ -1750,20 +2312,30 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) *result = NULL; return -1; } - PyDictObject*mp = (PyDictObject *)op; - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - *result = NULL; - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; + } + + return _PyDict_GetItemRef_KnownHash((PyDictObject *)op, key, hash, result); +} + +int +_PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result) +{ + ASSERT_DICT_LOCKED(op); + assert(PyUnicode_CheckExact(key)); + + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; } PyObject *value; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); assert(ix >= 0 || value == NULL); if (ix == DKIX_ERROR) { *result = NULL; @@ -1777,7 +2349,6 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) return 1; // key is present } - /* Variant of PyDict_GetItem() that doesn't suppress exceptions. This returns NULL *with* an exception set if an exception occurred. It returns NULL *without* an exception set if the key wasn't present. @@ -1794,15 +2365,17 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) PyErr_BadInternalCall(); return NULL; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - return NULL; - } + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } @@ -1811,7 +2384,7 @@ PyObject * _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) { assert(PyUnicode_CheckExact(kv)); - Py_hash_t hash = kv->ob_type->tp_hash(kv); + Py_hash_t hash = Py_TYPE(kv)->tp_hash(kv); if (hash == -1) { return NULL; } @@ -1830,6 +2403,19 @@ _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } +PyObject * +_PyDict_GetItemStringWithError(PyObject *v, const char *key) +{ + PyObject *kv, *rv; + kv = PyUnicode_FromString(key); + if (kv == NULL) { + return NULL; + } + rv = PyDict_GetItemWithError(v, kv); + Py_DECREF(kv); + return rv; +} + /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * @@ -1839,6 +2425,8 @@ _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) * Raise an exception and return NULL if an error occurred (ex: computing the * key hash failed, key comparison failed, ...). Return NULL if the key doesn't * exist. Return the value if the key exists. + * + * Returns a new reference. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) @@ -1847,42 +2435,42 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } /* namespace 1: globals */ - ix = _Py_dict_lookup(globals, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix != DKIX_EMPTY && value != NULL) return value; /* namespace 2: builtins */ - ix = _Py_dict_lookup(builtins, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); assert(ix >= 0 || value == NULL); return value; } /* Consumes references to key and value */ -int -_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +static int +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { + ASSERT_DICT_LOCKED(mp); + assert(key); assert(value); assert(PyDict_Check(mp)); - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - Py_DECREF(key); - Py_DECREF(value); - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + Py_DECREF(key); + Py_DECREF(value); + return -1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { return insert_to_emptydict(interp, mp, key, hash, value); } @@ -1890,6 +2478,16 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return insertdict(interp, mp, key, hash, value); } +int +_PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = setitem_take2_lock_held(mp, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the * dictionary if it's merely replacing the value for an existing key. * This means that it's safe to loop over a dictionary with PyDict_Next() @@ -1909,6 +2507,16 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } +static int +setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) +{ + assert(key); + assert(value); + return setitem_take2_lock_held(mp, + Py_NewRef(key), Py_NewRef(value)); +} + + int _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, Py_hash_t hash) @@ -1924,28 +2532,39 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); + + Py_BEGIN_CRITICAL_SECTION(mp); + if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } - /* insertdict() handles any resizing that might be necessary */ - return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + else { + /* insertdict() handles any resizing that might be necessary */ + res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + + Py_END_CRITICAL_SECTION(); + return res; } static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } static int @@ -1954,14 +2573,16 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, { PyObject *old_key; + ASSERT_DICT_LOCKED(mp); + Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - mp->ma_used--; + STORE_USED(mp, mp->ma_used - 1); mp->ma_version_tag = new_version; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); - mp->ma_values->values[ix] = NULL; + STORE_SPLIT_VALUE(mp, ix, NULL); assert(ix < SHARED_KEYS_MAX_SIZE); /* Update order */ delete_index_from_values(mp->ma_values, ix); @@ -1973,15 +2594,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - ep->me_hash = 0; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); + STORE_HASH(ep, 0); } Py_DECREF(old_key); } @@ -1994,19 +2615,17 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, int PyDict_DelItem(PyObject *op, PyObject *key) { - Py_hash_t hash; assert(key); - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return _PyDict_DelItem_KnownHash(op, key, hash); } -int -_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +static int +delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2016,6 +2635,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyErr_BadInternalCall(); return -1; } + + ASSERT_DICT_LOCKED(op); + assert(key); assert(hash != -1); mp = (PyDictObject *)op; @@ -2033,20 +2655,28 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return delitem_common(mp, hash, ix, old_value, new_version); } -/* This function promises that the predicate -> deletion sequence is atomic - * (i.e. protected by the GIL), assuming the predicate itself doesn't - * release the GIL. - */ int -_PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) +_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitem_knownhash_lock_held(op, key, hash); + Py_END_CRITICAL_SECTION(); + return res; +} + +static int +delitemif_lock_held(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) { - Py_ssize_t hashpos, ix; + Py_ssize_t ix; PyDictObject *mp; Py_hash_t hash; PyObject *old_value; int res; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -2068,28 +2698,41 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, if (res == -1) return -1; - hashpos = lookdict_index(mp->ma_keys, hash, ix); - assert(hashpos >= 0); - if (res > 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hashpos, ix, old_value, new_version); + return delitem_common(mp, hash, ix, old_value, new_version); } else { return 0; } } +/* This function promises that the predicate -> deletion sequence is atomic + * (i.e. protected by the GIL or the per-dict mutex in free threaded builds), + * assuming the predicate itself doesn't release the GIL (or cause re-entrancy + * which would release the per-dict mutex) + */ +int +_PyDict_DelItemIf(PyObject *op, PyObject *key, + int (*predicate)(PyObject *value)) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(op); + res = delitemif_lock_held(op, key, predicate); + Py_END_CRITICAL_SECTION(); + return res; +} - -void -PyDict_Clear(PyObject *op) +static void +clear_lock_held(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; + ASSERT_DICT_LOCKED(op); + if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2102,26 +2745,41 @@ PyDict_Clear(PyObject *op) PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); - dictkeys_incref(Py_EMPTY_KEYS); - mp->ma_keys = Py_EMPTY_KEYS; - mp->ma_values = NULL; - mp->ma_used = 0; + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); mp->ma_version_tag = new_version; - /* ...then clear the keys and values */ - if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues); - dictkeys_decref(interp, oldkeys); + STORE_USED(mp, 0); + if (oldvalues == NULL) { + set_keys(mp, Py_EMPTY_KEYS); + assert(oldkeys->dk_refcnt == 1); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } else { - assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) { + Py_CLEAR(oldvalues->values[i]); + } + if (oldvalues->embedded) { + oldvalues->size = 0; + } + else { + set_values(mp, NULL); + set_keys(mp, Py_EMPTY_KEYS); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); + } } ASSERT_CONSISTENT(mp); } +void +PyDict_Clear(PyObject *op) +{ + Py_BEGIN_CRITICAL_SECTION(op); + clear_lock_held(op); + Py_END_CRITICAL_SECTION(); +} + /* Internal version of PyDict_Next that returns a hash value in addition * to the key and value. * Return 1 on success, return 0 when the reached the end of the dictionary @@ -2138,16 +2796,16 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; + mp = (PyDictObject *)op; i = *ppos; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); if (i < 0 || i >= mp->ma_used) return 0; int index = get_index_from_order(mp, i); value = mp->ma_values->values[index]; - - key = DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key); hash = unicode_get_hash(key); assert(value != NULL); } @@ -2214,121 +2872,210 @@ PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) return _PyDict_Next(op, ppos, pkey, pvalue, NULL); } + /* Internal version of dict.pop(). */ -PyObject * -_PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *deflt) +int +_PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, + PyObject **result) { - Py_ssize_t ix; - PyObject *old_value; - PyDictObject *mp; - PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(PyDict_Check(mp)); - assert(PyDict_Check(dict)); - mp = (PyDictObject *)dict; + ASSERT_DICT_LOCKED(mp); if (mp->ma_used == 0) { - if (deflt) { - return Py_NewRef(deflt); + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + return 0; } - ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - return NULL; + + PyObject *old_value; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } + if (ix == DKIX_EMPTY || old_value == NULL) { - if (deflt) { - return Py_NewRef(deflt); + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + return 0; } + assert(old_value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); - return old_value; + if (result) { + *result = old_value; + } + else { + Py_DECREF(old_value); + } + return 1; } -PyObject * -_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *deflt) +static int +pop_lock_held(PyObject *op, PyObject *key, PyObject **result) { - Py_hash_t hash; + ASSERT_DICT_LOCKED(op); - if (((PyDictObject *)dict)->ma_used == 0) { - if (deflt) { - return Py_NewRef(deflt); + if (!PyDict_Check(op)) { + if (result) { + *result = NULL; } - _PyErr_SetKeyError(key); - return NULL; + PyErr_BadInternalCall(); + return -1; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + PyDictObject *dict = (PyDictObject *)op; + + if (dict->ma_used == 0) { + if (result) { + *result = NULL; + } + return 0; + } + + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; } - return _PyDict_Pop_KnownHash(dict, key, hash, deflt); + return _PyDict_Pop_KnownHash(dict, key, hash, result); } -/* Internal version of dict.from_keys(). It is subclass-friendly. */ -PyObject * -_PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) +int +PyDict_Pop(PyObject *op, PyObject *key, PyObject **result) { - PyObject *it; /* iter(iterable) */ - PyObject *key; - PyObject *d; - int status; - PyInterpreterState *interp = _PyInterpreterState_GET(); + int err; + Py_BEGIN_CRITICAL_SECTION(op); + err = pop_lock_held(op, key, result); + Py_END_CRITICAL_SECTION(); - d = _PyObject_CallNoArgs(cls); - if (d == NULL) + return err; +} + + +int +PyDict_PopString(PyObject *op, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(op, key_obj, result); + Py_DECREF(key_obj); + return res; +} + + +PyObject * +_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) +{ + PyObject *result; + if (PyDict_Pop(dict, key, &result) == 0) { + if (default_value != NULL) { + return Py_NewRef(default_value); + } + _PyErr_SetKeyError(key); + return NULL; + } + return result; +} + +static PyDictObject * +dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + PyObject *oldvalue; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, unicode)) { + Py_DECREF(mp); + return NULL; + } + + while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} + +static PyDictObject * +dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, + PyObject *iterable, PyObject *value) +{ + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + Py_DECREF(mp); + return NULL; + } + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(iterable); + while (_PySet_NextEntryRef(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, key, hash, Py_NewRef(value))) { + Py_DECREF(mp); + return NULL; + } + } + return mp; +} + +/* Internal version of dict.from_keys(). It is subclass-friendly. */ +PyObject * +_PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) +{ + PyObject *it; /* iter(iterable) */ + PyObject *key; + PyObject *d; + int status; + PyInterpreterState *interp = _PyInterpreterState_GET(); + + d = _PyObject_CallNoArgs(cls); + if (d == NULL) return NULL; - if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { + + if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - PyObject *oldvalue; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(PyDict_GET_SIZE(iterable)), - unicode)) { - Py_DECREF(d); - return NULL; - } - while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_dict_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } - if (PyAnySet_CheckExact(iterable)) { + else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { - Py_DECREF(d); - return NULL; - } - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { - Py_DECREF(d); - return NULL; - } - } + Py_BEGIN_CRITICAL_SECTION2(d, iterable); + d = (PyObject *)dict_set_fromkeys(interp, mp, iterable, value); + Py_END_CRITICAL_SECTION2(); return d; } } @@ -2340,12 +3087,17 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) } if (PyDict_CheckExact(d)) { + Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { - status = PyDict_SetItem(d, key, value); + status = setitem_lock_held((PyDictObject *)d, key, value); Py_DECREF(key); - if (status < 0) - goto Fail; + if (status < 0) { + assert(PyErr_Occurred()); + goto dict_iter_exit; + } } +dict_iter_exit:; + Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { status = PyObject_SetItem(d, key, value); @@ -2369,8 +3121,9 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) /* Methods */ static void -dict_dealloc(PyDictObject *mp) +dict_dealloc(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); @@ -2388,24 +3141,23 @@ dict_dealloc(PyDictObject *mp) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } -#if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(interp); -#ifdef Py_DEBUG - // new_dict() must not be called after _PyDict_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) { - state->free_list[state->numfree++] = mp; +#ifdef WITH_FREELISTS + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && + Py_IS_TYPE(mp, &PyDict_Type)) { + freelist->items[freelist->numfree++] = mp; OBJECT_STAT_INC(to_freelist); } else @@ -2418,13 +3170,16 @@ dict_dealloc(PyDictObject *mp) static PyObject * -dict_repr(PyDictObject *mp) +dict_repr_lock_held(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t i; PyObject *key = NULL, *value = NULL; _PyUnicodeWriter writer; int first; + ASSERT_DICT_LOCKED(mp); + i = Py_ReprEnter((PyObject *)mp); if (i != 0) { return i > 0 ? PyUnicode_FromString("{...}") : NULL; @@ -2447,7 +3202,7 @@ dict_repr(PyDictObject *mp) Note that repr may mutate the dict. */ i = 0; first = 1; - while (PyDict_Next((PyObject *)mp, &i, &key, &value)) { + while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { PyObject *s; int res; @@ -2500,25 +3255,35 @@ dict_repr(PyDictObject *mp) return NULL; } +static PyObject * +dict_repr(PyObject *self) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); + res = dict_repr_lock_held(self); + Py_END_CRITICAL_SECTION(); + return res; +} + static Py_ssize_t -dict_length(PyDictObject *mp) +dict_length(PyObject *self) { - return mp->ma_used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)self)->ma_used); } static PyObject * -dict_subscript(PyDictObject *mp, PyObject *key) +dict_subscript(PyObject *self, PyObject *key) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } - ix = _Py_dict_lookup(mp, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2538,27 +3303,34 @@ dict_subscript(PyDictObject *mp, PyObject *key) _PyErr_SetKeyError(key); return NULL; } - return Py_NewRef(value); + return value; } static int -dict_ass_sub(PyDictObject *mp, PyObject *v, PyObject *w) +dict_ass_sub(PyObject *mp, PyObject *v, PyObject *w) { if (w == NULL) - return PyDict_DelItem((PyObject *)mp, v); + return PyDict_DelItem(mp, v); else - return PyDict_SetItem((PyObject *)mp, v, w); + return PyDict_SetItem(mp, v, w); } static PyMappingMethods dict_as_mapping = { - (lenfunc)dict_length, /*mp_length*/ - (binaryfunc)dict_subscript, /*mp_subscript*/ - (objobjargproc)dict_ass_sub, /*mp_ass_subscript*/ + dict_length, /*mp_length*/ + dict_subscript, /*mp_subscript*/ + dict_ass_sub, /*mp_ass_subscript*/ }; static PyObject * -dict_keys(PyDictObject *mp) +keys_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2587,9 +3359,27 @@ dict_keys(PyDictObject *mp) return v; } +PyObject * +PyDict_Keys(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = keys_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + static PyObject * -dict_values(PyDictObject *mp) +values_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t n; @@ -2618,9 +3408,26 @@ dict_values(PyDictObject *mp) return v; } +PyObject * +PyDict_Values(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = values_lock_held(dict); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * -dict_items(PyDictObject *mp) +items_lock_held(PyObject *dict) { + ASSERT_DICT_LOCKED(dict); + + if (dict == NULL || !PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return NULL; + } + PyDictObject *mp = (PyDictObject *)dict; PyObject *v; Py_ssize_t i, n; PyObject *item; @@ -2664,6 +3471,17 @@ dict_items(PyDictObject *mp) return v; } +PyObject * +PyDict_Items(PyObject *dict) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(dict); + res = items_lock_held(dict); + Py_END_CRITICAL_SECTION(); + + return res; +} + /*[clinic input] @classmethod dict.fromkeys @@ -2742,8 +3560,8 @@ dict_update(PyObject *self, PyObject *args, PyObject *kwds) producing iterable objects of length 2. */ -int -PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +static int +merge_from_seq2_lock_held(PyObject *d, PyObject *seq2, int override) { PyObject *it; /* iter(seq2) */ Py_ssize_t i; /* index into seq2 of current element */ @@ -2795,14 +3613,14 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) Py_INCREF(key); Py_INCREF(value); if (override) { - if (PyDict_SetItem(d, key, value) < 0) { + if (setitem_lock_held((PyDictObject *)d, key, value) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; } } else { - if (PyDict_SetDefault(d, key, value) == NULL) { + if (dict_setdefault_ref_lock_held(d, key, value, NULL, 0) < 0) { Py_DECREF(key); Py_DECREF(value); goto Fail; @@ -2827,6 +3645,118 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) return Py_SAFE_DOWNCAST(i, Py_ssize_t, int); } +int +PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = merge_from_seq2_lock_held(d, seq2, override); + Py_END_CRITICAL_SECTION(); + + return res; +} + +static int +dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) +{ + ASSERT_DICT_LOCKED(mp); + ASSERT_DICT_LOCKED(other); + + if (other == mp || other->ma_used == 0) + /* a.update(a) or a.update({}); nothing to do */ + return 0; + if (mp->ma_used == 0) { + /* Since the target dict is empty, PyDict_GetItem() + * always returns NULL. Setting override to 1 + * skips the unnecessary test. + */ + override = 1; + PyDictKeysObject *okeys = other->ma_keys; + + // If other is clean, combined, and just allocated, just clone it. + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + PyDictKeysObject *keys = clone_combined_dict_keys(other); + if (keys == NULL) + return -1; + + ensure_shared_on_resize(mp); + dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + mp->ma_keys = keys; + STORE_USED(mp, other->ma_used); + mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + + if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { + /* Maintain tracking. */ + _PyObject_GC_TRACK(mp); + } + + return 0; + } + } + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new items. Expect + * that there will be no (or few) overlapping keys. + */ + if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { + int unicode = DK_IS_UNICODE(other->ma_keys); + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { + return -1; + } + } + + Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t pos = 0; + Py_hash_t hash; + PyObject *key, *value; + + while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { + int err = 0; + Py_INCREF(key); + Py_INCREF(value); + if (override == 1) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else { + err = _PyDict_Contains_KnownHash((PyObject *)mp, key, hash); + if (err == 0) { + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); + } + else if (err > 0) { + if (override != 0) { + _PyErr_SetKeyError(key); + Py_DECREF(value); + Py_DECREF(key); + return -1; + } + err = 0; + } + } + Py_DECREF(value); + Py_DECREF(key); + if (err != 0) + return -1; + + if (orig_size != other->ma_keys->dk_nentries) { + PyErr_SetString(PyExc_RuntimeError, + "dict mutated during update"); + return -1; + } + } + return 0; +} + static int dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { @@ -2844,123 +3774,40 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) return -1; } mp = (PyDictObject*)a; - if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == (getiterfunc)dict_iter)) { + int res = 0; + if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == dict_iter)) { other = (PyDictObject*)b; - if (other == mp || other->ma_used == 0) - /* a.update(a) or a.update({}); nothing to do */ - return 0; - if (mp->ma_used == 0) { - /* Since the target dict is empty, PyDict_GetItem() - * always returns NULL. Setting override to 1 - * skips the unnecessary test. - */ - override = 1; - PyDictKeysObject *okeys = other->ma_keys; - - // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, b, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); - if (keys == NULL) { - return -1; - } - - dictkeys_decref(interp, mp->ma_keys); - mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - - return 0; - } - } - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new items. Expect - * that there will be no (or few) overlapping keys. - */ - if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { - int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), - unicode)) { - return -1; - } - } - - Py_ssize_t orig_size = other->ma_keys->dk_nentries; - Py_ssize_t pos = 0; - Py_hash_t hash; - PyObject *key, *value; - - while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { - int err = 0; - Py_INCREF(key); - Py_INCREF(value); - if (override == 1) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - err = _PyDict_Contains_KnownHash(a, key, hash); - if (err == 0) { - err = insertdict(interp, mp, - Py_NewRef(key), hash, Py_NewRef(value)); - } - else if (err > 0) { - if (override != 0) { - _PyErr_SetKeyError(key); - Py_DECREF(value); - Py_DECREF(key); - return -1; - } - err = 0; - } - } - Py_DECREF(value); - Py_DECREF(key); - if (err != 0) - return -1; - - if (orig_size != other->ma_keys->dk_nentries) { - PyErr_SetString(PyExc_RuntimeError, - "dict mutated during update"); - return -1; - } - } + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_dict_merge(interp, (PyDictObject *)a, other, override); + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION2(); + return res; } else { /* Do it the generic, slower way */ + Py_BEGIN_CRITICAL_SECTION(a); PyObject *keys = PyMapping_Keys(b); PyObject *iter; PyObject *key, *value; int status; - if (keys == NULL) + if (keys == NULL) { /* Docstring says this is equivalent to E.keys() so * if E doesn't have a .keys() method we want * AttributeError to percolate up. Might as well * do the same for any other error. */ - return -1; + res = -1; + goto slow_exit; + } iter = PyObject_GetIter(keys); Py_DECREF(keys); - if (iter == NULL) - return -1; + if (iter == NULL) { + res = -1; + goto slow_exit; + } for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { @@ -2975,30 +3822,39 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) } Py_DECREF(key); Py_DECREF(iter); - return -1; + res = -1; + goto slow_exit; } } value = PyObject_GetItem(b, key); if (value == NULL) { Py_DECREF(iter); Py_DECREF(key); - return -1; + res = -1; + goto slow_exit; } - status = PyDict_SetItem(a, key, value); + status = setitem_lock_held(mp, key, value); Py_DECREF(key); Py_DECREF(value); if (status < 0) { Py_DECREF(iter); + res = -1; + goto slow_exit; return -1; } } Py_DECREF(iter); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { /* Iterator completed, via error */ - return -1; + res = -1; + goto slow_exit; + } + +slow_exit: + ASSERT_CONSISTENT(a); + Py_END_CRITICAL_SECTION(); + return res; } - ASSERT_CONSISTENT(a); - return 0; } int @@ -3023,23 +3879,48 @@ _PyDict_MergeEx(PyObject *a, PyObject *b, int override) return dict_merge(interp, a, b, override); } +/*[clinic input] +dict.copy + +Return a shallow copy of the dict. +[clinic start generated code]*/ + static PyObject * -dict_copy(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_copy_impl(PyDictObject *self) +/*[clinic end generated code: output=ffb782cf970a5c39 input=73935f042b639de4]*/ { - return PyDict_Copy((PyObject*)mp); + return PyDict_Copy((PyObject *)self); } -PyObject * -PyDict_Copy(PyObject *o) +/* Copies the values, but does not change the reference + * counts of the objects in the array. */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + PyErr_NoMemory(); + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + +static PyObject * +copy_lock_held(PyObject *o) { PyObject *copy; PyDictObject *mp; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (o == NULL || !PyDict_Check(o)) { - PyErr_BadInternalCall(); - return NULL; - } + ASSERT_DICT_LOCKED(o); mp = (PyDictObject *)o; if (mp->ma_used == 0) { @@ -3049,32 +3930,29 @@ PyDict_Copy(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { - free_values(newvalues); + free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; } - if (Py_TYPE(mp)->tp_iter == (getiterfunc)dict_iter && + if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL && (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3)) { @@ -3122,44 +4000,31 @@ PyDict_Copy(PyObject *o) return NULL; } -Py_ssize_t -PyDict_Size(PyObject *mp) +PyObject * +PyDict_Copy(PyObject *o) { - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return -1; - } - return ((PyDictObject *)mp)->ma_used; -} - -PyObject * -PyDict_Keys(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { + if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); return NULL; } - return dict_keys((PyDictObject *)mp); -} -PyObject * -PyDict_Values(PyObject *mp) -{ - if (mp == NULL || !PyDict_Check(mp)) { - PyErr_BadInternalCall(); - return NULL; - } - return dict_values((PyDictObject *)mp); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o); + + res = copy_lock_held(o); + + Py_END_CRITICAL_SECTION(); + return res; } -PyObject * -PyDict_Items(PyObject *mp) +Py_ssize_t +PyDict_Size(PyObject *mp) { if (mp == NULL || !PyDict_Check(mp)) { PyErr_BadInternalCall(); - return NULL; + return -1; } - return dict_items((PyDictObject *)mp); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)mp)->ma_used); } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -3167,15 +4032,18 @@ PyDict_Items(PyObject *mp) * Uses only Py_EQ comparison. */ static int -dict_equal(PyDictObject *a, PyDictObject *b) +dict_equal_lock_held(PyDictObject *a, PyDictObject *b) { Py_ssize_t i; + ASSERT_DICT_LOCKED(a); + ASSERT_DICT_LOCKED(b); + if (a->ma_used != b->ma_used) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < a->ma_keys->dk_nentries; i++) { + for (i = 0; i < LOAD_KEYS_NENTRIES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -3185,7 +4053,7 @@ dict_equal(PyDictObject *a, PyDictObject *b) continue; } hash = unicode_get_hash(key); - if (a->ma_values) + if (_PyDict_HasSplitTable(a)) aval = a->ma_values->values[i]; else aval = ep->me_value; @@ -3225,6 +4093,17 @@ dict_equal(PyDictObject *a, PyDictObject *b) return 1; } +static int +dict_equal(PyDictObject *a, PyDictObject *b) +{ + int res; + Py_BEGIN_CRITICAL_SECTION2(a, b); + res = dict_equal_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject * dict_richcompare(PyObject *v, PyObject *w, int op) { @@ -3260,25 +4139,18 @@ static PyObject * dict___contains__(PyDictObject *self, PyObject *key) /*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ { - register PyDictObject *mp = self; - Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + int contains = PyDict_Contains((PyObject *)self, key); + if (contains < 0) { return NULL; - if (ix == DKIX_EMPTY || value == NULL) - Py_RETURN_FALSE; - Py_RETURN_TRUE; + } + if (contains) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; } /*[clinic input] +@critical_section dict.get key: object @@ -3290,121 +4162,169 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=bba707729dee05bf input=279ddb5790b6b107]*/ +/*[clinic end generated code: output=bba707729dee05bf input=a631d3f18f584c60]*/ { PyObject *val = NULL; Py_hash_t hash; Py_ssize_t ix; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } - ix = _Py_dict_lookup(self, key, hash, &val); + ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || val == NULL) { - val = default_value; + val = Py_NewRef(default_value); } - return Py_NewRef(val); + return val; } -PyObject * -PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +static int +dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result, int incref_result) { PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(d); + if (!PyDict_Check(d)) { PyErr_BadInternalCall(); - return NULL; + if (result) { + *result = NULL; + } + return -1; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; + } + return -1; } if (mp->ma_keys == Py_EMPTY_KEYS) { if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, - Py_NewRef(defaultobj)) < 0) { - return NULL; + Py_NewRef(default_value)) < 0) { + if (result) { + *result = NULL; + } + return -1; } - return defaultobj; + if (result) { + *result = incref_result ? Py_NewRef(default_value) : default_value; + } + return 0; } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { if (insertion_resize(interp, mp, 0) < 0) { - return NULL; + if (result) { + *result = NULL; + } + return -1; + } + } + + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + PyObject *value = mp->ma_values->values[ix]; + int already_present = value != NULL; + if (!already_present) { + insert_split_value(interp, mp, key, default_value, ix); + value = default_value; + } + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return already_present; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto error; } } + assert(!_PyDict_HasSplitTable(mp)); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; + } if (ix == DKIX_EMPTY) { + assert(!_PyDict_HasSplitTable(mp)); uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); + interp, PyDict_EVENT_ADDED, mp, key, default_value); mp->ma_keys->dk_version = 0; - value = defaultobj; - if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(interp, mp, 1) < 0) { - return NULL; - } - } - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - assert(PyUnicode_CheckExact(key)); - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t index = (int)mp->ma_keys->dk_nentries; - assert(index < SHARED_KEYS_MAX_SIZE); - assert(mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - } - else { - ep->me_value = Py_NewRef(value); + value = default_value; + + if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; } } - else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - ep->me_hash = hash; - ep->me_value = Py_NewRef(value); - } + MAINTAIN_TRACKING(mp, key, value); - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); - } - else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, defaultobj); - value = defaultobj; - assert(_PyDict_HasSplitTable(mp)); - assert(mp->ma_values->values[ix] == NULL); - MAINTAIN_TRACKING(mp, key, value); - mp->ma_values->values[ix] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - mp->ma_version_tag = new_version; + ASSERT_CONSISTENT(mp); + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 0; } + assert(value != NULL); ASSERT_CONSISTENT(mp); - return value; + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return 1; + +error: + if (result) { + *result = NULL; + } + return -1; +} + +int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(d); + res = dict_setdefault_ref_lock_held(d, key, default_value, result, 1); + Py_END_CRITICAL_SECTION(); + return res; +} + +PyObject * +PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(d); + dict_setdefault_ref_lock_held(d, key, defaultobj, &result, 0); + Py_END_CRITICAL_SECTION(); + return result; } /*[clinic input] +@critical_section dict.setdefault key: object @@ -3419,18 +4339,25 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_setdefault_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f8c1101ebf69e220 input=0f063756e815fd9d]*/ +/*[clinic end generated code: output=f8c1101ebf69e220 input=9237af9a0a224302]*/ { PyObject *val; - - val = PyDict_SetDefault((PyObject *)self, key, default_value); - return Py_XNewRef(val); + dict_setdefault_ref_lock_held((PyObject *)self, key, default_value, &val, 1); + return val; } + +/*[clinic input] +dict.clear + +Remove all items from the dict. +[clinic start generated code]*/ + static PyObject * -dict_clear(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_clear_impl(PyDictObject *self) +/*[clinic end generated code: output=5139a830df00830a input=0bf729baba97a4c2]*/ { - PyDict_Clear((PyObject *)mp); + PyDict_Clear((PyObject *)self); Py_RETURN_NONE; } @@ -3455,6 +4382,7 @@ dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) } /*[clinic input] +@critical_section dict.popitem Remove and return a (key, value) pair as a 2-tuple. @@ -3465,13 +4393,15 @@ Raises KeyError if the dict is empty. static PyObject * dict_popitem_impl(PyDictObject *self) -/*[clinic end generated code: output=e65fcb04420d230d input=1c38a49f21f64941]*/ +/*[clinic end generated code: output=e65fcb04420d230d input=ef28b4da5f0f762e]*/ { Py_ssize_t i, j; PyObject *res; uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(self); + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -3490,8 +4420,8 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (_PyDict_HasSplitTable(self)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -3543,8 +4473,8 @@ dict_popitem_impl(PyDictObject *self) PyTuple_SET_ITEM(res, 0, key); PyTuple_SET_ITEM(res, 1, value); /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ - self->ma_keys->dk_nentries = i; - self->ma_used--; + STORE_KEYS_NENTRIES(self->ma_keys, i); + STORE_USED(self, self->ma_used - 1); self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; @@ -3558,9 +4488,11 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) Py_ssize_t i, n = keys->dk_nentries; if (DK_IS_UNICODE(keys)) { - if (mp->ma_values != NULL) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (_PyDict_HasSplitTable(mp)) { + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -3591,11 +4523,11 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -Py_ssize_t -_PyDict_SizeOf(PyDictObject *mp) +static Py_ssize_t +sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); } /* If the dictionary is split, the keys portion is accounted-for @@ -3607,6 +4539,17 @@ _PyDict_SizeOf(PyDictObject *mp) return (Py_ssize_t)res; } +Py_ssize_t +_PyDict_SizeOf(PyDictObject *mp) +{ + Py_ssize_t res; + Py_BEGIN_CRITICAL_SECTION(mp); + res = sizeof_lock_held(mp); + Py_END_CRITICAL_SECTION(); + + return res; +} + size_t _PyDict_KeysSize(PyDictKeysObject *keys) { @@ -3618,10 +4561,17 @@ _PyDict_KeysSize(PyDictKeysObject *keys) return size; } +/*[clinic input] +dict.__sizeof__ + +Return the size of the dict in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -dict_sizeof(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict___sizeof___impl(PyDictObject *self) +/*[clinic end generated code: output=44279379b3824bda input=4fec4ddfc44a4d1a]*/ { - return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); + return PyLong_FromSsize_t(_PyDict_SizeOf(self)); } static PyObject * @@ -3653,56 +4603,31 @@ dict_ior(PyObject *self, PyObject *other) PyDoc_STRVAR(getitem__doc__, "__getitem__($self, key, /)\n--\n\nReturn self[key]."); -PyDoc_STRVAR(sizeof__doc__, -"D.__sizeof__() -> size of D in memory, in bytes"); - PyDoc_STRVAR(update__doc__, "D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\ If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\ If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\ In either case, this is followed by: for k in F: D[k] = F[k]"); -PyDoc_STRVAR(clear__doc__, -"D.clear() -> None. Remove all items from D."); - -PyDoc_STRVAR(copy__doc__, -"D.copy() -> a shallow copy of D"); - /* Forward */ -static PyObject *dictkeys_new(PyObject *, PyObject *); -static PyObject *dictitems_new(PyObject *, PyObject *); -static PyObject *dictvalues_new(PyObject *, PyObject *); - -PyDoc_STRVAR(keys__doc__, - "D.keys() -> a set-like object providing a view on D's keys"); -PyDoc_STRVAR(items__doc__, - "D.items() -> a set-like object providing a view on D's items"); -PyDoc_STRVAR(values__doc__, - "D.values() -> an object providing a view on D's values"); static PyMethodDef mapp_methods[] = { DICT___CONTAINS___METHODDEF - {"__getitem__", _PyCFunction_CAST(dict_subscript), METH_O | METH_COEXIST, + {"__getitem__", dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", _PyCFunction_CAST(dict_sizeof), METH_NOARGS, - sizeof__doc__}, + DICT___SIZEOF___METHODDEF DICT_GET_METHODDEF DICT_SETDEFAULT_METHODDEF DICT_POP_METHODDEF DICT_POPITEM_METHODDEF - {"keys", dictkeys_new, METH_NOARGS, - keys__doc__}, - {"items", dictitems_new, METH_NOARGS, - items__doc__}, - {"values", dictvalues_new, METH_NOARGS, - values__doc__}, + DICT_KEYS_METHODDEF + DICT_ITEMS_METHODDEF + DICT_VALUES_METHODDEF {"update", _PyCFunction_CAST(dict_update), METH_VARARGS | METH_KEYWORDS, update__doc__}, DICT_FROMKEYS_METHODDEF - {"clear", (PyCFunction)dict_clear, METH_NOARGS, - clear__doc__}, - {"copy", (PyCFunction)dict_copy, METH_NOARGS, - copy__doc__}, + DICT_CLEAR_METHODDEF + DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -3712,20 +4637,13 @@ static PyMethodDef mapp_methods[] = { int PyDict_Contains(PyObject *op, PyObject *key) { - Py_hash_t hash; - Py_ssize_t ix; - PyDictObject *mp = (PyDictObject *)op; - PyObject *value; + Py_hash_t hash = _PyObject_HashFast(key); - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; - } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + if (hash == -1) { return -1; - return (ix != DKIX_EMPTY && value != NULL); + } + + return _PyDict_Contains_KnownHash(op, key, hash); } int @@ -3748,10 +4666,20 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -3856,8 +4784,9 @@ dict_vectorcall(PyObject *type, PyObject * const*args, } static PyObject * -dict_iter(PyDictObject *dict) +dict_iter(PyObject *self) { + PyDictObject *dict = (PyDictObject *)self; return dictiter_new(dict, &PyDictIterKey_Type); } @@ -3877,12 +4806,12 @@ PyTypeObject PyDict_Type = { "dict", sizeof(PyDictObject), 0, - (destructor)dict_dealloc, /* tp_dealloc */ + dict_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dict_repr, /* tp_repr */ + dict_repr, /* tp_repr */ &dict_as_number, /* tp_as_number */ &dict_as_sequence, /* tp_as_sequence */ &dict_as_mapping, /* tp_as_mapping */ @@ -3900,7 +4829,7 @@ PyTypeObject PyDict_Type = { dict_tp_clear, /* tp_clear */ dict_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dict_iter, /* tp_iter */ + dict_iter, /* tp_iter */ 0, /* tp_iternext */ mapp_methods, /* tp_methods */ 0, /* tp_members */ @@ -3925,10 +4854,14 @@ PyDict_GetItemString(PyObject *v, const char *key) PyObject *kv, *rv; kv = PyUnicode_FromString(key); if (kv == NULL) { - PyErr_Clear(); + PyErr_FormatUnraisable( + "Exception ignored in PyDict_GetItemString(); consider using " + "PyDict_GetItemRefString()"); return NULL; } - rv = PyDict_GetItem(v, kv); + rv = dict_getitem(v, kv, + "Exception ignored in PyDict_GetItemString(); consider using " + "PyDict_GetItemRefString()"); Py_DECREF(kv); return rv; // borrowed reference } @@ -3964,7 +4897,8 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) kv = PyUnicode_FromString(key); if (kv == NULL) return -1; - PyUnicode_InternInPlace(&kv); /* XXX Should we really? */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &kv); /* XXX Should we really? */ err = PyDict_SetItem(v, kv, item); Py_DECREF(kv); return err; @@ -4006,22 +4940,24 @@ typedef struct { static PyObject * dictiter_new(PyDictObject *dict, PyTypeObject *itertype) { + Py_ssize_t used; dictiterobject *di; di = PyObject_GC_New(dictiterobject, itertype); if (di == NULL) { return NULL; } di->di_dict = (PyDictObject*)Py_NewRef(dict); - di->di_used = dict->ma_used; - di->len = dict->ma_used; + used = FT_ATOMIC_LOAD_SSIZE_RELAXED(dict->ma_used); + di->di_used = used; + di->len = used; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { - if (dict->ma_values) { - di->di_pos = dict->ma_used - 1; + if (_PyDict_HasSplitTable(dict)) { + di->di_pos = used - 1; } else { - di->di_pos = dict->ma_keys->dk_nentries - 1; + di->di_pos = load_keys_nentries(dict) - 1; } } else { @@ -4043,8 +4979,9 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) } static void -dictiter_dealloc(dictiterobject *di) +dictiter_dealloc(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(di); Py_XDECREF(di->di_dict); @@ -4053,19 +4990,21 @@ dictiter_dealloc(dictiterobject *di) } static int -dictiter_traverse(dictiterobject *di, visitproc visit, void *arg) +dictiter_traverse(PyObject *self, visitproc visit, void *arg) { + dictiterobject *di = (dictiterobject *)self; Py_VISIT(di->di_dict); Py_VISIT(di->di_result); return 0; } static PyObject * -dictiter_len(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; - if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) - len = di->len; + if (di->di_dict != NULL && di->di_used == FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_dict->ma_used)) + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->len); return PyLong_FromSize_t(len); } @@ -4073,29 +5012,36 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)); +dictiter_reduce(PyObject *di, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef dictiter_methods[] = { - {"__length_hint__", _PyCFunction_CAST(dictiter_len), METH_NOARGS, + {"__length_hint__", dictiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", _PyCFunction_CAST(dictiter_reduce), METH_NOARGS, + {"__reduce__", dictiter_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; +#ifdef Py_GIL_DISABLED + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value); + +#else /* Py_GIL_DISABLED */ + static PyObject* -dictiter_iternextkey(dictiterobject *di) +dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *key; Py_ssize_t i; PyDictKeysObject *k; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4107,11 +5053,11 @@ dictiter_iternextkey(dictiterobject *di) i = di->di_pos; k = d->ma_keys; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); assert(d->ma_values->values[index] != NULL); } else { @@ -4153,13 +5099,36 @@ dictiter_iternextkey(dictiterobject *di) return NULL; } +#endif /* Py_GIL_DISABLED */ + +static PyObject* +dictiter_iternextkey(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &value, NULL) < 0) { + value = NULL; + } +#else + value = dictiter_iternextkey_lock_held(d, self); +#endif + + return value; +} + PyTypeObject PyDictIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_keyiterator", /* tp_name */ sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4176,26 +5145,27 @@ PyTypeObject PyDictIterKey_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextkey, /* tp_iternext */ + dictiter_iternextkey, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; +#ifndef Py_GIL_DISABLED + static PyObject * -dictiter_iternextvalue(dictiterobject *di) +dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4206,7 +5176,7 @@ dictiter_iternextvalue(dictiterobject *di) i = di->di_pos; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); @@ -4252,13 +5222,36 @@ dictiter_iternextvalue(dictiterobject *di) return NULL; } +#endif /* Py_GIL_DISABLED */ + +static PyObject * +dictiter_iternextvalue(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, NULL, &value) < 0) { + value = NULL; + } +#else + value = dictiter_iternextvalue_lock_held(d, self); +#endif + + return value; +} + PyTypeObject PyDictIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_valueiterator", /* tp_name */ sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4275,41 +5268,42 @@ PyTypeObject PyDictIterValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextvalue, /* tp_iternext */ + dictiter_iternextvalue, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; -static PyObject * -dictiter_iternextitem(dictiterobject *di) +static int +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) { - PyObject *key, *value, *result; + dictiterobject *di = (dictiterobject *)self; + PyObject *key, *value; Py_ssize_t i; - PyDictObject *d = di->di_dict; - if (d == NULL) - return NULL; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); di->di_used = -1; /* Make this state sticky */ - return NULL; + return -1; } - i = di->di_pos; + i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); + assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key); value = d->ma_values->values[index]; assert(value != NULL); } @@ -4346,65 +5340,251 @@ dictiter_iternextitem(dictiterobject *di) } di->di_pos = i+1; di->len--; - result = di->di_result; - if (Py_REFCNT(result) == 1) { - PyObject *oldkey = PyTuple_GET_ITEM(result, 0); - PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); - Py_INCREF(result); - Py_DECREF(oldkey); - Py_DECREF(oldvalue); - // bpo-42536: The GC may have untracked this result tuple. Since we're - // recycling it, make sure it's tracked again: - if (!_PyObject_GC_IS_TRACKED(result)) { - _PyObject_GC_TRACK(result); - } + if (out_key != NULL) { + *out_key = Py_NewRef(key); } - else { - result = PyTuple_New(2); - if (result == NULL) - return NULL; - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); + if (out_value != NULL) { + *out_value = Py_NewRef(value); } - return result; + return 0; fail: di->di_dict = NULL; Py_DECREF(d); - return NULL; + return -1; } -PyTypeObject PyDictIterItem_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "dict_itemiterator", /* tp_name */ - sizeof(dictiterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ +#ifdef Py_GIL_DISABLED + +// Grabs the key and/or value from the provided locations and if successful +// returns them with an increased reference count. If either one is unsuccessful +// nothing is incref'd and returns -1. +static int +acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, + PyObject **out_key, PyObject **out_value) +{ + if (out_key) { + *out_key = _Py_TryXGetRef(key_loc); + if (*out_key == NULL) { + return -1; + } + } + + if (out_value) { + if (!_Py_TryIncrefCompare(value_loc, value)) { + if (out_key) { + Py_DECREF(*out_key); + } + return -1; + } + *out_value = value; + } + + return 0; +} + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) +{ + int res; + dictiterobject *di = (dictiterobject *)self; + Py_ssize_t i; + PyDictKeysObject *k; + + assert (PyDict_Check(d)); + + if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + di->di_used = -1; /* Make this state sticky */ + return -1; + } + + ensure_shared_on_read(d); + + i = _Py_atomic_load_ssize_relaxed(&di->di_pos); + k = _Py_atomic_load_ptr_relaxed(&d->ma_keys); + assert(i >= 0); + if (_PyDict_HasSplitTable(d)) { + PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + if (values == NULL) { + goto concurrent_modification; + } + + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); + if (i >= used) { + goto fail; + } + + // We're racing against writes to the order from delete_index_from_values, but + // single threaded can suffer from concurrent modification to those as well and + // can have either duplicated or skipped attributes, so we strive to do no better + // here. + int index = get_index_from_order(d, i); + PyObject *value = _Py_atomic_load_ptr(&values->values[index]); + if (acquire_key_value(&DK_UNICODE_ENTRIES(k)[index].me_key, value, + &values->values[index], out_key, out_value) < 0) { + goto try_locked; + } + } + else { + Py_ssize_t n = _Py_atomic_load_ssize_relaxed(&k->dk_nentries); + if (DK_IS_UNICODE(k)) { + PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + else { + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + } + // We found an element (key), but did not expect it + Py_ssize_t len; + if ((len = _Py_atomic_load_ssize_relaxed(&di->len)) == 0) { + goto concurrent_modification; + } + + _Py_atomic_store_ssize_relaxed(&di->di_pos, i + 1); + _Py_atomic_store_ssize_relaxed(&di->len, len - 1); + return 0; + +concurrent_modification: + PyErr_SetString(PyExc_RuntimeError, + "dictionary keys changed during iteration"); + +fail: + di->di_dict = NULL; + Py_DECREF(d); + return -1; + +try_locked: + Py_BEGIN_CRITICAL_SECTION(d); + res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); + Py_END_CRITICAL_SECTION(); + return res; +} + +#endif + +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + +static bool +acquire_iter_result(PyObject *result) +{ + if (has_unique_reference(result)) { + Py_INCREF(result); + return true; + } + return false; +} + +static PyObject * +dictiter_iternextitem(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *key, *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &key, &value) == 0) { +#else + if (dictiter_iternextitem_lock_held(d, self, &key, &value) == 0) { + +#endif + PyObject *result = di->di_result; + if (acquire_iter_result(result)) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result == NULL) + return NULL; + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + } + return result; + } + return NULL; +} + +PyTypeObject PyDictIterItem_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "dict_itemiterator", /* tp_name */ + sizeof(dictiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + dictiter_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextitem, /* tp_iternext */ + dictiter_iternextitem, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; @@ -4413,14 +5593,12 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iternext(dictiterobject *di) +dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) { - PyDictObject *d = di->di_dict; + dictiterobject *di = (dictiterobject *)self; - if (d == NULL) { - return NULL; - } assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -4436,9 +5614,9 @@ dictreviter_iternext(dictiterobject *di) if (i < 0) { goto fail; } - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); value = d->ma_values->values[index]; assert (value != NULL); } @@ -4511,15 +5689,32 @@ dictreviter_iternext(dictiterobject *di) return NULL; } +static PyObject * +dictreviter_iternext(PyObject *self) +{ + dictiterobject *di = (dictiterobject *)self; + PyDictObject *d = di->di_dict; + + if (d == NULL) + return NULL; + + PyObject *value; + Py_BEGIN_CRITICAL_SECTION(d); + value = dictreviter_iter_lock_held(d, self); + Py_END_CRITICAL_SECTION(); + + return value; +} + PyTypeObject PyDictRevIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversekeyiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4539,8 +5734,9 @@ dict___reversed___impl(PyDictObject *self) } static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; /* copy the iterator state */ dictiterobject tmp = *di; Py_XINCREF(tmp.di_dict); @@ -4556,11 +5752,11 @@ PyTypeObject PyDictRevIterItem_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reverseitemiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4568,11 +5764,11 @@ PyTypeObject PyDictRevIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversevalueiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4583,8 +5779,9 @@ PyTypeObject PyDictRevIterValue_Type = { /* The instance lay-out is the same for all three; but the type differs. */ static void -dictview_dealloc(_PyDictViewObject *dv) +dictview_dealloc(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(dv); Py_XDECREF(dv->dv_dict); @@ -4592,18 +5789,20 @@ dictview_dealloc(_PyDictViewObject *dv) } static int -dictview_traverse(_PyDictViewObject *dv, visitproc visit, void *arg) +dictview_traverse(PyObject *self, visitproc visit, void *arg) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_VISIT(dv->dv_dict); return 0; } static Py_ssize_t -dictview_len(_PyDictViewObject *dv) +dictview_len(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) - len = dv->dv_dict->ma_used; + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(dv->dv_dict->ma_used); return len; } @@ -4740,8 +5939,9 @@ dictview_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -dictview_repr(_PyDictViewObject *dv) +dictview_repr(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; PyObject *seq; PyObject *result = NULL; Py_ssize_t rc; @@ -4765,8 +5965,9 @@ dictview_repr(_PyDictViewObject *dv) /*** dict_keys ***/ static PyObject * -dictkeys_iter(_PyDictViewObject *dv) +dictkeys_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -4774,22 +5975,23 @@ dictkeys_iter(_PyDictViewObject *dv) } static int -dictkeys_contains(_PyDictViewObject *dv, PyObject *obj) +dictkeys_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) return 0; return PyDict_Contains((PyObject *)dv->dv_dict, obj); } static PySequenceMethods dictkeys_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictkeys_contains, /* sq_contains */ + dictkeys_contains, /* sq_contains */ }; // Create a set object from dictviews object. @@ -4829,7 +6031,7 @@ dictviews_sub(PyObject *self, PyObject *other) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj); +dictitems_contains(PyObject *dv, PyObject *obj); PyObject * _PyDictView_Intersect(PyObject* self, PyObject *other) @@ -4839,7 +6041,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) PyObject *key; Py_ssize_t len_self; int rv; - int (*dict_contains)(_PyDictViewObject *, PyObject *); + objobjproc dict_contains; /* Python interpreter swaps parameters when dict view is on right side of & */ @@ -4849,7 +6051,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) self = tmp; } - len_self = dictview_len((_PyDictViewObject *)self); + len_self = dictview_len(self); /* if other is a set and self is smaller than other, reuse set intersection logic */ @@ -4861,7 +6063,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) /* if other is another dict view, and it is bigger than self, swap them */ if (PyDictViewSet_Check(other)) { - Py_ssize_t len_other = dictview_len((_PyDictViewObject *)other); + Py_ssize_t len_other = dictview_len(other); if (len_other > len_self) { PyObject *tmp = other; other = self; @@ -4891,7 +6093,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) } while ((key = PyIter_Next(it)) != NULL) { - rv = dict_contains((_PyDictViewObject *)self, key); + rv = dict_contains(self, key); if (rv < 0) { goto error; } @@ -4932,14 +6134,12 @@ dictviews_or(PyObject* self, PyObject *other) } static PyObject * -dictitems_xor(PyObject *self, PyObject *other) +dictitems_xor_lock_held(PyObject *d1, PyObject *d2) { - assert(PyDictItems_Check(self)); - assert(PyDictItems_Check(other)); - PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; - PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + ASSERT_DICT_LOCKED(d1); + ASSERT_DICT_LOCKED(d2); - PyObject *temp_dict = PyDict_Copy(d1); + PyObject *temp_dict = copy_lock_held(d1); if (temp_dict == NULL) { return NULL; } @@ -5017,6 +6217,22 @@ dictitems_xor(PyObject *self, PyObject *other) return NULL; } +static PyObject * +dictitems_xor(PyObject *self, PyObject *other) +{ + assert(PyDictItems_Check(self)); + assert(PyDictItems_Check(other)); + PyObject *d1 = (PyObject *)((_PyDictViewObject *)self)->dv_dict; + PyObject *d2 = (PyObject *)((_PyDictViewObject *)other)->dv_dict; + + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(d1, d2); + res = dictitems_xor_lock_held(d1, d2); + Py_END_CRITICAL_SECTION2(); + + return res; +} + static PyObject* dictviews_xor(PyObject* self, PyObject *other) { @@ -5065,7 +6281,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyObject *item = NULL; if (self == other) { - if (dictview_len((_PyDictViewObject *)self) == 0) + if (dictview_len(self) == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; @@ -5074,7 +6290,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) /* Iterate over the shorter object (only if other is a set, * because PySequence_Contains may be expensive otherwise): */ if (PyAnySet_Check(other) || PyDictViewSet_Check(other)) { - Py_ssize_t len_self = dictview_len((_PyDictViewObject *)self); + Py_ssize_t len_self = dictview_len(self); Py_ssize_t len_other = PyObject_Size(other); if (len_other == -1) return NULL; @@ -5112,15 +6328,15 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyDoc_STRVAR(isdisjoint_doc, "Return True if the view and the given iterable have a null intersection."); -static PyObject* dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictkeys_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_keys_doc, "Return a reverse iterator over the dict keys."); static PyMethodDef dictkeys_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", _PyCFunction_CAST(dictkeys_reversed), METH_NOARGS, + {"__reversed__", dictkeys_reversed, METH_NOARGS, reversed_keys_doc}, {NULL, NULL} /* sentinel */ }; @@ -5131,12 +6347,12 @@ PyTypeObject PyDictKeys_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictkeys_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5148,25 +6364,33 @@ PyTypeObject PyDictKeys_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictkeys_iter, /* tp_iter */ + dictkeys_iter, /* tp_iter */ 0, /* tp_iternext */ dictkeys_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.keys + +Return a set-like object providing a view on the dict's keys. +[clinic start generated code]*/ + static PyObject * -dictkeys_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_keys_impl(PyDictObject *self) +/*[clinic end generated code: output=aac2830c62990358 input=42f48a7a771212a7]*/ { - return _PyDictView_New(dict, &PyDictKeys_Type); + return _PyDictView_New((PyObject *)self, &PyDictKeys_Type); } static PyObject * -dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictkeys_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5176,8 +6400,9 @@ dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_items ***/ static PyObject * -dictitems_iter(_PyDictViewObject *dv) +dictitems_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5185,8 +6410,9 @@ dictitems_iter(_PyDictViewObject *dv) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj) +dictitems_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) @@ -5204,25 +6430,25 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) } static PySequenceMethods dictitems_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictitems_contains, /* sq_contains */ + dictitems_contains, /* sq_contains */ }; -static PyObject* dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictitems_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_items_doc, "Return a reverse iterator over the dict items."); static PyMethodDef dictitems_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", (PyCFunction)dictitems_reversed, METH_NOARGS, + {"__reversed__", dictitems_reversed, METH_NOARGS, reversed_items_doc}, {NULL, NULL} /* sentinel */ }; @@ -5233,12 +6459,12 @@ PyTypeObject PyDictItems_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictitems_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5250,25 +6476,33 @@ PyTypeObject PyDictItems_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictitems_iter, /* tp_iter */ + dictitems_iter, /* tp_iter */ 0, /* tp_iternext */ dictitems_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.items + +Return a set-like object providing a view on the dict's items. +[clinic start generated code]*/ + static PyObject * -dictitems_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_items_impl(PyDictObject *self) +/*[clinic end generated code: output=88c7db7150c7909a input=87c822872eb71f5a]*/ { - return _PyDictView_New(dict, &PyDictItems_Type); + return _PyDictView_New((PyObject *)self, &PyDictItems_Type); } static PyObject * -dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictitems_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5278,8 +6512,9 @@ dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_values ***/ static PyObject * -dictvalues_iter(_PyDictViewObject *dv) +dictvalues_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5287,7 +6522,7 @@ dictvalues_iter(_PyDictViewObject *dv) } static PySequenceMethods dictvalues_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ @@ -5297,13 +6532,13 @@ static PySequenceMethods dictvalues_as_sequence = { (objobjproc)0, /* sq_contains */ }; -static PyObject* dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictvalues_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_values_doc, "Return a reverse iterator over the dict values."); static PyMethodDef dictvalues_methods[] = { - {"__reversed__", (PyCFunction)dictvalues_reversed, METH_NOARGS, + {"__reversed__", dictvalues_reversed, METH_NOARGS, reversed_values_doc}, {NULL, NULL} /* sentinel */ }; @@ -5314,12 +6549,12 @@ PyTypeObject PyDictValues_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ 0, /* tp_as_number */ &dictvalues_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5331,25 +6566,33 @@ PyTypeObject PyDictValues_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictvalues_iter, /* tp_iter */ + dictvalues_iter, /* tp_iter */ 0, /* tp_iternext */ dictvalues_methods, /* tp_methods */ .tp_getset = dictview_getset, }; +/*[clinic input] +dict.values + +Return an object providing a view on the dict's values. +[clinic start generated code]*/ + static PyObject * -dictvalues_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) +dict_values_impl(PyDictObject *self) +/*[clinic end generated code: output=ce9f2e9e8a959dd4 input=b46944f85493b230]*/ { - return _PyDictView_New(dict, &PyDictValues_Type); + return _PyDictView_New((PyObject *)self, &PyDictValues_Type); } static PyObject * -dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5377,62 +6620,43 @@ _PyDict_NewKeysForClass(void) return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); +#ifdef Py_GIL_DISABLED + Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); + if (usable > 1) { + LOCK_KEYS(keys); + if (keys->dk_usable > 1) { + _Py_atomic_store_ssize(&keys->dk_usable, keys->dk_usable - 1); + } + UNLOCK_KEYS(keys); + } +#else if (keys->dk_usable > 1) { keys->dk_usable--; } +#endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; -} - -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { @@ -5447,95 +6671,134 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) { + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + + PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(interp, keys, values); + PyDictObject *dict = make_dict_from_instance_attributes(interp, keys, values); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return dict; } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) +PyDictObject * +_PyObject_MaterializeManagedDict(PyObject *obj) { - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + return dict; } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { - return false; + + Py_BEGIN_CRITICAL_SECTION(obj); + +#ifdef Py_GIL_DISABLED + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + // We raced with another thread creating the dict + goto exit; } - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; +#endif + dict = _PyObject_MaterializeManagedDict_LockHeld(obj); + +#ifdef Py_GIL_DISABLED +exit: +#endif + Py_END_CRITICAL_SECTION(); + return dict; } int -_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, +_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) +{ + if (value == NULL) { + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + return -1; + } + return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + } else { + return setitem_lock_held(dict, name, value); + } +} + +// Called with either the object's lock or the dict's lock held +// depending on whether or not a dict has been materialized for +// the object. +static int +store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + assert(dict == NULL || ((PyDictObject *)dict)->ma_values == values); if (PyUnicode_CheckExact(name)) { - ix = insert_into_dictkeys(keys, name); - } - if (ix == DKIX_EMPTY) { + Py_hash_t hash = unicode_get_hash(name); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(name); + assert(hash != -1); + } + + ix = insert_split_key(keys, name, hash); + #ifdef Py_STATS - if (PyUnicode_CheckExact(name)) { - if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { - OBJECT_STAT_INC(dict_materialized_too_big); + if (ix == DKIX_EMPTY) { + if (PyUnicode_CheckExact(name)) { + if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { + OBJECT_STAT_INC(dict_materialized_too_big); + } + else { + OBJECT_STAT_INC(dict_materialized_new_key); + } } else { - OBJECT_STAT_INC(dict_materialized_new_key); + OBJECT_STAT_INC(dict_materialized_str_subclass); } } - else { - OBJECT_STAT_INC(dict_materialized_str_subclass); - } #endif - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); + } + + if (ix == DKIX_EMPTY) { + int res; if (dict == NULL) { - return -1; - } - _PyObject_DictOrValuesPointer(obj)->dict = dict; - if (value == NULL) { - return PyDict_DelItem(dict, name); - } - else { - return PyDict_SetItem(dict, name, value); + // Make the dict but don't publish it in the object + // so that no one else will see it. + dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + if (dict == NULL || + _PyDict_SetItem_LockHeld(dict, name, value) < 0) { + Py_XDECREF(dict); + return -1; + } + + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return 0; } + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + + res = _PyDict_SetItem_LockHeld(dict, name, value); + return res; } + PyObject *old_value = values->values[ix]; - values->values[ix] = Py_XNewRef(value); + FT_ATOMIC_STORE_PTR_RELEASE(values->values[ix], Py_XNewRef(value)); + if (old_value == NULL) { if (value == NULL) { PyErr_Format(PyExc_AttributeError, @@ -5544,16 +6807,90 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return -1; } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used + 1); + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used - 1); + } } Py_DECREF(old_value); } return 0; } +static inline int +store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + if (dict->ma_values == values) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + else { + res = _PyDict_SetItem_LockHeld(dict, name, value); + } + Py_END_CRITICAL_SECTION(); + return res; +} + +int +_PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + dict = (PyDictObject *)PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return -1; + } + int res = store_instance_attr_dict(obj, dict, name, value); + Py_DECREF(dict); + return res; + } + return store_instance_attr_dict(obj, dict, name, value); + } + +#ifdef Py_GIL_DISABLED + // We have a valid inline values, at least for now... There are two potential + // races with having the values become invalid. One is the dictionary + // being detached from the object. The other is if someone is inserting + // into the dictionary directly and therefore causing it to resize. + // + // If we haven't materialized the dictionary yet we lock on the object, which + // will also be used to prevent the dictionary from being materialized while + // we're doing the insertion. If we race and the dictionary gets created + // then we'll need to release the object lock and lock the dictionary to + // prevent resizing. + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + int res; + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + + if (dict == NULL) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return res; + } + } + return store_instance_attr_dict(obj, dict, name, value); +#else + return store_instance_attr_lock_held(obj, values, name, value); +#endif +} + /* Sanity check for managed dicts */ #if 0 #define CHECK(val) assert(val); if (!(val)) { return 0; } @@ -5563,9 +6900,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); int size = ((uint8_t *)values)[-2]; int count = 0; PyDictKeysObject *keys = CACHED_KEYS(tp); @@ -5577,27 +6914,87 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) CHECK(size == count); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; } #endif -PyObject * -_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name) +// Attempts to get an instance attribute from the inline values. Returns true +// if successful, or false if the caller needs to lookup in the dictionary. +bool +_PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr) { assert(PyUnicode_CheckExact(name)); + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + return false; + } + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); if (ix == DKIX_EMPTY) { - return NULL; + *attr = NULL; + return true; } + +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); + if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { + *attr = value; + return true; + } + + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // No dict, lock the object to prevent one from being + // materialized... + bool success = false; + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // Still no dict, we can read from the values + assert(values->valid); + value = values->values[ix]; + *attr = Py_XNewRef(value); + success = true; + } + + Py_END_CRITICAL_SECTION(); + + if (success) { + return true; + } + } + + // We have a dictionary, we'll need to lock it to prevent + // the values from being resized. + assert(dict != NULL); + + bool success; + Py_BEGIN_CRITICAL_SECTION(dict); + + if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { + value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + *attr = Py_XNewRef(value); + success = true; + } else { + // Caller needs to lookup from the dictionary + success = false; + } + + Py_END_CRITICAL_SECTION(); + + return success; +#else PyObject *value = values->values[ix]; - return Py_XNewRef(value); + *attr = Py_XNewRef(value); + return true; +#endif } int @@ -5607,122 +7004,245 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_GetManagedDict(obj); + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dict = _PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; } - return ((PyDictObject *)dict)->ma_used == 0; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)dict)->ma_used) == 0; } -void -_PyObject_FreeInstanceAttributes(PyObject *self) +int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; + PyTypeObject *tp = Py_TYPE(obj); + if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return 0; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; + } + } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); + return 0; +} + +static void +set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + PyDictValues *values = _PyObject_InlineValues(obj); + + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + + if (values->valid) { + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } } - free_values(values); } int -PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict) { + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + int err = 0; PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return 0; - } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + set_dict_inline_values(obj, (PyDictObject *)new_dict); + } + + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return 0; + } +#else + set_dict_inline_values(obj, (PyDictObject *)new_dict); + return 0; +#endif + } + + Py_BEGIN_CRITICAL_SECTION2(dict, obj); + + // We've locked dict, but the actual dict could have changed + // since we locked it. + dict = _PyObject_ManagedDictPointer(obj)->dict; + err = _PyDict_DetachFromObject(dict, obj); + if (err == 0) { + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + } + Py_END_CRITICAL_SECTION2(); + + if (err == 0) { + Py_XDECREF(dict); } } else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); + PyDictObject *dict; + + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(dict); } - return 0; + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + return err; } void PyObject_ClearManagedDict(PyObject *obj) { - PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; + if (_PyObject_SetManagedDict(obj, NULL) < 0) { + PyErr_WriteUnraisable(NULL); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + + if (FT_ATOMIC_LOAD_PTR_RELAXED(mp->ma_values) != _PyObject_InlineValues(obj)) { + return 0; + } + + // We could be called with an unlocked dict when the caller knows the + // values are already detached, so we assert after inline values check. + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + + PyDictValues *values = copy_values(mp->ma_values); + + if (values == NULL) { + return -1; + } + mp->ma_values = values; + + FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0); + + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; +} + +static inline PyObject * +ensure_managed_dict(PyObject *obj) +{ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { + dict = _PyObject_MaterializeManagedDict(obj); + } + else { +#ifdef Py_GIL_DISABLED + // Check again that we're not racing with someone else creating the dict + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + goto done; + } +#endif + dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), + CACHED_KEYS(tp)); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } - dorv_ptr->dict = NULL; - free_values(values); } - else { - PyObject *dict = dorv_ptr->dict; - if (dict) { - dorv_ptr->dict = NULL; - Py_DECREF(dict); + return (PyObject *)dict; +} + +static inline PyObject * +ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) +{ + PyDictKeysObject *cached; + + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + dict = *dictptr; + if (dict != NULL) { + goto done; + } +#endif + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); + dict = new_dict_with_shared_keys(interp, cached); + } + else { + dict = PyDict_New(); } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } + return dict; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; - PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes( - interp, CACHED_KEYS(tp), values); - if (dict != NULL) { - dorv_ptr->dict = dict; - } - } - else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); - if (dict == NULL) { - dictkeys_incref(CACHED_KEYS(tp)); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; - } - } + return Py_XNewRef(ensure_managed_dict(obj)); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -5731,64 +7251,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( - interp, CACHED_KEYS(tp)); - } - else { - *dictptr = dict = PyDict_New(); - } - } + + return Py_XNewRef(ensure_nonmanaged_dict(obj, dictptr)); } - return Py_XNewRef(dict); } int -_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *key, PyObject *value) { PyObject *dict; int res; - PyDictKeysObject *cached; - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); - if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - assert(dictptr != NULL); - dict = *dictptr; - if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); - dictkeys_incref(cached); - dict = new_dict_with_shared_keys(interp, cached); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } - else { - res = PyDict_SetItem(dict, key, value); - } - } else { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } else { - res = PyDict_SetItem(dict, key, value); - } + dict = ensure_nonmanaged_dict(obj, dictptr); + if (dict == NULL) { + return -1; } + + Py_BEGIN_CRITICAL_SECTION(dict); + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, key, value); ASSERT_CONSISTENT(dict); + Py_END_CRITICAL_SECTION(); return res; } @@ -5796,7 +7280,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, @@ -5862,7 +7346,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + /* Start at 2, as 0 and 1 are reserved for CPython */ + for (int i = 2; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { interp->dict_state.watchers[i] = callback; return i; @@ -5912,16 +7397,32 @@ _PyDict_SendEvent(int watcher_bits, // unraisablehook keep a reference to it, so we don't pass the // dict as context, just an informative string message. Dict // repr can call arbitrary code, so we invent a simpler version. - PyObject *context = PyUnicode_FromFormat( - "%s watcher callback for ", + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for ", dict_event_name(event), mp); - if (context == NULL) { - context = Py_NewRef(Py_None); - } - PyErr_WriteUnraisable(context); - Py_DECREF(context); } } watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt deleted file mode 100644 index 387ef935ce739e..00000000000000 --- a/Objects/exception_handling_notes.txt +++ /dev/null @@ -1,182 +0,0 @@ -Description of exception handling in Python 3.11 ------------------------------------------------- - -Python 3.11 uses what is known as "zero-cost" exception handling. -Prior to 3.11, exceptions were handled by a runtime stack of "blocks". - -In zero-cost exception handling, the cost of supporting exceptions is minimized. -In the common case (where no exception is raised) the cost is reduced -to zero (or close to zero). -The cost of raising an exception is increased, but not by much. - -The following code: - -def f(): - try: - g(0) - except: - return "fail" - -compiles as follows in 3.10: - - 2 0 SETUP_FINALLY 7 (to 16) - - 3 2 LOAD_GLOBAL 0 (g) - 4 LOAD_CONST 1 (0) - 6 CALL_NO_KW 1 - 8 POP_TOP - 10 POP_BLOCK - 12 LOAD_CONST 0 (None) - 14 RETURN_VALUE - - 4 >> 16 POP_TOP - 18 POP_TOP - 20 POP_TOP - - 5 22 POP_EXCEPT - 24 LOAD_CONST 3 ('fail') - 26 RETURN_VALUE - -Note the explicit instructions to push and pop from the "block" stack: -SETUP_FINALLY and POP_BLOCK. - -In 3.11, the SETUP_FINALLY and POP_BLOCK are eliminated, replaced with -a table to determine where to jump to when an exception is raised. - - 1 0 RESUME 0 - - 2 2 NOP - - 3 4 LOAD_GLOBAL 1 (g + NULL) - 16 LOAD_CONST 1 (0) - 18 PRECALL 1 - 22 CALL 1 - 32 POP_TOP - 34 LOAD_CONST 0 (None) - 36 RETURN_VALUE - >> 38 PUSH_EXC_INFO - - 4 40 POP_TOP - - 5 42 POP_EXCEPT - 44 LOAD_CONST 2 ('fail') - 46 RETURN_VALUE - >> 48 COPY 3 - 50 POP_EXCEPT - 52 RERAISE 1 -ExceptionTable: - 4 to 32 -> 38 [0] - 38 to 40 -> 48 [1] lasti - -(Note this code is from 3.11, later versions may have slightly different bytecode.) - -If an instruction raises an exception then its offset is used to find the target to jump to. -For example, the CALL at offset 22, falls into the range 4 to 32. -So, if g() raises an exception, then control jumps to offset 38. - - -Unwinding ---------- - -When an exception is raised, the current instruction offset is used to find following: -target to jump to, stack depth, and 'lasti', which determines whether the instruction -offset of the raising instruction should be pushed. - -This information is stored in the exception table, described below. - -If there is no relevant entry, the exception bubbles up to the caller. - -If there is an entry, then: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if 'lasti' is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. - - -Format of the exception table ------------------------------ - -Conceptually, the exception table consists of a sequence of 5-tuples: - 1. start-offset (inclusive) - 2. end-offset (exclusive) - 3. target - 4. stack-depth - 5. push-lasti (boolean) - -All offsets and lengths are in instructions, not bytes. - -We want the format to be compact, but quickly searchable. -For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. -Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. - -It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - start, size, target, depth, push-lasti - -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each instruction takes 2 bytes. -It also happens that depth is generally quite small. - -So, we need to encode: - start (up to 30 bits) - size (up to 30 bits) - target (up to 30 bits) - depth (up to ~8 bits) - lasti (1 bit) - -We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. -Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. -Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a bit are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. - -In addition, we will combine depth and lasti into a single value, ((depth<<1)+lasti), before encoding. - -For example, the exception entry: - start: 20 - end: 28 - target: 100 - depth: 3 - lasti: False - -is encoded first by converting to the more compact four value form: - start: 20 - size: 8 - target: 100 - depth<<1+lasti: 6 - -which is then encoded as: - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 - -for a total of five bytes. - - - -Script to parse the exception table ------------------------------------ - -def parse_varint(iterator): - b = next(iterator) - val = b & 63 - while b&64: - val <<= 6 - b = next(iterator) - val |= b&63 - return val - -def parse_exception_table(code): - iterator = iter(code.co_exceptiontable) - try: - while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 - end = start + length - 2 # Present as inclusive, not exclusive - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl&1) - yield start, end, target, depth, lasti - except StopIteration: - return diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 62a44234b34047..6376f2f012a7d6 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -8,10 +8,11 @@ #include #include "pycore_abstract.h" // _PyObject_RealIsSubclass() #include "pycore_ceval.h" // _Py_EnterRecursiveCall -#include "pycore_pyerrors.h" // struct _PyErr_SetRaisedException #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_initconfig.h" +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_pyerrors.h" // struct _PyErr_SetRaisedException #include "osdefs.h" // SEP @@ -77,6 +78,40 @@ BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) return 0; } + +static PyObject * +BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + PyTypeObject *type = _PyType_CAST(type_obj); + if (!_PyArg_NoKwnames(type->tp_name, kwnames)) { + return NULL; + } + + PyBaseExceptionObject *self; + self = (PyBaseExceptionObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + + // The dict is created on the fly in PyObject_GenericSetAttr() + self->dict = NULL; + self->notes = NULL; + self->traceback = NULL; + self->cause = NULL; + self->context = NULL; + self->suppress_context = 0; + + self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + if (!self->args) { + Py_DECREF(self); + return NULL; + } + + return (PyObject *)self; +} + + static int BaseException_clear(PyBaseExceptionObject *self) { @@ -485,6 +520,7 @@ static PyTypeObject _PyExc_BaseException = { (initproc)BaseException_init, /* tp_init */ 0, /* tp_alloc */ BaseException_new, /* tp_new */ + .tp_vectorcall = BaseException_vectorcall, }; /* the CPython API expects exceptions to be (PyObject *) - both a hold-over from the previous implementation and also allowing Python objects to be used @@ -509,10 +545,10 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ }; \ PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME -#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ -static PyTypeObject _PyExc_ ## EXCNAME = { \ +#define MiddlingExtendsExceptionEx(EXCBASE, EXCNAME, PYEXCNAME, EXCSTORE, EXCDOC) \ +PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ - # EXCNAME, \ + # PYEXCNAME, \ sizeof(Py ## EXCSTORE ## Object), \ 0, (destructor)EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, \ @@ -521,8 +557,12 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \ 0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \ (initproc)EXCSTORE ## _init, 0, 0, \ -}; \ -PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME +}; + +#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ + static MiddlingExtendsExceptionEx( \ + EXCBASE, EXCNAME, EXCNAME, EXCSTORE, EXCDOC); \ + PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ @@ -875,13 +915,9 @@ BaseExceptionGroup_str(PyBaseExceptionGroupObject *self) } static PyObject * -BaseExceptionGroup_derive(PyObject *self_, PyObject *args) +BaseExceptionGroup_derive(PyObject *self_, PyObject *excs) { PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(self_); - PyObject *excs = NULL; - if (!PyArg_ParseTuple(args, "O", &excs)) { - return NULL; - } PyObject *init_args = PyTuple_Pack(2, self->msg, excs); if (!init_args) { return NULL; @@ -1175,13 +1211,8 @@ exceptiongroup_split_recursive(PyObject *exc, } static PyObject * -BaseExceptionGroup_split(PyObject *self, PyObject *args) +BaseExceptionGroup_split(PyObject *self, PyObject *matcher_value) { - PyObject *matcher_value = NULL; - if (!PyArg_UnpackTuple(args, "split", 1, 1, &matcher_value)) { - return NULL; - } - _exceptiongroup_split_matcher_type matcher_type; if (get_matcher_type(matcher_value, &matcher_type) < 0) { return NULL; @@ -1206,13 +1237,8 @@ BaseExceptionGroup_split(PyObject *self, PyObject *args) } static PyObject * -BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) +BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) { - PyObject *matcher_value = NULL; - if (!PyArg_UnpackTuple(args, "subgroup", 1, 1, &matcher_value)) { - return NULL; - } - _exceptiongroup_split_matcher_type matcher_type; if (get_matcher_type(matcher_value, &matcher_type) < 0) { return NULL; @@ -1487,9 +1513,9 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, - {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_VARARGS}, - {"split", (PyCFunction)BaseExceptionGroup_split, METH_VARARGS}, - {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_VARARGS}, + {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_O}, + {"split", (PyCFunction)BaseExceptionGroup_split, METH_O}, + {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_O}, {NULL} }; @@ -2190,6 +2216,10 @@ SimpleExtendsException(PyExc_Exception, RuntimeError, SimpleExtendsException(PyExc_RuntimeError, RecursionError, "Recursion limit exceeded."); +// PythonFinalizationError extends RuntimeError +SimpleExtendsException(PyExc_RuntimeError, PythonFinalizationError, + "Operation blocked during Python finalization."); + /* * NotImplementedError extends RuntimeError */ @@ -2579,6 +2609,11 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, "Improper mixture of spaces and tabs."); +/* + * IncompleteInputError extends SyntaxError + */ +MiddlingExtendsExceptionEx(PyExc_SyntaxError, IncompleteInputError, _IncompleteInputError, + SyntaxError, "incomplete input."); /* * LookupError extends Exception @@ -3547,7 +3582,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #undef EOPNOTSUPP #undef EPROTONOSUPPORT #undef EPROTOTYPE -#undef ETIMEDOUT #undef EWOULDBLOCK #if defined(WSAEALREADY) && !defined(EALREADY) @@ -3568,9 +3602,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning, #if defined(WSAESHUTDOWN) && !defined(ESHUTDOWN) #define ESHUTDOWN WSAESHUTDOWN #endif -#if defined(WSAETIMEDOUT) && !defined(ETIMEDOUT) -#define ETIMEDOUT WSAETIMEDOUT -#endif #if defined(WSAEWOULDBLOCK) && !defined(EWOULDBLOCK) #define EWOULDBLOCK WSAEWOULDBLOCK #endif @@ -3648,10 +3679,12 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) + {&_PyExc_IncompleteInputError, "_IncompleteInputError"}, // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) ITEM(NotImplementedError), // base: RuntimeError(Exception) + ITEM(PythonFinalizationError), // base: RuntimeError(Exception) ITEM(RecursionError), // base: RuntimeError(Exception) ITEM(UnboundLocalError), // base: NameError(Exception) ITEM(UnicodeError), // base: ValueError(Exception) @@ -3681,6 +3714,11 @@ _PyExc_InitTypes(PyInterpreterState *interp) if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } + if (exc->tp_new == BaseException_new + && exc->tp_init == (initproc)BaseException_init) + { + exc->tp_vectorcall = BaseException_vectorcall; + } } return 0; } @@ -3691,7 +3729,7 @@ _PyExc_FiniTypes(PyInterpreterState *interp) { for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(interp, exc); + _PyStaticType_FiniBuiltin(interp, exc); } } @@ -3699,10 +3737,6 @@ _PyExc_FiniTypes(PyInterpreterState *interp) PyStatus _PyExc_InitGlobalObjects(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return _PyStatus_OK(); - } - if (preallocate_memerrors() < 0) { return _PyStatus_NO_MEMORY(); } @@ -3758,6 +3792,9 @@ _PyExc_InitState(PyInterpreterState *interp) #endif ADD_ERRNO(ProcessLookupError, ESRCH); ADD_ERRNO(TimeoutError, ETIMEDOUT); +#ifdef WSAETIMEDOUT + ADD_ERRNO(TimeoutError, WSAETIMEDOUT); +#endif return _PyStatus_OK(); diff --git a/Objects/fileobject.c b/Objects/fileobject.c index 5522eba34eace9..bae49d367b65ee 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -80,13 +80,7 @@ PyFile_GetLine(PyObject *f, int n) "EOF when reading a line"); } else if (s[len-1] == '\n') { - if (Py_REFCNT(result) == 1) - _PyBytes_Resize(&result, len-1); - else { - PyObject *v; - v = PyBytes_FromStringAndSize(s, len-1); - Py_SETREF(result, v); - } + (void) _PyBytes_Resize(&result, len-1); } } if (n < 0 && result != NULL && PyUnicode_Check(result)) { @@ -174,6 +168,13 @@ PyObject_AsFileDescriptor(PyObject *o) PyObject *meth; if (PyLong_Check(o)) { + if (PyBool_Check(o)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(o); } else if (PyObject_GetOptionalAttr(o, &_Py_ID(fileno), &meth) < 0) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 211e2c699bac8d..9e1ef2a21b35c4 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -8,7 +8,7 @@ #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState.float_state +#include "pycore_interp.h" // _Py_float_freelist #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -26,17 +26,13 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif - - -#if PyFloat_MAXFREELIST > 0 -static struct _Py_float_state * -get_float_state(void) +#ifdef WITH_FREELISTS +static struct _Py_float_freelist * +get_float_freelist(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->float_state; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + assert(freelists != NULL); + return &freelists->floats; } #endif @@ -102,10 +98,18 @@ PyFloat_GetInfo(void) return NULL; } -#define SetIntFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyLong_FromLong(flag)) -#define SetDblFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyFloat_FromDouble(flag)) +#define SetFlag(CALL) \ + do { \ + PyObject *flag = (CALL); \ + if (flag == NULL) { \ + Py_CLEAR(floatinfo); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(floatinfo, pos++, flag); \ + } while (0) + +#define SetIntFlag(FLAG) SetFlag(PyLong_FromLong((FLAG))) +#define SetDblFlag(FLAG) SetFlag(PyFloat_FromDouble((FLAG))) SetDblFlag(DBL_MAX); SetIntFlag(DBL_MAX_EXP); @@ -120,11 +124,8 @@ PyFloat_GetInfo(void) SetIntFlag(FLT_ROUNDS); #undef SetIntFlag #undef SetDblFlag +#undef SetFlag - if (PyErr_Occurred()) { - Py_CLEAR(floatinfo); - return NULL; - } return floatinfo; } @@ -132,16 +133,12 @@ PyObject * PyFloat_FromDouble(double fval) { PyFloatObject *op; -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = get_float_state(); - op = state->free_list; +#ifdef WITH_FREELISTS + struct _Py_float_freelist *float_freelist = get_float_freelist(); + op = float_freelist->items; if (op != NULL) { -#ifdef Py_DEBUG - // PyFloat_FromDouble() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - state->free_list = (PyFloatObject *) Py_TYPE(op); - state->numfree--; + float_freelist->items = (PyFloatObject *) Py_TYPE(op); + float_freelist->numfree--; OBJECT_STAT_INC(from_freelist); } else @@ -252,19 +249,15 @@ _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); PyFloatObject *op = (PyFloatObject *)obj; -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = get_float_state(); -#ifdef Py_DEBUG - // float_dealloc() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree >= PyFloat_MAXFREELIST) { +#ifdef WITH_FREELISTS + struct _Py_float_freelist *float_freelist = get_float_freelist(); + if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) { PyObject_Free(op); return; } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; + float_freelist->numfree++; + Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); + float_freelist->items = op; OBJECT_STAT_INC(to_freelist); #else PyObject_Free(op); @@ -275,7 +268,7 @@ static void float_dealloc(PyObject *op) { assert(PyFloat_Check(op)); -#if PyFloat_MAXFREELIST > 0 +#ifdef WITH_FREELISTS if (PyFloat_CheckExact(op)) { _PyFloat_ExactDealloc(op); } @@ -425,7 +418,7 @@ float_richcompare(PyObject *v, PyObject *w, int op) if (PyFloat_Check(w)) j = PyFloat_AS_DOUBLE(w); - else if (!Py_IS_FINITE(i)) { + else if (!isfinite(i)) { if (PyLong_Check(w)) /* If i is an infinity, its magnitude exceeds any * finite integer, so it doesn't matter which int we @@ -630,7 +623,7 @@ float_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, b); if (b == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float division by zero"); + "division by zero"); return NULL; } a = a / b; @@ -646,7 +639,7 @@ float_rem(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float modulo"); + "division by zero"); return NULL; } mod = fmod(vx, wx); @@ -711,7 +704,7 @@ float_divmod(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float divmod()"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -726,7 +719,7 @@ float_floor_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float floor division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -756,13 +749,13 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) if (iw == 0) { /* v**0 is 1, even 0**0 */ return PyFloat_FromDouble(1.0); } - if (Py_IS_NAN(iv)) { /* nan**w = nan, unless w == 0 */ + if (isnan(iv)) { /* nan**w = nan, unless w == 0 */ return PyFloat_FromDouble(iv); } - if (Py_IS_NAN(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ + if (isnan(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ return PyFloat_FromDouble(iv == 1.0 ? 1.0 : iw); } - if (Py_IS_INFINITY(iw)) { + if (isinf(iw)) { /* v**inf is: 0.0 if abs(v) < 1; 1.0 if abs(v) == 1; inf if * abs(v) > 1 (including case where v infinite) * @@ -777,7 +770,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) else return PyFloat_FromDouble(0.0); } - if (Py_IS_INFINITY(iv)) { + if (isinf(iv)) { /* (+-inf)**w is: inf for w positive, 0 for w negative; in * both cases, we need to add the appropriate sign if w is * an odd integer. @@ -795,8 +788,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) int iw_is_odd = DOUBLE_IS_ODD_INTEGER(iw); if (iw < 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 cannot be raised to a " - "negative power"); + "zero to a negative power"); return NULL; } /* use correct sign if iw is odd */ @@ -892,7 +884,7 @@ float_is_integer_impl(PyObject *self) if (x == -1.0 && PyErr_Occurred()) return NULL; - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) Py_RETURN_FALSE; errno = 0; o = (floor(x) == x) ? Py_True : Py_False; @@ -1028,7 +1020,7 @@ double_round(double x, int ndigits) { } y = (x*pow1)*pow2; /* if y overflows, then rounded value is exactly x */ - if (!Py_IS_FINITE(y)) + if (!isfinite(y)) return PyFloat_FromDouble(x); } else { @@ -1048,7 +1040,7 @@ double_round(double x, int ndigits) { z *= pow1; /* if computation resulted in overflow, raise OverflowError */ - if (!Py_IS_FINITE(z)) { + if (!isfinite(z)) { PyErr_SetString(PyExc_OverflowError, "overflow occurred during round"); return NULL; @@ -1096,7 +1088,7 @@ float___round___impl(PyObject *self, PyObject *o_ndigits) return NULL; /* nans and infinities round to themselves */ - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) return PyFloat_FromDouble(x); /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x @@ -1150,69 +1142,39 @@ char_from_hex(int x) return Py_hexdigits[x]; } +/* This table maps characters to their hexadecimal values, only + * works with encodings whose lower half is ASCII (like UTF-8). + * '0' maps to 0, ..., '9' maps to 9. + * 'a' and 'A' map to 10, ..., 'f' and 'F' map to 15. + * All other indices map to -1. + */ +static const int +_CHAR_TO_HEX[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +/* Convert a character to its hexadecimal value, or -1 if it's not a + * valid hexadecimal character, only works with encodings whose lower + * half is ASCII (like UTF-8). + */ static int -hex_from_char(char c) { - int x; - switch(c) { - case '0': - x = 0; - break; - case '1': - x = 1; - break; - case '2': - x = 2; - break; - case '3': - x = 3; - break; - case '4': - x = 4; - break; - case '5': - x = 5; - break; - case '6': - x = 6; - break; - case '7': - x = 7; - break; - case '8': - x = 8; - break; - case '9': - x = 9; - break; - case 'a': - case 'A': - x = 10; - break; - case 'b': - case 'B': - x = 11; - break; - case 'c': - case 'C': - x = 12; - break; - case 'd': - case 'D': - x = 13; - break; - case 'e': - case 'E': - x = 14; - break; - case 'f': - case 'F': - x = 15; - break; - default: - x = -1; - break; - } - return x; +hex_from_char(unsigned char c) { + return _CHAR_TO_HEX[c]; } /* convert a float to a hexadecimal string */ @@ -1244,7 +1206,7 @@ float_hex_impl(PyObject *self) CONVERT_TO_DOUBLE(self, x); - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) + if (isnan(x) || isinf(x)) return float_repr((PyFloatObject *)self); if (x == 0.0) { @@ -1577,12 +1539,12 @@ float_as_integer_ratio_impl(PyObject *self) CONVERT_TO_DOUBLE(self, self_double); - if (Py_IS_INFINITY(self_double)) { + if (isinf(self_double)) { PyErr_SetString(PyExc_OverflowError, "cannot convert Infinity to integer ratio"); return NULL; } - if (Py_IS_NAN(self_double)) { + if (isnan(self_double)) { PyErr_SetString(PyExc_ValueError, "cannot convert NaN to integer ratio"); return NULL; @@ -2033,28 +1995,23 @@ _PyFloat_InitTypes(PyInterpreterState *interp) } void -_PyFloat_ClearFreeList(PyInterpreterState *interp) +_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; - PyFloatObject *f = state->free_list; +#ifdef WITH_FREELISTS + struct _Py_float_freelist *state = &freelists->floats; + PyFloatObject *f = state->items; while (f != NULL) { PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); PyObject_Free(f); f = next; } - state->free_list = NULL; - state->numfree = 0; -#endif -} - -void -_PyFloat_Fini(PyInterpreterState *interp) -{ - _PyFloat_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; - state->numfree = -1; + state->items = NULL; + if (is_finalization) { + state->numfree = -1; + } + else { + state->numfree = 0; + } #endif } @@ -2068,11 +2025,11 @@ _PyFloat_FiniType(PyInterpreterState *interp) void _PyFloat_DebugMallocStats(FILE *out) { -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = get_float_state(); +#ifdef WITH_FREELISTS + struct _Py_float_freelist *float_freelist = get_float_freelist(); _PyDebugAllocatorStats(out, "free PyFloatObject", - state->numfree, sizeof(PyFloatObject)); + float_freelist->numfree, sizeof(PyFloatObject)); #endif } @@ -2103,12 +2060,12 @@ PyFloat_Pack2(double x, char *data, int le) e = 0; bits = 0; } - else if (Py_IS_INFINITY(x)) { + else if (isinf(x)) { sign = (x < 0.0); e = 0x1f; bits = 0; } - else if (Py_IS_NAN(x)) { + else if (isnan(x)) { /* There are 2046 distinct half-precision NaNs (1022 signaling and 1024 quiet), but there are only two quiet NaNs that don't arise by quieting a signaling NaN; we get those by setting the topmost bit @@ -2277,7 +2234,7 @@ PyFloat_Pack4(double x, char *data, int le) float y = (float)x; int i, incr = 1; - if (Py_IS_INFINITY(y) && !Py_IS_INFINITY(x)) + if (isinf(y) && !isinf(x)) goto Overflow; unsigned char s[sizeof(float)]; diff --git a/Objects/frame_layout.md b/Objects/frame_layout.md deleted file mode 100644 index 2f95214db56498..00000000000000 --- a/Objects/frame_layout.md +++ /dev/null @@ -1,132 +0,0 @@ -# The Frame Stack - -Each call to a Python function has an activation record, -commonly known as a "frame". -Python semantics allows frames to outlive the activation, -so they have (before 3.11) been allocated on the heap. -This is expensive as it requires many allocations and -results in poor locality of reference. - -In 3.11, rather than have these frames scattered about memory, -as happens for heap-allocated objects, frames are allocated -contiguously in a per-thread stack. -This improves performance significantly for two reasons: -* It reduces allocation overhead to a pointer comparison and increment. -* Stack allocated data has the best possible locality and will always be in - CPU cache. - -Generator and coroutines still need heap allocated activation records, but -can be linked into the per-thread stack so as to not impact performance too much. - -## Layout - -Each activation record consists of four conceptual sections: - -* Local variables (including arguments, cells and free variables) -* Evaluation stack -* Specials: The per-frame object references needed by the VM: globals dict, - code object, etc. -* Linkage: Pointer to the previous activation record, stack depth, etc. - -### Layout - -The specials and linkage sections are a fixed size, so are grouped together. - -Each activation record is laid out as: -* Specials and linkage -* Locals -* Stack - -This seems to provide the best performance without excessive complexity. -It needs the interpreter to hold two pointers, a frame pointer and a stack pointer. - -#### Alternative layout - -An alternative layout that was used for part of 3.11 alpha was: - -* Locals -* Specials and linkage -* Stack - -This has the advantage that no copying is required when making a call, -as the arguments on the stack are (usually) already in the correct -location for the parameters. However, it requires the VM to maintain -an extra pointer for the locals, which can hurt performance. - -A variant that only needs the need two pointers is to reverse the numbering -of the locals, so that the last one is numbered `0`, and the first in memory -is numbered `N-1`. -This allows the locals, specials and linkage to accessed from the frame pointer. -We may implement this in the future. - -#### Note: - -> In a contiguous stack, we would need to save one fewer registers, as the -> top of the caller's activation record would be the same at the base of the -> callee's. However, since some activation records are kept on the heap we -> cannot do this. - -### Generators and Coroutines - -Generators and coroutines contain a `_PyInterpreterFrame` -The specials sections contains the following pointers: - -* Globals dict -* Builtins dict -* Locals dict (not the "fast" locals, but the locals for eval and class creation) -* Code object -* Heap allocated `PyFrameObject` for this activation record, if any. -* The function. - -The pointer to the function is not strictly required, but it is cheaper to -store a strong reference to the function and borrowed references to the globals -and builtins, than strong references to both globals and builtins. - -### Frame objects - -When creating a backtrace or when calling `sys._getframe()` the frame becomes -visible to Python code. When this happens a new `PyFrameObject` is created -and a strong reference to it placed in the `frame_obj` field of the specials -section. The `frame_obj` field is initially `NULL`. - -The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. -If it does then `_PyInterpreterFrame` is copied into the `PyFrameObject`, -except the evaluation stack which must be empty at this point. -The linkage section is updated to reflect the new location of the frame. - -This mechanism provides the appearance of persistent, heap-allocated -frames for each activation, but with low runtime overhead. - -### Generators and Coroutines - - -Generator objects have a `_PyInterpreterFrame` embedded in them. -This means that creating a generator requires only a single allocation, -reducing allocation overhead and improving locality of reference. -The embedded frame is linked into the per-thread frame when iterated or -awaited. - -If a frame object associated with a generator outlives the generator, then -the embedded `_PyInterpreterFrame` is copied into the frame object. - - -All the above applies to coroutines and async generators as well. - -### Field names - -Many of the fields in `_PyInterpreterFrame` were copied from the 3.10 `PyFrameObject`. -Thus, some of the field names may be a bit misleading. - -For example the `f_globals` field has a `f_` prefix implying it belongs to the -`PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct. -We may rationalize this naming scheme for 3.12. - - -### Shim frames - -On entry to `_PyEval_EvalFrameDefault()` a shim `_PyInterpreterFrame` is pushed. -This frame is stored on the C stack, and popped when `_PyEval_EvalFrameDefault()` -returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and -`RETURN_GENERATOR` do not need to check whether the current frame is the entry frame. -The shim frame points to a special code object containing the `INTERPRETER_EXIT` -instruction which cleans up the shim frame and returns. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d75444393f3697..3dc9ff058a5c9e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -16,12 +16,741 @@ #define OFF(x) offsetof(PyFrameObject, x) + +// Returns borrowed reference or NULL +static PyObject * +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) +{ + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = PyStackRef_AsPyObjectBorrow(fast[i]); + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns the fast locals index of the key + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + */ + + assert(PyUnicode_CheckExact(key)); + + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + int found_key = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + found_key = true; + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + if (!found_key) { + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (_PyUnicode_EQ(name, key)) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + } + + return -1; +} + +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return -1; + } + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i >= 0) { + _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyStackRef oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + cell = PyStackRef_AsPyObjectBorrow(oldvalue); + } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) { + PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); + if (PyCell_Check(as_obj)) { + cell = as_obj; + } + } + if (cell != NULL) { + PyObject *oldvalue_o = PyCell_GET(cell); + if (value != oldvalue_o) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue_o); + } + } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { + PyStackRef_XCLOSE(fast[i]); + fast[i] = PyStackRef_FromPyObjectNew(value); + } + return 0; + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + return PyDict_SetItem(extra, key, value); +} + +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + if (keys == NULL) { + return -1; + } + + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + return -1; + } + + PyObject *key = NULL; + PyObject *value = NULL; + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (framelocalsproxy_setitem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + Py_DECREF(iter); + + return 0; +} + +static PyObject * +framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *names = PyList_New(0); + if (names == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); + if (val) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (PyList_Append(names, name) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + if (PyList_Append(names, key) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + return names; +} + +static void +framelocalsproxy_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + PyFrameObject *frame = (PyFrameObject*)PyTuple_GET_ITEM(args, 0); + assert(PyFrame_Check(frame)); + + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyObject *self) +{ + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static PyObject * +framelocalsproxy_iter(PyObject *self) +{ + PyObject* keys = framelocalsproxy_keys(self, NULL); + if (keys == NULL) { + return NULL; + } + + PyObject* iter = PyObject_GetIter(keys); + Py_XDECREF(keys); + + return iter; +} + +static PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject *dct = PyDict_New(); + if (dct == NULL) { + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + return NULL; + } + + PyObject *result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + if (dct == NULL) { + Py_ReprLeave(self); + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + Py_ReprLeave(self); + return NULL; + } + + PyObject *repr = PyObject_Repr(dct); + Py_DECREF(dct); + + Py_ReprLeave(self); + + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + +static PyObject* +framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *values = PyList_New(0); + if (values == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + if (value) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + return values; +} + +static PyObject * +framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *items = PyList_New(0); + if (items == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + + if (value) { + PyObject *pair = PyTuple_Pack(2, name, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + return items; +} + +static Py_ssize_t +framelocalsproxy_length(PyObject *self) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + Py_ssize_t size = 0; + + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + size++; + } + } + return size; +} + +static int +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + return 1; + } + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + return PyDict_Contains(extra, key); + } + + return 0; +} + +static PyObject* framelocalsproxy___contains__(PyObject *self, PyObject *key) +{ + int result = framelocalsproxy_contains(self, key); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject* result = PyDict_New(); + + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + + if (result == NULL) { + return NULL; + } + + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + +static PySequenceMethods framelocalsproxy_as_sequence = { + .sq_contains = framelocalsproxy_contains, +}; + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, + NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, + NULL}, + {"copy", _PyCFunction_CAST(framelocalsproxy_copy), METH_NOARGS, + NULL}, + {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, + NULL}, + {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, + NULL}, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, + .tp_as_sequence = &framelocalsproxy_as_sequence, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + if (args == NULL) { + return NULL; + } + + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ }; - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -30,23 +759,43 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + if (f->f_frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + f->f_frame->f_locals = PyDict_New(); + if (f->f_frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + return _PyFrameLocalsProxy_New(f); } int PyFrame_GetLineNumber(PyFrameObject *f) { assert(f != NULL); - if (f->f_lineno != 0) { - return f->f_lineno; + if (f->f_lineno == -1) { + // We should calculate it once. If we can't get the line number, + // set f->f_lineno to 0. + f->f_lineno = PyUnstable_InterpreterFrame_GetLine(f->f_frame); + if (f->f_lineno < 0) { + f->f_lineno = 0; + return -1; + } } - else { - return PyUnstable_InterpreterFrame_GetLine(f->f_frame); + + if (f->f_lineno > 0) { + return f->f_lineno; } + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } static PyObject * @@ -127,10 +876,13 @@ frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignore } if (value == Py_True) { f->f_trace_opcodes = 1; - _PyInterpreterState_GET()->f_opcode_trace_set = true; + if (f->f_trace) { + return _PyEval_SetOpcodeTrace(f, true); + } } else { f->f_trace_opcodes = 0; + return _PyEval_SetOpcodeTrace(f, false); } return 0; } @@ -301,11 +1053,6 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i] = UNINITIALIZED; } stacks[0] = EMPTY_STACK; - if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - { - // Generators get sent None while starting: - stacks[0] = push_value(stacks[0], Object); - } int todo = 1; while (todo) { todo = 0; @@ -329,6 +1076,8 @@ mark_stacks(PyCodeObject *code_obj, int len) switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: { int64_t target_stack; int j = next_i + oparg; @@ -587,26 +1336,12 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_SUSPENDED; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); + return FRAME_STATE_SUSPENDED(gen->gi_frame_state); } return false; } @@ -778,7 +1513,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore for (int i = 0; i < code->co_nlocalsplus; i++) { // Counting every unbound local is overly-cautious, but a full flow // analysis (like we do in the compiler) is probably too expensive: - unbound += f->f_frame->localsplus[i] == NULL; + unbound += PyStackRef_IsNull(f->f_frame->localsplus[i]); } if (unbound) { const char *e = "assigning None to %d unbound local%s"; @@ -789,8 +1524,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore // Do this in a second pass to avoid writing a bunch of Nones when // warnings are being treated as errors and the previous bit raises: for (int i = 0; i < code->co_nlocalsplus; i++) { - if (f->f_frame->localsplus[i] == NULL) { - f->f_frame->localsplus[i] = Py_NewRef(Py_None); + if (PyStackRef_IsNull(f->f_frame->localsplus[i])) { + f->f_frame->localsplus[i] = PyStackRef_None; unbound--; } } @@ -803,23 +1538,19 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { /* Pop exception stack as well as the evaluation stack */ - PyThreadState *tstate = _PyThreadState_GET(); - _PyErr_StackItem *exc_info = tstate->exc_info; - PyObject *value = exc_info->exc_value; - PyObject *exc = _PyFrame_StackPop(f->f_frame); + PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(f->f_frame)); assert(PyExceptionInstance_Check(exc) || exc == Py_None); - exc_info->exc_value = exc; - Py_XDECREF(value); + PyThreadState *tstate = _PyThreadState_GET(); + Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { - PyObject *v = _PyFrame_StackPop(f->f_frame); - Py_XDECREF(v); + PyStackRef_XCLOSE(_PyFrame_StackPop(f->f_frame)); } start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ f->f_lineno = 0; - f->f_frame->prev_instr = _PyCode_CODE(code) + best_addr; + f->f_frame->instr_ptr = _PyCode_CODE(code) + best_addr; return 0; } @@ -840,6 +1571,9 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) } if (v != f->f_trace) { Py_XSETREF(f->f_trace, Py_XNewRef(v)); + if (v != NULL && f->f_trace_opcodes) { + return _PyEval_SetOpcodeTrace(f, true); + } } return 0; } @@ -885,13 +1619,14 @@ frame_dealloc(PyFrameObject *f) frame->f_executable = NULL; Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); - PyObject **locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyStackRef_CLEAR(locals[i]); } } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); PyObject_GC_Del(f); Py_XDECREF(co); Py_TRASHCAN_END; @@ -902,6 +1637,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -913,14 +1649,16 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); /* locals and stack */ - PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame); assert(f->f_frame->stacktop >= 0); for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyStackRef_CLEAR(locals[i]); } f->f_frame->stacktop = 0; + Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -928,10 +1666,13 @@ static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(f->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(f->f_frame); if (gen->gi_frame_state == FRAME_EXECUTING) { goto running; } + if (FRAME_STATE_SUSPENDED(gen->gi_frame_state)) { + goto suspended; + } _PyGen_Finalize((PyObject *)gen); } else if (f->f_frame->owner == FRAME_OWNED_BY_THREAD) { @@ -946,6 +1687,10 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) PyErr_SetString(PyExc_RuntimeError, "cannot clear an executing frame"); return NULL; +suspended: + PyErr_SetString(PyExc_RuntimeError, + "cannot clear a suspended frame"); + return NULL; } PyDoc_STRVAR(clear__doc__, @@ -1040,8 +1785,8 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; return f; } @@ -1077,39 +1822,13 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: - f->f_frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + f->f_frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; assert(!_PyFrame_IsIncomplete(f->f_frame)); Py_DECREF(func); _PyObject_GC_TRACK(f); return f; } -static int -_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) -{ - // This only works when opcode is a non-quickened form: - assert(_PyOpcode_Deopt[opcode] == opcode); - int check_oparg = 0; - for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->prev_instr; instruction++) - { - int check_opcode = _PyOpcode_Deopt[instruction->op.code]; - check_oparg |= instruction->op.arg; - if (check_opcode == opcode && check_oparg == oparg) { - return 1; - } - if (check_opcode == EXTENDED_ARG) { - check_oparg <<= 8; - } - else { - check_oparg = 0; - } - instruction += _PyOpcode_Caches[check_opcode]; - } - return 0; -} - - // Initialize frame free variables if needed static void frame_init_get_vars(_PyInterpreterFrame *frame) @@ -1127,13 +1846,13 @@ frame_init_get_vars(_PyInterpreterFrame *frame) /* Free vars have not been initialized -- Do that */ PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - int offset = PyCode_GetFirstFree(co); + int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); } @@ -1155,7 +1874,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 0; } - PyObject *value = frame->localsplus[i]; + PyObject *value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); if (frame->stacktop) { if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. @@ -1163,20 +1882,14 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, value = PyCell_GET(value); } else if (kind & CO_FAST_CELL) { - // Note that no *_DEREF ops can happen before MAKE_CELL - // executes. So there's no need to duplicate the work - // that MAKE_CELL would otherwise do later, if it hasn't - // run yet. if (value != NULL) { - if (PyCell_Check(value) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. + if (PyCell_Check(value)) { + assert(!_PyFrame_IsIncomplete(frame)); value = PyCell_GET(value); } // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), // with the initial value set when the frame was created... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). + // (unlikely) ...or it was set via the f_locals proxy. } } } @@ -1188,103 +1901,54 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } - } - PyObject *hidden = NULL; - - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } - } - - frame_init_get_vars(frame); + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } - continue; - } - if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; - } - } - } + PyObject* value = framelocalsproxy_getval(frame, co, i); - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; + if (value != NULL) { + return true; + } } - locals = innerlocals; - } - else { - Py_INCREF(locals); } - Py_CLEAR(hidden); - return locals; - - error: - Py_XDECREF(hidden); - return NULL; + return false; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + if (frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + frame->f_locals = PyDict_New(); + if (frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(frame->f_locals); } - Py_DECREF(locals); - return 0; + + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + return _PyFrameLocalsProxy_New(f); } @@ -1338,112 +2002,25 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); -} - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } int @@ -1521,7 +2098,7 @@ PyFrame_GetGenerator(PyFrameObject *frame) if (frame->f_frame->owner != FRAME_OWNED_BY_GENERATOR) { return NULL; } - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return Py_NewRef(gen); } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index e8ad486150da47..40211297be20c0 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,12 +3,12 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_long.h" // _PyLong_GetOne() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() -static PyObject* func_repr(PyFunctionObject *op); - static const char * func_event_name(PyFunction_WatchEvent event) { switch (event) { @@ -34,21 +34,9 @@ notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, func, new_value) < 0) { - // Don't risk resurrecting the func if an unraisablehook keeps a - // reference; pass a string as context. - PyObject *context = NULL; - PyObject *repr = func_repr(func); - if (repr != NULL) { - context = PyUnicode_FromFormat( - "%s watcher callback for %U", - func_event_name(event), repr); - Py_DECREF(repr); - } - if (context == NULL) { - context = Py_NewRef(Py_None); - } - PyErr_WriteUnraisable(context); - Py_DECREF(context); + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for function %U at %p", + func_event_name(event), func->func_qualname, func); } } i++; @@ -66,6 +54,15 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, if (interp->active_func_watchers) { notify_func_watchers(interp, event, func, new_value); } + switch (event) { + case PyFunction_EVENT_MODIFY_CODE: + case PyFunction_EVENT_MODIFY_DEFAULTS: + case PyFunction_EVENT_MODIFY_KWDEFAULTS: + RARE_EVENT_INTERP_INC(interp, func_modification); + break; + default: + break; + } } int @@ -105,8 +102,8 @@ PyFunction_ClearWatcher(int watcher_id) PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { - PyObject *module = Py_XNewRef(PyDict_GetItemWithError(constr->fc_globals, &_Py_ID(__name__))); - if (!module && PyErr_Occurred()) { + PyObject *module; + if (PyDict_GetItemRef(constr->fc_globals, &_Py_ID(__name__), &module) < 0) { return NULL; } @@ -128,9 +125,13 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; + // NOTE: functions created via FrameConstructor do not use deferred + // reference counting because they are typically not part of cycles + // nor accessed by multiple threads. _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -171,12 +172,11 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname Py_INCREF(doc); // __module__: Use globals['__name__'] if it exists, or NULL. - PyObject *module = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); + PyObject *module; PyObject *builtins = NULL; - if (module == NULL && _PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &module) < 0) { goto error; } - Py_XINCREF(module); builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref if (builtins == NULL) { @@ -204,9 +204,16 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; + if ((code_obj->co_flags & CO_NESTED) == 0) { + // Use deferred reference counting for top-level functions, but not + // nested functions because they are more likely to capture variables, + // which makes prompt deallocation more important. + _PyObject_SetDeferredRefcount((PyObject *)op); + } _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; @@ -223,92 +230,141 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname } /* -Function versions ------------------ +(This is purely internal documentation. There are no public APIs here.) + +Function (and code) versions +---------------------------- + +The Tier 1 specializer generates CALL variants that can be invalidated +by changes to critical function attributes: + +- __code__ +- __defaults__ +- __kwdefaults__ +- __closure__ + +For this purpose function objects have a 32-bit func_version member +that the specializer writes to the specialized instruction's inline +cache and which is checked by a guard on the specialized instructions. -Function versions are used to detect when a function object has been -updated, invalidating inline cache data used by the `CALL` bytecode -(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). +The MAKE_FUNCTION bytecode sets func_version from the code object's +co_version field. The latter is initialized from a counter in the +interpreter state (interp->func_state.next_version) and never changes. +When this counter overflows, it remains zero and the specializer loses +the ability to specialize calls to new functions. -They are also used by the Tier 2 superblock creation code to find -the function being called (and from there the code object). +The func_version is reset to zero when any of the critical attributes +is modified; after this point the specializer will no longer specialize +calls to this function, and the guard will always fail. -How does a function's `func_version` field get initialized? +The function and code version cache +----------------------------------- -- `PyFunction_New` and friends initialize it to 0. -- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. -- It is reset to 0 when various attributes like `__code__` are set. -- A new version is allocated by `_PyFunction_GetVersionForCurrentState` - when the specializer needs a version and the version is 0. +The Tier 2 optimizer now has a problem, since it needs to find the +function and code objects given only the version number from the inline +cache. Our solution is to maintain a cache mapping version numbers to +function and code objects. To limit the cache size we could hash +the version number, but for now we simply use it modulo the table size. -The latter allocates versions using a counter in the interpreter state; -when the counter wraps around to 0, no more versions are allocated. -There is one other special case: functions with a non-standard -`vectorcall` field are not given a version. +There are some corner cases (e.g. generator expressions) where we will +be unable to find the function object in the cache but we can still +find the code object. For this reason the cache stores both the +function object and the code object. -When the function version is 0, the `CALL` bytecode is not specialized. +The cache doesn't contain strong references; cache entries are +invalidated whenever the function or code object is deallocated. -Code object versions --------------------- +Invariants +---------- -So where to code objects get their `co_version`? -There is a per-interpreter counter, `next_func_version`. -This is initialized to 1 when the interpreter is created. +These should hold at any time except when one of the cache-mutating +functions is running. + +- For any slot s at index i: + - s->func == NULL or s->func->func_version % FUNC_VERSION_CACHE_SIZE == i + - s->code == NULL or s->code->co_version % FUNC_VERSION_CACHE_SIZE == i + if s->func != NULL, then s->func->func_code == s->code -Code objects get a new `co_version` allocated from this counter upon -creation. Since code objects are nominally immutable, `co_version` can -not be invalidated. The only way it can be 0 is when 2**32 or more -code objects have been created during the process's lifetime. -(The counter isn't reset by `fork()`, extending the lifetime.) */ void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); if (func->func_version != 0) { - PyFunctionObject **slot = + struct _func_version_cache_item *slot = interp->func_state.func_version_cache + (func->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == func) { - *slot = NULL; + if (slot->func == func) { + slot->func = NULL; + // Leave slot->code alone, there may be use for it. } } +#endif func->func_version = version; +#ifndef Py_GIL_DISABLED if (version != 0) { - interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE] = func; + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + slot->func = func; + slot->code = func->func_code; + } +#endif +} + +void +_PyFunction_ClearCodeByVersion(uint32_t version) +{ +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + slot->code = NULL; + slot->func = NULL; + } } +#endif } PyFunctionObject * -_PyFunction_LookupByVersion(uint32_t version) +_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { +#ifdef Py_GIL_DISABLED + return NULL; +#else PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject *func = interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE]; - if (func != NULL && func->func_version == version) { - return func; + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + *p_code = slot->code; + } + } + else { + *p_code = NULL; + } + if (slot->func && slot->func->func_version == version) { + assert(slot->func->func_code == slot->code); + return slot->func; } return NULL; +#endif } uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { - if (func->func_version != 0) { - return func->func_version; - } - if (func->vectorcall != _PyFunction_Vectorcall) { - return 0; - } - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->func_state.next_version == 0) { - return 0; - } - uint32_t v = interp->func_state.next_version++; - _PyFunction_SetVersion(func, v); - return v; + return func->func_version; } PyObject * @@ -459,7 +515,22 @@ static PyObject * func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { - return NULL; + if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { + Py_RETURN_NONE; + } + PyObject *one = _PyLong_GetOne(); + PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); + if (ann_dict == NULL) { + return NULL; + } + if (!PyDict_Check(ann_dict)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(ann_dict)->tp_name); + Py_DECREF(ann_dict); + return NULL; + } + Py_XSETREF(op->func_annotations, ann_dict); + return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { PyObject *ann_tuple = op->func_annotations; @@ -512,8 +583,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); - Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); + PyFunctionObject *func = (PyFunctionObject *)op; + Py_XSETREF(func->func_annotations, annotations); + Py_CLEAR(func->func_annotate); return 0; } @@ -570,6 +642,20 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) nclosure, nfree); return -1; } + + PyObject *func_code = PyFunction_GET_CODE(op); + int old_flags = ((PyCodeObject *)func_code)->co_flags; + int new_flags = ((PyCodeObject *)value)->co_flags; + int mask = CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR; + if ((old_flags & mask) != (new_flags & mask)) { + if (PyErr_Warn(PyExc_DeprecationWarning, + "Assigning a code object of non-matching type is deprecated " + "(e.g., from a generator to a plain function)") < 0) + { + return -1; + } + } + handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_code, Py_NewRef(value)); @@ -697,10 +783,44 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor return 0; } +static PyObject * +func_get_annotate(PyFunctionObject *op, void *Py_UNUSED(ignored)) +{ + if (op->func_annotate == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_annotate); +} + +static int +func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ cannot be deleted"); + return -1; + } + if (Py_IsNone(value)) { + Py_XSETREF(op->func_annotate, value); + return 0; + } + else if (PyCallable_Check(value)) { + Py_XSETREF(op->func_annotate, Py_XNewRef(value)); + Py_CLEAR(op->func_annotations); + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "__annotate__ must be callable or None"); + return -1; + } +} + static PyObject * func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) { - if (op->func_annotations == NULL) { + if (op->func_annotations == NULL && + (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; @@ -722,8 +842,8 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); + Py_CLEAR(op->func_annotate); return 0; } @@ -771,6 +891,7 @@ static PyGetSetDef func_getsetlist[] = { (setter)func_set_kwdefaults}, {"__annotations__", (getter)func_get_annotations, (setter)func_set_annotations}, + {"__annotate__", (getter)func_get_annotate, (setter)func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, {"__name__", (getter)func_get_name, (setter)func_set_name}, {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, @@ -809,14 +930,17 @@ function.__new__ as func_new a tuple that specifies the default argument values closure: object = None a tuple that supplies the bindings for free variables + kwdefaults: object = None + a dictionary that specifies the default keyword argument values Create a function object. [clinic start generated code]*/ static PyObject * func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, - PyObject *name, PyObject *defaults, PyObject *closure) -/*[clinic end generated code: output=99c6d9da3a24e3be input=93611752fc2daf11]*/ + PyObject *name, PyObject *defaults, PyObject *closure, + PyObject *kwdefaults) +/*[clinic end generated code: output=de72f4c22ac57144 input=20c9c9f04ad2d3f2]*/ { PyFunctionObject *newfunc; Py_ssize_t nclosure; @@ -843,6 +967,11 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, return NULL; } } + if (kwdefaults != Py_None && !PyDict_Check(kwdefaults)) { + PyErr_SetString(PyExc_TypeError, + "arg 6 (kwdefaults) must be None or dict"); + return NULL; + } /* check that the closure is well-formed */ nclosure = closure == Py_None ? 0 : PyTuple_GET_SIZE(closure); @@ -879,6 +1008,9 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, if (closure != Py_None) { newfunc->func_closure = Py_NewRef(closure); } + if (kwdefaults != Py_None) { + newfunc->func_kwdefaults = Py_NewRef(kwdefaults); + } return (PyObject *)newfunc; } @@ -896,6 +1028,7 @@ func_clear(PyFunctionObject *op) Py_CLEAR(op->func_dict); Py_CLEAR(op->func_closure); Py_CLEAR(op->func_annotations); + Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. @@ -952,6 +1085,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); return 0; @@ -1038,12 +1172,57 @@ functools_wraps(PyObject *wrapper, PyObject *wrapped) COPY_ATTR(__name__); COPY_ATTR(__qualname__); COPY_ATTR(__doc__); - COPY_ATTR(__annotations__); return 0; #undef COPY_ATTR } +// Used for wrapping __annotations__ and __annotate__ on classmethod +// and staticmethod objects. +static PyObject * +descriptor_get_wrapped_attribute(PyObject *wrapped, PyObject *dict, PyObject *name) +{ + PyObject *res; + if (PyDict_GetItemRef(dict, name, &res) < 0) { + return NULL; + } + if (res != NULL) { + return res; + } + res = PyObject_GetAttr(wrapped, name); + if (res == NULL) { + return NULL; + } + if (PyDict_SetItem(dict, name, res) < 0) { + Py_DECREF(res); + return NULL; + } + return res; +} + +static int +descriptor_set_wrapped_attribute(PyObject *dict, PyObject *name, PyObject *value, + char *type_name) +{ + if (value == NULL) { + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_AttributeError, + "'%.200s' object has no attribute '%U'", + type_name, name); + } + else { + return -1; + } + } + return 0; + } + else { + return PyDict_SetItem(dict, name, value); + } +} + /* Class method object */ @@ -1109,10 +1288,6 @@ cm_descr_get(PyObject *self, PyObject *obj, PyObject *type) } if (type == NULL) type = (PyObject *)(Py_TYPE(obj)); - if (Py_TYPE(cm->cm_callable)->tp_descr_get != NULL) { - return Py_TYPE(cm->cm_callable)->tp_descr_get(cm->cm_callable, type, - type); - } return PyMethod_New(cm->cm_callable, type); } @@ -1153,10 +1328,37 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) Py_RETURN_FALSE; } +static PyObject * +cm_get___annotations__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotations__)); +} + +static int +cm_set___annotations__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotations__), value, "classmethod"); +} + +static PyObject * +cm_get___annotate__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotate__)); +} + +static int +cm_set___annotate__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotate__), value, "classmethod"); +} + + static PyGetSetDef cm_getsetlist[] = { {"__isabstractmethod__", (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)cm_get___annotations__, (setter)cm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)cm_get___annotate__, (setter)cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1167,7 +1369,8 @@ cm_repr(classmethod *cm) } PyDoc_STRVAR(classmethod_doc, -"classmethod(function) -> method\n\ +"classmethod(function, /)\n\ +--\n\ \n\ Convert a function to be a class method.\n\ \n\ @@ -1348,10 +1551,36 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) Py_RETURN_FALSE; } +static PyObject * +sm_get___annotations__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotations__)); +} + +static int +sm_set___annotations__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotations__), value, "staticmethod"); +} + +static PyObject * +sm_get___annotate__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotate__)); +} + +static int +sm_set___annotate__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotate__), value, "staticmethod"); +} + static PyGetSetDef sm_getsetlist[] = { {"__isabstractmethod__", (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)sm_get___annotations__, (setter)sm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)sm_get___annotate__, (setter)sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1362,7 +1591,8 @@ sm_repr(staticmethod *sm) } PyDoc_STRVAR(staticmethod_doc, -"staticmethod(function) -> method\n\ +"staticmethod(function, /)\n\ +--\n\ \n\ Convert a function to be a static method.\n\ \n\ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index bf13ed3650bac5..96c96491501a2c 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -50,16 +51,15 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) } static int -ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) +ga_repr_item(PyUnicodeWriter *writer, PyObject *p) { PyObject *qualname = NULL; PyObject *module = NULL; - PyObject *r = NULL; int rc; if (p == Py_Ellipsis) { // The Ellipsis object - r = PyUnicode_FromString("..."); + rc = PyUnicodeWriter_WriteUTF8(writer, "...", 3); goto done; } @@ -70,17 +70,17 @@ ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) goto use_repr; } if (rc < 0) { - goto done; + goto error; } if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto done; + goto error; } if (qualname == NULL) { goto use_repr; } if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto done; + goto error; } if (module == NULL || module == Py_None) { goto use_repr; @@ -91,45 +91,42 @@ ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) _PyUnicode_EqualToASCIIString(module, "builtins")) { // builtins don't need a module name - r = PyObject_Str(qualname); + rc = PyUnicodeWriter_WriteStr(writer, qualname); goto done; } else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); + rc = PyUnicodeWriter_Format(writer, "%S.%S", module, qualname); goto done; } +error: + rc = -1; + goto done; + use_repr: - r = PyObject_Repr(p); + rc = PyUnicodeWriter_WriteRepr(writer, p); + goto done; done: Py_XDECREF(qualname); Py_XDECREF(module); - if (r == NULL) { - // error if any of the above PyObject_Repr/PyUnicode_From* fail - rc = -1; - } - else { - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); - } return rc; } static int -ga_repr_items_list(_PyUnicodeWriter *writer, PyObject *p) +ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) { assert(PyList_CheckExact(p)); Py_ssize_t len = PyList_GET_SIZE(p); - if (_PyUnicodeWriter_WriteASCIIString(writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { return -1; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { return -1; } } @@ -139,7 +136,7 @@ ga_repr_items_list(_PyUnicodeWriter *writer, PyObject *p) } } - if (_PyUnicodeWriter_WriteASCIIString(writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { return -1; } @@ -152,49 +149,55 @@ ga_repr(PyObject *self) gaobject *alias = (gaobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); + // Estimation based on the shortest format: "int[int, int, int]" + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 5) ? len * 5 : len; + estimate = 3 + 1 + estimate + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } if (alias->starred) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, "*", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '*') < 0) { goto error; } } - if (ga_repr_item(&writer, alias->origin) < 0) { + if (ga_repr_item(writer, alias->origin) < 0) { goto error; } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { goto error; } } PyObject *p = PyTuple_GET_ITEM(alias->args, i); if (PyList_CheckExact(p)) { // Looks like we are working with ParamSpec's list of type args: - if (ga_repr_items_list(&writer, p) < 0) { + if (ga_repr_items_list(writer, p) < 0) { goto error; } } - else if (ga_repr_item(&writer, p) < 0) { + else if (ga_repr_item(writer, p) < 0) { goto error; } } if (len == 0) { // for something like tuple[()] we should print a "()" - if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { goto error; } } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -536,6 +539,8 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } PyDoc_STRVAR(genericalias__doc__, +"GenericAlias(origin, args, /)\n" +"--\n\n" "Represent a PEP 585 generic type\n" "\n" "E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,)."); @@ -558,6 +563,10 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + if (res == NULL) { + Py_DECREF(newargs); + return NULL; + } ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); diff --git a/Objects/genobject.c b/Objects/genobject.c index 40033d10bea9ee..37b40530589d57 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,13 +6,15 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_freelist.h" // struct _Py_async_gen_freelist +#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "opcode.h" // SEND -#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); @@ -28,8 +30,7 @@ static const char *ASYNC_GEN_IGNORED_EXIT_MSG = /* Returns a borrowed reference */ static inline PyCodeObject * _PyGen_GetCode(PyGenObject *gen) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - return _PyFrame_GetCode(frame); + return _PyFrame_GetCode(&gen->gi_iframe); } PyCodeObject * @@ -40,20 +41,13 @@ PyGen_GetCode(PyGenObject *gen) { return res; } -static inline int -exc_state_traverse(_PyErr_StackItem *exc_state, visitproc visit, void *arg) -{ - Py_VISIT(exc_state->exc_value); - return 0; -} - static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) { Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); - if (gen->gi_frame_state < FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + if (gen->gi_frame_state != FRAME_CLEARED) { + _PyInterpreterFrame *frame = &gen->gi_iframe; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR); int err = _PyFrame_Traverse(frame, visit, arg); @@ -63,7 +57,8 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ - return exc_state_traverse(&gen->gi_exc_state, visit, arg); + Py_VISIT(gen->gi_exc_state.exc_value); + return 0; } void @@ -71,7 +66,7 @@ _PyGen_Finalize(PyObject *self) { PyGenObject *gen = (PyGenObject *)self; - if (gen->gi_frame_state >= FRAME_COMPLETED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { /* Generator isn't paused, so no need to close */ return; } @@ -144,8 +139,8 @@ gen_dealloc(PyGenObject *gen) and GC_Del. */ Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer); } - if (gen->gi_frame_state < FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + if (gen->gi_frame_state != FRAME_CLEARED) { + _PyInterpreterFrame *frame = &gen->gi_iframe; gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); @@ -167,8 +162,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc, int closing) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - PyObject *result; + _PyInterpreterFrame *frame = &gen->gi_iframe; *presult = NULL; if (gen->gi_frame_state == FRAME_CREATED && arg && arg != Py_None) { @@ -195,7 +189,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, PyErr_SetString(PyExc_ValueError, msg); return PYGEN_ERROR; } - if (gen->gi_frame_state >= FRAME_COMPLETED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { if (PyCoro_CheckExact(gen) && !closing) { /* `gen` is an exhausted coroutine: raise an error, except when called from gen_close(), which should @@ -213,10 +207,12 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, return PYGEN_ERROR; } - assert(gen->gi_frame_state < FRAME_EXECUTING); + assert((gen->gi_frame_state == FRAME_CREATED) || + FRAME_STATE_SUSPENDED(gen->gi_frame_state)); + /* Push arg onto the frame's value stack */ - result = arg ? arg : Py_None; - _PyFrame_StackPush(frame, Py_NewRef(result)); + PyObject *arg_obj = arg ? arg : Py_None; + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectNew(arg_obj)); _PyErr_StackItem *prev_exc_info = tstate->exc_info; gen->gi_exc_state.previous_item = prev_exc_info; @@ -229,7 +225,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, gen->gi_frame_state = FRAME_EXECUTING; EVAL_CALL_STAT_INC(EVAL_CALL_GENERATOR); - result = _PyEval_EvalFrame(tstate, frame, exc); + PyObject *result = _PyEval_EvalFrame(tstate, frame, exc); assert(tstate->exc_info == prev_exc_info); assert(gen->gi_exc_state.previous_item == NULL); assert(gen->gi_frame_state != FRAME_EXECUTING); @@ -238,7 +234,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ if (result) { - if (gen->gi_frame_state == FRAME_SUSPENDED) { + if (FRAME_STATE_SUSPENDED(gen->gi_frame_state)) { *presult = result; return PYGEN_NEXT; } @@ -333,60 +329,41 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } -static inline bool -is_yield(_Py_CODEUNIT *instr) -{ - return instr->op.code == YIELD_VALUE || instr->op.code == INSTRUMENTED_YIELD_VALUE; -} - PyObject * _PyGen_yf(PyGenObject *gen) { - PyObject *yf = NULL; - - if (gen->gi_frame_state < FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - - if (gen->gi_frame_state == FRAME_CREATED) { - /* Return immediately if the frame didn't start yet. SEND - always come after LOAD_CONST: a code object should not start - with SEND */ - assert(_PyCode_CODE(_PyGen_GetCode(gen))[0].op.code != SEND); - return NULL; - } - _Py_CODEUNIT next = frame->prev_instr[1]; - if (!is_resume(&next) || next.op.arg < 2) - { - /* Not in a yield from */ - return NULL; - } - yf = Py_NewRef(_PyFrame_StackPeek(frame)); + if (gen->gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { + _PyInterpreterFrame *frame = &gen->gi_iframe; + assert(is_resume(frame->instr_ptr)); + assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); } - - return yf; + return NULL; } static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyObject *yf = _PyGen_yf(gen); int err = 0; + if (gen->gi_frame_state == FRAME_CREATED) { gen->gi_frame_state = FRAME_COMPLETED; Py_RETURN_NONE; } - if (gen->gi_frame_state >= FRAME_COMPLETED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } + PyObject *yf = _PyGen_yf(gen); if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -394,17 +371,17 @@ gen_close(PyGenObject *gen, PyObject *args) gen->gi_frame_state = state; Py_DECREF(yf); } - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - /* It is possible for the previous instruction to not be a - * YIELD_VALUE if the debugger has changed the lineno. */ - if (err == 0 && is_yield(frame->prev_instr)) { - assert(is_resume(frame->prev_instr + 1)); - int exception_handler_depth = frame->prev_instr[0].op.code; - assert(exception_handler_depth > 0); + _PyInterpreterFrame *frame = &gen->gi_iframe; + if (is_resume(frame->instr_ptr)) { /* We can safely ignore the outermost try block - * as it automatically generated to handle + * as it is automatically generated to handle * StopIteration. */ - if (exception_handler_depth == 1) { + int oparg = frame->instr_ptr->op.arg; + if (oparg & RESUME_OPARG_DEPTH1_MASK) { + // RESUME after YIELD_VALUE and exception depth is 1 + assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); + gen->gi_frame_state = FRAME_COMPLETED; + _PyFrame_ClearLocals(&gen->gi_iframe); Py_RETURN_NONE; } } @@ -453,7 +430,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *yf = _PyGen_yf(gen); if (yf) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; PyObject *ret; int err; if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && @@ -749,7 +726,7 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) static PyObject * gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored)) { - return PyBool_FromLong(gen->gi_frame_state == FRAME_SUSPENDED); + return PyBool_FromLong(FRAME_STATE_SUSPENDED(gen->gi_frame_state)); } static PyObject * @@ -758,10 +735,10 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) { return NULL; } - if (gen->gi_frame_state == FRAME_CLEARED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } - return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((_PyInterpreterFrame *)gen->gi_iframe)); + return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(&gen->gi_iframe)); } static PyObject * @@ -821,6 +798,7 @@ static PyMethodDef gen_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -835,8 +813,7 @@ static PyAsyncMethods gen_as_async = { PyTypeObject PyGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "generator", /* tp_name */ - offsetof(PyGenObject, gi_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyGenObject, gi_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ (destructor)gen_dealloc, /* tp_dealloc */ @@ -970,7 +947,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, /* Copy the frame */ assert(f->f_frame->frame_obj == NULL); assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); @@ -1068,7 +1045,7 @@ _PyCoro_GetAwaitableIter(PyObject *o) } PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", + "'%.100s' object can't be awaited", ot->tp_name); return NULL; } @@ -1104,7 +1081,7 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) static PyObject * cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) { - if (coro->cr_frame_state == FRAME_SUSPENDED) { + if (FRAME_STATE_SUSPENDED(coro->cr_frame_state)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -1173,6 +1150,7 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -1186,8 +1164,7 @@ static PyAsyncMethods coro_as_async = { PyTypeObject PyCoro_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "coroutine", /* tp_name */ - offsetof(PyCoroObject, cr_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyCoroObject, cr_iframe.localsplus),/* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ (destructor)gen_dealloc, /* tp_dealloc */ @@ -1541,7 +1518,7 @@ ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) static PyObject * ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) { - if (ag->ag_frame_state == FRAME_SUSPENDED) { + if (FRAME_STATE_SUSPENDED(ag->ag_frame_state)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -1602,8 +1579,7 @@ static PyAsyncMethods async_gen_as_async = { PyTypeObject PyAsyncGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "async_generator", /* tp_name */ - offsetof(PyAsyncGenObject, ag_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyAsyncGenObject, ag_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ (destructor)gen_dealloc, /* tp_dealloc */ @@ -1653,12 +1629,19 @@ PyTypeObject PyAsyncGen_Type = { }; -#if _PyAsyncGen_MAXFREELIST > 0 -static struct _Py_async_gen_state * -get_async_gen_state(void) +#ifdef WITH_FREELISTS +static struct _Py_async_gen_freelist * +get_async_gen_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->async_gens; +} + +static struct _Py_async_gen_asend_freelist * +get_async_gen_asend_freelist(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->async_gen; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->async_gen_asends; } #endif @@ -1681,39 +1664,34 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) void -_PyAsyncGen_ClearFreeLists(PyInterpreterState *interp) +_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) { -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &interp->async_gen; +#ifdef WITH_FREELISTS + struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - while (state->value_numfree) { + while (freelist->numfree > 0) { _PyAsyncGenWrappedValue *o; - o = state->value_freelist[--state->value_numfree]; + o = freelist->items[--freelist->numfree]; assert(_PyAsyncGenWrappedValue_CheckExact(o)); PyObject_GC_Del(o); } - while (state->asend_numfree) { + struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; + + while (asend_freelist->numfree > 0) { PyAsyncGenASend *o; - o = state->asend_freelist[--state->asend_numfree]; + o = asend_freelist->items[--asend_freelist->numfree]; assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); PyObject_GC_Del(o); } -#endif -} -void -_PyAsyncGen_Fini(PyInterpreterState *interp) -{ - _PyAsyncGen_ClearFreeLists(interp); -#if defined(Py_DEBUG) && _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &interp->async_gen; - state->value_numfree = -1; - state->asend_numfree = -1; + if (is_finalization) { + freelist->numfree = -1; + asend_freelist->numfree = -1; + } #endif } - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { @@ -1757,15 +1735,12 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_asend_dealloc() must not be called after _PyAsyncGen_Fini() - assert(state->asend_numfree != -1); -#endif - if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) { +#ifdef WITH_FREELISTS + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); - state->asend_freelist[state->asend_numfree++] = o; + _PyGC_CLEAR_FINALIZED((PyObject *)o); + freelist->items[freelist->numfree++] = o; } else #endif @@ -1797,6 +1772,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) if (o->ags_state == AWAITABLE_STATE_INIT) { if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, "anext(): asynchronous generator is already running"); @@ -1840,10 +1816,24 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg return NULL; } + if (o->ags_state == AWAITABLE_STATE_INIT) { + if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + o->ags_state = AWAITABLE_STATE_ITER; + o->ags_gen->ag_running_async = 1; + } + result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { + o->ags_gen->ag_running_async = 0; o->ags_state = AWAITABLE_STATE_CLOSED; } @@ -1854,8 +1844,25 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) { - o->ags_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->ags_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_asend_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } static void @@ -1930,15 +1937,11 @@ static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { PyAsyncGenASend *o; -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_asend_new() must not be called after _PyAsyncGen_Fini() - assert(state->asend_numfree != -1); -#endif - if (state->asend_numfree) { - state->asend_numfree--; - o = state->asend_freelist[state->asend_numfree]; +#ifdef WITH_FREELISTS + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; _Py_NewReference((PyObject *)o); } else @@ -1969,15 +1972,11 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) { _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = get_async_gen_state(); -#ifdef Py_DEBUG - // async_gen_wrapped_val_dealloc() must not be called after _PyAsyncGen_Fini() - assert(state->value_numfree != -1); -#endif - if (state->value_numfree < _PyAsyncGen_MAXFREELIST) { +#ifdef WITH_FREELISTS + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); - state->value_freelist[state->value_numfree++] = o; + freelist->items[freelist->numfree++] = o; OBJECT_STAT_INC(to_freelist); } else @@ -2046,15 +2045,11 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) _PyAsyncGenWrappedValue *o; assert(val); -#if _PyAsyncGen_MAXFREELIST > 0 - struct _Py_async_gen_state *state = &tstate->interp->async_gen; -#ifdef Py_DEBUG - // _PyAsyncGenValueWrapperNew() must not be called after _PyAsyncGen_Fini() - assert(state->value_numfree != -1); -#endif - if (state->value_numfree) { - state->value_numfree--; - o = state->value_freelist[state->value_numfree]; +#ifdef WITH_FREELISTS + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; OBJECT_STAT_INC(from_freelist); assert(_PyAsyncGenWrappedValue_CheckExact(o)); _Py_NewReference((PyObject*)o); @@ -2113,7 +2108,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) return NULL; } - if (gen->gi_frame_state >= FRAME_COMPLETED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; @@ -2244,9 +2239,34 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } + if (o->agt_state == AWAITABLE_STATE_INIT) { + if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } + else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + o->agt_state = AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + } + retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { - return async_gen_unwrap_value(o->agt_gen, retval); + retval = async_gen_unwrap_value(o->agt_gen, retval); + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } + return retval; } else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { @@ -2256,6 +2276,10 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { @@ -2282,8 +2306,25 @@ async_gen_athrow_iternext(PyAsyncGenAThrow *o) static PyObject * async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) { - o->agt_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->agt_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_athrow_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c deleted file mode 100644 index 16e27b64c0c9c2..00000000000000 --- a/Objects/interpreteridobject.c +++ /dev/null @@ -1,294 +0,0 @@ -/* InterpreterID object */ - -#include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" - - -typedef struct interpid { - PyObject_HEAD - int64_t id; -} interpid; - -static interpid * -newinterpid(PyTypeObject *cls, int64_t id, int force) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp == NULL) { - if (force) { - PyErr_Clear(); - } - else { - return NULL; - } - } - - if (interp != NULL) { - if (_PyInterpreterState_IDIncref(interp) < 0) { - return NULL; - } - } - - interpid *self = PyObject_New(interpid, cls); - if (self == NULL) { - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - return NULL; - } - self->id = id; - - return self; -} - -static int -interp_id_converter(PyObject *arg, void *ptr) -{ - int64_t id; - if (PyObject_TypeCheck(arg, &PyInterpreterID_Type)) { - id = ((interpid *)arg)->id; - } - else if (_PyIndex_Check(arg)) { - id = PyLong_AsLongLong(arg); - if (id == -1 && PyErr_Occurred()) { - return 0; - } - if (id < 0) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", arg); - return 0; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return 0; - } - *(int64_t *)ptr = id; - return 1; -} - -static PyObject * -interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "force", NULL}; - int64_t id; - int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&|$p:InterpreterID.__init__", kwlist, - interp_id_converter, &id, &force)) { - return NULL; - } - - return (PyObject *)newinterpid(cls, id, force); -} - -static void -interpid_dealloc(PyObject *v) -{ - int64_t id = ((interpid *)v)->id; - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - else { - // already deleted - PyErr_Clear(); - } - Py_TYPE(v)->tp_free(v); -} - -static PyObject * -interpid_repr(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - const char *name = _PyType_Name(type); - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); -} - -static PyObject * -interpid_str(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%" PRId64 "", id->id); -} - -static PyObject * -interpid_int(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyLong_FromLongLong(id->id); -} - -static PyNumberMethods interpid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)interpid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)interpid_int, /* nb_index */ -}; - -static Py_hash_t -interpid_hash(PyObject *self) -{ - interpid *id = (interpid *)self; - PyObject *obj = PyLong_FromLongLong(id->id); - if (obj == NULL) { - return -1; - } - Py_hash_t hash = PyObject_Hash(obj); - Py_DECREF(obj); - return hash; -} - -static PyObject * -interpid_richcompare(PyObject *self, PyObject *other, int op) -{ - if (op != Py_EQ && op != Py_NE) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { - Py_RETURN_NOTIMPLEMENTED; - } - - interpid *id = (interpid *)self; - int equal; - if (PyObject_TypeCheck(other, &PyInterpreterID_Type)) { - interpid *otherid = (interpid *)other; - equal = (id->id == otherid->id); - } - else if (PyLong_CheckExact(other)) { - /* Fast path */ - int overflow; - long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow); - if (otherid == -1 && PyErr_Occurred()) { - return NULL; - } - equal = !overflow && (otherid >= 0) && (id->id == otherid); - } - else if (PyNumber_Check(other)) { - PyObject *pyid = PyLong_FromLongLong(id->id); - if (pyid == NULL) { - return NULL; - } - PyObject *res = PyObject_RichCompare(pyid, other, op); - Py_DECREF(pyid); - return res; - } - else { - Py_RETURN_NOTIMPLEMENTED; - } - - if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(interpid_doc, -"A interpreter ID identifies a interpreter and may be used as an int."); - -PyTypeObject PyInterpreterID_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "InterpreterID", /* tp_name */ - sizeof(interpid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)interpid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)interpid_repr, /* tp_repr */ - &interpid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - interpid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)interpid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - interpid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - interpid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - interpid_new, /* tp_new */ -}; - -PyObject *PyInterpreterID_New(int64_t id) -{ - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyObject * -PyInterpreterState_GetIDObject(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyInterpreterState * -PyInterpreterID_LookUp(PyObject *requested_id) -{ - int64_t id; - if (!interp_id_converter(requested_id, &id)) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} diff --git a/Objects/listobject.c b/Objects/listobject.c index ad1840b01226ec..f29f58dc25be04 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,12 +3,15 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_dict.h" // _PyDictViewObject +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list -#include "pycore_list.h" // struct _Py_list_state, _PyListIterObject +#include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_setobject.h" // _PySet_NextEntry() #include /*[clinic input] @@ -20,15 +23,59 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#if PyList_MAXFREELIST > 0 -static struct _Py_list_state * -get_list_state(void) +#ifdef WITH_FREELISTS +static struct _Py_list_freelist * +get_list_freelist(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->list; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + assert(freelists != NULL); + return &freelists->lists; } #endif +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + PyObject *ob_item[]; +} _PyListArray; + +static _PyListArray * +list_allocate_array(size_t capacity) +{ + if (capacity > PY_SSIZE_T_MAX/sizeof(PyObject*) - 1) { + return NULL; + } + _PyListArray *array = PyMem_Malloc(sizeof(_PyListArray) + capacity * sizeof(PyObject *)); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array; +} + +static Py_ssize_t +list_capacity(PyObject **items) +{ + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + return array->allocated; +} +#endif + +static void +free_list_items(PyObject** items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content @@ -46,8 +93,7 @@ get_list_state(void) static int list_resize(PyListObject *self, Py_ssize_t newsize) { - PyObject **items; - size_t new_allocated, num_allocated_bytes; + size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough @@ -79,9 +125,37 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(new_allocated); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + PyObject **old_items = self->ob_item; + if (self->ob_item) { + if (new_allocated < (size_t)allocated) { + target_bytes = new_allocated * sizeof(PyObject*); + } + else { + target_bytes = allocated * sizeof(PyObject*); + } + memcpy(array->ob_item, self->ob_item, target_bytes); + } + if (new_allocated > (size_t)allocated) { + memset(array->ob_item + allocated, 0, sizeof(PyObject *) * (new_allocated - allocated)); + } + _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); + self->allocated = new_allocated; + Py_SET_SIZE(self, newsize); + if (old_items != NULL) { + free_list_items(old_items, _PyObject_GC_IS_SHARED(self)); + } +#else + PyObject **items; if (new_allocated <= (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { - num_allocated_bytes = new_allocated * sizeof(PyObject *); - items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + target_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, target_bytes); } else { // integer overflow @@ -94,12 +168,14 @@ list_resize(PyListObject *self, Py_ssize_t newsize) self->ob_item = items; Py_SET_SIZE(self, newsize); self->allocated = new_allocated; +#endif return 0; } static int list_preallocate_exact(PyListObject *self, Py_ssize_t size) { + PyObject **items; assert(self->ob_item == NULL); assert(size > 0); @@ -109,36 +185,39 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) * allocated size up to the nearest even number. */ size = (size + 1) & ~(size_t)1; - PyObject **items = PyMem_New(PyObject*, size); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + items = array->ob_item; + memset(items, 0, size * sizeof(PyObject *)); +#else + items = PyMem_New(PyObject*, size); if (items == NULL) { PyErr_NoMemory(); return -1; } - self->ob_item = items; +#endif + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, items); self->allocated = size; return 0; } void -_PyList_ClearFreeList(PyInterpreterState *interp) +_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { -#if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - while (state->numfree) { - PyListObject *op = state->free_list[--state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_list_freelist *state = &freelists->lists; + while (state->numfree > 0) { + PyListObject *op = state->items[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } -#endif -} - -void -_PyList_Fini(PyInterpreterState *interp) -{ - _PyList_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &interp->list; - state->numfree = -1; + if (is_finalization) { + state->numfree = -1; + } #endif } @@ -146,11 +225,11 @@ _PyList_Fini(PyInterpreterState *interp) void _PyList_DebugMallocStats(FILE *out) { -#if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = get_list_state(); +#ifdef WITH_FREELISTS + struct _Py_list_freelist *list_freelist = get_list_freelist(); _PyDebugAllocatorStats(out, "free PyListObject", - state->numfree, sizeof(PyListObject)); + list_freelist->numfree, sizeof(PyListObject)); #endif } @@ -164,15 +243,11 @@ PyList_New(Py_ssize_t size) return NULL; } -#if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = get_list_state(); -#ifdef Py_DEBUG - // PyList_New() must not be called after _PyList_Fini() - assert(state->numfree != -1); -#endif - if (PyList_MAXFREELIST && state->numfree) { - state->numfree--; - op = state->free_list[state->numfree]; +#ifdef WITH_FREELISTS + struct _Py_list_freelist *list_freelist = get_list_freelist(); + if (PyList_MAXFREELIST && list_freelist->numfree > 0) { + list_freelist->numfree--; + op = list_freelist->items[list_freelist->numfree]; OBJECT_STAT_INC(from_freelist); _Py_NewReference((PyObject *)op); } @@ -188,7 +263,17 @@ PyList_New(Py_ssize_t size) op->ob_item = NULL; } else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + memset(&array->ob_item, 0, size * sizeof(PyObject *)); + op->ob_item = array->ob_item; +#else op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *)); +#endif if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -209,11 +294,20 @@ list_new_prealloc(Py_ssize_t size) return NULL; } assert(op->ob_item == NULL); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; +#else op->ob_item = PyMem_New(PyObject *, size); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } +#endif op->allocated = size; return (PyObject *) op; } @@ -225,8 +319,9 @@ PyList_Size(PyObject *op) PyErr_BadInternalCall(); return -1; } - else - return Py_SIZE(op); + else { + return PyList_GET_SIZE(op); + } } static inline int @@ -242,6 +337,67 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } +#ifdef Py_GIL_DISABLED + +static PyObject * +list_item_impl(PyListObject *self, Py_ssize_t idx) +{ + PyObject *item = NULL; + Py_BEGIN_CRITICAL_SECTION(self); + if (!_PyObject_GC_IS_SHARED(self)) { + _PyObject_GC_SET_SHARED(self); + } + Py_ssize_t size = Py_SIZE(self); + if (!valid_index(idx, size)) { + goto exit; + } +#ifdef Py_GIL_DISABLED + item = _Py_NewRefWithLock(self->ob_item[idx]); +#else + item = Py_NewRef(self->ob_item[idx]); +#endif +exit: + Py_END_CRITICAL_SECTION(); + return item; +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return list_item_impl(op, i); + } + // Need atomic operation for the getting size. + Py_ssize_t size = PyList_GET_SIZE(op); + if (!valid_index(i, size)) { + return NULL; + } + PyObject **ob_item = _Py_atomic_load_ptr(&op->ob_item); + if (ob_item == NULL) { + return NULL; + } + Py_ssize_t cap = list_capacity(ob_item); + assert(cap != -1 && cap >= size); + if (!valid_index(i, cap)) { + return NULL; + } + PyObject *item = _Py_TryXGetRef(&ob_item[i]); + if (item == NULL) { + return list_item_impl(op, i); + } + return item; +} +#else +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} +#endif + PyObject * PyList_GetItem(PyObject *op, Py_ssize_t i) { @@ -257,6 +413,22 @@ PyList_GetItem(PyObject *op, Py_ssize_t i) return ((PyListObject *)op) -> ob_item[i]; } +PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t i) +{ + if (!PyList_Check(op)) { + PyErr_SetString(PyExc_TypeError, "expected a list"); + return NULL; + } + PyObject *item = list_get_item_ref((PyListObject *)op, i); + if (item == NULL) { + _Py_DECLARE_STR(list_err, "list index out of range"); + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } + return item; +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) @@ -267,15 +439,22 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, PyErr_BadInternalCall(); return -1; } - if (!valid_index(i, Py_SIZE(op))) { + int ret; + PyListObject *self = ((PyListObject *)op); + Py_BEGIN_CRITICAL_SECTION(self); + if (!valid_index(i, Py_SIZE(self))) { Py_XDECREF(newitem); PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); - return -1; + ret = -1; + goto end; } - p = ((PyListObject *)op) -> ob_item + i; + p = self->ob_item + i; Py_XSETREF(*p, newitem); - return 0; + ret = 0; +end:; + Py_END_CRITICAL_SECTION(); + return ret; } static int @@ -313,20 +492,25 @@ PyList_Insert(PyObject *op, Py_ssize_t where, PyObject *newitem) PyErr_BadInternalCall(); return -1; } - return ins1((PyListObject *)op, where, newitem); + PyListObject *self = (PyListObject *)op; + int err; + Py_BEGIN_CRITICAL_SECTION(self); + err = ins1(self, where, newitem); + Py_END_CRITICAL_SECTION(); + return err; } /* internal, used by _PyList_AppendTakeRef */ int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) { - Py_ssize_t len = PyList_GET_SIZE(self); + Py_ssize_t len = Py_SIZE(self); assert(self->allocated == -1 || self->allocated == len); if (list_resize(self, len + 1) < 0) { Py_DECREF(newitem); return -1; } - PyList_SET_ITEM(self, len, newitem); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], newitem); return 0; } @@ -334,7 +518,11 @@ int PyList_Append(PyObject *op, PyObject *newitem) { if (PyList_Check(op) && (newitem != NULL)) { - return _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); + Py_END_CRITICAL_SECTION(); + return ret; } PyErr_BadInternalCall(); return -1; @@ -343,8 +531,9 @@ PyList_Append(PyObject *op, PyObject *newitem) /* Methods */ static void -list_dealloc(PyListObject *op) +list_dealloc(PyObject *self) { + PyListObject *op = (PyListObject *)self; Py_ssize_t i; PyObject_GC_UnTrack(op); Py_TRASHCAN_BEGIN(op, list_dealloc) @@ -357,16 +546,12 @@ list_dealloc(PyListObject *op) while (--i >= 0) { Py_XDECREF(op->ob_item[i]); } - PyMem_Free(op->ob_item); + free_list_items(op->ob_item, false); } -#if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = get_list_state(); -#ifdef Py_DEBUG - // list_dealloc() must not be called after _PyList_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyList_MAXFREELIST && PyList_CheckExact(op)) { - state->free_list[state->numfree++] = op; +#ifdef WITH_FREELISTS + struct _Py_list_freelist *list_freelist = get_list_freelist(); + if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { + list_freelist->items[list_freelist->numfree++] = op; OBJECT_STAT_INC(to_freelist); } else @@ -378,17 +563,11 @@ list_dealloc(PyListObject *op) } static PyObject * -list_repr(PyListObject *v) +list_repr_impl(PyListObject *v) { - Py_ssize_t i; PyObject *s; _PyUnicodeWriter writer; - - if (Py_SIZE(v) == 0) { - return PyUnicode_FromString("[]"); - } - - i = Py_ReprEnter((PyObject*)v); + Py_ssize_t i = Py_ReprEnter((PyObject*)v); if (i != 0) { return i > 0 ? PyUnicode_FromString("[...]") : NULL; } @@ -433,40 +612,68 @@ list_repr(PyListObject *v) return NULL; } +static PyObject * +list_repr(PyObject *self) +{ + if (PyList_GET_SIZE(self) == 0) { + return PyUnicode_FromString("[]"); + } + PyListObject *v = (PyListObject *)self; + PyObject *ret = NULL; + Py_BEGIN_CRITICAL_SECTION(v); + ret = list_repr_impl(v); + Py_END_CRITICAL_SECTION(); + return ret; +} + static Py_ssize_t -list_length(PyListObject *a) +list_length(PyObject *a) { - return Py_SIZE(a); + return PyList_GET_SIZE(a); } static int -list_contains(PyListObject *a, PyObject *el) +list_contains(PyObject *aa, PyObject *el) { - PyObject *item; - Py_ssize_t i; - int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) { - item = PyList_GET_ITEM(a, i); - Py_INCREF(item); - cmp = PyObject_RichCompareBool(item, el, Py_EQ); + for (Py_ssize_t i = 0; ; i++) { + PyObject *item = list_get_item_ref((PyListObject *)aa, i); + if (item == NULL) { + // out-of-bounds + return 0; + } + int cmp = PyObject_RichCompareBool(item, el, Py_EQ); Py_DECREF(item); + if (cmp != 0) { + return cmp; + } } - return cmp; + return 0; } static PyObject * -list_item(PyListObject *a, Py_ssize_t i) +list_item(PyObject *aa, Py_ssize_t i) { - if (!valid_index(i, Py_SIZE(a))) { + PyListObject *a = (PyListObject *)aa; + if (!valid_index(i, PyList_GET_SIZE(a))) { + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; + } + PyObject *item; +#ifdef Py_GIL_DISABLED + item = list_get_item_ref(a, i); + if (item == NULL) { PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } - return Py_NewRef(a->ob_item[i]); +#else + item = Py_NewRef(a->ob_item[i]); +#endif + return item; } static PyObject * -list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +list_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { PyListObject *np; PyObject **src, **dest; @@ -496,6 +703,8 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) PyErr_BadInternalCall(); return NULL; } + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(a); if (ilow < 0) { ilow = 0; } @@ -508,23 +717,18 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - return list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice_lock_held((PyListObject *)a, ilow, ihigh); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -list_concat(PyListObject *a, PyObject *bb) +list_concat_lock_held(PyListObject *a, PyListObject *b) { Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; PyListObject *np; - if (!PyList_Check(bb)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate list (not \"%.200s\") to list", - Py_TYPE(bb)->tp_name); - return NULL; - } -#define b ((PyListObject *)bb) assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { @@ -548,11 +752,28 @@ list_concat(PyListObject *a, PyObject *bb) } Py_SET_SIZE(np, size); return (PyObject *)np; -#undef b } static PyObject * -list_repeat(PyListObject *a, Py_ssize_t n) +list_concat(PyObject *aa, PyObject *bb) +{ + if (!PyList_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only concatenate list (not \"%.200s\") to list", + Py_TYPE(bb)->tp_name); + return NULL; + } + PyListObject *a = (PyListObject *)aa; + PyListObject *b = (PyListObject *)bb; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(a, b); + ret = list_concat_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +list_repeat_lock_held(PyListObject *a, Py_ssize_t n) { const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) @@ -592,26 +813,54 @@ list_repeat(PyListObject *a, Py_ssize_t n) return (PyObject *) np; } +static PyObject * +list_repeat(PyObject *aa, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_repeat_lock_held(a, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + +static void +list_clear_impl(PyListObject *a, bool is_resize) +{ + PyObject **items = a->ob_item; + if (items == NULL) { + return; + } + + /* Because XDECREF can recursively invoke operations on + this list, we make it empty first. */ + Py_ssize_t i = Py_SIZE(a); + Py_SET_SIZE(a, 0); + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item, NULL); + a->allocated = 0; + while (--i >= 0) { + Py_XDECREF(items[i]); + } +#ifdef Py_GIL_DISABLED + bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); +#else + bool use_qsbr = false; +#endif + free_list_items(items, use_qsbr); + // Note that there is no guarantee that the list is actually empty + // at this point, because XDECREF may have populated it indirectly again! +} + +static void +list_clear(PyListObject *a) +{ + list_clear_impl(a, true); +} + static int -_list_clear(PyListObject *a) +list_clear_slot(PyObject *self) { - Py_ssize_t i; - PyObject **item = a->ob_item; - if (item != NULL) { - /* Because XDECREF can recursively invoke operations on - this list, we make it empty first. */ - i = Py_SIZE(a); - Py_SET_SIZE(a, 0); - a->ob_item = NULL; - a->allocated = 0; - while (--i >= 0) { - Py_XDECREF(item[i]); - } - PyMem_Free(item); - } - /* Never fails; the return value can be ignored. - Note that there is no guarantee that the list is actually empty - at this point, because XDECREF may have populated it again! */ + list_clear_impl((PyListObject *)self, false); return 0; } @@ -622,7 +871,7 @@ _list_clear(PyListObject *a) * guaranteed the call cannot fail. */ static int -list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { /* Because [X]DECREF can recursively invoke list operations on this list, we must postpone all [X]DECREF activity until @@ -645,15 +894,6 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) if (v == NULL) n = 0; else { - if (a == b) { - /* Special case "a[i:j] = a" -- copy b first */ - v = list_slice(b, 0, Py_SIZE(b)); - if (v == NULL) - return result; - result = list_ass_slice(a, ilow, ihigh, v); - Py_DECREF(v); - return result; - } v_as_SF = PySequence_Fast(v, "can only assign an iterable"); if(v_as_SF == NULL) goto Error; @@ -675,7 +915,8 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) d = n - norig; if (Py_SIZE(a) + d == 0) { Py_XDECREF(v_as_SF); - return _list_clear(a); + list_clear(a); + return 0; } item = a->ob_item; /* recycle the items that we are about to remove */ @@ -726,6 +967,34 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) #undef b } +static int +list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +{ + int ret; + if (a == (PyListObject *)v) { + Py_BEGIN_CRITICAL_SECTION(a); + Py_ssize_t n = PyList_GET_SIZE(a); + PyObject *copy = list_slice_lock_held(a, 0, n); + if (copy == NULL) { + return -1; + } + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); + Py_END_CRITICAL_SECTION(); + } + else if (v != NULL && PyList_CheckExact(v)) { + Py_BEGIN_CRITICAL_SECTION2(a, v); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + int PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { @@ -736,26 +1005,28 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) return list_ass_slice((PyListObject *)a, ilow, ihigh, v); } -static PyObject * -list_inplace_repeat(PyListObject *self, Py_ssize_t n) +static int +list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { - return Py_NewRef(self); + return 0; } if (n < 1) { - (void)_list_clear(self); - return Py_NewRef(self); + list_clear(self); + return 0; } if (input_size > PY_SSIZE_T_MAX / n) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + return -1; } Py_ssize_t output_size = input_size * n; - if (list_resize(self, output_size) < 0) - return NULL; + if (list_resize(self, output_size) < 0) { + return -1; + } PyObject **items = self->ob_item; for (Py_ssize_t j = 0; j < input_size; j++) { @@ -763,25 +1034,61 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) } _Py_memory_repeat((char *)items, sizeof(PyObject *)*output_size, sizeof(PyObject *)*input_size); + return 0; +} - return Py_NewRef(self); +static PyObject * +list_inplace_repeat(PyObject *_self, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *self = (PyListObject *) _self; + Py_BEGIN_CRITICAL_SECTION(self); + if (list_inplace_repeat_lock_held(self, n) < 0) { + ret = NULL; + } + else { + ret = Py_NewRef(self); + } + Py_END_CRITICAL_SECTION(); + return ret; } static int -list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) +list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) { if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); return -1; } - if (v == NULL) - return list_ass_slice(a, i, i+1, v); - Py_SETREF(a->ob_item[i], Py_NewRef(v)); + PyObject *tmp = a->ob_item[i]; + if (v == NULL) { + Py_ssize_t size = Py_SIZE(a); + for (Py_ssize_t idx = i; idx < size - 1; idx++) { + FT_ATOMIC_STORE_PTR_RELAXED(a->ob_item[idx], a->ob_item[idx + 1]); + } + Py_SET_SIZE(a, size - 1); + } + else { + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[i], Py_NewRef(v)); + } + Py_DECREF(tmp); return 0; } +static int +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +{ + int ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_item_lock_held(a, i, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] +@critical_section list.insert index: Py_ssize_t @@ -793,28 +1100,31 @@ Insert object before index. static PyObject * list_insert_impl(PyListObject *self, Py_ssize_t index, PyObject *object) -/*[clinic end generated code: output=7f35e32f60c8cb78 input=858514cf894c7eab]*/ +/*[clinic end generated code: output=7f35e32f60c8cb78 input=b1987ca998a4ae2d]*/ { - if (ins1(self, index, object) == 0) + if (ins1(self, index, object) == 0) { Py_RETURN_NONE; + } return NULL; } /*[clinic input] -list.clear +@critical_section +list.clear as py_list_clear Remove all items from list. [clinic start generated code]*/ static PyObject * -list_clear_impl(PyListObject *self) -/*[clinic end generated code: output=67a1896c01f74362 input=ca3c1646856742f6]*/ +py_list_clear_impl(PyListObject *self) +/*[clinic end generated code: output=83726743807e3518 input=e285b7f09051a9ba]*/ { - _list_clear(self); + list_clear(self); Py_RETURN_NONE; } /*[clinic input] +@critical_section list.copy Return a shallow copy of the list. @@ -822,12 +1132,13 @@ Return a shallow copy of the list. static PyObject * list_copy_impl(PyListObject *self) -/*[clinic end generated code: output=ec6b72d6209d418e input=6453ab159e84771f]*/ +/*[clinic end generated code: output=ec6b72d6209d418e input=81c54b0c7bb4f73d]*/ { - return list_slice(self, 0, Py_SIZE(self)); + return list_slice_lock_held(self, 0, Py_SIZE(self)); } /*[clinic input] +@critical_section list.append object: object @@ -837,8 +1148,8 @@ Append object to the end of the list. [clinic start generated code]*/ static PyObject * -list_append(PyListObject *self, PyObject *object) -/*[clinic end generated code: output=7c096003a29c0eae input=43a3fe48a7066e91]*/ +list_append_impl(PyListObject *self, PyObject *object) +/*[clinic end generated code: output=78423561d92ed405 input=122b0853de54004f]*/ { if (_PyList_AppendTakeRef(self, Py_NewRef(object)) < 0) { return NULL; @@ -846,83 +1157,61 @@ list_append(PyListObject *self, PyObject *object) Py_RETURN_NONE; } -/*[clinic input] -list.extend - - iterable: object - / - -Extend list by appending elements from the iterable. -[clinic start generated code]*/ - -static PyObject * -list_extend(PyListObject *self, PyObject *iterable) -/*[clinic end generated code: output=630fb3bca0c8e789 input=9ec5ba3a81be3a4d]*/ +static int +list_extend_fast(PyListObject *self, PyObject *iterable) { - PyObject *it; /* iter(v) */ - Py_ssize_t m; /* size of self */ - Py_ssize_t n; /* guess for size of iterable */ - Py_ssize_t i; - PyObject *(*iternext)(PyObject *); + Py_ssize_t n = PySequence_Fast_GET_SIZE(iterable); + if (n == 0) { + /* short circuit when iterable is empty */ + return 0; + } - /* Special cases: - 1) lists and tuples which can use PySequence_Fast ops - 2) extending self to self requires making a copy first - */ - if (PyList_CheckExact(iterable) || PyTuple_CheckExact(iterable) || - (PyObject *)self == iterable) { - PyObject **src, **dest; - iterable = PySequence_Fast(iterable, "argument must be iterable"); - if (!iterable) - return NULL; - n = PySequence_Fast_GET_SIZE(iterable); - if (n == 0) { - /* short circuit when iterable is empty */ - Py_DECREF(iterable); - Py_RETURN_NONE; - } - m = Py_SIZE(self); - /* It should not be possible to allocate a list large enough to cause - an overflow on any relevant platform */ - assert(m < PY_SSIZE_T_MAX - n); - if (self->ob_item == NULL) { - if (list_preallocate_exact(self, n) < 0) { - return NULL; - } - Py_SET_SIZE(self, n); - } - else if (list_resize(self, m + n) < 0) { - Py_DECREF(iterable); - return NULL; - } - /* note that we may still have self == iterable here for the - * situation a.extend(a), but the following code works - * in that case too. Just make sure to resize self - * before calling PySequence_Fast_ITEMS. - */ - /* populate the end of self with iterable's items */ - src = PySequence_Fast_ITEMS(iterable); - dest = self->ob_item + m; - for (i = 0; i < n; i++) { - PyObject *o = src[i]; - dest[i] = Py_NewRef(o); + Py_ssize_t m = Py_SIZE(self); + // It should not be possible to allocate a list large enough to cause + // an overflow on any relevant platform. + assert(m < PY_SSIZE_T_MAX - n); + if (self->ob_item == NULL) { + if (list_preallocate_exact(self, n) < 0) { + return -1; } - Py_DECREF(iterable); - Py_RETURN_NONE; + Py_SET_SIZE(self, n); + } + else if (list_resize(self, m + n) < 0) { + return -1; } - it = PyObject_GetIter(iterable); - if (it == NULL) - return NULL; - iternext = *Py_TYPE(it)->tp_iternext; + // note that we may still have self == iterable here for the + // situation a.extend(a), but the following code works + // in that case too. Just make sure to resize self + // before calling PySequence_Fast_ITEMS. + // + // populate the end of self with iterable's items. + PyObject **src = PySequence_Fast_ITEMS(iterable); + PyObject **dest = self->ob_item + m; + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *o = src[i]; + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(o)); + } + return 0; +} + +static int +list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) +{ + PyObject *it = PyObject_GetIter(iterable); + if (it == NULL) { + return -1; + } + PyObject *(*iternext)(PyObject *) = *Py_TYPE(it)->tp_iternext; /* Guess a result list size. */ - n = PyObject_LengthHint(iterable, 8); + Py_ssize_t n = PyObject_LengthHint(iterable, 8); if (n < 0) { Py_DECREF(it); - return NULL; + return -1; } - m = Py_SIZE(self); + + Py_ssize_t m = Py_SIZE(self); if (m > PY_SSIZE_T_MAX - n) { /* m + n overflowed; on the chance that n lied, and there really * is enough room, ignore it. If n was telling the truth, we'll @@ -935,8 +1224,10 @@ list_extend(PyListObject *self, PyObject *iterable) } else { /* Make room. */ - if (list_resize(self, m + n) < 0) + if (list_resize(self, m + n) < 0) { goto error; + } + /* Make the list sane again. */ Py_SET_SIZE(self, m); } @@ -953,11 +1244,11 @@ list_extend(PyListObject *self, PyObject *iterable) } break; } + if (Py_SIZE(self) < self->allocated) { - /* steals ref */ Py_ssize_t len = Py_SIZE(self); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], item); // steals item ref Py_SET_SIZE(self, len + 1); - PyList_SET_ITEM(self, len, item); } else { if (_PyList_AppendTakeRef(self, item) < 0) @@ -972,11 +1263,173 @@ list_extend(PyListObject *self, PyObject *iterable) } Py_DECREF(it); - Py_RETURN_NONE; + return 0; error: Py_DECREF(it); - return NULL; + return -1; +} + +static int +list_extend_lock_held(PyListObject *self, PyObject *iterable) +{ + PyObject *seq = PySequence_Fast(iterable, "argument must be iterable"); + if (!seq) { + return -1; + } + + int res = list_extend_fast(self, seq); + Py_DECREF(seq); + return res; +} + +static int +list_extend_set(PyListObject *self, PySetObject *other) +{ + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PySet_GET_SIZE(other); + if (list_resize(self, m + n) < 0) { + return -1; + } + /* populate the end of self with iterable's items */ + Py_ssize_t setpos = 0; + Py_hash_t hash; + PyObject *key; + PyObject **dest = self->ob_item + m; + while (_PySet_NextEntryRef((PyObject *)other, &setpos, &key, &hash)) { + FT_ATOMIC_STORE_PTR_RELEASE(*dest, key); + dest++; + } + Py_SET_SIZE(self, m + n); + return 0; +} + +static int +list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) +{ + // which_item: 0 for keys and 1 for values + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + PyObject *keyvalue[2]; + while (_PyDict_Next((PyObject *)dict, &pos, &keyvalue[0], &keyvalue[1], NULL)) { + PyObject *obj = keyvalue[which_item]; + Py_INCREF(obj); + FT_ATOMIC_STORE_PTR_RELEASE(*dest, obj); + dest++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + +static int +list_extend_dictitems(PyListObject *self, PyDictObject *dict) +{ + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + Py_ssize_t i = 0; + PyObject *key, *value; + while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { + PyObject *item = PyTuple_Pack(2, key, value); + if (item == NULL) { + Py_SET_SIZE(self, m + i); + return -1; + } + FT_ATOMIC_STORE_PTR_RELEASE(*dest, item); + dest++; + i++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + +static int +_list_extend(PyListObject *self, PyObject *iterable) +{ + // Special case: + // lists and tuples which can use PySequence_Fast ops + int res = -1; + if ((PyObject *)self == iterable) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_inplace_repeat_lock_held(self, 2); + Py_END_CRITICAL_SECTION(); + } + else if (PyList_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyTuple_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyAnySet_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_set(self, (PySetObject *)iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyDict_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_dict(self, (PyDictObject *)iterable, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictValues_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 1 /*values*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictItems_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dictitems(self, dict); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_iter_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + return res; +} + +/*[clinic input] +list.extend as list_extend + + iterable: object + / + +Extend list by appending elements from the iterable. +[clinic start generated code]*/ + +static PyObject * +list_extend(PyListObject *self, PyObject *iterable) +/*[clinic end generated code: output=630fb3bca0c8e789 input=979da7597a515791]*/ +{ + if (_list_extend(self, iterable) < 0) { + return NULL; + } + Py_RETURN_NONE; } PyObject * @@ -985,19 +1438,41 @@ _PyList_Extend(PyListObject *self, PyObject *iterable) return list_extend(self, iterable); } -static PyObject * -list_inplace_concat(PyListObject *self, PyObject *other) +int +PyList_Extend(PyObject *self, PyObject *iterable) +{ + if (!PyList_Check(self)) { + PyErr_BadInternalCall(); + return -1; + } + return _list_extend((PyListObject*)self, iterable); +} + + +int +PyList_Clear(PyObject *self) { - PyObject *result; + if (!PyList_Check(self)) { + PyErr_BadInternalCall(); + return -1; + } + list_clear((PyListObject*)self); + return 0; +} - result = list_extend(self, other); - if (result == NULL) - return result; - Py_DECREF(result); + +static PyObject * +list_inplace_concat(PyObject *_self, PyObject *other) +{ + PyListObject *self = (PyListObject *)_self; + if (_list_extend(self, other) < 0) { + return NULL; + } return Py_NewRef(self); } /*[clinic input] +@critical_section list.pop index: Py_ssize_t = -1 @@ -1010,7 +1485,7 @@ Raises IndexError if list is empty or index is out of range. static PyObject * list_pop_impl(PyListObject *self, Py_ssize_t index) -/*[clinic end generated code: output=6bd69dcb3f17eca8 input=b83675976f329e6f]*/ +/*[clinic end generated code: output=6bd69dcb3f17eca8 input=c269141068ae4b8f]*/ { PyObject *v; int status; @@ -1032,7 +1507,8 @@ list_pop_impl(PyListObject *self, Py_ssize_t index) const Py_ssize_t size_after_pop = Py_SIZE(self) - 1; if (size_after_pop == 0) { Py_INCREF(v); - status = _list_clear(self); + list_clear(self); + status = 0; } else { if ((size_after_pop - index) > 0) { @@ -1165,6 +1641,15 @@ sortslice_advance(sortslice *slice, Py_ssize_t n) /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 +/* The largest value of minrun. This must be a power of 2, and >= 1, so that + * the compute_minrun() algorithm guarantees to return a result no larger than + * this, + */ +#define MAX_MINRUN 64 +#if ((MAX_MINRUN) < 1) || ((MAX_MINRUN) & ((MAX_MINRUN) - 1)) +#error "MAX_MINRUN must be a power of 2, and >= 1" +#endif + /* One MergeState exists on the stack per invocation of mergesort. It's just * a convenient way to pass state around among the helper functions. */ @@ -1222,123 +1707,248 @@ struct s_MergeState { int (*tuple_elem_compare)(PyObject *, PyObject *, MergeState *); }; -/* binarysort is the best method for sorting small arrays: it does - few compares, but can do data movement quadratic in the number of - elements. - [lo, hi) is a contiguous slice of a list, and is sorted via - binary insertion. This sort is stable. - On entry, must have lo <= start <= hi, and that [lo, start) is already - sorted (pass start == lo if you don't know!). - If islt() complains return -1, else 0. +/* binarysort is the best method for sorting small arrays: it does few + compares, but can do data movement quadratic in the number of elements. + ss->keys is viewed as an array of n kays, a[:n]. a[:ok] is already sorted. + Pass ok = 0 (or 1) if you don't know. + It's sorted in-place, by a stable binary insertion sort. If ss->values + isn't NULL, it's permuted in lockstap with ss->keys. + On entry, must have n >= 1, and 0 <= ok <= n <= MAX_MINRUN. + Return -1 if comparison raises an exception, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). */ static int -binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) +binarysort(MergeState *ms, const sortslice *ss, Py_ssize_t n, Py_ssize_t ok) { - Py_ssize_t k; - PyObject **l, **p, **r; + Py_ssize_t k; /* for IFLT macro expansion */ + PyObject ** const a = ss->keys; + PyObject ** const v = ss->values; + const bool has_values = v != NULL; PyObject *pivot; - - assert(lo.keys <= start && start <= hi); - /* assert [lo, start) is sorted */ - if (lo.keys == start) - ++start; - for (; start < hi; ++start) { - /* set l to where *start belongs */ - l = lo.keys; - r = start; - pivot = *r; - /* Invariants: - * pivot >= all in [lo, l). - * pivot < all in [r, start). - * The second is vacuously true at the start. + Py_ssize_t M; + + assert(0 <= ok && ok <= n && 1 <= n && n <= MAX_MINRUN); + /* assert a[:ok] is sorted */ + if (! ok) + ++ok; + /* Regular insertion sort has average- and worst-case O(n**2) cost + for both # of comparisons and number of bytes moved. But its branches + are highly predictable, and it loves sorted input (n-1 compares and no + data movement). This is significant in cases like sortperf.py's %sort, + where an out-of-order element near the start of a run is moved into + place slowly but then the remaining elements up to length minrun are + generally at worst one slot away from their correct position (so only + need 1 or 2 commpares to resolve). If comparisons are very fast (such + as for a list of Python floats), the simple inner loop leaves it + very competitive with binary insertion, despite that it does + significantly more compares overall on random data. + + Binary insertion sort has worst, average, and best case O(n log n) + cost for # of comparisons, but worst and average case O(n**2) cost + for data movement. The more expensive comparisons, the more important + the comparison advantage. But its branches are less predictable the + more "randomish" the data, and that's so significant its worst case + in real life is random input rather than reverse-ordered (which does + about twice the data movement than random input does). + + Note that the number of bytes moved doesn't seem to matter. MAX_MINRUN + of 64 is so small that the key and value pointers all fit in a corner + of L1 cache, and moving things around in that is very fast. */ +#if 0 // ordinary insertion sort. + PyObject * vpivot = NULL; + for (; ok < n; ++ok) { + pivot = a[ok]; + if (has_values) + vpivot = v[ok]; + for (M = ok - 1; M >= 0; --M) { + k = ISLT(pivot, a[M]); + if (k < 0) { + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + goto fail; + } + else if (k) { + a[M + 1] = a[M]; + if (has_values) + v[M + 1] = v[M]; + } + else + break; + } + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + } +#else // binary insertion sort + Py_ssize_t L, R; + for (; ok < n; ++ok) { + /* set L to where a[ok] belongs */ + L = 0; + R = ok; + pivot = a[ok]; + /* Slice invariants. vacuously true at the start: + * all a[0:L] <= pivot + * all a[L:R] unknown + * all a[R:ok] > pivot */ - assert(l < r); + assert(L < R); do { - p = l + ((r - l) >> 1); - IFLT(pivot, *p) - r = p; + /* don't do silly ;-) things to prevent overflow when finding + the midpoint; L and R are very far from filling a Py_ssize_t */ + M = (L + R) >> 1; +#if 1 // straightforward, but highly unpredictable branch on random data + IFLT(pivot, a[M]) + R = M; else - l = p+1; - } while (l < r); - assert(l == r); - /* The invariants still hold, so pivot >= all in [lo, l) and - pivot < all in [l, start), so pivot belongs at l. Note - that if there are elements equal to pivot, l points to the - first slot after them -- that's why this sort is stable. - Slide over to make room. - Caution: using memmove is much slower under MSVC 5; - we're not usually moving many slots. */ - for (p = start; p > l; --p) - *p = *(p-1); - *l = pivot; - if (lo.values != NULL) { - Py_ssize_t offset = lo.values - lo.keys; - p = start + offset; - pivot = *p; - l += offset; - for (p = start + offset; p > l; --p) - *p = *(p-1); - *l = pivot; + L = M + 1; +#else + /* Try to get compiler to generate conditional move instructions + instead. Works fine, but leaving it disabled for now because + it's not yielding consistently faster sorts. Needs more + investigation. More computation in the inner loop adds its own + costs, which can be significant when compares are fast. */ + k = ISLT(pivot, a[M]); + if (k < 0) + goto fail; + Py_ssize_t Mp1 = M + 1; + R = k ? M : R; + L = k ? L : Mp1; +#endif + } while (L < R); + assert(L == R); + /* a[:L] holds all elements from a[:ok] <= pivot now, so pivot belongs + at index L. Slide a[L:ok] to the right a slot to make room for it. + Caution: using memmove is much slower under MSVC 5; we're not + usually moving many slots. Years later: under Visual Studio 2022, + memmove seems just slightly slower than doing it "by hand". */ + for (M = ok; M > L; --M) + a[M] = a[M - 1]; + a[L] = pivot; + if (has_values) { + pivot = v[ok]; + for (M = ok; M > L; --M) + v[M] = v[M - 1]; + v[L] = pivot; } } +#endif // pick binary or regular insertion sort return 0; fail: return -1; } -/* -Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi -is required on entry. "A run" is the longest ascending sequence, with - - lo[0] <= lo[1] <= lo[2] <= ... - -or the longest descending sequence, with - - lo[0] > lo[1] > lo[2] > ... +static void +sortslice_reverse(sortslice *s, Py_ssize_t n) +{ + reverse_slice(s->keys, &s->keys[n]); + if (s->values != NULL) + reverse_slice(s->values, &s->values[n]); +} -Boolean *descending is set to 0 in the former case, or to 1 in the latter. -For its intended use in a stable mergesort, the strictness of the defn of -"descending" is needed so that the caller can safely reverse a descending -sequence without violating stability (strict > ensures there are no equal -elements to get out of order). +/* +Return the length of the run beginning at slo->keys, spanning no more than +nremaining elements. The run beginning there may be ascending or descending, +but the function permutes it in place, if needed, so that it's always ascending +upon return. Returns -1 in case of error. */ static Py_ssize_t -count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) +count_run(MergeState *ms, sortslice *slo, Py_ssize_t nremaining) { - Py_ssize_t k; + Py_ssize_t k; /* used by IFLT macro expansion */ Py_ssize_t n; + PyObject ** const lo = slo->keys; - assert(lo < hi); - *descending = 0; - ++lo; - if (lo == hi) - return 1; - - n = 2; - IFLT(*lo, *(lo-1)) { - *descending = 1; - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) - ; - else - break; - } + /* In general, as things go on we've established that the slice starts + with a monotone run of n elements, starting at lo. */ + + /* We're n elements into the slice, and the most recent neq+1 elements are + * all equal. This reverses them in-place, and resets neq for reuse. + */ +#define REVERSE_LAST_NEQ \ + if (neq) { \ + sortslice slice = *slo; \ + ++neq; \ + sortslice_advance(&slice, n - neq); \ + sortslice_reverse(&slice, neq); \ + neq = 0; \ + } + + /* Sticking to only __lt__ compares is confusing and error-prone. But in + * this routine, almost all uses of IFLT can be captured by tiny macros + * giving mnemonic names to the intent. Note that inline functions don't + * work for this (IFLT expands to code including `goto fail`). + */ +#define IF_NEXT_LARGER IFLT(lo[n-1], lo[n]) +#define IF_NEXT_SMALLER IFLT(lo[n], lo[n-1]) + + assert(nremaining); + /* try ascending run first */ + for (n = 1; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; } - else { - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) + if (n == nremaining) + return n; + /* lo[n] is strictly less */ + /* If n is 1 now, then the first compare established it's a descending + * run, so fall through to the descending case. But if n > 1, there are + * n elements in an ascending run terminated by the strictly less lo[n]. + * If the first key < lo[n-1], *somewhere* along the way the sequence + * increased, so we're done (there is no descending run). + * Else first key >= lo[n-1], which implies that the entire ascending run + * consists of equal elements. In that case, this is a descending run, + * and we reverse the all-equal prefix in-place. + */ + if (n > 1) { + IFLT(lo[0], lo[n-1]) + return n; + sortslice_reverse(slo, n); + } + ++n; /* in all cases it's been established that lo[n] has been resolved */ + + /* Finish descending run. All-squal subruns are reversed in-place on the + * fly. Their original order will be restored at the end by the whole-slice + * reversal. + */ + Py_ssize_t neq = 0; + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER { + /* This ends the most recent run of equal elements, but still in + * the "descending" direction. + */ + REVERSE_LAST_NEQ + } + else { + IF_NEXT_LARGER /* descending run is over */ break; + else /* not x < y and not y < x implies x == y */ + ++neq; } } + REVERSE_LAST_NEQ + sortslice_reverse(slo, n); /* transform to ascending run */ + + /* And after reversing, it's possible this can be extended by a + * naturally increasing suffix; e.g., [3, 2, 3, 4, 1] makes an + * ascending run from the first 4 elements. + */ + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; + } return n; fail: return -1; + +#undef REVERSE_LAST_NEQ +#undef IF_NEXT_SMALLER +#undef IF_NEXT_LARGER } /* @@ -2036,10 +2646,10 @@ merge_force_collapse(MergeState *ms) /* Compute a good value for the minimum run length; natural runs shorter * than this are boosted artificially via binary insertion. * - * If n < 64, return n (it's too small to bother with fancy stuff). - * Else if n is an exact power of 2, return 32. - * Else return an int k, 32 <= k <= 64, such that n/k is close to, but - * strictly less than, an exact power of 2. + * If n < MAX_MINRUN return n (it's too small to bother with fancy stuff). + * Else if n is an exact power of 2, return MAX_MINRUN / 2. + * Else return an int k, MAX_MINRUN / 2 <= k <= MAX_MINRUN, such that n/k is + * close to, but strictly less than, an exact power of 2. * * See listsort.txt for more info. */ @@ -2049,21 +2659,13 @@ merge_compute_minrun(Py_ssize_t n) Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ assert(n >= 0); - while (n >= 64) { + while (n >= MAX_MINRUN) { r |= n & 1; n >>= 1; } return n + r; } -static void -reverse_sortslice(sortslice *s, Py_ssize_t n) -{ - reverse_slice(s->keys, &s->keys[n]); - if (s->values != NULL) - reverse_slice(s->values, &s->values[n]); -} - /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ @@ -2232,6 +2834,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@critical_section list.sort * @@ -2251,7 +2854,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2278,7 +2881,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); - self->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { @@ -2430,20 +3033,17 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) */ minrun = merge_compute_minrun(nremaining); do { - int descending; Py_ssize_t n; /* Identify next run. */ - n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); + n = count_run(&ms, &lo, nremaining); if (n < 0) goto fail; - if (descending) - reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; - if (binarysort(&ms, lo, lo.keys + force, lo.keys + n) < 0) + if (binarysort(&ms, &lo, force, n) < 0) goto fail; n = force; } @@ -2498,15 +3098,20 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); - self->ob_item = saved_ob_item; - self->allocated = saved_allocated; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, saved_ob_item); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, saved_allocated); if (final_ob_item != NULL) { - /* we cannot use _list_clear() for this because it does not + /* we cannot use list_clear() for this because it does not guarantee that the list is really empty when it returns */ while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } - PyMem_Free(final_ob_item); +#ifdef Py_GIL_DISABLED + bool use_qsbr = _PyObject_GC_IS_SHARED(self); +#else + bool use_qsbr = false; +#endif + free_list_items(final_ob_item, use_qsbr); } return Py_XNewRef(result); } @@ -2520,7 +3125,9 @@ PyList_Sort(PyObject *v) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(v); v = list_sort_impl((PyListObject *)v, NULL, 0); + Py_END_CRITICAL_SECTION(); if (v == NULL) return -1; Py_DECREF(v); @@ -2528,6 +3135,7 @@ PyList_Sort(PyObject *v) } /*[clinic input] +@critical_section list.reverse Reverse *IN PLACE*. @@ -2535,7 +3143,7 @@ Reverse *IN PLACE*. static PyObject * list_reverse_impl(PyListObject *self) -/*[clinic end generated code: output=482544fc451abea9 input=eefd4c3ae1bc9887]*/ +/*[clinic end generated code: output=482544fc451abea9 input=04ac8e0c6a66e4d9]*/ { if (Py_SIZE(self) > 1) reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); @@ -2551,8 +3159,11 @@ PyList_Reverse(PyObject *v) PyErr_BadInternalCall(); return -1; } - if (Py_SIZE(self) > 1) + Py_BEGIN_CRITICAL_SECTION(self); + if (Py_SIZE(self) > 1) { reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); + } + Py_END_CRITICAL_SECTION() return 0; } @@ -2563,7 +3174,12 @@ PyList_AsTuple(PyObject *v) PyErr_BadInternalCall(); return NULL; } - return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); + PyObject *ret; + PyListObject *self = (PyListObject *)v; + Py_BEGIN_CRITICAL_SECTION(self); + ret = _PyTuple_FromArray(self->ob_item, Py_SIZE(v)); + Py_END_CRITICAL_SECTION(); + return ret; } PyObject * @@ -2605,8 +3221,6 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop) /*[clinic end generated code: output=ec51b88787e4e481 input=40ec5826303a0eb1]*/ { - Py_ssize_t i; - if (start < 0) { start += Py_SIZE(self); if (start < 0) @@ -2617,9 +3231,12 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, if (stop < 0) stop = 0; } - for (i = start; i < stop && i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; - Py_INCREF(obj); + for (Py_ssize_t i = start; i < stop; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2627,7 +3244,7 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.index(x): x not in list"); return NULL; } @@ -2645,15 +3262,17 @@ list_count(PyListObject *self, PyObject *value) /*[clinic end generated code: output=b1f5d284205ae714 input=3bdc3a5e6f749565]*/ { Py_ssize_t count = 0; - Py_ssize_t i; - - for (i = 0; i < Py_SIZE(self); i++) { - PyObject *obj = self->ob_item[i]; + for (Py_ssize_t i = 0; ; i++) { + PyObject *obj = list_get_item_ref(self, i); + if (obj == NULL) { + // out-of-bounds + break; + } if (obj == value) { count++; + Py_DECREF(obj); continue; } - Py_INCREF(obj); int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) @@ -2665,6 +3284,7 @@ list_count(PyListObject *self, PyObject *value) } /*[clinic input] +@critical_section list.remove value: object @@ -2676,8 +3296,8 @@ Raises ValueError if the value is not present. [clinic start generated code]*/ static PyObject * -list_remove(PyListObject *self, PyObject *value) -/*[clinic end generated code: output=f087e1951a5e30d1 input=2dc2ba5bb2fb1f82]*/ +list_remove_impl(PyListObject *self, PyObject *value) +/*[clinic end generated code: output=b9b76a6633b18778 input=26c813dbb95aa93b]*/ { Py_ssize_t i; @@ -2687,21 +3307,21 @@ list_remove(PyListObject *self, PyObject *value) int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) { - if (list_ass_slice(self, i, i+1, - (PyObject *)NULL) == 0) + if (list_ass_slice_lock_held(self, i, i+1, NULL) == 0) Py_RETURN_NONE; return NULL; } else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); return NULL; } static int -list_traverse(PyListObject *o, visitproc visit, void *arg) +list_traverse(PyObject *self, visitproc visit, void *arg) { + PyListObject *o = (PyListObject *)self; Py_ssize_t i; for (i = Py_SIZE(o); --i >= 0; ) @@ -2710,7 +3330,7 @@ list_traverse(PyListObject *o, visitproc visit, void *arg) } static PyObject * -list_richcompare(PyObject *v, PyObject *w, int op) +list_richcompare_impl(PyObject *v, PyObject *w, int op) { PyListObject *vl, *wl; Py_ssize_t i; @@ -2762,7 +3382,24 @@ list_richcompare(PyObject *v, PyObject *w, int op) } /* Compare the final item again using the proper operator */ - return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + PyObject *vitem = vl->ob_item[i]; + PyObject *witem = wl->ob_item[i]; + Py_INCREF(vitem); + Py_INCREF(witem); + PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; +} + +static PyObject * +list_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(v, w); + ret = list_richcompare_impl(v, w, op); + Py_END_CRITICAL_SECTION2() + return ret; } /*[clinic input] @@ -2789,13 +3426,12 @@ list___init___impl(PyListObject *self, PyObject *iterable) /* Empty previous contents */ if (self->ob_item != NULL) { - (void)_list_clear(self); + list_clear(self); } if (iterable != NULL) { - PyObject *rv = list_extend(self, iterable); - if (rv == NULL) + if (_list_extend(self, iterable) < 0) { return -1; - Py_DECREF(rv); + } } return 0; } @@ -2837,19 +3473,20 @@ list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)self->allocated * sizeof(void*); + Py_ssize_t allocated = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated); + res += (size_t)allocated * sizeof(void*); return PyLong_FromSize_t(res); } static PyObject *list_iter(PyObject *seq); -static PyObject *list_subscript(PyListObject*, PyObject*); +static PyObject *list_subscript(PyObject*, PyObject*); static PyMethodDef list_methods[] = { - {"__getitem__", (PyCFunction)list_subscript, METH_O|METH_COEXIST, + {"__getitem__", list_subscript, METH_O|METH_COEXIST, PyDoc_STR("__getitem__($self, index, /)\n--\n\nReturn self[index].")}, LIST___REVERSED___METHODDEF LIST___SIZEOF___METHODDEF - LIST_CLEAR_METHODDEF + PY_LIST_CLEAR_METHODDEF LIST_COPY_METHODDEF LIST_APPEND_METHODDEF LIST_INSERT_METHODDEF @@ -2865,21 +3502,61 @@ static PyMethodDef list_methods[] = { }; static PySequenceMethods list_as_sequence = { - (lenfunc)list_length, /* sq_length */ - (binaryfunc)list_concat, /* sq_concat */ - (ssizeargfunc)list_repeat, /* sq_repeat */ - (ssizeargfunc)list_item, /* sq_item */ + list_length, /* sq_length */ + list_concat, /* sq_concat */ + list_repeat, /* sq_repeat */ + list_item, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)list_ass_item, /* sq_ass_item */ + list_ass_item, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)list_contains, /* sq_contains */ - (binaryfunc)list_inplace_concat, /* sq_inplace_concat */ - (ssizeargfunc)list_inplace_repeat, /* sq_inplace_repeat */ + list_contains, /* sq_contains */ + list_inplace_concat, /* sq_inplace_concat */ + list_inplace_repeat, /* sq_inplace_repeat */ }; +static inline PyObject * +list_slice_step_lock_held(PyListObject *a, Py_ssize_t start, Py_ssize_t step, Py_ssize_t len) +{ + PyListObject *np = (PyListObject *)list_new_prealloc(len); + if (np == NULL) { + return NULL; + } + size_t cur; + Py_ssize_t i; + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (cur = start, i = 0; i < len; + cur += (size_t)step, i++) { + PyObject *v = src[cur]; + dest[i] = Py_NewRef(v); + } + Py_SET_SIZE(np, len); + return (PyObject *)np; +} + static PyObject * -list_subscript(PyListObject* self, PyObject* item) +list_slice_wrap(PyListObject *aa, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) { + PyObject *res = NULL; + Py_BEGIN_CRITICAL_SECTION(aa); + Py_ssize_t len = PySlice_AdjustIndices(Py_SIZE(aa), &start, &stop, step); + if (len <= 0) { + res = PyList_New(0); + } + else if (step == 1) { + res = list_slice_lock_held(aa, start, stop); + } + else { + res = list_slice_step_lock_held(aa, start, step, len); + } + Py_END_CRITICAL_SECTION(); + return res; +} + +static PyObject * +list_subscript(PyObject* _self, PyObject* item) +{ + PyListObject* self = (PyListObject*)_self; if (_PyIndex_Check(item)) { Py_ssize_t i; i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -2887,41 +3564,14 @@ list_subscript(PyListObject* self, PyObject* item) return NULL; if (i < 0) i += PyList_GET_SIZE(self); - return list_item(self, i); + return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, i; - size_t cur; - PyObject* result; - PyObject* it; - PyObject **src, **dest; - + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (slicelength <= 0) { - return PyList_New(0); - } - else if (step == 1) { - return list_slice(self, start, stop); - } - else { - result = list_new_prealloc(slicelength); - if (!result) return NULL; - - src = self->ob_item; - dest = ((PyListObject *)result)->ob_item; - for (cur = start, i = 0; i < slicelength; - cur += (size_t)step, i++) { - it = Py_NewRef(src[cur]); - dest[i] = it; - } - Py_SET_SIZE(result, slicelength); - return result; - } + return list_slice_wrap(self, start, stop, step); } else { PyErr_Format(PyExc_TypeError, @@ -2931,34 +3581,41 @@ list_subscript(PyListObject* self, PyObject* item) } } +static Py_ssize_t +adjust_slice_indexes(PyListObject *lst, + Py_ssize_t *start, Py_ssize_t *stop, + Py_ssize_t step) +{ + Py_ssize_t slicelength = PySlice_AdjustIndices(Py_SIZE(lst), start, stop, + step); + + /* Make sure s[5:2] = [..] inserts at the right place: + before 5, not before 2. */ + if ((step < 0 && *start < *stop) || + (step > 0 && *start > *stop)) + *stop = *start; + + return slicelength; +} + static int -list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) +list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) { + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item(self, i, value); + return list_ass_item((PyObject *)self, i, value); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength; + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (step == 1) - return list_ass_slice(self, start, stop, value); - - /* Make sure s[5:2] = [..] inserts at the right place: - before 5, not before 2. */ - if ((step < 0 && start < stop) || - (step > 0 && start > stop)) - stop = start; if (value == NULL) { /* delete slice */ @@ -2967,6 +3624,12 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) Py_ssize_t i; int res; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) + return list_ass_slice(self, start, stop, value); + if (slicelength <= 0) return 0; @@ -3031,8 +3694,10 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - seq = list_slice((PyListObject*)value, 0, - PyList_GET_SIZE(value)); + Py_BEGIN_CRITICAL_SECTION(value); + seq = list_slice_lock_held((PyListObject*)value, 0, + Py_SIZE(value)); + Py_END_CRITICAL_SECTION(); } else { seq = PySequence_Fast(value, @@ -3042,6 +3707,15 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) if (!seq) return -1; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) { + int res = list_ass_slice(self, start, stop, seq); + Py_DECREF(seq); + return res; + } + if (PySequence_Fast_GET_SIZE(seq) != slicelength) { PyErr_Format(PyExc_ValueError, "attempt to assign sequence of " @@ -3094,9 +3768,9 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) } static PyMappingMethods list_as_mapping = { - (lenfunc)list_length, - (binaryfunc)list_subscript, - (objobjargproc)list_ass_subscript + list_length, + list_subscript, + list_ass_subscript }; PyTypeObject PyList_Type = { @@ -3104,12 +3778,12 @@ PyTypeObject PyList_Type = { "list", sizeof(PyListObject), 0, - (destructor)list_dealloc, /* tp_dealloc */ + list_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)list_repr, /* tp_repr */ + list_repr, /* tp_repr */ 0, /* tp_as_number */ &list_as_sequence, /* tp_as_sequence */ &list_as_mapping, /* tp_as_mapping */ @@ -3123,8 +3797,8 @@ PyTypeObject PyList_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_LIST_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ list___init____doc__, /* tp_doc */ - (traverseproc)list_traverse, /* tp_traverse */ - (inquiry)_list_clear, /* tp_clear */ + list_traverse, /* tp_traverse */ + list_clear_slot, /* tp_clear */ list_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ list_iter, /* tp_iter */ @@ -3146,22 +3820,22 @@ PyTypeObject PyList_Type = { /*********************** List Iterator **************************/ -static void listiter_dealloc(_PyListIterObject *); -static int listiter_traverse(_PyListIterObject *, visitproc, void *); -static PyObject *listiter_next(_PyListIterObject *); -static PyObject *listiter_len(_PyListIterObject *, PyObject *); +static void listiter_dealloc(PyObject *); +static int listiter_traverse(PyObject *, visitproc, void *); +static PyObject *listiter_next(PyObject *); +static PyObject *listiter_len(PyObject *, PyObject *); static PyObject *listiter_reduce_general(void *_it, int forward); -static PyObject *listiter_reduce(_PyListIterObject *, PyObject *); -static PyObject *listiter_setstate(_PyListIterObject *, PyObject *state); +static PyObject *listiter_reduce(PyObject *, PyObject *); +static PyObject *listiter_setstate(PyObject *, PyObject *state); PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef listiter_methods[] = { - {"__length_hint__", (PyCFunction)listiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listiter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listiter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3171,7 +3845,7 @@ PyTypeObject PyListIter_Type = { sizeof(_PyListIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listiter_dealloc, /* tp_dealloc */ + listiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3188,12 +3862,12 @@ PyTypeObject PyListIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listiter_traverse, /* tp_traverse */ + listiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listiter_next, /* tp_iternext */ + listiter_next, /* tp_iternext */ listiter_methods, /* tp_methods */ 0, /* tp_members */ }; @@ -3218,49 +3892,53 @@ list_iter(PyObject *seq) } static void -listiter_dealloc(_PyListIterObject *it) +listiter_dealloc(PyObject *self) { + _PyListIterObject *it = (_PyListIterObject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listiter_traverse(_PyListIterObject *it, visitproc visit, void *arg) +listiter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((_PyListIterObject *)it)->it_seq); return 0; } static PyObject * -listiter_next(_PyListIterObject *it) +listiter_next(PyObject *self) { - PyListObject *seq; - PyObject *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) + _PyListIterObject *it = (_PyListIterObject *)self; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; - assert(PyList_Check(seq)); - - if (it->it_index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, it->it_index); - ++it->it_index; - return Py_NewRef(item); } - it->it_seq = NULL; - Py_DECREF(seq); - return NULL; + PyObject *item = list_get_item_ref(it->it_seq, index); + if (item == NULL) { + // out-of-bounds + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED + PyListObject *seq = it->it_seq; + it->it_seq = NULL; + Py_DECREF(seq); +#endif + return NULL; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1); + return item; } static PyObject * -listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t len; - if (it->it_seq) { - len = PyList_GET_SIZE(it->it_seq) - it->it_index; + assert(self != NULL); + _PyListIterObject *it = (_PyListIterObject *)self; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index >= 0) { + Py_ssize_t len = PyList_GET_SIZE(it->it_seq) - index; if (len >= 0) return PyLong_FromSsize_t(len); } @@ -3268,20 +3946,21 @@ listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -listiter_reduce(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 1); } static PyObject * -listiter_setstate(_PyListIterObject *it, PyObject *state) +listiter_setstate(PyObject *self, PyObject *state) { + _PyListIterObject *it = (_PyListIterObject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; if (it->it_seq != NULL) { - if (index < 0) - index = 0; + if (index < -1) + index = -1; else if (index > PyList_GET_SIZE(it->it_seq)) index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */ it->it_index = index; @@ -3297,17 +3976,17 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } listreviterobject; -static void listreviter_dealloc(listreviterobject *); -static int listreviter_traverse(listreviterobject *, visitproc, void *); -static PyObject *listreviter_next(listreviterobject *); -static PyObject *listreviter_len(listreviterobject *, PyObject *); -static PyObject *listreviter_reduce(listreviterobject *, PyObject *); -static PyObject *listreviter_setstate(listreviterobject *, PyObject *); +static void listreviter_dealloc(PyObject *); +static int listreviter_traverse(PyObject *, visitproc, void *); +static PyObject *listreviter_next(PyObject *); +static PyObject *listreviter_len(PyObject *, PyObject *); +static PyObject *listreviter_reduce(PyObject *, PyObject *); +static PyObject *listreviter_setstate(PyObject *, PyObject *); static PyMethodDef listreviter_methods[] = { - {"__length_hint__", (PyCFunction)listreviter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listreviter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listreviter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listreviter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listreviter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listreviter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3317,7 +3996,7 @@ PyTypeObject PyListRevIter_Type = { sizeof(listreviterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listreviter_dealloc, /* tp_dealloc */ + listreviter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3334,12 +4013,12 @@ PyTypeObject PyListRevIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listreviter_traverse, /* tp_traverse */ + listreviter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listreviter_next, /* tp_iternext */ + listreviter_next, /* tp_iternext */ listreviter_methods, /* tp_methods */ 0, }; @@ -3367,64 +4046,67 @@ list___reversed___impl(PyListObject *self) } static void -listreviter_dealloc(listreviterobject *it) +listreviter_dealloc(PyObject *self) { + listreviterobject *it = (listreviterobject *)self; PyObject_GC_UnTrack(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listreviter_traverse(listreviterobject *it, visitproc visit, void *arg) +listreviter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((listreviterobject *)it)->it_seq); return 0; } static PyObject * -listreviter_next(listreviterobject *it) +listreviter_next(PyObject *self) { - PyObject *item; - Py_ssize_t index; - PyListObject *seq; - + listreviterobject *it = (listreviterobject *)self; assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) { return NULL; } - assert(PyList_Check(seq)); - index = it->it_index; - if (index>=0 && index < PyList_GET_SIZE(seq)) { - item = PyList_GET_ITEM(seq, index); - it->it_index--; - return Py_NewRef(item); + PyListObject *seq = it->it_seq; + assert(PyList_Check(seq)); + PyObject *item = list_get_item_ref(seq, index); + if (item != NULL) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index - 1); + return item; } - it->it_index = -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED it->it_seq = NULL; Py_DECREF(seq); +#endif return NULL; } static PyObject * -listreviter_len(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t len = it->it_index + 1; + listreviterobject *it = (listreviterobject *)self; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + Py_ssize_t len = index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; return PyLong_FromSsize_t(len); } static PyObject * -listreviter_reduce(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 0); } static PyObject * -listreviter_setstate(listreviterobject *it, PyObject *state) +listreviter_setstate(PyObject *self, PyObject *state) { + listreviterobject *it = (listreviterobject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -3444,6 +4126,7 @@ static PyObject * listiter_reduce_general(void *_it, int forward) { PyObject *list; + PyObject *iter; /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. @@ -3451,29 +4134,21 @@ listiter_reduce_general(void *_it, int forward) /* the objects are not the same, index is of different types! */ if (forward) { - PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); - if (!iter) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(iter)); _PyListIterObject *it = (_PyListIterObject *)_it; - if (it->it_seq) { + if (it->it_index >= 0) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(iter); } else { - PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed)); - if (!reversed) { - return NULL; - } + iter = _PyEval_GetBuiltin(&_Py_ID(reversed)); listreviterobject *it = (listreviterobject *)_it; - if (it->it_seq) { - return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index); + if (it->it_index >= 0) { + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } - Py_DECREF(reversed); } /* empty iterator, create an empty list */ list = PyList_New(0); if (list == NULL) return NULL; - return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), list); + return Py_BuildValue("N(N)", iter, list); } diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 32a59e510f0754..f387d9c116e502 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -212,24 +212,43 @@ A detailed description of timsort follows. Runs ---- -count_run() returns the # of elements in the next run. A run is either -"ascending", which means non-decreasing: +count_run() returns the # of elements in the next run, and, if it's a +descending run, reverses it in-place. A run is either "ascending", which +means non-decreasing: a0 <= a1 <= a2 <= ... -or "descending", which means strictly decreasing: +or "descending", which means non-increasing: - a0 > a1 > a2 > ... + a0 >= a1 >= a2 >= ... Note that a run is always at least 2 long, unless we start at the array's -last element. - -The definition of descending is strict, because the main routine reverses -a descending run in-place, transforming a descending run into an ascending -run. Reversal is done via the obvious fast "swap elements starting at each -end, and converge at the middle" method, and that can violate stability if -the slice contains any equal elements. Using a strict definition of -descending ensures that a descending run contains distinct elements. +last element. If all elements in the array are equal, it can be viewed as +both ascending and descending. Upon return, the run count_run() identifies +is always ascending. + +Reversal is done via the obvious fast "swap elements starting at each +end, and converge at the middle" method. That can violate stability if +the slice contains any equal elements. For that reason, for a long time +the code used strict inequality (">" rather than ">=") in its definition +of descending. + +Removing that restriction required some complication: when processing a +descending run, all-equal sub-runs of elements are reversed in-place, on the +fly. Their original relative order is restored "by magic" via the final +"reverse the entire run" step. + +This makes processing descending runs a little more costly. We only use +`__lt__` comparisons, so that `x == y` has to be deduced from +`not x < y and not y < x`. But so long as a run remains strictly decreasing, +only one of those compares needs to be done per loop iteration. So the primsry +extra cost is paid only when there are equal elements, and they get some +compensating benefit by not needing to end the descending run. + +There's one more trick added since the original: after reversing a descending +run, it's possible that it can be extended by an adjacent ascending run. For +example, given [3, 2, 1, 3, 4, 5, 0], the 3-element descending prefix is +reversed in-place, and then extended by [3, 4, 5]. If an array is random, it's very unlikely we'll see long runs. If a natural run contains less than minrun elements (see next section), the main loop @@ -251,9 +270,9 @@ result. This has two primary good effects: Computing minrun ---------------- -If N < 64, minrun is N. IOW, binary insertion sort is used for the whole -array then; it's hard to beat that given the overheads of trying something -fancier (see note BINSORT). +If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the +whole array then; it's hard to beat that given the overheads of trying +something fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -291,12 +310,13 @@ place, and r < minrun is small compared to N), or q a little larger than a power of 2 regardless of r (then we've got a case similar to "2112", again leaving too little work for the last merge to do). -Instead we pick a minrun in range(32, 65) such that N/minrun is exactly a -power of 2, or if that isn't possible, is close to, but strictly less than, -a power of 2. This is easier to do than it may sound: take the first 6 -bits of N, and add 1 if any of the remaining bits are set. In fact, that -rule covers every case in this section, including small N and exact powers -of 2; merge_compute_minrun() is a deceptively simple function. +Instead we pick a minrun in range(MAX_MINRUN / 2, MAX_MINRUN + 1) such that +N/minrun is exactly a power of 2, or if that isn't possible, is close to, but +strictly less than, a power of 2. This is easier to do than it may sound: +take the first log2(MAX_MINRUN) bits of N, and add 1 if any of the remaining +bits are set. In fact, that rule covers every case in this section, including +small N and exact powers of 2; merge_compute_minrun() is a deceptively simple +function. The Merge Pattern diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index d45d09d4ab9a50..0f3599340318f0 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -60,7 +60,7 @@ Final form: Iterating over the table. ------------------------- -For the `co_lines` attribute we want to emit the full form, omitting the (350, 360, No line number) and empty entries. +For the `co_lines` method we want to emit the full form, omitting the (350, 360, No line number) and empty entries. The code is as follows: diff --git a/Objects/longobject.c b/Objects/longobject.c index e73de742229005..ace01ddeb37f66 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_call.h" // _PyObject_MakeTpCall #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -400,12 +401,12 @@ PyLong_FromDouble(double dval) double frac; int i, ndig, expo, neg; neg = 0; - if (Py_IS_INFINITY(dval)) { + if (isinf(dval)) { PyErr_SetString(PyExc_OverflowError, "cannot convert float infinity to integer"); return NULL; } - if (Py_IS_NAN(dval)) { + if (isnan(dval)) { PyErr_SetString(PyExc_ValueError, "cannot convert float NaN to integer"); return NULL; @@ -482,11 +483,18 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); - res = (long)tmp; - if (res != tmp) { - *overflow = tmp < 0 ? -1 : 1; +#if SIZEOF_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long)tmp; } #else res = _PyLong_CompactValue(v); @@ -631,14 +639,15 @@ PyLong_AsUnsignedLong(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); unsigned long res = (unsigned long)tmp; if (res != tmp) { goto overflow; } + return res; #else - return _PyLong_CompactValue(v); + return (unsigned long)(size_t)_PyLong_CompactValue(v); #endif } if (_PyLong_IsNegative(v)) { @@ -684,7 +693,7 @@ PyLong_AsSize_t(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { - return _PyLong_CompactValue(v); + return (size_t)_PyLong_CompactValue(v); } if (_PyLong_IsNegative(v)) { PyErr_SetString(PyExc_OverflowError, @@ -721,7 +730,11 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long)_PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + return (unsigned long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long)(long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); @@ -769,6 +782,18 @@ _PyLong_Sign(PyObject *vv) return _PyLong_NonCompactSign(v); } +int +PyLong_GetSign(PyObject *vv, int *sign) +{ + if (!PyLong_Check(vv)) { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + *sign = _PyLong_Sign(vv); + return 0; +} + static int bit_length_digit(digit x) { @@ -927,7 +952,8 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed) + int little_endian, int is_signed, + int with_exceptions) { Py_ssize_t i; /* index into v->long_value.ob_digit */ Py_ssize_t ndigits; /* number of digits */ @@ -944,8 +970,10 @@ _PyLong_AsByteArray(PyLongObject* v, ndigits = _PyLong_DigitCount(v); if (_PyLong_IsNegative(v)) { if (!is_signed) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative int to unsigned"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative int to unsigned"); + } return -1; } do_twos_comp = 1; @@ -966,7 +994,12 @@ _PyLong_AsByteArray(PyLongObject* v, /* Copy over all the Python digits. It's crucial that every Python digit except for the MSD contribute exactly PyLong_SHIFT bits to the total, so first assert that the int is - normalized. */ + normalized. + NOTE: PyLong_AsNativeBytes() assumes that this function will fill in 'n' + bytes even if it eventually fails to convert the whole number. Make sure + you account for that if you are changing this algorithm to return without + doing that. + */ assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); j = 0; accum = 0; @@ -1051,11 +1084,265 @@ _PyLong_AsByteArray(PyLongObject* v, return 0; Overflow: - PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + if (with_exceptions) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + } return -1; } +// Refactored out for readability, not reuse +static inline int +_fits_in_n_bits(Py_ssize_t v, Py_ssize_t n) +{ + if (n >= (Py_ssize_t)sizeof(Py_ssize_t) * 8) { + return 1; + } + // If all bits above n are the same, we fit. + // (Use n-1 if we require the sign bit to be consistent.) + Py_ssize_t v_extended = v >> ((int)n - 1); + return v_extended == 0 || v_extended == -1; +} + +static inline int +_resolve_endianness(int *endianness) +{ + if (*endianness == -1 || (*endianness & 2)) { + *endianness = PY_LITTLE_ENDIAN; + } else { + *endianness &= 1; + } + assert(*endianness == 0 || *endianness == 1); + return 0; +} + +Py_ssize_t +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) +{ + PyLongObject *v; + union { + Py_ssize_t v; + unsigned char b[sizeof(Py_ssize_t)]; + } cv; + int do_decref = 0; + Py_ssize_t res = 0; + + if (vv == NULL || n < 0) { + PyErr_BadInternalCall(); + return -1; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return -1; + } + + if (PyLong_Check(vv)) { + v = (PyLongObject *)vv; + } + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { + v = (PyLongObject *)_PyNumber_Index(vv); + if (v == NULL) { + return -1; + } + do_decref = 1; + } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } + + if (_PyLong_IsCompact(v)) { + res = 0; + cv.v = _PyLong_CompactValue(v); + /* Most paths result in res = sizeof(compact value). Only the case + * where 0 < n < sizeof(compact value) do we need to check and adjust + * our return value. */ + res = sizeof(cv.b); + if (n <= 0) { + // nothing to do! + } + else if (n <= (Py_ssize_t)sizeof(cv.b)) { +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, n); + } + else { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[n - i - 1] = cv.b[i]; + } + } +#else + if (little_endian) { + for (Py_ssize_t i = 0; i < n; ++i) { + ((unsigned char*)buffer)[i] = cv.b[sizeof(cv.b) - i - 1]; + } + } + else { + memcpy(buffer, &cv.b[sizeof(cv.b) - n], n); + } +#endif + + /* If we fit, return the requested number of bytes */ + if (_fits_in_n_bits(cv.v, n * 8)) { + res = n; + } else if (cv.v > 0 && _fits_in_n_bits(cv.v, n * 8 + 1)) { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } + else { + unsigned char fill = cv.v < 0 ? 0xFF : 0x00; +#if PY_LITTLE_ENDIAN + if (little_endian) { + memcpy(buffer, cv.b, sizeof(cv.b)); + memset((char *)buffer + sizeof(cv.b), fill, n - sizeof(cv.b)); + } + else { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + } +#else + if (little_endian) { + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { + *b++ = cv.b[i - 1]; + } + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { + *b++ = fill; + } + } + else { + memset(buffer, fill, n - sizeof(cv.b)); + memcpy((char *)buffer + n - sizeof(cv.b), cv.b, sizeof(cv.b)); + } +#endif + } + } + else { + if (n > 0) { + _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0); + } + + /* Calculates the number of bits required for the *absolute* value + * of v. This does not take sign into account, only magnitude. */ + size_t nb = _PyLong_NumBits((PyObject *)v); + if (nb == (size_t)-1) { + res = -1; + } else { + /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; + } + + /* Two edge cases exist that are best handled after extracting the + * bits. These may result in us reporting overflow when the value + * actually fits. + */ + if (n > 0 && res == n + 1 && nb % 8 == 0) { + if (_PyLong_IsNegative(v)) { + /* Values of 0x80...00 from negative values that use every + * available bit in the buffer do not require an additional + * bit to store the sign. */ + int is_edge_case = 1; + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n && is_edge_case; ++i, ++b) { + if (i == 0) { + is_edge_case = (*b == (little_endian ? 0 : 0x80)); + } else if (i < n - 1) { + is_edge_case = (*b == 0); + } else { + is_edge_case = (*b == (little_endian ? 0x80 : 0)); + } + } + if (is_edge_case) { + res = n; + } + } + else { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + unsigned char *b = (unsigned char *)buffer; + if (b[little_endian ? n - 1 : 0] & 0x80) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } + } + } + + if (do_decref) { + Py_DECREF(v); + } + + return res; +} + + +PyObject * +PyLong_FromNativeBytes(const void* buffer, size_t n, int flags) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray( + (const unsigned char *)buffer, + n, + little_endian, + (flags == -1 || !(flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) ? 1 : 0 + ); +} + + +PyObject * +PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int flags) +{ + if (!buffer) { + PyErr_BadInternalCall(); + return NULL; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return NULL; + } + + return _PyLong_FromByteArray((const unsigned char *)buffer, n, little_endian, 0); +} + + /* Create a new int object from a C pointer */ PyObject * @@ -1230,7 +1517,7 @@ PyLong_AsLongLong(PyObject *vv) } else { res = _PyLong_AsByteArray((PyLongObject *)v, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1, 1); } if (do_decref) { Py_DECREF(v); @@ -1265,11 +1552,22 @@ PyLong_AsUnsignedLongLong(PyObject *vv) v = (PyLongObject*)vv; if (_PyLong_IsNonNegativeCompact(v)) { res = 0; - bytes = _PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); + bytes = (unsigned long long)tmp; + if (bytes != tmp) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert " + "to C unsigned long long"); + res = -1; + } +#else + bytes = (unsigned long long)(size_t)_PyLong_CompactValue(v); +#endif } else { res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, - SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0); + SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0, 1); } /* Plan 9 can't handle long long in ? : expressions */ @@ -1296,7 +1594,11 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long long)(signed long long)_PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + return (unsigned long long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long long)(long long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); @@ -1368,7 +1670,22 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LLONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LLONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long long)tmp; + } +#else res = _PyLong_CompactValue(v); +#endif } else { i = _PyLong_DigitCount(v); @@ -1764,7 +2081,9 @@ long_to_decimal_string_internal(PyObject *aa, digit *pout, *pin, rem, tenpow; int negative; int d; - int kind; + + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { @@ -1877,7 +2196,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -1892,7 +2210,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = PyUnicode_KIND(str); } #define WRITE_DIGITS(p) \ @@ -1940,19 +2257,23 @@ long_to_decimal_string_internal(PyObject *aa, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(str); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -1993,11 +2314,12 @@ long_format_binary(PyObject *aa, int base, int alternate, PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; - int kind; int negative; int bits; assert(base == 2 || base == 8 || base == 16); + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); return -1; @@ -2045,7 +2367,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (writer) { if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); @@ -2056,7 +2377,6 @@ long_format_binary(PyObject *aa, int base, int alternate, v = PyUnicode_New(sz, 'x'); if (v == NULL) return -1; - kind = PyUnicode_KIND(v); } #define WRITE_DIGITS(p) \ @@ -2117,19 +2437,23 @@ long_format_binary(PyObject *aa, int base, int alternate, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(v); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -2843,8 +3167,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, PyLongObject *z; if (size_b == 0) { - PyErr_SetString(PyExc_ZeroDivisionError, - "integer division or modulo by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return -1; } if (size_a < size_b || @@ -2907,7 +3230,7 @@ long_rem(PyLongObject *a, PyLongObject *b, PyLongObject **prem) if (size_b == 0) { PyErr_SetString(PyExc_ZeroDivisionError, - "integer modulo by zero"); + "division by zero"); return -1; } if (size_a < size_b || @@ -3298,7 +3621,7 @@ long_hash(PyLongObject *v) int sign; if (_PyLong_IsCompact(v)) { - x = _PyLong_CompactValue(v); + x = (Py_uhash_t)_PyLong_CompactValue(v); if (x == (Py_uhash_t)-1) { x = (Py_uhash_t)-2; } @@ -3529,7 +3852,7 @@ x_mul(PyLongObject *a, PyLongObject *b) memset(z->long_value.ob_digit, 0, _PyLong_DigitCount(z) * sizeof(digit)); if (a == b) { /* Efficient squaring per HAC, Algorithm 14.16: - * http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf + * https://cacr.uwaterloo.ca/hac/about/chap14.pdf * Gives slightly less than a 2x speedup when a == b, * via exploiting that each entry in the multiplication * pyramid appears twice (except for the size_a squares). @@ -4737,7 +5060,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } else if (i <= HUGE_EXP_CUTOFF / PyLong_SHIFT ) { /* Left-to-right binary exponentiation (HAC Algorithm 14.79) */ - /* http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf */ + /* https://cacr.uwaterloo.ca/hac/about/chap14.pdf */ /* Find the first significant exponent bit. Search right to left * because we're primarily trying to cut overhead for small powers. @@ -5768,7 +6091,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /*[clinic input] int.__round__ - ndigits as o_ndigits: object = NULL + ndigits as o_ndigits: object = None / Rounding an Integral returns itself. @@ -5778,7 +6101,7 @@ Rounding with an ndigits argument also returns an integer. static PyObject * int___round___impl(PyObject *self, PyObject *o_ndigits) -/*[clinic end generated code: output=954fda6b18875998 input=1614cf23ec9e18c3]*/ +/*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/ { PyObject *temp, *result, *ndigits; @@ -5796,7 +6119,7 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) * * m - divmod_near(m, 10**n)[1]. */ - if (o_ndigits == NULL) + if (o_ndigits == Py_None) return long_long(self); ndigits = _PyNumber_Index(o_ndigits); @@ -6025,7 +6348,7 @@ int.to_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Determines whether two's complement is used to represent the integer. @@ -6038,7 +6361,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=d42ecfb545039d71]*/ +/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ { int little_endian; PyObject *bytes; @@ -6067,7 +6390,7 @@ int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, if (_PyLong_AsByteArray((PyLongObject *)self, (unsigned char *)PyBytes_AS_STRING(bytes), - length, little_endian, is_signed) < 0) { + length, little_endian, is_signed, 1) < 0) { Py_DECREF(bytes); return NULL; } @@ -6089,7 +6412,7 @@ int.from_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Indicates whether two's complement is used to represent the integer. @@ -6100,7 +6423,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=33326dccdd655553]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6152,6 +6475,29 @@ int_is_integer_impl(PyObject *self) Py_RETURN_TRUE; } +static PyObject * +long_vectorcall(PyObject *type, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (kwnames != NULL) { + PyThreadState *tstate = PyThreadState_GET(); + return _PyObject_MakeTpCall(tstate, type, args, nargs, kwnames); + } + switch (nargs) { + case 0: + return _PyLong_GetZero(); + case 1: + return PyNumber_Long(args[0]); + case 2: + return long_new_impl(_PyType_CAST(type), args[0], args[1]); + default: + return PyErr_Format(PyExc_TypeError, + "int expected at most 2 arguments, got %zd", + nargs); + } +} + static PyMethodDef long_methods[] = { {"conjugate", long_long_meth, METH_NOARGS, "Returns self, the complex conjugate of any int."}, @@ -6289,6 +6635,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_alloc */ long_new, /* tp_new */ PyObject_Free, /* tp_free */ + .tp_vectorcall = long_vectorcall, }; static PyTypeObject Int_InfoType; @@ -6371,12 +6718,12 @@ _PyLong_FiniTypes(PyInterpreterState *interp) int PyUnstable_Long_IsCompact(const PyLongObject* op) { - return _PyLong_IsCompact(op); + return _PyLong_IsCompact((PyLongObject*)op); } #undef PyUnstable_Long_CompactValue Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) { - return _PyLong_CompactValue(op); + return _PyLong_CompactValue((PyLongObject*)op); } diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index bcdd2ff0ceafe6..226bd6defdec5a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -119,8 +119,9 @@ mbuf_release(_PyManagedBufferObject *self) } static void -mbuf_dealloc(_PyManagedBufferObject *self) +mbuf_dealloc(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports == 0); mbuf_release(self); if (self->flags&_Py_MANAGED_BUFFER_FREE_FORMAT) @@ -129,15 +130,17 @@ mbuf_dealloc(_PyManagedBufferObject *self) } static int -mbuf_traverse(_PyManagedBufferObject *self, visitproc visit, void *arg) +mbuf_traverse(PyObject *_self, visitproc visit, void *arg) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; Py_VISIT(self->master.obj); return 0; } static int -mbuf_clear(_PyManagedBufferObject *self) +mbuf_clear(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports >= 0); mbuf_release(self); return 0; @@ -148,7 +151,7 @@ PyTypeObject _PyManagedBuffer_Type = { "managedbuffer", sizeof(_PyManagedBufferObject), 0, - (destructor)mbuf_dealloc, /* tp_dealloc */ + mbuf_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -165,8 +168,8 @@ PyTypeObject _PyManagedBuffer_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)mbuf_traverse, /* tp_traverse */ - (inquiry)mbuf_clear /* tp_clear */ + mbuf_traverse, /* tp_traverse */ + mbuf_clear /* tp_clear */ }; @@ -265,7 +268,7 @@ PyTypeObject _PyManagedBuffer_Type = { /* Assumptions: ndim >= 1. The macro tests for a corner case that should perhaps be explicitly forbidden in the PEP. */ #define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \ - (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0) + (view->suboffsets && view->suboffsets[view->ndim-1] >= 0) static inline int last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src) @@ -1137,8 +1140,9 @@ memoryview_release_impl(PyMemoryViewObject *self) } static void -memory_dealloc(PyMemoryViewObject *self) +memory_dealloc(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; assert(self->exports == 0); _PyObject_GC_UNTRACK(self); (void)_memory_release(self); @@ -1149,15 +1153,17 @@ memory_dealloc(PyMemoryViewObject *self) } static int -memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +memory_traverse(PyObject *_self, visitproc visit, void *arg) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_VISIT(self->mbuf); return 0; } static int -memory_clear(PyMemoryViewObject *self) +memory_clear(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; (void)_memory_release(self); Py_CLEAR(self->mbuf); return 0; @@ -1510,8 +1516,9 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /**************************************************************************/ static int -memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +memory_getbuf(PyObject *_self, Py_buffer *view, int flags) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *base = &self->view; int baseflags = self->flags; @@ -1589,8 +1596,9 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) } static void -memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +memory_releasebuf(PyObject *_self, Py_buffer *view) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; self->exports--; return; /* PyBuffer_Release() decrements view->obj after this function returns. */ @@ -1598,8 +1606,8 @@ memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) /* Buffer methods */ static PyBufferProcs memory_as_buffer = { - (getbufferproc)memory_getbuf, /* bf_getbuffer */ - (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ + memory_getbuf, /* bf_getbuffer */ + memory_releasebuf, /* bf_releasebuffer */ }; @@ -2344,8 +2352,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, } static PyObject * -memory_repr(PyMemoryViewObject *self) +memory_repr(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->flags & _Py_MEMORYVIEW_RELEASED) return PyUnicode_FromFormat("", self); else @@ -2421,8 +2430,9 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) with the type specified by view->format. Otherwise, the item is a sub-view. The function is used in memory_subscript() and memory_as_sequence. */ static PyObject * -memory_item(PyMemoryViewObject *self, Py_ssize_t index) +memory_item(PyObject *_self, Py_ssize_t index) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); const char *fmt; @@ -2546,8 +2556,9 @@ is_multiindex(PyObject *key) 0-d memoryview objects can be referenced using mv[...] or mv[()] but not with anything else. */ static PyObject * -memory_subscript(PyMemoryViewObject *self, PyObject *key) +memory_subscript(PyObject *_self, PyObject *key) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view; view = &(self->view); @@ -2575,7 +2586,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) index = PyNumber_AsSsize_t(key, PyExc_IndexError); if (index == -1 && PyErr_Occurred()) return NULL; - return memory_item(self, index); + return memory_item((PyObject *)self, index); } else if (PySlice_Check(key)) { CHECK_RESTRICTED(self); @@ -2608,8 +2619,9 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) } static int -memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) +memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); Py_buffer src; const char *fmt; @@ -2710,8 +2722,9 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) } static Py_ssize_t -memory_length(PyMemoryViewObject *self) +memory_length(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED_INT(self); if (self->view.ndim == 0) { PyErr_SetString(PyExc_TypeError, "0-dim memory has no length"); @@ -2722,17 +2735,17 @@ memory_length(PyMemoryViewObject *self) /* As mapping */ static PyMappingMethods memory_as_mapping = { - (lenfunc)memory_length, /* mp_length */ - (binaryfunc)memory_subscript, /* mp_subscript */ - (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ + memory_length, /* mp_length */ + memory_subscript, /* mp_subscript */ + memory_ass_sub, /* mp_ass_subscript */ }; /* As sequence */ static PySequenceMethods memory_as_sequence = { - (lenfunc)memory_length, /* sq_length */ + memory_length, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ - (ssizeargfunc)memory_item, /* sq_item */ + memory_item, /* sq_item */ }; @@ -3034,8 +3047,9 @@ memory_richcompare(PyObject *v, PyObject *w, int op) /**************************************************************************/ static Py_hash_t -memory_hash(PyMemoryViewObject *self) +memory_hash(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->hash == -1) { Py_buffer *view = &self->view; char *mem = view->buf; @@ -3112,8 +3126,9 @@ _IntTupleFromSsizet(int len, Py_ssize_t *vals) } static PyObject * -memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_obj_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &self->view; CHECK_RELEASED(self); @@ -3124,78 +3139,89 @@ memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) } static PyObject * -memory_nbytes_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_nbytes_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.len); } static PyObject * -memory_format_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_format_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyUnicode_FromString(self->view.format); } static PyObject * -memory_itemsize_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_itemsize_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.itemsize); } static PyObject * -memory_shape_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_shape_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.shape); } static PyObject * -memory_strides_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_strides_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.strides); } static PyObject * -memory_suboffsets_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_suboffsets_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); } static PyObject * -memory_readonly_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_readonly_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(self->view.readonly); } static PyObject * -memory_ndim_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_ndim_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromLong(self->view.ndim); } static PyObject * -memory_c_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_c_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_C_CONTIGUOUS(self->flags)); } static PyObject * -memory_f_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_f_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_F_CONTIGUOUS(self->flags)); } static PyObject * -memory_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_ANY_CONTIGUOUS(self->flags)); } @@ -3229,21 +3255,24 @@ PyDoc_STRVAR(memory_f_contiguous_doc, "A bool indicating whether the memory is Fortran contiguous."); PyDoc_STRVAR(memory_contiguous_doc, "A bool indicating whether the memory is contiguous."); +PyDoc_STRVAR(memory_exit_doc, + "__exit__($self, /, *exc_info)\n--\n\n" + "Release the underlying buffer exposed by the memoryview object."); static PyGetSetDef memory_getsetlist[] = { - {"obj", (getter)memory_obj_get, NULL, memory_obj_doc}, - {"nbytes", (getter)memory_nbytes_get, NULL, memory_nbytes_doc}, - {"readonly", (getter)memory_readonly_get, NULL, memory_readonly_doc}, - {"itemsize", (getter)memory_itemsize_get, NULL, memory_itemsize_doc}, - {"format", (getter)memory_format_get, NULL, memory_format_doc}, - {"ndim", (getter)memory_ndim_get, NULL, memory_ndim_doc}, - {"shape", (getter)memory_shape_get, NULL, memory_shape_doc}, - {"strides", (getter)memory_strides_get, NULL, memory_strides_doc}, - {"suboffsets", (getter)memory_suboffsets_get, NULL, memory_suboffsets_doc}, - {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, - {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, - {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + {"obj", memory_obj_get, NULL, memory_obj_doc}, + {"nbytes", memory_nbytes_get, NULL, memory_nbytes_doc}, + {"readonly", memory_readonly_get, NULL, memory_readonly_doc}, + {"itemsize", memory_itemsize_get, NULL, memory_itemsize_doc}, + {"format", memory_format_get, NULL, memory_format_doc}, + {"ndim", memory_ndim_get, NULL, memory_ndim_doc}, + {"shape", memory_shape_get, NULL, memory_shape_doc}, + {"strides", memory_strides_get, NULL, memory_strides_doc}, + {"suboffsets", memory_suboffsets_get, NULL, memory_suboffsets_doc}, + {"c_contiguous", memory_c_contiguous, NULL, memory_c_contiguous_doc}, + {"f_contiguous", memory_f_contiguous, NULL, memory_f_contiguous_doc}, + {"contiguous", memory_contiguous, NULL, memory_contiguous_doc}, {NULL, NULL, NULL, NULL}, }; @@ -3257,7 +3286,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, - {"__exit__", memory_exit, METH_VARARGS, NULL}, + {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, {NULL, NULL} }; @@ -3276,23 +3305,26 @@ typedef struct { } memoryiterobject; static void -memoryiter_dealloc(memoryiterobject *it) +memoryiter_dealloc(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -memoryiter_traverse(memoryiterobject *it, visitproc visit, void *arg) +memoryiter_traverse(PyObject *self, visitproc visit, void *arg) { + memoryiterobject *it = (memoryiterobject *)self; Py_VISIT(it->it_seq); return 0; } static PyObject * -memoryiter_next(memoryiterobject *it) +memoryiter_next(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; PyMemoryViewObject *seq; seq = it->it_seq; if (seq == NULL) { @@ -3347,7 +3379,7 @@ memory_iter(PyObject *seq) return NULL; } it->it_fmt = fmt; - it->it_length = memory_length(obj); + it->it_length = memory_length((PyObject *)obj); it->it_index = 0; it->it_seq = (PyMemoryViewObject*)Py_NewRef(obj); _PyObject_GC_TRACK(it); @@ -3359,12 +3391,12 @@ PyTypeObject _PyMemoryIter_Type = { .tp_name = "memory_iterator", .tp_basicsize = sizeof(memoryiterobject), // methods - .tp_dealloc = (destructor)memoryiter_dealloc, + .tp_dealloc = memoryiter_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)memoryiter_traverse, + .tp_traverse = memoryiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)memoryiter_next, + .tp_iternext = memoryiter_next, }; PyTypeObject PyMemoryView_Type = { @@ -3372,16 +3404,16 @@ PyTypeObject PyMemoryView_Type = { "memoryview", /* tp_name */ offsetof(PyMemoryViewObject, ob_array), /* tp_basicsize */ sizeof(Py_ssize_t), /* tp_itemsize */ - (destructor)memory_dealloc, /* tp_dealloc */ + memory_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)memory_repr, /* tp_repr */ + memory_repr, /* tp_repr */ 0, /* tp_as_number */ &memory_as_sequence, /* tp_as_sequence */ &memory_as_mapping, /* tp_as_mapping */ - (hashfunc)memory_hash, /* tp_hash */ + memory_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -3390,8 +3422,8 @@ PyTypeObject PyMemoryView_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_SEQUENCE, /* tp_flags */ memoryview__doc__, /* tp_doc */ - (traverseproc)memory_traverse, /* tp_traverse */ - (inquiry)memory_clear, /* tp_clear */ + memory_traverse, /* tp_traverse */ + memory_clear, /* tp_clear */ memory_richcompare, /* tp_richcompare */ offsetof(PyMemoryViewObject, weakreflist),/* tp_weaklistoffset */ memory_iter, /* tp_iter */ diff --git a/Objects/methodobject.c b/Objects/methodobject.c index b40b2821c3880d..d6773a264101dc 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -320,7 +320,7 @@ static Py_hash_t meth_hash(PyCFunctionObject *a) { Py_hash_t x, y; - x = _Py_HashPointer(a->m_self); + x = PyObject_GenericHash(a->m_self); y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); x ^= y; if (x == -1) @@ -417,7 +417,7 @@ cfunction_vectorcall_FASTCALL( return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFast meth = (_PyCFunctionFast) + PyCFunctionFast meth = (PyCFunctionFast) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -433,7 +433,7 @@ cfunction_vectorcall_FASTCALL_KEYWORDS( { PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) + PyCFunctionFastWithKeywords meth = (PyCFunctionFastWithKeywords) cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; @@ -552,4 +552,3 @@ cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs) } return _Py_CheckFunctionResult(tstate, func, result, NULL); } - diff --git a/Objects/mimalloc/alloc-aligned.c b/Objects/mimalloc/alloc-aligned.c new file mode 100644 index 00000000000000..4c15f4043ec117 --- /dev/null +++ b/Objects/mimalloc/alloc-aligned.c @@ -0,0 +1,298 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // mi_prim_get_default_heap + +#include // memset + +// ------------------------------------------------------ +// Aligned Allocation +// ------------------------------------------------------ + +// Fallback primitive aligned allocation -- split out for better codegen +static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +{ + mi_assert_internal(size <= PTRDIFF_MAX); + mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); + + const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; + + // use regular allocation if it is guaranteed to fit the alignment constraints + if (offset==0 && alignment<=padsize && padsize<=MI_MAX_ALIGN_GUARANTEE && (padsize&align_mask)==0) { + void* p = _mi_heap_malloc_zero(heap, size, zero); + mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); + return p; + } + + void* p; + size_t oversize; + if mi_unlikely(alignment > MI_ALIGNMENT_MAX) { + // use OS allocation for very large alignment and allocate inside a huge page (dedicated segment with 1 page) + // This can support alignments >= MI_SEGMENT_SIZE by ensuring the object can be aligned at a point in the + // first (and single) page such that the segment info is `MI_SEGMENT_SIZE` bytes before it (so it can be found by aligning the pointer down) + if mi_unlikely(offset != 0) { + // todo: cannot support offset alignment for very large alignments yet + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); + #endif + return NULL; + } + oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); + p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block + // zero afterwards as only the area from the aligned_p may be committed! + if (p == NULL) return NULL; + } + else { + // otherwise over-allocate + oversize = size + alignment - 1; + p = _mi_heap_malloc_zero(heap, oversize, zero); + if (p == NULL) return NULL; + } + + // .. and align within the allocation + const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask; + const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset); + mi_assert_internal(adjust < alignment); + void* aligned_p = (void*)((uintptr_t)p + adjust); + if (aligned_p != p) { + mi_page_t* page = _mi_ptr_page(p); + mi_page_set_has_aligned(page, true); + _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); + } + // todo: expand padding if overallocated ? + + mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); + mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); + mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); + mi_assert_internal(mi_usable_size(aligned_p)>=size); + mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); + + // now zero the block if needed + if (alignment > MI_ALIGNMENT_MAX) { + // for the tracker, on huge aligned allocations only from the start of the large block is defined + mi_track_mem_undefined(aligned_p, size); + if (zero) { + _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p)); + } + } + + if (p != aligned_p) { + mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); + } + return aligned_p; +} + +// Primitive aligned allocation +static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +{ + // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. + if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment); + #endif + return NULL; + } + + if mi_unlikely(size > PTRDIFF_MAX) { // we don't allocate more than PTRDIFF_MAX (see ) + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); + #endif + return NULL; + } + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; // note: cannot overflow due to earlier size > PTRDIFF_MAX check + + // try first if there happens to be a small block available with just the right alignment + if mi_likely(padsize <= MI_SMALL_SIZE_MAX && alignment <= padsize) { + mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); + const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; + if mi_likely(page->free != NULL && is_aligned) + { + #if MI_STAT>1 + mi_heap_stat_increase(heap, malloc, size); + #endif + void* p = _mi_page_malloc(heap, page, padsize, zero); // TODO: inline _mi_page_malloc + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_track_malloc(p,size,zero); + return p; + } + } + // fallback + return mi_heap_malloc_zero_aligned_at_fallback(heap, size, alignment, offset, zero); +} + + +// ------------------------------------------------------ +// Optimized mi_heap_malloc_aligned / mi_malloc_aligned +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { + if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) return NULL; + #if !MI_PADDING + // without padding, any small sized allocation is naturally aligned (see also `_mi_segment_page_start`) + if mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX) + #else + // with padding, we can only guarantee this for fixed alignments + if mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))) + && size <= MI_SMALL_SIZE_MAX) + #endif + { + // fast path for common alignment and size + return mi_heap_malloc_small(heap, size); + } + else { + return mi_heap_malloc_aligned_at(heap, size, alignment, 0); + } +} + +// ensure a definition is emitted +#if defined(__cplusplus) +static void* _mi_heap_malloc_aligned = (void*)&mi_heap_malloc_aligned; +#endif + +// ------------------------------------------------------ +// Aligned Allocation +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_zalloc_aligned_at(heap, size, alignment, 0); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_zalloc_aligned_at(heap, total, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_calloc_aligned_at(heap,count,size,alignment,0); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_malloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_calloc_aligned(mi_prim_get_default_heap(), count, size, alignment); +} + + +// ------------------------------------------------------ +// Aligned re-allocation +// ------------------------------------------------------ + +static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { + mi_assert(alignment > 0); + if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); + if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero); + size_t size = mi_usable_size(p); + if (newsize <= size && newsize >= (size - (size / 2)) + && (((uintptr_t)p + offset) % alignment) == 0) { + return p; // reallocation still fits, is aligned and not more than 50% waste + } + else { + // note: we don't zero allocate upfront so we only zero initialize the expanded part + void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset); + if (newp != NULL) { + if (zero && newsize > size) { + // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); + _mi_memzero((uint8_t*)newp + start, newsize - start); + } + _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); + mi_free(p); // only free if successful + } + return newp; + } +} + +static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { + mi_assert(alignment > 0); + if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); + size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL) + return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); +} + +mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false); +} + +mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false); +} + +mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true); +} + +mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true); +} + +mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(newcount, size, &total)) return NULL; + return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset); +} + +mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(newcount, size, &total)) return NULL; + return mi_heap_rezalloc_aligned(heap, p, total, alignment); +} + +mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_realloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_heap_realloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_rezalloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_heap_rezalloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_heap_recalloc_aligned_at(mi_prim_get_default_heap(), p, newcount, size, alignment, offset); +} + +mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { + return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment); +} diff --git a/Objects/mimalloc/alloc-override.c b/Objects/mimalloc/alloc-override.c new file mode 100644 index 00000000000000..873065dc63495c --- /dev/null +++ b/Objects/mimalloc/alloc-override.c @@ -0,0 +1,297 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#if !defined(MI_IN_ALLOC_C) +#error "this file should be included from 'alloc.c' (so aliases can work)" +#endif + +#if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL)) +#error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" +#endif + +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) + +#if defined(__APPLE__) +#include +mi_decl_externc void vfree(void* p); +mi_decl_externc size_t malloc_size(const void* p); +mi_decl_externc size_t malloc_good_size(size_t size); +#endif + +// helper definition for C override of C++ new +typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; + +// ------------------------------------------------------ +// Override system malloc +// ------------------------------------------------------ + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED + // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions + #if (defined(__GNUC__) && __GNUC__ >= 9) + #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); + #else + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); + #endif + #define MI_FORWARD1(fun,x) MI_FORWARD(fun) + #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) + #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) + #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) +#else + // otherwise use forwarding by calling our `mi_` function + #define MI_FORWARD1(fun,x) { return fun(x); } + #define MI_FORWARD2(fun,x,y) { return fun(x,y); } + #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } + #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD02(fun,x,y) { fun(x,y); } +#endif + + +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for + // functions that are interposed (or the interposing does not work) + #define MI_OSX_IS_INTERPOSED + + mi_decl_externc size_t mi_malloc_size_checked(void *p) { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); + } + + // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` + // See: + struct mi_interpose_s { + const void* replacement; + const void* target; + }; + #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } + #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) + + __attribute__((used)) static struct mi_interpose_s _mi_interposes[] __attribute__((section("__DATA, __interpose"))) = + { + MI_INTERPOSE_MI(malloc), + MI_INTERPOSE_MI(calloc), + MI_INTERPOSE_MI(realloc), + MI_INTERPOSE_MI(strdup), + MI_INTERPOSE_MI(strndup), + MI_INTERPOSE_MI(realpath), + MI_INTERPOSE_MI(posix_memalign), + MI_INTERPOSE_MI(reallocf), + MI_INTERPOSE_MI(valloc), + MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked), + MI_INTERPOSE_MI(malloc_good_size), + #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + MI_INTERPOSE_MI(aligned_alloc), + #endif + #ifdef MI_OSX_ZONE + // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely + MI_INTERPOSE_MI(free), + MI_INTERPOSE_FUN(vfree,mi_free), + #else + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), + #endif + }; + + #ifdef __cplusplus + extern "C" { + #endif + void _ZdlPv(void* p); // delete + void _ZdaPv(void* p); // delete[] + void _ZdlPvm(void* p, size_t n); // delete + void _ZdaPvm(void* p, size_t n); // delete[] + void* _Znwm(size_t n); // new + void* _Znam(size_t n); // new[] + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow + #ifdef __cplusplus + } + #endif + __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = + { + MI_INTERPOSE_FUN(_ZdlPv,mi_free), + MI_INTERPOSE_FUN(_ZdaPv,mi_free), + MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size), + MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size), + MI_INTERPOSE_FUN(_Znwm,mi_new), + MI_INTERPOSE_FUN(_Znam,mi_new), + MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow), + MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow), + }; + +#elif defined(_MSC_VER) + // cannot override malloc unless using a dll. + // we just override new/delete which does work in a static library. +#else + // On all other systems forward to our API + mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) + mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) + mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) + mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p) +#endif + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +#pragma GCC visibility push(default) +#endif + +// ------------------------------------------------------ +// Override new/delete +// This is not really necessary as they usually call +// malloc/free anyway, but it improves performance. +// ------------------------------------------------------ +#ifdef __cplusplus + // ------------------------------------------------------ + // With a C++ compiler we override the new/delete operators. + // see + // ------------------------------------------------------ + #include + + #ifndef MI_OSX_IS_INTERPOSED + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) + + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + #endif + #endif + + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + #endif + +#elif (defined(__GNUC__) || defined(__clang__)) + // ------------------------------------------------------ + // Override by defining the mangled C++ names of the operators (as + // used by GCC and CLang). + // See + // ------------------------------------------------------ + + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + + #if (MI_INTPTR_SIZE==8) + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + #elif (MI_INTPTR_SIZE==4) + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + #else + #error "define overloads for new/delete for this platform (just for performance, can be skipped)" + #endif +#endif // __cplusplus + +// ------------------------------------------------------ +// Further Posix & Unix functions definitions +// ------------------------------------------------------ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef MI_OSX_IS_INTERPOSED + // Forward Posix/Unix calls as well + void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) + size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) + #else + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + #endif + + // No forwarding here due to aliasing/name mangling issues + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + + // `aligned_alloc` is only available when __USE_ISOC11 is defined. + // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check + // for it if using glibc. + // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot + // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. + // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it + // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. + #if !defined(__GLIBC__) || __USE_ISOC11 + void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + #endif +#endif + +// no forwarding here due to aliasing/name mangling issues +void cfree(void* p) { mi_free(p); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + +#if defined(__wasi__) + // forward __libc interface (see PR #667) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) + void __libc_free(void* p) MI_FORWARD0(mi_free, p) + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } + +#elif defined(__GLIBC__) && defined(__linux__) + // forward __libc interface (needed for glibc-based Linux distributions) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) + + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } +#endif + +#ifdef __cplusplus +} +#endif + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +#pragma GCC visibility pop +#endif + +#endif // MI_MALLOC_OVERRIDE && !_WIN32 diff --git a/Objects/mimalloc/alloc-posix.c b/Objects/mimalloc/alloc-posix.c new file mode 100644 index 00000000000000..225752fd8707fe --- /dev/null +++ b/Objects/mimalloc/alloc-posix.c @@ -0,0 +1,185 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// ------------------------------------------------------------------------ +// mi prefixed publi definitions of various Posix, Unix, and C++ functions +// for convenience and used when overriding these functions. +// ------------------------------------------------------------------------ +#include "mimalloc.h" +#include "mimalloc/internal.h" + +// ------------------------------------------------------ +// Posix & Unix functions definitions +// ------------------------------------------------------ + +#include +#include // memset +#include // getenv + +#ifdef _MSC_VER +#pragma warning(disable:4996) // getenv _wgetenv +#endif + +#ifndef EINVAL +#define EINVAL 22 +#endif +#ifndef ENOMEM +#define ENOMEM 12 +#endif + + +mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { + // if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + // if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { + return mi_good_size(size); +} + +void mi_cfree(void* p) mi_attr_noexcept { + if (mi_is_in_heap_region(p)) { + mi_free(p); + } +} + +int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { + // Note: The spec dictates we should not modify `*p` on an error. (issue#27) + // + if (p == NULL) return EINVAL; + if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment + // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned` + if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 + void* q = mi_malloc_aligned(size, alignment); + if (q==NULL && size != 0) return ENOMEM; + mi_assert_internal(((uintptr_t)q % alignment) == 0); + *p = q; + return 0; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept { + void* p = mi_malloc_aligned(size, alignment); + mi_assert_internal(((uintptr_t)p % alignment) == 0); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept { + return mi_memalign( _mi_os_page_size(), size ); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept { + size_t psize = _mi_os_page_size(); + if (size >= SIZE_MAX - psize) return NULL; // overflow + size_t asize = _mi_align_up(size, psize); + return mi_malloc_aligned(asize, psize); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { + // C11 requires the size to be an integral multiple of the alignment, see . + // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check.. + /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment); + #endif + return NULL; + } + */ + // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned + void* p = mi_malloc_aligned(size, alignment); + mi_assert_internal(((uintptr_t)p % alignment) == 0); + return p; +} + +mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD + void* newp = mi_reallocn(p,count,size); + if (newp==NULL) { errno = ENOMEM; } + return newp; +} + +mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD + mi_assert(p != NULL); + if (p == NULL) { + errno = EINVAL; + return EINVAL; + } + void** op = (void**)p; + void* newp = mi_reallocarray(*op, count, size); + if mi_unlikely(newp == NULL) { return errno; } + *op = newp; + return 0; +} + +void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft + void* res = mi_expand(p, newsize); + if (res == NULL) { errno = ENOMEM; } + return res; +} + +mi_decl_nodiscard mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept { + if (s==NULL) return NULL; + size_t len; + for(len = 0; s[len] != 0; len++) { } + size_t size = (len+1)*sizeof(unsigned short); + unsigned short* p = (unsigned short*)mi_malloc(size); + if (p != NULL) { + _mi_memcpy(p,s,size); + } + return p; +} + +mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept { + return (unsigned char*)mi_strdup((const char*)s); +} + +int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { + if (buf==NULL || name==NULL) return EINVAL; + if (size != NULL) *size = 0; + char* p = getenv(name); // mscver warning 4996 + if (p==NULL) { + *buf = NULL; + } + else { + *buf = mi_strdup(p); + if (*buf==NULL) return ENOMEM; + if (size != NULL) *size = _mi_strlen(p); + } + return 0; +} + +int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept { + if (buf==NULL || name==NULL) return EINVAL; + if (size != NULL) *size = 0; +#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP)) + // not supported + *buf = NULL; + return EINVAL; +#else + unsigned short* p = (unsigned short*)_wgetenv((const wchar_t*)name); // msvc warning 4996 + if (p==NULL) { + *buf = NULL; + } + else { + *buf = mi_wcsdup(p); + if (*buf==NULL) return ENOMEM; + if (size != NULL) *size = wcslen((const wchar_t*)p); + } + return 0; +#endif +} + +mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft + return mi_recalloc_aligned_at(p, newcount, size, alignment, offset); +} + +mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft + return mi_recalloc_aligned(p, newcount, size, alignment); +} diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c new file mode 100644 index 00000000000000..c133f23fc9830d --- /dev/null +++ b/Objects/mimalloc/alloc.c @@ -0,0 +1,1074 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // _mi_prim_thread_id() + +#include // memset, strlen (for mi_strdup) +#include // malloc, abort + +#define _ZSt15get_new_handlerv _Py__ZSt15get_new_handlerv + +#define MI_IN_ALLOC_C +#include "alloc-override.c" +#undef MI_IN_ALLOC_C + +// ------------------------------------------------------ +// Allocation +// ------------------------------------------------------ + +#if (MI_DEBUG>0) +static inline void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { + size_t offset = (size_t)page->debug_offset; + if (offset < size) { + memset((char*)block + offset, c, size - offset); + } +} +#endif + +// Fast allocation in a page: just pop from the free list. +// Fall back to generic allocation only if the list is empty. +extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { + mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); + mi_block_t* const block = page->free; + if mi_unlikely(block == NULL) { + return _mi_malloc_generic(heap, size, zero, 0); + } + mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); + // pop from the free list + page->used++; + page->free = mi_block_next(page, block); + mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); + #if MI_DEBUG>3 + if (page->free_is_zero) { + mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); + } + #endif + + // allow use of the block internally + // note: when tracking we need to avoid ever touching the MI_PADDING since + // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`) + mi_track_mem_undefined(block, mi_page_usable_block_size(page)); + + // zero the block? note: we need to zero the full block size (issue #63) + if mi_unlikely(zero) { + mi_assert_internal(page->xblock_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic) + mi_assert_internal(page->xblock_size >= MI_PADDING_SIZE); + if (page->free_is_zero) { + block->next = 0; + mi_track_mem_defined(block, page->xblock_size - MI_PADDING_SIZE); + } + else { + _mi_memzero_aligned(block, page->xblock_size - MI_PADDING_SIZE); + } + } + +#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if (!zero && !mi_page_is_huge(page)) { + mi_debug_fill(page, block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); + } +#elif (MI_SECURE!=0) + if (!zero) { block->next = 0; } // don't leak internal data +#endif + +#if (MI_STAT>0) + const size_t bsize = mi_page_usable_block_size(page); + if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { + mi_heap_stat_increase(heap, normal, bsize); + mi_heap_stat_counter_increase(heap, normal_count, 1); +#if (MI_STAT>1) + const size_t bin = _mi_bin(bsize); + mi_heap_stat_increase(heap, normal_bins[bin], 1); +#endif + } +#endif + +#if MI_PADDING // && !MI_TRACK_ENABLED + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); + ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + #if (MI_DEBUG>=2) + mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); + #endif + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif +#endif + + return block; +} + +static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { + mi_assert(heap != NULL); + #if MI_DEBUG + const uintptr_t tid = _mi_thread_id(); + mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local + #endif + mi_assert(size <= MI_SMALL_SIZE_MAX); + #if (MI_PADDING) + if (size == 0) { size = sizeof(void*); } + #endif + mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); + void* const p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE, zero); + mi_track_malloc(p,size,zero); + #if MI_STAT>1 + if (p != NULL) { + if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } + mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + } + #endif + #if MI_DEBUG>3 + if (p != NULL && zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} + +// allocate a small block +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_heap_malloc_small_zero(heap, size, false); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { + return mi_heap_malloc_small(mi_prim_get_default_heap(), size); +} + +// The main allocation function +extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_assert_internal(huge_alignment == 0); + return mi_heap_malloc_small_zero(heap, size, zero); + } + else { + mi_assert(heap!=NULL); + mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local + void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic + mi_track_malloc(p,size,zero); + #if MI_STAT>1 + if (p != NULL) { + if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } + mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + } + #endif + #if MI_DEBUG>3 + if (p != NULL && zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; + } +} + +extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { + return _mi_heap_malloc_zero_ex(heap, size, zero, 0); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return _mi_heap_malloc_zero(heap, size, false); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { + return mi_heap_malloc(mi_prim_get_default_heap(), size); +} + +// zero initialized small block +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { + return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return _mi_heap_malloc_zero(heap, size, true); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { + return mi_heap_zalloc(mi_prim_get_default_heap(),size); +} + + +// ------------------------------------------------------ +// Check for double free in secure and debug mode +// This is somewhat expensive so only enabled for secure mode 4 +// ------------------------------------------------------ + +#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// linear check if the free list contains a specific element +static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { + while (list != NULL) { + if (elem==list) return true; + list = mi_block_next(page, list); + } + return false; +} + +static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { + // The decoded value is in the same page (or NULL). + // Walk the free lists to verify positively if it is already freed + if (mi_list_contains(page, page->free, block) || + mi_list_contains(page, page->local_free, block) || + mi_list_contains(page, mi_page_thread_free(page), block)) + { + _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); + return true; + } + return false; +} + +#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); } + +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + bool is_double_free = false; + mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field + if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? + { + // Suspicous: decoded value a in block is in the same page (or NULL) -- maybe a double free? + // (continue in separate function to improve code generation) + is_double_free = mi_check_is_double_freex(page, block); + } + return is_double_free; +} +#else +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(page); + MI_UNUSED(block); + return false; +} +#endif + +// --------------------------------------------------------------------------- +// Check for heap block overflow by setting up padding at the end of the block +// --------------------------------------------------------------------------- + +#if MI_PADDING // && !MI_TRACK_ENABLED +static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { + *bsize = mi_page_usable_block_size(page); + const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); + *delta = padding->delta; + uint32_t canary = padding->canary; + uintptr_t keys[2]; + keys[0] = page->keys[0]; + keys[1] = page->keys[1]; + bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize); + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); + return ok; +} + +// Return the exact usable size of a block. +static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + mi_assert_internal(ok); mi_assert_internal(delta <= bsize); + return (ok ? bsize - delta : 0); +} + +// When a non-thread-local block is freed, it becomes part of the thread delayed free +// list that is freed later by the owning heap. If the exact usable size is too small to +// contain the pointer for the delayed list, then shrink the padding (by decreasing delta) +// so it will later not trigger an overflow error in `mi_free_block`. +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + mi_assert_internal(ok); + if (!ok || (bsize - delta) >= min_size) return; // usually already enough space + mi_assert_internal(bsize >= min_size); + if (bsize < min_size) return; // should never happen + size_t new_delta = (bsize - min_size); + mi_assert_internal(new_delta < bsize); + mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); + padding->delta = (uint32_t)new_delta; + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); +} +#else +static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(block); + return mi_page_usable_block_size(page); +} + +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + MI_UNUSED(page); + MI_UNUSED(block); + MI_UNUSED(min_size); +} +#endif + +#if MI_PADDING && MI_PADDING_CHECK + +static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + *size = *wrong = bsize; + if (!ok) return false; + mi_assert_internal(bsize >= delta); + *size = bsize - delta; + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)block + bsize - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes + mi_track_mem_defined(fill, maxpad); + for (size_t i = 0; i < maxpad; i++) { + if (fill[i] != MI_DEBUG_PADDING) { + *wrong = bsize - delta + i; + ok = false; + break; + } + } + mi_track_mem_noaccess(fill, maxpad); + } + return ok; +} + +static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { + size_t size; + size_t wrong; + if (!mi_verify_padding(page,block,&size,&wrong)) { + _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, size, wrong ); + } +} + +#else + +static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(page); + MI_UNUSED(block); +} + +#endif + +// only maintain stats for smaller objects if requested +#if (MI_STAT>0) +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { + #if (MI_STAT < 2) + MI_UNUSED(block); + #endif + mi_heap_t* const heap = mi_heap_get_default(); + const size_t bsize = mi_page_usable_block_size(page); + #if (MI_STAT>1) + const size_t usize = mi_page_usable_size_of(page, block); + mi_heap_stat_decrease(heap, malloc, usize); + #endif + if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, normal, bsize); + #if (MI_STAT > 1) + mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); + #endif + } + else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, large, bsize); + } + else { + mi_heap_stat_decrease(heap, huge, bsize); + } +} +#else +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(page); MI_UNUSED(block); +} +#endif + +#if MI_HUGE_PAGE_ABANDON +#if (MI_STAT>0) +// maintain stats for huge objects +static void mi_stat_huge_free(const mi_page_t* page) { + mi_heap_t* const heap = mi_heap_get_default(); + const size_t bsize = mi_page_block_size(page); // to match stats in `page.c:mi_page_huge_alloc` + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, large, bsize); + } + else { + mi_heap_stat_decrease(heap, huge, bsize); + } +} +#else +static void mi_stat_huge_free(const mi_page_t* page) { + MI_UNUSED(page); +} +#endif +#endif + +// ------------------------------------------------------ +// Free +// ------------------------------------------------------ + +// multi-threaded free (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON) +static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) +{ + // The padding check may access the non-thread-owned page for the key values. + // that is safe as these are constant and the page won't be freed (as the block is not freed yet). + mi_check_padding(page, block); + _mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection + + // huge page segments are always abandoned and can be freed immediately + mi_segment_t* segment = _mi_page_segment(page); + if (segment->kind == MI_SEGMENT_HUGE) { + #if MI_HUGE_PAGE_ABANDON + // huge page segments are always abandoned and can be freed immediately + mi_stat_huge_free(page); + _mi_segment_huge_page_free(segment, page, block); + return; + #else + // huge pages are special as they occupy the entire segment + // as these are large we reset the memory occupied by the page so it is available to other threads + // (as the owning thread needs to actually free the memory later). + _mi_segment_huge_page_reset(segment, page, block); + #endif + } + + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading + if (segment->kind != MI_SEGMENT_HUGE) { // not for huge segments as we just reset the content + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_usable_size(block)); + } + #endif + + // Try to put the block on either the page-local thread free list, or the heap delayed free list. + mi_thread_free_t tfreex; + bool use_delayed; + mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); + do { + use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE); + if mi_unlikely(use_delayed) { + // unlikely: this only happens on the first concurrent free in a page that is in the full list + tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING); + } + else { + // usual: directly add to page thread_free list + mi_block_set_next(page, block, mi_tf_block(tfree)); + tfreex = mi_tf_set_block(tfree,block); + } + } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); + + if mi_unlikely(use_delayed) { + // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`) + mi_heap_t* const heap = (mi_heap_t*)(mi_atomic_load_acquire(&page->xheap)); //mi_page_heap(page); + mi_assert_internal(heap != NULL); + if (heap != NULL) { + // add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity) + mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); + do { + mi_block_set_nextx(heap,block,dfree, heap->keys); + } while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block)); + } + + // and reset the MI_DELAYED_FREEING flag + tfree = mi_atomic_load_relaxed(&page->xthread_free); + do { + tfreex = tfree; + mi_assert_internal(mi_tf_delayed(tfree) == MI_DELAYED_FREEING); + tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE); + } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); + } +} + +// regular free +static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block) +{ + // and push it on the free list + //const size_t bsize = mi_page_block_size(page); + if mi_likely(local) { + // owning thread can free a block directly + if mi_unlikely(mi_check_is_double_free(page, block)) return; + mi_check_padding(page, block); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if (!mi_page_is_huge(page)) { // huge page content may be already decommitted + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); + } + #endif + mi_block_set_next(page, block, page->local_free); + page->local_free = block; + page->used--; + if mi_unlikely(mi_page_all_free(page)) { + _mi_page_retire(page); + } + else if mi_unlikely(mi_page_is_in_full(page)) { + _mi_page_unfull(page); + } + } + else { + _mi_free_block_mt(page,block); + } +} + + +// Adjust a block that was allocated aligned, to the actual start of the block in the page. +mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) { + mi_assert_internal(page!=NULL && p!=NULL); + const size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL); + const size_t adjust = (diff % mi_page_block_size(page)); + return (mi_block_t*)((uintptr_t)p - adjust); +} + + +void mi_decl_noinline _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { + mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); + mi_stat_free(page, block); // stat_free may access the padding + mi_track_free_size(block, mi_page_usable_size_of(page,block)); + _mi_free_block(page, is_local, block); +} + +// Get the segment data belonging to a pointer +// This is just a single `and` in assembly but does further checks in debug mode +// (and secure mode) if this was a valid pointer. +static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg) +{ + MI_UNUSED(msg); + mi_assert(p != NULL); + +#if (MI_DEBUG>0) + if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { + _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); + return NULL; + } +#endif + + mi_segment_t* const segment = _mi_ptr_segment(p); + mi_assert_internal(segment != NULL); + +#if 0 && (MI_DEBUG>0) + if mi_unlikely(!mi_is_in_heap_region(p)) { + #if (MI_INTPTR_SIZE == 8 && defined(__linux__)) + if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640) + #else + { + #endif + _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" + "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); + if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { + _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); + } + } + } +#endif +#if (MI_DEBUG>0 || MI_SECURE>=4) + if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) { + _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); + return NULL; + } +#endif + + return segment; +} + +// Free a block +// fast path written carefully to prevent spilling on the stack +void mi_free(void* p) mi_attr_noexcept +{ + if mi_unlikely(p == NULL) return; + mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); + const bool is_local= (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_page_t* const page = _mi_segment_page_of(segment, p); + + if mi_likely(is_local) { // thread-local free? + if mi_likely(page->flags.full_aligned == 0) // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned) + { + mi_block_t* const block = (mi_block_t*)p; + if mi_unlikely(mi_check_is_double_free(page, block)) return; + mi_check_padding(page, block); + mi_stat_free(page, block); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); + #endif + mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned + mi_block_set_next(page, block, page->local_free); + page->local_free = block; + if mi_unlikely(--page->used == 0) { // using this expression generates better code than: page->used--; if (mi_page_all_free(page)) + _mi_page_retire(page); + } + } + else { + // page is full or contains (inner) aligned blocks; use generic path + _mi_free_generic(segment, page, true, p); + } + } + else { + // not thread-local; use generic path + _mi_free_generic(segment, page, false, p); + } +} + +// return true if successful +bool _mi_free_delayed_block(mi_block_t* block) { + // get segment and page + const mi_segment_t* const segment = _mi_ptr_segment(block); + mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); +#ifndef Py_GIL_DISABLED + // The GC traverses heaps of other threads, which can trigger this assert. + mi_assert_internal(_mi_thread_id() == segment->thread_id); +#endif + mi_page_t* const page = _mi_segment_page_of(segment, block); + + // Clear the no-delayed flag so delayed freeing is used again for this page. + // This must be done before collecting the free lists on this page -- otherwise + // some blocks may end up in the page `thread_free` list with no blocks in the + // heap `thread_delayed_free` list which may cause the page to be never freed! + // (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`) + if (!_mi_page_try_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */)) { + return false; + } + + // collect all other non-local frees to ensure up-to-date `used` count + _mi_page_free_collect(page, false); + + // and free the block (possibly freeing the page as well since used is updated) + _mi_free_block(page, true, block); + return true; +} + +// Bytes available in a block +mi_decl_noinline static size_t mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept { + const mi_block_t* block = _mi_page_ptr_unalign(segment, page, p); + const size_t size = mi_page_usable_size_of(page, block); + const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; + mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); + return (size - adjust); +} + +static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { + if (p == NULL) return 0; + const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); + const mi_page_t* const page = _mi_segment_page_of(segment, p); + if mi_likely(!mi_page_has_aligned(page)) { + const mi_block_t* block = (const mi_block_t*)p; + return mi_page_usable_size_of(page, block); + } + else { + // split out to separate routine for improved code generation + return mi_page_usable_aligned_size_of(segment, page, p); + } +} + +mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { + return _mi_usable_size(p, "mi_usable_size"); +} + + +// ------------------------------------------------------ +// Allocation extensions +// ------------------------------------------------------ + +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + MI_UNUSED_RELEASE(size); + mi_assert(p == NULL || size <= _mi_usable_size(p,"mi_free_size")); + mi_free(p); +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count,size,&total)) return NULL; + return mi_heap_zalloc(heap,total); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { + return mi_heap_calloc(mi_prim_get_default_heap(),count,size); +} + +// Uninitialized `calloc` +mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_malloc(heap, total); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { + return mi_heap_mallocn(mi_prim_get_default_heap(),count,size); +} + +// Expand (or shrink) in place (or fail) +void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { + #if MI_PADDING + // we do not shrink/expand with padding enabled + MI_UNUSED(p); MI_UNUSED(newsize); + return NULL; + #else + if (p == NULL) return NULL; + const size_t size = _mi_usable_size(p,"mi_expand"); + if (newsize > size) return NULL; + return p; // it fits + #endif +} + +void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + // if p == NULL then behave as malloc. + // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). + // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) + const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0) + if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) + mi_assert_internal(p!=NULL); + // todo: do not track as the usable size is still the same in the free; adjust potential padding? + // mi_track_resize(p,size,newsize) + // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } + return p; // reallocation still fits and not more than 50% waste + } + void* newp = mi_heap_malloc(heap,newsize); + if mi_likely(newp != NULL) { + if (zero && newsize > size) { + // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); + _mi_memzero((uint8_t*)newp + start, newsize - start); + } + else if (newsize == 0) { + ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) + } + if mi_likely(p != NULL) { + const size_t copysize = (newsize > size ? size : newsize); + mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking.. + _mi_memcpy(newp, p, copysize); + mi_free(p); // only free the original pointer if successful + } + } + return newp; +} + +mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + return _mi_heap_realloc_zero(heap, p, newsize, false); +} + +mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_realloc(heap, p, total); +} + + +// Reallocate but free `p` on errors +mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + void* newp = mi_heap_realloc(heap, p, newsize); + if (newp==NULL && p!=NULL) mi_free(p); + return newp; +} + +mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + return _mi_heap_realloc_zero(heap, p, newsize, true); +} + +mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_rezalloc(heap, p, total); +} + + +mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { + return mi_heap_realloc(mi_prim_get_default_heap(),p,newsize); +} + +mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size); +} + +// Reallocate but free `p` on errors +mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { + return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize); +} + +mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { + return mi_heap_rezalloc(mi_prim_get_default_heap(), p, newsize); +} + +mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size); +} + + + +// ------------------------------------------------------ +// strdup, strndup, and realpath +// ------------------------------------------------------ + +// `strdup` using mi_malloc +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { + if (s == NULL) return NULL; + size_t n = strlen(s); + char* t = (char*)mi_heap_malloc(heap,n+1); + if (t == NULL) return NULL; + _mi_memcpy(t, s, n); + t[n] = 0; + return t; +} + +mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { + return mi_heap_strdup(mi_prim_get_default_heap(), s); +} + +// `strndup` using mi_malloc +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { + if (s == NULL) return NULL; + const char* end = (const char*)memchr(s, 0, n); // find end of string in the first `n` characters (returns NULL if not found) + const size_t m = (end != NULL ? (size_t)(end - s) : n); // `m` is the minimum of `n` or the end-of-string + mi_assert_internal(m <= n); + char* t = (char*)mi_heap_malloc(heap, m+1); + if (t == NULL) return NULL; + _mi_memcpy(t, s, m); + t[m] = 0; + return t; +} + +mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { + return mi_heap_strndup(mi_prim_get_default_heap(),s,n); +} + +#ifndef __wasi__ +// `realpath` using mi_malloc +#ifdef _WIN32 +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif +#include +mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { + // todo: use GetFullPathNameW to allow longer file names + char buf[PATH_MAX]; + DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); + if (res == 0) { + errno = GetLastError(); return NULL; + } + else if (res > PATH_MAX) { + errno = EINVAL; return NULL; + } + else if (resolved_name != NULL) { + return resolved_name; + } + else { + return mi_heap_strndup(heap, buf, PATH_MAX); + } +} +#else +/* +#include // pathconf +static size_t mi_path_max(void) { + static size_t path_max = 0; + if (path_max <= 0) { + long m = pathconf("/",_PC_PATH_MAX); + if (m <= 0) path_max = 4096; // guess + else if (m < 256) path_max = 256; // at least 256 + else path_max = m; + } + return path_max; +} +*/ +char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { + if (resolved_name != NULL) { + return realpath(fname,resolved_name); + } + else { + char* rname = realpath(fname, NULL); + if (rname == NULL) return NULL; + char* result = mi_heap_strdup(heap, rname); + free(rname); // use regular free! (which may be redirected to our free but that's ok) + return result; + } + /* + const size_t n = mi_path_max(); + char* buf = (char*)mi_malloc(n+1); + if (buf == NULL) { + errno = ENOMEM; + return NULL; + } + char* rname = realpath(fname,buf); + char* result = mi_heap_strndup(heap,rname,n); // ok if `rname==NULL` + mi_free(buf); + return result; + } + */ +} +#endif + +mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { + return mi_heap_realpath(mi_prim_get_default_heap(),fname,resolved_name); +} +#endif + +/*------------------------------------------------------- +C++ new and new_aligned +The standard requires calling into `get_new_handler` and +throwing the bad_alloc exception on failure. If we compile +with a C++ compiler we can implement this precisely. If we +use a C compiler we cannot throw a `bad_alloc` exception +but we call `exit` instead (i.e. not returning). +-------------------------------------------------------*/ + +#ifdef __cplusplus +#include +static bool mi_try_new_handler(bool nothrow) { + #if defined(_MSC_VER) || (__cplusplus >= 201103L) + std::new_handler h = std::get_new_handler(); + #else + std::new_handler h = std::set_new_handler(); + std::set_new_handler(h); + #endif + if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); + if (!nothrow) { + throw std::bad_alloc(); + } + return false; + } + else { + h(); + return true; + } +} +#else +typedef void (*std_new_handler_t)(void); + +#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631 +std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) { + return NULL; +} +static std_new_handler_t mi_get_new_handler(void) { + return _ZSt15get_new_handlerv(); +} +#else +// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`. +static std_new_handler_t mi_get_new_handler() { + return NULL; +} +#endif + +static bool mi_try_new_handler(bool nothrow) { + std_new_handler_t h = mi_get_new_handler(); + if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); + if (!nothrow) { + abort(); // cannot throw in plain C, use abort + } + return false; + } + else { + h(); + return true; + } +} +#endif + +mi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) { + void* p = NULL; + while(p == NULL && mi_try_new_handler(nothrow)) { + p = mi_heap_malloc(heap,size); + } + return p; +} + +static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { + return mi_heap_try_new(mi_prim_get_default_heap(), size, nothrow); +} + + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { + void* p = mi_heap_malloc(heap,size); + if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { + return mi_heap_alloc_new(mi_prim_get_default_heap(), size); +} + + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { + size_t total; + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { + mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + return NULL; + } + else { + return mi_heap_alloc_new(heap,total); + } +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { + return mi_heap_alloc_new_n(mi_prim_get_default_heap(), size, count); +} + + +mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { + void* p = mi_malloc(size); + if mi_unlikely(p == NULL) return mi_try_new(size, true); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { + void* p; + do { + p = mi_malloc_aligned(size, alignment); + } + while(p == NULL && mi_try_new_handler(false)); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { + void* p; + do { + p = mi_malloc_aligned(size, alignment); + } + while(p == NULL && mi_try_new_handler(true)); + return p; +} + +mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { + void* q; + do { + q = mi_realloc(p, newsize); + } while (q == NULL && mi_try_new_handler(false)); + return q; +} + +mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { + size_t total; + if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { + mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + return NULL; + } + else { + return mi_new_realloc(p, total); + } +} + +// ------------------------------------------------------ +// ensure explicit external inline definitions are emitted! +// ------------------------------------------------------ + +#ifdef __cplusplus +void* _mi_externs[] = { + (void*)&_mi_page_malloc, + (void*)&_mi_heap_malloc_zero, + (void*)&_mi_heap_malloc_zero_ex, + (void*)&mi_malloc, + (void*)&mi_malloc_small, + (void*)&mi_zalloc_small, + (void*)&mi_heap_malloc, + (void*)&mi_heap_zalloc, + (void*)&mi_heap_malloc_small, + // (void*)&mi_heap_alloc_new, + // (void*)&mi_heap_alloc_new_n +}; +#endif diff --git a/Objects/mimalloc/arena.c b/Objects/mimalloc/arena.c new file mode 100644 index 00000000000000..f8883603860dce --- /dev/null +++ b/Objects/mimalloc/arena.c @@ -0,0 +1,935 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +"Arenas" are fixed area's of OS memory from which we can allocate +large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 4MiB). +In contrast to the rest of mimalloc, the arenas are shared between +threads and need to be accessed using atomic operations. + +Arenas are used to for huge OS page (1GiB) reservations or for reserving +OS memory upfront which can be improve performance or is sometimes needed +on embedded devices. We can also employ this with WASI or `sbrk` systems +to reserve large arenas upfront and be able to reuse the memory more effectively. + +The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" + +#include // memset +#include // ENOMEM + +#include "bitmap.h" // atomic bitmap + +/* ----------------------------------------------------------- + Arena allocation +----------------------------------------------------------- */ + +// Block info: bit 0 contains the `in_use` bit, the upper bits the +// size in count of arena blocks. +typedef uintptr_t mi_block_info_t; +#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN) +#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB +#define MI_MAX_ARENAS (112) // not more than 126 (since we use 7 bits in the memid and an arena index + 1) + +// A memory arena descriptor +typedef struct mi_arena_s { + mi_arena_id_t id; // arena id; 0 for non-specific + mi_memid_t memid; // memid of the memory area + _Atomic(uint8_t*) start; // the start of the memory area + size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) + size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) + size_t meta_size; // size of the arena structure itself (including its bitmaps) + mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) + int numa_node; // associated NUMA node + bool exclusive; // only allow allocations if specifically for this arena + bool is_large; // memory area consists of large- or huge OS pages (always committed) + _Atomic(size_t) search_idx; // optimization to start the search for free blocks + _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. + mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) + mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) + mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) +} mi_arena_t; + + +// The available arenas +static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; +static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 + + +//static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept; + +/* ----------------------------------------------------------- + Arena id's + id = arena_index + 1 +----------------------------------------------------------- */ + +static size_t mi_arena_id_index(mi_arena_id_t id) { + return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1); +} + +static mi_arena_id_t mi_arena_id_create(size_t arena_index) { + mi_assert_internal(arena_index < MI_MAX_ARENAS); + return (int)arena_index + 1; +} + +mi_arena_id_t _mi_arena_id_none(void) { + return 0; +} + +static bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclusive, mi_arena_id_t req_arena_id) { + return ((!arena_is_exclusive && req_arena_id == _mi_arena_id_none()) || + (arena_id == req_arena_id)); +} + +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id) { + if (memid.memkind == MI_MEM_ARENA) { + return mi_arena_id_is_suitable(memid.mem.arena.id, memid.mem.arena.is_exclusive, request_arena_id); + } + else { + return mi_arena_id_is_suitable(0, false, request_arena_id); + } +} + +bool _mi_arena_memid_is_os_allocated(mi_memid_t memid) { + return (memid.memkind == MI_MEM_OS); +} + +/* ----------------------------------------------------------- + Arena allocations get a (currently) 16-bit memory id where the + lower 8 bits are the arena id, and the upper bits the block index. +----------------------------------------------------------- */ + +static size_t mi_block_count_of_size(size_t size) { + return _mi_divide_up(size, MI_ARENA_BLOCK_SIZE); +} + +static size_t mi_arena_block_size(size_t bcount) { + return (bcount * MI_ARENA_BLOCK_SIZE); +} + +static size_t mi_arena_size(mi_arena_t* arena) { + return mi_arena_block_size(arena->block_count); +} + +static mi_memid_t mi_memid_create_arena(mi_arena_id_t id, bool is_exclusive, mi_bitmap_index_t bitmap_index) { + mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA); + memid.mem.arena.id = id; + memid.mem.arena.block_index = bitmap_index; + memid.mem.arena.is_exclusive = is_exclusive; + return memid; +} + +static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { + mi_assert_internal(memid.memkind == MI_MEM_ARENA); + *arena_index = mi_arena_id_index(memid.mem.arena.id); + *bitmap_index = memid.mem.arena.block_index; + return memid.mem.arena.is_exclusive; +} + + + +/* ----------------------------------------------------------- + Special static area for mimalloc internal structures + to avoid OS calls (for example, for the arena metadata) +----------------------------------------------------------- */ + +#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit + +static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; +static _Atomic(size_t) mi_arena_static_top; + +static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) { + *memid = _mi_memid_none(); + if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL; + if ((mi_atomic_load_relaxed(&mi_arena_static_top) + size) > MI_ARENA_STATIC_MAX) return NULL; + + // try to claim space + if (alignment == 0) { alignment = 1; } + const size_t oversize = size + alignment - 1; + if (oversize > MI_ARENA_STATIC_MAX) return NULL; + const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize); + size_t top = oldtop + oversize; + if (top > MI_ARENA_STATIC_MAX) { + // try to roll back, ok if this fails + mi_atomic_cas_strong_acq_rel(&mi_arena_static_top, &top, oldtop); + return NULL; + } + + // success + *memid = _mi_memid_create(MI_MEM_STATIC); + const size_t start = _mi_align_up(oldtop, alignment); + uint8_t* const p = &mi_arena_static[start]; + _mi_memzero(p, size); + return p; +} + +static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) { + *memid = _mi_memid_none(); + + // try static + void* p = mi_arena_static_zalloc(size, MI_ALIGNMENT_MAX, memid); + if (p != NULL) return p; + + // or fall back to the OS + return _mi_os_alloc(size, memid, stats); +} + +static void mi_arena_meta_free(void* p, mi_memid_t memid, size_t size, mi_stats_t* stats) { + if (mi_memkind_is_os(memid.memkind)) { + _mi_os_free(p, size, memid, stats); + } + else { + mi_assert(memid.memkind == MI_MEM_STATIC); + } +} + +static void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { + return (arena->start + mi_arena_block_size(mi_bitmap_index_bit(bindex))); +} + + +/* ----------------------------------------------------------- + Thread safe allocation in an arena +----------------------------------------------------------- */ + +// claim the `blocks_inuse` bits +static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx) +{ + size_t idx = 0; // mi_atomic_load_relaxed(&arena->search_idx); // start from last search; ok to be relaxed as the exact start does not matter + if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) { + mi_atomic_store_relaxed(&arena->search_idx, mi_bitmap_index_field(*bitmap_idx)); // start search from found location next time around + return true; + }; + return false; +} + + +/* ----------------------------------------------------------- + Arena Allocation +----------------------------------------------------------- */ + +static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t arena_index, size_t needed_bcount, + bool commit, mi_memid_t* memid, mi_os_tld_t* tld) +{ + MI_UNUSED(arena_index); + mi_assert_internal(mi_arena_id_index(arena->id) == arena_index); + + mi_bitmap_index_t bitmap_index; + if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index)) return NULL; + + // claimed it! + void* p = mi_arena_block_start(arena, bitmap_index); + *memid = mi_memid_create_arena(arena->id, arena->exclusive, bitmap_index); + memid->is_pinned = arena->memid.is_pinned; + + // none of the claimed blocks should be scheduled for a decommit + if (arena->blocks_purge != NULL) { + // this is thread safe as a potential purge only decommits parts that are not yet claimed as used (in `blocks_inuse`). + _mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, needed_bcount, bitmap_index); + } + + // set the dirty bits (todo: no need for an atomic op here?) + if (arena->memid.initially_zero && arena->blocks_dirty != NULL) { + memid->initially_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); + } + + // set commit state + if (arena->blocks_committed == NULL) { + // always committed + memid->initially_committed = true; + } + else if (commit) { + // commit requested, but the range may not be committed as a whole: ensure it is committed now + memid->initially_committed = true; + bool any_uncommitted; + _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted); + if (any_uncommitted) { + bool commit_zero = false; + if (!_mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero, tld->stats)) { + memid->initially_committed = false; + } + else { + if (commit_zero) { memid->initially_zero = true; } + } + } + } + else { + // no need to commit, but check if already fully committed + memid->initially_committed = _mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index); + } + + return p; +} + +// allocate in a speficic arena +static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment, + bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) +{ + MI_UNUSED_RELEASE(alignment); + mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + const size_t bcount = mi_block_count_of_size(size); + const size_t arena_index = mi_arena_id_index(arena_id); + mi_assert_internal(arena_index < mi_atomic_load_relaxed(&mi_arena_count)); + mi_assert_internal(size <= mi_arena_block_size(bcount)); + + // Check arena suitability + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]); + if (arena == NULL) return NULL; + if (!allow_large && arena->is_large) return NULL; + if (!mi_arena_id_is_suitable(arena->id, arena->exclusive, req_arena_id)) return NULL; + if (req_arena_id == _mi_arena_id_none()) { // in not specific, check numa affinity + const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node); + if (match_numa_node) { if (!numa_suitable) return NULL; } + else { if (numa_suitable) return NULL; } + } + + // try to allocate + void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid, tld); + mi_assert_internal(p == NULL || _mi_is_aligned(p, alignment)); + return p; +} + + +// allocate from an arena with fallback to the OS +static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, size_t alignment, + bool commit, bool allow_large, + mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) +{ + MI_UNUSED(alignment); + mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + if mi_likely(max_arena == 0) return NULL; + + if (req_arena_id != _mi_arena_id_none()) { + // try a specific arena if requested + if (mi_arena_id_index(req_arena_id) < max_arena) { + void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (p != NULL) return p; + } + } + else { + // try numa affine allocation + for (size_t i = 0; i < max_arena; i++) { + void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (p != NULL) return p; + } + + // try from another numa node instead.. + if (numa_node >= 0) { // if numa_node was < 0 (no specific affinity requested), all arena's have been tried already + for (size_t i = 0; i < max_arena; i++) { + void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (p != NULL) return p; + } + } + } + return NULL; +} + +// try to reserve a fresh arena space +static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t req_arena_id, mi_arena_id_t *arena_id) +{ + if (_mi_preloading()) return false; // use OS only while pre loading + if (req_arena_id != _mi_arena_id_none()) return false; + + const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); + if (arena_count > (MI_MAX_ARENAS - 4)) return false; + + size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve); + if (arena_reserve == 0) return false; + + if (!_mi_os_has_virtual_reserve()) { + arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for some embedded systems for example) + } + arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE); + if (arena_count >= 8 && arena_count <= 128) { + arena_reserve = ((size_t)1<<(arena_count/8)) * arena_reserve; // scale up the arena sizes exponentially + } + if (arena_reserve < req_size) return false; // should be able to at least handle the current allocation size + + // commit eagerly? + bool arena_commit = false; + if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = _mi_os_has_overcommit(); } + else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; } + + return (mi_reserve_os_memory_ex(arena_reserve, arena_commit, allow_large, false /* exclusive */, arena_id) == 0); +} + + +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, + mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld) +{ + mi_assert_internal(memid != NULL && tld != NULL); + mi_assert_internal(size > 0); + *memid = _mi_memid_none(); + + const int numa_node = _mi_os_numa_node(tld); // current numa node + + // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) + if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { + void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (p != NULL) return p; + + // otherwise, try to first eagerly reserve a new arena + if (req_arena_id == _mi_arena_id_none()) { + mi_arena_id_t arena_id = 0; + if (mi_arena_reserve(size, allow_large, req_arena_id, &arena_id)) { + // and try allocate in there + mi_assert_internal(req_arena_id == _mi_arena_id_none()); + p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (p != NULL) return p; + } + } + } + + // if we cannot use OS allocation, return NULL + if (mi_option_is_enabled(mi_option_limit_os_alloc) || req_arena_id != _mi_arena_id_none()) { + errno = ENOMEM; + return NULL; + } + + // finally, fall back to the OS + if (align_offset > 0) { + return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid, tld->stats); + } + else { + return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, tld->stats); + } +} + +void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld) +{ + return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid, tld); +} + + +void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { + if (size != NULL) *size = 0; + size_t arena_index = mi_arena_id_index(arena_id); + if (arena_index >= MI_MAX_ARENAS) return NULL; + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]); + if (arena == NULL) return NULL; + if (size != NULL) { *size = mi_arena_block_size(arena->block_count); } + return arena->start; +} + + +/* ----------------------------------------------------------- + Arena purge +----------------------------------------------------------- */ + +static long mi_arena_purge_delay(void) { + // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay + return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult)); +} + +// reset or decommit in an arena and update the committed/decommit bitmaps +// assumes we own the area (i.e. blocks_in_use is claimed by us) +static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) { + mi_assert_internal(arena->blocks_committed != NULL); + mi_assert_internal(arena->blocks_purge != NULL); + mi_assert_internal(!arena->memid.is_pinned); + const size_t size = mi_arena_block_size(blocks); + void* const p = mi_arena_block_start(arena, bitmap_idx); + bool needs_recommit; + if (_mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx)) { + // all blocks are committed, we can purge freely + needs_recommit = _mi_os_purge(p, size, stats); + } + else { + // some blocks are not committed -- this can happen when a partially committed block is freed + // in `_mi_arena_free` and it is conservatively marked as uncommitted but still scheduled for a purge + // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory), + // and also undo the decommit stats (as it was already adjusted) + mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits)); + needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, stats); + _mi_stat_increase(&stats->committed, size); + } + + // clear the purged blocks + _mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx); + // update committed bitmap + if (needs_recommit) { + _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); + } +} + +// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls. +// Note: assumes we (still) own the area as we may purge immediately +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) { + mi_assert_internal(arena->blocks_purge != NULL); + const long delay = mi_arena_purge_delay(); + if (delay < 0) return; // is purging allowed at all? + + if (_mi_preloading() || delay == 0) { + // decommit directly + mi_arena_purge(arena, bitmap_idx, blocks, stats); + } + else { + // schedule decommit + mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); + if (expire != 0) { + mi_atomic_addi64_acq_rel(&arena->purge_expire, delay/10); // add smallish extra delay + } + else { + mi_atomic_storei64_release(&arena->purge_expire, _mi_clock_now() + delay); + } + _mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL); + } +} + +// purge a range of blocks +// return true if the full range was purged. +// assumes we own the area (i.e. blocks_in_use is claimed by us) +static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge, mi_stats_t* stats) { + const size_t endidx = startidx + bitlen; + size_t bitidx = startidx; + bool all_purged = false; + while (bitidx < endidx) { + // count consequetive ones in the purge mask + size_t count = 0; + while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) { + count++; + } + if (count > 0) { + // found range to be purged + const mi_bitmap_index_t range_idx = mi_bitmap_index_create(idx, bitidx); + mi_arena_purge(arena, range_idx, count, stats); + if (count == bitlen) { + all_purged = true; + } + } + bitidx += (count+1); // +1 to skip the zero bit (or end) + } + return all_purged; +} + +// returns true if anything was purged +static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi_stats_t* stats) +{ + if (arena->memid.is_pinned || arena->blocks_purge == NULL) return false; + mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); + if (expire == 0) return false; + if (!force && expire > now) return false; + + // reset expire (if not already set concurrently) + mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, 0); + + // potential purges scheduled, walk through the bitmap + bool any_purged = false; + bool full_purge = true; + for (size_t i = 0; i < arena->field_count; i++) { + size_t purge = mi_atomic_load_relaxed(&arena->blocks_purge[i]); + if (purge != 0) { + size_t bitidx = 0; + while (bitidx < MI_BITMAP_FIELD_BITS) { + // find consequetive range of ones in the purge mask + size_t bitlen = 0; + while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) { + bitlen++; + } + // try to claim the longest range of corresponding in_use bits + const mi_bitmap_index_t bitmap_index = mi_bitmap_index_create(i, bitidx); + while( bitlen > 0 ) { + if (_mi_bitmap_try_claim(arena->blocks_inuse, arena->field_count, bitlen, bitmap_index)) { + break; + } + bitlen--; + } + // actual claimed bits at `in_use` + if (bitlen > 0) { + // read purge again now that we have the in_use bits + purge = mi_atomic_load_acquire(&arena->blocks_purge[i]); + if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge, stats)) { + full_purge = false; + } + any_purged = true; + // release the claimed `in_use` bits again + _mi_bitmap_unclaim(arena->blocks_inuse, arena->field_count, bitlen, bitmap_index); + } + bitidx += (bitlen+1); // +1 to skip the zero (or end) + } // while bitidx + } // purge != 0 + } + // if not fully purged, make sure to purge again in the future + if (!full_purge) { + const long delay = mi_arena_purge_delay(); + mi_msecs_t expected = 0; + mi_atomic_casi64_strong_acq_rel(&arena->purge_expire,&expected,_mi_clock_now() + delay); + } + return any_purged; +} + +static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats ) { + if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled + + const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count); + if (max_arena == 0) return; + + // allow only one thread to purge at a time + static mi_atomic_guard_t purge_guard; + mi_atomic_guard(&purge_guard) + { + mi_msecs_t now = _mi_clock_now(); + size_t max_purge_count = (visit_all ? max_arena : 1); + for (size_t i = 0; i < max_arena; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); + if (arena != NULL) { + if (mi_arena_try_purge(arena, now, force, stats)) { + if (max_purge_count <= 1) break; + max_purge_count--; + } + } + } + } +} + + +/* ----------------------------------------------------------- + Arena free +----------------------------------------------------------- */ + +void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid, mi_stats_t* stats) { + mi_assert_internal(size > 0 && stats != NULL); + mi_assert_internal(committed_size <= size); + if (p==NULL) return; + if (size==0) return; + const bool all_committed = (committed_size == size); + + if (mi_memkind_is_os(memid.memkind)) { + // was a direct OS allocation, pass through + if (!all_committed && committed_size > 0) { + // if partially committed, adjust the committed stats (as `_mi_os_free` will increase decommit by the full size) + _mi_stat_decrease(&stats->committed, committed_size); + } + _mi_os_free(p, size, memid, stats); + } + else if (memid.memkind == MI_MEM_ARENA) { + // allocated in an arena + size_t arena_idx; + size_t bitmap_idx; + mi_arena_memid_indices(memid, &arena_idx, &bitmap_idx); + mi_assert_internal(arena_idx < MI_MAX_ARENAS); + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t,&mi_arenas[arena_idx]); + mi_assert_internal(arena != NULL); + const size_t blocks = mi_block_count_of_size(size); + + // checks + if (arena == NULL) { + _mi_error_message(EINVAL, "trying to free from non-existent arena: %p, size %zu, memid: 0x%zx\n", p, size, memid); + return; + } + mi_assert_internal(arena->field_count > mi_bitmap_index_field(bitmap_idx)); + if (arena->field_count <= mi_bitmap_index_field(bitmap_idx)) { + _mi_error_message(EINVAL, "trying to free from non-existent arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid); + return; + } + + // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) + mi_track_mem_undefined(p,size); + + // potentially decommit + if (arena->memid.is_pinned || arena->blocks_committed == NULL) { + mi_assert_internal(all_committed); + } + else { + mi_assert_internal(arena->blocks_committed != NULL); + mi_assert_internal(arena->blocks_purge != NULL); + + if (!all_committed) { + // mark the entire range as no longer committed (so we recommit the full range when re-using) + _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); + mi_track_mem_noaccess(p,size); + if (committed_size > 0) { + // if partially committed, adjust the committed stats (is it will be recommitted when re-using) + // in the delayed purge, we now need to not count a decommit if the range is not marked as committed. + _mi_stat_decrease(&stats->committed, committed_size); + } + // note: if not all committed, it may be that the purge will reset/decommit the entire range + // that contains already decommitted parts. Since purge consistently uses reset or decommit that + // works (as we should never reset decommitted parts). + } + // (delay) purge the entire range + mi_arena_schedule_purge(arena, bitmap_idx, blocks, stats); + } + + // and make it available to others again + bool all_inuse = _mi_bitmap_unclaim_across(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx); + if (!all_inuse) { + _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", p, size); + return; + }; + } + else { + // arena was none, external, or static; nothing to do + mi_assert_internal(memid.memkind < MI_MEM_OS); + } + + // purge expired decommits + mi_arenas_try_purge(false, false, stats); +} + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +static void mi_arenas_unsafe_destroy(void) { + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + size_t new_max_arena = 0; + for (size_t i = 0; i < max_arena; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); + if (arena != NULL) { + if (arena->start != NULL && mi_memkind_is_os(arena->memid.memkind)) { + mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL); + _mi_os_free(arena->start, mi_arena_size(arena), arena->memid, &_mi_stats_main); + } + else { + new_max_arena = i; + } + mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size, &_mi_stats_main); + } + } + + // try to lower the max arena. + size_t expected = max_arena; + mi_atomic_cas_strong_acq_rel(&mi_arena_count, &expected, new_max_arena); +} + +// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired +void _mi_arena_collect(bool force_purge, mi_stats_t* stats) { + mi_arenas_try_purge(force_purge, true /* visit all */, stats); +} + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) { + mi_arenas_unsafe_destroy(); + _mi_arena_collect(true /* force purge */, stats); // purge non-owned arenas +} + +// Is a pointer inside any of our arenas? +bool _mi_arena_contains(const void* p) { + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + for (size_t i = 0; i < max_arena; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); + if (arena != NULL && arena->start <= (const uint8_t*)p && arena->start + mi_arena_block_size(arena->block_count) > (const uint8_t*)p) { + return true; + } + } + return false; +} + + +/* ----------------------------------------------------------- + Add an arena. +----------------------------------------------------------- */ + +static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id) { + mi_assert_internal(arena != NULL); + mi_assert_internal((uintptr_t)mi_atomic_load_ptr_relaxed(uint8_t,&arena->start) % MI_SEGMENT_ALIGN == 0); + mi_assert_internal(arena->block_count > 0); + if (arena_id != NULL) { *arena_id = -1; } + + size_t i = mi_atomic_increment_acq_rel(&mi_arena_count); + if (i >= MI_MAX_ARENAS) { + mi_atomic_decrement_acq_rel(&mi_arena_count); + return false; + } + arena->id = mi_arena_id_create(i); + mi_atomic_store_ptr_release(mi_arena_t,&mi_arenas[i], arena); + if (arena_id != NULL) { *arena_id = arena->id; } + return true; +} + +static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept +{ + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); + if (size < MI_ARENA_BLOCK_SIZE) return false; + + if (is_large) { + mi_assert_internal(memid.initially_committed && memid.is_pinned); + } + + const size_t bcount = size / MI_ARENA_BLOCK_SIZE; + const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); + const size_t bitmaps = (memid.is_pinned ? 2 : 4); + const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); + mi_memid_t meta_memid; + mi_arena_t* arena = (mi_arena_t*)mi_arena_meta_zalloc(asize, &meta_memid, &_mi_stats_main); // TODO: can we avoid allocating from the OS? + if (arena == NULL) return false; + + // already zero'd due to os_alloc + // _mi_memzero(arena, asize); + arena->id = _mi_arena_id_none(); + arena->memid = memid; + arena->exclusive = exclusive; + arena->meta_size = asize; + arena->meta_memid = meta_memid; + arena->block_count = bcount; + arena->field_count = fields; + arena->start = (uint8_t*)start; + arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) + arena->is_large = is_large; + arena->purge_expire = 0; + arena->search_idx = 0; + arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap + arena->blocks_committed = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + arena->blocks_purge = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[3*fields]); // just after committed bitmap + // initialize committed bitmap? + if (arena->blocks_committed != NULL && arena->memid.initially_committed) { + memset((void*)arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); // cast to void* to avoid atomic warning + } + + // and claim leftover blocks if needed (so we never allocate there) + ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; + mi_assert_internal(post >= 0); + if (post > 0) { + // don't use leftover bits at the end + mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post); + _mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL); + } + return mi_arena_add(arena, arena_id); + +} + +bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); + memid.initially_committed = is_committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_large; + return mi_manage_os_memory_ex2(start,size,is_large,numa_node,exclusive,memid, arena_id); +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); + size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block + mi_memid_t memid; + void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid, &_mi_stats_main); + if (start == NULL) return ENOMEM; + const bool is_large = memid.is_pinned; // todo: use separate is_large field? + if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) { + _mi_os_free_ex(start, size, commit, memid, &_mi_stats_main); + _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size, 1024)); + return ENOMEM; + } + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), is_large ? " (in large os pages)" : ""); + return 0; +} + + +// Manage a range of regular OS memory +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL); +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { + return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL); +} + + +/* ----------------------------------------------------------- + Debugging +----------------------------------------------------------- */ + +static size_t mi_debug_show_bitmap(const char* prefix, mi_bitmap_field_t* fields, size_t field_count ) { + size_t inuse_count = 0; + for (size_t i = 0; i < field_count; i++) { + char buf[MI_BITMAP_FIELD_BITS + 1]; + uintptr_t field = mi_atomic_load_relaxed(&fields[i]); + for (size_t bit = 0; bit < MI_BITMAP_FIELD_BITS; bit++) { + bool inuse = ((((uintptr_t)1 << bit) & field) != 0); + if (inuse) inuse_count++; + buf[MI_BITMAP_FIELD_BITS - 1 - bit] = (inuse ? 'x' : '.'); + } + buf[MI_BITMAP_FIELD_BITS] = 0; + _mi_verbose_message("%s%s\n", prefix, buf); + } + return inuse_count; +} + +void mi_debug_show_arenas(void) mi_attr_noexcept { + size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count); + for (size_t i = 0; i < max_arenas; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); + if (arena == NULL) break; + size_t inuse_count = 0; + _mi_verbose_message("arena %zu: %zu blocks with %zu fields\n", i, arena->block_count, arena->field_count); + inuse_count += mi_debug_show_bitmap(" ", arena->blocks_inuse, arena->field_count); + _mi_verbose_message(" blocks in use ('x'): %zu\n", inuse_count); + } +} + + +/* ----------------------------------------------------------- + Reserve a huge page arena. +----------------------------------------------------------- */ +// reserve at a specific numa node +int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = -1; + if (pages==0) return 0; + if (numa_node < -1) numa_node = -1; + if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); + size_t hsize = 0; + size_t pages_reserved = 0; + mi_memid_t memid; + void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); + if (p==NULL || pages_reserved==0) { + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); + return ENOMEM; + } + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); + + if (!mi_manage_os_memory_ex2(p, hsize, true, numa_node, exclusive, memid, arena_id)) { + _mi_os_free(p, hsize, memid, &_mi_stats_main); + return ENOMEM; + } + return 0; +} + +int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { + return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL); +} + +// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected) +int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept { + if (pages == 0) return 0; + + // pages per numa node + size_t numa_count = (numa_nodes > 0 ? numa_nodes : _mi_os_numa_node_count()); + if (numa_count <= 0) numa_count = 1; + const size_t pages_per = pages / numa_count; + const size_t pages_mod = pages % numa_count; + const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50); + + // reserve evenly among numa nodes + for (size_t numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) { + size_t node_pages = pages_per; // can be 0 + if (numa_node < pages_mod) node_pages++; + int err = mi_reserve_huge_os_pages_at(node_pages, (int)numa_node, timeout_per); + if (err) return err; + if (pages < node_pages) { + pages = 0; + } + else { + pages -= node_pages; + } + } + + return 0; +} + +int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { + MI_UNUSED(max_secs); + _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); + if (pages_reserved != NULL) *pages_reserved = 0; + int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); + if (err==0 && pages_reserved!=NULL) *pages_reserved = pages; + return err; +} diff --git a/Objects/mimalloc/bitmap.c b/Objects/mimalloc/bitmap.c new file mode 100644 index 00000000000000..31830756f58c7c --- /dev/null +++ b/Objects/mimalloc/bitmap.c @@ -0,0 +1,432 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2023 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically, +represented as an array of fields where each field is a machine word (`size_t`) + +There are two api's; the standard one cannot have sequences that cross +between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). + +The `_across` postfixed functions do allow sequences that can cross over +between the fields. (This is used in arena allocation) +---------------------------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "bitmap.h" + +/* ----------------------------------------------------------- + Bitmap definition +----------------------------------------------------------- */ + +// The bit mask for a given number of blocks at a specified bit index. +static inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) { + mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS); + mi_assert_internal(count > 0); + if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL; + if (count == 0) return 0; + return ((((size_t)1 << count) - 1) << bitidx); +} + + +/* ----------------------------------------------------------- + Claim a bit sequence atomically +----------------------------------------------------------- */ + +// Try to atomically claim a sequence of `count` bits in a single +// field at `idx` in `bitmap`. Returns `true` on success. +inline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx) +{ + mi_assert_internal(bitmap_idx != NULL); + mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); + mi_assert_internal(count > 0); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); + if (map==MI_BITMAP_FIELD_FULL) return false; // short cut + + // search for 0-bit sequence of length count + const size_t mask = mi_bitmap_mask_(count, 0); + const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; + +#ifdef MI_HAVE_FAST_BITSCAN + size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible +#else + size_t bitidx = 0; // otherwise start at 0 +#endif + size_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx + + // scan linearly for a free range of zero bits + while (bitidx <= bitidx_max) { + const size_t mapm = (map & m); + if (mapm == 0) { // are the mask bits free at bitidx? + mi_assert_internal((m >> bitidx) == mask); // no overflow? + const size_t newmap = (map | m); + mi_assert_internal((newmap^map) >> bitidx == mask); + if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { // TODO: use weak cas here? + // no success, another thread claimed concurrently.. keep going (with updated `map`) + continue; + } + else { + // success, we claimed the bits! + *bitmap_idx = mi_bitmap_index_create(idx, bitidx); + return true; + } + } + else { + // on to the next bit range +#ifdef MI_HAVE_FAST_BITSCAN + mi_assert_internal(mapm != 0); + const size_t shift = (count == 1 ? 1 : (MI_INTPTR_BITS - mi_clz(mapm) - bitidx)); + mi_assert_internal(shift > 0 && shift <= count); +#else + const size_t shift = 1; +#endif + bitidx += shift; + m <<= shift; + } + } + // no bits found + return false; +} + +// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success. +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +// `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. +bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) { idx = 0; } // wrap + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + return true; + } + } + return false; +} + +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fulfilled +bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, + const size_t start_field_idx, const size_t count, + mi_bitmap_pred_fun_t pred_fun, void* pred_arg, + mi_bitmap_index_t* bitmap_idx) { + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) idx = 0; // wrap + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + if (pred_fun == NULL || pred_fun(*bitmap_idx, pred_arg)) { + return true; + } + // predicate returned false, unclaim and look further + _mi_bitmap_unclaim(bitmap, bitmap_fields, count, *bitmap_idx); + } + } + return false; +} + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const size_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); + // mi_assert_internal((bitmap[idx] & mask) == mask); + const size_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); + return ((prev & mask) == mask); +} + + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const size_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); + //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); + size_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); + if (any_zero != NULL) { *any_zero = ((prev & mask) != mask); } + return ((prev & mask) == 0); +} + +// Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one. +static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const size_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); + const size_t field = mi_atomic_load_relaxed(&bitmap[idx]); + if (any_ones != NULL) { *any_ones = ((field & mask) != 0); } + return ((field & mask) == mask); +} + +// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically. +// Returns `true` if successful when all previous `count` bits were 0. +bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const size_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); + size_t expected = mi_atomic_load_relaxed(&bitmap[idx]); + do { + if ((expected & mask) != 0) return false; + } + while (!mi_atomic_cas_strong_acq_rel(&bitmap[idx], &expected, expected | mask)); + mi_assert_internal((expected & mask) == 0); + return true; +} + + +bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL); +} + +bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + bool any_ones; + mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); + return any_ones; +} + + +//-------------------------------------------------------------------------- +// the `_across` functions work on bitmaps where sequences can cross over +// between the fields. This is used in arena allocation +//-------------------------------------------------------------------------- + +// Try to atomically claim a sequence of `count` bits starting from the field +// at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success. +// Only needs to consider crossing into the next fields (see `mi_bitmap_try_find_from_claim_across`) +static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx) +{ + mi_assert_internal(bitmap_idx != NULL); + + // check initial trailing zeros + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); + const size_t initial = mi_clz(map); // count of initial zeros starting at idx + mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); + if (initial == 0) return false; + if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields (this case won't happen for us) + if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries + + // scan ahead + size_t found = initial; + size_t mask = 0; // mask bits for the final field + while(found < count) { + field++; + map = mi_atomic_load_relaxed(field); + const size_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); + mi_assert_internal(mask_bits > 0 && mask_bits <= MI_BITMAP_FIELD_BITS); + mask = mi_bitmap_mask_(mask_bits, 0); + if ((map & mask) != 0) return false; // some part is already claimed + found += mask_bits; + } + mi_assert_internal(field < &bitmap[bitmap_fields]); + + // we found a range of contiguous zeros up to the final field; mask contains mask in the final field + // now try to claim the range atomically + mi_bitmap_field_t* const final_field = field; + const size_t final_mask = mask; + mi_bitmap_field_t* const initial_field = &bitmap[idx]; + const size_t initial_idx = MI_BITMAP_FIELD_BITS - initial; + const size_t initial_mask = mi_bitmap_mask_(initial, initial_idx); + + // initial field + size_t newmap; + field = initial_field; + map = mi_atomic_load_relaxed(field); + do { + newmap = (map | initial_mask); + if ((map & initial_mask) != 0) { goto rollback; }; + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + + // intermediate fields + while (++field < final_field) { + newmap = MI_BITMAP_FIELD_FULL; + map = 0; + if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; } + } + + // final field + mi_assert_internal(field == final_field); + map = mi_atomic_load_relaxed(field); + do { + newmap = (map | final_mask); + if ((map & final_mask) != 0) { goto rollback; } + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + + // claimed! + *bitmap_idx = mi_bitmap_index_create(idx, initial_idx); + return true; + +rollback: + // roll back intermediate fields + // (we just failed to claim `field` so decrement first) + while (--field > initial_field) { + newmap = 0; + map = MI_BITMAP_FIELD_FULL; + mi_assert_internal(mi_atomic_load_relaxed(field) == map); + mi_atomic_store_release(field, newmap); + } + if (field == initial_field) { // (if we failed on the initial field, `field + 1 == initial_field`) + map = mi_atomic_load_relaxed(field); + do { + mi_assert_internal((map & initial_mask) == initial_mask); + newmap = (map & ~initial_mask); + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + } + // retry? (we make a recursive call instead of goto to be able to use const declarations) + if (retries <= 2) { + return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx); + } + else { + return false; + } +} + + +// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { + mi_assert_internal(count > 0); + if (count <= 2) { + // we don't bother with crossover fields for small counts + return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, start_field_idx, count, bitmap_idx); + } + + // visit the fields + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) { idx = 0; } // wrap + // first try to claim inside a field + if (count <= MI_BITMAP_FIELD_BITS) { + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + return true; + } + } + // if that fails, then try to claim across fields + if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) { + return true; + } + } + return false; +} + +// Helper for masks across fields; returns the mid count, post_mask may be 0 +static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) { + MI_UNUSED(bitmap_fields); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + if mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS) { + *pre_mask = mi_bitmap_mask_(count, bitidx); + *mid_mask = 0; + *post_mask = 0; + mi_assert_internal(mi_bitmap_index_field(bitmap_idx) < bitmap_fields); + return 0; + } + else { + const size_t pre_bits = MI_BITMAP_FIELD_BITS - bitidx; + mi_assert_internal(pre_bits < count); + *pre_mask = mi_bitmap_mask_(pre_bits, bitidx); + count -= pre_bits; + const size_t mid_count = (count / MI_BITMAP_FIELD_BITS); + *mid_mask = MI_BITMAP_FIELD_FULL; + count %= MI_BITMAP_FIELD_BITS; + *post_mask = (count==0 ? 0 : mi_bitmap_mask_(count, 0)); + mi_assert_internal(mi_bitmap_index_field(bitmap_idx) + mid_count + (count==0 ? 0 : 1) < bitmap_fields); + return mid_count; + } +} + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + size_t pre_mask; + size_t mid_mask; + size_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_one = true; + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); // clear first part + if ((prev & pre_mask) != pre_mask) all_one = false; + while(mid_count-- > 0) { + prev = mi_atomic_and_acq_rel(field++, ~mid_mask); // clear mid part + if ((prev & mid_mask) != mid_mask) all_one = false; + } + if (post_mask!=0) { + prev = mi_atomic_and_acq_rel(field, ~post_mask); // clear end part + if ((prev & post_mask) != post_mask) all_one = false; + } + return all_one; +} + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + size_t pre_mask; + size_t mid_mask; + size_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_zero = true; + bool any_zero = false; + _Atomic(size_t)*field = &bitmap[idx]; + size_t prev = mi_atomic_or_acq_rel(field++, pre_mask); + if ((prev & pre_mask) != 0) all_zero = false; + if ((prev & pre_mask) != pre_mask) any_zero = true; + while (mid_count-- > 0) { + prev = mi_atomic_or_acq_rel(field++, mid_mask); + if ((prev & mid_mask) != 0) all_zero = false; + if ((prev & mid_mask) != mid_mask) any_zero = true; + } + if (post_mask!=0) { + prev = mi_atomic_or_acq_rel(field, post_mask); + if ((prev & post_mask) != 0) all_zero = false; + if ((prev & post_mask) != post_mask) any_zero = true; + } + if (pany_zero != NULL) { *pany_zero = any_zero; } + return all_zero; +} + + +// Returns `true` if all `count` bits were 1. +// `any_ones` is `true` if there was at least one bit set to one. +static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + size_t pre_mask; + size_t mid_mask; + size_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_ones = true; + bool any_ones = false; + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_load_relaxed(field++); + if ((prev & pre_mask) != pre_mask) all_ones = false; + if ((prev & pre_mask) != 0) any_ones = true; + while (mid_count-- > 0) { + prev = mi_atomic_load_relaxed(field++); + if ((prev & mid_mask) != mid_mask) all_ones = false; + if ((prev & mid_mask) != 0) any_ones = true; + } + if (post_mask!=0) { + prev = mi_atomic_load_relaxed(field); + if ((prev & post_mask) != post_mask) all_ones = false; + if ((prev & post_mask) != 0) any_ones = true; + } + if (pany_ones != NULL) { *pany_ones = any_ones; } + return all_ones; +} + +bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + return mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, NULL); +} + +bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + bool any_ones; + mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); + return any_ones; +} diff --git a/Objects/mimalloc/bitmap.h b/Objects/mimalloc/bitmap.h new file mode 100644 index 00000000000000..9ba15d5d6f09ea --- /dev/null +++ b/Objects/mimalloc/bitmap.h @@ -0,0 +1,115 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2023 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically, +represeted as an array of fields where each field is a machine word (`size_t`) + +There are two api's; the standard one cannot have sequences that cross +between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). +(this is used in region allocation) + +The `_across` postfixed functions do allow sequences that can cross over +between the fields. (This is used in arena allocation) +---------------------------------------------------------------------------- */ +#pragma once +#ifndef MI_BITMAP_H +#define MI_BITMAP_H + +/* ----------------------------------------------------------- + Bitmap definition +----------------------------------------------------------- */ + +#define MI_BITMAP_FIELD_BITS (8*MI_SIZE_SIZE) +#define MI_BITMAP_FIELD_FULL (~((size_t)0)) // all bits set + +// An atomic bitmap of `size_t` fields +typedef _Atomic(size_t) mi_bitmap_field_t; +typedef mi_bitmap_field_t* mi_bitmap_t; + +// A bitmap index is the index of the bit in a bitmap. +typedef size_t mi_bitmap_index_t; + +// Create a bit index. +static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) { + mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS); + return (idx*MI_BITMAP_FIELD_BITS) + bitidx; +} + +// Create a bit index. +static inline mi_bitmap_index_t mi_bitmap_index_create_from_bit(size_t full_bitidx) { + return mi_bitmap_index_create(full_bitidx / MI_BITMAP_FIELD_BITS, full_bitidx % MI_BITMAP_FIELD_BITS); +} + +// Get the field index from a bit index. +static inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) { + return (bitmap_idx / MI_BITMAP_FIELD_BITS); +} + +// Get the bit index in a bitmap field +static inline size_t mi_bitmap_index_bit_in_field(mi_bitmap_index_t bitmap_idx) { + return (bitmap_idx % MI_BITMAP_FIELD_BITS); +} + +// Get the full bit index +static inline size_t mi_bitmap_index_bit(mi_bitmap_index_t bitmap_idx) { + return bitmap_idx; +} + +/* ----------------------------------------------------------- + Claim a bit sequence atomically +----------------------------------------------------------- */ + +// Try to atomically claim a sequence of `count` bits in a single +// field at `idx` in `bitmap`. Returns `true` on success. +bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. +bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); +bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically. +// Returns `true` if successful when all previous `count` bits were 0. +bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero); + +bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); +bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + + +//-------------------------------------------------------------------------- +// the `_across` functions work on bitmaps where sequences can cross over +// between the fields. This is used in arena allocation +//-------------------------------------------------------------------------- + +// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero); + +bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); +bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +#endif diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c new file mode 100644 index 00000000000000..d92dc768e5ec28 --- /dev/null +++ b/Objects/mimalloc/heap.c @@ -0,0 +1,693 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // mi_prim_get_default_heap + +#include // memset, memcpy + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +#pragma warning(disable:4204) // non-constant aggregate initializer +#endif + +/* ----------------------------------------------------------- + Helpers +----------------------------------------------------------- */ + +// return `true` if ok, `false` to break +typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2); + +// Visit all pages in a heap; returns `false` if break was called. +static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void* arg1, void* arg2) +{ + if (heap==NULL || heap->page_count==0) return true; + + // visit all pages + #if MI_DEBUG>1 + size_t total = heap->page_count; + size_t count = 0; + #endif + + for (size_t i = 0; i <= MI_BIN_FULL; i++) { + mi_page_queue_t* pq = &heap->pages[i]; + mi_page_t* page = pq->first; + while(page != NULL) { + mi_page_t* next = page->next; // save next in case the page gets removed from the queue + mi_assert_internal(mi_page_heap(page) == heap); + #if MI_DEBUG>1 + count++; + #endif + if (!fn(heap, pq, page, arg1, arg2)) return false; + page = next; // and continue + } + } + mi_assert_internal(count == total); + return true; +} + + +#if MI_DEBUG>=2 +static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(pq); + mi_assert_internal(mi_page_heap(page) == heap); + mi_segment_t* segment = _mi_page_segment(page); + mi_assert_internal(segment->thread_id == heap->thread_id); + mi_assert_expensive(_mi_page_is_valid(page)); + return true; +} +#endif +#if MI_DEBUG>=3 +static bool mi_heap_is_valid(mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL); + return true; +} +#endif + + + + +/* ----------------------------------------------------------- + "Collect" pages by migrating `local_free` and `thread_free` + lists and freeing empty pages. This is done when a thread + stops (and in that case abandons pages if there are still + blocks alive) +----------------------------------------------------------- */ + +typedef enum mi_collect_e { + MI_NORMAL, + MI_FORCE, + MI_ABANDON +} mi_collect_t; + + +static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { + MI_UNUSED(arg2); + MI_UNUSED(heap); + mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); + mi_collect_t collect = *((mi_collect_t*)arg_collect); + _mi_page_free_collect(page, collect >= MI_FORCE); + if (mi_page_all_free(page)) { + // no more used blocks, free the page. + // note: this will free retired pages as well. + bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); + if (!freed && collect == MI_ABANDON) { + _mi_page_abandon(page, pq); + } + } + else if (collect == MI_ABANDON) { + // still used blocks but the thread is done; abandon the page + _mi_page_abandon(page, pq); + } + return true; // don't break +} + +static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); + return true; // don't break +} + +static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) +{ + if (heap==NULL || !mi_heap_is_initialized(heap)) return; + + const bool force = collect >= MI_FORCE; + _mi_deferred_free(heap, force); + + // gh-112532: we may be called from a thread that is not the owner of the heap + bool is_main_thread = _mi_is_main_thread() && heap->thread_id == _mi_thread_id(); + + // note: never reclaim on collect but leave it to threads that need storage to reclaim + const bool force_main = + #ifdef NDEBUG + collect == MI_FORCE + #else + collect >= MI_FORCE + #endif + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim; + + if (force_main) { + // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. + // if all memory is freed by now, all segments should be freed. + _mi_abandoned_reclaim_all(heap, &heap->tld->segments); + } + + // if abandoning, mark all pages to no longer add to delayed_free + if (collect == MI_ABANDON) { + mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); + } + + // free all current thread delayed blocks. + // (if abandoning, after this there are no more thread-delayed references into the pages.) + _mi_heap_delayed_free_all(heap); + + // collect retired pages + _mi_heap_collect_retired(heap, force); + + // free pages that were delayed with QSBR + _PyMem_mi_heap_collect_qsbr(heap); + + // collect all pages owned by this thread + mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); + mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); + + // collect abandoned segments (in particular, purge expired parts of segments in the abandoned segment list) + // note: forced purge can be quite expensive if many threads are created/destroyed so we do not force on abandonment + _mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments); + + // collect segment local caches + if (force) { + _mi_segment_thread_collect(&heap->tld->segments); + } + + // collect regions on program-exit (or shared library unload) + if (force && is_main_thread && mi_heap_is_backing(heap)) { + _mi_thread_data_collect(); // collect thread data cache + _mi_arena_collect(true /* force purge */, &heap->tld->stats); + } +} + +void _mi_heap_collect_abandon(mi_heap_t* heap) { + mi_heap_collect_ex(heap, MI_ABANDON); +} + +void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept { + mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL)); +} + +void mi_collect(bool force) mi_attr_noexcept { + mi_heap_collect(mi_prim_get_default_heap(), force); +} + + +/* ----------------------------------------------------------- + Heap new +----------------------------------------------------------- */ + +mi_heap_t* mi_heap_get_default(void) { + mi_thread_init(); + return mi_prim_get_default_heap(); +} + +static bool mi_heap_is_default(const mi_heap_t* heap) { + return (heap == mi_prim_get_default_heap()); +} + + +mi_heap_t* mi_heap_get_backing(void) { + mi_heap_t* heap = mi_heap_get_default(); + mi_assert_internal(heap!=NULL); + mi_heap_t* bheap = heap->tld->heap_backing; + mi_assert_internal(bheap!=NULL); + mi_assert_internal(bheap->thread_id == _mi_thread_id()); + return bheap; +} + +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool no_reclaim, uint8_t tag) +{ + _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); + heap->tld = tld; + heap->thread_id = _mi_thread_id(); + heap->arena_id = arena_id; + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } + heap->cookie = _mi_heap_random_next(heap) | 1; + heap->keys[0] = _mi_heap_random_next(heap); + heap->keys[1] = _mi_heap_random_next(heap); + heap->no_reclaim = no_reclaim; + heap->tag = tag; + // push on the thread local heaps list + heap->next = heap->tld->heaps; + heap->tld->heaps = heap; +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? + if (heap == NULL) return NULL; + // don't reclaim abandoned pages or otherwise destroy is unsafe + _mi_heap_init_ex(heap, bheap->tld, arena_id, true, 0); + return heap; +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new(void) { + return mi_heap_new_in_arena(_mi_arena_id_none()); +} + +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) { + return _mi_arena_memid_is_suitable(memid, heap->arena_id); +} + +uintptr_t _mi_heap_random_next(mi_heap_t* heap) { + return _mi_random_next(&heap->random); +} + +// zero out the page queues +static void mi_heap_reset_pages(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + mi_assert_internal(mi_heap_is_initialized(heap)); + // TODO: copy full empty heap instead? + memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct)); + _mi_memcpy_aligned(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); + heap->thread_delayed_free = NULL; + heap->page_count = 0; +} + +// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. +static void mi_heap_free(mi_heap_t* heap) { + mi_assert(heap != NULL); + mi_assert_internal(mi_heap_is_initialized(heap)); + if (heap==NULL || !mi_heap_is_initialized(heap)) return; + if (mi_heap_is_backing(heap)) return; // dont free the backing heap + + // reset default + if (mi_heap_is_default(heap)) { + _mi_heap_set_default_direct(heap->tld->heap_backing); + } + + // remove ourselves from the thread local heaps list + // linear search but we expect the number of heaps to be relatively small + mi_heap_t* prev = NULL; + mi_heap_t* curr = heap->tld->heaps; + while (curr != heap && curr != NULL) { + prev = curr; + curr = curr->next; + } + mi_assert_internal(curr == heap); + if (curr == heap) { + if (prev != NULL) { prev->next = heap->next; } + else { heap->tld->heaps = heap->next; } + } + mi_assert_internal(heap->tld->heaps != NULL); + + // and free the used memory + mi_free(heap); +} + + +/* ----------------------------------------------------------- + Heap destroy +----------------------------------------------------------- */ + +static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); + + // ensure no more thread_delayed_free will be added + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); + + // stats + const size_t bsize = mi_page_block_size(page); + if (bsize > MI_MEDIUM_OBJ_SIZE_MAX) { + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, large, bsize); + } + else { + mi_heap_stat_decrease(heap, huge, bsize); + } + } +#if (MI_STAT) + _mi_page_free_collect(page, false); // update used count + const size_t inuse = page->used; + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, normal, bsize * inuse); +#if (MI_STAT>1) + mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); +#endif + } + mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks... +#endif + + /// pretend it is all free now + mi_assert_internal(mi_page_thread_free(page) == NULL); + page->used = 0; + + // and free the page + // mi_page_free(page,false); + page->next = NULL; + page->prev = NULL; + _mi_segment_page_free(page,false /* no force? */, &heap->tld->segments); + + return true; // keep going +} + +void _mi_heap_destroy_pages(mi_heap_t* heap) { + mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL); + mi_heap_reset_pages(heap); +} + +#if MI_TRACK_HEAP_DESTROY +static bool mi_cdecl mi_heap_track_block_free(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + MI_UNUSED(heap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size); + mi_track_free_size(block,mi_usable_size(block)); + return true; +} +#endif + +void mi_heap_destroy(mi_heap_t* heap) { + mi_assert(heap != NULL); + mi_assert(mi_heap_is_initialized(heap)); + mi_assert(heap->no_reclaim); + mi_assert_expensive(mi_heap_is_valid(heap)); + if (heap==NULL || !mi_heap_is_initialized(heap)) return; + if (!heap->no_reclaim) { + // don't free in case it may contain reclaimed pages + mi_heap_delete(heap); + } + else { + // track all blocks as freed + #if MI_TRACK_HEAP_DESTROY + mi_heap_visit_blocks(heap, true, mi_heap_track_block_free, NULL); + #endif + // free all pages + _mi_heap_destroy_pages(heap); + mi_heap_free(heap); + } +} + +// forcefully destroy all heaps in the current thread +void _mi_heap_unsafe_destroy_all(void) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* curr = bheap->tld->heaps; + while (curr != NULL) { + mi_heap_t* next = curr->next; + if (curr->no_reclaim) { + mi_heap_destroy(curr); + } + else { + _mi_heap_destroy_pages(curr); + } + curr = next; + } +} + +/* ----------------------------------------------------------- + Safe Heap delete +----------------------------------------------------------- */ + +// Transfer the pages from one heap to the other +static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { + mi_assert_internal(heap!=NULL); + if (from==NULL || from->page_count == 0) return; + + // reduce the size of the delayed frees + _mi_heap_delayed_free_partial(from); + + // transfer all pages by appending the queues; this will set a new heap field + // so threads may do delayed frees in either heap for a while. + // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state + // so after this only the new heap will get delayed frees + for (size_t i = 0; i <= MI_BIN_FULL; i++) { + mi_page_queue_t* pq = &heap->pages[i]; + mi_page_queue_t* append = &from->pages[i]; + size_t pcount = _mi_page_queue_append(heap, pq, append); + heap->page_count += pcount; + from->page_count -= pcount; + } + mi_assert_internal(from->page_count == 0); + + // and do outstanding delayed frees in the `from` heap + // note: be careful here as the `heap` field in all those pages no longer point to `from`, + // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a + // the regular `_mi_free_delayed_block` which is safe. + _mi_heap_delayed_free_all(from); + #if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353 + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL); + #endif + + // and reset the `from` heap + mi_heap_reset_pages(from); +} + +// Safe delete a heap without freeing any still allocated blocks in that heap. +void mi_heap_delete(mi_heap_t* heap) +{ + mi_assert(heap != NULL); + mi_assert(mi_heap_is_initialized(heap)); + mi_assert_expensive(mi_heap_is_valid(heap)); + if (heap==NULL || !mi_heap_is_initialized(heap)) return; + + if (!mi_heap_is_backing(heap)) { + // transfer still used pages to the backing heap + mi_heap_absorb(heap->tld->heap_backing, heap); + } + else { + // the backing heap abandons its pages + _mi_heap_collect_abandon(heap); + } + mi_assert_internal(heap->page_count==0); + mi_heap_free(heap); +} + +mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { + mi_assert(heap != NULL); + mi_assert(mi_heap_is_initialized(heap)); + if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL; + mi_assert_expensive(mi_heap_is_valid(heap)); + mi_heap_t* old = mi_prim_get_default_heap(); + _mi_heap_set_default_direct(heap); + return old; +} + + + + +/* ----------------------------------------------------------- + Analysis +----------------------------------------------------------- */ + +// static since it is not thread safe to access heaps from other threads. +static mi_heap_t* mi_heap_of_block(const void* p) { + if (p == NULL) return NULL; + mi_segment_t* segment = _mi_ptr_segment(p); + bool valid = (_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(valid); + if mi_unlikely(!valid) return NULL; + return mi_page_heap(_mi_segment_page_of(segment,p)); +} + +bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { + mi_assert(heap != NULL); + if (heap==NULL || !mi_heap_is_initialized(heap)) return false; + return (heap == mi_heap_of_block(p)); +} + + +static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { + MI_UNUSED(heap); + MI_UNUSED(pq); + bool* found = (bool*)vfound; + mi_segment_t* segment = _mi_page_segment(page); + void* start = _mi_page_start(segment, page, NULL); + void* end = (uint8_t*)start + (page->capacity * mi_page_block_size(page)); + *found = (p >= start && p < end); + return (!*found); // continue if not found +} + +bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { + mi_assert(heap != NULL); + if (heap==NULL || !mi_heap_is_initialized(heap)) return false; + if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers + bool found = false; + mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void*)p, &found); + return found; +} + +bool mi_check_owned(const void* p) { + return mi_heap_check_owned(mi_prim_get_default_heap(), p); +} + +/* ----------------------------------------------------------- + Visit all heap blocks and areas + Todo: enable visiting abandoned pages, and + enable visiting all blocks of all heaps across threads +----------------------------------------------------------- */ + +// Separate struct to keep `mi_page_t` out of the public interface +typedef struct mi_heap_area_ex_s { + mi_heap_area_t area; + mi_page_t* page; +} mi_heap_area_ex_t; + +static void mi_fast_divisor(size_t divisor, size_t* magic, size_t* shift) { + mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); + *shift = MI_INTPTR_BITS - mi_clz(divisor - 1); + *magic = (size_t)(((1ULL << 32) * ((1ULL << *shift) - divisor)) / divisor + 1); +} + +static size_t mi_fast_divide(size_t n, size_t magic, size_t shift) { + mi_assert_internal(n <= UINT32_MAX); + return ((((uint64_t) n * magic) >> 32) + n) >> shift; +} + +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_block_visit_fun* visitor, void* arg) { + mi_assert(area != NULL); + if (area==NULL) return true; + mi_assert(page != NULL); + if (page == NULL) return true; + + mi_assert_internal(page->local_free == NULL); + if (page->used == 0) return true; + + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); // without padding + size_t psize; + uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); + mi_heap_t* heap = mi_page_heap(page); + + if (page->capacity == 1) { + // optimize page with one block + mi_assert_internal(page->used == 1 && page->free == NULL); + return visitor(heap, area, pstart, ubsize, arg); + } + + if (page->used == page->capacity) { + // optimize full pages + uint8_t* block = pstart; + for (size_t i = 0; i < page->capacity; i++) { + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + return true; + } + + // create a bitmap of free blocks. + #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) + uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; + size_t bmapsize = (page->capacity + MI_INTPTR_BITS - 1) / MI_INTPTR_BITS; + memset(free_map, 0, bmapsize * sizeof(uintptr_t)); + + if (page->capacity % MI_INTPTR_BITS != 0) { + size_t shift = (page->capacity % MI_INTPTR_BITS); + uintptr_t mask = (UINTPTR_MAX << shift); + free_map[bmapsize-1] = mask; + } + + // fast repeated division by the block size + size_t magic, shift; + mi_fast_divisor(bsize, &magic, &shift); + + #if MI_DEBUG>1 + size_t free_count = 0; + #endif + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + #if MI_DEBUG>1 + free_count++; + #endif + mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); + size_t offset = (uint8_t*)block - pstart; + mi_assert_internal(offset % bsize == 0); + size_t blockidx = mi_fast_divide(offset, magic, shift); + mi_assert_internal(blockidx == offset / bsize); + mi_assert_internal(blockidx < MI_MAX_BLOCKS); + size_t bitidx = (blockidx / MI_INTPTR_BITS); + size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); + free_map[bitidx] |= ((uintptr_t)1 << bit); + } + mi_assert_internal(page->capacity == (free_count + page->used)); + + // walk through all blocks skipping the free ones + #if MI_DEBUG>1 + size_t used_count = 0; + #endif + uint8_t* block = pstart; + for (size_t i = 0; i < bmapsize; i++) { + if (free_map[i] == 0) { + // every block is in use + for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + #if MI_DEBUG>1 + used_count++; + #endif + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + } + else { + uintptr_t m = ~free_map[i]; + while (m) { + #if MI_DEBUG>1 + used_count++; + #endif + size_t bitidx = mi_ctz(m); + if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + m &= m - 1; + } + block += bsize * MI_INTPTR_BITS; + } + } + mi_assert_internal(page->used == used_count); + return true; +} + +typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); + +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + _mi_page_free_collect(page,true); + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); + area->reserved = page->reserved * bsize; + area->committed = page->capacity * bsize; + area->blocks = _mi_page_start(_mi_page_segment(page), page, NULL); + area->used = page->used; // number of blocks in use (#553) + area->block_size = ubsize; + area->full_block_size = bsize; +} + +static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { + MI_UNUSED(heap); + MI_UNUSED(pq); + mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; + mi_heap_area_ex_t xarea; + xarea.page = page; + _mi_heap_area_init(&xarea.area, page); + return fun(heap, &xarea, arg); +} + +// Visit all heap pages as areas +static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { + if (visitor == NULL) return false; + return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{ +} + +// Just to pass arguments +typedef struct mi_visit_blocks_args_s { + bool visit_blocks; + mi_block_visit_fun* visitor; + void* arg; +} mi_visit_blocks_args_t; + +static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) { + mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; + if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; + if (args->visit_blocks) { + return _mi_heap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); + } + else { + return true; + } +} + +// Visit all blocks in a heap +bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; + _mi_heap_delayed_free_partial((mi_heap_t *)heap); + return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args); +} diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c new file mode 100644 index 00000000000000..cb0ef6642803cc --- /dev/null +++ b/Objects/mimalloc/init.c @@ -0,0 +1,687 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" + +#include // memcpy, memset +#include // atexit + + +// Empty page used to initialize the small free pages array +const mi_page_t _mi_page_empty; + +#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) + +#if (MI_SMALL_WSIZE_MAX==128) +#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) +#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#elif (MI_PADDING>0) +#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#else +#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } +#endif +#else +#error "define right initialization sizes corresponding to MI_SMALL_WSIZE_MAX" +#endif + +// Empty page queues for every bin +#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } +#define MI_PAGE_QUEUES_EMPTY \ + { QNULL(1), \ + QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ + QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ + QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ + QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ + QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ + QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ + QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ + QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ + QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ + QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ + QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ } + +#define MI_STAT_COUNT_NULL() {0,0,0,0} + +// Empty statistics +#if MI_STAT>1 +#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } +#else +#define MI_STAT_COUNT_END_NULL() +#endif + +#define MI_STATS_NULL \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ + MI_STAT_COUNT_END_NULL() + + +// Empty slice span queues for every bin +#define SQNULL(sz) { NULL, NULL, sz } +#define MI_SEGMENT_SPAN_QUEUES_EMPTY \ + { SQNULL(1), \ + SQNULL( 1), SQNULL( 2), SQNULL( 3), SQNULL( 4), SQNULL( 5), SQNULL( 6), SQNULL( 7), SQNULL( 10), /* 8 */ \ + SQNULL( 12), SQNULL( 14), SQNULL( 16), SQNULL( 20), SQNULL( 24), SQNULL( 28), SQNULL( 32), SQNULL( 40), /* 16 */ \ + SQNULL( 48), SQNULL( 56), SQNULL( 64), SQNULL( 80), SQNULL( 96), SQNULL( 112), SQNULL( 128), SQNULL( 160), /* 24 */ \ + SQNULL( 192), SQNULL( 224), SQNULL( 256), SQNULL( 320), SQNULL( 384), SQNULL( 448), SQNULL( 512), SQNULL( 640), /* 32 */ \ + SQNULL( 768), SQNULL( 896), SQNULL( 1024) /* 35 */ } + + +// -------------------------------------------------------- +// Statically allocate an empty heap as the initial +// thread local value for the default heap, +// and statically allocate the backing heap for the main +// thread so it can function without doing any allocation +// itself (as accessing a thread local for the first time +// may lead to allocation itself on some platforms) +// -------------------------------------------------------- + +mi_decl_cache_align const mi_heap_t _mi_heap_empty = { + NULL, + MI_SMALL_PAGES_EMPTY, + MI_PAGE_QUEUES_EMPTY, + MI_ATOMIC_VAR_INIT(NULL), + 0, // tid + 0, // cookie + 0, // arena id + { 0, 0 }, // keys + { {0}, {0}, 0, true }, // random + 0, // page count + MI_BIN_FULL, 0, // page retired min/max + NULL, // next + false, + 0, + 0 +}; + +#define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats))) +#define tld_empty_os ((mi_os_tld_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,os))) + +mi_decl_cache_align static const mi_tld_t tld_empty = { + 0, + false, + NULL, NULL, + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, tld_empty_stats, tld_empty_os, &_mi_abandoned_default }, // segments + { 0, tld_empty_stats }, // os + { MI_STATS_NULL } // stats +}; + +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { + return _mi_prim_thread_id(); +} + +// the thread-local default heap for allocation +mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; + +extern mi_heap_t _mi_heap_main; + +static mi_tld_t tld_main = { + 0, false, + &_mi_heap_main, & _mi_heap_main, + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, &tld_main.stats, &tld_main.os, &_mi_abandoned_default }, // segments + { 0, &tld_main.stats }, // os + { MI_STATS_NULL } // stats +}; + +mi_heap_t _mi_heap_main = { + &tld_main, + MI_SMALL_PAGES_EMPTY, + MI_PAGE_QUEUES_EMPTY, + MI_ATOMIC_VAR_INIT(NULL), + 0, // thread id + 0, // initial cookie + 0, // arena id + { 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) + { {0x846ca68b}, {0}, 0, true }, // random + 0, // page count + MI_BIN_FULL, 0, // page retired min/max + NULL, // next heap + false // can reclaim +}; + +bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. + +mi_stats_t _mi_stats_main = { MI_STATS_NULL }; + + +static void mi_heap_main_init(void) { + if (_mi_heap_main.cookie == 0) { + _mi_heap_main.thread_id = _mi_thread_id(); + _mi_heap_main.cookie = 1; + #if defined(_WIN32) && !defined(MI_SHARED_LIB) + _mi_random_init_weak(&_mi_heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking + #else + _mi_random_init(&_mi_heap_main.random); + #endif + _mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); + _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); + _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); + } +} + +mi_heap_t* _mi_heap_main_get(void) { + mi_heap_main_init(); + return &_mi_heap_main; +} + + +/* ----------------------------------------------------------- + Initialization and freeing of the thread local heaps +----------------------------------------------------------- */ + +// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). +typedef struct mi_thread_data_s { + mi_heap_t heap; // must come first due to cast in `_mi_heap_done` + mi_tld_t tld; + mi_memid_t memid; +} mi_thread_data_t; + + +// Thread meta-data is allocated directly from the OS. For +// some programs that do not use thread pools and allocate and +// destroy many OS threads, this may causes too much overhead +// per thread so we maintain a small cache of recently freed metadata. + +#define TD_CACHE_SIZE (16) +static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; + +static mi_thread_data_t* mi_thread_data_zalloc(void) { + // try to find thread metadata in the cache + bool is_zero = false; + mi_thread_data_t* td = NULL; + for (int i = 0; i < TD_CACHE_SIZE; i++) { + td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); + if (td != NULL) { + // found cached allocation, try use it + td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); + if (td != NULL) { + break; + } + } + } + + // if that fails, allocate as meta data + if (td == NULL) { + mi_memid_t memid; + td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); + if (td == NULL) { + // if this fails, try once more. (issue #257) + td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); + if (td == NULL) { + // really out of memory + _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); + } + } + if (td != NULL) { + td->memid = memid; + is_zero = memid.initially_zero; + } + } + + if (td != NULL && !is_zero) { + _mi_memzero_aligned(td, sizeof(*td)); + } + return td; +} + +static void mi_thread_data_free( mi_thread_data_t* tdfree ) { + // try to add the thread metadata to the cache + for (int i = 0; i < TD_CACHE_SIZE; i++) { + mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); + if (td == NULL) { + mi_thread_data_t* expected = NULL; + if (mi_atomic_cas_ptr_weak_acq_rel(mi_thread_data_t, &td_cache[i], &expected, tdfree)) { + return; + } + } + } + // if that fails, just free it directly + _mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid, &_mi_stats_main); +} + +void _mi_thread_data_collect(void) { + // free all thread metadata from the cache + for (int i = 0; i < TD_CACHE_SIZE; i++) { + mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); + if (td != NULL) { + td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); + if (td != NULL) { + _mi_os_free(td, sizeof(mi_thread_data_t), td->memid, &_mi_stats_main); + } + } + } +} + +// Initialize the thread local default heap, called from `mi_thread_init` +static bool _mi_heap_init(void) { + if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; + if (_mi_is_main_thread()) { + // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization + // the main heap is statically allocated + mi_heap_main_init(); + _mi_heap_set_default_direct(&_mi_heap_main); + //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap()); + } + else { + // use `_mi_os_alloc` to allocate directly from the OS + mi_thread_data_t* td = mi_thread_data_zalloc(); + if (td == NULL) return false; + + _mi_tld_init(&td->tld, &td->heap); + _mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none(), false, 0); + _mi_heap_set_default_direct(&td->heap); + } + return false; +} + +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { + _mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); + tld->segments.stats = &tld->stats; + tld->segments.os = &tld->os; + tld->segments.abandoned = &_mi_abandoned_default; + tld->os.stats = &tld->stats; + tld->heap_backing = bheap; +} + +// Free the thread local default heap (called from `mi_thread_done`) +static bool _mi_heap_done(mi_heap_t* heap) { + if (!mi_heap_is_initialized(heap)) return true; + + // reset default heap + _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); + + // switch to backing heap + heap = heap->tld->heap_backing; + if (!mi_heap_is_initialized(heap)) return false; + + // delete all non-backing heaps in this thread + mi_heap_t* curr = heap->tld->heaps; + while (curr != NULL) { + mi_heap_t* next = curr->next; // save `next` as `curr` will be freed + if (curr != heap) { + mi_assert_internal(!mi_heap_is_backing(curr)); + mi_heap_delete(curr); + } + curr = next; + } + mi_assert_internal(heap->tld->heaps == heap && heap->next == NULL); + mi_assert_internal(mi_heap_is_backing(heap)); + + // collect if not the main thread + if (heap != &_mi_heap_main) { + _mi_heap_collect_abandon(heap); + } + + // merge stats + _mi_stats_done(&heap->tld->stats); + + // free if not the main thread + if (heap != &_mi_heap_main) { + // the following assertion does not always hold for huge segments as those are always treated + // as abondened: one may allocate it in one thread, but deallocate in another in which case + // the count can be too large or negative. todo: perhaps not count huge segments? see issue #363 + // mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); + mi_thread_data_free((mi_thread_data_t*)heap); + } + else { + #if 0 + // never free the main thread even in debug mode; if a dll is linked statically with mimalloc, + // there may still be delete/free calls after the mi_fls_done is called. Issue #207 + _mi_heap_destroy_pages(heap); + mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main); + #endif + } + return false; +} + + + +// -------------------------------------------------------- +// Try to run `mi_thread_done()` automatically so any memory +// owned by the thread but not yet released can be abandoned +// and re-owned by another thread. +// +// 1. windows dynamic library: +// call from DllMain on DLL_THREAD_DETACH +// 2. windows static library: +// use `FlsAlloc` to call a destructor when the thread is done +// 3. unix, pthreads: +// use a pthread key to call a destructor when a pthread is done +// +// In the last two cases we also need to call `mi_process_init` +// to set up the thread local keys. +// -------------------------------------------------------- + +// Set up handlers so `mi_thread_done` is called automatically +static void mi_process_setup_auto_thread_done(void) { + static bool tls_initialized = false; // fine if it races + if (tls_initialized) return; + tls_initialized = true; + _mi_prim_thread_init_auto_done(); + _mi_heap_set_default_direct(&_mi_heap_main); +} + + +bool _mi_is_main_thread(void) { + return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); +} + +static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1); + +size_t _mi_current_thread_count(void) { + return mi_atomic_load_relaxed(&thread_count); +} + +// This is called from the `mi_malloc_generic` +void mi_thread_init(void) mi_attr_noexcept +{ + // ensure our process has started already + mi_process_init(); + + // initialize the thread local default heap + // (this will call `_mi_heap_set_default_direct` and thus set the + // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) + if (_mi_heap_init()) return; // returns true if already initialized + + _mi_stat_increase(&_mi_stats_main.threads, 1); + mi_atomic_increment_relaxed(&thread_count); + //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); +} + +void mi_thread_done(void) mi_attr_noexcept { + _mi_thread_done(NULL); +} + +void _mi_thread_done(mi_heap_t* heap) +{ + // calling with NULL implies using the default heap + if (heap == NULL) { + heap = mi_prim_get_default_heap(); + if (heap == NULL) return; + } + + // prevent re-entrancy through heap_done/heap_set_default_direct (issue #699) + if (!mi_heap_is_initialized(heap)) { + return; + } + + // adjust stats + mi_atomic_decrement_relaxed(&thread_count); + _mi_stat_decrease(&_mi_stats_main.threads, 1); + + // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... + if (heap->thread_id != _mi_thread_id()) return; + + // abandon the thread local heap + if (_mi_heap_done(heap)) return; // returns true if already ran +} + +void _mi_heap_set_default_direct(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + #if defined(MI_TLS_SLOT) + mi_prim_tls_slot_set(MI_TLS_SLOT,heap); + #elif defined(MI_TLS_PTHREAD_SLOT_OFS) + *mi_tls_pthread_heap_slot() = heap; + #elif defined(MI_TLS_PTHREAD) + // we use _mi_heap_default_key + #else + _mi_heap_default = heap; + #endif + + // ensure the default heap is passed to `_mi_thread_done` + // setting to a non-NULL value also ensures `mi_thread_done` is called. + _mi_prim_thread_associate_default_heap(heap); +} + + +// -------------------------------------------------------- +// Run functions on process init/done, and thread init/done +// -------------------------------------------------------- +static void mi_cdecl mi_process_done(void); + +static bool os_preloading = true; // true until this module is initialized +static bool mi_redirected = false; // true if malloc redirects to mi_malloc + +// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. +bool mi_decl_noinline _mi_preloading(void) { + return os_preloading; +} + +mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { + return mi_redirected; +} + +// Communicate with the redirection module on Windows +#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) +#ifdef __cplusplus +extern "C" { +#endif +mi_decl_export void _mi_redirect_entry(DWORD reason) { + // called on redirection; careful as this may be called before DllMain + if (reason == DLL_PROCESS_ATTACH) { + mi_redirected = true; + } + else if (reason == DLL_PROCESS_DETACH) { + mi_redirected = false; + } + else if (reason == DLL_THREAD_DETACH) { + mi_thread_done(); + } +} +__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); +__declspec(dllimport) void mi_cdecl mi_allocator_done(void); +#ifdef __cplusplus +} +#endif +#else +static bool mi_allocator_init(const char** message) { + if (message != NULL) *message = NULL; + return true; +} +static void mi_allocator_done(void) { + // nothing to do +} +#endif + +// Called once by the process loader +static void mi_process_load(void) { + mi_heap_main_init(); + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) + volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; + if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) + #endif + os_preloading = false; + mi_assert_internal(_mi_is_main_thread()); + #if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521) + atexit(&mi_process_done); + #endif + _mi_options_init(); + mi_process_setup_auto_thread_done(); + mi_process_init(); + if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); + + // show message from the redirector (if present) + const char* msg = NULL; + mi_allocator_init(&msg); + if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { + _mi_fputs(NULL,NULL,NULL,msg); + } + + // reseed random + _mi_random_reinit_if_weak(&_mi_heap_main.random); +} + +#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#include +mi_decl_cache_align bool _mi_cpu_has_fsrm = false; + +static void mi_detect_cpu_features(void) { + // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + int32_t cpu_info[4]; + __cpuid(cpu_info, 7); + _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see +} +#else +static void mi_detect_cpu_features(void) { + // nothing +} +#endif + +// Initialize the process; called by thread_init or the process loader +void mi_process_init(void) mi_attr_noexcept { + // ensure we are called once + static mi_atomic_once_t process_init; + #if _MSC_VER < 1920 + mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main + #endif + if (!mi_atomic_once(&process_init)) return; + _mi_process_is_initialized = true; + _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + mi_process_setup_auto_thread_done(); + + mi_detect_cpu_features(); + _mi_os_init(); + mi_heap_main_init(); + #if MI_DEBUG + _mi_verbose_message("debug level : %d\n", MI_DEBUG); + #endif + _mi_verbose_message("secure level: %d\n", MI_SECURE); + _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); + #if MI_TSAN + _mi_verbose_message("thread santizer enabled\n"); + #endif + mi_thread_init(); + + #if defined(_WIN32) + // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. + // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup + // will not call _mi_thread_done on the (still executing) main thread. See issue #508. + _mi_prim_thread_associate_default_heap(NULL); + #endif + + mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) + mi_track_init(); + + if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { + size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); + long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); + if (reserve_at != -1) { + mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); + } else { + mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + } + } + if (mi_option_is_enabled(mi_option_reserve_os_memory)) { + long ksize = mi_option_get(mi_option_reserve_os_memory); + if (ksize > 0) { + mi_reserve_os_memory((size_t)ksize*MI_KiB, true /* commit? */, true /* allow large pages? */); + } + } +} + +// Called when the process is done (through `at_exit`) +static void mi_cdecl mi_process_done(void) { + // only shutdown if we were initialized + if (!_mi_process_is_initialized) return; + // ensure we are called once + static bool process_done = false; + if (process_done) return; + process_done = true; + + // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread + _mi_prim_thread_done_auto_done(); + + #ifndef MI_SKIP_COLLECT_ON_EXIT + #if (MI_DEBUG || !defined(MI_SHARED_LIB)) + // free all memory if possible on process exit. This is not needed for a stand-alone process + // but should be done if mimalloc is statically linked into another shared library which + // is repeatedly loaded/unloaded, see issue #281. + mi_collect(true /* force */ ); + #endif + #endif + + // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free + // since after process_done there might still be other code running that calls `free` (like at_exit routines, + // or C-runtime termination code. + if (mi_option_is_enabled(mi_option_destroy_on_exit)) { + mi_collect(true /* force */); + _mi_heap_unsafe_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!) + _mi_arena_unsafe_destroy_all(& _mi_heap_main_get()->tld->stats); + } + + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_stats_print(NULL); + } + mi_allocator_done(); + _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); + os_preloading = true; // don't call the C runtime anymore +} + + + +#if defined(_WIN32) && defined(MI_SHARED_LIB) + // Windows DLL: easy to hook into process_init and thread_done + __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(inst); + if (reason==DLL_PROCESS_ATTACH) { + mi_process_load(); + } + else if (reason==DLL_PROCESS_DETACH) { + mi_process_done(); + } + else if (reason==DLL_THREAD_DETACH) { + if (!mi_is_redirected()) { + mi_thread_done(); + } + } + return TRUE; + } + +#elif defined(_MSC_VER) + // MSVC: use data section magic for static libraries + // See + static int _mi_process_init(void) { + mi_process_load(); + return 0; + } + typedef int(*_mi_crt_callback_t)(void); + #if defined(_M_X64) || defined(_M_ARM64) + __pragma(comment(linker, "/include:" "_mi_msvc_initu")) + #pragma section(".CRT$XIU", long, read) + #else + __pragma(comment(linker, "/include:" "__mi_msvc_initu")) + #endif + #pragma data_seg(".CRT$XIU") + mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; + #pragma data_seg() + +#elif defined(__cplusplus) + // C++: use static initialization to detect process start + static bool _mi_process_init(void) { + mi_process_load(); + return (_mi_heap_main.thread_id != 0); + } + static bool mi_initialized = _mi_process_init(); + +#elif defined(__GNUC__) || defined(__clang__) + // GCC,Clang: use the constructor attribute + static void __attribute__((constructor)) _mi_process_init(void) { + mi_process_load(); + } + +#else +#pragma message("define a way to call mi_process_load on your platform") +#endif diff --git a/Objects/mimalloc/options.c b/Objects/mimalloc/options.c new file mode 100644 index 00000000000000..345b560e3e7f4c --- /dev/null +++ b/Objects/mimalloc/options.c @@ -0,0 +1,571 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // mi_prim_out_stderr + +#include // FILE +#include // abort +#include + + +static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit) +static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit) + +static void mi_add_stderr_output(void); + +int mi_version(void) mi_attr_noexcept { + return MI_MALLOC_VERSION; +} + + +// -------------------------------------------------------- +// Options +// These can be accessed by multiple threads and may be +// concurrently initialized, but an initializing data race +// is ok since they resolve to the same value. +// -------------------------------------------------------- +typedef enum mi_init_e { + UNINIT, // not yet initialized + DEFAULTED, // not found in the environment, use default value + INITIALIZED // found in environment or set explicitly +} mi_init_t; + +typedef struct mi_option_desc_s { + long value; // the value + mi_init_t init; // is it initialized yet? (from the environment) + mi_option_t option; // for debugging: the option index should match the option + const char* name; // option name without `mimalloc_` prefix + const char* legacy_name; // potential legacy option name +} mi_option_desc_t; + +#define MI_OPTION(opt) mi_option_##opt, #opt, NULL +#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy + +static mi_option_desc_t options[_mi_option_last] = +{ + // stable options + #if MI_DEBUG || defined(MI_SHOW_ERRORS) + { 1, UNINIT, MI_OPTION(show_errors) }, + #else + { 0, UNINIT, MI_OPTION(show_errors) }, + #endif + { 0, UNINIT, MI_OPTION(show_stats) }, + { 0, UNINIT, MI_OPTION(verbose) }, + + // the following options are experimental and not all combinations make sense. + { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) + { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) + { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N + { 0, UNINIT, MI_OPTION(reserve_os_memory) }, + { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread + { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free + { 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates + { 0, UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit) +#if defined(__NetBSD__) + { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed +#else + { 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) +#endif + { 10, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds + { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. + { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) + { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose + { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output + { 8, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. number of segment reclaims from the abandoned segments per try. + { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + #if (MI_INTPTR_SIZE>4) + { 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time + #else + { 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, + #endif + { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, +}; + +static void mi_option_init(mi_option_desc_t* desc); + +void _mi_options_init(void) { + // called on process load; should not be called before the CRT is initialized! + // (e.g. do not call this from process_init as that may run before CRT initialization) + mi_add_stderr_output(); // now it safe to use stderr for output + for(int i = 0; i < _mi_option_last; i++ ) { + mi_option_t option = (mi_option_t)i; + long l = mi_option_get(option); MI_UNUSED(l); // initialize + // if (option != mi_option_verbose) + { + mi_option_desc_t* desc = &options[option]; + _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); + } + } + mi_max_error_count = mi_option_get(mi_option_max_errors); + mi_max_warning_count = mi_option_get(mi_option_max_warnings); +} + +mi_decl_nodiscard long mi_option_get(mi_option_t option) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return 0; + mi_option_desc_t* desc = &options[option]; + mi_assert(desc->option == option); // index should match the option + if mi_unlikely(desc->init == UNINIT) { + mi_option_init(desc); + } + return desc->value; +} + +mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) { + long x = mi_option_get(option); + return (x < min ? min : (x > max ? max : x)); +} + +mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { + mi_assert_internal(option == mi_option_reserve_os_memory || option == mi_option_arena_reserve); + long x = mi_option_get(option); + return (x < 0 ? 0 : (size_t)x * MI_KiB); +} + +void mi_option_set(mi_option_t option, long value) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return; + mi_option_desc_t* desc = &options[option]; + mi_assert(desc->option == option); // index should match the option + desc->value = value; + desc->init = INITIALIZED; +} + +void mi_option_set_default(mi_option_t option, long value) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return; + mi_option_desc_t* desc = &options[option]; + if (desc->init != INITIALIZED) { + desc->value = value; + } +} + +mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) { + return (mi_option_get(option) != 0); +} + +void mi_option_set_enabled(mi_option_t option, bool enable) { + mi_option_set(option, (enable ? 1 : 0)); +} + +void mi_option_set_enabled_default(mi_option_t option, bool enable) { + mi_option_set_default(option, (enable ? 1 : 0)); +} + +void mi_option_enable(mi_option_t option) { + mi_option_set_enabled(option,true); +} + +void mi_option_disable(mi_option_t option) { + mi_option_set_enabled(option,false); +} + +static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { + MI_UNUSED(arg); + if (msg != NULL && msg[0] != 0) { + _mi_prim_out_stderr(msg); + } +} + +// Since an output function can be registered earliest in the `main` +// function we also buffer output that happens earlier. When +// an output function is registered it is called immediately with +// the output up to that point. +#ifndef MI_MAX_DELAY_OUTPUT +#define MI_MAX_DELAY_OUTPUT ((size_t)(32*1024)) +#endif +static char out_buf[MI_MAX_DELAY_OUTPUT+1]; +static _Atomic(size_t) out_len; + +static void mi_cdecl mi_out_buf(const char* msg, void* arg) { + MI_UNUSED(arg); + if (msg==NULL) return; + if (mi_atomic_load_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; + size_t n = _mi_strlen(msg); + if (n==0) return; + // claim space + size_t start = mi_atomic_add_acq_rel(&out_len, n); + if (start >= MI_MAX_DELAY_OUTPUT) return; + // check bound + if (start+n >= MI_MAX_DELAY_OUTPUT) { + n = MI_MAX_DELAY_OUTPUT-start-1; + } + _mi_memcpy(&out_buf[start], msg, n); +} + +static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { + if (out==NULL) return; + // claim (if `no_more_buf == true`, no more output will be added after this point) + size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); + // and output the current contents + if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; + out_buf[count] = 0; + out(out_buf,arg); + if (!no_more_buf) { + out_buf[count] = '\n'; // if continue with the buffer, insert a newline + } +} + + +// Once this module is loaded, switch to this routine +// which outputs to stderr and the delayed output buffer. +static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) { + mi_out_stderr(msg,arg); + mi_out_buf(msg,arg); +} + + + +// -------------------------------------------------------- +// Default output handler +// -------------------------------------------------------- + +// Should be atomic but gives errors on many platforms as generally we cannot cast a function pointer to a uintptr_t. +// For now, don't register output from multiple threads. +static mi_output_fun* volatile mi_out_default; // = NULL +static _Atomic(void*) mi_out_arg; // = NULL + +static mi_output_fun* mi_out_get_default(void** parg) { + if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); } + mi_output_fun* out = mi_out_default; + return (out == NULL ? &mi_out_buf : out); +} + +void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer + mi_atomic_store_ptr_release(void,&mi_out_arg, arg); + if (out!=NULL) mi_out_buf_flush(out,true,arg); // output all the delayed output now +} + +// add stderr to the delayed output after the module is loaded +static void mi_add_stderr_output(void) { + mi_assert_internal(mi_out_default == NULL); + mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr + mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output +} + +// -------------------------------------------------------- +// Messages, all end up calling `_mi_fputs`. +// -------------------------------------------------------- +static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings + +// When overriding malloc, we may recurse into mi_vfprintf if an allocation +// inside the C runtime causes another message. +// In some cases (like on macOS) the loader already allocates which +// calls into mimalloc; if we then access thread locals (like `recurse`) +// this may crash as the access may call _tlv_bootstrap that tries to +// (recursively) invoke malloc again to allocate space for the thread local +// variables on demand. This is why we use a _mi_preloading test on such +// platforms. However, C code generator may move the initial thread local address +// load before the `if` and we therefore split it out in a separate funcion. +static mi_decl_thread bool recurse = false; + +static mi_decl_noinline bool mi_recurse_enter_prim(void) { + if (recurse) return false; + recurse = true; + return true; +} + +static mi_decl_noinline void mi_recurse_exit_prim(void) { + recurse = false; +} + +static bool mi_recurse_enter(void) { + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) + if (_mi_preloading()) return false; + #endif + return mi_recurse_enter_prim(); +} + +static void mi_recurse_exit(void) { + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) + if (_mi_preloading()) return; + #endif + mi_recurse_exit_prim(); +} + +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) { + if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr? + if (!mi_recurse_enter()) return; + out = mi_out_get_default(&arg); + if (prefix != NULL) out(prefix, arg); + out(message, arg); + mi_recurse_exit(); + } + else { + if (prefix != NULL) out(prefix, arg); + out(message, arg); + } +} + +// Define our own limited `fprintf` that avoids memory allocation. +// We do this using `snprintf` with a limited buffer. +static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) { + char buf[512]; + if (fmt==NULL) return; + if (!mi_recurse_enter()) return; + vsnprintf(buf,sizeof(buf)-1,fmt,args); + mi_recurse_exit(); + _mi_fputs(out,arg,prefix,buf); +} + +void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { + va_list args; + va_start(args,fmt); + mi_vfprintf(out,arg,NULL,fmt,args); + va_end(args); +} + +static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { + if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) { + char tprefix[64]; + snprintf(tprefix, sizeof(tprefix), "%sthread 0x%llx: ", prefix, (unsigned long long)_mi_thread_id()); + mi_vfprintf(out, arg, tprefix, fmt, args); + } + else { + mi_vfprintf(out, arg, prefix, fmt, args); + } +} + +void _mi_trace_message(const char* fmt, ...) { + if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher + va_list args; + va_start(args, fmt); + mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args); + va_end(args); +} + +void _mi_verbose_message(const char* fmt, ...) { + if (!mi_option_is_enabled(mi_option_verbose)) return; + va_list args; + va_start(args,fmt); + mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args); + va_end(args); +} + +static void mi_show_error_message(const char* fmt, va_list args) { + if (!mi_option_is_enabled(mi_option_verbose)) { + if (!mi_option_is_enabled(mi_option_show_errors)) return; + if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return; + } + mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args); +} + +void _mi_warning_message(const char* fmt, ...) { + if (!mi_option_is_enabled(mi_option_verbose)) { + if (!mi_option_is_enabled(mi_option_show_errors)) return; + if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return; + } + va_list args; + va_start(args,fmt); + mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args); + va_end(args); +} + + +#if MI_DEBUG +void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) { + _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); + abort(); +} +#endif + +// -------------------------------------------------------- +// Errors +// -------------------------------------------------------- + +static mi_error_fun* volatile mi_error_handler; // = NULL +static _Atomic(void*) mi_error_arg; // = NULL + +static void mi_error_default(int err) { + MI_UNUSED(err); +#if (MI_DEBUG>0) + if (err==EFAULT) { + #ifdef _MSC_VER + __debugbreak(); + #endif + abort(); + } +#endif +#if (MI_SECURE>0) + if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) + abort(); + } +#endif +#if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode + abort(); + } +#endif +} + +void mi_register_error(mi_error_fun* fun, void* arg) { + mi_error_handler = fun; // can be NULL + mi_atomic_store_ptr_release(void,&mi_error_arg, arg); +} + +void _mi_error_message(int err, const char* fmt, ...) { + // show detailed error message + va_list args; + va_start(args, fmt); + mi_show_error_message(fmt, args); + va_end(args); + // and call the error handler which may abort (or return normally) + if (mi_error_handler != NULL) { + mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg)); + } + else { + mi_error_default(err); + } +} + +// -------------------------------------------------------- +// Initialize options by checking the environment +// -------------------------------------------------------- +char _mi_toupper(char c) { + if (c >= 'a' && c <= 'z') return (c - 'a' + 'A'); + else return c; +} + +int _mi_strnicmp(const char* s, const char* t, size_t n) { + if (n == 0) return 0; + for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { + if (_mi_toupper(*s) != _mi_toupper(*t)) break; + } + return (n == 0 ? 0 : *s - *t); +} + +void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return; + // copy until end of src, or when dest is (almost) full + while (*src != 0 && dest_size > 1) { + *dest++ = *src++; + dest_size--; + } + // always zero terminate + *dest = 0; +} + +void _mi_strlcat(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return; + // find end of string in the dest buffer + while (*dest != 0 && dest_size > 1) { + dest++; + dest_size--; + } + // and catenate + _mi_strlcpy(dest, src, dest_size); +} + +size_t _mi_strlen(const char* s) { + if (s==NULL) return 0; + size_t len = 0; + while(s[len] != 0) { len++; } + return len; +} + +size_t _mi_strnlen(const char* s, size_t max_len) { + if (s==NULL) return 0; + size_t len = 0; + while(s[len] != 0 && len < max_len) { len++; } + return len; +} + +#ifdef MI_NO_GETENV +static bool mi_getenv(const char* name, char* result, size_t result_size) { + MI_UNUSED(name); + MI_UNUSED(result); + MI_UNUSED(result_size); + return false; +} +#else +static bool mi_getenv(const char* name, char* result, size_t result_size) { + if (name==NULL || result == NULL || result_size < 64) return false; + return _mi_prim_getenv(name,result,result_size); +} +#endif + +// TODO: implement ourselves to reduce dependencies on the C runtime +#include // strtol +#include // strstr + + +static void mi_option_init(mi_option_desc_t* desc) { + // Read option value from the environment + char s[64 + 1]; + char buf[64+1]; + _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + _mi_strlcat(buf, desc->name, sizeof(buf)); + bool found = mi_getenv(buf, s, sizeof(s)); + if (!found && desc->legacy_name != NULL) { + _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + _mi_strlcat(buf, desc->legacy_name, sizeof(buf)); + found = mi_getenv(buf, s, sizeof(s)); + if (found) { + _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name); + } + } + + if (found) { + size_t len = _mi_strnlen(s, sizeof(buf) - 1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(s[i]); + } + buf[len] = 0; + if (buf[0] == 0 || strstr("1;TRUE;YES;ON", buf) != NULL) { + desc->value = 1; + desc->init = INITIALIZED; + } + else if (strstr("0;FALSE;NO;OFF", buf) != NULL) { + desc->value = 0; + desc->init = INITIALIZED; + } + else { + char* end = buf; + long value = strtol(buf, &end, 10); + if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_arena_reserve) { + // this option is interpreted in KiB to prevent overflow of `long` + if (*end == 'K') { end++; } + else if (*end == 'M') { value *= MI_KiB; end++; } + else if (*end == 'G') { value *= MI_MiB; end++; } + else { value = (value + MI_KiB - 1) / MI_KiB; } + if (end[0] == 'I' && end[1] == 'B') { end += 2; } + else if (*end == 'B') { end++; } + } + if (*end == 0) { + desc->value = value; + desc->init = INITIALIZED; + } + else { + // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose. + desc->init = DEFAULTED; + if (desc->option == mi_option_verbose && desc->value == 0) { + // if the 'mimalloc_verbose' env var has a bogus value we'd never know + // (since the value defaults to 'off') so in that case briefly enable verbose + desc->value = 1; + _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); + desc->value = 0; + } + else { + _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); + } + } + } + mi_assert_internal(desc->init != UNINIT); + } + else if (!_mi_preloading()) { + desc->init = DEFAULTED; + } +} diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c new file mode 100644 index 00000000000000..f3bc7184c41c5b --- /dev/null +++ b/Objects/mimalloc/os.c @@ -0,0 +1,690 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" + + +/* ----------------------------------------------------------- + Initialization. + On windows initializes support for aligned allocation and + large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). +----------------------------------------------------------- */ + +static mi_os_mem_config_t mi_os_mem_config = { + 4096, // page size + 0, // large page size (usually 2MiB) + 4096, // allocation granularity + true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) + false, // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) +}; + +bool _mi_os_has_overcommit(void) { + return mi_os_mem_config.has_overcommit; +} + +bool _mi_os_has_virtual_reserve(void) { + return mi_os_mem_config.has_virtual_reserve; +} + + +// OS (small) page size +size_t _mi_os_page_size(void) { + return mi_os_mem_config.page_size; +} + +// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) +size_t _mi_os_large_page_size(void) { + return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size()); +} + +bool _mi_os_use_large_page(size_t size, size_t alignment) { + // if we have access, check the size and alignment requirements + if (mi_os_mem_config.large_page_size == 0 || !mi_option_is_enabled(mi_option_allow_large_os_pages)) return false; + return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); +} + +// round to a good OS allocation size (bounded by max 12.5% waste) +size_t _mi_os_good_alloc_size(size_t size) { + size_t align_size; + if (size < 512*MI_KiB) align_size = _mi_os_page_size(); + else if (size < 2*MI_MiB) align_size = 64*MI_KiB; + else if (size < 8*MI_MiB) align_size = 256*MI_KiB; + else if (size < 32*MI_MiB) align_size = 1*MI_MiB; + else align_size = 4*MI_MiB; + if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? + return _mi_align_up(size, align_size); +} + +void _mi_os_init(void) { + _mi_prim_mem_init(&mi_os_mem_config); +} + + +/* ----------------------------------------------------------- + Util +-------------------------------------------------------------- */ +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); +bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); + +static void* mi_align_up_ptr(void* p, size_t alignment) { + return (void*)_mi_align_up((uintptr_t)p, alignment); +} + +static void* mi_align_down_ptr(void* p, size_t alignment) { + return (void*)_mi_align_down((uintptr_t)p, alignment); +} + + +/* ----------------------------------------------------------- + aligned hinting +-------------------------------------------------------------- */ + +// On 64-bit systems, we can do efficient aligned allocation by using +// the 2TiB to 30TiB area to allocate those. +#if (MI_INTPTR_SIZE >= 8) +static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; + +// Return a MI_SEGMENT_SIZE aligned address that is probably available. +// If this returns NULL, the OS will determine the address but on some OS's that may not be +// properly aligned which can be more costly as it needs to be adjusted afterwards. +// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; +// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses +// in the middle of the 2TiB - 6TiB address range (see issue #372)) + +#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) +#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) +{ + if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; + size = _mi_align_up(size, MI_SEGMENT_SIZE); + if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. + #if (MI_SECURE>0) + size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. + #endif + + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); + if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize + uintptr_t init = MI_HINT_BASE; + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode + uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + #endif + uintptr_t expected = hint + size; + mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); + hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + } + if (hint%try_alignment != 0) return NULL; + return (void*)hint; +} +#else +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { + MI_UNUSED(try_alignment); MI_UNUSED(size); + return NULL; +} +#endif + + +/* ----------------------------------------------------------- + Free memory +-------------------------------------------------------------- */ + +static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats); + +static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_stats_t* tld_stats) { + MI_UNUSED(tld_stats); + mi_assert_internal((size % _mi_os_page_size()) == 0); + if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) + int err = _mi_prim_free(addr, size); + if (err != 0) { + _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); + } + mi_stats_t* stats = &_mi_stats_main; + if (still_committed) { _mi_stat_decrease(&stats->committed, size); } + _mi_stat_decrease(&stats->reserved, size); +} + +void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { + if (mi_memkind_is_os(memid.memkind)) { + size_t csize = _mi_os_good_alloc_size(size); + void* base = addr; + // different base? (due to alignment) + if (memid.mem.os.base != NULL) { + mi_assert(memid.mem.os.base <= addr); + mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr); + base = memid.mem.os.base; + csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base); + } + // free it + if (memid.memkind == MI_MEM_OS_HUGE) { + mi_assert(memid.is_pinned); + mi_os_free_huge_os_pages(base, csize, tld_stats); + } + else { + mi_os_prim_free(base, csize, still_committed, tld_stats); + } + } + else { + // nothing to do + mi_assert(memid.memkind < MI_MEM_OS); + } +} + +void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) { + _mi_os_free_ex(p, size, true, memid, tld_stats); +} + + +/* ----------------------------------------------------------- + Primitive allocation from the OS. +-------------------------------------------------------------- */ + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(is_zero != NULL); + mi_assert_internal(is_large != NULL); + if (size == 0) return NULL; + if (!commit) { allow_large = false; } + if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning + + *is_zero = false; + void* p = NULL; + int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p); + if (err != 0) { + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); + } + mi_stat_counter_increase(stats->mmap_calls, 1); + if (p != NULL) { + _mi_stat_increase(&stats->reserved, size); + if (commit) { + _mi_stat_increase(&stats->committed, size); + // seems needed for asan (or `mimalloc-test-api` fails) + #ifdef MI_TRACK_ASAN + if (*is_zero) { mi_track_mem_defined(p,size); } + else { mi_track_mem_undefined(p,size); } + #endif + } + } + return p; +} + + +// Primitive aligned allocation from the OS. +// This function guarantees the allocated memory is aligned. +static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) { + mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(is_large != NULL); + mi_assert_internal(is_zero != NULL); + mi_assert_internal(base != NULL); + if (!commit) allow_large = false; + if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; + size = _mi_align_up(size, _mi_os_page_size()); + + // try first with a hint (this will be aligned directly on Win 10+ or BSD) + void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); + if (p == NULL) return NULL; + + // aligned already? + if (((uintptr_t)p % alignment) == 0) { + *base = p; + } + else { + // if not aligned, free it, overallocate, and unmap around it + // NOTE(sgross): this warning causes issues in Python tests + // _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); + mi_os_prim_free(p, size, commit, stats); + if (size >= (SIZE_MAX - alignment)) return NULL; // overflow + const size_t over_size = size + alignment; + + if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block + // over-allocate uncommitted (virtual) memory + p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats); + if (p == NULL) return NULL; + + // set p to the aligned part in the full region + // note: this is dangerous on Windows as VirtualFree needs the actual base pointer + // this is handled though by having the `base` field in the memid's + *base = p; // remember the base + p = mi_align_up_ptr(p, alignment); + + // explicitly commit only the aligned part + if (commit) { + _mi_os_commit(p, size, NULL, stats); + } + } + else { // mmap can free inside an allocation + // overallocate... + p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); + if (p == NULL) return NULL; + + // and selectively unmap parts around the over-allocated area. (noop on sbrk) + void* aligned_p = mi_align_up_ptr(p, alignment); + size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; + size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + size_t post_size = over_size - pre_size - mid_size; + mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); + if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); } + // we can return the aligned pointer on `mmap` (and sbrk) systems + p = aligned_p; + *base = aligned_p; // since we freed the pre part, `*base == p`. + } + } + + mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0)); + return p; +} + + +/* ----------------------------------------------------------- + OS API: alloc and alloc_aligned +----------------------------------------------------------- */ + +void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) { + MI_UNUSED(tld_stats); + *memid = _mi_memid_none(); + mi_stats_t* stats = &_mi_stats_main; + if (size == 0) return NULL; + size = _mi_os_good_alloc_size(size); + bool os_is_large = false; + bool os_is_zero = false; + void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats); + if (p != NULL) { + *memid = _mi_memid_create_os(true, os_is_zero, os_is_large); + } + return p; +} + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats) +{ + MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings + MI_UNUSED(tld_stats); + *memid = _mi_memid_none(); + if (size == 0) return NULL; + size = _mi_os_good_alloc_size(size); + alignment = _mi_align_up(alignment, _mi_os_page_size()); + + bool os_is_large = false; + bool os_is_zero = false; + void* os_base = NULL; + void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &_mi_stats_main /*tld->stats*/ ); + if (p != NULL) { + *memid = _mi_memid_create_os(commit, os_is_zero, os_is_large); + memid->mem.os.base = os_base; + memid->mem.os.alignment = alignment; + } + return p; +} + +/* ----------------------------------------------------------- + OS aligned allocation with an offset. This is used + for large alignments > MI_ALIGNMENT_MAX. We use a large mimalloc + page where the object can be aligned at an offset from the start of the segment. + As we may need to overallocate, we need to free such pointers using `mi_free_aligned` + to use the actual start of the memory region. +----------------------------------------------------------- */ + +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats) { + mi_assert(offset <= MI_SEGMENT_SIZE); + mi_assert(offset <= size); + mi_assert((alignment % _mi_os_page_size()) == 0); + *memid = _mi_memid_none(); + if (offset > MI_SEGMENT_SIZE) return NULL; + if (offset == 0) { + // regular aligned allocation + return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, tld_stats); + } + else { + // overallocate to align at an offset + const size_t extra = _mi_align_up(offset, alignment) - offset; + const size_t oversize = size + extra; + void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats); + if (start == NULL) return NULL; + + void* const p = (uint8_t*)start + extra; + mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); + // decommit the overallocation at the start + if (commit && extra > _mi_os_page_size()) { + _mi_os_decommit(start, extra, tld_stats); + } + return p; + } +} + +/* ----------------------------------------------------------- + OS memory API: reset, commit, decommit, protect, unprotect. +----------------------------------------------------------- */ + +// OS page align within a given area, either conservative (pages inside the area only), +// or not (straddling pages outside the area is possible) +static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { + mi_assert(addr != NULL && size > 0); + if (newsize != NULL) *newsize = 0; + if (size == 0 || addr == NULL) return NULL; + + // page align conservatively within the range + void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size()) + : mi_align_down_ptr(addr, _mi_os_page_size())); + void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) + : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); + ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; + if (diff <= 0) return NULL; + + mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size)); + if (newsize != NULL) *newsize = (size_t)diff; + return start; +} + +static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) { + return mi_os_page_align_areax(true, addr, size, newsize); +} + +bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { + MI_UNUSED(tld_stats); + mi_stats_t* stats = &_mi_stats_main; + if (is_zero != NULL) { *is_zero = false; } + _mi_stat_increase(&stats->committed, size); // use size for precise commit vs. decommit + _mi_stat_counter_increase(&stats->commit_calls, 1); + + // page align range + size_t csize; + void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize); + if (csize == 0) return true; + + // commit + bool os_is_zero = false; + int err = _mi_prim_commit(start, csize, &os_is_zero); + if (err != 0) { + _mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + return false; + } + if (os_is_zero && is_zero != NULL) { + *is_zero = true; + mi_assert_expensive(mi_mem_is_zero(start, csize)); + } + // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails) + #ifdef MI_TRACK_ASAN + if (os_is_zero) { mi_track_mem_defined(start,csize); } + else { mi_track_mem_undefined(start,csize); } + #endif + return true; +} + +static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_stats_t* tld_stats) { + MI_UNUSED(tld_stats); + mi_stats_t* stats = &_mi_stats_main; + mi_assert_internal(needs_recommit!=NULL); + _mi_stat_decrease(&stats->committed, size); + + // page align + size_t csize; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return true; + + // decommit + *needs_recommit = true; + int err = _mi_prim_decommit(start,csize,needs_recommit); + if (err != 0) { + _mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + } + mi_assert_internal(err == 0); + return (err == 0); +} + +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { + bool needs_recommit; + return mi_os_decommit_ex(addr, size, &needs_recommit, tld_stats); +} + + +// Signal to the OS that the address range is no longer in use +// but may be used later again. This will release physical memory +// pages and reduce swapping while keeping the memory committed. +// We page align to a conservative area inside the range to reset. +bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { + // page align conservatively within the range + size_t csize; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) + _mi_stat_increase(&stats->reset, csize); + _mi_stat_counter_increase(&stats->reset_calls, 1); + + #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN + memset(start, 0, csize); // pretend it is eagerly reset + #endif + + int err = _mi_prim_reset(start, csize); + if (err != 0) { + _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + } + return (err == 0); +} + + +// either resets or decommits memory, returns true if the memory needs +// to be recommitted if it is to be re-used later on. +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats) +{ + if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? + _mi_stat_counter_increase(&stats->purge_calls, 1); + _mi_stat_increase(&stats->purged, size); + + if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? + !_mi_preloading()) // don't decommit during preloading (unsafe) + { + bool needs_recommit = true; + mi_os_decommit_ex(p, size, &needs_recommit, stats); + return needs_recommit; + } + else { + if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed + _mi_os_reset(p, size, stats); + } + return false; // needs no recommit + } +} + +// either resets or decommits memory, returns true if the memory needs +// to be recommitted if it is to be re-used later on. +bool _mi_os_purge(void* p, size_t size, mi_stats_t * stats) { + return _mi_os_purge_ex(p, size, true, stats); +} + +// Protect a region in memory to be not accessible. +static bool mi_os_protectx(void* addr, size_t size, bool protect) { + // page align conservatively within the range + size_t csize = 0; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return false; + /* + if (_mi_os_is_huge_reserved(addr)) { + _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); + } + */ + int err = _mi_prim_protect(start,csize,protect); + if (err != 0) { + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); + } + return (err == 0); +} + +bool _mi_os_protect(void* addr, size_t size) { + return mi_os_protectx(addr, size, true); +} + +bool _mi_os_unprotect(void* addr, size_t size) { + return mi_os_protectx(addr, size, false); +} + + + +/* ---------------------------------------------------------------------------- +Support for allocating huge OS pages (1Gib) that are reserved up-front +and possibly associated with a specific NUMA node. (use `numa_node>=0`) +-----------------------------------------------------------------------------*/ +#define MI_HUGE_OS_PAGE_SIZE (MI_GiB) + + +#if (MI_INTPTR_SIZE >= 8) +// To ensure proper alignment, use our own area for huge OS pages +static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 + +// Claim an aligned address range for huge pages +static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { + if (total_size != NULL) *total_size = 0; + const size_t size = pages * MI_HUGE_OS_PAGE_SIZE; + + uintptr_t start = 0; + uintptr_t end = 0; + uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start); + do { + start = huge_start; + if (start == 0) { + // Initialize the start address after the 32TiB area + start = ((uintptr_t)32 << 40); // 32TiB virtual start address + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode + uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + #endif + } + end = start + size; + mi_assert_internal(end % MI_SEGMENT_SIZE == 0); + } while (!mi_atomic_cas_strong_acq_rel(&mi_huge_start, &huge_start, end)); + + if (total_size != NULL) *total_size = size; + return (uint8_t*)start; +} +#else +static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { + MI_UNUSED(pages); + if (total_size != NULL) *total_size = 0; + return NULL; +} +#endif + +// Allocate MI_SEGMENT_SIZE aligned huge pages +void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) { + *memid = _mi_memid_none(); + if (psize != NULL) *psize = 0; + if (pages_reserved != NULL) *pages_reserved = 0; + size_t size = 0; + uint8_t* start = mi_os_claim_huge_pages(pages, &size); + if (start == NULL) return NULL; // or 32-bit systems + + // Allocate one page at the time but try to place them contiguously + // We allocate one page at the time to be able to abort if it takes too long + // or to at least allocate as many as available on the system. + mi_msecs_t start_t = _mi_clock_start(); + size_t page = 0; + bool all_zero = true; + while (page < pages) { + // allocate a page + bool is_zero = false; + void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); + void* p = NULL; + int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p); + if (!is_zero) { all_zero = false; } + if (err != 0) { + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + break; + } + + // Did we succeed at a contiguous address? + if (p != addr) { + // no success, issue a warning and break + if (p != NULL) { + _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); + mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true, &_mi_stats_main); + } + break; + } + + // success, record it + page++; // increase before timeout check (see issue #711) + _mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE); + _mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE); + + // check for timeout + if (max_msecs > 0) { + mi_msecs_t elapsed = _mi_clock_end(start_t); + if (page >= 1) { + mi_msecs_t estimate = ((elapsed / (page+1)) * pages); + if (estimate > 2*max_msecs) { // seems like we are going to timeout, break + elapsed = max_msecs + 1; + } + } + if (elapsed > max_msecs) { + _mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page); + break; + } + } + } + mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size); + if (pages_reserved != NULL) { *pages_reserved = page; } + if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; } + if (page != 0) { + mi_assert(start != NULL); + *memid = _mi_memid_create_os(true /* is committed */, all_zero, true /* is_large */); + memid->memkind = MI_MEM_OS_HUGE; + mi_assert(memid->is_pinned); + #ifdef MI_TRACK_ASAN + if (all_zero) { mi_track_mem_defined(start,size); } + #endif + } + return (page == 0 ? NULL : start); +} + +// free every huge page in a range individually (as we allocated per page) +// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. +static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats) { + if (p==NULL || size==0) return; + uint8_t* base = (uint8_t*)p; + while (size >= MI_HUGE_OS_PAGE_SIZE) { + mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true, stats); + size -= MI_HUGE_OS_PAGE_SIZE; + base += MI_HUGE_OS_PAGE_SIZE; + } +} + +/* ---------------------------------------------------------------------------- +Support NUMA aware allocation +-----------------------------------------------------------------------------*/ + +_Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count + +size_t _mi_os_numa_node_count_get(void) { + size_t count = mi_atomic_load_acquire(&_mi_numa_node_count); + if (count <= 0) { + long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? + if (ncount > 0) { + count = (size_t)ncount; + } + else { + count = _mi_prim_numa_node_count(); // or detect dynamically + if (count == 0) count = 1; + } + mi_atomic_store_release(&_mi_numa_node_count, count); // save it + _mi_verbose_message("using %zd numa regions\n", count); + } + return count; +} + +int _mi_os_numa_node_get(mi_os_tld_t* tld) { + MI_UNUSED(tld); + size_t numa_count = _mi_os_numa_node_count(); + if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 + // never more than the node count and >= 0 + size_t numa_node = _mi_prim_numa_node(); + if (numa_node >= numa_count) { numa_node = numa_node % numa_count; } + return (int)numa_node; +} diff --git a/Objects/mimalloc/page-queue.c b/Objects/mimalloc/page-queue.c new file mode 100644 index 00000000000000..cb54b3740196e9 --- /dev/null +++ b/Objects/mimalloc/page-queue.c @@ -0,0 +1,332 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + Definition of page queues for each block size +----------------------------------------------------------- */ + +#ifndef MI_IN_PAGE_C +#error "this file should be included from 'page.c'" +#endif + +/* ----------------------------------------------------------- + Minimal alignment in machine words (i.e. `sizeof(void*)`) +----------------------------------------------------------- */ + +#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) + #error "define alignment for more than 4x word size for this platform" +#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) + #define MI_ALIGN4W // 4 machine words minimal alignment +#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) + #define MI_ALIGN2W // 2 machine words minimal alignment +#else + // ok, default alignment is 1 word +#endif + + +/* ----------------------------------------------------------- + Queue query +----------------------------------------------------------- */ + + +static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t))); +} + +static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t)))); +} + +static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { + return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX); +} + +/* ----------------------------------------------------------- + Bins +----------------------------------------------------------- */ + +// Return the bin for a given field size. +// Returns MI_BIN_HUGE if the size is too large. +// We use `wsize` for the size in "machine word sizes", +// i.e. byte size == `wsize*sizeof(void*)`. +static inline uint8_t mi_bin(size_t size) { + size_t wsize = _mi_wsize_from_size(size); + uint8_t bin; + if (wsize <= 1) { + bin = 1; + } + #if defined(MI_ALIGN4W) + else if (wsize <= 4) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #elif defined(MI_ALIGN2W) + else if (wsize <= 8) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #else + else if (wsize <= 8) { + bin = (uint8_t)wsize; + } + #endif + else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) { + bin = MI_BIN_HUGE; + } + else { + #if defined(MI_ALIGN4W) + if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes + #endif + wsize--; + // find the highest bit + uint8_t b = (uint8_t)mi_bsr(wsize); // note: wsize != 0 + // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). + // - adjust with 3 because we use do not round the first 8 sizes + // which each get an exact bin + bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; + mi_assert_internal(bin < MI_BIN_HUGE); + } + mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE); + return bin; +} + + + +/* ----------------------------------------------------------- + Queue of pages with free blocks +----------------------------------------------------------- */ + +uint8_t _mi_bin(size_t size) { + return mi_bin(size); +} + +size_t _mi_bin_size(uint8_t bin) { + return _mi_heap_empty.pages[bin].block_size; +} + +// Good size for allocation +size_t mi_good_size(size_t size) mi_attr_noexcept { + if (size <= MI_MEDIUM_OBJ_SIZE_MAX) { + return _mi_bin_size(mi_bin(size)); + } + else { + return _mi_align_up(size,_mi_os_page_size()); + } +} + +#if (MI_DEBUG>1) +static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_page_t* list = queue->first; + while (list != NULL) { + mi_assert_internal(list->next == NULL || list->next->prev == list); + mi_assert_internal(list->prev == NULL || list->prev->next == list); + if (list == page) break; + list = list->next; + } + return (list == page); +} + +#endif + +#if (MI_DEBUG>1) +static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* pq) { + return (pq >= &heap->pages[0] && pq <= &heap->pages[MI_BIN_FULL]); +} +#endif + +static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); + mi_heap_t* heap = mi_page_heap(page); + mi_assert_internal(heap != NULL && bin <= MI_BIN_FULL); + mi_page_queue_t* pq = &heap->pages[bin]; + mi_assert_internal(bin >= MI_BIN_HUGE || page->xblock_size == pq->block_size); + mi_assert_expensive(mi_page_queue_contains(pq, page)); + return pq; +} + +static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); + mi_assert_internal(bin <= MI_BIN_FULL); + mi_page_queue_t* pq = &heap->pages[bin]; + mi_assert_internal(mi_page_is_in_full(page) || page->xblock_size == pq->block_size); + return pq; +} + +// The current small page array is for efficiency and for each +// small size (up to 256) it points directly to the page for that +// size without having to compute the bin. This means when the +// current free page queue is updated for a small bin, we need to update a +// range of entries in `_mi_page_small_free`. +static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_queue_t* pq) { + mi_assert_internal(mi_heap_contains_queue(heap,pq)); + size_t size = pq->block_size; + if (size > MI_SMALL_SIZE_MAX) return; + + mi_page_t* page = pq->first; + if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty; + + // find index in the right direct page array + size_t start; + size_t idx = _mi_wsize_from_size(size); + mi_page_t** pages_free = heap->pages_free_direct; + + if (pages_free[idx] == page) return; // already set + + // find start slot + if (idx<=1) { + start = 0; + } + else { + // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped + uint8_t bin = mi_bin(size); + const mi_page_queue_t* prev = pq - 1; + while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) { + prev--; + } + start = 1 + _mi_wsize_from_size(prev->block_size); + if (start > idx) start = idx; + } + + // set size range to the right page + mi_assert(start <= idx); + for (size_t sz = start; sz <= idx; sz++) { + pages_free[sz] = page; + } +} + +/* +static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { + return (queue->first == NULL); +} +*/ + +static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_expensive(mi_page_queue_contains(queue, page)); + mi_assert_internal(page->xblock_size == queue->block_size || (page->xblock_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + mi_heap_t* heap = mi_page_heap(page); + + if (page->prev != NULL) page->prev->next = page->next; + if (page->next != NULL) page->next->prev = page->prev; + if (page == queue->last) queue->last = page->prev; + if (page == queue->first) { + queue->first = page->next; + // update first + mi_assert_internal(mi_heap_contains_queue(heap, queue)); + mi_heap_queue_first_update(heap,queue); + } + heap->page_count--; + page->next = NULL; + page->prev = NULL; + // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), NULL); + mi_page_set_in_full(page,false); +} + + +static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_heap(page) == heap); + mi_assert_internal(!mi_page_queue_contains(queue, page)); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + #endif + mi_assert_internal(page->xblock_size == queue->block_size || + (page->xblock_size > MI_MEDIUM_OBJ_SIZE_MAX) || + (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + + mi_page_set_in_full(page, mi_page_queue_is_full(queue)); + // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), heap); + page->next = queue->first; + page->prev = NULL; + if (queue->first != NULL) { + mi_assert_internal(queue->first->prev == NULL); + queue->first->prev = page; + queue->first = page; + } + else { + queue->first = queue->last = page; + } + + // update direct + mi_heap_queue_first_update(heap, queue); + heap->page_count++; +} + + +static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_expensive(mi_page_queue_contains(from, page)); + mi_assert_expensive(!mi_page_queue_contains(to, page)); + + mi_assert_internal((page->xblock_size == to->block_size && page->xblock_size == from->block_size) || + (page->xblock_size == to->block_size && mi_page_queue_is_full(from)) || + (page->xblock_size == from->block_size && mi_page_queue_is_full(to)) || + (page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) || + (page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_full(to))); + + mi_heap_t* heap = mi_page_heap(page); + if (page->prev != NULL) page->prev->next = page->next; + if (page->next != NULL) page->next->prev = page->prev; + if (page == from->last) from->last = page->prev; + if (page == from->first) { + from->first = page->next; + // update first + mi_assert_internal(mi_heap_contains_queue(heap, from)); + mi_heap_queue_first_update(heap, from); + } + + page->prev = to->last; + page->next = NULL; + if (to->last != NULL) { + mi_assert_internal(heap == mi_page_heap(to->last)); + to->last->next = page; + to->last = page; + } + else { + to->first = page; + to->last = page; + mi_heap_queue_first_update(heap, to); + } + + mi_page_set_in_full(page, mi_page_queue_is_full(to)); +} + +// Only called from `mi_heap_absorb`. +size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { + mi_assert_internal(mi_heap_contains_queue(heap,pq)); + mi_assert_internal(pq->block_size == append->block_size); + + if (append->first==NULL) return 0; + + // set append pages to new heap and count + size_t count = 0; + for (mi_page_t* page = append->first; page != NULL; page = page->next) { + // inline `mi_page_set_heap` to avoid wrong assertion during absorption; + // in this case it is ok to be delayed freeing since both "to" and "from" heap are still alive. + mi_atomic_store_release(&page->xheap, (uintptr_t)heap); + // set the flag to delayed free (not overriding NEVER_DELAYED_FREE) which has as a + // side effect that it spins until any DELAYED_FREEING is finished. This ensures + // that after appending only the new heap will be used for delayed free operations. + _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false); + count++; + } + + if (pq->last==NULL) { + // take over afresh + mi_assert_internal(pq->first==NULL); + pq->first = append->first; + pq->last = append->last; + mi_heap_queue_first_update(heap, pq); + } + else { + // append to end + mi_assert_internal(pq->last!=NULL); + mi_assert_internal(append->first!=NULL); + pq->last->next = append->first; + append->first->prev = pq->last; + pq->last = append->last; + } + return count; +} diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c new file mode 100644 index 00000000000000..25ecd6ec7d7983 --- /dev/null +++ b/Objects/mimalloc/page.c @@ -0,0 +1,972 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + The core of the allocator. Every segment contains + pages of a certain block size. The main function + exported is `mi_malloc_generic`. +----------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" + +/* ----------------------------------------------------------- + Definition of page queues for each block size +----------------------------------------------------------- */ + +#define MI_IN_PAGE_C +#include "page-queue.c" +#undef MI_IN_PAGE_C + + +/* ----------------------------------------------------------- + Page helpers +----------------------------------------------------------- */ + +// Index a block in a page +static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) { + MI_UNUSED(page); + mi_assert_internal(page != NULL); + mi_assert_internal(i <= page->reserved); + return (mi_block_t*)((uint8_t*)page_start + (i * block_size)); +} + +static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t* tld); +static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld); + +#if (MI_DEBUG>=3) +static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) { + size_t count = 0; + while (head != NULL) { + mi_assert_internal(page == _mi_ptr_page(head)); + count++; + head = mi_block_next(page, head); + } + return count; +} + +/* +// Start of the page available memory +static inline uint8_t* mi_page_area(const mi_page_t* page) { + return _mi_page_start(_mi_page_segment(page), page, NULL); +} +*/ + +static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { + size_t psize; + uint8_t* page_area = _mi_page_start(_mi_page_segment(page), page, &psize); + mi_block_t* start = (mi_block_t*)page_area; + mi_block_t* end = (mi_block_t*)(page_area + psize); + while(p != NULL) { + if (p < start || p >= end) return false; + p = mi_block_next(page, p); + } +#if MI_DEBUG>3 // generally too expensive to check this + if (page->free_is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); + } + } +#endif + return true; +} + +static bool mi_page_is_valid_init(mi_page_t* page) { + mi_assert_internal(page->xblock_size > 0); + mi_assert_internal(page->used <= page->capacity); + mi_assert_internal(page->capacity <= page->reserved); + + mi_segment_t* segment = _mi_page_segment(page); + uint8_t* start = _mi_page_start(segment,page,NULL); + mi_assert_internal(start == _mi_segment_page_start(segment,page,NULL)); + //const size_t bsize = mi_page_block_size(page); + //mi_assert_internal(start + page->capacity*page->block_size == page->top); + + mi_assert_internal(mi_page_list_is_valid(page,page->free)); + mi_assert_internal(mi_page_list_is_valid(page,page->local_free)); + + #if MI_DEBUG>3 // generally too expensive to check this + if (page->free_is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); + } + } + #endif + + #if !MI_TRACK_ENABLED && !MI_TSAN + mi_block_t* tfree = mi_page_thread_free(page); + mi_assert_internal(mi_page_list_is_valid(page, tfree)); + //size_t tfree_count = mi_page_list_count(page, tfree); + //mi_assert_internal(tfree_count <= page->thread_freed + 1); + #endif + + size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); + mi_assert_internal(page->used + free_count == page->capacity); + + return true; +} + +extern bool _mi_process_is_initialized; // has mi_process_init been called? + +bool _mi_page_is_valid(mi_page_t* page) { + mi_assert_internal(mi_page_is_valid_init(page)); + #if MI_SECURE + mi_assert_internal(page->keys[0] != 0); + #endif + if (mi_page_heap(page)!=NULL) { + mi_segment_t* segment = _mi_page_segment(page); + + mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == mi_page_heap(page)->thread_id); + #if MI_HUGE_PAGE_ABANDON + if (segment->kind != MI_SEGMENT_HUGE) + #endif + { + mi_page_queue_t* pq = mi_page_queue_of(page); + mi_assert_internal(mi_page_queue_contains(pq, page)); + mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_in_full(page)); + mi_assert_internal(mi_heap_contains_queue(mi_page_heap(page),pq)); + } + } + return true; +} +#endif + +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { + while (!_mi_page_try_use_delayed_free(page, delay, override_never)) { + mi_atomic_yield(); + } +} + +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { + mi_thread_free_t tfreex; + mi_delayed_t old_delay; + mi_thread_free_t tfree; + size_t yield_count = 0; + do { + tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS; + tfreex = mi_tf_set_delayed(tfree, delay); + old_delay = mi_tf_delayed(tfree); + if mi_unlikely(old_delay == MI_DELAYED_FREEING) { + if (yield_count >= 4) return false; // give up after 4 tries + yield_count++; + mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. + // tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail + } + else if (delay == old_delay) { + break; // avoid atomic operation if already equal + } + else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE) { + break; // leave never-delayed flag set + } + } while ((old_delay == MI_DELAYED_FREEING) || + !mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex)); + + return true; // success +} + +/* ----------------------------------------------------------- + Page collect the `local_free` and `thread_free` lists +----------------------------------------------------------- */ + +// Collect the local `thread_free` list using an atomic exchange. +// Note: The exchange must be done atomically as this is used right after +// moving to the full list in `mi_page_collect_ex` and we need to +// ensure that there was no race where the page became unfull just before the move. +static void _mi_page_thread_free_collect(mi_page_t* page) +{ + mi_block_t* head; + mi_thread_free_t tfreex; + mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); + do { + head = mi_tf_block(tfree); + tfreex = mi_tf_set_block(tfree,NULL); + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); + + // return if the list is empty + if (head == NULL) return; + + // find the tail -- also to get a proper count (without data races) + uint32_t max_count = page->capacity; // cannot collect more than capacity + uint32_t count = 1; + mi_block_t* tail = head; + mi_block_t* next; + while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) { + count++; + tail = next; + } + // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) + if (count > max_count) { + _mi_error_message(EFAULT, "corrupted thread-free list\n"); + return; // the thread-free items cannot be freed + } + + // and append the current local free list + mi_block_set_next(page,tail, page->local_free); + page->local_free = head; + + // update counts now + page->used -= count; +} + +void _mi_page_free_collect(mi_page_t* page, bool force) { + mi_assert_internal(page!=NULL); + + // collect the thread free list + if (force || mi_page_thread_free(page) != NULL) { // quick test to avoid an atomic operation + _mi_page_thread_free_collect(page); + } + + // and the local free list + if (page->local_free != NULL) { + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + + if mi_likely(page->free == NULL) { + // usual case + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + } + else if (force) { + // append -- only on shutdown (force) as this is a linear operation + mi_block_t* tail = page->local_free; + mi_block_t* next; + while ((next = mi_block_next(page, tail)) != NULL) { + tail = next; + } + mi_block_set_next(page, tail, page->free); + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + } + } + + mi_assert_internal(!force || page->local_free == NULL); +} + + + +/* ----------------------------------------------------------- + Page fresh and retire +----------------------------------------------------------- */ + +// called from segments when reclaiming abandoned pages +void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { + mi_assert_expensive(mi_page_is_valid_init(page)); + + mi_assert_internal(mi_page_heap(page) == heap); + mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + #endif + + // TODO: push on full queue immediately if it is full? + mi_page_queue_t* pq = mi_page_queue(heap, mi_page_block_size(page)); + mi_page_queue_push(heap, pq, page); + _PyMem_mi_page_reclaimed(page); + mi_assert_expensive(_mi_page_is_valid(page)); +} + +// allocate a fresh page from a segment +static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) { + #if !MI_HUGE_PAGE_ABANDON + mi_assert_internal(pq != NULL); + mi_assert_internal(mi_heap_contains_queue(heap, pq)); + mi_assert_internal(page_alignment > 0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || block_size == pq->block_size); + #endif + mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments, &heap->tld->os); + if (page == NULL) { + // this may be out-of-memory, or an abandoned page was reclaimed (and in our queue) + return NULL; + } + mi_assert_internal(page_alignment >0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || _mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + mi_assert_internal(pq!=NULL || page->xblock_size != 0); + mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size); + // a fresh page was found, initialize it + const size_t full_block_size = ((pq == NULL || mi_page_queue_is_huge(pq)) ? mi_page_block_size(page) : block_size); // see also: mi_segment_huge_page_alloc + mi_assert_internal(full_block_size >= block_size); + mi_page_init(heap, page, full_block_size, heap->tld); + mi_heap_stat_increase(heap, pages, 1); + if (pq != NULL) { mi_page_queue_push(heap, pq, page); } + mi_assert_expensive(_mi_page_is_valid(page)); + return page; +} + +// Get a fresh page to use +static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) { + mi_assert_internal(mi_heap_contains_queue(heap, pq)); + mi_page_t* page = mi_page_fresh_alloc(heap, pq, pq->block_size, 0); + if (page==NULL) return NULL; + mi_assert_internal(pq->block_size==mi_page_block_size(page)); + mi_assert_internal(pq==mi_page_queue(heap, mi_page_block_size(page))); + return page; +} + +/* ----------------------------------------------------------- + Do any delayed frees + (put there by other threads if they deallocated in a full page) +----------------------------------------------------------- */ +void _mi_heap_delayed_free_all(mi_heap_t* heap) { + while (!_mi_heap_delayed_free_partial(heap)) { + mi_atomic_yield(); + } +} + +// returns true if all delayed frees were processed +bool _mi_heap_delayed_free_partial(mi_heap_t* heap) { + // take over the list (note: no atomic exchange since it is often NULL) + mi_block_t* block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); + while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL)) { /* nothing */ }; + bool all_freed = true; + + // and free them all + while(block != NULL) { + mi_block_t* next = mi_block_nextx(heap,block, heap->keys); + // use internal free instead of regular one to keep stats etc correct + if (!_mi_free_delayed_block(block)) { + // we might already start delayed freeing while another thread has not yet + // reset the delayed_freeing flag; in that case delay it further by reinserting the current block + // into the delayed free list + all_freed = false; + mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); + do { + mi_block_set_nextx(heap, block, dfree, heap->keys); + } while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block)); + } + block = next; + } + return all_freed; +} + +/* ----------------------------------------------------------- + Unfull, abandon, free and retire +----------------------------------------------------------- */ + +// Move a page from the full list back to a regular list +void _mi_page_unfull(mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(mi_page_is_in_full(page)); + if (!mi_page_is_in_full(page)) return; + + mi_heap_t* heap = mi_page_heap(page); + mi_page_queue_t* pqfull = &heap->pages[MI_BIN_FULL]; + mi_page_set_in_full(page, false); // to get the right queue + mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page); + mi_page_set_in_full(page, true); + mi_page_queue_enqueue_from(pq, pqfull, page); +} + +static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { + mi_assert_internal(pq == mi_page_queue_of(page)); + mi_assert_internal(!mi_page_immediate_available(page)); + mi_assert_internal(!mi_page_is_in_full(page)); + + if (mi_page_is_in_full(page)) return; + mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page); + _mi_page_free_collect(page,false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set +} + + +// Abandon a page with used blocks at the end of a thread. +// Note: only call if it is ensured that no references exist from +// the `page->heap->thread_delayed_free` into this page. +// Currently only called through `mi_heap_collect_ex` which ensures this. +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(pq == mi_page_queue_of(page)); + mi_assert_internal(mi_page_heap(page) != NULL); + + mi_heap_t* pheap = mi_page_heap(page); + +#ifdef Py_GIL_DISABLED + if (page->qsbr_node.next != NULL) { + // remove from QSBR queue, but keep the goal + llist_remove(&page->qsbr_node); + } +#endif + + // remove from our page list + mi_segments_tld_t* segments_tld = &pheap->tld->segments; + mi_page_queue_remove(pq, page); + + // page is no longer associated with our heap + mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); + mi_page_set_heap(page, NULL); + +#if (MI_DEBUG>1) && !MI_TRACK_ENABLED + // check there are no references left.. + for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys)) { + mi_assert_internal(_mi_ptr_page(block) != page); + } +#endif + + // and abandon it + mi_assert_internal(mi_page_heap(page) == NULL); + _mi_segment_page_abandon(page,segments_tld); +} + + +// Free a page with no more free blocks +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(pq == mi_page_queue_of(page)); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING); + + // no more aligned blocks in here + mi_page_set_has_aligned(page, false); + + mi_heap_t* heap = mi_page_heap(page); + +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif + + // remove from the page list + // (no need to do _mi_heap_delayed_free first as all blocks are already free) + mi_segments_tld_t* segments_tld = &heap->tld->segments; + mi_page_queue_remove(pq, page); + + // and free it + mi_page_set_heap(page,NULL); + _mi_segment_page_free(page, force, segments_tld); +} + +// Retire parameters +#define MI_MAX_RETIRE_SIZE (MI_MEDIUM_OBJ_SIZE_MAX) +#define MI_RETIRE_CYCLES (16) + +// Retire a page with no more used blocks +// Important to not retire too quickly though as new +// allocations might coming. +// Note: called from `mi_free` and benchmarks often +// trigger this due to freeing everything and then +// allocating again so careful when changing this. +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(mi_page_all_free(page)); + + mi_page_set_has_aligned(page, false); + + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + + // don't retire too often.. + // (or we end up retiring and re-allocating most of the time) + // NOTE: refine this more: we should not retire if this + // is the only page left with free blocks. It is not clear + // how to check this efficiently though... + // for now, we don't retire if it is the only page left of this size class. + mi_page_queue_t* pq = mi_page_queue_of(page); + if mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_queue_is_special(pq)) { // not too large && not full or huge queue? + if (pq->last==page && pq->first==page) { // the only page in the queue? + mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); + page->retire_expire = 1 + (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); + mi_heap_t* heap = mi_page_heap(page); + mi_assert_internal(pq >= heap->pages); + const size_t index = pq - heap->pages; + mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE); + if (index < heap->page_retired_min) heap->page_retired_min = index; + if (index > heap->page_retired_max) heap->page_retired_max = index; + mi_assert_internal(mi_page_all_free(page)); + return; // dont't free after all + } + } + _PyMem_mi_page_maybe_free(page, pq, false); +} + +// free retired pages: we don't need to look at the entire queues +// since we only retire pages that are at the head position in a queue. +void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { + size_t min = MI_BIN_FULL; + size_t max = 0; + for(size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++) { + mi_page_queue_t* pq = &heap->pages[bin]; + mi_page_t* page = pq->first; + if (page != NULL && page->retire_expire != 0) { + if (mi_page_all_free(page)) { + page->retire_expire--; + if (force || page->retire_expire == 0) { +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); +#endif + _PyMem_mi_page_maybe_free(page, pq, force); + } + else { + // keep retired, update min/max + if (bin < min) min = bin; + if (bin > max) max = bin; + } + } + else { + page->retire_expire = 0; + } + } + } + heap->page_retired_min = min; + heap->page_retired_max = max; +} + + +/* ----------------------------------------------------------- + Initialize the initial free list in a page. + In secure mode we initialize a randomized list by + alternating between slices. +----------------------------------------------------------- */ + +#define MI_MAX_SLICE_SHIFT (6) // at most 64 slices +#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT) +#define MI_MIN_SLICES (2) + +static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { + MI_UNUSED(stats); + #if (MI_SECURE<=2) + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + #endif + mi_assert_internal(page->capacity + extend <= page->reserved); + mi_assert_internal(bsize == mi_page_block_size(page)); + void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL); + + // initialize a randomized free list + // set up `slice_count` slices to alternate between + size_t shift = MI_MAX_SLICE_SHIFT; + while ((extend >> shift) == 0) { + shift--; + } + const size_t slice_count = (size_t)1U << shift; + const size_t slice_extend = extend / slice_count; + mi_assert_internal(slice_extend >= 1); + mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice + size_t counts[MI_MAX_SLICES]; // available objects in the slice + for (size_t i = 0; i < slice_count; i++) { + blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend); + counts[i] = slice_extend; + } + counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?) + + // and initialize the free list by randomly threading through them + // set up first element + const uintptr_t r = _mi_heap_random_next(heap); + size_t current = r % slice_count; + counts[current]--; + mi_block_t* const free_start = blocks[current]; + // and iterate through the rest; use `random_shuffle` for performance + uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + for (size_t i = 1; i < extend; i++) { + // call random_shuffle only every INTPTR_SIZE rounds + const size_t round = i%MI_INTPTR_SIZE; + if (round == 0) rnd = _mi_random_shuffle(rnd); + // select a random next slice index + size_t next = ((rnd >> 8*round) & (slice_count-1)); + while (counts[next]==0) { // ensure it still has space + next++; + if (next==slice_count) next = 0; + } + // and link the current block to it + counts[next]--; + mi_block_t* const block = blocks[current]; + blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block + mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` + current = next; + } + // prepend to the free list (usually NULL) + mi_block_set_next(page, blocks[current], page->free); // end of the list + page->free = free_start; +} + +static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) +{ + MI_UNUSED(stats); + #if (MI_SECURE <= 2) + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + #endif + mi_assert_internal(page->capacity + extend <= page->reserved); + mi_assert_internal(bsize == mi_page_block_size(page)); + void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL ); + + mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity); + + // initialize a sequential free list + mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1); + mi_block_t* block = start; + while(block <= last) { + mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize); + mi_block_set_next(page,block,next); + block = next; + } + // prepend to free list (usually `NULL`) + mi_block_set_next(page, last, page->free); + page->free = start; +} + +/* ----------------------------------------------------------- + Page initialize and extend the capacity +----------------------------------------------------------- */ + +#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well. +#if (MI_SECURE>0) +#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many +#else +#define MI_MIN_EXTEND (4) +#endif + +// Extend the capacity (up to reserved) by initializing a free list +// We do at most `MI_MAX_EXTEND` to avoid touching too much memory +// Note: we also experimented with "bump" allocation on the first +// allocations but this did not speed up any benchmark (due to an +// extra test in malloc? or cache effects?) +static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) { + MI_UNUSED(tld); + mi_assert_expensive(mi_page_is_valid_init(page)); + #if (MI_SECURE<=2) + mi_assert(page->free == NULL); + mi_assert(page->local_free == NULL); + if (page->free != NULL) return; + #endif + if (page->capacity >= page->reserved) return; + + size_t page_size; + _mi_page_start(_mi_page_segment(page), page, &page_size); + mi_stat_counter_increase(tld->stats.pages_extended, 1); + + // calculate the extend count + const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size); + size_t extend = page->reserved - page->capacity; + mi_assert_internal(extend > 0); + + size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)bsize); + if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; } + mi_assert_internal(max_extend > 0); + + if (extend > max_extend) { + // ensure we don't touch memory beyond the page to reduce page commit. + // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%. + extend = max_extend; + } + + mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved); + mi_assert_internal(extend < (1UL<<16)); + + // and append the extend the free list + if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) { + mi_page_free_list_extend(page, bsize, extend, &tld->stats ); + } + else { + mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats); + } + // enable the new free list + page->capacity += (uint16_t)extend; + mi_stat_increase(tld->stats.page_committed, extend * bsize); + mi_assert_expensive(mi_page_is_valid_init(page)); +} + +// Initialize a fresh page +static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_tld_t* tld) { + mi_assert(page != NULL); + mi_segment_t* segment = _mi_page_segment(page); + mi_assert(segment != NULL); + mi_assert_internal(block_size > 0); + // set fields + mi_page_set_heap(page, heap); + page->tag = heap->tag; + page->use_qsbr = heap->page_use_qsbr; + page->debug_offset = heap->debug_offset; + page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start + size_t page_size; + const void* page_start = _mi_segment_page_start(segment, page, &page_size); + MI_UNUSED(page_start); + mi_track_mem_noaccess(page_start,page_size); + mi_assert_internal(mi_page_block_size(page) <= page_size); + mi_assert_internal(page_size <= page->slice_count*MI_SEGMENT_SLICE_SIZE); + mi_assert_internal(page_size / block_size < (1L<<16)); + page->reserved = (uint16_t)(page_size / block_size); + mi_assert_internal(page->reserved > 0); + #if (MI_PADDING || MI_ENCODE_FREELIST) + page->keys[0] = _mi_heap_random_next(heap); + page->keys[1] = _mi_heap_random_next(heap); + #endif + page->free_is_zero = page->is_zero_init; + #if MI_DEBUG>2 + if (page->is_zero_init) { + mi_track_mem_defined(page_start, page_size); + mi_assert_expensive(mi_mem_is_zero(page_start, page_size)); + } + #endif + + mi_assert_internal(page->is_committed); + mi_assert_internal(page->capacity == 0); + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->used == 0); + mi_assert_internal(page->xthread_free == 0); + mi_assert_internal(page->next == NULL); + mi_assert_internal(page->prev == NULL); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif + mi_assert_internal(page->retire_expire == 0); + mi_assert_internal(!mi_page_has_aligned(page)); + #if (MI_PADDING || MI_ENCODE_FREELIST) + mi_assert_internal(page->keys[0] != 0); + mi_assert_internal(page->keys[1] != 0); + #endif + mi_assert_expensive(mi_page_is_valid_init(page)); + + // initialize an initial free list + mi_page_extend_free(heap,page,tld); + mi_assert(mi_page_immediate_available(page)); +} + + +/* ----------------------------------------------------------- + Find pages with free blocks +-------------------------------------------------------------*/ + +// Find a page with free blocks of `page->block_size`. +static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq, bool first_try) +{ + // search through the pages in "next fit" order + #if MI_STAT + size_t count = 0; + #endif + mi_page_t* page = pq->first; + while (page != NULL) + { + mi_page_t* next = page->next; // remember next + #if MI_STAT + count++; + #endif + + // 0. collect freed blocks by us and other threads + _mi_page_free_collect(page, false); + + // 1. if the page contains free blocks, we are done + if (mi_page_immediate_available(page)) { + break; // pick this one + } + + // 2. Try to extend + if (page->capacity < page->reserved) { + mi_page_extend_free(heap, page, heap->tld); + mi_assert_internal(mi_page_immediate_available(page)); + break; + } + + // 3. If the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + mi_page_to_full(page, pq); + + page = next; + } // for each page + + mi_heap_stat_counter_increase(heap, searches, count); + + if (page == NULL) { + _PyMem_mi_heap_collect_qsbr(heap); // some pages might be safe to free now + _mi_heap_collect_retired(heap, false); // perhaps make a page available? + page = mi_page_fresh(heap, pq); + if (page == NULL && first_try) { + // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again + page = mi_page_queue_find_free_ex(heap, pq, false); + } + } + else { + mi_assert(pq->first == page); + page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); + } + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + return page; +} + + + +// Find a page with free blocks of `size`. +static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { + mi_page_queue_t* pq = mi_page_queue(heap,size); + mi_page_t* page = pq->first; + if (page != NULL) { + #if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness + if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) { + mi_page_extend_free(heap, page, heap->tld); + mi_assert_internal(mi_page_immediate_available(page)); + } + else + #endif + { + _mi_page_free_collect(page,false); + } + + if (mi_page_immediate_available(page)) { + page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); + return page; // fast path + } + } + return mi_page_queue_find_free_ex(heap, pq, true); +} + + +/* ----------------------------------------------------------- + Users can register a deferred free function called + when the `free` list is empty. Since the `local_free` + is separate this is deterministically called after + a certain number of allocations. +----------------------------------------------------------- */ + +static mi_deferred_free_fun* volatile deferred_free = NULL; +static _Atomic(void*) deferred_arg; // = NULL + +void _mi_deferred_free(mi_heap_t* heap, bool force) { + heap->tld->heartbeat++; + if (deferred_free != NULL && !heap->tld->recurse) { + heap->tld->recurse = true; + deferred_free(force, heap->tld->heartbeat, mi_atomic_load_ptr_relaxed(void,&deferred_arg)); + heap->tld->recurse = false; + } +} + +void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { + deferred_free = fn; + mi_atomic_store_ptr_release(void,&deferred_arg, arg); +} + + +/* ----------------------------------------------------------- + General allocation +----------------------------------------------------------- */ + +// Large and huge page allocation. +// Huge pages are allocated directly without being in a queue. +// Because huge pages contain just one block, and the segment contains +// just that page, we always treat them as abandoned and any thread +// that frees the block can free the whole page and segment directly. +// Huge pages are also use if the requested alignment is very large (> MI_ALIGNMENT_MAX). +static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) { + size_t block_size = _mi_os_good_alloc_size(size); + mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); + bool is_huge = (block_size > MI_LARGE_OBJ_SIZE_MAX || page_alignment > 0); + #if MI_HUGE_PAGE_ABANDON + mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap, block_size)); + #else + mi_page_queue_t* pq = mi_page_queue(heap, is_huge ? MI_HUGE_BLOCK_SIZE : block_size); // not block_size as that can be low if the page_alignment > 0 + mi_assert_internal(!is_huge || mi_page_queue_is_huge(pq)); + #endif + mi_page_t* page = mi_page_fresh_alloc(heap, pq, block_size, page_alignment); + if (page != NULL) { + mi_assert_internal(mi_page_immediate_available(page)); + + if (is_huge) { + mi_assert_internal(_mi_page_segment(page)->kind == MI_SEGMENT_HUGE); + mi_assert_internal(_mi_page_segment(page)->used==1); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue + mi_page_set_heap(page, NULL); + #endif + } + else { + mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE); + } + + const size_t bsize = mi_page_usable_block_size(page); // note: not `mi_page_block_size` to account for padding + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_increase(heap, large, bsize); + mi_heap_stat_counter_increase(heap, large_count, 1); + } + else { + mi_heap_stat_increase(heap, huge, bsize); + mi_heap_stat_counter_increase(heap, huge_count, 1); + } + } + return page; +} + + +// Allocate a page +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignment) mi_attr_noexcept { + // huge allocation? + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if mi_unlikely(req_size > (MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE) || huge_alignment > 0) { + if mi_unlikely(req_size > PTRDIFF_MAX) { // we don't allocate more than PTRDIFF_MAX (see ) + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + else { + _PyMem_mi_heap_collect_qsbr(heap); + return mi_large_huge_page_alloc(heap,size,huge_alignment); + } + } + else { + // otherwise find a page with free blocks in our size segregated queues + #if MI_PADDING + mi_assert_internal(size >= MI_PADDING_SIZE); + #endif + return mi_find_free_page(heap, size); + } +} + +// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SEGMENT_SIZE for +// very large requested alignments in which case we use a huge segment. +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept +{ + mi_assert_internal(heap != NULL); + + // initialize if necessary + if mi_unlikely(!mi_heap_is_initialized(heap)) { + heap = mi_heap_get_default(); // calls mi_thread_init + if mi_unlikely(!mi_heap_is_initialized(heap)) { return NULL; } + } + mi_assert_internal(mi_heap_is_initialized(heap)); + + // call potential deferred free routines + _mi_deferred_free(heap, false); + + // free delayed frees from other threads (but skip contended ones) + _mi_heap_delayed_free_partial(heap); + + // find (or allocate) a page of the right size + mi_page_t* page = mi_find_page(heap, size, huge_alignment); + if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more + mi_heap_collect(heap, true /* force */); + page = mi_find_page(heap, size, huge_alignment); + } + + if mi_unlikely(page == NULL) { // out of memory + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); + return NULL; + } + + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_page_block_size(page) >= size); + + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + if mi_unlikely(zero && page->xblock_size == 0) { + // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case. + void* p = _mi_page_malloc(heap, page, size, false); + mi_assert_internal(p != NULL); + _mi_memzero_aligned(p, mi_page_usable_block_size(page)); + return p; + } + else { + return _mi_page_malloc(heap, page, size, zero); + } +} diff --git a/Objects/mimalloc/prim/osx/alloc-override-zone.c b/Objects/mimalloc/prim/osx/alloc-override-zone.c new file mode 100644 index 00000000000000..0e0a99d9388c8a --- /dev/null +++ b/Objects/mimalloc/prim/osx/alloc-override-zone.c @@ -0,0 +1,458 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" + +#if defined(MI_MALLOC_OVERRIDE) + +#if !defined(__APPLE__) +#error "this file should only be included on macOS" +#endif + +/* ------------------------------------------------------ + Override system malloc on macOS + This is done through the malloc zone interface. + It seems to be most robust in combination with interposing + though or otherwise we may get zone errors as there are could + be allocations done by the time we take over the + zone. +------------------------------------------------------ */ + +#include +#include +#include // memset +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) +// only available from OSX 10.6 +extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); +#endif + +/* ------------------------------------------------------ + malloc zone members +------------------------------------------------------ */ + +static size_t zone_size(malloc_zone_t* zone, const void* p) { + MI_UNUSED(zone); + if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out + return mi_usable_size(p); +} + +static void* zone_malloc(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_malloc(size); +} + +static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { + MI_UNUSED(zone); + return mi_calloc(count, size); +} + +static void* zone_valloc(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_malloc_aligned(size, _mi_os_page_size()); +} + +static void zone_free(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); + mi_cfree(p); +} + +static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { + MI_UNUSED(zone); + return mi_realloc(p, newsize); +} + +static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { + MI_UNUSED(zone); + return mi_malloc_aligned(size,alignment); +} + +static void zone_destroy(malloc_zone_t* zone) { + MI_UNUSED(zone); + // todo: ignore for now? +} + +static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { + size_t i; + for (i = 0; i < count; i++) { + ps[i] = zone_malloc(zone, size); + if (ps[i] == NULL) break; + } + return i; +} + +static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { + for(size_t i = 0; i < count; i++) { + zone_free(zone, ps[i]); + ps[i] = NULL; + } +} + +static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); MI_UNUSED(size); + mi_collect(false); + return 0; +} + +static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { + MI_UNUSED(size); + zone_free(zone,p); +} + +static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); + return mi_is_in_heap_region(p); +} + + +/* ------------------------------------------------------ + Introspection members +------------------------------------------------------ */ + +static kern_return_t intro_enumerator(task_t task, void* p, + unsigned type_mask, vm_address_t zone_address, + memory_reader_t reader, + vm_range_recorder_t recorder) +{ + // todo: enumerate all memory + MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); + MI_UNUSED(reader); MI_UNUSED(recorder); + return KERN_SUCCESS; +} + +static size_t intro_good_size(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_good_size(size); +} + +static boolean_t intro_check(malloc_zone_t* zone) { + MI_UNUSED(zone); + return true; +} + +static void intro_print(malloc_zone_t* zone, boolean_t verbose) { + MI_UNUSED(zone); MI_UNUSED(verbose); + mi_stats_print(NULL); +} + +static void intro_log(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); MI_UNUSED(p); + // todo? +} + +static void intro_force_lock(malloc_zone_t* zone) { + MI_UNUSED(zone); + // todo? +} + +static void intro_force_unlock(malloc_zone_t* zone) { + MI_UNUSED(zone); + // todo? +} + +static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { + MI_UNUSED(zone); + // todo... + stats->blocks_in_use = 0; + stats->size_in_use = 0; + stats->max_size_in_use = 0; + stats->size_allocated = 0; +} + +static boolean_t intro_zone_locked(malloc_zone_t* zone) { + MI_UNUSED(zone); + return false; +} + + +/* ------------------------------------------------------ + At process start, override the default allocator +------------------------------------------------------ */ + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wc99-extensions" +#endif + +static malloc_introspection_t mi_introspect = { + .enumerator = &intro_enumerator, + .good_size = &intro_good_size, + .check = &intro_check, + .print = &intro_print, + .log = &intro_log, + .force_lock = &intro_force_lock, + .force_unlock = &intro_force_unlock, +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) + .statistics = &intro_statistics, + .zone_locked = &intro_zone_locked, +#endif +}; + +static malloc_zone_t mi_malloc_zone = { + // note: even with designators, the order is important for C++ compilation + //.reserved1 = NULL, + //.reserved2 = NULL, + .size = &zone_size, + .malloc = &zone_malloc, + .calloc = &zone_calloc, + .valloc = &zone_valloc, + .free = &zone_free, + .realloc = &zone_realloc, + .destroy = &zone_destroy, + .zone_name = "mimalloc", + .batch_malloc = &zone_batch_malloc, + .batch_free = &zone_batch_free, + .introspect = &mi_introspect, +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) + .version = 10, + #else + .version = 9, + #endif + // switch to version 9+ on OSX 10.6 to support memalign. + .memalign = &zone_memalign, + .free_definite_size = &zone_free_definite_size, + .pressure_relief = &zone_pressure_relief, + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) + .claimed_address = &zone_claimed_address, + #endif +#else + .version = 4, +#endif +}; + +#ifdef __cplusplus +} +#endif + + +#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) + +// ------------------------------------------------------ +// Override malloc_xxx and malloc_zone_xxx api's to use only +// our mimalloc zone. Since even the loader uses malloc +// on macOS, this ensures that all allocations go through +// mimalloc (as all calls are interposed). +// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, +// Here, we also override macOS specific API's like +// `malloc_zone_calloc` etc. see +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + static bool init; + if mi_unlikely(!init) { + init = true; + malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see ) + } + return &mi_malloc_zone; +} + +mi_decl_externc int malloc_jumpstart(uintptr_t cookie); +mi_decl_externc void _malloc_fork_prepare(void); +mi_decl_externc void _malloc_fork_parent(void); +mi_decl_externc void _malloc_fork_child(void); + + +static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { + MI_UNUSED(size); MI_UNUSED(flags); + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_zone (void) { + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { + return mi_get_default_zone(); +} + +static void mi_malloc_destroy_zone(malloc_zone_t* zone) { + MI_UNUSED(zone); + // nothing. +} + +static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { + MI_UNUSED(task); MI_UNUSED(mr); + if (addresses != NULL) *addresses = NULL; + if (count != NULL) *count = 0; + return KERN_SUCCESS; +} + +static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { + return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); +} + +static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { + MI_UNUSED(zone); MI_UNUSED(name); +} + +static int mi_malloc_jumpstart(uintptr_t cookie) { + MI_UNUSED(cookie); + return 1; // or 0 for no error? +} + +static void mi__malloc_fork_prepare(void) { + // nothing +} +static void mi__malloc_fork_parent(void) { + // nothing +} +static void mi__malloc_fork_child(void) { + // nothing +} + +static void mi_malloc_printf(const char* fmt, ...) { + MI_UNUSED(fmt); +} + +static bool zone_check(malloc_zone_t* zone) { + MI_UNUSED(zone); + return true; +} + +static malloc_zone_t* zone_from_ptr(const void* p) { + MI_UNUSED(p); + return mi_get_default_zone(); +} + +static void zone_log(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); MI_UNUSED(p); +} + +static void zone_print(malloc_zone_t* zone, bool b) { + MI_UNUSED(zone); MI_UNUSED(b); +} + +static void zone_print_ptr_info(void* p) { + MI_UNUSED(p); +} + +static void zone_register(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + +static void zone_unregister(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + +// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` +// See: +struct mi_interpose_s { + const void* replacement; + const void* target; +}; +#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } +#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) +#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) +__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = +{ + + MI_INTERPOSE_MI(malloc_create_zone), + MI_INTERPOSE_MI(malloc_default_purgeable_zone), + MI_INTERPOSE_MI(malloc_default_zone), + MI_INTERPOSE_MI(malloc_destroy_zone), + MI_INTERPOSE_MI(malloc_get_all_zones), + MI_INTERPOSE_MI(malloc_get_zone_name), + MI_INTERPOSE_MI(malloc_jumpstart), + MI_INTERPOSE_MI(malloc_printf), + MI_INTERPOSE_MI(malloc_set_zone_name), + MI_INTERPOSE_MI(_malloc_fork_child), + MI_INTERPOSE_MI(_malloc_fork_parent), + MI_INTERPOSE_MI(_malloc_fork_prepare), + + MI_INTERPOSE_ZONE(zone_batch_free), + MI_INTERPOSE_ZONE(zone_batch_malloc), + MI_INTERPOSE_ZONE(zone_calloc), + MI_INTERPOSE_ZONE(zone_check), + MI_INTERPOSE_ZONE(zone_free), + MI_INTERPOSE_ZONE(zone_from_ptr), + MI_INTERPOSE_ZONE(zone_log), + MI_INTERPOSE_ZONE(zone_malloc), + MI_INTERPOSE_ZONE(zone_memalign), + MI_INTERPOSE_ZONE(zone_print), + MI_INTERPOSE_ZONE(zone_print_ptr_info), + MI_INTERPOSE_ZONE(zone_realloc), + MI_INTERPOSE_ZONE(zone_register), + MI_INTERPOSE_ZONE(zone_unregister), + MI_INTERPOSE_ZONE(zone_valloc) +}; + + +#else + +// ------------------------------------------------------ +// hook into the zone api's without interposing +// This is the official way of adding an allocator but +// it seems less robust than using interpose. +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + // The first returned zone is the real default + malloc_zone_t** zones = NULL; + unsigned count = 0; + kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); + if (ret == KERN_SUCCESS && count > 0) { + return zones[0]; + } + else { + // fallback + return malloc_default_zone(); + } +} + +#if defined(__clang__) +__attribute__((constructor(0))) +#else +__attribute__((constructor)) // seems not supported by g++-11 on the M1 +#endif +static void _mi_macos_override_malloc(void) { + malloc_zone_t* purgeable_zone = NULL; + + #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) + // force the purgeable zone to exist to avoid strange bugs + if (malloc_default_purgeable_zone) { + purgeable_zone = malloc_default_purgeable_zone(); + } + #endif + + // Register our zone. + // thomcc: I think this is still needed to put us in the zone list. + malloc_zone_register(&mi_malloc_zone); + // Unregister the default zone, this makes our zone the new default + // as that was the last registered. + malloc_zone_t *default_zone = mi_get_default_zone(); + // thomcc: Unsure if the next test is *always* false or just false in the + // cases I've tried. I'm also unsure if the code inside is needed. at all + if (default_zone != &mi_malloc_zone) { + malloc_zone_unregister(default_zone); + + // Reregister the default zone so free and realloc in that zone keep working. + malloc_zone_register(default_zone); + } + + // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs + // earlier than the default zone. + if (purgeable_zone != NULL) { + malloc_zone_unregister(purgeable_zone); + malloc_zone_register(purgeable_zone); + } + +} +#endif // MI_OSX_INTERPOSE + +#endif // MI_MALLOC_OVERRIDE diff --git a/Objects/mimalloc/prim/osx/prim.c b/Objects/mimalloc/prim/osx/prim.c new file mode 100644 index 00000000000000..8a2f4e8aa47316 --- /dev/null +++ b/Objects/mimalloc/prim/osx/prim.c @@ -0,0 +1,9 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// We use the unix/prim.c with the mmap API on macOSX +#include "../unix/prim.c" diff --git a/Objects/mimalloc/prim/prim.c b/Objects/mimalloc/prim/prim.c new file mode 100644 index 00000000000000..9a597d8eb603e9 --- /dev/null +++ b/Objects/mimalloc/prim/prim.c @@ -0,0 +1,24 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Select the implementation of the primitives +// depending on the OS. + +#if defined(_WIN32) +#include "windows/prim.c" // VirtualAlloc (Windows) + +#elif defined(__APPLE__) +#include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c) + +#elif defined(__wasi__) +#define MI_USE_SBRK +#include "wasi/prim.c" // memory-grow or sbrk (Wasm) + +#else +#include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) + +#endif diff --git a/Objects/mimalloc/prim/readme.md b/Objects/mimalloc/prim/readme.md new file mode 100644 index 00000000000000..380dd3a7178419 --- /dev/null +++ b/Objects/mimalloc/prim/readme.md @@ -0,0 +1,9 @@ +## Portability Primitives + +This is the portability layer where all primitives needed from the OS are defined. + +- `include/mimalloc/prim.h`: primitive portability API definition. +- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform + (and on macOS, `osx/prim.c` defers to `unix/prim.c`). + +Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c new file mode 100644 index 00000000000000..c4816af1a0d6e5 --- /dev/null +++ b/Objects/mimalloc/prim/unix/prim.c @@ -0,0 +1,860 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined +#endif + +#if defined(__sun) +// illumos provides new mman.h api when any of these are defined +// otherwise the old api based on caddr_t which predates the void pointers one. +// stock solaris provides only the former, chose to atomically to discard those +// flags only here rather than project wide tough. +#undef _XOPEN_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" + +#include // mmap +#include // sysconf +#include // open, close, read, access + +#if defined(__linux__) + #include + #include + #if defined(__GLIBC__) + #include // linux mmap flags + #else + #include + #endif +#elif defined(__APPLE__) + #include + #if !TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR + #include + #endif +#elif defined(__FreeBSD__) || defined(__DragonFly__) + #include + #if __FreeBSD_version >= 1200000 + #include + #include + #endif + #include +#endif + +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) + #define MI_HAS_SYSCALL_H + #include +#endif + +//------------------------------------------------------------------------------------ +// Use syscalls for some primitives to allow for libraries that override open/read/close etc. +// and do allocation themselves; using syscalls prevents recursion when mimalloc is +// still initializing (issue #713) +//------------------------------------------------------------------------------------ + +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) + +static int mi_prim_open(const char* fpath, int open_flags) { + return syscall(SYS_open,fpath,open_flags,0); +} +static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return syscall(SYS_read,fd,buf,bufsize); +} +static int mi_prim_close(int fd) { + return syscall(SYS_close,fd); +} +static int mi_prim_access(const char *fpath, int mode) { + return syscall(SYS_access,fpath,mode); +} + +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings + +static int mi_prim_open(const char* fpath, int open_flags) { + return open(fpath,open_flags); +} +static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return read(fd,buf,bufsize); +} +static int mi_prim_close(int fd) { + return close(fd); +} +static int mi_prim_access(const char *fpath, int mode) { + return access(fpath,mode); +} + +#endif + + + +//--------------------------------------------- +// init +//--------------------------------------------- + +static bool unix_detect_overcommit(void) { + bool os_overcommit = true; +#if defined(__linux__) + int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd >= 0) { + char buf[32] = {0}; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } +#else + // default: overcommit is true +#endif + return os_overcommit; +} + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { + long psize = sysconf(_SC_PAGESIZE); + if (psize > 0) { + config->page_size = (size_t)psize; + config->alloc_granularity = (size_t)psize; + } + config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? + config->has_overcommit = unix_detect_overcommit(); + config->must_free_whole = false; // mmap can free in parts + config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE) +} + + +//--------------------------------------------- +// free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + bool err = (munmap(addr, size) == -1); + return (err ? errno : 0); +} + + +//--------------------------------------------- +// mmap +//--------------------------------------------- + +static int unix_madvise(void* addr, size_t size, int advice) { + #if defined(__sun) + return madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) + #else + return madvise(addr, size, advice); + #endif +} + +static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + MI_UNUSED(try_alignment); + void* p = NULL; + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_verbose_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); + } + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif + #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? + int err = 0; + #else + int err = errno; + #endif + _mi_verbose_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #endif + // regular mmap + p = mmap(addr, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; +} + +static int unix_mmap_fd(void) { + #if defined(VM_MAKE_TAG) + // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) + int os_tag = (int)mi_option_get(mi_option_os_tag); + if (os_tag < 100 || os_tag > 255) { os_tag = 100; } + return VM_MAKE_TAG(os_tag); + #else + return -1; + #endif +} + +static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { + #if !defined(MAP_ANONYMOUS) + #define MAP_ANONYMOUS MAP_ANON + #endif + #if !defined(MAP_NORESERVE) + #define MAP_NORESERVE 0 + #endif + void* p = NULL; + const int fd = unix_mmap_fd(); + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + if (_mi_os_has_overcommit()) { + flags |= MAP_NORESERVE; + } + #if defined(PROT_MAX) + protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD + #endif + // huge page allocation + if ((large_only || _mi_os_use_large_page(size, try_alignment)) && allow_large) { + static _Atomic(size_t) large_page_try_ok; // = 0; + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // If the OS is not configured for large OS pages, or the user does not have + // enough permission, the `mmap` will always fail (but it might also fail for other reasons). + // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times + // to avoid too many failing calls to mmap. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux + int lfd = fd; + #ifdef MAP_ALIGNED_SUPER + lflags |= MAP_ALIGNED_SUPER; + #endif + #ifdef MAP_HUGETLB + lflags |= MAP_HUGETLB; + #endif + #ifdef MAP_HUGE_1GB + static bool mi_huge_pages_available = true; + if ((size % MI_GiB) == 0 && mi_huge_pages_available) { + lflags |= MAP_HUGE_1GB; + } + else + #endif + { + #ifdef MAP_HUGE_2MB + lflags |= MAP_HUGE_2MB; + #endif + } + #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB + lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; + #endif + if (large_only || lflags != flags) { + // try large OS page allocation + *is_large = true; + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); + #ifdef MAP_HUGE_1GB + if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { + mi_huge_pages_available = false; // don't try huge 1GiB pages again + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); + lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); + } + #endif + if (large_only) return p; + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations + } + } + } + } + // regular allocation + if (p == NULL) { + *is_large = false; + p = unix_mmap_prim(addr, size, try_alignment, protect_flags, flags, fd); + if (p != NULL) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (allow_large && _mi_os_use_large_page(size, try_alignment)) { + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; + } + #elif defined(__sun) + if (allow_large && _mi_os_use_large_page(size, try_alignment)) { + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = 2*MI_MiB; + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + *is_large = true; + } + } + #endif + } + } + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + + *is_zero = true; + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : errno); +} + + +//--------------------------------------------- +// Commit/Reset +//--------------------------------------------- + +static void unix_mprotect_hint(int err) { + #if defined(__linux__) && (MI_SECURE>=2) // guard page around every mimalloc page + if (err == ENOMEM) { + _mi_warning_message("The next warning may be caused by a low memory map limit.\n" + " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" + " For example: sudo sysctl -w vm.max_map_count=262144\n"); + } + #else + MI_UNUSED(err); + #endif +} + +int _mi_prim_commit(void* start, size_t size, bool* is_zero) { + // commit: ensure we can access the area + // note: we may think that *is_zero can be true since the memory + // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but + // we sometimes call commit on a range with still partially committed + // memory and `mprotect` does not zero the range. + *is_zero = false; + int err = mprotect(start, size, (PROT_READ | PROT_WRITE)); + if (err != 0) { + err = errno; + unix_mprotect_hint(err); + } + return err; +} + +int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { + int err = 0; + // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + err = unix_madvise(start, size, MADV_DONTNEED); + #if !MI_DEBUG && !MI_SECURE + *needs_recommit = false; + #else + *needs_recommit = true; + mprotect(start, size, PROT_NONE); + #endif + /* + // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss) + *needs_recommit = true; + const int fd = unix_mmap_fd(); + void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); + if (p != start) { err = errno; } + */ + return err; +} + +int _mi_prim_reset(void* start, size_t size) { + // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // will not reduce the `rss` stats in tools like `top` even though the memory is available + // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by + // default `MADV_DONTNEED` is used though. + #if defined(MADV_FREE) + static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); + int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; + while ((err = unix_madvise(start, size, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); + err = unix_madvise(start, size, MADV_DONTNEED); + } + #else + int err = unix_madvise(start, size, MADV_DONTNEED); + #endif + return err; +} + +int _mi_prim_protect(void* start, size_t size, bool protect) { + int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); + if (err != 0) { err = errno; } + unix_mprotect_hint(err); + return err; +} + + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) + +#ifndef MPOL_PREFERRED +#define MPOL_PREFERRED 1 +#endif + +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind) +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags); +} +#else +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); + return 0; +} +#endif + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + bool is_large = true; + *is_zero = true; + *addr = unix_mmap(hint_addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); + if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + unsigned long numa_mask = (1UL << numa_node); + // TODO: does `mbind` work correctly for huge OS pages? should we + // use `set_mempolicy` before calling mmap instead? + // see: + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + if (err != 0) { + err = errno; + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); + } + } + return (*addr != NULL ? 0 : errno); +} + +#else + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *is_zero = false; + *addr = NULL; + return ENOMEM; +} + +#endif + +//--------------------------------------------- +// NUMA nodes +//--------------------------------------------- + +#if defined(__linux__) + +#include // snprintf + +size_t _mi_prim_numa_node(void) { + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu) + unsigned long node = 0; + unsigned long ncpu = 0; + long err = syscall(SYS_getcpu, &ncpu, &node, NULL); + if (err != 0) return 0; + return node; + #else + return 0; + #endif +} + +size_t _mi_prim_numa_node_count(void) { + char buf[128]; + unsigned node = 0; + for(node = 0; node < 256; node++) { + // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) + snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); + if (mi_prim_access(buf,R_OK) != 0) break; + } + return (node+1); +} + +#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 + +size_t _mi_prim_numa_node(void) { + domainset_t dom; + size_t node; + int policy; + if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; + for (node = 0; node < MAXMEMDOM; node++) { + if (DOMAINSET_ISSET(node, &dom)) return node; + } + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ndomains = 0; + size_t len = sizeof(ndomains); + if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; + return ndomains; +} + +#elif defined(__DragonFly__) + +size_t _mi_prim_numa_node(void) { + // TODO: DragonFly does not seem to provide any userland means to get this information. + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ncpus = 0, nvirtcoresperphys = 0; + size_t len = sizeof(size_t); + if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; + if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; + return nvirtcoresperphys * ncpus; +} + +#else + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + +#endif + +// ---------------------------------------------------------------- +// Clock +// ---------------------------------------------------------------- + +#include + +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + +mi_msecs_t _mi_prim_clock_now(void) { + struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else + clock_gettime(CLOCK_REALTIME, &t); + #endif + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); +} + +#else + +// low resolution timer +mi_msecs_t _mi_prim_clock_now(void) { + #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) + return (mi_msecs_t)clock(); + #elif (CLOCKS_PER_SEC < 1000) + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + #else + return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + #endif +} + +#endif + + + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) +#include +#include +#include + +#if defined(__APPLE__) +#include +#endif + +#if defined(__HAIKU__) +#include +#endif + +static mi_msecs_t timeval_secs(const struct timeval* tv) { + return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); +} + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + struct rusage rusage; + getrusage(RUSAGE_SELF, &rusage); + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); +#if !defined(__HAIKU__) + pinfo->page_faults = rusage.ru_majflt; +#endif +#if defined(__HAIKU__) + // Haiku does not have (yet?) a way to + // get these stats per process + thread_info tid; + area_info mem; + ssize_t c; + get_thread_info(find_thread(0), &tid); + while (get_next_area_info(tid.team, &c, &mem) == B_OK) { + pinfo->peak_rss += mem.ram_size; + } + pinfo->page_faults = 0; +#elif defined(__APPLE__) + pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes + #ifdef MACH_TASK_BASIC_INFO + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #else + struct task_basic_info info; + mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #endif +#else + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB +#endif + // use defaults for commit +} + +#else + +#ifndef __wasi__ +// WebAssembly instances are not processes +#pragma message("define a way to get process info") +#endif + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + // use defaults + MI_UNUSED(pinfo); +} + +#endif + + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) +// On Posix systemsr use `environ` to access environment variables +// even before the C runtime is initialized. +#if defined(__APPLE__) && defined(__has_include) && __has_include() +#include +static char** mi_get_environ(void) { + return (*_NSGetEnviron()); +} +#else +extern char** environ; +static char** mi_get_environ(void) { + return environ; +} +#endif +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + if (name==NULL) return false; + const size_t len = _mi_strlen(name); + if (len == 0) return false; + char** env = mi_get_environ(); + if (env == NULL) return false; + // compare up to 10000 entries + for (int i = 0; i < 10000 && env[i] != NULL; i++) { + const char* s = env[i]; + if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive + // found it + _mi_strlcpy(result, s + len + 1, result_size); + return true; + } + } + return false; +} +#else +// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return false; + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + _mi_strlcpy(result, s, result_size); + return true; +} +#endif // !MI_USE_ENVIRON + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(__APPLE__) + +#include +#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 +#include +#include +#endif +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // may fail silently on macOS. See PR #390, and + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + #else + // fall back on older macOS + arc4random_buf(buf, buf_len); + return true; + #endif +} + +#elif defined(__ANDROID__) || defined(__DragonFly__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__sun) + +#include +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + arc4random_buf(buf, buf_len); + return true; +} + +#elif defined(__linux__) || defined(__HAIKU__) + +#include +#include +#include +#include + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h` + // and for the latter the actual `getrandom` call is not always defined. + // (see ) + // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed. + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom) + #ifndef GRND_NONBLOCK + #define GRND_NONBLOCK (1) + #endif + static _Atomic(uintptr_t) no_getrandom; // = 0 + if (mi_atomic_load_acquire(&no_getrandom)==0) { + ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); + if (ret >= 0) return (buf_len == (size_t)ret); + if (errno != ENOSYS) return false; + mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom + } + #endif + int flags = O_RDONLY; + #if defined(O_CLOEXEC) + flags |= O_CLOEXEC; + #endif + int fd = mi_prim_open("/dev/urandom", flags); + if (fd < 0) return false; + size_t count = 0; + while(count < buf_len) { + ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count); + if (ret<=0) { + if (errno!=EAGAIN && errno!=EINTR) break; + } + else { + count += ret; + } + } + mi_prim_close(fd); + return (count==buf_len); +} + +#else + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} + +#endif + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if defined(MI_USE_PTHREADS) + +// use pthread local storage keys to detect thread ending +// (and used with MI_TLS_PTHREADS for the default heap) +pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); + +static void mi_pthread_done(void* value) { + if (value!=NULL) { + _mi_thread_done((mi_heap_t*)value); + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); + pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing to do +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD + pthread_setspecific(_mi_heap_default_key, heap); + } +} + +#else + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} + +#endif diff --git a/Objects/mimalloc/prim/wasi/prim.c b/Objects/mimalloc/prim/wasi/prim.c new file mode 100644 index 00000000000000..640dac0493f573 --- /dev/null +++ b/Objects/mimalloc/prim/wasi/prim.c @@ -0,0 +1,276 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" +#include // sbrk() + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { + config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB + config->alloc_granularity = 16; + config->has_overcommit = false; + config->must_free_whole = true; + config->has_virtual_reserve = false; +} + +//--------------------------------------------- +// Free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(addr); MI_UNUSED(size); + // wasi heap cannot be shrunk + return 0; +} + + +//--------------------------------------------- +// Allocation: sbrk or memory_grow +//--------------------------------------------- + +#if defined(MI_USE_SBRK) + static void* mi_memory_grow( size_t size ) { + void* p = sbrk(size); + if (p == (void*)(-1)) return NULL; + #if !defined(__wasi__) // on wasi this is always zero initialized already (?) + memset(p,0,size); + #endif + return p; + } +#elif defined(__wasi__) + static void* mi_memory_grow( size_t size ) { + size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) + : __builtin_wasm_memory_size(0)); + if (base == SIZE_MAX) return NULL; + return (void*)(base * _mi_os_page_size()); + } +#endif + +#if defined(MI_USE_PTHREADS) +static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + +static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { + void* p = NULL; + if (try_alignment <= 1) { + // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + p = mi_memory_grow(size); + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + } + else { + void* base = NULL; + size_t alloc_size = 0; + // to allocate aligned use a lock to try to avoid thread interaction + // between getting the current size and actual allocation + // (also, `sbrk` is not thread safe in general) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + { + void* current = mi_memory_grow(0); // get current size + if (current != NULL) { + void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space + alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); + base = mi_memory_grow(alloc_size); + } + } + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + if (base != NULL) { + p = mi_align_up_ptr(base, try_alignment); + if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { + // another thread used wasm_memory_grow/sbrk in-between and we do not have enough + // space after alignment. Give up (and waste the space as we cannot shrink :-( ) + // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) + p = NULL; + } + } + } + /* + if (p == NULL) { + _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); + errno = ENOMEM; + return NULL; + } + */ + mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(allow_large); MI_UNUSED(commit); + *is_large = false; + *is_zero = false; + *addr = mi_prim_mem_grow(size, try_alignment); + return (*addr != NULL ? 0 : ENOMEM); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- + +int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { + MI_UNUSED(addr); MI_UNUSED(size); + *is_zero = false; + return 0; +} + +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { + MI_UNUSED(addr); MI_UNUSED(size); + *needs_recommit = false; + return 0; +} + +int _mi_prim_reset(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); + return 0; +} + + +//--------------------------------------------- +// Huge pages and NUMA nodes +//--------------------------------------------- + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *is_zero = true; + *addr = NULL; + return ENOSYS; +} + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +#include + +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + +mi_msecs_t _mi_prim_clock_now(void) { + struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else + clock_gettime(CLOCK_REALTIME, &t); + #endif + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); +} + +#else + +// low resolution timer +mi_msecs_t _mi_prim_clock_now(void) { + #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) + return (mi_msecs_t)clock(); + #elif (CLOCKS_PER_SEC < 1000) + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + #else + return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + #endif +} + +#endif + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + // use defaults + MI_UNUSED(pinfo); +} + + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return false; + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false; + _mi_strlcpy(result, s, result_size); + return true; +} + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} diff --git a/Objects/mimalloc/prim/windows/etw-mimalloc.wprp b/Objects/mimalloc/prim/windows/etw-mimalloc.wprp new file mode 100644 index 00000000000000..b00cd7adf2285c --- /dev/null +++ b/Objects/mimalloc/prim/windows/etw-mimalloc.wprp @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Objects/mimalloc/prim/windows/etw.h b/Objects/mimalloc/prim/windows/etw.h new file mode 100644 index 00000000000000..e9ec35fc9ed7eb --- /dev/null +++ b/Objects/mimalloc/prim/windows/etw.h @@ -0,0 +1,905 @@ +//**********************************************************************` +//* This is an include file generated by Message Compiler. *` +//* *` +//* Copyright (c) Microsoft Corporation. All Rights Reserved. *` +//**********************************************************************` +#pragma once + +//***************************************************************************** +// +// Notes on the ETW event code generated by MC: +// +// - Structures and arrays of structures are treated as an opaque binary blob. +// The caller is responsible for packing the data for the structure into a +// single region of memory, with no padding between values. The macro will +// have an extra parameter for the length of the blob. +// - Arrays of nul-terminated strings must be packed by the caller into a +// single binary blob containing the correct number of strings, with a nul +// after each string. The size of the blob is specified in characters, and +// includes the final nul. +// - Arrays of SID are treated as a single binary blob. The caller is +// responsible for packing the SID values into a single region of memory with +// no padding. +// - The length attribute on the data element in the manifest is significant +// for values with intype win:UnicodeString, win:AnsiString, or win:Binary. +// The length attribute must be specified for win:Binary, and is optional for +// win:UnicodeString and win:AnsiString (if no length is given, the strings +// are assumed to be nul-terminated). For win:UnicodeString, the length is +// measured in characters, not bytes. +// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the +// length attribute applies to every value in the array, so every value in +// the array must have the same length. The values in the array are provided +// to the macro via a single pointer -- the caller is responsible for packing +// all of the values into a single region of memory with no padding between +// values. +// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary can be generated and collected on Vista or later. +// However, they may not decode properly without the Windows 10 2018 Fall +// Update. +// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary must be packed by the caller into a single region of +// memory. The format for each item is a UINT16 byte-count followed by that +// many bytes of data. When providing the array to the generated macro, you +// must provide the total size of the packed array data, including the UINT16 +// sizes for each item. In the case of win:CountedUnicodeString, the data +// size is specified in WCHAR (16-bit) units. In the case of +// win:CountedAnsiString and win:CountedBinary, the data size is specified in +// bytes. +// +//***************************************************************************** + +#include +#include +#include + +#ifndef ETW_INLINE + #ifdef _ETW_KM_ + // In kernel mode, save stack space by never inlining templates. + #define ETW_INLINE DECLSPEC_NOINLINE __inline + #else + // In user mode, save code size by inlining templates as appropriate. + #define ETW_INLINE __inline + #endif +#endif // ETW_INLINE + +#if defined(__cplusplus) +extern "C" { +#endif + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_USE_KERNEL_MODE_APIS macro: +// Controls whether the generated code uses kernel-mode or user-mode APIs. +// - Set to 0 to use Windows user-mode APIs such as EventRegister. +// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister. +// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h). +// Note that the APIs can also be overridden directly, e.g. by setting the +// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros. +// +#ifndef MCGEN_USE_KERNEL_MODE_APIS + #ifdef _ETW_KM_ + #define MCGEN_USE_KERNEL_MODE_APIS 1 + #else + #define MCGEN_USE_KERNEL_MODE_APIS 0 + #endif +#endif // MCGEN_USE_KERNEL_MODE_APIS + +// +// MCGEN_HAVE_EVENTSETINFORMATION macro: +// Controls how McGenEventSetInformation uses the EventSetInformation API. +// - Set to 0 to disable the use of EventSetInformation +// (McGenEventSetInformation will always return an error). +// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION. +// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress +// (user-mode) or MmGetSystemRoutineAddress (kernel-mode). +// Default is determined as follows: +// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else if the target OS version has EventSetInformation, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else set to 2 (i.e. try to dynamically locate EventSetInformation). +// Note that an McGenEventSetInformation function will only be generated if one +// or more provider in a manifest has provider traits. +// +#ifndef MCGEN_HAVE_EVENTSETINFORMATION + #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs, + #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EtwSetInformation" via MmGetSystemRoutineAddress. + #endif // else (using user-mode APIs) + #else // if target OS and SDK is Windows 8 or later, + #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED) + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EventSetInformation" via GetModuleHandleExW/GetProcAddress. + #endif + #endif +#endif // MCGEN_HAVE_EVENTSETINFORMATION + +// +// MCGEN Override Macros +// +// The following override macros may be defined before including this header +// to control the APIs used by this header: +// +// - MCGEN_EVENTREGISTER +// - MCGEN_EVENTUNREGISTER +// - MCGEN_EVENTSETINFORMATION +// - MCGEN_EVENTWRITETRANSFER +// +// If the macro is undefined, the MC implementation will default to the +// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is +// undefined, the EventRegister[MyProviderName] macro will use EventRegister +// in user mode and will use EtwRegister in kernel mode. +// +// To prevent issues from conflicting definitions of these macros, the value +// of the override macro will be used as a suffix in certain internal function +// names. Because of this, the override macros must follow certain rules: +// +// - The macro must be defined before any MC-generated header is included and +// must not be undefined or redefined after any MC-generated header is +// included. Different translation units (i.e. different .c or .cpp files) +// may set the macros to different values, but within a translation unit +// (within a single .c or .cpp file), the macro must be set once and not +// changed. +// - The override must be an object-like macro, not a function-like macro +// (i.e. the override macro must not have a parameter list). +// - The override macro's value must be a simple identifier, i.e. must be +// something that starts with a letter or '_' and contains only letters, +// numbers, and '_' characters. +// - If the override macro's value is the name of a second object-like macro, +// the second object-like macro must follow the same rules. (The override +// macro's value can also be the name of a function-like macro, in which +// case the function-like macro does not need to follow the same rules.) +// +// For example, the following will cause compile errors: +// +// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon). +// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses). +// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro. +// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon). +// +// The following would be ok: +// +// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be "MyEventWriteFunction1". +// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2 +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be "MyEventWriteFunction2". +// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d) +// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO". +// +#ifndef MCGEN_EVENTREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTREGISTER EtwRegister + #else + #define MCGEN_EVENTREGISTER EventRegister + #endif +#endif // MCGEN_EVENTREGISTER +#ifndef MCGEN_EVENTUNREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTUNREGISTER EtwUnregister + #else + #define MCGEN_EVENTUNREGISTER EventUnregister + #endif +#endif // MCGEN_EVENTUNREGISTER +#ifndef MCGEN_EVENTSETINFORMATION + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTSETINFORMATION EtwSetInformation + #else + #define MCGEN_EVENTSETINFORMATION EventSetInformation + #endif +#endif // MCGEN_EVENTSETINFORMATION +#ifndef MCGEN_EVENTWRITETRANSFER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer + #else + #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer + #endif +#endif // MCGEN_EVENTWRITETRANSFER + +// +// MCGEN_EVENT_ENABLED macro: +// Override to control how the EventWrite[EventName] macros determine whether +// an event is enabled. The default behavior is for EventWrite[EventName] to +// use the EventEnabled[EventName] macros. +// +#ifndef MCGEN_EVENT_ENABLED +#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName() +#endif + +// +// MCGEN_EVENT_ENABLED_FORCONTEXT macro: +// Override to control how the EventWrite[EventName]_ForContext macros +// determine whether an event is enabled. The default behavior is for +// EventWrite[EventName]_ForContext to use the +// EventEnabled[EventName]_ForContext macros. +// +#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT +#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext) +#endif + +// +// MCGEN_ENABLE_CHECK macro: +// Determines whether the specified event would be considered as enabled +// based on the state of the specified context. Slightly faster than calling +// McGenEventEnabled directly. +// +#ifndef MCGEN_ENABLE_CHECK +#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor)) +#endif + +#if !defined(MCGEN_TRACE_CONTEXT_DEF) +#define MCGEN_TRACE_CONTEXT_DEF +// This structure is for use by MC-generated code and should not be used directly. +typedef struct _MCGEN_TRACE_CONTEXT +{ + TRACEHANDLE RegistrationHandle; + TRACEHANDLE Logger; // Used as pointer to provider traits. + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + ULONG Flags; + ULONG IsEnabled; + UCHAR Level; + UCHAR Reserve; + USHORT EnableBitsCount; + PULONG EnableBitMask; + const ULONGLONG* EnableKeyWords; + const UCHAR* EnableLevel; +} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT; +#endif // MCGEN_TRACE_CONTEXT_DEF + +#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF) +#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF +// +// Determines whether an event with a given Level and Keyword would be +// considered as enabled based on the state of the specified context. +// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this +// function directly. +// +FORCEINLINE +BOOLEAN +McGenLevelKeywordEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ UCHAR Level, + _In_ ULONGLONG Keyword + ) +{ + // + // Check if the event Level is lower than the level at which + // the channel is enabled. + // If the event Level is 0 or the channel is enabled at level 0, + // all levels are enabled. + // + + if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0. + (EnableInfo->Level == 0)) { + + // + // Check if Keyword is enabled + // + + if ((Keyword == (ULONGLONG)0) || + ((Keyword & EnableInfo->MatchAnyKeyword) && + ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) { + return TRUE; + } + } + + return FALSE; +} +#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF + +#if !defined(MCGEN_EVENT_ENABLED_DEF) +#define MCGEN_EVENT_ENABLED_DEF +// +// Determines whether the specified event would be considered as enabled based +// on the state of the specified context. Note that you may want to use +// MCGEN_ENABLE_CHECK instead of calling this function directly. +// +FORCEINLINE +BOOLEAN +McGenEventEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ PCEVENT_DESCRIPTOR EventDescriptor + ) +{ + return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword); +} +#endif // MCGEN_EVENT_ENABLED_DEF + +#if !defined(MCGEN_CONTROL_CALLBACK) +#define MCGEN_CONTROL_CALLBACK + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +VOID +__stdcall +McGenControlCallbackV2( + _In_ LPCGUID SourceId, + _In_ ULONG ControlCode, + _In_ UCHAR Level, + _In_ ULONGLONG MatchAnyKeyword, + _In_ ULONGLONG MatchAllKeyword, + _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData, + _Inout_opt_ PVOID CallbackContext + ) +/*++ + +Routine Description: + + This is the notification callback for Windows Vista and later. + +Arguments: + + SourceId - The GUID that identifies the session that enabled the provider. + + ControlCode - The parameter indicates whether the provider + is being enabled or disabled. + + Level - The level at which the event is enabled. + + MatchAnyKeyword - The bitmask of keywords that the provider uses to + determine the category of events that it writes. + + MatchAllKeyword - This bitmask additionally restricts the category + of events that the provider writes. + + FilterData - The provider-defined data. + + CallbackContext - The context of the callback that is defined when the provider + called EtwRegister to register itself. + +Remarks: + + ETW calls this function to notify provider of enable/disable + +--*/ +{ + PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext; + ULONG Ix; +#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + UNREFERENCED_PARAMETER(SourceId); + UNREFERENCED_PARAMETER(FilterData); +#endif + + if (Ctx == NULL) { + return; + } + + switch (ControlCode) { + + case EVENT_CONTROL_CODE_ENABLE_PROVIDER: + Ctx->Level = Level; + Ctx->MatchAnyKeyword = MatchAnyKeyword; + Ctx->MatchAllKeyword = MatchAllKeyword; + Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER; + + for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) { + if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) { + Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32)); + } else { + Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32)); + } + } + break; + + case EVENT_CONTROL_CODE_DISABLE_PROVIDER: + Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER; + Ctx->Level = 0; + Ctx->MatchAnyKeyword = 0; + Ctx->MatchAllKeyword = 0; + if (Ctx->EnableBitsCount > 0) { +#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount + RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG)); + } + break; + + default: + break; + } + +#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + // + // Call user defined callback + // + MCGEN_PRIVATE_ENABLE_CALLBACK_V2( + SourceId, + ControlCode, + Level, + MatchAnyKeyword, + MatchAllKeyword, + FilterData, + CallbackContext + ); +#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + + return; +} + +#endif // MCGEN_CONTROL_CALLBACK + +#ifndef _mcgen_PENABLECALLBACK + #if MCGEN_USE_KERNEL_MODE_APIS + #define _mcgen_PENABLECALLBACK PETWENABLECALLBACK + #else + #define _mcgen_PENABLECALLBACK PENABLECALLBACK + #endif +#endif // _mcgen_PENABLECALLBACK + +#if !defined(_mcgen_PASTE2) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b) +#define _mcgen_PASTE2_imp(a, b) a##b +#endif // _mcgen_PASTE2 + +#if !defined(_mcgen_PASTE3) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c) +#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c +#endif // _mcgen_PASTE3 + +// +// Macro validation +// + +// Validate MCGEN_EVENTREGISTER: + +// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER); + +// Trigger an error if MCGEN_EVENTREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER) + MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER); + +// Validate MCGEN_EVENTUNREGISTER: + +// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER); + +// Trigger an error if MCGEN_EVENTUNREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER) + MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER); + +// Validate MCGEN_EVENTSETINFORMATION: + +// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION); + +// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION) + MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro: +typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION; +typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION); + +// Validate MCGEN_EVENTWRITETRANSFER: + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER); + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER) + MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;; + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro: +typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER; +typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER); + +#ifndef McGenEventWrite_def +#define McGenEventWrite_def + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventWrite( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_opt_ LPCGUID ActivityId, + _In_range_(1, 128) ULONG EventDataCount, + _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData + ) +{ + const USHORT UNALIGNED* Traits; + + // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId. + UNREFERENCED_PARAMETER(ActivityId); + + Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger; + + if (Traits == NULL) { + EventData[0].Ptr = 0; + EventData[0].Size = 0; + EventData[0].Reserved = 0; + } else { + EventData[0].Ptr = (ULONG_PTR)Traits; + EventData[0].Size = *Traits; + EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA + } + + return MCGEN_EVENTWRITETRANSFER( + Context->RegistrationHandle, + Descriptor, + ActivityId, + NULL, + EventDataCount, + EventData); +} +#endif // McGenEventWrite_def + +#if !defined(McGenEventRegisterUnregister) +#define McGenEventRegisterUnregister + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER) + +#pragma warning(push) +#pragma warning(disable:6103) +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventRegister( + _In_ LPCGUID ProviderId, + _In_opt_ _mcgen_PENABLECALLBACK EnableCallback, + _In_opt_ PVOID CallbackContext, + _Inout_ PREGHANDLE RegHandle + ) +/*++ + +Routine Description: + + This function registers the provider with ETW. + +Arguments: + + ProviderId - Provider ID to register with ETW. + + EnableCallback - Callback to be used. + + CallbackContext - Context for the callback. + + RegHandle - Pointer to registration handle. + +Remarks: + + Should not be called if the provider is already registered (i.e. should not + be called if *RegHandle != 0). Repeatedly registering a provider is a bug + and may indicate a race condition. However, for compatibility with previous + behavior, this function will return SUCCESS in this case. + +--*/ +{ + ULONG Error; + + if (*RegHandle != 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle); + } + + return Error; +} +#pragma warning(pop) + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventUnregister(_Inout_ PREGHANDLE RegHandle) +/*++ + +Routine Description: + + Unregister from ETW and set *RegHandle = 0. + +Arguments: + + RegHandle - the pointer to the provider registration handle + +Remarks: + + If provider has not been registered (i.e. if *RegHandle == 0), + return SUCCESS. It is safe to call McGenEventUnregister even if the + call to McGenEventRegister returned an error. + +--*/ +{ + ULONG Error; + + if(*RegHandle == 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTUNREGISTER(*RegHandle); + *RegHandle = (REGHANDLE)0; + } + + return Error; +} + +#endif // McGenEventRegisterUnregister + +#ifndef _mcgen_EVENT_BIT_SET + #if defined(_M_IX86) || defined(_M_X64) + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0) + #else // CPU type + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0) + #endif // CPU type +#endif // _mcgen_EVENT_BIT_SET + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// Provider "microsoft-windows-mimalloc" event count 2 +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779 +EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}}; + +#ifndef ETW_MI_Provider_Traits +#define ETW_MI_Provider_Traits NULL +#endif // ETW_MI_Provider_Traits + +// +// Event Descriptors +// +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_ALLOC_value 0x64 +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_FREE_value 0x65 + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Event Enablement Bits +// These variables are for use by MC-generated code and should not be used directly. +// +EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1]; +EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0}; +EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4}; + +// +// Provider context +// +EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels}; + +// +// Provider REGHANDLE +// +#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle) + +// +// This macro is set to 0, indicating that the EventWrite[Name] macros do not +// have an Activity parameter. This is controlled by the -km and -um options. +// +#define ETW_MI_Provider_EventWriteActivity 0 + +// +// Register with ETW using the control GUID specified in the manifest. +// Invoke this macro during module initialization (i.e. program startup, +// DLL process attach, or driver load) to initialize the provider. +// Note that if this function returns an error, the error means that +// will not work, but no action needs to be taken -- even if EventRegister +// returns an error, it is generally safe to use EventWrite and +// EventUnregister macros (they will be no-ops if EventRegister failed). +// +#ifndef EventRegistermicrosoft_windows_mimalloc +#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Register with ETW using a specific control GUID (i.e. a GUID other than what +// is specified in the manifest). Advanced scenarios only. +// +#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc +#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Unregister with ETW and close the provider. +// Invoke this macro during module shutdown (i.e. program exit, DLL process +// detach, or driver unload) to unregister the provider. +// Note that you MUST call EventUnregister before DLL or driver unload +// (not optional): failure to unregister a provider before DLL or driver unload +// will result in crashes. +// +#ifndef EventUnregistermicrosoft_windows_mimalloc +#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle) +#endif + +// +// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro: +// Define this macro to enable support for caller-allocated provider context. +// +#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Advanced scenarios: Caller-allocated provider context. +// Use when multiple differently-configured provider handles are needed, +// e.g. for container-aware drivers, one context per container. +// +// Usage: +// +// - Caller enables the feature before including this header, e.g. +// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1 +// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc)); +// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...); +// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller frees memory, e.g. free(pContext); +// + +typedef struct tagMcGenContext_microsoft_windows_mimalloc { + // The fields of this structure are subject to change and should + // not be accessed directly. To access the provider's REGHANDLE, + // use microsoft_windows_mimallocHandle_ForContext(pContext). + MCGEN_TRACE_CONTEXT Context; + ULONG EnableBits[1]; +} McGenContext_microsoft_windows_mimalloc; + +#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext) +#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext) +#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle) + +// +// Provider REGHANDLE for caller-allocated context. +// +#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle) + +// This function is for use by MC-generated code and should not be used directly. +// Initialize and register the caller-allocated context. +__inline +ULONG __stdcall +_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)( + _In_ LPCGUID pProviderId, + _Out_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + RtlZeroMemory(pContext, sizeof(*pContext)); + pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits; + pContext->Context.EnableBitsCount = 1; + pContext->Context.EnableBitMask = pContext->EnableBits; + pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords; + pContext->Context.EnableLevel = microsoft_windows_mimallocLevels; + return McGenEventRegister( + pProviderId, + McGenControlCallbackV2, + &pContext->Context, + &pContext->Context.RegistrationHandle); +} + +// This function is for use by MC-generated code and should not be used directly. +// Trigger a compile error if called with the wrong parameter type. +FORCEINLINE +_Ret_ McGenContext_microsoft_windows_mimalloc* +_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + return pContext; +} + +#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Enablement check macro for event "ETW_MI_ALLOC" +// +#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_ALLOC" +// +#define EventWriteETW_MI_ALLOC(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) +#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +// +// Enablement check macro for event "ETW_MI_FREE" +// +#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_FREE" +// +#define EventWriteETW_MI_FREE(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) +#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Template Functions +// + +// +// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others). +// This function is for use by MC-generated code and should not be used directly. +// +#ifndef McTemplateU0xx_def +#define McTemplateU0xx_def +ETW_INLINE +ULONG +_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_ const unsigned __int64 _Arg0, + _In_ const unsigned __int64 _Arg1 + ) +{ +#define McTemplateU0xx_ARGCOUNT 2 + + EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1]; + + EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) ); + + EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) ); + + return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData); +} +#endif // McTemplateU0xx_def + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +#if defined(__cplusplus) +} +#endif diff --git a/Objects/mimalloc/prim/windows/etw.man b/Objects/mimalloc/prim/windows/etw.man new file mode 100644 index 00000000000000..cfd1f8a9eaacd5 Binary files /dev/null and b/Objects/mimalloc/prim/windows/etw.man differ diff --git a/Objects/mimalloc/prim/windows/prim.c b/Objects/mimalloc/prim/windows/prim.c new file mode 100644 index 00000000000000..a038277ad21cb0 --- /dev/null +++ b/Objects/mimalloc/prim/windows/prim.c @@ -0,0 +1,622 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" +#include // fputs, stderr + + +//--------------------------------------------- +// Dynamically bind Windows API points for portability +//--------------------------------------------- + +// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. +// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) +// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB) +// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's. +typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E { + MiMemExtendedParameterInvalidType = 0, + MiMemExtendedParameterAddressRequirements, + MiMemExtendedParameterNumaNode, + MiMemExtendedParameterPartitionHandle, + MiMemExtendedParameterUserPhysicalHandle, + MiMemExtendedParameterAttributeFlags, + MiMemExtendedParameterMax +} MI_MEM_EXTENDED_PARAMETER_TYPE; + +typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S { + struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type; + union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg; +} MI_MEM_EXTENDED_PARAMETER; + +typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S { + PVOID LowestStartingAddress; + PVOID HighestEndingAddress; + SIZE_T Alignment; +} MI_MEM_ADDRESS_REQUIREMENTS; + +#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010 + +#include +typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); +typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); +static PVirtualAlloc2 pVirtualAlloc2 = NULL; +static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; + +// Similarly, GetNumaProcesorNodeEx is only supported since Windows 7 +typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER; + +typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber); +typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber); +typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask); +typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber); +static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL; +static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL; +static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL; +static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; + +//--------------------------------------------- +// Enable large page support dynamically (if possible) +//--------------------------------------------- + +static bool win_enable_large_os_pages(size_t* large_page_size) +{ + static bool large_initialized = false; + if (large_initialized) return (_mi_os_large_page_size() > 0); + large_initialized = true; + + // Try to see if large OS pages are supported + // To use large pages on Windows, we first need access permission + // Set "Lock pages in memory" permission in the group policy editor + // + unsigned long err = 0; + HANDLE token = NULL; + BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + if (ok) { + TOKEN_PRIVILEGES tp; + ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + if (ok) { + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + if (ok) { + err = GetLastError(); + ok = (err == ERROR_SUCCESS); + if (ok && large_page_size != NULL) { + *large_page_size = GetLargePageMinimum(); + } + } + } + CloseHandle(token); + } + if (!ok) { + if (err == 0) err = GetLastError(); + _mi_warning_message("cannot enable large OS page support, error %lu\n", err); + } + return (ok!=0); +} + + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) +{ + config->has_overcommit = false; + config->must_free_whole = true; + config->has_virtual_reserve = true; + // get the page size + SYSTEM_INFO si; + GetSystemInfo(&si); + if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; } + if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; } + // get the VirtualAlloc2 function + HINSTANCE hDll; + hDll = LoadLibrary(TEXT("kernelbase.dll")); + if (hDll != NULL) { + // use VirtualAlloc2FromApp if possible as it is available to Windows store apps + pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); + if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); + FreeLibrary(hDll); + } + // NtAllocateVirtualMemoryEx is used for huge page allocation + hDll = LoadLibrary(TEXT("ntdll.dll")); + if (hDll != NULL) { + pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); + FreeLibrary(hDll); + } + // Try to use Win7+ numa API + hDll = LoadLibrary(TEXT("kernel32.dll")); + if (hDll != NULL) { + pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); + pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); + pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx"); + pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode"); + FreeLibrary(hDll); + } + if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { + win_enable_large_os_pages(&config->large_page_size); + } +} + + +//--------------------------------------------- +// Free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(size); + DWORD errcode = 0; + bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + if (errcode == ERROR_INVALID_ADDRESS) { + // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside + // the memory region returned by VirtualAlloc; in that case we need to free using + // the start of the region. + MEMORY_BASIC_INFORMATION info = { 0 }; + VirtualQuery(addr, &info, sizeof(info)); + if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) { + errcode = 0; + err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + } + } + return (int)errcode; +} + + +//--------------------------------------------- +// VirtualAlloc +//--------------------------------------------- + +static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { + #if (MI_INTPTR_SIZE >= 8) + // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags); + // fall through on error + } + } + #endif + // on modern Windows try use VirtualAlloc2 for aligned allocation + if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; + reqs.Alignment = try_alignment; + MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; + param.Type.Type = MiMemExtendedParameterAddressRequirements; + param.Arg.Pointer = &reqs; + void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + if (p != NULL) return p; + _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); + // fall through on error + } + // last resort + return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +} + +static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { + mi_assert_internal(!(large_only && !allow_large)); + static _Atomic(size_t) large_page_try_ok; // = 0; + void* p = NULL; + // Try to allocate large OS pages (2MiB) if allowed or required. + if ((large_only || _mi_os_use_large_page(size, try_alignment)) + && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. + // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + // large OS pages must always reserve and commit. + *is_large = true; + p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES); + if (large_only) return p; + // fall back to non-large page allocation on error (`p == NULL`). + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations + } + } + } + // Fall back to regular page allocation + if (p == NULL) { + *is_large = ((flags&MEM_LARGE_PAGES) != 0); + p = win_virtual_alloc_prim(addr, size, try_alignment, flags); + } + //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); } + return p; +} + +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + *is_zero = true; + int flags = MEM_RESERVE; + if (commit) { flags |= MEM_COMMIT; } + *addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- +#ifdef _MSC_VER +#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit) +#endif + +int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { + *is_zero = false; + /* + // zero'ing only happens on an initial commit... but checking upfront seems expensive.. + _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo); + if (VirtualQuery(addr, &meminfo, size) > 0) { + if ((meminfo.State & MEM_COMMIT) == 0) { + *is_zero = true; + } + } + */ + // commit + void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE); + if (p == NULL) return (int)GetLastError(); + return 0; +} + +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { + BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT); + *needs_recommit = true; // for safety, assume always decommitted even in the case of an error. + return (ok ? 0 : (int)GetLastError()); +} + +int _mi_prim_reset(void* addr, size_t size) { + void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); + mi_assert_internal(p == addr); + #if 0 + if (p != NULL) { + VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set + } + #endif + return (p != NULL ? 0 : (int)GetLastError()); +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + DWORD oldprotect = 0; + BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); + return (ok ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node) +{ + const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; + + win_enable_large_os_pages(NULL); + + MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} }; + // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages + static bool mi_huge_pages_available = true; + if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) { + params[0].Type.Type = MiMemExtendedParameterAttributeFlags; + params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; + ULONG param_count = 1; + if (numa_node >= 0) { + param_count++; + params[1].Type.Type = MiMemExtendedParameterNumaNode; + params[1].Arg.ULong = (unsigned)numa_node; + } + SIZE_T psize = size; + void* base = hint_addr; + NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); + if (err == 0 && base != NULL) { + return base; + } + else { + // fall back to regular large pages + mi_huge_pages_available = false; // don't try further huge pages + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); + } + } + // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation + if (pVirtualAlloc2 != NULL && numa_node >= 0) { + params[0].Type.Type = MiMemExtendedParameterNumaNode; + params[0].Arg.ULong = (unsigned)numa_node; + return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1); + } + + // otherwise use regular virtual alloc on older windows + return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE); +} + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + *is_zero = true; + *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node); + return (*addr != NULL ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Numa nodes +//--------------------------------------------- + +size_t _mi_prim_numa_node(void) { + USHORT numa_node = 0; + if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) { + // Extended API is supported + MI_PROCESSOR_NUMBER pnum; + (*pGetCurrentProcessorNumberEx)(&pnum); + USHORT nnode = 0; + BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode); + if (ok) { numa_node = nnode; } + } + else if (pGetNumaProcessorNode != NULL) { + // Vista or earlier, use older API that is limited to 64 processors. Issue #277 + DWORD pnum = GetCurrentProcessorNumber(); + UCHAR nnode = 0; + BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode); + if (ok) { numa_node = nnode; } + } + return numa_node; +} + +size_t _mi_prim_numa_node_count(void) { + ULONG numa_max = 0; + GetNumaHighestNodeNumber(&numa_max); + // find the highest node number that has actual processors assigned to it. Issue #282 + while(numa_max > 0) { + if (pGetNumaNodeProcessorMaskEx != NULL) { + // Extended API is supported + GROUP_AFFINITY affinity; + if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) { + if (affinity.Mask != 0) break; // found the maximum non-empty node + } + } + else { + // Vista or earlier, use older API that is limited to 64 processors. + ULONGLONG mask; + if (GetNumaNodeProcessorMask((UCHAR)numa_max, &mask)) { + if (mask != 0) break; // found the maximum non-empty node + }; + } + // max node was invalid or had no processor assigned, try again + numa_max--; + } + return ((size_t)numa_max + 1); +} + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { + static LARGE_INTEGER mfreq; // = 0 + if (mfreq.QuadPart == 0LL) { + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + mfreq.QuadPart = f.QuadPart/1000LL; + if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; + } + return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); +} + +mi_msecs_t _mi_prim_clock_now(void) { + LARGE_INTEGER t; + QueryPerformanceCounter(&t); + return mi_to_msecs(t); +} + + +//---------------------------------------------------------------- +// Process Info +//---------------------------------------------------------------- + +#include +#include + +static mi_msecs_t filetime_msecs(const FILETIME* ftime) { + ULARGE_INTEGER i; + i.LowPart = ftime->dwLowDateTime; + i.HighPart = ftime->dwHighDateTime; + mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds + return msecs; +} + +typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); +static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + FILETIME ct; + FILETIME ut; + FILETIME st; + FILETIME et; + GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); + + // load psapi on demand + if (pGetProcessMemoryInfo == NULL) { + HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); + if (hDll != NULL) { + pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); + } + } + + // get process info + PROCESS_MEMORY_COUNTERS info; + memset(&info, 0, sizeof(info)); + if (pGetProcessMemoryInfo != NULL) { + pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); + } + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; +} + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) +{ + // on windows with redirection, the C runtime cannot handle locale dependent output + // after the main thread closes so we use direct console output. + if (!_mi_preloading()) { + // _cputs(msg); // _cputs cannot be used at is aborts if it fails to lock the console + static HANDLE hcon = INVALID_HANDLE_VALUE; + static bool hconIsConsole; + if (hcon == INVALID_HANDLE_VALUE) { + CONSOLE_SCREEN_BUFFER_INFO sbi; + hcon = GetStdHandle(STD_ERROR_HANDLE); + hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); + } + const size_t len = _mi_strlen(msg); + if (len > 0 && len < UINT32_MAX) { + DWORD written = 0; + if (hconIsConsole) { + WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + } + else if (hcon != INVALID_HANDLE_VALUE) { + // use direct write if stderr was redirected + WriteFile(hcon, msg, (DWORD)len, &written, NULL); + } + else { + // finally fall back to fputs after all + fputs(msg, stderr); + } + } + } +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +// On Windows use GetEnvironmentVariable instead of getenv to work +// reliably even when this is invoked before the C runtime is initialized. +// i.e. when `_mi_preloading() == true`. +// Note: on windows, environment names are not case sensitive. +bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { + result[0] = 0; + size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); + return (len > 0 && len < result_size); +} + + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) +// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using +// dynamic overriding, we observed it can raise an exception when compiled with C++, and +// sometimes deadlocks when also running under the VS debugger. +// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. +// To be continued.. +#pragma comment (lib,"advapi32.lib") +#define RtlGenRandom SystemFunction036 +mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return (RtlGenRandom(buf, (ULONG)buf_len) != 0); +} + +#else + +#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG +#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002 +#endif + +typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG); +static PBCryptGenRandom pBCryptGenRandom = NULL; + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + if (pBCryptGenRandom == NULL) { + HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll")); + if (hDll != NULL) { + pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); + } + if (pBCryptGenRandom == NULL) return false; + } + return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); +} + +#endif // MI_USE_RTLGENRANDOM + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if !defined(MI_SHARED_LIB) + +// use thread local storage keys to detect thread ending +#include +#if (_WIN32_WINNT < 0x600) // before Windows Vista +WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); +WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); +WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); +WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); +#endif + +static DWORD mi_fls_key = (DWORD)(-1); + +static void NTAPI mi_fls_done(PVOID value) { + mi_heap_t* heap = (mi_heap_t*)value; + if (heap != NULL) { + _mi_thread_done(heap); + FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_fls_key = FlsAlloc(&mi_fls_done); +} + +void _mi_prim_thread_done_auto_done(void) { + // call thread-done on all threads (except the main thread) to prevent + // dangling callback pointer if statically linked with a DLL; Issue #208 + FlsFree(mi_fls_key); +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + mi_assert_internal(mi_fls_key != (DWORD)(-1)); + FlsSetValue(mi_fls_key, heap); +} + +#else + +// Dll; nothing to do as in that case thread_done is handled through the DLL_THREAD_DETACH event. + +void _mi_prim_thread_init_auto_done(void) { +} + +void _mi_prim_thread_done_auto_done(void) { +} + +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); +} + +#endif diff --git a/Objects/mimalloc/prim/windows/readme.md b/Objects/mimalloc/prim/windows/readme.md new file mode 100644 index 00000000000000..217c3d174db473 --- /dev/null +++ b/Objects/mimalloc/prim/windows/readme.md @@ -0,0 +1,17 @@ +## Primitives: + +- `prim.c` contains Windows primitives for OS allocation. + +## Event Tracing for Windows (ETW) + +- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events. + (100 is an allocation, 101 is for a free) + +- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). + In an admin prompt, you can use: + ``` + > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode + > + > wpr -stop test.etl + ``` + and then open `test.etl` in the Windows Performance Analyzer (WPA). \ No newline at end of file diff --git a/Objects/mimalloc/random.c b/Objects/mimalloc/random.c new file mode 100644 index 00000000000000..35a0633a800d6c --- /dev/null +++ b/Objects/mimalloc/random.c @@ -0,0 +1,254 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // _mi_prim_random_buf +#include // memset + +/* ---------------------------------------------------------------------------- +We use our own PRNG to keep predictable performance of random number generation +and to avoid implementations that use a lock. We only use the OS provided +random source to initialize the initial seeds. Since we do not need ultimate +performance but we do rely on the security (for secret cookies in secure mode) +we use a cryptographically secure generator (chacha20). +-----------------------------------------------------------------------------*/ + +#define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance? + + +/* ---------------------------------------------------------------------------- +Chacha20 implementation as the original algorithm with a 64-bit nonce +and counter: https://en.wikipedia.org/wiki/Salsa20 +The input matrix has sixteen 32-bit values: +Position 0 to 3: constant key +Position 4 to 11: the key +Position 12 to 13: the counter. +Position 14 to 15: the nonce. + +The implementation uses regular C code which compiles very well on modern compilers. +(gcc x64 has no register spills, and clang 6+ uses SSE instructions) +-----------------------------------------------------------------------------*/ + +static inline uint32_t rotl(uint32_t x, uint32_t shift) { + return (x << shift) | (x >> (32 - shift)); +} + +static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) { + x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 16); + x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 12); + x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 8); + x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 7); +} + +static void chacha_block(mi_random_ctx_t* ctx) +{ + // scramble into `x` + uint32_t x[16]; + for (size_t i = 0; i < 16; i++) { + x[i] = ctx->input[i]; + } + for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) { + qround(x, 0, 4, 8, 12); + qround(x, 1, 5, 9, 13); + qround(x, 2, 6, 10, 14); + qround(x, 3, 7, 11, 15); + qround(x, 0, 5, 10, 15); + qround(x, 1, 6, 11, 12); + qround(x, 2, 7, 8, 13); + qround(x, 3, 4, 9, 14); + } + + // add scrambled data to the initial state + for (size_t i = 0; i < 16; i++) { + ctx->output[i] = x[i] + ctx->input[i]; + } + ctx->output_available = 16; + + // increment the counter for the next round + ctx->input[12] += 1; + if (ctx->input[12] == 0) { + ctx->input[13] += 1; + if (ctx->input[13] == 0) { // and keep increasing into the nonce + ctx->input[14] += 1; + } + } +} + +static uint32_t chacha_next32(mi_random_ctx_t* ctx) { + if (ctx->output_available <= 0) { + chacha_block(ctx); + ctx->output_available = 16; // (assign again to suppress static analysis warning) + } + const uint32_t x = ctx->output[16 - ctx->output_available]; + ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out + ctx->output_available--; + return x; +} + +static inline uint32_t read32(const uint8_t* p, size_t idx32) { + const size_t i = 4*idx32; + return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24); +} + +static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce) +{ + // since we only use chacha for randomness (and not encryption) we + // do not _need_ to read 32-bit values as little endian but we do anyways + // just for being compatible :-) + memset(ctx, 0, sizeof(*ctx)); + for (size_t i = 0; i < 4; i++) { + const uint8_t* sigma = (uint8_t*)"expand 32-byte k"; + ctx->input[i] = read32(sigma,i); + } + for (size_t i = 0; i < 8; i++) { + ctx->input[i + 4] = read32(key,i); + } + ctx->input[12] = 0; + ctx->input[13] = 0; + ctx->input[14] = (uint32_t)nonce; + ctx->input[15] = (uint32_t)(nonce >> 32); +} + +static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { + memset(ctx_new, 0, sizeof(*ctx_new)); + _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + ctx_new->input[12] = 0; + ctx_new->input[13] = 0; + ctx_new->input[14] = (uint32_t)nonce; + ctx_new->input[15] = (uint32_t)(nonce >> 32); + mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces! + chacha_block(ctx_new); +} + + +/* ---------------------------------------------------------------------------- +Random interface +-----------------------------------------------------------------------------*/ + +#if MI_DEBUG>1 +static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { + return (ctx != NULL && ctx->input[0] != 0); +} +#endif + +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { + mi_assert_internal(mi_random_is_initialized(ctx)); + mi_assert_internal(ctx != ctx_new); + chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new); +} + +uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { + mi_assert_internal(mi_random_is_initialized(ctx)); + #if MI_INTPTR_SIZE <= 4 + return chacha_next32(ctx); + #elif MI_INTPTR_SIZE == 8 + return (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); + #else + # error "define mi_random_next for this platform" + #endif +} + + +/* ---------------------------------------------------------------------------- +To initialize a fresh random context. +If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. +-----------------------------------------------------------------------------*/ + +uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { + uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random + x ^= _mi_prim_clock_now(); + // and do a few randomization steps + uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; + for (uintptr_t i = 0; i < max; i++) { + x = _mi_random_shuffle(x); + } + mi_assert_internal(x != 0); + return x; +} + +static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { + uint8_t key[32] = {0}; + if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) { + // if we fail to get random data from the OS, we fall back to a + // weak random source based on the current time + #if !defined(__wasi__) + if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } + #endif + uintptr_t x = _mi_os_random_weak(0); + for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. + x = _mi_random_shuffle(x); + ((uint32_t*)key)[i] = (uint32_t)x; + } + ctx->weak = true; + } + else { + ctx->weak = false; + } + chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); +} + +void _mi_random_init(mi_random_ctx_t* ctx) { + mi_random_init_ex(ctx, false); +} + +void _mi_random_init_weak(mi_random_ctx_t * ctx) { + mi_random_init_ex(ctx, true); +} + +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) { + if (ctx->weak) { + _mi_random_init(ctx); + } +} + +/* -------------------------------------------------------- +test vectors from +----------------------------------------------------------- */ +/* +static bool array_equals(uint32_t* x, uint32_t* y, size_t n) { + for (size_t i = 0; i < n; i++) { + if (x[i] != y[i]) return false; + } + return true; +} +static void chacha_test(void) +{ + uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 }; + uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb }; + qround(x, 0, 1, 2, 3); + mi_assert_internal(array_equals(x, x_out, 4)); + + uint32_t y[16] = { + 0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a, + 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c, + 0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963, + 0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 }; + uint32_t y_out[16] = { + 0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a, + 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2, + 0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963, + 0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 }; + qround(y, 2, 7, 8, 13); + mi_assert_internal(array_equals(y, y_out, 16)); + + mi_random_ctx_t r = { + { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574, + 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c, + 0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c, + 0x00000001, 0x09000000, 0x4a000000, 0x00000000 }, + {0}, + 0 + }; + uint32_t r_out[16] = { + 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3, + 0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3, + 0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9, + 0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 }; + chacha_block(&r); + mi_assert_internal(array_equals(r.output, r_out, 16)); +} +*/ diff --git a/Objects/mimalloc/segment-map.c b/Objects/mimalloc/segment-map.c new file mode 100644 index 00000000000000..3cd2127e56c1a7 --- /dev/null +++ b/Objects/mimalloc/segment-map.c @@ -0,0 +1,153 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + The following functions are to reliably find the segment or + block that encompasses any pointer p (or NULL if it is not + in any of our segments). + We maintain a bitmap of all memory with 1 bit per MI_SEGMENT_SIZE (64MiB) + set to 1 if it contains the segment meta data. +----------------------------------------------------------- */ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" + +#if (MI_INTPTR_SIZE==8) +#define MI_MAX_ADDRESS ((size_t)40 << 40) // 40TB (to include huge page areas) +#else +#define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb +#endif + +#define MI_SEGMENT_MAP_BITS (MI_MAX_ADDRESS / MI_SEGMENT_SIZE) +#define MI_SEGMENT_MAP_SIZE (MI_SEGMENT_MAP_BITS / 8) +#define MI_SEGMENT_MAP_WSIZE (MI_SEGMENT_MAP_SIZE / MI_INTPTR_SIZE) + +static _Atomic(uintptr_t) mi_segment_map[MI_SEGMENT_MAP_WSIZE + 1]; // 2KiB per TB with 64MiB segments + +static size_t mi_segment_map_index_of(const mi_segment_t* segment, size_t* bitidx) { + mi_assert_internal(_mi_ptr_segment(segment + 1) == segment); // is it aligned on MI_SEGMENT_SIZE? + if ((uintptr_t)segment >= MI_MAX_ADDRESS) { + *bitidx = 0; + return MI_SEGMENT_MAP_WSIZE; + } + else { + const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_SIZE; + *bitidx = segindex % MI_INTPTR_BITS; + const size_t mapindex = segindex / MI_INTPTR_BITS; + mi_assert_internal(mapindex < MI_SEGMENT_MAP_WSIZE); + return mapindex; + } +} + +void _mi_segment_map_allocated_at(const mi_segment_t* segment) { + size_t bitidx; + size_t index = mi_segment_map_index_of(segment, &bitidx); + mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); + if (index==MI_SEGMENT_MAP_WSIZE) return; + uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + uintptr_t newmask; + do { + newmask = (mask | ((uintptr_t)1 << bitidx)); + } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); +} + +void _mi_segment_map_freed_at(const mi_segment_t* segment) { + size_t bitidx; + size_t index = mi_segment_map_index_of(segment, &bitidx); + mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); + if (index == MI_SEGMENT_MAP_WSIZE) return; + uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + uintptr_t newmask; + do { + newmask = (mask & ~((uintptr_t)1 << bitidx)); + } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); +} + +// Determine the segment belonging to a pointer or NULL if it is not in a valid segment. +static mi_segment_t* _mi_segment_of(const void* p) { + if (p == NULL) return NULL; + mi_segment_t* segment = _mi_ptr_segment(p); + mi_assert_internal(segment != NULL); + size_t bitidx; + size_t index = mi_segment_map_index_of(segment, &bitidx); + // fast path: for any pointer to valid small/medium/large object or first MI_SEGMENT_SIZE in huge + const uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) { + return segment; // yes, allocated by us + } + if (index==MI_SEGMENT_MAP_WSIZE) return NULL; + + // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? + + // search downwards for the first segment in case it is an interior pointer + // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough + // valid huge objects + // note: we could maintain a lowest index to speed up the path for invalid pointers? + size_t lobitidx; + size_t loindex; + uintptr_t lobits = mask & (((uintptr_t)1 << bitidx) - 1); + if (lobits != 0) { + loindex = index; + lobitidx = mi_bsr(lobits); // lobits != 0 + } + else if (index == 0) { + return NULL; + } + else { + mi_assert_internal(index > 0); + uintptr_t lomask = mask; + loindex = index; + do { + loindex--; + lomask = mi_atomic_load_relaxed(&mi_segment_map[loindex]); + } while (lomask != 0 && loindex > 0); + if (lomask == 0) return NULL; + lobitidx = mi_bsr(lomask); // lomask != 0 + } + mi_assert_internal(loindex < MI_SEGMENT_MAP_WSIZE); + // take difference as the addresses could be larger than the MAX_ADDRESS space. + size_t diff = (((index - loindex) * (8*MI_INTPTR_SIZE)) + bitidx - lobitidx) * MI_SEGMENT_SIZE; + segment = (mi_segment_t*)((uint8_t*)segment - diff); + + if (segment == NULL) return NULL; + mi_assert_internal((void*)segment < p); + bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(cookie_ok); + if mi_unlikely(!cookie_ok) return NULL; + if (((uint8_t*)segment + mi_segment_size(segment)) <= (uint8_t*)p) return NULL; // outside the range + mi_assert_internal(p >= (void*)segment && (uint8_t*)p < (uint8_t*)segment + mi_segment_size(segment)); + return segment; +} + +// Is this a valid pointer in our heap? +static bool mi_is_valid_pointer(const void* p) { + return ((_mi_segment_of(p) != NULL) || (_mi_arena_contains(p))); +} + +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { + return mi_is_valid_pointer(p); +} + +/* +// Return the full segment range belonging to a pointer +static void* mi_segment_range_of(const void* p, size_t* size) { + mi_segment_t* segment = _mi_segment_of(p); + if (segment == NULL) { + if (size != NULL) *size = 0; + return NULL; + } + else { + if (size != NULL) *size = segment->segment_size; + return segment; + } + mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); + mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size); + mi_reset_delayed(tld); + mi_assert_internal(page == NULL || mi_page_not_in_queue(page, tld)); + return page; +} +*/ diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c new file mode 100644 index 00000000000000..0b4d3abc07a93c --- /dev/null +++ b/Objects/mimalloc/segment.c @@ -0,0 +1,1682 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" + +#include // memset +#include + +#define MI_PAGE_HUGE_ALIGN (256*1024) + +static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats); + + +// ------------------------------------------------------------------- +// commit mask +// ------------------------------------------------------------------- + +static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false; + } + return true; +} + +static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if ((commit->mask[i] & cm->mask[i]) != 0) return true; + } + return false; +} + +static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] = (commit->mask[i] & cm->mask[i]); + } +} + +static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] &= ~(cm->mask[i]); + } +} + +static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] |= cm->mask[i]; + } +} + +static void mi_commit_mask_create(size_t bitidx, size_t bitcount, mi_commit_mask_t* cm) { + mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); + mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); + if (bitcount == MI_COMMIT_MASK_BITS) { + mi_assert_internal(bitidx==0); + mi_commit_mask_create_full(cm); + } + else if (bitcount == 0) { + mi_commit_mask_create_empty(cm); + } + else { + mi_commit_mask_create_empty(cm); + size_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS; + size_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS; + while (bitcount > 0) { + mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT); + size_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs; + size_t count = (bitcount > avail ? avail : bitcount); + size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs); + cm->mask[i] = mask; + bitcount -= count; + ofs = 0; + i++; + } + } +} + +size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { + mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); + size_t count = 0; + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + size_t mask = cm->mask[i]; + if (~mask == 0) { + count += MI_COMMIT_MASK_FIELD_BITS; + } + else { + for (; mask != 0; mask >>= 1) { // todo: use popcount + if ((mask&1)!=0) count++; + } + } + } + // we use total since for huge segments each commit bit may represent a larger size + return ((total / MI_COMMIT_MASK_BITS) * count); +} + + +size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) { + size_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS; + size_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS; + size_t mask = 0; + // find first ones + while (i < MI_COMMIT_MASK_FIELD_COUNT) { + mask = cm->mask[i]; + mask >>= ofs; + if (mask != 0) { + while ((mask&1) == 0) { + mask >>= 1; + ofs++; + } + break; + } + i++; + ofs = 0; + } + if (i >= MI_COMMIT_MASK_FIELD_COUNT) { + // not found + *idx = MI_COMMIT_MASK_BITS; + return 0; + } + else { + // found, count ones + size_t count = 0; + *idx = (i*MI_COMMIT_MASK_FIELD_BITS) + ofs; + do { + mi_assert_internal(ofs < MI_COMMIT_MASK_FIELD_BITS && (mask&1) == 1); + do { + count++; + mask >>= 1; + } while ((mask&1) == 1); + if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) { + i++; + if (i >= MI_COMMIT_MASK_FIELD_COUNT) break; + mask = cm->mask[i]; + ofs = 0; + } + } while ((mask&1) == 1); + mi_assert_internal(count > 0); + return count; + } +} + + +/* -------------------------------------------------------------------------------- + Segment allocation + + If a thread ends, it "abandons" pages with used blocks + and there is an abandoned segment list whose segments can + be reclaimed by still running threads, much like work-stealing. +-------------------------------------------------------------------------------- */ + + +/* ----------------------------------------------------------- + Slices +----------------------------------------------------------- */ + + +static const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment) { + return &segment->slices[segment->slice_entries]; +} + +static uint8_t* mi_slice_start(const mi_slice_t* slice) { + mi_segment_t* segment = _mi_ptr_segment(slice); + mi_assert_internal(slice >= segment->slices && slice < mi_segment_slices_end(segment)); + return ((uint8_t*)segment + ((slice - segment->slices)*MI_SEGMENT_SLICE_SIZE)); +} + + +/* ----------------------------------------------------------- + Bins +----------------------------------------------------------- */ +// Use bit scan forward to quickly find the first zero bit if it is available + +static inline size_t mi_slice_bin8(size_t slice_count) { + if (slice_count<=1) return slice_count; + mi_assert_internal(slice_count <= MI_SLICES_PER_SEGMENT); + slice_count--; + size_t s = mi_bsr(slice_count); // slice_count > 1 + if (s <= 2) return slice_count + 1; + size_t bin = ((s << 2) | ((slice_count >> (s - 2))&0x03)) - 4; + return bin; +} + +static inline size_t mi_slice_bin(size_t slice_count) { + mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_SEGMENT_SIZE); + mi_assert_internal(mi_slice_bin8(MI_SLICES_PER_SEGMENT) <= MI_SEGMENT_BIN_MAX); + size_t bin = mi_slice_bin8(slice_count); + mi_assert_internal(bin <= MI_SEGMENT_BIN_MAX); + return bin; +} + +static inline size_t mi_slice_index(const mi_slice_t* slice) { + mi_segment_t* segment = _mi_ptr_segment(slice); + ptrdiff_t index = slice - segment->slices; + mi_assert_internal(index >= 0 && index < (ptrdiff_t)segment->slice_entries); + return index; +} + + +/* ----------------------------------------------------------- + Slice span queues +----------------------------------------------------------- */ + +static void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice) { + // todo: or push to the end? + mi_assert_internal(slice->prev == NULL && slice->next==NULL); + slice->prev = NULL; // paranoia + slice->next = sq->first; + sq->first = slice; + if (slice->next != NULL) slice->next->prev = slice; + else sq->last = slice; + slice->xblock_size = 0; // free +} + +static mi_span_queue_t* mi_span_queue_for(size_t slice_count, mi_segments_tld_t* tld) { + size_t bin = mi_slice_bin(slice_count); + mi_span_queue_t* sq = &tld->spans[bin]; + mi_assert_internal(sq->slice_count >= slice_count); + return sq; +} + +static void mi_span_queue_delete(mi_span_queue_t* sq, mi_slice_t* slice) { + mi_assert_internal(slice->xblock_size==0 && slice->slice_count>0 && slice->slice_offset==0); + // should work too if the queue does not contain slice (which can happen during reclaim) + if (slice->prev != NULL) slice->prev->next = slice->next; + if (slice == sq->first) sq->first = slice->next; + if (slice->next != NULL) slice->next->prev = slice->prev; + if (slice == sq->last) sq->last = slice->prev; + slice->prev = NULL; + slice->next = NULL; + slice->xblock_size = 1; // no more free +} + + +/* ----------------------------------------------------------- + Invariant checking +----------------------------------------------------------- */ + +static bool mi_slice_is_used(const mi_slice_t* slice) { + return (slice->xblock_size > 0); +} + + +#if (MI_DEBUG>=3) +static bool mi_span_queue_contains(mi_span_queue_t* sq, mi_slice_t* slice) { + for (mi_slice_t* s = sq->first; s != NULL; s = s->next) { + if (s==slice) return true; + } + return false; +} + +static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_assert_internal(segment != NULL); + mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(segment->abandoned <= segment->used); + mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id()); + mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); // can only decommit committed blocks + //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0); + mi_slice_t* slice = &segment->slices[0]; + const mi_slice_t* end = mi_segment_slices_end(segment); + size_t used_count = 0; + mi_span_queue_t* sq; + while(slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + size_t index = mi_slice_index(slice); + size_t maxindex = (index + slice->slice_count >= segment->slice_entries ? segment->slice_entries : index + slice->slice_count) - 1; + if (mi_slice_is_used(slice)) { // a page in use, we need at least MAX_SLICE_OFFSET valid back offsets + used_count++; + for (size_t i = 0; i <= MI_MAX_SLICE_OFFSET && index + i <= maxindex; i++) { + mi_assert_internal(segment->slices[index + i].slice_offset == i*sizeof(mi_slice_t)); + mi_assert_internal(i==0 || segment->slices[index + i].slice_count == 0); + mi_assert_internal(i==0 || segment->slices[index + i].xblock_size == 1); + } + // and the last entry as well (for coalescing) + const mi_slice_t* last = slice + slice->slice_count - 1; + if (last > slice && last < mi_segment_slices_end(segment)) { + mi_assert_internal(last->slice_offset == (slice->slice_count-1)*sizeof(mi_slice_t)); + mi_assert_internal(last->slice_count == 0); + mi_assert_internal(last->xblock_size == 1); + } + } + else { // free range of slices; only last slice needs a valid back offset + mi_slice_t* last = &segment->slices[maxindex]; + if (segment->kind != MI_SEGMENT_HUGE || slice->slice_count <= (segment->slice_entries - segment->segment_info_slices)) { + mi_assert_internal((uint8_t*)slice == (uint8_t*)last - last->slice_offset); + } + mi_assert_internal(slice == last || last->slice_count == 0 ); + mi_assert_internal(last->xblock_size == 0 || (segment->kind==MI_SEGMENT_HUGE && last->xblock_size==1)); + if (segment->kind != MI_SEGMENT_HUGE && segment->thread_id != 0) { // segment is not huge or abandoned + sq = mi_span_queue_for(slice->slice_count,tld); + mi_assert_internal(mi_span_queue_contains(sq,slice)); + } + } + slice = &segment->slices[maxindex+1]; + } + mi_assert_internal(slice == end); + mi_assert_internal(used_count == segment->used + 1); + return true; +} +#endif + +/* ----------------------------------------------------------- + Segment size calculations +----------------------------------------------------------- */ + +static size_t mi_segment_info_size(mi_segment_t* segment) { + return segment->segment_info_slices * MI_SEGMENT_SLICE_SIZE; +} + +static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t xblock_size, size_t* page_size) +{ + ptrdiff_t idx = slice - segment->slices; + size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE; + // make the start not OS page aligned for smaller blocks to avoid page/cache effects + // note: the offset must always be an xblock_size multiple since we assume small allocations + // are aligned (see `mi_heap_malloc_aligned`). + size_t start_offset = 0; + if (xblock_size >= MI_INTPTR_SIZE) { + if (xblock_size <= 64) { start_offset = 3*xblock_size; } + else if (xblock_size <= 512) { start_offset = xblock_size; } + } + if (page_size != NULL) { *page_size = psize - start_offset; } + return (uint8_t*)segment + ((idx*MI_SEGMENT_SLICE_SIZE) + start_offset); +} + +// Start of the page available memory; can be used on uninitialized pages +uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) +{ + const mi_slice_t* slice = mi_page_to_slice((mi_page_t*)page); + uint8_t* p = _mi_segment_page_start_from_slice(segment, slice, page->xblock_size, page_size); + mi_assert_internal(page->xblock_size > 0 || _mi_ptr_page(p) == page); + mi_assert_internal(_mi_ptr_segment(p) == segment); + return p; +} + + +static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) { + size_t page_size = _mi_os_page_size(); + size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size); + size_t guardsize = 0; + + if (MI_SECURE>0) { + // in secure mode, we set up a protected page in between the segment info + // and the page data (and one at the end of the segment) + guardsize = page_size; + if (required > 0) { + required = _mi_align_up(required, MI_SEGMENT_SLICE_SIZE) + page_size; + } + } + + if (pre_size != NULL) *pre_size = isize; + isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE); + if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE; + size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) ); + mi_assert_internal(segment_size % MI_SEGMENT_SLICE_SIZE == 0); + return (segment_size / MI_SEGMENT_SLICE_SIZE); +} + + +/* ---------------------------------------------------------------------------- +Segment caches +We keep a small segment cache per thread to increase local +reuse and avoid setting/clearing guard pages in secure mode. +------------------------------------------------------------------------------- */ + +static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) { + if (segment_size>=0) _mi_stat_increase(&tld->stats->segments,1); + else _mi_stat_decrease(&tld->stats->segments,1); + tld->count += (segment_size >= 0 ? 1 : -1); + if (tld->count > tld->peak_count) tld->peak_count = tld->count; + tld->current_size += segment_size; + if (tld->current_size > tld->peak_size) tld->peak_size = tld->current_size; +} + +static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { + segment->thread_id = 0; + _mi_segment_map_freed_at(segment); + mi_segments_track_size(-((long)mi_segment_size(segment)),tld); + if (MI_SECURE>0) { + // _mi_os_unprotect(segment, mi_segment_size(segment)); // ensure no more guard pages are set + // unprotect the guard pages; we cannot just unprotect the whole segment size as part may be decommitted + size_t os_pagesize = _mi_os_page_size(); + _mi_os_unprotect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); + uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; + _mi_os_unprotect(end, os_pagesize); + } + + // purge delayed decommits now? (no, leave it to the arena) + // mi_segment_try_purge(segment,true,tld->stats); + + const size_t size = mi_segment_size(segment); + const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); + + _mi_abandoned_await_readers(tld->abandoned); // wait until safe to free + _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid, tld->stats); +} + +// called by threads that are terminating +void _mi_segment_thread_collect(mi_segments_tld_t* tld) { + MI_UNUSED(tld); + // nothing to do +} + + +/* ----------------------------------------------------------- + Commit/Decommit ranges +----------------------------------------------------------- */ + +static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) { + mi_assert_internal(_mi_ptr_segment(p + 1) == segment); + mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); + mi_commit_mask_create_empty(cm); + if (size == 0 || size > MI_SEGMENT_SIZE || segment->kind == MI_SEGMENT_HUGE) return; + const size_t segstart = mi_segment_info_size(segment); + const size_t segsize = mi_segment_size(segment); + if (p >= (uint8_t*)segment + segsize) return; + + size_t pstart = (p - (uint8_t*)segment); + mi_assert_internal(pstart + size <= segsize); + + size_t start; + size_t end; + if (conservative) { + // decommit conservative + start = _mi_align_up(pstart, MI_COMMIT_SIZE); + end = _mi_align_down(pstart + size, MI_COMMIT_SIZE); + mi_assert_internal(start >= segstart); + mi_assert_internal(end <= segsize); + } + else { + // commit liberal + start = _mi_align_down(pstart, MI_MINIMAL_COMMIT_SIZE); + end = _mi_align_up(pstart + size, MI_MINIMAL_COMMIT_SIZE); + } + if (pstart >= segstart && start < segstart) { // note: the mask is also calculated for an initial commit of the info area + start = segstart; + } + if (end > segsize) { + end = segsize; + } + + mi_assert_internal(start <= pstart && (pstart + size) <= end); + mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0); + *start_p = (uint8_t*)segment + start; + *full_size = (end > start ? end - start : 0); + if (*full_size == 0) return; + + size_t bitidx = start / MI_COMMIT_SIZE; + mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); + + size_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0 + if (bitidx + bitcount > MI_COMMIT_MASK_BITS) { + _mi_warning_message("commit mask overflow: idx=%zu count=%zu start=%zx end=%zx p=0x%p size=%zu fullsize=%zu\n", bitidx, bitcount, start, end, p, size, *full_size); + } + mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); + mi_commit_mask_create(bitidx, bitcount, cm); +} + +static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { + mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); + + // commit liberal + uint8_t* start = NULL; + size_t full_size = 0; + mi_commit_mask_t mask; + mi_segment_commit_mask(segment, false /* conservative? */, p, size, &start, &full_size, &mask); + if (mi_commit_mask_is_empty(&mask) || full_size == 0) return true; + + if (!mi_commit_mask_all_set(&segment->commit_mask, &mask)) { + // committing + bool is_zero = false; + mi_commit_mask_t cmask; + mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); + _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap + if (!_mi_os_commit(start, full_size, &is_zero, stats)) return false; + mi_commit_mask_set(&segment->commit_mask, &mask); + } + + // increase purge expiration when using part of delayed purges -- we assume more allocations are coming soon. + if (mi_commit_mask_any_set(&segment->purge_mask, &mask)) { + segment->purge_expire = _mi_clock_now() + mi_option_get(mi_option_purge_delay); + } + + // always clear any delayed purges in our range (as they are either committed now) + mi_commit_mask_clear(&segment->purge_mask, &mask); + return true; +} + +static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { + mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); + // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow + if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed + mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); + return mi_segment_commit(segment, p, size, stats); +} + +static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { + mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); + if (!segment->allow_purge) return true; + + // purge conservative + uint8_t* start = NULL; + size_t full_size = 0; + mi_commit_mask_t mask; + mi_segment_commit_mask(segment, true /* conservative? */, p, size, &start, &full_size, &mask); + if (mi_commit_mask_is_empty(&mask) || full_size==0) return true; + + if (mi_commit_mask_any_set(&segment->commit_mask, &mask)) { + // purging + mi_assert_internal((void*)start != (void*)segment); + mi_assert_internal(segment->allow_decommit); + const bool decommitted = _mi_os_purge(start, full_size, stats); // reset or decommit + if (decommitted) { + mi_commit_mask_t cmask; + mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); + _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for double counting + mi_commit_mask_clear(&segment->commit_mask, &mask); + } + } + + // always clear any scheduled purges in our range + mi_commit_mask_clear(&segment->purge_mask, &mask); + return true; +} + +static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { + if (!segment->allow_purge) return; + + if (mi_option_get(mi_option_purge_delay) == 0) { + mi_segment_purge(segment, p, size, stats); + } + else { + // register for future purge in the purge mask + uint8_t* start = NULL; + size_t full_size = 0; + mi_commit_mask_t mask; + mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size, &mask); + if (mi_commit_mask_is_empty(&mask) || full_size==0) return; + + // update delayed commit + mi_assert_internal(segment->purge_expire > 0 || mi_commit_mask_is_empty(&segment->purge_mask)); + mi_commit_mask_t cmask; + mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); // only purge what is committed; span_free may try to decommit more + mi_commit_mask_set(&segment->purge_mask, &cmask); + mi_msecs_t now = _mi_clock_now(); + if (segment->purge_expire == 0) { + // no previous purgess, initialize now + segment->purge_expire = now + mi_option_get(mi_option_purge_delay); + } + else if (segment->purge_expire <= now) { + // previous purge mask already expired + if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) { + mi_segment_try_purge(segment, true, stats); + } + else { + segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's + } + } + else { + // previous purge mask is not yet expired, increase the expiration by a bit. + segment->purge_expire += mi_option_get(mi_option_purge_extend_delay); + } + } +} + +static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { + if (!segment->allow_purge || mi_commit_mask_is_empty(&segment->purge_mask)) return; + mi_msecs_t now = _mi_clock_now(); + if (!force && now < segment->purge_expire) return; + + mi_commit_mask_t mask = segment->purge_mask; + segment->purge_expire = 0; + mi_commit_mask_create_empty(&segment->purge_mask); + + size_t idx; + size_t count; + mi_commit_mask_foreach(&mask, idx, count) { + // if found, decommit that sequence + if (count > 0) { + uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE); + size_t size = count * MI_COMMIT_SIZE; + mi_segment_purge(segment, p, size, stats); + } + } + mi_commit_mask_foreach_end() + mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); +} + + +/* ----------------------------------------------------------- + Span free +----------------------------------------------------------- */ + +static bool mi_segment_is_abandoned(mi_segment_t* segment) { + return (segment->thread_id == 0); +} + +// note: can be called on abandoned segments +static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size_t slice_count, bool allow_purge, mi_segments_tld_t* tld) { + mi_assert_internal(slice_index < segment->slice_entries); + mi_span_queue_t* sq = (segment->kind == MI_SEGMENT_HUGE || mi_segment_is_abandoned(segment) + ? NULL : mi_span_queue_for(slice_count,tld)); + if (slice_count==0) slice_count = 1; + mi_assert_internal(slice_index + slice_count - 1 < segment->slice_entries); + + // set first and last slice (the intermediates can be undetermined) + mi_slice_t* slice = &segment->slices[slice_index]; + slice->slice_count = (uint32_t)slice_count; + mi_assert_internal(slice->slice_count == slice_count); // no overflow? + slice->slice_offset = 0; + if (slice_count > 1) { + mi_slice_t* last = &segment->slices[slice_index + slice_count - 1]; + last->slice_count = 0; + last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1)); + last->xblock_size = 0; + } + + // perhaps decommit + if (allow_purge) { + mi_segment_schedule_purge(segment, mi_slice_start(slice), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats); + } + + // and push it on the free page queue (if it was not a huge page) + if (sq != NULL) mi_span_queue_push( sq, slice ); + else slice->xblock_size = 0; // mark huge page as free anyways +} + +/* +// called from reclaim to add existing free spans +static void mi_segment_span_add_free(mi_slice_t* slice, mi_segments_tld_t* tld) { + mi_segment_t* segment = _mi_ptr_segment(slice); + mi_assert_internal(slice->xblock_size==0 && slice->slice_count>0 && slice->slice_offset==0); + size_t slice_index = mi_slice_index(slice); + mi_segment_span_free(segment,slice_index,slice->slice_count,tld); +} +*/ + +static void mi_segment_span_remove_from_queue(mi_slice_t* slice, mi_segments_tld_t* tld) { + mi_assert_internal(slice->slice_count > 0 && slice->slice_offset==0 && slice->xblock_size==0); + mi_assert_internal(_mi_ptr_segment(slice)->kind != MI_SEGMENT_HUGE); + mi_span_queue_t* sq = mi_span_queue_for(slice->slice_count, tld); + mi_span_queue_delete(sq, slice); +} + +// note: can be called on abandoned segments +static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_tld_t* tld) { + mi_assert_internal(slice != NULL && slice->slice_count > 0 && slice->slice_offset == 0); + mi_segment_t* segment = _mi_ptr_segment(slice); + bool is_abandoned = mi_segment_is_abandoned(segment); + + // for huge pages, just mark as free but don't add to the queues + if (segment->kind == MI_SEGMENT_HUGE) { + // issue #691: segment->used can be 0 if the huge page block was freed while abandoned (reclaim will get here in that case) + mi_assert_internal((segment->used==0 && slice->xblock_size==0) || segment->used == 1); // decreased right after this call in `mi_segment_page_clear` + slice->xblock_size = 0; // mark as free anyways + // we should mark the last slice `xblock_size=0` now to maintain invariants but we skip it to + // avoid a possible cache miss (and the segment is about to be freed) + return slice; + } + + // otherwise coalesce the span and add to the free span queues + size_t slice_count = slice->slice_count; + mi_slice_t* next = slice + slice->slice_count; + mi_assert_internal(next <= mi_segment_slices_end(segment)); + if (next < mi_segment_slices_end(segment) && next->xblock_size==0) { + // free next block -- remove it from free and merge + mi_assert_internal(next->slice_count > 0 && next->slice_offset==0); + slice_count += next->slice_count; // extend + if (!is_abandoned) { mi_segment_span_remove_from_queue(next, tld); } + } + if (slice > segment->slices) { + mi_slice_t* prev = mi_slice_first(slice - 1); + mi_assert_internal(prev >= segment->slices); + if (prev->xblock_size==0) { + // free previous slice -- remove it from free and merge + mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0); + slice_count += prev->slice_count; + if (!is_abandoned) { mi_segment_span_remove_from_queue(prev, tld); } + slice = prev; + } + } + + // and add the new free page + mi_segment_span_free(segment, mi_slice_index(slice), slice_count, true, tld); + return slice; +} + + + +/* ----------------------------------------------------------- + Page allocation +----------------------------------------------------------- */ + +// Note: may still return NULL if committing the memory failed +static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count, mi_segments_tld_t* tld) { + mi_assert_internal(slice_index < segment->slice_entries); + mi_slice_t* const slice = &segment->slices[slice_index]; + mi_assert_internal(slice->xblock_size==0 || slice->xblock_size==1); + + // commit before changing the slice data + if (!mi_segment_ensure_committed(segment, _mi_segment_page_start_from_slice(segment, slice, 0, NULL), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats)) { + return NULL; // commit failed! + } + + // convert the slices to a page + slice->slice_offset = 0; + slice->slice_count = (uint32_t)slice_count; + mi_assert_internal(slice->slice_count == slice_count); + const size_t bsize = slice_count * MI_SEGMENT_SLICE_SIZE; + slice->xblock_size = (uint32_t)(bsize >= MI_HUGE_BLOCK_SIZE ? MI_HUGE_BLOCK_SIZE : bsize); + mi_page_t* page = mi_slice_to_page(slice); + mi_assert_internal(mi_page_block_size(page) == bsize); + + // set slice back pointers for the first MI_MAX_SLICE_OFFSET entries + size_t extra = slice_count-1; + if (extra > MI_MAX_SLICE_OFFSET) extra = MI_MAX_SLICE_OFFSET; + if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than avaiable entries in the segment->slices + + mi_slice_t* slice_next = slice + 1; + for (size_t i = 1; i <= extra; i++, slice_next++) { + slice_next->slice_offset = (uint32_t)(sizeof(mi_slice_t)*i); + slice_next->slice_count = 0; + slice_next->xblock_size = 1; + } + + // and also for the last one (if not set already) (the last one is needed for coalescing and for large alignments) + // note: the cast is needed for ubsan since the index can be larger than MI_SLICES_PER_SEGMENT for huge allocations (see #543) + mi_slice_t* last = slice + slice_count - 1; + mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment); + if (last > end) last = end; + if (last > slice) { + last->slice_offset = (uint32_t)(sizeof(mi_slice_t) * (last - slice)); + last->slice_count = 0; + last->xblock_size = 1; + } + + // and initialize the page + page->is_committed = true; + segment->used++; + return page; +} + +static void mi_segment_slice_split(mi_segment_t* segment, mi_slice_t* slice, size_t slice_count, mi_segments_tld_t* tld) { + mi_assert_internal(_mi_ptr_segment(slice) == segment); + mi_assert_internal(slice->slice_count >= slice_count); + mi_assert_internal(slice->xblock_size > 0); // no more in free queue + if (slice->slice_count <= slice_count) return; + mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); + size_t next_index = mi_slice_index(slice) + slice_count; + size_t next_count = slice->slice_count - slice_count; + mi_segment_span_free(segment, next_index, next_count, false /* don't purge left-over part */, tld); + slice->slice_count = (uint32_t)slice_count; +} + +static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) { + mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX); + // search from best fit up + mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld); + if (slice_count == 0) slice_count = 1; + while (sq <= &tld->spans[MI_SEGMENT_BIN_MAX]) { + for (mi_slice_t* slice = sq->first; slice != NULL; slice = slice->next) { + if (slice->slice_count >= slice_count) { + // found one + mi_segment_t* segment = _mi_ptr_segment(slice); + if (_mi_arena_memid_is_suitable(segment->memid, req_arena_id)) { + // found a suitable page span + mi_span_queue_delete(sq, slice); + + if (slice->slice_count > slice_count) { + mi_segment_slice_split(segment, slice, slice_count, tld); + } + mi_assert_internal(slice != NULL && slice->slice_count == slice_count && slice->xblock_size > 0); + mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count, tld); + if (page == NULL) { + // commit failed; return NULL but first restore the slice + mi_segment_span_free_coalesce(slice, tld); + return NULL; + } + return page; + } + } + } + sq++; + } + // could not find a page.. + return NULL; +} + + +/* ----------------------------------------------------------- + Segment allocation +----------------------------------------------------------- */ + +static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id, + size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices, + bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) + +{ + mi_memid_t memid; + bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy + size_t align_offset = 0; + size_t alignment = MI_SEGMENT_ALIGN; + + if (page_alignment > 0) { + // mi_assert_internal(huge_page != NULL); + mi_assert_internal(page_alignment >= MI_SEGMENT_ALIGN); + alignment = page_alignment; + const size_t info_size = (*pinfo_slices) * MI_SEGMENT_SLICE_SIZE; + align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN ); + const size_t extra = align_offset - info_size; + // recalculate due to potential guard pages + *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); + + // mi_page_t.slice_count type is uint32_t + if (*psegment_slices > (size_t)UINT32_MAX) return NULL; + } + + const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; + mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, os_tld); + if (segment == NULL) { + return NULL; // failed to allocate + } + + // ensure metadata part of the segment is committed + mi_commit_mask_t commit_mask; + if (memid.initially_committed) { + mi_commit_mask_create_full(&commit_mask); + } + else { + // at least commit the info slices + const size_t commit_needed = _mi_divide_up((*pinfo_slices)*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE); + mi_assert_internal(commit_needed>0); + mi_commit_mask_create(0, commit_needed, &commit_mask); + mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= (*pinfo_slices)*MI_SEGMENT_SLICE_SIZE); + if (!_mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, NULL, tld->stats)) { + _mi_arena_free(segment,segment_size,0,memid,tld->stats); + return NULL; + } + } + mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); + + segment->memid = memid; + segment->allow_decommit = !memid.is_pinned; + segment->allow_purge = segment->allow_decommit && (mi_option_get(mi_option_purge_delay) >= 0); + segment->segment_size = segment_size; + segment->commit_mask = commit_mask; + segment->purge_expire = 0; + mi_commit_mask_create_empty(&segment->purge_mask); + mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan + + mi_segments_track_size((long)(segment_size), tld); + _mi_segment_map_allocated_at(segment); + return segment; +} + + +// Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . +static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) +{ + mi_assert_internal((required==0 && huge_page==NULL) || (required>0 && huge_page != NULL)); + + // calculate needed sizes first + size_t info_slices; + size_t pre_size; + size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices); + + // mi_page_t.slice_count type is uint32_t + if (segment_slices > (size_t)UINT32_MAX) return NULL; + + // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) + const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems + _mi_current_thread_count() > 1 && // do not delay for the first N threads + tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); + const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit); + bool commit = eager || (required > 0); + + // Allocate the segment from the OS + mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id, + &segment_slices, &pre_size, &info_slices, commit, tld, os_tld); + if (segment == NULL) return NULL; + + // zero the segment info? -- not always needed as it may be zero initialized from the OS + if (!segment->memid.initially_zero) { + ptrdiff_t ofs = offsetof(mi_segment_t, next); + size_t prefix = offsetof(mi_segment_t, slices) - ofs; + size_t zsize = prefix + (sizeof(mi_slice_t) * (segment_slices + 1)); // one more + _mi_memzero((uint8_t*)segment + ofs, zsize); + } + + // initialize the rest of the segment info + const size_t slice_entries = (segment_slices > MI_SLICES_PER_SEGMENT ? MI_SLICES_PER_SEGMENT : segment_slices); + segment->segment_slices = segment_slices; + segment->segment_info_slices = info_slices; + segment->thread_id = _mi_thread_id(); + segment->cookie = _mi_ptr_cookie(segment); + segment->slice_entries = slice_entries; + segment->kind = (required == 0 ? MI_SEGMENT_NORMAL : MI_SEGMENT_HUGE); + + // _mi_memzero(segment->slices, sizeof(mi_slice_t)*(info_slices+1)); + _mi_stat_increase(&tld->stats->page_committed, mi_segment_info_size(segment)); + + // set up guard pages + size_t guard_slices = 0; + if (MI_SECURE>0) { + // in secure mode, we set up a protected page in between the segment info + // and the page data, and at the end of the segment. + size_t os_pagesize = _mi_os_page_size(); + mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size); + _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); + uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; + mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats); + _mi_os_protect(end, os_pagesize); + if (slice_entries == segment_slices) segment->slice_entries--; // don't use the last slice :-( + guard_slices = 1; + } + + // reserve first slices for segment info + mi_page_t* page0 = mi_segment_span_allocate(segment, 0, info_slices, tld); + mi_assert_internal(page0!=NULL); if (page0==NULL) return NULL; // cannot fail as we always commit in advance + mi_assert_internal(segment->used == 1); + segment->used = 0; // don't count our internal slices towards usage + + // initialize initial free pages + if (segment->kind == MI_SEGMENT_NORMAL) { // not a huge page + mi_assert_internal(huge_page==NULL); + mi_segment_span_free(segment, info_slices, segment->slice_entries - info_slices, false /* don't purge */, tld); + } + else { + mi_assert_internal(huge_page!=NULL); + mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); + mi_assert_internal(mi_commit_mask_is_full(&segment->commit_mask)); + *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices, tld); + mi_assert_internal(*huge_page != NULL); // cannot fail as we commit in advance + } + + mi_assert_expensive(mi_segment_is_valid(segment,tld)); + return segment; +} + + +static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { + MI_UNUSED(force); + mi_assert_internal(segment != NULL); + mi_assert_internal(segment->next == NULL); + mi_assert_internal(segment->used == 0); + + // Remove the free pages + mi_slice_t* slice = &segment->slices[0]; + const mi_slice_t* end = mi_segment_slices_end(segment); + #if MI_DEBUG>1 + size_t page_count = 0; + #endif + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + mi_assert_internal(mi_slice_index(slice)==0 || slice->xblock_size == 0); // no more used pages .. + if (slice->xblock_size == 0 && segment->kind != MI_SEGMENT_HUGE) { + mi_segment_span_remove_from_queue(slice, tld); + } + #if MI_DEBUG>1 + page_count++; + #endif + slice = slice + slice->slice_count; + } + mi_assert_internal(page_count == 2); // first page is allocated by the segment itself + + // stats + _mi_stat_decrease(&tld->stats->page_committed, mi_segment_info_size(segment)); + + // return it to the OS + mi_segment_os_free(segment, tld); +} + + +/* ----------------------------------------------------------- + Page Free +----------------------------------------------------------- */ + +static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld); + +// note: can be called on abandoned pages +static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) { + mi_assert_internal(page->xblock_size > 0); + mi_assert_internal(mi_page_all_free(page)); + mi_segment_t* segment = _mi_ptr_segment(page); + mi_assert_internal(segment->used > 0); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif + + size_t inuse = page->capacity * mi_page_block_size(page); + _mi_stat_decrease(&tld->stats->page_committed, inuse); + _mi_stat_decrease(&tld->stats->pages, 1); + + // reset the page memory to reduce memory pressure? + if (segment->allow_decommit && mi_option_is_enabled(mi_option_deprecated_page_reset)) { + size_t psize; + uint8_t* start = _mi_page_start(segment, page, &psize); + _mi_os_reset(start, psize, tld->stats); + } + + // zero the page data, but not the segment fields + page->is_zero_init = false; + ptrdiff_t ofs = offsetof(mi_page_t, capacity); + _mi_memzero((uint8_t*)page + ofs, sizeof(*page) - ofs); + page->xblock_size = 1; + + // and free it + mi_slice_t* slice = mi_segment_span_free_coalesce(mi_page_to_slice(page), tld); + segment->used--; + // cannot assert segment valid as it is called during reclaim + // mi_assert_expensive(mi_segment_is_valid(segment, tld)); + return slice; +} + +void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) +{ + mi_assert(page != NULL); + + mi_segment_t* segment = _mi_page_segment(page); + mi_assert_expensive(mi_segment_is_valid(segment,tld)); + + // mark it as free now + mi_segment_page_clear(page, tld); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); + + if (segment->used == 0) { + // no more used pages; remove from the free list and free the segment + mi_segment_free(segment, force, tld); + } + else if (segment->used == segment->abandoned) { + // only abandoned pages; remove from free list and abandon + mi_segment_abandon(segment,tld); + } +} + + +/* ----------------------------------------------------------- +Abandonment + +When threads terminate, they can leave segments with +live blocks (reachable through other threads). Such segments +are "abandoned" and will be reclaimed by other threads to +reuse their pages and/or free them eventually + +We maintain a global list of abandoned segments that are +reclaimed on demand. Since this is shared among threads +the implementation needs to avoid the A-B-A problem on +popping abandoned segments: +We use tagged pointers to avoid accidentally identifying +reused segments, much like stamped references in Java. +Secondly, we maintain a reader counter to avoid resetting +or decommitting segments that have a pending read operation. + +Note: the current implementation is one possible design; +another way might be to keep track of abandoned segments +in the arenas/segment_cache's. This would have the advantage of keeping +all concurrent code in one place and not needing to deal +with ABA issues. The drawback is that it is unclear how to +scan abandoned segments efficiently in that case as they +would be spread among all other segments in the arenas. +----------------------------------------------------------- */ + +// Use the bottom 20-bits (on 64-bit) of the aligned segment pointers +// to put in a tag that increments on update to avoid the A-B-A problem. +#define MI_TAGGED_MASK MI_SEGMENT_MASK + +static mi_segment_t* mi_tagged_segment_ptr(mi_tagged_segment_t ts) { + return (mi_segment_t*)(ts & ~MI_TAGGED_MASK); +} + +static mi_tagged_segment_t mi_tagged_segment(mi_segment_t* segment, mi_tagged_segment_t ts) { + mi_assert_internal(((uintptr_t)segment & MI_TAGGED_MASK) == 0); + uintptr_t tag = ((ts & MI_TAGGED_MASK) + 1) & MI_TAGGED_MASK; + return ((uintptr_t)segment | tag); +} + +mi_abandoned_pool_t _mi_abandoned_default; + +// Push on the visited list +static void mi_abandoned_visited_push(mi_abandoned_pool_t *pool, mi_segment_t* segment) { + mi_assert_internal(segment->thread_id == 0); + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t,&segment->abandoned_next) == NULL); + mi_assert_internal(segment->next == NULL); + mi_assert_internal(segment->used > 0); + mi_segment_t* anext = mi_atomic_load_ptr_relaxed(mi_segment_t, &pool->abandoned_visited); + do { + mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, anext); + } while (!mi_atomic_cas_ptr_weak_release(mi_segment_t, &pool->abandoned_visited, &anext, segment)); + mi_atomic_increment_relaxed(&pool->abandoned_visited_count); +} + +// Move the visited list to the abandoned list. +static bool mi_abandoned_visited_revisit(mi_abandoned_pool_t *pool) +{ + // quick check if the visited list is empty + if (mi_atomic_load_ptr_relaxed(mi_segment_t, &pool->abandoned_visited) == NULL) return false; + + // grab the whole visited list + mi_segment_t* first = mi_atomic_exchange_ptr_acq_rel(mi_segment_t, &pool->abandoned_visited, NULL); + if (first == NULL) return false; + + // first try to swap directly if the abandoned list happens to be NULL + mi_tagged_segment_t afirst; + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); + if (mi_tagged_segment_ptr(ts)==NULL) { + size_t count = mi_atomic_load_relaxed(&pool->abandoned_visited_count); + afirst = mi_tagged_segment(first, ts); + if (mi_atomic_cas_strong_acq_rel(&pool->abandoned, &ts, afirst)) { + mi_atomic_add_relaxed(&pool->abandoned_count, count); + mi_atomic_sub_relaxed(&pool->abandoned_visited_count, count); + return true; + } + } + + // find the last element of the visited list: O(n) + mi_segment_t* last = first; + mi_segment_t* next; + while ((next = mi_atomic_load_ptr_relaxed(mi_segment_t, &last->abandoned_next)) != NULL) { + last = next; + } + + // and atomically prepend to the abandoned list + // (no need to increase the readers as we don't access the abandoned segments) + mi_tagged_segment_t anext = mi_atomic_load_relaxed(&pool->abandoned); + size_t count; + do { + count = mi_atomic_load_relaxed(&pool->abandoned_visited_count); + mi_atomic_store_ptr_release(mi_segment_t, &last->abandoned_next, mi_tagged_segment_ptr(anext)); + afirst = mi_tagged_segment(first, anext); + } while (!mi_atomic_cas_weak_release(&pool->abandoned, &anext, afirst)); + mi_atomic_add_relaxed(&pool->abandoned_count, count); + mi_atomic_sub_relaxed(&pool->abandoned_visited_count, count); + return true; +} + +// Push on the abandoned list. +static void mi_abandoned_push(mi_abandoned_pool_t* pool, mi_segment_t* segment) { + mi_assert_internal(segment->thread_id == 0); + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next) == NULL); + mi_assert_internal(segment->next == NULL); + mi_assert_internal(segment->used > 0); + mi_tagged_segment_t next; + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); + do { + mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, mi_tagged_segment_ptr(ts)); + next = mi_tagged_segment(segment, ts); + } while (!mi_atomic_cas_weak_release(&pool->abandoned, &ts, next)); + mi_atomic_increment_relaxed(&pool->abandoned_count); +} + +// Wait until there are no more pending reads on segments that used to be in the abandoned list +// called for example from `arena.c` before decommitting +void _mi_abandoned_await_readers(mi_abandoned_pool_t* pool) { + size_t n; + do { + n = mi_atomic_load_acquire(&pool->abandoned_readers); + if (n != 0) mi_atomic_yield(); + } while (n != 0); +} + +// Pop from the abandoned list +static mi_segment_t* mi_abandoned_pop(mi_abandoned_pool_t* pool) { + mi_segment_t* segment; + // Check efficiently if it is empty (or if the visited list needs to be moved) + mi_tagged_segment_t ts = mi_atomic_load_relaxed(&pool->abandoned); + segment = mi_tagged_segment_ptr(ts); + if mi_likely(segment == NULL) { + if mi_likely(!mi_abandoned_visited_revisit(pool)) { // try to swap in the visited list on NULL + return NULL; + } + } + + // Do a pop. We use a reader count to prevent + // a segment to be decommitted while a read is still pending, + // and a tagged pointer to prevent A-B-A link corruption. + // (this is called from `region.c:_mi_mem_free` for example) + mi_atomic_increment_relaxed(&pool->abandoned_readers); // ensure no segment gets decommitted + mi_tagged_segment_t next = 0; + ts = mi_atomic_load_acquire(&pool->abandoned); + do { + segment = mi_tagged_segment_ptr(ts); + if (segment != NULL) { + mi_segment_t* anext = mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next); + next = mi_tagged_segment(anext, ts); // note: reads the segment's `abandoned_next` field so should not be decommitted + } + } while (segment != NULL && !mi_atomic_cas_weak_acq_rel(&pool->abandoned, &ts, next)); + mi_atomic_decrement_relaxed(&pool->abandoned_readers); // release reader lock + if (segment != NULL) { + mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); + mi_atomic_decrement_relaxed(&pool->abandoned_count); + } + return segment; +} + +/* ----------------------------------------------------------- + Abandon segment/page +----------------------------------------------------------- */ + +static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { + mi_assert_internal(segment->used == segment->abandoned); + mi_assert_internal(segment->used > 0); + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next) == NULL); + mi_assert_internal(segment->abandoned_visits == 0); + mi_assert_expensive(mi_segment_is_valid(segment,tld)); + + // remove the free pages from the free page queues + mi_slice_t* slice = &segment->slices[0]; + const mi_slice_t* end = mi_segment_slices_end(segment); + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + if (slice->xblock_size == 0) { // a free page + mi_segment_span_remove_from_queue(slice,tld); + slice->xblock_size = 0; // but keep it free + } + slice = slice + slice->slice_count; + } + + // perform delayed decommits (forcing is much slower on mstress) + mi_segment_try_purge(segment, mi_option_is_enabled(mi_option_abandoned_page_purge) /* force? */, tld->stats); + + // all pages in the segment are abandoned; add it to the abandoned list + _mi_stat_increase(&tld->stats->segments_abandoned, 1); + mi_segments_track_size(-((long)mi_segment_size(segment)), tld); + segment->thread_id = 0; + mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); + segment->abandoned_visits = 1; // from 0 to 1 to signify it is abandoned + mi_abandoned_push(tld->abandoned, segment); +} + +void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { + mi_assert(page != NULL); + mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); + mi_assert_internal(mi_page_heap(page) == NULL); + mi_segment_t* segment = _mi_page_segment(page); + + mi_assert_expensive(mi_segment_is_valid(segment,tld)); + segment->abandoned++; + + _mi_stat_increase(&tld->stats->pages_abandoned, 1); + mi_assert_internal(segment->abandoned <= segment->used); + if (segment->used == segment->abandoned) { + // all pages are abandoned, abandon the entire segment + mi_segment_abandon(segment, tld); + } +} + +/* ----------------------------------------------------------- + Reclaim abandoned pages +----------------------------------------------------------- */ + +static mi_slice_t* mi_slices_start_iterate(mi_segment_t* segment, const mi_slice_t** end) { + mi_slice_t* slice = &segment->slices[0]; + *end = mi_segment_slices_end(segment); + mi_assert_internal(slice->slice_count>0 && slice->xblock_size>0); // segment allocated page + slice = slice + slice->slice_count; // skip the first segment allocated page + return slice; +} + +// Possibly free pages and check if free space is available +static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, size_t block_size, mi_segments_tld_t* tld) +{ + mi_assert_internal(block_size < MI_HUGE_BLOCK_SIZE); + mi_assert_internal(mi_segment_is_abandoned(segment)); + bool has_page = false; + + // for all slices + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + if (mi_slice_is_used(slice)) { // used page + // ensure used count is up to date and collect potential concurrent frees + mi_page_t* const page = mi_slice_to_page(slice); + _mi_page_free_collect(page, false); + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { + // if this page is all free now, free it without adding to any queues (yet) + mi_assert_internal(page->next == NULL && page->prev==NULL); + _mi_stat_decrease(&tld->stats->pages_abandoned, 1); +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif + segment->abandoned--; + slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! + mi_assert_internal(!mi_slice_is_used(slice)); + if (slice->slice_count >= slices_needed) { + has_page = true; + } + } + else { + if (page->xblock_size == block_size && mi_page_has_any_available(page)) { + // a page has available free blocks of the right size + has_page = true; + } + } + } + else { + // empty span + if (slice->slice_count >= slices_needed) { + has_page = true; + } + } + slice = slice + slice->slice_count; + } + return has_page; +} + +static mi_heap_t* mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { + if (heap->tag == tag) { + return heap; + } + for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) { + if (curr->tag == tag) { + return curr; + } + } + return NULL; +} + +// Reclaim an abandoned segment; returns NULL if the segment was freed +// set `right_page_reclaimed` to `true` if it reclaimed a page of the right `block_size` that was not full. +static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, size_t requested_block_size, bool* right_page_reclaimed, mi_segments_tld_t* tld) { + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_segment_t, &segment->abandoned_next) == NULL); + mi_assert_expensive(mi_segment_is_valid(segment, tld)); + if (right_page_reclaimed != NULL) { *right_page_reclaimed = false; } + + segment->thread_id = _mi_thread_id(); + segment->abandoned_visits = 0; + mi_segments_track_size((long)mi_segment_size(segment), tld); + mi_assert_internal(segment->next == NULL); + _mi_stat_decrease(&tld->stats->segments_abandoned, 1); + + // for all slices + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + if (mi_slice_is_used(slice)) { + // in use: reclaim the page in our heap + mi_page_t* page = mi_slice_to_page(slice); + mi_heap_t* target_heap = mi_heap_by_tag(heap, page->tag); + mi_assert_internal(page->is_committed); + mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE); + mi_assert_internal(mi_page_heap(page) == NULL); + mi_assert_internal(page->next == NULL && page->prev==NULL); + _mi_stat_decrease(&tld->stats->pages_abandoned, 1); + segment->abandoned--; + // set the heap again and allow delayed free again + mi_page_set_heap(page, target_heap); + _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) + _mi_page_free_collect(page, false); // ensure used count is up to date + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { + // if everything free by now, free the page +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif + slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing + } + else { + // otherwise reclaim it into the heap + _mi_page_reclaim(target_heap, page); + if (requested_block_size == page->xblock_size && mi_page_has_any_available(page) && + requested_block_size <= MI_MEDIUM_OBJ_SIZE_MAX && heap == target_heap) { + if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } + } + } + } + else { + // the span is free, add it to our page queues + slice = mi_segment_span_free_coalesce(slice, tld); // set slice again due to coalesceing + } + mi_assert_internal(slice->slice_count>0 && slice->slice_offset==0); + slice = slice + slice->slice_count; + } + + mi_assert(segment->abandoned == 0); + if (segment->used == 0) { // due to page_clear + mi_assert_internal(right_page_reclaimed == NULL || !(*right_page_reclaimed)); + mi_segment_free(segment, false, tld); + return NULL; + } + else { + return segment; + } +} + + +void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { + mi_segment_t* segment; + while ((segment = mi_abandoned_pop(tld->abandoned)) != NULL) { + mi_segment_reclaim(segment, heap, 0, NULL, tld); + } +} + +static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slices, size_t block_size, bool* reclaimed, mi_segments_tld_t* tld) +{ + *reclaimed = false; + mi_segment_t* segment; + long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 8, 1024); // limit the work to bound allocation times + while ((max_tries-- > 0) && ((segment = mi_abandoned_pop(tld->abandoned)) != NULL)) { + segment->abandoned_visits++; + // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments + // and push them into the visited list and use many tries. Perhaps we can skip non-suitable ones in a better way? + bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid); + bool has_page = mi_segment_check_free(segment,needed_slices,block_size,tld); // try to free up pages (due to concurrent frees) + if (segment->used == 0) { + // free the segment (by forced reclaim) to make it available to other threads. + // note1: we prefer to free a segment as that might lead to reclaiming another + // segment that is still partially used. + // note2: we could in principle optimize this by skipping reclaim and directly + // freeing but that would violate some invariants temporarily) + mi_segment_reclaim(segment, heap, 0, NULL, tld); + } + else if (has_page && is_suitable) { + // found a large enough free span, or a page of the right block_size with free space + // we return the result of reclaim (which is usually `segment`) as it might free + // the segment due to concurrent frees (in which case `NULL` is returned). + return mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + } + else if (segment->abandoned_visits > 3 && is_suitable) { + // always reclaim on 3rd visit to limit the abandoned queue length. + mi_segment_reclaim(segment, heap, 0, NULL, tld); + } + else { + // otherwise, push on the visited list so it gets not looked at too quickly again + mi_segment_try_purge(segment, true /* force? */, tld->stats); // force purge if needed as we may not visit soon again + mi_abandoned_visited_push(tld->abandoned, segment); + } + } + return NULL; +} + + +void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) +{ + mi_segment_t* segment; + mi_abandoned_pool_t* pool = tld->abandoned; + int max_tries = (force ? 16*1024 : 1024); // limit latency + if (force) { + mi_abandoned_visited_revisit(pool); + } + while ((max_tries-- > 0) && ((segment = mi_abandoned_pop(pool)) != NULL)) { + mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees) + if (segment->used == 0) { + // free the segment (by forced reclaim) to make it available to other threads. + // note: we could in principle optimize this by skipping reclaim and directly + // freeing but that would violate some invariants temporarily) + mi_segment_reclaim(segment, heap, 0, NULL, tld); + } + else { + // otherwise, purge if needed and push on the visited list + // note: forced purge can be expensive if many threads are destroyed/created as in mstress. + mi_segment_try_purge(segment, force, tld->stats); + mi_abandoned_visited_push(pool, segment); + } + } +} + +/* ----------------------------------------------------------- + Reclaim or allocate +----------------------------------------------------------- */ + +static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +{ + mi_assert_internal(block_size < MI_HUGE_BLOCK_SIZE); + mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX); + + // 1. try to reclaim an abandoned segment + bool reclaimed; + mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld); + if (reclaimed) { + // reclaimed the right page right into the heap + mi_assert_internal(segment != NULL); + return NULL; // pretend out-of-memory as the page will be in the page queue of the heap with available blocks + } + else if (segment != NULL) { + // reclaimed a segment with a large enough empty span in it + return segment; + } + // 2. otherwise allocate a fresh segment + return mi_segment_alloc(0, 0, heap->arena_id, tld, os_tld, NULL); +} + + +/* ----------------------------------------------------------- + Page allocation +----------------------------------------------------------- */ + +static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +{ + mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE); + + // find a free page + size_t page_size = _mi_align_up(required, (required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE)); + size_t slices_needed = page_size / MI_SEGMENT_SLICE_SIZE; + mi_assert_internal(slices_needed * MI_SEGMENT_SLICE_SIZE == page_size); + mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, heap->arena_id, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld); + if (page==NULL) { + // no free page, allocate a new segment and try again + if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld, os_tld) == NULL) { + // OOM or reclaimed a good page in the heap + return NULL; + } + else { + // otherwise try again + return mi_segments_page_alloc(heap, page_kind, required, block_size, tld, os_tld); + } + } + mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size); + mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id()); + mi_segment_try_purge(_mi_ptr_segment(page), false, tld->stats); + return page; +} + + + +/* ----------------------------------------------------------- + Huge page allocation +----------------------------------------------------------- */ + +static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +{ + mi_page_t* page = NULL; + mi_segment_t* segment = mi_segment_alloc(size,page_alignment,req_arena_id,tld,os_tld,&page); + if (segment == NULL || page==NULL) return NULL; + mi_assert_internal(segment->used==1); + mi_assert_internal(mi_page_block_size(page) >= size); + #if MI_HUGE_PAGE_ABANDON + segment->thread_id = 0; // huge segments are immediately abandoned + #endif + + // for huge pages we initialize the xblock_size as we may + // overallocate to accommodate large alignments. + size_t psize; + uint8_t* start = _mi_segment_page_start(segment, page, &psize); + page->xblock_size = (psize > MI_HUGE_BLOCK_SIZE ? MI_HUGE_BLOCK_SIZE : (uint32_t)psize); + + // decommit the part of the prefix of a page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE) + if (page_alignment > 0 && segment->allow_decommit) { + uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment); + mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment)); + mi_assert_internal(psize - (aligned_p - start) >= size); + uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list + ptrdiff_t decommit_size = aligned_p - decommit_start; + _mi_os_reset(decommit_start, decommit_size, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + } + + return page; +} + +#if MI_HUGE_PAGE_ABANDON +// free huge block from another thread +void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) { + // huge page segments are always abandoned and can be freed immediately by any thread + mi_assert_internal(segment->kind==MI_SEGMENT_HUGE); + mi_assert_internal(segment == _mi_page_segment(page)); + mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id)==0); + + // claim it and free + mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized. + // paranoia: if this it the last reference, the cas should always succeed + size_t expected_tid = 0; + if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected_tid, heap->thread_id)) { + mi_block_set_next(page, block, page->free); + page->free = block; + page->used--; + page->is_zero = false; + mi_assert(page->used == 0); + mi_tld_t* tld = heap->tld; + _mi_segment_page_free(page, true, &tld->segments); + } +#if (MI_DEBUG!=0) + else { + mi_assert_internal(false); + } +#endif +} + +#else +// reset memory of a huge block from another thread +void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) { + MI_UNUSED(page); + mi_assert_internal(segment->kind == MI_SEGMENT_HUGE); + mi_assert_internal(segment == _mi_page_segment(page)); + mi_assert_internal(page->used == 1); // this is called just before the free + mi_assert_internal(page->free == NULL); + if (segment->allow_decommit) { + size_t csize = mi_usable_size(block); + if (csize > sizeof(mi_block_t)) { + csize = csize - sizeof(mi_block_t); + uint8_t* p = (uint8_t*)block + sizeof(mi_block_t); + _mi_os_reset(p, csize, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + } + } +} +#endif + +/* ----------------------------------------------------------- + Page allocation and free +----------------------------------------------------------- */ +mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { + mi_page_t* page; + if mi_unlikely(page_alignment > MI_ALIGNMENT_MAX) { + mi_assert_internal(_mi_is_power_of_two(page_alignment)); + mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE); + if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } + page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + } + else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { + page = mi_segments_page_alloc(heap,MI_PAGE_SMALL,block_size,block_size,tld,os_tld); + } + else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) { + page = mi_segments_page_alloc(heap,MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,block_size,tld, os_tld); + } + else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { + page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld, os_tld); + } + else { + page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + } + mi_assert_internal(page == NULL || _mi_heap_memid_is_suitable(heap, _mi_page_segment(page)->memid)); + mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); + return page; +} + +/* ----------------------------------------------------------- + Visit blocks in abandoned segments +----------------------------------------------------------- */ + +static bool mi_segment_visit_page(mi_segment_t* segment, mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) +{ + mi_heap_area_t area; + _mi_heap_area_init(&area, page); + if (!visitor(NULL, &area, NULL, area.block_size, arg)) return false; + if (visit_blocks) { + return _mi_heap_area_visit_blocks(&area, page, visitor, arg); + } + else { + return true; + } +} + +static bool mi_segment_visit_pages(mi_segment_t* segment, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + if (mi_slice_is_used(slice)) { + mi_page_t* const page = mi_slice_to_page(slice); + if (page->tag == page_tag) { + if (!mi_segment_visit_page(segment, page, visit_blocks, visitor, arg)) return false; + } + } + slice = slice + slice->slice_count; + } + return true; +} + +// Visit all blocks in a abandoned segments +bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + // Note: this is not safe in any other thread is abandoning or claiming segments from the pool + mi_segment_t* segment = mi_tagged_segment_ptr(pool->abandoned); + while (segment != NULL) { + if (!mi_segment_visit_pages(segment, page_tag, visit_blocks, visitor, arg)) return false; + segment = segment->abandoned_next; + } + + segment = pool->abandoned_visited; + while (segment != NULL) { + if (!mi_segment_visit_pages(segment, page_tag, visit_blocks, visitor, arg)) return false; + segment = segment->abandoned_next; + } + + return true; +} diff --git a/Objects/mimalloc/static.c b/Objects/mimalloc/static.c new file mode 100644 index 00000000000000..98c4aa18e450cd --- /dev/null +++ b/Objects/mimalloc/static.c @@ -0,0 +1,40 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#if defined(__sun) +// same remarks as os.c for the static's context. +#undef _XOPEN_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" + +// For a static override we create a single object file +// containing the whole library. If it is linked first +// it will override all the standard library allocation +// functions (on Unix's). +#include "alloc.c" // includes alloc-override.c +#include "alloc-aligned.c" +#include "alloc-posix.c" +#include "arena.c" +#include "bitmap.c" +#include "heap.c" +#include "init.c" +#include "options.c" +#include "os.c" +#include "page.c" // includes page-queue.c +#include "random.c" +#include "segment.c" +#include "segment-map.c" +#include "stats.c" +#include "prim/prim.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif diff --git a/Objects/mimalloc/stats.c b/Objects/mimalloc/stats.c new file mode 100644 index 00000000000000..e7ea397ec4c384 --- /dev/null +++ b/Objects/mimalloc/stats.c @@ -0,0 +1,467 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" + +#include // snprintf +#include // memset + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +#pragma warning(disable:4204) // non-constant aggregate initializer +#endif + +/* ----------------------------------------------------------- + Statistics operations +----------------------------------------------------------- */ + +static bool mi_is_in_main(void* stat) { + return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main + && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); +} + +static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + if (mi_is_in_main(stat)) + { + // add atomically (for abandoned pages) + int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); + mi_atomic_maxi64_relaxed(&stat->peak, current + amount); + if (amount > 0) { + mi_atomic_addi64_relaxed(&stat->allocated,amount); + } + else { + mi_atomic_addi64_relaxed(&stat->freed, -amount); + } + } + else { + // add thread local + stat->current += amount; + if (stat->current > stat->peak) stat->peak = stat->current; + if (amount > 0) { + stat->allocated += amount; + } + else { + stat->freed += -amount; + } + } +} + +void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { + if (mi_is_in_main(stat)) { + mi_atomic_addi64_relaxed( &stat->count, 1 ); + mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount ); + } + else { + stat->count++; + stat->total += amount; + } +} + +void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) { + mi_stat_update(stat, (int64_t)amount); +} + +void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { + mi_stat_update(stat, -((int64_t)amount)); +} + +// must be thread safe as it is called from stats_merge +static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { + if (stat==src) return; + if (src->allocated==0 && src->freed==0) return; + mi_atomic_addi64_relaxed( &stat->allocated, src->allocated * unit); + mi_atomic_addi64_relaxed( &stat->current, src->current * unit); + mi_atomic_addi64_relaxed( &stat->freed, src->freed * unit); + // peak scores do not work across threads.. + mi_atomic_addi64_relaxed( &stat->peak, src->peak * unit); +} + +static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t* src, int64_t unit) { + if (stat==src) return; + mi_atomic_addi64_relaxed( &stat->total, src->total * unit); + mi_atomic_addi64_relaxed( &stat->count, src->count * unit); +} + +// must be thread safe as it is called from stats_merge +static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { + if (stats==src) return; + mi_stat_add(&stats->segments, &src->segments,1); + mi_stat_add(&stats->pages, &src->pages,1); + mi_stat_add(&stats->reserved, &src->reserved, 1); + mi_stat_add(&stats->committed, &src->committed, 1); + mi_stat_add(&stats->reset, &src->reset, 1); + mi_stat_add(&stats->purged, &src->purged, 1); + mi_stat_add(&stats->page_committed, &src->page_committed, 1); + + mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); + mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); + mi_stat_add(&stats->threads, &src->threads, 1); + + mi_stat_add(&stats->malloc, &src->malloc, 1); + mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); + mi_stat_add(&stats->normal, &src->normal, 1); + mi_stat_add(&stats->huge, &src->huge, 1); + mi_stat_add(&stats->large, &src->large, 1); + + mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1); + mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1); + mi_stat_counter_add(&stats->commit_calls, &src->commit_calls, 1); + mi_stat_counter_add(&stats->reset_calls, &src->reset_calls, 1); + mi_stat_counter_add(&stats->purge_calls, &src->purge_calls, 1); + + mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); + mi_stat_counter_add(&stats->searches, &src->searches, 1); + mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); + mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); + mi_stat_counter_add(&stats->large_count, &src->large_count, 1); +#if MI_STAT>1 + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { + mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i], 1); + } + } +#endif +} + +/* ----------------------------------------------------------- + Display statistics +----------------------------------------------------------- */ + +// unit > 0 : size in binary bytes +// unit == 0: count as decimal +// unit < 0 : count in binary +static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { + char buf[32]; buf[0] = 0; + int len = 32; + const char* suffix = (unit <= 0 ? " " : "B"); + const int64_t base = (unit == 0 ? 1000 : 1024); + if (unit>0) n *= unit; + + const int64_t pos = (n < 0 ? -n : n); + if (pos < base) { + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); + } + } + else { + int64_t divider = base; + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } + const int64_t tens = (n / (divider/10)); + const long whole = (long)(tens/10); + const long frac1 = (long)(tens%10); + char unitdesc[8]; + snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); + } + _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf); +} + + +static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { + mi_printf_amount(n,unit,out,arg,NULL); +} + +static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { + if (unit==1) _mi_fprintf(out, arg, "%12s"," "); + else mi_print_amount(n,0,out,arg); +} + +static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { + _mi_fprintf(out, arg,"%10s:", msg); + if (unit > 0) { + mi_print_amount(stat->peak, unit, out, arg); + mi_print_amount(stat->allocated, unit, out, arg); + mi_print_amount(stat->freed, unit, out, arg); + mi_print_amount(stat->current, unit, out, arg); + mi_print_amount(unit, 1, out, arg); + mi_print_count(stat->allocated, unit, out, arg); + if (stat->allocated > stat->freed) { + _mi_fprintf(out, arg, " "); + _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok)); + _mi_fprintf(out, arg, "\n"); + } + else { + _mi_fprintf(out, arg, " ok\n"); + } + } + else if (unit<0) { + mi_print_amount(stat->peak, -1, out, arg); + mi_print_amount(stat->allocated, -1, out, arg); + mi_print_amount(stat->freed, -1, out, arg); + mi_print_amount(stat->current, -1, out, arg); + if (unit==-1) { + _mi_fprintf(out, arg, "%24s", ""); + } + else { + mi_print_amount(-unit, 1, out, arg); + mi_print_count((stat->allocated / -unit), 0, out, arg); + } + if (stat->allocated > stat->freed) + _mi_fprintf(out, arg, " not all freed!\n"); + else + _mi_fprintf(out, arg, " ok\n"); + } + else { + mi_print_amount(stat->peak, 1, out, arg); + mi_print_amount(stat->allocated, 1, out, arg); + _mi_fprintf(out, arg, "%11s", " "); // no freed + mi_print_amount(stat->current, 1, out, arg); + _mi_fprintf(out, arg, "\n"); + } +} + +static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { + mi_stat_print_ex(stat, msg, unit, out, arg, NULL); +} + +static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, "%10s:", msg); + mi_print_amount(stat->peak, unit, out, arg); + _mi_fprintf(out, arg, "\n"); +} + +static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, "%10s:", msg); + mi_print_amount(stat->total, -1, out, arg); + _mi_fprintf(out, arg, "\n"); +} + + +static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { + const int64_t avg_tens = (stat->count == 0 ? 0 : (stat->total*10 / stat->count)); + const long avg_whole = (long)(avg_tens/10); + const long avg_frac1 = (long)(avg_tens%10); + _mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1); +} + + +static void mi_print_header(mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s %11s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); +} + +#if MI_STAT>1 +static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) { + bool found = false; + char buf[64]; + for (size_t i = 0; i <= max; i++) { + if (bins[i].allocated > 0) { + found = true; + int64_t unit = _mi_bin_size((uint8_t)i); + snprintf(buf, 64, "%s %3lu", fmt, (long)i); + mi_stat_print(&bins[i], buf, unit, out, arg); + } + } + if (found) { + _mi_fprintf(out, arg, "\n"); + mi_print_header(out, arg); + } +} +#endif + + + +//------------------------------------------------------------ +// Use an output wrapper for line-buffered output +// (which is nice when using loggers etc.) +//------------------------------------------------------------ +typedef struct buffered_s { + mi_output_fun* out; // original output function + void* arg; // and state + char* buf; // local buffer of at least size `count+1` + size_t used; // currently used chars `used <= count` + size_t count; // total chars available for output +} buffered_t; + +static void mi_buffered_flush(buffered_t* buf) { + buf->buf[buf->used] = 0; + _mi_fputs(buf->out, buf->arg, NULL, buf->buf); + buf->used = 0; +} + +static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { + buffered_t* buf = (buffered_t*)arg; + if (msg==NULL || buf==NULL) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (buf->used >= buf->count) mi_buffered_flush(buf); + mi_assert_internal(buf->used < buf->count); + buf->buf[buf->used++] = c; + if (c == '\n') mi_buffered_flush(buf); + } +} + +//------------------------------------------------------------ +// Print statistics +//------------------------------------------------------------ + +static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { + // wrap the output function to be line buffered + char buf[256]; + buffered_t buffer = { out0, arg0, NULL, 0, 255 }; + buffer.buf = buf; + mi_output_fun* out = &mi_buffered_out; + void* arg = &buffer; + + // and print using that + mi_print_header(out,arg); + #if MI_STAT>1 + mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg); + #endif + #if MI_STAT + mi_stat_print(&stats->normal, "normal", (stats->normal_count.count == 0 ? 1 : -(stats->normal.allocated / stats->normal_count.count)), out, arg); + mi_stat_print(&stats->large, "large", (stats->large_count.count == 0 ? 1 : -(stats->large.allocated / stats->large_count.count)), out, arg); + mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg); + mi_stat_count_t total = { 0,0,0,0 }; + mi_stat_add(&total, &stats->normal, 1); + mi_stat_add(&total, &stats->large, 1); + mi_stat_add(&total, &stats->huge, 1); + mi_stat_print(&total, "total", 1, out, arg); + #endif + #if MI_STAT>1 + mi_stat_print(&stats->malloc, "malloc req", 1, out, arg); + _mi_fprintf(out, arg, "\n"); + #endif + mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); + mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); + mi_stat_peak_print(&stats->reset, "reset", 1, out, arg ); + mi_stat_peak_print(&stats->purged, "purged", 1, out, arg ); + mi_stat_print(&stats->page_committed, "touched", 1, out, arg); + mi_stat_print(&stats->segments, "segments", -1, out, arg); + mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); + mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); + mi_stat_print(&stats->pages, "pages", -1, out, arg); + mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg); + mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg); + mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg); + mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); + mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); + mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); + mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); + mi_stat_print(&stats->threads, "threads", -1, out, arg); + mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); + _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); + + size_t elapsed; + size_t user_time; + size_t sys_time; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; + mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); + _mi_fprintf(out, arg, "%10s: %5ld.%03ld s\n", "elapsed", elapsed/1000, elapsed%1000); + _mi_fprintf(out, arg, "%10s: user: %ld.%03ld s, system: %ld.%03ld s, faults: %lu, rss: ", "process", + user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, (unsigned long)page_faults ); + mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s"); + if (peak_commit > 0) { + _mi_fprintf(out, arg, ", commit: "); + mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); + } + _mi_fprintf(out, arg, "\n"); +} + +static mi_msecs_t mi_process_start; // = 0 + +static mi_stats_t* mi_stats_get_default(void) { + mi_heap_t* heap = mi_heap_get_default(); + return &heap->tld->stats; +} + +static void mi_stats_merge_from(mi_stats_t* stats) { + if (stats != &_mi_stats_main) { + mi_stats_add(&_mi_stats_main, stats); + memset(stats, 0, sizeof(mi_stats_t)); + } +} + +void mi_stats_reset(void) mi_attr_noexcept { + mi_stats_t* stats = mi_stats_get_default(); + if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); } + memset(&_mi_stats_main, 0, sizeof(mi_stats_t)); + if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); }; +} + +void mi_stats_merge(void) mi_attr_noexcept { + mi_stats_merge_from( mi_stats_get_default() ); +} + +void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` + mi_stats_merge_from(stats); +} + +void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_stats_merge_from(mi_stats_get_default()); + _mi_stats_print(&_mi_stats_main, out, arg); +} + +void mi_stats_print(void* out) mi_attr_noexcept { + // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`) + mi_stats_print_out((mi_output_fun*)out, NULL); +} + +void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { + _mi_stats_print(mi_stats_get_default(), out, arg); +} + + +// ---------------------------------------------------------------- +// Basic timer for convenience; use milli-seconds to avoid doubles +// ---------------------------------------------------------------- + +static mi_msecs_t mi_clock_diff; + +mi_msecs_t _mi_clock_now(void) { + return _mi_prim_clock_now(); +} + +mi_msecs_t _mi_clock_start(void) { + if (mi_clock_diff == 0.0) { + mi_msecs_t t0 = _mi_clock_now(); + mi_clock_diff = _mi_clock_now() - t0; + } + return _mi_clock_now(); +} + +mi_msecs_t _mi_clock_end(mi_msecs_t start) { + mi_msecs_t end = _mi_clock_now(); + return (end - start - mi_clock_diff); +} + + +// -------------------------------------------------------- +// Basic process statistics +// -------------------------------------------------------- + +mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept +{ + mi_process_info_t pinfo; + _mi_memzero_var(pinfo); + pinfo.elapsed = _mi_clock_end(mi_process_start); + pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); + pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); + pinfo.current_rss = pinfo.current_commit; + pinfo.peak_rss = pinfo.peak_commit; + pinfo.utime = 0; + pinfo.stime = 0; + pinfo.page_faults = 0; + + _mi_prim_process_info(&pinfo); + + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); + if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); + if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); + if (current_rss!=NULL) *current_rss = pinfo.current_rss; + if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss; + if (current_commit!=NULL) *current_commit = pinfo.current_commit; + if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; + if (page_faults!=NULL) *page_faults = pinfo.page_faults; +} diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 5b9f74dbddf031..73ad9711b6b0fc 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -3,13 +3,16 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "osdefs.h" // MAXPATHLEN static PyMemberDef module_members[] = { @@ -88,21 +91,31 @@ new_module_notrack(PyTypeObject *mt) m->md_weaklist = NULL; m->md_name = NULL; m->md_dict = PyDict_New(); - if (m->md_dict != NULL) { - return m; + if (m->md_dict == NULL) { + Py_DECREF(m); + return NULL; } - Py_DECREF(m); - return NULL; + return m; +} + +static void +track_module(PyModuleObject *m) +{ + _PyObject_SetDeferredRefcount(m->md_dict); + PyObject_GC_Track(m->md_dict); + + _PyObject_SetDeferredRefcount((PyObject *)m); + PyObject_GC_Track(m); } static PyObject * new_module(PyTypeObject *mt, PyObject *args, PyObject *kws) { - PyObject *m = (PyObject *)new_module_notrack(mt); + PyModuleObject *m = new_module_notrack(mt); if (m != NULL) { - PyObject_GC_Track(m); + track_module(m); } - return m; + return (PyObject *)m; } PyObject * @@ -113,7 +126,7 @@ PyModule_NewObject(PyObject *name) return NULL; if (module_init_dict(m, m->md_dict, name, NULL) != 0) goto fail; - PyObject_GC_Track(m); + track_module(m); return (PyObject *)m; fail: @@ -171,6 +184,7 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) if (func == NULL) { return -1; } + _PyObject_SetDeferredRefcount(func); if (PyObject_SetAttrString(module, fdef->ml_name, func) != 0) { Py_DECREF(func); return -1; @@ -237,6 +251,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; +#ifdef Py_GIL_DISABLE + m->md_gil = Py_MOD_GIL_USED; +#endif return (PyObject*)m; } @@ -249,6 +266,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *m = NULL; int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; + int has_gil_slot = 0; + void *gil_slot = Py_MOD_GIL_USED; int has_execution_slots = 0; const char *name; int ret; @@ -303,6 +322,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = cur_slot->value; has_multiple_interpreters_slot = 1; break; + case Py_mod_gil: + if (has_gil_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'gil' slot", + name); + goto error; + } + gil_slot = cur_slot->value; + has_gil_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -362,6 +392,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio if (PyModule_Check(m)) { ((PyModuleObject*)m)->md_state = NULL; ((PyModuleObject*)m)->md_def = def; +#ifdef Py_GIL_DISABLED + ((PyModuleObject*)m)->md_gil = gil_slot; +#else + (void)gil_slot; +#endif } else { if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { PyErr_Format( @@ -403,6 +438,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +#ifdef Py_GIL_DISABLED +int +PyUnstable_Module_SetGIL(PyObject *module, void *gil) +{ + if (!PyModule_Check(module)) { + PyErr_BadInternalCall(); + return -1; + } + ((PyModuleObject *)module)->md_gil = gil; + return 0; +} +#endif + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { @@ -458,6 +506,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) } break; case Py_mod_multiple_interpreters: + case Py_mod_gil: /* handled in PyModule_FromDefAndSpec2 */ break; default: @@ -522,6 +571,7 @@ PyModule_GetNameObject(PyObject *mod) } PyObject *name; if (PyDict_GetItemRef(dict, &_Py_ID(__name__), &name) <= 0) { + // error or not found goto error; } if (!PyUnicode_Check(name)) { @@ -562,6 +612,7 @@ PyModule_GetFilenameObject(PyObject *mod) } PyObject *fileobj; if (PyDict_GetItemRef(dict, &_Py_ID(__file__), &fileobj) <= 0) { + // error or not found goto error; } if (!PyUnicode_Check(fileobj)) { @@ -647,7 +698,7 @@ _PyModule_ClearDict(PyObject *d) PyErr_Clear(); } if (PyDict_SetItem(d, key, Py_None) != 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing module dict"); } } } @@ -668,7 +719,7 @@ _PyModule_ClearDict(PyObject *d) PyErr_Clear(); } if (PyDict_SetItem(d, key, Py_None) != 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing module dict"); } } } @@ -703,16 +754,7 @@ static int module___init___impl(PyModuleObject *self, PyObject *name, PyObject *doc) /*[clinic end generated code: output=e7e721c26ce7aad7 input=57f9e177401e5e1e]*/ { - PyObject *dict = self->md_dict; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - self->md_dict = dict; - } - if (module_init_dict(self, dict, name, doc) < 0) - return -1; - return 0; + return module_init_dict(self, self->md_dict, name, doc); } static void @@ -747,27 +789,20 @@ module_repr(PyModuleObject *m) } /* Check if the "_initializing" attribute of the module spec is set to true. - Clear the exception and return 0 if spec is NULL. */ int _PyModuleSpec_IsInitializing(PyObject *spec) { - if (spec != NULL) { - PyObject *value; - int ok = PyObject_GetOptionalAttr(spec, &_Py_ID(_initializing), &value); - if (ok == 0) { - return 0; - } - if (value != NULL) { - int initializing = PyObject_IsTrue(value); - Py_DECREF(value); - if (initializing >= 0) { - return initializing; - } - } + if (spec == NULL) { + return 0; } - PyErr_Clear(); - return 0; + PyObject *value; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(_initializing), &value); + if (rc > 0) { + rc = PyObject_IsTrue(value); + Py_DECREF(value); + } + return rc; } /* Check if the submodule name is in the "_uninitialized_submodules" attribute @@ -780,17 +815,106 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return 0; } - PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_uninitialized_submodules)); - if (value == NULL) { + PyObject *value; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(_uninitialized_submodules), &value); + if (rc > 0) { + rc = PySequence_Contains(value, name); + Py_DECREF(value); + } + return rc; +} + +static int +_get_file_origin_from_spec(PyObject *spec, PyObject **p_origin) +{ + PyObject *has_location = NULL; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(has_location), &has_location); + if (rc <= 0) { + return rc; + } + // If origin is not a location, or doesn't exist, or is not a str), we could consider falling + // back to module.__file__. But the cases in which module.__file__ is not __spec__.origin + // are cases in which we probably shouldn't be guessing. + rc = PyObject_IsTrue(has_location); + Py_DECREF(has_location); + if (rc <= 0) { + return rc; + } + // has_location is true, so origin is a location + PyObject *origin = NULL; + rc = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); + if (rc <= 0) { + return rc; + } + assert(origin != NULL); + if (!PyUnicode_Check(origin)) { + Py_DECREF(origin); return 0; } + *p_origin = origin; + return 1; +} + +static int +_is_module_possibly_shadowing(PyObject *origin) +{ + // origin must be a unicode subtype + // Returns 1 if the module at origin could be shadowing a module of the + // same name later in the module search path. The condition we check is basically: + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + // return not sys.flags.safe_path and root == (sys.path[0] or os.getcwd()) + // Returns 0 otherwise (or if we aren't sure) + // Returns -1 if an error occurred that should be propagated + if (origin == NULL) { + return 0; + } + + // not sys.flags.safe_path + const PyConfig *config = _Py_GetConfig(); + if (config->safe_path) { + return 0; + } + + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + wchar_t root[MAXPATHLEN + 1]; + Py_ssize_t size = PyUnicode_AsWideChar(origin, root, MAXPATHLEN); + if (size < 0) { + return -1; + } + assert(size <= MAXPATHLEN); + root[size] = L'\0'; + + wchar_t *sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + // If it's a package then we need to look one directory further up + if (wcscmp(sep + 1, L"__init__.py") == 0) { + *sep = L'\0'; + sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + } + *sep = L'\0'; - int is_uninitialized = PySequence_Contains(value, name); - Py_DECREF(value); - if (is_uninitialized == -1) { + // sys.path[0] or os.getcwd() + wchar_t *sys_path_0 = config->sys_path_0; + if (!sys_path_0) { return 0; } - return is_uninitialized; + + wchar_t sys_path_0_buf[MAXPATHLEN]; + if (sys_path_0[0] == L'\0') { + // if sys.path[0] == "", treat it as if it were the current directory + if (!_Py_wgetcwd(sys_path_0_buf, MAXPATHLEN)) { + return -1; + } + sys_path_0 = sys_path_0_buf; + } + + int result = wcscmp(sys_path_0, root) == 0; + return result; } PyObject* @@ -816,58 +940,123 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); - getattr = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__getattr__)); + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + return NULL; + } if (getattr) { PyObject *result = PyObject_CallOneArg(getattr, name); if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { // suppress AttributeError PyErr_Clear(); } + Py_DECREF(getattr); return result; } - if (PyErr_Occurred()) { + + // The attribute was not found. We make a best effort attempt at a useful error message, + // but only if we're not suppressing AttributeError. + if (suppress == 1) { + return NULL; + } + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name) < 0) { return NULL; } - mod_name = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__name__)); - if (mod_name && PyUnicode_Check(mod_name)) { - Py_INCREF(mod_name); - PyObject *spec = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__spec__)); - if (spec == NULL && PyErr_Occurred()) { - Py_DECREF(mod_name); - return NULL; + if (!mod_name || !PyUnicode_Check(mod_name)) { + Py_XDECREF(mod_name); + PyErr_Format(PyExc_AttributeError, + "module has no attribute '%U'", name); + return NULL; + } + PyObject *spec; + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(mod_name); + return NULL; + } + if (spec == NULL) { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); + Py_DECREF(mod_name); + return NULL; + } + + PyObject *origin = NULL; + if (_get_file_origin_from_spec(spec, &origin) < 0) { + goto done; + } + + int is_possibly_shadowing = _is_module_possibly_shadowing(origin); + if (is_possibly_shadowing < 0) { + goto done; + } + int is_possibly_shadowing_stdlib = 0; + if (is_possibly_shadowing) { + PyObject *stdlib_modules = PySys_GetObject("stdlib_module_names"); + if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { + is_possibly_shadowing_stdlib = PySet_Contains(stdlib_modules, mod_name); + if (is_possibly_shadowing_stdlib < 0) { + goto done; + } } - if (suppress != 1) { - Py_XINCREF(spec); - if (_PyModuleSpec_IsInitializing(spec)) { + } + + if (is_possibly_shadowing_stdlib) { + assert(origin); + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U' " + "(consider renaming '%U' since it has the same " + "name as the standard library module named '%U' " + "and the import system gives it precedence)", + mod_name, name, origin, mod_name); + } + else { + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc > 0) { + if (is_possibly_shadowing) { + assert(origin); + // For third-party modules, only mention the possibility of + // shadowing if the module is being initialized. PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, name); + "module '%U' has no attribute '%U' " + "(consider renaming '%U' if it has the same name " + "as a third-party module you intended to import)", + mod_name, name, origin); } - else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { + else if (origin) { PyErr_Format(PyExc_AttributeError, - "cannot access submodule '%U' of module '%U' " - "(most likely due to a circular import)", - name, mod_name); + "partially initialized " + "module '%U' from '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, origin, name); } else { PyErr_Format(PyExc_AttributeError, - "module '%U' has no attribute '%U'", - mod_name, name); + "partially initialized " + "module '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, name); + } + } + else if (rc == 0) { + rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); + if (rc > 0) { + PyErr_Format(PyExc_AttributeError, + "cannot access submodule '%U' of module '%U' " + "(most likely due to a circular import)", + name, mod_name); + } + else if (rc == 0) { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); } - Py_XDECREF(spec); } - Py_DECREF(mod_name); - return NULL; - } - else if (PyErr_Occurred()) { - return NULL; - } - if (suppress != 1) { - PyErr_Format(PyExc_AttributeError, - "module has no attribute '%U'", name); } + +done: + Py_XDECREF(origin); + Py_DECREF(spec); + Py_DECREF(mod_name); return NULL; } @@ -902,10 +1091,9 @@ module_clear(PyModuleObject *m) { int res = m->md_def->m_clear((PyObject*)m); if (PyErr_Occurred()) { - PySys_FormatStderr("Exception ignored in m_clear of module%s%V\n", - m->md_name ? " " : "", - m->md_name, ""); - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V", + m->md_name ? " " : "", + m->md_name, ""); } if (res) return res; @@ -946,7 +1134,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -957,13 +1145,97 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + return dict; +} - PyObject *annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); - if (annotations) { - Py_INCREF(annotations); +static PyObject * +module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotate; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { + annotate = Py_None; + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate) == -1) { + Py_CLEAR(annotate); + } } - else if (!PyErr_Occurred()) { - annotations = PyDict_New(); + Py_DECREF(dict); + return annotate; +} + +static int +module_set_annotate(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + Py_DECREF(dict); + return -1; + } + + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), value) == -1) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; +} + +static PyObject * +module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotations; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { + PyObject *annotate; + int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); + if (annotate_result < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate_result == 1 && PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotate); + Py_DECREF(annotations); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_XDECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -980,14 +1252,10 @@ static int module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) { int ret = -1; - PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } - if (!PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - goto exit; - } if (value != NULL) { /* set */ @@ -995,13 +1263,19 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor } else { /* delete */ - ret = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (ret < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + ret = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (ret == 0) { + PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__annotations__)); + ret = -1; } + else if (ret > 0) { + ret = 0; + } + } + if (ret == 0 && PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; } -exit: Py_DECREF(dict); return ret; } @@ -1009,6 +1283,7 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor static PyGetSetDef module_getsets[] = { {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {"__annotate__", (getter)module_get_annotate, (setter)module_set_annotate}, {NULL} }; diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 204c114fd9de2d..5b7547103a2b3f 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -1,12 +1,12 @@ // namespace object implementation #include "Python.h" +#include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_namespace.h" // _PyNamespace_Type #include // offsetof() - typedef struct { PyObject_HEAD PyObject *ns_dict; @@ -43,10 +43,28 @@ namespace_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int namespace_init(_PyNamespaceObject *ns, PyObject *args, PyObject *kwds) { - if (PyTuple_GET_SIZE(args) != 0) { - PyErr_Format(PyExc_TypeError, "no positional arguments expected"); + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, _PyType_Name(Py_TYPE(ns)), 0, 1, &arg)) { return -1; } + if (arg != NULL) { + PyObject *dict; + if (PyDict_CheckExact(arg)) { + dict = Py_NewRef(arg); + } + else { + dict = PyObject_CallOneArg((PyObject *)&PyDict_Type, arg); + if (dict == NULL) { + return -1; + } + } + int err = (!PyArg_ValidateKeywordArguments(dict) || + PyDict_Update(ns->ns_dict, dict) < 0); + Py_DECREF(dict); + if (err) { + return -1; + } + } if (kwds == NULL) { return 0; } @@ -219,15 +237,17 @@ namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) static PyMethodDef namespace_methods[] = { {"__reduce__", (PyCFunction)namespace_reduce, METH_NOARGS, namespace_reduce__doc__}, - {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the namespace object with new values for the specified attributes.")}, {NULL, NULL} // sentinel }; PyDoc_STRVAR(namespace_doc, -"A simple attribute-based namespace.\n\ -\n\ -SimpleNamespace(**kwargs)"); +"SimpleNamespace(mapping_or_iterable=(), /, **kwargs)\n\ +--\n\n\ +A simple attribute-based namespace."); PyTypeObject _PyNamespace_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Objects/object.c b/Objects/object.c index 3ed272afdced7c..c4622359bb1035 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,17 +2,22 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_brc.h" // _Py_brc_queue_object() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -21,7 +26,6 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() @@ -72,21 +76,16 @@ get_legacy_reftotal(void) interp->object_state.reftotal static inline void -reftotal_increment(PyInterpreterState *interp) +reftotal_add(PyThreadState *tstate, Py_ssize_t n) { - REFTOTAL(interp)++; -} - -static inline void -reftotal_decrement(PyInterpreterState *interp) -{ - REFTOTAL(interp)--; -} - -static inline void -reftotal_add(PyInterpreterState *interp, Py_ssize_t n) -{ - REFTOTAL(interp) += n; +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + // relaxed store to avoid data race with read in get_reftotal() + Py_ssize_t reftotal = tstate_impl->reftotal + n; + _Py_atomic_store_ssize_relaxed(&tstate_impl->reftotal, reftotal); +#else + REFTOTAL(tstate->interp) += n; +#endif } static inline Py_ssize_t get_global_reftotal(_PyRuntimeState *); @@ -116,7 +115,15 @@ get_reftotal(PyInterpreterState *interp) { /* For a single interpreter, we ignore the legacy _Py_RefTotal, since we can't determine which interpreter updated it. */ - return REFTOTAL(interp); + Py_ssize_t total = REFTOTAL(interp); +#ifdef Py_GIL_DISABLED + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + /* This may race with other threads modifications to their reftotal */ + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)p; + total += _Py_atomic_load_ssize_relaxed(&tstate_impl->reftotal); + } +#endif + return total; } static inline Py_ssize_t @@ -128,7 +135,7 @@ get_global_reftotal(_PyRuntimeState *runtime) HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = PyInterpreterState_Head(); for (; interp != NULL; interp = PyInterpreterState_Next(interp)) { - total += REFTOTAL(interp); + total += get_reftotal(interp); } HEAD_UNLOCK(&_PyRuntime); @@ -221,32 +228,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) void _Py_INCREF_IncRefTotal(void) { - reftotal_increment(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), 1); } /* This is used strictly by Py_DECREF(). */ void _Py_DECREF_DecRefTotal(void) { - reftotal_decrement(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), -1); } void -_Py_IncRefTotal(PyInterpreterState *interp) +_Py_IncRefTotal(PyThreadState *tstate) { - reftotal_increment(interp); + reftotal_add(tstate, 1); } void -_Py_DecRefTotal(PyInterpreterState *interp) +_Py_DecRefTotal(PyThreadState *tstate) { - reftotal_decrement(interp); + reftotal_add(tstate, -1); } void -_Py_AddRefTotal(PyInterpreterState *interp, Py_ssize_t n) +_Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { - reftotal_add(interp, n); + reftotal_add(tstate, n); } /* This includes the legacy total @@ -266,7 +273,10 @@ _Py_GetLegacyRefTotal(void) Py_ssize_t _PyInterpreterState_GetRefTotal(PyInterpreterState *interp) { - return get_reftotal(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t total = get_reftotal(interp); + HEAD_UNLOCK(&_PyRuntime); + return total; } #endif /* Py_REF_DEBUG */ @@ -295,6 +305,127 @@ _Py_DecRef(PyObject *o) Py_DECREF(o); } +#ifdef Py_GIL_DISABLED +# ifdef Py_REF_DEBUG +static int +is_dead(PyObject *o) +{ +# if SIZEOF_SIZE_T == 8 + return (uintptr_t)o->ob_type == 0xDDDDDDDDDDDDDDDD; +# else + return (uintptr_t)o->ob_type == 0xDDDDDDDD; +# endif +} +# endif + +void +_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +{ + // Should we queue the object for the owning thread to merge? + int should_queue; + + Py_ssize_t new_shared; + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&o->ob_ref_shared); + do { + should_queue = (shared == 0 || shared == _Py_REF_MAYBE_WEAKREF); + + if (should_queue) { + // If the object had refcount zero, not queued, and not merged, + // then we enqueue the object to be merged by the owning thread. + // In this case, we don't subtract one from the reference count + // because the queue holds a reference. + new_shared = _Py_REF_QUEUED; + } + else { + // Otherwise, subtract one from the reference count. This might + // be negative! + new_shared = shared - (1 << _Py_REF_SHARED_SHIFT); + } + +#ifdef Py_REF_DEBUG + if ((new_shared < 0 && _Py_REF_IS_MERGED(new_shared)) || + (should_queue && is_dead(o))) + { + _Py_NegativeRefcount(filename, lineno, o); + } +#endif + } while (!_Py_atomic_compare_exchange_ssize(&o->ob_ref_shared, + &shared, new_shared)); + + if (should_queue) { +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + _Py_brc_queue_object(o); + } + else if (new_shared == _Py_REF_MERGED) { + // refcount is zero AND merged + _Py_Dealloc(o); + } +} + +void +_Py_DecRefShared(PyObject *o) +{ + _Py_DecRefSharedDebug(o, NULL, 0); +} + +void +_Py_MergeZeroLocalRefcount(PyObject *op) +{ + assert(op->ob_ref_local == 0); + + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); + if (shared == 0) { + // Fast-path: shared refcount is zero (including flags) + _Py_Dealloc(op); + return; + } + + // Slow-path: atomically set the flags (low two bits) to _Py_REF_MERGED. + Py_ssize_t new_shared; + do { + new_shared = (shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_MERGED; + } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, + &shared, new_shared)); + + if (new_shared == _Py_REF_MERGED) { + // i.e., the shared refcount is zero (only the flags are set) so we + // deallocate the object. + _Py_Dealloc(op); + } +} + +Py_ssize_t +_Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) +{ + assert(!_Py_IsImmortal(op)); + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // gh-119999: Write to ob_ref_local and ob_tid before merging the refcount. + Py_ssize_t local = (Py_ssize_t)op->ob_ref_local; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + + Py_ssize_t refcnt; + Py_ssize_t new_shared; + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + do { + refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + refcnt += local; + refcnt += extra; + + new_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, + &shared, new_shared)); + return refcnt; +} +#endif /* Py_GIL_DISABLED */ + /**************************************/ @@ -388,9 +519,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); + _Py_ResurrectReference(self); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) @@ -675,6 +804,21 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +void +_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +{ + // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() + // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() + _PyFloat_ClearFreeList(freelists, is_finalization); + _PyTuple_ClearFreeList(freelists, is_finalization); + _PyList_ClearFreeList(freelists, is_finalization); + _PyDict_ClearFreeList(freelists, is_finalization); + _PyContext_ClearFreeList(freelists, is_finalization); + _PyAsyncGen_ClearFreeLists(freelists, is_finalization); + // Only be cleared if is_finalization is true. + _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); + _PySlice_ClearFreeList(freelists, is_finalization); +} /* def _PyObject_FunctionStr(x): @@ -921,7 +1065,10 @@ PyObject_HasAttrString(PyObject *obj, const char *name) { int rc = PyObject_HasAttrStringWithError(obj, name); if (rc < 0) { - PyErr_Clear(); + PyErr_FormatUnraisable( + "Exception ignored in PyObject_HasAttrString(); consider using " + "PyObject_HasAttrStringWithError(), " + "PyObject_GetOptionalAttrString() or PyObject_GetAttrString()"); return 0; } return rc; @@ -978,18 +1125,7 @@ _PyObject_GetAttrId(PyObject *v, _Py_Identifier *name) } int -_PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) -{ - int result; - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) - return -1; - result = PyObject_SetAttr(v, oname, w); - return result; -} - -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1044,7 +1180,7 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } @@ -1072,7 +1208,7 @@ PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) } return 0; } - if (tp->tp_getattro == (getattrofunc)_Py_type_getattro) { + if (tp->tp_getattro == _Py_type_getattro) { int supress_missing_attribute_exception = 0; *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); if (supress_missing_attribute_exception) { @@ -1156,7 +1292,10 @@ PyObject_HasAttr(PyObject *obj, PyObject *name) { int rc = PyObject_HasAttrWithError(obj, name); if (rc < 0) { - PyErr_Clear(); + PyErr_FormatUnraisable( + "Exception ignored in PyObject_HasAttr(); consider using " + "PyObject_HasAttrWithError(), " + "PyObject_GetOptionalAttr() or PyObject_GetAttr()"); return 0; } return rc; @@ -1176,7 +1315,8 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) } Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (tp->tp_setattro != NULL) { err = (*tp->tp_setattro)(v, name, value); Py_DECREF(name); @@ -1258,16 +1398,15 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + dict = _PyObject_MaterializeManagedDict(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict; } PyObject * @@ -1320,13 +1459,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1335,22 +1474,18 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1363,19 +1498,14 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } if (dict != NULL) { Py_INCREF(dict); - PyObject *attr = PyDict_GetItemWithError(dict, name); - if (attr != NULL) { - *method = Py_NewRef(attr); + if (PyDict_GetItemRef(dict, name, method) != 0) { + // found or error Py_DECREF(dict); Py_XDECREF(descr); return 0; } + // not found Py_DECREF(dict); - - if (PyErr_Occurred()) { - Py_XDECREF(descr); - return 0; - } } if (meth_found) { @@ -1398,7 +1528,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1432,11 +1562,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1448,29 +1577,24 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + if (PyUnicode_CheckExact(name) && + _PyObject_TryGetInstanceAttribute(obj, name, &res)) { + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MaterializeManagedDict(obj); + if (dict == NULL) { + res = NULL; + goto done; + } } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { @@ -1480,21 +1604,17 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } if (dict != NULL) { Py_INCREF(dict); - res = PyDict_GetItemWithError(dict, name); + int rc = PyDict_GetItemRef(dict, name, &res); + Py_DECREF(dict); if (res != NULL) { - Py_INCREF(res); - Py_DECREF(dict); goto done; } - else { - Py_DECREF(dict); - if (PyErr_Occurred()) { - if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - } - else { - goto done; - } + else if (rc < 0) { + if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + goto done; } } } @@ -1519,7 +1639,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1542,6 +1662,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1555,10 +1676,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1568,22 +1688,15 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; + + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + res = _PyObject_StoreInstanceAttribute(obj, name, value); + goto error_check; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; - if (*dictptr == NULL) { - if (_PyObject_InitInlineValues(obj, tp) < 0) { - goto done; - } - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -1601,7 +1714,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1611,7 +1724,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1624,17 +1737,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } - set_attribute_error_context(obj, name); + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1654,9 +1760,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_GetManagedDict(obj) == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } @@ -1676,7 +1782,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -1884,6 +1992,11 @@ static PyNumberMethods none_as_number = { 0, /* nb_index */ }; +PyDoc_STRVAR(none_doc, +"NoneType()\n" +"--\n\n" +"The type of the None singleton."); + PyTypeObject _PyNone_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NoneType", @@ -1905,10 +2018,10 @@ PyTypeObject _PyNone_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + none_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ - 0, /*tp_richcompare */ + _Py_BaseObject_RichCompare, /*tp_richcompare */ 0, /*tp_weaklistoffset */ 0, /*tp_iter */ 0, /*tp_iternext */ @@ -1925,10 +2038,7 @@ PyTypeObject _PyNone_Type = { none_new, /*tp_new */ }; -PyObject _Py_NoneStruct = { - { _Py_IMMORTAL_REFCNT }, - &_PyNone_Type -}; +PyObject _Py_NoneStruct = _PyObject_HEAD_INIT(&_PyNone_Type); /* NotImplemented is an object that can be used to signal that an operation is not implemented for the given type combination. */ @@ -1973,19 +2083,20 @@ notimplemented_dealloc(PyObject *notimplemented) static int notimplemented_bool(PyObject *v) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "NotImplemented should not be used in a boolean context", - 1) < 0) - { - return -1; - } - return 1; + PyErr_SetString(PyExc_TypeError, + "NotImplemented should not be used in a boolean context"); + return -1; } static PyNumberMethods notimplemented_as_number = { .nb_bool = notimplemented_bool, }; +PyDoc_STRVAR(notimplemented_doc, +"NotImplementedType()\n" +"--\n\n" +"The type of the NotImplemented singleton."); + PyTypeObject _PyNotImplemented_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NotImplementedType", @@ -2007,7 +2118,7 @@ PyTypeObject _PyNotImplemented_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + notimplemented_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ 0, /*tp_richcompare */ @@ -2027,10 +2138,7 @@ PyTypeObject _PyNotImplemented_Type = { notimplemented_new, /*tp_new */ }; -PyObject _Py_NotImplementedStruct = { - { _Py_IMMORTAL_REFCNT }, - &_PyNotImplemented_Type -}; +PyObject _Py_NotImplementedStruct = _PyObject_HEAD_INIT(&_PyNotImplemented_Type); PyStatus @@ -2110,12 +2218,12 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, - &PyInterpreterID_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, @@ -2157,6 +2265,11 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, +#ifdef _Py_TIER2 + &_PyCounterExecutor_Type, + &_PyCounterOptimizer_Type, + &_PyDefaultOptimizer_Type, +#endif &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2165,6 +2278,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, &_PyLineIterator, &_PyManagedBuffer_Type, @@ -2176,6 +2290,10 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 + &_PyUOpExecutor_Type, + &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, @@ -2230,7 +2348,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); } } @@ -2238,21 +2356,32 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 +#if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; +#else + op->ob_tid = _Py_ThreadId(); + op->_padding = 0; + op->ob_mutex = (PyMutex){ 0 }; + op->ob_gc_bits = 0; + op->ob_ref_local = 1; + op->ob_ref_shared = 0; +#endif #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_CREATE, data); + } } void _Py_NewReference(PyObject *op) { #ifdef Py_REF_DEBUG - reftotal_increment(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif new_reference(op); } @@ -2263,6 +2392,66 @@ _Py_NewReferenceNoTotal(PyObject *op) new_reference(op); } +void +_Py_SetImmortalUntracked(PyObject *op) +{ +#ifdef Py_DEBUG + // For strings, use _PyUnicode_InternImmortal instead. + if (PyUnicode_CheckExact(op)) { + assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL + || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); + } +#endif +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else + op->ob_refcnt = _Py_IMMORTAL_REFCNT; +#endif +} + +void +_Py_SetImmortal(PyObject *op) +{ + if (PyObject_IS_GC(op) && _PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + _Py_SetImmortalUntracked(op); +} + +void +_PyObject_SetDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + assert(PyType_IS_GC(Py_TYPE(op))); + assert(_Py_IsOwnedByCurrentThread(op)); + assert(op->ob_ref_shared == 0); + _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_atomic_load_int_relaxed(&interp->gc.immortalize) == 1) { + // gh-117696: immortalize objects instead of using deferred reference + // counting for now. + _Py_SetImmortal(op); + return; + } + op->ob_ref_local += 1; + op->ob_ref_shared = _Py_REF_QUEUED; +#endif +} + +void +_Py_ResurrectReference(PyObject *op) +{ +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(op); +#endif + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); + } +} + #ifdef Py_TRACE_REFS void @@ -2524,28 +2713,31 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#define _PyTrash_UNWIND_LEVEL 50 - /* Add op to the gcstate->trash_delete_later list. Called when the current * call-stack depth gets large. op must be a currently untracked gc'ed * object, with refcount 0. Py_DECREF must already have been called on it. */ -static void -_PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) +void +_PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); - _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); - trash->delete_later = op; +#ifdef Py_GIL_DISABLED + _PyObject_ASSERT(op, op->ob_tid == 0); + op->ob_tid = (uintptr_t)tstate->delete_later; +#else + _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)tstate->delete_later); +#endif + tstate->delete_later = op; } /* Deallocate all the objects in the gcstate->trash_delete_later list. * Called when the call-stack unwinds again. */ -static void -_PyTrash_thread_destroy_chain(struct _py_trashcan *trash) +void +_PyTrash_thread_destroy_chain(PyThreadState *tstate) { - /* We need to increase trash_delete_nesting here, otherwise, + /* We need to increase c_recursion_remaining here, otherwise, _PyTrash_thread_destroy_chain will be called recursively and then possibly crash. An example that may crash without increase: @@ -2556,14 +2748,18 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) tups = [(tup,) for tup in tups] del tups */ - assert(trash->delete_nesting == 0); - ++trash->delete_nesting; - while (trash->delete_later) { - PyObject *op = trash->delete_later; + assert(tstate->c_recursion_remaining > Py_TRASHCAN_HEADROOM); + tstate->c_recursion_remaining--; + while (tstate->delete_later) { + PyObject *op = tstate->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; - trash->delete_later = - (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#ifdef Py_GIL_DISABLED + tstate->delete_later = (PyObject*) op->ob_tid; + op->ob_tid = 0; +#else + tstate->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#endif /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but @@ -2573,92 +2769,10 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) */ _PyObject_ASSERT(op, Py_REFCNT(op) == 0); (*dealloc)(op); - assert(trash->delete_nesting == 1); } - --trash->delete_nesting; + tstate->c_recursion_remaining++; } - -static struct _py_trashcan * -_PyTrash_get_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - return &tstate->trash; - } - // The current thread must be finalizing. - // Fall back to using thread-local state. - // XXX Use thread-local variable syntax? - assert(PyThread_tss_is_created(&_PyRuntime.trashTSSkey)); - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash == NULL) { - trash = PyMem_RawMalloc(sizeof(struct _py_trashcan)); - if (trash == NULL) { - Py_FatalError("Out of memory"); - } - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)trash); - } - return trash; -} - -static void -_PyTrash_clear_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - assert(tstate->trash.delete_later == NULL); - return; - } - if (PyThread_tss_is_created(&_PyRuntime.trashTSSkey)) { - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash != NULL) { - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)NULL); - PyMem_RawFree(trash); - } - } -} - - -int -_PyTrash_begin(PyThreadState *tstate, PyObject *op) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - if (trash->delete_nesting >= _PyTrash_UNWIND_LEVEL) { - /* Store the object (to be deallocated later) and jump past - * Py_TRASHCAN_END, skipping the body of the deallocator */ - _PyTrash_thread_deposit_object(trash, op); - return 1; - } - ++trash->delete_nesting; - return 0; -} - - -void -_PyTrash_end(PyThreadState *tstate) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - --trash->delete_nesting; - if (trash->delete_nesting <= 0) { - if (trash->delete_later != NULL) { - _PyTrash_thread_destroy_chain(trash); - } - _PyTrash_clear_state(tstate); - } -} - - -/* bpo-40170: It's only be used in Py_TRASHCAN_BEGIN macro to hide - implementation details. */ -int -_PyTrash_cond(PyObject *op, destructor dealloc) -{ - return Py_TYPE(op)->tp_dealloc == dealloc; -} - - void _Py_NO_RETURN _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, const char *file, int line, const char *function) @@ -2725,6 +2839,12 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif @@ -2804,3 +2924,85 @@ int Py_IsFalse(PyObject *x) { return Py_Is(x, Py_False); } + + +// Py_SET_REFCNT() implementation for stable ABI +void +_Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) +{ + Py_SET_REFCNT(ob, refcnt); +} + +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; +} + + + +static PyObject* constants[] = { + &_Py_NoneStruct, // Py_CONSTANT_NONE + (PyObject*)(&_Py_FalseStruct), // Py_CONSTANT_FALSE + (PyObject*)(&_Py_TrueStruct), // Py_CONSTANT_TRUE + &_Py_EllipsisObject, // Py_CONSTANT_ELLIPSIS + &_Py_NotImplementedStruct, // Py_CONSTANT_NOT_IMPLEMENTED + NULL, // Py_CONSTANT_ZERO + NULL, // Py_CONSTANT_ONE + NULL, // Py_CONSTANT_EMPTY_STR + NULL, // Py_CONSTANT_EMPTY_BYTES + NULL, // Py_CONSTANT_EMPTY_TUPLE +}; + +void +_Py_GetConstant_Init(void) +{ + constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); + constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); +#ifndef NDEBUG + for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { + assert(constants[i] != NULL); + assert(_Py_IsImmortal(constants[i])); + } +#endif +} + +PyObject* +Py_GetConstant(unsigned int constant_id) +{ + if (constant_id < Py_ARRAY_LENGTH(constants)) { + return constants[constant_id]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + + +PyObject* +Py_GetConstantBorrowed(unsigned int constant_id) +{ + // All constants are immortal + return Py_GetConstant(constant_id); +} + + +// Py_TYPE() implementation for the stable ABI +#undef Py_TYPE +PyTypeObject* Py_TYPE(PyObject *ob) +{ + return _Py_TYPE(ob); +} diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 3f7d72eb22f224..352409425ee802 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -7,7 +7,7 @@ Each Python object starts with two fields: * ob_refcnt * ob_type -which the form the header common to all Python objects, for all versions, +which form the header common to all Python objects, for all versions, and hold the reference count and class of the object, respectively. ## Pre-header @@ -16,7 +16,45 @@ Since the introduction of the cycle GC, there has also been a pre-header. Before 3.11, this pre-header was two words in size. It should be considered opaque to all code except the cycle GC. -## 3.11 pre-header +### 3.13 + +In 3.13, the values array is embedded into the object, so there is no +need for a values pointer (it is just a fixed offset into the object). +So the pre-header is these two fields: + +* weakreflist +* dict_pointer + +If the object has no physical dictionary, then the ``dict_pointer`` +is set to `NULL`. + + +
    + 3.12 + +### 3.12 + +In 3.12, the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictionary, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. +
    + +
    + 3.11 + +### 3.11 In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. The reason for moving the ``__dict__`` to the pre-header is that it allows @@ -33,27 +71,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the values of the objects's attributes. Should the dictionary be needed, then ``values`` is set to ``NULL`` and the ``dict`` field points to the dictionary. +
    -## 3.12 pre-header +## Layout of a "normal" Python object -In 3.12 the pointer to the list of weak references is added to the -pre-header. In order to make space for it, the ``dict`` and ``values`` -pointers are combined into a single tagged pointer: +A "normal" Python object is one that doesn't inherit from a builtin +class, doesn't have slots. + +### 3.13 + +In 3.13 the values are embedded into the object, as follows: * weakreflist * dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type +* Inlined values: + * Flags + * values 0 + * values 1 + * ... + * Insertion order bytes -If the object has no physical dictionary, then the ``dict_or_values`` -has its low bit set to one, and points to the values array. -If the object has a physical dictionary, then the ``dict_or_values`` -has its low bit set to zero, and points to the dictionary. +This has all the advantages of the layout used in 3.12, plus: +* Access to values is even faster as there is one less load +* Fast access is mostly maintained when the `__dict__` is materialized -The untagged form is chosen for the dictionary pointer, rather than -the values pointer, to enable the (legacy) C-API function -`_PyObject_GetDictPtr(PyObject *obj)` to work. +![Layout of "normal" object in 3.13](./object_layout_313.png) + +For objects with opaque parts defined by a C extension, +the layout is much the same as for 3.12 +![Layout of "full" object in 3.13](./object_layout_full_313.png) -## Layout of a "normal" Python object in 3.12: + +
    + 3.12 + +### 3.12: + +In 3.12, the header and pre-header form the entire object for "normal" +Python objects: * weakreflist * dict_or_values @@ -62,9 +122,6 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, that is one that doesn't inherit from a builtin -class or have slots, the header and pre-header form the entire object. - ![Layout of "normal" object in 3.12](./object_layout_312.png) There are several advantages to this layout: @@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension, and `__slots__` looks like this: ![Layout of "full" object in 3.12](./object_layout_full_312.png) +
    + diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv index c0068d78568524..731a25332b3ace 100644 --- a/Objects/object_layout_312.gv +++ b/Objects/object_layout_312.gv @@ -20,6 +20,7 @@ digraph ideal { shape = none label = < + diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index 396dab183b3e9b..a63d095ea0b19e 100644 Binary files a/Objects/object_layout_312.png and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv new file mode 100644 index 00000000000000..d85352876f273b --- /dev/null +++ b/Objects/object_layout_313.gv @@ -0,0 +1,45 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <
    values
    Insertion order
    values[0]
    values[1]
    ...
    + + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    values flags
    values[0]
    values[1]
    ...
    Insertion order
    > + + ] + + class [ + shape = none + label = < + + + + + +
    class
    ...
    dict_offset
    ...
    cached_keys
    > + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> NULL + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png new file mode 100644 index 00000000000000..f7059c286c84e6 Binary files /dev/null and b/Objects/object_layout_313.png differ diff --git a/Objects/object_layout_full_313.gv b/Objects/object_layout_full_313.gv new file mode 100644 index 00000000000000..551312d51a7012 --- /dev/null +++ b/Objects/object_layout_full_313.gv @@ -0,0 +1,25 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = < + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    opaque (extension) data
    ...
    __slot__ 0
    ...
    > + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png new file mode 100644 index 00000000000000..7352ec69e5d8db Binary files /dev/null and b/Objects/object_layout_full_313.png differ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7d552ff57c8f1e..a6a71802ef8e01 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -7,9 +7,25 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" #include "pycore_pystate.h" // _PyInterpreterState_GET +#include "pycore_obmalloc_init.h" #include // malloc() #include +#ifdef WITH_MIMALLOC +// Forward declarations of functions used in our mimalloc modifications +static void _PyMem_mi_page_clear_qsbr(mi_page_t *page); +static bool _PyMem_mi_page_is_safe_to_free(mi_page_t *page); +static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force); +static void _PyMem_mi_page_reclaimed(mi_page_t *page); +static void _PyMem_mi_heap_collect_qsbr(mi_heap_t *heap); +# include "pycore_mimalloc.h" +# include "mimalloc/static.c" +# include "mimalloc/internal.h" // for stats +#endif + +#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC) +# error "Py_GIL_DISABLED requires WITH_MIMALLOC" +#endif #undef uint #define uint pymem_uint @@ -74,25 +90,238 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) free(ptr); } +#ifdef WITH_MIMALLOC + +static void +_PyMem_mi_page_clear_qsbr(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + // Clear the QSBR goal and remove the page from the QSBR linked list. + page->qsbr_goal = 0; + if (page->qsbr_node.next != NULL) { + llist_remove(&page->qsbr_node); + } +#endif +} + +// Check if an empty, newly reclaimed page is safe to free now. +static bool +_PyMem_mi_page_is_safe_to_free(mi_page_t *page) +{ + assert(mi_page_all_free(page)); +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->use_qsbr && page->qsbr_goal != 0) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (tstate == NULL) { + return false; + } + return _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal); + } +#endif + return true; + +} + +static bool +_PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) +{ +#ifdef Py_GIL_DISABLED + assert(mi_page_all_free(page)); + if (page->use_qsbr) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, pq, force); + return true; + } + + _PyMem_mi_page_clear_qsbr(page); + page->retire_expire = 0; + page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + return false; + } +#endif + _mi_page_free(page, pq, force); + return true; +} + +static void +_PyMem_mi_page_reclaimed(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->qsbr_goal != 0) { + if (mi_page_all_free(page)) { + assert(page->qsbr_node.next == NULL); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + page->retire_expire = 0; + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + } + else { + page->qsbr_goal = 0; + } + } +#endif +} + +static void +_PyMem_mi_heap_collect_qsbr(mi_heap_t *heap) +{ +#ifdef Py_GIL_DISABLED + if (!heap->page_use_qsbr) { + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mimalloc.page_list; + if (llist_empty(head)) { + return; + } + + struct llist_node *node; + llist_for_each_safe(node, head) { + mi_page_t *page = llist_data(node, mi_page_t, qsbr_node); + if (!mi_page_all_free(page)) { + // We allocated from this page some point after the delayed free + _PyMem_mi_page_clear_qsbr(page); + continue; + } + + if (!_Py_qsbr_poll(tstate->qsbr, page->qsbr_goal)) { + return; + } + + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, mi_page_queue_of(page), false); + } +#endif +} + +void * +_PyMem_MiMalloc(void *ctx, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_malloc(heap, size); +#else + return mi_malloc(size); +#endif +} + +void * +_PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_calloc(heap, nelem, elsize); +#else + return mi_calloc(nelem, elsize); +#endif +} + +void * +_PyMem_MiRealloc(void *ctx, void *ptr, size_t size) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_realloc(heap, ptr, size); +#else + return mi_realloc(ptr, size); +#endif +} + +void +_PyMem_MiFree(void *ctx, void *ptr) +{ + mi_free(ptr); +} + +void * +_PyObject_MiMalloc(void *ctx, size_t nbytes) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_malloc(heap, nbytes); +#else + return mi_malloc(nbytes); +#endif +} + +void * +_PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_calloc(heap, nelem, elsize); +#else + return mi_calloc(nelem, elsize); +#endif +} + + +void * +_PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_realloc(heap, ptr, nbytes); +#else + return mi_realloc(ptr, nbytes); +#endif +} + +void +_PyObject_MiFree(void *ctx, void *ptr) +{ + mi_free(ptr); +} + +#endif // WITH_MIMALLOC + + #define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} -#define PYRAW_ALLOC MALLOC_ALLOC -/* the default object allocator */ + +#ifdef WITH_MIMALLOC +# define MIMALLOC_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +# define MIMALLOC_OBJALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree} +#endif + +/* the pymalloc allocator */ // The actual implementation is further down. -#ifdef WITH_PYMALLOC +#if defined(WITH_PYMALLOC) void* _PyObject_Malloc(void *ctx, size_t size); void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize); void _PyObject_Free(void *ctx, void *p); void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); # define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} +#endif // WITH_PYMALLOC + +#if defined(Py_GIL_DISABLED) +// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains. +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC MIMALLOC_ALLOC +# define PYOBJ_ALLOC MIMALLOC_OBJALLOC +#elif defined(WITH_PYMALLOC) +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC PYMALLOC_ALLOC # define PYOBJ_ALLOC PYMALLOC_ALLOC #else +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC MALLOC_ALLOC # define PYOBJ_ALLOC MALLOC_ALLOC -#endif // WITH_PYMALLOC +#endif -#define PYMEM_ALLOC PYOBJ_ALLOC /* the default debug allocators */ @@ -157,8 +386,16 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, ) { #ifdef MS_WINDOWS + /* Unlike free(), VirtualFree() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } VirtualFree(ptr, 0, MEM_RELEASE); #elif defined(ARENAS_USE_MMAP) + /* Unlike free(), munmap() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } munmap(ptr, size); #else free(ptr); @@ -259,13 +496,9 @@ int _PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *old_alloc) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must be initializing. */ - return set_default_allocator_unlocked(domain, pydebug, old_alloc); - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); int res = set_default_allocator_unlocked(domain, pydebug, old_alloc); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return res; } @@ -284,7 +517,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) else if (strcmp(name, "debug") == 0) { *allocator = PYMEM_ALLOCATOR_DEBUG; } -#ifdef WITH_PYMALLOC +#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED) else if (strcmp(name, "pymalloc") == 0) { *allocator = PYMEM_ALLOCATOR_PYMALLOC; } @@ -292,12 +525,22 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG; } #endif +#ifdef WITH_MIMALLOC + else if (strcmp(name, "mimalloc") == 0) { + *allocator = PYMEM_ALLOCATOR_MIMALLOC; + } + else if (strcmp(name, "mimalloc_debug") == 0) { + *allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG; + } +#endif +#ifndef Py_GIL_DISABLED else if (strcmp(name, "malloc") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC; } else if (strcmp(name, "malloc_debug") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; } +#endif else { /* unknown allocator */ return -1; @@ -318,12 +561,14 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) (void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, pydebug, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, pydebug, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, pydebug, NULL); + _PyRuntime.allocators.is_debug_enabled = pydebug; break; case PYMEM_ALLOCATOR_DEBUG: (void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, 1, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, 1, NULL); (void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, 1, NULL); + _PyRuntime.allocators.is_debug_enabled = 1; break; #ifdef WITH_PYMALLOC @@ -337,12 +582,36 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc); set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &pymalloc); - if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) { + int is_debug = (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { set_up_debug_hooks_unlocked(); } break; } #endif +#ifdef WITH_MIMALLOC + case PYMEM_ALLOCATOR_MIMALLOC: + case PYMEM_ALLOCATOR_MIMALLOC_DEBUG: + { + PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; + set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc); + + PyMemAllocatorEx pymalloc = MIMALLOC_ALLOC; + set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc); + + PyMemAllocatorEx objmalloc = MIMALLOC_OBJALLOC; + set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &objmalloc); + + int is_debug = (allocator == PYMEM_ALLOCATOR_MIMALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { + set_up_debug_hooks_unlocked(); + } + + break; + } +#endif case PYMEM_ALLOCATOR_MALLOC: case PYMEM_ALLOCATOR_MALLOC_DEBUG: @@ -352,7 +621,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) set_allocator_unlocked(PYMEM_DOMAIN_MEM, &malloc_alloc); set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &malloc_alloc); - if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) { + int is_debug = (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG); + _PyRuntime.allocators.is_debug_enabled = is_debug; + if (is_debug) { set_up_debug_hooks_unlocked(); } break; @@ -369,9 +640,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) int _PyMem_SetupAllocators(PyMemAllocatorName allocator) { - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); int res = set_up_allocators_unlocked(allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return res; } @@ -390,6 +661,10 @@ get_current_allocator_name_unlocked(void) #ifdef WITH_PYMALLOC PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC; #endif +#ifdef WITH_MIMALLOC + PyMemAllocatorEx mimalloc = MIMALLOC_ALLOC; + PyMemAllocatorEx mimalloc_obj = MIMALLOC_OBJALLOC; +#endif if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && pymemallocator_eq(&_PyMem, &malloc_alloc) && @@ -405,6 +680,14 @@ get_current_allocator_name_unlocked(void) return "pymalloc"; } #endif +#ifdef WITH_MIMALLOC + if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && + pymemallocator_eq(&_PyMem, &mimalloc) && + pymemallocator_eq(&_PyObject, &mimalloc_obj)) + { + return "mimalloc"; + } +#endif PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC; PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC; @@ -428,6 +711,14 @@ get_current_allocator_name_unlocked(void) { return "pymalloc_debug"; } +#endif +#ifdef WITH_MIMALLOC + if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) && + pymemallocator_eq(&_PyMem_Debug.mem.alloc, &mimalloc) && + pymemallocator_eq(&_PyMem_Debug.obj.alloc, &mimalloc_obj)) + { + return "mimalloc_debug"; + } #endif } return NULL; @@ -436,20 +727,20 @@ get_current_allocator_name_unlocked(void) const char* _PyMem_GetCurrentAllocatorName(void) { - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); const char *name = get_current_allocator_name_unlocked(); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return name; } -#ifdef WITH_PYMALLOC -static int +int _PyMem_DebugEnabled(void) { - return (_PyObject.malloc == _PyMem_DebugMalloc); + return _PyRuntime.allocators.is_debug_enabled; } +#ifdef WITH_PYMALLOC static int _PyMem_PymallocEnabled(void) { @@ -460,7 +751,25 @@ _PyMem_PymallocEnabled(void) return (_PyObject.malloc == _PyObject_Malloc); } } + +#ifdef WITH_MIMALLOC +static int +_PyMem_MimallocEnabled(void) +{ +#ifdef Py_GIL_DISABLED + return 1; +#else + if (_PyMem_DebugEnabled()) { + return (_PyMem_Debug.obj.alloc.malloc == _PyObject_MiMalloc); + } + else { + return (_PyObject.malloc == _PyObject_MiMalloc); + } #endif +} +#endif // WITH_MIMALLOC + +#endif // WITH_PYMALLOC static void @@ -516,19 +825,15 @@ set_up_debug_hooks_unlocked(void) set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_RAW); set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_MEM); set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_OBJ); + _PyRuntime.allocators.is_debug_enabled = 1; } void PyMem_SetupDebugHooks(void) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - set_up_debug_hooks_unlocked(); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); set_up_debug_hooks_unlocked(); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } static void @@ -564,53 +869,33 @@ set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - get_allocator_unlocked(domain, allocator); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); get_allocator_unlocked(domain, allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - set_allocator_unlocked(domain, allocator); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); set_allocator_unlocked(domain, allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - *allocator = _PyObject_Arena; - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); *allocator = _PyObject_Arena; - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - _PyObject_Arena = *allocator; - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); _PyObject_Arena = *allocator; - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } @@ -784,6 +1069,221 @@ _PyMem_Strdup(const char *str) return copy; } +/***********************************************/ +/* Delayed freeing support for Py_GIL_DISABLED */ +/***********************************************/ + +// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms. +#define WORK_ITEMS_PER_CHUNK 254 + +// A pointer to be freed once the QSBR read sequence reaches qsbr_goal. +struct _mem_work_item { + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free + uint64_t qsbr_goal; +}; + +// A fixed-size buffer of pointers to be freed +struct _mem_work_chunk { + // Linked list node of chunks in queue + struct llist_node node; + + Py_ssize_t rd_idx; // index of next item to read + Py_ssize_t wr_idx; // index of next item to write + struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; +}; + +static void +free_work_item(uintptr_t ptr) +{ + if (ptr & 0x01) { + PyObject_Free((char *)(ptr - 1)); + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) +{ +#ifndef Py_GIL_DISABLED + free_work_item(ptr); +#else + if (_PyRuntime.stoptheworld.world_stopped) { + // Free immediately if the world is stopped, including during + // interpreter shutdown. + free_work_item(ptr); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mem_free_queue; + + struct _mem_work_chunk *buf = NULL; + if (!llist_empty(head)) { + // Try to re-use the last buffer + buf = llist_data(head->prev, struct _mem_work_chunk, node); + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // already full + buf = NULL; + } + } + + if (buf == NULL) { + buf = PyMem_Calloc(1, sizeof(*buf)); + if (buf != NULL) { + llist_insert_tail(head, &buf->node); + } + } + + if (buf == NULL) { + // failed to allocate a buffer, free immediately + _PyEval_StopTheWorld(tstate->base.interp); + free_work_item(ptr); + _PyEval_StartTheWorld(tstate->base.interp); + return; + } + + assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); + uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + buf->array[buf->wr_idx].ptr = ptr; + buf->array[buf->wr_idx].qsbr_goal = seq; + buf->wr_idx++; + + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + _PyMem_ProcessDelayed((PyThreadState *)tstate); + } +#endif +} + +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +void +_PyObject_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed(((uintptr_t)ptr)|0x01); +} + +static struct _mem_work_chunk * +work_queue_first(struct llist_node *head) +{ + return llist_data(head->next, struct _mem_work_chunk, node); +} + +static void +process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, + bool keep_empty) +{ + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + return; + } + + free_work_item(item->ptr); + buf->rd_idx++; + } + + assert(buf->rd_idx == buf->wr_idx); + if (keep_empty && buf->node.next == head) { + // Keep the last buffer in the queue to reduce re-allocations + buf->rd_idx = buf->wr_idx = 0; + return; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} + +static void +process_interp_queue(struct _Py_mem_interp_free_queue *queue, + struct _qsbr_thread_state *qsbr) +{ + if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { + return; + } + + // Try to acquire the lock, but don't block if it's already held. + if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { + process_queue(&queue->head, qsbr, false); + + int more_work = !llist_empty(&queue->head); + _Py_atomic_store_int_relaxed(&queue->has_work, more_work); + + PyMutex_Unlock(&queue->mutex); + } +} + +void +_PyMem_ProcessDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + + // Process thread-local work + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true); + + // Process shared interpreter work + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr); +} + +void +_PyMem_AbandonDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue; + + if (llist_empty(queue)) { + return; + } + + // Check if the queue contains one empty buffer + struct _mem_work_chunk *buf = work_queue_first(queue); + if (buf->rd_idx == buf->wr_idx) { + llist_remove(&buf->node); + PyMem_Free(buf); + assert(llist_empty(queue)); + return; + } + + // Merge the thread's work queue into the interpreter's work queue. + PyMutex_Lock(&interp->mem_free_queue.mutex); + llist_concat(&interp->mem_free_queue.head, queue); + _Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1); + PyMutex_Unlock(&interp->mem_free_queue.mutex); + + assert(llist_empty(queue)); // the thread's queue is now empty +} + +void +_PyMem_FiniDelayed(PyInterpreterState *interp) +{ + struct llist_node *head = &interp->mem_free_queue.head; + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + // Free the remaining items immediately. There should be no other + // threads accessing the memory at this point during shutdown. + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + free_work_item(item->ptr); + buf->rd_idx++; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} /**************************/ /* the "object" allocator */ @@ -853,6 +1353,13 @@ static int running_on_valgrind = -1; typedef struct _obmalloc_state OMState; +/* obmalloc state for main interpreter and shared by all interpreters without + * their own obmalloc state. By not explicitly initializing this structure, it + * will be allocated in the BSS which is a small performance win. The radix + * tree arrays are fairly large but are sparsely used. */ +static struct _obmalloc_state obmalloc_state_main; +static bool obmalloc_state_initialized; + static inline int has_own_state(PyInterpreterState *interp) { @@ -865,10 +1372,8 @@ static inline OMState * get_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_own_state(interp)) { - interp = _PyInterpreterState_Main(); - } - return &interp->obmalloc; + assert(interp->obmalloc != NULL); // otherwise not initialized or freed + return interp->obmalloc; } // These macros all rely on a local "state" variable. @@ -883,9 +1388,51 @@ get_state(void) #define narenas_highwater (state->mgmt.narenas_highwater) #define raw_allocated_blocks (state->mgmt.raw_allocated_blocks) +#ifdef WITH_MIMALLOC +static bool count_blocks( + const mi_heap_t* heap, const mi_heap_area_t* area, + void* block, size_t block_size, void* allocated_blocks) +{ + *(size_t *)allocated_blocks += area->used; + return 1; +} + +static Py_ssize_t +get_mimalloc_allocated_blocks(PyInterpreterState *interp) +{ + size_t allocated_blocks = 0; +#ifdef Py_GIL_DISABLED + for (PyThreadState *t = interp->threads.head; t != NULL; t = t->next) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)t; + for (int i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + mi_heap_t *heap = &tstate->mimalloc.heaps[i]; + mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); + } + } + + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + for (uint8_t tag = 0; tag < _Py_MIMALLOC_HEAP_COUNT; tag++) { + _mi_abandoned_pool_visit_blocks(pool, tag, false, &count_blocks, + &allocated_blocks); + } +#else + // TODO(sgross): this only counts the current thread's blocks. + mi_heap_t *heap = mi_heap_get_default(); + mi_heap_visit_blocks(heap, false, &count_blocks, &allocated_blocks); +#endif + return allocated_blocks; +} +#endif + Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) { +#ifdef WITH_MIMALLOC + if (_PyMem_MimallocEnabled()) { + return get_mimalloc_allocated_blocks(interp); + } +#endif + #ifdef Py_DEBUG assert(has_own_state(interp)); #else @@ -894,7 +1441,11 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) "the interpreter doesn't have its own allocator"); } #endif - OMState *state = &interp->obmalloc; + OMState *state = interp->obmalloc; + + if (state == NULL) { + return 0; + } Py_ssize_t n = raw_allocated_blocks; /* add up allocated blocks for used pools */ @@ -916,19 +1467,36 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp) return n; } +static void free_obmalloc_arenas(PyInterpreterState *interp); + void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { - if (has_own_state(interp)) { +#ifdef WITH_MIMALLOC + if (_PyMem_MimallocEnabled()) { + return; + } +#endif + if (has_own_state(interp) && interp->obmalloc != NULL) { Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); assert(has_own_state(interp) || leaked == 0); interp->runtime->obmalloc.interpreter_leaks += leaked; + if (_PyMem_obmalloc_state_on_heap(interp) && leaked == 0) { + // free the obmalloc arenas and radix tree nodes. If leaked > 0 + // then some of the memory allocated by obmalloc has not been + // freed. It might be safe to free the arenas in that case but + // it's possible that extension modules are still using that + // memory. So, it is safer to not free and to leak. Perhaps there + // should be warning when this happens. It should be possible to + // use a tool like "-fsanitize=address" to track down these leaks. + free_obmalloc_arenas(interp); + } } } static Py_ssize_t get_num_global_allocated_blocks(_PyRuntimeState *); -/* We preserve the number of blockss leaked during runtime finalization, +/* We preserve the number of blocks leaked during runtime finalization, so they can be reported if the runtime is initialized again. */ // XXX We don't lose any information by dropping this, // so we should consider doing so. @@ -962,6 +1530,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } else { + _PyEval_StopTheWorldAll(&_PyRuntime); HEAD_LOCK(runtime); PyInterpreterState *interp = PyInterpreterState_Head(); assert(interp != NULL); @@ -981,6 +1550,7 @@ get_num_global_allocated_blocks(_PyRuntimeState *runtime) } } HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(&_PyRuntime); #ifdef Py_DEBUG assert(got_main); #endif @@ -2035,6 +2605,33 @@ write_size_t(void *p, size_t n) } } +static void +fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes, + bool is_alloc) +{ +#ifdef Py_GIL_DISABLED + if (api->api_id == 'o') { + // Don't overwrite the first few bytes of a PyObject allocation in the + // free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + size_t debug_offset; + if (is_alloc) { + debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + } + else { + char *alloc = (char *)data - 2*SST; // start of the allocation + debug_offset = _mi_ptr_page(alloc)->debug_offset; + } + debug_offset -= 2*SST; // account for pymalloc extra bytes + if (debug_offset < nbytes) { + memset((char *)data + debug_offset, c, nbytes - debug_offset); + } + return; + } +#endif + memset(data, c, nbytes); +} + /* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2111,7 +2708,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); + fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes, true); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ @@ -2159,8 +2756,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(__func__, api->api_id, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); + nbytes += PYMEM_DEBUG_EXTRA_BYTES - 2*SST; + memset(q, PYMEM_DEADBYTE, 2*SST); + fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes, false); api->alloc.free(api->alloc.ctx, q); } @@ -2180,7 +2778,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; #define ERASED_SIZE 64 - uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ _PyMem_DebugCheckAddress(__func__, api->api_id, p); @@ -2197,9 +2794,11 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) #ifdef PYMEM_DEBUG_SERIALNO size_t block_serialno = read_size_t(tail + SST); #endif +#ifndef Py_GIL_DISABLED /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ + uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); memset(data - 2 * SST, PYMEM_DEADBYTE, @@ -2212,6 +2811,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } +#endif /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); @@ -2239,6 +2839,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) write_size_t(tail + SST, block_serialno); #endif +#ifndef Py_GIL_DISABLED /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(data, save, Py_MIN(nbytes, original_nbytes)); @@ -2251,6 +2852,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) Py_MIN(nbytes - i, ERASED_SIZE)); } } +#endif if (r == NULL) { return NULL; @@ -2512,8 +3114,95 @@ _PyDebugAllocatorStats(FILE *out, (void)printone(out, buf2, num_blocks * sizeof_block); } +// Return true if the obmalloc state structure is heap allocated, +// by PyMem_RawCalloc(). For the main interpreter, this structure +// allocated in the BSS. Allocating that way gives some memory savings +// and a small performance win (at least on a demand paged OS). On +// 64-bit platforms, the obmalloc structure is 256 kB. Most of that +// memory is for the arena_map_top array. Since normally only one entry +// of that array is used, only one page of resident memory is actually +// used, rather than the full 256 kB. +bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp) +{ +#if WITH_PYMALLOC + return interp->obmalloc && interp->obmalloc != &obmalloc_state_main; +#else + return false; +#endif +} #ifdef WITH_PYMALLOC +static void +init_obmalloc_pools(PyInterpreterState *interp) +{ + // initialize the obmalloc->pools structure. This must be done + // before the obmalloc alloc/free functions can be called. + poolp temp[OBMALLOC_USED_POOLS_SIZE] = + _obmalloc_pools_INIT(interp->obmalloc->pools); + memcpy(&interp->obmalloc->pools.used, temp, sizeof(temp)); +} +#endif /* WITH_PYMALLOC */ + +int _PyMem_init_obmalloc(PyInterpreterState *interp) +{ +#ifdef WITH_PYMALLOC + /* Initialize obmalloc, but only for subinterpreters, + since the main interpreter is initialized statically. */ + if (_Py_IsMainInterpreter(interp) + || _PyInterpreterState_HasFeature(interp, + Py_RTFLAGS_USE_MAIN_OBMALLOC)) { + interp->obmalloc = &obmalloc_state_main; + if (!obmalloc_state_initialized) { + init_obmalloc_pools(interp); + obmalloc_state_initialized = true; + } + } else { + interp->obmalloc = PyMem_RawCalloc(1, sizeof(struct _obmalloc_state)); + if (interp->obmalloc == NULL) { + return -1; + } + init_obmalloc_pools(interp); + } +#endif /* WITH_PYMALLOC */ + return 0; // success +} + + +#ifdef WITH_PYMALLOC + +static void +free_obmalloc_arenas(PyInterpreterState *interp) +{ + OMState *state = interp->obmalloc; + for (uint i = 0; i < maxarenas; ++i) { + // free each obmalloc memory arena + struct arena_object *ao = &allarenas[i]; + _PyObject_Arena.free(_PyObject_Arena.ctx, + (void *)ao->address, ARENA_SIZE); + } + // free the array containing pointers to all arenas + PyMem_RawFree(allarenas); +#if WITH_PYMALLOC_RADIX_TREE +#ifdef USE_INTERIOR_NODES + // Free the middle and bottom nodes of the radix tree. These are allocated + // by arena_map_mark_used() but not freed when arenas are freed. + for (int i1 = 0; i1 < MAP_TOP_LENGTH; i1++) { + arena_map_mid_t *mid = arena_map_root.ptrs[i1]; + if (mid == NULL) { + continue; + } + for (int i2 = 0; i2 < MAP_MID_LENGTH; i2++) { + arena_map_bot_t *bot = arena_map_root.ptrs[i1]->ptrs[i2]; + if (bot == NULL) { + continue; + } + PyMem_RawFree(bot); + } + PyMem_RawFree(mid); + } +#endif +#endif +} #ifdef Py_DEBUG /* Is target in the list? The list is traversed via the nextpool pointers. @@ -2536,19 +3225,55 @@ pool_is_in_list(const poolp target, poolp list) } #endif -/* Print summary info to "out" about the state of pymalloc's structures. - * In Py_DEBUG mode, also perform some expensive internal consistency - * checks. - * - * Return 0 if the memory debug hooks are not installed or no statistics was - * written into out, return 1 otherwise. - */ -int -_PyObject_DebugMallocStats(FILE *out) +#ifdef WITH_MIMALLOC +struct _alloc_stats { + size_t allocated_blocks; + size_t allocated_bytes; + size_t allocated_with_overhead; + size_t bytes_reserved; + size_t bytes_committed; +}; + +static bool _collect_alloc_stats( + const mi_heap_t* heap, const mi_heap_area_t* area, + void* block, size_t block_size, void* arg) +{ + struct _alloc_stats *stats = (struct _alloc_stats *)arg; + stats->allocated_blocks += area->used; + stats->allocated_bytes += area->used * area->block_size; + stats->allocated_with_overhead += area->used * area->full_block_size; + stats->bytes_reserved += area->reserved; + stats->bytes_committed += area->committed; + return 1; +} + +static void +py_mimalloc_print_stats(FILE *out) +{ + fprintf(out, "Small block threshold = %zd, in %u size classes.\n", + MI_SMALL_OBJ_SIZE_MAX, MI_BIN_HUGE); + fprintf(out, "Medium block threshold = %zd\n", + MI_MEDIUM_OBJ_SIZE_MAX); + fprintf(out, "Large object max size = %zd\n", + MI_LARGE_OBJ_SIZE_MAX); + + mi_heap_t *heap = mi_heap_get_default(); + struct _alloc_stats stats; + memset(&stats, 0, sizeof(stats)); + mi_heap_visit_blocks(heap, false, &_collect_alloc_stats, &stats); + + fprintf(out, " Allocated Blocks: %zd\n", stats.allocated_blocks); + fprintf(out, " Allocated Bytes: %zd\n", stats.allocated_bytes); + fprintf(out, " Allocated Bytes w/ Overhead: %zd\n", stats.allocated_with_overhead); + fprintf(out, " Bytes Reserved: %zd\n", stats.bytes_reserved); + fprintf(out, " Bytes Committed: %zd\n", stats.bytes_committed); +} +#endif + + +static void +pymalloc_print_stats(FILE *out) { - if (!_PyMem_PymallocEnabled()) { - return 0; - } OMState *state = get_state(); uint i; @@ -2701,7 +3426,32 @@ _PyObject_DebugMallocStats(FILE *out) #endif #endif - return 1; +} + +/* Print summary info to "out" about the state of pymalloc's structures. + * In Py_DEBUG mode, also perform some expensive internal consistency + * checks. + * + * Return 0 if the memory debug hooks are not installed or no statistics was + * written into out, return 1 otherwise. + */ +int +_PyObject_DebugMallocStats(FILE *out) +{ +#ifdef WITH_MIMALLOC + if (_PyMem_MimallocEnabled()) { + py_mimalloc_print_stats(out); + return 1; + } + else +#endif + if (_PyMem_PymallocEnabled()) { + pymalloc_print_stats(out); + return 1; + } + else { + return 0; + } } #endif /* #ifdef WITH_PYMALLOC */ diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b99896319e0136..30277aa0c23883 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -200,7 +200,7 @@ Here are some ways to address this challenge: Adding the checks to the concrete API would help make any interpreter switch to OrderedDict less painful for extension modules. However, this won't work. The equivalent C API call to `dict.__setitem__(obj, k, v)` -is 'PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call +is `PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call the base class's methods, since there is no equivalent of super() in the C API. Calling into Python for parent class API would work, but some extension modules already rely on this feature of the concrete API. @@ -465,12 +465,13 @@ Potential Optimizations */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _Py_dict_lookup() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include // offsetof() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" //_Py_BEGIN_CRITICAL_SECTION +#include "pycore_dict.h" // _Py_dict_lookup() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include // offsetof() #include "clinic/odictobject.c.h" @@ -534,8 +535,12 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; - +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe((PyDictObject *)od, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup((PyDictObject *)od, key, hash, &value); +#endif if (ix == DKIX_EMPTY) { return keys->dk_nentries; /* index of new entry */ } @@ -1039,6 +1044,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, { PyObject *value = NULL; + Py_BEGIN_CRITICAL_SECTION(od); + _ODictNode *node = _odict_find_node_hash((PyODictObject *)od, key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1046,10 +1053,13 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, resolution. */ int res = _odict_clear_node((PyODictObject *)od, node, key, hash); if (res < 0) { - return NULL; + goto done; } /* Now delete the value from the dict. */ - value = _PyDict_Pop_KnownHash(od, key, hash, failobj); + if (_PyDict_Pop_KnownHash((PyDictObject *)od, key, hash, + &value) == 0) { + value = Py_NewRef(failobj); + } } else if (value == NULL && !PyErr_Occurred()) { /* Apply the fallback value, if necessary. */ @@ -1060,6 +1070,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } + Py_END_CRITICAL_SECTION(); +done: return value; } diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index ce9eef69ad75a8..9727b4f47b53a1 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -83,7 +83,7 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) switch (num_args) { case 3: step = args[2]; - /* fallthrough */ + _Py_FALLTHROUGH; case 2: /* Convert borrowed refs to owned refs */ start = PyNumber_Index(args[0]); @@ -655,7 +655,7 @@ range_index(rangeobject *r, PyObject *ob) } /* object is not in the range */ - PyErr_Format(PyExc_ValueError, "%R is not in range", ob); + PyErr_SetString(PyExc_ValueError, "range.index(x): x not in range"); return NULL; } @@ -751,7 +751,7 @@ PyDoc_STRVAR(index_doc, static PyMethodDef range_methods[] = { {"__reversed__", range_reverse, METH_NOARGS, reverse_doc}, - {"__reduce__", (PyCFunction)range_reduce, METH_VARARGS}, + {"__reduce__", (PyCFunction)range_reduce, METH_NOARGS}, {"count", (PyCFunction)range_count, METH_O, count_doc}, {"index", (PyCFunction)range_index, METH_O, index_doc}, {NULL, NULL} /* sentinel */ diff --git a/Objects/setobject.c b/Objects/setobject.c index 2a3514f2028c3d..5d7ad395d08c90 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2,7 +2,7 @@ /* set object implementation Written and maintained by Raymond D. Hettinger - Derived from Lib/sets.py and Objects/dictobject.c. + Derived from Objects/dictobject.c. The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -32,13 +32,28 @@ */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_dict.h" // _PyDict_Contains_KnownHash() -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_SetKeyError() -#include "pycore_setobject.h" // _PySet_NextEntry() definition -#include // offsetof() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() +#include "pycore_pyerrors.h" // _PyErr_SetKeyError() +#include "pycore_setobject.h" // _PySet_NextEntry() definition +#include // offsetof() +#include "clinic/setobject.c.h" + +/*[clinic input] +class set "PySetObject *" "&PySet_Type" +class frozenset "PySetObject *" "&PyFrozenSet_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=97ad1d3e9f117079]*/ + +/*[python input] +class setobject_converter(self_converter): + type = "PySetObject *" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=33a44506d4d57793]*/ /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -116,6 +131,8 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + /* Pre-increment is necessary to prevent arbitrary code in the rich comparison from deallocating the key just before the insertion. */ Py_INCREF(key); @@ -167,14 +184,14 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); freeslot->key = key; freeslot->hash = hash; return 0; found_unused: so->fill++; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); entry->key = key; entry->hash = hash; if ((size_t)so->fill*5 < mask*3) @@ -340,7 +357,7 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) old_key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); Py_DECREF(old_key); return DISCARD_FOUND; } @@ -348,13 +365,9 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_add_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_add_entry(so, key, hash); } @@ -362,13 +375,9 @@ set_add_key(PySetObject *so, PyObject *key) static int set_contains_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_contains_entry(so, key, hash); } @@ -376,13 +385,9 @@ set_contains_key(PySetObject *so, PyObject *key) static int set_discard_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_discard_entry(so, key, hash); } @@ -392,7 +397,7 @@ set_empty_to_minsize(PySetObject *so) { memset(so->smalltable, 0, sizeof(so->smalltable)); so->fill = 0; - so->used = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, 0); so->mask = PySet_MINSIZE - 1; so->table = so->smalltable; so->hash = -1; @@ -509,7 +514,7 @@ set_dealloc(PySetObject *so) } static PyObject * -set_repr(PySetObject *so) +set_repr_lock_held(PySetObject *so) { PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); @@ -553,14 +558,24 @@ set_repr(PySetObject *so) return result; } +static PyObject * +set_repr(PySetObject *so) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(so); + result = set_repr_lock_held(so); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t -set_len(PyObject *so) +set_len(PySetObject *so) { - return ((PySetObject *)so)->used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); } static int -set_merge(PySetObject *so, PyObject *otherset) +set_merge_lock_held(PySetObject *so, PyObject *otherset) { PySetObject *other; PyObject *key; @@ -570,6 +585,8 @@ set_merge(PySetObject *so, PyObject *otherset) assert (PyAnySet_Check(so)); assert (PyAnySet_Check(otherset)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(otherset); other = (PySetObject*)otherset; if (other == so || other->used == 0) @@ -598,7 +615,7 @@ set_merge(PySetObject *so, PyObject *otherset) } } so->fill = other->fill; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); return 0; } @@ -607,7 +624,7 @@ set_merge(PySetObject *so, PyObject *otherset) setentry *newtable = so->table; size_t newmask = (size_t)so->mask; so->fill = other->used; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { @@ -630,8 +647,19 @@ set_merge(PySetObject *so, PyObject *otherset) return 0; } +/*[clinic input] +@critical_section +set.pop + so: setobject + +Remove and return an arbitrary set element. + +Raises KeyError if the set is empty. +[clinic start generated code]*/ + static PyObject * -set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_pop_impl(PySetObject *so) +/*[clinic end generated code: output=4d65180f1271871b input=9296c84921125060]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -650,14 +678,11 @@ set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); so->finger = entry - so->table + 1; /* next place to start */ return key; } -PyDoc_STRVAR(pop_doc, "Remove and return an arbitrary set element.\n\ -Raises KeyError if the set is empty."); - static int set_traverse(PySetObject *so, visitproc visit, void *arg) { @@ -797,7 +822,7 @@ static PyMethodDef setiter_methods[] = { static PyObject *setiter_iternext(setiterobject *si) { - PyObject *key; + PyObject *key = NULL; Py_ssize_t i, mask; setentry *entry; PySetObject *so = si->si_set; @@ -806,30 +831,35 @@ static PyObject *setiter_iternext(setiterobject *si) return NULL; assert (PyAnySet_Check(so)); - if (si->si_used != so->used) { + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE(so->used); + Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE(si->si_used); + if (si_used != so_used) { PyErr_SetString(PyExc_RuntimeError, "Set changed size during iteration"); si->si_used = -1; /* Make this state sticky */ return NULL; } + Py_BEGIN_CRITICAL_SECTION(so); i = si->si_pos; assert(i>=0); entry = so->table; mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { i++; + } + if (i <= mask) { + key = Py_NewRef(entry[i].key); + } + Py_END_CRITICAL_SECTION(); si->si_pos = i+1; - if (i > mask) - goto fail; + if (key == NULL) { + si->si_set = NULL; + Py_DECREF(so); + return NULL; + } si->len--; - key = entry[i].key; - return Py_NewRef(key); - -fail: - si->si_set = NULL; - Py_DECREF(so); - return NULL; + return key; } PyTypeObject PySetIter_Type = { @@ -868,52 +898,60 @@ PyTypeObject PySetIter_Type = { static PyObject * set_iter(PySetObject *so) { + Py_ssize_t size = set_len(so); setiterobject *si = PyObject_GC_New(setiterobject, &PySetIter_Type); if (si == NULL) return NULL; si->si_set = (PySetObject*)Py_NewRef(so); - si->si_used = so->used; + si->si_used = size; si->si_pos = 0; - si->len = so->used; + si->len = size; _PyObject_GC_TRACK(si); return (PyObject *)si; } static int -set_update_internal(PySetObject *so, PyObject *other) +set_update_dict_lock_held(PySetObject *so, PyObject *other) { - PyObject *key, *it; + assert(PyDict_CheckExact(other)); - if (PyAnySet_Check(other)) - return set_merge(so, other); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); - if (PyDict_CheckExact(other)) { - PyObject *value; - Py_ssize_t pos = 0; - Py_hash_t hash; - Py_ssize_t dictsize = PyDict_GET_SIZE(other); - - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new keys. Expect - * that there will be no (or few) overlapping keys. - */ - if (dictsize < 0) + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new keys. Expect + * that there will be no (or few) overlapping keys. + */ + Py_ssize_t dictsize = PyDict_GET_SIZE(other); + if ((so->fill + dictsize)*5 >= so->mask*3) { + if (set_table_resize(so, (so->used + dictsize)*2) != 0) { return -1; - if ((so->fill + dictsize)*5 >= so->mask*3) { - if (set_table_resize(so, (so->used + dictsize)*2) != 0) - return -1; } - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) - return -1; + } + + Py_ssize_t pos = 0; + PyObject *key; + PyObject *value; + Py_hash_t hash; + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + if (set_add_entry(so, key, hash)) { + return -1; } - return 0; } + return 0; +} - it = PyObject_GetIter(other); - if (it == NULL) +static int +set_update_iterable_lock_held(PySetObject *so, PyObject *other) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + + PyObject *it = PyObject_GetIter(other); + if (it == NULL) { return -1; + } + PyObject *key; while ((key = PyIter_Next(it)) != NULL) { if (set_add_key(so, key)) { Py_DECREF(it); @@ -928,8 +966,81 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +static int +set_update_lock_held(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + return set_merge_lock_held(so, other); + } + else if (PyDict_CheckExact(other)) { + return set_update_dict_lock_held(so, other); + } + return set_update_iterable_lock_held(so, other); +} + +// set_update for a `so` that is only visible to the current thread +static int +set_update_local(PySetObject *so, PyObject *other) +{ + assert(Py_REFCNT(so) == 1); + if (PyAnySet_Check(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + return set_update_iterable_lock_held(so, other); +} + +static int +set_update_internal(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + if (Py_Is((PyObject *)so, other)) { + return 0; + } + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else { + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_update_iterable_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } +} + +/*[clinic input] +set.update + so: setobject + *others as args: object + / + +Update the set, adding elements from all others. +[clinic start generated code]*/ + static PyObject * -set_update(PySetObject *so, PyObject *args) +set_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ { Py_ssize_t i; @@ -941,9 +1052,6 @@ set_update(PySetObject *so, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(update_doc, -"Update a set with the union of itself and others."); - /* XXX Todo: If aligned memory allocations become available, make the set object 64 byte aligned so that most of the fields @@ -969,7 +1077,7 @@ make_new_set(PyTypeObject *type, PyObject *iterable) so->weakreflist = NULL; if (iterable != NULL) { - if (set_update_internal(so, iterable)) { + if (set_update_local(so, iterable)) { Py_DECREF(so); return NULL; } @@ -1065,7 +1173,9 @@ set_swap_bodies(PySetObject *a, PySetObject *b) Py_hash_t h; t = a->fill; a->fill = b->fill; b->fill = t; - t = a->used; a->used = b->used; b->used = t; + t = a->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(a->used, b->used); + FT_ATOMIC_STORE_SSIZE_RELAXED(b->used, t); t = a->mask; a->mask = b->mask; b->mask = t; u = a->table; @@ -1091,34 +1201,76 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } } +/*[clinic input] +@critical_section +set.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_copy_impl(PySetObject *so) +/*[clinic end generated code: output=c9223a1e1cc6b041 input=c169a4fbb8209257]*/ { - return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + PyObject *copy = make_new_set_basetype(Py_TYPE(so), NULL); + if (copy == NULL) { + return NULL; + } + if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { + Py_DECREF(copy); + return NULL; + } + return copy; } +/*[clinic input] +@critical_section +frozenset.copy + so: setobject + +Return a shallow copy of a set. +[clinic start generated code]*/ + static PyObject * -frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +frozenset_copy_impl(PySetObject *so) +/*[clinic end generated code: output=b356263526af9e70 input=fbf5bef131268dd7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); } - return set_copy(so, NULL); + return set_copy_impl(so); } -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a set."); +/*[clinic input] +@critical_section +set.clear + so: setobject + +Remove all elements from this set. +[clinic start generated code]*/ static PyObject * -set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_clear_impl(PySetObject *so) +/*[clinic end generated code: output=4e71d5a83904161a input=c6f831b366111950]*/ { set_clear_internal(so); Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "Remove all elements from this set."); +/*[clinic input] +set.union + so: setobject + *others as args: object + / + +Return a new set with elements from the set and all others. +[clinic start generated code]*/ static PyObject * -set_union(PySetObject *so, PyObject *args) +set_union_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ { PySetObject *result; PyObject *other; @@ -1132,7 +1284,7 @@ set_union(PySetObject *so, PyObject *args) other = PyTuple_GET_ITEM(args, i); if ((PyObject *)so == other) continue; - if (set_update_internal(result, other)) { + if (set_update_local(result, other)) { Py_DECREF(result); return NULL; } @@ -1140,11 +1292,6 @@ set_union(PySetObject *so, PyObject *args) return (PyObject *)result; } -PyDoc_STRVAR(union_doc, - "Return the union of sets as a new set.\n\ -\n\ -(i.e. all elements that are in either set.)"); - static PyObject * set_or(PySetObject *so, PyObject *other) { @@ -1154,11 +1301,13 @@ set_or(PySetObject *so, PyObject *other) Py_RETURN_NOTIMPLEMENTED; result = (PySetObject *)set_copy(so, NULL); - if (result == NULL) + if (result == NULL) { return NULL; - if ((PyObject *)so == other) + } + if (Py_Is((PyObject *)so, other)) { return (PyObject *)result; - if (set_update_internal(result, other)) { + } + if (set_update_local(result, other)) { Py_DECREF(result); return NULL; } @@ -1171,8 +1320,9 @@ set_ior(PySetObject *so, PyObject *other) if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; - if (set_update_internal(so, other)) + if (set_update_internal(so, other)) { return NULL; + } return Py_NewRef(so); } @@ -1185,7 +1335,7 @@ set_intersection(PySetObject *so, PyObject *other) int rv; if ((PyObject *)so == other) - return set_copy(so, NULL); + return set_copy_impl(so); result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); if (result == NULL) @@ -1259,18 +1409,32 @@ set_intersection(PySetObject *so, PyObject *other) return NULL; } +/*[clinic input] +set.intersection as set_intersection_multi + so: setobject + *others as args: object + / + +Return a new set with elements common to the set and all others. +[clinic start generated code]*/ + static PyObject * -set_intersection_multi(PySetObject *so, PyObject *args) +set_intersection_multi_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=2406ef3387adbe2f input=04108ea6d7f0532b]*/ { Py_ssize_t i; - if (PyTuple_GET_SIZE(args) == 0) + if (PyTuple_GET_SIZE(args) == 0) { return set_copy(so, NULL); + } PyObject *result = Py_NewRef(so); for (i=0 ; iused>50000 ? so->used*2 : so->used*4); } +/*[clinic input] +set.difference_update + so: setobject + *others as args: object + / + +Update the set, removing elements found in others. +[clinic start generated code]*/ + static PyObject * -set_difference_update(PySetObject *so, PyObject *args) +set_difference_update_impl(PySetObject *so, PyObject *args) +/*[clinic end generated code: output=28685b2fc63e41c4 input=e7abb43c9f2c5a73]*/ { Py_ssize_t i; for (i=0 ; ikey; - hash = entry->hash; - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); + Py_ssize_t pos = 0; + setentry *entry; + while (set_next(other, &pos, &entry)) { + PyObject *key = Py_NewRef(entry->key); + Py_hash_t hash = entry->hash; + int rv = set_discard_entry(so, key, hash); if (rv < 0) { - Py_DECREF(otherset); Py_DECREF(key); - return NULL; + return -1; } if (rv == DISCARD_NOTFOUND) { if (set_add_entry(so, key, hash)) { - Py_DECREF(otherset); Py_DECREF(key); - return NULL; + return -1; } } Py_DECREF(key); } - Py_DECREF(otherset); - Py_RETURN_NONE; + return 0; } -PyDoc_STRVAR(symmetric_difference_update_doc, -"Update a set with the symmetric difference of itself and another."); +/*[clinic input] +set.symmetric_difference_update + so: setobject + other: object + / + +Update the set, keeping only elements found in either set, but not in both. +[clinic start generated code]*/ static PyObject * -set_symmetric_difference(PySetObject *so, PyObject *other) +set_symmetric_difference_update(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { - PyObject *rv; - PySetObject *otherset; + if (Py_Is((PyObject *)so, other)) { + return set_clear(so, NULL); + } + + int rv; + if (PyDict_CheckExact(other)) { + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION2(); + } + else if (PyAnySet_Check(other)) { + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_set(so, (PySetObject *)other); + Py_END_CRITICAL_SECTION2(); + } + else { + PySetObject *otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); + if (otherset == NULL) { + return NULL; + } + + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_symmetric_difference_update_set(so, otherset); + Py_END_CRITICAL_SECTION(); - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) - return NULL; - rv = set_symmetric_difference_update(otherset, (PyObject *)so); - if (rv == NULL) { Py_DECREF(otherset); + } + if (rv < 0) { return NULL; } - Py_DECREF(rv); - return (PyObject *)otherset; + Py_RETURN_NONE; } -PyDoc_STRVAR(symmetric_difference_doc, -"Return the symmetric difference of two sets as a new set.\n\ -\n\ -(i.e. all elements that are in exactly one of the sets.)"); +/*[clinic input] +@critical_section so other +set.symmetric_difference + so: setobject + other: object + / + +Return a new set with elements in either the set or other but not both. +[clinic start generated code]*/ + +static PyObject * +set_symmetric_difference_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ +{ + PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + if (result == NULL) { + return NULL; + } + if (set_update_lock_held(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + if (set_symmetric_difference_update_set(result, so) < 0) { + Py_DECREF(result); + return NULL; + } + return (PyObject *)result; +} static PyObject * set_xor(PySetObject *so, PyObject *other) @@ -1720,8 +1984,19 @@ set_ixor(PySetObject *so, PyObject *other) return Py_NewRef(so); } +/*[clinic input] +@critical_section so other +set.issubset + so: setobject + other: object + / + +Report whether another set contains this set. +[clinic start generated code]*/ + static PyObject * -set_issubset(PySetObject *so, PyObject *other) +set_issubset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=b2b59d5f314555ce input=f2a4fd0f2537758b]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1754,10 +2029,19 @@ set_issubset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issubset_doc, "Report whether another set contains this set."); +/*[clinic input] +@critical_section so other +set.issuperset + so: setobject + other: object + / + +Report whether this set contains another set. +[clinic start generated code]*/ static PyObject * -set_issuperset(PySetObject *so, PyObject *other) +set_issuperset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=ecf00ce552c09461 input=5f2e1f262e6e4ccc]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1786,8 +2070,6 @@ set_issuperset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } -PyDoc_STRVAR(issuperset_doc, "Report whether this set contains another set."); - static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { @@ -1831,21 +2113,29 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_NOTIMPLEMENTED; } +/*[clinic input] +@critical_section +set.add + so: setobject + object as key: object + / + +Add an element to a set. + +This has no effect if the element is already present. +[clinic start generated code]*/ + static PyObject * -set_add(PySetObject *so, PyObject *key) +set_add_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=4cc4a937f1425c96 input=03baf62cb0e66514]*/ { if (set_add_key(so, key)) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(add_doc, -"Add an element to a set.\n\ -\n\ -This has no effect if the element is already present."); - static int -set_contains(PySetObject *so, PyObject *key) +set_contains_lock_held(PySetObject *so, PyObject *key) { PyObject *tmpkey; int rv; @@ -1864,21 +2154,54 @@ set_contains(PySetObject *so, PyObject *key) return rv; } +int +_PySet_Contains(PySetObject *so, PyObject *key) +{ + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + return rv; +} + +/*[clinic input] +@critical_section +@coexist +set.__contains__ + so: setobject + object as key: object + / + +x.__contains__(y) <==> y in x. +[clinic start generated code]*/ + static PyObject * -set_direct_contains(PySetObject *so, PyObject *key) +set___contains___impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b44863d034b3c70e input=4a7d568459617f24]*/ { long result; - result = set_contains(so, key); + result = set_contains_lock_held(so, key); if (result < 0) return NULL; return PyBool_FromLong(result); } -PyDoc_STRVAR(contains_doc, "x.__contains__(y) <==> y in x."); +/*[clinic input] +@critical_section +set.remove + so: setobject + object as key: object + / + +Remove an element from a set; it must be a member. + +If the element is not a member, raise a KeyError. +[clinic start generated code]*/ static PyObject * -set_remove(PySetObject *so, PyObject *key) +set_remove_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=0b9134a2a2200363 input=893e1cb1df98227a]*/ { PyObject *tmpkey; int rv; @@ -1904,13 +2227,22 @@ set_remove(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(remove_doc, -"Remove an element from a set; it must be a member.\n\ -\n\ -If the element is not a member, raise a KeyError."); +/*[clinic input] +@critical_section +set.discard + so: setobject + object as key: object + / + +Remove an element from a set if it is a member. + +Unlike set.remove(), the discard() method does not raise +an exception when an element is missing from the set. +[clinic start generated code]*/ static PyObject * -set_discard(PySetObject *so, PyObject *key) +set_discard_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=eec3b687bf32759e input=861cb7fb69b4def0]*/ { PyObject *tmpkey; int rv; @@ -1931,14 +2263,17 @@ set_discard(PySetObject *so, PyObject *key) Py_RETURN_NONE; } -PyDoc_STRVAR(discard_doc, -"Remove an element from a set if it is a member.\n\ -\n\ -Unlike set.remove(), the discard() method does not raise\n\ -an exception when an element is missing from the set."); +/*[clinic input] +@critical_section +set.__reduce__ + so: setobject + +Return state information for pickling. +[clinic start generated code]*/ static PyObject * -set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___reduce___impl(PySetObject *so) +/*[clinic end generated code: output=9af7d0e029df87ee input=59405a4249e82f71]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -1959,8 +2294,17 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) return result; } +/*[clinic input] +@critical_section +set.__sizeof__ + so: setobject + +S.__sizeof__() -> size of S in memory, in bytes. +[clinic start generated code]*/ + static PyObject * -set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___sizeof___impl(PySetObject *so) +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=09e1a09f168eaa23]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -1969,19 +2313,29 @@ set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) return PyLong_FromSize_t(res); } -PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes"); static int set_init(PySetObject *self, PyObject *args, PyObject *kwds) { PyObject *iterable = NULL; - if (!_PyArg_NoKeywords("set", kwds)) + if (!_PyArg_NoKeywords("set", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; + + if (Py_REFCNT(self) == 1 && self->fill == 0) { + self->hash = -1; + if (iterable == NULL) { + return 0; + } + return set_update_local(self, iterable); + } + Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) set_clear_internal(self); self->hash = -1; + Py_END_CRITICAL_SECTION(); + if (iterable == NULL) return 0; return set_update_internal(self, iterable); @@ -2010,59 +2364,39 @@ set_vectorcall(PyObject *type, PyObject * const*args, } static PySequenceMethods set_as_sequence = { - set_len, /* sq_length */ + (lenfunc)set_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)set_contains, /* sq_contains */ + (objobjproc)_PySet_Contains, /* sq_contains */ }; /* set object ********************************************************/ static PyMethodDef set_methods[] = { - {"add", (PyCFunction)set_add, METH_O, - add_doc}, - {"clear", (PyCFunction)set_clear, METH_NOARGS, - clear_doc}, - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)set_copy, METH_NOARGS, - copy_doc}, - {"discard", (PyCFunction)set_discard, METH_O, - discard_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"difference_update", (PyCFunction)set_difference_update, METH_VARARGS, - difference_update_doc}, - {"intersection",(PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"intersection_update",(PyCFunction)set_intersection_update_multi, METH_VARARGS, - intersection_update_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"pop", (PyCFunction)set_pop, METH_NOARGS, - pop_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"remove", (PyCFunction)set_remove, METH_O, - remove_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"symmetric_difference_update",(PyCFunction)set_symmetric_difference_update, METH_O, - symmetric_difference_update_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, - {"update", (PyCFunction)set_update, METH_VARARGS, - update_doc}, + SET_ADD_METHODDEF + SET_CLEAR_METHODDEF + SET___CONTAINS___METHODDEF + SET_COPY_METHODDEF + SET_DISCARD_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_DIFFERENCE_UPDATE_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_INTERSECTION_UPDATE_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET_POP_METHODDEF + SET___REDUCE___METHODDEF + SET_REMOVE_METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF + SET_UNION_METHODDEF + SET_UPDATE_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2100,8 +2434,8 @@ static PyNumberMethods set_as_number = { }; PyDoc_STRVAR(set_doc, -"set() -> new empty set object\n\ -set(iterable) -> new set object\n\ +"set(iterable=(), /)\n\ +--\n\ \n\ Build an unordered collection of unique elements."); @@ -2155,28 +2489,17 @@ PyTypeObject PySet_Type = { static PyMethodDef frozenset_methods[] = { - {"__contains__",(PyCFunction)set_direct_contains, METH_O | METH_COEXIST, - contains_doc}, - {"copy", (PyCFunction)frozenset_copy, METH_NOARGS, - copy_doc}, - {"difference", (PyCFunction)set_difference_multi, METH_VARARGS, - difference_doc}, - {"intersection", (PyCFunction)set_intersection_multi, METH_VARARGS, - intersection_doc}, - {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, - isdisjoint_doc}, - {"issubset", (PyCFunction)set_issubset, METH_O, - issubset_doc}, - {"issuperset", (PyCFunction)set_issuperset, METH_O, - issuperset_doc}, - {"__reduce__", (PyCFunction)set_reduce, METH_NOARGS, - reduce_doc}, - {"__sizeof__", (PyCFunction)set_sizeof, METH_NOARGS, - sizeof_doc}, - {"symmetric_difference",(PyCFunction)set_symmetric_difference, METH_O, - symmetric_difference_doc}, - {"union", (PyCFunction)set_union, METH_VARARGS, - union_doc}, + SET___CONTAINS___METHODDEF + FROZENSET_COPY_METHODDEF + SET_DIFFERENCE_MULTI_METHODDEF + SET_INTERSECTION_MULTI_METHODDEF + SET_ISDISJOINT_METHODDEF + SET_ISSUBSET_METHODDEF + SET_ISSUPERSET_METHODDEF + SET___REDUCE___METHODDEF + SET___SIZEOF___METHODDEF + SET_SYMMETRIC_DIFFERENCE_METHODDEF + SET_UNION_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2201,8 +2524,8 @@ static PyNumberMethods frozenset_as_number = { }; PyDoc_STRVAR(frozenset_doc, -"frozenset() -> empty frozenset object\n\ -frozenset(iterable) -> frozenset object\n\ +"frozenset(iterable=(), /)\n\ +--\n\ \n\ Build an immutable unordered collection of unique elements."); @@ -2274,7 +2597,7 @@ PySet_Size(PyObject *anyset) PyErr_BadInternalCall(); return -1; } - return PySet_GET_SIZE(anyset); + return set_len((PySetObject *)anyset); } int @@ -2284,7 +2607,14 @@ PySet_Clear(PyObject *set) PyErr_BadInternalCall(); return -1; } - return set_clear_internal((PySetObject *)set); + (void)set_clear((PySetObject *)set, NULL); + return 0; +} + +void +_PySet_ClearInternal(PySetObject *so) +{ + (void)set_clear_internal(so); } int @@ -2294,7 +2624,12 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_contains_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_contains_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2304,7 +2639,12 @@ PySet_Discard(PyObject *set, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_discard_key((PySetObject *)set, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(set); + rv = set_discard_key((PySetObject *)set, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2315,7 +2655,12 @@ PySet_Add(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_add_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2334,6 +2679,23 @@ _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash return 1; } +int +_PySet_NextEntryRef(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) +{ + setentry *entry; + + if (!PyAnySet_Check(set)) { + PyErr_BadInternalCall(); + return -1; + } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(set); + if (set_next((PySetObject *)set, pos, &entry) == 0) + return 0; + *key = Py_NewRef(entry->key); + *hash = entry->hash; + return 1; +} + PyObject * PySet_Pop(PyObject *set) { @@ -2394,7 +2756,4 @@ static PyTypeObject _PySetDummy_Type = { Py_TPFLAGS_DEFAULT, /*tp_flags */ }; -static PyObject _dummy_struct = { - { _Py_IMMORTAL_REFCNT }, - &_PySetDummy_Type -}; +static PyObject _dummy_struct = _PyObject_HEAD_INIT(&_PySetDummy_Type); diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 5ffc52ae674c82..245bea98d58509 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -16,6 +16,7 @@ this type and there is exactly one in existence. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -56,6 +57,11 @@ static PyMethodDef ellipsis_methods[] = { {NULL, NULL} }; +PyDoc_STRVAR(ellipsis_doc, +"ellipsis()\n" +"--\n\n" +"The type of the Ellipsis singleton."); + PyTypeObject PyEllipsis_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "ellipsis", /* tp_name */ @@ -77,7 +83,7 @@ PyTypeObject PyEllipsis_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ + ellipsis_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -97,22 +103,23 @@ PyTypeObject PyEllipsis_Type = { ellipsis_new, /* tp_new */ }; -PyObject _Py_EllipsisObject = { - { _Py_IMMORTAL_REFCNT }, - &PyEllipsis_Type -}; +PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ - -void _PySlice_Fini(PyInterpreterState *interp) +void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { - PySliceObject *obj = interp->slice_cache; + if (!is_finalization) { + return; + } +#ifdef WITH_FREELISTS + PySliceObject *obj = freelists->slices.slice_cache; if (obj != NULL) { - interp->slice_cache = NULL; + freelists->slices.slice_cache = NULL; PyObject_GC_Del(obj); } +#endif } /* start, stop, and step are python objects with None indicating no @@ -123,15 +130,17 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - - PyInterpreterState *interp = _PyInterpreterState_GET(); PySliceObject *obj; - if (interp->slice_cache != NULL) { - obj = interp->slice_cache; - interp->slice_cache = NULL; +#ifdef WITH_FREELISTS + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + if (freelists->slices.slice_cache != NULL) { + obj = freelists->slices.slice_cache; + freelists->slices.slice_cache = NULL; _Py_NewReference((PyObject *)obj); } - else { + else +#endif + { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -356,15 +365,18 @@ Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) { - PyInterpreterState *interp = _PyInterpreterState_GET(); _PyObject_GC_UNTRACK(r); Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); - if (interp->slice_cache == NULL) { - interp->slice_cache = r; +#ifdef WITH_FREELISTS + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + if (freelists->slices.slice_cache == NULL) { + freelists->slices.slice_cache = r; } - else { + else +#endif + { PyObject_GC_Del(r); } } diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index e81c5916e6f196..cef7a9496fa874 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(stringlib_expandtabs__doc__, "expandtabs($self, /, tabsize=8)\n" @@ -112,10 +113,24 @@ stringlib_ljust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -168,10 +183,24 @@ stringlib_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -224,10 +253,24 @@ stringlib_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -278,4 +321,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=39cd1ee983137188 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06dd79019356b6bb input=a9049054013a1b77]*/ diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index f98e71c3fc6ecf..440410d0aef17d 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -331,7 +331,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, case _Py_ERROR_REPLACE: memset(p, '?', endpos - startpos); p += (endpos - startpos); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: i += (endpos - startpos - 1); break; @@ -379,7 +379,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, } startpos = k; assert(startpos < endpos); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler( errors, &error_handler_obj, "utf-8", "surrogates not allowed", diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 257b7bd6788ad2..05e700b06258f0 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -256,7 +256,7 @@ STRINGLIB(_factorize)(const STRINGLIB_CHAR *needle, The local period of the cut is the minimal length of a string w such that (left endswith w or w endswith left) - and (right startswith w or w startswith left). + and (right startswith w or w startswith right). The Critical Factorization Theorem says that this maximal local period is the global period of the string. @@ -337,21 +337,20 @@ STRINGLIB(_preprocess)(const STRINGLIB_CHAR *needle, Py_ssize_t len_needle, if (p->is_periodic) { assert(p->cut <= len_needle/2); assert(p->cut < p->period); - p->gap = 0; // unused } else { // A lower bound on the period p->period = Py_MAX(p->cut, len_needle - p->cut) + 1; - // The gap between the last character and the previous - // occurrence of an equivalent character (modulo TABLE_SIZE) - p->gap = len_needle; - STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; - for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { - STRINGLIB_CHAR x = needle[i] & TABLE_MASK; - if (x == last) { - p->gap = len_needle - 1 - i; - break; - } + } + // The gap between the last character and the previous + // occurrence of an equivalent character (modulo TABLE_SIZE) + p->gap = len_needle; + STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; + for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { + STRINGLIB_CHAR x = needle[i] & TABLE_MASK; + if (x == last) { + p->gap = len_needle - 1 - i; + break; } } // Fill up a compressed Boyer-Moore "Bad Character" table @@ -383,6 +382,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, const STRINGLIB_CHAR *window; LOG("===== Two-way: \"%s\" in \"%s\". =====\n", needle, haystack); + Py_ssize_t gap = p->gap; + Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); if (p->is_periodic) { LOG("Needle is periodic.\n"); Py_ssize_t memory = 0; @@ -408,8 +409,16 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, Py_ssize_t i = Py_MAX(cut, memory); for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Right half does not match.\n"); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } memory = 0; goto periodicwindowloop; } @@ -442,10 +451,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, } } else { - Py_ssize_t gap = p->gap; period = Py_MAX(gap, period); LOG("Needle is not periodic.\n"); - Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); windowloop: while (window_last < haystack_end) { for (;;) { @@ -463,19 +470,19 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, window = window_last - len_needle + 1; assert((window[len_needle - 1] & TABLE_MASK) == (needle[len_needle - 1] & TABLE_MASK)); - for (Py_ssize_t i = cut; i < gap_jump_end; i++) { - if (needle[i] != window[i]) { - LOG("Early right half mismatch: jump by gap.\n"); - assert(gap >= i - cut + 1); - window_last += gap; - goto windowloop; - } - } - for (Py_ssize_t i = gap_jump_end; i < len_needle; i++) { + Py_ssize_t i = cut; + for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Late right half mismatch.\n"); - assert(i - cut + 1 > gap); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } goto windowloop; } } @@ -746,6 +753,22 @@ STRINGLIB(count_char)(const STRINGLIB_CHAR *s, Py_ssize_t n, } +static inline Py_ssize_t +STRINGLIB(count_char_no_maxcount)(const STRINGLIB_CHAR *s, Py_ssize_t n, + const STRINGLIB_CHAR p0) +/* A specialized function of count_char that does not cut off at a maximum. + As a result, the compiler is able to vectorize the loop. */ +{ + Py_ssize_t count = 0; + for (Py_ssize_t i = 0; i < n; i++) { + if (s[i] == p0) { + count++; + } + } + return count; +} + + Py_LOCAL_INLINE(Py_ssize_t) FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, @@ -766,6 +789,9 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, else if (mode == FAST_RSEARCH) return STRINGLIB(rfind_char)(s, n, p[0]); else { + if (maxcount == PY_SSIZE_T_MAX) { + return STRINGLIB(count_char_no_maxcount)(s, n, p[0]); + } return STRINGLIB(count_char)(s, n, p[0], maxcount); } } diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index 509b9297396be8..c385718a5b2692 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -70,50 +70,3 @@ STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) } #endif /* STRINGLIB_WANT_CONTAINS_OBJ */ - -/* -This function is a helper for the "find" family (find, rfind, index, -rindex) and for count, startswith and endswith, because they all have -the same behaviour for the arguments. - -It does not touch the variables received until it knows everything -is ok. -*/ - -#define FORMAT_BUFFER_SIZE 50 - -Py_LOCAL_INLINE(int) -STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, - PyObject **subobj, - Py_ssize_t *start, Py_ssize_t *end) -{ - PyObject *tmp_subobj; - Py_ssize_t tmp_start = 0; - Py_ssize_t tmp_end = PY_SSIZE_T_MAX; - PyObject *obj_start=Py_None, *obj_end=Py_None; - char format[FORMAT_BUFFER_SIZE] = "O|OO:"; - size_t len = strlen(format); - - strncpy(format + len, function_name, FORMAT_BUFFER_SIZE - len - 1); - format[FORMAT_BUFFER_SIZE - 1] = '\0'; - - if (!PyArg_ParseTuple(args, format, &tmp_subobj, &obj_start, &obj_end)) - return 0; - - /* To support None in "start" and "end" arguments, meaning - the same as if they were not passed. - */ - if (obj_start != Py_None) - if (!_PyEval_SliceIndex(obj_start, &tmp_start)) - return 0; - if (obj_end != Py_None) - if (!_PyEval_SliceIndex(obj_end, &tmp_end)) - return 0; - - *start = tmp_start; - *end = tmp_end; - *subobj = tmp_subobj; - return 1; -} - -#undef FORMAT_BUFFER_SIZE diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index b9ffdfc2e352ce..7ab3fc88b331b1 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -1,6 +1,7 @@ /* Finding the optimal width of unicode characters in a buffer */ -#if !STRINGLIB_IS_UNICODE +/* find_max_char for one-byte will work for bytes objects as well. */ +#if !STRINGLIB_IS_UNICODE && STRINGLIB_SIZEOF_CHAR > 1 # error "find_max_char.h is specific to Unicode" #endif @@ -20,19 +21,20 @@ Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) { const unsigned char *p = (const unsigned char *) begin; + const unsigned char *_end = (const unsigned char *)end; - while (p < end) { + while (p < _end) { if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { /* Help register allocation */ const unsigned char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { + while (_p + SIZEOF_SIZE_T <= _end) { size_t value = *(const size_t *) _p; if (value & UCS1_ASCII_CHAR_MASK) return 255; _p += SIZEOF_SIZE_T; } p = _p; - if (p == end) + if (p == _end) break; } if (*p++ & 0x80) diff --git a/Objects/stringlib/repr.h b/Objects/stringlib/repr.h new file mode 100644 index 00000000000000..87b1a8ba629dc6 --- /dev/null +++ b/Objects/stringlib/repr.h @@ -0,0 +1,95 @@ +/* stringlib: repr() implementation */ + +#ifndef STRINGLIB_FASTSEARCH_H +#error must include "stringlib/fastsearch.h" before including this module +#endif + + +static void +STRINGLIB(repr)(PyObject *unicode, Py_UCS4 quote, + STRINGLIB_CHAR *odata) +{ + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); + int ikind = PyUnicode_KIND(unicode); + + *odata++ = quote; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + + /* Escape quotes and backslashes */ + if ((ch == quote) || (ch == '\\')) { + *odata++ = '\\'; + *odata++ = ch; + continue; + } + + /* Map special whitespace to '\t', \n', '\r' */ + if (ch == '\t') { + *odata++ = '\\'; + *odata++ = 't'; + } + else if (ch == '\n') { + *odata++ = '\\'; + *odata++ = 'n'; + } + else if (ch == '\r') { + *odata++ = '\\'; + *odata++ = 'r'; + } + + /* Map non-printable US ASCII to '\xhh' */ + else if (ch < ' ' || ch == 0x7F) { + *odata++ = '\\'; + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + + /* Copy ASCII characters as-is */ + else if (ch < 0x7F) { + *odata++ = ch; + } + + /* Non-ASCII characters */ + else { + /* Map Unicode whitespace and control characters + (categories Z* and C* except ASCII space) + */ + if (!Py_UNICODE_ISPRINTABLE(ch)) { + *odata++ = '\\'; + /* Map 8-bit characters to '\xhh' */ + if (ch <= 0xff) { + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + /* Map 16-bit characters to '\uxxxx' */ + else if (ch <= 0xffff) { + *odata++ = 'u'; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + /* Map 21-bit characters to '\U00xxxxxx' */ + else { + *odata++ = 'U'; + *odata++ = Py_hexdigits[(ch >> 28) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 24) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 20) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 16) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + } + /* Copy characters as-is */ + else { + *odata++ = ch; + } + } + } + *odata = quote; +} diff --git a/Objects/structseq.c b/Objects/structseq.c index e4a4b45a8db626..d8289f2638db0f 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,12 +8,11 @@ */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_Pop() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_object.h" // _PyObject_GC_TRACK() - #include "pycore_structseq.h" // PyStructSequence_InitType() -#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_tuple.h" // _PyTuple_FromArray() static const char visible_length_key[] = "n_sequence_fields"; static const char real_length_key[] = "n_fields"; @@ -417,14 +416,13 @@ structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) // We do not support types with unnamed fields, so we can iterate over // i >= n_visible_fields case without slicing with (i - n_unnamed_fields). for (i = 0; i < n_fields; ++i) { - PyObject *key = PyUnicode_FromString(Py_TYPE(self)->tp_members[i].name); - if (!key) { + PyObject *ob; + if (PyDict_PopString(kwargs, Py_TYPE(self)->tp_members[i].name, + &ob) < 0) { goto error; } - PyObject *ob = _PyDict_Pop(kwargs, key, self->ob_item[i]); - Py_DECREF(key); - if (!ob) { - goto error; + if (ob == NULL) { + ob = Py_NewRef(self->ob_item[i]); } result->ob_item[i] = ob; } @@ -455,7 +453,9 @@ structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) static PyMethodDef structseq_methods[] = { {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL}, - {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the structure with new values for the specified fields.")}, {NULL, NULL} // sentinel }; @@ -605,6 +605,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, PyStructSequence_Desc *desc, unsigned long tp_flags) { + if (Py_TYPE(type) == NULL) { + Py_SET_TYPE(type, &PyType_Type); + } Py_ssize_t n_unnamed_members; Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; @@ -620,7 +623,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, } initialize_static_fields(type, desc, members, tp_flags); - _Py_SetImmortal(type); + _Py_SetImmortal((PyObject *)type); } #ifndef NDEBUG else { @@ -715,7 +718,7 @@ _PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) return; } - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); if (_Py_IsMainInterpreter(interp)) { // Undo _PyStructSequence_InitBuiltinWithFlags(). diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d567839c5e3a0b..3704d095a977ea 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -390,6 +390,27 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return (PyObject *)tuple; } +PyObject * +_PyTuple_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) +{ + if (n == 0) { + return tuple_get_empty(); + } + PyTupleObject *tuple = tuple_alloc(n); + if (tuple == NULL) { + for (Py_ssize_t i = 0; i < n; i++) { + PyStackRef_CLOSE(src[i]); + } + return NULL; + } + PyObject **dst = tuple->ob_item; + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } + _PyObject_GC_TRACK(tuple); + return (PyObject *)tuple; +} + PyObject * _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) { @@ -946,7 +967,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) if (sv == NULL) { *pv = NULL; #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_GC_Del(v); return -1; @@ -962,18 +983,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) } -static void maybe_freelist_clear(PyInterpreterState *, int); +static void maybe_freelist_clear(struct _Py_object_freelists *, int); -void -_PyTuple_Fini(PyInterpreterState *interp) -{ - maybe_freelist_clear(interp, 1); -} void -_PyTuple_ClearFreeList(PyInterpreterState *interp) +_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { - maybe_freelist_clear(interp, 0); + maybe_freelist_clear(freelists, is_finalization); } /*********************** Tuple Iterator **************************/ @@ -1125,30 +1141,26 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define STATE (interp->tuple) -#define FREELIST_FINALIZED (STATE.numfree[0] < 0) +#define TUPLE_FREELIST (freelists->tuples) +#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0) static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t size) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); -#ifdef Py_DEBUG - /* maybe_freelist_pop() must not be called after maybe_freelist_fini(). */ - assert(!FREELIST_FINALIZED); -#endif +#ifdef WITH_FREELISTS + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); if (size == 0) { return NULL; } assert(size > 0); - if (size < PyTuple_MAXSAVESIZE) { + if (size <= PyTuple_MAXSAVESIZE) { Py_ssize_t index = size - 1; - PyTupleObject *op = STATE.free_list[index]; + PyTupleObject *op = TUPLE_FREELIST.items[index]; if (op != NULL) { /* op is the head of a linked list, with the first item pointing to the next node. Here we pop off the old head. */ - STATE.free_list[index] = (PyTupleObject *) op->ob_item[0]; - STATE.numfree[index]--; + TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; + TUPLE_FREELIST.numfree[index]--; /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ #ifdef Py_TRACE_REFS /* maybe_freelist_push() ensures these were already set. */ @@ -1169,25 +1181,22 @@ maybe_freelist_pop(Py_ssize_t size) static inline int maybe_freelist_push(PyTupleObject *op) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); -#ifdef Py_DEBUG - /* maybe_freelist_push() must not be called after maybe_freelist_fini(). */ - assert(!FREELIST_FINALIZED); -#endif +#ifdef WITH_FREELISTS + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); if (Py_SIZE(op) == 0) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; if (index < PyTuple_NFREELISTS - && STATE.numfree[index] < PyTuple_MAXFREELIST + && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST + && TUPLE_FREELIST.numfree[index] >= 0 && Py_IS_TYPE(op, &PyTuple_Type)) { /* op is the head of a linked list, with the first item pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) STATE.free_list[index]; - STATE.free_list[index] = op; - STATE.numfree[index]++; + op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; + TUPLE_FREELIST.items[index] = op; + TUPLE_FREELIST.numfree[index]++; OBJECT_STAT_INC(to_freelist); return 1; } @@ -1196,13 +1205,13 @@ maybe_freelist_push(PyTupleObject *op) } static void -maybe_freelist_clear(PyInterpreterState *interp, int fini) +maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) { -#if PyTuple_NFREELISTS > 0 +#ifdef WITH_FREELISTS for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = STATE.free_list[i]; - STATE.free_list[i] = NULL; - STATE.numfree[i] = fini ? -1 : 0; + PyTupleObject *p = TUPLE_FREELIST.items[i]; + TUPLE_FREELIST.items[i] = NULL; + TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; while (p) { PyTupleObject *q = p; p = (PyTupleObject *)(p->ob_item[0]); @@ -1216,14 +1225,14 @@ maybe_freelist_clear(PyInterpreterState *interp, int fini) void _PyTuple_DebugMallocStats(FILE *out) { -#if PyTuple_NFREELISTS > 0 - PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef WITH_FREELISTS + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); for (int i = 0; i < PyTuple_NFREELISTS; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, STATE.numfree[i], + _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, len)); } #endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3261a14a053dc8..7d01b680605a38 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,11 +6,13 @@ #include "pycore_code.h" // CO_FAST_FREE #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_lock.h" // _PySeqLock_* +#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // _Py_Mangle() @@ -41,7 +43,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -51,6 +54,36 @@ class object "PyObject *" "&PyBaseObject_Type" #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag +#ifdef Py_GIL_DISABLED + +// There's a global lock for mutation of types. This avoids having to take +// additional locks while doing various subclass processing which may result +// in odd behaviors w.r.t. running with the GIL as the outer type lock could +// be released and reacquired during a subclass update if there's contention +// on the subclass lock. +#define TYPE_LOCK &PyInterpreterState_Get()->types.mutex +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() + +#define BEGIN_TYPE_DICT_LOCK(d) \ + Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + +#define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() + +#define ASSERT_TYPE_LOCK_HELD() \ + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) + +#else + +#define BEGIN_TYPE_LOCK() +#define END_TYPE_LOCK() +#define BEGIN_TYPE_DICT_LOCK(d) +#define END_TYPE_DICT_LOCK() +#define ASSERT_TYPE_LOCK_HELD() + +#endif + + typedef struct PySlot_Offset { short subslot_offset; short slot_offset; @@ -85,88 +118,213 @@ type_from_ref(PyObject *ref) /* helpers for for static builtin types */ +#ifndef NDEBUG static inline int -static_builtin_index_is_set(PyTypeObject *self) +managed_static_type_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } +#endif static inline size_t -static_builtin_index_get(PyTypeObject *self) +managed_static_type_index_get(PyTypeObject *self) { - assert(static_builtin_index_is_set(self)); + assert(managed_static_type_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ return (size_t)self->tp_subclasses - 1; } static inline void -static_builtin_index_set(PyTypeObject *self, size_t index) +managed_static_type_index_set(PyTypeObject *self, size_t index) { - assert(index < _Py_MAX_STATIC_BUILTIN_TYPES); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); /* We store a 1-based index so 0 can mean "not initialized". */ self->tp_subclasses = (PyObject *)(index + 1); } static inline void -static_builtin_index_clear(PyTypeObject *self) +managed_static_type_index_clear(PyTypeObject *self) { self->tp_subclasses = NULL; } -static inline static_builtin_state * -static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) +static PyTypeObject * +static_ext_type_lookup(PyInterpreterState *interp, size_t index, + int64_t *p_interp_count) +{ + assert(interp->runtime == &_PyRuntime); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + + size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + int64_t interp_count = + _PyRuntime.types.managed_static.types[full_index].interp_count; + assert((interp_count == 0) == + (_PyRuntime.types.managed_static.types[full_index].type == NULL)); + *p_interp_count = interp_count; + + PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + if (type == NULL) { + return NULL; + } + assert(!interp->types.for_extensions.initialized[index].isbuiltin); + assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(managed_static_type_index_is_set(type)); + return type; +} + +static managed_static_type_state * +managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self) { - return &(interp->types.builtins[static_builtin_index_get(self)]); + // It's probably a builtin type. + size_t index = managed_static_type_index_get(self); + managed_static_type_state *state = + &(interp->types.builtins.initialized[index]); + if (state->type == self) { + return state; + } + if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) { + return state; + } + return &(interp->types.for_extensions.initialized[index]); } /* For static types we store some state in an array on each interpreter. */ -static_builtin_state * +managed_static_type_state * _PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - return static_builtin_state_get(interp, self); + return managed_static_type_state_get(interp, self); } /* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { - if (!static_builtin_index_is_set(self)) { - static_builtin_index_set(self, interp->types.num_builtins_initialized); + assert(interp->runtime == &_PyRuntime); + + size_t index; + if (initial) { + assert(!managed_static_type_index_is_set(self)); + if (isbuiltin) { + index = interp->types.builtins.num_initialized; + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + PyMutex_Lock(&interp->types.mutex); + index = interp->types.for_extensions.next_index; + interp->types.for_extensions.next_index++; + PyMutex_Unlock(&interp->types.mutex); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + managed_static_type_index_set(self, index); + } + else { + index = managed_static_type_index_get(self); + if (isbuiltin) { + assert(index == interp->types.builtins.num_initialized); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + } + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + assert((initial == 1) == + (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); + + if (initial) { + assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); + _PyRuntime.types.managed_static.types[full_index].type = self; + } + else { + assert(_PyRuntime.types.managed_static.types[full_index].type == self); } - static_builtin_state *state = static_builtin_state_get(interp, self); - /* It should only be called once for each builtin type. */ + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + + /* It should only be called once for each builtin type per interpreter. */ assert(state->type == NULL); state->type = self; + state->isbuiltin = isbuiltin; /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ - interp->types.num_builtins_initialized++; + if (isbuiltin) { + interp->types.builtins.num_initialized++; + } + else { + interp->types.for_extensions.num_initialized++; + } } /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ + This basically undoes what managed_static_type_state_init() did. */ static void -static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int final) { - static_builtin_state *state = static_builtin_state_get(interp, self); + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + assert(state != NULL); + + assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - if (_Py_IsMainInterpreter(interp)) { - static_builtin_index_clear(self); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); + if (final) { + assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + _PyRuntime.types.managed_static.types[full_index].type = NULL; + + managed_static_type_index_clear(self); } - assert(interp->types.num_builtins_initialized > 0); - interp->types.num_builtins_initialized--; + if (isbuiltin) { + assert(interp->types.builtins.num_initialized > 0); + interp->types.builtins.num_initialized--; + } + else { + PyMutex_Lock(&interp->types.mutex); + assert(interp->types.for_extensions.num_initialized > 0); + interp->types.for_extensions.num_initialized--; + if (interp->types.for_extensions.num_initialized == 0) { + interp->types.for_extensions.next_index = 0; + } + PyMutex_Unlock(&interp->types.mutex); + } +} + +static PyTypeObject * +managed_static_type_get_def(PyTypeObject *self, int isbuiltin) +{ + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + return &_PyRuntime.types.managed_static.types[full_index].def; } -// Also see _PyStaticType_InitBuiltin() and _PyStaticType_Dealloc(). +// Also see _PyStaticType_InitBuiltin() and _PyStaticType_FiniBuiltin(). /* end static builtin helpers */ @@ -176,7 +334,7 @@ start_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(!state->readying); state->readying = 1; @@ -191,7 +349,7 @@ stop_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(state->readying); state->readying = 0; @@ -206,7 +364,7 @@ is_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); return state->readying; } @@ -221,7 +379,7 @@ lookup_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_dict; } @@ -247,7 +405,7 @@ set_tp_dict(PyTypeObject *self, PyObject *dict) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); state->tp_dict = dict; return; @@ -260,7 +418,7 @@ clear_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); Py_CLEAR(state->tp_dict); return; @@ -278,18 +436,24 @@ lookup_tp_bases(PyTypeObject *self) PyObject * _PyType_GetBases(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_bases(self); + PyObject *res; + + BEGIN_TYPE_LOCK(); + res = lookup_tp_bases(self); + Py_INCREF(res); + END_TYPE_LOCK(); + + return res; } static inline void -set_tp_bases(PyTypeObject *self, PyObject *bases) +set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) { assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_bases can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_bases == NULL); if (PyTuple_GET_SIZE(bases) == 0) { assert(self->tp_base == NULL); @@ -306,10 +470,10 @@ set_tp_bases(PyTypeObject *self, PyObject *bases) } static inline void -clear_tp_bases(PyTypeObject *self) +clear_tp_bases(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_bases != NULL) { if (PyTuple_GET_SIZE(self->tp_bases) == 0) { Py_CLEAR(self->tp_bases); @@ -329,24 +493,40 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { + ASSERT_TYPE_LOCK_HELD(); return self->tp_mro; } PyObject * _PyType_GetMRO(PyTypeObject *self) { - /* It returns a borrowed reference. */ - return lookup_tp_mro(self); +#ifdef Py_GIL_DISABLED + PyObject *mro = _Py_atomic_load_ptr_relaxed(&self->tp_mro); + if (mro == NULL) { + return NULL; + } + if (_Py_TryIncrefCompare(&self->tp_mro, mro)) { + return mro; + } + + BEGIN_TYPE_LOCK(); + mro = lookup_tp_mro(self); + Py_XINCREF(mro); + END_TYPE_LOCK(); + return mro; +#else + return Py_XNewRef(lookup_tp_mro(self)); +#endif } static inline void -set_tp_mro(PyTypeObject *self, PyObject *mro) +set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) { assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_mro can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_mro == NULL); /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); @@ -355,10 +535,10 @@ set_tp_mro(PyTypeObject *self, PyObject *mro) } static inline void -clear_tp_mro(PyTypeObject *self) +clear_tp_mro(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_mro != NULL) { if (PyTuple_GET_SIZE(self->tp_mro) == 0) { Py_CLEAR(self->tp_mro); @@ -384,7 +564,7 @@ init_tp_subclasses(PyTypeObject *self) } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -400,7 +580,7 @@ clear_tp_subclasses(PyTypeObject *self) has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } @@ -412,7 +592,7 @@ lookup_tp_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -645,9 +825,15 @@ type_cache_clear(struct type_cache *cache, PyObject *value) { for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { struct type_cache_entry *entry = &cache->hashtable[i]; +#ifdef Py_GIL_DISABLED + _PySeqLock_LockWrite(&entry->sequence); +#endif entry->version = 0; Py_XSETREF(entry->name, _Py_XNewRef(value)); entry->value = NULL; +#ifdef Py_GIL_DISABLED + _PySeqLock_UnlockWrite(&entry->sequence); +#endif } } @@ -661,7 +847,7 @@ _PyType_InitCache(PyInterpreterState *interp) assert(entry->name == NULL); entry->version = 0; - // Set to None so _PyType_Lookup() can use Py_SETREF(), + // Set to None so _PyType_LookupRef() can use Py_SETREF(), // rather than using slower Py_XSETREF(). entry->name = Py_None; entry->value = NULL; @@ -673,7 +859,7 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_Lookup() can + // Set to None, rather than NULL, so _PyType_LookupRef() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -695,10 +881,14 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); - assert(interp->types.num_builtins_initialized == 0); - // All the static builtin types should have been finalized already. - for (size_t i = 0; i < _Py_MAX_STATIC_BUILTIN_TYPES; i++) { - assert(interp->types.builtins[i].type == NULL); + // All the managed static types should have been finalized already. + assert(interp->types.for_extensions.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + assert(interp->types.for_extensions.initialized[i].type == NULL); + } + assert(interp->types.builtins.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + assert(interp->types.builtins.initialized[i].type == NULL); } } @@ -708,7 +898,8 @@ PyType_AddWatcher(PyType_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + // start at 1, 0 is reserved for cpython optimizer + for (int i = 1; i < TYPE_MAX_WATCHERS; i++) { if (!interp->type_watchers[i]) { interp->type_watchers[i] = callback; return i; @@ -759,8 +950,10 @@ PyType_Watch(int watcher_id, PyObject* obj) return -1; } // ensure we will get a callback on the next modification + BEGIN_TYPE_LOCK(); assign_version_tag(interp, type); type->tp_watched |= (1 << watcher_id); + END_TYPE_LOCK(); return 0; } @@ -780,8 +973,40 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } -void -PyType_Modified(PyTypeObject *type) +static void +set_version_unlocked(PyTypeObject *tp, unsigned int version) +{ + ASSERT_TYPE_LOCK_HELD(); +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + // lookup the old version and set to null + if (tp->tp_version_tag != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE); + *slot = NULL; + } + if (version) { + tp->tp_versions_used++; + } +#else + if (version) { + _Py_atomic_add_uint16(&tp->tp_versions_used, 1); + } +#endif + FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version); +#ifndef Py_GIL_DISABLED + if (version != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + *slot = tp; + } +#endif +} + +static void +type_modified_unlocked(PyTypeObject *type) { /* Invalidate any cached data for the specified type and all subclasses. This function is called after the base @@ -789,16 +1014,16 @@ PyType_Modified(PyTypeObject *type) Invariants: - - before Py_TPFLAGS_VALID_VERSION_TAG can be set on a type, + - before tp_version_tag can be set on a type, it must first be set on all super types. - This function clears the Py_TPFLAGS_VALID_VERSION_TAG of a + This function clears the tp_version_tag of a type (so it must first clear it on all subclasses). The - tp_version_tag value is meaningless unless this flag is set. + tp_version_tag value is meaningless when equal to zero. We don't assign new version tags eagerly, but only as needed. */ - if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag == 0) { return; } @@ -813,7 +1038,7 @@ PyType_Modified(PyTypeObject *type) if (subclass == NULL) { continue; } - PyType_Modified(subclass); + type_modified_unlocked(subclass); Py_DECREF(subclass); } } @@ -828,7 +1053,9 @@ PyType_Modified(PyTypeObject *type) if (bits & 1) { PyType_WatchCallback cb = interp->type_watchers[i]; if (cb && (cb(type) < 0)) { - PyErr_WriteUnraisable((PyObject *)type); + PyErr_FormatUnraisable( + "Exception ignored in type watcher callback #%d for %R", + i, type); } } i++; @@ -836,8 +1063,7 @@ PyType_Modified(PyTypeObject *type) } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -845,6 +1071,22 @@ PyType_Modified(PyTypeObject *type) } } +void +PyType_Modified(PyTypeObject *type) +{ + // Quick check without the lock held + if (type->tp_version_tag == 0) { + return; + } + + BEGIN_TYPE_LOCK(); + type_modified_unlocked(type); + END_TYPE_LOCK(); +} + +static int +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b); + static void type_mro_modified(PyTypeObject *type, PyObject *bases) { /* @@ -863,6 +1105,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { int custom = !Py_IS_TYPE(type, &PyType_Type); int unbound; + ASSERT_TYPE_LOCK_HELD(); if (custom) { PyObject *mro_meth, *type_mro_meth; mro_meth = lookup_maybe_method( @@ -888,7 +1131,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); - if (!PyType_IsSubtype(type, cls)) { + if (!is_subtype_with_mro(lookup_tp_mro(type), type, cls)) { goto clear; } } @@ -896,8 +1139,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -905,28 +1147,85 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +/* +The Tier 2 interpreter requires looking up the type object by the type version, so it can install +watchers to understand when they change. + +So we add a global cache from type version to borrowed references of type objects. + +This is similar to func_version_cache. +*/ + +void +_PyType_SetVersion(PyTypeObject *tp, unsigned int version) +{ + + BEGIN_TYPE_LOCK(); + set_version_unlocked(tp, version); + END_TYPE_LOCK(); +} + +PyTypeObject * +_PyType_LookupByVersion(unsigned int version) +{ +#ifdef Py_GIL_DISABLED + return NULL; +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + if (*slot && (*slot)->tp_version_tag == version) { + return *slot; + } + return NULL; +#endif +} + +unsigned int +_PyType_GetVersionForCurrentState(PyTypeObject *tp) +{ + return tp->tp_version_tag; +} + + + +#define MAX_VERSIONS_PER_CLASS 1000 + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { - /* Ensure that the tp_version_tag is valid and set - Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this - must first be done on all super classes. Return 0 if this - cannot be done, 1 if Py_TPFLAGS_VALID_VERSION_TAG. + ASSERT_TYPE_LOCK_HELD(); + + /* Ensure that the tp_version_tag is valid. + * To respect the invariant, this must first be done on all super classes. + * Return 0 if this cannot be done, 1 if tp_version_tag is set. */ - if (_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag != 0) { return 1; } if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { return 0; } + if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + return 0; + } + PyObject *bases = lookup_tp_bases(type); + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *b = PyTuple_GET_ITEM(bases, i); + if (!assign_version_tag(interp, _PyType_CAST(b))) { + return 0; + } + } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -935,25 +1234,20 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + set_version_unlocked(type, NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } - - PyObject *bases = lookup_tp_bases(type); - Py_ssize_t n = PyTuple_GET_SIZE(bases); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *b = PyTuple_GET_ITEM(bases, i); - if (!assign_version_tag(interp, _PyType_CAST(b))) - return 0; - } - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; return 1; } int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return assign_version_tag(interp, type); + int assigned; + BEGIN_TYPE_LOCK(); + assigned = assign_version_tag(interp, type); + END_TYPE_LOCK(); + return assigned; } @@ -1084,28 +1378,24 @@ type_set_qualname(PyTypeObject *type, PyObject *value, void *context) } static PyObject * -type_module(PyTypeObject *type, void *context) +type_module(PyTypeObject *type) { PyObject *mod; - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyObject *dict = lookup_tp_dict(type); - mod = PyDict_GetItemWithError(dict, &_Py_ID(__module__)); - if (mod == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, "__module__"); - } - return NULL; + if (PyDict_GetItemRef(dict, &_Py_ID(__module__), &mod) == 0) { + PyErr_Format(PyExc_AttributeError, "__module__"); } - Py_INCREF(mod); } else { const char *s = strrchr(type->tp_name, '.'); if (s != NULL) { mod = PyUnicode_FromStringAndSize( type->tp_name, (Py_ssize_t)(s - type->tp_name)); - if (mod != NULL) - PyUnicode_InternInPlace(&mod); + if (mod != NULL) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &mod); + } } else { mod = &_Py_ID(builtins); @@ -1114,6 +1404,12 @@ type_module(PyTypeObject *type, void *context) return mod; } +static PyObject * +type_get_module(PyTypeObject *type, void *context) +{ + return type_module(type); +} + static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { @@ -1126,23 +1422,63 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__module__), value); } + +PyObject * +_PyType_GetFullyQualifiedName(PyTypeObject *type, char sep) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return PyUnicode_FromString(type->tp_name); + } + + PyObject *qualname = type_qualname(type, NULL); + if (qualname == NULL) { + return NULL; + } + + PyObject *module = type_module(type); + if (module == NULL) { + Py_DECREF(qualname); + return NULL; + } + + PyObject *result; + if (PyUnicode_Check(module) + && !_PyUnicode_Equal(module, &_Py_ID(builtins)) + && !_PyUnicode_Equal(module, &_Py_ID(__main__))) + { + result = PyUnicode_FromFormat("%U%c%U", module, sep, qualname); + } + else { + result = Py_NewRef(qualname); + } + Py_DECREF(module); + Py_DECREF(qualname); + return result; +} + +PyObject * +PyType_GetFullyQualifiedName(PyTypeObject *type) +{ + return _PyType_GetFullyQualifiedName(type, '.'); +} + + static PyObject * type_abstractmethods(PyTypeObject *type, void *context) { PyObject *mod = NULL; /* type itself has an __abstractmethods__ descriptor (this). Don't return that. */ - if (type != &PyType_Type) { - PyObject *dict = lookup_tp_dict(type); - mod = PyDict_GetItemWithError(dict, &_Py_ID(__abstractmethods__)); + if (type == &PyType_Type) { + PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); } - if (!mod) { - if (!PyErr_Occurred()) { + else { + PyObject *dict = lookup_tp_dict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__abstractmethods__), &mod) == 0) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); } - return NULL; } - return Py_NewRef(mod); + return mod; } static int @@ -1162,40 +1498,49 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) } else { abstract = 0; - res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); - if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { + res = PyDict_Pop(dict, &_Py_ID(__abstractmethods__), NULL); + if (res == 0) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; } } - if (res == 0) { - PyType_Modified(type); - if (abstract) - type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; - else - type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + if (res < 0) { + return -1; } - return res; + + PyType_Modified(type); + if (abstract) + type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; + else + type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + return 0; } static PyObject * type_get_bases(PyTypeObject *type, void *context) { - PyObject *bases = lookup_tp_bases(type); + PyObject *bases = _PyType_GetBases(type); if (bases == NULL) { Py_RETURN_NONE; } - return Py_NewRef(bases); + return bases; } static PyObject * type_get_mro(PyTypeObject *type, void *context) { - PyObject *mro = lookup_tp_mro(type); + PyObject *mro; + + BEGIN_TYPE_LOCK(); + mro = lookup_tp_mro(type); if (mro == NULL) { - Py_RETURN_NONE; + mro = Py_None; + } else { + Py_INCREF(mro); } - return Py_NewRef(mro); + + END_TYPE_LOCK(); + return mro; } static PyTypeObject *best_base(PyObject *); @@ -1217,6 +1562,8 @@ static int recurse_down_subclasses(PyTypeObject *type, PyObject *name, static int mro_hierarchy(PyTypeObject *type, PyObject *temp) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *old_mro; int res = mro_internal(type, &old_mro); if (res <= 0) { @@ -1242,7 +1589,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - set_tp_mro(type, old_mro); + set_tp_mro(type, old_mro, 0); Py_DECREF(new_mro); return -1; } @@ -1280,7 +1627,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) } static int -type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) { // Check arguments if (!check_set_special_type_attr(type, new_bases, "__bases__")) { @@ -1311,7 +1658,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) } PyTypeObject *base = (PyTypeObject*)ob; - if (PyType_IsSubtype(base, type) || + if (is_subtype_with_mro(lookup_tp_mro(base), base, type) || /* In case of reentering here again through a custom mro() the above check is not enough since it relies on base->tp_mro which would gonna be updated inside @@ -1343,7 +1690,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - set_tp_bases(type, Py_NewRef(new_bases)); + set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -1374,6 +1721,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) res = 0; } + RARE_EVENT_INC(set_bases); Py_DECREF(old_bases); Py_DECREF(old_base); @@ -1390,7 +1738,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ if (lookup_tp_mro(cls) == new_mro) { - set_tp_mro(cls, Py_XNewRef(old_mro)); + set_tp_mro(cls, Py_XNewRef(old_mro), 0); Py_DECREF(new_mro); } } @@ -1400,7 +1748,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - set_tp_bases(type, old_bases); + set_tp_bases(type, old_bases, 0); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1415,6 +1763,16 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } +static int +type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) +{ + int res; + BEGIN_TYPE_LOCK(); + res = type_set_bases_unlocked(type, new_bases, context); + END_TYPE_LOCK(); + return res; +} + static PyObject * type_dict(PyTypeObject *type, void *context) { @@ -1433,18 +1791,14 @@ type_get_doc(PyTypeObject *type, void *context) return _PyType_GetDocFromInternalDoc(type->tp_name, type->tp_doc); } PyObject *dict = lookup_tp_dict(type); - result = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); - if (result == NULL) { - if (!PyErr_Occurred()) { - result = Py_NewRef(Py_None); - } + if (PyDict_GetItemRef(dict, &_Py_ID(__doc__), &result) == 0) { + result = Py_NewRef(Py_None); } - else if (Py_TYPE(result)->tp_descr_get) { - result = Py_TYPE(result)->tp_descr_get(result, NULL, - (PyObject *)type); - } - else { - Py_INCREF(result); + else if (result) { + descrgetfunc descr_get = Py_TYPE(result)->tp_descr_get; + if (descr_get) { + Py_SETREF(result, descr_get(result, NULL, (PyObject *)type)); + } } return result; } @@ -1465,6 +1819,76 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } +static PyObject * +type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotate__'", type->tp_name); + return NULL; + } + + PyObject *annotate; + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate) { + descrgetfunc get = Py_TYPE(annotate)->tp_descr_get; + if (get) { + Py_SETREF(annotate, get(annotate, NULL, (PyObject *)type)); + } + } + else { + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result < 0) { + Py_DECREF(dict); + return NULL; + } + } + Py_DECREF(dict); + return annotate; +} + +static int +type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_Format(PyExc_TypeError, + "cannot set '__annotate__' attribute of immutable type '%s'", + type->tp_name); + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + + PyObject *dict = PyType_GetDict(type); + assert(PyDict_Check(dict)); + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + PyType_Modified(type); + return -1; + } + } + Py_DECREF(dict); + PyType_Modified(type); + return 0; +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1474,18 +1898,44 @@ type_get_annotations(PyTypeObject *type, void *context) } PyObject *annotations; - PyObject *dict = lookup_tp_dict(type); - annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + Py_DECREF(dict); + return NULL; + } if (annotations) { - if (Py_TYPE(annotations)->tp_descr_get) { - annotations = Py_TYPE(annotations)->tp_descr_get( - annotations, NULL, (PyObject *)type); - } else { - Py_INCREF(annotations); + descrgetfunc get = Py_TYPE(annotations)->tp_descr_get; + if (get) { + Py_SETREF(annotations, get(annotations, NULL, (PyObject *)type)); } } - else if (!PyErr_Occurred()) { - annotations = PyDict_New(); + else { + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(dict); + Py_DECREF(annotate); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_DECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1496,6 +1946,7 @@ type_get_annotations(PyTypeObject *type, void *context) } } } + Py_DECREF(dict); return annotations; } @@ -1510,37 +1961,47 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (result < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; } } - - if (result == 0) { - PyType_Modified(type); + if (result < 0) { + Py_DECREF(dict); + return -1; } - return result; + else if (result == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + PyType_Modified(type); + Py_DECREF(dict); + return -1; + } + } + PyType_Modified(type); + Py_DECREF(dict); + return 0; } static PyObject * type_get_type_params(PyTypeObject *type, void *context) { - PyObject *params = PyDict_GetItemWithError(lookup_tp_dict(type), &_Py_ID(__type_params__)); - - if (params) { - return Py_NewRef(params); - } - if (PyErr_Occurred()) { - return NULL; + if (type == &PyType_Type) { + return PyTuple_New(0); } - return PyTuple_New(0); + PyObject *params; + if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { + return PyTuple_New(0); + } + return params; } static int @@ -1598,53 +2059,59 @@ static PyGetSetDef type_getsets[] = { {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, {"__mro__", (getter)type_get_mro, NULL, NULL}, - {"__module__", (getter)type_module, (setter)type_set_module, NULL}, + {"__module__", (getter)type_get_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, {"__dict__", (getter)type_dict, NULL, NULL}, {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, + {"__annotate__", (getter)type_get_annotate, (setter)type_set_annotate, NULL}, {"__type_params__", (getter)type_get_type_params, (setter)type_set_type_params, NULL}, {0} }; static PyObject * -type_repr(PyTypeObject *type) +type_repr(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; if (type->tp_name == NULL) { // type_repr() called before the type is fully initialized // by PyType_Ready(). return PyUnicode_FromFormat("", type); } - PyObject *mod, *name, *rtn; - - mod = type_module(type, NULL); - if (mod == NULL) + PyObject *mod = type_module(type); + if (mod == NULL) { PyErr_Clear(); + } else if (!PyUnicode_Check(mod)) { - Py_SETREF(mod, NULL); + Py_CLEAR(mod); } - name = type_qualname(type, NULL); + + PyObject *name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; - } - - if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) - rtn = PyUnicode_FromFormat("", mod, name); - else - rtn = PyUnicode_FromFormat("", type->tp_name); + } + PyObject *result; + if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) { + result = PyUnicode_FromFormat("", mod, name); + } + else { + result = PyUnicode_FromFormat("", type->tp_name); + } Py_XDECREF(mod); Py_DECREF(name); - return rtn; + + return result; } static PyObject * -type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) +type_call(PyObject *self, PyObject *args, PyObject *kwds) { + PyTypeObject *type = (PyTypeObject *)self; PyObject *obj; PyThreadState *tstate = _PyThreadState_GET(); @@ -1710,7 +2177,7 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject * _PyType_NewManagedObject(PyTypeObject *type) { - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(_PyType_IS_GC(type)); assert(type->tp_new == PyBaseObject_Type.tp_new); assert(type->tp_alloc == PyType_GenericAlloc); @@ -1719,11 +2186,6 @@ _PyType_NewManagedObject(PyTypeObject *type) if (obj == NULL) { return PyErr_NoMemory(); } - _PyObject_DictOrValuesPointer(obj)->dict = NULL; - if (_PyObject_InitInlineValues(obj, type)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -1737,10 +2199,14 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) * flag to indicate when that is safe) it does not seem worth the memory * savings. An example type that doesn't need the +1 is a subclass of * tuple. See GH-100659 and GH-81381. */ - const size_t size = _PyObject_VAR_SIZE(type, nitems+1); + size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); - char *alloc = PyObject_Malloc(size + presize); + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + assert(type->tp_itemsize == 0); + size += _PyInlineValuesSize(type); + } + char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); } @@ -1748,6 +2214,8 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) if (presize) { ((PyObject **)alloc)[0] = NULL; ((PyObject **)alloc)[1] = NULL; + } + if (PyType_IS_GC(type)) { _PyObject_GC_Link(obj); } memset(obj, '\0', size); @@ -1758,6 +2226,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) else { _PyObject_InitVar((PyVarObject *)obj, type, nitems); } + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(obj, type); + } return obj; } @@ -1907,6 +2378,10 @@ subtype_clear(PyObject *self) if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { PyObject_ClearManagedDict(self); } + else { + assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) == + (type->tp_flags & Py_TPFLAGS_INLINE_VALUES)); + } } else if (type->tp_dictoffset != base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2034,15 +2509,7 @@ subtype_dealloc(PyObject *self) finalizers since they might rely on part of the object being finalized that has already been destroyed. */ if (type->tp_weaklistoffset && !base->tp_weaklistoffset) { - /* Modeled after GET_WEAKREFS_LISTPTR(). - - This is never triggered for static types so we can avoid the - (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ - PyWeakReference **list = \ - _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(self); - while (*list) { - _PyWeakref_ClearRef(*list); - } + _PyWeakref_ClearWeakRefsNoCallbacks(self); } } @@ -2057,14 +2524,7 @@ subtype_dealloc(PyObject *self) /* If we added a dict, DECREF it, or free inline values. */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - _PyObject_FreeInstanceAttributes(self); - } - else { - Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr)); - } - dorv_ptr->values = NULL; + PyObject_ClearManagedDict(self); } else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2160,27 +2620,35 @@ type_is_subtype_base_chain(PyTypeObject *a, PyTypeObject *b) return (b == &PyBaseObject_Type); } -int -PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +static int +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - - mro = lookup_tp_mro(a); - if (mro != NULL) { + int res; + if (a_mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ Py_ssize_t i, n; - assert(PyTuple_Check(mro)); - n = PyTuple_GET_SIZE(mro); + assert(PyTuple_Check(a_mro)); + n = PyTuple_GET_SIZE(a_mro); + res = 0; for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; + if (PyTuple_GET_ITEM(a_mro, i) == (PyObject *)b) { + res = 1; + break; + } } - return 0; } - else + else { /* a is not completely initialized yet; follow tp_base */ - return type_is_subtype_base_chain(a, b); + res = type_is_subtype_base_chain(a, b); + } + return res; +} + +int +PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) +{ + return is_subtype_with_mro(a->tp_mro, a, b); } /* Routines to do a method lookup in the type without looking in the @@ -2190,7 +2658,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) Variants: - _PyObject_LookupSpecial() returns NULL without raising an exception - when the _PyType_Lookup() call fails; + when the _PyType_LookupRef() call fails; - lookup_maybe_method() and lookup_method() are internal routines similar to _PyObject_LookupSpecial(), but can return unbound PyFunction @@ -2203,30 +2671,48 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) { PyObject *res; - res = _PyType_Lookup(Py_TYPE(self), attr); + res = _PyType_LookupRef(Py_TYPE(self), attr); if (res != NULL) { descrgetfunc f; - if ((f = Py_TYPE(res)->tp_descr_get) == NULL) - Py_INCREF(res); - else - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if ((f = Py_TYPE(res)->tp_descr_get) != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } } return res; } +/* Steals a reference to self */ PyObject * -_PyObject_LookupSpecialId(PyObject *self, _Py_Identifier *attrid) +_PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null) { - PyObject *attr = _PyUnicode_FromId(attrid); /* borrowed */ - if (attr == NULL) + PyObject *res; + + res = _PyType_LookupRef(Py_TYPE(self), attr); + if (res == NULL) { + Py_DECREF(self); + *self_or_null = NULL; return NULL; - return _PyObject_LookupSpecial(self, attr); + } + + if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + /* Avoid temporary PyMethodObject */ + *self_or_null = self; + } + else { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } + *self_or_null = NULL; + Py_DECREF(self); + } + return res; } static PyObject * lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) { - PyObject *res = _PyType_Lookup(Py_TYPE(self), attr); + PyObject *res = _PyType_LookupRef(Py_TYPE(self), attr); if (res == NULL) { return NULL; } @@ -2234,16 +2720,12 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ *unbound = 1; - Py_INCREF(res); } else { *unbound = 0; descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f == NULL) { - Py_INCREF(res); - } - else { - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); } } return res; @@ -2542,8 +3024,10 @@ pmerge(PyObject *acc, PyObject **to_merge, Py_ssize_t to_merge_size) } static PyObject * -mro_implementation(PyTypeObject *type) +mro_implementation_unlocked(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + if (!_PyType_IsReady(type)) { if (PyType_Ready(type) < 0) return NULL; @@ -2623,6 +3107,16 @@ mro_implementation(PyTypeObject *type) return result; } +static PyObject * +mro_implementation(PyTypeObject *type) +{ + PyObject *mro; + BEGIN_TYPE_LOCK(); + mro = mro_implementation_unlocked(type); + END_TYPE_LOCK(); + return mro; +} + /*[clinic input] type.mro @@ -2661,7 +3155,7 @@ mro_check(PyTypeObject *type, PyObject *mro) } PyTypeObject *base = (PyTypeObject*)obj; - if (!PyType_IsSubtype(solid, solid_base(base))) { + if (!is_subtype_with_mro(lookup_tp_mro(solid), solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, "mro() returned base with unsuitable layout ('%.500s')", @@ -2692,6 +3186,9 @@ mro_invoke(PyTypeObject *type) { PyObject *mro_result; PyObject *new_mro; + + ASSERT_TYPE_LOCK_HELD(); + const int custom = !Py_IS_TYPE(type, &PyType_Type); if (custom) { @@ -2704,7 +3201,7 @@ mro_invoke(PyTypeObject *type) Py_DECREF(mro_meth); } else { - mro_result = mro_implementation(type); + mro_result = mro_implementation_unlocked(type); } if (mro_result == NULL) return NULL; @@ -2751,8 +3248,10 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal(PyTypeObject *type, PyObject **p_old_mro) +mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *new_mro, *old_mro; int reent; @@ -2772,7 +3271,7 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 0; } - set_tp_mro(type, new_mro); + set_tp_mro(type, new_mro, initial); type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden @@ -2786,7 +3285,7 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) else { /* For static builtin types, this is only called during init before the method cache has been populated. */ - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); } if (p_old_mro != NULL) @@ -2797,6 +3296,16 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 1; } +static int +mro_internal(PyTypeObject *type, PyObject **p_old_mro) +{ + int res; + BEGIN_TYPE_LOCK(); + res = mro_internal_unlocked(type, 0, p_old_mro); + END_TYPE_LOCK(); + return res; +} + /* Calculate the best base amongst multiple base classes. This is the first one that's on the path to the "solid base". */ @@ -2973,19 +3482,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) return func(descr, obj, value); } /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } if (value != NULL && !PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - Py_XSETREF(*dictptr, Py_XNewRef(value)); + + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + else { + dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_CLEAR(*dictptr); + *dictptr = Py_XNewRef(value); + } return 0; } @@ -3019,21 +3535,21 @@ subtype_getweakref(PyObject *obj, void *context) static PyGetSetDef subtype_getsets_full[] = { {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables (if defined)")}, + PyDoc_STR("dictionary for instance variables")}, {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object (if defined)")}, + PyDoc_STR("list of weak references to the object")}, {0} }; static PyGetSetDef subtype_getsets_dict_only[] = { {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables (if defined)")}, + PyDoc_STR("dictionary for instance variables")}, {0} }; static PyGetSetDef subtype_getsets_weakref_only[] = { {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object (if defined)")}, + PyDoc_STR("list of weak references to the object")}, {0} }; @@ -3080,7 +3596,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return type->tp_flags; + return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); } @@ -3380,7 +3896,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - set_tp_bases(type, Py_NewRef(ctx->bases)); + set_tp_bases(type, Py_NewRef(ctx->bases), 1); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3395,6 +3911,8 @@ type_new_alloc(type_new_ctx *ctx) et->ht_module = NULL; et->_ht_tpname = NULL; + _PyObject_SetDeferredRefcount((PyObject *)et); + return type; } @@ -3434,18 +3952,13 @@ type_new_set_module(PyTypeObject *type) return 0; } - PyObject *module = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); - if (module == NULL) { - if (PyErr_Occurred()) { - return -1; - } - return 0; - } - - if (PyDict_SetItem(dict, &_Py_ID(__module__), module) < 0) { - return -1; + PyObject *module; + r = PyDict_GetItemRef(globals, &_Py_ID(__name__), &module); + if (module) { + r = PyDict_SetItem(dict, &_Py_ID(__module__), module); + Py_DECREF(module); } - return 0; + return r; } @@ -3456,23 +3969,24 @@ type_new_set_ht_name(PyTypeObject *type) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; PyObject *dict = lookup_tp_dict(type); - PyObject *qualname = PyDict_GetItemWithError(dict, &_Py_ID(__qualname__)); + PyObject *qualname; + if (PyDict_GetItemRef(dict, &_Py_ID(__qualname__), &qualname) < 0) { + return -1; + } if (qualname != NULL) { if (!PyUnicode_Check(qualname)) { PyErr_Format(PyExc_TypeError, "type __qualname__ must be a str, not %s", Py_TYPE(qualname)->tp_name); + Py_DECREF(qualname); return -1; } - et->ht_qualname = Py_NewRef(qualname); + et->ht_qualname = qualname; if (PyDict_DelItem(dict, &_Py_ID(__qualname__)) < 0) { return -1; } } else { - if (PyErr_Occurred()) { - return -1; - } et->ht_qualname = Py_NewRef(et->ht_name); } return 0; @@ -3506,7 +4020,7 @@ type_new_set_doc(PyTypeObject *type) // Silently truncate the docstring if it contains a null byte Py_ssize_t size = strlen(doc_str) + 1; - char *tp_doc = (char *)PyObject_Malloc(size); + char *tp_doc = (char *)PyMem_Malloc(size); if (tp_doc == NULL) { PyErr_NoMemory(); return -1; @@ -3841,6 +4355,17 @@ type_new_impl(type_new_ctx *ctx) // Put the proper slots in place fixup_slot_dispatchers(type); + if (!_PyDict_HasOnlyStringKeys(type->tp_dict)) { + if (PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "non-string key in the __dict__ of class %.200s", + type->tp_name) == -1) + { + goto error; + } + } + if (type_new_set_names(type) < 0) { goto error; } @@ -4179,12 +4704,12 @@ _PyType_FromMetaclass_impl( goto finally; } if (slot->pfunc == NULL) { - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); tp_doc = NULL; } else { size_t len = strlen(slot->pfunc)+1; - tp_doc = PyObject_Malloc(len); + tp_doc = PyMem_Malloc(len); if (tp_doc == NULL) { PyErr_NoMemory(); goto finally; @@ -4252,16 +4777,12 @@ _PyType_FromMetaclass_impl( goto finally; } if (!_PyType_HasFeature(b, Py_TPFLAGS_IMMUTABLETYPE)) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, - 0, - "Creating immutable type %s from mutable base %s is " - "deprecated, and slated to be disallowed in Python 3.14.", - spec->name, - b->tp_name)) - { - goto finally; - } + PyErr_Format( + PyExc_TypeError, + "Creating immutable type %s from mutable base %N", + spec->name, b + ); + goto finally; } } } @@ -4365,7 +4886,7 @@ _PyType_FromMetaclass_impl( /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -4514,7 +5035,7 @@ _PyType_FromMetaclass_impl( Py_CLEAR(res); } Py_XDECREF(bases); - PyObject_Free(tp_doc); + PyMem_Free(tp_doc); Py_XDECREF(ht_name); PyMem_Free(_ht_tpname); return (PyObject*)res; @@ -4557,6 +5078,12 @@ PyType_GetQualName(PyTypeObject *type) return type_qualname(type, NULL); } +PyObject * +PyType_GetModuleName(PyTypeObject *type) +{ + return type_module(type); +} + void * PyType_GetSlot(PyTypeObject *type, int slot) { @@ -4617,21 +5144,39 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +static inline PyObject * +get_module_by_def(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // type_ready_mro() ensures that no heap type is + // contained in a static type MRO. + return NULL; + } + else { + PyHeapTypeObject *ht = (PyHeapTypeObject*)type; + PyObject *module = ht->ht_module; + if (module && _PyModule_GetDef(module) == def) { + return module; + } + } + + PyObject *res = NULL; + BEGIN_TYPE_LOCK(); + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); - // mro_invoke() ensures that the type MRO cannot be empty, so we don't have - // to check i < PyTuple_GET_SIZE(mro) at the first loop iteration. + // mro_invoke() ensures that the type MRO cannot be empty. assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 0; i < n; i++) { + for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped @@ -4641,15 +5186,42 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; if (module && _PyModule_GetDef(module) == def) { - return module; + res = module; + break; } } + END_TYPE_LOCK(); + return res; +} - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' has the given module", - type->tp_name); - return NULL; +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + PyObject *module = get_module_by_def(type, def); + if (module == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' has the given module", + type->tp_name); + } + return module; +} + +PyObject * +_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, + PyModuleDef *def) +{ + PyObject *module = get_module_by_def(left, def); + if (module == NULL) { + module = get_module_by_def(right, def); + if (module == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " + "the given module", left->tp_name, right->tp_name); + } + } + return module; } void * @@ -4687,15 +5259,12 @@ PyObject_GetItemData(PyObject *obj) static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { - Py_hash_t hash; - if (!PyUnicode_CheckExact(name) || - (hash = _PyASCIIObject_CAST(name)->hash) == -1) - { - hash = PyObject_Hash(name); - if (hash == -1) { - *error = -1; - return NULL; - } + ASSERT_TYPE_LOCK_HELD(); + + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + *error = -1; + return NULL; } /* Look in tp_dict of types in MRO */ @@ -4723,14 +5292,13 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - res = _PyDict_GetItem_KnownHash(dict, name, hash); - if (res != NULL) { - break; - } - if (PyErr_Occurred()) { + if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; } + if (res != NULL) { + break; + } } *error = 0; done: @@ -4756,10 +5324,68 @@ is_dunder_name(PyObject *name) return 0; } +static PyObject * +update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) +{ + _Py_atomic_store_uint32_relaxed(&entry->version, version_tag); + _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ + assert(_PyASCIIObject_CAST(name)->hash != -1); + OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); + // We're releasing this under the lock for simplicity sake because it's always a + // exact unicode object or Py_None so it's safe to do so. + PyObject *old_name = entry->name; + _Py_atomic_store_ptr_relaxed(&entry->name, Py_NewRef(name)); + return old_name; +} + +#if Py_GIL_DISABLED + +static void +update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, + unsigned int version_tag, PyObject *value) +{ + _PySeqLock_LockWrite(&entry->sequence); + + // update the entry + if (entry->name == name && + entry->value == value && + entry->version == version_tag) { + // We raced with another update, bail and restore previous sequence. + _PySeqLock_AbandonWrite(&entry->sequence); + return; + } + + PyObject *old_value = update_cache(entry, name, version_tag, value); + + // Then update sequence to the next valid value + _PySeqLock_UnlockWrite(&entry->sequence); + + Py_DECREF(old_value); +} + +#endif + +void +_PyTypes_AfterFork(void) +{ +#ifdef Py_GIL_DISABLED + struct type_cache *cache = get_type_cache(); + for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { + struct type_cache_entry *entry = &cache->hashtable[i]; + if (_PySeqLock_AfterFork(&entry->sequence)) { + // Entry was in the process of updating while forking, clear it... + entry->value = NULL; + Py_SETREF(entry->name, Py_None); + entry->version = 0; + } + } +#endif +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +_PyType_LookupRef(PyTypeObject *type, PyObject *name) { PyObject *res; int error; @@ -4768,20 +5394,63 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) unsigned int h = MCACHE_HASH_METHOD(type, name); struct type_cache *cache = get_type_cache(); struct type_cache_entry *entry = &cache->hashtable[h]; +#ifdef Py_GIL_DISABLED + // synchronize-with other writing threads by doing an acquire load on the sequence + while (1) { + uint32_t sequence = _PySeqLock_BeginRead(&entry->sequence); + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); + if (entry_version == type_version && + _Py_atomic_load_ptr_relaxed(&entry->name) == name) { + OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); + OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); + // If the sequence is still valid then we're done + if (value == NULL || _Py_TryIncref(value)) { + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + return value; + } + Py_XDECREF(value); + } + else { + // If we can't incref the object we need to fallback to locking + break; + } + } + else { + // cache miss + break; + } + } +#else if (entry->version == type->tp_version_tag && entry->name == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + Py_XINCREF(entry->value); return entry->value; } +#endif OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); + // We need to atomically do the lookup and capture the version before + // anyone else can modify our mro or mutate the type. + + int has_version = 0; + int version = 0; + BEGIN_TYPE_LOCK(); res = find_name_in_mro(type, name, &error); + if (MCACHE_CACHEABLE_NAME(name)) { + has_version = assign_version_tag(interp, type); + version = type->tp_version_tag; + } + END_TYPE_LOCK(); + /* Only put NULL results into cache if there was no error. */ if (error) { /* It's not ideal to clear the error condition, @@ -4798,31 +5467,73 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return NULL; } - if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(interp, type)) { - h = MCACHE_HASH_METHOD(type, name); - struct type_cache_entry *entry = &cache->hashtable[h]; - entry->version = type->tp_version_tag; - entry->value = res; /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); - OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); - Py_SETREF(entry->name, Py_NewRef(name)); + if (has_version) { +#if Py_GIL_DISABLED + update_cache_gil_disabled(entry, name, version, res); +#else + PyObject *old_value = update_cache(entry, name, version, res); + Py_DECREF(old_value); +#endif } return res; } PyObject * -_PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) +_PyType_Lookup(PyTypeObject *type, PyObject *name) { - PyObject *oname; - oname = _PyUnicode_FromId(name); /* borrowed */ - if (oname == NULL) - return NULL; - return _PyType_Lookup(type, oname); + PyObject *res = _PyType_LookupRef(type, name); + Py_XDECREF(res); + return res; +} + +static void +set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + ASSERT_TYPE_LOCK_HELD(); + self->tp_flags = (self->tp_flags & ~mask) | flags; +} + +void +_PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags(self, mask, flags); + END_TYPE_LOCK(); +} + +static void +set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + if (PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) || + (self->tp_flags & mask) == flags) + { + return; + } + + set_flags(self, mask, flags); + + PyObject *children = _PyType_GetSubclasses(self); + if (children == NULL) { + return; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(children); i++) { + PyObject *child = PyList_GET_ITEM(children, i); + set_flags_recursive((PyTypeObject *)child, mask, flags); + } + Py_DECREF(children); +} + +void +_PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags_recursive(self, mask, flags); + END_TYPE_LOCK(); } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. The argument suppress_missing_attribute is used to provide a fast path for hasattr. The possible values are: @@ -4858,10 +5569,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin meta_get = NULL; /* Look for the attribute in the metatype */ - meta_attribute = _PyType_Lookup(metatype, name); + meta_attribute = _PyType_LookupRef(metatype, name); if (meta_attribute != NULL) { - Py_INCREF(meta_attribute); meta_get = Py_TYPE(meta_attribute)->tp_descr_get; if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { @@ -4878,10 +5588,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* No data descriptor found on metatype. Look in tp_dict of this * type and its bases */ - attribute = _PyType_Lookup(type, name); + attribute = _PyType_LookupRef(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ - Py_INCREF(attribute); descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; Py_XDECREF(meta_attribute); @@ -4922,20 +5631,57 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin // signal the caller we have not set an PyExc_AttributeError and gave up *suppress_missing_attribute = 1; } - return NULL; -} + return NULL; +} + +/* This is similar to PyObject_GenericGetAttr(), + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. */ +PyObject * +_Py_type_getattro(PyObject *type, PyObject *name) +{ + return _Py_type_getattro_impl((PyTypeObject *)type, name, NULL); +} + +static int +type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, + PyObject *value, PyObject **old_value) +{ + // We don't want any re-entrancy between when we update the dict + // and call type_modified_unlocked, including running the destructor + // of the current value as it can observe the cache in an inconsistent + // state. Because we have an exact unicode and our dict has exact + // unicodes we know that this will all complete without releasing + // the locks. + if (_PyDict_GetItemRef_Unicode_LockHeld(dict, name, old_value) < 0) { + return -1; + } + + /* Clear the VALID_VERSION flag of 'type' and all its + subclasses. This could possibly be unified with the + update_subclasses() recursion in update_slot(), but carefully: + they each have their own conditions on which to stop + recursing into subclasses. */ + type_modified_unlocked(type); + + if (_PyDict_SetItem_LockHeld(dict, name, value) < 0) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + ((PyTypeObject*)type)->tp_name, name); + _PyObject_SetAttributeErrorContext((PyObject *)type, name); + return -1; + } + + if (is_dunder_name(name)) { + return update_slot(type, name); + } -/* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ -PyObject * -_Py_type_getattro(PyTypeObject *type, PyObject *name) -{ - return _Py_type_getattro_impl(type, name, NULL); + return 0; } static int -type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) +type_setattro(PyObject *self, PyObject *name, PyObject *value) { + PyTypeObject *type = (PyTypeObject *)self; int res; if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { PyErr_Format( @@ -4944,45 +5690,70 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) name, type->tp_name); return -1; } - if (PyUnicode_Check(name)) { - if (PyUnicode_CheckExact(name)) { - Py_INCREF(name); - } - else { - name = _PyUnicode_Copy(name); - if (name == NULL) - return -1; - } - /* bpo-40521: Interned strings are shared by all subinterpreters */ + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return -1; + } + + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + if (!PyUnicode_CHECK_INTERNED(name)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (!PyUnicode_CHECK_INTERNED(name)) { - PyUnicode_InternInPlace(&name); - if (!PyUnicode_CHECK_INTERNED(name)) { - PyErr_SetString(PyExc_MemoryError, - "Out of memory interning an attribute name"); - Py_DECREF(name); - return -1; - } + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; } } - else { - /* Will fail in _PyObject_GenericSetAttrWithDict. */ - Py_INCREF(name); + + PyTypeObject *metatype = Py_TYPE(type); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); + + PyObject *old_value = NULL; + PyObject *descr = _PyType_LookupRef(metatype, name); + if (descr != NULL) { + descrsetfunc f = Py_TYPE(descr)->tp_descr_set; + if (f != NULL) { + res = f(descr, (PyObject *)type, value); + goto done; + } } - res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); - if (res == 0) { - /* Clear the VALID_VERSION flag of 'type' and all its - subclasses. This could possibly be unified with the - update_subclasses() recursion in update_slot(), but carefully: - they each have their own conditions on which to stop - recursing into subclasses. */ - PyType_Modified(type); - if (is_dunder_name(name)) { - res = update_slot(type, name); + PyObject *dict = type->tp_dict; + if (dict == NULL) { + // We don't just do PyType_Ready because we could already be readying + BEGIN_TYPE_LOCK(); + dict = type->tp_dict; + if (dict == NULL) { + dict = type->tp_dict = PyDict_New(); + } + END_TYPE_LOCK(); + if (dict == NULL) { + res = -1; + goto done; } - assert(_PyType_CheckConsistency(type)); } + + BEGIN_TYPE_DICT_LOCK(dict); + res = type_update_dict(type, (PyDictObject *)dict, name, value, &old_value); + assert(_PyType_CheckConsistency(type)); + END_TYPE_DICT_LOCK(); + +done: Py_DECREF(name); + Py_XDECREF(descr); + Py_XDECREF(old_value); return res; } @@ -5000,7 +5771,7 @@ type_dealloc_common(PyTypeObject *type) static void -clear_static_tp_subclasses(PyTypeObject *type) +clear_static_tp_subclasses(PyTypeObject *type, int isbuiltin) { PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { @@ -5037,56 +5808,83 @@ clear_static_tp_subclasses(PyTypeObject *type) continue; } // All static builtin subtypes should have been finalized already. - assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(!isbuiltin || !(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); Py_DECREF(subclass); } +#else + (void)isbuiltin; #endif clear_tp_subclasses(type); } static void -clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { - if (_Py_IsMainInterpreter(interp)) { + if (final) { Py_CLEAR(type->tp_cache); } clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); - clear_static_tp_subclasses(type); + clear_tp_bases(type, final); + clear_tp_mro(type, final); + clear_static_tp_subclasses(type, isbuiltin); } -void -_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) + +static void +fini_static_type(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - clear_static_type_objects(interp, type); + clear_static_type_objects(interp, type, isbuiltin, final); - if (_Py_IsMainInterpreter(interp)) { + if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; + _PyType_SetVersion(type, 0); } _PyStaticType_ClearWeakRefs(interp, type); - static_builtin_state_clear(interp, type); - /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ + managed_static_type_state_clear(interp, type, isbuiltin, final); +} + +void +_PyTypes_FiniExtTypes(PyInterpreterState *interp) +{ + for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { + if (interp->types.for_extensions.num_initialized == 0) { + break; + } + int64_t count = 0; + PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); + if (type == NULL) { + continue; + } + int final = (count == 1); + fini_static_type(interp, type, 0, final); + } +} + +void +_PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) +{ + fini_static_type(interp, type, 1, _Py_IsMainInterpreter(interp)); } static void -type_dealloc(PyTypeObject *type) +type_dealloc(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; + // Assert this is a heap-allocated type object _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); _PyObject_GC_UNTRACK(type); - type_dealloc_common(type); // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 @@ -5103,7 +5901,7 @@ type_dealloc(PyTypeObject *type) /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. */ - PyObject_Free((char *)type->tp_doc); + PyMem_Free((char *)type->tp_doc); PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_XDECREF(et->ht_name); @@ -5252,8 +6050,10 @@ static PyMethodDef type_methods[] = { TYPE___SUBCLASSES___METHODDEF {"__prepare__", _PyCFunction_CAST(type_prepare), METH_FASTCALL | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("__prepare__() -> dict\n" - "used to create the namespace for the class statement")}, + PyDoc_STR("__prepare__($cls, name, bases, /, **kwds)\n" + "--\n" + "\n" + "Create the namespace for the class statement")}, TYPE___INSTANCECHECK___METHODDEF TYPE___SUBCLASSCHECK___METHODDEF TYPE___DIR___METHODDEF @@ -5266,8 +6066,10 @@ PyDoc_STRVAR(type_doc, "type(name, bases, dict, **kwds) -> a new type"); static int -type_traverse(PyTypeObject *type, visitproc visit, void *arg) +type_traverse(PyObject *self, visitproc visit, void *arg) { + PyTypeObject *type = (PyTypeObject *)self; + /* Because of type_is_gc(), the collector only calls this for heaptypes. */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { @@ -5295,8 +6097,10 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) } static int -type_clear(PyTypeObject *type) +type_clear(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; + /* Because of type_is_gc(), the collector only calls this for heaptypes. */ _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -5305,7 +6109,7 @@ type_clear(PyTypeObject *type) the dict, so that other objects caught in a reference cycle don't start calling destroyed methods. - Otherwise, the we need to clear tp_mro, which is + Otherwise, we need to clear tp_mro, which is part of a hard cycle (its first element is the class itself) that won't be broken otherwise (it's a tuple and tuples don't have a tp_clear handler). @@ -5343,9 +6147,9 @@ type_clear(PyTypeObject *type) } static int -type_is_gc(PyTypeObject *type) +type_is_gc(PyObject *type) { - return type->tp_flags & Py_TPFLAGS_HEAPTYPE; + return ((PyTypeObject *)type)->tp_flags & Py_TPFLAGS_HEAPTYPE; } @@ -5358,28 +6162,28 @@ PyTypeObject PyType_Type = { "type", /* tp_name */ sizeof(PyHeapTypeObject), /* tp_basicsize */ sizeof(PyMemberDef), /* tp_itemsize */ - (destructor)type_dealloc, /* tp_dealloc */ + type_dealloc, /* tp_dealloc */ offsetof(PyTypeObject, tp_vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)type_repr, /* tp_repr */ + type_repr, /* tp_repr */ &type_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)type_call, /* tp_call */ + type_call, /* tp_call */ 0, /* tp_str */ - (getattrofunc)_Py_type_getattro, /* tp_getattro */ - (setattrofunc)type_setattro, /* tp_setattro */ + _Py_type_getattro, /* tp_getattro */ + type_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_ITEMS_AT_END, /* tp_flags */ type_doc, /* tp_doc */ - (traverseproc)type_traverse, /* tp_traverse */ - (inquiry)type_clear, /* tp_clear */ + type_traverse, /* tp_traverse */ + type_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyTypeObject, tp_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -5396,7 +6200,7 @@ PyTypeObject PyType_Type = { 0, /* tp_alloc */ type_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - (inquiry)type_is_gc, /* tp_is_gc */ + type_is_gc, /* tp_is_gc */ .tp_vectorcall = type_vectorcall, }; @@ -5511,15 +6315,19 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } comma_w_quotes_sep = PyUnicode_FromString("', '"); + if (!comma_w_quotes_sep) { + Py_DECREF(sorted_methods); + return NULL; + } joined = PyUnicode_Join(comma_w_quotes_sep, sorted_methods); - method_count = PyObject_Length(sorted_methods); - Py_DECREF(sorted_methods); + Py_DECREF(comma_w_quotes_sep); if (joined == NULL) { - Py_DECREF(comma_w_quotes_sep); + Py_DECREF(sorted_methods); return NULL; } + method_count = PyObject_Length(sorted_methods); + Py_DECREF(sorted_methods); if (method_count == -1) { - Py_DECREF(comma_w_quotes_sep); Py_DECREF(joined); return NULL; } @@ -5531,17 +6339,12 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) method_count > 1 ? "s" : "", joined); Py_DECREF(joined); - Py_DECREF(comma_w_quotes_sep); return NULL; } PyObject *obj = type->tp_alloc(type, 0); if (obj == NULL) { return NULL; } - if (_PyObject_InitializeDict(obj)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -5558,7 +6361,7 @@ object_repr(PyObject *self) PyObject *mod, *name, *rtn; type = Py_TYPE(self); - mod = type_module(type, NULL); + mod = type_module(type); if (mod == NULL) PyErr_Clear(); else if (!PyUnicode_Check(mod)) { @@ -5634,6 +6437,12 @@ object_richcompare(PyObject *self, PyObject *other, int op) return res; } +PyObject* +_Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op) +{ + return object_richcompare(self, other, op); +} + static PyObject * object_get_class(PyObject *self, void *closure) { @@ -5719,6 +6528,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } + if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != + ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) + { + goto differs; + } /* The above does not check for the preheader */ if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) @@ -5735,29 +6549,13 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* return 0; } + + static int -object_set_class(PyObject *self, PyObject *value, void *closure) +object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) { PyTypeObject *oldto = Py_TYPE(self); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "can't delete __class__ attribute"); - return -1; - } - if (!PyType_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__class__ must be set to a class, not '%s' object", - Py_TYPE(value)->tp_name); - return -1; - } - PyTypeObject *newto = (PyTypeObject *)value; - - if (PySys_Audit("object.__setattr__", "OsO", - self, "__class__", value) < 0) { - return -1; - } - /* In versions of CPython prior to 3.5, the code in compatible_for_assignment was not set up to correctly check for memory layout / slot / etc. compatibility for non-HEAPTYPE classes, so we just @@ -5820,21 +6618,28 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); - _PyObject_GetDictPtr(self); - if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) - { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - return -1; + if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_GetManagedDict(self); + if (dict == NULL) { + dict = _PyObject_MaterializeManagedDict_LockHeld(self); + if (dict == NULL) { + return -1; + } + } + + assert(_PyObject_GetManagedDict(self) == dict); + + if (_PyDict_DetachFromObject(dict, self) < 0) { + return -1; + } + } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } + Py_SET_TYPE(self, newto); - if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) - Py_DECREF(oldto); + return 0; } else { @@ -5842,6 +6647,48 @@ object_set_class(PyObject *self, PyObject *value, void *closure) } } +static int +object_set_class(PyObject *self, PyObject *value, void *closure) +{ + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "can't delete __class__ attribute"); + return -1; + } + if (!PyType_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__class__ must be set to a class, not '%s' object", + Py_TYPE(value)->tp_name); + return -1; + } + PyTypeObject *newto = (PyTypeObject *)value; + + if (PySys_Audit("object.__setattr__", "OsO", + self, "__class__", value) < 0) { + return -1; + } + +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +#endif + PyTypeObject *oldto = Py_TYPE(self); + int res = object_set_class_world_stopped(self, newto); +#ifdef Py_GIL_DISABLED + _PyEval_StartTheWorld(interp); +#endif + if (res == 0) { + if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_DECREF(oldto); + } + + RARE_EVENT_INC(set_class); + return 0; + } + return res; +} + static PyGetSetDef object_getsets[] = { {"__class__", object_get_class, object_set_class, PyDoc_STR("the object's class")}, @@ -5884,24 +6731,22 @@ _PyType_GetSlotNames(PyTypeObject *cls) /* Get the slot names from the cache in the class if possible. */ PyObject *dict = lookup_tp_dict(cls); - slotnames = PyDict_GetItemWithError(dict, &_Py_ID(__slotnames__)); + if (PyDict_GetItemRef(dict, &_Py_ID(__slotnames__), &slotnames) < 0) { + return NULL; + } if (slotnames != NULL) { if (slotnames != Py_None && !PyList_Check(slotnames)) { PyErr_Format(PyExc_TypeError, "%.200s.__slotnames__ should be a list or None, " "not %.200s", cls->tp_name, Py_TYPE(slotnames)->tp_name); + Py_DECREF(slotnames); return NULL; } - return Py_NewRef(slotnames); - } - else { - if (PyErr_Occurred()) { - return NULL; - } - /* The class does not have the slot names cached yet. */ + return slotnames; } + /* The class does not have the slot names cached yet. */ copyreg = import_copyreg(); if (copyreg == NULL) return NULL; @@ -6293,6 +7138,7 @@ reduce_newobj(PyObject *obj) } else { /* args == NULL */ + Py_DECREF(copyreg); Py_DECREF(kwargs); PyErr_BadInternalCall(); return NULL; @@ -6485,8 +7331,11 @@ object___sizeof___impl(PyObject *self) res = 0; isize = Py_TYPE(self)->tp_itemsize; - if (isize > 0) - res = Py_SIZE(self) * isize; + if (isize > 0) { + /* This assumes that ob_size is valid if tp_itemsize is not 0, + which isn't true for PyLongObject. */ + res = _PyVarObject_CAST(self)->ob_size * isize; + } res += Py_TYPE(self)->tp_basicsize; return PyLong_FromSsize_t(res); @@ -6550,7 +7399,7 @@ static PyMethodDef object_methods[] = { OBJECT___REDUCE_EX___METHODDEF OBJECT___REDUCE___METHODDEF OBJECT___GETSTATE___METHODDEF - {"__subclasshook__", object_subclasshook, METH_CLASS | METH_VARARGS, + {"__subclasshook__", object_subclasshook, METH_CLASS | METH_O, object_subclasshook_doc}, {"__init_subclass__", object_init_subclass, METH_CLASS | METH_NOARGS, object_init_subclass_doc}, @@ -6580,7 +7429,7 @@ PyTypeObject PyBaseObject_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)_Py_HashPointer, /* tp_hash */ + PyObject_GenericHash, /* tp_hash */ 0, /* tp_call */ object_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -6653,7 +7502,7 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) int err; PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(dict, name, descr) == NULL; + err = PyDict_SetDefaultRef(dict, name, descr, NULL) < 0; } else { err = PyDict_SetItem(dict, name, descr) < 0; @@ -6701,7 +7550,7 @@ type_add_members(PyTypeObject *type) if (descr == NULL) return -1; - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6726,7 +7575,7 @@ type_add_getset(PyTypeObject *type) return -1; } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { Py_DECREF(descr); return -1; } @@ -6766,28 +7615,29 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) #undef COPYVAL /* Setup fast subclass flags */ - if (PyType_IsSubtype(base, (PyTypeObject*)PyExc_BaseException)) { + PyObject *mro = lookup_tp_mro(base); + if (is_subtype_with_mro(mro, base, (PyTypeObject*)PyExc_BaseException)) { type->tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyType_Type)) { + else if (is_subtype_with_mro(mro, base, &PyType_Type)) { type->tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyLong_Type)) { + else if (is_subtype_with_mro(mro, base, &PyLong_Type)) { type->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyBytes_Type)) { + else if (is_subtype_with_mro(mro, base, &PyBytes_Type)) { type->tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyUnicode_Type)) { + else if (is_subtype_with_mro(mro, base, &PyUnicode_Type)) { type->tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyTuple_Type)) { + else if (is_subtype_with_mro(mro, base, &PyTuple_Type)) { type->tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyList_Type)) { + else if (is_subtype_with_mro(mro, base, &PyList_Type)) { type->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS; } - else if (PyType_IsSubtype(base, &PyDict_Type)) { + else if (is_subtype_with_mro(mro, base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } @@ -7009,7 +7859,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) return 0; } -static int add_operators(PyTypeObject *); +static int add_operators(PyTypeObject *, PyTypeObject *); static int add_tp_new_wrapper(PyTypeObject *type); #define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING) @@ -7092,10 +7942,10 @@ type_ready_set_type(PyTypeObject *type) } static int -type_ready_set_bases(PyTypeObject *type) +type_ready_set_bases(PyTypeObject *type, int initial) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_bases(type) != NULL); return 0; } @@ -7114,7 +7964,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); } return 0; } @@ -7174,10 +8024,10 @@ type_dict_set_doc(PyTypeObject *type) static int -type_ready_fill_dict(PyTypeObject *type) +type_ready_fill_dict(PyTypeObject *type, PyTypeObject *def) { /* Add type-specific descriptors to tp_dict */ - if (add_operators(type) < 0) { + if (add_operators(type, def) < 0) { return -1; } if (type_add_methods(type) < 0) { @@ -7224,10 +8074,12 @@ type_ready_preheader(PyTypeObject *type) } static int -type_ready_mro(PyTypeObject *type) +type_ready_mro(PyTypeObject *type, int initial) { + ASSERT_TYPE_LOCK_HELD(); + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_mro(type) != NULL); return 0; } @@ -7235,7 +8087,7 @@ type_ready_mro(PyTypeObject *type) } /* Calculate method resolution order */ - if (mro_internal(type, NULL) < 0) { + if (mro_internal_unlocked(type, initial, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -7299,6 +8151,8 @@ inherit_patma_flags(PyTypeObject *type, PyTypeObject *base) { static int type_ready_inherit(PyTypeObject *type) { + ASSERT_TYPE_LOCK_HELD(); + /* Inherit special flags from dominant base */ PyTypeObject *base = type->tp_base; if (base != NULL) { @@ -7388,7 +8242,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type, int rerunbuiltin) +type_ready_set_new(PyTypeObject *type, int initial) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -7410,7 +8264,7 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + if (initial || base == NULL || type->tp_new != base->tp_new) { // If "__new__" key does not exists in the type dictionary, // set it to tp_new_wrapper(). if (add_tp_new_wrapper(type) < 0) { @@ -7451,6 +8305,9 @@ type_ready_managed_dict(PyTypeObject *type) return -1; } } + if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) { + type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; + } return 0; } @@ -7489,8 +8346,10 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type, int rerunbuiltin) +type_ready(PyTypeObject *type, PyTypeObject *def, int initial) { + ASSERT_TYPE_LOCK_HELD(); + _PyObject_ASSERT((PyObject *)type, !is_readying(type)); start_readying(type); @@ -7517,19 +8376,19 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_set_type(type) < 0) { goto error; } - if (type_ready_set_bases(type) < 0) { + if (type_ready_set_bases(type, initial) < 0) { goto error; } - if (type_ready_mro(type) < 0) { + if (type_ready_mro(type, initial) < 0) { goto error; } - if (type_ready_set_new(type, rerunbuiltin) < 0) { + if (type_ready_set_new(type, initial) < 0) { goto error; } - if (type_ready_fill_dict(type) < 0) { + if (type_ready_fill_dict(type, def) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_inherit(type) < 0) { goto error; } @@ -7543,7 +8402,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_managed_dict(type) < 0) { goto error; } @@ -7553,7 +8412,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) } /* All done -- set the ready flag */ - type->tp_flags = type->tp_flags | Py_TPFLAGS_READY; + type->tp_flags |= Py_TPFLAGS_READY; stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -7576,45 +8435,78 @@ PyType_Ready(PyTypeObject *type) /* Historically, all static types were immutable. See bpo-43908 */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + /* Static types must be immortal */ + _Py_SetImmortalUntracked((PyObject *)type); } - return type_ready(type, 0); + int res; + BEGIN_TYPE_LOCK(); + if (!(type->tp_flags & Py_TPFLAGS_READY)) { + res = type_ready(type, NULL, 1); + } else { + res = 0; + assert(_PyType_CheckConsistency(type)); + } + END_TYPE_LOCK(); + return res; } -int -_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) + +static int +init_static_type(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - int ismain = _Py_IsMainInterpreter(interp); if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(ismain); + assert(initial); self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; - self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); } else { - assert(!ismain); + assert(!initial); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + assert(self->tp_version_tag != 0); } - static_builtin_state_init(interp, self); + managed_static_type_state_init(interp, self, isbuiltin, initial); + + PyTypeObject *def = managed_static_type_get_def(self, isbuiltin); + if (initial) { + memcpy(def, self, sizeof(PyTypeObject)); + } - int res = type_ready(self, !ismain); + int res; + BEGIN_TYPE_LOCK(); + res = type_ready(self, def, initial); + END_TYPE_LOCK(); if (res < 0) { - static_builtin_state_clear(interp, self); + _PyStaticType_ClearWeakRefs(interp, self); + managed_static_type_state_clear(interp, self, isbuiltin, initial); } + return res; } +int +_PyStaticType_InitForExtension(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 0, ((self->tp_flags & Py_TPFLAGS_READY) == 0)); +} + +int +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 1, _Py_IsMainInterpreter(interp)); +} + static int add_subclass(PyTypeObject *base, PyTypeObject *type) @@ -8010,9 +8902,12 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) https://mail.python.org/pipermail/python-dev/2003-April/034535.html */ static int -hackcheck(PyObject *self, setattrofunc func, const char *what) +hackcheck_unlocked(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); + + ASSERT_TYPE_LOCK_HELD(); + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ @@ -8054,6 +8949,20 @@ hackcheck(PyObject *self, setattrofunc func, const char *what) return 1; } +static int +hackcheck(PyObject *self, setattrofunc func, const char *what) +{ + if (!PyType_Check(self)) { + return 1; + } + + int res; + BEGIN_TYPE_LOCK(); + res = hackcheck_unlocked(self, func, what); + END_TYPE_LOCK(); + return res; +} + static PyObject * wrap_setattr(PyObject *self, PyObject *args, void *wrapped) { @@ -8868,33 +9777,33 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use - _PyType_Lookup and create the method only when needed, with + _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, &_Py_ID(__getattr__)); + getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = _Py_slot_tp_getattro; return _Py_slot_tp_getattro(self, name); } - Py_INCREF(getattr); /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when self has the default __getattribute__ - method. So we use _PyType_Lookup and create the method only when + method. So we use _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, &_Py_ID(__getattribute__)); + getattribute = _PyType_LookupRef(tp, &_Py_ID(__getattribute__)); if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) { + Py_XDECREF(getattribute); res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); /* if res == NULL with no exception set, then it must be an AttributeError suppressed by us. */ if (res == NULL && !PyErr_Occurred()) { res = call_attribute(self, getattr, name); } - } else { - Py_INCREF(getattribute); + } + else { res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -9001,7 +9910,7 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyTypeObject *tp = Py_TYPE(self); PyObject *get; - get = _PyType_Lookup(tp, &_Py_ID(__get__)); + get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -9013,7 +9922,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (type == NULL) type = Py_None; PyObject *stack[3] = {self, obj, type}; - return PyObject_Vectorcall(get, stack, 3, NULL); + PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); + Py_DECREF(get); + return res; } static int @@ -9222,13 +10133,13 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } -static int -releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +static releasebufferproc +releasebuffer_maybe_call_super_unlocked(PyObject *self, Py_buffer *buffer) { PyTypeObject *self_type = Py_TYPE(self); PyObject *mro = lookup_tp_mro(self_type); if (mro == NULL) { - return -1; + return NULL; } assert(PyTuple_Check(mro)); @@ -9242,9 +10153,8 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) } i++; /* skip self_type */ if (i >= n) - return -1; + return NULL; - releasebufferproc base_releasebuffer = NULL; for (; i < n; i++) { PyObject *obj = PyTuple_GET_ITEM(mro, i); if (!PyType_Check(obj)) { @@ -9254,15 +10164,25 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) if (base_type->tp_as_buffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != NULL && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { - base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; - break; + return base_type->tp_as_buffer->bf_releasebuffer; } } + return NULL; +} + +static void +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + releasebufferproc base_releasebuffer; + + BEGIN_TYPE_LOCK(); + base_releasebuffer = releasebuffer_maybe_call_super_unlocked(self, buffer); + END_TYPE_LOCK(); + if (base_releasebuffer != NULL) { base_releasebuffer(self, buffer); } - return 0; } static void @@ -9290,7 +10210,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) // from a Python __buffer__ function. mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { - PyErr_WriteUnraisable(self); + PyErr_FormatUnraisable("Exception ignored in bf_releasebuffer of %s", Py_TYPE(self)->tp_name); goto end; } // Set the memoryview to restricted mode, which forbids @@ -9303,7 +10223,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); if (ret == NULL) { - PyErr_WriteUnraisable(self); + PyErr_FormatUnraisable("Exception ignored in __release_buffer__ of %s", Py_TYPE(self)->tp_name); } else { Py_DECREF(ret); @@ -9311,7 +10231,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) if (!is_buffer_wrapper) { PyObject *res = PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); if (res == NULL) { - PyErr_WriteUnraisable(self); + PyErr_FormatUnraisable("Exception ignored in bf_releasebuffer of %s", Py_TYPE(self)->tp_name); } else { Py_DECREF(res); @@ -9339,11 +10259,7 @@ static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) { releasebuffer_call_python(self, buffer); - if (releasebuffer_maybe_call_super(self, buffer) < 0) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); - } - } + releasebuffer_maybe_call_super(self, buffer); } static PyObject * @@ -9479,7 +10395,8 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__getattribute__, tp_getattro, _Py_slot_tp_getattr_hook, wrap_binaryfunc, "__getattribute__($self, name, /)\n--\n\nReturn getattr(self, name)."), - TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, ""), + TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, + "__getattr__($self, name, /)\n--\n\nImplement getattr(self, name)."), TPSLOT(__setattr__, tp_setattro, slot_tp_setattro, wrap_setattr, "__setattr__($self, name, value, /)\n--\n\nImplement setattr(self, name, value)."), TPSLOT(__delattr__, tp_setattro, slot_tp_setattro, wrap_delattr, @@ -9514,7 +10431,9 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__new__, tp_new, slot_tp_new, NULL, "__new__(type, /, *args, **kwargs)\n--\n\n" "Create and return new object. See help(type) for accurate signature."), - TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, + "__del__($self, /)\n--\n\n" + "Called when the instance is about to be destroyed."), BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, "__buffer__($self, flags, /)\n--\n\n" @@ -9798,6 +10717,8 @@ resolve_slotdups(PyTypeObject *type, PyObject *name) static pytype_slotdef * update_one_slot(PyTypeObject *type, pytype_slotdef *p) { + ASSERT_TYPE_LOCK_HELD(); + PyObject *descr; PyWrapperDescrObject *d; @@ -9846,7 +10767,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) d = (PyWrapperDescrObject *)descr; if ((specific == NULL || specific == d->d_wrapped) && d->d_base->wrapper == p->wrapper && - PyType_IsSubtype(type, PyDescr_TYPE(d))) + is_subtype_with_mro(lookup_tp_mro(type), type, PyDescr_TYPE(d))) { specific = d->d_wrapped; } @@ -9898,6 +10819,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type->tp_flags &= ~Py_TPFLAGS_HAVE_VECTORCALL; } } + Py_DECREF(descr); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; @@ -9911,6 +10833,8 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) static int update_slots_callback(PyTypeObject *type, void *data) { + ASSERT_TYPE_LOCK_HELD(); + pytype_slotdef **pp = (pytype_slotdef **)data; for (; *pp; pp++) { update_one_slot(type, *pp); @@ -9927,6 +10851,7 @@ update_slot(PyTypeObject *type, PyObject *name) pytype_slotdef **pp; int offset; + ASSERT_TYPE_LOCK_HELD(); assert(PyUnicode_CheckExact(name)); assert(PyUnicode_CHECK_INTERNED(name)); @@ -9960,10 +10885,17 @@ update_slot(PyTypeObject *type, PyObject *name) static void fixup_slot_dispatchers(PyTypeObject *type) { + // This lock isn't strictly necessary because the type has not been + // exposed to anyone else yet, but update_ont_slot calls find_name_in_mro + // where we'd like to assert that the type is locked. + BEGIN_TYPE_LOCK(); + assert(!PyErr_Occurred()); for (pytype_slotdef *p = slotdefs; p->name; ) { p = update_one_slot(type, p); } + + END_TYPE_LOCK(); } static void @@ -9971,6 +10903,8 @@ update_all_slots(PyTypeObject* type) { pytype_slotdef *p; + ASSERT_TYPE_LOCK_HELD(); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. */ PyType_Modified(type); @@ -10145,17 +11079,22 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, infinite recursion here.) */ static int -add_operators(PyTypeObject *type) +add_operators(PyTypeObject *type, PyTypeObject *def) { PyObject *dict = lookup_tp_dict(type); pytype_slotdef *p; PyObject *descr; void **ptr; + assert(def == NULL || (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + if (def == NULL) { + def = type; + } + for (p = slotdefs; p->name; p++) { if (p->wrapper == NULL) continue; - ptr = slotptr(type, p->offset); + ptr = slotptr(def, p->offset); if (!ptr || !*ptr) continue; int r = PyDict_Contains(dict, p->name_strobj); @@ -10243,7 +11182,15 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(su_obj_type); + /* keep a strong reference to mro because su_obj_type->tp_mro can be + replaced during PyDict_GetItemRef(dict, name, &res) and because + another thread can modify it after we end the critical section + below */ + Py_XINCREF(mro); + END_TYPE_LOCK(); + if (mro == NULL) return NULL; @@ -10256,27 +11203,21 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * break; } i++; /* skip su->type (if any) */ - if (i >= n) + if (i >= n) { + Py_DECREF(mro); return NULL; + } - /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemWithError(dict, name) */ - Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); assert(dict != NULL && PyDict_Check(dict)); - res = PyDict_GetItemWithError(dict, name); - if (res != NULL) { - Py_INCREF(res); + if (PyDict_GetItemRef(dict, name, &res) != 0) { + // found or error Py_DECREF(mro); return res; } - else if (PyErr_Occurred()) { - Py_DECREF(mro); - return NULL; - } i++; } while (i < n); @@ -10399,9 +11340,22 @@ supercheck(PyTypeObject *type, PyObject *obj) Py_XDECREF(class_attr); } - PyErr_SetString(PyExc_TypeError, - "super(type, obj): " - "obj must be an instance or subtype of type"); + const char *type_or_instance, *obj_str; + + if (PyType_Check(obj)) { + type_or_instance = "type"; + obj_str = ((PyTypeObject*)obj)->tp_name; + } + else { + type_or_instance = "instance of"; + obj_str = Py_TYPE(obj)->tp_name; + } + + PyErr_Format(PyExc_TypeError, + "super(type, obj): obj (%s %.200s) is not " + "an instance or subtype of type (%.200s).", + type_or_instance, obj_str, type->tp_name); + return NULL; } @@ -10461,7 +11415,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, } assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); - PyObject *firstarg = _PyFrame_GetLocalsArray(cframe)[0]; + PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); // The first argument might be a cell. if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { // "firstarg" is a cell here unless (very unlikely) super() @@ -10483,13 +11437,13 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // Look for __class__ in the free vars. PyTypeObject *type = NULL; - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; i++) { assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_Equal(name, &_Py_ID(__class__))) { - PyObject *cell = _PyFrame_GetLocalsArray(cframe)[i]; + PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]); if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 8a20b23c6866d7..c8ab14053de418 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -23,6 +23,8 @@ typedef struct { PyObject *evaluate_bound; PyObject *constraints; PyObject *evaluate_constraints; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -31,12 +33,16 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *default_value; + PyObject *evaluate_default; } typevartupleobject; typedef struct { PyObject_HEAD PyObject *name; PyObject *bound; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -53,6 +59,64 @@ typedef struct { #include "clinic/typevarobject.c.h" +/* NoDefault is a marker object to indicate that a parameter has no default. */ + +static PyObject * +NoDefault_repr(PyObject *op) +{ + return PyUnicode_FromString("typing.NoDefault"); +} + +static PyObject * +NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("NoDefault"); +} + +static PyMethodDef nodefault_methods[] = { + {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyObject * +nodefault_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoDefaultType takes no arguments"); + return NULL; + } + return &_Py_NoDefaultStruct; +} + +static void +nodefault_dealloc(PyObject *nodefault) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref NoDefault out of existence. Instead, + * since NoDefault is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(nodefault); +} + +PyDoc_STRVAR(nodefault_doc, +"NoDefaultType()\n" +"--\n\n" +"The type of the NoDefault singleton."); + +PyTypeObject _PyNoDefault_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "NoDefaultType", + .tp_dealloc = nodefault_dealloc, + .tp_repr = NoDefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = nodefault_doc, + .tp_methods = nodefault_methods, + .tp_new = nodefault_new, +}; + +PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); + + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -200,6 +264,8 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); + Py_XDECREF(tv->default_value); + Py_XDECREF(tv->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -216,6 +282,8 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); + Py_VISIT(tv->default_value); + Py_VISIT(tv->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,6 +295,8 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -266,6 +336,20 @@ typevar_bound(typevarobject *self, void *Py_UNUSED(ignored)) return bound; } +static PyObject * +typevar_default(typevarobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyObject * typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) { @@ -283,12 +367,14 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, + {"__default__", (getter)typevar_default, NULL, NULL, NULL}, {0} }; static typevarobject * typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, PyObject *constraints, PyObject *evaluate_constraints, + PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { @@ -305,6 +391,8 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_bound = Py_XNewRef(evaluate_bound); tv->constraints = Py_XNewRef(constraints); tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); + tv->default_value = Py_XNewRef(default_value); + tv->evaluate_default = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -327,8 +415,8 @@ typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") *constraints: object - * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -338,9 +426,9 @@ Create a TypeVar. static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance) -/*[clinic end generated code: output=1d200450ee99226d input=2c07ab87c94f462b]*/ + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance) +/*[clinic end generated code: output=d2b248ff074eaab6 input=836f97f631d7293a]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, @@ -364,26 +452,20 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, } } - if (constraints != NULL) { - if (!PyTuple_CheckExact(constraints)) { - PyErr_SetString(PyExc_TypeError, - "constraints must be a tuple"); - return NULL; - } - Py_ssize_t n_constraints = PyTuple_GET_SIZE(constraints); - if (n_constraints == 1) { - PyErr_SetString(PyExc_TypeError, - "A single constraint is not allowed"); - Py_XDECREF(bound); - return NULL; - } else if (n_constraints == 0) { - constraints = NULL; - } else if (bound != NULL) { - PyErr_SetString(PyExc_TypeError, - "Constraints cannot be combined with bound=..."); - Py_XDECREF(bound); - return NULL; - } + assert(PyTuple_CheckExact(constraints)); + Py_ssize_t n_constraints = PyTuple_GET_SIZE(constraints); + if (n_constraints == 1) { + PyErr_SetString(PyExc_TypeError, + "A single constraint is not allowed"); + Py_XDECREF(bound); + return NULL; + } else if (n_constraints == 0) { + constraints = NULL; + } else if (bound != NULL) { + PyErr_SetString(PyExc_TypeError, + "Constraints cannot be combined with bound=..."); + Py_XDECREF(bound); + return NULL; } PyObject *module = caller(); if (module == NULL) { @@ -393,6 +475,7 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *tv = (PyObject *)typevar_alloc(name, bound, NULL, constraints, NULL, + default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); @@ -417,6 +500,66 @@ typevar_typing_subst(typevarobject *self, PyObject *arg) return result; } +/*[clinic input] +typevar.__typing_prepare_subst__ as typevar_typing_prepare_subst + + alias: object + args: object + / + +[clinic start generated code]*/ + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args) +/*[clinic end generated code: output=82c3f4691e0ded22 input=201a750415d14ffb]*/ +{ + PyObject *params = PyObject_GetAttrString(alias, "__parameters__"); + if (params == NULL) { + return NULL; + } + Py_ssize_t i = PySequence_Index(params, (PyObject *)self); + if (i == -1) { + Py_DECREF(params); + return NULL; + } + Py_ssize_t args_len = PySequence_Length(args); + if (args_len == -1) { + Py_DECREF(params); + return NULL; + } + if (i < args_len) { + // We already have a value for our TypeVar + Py_DECREF(params); + return Py_NewRef(args); + } + else if (i == args_len) { + // If the TypeVar has a default, use it. + PyObject *dflt = typevar_default(self, NULL); + if (dflt == NULL) { + Py_DECREF(params); + return NULL; + } + if (dflt != &_Py_NoDefaultStruct) { + PyObject *new_args = PyTuple_Pack(1, dflt); + Py_DECREF(dflt); + if (new_args == NULL) { + Py_DECREF(params); + return NULL; + } + PyObject *result = PySequence_Concat(args, new_args); + Py_DECREF(params); + Py_DECREF(new_args); + return result; + } + } + Py_DECREF(params); + PyErr_Format(PyExc_TypeError, + "Too few arguments for %S; actual %d, expected at least %d", + alias, args_len, i + 1); + return NULL; +} + /*[clinic input] typevar.__reduce__ as typevar_reduce @@ -429,6 +572,23 @@ typevar_reduce_impl(typevarobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevar.has_default as typevar_has_default + +[clinic start generated code]*/ + +static PyObject * +typevar_has_default_impl(typevarobject *self) +/*[clinic end generated code: output=76bf0b8dc98b97dd input=31024aa030761cf6]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevar_mro_entries(PyObject *self, PyObject *args) { @@ -439,7 +599,9 @@ typevar_mro_entries(PyObject *self, PyObject *args) static PyMethodDef typevar_methods[] = { TYPEVAR_TYPING_SUBST_METHODDEF + TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF TYPEVAR_REDUCE_METHODDEF + TYPEVAR_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevar_mro_entries, METH_O}, {0} }; @@ -464,12 +626,18 @@ variables::\n\ class StrOrBytesSequence[A: (str, bytes)]:\n\ ...\n\ \n\ +Type variables can also have defaults:\n\ +\n\ + class IntDefault[T = int]:\n\ + ...\n\ +\n\ However, if desired, reusable type variables can also be constructed\n\ manually, like so::\n\ \n\ T = TypeVar('T') # Can be anything\n\ S = TypeVar('S', bound=str) # Can be any subtype of str\n\ A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n\ + D = TypeVar('D', default=int) # Defaults to int\n\ \n\ Type variables exist primarily for the benefit of static type\n\ checkers. They serve as the parameters for generic types as well\n\ @@ -746,6 +914,8 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); + Py_XDECREF(ps->default_value); + Py_XDECREF(ps->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -759,6 +929,8 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); + Py_VISIT(ps->default_value); + Py_VISIT(ps->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -767,6 +939,8 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -807,14 +981,29 @@ paramspec_kwargs(PyObject *self, void *unused) return (PyObject *)paramspecattr_new(tp, self); } +static PyObject * +paramspec_default(paramspecobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, + {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, {0}, }; static paramspecobject * -paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, +paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; @@ -827,6 +1016,8 @@ paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, ps->covariant = covariant; ps->contravariant = contravariant; ps->infer_variance = infer_variance; + ps->default_value = Py_XNewRef(default_value); + ps->evaluate_default = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -844,6 +1035,7 @@ paramspec.__new__ as paramspec_new name: object(subclass_of="&PyUnicode_Type") * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -853,8 +1045,9 @@ Create a ParamSpec object. static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=fd2daab79cba62da input=57c49c581979b952]*/ + PyObject *default_value, int covariant, int contravariant, + int infer_variance) +/*[clinic end generated code: output=47ca9d63fa5a094d input=495e1565bc067ab9]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); @@ -876,7 +1069,7 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( - name, bound, covariant, contravariant, infer_variance, module); + name, bound, default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); Py_DECREF(module); return ps; @@ -932,6 +1125,22 @@ paramspec_reduce_impl(paramspecobject *self) return Py_NewRef(self->name); } +/*[clinic input] +paramspec.has_default as paramspec_has_default + +[clinic start generated code]*/ + +static PyObject * +paramspec_has_default_impl(paramspecobject *self) +/*[clinic end generated code: output=daaae7467a6a4368 input=2112e97eeb76cd59]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * paramspec_mro_entries(PyObject *self, PyObject *args) { @@ -943,6 +1152,7 @@ paramspec_mro_entries(PyObject *self, PyObject *args) static PyMethodDef paramspec_methods[] = { PARAMSPEC_TYPING_SUBST_METHODDEF PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF + PARAMSPEC_HAS_DEFAULT_METHODDEF PARAMSPEC_REDUCE_METHODDEF {"__mro_entries__", paramspec_mro_entries, METH_O}, {0} @@ -957,10 +1167,17 @@ where the use of '**' creates a parameter specification::\n\ \n\ type IntFunc[**P] = Callable[P, int]\n\ \n\ +The following syntax creates a parameter specification that defaults\n\ +to a callable accepting two positional-only arguments of types int\n\ +and str:\n\ +\n\ + type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ +\n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ + DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1028,6 +1245,8 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); + Py_XDECREF(tvt->default_value); + Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1067,7 +1286,7 @@ static PyMemberDef typevartuple_members[] = { }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module) +typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1075,6 +1294,8 @@ typevartuple_alloc(PyObject *name, PyObject *module) return NULL; } tvt->name = Py_NewRef(name); + tvt->default_value = Py_XNewRef(default_value); + tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1090,19 +1311,22 @@ typevartuple_alloc(PyObject *name, PyObject *module) typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") + * + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=09d417a28f976202 input=00d28abcf1fc96bb]*/ +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ { PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module); + PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); Py_DECREF(module); return result; } @@ -1155,6 +1379,23 @@ typevartuple_reduce_impl(typevartupleobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevartuple.has_default as typevartuple_has_default + +[clinic start generated code]*/ + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self) +/*[clinic end generated code: output=4895f602f56a5e29 input=9ef3250ddb2c1851]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevartuple_mro_entries(PyObject *self, PyObject *args) { @@ -1167,6 +1408,8 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(((typevartupleobject *)self)->default_value); + Py_VISIT(((typevartupleobject *)self)->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1174,14 +1417,36 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) static int typevartuple_clear(PyObject *self) { + Py_CLEAR(((typevartupleobject *)self)->default_value); + Py_CLEAR(((typevartupleobject *)self)->evaluate_default); PyObject_ClearManagedDict(self); return 0; } +static PyObject * +typevartuple_default(typevartupleobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyGetSetDef typevartuple_getset[] = { + {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {0}, +}; + static PyMethodDef typevartuple_methods[] = { TYPEVARTUPLE_TYPING_SUBST_METHODDEF TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF TYPEVARTUPLE_REDUCE_METHODDEF + TYPEVARTUPLE_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevartuple_mro_entries, METH_O}, {0} }; @@ -1197,10 +1462,15 @@ where a single '*' indicates a type variable tuple::\n\ def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n\ return (*tup[1:], tup[0])\n\ \n\ +Type variables tuples can have default values:\n\ +\n\ + type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\ +\n\ For compatibility with Python 3.11 and earlier, TypeVarTuple objects\n\ can also be created as follows::\n\ \n\ Ts = TypeVarTuple('Ts') # Can be given any name\n\ + DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\ \n\ Just as a TypeVar (type variable) is a placeholder for a single type,\n\ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For\n\ @@ -1225,6 +1495,7 @@ PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, + {Py_tp_getset, typevartuple_getset}, {Py_tp_new, typevartuple}, {Py_tp_iter, typevartuple_iter}, {Py_tp_repr, typevartuple_repr}, @@ -1248,21 +1519,21 @@ PyObject * _Py_make_typevar(PyObject *name, PyObject *evaluate_bound, PyObject *evaluate_constraints) { return (PyObject *)typevar_alloc(name, NULL, evaluate_bound, NULL, evaluate_constraints, - false, false, true, NULL); + NULL, false, false, true, NULL); } PyObject * _Py_make_paramspec(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)paramspec_alloc(v, NULL, false, false, true, NULL); + return (PyObject *)paramspec_alloc(v, NULL, NULL, false, false, true, NULL); } PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL); } static void @@ -1694,3 +1965,24 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); } + +PyObject * +_Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *evaluate_default) +{ + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + Py_XSETREF(((typevarobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + Py_XSETREF(((paramspecobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + Py_XSETREF(((typevartupleobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 33cbc987d43282..394ea888fc9231 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -44,6 +44,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_bytesobject.h" // _PyBytes_Repeat() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec @@ -176,10 +177,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently *_to++ = (to_type) *_iter++; \ } while (0) -#define LATIN1(ch) \ - (ch < 128 \ - ? (PyObject*)&_Py_SINGLETON(strings).ascii[ch] \ - : (PyObject*)&_Py_SINGLETON(strings).latin1[ch - 128]) +#define LATIN1 _Py_LATIN1_CHR #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ @@ -201,6 +199,11 @@ static PyObject * unicode_decode_utf8(const char *s, Py_ssize_t size, _Py_error_handler error_handler, const char *errors, Py_ssize_t *consumed); +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed); #ifdef Py_DEBUG static inline int unicode_is_finalizing(void); static int unicode_is_singleton(PyObject *unicode); @@ -214,18 +217,20 @@ static inline PyObject* unicode_get_empty(void) return &_Py_STR(empty); } -/* This dictionary holds all interned unicode strings. Note that references - to strings in this dictionary are *not* counted in the string's ob_refcnt. - When the interned string reaches a refcnt of 0 the string deallocation - function will delete the reference from this dictionary. -*/ +/* This dictionary holds per-interpreter interned strings. + * See InternalDocs/string_interning.md for details. + */ static inline PyObject *get_interned_dict(PyInterpreterState *interp) { return _Py_INTERP_CACHED_OBJECT(interp, interned_strings); } +/* This hashtable holds statically allocated interned strings. + * See InternalDocs/string_interning.md for details. + */ #define INTERNED_STRINGS _PyRuntime.cached_objects.interned_strings +/* Get number of all interned strings for the current interpreter. */ Py_ssize_t _PyUnicode_InternedSize(void) { @@ -233,6 +238,27 @@ _PyUnicode_InternedSize(void) return _Py_hashtable_len(INTERNED_STRINGS) + PyDict_GET_SIZE(dict); } +/* Get number of immortal interned strings for the current interpreter. */ +Py_ssize_t +_PyUnicode_InternedSize_Immortal(void) +{ + PyObject *dict = get_interned_dict(_PyInterpreterState_GET()); + PyObject *key, *value; + Py_ssize_t pos = 0; + Py_ssize_t count = 0; + + // It's tempting to keep a count and avoid a loop here. But, this function + // is intended for refleak tests. It spends extra work to report the true + // value, to help detect bugs in optimizations. + + while (PyDict_Next(dict, &pos, &key, &value)) { + if (_Py_IsImmortal(key)) { + count++; + } + } + return _Py_hashtable_len(INTERNED_STRINGS) + count; +} + static Py_hash_t unicode_hash(PyObject *); static int unicode_compare_eq(PyObject *, PyObject *); @@ -258,20 +284,6 @@ hashtable_unicode_compare(const void *key1, const void *key2) static int init_interned_dict(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - assert(INTERNED_STRINGS == NULL); - _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; - INTERNED_STRINGS = _Py_hashtable_new_full( - hashtable_unicode_hash, - hashtable_unicode_compare, - NULL, - NULL, - &hashtable_alloc - ); - if (INTERNED_STRINGS == NULL) { - return -1; - } - } assert(get_interned_dict(interp) == NULL); PyObject *interned = interned = PyDict_New(); if (interned == NULL) { @@ -290,7 +302,57 @@ clear_interned_dict(PyInterpreterState *interp) Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } - if (_Py_IsMainInterpreter(interp) && INTERNED_STRINGS != NULL) { +} + +static PyStatus +init_global_interned_strings(PyInterpreterState *interp) +{ + assert(INTERNED_STRINGS == NULL); + _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; + + INTERNED_STRINGS = _Py_hashtable_new_full( + hashtable_unicode_hash, + hashtable_unicode_compare, + // Objects stored here are immortal and statically allocated, + // so we don't need key_destroy_func & value_destroy_func: + NULL, + NULL, + &hashtable_alloc + ); + if (INTERNED_STRINGS == NULL) { + PyErr_Clear(); + return _PyStatus_ERR("failed to create global interned dict"); + } + + /* Intern statically allocated string identifiers and deepfreeze strings. + * This must be done before any module initialization so that statically + * allocated string identifiers are used instead of heap allocated strings. + * Deepfreeze uses the interned identifiers if present to save space + * else generates them and they are interned to speed up dict lookups. + */ + _PyUnicode_InitStaticStrings(interp); + +#ifdef Py_GIL_DISABLED +// In the free-threaded build, intern the 1-byte strings as well + for (int i = 0; i < 256; i++) { + PyObject *s = LATIN1(i); + _PyUnicode_InternStatic(interp, &s); + assert(s == LATIN1(i)); + } +#endif +#ifdef Py_DEBUG + assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); + + for (int i = 0; i < 256; i++) { + assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); + } +#endif + return _PyStatus_OK(); +} + +static void clear_global_interned_strings(void) +{ + if (INTERNED_STRINGS != NULL) { _Py_hashtable_destroy(INTERNED_STRINGS); INTERNED_STRINGS = NULL; } @@ -505,7 +567,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } @@ -623,6 +685,39 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) } CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); } + + /* Check interning state */ +#ifdef Py_DEBUG + switch (PyUnicode_CHECK_INTERNED(op)) { + case SSTATE_NOT_INTERNED: + if (ascii->state.statically_allocated) { + CHECK(_Py_IsImmortal(op)); + // This state is for two exceptions: + // - strings are currently checked before they're interned + // - the 256 one-latin1-character strings + // are static but use SSTATE_NOT_INTERNED + } + else { + CHECK(!_Py_IsImmortal(op)); + } + break; + case SSTATE_INTERNED_MORTAL: + CHECK(!ascii->state.statically_allocated); + CHECK(!_Py_IsImmortal(op)); + break; + case SSTATE_INTERNED_IMMORTAL: + CHECK(!ascii->state.statically_allocated); + CHECK(_Py_IsImmortal(op)); + break; + case SSTATE_INTERNED_IMMORTAL_STATIC: + CHECK(ascii->state.statically_allocated); + CHECK(_Py_IsImmortal(op)); + break; + default: + Py_UNREACHABLE(); + } +#endif + return 1; #undef CHECK @@ -893,6 +988,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -903,6 +999,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -913,6 +1010,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -996,7 +1094,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) new_size = (struct_size + (length + 1) * char_size); if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1049,7 +1147,7 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } @@ -1276,46 +1374,6 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) return obj; } -#if SIZEOF_WCHAR_T == 2 -/* Helper function to convert a 16-bits wchar_t representation to UCS4, this - will decode surrogate pairs, the other conversions are implemented as macros - for efficiency. - - This function assumes that unicode can hold one more code point than wstr - characters for a terminating null character. */ -static void -unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, - PyObject *unicode) -{ - const wchar_t *iter; - Py_UCS4 *ucs4_out; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - ucs4_out = PyUnicode_4BYTE_DATA(unicode); - - for (iter = begin; iter < end; ) { - assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) - && (iter+1) < end - && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) - { - *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); - iter += 2; - } - else { - *ucs4_out++ = *iter; - iter++; - } - } - assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - -} -#endif - static int unicode_check_modifiable(PyObject *unicode) { @@ -1579,21 +1637,79 @@ unicode_dealloc(PyObject *unicode) _Py_FatalRefcountError("deallocating an Unicode singleton"); } #endif - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref an immortal string out of existence. Since - * the string is an immortal object, just re-set the reference count. - */ - if (PyUnicode_CHECK_INTERNED(unicode) - || _PyUnicode_STATE(unicode).statically_allocated) - { + if (_PyUnicode_STATE(unicode).statically_allocated) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref an immortal string out of existence. Since + * the string is an immortal object, just re-set the reference count. + */ +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif _Py_SetImmortal(unicode); return; } + switch (_PyUnicode_STATE(unicode).interned) { + case SSTATE_NOT_INTERNED: + break; + case SSTATE_INTERNED_MORTAL: + /* Remove the object from the intern dict. + * Before doing so, we set the refcount to 2: the key and value + * in the interned_dict. + */ + assert(Py_REFCNT(unicode) == 0); + Py_SET_REFCNT(unicode, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *interned = get_interned_dict(interp); + assert(interned != NULL); + PyObject *popped; + int r = PyDict_Pop(interned, unicode, &popped); + if (r == -1) { + PyErr_WriteUnraisable(unicode); + // We don't know what happened to the string. It's probably + // best to leak it: + // - if it was popped, there are no more references to it + // so it can't cause trouble (except wasted memory) + // - if it wasn't popped, it'll remain interned + _Py_SetImmortal(unicode); + _PyUnicode_STATE(unicode).interned = SSTATE_INTERNED_IMMORTAL; + return; + } + if (r == 0) { + // The interned string was not found in the interned_dict. +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } + // Successfully popped. + assert(popped == unicode); + // Only our `popped` reference should be left; remove it too. + assert(Py_REFCNT(unicode) == 1); + Py_SET_REFCNT(unicode, 0); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + break; + default: + // As with `statically_allocated` above. +#ifdef Py_REF_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_Free(_PyUnicode_UTF8(unicode)); + PyMem_Free(_PyUnicode_UTF8(unicode)); } if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) { - PyObject_Free(_PyUnicode_DATA_ANY(unicode)); + PyMem_Free(_PyUnicode_DATA_ANY(unicode)); } Py_TYPE(unicode)->tp_free(unicode); @@ -1624,7 +1740,7 @@ unicode_modifiable(PyObject *unicode) assert(_PyUnicode_CHECK(unicode)); if (Py_REFCNT(unicode) != 1) return 0; - if (_PyUnicode_HASH(unicode) != -1) + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(unicode)) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; @@ -1781,6 +1897,62 @@ unicode_char(Py_UCS4 ch) return unicode; } + +static inline void +unicode_write_widechar(int kind, void *data, + const wchar_t *u, Py_ssize_t size, + Py_ssize_t num_surrogates) +{ + switch (kind) { + case PyUnicode_1BYTE_KIND: + _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, u, u + size, data); + break; + + case PyUnicode_2BYTE_KIND: +#if SIZEOF_WCHAR_T == 2 + memcpy(data, u, size * 2); +#else + _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, u, u + size, data); +#endif + break; + + case PyUnicode_4BYTE_KIND: + { +#if SIZEOF_WCHAR_T == 2 + // Convert a 16-bits wchar_t representation to UCS4, this will decode + // surrogate pairs. + const wchar_t *end = u + size; + Py_UCS4 *ucs4_out = (Py_UCS4*)data; +# ifndef NDEBUG + Py_UCS4 *ucs4_end = (Py_UCS4*)data + (size - num_surrogates); +# endif + for (const wchar_t *iter = u; iter < end; ) { + assert(ucs4_out < ucs4_end); + if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) + && (iter+1) < end + && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) + { + *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); + iter += 2; + } + else { + *ucs4_out++ = *iter; + iter++; + } + } + assert(ucs4_out == ucs4_end); +#else + assert(num_surrogates == 0); + memcpy(data, u, size * 4); +#endif + break; + } + default: + Py_UNREACHABLE(); + } +} + + PyObject * PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) { @@ -1833,36 +2005,63 @@ PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) if (!unicode) return NULL; - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, - u, u + size, PyUnicode_1BYTE_DATA(unicode)); - break; - case PyUnicode_2BYTE_KIND: -#if Py_UNICODE_SIZE == 2 - memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2); -#else - _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, - u, u + size, PyUnicode_2BYTE_DATA(unicode)); -#endif - break; - case PyUnicode_4BYTE_KIND: -#if SIZEOF_WCHAR_T == 2 - /* This is the only case which has to process surrogates, thus - a simple copy loop is not enough and we need a function. */ - unicode_convert_wchar_to_ucs4(u, u + size, unicode); -#else - assert(num_surrogates == 0); - memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4); + unicode_write_widechar(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + u, size, num_surrogates); + + return unicode_result(unicode); +} + + +int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *pub_writer, + const wchar_t *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + if (size < 0) { + size = wcslen(str); + } + + if (size == 0) { + return 0; + } + +#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION + /* Oracle Solaris uses non-Unicode internal wchar_t form for + non-Unicode locales and hence needs conversion to UCS-4 first. */ + if (_Py_LocaleUsesNonUnicodeWchar()) { + wchar_t* converted = _Py_DecodeNonUnicodeWchar(str, size); + if (!converted) { + return -1; + } + + int res = PyUnicodeWriter_WriteUCS4(pub_writer, converted, size); + PyMem_Free(converted); + return res; + } #endif - break; - default: - Py_UNREACHABLE(); + + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; + if (find_maxchar_surrogates(str, str + size, + &maxchar, &num_surrogates) == -1) { + return -1; } - return unicode_result(unicode); + if (_PyUnicodeWriter_Prepare(writer, size - num_surrogates, maxchar) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + unicode_write_widechar(kind, data, str, size, num_surrogates); + + writer->pos += size - num_surrogates; + return 0; } + PyObject * PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) { @@ -1897,6 +2096,7 @@ PyUnicode_FromString(const char *u) PyObject * _PyUnicode_FromId(_Py_Identifier *id) { + PyMutex_Lock((PyMutex *)&id->mutex); PyInterpreterState *interp = _PyInterpreterState_GET(); struct _Py_unicode_ids *ids = &interp->unicode.ids; @@ -1904,7 +2104,7 @@ _PyUnicode_FromId(_Py_Identifier *id) if (index < 0) { struct _Py_unicode_runtime_ids *rt_ids = &interp->runtime->unicode_state.ids; - PyThread_acquire_lock(rt_ids->lock, WAIT_LOCK); + PyMutex_Lock(&rt_ids->mutex); // Check again to detect concurrent access. Another thread can have // initialized the index while this thread waited for the lock. index = _Py_atomic_load_ssize(&id->index); @@ -1914,7 +2114,7 @@ _PyUnicode_FromId(_Py_Identifier *id) rt_ids->next_index++; _Py_atomic_store_ssize(&id->index, index); } - PyThread_release_lock(rt_ids->lock); + PyMutex_Unlock(&rt_ids->mutex); } assert(index >= 0); @@ -1923,16 +2123,16 @@ _PyUnicode_FromId(_Py_Identifier *id) obj = ids->array[index]; if (obj) { // Return a borrowed reference - return obj; + goto end; } } obj = PyUnicode_DecodeUTF8Stateful(id->string, strlen(id->string), NULL, NULL); if (!obj) { - return NULL; + goto end; } - PyUnicode_InternInPlace(&obj); + _PyUnicode_InternImmortal(interp, &obj); if (index >= ids->size) { // Overallocate to reduce the number of realloc @@ -1941,7 +2141,8 @@ _PyUnicode_FromId(_Py_Identifier *id) PyObject **new_array = PyMem_Realloc(ids->array, new_size * item_size); if (new_array == NULL) { PyErr_NoMemory(); - return NULL; + obj = NULL; + goto end; } memset(&new_array[ids->size], 0, (new_size - ids->size) * item_size); ids->array = new_array; @@ -1951,6 +2152,8 @@ _PyUnicode_FromId(_Py_Identifier *id) // The array stores a strong reference ids->array[index] = obj; +end: + PyMutex_Unlock((PyMutex *)&id->mutex); // Return a borrowed reference return obj; } @@ -2084,6 +2287,51 @@ _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) return res; } + +int +PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer, + Py_UCS4 *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, + "size must be positive"); + return -1; + } + + if (size == 0) { + return 0; + } + + Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size); + + if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + if (kind == PyUnicode_1BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, + str, str + size, + data); + } + else if (kind == PyUnicode_2BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, + str, str + size, + data); + } + else { + memcpy(data, str, size * sizeof(Py_UCS4)); + } + writer->pos += size; + + return 0; +} + + PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) { @@ -2372,14 +2620,12 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, } static int -unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, +unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ + Py_ssize_t *pconsumed = NULL; Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = strlen(str); } @@ -2388,12 +2634,28 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, while (length < precision && str[length]) { length++; } + if (length == precision) { + /* The input string is not NUL-terminated. If it ends with an + * incomplete UTF-8 sequence, truncate the string just before it. + * Incomplete sequences in the middle and sequences which cannot + * be valid prefixes are still treated as errors and replaced + * with \xfffd. */ + pconsumed = &length; + } + } + + if (width < 0) { + return unicode_decode_utf8_writer(writer, str, length, + _Py_ERROR_REPLACE, "replace", pconsumed); } - unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); + + PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, + "replace", pconsumed); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, + width, -1, flags); Py_DECREF(unicode); return res; } @@ -2402,11 +2664,7 @@ static int unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t width, Py_ssize_t precision, int flags) { - /* UTF-8 */ Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = wcslen(str); } @@ -2416,11 +2674,17 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, length++; } } - unicode = PyUnicode_FromWideChar(str, length); + + if (width < 0) { + return PyUnicodeWriter_WriteWideChar((PyUnicodeWriter*)writer, + str, length); + } + + PyObject *unicode = PyUnicode_FromWideChar(str, length); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); Py_DECREF(unicode); return res; } @@ -2464,6 +2728,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, switch (*f++) { case '-': flags |= F_LJUST; continue; case '0': flags |= F_ZERO; continue; + case '#': flags |= F_ALT; continue; } f--; break; @@ -2694,7 +2959,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, else { /* UTF-8 */ const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, s, width, precision, flags) < 0) return NULL; } break; @@ -2733,7 +2998,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -2787,71 +3052,137 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } - default: - invalid_format: - PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); - return NULL; - } + case 'T': + { + PyObject *obj = va_arg(*vargs, PyObject *); + PyTypeObject *type = (PyTypeObject *)Py_NewRef(Py_TYPE(obj)); - f++; - return f; -} + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + Py_DECREF(type); + if (!type_name) { + return NULL; + } -PyObject * -PyUnicode_FromFormatV(const char *format, va_list vargs) -{ - va_list vargs2; - const char *f; - _PyUnicodeWriter writer; + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } - _PyUnicodeWriter_Init(&writer); - writer.min_length = strlen(format) + 100; - writer.overallocate = 1; + case 'N': + { + PyObject *type_raw = va_arg(*vargs, PyObject *); + assert(type_raw != NULL); - // Copy varags to be able to pass a reference to a subfunction. + if (!PyType_Check(type_raw)) { + PyErr_SetString(PyExc_TypeError, "%N argument must be a type"); + return NULL; + } + PyTypeObject *type = (PyTypeObject*)type_raw; + + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + if (!type_name) { + return NULL; + } + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + + default: + invalid_format: + PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); + return NULL; + } + + f++; + return f; +} + +static int +unicode_from_format(_PyUnicodeWriter *writer, const char *format, va_list vargs) +{ + Py_ssize_t len = strlen(format); + writer->min_length += len + 100; + writer->overallocate = 1; + + // Copy varags to be able to pass a reference to a subfunction. + va_list vargs2; va_copy(vargs2, vargs); - for (f = format; *f; ) { + // _PyUnicodeWriter_WriteASCIIString() below requires the format string + // to be encoded to ASCII. + int is_ascii = (ucs1lib_find_max_char((Py_UCS1*)format, (Py_UCS1*)format + len) < 128); + if (!is_ascii) { + Py_ssize_t i; + for (i=0; i < len && (unsigned char)format[i] <= 127; i++); + PyErr_Format(PyExc_ValueError, + "PyUnicode_FromFormatV() expects an ASCII-encoded format " + "string, got a non-ASCII byte: 0x%02x", + (unsigned char)format[i]); + goto fail; + } + + for (const char *f = format; *f; ) { if (*f == '%') { - f = unicode_fromformat_arg(&writer, f, &vargs2); + f = unicode_fromformat_arg(writer, f, &vargs2); if (f == NULL) goto fail; } else { - const char *p; - Py_ssize_t len; - - p = f; - do - { - if ((unsigned char)*p > 127) { - PyErr_Format(PyExc_ValueError, - "PyUnicode_FromFormatV() expects an ASCII-encoded format " - "string, got a non-ASCII byte: 0x%02x", - (unsigned char)*p); - goto fail; - } - p++; + const char *p = strchr(f, '%'); + if (p != NULL) { + len = p - f; + } + else { + len = strlen(f); + writer->overallocate = 0; } - while (*p != '\0' && *p != '%'); - len = p - f; - - if (*p == '\0') - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0) + if (_PyUnicodeWriter_WriteASCIIString(writer, f, len) < 0) { goto fail; - - f = p; + } + f += len; } } va_end(vargs2); - return _PyUnicodeWriter_Finish(&writer); + return 0; fail: va_end(vargs2); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + +PyObject * +PyUnicode_FromFormatV(const char *format, va_list vargs) +{ + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (unicode_from_format(&writer, format, vargs) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyObject * @@ -2866,6 +3197,23 @@ PyUnicode_FromFormat(const char *format, ...) return ret; } +int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + + va_list vargs; + va_start(vargs, format); + int res = unicode_from_format(_writer, format, vargs); + va_end(vargs); + + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + static Py_ssize_t unicode_get_widechar_size(PyObject *unicode) { @@ -3820,17 +4168,24 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); + if (psize) { + *psize = -1; + } return NULL; } if (PyUnicode_UTF8(unicode) == NULL) { if (unicode_fill_utf8(unicode) == -1) { + if (psize) { + *psize = -1; + } return NULL; } } - if (psize) + if (psize) { *psize = PyUnicode_UTF8_LENGTH(unicode); + } return PyUnicode_UTF8(unicode); } @@ -3840,6 +4195,18 @@ PyUnicode_AsUTF8(PyObject *unicode) return PyUnicode_AsUTF8AndSize(unicode, NULL); } +const char * +_PyUnicode_AsUTF8NoNUL(PyObject *unicode) +{ + Py_ssize_t size; + const char *s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s && strlen(s) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + return s; +} + /* PyUnicode_GetSize() has been deprecated since Python 3.3 because it returned length of Py_UNICODE. @@ -4608,8 +4975,9 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) const char *p = start; #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - assert(_Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)); - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { + if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T) + && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) + { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ /* Help allocation */ @@ -4656,46 +5024,14 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) -{ - if (size == 0) { - if (consumed) - *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); - } - - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) { - if (consumed) { - *consumed = 1; - } - return get_latin1_char((unsigned char)s[0]); - } - - const char *starts = s; - const char *end = s + size; - - // fast path: try ASCII string. - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; - } - s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (s == end) { - if (consumed) { - *consumed = size; - } - return u; - } - - // Use _PyUnicodeWriter after fast path is failed. - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = s - starts; +static int +unicode_decode_utf8_impl(_PyUnicodeWriter *writer, + const char *starts, const char *s, const char *end, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed) +{ Py_ssize_t startinpos, endinpos; const char *errmsg = ""; PyObject *error_handler_obj = NULL; @@ -4703,18 +5039,18 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, while (s < end) { Py_UCS4 ch; - int kind = writer.kind; + int kind = writer->kind; if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); + if (PyUnicode_IS_ASCII(writer->buffer)) + ch = asciilib_utf8_decode(&s, end, writer->data, &writer->pos); else - ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs1lib_utf8_decode(&s, end, writer->data, &writer->pos); } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs2lib_utf8_decode(&s, end, writer->data, &writer->pos); } else { assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs4lib_utf8_decode(&s, end, writer->data, &writer->pos); } switch (ch) { @@ -4737,7 +5073,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, /* Truncated surrogate code in range D800-DFFF */ goto End; } - /* fall through */ + _Py_FALLTHROUGH; case 3: case 4: errmsg = "invalid continuation byte"; @@ -4745,7 +5081,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, endinpos = startinpos + ch - 1; break; default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) + // ch doesn't fit into kind, so change the buffer kind to write + // the character + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) goto onError; continue; } @@ -4759,7 +5097,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, break; case _Py_ERROR_REPLACE: - if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + if (_PyUnicodeWriter_WriteCharInline(writer, 0xfffd) < 0) goto onError; s += (endinpos - startinpos); break; @@ -4768,13 +5106,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, { Py_ssize_t i; - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + if (_PyUnicodeWriter_PrepareKind(writer, PyUnicode_2BYTE_KIND) < 0) goto onError; for (i=startinpos; ikind, writer->data, writer->pos, ch + 0xdc00); - writer.pos++; + writer->pos++; } s += (endinpos - startinpos); break; @@ -4785,8 +5123,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, errors, &error_handler_obj, "utf-8", errmsg, &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) + writer)) { goto onError; + } + + if (_PyUnicodeWriter_Prepare(writer, end - s, 127) < 0) { + return -1; + } } } @@ -4796,13 +5139,106 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return 0; onError: Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + + +static PyObject * +unicode_decode_utf8(const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + _Py_RETURN_UNICODE_EMPTY(); + } + + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) { + if (consumed) { + *consumed = 1; + } + return get_latin1_char((unsigned char)s[0]); + } + + // fast path: try ASCII string. + const char *starts = s; + const char *end = s + size; + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + Py_ssize_t decoded = ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return u; + } + s += decoded; + size -= decoded; + + // Use _PyUnicodeWriter after fast path is failed. + _PyUnicodeWriter writer; + _PyUnicodeWriter_InitWithBuffer(&writer, u); + writer.pos = decoded; + + if (unicode_decode_utf8_impl(&writer, starts, s, end, + error_handler, errors, + consumed) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} + + +// Used by PyUnicodeWriter_WriteUTF8() implementation +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + return 0; + } + + // fast path: try ASCII string. + if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { + return -1; + } + + const char *starts = s; + const char *end = s + size; + Py_ssize_t decoded = 0; + Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; + if (writer->kind == PyUnicode_1BYTE_KIND) { + decoded = ascii_decode(s, end, dest); + writer->pos += decoded; + + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return 0; + } + s += decoded; + size -= decoded; + } + + return unicode_decode_utf8_impl(writer, starts, s, end, + error_handler, errors, consumed); } @@ -5180,7 +5616,7 @@ unicode_fill_utf8(PyObject *unicode) PyBytes_AS_STRING(writer.buffer); Py_ssize_t len = end - start; - char *cache = PyObject_Malloc(len + 1); + char *cache = PyMem_Malloc(len + 1); if (cache == NULL) { _PyBytesWriter_Dealloc(&writer); PyErr_NoMemory(); @@ -5850,6 +6286,23 @@ PyUnicode_AsUTF16String(PyObject *unicode) return _PyUnicode_EncodeUTF16(unicode, NULL, 0); } +_PyUnicode_Name_CAPI * +_PyUnicode_GetNameCAPI(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_Name_CAPI *ucnhash_capi; + + ucnhash_capi = _Py_atomic_load_ptr(&interp->unicode.ucnhash_capi); + if (ucnhash_capi == NULL) { + ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( + PyUnicodeData_CAPSULE_NAME, 1); + + // It's fine if we overwrite the value here. It's always the same value. + _Py_atomic_store_ptr(&interp->unicode.ucnhash_capi, ucnhash_capi); + } + return ucnhash_capi; +} + /* --- Unicode Escape Codec ----------------------------------------------- */ PyObject * @@ -5865,7 +6318,6 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; _PyUnicode_Name_CAPI *ucnhash_capi; - PyInterpreterState *interp = _PyInterpreterState_GET(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -6013,19 +6465,13 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': - ucnhash_capi = interp->unicode.ucnhash_capi; + ucnhash_capi = _PyUnicode_GetNameCAPI(); if (ucnhash_capi == NULL) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (ucnhash_capi == NULL) { - PyErr_SetString( + PyErr_SetString( PyExc_UnicodeError, "\\N escapes not supported (can't load unicodedata module)" - ); - goto onError; - } - interp->unicode.ucnhash_capi = ucnhash_capi; + ); + goto onError; } message = "malformed \\N character escape"; @@ -6662,7 +7108,7 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_REPLACE: memset(str, '?', collend - collstart); str += (collend - collstart); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: pos = collend; break; @@ -6701,7 +7147,7 @@ unicode_encode_ucs1(PyObject *unicode, break; collstart = pos; assert(collstart != collend); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, @@ -7927,7 +8373,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) int count2 = 0, count3 = 0; int kind; const void *data; - Py_ssize_t length; + int length; Py_UCS4 ch; if (!PyUnicode_Check(string) || !PyUnicode_GET_LENGTH(string)) { @@ -7936,8 +8382,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) } kind = PyUnicode_KIND(string); data = PyUnicode_DATA(string); - length = PyUnicode_GET_LENGTH(string); - length = Py_MIN(length, 256); + length = (int)Py_MIN(PyUnicode_GET_LENGTH(string), 256); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); @@ -8254,7 +8699,7 @@ charmap_encoding_error( return -1; } } - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: *inpos = collendpos; break; @@ -8870,19 +9315,24 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) /* --- Helpers ------------------------------------------------------------ */ /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) static Py_ssize_t any_find_slice(PyObject* s1, PyObject* s2, @@ -9103,99 +9553,30 @@ _PyUnicode_InsertThousandsGrouping( return count; } -static Py_ssize_t -unicode_count_impl(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) +Py_ssize_t +PyUnicode_Count(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end) { - assert(PyUnicode_Check(str)); - assert(PyUnicode_Check(substr)); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) + return -1; - Py_ssize_t result; - int kind1, kind2; - const void *buf1 = NULL, *buf2 = NULL; - Py_ssize_t len1, len2; + return unicode_count_impl(str, substr, start, end); +} - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; +Py_ssize_t +PyUnicode_Find(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) + return -2; - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return 0; - - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (kind2 != kind1) { - buf2 = unicode_askind(kind2, buf2, len2, kind1); - if (!buf2) - goto onError; - } - - // We don't reuse `anylib_count` here because of the explicit casts. - switch (kind1) { - case PyUnicode_1BYTE_KIND: - result = ucs1lib_count( - ((const Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_count( - ((const Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_count( - ((const Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - default: - Py_UNREACHABLE(); - } - - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - - return result; - onError: - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - return -1; -} - -Py_ssize_t -PyUnicode_Count(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) -{ - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -1; - - return unicode_count_impl(str, substr, start, end); -} - -Py_ssize_t -PyUnicode_Find(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -2; - - return any_find_slice(str, substr, start, end, direction); -} + return any_find_slice(str, substr, start, end, direction); +} Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, @@ -9538,13 +9919,14 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) return NULL; } - /* NOTE: the following code can't call back into Python code, - * so we are sure that fseq won't be mutated. - */ + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); res = _PyUnicode_JoinArray(separator, items, seqlen); + + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + Py_DECREF(fseq); return res; } @@ -10793,12 +11175,15 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (left == right_uni) return 1; - if (PyUnicode_CHECK_INTERNED(left)) + assert(PyUnicode_CHECK_INTERNED(right_uni)); + if (PyUnicode_CHECK_INTERNED(left)) { return 0; + } - assert(_PyUnicode_HASH(right_uni) != -1); - Py_hash_t hash = _PyUnicode_HASH(left); - if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { + Py_hash_t right_hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(right_uni)); + assert(right_hash != -1); + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(left)); + if (hash != -1 && hash != right_hash) { return 0; } @@ -11040,47 +11425,87 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } -/* -Wraps asciilib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +str.count as unicode_count -> Py_ssize_t -static inline int -parse_args_finds_unicode(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if (asciilib_parse_args_finds(function_name, args, substring, start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; -} + self as str: self + sub as substr: unicode + start: slice_index(accept={int, NoneType}, c_default='0') = None + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + / -PyDoc_STRVAR(count__doc__, - "S.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are\n\ -interpreted as in slice notation."); +Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -static PyObject * -unicode_count(PyObject *self, PyObject *args) +Optional arguments start and end are interpreted as in slice notation. +[clinic start generated code]*/ + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ { - PyObject *substring = NULL; /* initialize to fix a compiler warning */ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; + assert(PyUnicode_Check(str)); + assert(PyUnicode_Check(substr)); + Py_ssize_t result; + int kind1, kind2; + const void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; - if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) - return NULL; + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) + return 0; - result = unicode_count_impl(self, substring, start, end); - if (result == -1) - return NULL; + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); + ADJUST_INDICES(start, end, len1); + if (end - start < len2) + return 0; + + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); + if (kind2 != kind1) { + buf2 = unicode_askind(kind2, buf2, len2, kind1); + if (!buf2) + goto onError; + } + + // We don't reuse `anylib_count` here because of the explicit casts. + switch (kind1) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_count( + ((const Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((const Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((const Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + Py_UNREACHABLE(); + } + + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); - return PyLong_FromSsize_t(result); + return result; + onError: + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); + return -1; } /*[clinic input] @@ -11191,33 +11616,25 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) return NULL; } -PyDoc_STRVAR(find__doc__, - "S.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_find(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.find as unicode_find = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ - return PyLong_FromSsize_t(result); +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result < 0) { + return -1; + } + return result; } static PyObject * @@ -11251,47 +11668,39 @@ unicode_hash(PyObject *self) #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (_PyUnicode_HASH(self) != -1) - return _PyUnicode_HASH(self); - + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(self)); + if (hash != -1) { + return hash; + } x = _Py_HashBytes(PyUnicode_DATA(self), PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - _PyUnicode_HASH(self) = x; + + FT_ATOMIC_STORE_SSIZE_RELAXED(_PyUnicode_HASH(self), x); return x; } -PyDoc_STRVAR(index__doc__, - "S.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); - -static PyObject * -unicode_index(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - Py_ssize_t result; - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - - if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.index as unicode_index = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12212,24 +12621,17 @@ unicode_removesuffix_impl(PyObject *self, PyObject *suffix) static PyObject * unicode_repr(PyObject *unicode) { - PyObject *repr; - Py_ssize_t isize; - Py_ssize_t osize, squote, dquote, i, o; - Py_UCS4 max, quote; - int ikind, okind, unchanged; - const void *idata; - void *odata; - - isize = PyUnicode_GET_LENGTH(unicode); - idata = PyUnicode_DATA(unicode); + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); /* Compute length of output, quote characters, and maximum character */ - osize = 0; - max = 127; - squote = dquote = 0; - ikind = PyUnicode_KIND(unicode); - for (i = 0; i < isize; i++) { + Py_ssize_t osize = 0; + Py_UCS4 maxch = 127; + Py_ssize_t squote = 0; + Py_ssize_t dquote = 0; + int ikind = PyUnicode_KIND(unicode); + for (Py_ssize_t i = 0; i < isize; i++) { Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); Py_ssize_t incr = 1; switch (ch) { @@ -12245,7 +12647,7 @@ unicode_repr(PyObject *unicode) else if (ch < 0x7f) ; else if (Py_UNICODE_ISPRINTABLE(ch)) - max = ch > max ? ch : max; + maxch = (ch > maxch) ? ch : maxch; else if (ch < 0x100) incr = 4; /* \xHH */ else if (ch < 0x10000) @@ -12261,10 +12663,10 @@ unicode_repr(PyObject *unicode) osize += incr; } - quote = '\''; - unchanged = (osize == isize); + Py_UCS4 quote = '\''; + int changed = (osize != isize); if (squote) { - unchanged = 0; + changed = 1; if (dquote) /* Both squote and dquote present. Use squote, and escape them */ @@ -12274,170 +12676,88 @@ unicode_repr(PyObject *unicode) } osize += 2; /* quotes */ - repr = PyUnicode_New(osize, max); + PyObject *repr = PyUnicode_New(osize, maxch); if (repr == NULL) return NULL; - okind = PyUnicode_KIND(repr); - odata = PyUnicode_DATA(repr); + int okind = PyUnicode_KIND(repr); + void *odata = PyUnicode_DATA(repr); + + if (!changed) { + PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, osize-1, quote); - if (unchanged) { _PyUnicode_FastCopyCharacters(repr, 1, unicode, 0, isize); + + PyUnicode_WRITE(okind, odata, osize-1, quote); } else { - for (i = 0, o = 1; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - - /* Escape quotes and backslashes */ - if ((ch == quote) || (ch == '\\')) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, ch); - continue; - } - - /* Map special whitespace to '\t', \n', '\r' */ - if (ch == '\t') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 't'); - } - else if (ch == '\n') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'n'); - } - else if (ch == '\r') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'r'); - } - - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch == 0x7F) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - - /* Copy ASCII characters as-is */ - else if (ch < 0x7F) { - PyUnicode_WRITE(okind, odata, o++, ch); - } - - /* Non-ASCII characters */ - else { - /* Map Unicode whitespace and control characters - (categories Z* and C* except ASCII space) - */ - if (!Py_UNICODE_ISPRINTABLE(ch)) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - /* Map 8-bit characters to '\xhh' */ - if (ch <= 0xff) { - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch <= 0xffff) { - PyUnicode_WRITE(okind, odata, o++, 'u'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else { - PyUnicode_WRITE(okind, odata, o++, 'U'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - } - /* Copy characters as-is */ - else { - PyUnicode_WRITE(okind, odata, o++, ch); - } - } + switch (okind) { + case PyUnicode_1BYTE_KIND: + ucs1lib_repr(unicode, quote, odata); + break; + case PyUnicode_2BYTE_KIND: + ucs2lib_repr(unicode, quote, odata); + break; + default: + assert(okind == PyUnicode_4BYTE_KIND); + ucs4lib_repr(unicode, quote, odata); } } - /* Closing quote already added at the beginning */ + assert(_PyUnicode_CheckConsistency(repr, 1)); return repr; } -PyDoc_STRVAR(rfind__doc__, - "S.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_rfind(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) - return NULL; - - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; +/*[clinic input] +str.rfind as unicode_rfind = str.count - return PyLong_FromSsize_t(result); -} +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -PyDoc_STRVAR(rindex__doc__, - "S.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ -static PyObject * -unicode_rindex(PyObject *self, PyObject *args) +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) - return NULL; - - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; - + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { - PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; + return -1; } - - return PyLong_FromSsize_t(result); + return result; } /*[clinic input] -str.rjust as unicode_rjust +str.rindex as unicode_rindex = str.count - width: Py_ssize_t +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. + +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ + +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result == -1) { + PyErr_SetString(PyExc_ValueError, "substring not found"); + } + else if (result < 0) { + return -1; + } + return result; +} + +/*[clinic input] +str.rjust as unicode_rjust + + width: Py_ssize_t fillchar: Py_UCS4 = ' ' / @@ -12475,11 +12795,13 @@ str.split as unicode_split character (including \n \r \t \f and spaces) and will discard empty strings from the result. maxsplit: Py_ssize_t = -1 - Maximum number of splits (starting from the left). + Maximum number of splits. -1 (the default value) means no limit. Return a list of the substrings in the string, using sep as the separator string. +Splitting starts at the front of the string and works to the end. + Note, str.split() is mainly useful for data that has been intentionally delimited. With natural text that includes punctuation, consider using the regular expression module. @@ -12488,7 +12810,7 @@ the regular expression module. static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=07b9040d98c5fe8d]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=a29bcc0c7a5af0eb]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -12928,30 +13250,30 @@ unicode_zfill_impl(PyObject *self, Py_ssize_t width) return u; } -PyDoc_STRVAR(startswith__doc__, - "S.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if S starts with the specified prefix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -prefix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +str.startswith as unicode_startswith + + prefix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string starts with the specified prefix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_startswith(PyObject *self, - PyObject *args) +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=4bd7cfd0803051d4 input=5f918b5f5f89d856]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for startswith must only contain str, " @@ -12959,9 +13281,10 @@ unicode_startswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - if (result == -1) + int result = tailmatch(self, substring, start, end, -1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -12975,37 +13298,38 @@ unicode_startswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, -1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, -1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } -PyDoc_STRVAR(endswith__doc__, - "S.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if S ends with the specified suffix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -suffix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +str.endswith as unicode_endswith + + suffix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string ends with the specified suffix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_endswith(PyObject *self, - PyObject *args) +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=00fbdc774a7d4d71]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for endswith must only contain str, " @@ -13013,9 +13337,10 @@ unicode_endswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - if (result == -1) + int result = tailmatch(self, substring, start, end, +1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13028,12 +13353,14 @@ unicode_endswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, +1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, +1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } + static inline void _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { @@ -13057,6 +13384,7 @@ _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) } } + void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { @@ -13065,12 +13393,47 @@ _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) /* ASCII is the bare minimum */ writer->min_char = 127; - /* use a value smaller than PyUnicode_1BYTE_KIND() so + /* use a kind value smaller than PyUnicode_1BYTE_KIND so _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = 0; - assert(writer->kind <= PyUnicode_1BYTE_KIND); + assert(writer->kind == 0); + assert(writer->kind < PyUnicode_1BYTE_KIND); +} + + +PyUnicodeWriter* +PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == NULL) { + return (PyUnicodeWriter *)PyErr_NoMemory(); + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + + return pub_writer; } + +void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + + // Initialize _PyUnicodeWriter with initial buffer static inline void _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) @@ -13081,6 +13444,7 @@ _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) writer->min_length = writer->size; } + int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) @@ -13088,6 +13452,7 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t newlen; PyObject *newbuffer; + assert(length >= 0); assert(maxchar <= MAX_UNICODE); /* ensure that the _PyUnicodeWriter_Prepare macro was used */ @@ -13196,9 +13561,23 @@ _PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) return _PyUnicodeWriter_WriteCharInline(writer, ch); } +int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > MAX_UNICODE) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + int _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) { + assert(PyUnicode_Check(str)); + Py_UCS4 maxchar; Py_ssize_t len; @@ -13224,6 +13603,34 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) return 0; } +int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + + +int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *repr = PyObject_Repr(obj); + if (repr == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); + Py_DECREF(repr); + return res; +} + + int _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) @@ -13256,6 +13663,29 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, return 0; } + +int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + + int _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, const char *ascii, Py_ssize_t len) @@ -13316,6 +13746,51 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } +int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + if (size < 0) { + size = strlen(str); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, str, size, + _Py_ERROR_STRICT, NULL, NULL); + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + + +int +PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, + const char *string, + Py_ssize_t length, + const char *errors, + Py_ssize_t *consumed) +{ + if (length < 0) { + length = strlen(string); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, string, length, + _Py_ERROR_UNKNOWN, errors, consumed); + if (res < 0) { + _writer->pos = old_pos; + if (consumed) { + *consumed = 0; + } + } + return res; +} + + int _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, const char *str, Py_ssize_t len) @@ -13362,6 +13837,17 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) return unicode_result(str); } + +PyObject* +PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + + void _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) { @@ -13371,15 +13857,17 @@ _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) #include "stringlib/unicode_format.h" PyDoc_STRVAR(format__doc__, - "S.format(*args, **kwargs) -> str\n\ + "format($self, /, *args, **kwargs)\n\ +--\n\ \n\ -Return a formatted version of S, using substitutions from args and kwargs.\n\ +Return a formatted version of the string, using substitutions from args and kwargs.\n\ The substitutions are identified by braces ('{' and '}')."); PyDoc_STRVAR(format_map__doc__, - "S.format_map(mapping) -> str\n\ + "format_map($self, mapping, /)\n\ +--\n\ \n\ -Return a formatted version of S, using substitutions from mapping.\n\ +Return a formatted version of the string, using substitutions from mapping.\n\ The substitutions are identified by braces ('{' and '}')."); /*[clinic input] @@ -13463,16 +13951,16 @@ static PyMethodDef unicode_methods[] = { UNICODE_CASEFOLD_METHODDEF UNICODE_TITLE_METHODDEF UNICODE_CENTER_METHODDEF - {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, + UNICODE_COUNT_METHODDEF UNICODE_EXPANDTABS_METHODDEF - {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__}, + UNICODE_FIND_METHODDEF UNICODE_PARTITION_METHODDEF - {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__}, + UNICODE_INDEX_METHODDEF UNICODE_LJUST_METHODDEF UNICODE_LOWER_METHODDEF UNICODE_LSTRIP_METHODDEF - {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__}, + UNICODE_RFIND_METHODDEF + UNICODE_RINDEX_METHODDEF UNICODE_RJUST_METHODDEF UNICODE_RSTRIP_METHODDEF UNICODE_RPARTITION_METHODDEF @@ -13481,8 +13969,8 @@ static PyMethodDef unicode_methods[] = { UNICODE_SWAPCASE_METHODDEF UNICODE_TRANSLATE_METHODDEF UNICODE_UPPER_METHODDEF - {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__}, - {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__}, + UNICODE_STARTSWITH_METHODDEF + UNICODE_ENDSWITH_METHODDEF UNICODE_REMOVEPREFIX_METHODDEF UNICODE_REMOVESUFFIX_METHODDEF UNICODE_ISASCII_METHODDEF @@ -13943,14 +14431,21 @@ formatchar(PyObject *v) if (PyUnicode_GET_LENGTH(v) == 1) { return PyUnicode_READ_CHAR(v, 0); } - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; } else { int overflow; long x = PyLong_AsLongAndOverflow(v, &overflow); if (x == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; } return (Py_UCS4) -1; } @@ -13964,11 +14459,6 @@ formatchar(PyObject *v) return (Py_UCS4) x; } - - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires int or char"); - return (Py_UCS4) -1; } /* Parse options of an argument: flags, width, precision. @@ -14584,6 +15074,56 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, return unicode; } +static const char * +arg_as_utf8(PyObject *obj, const char *name) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "str() argument '%s' must be str, not %T", + name, obj); + return NULL; + } + return _PyUnicode_AsUTF8NoNUL(obj); +} + +static PyObject * +unicode_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + assert(Py_Is(_PyType_CAST(type), &PyUnicode_Type)); + + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { + // Fallback to unicode_new() + PyObject *tuple = _PyTuple_FromArray(args, nargs); + if (tuple == NULL) { + return NULL; + } + PyObject *dict = _PyStack_AsDict(args + nargs, kwnames); + if (dict == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyObject *ret = unicode_new(_PyType_CAST(type), tuple, dict); + Py_DECREF(tuple); + Py_DECREF(dict); + return ret; + } + if (!_PyArg_CheckPositional("str", nargs, 0, 3)) { + return NULL; + } + if (nargs == 0) { + return unicode_get_empty(); + } + PyObject *object = args[0]; + if (nargs == 1) { + return PyObject_Str(object); + } + const char *encoding = arg_as_utf8(args[1], "encoding"); + const char *errors = (nargs == 3) ? arg_as_utf8(args[2], "errors") : NULL; + return PyUnicode_FromEncodedObject(object, encoding, errors); +} + static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *unicode) { @@ -14637,7 +15177,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) PyErr_NoMemory(); goto onError; } - data = PyObject_Malloc((length + 1) * char_size); + data = PyMem_Malloc((length + 1) * char_size); if (data == NULL) { PyErr_NoMemory(); goto onError; @@ -14677,7 +15217,7 @@ errors is specified, then the object must expose a data buffer\n\ that will be decoded using the given encoding and error handler.\n\ Otherwise, returns the result of object.__str__() (if defined)\n\ or repr(object).\n\ -encoding defaults to sys.getdefaultencoding().\n\ +encoding defaults to 'utf-8'.\n\ errors defaults to 'strict'."); static PyObject *unicode_iter(PyObject *seq); @@ -14725,6 +15265,7 @@ PyTypeObject PyUnicode_Type = { 0, /* tp_alloc */ unicode_new, /* tp_new */ PyObject_Del, /* tp_free */ + .tp_vectorcall = unicode_vectorcall, }; /* Initialize the Unicode implementation */ @@ -14767,30 +15308,19 @@ _PyUnicode_InitState(PyInterpreterState *interp) PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) { - // Initialize the global interned dict + if (_Py_IsMainInterpreter(interp)) { + PyStatus status = init_global_interned_strings(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + assert(INTERNED_STRINGS); + if (init_interned_dict(interp)) { PyErr_Clear(); return _PyStatus_ERR("failed to create interned dict"); } - if (_Py_IsMainInterpreter(interp)) { - /* Intern statically allocated string identifiers and deepfreeze strings. - * This must be done before any module initialization so that statically - * allocated string identifiers are used instead of heap allocated strings. - * Deepfreeze uses the interned identifiers if present to save space - * else generates them and they are interned to speed up dict lookups. - */ - _PyUnicode_InitStaticStrings(interp); - -#ifdef Py_DEBUG - assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); - - for (int i = 0; i < 256; i++) { - assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); - } -#endif - } - return _PyStatus_OK(); } @@ -14813,104 +15343,267 @@ _PyUnicode_InitTypes(PyInterpreterState *interp) return _PyStatus_ERR("Can't initialize unicode types"); } +static /* non-null */ PyObject* +intern_static(PyInterpreterState *interp, PyObject *s /* stolen */) +{ + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + + assert(s != NULL); + assert(_PyUnicode_CHECK(s)); + assert(_PyUnicode_STATE(s).statically_allocated); + assert(_Py_IsImmortal(s)); + + switch (PyUnicode_CHECK_INTERNED(s)) { + case SSTATE_NOT_INTERNED: + break; + case SSTATE_INTERNED_IMMORTAL_STATIC: + return s; + default: + Py_FatalError("_PyUnicode_InternStatic called on wrong string"); + } + +#ifdef Py_DEBUG + /* We must not add process-global interned string if there's already a + * per-interpreter interned_dict, which might contain duplicates. + * Except "short string" singletons: those are special-cased. */ + PyObject *interned = get_interned_dict(interp); + assert(interned == NULL || unicode_is_singleton(s)); +#ifdef Py_GIL_DISABLED + // In the free-threaded build, don't allow even the short strings. + assert(interned == NULL); +#endif +#endif + + /* Look in the global cache first. */ + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + /* We should only init each string once */ + assert(r == NULL); + /* but just in case (for the non-debug build), handle this */ + if (r != NULL && r != s) { + assert(_PyUnicode_STATE(r).interned == SSTATE_INTERNED_IMMORTAL_STATIC); + assert(_PyUnicode_CHECK(r)); + Py_DECREF(s); + return Py_NewRef(r); + } + + if (_Py_hashtable_set(INTERNED_STRINGS, s, s) < -1) { + Py_FatalError("failed to intern static string"); + } + + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + return s; +} void -_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +_PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **p) +{ + // This should only be called as part of runtime initialization + assert(!Py_IsInitialized()); + + *p = intern_static(interp, *p); + assert(*p); +} + +static void +immortalize_interned(PyObject *s) +{ + assert(PyUnicode_CHECK_INTERNED(s) == SSTATE_INTERNED_MORTAL); + assert(!_Py_IsImmortal(s)); +#ifdef Py_REF_DEBUG + /* The reference count value should be excluded from the RefTotal. + The decrements to these objects will not be registered so they + need to be accounted for in here. */ + for (Py_ssize_t i = 0; i < Py_REFCNT(s); i++) { + _Py_DecRefTotal(_PyThreadState_GET()); + } +#endif + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL; + _Py_SetImmortal(s); +} + +static /* non-null */ PyObject* +intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, + bool immortalize) { - PyObject *s = *p; + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + #ifdef Py_DEBUG assert(s != NULL); assert(_PyUnicode_CHECK(s)); #else if (s == NULL || !PyUnicode_Check(s)) { - return; + return s; } #endif /* If it's a subclass, we don't really know what putting it in the interned dict might do. */ if (!PyUnicode_CheckExact(s)) { - return; + return s; } - if (PyUnicode_CHECK_INTERNED(s)) { - return; + /* Handle statically allocated strings. */ + if (_PyUnicode_STATE(s).statically_allocated) { + return intern_static(interp, s); } - /* Look in the global cache first. */ - PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); - if (r != NULL && r != s) { - Py_SETREF(*p, Py_NewRef(r)); - return; + /* Is it already interned? */ + switch (PyUnicode_CHECK_INTERNED(s)) { + case SSTATE_NOT_INTERNED: + // no, go on + break; + case SSTATE_INTERNED_MORTAL: + // yes but we might need to make it immortal + if (immortalize) { + immortalize_interned(s); + } + return s; + default: + // all done + return s; } - /* Handle statically allocated strings. */ - if (_PyUnicode_STATE(s).statically_allocated) { - assert(_Py_IsImmortal(s)); - if (_Py_hashtable_set(INTERNED_STRINGS, s, s) == 0) { - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; +#if Py_GIL_DISABLED + /* In the free-threaded build, all interned strings are immortal */ + immortalize = 1; +#endif + + /* If it's already immortal, intern it as such */ + if (_Py_IsImmortal(s)) { + immortalize = 1; + } + + /* if it's a short string, get the singleton -- and intern it */ + if (PyUnicode_GET_LENGTH(s) == 1 && + PyUnicode_KIND(s) == PyUnicode_1BYTE_KIND) { + PyObject *r = LATIN1(*(unsigned char*)PyUnicode_DATA(s)); + if (!PyUnicode_CHECK_INTERNED(r)) { + r = intern_static(interp, r); + } + Py_DECREF(s); + return r; + } +#ifdef Py_DEBUG + assert(!unicode_is_singleton(s)); +#endif + + /* Look in the global cache now. */ + { + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + if (r != NULL) { + assert(_Py_IsImmortal(r)); + assert(r != s); // r must be statically_allocated; s is not + Py_DECREF(s); + return Py_NewRef(r); } - return; } - /* Look in the per-interpreter cache. */ + /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - PyObject *t = PyDict_SetDefault(interned, s, s); - if (t == NULL) { - PyErr_Clear(); - return; + PyObject *t; + { + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { + PyErr_Clear(); + return s; + } + else if (res == 1) { + // value was already present (not inserted) + Py_DECREF(s); + if (immortalize && + PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { + immortalize_interned(t); + } + return t; + } + else { + // value was newly inserted + assert (s == t); + Py_DECREF(t); + } } - if (t != s) { - Py_SETREF(*p, Py_NewRef(t)); - return; - } + /* NOT_INTERNED -> INTERNED_MORTAL */ - if (_Py_IsImmortal(s)) { - // XXX Restrict this to the main interpreter? - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; - return; - } + assert(_PyUnicode_STATE(s).interned == SSTATE_NOT_INTERNED); + if (!_Py_IsImmortal(s)) { + /* The two references in interned dict (key and value) are not counted. + unicode_dealloc() and _PyUnicode_ClearInterned() take care of this. */ + Py_SET_REFCNT(s, Py_REFCNT(s) - 2); #ifdef Py_REF_DEBUG - /* The reference count value excluding the 2 references from the - interned dictionary should be excluded from the RefTotal. The - decrements to these objects will not be registered so they - need to be accounted for in here. */ - for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) { - _Py_DecRefTotal(_PyInterpreterState_GET()); + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + } + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; + + /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ + +#ifdef Py_DEBUG + if (_Py_IsImmortal(s)) { + assert(immortalize); } #endif - _Py_SetImmortal(s); - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; + if (immortalize) { + immortalize_interned(s); + } + + return s; +} + +void +_PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 1); + assert(*p); +} + +void +_PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 0); + assert(*p); +} + + +void +_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +{ + _PyUnicode_InternImmortal(interp, p); + return; } void PyUnicode_InternInPlace(PyObject **p) { PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternInPlace(interp, p); + _PyUnicode_InternImmortal(interp, p); } -// Function kept for the stable ABI. +// Public-looking name kept for the stable ABI; user should not call this: PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **); void PyUnicode_InternImmortal(PyObject **p) { - PyUnicode_InternInPlace(p); - // Leak a reference on purpose - Py_INCREF(*p); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, p); } PyObject * PyUnicode_InternFromString(const char *cp) { PyObject *s = PyUnicode_FromString(cp); - if (s == NULL) + if (s == NULL) { return NULL; - PyUnicode_InternInPlace(&s); + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &s); return s; } @@ -14924,20 +15617,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); - /* TODO: - * Currently, the runtime is not able to guarantee that it can exit without - * allocations that carry over to a future initialization of Python within - * the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * Therefore, this should remain disabled for until there is a strict guarantee - * that no memory will be left after `Py_Finalize`. - */ -#ifdef Py_DEBUG - /* For all non-singleton interned strings, restore the two valid references - to that instance from within the intern string dictionary and let the - normal reference counting process clean up these instances. */ #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -14951,13 +15630,32 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: + /* Make immortal interned strings mortal again. + * + * Currently, the runtime is not able to guarantee that it can exit + * without allocations that carry over to a future initialization + * of Python within the same process. i.e: + * ./python -X showrefcount -c 'import itertools' + * [237 refs, 237 blocks] + * + * This should remain disabled (`Py_DEBUG` only) until there is a + * strict guarantee that no memory will be left after + * `Py_Finalize`. + */ +#ifdef Py_DEBUG // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). - s->ob_refcnt = 2; + _Py_SetMortal(s, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif #ifdef INTERNED_STATS total_length += PyUnicode_GET_LENGTH(s); #endif +#endif // Py_DEBUG break; case SSTATE_INTERNED_IMMORTAL_STATIC: /* It is shared between interpreters, so we should unmark it @@ -14970,9 +15668,17 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } break; case SSTATE_INTERNED_MORTAL: - /* fall through */ + // Restore 2 references held by the interned dict; these will + // be decref'd by clear_interned_dict's PyDict_Clear. + Py_SET_REFCNT(s, Py_REFCNT(s) + 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + break; case SSTATE_NOT_INTERNED: - /* fall through */ + _Py_FALLTHROUGH; default: Py_UNREACHABLE(); } @@ -14991,8 +15697,10 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) for (Py_ssize_t i=0; i < ids->size; i++) { Py_XINCREF(ids->array[i]); } -#endif /* Py_DEBUG */ clear_interned_dict(interp); + if (_Py_IsMainInterpreter(interp)) { + clear_global_interned_strings(); + } } @@ -15355,7 +16063,11 @@ init_fs_encoding(PyThreadState *tstate) PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate) { - PyStatus status = init_fs_encoding(tstate); + PyStatus status = _PyCodec_InitRegistry(tstate->interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = init_fs_encoding(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -15415,9 +16127,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - _PyStaticType_Dealloc(interp, &EncodingMapType); - _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); - _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); + _PyStaticType_FiniBuiltin(interp, &EncodingMapType); + _PyStaticType_FiniBuiltin(interp, &PyFieldNameIter_Type); + _PyStaticType_FiniBuiltin(interp, &PyFormatterIter_Type); } @@ -15451,6 +16163,7 @@ static PyMethodDef _string_methods[] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index 39a567dc46e89a..e6dbeffbe2aa3e 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -1,4 +1,4 @@ -/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ /* a list of unique character type descriptors */ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index bf5605686f8df7..7931f4345f7fdd 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -81,7 +81,7 @@ is_same(PyObject *left, PyObject *right) static int contains(PyObject **items, Py_ssize_t size, PyObject *obj) { - for (int i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < size; i++) { int is_duplicate = is_same(items[i], obj); if (is_duplicate) { // -1 or 1 return is_duplicate; @@ -97,7 +97,7 @@ merge(PyObject **items1, Py_ssize_t size1, PyObject *tuple = NULL; Py_ssize_t pos = 0; - for (int i = 0; i < size2; i++) { + for (Py_ssize_t i = 0; i < size2; i++) { PyObject *arg = items2[i]; int is_duplicate = contains(items1, size1, arg); if (is_duplicate < 0) { @@ -182,15 +182,14 @@ _Py_union_type_or(PyObject* self, PyObject* other) } static int -union_repr_item(_PyUnicodeWriter *writer, PyObject *p) +union_repr_item(PyUnicodeWriter *writer, PyObject *p) { PyObject *qualname = NULL; PyObject *module = NULL; - PyObject *r = NULL; int rc; if (p == (PyObject *)&_PyNone_Type) { - return _PyUnicodeWriter_WriteASCIIString(writer, "None", 4); + return PyUnicodeWriter_WriteUTF8(writer, "None", 4); } if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && @@ -200,17 +199,17 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) goto use_repr; } if (rc < 0) { - goto exit; + goto error; } if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto exit; + goto error; } if (qualname == NULL) { goto use_repr; } if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto exit; + goto error; } if (module == NULL || module == Py_None) { goto use_repr; @@ -221,24 +220,25 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) _PyUnicode_EqualToASCIIString(module, "builtins")) { // builtins don't need a module name - r = PyObject_Str(qualname); - goto exit; + rc = PyUnicodeWriter_WriteStr(writer, qualname); + goto done; } else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto exit; + rc = PyUnicodeWriter_Format(writer, "%S.%S", module, qualname); + goto done; } +error: + rc = -1; + goto done; + use_repr: - r = PyObject_Repr(p); -exit: + rc = PyUnicodeWriter_WriteRepr(writer, p); + goto done; + +done: Py_XDECREF(qualname); Py_XDECREF(module); - if (r == NULL) { - return -1; - } - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); return rc; } @@ -248,20 +248,26 @@ union_repr(PyObject *self) unionobject *alias = (unionobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && _PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { + // Shortest type name "int" (3 chars) + " | " (3 chars) separator + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 6) ? len * 6 : len; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (union_repr_item(&writer, p) < 0) { + if (union_repr_item(writer, p) < 0) { goto error; } } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index df74be6aba7244..61f05514a48023 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,24 +1,58 @@ #include "Python.h" +#include "pycore_critical_section.h" +#include "pycore_lock.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#ifdef Py_GIL_DISABLED +/* + * Thread-safety for free-threaded builds + * ====================================== + * + * In free-threaded builds we need to protect mutable state of: + * + * - The weakref (wr_object, hash, wr_callback) + * - The referenced object (its head-of-list pointer) + * - The linked list of weakrefs + * + * For now we've chosen to address this in a straightforward way: + * + * - The weakref's hash is protected using the weakref's per-object lock. + * - The other mutable is protected by a striped lock keyed on the referenced + * object's address. + * - The striped lock must be locked using `_Py_LOCK_DONT_DETACH` in order to + * support atomic deletion from WeakValueDictionaries. As a result, we must + * be careful not to perform any operations that could suspend while the + * lock is held. + * + * Since the world is stopped when the GC runs, it is free to clear weakrefs + * without acquiring any locks. + */ +#endif #define GET_WEAKREFS_LISTPTR(o) \ ((PyWeakReference **) _PyObject_GET_WEAKREFS_LISTPTR(o)) Py_ssize_t -_PyWeakref_GetWeakrefCount(PyWeakReference *head) +_PyWeakref_GetWeakrefCount(PyObject *obj) { - Py_ssize_t count = 0; + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + return 0; + } + LOCK_WEAKREFS(obj); + Py_ssize_t count = 0; + PyWeakReference *head = *GET_WEAKREFS_LISTPTR(obj); while (head != NULL) { ++count; head = head->wr_next; } + UNLOCK_WEAKREFS(obj); return count; } @@ -33,54 +67,55 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) self->wr_next = NULL; self->wr_callback = Py_XNewRef(callback); self->vectorcall = weakref_vectorcall; +#ifdef Py_GIL_DISABLED + self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); + _PyObject_SetMaybeWeakref(ob); + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif } -static PyWeakReference * -new_weakref(PyObject *ob, PyObject *callback) -{ - PyWeakReference *result; - - result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType); - if (result) { - init_weakref(result, ob, callback); - PyObject_GC_Track(result); - } - return result; -} - - -/* This function clears the passed-in reference and removes it from the - * list of weak references for the referent. This is the only code that - * removes an item from the doubly-linked list of weak references for an - * object; it is also responsible for clearing the callback slot. - */ +// Clear the weakref and steal its callback into `callback`, if provided. static void -clear_weakref(PyWeakReference *self) +clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) { - PyObject *callback = self->wr_callback; - if (self->wr_object != Py_None) { PyWeakReference **list = GET_WEAKREFS_LISTPTR(self->wr_object); - - if (*list == self) - /* If 'self' is the end of the list (and thus self->wr_next == NULL) - then the weakref list itself (and thus the value of *list) will - end up being set to NULL. */ - *list = self->wr_next; - self->wr_object = Py_None; - if (self->wr_prev != NULL) + if (*list == self) { + /* If 'self' is the end of the list (and thus self->wr_next == + NULL) then the weakref list itself (and thus the value of *list) + will end up being set to NULL. */ + FT_ATOMIC_STORE_PTR(*list, self->wr_next); + } + FT_ATOMIC_STORE_PTR(self->wr_object, Py_None); + if (self->wr_prev != NULL) { self->wr_prev->wr_next = self->wr_next; - if (self->wr_next != NULL) + } + if (self->wr_next != NULL) { self->wr_next->wr_prev = self->wr_prev; + } self->wr_prev = NULL; self->wr_next = NULL; } if (callback != NULL) { - Py_DECREF(callback); + *callback = self->wr_callback; self->wr_callback = NULL; } } +// Clear the weakref and its callback +static void +clear_weakref(PyWeakReference *self) +{ + PyObject *callback = NULL; + // self->wr_object may be Py_None if the GC cleared the weakref, so lock + // using the pointer in the weakref. + LOCK_WEAKREFS_FOR_WR(self); + clear_weakref_lock_held(self, &callback); + UNLOCK_WEAKREFS_FOR_WR(self); + Py_XDECREF(callback); +} + + /* Cyclic gc uses this to *just* clear the passed-in reference, leaving * the callback intact and uncalled. It must be possible to call self's * tp_dealloc() after calling this, so self has to be left in a sane enough @@ -95,15 +130,9 @@ clear_weakref(PyWeakReference *self) void _PyWeakref_ClearRef(PyWeakReference *self) { - PyObject *callback; - assert(self != NULL); assert(PyWeakref_Check(self)); - /* Preserve and restore the callback around clear_weakref. */ - callback = self->wr_callback; - self->wr_callback = NULL; - clear_weakref(self); - self->wr_callback = callback; + clear_weakref_lock_held(self, NULL); } static void @@ -126,7 +155,11 @@ gc_traverse(PyWeakReference *self, visitproc visit, void *arg) static int gc_clear(PyWeakReference *self) { - clear_weakref(self); + PyObject *callback; + // The world is stopped during GC in free-threaded builds. It's safe to + // call this without holding the lock. + clear_weakref_lock_held(self, &callback); + Py_XDECREF(callback); return 0; } @@ -150,7 +183,7 @@ weakref_vectorcall(PyObject *self, PyObject *const *args, } static Py_hash_t -weakref_hash(PyWeakReference *self) +weakref_hash_lock_held(PyWeakReference *self) { if (self->hash != -1) return self->hash; @@ -164,6 +197,15 @@ weakref_hash(PyWeakReference *self) return self->hash; } +static Py_hash_t +weakref_hash(PyWeakReference *self) +{ + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(self); + hash = weakref_hash_lock_held(self); + Py_END_CRITICAL_SECTION(); + return hash; +} static PyObject * weakref_repr(PyObject *self) @@ -177,13 +219,13 @@ weakref_repr(PyObject *self) PyObject *repr; if (name == NULL || !PyUnicode_Check(name)) { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj); + "", + self, obj, obj); } else { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj, name); + "", + self, obj, obj, name); } Py_DECREF(obj); Py_XDECREF(name); @@ -276,6 +318,135 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) *list = newref; } +/* See if we can reuse either the basic ref or proxy in list instead of + * creating a new weakref + */ +static PyWeakReference * +try_reuse_basic_ref(PyWeakReference *list, PyTypeObject *type, + PyObject *callback) +{ + if (callback != NULL) { + return NULL; + } + + PyWeakReference *ref, *proxy; + get_basic_refs(list, &ref, &proxy); + + PyWeakReference *cand = NULL; + if (type == &_PyWeakref_RefType) { + cand = ref; + } + if ((type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) { + cand = proxy; + } + + if (cand != NULL && _Py_TryIncref((PyObject *) cand)) { + return cand; + } + return NULL; +} + +static int +is_basic_ref(PyWeakReference *ref) +{ + return (ref->wr_callback == NULL) && PyWeakref_CheckRefExact(ref); +} + +static int +is_basic_proxy(PyWeakReference *proxy) +{ + return (proxy->wr_callback == NULL) && PyWeakref_CheckProxy(proxy); +} + +static int +is_basic_ref_or_proxy(PyWeakReference *wr) +{ + return is_basic_ref(wr) || is_basic_proxy(wr); +} + +/* Insert `newref` in the appropriate position in `list` */ +static void +insert_weakref(PyWeakReference *newref, PyWeakReference **list) +{ + PyWeakReference *ref, *proxy; + get_basic_refs(*list, &ref, &proxy); + + PyWeakReference *prev; + if (is_basic_ref(newref)) { + prev = NULL; + } + else if (is_basic_proxy(newref)) { + prev = ref; + } + else { + prev = (proxy == NULL) ? ref : proxy; + } + + if (prev == NULL) { + insert_head(newref, list); + } + else { + insert_after(newref, prev); + } +} + +static PyWeakReference * +allocate_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + PyWeakReference *newref = (PyWeakReference *) type->tp_alloc(type, 0); + if (newref == NULL) { + return NULL; + } + init_weakref(newref, obj, callback); + return newref; +} + +static PyWeakReference * +get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + PyErr_Format(PyExc_TypeError, + "cannot create weak reference to '%s' object", + Py_TYPE(obj)->tp_name); + return NULL; + } + if (callback == Py_None) + callback = NULL; + + PyWeakReference **list = GET_WEAKREFS_LISTPTR(obj); + if ((type == &_PyWeakref_RefType) || + (type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) + { + LOCK_WEAKREFS(obj); + PyWeakReference *basic_ref = try_reuse_basic_ref(*list, type, callback); + if (basic_ref != NULL) { + UNLOCK_WEAKREFS(obj); + return basic_ref; + } + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + UNLOCK_WEAKREFS(obj); + return NULL; + } + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } + else { + // We may not be able to safely allocate inside the lock + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + return NULL; + } + LOCK_WEAKREFS(obj); + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } +} + static int parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) @@ -286,54 +457,11 @@ parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, static PyObject * weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - PyWeakReference *self = NULL; PyObject *ob, *callback = NULL; - if (parse_weakref_init_args("__new__", args, kwargs, &ob, &callback)) { - PyWeakReference *ref, *proxy; - PyWeakReference **list; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - if (callback == Py_None) - callback = NULL; - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL && type == &_PyWeakref_RefType) { - if (ref != NULL) { - /* We can re-use an existing reference. */ - return Py_NewRef(ref); - } - } - /* We have to create a new reference. */ - /* Note: the tp_alloc() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - self = (PyWeakReference *) (type->tp_alloc(type, 0)); - if (self != NULL) { - init_weakref(self, ob, callback); - if (callback == NULL && type == &_PyWeakref_RefType) { - insert_head(self, list); - } - else { - PyWeakReference *prev; - - get_basic_refs(*list, &ref, &proxy); - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(self, list); - else - insert_after(self, prev); - } - } + return (PyObject *)get_or_create_weakref(type, ob, callback); } - return (PyObject *)self; + return NULL; } static int @@ -471,10 +599,18 @@ static PyObject * proxy_repr(PyObject *proxy) { PyObject *obj = _PyWeakref_GET_REF(proxy); - PyObject *repr = PyUnicode_FromFormat( - "", - proxy, Py_TYPE(obj)->tp_name, obj); - Py_DECREF(obj); + PyObject *repr; + if (obj != NULL) { + repr = PyUnicode_FromFormat( + "", + proxy, obj, obj); + Py_DECREF(obj); + } + else { + repr = PyUnicode_FromFormat( + "", + proxy); + } return repr; } @@ -554,8 +690,6 @@ static void proxy_dealloc(PyWeakReference *self) { PyObject_GC_UnTrack(self); - if (self->wr_callback != NULL) - PyObject_GC_UnTrack((PyObject *)self); clear_weakref(self); PyObject_GC_Del(self); } @@ -776,127 +910,21 @@ _PyWeakref_CallableProxyType = { proxy_iternext, /* tp_iternext */ }; - - PyObject * PyWeakref_NewRef(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* return existing weak reference if it exists */ - result = ref; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (ref == NULL) - insert_head(result, list); - else { - /* Someone else added a ref without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(ref)); - } - } - else { - PyWeakReference *prev; - - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - } - } - } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(&_PyWeakref_RefType, ob, + callback); } - PyObject * PyWeakref_NewProxy(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* attempt to return an existing weak reference if it exists */ - result = proxy; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - PyWeakReference *prev; - - if (PyCallable_Check(ob)) { - Py_SET_TYPE(result, &_PyWeakref_CallableProxyType); - } - else { - Py_SET_TYPE(result, &_PyWeakref_ProxyType); - } - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (proxy != NULL) { - /* Someone else added a proxy without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(proxy)); - goto skip_insert; - } - prev = ref; - } - else - prev = (proxy == NULL) ? ref : proxy; - - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - skip_insert: - ; - } + PyTypeObject *type = &_PyWeakref_ProxyType; + if (PyCallable_Check(ob)) { + type = &_PyWeakref_CallableProxyType; } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(type, ob, callback); } @@ -965,67 +993,82 @@ PyObject_ClearWeakRefs(PyObject *object) PyErr_BadInternalCall(); return; } + list = GET_WEAKREFS_LISTPTR(object); - /* Remove the callback-less basic and proxy references */ - if (*list != NULL && (*list)->wr_callback == NULL) { - clear_weakref(*list); - if (*list != NULL && (*list)->wr_callback == NULL) - clear_weakref(*list); + if (FT_ATOMIC_LOAD_PTR(*list) == NULL) { + // Fast path for the common case + return; } - if (*list != NULL) { - PyWeakReference *current = *list; - Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *exc = PyErr_GetRaisedException(); - - if (count == 1) { - PyObject *callback = current->wr_callback; - - current->wr_callback = NULL; - clear_weakref(current); - if (callback != NULL) { - if (Py_REFCNT((PyObject *)current) > 0) { - handle_callback(current, callback); - } - Py_DECREF(callback); - } + + /* Remove the callback-less basic and proxy references, which always appear + at the head of the list. + */ + for (int done = 0; !done;) { + LOCK_WEAKREFS(object); + if (*list != NULL && is_basic_ref_or_proxy(*list)) { + PyObject *callback; + clear_weakref_lock_held(*list, &callback); + assert(callback == NULL); } - else { - PyObject *tuple; - Py_ssize_t i = 0; - - tuple = PyTuple_New(count * 2); - if (tuple == NULL) { - _PyErr_ChainExceptions1(exc); - return; - } + done = (*list == NULL) || !is_basic_ref_or_proxy(*list); + UNLOCK_WEAKREFS(object); + } - for (i = 0; i < count; ++i) { - PyWeakReference *next = current->wr_next; - - if (Py_REFCNT((PyObject *)current) > 0) { - PyTuple_SET_ITEM(tuple, i * 2, Py_NewRef(current)); - PyTuple_SET_ITEM(tuple, i * 2 + 1, current->wr_callback); - } - else { - Py_DECREF(current->wr_callback); - } - current->wr_callback = NULL; - clear_weakref(current); - current = next; - } - for (i = 0; i < count; ++i) { - PyObject *callback = PyTuple_GET_ITEM(tuple, i * 2 + 1); - - /* The tuple may have slots left to NULL */ - if (callback != NULL) { - PyObject *item = PyTuple_GET_ITEM(tuple, i * 2); - handle_callback((PyWeakReference *)item, callback); - } + /* Deal with non-canonical (subtypes or refs with callbacks) references. */ + Py_ssize_t num_weakrefs = _PyWeakref_GetWeakrefCount(object); + if (num_weakrefs == 0) { + return; + } + + PyObject *exc = PyErr_GetRaisedException(); + PyObject *tuple = PyTuple_New(num_weakrefs * 2); + if (tuple == NULL) { + _PyWeakref_ClearWeakRefsNoCallbacks(object); + PyErr_WriteUnraisable(NULL); + PyErr_SetRaisedException(exc); + return; + } + + Py_ssize_t num_items = 0; + for (int done = 0; !done;) { + PyObject *callback = NULL; + LOCK_WEAKREFS(object); + PyWeakReference *cur = *list; + if (cur != NULL) { + clear_weakref_lock_held(cur, &callback); + if (_Py_TryIncref((PyObject *) cur)) { + assert(num_items / 2 < num_weakrefs); + PyTuple_SET_ITEM(tuple, num_items, (PyObject *) cur); + PyTuple_SET_ITEM(tuple, num_items + 1, callback); + num_items += 2; + callback = NULL; } - Py_DECREF(tuple); } - assert(!PyErr_Occurred()); - PyErr_SetRaisedException(exc); + done = (*list == NULL); + UNLOCK_WEAKREFS(object); + + Py_XDECREF(callback); + } + + for (Py_ssize_t i = 0; i < num_items; i += 2) { + PyObject *callback = PyTuple_GET_ITEM(tuple, i + 1); + if (callback != NULL) { + PyObject *weakref = PyTuple_GET_ITEM(tuple, i); + handle_callback((PyWeakReference *)weakref, callback); + } + } + + Py_DECREF(tuple); + + assert(!PyErr_Occurred()); + PyErr_SetRaisedException(exc); +} + +void +PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + if (_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + _PyWeakref_ClearWeakRefsNoCallbacks(obj); } } @@ -1038,12 +1081,32 @@ PyObject_ClearWeakRefs(PyObject *object) void _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(interp, type); + managed_static_type_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); - while (*list != NULL) { - /* Note that clear_weakref() pops the first ref off the type's - weaklist before clearing its wr_object and wr_callback. - That is how we're able to loop over the list. */ - clear_weakref((PyWeakReference *)*list); + // This is safe to do without holding the lock in free-threaded builds; + // there is only one thread running and no new threads can be created. + while (*list) { + _PyWeakref_ClearRef((PyWeakReference *)*list); + } +} + +void +_PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + /* Modeled after GET_WEAKREFS_LISTPTR(). + + This is never triggered for static types so we can avoid the + (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + while (*list) { + _PyWeakref_ClearRef(*list); } + UNLOCK_WEAKREFS(obj); +} + +int +_PyWeakref_IsDead(PyObject *weakref) +{ + return _PyWeakref_IS_DEAD(weakref); } diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 1dc0d230c4d7c3..0dcea866f65d35 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,17 +1,16 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" #ifdef MS_WINDOWS -#include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "pycore_runtime.h" // _Py_ID() - #define WIN32_LEAN_AND_MEAN #include #include @@ -32,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -57,20 +57,24 @@ module _testconsole _testconsole.write_input file: object - s: PyBytesObject + s: Py_buffer Writes UTF-16-LE encoded bytes to the console as if typed by a user. [clinic start generated code]*/ static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s) -/*[clinic end generated code: output=48f9563db34aedb3 input=4c774f2d05770bc6]*/ +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s) +/*[clinic end generated code: output=58631a8985426ad3 input=68062f1bb2e52206]*/ { INPUT_RECORD *rec = NULL; - PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( - &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + PyObject *mod = PyImport_ImportModule("_io"); + if (mod == NULL) { + return NULL; + } + + PyTypeObject *winconsoleio_type = (PyTypeObject *)PyObject_GetAttrString(mod, "_WindowsConsoleIO"); + Py_DECREF(mod); if (winconsoleio_type == NULL) { return NULL; } @@ -81,8 +85,8 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, return NULL; } - const wchar_t *p = (const wchar_t *)PyBytes_AS_STRING(s); - DWORD size = (DWORD)PyBytes_GET_SIZE(s) / sizeof(wchar_t); + const wchar_t *p = (const wchar_t *)s->buf; + DWORD size = (DWORD)s->len / sizeof(wchar_t); rec = (INPUT_RECORD*)PyMem_Calloc(size, sizeof(INPUT_RECORD)); if (!rec) @@ -96,9 +100,11 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, prec->Event.KeyEvent.uChar.UnicodeChar = *p; } - HANDLE hInput = _Py_get_osfhandle(((winconsoleio*)file)->fd); - if (hInput == INVALID_HANDLE_VALUE) + HANDLE hInput = (HANDLE)_get_osfhandle(((winconsoleio*)file)->fd); + if (hInput == INVALID_HANDLE_VALUE) { + PyErr_SetFromErrno(PyExc_OSError); goto error; + } DWORD total = 0; while (total < size) { diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 310aa86d94d9b6..48863b90f4cc27 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -8,6 +8,11 @@ // Version history // 2022-08: Initial contribution (Steve Dower) +// clinic/_wmimodule.cpp.h uses internal pycore_modsupport.h API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #define _WIN32_DCOM #include #include @@ -39,6 +44,8 @@ struct _query_data { LPCWSTR query; HANDLE writePipe; HANDLE readPipe; + HANDLE initEvent; + HANDLE connectEvent; }; @@ -75,12 +82,18 @@ _query_thread(LPVOID param) IID_IWbemLocator, (LPVOID *)&locator ); } + if (SUCCEEDED(hr) && !SetEvent(data->initEvent)) { + hr = HRESULT_FROM_WIN32(GetLastError()); + } if (SUCCEEDED(hr)) { hr = locator->ConnectServer( bstr_t(L"ROOT\\CIMV2"), NULL, NULL, 0, NULL, 0, 0, &services ); } + if (SUCCEEDED(hr) && !SetEvent(data->connectEvent)) { + hr = HRESULT_FROM_WIN32(GetLastError()); + } if (SUCCEEDED(hr)) { hr = CoSetProxyBlanket( services, RPC_C_AUTHN_WINNT, RPC_C_AUTHZ_NONE, NULL, @@ -184,6 +197,24 @@ _query_thread(LPVOID param) } +static DWORD +wait_event(HANDLE event, DWORD timeout) +{ + DWORD err = 0; + switch (WaitForSingleObject(event, timeout)) { + case WAIT_OBJECT_0: + break; + case WAIT_TIMEOUT: + err = WAIT_TIMEOUT; + break; + default: + err = GetLastError(); + break; + } + return err; +} + + /*[clinic input] _wmi.exec_query @@ -226,7 +257,11 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) Py_BEGIN_ALLOW_THREADS - if (!CreatePipe(&data.readPipe, &data.writePipe, NULL, 0)) { + data.initEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + data.connectEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!data.initEvent || !data.connectEvent || + !CreatePipe(&data.readPipe, &data.writePipe, NULL, 0)) + { err = GetLastError(); } else { hThread = CreateThread(NULL, 0, _query_thread, (LPVOID*)&data, 0, NULL); @@ -238,6 +273,19 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) } } + // gh-112278: If current user doesn't have permission to query the WMI, the + // function IWbemLocator::ConnectServer will hang for 5 seconds, and there + // is no way to specify the timeout. So we use an Event object to simulate + // a timeout. The initEvent will be set after COM initialization, it will + // take a longer time when first initialized. The connectEvent will be set + // after connected to WMI. + if (!err) { + err = wait_event(data.initEvent, 1000); + if (!err) { + err = wait_event(data.connectEvent, 100); + } + } + while (!err) { if (ReadFile( data.readPipe, @@ -259,28 +307,35 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) CloseHandle(data.readPipe); } - // Allow the thread some time to clean up - switch (WaitForSingleObject(hThread, 1000)) { - case WAIT_OBJECT_0: - // Thread ended cleanly - if (!GetExitCodeThread(hThread, (LPDWORD)&err)) { - err = GetLastError(); - } - break; - case WAIT_TIMEOUT: - // Probably stuck - there's not much we can do, unfortunately - if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = WAIT_TIMEOUT; + if (hThread) { + // Allow the thread some time to clean up + int thread_err; + switch (WaitForSingleObject(hThread, 100)) { + case WAIT_OBJECT_0: + // Thread ended cleanly + if (!GetExitCodeThread(hThread, (LPDWORD)&thread_err)) { + thread_err = GetLastError(); + } + break; + case WAIT_TIMEOUT: + // Probably stuck - there's not much we can do, unfortunately + thread_err = WAIT_TIMEOUT; + break; + default: + thread_err = GetLastError(); + break; } - break; - default: + // An error on our side is more likely to be relevant than one from + // the thread, but if we don't have one on our side we'll take theirs. if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = GetLastError(); + err = thread_err; } - break; + + CloseHandle(hThread); } - CloseHandle(hThread); + CloseHandle(data.initEvent); + CloseHandle(data.connectEvent); hThread = NULL; Py_END_ALLOW_THREADS @@ -307,12 +362,18 @@ static PyMethodDef wmi_functions[] = { { NULL, NULL, 0, NULL } }; +static PyModuleDef_Slot wmi_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef wmi_def = { PyModuleDef_HEAD_INIT, "_wmi", - NULL, // doc - 0, // m_size - wmi_functions + NULL, // doc + 0, // m_size + wmi_functions, // m_methods + wmi_slots, // m_slots }; extern "C" { diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index 99cd302ff34698..4c11e545499ac5 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -2,11 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - #if defined(MS_WINDOWS) PyDoc_STRVAR(_testconsole_write_input__doc__, @@ -16,58 +11,30 @@ PyDoc_STRVAR(_testconsole_write_input__doc__, "Writes UTF-16-LE encoded bytes to the console as if typed by a user."); #define _TESTCONSOLE_WRITE_INPUT_METHODDEF \ - {"write_input", _PyCFunction_CAST(_testconsole_write_input), METH_FASTCALL|METH_KEYWORDS, _testconsole_write_input__doc__}, + {"write_input", (PyCFunction)(void(*)(void))_testconsole_write_input, METH_VARARGS|METH_KEYWORDS, _testconsole_write_input__doc__}, static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s); +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s); static PyObject * -_testconsole_write_input(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_write_input(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), &_Py_ID(s), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", "s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "write_input", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"file", "s", NULL}; PyObject *file; - PyBytesObject *s; + Py_buffer s = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - file = args[0]; - if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("write_input", "argument 's'", "bytes", args[1]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*:write_input", _keywords, + &file, &s)) goto exit; - } - s = (PyBytesObject *)args[1]; - return_value = _testconsole_write_input_impl(module, file, s); + return_value = _testconsole_write_input_impl(module, file, &s); exit: + /* Cleanup for s */ + if (s.obj) { + PyBuffer_Release(&s); + } + return return_value; } @@ -82,48 +49,21 @@ PyDoc_STRVAR(_testconsole_read_output__doc__, "Reads a str from the console as written to stdout."); #define _TESTCONSOLE_READ_OUTPUT_METHODDEF \ - {"read_output", _PyCFunction_CAST(_testconsole_read_output), METH_FASTCALL|METH_KEYWORDS, _testconsole_read_output__doc__}, + {"read_output", (PyCFunction)(void(*)(void))_testconsole_read_output, METH_VARARGS|METH_KEYWORDS, _testconsole_read_output__doc__}, static PyObject * _testconsole_read_output_impl(PyObject *module, PyObject *file); static PyObject * -_testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_read_output(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "read_output", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"file", NULL}; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:read_output", _keywords, + &file)) goto exit; - } - file = args[0]; return_value = _testconsole_read_output_impl(module, file); exit: @@ -139,4 +79,4 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ -/*[clinic end generated code: output=f59fe72cd4e73704 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d60ce07157e3741a input=a9049054013a1b77]*/ diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index 3ece5e6823b462..fccf3d795d5340 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_wmi_exec_query__doc__, "exec_query($module, /, query)\n" @@ -68,4 +69,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=53821e597fc2aca4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ba04920d127f3ceb input=a9049054013a1b77]*/ diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index 54ae61bd4f10a9..a77d0855af293f 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(msvcrt_heapmin__doc__, "heapmin($module, /)\n" "--\n" @@ -353,10 +355,24 @@ msvcrt_putch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -394,7 +410,10 @@ msvcrt_putwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("putwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "putwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -428,10 +447,24 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -469,7 +502,10 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("ungetwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "ungetwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -695,4 +731,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=525ec6ac4e3cb4f2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 7507c1151a9840..7e5545e113e718 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) @@ -1761,4 +1762,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=d7ae41899af53d7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ee4098b2f143b6a input=a9049054013a1b77]*/ diff --git a/PC/clinic/winsound.c.h b/PC/clinic/winsound.c.h index b8c8e7d04a699f..6cc989565b638e 100644 --- a/PC/clinic/winsound.c.h +++ b/PC/clinic/winsound.c.h @@ -2,11 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - PyDoc_STRVAR(winsound_PlaySound__doc__, "PlaySound($module, /, sound, flags)\n" "--\n" @@ -19,53 +14,22 @@ PyDoc_STRVAR(winsound_PlaySound__doc__, " Flag values, ored together. See module documentation."); #define WINSOUND_PLAYSOUND_METHODDEF \ - {"PlaySound", _PyCFunction_CAST(winsound_PlaySound), METH_FASTCALL|METH_KEYWORDS, winsound_PlaySound__doc__}, + {"PlaySound", (PyCFunction)(void(*)(void))winsound_PlaySound, METH_VARARGS|METH_KEYWORDS, winsound_PlaySound__doc__}, static PyObject * winsound_PlaySound_impl(PyObject *module, PyObject *sound, int flags); static PyObject * -winsound_PlaySound(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +winsound_PlaySound(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(sound), &_Py_ID(flags), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"sound", "flags", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "PlaySound", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"sound", "flags", NULL}; PyObject *sound; int flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - sound = args[0]; - flags = PyLong_AsInt(args[1]); - if (flags == -1 && PyErr_Occurred()) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oi:PlaySound", _keywords, + &sound, &flags)) goto exit; - } return_value = winsound_PlaySound_impl(module, sound, flags); exit: @@ -85,56 +49,22 @@ PyDoc_STRVAR(winsound_Beep__doc__, " How long the sound should play, in milliseconds."); #define WINSOUND_BEEP_METHODDEF \ - {"Beep", _PyCFunction_CAST(winsound_Beep), METH_FASTCALL|METH_KEYWORDS, winsound_Beep__doc__}, + {"Beep", (PyCFunction)(void(*)(void))winsound_Beep, METH_VARARGS|METH_KEYWORDS, winsound_Beep__doc__}, static PyObject * winsound_Beep_impl(PyObject *module, int frequency, int duration); static PyObject * -winsound_Beep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +winsound_Beep(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(frequency), &_Py_ID(duration), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"frequency", "duration", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "Beep", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"frequency", "duration", NULL}; int frequency; int duration; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - frequency = PyLong_AsInt(args[0]); - if (frequency == -1 && PyErr_Occurred()) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ii:Beep", _keywords, + &frequency, &duration)) goto exit; - } - duration = PyLong_AsInt(args[1]); - if (duration == -1 && PyErr_Occurred()) { - goto exit; - } return_value = winsound_Beep_impl(module, frequency, duration); exit: @@ -150,59 +80,24 @@ PyDoc_STRVAR(winsound_MessageBeep__doc__, "x defaults to MB_OK."); #define WINSOUND_MESSAGEBEEP_METHODDEF \ - {"MessageBeep", _PyCFunction_CAST(winsound_MessageBeep), METH_FASTCALL|METH_KEYWORDS, winsound_MessageBeep__doc__}, + {"MessageBeep", (PyCFunction)(void(*)(void))winsound_MessageBeep, METH_VARARGS|METH_KEYWORDS, winsound_MessageBeep__doc__}, static PyObject * winsound_MessageBeep_impl(PyObject *module, int type); static PyObject * -winsound_MessageBeep(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +winsound_MessageBeep(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(type), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"type", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "MessageBeep", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + static char *_keywords[] = {"type", NULL}; int type = MB_OK; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - type = PyLong_AsInt(args[0]); - if (type == -1 && PyErr_Occurred()) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:MessageBeep", _keywords, + &type)) goto exit; - } -skip_optional_pos: return_value = winsound_MessageBeep_impl(module, type); exit: return return_value; } -/*[clinic end generated code: output=21584101f656198f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18a3771b34cdf97d input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index da2bde640961e0..b744f711b0d636 100644 --- a/PC/config.c +++ b/PC/config.c @@ -35,17 +35,16 @@ extern PyObject* PyInit__codecs(void); extern PyObject* PyInit__weakref(void); /* XXX: These two should really be extracted to standalone extensions. */ extern PyObject* PyInit_xxsubtype(void); -extern PyObject* PyInit__xxsubinterpreters(void); -extern PyObject* PyInit__xxinterpchannels(void); +extern PyObject* PyInit__interpreters(void); +extern PyObject* PyInit__interpchannels(void); +extern PyObject* PyInit__interpqueues(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); -#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); -#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) @@ -140,8 +139,9 @@ struct _inittab _PyImport_Inittab[] = { {"_json", PyInit__json}, {"xxsubtype", PyInit_xxsubtype}, - {"_xxsubinterpreters", PyInit__xxsubinterpreters}, - {"_xxinterpchannels", PyInit__xxinterpchannels}, + {"_interpreters", PyInit__interpreters}, + {"_interpchannels", PyInit__interpchannels}, + {"_interpqueues", PyInit__interpqueues}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/PC/errmap.h b/PC/errmap.h index a7489ab75c6561..a064ecb80b1ed9 100644 --- a/PC/errmap.h +++ b/PC/errmap.h @@ -129,6 +129,9 @@ winerror_to_errno(int winerror) case ERROR_NO_UNICODE_TRANSLATION: // 1113 return EILSEQ; + case WAIT_TIMEOUT: // 258 + return ETIMEDOUT; + case ERROR_INVALID_FUNCTION: // 1 case ERROR_INVALID_ACCESS: // 12 case ERROR_INVALID_DATA: // 13 diff --git a/PC/launcher.c b/PC/launcher.c index 8e60ab9303cb95..47fafbc3bf6bad 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1271,6 +1271,7 @@ static PYC_MAGIC magic_values[] = { { 3450, 3499, L"3.11" }, { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, + { 3600, 3649, L"3.14" }, { 0 } }; diff --git a/PC/launcher2.c b/PC/launcher2.c index 116091f01227b8..b372044e353202 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -438,7 +438,7 @@ typedef struct { bool list; // if true, only list detected runtimes with paths without launching bool listPaths; - // if true, display help message before contiuning + // if true, display help message before continuing bool help; // if set, limits search to registry keys with the specified Company // This is intended for debugging and testing only @@ -572,6 +572,21 @@ findArgv0End(const wchar_t *buffer, int bufferLength) *** COMMAND-LINE PARSING *** \******************************************************************************/ +// Adapted from https://stackoverflow.com/a/65583702 +typedef struct AppExecLinkFile { // For tag IO_REPARSE_TAG_APPEXECLINK + DWORD reparseTag; + WORD reparseDataLength; + WORD reserved; + ULONG version; + wchar_t stringList[MAX_PATH * 4]; // Multistring (Consecutive UTF-16 strings each ending with a NUL) + /* There are normally 4 strings here. Ex: + Package ID: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe" + Entry Point: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe!PythonRedirector" + Executable: L"C:\Program Files\WindowsApps\Microsoft.DesktopAppInstaller_1.17.106910_x64__8wekyb3d8bbwe\AppInstallerPythonRedirector.exe" + Applic. Type: L"0" // Integer as ASCII. "0" = Desktop bridge application; Else sandboxed UWP application + */ +} AppExecLinkFile; + int parseCommandLine(SearchInfo *search) @@ -763,6 +778,55 @@ _shebangStartsWith(const wchar_t *buffer, int bufferLength, const wchar_t *prefi } +int +ensure_no_redirector_stub(wchar_t* filename, wchar_t* buffer) +{ + // Make sure we didn't find a reparse point that will open the Microsoft Store + // If we did, pretend there was no shebang and let normal handling take over + WIN32_FIND_DATAW findData; + HANDLE hFind = FindFirstFileW(buffer, &findData); + if (!hFind) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + FindClose(hFind); + + if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + findData.dwReserved0 & IO_REPARSE_TAG_APPEXECLINK)) { + return 0; + } + + HANDLE hReparsePoint = CreateFileW(buffer, 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (!hReparsePoint) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + return RC_NO_SHEBANG; + } + + AppExecLinkFile appExecLink; + + if (!DeviceIoControl(hReparsePoint, FSCTL_GET_REPARSE_POINT, NULL, 0, &appExecLink, sizeof(appExecLink), NULL, NULL)) { + // Let normal handling take over + debug(L"# Did not find %s on PATH\n", filename); + CloseHandle(hReparsePoint); + return RC_NO_SHEBANG; + } + + CloseHandle(hReparsePoint); + + const wchar_t* redirectorPackageId = L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe"; + + if (0 == wcscmp(appExecLink.stringList, redirectorPackageId)) { + debug(L"# ignoring redirector that would launch store\n"); + return RC_NO_SHEBANG; + } + + return 0; +} + + int searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) { @@ -789,7 +853,7 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) } wchar_t filename[MAXLEN]; - if (wcsncpy_s(filename, MAXLEN, command, lastDot)) { + if (wcsncpy_s(filename, MAXLEN, command, commandLength)) { return RC_BAD_VIRTUAL_PATH; } @@ -826,6 +890,11 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) return RC_BAD_VIRTUAL_PATH; } + int result = ensure_no_redirector_stub(filename, buffer); + if (result) { + return result; + } + // Check that we aren't going to call ourselves again // If we are, pretend there was no shebang and let normal handling take over if (GetModuleFileNameW(NULL, filename, MAXLEN) && @@ -1525,6 +1594,7 @@ _registryReadLegacyEnvironment(const SearchInfo *search, HKEY root, EnvironmentI int count = swprintf_s(realTag, tagLength + 4, L"%s-32", env->tag); if (count == -1) { + debug(L"# Failed to generate 32bit tag\n"); free(realTag); return RC_INTERNAL_ERROR; } @@ -1680,10 +1750,18 @@ appxSearch(const SearchInfo *search, EnvironmentInfo **result, const wchar_t *pa exeName = search->windowed ? L"pythonw.exe" : L"python.exe"; } - if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer)) || - !join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || + // Failure to get LocalAppData may just mean we're running as a user who + // doesn't have a profile directory. + // In this case, return "not found", but don't fail. + // Chances are they can't launch Store installs anyway. + if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer))) { + return RC_NO_PYTHON; + } + + if (!join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || !join(buffer, MAXLEN, packageFamilyName) || !join(buffer, MAXLEN, exeName)) { + debug(L"# Failed to construct App Execution Alias path\n"); return RC_INTERNAL_ERROR; } @@ -1884,6 +1962,7 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, @@ -1893,6 +1972,7 @@ struct AppxSearchInfo APPX_SEARCH[] = { // Side-loadable releases. Note that the publisher ID changes whenever we // renew our code-signing certificate, so the newer ID has a higher // priority (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, @@ -1913,6 +1993,7 @@ collectEnvironments(const SearchInfo *search, EnvironmentInfo **result) EnvironmentInfo *env = NULL; if (!result) { + debug(L"# collectEnvironments() was passed a NULL result\n"); return RC_INTERNAL_ERROR; } *result = NULL; @@ -1973,7 +2054,8 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.11 */ L"9NRWMJP3717K" }, + { L"3", /* 3.12 */ L"9NCVDN91XZQP" }, + { L"3.13", L"9PNRBTZXMB4Z" }, { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, { L"3.10", L"9PJPW5LDXLZ5" }, @@ -2207,6 +2289,7 @@ int selectEnvironment(const SearchInfo *search, EnvironmentInfo *root, EnvironmentInfo **best) { if (!best) { + debug(L"# selectEnvironment() was passed a NULL best\n"); return RC_INTERNAL_ERROR; } if (!root) { @@ -2624,6 +2707,11 @@ process(int argc, wchar_t ** argv) DWORD len = GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", NULL, 0); if (len > 1) { wchar_t *limitToCompany = allocSearchInfoBuffer(&search, len); + if (!limitToCompany) { + exitCode = RC_NO_MEMORY; + winerror(0, L"Failed to allocate internal buffer"); + goto abort; + } search.limitToCompany = limitToCompany; if (0 == GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", limitToCompany, len)) { exitCode = RC_INTERNAL_ERROR; diff --git a/PC/layout/__main__.py b/PC/layout/__main__.py index f7aa1e6d261f4a..05a059eee7c1d7 100644 --- a/PC/layout/__main__.py +++ b/PC/layout/__main__.py @@ -1,7 +1,7 @@ import sys try: - import layout + import layout # noqa: F401 except ImportError: # Failed to import our package, which likely means we were started directly # Add the additional search path needed to locate our module. diff --git a/PC/layout/main.py b/PC/layout/main.py index cb2e4878da26b1..0350ed7af3f9b5 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -41,7 +41,7 @@ VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") -EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") +EXCLUDE_FROM_DLLS = FileStemSet("python*", "pyshellext", "vcruntime*") EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") @@ -73,7 +73,10 @@ def copy_if_modified(src, dest): ) if do_copy: - shutil.copy2(src, dest) + try: + shutil.copy2(src, dest) + except FileNotFoundError: + raise FileNotFoundError(src) from None def get_lib_layout(ns): @@ -118,14 +121,14 @@ def get_tcltk_lib(ns): def get_layout(ns): - def in_build(f, dest="", new_name=None): + def in_build(f, dest="", new_name=None, no_lib=False): n, _, x = f.rpartition(".") n = new_name or n src = ns.build / f if ns.debug and src not in REQUIRED_DLLS: - if not src.stem.endswith("_d"): + if not "_d." in src.name: src = src.parent / (src.stem + "_d" + src.suffix) - if not n.endswith("_d"): + if "_d." not in f: n += "_d" f = n + "." + x yield dest + n + "." + x, src @@ -133,22 +136,50 @@ def in_build(f, dest="", new_name=None): pdb = src.with_suffix(".pdb") if pdb.is_file(): yield dest + n + ".pdb", pdb - if ns.include_dev: + if ns.include_dev and not no_lib: lib = src.with_suffix(".lib") if lib.is_file(): yield "libs/" + n + ".lib", lib + source = "python.exe" + sourcew = "pythonw.exe" + alias = [ + "python", + "python{}".format(VER_MAJOR) if ns.include_alias3 else "", + "python{}".format(VER_DOT) if ns.include_alias3x else "", + ] + aliasw = [ + "pythonw", + "pythonw{}".format(VER_MAJOR) if ns.include_alias3 else "", + "pythonw{}".format(VER_DOT) if ns.include_alias3x else "", + ] if ns.include_appxmanifest: - yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) - yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) - # For backwards compatibility, but we don't reference these ourselves. - yield from in_build("python_uwp.exe", new_name="python") - yield from in_build("pythonw_uwp.exe", new_name="pythonw") + source = "python_uwp.exe" + sourcew = "pythonw_uwp.exe" + elif ns.include_freethreaded: + source = "python{}t.exe".format(VER_DOT) + sourcew = "pythonw{}t.exe".format(VER_DOT) + if not ns.include_alias: + alias = [] + aliasw = [] + alias.extend([ + "python{}t".format(VER_DOT), + "python{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + aliasw.extend([ + "pythonw{}t".format(VER_DOT), + "pythonw{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + + for a in filter(None, alias): + yield from in_build(source, new_name=a) + for a in filter(None, aliasw): + yield from in_build(sourcew, new_name=a) + + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_DLL_NAME) else: - yield from in_build("python.exe", new_name="python") - yield from in_build("pythonw.exe", new_name="pythonw") - - yield from in_build(PYTHON_DLL_NAME) + yield from in_build(PYTHON_DLL_NAME) if ns.include_launchers and ns.include_appxmanifest: if ns.include_pip: @@ -157,7 +188,10 @@ def in_build(f, dest="", new_name=None): yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) if ns.include_stable: - yield from in_build(PYTHON_STABLE_DLL_NAME) + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_STABLE_DLL_NAME) + else: + yield from in_build(PYTHON_STABLE_DLL_NAME) found_any = False for dest, src in rglob(ns.build, "vcruntime*.dll"): @@ -168,17 +202,31 @@ def in_build(f, dest="", new_name=None): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): - if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: - continue - if src in EXCLUDE_FROM_PYDS: - continue + dest = "" if ns.flat_dlls else "DLLs/" + + for _, src in rglob(ns.build, "*.pyd"): + if ns.include_freethreaded: + if not src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.cp*.pyd")) != bool(ns.debug): + continue + else: + if src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.pyd")) != bool(ns.debug): + continue if src in TEST_PYDS_ONLY and not ns.include_tests: continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue + yield from in_build(src.name, dest=dest, no_lib=True) - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + for _, src in rglob(ns.build, "*.dll"): + if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: + continue + if src in EXCLUDE_FROM_DLLS: + continue + yield from in_build(src.name, dest=dest, no_lib=True) if ns.zip_lib: zip_name = PYTHON_ZIP_NAME @@ -188,8 +236,12 @@ def in_build(f, dest="", new_name=None): yield "Lib/{}".format(dest), src if ns.include_venv: - yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") - yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") + if ns.include_freethreaded: + yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") + else: + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") if ns.include_tools: @@ -205,11 +257,9 @@ def _c(d): yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME if ns.include_dev: - for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src - src = ns.source / "PC" / "pyconfig.h" - yield "include/pyconfig.h", src + yield "include/pyconfig.h", ns.build / "pyconfig.h" for dest, src in get_tcltk_lib(ns): yield dest, src @@ -550,8 +600,16 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent - ns.temp = ns.temp or Path(tempfile.mkdtemp()) ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") + if ns.copy and not ns.copy.is_absolute(): + ns.copy = (Path.cwd() / ns.copy).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() if not ns.build.is_absolute(): @@ -563,10 +621,13 @@ def main(): if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() if not ns.arch: - ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" + if sys.winver.endswith("-arm64"): + ns.arch = "arm64" + elif sys.winver.endswith("-32"): + ns.arch = "win32" + else: + ns.arch = "amd64" - if ns.copy and not ns.copy.is_absolute(): - ns.copy = (Path.cwd() / ns.copy).resolve() if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): @@ -600,6 +661,12 @@ def main(): log_warning("Assuming --include-tcltk to support --include-idle") ns.include_tcltk = True + if not (ns.include_alias or ns.include_alias3 or ns.include_alias3x): + if ns.include_freethreaded: + ns.include_alias3x = True + else: + ns.include_alias = True + try: generate_source_files(ns) files = list(get_layout(ns)) diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index 1fb03380278f43..53977beb8af834 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -209,7 +209,7 @@ class PACKAGE_ID(ctypes.Structure): result = ctypes.create_unicode_buffer(256) result_len = ctypes.c_uint32(256) r = ctypes.windll.kernel32.PackageFamilyNameFromId( - pid, ctypes.byref(result_len), result + ctypes.byref(pid), ctypes.byref(result_len), result ) if r: raise OSError(r, "failed to get package family name") diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 8195c3dc30cdc7..ae22aa16ebfa5d 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -39,3 +39,6 @@ def _get_suffix(field4): PYTHON_CHM_NAME = "python{}{}{}{}.chm".format( VER_MAJOR, VER_MINOR, VER_MICRO, VER_SUFFIX ) + +FREETHREADED_PYTHON_DLL_NAME = "python{}{}t.dll".format(VER_MAJOR, VER_MINOR) +FREETHREADED_PYTHON_STABLE_DLL_NAME = "python{}t.dll".format(VER_MAJOR) diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py index dbcb713ef9d0c0..a87e0bea049427 100644 --- a/PC/layout/support/nuspec.py +++ b/PC/layout/support/nuspec.py @@ -24,6 +24,10 @@ amd64=("64-bit", "python", "Python"), arm32=("ARM", "pythonarm", "Python (ARM)"), arm64=("ARM64", "pythonarm64", "Python (ARM64)"), + win32t=("32-bit free-threaded", "pythonx86-freethreaded", "Python (32-bit, free-threaded)"), + amd64t=("64-bit free-threaded", "python-freethreaded", "Python (free-threaded)"), + arm32t=("ARM free-threaded", "pythonarm-freethreaded", "Python (ARM, free-threaded)"), + arm64t=("ARM64 free-threaded", "pythonarm64-freethreaded", "Python (ARM64, free-threaded)"), ) if not NUSPEC_DATA["PYTHON_VERSION"]: @@ -58,7 +62,10 @@ def _get_nuspec_data_overrides(ns): - for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[ns.arch]): + arch = ns.arch + if ns.include_freethreaded: + arch += "t" + for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[arch]): ev = os.getenv("PYTHON_NUSPEC_" + k) if ev: yield k, ev diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 60256fb32fe329..f1a8eb0b317744 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -32,6 +32,10 @@ def public(f): "nuspec": {"help": "a python.nuspec file"}, "chm": {"help": "the CHM documentation"}, "html-doc": {"help": "the HTML documentation"}, + "freethreaded": {"help": "freethreaded binaries", "not-in-all": True}, + "alias": {"help": "aliased python.exe entry-point binaries"}, + "alias3": {"help": "aliased python3.exe entry-point binaries"}, + "alias3x": {"help": "aliased python3.x.exe entry-point binaries"}, } @@ -47,6 +51,8 @@ def public(f): "dev", "launchers", "appxmanifest", + "alias", + "alias3x", # XXX: Disabled for now "precompile", ], }, @@ -59,9 +65,10 @@ def public(f): "venv", "props", "nuspec", + "alias", ], }, - "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]}, + "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, "default": { "help": "development kit package", "options": [ @@ -74,11 +81,19 @@ def public(f): "dev", "symbols", "html-doc", + "alias", ], }, "embed": { "help": "embeddable package", - "options": ["stable", "zip-lib", "flat-dlls", "underpth", "precompile"], + "options": [ + "alias", + "stable", + "zip-lib", + "flat-dlls", + "underpth", + "precompile", + ], }, } diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 5ff703217b421f..b170e06b47dd59 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -656,6 +656,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/pyconfig.h b/PC/pyconfig.h.in similarity index 95% rename from PC/pyconfig.h rename to PC/pyconfig.h.in index ac20129cd30fcc..f44e41c2e72f84 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h.in @@ -94,6 +94,14 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ +/* Define to 1 if you want to disable the GIL */ +/* Uncomment the definition for free-threaded builds, or define it manually + * when compiling extension modules. Note that we test with #ifdef, so + * defining as 0 will still disable the GIL. */ +#ifndef Py_GIL_DISABLED +/* #define Py_GIL_DISABLED 1 */ +#endif + /* Compiler specific defines */ /* ------------------------------------------------------------------------*/ @@ -305,15 +313,24 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* not building the core - must be an ext */ # if defined(_MSC_VER) /* So MSVC users need not specify the .lib - file in their Makefile (other compilers are - generally taken care of by distutils.) */ + file in their Makefile */ +# if defined(Py_GIL_DISABLED) # if defined(_DEBUG) -# pragma comment(lib,"python313_d.lib") +# pragma comment(lib,"python314t_d.lib") +# elif defined(Py_LIMITED_API) +# pragma comment(lib,"python3t.lib") +# else +# pragma comment(lib,"python314t.lib") +# endif /* _DEBUG */ +# else /* Py_GIL_DISABLED */ +# if defined(_DEBUG) +# pragma comment(lib,"python314_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python313.lib") +# pragma comment(lib,"python314.lib") # endif /* _DEBUG */ +# endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ @@ -511,6 +528,9 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Use Python's own small-block memory-allocator. */ #define WITH_PYMALLOC 1 +/* Define if you want to compile in mimalloc memory allocator. */ +#define WITH_MIMALLOC 1 + /* Define if you want to compile in object freelists optimization */ #define WITH_FREELISTS 1 diff --git a/PC/python3dll.c b/PC/python3dll.c index 7ee11746770442..aa3c3965908ff4 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -19,6 +19,7 @@ EXPORT_FUNC(_Py_Dealloc) EXPORT_FUNC(_Py_DecRef) EXPORT_FUNC(_Py_IncRef) EXPORT_FUNC(_Py_NegativeRefcount) +EXPORT_FUNC(_Py_SetRefcnt) EXPORT_FUNC(_Py_VaBuildValue_SizeT) EXPORT_FUNC(_PyArg_Parse_SizeT) EXPORT_FUNC(_PyArg_ParseTuple_SizeT) @@ -54,6 +55,8 @@ EXPORT_FUNC(Py_GenericAlias) EXPORT_FUNC(Py_GetArgcArgv) EXPORT_FUNC(Py_GetBuildInfo) EXPORT_FUNC(Py_GetCompiler) +EXPORT_FUNC(Py_GetConstant) +EXPORT_FUNC(Py_GetConstantBorrowed) EXPORT_FUNC(Py_GetCopyright) EXPORT_FUNC(Py_GetExecPrefix) EXPORT_FUNC(Py_GetPath) @@ -84,6 +87,7 @@ EXPORT_FUNC(Py_SetPath) EXPORT_FUNC(Py_SetProgramName) EXPORT_FUNC(Py_SetPythonHome) EXPORT_FUNC(Py_SetRecursionLimit) +EXPORT_FUNC(Py_TYPE) EXPORT_FUNC(Py_VaBuildValue) EXPORT_FUNC(Py_XNewRef) EXPORT_FUNC(PyAIter_Check) @@ -250,6 +254,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) @@ -323,6 +330,7 @@ EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) EXPORT_FUNC(PyList_GetItem) +EXPORT_FUNC(PyList_GetItemRef) EXPORT_FUNC(PyList_GetSlice) EXPORT_FUNC(PyList_Insert) EXPORT_FUNC(PyList_New) @@ -373,6 +381,10 @@ EXPORT_FUNC(PyMarshal_WriteObjectToString) EXPORT_FUNC(PyMem_Calloc) EXPORT_FUNC(PyMem_Free) EXPORT_FUNC(PyMem_Malloc) +EXPORT_FUNC(PyMem_RawCalloc) +EXPORT_FUNC(PyMem_RawFree) +EXPORT_FUNC(PyMem_RawMalloc) +EXPORT_FUNC(PyMem_RawRealloc) EXPORT_FUNC(PyMem_Realloc) EXPORT_FUNC(PyMember_GetOne) EXPORT_FUNC(PyMember_SetOne) @@ -564,6 +576,8 @@ EXPORT_FUNC(PyStructSequence_SetItem) EXPORT_FUNC(PySys_AddWarnOption) EXPORT_FUNC(PySys_AddWarnOptionUnicode) EXPORT_FUNC(PySys_AddXOption) +EXPORT_FUNC(PySys_Audit) +EXPORT_FUNC(PySys_AuditTuple) EXPORT_FUNC(PySys_FormatStderr) EXPORT_FUNC(PySys_FormatStdout) EXPORT_FUNC(PySys_GetObject) @@ -629,7 +643,10 @@ EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) EXPORT_FUNC(PyType_GetFlags) +EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) +EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index 5b55b810cd2152..08509f96ed1db8 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -5,7 +5,7 @@ #include "winver.h" #define PYTHON_COMPANY "Python Software Foundation" -#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2023 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." +#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2024 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." #define MS_WINDOWS #include "modsupport.h" diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c new file mode 100644 index 00000000000000..b1c8d0763d8c76 --- /dev/null +++ b/PC/venvlauncher.c @@ -0,0 +1,510 @@ +/* + * venv redirector for Windows + * + * This launcher looks for a nearby pyvenv.cfg to find the correct home + * directory, and then launches the original Python executable from it. + * The name of this executable is passed as argv[0]. + */ + +#define __STDC_WANT_LIB_EXT1__ 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MS_WINDOWS +#include "patchlevel.h" + +#define MAXLEN PATHCCH_MAX_CCH +#define MSGSIZE 1024 + +#define RC_NO_STD_HANDLES 100 +#define RC_CREATE_PROCESS 101 +#define RC_NO_PYTHON 103 +#define RC_NO_MEMORY 104 +#define RC_NO_VENV_CFG 106 +#define RC_BAD_VENV_CFG 107 +#define RC_NO_COMMANDLINE 108 +#define RC_INTERNAL_ERROR 109 + +// This should always be defined when we build for real, +// but it's handy to have a definition for quick testing +#ifndef EXENAME +#define EXENAME L"python.exe" +#endif + +#ifndef CFGNAME +#define CFGNAME L"pyvenv.cfg" +#endif + +static FILE * log_fp = NULL; + +void +debug(wchar_t * format, ...) +{ + va_list va; + + if (log_fp != NULL) { + wchar_t buffer[MAXLEN]; + int r = 0; + va_start(va, format); + r = vswprintf_s(buffer, MAXLEN, format, va); + va_end(va); + + if (r <= 0) { + return; + } + fwprintf(log_fp, L"%ls\n", buffer); + while (r && isspace(buffer[r])) { + buffer[r--] = L'\0'; + } + if (buffer[0]) { + OutputDebugStringW(buffer); + } + } +} + + +void +formatWinerror(int rc, wchar_t * message, int size) +{ + FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, rc, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + message, size, NULL); +} + + +void +winerror(int err, wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + wchar_t win_message[MSGSIZE]; + int len; + + if (err == 0) { + err = GetLastError(); + } + + va_start(va, format); + len = _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + + formatWinerror(err, win_message, MSGSIZE); + if (len >= 0) { + _snwprintf_s(&message[len], MSGSIZE - len, _TRUNCATE, L": %ls", + win_message); + } + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +void +error(wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + + va_start(va, format); + _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +bool +isEnvVarSet(const wchar_t *name) +{ + /* only looking for non-empty, which means at least one character + and the null terminator */ + return GetEnvironmentVariableW(name, NULL, 0) >= 2; +} + + +bool +join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment) +{ + if (SUCCEEDED(PathCchCombineEx(buffer, bufferLength, buffer, fragment, PATHCCH_ALLOW_LONG_PATHS))) { + return true; + } + return false; +} + + +bool +split_parent(wchar_t *buffer, size_t bufferLength) +{ + return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength)); +} + + +/* + * Path calculation + */ + +int +calculate_pyvenvcfg_path(wchar_t *pyvenvcfg_path, size_t maxlen) +{ + if (!pyvenvcfg_path) { + error(L"invalid buffer provided"); + return RC_INTERNAL_ERROR; + } + if ((DWORD)maxlen != maxlen) { + error(L"path buffer is too large"); + return RC_INTERNAL_ERROR; + } + if (!GetModuleFileNameW(NULL, pyvenvcfg_path, (DWORD)maxlen)) { + winerror(GetLastError(), L"failed to read executable directory"); + return RC_NO_COMMANDLINE; + } + // Remove 'python.exe' from our path + if (!split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segment from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace with 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, remove 'pyvenv.cfg' and (probably) 'Scripts' + if (!split_parent(pyvenvcfg_path, maxlen) || + !split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segments from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, we fail + winerror(GetLastError(), L"failed to locate %ls", CFGNAME); + return RC_NO_VENV_CFG; +} + + +/* + * pyvenv.cfg parsing + */ + +static int +find_home_value(const char *buffer, DWORD maxlen, const char **start, DWORD *length) +{ + if (!buffer || !start || !length) { + error(L"invalid find_home_value parameters()"); + return 0; + } + for (const char *s = strstr(buffer, "home"); + s && ((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; + s = strstr(s + 1, "\nhome") + ) { + if (*s == '\n') { + ++s; + } + for (int i = 4; i > 0 && *s; --i, ++s); + + while (*s && iswspace(*s)) { + ++s; + } + if (*s != L'=') { + continue; + } + + do { + ++s; + } while (*s && iswspace(*s)); + + *start = s; + char *nl = strchr(s, '\n'); + if (nl) { + while (nl != s && iswspace(nl[-1])) { + --nl; + } + *length = (DWORD)((ptrdiff_t)nl - (ptrdiff_t)s); + } else { + *length = (DWORD)strlen(s); + } + return 1; + } + return 0; +} + + +int +read_home(const wchar_t *pyvenv_cfg, wchar_t *home_path, size_t maxlen) +{ + HANDLE hFile = CreateFileW(pyvenv_cfg, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, 0, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + winerror(GetLastError(), L"failed to open '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + // 8192 characters ought to be enough for anyone + // (doubled compared to the old implementation!) + char buffer[8192]; + DWORD len; + if (!ReadFile(hFile, buffer, sizeof(buffer) - 1, &len, NULL)) { + winerror(GetLastError(), L"failed to read '%ls'", pyvenv_cfg); + CloseHandle(hFile); + return RC_BAD_VENV_CFG; + } + CloseHandle(hFile); + // Ensure null termination + buffer[len] = '\0'; + + char *home; + DWORD home_len; + if (!find_home_value(buffer, sizeof(buffer), &home, &home_len)) { + error(L"no home= specified in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + if ((DWORD)maxlen != maxlen) { + maxlen = 8192; + } + len = MultiByteToWideChar(CP_UTF8, 0, home, home_len, home_path, (DWORD)maxlen); + if (!len) { + winerror(GetLastError(), L"failed to decode home setting in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + home_path[len] = L'\0'; + + return 0; +} + + +int +locate_python(wchar_t *path, size_t maxlen) +{ + if (!join(path, maxlen, EXENAME)) { + error(L"failed to append %ls to '%ls'", EXENAME, path); + return RC_NO_MEMORY; + } + + if (GetFileAttributesW(path) == INVALID_FILE_ATTRIBUTES) { + winerror(GetLastError(), L"did not find executable at '%ls'", path); + return RC_NO_PYTHON; + } + + return 0; +} + + +int +smuggle_path() +{ + wchar_t buffer[MAXLEN]; + // We could use argv[0], but that may be wrong in certain rare cases (if the + // user is doing something weird like symlinks to venv redirectors), and + // what we _really_ want is the directory of the venv. We always copy the + // redirectors, so if we've made the venv, this will be correct. + DWORD len = GetModuleFileNameW(NULL, buffer, MAXLEN); + if (!len) { + winerror(GetLastError(), L"Failed to get own executable path"); + return RC_INTERNAL_ERROR; + } + buffer[len] = L'\0'; + debug(L"Setting __PYVENV_LAUNCHER__ = '%s'", buffer); + + if (!SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", buffer)) { + winerror(GetLastError(), L"Failed to set launcher environment"); + return RC_INTERNAL_ERROR; + } + + return 0; +} + +/* + * Process creation + */ + +static BOOL +safe_duplicate_handle(HANDLE in, HANDLE * pout, const wchar_t *name) +{ + BOOL ok; + HANDLE process = GetCurrentProcess(); + DWORD rc; + + *pout = NULL; + ok = DuplicateHandle(process, in, process, pout, 0, TRUE, + DUPLICATE_SAME_ACCESS); + if (!ok) { + rc = GetLastError(); + if (rc == ERROR_INVALID_HANDLE) { + debug(L"DuplicateHandle(%ls) returned ERROR_INVALID_HANDLE\n", name); + ok = TRUE; + } + else { + debug(L"DuplicateHandle(%ls) returned %d\n", name, rc); + } + } + return ok; +} + +static BOOL WINAPI +ctrl_c_handler(DWORD code) +{ + return TRUE; /* We just ignore all control events. */ +} + +static int +launch(const wchar_t *executable, wchar_t *cmdline) +{ + HANDLE job; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; + DWORD rc; + BOOL ok; + STARTUPINFOW si; + PROCESS_INFORMATION pi; + +#if defined(_WINDOWS) + /* + When explorer launches a Windows (GUI) application, it displays + the "app starting" (the "pointer + hourglass") cursor for a number + of seconds, or until the app does something UI-ish (eg, creating a + window, or fetching a message). As this launcher doesn't do this + directly, that cursor remains even after the child process does these + things. We avoid that by doing a simple post+get message. + See http://bugs.python.org/issue17290 + */ + MSG msg; + + PostMessage(0, 0, 0, 0); + GetMessage(&msg, 0, 0, 0); +#endif + + debug(L"run_child: about to run '%ls' with '%ls'\n", executable, cmdline); + job = CreateJobObject(NULL, NULL); + ok = QueryInformationJobObject(job, JobObjectExtendedLimitInformation, + &info, sizeof(info), &rc); + if (!ok || (rc != sizeof(info)) || !job) { + winerror(GetLastError(), L"Job information querying failed"); + return RC_CREATE_PROCESS; + } + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | + JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; + ok = SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, + sizeof(info)); + if (!ok) { + winerror(GetLastError(), L"Job information setting failed"); + return RC_CREATE_PROCESS; + } + memset(&si, 0, sizeof(si)); + GetStartupInfoW(&si); + ok = safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput, L"stdin"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_OUTPUT_HANDLE), &si.hStdOutput, L"stdout"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_ERROR_HANDLE), &si.hStdError, L"stderr"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + + ok = SetConsoleCtrlHandler(ctrl_c_handler, TRUE); + if (!ok) { + winerror(GetLastError(), L"control handler setting failed"); + return RC_CREATE_PROCESS; + } + + si.dwFlags = STARTF_USESTDHANDLES; + ok = CreateProcessW(executable, cmdline, NULL, NULL, TRUE, + 0, NULL, NULL, &si, &pi); + if (!ok) { + winerror(GetLastError(), L"Unable to create process using '%ls'", cmdline); + return RC_CREATE_PROCESS; + } + AssignProcessToJobObject(job, pi.hProcess); + CloseHandle(pi.hThread); + WaitForSingleObjectEx(pi.hProcess, INFINITE, FALSE); + ok = GetExitCodeProcess(pi.hProcess, &rc); + if (!ok) { + winerror(GetLastError(), L"Failed to get exit code of process"); + return RC_CREATE_PROCESS; + } + debug(L"child process exit code: %d", rc); + return rc; +} + + +int +process(int argc, wchar_t ** argv) +{ + int exitCode; + wchar_t pyvenvcfg_path[MAXLEN]; + wchar_t home_path[MAXLEN]; + + if (isEnvVarSet(L"PYLAUNCHER_DEBUG")) { + setvbuf(stderr, (char *)NULL, _IONBF, 0); + log_fp = stderr; + } + + exitCode = calculate_pyvenvcfg_path(pyvenvcfg_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = read_home(pyvenvcfg_path, home_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = locate_python(home_path, MAXLEN); + if (exitCode) return exitCode; + + // We do not update argv[0] to point at the target runtime, and so we do not + // pass through our original argv[0] in an environment variable. + exitCode = smuggle_path(); + if (exitCode) return exitCode; + + exitCode = launch(home_path, GetCommandLineW()); + return exitCode; +} + + +#if defined(_WINDOWS) + +int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, + LPWSTR lpstrCmd, int nShow) +{ + return process(__argc, __wargv); +} + +#else + +int cdecl wmain(int argc, wchar_t ** argv) +{ + return process(argc, argv); +} + +#endif diff --git a/PC/winreg.c b/PC/winreg.c index 77b80217ac0ab1..efdf8addc06186 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -200,7 +200,7 @@ PyHKEY_hashFunc(PyObject *ob) /* Just use the address. XXX - should we use the handle value? */ - return _Py_HashPointer(ob); + return PyObject_GenericHash(ob); } @@ -2179,6 +2179,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 68a917810f884d..094c77ae34d678 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,6 +35,12 @@ winsound.PlaySound(None, 0) */ +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include #include #include @@ -95,9 +101,13 @@ winsound_PlaySound_impl(PyObject *module, PyObject *sound, int flags) } wsound = (wchar_t *)view.buf; } else if (PyBytes_Check(sound)) { - PyErr_Format(PyExc_TypeError, - "'sound' must be str, os.PathLike, or None, not '%s'", - Py_TYPE(sound)->tp_name); + PyObject *type_name = PyType_GetQualName(Py_TYPE(sound)); + if (type_name != NULL) { + PyErr_Format(PyExc_TypeError, + "'sound' must be str, os.PathLike, or None, not %S", + type_name); + Py_DECREF(type_name); + } return NULL; } else { PyObject *obj = PyOS_FSPath(sound); @@ -236,6 +246,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PCbuild/_asyncio.vcxproj b/PCbuild/_asyncio.vcxproj index ed1e1bc0a420dc..76b0ffd660dba0 100644 --- a/PCbuild/_asyncio.vcxproj +++ b/PCbuild/_asyncio.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj index 3fe95fbf83993a..e0dc6ec187a08d 100644 --- a/PCbuild/_bz2.vcxproj +++ b/PCbuild/_bz2.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 253da31e9ce182..63d5fa49cd4e17 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 8a01e743a4d86f..f15b80852e362a 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -80,13 +80,14 @@ - .pyd + $(PyStdlibPydExt) + @@ -94,6 +95,7 @@ + @@ -109,4 +111,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index 5174196c52e4d0..618cfb32115e99 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -15,6 +15,9 @@ Header Files + + Header Files + @@ -26,4 +29,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 0916f1a2d37887..ee7421484b5312 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) @@ -93,51 +93,55 @@ - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions) CONFIG_32;ANSI;%(PreprocessorDefinitions) CONFIG_64;ANSI;%(PreprocessorDefinitions) CONFIG_64;MASM;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + ..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories) - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + true true true diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 0cbd3d0736c241..e4bdb64ec1fb9f 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -21,49 +21,55 @@ Header Files - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + + Header Files\libmpdec + + + Header Files\libmpdec + + Header Files\libmpdec @@ -71,46 +77,46 @@ Source Files - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec @@ -120,8 +126,8 @@ - + Source Files\libmpdec - \ No newline at end of file + diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 8da5244bac0cb6..8c9c0e42f7fe3e 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index aaa63fe9456fb7..e5e18de60ec349 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -89,6 +89,7 @@ Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) + $(IntDir);%(AdditionalIncludeDirectories) Disabled false @@ -141,7 +142,6 @@ - @@ -190,21 +190,26 @@ + + + - + + + @@ -217,17 +222,25 @@ + + + + + + + + @@ -243,6 +256,7 @@ + @@ -254,122 +268,125 @@ + + + importlib._bootstrap $(IntDir)importlib._bootstrap.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h zipimport $(IntDir)zipimport.g.h - $(PySourcePath)Python\frozen_modules\zipimport.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h abc $(IntDir)abc.g.h - $(PySourcePath)Python\frozen_modules\abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h codecs $(IntDir)codecs.g.h - $(PySourcePath)Python\frozen_modules\codecs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h io $(IntDir)io.g.h - $(PySourcePath)Python\frozen_modules\io.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\io.h _collections_abc $(IntDir)_collections_abc.g.h - $(PySourcePath)Python\frozen_modules\_collections_abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h _sitebuiltins $(IntDir)_sitebuiltins.g.h - $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h genericpath $(IntDir)genericpath.g.h - $(PySourcePath)Python\frozen_modules\genericpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h ntpath $(IntDir)ntpath.g.h - $(PySourcePath)Python\frozen_modules\ntpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h posixpath $(IntDir)posixpath.g.h - $(PySourcePath)Python\frozen_modules\posixpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h os $(IntDir)os.g.h - $(PySourcePath)Python\frozen_modules\os.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\os.h site $(IntDir)site.g.h - $(PySourcePath)Python\frozen_modules\site.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\site.h stat $(IntDir)stat.g.h - $(PySourcePath)Python\frozen_modules\stat.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h importlib.util $(IntDir)importlib.util.g.h - $(PySourcePath)Python\frozen_modules\importlib.util.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h importlib.machinery $(IntDir)importlib.machinery.g.h - $(PySourcePath)Python\frozen_modules\importlib.machinery.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h runpy $(IntDir)runpy.g.h - $(PySourcePath)Python\frozen_modules\runpy.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h __hello__ $(IntDir)__hello__.g.h - $(PySourcePath)Python\frozen_modules\__hello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h __phello__ $(IntDir)__phello__.g.h - $(PySourcePath)Python\frozen_modules\__phello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h __phello__.ham $(IntDir)__phello__.ham.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h __phello__.spam $(IntDir)__phello__.spam.g.h - $(PySourcePath)Python\frozen_modules\__phello__.spam.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h frozen_only $(IntDir)frozen_only.g.h - $(PySourcePath)Python\frozen_modules\frozen_only.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h @@ -378,12 +395,65 @@ getpath $(IntDir)getpath.g.h - $(PySourcePath)Python\frozen_modules\getpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\getpath.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + @@ -408,45 +478,13 @@ - - - - - - + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 279736b0fbbb13..9630f54ae4ea29 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -46,6 +46,9 @@ Source Files + + Python + Source Files @@ -91,6 +94,9 @@ Source Files + + Source Files + Source Files @@ -103,6 +109,12 @@ Source Files + + Source Files + + + Source Files + Source Files @@ -124,9 +136,6 @@ Source Files - - Source Files - Source Files @@ -163,6 +172,15 @@ Source Files + + Source Files + + + Source Files + + + Source Files + Source Files @@ -214,6 +232,10 @@ Source Files + + + Source Files + Source Files @@ -223,7 +245,7 @@ Source Files - + Source Files @@ -235,6 +257,9 @@ Source Files + + Source Files + Source Files @@ -274,6 +299,9 @@ Source Files + + Source Files + Source Files @@ -286,9 +314,15 @@ Source Files + + Python + Source Files + + Source Files + Source Files @@ -349,6 +383,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 6dad8183c57ae3..2cd205224bc089 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index fe076a6fc57168..40107d4b76cd53 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_multiprocessing.vcxproj b/PCbuild/_multiprocessing.vcxproj index 77b6bfc8e1e483..a65397f532aa86 100644 --- a/PCbuild/_multiprocessing.vcxproj +++ b/PCbuild/_multiprocessing.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_overlapped.vcxproj b/PCbuild/_overlapped.vcxproj index 9e60d3b5db336c..224bf05d5303a0 100644 --- a/PCbuild/_overlapped.vcxproj +++ b/PCbuild/_overlapped.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_queue.vcxproj b/PCbuild/_queue.vcxproj index 8065b235851686..80a1c3c6a4ad3e 100644 --- a/PCbuild/_queue.vcxproj +++ b/PCbuild/_queue.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_socket.vcxproj b/PCbuild/_socket.vcxproj index 78fa4d6729abb9..41af0895921bbb 100644 --- a/PCbuild/_socket.vcxproj +++ b/PCbuild/_socket.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 57c7413671e54e..9ae0a0fc3a009d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index 226ff506f8c62b..d4e1affab031d7 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testbuffer.vcxproj b/PCbuild/_testbuffer.vcxproj index 917d7ae50feb14..4e721e8ce09f0c 100644 --- a/PCbuild/_testbuffer.vcxproj +++ b/PCbuild/_testbuffer.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 0f33c5a76ade9d..44dbf2348137e1 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) @@ -96,27 +96,36 @@ - - + + + + + - + + + + + + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 4ba6011d8af5b9..cae44bc955f7f1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,12 +30,24 @@ Source Files + + Source Files + Source Files Source Files + + Source Files + + + Source Files + + + Source Files + Source Files @@ -54,6 +66,12 @@ Source Files + + Source Files + + + Source Files + Source Files @@ -69,12 +87,30 @@ Source Files - + + Source Files + + + Source Files + + + Source Files + + + Source Files + + Source Files Source Files + + Source Files + + + Source Files + diff --git a/PCbuild/_testclinic.vcxproj b/PCbuild/_testclinic.vcxproj index e319b3c0f42e0f..ef981332c6ab03 100644 --- a/PCbuild/_testclinic.vcxproj +++ b/PCbuild/_testclinic.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) @@ -107,4 +107,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index b00b2be491b423..183a55080e8693 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testconsole.vcxproj b/PCbuild/_testconsole.vcxproj index 5d7e14eff10294..69d312b17a5a62 100644 --- a/PCbuild/_testconsole.vcxproj +++ b/PCbuild/_testconsole.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testimportmultiple.vcxproj b/PCbuild/_testimportmultiple.vcxproj index 6d80d5779f24d8..c35ac83c1c739f 100644 --- a/PCbuild/_testimportmultiple.vcxproj +++ b/PCbuild/_testimportmultiple.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index a729ab3877d91f..87db569423de2a 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) @@ -95,6 +95,7 @@ + @@ -107,6 +108,12 @@ false + + + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index 9c8a5d793ee0f4..27429ea5833077 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -15,6 +15,9 @@ Source Files + + Source Files + Source Files @@ -24,4 +27,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj new file mode 100644 index 00000000000000..252039d93103bd --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -0,0 +1,129 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933} + _testlimitedcapi + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + false + + + + + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters new file mode 100644 index 00000000000000..7efbb0acf8f960 --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -0,0 +1,35 @@ + + + + + {a76a90d8-8e8b-4c36-8f58-8bd46abe9f5e} + + + {071b2ff4-e5a1-4e79-b0c5-cf46b0094a80} + + + + + + + + + + + + + + + + + + + + + + + + Resource Files + + + diff --git a/PCbuild/_testmultiphase.vcxproj b/PCbuild/_testmultiphase.vcxproj index 430eb528cc3927..e730fe308ab835 100644 --- a/PCbuild/_testmultiphase.vcxproj +++ b/PCbuild/_testmultiphase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testsinglephase.vcxproj b/PCbuild/_testsinglephase.vcxproj index fb4bcd953923f8..bf4dabf66c1040 100644 --- a/PCbuild/_testsinglephase.vcxproj +++ b/PCbuild/_testsinglephase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 30cedcbb43de76..117488a01621cc 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_uuid.vcxproj b/PCbuild/_uuid.vcxproj index 2437b7eb2d9399..50d81cc7916dbd 100644 --- a/PCbuild/_uuid.vcxproj +++ b/PCbuild/_uuid.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_wmi.vcxproj b/PCbuild/_wmi.vcxproj index c1914a3fa5a1bf..22fa8960982528 100644 --- a/PCbuild/_wmi.vcxproj +++ b/PCbuild/_wmi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_zoneinfo.vcxproj b/PCbuild/_zoneinfo.vcxproj index 6e6389c3773397..47b5bfa5b8815a 100644 --- a/PCbuild/_zoneinfo.vcxproj +++ b/PCbuild/_zoneinfo.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index e61267b5852a8f..6c76f09a071312 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,6 +36,9 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. +echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). +echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -65,6 +68,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo +set UseTIER2= :CheckOpts if "%~1"=="-h" goto Usage @@ -85,6 +89,10 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -176,6 +184,8 @@ echo on /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + /p:UseJIT=%UseJIT%^ + /p:UseTIER2=%UseTIER2%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d3f62c93869003..af85f6d362466e 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -29,6 +29,9 @@ :begin_search @set PYTHON= +@rem If PYTHON_FOR_BUILD is set, use that +@if NOT "%PYTHON_FOR_BUILD%"=="" @(set PYTHON="%PYTHON_FOR_BUILD%") && (set _Py_Python_Source=found as PYTHON_FOR_BUILD) && goto :found + @rem If there is an active virtual env, use that one @if NOT "%VIRTUAL_ENV%"=="" (set PYTHON="%VIRTUAL_ENV%\Scripts\python.exe") & (set _Py_Python_Source=found in virtual env) & goto :found @@ -42,7 +45,9 @@ @if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@rem It is fine to add new versions to this list when they have released, +@rem but we do not use prerelease builds here. +@for %%p in (3.12 3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 94437f054d788c..1927938ef0821c 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,12 +53,13 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.43.1.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% mpdecimal-4.0.0 +set libraries=%libraries% sqlite-3.45.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.14.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.14.0 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.2.13 +set libraries=%libraries% zlib-1.3.1 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( @@ -76,8 +77,8 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.14.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index b7b78be768d7ec..a2a637a3044373 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -77,7 +77,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index a0b5fbd1304b41..d10e1c46a91480 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30028.174 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34525.116 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject @@ -159,6 +159,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_wmi", "_wmi.vcxproj", "{54 EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic_limited", "_testclinic_limited.vcxproj", "{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testlimitedcapi", "_testlimitedcapi.vcxproj", "{7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1648,6 +1650,38 @@ Global {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|Win32.Build.0 = Release|Win32 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.ActiveCfg = Release|x64 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.Build.0 = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.ActiveCfg = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.Build.0 = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.Build.0 = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.ActiveCfg = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.Build.0 = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.ActiveCfg = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.Build.0 = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.ActiveCfg = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.Build.0 = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.ActiveCfg = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.Build.0 = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.ActiveCfg = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.Build.0 = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.ActiveCfg = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 001f8afd89b9e9..dc9161a8b290f9 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index b8d2d3d265543f..9c85e5efa4af4a 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,6 +10,9 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) + + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -21,11 +24,19 @@ false + + $(TargetName)$(TargetExt) + <_TargetNameSep>$(TargetNameExt.LastIndexOf(`.`)) + $(TargetNameExt.Substring(0, $(_TargetNameSep))) + $(TargetNameExt.Substring($(_TargetNameSep))) + + <_VCToolsVersion>$([System.Version]::Parse(`$(VCToolsVersion)`).Major).$([System.Version]::Parse(`$(VCToolsVersion)`).Minor) true + true @@ -34,13 +45,13 @@ <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) - <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)"; + <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)"; - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - Py_NOGIL=1;%(PreprocessorDefinitions) + _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed true @@ -59,6 +70,8 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) + -d2ssa-patterns-all- %(AdditionalOptions) + /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) OnlyExplicitInline @@ -152,8 +165,8 @@ public override bool Execute() { - - + + $([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\)) $(PySourcePath)\ - + $(PySourcePath)PCbuild\win32\ $(Py_OutDir)\win32\ @@ -52,7 +52,7 @@ $(PySourcePath)PCbuild\$(ArchName)\ $(BuildPath)\ $(BuildPath)instrumented\ - + ..\\.. ..\\..\\.. @@ -68,38 +68,36 @@ - $(ExternalsDir)sqlite-3.43.1.0\ + $(ExternalsDir)sqlite-3.45.3.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-3.0.11\ - $(ExternalsDir)openssl-bin-3.0.11\$(ArchName)\ + $(ExternalsDir)\mpdecimal-4.0.0\ + $(ExternalsDir)openssl-3.0.13\ + $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ - $(ExternalsDir)\zlib-1.2.13\ + $(ExternalsDir)\zlib-1.3.1\ _d - + -test - + -32 -arm32 -arm64 - - - $(BuildPath)python$(PyDebugExt).exe true - + true @@ -141,7 +139,7 @@ @@ -223,22 +221,55 @@ )) )) $([msbuild]::Add($(Field3Value), 9000)) - + + + python$(MajorVersionNumber).$(MinorVersionNumber)t + python + $(BuildPath)$(PyExeName)$(PyDebugExt).exe + pythonw$(MajorVersionNumber).$(MinorVersionNumber)t + pythonw + - python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)t$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) - python3$(PyDebugExt) + python3t + python3 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm64 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64 - + $(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt) + + + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm64 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_amd64 + + + $(MajorVersionNumber).$(MinorVersionNumber)t$(PyArchExt)$(PyTestExt) + + + + + .pyd - + + + $(FreethreadedPydTag) + + + $(PydTag).pyd + + + $(FreethreadedSysWinVer) + + diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index f4640454a73070..4a99ffc677c287 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -72,6 +72,7 @@ + $(PyExeName) Application false MultiByte @@ -94,8 +95,11 @@ Console - 2000000 - 8000000 + 2000000 + 12000000 + 12000000 + + 3000000 @@ -129,7 +133,7 @@ +"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" />
    @@ -142,7 +146,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) -@"$(OutDir)python$(PyDebugExt).exe" %* +@"$(OutDir)$(PyExeName)$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/PCbuild/python3dll.vcxproj b/PCbuild/python3dll.vcxproj index ec22e6fc76e584..235ea1cf9d33fb 100644 --- a/PCbuild/python3dll.vcxproj +++ b/PCbuild/python3dll.vcxproj @@ -70,12 +70,12 @@ {885D4898-D08D-4091-9C40-C700CFE3FC5A} python3dll Win32Proj - python3 false + $(Py3DllName) DynamicLibrary diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f240779d2a5d4e..f36fcb8caece33 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -104,6 +104,8 @@ $(zlibDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) @@ -111,6 +113,7 @@ + $(GeneratedFrozenModulesDir);%(AdditionalIncludeDirectories) PREFIX=NULL; EXEC_PREFIX=NULL; @@ -132,6 +135,7 @@ + @@ -142,6 +146,7 @@ + @@ -152,8 +157,8 @@ - + @@ -162,7 +167,6 @@ - @@ -173,6 +177,7 @@ + @@ -203,12 +208,14 @@ - + + + @@ -218,6 +225,8 @@ + + @@ -229,6 +238,7 @@ + @@ -242,8 +252,10 @@ + + @@ -252,6 +264,8 @@ + + @@ -259,6 +273,7 @@ + @@ -268,6 +283,7 @@ + @@ -280,10 +296,12 @@ + + @@ -291,17 +309,22 @@ - - + + + + + + + @@ -331,6 +354,7 @@ + @@ -370,7 +394,7 @@ - + @@ -415,6 +439,7 @@ + @@ -448,8 +473,9 @@ - - + + + @@ -487,7 +513,6 @@ - @@ -539,20 +564,27 @@ + + + - - + + $(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories) + + + + @@ -565,19 +597,25 @@ + + + + + + @@ -589,6 +627,7 @@ + @@ -604,11 +643,6 @@ - - - - - @@ -636,6 +670,35 @@ + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) + + + + + + + + + + + git diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6e13eb81018f9b..a1b43addf9e36a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -4,6 +4,9 @@ {086b0afb-270c-4603-a02a-63d46f0b2b92} + + {1dc8d8bd-d493-478e-9b7a-7eb108bb8bd5} + {8e81609f-13ca-4eae-9fdb-f8af20c710c7} @@ -19,6 +22,9 @@ {ab29a558-143d-4fe7-a039-b431fb429856} + + {ad449c55-23cb-44f8-ada4-04b23c9ec388} + {97349fee-0abf-48b0-a8f5-771bf39b8aee} @@ -66,6 +72,9 @@ Include + + Include + Include @@ -108,6 +117,9 @@ Include + + Include + Include @@ -201,6 +213,9 @@ Include + + Include + Include @@ -324,9 +339,6 @@ Include - - Include - Modules @@ -369,6 +381,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -390,6 +405,9 @@ Include\cpython + + Include + Include @@ -402,9 +420,6 @@ Include - - Include - Include\cpython @@ -465,6 +480,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -486,9 +504,6 @@ Include - - Include\cpython - Include\cpython @@ -537,10 +552,10 @@ Include\internal - + Include\internal - + Include\internal @@ -555,6 +570,9 @@ Include\internal + + Include\internal + Include\internal @@ -579,6 +597,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -609,6 +633,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -648,12 +678,18 @@ Include\internal + + Include\internal + Include\internal Include\cpython + + Include\internal + Include\internal @@ -678,6 +714,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -699,6 +741,9 @@ Include\internal + + Include\internal + Include\internal @@ -726,6 +771,9 @@ Include\internal + + Include\internal + Include\internal @@ -759,6 +807,9 @@ Include\internal + + Include\internal + Include\internal @@ -771,6 +822,9 @@ Include\internal + + Include\internal + Include\internal @@ -786,8 +840,23 @@ Include\internal - - Include\internal + + Include\internal\mimalloc + + + Include\internal\mimalloc + + + Include\internal\mimalloc + + + Include\internal\mimalloc + + + Include\internal\mimalloc + + + Include\internal\mimalloc Modules\zlib @@ -902,6 +971,9 @@ Modules + + Modules + Modules @@ -1211,6 +1283,9 @@ Python + + Python + Python @@ -1220,6 +1295,12 @@ Python + + Python + + + Source Files + Python @@ -1229,9 +1310,6 @@ Python - - Python - Python @@ -1247,6 +1325,15 @@ Python + + Python + + + Python + + + Python + Python @@ -1280,12 +1367,21 @@ Python + + Python + Source Files + + Source Files + Source Files + + Python + Source Files @@ -1304,12 +1400,18 @@ Python + + Python + Python Python + + Python + Python @@ -1319,6 +1421,9 @@ Python + + Python + Python @@ -1352,6 +1457,9 @@ Python + + Python + Python @@ -1397,9 +1505,6 @@ Objects - - Objects - Modules @@ -1466,10 +1571,13 @@ Parser - + + Modules + + Modules - + Modules diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index e23635e5ea9411..d08c210ef8a1dc 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -73,6 +73,7 @@ + $(PyWExeName) Application false @@ -89,8 +90,11 @@ - 2000000 - 8000000 + 2000000 + 12000000 + 12000000 + + 3000000 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 175ce918ac20f6..f4dfe0e9015f02 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -131,29 +131,31 @@ xxlimited_35 The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to .pyd) of the same name as the project: -_asyncio -_ctypes -_ctypes_test -_zoneinfo -_decimal -_elementtree -_hashlib -_multiprocessing -_overlapped -_socket -_testbuffer -_testcapi -_testclinic -_testclinic_limited -_testconsole -_testimportmultiple -_testmultiphase -_testsinglephase -_tkinter -pyexpat -select -unicodedata -winsound + * _asyncio + * _ctypes + * _ctypes_test + * _zoneinfo + * _decimal + * _elementtree + * _hashlib + * _multiprocessing + * _overlapped + * _socket + * _testbuffer + * _testcapi + * _testlimitedcapi + * _testinternalcapi + * _testclinic + * _testclinic_limited + * _testconsole + * _testimportmultiple + * _testmultiphase + * _testsinglephase + * _tkinter + * pyexpat + * select + * unicodedata + * winsound The following Python-controlled sub-projects wrap external projects. Note that these external libraries are not necessary for a working @@ -189,7 +191,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.43.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.3, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter @@ -226,12 +228,18 @@ directory. This script extracts all the external sub-projects from and https://github.com/python/cpython-bin-deps via a Python script called "get_external.py", located in this directory. -If Python 3.6 or later is not available via the "py.exe" launcher, the -path or command to use for Python can be provided in the PYTHON_FOR_BUILD -environment variable, or get_externals.bat will download the latest -version of NuGet and use it to download the latest "pythonx86" package -for use with get_external.py. Everything downloaded by these scripts is -stored in ..\externals (relative to this directory). +Everything downloaded by these scripts is stored in ..\externals +(relative to this directory), or the path specified by the EXTERNALS_DIR +environment variable. + +The path or command to use for Python can be provided with the +PYTHON_FOR_BUILD environment variable. If this is not set, an active +virtual environment will be used. If none is active, and HOST_PYTHON is +set to a recent enough version or "py.exe" is able to find a recent +enough version, those will be used. If all else fails, a copy of Python +will be downloaded from NuGet and extracted to the externals directory. +This will then be used for later builds (see PCbuild/find_python.bat +for the full logic). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild @@ -303,6 +311,8 @@ _testclinic_limited extension, the file Modules/_testclinic_limited.c: * In PCbuild/, copy _testclinic.vcxproj to _testclinic_limited.vcxproj, replace RootNamespace value with `_testclinic_limited`, replace `_asyncio.c` with `_testclinic_limited.c`. +* In PCbuild/, copy _testclinic.vcxproj.filters to + _testclinic_limited.vcxproj.filters, edit the list of files in the new file. * Open Visual Studio, open PCbuild\pcbuild.sln solution, add the PCbuild\_testclinic_limited.vcxproj project to the solution ("add existing project). diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index cc9469c7ddd726..416241d9d0df10 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -28,11 +28,20 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> + + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> + <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> + <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + json + - @@ -77,9 +86,58 @@ WorkingDirectory="$(PySourcePath)" /> - + + + + + + + + + + + + + + + aarch64-pc-windows-msvc + i686-pc-windows-msvc + x86_64-pc-windows-msvc + $(JITArgs) --debug + + + + + + + + + + + + @@ -103,9 +161,7 @@ Condition="($(Platform) == 'Win32' or $(Platform) == 'x64') and $(Configuration) != 'PGInstrument' and $(Configuration) != 'PGUpdate'"> - diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 332ba5edcf4082..c436215780fda0 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -9,6 +9,7 @@ rem -q runs the tests just once, and without deleting .pyc files. rem -p or -win32, -x64, -arm32, -arm64 rem Run the specified architecture of python (or python_d if -d rem was specified). If omitted, uses %PREFIX% if set or 64-bit. +rem --disable-gil Run free-threaded build. rem All leading instances of these switches are shifted off, and rem whatever remains (up to 9 arguments) is passed to regrtest.py. rem For example, @@ -29,6 +30,7 @@ rem rt -u "network,largefile" setlocal set pcbuild=%~dp0 +set pyname=python set suffix= set qmode= set dashO= @@ -36,18 +38,21 @@ set regrtestargs=--fast-ci set exe= :CheckOpts -if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts -if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts -if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts -if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts -if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts -if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts -if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts -if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts +if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts +if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts +if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts +rem HACK: Need some way to infer the version number in this script +if "%~1"=="--disable-gil" (set pyname=python3.14t) & shift & goto CheckOpts +if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts +if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts +if "%~1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts +if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts +if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 -set exe=%prefix%\python%suffix%.exe +set exe=%prefix%\%pyname%%suffix%.exe set cmd="%exe%" %dashO% -m test %regrtestargs% if defined qmode goto Qmode diff --git a/PCbuild/select.vcxproj b/PCbuild/select.vcxproj index 750a713949919a..d7448fd4d72380 100644 --- a/PCbuild/select.vcxproj +++ b/PCbuild/select.vcxproj @@ -78,7 +78,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index c502d51833b91a..6bcc4e913c8e77 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -69,12 +69,12 @@ {A1A295E5-463C-437F-81CA-1F32367685DA} sqlite3 - .pyd false + $(PyStdlibPydExt) DynamicLibrary NotSet diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 96dd289face6a5..95b699b4cac0aa 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.0 + 8.6.14.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) diff --git a/PCbuild/unicodedata.vcxproj b/PCbuild/unicodedata.vcxproj index addef753359ed6..781f938e2ab78e 100644 --- a/PCbuild/unicodedata.vcxproj +++ b/PCbuild/unicodedata.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index 123e84ec4e3682..1193e032245c94 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -69,12 +69,13 @@ {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} venvlauncher - venvlauncher false + venvlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _CONSOLE;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions) MultiThreaded PY_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Console - + diff --git a/PCbuild/venvlauncher.vcxproj.filters b/PCbuild/venvlauncher.vcxproj.filters index ec13936bf6cb7e..56a0f005a3fa2a 100644 --- a/PCbuild/venvlauncher.vcxproj.filters +++ b/PCbuild/venvlauncher.vcxproj.filters @@ -19,7 +19,7 @@ - + Source Files diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index b8504d5d08e52f..1b61718201367f 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -69,12 +69,13 @@ {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} venvwlauncher - venvwlauncher false + venvwlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _WINDOWS;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions) MultiThreaded PYW_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Windows - + diff --git a/PCbuild/venvwlauncher.vcxproj.filters b/PCbuild/venvwlauncher.vcxproj.filters index 8addc13e977e7a..61a514395e82dc 100644 --- a/PCbuild/venvwlauncher.vcxproj.filters +++ b/PCbuild/venvwlauncher.vcxproj.filters @@ -9,7 +9,7 @@ - + Source Files diff --git a/PCbuild/winsound.vcxproj b/PCbuild/winsound.vcxproj index 32cedc9b444902..c26029b15a339f 100644 --- a/PCbuild/winsound.vcxproj +++ b/PCbuild/winsound.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited.vcxproj b/PCbuild/xxlimited.vcxproj index 1c776fb0da3e72..093e6920c0b76c 100644 --- a/PCbuild/xxlimited.vcxproj +++ b/PCbuild/xxlimited.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited_35.vcxproj b/PCbuild/xxlimited_35.vcxproj index dd830b3b6aaa91..3f4d4463f24af0 100644 --- a/PCbuild/xxlimited_35.vcxproj +++ b/PCbuild/xxlimited_35.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 0d154867276c36..80776ffe449393 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -145,8 +145,8 @@ module Python type_ignore = TypeIgnore(int lineno, string tag) - type_param = TypeVar(identifier name, expr? bound) - | ParamSpec(identifier name) - | TypeVarTuple(identifier name) + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) } diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index fb2d56af3fc372..44bf87da8288a6 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -3,6 +3,7 @@ #include "pegen.h" #include "string_parser.h" #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pystate.h" // _PyInterpreterState_GET() void * _PyPegen_dummy_name(Parser *p, ...) @@ -118,42 +119,13 @@ expr_ty _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name) { assert(first_name != NULL && second_name != NULL); - PyObject *first_identifier = first_name->v.Name.id; - PyObject *second_identifier = second_name->v.Name.id; - - const char *first_str = PyUnicode_AsUTF8(first_identifier); - if (!first_str) { - return NULL; - } - const char *second_str = PyUnicode_AsUTF8(second_identifier); - if (!second_str) { - return NULL; - } - Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot - - PyObject *str = PyBytes_FromStringAndSize(NULL, len); - if (!str) { - return NULL; - } - - char *s = PyBytes_AS_STRING(str); - if (!s) { - return NULL; - } - - strcpy(s, first_str); - s += strlen(first_str); - *s++ = '.'; - strcpy(s, second_str); - s += strlen(second_str); - *s = '\0'; - - PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL); - Py_DECREF(str); + PyObject *uni = PyUnicode_FromFormat("%U.%U", + first_name->v.Name.id, second_name->v.Name.id); if (!uni) { return NULL; } - PyUnicode_InternInPlace(&uni); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &uni); if (_PyArena_AddPyObject(p->arena, uni) < 0) { Py_DECREF(uni); return NULL; @@ -573,22 +545,30 @@ _make_posargs(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_default, asdl_arg_seq **posargs) { - if (plain_names != NULL && names_with_default != NULL) { - asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default); - if (!names_with_default_names) { - return -1; + + if (names_with_default != NULL) { + if (plain_names != NULL) { + asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default); + if (!names_with_default_names) { + return -1; + } + *posargs = (asdl_arg_seq*)_PyPegen_join_sequences( + p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names); + } + else { + *posargs = _get_names(p, names_with_default); } - *posargs = (asdl_arg_seq*)_PyPegen_join_sequences( - p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names); - } - else if (plain_names == NULL && names_with_default != NULL) { - *posargs = _get_names(p, names_with_default); - } - else if (plain_names != NULL && names_with_default == NULL) { - *posargs = plain_names; } else { - *posargs = _Py_asdl_arg_seq_new(0, p->arena); + if (plain_names != NULL) { + // With the current grammar, we never get here. + // If that has changed, remove the assert, and test thoroughly. + assert(0); + *posargs = plain_names; + } + else { + *posargs = _Py_asdl_arg_seq_new(0, p->arena); + } } return *posargs == NULL ? -1 : 0; } @@ -884,7 +864,7 @@ _PyPegen_make_module(Parser *p, asdl_stmt_seq *a) { if (type_ignores == NULL) { return NULL; } - for (int i = 0; i < num; i++) { + for (Py_ssize_t i = 0; i < num; i++) { PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment); if (tag == NULL) { return NULL; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f61099b97055ad..e6867f138a5ccb 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -15,6 +15,13 @@ MAX_COL = 80 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" +builtin_type_to_c_type = { + "identifier": "PyUnicode_Type", + "string": "PyUnicode_Type", + "int": "PyLong_Type", + "constant": "PyBaseObject_Type", +} + def get_c_type(name): """Return a string for the C name of the type. @@ -518,7 +525,7 @@ def sumTrailer(self, name, add_label=False): if add_label: self.emit("failed:", 1) self.emit("Py_XDECREF(tmp);", 1) - self.emit("return 1;", 1) + self.emit("return -1;", 1) self.emit("}", 0) self.emit("", 0) @@ -529,7 +536,7 @@ def simpleSum(self, sum, name): "state->%s_type);") self.emit(line % (t.name,), 1) self.emit("if (isinstance == -1) {", 1) - self.emit("return 1;", 2) + self.emit("return -1;", 2) self.emit("}", 1) self.emit("if (isinstance) {", 1) self.emit("*out = %s;" % t.name, 2) @@ -558,7 +565,7 @@ def complexSum(self, sum, name): self.emit("tp = state->%s_type;" % (t.name,), 1) self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1) self.emit("if (isinstance == -1) {", 1) - self.emit("return 1;", 2) + self.emit("return -1;", 2) self.emit("}", 1) self.emit("if (isinstance) {", 1) for f in t.fields: @@ -605,7 +612,7 @@ def visitProduct(self, prod, name): self.emit("return 0;", 1) self.emit("failed:", 0) self.emit("Py_XDECREF(tmp);", 1) - self.emit("return 1;", 1) + self.emit("return -1;", 1) self.emit("}", 0) self.emit("", 0) @@ -631,13 +638,13 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): ctype = get_c_type(field.type) line = "if (PyObject_GetOptionalAttr(obj, state->%s, &tmp) < 0) {" self.emit(line % field.name, depth) - self.emit("return 1;", depth+1) + self.emit("return -1;", depth+1) self.emit("}", depth) if field.seq: self.emit("if (tmp == NULL) {", depth) self.emit("tmp = PyList_New(0);", depth+1) self.emit("if (tmp == NULL) {", depth+1) - self.emit("return 1;", depth+2) + self.emit("return -1;", depth+2) self.emit("}", depth+1) self.emit("}", depth) self.emit("{", depth) @@ -647,7 +654,7 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): message = "required field \\\"%s\\\" missing from %s" % (field.name, name) format = "PyErr_SetString(PyExc_TypeError, \"%s\");" self.emit(format % message, depth+1, reflow=False) - self.emit("return 1;", depth+1) + self.emit("return -1;", depth+1) else: self.emit("if (tmp == NULL || tmp == Py_None) {", depth) self.emit("Py_CLEAR(tmp);", depth+1) @@ -731,7 +738,7 @@ def emit_sequence_constructor(self, name, type): class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, void*);" % name, 0) if prod.attributes: self.emit("static const char * const %s_attributes[] = {" % name, 0) for a in prod.attributes: @@ -752,7 +759,7 @@ def visitSum(self, sum, name): ptype = "void*" if is_simple(sum): ptype = get_c_type(name) - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -764,6 +771,67 @@ def visitConstructor(self, cons, name): self.emit("};",0) +class AnnotationsVisitor(PickleVisitor): + def visitModule(self, mod): + self.file.write(textwrap.dedent(''' + static int + add_ast_annotations(struct ast_state *state) + { + bool cond; + ''')) + for dfn in mod.dfns: + self.visit(dfn) + self.file.write(textwrap.dedent(''' + return 1; + } + ''')) + + def visitProduct(self, prod, name): + self.emit_annotations(name, prod.fields) + + def visitSum(self, sum, name): + for t in sum.types: + self.visitConstructor(t, name) + + def visitConstructor(self, cons, name): + self.emit_annotations(cons.name, cons.fields) + + def emit_annotations(self, name, fields): + self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1) + self.emit(f"if (!{name}_annotations) return 0;", 1) + for field in fields: + self.emit("{", 1) + if field.type in builtin_type_to_c_type: + self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2) + else: + self.emit(f"PyObject *type = state->{field.type}_type;", 2) + if field.opt: + self.emit("type = _Py_union_type_or(type, Py_None);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + elif field.seq: + self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + else: + self.emit("Py_INCREF(type);", 2) + self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2) + self.emit("Py_DECREF(type);", 2) + self.emit_annotations_error(name, 2) + self.emit("}", 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f"Py_DECREF({name}_annotations);", 1) + + def emit_annotations_error(self, name, depth): + self.emit("if (!cond) {", depth) + self.emit(f"Py_DECREF({name}_annotations);", depth + 1) + self.emit("return 0;", depth + 1) + self.emit("}", depth) + + class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): @@ -812,7 +880,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -821,6 +889,13 @@ def visitModule(self, mod): if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -840,6 +915,11 @@ def visitModule(self, mod): goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -852,13 +932,14 @@ def visitModule(self, mod): if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -866,15 +947,123 @@ def visitModule(self, mod): goto cleanup; } } + else { + // Lazily initialize "attributes" + if (attributes == NULL) { + attributes = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_attributes); + if (attributes == NULL) { + res = -1; + goto cleanup; + } + } + int contains = PySequence_Contains(attributes, key); + if (contains == -1) { + res = -1; + goto cleanup; + } + else if (contains == 0) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types) < 0) { + res = -1; + goto cleanup; + } + if (field_types == NULL) { + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } + } + } + else if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + goto set_remaining_cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: + Py_XDECREF(attributes); Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -886,14 +1075,345 @@ def visitModule(self, mod): return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; + if (dict) { + // Unpickling (or copying) works as follows: + // - Construct the object with only positional arguments + // - Set the fields from the dict + // We have two constraints: + // - We must set all the required fields in the initial constructor call, + // or the unpickling or deepcopying of the object will trigger DeprecationWarnings. + // - We must not include child nodes in the positional args, because + // that may trigger runaway recursion during copying (gh-120108). + // To satisfy both constraints, we set all the fields to None in the + // initial list of positional args, and then set the fields from the dict. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value; + int rc = PyDict_GetItemRef(dict, name, &value); + Py_DECREF(name); + if (rc < 0) { + goto cleanup; + } + if (!value) { + break; + } + rc = PyList_Append(positional_args, Py_None); + Py_DECREF(value); + if (rc < 0) { + goto cleanup; + } + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONN", Py_TYPE(self), args_tuple, dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); + } +cleanup: + Py_XDECREF(fields); + Py_XDECREF(positional_args); + return result; +} + +/* + * Perform the following validations: + * + * - All keyword arguments are known 'fields' or 'attributes'. + * - No field or attribute would be left unfilled after copy.replace(). + * + * On success, this returns 1. Otherwise, set a TypeError + * exception and returns -1 (no exception is set if some + * other internal errors occur). + * + * Parameters + * + * self The AST node instance. + * dict The AST node instance dictionary (self.__dict__). + * fields The list of fields (self._fields). + * attributes The list of attributes (self._attributes). + * kwargs Keyword arguments passed to ast_type_replace(). + * + * The 'dict', 'fields', 'attributes' and 'kwargs' arguments can be NULL. + * + * Note: this function can be removed in 3.15 since the verification + * will be done inside the constructor. + */ +static inline int +ast_type_replace_check(PyObject *self, + PyObject *dict, + PyObject *fields, + PyObject *attributes, + PyObject *kwargs) +{ + // While it is possible to make some fast paths that would avoid + // allocating objects on the stack, this would cost us readability. + // For instance, if 'fields' and 'attributes' are both empty, and + // 'kwargs' is not empty, we could raise a TypeError immediately. + PyObject *expecting = PySet_New(fields); + if (expecting == NULL) { + return -1; + } + if (attributes) { + if (_PySet_Update(expecting, attributes) < 0) { + Py_DECREF(expecting); + return -1; + } + } + // Any keyword argument that is neither a field nor attribute is rejected. + // We first need to check whether a keyword argument is accepted or not. + // If all keyword arguments are accepted, we compute the required fields + // and attributes. A field or attribute is not needed if: + // + // 1) it is given in 'kwargs', or + // 2) it already exists on 'self'. + if (kwargs) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(kwargs, &pos, &key, &value)) { + int rc = PySet_Discard(expecting, key); + if (rc < 0) { + Py_DECREF(expecting); + return -1; + } + if (rc == 0) { + PyErr_Format(PyExc_TypeError, + "%.400s.__replace__ got an unexpected keyword " + "argument '%U'.", Py_TYPE(self)->tp_name, key); + Py_DECREF(expecting); + return -1; + } + } + } + // check that the remaining fields or attributes would be filled if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + // Mark fields or attributes that are found on the instance + // as non-mandatory. If they are not given in 'kwargs', they + // will be shallow-coied; otherwise, they would be replaced + // (not in this function). + if (PySet_Discard(expecting, key) < 0) { + Py_DECREF(expecting); + return -1; + } + } + if (attributes) { + // Some attributes may or may not be present at runtime. + // In particular, now that we checked whether 'kwargs' + // is correct or not, we allow any attribute to be missing. + // + // Note that fields must still be entirely determined when + // calling the constructor later. + PyObject *unused = PyObject_CallMethodOneArg(expecting, + &_Py_ID(difference_update), + attributes); + if (unused == NULL) { + Py_DECREF(expecting); + return -1; + } + Py_DECREF(unused); + } + } + // Now 'expecting' contains the fields or attributes + // that would not be filled inside ast_type_replace(). + Py_ssize_t m = PySet_GET_SIZE(expecting); + if (m > 0) { + PyObject *names = PyList_New(m); + if (names == NULL) { + Py_DECREF(expecting); + return -1; + } + Py_ssize_t i = 0, pos = 0; + PyObject *item; + Py_hash_t hash; + while (_PySet_NextEntry(expecting, &pos, &item, &hash)) { + PyObject *name = PyObject_Repr(item); + if (name == NULL) { + Py_DECREF(expecting); + Py_DECREF(names); + return -1; + } + // steal the reference 'name' + PyList_SET_ITEM(names, i++, name); + } + Py_DECREF(expecting); + if (PyList_Sort(names) < 0) { + Py_DECREF(names); + return -1; + } + PyObject *sep = PyUnicode_FromString(", "); + if (sep == NULL) { + Py_DECREF(names); + return -1; + } + PyObject *str_names = PyUnicode_Join(sep, names); + Py_DECREF(sep); + Py_DECREF(names); + if (str_names == NULL) { + return -1; + } + PyErr_Format(PyExc_TypeError, + "%.400s.__replace__ missing %ld keyword argument%s: %U.", + Py_TYPE(self)->tp_name, m, m == 1 ? "" : "s", str_names); + Py_DECREF(str_names); + return -1; + } + else { + Py_DECREF(expecting); + return 1; } - return Py_BuildValue("O()", Py_TYPE(self)); +} + +/* + * Python equivalent: + * + * for key in keys: + * if hasattr(self, key): + * payload[key] = getattr(self, key) + * + * The 'keys' argument is a sequence corresponding to + * the '_fields' or the '_attributes' of an AST node. + * + * This returns -1 if an error occurs and 0 otherwise. + * + * Parameters + * + * payload A dictionary to fill. + * keys A sequence of keys or NULL for an empty sequence. + * dict The AST node instance dictionary (must not be NULL). + */ +static inline int +ast_type_replace_update_payload(PyObject *payload, + PyObject *keys, + PyObject *dict) +{ + assert(dict != NULL); + if (keys == NULL) { + return 0; + } + Py_ssize_t n = PySequence_Size(keys); + if (n == -1) { + return -1; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *key = PySequence_GetItem(keys, i); + if (key == NULL) { + return -1; + } + PyObject *value; + if (PyDict_GetItemRef(dict, key, &value) < 0) { + Py_DECREF(key); + return -1; + } + if (value == NULL) { + Py_DECREF(key); + // If a field or attribute is not present at runtime, it should + // be explicitly given in 'kwargs'. If not, the constructor will + // issue a warning (which becomes an error in 3.15). + continue; + } + int rc = PyDict_SetItem(payload, key, value); + Py_DECREF(key); + Py_DECREF(value); + if (rc < 0) { + return -1; + } + } + return 0; +} + +/* copy.replace() support (shallow copy) */ +static PyObject * +ast_type_replace(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoPositional("__replace__", args)) { + return NULL; + } + + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + PyObject *result = NULL; + // known AST class fields and attributes + PyObject *fields = NULL, *attributes = NULL; + // current instance dictionary + PyObject *dict = NULL; + // constructor positional and keyword arguments + PyObject *empty_tuple = NULL, *payload = NULL; + + PyObject *type = (PyObject *)Py_TYPE(self); + if (PyObject_GetOptionalAttr(type, state->_fields, &fields) < 0) { + goto cleanup; + } + if (PyObject_GetOptionalAttr(type, state->_attributes, &attributes) < 0) { + goto cleanup; + } + if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { + goto cleanup; + } + if (ast_type_replace_check(self, dict, fields, attributes, kwargs) < 0) { + goto cleanup; + } + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + goto cleanup; + } + payload = PyDict_New(); + if (payload == NULL) { + goto cleanup; + } + if (dict) { // in case __dict__ is missing (for some obscure reason) + // copy the instance's fields (possibly NULL) + if (ast_type_replace_update_payload(payload, fields, dict) < 0) { + goto cleanup; + } + // copy the instance's attributes (possibly NULL) + if (ast_type_replace_update_payload(payload, attributes, dict) < 0) { + goto cleanup; + } + } + if (kwargs && PyDict_Update(payload, kwargs) < 0) { + goto cleanup; + } + result = PyObject_Call(type, empty_tuple, payload); +cleanup: + Py_XDECREF(payload); + Py_XDECREF(empty_tuple); + Py_XDECREF(dict); + Py_XDECREF(attributes); + Py_XDECREF(fields); + return result; } static PyMemberDef ast_type_members[] = { @@ -903,6 +1423,10 @@ def visitModule(self, mod): static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, + {"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **fields)\\n--\\n\\n" + "Return a copy of the AST node with new values " + "for the specified fields.")}, {NULL} }; @@ -968,23 +1492,24 @@ def visitModule(self, mod): int i, result; PyObject *s, *l = PyTuple_New(num_fields); if (!l) - return 0; + return -1; for (i = 0; i < num_fields; i++) { s = PyUnicode_InternFromString(attrs[i]); if (!s) { Py_DECREF(l); - return 0; + return -1; } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttr(type, state->_attributes, l) >= 0; + result = PyObject_SetAttr(type, state->_attributes, l); Py_DECREF(l); return result; } /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -992,7 +1517,7 @@ def visitModule(self, mod): if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1002,7 +1527,7 @@ def visitModule(self, mod): return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1014,7 +1539,7 @@ def visitModule(self, mod): #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1052,7 +1577,7 @@ def visitModule(self, mod): { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1061,7 +1586,7 @@ def visitModule(self, mod): { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1071,12 +1596,12 @@ def visitModule(self, mod): int i; if (!PyLong_Check(obj)) { PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); - return 1; + return -1; } i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) - return 1; + return -1; *out = i; return 0; } @@ -1102,31 +1627,24 @@ def visitModule(self, mod): static int init_types(struct ast_state *state) { - // init_types() must not be called after _PyAST_Fini() - // has been called - assert(state->initialized >= 0); - - if (state->initialized) { - return 1; - } if (init_identifiers(state) < 0) { - return 0; + return -1; } state->AST_type = PyType_FromSpec(&AST_type_spec); if (!state->AST_type) { - return 0; + return -1; } if (add_ast_fields(state) < 0) { - return 0; + return -1; } ''')) for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' - state->recursion_depth = 0; - state->recursion_limit = 0; - state->initialized = 1; - return 1; + if (!add_ast_annotations(state)) { + return -1; + } + return 0; } ''')) @@ -1138,12 +1656,12 @@ def visitProduct(self, prod, name): self.emit('state->%s_type = make_type(state, "%s", state->AST_type, %s, %d,' % (name, name, fields, len(prod.fields)), 1) self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % name, 1) + self.emit("if (!state->%s_type) return -1;" % name, 1) if prod.attributes: - self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" % + self.emit("if (add_attributes(state, state->%s_type, %s_attributes, %d) < 0) return -1;" % (name, name, len(prod.attributes)), 1) else: - self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) self.emit_defaults(name, prod.fields, 1) self.emit_defaults(name, prod.attributes, 1) @@ -1151,12 +1669,12 @@ def visitSum(self, sum, name): self.emit('state->%s_type = make_type(state, "%s", state->AST_type, NULL, 0,' % (name, name), 1) self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % name, 1) + self.emit("if (!state->%s_type) return -1;" % name, 1) if sum.attributes: - self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" % + self.emit("if (add_attributes(state, state->%s_type, %s_attributes, %d) < 0) return -1;" % (name, name, len(sum.attributes)), 1) else: - self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1) + self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) self.emit_defaults(name, sum.attributes, 1) simple = is_simple(sum) for t in sum.types: @@ -1170,20 +1688,20 @@ def visitConstructor(self, cons, name, simple): self.emit('state->%s_type = make_type(state, "%s", state->%s_type, %s, %d,' % (cons.name, cons.name, name, fields, len(cons.fields)), 1) self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False) - self.emit("if (!state->%s_type) return 0;" % cons.name, 1) + self.emit("if (!state->%s_type) return -1;" % cons.name, 1) self.emit_defaults(cons.name, cons.fields, 1) if simple: self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)" "state->%s_type, NULL, NULL);" % (cons.name, cons.name), 1) - self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1) + self.emit("if (!state->%s_singleton) return -1;" % cons.name, 1) def emit_defaults(self, name, fields, depth): for field in fields: if field.opt: self.emit('if (PyObject_SetAttr(state->%s_type, state->%s, Py_None) == -1)' % (name, field.name), depth) - self.emit("return 0;", depth+1) + self.emit("return -1;", depth+1) class ASTModuleVisitor(PickleVisitor): @@ -1220,6 +1738,7 @@ def visitModule(self, mod): static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -1268,7 +1787,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, struct validator *vstate, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1276,16 +1795,17 @@ def func_begin(self, name): self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) - self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) + self.emit("if (++vstate->recursion_depth > vstate->recursion_limit) {", 1) self.emit("PyErr_SetString(PyExc_RecursionError,", 2) self.emit('"maximum recursion depth exceeded during ast construction");', 3) - self.emit("return 0;", 2) + self.emit("return NULL;", 2) self.emit("}", 1) def func_end(self): - self.emit("state->recursion_depth--;", 1) + self.emit("vstate->recursion_depth--;", 1) self.emit("return result;", 1) self.emit("failed:", 0) + self.emit("vstate->recursion_depth--;", 1) self.emit("Py_XDECREF(value);", 1) self.emit("Py_XDECREF(result);", 1) self.emit("return NULL;", 1) @@ -1303,7 +1823,7 @@ def visitSum(self, sum, name): self.visitConstructor(t, i + 1, name) self.emit("}", 1) for a in sum.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit('if (PyObject_SetAttr(result, state->%s, value) < 0)' % a.name, 1) self.emit('goto failed;', 2) @@ -1311,7 +1831,7 @@ def visitSum(self, sum, name): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1329,7 +1849,7 @@ def visitProduct(self, prod, name): for field in prod.fields: self.visitField(field, name, 1, True) for a in prod.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit("if (PyObject_SetAttr(result, state->%s, value) < 0)" % a.name, 1) self.emit('goto failed;', 2) @@ -1370,7 +1890,7 @@ def set(self, field, value, depth): self.emit("for(i = 0; i < n; i++)", depth+1) # This cannot fail, so no need for error handling self.emit( - "PyList_SET_ITEM(value, i, ast2obj_{0}(state, ({0}_ty)asdl_seq_GET({1}, i)));".format( + "PyList_SET_ITEM(value, i, ast2obj_{0}(state, vstate, ({0}_ty)asdl_seq_GET({1}, i)));".format( field.type, value ), @@ -1379,9 +1899,9 @@ def set(self, field, value, depth): ) self.emit("}", depth) else: - self.emit("value = ast2obj_list(state, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) + self.emit("value = ast2obj_list(state, vstate, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) else: - self.emit("value = ast2obj_%s(state, %s);" % (field.type, value), depth, reflow=False) + self.emit("value = ast2obj_%s(state, vstate, %s);" % (field.type, value), depth, reflow=False) class PartingShots(StaticVisitor): @@ -1396,24 +1916,24 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { - return 0; + return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + starting_recursion_depth = recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; + starting_recursion_depth, vstate.recursion_depth); + return NULL; } return result; } @@ -1480,9 +2000,8 @@ def visit(self, object): def generate_ast_state(module_state, f): f.write('struct ast_state {\n') - f.write(' int initialized;\n') - f.write(' int recursion_depth;\n') - f.write(' int recursion_limit;\n') + f.write(' _PyOnceFlag once;\n') + f.write(' int finalized;\n') for s in module_state: f.write(' PyObject *' + s + ';\n') f.write('};') @@ -1500,11 +2019,8 @@ def generate_ast_fini(module_state, f): f.write(textwrap.dedent(""" Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); - #if !defined(NDEBUG) - state->initialized = -1; - #else - state->initialized = 0; - #endif + state->finalized = 1; + state->once = (_PyOnceFlag){0}; } """)) @@ -1543,10 +2059,20 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall + #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast + #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "pycore_setobject.h" // _PySet_NextEntry(), _PySet_Update() + #include "pycore_unionobject.h" // _Py_union_type_or + #include "structmember.h" #include + struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ + }; + // Forward declaration static int init_types(struct ast_state *state); @@ -1555,7 +2081,8 @@ def generate_module_def(mod, metadata, f, internal_h): { PyInterpreterState *interp = _PyInterpreterState_GET(); struct ast_state *state = &interp->ast; - if (!init_types(state)) { + assert(!state->finalized); + if (_PyOnceFlag_CallOnce(&state->once, (_Py_once_fn_t *)&init_types, state) < 0) { return NULL; } return state; @@ -1569,8 +2096,8 @@ def generate_module_def(mod, metadata, f, internal_h): for identifier in state_strings: f.write(' if ((state->' + identifier) f.write(' = PyUnicode_InternFromString("') - f.write(identifier + '")) == NULL) return 0;\n') - f.write(' return 1;\n') + f.write(identifier + '")) == NULL) return -1;\n') + f.write(' return 0;\n') f.write('};\n\n') def write_header(mod, metadata, f): @@ -1628,6 +2155,9 @@ def write_internal_h_header(mod, f): print(textwrap.dedent(""" #ifndef Py_INTERNAL_AST_STATE_H #define Py_INTERNAL_AST_STATE_H + + #include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -1653,6 +2183,7 @@ def write_source(mod, metadata, f, internal_h_file): v = ChainOfVisitors( SequenceConstructorVisitor(f), PyTypesDeclareVisitor(f), + AnnotationsVisitor(f), PyTypesVisitor(f), Obj2ModPrototypeVisitor(f), FunctionVisitor(f), diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index c7134ab868bfbd..82b0e4ee352d62 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -59,6 +59,10 @@ tok_nextc(struct tok_state *tok) int rc; for (;;) { if (tok->cur != tok->inp) { + if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) { + tok->done = E_COLUMNOVERFLOW; + return EOF; + } tok->col_offset++; return Py_CHARMASK(*tok->cur++); /* Fast path */ } @@ -112,13 +116,56 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { if (!tok_mode->f_string_debug || token->metadata) { return 0; } + PyObject *res = NULL; + + // Check if there is a # character in the expression + int hash_detected = 0; + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { + if (tok_mode->last_expr_buffer[i] == '#') { + hash_detected = 1; + break; + } + } + + if (hash_detected) { + Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; + char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); + if (!result) { + return -1; + } - PyObject *res = PyUnicode_DecodeUTF8( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, - NULL - ); - if (!res) { + Py_ssize_t i = 0; + Py_ssize_t j = 0; + + for (i = 0, j = 0; i < input_length; i++) { + if (tok_mode->last_expr_buffer[i] == '#') { + // Skip characters until newline or end of string + while (tok_mode->last_expr_buffer[i] != '\0' && i < input_length) { + if (tok_mode->last_expr_buffer[i] == '\n') { + result[j++] = tok_mode->last_expr_buffer[i]; + break; + } + i++; + } + } else { + result[j++] = tok_mode->last_expr_buffer[i]; + } + } + + result[j] = '\0'; // Null-terminate the result string + res = PyUnicode_DecodeUTF8(result, j, NULL); + PyMem_Free(result); + } else { + res = PyUnicode_DecodeUTF8( + tok_mode->last_expr_buffer, + tok_mode->last_expr_size - tok_mode->last_expr_end, + NULL + ); + + } + + + if (!res) { return -1; } token->metadata = res; @@ -968,6 +1015,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t int quote = c; int quote_size = 1; /* 1 or 3 */ int end_quote_size = 0; + int has_escaped_quote = 0; /* Nodes of type STRING, especially multi line strings must be handled differently in order to get both @@ -1033,8 +1081,18 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } else { - _PyTokenizer_syntaxerror(tok, "unterminated string literal (detected at" - " line %d)", start); + if (has_escaped_quote) { + _PyTokenizer_syntaxerror( + tok, + "unterminated string literal (detected at line %d); " + "perhaps you escaped the end quote?", + start + ); + } else { + _PyTokenizer_syntaxerror( + tok, "unterminated string literal (detected at line %d)", start + ); + } if (c != '\n') { tok->done = E_EOLS; } @@ -1048,6 +1106,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t end_quote_size = 0; if (c == '\\') { c = tok_nextc(tok); /* skip escaped char */ + if (c == quote) { /* but record whether the escaped char was a quote */ + has_escaped_quote = 1; + } if (c == '\r') { c = tok_nextc(tok); } @@ -1294,9 +1355,13 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok->lineno = the_current_tok->f_string_line_start; if (current_tok->f_string_quote_size == 3) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted f-string literal" - " (detected at line %d)", start)); + " (detected at line %d)", start); + if (c != '\n') { + tok->done = E_EOFS; + } + return MAKE_TOKEN(ERRORTOKEN); } else { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, diff --git a/Parser/parser.c b/Parser/parser.c index b10bf69e4df2ef..cbbe9a9be87178 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -8,7 +8,11 @@ #endif #ifdef __wasi__ -# define MAXSTACK 4000 +# ifdef Py_DEBUG +# define MAXSTACK 1000 +# else +# define MAXSTACK 4000 +# endif #else # define MAXSTACK 6000 #endif @@ -17,28 +21,28 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 656}, - {"as", 654}, - {"in", 667}, + {"if", 660}, + {"as", 658}, + {"in", 671}, {"or", 581}, {"is", 589}, {NULL, -1}, }, (KeywordToken[]) { {"del", 613}, - {"def", 669}, - {"for", 666}, - {"try", 638}, + {"def", 675}, + {"for", 670}, + {"try", 642}, {"and", 582}, - {"not", 588}, + {"not", 679}, {NULL, -1}, }, (KeywordToken[]) { - {"from", 618}, + {"from", 621}, {"pass", 504}, - {"with", 629}, - {"elif", 658}, - {"else", 659}, + {"with", 633}, + {"elif", 662}, + {"else", 663}, {"None", 611}, {"True", 610}, {NULL, -1}, @@ -47,24 +51,24 @@ static KeywordToken *reserved_keywords[] = { {"raise", 525}, {"yield", 580}, {"break", 508}, - {"async", 668}, - {"class", 671}, - {"while", 661}, + {"async", 674}, + {"class", 677}, + {"while", 665}, {"False", 612}, {"await", 590}, {NULL, -1}, }, (KeywordToken[]) { {"return", 522}, - {"import", 617}, + {"import", 622}, {"assert", 529}, {"global", 526}, - {"except", 651}, + {"except", 655}, {"lambda", 609}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 647}, + {"finally", 651}, {NULL, -1}, }, (KeywordToken[]) { @@ -182,429 +186,437 @@ static char *soft_keywords[] = { #define type_param_seq_type 1099 #define type_param_type 1100 #define type_param_bound_type 1101 -#define expressions_type 1102 -#define expression_type 1103 -#define yield_expr_type 1104 -#define star_expressions_type 1105 -#define star_expression_type 1106 -#define star_named_expressions_type 1107 -#define star_named_expression_type 1108 -#define assignment_expression_type 1109 -#define named_expression_type 1110 -#define disjunction_type 1111 -#define conjunction_type 1112 -#define inversion_type 1113 -#define comparison_type 1114 -#define compare_op_bitwise_or_pair_type 1115 -#define eq_bitwise_or_type 1116 -#define noteq_bitwise_or_type 1117 -#define lte_bitwise_or_type 1118 -#define lt_bitwise_or_type 1119 -#define gte_bitwise_or_type 1120 -#define gt_bitwise_or_type 1121 -#define notin_bitwise_or_type 1122 -#define in_bitwise_or_type 1123 -#define isnot_bitwise_or_type 1124 -#define is_bitwise_or_type 1125 -#define bitwise_or_type 1126 // Left-recursive -#define bitwise_xor_type 1127 // Left-recursive -#define bitwise_and_type 1128 // Left-recursive -#define shift_expr_type 1129 // Left-recursive -#define sum_type 1130 // Left-recursive -#define term_type 1131 // Left-recursive -#define factor_type 1132 -#define power_type 1133 -#define await_primary_type 1134 -#define primary_type 1135 // Left-recursive -#define slices_type 1136 -#define slice_type 1137 -#define atom_type 1138 -#define group_type 1139 -#define lambdef_type 1140 -#define lambda_params_type 1141 -#define lambda_parameters_type 1142 -#define lambda_slash_no_default_type 1143 -#define lambda_slash_with_default_type 1144 -#define lambda_star_etc_type 1145 -#define lambda_kwds_type 1146 -#define lambda_param_no_default_type 1147 -#define lambda_param_with_default_type 1148 -#define lambda_param_maybe_default_type 1149 -#define lambda_param_type 1150 -#define fstring_middle_type 1151 -#define fstring_replacement_field_type 1152 -#define fstring_conversion_type 1153 -#define fstring_full_format_spec_type 1154 -#define fstring_format_spec_type 1155 -#define fstring_type 1156 -#define string_type 1157 -#define strings_type 1158 -#define list_type 1159 -#define tuple_type 1160 -#define set_type 1161 -#define dict_type 1162 -#define double_starred_kvpairs_type 1163 -#define double_starred_kvpair_type 1164 -#define kvpair_type 1165 -#define for_if_clauses_type 1166 -#define for_if_clause_type 1167 -#define listcomp_type 1168 -#define setcomp_type 1169 -#define genexp_type 1170 -#define dictcomp_type 1171 -#define arguments_type 1172 -#define args_type 1173 -#define kwargs_type 1174 -#define starred_expression_type 1175 -#define kwarg_or_starred_type 1176 -#define kwarg_or_double_starred_type 1177 -#define star_targets_type 1178 -#define star_targets_list_seq_type 1179 -#define star_targets_tuple_seq_type 1180 -#define star_target_type 1181 -#define target_with_star_atom_type 1182 -#define star_atom_type 1183 -#define single_target_type 1184 -#define single_subscript_attribute_target_type 1185 -#define t_primary_type 1186 // Left-recursive -#define t_lookahead_type 1187 -#define del_targets_type 1188 -#define del_target_type 1189 -#define del_t_atom_type 1190 -#define type_expressions_type 1191 -#define func_type_comment_type 1192 -#define invalid_arguments_type 1193 -#define invalid_kwarg_type 1194 -#define expression_without_invalid_type 1195 -#define invalid_legacy_expression_type 1196 -#define invalid_expression_type 1197 -#define invalid_named_expression_type 1198 -#define invalid_assignment_type 1199 -#define invalid_ann_assign_target_type 1200 -#define invalid_del_stmt_type 1201 -#define invalid_block_type 1202 -#define invalid_comprehension_type 1203 -#define invalid_dict_comprehension_type 1204 -#define invalid_parameters_type 1205 -#define invalid_default_type 1206 -#define invalid_star_etc_type 1207 -#define invalid_kwds_type 1208 -#define invalid_parameters_helper_type 1209 -#define invalid_lambda_parameters_type 1210 -#define invalid_lambda_parameters_helper_type 1211 -#define invalid_lambda_star_etc_type 1212 -#define invalid_lambda_kwds_type 1213 -#define invalid_double_type_comments_type 1214 -#define invalid_with_item_type 1215 -#define invalid_for_target_type 1216 -#define invalid_group_type 1217 -#define invalid_import_type 1218 -#define invalid_import_from_targets_type 1219 -#define invalid_with_stmt_type 1220 -#define invalid_with_stmt_indent_type 1221 -#define invalid_try_stmt_type 1222 -#define invalid_except_stmt_type 1223 -#define invalid_finally_stmt_type 1224 -#define invalid_except_stmt_indent_type 1225 -#define invalid_except_star_stmt_indent_type 1226 -#define invalid_match_stmt_type 1227 -#define invalid_case_block_type 1228 -#define invalid_as_pattern_type 1229 -#define invalid_class_pattern_type 1230 -#define invalid_class_argument_pattern_type 1231 -#define invalid_if_stmt_type 1232 -#define invalid_elif_stmt_type 1233 -#define invalid_else_stmt_type 1234 -#define invalid_while_stmt_type 1235 -#define invalid_for_stmt_type 1236 -#define invalid_def_raw_type 1237 -#define invalid_class_def_raw_type 1238 -#define invalid_double_starred_kvpairs_type 1239 -#define invalid_kvpair_type 1240 -#define invalid_starred_expression_type 1241 -#define invalid_replacement_field_type 1242 -#define invalid_conversion_character_type 1243 -#define _loop0_1_type 1244 -#define _loop0_2_type 1245 -#define _loop1_3_type 1246 -#define _loop0_5_type 1247 -#define _gather_4_type 1248 -#define _tmp_6_type 1249 -#define _tmp_7_type 1250 -#define _tmp_8_type 1251 -#define _tmp_9_type 1252 -#define _tmp_10_type 1253 -#define _tmp_11_type 1254 -#define _tmp_12_type 1255 -#define _tmp_13_type 1256 -#define _loop1_14_type 1257 -#define _tmp_15_type 1258 -#define _tmp_16_type 1259 -#define _tmp_17_type 1260 -#define _loop0_19_type 1261 -#define _gather_18_type 1262 -#define _loop0_21_type 1263 -#define _gather_20_type 1264 -#define _tmp_22_type 1265 -#define _tmp_23_type 1266 -#define _loop0_24_type 1267 -#define _loop1_25_type 1268 -#define _loop0_27_type 1269 -#define _gather_26_type 1270 -#define _tmp_28_type 1271 -#define _loop0_30_type 1272 -#define _gather_29_type 1273 -#define _tmp_31_type 1274 -#define _loop1_32_type 1275 -#define _tmp_33_type 1276 -#define _tmp_34_type 1277 -#define _tmp_35_type 1278 -#define _loop0_36_type 1279 -#define _loop0_37_type 1280 -#define _loop0_38_type 1281 -#define _loop1_39_type 1282 -#define _loop0_40_type 1283 -#define _loop1_41_type 1284 -#define _loop1_42_type 1285 -#define _loop1_43_type 1286 -#define _loop0_44_type 1287 -#define _loop1_45_type 1288 -#define _loop0_46_type 1289 -#define _loop1_47_type 1290 -#define _loop0_48_type 1291 -#define _loop0_49_type 1292 -#define _loop1_50_type 1293 -#define _loop0_52_type 1294 -#define _gather_51_type 1295 -#define _loop0_54_type 1296 -#define _gather_53_type 1297 -#define _loop0_56_type 1298 -#define _gather_55_type 1299 -#define _loop0_58_type 1300 -#define _gather_57_type 1301 -#define _tmp_59_type 1302 -#define _loop1_60_type 1303 -#define _loop1_61_type 1304 -#define _tmp_62_type 1305 -#define _tmp_63_type 1306 -#define _loop1_64_type 1307 -#define _loop0_66_type 1308 -#define _gather_65_type 1309 -#define _tmp_67_type 1310 -#define _tmp_68_type 1311 -#define _tmp_69_type 1312 -#define _tmp_70_type 1313 -#define _loop0_72_type 1314 -#define _gather_71_type 1315 -#define _loop0_74_type 1316 -#define _gather_73_type 1317 -#define _tmp_75_type 1318 -#define _loop0_77_type 1319 -#define _gather_76_type 1320 -#define _loop0_79_type 1321 -#define _gather_78_type 1322 -#define _loop0_81_type 1323 -#define _gather_80_type 1324 -#define _loop1_82_type 1325 -#define _loop1_83_type 1326 -#define _loop0_85_type 1327 -#define _gather_84_type 1328 -#define _loop1_86_type 1329 -#define _loop1_87_type 1330 -#define _loop1_88_type 1331 -#define _tmp_89_type 1332 -#define _loop0_91_type 1333 -#define _gather_90_type 1334 -#define _tmp_92_type 1335 -#define _tmp_93_type 1336 -#define _tmp_94_type 1337 -#define _tmp_95_type 1338 -#define _tmp_96_type 1339 -#define _tmp_97_type 1340 -#define _loop0_98_type 1341 -#define _loop0_99_type 1342 -#define _loop0_100_type 1343 -#define _loop1_101_type 1344 -#define _loop0_102_type 1345 -#define _loop1_103_type 1346 -#define _loop1_104_type 1347 -#define _loop1_105_type 1348 -#define _loop0_106_type 1349 -#define _loop1_107_type 1350 -#define _loop0_108_type 1351 -#define _loop1_109_type 1352 -#define _loop0_110_type 1353 -#define _loop1_111_type 1354 -#define _tmp_112_type 1355 -#define _loop0_113_type 1356 -#define _loop0_114_type 1357 -#define _loop1_115_type 1358 -#define _tmp_116_type 1359 -#define _loop0_118_type 1360 -#define _gather_117_type 1361 -#define _loop1_119_type 1362 -#define _loop0_120_type 1363 -#define _loop0_121_type 1364 -#define _tmp_122_type 1365 -#define _loop0_124_type 1366 -#define _gather_123_type 1367 -#define _tmp_125_type 1368 -#define _loop0_127_type 1369 -#define _gather_126_type 1370 -#define _loop0_129_type 1371 -#define _gather_128_type 1372 -#define _loop0_131_type 1373 -#define _gather_130_type 1374 -#define _loop0_133_type 1375 -#define _gather_132_type 1376 -#define _loop0_134_type 1377 -#define _loop0_136_type 1378 -#define _gather_135_type 1379 -#define _loop1_137_type 1380 -#define _tmp_138_type 1381 -#define _loop0_140_type 1382 -#define _gather_139_type 1383 -#define _loop0_142_type 1384 -#define _gather_141_type 1385 -#define _loop0_144_type 1386 -#define _gather_143_type 1387 -#define _loop0_146_type 1388 -#define _gather_145_type 1389 -#define _loop0_148_type 1390 -#define _gather_147_type 1391 -#define _tmp_149_type 1392 -#define _tmp_150_type 1393 -#define _tmp_151_type 1394 -#define _tmp_152_type 1395 -#define _tmp_153_type 1396 -#define _tmp_154_type 1397 -#define _tmp_155_type 1398 -#define _tmp_156_type 1399 -#define _tmp_157_type 1400 -#define _tmp_158_type 1401 -#define _tmp_159_type 1402 -#define _tmp_160_type 1403 -#define _loop0_161_type 1404 -#define _loop0_162_type 1405 -#define _loop0_163_type 1406 -#define _tmp_164_type 1407 -#define _tmp_165_type 1408 -#define _tmp_166_type 1409 -#define _tmp_167_type 1410 -#define _tmp_168_type 1411 -#define _loop0_169_type 1412 -#define _loop0_170_type 1413 -#define _loop0_171_type 1414 -#define _loop1_172_type 1415 -#define _tmp_173_type 1416 -#define _loop0_174_type 1417 -#define _tmp_175_type 1418 -#define _loop0_176_type 1419 -#define _loop1_177_type 1420 -#define _tmp_178_type 1421 -#define _tmp_179_type 1422 -#define _tmp_180_type 1423 -#define _loop0_181_type 1424 -#define _tmp_182_type 1425 -#define _tmp_183_type 1426 -#define _loop1_184_type 1427 -#define _tmp_185_type 1428 -#define _loop0_186_type 1429 -#define _loop0_187_type 1430 -#define _loop0_188_type 1431 -#define _loop0_190_type 1432 -#define _gather_189_type 1433 -#define _tmp_191_type 1434 -#define _loop0_192_type 1435 -#define _tmp_193_type 1436 -#define _loop0_194_type 1437 -#define _loop1_195_type 1438 -#define _loop1_196_type 1439 -#define _tmp_197_type 1440 -#define _tmp_198_type 1441 -#define _loop0_199_type 1442 -#define _tmp_200_type 1443 -#define _tmp_201_type 1444 -#define _tmp_202_type 1445 -#define _loop0_204_type 1446 -#define _gather_203_type 1447 -#define _loop0_206_type 1448 -#define _gather_205_type 1449 -#define _loop0_208_type 1450 -#define _gather_207_type 1451 -#define _loop0_210_type 1452 -#define _gather_209_type 1453 -#define _loop0_212_type 1454 -#define _gather_211_type 1455 -#define _tmp_213_type 1456 -#define _loop0_214_type 1457 -#define _loop1_215_type 1458 -#define _tmp_216_type 1459 -#define _loop0_217_type 1460 -#define _loop1_218_type 1461 -#define _tmp_219_type 1462 -#define _tmp_220_type 1463 -#define _tmp_221_type 1464 -#define _tmp_222_type 1465 -#define _tmp_223_type 1466 -#define _tmp_224_type 1467 -#define _tmp_225_type 1468 -#define _tmp_226_type 1469 -#define _tmp_227_type 1470 -#define _tmp_228_type 1471 -#define _loop0_230_type 1472 -#define _gather_229_type 1473 -#define _tmp_231_type 1474 -#define _tmp_232_type 1475 -#define _tmp_233_type 1476 -#define _tmp_234_type 1477 -#define _tmp_235_type 1478 -#define _tmp_236_type 1479 -#define _tmp_237_type 1480 -#define _tmp_238_type 1481 -#define _tmp_239_type 1482 -#define _tmp_240_type 1483 -#define _tmp_241_type 1484 -#define _tmp_242_type 1485 -#define _tmp_243_type 1486 -#define _loop0_244_type 1487 -#define _tmp_245_type 1488 -#define _tmp_246_type 1489 -#define _tmp_247_type 1490 -#define _tmp_248_type 1491 -#define _tmp_249_type 1492 -#define _tmp_250_type 1493 -#define _tmp_251_type 1494 -#define _tmp_252_type 1495 -#define _tmp_253_type 1496 -#define _tmp_254_type 1497 -#define _tmp_255_type 1498 -#define _tmp_256_type 1499 -#define _tmp_257_type 1500 -#define _tmp_258_type 1501 -#define _tmp_259_type 1502 -#define _tmp_260_type 1503 -#define _tmp_261_type 1504 -#define _tmp_262_type 1505 -#define _tmp_263_type 1506 -#define _tmp_264_type 1507 -#define _tmp_265_type 1508 -#define _tmp_266_type 1509 -#define _tmp_267_type 1510 -#define _tmp_268_type 1511 -#define _tmp_269_type 1512 -#define _tmp_270_type 1513 -#define _tmp_271_type 1514 -#define _tmp_272_type 1515 -#define _tmp_273_type 1516 -#define _loop0_275_type 1517 -#define _gather_274_type 1518 -#define _tmp_276_type 1519 -#define _tmp_277_type 1520 -#define _tmp_278_type 1521 -#define _tmp_279_type 1522 -#define _tmp_280_type 1523 -#define _tmp_281_type 1524 +#define type_param_default_type 1102 +#define type_param_starred_default_type 1103 +#define expressions_type 1104 +#define expression_type 1105 +#define yield_expr_type 1106 +#define star_expressions_type 1107 +#define star_expression_type 1108 +#define star_named_expressions_type 1109 +#define star_named_expression_type 1110 +#define assignment_expression_type 1111 +#define named_expression_type 1112 +#define disjunction_type 1113 +#define conjunction_type 1114 +#define inversion_type 1115 +#define comparison_type 1116 +#define compare_op_bitwise_or_pair_type 1117 +#define eq_bitwise_or_type 1118 +#define noteq_bitwise_or_type 1119 +#define lte_bitwise_or_type 1120 +#define lt_bitwise_or_type 1121 +#define gte_bitwise_or_type 1122 +#define gt_bitwise_or_type 1123 +#define notin_bitwise_or_type 1124 +#define in_bitwise_or_type 1125 +#define isnot_bitwise_or_type 1126 +#define is_bitwise_or_type 1127 +#define bitwise_or_type 1128 // Left-recursive +#define bitwise_xor_type 1129 // Left-recursive +#define bitwise_and_type 1130 // Left-recursive +#define shift_expr_type 1131 // Left-recursive +#define sum_type 1132 // Left-recursive +#define term_type 1133 // Left-recursive +#define factor_type 1134 +#define power_type 1135 +#define await_primary_type 1136 +#define primary_type 1137 // Left-recursive +#define slices_type 1138 +#define slice_type 1139 +#define atom_type 1140 +#define group_type 1141 +#define lambdef_type 1142 +#define lambda_params_type 1143 +#define lambda_parameters_type 1144 +#define lambda_slash_no_default_type 1145 +#define lambda_slash_with_default_type 1146 +#define lambda_star_etc_type 1147 +#define lambda_kwds_type 1148 +#define lambda_param_no_default_type 1149 +#define lambda_param_with_default_type 1150 +#define lambda_param_maybe_default_type 1151 +#define lambda_param_type 1152 +#define fstring_middle_type 1153 +#define fstring_replacement_field_type 1154 +#define fstring_conversion_type 1155 +#define fstring_full_format_spec_type 1156 +#define fstring_format_spec_type 1157 +#define fstring_type 1158 +#define string_type 1159 +#define strings_type 1160 +#define list_type 1161 +#define tuple_type 1162 +#define set_type 1163 +#define dict_type 1164 +#define double_starred_kvpairs_type 1165 +#define double_starred_kvpair_type 1166 +#define kvpair_type 1167 +#define for_if_clauses_type 1168 +#define for_if_clause_type 1169 +#define listcomp_type 1170 +#define setcomp_type 1171 +#define genexp_type 1172 +#define dictcomp_type 1173 +#define arguments_type 1174 +#define args_type 1175 +#define kwargs_type 1176 +#define starred_expression_type 1177 +#define kwarg_or_starred_type 1178 +#define kwarg_or_double_starred_type 1179 +#define star_targets_type 1180 +#define star_targets_list_seq_type 1181 +#define star_targets_tuple_seq_type 1182 +#define star_target_type 1183 +#define target_with_star_atom_type 1184 +#define star_atom_type 1185 +#define single_target_type 1186 +#define single_subscript_attribute_target_type 1187 +#define t_primary_type 1188 // Left-recursive +#define t_lookahead_type 1189 +#define del_targets_type 1190 +#define del_target_type 1191 +#define del_t_atom_type 1192 +#define type_expressions_type 1193 +#define func_type_comment_type 1194 +#define invalid_arguments_type 1195 +#define invalid_kwarg_type 1196 +#define expression_without_invalid_type 1197 +#define invalid_legacy_expression_type 1198 +#define invalid_type_param_type 1199 +#define invalid_expression_type 1200 +#define invalid_named_expression_type 1201 +#define invalid_assignment_type 1202 +#define invalid_ann_assign_target_type 1203 +#define invalid_del_stmt_type 1204 +#define invalid_block_type 1205 +#define invalid_comprehension_type 1206 +#define invalid_dict_comprehension_type 1207 +#define invalid_parameters_type 1208 +#define invalid_default_type 1209 +#define invalid_star_etc_type 1210 +#define invalid_kwds_type 1211 +#define invalid_parameters_helper_type 1212 +#define invalid_lambda_parameters_type 1213 +#define invalid_lambda_parameters_helper_type 1214 +#define invalid_lambda_star_etc_type 1215 +#define invalid_lambda_kwds_type 1216 +#define invalid_double_type_comments_type 1217 +#define invalid_with_item_type 1218 +#define invalid_for_if_clause_type 1219 +#define invalid_for_target_type 1220 +#define invalid_group_type 1221 +#define invalid_import_type 1222 +#define invalid_import_from_targets_type 1223 +#define invalid_with_stmt_type 1224 +#define invalid_with_stmt_indent_type 1225 +#define invalid_try_stmt_type 1226 +#define invalid_except_stmt_type 1227 +#define invalid_finally_stmt_type 1228 +#define invalid_except_stmt_indent_type 1229 +#define invalid_except_star_stmt_indent_type 1230 +#define invalid_match_stmt_type 1231 +#define invalid_case_block_type 1232 +#define invalid_as_pattern_type 1233 +#define invalid_class_pattern_type 1234 +#define invalid_class_argument_pattern_type 1235 +#define invalid_if_stmt_type 1236 +#define invalid_elif_stmt_type 1237 +#define invalid_else_stmt_type 1238 +#define invalid_while_stmt_type 1239 +#define invalid_for_stmt_type 1240 +#define invalid_def_raw_type 1241 +#define invalid_class_def_raw_type 1242 +#define invalid_double_starred_kvpairs_type 1243 +#define invalid_kvpair_type 1244 +#define invalid_starred_expression_unpacking_type 1245 +#define invalid_starred_expression_type 1246 +#define invalid_replacement_field_type 1247 +#define invalid_conversion_character_type 1248 +#define invalid_arithmetic_type 1249 +#define invalid_factor_type 1250 +#define invalid_type_params_type 1251 +#define _loop0_1_type 1252 +#define _loop0_2_type 1253 +#define _loop1_3_type 1254 +#define _loop0_5_type 1255 +#define _gather_4_type 1256 +#define _tmp_6_type 1257 +#define _tmp_7_type 1258 +#define _tmp_8_type 1259 +#define _tmp_9_type 1260 +#define _tmp_10_type 1261 +#define _tmp_11_type 1262 +#define _tmp_12_type 1263 +#define _tmp_13_type 1264 +#define _loop1_14_type 1265 +#define _tmp_15_type 1266 +#define _tmp_16_type 1267 +#define _tmp_17_type 1268 +#define _loop0_19_type 1269 +#define _gather_18_type 1270 +#define _loop0_21_type 1271 +#define _gather_20_type 1272 +#define _tmp_22_type 1273 +#define _tmp_23_type 1274 +#define _loop0_24_type 1275 +#define _loop1_25_type 1276 +#define _loop0_27_type 1277 +#define _gather_26_type 1278 +#define _tmp_28_type 1279 +#define _loop0_30_type 1280 +#define _gather_29_type 1281 +#define _tmp_31_type 1282 +#define _loop1_32_type 1283 +#define _tmp_33_type 1284 +#define _tmp_34_type 1285 +#define _tmp_35_type 1286 +#define _loop0_36_type 1287 +#define _loop0_37_type 1288 +#define _loop0_38_type 1289 +#define _loop1_39_type 1290 +#define _loop0_40_type 1291 +#define _loop1_41_type 1292 +#define _loop1_42_type 1293 +#define _loop1_43_type 1294 +#define _loop0_44_type 1295 +#define _loop1_45_type 1296 +#define _loop0_46_type 1297 +#define _loop1_47_type 1298 +#define _loop0_48_type 1299 +#define _loop0_49_type 1300 +#define _loop1_50_type 1301 +#define _loop0_52_type 1302 +#define _gather_51_type 1303 +#define _loop0_54_type 1304 +#define _gather_53_type 1305 +#define _loop0_56_type 1306 +#define _gather_55_type 1307 +#define _loop0_58_type 1308 +#define _gather_57_type 1309 +#define _tmp_59_type 1310 +#define _loop1_60_type 1311 +#define _loop1_61_type 1312 +#define _tmp_62_type 1313 +#define _tmp_63_type 1314 +#define _loop1_64_type 1315 +#define _loop0_66_type 1316 +#define _gather_65_type 1317 +#define _tmp_67_type 1318 +#define _tmp_68_type 1319 +#define _tmp_69_type 1320 +#define _tmp_70_type 1321 +#define _loop0_72_type 1322 +#define _gather_71_type 1323 +#define _loop0_74_type 1324 +#define _gather_73_type 1325 +#define _tmp_75_type 1326 +#define _loop0_77_type 1327 +#define _gather_76_type 1328 +#define _loop0_79_type 1329 +#define _gather_78_type 1330 +#define _loop0_81_type 1331 +#define _gather_80_type 1332 +#define _loop1_82_type 1333 +#define _loop1_83_type 1334 +#define _loop0_85_type 1335 +#define _gather_84_type 1336 +#define _loop1_86_type 1337 +#define _loop1_87_type 1338 +#define _loop1_88_type 1339 +#define _tmp_89_type 1340 +#define _loop0_91_type 1341 +#define _gather_90_type 1342 +#define _tmp_92_type 1343 +#define _tmp_93_type 1344 +#define _tmp_94_type 1345 +#define _tmp_95_type 1346 +#define _tmp_96_type 1347 +#define _tmp_97_type 1348 +#define _loop0_98_type 1349 +#define _loop0_99_type 1350 +#define _loop0_100_type 1351 +#define _loop1_101_type 1352 +#define _loop0_102_type 1353 +#define _loop1_103_type 1354 +#define _loop1_104_type 1355 +#define _loop1_105_type 1356 +#define _loop0_106_type 1357 +#define _loop1_107_type 1358 +#define _loop0_108_type 1359 +#define _loop1_109_type 1360 +#define _loop0_110_type 1361 +#define _loop1_111_type 1362 +#define _loop0_112_type 1363 +#define _loop0_113_type 1364 +#define _loop1_114_type 1365 +#define _tmp_115_type 1366 +#define _loop0_117_type 1367 +#define _gather_116_type 1368 +#define _loop1_118_type 1369 +#define _loop0_119_type 1370 +#define _loop0_120_type 1371 +#define _tmp_121_type 1372 +#define _loop0_123_type 1373 +#define _gather_122_type 1374 +#define _tmp_124_type 1375 +#define _loop0_126_type 1376 +#define _gather_125_type 1377 +#define _loop0_128_type 1378 +#define _gather_127_type 1379 +#define _loop0_130_type 1380 +#define _gather_129_type 1381 +#define _loop0_132_type 1382 +#define _gather_131_type 1383 +#define _loop0_133_type 1384 +#define _loop0_135_type 1385 +#define _gather_134_type 1386 +#define _loop1_136_type 1387 +#define _tmp_137_type 1388 +#define _loop0_139_type 1389 +#define _gather_138_type 1390 +#define _loop0_141_type 1391 +#define _gather_140_type 1392 +#define _loop0_143_type 1393 +#define _gather_142_type 1394 +#define _loop0_145_type 1395 +#define _gather_144_type 1396 +#define _loop0_147_type 1397 +#define _gather_146_type 1398 +#define _tmp_148_type 1399 +#define _tmp_149_type 1400 +#define _loop0_151_type 1401 +#define _gather_150_type 1402 +#define _tmp_152_type 1403 +#define _tmp_153_type 1404 +#define _tmp_154_type 1405 +#define _tmp_155_type 1406 +#define _tmp_156_type 1407 +#define _tmp_157_type 1408 +#define _tmp_158_type 1409 +#define _tmp_159_type 1410 +#define _tmp_160_type 1411 +#define _tmp_161_type 1412 +#define _loop0_162_type 1413 +#define _loop0_163_type 1414 +#define _loop0_164_type 1415 +#define _tmp_165_type 1416 +#define _tmp_166_type 1417 +#define _tmp_167_type 1418 +#define _tmp_168_type 1419 +#define _loop0_169_type 1420 +#define _loop0_170_type 1421 +#define _loop0_171_type 1422 +#define _loop1_172_type 1423 +#define _tmp_173_type 1424 +#define _loop0_174_type 1425 +#define _tmp_175_type 1426 +#define _loop0_176_type 1427 +#define _loop1_177_type 1428 +#define _tmp_178_type 1429 +#define _tmp_179_type 1430 +#define _tmp_180_type 1431 +#define _loop0_181_type 1432 +#define _tmp_182_type 1433 +#define _tmp_183_type 1434 +#define _loop1_184_type 1435 +#define _tmp_185_type 1436 +#define _loop0_186_type 1437 +#define _loop0_187_type 1438 +#define _loop0_188_type 1439 +#define _loop0_190_type 1440 +#define _gather_189_type 1441 +#define _tmp_191_type 1442 +#define _loop0_192_type 1443 +#define _tmp_193_type 1444 +#define _loop0_194_type 1445 +#define _loop1_195_type 1446 +#define _loop1_196_type 1447 +#define _tmp_197_type 1448 +#define _tmp_198_type 1449 +#define _loop0_199_type 1450 +#define _tmp_200_type 1451 +#define _tmp_201_type 1452 +#define _tmp_202_type 1453 +#define _tmp_203_type 1454 +#define _loop0_205_type 1455 +#define _gather_204_type 1456 +#define _loop0_207_type 1457 +#define _gather_206_type 1458 +#define _loop0_209_type 1459 +#define _gather_208_type 1460 +#define _loop0_211_type 1461 +#define _gather_210_type 1462 +#define _loop0_213_type 1463 +#define _gather_212_type 1464 +#define _tmp_214_type 1465 +#define _loop0_215_type 1466 +#define _loop1_216_type 1467 +#define _tmp_217_type 1468 +#define _loop0_218_type 1469 +#define _loop1_219_type 1470 +#define _tmp_220_type 1471 +#define _tmp_221_type 1472 +#define _tmp_222_type 1473 +#define _tmp_223_type 1474 +#define _tmp_224_type 1475 +#define _tmp_225_type 1476 +#define _tmp_226_type 1477 +#define _tmp_227_type 1478 +#define _tmp_228_type 1479 +#define _tmp_229_type 1480 +#define _tmp_230_type 1481 +#define _loop0_232_type 1482 +#define _gather_231_type 1483 +#define _tmp_233_type 1484 +#define _tmp_234_type 1485 +#define _tmp_235_type 1486 +#define _tmp_236_type 1487 +#define _tmp_237_type 1488 +#define _tmp_238_type 1489 +#define _tmp_239_type 1490 +#define _loop0_240_type 1491 +#define _tmp_241_type 1492 +#define _tmp_242_type 1493 +#define _tmp_243_type 1494 +#define _tmp_244_type 1495 +#define _tmp_245_type 1496 +#define _tmp_246_type 1497 +#define _tmp_247_type 1498 +#define _tmp_248_type 1499 +#define _tmp_249_type 1500 +#define _tmp_250_type 1501 +#define _tmp_251_type 1502 +#define _tmp_252_type 1503 +#define _tmp_253_type 1504 +#define _tmp_254_type 1505 +#define _tmp_255_type 1506 +#define _tmp_256_type 1507 +#define _tmp_257_type 1508 +#define _tmp_258_type 1509 +#define _tmp_259_type 1510 +#define _tmp_260_type 1511 +#define _tmp_261_type 1512 +#define _tmp_262_type 1513 +#define _tmp_263_type 1514 +#define _tmp_264_type 1515 +#define _tmp_265_type 1516 +#define _loop0_266_type 1517 +#define _tmp_267_type 1518 +#define _tmp_268_type 1519 +#define _tmp_269_type 1520 +#define _tmp_270_type 1521 +#define _tmp_271_type 1522 +#define _tmp_272_type 1523 +#define _loop0_274_type 1524 +#define _gather_273_type 1525 +#define _tmp_275_type 1526 +#define _tmp_276_type 1527 +#define _tmp_277_type 1528 +#define _tmp_278_type 1529 +#define _tmp_279_type 1530 +#define _tmp_280_type 1531 +#define _tmp_281_type 1532 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -708,6 +720,8 @@ static asdl_type_param_seq* type_params_rule(Parser *p); static asdl_type_param_seq* type_param_seq_rule(Parser *p); static type_param_ty type_param_rule(Parser *p); static expr_ty type_param_bound_rule(Parser *p); +static expr_ty type_param_default_rule(Parser *p); +static expr_ty type_param_starred_default_rule(Parser *p); static expr_ty expressions_rule(Parser *p); static expr_ty expression_rule(Parser *p); static expr_ty yield_expr_rule(Parser *p); @@ -803,6 +817,7 @@ static void *invalid_arguments_rule(Parser *p); static void *invalid_kwarg_rule(Parser *p); static expr_ty expression_without_invalid_rule(Parser *p); static void *invalid_legacy_expression_rule(Parser *p); +static void *invalid_type_param_rule(Parser *p); static void *invalid_expression_rule(Parser *p); static void *invalid_named_expression_rule(Parser *p); static void *invalid_assignment_rule(Parser *p); @@ -822,6 +837,7 @@ static void *invalid_lambda_star_etc_rule(Parser *p); static void *invalid_lambda_kwds_rule(Parser *p); static void *invalid_double_type_comments_rule(Parser *p); static void *invalid_with_item_rule(Parser *p); +static void *invalid_for_if_clause_rule(Parser *p); static void *invalid_for_target_rule(Parser *p); static void *invalid_group_rule(Parser *p); static void *invalid_import_rule(Parser *p); @@ -847,9 +863,13 @@ static void *invalid_def_raw_rule(Parser *p); static void *invalid_class_def_raw_rule(Parser *p); static void *invalid_double_starred_kvpairs_rule(Parser *p); static void *invalid_kvpair_rule(Parser *p); +static void *invalid_starred_expression_unpacking_rule(Parser *p); static void *invalid_starred_expression_rule(Parser *p); static void *invalid_replacement_field_rule(Parser *p); static void *invalid_conversion_character_rule(Parser *p); +static void *invalid_arithmetic_rule(Parser *p); +static void *invalid_factor_rule(Parser *p); +static void *invalid_type_params_rule(Parser *p); static asdl_seq *_loop0_1_rule(Parser *p); static asdl_seq *_loop0_2_rule(Parser *p); static asdl_seq *_loop1_3_rule(Parser *p); @@ -961,46 +981,46 @@ static asdl_seq *_loop0_108_rule(Parser *p); static asdl_seq *_loop1_109_rule(Parser *p); static asdl_seq *_loop0_110_rule(Parser *p); static asdl_seq *_loop1_111_rule(Parser *p); -static void *_tmp_112_rule(Parser *p); +static asdl_seq *_loop0_112_rule(Parser *p); static asdl_seq *_loop0_113_rule(Parser *p); -static asdl_seq *_loop0_114_rule(Parser *p); -static asdl_seq *_loop1_115_rule(Parser *p); -static void *_tmp_116_rule(Parser *p); -static asdl_seq *_loop0_118_rule(Parser *p); -static asdl_seq *_gather_117_rule(Parser *p); -static asdl_seq *_loop1_119_rule(Parser *p); +static asdl_seq *_loop1_114_rule(Parser *p); +static void *_tmp_115_rule(Parser *p); +static asdl_seq *_loop0_117_rule(Parser *p); +static asdl_seq *_gather_116_rule(Parser *p); +static asdl_seq *_loop1_118_rule(Parser *p); +static asdl_seq *_loop0_119_rule(Parser *p); static asdl_seq *_loop0_120_rule(Parser *p); -static asdl_seq *_loop0_121_rule(Parser *p); -static void *_tmp_122_rule(Parser *p); -static asdl_seq *_loop0_124_rule(Parser *p); -static asdl_seq *_gather_123_rule(Parser *p); -static void *_tmp_125_rule(Parser *p); -static asdl_seq *_loop0_127_rule(Parser *p); -static asdl_seq *_gather_126_rule(Parser *p); -static asdl_seq *_loop0_129_rule(Parser *p); -static asdl_seq *_gather_128_rule(Parser *p); -static asdl_seq *_loop0_131_rule(Parser *p); -static asdl_seq *_gather_130_rule(Parser *p); +static void *_tmp_121_rule(Parser *p); +static asdl_seq *_loop0_123_rule(Parser *p); +static asdl_seq *_gather_122_rule(Parser *p); +static void *_tmp_124_rule(Parser *p); +static asdl_seq *_loop0_126_rule(Parser *p); +static asdl_seq *_gather_125_rule(Parser *p); +static asdl_seq *_loop0_128_rule(Parser *p); +static asdl_seq *_gather_127_rule(Parser *p); +static asdl_seq *_loop0_130_rule(Parser *p); +static asdl_seq *_gather_129_rule(Parser *p); +static asdl_seq *_loop0_132_rule(Parser *p); +static asdl_seq *_gather_131_rule(Parser *p); static asdl_seq *_loop0_133_rule(Parser *p); -static asdl_seq *_gather_132_rule(Parser *p); -static asdl_seq *_loop0_134_rule(Parser *p); -static asdl_seq *_loop0_136_rule(Parser *p); -static asdl_seq *_gather_135_rule(Parser *p); -static asdl_seq *_loop1_137_rule(Parser *p); -static void *_tmp_138_rule(Parser *p); -static asdl_seq *_loop0_140_rule(Parser *p); -static asdl_seq *_gather_139_rule(Parser *p); -static asdl_seq *_loop0_142_rule(Parser *p); -static asdl_seq *_gather_141_rule(Parser *p); -static asdl_seq *_loop0_144_rule(Parser *p); -static asdl_seq *_gather_143_rule(Parser *p); -static asdl_seq *_loop0_146_rule(Parser *p); -static asdl_seq *_gather_145_rule(Parser *p); -static asdl_seq *_loop0_148_rule(Parser *p); -static asdl_seq *_gather_147_rule(Parser *p); +static asdl_seq *_loop0_135_rule(Parser *p); +static asdl_seq *_gather_134_rule(Parser *p); +static asdl_seq *_loop1_136_rule(Parser *p); +static void *_tmp_137_rule(Parser *p); +static asdl_seq *_loop0_139_rule(Parser *p); +static asdl_seq *_gather_138_rule(Parser *p); +static asdl_seq *_loop0_141_rule(Parser *p); +static asdl_seq *_gather_140_rule(Parser *p); +static asdl_seq *_loop0_143_rule(Parser *p); +static asdl_seq *_gather_142_rule(Parser *p); +static asdl_seq *_loop0_145_rule(Parser *p); +static asdl_seq *_gather_144_rule(Parser *p); +static asdl_seq *_loop0_147_rule(Parser *p); +static asdl_seq *_gather_146_rule(Parser *p); +static void *_tmp_148_rule(Parser *p); static void *_tmp_149_rule(Parser *p); -static void *_tmp_150_rule(Parser *p); -static void *_tmp_151_rule(Parser *p); +static asdl_seq *_loop0_151_rule(Parser *p); +static asdl_seq *_gather_150_rule(Parser *p); static void *_tmp_152_rule(Parser *p); static void *_tmp_153_rule(Parser *p); static void *_tmp_154_rule(Parser *p); @@ -1010,10 +1030,10 @@ static void *_tmp_157_rule(Parser *p); static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); static void *_tmp_160_rule(Parser *p); -static asdl_seq *_loop0_161_rule(Parser *p); +static void *_tmp_161_rule(Parser *p); static asdl_seq *_loop0_162_rule(Parser *p); static asdl_seq *_loop0_163_rule(Parser *p); -static void *_tmp_164_rule(Parser *p); +static asdl_seq *_loop0_164_rule(Parser *p); static void *_tmp_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); @@ -1052,23 +1072,23 @@ static asdl_seq *_loop0_199_rule(Parser *p); static void *_tmp_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); static void *_tmp_202_rule(Parser *p); -static asdl_seq *_loop0_204_rule(Parser *p); -static asdl_seq *_gather_203_rule(Parser *p); -static asdl_seq *_loop0_206_rule(Parser *p); -static asdl_seq *_gather_205_rule(Parser *p); -static asdl_seq *_loop0_208_rule(Parser *p); -static asdl_seq *_gather_207_rule(Parser *p); -static asdl_seq *_loop0_210_rule(Parser *p); -static asdl_seq *_gather_209_rule(Parser *p); -static asdl_seq *_loop0_212_rule(Parser *p); -static asdl_seq *_gather_211_rule(Parser *p); -static void *_tmp_213_rule(Parser *p); -static asdl_seq *_loop0_214_rule(Parser *p); -static asdl_seq *_loop1_215_rule(Parser *p); -static void *_tmp_216_rule(Parser *p); -static asdl_seq *_loop0_217_rule(Parser *p); -static asdl_seq *_loop1_218_rule(Parser *p); -static void *_tmp_219_rule(Parser *p); +static void *_tmp_203_rule(Parser *p); +static asdl_seq *_loop0_205_rule(Parser *p); +static asdl_seq *_gather_204_rule(Parser *p); +static asdl_seq *_loop0_207_rule(Parser *p); +static asdl_seq *_gather_206_rule(Parser *p); +static asdl_seq *_loop0_209_rule(Parser *p); +static asdl_seq *_gather_208_rule(Parser *p); +static asdl_seq *_loop0_211_rule(Parser *p); +static asdl_seq *_gather_210_rule(Parser *p); +static asdl_seq *_loop0_213_rule(Parser *p); +static asdl_seq *_gather_212_rule(Parser *p); +static void *_tmp_214_rule(Parser *p); +static asdl_seq *_loop0_215_rule(Parser *p); +static asdl_seq *_loop1_216_rule(Parser *p); +static void *_tmp_217_rule(Parser *p); +static asdl_seq *_loop0_218_rule(Parser *p); +static asdl_seq *_loop1_219_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); static void *_tmp_222_rule(Parser *p); @@ -1078,10 +1098,10 @@ static void *_tmp_225_rule(Parser *p); static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); static void *_tmp_228_rule(Parser *p); -static asdl_seq *_loop0_230_rule(Parser *p); -static asdl_seq *_gather_229_rule(Parser *p); -static void *_tmp_231_rule(Parser *p); -static void *_tmp_232_rule(Parser *p); +static void *_tmp_229_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); +static asdl_seq *_loop0_232_rule(Parser *p); +static asdl_seq *_gather_231_rule(Parser *p); static void *_tmp_233_rule(Parser *p); static void *_tmp_234_rule(Parser *p); static void *_tmp_235_rule(Parser *p); @@ -1089,11 +1109,11 @@ static void *_tmp_236_rule(Parser *p); static void *_tmp_237_rule(Parser *p); static void *_tmp_238_rule(Parser *p); static void *_tmp_239_rule(Parser *p); -static void *_tmp_240_rule(Parser *p); +static asdl_seq *_loop0_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); -static asdl_seq *_loop0_244_rule(Parser *p); +static void *_tmp_244_rule(Parser *p); static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); @@ -1115,16 +1135,16 @@ static void *_tmp_262_rule(Parser *p); static void *_tmp_263_rule(Parser *p); static void *_tmp_264_rule(Parser *p); static void *_tmp_265_rule(Parser *p); -static void *_tmp_266_rule(Parser *p); +static asdl_seq *_loop0_266_rule(Parser *p); static void *_tmp_267_rule(Parser *p); static void *_tmp_268_rule(Parser *p); static void *_tmp_269_rule(Parser *p); static void *_tmp_270_rule(Parser *p); static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); -static void *_tmp_273_rule(Parser *p); -static asdl_seq *_loop0_275_rule(Parser *p); -static asdl_seq *_gather_274_rule(Parser *p); +static asdl_seq *_loop0_274_rule(Parser *p); +static asdl_seq *_gather_273_rule(Parser *p); +static void *_tmp_275_rule(Parser *p); static void *_tmp_276_rule(Parser *p); static void *_tmp_277_rule(Parser *p); static void *_tmp_278_rule(Parser *p); @@ -1794,7 +1814,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); stmt_ty import_stmt_var; if ( - _PyPegen_lookahead(1, _tmp_6_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) && (import_stmt_var = import_stmt_rule(p)) // import_stmt ) @@ -2068,7 +2088,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('def' | '@' | 'async') function_def")); stmt_ty function_def_var; if ( - _PyPegen_lookahead(1, _tmp_7_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) && (function_def_var = function_def_rule(p)) // function_def ) @@ -2089,7 +2109,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 656) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 660) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2110,7 +2130,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('class' | '@') class_def")); stmt_ty class_def_var; if ( - _PyPegen_lookahead(1, _tmp_8_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) && (class_def_var = class_def_rule(p)) // class_def ) @@ -2131,7 +2151,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('with' | 'async') with_stmt")); stmt_ty with_stmt_var; if ( - _PyPegen_lookahead(1, _tmp_9_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) && (with_stmt_var = with_stmt_rule(p)) // with_stmt ) @@ -2152,7 +2172,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('for' | 'async') for_stmt")); stmt_ty for_stmt_var; if ( - _PyPegen_lookahead(1, _tmp_10_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_10_rule, p) && (for_stmt_var = for_stmt_rule(p)) // for_stmt ) @@ -2173,7 +2193,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 638) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 642) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2194,7 +2214,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 661) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 665) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -3183,7 +3203,7 @@ del_stmt_rule(Parser *p) && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, _tmp_22_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_22_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -3468,7 +3488,7 @@ import_name_rule(Parser *p) Token * _keyword; asdl_alias_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword = _PyPegen_expect_token(p, 622)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) @@ -3537,13 +3557,13 @@ import_from_rule(Parser *p) expr_ty b; asdl_alias_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (a = _loop0_24_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && - (_keyword_1 = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 622)) // token='import' && (c = import_from_targets_rule(p)) // import_from_targets ) @@ -3581,11 +3601,11 @@ import_from_rule(Parser *p) asdl_seq * a; asdl_alias_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (a = _loop1_25_rule(p)) // (('.' | '...'))+ && - (_keyword_1 = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 622)) // token='import' && (b = import_from_targets_rule(p)) // import_from_targets ) @@ -4334,7 +4354,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4442,8 +4462,8 @@ function_def_rule(Parser *p) // function_def_raw: // | invalid_def_raw -// | 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block -// | 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block +// | 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block +// | 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block static stmt_ty function_def_raw_rule(Parser *p) { @@ -4484,12 +4504,12 @@ function_def_raw_rule(Parser *p) D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_def_raw")); } - { // 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _literal; Token * _literal_1; @@ -4501,13 +4521,13 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 669)) // token='def' + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4515,14 +4535,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_34_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4542,14 +4562,14 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } - { // 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _keyword_1; Token * _literal; @@ -4562,15 +4582,15 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 669)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 675)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4578,14 +4598,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_35_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4605,7 +4625,7 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } _res = NULL; done: @@ -5902,7 +5922,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -5947,7 +5967,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6042,7 +6062,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6087,7 +6107,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6168,7 +6188,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 659)) // token='else' + (_keyword = _PyPegen_expect_token(p, 663)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6247,7 +6267,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 661)) // token='while' + (_keyword = _PyPegen_expect_token(p, 665)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6347,11 +6367,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -6409,13 +6429,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 670)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -6483,7 +6503,7 @@ for_stmt_rule(Parser *p) // with_stmt: // | invalid_with_stmt_indent -// | 'with' '(' ','.with_item+ ','? ')' ':' block +// | 'with' '(' ','.with_item+ ','? ')' ':' TYPE_COMMENT? block // | 'with' ','.with_item+ ':' TYPE_COMMENT? block // | 'async' 'with' '(' ','.with_item+ ','? ')' ':' block // | 'async' 'with' ','.with_item+ ':' TYPE_COMMENT? block @@ -6528,12 +6548,12 @@ with_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s with_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_with_stmt_indent")); } - { // 'with' '(' ','.with_item+ ','? ')' ':' block + { // 'with' '(' ','.with_item+ ','? ')' ':' TYPE_COMMENT? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' block")); + D(fprintf(stderr, "%*c> with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' TYPE_COMMENT? block")); Token * _keyword; Token * _literal; Token * _literal_1; @@ -6542,8 +6562,9 @@ with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_withitem_seq* a; asdl_stmt_seq* b; + void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6555,10 +6576,12 @@ with_stmt_rule(Parser *p) && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? + && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ with_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' block")); + D(fprintf(stderr, "%*c+ with_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' TYPE_COMMENT? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -6568,7 +6591,7 @@ with_stmt_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = CHECK_VERSION ( stmt_ty , 9 , "Parenthesized context managers are" , _PyAST_With ( a , b , NULL , EXTRA ) ); + _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -6578,7 +6601,7 @@ with_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s with_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'with' '(' ','.with_item+ ','? ')' ':' TYPE_COMMENT? block")); } { // 'with' ','.with_item+ ':' TYPE_COMMENT? block if (p->error_indicator) { @@ -6592,7 +6615,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (a = (asdl_withitem_seq*)_gather_53_rule(p)) // ','.with_item+ && @@ -6641,9 +6664,9 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6693,9 +6716,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='with' && (a = (asdl_withitem_seq*)_gather_57_rule(p)) // ','.with_item+ && @@ -6781,11 +6804,11 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, _tmp_59_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_59_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); @@ -6906,7 +6929,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6950,7 +6973,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6998,7 +7021,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7096,7 +7119,7 @@ except_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (e = expression_rule(p)) // expression && @@ -7139,7 +7162,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7250,7 +7273,7 @@ except_star_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7352,7 +7375,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 651)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7660,7 +7683,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -7855,7 +7878,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -8190,7 +8213,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, _tmp_67_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_67_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8424,7 +8447,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, _tmp_68_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_68_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -9024,7 +9047,7 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, _tmp_69_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_69_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); @@ -9139,7 +9162,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, _tmp_70_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_70_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -10518,7 +10541,7 @@ type_alias_rule(Parser *p) return _res; } -// type_params: '[' type_param_seq ']' +// type_params: invalid_type_params | '[' type_param_seq ']' static asdl_type_param_seq* type_params_rule(Parser *p) { @@ -10531,6 +10554,25 @@ type_params_rule(Parser *p) } asdl_type_param_seq* _res = NULL; int _mark = p->mark; + if (p->call_invalid_rules) { // invalid_type_params + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + void *invalid_type_params_var; + if ( + (invalid_type_params_var = invalid_type_params_rule(p)) // invalid_type_params + ) + { + D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + _res = invalid_type_params_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_type_params")); + } { // '[' type_param_seq ']' if (p->error_indicator) { p->level--; @@ -10615,11 +10657,10 @@ type_param_seq_rule(Parser *p) } // type_param: -// | NAME type_param_bound? -// | '*' NAME ':' expression -// | '*' NAME -// | '**' NAME ':' expression -// | '**' NAME +// | NAME type_param_bound? type_param_default? +// | invalid_type_param +// | '*' NAME type_param_starred_default? +// | '**' NAME type_param_default? static type_param_ty type_param_rule(Parser *p) { @@ -10645,21 +10686,24 @@ type_param_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // NAME type_param_bound? + { // NAME type_param_bound? type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); expr_ty a; void *b; + void *c; if ( (a = _PyPegen_name_token(p)) // NAME && (b = type_param_bound_rule(p), !p->error_indicator) // type_param_bound? + && + (c = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10669,7 +10713,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVar ( a -> v . Name . id , b , EXTRA ); + _res = _PyAST_TypeVar ( a -> v . Name . id , b , c , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10679,30 +10723,55 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound?")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound? type_param_default?")); } - { // '*' NAME ':' expression + if (p->call_invalid_rules) { // invalid_type_param + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_type_param")); + void *invalid_type_param_var; + if ( + (invalid_type_param_var = invalid_type_param_rule(p)) // invalid_type_param + ) + { + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_type_param")); + _res = invalid_type_param_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_type_param")); + } + { // '*' NAME type_param_starred_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); Token * _literal; expr_ty a; - Token * colon; - expr_ty e; + void *b; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (a = _PyPegen_name_token(p)) // NAME && - (colon = _PyPegen_expect_token(p, 11)) // token=':' - && - (e = expression_rule(p)) // expression + (b = type_param_starred_default_rule(p), !p->error_indicator) // type_param_starred_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple" ); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_TypeVarTuple ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10712,23 +10781,26 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME type_param_starred_default?")); } - { // '*' NAME + { // '**' NAME type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token * _literal; expr_ty a; + void *b; if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' + (_literal = _PyPegen_expect_token(p, 35)) // token='**' && (a = _PyPegen_name_token(p)) // NAME + && + (b = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10738,7 +10810,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVarTuple ( a -> v . Name . id , EXTRA ); + _res = _PyAST_ParamSpec ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10748,30 +10820,44 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME type_param_default?")); } - { // '**' NAME ':' expression + _res = NULL; + done: + _PyPegen_insert_memo(p, _mark, type_param_type, _res); + p->level--; + return _res; +} + +// type_param_bound: ':' expression +static expr_ty +type_param_bound_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // ':' expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); + D(fprintf(stderr, "%*c> type_param_bound[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression")); Token * _literal; - expr_ty a; - Token * colon; expr_ty e; if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' - && - (a = _PyPegen_name_token(p)) // NAME - && - (colon = _PyPegen_expect_token(p, 11)) // token=':' + (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (e = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec" ); + D(fprintf(stderr, "%*c+ type_param_bound[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression")); + _res = e; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10780,34 +10866,44 @@ type_param_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); + D(fprintf(stderr, "%*c%s type_param_bound[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// type_param_default: '=' expression +static expr_ty +type_param_default_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); } - { // '**' NAME + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // '=' expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c> type_param_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' expression")); Token * _literal; - expr_ty a; + expr_ty e; if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' + (_literal = _PyPegen_expect_token(p, 22)) // token='=' && - (a = _PyPegen_name_token(p)) // NAME + (e = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME")); - Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); - if (_token == NULL) { - p->level--; - return NULL; - } - int _end_lineno = _token->end_lineno; - UNUSED(_end_lineno); // Only used by EXTRA macro - int _end_col_offset = _token->end_col_offset; - UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_ParamSpec ( a -> v . Name . id , EXTRA ); + D(fprintf(stderr, "%*c+ type_param_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10816,19 +10912,18 @@ type_param_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c%s type_param_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' expression")); } _res = NULL; done: - _PyPegen_insert_memo(p, _mark, type_param_type, _res); p->level--; return _res; } -// type_param_bound: ':' expression +// type_param_starred_default: '=' star_expression static expr_ty -type_param_bound_rule(Parser *p) +type_param_starred_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -10839,22 +10934,22 @@ type_param_bound_rule(Parser *p) } expr_ty _res = NULL; int _mark = p->mark; - { // ':' expression + { // '=' star_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param_bound[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression")); + D(fprintf(stderr, "%*c> type_param_starred_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); Token * _literal; expr_ty e; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (_literal = _PyPegen_expect_token(p, 22)) // token='=' && - (e = expression_rule(p)) // expression + (e = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ type_param_bound[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression")); - _res = e; + D(fprintf(stderr, "%*c+ type_param_starred_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10863,8 +10958,8 @@ type_param_bound_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s type_param_bound[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression")); + D(fprintf(stderr, "%*c%s type_param_starred_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' star_expression")); } _res = NULL; done: @@ -11078,11 +11173,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 659)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 663)) // token='else' && (c = expression_rule(p)) // expression ) @@ -11188,7 +11283,7 @@ yield_expr_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 580)) // token='yield' && - (_keyword_1 = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword_1 = _PyPegen_expect_token(p, 621)) // token='from' && (a = expression_rule(p)) // expression ) @@ -11964,7 +12059,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 588)) // token='not' + (_keyword = _PyPegen_expect_token(p, 679)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12618,9 +12713,9 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 588)) // token='not' + (_keyword = _PyPegen_expect_token(p, 679)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12666,7 +12761,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword = _PyPegen_expect_token(p, 671)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12715,7 +12810,7 @@ isnot_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 589)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 588)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 679)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13152,7 +13247,7 @@ bitwise_and_raw(Parser *p) } // Left-recursive -// shift_expr: shift_expr '<<' sum | shift_expr '>>' sum | sum +// shift_expr: shift_expr '<<' sum | shift_expr '>>' sum | invalid_arithmetic | sum static expr_ty shift_expr_raw(Parser *); static expr_ty shift_expr_rule(Parser *p) @@ -13287,6 +13382,25 @@ shift_expr_raw(Parser *p) D(fprintf(stderr, "%*c%s shift_expr[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "shift_expr '>>' sum")); } + if (p->call_invalid_rules) { // invalid_arithmetic + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> shift_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_arithmetic")); + void *invalid_arithmetic_var; + if ( + (invalid_arithmetic_var = invalid_arithmetic_rule(p)) // invalid_arithmetic + ) + { + D(fprintf(stderr, "%*c+ shift_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_arithmetic")); + _res = invalid_arithmetic_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s shift_expr[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_arithmetic")); + } { // sum if (p->error_indicator) { p->level--; @@ -13480,6 +13594,7 @@ sum_raw(Parser *p) // | term '//' factor // | term '%' factor // | term '@' factor +// | invalid_factor // | factor static expr_ty term_raw(Parser *); static expr_ty @@ -13732,6 +13847,25 @@ term_raw(Parser *p) D(fprintf(stderr, "%*c%s term[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "term '@' factor")); } + if (p->call_invalid_rules) { // invalid_factor + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> term[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_factor")); + void *invalid_factor_var; + if ( + (invalid_factor_var = invalid_factor_rule(p)) // invalid_factor + ) + { + D(fprintf(stderr, "%*c+ term[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_factor")); + _res = invalid_factor_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s term[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_factor")); + } { // factor if (p->error_indicator) { p->level--; @@ -14681,7 +14815,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, _tmp_93_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_93_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -15893,7 +16027,7 @@ fstring_middle_rule(Parser *p) } // fstring_replacement_field: -// | '{' (yield_expr | star_expressions) '='? fstring_conversion? fstring_full_format_spec? '}' +// | '{' annotated_rhs '='? fstring_conversion? fstring_full_format_spec? '}' // | invalid_replacement_field static expr_ty fstring_replacement_field_rule(Parser *p) @@ -15916,14 +16050,14 @@ fstring_replacement_field_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // '{' (yield_expr | star_expressions) '='? fstring_conversion? fstring_full_format_spec? '}' + { // '{' annotated_rhs '='? fstring_conversion? fstring_full_format_spec? '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> fstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? fstring_conversion? fstring_full_format_spec? '}'")); + D(fprintf(stderr, "%*c> fstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? fstring_conversion? fstring_full_format_spec? '}'")); Token * _literal; - void *a; + expr_ty a; void *conversion; void *debug_expr; void *format; @@ -15931,7 +16065,7 @@ fstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (a = _tmp_112_rule(p)) // yield_expr | star_expressions + (a = annotated_rhs_rule(p)) // annotated_rhs && (debug_expr = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -15942,7 +16076,7 @@ fstring_replacement_field_rule(Parser *p) (rbrace = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? fstring_conversion? fstring_full_format_spec? '}'")); + D(fprintf(stderr, "%*c+ fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? fstring_conversion? fstring_full_format_spec? '}'")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -15962,7 +16096,7 @@ fstring_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s fstring_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '='? fstring_conversion? fstring_full_format_spec? '}'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '='? fstring_conversion? fstring_full_format_spec? '}'")); } if (p->call_invalid_rules) { // invalid_replacement_field if (p->error_indicator) { @@ -16068,7 +16202,7 @@ fstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_113_rule(p)) // fstring_format_spec* + (spec = _loop0_112_rule(p)) // fstring_format_spec* ) { D(fprintf(stderr, "%*c+ fstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' fstring_format_spec*")); @@ -16186,7 +16320,7 @@ fstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' && - (b = _loop0_114_rule(p)) // fstring_middle* + (b = _loop0_113_rule(p)) // fstring_middle* && (c = _PyPegen_expect_token(p, FSTRING_END)) // token='FSTRING_END' ) @@ -16287,7 +16421,7 @@ strings_rule(Parser *p) D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_115_rule(p)) // ((fstring | string))+ + (a = (asdl_expr_seq*)_loop1_114_rule(p)) // ((fstring | string))+ ) { D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); @@ -16420,7 +16554,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_116_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_115_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -16635,7 +16769,7 @@ double_starred_kvpairs_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * a; if ( - (a = _gather_117_rule(p)) // ','.double_starred_kvpair+ + (a = _gather_116_rule(p)) // ','.double_starred_kvpair+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -16794,7 +16928,7 @@ for_if_clauses_rule(Parser *p) D(fprintf(stderr, "%*c> for_if_clauses[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); asdl_comprehension_seq* a; if ( - (a = (asdl_comprehension_seq*)_loop1_119_rule(p)) // for_if_clause+ + (a = (asdl_comprehension_seq*)_loop1_118_rule(p)) // for_if_clause+ ) { D(fprintf(stderr, "%*c+ for_if_clauses[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); @@ -16819,6 +16953,7 @@ for_if_clauses_rule(Parser *p) // for_if_clause: // | 'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))* // | 'for' star_targets 'in' ~ disjunction (('if' disjunction))* +// | invalid_for_if_clause // | invalid_for_target static comprehension_ty for_if_clause_rule(Parser *p) @@ -16846,19 +16981,19 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_120_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_119_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -16891,17 +17026,17 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_121_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_120_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -16921,6 +17056,25 @@ for_if_clause_rule(Parser *p) return NULL; } } + if (p->call_invalid_rules) { // invalid_for_if_clause + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_for_if_clause")); + void *invalid_for_if_clause_var; + if ( + (invalid_for_if_clause_var = invalid_for_if_clause_rule(p)) // invalid_for_if_clause + ) + { + D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_for_if_clause")); + _res = invalid_for_if_clause_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s for_if_clause[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_for_if_clause")); + } if (p->call_invalid_rules) { // invalid_for_target if (p->error_indicator) { p->level--; @@ -17161,7 +17315,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_122_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_121_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -17410,9 +17564,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_123_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_122_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_125_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_124_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -17502,11 +17656,11 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_126_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_125_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_128_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_127_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); @@ -17528,13 +17682,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_130_var; + asdl_seq * _gather_129_var; if ( - (_gather_130_var = _gather_130_rule(p)) // ','.kwarg_or_starred+ + (_gather_129_var = _gather_129_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_130_var; + _res = _gather_129_var; goto done; } p->mark = _mark; @@ -17547,13 +17701,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_132_var; + asdl_seq * _gather_131_var; if ( - (_gather_132_var = _gather_132_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_131_var = _gather_131_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_132_var; + _res = _gather_131_var; goto done; } p->mark = _mark; @@ -17566,7 +17720,10 @@ kwargs_rule(Parser *p) return _res; } -// starred_expression: invalid_starred_expression | '*' expression +// starred_expression: +// | invalid_starred_expression_unpacking +// | '*' expression +// | invalid_starred_expression static expr_ty starred_expression_rule(Parser *p) { @@ -17588,24 +17745,24 @@ starred_expression_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - if (p->call_invalid_rules) { // invalid_starred_expression + if (p->call_invalid_rules) { // invalid_starred_expression_unpacking if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); - void *invalid_starred_expression_var; + D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression_unpacking")); + void *invalid_starred_expression_unpacking_var; if ( - (invalid_starred_expression_var = invalid_starred_expression_rule(p)) // invalid_starred_expression + (invalid_starred_expression_unpacking_var = invalid_starred_expression_unpacking_rule(p)) // invalid_starred_expression_unpacking ) { - D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); - _res = invalid_starred_expression_var; + D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression_unpacking")); + _res = invalid_starred_expression_unpacking_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression_unpacking")); } { // '*' expression if (p->error_indicator) { @@ -17643,6 +17800,25 @@ starred_expression_rule(Parser *p) D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' expression")); } + if (p->call_invalid_rules) { // invalid_starred_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); + void *invalid_starred_expression_var; + if ( + (invalid_starred_expression_var = invalid_starred_expression_rule(p)) // invalid_starred_expression + ) + { + D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); + _res = invalid_starred_expression_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression")); + } _res = NULL; done: p->level--; @@ -17942,7 +18118,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_134_rule(p)) // ((',' star_target))* + (b = _loop0_133_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -17998,7 +18174,7 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_135_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_134_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18048,7 +18224,7 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_137_rule(p)) // ((',' star_target))+ + (b = _loop1_136_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18136,7 +18312,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_138_rule(p)) // !'*' star_target + (a = _tmp_137_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -18232,7 +18408,7 @@ target_with_star_atom_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -18276,7 +18452,7 @@ target_with_star_atom_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -18623,7 +18799,7 @@ single_subscript_attribute_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -18667,7 +18843,7 @@ single_subscript_attribute_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -18777,7 +18953,7 @@ t_primary_raw(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(1, t_lookahead_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME &t_lookahead")); @@ -18821,7 +18997,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(1, t_lookahead_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' &t_lookahead")); @@ -18859,7 +19035,7 @@ t_primary_raw(Parser *p) && (b = genexp_rule(p)) // genexp && - _PyPegen_lookahead(1, t_lookahead_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary genexp &t_lookahead")); @@ -18903,7 +19079,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - _PyPegen_lookahead(1, t_lookahead_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '(' arguments? ')' &t_lookahead")); @@ -18938,7 +19114,7 @@ t_primary_raw(Parser *p) if ( (a = atom_rule(p)) // atom && - _PyPegen_lookahead(1, t_lookahead_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "atom &t_lookahead")); @@ -19059,7 +19235,7 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_139_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_138_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -19128,7 +19304,7 @@ del_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19172,7 +19348,7 @@ del_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, t_lookahead_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19417,7 +19593,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_141_rule(p)) // ','.expression+ + (a = _gather_140_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19456,7 +19632,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_143_rule(p)) // ','.expression+ + (a = _gather_142_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19489,7 +19665,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_145_rule(p)) // ','.expression+ + (a = _gather_144_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19609,7 +19785,7 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_147_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_146_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); @@ -19660,7 +19836,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, _tmp_149_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -19721,7 +19897,7 @@ func_type_comment_rule(Parser *p) } // invalid_arguments: -// | ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*' +// | ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+ // | expression for_if_clauses ',' [args | expression for_if_clauses] // | NAME '=' expression for_if_clauses // | [(args ',')] NAME '=' &(',' | ')') @@ -19740,25 +19916,25 @@ invalid_arguments_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*' + { // ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+ if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); - Token * _literal; - void *_tmp_150_var; - Token * b; + D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); + asdl_seq * _gather_150_var; + void *_tmp_149_var; + Token * a; if ( - (_tmp_150_var = _tmp_150_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_149_var = _tmp_149_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (a = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _PyPegen_expect_token(p, 16)) // token='*' + (_gather_150_var = _gather_150_rule(p)) // ','.(starred_expression !'=')+ ) { - D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( b , "iterable argument unpacking follows keyword argument unpacking" ); + D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "iterable argument unpacking follows keyword argument unpacking" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -19768,7 +19944,7 @@ invalid_arguments_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_arguments[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); } { // expression for_if_clauses ',' [args | expression for_if_clauses] if (p->error_indicator) { @@ -19788,7 +19964,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_151_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -19848,13 +20024,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_153_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_154_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -19992,7 +20168,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_154_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_155_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20052,7 +20228,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, _tmp_155_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_156_rule, p) && (a = expression_rule(p)) // expression && @@ -20155,11 +20331,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 659)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 663)) // token='else' && (c = expression_rule(p)) // expression ) @@ -20283,6 +20459,91 @@ invalid_legacy_expression_rule(Parser *p) return _res; } +// invalid_type_param: '*' NAME ':' expression | '**' NAME ':' expression +static void * +invalid_type_param_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '*' NAME ':' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); + Token * _literal; + expr_ty a; + Token * colon; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + && + (a = _PyPegen_name_token(p)) // NAME + && + (colon = _PyPegen_expect_token(p, 11)) // token=':' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_param[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); + } + { // '**' NAME ':' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); + Token * _literal; + expr_ty a; + Token * colon; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 35)) // token='**' + && + (a = _PyPegen_name_token(p)) // NAME + && + (colon = _PyPegen_expect_token(p, 11)) // token=':' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_param[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_expression: // | !(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid // | disjunction 'if' disjunction !('else' | ':') @@ -20308,7 +20569,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, _tmp_156_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_157_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -20340,11 +20601,11 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, _tmp_157_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_158_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -20465,7 +20726,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_158_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_159_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20491,7 +20752,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_159_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_160_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20499,7 +20760,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_160_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_161_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20528,7 +20789,7 @@ invalid_named_expression_rule(Parser *p) // | expression ':' expression // | ((star_targets '='))* star_expressions '=' // | ((star_targets '='))* yield_expr '=' -// | star_expressions augassign (yield_expr | star_expressions) +// | star_expressions augassign annotated_rhs static void * invalid_assignment_rule(Parser *p) { @@ -20579,7 +20840,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_161_var; + asdl_seq * _loop0_162_var; expr_ty a; expr_ty expression_var; if ( @@ -20587,7 +20848,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_161_var = _loop0_161_rule(p)) // star_named_expressions* + (_loop0_162_var = _loop0_162_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20644,10 +20905,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_163_var; expr_ty a; if ( - (_loop0_162_var = _loop0_162_rule(p)) // ((star_targets '='))* + (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20674,10 +20935,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_164_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -20697,24 +20958,24 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_assignment[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((star_targets '='))* yield_expr '='")); } - { // star_expressions augassign (yield_expr | star_expressions) + { // star_expressions augassign annotated_rhs if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); - void *_tmp_164_var; + D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign annotated_rhs")); expr_ty a; + expr_ty annotated_rhs_var; AugOperator* augassign_var; if ( (a = star_expressions_rule(p)) // star_expressions && (augassign_var = augassign_rule(p)) // augassign && - (_tmp_164_var = _tmp_164_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs ) { - D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); + D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign annotated_rhs")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "'%s' is an illegal expression for augmented assignment" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -20725,7 +20986,7 @@ invalid_assignment_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_assignment[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions augassign annotated_rhs")); } _res = NULL; done: @@ -21333,7 +21594,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_178_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_178_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -22278,11 +22539,11 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_202_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_202_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22304,6 +22565,58 @@ invalid_with_item_rule(Parser *p) return _res; } +// invalid_for_if_clause: 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' +static void * +invalid_for_if_clause_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + Token * _keyword; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_tmp_203_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' + && + (_tmp_203_var = _tmp_203_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 671) // token='in' + ) + { + D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + _res = RAISE_SYNTAX_ERROR ( "'in' expected after for-loop variables" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_for_if_clause[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_for_target: 'async'? 'for' star_expressions static void * invalid_for_target_rule(Parser *p) @@ -22328,9 +22641,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -22436,7 +22749,7 @@ invalid_group_rule(Parser *p) return _res; } -// invalid_import: 'import' ','.dotted_name+ 'from' dotted_name +// invalid_import: 'import' ','.dotted_name+ 'from' dotted_name | 'import' NEWLINE static void * invalid_import_rule(Parser *p) { @@ -22455,16 +22768,16 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_203_var; + asdl_seq * _gather_204_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( - (a = _PyPegen_expect_token(p, 617)) // token='import' + (a = _PyPegen_expect_token(p, 622)) // token='import' && - (_gather_203_var = _gather_203_rule(p)) // ','.dotted_name+ + (_gather_204_var = _gather_204_rule(p)) // ','.dotted_name+ && - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (dotted_name_var = dotted_name_rule(p)) // dotted_name ) @@ -22482,13 +22795,40 @@ invalid_import_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_import[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); } + { // 'import' NEWLINE + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' NEWLINE")); + Token * _keyword; + Token * token; + if ( + (_keyword = _PyPegen_expect_token(p, 622)) // token='import' + && + (token = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + ) + { + D(fprintf(stderr, "%*c+ invalid_import[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' NEWLINE")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Expected one or more names after 'import'" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_import[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import' NEWLINE")); + } _res = NULL; done: p->level--; return _res; } -// invalid_import_from_targets: import_from_as_names ',' NEWLINE +// invalid_import_from_targets: import_from_as_names ',' NEWLINE | NEWLINE static void * invalid_import_from_targets_rule(Parser *p) { @@ -22531,6 +22871,30 @@ invalid_import_from_targets_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_import_from_targets[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_names ',' NEWLINE")); } + { // NEWLINE + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_import_from_targets[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + Token * token; + if ( + (token = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + ) + { + D(fprintf(stderr, "%*c+ invalid_import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Expected one or more names after 'import'" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_import_from_targets[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); + } _res = NULL; done: p->level--; @@ -22558,17 +22922,17 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_205_var; + asdl_seq * _gather_206_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && - (_gather_205_var = _gather_205_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_206_var = _gather_206_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22592,7 +22956,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_207_var; + asdl_seq * _gather_208_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22602,13 +22966,13 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_207_var = _gather_207_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_208_var = _gather_208_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22657,18 +23021,18 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_209_var; + asdl_seq * _gather_210_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 629)) // token='with' + (a = _PyPegen_expect_token(p, 633)) // token='with' && - (_gather_209_var = _gather_209_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_210_var = _gather_210_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22696,7 +23060,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_211_var; + asdl_seq * _gather_212_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22707,13 +23071,13 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 629)) // token='with' + (a = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_211_var = _gather_211_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_212_var = _gather_212_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22772,7 +23136,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 638)) // token='try' + (a = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22804,13 +23168,13 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_213_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_214_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -22835,29 +23199,29 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_214_var; - asdl_seq * _loop1_215_var; + asdl_seq * _loop0_215_var; + asdl_seq * _loop1_216_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_214_var = _loop0_214_rule(p)) // block* + (_loop0_215_var = _loop0_215_rule(p)) // block* && - (_loop1_215_var = _loop1_215_rule(p)) // except_block+ + (_loop1_216_var = _loop1_216_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_217_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22884,23 +23248,23 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_217_var; - asdl_seq * _loop1_218_var; + asdl_seq * _loop0_218_var; + asdl_seq * _loop1_219_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_217_var = _loop0_217_rule(p)) // block* + (_loop0_218_var = _loop0_218_rule(p)) // block* && - (_loop1_218_var = _loop1_218_rule(p)) // except_star_block+ + (_loop1_219_var = _loop1_219_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && - (_opt_var = _tmp_219_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_220_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22957,7 +23321,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty a; expr_ty expressions_var; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && @@ -22967,7 +23331,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_220_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22999,13 +23363,13 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23032,7 +23396,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23057,14 +23421,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_222_var; + void *_tmp_223_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_222_var = _tmp_222_rule(p)) // NEWLINE | ':' + (_tmp_223_var = _tmp_223_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23109,7 +23473,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 647)) // token='finally' + (a = _PyPegen_expect_token(p, 651)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23165,11 +23529,11 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23201,7 +23565,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23257,13 +23621,13 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 651)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_225_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23496,7 +23860,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -23526,7 +23890,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) && @@ -23627,7 +23991,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_226_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -23680,7 +24044,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23711,7 +24075,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 656)) // token='if' + (a = _PyPegen_expect_token(p, 660)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -23766,7 +24130,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 658)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23797,7 +24161,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 658)) // token='elif' + (a = _PyPegen_expect_token(p, 662)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23850,7 +24214,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 659)) // token='else' + (a = _PyPegen_expect_token(p, 663)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23903,7 +24267,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 661)) // token='while' + (_keyword = _PyPegen_expect_token(p, 665)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23934,7 +24298,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 661)) // token='while' + (a = _PyPegen_expect_token(p, 665)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23993,13 +24357,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24034,13 +24398,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 666)) // token='for' + (a = _PyPegen_expect_token(p, 670)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 667)) // token='in' + (_keyword = _PyPegen_expect_token(p, 671)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24071,7 +24435,8 @@ invalid_for_stmt_rule(Parser *p) } // invalid_def_raw: -// | 'async'? 'def' NAME '(' params? ')' ['->' expression] ':' NEWLINE !INDENT +// | 'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT +// | 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block static void * invalid_def_raw_rule(Parser *p) { @@ -24084,12 +24449,12 @@ invalid_def_raw_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'async'? 'def' NAME '(' params? ')' ['->' expression] ':' NEWLINE !INDENT + { // 'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); + D(fprintf(stderr, "%*c> invalid_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); Token * _literal; Token * _literal_1; Token * _literal_2; @@ -24099,23 +24464,27 @@ invalid_def_raw_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings void *_opt_var_2; UNUSED(_opt_var_2); // Silence compiler warnings + void *_opt_var_3; + UNUSED(_opt_var_3); // Silence compiler warnings Token * a; expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 668), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 669)) // token='def' + (a = _PyPegen_expect_token(p, 675)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && + (_opt_var_1 = type_params_rule(p), !p->error_indicator) // type_params? + && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_opt_var_1 = params_rule(p), !p->error_indicator) // params? + (_opt_var_2 = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_226_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_227_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24124,7 +24493,7 @@ invalid_def_raw_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, INDENT) // token=INDENT ) { - D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); + D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after function definition on line %d" , a -> lineno ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24135,7 +24504,61 @@ invalid_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); + } + { // 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + Token * _literal_2; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings + void *_opt_var_2; + UNUSED(_opt_var_2); // Silence compiler warnings + void *_opt_var_3; + UNUSED(_opt_var_3); // Silence compiler warnings + void *_opt_var_4; + UNUSED(_opt_var_4); // Silence compiler warnings + asdl_stmt_seq* block_var; + expr_ty name_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' + && + (name_var = _PyPegen_name_token(p)) // NAME + && + (_opt_var_1 = type_params_rule(p), !p->error_indicator) // type_params? + && + (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + && + (_opt_var_2 = params_rule(p), !p->error_indicator) // params? + && + (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' + && + (_opt_var_3 = _tmp_228_rule(p), !p->error_indicator) // ['->' expression] + && + (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + && + (_opt_var_4 = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? + && + (block_var = block_rule(p)) // block + ) + { + D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + _res = _PyPegen_dummy_name(p, _opt_var, _keyword, name_var, _opt_var_1, _literal, _opt_var_2, _literal_1, _opt_var_3, _literal_2, _opt_var_4, block_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); } _res = NULL; done: @@ -24144,8 +24567,8 @@ invalid_def_raw_rule(Parser *p) } // invalid_class_def_raw: -// | 'class' NAME ['(' arguments? ')'] NEWLINE -// | 'class' NAME ['(' arguments? ')'] ':' NEWLINE !INDENT +// | 'class' NAME type_params? ['(' arguments? ')'] NEWLINE +// | 'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT static void * invalid_class_def_raw_rule(Parser *p) { @@ -24158,28 +24581,32 @@ invalid_class_def_raw_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'class' NAME ['(' arguments? ')'] NEWLINE + { // 'class' NAME type_params? ['(' arguments? ')'] NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_class_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class' NAME ['(' arguments? ')'] NEWLINE")); + D(fprintf(stderr, "%*c> invalid_class_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] NEWLINE")); Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? + && + (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME ['(' arguments? ')'] NEWLINE")); + D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24190,26 +24617,30 @@ invalid_class_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_class_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'class' NAME ['(' arguments? ')'] NEWLINE")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] NEWLINE")); } - { // 'class' NAME ['(' arguments? ')'] ':' NEWLINE !INDENT + { // 'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_class_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class' NAME ['(' arguments? ')'] ':' NEWLINE !INDENT")); + D(fprintf(stderr, "%*c> invalid_class_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT")); Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings Token * a; expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 671)) // token='class' + (a = _PyPegen_expect_token(p, 677)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? + && + (_opt_var_1 = _tmp_230_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24218,7 +24649,7 @@ invalid_class_def_raw_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, INDENT) // token=INDENT ) { - D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME ['(' arguments? ')'] ':' NEWLINE !INDENT")); + D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after class definition on line %d" , a -> lineno ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24229,7 +24660,7 @@ invalid_class_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_class_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'class' NAME ['(' arguments? ')'] ':' NEWLINE !INDENT")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT")); } _res = NULL; done: @@ -24259,11 +24690,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_229_var; + asdl_seq * _gather_231_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_229_var = _gather_229_rule(p)) // ','.double_starred_kvpair+ + (_gather_231_var = _gather_231_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24271,7 +24702,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_229_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_231_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24324,7 +24755,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_231_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_233_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24434,7 +24865,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_232_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24456,9 +24887,9 @@ invalid_kvpair_rule(Parser *p) return _res; } -// invalid_starred_expression: '*' expression '=' expression +// invalid_starred_expression_unpacking: '*' expression '=' expression static void * -invalid_starred_expression_rule(Parser *p) +invalid_starred_expression_unpacking_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -24474,7 +24905,7 @@ invalid_starred_expression_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); + D(fprintf(stderr, "%*c> invalid_starred_expression_unpacking[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); Token * _literal; Token * a; expr_ty b; @@ -24489,7 +24920,7 @@ invalid_starred_expression_rule(Parser *p) (b = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); + D(fprintf(stderr, "%*c+ invalid_starred_expression_unpacking[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to iterable argument unpacking" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24499,7 +24930,7 @@ invalid_starred_expression_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s invalid_starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s invalid_starred_expression_unpacking[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' expression '=' expression")); } _res = NULL; @@ -24508,18 +24939,61 @@ invalid_starred_expression_rule(Parser *p) return _res; } +// invalid_starred_expression: '*' +static void * +invalid_starred_expression_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ invalid_starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = RAISE_SYNTAX_ERROR ( "Invalid star expression" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_replacement_field: // | '{' '=' // | '{' '!' // | '{' ':' // | '{' '}' -// | '{' !(yield_expr | star_expressions) -// | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') -// | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') -// | '{' (yield_expr | star_expressions) '='? invalid_conversion_character -// | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') -// | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' -// | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' +// | '{' !annotated_rhs +// | '{' annotated_rhs !('=' | '!' | ':' | '}') +// | '{' annotated_rhs '=' !('!' | ':' | '}') +// | '{' annotated_rhs '='? invalid_conversion_character +// | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') +// | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' +// | '{' annotated_rhs '='? ['!' NAME] !'}' static void * invalid_replacement_field_rule(Parser *p) { @@ -24640,20 +25114,20 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' '}'")); } - { // '{' !(yield_expr | star_expressions) + { // '{' !annotated_rhs if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_233_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting a valid expression after '{'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24664,25 +25138,25 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' !(yield_expr | star_expressions)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' !annotated_rhs")); } - { // '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') + { // '{' annotated_rhs !('=' | '!' | ':' | '}') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_234_var; + expr_ty annotated_rhs_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_234_var = _tmp_234_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, _tmp_235_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_235_rule, p) ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '=', or '!', or ':', or '}'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24693,28 +25167,28 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); } - { // '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') + { // '{' annotated_rhs '=' !('!' | ':' | '}') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_236_var; + expr_ty annotated_rhs_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_236_var = _tmp_236_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_237_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_236_rule, p) ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '!', or ':', or '}'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24725,62 +25199,62 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); } - { // '{' (yield_expr | star_expressions) '='? invalid_conversion_character + { // '{' annotated_rhs '='? invalid_conversion_character if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? invalid_conversion_character")); Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_238_var; + expr_ty annotated_rhs_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_238_var = _tmp_238_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && (invalid_conversion_character_var = invalid_conversion_character_rule(p)) // invalid_conversion_character ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var, _opt_var, invalid_conversion_character_var); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? invalid_conversion_character")); + _res = _PyPegen_dummy_name(p, _literal, annotated_rhs_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '='? invalid_conversion_character")); } - { // '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') + { // '{' annotated_rhs '='? ['!' NAME] !(':' | '}') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_239_var; + expr_ty annotated_rhs_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_237_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_241_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_238_rule, p) ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting ':' or '}'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24791,39 +25265,39 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); } - { // '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' + { // '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_244_var; + asdl_seq * _loop0_240_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_242_var; + expr_ty annotated_rhs_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_244_var = _loop0_244_rule(p)) // fstring_format_spec* + (_loop0_240_var = _loop0_240_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '}', or format specs" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24834,33 +25308,33 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); } - { // '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' + { // '{' annotated_rhs '='? ['!' NAME] !'}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !'}'")); + D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !'}'")); Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_245_var; + expr_ty annotated_rhs_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions + (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) { - D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !'}'")); + D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '}'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24871,7 +25345,7 @@ invalid_replacement_field_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_replacement_field[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !'}'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !'}'")); } _res = NULL; done: @@ -24902,7 +25376,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_247_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_242_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -24950,6 +25424,153 @@ invalid_conversion_character_rule(Parser *p) return _res; } +// invalid_arithmetic: sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion +static void * +invalid_arithmetic_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); + void *_tmp_243_var; + Token * a; + expr_ty b; + expr_ty sum_var; + if ( + (sum_var = sum_rule(p)) // sum + && + (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + && + (a = _PyPegen_expect_token(p, 679)) // token='not' + && + (b = inversion_rule(p)) // inversion + ) + { + D(fprintf(stderr, "%*c+ invalid_arithmetic[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "'not' after an operator must be parenthesized" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_arithmetic[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// invalid_factor: ('+' | '-' | '~') 'not' factor +static void * +invalid_factor_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ('+' | '-' | '~') 'not' factor + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); + void *_tmp_244_var; + Token * a; + expr_ty b; + if ( + (_tmp_244_var = _tmp_244_rule(p)) // '+' | '-' | '~' + && + (a = _PyPegen_expect_token(p, 679)) // token='not' + && + (b = factor_rule(p)) // factor + ) + { + D(fprintf(stderr, "%*c+ invalid_factor[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "'not' after an operator must be parenthesized" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_factor[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('+' | '-' | '~') 'not' factor")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// invalid_type_params: '[' ']' +static void * +invalid_type_params_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '[' ']' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + Token * _literal; + Token * token; + if ( + (_literal = _PyPegen_expect_token(p, 9)) // token='[' + && + (token = _PyPegen_expect_token(p, 10)) // token=']' + ) + { + D(fprintf(stderr, "%*c+ invalid_type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Type parameter list cannot be empty" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'[' ']'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // _loop0_1: NEWLINE static asdl_seq * _loop0_1_rule(Parser *p) @@ -25011,7 +25632,7 @@ _loop0_1_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -25078,7 +25699,7 @@ _loop0_2_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -25150,7 +25771,7 @@ _loop1_3_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -25226,7 +25847,7 @@ _loop0_5_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -25294,7 +25915,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 617)) // token='import' + (_keyword = _PyPegen_expect_token(p, 622)) // token='import' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); @@ -25313,7 +25934,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); @@ -25351,7 +25972,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 669)) // token='def' + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -25389,7 +26010,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25427,7 +26048,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -25484,7 +26105,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -25503,7 +26124,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25541,7 +26162,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -25560,7 +26181,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25765,12 +26386,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_248_var; + void *_tmp_245_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // star_targets '=' + (_tmp_245_var = _tmp_245_rule(p)) // star_targets '=' ) { - _res = _tmp_248_var; + _res = _tmp_245_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -25803,7 +26424,7 @@ _loop1_14_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -25945,7 +26566,7 @@ _tmp_17_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='from' + (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && (z = expression_rule(p)) // expression ) @@ -26039,7 +26660,7 @@ _loop0_19_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26156,7 +26777,7 @@ _loop0_21_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26334,12 +26955,12 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_249_var; + void *_tmp_246_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' + (_tmp_246_var = _tmp_246_rule(p)) // '.' | '...' ) { - _res = _tmp_249_var; + _res = _tmp_246_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26367,7 +26988,7 @@ _loop0_24_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26401,12 +27022,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_250_var; + void *_tmp_247_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // '.' | '...' + (_tmp_247_var = _tmp_247_rule(p)) // '.' | '...' ) { - _res = _tmp_250_var; + _res = _tmp_247_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26439,7 +27060,7 @@ _loop1_25_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26515,7 +27136,7 @@ _loop0_27_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26584,7 +27205,7 @@ _tmp_28_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -26678,7 +27299,7 @@ _loop0_30_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -26747,7 +27368,7 @@ _tmp_31_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -26799,12 +27420,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_251_var; + void *_tmp_248_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // '@' named_expression NEWLINE + (_tmp_248_var = _tmp_248_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_251_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26837,7 +27458,7 @@ _loop1_32_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27045,7 +27666,7 @@ _loop0_36_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27112,7 +27733,7 @@ _loop0_37_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27179,7 +27800,7 @@ _loop0_38_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27251,7 +27872,7 @@ _loop1_39_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27318,7 +27939,7 @@ _loop0_40_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27390,7 +28011,7 @@ _loop1_41_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27462,7 +28083,7 @@ _loop1_42_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27534,7 +28155,7 @@ _loop1_43_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27601,7 +28222,7 @@ _loop0_44_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27673,7 +28294,7 @@ _loop1_45_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27740,7 +28361,7 @@ _loop0_46_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27812,7 +28433,7 @@ _loop1_47_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27879,7 +28500,7 @@ _loop0_48_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -27946,7 +28567,7 @@ _loop0_49_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28018,7 +28639,7 @@ _loop1_50_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28094,7 +28715,7 @@ _loop0_52_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28211,7 +28832,7 @@ _loop0_54_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28328,7 +28949,7 @@ _loop0_56_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28445,7 +29066,7 @@ _loop0_58_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28634,7 +29255,7 @@ _loop1_60_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28706,7 +29327,7 @@ _loop1_61_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28734,7 +29355,7 @@ _tmp_62_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -28780,7 +29401,7 @@ _tmp_63_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -28870,7 +29491,7 @@ _loop1_64_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -28946,7 +29567,7 @@ _loop0_66_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29329,7 +29950,7 @@ _loop0_72_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29446,7 +30067,7 @@ _loop0_74_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29620,7 +30241,7 @@ _loop0_77_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29737,7 +30358,7 @@ _loop0_79_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29854,7 +30475,7 @@ _loop0_81_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -29929,12 +30550,12 @@ _loop1_82_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_252_var; + void *_tmp_249_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // ',' expression + (_tmp_249_var = _tmp_249_rule(p)) // ',' expression ) { - _res = _tmp_252_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29967,7 +30588,7 @@ _loop1_82_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30001,12 +30622,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_253_var; + void *_tmp_250_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // ',' star_expression + (_tmp_250_var = _tmp_250_rule(p)) // ',' star_expression ) { - _res = _tmp_253_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30039,7 +30660,7 @@ _loop1_83_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30115,7 +30736,7 @@ _loop0_85_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30190,12 +30811,12 @@ _loop1_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_254_var; + void *_tmp_251_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'or' conjunction + (_tmp_251_var = _tmp_251_rule(p)) // 'or' conjunction ) { - _res = _tmp_254_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30228,7 +30849,7 @@ _loop1_86_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30262,12 +30883,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_255_var; + void *_tmp_252_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'and' inversion + (_tmp_252_var = _tmp_252_rule(p)) // 'and' inversion ) { - _res = _tmp_255_var; + _res = _tmp_252_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30300,7 +30921,7 @@ _loop1_87_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30372,7 +30993,7 @@ _loop1_88_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30454,7 +31075,7 @@ _loop0_91_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30491,7 +31112,7 @@ _loop0_91_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -30519,7 +31140,7 @@ _gather_90_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_256_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression && (seq = _loop0_91_rule(p)) // _loop0_91 ) @@ -30987,7 +31608,7 @@ _loop0_98_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31054,7 +31675,7 @@ _loop0_99_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31121,7 +31742,7 @@ _loop0_100_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31193,7 +31814,7 @@ _loop1_101_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31260,7 +31881,7 @@ _loop0_102_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31332,7 +31953,7 @@ _loop1_103_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31404,7 +32025,7 @@ _loop1_104_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31476,7 +32097,7 @@ _loop1_105_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31543,7 +32164,7 @@ _loop0_106_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31615,7 +32236,7 @@ _loop1_107_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31682,7 +32303,7 @@ _loop0_108_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31754,7 +32375,7 @@ _loop1_109_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31821,7 +32442,7 @@ _loop0_110_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -31893,15 +32514,15 @@ _loop1_111_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_112: yield_expr | star_expressions -static void * -_tmp_112_rule(Parser *p) +// _loop0_112: fstring_format_spec +static asdl_seq * +_loop0_112_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31910,45 +32531,243 @@ _tmp_112_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // yield_expr + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // fstring_format_spec if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr + D(fprintf(stderr, "%*c> _loop0_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + expr_ty fstring_format_spec_var; + while ( + (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec ) { - D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; + _res = fstring_format_spec_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c%s _loop0_112[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } - { // star_expressions + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop0_113: fstring_middle +static asdl_seq * +_loop0_113_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // fstring_middle if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; + D(fprintf(stderr, "%*c> _loop0_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); + expr_ty fstring_middle_var; + while ( + (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle + ) + { + _res = fstring_middle_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_113[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_114: (fstring | string) +static asdl_seq * +_loop1_114_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_254_var; + while ( + (_tmp_254_var = _tmp_254_rule(p)) // fstring | string + ) + { + _res = _tmp_254_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_115: star_named_expression ',' star_named_expressions? +static void * +_tmp_115_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // star_named_expression ',' star_named_expressions? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + Token * _literal; + expr_ty y; + void *z; if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions + (y = star_named_expression_rule(p)) // star_named_expression + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { - D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; + D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + _res = _PyPegen_seq_insert_in_front ( p , y , z ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); } _res = NULL; done: @@ -31956,9 +32775,9 @@ _tmp_112_rule(Parser *p) return _res; } -// _loop0_113: fstring_format_spec +// _loop0_117: ',' double_starred_kvpair static asdl_seq * -_loop0_113_rule(Parser *p) +_loop0_117_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31978,18 +32797,27 @@ _loop0_113_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // fstring_format_spec + { // ',' double_starred_kvpair if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); - expr_ty fstring_format_spec_var; + D(fprintf(stderr, "%*c> _loop0_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + Token * _literal; + KeyValuePair* elem; while ( - (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair ) { - _res = fstring_format_spec_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32006,8 +32834,8 @@ _loop0_113_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_113[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c%s _loop0_117[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -32017,279 +32845,15 @@ _loop0_113_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_114: fstring_middle +// _gather_116: double_starred_kvpair _loop0_117 static asdl_seq * -_loop0_114_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // fstring_middle - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); - expr_ty fstring_middle_var; - while ( - (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle - ) - { - _res = fstring_middle_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_114[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_115: (fstring | string) -static asdl_seq * -_loop1_115_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (fstring | string) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_257_var; - while ( - (_tmp_257_var = _tmp_257_rule(p)) // fstring | string - ) - { - _res = _tmp_257_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_115[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_116: star_named_expression ',' star_named_expressions? -static void * -_tmp_116_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // star_named_expression ',' star_named_expressions? - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - Token * _literal; - expr_ty y; - void *z; - if ( - (y = star_named_expression_rule(p)) // star_named_expression - && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? - ) - { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - _res = _PyPegen_seq_insert_in_front ( p , y , z ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_118: ',' double_starred_kvpair -static asdl_seq * -_loop0_118_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' double_starred_kvpair - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); - Token * _literal; - KeyValuePair* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_118[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_117: double_starred_kvpair _loop0_118 -static asdl_seq * -_gather_117_rule(Parser *p) +_gather_116_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32300,27 +32864,27 @@ _gather_117_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_118 + { // double_starred_kvpair _loop0_117 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_118")); + D(fprintf(stderr, "%*c> _gather_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_117")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_118_rule(p)) // _loop0_118 + (seq = _loop0_117_rule(p)) // _loop0_117 ) { - D(fprintf(stderr, "%*c+ _gather_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_118")); + D(fprintf(stderr, "%*c+ _gather_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_117")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_117[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_118")); + D(fprintf(stderr, "%*c%s _gather_116[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_117")); } _res = NULL; done: @@ -32328,9 +32892,9 @@ _gather_117_rule(Parser *p) return _res; } -// _loop1_119: for_if_clause +// _loop1_118: for_if_clause static asdl_seq * -_loop1_119_rule(Parser *p) +_loop1_118_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32355,7 +32919,7 @@ _loop1_119_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); + D(fprintf(stderr, "%*c> _loop1_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); comprehension_ty for_if_clause_var; while ( (for_if_clause_var = for_if_clause_rule(p)) // for_if_clause @@ -32378,7 +32942,7 @@ _loop1_119_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_119[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_118[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "for_if_clause")); } if (_n == 0 || p->error_indicator) { @@ -32394,15 +32958,15 @@ _loop1_119_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_120: ('if' disjunction) +// _loop0_119: ('if' disjunction) static asdl_seq * -_loop0_120_rule(Parser *p) +_loop0_119_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32427,13 +32991,13 @@ _loop0_120_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_258_var; + D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); + void *_tmp_255_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction + (_tmp_255_var = _tmp_255_rule(p)) // 'if' disjunction ) { - _res = _tmp_258_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32450,7 +33014,7 @@ _loop0_120_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32461,15 +33025,15 @@ _loop0_120_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_121: ('if' disjunction) +// _loop0_120: ('if' disjunction) static asdl_seq * -_loop0_121_rule(Parser *p) +_loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32494,13 +33058,13 @@ _loop0_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_259_var; + D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); + void *_tmp_256_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // 'if' disjunction + (_tmp_256_var = _tmp_256_rule(p)) // 'if' disjunction ) { - _res = _tmp_259_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32517,7 +33081,7 @@ _loop0_121_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32528,15 +33092,15 @@ _loop0_121_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_122: assignment_expression | expression !':=' +// _tmp_121: assignment_expression | expression !':=' static void * -_tmp_122_rule(Parser *p) +_tmp_121_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32552,18 +33116,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -32571,7 +33135,7 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -32579,12 +33143,12 @@ _tmp_122_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -32593,9 +33157,9 @@ _tmp_122_rule(Parser *p) return _res; } -// _loop0_124: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_123: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_124_rule(Parser *p) +_loop0_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32620,13 +33184,13 @@ _loop0_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32652,7 +33216,7 @@ _loop0_124_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32663,16 +33227,16 @@ _loop0_124_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_123: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 +// _gather_122: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 static asdl_seq * -_gather_123_rule(Parser *p) +_gather_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32683,27 +33247,27 @@ _gather_123_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c> _gather_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_260_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_124_rule(p)) // _loop0_124 + (seq = _loop0_123_rule(p)) // _loop0_123 ) { - D(fprintf(stderr, "%*c+ _gather_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c+ _gather_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_123[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c%s _gather_122[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); } _res = NULL; done: @@ -32711,9 +33275,9 @@ _gather_123_rule(Parser *p) return _res; } -// _tmp_125: ',' kwargs +// _tmp_124: ',' kwargs static void * -_tmp_125_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32729,7 +33293,7 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); Token * _literal; asdl_seq* k; if ( @@ -32738,7 +33302,7 @@ _tmp_125_rule(Parser *p) (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); _res = k; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -32748,7 +33312,7 @@ _tmp_125_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; @@ -32757,9 +33321,9 @@ _tmp_125_rule(Parser *p) return _res; } -// _loop0_127: ',' kwarg_or_starred +// _loop0_126: ',' kwarg_or_starred static asdl_seq * -_loop0_127_rule(Parser *p) +_loop0_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32784,7 +33348,7 @@ _loop0_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32816,7 +33380,7 @@ _loop0_127_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32827,15 +33391,15 @@ _loop0_127_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_126: kwarg_or_starred _loop0_127 +// _gather_125: kwarg_or_starred _loop0_126 static asdl_seq * -_gather_126_rule(Parser *p) +_gather_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32846,27 +33410,27 @@ _gather_126_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_127 + { // kwarg_or_starred _loop0_126 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c> _gather_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_127_rule(p)) // _loop0_127 + (seq = _loop0_126_rule(p)) // _loop0_126 ) { - D(fprintf(stderr, "%*c+ _gather_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c+ _gather_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_126[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c%s _gather_125[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_126")); } _res = NULL; done: @@ -32874,9 +33438,9 @@ _gather_126_rule(Parser *p) return _res; } -// _loop0_129: ',' kwarg_or_double_starred +// _loop0_128: ',' kwarg_or_double_starred static asdl_seq * -_loop0_129_rule(Parser *p) +_loop0_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32901,7 +33465,7 @@ _loop0_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32933,7 +33497,7 @@ _loop0_129_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32944,15 +33508,15 @@ _loop0_129_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_128: kwarg_or_double_starred _loop0_129 +// _gather_127: kwarg_or_double_starred _loop0_128 static asdl_seq * -_gather_128_rule(Parser *p) +_gather_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32963,27 +33527,27 @@ _gather_128_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_129 + { // kwarg_or_double_starred _loop0_128 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c> _gather_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_129_rule(p)) // _loop0_129 + (seq = _loop0_128_rule(p)) // _loop0_128 ) { - D(fprintf(stderr, "%*c+ _gather_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c+ _gather_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_128[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c%s _gather_127[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_128")); } _res = NULL; done: @@ -32991,9 +33555,9 @@ _gather_128_rule(Parser *p) return _res; } -// _loop0_131: ',' kwarg_or_starred +// _loop0_130: ',' kwarg_or_starred static asdl_seq * -_loop0_131_rule(Parser *p) +_loop0_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33018,7 +33582,7 @@ _loop0_131_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33050,7 +33614,7 @@ _loop0_131_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_131[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33061,15 +33625,15 @@ _loop0_131_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_130: kwarg_or_starred _loop0_131 +// _gather_129: kwarg_or_starred _loop0_130 static asdl_seq * -_gather_130_rule(Parser *p) +_gather_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33080,27 +33644,27 @@ _gather_130_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_131 + { // kwarg_or_starred _loop0_130 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c> _gather_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_131_rule(p)) // _loop0_131 + (seq = _loop0_130_rule(p)) // _loop0_130 ) { - D(fprintf(stderr, "%*c+ _gather_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c+ _gather_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_130[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c%s _gather_129[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_130")); } _res = NULL; done: @@ -33108,9 +33672,9 @@ _gather_130_rule(Parser *p) return _res; } -// _loop0_133: ',' kwarg_or_double_starred +// _loop0_132: ',' kwarg_or_double_starred static asdl_seq * -_loop0_133_rule(Parser *p) +_loop0_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33135,7 +33699,7 @@ _loop0_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33167,7 +33731,7 @@ _loop0_133_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_132[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33178,15 +33742,15 @@ _loop0_133_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_132: kwarg_or_double_starred _loop0_133 +// _gather_131: kwarg_or_double_starred _loop0_132 static asdl_seq * -_gather_132_rule(Parser *p) +_gather_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33197,27 +33761,27 @@ _gather_132_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_133 + { // kwarg_or_double_starred _loop0_132 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_133_rule(p)) // _loop0_133 + (seq = _loop0_132_rule(p)) // _loop0_132 ) { - D(fprintf(stderr, "%*c+ _gather_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_132[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_132")); } _res = NULL; done: @@ -33225,9 +33789,9 @@ _gather_132_rule(Parser *p) return _res; } -// _loop0_134: (',' star_target) +// _loop0_133: (',' star_target) static asdl_seq * -_loop0_134_rule(Parser *p) +_loop0_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33252,13 +33816,13 @@ _loop0_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_261_var; + D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_258_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target + (_tmp_258_var = _tmp_258_rule(p)) // ',' star_target ) { - _res = _tmp_261_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33275,7 +33839,7 @@ _loop0_134_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33286,15 +33850,15 @@ _loop0_134_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_136: ',' star_target +// _loop0_135: ',' star_target static asdl_seq * -_loop0_136_rule(Parser *p) +_loop0_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33319,7 +33883,7 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _loop0_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty elem; while ( @@ -33351,7 +33915,7 @@ _loop0_136_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33362,15 +33926,15 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_135: star_target _loop0_136 +// _gather_134: star_target _loop0_135 static asdl_seq * -_gather_135_rule(Parser *p) +_gather_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33381,27 +33945,27 @@ _gather_135_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_target _loop0_136 + { // star_target _loop0_135 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c> _gather_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) // star_target && - (seq = _loop0_136_rule(p)) // _loop0_136 + (seq = _loop0_135_rule(p)) // _loop0_135 ) { - D(fprintf(stderr, "%*c+ _gather_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c+ _gather_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_135[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c%s _gather_134[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_135")); } _res = NULL; done: @@ -33409,9 +33973,9 @@ _gather_135_rule(Parser *p) return _res; } -// _loop1_137: (',' star_target) +// _loop1_136: (',' star_target) static asdl_seq * -_loop1_137_rule(Parser *p) +_loop1_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33436,13 +34000,13 @@ _loop1_137_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_262_var; + D(fprintf(stderr, "%*c> _loop1_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_259_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // ',' star_target + (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target ) { - _res = _tmp_262_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33459,7 +34023,7 @@ _loop1_137_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_137[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_136[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { @@ -33475,15 +34039,15 @@ _loop1_137_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_138: !'*' star_target +// _tmp_137: !'*' star_target static void * -_tmp_138_rule(Parser *p) +_tmp_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33499,7 +34063,7 @@ _tmp_138_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c> _tmp_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); expr_ty star_target_var; if ( _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' @@ -33507,12 +34071,12 @@ _tmp_138_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c+ _tmp_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); _res = star_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_137[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } _res = NULL; @@ -33521,9 +34085,9 @@ _tmp_138_rule(Parser *p) return _res; } -// _loop0_140: ',' del_target +// _loop0_139: ',' del_target static asdl_seq * -_loop0_140_rule(Parser *p) +_loop0_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33548,7 +34112,7 @@ _loop0_140_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); Token * _literal; expr_ty elem; while ( @@ -33580,7 +34144,7 @@ _loop0_140_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_140[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33591,15 +34155,15 @@ _loop0_140_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_139: del_target _loop0_140 +// _gather_138: del_target _loop0_139 static asdl_seq * -_gather_139_rule(Parser *p) +_gather_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33610,27 +34174,27 @@ _gather_139_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // del_target _loop0_140 + { // del_target _loop0_139 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c> _gather_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) // del_target && - (seq = _loop0_140_rule(p)) // _loop0_140 + (seq = _loop0_139_rule(p)) // _loop0_139 ) { - D(fprintf(stderr, "%*c+ _gather_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c+ _gather_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_139[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c%s _gather_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_139")); } _res = NULL; done: @@ -33638,9 +34202,9 @@ _gather_139_rule(Parser *p) return _res; } -// _loop0_142: ',' expression +// _loop0_141: ',' expression static asdl_seq * -_loop0_142_rule(Parser *p) +_loop0_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33665,7 +34229,7 @@ _loop0_142_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33697,7 +34261,7 @@ _loop0_142_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_142[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33708,15 +34272,15 @@ _loop0_142_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_141: expression _loop0_142 +// _gather_140: expression _loop0_141 static asdl_seq * -_gather_141_rule(Parser *p) +_gather_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33727,27 +34291,27 @@ _gather_141_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_142 + { // expression _loop0_141 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_142_rule(p)) // _loop0_142 + (seq = _loop0_141_rule(p)) // _loop0_141 ) { - D(fprintf(stderr, "%*c+ _gather_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_141[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_141")); } _res = NULL; done: @@ -33755,9 +34319,9 @@ _gather_141_rule(Parser *p) return _res; } -// _loop0_144: ',' expression +// _loop0_143: ',' expression static asdl_seq * -_loop0_144_rule(Parser *p) +_loop0_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33782,7 +34346,7 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33814,7 +34378,7 @@ _loop0_144_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_144[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_143[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33825,15 +34389,15 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_143: expression _loop0_144 +// _gather_142: expression _loop0_143 static asdl_seq * -_gather_143_rule(Parser *p) +_gather_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33844,27 +34408,27 @@ _gather_143_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_144 + { // expression _loop0_143 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_144_rule(p)) // _loop0_144 + (seq = _loop0_143_rule(p)) // _loop0_143 ) { - D(fprintf(stderr, "%*c+ _gather_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_143[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_143")); } _res = NULL; done: @@ -33872,9 +34436,9 @@ _gather_143_rule(Parser *p) return _res; } -// _loop0_146: ',' expression +// _loop0_145: ',' expression static asdl_seq * -_loop0_146_rule(Parser *p) +_loop0_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33899,7 +34463,7 @@ _loop0_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33931,7 +34495,7 @@ _loop0_146_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33942,15 +34506,15 @@ _loop0_146_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_145: expression _loop0_146 +// _gather_144: expression _loop0_145 static asdl_seq * -_gather_145_rule(Parser *p) +_gather_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33961,27 +34525,27 @@ _gather_145_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_146 + { // expression _loop0_145 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c> _gather_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_146_rule(p)) // _loop0_146 + (seq = _loop0_145_rule(p)) // _loop0_145 ) { - D(fprintf(stderr, "%*c+ _gather_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c+ _gather_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_145[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c%s _gather_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_145")); } _res = NULL; done: @@ -33989,9 +34553,9 @@ _gather_145_rule(Parser *p) return _res; } -// _loop0_148: ',' expression +// _loop0_147: ',' expression static asdl_seq * -_loop0_148_rule(Parser *p) +_loop0_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34016,7 +34580,7 @@ _loop0_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34048,7 +34612,7 @@ _loop0_148_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34059,15 +34623,15 @@ _loop0_148_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_147: expression _loop0_148 +// _gather_146: expression _loop0_147 static asdl_seq * -_gather_147_rule(Parser *p) +_gather_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34078,27 +34642,27 @@ _gather_147_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_148 + { // expression _loop0_147 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c> _gather_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_148_rule(p)) // _loop0_148 + (seq = _loop0_147_rule(p)) // _loop0_147 ) { - D(fprintf(stderr, "%*c+ _gather_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c+ _gather_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_147[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c%s _gather_146[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_147")); } _res = NULL; done: @@ -34106,9 +34670,9 @@ _gather_147_rule(Parser *p) return _res; } -// _tmp_149: NEWLINE INDENT +// _tmp_148: NEWLINE INDENT static void * -_tmp_149_rule(Parser *p) +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34124,7 +34688,7 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); Token * indent_var; Token * newline_var; if ( @@ -34133,12 +34697,12 @@ _tmp_149_rule(Parser *p) (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); _res = _PyPegen_dummy_name(p, newline_var, indent_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } _res = NULL; @@ -34147,11 +34711,11 @@ _tmp_149_rule(Parser *p) return _res; } -// _tmp_150: +// _tmp_149: // | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) // | kwargs static void * -_tmp_150_rule(Parser *p) +_tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34167,18 +34731,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_263_var; + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_260_var; if ( - (_tmp_263_var = _tmp_263_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_260_var = _tmp_260_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_263_var; + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_260_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } { // kwargs @@ -34186,18 +34750,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); asdl_seq* kwargs_var; if ( (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); _res = kwargs_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } _res = NULL; @@ -34206,9 +34770,126 @@ _tmp_150_rule(Parser *p) return _res; } -// _tmp_151: args | expression for_if_clauses +// _loop0_151: ',' (starred_expression !'=') +static asdl_seq * +_loop0_151_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' (starred_expression !'=') + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + Token * _literal; + void *elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_261_rule(p)) // starred_expression !'=' + ) + { + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_150: (starred_expression !'=') _loop0_151 +static asdl_seq * +_gather_150_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // (starred_expression !'=') _loop0_151 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_261_rule(p)) // starred_expression !'=' + && + (seq = _loop0_151_rule(p)) // _loop0_151 + ) + { + D(fprintf(stderr, "%*c+ _gather_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_150[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_151")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_152: args | expression for_if_clauses static void * -_tmp_151_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34224,18 +34905,18 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -34243,7 +34924,7 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -34252,12 +34933,12 @@ _tmp_151_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -34266,9 +34947,9 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: args ',' +// _tmp_153: args ',' static void * -_tmp_152_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34284,7 +34965,7 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -34293,12 +34974,12 @@ _tmp_152_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -34307,9 +34988,9 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: ',' | ')' +// _tmp_154: ',' | ')' static void * -_tmp_153_rule(Parser *p) +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34325,18 +35006,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -34344,18 +35025,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -34364,9 +35045,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: 'True' | 'False' | 'None' +// _tmp_155: 'True' | 'False' | 'None' static void * -_tmp_154_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34382,18 +35063,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -34401,18 +35082,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -34420,18 +35101,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -34440,9 +35121,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: NAME '=' +// _tmp_156: NAME '=' static void * -_tmp_155_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34458,7 +35139,7 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -34467,12 +35148,12 @@ _tmp_155_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -34481,9 +35162,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: NAME STRING | SOFT_KEYWORD +// _tmp_157: NAME STRING | SOFT_KEYWORD static void * -_tmp_156_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34499,7 +35180,7 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -34508,12 +35189,12 @@ _tmp_156_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -34521,18 +35202,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -34541,9 +35222,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: 'else' | ':' +// _tmp_158: 'else' | ':' static void * -_tmp_157_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34559,18 +35240,18 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 659)) // token='else' + (_keyword = _PyPegen_expect_token(p, 663)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -34578,18 +35259,18 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -34598,9 +35279,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: '=' | ':=' +// _tmp_159: '=' | ':=' static void * -_tmp_158_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34616,18 +35297,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34635,18 +35316,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34655,9 +35336,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_160: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_159_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34673,18 +35354,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34692,18 +35373,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34711,18 +35392,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34730,18 +35411,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34749,18 +35430,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34768,18 +35449,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34788,9 +35469,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: '=' | ':=' +// _tmp_161: '=' | ':=' static void * -_tmp_160_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34806,18 +35487,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34825,18 +35506,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34845,9 +35526,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _loop0_161: star_named_expressions +// _loop0_162: star_named_expressions static asdl_seq * -_loop0_161_rule(Parser *p) +_loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34872,7 +35553,7 @@ _loop0_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -34895,7 +35576,7 @@ _loop0_161_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34906,15 +35587,15 @@ _loop0_161_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_162: (star_targets '=') +// _loop0_163: (star_targets '=') static asdl_seq * -_loop0_162_rule(Parser *p) +_loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34939,13 +35620,13 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; + D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_262_var; while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' + (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' ) { - _res = _tmp_264_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34962,7 +35643,7 @@ _loop0_162_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34973,15 +35654,15 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop0_163: (star_targets '=') +// _loop0_164: (star_targets '=') static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35006,13 +35687,13 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_265_var; + D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_263_var; while ( - (_tmp_265_var = _tmp_265_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_265_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35029,7 +35710,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35040,69 +35721,12 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_164: yield_expr | star_expressions -static void * -_tmp_164_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); - } - _res = NULL; - done: - p->level--; - return _res; -} - // _tmp_165: '[' | '(' | '{' static void * _tmp_165_rule(Parser *p) @@ -35411,7 +36035,7 @@ _loop0_169_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35478,7 +36102,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35545,7 +36169,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35617,7 +36241,7 @@ _loop1_172_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35741,7 +36365,7 @@ _loop0_174_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35865,7 +36489,7 @@ _loop0_176_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -35937,7 +36561,7 @@ _loop1_177_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36039,15 +36663,15 @@ _tmp_179_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_266_var; + void *_tmp_264_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_266_var = _tmp_266_rule(p)) // ')' | '**' + (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); goto done; } p->mark = _mark; @@ -36178,7 +36802,7 @@ _loop0_181_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36383,7 +37007,7 @@ _loop1_184_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36507,7 +37131,7 @@ _loop0_186_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36574,7 +37198,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36641,7 +37265,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36717,7 +37341,7 @@ _loop0_190_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -36882,7 +37506,7 @@ _loop0_192_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -37006,7 +37630,7 @@ _loop0_194_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -37078,7 +37702,7 @@ _loop1_195_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -37150,7 +37774,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -37195,15 +37819,15 @@ _tmp_197_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_267_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_267_var = _tmp_267_rule(p)) // ':' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_267_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; @@ -37334,7 +37958,7 @@ _loop0_199_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; @@ -37549,9 +38173,54 @@ _tmp_202_rule(Parser *p) return _res; } -// _loop0_204: ',' dotted_name +// _tmp_203: bitwise_or ((',' bitwise_or))* ','? +static void * +_tmp_203_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // bitwise_or ((',' bitwise_or))* ','? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_266_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty bitwise_or_var; + if ( + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + && + (_loop0_266_var = _loop0_266_rule(p)) // ((',' bitwise_or))* + && + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? + ) + { + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_266_var, _opt_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_205: ',' dotted_name static asdl_seq * -_loop0_204_rule(Parser *p) +_loop0_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37576,7 +38245,7 @@ _loop0_204_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -37608,7 +38277,7 @@ _loop0_204_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_204[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37619,15 +38288,15 @@ _loop0_204_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_203: dotted_name _loop0_204 +// _gather_204: dotted_name _loop0_205 static asdl_seq * -_gather_203_rule(Parser *p) +_gather_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37638,27 +38307,27 @@ _gather_203_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_204 + { // dotted_name _loop0_205 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_204_rule(p)) // _loop0_204 + (seq = _loop0_205_rule(p)) // _loop0_205 ) { - D(fprintf(stderr, "%*c+ _gather_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_204")); + D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_205")); } _res = NULL; done: @@ -37666,9 +38335,9 @@ _gather_203_rule(Parser *p) return _res; } -// _loop0_206: ',' (expression ['as' star_target]) +// _loop0_207: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_206_rule(Parser *p) +_loop0_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37693,13 +38362,13 @@ _loop0_206_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37725,7 +38394,7 @@ _loop0_206_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37736,15 +38405,15 @@ _loop0_206_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_205: (expression ['as' star_target]) _loop0_206 +// _gather_206: (expression ['as' star_target]) _loop0_207 static asdl_seq * -_gather_205_rule(Parser *p) +_gather_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37755,27 +38424,27 @@ _gather_205_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_206 + { // (expression ['as' star_target]) _loop0_207 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] && - (seq = _loop0_206_rule(p)) // _loop0_206 + (seq = _loop0_207_rule(p)) // _loop0_207 ) { - D(fprintf(stderr, "%*c+ _gather_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_206")); + D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); } _res = NULL; done: @@ -37783,9 +38452,9 @@ _gather_205_rule(Parser *p) return _res; } -// _loop0_208: ',' (expressions ['as' star_target]) +// _loop0_209: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_208_rule(Parser *p) +_loop0_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37810,13 +38479,13 @@ _loop0_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -37842,7 +38511,7 @@ _loop0_208_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37853,15 +38522,15 @@ _loop0_208_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_207: (expressions ['as' star_target]) _loop0_208 +// _gather_208: (expressions ['as' star_target]) _loop0_209 static asdl_seq * -_gather_207_rule(Parser *p) +_gather_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37872,27 +38541,27 @@ _gather_207_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_208 + { // (expressions ['as' star_target]) _loop0_209 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_208_rule(p)) // _loop0_208 + (seq = _loop0_209_rule(p)) // _loop0_209 ) { - D(fprintf(stderr, "%*c+ _gather_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_208")); + D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); } _res = NULL; done: @@ -37900,9 +38569,9 @@ _gather_207_rule(Parser *p) return _res; } -// _loop0_210: ',' (expression ['as' star_target]) +// _loop0_211: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_210_rule(Parser *p) +_loop0_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37927,13 +38596,13 @@ _loop0_210_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_270_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -37959,7 +38628,7 @@ _loop0_210_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_210[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37970,15 +38639,15 @@ _loop0_210_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_209: (expression ['as' star_target]) _loop0_210 +// _gather_210: (expression ['as' star_target]) _loop0_211 static asdl_seq * -_gather_209_rule(Parser *p) +_gather_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37989,27 +38658,27 @@ _gather_209_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_210 + { // (expression ['as' star_target]) _loop0_211 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_270_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] && - (seq = _loop0_210_rule(p)) // _loop0_210 + (seq = _loop0_211_rule(p)) // _loop0_211 ) { - D(fprintf(stderr, "%*c+ _gather_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_210")); + D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); } _res = NULL; done: @@ -38017,9 +38686,9 @@ _gather_209_rule(Parser *p) return _res; } -// _loop0_212: ',' (expressions ['as' star_target]) +// _loop0_213: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_212_rule(Parser *p) +_loop0_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38044,13 +38713,13 @@ _loop0_212_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_271_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38076,7 +38745,7 @@ _loop0_212_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_212[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38087,15 +38756,15 @@ _loop0_212_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_211: (expressions ['as' star_target]) _loop0_212 +// _gather_212: (expressions ['as' star_target]) _loop0_213 static asdl_seq * -_gather_211_rule(Parser *p) +_gather_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38106,27 +38775,27 @@ _gather_211_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_212 + { // (expressions ['as' star_target]) _loop0_213 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c> _gather_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_271_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_212_rule(p)) // _loop0_212 + (seq = _loop0_213_rule(p)) // _loop0_213 ) { - D(fprintf(stderr, "%*c+ _gather_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c+ _gather_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_212")); + D(fprintf(stderr, "%*c%s _gather_212[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); } _res = NULL; done: @@ -38134,9 +38803,9 @@ _gather_211_rule(Parser *p) return _res; } -// _tmp_213: 'except' | 'finally' +// _tmp_214: 'except' | 'finally' static void * -_tmp_213_rule(Parser *p) +_tmp_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38152,18 +38821,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -38171,18 +38840,18 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 651)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -38191,9 +38860,9 @@ _tmp_213_rule(Parser *p) return _res; } -// _loop0_214: block +// _loop0_215: block static asdl_seq * -_loop0_214_rule(Parser *p) +_loop0_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38218,7 +38887,7 @@ _loop0_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38241,7 +38910,7 @@ _loop0_214_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38252,15 +38921,15 @@ _loop0_214_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop1_215: except_block +// _loop1_216: except_block static asdl_seq * -_loop1_215_rule(Parser *p) +_loop1_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38285,7 +38954,7 @@ _loop1_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -38308,7 +38977,7 @@ _loop1_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -38324,15 +38993,15 @@ _loop1_215_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_216: 'as' NAME +// _tmp_217: 'as' NAME static void * -_tmp_216_rule(Parser *p) +_tmp_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38348,21 +39017,21 @@ _tmp_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38371,9 +39040,9 @@ _tmp_216_rule(Parser *p) return _res; } -// _loop0_217: block +// _loop0_218: block static asdl_seq * -_loop0_217_rule(Parser *p) +_loop0_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38398,7 +39067,7 @@ _loop0_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38421,7 +39090,7 @@ _loop0_217_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_218[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38432,15 +39101,15 @@ _loop0_217_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _loop1_218: except_star_block +// _loop1_219: except_star_block static asdl_seq * -_loop1_218_rule(Parser *p) +_loop1_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38465,7 +39134,7 @@ _loop1_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -38488,7 +39157,7 @@ _loop1_218_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -38504,15 +39173,15 @@ _loop1_218_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _tmp_219: expression ['as' NAME] +// _tmp_220: expression ['as' NAME] static void * -_tmp_219_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38528,22 +39197,22 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_272_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38552,9 +39221,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: 'as' NAME +// _tmp_221: 'as' NAME static void * -_tmp_220_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38570,21 +39239,21 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38593,9 +39262,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_222: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38611,21 +39280,21 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38634,9 +39303,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: NEWLINE | ':' +// _tmp_223: NEWLINE | ':' static void * -_tmp_222_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38652,18 +39321,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -38671,18 +39340,18 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -38691,9 +39360,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: 'as' NAME +// _tmp_224: 'as' NAME static void * -_tmp_223_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38709,21 +39378,21 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38732,9 +39401,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_225: 'as' NAME static void * -_tmp_224_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38750,21 +39419,21 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38773,9 +39442,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: positional_patterns ',' +// _tmp_226: positional_patterns ',' static void * -_tmp_225_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38791,7 +39460,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -38800,12 +39469,12 @@ _tmp_225_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -38814,9 +39483,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: '->' expression +// _tmp_227: '->' expression static void * -_tmp_226_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38832,7 +39501,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -38841,12 +39510,12 @@ _tmp_226_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -38855,9 +39524,50 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: '(' arguments? ')' +// _tmp_228: '->' expression static void * -_tmp_227_rule(Parser *p) +_tmp_228_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '->' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + Token * _literal; + expr_ty expression_var; + if ( + (_literal = _PyPegen_expect_token(p, 51)) // token='->' + && + (expression_var = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + _res = _PyPegen_dummy_name(p, _literal, expression_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_229: '(' arguments? ')' +static void * +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38873,7 +39583,7 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38886,12 +39596,12 @@ _tmp_227_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38900,9 +39610,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_230: '(' arguments? ')' static void * -_tmp_228_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38918,7 +39628,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -38931,12 +39641,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -38945,9 +39655,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _loop0_230: ',' double_starred_kvpair +// _loop0_232: ',' double_starred_kvpair static asdl_seq * -_loop0_230_rule(Parser *p) +_loop0_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38972,7 +39682,7 @@ _loop0_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39004,7 +39714,7 @@ _loop0_230_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39015,15 +39725,15 @@ _loop0_230_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_229: double_starred_kvpair _loop0_230 +// _gather_231: double_starred_kvpair _loop0_232 static asdl_seq * -_gather_229_rule(Parser *p) +_gather_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39034,27 +39744,27 @@ _gather_229_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_230 + { // double_starred_kvpair _loop0_232 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c> _gather_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_230_rule(p)) // _loop0_230 + (seq = _loop0_232_rule(p)) // _loop0_232 ) { - D(fprintf(stderr, "%*c+ _gather_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c+ _gather_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_229[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_230")); + D(fprintf(stderr, "%*c%s _gather_231[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_232")); } _res = NULL; done: @@ -39062,9 +39772,9 @@ _gather_229_rule(Parser *p) return _res; } -// _tmp_231: '}' | ',' +// _tmp_233: '}' | ',' static void * -_tmp_231_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39080,18 +39790,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39099,18 +39809,18 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39119,9 +39829,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_234: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39137,18 +39847,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39156,133 +39866,19 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_233: yield_expr | star_expressions -static void * -_tmp_233_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_234: yield_expr | star_expressions -static void * -_tmp_234_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; done: @@ -39385,66 +39981,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: yield_expr | star_expressions +// _tmp_236: '!' | ':' | '}' static void * _tmp_236_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_237: '!' | ':' | '}' -static void * -_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39460,18 +39999,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39479,18 +40018,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39498,18 +40037,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39518,9 +40057,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: yield_expr | star_expressions +// _tmp_237: '!' NAME static void * -_tmp_238_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39531,43 +40070,27 @@ _tmp_238_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions + { // '!' NAME if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + Token * _literal; + expr_ty name_var; if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions + (_literal = _PyPegen_expect_token(p, 54)) // token='!' + && + (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; done: @@ -39575,9 +40098,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: yield_expr | star_expressions +// _tmp_238: ':' | '}' static void * -_tmp_239_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39588,43 +40111,43 @@ _tmp_239_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // yield_expr + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - { // star_expressions + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + Token * _literal; if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; done: @@ -39632,9 +40155,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: '!' NAME +// _tmp_239: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39650,7 +40173,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39659,12 +40182,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39673,9 +40196,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' -static void * -_tmp_241_rule(Parser *p) +// _loop0_240: fstring_format_spec +static asdl_seq * +_loop0_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39684,45 +40207,96 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // ':' + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // fstring_format_spec if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + D(fprintf(stderr, "%*c> _loop0_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + expr_ty fstring_format_spec_var; + while ( + (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; + _res = fstring_format_spec_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _loop0_240[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } - { // '}' + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_241: '!' NAME +static void * +_tmp_241_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '!' NAME if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; + expr_ty name_var; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_literal = _PyPegen_expect_token(p, 54)) // token='!' + && + (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; done: @@ -39730,7 +40304,7 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: yield_expr | star_expressions +// _tmp_242: ':' | '}' static void * _tmp_242_rule(Parser *p) { @@ -39743,43 +40317,43 @@ _tmp_242_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // yield_expr + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - { // star_expressions + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + Token * _literal; if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; done: @@ -39787,7 +40361,7 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '!' NAME +// _tmp_243: '+' | '-' | '*' | '/' | '%' | '//' | '@' static void * _tmp_243_rule(Parser *p) { @@ -39800,151 +40374,138 @@ _tmp_243_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '!' NAME + { // '+' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; - expr_ty name_var; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - _res = _PyPegen_dummy_name(p, _literal, name_var); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_244: fstring_format_spec -static asdl_seq * -_loop0_244_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // fstring_format_spec + { // '-' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); - expr_ty fstring_format_spec_var; - while ( - (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - _res = fstring_format_spec_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_244[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_245: yield_expr | star_expressions -static void * -_tmp_245_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + { // '/' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 17)) // token='/' + ) + { + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // '%' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 24)) // token='%' + ) + { + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr + { // '//' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + Token * _literal; if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr + (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } - { // star_expressions + { // '@' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + Token * _literal; if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions + (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; done: @@ -39952,9 +40513,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '!' NAME +// _tmp_244: '+' | '-' | '~' static void * -_tmp_246_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39965,84 +40526,62 @@ _tmp_246_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '!' NAME + { // '+' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; - expr_ty name_var; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - _res = _PyPegen_dummy_name(p, _literal, name_var); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_247: ':' | '}' -static void * -_tmp_247_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } - void * _res = NULL; - int _mark = p->mark; - { // ':' + { // '-' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } - { // '}' + { // '~' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; done: @@ -40050,9 +40589,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: star_targets '=' +// _tmp_245: star_targets '=' static void * -_tmp_248_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40068,7 +40607,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40077,7 +40616,7 @@ _tmp_248_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40087,7 +40626,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40096,9 +40635,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '.' | '...' +// _tmp_246: '.' | '...' static void * -_tmp_249_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40114,18 +40653,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40133,18 +40672,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40153,9 +40692,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: '.' | '...' +// _tmp_247: '.' | '...' static void * -_tmp_250_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40171,18 +40710,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40190,18 +40729,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40210,9 +40749,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: '@' named_expression NEWLINE +// _tmp_248: '@' named_expression NEWLINE static void * -_tmp_251_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40228,7 +40767,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40240,7 +40779,7 @@ _tmp_251_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40250,7 +40789,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40259,9 +40798,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: ',' expression +// _tmp_249: ',' expression static void * -_tmp_252_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40277,7 +40816,7 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40286,7 +40825,7 @@ _tmp_252_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40296,7 +40835,7 @@ _tmp_252_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40305,9 +40844,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: ',' star_expression +// _tmp_250: ',' star_expression static void * -_tmp_253_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40323,7 +40862,7 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40332,7 +40871,7 @@ _tmp_253_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40342,7 +40881,7 @@ _tmp_253_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40351,9 +40890,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'or' conjunction +// _tmp_251: 'or' conjunction static void * -_tmp_254_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40369,7 +40908,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40378,7 +40917,7 @@ _tmp_254_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40388,7 +40927,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40397,9 +40936,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'and' inversion +// _tmp_252: 'and' inversion static void * -_tmp_255_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40415,7 +40954,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40424,7 +40963,7 @@ _tmp_255_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40434,7 +40973,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40443,9 +40982,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: slice | starred_expression +// _tmp_253: slice | starred_expression static void * -_tmp_256_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40461,18 +41000,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40480,18 +41019,18 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40500,9 +41039,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: fstring | string +// _tmp_254: fstring | string static void * -_tmp_257_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40518,18 +41057,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40537,18 +41076,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40557,9 +41096,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: 'if' disjunction +// _tmp_255: 'if' disjunction static void * -_tmp_258_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40575,16 +41114,16 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40594,7 +41133,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40603,9 +41142,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: 'if' disjunction +// _tmp_256: 'if' disjunction static void * -_tmp_259_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40621,16 +41160,16 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40640,7 +41179,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40649,9 +41188,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_257: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_260_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40667,18 +41206,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -40686,20 +41225,20 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_273_var; + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_272_var; if ( - (_tmp_273_var = _tmp_273_rule(p)) // assignment_expression | expression !':=' + (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_273_var; + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_272_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -40708,9 +41247,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: ',' star_target +// _tmp_258: ',' star_target static void * -_tmp_261_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40726,7 +41265,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40735,7 +41274,7 @@ _tmp_261_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40745,7 +41284,7 @@ _tmp_261_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40754,9 +41293,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: ',' star_target +// _tmp_259: ',' star_target static void * -_tmp_262_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40772,7 +41311,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -40781,7 +41320,7 @@ _tmp_262_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40791,7 +41330,7 @@ _tmp_262_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -40800,10 +41339,10 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: +// _tmp_260: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_263_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40819,24 +41358,24 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_274_var; + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_273_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_274_var = _gather_274_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_273_var = _gather_273_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_274_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_273_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -40845,9 +41384,49 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_261: starred_expression !'=' static void * -_tmp_264_rule(Parser *p) +_tmp_261_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // starred_expression !'=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + expr_ty starred_expression_var; + if ( + (starred_expression_var = starred_expression_rule(p)) // starred_expression + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' + ) + { + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + _res = starred_expression_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_262: star_targets '=' +static void * +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40863,7 +41442,7 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40872,12 +41451,12 @@ _tmp_264_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40886,9 +41465,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_265_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40904,7 +41483,7 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -40913,12 +41492,12 @@ _tmp_265_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40927,9 +41506,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ')' | '**' +// _tmp_264: ')' | '**' static void * -_tmp_266_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40945,18 +41524,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -40964,18 +41543,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -40984,9 +41563,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: ':' | '**' +// _tmp_265: ':' | '**' static void * -_tmp_267_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41002,18 +41581,18 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41021,18 +41600,18 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41041,9 +41620,76 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expression ['as' star_target] +// _loop0_266: (',' bitwise_or) +static asdl_seq * +_loop0_266_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (',' bitwise_or) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_275_var; + while ( + (_tmp_275_var = _tmp_275_rule(p)) // ',' bitwise_or + ) + { + _res = _tmp_275_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_266[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_267: expression ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41059,7 +41705,7 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; @@ -41069,12 +41715,12 @@ _tmp_268_rule(Parser *p) (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41083,9 +41729,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expressions ['as' star_target] +// _tmp_268: expressions ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41101,7 +41747,7 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; @@ -41111,12 +41757,12 @@ _tmp_269_rule(Parser *p) (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41125,9 +41771,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: expression ['as' star_target] +// _tmp_269: expression ['as' star_target] static void * -_tmp_270_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41143,7 +41789,7 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; @@ -41153,12 +41799,12 @@ _tmp_270_rule(Parser *p) (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41167,9 +41813,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: expressions ['as' star_target] +// _tmp_270: expressions ['as' star_target] static void * -_tmp_271_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41185,7 +41831,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; @@ -41195,12 +41841,12 @@ _tmp_271_rule(Parser *p) (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41209,9 +41855,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: 'as' NAME +// _tmp_271: 'as' NAME static void * -_tmp_272_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41227,21 +41873,21 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41250,9 +41896,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: assignment_expression | expression !':=' +// _tmp_272: assignment_expression | expression !':=' static void * -_tmp_273_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41268,18 +41914,18 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41287,7 +41933,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41295,12 +41941,12 @@ _tmp_273_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41309,9 +41955,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _loop0_275: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_274: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_275_rule(Parser *p) +_loop0_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41336,7 +41982,7 @@ _loop0_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( @@ -41368,7 +42014,7 @@ _loop0_275_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41379,16 +42025,16 @@ _loop0_275_rule(Parser *p) p->level--; return NULL; } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); p->level--; return _seq; } -// _gather_274: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 +// _gather_273: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 static asdl_seq * -_gather_274_rule(Parser *p) +_gather_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41399,27 +42045,68 @@ _gather_274_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c> _gather_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); void *elem; asdl_seq * seq; if ( (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_275_rule(p)) // _loop0_275 + (seq = _loop0_274_rule(p)) // _loop0_274 ) { - D(fprintf(stderr, "%*c+ _gather_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c+ _gather_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_274[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c%s _gather_273[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_275: ',' bitwise_or +static void * +_tmp_275_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ',' bitwise_or + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + Token * _literal; + expr_ty bitwise_or_var; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + ) + { + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; done: @@ -41449,7 +42136,7 @@ _tmp_276_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -41490,7 +42177,7 @@ _tmp_277_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -41531,7 +42218,7 @@ _tmp_278_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -41572,7 +42259,7 @@ _tmp_279_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 654)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) diff --git a/Parser/pegen.c b/Parser/pegen.c index 0c60394e4f199b..6efb5477c7b80f 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1,6 +1,7 @@ #include #include "pycore_ast.h" // _PyAST_Validate(), #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pyerrors.h" // PyExc_IncompleteInputError #include #include "lexer/lexer.h" @@ -19,12 +20,33 @@ _PyPegen_interactive_exit(Parser *p) } Py_ssize_t -_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) -{ - const char *str = PyUnicode_AsUTF8(line); - if (!str) { - return -1; +_PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) +{ + const char *data = PyUnicode_AsUTF8(line); + + Py_ssize_t len = 0; + while (col_offset < end_col_offset) { + Py_UCS4 ch = data[col_offset]; + if (ch < 0x80) { + col_offset += 1; + } else if ((ch & 0xe0) == 0xc0) { + col_offset += 2; + } else if ((ch & 0xf0) == 0xe0) { + col_offset += 3; + } else if ((ch & 0xf8) == 0xf0) { + col_offset += 4; + } else { + PyErr_SetString(PyExc_ValueError, "Invalid UTF-8 sequence"); + return -1; + } + len++; } + return len; +} + +Py_ssize_t +_PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset) +{ Py_ssize_t len = strlen(str); if (col_offset > len + 1) { col_offset = len + 1; @@ -39,6 +61,16 @@ _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) return size; } +Py_ssize_t +_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) +{ + const char *str = PyUnicode_AsUTF8(line); + if (!str) { + return -1; + } + return _PyPegen_byte_offset_to_character_offset_raw(str, col_offset); +} + // Here, mark is the start of the node, while p->mark is the end. // If node==NULL, they should be the same. int @@ -496,7 +528,8 @@ _PyPegen_new_identifier(Parser *p, const char *n) } id = id2; } - PyUnicode_InternInPlace(&id); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &id); if (_PyArena_AddPyObject(p->arena, id) < 0) { Py_DECREF(id); @@ -838,7 +871,7 @@ _PyPegen_run_parser(Parser *p) if (res == NULL) { if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { PyErr_Clear(); - return RAISE_SYNTAX_ERROR("incomplete input"); + return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input"); } if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { return NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 424f80acd7be3b..32c64e7774b878 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -148,7 +148,9 @@ int _PyPegen_fill_token(Parser *p); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); +Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); +Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); // Error handling functions and APIs typedef enum { diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 15e99e23d8490f..e8f11a67e50fa0 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -68,6 +68,7 @@ _Pypegen_tokenizer_error(Parser *p) const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; Py_ssize_t col_offset = -1; + p->error_indicator = 1; switch (p->tok->done) { case E_TOKEN: msg = "invalid token"; @@ -103,6 +104,10 @@ _Pypegen_tokenizer_error(Parser *p) msg = "unexpected character after line continuation character"; break; } + case E_COLUMNOVERFLOW: + PyErr_SetString(PyExc_OverflowError, + "Parser column offset overflow - source line is too big"); + return -1; default: msg = "unknown parsing error"; } @@ -214,6 +219,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { void * _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } if (p->fill == 0) { va_list va; va_start(va, errmsg); @@ -235,12 +244,6 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err col_offset = 0; } else { const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf; - if (p->tok->cur - start > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "Parser column offset overflow - source line is too big"); - p->error_indicator = 1; - return NULL; - } col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int); } } else { @@ -279,6 +282,10 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; + if (buf_end < cur_line) { + buf_end = cur_line + strlen(cur_line); + } + for (int i = 0; i < relative_lineno - 1; i++) { char *new_line = strchr(cur_line, '\n'); // The assert is here for debug builds but the conditional that @@ -304,6 +311,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } PyObject *value = NULL; PyObject *errstr = NULL; PyObject *error_line = NULL; @@ -358,20 +369,18 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t col_number = col_offset; Py_ssize_t end_col_number = end_col_offset; - if (p->tok->encoding != NULL) { - col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); - if (col_number < 0) { + col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); + if (col_number < 0) { + goto error; + } + + if (end_col_offset > 0) { + end_col_number = _PyPegen_byte_offset_to_character_offset(error_line, end_col_offset); + if (end_col_number < 0) { goto error; } - if (end_col_number > 0) { - Py_ssize_t end_col_offset = _PyPegen_byte_offset_to_character_offset(error_line, end_col_number); - if (end_col_offset < 0) { - goto error; - } else { - end_col_number = end_col_offset; - } - } } + tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number); if (!tmp) { goto error; @@ -395,7 +404,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, void _Pypegen_set_syntax_error(Parser* p, Token* last_token) { - // Existing sintax error + // Existing syntax error if (PyErr_Occurred()) { // Prioritize tokenizer errors to custom syntax errors raised // on the second phase only if the errors come from the parser. diff --git a/Parser/string_parser.c b/Parser/string_parser.c index f1e027765c86b9..93ad92b823581e 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -13,6 +13,11 @@ static int warn_invalid_escape_sequence(Parser *p, const char *first_invalid_escape, Token *t) { + if (p->call_invalid_rules) { + // Do not report warnings if we are in the second pass of the parser + // to avoid showing the warning twice. + return 0; + } unsigned char c = *first_invalid_escape; if ((t->type == FSTRING_MIDDLE || t->type == FSTRING_END) && (c == '{' || c == '}')) { // in this case the tokenizer has already emitted a warning, @@ -224,9 +229,14 @@ _PyPegen_parse_string(Parser *p, Token *t) PyErr_BadInternalCall(); return NULL; } + /* Skip the leading quote char. */ s++; len = strlen(s); + // gh-120155: 's' contains at least the trailing quote, + // so the code '--len' below is safe. + assert(len >= 1); + if (len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "string to parse is too long"); return NULL; diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 34f79191b4e8d7..6443d814a22dab 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -15,17 +15,6 @@ #include "Python/frozen_modules/zipimport.h" /* End includes */ -/* Empty initializer for deepfrozen modules */ -int _Py_Deepfreeze_Init(void) -{ - return 0; -} -/* Empty finalizer for deepfrozen modules */ -void -_Py_Deepfreeze_Fini(void) -{ -} - /* Note that a negative size indicates a package. */ static const struct _frozen bootstrap_modules[] = { diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 3de6c6816c1e61..2a462a42cdad7c 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -22,17 +22,6 @@ # include #endif -/* Empty initializer for deepfrozen modules */ -int _Py_Deepfreeze_Init(void) -{ - return 0; -} -/* Empty finalizer for deepfrozen modules */ -void -_Py_Deepfreeze_Fini(void) -{ -} - /* To avoid a circular dependency on frozen.o, we create our own structure of frozen modules instead, left deliberately blank so as to avoid unintentional import of a stale version of _frozen_importlib. */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 1f9aa4b3d449a1..d149b6a0c5cd21 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -16,11 +16,9 @@ // These functions were removed from Python 3.13 API but are still exported // for the stable ABI. We want to test them in this program. -extern void Py_SetProgramName(const wchar_t *program_name); extern void PySys_AddWarnOption(const wchar_t *s); extern void PySys_AddXOption(const wchar_t *s); extern void Py_SetPath(const wchar_t *path); -extern void Py_SetPythonHome(const wchar_t *home); int main_argc; @@ -576,7 +574,11 @@ static int test_init_from_config(void) _PyPreConfig_InitCompatConfig(&preconfig); putenv("PYTHONMALLOC=malloc_debug"); +#ifndef Py_GIL_DISABLED preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; +#else + preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC; +#endif putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; @@ -765,7 +767,11 @@ static int test_init_dont_parse_argv(void) static void set_most_env_vars(void) { putenv("PYTHONHASHSEED=42"); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif putenv("PYTHONTRACEMALLOC=2"); putenv("PYTHONPROFILEIMPORTTIME=1"); putenv("PYTHONNODEBUGRANGES=1"); @@ -851,7 +857,11 @@ static int test_init_env_dev_mode_alloc(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 4fb78cf632d70e..b2a7196bd6081c 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,38 +1,37 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,149,0,83,0,83,1, - 75,0,114,0,83,0,83,1,75,1,114,1,92,2,34,0, - 83,2,53,1,0,0,0,0,0,0,32,0,92,2,34,0, - 83,3,92,0,82,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,53,2,0,0,0,0,0,0, - 32,0,92,1,82,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,34,0,53,0,0,0,0,0, - 0,0,83,4,5,0,0,0,114,5,83,5,19,0,72,20, - 0,0,114,6,92,2,34,0,83,6,92,6,14,0,83,7, - 92,5,92,6,5,0,0,0,14,0,51,4,53,1,0,0, - 0,0,0,0,32,0,77,22,0,0,11,0,103,1,41,8, - 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, - 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, - 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, - 112,114,111,103,114,97,109,95,110,97,109,101,218,10,101,120, - 101,99,117,116,97,98,108,101,218,15,117,115,101,95,101,110, - 118,105,114,111,110,109,101,110,116,218,17,99,111,110,102,105, - 103,117,114,101,95,99,95,115,116,100,105,111,218,14,98,117, - 102,102,101,114,101,100,95,115,116,100,105,111,122,7,99,111, - 110,102,105,103,32,122,2,58,32,41,7,218,3,115,121,115, - 218,17,95,116,101,115,116,105,110,116,101,114,110,97,108,99, - 97,112,105,218,5,112,114,105,110,116,218,4,97,114,103,118, - 218,11,103,101,116,95,99,111,110,102,105,103,115,114,3,0, - 0,0,218,3,107,101,121,169,0,243,0,0,0,0,250,18, - 116,101,115,116,95,102,114,111,122,101,110,109,97,105,110,46, - 112,121,250,8,60,109,111,100,117,108,101,62,114,18,0,0, - 0,1,0,0,0,115,99,0,0,0,240,3,1,1,1,243, - 8,0,1,11,219,0,24,225,0,5,208,6,26,212,0,27, - 217,0,5,128,106,144,35,151,40,145,40,212,0,27,216,9, - 26,215,9,38,210,9,38,211,9,40,168,24,209,9,50,128, - 6,240,2,6,12,2,242,0,7,1,42,128,67,241,14,0, - 5,10,136,71,144,67,144,53,152,2,152,54,160,35,153,59, - 152,45,208,10,40,214,4,41,241,15,7,1,42,114,16,0, - 0,0, + 0,0,0,0,0,243,166,0,0,0,149,0,81,0,81,1, + 72,0,113,0,81,0,81,1,72,1,113,1,90,2,31,0, + 81,2,50,1,0,0,0,0,0,0,29,0,90,2,31,0, + 81,3,90,0,79,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,50,2,0,0,0,0,0,0, + 29,0,90,1,79,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,31,0,50,0,0,0,0,0, + 0,0,81,4,2,0,0,0,113,5,81,5,16,0,69,20, + 0,0,113,6,90,2,31,0,81,6,90,6,12,0,81,7, + 90,5,90,6,2,0,0,0,12,0,48,4,50,1,0,0, + 0,0,0,0,29,0,74,22,0,0,9,0,29,0,102,1, + 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, + 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, + 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, + 218,12,112,114,111,103,114,97,109,95,110,97,109,101,218,10, + 101,120,101,99,117,116,97,98,108,101,218,15,117,115,101,95, + 101,110,118,105,114,111,110,109,101,110,116,218,17,99,111,110, + 102,105,103,117,114,101,95,99,95,115,116,100,105,111,218,14, + 98,117,102,102,101,114,101,100,95,115,116,100,105,111,122,7, + 99,111,110,102,105,103,32,122,2,58,32,41,7,218,3,115, + 121,115,218,17,95,116,101,115,116,105,110,116,101,114,110,97, + 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, + 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, + 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, + 218,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,218,8,60,109,111,100,117,108,101,62,114,18, + 0,0,0,1,0,0,0,115,94,0,0,0,240,3,1,1, + 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, + 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, + 216,9,26,215,9,38,210,9,38,211,9,40,168,24,209,9, + 50,128,6,243,2,6,12,2,128,67,241,14,0,5,10,136, + 71,144,67,144,53,152,2,152,54,160,35,153,59,152,45,208, + 10,40,214,4,41,242,15,6,12,2,114,16,0,0,0, }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a197d44868b5aa..4d0db457a8b172 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -4,10 +4,20 @@ #include "pycore_ast.h" #include "pycore_ast_state.h" // struct ast_state #include "pycore_ceval.h" // _Py_EnterRecursiveCall +#include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast +#include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry(), _PySet_Update() +#include "pycore_unionobject.h" // _Py_union_type_or +#include "structmember.h" #include +struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ +}; + // Forward declaration static int init_types(struct ast_state *state); @@ -16,7 +26,8 @@ get_ast_state(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); struct ast_state *state = &interp->ast; - if (!init_types(state)) { + assert(!state->finalized); + if (_PyOnceFlag_CallOnce(&state->once, (_Py_once_fn_t *)&init_types, state) < 0) { return NULL; } return state; @@ -194,6 +205,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->conversion); Py_CLEAR(state->ctx); Py_CLEAR(state->decorator_list); + Py_CLEAR(state->default_value); Py_CLEAR(state->defaults); Py_CLEAR(state->elt); Py_CLEAR(state->elts); @@ -271,102 +283,100 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); -#if !defined(NDEBUG) - state->initialized = -1; -#else - state->initialized = 0; -#endif + state->finalized = 1; + state->once = (_PyOnceFlag){0}; } static int init_identifiers(struct ast_state *state) { - if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; - if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; - if ((state->__match_args__ = PyUnicode_InternFromString("__match_args__")) == NULL) return 0; - if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return 0; - if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return 0; - if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return 0; - if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return 0; - if ((state->arg = PyUnicode_InternFromString("arg")) == NULL) return 0; - if ((state->args = PyUnicode_InternFromString("args")) == NULL) return 0; - if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return 0; - if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return 0; - if ((state->ast = PyUnicode_InternFromString("ast")) == NULL) return 0; - if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return 0; - if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return 0; - if ((state->body = PyUnicode_InternFromString("body")) == NULL) return 0; - if ((state->bound = PyUnicode_InternFromString("bound")) == NULL) return 0; - if ((state->cases = PyUnicode_InternFromString("cases")) == NULL) return 0; - if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return 0; - if ((state->cls = PyUnicode_InternFromString("cls")) == NULL) return 0; - if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return 0; - if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return 0; - if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return 0; - if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return 0; - if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return 0; - if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return 0; - if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return 0; - if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return 0; - if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return 0; - if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0; - if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return 0; - if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return 0; - if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return 0; - if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return 0; - if ((state->func = PyUnicode_InternFromString("func")) == NULL) return 0; - if ((state->generators = PyUnicode_InternFromString("generators")) == NULL) return 0; - if ((state->guard = PyUnicode_InternFromString("guard")) == NULL) return 0; - if ((state->handlers = PyUnicode_InternFromString("handlers")) == NULL) return 0; - if ((state->id = PyUnicode_InternFromString("id")) == NULL) return 0; - if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return 0; - if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return 0; - if ((state->items = PyUnicode_InternFromString("items")) == NULL) return 0; - if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return 0; - if ((state->key = PyUnicode_InternFromString("key")) == NULL) return 0; - if ((state->keys = PyUnicode_InternFromString("keys")) == NULL) return 0; - if ((state->keywords = PyUnicode_InternFromString("keywords")) == NULL) return 0; - if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return 0; - if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return 0; - if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return 0; - if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return 0; - if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return 0; - if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return 0; - if ((state->left = PyUnicode_InternFromString("left")) == NULL) return 0; - if ((state->level = PyUnicode_InternFromString("level")) == NULL) return 0; - if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return 0; - if ((state->lower = PyUnicode_InternFromString("lower")) == NULL) return 0; - if ((state->module = PyUnicode_InternFromString("module")) == NULL) return 0; - if ((state->msg = PyUnicode_InternFromString("msg")) == NULL) return 0; - if ((state->name = PyUnicode_InternFromString("name")) == NULL) return 0; - if ((state->names = PyUnicode_InternFromString("names")) == NULL) return 0; - if ((state->op = PyUnicode_InternFromString("op")) == NULL) return 0; - if ((state->operand = PyUnicode_InternFromString("operand")) == NULL) return 0; - if ((state->ops = PyUnicode_InternFromString("ops")) == NULL) return 0; - if ((state->optional_vars = PyUnicode_InternFromString("optional_vars")) == NULL) return 0; - if ((state->orelse = PyUnicode_InternFromString("orelse")) == NULL) return 0; - if ((state->pattern = PyUnicode_InternFromString("pattern")) == NULL) return 0; - if ((state->patterns = PyUnicode_InternFromString("patterns")) == NULL) return 0; - if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return 0; - if ((state->rest = PyUnicode_InternFromString("rest")) == NULL) return 0; - if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return 0; - if ((state->right = PyUnicode_InternFromString("right")) == NULL) return 0; - if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return 0; - if ((state->slice = PyUnicode_InternFromString("slice")) == NULL) return 0; - if ((state->step = PyUnicode_InternFromString("step")) == NULL) return 0; - if ((state->subject = PyUnicode_InternFromString("subject")) == NULL) return 0; - if ((state->tag = PyUnicode_InternFromString("tag")) == NULL) return 0; - if ((state->target = PyUnicode_InternFromString("target")) == NULL) return 0; - if ((state->targets = PyUnicode_InternFromString("targets")) == NULL) return 0; - if ((state->test = PyUnicode_InternFromString("test")) == NULL) return 0; - if ((state->type = PyUnicode_InternFromString("type")) == NULL) return 0; - if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return 0; - if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return 0; - if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return 0; - if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return 0; - if ((state->value = PyUnicode_InternFromString("value")) == NULL) return 0; - if ((state->values = PyUnicode_InternFromString("values")) == NULL) return 0; - if ((state->vararg = PyUnicode_InternFromString("vararg")) == NULL) return 0; - return 1; + if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return -1; + if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return -1; + if ((state->__match_args__ = PyUnicode_InternFromString("__match_args__")) == NULL) return -1; + if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return -1; + if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return -1; + if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return -1; + if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return -1; + if ((state->arg = PyUnicode_InternFromString("arg")) == NULL) return -1; + if ((state->args = PyUnicode_InternFromString("args")) == NULL) return -1; + if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return -1; + if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return -1; + if ((state->ast = PyUnicode_InternFromString("ast")) == NULL) return -1; + if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return -1; + if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return -1; + if ((state->body = PyUnicode_InternFromString("body")) == NULL) return -1; + if ((state->bound = PyUnicode_InternFromString("bound")) == NULL) return -1; + if ((state->cases = PyUnicode_InternFromString("cases")) == NULL) return -1; + if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return -1; + if ((state->cls = PyUnicode_InternFromString("cls")) == NULL) return -1; + if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return -1; + if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return -1; + if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return -1; + if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return -1; + if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return -1; + if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return -1; + if ((state->default_value = PyUnicode_InternFromString("default_value")) == NULL) return -1; + if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return -1; + if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return -1; + if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return -1; + if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return -1; + if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return -1; + if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return -1; + if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return -1; + if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return -1; + if ((state->func = PyUnicode_InternFromString("func")) == NULL) return -1; + if ((state->generators = PyUnicode_InternFromString("generators")) == NULL) return -1; + if ((state->guard = PyUnicode_InternFromString("guard")) == NULL) return -1; + if ((state->handlers = PyUnicode_InternFromString("handlers")) == NULL) return -1; + if ((state->id = PyUnicode_InternFromString("id")) == NULL) return -1; + if ((state->ifs = PyUnicode_InternFromString("ifs")) == NULL) return -1; + if ((state->is_async = PyUnicode_InternFromString("is_async")) == NULL) return -1; + if ((state->items = PyUnicode_InternFromString("items")) == NULL) return -1; + if ((state->iter = PyUnicode_InternFromString("iter")) == NULL) return -1; + if ((state->key = PyUnicode_InternFromString("key")) == NULL) return -1; + if ((state->keys = PyUnicode_InternFromString("keys")) == NULL) return -1; + if ((state->keywords = PyUnicode_InternFromString("keywords")) == NULL) return -1; + if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return -1; + if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return -1; + if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return -1; + if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return -1; + if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return -1; + if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return -1; + if ((state->left = PyUnicode_InternFromString("left")) == NULL) return -1; + if ((state->level = PyUnicode_InternFromString("level")) == NULL) return -1; + if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return -1; + if ((state->lower = PyUnicode_InternFromString("lower")) == NULL) return -1; + if ((state->module = PyUnicode_InternFromString("module")) == NULL) return -1; + if ((state->msg = PyUnicode_InternFromString("msg")) == NULL) return -1; + if ((state->name = PyUnicode_InternFromString("name")) == NULL) return -1; + if ((state->names = PyUnicode_InternFromString("names")) == NULL) return -1; + if ((state->op = PyUnicode_InternFromString("op")) == NULL) return -1; + if ((state->operand = PyUnicode_InternFromString("operand")) == NULL) return -1; + if ((state->ops = PyUnicode_InternFromString("ops")) == NULL) return -1; + if ((state->optional_vars = PyUnicode_InternFromString("optional_vars")) == NULL) return -1; + if ((state->orelse = PyUnicode_InternFromString("orelse")) == NULL) return -1; + if ((state->pattern = PyUnicode_InternFromString("pattern")) == NULL) return -1; + if ((state->patterns = PyUnicode_InternFromString("patterns")) == NULL) return -1; + if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return -1; + if ((state->rest = PyUnicode_InternFromString("rest")) == NULL) return -1; + if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return -1; + if ((state->right = PyUnicode_InternFromString("right")) == NULL) return -1; + if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return -1; + if ((state->slice = PyUnicode_InternFromString("slice")) == NULL) return -1; + if ((state->step = PyUnicode_InternFromString("step")) == NULL) return -1; + if ((state->subject = PyUnicode_InternFromString("subject")) == NULL) return -1; + if ((state->tag = PyUnicode_InternFromString("tag")) == NULL) return -1; + if ((state->target = PyUnicode_InternFromString("target")) == NULL) return -1; + if ((state->targets = PyUnicode_InternFromString("targets")) == NULL) return -1; + if ((state->test = PyUnicode_InternFromString("test")) == NULL) return -1; + if ((state->type = PyUnicode_InternFromString("type")) == NULL) return -1; + if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return -1; + if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return -1; + if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return -1; + if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return -1; + if ((state->value = PyUnicode_InternFromString("value")) == NULL) return -1; + if ((state->values = PyUnicode_InternFromString("values")) == NULL) return -1; + if ((state->vararg = PyUnicode_InternFromString("vararg")) == NULL) return -1; + return 0; }; GENERATE_ASDL_SEQ_CONSTRUCTOR(mod, mod_ty) @@ -384,7 +394,8 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_param, type_param_ty) -static PyObject* ast2obj_mod(struct ast_state *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, struct validator *vstate, + void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -405,7 +416,8 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(struct ast_state *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, struct validator + *vstate, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -540,7 +552,8 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(struct ast_state *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, struct validator + *vstate, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -653,12 +666,18 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); -static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); -static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); -static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); -static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); -static PyObject* ast2obj_comprehension(struct ast_state *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, struct validator + *vstate, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, struct validator + *vstate, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, struct validator + *vstate, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, struct validator + *vstate, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, struct + validator *vstate, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -671,13 +690,15 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, struct + validator *vstate, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(struct ast_state *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, struct validator + *vstate, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -687,7 +708,8 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(struct ast_state *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, struct validator *vstate, + void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -699,7 +721,8 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(struct ast_state *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, struct validator + *vstate, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -710,7 +733,8 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(struct ast_state *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, struct validator + *vstate, void*); static const char * const alias_attributes[] = { "lineno", "col_offset", @@ -721,12 +745,14 @@ static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(struct ast_state *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, struct validator + *vstate, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_match_case(struct ast_state *state, void*); +static PyObject* ast2obj_match_case(struct ast_state *state, struct validator + *vstate, void*); static const char * const match_case_fields[]={ "pattern", "guard", @@ -738,7 +764,8 @@ static const char * const pattern_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_pattern(struct ast_state *state, void*); +static PyObject* ast2obj_pattern(struct ast_state *state, struct validator + *vstate, void*); static const char * const MatchValue_fields[]={ "value", }; @@ -769,7 +796,8 @@ static const char * const MatchAs_fields[]={ static const char * const MatchOr_fields[]={ "patterns", }; -static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -780,19 +808,4235 @@ static const char * const type_param_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_type_param(struct ast_state *state, void*); +static PyObject* ast2obj_type_param(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeVar_fields[]={ "name", "bound", + "default_value", }; static const char * const ParamSpec_fields[]={ "name", + "default_value", }; static const char * const TypeVarTuple_fields[]={ "name", + "default_value", }; +static int +add_ast_annotations(struct ast_state *state) +{ + bool cond; + PyObject *Module_annotations = PyDict_New(); + if (!Module_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + { + PyObject *type = state->type_ignore_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "type_ignores", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Module_type, "_field_types", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Module_type, "__annotations__", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + Py_DECREF(Module_annotations); + PyObject *Interactive_annotations = PyDict_New(); + if (!Interactive_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyDict_SetItemString(Interactive_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Interactive_type, "_field_types", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Interactive_type, "__annotations__", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + Py_DECREF(Interactive_annotations); + PyObject *Expression_annotations = PyDict_New(); + if (!Expression_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expression_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expression_type, "_field_types", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expression_type, "__annotations__", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + Py_DECREF(Expression_annotations); + PyObject *FunctionType_annotations = PyDict_New(); + if (!FunctionType_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionType_annotations, "argtypes", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionType_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionType_type, "_field_types", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionType_type, "__annotations__", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + Py_DECREF(FunctionType_annotations); + PyObject *FunctionDef_annotations = PyDict_New(); + if (!FunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_params", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "_field_types", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "__annotations__", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + Py_DECREF(FunctionDef_annotations); + PyObject *AsyncFunctionDef_annotations = PyDict_New(); + if (!AsyncFunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "name", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "args", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "body", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "decorator_list", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "returns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_params", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, "_field_types", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, + "__annotations__", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + Py_DECREF(AsyncFunctionDef_annotations); + PyObject *ClassDef_annotations = PyDict_New(); + if (!ClassDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ClassDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "bases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "keywords", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ClassDef_type, "_field_types", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ClassDef_type, "__annotations__", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + Py_DECREF(ClassDef_annotations); + PyObject *Return_annotations = PyDict_New(); + if (!Return_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyDict_SetItemString(Return_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Return_type, "_field_types", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Return_type, "__annotations__", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + Py_DECREF(Return_annotations); + PyObject *Delete_annotations = PyDict_New(); + if (!Delete_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyDict_SetItemString(Delete_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Delete_type, "_field_types", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Delete_type, "__annotations__", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + Py_DECREF(Delete_annotations); + PyObject *Assign_annotations = PyDict_New(); + if (!Assign_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assign_type, "_field_types", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assign_type, "__annotations__", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + Py_DECREF(Assign_annotations); + PyObject *TypeAlias_annotations = PyDict_New(); + if (!TypeAlias_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeAlias_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "_field_types", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "__annotations__", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + Py_DECREF(TypeAlias_annotations); + PyObject *AugAssign_annotations = PyDict_New(); + if (!AugAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AugAssign_type, "_field_types", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AugAssign_type, "__annotations__", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + Py_DECREF(AugAssign_annotations); + PyObject *AnnAssign_annotations = PyDict_New(); + if (!AnnAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "annotation", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyDict_SetItemString(AnnAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "simple", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "_field_types", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "__annotations__", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + Py_DECREF(AnnAssign_annotations); + PyObject *For_annotations = PyDict_New(); + if (!For_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->For_type, "_field_types", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->For_type, "__annotations__", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + Py_DECREF(For_annotations); + PyObject *AsyncFor_annotations = PyDict_New(); + if (!AsyncFor_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "_field_types", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "__annotations__", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + Py_DECREF(AsyncFor_annotations); + PyObject *While_annotations = PyDict_New(); + if (!While_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(While_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->While_type, "_field_types", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->While_type, "__annotations__", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + Py_DECREF(While_annotations); + PyObject *If_annotations = PyDict_New(); + if (!If_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(If_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->If_type, "_field_types", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->If_type, "__annotations__", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + Py_DECREF(If_annotations); + PyObject *With_annotations = PyDict_New(); + if (!With_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "type_comment", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->With_type, "_field_types", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->With_type, "__annotations__", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + Py_DECREF(With_annotations); + PyObject *AsyncWith_annotations = PyDict_New(); + if (!AsyncWith_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "_field_types", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "__annotations__", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + Py_DECREF(AsyncWith_annotations); + PyObject *Match_annotations = PyDict_New(); + if (!Match_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Match_annotations, "subject", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + { + PyObject *type = state->match_case_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyDict_SetItemString(Match_annotations, "cases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Match_type, "_field_types", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Match_type, "__annotations__", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + Py_DECREF(Match_annotations); + PyObject *Raise_annotations = PyDict_New(); + if (!Raise_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "exc", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "cause", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Raise_type, "_field_types", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Raise_type, "__annotations__", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + Py_DECREF(Raise_annotations); + PyObject *Try_annotations = PyDict_New(); + if (!Try_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "finalbody", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Try_type, "_field_types", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Try_type, "__annotations__", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + Py_DECREF(Try_annotations); + PyObject *TryStar_annotations = PyDict_New(); + if (!TryStar_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "finalbody", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TryStar_type, "_field_types", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TryStar_type, "__annotations__", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + Py_DECREF(TryStar_annotations); + PyObject *Assert_annotations = PyDict_New(); + if (!Assert_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assert_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyDict_SetItemString(Assert_annotations, "msg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assert_type, "_field_types", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assert_type, "__annotations__", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + Py_DECREF(Assert_annotations); + PyObject *Import_annotations = PyDict_New(); + if (!Import_annotations) return 0; + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyDict_SetItemString(Import_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Import_type, "_field_types", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Import_type, "__annotations__", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + Py_DECREF(Import_annotations); + PyObject *ImportFrom_annotations = PyDict_New(); + if (!ImportFrom_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "module", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "level", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "_field_types", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "__annotations__", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + Py_DECREF(ImportFrom_annotations); + PyObject *Global_annotations = PyDict_New(); + if (!Global_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyDict_SetItemString(Global_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Global_type, "_field_types", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Global_type, "__annotations__", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + Py_DECREF(Global_annotations); + PyObject *Nonlocal_annotations = PyDict_New(); + if (!Nonlocal_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyDict_SetItemString(Nonlocal_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "_field_types", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "__annotations__", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + Py_DECREF(Nonlocal_annotations); + PyObject *Expr_annotations = PyDict_New(); + if (!Expr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expr_type, "_field_types", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expr_type, "__annotations__", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + Py_DECREF(Expr_annotations); + PyObject *Pass_annotations = PyDict_New(); + if (!Pass_annotations) return 0; + cond = PyObject_SetAttrString(state->Pass_type, "_field_types", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pass_type, "__annotations__", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + Py_DECREF(Pass_annotations); + PyObject *Break_annotations = PyDict_New(); + if (!Break_annotations) return 0; + cond = PyObject_SetAttrString(state->Break_type, "_field_types", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Break_type, "__annotations__", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + Py_DECREF(Break_annotations); + PyObject *Continue_annotations = PyDict_New(); + if (!Continue_annotations) return 0; + cond = PyObject_SetAttrString(state->Continue_type, "_field_types", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Continue_type, "__annotations__", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + Py_DECREF(Continue_annotations); + PyObject *BoolOp_annotations = PyDict_New(); + if (!BoolOp_annotations) return 0; + { + PyObject *type = state->boolop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BoolOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyDict_SetItemString(BoolOp_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BoolOp_type, "_field_types", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BoolOp_type, "__annotations__", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + Py_DECREF(BoolOp_annotations); + PyObject *NamedExpr_annotations = PyDict_New(); + if (!NamedExpr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "_field_types", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "__annotations__", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + Py_DECREF(NamedExpr_annotations); + PyObject *BinOp_annotations = PyDict_New(); + if (!BinOp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "right", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BinOp_type, "_field_types", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BinOp_type, "__annotations__", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + Py_DECREF(BinOp_annotations); + PyObject *UnaryOp_annotations = PyDict_New(); + if (!UnaryOp_annotations) return 0; + { + PyObject *type = state->unaryop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "operand", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "_field_types", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "__annotations__", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + Py_DECREF(UnaryOp_annotations); + PyObject *Lambda_annotations = PyDict_New(); + if (!Lambda_annotations) return 0; + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Lambda_type, "_field_types", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lambda_type, "__annotations__", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + Py_DECREF(Lambda_annotations); + PyObject *IfExp_annotations = PyDict_New(); + if (!IfExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->IfExp_type, "_field_types", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IfExp_type, "__annotations__", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + Py_DECREF(IfExp_annotations); + PyObject *Dict_annotations = PyDict_New(); + if (!Dict_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "keys", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Dict_type, "_field_types", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Dict_type, "__annotations__", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + Py_DECREF(Dict_annotations); + PyObject *Set_annotations = PyDict_New(); + if (!Set_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyDict_SetItemString(Set_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Set_type, "_field_types", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Set_type, "__annotations__", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + Py_DECREF(Set_annotations); + PyObject *ListComp_annotations = PyDict_New(); + if (!ListComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(ListComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyDict_SetItemString(ListComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ListComp_type, "_field_types", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ListComp_type, "__annotations__", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + Py_DECREF(ListComp_annotations); + PyObject *SetComp_annotations = PyDict_New(); + if (!SetComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(SetComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyDict_SetItemString(SetComp_annotations, "generators", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->SetComp_type, "_field_types", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->SetComp_type, "__annotations__", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + Py_DECREF(SetComp_annotations); + PyObject *DictComp_annotations = PyDict_New(); + if (!DictComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "key", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyDict_SetItemString(DictComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->DictComp_type, "_field_types", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->DictComp_type, "__annotations__", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + Py_DECREF(DictComp_annotations); + PyObject *GeneratorExp_annotations = PyDict_New(); + if (!GeneratorExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(GeneratorExp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyDict_SetItemString(GeneratorExp_annotations, "generators", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "_field_types", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "__annotations__", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + Py_DECREF(GeneratorExp_annotations); + PyObject *Await_annotations = PyDict_New(); + if (!Await_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Await_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Await_type, "_field_types", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Await_type, "__annotations__", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + Py_DECREF(Await_annotations); + PyObject *Yield_annotations = PyDict_New(); + if (!Yield_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyDict_SetItemString(Yield_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Yield_type, "_field_types", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Yield_type, "__annotations__", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + Py_DECREF(Yield_annotations); + PyObject *YieldFrom_annotations = PyDict_New(); + if (!YieldFrom_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(YieldFrom_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "_field_types", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "__annotations__", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + Py_DECREF(YieldFrom_annotations); + PyObject *Compare_annotations = PyDict_New(); + if (!Compare_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Compare_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->cmpop_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "ops", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "comparators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Compare_type, "_field_types", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Compare_type, "__annotations__", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + Py_DECREF(Compare_annotations); + PyObject *Call_annotations = PyDict_New(); + if (!Call_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Call_annotations, "func", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "keywords", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Call_type, "_field_types", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Call_type, "__annotations__", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + Py_DECREF(Call_annotations); + PyObject *FormattedValue_annotations = PyDict_New(); + if (!FormattedValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "conversion", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyDict_SetItemString(FormattedValue_annotations, "format_spec", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FormattedValue_type, "_field_types", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FormattedValue_type, + "__annotations__", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + Py_DECREF(FormattedValue_annotations); + PyObject *JoinedStr_annotations = PyDict_New(); + if (!JoinedStr_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyDict_SetItemString(JoinedStr_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "_field_types", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "__annotations__", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + Py_DECREF(JoinedStr_annotations); + PyObject *Constant_annotations = PyDict_New(); + if (!Constant_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Constant_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyDict_SetItemString(Constant_annotations, "kind", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Constant_type, "_field_types", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Constant_type, "__annotations__", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + Py_DECREF(Constant_annotations); + PyObject *Attribute_annotations = PyDict_New(); + if (!Attribute_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "attr", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Attribute_type, "_field_types", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Attribute_type, "__annotations__", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + Py_DECREF(Attribute_annotations); + PyObject *Subscript_annotations = PyDict_New(); + if (!Subscript_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "slice", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Subscript_type, "_field_types", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Subscript_type, "__annotations__", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + Py_DECREF(Subscript_annotations); + PyObject *Starred_annotations = PyDict_New(); + if (!Starred_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Starred_type, "_field_types", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Starred_type, "__annotations__", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + Py_DECREF(Starred_annotations); + PyObject *Name_annotations = PyDict_New(); + if (!Name_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "id", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Name_type, "_field_types", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Name_type, "__annotations__", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + Py_DECREF(Name_annotations); + PyObject *List_annotations = PyDict_New(); + if (!List_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyDict_SetItemString(List_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(List_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->List_type, "_field_types", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->List_type, "__annotations__", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + Py_DECREF(List_annotations); + PyObject *Tuple_annotations = PyDict_New(); + if (!Tuple_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyDict_SetItemString(Tuple_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Tuple_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Tuple_type, "_field_types", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Tuple_type, "__annotations__", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + Py_DECREF(Tuple_annotations); + PyObject *Slice_annotations = PyDict_New(); + if (!Slice_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "lower", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "upper", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "step", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Slice_type, "_field_types", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Slice_type, "__annotations__", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + Py_DECREF(Slice_annotations); + PyObject *Load_annotations = PyDict_New(); + if (!Load_annotations) return 0; + cond = PyObject_SetAttrString(state->Load_type, "_field_types", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Load_type, "__annotations__", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + Py_DECREF(Load_annotations); + PyObject *Store_annotations = PyDict_New(); + if (!Store_annotations) return 0; + cond = PyObject_SetAttrString(state->Store_type, "_field_types", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Store_type, "__annotations__", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + Py_DECREF(Store_annotations); + PyObject *Del_annotations = PyDict_New(); + if (!Del_annotations) return 0; + cond = PyObject_SetAttrString(state->Del_type, "_field_types", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Del_type, "__annotations__", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + Py_DECREF(Del_annotations); + PyObject *And_annotations = PyDict_New(); + if (!And_annotations) return 0; + cond = PyObject_SetAttrString(state->And_type, "_field_types", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->And_type, "__annotations__", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + Py_DECREF(And_annotations); + PyObject *Or_annotations = PyDict_New(); + if (!Or_annotations) return 0; + cond = PyObject_SetAttrString(state->Or_type, "_field_types", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Or_type, "__annotations__", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + Py_DECREF(Or_annotations); + PyObject *Add_annotations = PyDict_New(); + if (!Add_annotations) return 0; + cond = PyObject_SetAttrString(state->Add_type, "_field_types", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Add_type, "__annotations__", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + Py_DECREF(Add_annotations); + PyObject *Sub_annotations = PyDict_New(); + if (!Sub_annotations) return 0; + cond = PyObject_SetAttrString(state->Sub_type, "_field_types", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Sub_type, "__annotations__", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + Py_DECREF(Sub_annotations); + PyObject *Mult_annotations = PyDict_New(); + if (!Mult_annotations) return 0; + cond = PyObject_SetAttrString(state->Mult_type, "_field_types", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mult_type, "__annotations__", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + Py_DECREF(Mult_annotations); + PyObject *MatMult_annotations = PyDict_New(); + if (!MatMult_annotations) return 0; + cond = PyObject_SetAttrString(state->MatMult_type, "_field_types", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatMult_type, "__annotations__", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + Py_DECREF(MatMult_annotations); + PyObject *Div_annotations = PyDict_New(); + if (!Div_annotations) return 0; + cond = PyObject_SetAttrString(state->Div_type, "_field_types", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Div_type, "__annotations__", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + Py_DECREF(Div_annotations); + PyObject *Mod_annotations = PyDict_New(); + if (!Mod_annotations) return 0; + cond = PyObject_SetAttrString(state->Mod_type, "_field_types", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mod_type, "__annotations__", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + Py_DECREF(Mod_annotations); + PyObject *Pow_annotations = PyDict_New(); + if (!Pow_annotations) return 0; + cond = PyObject_SetAttrString(state->Pow_type, "_field_types", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pow_type, "__annotations__", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + Py_DECREF(Pow_annotations); + PyObject *LShift_annotations = PyDict_New(); + if (!LShift_annotations) return 0; + cond = PyObject_SetAttrString(state->LShift_type, "_field_types", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LShift_type, "__annotations__", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + Py_DECREF(LShift_annotations); + PyObject *RShift_annotations = PyDict_New(); + if (!RShift_annotations) return 0; + cond = PyObject_SetAttrString(state->RShift_type, "_field_types", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->RShift_type, "__annotations__", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + Py_DECREF(RShift_annotations); + PyObject *BitOr_annotations = PyDict_New(); + if (!BitOr_annotations) return 0; + cond = PyObject_SetAttrString(state->BitOr_type, "_field_types", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitOr_type, "__annotations__", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + Py_DECREF(BitOr_annotations); + PyObject *BitXor_annotations = PyDict_New(); + if (!BitXor_annotations) return 0; + cond = PyObject_SetAttrString(state->BitXor_type, "_field_types", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitXor_type, "__annotations__", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + Py_DECREF(BitXor_annotations); + PyObject *BitAnd_annotations = PyDict_New(); + if (!BitAnd_annotations) return 0; + cond = PyObject_SetAttrString(state->BitAnd_type, "_field_types", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitAnd_type, "__annotations__", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + Py_DECREF(BitAnd_annotations); + PyObject *FloorDiv_annotations = PyDict_New(); + if (!FloorDiv_annotations) return 0; + cond = PyObject_SetAttrString(state->FloorDiv_type, "_field_types", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FloorDiv_type, "__annotations__", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + Py_DECREF(FloorDiv_annotations); + PyObject *Invert_annotations = PyDict_New(); + if (!Invert_annotations) return 0; + cond = PyObject_SetAttrString(state->Invert_type, "_field_types", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Invert_type, "__annotations__", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + Py_DECREF(Invert_annotations); + PyObject *Not_annotations = PyDict_New(); + if (!Not_annotations) return 0; + cond = PyObject_SetAttrString(state->Not_type, "_field_types", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Not_type, "__annotations__", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + Py_DECREF(Not_annotations); + PyObject *UAdd_annotations = PyDict_New(); + if (!UAdd_annotations) return 0; + cond = PyObject_SetAttrString(state->UAdd_type, "_field_types", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UAdd_type, "__annotations__", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + Py_DECREF(UAdd_annotations); + PyObject *USub_annotations = PyDict_New(); + if (!USub_annotations) return 0; + cond = PyObject_SetAttrString(state->USub_type, "_field_types", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->USub_type, "__annotations__", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + Py_DECREF(USub_annotations); + PyObject *Eq_annotations = PyDict_New(); + if (!Eq_annotations) return 0; + cond = PyObject_SetAttrString(state->Eq_type, "_field_types", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Eq_type, "__annotations__", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + Py_DECREF(Eq_annotations); + PyObject *NotEq_annotations = PyDict_New(); + if (!NotEq_annotations) return 0; + cond = PyObject_SetAttrString(state->NotEq_type, "_field_types", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotEq_type, "__annotations__", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + Py_DECREF(NotEq_annotations); + PyObject *Lt_annotations = PyDict_New(); + if (!Lt_annotations) return 0; + cond = PyObject_SetAttrString(state->Lt_type, "_field_types", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lt_type, "__annotations__", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + Py_DECREF(Lt_annotations); + PyObject *LtE_annotations = PyDict_New(); + if (!LtE_annotations) return 0; + cond = PyObject_SetAttrString(state->LtE_type, "_field_types", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LtE_type, "__annotations__", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + Py_DECREF(LtE_annotations); + PyObject *Gt_annotations = PyDict_New(); + if (!Gt_annotations) return 0; + cond = PyObject_SetAttrString(state->Gt_type, "_field_types", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Gt_type, "__annotations__", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + Py_DECREF(Gt_annotations); + PyObject *GtE_annotations = PyDict_New(); + if (!GtE_annotations) return 0; + cond = PyObject_SetAttrString(state->GtE_type, "_field_types", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GtE_type, "__annotations__", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + Py_DECREF(GtE_annotations); + PyObject *Is_annotations = PyDict_New(); + if (!Is_annotations) return 0; + cond = PyObject_SetAttrString(state->Is_type, "_field_types", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Is_type, "__annotations__", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + Py_DECREF(Is_annotations); + PyObject *IsNot_annotations = PyDict_New(); + if (!IsNot_annotations) return 0; + cond = PyObject_SetAttrString(state->IsNot_type, "_field_types", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IsNot_type, "__annotations__", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + Py_DECREF(IsNot_annotations); + PyObject *In_annotations = PyDict_New(); + if (!In_annotations) return 0; + cond = PyObject_SetAttrString(state->In_type, "_field_types", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->In_type, "__annotations__", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + Py_DECREF(In_annotations); + PyObject *NotIn_annotations = PyDict_New(); + if (!NotIn_annotations) return 0; + cond = PyObject_SetAttrString(state->NotIn_type, "_field_types", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotIn_type, "__annotations__", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + Py_DECREF(NotIn_annotations); + PyObject *comprehension_annotations = PyDict_New(); + if (!comprehension_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "target", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "iter", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyDict_SetItemString(comprehension_annotations, "ifs", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "is_async", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->comprehension_type, "_field_types", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->comprehension_type, "__annotations__", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + Py_DECREF(comprehension_annotations); + PyObject *ExceptHandler_annotations = PyDict_New(); + if (!ExceptHandler_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "type", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "body", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "_field_types", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "__annotations__", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + Py_DECREF(ExceptHandler_annotations); + PyObject *arguments_annotations = PyDict_New(); + if (!arguments_annotations) return 0; + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "posonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "vararg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kw_defaults", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwarg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "defaults", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arguments_type, "_field_types", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arguments_type, "__annotations__", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + Py_DECREF(arguments_annotations); + PyObject *arg_annotations = PyDict_New(); + if (!arg_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(arg_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "annotation", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arg_type, "_field_types", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arg_type, "__annotations__", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + Py_DECREF(arg_annotations); + PyObject *keyword_annotations = PyDict_New(); + if (!keyword_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyDict_SetItemString(keyword_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(keyword_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->keyword_type, "_field_types", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->keyword_type, "__annotations__", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + Py_DECREF(keyword_annotations); + PyObject *alias_annotations = PyDict_New(); + if (!alias_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(alias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyDict_SetItemString(alias_annotations, "asname", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->alias_type, "_field_types", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->alias_type, "__annotations__", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + Py_DECREF(alias_annotations); + PyObject *withitem_annotations = PyDict_New(); + if (!withitem_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(withitem_annotations, "context_expr", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyDict_SetItemString(withitem_annotations, "optional_vars", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->withitem_type, "_field_types", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->withitem_type, "__annotations__", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + Py_DECREF(withitem_annotations); + PyObject *match_case_annotations = PyDict_New(); + if (!match_case_annotations) return 0; + { + PyObject *type = state->pattern_type; + Py_INCREF(type); + cond = PyDict_SetItemString(match_case_annotations, "pattern", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "guard", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->match_case_type, "_field_types", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->match_case_type, "__annotations__", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + Py_DECREF(match_case_annotations); + PyObject *MatchValue_annotations = PyDict_New(); + if (!MatchValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchValue_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchValue_type, "_field_types", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchValue_type, "__annotations__", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + Py_DECREF(MatchValue_annotations); + PyObject *MatchSingleton_annotations = PyDict_New(); + if (!MatchSingleton_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchSingleton_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, "_field_types", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, + "__annotations__", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + Py_DECREF(MatchSingleton_annotations); + PyObject *MatchSequence_annotations = PyDict_New(); + if (!MatchSequence_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchSequence_annotations, "patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "_field_types", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "__annotations__", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + Py_DECREF(MatchSequence_annotations); + PyObject *MatchMapping_annotations = PyDict_New(); + if (!MatchMapping_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "keys", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "rest", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "_field_types", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "__annotations__", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + Py_DECREF(MatchMapping_annotations); + PyObject *MatchClass_annotations = PyDict_New(); + if (!MatchClass_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchClass_annotations, "cls", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_attrs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchClass_type, "_field_types", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchClass_type, "__annotations__", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + Py_DECREF(MatchClass_annotations); + PyObject *MatchStar_annotations = PyDict_New(); + if (!MatchStar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchStar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchStar_type, "_field_types", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchStar_type, "__annotations__", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + Py_DECREF(MatchStar_annotations); + PyObject *MatchAs_annotations = PyDict_New(); + if (!MatchAs_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "pattern", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchAs_type, "_field_types", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchAs_type, "__annotations__", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + Py_DECREF(MatchAs_annotations); + PyObject *MatchOr_annotations = PyDict_New(); + if (!MatchOr_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchOr_annotations, "patterns", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchOr_type, "_field_types", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchOr_type, "__annotations__", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + Py_DECREF(MatchOr_annotations); + PyObject *TypeIgnore_annotations = PyDict_New(); + if (!TypeIgnore_annotations) return 0; + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "lineno", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "tag", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "_field_types", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "__annotations__", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + Py_DECREF(TypeIgnore_annotations); + PyObject *TypeVar_annotations = PyDict_New(); + if (!TypeVar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "bound", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "default_value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVar_type, "__annotations__", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + Py_DECREF(TypeVar_annotations); + PyObject *ParamSpec_annotations = PyDict_New(); + if (!ParamSpec_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ParamSpec_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyDict_SetItemString(ParamSpec_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "__annotations__", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + Py_DECREF(ParamSpec_annotations); + PyObject *TypeVarTuple_annotations = PyDict_New(); + if (!TypeVarTuple_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVarTuple_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVarTuple_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "__annotations__", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + Py_DECREF(TypeVarTuple_annotations); + + return 1; +} + + typedef struct { PyObject_HEAD @@ -837,7 +5081,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -846,6 +5090,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -865,6 +5116,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -877,13 +5133,14 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -891,15 +5148,123 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } } + else { + // Lazily initialize "attributes" + if (attributes == NULL) { + attributes = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_attributes); + if (attributes == NULL) { + res = -1; + goto cleanup; + } + } + int contains = PySequence_Contains(attributes, key); + if (contains == -1) { + res = -1; + goto cleanup; + } + else if (contains == 0) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types) < 0) { + res = -1; + goto cleanup; + } + if (field_types == NULL) { + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } + } + } + else if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + goto set_remaining_cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: + Py_XDECREF(attributes); Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -911,14 +5276,345 @@ ast_type_reduce(PyObject *self, PyObject *unused) return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; + if (dict) { + // Unpickling (or copying) works as follows: + // - Construct the object with only positional arguments + // - Set the fields from the dict + // We have two constraints: + // - We must set all the required fields in the initial constructor call, + // or the unpickling or deepcopying of the object will trigger DeprecationWarnings. + // - We must not include child nodes in the positional args, because + // that may trigger runaway recursion during copying (gh-120108). + // To satisfy both constraints, we set all the fields to None in the + // initial list of positional args, and then set the fields from the dict. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value; + int rc = PyDict_GetItemRef(dict, name, &value); + Py_DECREF(name); + if (rc < 0) { + goto cleanup; + } + if (!value) { + break; + } + rc = PyList_Append(positional_args, Py_None); + Py_DECREF(value); + if (rc < 0) { + goto cleanup; + } + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONN", Py_TYPE(self), args_tuple, dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); + } +cleanup: + Py_XDECREF(fields); + Py_XDECREF(positional_args); + return result; +} + +/* + * Perform the following validations: + * + * - All keyword arguments are known 'fields' or 'attributes'. + * - No field or attribute would be left unfilled after copy.replace(). + * + * On success, this returns 1. Otherwise, set a TypeError + * exception and returns -1 (no exception is set if some + * other internal errors occur). + * + * Parameters + * + * self The AST node instance. + * dict The AST node instance dictionary (self.__dict__). + * fields The list of fields (self._fields). + * attributes The list of attributes (self._attributes). + * kwargs Keyword arguments passed to ast_type_replace(). + * + * The 'dict', 'fields', 'attributes' and 'kwargs' arguments can be NULL. + * + * Note: this function can be removed in 3.15 since the verification + * will be done inside the constructor. + */ +static inline int +ast_type_replace_check(PyObject *self, + PyObject *dict, + PyObject *fields, + PyObject *attributes, + PyObject *kwargs) +{ + // While it is possible to make some fast paths that would avoid + // allocating objects on the stack, this would cost us readability. + // For instance, if 'fields' and 'attributes' are both empty, and + // 'kwargs' is not empty, we could raise a TypeError immediately. + PyObject *expecting = PySet_New(fields); + if (expecting == NULL) { + return -1; + } + if (attributes) { + if (_PySet_Update(expecting, attributes) < 0) { + Py_DECREF(expecting); + return -1; + } + } + // Any keyword argument that is neither a field nor attribute is rejected. + // We first need to check whether a keyword argument is accepted or not. + // If all keyword arguments are accepted, we compute the required fields + // and attributes. A field or attribute is not needed if: + // + // 1) it is given in 'kwargs', or + // 2) it already exists on 'self'. + if (kwargs) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(kwargs, &pos, &key, &value)) { + int rc = PySet_Discard(expecting, key); + if (rc < 0) { + Py_DECREF(expecting); + return -1; + } + if (rc == 0) { + PyErr_Format(PyExc_TypeError, + "%.400s.__replace__ got an unexpected keyword " + "argument '%U'.", Py_TYPE(self)->tp_name, key); + Py_DECREF(expecting); + return -1; + } + } + } + // check that the remaining fields or attributes would be filled if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + // Mark fields or attributes that are found on the instance + // as non-mandatory. If they are not given in 'kwargs', they + // will be shallow-coied; otherwise, they would be replaced + // (not in this function). + if (PySet_Discard(expecting, key) < 0) { + Py_DECREF(expecting); + return -1; + } + } + if (attributes) { + // Some attributes may or may not be present at runtime. + // In particular, now that we checked whether 'kwargs' + // is correct or not, we allow any attribute to be missing. + // + // Note that fields must still be entirely determined when + // calling the constructor later. + PyObject *unused = PyObject_CallMethodOneArg(expecting, + &_Py_ID(difference_update), + attributes); + if (unused == NULL) { + Py_DECREF(expecting); + return -1; + } + Py_DECREF(unused); + } + } + // Now 'expecting' contains the fields or attributes + // that would not be filled inside ast_type_replace(). + Py_ssize_t m = PySet_GET_SIZE(expecting); + if (m > 0) { + PyObject *names = PyList_New(m); + if (names == NULL) { + Py_DECREF(expecting); + return -1; + } + Py_ssize_t i = 0, pos = 0; + PyObject *item; + Py_hash_t hash; + while (_PySet_NextEntry(expecting, &pos, &item, &hash)) { + PyObject *name = PyObject_Repr(item); + if (name == NULL) { + Py_DECREF(expecting); + Py_DECREF(names); + return -1; + } + // steal the reference 'name' + PyList_SET_ITEM(names, i++, name); + } + Py_DECREF(expecting); + if (PyList_Sort(names) < 0) { + Py_DECREF(names); + return -1; + } + PyObject *sep = PyUnicode_FromString(", "); + if (sep == NULL) { + Py_DECREF(names); + return -1; + } + PyObject *str_names = PyUnicode_Join(sep, names); + Py_DECREF(sep); + Py_DECREF(names); + if (str_names == NULL) { + return -1; + } + PyErr_Format(PyExc_TypeError, + "%.400s.__replace__ missing %ld keyword argument%s: %U.", + Py_TYPE(self)->tp_name, m, m == 1 ? "" : "s", str_names); + Py_DECREF(str_names); + return -1; + } + else { + Py_DECREF(expecting); + return 1; + } +} + +/* + * Python equivalent: + * + * for key in keys: + * if hasattr(self, key): + * payload[key] = getattr(self, key) + * + * The 'keys' argument is a sequence corresponding to + * the '_fields' or the '_attributes' of an AST node. + * + * This returns -1 if an error occurs and 0 otherwise. + * + * Parameters + * + * payload A dictionary to fill. + * keys A sequence of keys or NULL for an empty sequence. + * dict The AST node instance dictionary (must not be NULL). + */ +static inline int +ast_type_replace_update_payload(PyObject *payload, + PyObject *keys, + PyObject *dict) +{ + assert(dict != NULL); + if (keys == NULL) { + return 0; + } + Py_ssize_t n = PySequence_Size(keys); + if (n == -1) { + return -1; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *key = PySequence_GetItem(keys, i); + if (key == NULL) { + return -1; + } + PyObject *value; + if (PyDict_GetItemRef(dict, key, &value) < 0) { + Py_DECREF(key); + return -1; + } + if (value == NULL) { + Py_DECREF(key); + // If a field or attribute is not present at runtime, it should + // be explicitly given in 'kwargs'. If not, the constructor will + // issue a warning (which becomes an error in 3.15). + continue; + } + int rc = PyDict_SetItem(payload, key, value); + Py_DECREF(key); + Py_DECREF(value); + if (rc < 0) { + return -1; + } + } + return 0; +} + +/* copy.replace() support (shallow copy) */ +static PyObject * +ast_type_replace(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoPositional("__replace__", args)) { + return NULL; + } + + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + + PyObject *result = NULL; + // known AST class fields and attributes + PyObject *fields = NULL, *attributes = NULL; + // current instance dictionary + PyObject *dict = NULL; + // constructor positional and keyword arguments + PyObject *empty_tuple = NULL, *payload = NULL; + + PyObject *type = (PyObject *)Py_TYPE(self); + if (PyObject_GetOptionalAttr(type, state->_fields, &fields) < 0) { + goto cleanup; + } + if (PyObject_GetOptionalAttr(type, state->_attributes, &attributes) < 0) { + goto cleanup; + } + if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { + goto cleanup; } - return Py_BuildValue("O()", Py_TYPE(self)); + if (ast_type_replace_check(self, dict, fields, attributes, kwargs) < 0) { + goto cleanup; + } + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + goto cleanup; + } + payload = PyDict_New(); + if (payload == NULL) { + goto cleanup; + } + if (dict) { // in case __dict__ is missing (for some obscure reason) + // copy the instance's fields (possibly NULL) + if (ast_type_replace_update_payload(payload, fields, dict) < 0) { + goto cleanup; + } + // copy the instance's attributes (possibly NULL) + if (ast_type_replace_update_payload(payload, attributes, dict) < 0) { + goto cleanup; + } + } + if (kwargs && PyDict_Update(payload, kwargs) < 0) { + goto cleanup; + } + result = PyObject_Call(type, empty_tuple, payload); +cleanup: + Py_XDECREF(payload); + Py_XDECREF(empty_tuple); + Py_XDECREF(dict); + Py_XDECREF(attributes); + Py_XDECREF(fields); + return result; } static PyMemberDef ast_type_members[] = { @@ -928,6 +5624,10 @@ static PyMemberDef ast_type_members[] = { static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, + {"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **fields)\n--\n\n" + "Return a copy of the AST node with new values " + "for the specified fields.")}, {NULL} }; @@ -993,23 +5693,24 @@ add_attributes(struct ast_state *state, PyObject *type, const char * const *attr int i, result; PyObject *s, *l = PyTuple_New(num_fields); if (!l) - return 0; + return -1; for (i = 0; i < num_fields; i++) { s = PyUnicode_InternFromString(attrs[i]); if (!s) { Py_DECREF(l); - return 0; + return -1; } PyTuple_SET_ITEM(l, i, s); } - result = PyObject_SetAttr(type, state->_attributes, l) >= 0; + result = PyObject_SetAttr(type, state->_attributes, l); Py_DECREF(l); return result; } /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1017,7 +5718,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1027,7 +5728,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1039,7 +5740,7 @@ static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1077,7 +5778,7 @@ static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1086,7 +5787,7 @@ static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); - return 1; + return -1; } return obj2ast_object(state, obj, out, arena); } @@ -1096,12 +5797,12 @@ static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* o int i; if (!PyLong_Check(obj)) { PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); - return 1; + return -1; } i = PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) - return 1; + return -1; *out = i; return 0; } @@ -1126,47 +5827,40 @@ static int add_ast_fields(struct ast_state *state) static int init_types(struct ast_state *state) { - // init_types() must not be called after _PyAST_Fini() - // has been called - assert(state->initialized >= 0); - - if (state->initialized) { - return 1; - } if (init_identifiers(state) < 0) { - return 0; + return -1; } state->AST_type = PyType_FromSpec(&AST_type_spec); if (!state->AST_type) { - return 0; + return -1; } if (add_ast_fields(state) < 0) { - return 0; + return -1; } state->mod_type = make_type(state, "mod", state->AST_type, NULL, 0, "mod = Module(stmt* body, type_ignore* type_ignores)\n" " | Interactive(stmt* body)\n" " | Expression(expr body)\n" " | FunctionType(expr* argtypes, expr returns)"); - if (!state->mod_type) return 0; - if (!add_attributes(state, state->mod_type, NULL, 0)) return 0; + if (!state->mod_type) return -1; + if (add_attributes(state, state->mod_type, NULL, 0) < 0) return -1; state->Module_type = make_type(state, "Module", state->mod_type, Module_fields, 2, "Module(stmt* body, type_ignore* type_ignores)"); - if (!state->Module_type) return 0; + if (!state->Module_type) return -1; state->Interactive_type = make_type(state, "Interactive", state->mod_type, Interactive_fields, 1, "Interactive(stmt* body)"); - if (!state->Interactive_type) return 0; + if (!state->Interactive_type) return -1; state->Expression_type = make_type(state, "Expression", state->mod_type, Expression_fields, 1, "Expression(expr body)"); - if (!state->Expression_type) return 0; + if (!state->Expression_type) return -1; state->FunctionType_type = make_type(state, "FunctionType", state->mod_type, FunctionType_fields, 2, "FunctionType(expr* argtypes, expr returns)"); - if (!state->FunctionType_type) return 0; + if (!state->FunctionType_type) return -1; state->stmt_type = make_type(state, "stmt", state->AST_type, NULL, 0, "stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n" " | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n" @@ -1196,160 +5890,161 @@ init_types(struct ast_state *state) " | Pass\n" " | Break\n" " | Continue"); - if (!state->stmt_type) return 0; - if (!add_attributes(state, state->stmt_type, stmt_attributes, 4)) return 0; + if (!state->stmt_type) return -1; + if (add_attributes(state, state->stmt_type, stmt_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->stmt_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->stmt_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->FunctionDef_type = make_type(state, "FunctionDef", state->stmt_type, FunctionDef_fields, 7, "FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)"); - if (!state->FunctionDef_type) return 0; + if (!state->FunctionDef_type) return -1; if (PyObject_SetAttr(state->FunctionDef_type, state->returns, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->FunctionDef_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncFunctionDef_type = make_type(state, "AsyncFunctionDef", state->stmt_type, AsyncFunctionDef_fields, 7, "AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)"); - if (!state->AsyncFunctionDef_type) return 0; + if (!state->AsyncFunctionDef_type) return -1; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->returns, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->ClassDef_type = make_type(state, "ClassDef", state->stmt_type, ClassDef_fields, 6, "ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)"); - if (!state->ClassDef_type) return 0; + if (!state->ClassDef_type) return -1; state->Return_type = make_type(state, "Return", state->stmt_type, Return_fields, 1, "Return(expr? value)"); - if (!state->Return_type) return 0; + if (!state->Return_type) return -1; if (PyObject_SetAttr(state->Return_type, state->value, Py_None) == -1) - return 0; + return -1; state->Delete_type = make_type(state, "Delete", state->stmt_type, Delete_fields, 1, "Delete(expr* targets)"); - if (!state->Delete_type) return 0; + if (!state->Delete_type) return -1; state->Assign_type = make_type(state, "Assign", state->stmt_type, Assign_fields, 3, "Assign(expr* targets, expr value, string? type_comment)"); - if (!state->Assign_type) return 0; + if (!state->Assign_type) return -1; if (PyObject_SetAttr(state->Assign_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->TypeAlias_type = make_type(state, "TypeAlias", state->stmt_type, TypeAlias_fields, 3, "TypeAlias(expr name, type_param* type_params, expr value)"); - if (!state->TypeAlias_type) return 0; + if (!state->TypeAlias_type) return -1; state->AugAssign_type = make_type(state, "AugAssign", state->stmt_type, AugAssign_fields, 3, "AugAssign(expr target, operator op, expr value)"); - if (!state->AugAssign_type) return 0; + if (!state->AugAssign_type) return -1; state->AnnAssign_type = make_type(state, "AnnAssign", state->stmt_type, AnnAssign_fields, 4, "AnnAssign(expr target, expr annotation, expr? value, int simple)"); - if (!state->AnnAssign_type) return 0; + if (!state->AnnAssign_type) return -1; if (PyObject_SetAttr(state->AnnAssign_type, state->value, Py_None) == -1) - return 0; + return -1; state->For_type = make_type(state, "For", state->stmt_type, For_fields, 5, "For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)"); - if (!state->For_type) return 0; + if (!state->For_type) return -1; if (PyObject_SetAttr(state->For_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncFor_type = make_type(state, "AsyncFor", state->stmt_type, AsyncFor_fields, 5, "AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)"); - if (!state->AsyncFor_type) return 0; + if (!state->AsyncFor_type) return -1; if (PyObject_SetAttr(state->AsyncFor_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->While_type = make_type(state, "While", state->stmt_type, While_fields, 3, "While(expr test, stmt* body, stmt* orelse)"); - if (!state->While_type) return 0; + if (!state->While_type) return -1; state->If_type = make_type(state, "If", state->stmt_type, If_fields, 3, "If(expr test, stmt* body, stmt* orelse)"); - if (!state->If_type) return 0; + if (!state->If_type) return -1; state->With_type = make_type(state, "With", state->stmt_type, With_fields, 3, "With(withitem* items, stmt* body, string? type_comment)"); - if (!state->With_type) return 0; + if (!state->With_type) return -1; if (PyObject_SetAttr(state->With_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->AsyncWith_type = make_type(state, "AsyncWith", state->stmt_type, AsyncWith_fields, 3, "AsyncWith(withitem* items, stmt* body, string? type_comment)"); - if (!state->AsyncWith_type) return 0; + if (!state->AsyncWith_type) return -1; if (PyObject_SetAttr(state->AsyncWith_type, state->type_comment, Py_None) == -1) - return 0; + return -1; state->Match_type = make_type(state, "Match", state->stmt_type, Match_fields, 2, "Match(expr subject, match_case* cases)"); - if (!state->Match_type) return 0; + if (!state->Match_type) return -1; state->Raise_type = make_type(state, "Raise", state->stmt_type, Raise_fields, 2, "Raise(expr? exc, expr? cause)"); - if (!state->Raise_type) return 0; + if (!state->Raise_type) return -1; if (PyObject_SetAttr(state->Raise_type, state->exc, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Raise_type, state->cause, Py_None) == -1) - return 0; + return -1; state->Try_type = make_type(state, "Try", state->stmt_type, Try_fields, 4, "Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)"); - if (!state->Try_type) return 0; + if (!state->Try_type) return -1; state->TryStar_type = make_type(state, "TryStar", state->stmt_type, TryStar_fields, 4, "TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)"); - if (!state->TryStar_type) return 0; + if (!state->TryStar_type) return -1; state->Assert_type = make_type(state, "Assert", state->stmt_type, Assert_fields, 2, "Assert(expr test, expr? msg)"); - if (!state->Assert_type) return 0; + if (!state->Assert_type) return -1; if (PyObject_SetAttr(state->Assert_type, state->msg, Py_None) == -1) - return 0; + return -1; state->Import_type = make_type(state, "Import", state->stmt_type, Import_fields, 1, "Import(alias* names)"); - if (!state->Import_type) return 0; + if (!state->Import_type) return -1; state->ImportFrom_type = make_type(state, "ImportFrom", state->stmt_type, ImportFrom_fields, 3, "ImportFrom(identifier? module, alias* names, int? level)"); - if (!state->ImportFrom_type) return 0; + if (!state->ImportFrom_type) return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->module, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->ImportFrom_type, state->level, Py_None) == -1) - return 0; + return -1; state->Global_type = make_type(state, "Global", state->stmt_type, Global_fields, 1, "Global(identifier* names)"); - if (!state->Global_type) return 0; + if (!state->Global_type) return -1; state->Nonlocal_type = make_type(state, "Nonlocal", state->stmt_type, Nonlocal_fields, 1, "Nonlocal(identifier* names)"); - if (!state->Nonlocal_type) return 0; + if (!state->Nonlocal_type) return -1; state->Expr_type = make_type(state, "Expr", state->stmt_type, Expr_fields, 1, "Expr(expr value)"); - if (!state->Expr_type) return 0; + if (!state->Expr_type) return -1; state->Pass_type = make_type(state, "Pass", state->stmt_type, NULL, 0, "Pass"); - if (!state->Pass_type) return 0; + if (!state->Pass_type) return -1; state->Break_type = make_type(state, "Break", state->stmt_type, NULL, 0, "Break"); - if (!state->Break_type) return 0; + if (!state->Break_type) return -1; state->Continue_type = make_type(state, "Continue", state->stmt_type, NULL, 0, "Continue"); - if (!state->Continue_type) return 0; + if (!state->Continue_type) return -1; state->expr_type = make_type(state, "expr", state->AST_type, NULL, 0, "expr = BoolOp(boolop op, expr* values)\n" " | NamedExpr(expr target, expr value)\n" @@ -1378,454 +6073,457 @@ init_types(struct ast_state *state) " | List(expr* elts, expr_context ctx)\n" " | Tuple(expr* elts, expr_context ctx)\n" " | Slice(expr? lower, expr? upper, expr? step)"); - if (!state->expr_type) return 0; - if (!add_attributes(state, state->expr_type, expr_attributes, 4)) return 0; + if (!state->expr_type) return -1; + if (add_attributes(state, state->expr_type, expr_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->expr_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->BoolOp_type = make_type(state, "BoolOp", state->expr_type, BoolOp_fields, 2, "BoolOp(boolop op, expr* values)"); - if (!state->BoolOp_type) return 0; + if (!state->BoolOp_type) return -1; state->NamedExpr_type = make_type(state, "NamedExpr", state->expr_type, NamedExpr_fields, 2, "NamedExpr(expr target, expr value)"); - if (!state->NamedExpr_type) return 0; + if (!state->NamedExpr_type) return -1; state->BinOp_type = make_type(state, "BinOp", state->expr_type, BinOp_fields, 3, "BinOp(expr left, operator op, expr right)"); - if (!state->BinOp_type) return 0; + if (!state->BinOp_type) return -1; state->UnaryOp_type = make_type(state, "UnaryOp", state->expr_type, UnaryOp_fields, 2, "UnaryOp(unaryop op, expr operand)"); - if (!state->UnaryOp_type) return 0; + if (!state->UnaryOp_type) return -1; state->Lambda_type = make_type(state, "Lambda", state->expr_type, Lambda_fields, 2, "Lambda(arguments args, expr body)"); - if (!state->Lambda_type) return 0; + if (!state->Lambda_type) return -1; state->IfExp_type = make_type(state, "IfExp", state->expr_type, IfExp_fields, 3, "IfExp(expr test, expr body, expr orelse)"); - if (!state->IfExp_type) return 0; + if (!state->IfExp_type) return -1; state->Dict_type = make_type(state, "Dict", state->expr_type, Dict_fields, 2, "Dict(expr* keys, expr* values)"); - if (!state->Dict_type) return 0; + if (!state->Dict_type) return -1; state->Set_type = make_type(state, "Set", state->expr_type, Set_fields, 1, "Set(expr* elts)"); - if (!state->Set_type) return 0; + if (!state->Set_type) return -1; state->ListComp_type = make_type(state, "ListComp", state->expr_type, ListComp_fields, 2, "ListComp(expr elt, comprehension* generators)"); - if (!state->ListComp_type) return 0; + if (!state->ListComp_type) return -1; state->SetComp_type = make_type(state, "SetComp", state->expr_type, SetComp_fields, 2, "SetComp(expr elt, comprehension* generators)"); - if (!state->SetComp_type) return 0; + if (!state->SetComp_type) return -1; state->DictComp_type = make_type(state, "DictComp", state->expr_type, DictComp_fields, 3, "DictComp(expr key, expr value, comprehension* generators)"); - if (!state->DictComp_type) return 0; + if (!state->DictComp_type) return -1; state->GeneratorExp_type = make_type(state, "GeneratorExp", state->expr_type, GeneratorExp_fields, 2, "GeneratorExp(expr elt, comprehension* generators)"); - if (!state->GeneratorExp_type) return 0; + if (!state->GeneratorExp_type) return -1; state->Await_type = make_type(state, "Await", state->expr_type, Await_fields, 1, "Await(expr value)"); - if (!state->Await_type) return 0; + if (!state->Await_type) return -1; state->Yield_type = make_type(state, "Yield", state->expr_type, Yield_fields, 1, "Yield(expr? value)"); - if (!state->Yield_type) return 0; + if (!state->Yield_type) return -1; if (PyObject_SetAttr(state->Yield_type, state->value, Py_None) == -1) - return 0; + return -1; state->YieldFrom_type = make_type(state, "YieldFrom", state->expr_type, YieldFrom_fields, 1, "YieldFrom(expr value)"); - if (!state->YieldFrom_type) return 0; + if (!state->YieldFrom_type) return -1; state->Compare_type = make_type(state, "Compare", state->expr_type, Compare_fields, 3, "Compare(expr left, cmpop* ops, expr* comparators)"); - if (!state->Compare_type) return 0; + if (!state->Compare_type) return -1; state->Call_type = make_type(state, "Call", state->expr_type, Call_fields, 3, "Call(expr func, expr* args, keyword* keywords)"); - if (!state->Call_type) return 0; + if (!state->Call_type) return -1; state->FormattedValue_type = make_type(state, "FormattedValue", state->expr_type, FormattedValue_fields, 3, "FormattedValue(expr value, int conversion, expr? format_spec)"); - if (!state->FormattedValue_type) return 0; + if (!state->FormattedValue_type) return -1; if (PyObject_SetAttr(state->FormattedValue_type, state->format_spec, Py_None) == -1) - return 0; + return -1; state->JoinedStr_type = make_type(state, "JoinedStr", state->expr_type, JoinedStr_fields, 1, "JoinedStr(expr* values)"); - if (!state->JoinedStr_type) return 0; + if (!state->JoinedStr_type) return -1; state->Constant_type = make_type(state, "Constant", state->expr_type, Constant_fields, 2, "Constant(constant value, string? kind)"); - if (!state->Constant_type) return 0; + if (!state->Constant_type) return -1; if (PyObject_SetAttr(state->Constant_type, state->kind, Py_None) == -1) - return 0; + return -1; state->Attribute_type = make_type(state, "Attribute", state->expr_type, Attribute_fields, 3, "Attribute(expr value, identifier attr, expr_context ctx)"); - if (!state->Attribute_type) return 0; + if (!state->Attribute_type) return -1; state->Subscript_type = make_type(state, "Subscript", state->expr_type, Subscript_fields, 3, "Subscript(expr value, expr slice, expr_context ctx)"); - if (!state->Subscript_type) return 0; + if (!state->Subscript_type) return -1; state->Starred_type = make_type(state, "Starred", state->expr_type, Starred_fields, 2, "Starred(expr value, expr_context ctx)"); - if (!state->Starred_type) return 0; + if (!state->Starred_type) return -1; state->Name_type = make_type(state, "Name", state->expr_type, Name_fields, 2, "Name(identifier id, expr_context ctx)"); - if (!state->Name_type) return 0; + if (!state->Name_type) return -1; state->List_type = make_type(state, "List", state->expr_type, List_fields, 2, "List(expr* elts, expr_context ctx)"); - if (!state->List_type) return 0; + if (!state->List_type) return -1; state->Tuple_type = make_type(state, "Tuple", state->expr_type, Tuple_fields, 2, "Tuple(expr* elts, expr_context ctx)"); - if (!state->Tuple_type) return 0; + if (!state->Tuple_type) return -1; state->Slice_type = make_type(state, "Slice", state->expr_type, Slice_fields, 3, "Slice(expr? lower, expr? upper, expr? step)"); - if (!state->Slice_type) return 0; + if (!state->Slice_type) return -1; if (PyObject_SetAttr(state->Slice_type, state->lower, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Slice_type, state->upper, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->Slice_type, state->step, Py_None) == -1) - return 0; + return -1; state->expr_context_type = make_type(state, "expr_context", state->AST_type, NULL, 0, "expr_context = Load | Store | Del"); - if (!state->expr_context_type) return 0; - if (!add_attributes(state, state->expr_context_type, NULL, 0)) return 0; + if (!state->expr_context_type) return -1; + if (add_attributes(state, state->expr_context_type, NULL, 0) < 0) return -1; state->Load_type = make_type(state, "Load", state->expr_context_type, NULL, 0, "Load"); - if (!state->Load_type) return 0; + if (!state->Load_type) return -1; state->Load_singleton = PyType_GenericNew((PyTypeObject *)state->Load_type, NULL, NULL); - if (!state->Load_singleton) return 0; + if (!state->Load_singleton) return -1; state->Store_type = make_type(state, "Store", state->expr_context_type, NULL, 0, "Store"); - if (!state->Store_type) return 0; + if (!state->Store_type) return -1; state->Store_singleton = PyType_GenericNew((PyTypeObject *)state->Store_type, NULL, NULL); - if (!state->Store_singleton) return 0; + if (!state->Store_singleton) return -1; state->Del_type = make_type(state, "Del", state->expr_context_type, NULL, 0, "Del"); - if (!state->Del_type) return 0; + if (!state->Del_type) return -1; state->Del_singleton = PyType_GenericNew((PyTypeObject *)state->Del_type, NULL, NULL); - if (!state->Del_singleton) return 0; + if (!state->Del_singleton) return -1; state->boolop_type = make_type(state, "boolop", state->AST_type, NULL, 0, "boolop = And | Or"); - if (!state->boolop_type) return 0; - if (!add_attributes(state, state->boolop_type, NULL, 0)) return 0; + if (!state->boolop_type) return -1; + if (add_attributes(state, state->boolop_type, NULL, 0) < 0) return -1; state->And_type = make_type(state, "And", state->boolop_type, NULL, 0, "And"); - if (!state->And_type) return 0; + if (!state->And_type) return -1; state->And_singleton = PyType_GenericNew((PyTypeObject *)state->And_type, NULL, NULL); - if (!state->And_singleton) return 0; + if (!state->And_singleton) return -1; state->Or_type = make_type(state, "Or", state->boolop_type, NULL, 0, "Or"); - if (!state->Or_type) return 0; + if (!state->Or_type) return -1; state->Or_singleton = PyType_GenericNew((PyTypeObject *)state->Or_type, NULL, NULL); - if (!state->Or_singleton) return 0; + if (!state->Or_singleton) return -1; state->operator_type = make_type(state, "operator", state->AST_type, NULL, 0, "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv"); - if (!state->operator_type) return 0; - if (!add_attributes(state, state->operator_type, NULL, 0)) return 0; + if (!state->operator_type) return -1; + if (add_attributes(state, state->operator_type, NULL, 0) < 0) return -1; state->Add_type = make_type(state, "Add", state->operator_type, NULL, 0, "Add"); - if (!state->Add_type) return 0; + if (!state->Add_type) return -1; state->Add_singleton = PyType_GenericNew((PyTypeObject *)state->Add_type, NULL, NULL); - if (!state->Add_singleton) return 0; + if (!state->Add_singleton) return -1; state->Sub_type = make_type(state, "Sub", state->operator_type, NULL, 0, "Sub"); - if (!state->Sub_type) return 0; + if (!state->Sub_type) return -1; state->Sub_singleton = PyType_GenericNew((PyTypeObject *)state->Sub_type, NULL, NULL); - if (!state->Sub_singleton) return 0; + if (!state->Sub_singleton) return -1; state->Mult_type = make_type(state, "Mult", state->operator_type, NULL, 0, "Mult"); - if (!state->Mult_type) return 0; + if (!state->Mult_type) return -1; state->Mult_singleton = PyType_GenericNew((PyTypeObject *)state->Mult_type, NULL, NULL); - if (!state->Mult_singleton) return 0; + if (!state->Mult_singleton) return -1; state->MatMult_type = make_type(state, "MatMult", state->operator_type, NULL, 0, "MatMult"); - if (!state->MatMult_type) return 0; + if (!state->MatMult_type) return -1; state->MatMult_singleton = PyType_GenericNew((PyTypeObject *)state->MatMult_type, NULL, NULL); - if (!state->MatMult_singleton) return 0; + if (!state->MatMult_singleton) return -1; state->Div_type = make_type(state, "Div", state->operator_type, NULL, 0, "Div"); - if (!state->Div_type) return 0; + if (!state->Div_type) return -1; state->Div_singleton = PyType_GenericNew((PyTypeObject *)state->Div_type, NULL, NULL); - if (!state->Div_singleton) return 0; + if (!state->Div_singleton) return -1; state->Mod_type = make_type(state, "Mod", state->operator_type, NULL, 0, "Mod"); - if (!state->Mod_type) return 0; + if (!state->Mod_type) return -1; state->Mod_singleton = PyType_GenericNew((PyTypeObject *)state->Mod_type, NULL, NULL); - if (!state->Mod_singleton) return 0; + if (!state->Mod_singleton) return -1; state->Pow_type = make_type(state, "Pow", state->operator_type, NULL, 0, "Pow"); - if (!state->Pow_type) return 0; + if (!state->Pow_type) return -1; state->Pow_singleton = PyType_GenericNew((PyTypeObject *)state->Pow_type, NULL, NULL); - if (!state->Pow_singleton) return 0; + if (!state->Pow_singleton) return -1; state->LShift_type = make_type(state, "LShift", state->operator_type, NULL, 0, "LShift"); - if (!state->LShift_type) return 0; + if (!state->LShift_type) return -1; state->LShift_singleton = PyType_GenericNew((PyTypeObject *)state->LShift_type, NULL, NULL); - if (!state->LShift_singleton) return 0; + if (!state->LShift_singleton) return -1; state->RShift_type = make_type(state, "RShift", state->operator_type, NULL, 0, "RShift"); - if (!state->RShift_type) return 0; + if (!state->RShift_type) return -1; state->RShift_singleton = PyType_GenericNew((PyTypeObject *)state->RShift_type, NULL, NULL); - if (!state->RShift_singleton) return 0; + if (!state->RShift_singleton) return -1; state->BitOr_type = make_type(state, "BitOr", state->operator_type, NULL, 0, "BitOr"); - if (!state->BitOr_type) return 0; + if (!state->BitOr_type) return -1; state->BitOr_singleton = PyType_GenericNew((PyTypeObject *)state->BitOr_type, NULL, NULL); - if (!state->BitOr_singleton) return 0; + if (!state->BitOr_singleton) return -1; state->BitXor_type = make_type(state, "BitXor", state->operator_type, NULL, 0, "BitXor"); - if (!state->BitXor_type) return 0; + if (!state->BitXor_type) return -1; state->BitXor_singleton = PyType_GenericNew((PyTypeObject *)state->BitXor_type, NULL, NULL); - if (!state->BitXor_singleton) return 0; + if (!state->BitXor_singleton) return -1; state->BitAnd_type = make_type(state, "BitAnd", state->operator_type, NULL, 0, "BitAnd"); - if (!state->BitAnd_type) return 0; + if (!state->BitAnd_type) return -1; state->BitAnd_singleton = PyType_GenericNew((PyTypeObject *)state->BitAnd_type, NULL, NULL); - if (!state->BitAnd_singleton) return 0; + if (!state->BitAnd_singleton) return -1; state->FloorDiv_type = make_type(state, "FloorDiv", state->operator_type, NULL, 0, "FloorDiv"); - if (!state->FloorDiv_type) return 0; + if (!state->FloorDiv_type) return -1; state->FloorDiv_singleton = PyType_GenericNew((PyTypeObject *)state->FloorDiv_type, NULL, NULL); - if (!state->FloorDiv_singleton) return 0; + if (!state->FloorDiv_singleton) return -1; state->unaryop_type = make_type(state, "unaryop", state->AST_type, NULL, 0, "unaryop = Invert | Not | UAdd | USub"); - if (!state->unaryop_type) return 0; - if (!add_attributes(state, state->unaryop_type, NULL, 0)) return 0; + if (!state->unaryop_type) return -1; + if (add_attributes(state, state->unaryop_type, NULL, 0) < 0) return -1; state->Invert_type = make_type(state, "Invert", state->unaryop_type, NULL, 0, "Invert"); - if (!state->Invert_type) return 0; + if (!state->Invert_type) return -1; state->Invert_singleton = PyType_GenericNew((PyTypeObject *)state->Invert_type, NULL, NULL); - if (!state->Invert_singleton) return 0; + if (!state->Invert_singleton) return -1; state->Not_type = make_type(state, "Not", state->unaryop_type, NULL, 0, "Not"); - if (!state->Not_type) return 0; + if (!state->Not_type) return -1; state->Not_singleton = PyType_GenericNew((PyTypeObject *)state->Not_type, NULL, NULL); - if (!state->Not_singleton) return 0; + if (!state->Not_singleton) return -1; state->UAdd_type = make_type(state, "UAdd", state->unaryop_type, NULL, 0, "UAdd"); - if (!state->UAdd_type) return 0; + if (!state->UAdd_type) return -1; state->UAdd_singleton = PyType_GenericNew((PyTypeObject *)state->UAdd_type, NULL, NULL); - if (!state->UAdd_singleton) return 0; + if (!state->UAdd_singleton) return -1; state->USub_type = make_type(state, "USub", state->unaryop_type, NULL, 0, "USub"); - if (!state->USub_type) return 0; + if (!state->USub_type) return -1; state->USub_singleton = PyType_GenericNew((PyTypeObject *)state->USub_type, NULL, NULL); - if (!state->USub_singleton) return 0; + if (!state->USub_singleton) return -1; state->cmpop_type = make_type(state, "cmpop", state->AST_type, NULL, 0, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn"); - if (!state->cmpop_type) return 0; - if (!add_attributes(state, state->cmpop_type, NULL, 0)) return 0; + if (!state->cmpop_type) return -1; + if (add_attributes(state, state->cmpop_type, NULL, 0) < 0) return -1; state->Eq_type = make_type(state, "Eq", state->cmpop_type, NULL, 0, "Eq"); - if (!state->Eq_type) return 0; + if (!state->Eq_type) return -1; state->Eq_singleton = PyType_GenericNew((PyTypeObject *)state->Eq_type, NULL, NULL); - if (!state->Eq_singleton) return 0; + if (!state->Eq_singleton) return -1; state->NotEq_type = make_type(state, "NotEq", state->cmpop_type, NULL, 0, "NotEq"); - if (!state->NotEq_type) return 0; + if (!state->NotEq_type) return -1; state->NotEq_singleton = PyType_GenericNew((PyTypeObject *)state->NotEq_type, NULL, NULL); - if (!state->NotEq_singleton) return 0; + if (!state->NotEq_singleton) return -1; state->Lt_type = make_type(state, "Lt", state->cmpop_type, NULL, 0, "Lt"); - if (!state->Lt_type) return 0; + if (!state->Lt_type) return -1; state->Lt_singleton = PyType_GenericNew((PyTypeObject *)state->Lt_type, NULL, NULL); - if (!state->Lt_singleton) return 0; + if (!state->Lt_singleton) return -1; state->LtE_type = make_type(state, "LtE", state->cmpop_type, NULL, 0, "LtE"); - if (!state->LtE_type) return 0; + if (!state->LtE_type) return -1; state->LtE_singleton = PyType_GenericNew((PyTypeObject *)state->LtE_type, NULL, NULL); - if (!state->LtE_singleton) return 0; + if (!state->LtE_singleton) return -1; state->Gt_type = make_type(state, "Gt", state->cmpop_type, NULL, 0, "Gt"); - if (!state->Gt_type) return 0; + if (!state->Gt_type) return -1; state->Gt_singleton = PyType_GenericNew((PyTypeObject *)state->Gt_type, NULL, NULL); - if (!state->Gt_singleton) return 0; + if (!state->Gt_singleton) return -1; state->GtE_type = make_type(state, "GtE", state->cmpop_type, NULL, 0, "GtE"); - if (!state->GtE_type) return 0; + if (!state->GtE_type) return -1; state->GtE_singleton = PyType_GenericNew((PyTypeObject *)state->GtE_type, NULL, NULL); - if (!state->GtE_singleton) return 0; + if (!state->GtE_singleton) return -1; state->Is_type = make_type(state, "Is", state->cmpop_type, NULL, 0, "Is"); - if (!state->Is_type) return 0; + if (!state->Is_type) return -1; state->Is_singleton = PyType_GenericNew((PyTypeObject *)state->Is_type, NULL, NULL); - if (!state->Is_singleton) return 0; + if (!state->Is_singleton) return -1; state->IsNot_type = make_type(state, "IsNot", state->cmpop_type, NULL, 0, "IsNot"); - if (!state->IsNot_type) return 0; + if (!state->IsNot_type) return -1; state->IsNot_singleton = PyType_GenericNew((PyTypeObject *)state->IsNot_type, NULL, NULL); - if (!state->IsNot_singleton) return 0; + if (!state->IsNot_singleton) return -1; state->In_type = make_type(state, "In", state->cmpop_type, NULL, 0, "In"); - if (!state->In_type) return 0; + if (!state->In_type) return -1; state->In_singleton = PyType_GenericNew((PyTypeObject *)state->In_type, NULL, NULL); - if (!state->In_singleton) return 0; + if (!state->In_singleton) return -1; state->NotIn_type = make_type(state, "NotIn", state->cmpop_type, NULL, 0, "NotIn"); - if (!state->NotIn_type) return 0; + if (!state->NotIn_type) return -1; state->NotIn_singleton = PyType_GenericNew((PyTypeObject *)state->NotIn_type, NULL, NULL); - if (!state->NotIn_singleton) return 0; + if (!state->NotIn_singleton) return -1; state->comprehension_type = make_type(state, "comprehension", state->AST_type, comprehension_fields, 4, "comprehension(expr target, expr iter, expr* ifs, int is_async)"); - if (!state->comprehension_type) return 0; - if (!add_attributes(state, state->comprehension_type, NULL, 0)) return 0; + if (!state->comprehension_type) return -1; + if (add_attributes(state, state->comprehension_type, NULL, 0) < 0) return + -1; state->excepthandler_type = make_type(state, "excepthandler", state->AST_type, NULL, 0, "excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)"); - if (!state->excepthandler_type) return 0; - if (!add_attributes(state, state->excepthandler_type, - excepthandler_attributes, 4)) return 0; + if (!state->excepthandler_type) return -1; + if (add_attributes(state, state->excepthandler_type, + excepthandler_attributes, 4) < 0) return -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->ExceptHandler_type = make_type(state, "ExceptHandler", state->excepthandler_type, ExceptHandler_fields, 3, "ExceptHandler(expr? type, identifier? name, stmt* body)"); - if (!state->ExceptHandler_type) return 0; + if (!state->ExceptHandler_type) return -1; if (PyObject_SetAttr(state->ExceptHandler_type, state->type, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->ExceptHandler_type, state->name, Py_None) == -1) - return 0; + return -1; state->arguments_type = make_type(state, "arguments", state->AST_type, arguments_fields, 7, "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults)"); - if (!state->arguments_type) return 0; - if (!add_attributes(state, state->arguments_type, NULL, 0)) return 0; + if (!state->arguments_type) return -1; + if (add_attributes(state, state->arguments_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->arguments_type, state->vararg, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arguments_type, state->kwarg, Py_None) == -1) - return 0; + return -1; state->arg_type = make_type(state, "arg", state->AST_type, arg_fields, 3, "arg(identifier arg, expr? annotation, string? type_comment)"); - if (!state->arg_type) return 0; - if (!add_attributes(state, state->arg_type, arg_attributes, 4)) return 0; + if (!state->arg_type) return -1; + if (add_attributes(state, state->arg_type, arg_attributes, 4) < 0) return + -1; if (PyObject_SetAttr(state->arg_type, state->annotation, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->type_comment, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->arg_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->keyword_type = make_type(state, "keyword", state->AST_type, keyword_fields, 2, "keyword(identifier? arg, expr value)"); - if (!state->keyword_type) return 0; - if (!add_attributes(state, state->keyword_type, keyword_attributes, 4)) - return 0; + if (!state->keyword_type) return -1; + if (add_attributes(state, state->keyword_type, keyword_attributes, 4) < 0) + return -1; if (PyObject_SetAttr(state->keyword_type, state->arg, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->keyword_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->keyword_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->alias_type = make_type(state, "alias", state->AST_type, alias_fields, 2, "alias(identifier name, identifier? asname)"); - if (!state->alias_type) return 0; - if (!add_attributes(state, state->alias_type, alias_attributes, 4)) return - 0; + if (!state->alias_type) return -1; + if (add_attributes(state, state->alias_type, alias_attributes, 4) < 0) + return -1; if (PyObject_SetAttr(state->alias_type, state->asname, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->alias_type, state->end_lineno, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->alias_type, state->end_col_offset, Py_None) == -1) - return 0; + return -1; state->withitem_type = make_type(state, "withitem", state->AST_type, withitem_fields, 2, "withitem(expr context_expr, expr? optional_vars)"); - if (!state->withitem_type) return 0; - if (!add_attributes(state, state->withitem_type, NULL, 0)) return 0; + if (!state->withitem_type) return -1; + if (add_attributes(state, state->withitem_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->withitem_type, state->optional_vars, Py_None) == -1) - return 0; + return -1; state->match_case_type = make_type(state, "match_case", state->AST_type, match_case_fields, 3, "match_case(pattern pattern, expr? guard, stmt* body)"); - if (!state->match_case_type) return 0; - if (!add_attributes(state, state->match_case_type, NULL, 0)) return 0; + if (!state->match_case_type) return -1; + if (add_attributes(state, state->match_case_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->match_case_type, state->guard, Py_None) == -1) - return 0; + return -1; state->pattern_type = make_type(state, "pattern", state->AST_type, NULL, 0, "pattern = MatchValue(expr value)\n" " | MatchSingleton(constant value)\n" @@ -1835,93 +6533,102 @@ init_types(struct ast_state *state) " | MatchStar(identifier? name)\n" " | MatchAs(pattern? pattern, identifier? name)\n" " | MatchOr(pattern* patterns)"); - if (!state->pattern_type) return 0; - if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) - return 0; + if (!state->pattern_type) return -1; + if (add_attributes(state, state->pattern_type, pattern_attributes, 4) < 0) + return -1; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, "MatchValue(expr value)"); - if (!state->MatchValue_type) return 0; + if (!state->MatchValue_type) return -1; state->MatchSingleton_type = make_type(state, "MatchSingleton", state->pattern_type, MatchSingleton_fields, 1, "MatchSingleton(constant value)"); - if (!state->MatchSingleton_type) return 0; + if (!state->MatchSingleton_type) return -1; state->MatchSequence_type = make_type(state, "MatchSequence", state->pattern_type, MatchSequence_fields, 1, "MatchSequence(pattern* patterns)"); - if (!state->MatchSequence_type) return 0; + if (!state->MatchSequence_type) return -1; state->MatchMapping_type = make_type(state, "MatchMapping", state->pattern_type, MatchMapping_fields, 3, "MatchMapping(expr* keys, pattern* patterns, identifier? rest)"); - if (!state->MatchMapping_type) return 0; + if (!state->MatchMapping_type) return -1; if (PyObject_SetAttr(state->MatchMapping_type, state->rest, Py_None) == -1) - return 0; + return -1; state->MatchClass_type = make_type(state, "MatchClass", state->pattern_type, MatchClass_fields, 4, "MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)"); - if (!state->MatchClass_type) return 0; + if (!state->MatchClass_type) return -1; state->MatchStar_type = make_type(state, "MatchStar", state->pattern_type, MatchStar_fields, 1, "MatchStar(identifier? name)"); - if (!state->MatchStar_type) return 0; + if (!state->MatchStar_type) return -1; if (PyObject_SetAttr(state->MatchStar_type, state->name, Py_None) == -1) - return 0; + return -1; state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, MatchAs_fields, 2, "MatchAs(pattern? pattern, identifier? name)"); - if (!state->MatchAs_type) return 0; + if (!state->MatchAs_type) return -1; if (PyObject_SetAttr(state->MatchAs_type, state->pattern, Py_None) == -1) - return 0; + return -1; if (PyObject_SetAttr(state->MatchAs_type, state->name, Py_None) == -1) - return 0; + return -1; state->MatchOr_type = make_type(state, "MatchOr", state->pattern_type, MatchOr_fields, 1, "MatchOr(pattern* patterns)"); - if (!state->MatchOr_type) return 0; + if (!state->MatchOr_type) return -1; state->type_ignore_type = make_type(state, "type_ignore", state->AST_type, NULL, 0, "type_ignore = TypeIgnore(int lineno, string tag)"); - if (!state->type_ignore_type) return 0; - if (!add_attributes(state, state->type_ignore_type, NULL, 0)) return 0; + if (!state->type_ignore_type) return -1; + if (add_attributes(state, state->type_ignore_type, NULL, 0) < 0) return -1; state->TypeIgnore_type = make_type(state, "TypeIgnore", state->type_ignore_type, TypeIgnore_fields, 2, "TypeIgnore(int lineno, string tag)"); - if (!state->TypeIgnore_type) return 0; + if (!state->TypeIgnore_type) return -1; state->type_param_type = make_type(state, "type_param", state->AST_type, NULL, 0, - "type_param = TypeVar(identifier name, expr? bound)\n" - " | ParamSpec(identifier name)\n" - " | TypeVarTuple(identifier name)"); - if (!state->type_param_type) return 0; - if (!add_attributes(state, state->type_param_type, type_param_attributes, - 4)) return 0; + "type_param = TypeVar(identifier name, expr? bound, expr? default_value)\n" + " | ParamSpec(identifier name, expr? default_value)\n" + " | TypeVarTuple(identifier name, expr? default_value)"); + if (!state->type_param_type) return -1; + if (add_attributes(state, state->type_param_type, type_param_attributes, 4) + < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, - TypeVar_fields, 2, - "TypeVar(identifier name, expr? bound)"); - if (!state->TypeVar_type) return 0; + TypeVar_fields, 3, + "TypeVar(identifier name, expr? bound, expr? default_value)"); + if (!state->TypeVar_type) return -1; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) - return 0; + return -1; + if (PyObject_SetAttr(state->TypeVar_type, state->default_value, Py_None) == + -1) + return -1; state->ParamSpec_type = make_type(state, "ParamSpec", state->type_param_type, ParamSpec_fields, - 1, - "ParamSpec(identifier name)"); - if (!state->ParamSpec_type) return 0; + 2, + "ParamSpec(identifier name, expr? default_value)"); + if (!state->ParamSpec_type) return -1; + if (PyObject_SetAttr(state->ParamSpec_type, state->default_value, Py_None) + == -1) + return -1; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", state->type_param_type, - TypeVarTuple_fields, 1, - "TypeVarTuple(identifier name)"); - if (!state->TypeVarTuple_type) return 0; + TypeVarTuple_fields, 2, + "TypeVarTuple(identifier name, expr? default_value)"); + if (!state->TypeVarTuple_type) return -1; + if (PyObject_SetAttr(state->TypeVarTuple_type, state->default_value, + Py_None) == -1) + return -1; - state->recursion_depth = 0; - state->recursion_limit = 0; - state->initialized = 1; - return 1; + if (!add_ast_annotations(state)) { + return -1; + } + return 0; } static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, @@ -3707,8 +8414,9 @@ _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) } type_param_ty -_PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVar(identifier name, expr_ty bound, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { type_param_ty p; if (!name) { @@ -3722,6 +8430,7 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int p->kind = TypeVar_kind; p->v.TypeVar.name = name; p->v.TypeVar.bound = bound; + p->v.TypeVar.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3730,8 +8439,8 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int } type_param_ty -_PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena) +_PyAST_ParamSpec(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { type_param_ty p; if (!name) { @@ -3744,6 +8453,7 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, return NULL; p->kind = ParamSpec_kind; p->v.ParamSpec.name = name; + p->v.ParamSpec.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3752,8 +8462,9 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, } type_param_ty -_PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVarTuple(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { type_param_ty p; if (!name) { @@ -3766,6 +8477,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int return NULL; p->kind = TypeVarTuple_kind; p->v.TypeVarTuple.name = name; + p->v.TypeVarTuple.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3775,7 +8487,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int PyObject* -ast2obj_mod(struct ast_state *state, void* _o) +ast2obj_mod(struct ast_state *state, struct validator *vstate, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3783,22 +8495,24 @@ ast2obj_mod(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case Module_kind: tp = (PyTypeObject *)state->Module_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Module.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Module.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Module.type_ignores, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Module.type_ignores, ast2obj_type_ignore); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_ignores, value) == -1) @@ -3809,7 +8523,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Interactive_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Interactive.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Interactive.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -3820,7 +8534,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expression_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expression.body); + value = ast2obj_expr(state, vstate, o->v.Expression.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -3830,29 +8544,31 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FunctionType_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionType.argtypes, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionType.argtypes, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->argtypes, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionType.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionType.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_stmt(struct ast_state *state, void* _o) +ast2obj_stmt(struct ast_state *state, struct validator *vstate, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3860,49 +8576,51 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case FunctionDef_kind: tp = (PyTypeObject *)state->FunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.FunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.FunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.FunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.FunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.FunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.FunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.FunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3913,40 +8631,41 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.AsyncFunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.AsyncFunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.AsyncFunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.AsyncFunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFunctionDef.body, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.AsyncFunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.AsyncFunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; @@ -3958,36 +8677,38 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ClassDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ClassDef.name); + value = ast2obj_identifier(state, vstate, o->v.ClassDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.bases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.bases, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->bases, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3998,7 +8719,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Return_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Return.value); + value = ast2obj_expr(state, vstate, o->v.Return.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4008,7 +8729,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Delete_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Delete.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Delete.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) @@ -4019,18 +8740,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Assign.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Assign.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assign.value); + value = ast2obj_expr(state, vstate, o->v.Assign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Assign.type_comment); + value = ast2obj_string(state, vstate, o->v.Assign.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4040,18 +8761,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeAlias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.TypeAlias.name); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.TypeAlias.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeAlias.value); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4061,17 +8783,17 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AugAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AugAssign.target); + value = ast2obj_expr(state, vstate, o->v.AugAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.AugAssign.op); + value = ast2obj_operator(state, vstate, o->v.AugAssign.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AugAssign.value); + value = ast2obj_expr(state, vstate, o->v.AugAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4081,22 +8803,22 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AnnAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AnnAssign.target); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.annotation); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.value); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.AnnAssign.simple); + value = ast2obj_int(state, vstate, o->v.AnnAssign.simple); if (!value) goto failed; if (PyObject_SetAttr(result, state->simple, value) == -1) goto failed; @@ -4106,27 +8828,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->For_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.For.target); + value = ast2obj_expr(state, vstate, o->v.For.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.For.iter); + value = ast2obj_expr(state, vstate, o->v.For.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.For.type_comment); + value = ast2obj_string(state, vstate, o->v.For.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4136,29 +8860,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFor_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AsyncFor.target); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFor.iter); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFor.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFor.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4168,17 +8892,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->While_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.While.test); + value = ast2obj_expr(state, vstate, o->v.While.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4188,17 +8914,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->If_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.If.test); + value = ast2obj_expr(state, vstate, o->v.If.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4208,18 +8936,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->With_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.With.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.With.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.With.type_comment); + value = ast2obj_string(state, vstate, o->v.With.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4229,19 +8958,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncWith_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncWith.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncWith.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4251,12 +8980,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Match_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Match.subject); + value = ast2obj_expr(state, vstate, o->v.Match.subject); if (!value) goto failed; if (PyObject_SetAttr(result, state->subject, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Match.cases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Match.cases, ast2obj_match_case); if (!value) goto failed; if (PyObject_SetAttr(result, state->cases, value) == -1) @@ -4267,12 +8996,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Raise_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Raise.exc); + value = ast2obj_expr(state, vstate, o->v.Raise.exc); if (!value) goto failed; if (PyObject_SetAttr(result, state->exc, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Raise.cause); + value = ast2obj_expr(state, vstate, o->v.Raise.cause); if (!value) goto failed; if (PyObject_SetAttr(result, state->cause, value) == -1) goto failed; @@ -4282,23 +9011,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Try_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Try.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4309,24 +9040,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TryStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4337,12 +9069,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assert_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Assert.test); + value = ast2obj_expr(state, vstate, o->v.Assert.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assert.msg); + value = ast2obj_expr(state, vstate, o->v.Assert.msg); if (!value) goto failed; if (PyObject_SetAttr(result, state->msg, value) == -1) goto failed; @@ -4352,7 +9084,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Import_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Import.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Import.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4363,18 +9095,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ImportFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ImportFrom.module); + value = ast2obj_identifier(state, vstate, o->v.ImportFrom.module); if (!value) goto failed; if (PyObject_SetAttr(result, state->module, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ImportFrom.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ImportFrom.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.ImportFrom.level); + value = ast2obj_int(state, vstate, o->v.ImportFrom.level); if (!value) goto failed; if (PyObject_SetAttr(result, state->level, value) == -1) goto failed; @@ -4384,7 +9116,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Global_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Global.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Global.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4395,7 +9127,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Nonlocal_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Nonlocal.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Nonlocal.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4406,7 +9138,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expr.value); + value = ast2obj_expr(state, vstate, o->v.Expr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4428,36 +9160,37 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!result) goto failed; break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_expr(struct ast_state *state, void* _o) +ast2obj_expr(struct ast_state *state, struct validator *vstate, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4465,22 +9198,22 @@ ast2obj_expr(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case BoolOp_kind: tp = (PyTypeObject *)state->BoolOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_boolop(state, o->v.BoolOp.op); + value = ast2obj_boolop(state, vstate, o->v.BoolOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.BoolOp.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.BoolOp.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4491,12 +9224,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->NamedExpr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.NamedExpr.target); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.NamedExpr.value); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4506,17 +9239,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->BinOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.BinOp.left); + value = ast2obj_expr(state, vstate, o->v.BinOp.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.BinOp.op); + value = ast2obj_operator(state, vstate, o->v.BinOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.BinOp.right); + value = ast2obj_expr(state, vstate, o->v.BinOp.right); if (!value) goto failed; if (PyObject_SetAttr(result, state->right, value) == -1) goto failed; @@ -4526,12 +9259,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->UnaryOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_unaryop(state, o->v.UnaryOp.op); + value = ast2obj_unaryop(state, vstate, o->v.UnaryOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.UnaryOp.operand); + value = ast2obj_expr(state, vstate, o->v.UnaryOp.operand); if (!value) goto failed; if (PyObject_SetAttr(result, state->operand, value) == -1) goto failed; @@ -4541,12 +9274,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Lambda_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_arguments(state, o->v.Lambda.args); + value = ast2obj_arguments(state, vstate, o->v.Lambda.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Lambda.body); + value = ast2obj_expr(state, vstate, o->v.Lambda.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -4556,17 +9289,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->IfExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.IfExp.test); + value = ast2obj_expr(state, vstate, o->v.IfExp.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.body); + value = ast2obj_expr(state, vstate, o->v.IfExp.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.orelse); + value = ast2obj_expr(state, vstate, o->v.IfExp.orelse); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4576,12 +9309,14 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Dict_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.keys, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.keys, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.values, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.values, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) goto failed; @@ -4591,7 +9326,8 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Set_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Set.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Set.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; @@ -4601,12 +9337,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ListComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ListComp.elt); + value = ast2obj_expr(state, vstate, o->v.ListComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ListComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ListComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4617,12 +9354,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->SetComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.SetComp.elt); + value = ast2obj_expr(state, vstate, o->v.SetComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.SetComp.generators, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.SetComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4633,17 +9370,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->DictComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.DictComp.key); + value = ast2obj_expr(state, vstate, o->v.DictComp.key); if (!value) goto failed; if (PyObject_SetAttr(result, state->key, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.DictComp.value); + value = ast2obj_expr(state, vstate, o->v.DictComp.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.DictComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.DictComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4654,12 +9392,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->GeneratorExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.GeneratorExp.elt); + value = ast2obj_expr(state, vstate, o->v.GeneratorExp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.GeneratorExp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.GeneratorExp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4670,7 +9409,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Await_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Await.value); + value = ast2obj_expr(state, vstate, o->v.Await.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4680,7 +9419,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Yield_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Yield.value); + value = ast2obj_expr(state, vstate, o->v.Yield.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4690,7 +9429,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->YieldFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.YieldFrom.value); + value = ast2obj_expr(state, vstate, o->v.YieldFrom.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4700,7 +9439,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Compare_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Compare.left); + value = ast2obj_expr(state, vstate, o->v.Compare.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; @@ -4710,14 +9449,14 @@ ast2obj_expr(struct ast_state *state, void* _o) value = PyList_New(n); if (!value) goto failed; for(i = 0; i < n; i++) - PyList_SET_ITEM(value, i, ast2obj_cmpop(state, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); + PyList_SET_ITEM(value, i, ast2obj_cmpop(state, vstate, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); } if (!value) goto failed; if (PyObject_SetAttr(result, state->ops, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Compare.comparators, - ast2obj_expr); + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Compare.comparators, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->comparators, value) == -1) goto failed; @@ -4727,17 +9466,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Call_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Call.func); + value = ast2obj_expr(state, vstate, o->v.Call.func); if (!value) goto failed; if (PyObject_SetAttr(result, state->func, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.args, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.args, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) @@ -4748,17 +9488,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FormattedValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.FormattedValue.value); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.FormattedValue.conversion); + value = ast2obj_int(state, vstate, o->v.FormattedValue.conversion); if (!value) goto failed; if (PyObject_SetAttr(result, state->conversion, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FormattedValue.format_spec); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.format_spec); if (!value) goto failed; if (PyObject_SetAttr(result, state->format_spec, value) == -1) goto failed; @@ -4768,7 +9508,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->JoinedStr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.JoinedStr.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.JoinedStr.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4779,12 +9519,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Constant_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.Constant.value); + value = ast2obj_constant(state, vstate, o->v.Constant.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Constant.kind); + value = ast2obj_string(state, vstate, o->v.Constant.kind); if (!value) goto failed; if (PyObject_SetAttr(result, state->kind, value) == -1) goto failed; @@ -4794,17 +9534,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Attribute_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Attribute.value); + value = ast2obj_expr(state, vstate, o->v.Attribute.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.Attribute.attr); + value = ast2obj_identifier(state, vstate, o->v.Attribute.attr); if (!value) goto failed; if (PyObject_SetAttr(result, state->attr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Attribute.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Attribute.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4814,17 +9554,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Subscript_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Subscript.value); + value = ast2obj_expr(state, vstate, o->v.Subscript.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Subscript.slice); + value = ast2obj_expr(state, vstate, o->v.Subscript.slice); if (!value) goto failed; if (PyObject_SetAttr(result, state->slice, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Subscript.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Subscript.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4834,12 +9574,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Starred_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Starred.value); + value = ast2obj_expr(state, vstate, o->v.Starred.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Starred.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Starred.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4849,12 +9589,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Name_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.Name.id); + value = ast2obj_identifier(state, vstate, o->v.Name.id); if (!value) goto failed; if (PyObject_SetAttr(result, state->id, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Name.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Name.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4864,12 +9604,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->List_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.List.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.List.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.List.ctx); + value = ast2obj_expr_context(state, vstate, o->v.List.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4879,12 +9620,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Tuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Tuple.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Tuple.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Tuple.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Tuple.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4894,52 +9636,54 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Slice_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Slice.lower); + value = ast2obj_expr(state, vstate, o->v.Slice.lower); if (!value) goto failed; if (PyObject_SetAttr(result, state->lower, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.upper); + value = ast2obj_expr(state, vstate, o->v.Slice.upper); if (!value) goto failed; if (PyObject_SetAttr(result, state->upper, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.step); + value = ast2obj_expr(state, vstate, o->v.Slice.step); if (!value) goto failed; if (PyObject_SetAttr(result, state->step, value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } -PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty o) { switch(o) { case Load: @@ -4951,7 +9695,8 @@ PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, struct validator *vstate, + boolop_ty o) { switch(o) { case And: @@ -4961,7 +9706,8 @@ PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, struct validator *vstate, + operator_ty o) { switch(o) { case Add: @@ -4993,7 +9739,8 @@ PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, struct validator *vstate, + unaryop_ty o) { switch(o) { case Invert: @@ -5007,7 +9754,8 @@ PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, struct validator *vstate, + cmpop_ty o) { switch(o) { case Eq: @@ -5034,7 +9782,8 @@ PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(struct ast_state *state, void* _o) +ast2obj_comprehension(struct ast_state *state, struct validator *vstate, void* + _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5042,44 +9791,46 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->comprehension_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->target); + value = ast2obj_expr(state, vstate, o->target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->iter); + value = ast2obj_expr(state, vstate, o->iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->ifs, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->ifs, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->ifs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->is_async); + value = ast2obj_int(state, vstate, o->is_async); if (!value) goto failed; if (PyObject_SetAttr(result, state->is_async, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_excepthandler(struct ast_state *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, struct validator *vstate, void* + _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5087,27 +9838,27 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case ExceptHandler_kind: tp = (PyTypeObject *)state->ExceptHandler_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ExceptHandler.type); + value = ast2obj_expr(state, vstate, o->v.ExceptHandler.type); if (!value) goto failed; if (PyObject_SetAttr(result, state->type, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.ExceptHandler.name); + value = ast2obj_identifier(state, vstate, o->v.ExceptHandler.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ExceptHandler.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ExceptHandler.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -5115,36 +9866,37 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arguments(struct ast_state *state, void* _o) +ast2obj_arguments(struct ast_state *state, struct validator *vstate, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5152,59 +9904,61 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->arguments_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_list(state, (asdl_seq*)o->posonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->posonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->posonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->args, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->args, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->vararg); + value = ast2obj_arg(state, vstate, o->vararg); if (!value) goto failed; if (PyObject_SetAttr(result, state->vararg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kwonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kwonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kw_defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kw_defaults, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->kw_defaults, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->kwarg); + value = ast2obj_arg(state, vstate, o->kwarg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwarg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->defaults, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->defaults, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arg(struct ast_state *state, void* _o) +ast2obj_arg(struct ast_state *state, struct validator *vstate, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5212,59 +9966,60 @@ ast2obj_arg(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->arg_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->annotation); + value = ast2obj_expr(state, vstate, o->annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->type_comment); + value = ast2obj_string(state, vstate, o->type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_keyword(struct ast_state *state, void* _o) +ast2obj_keyword(struct ast_state *state, struct validator *vstate, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5272,54 +10027,55 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->keyword_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->value); + value = ast2obj_expr(state, vstate, o->value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_alias(struct ast_state *state, void* _o) +ast2obj_alias(struct ast_state *state, struct validator *vstate, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5327,54 +10083,55 @@ ast2obj_alias(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->alias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->name); + value = ast2obj_identifier(state, vstate, o->name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->asname); + value = ast2obj_identifier(state, vstate, o->asname); if (!value) goto failed; if (PyObject_SetAttr(result, state->asname, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_withitem(struct ast_state *state, void* _o) +ast2obj_withitem(struct ast_state *state, struct validator *vstate, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5382,34 +10139,35 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->withitem_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->context_expr); + value = ast2obj_expr(state, vstate, o->context_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->context_expr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->optional_vars); + value = ast2obj_expr(state, vstate, o->optional_vars); if (!value) goto failed; if (PyObject_SetAttr(result, state->optional_vars, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_match_case(struct ast_state *state, void* _o) +ast2obj_match_case(struct ast_state *state, struct validator *vstate, void* _o) { match_case_ty o = (match_case_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5417,39 +10175,40 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } tp = (PyTypeObject *)state->match_case_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_pattern(state, o->pattern); + value = ast2obj_pattern(state, vstate, o->pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->guard); + value = ast2obj_expr(state, vstate, o->guard); if (!value) goto failed; if (PyObject_SetAttr(result, state->guard, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_pattern(struct ast_state *state, void* _o) +ast2obj_pattern(struct ast_state *state, struct validator *vstate, void* _o) { pattern_ty o = (pattern_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5457,17 +10216,17 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case MatchValue_kind: tp = (PyTypeObject *)state->MatchValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchValue.value); + value = ast2obj_expr(state, vstate, o->v.MatchValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5477,7 +10236,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSingleton_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.MatchSingleton.value); + value = ast2obj_constant(state, vstate, o->v.MatchSingleton.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5487,7 +10246,8 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSequence_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchSequence.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchSequence.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5498,19 +10258,20 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchMapping_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.keys, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchMapping.keys, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchMapping.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchMapping.rest); + value = ast2obj_identifier(state, vstate, o->v.MatchMapping.rest); if (!value) goto failed; if (PyObject_SetAttr(result, state->rest, value) == -1) goto failed; @@ -5520,24 +10281,27 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchClass_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchClass.cls); + value = ast2obj_expr(state, vstate, o->v.MatchClass.cls); if (!value) goto failed; if (PyObject_SetAttr(result, state->cls, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_attrs, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_attrs, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_attrs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_patterns, value) == -1) @@ -5548,7 +10312,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.MatchStar.name); + value = ast2obj_identifier(state, vstate, o->v.MatchStar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5558,12 +10322,12 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchAs_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_pattern(state, o->v.MatchAs.pattern); + value = ast2obj_pattern(state, vstate, o->v.MatchAs.pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchAs.name); + value = ast2obj_identifier(state, vstate, o->v.MatchAs.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5573,7 +10337,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchOr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchOr.patterns, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchOr.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5581,36 +10345,37 @@ ast2obj_pattern(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_ignore(struct ast_state *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, struct validator *vstate, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5618,38 +10383,39 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case TypeIgnore_kind: tp = (PyTypeObject *)state->TypeIgnore_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_int(state, o->v.TypeIgnore.lineno); + value = ast2obj_int(state, vstate, o->v.TypeIgnore.lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.TypeIgnore.tag); + value = ast2obj_string(state, vstate, o->v.TypeIgnore.tag); if (!value) goto failed; if (PyObject_SetAttr(result, state->tag, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_param(struct ast_state *state, void* _o) +ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) { type_param_ty o = (type_param_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5657,71 +10423,87 @@ ast2obj_type_param(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); - return 0; + return NULL; } switch (o->kind) { case TypeVar_kind: tp = (PyTypeObject *)state->TypeVar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVar.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeVar.bound); + value = ast2obj_expr(state, vstate, o->v.TypeVar.bound); if (!value) goto failed; if (PyObject_SetAttr(result, state->bound, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVar.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case ParamSpec_kind: tp = (PyTypeObject *)state->ParamSpec_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ParamSpec.name); + value = ast2obj_identifier(state, vstate, o->v.ParamSpec.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.ParamSpec.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case TypeVarTuple_kind: tp = (PyTypeObject *)state->TypeVarTuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVarTuple.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVarTuple.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVarTuple.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; @@ -5743,19 +10525,19 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Module_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; asdl_type_ignore_seq* type_ignores; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5788,12 +10570,12 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_ignores, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5832,18 +10614,18 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Interactive_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5882,17 +10664,17 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->Expression_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty body; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Expression"); - return 1; + return -1; } else { int res; @@ -5911,19 +10693,19 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) tp = state->FunctionType_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* argtypes; expr_ty returns; if (PyObject_GetOptionalAttr(obj, state->argtypes, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -5956,11 +10738,11 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"returns\" missing from FunctionType"); - return 1; + return -1; } else { int res; @@ -5980,7 +10762,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -6001,11 +10783,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from stmt"); - return 1; + return -1; } else { int res; @@ -6018,11 +10800,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from stmt"); - return 1; + return -1; } else { int res; @@ -6035,7 +10817,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6052,7 +10834,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6071,7 +10853,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->FunctionDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6083,11 +10865,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from FunctionDef"); - return 1; + return -1; } else { int res; @@ -6100,11 +10882,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from FunctionDef"); - return 1; + return -1; } else { int res; @@ -6117,12 +10899,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6155,12 +10937,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6193,7 +10975,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6210,7 +10992,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6227,12 +11009,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6274,7 +11056,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncFunctionDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6286,11 +11068,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef"); - return 1; + return -1; } else { int res; @@ -6303,11 +11085,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef"); - return 1; + return -1; } else { int res; @@ -6320,12 +11102,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6358,12 +11140,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6396,7 +11178,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->returns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6413,7 +11195,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6430,12 +11212,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6477,7 +11259,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->ClassDef_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; @@ -6488,11 +11270,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_type_param_seq* type_params; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ClassDef"); - return 1; + return -1; } else { int res; @@ -6505,12 +11287,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->bases, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6543,12 +11325,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->keywords, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6581,12 +11363,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6619,12 +11401,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->decorator_list, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6657,12 +11439,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6703,13 +11485,13 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Return_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6733,18 +11515,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Delete_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* targets; if (PyObject_GetOptionalAttr(obj, state->targets, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6784,7 +11566,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Assign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* targets; @@ -6792,12 +11574,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->targets, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6830,11 +11612,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Assign"); - return 1; + return -1; } else { int res; @@ -6847,7 +11629,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -6871,7 +11653,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->TypeAlias_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty name; @@ -6879,11 +11661,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* expr_ty value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeAlias"); - return 1; + return -1; } else { int res; @@ -6896,12 +11678,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_params, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -6934,11 +11716,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from TypeAlias"); - return 1; + return -1; } else { int res; @@ -6958,7 +11740,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AugAssign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -6966,11 +11748,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* expr_ty value; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -6983,11 +11765,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -7000,11 +11782,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from AugAssign"); - return 1; + return -1; } else { int res; @@ -7024,7 +11806,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AnnAssign_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7033,11 +11815,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* int simple; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7050,11 +11832,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->annotation, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"annotation\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7067,7 +11849,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7084,11 +11866,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->simple, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"simple\" missing from AnnAssign"); - return 1; + return -1; } else { int res; @@ -7108,7 +11890,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->For_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7118,11 +11900,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from For"); - return 1; + return -1; } else { int res; @@ -7135,11 +11917,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from For"); - return 1; + return -1; } else { int res; @@ -7152,12 +11934,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7190,12 +11972,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7228,7 +12010,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7252,7 +12034,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncFor_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; @@ -7262,11 +12044,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor"); - return 1; + return -1; } else { int res; @@ -7279,11 +12061,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor"); - return 1; + return -1; } else { int res; @@ -7296,12 +12078,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7334,12 +12116,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7372,7 +12154,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7397,7 +12179,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->While_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -7405,11 +12187,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from While"); - return 1; + return -1; } else { int res; @@ -7422,12 +12204,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7460,12 +12242,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7505,7 +12287,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->If_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -7513,11 +12295,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from If"); - return 1; + return -1; } else { int res; @@ -7530,12 +12312,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7568,12 +12350,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7613,7 +12395,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->With_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_withitem_seq* items; @@ -7621,12 +12403,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->items, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7659,12 +12441,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7697,7 +12479,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7721,7 +12503,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->AsyncWith_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_withitem_seq* items; @@ -7729,12 +12511,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* string type_comment; if (PyObject_GetOptionalAttr(obj, state->items, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7767,12 +12549,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7805,7 +12587,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7829,18 +12611,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Match_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty subject; asdl_match_case_seq* cases; if (PyObject_GetOptionalAttr(obj, state->subject, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"subject\" missing from Match"); - return 1; + return -1; } else { int res; @@ -7853,12 +12635,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->cases, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7898,14 +12680,14 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Raise_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty exc; expr_ty cause; if (PyObject_GetOptionalAttr(obj, state->exc, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7922,7 +12704,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->cause, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -7946,7 +12728,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Try_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; @@ -7955,12 +12737,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* finalbody; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -7993,12 +12775,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->handlers, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8031,12 +12813,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8069,12 +12851,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->finalbody, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8114,7 +12896,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->TryStar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_stmt_seq* body; @@ -8123,12 +12905,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* asdl_stmt_seq* finalbody; if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8161,12 +12943,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->handlers, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8199,12 +12981,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8237,12 +13019,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->finalbody, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8282,18 +13064,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Assert_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; expr_ty msg; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from Assert"); - return 1; + return -1; } else { int res; @@ -8306,7 +13088,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->msg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8330,18 +13112,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Import_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_alias_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8381,7 +13163,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->ImportFrom_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier module; @@ -8389,7 +13171,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* int level; if (PyObject_GetOptionalAttr(obj, state->module, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8406,12 +13188,12 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8444,7 +13226,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->level, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8468,18 +13250,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Global_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_identifier_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8519,18 +13301,18 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Nonlocal_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_identifier_seq* names; if (PyObject_GetOptionalAttr(obj, state->names, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8570,17 +13352,17 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Expr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Expr"); - return 1; + return -1; } else { int res; @@ -8600,7 +13382,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Pass_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8612,7 +13394,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Break_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8624,7 +13406,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* tp = state->Continue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { @@ -8637,7 +13419,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -8658,11 +13440,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from expr"); - return 1; + return -1; } else { int res; @@ -8675,11 +13457,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from expr"); - return 1; + return -1; } else { int res; @@ -8692,7 +13474,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8709,7 +13491,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -8728,18 +13510,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->BoolOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { boolop_ty op; asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BoolOp"); - return 1; + return -1; } else { int res; @@ -8752,12 +13534,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -8797,18 +13579,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->NamedExpr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty target; expr_ty value; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from NamedExpr"); - return 1; + return -1; } else { int res; @@ -8821,11 +13603,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NamedExpr"); - return 1; + return -1; } else { int res; @@ -8845,7 +13627,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->BinOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty left; @@ -8853,11 +13635,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty right; if (PyObject_GetOptionalAttr(obj, state->left, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8870,11 +13652,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8887,11 +13669,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->right, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"right\" missing from BinOp"); - return 1; + return -1; } else { int res; @@ -8911,18 +13693,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->UnaryOp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { unaryop_ty op; expr_ty operand; if (PyObject_GetOptionalAttr(obj, state->op, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from UnaryOp"); - return 1; + return -1; } else { int res; @@ -8935,11 +13717,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->operand, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"operand\" missing from UnaryOp"); - return 1; + return -1; } else { int res; @@ -8959,18 +13741,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Lambda_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { arguments_ty args; expr_ty body; if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Lambda"); - return 1; + return -1; } else { int res; @@ -8983,11 +13765,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Lambda"); - return 1; + return -1; } else { int res; @@ -9007,7 +13789,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->IfExp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty test; @@ -9015,11 +13797,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty orelse; if (PyObject_GetOptionalAttr(obj, state->test, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9032,11 +13814,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9049,11 +13831,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->orelse, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from IfExp"); - return 1; + return -1; } else { int res; @@ -9073,19 +13855,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Dict_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* keys; asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->keys, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9118,12 +13900,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9163,18 +13945,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Set_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9214,18 +13996,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->ListComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from ListComp"); - return 1; + return -1; } else { int res; @@ -9238,12 +14020,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9283,18 +14065,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->SetComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); - return 1; + return -1; } else { int res; @@ -9307,12 +14089,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9352,7 +14134,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->DictComp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty key; @@ -9360,11 +14142,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->key, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); - return 1; + return -1; } else { int res; @@ -9377,11 +14159,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); - return 1; + return -1; } else { int res; @@ -9394,12 +14176,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9439,18 +14221,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->GeneratorExp_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty elt; asdl_comprehension_seq* generators; if (PyObject_GetOptionalAttr(obj, state->elt, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from GeneratorExp"); - return 1; + return -1; } else { int res; @@ -9463,12 +14245,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->generators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9508,17 +14290,17 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Await_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await"); - return 1; + return -1; } else { int res; @@ -9538,13 +14320,13 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Yield_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9568,17 +14350,17 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->YieldFrom_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from YieldFrom"); - return 1; + return -1; } else { int res; @@ -9598,7 +14380,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Compare_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty left; @@ -9606,11 +14388,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_expr_seq* comparators; if (PyObject_GetOptionalAttr(obj, state->left, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from Compare"); - return 1; + return -1; } else { int res; @@ -9623,12 +14405,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ops, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9661,12 +14443,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->comparators, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9706,7 +14488,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Call_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty func; @@ -9714,11 +14496,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* asdl_keyword_seq* keywords; if (PyObject_GetOptionalAttr(obj, state->func, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"func\" missing from Call"); - return 1; + return -1; } else { int res; @@ -9731,12 +14513,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9769,12 +14551,12 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->keywords, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9814,7 +14596,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->FormattedValue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -9822,11 +14604,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty format_spec; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); - return 1; + return -1; } else { int res; @@ -9839,11 +14621,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->conversion, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"conversion\" missing from FormattedValue"); - return 1; + return -1; } else { int res; @@ -9856,7 +14638,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->format_spec, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9881,18 +14663,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->JoinedStr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* values; if (PyObject_GetOptionalAttr(obj, state->values, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -9932,18 +14714,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Constant_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { constant value; string kind; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); - return 1; + return -1; } else { int res; @@ -9956,7 +14738,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kind, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -9980,7 +14762,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Attribute_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -9988,11 +14770,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10005,11 +14787,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->attr, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"attr\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10022,11 +14804,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Attribute"); - return 1; + return -1; } else { int res; @@ -10046,7 +14828,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Subscript_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; @@ -10054,11 +14836,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10071,11 +14853,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->slice, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"slice\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10088,11 +14870,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Subscript"); - return 1; + return -1; } else { int res; @@ -10112,18 +14894,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Starred_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); - return 1; + return -1; } else { int res; @@ -10136,11 +14918,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Starred"); - return 1; + return -1; } else { int res; @@ -10160,18 +14942,18 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Name_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier id; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->id, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"id\" missing from Name"); - return 1; + return -1; } else { int res; @@ -10184,11 +14966,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Name"); - return 1; + return -1; } else { int res; @@ -10208,19 +14990,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->List_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10253,11 +15035,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from List"); - return 1; + return -1; } else { int res; @@ -10277,19 +15059,19 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Tuple_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* elts; expr_context_ty ctx; if (PyObject_GetOptionalAttr(obj, state->elts, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10322,11 +15104,11 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ctx, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Tuple"); - return 1; + return -1; } else { int res; @@ -10346,7 +15128,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* tp = state->Slice_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty lower; @@ -10354,7 +15136,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* expr_ty step; if (PyObject_GetOptionalAttr(obj, state->lower, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10371,7 +15153,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->upper, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10388,7 +15170,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->step, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10413,7 +15195,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -10424,7 +15206,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* isinstance = PyObject_IsInstance(obj, state->Load_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Load; @@ -10432,7 +15214,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } isinstance = PyObject_IsInstance(obj, state->Store_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Store; @@ -10440,7 +15222,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } isinstance = PyObject_IsInstance(obj, state->Del_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Del; @@ -10448,7 +15230,7 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %R", obj); - return 1; + return -1; } int @@ -10459,7 +15241,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* isinstance = PyObject_IsInstance(obj, state->And_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = And; @@ -10467,7 +15249,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Or_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Or; @@ -10475,7 +15257,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* } PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %R", obj); - return 1; + return -1; } int @@ -10486,7 +15268,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, isinstance = PyObject_IsInstance(obj, state->Add_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Add; @@ -10494,7 +15276,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Sub_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Sub; @@ -10502,7 +15284,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Mult_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Mult; @@ -10510,7 +15292,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->MatMult_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = MatMult; @@ -10518,7 +15300,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Div_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Div; @@ -10526,7 +15308,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Mod_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Mod; @@ -10534,7 +15316,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->Pow_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Pow; @@ -10542,7 +15324,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->LShift_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = LShift; @@ -10550,7 +15332,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->RShift_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = RShift; @@ -10558,7 +15340,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitOr_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitOr; @@ -10566,7 +15348,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitXor_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitXor; @@ -10574,7 +15356,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->BitAnd_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = BitAnd; @@ -10582,7 +15364,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } isinstance = PyObject_IsInstance(obj, state->FloorDiv_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = FloorDiv; @@ -10590,7 +15372,7 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %R", obj); - return 1; + return -1; } int @@ -10601,7 +15383,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, isinstance = PyObject_IsInstance(obj, state->Invert_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Invert; @@ -10609,7 +15391,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->Not_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Not; @@ -10617,7 +15399,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->UAdd_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = UAdd; @@ -10625,7 +15407,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } isinstance = PyObject_IsInstance(obj, state->USub_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = USub; @@ -10633,7 +15415,7 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %R", obj); - return 1; + return -1; } int @@ -10644,7 +15426,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* isinstance = PyObject_IsInstance(obj, state->Eq_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Eq; @@ -10652,7 +15434,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->NotEq_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = NotEq; @@ -10660,7 +15442,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Lt_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Lt; @@ -10668,7 +15450,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->LtE_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = LtE; @@ -10676,7 +15458,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Gt_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Gt; @@ -10684,7 +15466,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->GtE_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = GtE; @@ -10692,7 +15474,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->Is_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = Is; @@ -10700,7 +15482,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->IsNot_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = IsNot; @@ -10708,7 +15490,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->In_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = In; @@ -10716,7 +15498,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } isinstance = PyObject_IsInstance(obj, state->NotIn_type); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { *out = NotIn; @@ -10724,7 +15506,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* } PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %R", obj); - return 1; + return -1; } int @@ -10738,11 +15520,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* int is_async; if (PyObject_GetOptionalAttr(obj, state->target, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10755,11 +15537,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->iter, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10772,12 +15554,12 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->ifs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -10810,11 +15592,11 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->is_async, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"is_async\" missing from comprehension"); - return 1; + return -1; } else { int res; @@ -10831,7 +15613,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -10852,11 +15634,11 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler"); - return 1; + return -1; } else { int res; @@ -10869,11 +15651,11 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from excepthandler"); - return 1; + return -1; } else { int res; @@ -10886,7 +15668,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10903,7 +15685,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10922,7 +15704,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* tp = state->ExceptHandler_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty type; @@ -10930,7 +15712,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->type, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10947,7 +15729,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -10964,12 +15746,12 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11010,7 +15792,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11027,12 +15809,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, asdl_expr_seq* defaults; if (PyObject_GetOptionalAttr(obj, state->posonlyargs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11065,12 +15847,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->args, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11103,7 +15885,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->vararg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11120,12 +15902,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwonlyargs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11158,12 +15940,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kw_defaults, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11196,7 +15978,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwarg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11213,12 +15995,12 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->defaults, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11256,7 +16038,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11272,11 +16054,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->arg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11289,7 +16071,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->annotation, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11306,7 +16088,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->type_comment, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11323,11 +16105,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11340,11 +16122,11 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); - return 1; + return -1; } else { int res; @@ -11357,7 +16139,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11374,7 +16156,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11396,7 +16178,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11412,7 +16194,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->arg, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11429,11 +16211,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11446,11 +16228,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11463,11 +16245,11 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from keyword"); - return 1; + return -1; } else { int res; @@ -11480,7 +16262,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11497,7 +16279,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11519,7 +16301,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11535,11 +16317,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* int end_col_offset; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11552,7 +16334,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->asname, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11569,11 +16351,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11586,11 +16368,11 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from alias"); - return 1; + return -1; } else { int res; @@ -11603,7 +16385,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11620,7 +16402,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11642,7 +16424,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11654,11 +16436,11 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, expr_ty optional_vars; if (PyObject_GetOptionalAttr(obj, state->context_expr, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); - return 1; + return -1; } else { int res; @@ -11671,7 +16453,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->optional_vars, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11692,7 +16474,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11705,11 +16487,11 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, asdl_stmt_seq* body; if (PyObject_GetOptionalAttr(obj, state->pattern, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"pattern\" missing from match_case"); - return 1; + return -1; } else { int res; @@ -11722,7 +16504,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->guard, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -11739,12 +16521,12 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->body, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11781,7 +16563,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, return 0; failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -11802,11 +16584,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11819,11 +16601,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11836,11 +16618,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11853,11 +16635,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from pattern"); - return 1; + return -1; } else { int res; @@ -11872,17 +16654,17 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchValue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchValue"); - return 1; + return -1; } else { int res; @@ -11902,17 +16684,17 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchSingleton_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { constant value; if (PyObject_GetOptionalAttr(obj, state->value, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchSingleton"); - return 1; + return -1; } else { int res; @@ -11932,18 +16714,18 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchSequence_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_pattern_seq* patterns; if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -11983,7 +16765,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchMapping_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_expr_seq* keys; @@ -11991,12 +16773,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, identifier rest; if (PyObject_GetOptionalAttr(obj, state->keys, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12029,12 +16811,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12067,7 +16849,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->rest, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12091,7 +16873,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchClass_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { expr_ty cls; @@ -12100,11 +16882,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, asdl_pattern_seq* kwd_patterns; if (PyObject_GetOptionalAttr(obj, state->cls, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"cls\" missing from MatchClass"); - return 1; + return -1; } else { int res; @@ -12117,12 +16899,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12155,12 +16937,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwd_attrs, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12193,12 +16975,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->kwd_patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12239,13 +17021,13 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchStar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12269,14 +17051,14 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchAs_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { pattern_ty pattern; identifier name; if (PyObject_GetOptionalAttr(obj, state->pattern, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12293,7 +17075,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12317,18 +17099,18 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, tp = state->MatchOr_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { asdl_pattern_seq* patterns; if (PyObject_GetOptionalAttr(obj, state->patterns, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { tmp = PyList_New(0); if (tmp == NULL) { - return 1; + return -1; } } { @@ -12369,7 +17151,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, PyErr_Format(PyExc_TypeError, "expected some sort of pattern, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -12388,18 +17170,18 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* tp = state->TypeIgnore_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { int lineno; string tag; if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore"); - return 1; + return -1; } else { int res; @@ -12412,11 +17194,11 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->tag, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore"); - return 1; + return -1; } else { int res; @@ -12436,7 +17218,7 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* PyErr_Format(PyExc_TypeError, "expected some sort of type_ignore, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } int @@ -12457,11 +17239,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, return 0; } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12474,11 +17256,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12491,11 +17273,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_lineno, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12508,11 +17290,11 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->end_col_offset, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from type_param"); - return 1; + return -1; } else { int res; @@ -12527,18 +17309,19 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, tp = state->TypeVar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; expr_ty bound; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeVar"); - return 1; + return -1; } else { int res; @@ -12551,7 +17334,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, Py_CLEAR(tmp); } if (PyObject_GetOptionalAttr(obj, state->bound, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); @@ -12567,25 +17350,43 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVar(name, bound, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVar(name, bound, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } tp = state->ParamSpec_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ParamSpec"); - return 1; + return -1; } else { int res; @@ -12597,25 +17398,43 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_ParamSpec(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'ParamSpec' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_ParamSpec(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } tp = state->TypeVarTuple_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { - return 1; + return -1; } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { - return 1; + return -1; } if (tmp == NULL) { PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from TypeVarTuple"); - return 1; + return -1; } else { int res; @@ -12627,8 +17446,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVarTuple(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVarTuple' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVarTuple(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -12636,7 +17472,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, PyErr_Format(PyExc_TypeError, "expected some sort of type_param, but got %R", obj); failed: Py_XDECREF(tmp); - return 1; + return -1; } @@ -13049,6 +17885,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -13076,24 +17913,24 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { - return 0; + return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + starting_recursion_depth = recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; + starting_recursion_depth, vstate.recursion_depth); + return NULL; } return result; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 83b4aa4b1a7e11..55c821754c2031 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -4,7 +4,6 @@ #include "../Parser/lexer/lexer.h" #include "../Parser/tokenizer/tokenizer.h" #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() -#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() static struct PyModuleDef _tokenizemodule; @@ -33,6 +32,12 @@ typedef struct { PyObject_HEAD struct tok_state *tok; int done; + + /* Needed to cache line for performance */ + PyObject *last_line; + Py_ssize_t last_lineno; + Py_ssize_t last_end_lineno; + Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; /*[clinic input] @@ -69,6 +74,12 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->tok->tok_extra_tokens = 1; } self->done = 0; + + self->last_line = NULL; + self->byte_col_offset_diff = 0; + self->last_lineno = 0; + self->last_end_lineno = 0; + return (PyObject *)self; } @@ -204,6 +215,7 @@ tokenizeriter_next(tokenizeriterobject *it) const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; PyObject* line = NULL; + int line_changed = 1; if (it->tok->tok_extra_tokens && is_trailing_token) { line = PyUnicode_FromString(""); } else { @@ -211,7 +223,19 @@ tokenizeriter_next(tokenizeriterobject *it) if (size >= 1 && it->tok->implicit_newline) { size -= 1; } - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); + + if (it->tok->lineno != it->last_lineno) { + // Line has changed since last token, so we fetch the new line and cache it + // in the iter object. + Py_XDECREF(it->last_line); + line = PyUnicode_DecodeUTF8(line_start, size, "replace"); + it->last_line = line; + it->byte_col_offset_diff = 0; + } else { + // Line hasn't changed so we reuse the cached one. + line = it->last_line; + line_changed = 0; + } } if (line == NULL) { Py_DECREF(str); @@ -220,13 +244,35 @@ tokenizeriter_next(tokenizeriterobject *it) Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; Py_ssize_t end_lineno = it->tok->lineno; + it->last_lineno = lineno; + it->last_end_lineno = end_lineno; + Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; + Py_ssize_t byte_offset = -1; if (token.start != NULL && token.start >= line_start) { - col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start); + byte_offset = token.start - line_start; + if (line_changed) { + col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + it->byte_col_offset_diff = byte_offset - col_offset; + } + else { + col_offset = byte_offset - it->byte_col_offset_diff; + } } if (token.end != NULL && token.end >= it->tok->line_start) { - end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); + Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (lineno == end_lineno) { + // If the whole token is at the same line, we can just use the token.start + // buffer for figuring out the new column offset, since using line is not + // performant for very long lines. + Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + end_col_offset = col_offset + token_col_offset; + it->byte_col_offset_diff += token.end - token.start - token_col_offset; + } else { + end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); + it->byte_col_offset_diff += end_byte_offset - end_col_offset; + } } if (it->tok->tok_extra_tokens) { @@ -263,7 +309,7 @@ tokenizeriter_next(tokenizeriterobject *it) } } - result = Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); + result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); exit: _PyToken_Free(&token); if (type == ENDMARKER) { @@ -276,6 +322,7 @@ static void tokenizeriter_dealloc(tokenizeriterobject *it) { PyTypeObject *tp = Py_TYPE(it); + Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); @@ -323,6 +370,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index 4b7fb888247145..3f9e73b5376223 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_dict.h" // _PyDict_GetItemWithError() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_MUT() #include "pycore_interp.h" // PyInterpreterState.warnings #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -8,6 +8,8 @@ #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DisplaySourceLine() +#include + #include "clinic/_warnings.c.h" #define MODULE_NAME "_warnings" @@ -234,14 +236,12 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import) static PyObject * get_once_registry(PyInterpreterState *interp) { - PyObject *registry; - WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return NULL; - } + assert(st != NULL); + + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); - registry = GET_WARNINGS_ATTR(interp, onceregistry, 0); + PyObject *registry = GET_WARNINGS_ATTR(interp, onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -264,14 +264,12 @@ get_once_registry(PyInterpreterState *interp) static PyObject * get_default_action(PyInterpreterState *interp) { - PyObject *default_action; - WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return NULL; - } + assert(st != NULL); + + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); - default_action = GET_WARNINGS_ATTR(interp, defaultaction, 0); + PyObject *default_action = GET_WARNINGS_ATTR(interp, defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; @@ -298,15 +296,12 @@ get_filter(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *module, PyObject **item) { - PyObject *action; - Py_ssize_t i; - PyObject *warnings_filters; WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return NULL; - } + assert(st != NULL); + + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); - warnings_filters = GET_WARNINGS_ATTR(interp, filters, 0); + PyObject *warnings_filters = GET_WARNINGS_ATTR(interp, filters, 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; @@ -323,7 +318,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, } /* WarningsState.filters could change while we are iterating over it. */ - for (i = 0; i < PyList_GET_SIZE(filters); i++) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(filters); i++) { PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; Py_ssize_t ln; int is_subclass, good_msg, good_mod; @@ -383,7 +378,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, Py_DECREF(tmp_item); } - action = get_default_action(interp); + PyObject *action = get_default_action(interp); if (action != NULL) { *item = Py_NewRef(Py_None); return action; @@ -397,23 +392,26 @@ static int already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, int should_set) { - PyObject *version_obj, *already_warned; + PyObject *already_warned; if (key == NULL) return -1; WarningsState *st = warnings_get_state(interp); - if (st == NULL) { + assert(st != NULL); + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + + PyObject *version_obj; + if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { return -1; } - version_obj = _PyDict_GetItemWithError(registry, &_Py_ID(version)); - if (version_obj == NULL + bool should_update_version = ( + version_obj == NULL || !PyLong_CheckExact(version_obj) - || PyLong_AsLong(version_obj) != st->filters_version) - { - if (PyErr_Occurred()) { - return -1; - } + || PyLong_AsLong(version_obj) != st->filters_version + ); + Py_XDECREF(version_obj); + if (should_update_version) { PyDict_Clear(registry); version_obj = PyLong_FromLong(st->filters_version); if (version_obj == NULL) @@ -425,15 +423,15 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, Py_DECREF(version_obj); } else { - already_warned = PyDict_GetItemWithError(registry, key); + if (PyDict_GetItemRef(registry, key, &already_warned) < 0) { + return -1; + } if (already_warned != NULL) { int rc = PyObject_IsTrue(already_warned); + Py_DECREF(already_warned); if (rc != 0) return rc; } - else if (PyErr_Occurred()) { - return -1; - } } /* This warning wasn't found in the registry, set it. */ @@ -571,10 +569,9 @@ call_show_warning(PyThreadState *tstate, PyObject *category, PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; PyInterpreterState *interp = tstate->interp; - /* If the source parameter is set, try to get the Python implementation. - The Python implementation is able to log the traceback where the source + /* The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = GET_WARNINGS_ATTR(interp, _showwarnmsg, source != NULL); + show_fn = GET_WARNINGS_ATTR(interp, _showwarnmsg, 1); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -707,9 +704,9 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, } /* Store in the registry that we've been here, *except* when the action - is "always". */ + is "always" or "all". */ rc = 0; - if (!_PyUnicode_EqualToASCIIString(action, "always")) { + if (!_PyUnicode_EqualToASCIIString(action, "always") && !_PyUnicode_EqualToASCIIString(action, "all")) { if (registry != NULL && registry != Py_None && PyDict_SetItem(registry, key, Py_True) < 0) { @@ -896,8 +893,8 @@ setup_context(Py_ssize_t stack_level, if (f == NULL) { globals = interp->sysdict; - *filename = PyUnicode_FromString("sys"); - *lineno = 1; + *filename = PyUnicode_FromString(""); + *lineno = 0; } else { globals = f->f_frame->f_globals; @@ -911,13 +908,12 @@ setup_context(Py_ssize_t stack_level, /* Setup registry. */ assert(globals != NULL); assert(PyDict_Check(globals)); - *registry = _PyDict_GetItemWithError(globals, &_Py_ID(__warningregistry__)); + int rc = PyDict_GetItemRef(globals, &_Py_ID(__warningregistry__), + registry); + if (rc < 0) { + goto handle_error; + } if (*registry == NULL) { - int rc; - - if (_PyErr_Occurred(tstate)) { - goto handle_error; - } *registry = PyDict_New(); if (*registry == NULL) goto handle_error; @@ -926,21 +922,21 @@ setup_context(Py_ssize_t stack_level, if (rc < 0) goto handle_error; } - else - Py_INCREF(*registry); /* Setup module. */ - *module = _PyDict_GetItemWithError(globals, &_Py_ID(__name__)); - if (*module == Py_None || (*module != NULL && PyUnicode_Check(*module))) { - Py_INCREF(*module); - } - else if (_PyErr_Occurred(tstate)) { + rc = PyDict_GetItemRef(globals, &_Py_ID(__name__), module); + if (rc < 0) { goto handle_error; } - else { - *module = PyUnicode_FromString(""); - if (*module == NULL) - goto handle_error; + if (rc > 0) { + if (Py_IsNone(*module) || PyUnicode_Check(*module)) { + return 1; + } + Py_DECREF(*module); + } + *module = PyUnicode_FromString(""); + if (*module == NULL) { + goto handle_error; } return 1; @@ -997,8 +993,15 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, &filename, &lineno, &module, ®istry)) return NULL; +#ifdef Py_GIL_DISABLED + WarningsState *st = warnings_get_state(tstate->interp); + assert(st != NULL); +#endif + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, source); + Py_END_CRITICAL_SECTION(); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -1063,12 +1066,12 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno return NULL; } - module_name = _PyDict_GetItemWithError(module_globals, &_Py_ID(__name__)); - if (!module_name) { + int rc = PyDict_GetItemRef(module_globals, &_Py_ID(__name__), + &module_name); + if (rc < 0 || rc == 0) { Py_DECREF(loader); return NULL; } - Py_INCREF(module_name); /* Make sure the loader implements the optional get_source() method. */ (void)PyObject_GetOptionalAttr(loader, &_Py_ID(get_source), &get_source); @@ -1146,8 +1149,16 @@ warnings_warn_explicit_impl(PyObject *module, PyObject *message, return NULL; } } + +#ifdef Py_GIL_DISABLED + WarningsState *st = warnings_get_state(tstate->interp); + assert(st != NULL); +#endif + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); returned = warn_explicit(tstate, category, message, filename, lineno, mod, registry, source_line, sourceobj); + Py_END_CRITICAL_SECTION(); Py_XDECREF(source_line); return returned; } @@ -1165,11 +1176,14 @@ warnings_filters_mutated_impl(PyObject *module) if (interp == NULL) { return NULL; } + WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return NULL; - } + assert(st != NULL); + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); st->filters_version++; + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; } @@ -1287,8 +1301,16 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, if (tstate == NULL) { return -1; } + +#ifdef Py_GIL_DISABLED + WarningsState *st = warnings_get_state(tstate->interp); + assert(st != NULL); +#endif + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); + Py_END_CRITICAL_SECTION(); if (res == NULL) return -1; Py_DECREF(res); @@ -1353,8 +1375,15 @@ PyErr_WarnExplicitFormat(PyObject *category, PyObject *res; PyThreadState *tstate = get_current_tstate(); if (tstate != NULL) { +#ifdef Py_GIL_DISABLED + WarningsState *st = warnings_get_state(tstate->interp); + assert(st != NULL); +#endif + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); + Py_END_CRITICAL_SECTION(); Py_DECREF(message); if (res != NULL) { Py_DECREF(res); @@ -1468,6 +1497,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h deleted file mode 100644 index 44115da8629e42..00000000000000 --- a/Python/abstract_interp_cases.c.h +++ /dev/null @@ -1,930 +0,0 @@ -// This file is generated by Tools/cases_generator/generate_cases.py -// from: -// Python/bytecodes.c -// Do not edit! - - case NOP: { - break; - } - - case RESUME_CHECK: { - break; - } - - case POP_TOP: { - STACK_SHRINK(1); - break; - } - - case PUSH_NULL: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case END_SEND: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_NEGATIVE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_NOT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_BOOL: { - break; - } - - case TO_BOOL_INT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_LIST: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_NONE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_STR: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_ALWAYS_TRUE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_INVERT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_BOTH_INT: { - break; - } - - case _GUARD_BOTH_FLOAT: { - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_OP_ADD_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_BOTH_UNICODE: { - break; - } - - case _BINARY_OP_ADD_UNICODE: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SLICE: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_SLICE: { - STACK_SHRINK(4); - break; - } - - case BINARY_SUBSCR_LIST_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_STR_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_TUPLE_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_DICT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LIST_APPEND: { - STACK_SHRINK(1); - break; - } - - case SET_ADD: { - STACK_SHRINK(1); - break; - } - - case STORE_SUBSCR: { - STACK_SHRINK(3); - break; - } - - case STORE_SUBSCR_LIST_INT: { - STACK_SHRINK(3); - break; - } - - case STORE_SUBSCR_DICT: { - STACK_SHRINK(3); - break; - } - - case DELETE_SUBSCR: { - STACK_SHRINK(2); - break; - } - - case CALL_INTRINSIC_1: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_INTRINSIC_2: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _POP_FRAME: { - STACK_SHRINK(1); - break; - } - - case GET_AITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_ANEXT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_AWAITABLE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case POP_EXCEPT: { - STACK_SHRINK(1); - break; - } - - case LOAD_ASSERTION_ERROR: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_BUILD_CLASS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_NAME: { - STACK_SHRINK(1); - break; - } - - case DELETE_NAME: { - break; - } - - case UNPACK_SEQUENCE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_TWO_TUPLE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_TUPLE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_LIST: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_EX: { - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg >> 8))), true); - break; - } - - case STORE_ATTR: { - STACK_SHRINK(2); - break; - } - - case DELETE_ATTR: { - STACK_SHRINK(1); - break; - } - - case STORE_GLOBAL: { - STACK_SHRINK(1); - break; - } - - case DELETE_GLOBAL: { - break; - } - - case LOAD_LOCALS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_FROM_DICT_OR_GLOBALS: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_NAME: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_GLOBAL: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_GLOBALS_VERSION: { - break; - } - - case _GUARD_BUILTINS_VERSION: { - break; - } - - case _LOAD_GLOBAL_MODULE: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _LOAD_GLOBAL_BUILTINS: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case DELETE_FAST: { - break; - } - - case DELETE_DEREF: { - break; - } - - case LOAD_FROM_DICT_OR_DEREF: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_DEREF: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_DEREF: { - STACK_SHRINK(1); - break; - } - - case COPY_FREE_VARS: { - break; - } - - case BUILD_STRING: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_TUPLE: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_LIST: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LIST_EXTEND: { - STACK_SHRINK(1); - break; - } - - case SET_UPDATE: { - STACK_SHRINK(1); - break; - } - - case BUILD_SET: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_MAP: { - STACK_SHRINK(oparg*2); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SETUP_ANNOTATIONS: { - break; - } - - case BUILD_CONST_KEY_MAP: { - STACK_SHRINK(oparg); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case DICT_UPDATE: { - STACK_SHRINK(1); - break; - } - - case DICT_MERGE: { - STACK_SHRINK(1); - break; - } - - case MAP_ADD: { - STACK_SHRINK(2); - break; - } - - case LOAD_SUPER_ATTR_ATTR: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case LOAD_SUPER_ATTR_METHOD: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_ATTR: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_TYPE_VERSION: { - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_MODULE: { - break; - } - - case _LOAD_ATTR_MODULE: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_WITH_HINT: { - break; - } - - case _LOAD_ATTR_WITH_HINT: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _LOAD_ATTR_SLOT: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_CLASS: { - break; - } - - case _LOAD_ATTR_CLASS: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_DORV_VALUES: { - break; - } - - case _STORE_ATTR_INSTANCE_VALUE: { - STACK_SHRINK(2); - break; - } - - case _STORE_ATTR_SLOT: { - STACK_SHRINK(2); - break; - } - - case COMPARE_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_STR: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case IS_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CONTAINS_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CHECK_EG_MATCH: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CHECK_EXC_MATCH: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _IS_NONE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_LEN: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_CLASS: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_MAPPING: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_SEQUENCE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_KEYS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_ITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_YIELD_FROM_ITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_LIST: { - break; - } - - case _IS_ITER_EXHAUSTED_LIST: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_NEXT_LIST: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_TUPLE: { - break; - } - - case _IS_ITER_EXHAUSTED_TUPLE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_NEXT_TUPLE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_RANGE: { - break; - } - - case _IS_ITER_EXHAUSTED_RANGE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_NEXT_RANGE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case WITH_EXCEPT_START: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case PUSH_EXC_INFO: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - break; - } - - case _GUARD_KEYS_VERSION: { - break; - } - - case _LOAD_ATTR_METHOD_WITH_VALUES: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_ATTR_METHOD_NO_DICT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT: { - break; - } - - case _LOAD_ATTR_METHOD_LAZY_DICT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - break; - } - - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - oparg)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); - break; - } - - case _CHECK_PEP_523: { - break; - } - - case _CHECK_FUNCTION_EXACT_ARGS: { - break; - } - - case _CHECK_STACK_SPACE: { - break; - } - - case _INIT_CALL_PY_EXACT_ARGS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _PUSH_FRAME: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_TYPE_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_STR_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_TUPLE_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case EXIT_INIT_CHECK: { - STACK_SHRINK(1); - break; - } - - case CALL_BUILTIN_CLASS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_O: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_FAST: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_LEN: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_ISINSTANCE: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_O: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_NOARGS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MAKE_FUNCTION: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SET_FUNCTION_ATTRIBUTE: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_SLICE: { - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CONVERT_VALUE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case FORMAT_SIMPLE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case FORMAT_WITH_SPEC: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SWAP: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - (oparg-2))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _POP_JUMP_IF_FALSE: { - STACK_SHRINK(1); - break; - } - - case _POP_JUMP_IF_TRUE: { - STACK_SHRINK(1); - break; - } - - case _JUMP_TO_TOP: { - break; - } - - case _SET_IP: { - break; - } - - case _EXIT_TRACE: { - break; - } - - case _INSERT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); - break; - } diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 460707717df003..0a3265dfeee204 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -22,6 +22,14 @@ _Py_trampoline_func_start: blr x3 ldp x29, x30, [sp], 16 ret +#endif +#ifdef __riscv + addi sp,sp,-16 + sd ra,8(sp) + jalr a3 + ld ra,8(sp) + addi sp,sp,16 + jr ra #endif .globl _Py_trampoline_func_end _Py_trampoline_func_end: diff --git a/Python/assemble.c b/Python/assemble.c index b6fb432aed4a3b..f7b88b519f5f71 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -3,8 +3,10 @@ #include "Python.h" #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" +#include "pycore_instruction_sequence.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE -#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches +#include "pycore_symtable.h" // _Py_SourceLocation #define DEFAULT_CODE_SIZE 128 @@ -21,9 +23,9 @@ return ERROR; \ } -typedef _PyCompilerSrcLocation location; -typedef _PyCompile_Instruction instruction; -typedef _PyCompile_InstructionSequence instr_sequence; +typedef _Py_SourceLocation location; +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; static inline bool same_location(location a, location b) @@ -131,7 +133,7 @@ assemble_emit_exception_table_item(struct assembler *a, int value, int msb) static int assemble_emit_exception_table_entry(struct assembler *a, int start, int end, int handler_offset, - _PyCompile_ExceptHandlerInfo *handler) + _PyExceptHandlerInfo *handler) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { @@ -157,7 +159,7 @@ static int assemble_exception_table(struct assembler *a, instr_sequence *instrs) { int ioffset = 0; - _PyCompile_ExceptHandlerInfo handler; + _PyExceptHandlerInfo handler; handler.h_label = -1; handler.h_startdepth = -1; handler.h_preserve_lasti = -1; @@ -367,17 +369,17 @@ write_instr(_Py_CODEUNIT *codestr, instruction *instr, int ilen) codestr->op.code = EXTENDED_ARG; codestr->op.arg = (oparg >> 24) & 0xFF; codestr++; - /* fall through */ + _Py_FALLTHROUGH; case 3: codestr->op.code = EXTENDED_ARG; codestr->op.arg = (oparg >> 16) & 0xFF; codestr++; - /* fall through */ + _Py_FALLTHROUGH; case 2: codestr->op.code = EXTENDED_ARG; codestr->op.arg = (oparg >> 8) & 0xFF; codestr++; - /* fall through */ + _Py_FALLTHROUGH; case 1: codestr->op.code = opcode; codestr->op.arg = oparg & 0xFF; @@ -710,13 +712,13 @@ resolve_unconditional_jumps(instr_sequence *instrs) bool is_forward = (instr->i_oparg > i); switch(instr->i_opcode) { case JUMP: - assert(SAME_OPCODE_METADATA(JUMP, JUMP_FORWARD)); - assert(SAME_OPCODE_METADATA(JUMP, JUMP_BACKWARD)); + assert(is_pseudo_target(JUMP, JUMP_FORWARD)); + assert(is_pseudo_target(JUMP, JUMP_BACKWARD)); instr->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD; break; case JUMP_NO_INTERRUPT: - assert(SAME_OPCODE_METADATA(JUMP_NO_INTERRUPT, JUMP_FORWARD)); - assert(SAME_OPCODE_METADATA(JUMP_NO_INTERRUPT, JUMP_BACKWARD_NO_INTERRUPT)); + assert(is_pseudo_target(JUMP_NO_INTERRUPT, JUMP_FORWARD)); + assert(is_pseudo_target(JUMP_NO_INTERRUPT, JUMP_BACKWARD_NO_INTERRUPT)); instr->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT; break; @@ -735,7 +737,9 @@ _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *umd, PyObject *const_cac PyObject *consts, int maxdepth, instr_sequence *instrs, int nlocalsplus, int code_flags, PyObject *filename) { - + if (_PyInstructionSequence_ApplyLabelMap(instrs) < 0) { + return NULL; + } if (resolve_unconditional_jumps(instrs) < 0) { return NULL; } diff --git a/Python/ast.c b/Python/ast.c index 5f46d4149c2ed0..1d1a48ec885686 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1011,13 +1011,19 @@ validate_typeparam(struct validator *state, type_param_ty tp) case TypeVar_kind: ret = validate_name(tp->v.TypeVar.name) && (!tp->v.TypeVar.bound || - validate_expr(state, tp->v.TypeVar.bound, Load)); + validate_expr(state, tp->v.TypeVar.bound, Load)) && + (!tp->v.TypeVar.default_value || + validate_expr(state, tp->v.TypeVar.default_value, Load)); break; case ParamSpec_kind: - ret = validate_name(tp->v.ParamSpec.name); + ret = validate_name(tp->v.ParamSpec.name) && + (!tp->v.ParamSpec.default_value || + validate_expr(state, tp->v.ParamSpec.default_value, Load)); break; case TypeVarTuple_kind: - ret = validate_name(tp->v.TypeVarTuple.name); + ret = validate_name(tp->v.TypeVarTuple.name) && + (!tp->v.TypeVarTuple.default_value || + validate_expr(state, tp->v.TypeVarTuple.default_value, Load)); break; } return ret; @@ -1037,10 +1043,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) return 1; } - -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1059,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 41f48eba08afc4..2e2c78b9d4d7d2 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -142,15 +142,6 @@ check_complexity(PyObject *obj, Py_ssize_t limit) } return limit; } - else if (PyFrozenSet_Check(obj)) { - Py_ssize_t i = 0; - PyObject *item; - Py_hash_t hash; - limit -= PySet_GET_SIZE(obj); - while (limit >= 0 && _PySet_NextEntry(obj, &i, &item, &hash)) { - limit = check_complexity(item, limit); - } - } return limit; } @@ -174,9 +165,8 @@ safe_multiply(PyObject *v, PyObject *w) return NULL; } } - else if (PyLong_Check(v) && (PyTuple_Check(w) || PyFrozenSet_Check(w))) { - Py_ssize_t size = PyTuple_Check(w) ? PyTuple_GET_SIZE(w) : - PySet_GET_SIZE(w); + else if (PyLong_Check(v) && PyTuple_Check(w)) { + Py_ssize_t size = PyTuple_GET_SIZE(w); if (size) { long n = PyLong_AsLong(v); if (n < 0 || n > MAX_COLLECTION_SIZE / size) { @@ -198,8 +188,7 @@ safe_multiply(PyObject *v, PyObject *w) } } else if (PyLong_Check(w) && - (PyTuple_Check(v) || PyFrozenSet_Check(v) || - PyUnicode_Check(v) || PyBytes_Check(v))) + (PyTuple_Check(v) || PyUnicode_Check(v) || PyBytes_Check(v))) { return safe_multiply(w, v); } @@ -284,10 +273,9 @@ parse_literal(PyObject *fmt, Py_ssize_t *ppos, PyArena *arena) PyObject *str = PyUnicode_Substring(fmt, start, pos); /* str = str.replace('%%', '%') */ if (str && has_percents) { - _Py_DECLARE_STR(percent, "%"); _Py_DECLARE_STR(dbl_percent, "%%"); Py_SETREF(str, PyUnicode_Replace(str, &_Py_STR(dbl_percent), - &_Py_STR(percent), -1)); + _Py_LATIN1_CHR('%'), -1)); } if (!str) { return NULL; @@ -533,7 +521,7 @@ fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) static PyObject* make_const_tuple(asdl_expr_seq *elts) { - for (int i = 0; i < asdl_seq_LEN(elts); i++) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) { expr_ty e = (expr_ty)asdl_seq_GET(elts, i); if (e->kind != Constant_kind) { return NULL; @@ -545,7 +533,7 @@ make_const_tuple(asdl_expr_seq *elts) return NULL; } - for (int i = 0; i < asdl_seq_LEN(elts); i++) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) { expr_ty e = (expr_ty)asdl_seq_GET(elts, i); PyObject *v = e->v.Constant.value; PyTuple_SET_ITEM(newval, i, Py_NewRef(v)); @@ -662,7 +650,7 @@ static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimize return 0; #define CALL_SEQ(FUNC, TYPE, ARG) { \ - int i; \ + Py_ssize_t i; \ asdl_ ## TYPE ## _seq *seq = (ARG); /* avoid variable capture */ \ for (i = 0; i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ @@ -1111,9 +1099,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) { @@ -1131,9 +1116,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 8aff045101cc72..86f7a582b981a3 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -10,9 +10,7 @@ * See ast.unparse for a full unparser (written in Python) */ -_Py_DECLARE_STR(open_br, "{"); _Py_DECLARE_STR(dbl_open_br, "{{"); -_Py_DECLARE_STR(close_br, "}"); _Py_DECLARE_STR(dbl_close_br, "}}"); /* We would statically initialize this if doing so were simple enough. */ @@ -79,7 +77,7 @@ append_repr(_PyUnicodeWriter *writer, PyObject *obj) return -1; } - if ((PyFloat_CheckExact(obj) && Py_IS_INFINITY(PyFloat_AS_DOUBLE(obj))) || + if ((PyFloat_CheckExact(obj) && isinf(PyFloat_AS_DOUBLE(obj))) || PyComplex_CheckExact(obj)) { PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -580,11 +578,13 @@ escape_braces(PyObject *orig) { PyObject *temp; PyObject *result; - temp = PyUnicode_Replace(orig, &_Py_STR(open_br), &_Py_STR(dbl_open_br), -1); + temp = PyUnicode_Replace(orig, _Py_LATIN1_CHR('{'), + &_Py_STR(dbl_open_br), -1); if (!temp) { return NULL; } - result = PyUnicode_Replace(temp, &_Py_STR(close_br), &_Py_STR(dbl_close_br), -1); + result = PyUnicode_Replace(temp, _Py_LATIN1_CHR('}'), + &_Py_STR(dbl_close_br), -1); Py_DECREF(temp); return result; } @@ -678,7 +678,7 @@ append_formattedvalue(_PyUnicodeWriter *writer, expr_ty e) if (!temp_fv_str) { return -1; } - if (PyUnicode_Find(temp_fv_str, &_Py_STR(open_br), 0, 1, 1) == 0) { + if (PyUnicode_Find(temp_fv_str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { /* Expression starts with a brace, split it with a space from the outer one. */ outer_brace = "{ "; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c373585c0986ce..3f7bf4d568ee46 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -5,7 +5,6 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -141,18 +140,13 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } - meta = _PyDict_GetItemWithError(mkw, &_Py_ID(metaclass)); + if (PyDict_Pop(mkw, &_Py_ID(metaclass), &meta) < 0) { + goto error; + } if (meta != NULL) { - Py_INCREF(meta); - if (PyDict_DelItem(mkw, &_Py_ID(metaclass)) < 0) { - goto error; - } /* metaclass is explicitly given, check if it's indeed a class */ isclass = PyType_Check(meta); } - else if (PyErr_Occurred()) { - goto error; - } } if (meta == NULL) { /* if there are no bases, use type: */ @@ -481,7 +475,8 @@ builtin_breakpoint(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb } PyDoc_STRVAR(breakpoint_doc, -"breakpoint(*args, **kws)\n\ +"breakpoint($module, /, *args, **kws)\n\ +--\n\ \n\ Call sys.breakpointhook(*args, **kws). sys.breakpointhook() must accept\n\ whatever arguments are passed.\n\ @@ -626,7 +621,8 @@ static PyMethodDef filter_methods[] = { }; PyDoc_STRVAR(filter_doc, -"filter(function or None, iterable) --> filter object\n\ +"filter(function, iterable, /)\n\ +--\n\ \n\ Return an iterator yielding those items of iterable for which function(item)\n\ is true. If function is None, return the items that are true."); @@ -704,17 +700,34 @@ builtin_format_impl(PyObject *module, PyObject *value, PyObject *format_spec) /*[clinic input] chr as builtin_chr - i: int + i: object / Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. [clinic start generated code]*/ static PyObject * -builtin_chr_impl(PyObject *module, int i) -/*[clinic end generated code: output=c733afcd200afcb7 input=3f604ef45a70750d]*/ +builtin_chr(PyObject *module, PyObject *i) +/*[clinic end generated code: output=d34f25b8035a9b10 input=f919867f0ba2f496]*/ { - return PyUnicode_FromOrdinal(i); + int overflow; + long v = PyLong_AsLongAndOverflow(i, &overflow); + if (v == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow) { + v = overflow < 0 ? INT_MIN : INT_MAX; + /* Allow PyUnicode_FromOrdinal() to raise an exception */ + } +#if SIZEOF_INT < SIZEOF_LONG + else if (v < INT_MIN) { + v = INT_MIN; + } + else if (v > INT_MAX) { + v = INT_MAX; + } +#endif + return PyUnicode_FromOrdinal(v); } @@ -853,8 +866,21 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; +#ifdef Py_GIL_DISABLED + // gh-118527: Disable immortalization of code constants for explicit + // compile() calls to get consistent frozen outputs between the default + // and free-threaded builds. + // Subtract two to suppress immortalization (so that 1 -> -1) + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_atomic_add_int(&interp->gc.immortalize, -2); +#endif + result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); +#ifdef Py_GIL_DISABLED + _Py_atomic_add_int(&interp->gc.immortalize, 2); +#endif + Py_XDECREF(source_copy); goto finally; @@ -912,9 +938,9 @@ builtin_divmod_impl(PyObject *module, PyObject *x, PyObject *y) eval as builtin_eval source: object + / globals: object = None locals: object = None - / Evaluate the given source in the context of globals and locals. @@ -928,7 +954,7 @@ If only globals is given, locals defaults to it. static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) -/*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ +/*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { PyObject *result = NULL, *source_copy; const char *str; @@ -1011,9 +1037,9 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, exec as builtin_exec source: object + / globals: object = None locals: object = None - / * closure: object(c_default="NULL") = None @@ -1031,7 +1057,7 @@ when source is a code object requiring exactly that many cellvars. static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) -/*[clinic end generated code: output=7579eb4e7646743d input=f13a7e2b503d1d9a]*/ +/*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { PyObject *v; @@ -1449,7 +1475,8 @@ static PyMethodDef map_methods[] = { PyDoc_STRVAR(map_doc, -"map(func, *iterables) --> map object\n\ +"map(function, iterable, /, *iterables)\n\ +--\n\ \n\ Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted."); @@ -1689,16 +1716,16 @@ anext as builtin_anext default: object = NULL / -async anext(aiterator[, default]) +Return the next item from the async iterator. -Return the next item from the async iterator. If default is given and the async -iterator is exhausted, it is returned instead of raising StopAsyncIteration. +If default is given and the async iterator is exhausted, +it is returned instead of raising StopAsyncIteration. [clinic start generated code]*/ static PyObject * builtin_anext_impl(PyObject *module, PyObject *aiterator, PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=8f63f4f78590bb4c]*/ +/*[clinic end generated code: output=f02c060c163a81fa input=2900e4a370d39550]*/ { PyTypeObject *t; PyObject *awaitable; @@ -1766,35 +1793,27 @@ builtin_locals_impl(PyObject *module) static PyObject * -min_max(PyObject *args, PyObject *kwds, int op) +min_max(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, int op) { - PyObject *v, *it, *item, *val, *maxitem, *maxval, *keyfunc=NULL; - PyObject *emptytuple, *defaultval = NULL; - static char *kwlist[] = {"key", "default", NULL}; - const char *name = op == Py_LT ? "min" : "max"; - const int positional = PyTuple_Size(args) > 1; - int ret; + PyObject *it = NULL, *item, *val, *maxitem, *maxval, *keyfunc=NULL; + PyObject *defaultval = NULL; + static const char * const keywords[] = {"key", "default", NULL}; + static _PyArg_Parser _parser_min = {"|$OO:min", keywords, 0}; + static _PyArg_Parser _parser_max = {"|$OO:max", keywords, 0}; + const char *name = (op == Py_LT) ? "min" : "max"; + _PyArg_Parser *_parser = (op == Py_LT) ? &_parser_min : &_parser_max; - if (positional) { - v = args; - } - else if (!PyArg_UnpackTuple(args, name, 1, 1, &v)) { - if (PyExceptionClass_Check(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); - } + if (nargs == 0) { + PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); return NULL; } - emptytuple = PyTuple_New(0); - if (emptytuple == NULL) - return NULL; - ret = PyArg_ParseTupleAndKeywords(emptytuple, kwds, - (op == Py_LT) ? "|$OO:min" : "|$OO:max", - kwlist, &keyfunc, &defaultval); - Py_DECREF(emptytuple); - if (!ret) + if (kwnames != NULL && !_PyArg_ParseStackAndKeywords(args + nargs, 0, kwnames, _parser, + &keyfunc, &defaultval)) { return NULL; + } + const int positional = nargs > 1; // False iff nargs == 1 if (positional && defaultval != NULL) { PyErr_Format(PyExc_TypeError, "Cannot specify a default for %s() with multiple " @@ -1802,9 +1821,11 @@ min_max(PyObject *args, PyObject *kwds, int op) return NULL; } - it = PyObject_GetIter(v); - if (it == NULL) { - return NULL; + if (!positional) { + it = PyObject_GetIter(args[0]); + if (it == NULL) { + return NULL; + } } if (keyfunc == Py_None) { @@ -1813,7 +1834,24 @@ min_max(PyObject *args, PyObject *kwds, int op) maxitem = NULL; /* the result */ maxval = NULL; /* the value associated with the result */ - while (( item = PyIter_Next(it) )) { + while (1) { + if (it == NULL) { + if (nargs-- <= 0) { + break; + } + item = *args++; + Py_INCREF(item); + } + else { + item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + goto Fail_it; + } + break; + } + } + /* get the value from the key function */ if (keyfunc != NULL) { val = PyObject_CallOneArg(keyfunc, item); @@ -1847,8 +1885,6 @@ min_max(PyObject *args, PyObject *kwds, int op) } } } - if (PyErr_Occurred()) - goto Fail_it; if (maxval == NULL) { assert(maxitem == NULL); if (defaultval != NULL) { @@ -1860,7 +1896,7 @@ min_max(PyObject *args, PyObject *kwds, int op) } else Py_DECREF(maxval); - Py_DECREF(it); + Py_XDECREF(it); return maxitem; Fail_it_item_and_val: @@ -1870,15 +1906,15 @@ min_max(PyObject *args, PyObject *kwds, int op) Fail_it: Py_XDECREF(maxval); Py_XDECREF(maxitem); - Py_DECREF(it); + Py_XDECREF(it); return NULL; } /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_min(PyObject *self, PyObject *args, PyObject *kwds) +builtin_min(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_LT); + return min_max(args, nargs, kwnames, Py_LT); } PyDoc_STRVAR(min_doc, @@ -1888,14 +1924,14 @@ min(arg1, arg2, *args, *[, key=func]) -> value\n\ With a single iterable argument, return its smallest item. The\n\ default keyword-only argument specifies an object to return if\n\ the provided iterable is empty.\n\ -With two or more arguments, return the smallest argument."); +With two or more positional arguments, return the smallest argument."); /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_max(PyObject *self, PyObject *args, PyObject *kwds) +builtin_max(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_GT); + return min_max(args, nargs, kwnames, Py_GT); } PyDoc_STRVAR(max_doc, @@ -1905,7 +1941,7 @@ max(arg1, arg2, *args, *[, key=func]) -> value\n\ With a single iterable argument, return its biggest item. The\n\ default keyword-only argument specifies an object to return if\n\ the provided iterable is empty.\n\ -With two or more arguments, return the largest argument."); +With two or more positional arguments, return the largest argument."); /*[clinic input] @@ -2262,6 +2298,11 @@ builtin_input_impl(PyObject *module, PyObject *prompt) goto _readline_errors; assert(PyBytes_Check(po)); promptstr = PyBytes_AS_STRING(po); + if ((Py_ssize_t)strlen(promptstr) != PyBytes_GET_SIZE(po)) { + PyErr_SetString(PyExc_ValueError, + "input: prompt string cannot contain null characters"); + goto _readline_errors; + } } else { po = NULL; @@ -2368,11 +2409,6 @@ builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) { PyObject *round, *result; - if (!_PyType_IsReady(Py_TYPE(number))) { - if (PyType_Ready(Py_TYPE(number)) < 0) - return NULL; - } - round = _PyObject_LookupSpecial(number, &_Py_ID(__round__)); if (round == NULL) { if (!PyErr_Occurred()) @@ -2480,6 +2516,49 @@ Without arguments, equivalent to locals().\n\ With an argument, equivalent to object.__dict__."); +/* Improved Kahan–BabuÅ¡ka algorithm by Arnold Neumaier + Neumaier, A. (1974), Rundungsfehleranalyse einiger Verfahren + zur Summation endlicher Summen. Z. angew. Math. Mech., + 54: 39-51. https://doi.org/10.1002/zamm.19740540106 + https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements + */ + +typedef struct { + double hi; /* high-order bits for a running sum */ + double lo; /* a running compensation for lost low-order bits */ +} CompensatedSum; + +static inline CompensatedSum +cs_from_double(double x) +{ + return (CompensatedSum) {x}; +} + +static inline CompensatedSum +cs_add(CompensatedSum total, double x) +{ + double t = total.hi + x; + if (fabs(total.hi) >= fabs(x)) { + total.lo += (total.hi - t) + x; + } + else { + total.lo += (x - t) + total.hi; + } + return (CompensatedSum) {t, total.lo}; +} + +static inline double +cs_to_double(CompensatedSum total) +{ + /* Avoid losing the sign on a negative result, + and don't let adding the compensation convert + an infinite or overflowed sum to a NaN. */ + if (total.lo && isfinite(total.lo)) { + return total.hi + total.lo; + } + return total.hi; +} + /*[clinic input] sum as builtin_sum @@ -2565,8 +2644,8 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) b = PyLong_AsLongAndOverflow(item, &overflow); } if (overflow == 0 && - (i_result >= 0 ? (b <= LONG_MAX - i_result) - : (b >= LONG_MIN - i_result))) + (i_result >= 0 ? (b <= PY_SSIZE_T_MAX - i_result) + : (b >= PY_SSIZE_T_MIN - i_result))) { i_result += b; Py_DECREF(item); @@ -2592,8 +2671,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(result)) { - double f_result = PyFloat_AS_DOUBLE(result); - double c = 0.0; + CompensatedSum re_sum = cs_from_double(PyFloat_AS_DOUBLE(result)); Py_SETREF(result, NULL); while(result == NULL) { item = PyIter_Next(iter); @@ -2601,25 +2679,10 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) Py_DECREF(iter); if (PyErr_Occurred()) return NULL; - /* Avoid losing the sign on a negative result, - and don't let adding the compensation convert - an infinite or overflowed sum to a NaN. */ - if (c && Py_IS_FINITE(c)) { - f_result += c; - } - return PyFloat_FromDouble(f_result); + return PyFloat_FromDouble(cs_to_double(re_sum)); } if (PyFloat_CheckExact(item)) { - // Improved Kahan–BabuÅ¡ka algorithm by Arnold Neumaier - // https://www.mat.univie.ac.at/~neum/scan/01.pdf - double x = PyFloat_AS_DOUBLE(item); - double t = f_result + x; - if (fabs(f_result) >= fabs(x)) { - c += (f_result - t) + x; - } else { - c += (x - t) + f_result; - } - f_result = t; + re_sum = cs_add(re_sum, PyFloat_AS_DOUBLE(item)); _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } @@ -2628,15 +2691,70 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) int overflow; value = PyLong_AsLongAndOverflow(item, &overflow); if (!overflow) { - f_result += (double)value; + re_sum.hi += (double)value; Py_DECREF(item); continue; } } - if (c && Py_IS_FINITE(c)) { - f_result += c; + result = PyFloat_FromDouble(cs_to_double(re_sum)); + if (result == NULL) { + Py_DECREF(item); + Py_DECREF(iter); + return NULL; } - result = PyFloat_FromDouble(f_result); + temp = PyNumber_Add(result, item); + Py_DECREF(result); + Py_DECREF(item); + result = temp; + if (result == NULL) { + Py_DECREF(iter); + return NULL; + } + } + } + + if (PyComplex_CheckExact(result)) { + Py_complex z = PyComplex_AsCComplex(result); + CompensatedSum re_sum = cs_from_double(z.real); + CompensatedSum im_sum = cs_from_double(z.imag); + Py_SETREF(result, NULL); + while (result == NULL) { + item = PyIter_Next(iter); + if (item == NULL) { + Py_DECREF(iter); + if (PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(cs_to_double(re_sum), + cs_to_double(im_sum)); + } + if (PyComplex_CheckExact(item)) { + z = PyComplex_AsCComplex(item); + re_sum = cs_add(re_sum, z.real); + im_sum = cs_add(im_sum, z.imag); + Py_DECREF(item); + continue; + } + if (PyLong_Check(item)) { + long value; + int overflow; + value = PyLong_AsLongAndOverflow(item, &overflow); + if (!overflow) { + re_sum.hi += (double)value; + im_sum.hi += 0.0; + Py_DECREF(item); + continue; + } + } + if (PyFloat_Check(item)) { + double value = PyFloat_AS_DOUBLE(item); + re_sum.hi += value; + im_sum.hi += 0.0; + _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); + continue; + } + result = PyComplex_FromDoubles(cs_to_double(re_sum), + cs_to_double(im_sum)); if (result == NULL) { Py_DECREF(item); Py_DECREF(iter); @@ -2955,10 +3073,8 @@ static PyMethodDef zip_methods[] = { }; PyDoc_STRVAR(zip_doc, -"zip(*iterables, strict=False) --> Yield tuples until an input is exhausted.\n\ -\n\ - >>> list(zip('abcdefg', range(3), range(4)))\n\ - [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]\n\ +"zip(*iterables, strict=False)\n\ +--\n\ \n\ The zip object yields n-length tuples, where n is the number of iterables\n\ passed as positional arguments to zip(). The i-th element in every tuple\n\ @@ -2966,7 +3082,10 @@ comes from the i-th iterable argument to zip(). This continues until the\n\ shortest argument is exhausted.\n\ \n\ If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError."); +raise a ValueError.\n\ +\n\ + >>> list(zip('abcdefg', range(3), range(4)))\n\ + [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]"); PyTypeObject PyZip_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -3045,8 +3164,8 @@ static PyMethodDef builtin_methods[] = { BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF - {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, - {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, + {"max", _PyCFunction_CAST(builtin_max), METH_FASTCALL | METH_KEYWORDS, max_doc}, + {"min", _PyCFunction_CAST(builtin_min), METH_FASTCALL | METH_KEYWORDS, min_doc}, {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF @@ -3097,6 +3216,9 @@ _PyBuiltin_Init(PyInterpreterState *interp) mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif dict = PyModule_GetDict(mod); #ifdef Py_TRACE_REFS diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 92f2301a012c0a..1dc5bffe1b0003 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -199,7 +199,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) getentropy() is retried if it failed with EINTR: interrupted by a signal. */ -#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +#if defined(__APPLE__) && _Py__has_attribute(availability) static int py_getentropy(char *buffer, Py_ssize_t size, int raise) __attribute__((availability(macos,introduced=10.12))) diff --git a/Python/brc.c b/Python/brc.c new file mode 100644 index 00000000000000..d27687052aec19 --- /dev/null +++ b/Python/brc.c @@ -0,0 +1,208 @@ +// Implementation of biased reference counting inter-thread queue. +// +// Biased reference counting maintains two refcount fields in each object: +// ob_ref_local and ob_ref_shared. The true refcount is the sum of these two +// fields. In some cases, when refcounting operations are split across threads, +// the ob_ref_shared field can be negative (although the total refcount must +// be at least zero). In this case, the thread that decremented the refcount +// requests that the owning thread give up ownership and merge the refcount +// fields. This file implements the mechanism for doing so. +// +// Each thread state maintains a queue of objects whose refcounts it should +// merge. The thread states are stored in a per-interpreter hash table by +// thread id. The hash table has a fixed size and uses a linked list to store +// thread states within each bucket. +// +// The queueing thread uses the eval breaker mechanism to notify the owning +// thread that it has objects to merge. Additionally, all queued objects are +// merged during GC. +#include "Python.h" +#include "pycore_object.h" // _Py_ExplicitMergeRefcount +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit +#include "pycore_llist.h" // struct llist_node +#include "pycore_pystate.h" // _PyThreadStateImpl + +#ifdef Py_GIL_DISABLED + +// Get the hashtable bucket for a given thread id. +static struct _brc_bucket * +get_bucket(PyInterpreterState *interp, uintptr_t tid) +{ + return &interp->brc.table[tid % _Py_BRC_NUM_BUCKETS]; +} + +// Find the thread state in a hash table bucket by thread id. +static _PyThreadStateImpl * +find_thread_state(struct _brc_bucket *bucket, uintptr_t thread_id) +{ + struct llist_node *node; + llist_for_each(node, &bucket->root) { + // Get the containing _PyThreadStateImpl from the linked-list node. + _PyThreadStateImpl *ts = llist_data(node, _PyThreadStateImpl, + brc.bucket_node); + if (ts->brc.tid == thread_id) { + return ts; + } + } + return NULL; +} + +// Enqueue an object to be merged by the owning thread. This steals a +// reference to the object. +void +_Py_brc_queue_object(PyObject *ob) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + uintptr_t ob_tid = _Py_atomic_load_uintptr(&ob->ob_tid); + if (ob_tid == 0) { + // The owning thread may have concurrently decided to merge the + // refcount fields. + Py_DECREF(ob); + return; + } + + struct _brc_bucket *bucket = get_bucket(interp, ob_tid); + PyMutex_Lock(&bucket->mutex); + _PyThreadStateImpl *tstate = find_thread_state(bucket, ob_tid); + if (tstate == NULL) { + // If we didn't find the owning thread then it must have already exited. + // It's safe (and necessary) to merge the refcount. Subtract one when + // merging because we've stolen a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + PyMutex_Unlock(&bucket->mutex); + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + if (_PyObjectStack_Push(&tstate->brc.objects_to_merge, ob) < 0) { + PyMutex_Unlock(&bucket->mutex); + + // Fall back to stopping all threads and manually merging the refcount + // if we can't enqueue the object to be merged. + _PyEval_StopTheWorld(interp); + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + _PyEval_StartTheWorld(interp); + + if (refcount == 0) { + _Py_Dealloc(ob); + } + return; + } + + // Notify owning thread + _Py_set_eval_breaker_bit(&tstate->base, _PY_EVAL_EXPLICIT_MERGE_BIT); + + PyMutex_Unlock(&bucket->mutex); +} + +static void +merge_queued_objects(_PyObjectStack *to_merge) +{ + PyObject *ob; + while ((ob = _PyObjectStack_Pop(to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1); + if (refcount == 0) { + _Py_Dealloc(ob); + } + } +} + +// Process this thread's queue of objects to merge. +void +_Py_brc_merge_refcounts(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + assert(brc->tid == _Py_ThreadId()); + + // Append all objects into a local stack. We don't want to hold the lock + // while calling destructors. + PyMutex_Lock(&bucket->mutex); + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + PyMutex_Unlock(&bucket->mutex); + + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); +} + +void +_Py_brc_init_state(PyInterpreterState *interp) +{ + struct _brc_state *brc = &interp->brc; + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + llist_init(&brc->table[i].root); + } +} + +void +_Py_brc_init_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + uintptr_t tid = _Py_ThreadId(); + + // Add ourself to the hashtable + struct _brc_bucket *bucket = get_bucket(tstate->interp, tid); + PyMutex_Lock(&bucket->mutex); + brc->tid = tid; + llist_insert_tail(&bucket->root, &brc->bucket_node); + PyMutex_Unlock(&bucket->mutex); +} + +void +_Py_brc_remove_thread(PyThreadState *tstate) +{ + struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc; + if (brc->tid == 0) { + // The thread state may have been created, but never bound to a native + // thread and therefore never added to the hashtable. + assert(tstate->_status.bound == 0); + return; + } + + struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid); + + // We need to fully process any objects to merge before removing ourself + // from the hashtable. It is not safe to perform any refcount operations + // after we are removed. After that point, other threads treat our objects + // as abandoned and may merge the objects' refcounts directly. + bool empty = false; + while (!empty) { + // Process the local stack until it's empty + merge_queued_objects(&brc->local_objects_to_merge); + + PyMutex_Lock(&bucket->mutex); + empty = (brc->objects_to_merge.head == NULL); + if (empty) { + llist_remove(&brc->bucket_node); + } + else { + _PyObjectStack_Merge(&brc->local_objects_to_merge, + &brc->objects_to_merge); + } + PyMutex_Unlock(&bucket->mutex); + } + + assert(brc->local_objects_to_merge.head == NULL); + assert(brc->objects_to_merge.head == NULL); +} + +void +_Py_brc_after_fork(PyInterpreterState *interp) +{ + // Unlock all bucket mutexes. Some of the buckets may be locked because + // locks can be handed off to a parked thread (see lock.c). We don't have + // to worry about consistency here, because no thread can be actively + // modifying a bucket, but it might be paused (not yet woken up) on a + // PyMutex_Lock while holding that lock. + for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { + _PyMutex_at_fork_reinit(&interp->brc.table[i].mutex); + } +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d7e2ecdd24dcee..84241c64ffae88 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1,6 +1,6 @@ // This file contains instruction definitions. -// It is read by Tools/cases_generator/generate_cases.py -// to generate Python/generated_cases.c.h. +// It is read by generators stored in Tools/cases_generator/ +// to generate Python/generated_cases.c.h and others. // Note that there is some dummy C code at the top and bottom of the file // to fool text editors like VS Code into believing this is valid C code. // The actual instruction definitions start at // BEGIN BYTECODES //. @@ -8,7 +8,8 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() +#include "pycore_backoff.h" +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -19,6 +20,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -50,6 +52,13 @@ #define family(name, ...) static int family_##name #define pseudo(name) static int pseudo_##name +/* Annotations */ +#define guard +#define override +#define specializing +#define split +#define replicate(TIMES) + // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2; @@ -63,6 +72,7 @@ static size_t jump; static uint16_t invert, counter, index, hint; #define unused 0 // Used in a macro def, can't be static static uint32_t type_version; +static _PyExecutorObject *current_executor; static PyObject * dummy_func( @@ -80,6 +90,7 @@ dummy_func( pop_1_error: // Dummy locals. PyObject *dummy; + _Py_CODEUNIT *this_instr; PyObject *attr; PyObject *attrs; PyObject *bottom; @@ -121,36 +132,47 @@ dummy_func( PyObject *top; PyObject *type; PyObject *typevars; + PyObject *val0; + PyObject *val1; int values_or_none; switch (opcode) { // BEGIN BYTECODES // - inst(NOP, (--)) { + pure inst(NOP, (--)) { } family(RESUME, 0) = { RESUME_CHECK, }; - inst(RESUME, (--)) { - TIER_ONE_ONLY + tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - ERROR_IF(err, error); - next_instr--; - } - else { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; + DISPATCH(); + } + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } - next_instr[-1].op.code = RESUME_CHECK; + } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } } @@ -159,79 +181,81 @@ dummy_func( DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version); } inst(INSTRUMENTED_RESUME, (--)) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; + ERROR_NO_POP(); } - next_instr--; + next_instr = this_instr; } else { - if (oparg < 2) { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->prev_instr != next_instr-1) { + if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } } - pseudo(LOAD_CLOSURE) = { + pseudo(LOAD_CLOSURE, (-- unused)) = { LOAD_FAST, }; inst(LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - ERROR_IF(value == NULL, unbound_local_error); - Py_INCREF(value); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } + value = PyStackRef_DUP(value_s); } - inst(LOAD_FAST, (-- value)) { - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + replicate(8) pure inst(LOAD_FAST, (-- value)) { + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); } inst(LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; } inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) { uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } - inst(LOAD_CONST, (-- value)) { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + pure inst(LOAD_CONST, (-- value)) { + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - inst(STORE_FAST, (value --)) { + replicate(8) inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } - pseudo(STORE_FAST_MAYBE_NULL) = { + pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = { STORE_FAST, }; @@ -239,8 +263,7 @@ dummy_func( uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { @@ -250,53 +273,53 @@ dummy_func( SETLOCAL(oparg2, value2); } - inst(POP_TOP, (value --)) { + pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); } - inst(PUSH_NULL, (-- res)) { - res = NULL; + pure inst(PUSH_NULL, (-- res)) { + res = PyStackRef_NULL; } - macro(END_FOR) = POP_TOP + POP_TOP; + macro(END_FOR) = POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + ERROR_NO_POP(); } - PyErr_SetRaisedException(NULL); } DECREF_INPUTS(); } - inst(END_SEND, (receiver, value -- value)) { - Py_DECREF(receiver); + pure inst(END_SEND, (receiver, value -- value)) { + (void)receiver; + PyStackRef_CLOSE(receiver); } - inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + ERROR_NO_POP(); } - PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); + PyStackRef_CLOSE(receiver); } inst(UNARY_NEGATIVE, (value -- res)) { - res = PyNumber_Negative(value); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(UNARY_NOT, (value -- res)) { - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + pure inst(UNARY_NOT, (value -- res)) { + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; } family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = { @@ -308,82 +331,91 @@ dummy_func( TO_BOOL_STR, }; - inst(TO_BOOL, (unused/1, unused/2, value -- res)) { + specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ - int err = PyObject_IsTrue(value); + } + + op(_TO_BOOL, (value -- res)) { + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); ERROR_IF(err < 0, error); - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; } + macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; + inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; DECREF_INPUTS(); } inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value)); + EXIT_IF(!PyStackRef_Is(value, PyStackRef_None)); STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyUnicode_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); + assert(Py_SIZE(value_o)); DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } - inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); - STAT_INC(TO_BOOL, hit); + op(_REPLACE_WITH_TRUE, (value -- res)) { DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } + macro(TO_BOOL_ALWAYS_TRUE) = + unused/1 + + _GUARD_TYPE_VERSION + + _REPLACE_WITH_TRUE; + inst(UNARY_INVERT, (value -- res)) { - res = PyNumber_Invert(value); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = { @@ -398,92 +430,150 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckExact(right_o)); + } + + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyLong_CheckExact(left_o)); + } + + op(_GUARD_TOS_INT, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); } - op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_ADD_INT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_INT) = - _GUARD_BOTH_INT + _BINARY_OP_MULTIPLY_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_MULTIPLY_INT; macro(BINARY_OP_ADD_INT) = - _GUARD_BOTH_INT + _BINARY_OP_ADD_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_ADD_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyFloat_CheckExact(left_o)); + EXIT_IF(!PyFloat_CheckExact(right_o)); + } + + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyFloat_CheckExact(left_o)); + } + + op(_GUARD_TOS_FLOAT, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyFloat_CheckExact(value_o)); } - op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_ADD_FLOAT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } - op(_BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_MULTIPLY_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_ADD_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_ADD_FLOAT; macro(BINARY_OP_SUBTRACT_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + EXIT_IF(!PyUnicode_CheckExact(left_o)); + EXIT_IF(!PyUnicode_CheckExact(right_o)); } - op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { + pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + _BINARY_OP_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_ADD_UNICODE; // This is a subtle one. It's a super-instruction for // BINARY_OP_ADD_UNICODE followed by STORE_FAST @@ -491,11 +581,13 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left); + tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + assert(next_instr->op.code == STORE_FAST); + _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(!PyStackRef_Is(*target_local, left)); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -508,17 +600,20 @@ dummy_func( * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(*target_local == NULL, error); + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(PyStackRef_IsNull(*target_local), error); // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); } macro(BINARY_OP_INPLACE_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + _BINARY_OP_INPLACE_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_INPLACE_ADD_UNICODE; family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { BINARY_SUBSCR_DICT, @@ -528,53 +623,68 @@ dummy_func( BINARY_SUBSCR_TUPLE_INT, }; - inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) { + specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ - res = PyObject_GetItem(container, sub); + } + + op(_BINARY_SUBSCR, (container, sub -- res)) { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + + PyObject *res_o = PyObject_GetItem(container_o, sub_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } + macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; + inst(BINARY_SLICE, (container, start, stop -- res)) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(container); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(STORE_SLICE, (v, container, start, stop -- )) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); ERROR_IF(err, error); } - inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { + inst(BINARY_SUBSCR_LIST_INT, (unused/1, list_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -583,14 +693,18 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { + inst(BINARY_SUBSCR_STR_INT, (unused/1, str_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyUnicode_CheckExact(str)); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -600,12 +714,16 @@ dummy_func( Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { + inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyTuple_CheckExact(tuple)); @@ -614,29 +732,33 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { + inst(BINARY_SUBSCR_DICT, (unused/1, dict_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - DECREF_INPUTS(); - ERROR_IF(true, error); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); + if (rc == 0) { + _PyErr_SetKeyError(sub); } - Py_INCREF(res); // Do this before DECREF'ing dict, sub DECREF_INPUTS(); + ERROR_IF(rc <= 0, error); // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { + inst(BINARY_SUBSCR_GETITEM, (unused/1, container_st, sub_st -- unused)) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); @@ -654,19 +776,20 @@ dummy_func( Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; + new_frame->localsplus[0] = container_st; + new_frame->localsplus[1] = sub_st; + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) { - ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error); + ERROR_IF(_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0, error); } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { - int err = PySet_Add(set, v); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -676,24 +799,31 @@ dummy_func( STORE_SUBSCR_LIST_INT, }; - inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { + specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } - inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { + macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -705,55 +835,64 @@ dummy_func( STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); } - inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { + inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); ERROR_IF(err, error); } inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ - int err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); DECREF_INPUTS(); ERROR_IF(err, error); } inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { + inst(CALL_INTRINSIC_2, (value2_st, value1_st -- res)) { assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(RAISE_VARARGS, (args[oparg] -- )) { + tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: - cause = args[1]; - /* fall through */ + cause = PyStackRef_AsPyObjectSteal(args[1]); + _Py_FALLTHROUGH; case 1: - exc = args[0]; - /* fall through */ + exc = PyStackRef_AsPyObjectSteal(args[0]); + _Py_FALLTHROUGH; case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; @@ -765,52 +904,43 @@ dummy_func( ERROR_IF(true, error); } - inst(INTERPRETER_EXIT, (retval --)) { + tier1 inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + return PyStackRef_AsPyObjectSteal(retval); } // The stack effect here is ambiguous. // We definitely pop the return value off the stack on entry. // We also push it onto the stack on exit, but that's a // different frame, and it's accounted for by _PUSH_FRAME. - op(_POP_FRAME, (retval --)) { - assert(EMPTY()); + inst(RETURN_VALUE, (retval -- res)) { #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); } - macro(RETURN_VALUE) = - _SAVE_CURRENT_IP + // Sets frame->prev_instr - _POP_FRAME; - inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -820,22 +950,21 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } macro(RETURN_CONST) = LOAD_CONST + - _SAVE_CURRENT_IP + // Sets frame->prev_instr - _POP_FRAME; + RETURN_VALUE; inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; + frame, this_instr, retval); + if (err) ERROR_NO_POP(); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -845,14 +974,16 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); + LOAD_IP(frame->return_offset); goto resume_frame; } inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; @@ -867,31 +998,34 @@ dummy_func( ERROR_IF(true, error); } - iter = (*getter)(obj); + iter_o = (*getter)(obj_o); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); ERROR_IF(true, error); } + iter = PyStackRef_FromPyObjectSteal(iter_o); } inst(GET_ANEXT, (aiter -- aiter, awaitable)) { unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - goto error; + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); + + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { + ERROR_NO_POP(); } } else { if (type->tp_as_async != NULL){ @@ -899,9 +1033,9 @@ dummy_func( } if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { - goto error; + ERROR_NO_POP(); } } else { @@ -909,11 +1043,11 @@ dummy_func( "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - goto error; + ERROR_NO_POP(); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " @@ -921,119 +1055,134 @@ dummy_func( Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - goto error; + ERROR_NO_POP(); } else { Py_DECREF(next_iter); } } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } inst(GET_AWAITABLE, (iterable -- iter)) { - iter = _PyCoro_GetAwaitableIter(iterable); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); } DECREF_INPUTS(); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { SEND_GEN, }; - inst(SEND, (unused/1, receiver, v -- receiver, retval)) { + specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_SEND, (receiver, v -- receiver, retval)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + + PyObject *retval_o; assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); } - if (retval == NULL) { + if (retval_o == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); + if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + assert(retval_o != NULL); JUMPBY(oparg); } else { - goto error; + ERROR_NO_POP(); } } - Py_DECREF(v); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } + macro(SEND) = _SPECIALIZE_SEND + _SEND; + inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { DEOPT_IF(tstate->interp->eval_frame); - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); - if (err) goto error; + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + if (err) ERROR_NO_POP(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1041,55 +1190,81 @@ dummy_func( frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - inst(YIELD_VALUE, (retval -- unused)) { + inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + #if TIER_ONE assert(frame != &entry_frame); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); } inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); + Py_XSETREF(exc_info->exc_value, + PyStackRef_Is(exc_value, PyStackRef_None) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); } - inst(RERAISE, (values[oparg], exc -- values[oparg])) { + tier1 inst(RERAISE, (values[oparg], exc_st -- values[oparg])) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = values[0]; + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - goto error; + ERROR_NO_POP(); } } assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - inst(END_ASYNC_FOR, (awaitable, exc -- )) { + tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1097,40 +1272,53 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } - inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); - none = Py_None; + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } - inst(LOAD_ASSERTION_ERROR, ( -- value)) { - value = Py_NewRef(PyExc_AssertionError); + inst(LOAD_COMMON_CONSTANT, ( -- value)) { + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } } inst(LOAD_BUILD_CLASS, ( -- bc)) { - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error); - if (bc == NULL) { + PyObject *bc_o; + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0, error); + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); ERROR_IF(true, error); } + bc = PyStackRef_FromPyObjectSteal(bc_o); } - inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -1142,9 +1330,9 @@ dummy_func( ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1156,7 +1344,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; + ERROR_NO_POP(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1164,7 +1352,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + ERROR_NO_POP(); } } @@ -1174,59 +1362,68 @@ dummy_func( UNPACK_SEQUENCE_LIST, }; - inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) { + specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + (void)seq; + (void)counter; + } + + op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } - inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); + macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; + + inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq)); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o)); + DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1237,91 +1434,112 @@ dummy_func( STORE_ATTR_WITH_HINT, }; - inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { + specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } + macro(STORE_ATTR) = _SPECIALIZE_STORE_ATTR + unused/3 + _STORE_ATTR; + inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); + int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - goto error; + if (err < 0) { + ERROR_NO_POP(); + } + if (err == 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + ERROR_NO_POP(); } } inst(LOAD_LOCALS, ( -- locals)) { - locals = LOCALS(); - if (locals == NULL) { + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); ERROR_IF(true, error); } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l);; } inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + PyObject *v_o; + if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + ERROR_NO_POP(); } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + ERROR_NO_POP(); + } } else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0, error); + if (v_o == NULL) { + /* namespace 2: builtins */ + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); + if (v_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + ERROR_IF(true, error); + } } } } DECREF_INPUTS(); + v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { + PyObject *v_o; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1329,29 +1547,26 @@ dummy_func( ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { + ERROR_NO_POP(); } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { + ERROR_NO_POP(); } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { + ERROR_NO_POP(); } - if (v == NULL) { + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + ERROR_NO_POP(); } } } + v = PyStackRef_FromPyObjectSteal(v_o); } family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { @@ -1359,26 +1574,29 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { + specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_LOAD_GLOBAL, ( -- res, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { + if (res_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1387,17 +1605,15 @@ dummy_func( } ERROR_IF(true, error); } - Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0, error); + if (res_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0, error); + if (res_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1405,9 +1621,17 @@ dummy_func( } } } - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } + macro(LOAD_GLOBAL) = + _SPECIALIZE_LOAD_GLOBAL + + counter/1 + + globals_version/1 + + builtins_version/1 + + _LOAD_GLOBAL; + op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict)); @@ -1425,21 +1649,23 @@ dummy_func( op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } macro(LOAD_GLOBAL_MODULE) = @@ -1455,71 +1681,76 @@ dummy_func( _LOAD_GLOBAL_BUILTINS; inst(DELETE_FAST, (--)) { - PyObject *v = GETLOCAL(oparg); - ERROR_IF(v == NULL, unbound_local_error); - SETLOCAL(oparg, NULL); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } + SETLOCAL(oparg, PyStackRef_NULL); } inst(MAKE_CELL, (--)) { // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto resume_with_error; + ERROR_NO_POP(); } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); } inst(DELETE_DEREF, (--)) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + ERROR_NO_POP(); } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); } - inst(LOAD_FROM_DICT_OR_DEREF, (class_dict -- value)) { + inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - Py_DECREF(class_dict); - goto error; - } - Py_DECREF(class_dict); - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + ERROR_NO_POP(); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + ERROR_NO_POP(); } - Py_INCREF(value); } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); } inst(LOAD_DEREF, ( -- value)) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } - Py_INCREF(value); + value = PyStackRef_FromPyObjectSteal(value_o); } inst(STORE_DEREF, (v --)) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); } inst(COPY_FREE_VARS, (--)) { @@ -1531,27 +1762,45 @@ dummy_func( int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } } inst(BUILD_STRING, (pieces[oparg] -- str)) { - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); DECREF_INPUTS(); - ERROR_IF(str == NULL, error); + ERROR_IF(str_o == NULL, error); + str = PyStackRef_FromPyObjectSteal(str_o); } inst(BUILD_TUPLE, (values[oparg] -- tup)) { - tup = _PyTuple_FromArraySteal(values, oparg); - ERROR_IF(tup == NULL, error); + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + ERROR_IF(tup_o == NULL, error); + tup = PyStackRef_FromPyObjectSteal(tup_o); } inst(BUILD_LIST, (values[oparg] -- list)) { - list = _PyList_FromArraySteal(values, oparg); - ERROR_IF(list == NULL, error); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + ERROR_IF(list_o == NULL, error); + list = PyStackRef_FromPyObjectSteal(list_o); } - inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) { + inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) { + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1570,35 +1819,46 @@ dummy_func( } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { - int err = _PySet_Update(set, iterable); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); DECREF_INPUTS(); ERROR_IF(err < 0, error); } inst(BUILD_SET, (values[oparg] -- set)) { - set = PySet_New(NULL); - if (set == NULL) - goto error; + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { + ERROR_NO_POP(); + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } Py_DECREF(item); } if (err != 0) { - Py_DECREF(set); + Py_DECREF(set_o); ERROR_IF(true, error); } + set = PyStackRef_FromPyObjectSteal(set_o); } inst(BUILD_MAP, (values[oparg*2] -- map)) { - map = _PyDict_FromItems( - values, 2, - values+1, 2, + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(SETUP_ANNOTATIONS, (--)) { @@ -1610,57 +1870,48 @@ dummy_func( ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - ERROR_IF(_PyErr_Occurred(tstate), error); - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - ERROR_IF(ann_dict == NULL, error); - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - ERROR_IF(err, error); - } + ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + ERROR_IF(ann_dict == NULL, error); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + ERROR_IF(err, error); } else { - /* do the same if locals() is not a dict */ - ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - ERROR_IF(ann_dict == NULL, error); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - ERROR_IF(err, error); - } - else { - Py_DECREF(ann_dict); - } + Py_DECREF(ann_dict); } } inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); } DECREF_INPUTS(); ERROR_IF(true, error); @@ -1669,26 +1920,30 @@ dummy_func( } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); DECREF_INPUTS(); ERROR_IF(true, error); } DECREF_INPUTS(); } - inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) { + inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); + ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0, error); } - inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1697,28 +1952,31 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; + specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); ERROR_IF(err, error); } - // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; @@ -1728,12 +1986,12 @@ dummy_func( if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -1741,36 +1999,36 @@ dummy_func( } DECREF_INPUTS(); ERROR_IF(super == NULL, error); - attr = PyObject_GetAttr(super, name); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); Py_DECREF(super); - ERROR_IF(attr == NULL, error); - null = NULL; + ERROR_IF(PyStackRef_IsNull(attr), error); + null = PyStackRef_NULL; } - pseudo(LOAD_SUPER_METHOD) = { - LOAD_SUPER_ATTR, - }; - - pseudo(LOAD_ZERO_SUPER_METHOD) = { - LOAD_SUPER_ATTR, - }; + macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; - pseudo(LOAD_ZERO_SUPER_ATTR) = { - LOAD_SUPER_ATTR, - }; + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); + attr_st = PyStackRef_FromPyObjectSteal(attr); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); @@ -1778,20 +2036,22 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, + PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); ERROR_IF(true, error); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + + attr = PyStackRef_FromPyObjectSteal(attr_o); } family(LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { @@ -1809,29 +2069,31 @@ dummy_func( LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { + specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. - meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -1839,46 +2101,48 @@ dummy_func( something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - - NULL | meth | arg1 | ... | argN + meth | NULL | arg1 | ... | argN */ DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); - self_or_null = NULL; + ERROR_IF(attr_o == NULL, error); + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); + ERROR_IF(attr_o == NULL, error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - pseudo(LOAD_METHOD) = { - LOAD_ATTR, - }; + macro(LOAD_ATTR) = + _SPECIALIZE_LOAD_ATTR + + unused/8 + + _LOAD_ATTR; op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); + EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL); + split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -1889,23 +2153,26 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - op(_CHECK_ATTR_MODULE, (type_version/2, owner -- owner)) { - DEOPT_IF(!PyModule_CheckExact(owner)); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyModule_CheckExact(owner_o)); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version); + DEOPT_IF(dict->ma_keys->dk_version != dict_version); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL); + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -1916,33 +2183,36 @@ dummy_func( unused/5; op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } - DEOPT_IF(attr == NULL); + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -1953,13 +2223,15 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL); + split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -1970,17 +2242,19 @@ dummy_func( unused/5; op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { - DEOPT_IF(!PyType_Check(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + DEOPT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -1991,12 +2265,14 @@ dummy_func( _LOAD_ATTR_CLASS; inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); @@ -2010,17 +2286,18 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -2036,47 +2313,48 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); + op(_GUARD_DORV_NO_DICT, (owner -- owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner_o)); + DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); + PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + _GUARD_TYPE_VERSION + - _GUARD_DORV_VALUES + + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; - inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner --)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2088,35 +2366,42 @@ dummy_func( DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { _PyObject_GC_TRACK(dict); } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } + macro(STORE_ATTR_WITH_HINT) = + unused/1 + + _GUARD_TYPE_VERSION + + _STORE_ATTR_WITH_HINT; + op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_SLOT) = @@ -2130,225 +2415,336 @@ dummy_func( COMPARE_OP_STR, }; - inst(COMPARE_OP, (unused/1, left, right -- res)) { + specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_COMPARE_OP, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); ERROR_IF(res_bool < 0, error); - res = res_bool ? Py_True : Py_False; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); } } - inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; + + macro(COMPARE_OP_FLOAT) = + _GUARD_BOTH_FLOAT + unused/1 + _COMPARE_OP_FLOAT; + + macro(COMPARE_OP_INT) = + _GUARD_BOTH_INT + unused/1 + _COMPARE_OP_INT; + + macro(COMPARE_OP_STR) = + _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; + + op(_COMPARE_OP_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT - inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); + op(_COMPARE_OP_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT, but for ==, != only - inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + op(_COMPARE_OP_STR, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } inst(IS_OP, (left, right -- b)) { - int res = Py_Is(left, right) ^ oparg; +#ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; +#else + int res = PyStackRef_Is(left, right) ^ oparg; +#endif + DECREF_INPUTS(); + b = res ? PyStackRef_True : PyStackRef_False; + } + + family(CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP) = { + CONTAINS_OP_SET, + CONTAINS_OP_DICT, + }; + + op(_CONTAINS_OP, (left, right -- b)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + int res = PySequence_Contains(right_o, left_o); + DECREF_INPUTS(); + ERROR_IF(res < 0, error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + + specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ContainsOp(right, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CONTAINS_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + + macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP; + + inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + int res = _PySet_Contains((PySetObject *)right_o, left_o); DECREF_INPUTS(); - b = res ? Py_True : Py_False; + ERROR_IF(res < 0, error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - inst(CONTAINS_OP, (left, right -- b)) { - int res = PySequence_Contains(right, left); + inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + int res = PyDict_Contains(right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) { + inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match_o, &rest_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - assert((match == NULL) == (rest == NULL)); - ERROR_IF(match == NULL, error); + assert((match_o == NULL) == (rest_o == NULL)); + ERROR_IF(match_o == NULL, error); - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); } inst(CHECK_EXC_MATCH, (left, right -- left, b)) { - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - int res = PyErr_GivenExceptionMatches(left, right); + int res = PyErr_GivenExceptionMatches(left_o, right_o); DECREF_INPUTS(); - b = res ? Py_True : Py_False; + b = res ? PyStackRef_True : PyStackRef_False; } - inst(IMPORT_NAME, (level, fromlist -- res)) { + tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); + PyObject *res_o = import_name(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(IMPORT_FROM, (from -- from, res)) { + tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - ERROR_IF(res == NULL, error); + PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(JUMP_FORWARD, (--)) { + tier1 inst(JUMP_FORWARD, (--)) { JUMPBY(oparg); } - inst(JUMP_BACKWARD, (--)) { + tier1 inst(JUMP_BACKWARD, (unused/1 --)) { CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); - JUMPBY(1-oparg); + JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && - // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) - { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + else { + this_instr[1].counter = restart_backoff_counter(counter); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } - pseudo(JUMP) = { + pseudo(JUMP, (--)) = { JUMP_FORWARD, JUMP_BACKWARD, }; - pseudo(JUMP_NO_INTERRUPT) = { + pseudo(JUMP_NO_INTERRUPT, (--)) = { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, }; - inst(ENTER_EXECUTOR, (--)) { - CHECK_EVAL_BREAKER(); - + tier1 inst(ENTER_EXECUTOR, (--)) { + #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; - Py_INCREF(executor); - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); } - goto resume_frame; + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ } - inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } - inst(POP_JUMP_IF_TRUE, (unused/1, cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } op(_IS_NONE, (value -- b)) { - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; + b = PyStackRef_False; DECREF_INPUTS(); } } - macro(POP_JUMP_IF_NONE) = _IS_NONE + POP_JUMP_IF_TRUE; + macro(POP_JUMP_IF_TRUE) = unused/1 + _POP_JUMP_IF_TRUE; + + macro(POP_JUMP_IF_FALSE) = unused/1 + _POP_JUMP_IF_FALSE; + + macro(POP_JUMP_IF_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_TRUE; - macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + POP_JUMP_IF_FALSE; + macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -2357,55 +2753,63 @@ dummy_func( JUMPBY(-oparg); } - inst(GET_LEN, (obj -- obj, len_o)) { + inst(GET_LEN, (obj -- obj, len)) { // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); ERROR_IF(len_i < 0, error); - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); ERROR_IF(len_o == NULL, error); + len = PyStackRef_FromPyObjectSteal(len_o); } inst(MATCH_CLASS, (subject, type, names -- attrs)) { // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); DECREF_INPUTS(); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { ERROR_IF(_PyErr_Occurred(tstate), error); // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } } inst(MATCH_MAPPING, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_SEQUENCE, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - ERROR_IF(values_or_none == NULL, error); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + ERROR_IF(values_or_none_o == NULL, error); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(PyStackRef_IsNull(iter), error); } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -2413,18 +2817,18 @@ dummy_func( _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - goto error; + ERROR_NO_POP(); } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - goto error; + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { + ERROR_NO_POP(); } DECREF_INPUTS(); } @@ -2443,116 +2847,140 @@ dummy_func( FOR_ITER_GEN, }; - inst(FOR_ITER, (unused/1, iter -- iter, next)) { + specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ + } + + replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + ERROR_NO_POP(); } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } - inst(INSTRUMENTED_FOR_ITER, ( -- )) { - _Py_CODEUNIT *here = next_instr-1; - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; - } - else { + op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + ERROR_NO_POP(); } - monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + /* The translator sets the deopt target just past the matching END_FOR */ + DEOPT_IF(true); } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } - op(_ITER_CHECK_LIST, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type); - } + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; - op(_ITER_JUMP_LIST, (iter -- iter)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { + _Py_CODEUNIT *target; + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + ERROR_NO_POP(); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + PyStackRef_CLOSE(iter_stackref); + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + } + + op(_ITER_CHECK_LIST, (iter -- iter)) { + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type); + } + + replaced op(_ITER_JUMP_LIST, (iter -- iter)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; Py_DECREF(seq); } - Py_DECREF(iter); + #endif + PyStackRef_CLOSE(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } // Only used by Tier 2 - op(_IS_ITER_EXHAUSTED_LIST, (iter -- iter, exhausted)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) { - exhausted = Py_True; - } - else if (it->it_index >= PyList_GET_SIZE(seq)) { - Py_DECREF(seq); - it->it_seq = NULL; - exhausted = Py_True; - } - else { - exhausted = Py_False; + EXIT_IF(seq == NULL); + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + EXIT_IF(1); } } op(_ITER_NEXT_LIST, (iter -- iter, next)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_LIST) = @@ -2562,12 +2990,13 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type); } - op(_ITER_JUMP_TUPLE, (iter -- iter)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { @@ -2575,40 +3004,32 @@ dummy_func( it->it_seq = NULL; Py_DECREF(seq); } - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); DISPATCH(); } } // Only used by Tier 2 - op(_IS_ITER_EXHAUSTED_TUPLE, (iter -- iter, exhausted)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) { - exhausted = Py_True; - } - else if (it->it_index >= PyTuple_GET_SIZE(seq)) { - Py_DECREF(seq); - it->it_seq = NULL; - exhausted = Py_True; - } - else { - exhausted = Py_False; - } + EXIT_IF(seq == NULL); + EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); } op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_TUPLE) = @@ -2618,40 +3039,40 @@ dummy_func( _ITER_NEXT_TUPLE; op(_ITER_CHECK_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type); + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); } - op(_ITER_JUMP_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + PyStackRef_CLOSE(iter); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); DISPATCH(); } } // Only used by Tier 2 - op(_IS_ITER_EXHAUSTED_RANGE, (iter -- iter, exhausted)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - exhausted = r->len <= 0 ? Py_True : Py_False; + EXIT_IF(r->len <= 0); } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - ERROR_IF(next == NULL, error); + PyObject *res = PyLong_FromLong(value); + ERROR_IF(res == NULL, error); + next = PyStackRef_FromPyObjectSteal(res); } macro(FOR_ITER_RANGE) = @@ -2660,166 +3081,132 @@ dummy_func( _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; - inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { - DEOPT_IF(tstate->interp->eval_frame); - PyGenObject *gen = (PyGenObject *)iter; + op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; - DISPATCH_INLINED(gen_frame); + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } - inst(BEFORE_ASYNC_WITH, (mgr -- exit, res)) { - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - goto error; - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - goto error; - } - DECREF_INPUTS(); - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - ERROR_IF(true, error); - } - } + macro(FOR_ITER_GEN) = + unused/1 + + _CHECK_PEP_523 + + _FOR_ITER_GEN_FRAME + + _PUSH_FRAME; - inst(BEFORE_WITH, (mgr -- exit, res)) { - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - goto error; - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { + inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) { + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); } - Py_DECREF(enter); - goto error; - } - DECREF_INPUTS(); - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - ERROR_IF(true, error); } + ERROR_IF(PyStackRef_IsNull(attr), error); + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); } - inst(WITH_EXCEPT_START, (exit_func, lasti, unused, val -- exit_func, lasti, unused, val, res)) { + inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - ERROR_IF(res == NULL, error); + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + ERROR_IF(PyStackRef_IsNull(res), error); } - pseudo(SETUP_FINALLY) = { + pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { + /* If an exception is raised, restore the stack position + * and push one value before jumping to the handler. + */ NOP, }; - pseudo(SETUP_CLEANUP) = { + pseudo(SETUP_CLEANUP, (-- unused, unused), (HAS_ARG)) = { + /* As SETUP_FINALLY, but push lasti as well */ NOP, }; - pseudo(SETUP_WITH) = { + pseudo(SETUP_WITH, (-- unused), (HAS_ARG)) = { + /* If an exception is raised, restore the stack position to the + * position before the result of __(a)enter__ and push 2 values + * before jumping to the handler. + */ NOP, }; - pseudo(POP_BLOCK) = { + pseudo(POP_BLOCK, (--)) = { NOP, }; inst(PUSH_EXC_INFO, (new_exc -- prev_exc, new_exc)) { + _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2832,11 +3219,11 @@ dummy_func( op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2851,7 +3238,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) = @@ -2863,11 +3250,11 @@ dummy_func( op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) = @@ -2876,10 +3263,9 @@ dummy_func( unused/2 + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } @@ -2889,7 +3275,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2897,21 +3283,20 @@ dummy_func( unused/1 + _GUARD_TYPE_VERSION + _CHECK_ATTR_METHOD_LAZY_DICT + - unused/2 + + unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, ( -- )) { - int is_meth = PEEK(oparg + 1) != NULL; + inst(INSTRUMENTED_CALL, (unused/3 -- )) { + int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); } @@ -2920,7 +3305,6 @@ dummy_func( family(CALL, INLINE_CACHE_ENTRIES_CALL) = { CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, - CALL_PY_WITH_DEFAULTS, CALL_TYPE_1, CALL_STR_1, CALL_TUPLE_1, @@ -2936,45 +3320,54 @@ dummy_func( CALL_METHOD_DESCRIPTOR_NOARGS, CALL_METHOD_DESCRIPTOR_FAST, CALL_ALLOC_AND_ENTER_INIT, + CALL_PY_GENERAL, + CALL_BOUND_METHOD_GENERAL, + CALL_NON_PY_GENERAL, }; + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // When calling Python, inline the call using DISPATCH_INLINED(). - inst(CALL, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_Call(callable, next_instr, total_args); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -2982,111 +3375,233 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + ERROR_NO_POP(); } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + + op(_CHECK_PERIODIC, (--)) { CHECK_EVAL_BREAKER(); } + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_PY_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(!PyStackRef_IsNull(null)); + } + + op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); + } + + macro(CALL_BOUND_METHOD_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); + } + + op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + + macro(CALL_NON_PY_GENERAL) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE + + _CALL_NON_PY_GENERAL + + _CHECK_PERIODIC; + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); + EXIT_IF(!PyStackRef_IsNull(null)); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; } op(_CHECK_PEP_523, (--)) { DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - DEOPT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version); + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL)); + EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); DEOPT_IF(tstate->py_recursion_remaining <= 1); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } - // The 'unused' output effect represents the return value - // (which will be pushed when the frame returns). - // It is needed so CALL_PY_EXACT_ARGS matches its family. - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- )) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } macro(CALL_BOUND_METHOD_EXACT_ARGS) = @@ -3094,105 +3609,87 @@ dummy_func( _CHECK_PEP_523 + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + _CHECK_FUNCTION_VERSION + _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; macro(CALL_PY_EXACT_ARGS) = unused/1 + // Skip over the counter _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount); - DEOPT_IF(argcount < min_args); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } + inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); } - inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + op(_CALL_STR_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } - inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + macro(CALL_STR_1) = + unused/1 + + unused/2 + + _CALL_STR_1 + + _CHECK_PERIODIC; + + op(_CALL_TUPLE_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } + macro(CALL_TUPLE_1) = + unused/1 + + unused/2 + + _CALL_TUPLE_1 + + _CHECK_PERIODIC; + inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; - DEOPT_IF(null != NULL); - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; DEOPT_IF(code->co_argcount != oparg+1); @@ -3200,25 +3697,23 @@ dummy_func( STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - goto error; + ERROR_NO_POP(); } - Py_DECREF(tp); + PyStackRef_CLOSE(callable); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); - shim->localsplus[0] = self; + shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; + init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; + frame->return_offset = (uint16_t)(next_instr - this_instr); STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); /* Link frames */ @@ -3235,390 +3730,504 @@ dummy_func( inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - goto error; + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + ERROR_NO_POP(); } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_BUILTIN_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_CLASS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_CLASS + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_BUILTIN_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_O) = + unused/1 + + unused/2 + + _CALL_BUILTIN_O + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_FAST) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } + macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len); + DEOPT_IF(callable_o != interp->callable_cache.len); STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - goto error; + ERROR_NO_POP(); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(callable); - Py_DECREF(arg); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + GOTO_ERROR(error); + } + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance); + DEOPT_IF(callable_o != interp->callable_cache.isinstance); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { - goto error; + ERROR_NO_POP(); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); } // This is secretly a super-instruction - inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { + tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self)); + DEOPT_IF(callable_o != interp->callable_cache.list_append); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } - inst(CALL_METHOD_DESCRIPTOR_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(total_args != 2); DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_O) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_O + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); DEOPT_IF(meth->ml_flags != METH_NOARGS); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_NOARGS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_NOARGS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL); - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } + macro(CALL_METHOD_DESCRIPTOR_FAST) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST + + _CHECK_PERIODIC; + inst(INSTRUMENTED_CALL_KW, ( -- )) { - int is_meth = PEEK(oparg + 2) != NULL; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o ); - Py_DECREF(kwnames); + PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + ERROR_NO_POP(); } - frame->return_offset = 0; + assert(next_instr - this_instr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); CHECK_EVAL_BREAKER(); } @@ -3626,43 +4235,50 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) { + inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - goto error; + ERROR_NO_POP(); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - goto error; + ERROR_NO_POP(); } - Py_SETREF(callargs, tuple); + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); - if (err) goto error; - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); - if (err < 0) { - Py_CLEAR(result); + frame, this_instr, func, arg); + if (err) ERROR_NO_POP(); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (PyStackRef_IsNull(result)) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + PyStackRef_CLEAR(result); + } } } } @@ -3676,40 +4292,45 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - goto error; + ERROR_NO_POP(); } - frame->return_offset = 0; + assert(next_instr - this_instr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); } DECREF_INPUTS(); - assert(PEEK(2 + (oparg & 1)) == NULL); - ERROR_IF(result == NULL, error); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + ERROR_IF(PyStackRef_IsNull(result), error); CHECK_EVAL_BREAKER(); } - inst(MAKE_FUNCTION, (codeobj -- func)) { + inst(MAKE_FUNCTION, (codeobj_st -- func)) { + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { - goto error; + ERROR_NO_POP(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); } - inst(SET_FUNCTION_ATTRIBUTE, (attr, func -- func)) { + inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_st -- func_st)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -3731,56 +4352,70 @@ dummy_func( assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } } - inst(RETURN_GENERATOR, (--)) { + inst(RETURN_GENERATOR, (-- res)) { assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - goto error; + ERROR_NO_POP(); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - goto resume_frame; + LOAD_IP(frame->return_offset); + LOAD_SP(); + LLTRACE_RESUME_FRAME(); } inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { - slice = PySlice_New(start, stop, step); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); - ERROR_IF(slice == NULL, error); + ERROR_IF(slice_o == NULL, error); + slice = PyStackRef_FromPyObjectSteal(slice_o); } inst(CONVERT_VALUE, (value -- result)) { - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - ERROR_IF(result == NULL, error); + conv_fn = _PyEval_ConversionFuncs[oparg]; + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + ERROR_IF(result_o == NULL, error); + result = PyStackRef_FromPyObjectSteal(result_o); } inst(FORMAT_SIMPLE, (value -- res)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - ERROR_IF(res == NULL, error); + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + ERROR_IF(PyStackRef_IsNull(res), error); } else { res = value; @@ -3788,49 +4423,57 @@ dummy_func( } inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); } - inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { + specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); + } + + op(_BINARY_OP, (lhs, rhs -- res)) { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(SWAP, (bottom, unused[oparg-2], top -- + macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; + + pure inst(SWAP, (bottom, unused[oparg-2], top -- top, unused[oparg-2], bottom)) { assert(oparg >= 2); } inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); + tstate, frame, this_instr); ERROR_IF(next_opcode < 0, error); - next_instr--; + next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3838,75 +4481,71 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } - inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { + inst(INSTRUMENTED_JUMP_BACKWARD, (unused/1 -- )) { CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsTrue(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsFalse(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsNone(value); + _PyStackRef value_stackref = POP(); + int flag = PyStackRef_Is(value_stackref, PyStackRef_None); int offset; if (flag) { offset = oparg; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = 0; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; + _PyStackRef value_stackref = POP(); int offset; - int nflag = Py_IsNone(value); + int nflag = PyStackRef_Is(value_stackref, PyStackRef_None); if (nflag) { offset = 0; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = oparg; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } - inst(EXTENDED_ARG, ( -- )) { + tier1 inst(EXTENDED_ARG, ( -- )) { assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -3914,58 +4553,235 @@ dummy_func( DISPATCH_GOTO(); } - inst(CACHE, (--)) { + tier1 inst(CACHE, (--)) { assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + Py_FatalError("Executing a cache."); } - inst(RESERVED, (--)) { + tier1 inst(RESERVED, (--)) { assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + Py_FatalError("Executing RESERVED instruction."); } ///////// Tier-2 only opcodes ///////// - op(_POP_JUMP_IF_FALSE, (flag -- )) { - if (Py_IsFalse(flag)) { - pc = oparg; - } + op (_GUARD_IS_TRUE_POP, (flag -- )) { + SYNC_SP(); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_True)); + assert(PyStackRef_Is(flag, PyStackRef_True)); + } + + op (_GUARD_IS_FALSE_POP, (flag -- )) { + SYNC_SP(); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_False)); + assert(PyStackRef_Is(flag, PyStackRef_False)); } - op(_POP_JUMP_IF_TRUE, (flag -- )) { - if (Py_IsTrue(flag)) { - pc = oparg; + op (_GUARD_IS_NONE_POP, (val -- )) { + SYNC_SP(); + if (!PyStackRef_Is(val, PyStackRef_None)) { + PyStackRef_CLOSE(val); + EXIT_IF(1); } } + op (_GUARD_IS_NOT_NONE_POP, (val -- )) { + SYNC_SP(); + EXIT_IF(PyStackRef_Is(val, PyStackRef_None)); + PyStackRef_CLOSE(val); + } + op(_JUMP_TO_TOP, (--)) { - pc = 0; - CHECK_EVAL_BREAKER(); + JUMP_TO_JUMP_TARGET(); } - op(_SET_IP, (--)) { - TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + tier2 op(_SET_IP, (instr_ptr/4 --)) { + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } - op(_SAVE_CURRENT_IP, (--)) { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + tier2 op(_CHECK_STACK_SPACE_OPERAND, (framesize/2 --)) { + assert(framesize <= INT_MAX); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); } - op(_EXIT_TRACE, (--)) { - frame->prev_instr--; // Back up to just before destination - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); + op(_SAVE_RETURN_OFFSET, (--)) { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + + tier2 op(_EXIT_TRACE, (--)) { + _PyExitData *exit = ¤t_executor->exits[oparg]; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + oparg, exit->temperature.as_counter, + (int)(target - _PyCode_CODE(code)), + _PyOpcode_OpName[target->op.code]); + } + #endif + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + if (optimized < 0) { + Py_DECREF(current_executor); + tstate->previous_executor = Py_None; + GOTO_UNWIND(); + } + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + } + exit->executor = executor; + } + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + } + + tier2 op(_CHECK_VALIDITY, (--)) { + DEOPT_IF(!current_executor->vm_data.valid); + } + + tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = PyStackRef_FromPyObjectNew(ptr); + } + + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = PyStackRef_FromPyObjectImmortal(ptr); + } + + tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); + } + + tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + value = PyStackRef_FromPyObjectNew(ptr); + null = PyStackRef_NULL; + } + + tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + value = PyStackRef_FromPyObjectImmortal(ptr); + null = PyStackRef_NULL; + } + + tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { + assert(PyFunction_Check(frame->f_funcobj)); + DEOPT_IF(((PyFunctionObject *)frame->f_funcobj)->func_version != func_version); + } + + /* Internal -- for testing executors */ + op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); + exe->count++; + } + + tier2 op(_DYNAMIC_EXIT, (--)) { + tstate->previous_executor = (PyObject *)current_executor; + _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _Py_CODEUNIT *target = frame->instr_ptr; + #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - return frame; + if (lltrace >= 2) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + oparg, exit->temperature.as_counter, + (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), + _PyOpcode_OpName[target->op.code]); + } + #endif + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + if (!backoff_counter_triggers(exit->temperature)) { + exit->temperature = advance_backoff_counter(exit->temperature); + GOTO_TIER_ONE(target); + } + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + if (optimized < 0) { + Py_DECREF(current_executor); + tstate->previous_executor = Py_None; + GOTO_UNWIND(); + } + GOTO_TIER_ONE(target); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + GOTO_TIER_TWO(executor); + } + + tier2 op(_START_EXECUTOR, (executor/4 --)) { + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; +#ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; +#endif + DEOPT_IF(!((_PyExecutorObject *)executor)->vm_data.valid); + } + + tier2 op(_FATAL_ERROR, (--)) { + assert(0); + Py_FatalError("Fatal error uop executed."); + } + + tier2 op(_CHECK_VALIDITY_AND_SET_IP, (instr_ptr/4 --)) { + DEOPT_IF(!current_executor->vm_data.valid); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + } + + tier2 op(_DEOPT, (--)) { + EXIT_TO_TIER1(); } - op(_INSERT, (unused[oparg], top -- top, unused[oparg])) { - // Inserts TOS at position specified by oparg; - memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + tier2 op(_ERROR_POP_N, (target/2, unused[oparg] --)) { + frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; + SYNC_SP(); + GOTO_UNWIND(); } + /* Progress is guaranteed if we DEOPT on the eval breaker, because + * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ + tier2 op(_TIER2_RESUME_CHECK, (--)) { +#if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index ac40425263931f..d8bc830f8e80c1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4,18 +4,23 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" #include "pycore_instruments.h" #include "pycore_intrinsics.h" +#include "pycore_jit.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -24,7 +29,7 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() -#include "pycore_uops.h" // _PyUOpExecutorObject +#include "pycore_uop_ids.h" // Uops #include "pycore_pyerrors.h" #include "pycore_dict.h" @@ -34,6 +39,7 @@ #include "opcode.h" #include "pydtrace.h" #include "setobject.h" +#include "pycore_stackref.h" #include // bool @@ -46,12 +52,13 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif -#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED) // GH-89279: The MSVC compiler does not inline these static inline functions // in PGO build in _PyEval_EvalFrameDefault(), because this function is over // the limit of PGO, and that limit cannot be configured. // Define them as macros to make sure that they are always inlined by the // preprocessor. +// TODO: implement Py_DECREF macro for Py_GIL_DISABLED #undef Py_DECREF #define Py_DECREF(arg) \ @@ -98,33 +105,34 @@ #ifdef LLTRACE static void -dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - PyObject **stack_base = _PyFrame_Stackbase(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); - for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) { + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { if (ptr != stack_base) { printf(", "); } - if (*ptr == NULL) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr); + if (obj == NULL) { printf(""); continue; } if ( - *ptr == Py_None - || PyBool_Check(*ptr) - || PyLong_CheckExact(*ptr) - || PyFloat_CheckExact(*ptr) - || PyUnicode_CheckExact(*ptr) + obj == Py_None + || PyBool_Check(obj) + || PyLong_CheckExact(obj) + || PyFloat_CheckExact(obj) + || PyUnicode_CheckExact(obj) ) { - if (PyObject_Print(*ptr, stdout, 0) == 0) { + if (PyObject_Print(obj, stdout, 0) == 0) { continue; } PyErr_Clear(); } // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits)); } printf("]\n"); fflush(stdout); @@ -133,15 +141,15 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) static void lltrace_instruction(_PyInterpreterFrame *frame, - PyObject **stack_pointer, - _Py_CODEUNIT *next_instr) + _PyStackRef *stack_pointer, + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) { if (frame->owner == FRAME_OWNED_BY_CSTACK) { return; } dump_stack(frame, stack_pointer); - int oparg = next_instr->op.arg; - int opcode = next_instr->op.code; const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); int offset = (int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame))); @@ -201,15 +209,15 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip if (r < 0) { return -1; } - int lltrace = r; + int lltrace = r * 5; // Levels 1-4 only trace uops if (!lltrace) { - // When tracing executed uops, also trace bytecode - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that + // Can also be controlled by environment variable + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } - if (lltrace) { + if (lltrace >= 5) { lltrace_resume_frame(frame); } return lltrace; @@ -225,7 +233,8 @@ static void monitor_reraise(PyThreadState *tstate, _Py_CODEUNIT *instr); static int monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, + PyObject *value); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -241,10 +250,6 @@ static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); @@ -334,6 +339,35 @@ const binaryfunc _PyEval_BinaryOps[] = { [NB_INPLACE_XOR] = PyNumber_InPlaceXor, }; +const conversion_func _PyEval_ConversionFuncs[4] = { + [FVC_STR] = PyObject_Str, + [FVC_REPR] = PyObject_Repr, + [FVC_ASCII] = PyObject_ASCII +}; + +const _Py_SpecialMethod _Py_SpecialMethods[] = { + [SPECIAL___ENTER__] = { + .name = &_Py_ID(__enter__), + .error = "'%.200s' object does not support the " + "context manager protocol (missed __enter__ method)", + }, + [SPECIAL___EXIT__] = { + .name = &_Py_ID(__exit__), + .error = "'%.200s' object does not support the " + "context manager protocol (missed __exit__ method)", + }, + [SPECIAL___AENTER__] = { + .name = &_Py_ID(__aenter__), + .error = "'%.200s' object does not support the asynchronous " + "context manager protocol (missed __aenter__ method)", + }, + [SPECIAL___AEXIT__] = { + .name = &_Py_ID(__aexit__), + .error = "'%.200s' object does not support the asynchronous " + "context manager protocol (missed __aexit__ method)", + } +}; + // PEP 634: Structural Pattern Matching @@ -610,10 +644,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); } -#define TIER_ONE 1 #include "ceval_macros.h" - int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -635,17 +667,23 @@ int _Py_CheckRecursiveCallPy( return 0; } - static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, - { .op.code = RESUME, .op.arg = 0 } + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ + { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; extern const struct _PyCode_DEF(8) _Py_InitCleanup; +#ifdef Py_DEBUG +extern void _PyUOpPrint(const _PyUOpInstruction *uop); +#endif + + /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -659,6 +697,35 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; #endif +PyObject ** +_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) +{ + PyObject **result; + if (nargs > MAX_STACKREF_SCRATCH) { + // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set. + result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); + if (result == NULL) { + return NULL; + } + result++; + } + else { + result = scratch; + } + for (int i = 0; i < nargs; i++) { + result[i] = PyStackRef_AsPyObjectBorrow(input[i]); + } + return result; +} + +void +_PyObjectArray_Free(PyObject **array, PyObject **scratch) +{ + if (array != scratch) { + PyMem_Free(array); + } +} + /* _PyEval_EvalFrameDefault() is a *big* function, * so consume 3 units of C stack */ #define PY_EVAL_C_STACK_UNITS 2 @@ -677,9 +744,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif - // opcode is an 8-bit value to improve the code generated by MSVC - // for the big switch below (in combination with the EXTRA_CASES macro). - uint8_t opcode; /* Current opcode */ + uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ #ifdef LLTRACE int lltrace = 0; @@ -698,7 +763,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.f_builtins = (PyObject*)0xaaa4; #endif entry_frame.f_executable = Py_None; - entry_frame.prev_instr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stacktop = 0; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; @@ -722,30 +787,30 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, * we need to update instrumentation */ _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - monitor_throw(tstate, frame, frame->prev_instr); + monitor_throw(tstate, frame, frame->instr_ptr); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } /* Local "register" variables. * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; - PyObject **stack_pointer; + _PyStackRef *stack_pointer; -/* Sets the above local variables from the frame */ -#define SET_LOCALS_FROM_FRAME() \ - /* Jump back to the last instruction executed... */ \ - next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); +#if defined(_Py_TIER2) && !defined(_Py_JIT) + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif start_frame: if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } + next_instr = frame->instr_ptr; resume_frame: - SET_LOCALS_FROM_FRAME(); + stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); @@ -774,7 +839,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->prev_instr before it is updated, + * because it needs to capture frame->instr_ptr before it is updated, * as happens in the standard instruction prologue. */ #if USE_COMPUTED_GOTOS @@ -783,25 +848,31 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int case INSTRUMENTED_LINE: #endif { - _Py_CODEUNIT *prev = frame->prev_instr; - _Py_CODEUNIT *here = frame->prev_instr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->prev_instr; - if (next_instr != here) { - DISPATCH(); + _Py_CODEUNIT *prev = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); /* Prevent the underlying instruction from specializing * and overwriting the instrumentation. */ - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; DISPATCH_GOTO(); @@ -811,7 +882,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if USE_COMPUTED_GOTOS _unknown_opcode: #else - EXTRA_CASES // From pycore_opcode.h, a 'case' for each unused opcode + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode #endif /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ @@ -829,15 +900,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); -unbound_local_error: - { - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - } - pop_4_error: STACK_SHRINK(1); pop_3_error: @@ -868,7 +930,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int monitor_raise(tstate, frame, next_instr-1); exception_unwind: { - /* We can't use frame->f_lasti here, as RERAISE may have set it */ + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { @@ -876,10 +938,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyErr_Occurred(tstate)); /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); + PyStackRef_XCLOSE(POP()); } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -888,10 +949,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); + PyStackRef_XCLOSE(POP()); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -899,7 +959,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (lasti == NULL) { goto exception_unwind; } - PUSH(lasti); + PUSH(PyStackRef_FromPyObjectSteal(lasti)); } /* Make the raw exception data @@ -907,14 +967,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int so a program can emulate the Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(exc); - JUMPTO(handler); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { goto exception_unwind; } /* Resume normal execution */ #ifdef LLTRACE - if (lltrace) { + if (lltrace >= 5) { lltrace_resume_frame(frame); } #endif @@ -939,10 +1000,151 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } resume_with_error: - SET_LOCALS_FROM_FRAME(); + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; + +#ifdef _Py_TIER2 + +// Tier 2 is also here! +enter_tier_two: + +#ifdef _Py_JIT + assert(0); +#else + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 + +#ifdef Py_DEBUG + #define DPRINTF(level, ...) \ + if (lltrace >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + ; // dummy statement after a label, before a declaration + uint16_t uopcode; +#ifdef Py_STATS + int lastuop = 0; + uint64_t trace_uop_execution_counter = 0; +#endif + + assert(next_uop->opcode == _START_EXECUTOR); +tier2_dispatch: + for (;;) { + uopcode = next_uop->opcode; +#ifdef Py_DEBUG + if (lltrace >= 3) { + if (next_uop->opcode == _START_EXECUTOR) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } + _PyUOpPrint(next_uop); + printf(" stack_level=%d\n", + (int)(stack_pointer - _PyFrame_Stackbase(frame))); + } +#endif + next_uop++; + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + UOP_PAIR_INC(uopcode, lastuop); +#ifdef Py_STATS + trace_uop_execution_counter++; +#endif + + switch (uopcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif + + } + } + +jump_to_error_target: +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + assert (next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +error_tier_two: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + frame->return_offset = 0; // Don't leave this random + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + goto resume_with_error; + +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +exit_to_tier1_dynamic: + next_instr = frame->instr_ptr; + goto goto_to_tier1; +exit_to_tier1: + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); +goto_to_tier1: +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("DEOPT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" -> %s]\n", + _PyOpcode_OpName[next_instr->op.code]); + } +#endif + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + DISPATCH(); + +#endif // _Py_JIT + +#endif // _Py_TIER2 + } + #if defined(__GNUC__) # pragma GCC diagnostic pop #elif defined(_MSC_VER) /* MS_WINDOWS */ @@ -1015,7 +1217,7 @@ format_missing(PyThreadState *tstate, const char *kind, static void missing_arguments(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { Py_ssize_t i, j = 0; Py_ssize_t start, end; @@ -1036,7 +1238,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, end = start + co->co_kwonlyargcount; } for (i = start; i < end; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *name = PyObject_Repr(raw); if (name == NULL) { @@ -1054,7 +1256,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, static void too_many_positional(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t given, PyObject *defaults, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { int plural; Py_ssize_t kwonly_given = 0; @@ -1065,7 +1267,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, assert((co->co_flags & CO_VARARGS) == 0); /* Count missing keyword-only args. */ for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { - if (localsplus[i] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) { kwonly_given++; } } @@ -1243,7 +1445,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i static int initialize_locals(PyThreadState *tstate, PyFunctionObject *func, - PyObject **localsplus, PyObject *const *args, + _PyStackRef *localsplus, _PyStackRef const *args, Py_ssize_t argcount, PyObject *kwnames) { PyCodeObject *co = (PyCodeObject*)func->func_code; @@ -1261,8 +1463,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { i++; } - assert(localsplus[i] == NULL); - localsplus[i] = kwdict; + assert(PyStackRef_IsNull(localsplus[i])); + localsplus[i] = PyStackRef_FromPyObjectSteal(kwdict); } else { kwdict = NULL; @@ -1277,9 +1479,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, n = argcount; } for (j = 0; j < n; j++) { - PyObject *x = args[j]; - assert(localsplus[j] == NULL); - localsplus[j] = x; + assert(PyStackRef_IsNull(localsplus[j])); + localsplus[j] = args[j]; } /* Pack other positional arguments into the *args argument */ @@ -1289,19 +1490,18 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, u = (PyObject *)&_Py_SINGLETON(tuple_empty); } else { - assert(args != NULL); - u = _PyTuple_FromArraySteal(args + n, argcount - n); + u = _PyTuple_FromStackRefSteal(args + n, argcount - n); } if (u == NULL) { goto fail_post_positional; } - assert(localsplus[total_args] == NULL); - localsplus[total_args] = u; + assert(PyStackRef_AsPyObjectBorrow(localsplus[total_args]) == NULL); + localsplus[total_args] = PyStackRef_FromPyObjectSteal(u); } else if (argcount > n) { - /* Too many postional args. Error is reported later */ + /* Too many positional args. Error is reported later */ for (j = n; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } @@ -1311,7 +1511,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, for (i = 0; i < kwcount; i++) { PyObject **co_varnames; PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); - PyObject *value = args[i+argcount]; + _PyStackRef value_stackref = args[i+argcount]; Py_ssize_t j; if (keyword == NULL || !PyUnicode_Check(keyword)) { @@ -1384,27 +1584,26 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - if (PyDict_SetItem(kwdict, keyword, value) == -1) { + if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectBorrow(value_stackref)) == -1) { goto kw_fail; } - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); continue; kw_fail: for (;i < kwcount; i++) { - PyObject *value = args[i+argcount]; - Py_DECREF(value); + PyStackRef_CLOSE(args[i+argcount]); } goto fail_post_args; kw_found: - if (localsplus[j] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[j]) != NULL) { _PyErr_Format(tstate, PyExc_TypeError, "%U() got multiple values for argument '%S'", func->func_qualname, keyword); goto kw_fail; } - localsplus[j] = value; + localsplus[j] = value_stackref; } } @@ -1421,7 +1620,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, Py_ssize_t m = co->co_argcount - defcount; Py_ssize_t missing = 0; for (i = argcount; i < m; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { missing++; } } @@ -1437,9 +1636,9 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (defcount) { PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); for (; i < defcount; i++) { - if (localsplus[m+i] == NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[m+i]) == NULL) { PyObject *def = defs[i]; - localsplus[m+i] = Py_NewRef(def); + localsplus[m+i] = PyStackRef_FromPyObjectNew(def); } } } @@ -1449,18 +1648,18 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_kwonlyargcount > 0) { Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { - if (localsplus[i] != NULL) + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) continue; PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (func->func_kwdefaults != NULL) { - PyObject *def = PyDict_GetItemWithError(func->func_kwdefaults, varname); + PyObject *def; + if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { + goto fail_post_args; + } if (def) { - localsplus[i] = Py_NewRef(def); + localsplus[i] = PyStackRef_FromPyObjectSteal(def); continue; } - else if (_PyErr_Occurred(tstate)) { - goto fail_post_args; - } } missing++; } @@ -1474,14 +1673,14 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, fail_pre_positional: for (j = 0; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } /* fall through */ fail_post_positional: if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (j = argcount; j < argcount+kwcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } /* fall through */ @@ -1509,7 +1708,7 @@ static void clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - PyGenObject *gen = _PyFrame_GetGenerator(frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); gen->gi_frame_state = FRAME_CLEARED; assert(tstate->exc_info == &gen->gi_exc_state); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -1534,9 +1733,9 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) } /* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * +_PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, + PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames) { PyCodeObject * code = (PyCodeObject *)func->func_code; @@ -1554,19 +1753,48 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return frame; fail: /* Consume the references */ + Py_DECREF(func); + Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); + PyStackRef_CLOSE(args[i+argcount]); } } PyErr_NoMemory(); return NULL; } +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject *const* args, + size_t argcount, PyObject *kwnames) +{ +#if defined(Py_GIL_DISABLED) + size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + size_t total_argcount = argcount + kw_count; + _PyStackRef *tagged_args_buffer = PyMem_Malloc(sizeof(_PyStackRef) * total_argcount); + if (tagged_args_buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (size_t i = 0; i < argcount; i++) { + tagged_args_buffer[i] = PyStackRef_FromPyObjectSteal(args[i]); + } + for (size_t i = 0; i < kw_count; i++) { + tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); + } + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + PyMem_Free(tagged_args_buffer); + return res; +#else + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); +#endif +} + /* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. Steals references to func, callargs and kwargs. */ @@ -1591,7 +1819,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(PyTuple_GET_ITEM(callargs, i)); } } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, newargs, nargs, kwnames ); @@ -1629,7 +1857,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(args[i+argcount]); } } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( tstate, func, locals, args, argcount, kwnames); if (frame == NULL) { return NULL; @@ -1775,6 +2003,13 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) fixed_cause = _PyObject_CallNoArgs(cause); if (fixed_cause == NULL) goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + goto raise_error; + } Py_DECREF(cause); } else if (PyExceptionInstance_Check(cause)) { @@ -1874,8 +2109,8 @@ _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, */ int -_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, + int argcnt, int argcntafter, _PyStackRef *sp) { int i = 0, j = 0; Py_ssize_t ll = 0; @@ -1883,6 +2118,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, PyObject *w; PyObject *l = NULL; /* variable list */ + PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref); assert(v != NULL); it = PyObject_GetIter(v); @@ -1917,7 +2153,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, } goto Error; } - *--sp = w; + *--sp = PyStackRef_FromPyObjectSteal(w); } if (argcntafter == -1) { @@ -1939,7 +2175,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, l = PySequence_List(it); if (l == NULL) goto Error; - *--sp = l; + *--sp = PyStackRef_FromPyObjectSteal(l); i++; ll = PyList_GET_SIZE(l); @@ -1952,7 +2188,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, /* Pop the "after-variable" args off the list. */ for (j = argcntafter; j > 0; j--, i++) { - *--sp = PyList_GET_ITEM(l, ll - j); + *--sp = PyStackRef_FromPyObjectSteal(PyList_GET_ITEM(l, ll - j)); } /* Resize the list. */ Py_SET_SIZE(l, ll - argcntafter); @@ -1960,8 +2196,9 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, return 1; Error: - for (; i > 0; i--, sp++) - Py_DECREF(*sp); + for (; i > 0; i--, sp++) { + PyStackRef_CLOSE(*sp); + } Py_XDECREF(it); return 0; } @@ -1971,6 +2208,9 @@ do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); @@ -2025,12 +2265,19 @@ monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, static int monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, PyObject *value) { if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; } - return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + if (res < 0) { + return res; + } + PyErr_SetRaisedException(NULL); + return 0; } static void @@ -2104,7 +2351,7 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg) PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetProfile(tstate, func, arg) < 0) { /* Log _PySys_Audit() error */ - _PyErr_WriteUnraisableMsg("in PyEval_SetProfile", NULL); + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfile"); } } @@ -2121,7 +2368,7 @@ PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) while (ts) { if (_PyEval_SetProfile(ts, func, arg) < 0) { - _PyErr_WriteUnraisableMsg("in PyEval_SetProfileAllThreads", NULL); + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } HEAD_LOCK(runtime); ts = PyThreadState_Next(ts); @@ -2135,7 +2382,7 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) PyThreadState *tstate = _PyThreadState_GET(); if (_PyEval_SetTrace(tstate, func, arg) < 0) { /* Log _PySys_Audit() error */ - _PyErr_WriteUnraisableMsg("in PyEval_SetTrace", NULL); + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTrace"); } } @@ -2152,7 +2399,7 @@ PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) while (ts) { if (_PyEval_SetTrace(ts, func, arg) < 0) { - _PyErr_WriteUnraisableMsg("in PyEval_SetTraceAllThreads", NULL); + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } HEAD_LOCK(runtime); ts = PyThreadState_Next(ts); @@ -2262,13 +2509,9 @@ PyEval_GetBuiltins(void) PyObject * _PyEval_GetBuiltin(PyObject *name) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *attr = PyDict_GetItemWithError(PyEval_GetBuiltins(), name); - if (attr) { - Py_INCREF(attr); - } - else if (!_PyErr_Occurred(tstate)) { - _PyErr_SetObject(tstate, PyExc_AttributeError, name); + PyObject *attr; + if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { + PyErr_SetObject(PyExc_AttributeError, name); } return attr; } @@ -2289,12 +2532,7 @@ PyEval_GetLocals(void) return NULL; } - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { - return NULL; - } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); + PyObject *locals = _PyEval_GetFrameLocals(); return locals; } @@ -2308,7 +2546,28 @@ _PyEval_GetFrameLocals(void) return NULL; } - return _PyFrame_GetLocals(current_frame, 1); + PyObject *locals = _PyFrame_GetLocals(current_frame); + if (locals == NULL) { + return NULL; + } + + if (PyFrameLocalsProxy_Check(locals)) { + PyObject* ret = PyDict_New(); + if (ret == NULL) { + Py_DECREF(locals); + return NULL; + } + if (PyDict_Update(ret, locals) < 0) { + Py_DECREF(ret); + Py_DECREF(locals); + return NULL; + } + Py_DECREF(locals); + return ret; + } + + assert(PyMapping_Check(locals)); + return locals; } PyObject * @@ -2322,6 +2581,28 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject* +PyEval_GetFrameLocals(void) +{ + return _PyEval_GetFrameLocals(); +} + +PyObject* PyEval_GetFrameGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; + } + return Py_XNewRef(current_frame->f_globals); +} + +PyObject* PyEval_GetFrameBuiltins(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return Py_XNewRef(_PyEval_GetBuiltins(tstate)); +} + int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { @@ -2419,12 +2700,12 @@ static PyObject * import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { - PyObject *import_func = _PyDict_GetItemWithError(frame->f_builtins, - &_Py_ID(__import__)); + PyObject *import_func; + if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + return NULL; + } if (import_func == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); - } + _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); return NULL; } @@ -2435,6 +2716,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, /* Fast path for not overloaded __import__. */ if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + Py_DECREF(import_func); int ilevel = PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; @@ -2448,7 +2730,6 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, } PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; - Py_INCREF(import_func); PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); Py_DECREF(import_func); return res; @@ -2466,11 +2747,10 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name) /* Issue #17636: in case this failed because of a circular relative import, try to fallback on reading the module directly from sys.modules. */ - pkgname = PyObject_GetAttr(v, &_Py_ID(__name__)); - if (pkgname == NULL) { - goto error; + if (PyObject_GetOptionalAttr(v, &_Py_ID(__name__), &pkgname) < 0) { + return NULL; } - if (!PyUnicode_Check(pkgname)) { + if (pkgname == NULL || !PyUnicode_Check(pkgname)) { Py_CLEAR(pkgname); goto error; } @@ -2487,42 +2767,59 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name) Py_DECREF(pkgname); return x; error: - pkgpath = PyModule_GetFilenameObject(v); if (pkgname == NULL) { pkgname_or_unknown = PyUnicode_FromString(""); if (pkgname_or_unknown == NULL) { - Py_XDECREF(pkgpath); return NULL; } } else { pkgname_or_unknown = pkgname; } + pkgpath = NULL; + if (PyModule_Check(v)) { + pkgpath = PyModule_GetFilenameObject(v); + if (pkgpath == NULL) { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(pkgname_or_unknown); + return NULL; + } + // module filename missing + _PyErr_Clear(tstate); + } + } if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) { - _PyErr_Clear(tstate); + Py_CLEAR(pkgpath); errmsg = PyUnicode_FromFormat( "cannot import name %R from %R (unknown location)", name, pkgname_or_unknown ); - /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, NULL, name); } else { - PyObject *spec = PyObject_GetAttr(v, &_Py_ID(__spec__)); + PyObject *spec; + int rc = PyObject_GetOptionalAttr(v, &_Py_ID(__spec__), &spec); + if (rc > 0) { + rc = _PyModuleSpec_IsInitializing(spec); + Py_DECREF(spec); + } + if (rc < 0) { + Py_DECREF(pkgname_or_unknown); + Py_DECREF(pkgpath); + return NULL; + } const char *fmt = - _PyModuleSpec_IsInitializing(spec) ? + rc ? "cannot import name %R from partially initialized module %R " "(most likely due to a circular import) (%S)" : "cannot import name %R from %R (%S)"; - Py_XDECREF(spec); errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath); - /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); } + /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ + _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); Py_XDECREF(errmsg); - Py_XDECREF(pkgname_or_unknown); + Py_DECREF(pkgname_or_unknown); Py_XDECREF(pkgpath); return NULL; } @@ -2694,7 +2991,7 @@ _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) if (_PyErr_Occurred(tstate)) return; name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - if (oparg < PyCode_GetFirstFree(co)) { + if (oparg < PyUnstable_Code_GetFirstFree(co)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, name); } else { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index bbb1e784dfa04e..5617504a495686 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1,6 +1,5 @@ #include "Python.h" -#include "pycore_atomic.h" // _Py_atomic_int #include "pycore_ceval.h" // _PyEval_SignalReceived() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() @@ -57,60 +56,52 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* bpo-40010: eval_breaker should be recomputed if there - is a pending signal: signal received by another thread which cannot - handle signals. - Similarly, we set CALLS_TO_DO and ASYNC_EXCEPTION to match the thread. -*/ +// Atomically copy the bits indicated by mask between two values. static inline void -update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState *tstate) +copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask) { - if (tstate == NULL) { + uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask; + uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to); + uintptr_t to_bits = old_value & mask; + if (from_bits == to_bits) { return; } - if (_Py_IsMainThread()) { - int32_t calls_to_do = _Py_atomic_load_int32_relaxed( - &_PyRuntime.ceval.pending_mainthread.calls_to_do); - if (calls_to_do) { - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); - } - if (_Py_ThreadCanHandleSignals(interp)) { - if (_Py_atomic_load_int(&_PyRuntime.signals.is_tripped)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } - } - } - if (tstate->async_exc != NULL) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); - } -} - -static inline void -SET_GIL_DROP_REQUEST(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1); -} - - -static inline void -RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0); + uintptr_t new_value; + do { + new_value = (old_value & ~mask) | from_bits; + } while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value)); } - +// When attaching a thread, set the global instrumentation version and +// _PY_CALLS_TO_DO_BIT from the current state of the interpreter. static inline void -SIGNAL_PENDING_CALLS(PyInterpreterState *interp) +update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); -} +#ifdef Py_GIL_DISABLED + // Free-threaded builds eagerly update the eval_breaker on *all* threads as + // needed, so this function doesn't apply. + return; +#endif + int32_t npending = _Py_atomic_load_int32_relaxed( + &interp->ceval.pending.npending); + if (npending) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + else if (_Py_IsMainThread()) { + npending = _Py_atomic_load_int32_relaxed( + &_PyRuntime.ceval.pending_mainthread.npending); + if (npending) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + } -static inline void -UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0); + // _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits + // we copy from our interpreter's state are the instrumentation version. + copy_eval_breaker_bits(&interp->ceval.instrumentation_version, + &tstate->eval_breaker, + ~_PY_EVAL_EVENTS_MASK); } /* @@ -120,9 +111,6 @@ UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) #include #include -#include "pycore_atomic.h" - - #include "condvar.h" #define MUTEX_INIT(mut) \ @@ -166,8 +154,7 @@ UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) static void _gil_initialize(struct _gil_runtime_state *gil) { - _Py_atomic_int uninitialized = {-1}; - gil->locked = uninitialized; + gil->locked = -1; gil->interval = DEFAULT_INTERVAL; } @@ -176,7 +163,7 @@ static int gil_created(struct _gil_runtime_state *gil) if (gil == NULL) { return 0; } - return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); + return (_Py_atomic_load_int_acquire(&gil->locked) >= 0); } static void create_gil(struct _gil_runtime_state *gil) @@ -191,7 +178,7 @@ static void create_gil(struct _gil_runtime_state *gil) #endif _Py_atomic_store_ptr_relaxed(&gil->last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked); - _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, 0); } static void destroy_gil(struct _gil_runtime_state *gil) @@ -205,8 +192,7 @@ static void destroy_gil(struct _gil_runtime_state *gil) COND_FINI(gil->switch_cond); MUTEX_FINI(gil->switch_mutex); #endif - _Py_atomic_store_explicit(&gil->locked, -1, - _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, -1); _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); } @@ -219,54 +205,70 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif +static inline void +drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) +{ + MUTEX_LOCK(gil->mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); + _Py_atomic_store_int_relaxed(&gil->locked, 0); + if (tstate != NULL) { + tstate->_status.holds_gil = 0; + } + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); +} + static void -drop_gil(PyInterpreterState *interp, PyThreadState *tstate) +drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { struct _ceval_state *ceval = &interp->ceval; - /* If tstate is NULL, the caller is indicating that we're releasing + /* If final_release is true, the caller is indicating that we're releasing the GIL for the last time in this thread. This is particularly relevant when the current thread state is finalizing or its interpreter is finalizing (either may be in an inconsistent state). In that case the current thread will definitely never try to acquire the GIL again. */ // XXX It may be more correct to check tstate->_status.finalizing. - // XXX assert(tstate == NULL || !tstate->_status.cleared); + // XXX assert(final_release || !tstate->_status.cleared); + assert(final_release || tstate != NULL); struct _gil_runtime_state *gil = ceval->gil; - if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { +#ifdef Py_GIL_DISABLED + // Check if we have the GIL before dropping it. tstate will be NULL if + // take_gil() detected that this thread has been destroyed, in which case + // we know we have the GIL. + if (tstate != NULL && !tstate->_status.holds_gil) { + return; + } +#endif + if (!_Py_atomic_load_int_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } - /* tstate is allowed to be NULL (early interpreter init) */ - if (tstate != NULL) { + if (!final_release) { /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - MUTEX_LOCK(gil->mutex); - _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); - _Py_atomic_store_relaxed(&gil->locked, 0); - COND_SIGNAL(gil->cond); - MUTEX_UNLOCK(gil->mutex); + drop_gil_impl(tstate, gil); #ifdef FORCE_SWITCHING - /* We check tstate first in case we might be releasing the GIL for - the last time in this thread. In that case there's a possible - race with tstate->interp getting deleted after gil->mutex is - unlocked and before the following code runs, leading to a crash. - We can use (tstate == NULL) to indicate the thread is done with - the GIL, and that's the only time we might delete the - interpreter, so checking tstate first prevents the crash. - See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + /* We might be releasing the GIL for the last time in this thread. In that + case there's a possible race with tstate->interp getting deleted after + gil->mutex is unlocked and before the following code runs, leading to a + crash. We can use final_release to indicate the thread is done with the + GIL, and that's the only time we might delete the interpreter. See + https://github.com/python/cpython/issues/104341. */ + if (!final_release && + _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(tstate->interp); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take the GIL and drop it again, and reset the condition @@ -283,7 +285,9 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) The function saves errno at entry and restores its value at exit. - tstate must be non-NULL. */ + tstate must be non-NULL. + + Returns 1 if the GIL was acquired, or 0 if not. */ static void take_gil(PyThreadState *tstate) { @@ -307,18 +311,19 @@ take_gil(PyThreadState *tstate) assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + return; + } +#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); MUTEX_LOCK(gil->mutex); - if (!_Py_atomic_load_relaxed(&gil->locked)) { - goto _ready; - } - int drop_requested = 0; - while (_Py_atomic_load_relaxed(&gil->locked)) { + while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; unsigned long interval = (gil->interval >= 1 ? gil->interval : 1); @@ -328,9 +333,11 @@ take_gil(PyThreadState *tstate) /* If we timed out and no switch occurred in the meantime, it is time to ask the GIL-holding thread to drop it. */ if (timed_out && - _Py_atomic_load_relaxed(&gil->locked) && + _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { + PyThreadState *holder_tstate = + (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); if (_PyThreadState_MustExit(tstate)) { MUTEX_UNLOCK(gil->mutex); // gh-96387: If the loop requested a drop request in a previous @@ -340,25 +347,35 @@ take_gil(PyThreadState *tstate) // may have to request again a drop request (iterate one more // time). if (drop_requested) { - RESET_GIL_DROP_REQUEST(interp); + _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - SET_GIL_DROP_REQUEST(interp); + _Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); drop_requested = 1; } } -_ready: +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Another thread disabled the GIL between our check above and + // now. Don't take the GIL, signal any other waiting threads, and + // return. + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); + return; + } +#endif + #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ MUTEX_LOCK(gil->switch_mutex); #endif /* We now hold the GIL */ - _Py_atomic_store_relaxed(&gil->locked, 1); + _Py_atomic_store_int_relaxed(&gil->locked, 1); _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1); if (tstate != (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) { @@ -380,17 +397,21 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(interp, tstate); + /* tstate could be a dangling pointer, so don't pass it to + drop_gil(). */ + drop_gil(interp, NULL, 1); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(interp); - update_eval_breaker_from_thread(interp, tstate); + tstate->_status.holds_gil = 1; + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); errno = err; + return; } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -431,14 +452,23 @@ PyEval_ThreadsInitialized(void) return _PyEval_ThreadsInitialized(); } +#ifndef NDEBUG static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { - if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) != tstate) { - return 0; - } - return _Py_atomic_load_relaxed(&gil->locked); + int holds_gil = tstate->_status.holds_gil; + + // holds_gil is the source of truth; check that last_holder and gil->locked + // are consistent with it. + int locked = _Py_atomic_load_int_relaxed(&gil->locked); + int is_last_holder = + ((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate; + assert(!holds_gil || locked); + assert(!holds_gil || is_last_holder); + + return holds_gil; } +#endif static void init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) @@ -452,13 +482,17 @@ static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); +#ifdef Py_GIL_DISABLED + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gil->enabled = config->enable_gil == _PyConfig_GIL_ENABLE ? INT_MAX : 0; +#endif create_gil(gil); assert(gil_created(gil)); interp->ceval.gil = gil; interp->ceval.own_gil = 1; } -PyStatus +void _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); @@ -477,8 +511,6 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) // Lock the GIL and mark the current thread as attached. _PyThreadState_Attach(tstate); - - return _PyStatus_OK(); } void @@ -511,8 +543,7 @@ _PyEval_FiniGIL(PyInterpreterState *interp) interp->ceval.gil = NULL; } -// Function removed in the Python 3.13 API but kept in the stable ABI. -PyAPI_FUNC(void) +void PyEval_InitThreads(void) { /* Do nothing: kept for backward compatibility */ @@ -544,7 +575,7 @@ PyEval_ReleaseLock(void) /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - drop_gil(tstate->interp, tstate); + drop_gil(tstate->interp, tstate, 0); } void @@ -555,12 +586,13 @@ _PyEval_AcquireLock(PyThreadState *tstate) } void -_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate) +_PyEval_ReleaseLock(PyInterpreterState *interp, + PyThreadState *tstate, + int final_release) { - /* If tstate is NULL then we do not expect the current thread - to acquire the GIL ever again. */ - assert(tstate == NULL || tstate->interp == interp); - drop_gil(interp, tstate); + assert(tstate != NULL); + assert(tstate->interp == interp); + drop_gil(interp, tstate, final_release); } void @@ -578,9 +610,8 @@ PyEval_ReleaseThread(PyThreadState *tstate) } #ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child to destroy all threads - which are not running in the child process, and clear internal locks - which might be held by those threads. */ +/* This function is called from PyOS_AfterFork_Child to re-initialize the + GIL and pending calls lock. */ PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { @@ -595,25 +626,12 @@ _PyEval_ReInitThreads(PyThreadState *tstate) take_gil(tstate); struct _pending_calls *pending = &tstate->interp->ceval.pending; - if (_PyThread_at_fork_reinit(&pending->lock) < 0) { - return _PyStatus_ERR("Can't reinitialize pending calls lock"); - } + _PyMutex_at_fork_reinit(&pending->mutex); - /* Destroy all threads except the current one */ - _PyThreadState_DeleteExcept(tstate); return _PyStatus_OK(); } #endif -/* This function is used to signal that async exceptions are waiting to be - raised. */ - -void -_PyEval_SignalAsyncExc(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); -} - PyThreadState * PyEval_SaveThread(void) { @@ -625,11 +643,47 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + _Py_EnsureTstateNotNULL(tstate); _PyThreadState_Attach(tstate); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif } +void +_PyEval_SignalReceived(void) +{ + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT); +} + + +#ifndef Py_GIL_DISABLED +static void +signal_active_thread(PyInterpreterState *interp, uintptr_t bit) +{ + struct _gil_runtime_state *gil = interp->ceval.gil; + + // If a thread from the targeted interpreter is holding the GIL, signal + // that thread. Otherwise, the next thread to run from the targeted + // interpreter will have its bit set as part of taking the GIL. + MUTEX_LOCK(gil->mutex); + if (_Py_atomic_load_int_relaxed(&gil->locked)) { + PyThreadState *holder = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); + if (holder->interp == interp) { + _Py_set_eval_breaker_bit(holder, bit); + } + } + MUTEX_UNLOCK(gil->mutex); +} +#endif + + /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX signal handlers or Mac I/O completion routines) can schedule calls to a function to be called synchronously. @@ -652,31 +706,31 @@ PyEval_RestoreThread(PyThreadState *tstate) threadstate. */ -void -_PyEval_SignalReceived(PyInterpreterState *interp) -{ - if (_Py_ThreadCanHandleSignals(interp)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } -} - /* Push one item onto the queue while holding the lock. */ static int _push_pending_call(struct _pending_calls *pending, _Py_pending_call_func func, void *arg, int flags) { - int i = pending->last; - int j = (i + 1) % NPENDINGCALLS; - if (j == pending->first) { - return -1; /* Queue full */ + if (pending->npending == pending->max) { + return _Py_ADD_PENDING_FULL; } + assert(pending->npending < pending->max); + + int i = pending->next; + assert(pending->calls[i].func == NULL); + pending->calls[i].func = func; pending->calls[i].arg = arg; pending->calls[i].flags = flags; - pending->last = j; - assert(pending->calls_to_do < NPENDINGCALLS); - pending->calls_to_do++; - return 0; + + assert(pending->npending < PENDINGCALLSARRAYSIZE); + _Py_atomic_add_int32(&pending->npending, 1); + + pending->next = (i + 1) % PENDINGCALLSARRAYSIZE; + assert(pending->next != pending->first + || pending->npending == pending->max); + + return _Py_ADD_PENDING_SUCCESS; } static int @@ -684,8 +738,9 @@ _next_pending_call(struct _pending_calls *pending, int (**func)(void *), void **arg, int *flags) { int i = pending->first; - if (i == pending->last) { + if (pending->npending == 0) { /* Queue empty */ + assert(i == pending->next); assert(pending->calls[i].func == NULL); return -1; } @@ -703,9 +758,9 @@ _pop_pending_call(struct _pending_calls *pending, int i = _next_pending_call(pending, func, arg, flags); if (i >= 0) { pending->calls[i] = (struct _pending_call){0}; - pending->first = (i + 1) % NPENDINGCALLS; - assert(pending->calls_to_do > 0); - pending->calls_to_do--; + pending->first = (i + 1) % PENDINGCALLSARRAYSIZE; + assert(pending->npending > 0); + _Py_atomic_add_int32(&pending->npending, -1); } } @@ -714,28 +769,34 @@ _pop_pending_call(struct _pending_calls *pending, callback. */ -int +_Py_add_pending_call_result _PyEval_AddPendingCall(PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags) { - assert(!(flags & _Py_PENDING_MAINTHREADONLY) - || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; - if (flags & _Py_PENDING_MAINTHREADONLY) { + int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0; + if (main_only) { /* The main thread only exists in the main interpreter. */ assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; } - /* Ensure that _PyEval_InitState() was called - and that _PyEval_FiniState() is not called yet. */ - assert(pending->lock != NULL); - PyThread_acquire_lock(pending->lock, WAIT_LOCK); - int result = _push_pending_call(pending, func, arg, flags); - PyThread_release_lock(pending->lock); + PyMutex_Lock(&pending->mutex); + _Py_add_pending_call_result result = + _push_pending_call(pending, func, arg, flags); + PyMutex_Unlock(&pending->mutex); + + if (main_only) { + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT); + } + else { +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + signal_active_thread(interp, _PY_CALLS_TO_DO_BIT); +#endif + } - /* signal main loop */ - SIGNAL_PENDING_CALLS(interp); return result; } @@ -745,111 +806,192 @@ Py_AddPendingCall(_Py_pending_call_func func, void *arg) /* Legacy users of this API will continue to target the main thread (of the main interpreter). */ PyInterpreterState *interp = _PyInterpreterState_Main(); - return _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY); + _Py_add_pending_call_result r = + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY); + if (r == _Py_ADD_PENDING_FULL) { + return -1; + } + else { + assert(r == _Py_ADD_PENDING_SUCCESS); + return 0; + } } static int handle_signals(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0); + _Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } if (_PyErr_CheckSignalsTstate(tstate) < 0) { /* On failure, re-schedule a call to handle_signals(). */ - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1); + _Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); return -1; } return 0; } static int -_make_pending_calls(struct _pending_calls *pending) +_make_pending_calls(struct _pending_calls *pending, int32_t *p_npending) { + int res = 0; + int32_t npending = -1; + + assert(sizeof(pending->max) <= sizeof(size_t) + && ((size_t)pending->max) <= Py_ARRAY_LENGTH(pending->calls)); + int32_t maxloop = pending->maxloop; + if (maxloop == 0) { + maxloop = pending->max; + } + assert(maxloop > 0 && maxloop <= pending->max); + /* perform a bounded number of calls, in case of recursion */ - for (int i=0; ilock, WAIT_LOCK); + PyMutex_Lock(&pending->mutex); _pop_pending_call(pending, &func, &arg, &flags); - PyThread_release_lock(pending->lock); + npending = pending->npending; + PyMutex_Unlock(&pending->mutex); - /* having released the lock, perform the callback */ + /* Check if there are any more pending calls. */ if (func == NULL) { + assert(npending == 0); break; } - int res = func(arg); + + /* having released the lock, perform the callback */ + res = func(arg); if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) { PyMem_RawFree(arg); } if (res != 0) { - return -1; + res = -1; + goto finally; } } - return 0; + +finally: + *p_npending = npending; + return res; +} + +static void +signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + +static void +unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif } static int -make_pending_calls(PyInterpreterState *interp) +make_pending_calls(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; struct _pending_calls *pending = &interp->ceval.pending; struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread; /* Only one thread (per interpreter) may run the pending calls at once. In the same way, we don't do recursive pending calls. */ - PyThread_acquire_lock(pending->lock, WAIT_LOCK); - if (pending->busy) { + PyMutex_Lock(&pending->mutex); + if (pending->handling_thread != NULL) { /* A pending call was added after another thread was already handling the pending calls (and had already "unsignaled"). Once that thread is done, it may have taken care of all the pending calls, or there might be some still waiting. - Regardless, this interpreter's pending calls will stay - "signaled" until that first thread has finished. At that - point the next thread to trip the eval breaker will take - care of any remaining pending calls. Until then, though, - all the interpreter's threads will be tripping the eval - breaker every time it's checked. */ - PyThread_release_lock(pending->lock); + To avoid all threads constantly stopping on the eval breaker, + we clear the bit for this thread and make sure it is set + for the thread currently handling the pending call. */ + _Py_set_eval_breaker_bit(pending->handling_thread, _PY_CALLS_TO_DO_BIT); + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + PyMutex_Unlock(&pending->mutex); return 0; } - pending->busy = 1; - PyThread_release_lock(pending->lock); + pending->handling_thread = tstate; + PyMutex_Unlock(&pending->mutex); /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(interp); + unsignal_pending_calls(tstate, interp); - if (_make_pending_calls(pending) != 0) { - pending->busy = 0; + int32_t npending; + if (_make_pending_calls(pending, &npending) != 0) { + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } + if (npending > 0) { + /* We hit pending->maxloop. */ + signal_pending_calls(tstate, interp); + } if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { - if (_make_pending_calls(pending_main) != 0) { - pending->busy = 0; + if (_make_pending_calls(pending_main, &npending) != 0) { + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } + if (npending > 0) { + /* We hit pending_main->maxloop. */ + signal_pending_calls(tstate, interp); + } } - pending->busy = 0; + pending->handling_thread = NULL; return 0; } + +void +_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_set_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + +void +_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_unset_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); assert(_PyThreadState_CheckConsistency(tstate)); - if (make_pending_calls(tstate->interp) < 0) { + if (make_pending_calls(tstate) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyErr_BadInternalCall(); _PyErr_ChainExceptions1(exc); @@ -872,7 +1014,7 @@ _PyEval_MakePendingCalls(PyThreadState *tstate) } } - res = make_pending_calls(tstate->interp); + res = make_pending_calls(tstate); if (res != 0) { return res; } @@ -898,25 +1040,127 @@ Py_MakePendingCalls(void) } void -_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) +_PyEval_InitState(PyInterpreterState *interp) { _gil_initialize(&interp->_gil); +} - struct _pending_calls *pending = &interp->ceval.pending; - assert(pending->lock == NULL); - pending->lock = pending_lock; +#ifdef Py_GIL_DISABLED +int +_PyEval_EnableGILTransient(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + // The GIL is already enabled permanently. + return 0; + } + if (enabled == INT_MAX - 1) { + Py_FatalError("Too many transient requests to enable the GIL"); + } + if (enabled > 0) { + // If enabled is nonzero, we know we hold the GIL. This means that no + // other threads are attached, and nobody else can be concurrently + // mutating it. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + return 0; + } + + // Enabling the GIL changes what it means to be an "attached" thread. To + // safely make this transition, we: + // 1. Detach the current thread. + // 2. Stop the world to detach (and suspend) all other threads. + // 3. Enable the GIL, if nobody else did between our check above and when + // our stop-the-world begins. + // 4. Start the world. + // 5. Attach the current thread. Other threads may attach and hold the GIL + // before this thread, which is harmless. + _PyThreadState_Detach(tstate); + + // This could be an interpreter-local stop-the-world in situations where we + // know that this interpreter's GIL is not shared, and that it won't become + // shared before the stop-the-world begins. For now, we always stop all + // interpreters for simplicity. + _PyEval_StopTheWorldAll(&_PyRuntime); + + enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + int this_thread_enabled = enabled == 0; + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_Attach(tstate); + + return this_thread_enabled; } -void -_PyEval_FiniState(struct _ceval_state *ceval) +int +_PyEval_EnableGILPermanent(PyThreadState *tstate) { - struct _pending_calls *pending = &ceval->pending; - if (pending->lock != NULL) { - PyThread_free_lock(pending->lock); - pending->lock = NULL; + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; } + + _Py_atomic_store_int_relaxed(&gil->enabled, INT_MAX); + return 1; } +int +_PyEval_DisableGIL(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + assert(enabled >= 1); + enabled--; + + // Disabling the GIL is much simpler than enabling it, since we know we are + // the only attached thread. Other threads may start free-threading as soon + // as this store is complete, if it sets gil->enabled to 0. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled); + + if (enabled == 0) { + // We're attached, so we know the GIL will remain disabled until at + // least the next time we detach, which must be after this function + // returns. + // + // Drop the GIL, which will wake up any threads waiting in take_gil() + // and let them resume execution without the GIL. + drop_gil_impl(tstate, gil); + + // If another thread asked us to drop the GIL, they should be + // free-threading by now. Remove any such request so we have a clean + // slate if/when the GIL is enabled again. + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + return 1; + } + return 0; +} +#endif + /* Do periodic things, like check for signals and async I/0. * We need to do reasonably frequently, but not too frequently. @@ -979,30 +1223,47 @@ _PyEval_FiniState(struct _ceval_state *ceval) int _Py_HandlePending(PyThreadState *tstate) { - PyInterpreterState *interp = tstate->interp; + uintptr_t breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + + /* Stop-the-world */ + if ((breaker & _PY_EVAL_PLEASE_STOP_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PLEASE_STOP_BIT); + _PyThreadState_Suspend(tstate); + + /* The attach blocks until the stop-the-world event is complete. */ + _PyThreadState_Attach(tstate); + } /* Pending signals */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) { + if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) { if (handle_signals(tstate) != 0) { return -1; } } /* Pending calls */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) { - if (make_pending_calls(interp) != 0) { + if ((breaker & _PY_CALLS_TO_DO_BIT) != 0) { + if (make_pending_calls(tstate) != 0) { return -1; } } +#ifdef Py_GIL_DISABLED + /* Objects with refcounts to merge */ + if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); + _Py_brc_merge_refcounts(tstate); + } +#endif + /* GC scheduled to run */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); + if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } /* GIL drop request */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ _PyThreadState_Detach(tstate); @@ -1012,11 +1273,10 @@ _Py_HandlePending(PyThreadState *tstate) } /* Check for asynchronous exception. */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_ASYNC_EXCEPTION_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 0); - if (tstate->async_exc != NULL) { - PyObject *exc = tstate->async_exc; - tstate->async_exc = NULL; + if ((breaker & _PY_ASYNC_EXCEPTION_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); + PyObject *exc = _Py_atomic_exchange_ptr(&tstate->async_exc, NULL); + if (exc != NULL) { _PyErr_SetNone(tstate, exc); Py_DECREF(exc); return -1; @@ -1024,4 +1284,3 @@ _Py_HandlePending(PyThreadState *tstate) } return 0; } - diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bd28126b0b7dfb..595b72bfaf9613 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -1,4 +1,4 @@ -// Macros and other things needed by ceval.c, executor.c, and bytecodes.c +// Macros and other things needed by ceval.c, and bytecodes.c /* Computed GOTOs, or the-optimization-commonly-but-improperly-known-as-"threaded code" @@ -60,33 +60,50 @@ #endif #ifdef Py_STATS -#define INSTRUCTION_START(op) \ +#define INSTRUCTION_STATS(op) \ do { \ - frame->prev_instr = next_instr++; \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else -#define INSTRUCTION_START(op) (frame->prev_instr = next_instr++) +#define INSTRUCTION_STATS(op) ((void)0) #endif #if USE_COMPUTED_GOTOS -# define TARGET(op) TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] #else -# define TARGET(op) case op: TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE -#define PRE_DISPATCH_GOTO() if (lltrace) { \ - lltrace_instruction(frame, stack_pointer, next_instr); } +#define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } #else #define PRE_DISPATCH_GOTO() ((void)0) #endif +#if LLTRACE +#define LLTRACE_RESUME_FRAME() \ +do { \ + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); \ + if (lltrace < 0) { \ + goto exit_unwind; \ + } \ +} while (0) +#else +#define LLTRACE_RESUME_FRAME() ((void)0) +#endif + +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ @@ -107,18 +124,21 @@ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) +// Use this instead of 'goto error' so Tier 2 can go to a different label +#define GOTO_ERROR(LABEL) goto LABEL + #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ - goto error; \ + GOTO_ERROR(error); \ } \ } @@ -142,11 +162,10 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) #define NEXTOPARG() do { \ - _Py_CODEUNIT word = *next_instr; \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) -#define JUMPTO(x) (next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + (x)) /* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is * for advancing to the next instruction, taking into account cache entries @@ -227,6 +246,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define STACK_SHRINK(n) BASIC_STACKADJ(-(n)) #endif +#define WITHIN_STACK_BOUNDS() \ + (frame == &entry_frame || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) /* Data access macros */ #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) @@ -243,9 +264,9 @@ GETITEM(PyObject *v, Py_ssize_t i) { This is because it is possible that during the DECREF the frame is accessed by other code (e.g. a __del__ method or gc.collect()) and the variable would be pointing to already-freed memory. */ -#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \ +#define SETLOCAL(i, value) do { _PyStackRef tmp = GETLOCAL(i); \ GETLOCAL(i) = value; \ - Py_XDECREF(tmp); } while (0) + PyStackRef_XCLOSE(tmp); } while (0) #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) @@ -255,13 +276,9 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ - else { \ - /* This is about to be (incorrectly) incremented: */ \ - STAT_DEC((INSTNAME), deferred); \ - } \ } while (0) #else #define UPDATE_MISS_STATS(INSTNAME) ((void)0) @@ -287,21 +304,28 @@ GETITEM(PyObject *v, Py_ssize_t i) { dtrace_function_entry(frame); \ } -#define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ - (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) - -#define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ - (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) +/* This takes a uint16_t instead of a _Py_BackoffCounter, + * because it is used directly on the cache entry in generated code, + * which is always an integral type. */ +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) -#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ - (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ +#ifdef Py_GIL_DISABLED +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + /* gh-115999 tracks progress on addressing this. */ \ + static_assert(0, "The specializing interpreter is not yet thread-safe"); \ + } while (0); +#else +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = advance_backoff_counter((COUNTER)); \ } while (0); +#endif -#define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = pause_backoff_counter((COUNTER)); \ } while (0); #define UNBOUNDLOCAL_ERROR_MSG \ @@ -325,7 +349,7 @@ do { \ }\ else { \ result = PyFloat_FromDouble(dval); \ - if ((result) == NULL) goto error; \ + if ((result) == NULL) GOTO_ERROR(error); \ _Py_DECREF_NO_DEALLOC(left); \ _Py_DECREF_NO_DEALLOC(right); \ } \ @@ -336,22 +360,19 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ + if (tstate->tracing) {\ + next_instr = dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ } \ } while (0); -typedef PyObject *(*convertion_func_ptr)(PyObject *); - -static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { - [FVC_STR] = PyObject_Str, - [FVC_REPR] = PyObject_Repr, - [FVC_ASCII] = PyObject_ASCII -}; // GH-89279: Force inlining by using a macro. #if defined(_MSC_VER) && SIZEOF_INT == 4 @@ -369,43 +390,92 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } -/* Marker to specify tier 1 only instructions */ -#define TIER_ONE_ONLY - -/* Marker to specify tier 2 only instructions */ -#define TIER_TWO_ONLY - /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. - * These need to treated differently by tier 1 and 2. */ + * These need to treated differently by tier 1 and 2. + * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ -#if TIER_ONE - -#define LOAD_IP() \ -do { next_instr = frame->prev_instr+1; } while (0) +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) -#define STORE_SP() \ -_PyFrame_SetStackPointer(frame, stack_pointer) +/* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ stack_pointer = _PyFrame_GetStackPointer(frame); +/* Tier-switching macros. */ + +#ifdef _Py_JIT +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + jit_func jitted = (EXECUTOR)->jit_code; \ + next_instr = jitted(frame, stack_pointer, tstate); \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + frame = tstate->current_frame; \ + if (next_instr == NULL) { \ + goto resume_with_error; \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + DISPATCH(); \ +} while (0) +#else +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + next_uop = (EXECUTOR)->trace; \ + assert(next_uop->opcode == _START_EXECUTOR); \ + goto enter_tier_two; \ +} while (0) #endif +#define GOTO_TIER_ONE(TARGET) \ +do { \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + next_instr = target; \ + DISPATCH(); \ +} while (0) -#if TIER_TWO - -#define LOAD_IP() \ -do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0) +#define CURRENT_OPARG() (next_uop[-1].oparg) -#define STORE_SP() \ -_PyFrame_SetStackPointer(frame, stack_pointer) +#define CURRENT_OPERAND() (next_uop[-1].operand) -#define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target +#define GOTO_UNWIND() goto error_tier_two +#define EXIT_TO_TIER1() goto exit_to_tier1 +#define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic; -#endif +/* Stackref macros */ +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 +#ifdef Py_GIL_DISABLED +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); +#else +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + PyObject **NAME = (PyObject **)ARGS; \ + assert(NAME != NULL); +#endif +#ifdef Py_GIL_DISABLED +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + /* +1 because we +1 previously */ \ + _PyObjectArray_Free(NAME - 1, NAME##_temp); +#else +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + (void)(NAME); +#endif +#ifdef Py_GIL_DISABLED +#define CONVERSION_FAILED(NAME) ((NAME) == NULL) +#else +#define CONVERSION_FAILED(NAME) (0) +#endif diff --git a/Python/clinic/Python-tokenize.c.h b/Python/clinic/Python-tokenize.c.h index a8453e25807b0d..730fa8ef2a2154 100644 --- a/Python/clinic/Python-tokenize.c.h +++ b/Python/clinic/Python-tokenize.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, @@ -79,4 +80,4 @@ tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=406b5a433a59069c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dcd6ec48f06a092e input=a9049054013a1b77]*/ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index cf3c73b1cf0e6e..98dc49db3059b1 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(warnings_warn__doc__, "warn($module, /, message, category=None, stacklevel=1, source=None, *,\n" @@ -243,4 +244,4 @@ warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) { return warnings_filters_mutated_impl(module); } -/*[clinic end generated code: output=8c396a721ef75739 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f2d4214c382717a6 input=a9049054013a1b77]*/ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index b4dcc6bf1d2978..f75a8d4ac0ccd4 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(builtin___import____doc__, "__import__($module, /, name, globals=None, locals=None, fromlist=(),\n" @@ -232,25 +233,6 @@ PyDoc_STRVAR(builtin_chr__doc__, #define BUILTIN_CHR_METHODDEF \ {"chr", (PyCFunction)builtin_chr, METH_O, builtin_chr__doc__}, -static PyObject * -builtin_chr_impl(PyObject *module, int i); - -static PyObject * -builtin_chr(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - int i; - - i = PyLong_AsInt(arg); - if (i == -1 && PyErr_Occurred()) { - goto exit; - } - return_value = builtin_chr_impl(module, i); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" " dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" @@ -413,7 +395,7 @@ builtin_divmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_eval__doc__, -"eval($module, source, globals=None, locals=None, /)\n" +"eval($module, source, /, globals=None, locals=None)\n" "--\n" "\n" "Evaluate the given source in the context of globals and locals.\n" @@ -425,33 +407,63 @@ PyDoc_STRVAR(builtin_eval__doc__, "If only globals is given, locals defaults to it."); #define BUILTIN_EVAL_METHODDEF \ - {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL, builtin_eval__doc__}, + {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL|METH_KEYWORDS, builtin_eval__doc__}, static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals); static PyObject * -builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "globals", "locals", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "eval", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *source; PyObject *globals = Py_None; PyObject *locals = Py_None; - if (!_PyArg_CheckPositional("eval", nargs, 1, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional; + if (!noptargs) { + goto skip_optional_pos; } - globals = args[1]; - if (nargs < 3) { - goto skip_optional; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } locals = args[2]; -skip_optional: +skip_optional_pos: return_value = builtin_eval_impl(module, source, globals, locals); exit: @@ -459,7 +471,7 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_exec__doc__, -"exec($module, source, globals=None, locals=None, /, *, closure=None)\n" +"exec($module, source, /, globals=None, locals=None, *, closure=None)\n" "--\n" "\n" "Execute the given source in the context of globals and locals.\n" @@ -485,14 +497,14 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(closure), }, + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), &_Py_ID(closure), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -501,7 +513,7 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "", "", "closure", NULL}; + static const char * const _keywords[] = {"", "globals", "locals", "closure", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "exec", @@ -520,17 +532,22 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - globals = args[1]; - if (nargs < 3) { - goto skip_optional_posonly; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } - noptargs--; - locals = args[2]; -skip_optional_posonly: + if (args[2]) { + locals = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -711,10 +728,10 @@ PyDoc_STRVAR(builtin_anext__doc__, "anext($module, aiterator, default=, /)\n" "--\n" "\n" -"async anext(aiterator[, default])\n" +"Return the next item from the async iterator.\n" "\n" -"Return the next item from the async iterator. If default is given and the async\n" -"iterator is exhausted, it is returned instead of raising StopAsyncIteration."); +"If default is given and the async iterator is exhausted,\n" +"it is returned instead of raising StopAsyncIteration."); #define BUILTIN_ANEXT_METHODDEF \ {"anext", _PyCFunction_CAST(builtin_anext), METH_FASTCALL, builtin_anext__doc__}, @@ -1211,4 +1228,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=607c62f2341ebfc0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=435d3f286a863c49 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 292d3f7f4ff49c..997ac6f63384a9 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_contextvars_Context_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -177,4 +179,4 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, -/*[clinic end generated code: output=2436b16a92452869 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b667826178444c3f input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 890c3026fc9996..5edeaef656ef62 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_imp_lock_held__doc__, "lock_held($module, /)\n" @@ -604,10 +605,6 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[1], &source, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&source, 'C')) { - _PyArg_BadArgument("source_hash", "argument 'source'", "contiguous buffer", args[1]); - goto exit; - } return_value = _imp_source_hash_impl(module, key, &source); exit: @@ -626,4 +623,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=058f6aa1c9f4ebe4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dbd63707bd40b07c input=a9049054013a1b77]*/ diff --git a/Python/clinic/instruction_sequence.c.h b/Python/clinic/instruction_sequence.c.h new file mode 100644 index 00000000000000..66c2ccadfa03c9 --- /dev/null +++ b/Python/clinic/instruction_sequence.c.h @@ -0,0 +1,304 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +PyDoc_STRVAR(inst_seq_new__doc__, +"InstructionSequenceType()\n" +"--\n" +"\n" +"Create a new InstructionSequence object."); + +static PyObject * +inst_seq_new_impl(PyTypeObject *type); + +static PyObject * +inst_seq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &_PyInstructionSequence_Type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("InstructionSequenceType", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("InstructionSequenceType", kwargs)) { + goto exit; + } + return_value = inst_seq_new_impl(type); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_use_label__doc__, +"use_label($self, /, label)\n" +"--\n" +"\n" +"Place label at current location."); + +#define INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF \ + {"use_label", _PyCFunction_CAST(InstructionSequenceType_use_label), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_use_label__doc__}, + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label); + +static PyObject * +InstructionSequenceType_use_label(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(label), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"label", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "use_label", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int label; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + label = PyLong_AsInt(args[0]); + if (label == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_use_label_impl(self, label); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_addop__doc__, +"addop($self, /, opcode, oparg, lineno, col_offset, end_lineno,\n" +" end_col_offset)\n" +"--\n" +"\n" +"Append an instruction."); + +#define INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF \ + {"addop", _PyCFunction_CAST(InstructionSequenceType_addop), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_addop__doc__}, + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset); + +static PyObject * +InstructionSequenceType_addop(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), &_Py_ID(oparg), &_Py_ID(lineno), &_Py_ID(col_offset), &_Py_ID(end_lineno), &_Py_ID(end_col_offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", "oparg", "lineno", "col_offset", "end_lineno", "end_col_offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "addop", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + int opcode; + int oparg; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + oparg = PyLong_AsInt(args[1]); + if (oparg == -1 && PyErr_Occurred()) { + goto exit; + } + lineno = PyLong_AsInt(args[2]); + if (lineno == -1 && PyErr_Occurred()) { + goto exit; + } + col_offset = PyLong_AsInt(args[3]); + if (col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + end_lineno = PyLong_AsInt(args[4]); + if (end_lineno == -1 && PyErr_Occurred()) { + goto exit; + } + end_col_offset = PyLong_AsInt(args[5]); + if (end_col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_addop_impl(self, opcode, oparg, lineno, col_offset, end_lineno, end_col_offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_new_label__doc__, +"new_label($self, /)\n" +"--\n" +"\n" +"Return a new label."); + +#define INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF \ + {"new_label", (PyCFunction)InstructionSequenceType_new_label, METH_NOARGS, InstructionSequenceType_new_label__doc__}, + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_new_label(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = InstructionSequenceType_new_label_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_add_nested__doc__, +"add_nested($self, /, nested)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF \ + {"add_nested", _PyCFunction_CAST(InstructionSequenceType_add_nested), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_add_nested__doc__}, + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested); + +static PyObject * +InstructionSequenceType_add_nested(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(nested), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"nested", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "add_nested", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *nested; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + nested = args[0]; + return_value = InstructionSequenceType_add_nested_impl(self, nested); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_get_nested__doc__, +"get_nested($self, /)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF \ + {"get_nested", (PyCFunction)InstructionSequenceType_get_nested, METH_NOARGS, InstructionSequenceType_get_nested__doc__}, + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_nested(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_nested_impl(self); +} + +PyDoc_STRVAR(InstructionSequenceType_get_instructions__doc__, +"get_instructions($self, /)\n" +"--\n" +"\n" +"Return the instructions as a list of tuples or labels."); + +#define INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF \ + {"get_instructions", (PyCFunction)InstructionSequenceType_get_instructions, METH_NOARGS, InstructionSequenceType_get_instructions__doc__}, + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_instructions(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_instructions_impl(self); +} +/*[clinic end generated code: output=8809d7aa11d9b2bb input=a9049054013a1b77]*/ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 1b1b6d048b0dd0..8dae747c44a543 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(monitoring_use_tool_id__doc__, "use_tool_id($module, tool_id, name, /)\n" "--\n" @@ -302,4 +304,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=46f449b18195f976 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=14ffc0884a6de50a input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index 16e48f77c87fdb..c19a3ed5050ed3 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -2,8 +2,14 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + PyDoc_STRVAR(marshal_dump__doc__, -"dump($module, value, file, version=version, /)\n" +"dump($module, value, file, version=version, /, *, allow_code=True)\n" "--\n" "\n" "Write the value on the open file.\n" @@ -14,53 +20,95 @@ PyDoc_STRVAR(marshal_dump__doc__, " Must be a writeable binary file.\n" " version\n" " Indicates the data format that dump should use.\n" +" allow_code\n" +" Allow to write code objects.\n" "\n" "If the value has (or contains an object that has) an unsupported type, a\n" "ValueError exception is raised - but garbage data will also be written\n" "to the file. The object will not be properly read back by load()."); #define MARSHAL_DUMP_METHODDEF \ - {"dump", _PyCFunction_CAST(marshal_dump), METH_FASTCALL, marshal_dump__doc__}, + {"dump", _PyCFunction_CAST(marshal_dump), METH_FASTCALL|METH_KEYWORDS, marshal_dump__doc__}, static PyObject * marshal_dump_impl(PyObject *module, PyObject *value, PyObject *file, - int version); + int version, int allow_code); static PyObject * -marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +marshal_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; PyObject *value; PyObject *file; int version = Py_MARSHAL_VERSION; + int allow_code = 1; - if (!_PyArg_CheckPositional("dump", nargs, 2, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { goto exit; } value = args[0]; file = args[1]; if (nargs < 3) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; version = PyLong_AsInt(args[2]); if (version == -1 && PyErr_Occurred()) { goto exit; } -skip_optional: - return_value = marshal_dump_impl(module, value, file, version); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[3]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_dump_impl(module, value, file, version, allow_code); exit: return return_value; } PyDoc_STRVAR(marshal_load__doc__, -"load($module, file, /)\n" +"load($module, file, /, *, allow_code=True)\n" "--\n" "\n" "Read one value from the open file and return it.\n" "\n" " file\n" " Must be readable binary file.\n" +" allow_code\n" +" Allow to load code objects.\n" "\n" "If no valid value is read (e.g. because the data has a different Python\n" "version\'s incompatible marshal format), raise EOFError, ValueError or\n" @@ -70,10 +118,66 @@ PyDoc_STRVAR(marshal_load__doc__, "dump(), load() will substitute None for the unmarshallable type."); #define MARSHAL_LOAD_METHODDEF \ - {"load", (PyCFunction)marshal_load, METH_O, marshal_load__doc__}, + {"load", _PyCFunction_CAST(marshal_load), METH_FASTCALL|METH_KEYWORDS, marshal_load__doc__}, + +static PyObject * +marshal_load_impl(PyObject *module, PyObject *file, int allow_code); + +static PyObject * +marshal_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "load", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *file; + int allow_code = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + file = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[1]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_load_impl(module, file, allow_code); + +exit: + return return_value; +} PyDoc_STRVAR(marshal_dumps__doc__, -"dumps($module, value, version=version, /)\n" +"dumps($module, value, version=version, /, *, allow_code=True)\n" "--\n" "\n" "Return the bytes object that would be written to a file by dump(value, file).\n" @@ -82,70 +186,150 @@ PyDoc_STRVAR(marshal_dumps__doc__, " Must be a supported type.\n" " version\n" " Indicates the data format that dumps should use.\n" +" allow_code\n" +" Allow to write code objects.\n" "\n" "Raise a ValueError exception if value has (or contains an object that has) an\n" "unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ - {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL, marshal_dumps__doc__}, + {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, static PyObject * -marshal_dumps_impl(PyObject *module, PyObject *value, int version); +marshal_dumps_impl(PyObject *module, PyObject *value, int version, + int allow_code); static PyObject * -marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +marshal_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dumps", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *value; int version = Py_MARSHAL_VERSION; + int allow_code = 1; - if (!_PyArg_CheckPositional("dumps", nargs, 1, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { goto exit; } value = args[0]; if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; version = PyLong_AsInt(args[1]); if (version == -1 && PyErr_Occurred()) { goto exit; } -skip_optional: - return_value = marshal_dumps_impl(module, value, version); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[2]); + if (allow_code < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = marshal_dumps_impl(module, value, version, allow_code); exit: return return_value; } PyDoc_STRVAR(marshal_loads__doc__, -"loads($module, bytes, /)\n" +"loads($module, bytes, /, *, allow_code=True)\n" "--\n" "\n" "Convert the bytes-like object to a value.\n" "\n" +" allow_code\n" +" Allow to load code objects.\n" +"\n" "If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" "bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ - {"loads", (PyCFunction)marshal_loads, METH_O, marshal_loads__doc__}, + {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, static PyObject * -marshal_loads_impl(PyObject *module, Py_buffer *bytes); +marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code); static PyObject * -marshal_loads(PyObject *module, PyObject *arg) +marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(allow_code), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "allow_code", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "loads", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer bytes = {NULL, NULL}; + int allow_code = 1; - if (PyObject_GetBuffer(arg, &bytes, PyBUF_SIMPLE) != 0) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { goto exit; } - if (!PyBuffer_IsContiguous(&bytes, 'C')) { - _PyArg_BadArgument("loads", "argument", "contiguous buffer", arg); + if (PyObject_GetBuffer(args[0], &bytes, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + allow_code = PyObject_IsTrue(args[1]); + if (allow_code < 0) { goto exit; } - return_value = marshal_loads_impl(module, &bytes); +skip_optional_kwonly: + return_value = marshal_loads_impl(module, &bytes, allow_code); exit: /* Cleanup for bytes */ @@ -155,4 +339,4 @@ marshal_loads(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=23091e077319f596 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1575b9a3ae48ad3d input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 06105e221c1f89..8277d286cf51ef 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" @@ -288,8 +289,54 @@ sys_intern(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(sys__is_interned__doc__, +"_is_interned($module, string, /)\n" +"--\n" +"\n" +"Return True if the given string is \"interned\"."); + +#define SYS__IS_INTERNED_METHODDEF \ + {"_is_interned", (PyCFunction)sys__is_interned, METH_O, sys__is_interned__doc__}, + +static int +sys__is_interned_impl(PyObject *module, PyObject *string); + +static PyObject * +sys__is_interned(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_is_interned", "argument", "str", arg); + goto exit; + } + string = arg; + _return_value = sys__is_interned_impl(module, string); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(sys_settrace__doc__, +"settrace($module, function, /)\n" +"--\n" +"\n" +"Set the global debug tracing function.\n" +"\n" +"It will be called on each function call. See the debugger chapter\n" +"in the library manual."); + +#define SYS_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)sys_settrace, METH_O, sys_settrace__doc__}, + PyDoc_STRVAR(sys__settraceallthreads__doc__, -"_settraceallthreads($module, arg, /)\n" +"_settraceallthreads($module, function, /)\n" "--\n" "\n" "Set the global debug tracing function in all running threads belonging to the current interpreter.\n" @@ -320,14 +367,26 @@ sys_gettrace(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys_gettrace_impl(module); } +PyDoc_STRVAR(sys_setprofile__doc__, +"setprofile($module, function, /)\n" +"--\n" +"\n" +"Set the profiling function.\n" +"\n" +"It will be called on each function call and return. See the profiler\n" +"chapter in the library manual."); + +#define SYS_SETPROFILE_METHODDEF \ + {"setprofile", (PyCFunction)sys_setprofile, METH_O, sys_setprofile__doc__}, + PyDoc_STRVAR(sys__setprofileallthreads__doc__, -"_setprofileallthreads($module, arg, /)\n" +"_setprofileallthreads($module, function, /)\n" "--\n" "\n" "Set the profiling function in all running threads belonging to the current interpreter.\n" "\n" -"It will be called on each function call and return. See the profiler chapter\n" -"in the library manual."); +"It will be called on each function call and return. See the profiler\n" +"chapter in the library manual."); #define SYS__SETPROFILEALLTHREADS_METHODDEF \ {"_setprofileallthreads", (PyCFunction)sys__setprofileallthreads, METH_O, sys__setprofileallthreads__doc__}, @@ -909,24 +968,64 @@ sys_getallocatedblocks(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(sys_getunicodeinternedsize__doc__, -"getunicodeinternedsize($module, /)\n" +"getunicodeinternedsize($module, /, *, _only_immortal=False)\n" "--\n" "\n" "Return the number of elements of the unicode interned dictionary"); #define SYS_GETUNICODEINTERNEDSIZE_METHODDEF \ - {"getunicodeinternedsize", (PyCFunction)sys_getunicodeinternedsize, METH_NOARGS, sys_getunicodeinternedsize__doc__}, + {"getunicodeinternedsize", _PyCFunction_CAST(sys_getunicodeinternedsize), METH_FASTCALL|METH_KEYWORDS, sys_getunicodeinternedsize__doc__}, static Py_ssize_t -sys_getunicodeinternedsize_impl(PyObject *module); +sys_getunicodeinternedsize_impl(PyObject *module, int _only_immortal); static PyObject * -sys_getunicodeinternedsize(PyObject *module, PyObject *Py_UNUSED(ignored)) +sys_getunicodeinternedsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(_only_immortal), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"_only_immortal", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "getunicodeinternedsize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int _only_immortal = 0; Py_ssize_t _return_value; - _return_value = sys_getunicodeinternedsize_impl(module); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + _only_immortal = PyObject_IsTrue(args[0]); + if (_only_immortal < 0) { + goto exit; + } +skip_optional_kwonly: + _return_value = sys_getunicodeinternedsize_impl(module, _only_immortal); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -1096,6 +1195,24 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__clear_type_cache_impl(module); } +PyDoc_STRVAR(sys__clear_internal_caches__doc__, +"_clear_internal_caches($module, /)\n" +"--\n" +"\n" +"Clear all internal performance-related caches."); + +#define SYS__CLEAR_INTERNAL_CACHES_METHODDEF \ + {"_clear_internal_caches", (PyCFunction)sys__clear_internal_caches, METH_NOARGS, sys__clear_internal_caches__doc__}, + +static PyObject * +sys__clear_internal_caches_impl(PyObject *module); + +static PyObject * +sys__clear_internal_caches(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__clear_internal_caches_impl(module); +} + PyDoc_STRVAR(sys_is_finalizing__doc__, "is_finalizing($module, /)\n" "--\n" @@ -1408,6 +1525,52 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(sys__baserepl__doc__, +"_baserepl($module, /)\n" +"--\n" +"\n" +"Private function for getting the base REPL"); + +#define SYS__BASEREPL_METHODDEF \ + {"_baserepl", (PyCFunction)sys__baserepl, METH_NOARGS, sys__baserepl__doc__}, + +static PyObject * +sys__baserepl_impl(PyObject *module); + +static PyObject * +sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__baserepl_impl(module); +} + +PyDoc_STRVAR(sys__is_gil_enabled__doc__, +"_is_gil_enabled($module, /)\n" +"--\n" +"\n" +"Return True if the GIL is currently enabled and False otherwise."); + +#define SYS__IS_GIL_ENABLED_METHODDEF \ + {"_is_gil_enabled", (PyCFunction)sys__is_gil_enabled, METH_NOARGS, sys__is_gil_enabled__doc__}, + +static int +sys__is_gil_enabled_impl(PyObject *module); + +static PyObject * +sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__is_gil_enabled_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + #ifndef SYS_GETWINDOWSVERSION_METHODDEF #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ @@ -1451,4 +1614,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=3a7d3fbbcb281c22 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9cc9069aef1482bc input=a9049054013a1b77]*/ diff --git a/Python/clinic/traceback.c.h b/Python/clinic/traceback.c.h index c5687e30748782..fe53a2786d1ad6 100644 --- a/Python/clinic/traceback.c.h +++ b/Python/clinic/traceback.c.h @@ -6,9 +6,10 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(tb_new__doc__, -"TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno)\n" +"traceback(tb_next, tb_frame, tb_lasti, tb_lineno)\n" "--\n" "\n" "Create a new traceback object."); @@ -42,7 +43,7 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) static const char * const _keywords[] = {"tb_next", "tb_frame", "tb_lasti", "tb_lineno", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "TracebackType", + .fname = "traceback", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -60,7 +61,7 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } tb_next = fastargs[0]; if (!PyObject_TypeCheck(fastargs[1], &PyFrame_Type)) { - _PyArg_BadArgument("TracebackType", "argument 'tb_frame'", (&PyFrame_Type)->tp_name, fastargs[1]); + _PyArg_BadArgument("traceback", "argument 'tb_frame'", (&PyFrame_Type)->tp_name, fastargs[1]); goto exit; } tb_frame = (PyFrameObject *)fastargs[1]; @@ -77,4 +78,4 @@ tb_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=57e27eb68d04d842 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=916a759875507c5a input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index b79bf555f2f22a..9c0a3fad314cb5 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -11,6 +11,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.codec_search_path +#include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -19,24 +20,10 @@ const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ -/* Import the standard encodings package which will register the first - codec search function. - - This is done in a lazy way so that the Unicode implementation does - not downgrade startup time of scripts not needing it. - - ImportErrors are silently ignored by this function. Only one try is - made. - -*/ - -static int _PyCodecRegistry_Init(void); /* Forward */ - int PyCodec_Register(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - goto onError; + assert(interp->codecs.initialized); if (search_function == NULL) { PyErr_BadArgument(); goto onError; @@ -45,7 +32,14 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } - return PyList_Append(interp->codec_search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + int ret = PyList_Append(interp->codecs.search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + return ret; onError: return -1; @@ -55,22 +49,34 @@ int PyCodec_Unregister(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *codec_search_path = interp->codec_search_path; - /* Do nothing if codec_search_path is not created yet or was cleared. */ - if (codec_search_path == NULL) { + if (interp->codecs.initialized != 1) { + /* Do nothing if codecs state was cleared (only possible during + interpreter shutdown). */ return 0; } + PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); - Py_ssize_t n = PyList_GET_SIZE(codec_search_path); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *item = PyList_GET_ITEM(codec_search_path, i); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + PyObject *item = PyList_GetItemRef(codec_search_path, i); + int ret = 1; if (item == search_function) { - if (interp->codec_search_cache != NULL) { - assert(PyDict_CheckExact(interp->codec_search_cache)); - PyDict_Clear(interp->codec_search_cache); - } - return PyList_SetSlice(codec_search_path, i, i+1, NULL); + // We hold a reference to the item, so its destructor can't run + // while we hold search_path_mutex. + ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + Py_DECREF(item); + if (ret != 1) { + assert(interp->codecs.search_cache != NULL); + assert(PyDict_CheckExact(interp->codecs.search_cache)); + PyDict_Clear(interp->codecs.search_cache); + return ret; } } return 0; @@ -132,9 +138,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) { - return NULL; - } + assert(interp->codecs.initialized); /* Convert the encoding to a normalized Python string: all characters are converted to lower case, spaces and hyphens are @@ -143,21 +147,22 @@ PyObject *_PyCodec_Lookup(const char *encoding) if (v == NULL) { return NULL; } - PyUnicode_InternInPlace(&v); + + /* Intern the string. We'll make it immortal later if lookup succeeds. */ + _PyUnicode_InternMortal(interp, &v); /* First, try to lookup the name in the registry dictionary */ - PyObject *result = PyDict_GetItemWithError(interp->codec_search_cache, v); + PyObject *result; + if (PyDict_GetItemRef(interp->codecs.search_cache, v, &result) < 0) { + goto onError; + } if (result != NULL) { - Py_INCREF(result); Py_DECREF(v); return result; } - else if (PyErr_Occurred()) { - goto onError; - } /* Next, scan the search functions in order of registration */ - const Py_ssize_t len = PyList_Size(interp->codec_search_path); + const Py_ssize_t len = PyList_Size(interp->codecs.search_path); if (len < 0) goto onError; if (len == 0) { @@ -171,14 +176,15 @@ PyObject *_PyCodec_Lookup(const char *encoding) for (i = 0; i < len; i++) { PyObject *func; - func = PyList_GetItem(interp->codec_search_path, i); + func = PyList_GetItemRef(interp->codecs.search_path, i); if (func == NULL) goto onError; result = PyObject_CallOneArg(func, v); + Py_DECREF(func); if (result == NULL) goto onError; if (result == Py_None) { - Py_DECREF(result); + Py_CLEAR(result); continue; } if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 4) { @@ -189,15 +195,17 @@ PyObject *_PyCodec_Lookup(const char *encoding) } break; } - if (i == len) { + if (result == NULL) { /* XXX Perhaps we should cache misses too ? */ PyErr_Format(PyExc_LookupError, "unknown encoding: %s", encoding); goto onError; } + _PyUnicode_InternImmortal(interp, &v); + /* Cache and return the result */ - if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) { + if (PyDict_SetItem(interp->codecs.search_cache, v, result) < 0) { Py_DECREF(result); goto onError; } @@ -601,13 +609,12 @@ PyObject *_PyCodec_DecodeText(PyObject *object, int PyCodec_RegisterError(const char *name, PyObject *error) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return -1; + assert(interp->codecs.initialized); if (!PyCallable_Check(error)) { PyErr_SetString(PyExc_TypeError, "handler must be callable"); return -1; } - return PyDict_SetItemString(interp->codec_error_registry, + return PyDict_SetItemString(interp->codecs.error_registry, name, error); } @@ -617,13 +624,12 @@ int PyCodec_RegisterError(const char *name, PyObject *error) PyObject *PyCodec_LookupError(const char *name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return NULL; + assert(interp->codecs.initialized); if (name==NULL) name = "strict"; PyObject *handler; - if (PyDict_GetItemStringRef(interp->codec_error_registry, name, &handler) < 0) { + if (PyDict_GetItemStringRef(interp->codecs.error_registry, name, &handler) < 0) { return NULL; } if (handler == NULL) { @@ -932,8 +938,6 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) return Py_BuildValue("(Nn)", res, end); } -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject *PyCodec_NameReplaceErrors(PyObject *exc) { if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { @@ -954,13 +958,9 @@ PyObject *PyCodec_NameReplaceErrors(PyObject *exc) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - if (!ucnhash_capi) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (!ucnhash_capi) { - return NULL; - } + _PyUnicode_Name_CAPI *ucnhash_capi = _PyUnicode_GetNameCAPI(); + if (ucnhash_capi == NULL) { + return NULL; } for (i = start, ressize = 0; i < end; ++i) { /* object is guaranteed to be "ready" */ @@ -1382,7 +1382,8 @@ static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) return PyCodec_SurrogateEscapeErrors(exc); } -static int _PyCodecRegistry_Init(void) +PyStatus +_PyCodec_InitRegistry(PyInterpreterState *interp) { static struct { const char *name; @@ -1470,45 +1471,51 @@ static int _PyCodecRegistry_Init(void) } }; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *mod; - - if (interp->codec_search_path != NULL) - return 0; - - interp->codec_search_path = PyList_New(0); - if (interp->codec_search_path == NULL) { - return -1; + assert(interp->codecs.initialized == 0); + interp->codecs.search_path = PyList_New(0); + if (interp->codecs.search_path == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_search_cache = PyDict_New(); - if (interp->codec_search_cache == NULL) { - return -1; + interp->codecs.search_cache = PyDict_New(); + if (interp->codecs.search_cache == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_error_registry = PyDict_New(); - if (interp->codec_error_registry == NULL) { - return -1; + interp->codecs.error_registry = PyDict_New(); + if (interp->codecs.error_registry == NULL) { + return PyStatus_NoMemory(); } - for (size_t i = 0; i < Py_ARRAY_LENGTH(methods); ++i) { PyObject *func = PyCFunction_NewEx(&methods[i].def, NULL, NULL); - if (!func) { - return -1; + if (func == NULL) { + return PyStatus_NoMemory(); } - int res = PyCodec_RegisterError(methods[i].name, func); + int res = PyDict_SetItemString(interp->codecs.error_registry, + methods[i].name, func); Py_DECREF(func); - if (res) { - return -1; + if (res < 0) { + return PyStatus_Error("Failed to insert into codec error registry"); } } - mod = PyImport_ImportModule("encodings"); + interp->codecs.initialized = 1; + + // Importing `encodings' will call back into this module to register codec + // search functions, so this is done after everything else is initialized. + PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { - return -1; + return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); - interp->codecs_initialized = 1; - return 0; + + return PyStatus_Ok(); +} + +void +_PyCodec_Fini(PyInterpreterState *interp) +{ + Py_CLEAR(interp->codecs.search_path); + Py_CLEAR(interp->codecs.search_cache); + Py_CLEAR(interp->codecs.error_registry); + interp->codecs.initialized = 0; } diff --git a/Python/compile.c b/Python/compile.c index 1d9ae626677310..ca64b5cd376521 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -32,9 +32,9 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_compile.h" #include "pycore_flowgraph.h" +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() @@ -71,23 +71,32 @@ ((C)->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && ((C)->u->u_ste->ste_type == ModuleBlock)) -typedef _PyCompilerSrcLocation location; +struct compiler; + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; + +static instr_sequence *compiler_instr_sequence(struct compiler *c); +static int compiler_future_features(struct compiler *c); +static struct symtable *compiler_symtable(struct compiler *c); +static PySTEntryObject *compiler_symtable_entry(struct compiler *c); + +#define INSTR_SEQUENCE(C) compiler_instr_sequence(C) +#define FUTURE_FEATURES(C) compiler_future_features(C) +#define SYMTABLE(C) compiler_symtable(C) +#define SYMTABLE_ENTRY(C) compiler_symtable_entry(C) + +typedef _Py_SourceLocation location; typedef struct _PyCfgBuilder cfg_builder; -#define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) +static PyObject *compiler_maybe_mangle(struct compiler *c, PyObject *name); -/* Return true if loc1 starts after loc2 ends. */ -static inline bool -location_is_after(location loc1, location loc2) { - return (loc1.lineno > loc2.end_lineno) || - ((loc1.lineno == loc2.end_lineno) && - (loc1.col_offset > loc2.end_col_offset)); -} +#define LOCATION(LNO, END_LNO, COL, END_COL) \ + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) #define LOC(x) SRC_LOCATION_FROM_AST(x) -typedef _PyCfgJumpTargetLabel jump_target_label; +typedef _PyJumpTargetLabel jump_target_label; static jump_target_label NO_LABEL = {-1}; @@ -95,13 +104,13 @@ static jump_target_label NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ + jump_target_label NAME = _PyInstructionSequence_NewLabel(INSTR_SEQUENCE(C)); \ if (!IS_LABEL(NAME)) { \ return ERROR; \ } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) /* fblockinfo tracks the current frame block. @@ -113,11 +122,13 @@ compiler IR. enum fblocktype { WHILE_LOOP, FOR_LOOP, TRY_EXCEPT, FINALLY_TRY, FINALLY_END, WITH, ASYNC_WITH, HANDLER_CLEANUP, POP_VALUE, EXCEPTION_HANDLER, - EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR }; + EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR, + STOP_ITERATION }; struct fblockinfo { enum fblocktype fb_type; jump_target_label fb_block; + location fb_loc; /* (optional) type-specific exit or cleanup block */ jump_target_label fb_exit; /* (optional) additional information required for unwinding */ @@ -131,15 +142,18 @@ enum { COMPILER_SCOPE_ASYNC_FUNCTION, COMPILER_SCOPE_LAMBDA, COMPILER_SCOPE_COMPREHENSION, - COMPILER_SCOPE_TYPEPARAMS, + COMPILER_SCOPE_ANNOTATIONS, }; -typedef _PyCompile_Instruction instruction; -typedef _PyCompile_InstructionSequence instr_sequence; - -#define INITIAL_INSTR_SEQUENCE_SIZE 100 -#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 +static const int compare_masks[] = { + [Py_LT] = COMPARISON_LESS_THAN, + [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, + [Py_EQ] = COMPARISON_EQUALS, + [Py_NE] = COMPARISON_NOT_EQUALS, + [Py_GT] = COMPARISON_GREATER_THAN, + [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, +}; /* * Resize the array if index is out of range. @@ -161,7 +175,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, if (idx >= new_alloc) { new_alloc = idx + default_alloc; } - arr = PyObject_Calloc(new_alloc, item_size); + arr = PyMem_Calloc(new_alloc, item_size); if (arr == NULL) { PyErr_NoMemory(); return ERROR; @@ -182,7 +196,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, } assert(newsize > 0); - void *tmp = PyObject_Realloc(arr, newsize); + void *tmp = PyMem_Realloc(arr, newsize); if (tmp == NULL) { PyErr_NoMemory(); return ERROR; @@ -196,160 +210,6 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, return SUCCESS; } -static int -instr_sequence_next_inst(instr_sequence *seq) { - assert(seq->s_instrs != NULL || seq->s_used == 0); - - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, - (void**)&seq->s_instrs, - &seq->s_allocated, - INITIAL_INSTR_SEQUENCE_SIZE, - sizeof(instruction))); - assert(seq->s_allocated >= 0); - assert(seq->s_used < seq->s_allocated); - return seq->s_used++; -} - -static jump_target_label -instr_sequence_new_label(instr_sequence *seq) -{ - jump_target_label lbl = {++seq->s_next_free_label}; - return lbl; -} - -int -_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) -{ - int old_size = seq->s_labelmap_size; - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(lbl, - (void**)&seq->s_labelmap, - &seq->s_labelmap_size, - INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, - sizeof(int))); - - for(int i = old_size; i < seq->s_labelmap_size; i++) { - seq->s_labelmap[i] = -111; /* something weird, for debugging */ - } - seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ - return SUCCESS; -} - - -#define MAX_OPCODE 511 - -int -_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, - location loc) -{ - assert(0 <= opcode && opcode <= MAX_OPCODE); - assert(IS_WITHIN_OPCODE_RANGE(opcode)); - assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); - assert(0 <= oparg && oparg < (1 << 30)); - - int idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(idx); - instruction *ci = &seq->s_instrs[idx]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - return SUCCESS; -} - -static int -instr_sequence_insert_instruction(instr_sequence *seq, int pos, - int opcode, int oparg, location loc) -{ - assert(pos >= 0 && pos <= seq->s_used); - int last_idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(last_idx); - for (int i=last_idx-1; i >= pos; i--) { - seq->s_instrs[i+1] = seq->s_instrs[i]; - } - instruction *ci = &seq->s_instrs[pos]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - - /* fix the labels map */ - for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - if (seq->s_labelmap[lbl] >= pos) { - seq->s_labelmap[lbl]++; - } - } - return SUCCESS; -} - -static void -instr_sequence_fini(instr_sequence *seq) { - PyObject_Free(seq->s_labelmap); - seq->s_labelmap = NULL; - - PyObject_Free(seq->s_instrs); - seq->s_instrs = NULL; -} - -static cfg_builder* -instr_sequence_to_cfg(instr_sequence *seq) { - cfg_builder *g = _PyCfgBuilder_New(); - if (g == NULL) { - return NULL; - } - - /* There can be more than one label for the same offset. The - * offset2lbl maping selects one of them which we use consistently. - */ - - int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int)); - if (offset2lbl == NULL) { - PyErr_NoMemory(); - goto error; - } - for (int i = 0; i < seq->s_used; i++) { - offset2lbl[i] = -1; - } - for (int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - int offset = seq->s_labelmap[lbl]; - if (offset >= 0) { - assert(offset < seq->s_used); - offset2lbl[offset] = lbl; - } - } - - for (int i = 0; i < seq->s_used; i++) { - int lbl = offset2lbl[i]; - if (lbl >= 0) { - assert (lbl < seq->s_labelmap_size); - jump_target_label lbl_ = {lbl}; - if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { - goto error; - } - } - instruction *instr = &seq->s_instrs[i]; - int opcode = instr->i_opcode; - int oparg = instr->i_oparg; - if (HAS_TARGET(opcode)) { - int offset = seq->s_labelmap[oparg]; - assert(offset >= 0 && offset < seq->s_used); - int lbl = offset2lbl[offset]; - assert(lbl >= 0 && lbl < seq->s_labelmap_size); - oparg = lbl; - } - if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { - goto error; - } - } - if (_PyCfgBuilder_CheckSize(g) < 0) { - goto error; - } - PyMem_Free(offset2lbl); - return g; -error: - _PyCfgBuilder_Free(g); - PyMem_Free(offset2lbl); - return NULL; -} /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. @@ -359,9 +219,11 @@ struct compiler_unit { int u_scope_type; - PyObject *u_private; /* for private name mangling */ + PyObject *u_private; /* for private name mangling */ + PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */ - instr_sequence u_instr_sequence; /* codegen output */ + instr_sequence *u_instr_sequence; /* codegen output */ int u_nfblocks; int u_in_inlined_comp; @@ -386,7 +248,7 @@ handled by the symbol analysis pass. struct compiler { PyObject *c_filename; struct symtable *c_st; - PyFutureFeatures c_future; /* module's __future__ */ + _PyFutureFeatures c_future; /* module's __future__ */ PyCompilerFlags c_flags; int c_optimize; /* optimization level */ @@ -394,12 +256,14 @@ struct compiler { int c_nestlevel; PyObject *c_const_cache; /* Python dict holding all constants, including names tuple */ - struct compiler_unit *u; /* compiler state for current block */ + struct compiler_unit *u; /* compiler state for current block */ PyObject *c_stack; /* Python list holding compiler_unit ptrs */ PyArena *c_arena; /* pointer to memory allocation arena */ -}; -#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) + bool c_save_nested_seqs; /* if true, construct recursive instruction sequences + * (including instructions for nested code objects) + */ +}; typedef struct { @@ -458,7 +322,6 @@ static int compiler_call_helper(struct compiler *c, location loc, asdl_keyword_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); static int compiler_try_star_except(struct compiler *, stmt_ty); -static int compiler_set_qualname(struct compiler *); static int compiler_sync_comprehension_generator( struct compiler *c, location loc, @@ -478,6 +341,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); static int compiler_pattern_subpattern(struct compiler *, pattern_ty, pattern_context *); +static int compiler_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -514,6 +379,7 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, c->c_flags = *flags; c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_nestlevel = 0; + c->c_save_nested_seqs = false; if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { return ERROR; @@ -563,7 +429,7 @@ int _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, int optimize, PyArena *arena) { - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { return -1; } @@ -680,7 +546,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static void compiler_unit_free(struct compiler_unit *u) { - instr_sequence_fini(&u->u_instr_sequence); + Py_CLEAR(u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -691,7 +557,27 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); - PyObject_Free(u); + Py_CLEAR(u->u_static_attributes); + Py_CLEAR(u->u_deferred_annotations); + PyMem_Free(u); +} + +static int +compiler_add_static_attribute_to_class(struct compiler *c, PyObject *attr) +{ + Py_ssize_t stack_size = PyList_GET_SIZE(c->c_stack); + for (Py_ssize_t i = stack_size - 1; i >= 0; i--) { + PyObject *capsule = PyList_GET_ITEM(c->c_stack, i); + struct compiler_unit *u = (struct compiler_unit *)PyCapsule_GetPointer( + capsule, CAPSULE_NAME); + assert(u); + if (u->u_scope_type == COMPILER_SCOPE_CLASS) { + assert(u->u_static_attributes); + RETURN_IF_ERROR(PySet_Add(u->u_static_attributes, attr)); + break; + } + } + return SUCCESS; } static int @@ -712,8 +598,8 @@ compiler_set_qualname(struct compiler *c) capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); assert(parent); - if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) { - /* The parent is a type parameter scope, so we need to + if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) { + /* The parent is an annotation scope, so we need to look at the grandparent. */ if (stack_size == 2) { // If we're immediately within the module, we can skip @@ -761,8 +647,7 @@ compiler_set_qualname(struct compiler *c) } if (base != NULL) { - _Py_DECLARE_STR(dot, "."); - name = PyUnicode_Concat(base, &_Py_STR(dot)); + name = PyUnicode_Concat(base, _Py_LATIN1_CHR('.')); Py_DECREF(base); if (name == NULL) { return ERROR; @@ -780,163 +665,12 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -/* Return the stack effect of opcode with argument oparg. - - Some opcodes have different stack effect when jump to the target and - when not jump. The 'jump' parameter specifies the case: - - * 0 -- when not jump - * 1 -- when jump - * -1 -- maximal - */ -static int -stack_effect(int opcode, int oparg, int jump) -{ - if (0 <= opcode && opcode <= MAX_REAL_OPCODE) { - if (_PyOpcode_Deopt[opcode] != opcode) { - // Specialized instructions are not supported. - return PY_INVALID_STACK_EFFECT; - } - int popped, pushed; - if (jump > 0) { - popped = _PyOpcode_num_popped(opcode, oparg, true); - pushed = _PyOpcode_num_pushed(opcode, oparg, true); - } - else { - popped = _PyOpcode_num_popped(opcode, oparg, false); - pushed = _PyOpcode_num_pushed(opcode, oparg, false); - } - if (popped < 0 || pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - if (jump >= 0) { - return pushed - popped; - } - if (jump < 0) { - // Compute max(pushed - popped, alt_pushed - alt_popped) - int alt_popped = _PyOpcode_num_popped(opcode, oparg, true); - int alt_pushed = _PyOpcode_num_pushed(opcode, oparg, true); - if (alt_popped < 0 || alt_pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - int diff = pushed - popped; - int alt_diff = alt_pushed - alt_popped; - if (alt_diff > diff) { - return alt_diff; - } - return diff; - } - } - - // Pseudo ops - switch (opcode) { - case POP_BLOCK: - case JUMP: - case JUMP_NO_INTERRUPT: - return 0; - - case EXIT_INIT_CHECK: - return -1; - - /* Exception handling pseudo-instructions */ - case SETUP_FINALLY: - /* 0 in the normal flow. - * Restore the stack position and push 1 value before jumping to - * the handler if an exception be raised. */ - return jump ? 1 : 0; - case SETUP_CLEANUP: - /* As SETUP_FINALLY, but pushes lasti as well */ - return jump ? 2 : 0; - case SETUP_WITH: - /* 0 in the normal flow. - * Restore the stack position to the position before the result - * of __(a)enter__ and push 2 values before jumping to the handler - * if an exception be raised. */ - return jump ? 1 : 0; - - case STORE_FAST_MAYBE_NULL: - return -1; - case LOAD_CLOSURE: - return 1; - case LOAD_METHOD: - return 1; - case LOAD_SUPER_METHOD: - case LOAD_ZERO_SUPER_METHOD: - case LOAD_ZERO_SUPER_ATTR: - return -1; - default: - return PY_INVALID_STACK_EFFECT; - } - - return PY_INVALID_STACK_EFFECT; /* not reachable */ -} - -int -PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) -{ - return stack_effect(opcode, oparg, jump); -} - -int -PyCompile_OpcodeStackEffect(int opcode, int oparg) -{ - return stack_effect(opcode, oparg, -1); -} - -int -PyUnstable_OpcodeIsValid(int opcode) -{ - return IS_VALID_OPCODE(opcode); -} - -int -PyUnstable_OpcodeHasArg(int opcode) -{ - return OPCODE_HAS_ARG(opcode); -} - -int -PyUnstable_OpcodeHasConst(int opcode) -{ - return OPCODE_HAS_CONST(opcode); -} - -int -PyUnstable_OpcodeHasName(int opcode) -{ - return OPCODE_HAS_NAME(opcode); -} - -int -PyUnstable_OpcodeHasJump(int opcode) -{ - return OPCODE_HAS_JUMP(opcode); -} - -int -PyUnstable_OpcodeHasFree(int opcode) -{ - return OPCODE_HAS_FREE(opcode); -} - -int -PyUnstable_OpcodeHasLocal(int opcode) -{ - return OPCODE_HAS_LOCAL(opcode); -} - -int -PyUnstable_OpcodeHasExc(int opcode) -{ - return IS_BLOCK_PUSH_OPCODE(opcode); -} - static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!OPCODE_HAS_ARG(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc); + return _PyInstructionSequence_Addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -945,11 +679,10 @@ dict_add_o(PyObject *dict, PyObject *o) PyObject *v; Py_ssize_t arg; - v = PyDict_GetItemWithError(dict, o); + if (PyDict_GetItemRef(dict, o, &v) < 0) { + return ERROR; + } if (!v) { - if (PyErr_Occurred()) { - return ERROR; - } arg = PyDict_GET_SIZE(dict); v = PyLong_FromSsize_t(arg); if (!v) { @@ -959,16 +692,18 @@ dict_add_o(PyObject *dict, PyObject *o) Py_DECREF(v); return ERROR; } - Py_DECREF(v); } else arg = PyLong_AsLong(v); + Py_DECREF(v); return arg; } -// Merge const *o* recursively and return constant key object. +/* Merge const *o* and return constant key object. + * If recursive, insert all elements if o is a tuple or frozen set. + */ static PyObject* -merge_consts_recursive(PyObject *const_cache, PyObject *o) +const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) { assert(PyDict_CheckExact(const_cache)); // None and Ellipsis are immortal objects, and key is the singleton. @@ -982,14 +717,19 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return NULL; } - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); - if (t != key) { - // o is registered in const_cache. Just use it. - Py_XINCREF(t); + PyObject *t; + int res = PyDict_SetDefaultRef(const_cache, key, key, &t); + if (res != 0) { + // o was not inserted into const_cache. t is either the existing value + // or NULL (on error). Py_DECREF(key); return t; } + Py_DECREF(t); + + if (!recursive) { + return key; + } // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its @@ -998,7 +738,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) Py_ssize_t len = PyTuple_GET_SIZE(o); for (Py_ssize_t i = 0; i < len; i++) { PyObject *item = PyTuple_GET_ITEM(o, i); - PyObject *u = merge_consts_recursive(const_cache, item); + PyObject *u = const_cache_insert(const_cache, item, recursive); if (u == NULL) { Py_DECREF(key); return NULL; @@ -1040,7 +780,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) PyObject *item; Py_hash_t hash; while (_PySet_NextEntry(o, &pos, &item, &hash)) { - PyObject *k = merge_consts_recursive(const_cache, item); + PyObject *k = const_cache_insert(const_cache, item, recursive); if (k == NULL) { Py_DECREF(tuple); Py_DECREF(key); @@ -1074,46 +814,56 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return key; } +static PyObject* +merge_consts_recursive(PyObject *const_cache, PyObject *o) +{ + return const_cache_insert(const_cache, o, true); +} + static Py_ssize_t -compiler_add_const(PyObject *const_cache, struct compiler_unit *u, PyObject *o) +compiler_add_const(struct compiler *c, PyObject *o) { - assert(PyDict_CheckExact(const_cache)); - PyObject *key = merge_consts_recursive(const_cache, o); + PyObject *key = merge_consts_recursive(c->c_const_cache, o); if (key == NULL) { return ERROR; } - Py_ssize_t arg = dict_add_o(u->u_metadata.u_consts, key); + Py_ssize_t arg = dict_add_o(c->u->u_metadata.u_consts, key); Py_DECREF(key); return arg; } static int -compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, location loc, PyObject *o) +compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) { - Py_ssize_t arg = compiler_add_const(const_cache, u, o); + Py_ssize_t arg = compiler_add_const(c, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); } static int -compiler_addop_o(struct compiler_unit *u, location loc, +compiler_addop_o(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } +#define LOAD_METHOD -1 +#define LOAD_SUPER_METHOD -2 +#define LOAD_ZERO_SUPER_ATTR -3 +#define LOAD_ZERO_SUPER_METHOD -4 + static int -compiler_addop_name(struct compiler_unit *u, location loc, +compiler_addop_name(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { - PyObject *mangled = _Py_Mangle(u->u_private, o); + PyObject *mangled = compiler_maybe_mangle(c, o); if (!mangled) { return ERROR; } @@ -1126,7 +876,6 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 1; } if (opcode == LOAD_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_METHOD, LOAD_ATTR)); opcode = LOAD_ATTR; arg <<= 1; arg |= 1; @@ -1136,23 +885,20 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg |= 2; } if (opcode == LOAD_SUPER_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 3; } if (opcode == LOAD_ZERO_SUPER_ATTR) { - assert(SAME_OPCODE_METADATA(LOAD_ZERO_SUPER_ATTR, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_ZERO_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 1; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -1169,7 +915,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc); + return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); } static int @@ -1179,7 +925,7 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc); + return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); } #define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ @@ -1195,7 +941,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_IN_SCOPE(C, LOC, OP) RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) #define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), (O))) + RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ @@ -1203,7 +949,7 @@ codegen_addop_j(instr_sequence *seq, location loc, if (__new_const == NULL) { \ return ERROR; \ } \ - if (compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), __new_const) < 0) { \ + if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ return ERROR; \ } \ @@ -1212,7 +958,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ + if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ return ERROR; \ } \ @@ -1220,7 +966,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) + RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -1279,15 +1025,14 @@ codegen_addop_j(instr_sequence *seq, location loc, static int -compiler_enter_scope(struct compiler *c, identifier name, - int scope_type, void *key, int lineno) +compiler_enter_scope(struct compiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private) { location loc = LOCATION(lineno, lineno, 0, 0); struct compiler_unit *u; - u = (struct compiler_unit *)PyObject_Calloc(1, sizeof( - struct compiler_unit)); + u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit)); if (!u) { PyErr_NoMemory(); return ERROR; @@ -1360,7 +1105,23 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } - u->u_private = NULL; + u->u_deferred_annotations = NULL; + if (scope_type == COMPILER_SCOPE_CLASS) { + u->u_static_attributes = PySet_New(0); + if (!u->u_static_attributes) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_static_attributes = NULL; + } + + u->u_instr_sequence = (instr_sequence*)_PyInstructionSequence_New(); + if (!u->u_instr_sequence) { + compiler_unit_free(u); + return ERROR; + } /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -1371,8 +1132,13 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } Py_DECREF(capsule); - u->u_private = Py_XNewRef(c->u->u_private); + if (private == NULL) { + private = c->u->u_private; + } } + + u->u_private = Py_XNewRef(private); + c->u = u; c->c_nestlevel++; @@ -1383,7 +1149,7 @@ compiler_enter_scope(struct compiler *c, identifier name, else { RETURN_IF_ERROR(compiler_set_qualname(c)); } - ADDOP_I(c, loc, RESUME, 0); + ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = -1; @@ -1397,6 +1163,11 @@ compiler_exit_scope(struct compiler *c) // Don't call PySequence_DelItem() with an exception raised PyObject *exc = PyErr_GetRaisedException(); + instr_sequence *nested_seq = NULL; + if (c->c_save_nested_seqs) { + nested_seq = c->u->u_instr_sequence; + Py_INCREF(nested_seq); + } c->c_nestlevel--; compiler_unit_free(c->u); /* Restore c->u to the parent unit. */ @@ -1407,96 +1178,24 @@ compiler_exit_scope(struct compiler *c) assert(c->u); /* we are deleting from a list so this really shouldn't fail */ if (PySequence_DelItem(c->c_stack, n) < 0) { - _PyErr_WriteUnraisableMsg("on removing the last compiler " - "stack item", NULL); + PyErr_FormatUnraisable("Exception ignored on removing " + "the last compiler stack item"); + } + if (nested_seq != NULL) { + if (_PyInstructionSequence_AddNested(c->u->u_instr_sequence, nested_seq) < 0) { + PyErr_FormatUnraisable("Exception ignored on appending " + "nested instruction sequence"); + } } } else { c->u = NULL; } + Py_XDECREF(nested_seq); PyErr_SetRaisedException(exc); } -/* Search if variable annotations are present statically in a block. */ - -static bool -find_ann(asdl_stmt_seq *stmts) -{ - int i, j, res = 0; - stmt_ty st; - - for (i = 0; i < asdl_seq_LEN(stmts); i++) { - st = (stmt_ty)asdl_seq_GET(stmts, i); - switch (st->kind) { - case AnnAssign_kind: - return true; - case For_kind: - res = find_ann(st->v.For.body) || - find_ann(st->v.For.orelse); - break; - case AsyncFor_kind: - res = find_ann(st->v.AsyncFor.body) || - find_ann(st->v.AsyncFor.orelse); - break; - case While_kind: - res = find_ann(st->v.While.body) || - find_ann(st->v.While.orelse); - break; - case If_kind: - res = find_ann(st->v.If.body) || - find_ann(st->v.If.orelse); - break; - case With_kind: - res = find_ann(st->v.With.body); - break; - case AsyncWith_kind: - res = find_ann(st->v.AsyncWith.body); - break; - case Try_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.Try.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.Try.body) || - find_ann(st->v.Try.finalbody) || - find_ann(st->v.Try.orelse); - break; - case TryStar_kind: - for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.TryStar.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.TryStar.body) || - find_ann(st->v.TryStar.finalbody) || - find_ann(st->v.TryStar.orelse); - break; - case Match_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) { - match_case_ty match_case = (match_case_ty)asdl_seq_GET( - st->v.Match.cases, j); - if (find_ann(match_case->body)) { - return true; - } - } - break; - default: - res = false; - break; - } - if (res) { - break; - } - } - return res; -} - /* * Frame block handling functions */ @@ -1513,6 +1212,7 @@ compiler_push_fblock(struct compiler *c, location loc, f = &c->u->u_fblock[c->u->u_nfblocks++]; f->fb_type = t; f->fb_block = block_label; + f->fb_loc = loc; f->fb_exit = exit; f->fb_datum = datum; return SUCCESS; @@ -1534,7 +1234,7 @@ compiler_call_exit_with_nones(struct compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 2); + ADDOP_I(c, loc, CALL, 3); return SUCCESS; } @@ -1550,9 +1250,9 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) // Set up a virtual try/except to handle when StopIteration is raised during // a close or throw call. The only way YIELD_VALUE raises if they do! ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); - ADDOP_I(c, loc, YIELD_VALUE, 0); + ADDOP_I(c, loc, YIELD_VALUE, 1); ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_I(c, loc, RESUME, await ? 3 : 2); + ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM); ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send); USE_LABEL(c, fail); @@ -1593,6 +1293,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case EXCEPTION_HANDLER: case EXCEPTION_GROUP_HANDLER: case ASYNC_COMPREHENSION_GENERATOR: + case STOP_ITERATION: return SUCCESS; case FOR_LOOP: @@ -1639,9 +1340,10 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case WITH: case ASYNC_WITH: - *ploc = LOC((stmt_ty)info->fb_datum); + *ploc = info->fb_loc; ADDOP(c, *ploc, POP_BLOCK); if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 3); ADDOP_I(c, *ploc, SWAP, 2); } RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); @@ -1710,79 +1412,157 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, return SUCCESS; } +static int +compiler_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno, NULL) == -1) { + return ERROR; + } + c->u->u_metadata.u_posonlyargcount = 1; + // if .format != 1: raise NotImplementedError + _Py_DECLARE_STR(format, ".format"); + ADDOP_I(c, loc, LOAD_FAST, 0); + ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); + ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + NEW_JUMP_TARGET_LABEL(c, body); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); + ADDOP_I(c, loc, RAISE_VARARGS, 1); + USE_LABEL(c, body); + return 0; +} + +static int +compiler_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) +{ + ADDOP_I(c, loc, BUILD_MAP, annotations_len); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, loc, co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return 0; +} + /* Compile a sequence of statements, checking for a docstring and for annotations. */ static int compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) { - int i = 0; - stmt_ty st; - PyObject *docstring; /* Set current line number to the line number of first statement. This way line number for SETUP_ANNOTATIONS will always coincide with the line number of first "real" statement in module. If body is empty, then lineno will be set later in optimize_and_assemble. */ if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && asdl_seq_LEN(stmts)) { - st = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); loc = LOC(st); } - /* Every annotated class and module should have __annotations__. */ - if (find_ann(stmts)) { + /* If from __future__ import annotations is active, + * every annotated class and module should have __annotations__. + * Else __annotate__ is created when necessary. */ + if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && SYMTABLE_ENTRY(c)->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { return SUCCESS; } - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - docstring = _PyAST_GetDocString(stmts); + Py_ssize_t first_instr = 0; + if (!c->c_interactive) { + PyObject *docstring = _PyAST_GetDocString(stmts); if (docstring) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; + first_instr = 1; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2) { + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } - i = 1; - st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } - for (; i < asdl_seq_LEN(stmts); i++) { + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); } + // If there are annotations and the future import is not on, we + // collect the annotations in a separate pass and generate an + // __annotate__ function. See PEP 649. + if (!(FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && + c->u->u_deferred_annotations != NULL) { + + // It's possible that ste_annotations_block is set but + // u_deferred_annotations is not, because the former is still + // set if there are only non-simple annotations (i.e., annotations + // for attributes, subscripts, or parenthesized names). However, the + // reverse should not be possible. + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + assert(ste->ste_annotation_block != NULL); + PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); + void *key = (void *)((uintptr_t)ste->ste_id + 1); + if (compiler_setup_annotations_scope(c, loc, key, + ste->ste_annotation_block->ste_name) == -1) { + Py_DECREF(deferred_anno); + return ERROR; + } + Py_ssize_t annotations_len = PyList_Size(deferred_anno); + for (Py_ssize_t i = 0; i < annotations_len; i++) { + PyObject *ptr = PyList_GET_ITEM(deferred_anno, i); + stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr); + if (st == NULL) { + compiler_exit_scope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled); + VISIT(c, expr, st->v.AnnAssign.annotation); + } + Py_DECREF(deferred_anno); + + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + RETURN_IF_ERROR( + compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) + ); + } return SUCCESS; } static int compiler_codegen(struct compiler *c, mod_ty mod) { - _Py_DECLARE_STR(anon_module, ""); - RETURN_IF_ERROR( - compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)); - location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: if (compiler_body(c, loc, mod->v.Module.body) < 0) { - compiler_exit_scope(c); return ERROR; } break; case Interactive_kind: - if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } c->c_interactive = 1; - VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body); + if (compiler_body(c, loc, mod->v.Interactive.body) < 0) { + return ERROR; + } break; case Expression_kind: - VISIT_IN_SCOPE(c, expr, mod->v.Expression.body); + VISIT(c, expr, mod->v.Expression.body); break; default: PyErr_Format(PyExc_SystemError, @@ -1793,25 +1573,35 @@ compiler_codegen(struct compiler *c, mod_ty mod) return SUCCESS; } +static int +compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) +{ + _Py_DECLARE_STR(anon_module, ""); + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1, NULL)); + return SUCCESS; +} + static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { + PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; - if (compiler_codegen(c, mod) < 0) { + if (compiler_enter_anonymous_scope(c, mod) < 0) { return NULL; } - PyCodeObject *co = optimize_and_assemble(c, addNone); + if (compiler_codegen(c, mod) < 0) { + goto finally; + } + co = optimize_and_assemble(c, addNone); +finally: compiler_exit_scope(c); return co; } -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. -*/ - static int -get_ref_type(struct compiler *c, PyObject *name) +compiler_get_ref_type(struct compiler *c, PyObject *name) { int scope; if (c->u->u_scope_type == COMPILER_SCOPE_CLASS && @@ -1819,15 +1609,16 @@ get_ref_type(struct compiler *c, PyObject *name) _PyUnicode_EqualToASCIIString(name, "__classdict__"))) { return CELL; } - scope = _PyST_GetScope(c->u->u_ste, name); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + scope = _PyST_GetScope(ste, name); if (scope == 0) { PyErr_Format(PyExc_SystemError, "_PyST_GetScope(name=%R) failed: " "unknown scope in unit %S (%R); " "symbols: %R; locals: %R; globals: %R", name, - c->u->u_metadata.u_name, c->u->u_ste->ste_id, - c->u->u_ste->ste_symbols, c->u->u_metadata.u_varnames, c->u->u_metadata.u_names); + c->u->u_metadata.u_name, ste->ste_id, + ste->ste_symbols, c->u->u_metadata.u_varnames, c->u->u_metadata.u_names); return ERROR; } return scope; @@ -1848,7 +1639,7 @@ compiler_make_closure(struct compiler *c, location loc, PyCodeObject *co, Py_ssize_t flags) { if (co->co_nfreevars) { - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; ++i) { /* Bypass com_addop_varname because it will generate LOAD_DEREF but LOAD_CLOSURE is needed. @@ -1861,7 +1652,7 @@ compiler_make_closure(struct compiler *c, location loc, class. It should be handled by the closure, as well as by the normal name lookup logic. */ - int reftype = get_ref_type(c, name); + int reftype = compiler_get_ref_type(c, name); if (reftype == -1) { return ERROR; } @@ -1903,6 +1694,9 @@ compiler_make_closure(struct compiler *c, location loc, if (flags & MAKE_FUNCTION_ANNOTATIONS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); } + if (flags & MAKE_FUNCTION_ANNOTATE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); + } if (flags & MAKE_FUNCTION_KWDEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); } @@ -1940,7 +1734,7 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) } static int -compiler_visit_kwonlydefaults(struct compiler *c, location loc, +compiler_kwonlydefaults(struct compiler *c, location loc, asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) { /* Push a dict of keyword-only default values. @@ -1954,7 +1748,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { - PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); + PyObject *mangled = compiler_maybe_mangle(c, arg->arg); if (!mangled) { goto error; } @@ -2005,20 +1799,20 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation) } static int -compiler_visit_argannotation(struct compiler *c, identifier id, +compiler_argannotation(struct compiler *c, identifier id, expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { return SUCCESS; } - PyObject *mangled = _Py_Mangle(c->u->u_private, id); + PyObject *mangled = compiler_maybe_mangle(c, id); if (!mangled) { return ERROR; } ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, annotation); } else { @@ -2034,19 +1828,19 @@ compiler_visit_argannotation(struct compiler *c, identifier id, VISIT(c, expr, annotation); } } - *annotations_len += 2; + *annotations_len += 1; return SUCCESS; } static int -compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, - Py_ssize_t *annotations_len, location loc) +compiler_argannotations(struct compiler *c, asdl_arg_seq* args, + Py_ssize_t *annotations_len, location loc) { int i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); RETURN_IF_ERROR( - compiler_visit_argannotation( + compiler_argannotation( c, arg->arg, arg->annotation, @@ -2057,50 +1851,83 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_visit_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +compiler_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { - /* Push arg annotation names and values. - The expressions are evaluated out-of-order wrt the source code. - - Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. - */ - Py_ssize_t annotations_len = 0; - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->args, &annotations_len, loc)); + compiler_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + compiler_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( - compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)); + compiler_argannotation(c, args->vararg->arg, + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + compiler_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( - compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)); + compiler_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + compiler_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); - if (annotations_len) { - ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); - return 1; + return 0; +} + +static int +compiler_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) +{ + /* Push arg annotation names and values. + The expressions are evaluated separately from the rest of the source code. + + Return -1 on error, or a combination of flags to add to the function. + */ + Py_ssize_t annotations_len = 0; + + PySTEntryObject *ste; + if (_PySymtable_LookupOptional(SYMTABLE(c), args, &ste) < 0) { + return ERROR; + } + assert(ste != NULL); + bool annotations_used = ste->ste_annotations_used; + + if (annotations_used) { + if (compiler_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { + Py_DECREF(ste); + return ERROR; + } + } + Py_DECREF(ste); + + if (compiler_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (annotations_used) { + compiler_exit_scope(c); + } + return ERROR; + } + + if (annotations_used) { + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + return MAKE_FUNCTION_ANNOTATE; } return 0; } static int -compiler_visit_defaults(struct compiler *c, arguments_ty args, +compiler_defaults(struct compiler *c, arguments_ty args, location loc) { VISIT_SEQ(c, expr, args->defaults); @@ -2114,13 +1941,13 @@ compiler_default_arguments(struct compiler *c, location loc, { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - RETURN_IF_ERROR(compiler_visit_defaults(c, args, loc)); + RETURN_IF_ERROR(compiler_defaults(c, args, loc)); funcflags |= MAKE_FUNCTION_DEFAULTS; } if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, loc, - args->kwonlyargs, - args->kw_defaults); + int res = compiler_kwonlydefaults(c, loc, + args->kwonlyargs, + args->kw_defaults); RETURN_IF_ERROR(res); if (res > 0) { funcflags |= MAKE_FUNCTION_KWDEFAULTS; @@ -2185,7 +2012,7 @@ wrap_in_stopiteration_handler(struct compiler *c) /* Insert SETUP_CLEANUP at start */ RETURN_IF_ERROR( - instr_sequence_insert_instruction( + _PyInstructionSequence_InsertInstruction( INSTR_SEQUENCE(c), 0, SETUP_CLEANUP, handler.id, NO_LOCATION)); @@ -2197,6 +2024,36 @@ wrap_in_stopiteration_handler(struct compiler *c) return SUCCESS; } +static int +compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, e->lineno, NULL) == -1) { + return ERROR; + } + if (allow_starred && e->kind == Starred_kind) { + VISIT(c, expr, e->v.Starred.value); + ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + } + else { + VISIT(c, expr, e); + } + ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, LOC(e), co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return SUCCESS; +} + static int compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) { @@ -2204,6 +2061,7 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) return SUCCESS; } Py_ssize_t n = asdl_seq_LEN(type_params); + bool seen_default = false; for (Py_ssize_t i = 0; i < n; i++) { type_param_ty typeparam = asdl_seq_GET(type_params, i); @@ -2213,22 +2071,10 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); if (typeparam->v.TypeVar.bound) { expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_enter_scope(c, typeparam->v.TypeVar.name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparam, bound->lineno) == -1) { + if (compiler_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false) < 0) { return ERROR; } - VISIT_IN_SCOPE(c, expr, bound); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); int intrinsic = bound->kind == Tuple_kind ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS @@ -2238,18 +2084,60 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) else { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVAR); } + if (typeparam->v.TypeVar.default_value) { + seen_default = true; + expr_ty default_ = typeparam->v.TypeVar.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVar.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVar.name, Store)); break; case TypeVarTuple_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVarTuple.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); + if (typeparam->v.TypeVarTuple.default_value) { + expr_ty default_ = typeparam->v.TypeVarTuple.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVarTuple.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVarTuple.name, Store)); break; case ParamSpec_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.ParamSpec.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); + if (typeparam->v.ParamSpec.default_value) { + expr_ty default_ = typeparam->v.ParamSpec.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.ParamSpec.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); break; @@ -2263,7 +2151,6 @@ static int compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags, int firstlineno) { - PyObject *docstring = NULL; arguments_ty args; identifier name; asdl_stmt_seq *body; @@ -2288,37 +2175,55 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f } RETURN_IF_ERROR( - compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); - - /* if not -OO mode, add docstring */ - if (c->c_optimize < 2) { - docstring = _PyAST_GetDocString(body); - if (docstring) { + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL)); + + Py_ssize_t first_instr = 0; + PyObject *docstring = _PyAST_GetDocString(body); + if (docstring) { + first_instr = 1; + /* if not -OO mode, add docstring */ + if (c->c_optimize < 2) { docstring = _PyCompile_CleanDoc(docstring); if (docstring == NULL) { compiler_exit_scope(c); return ERROR; } } + else { + docstring = NULL; + } } - if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { + if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { Py_XDECREF(docstring); compiler_exit_scope(c); return ERROR; } - Py_XDECREF(docstring); + Py_CLEAR(docstring); c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); - for (Py_ssize_t i = docstring ? 1 : 0; i < asdl_seq_LEN(body); i++) { + + NEW_JUMP_TARGET_LABEL(c, start); + USE_LABEL(c, start); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; + if (add_stopiteration_handler) { + /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ + RETURN_IF_ERROR( + compiler_push_fblock(c, NO_LOCATION, STOP_ITERATION, + start, NO_LABEL, NULL)); + } + + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } - if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { + if (add_stopiteration_handler) { if (wrap_in_stopiteration_handler(c) < 0) { compiler_exit_scope(c); return ERROR; } + compiler_pop_fblock(c, STOP_ITERATION, start); } PyCodeObject *co = optimize_and_assemble(c, 1); compiler_exit_scope(c); @@ -2344,7 +2249,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_expr_seq *decos; asdl_type_param_seq *type_params; Py_ssize_t funcflags; - int annotations; int firstlineno; if (is_async) { @@ -2398,8 +2302,8 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, firstlineno) == -1) { + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, + (void *)type_params, firstlineno, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } @@ -2410,16 +2314,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations < 0) { + int annotations_flag = compiler_annotations(c, loc, args, returns); + if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); } return ERROR; } - if (annotations > 0) { - funcflags |= MAKE_FUNCTION_ANNOTATIONS; - } + funcflags |= annotations_flag; if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) { if (is_generic) { @@ -2484,12 +2386,10 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) /* 1. compile the class body into a code object */ RETURN_IF_ERROR( - compiler_enter_scope(c, s->v.ClassDef.name, - COMPILER_SCOPE_CLASS, (void *)s, firstlineno)); + compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, + (void *)s, firstlineno, s->v.ClassDef.name)); location loc = LOCATION(firstlineno, firstlineno, 0, 0); - /* use the class name for name mangling */ - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); /* load (global) __name__ ... */ if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { compiler_exit_scope(c); @@ -2506,6 +2406,11 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(c->u->u_metadata.u_firstlineno)); + if (compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store) < 0) { + compiler_exit_scope(c); + return ERROR; + } asdl_type_param_seq *type_params = s->v.ClassDef.type_params; if (asdl_seq_LEN(type_params) > 0) { if (!compiler_set_type_params_in_class(c, loc)) { @@ -2513,14 +2418,14 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) return ERROR; } } - if (c->u->u_ste->ste_needs_classdict) { + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { ADDOP(c, loc, LOAD_LOCALS); // We can't use compiler_nameop here because we need to generate a // STORE_DEREF in a class namespace, and compiler_nameop() won't do // that by default. PyObject *cellvars = c->u->u_metadata.u_cellvars; - if (compiler_addop_o(c->u, loc, STORE_DEREF, cellvars, + if (compiler_addop_o(c, loc, STORE_DEREF, cellvars, &_Py_ID(__classdict__)) < 0) { compiler_exit_scope(c); return ERROR; @@ -2531,9 +2436,21 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } + assert(c->u->u_static_attributes); + PyObject *static_attributes = PySequence_Tuple(c->u->u_static_attributes); + if (static_attributes == NULL) { + compiler_exit_scope(c); + return ERROR; + } + ADDOP_LOAD_CONST(c, NO_LOCATION, static_attributes); + Py_CLEAR(static_attributes); + if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store) < 0) { + compiler_exit_scope(c); + return ERROR; + } /* The following code is artificial */ /* Set __classdictcell__ if necessary */ - if (c->u->u_ste->ste_needs_classdict) { + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { /* Store __classdictcell__ into class namespace */ int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); if (i < 0) { @@ -2547,7 +2464,7 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } } /* Return __classcell__ if it is referenced, otherwise return None */ - if (c->u->u_ste->ste_needs_class_closure) { + if (SYMTABLE_ENTRY(c)->ste_needs_class_closure) { /* Store __classcell__ into class namespace & return it */ int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); if (i < 0) { @@ -2612,14 +2529,13 @@ compiler_class(struct compiler *c, stmt_ty s) asdl_type_param_seq *type_params = s->v.ClassDef.type_params; int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, firstlineno) == -1) { + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, + (void *)type_params, firstlineno, s->v.ClassDef.name) == -1) { Py_DECREF(type_params_name); return ERROR; } @@ -2671,6 +2587,7 @@ compiler_class(struct compiler *c, stmt_ty s) s->v.ClassDef.keywords)); PyCodeObject *co = optimize_and_assemble(c, 0); + compiler_exit_scope(c); if (co == NULL) { return ERROR; @@ -2702,10 +2619,10 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) location loc = LOC(s); PyObject *name = s->v.TypeAlias.name->v.Name.id; RETURN_IF_ERROR( - compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION, s, loc.lineno)); + compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION, s, loc.lineno, NULL)); /* Make None the first constant, so the evaluate function can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); PyCodeObject *co = optimize_and_assemble(c, 0); @@ -2736,14 +2653,14 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, loc.lineno) == -1) { + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, + (void *)type_params, loc.lineno, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); RETURN_IF_ERROR_IN_SCOPE( - c, compiler_addop_load_const(c->c_const_cache, c->u, loc, name) + c, compiler_addop_load_const(c, loc, name) ); RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); } @@ -2825,15 +2742,6 @@ check_compare(struct compiler *c, expr_ty e) return SUCCESS; } -static const int compare_masks[] = { - [Py_LT] = COMPARISON_LESS_THAN, - [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, - [Py_EQ] = COMPARISON_EQUALS, - [Py_NE] = COMPARISON_NOT_EQUALS, - [Py_GT] = COMPARISON_GREATER_THAN, - [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, -}; - static int compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) { @@ -2921,7 +2829,7 @@ compiler_jump_if(struct compiler *c, location loc, compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); RETURN_IF_ERROR( compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next2); RETURN_IF_ERROR( @@ -2950,12 +2858,12 @@ compiler_jump_if(struct compiler *c, location loc, ADDOP(c, LOC(e), TO_BOOL); ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, cleanup); ADDOP(c, LOC(e), POP_TOP); if (!cond) { - ADDOP_JUMP(c, NO_LOCATION, JUMP, next); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, next); } USE_LABEL(c, end); @@ -2987,7 +2895,7 @@ compiler_ifexp(struct compiler *c, expr_ty e) compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); VISIT(c, expr, e->v.IfExp.body); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next); VISIT(c, expr, e->v.IfExp.orelse); @@ -3015,21 +2923,21 @@ compiler_lambda(struct compiler *c, expr_ty e) _Py_DECLARE_STR(anon_lambda, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno)); + (void *)e, e->lineno, NULL)); /* Make None the first constant, so the lambda can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); - if (c->u->u_ste->ste_generator) { + if (SYMTABLE_ENTRY(c)->ste_generator) { co = optimize_and_assemble(c, 0); } else { - location loc = LOCATION(e->lineno, e->lineno, 0, 0); + location loc = LOC(e->v.Lambda.body); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); co = optimize_and_assemble(c, 1); } @@ -3065,7 +2973,7 @@ compiler_if(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); @@ -3087,11 +2995,18 @@ compiler_for(struct compiler *c, stmt_ty s) RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); VISIT(c, expr, s->v.For.iter); + + loc = LOC(s->v.For.iter); ADDOP(c, loc, GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); + /* Add NOP to ensure correct line tracing of multiline for statements. + * It will be removed later if redundant. + */ + ADDOP(c, LOC(s->v.For.target), NOP); + USE_LABEL(c, body); VISIT(c, expr, s->v.For.target); VISIT_SEQ(c, stmt, s->v.For.body); @@ -3099,7 +3014,12 @@ compiler_for(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, NO_LOCATION, JUMP, start); USE_LABEL(c, cleanup); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); compiler_pop_fblock(c, FOR_LOOP, start); @@ -3114,11 +3034,6 @@ static int compiler_async_for(struct compiler *c, stmt_ty s) { location loc = LOC(s); - if (IS_TOP_LEVEL_AWAIT(c)){ - c->u->u_ste->ste_coroutine = 1; - } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { - return compiler_error(c, loc, "'async for' outside async function"); - } NEW_JUMP_TARGET_LABEL(c, start); NEW_JUMP_TARGET_LABEL(c, except); @@ -3194,12 +3109,12 @@ compiler_return(struct compiler *c, stmt_ty s) location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && (s->v.Return.value->kind != Constant_kind)); - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (!_PyST_IsFunctionLike(ste)) { return compiler_error(c, loc, "'return' outside function"); } - if (s->v.Return.value != NULL && - c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) - { + if (s->v.Return.value != NULL && ste->ste_coroutine && ste->ste_generator) { return compiler_error(c, loc, "'return' with value in async generator"); } @@ -3318,7 +3233,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.Try.finalbody); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); /* `finally` block */ USE_LABEL(c, end); @@ -3368,7 +3283,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); /* `finally` block */ USE_LABEL(c, end); @@ -3443,7 +3358,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) if (s->v.Try.orelse && asdl_seq_LEN(s->v.Try.orelse)) { VISIT_SEQ(c, stmt, s->v.Try.orelse); } - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); n = asdl_seq_LEN(s->v.Try.handlers); USE_LABEL(c, except); @@ -3507,7 +3422,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); RETURN_IF_ERROR( compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); /* except: */ USE_LABEL(c, cleanup_end); @@ -3535,7 +3450,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); } USE_LABEL(c, except); @@ -3623,7 +3538,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) VISIT_SEQ(c, stmt, s->v.TryStar.body); compiler_pop_fblock(c, TRY_EXCEPT, body); ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP, orelse); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, orelse); Py_ssize_t n = asdl_seq_LEN(s->v.TryStar.handlers); USE_LABEL(c, except); @@ -3705,7 +3620,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) RETURN_IF_ERROR( compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); } - ADDOP_JUMP(c, NO_LOCATION, JUMP, except); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except); /* except: */ USE_LABEL(c, cleanup_end); @@ -3722,11 +3637,11 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) /* add exception raised to the res list */ ADDOP_I(c, NO_LOCATION, LIST_APPEND, 3); // exc ADDOP(c, NO_LOCATION, POP_TOP); // lasti - ADDOP_JUMP(c, NO_LOCATION, JUMP, except_with_error); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); USE_LABEL(c, except); ADDOP(c, NO_LOCATION, NOP); // to hold a propagated location info - ADDOP_JUMP(c, NO_LOCATION, JUMP, except_with_error); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); USE_LABEL(c, no_match); ADDOP(c, loc, POP_TOP); // match (None) @@ -3736,7 +3651,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) if (i == n - 1) { /* Add exc to the list (if not None it's the unhandled part of the EG) */ ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); - ADDOP_JUMP(c, NO_LOCATION, JUMP, reraise_star); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, reraise_star); } } /* artificial */ @@ -3752,7 +3667,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) ADDOP(c, NO_LOCATION, POP_TOP); ADDOP(c, NO_LOCATION, POP_BLOCK); ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, reraise); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -3895,14 +3810,6 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyTuple_SET_ITEM(names, i, Py_NewRef(alias->name)); } - if (location_is_after(LOC(s), c->c_future.ff_location) && - s->v.ImportFrom.module && - _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) - { - Py_DECREF(names); - return compiler_error(c, LOC(s), "from __future__ imports must occur " - "at the beginning of the file"); - } ADDOP_LOAD_CONST_NEW(c, LOC(s), names); if (s->v.ImportFrom.module) { @@ -3955,7 +3862,7 @@ compiler_assert(struct compiler *c, stmt_ty s) } NEW_JUMP_TARGET_LABEL(c, end); RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); - ADDOP(c, LOC(s), LOAD_ASSERTION_ERROR); + ADDOP_I(c, LOC(s), LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); ADDOP_I(c, LOC(s), CALL, 0); @@ -4157,11 +4064,12 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, static int addop_yield(struct compiler *c, location loc) { - if (c->u->u_ste->ste_generator && c->u->u_ste->ste_coroutine) { + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (ste->ste_generator && ste->ste_coroutine) { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); } ADDOP_I(c, loc, YIELD_VALUE, 0); - ADDOP_I(c, loc, RESUME, 1); + ADDOP_I(c, loc, RESUME, RESUME_AFTER_YIELD); return SUCCESS; } @@ -4184,14 +4092,14 @@ compiler_nameop(struct compiler *c, location loc, return ERROR; } - mangled = _Py_Mangle(c->u->u_private, name); + mangled = compiler_maybe_mangle(c, name); if (!mangled) { return ERROR; } op = 0; optype = OP_NAME; - scope = _PyST_GetScope(c->u->u_ste, mangled); + scope = _PyST_GetScope(SYMTABLE_ENTRY(c), mangled); switch (scope) { case FREE: dict = c->u->u_metadata.u_freevars; @@ -4202,7 +4110,7 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { optype = OP_FAST; } else { @@ -4218,7 +4126,7 @@ compiler_nameop(struct compiler *c, location loc, } break; case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) + if (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) optype = OP_GLOBAL; break; case GLOBAL_EXPLICIT: @@ -4236,17 +4144,17 @@ compiler_nameop(struct compiler *c, location loc, case OP_DEREF: switch (ctx) { case Load: - if (c->u->u_ste->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { goto error; } } - else if (c->u->u_ste->ste_can_see_class_scope) { + else if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope) { op = LOAD_FROM_DICT_OR_DEREF; // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, + if (compiler_addop_o(c, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { goto error; } @@ -4270,10 +4178,10 @@ compiler_nameop(struct compiler *c, location loc, case OP_GLOBAL: switch (ctx) { case Load: - if (c->u->u_ste->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { + if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { op = LOAD_FROM_DICT_OR_GLOBALS; // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, + if (compiler_addop_o(c, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { goto error; } @@ -4288,7 +4196,7 @@ compiler_nameop(struct compiler *c, location loc, case OP_NAME: switch (ctx) { case Load: - op = (c->u->u_ste->ste_type == ClassBlock + op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && c->u->u_in_inlined_comp) ? LOAD_GLOBAL : LOAD_NAME; @@ -4650,7 +4558,7 @@ compiler_compare(struct compiler *c, expr_ty e) VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, n)); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); USE_LABEL(c, cleanup); ADDOP_I(c, loc, SWAP, 2); @@ -4729,7 +4637,7 @@ check_subscripter(struct compiler *c, expr_ty e) { return SUCCESS; } - /* fall through */ + _Py_FALLTHROUGH; case Set_kind: case SetComp_kind: case GeneratorExp_kind: @@ -4762,7 +4670,7 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) if (!(PyUnicode_Check(v) || PyBytes_Check(v) || PyTuple_Check(v))) { return SUCCESS; } - /* fall through */ + _Py_FALLTHROUGH; case Tuple_kind: case List_kind: case ListComp_kind: @@ -4793,7 +4701,7 @@ is_import_originated(struct compiler *c, expr_ty e) return 0; } - long flags = _PyST_GetSymbol(c->c_st->st_top, e->v.Name.id); + long flags = _PyST_GetSymbol(SYMTABLE(c)->st_top, e->v.Name.id); return flags & DEF_IMPORT; } @@ -4812,11 +4720,11 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) PyObject *super_name = e->v.Call.func->v.Name.id; // detect statically-visible shadowing of 'super' name - int scope = _PyST_GetScope(c->u->u_ste, super_name); + int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), super_name); if (scope != GLOBAL_IMPLICIT) { return 0; } - scope = _PyST_GetScope(c->c_st->st_top, super_name); + scope = _PyST_GetScope(SYMTABLE(c)->st_top, super_name); if (scope != 0) { return 0; } @@ -4844,7 +4752,7 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) return 0; } // __class__ cell should be available - if (get_ref_type(c, &_Py_ID(__class__)) == FREE) { + if (compiler_get_ref_type(c, &_Py_ID(__class__)) == FREE) { return 1; } return 0; @@ -4866,7 +4774,7 @@ load_args_for_super(struct compiler *c, expr_ty e) { // load __class__ cell PyObject *name = &_Py_ID(__class__); - assert(get_ref_type(c, name) == FREE); + assert(compiler_get_ref_type(c, name) == FREE); RETURN_IF_ERROR(compiler_nameop(c, loc, name, Load)); // load self (first argument) @@ -5043,8 +4951,12 @@ compiler_joined_str(struct compiler *c, expr_ty e) } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); - if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) { - ADDOP_I(c, loc, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); + if (value_count > 1) { + ADDOP_I(c, loc, BUILD_STRING, value_count); + } + else if (value_count == 0) { + _Py_DECLARE_STR(empty, ""); + ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); } } return SUCCESS; @@ -5153,7 +5065,7 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, if (names == NULL) { return ERROR; } - for (int i = 0; i < nkwelts; i++) { + for (Py_ssize_t i = 0; i < nkwelts; i++) { keyword_ty kw = asdl_seq_GET(keywords, i); PyTuple_SET_ITEM(names, i, Py_NewRef(kw->arg)); } @@ -5410,7 +5322,12 @@ compiler_sync_comprehension_generator(struct compiler *c, location loc, ADDOP_JUMP(c, elt_loc, JUMP, start); USE_LABEL(c, anchor); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_TOP); } return SUCCESS; @@ -5529,7 +5446,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, PySTEntryObject *entry, inlined_comprehension_state *state) { - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; c->u->u_in_inlined_comp++; // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed @@ -5538,12 +5455,50 @@ push_inlined_comprehension_state(struct compiler *c, location loc, while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { assert(PyLong_Check(v)); long symbol = PyLong_AS_LONG(v); - // only values bound in the comprehension (DEF_LOCAL) need to be handled - // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an - // assignment expression to a nonlocal in the comprehension, these don't - // need handling here since they shouldn't be isolated + long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; + PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); + if (outv == NULL) { + if (PyErr_Occurred()) { + return ERROR; + } + outv = _PyLong_GetZero(); + } + assert(PyLong_CheckExact(outv)); + long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + // If a name has different scope inside than outside the comprehension, + // we need to temporarily handle it with the right scope while + // compiling the comprehension. If it's free in the comprehension + // scope, no special handling; it should be handled the same as the + // enclosing scope. (If it's free in outer scope and cell in inner + // scope, we can't treat it as both cell and free in the same function, + // but treating it as free throughout is fine; it's *_DEREF + // either way.) + if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) + || in_class_block) { + if (state->temp_symbols == NULL) { + state->temp_symbols = PyDict_New(); + if (state->temp_symbols == NULL) { + return ERROR; + } + } + // update the symbol to the in-comprehension version and save + // the outer version; we'll restore it after running the + // comprehension + Py_INCREF(outv); + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v) < 0) { + Py_DECREF(outv); + return ERROR; + } + if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { + Py_DECREF(outv); + return ERROR; + } + Py_DECREF(outv); + } + // locals handling for names bound in comprehension (DEF_LOCAL | + // DEF_NONLOCAL occurs in assignment expression to nonlocal) if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { // non-function scope: override this name to use fast locals PyObject *orig; if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { @@ -5566,41 +5521,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } } } - long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; - PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); - if (outv == NULL) { - outv = _PyLong_GetZero(); - } - assert(PyLong_Check(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc && !(scope == CELL && outsc == FREE)) { - // If a name has different scope inside than outside the - // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. (If it's free - // in outer scope and cell in inner scope, we can't treat it as - // both cell and free in the same function, but treating it as - // free throughout is fine; it's *_DEREF either way.) - - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - Py_INCREF(outv); - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - Py_DECREF(outv); - return ERROR; - } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); - return ERROR; - } - Py_DECREF(outv); - } // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -5680,7 +5600,7 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, Py_ssize_t pos = 0; if (state.temp_symbols) { while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { return ERROR; } } @@ -5688,7 +5608,7 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } if (state.pushed_locals) { ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP, state.end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, state.end); // cleanup from an exception inside the comprehension USE_LABEL(c, state.cleanup); @@ -5749,7 +5669,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, comprehension_ty outermost; int scope_type = c->u->u_scope_type; int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); - PySTEntryObject *entry = _PySymtable_Lookup(c->c_st, (void *)e); + PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; } @@ -5769,7 +5689,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } else { if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno) < 0) + (void *)e, e->lineno, NULL) < 0) { goto error; } @@ -5833,9 +5753,6 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, co = optimize_and_assemble(c, 1); compiler_exit_scope(c); - if (is_top_level_await && is_async_generator){ - c->u->u_ste->ste_coroutine = 1; - } if (co == NULL) { goto error; } @@ -5935,8 +5852,9 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { ADDOP(c, NO_LOCATION, POP_EXCEPT); ADDOP(c, NO_LOCATION, POP_TOP); ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); NEW_JUMP_TARGET_LABEL(c, exit); - ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); USE_LABEL(c, cleanup); POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); @@ -5976,11 +5894,6 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); assert(s->kind == AsyncWith_kind); - if (IS_TOP_LEVEL_AWAIT(c)){ - c->u->u_ste->ste_coroutine = 1; - } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){ - return compiler_error(c, loc, "'async with' outside async function"); - } NEW_JUMP_TARGET_LABEL(c, block); NEW_JUMP_TARGET_LABEL(c, final); @@ -5989,8 +5902,13 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); - - ADDOP(c, loc, BEFORE_ASYNC_WITH); + loc = LOC(item->context_expr); + ADDOP_I(c, loc, COPY, 1); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AEXIT__); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); + ADDOP_I(c, loc, CALL, 0); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -6087,8 +6005,13 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); /* Will push bound __exit__ */ - location loc = LOC(s); - ADDOP(c, loc, BEFORE_WITH); + location loc = LOC(item->context_expr); + ADDOP_I(c, loc, COPY, 1); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___EXIT__); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___ENTER__); + ADDOP_I(c, loc, CALL, 0); ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ @@ -6120,7 +6043,6 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - loc = LOC(s); RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -6138,7 +6060,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) } static int -compiler_visit_expr1(struct compiler *c, expr_ty e) +compiler_visit_expr(struct compiler *c, expr_ty e) { location loc = LOC(e); switch (e->kind) { @@ -6184,7 +6106,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { return compiler_error(c, loc, "'yield' outside function"); } if (e->v.Yield.value) { @@ -6196,8 +6118,8 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP_YIELD(c, loc); break; case YieldFrom_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'yield' outside function"); + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { + return compiler_error(c, loc, "'yield from' outside function"); } if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) { return compiler_error(c, loc, "'yield from' inside async function"); @@ -6209,7 +6131,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) break; case Await_kind: if (!IS_TOP_LEVEL_AWAIT(c)){ - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { return compiler_error(c, loc, "'await' outside function"); } @@ -6246,6 +6168,11 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP(c, loc, NOP); return SUCCESS; } + if (e->v.Attribute.value->kind == Name_kind && + _PyUnicode_EqualToASCIIString(e->v.Attribute.value->v.Name.id, "self")) + { + RETURN_IF_ERROR(compiler_add_static_attribute_to_class(c, e->v.Attribute.attr)); + } VISIT(c, expr, e->v.Attribute.value); loc = LOC(e); loc = update_start_location_to_match_attr(c, loc, e); @@ -6296,13 +6223,6 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) return SUCCESS; } -static int -compiler_visit_expr(struct compiler *c, expr_ty e) -{ - int res = compiler_visit_expr1(c, e); - return res; -} - static bool is_two_element_slice(expr_ty s) { @@ -6398,7 +6318,7 @@ check_annotation(struct compiler *c, stmt_ty s) { /* Annotations of complex targets does not produce anything under annotations future */ - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { return SUCCESS; } @@ -6445,7 +6365,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) { location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; - PyObject* mangled; + bool future_annotations = FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS; + PyObject *mangled; assert(s->kind == AnnAssign_kind); @@ -6463,16 +6384,30 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, s->v.AnnAssign.annotation) + if (future_annotations) { + VISIT(c, annexpr, s->v.AnnAssign.annotation); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); + mangled = compiler_maybe_mangle(c, targ->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); } else { - VISIT(c, expr, s->v.AnnAssign.annotation); + if (c->u->u_deferred_annotations == NULL) { + c->u->u_deferred_annotations = PyList_New(0); + if (c->u->u_deferred_annotations == NULL) { + return ERROR; + } + } + PyObject *ptr = PyLong_FromVoidPtr((void *)s); + if (ptr == NULL) { + return ERROR; + } + if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) { + Py_DECREF(ptr); + return ERROR; + } + Py_DECREF(ptr); } - ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, loc, mangled); - ADDOP(c, loc, STORE_SUBSCR); } break; case Attribute_kind: @@ -6498,7 +6433,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; @@ -6667,7 +6602,7 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) return SUCCESS; } Py_ssize_t needed = sizeof(jump_target_label) * size; - jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed); + jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); if (resized == NULL) { PyErr_NoMemory(); return ERROR; @@ -6706,13 +6641,13 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return ERROR; } } USE_LABEL(c, pc->fail_pop[0]); - PyObject_Free(pc->fail_pop); + PyMem_Free(pc->fail_pop); pc->fail_pop = NULL; return SUCCESS; } @@ -7216,7 +7151,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_DECREF(pc->stores); *pc = old_pc; Py_INCREF(pc->stores); - // Need to NULL this for the PyObject_Free call in the error block. + // Need to NULL this for the PyMem_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || @@ -7262,7 +7197,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) diff: compiler_error(c, LOC(p), "alternative patterns bind different names"); error: - PyObject_Free(old_pc.fail_pop); + PyMem_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); Py_XDECREF(control); return ERROR; @@ -7430,7 +7365,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) ADDOP(c, LOC(m->pattern), POP_TOP); } VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body @@ -7463,7 +7398,7 @@ compiler_match(struct compiler *c, stmt_ty s) pattern_context pc; pc.fail_pop = NULL; int result = compiler_match_inner(c, s, &pc); - PyObject_Free(pc.fail_pop); + PyMem_Free(pc.fail_pop); return result; } @@ -7494,19 +7429,47 @@ consts_dict_keys_inorder(PyObject *dict) return consts; } +static PyObject * +compiler_maybe_mangle(struct compiler *c, PyObject *name) +{ + return _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name); +} + +static instr_sequence * +compiler_instr_sequence(struct compiler *c) +{ + return c->u->u_instr_sequence; +} + +static int +compiler_future_features(struct compiler *c) +{ + return c->c_future.ff_features; +} + +static struct symtable * +compiler_symtable(struct compiler *c) +{ + return c->c_st; +} + +static PySTEntryObject * +compiler_symtable_entry(struct compiler *c) +{ + return c->u->u_ste; +} + static int compute_code_flags(struct compiler *c) { - PySTEntryObject *ste = c->u->u_ste; + PySTEntryObject *ste = SYMTABLE_ENTRY(c); int flags = 0; - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(ste)) { flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; - if (!ste->ste_generator && ste->ste_coroutine) - flags |= CO_COROUTINE; if (ste->ste_generator && ste->ste_coroutine) flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) @@ -7515,45 +7478,33 @@ compute_code_flags(struct compiler *c) flags |= CO_VARKEYWORDS; } - /* (Only) inherit compilerflags in PyCF_MASK */ - flags |= (c->c_flags.cf_flags & PyCF_MASK); - - if ((IS_TOP_LEVEL_AWAIT(c)) && - ste->ste_coroutine && - !ste->ste_generator) { + if (ste->ste_coroutine && !ste->ste_generator) { + assert (IS_TOP_LEVEL_AWAIT(c) || _PyST_IsFunctionLike(ste)); flags |= CO_COROUTINE; } + /* (Only) inherit compilerflags in PyCF_MASK */ + flags |= (c->c_flags.cf_flags & PyCF_MASK); + return flags; } -// Merge *obj* with constant cache. -// Unlike merge_consts_recursive(), this function doesn't work recursively. +// Merge *obj* with constant cache, without recursion. int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) { - assert(PyDict_CheckExact(const_cache)); - PyObject *key = _PyCode_ConstantKey(*obj); + PyObject *key = const_cache_insert(const_cache, *obj, false); if (key == NULL) { return ERROR; } - - // t is borrowed reference - PyObject *t = PyDict_SetDefault(const_cache, key, key); - Py_DECREF(key); - if (t == NULL) { - return ERROR; - } - if (t == key) { // obj is new constant. - return SUCCESS; + if (PyTuple_CheckExact(key)) { + PyObject *item = PyTuple_GET_ITEM(key, 1); + Py_SETREF(*obj, Py_NewRef(item)); + Py_DECREF(key); } - - if (PyTuple_CheckExact(t)) { - // t is still borrowed reference - t = PyTuple_GET_ITEM(t, 1); + else { + Py_SETREF(*obj, key); } - - Py_SETREF(*obj, Py_NewRef(t)); return SUCCESS; } @@ -7583,7 +7534,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - g = instr_sequence_to_cfg(&u->u_instr_sequence); + g = _PyCfg_FromInstructionSequence(u->u_instr_sequence); if (g == NULL) { goto error; } @@ -7612,7 +7563,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, error: Py_XDECREF(consts); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); _PyCfgBuilder_Free(g); return co; } @@ -7636,174 +7587,6 @@ optimize_and_assemble(struct compiler *c, int addNone) return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } -/* Access to compiler optimizations for unit tests. - * - * _PyCompile_CodeGen takes and AST, applies code-gen and - * returns the unoptimized CFG as an instruction list. - * - * _PyCompile_OptimizeCfg takes an instruction list, constructs - * a CFG, optimizes it and converts back to an instruction list. - * - * An instruction list is a PyList where each item is either - * a tuple describing a single instruction: - * (opcode, oparg, lineno, end_lineno, col, end_col), or - * a jump target label marking the beginning of a basic block. - */ - -static int -instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) -{ - assert(PyList_Check(instructions)); - - Py_ssize_t num_insts = PyList_GET_SIZE(instructions); - bool *is_target = PyMem_Calloc(num_insts, sizeof(bool)); - if (is_target == NULL) { - return ERROR; - } - for (Py_ssize_t i = 0; i < num_insts; i++) { - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - if (HAS_TARGET(opcode)) { - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - if (oparg < 0 || oparg >= num_insts) { - PyErr_SetString(PyExc_ValueError, "label out of range"); - goto error; - } - is_target[oparg] = true; - } - } - - for (int i = 0; i < num_insts; i++) { - if (is_target[i]) { - if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) { - goto error; - } - } - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - int oparg; - if (OPCODE_HAS_ARG(opcode)) { - oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - } - else { - oparg = 0; - } - location loc; - loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); - if (PyErr_Occurred()) { - goto error; - } - loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); - if (PyErr_Occurred()) { - goto error; - } - if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { - goto error; - } - } - PyMem_Free(is_target); - return SUCCESS; -error: - PyMem_Free(is_target); - return ERROR; -} - -static cfg_builder* -instructions_to_cfg(PyObject *instructions) -{ - cfg_builder *g = NULL; - instr_sequence seq; - memset(&seq, 0, sizeof(instr_sequence)); - - if (instructions_to_instr_sequence(instructions, &seq) < 0) { - goto error; - } - g = instr_sequence_to_cfg(&seq); - if (g == NULL) { - goto error; - } - instr_sequence_fini(&seq); - return g; -error: - _PyCfgBuilder_Free(g); - instr_sequence_fini(&seq); - return NULL; -} - -static PyObject * -instr_sequence_to_instructions(instr_sequence *seq) -{ - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { - return NULL; - } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? - seq->s_labelmap[instr->i_oparg] : instr->i_oparg; - - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, arg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { - goto error; - } - - int res = PyList_Append(instructions, inst_tuple); - Py_DECREF(inst_tuple); - if (res != 0) { - goto error; - } - } - return instructions; -error: - Py_XDECREF(instructions); - return NULL; -} - -static PyObject * -cfg_to_instructions(cfg_builder *g) -{ - instr_sequence seq; - memset(&seq, 0, sizeof(seq)); - if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { - return NULL; - } - PyObject *res = instr_sequence_to_instructions(&seq); - instr_sequence_fini(&seq); - return res; -} - // C implementation of inspect.cleandoc() // // Difference from inspect.cleandoc(): @@ -7894,6 +7677,12 @@ _PyCompile_CleanDoc(PyObject *doc) return res; } +/* Access to compiler optimizations for unit tests. + * + * _PyCompile_CodeGen takes an AST, applies code-gen and + * returns the unoptimized CFG as an instruction list. + * + */ PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, @@ -7923,16 +7712,22 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, _PyArena_Free(arena); return NULL; } + c->c_save_nested_seqs = true; + + metadata = PyDict_New(); + if (metadata == NULL) { + return NULL; + } + if (compiler_enter_anonymous_scope(c, mod) < 0) { + return NULL; + } if (compiler_codegen(c, mod) < 0) { goto finally; } _PyCompile_CodeUnitMetadata *umd = &c->u->u_metadata; - metadata = PyDict_New(); - if (metadata == NULL) { - goto finally; - } + #define SET_MATADATA_ITEM(key, value) \ if (value != NULL) { \ if (PyDict_SetItemString(metadata, key, value) < 0) goto finally; \ @@ -7965,12 +7760,11 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); - if (insts == NULL) { - goto finally; + if (_PyInstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { + return NULL; } - res = PyTuple_Pack(2, insts, metadata); - Py_DECREF(insts); + /* Allocate a copy of the instruction sequence on the heap */ + res = PyTuple_Pack(2, INSTR_SEQUENCE(c), metadata); finally: Py_XDECREF(metadata); @@ -7980,38 +7774,16 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return res; } -PyObject * -_PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) -{ - cfg_builder *g = NULL; - PyObject *res = NULL; - PyObject *const_cache = PyDict_New(); - if (const_cache == NULL) { - return NULL; - } - - g = instructions_to_cfg(instructions); - if (g == NULL) { - goto error; - } - int nparams = 0, firstlineno = 1; - if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, - nparams, firstlineno) < 0) { - goto error; - } - res = cfg_to_instructions(g); -error: - Py_DECREF(const_cache); - _PyCfgBuilder_Free(g); - return res; -} - int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, - PyObject *instructions) + PyObject *seq) { + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, "expected an instruction sequence"); + return NULL; + } cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; @@ -8022,7 +7794,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - g = instructions_to_cfg(instructions); + g = _PyCfg_FromInstructionSequence((instr_sequence*)seq); if (g == NULL) { goto error; } @@ -8051,7 +7823,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, error: Py_DECREF(const_cache); _PyCfgBuilder_Free(g); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); return co; } diff --git a/Python/condvar.h b/Python/condvar.h index d54db94f2c871d..dcabed6d55928c 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -260,13 +260,13 @@ PyMUTEX_UNLOCK(PyMUTEX_T *cs) return 0; } - Py_LOCAL_INLINE(int) PyCOND_INIT(PyCOND_T *cv) { InitializeConditionVariable(cv); return 0; } + Py_LOCAL_INLINE(int) PyCOND_FINI(PyCOND_T *cv) { @@ -279,27 +279,32 @@ PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) return SleepConditionVariableSRW(cv, cs, INFINITE, 0) ? 0 : -1; } -/* This implementation makes no distinction about timeouts. Signal - * 2 to indicate that we don't know. - */ +/* return 0 for success, 1 on timeout, -1 on error */ Py_LOCAL_INLINE(int) PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, long long us) { - return SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0) ? 2 : -1; + BOOL success = SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0); + if (!success) { + if (GetLastError() == ERROR_TIMEOUT) { + return 1; + } + return -1; + } + return 0; } Py_LOCAL_INLINE(int) PyCOND_SIGNAL(PyCOND_T *cv) { - WakeConditionVariable(cv); - return 0; + WakeConditionVariable(cv); + return 0; } Py_LOCAL_INLINE(int) PyCOND_BROADCAST(PyCOND_T *cv) { - WakeAllConditionVariable(cv); - return 0; + WakeAllConditionVariable(cv); + return 0; } diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000000..e749bd4bf0dc68 --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/context.c b/Python/context.c index c94c014219d0e4..a3830be17908fe 100644 --- a/Python/context.c +++ b/Python/context.c @@ -64,12 +64,12 @@ static int contextvar_del(PyContextVar *var); -#if PyContext_MAXFREELIST > 0 -static struct _Py_context_state * -get_context_state(void) +#ifdef WITH_FREELISTS +static struct _Py_context_freelist * +get_context_freelist(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->context; + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->contexts; } #endif @@ -340,16 +340,12 @@ static inline PyContext * _context_alloc(void) { PyContext *ctx; -#if PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = get_context_state(); -#ifdef Py_DEBUG - // _context_alloc() must not be called after _PyContext_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree) { - state->numfree--; - ctx = state->freelist; - state->freelist = (PyContext *)ctx->ctx_weakreflist; +#ifdef WITH_FREELISTS + struct _Py_context_freelist *context_freelist = get_context_freelist(); + if (context_freelist->numfree > 0) { + context_freelist->numfree--; + ctx = context_freelist->items; + context_freelist->items = (PyContext *)ctx->ctx_weakreflist; OBJECT_STAT_INC(from_freelist); ctx->ctx_weakreflist = NULL; _Py_NewReference((PyObject *)ctx); @@ -471,16 +467,12 @@ context_tp_dealloc(PyContext *self) } (void)context_tp_clear(self); -#if PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = get_context_state(); -#ifdef Py_DEBUG - // _context_alloc() must not be called after _PyContext_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree < PyContext_MAXFREELIST) { - state->numfree++; - self->ctx_weakreflist = (PyObject *)state->freelist; - state->freelist = self; +#ifdef WITH_FREELISTS + struct _Py_context_freelist *context_freelist = get_context_freelist(); + if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { + context_freelist->numfree++; + self->ctx_weakreflist = (PyObject *)context_freelist->items; + context_freelist->items = self; OBJECT_STAT_INC(to_freelist); } else @@ -669,6 +661,7 @@ context_run(PyContext *self, PyObject *const *args, ts, args[0], args + 1, nargs - 1, kwnames); if (_PyContext_Exit(ts, (PyObject *)self)) { + Py_XDECREF(call_result); return NULL; } @@ -901,56 +894,39 @@ contextvar_tp_hash(PyContextVar *self) static PyObject * contextvar_tp_repr(PyContextVar *self) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - - if (_PyUnicodeWriter_WriteASCIIString( - &writer, "" + // "" + Py_ssize_t estimate = self->var_default ? 53 : 43; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; } - PyObject *name = PyObject_Repr(self->var_name); - if (name == NULL) { + if (PyUnicodeWriter_WriteUTF8(writer, "", self); - if (addr == NULL) { + if (PyUnicodeWriter_Format(writer, " at %p>", self) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, addr) < 0) { - Py_DECREF(addr); - goto error; - } - Py_DECREF(addr); - - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -1275,27 +1251,19 @@ get_token_missing(void) void -_PyContext_ClearFreeList(PyInterpreterState *interp) +_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { -#if PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = &interp->context; - for (; state->numfree; state->numfree--) { - PyContext *ctx = state->freelist; - state->freelist = (PyContext *)ctx->ctx_weakreflist; +#ifdef WITH_FREELISTS + struct _Py_context_freelist *state = &freelists->contexts; + for (; state->numfree > 0; state->numfree--) { + PyContext *ctx = state->items; + state->items = (PyContext *)ctx->ctx_weakreflist; ctx->ctx_weakreflist = NULL; PyObject_GC_Del(ctx); } -#endif -} - - -void -_PyContext_Fini(PyInterpreterState *interp) -{ - _PyContext_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyContext_MAXFREELIST > 0 - struct _Py_context_state *state = &interp->context; - state->numfree = -1; + if (is_finalization) { + state->numfree = -1; + } #endif } diff --git a/Python/critical_section.c b/Python/critical_section.c new file mode 100644 index 00000000000000..62ed25523fd6dc --- /dev/null +++ b/Python/critical_section.c @@ -0,0 +1,148 @@ +#include "Python.h" + +#include "pycore_lock.h" +#include "pycore_critical_section.h" + +#ifdef Py_GIL_DISABLED +static_assert(_Alignof(PyCriticalSection) >= 4, + "critical section must be aligned to at least 4 bytes"); +#endif + +void +_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) +{ +#ifdef Py_GIL_DISABLED + PyThreadState *tstate = _PyThreadState_GET(); + c->_cs_mutex = NULL; + c->_cs_prev = (uintptr_t)tstate->critical_section; + tstate->critical_section = (uintptr_t)c; + + PyMutex_Lock(m); + c->_cs_mutex = m; +#endif +} + +void +_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, + int is_m1_locked) +{ +#ifdef Py_GIL_DISABLED + PyThreadState *tstate = _PyThreadState_GET(); + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = tstate->critical_section; + tstate->critical_section = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; + + if (!is_m1_locked) { + PyMutex_Lock(m1); + } + PyMutex_Lock(m2); + c->_cs_base._cs_mutex = m1; + c->_cs_mutex2 = m2; +#endif +} + +#ifdef Py_GIL_DISABLED +static PyCriticalSection * +untag_critical_section(uintptr_t tag) +{ + return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); +} +#endif + +// Release all locks held by critical sections. This is called by +// _PyThreadState_Detach. +void +_PyCriticalSection_SuspendAll(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + uintptr_t *tagptr = &tstate->critical_section; + while (_PyCriticalSection_IsActive(*tagptr)) { + PyCriticalSection *c = untag_critical_section(*tagptr); + + if (c->_cs_mutex) { + PyMutex_Unlock(c->_cs_mutex); + if ((*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + if (c2->_cs_mutex2) { + PyMutex_Unlock(c2->_cs_mutex2); + } + } + } + + *tagptr |= _Py_CRITICAL_SECTION_INACTIVE; + tagptr = &c->_cs_prev; + } +#endif +} + +void +_PyCriticalSection_Resume(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + uintptr_t p = tstate->critical_section; + PyCriticalSection *c = untag_critical_section(p); + assert(!_PyCriticalSection_IsActive(p)); + + PyMutex *m1 = c->_cs_mutex; + c->_cs_mutex = NULL; + + PyMutex *m2 = NULL; + PyCriticalSection2 *c2 = NULL; + if ((p & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + c2 = (PyCriticalSection2 *)c; + m2 = c2->_cs_mutex2; + c2->_cs_mutex2 = NULL; + } + + if (m1) { + PyMutex_Lock(m1); + } + if (m2) { + PyMutex_Lock(m2); + } + + c->_cs_mutex = m1; + if (m2) { + c2->_cs_mutex2 = m2; + } + + tstate->critical_section &= ~_Py_CRITICAL_SECTION_INACTIVE; +#endif +} + +#undef PyCriticalSection_Begin +void +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_Begin(c, op); +#endif +} + +#undef PyCriticalSection_End +void +PyCriticalSection_End(PyCriticalSection *c) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_End(c); +#endif +} + +#undef PyCriticalSection2_Begin +void +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_Begin(c, a, b); +#endif +} + +#undef PyCriticalSection2_End +void +PyCriticalSection2_End(PyCriticalSection2 *c) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_End(c); +#endif +} diff --git a/Python/crossinterp.c b/Python/crossinterp.c new file mode 100644 index 00000000000000..acb372af42408e --- /dev/null +++ b/Python/crossinterp.c @@ -0,0 +1,1916 @@ + +/* API for managing interactions between isolated interpreters */ + +#include "Python.h" +#include "pycore_ceval.h" // _Py_simple_func +#include "pycore_crossinterp.h" // struct _xid +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_namespace.h" //_PyNamespace_New() +#include "pycore_pyerrors.h" // _PyErr_Clear() +#include "pycore_weakref.h" // _PyWeakref_GET_REF() + + +/**************/ +/* exceptions */ +/**************/ + +static int init_exceptions(PyInterpreterState *); +static void fini_exceptions(PyInterpreterState *); +static int _init_not_shareable_error_type(PyInterpreterState *); +static void _fini_not_shareable_error_type(PyInterpreterState *); +static PyObject * _get_not_shareable_error_type(PyInterpreterState *); +#include "crossinterp_exceptions.h" + + +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +int +_Py_CallInInterpreter(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == PyInterpreterState_Get()) { + return func(arg); + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); + return 0; +} + +int +_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == PyInterpreterState_Get()) { + int res = func(arg); + PyMem_RawFree(arg); + return res; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); + return 0; +} + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +static void xid_lookup_init(PyInterpreterState *); +static void xid_lookup_fini(PyInterpreterState *); +static crossinterpdatafunc lookup_getdata(PyInterpreterState *, PyObject *); +#include "crossinterp_data_lookup.h" + + +/* lifecycle */ + +_PyCrossInterpreterData * +_PyCrossInterpreterData_New(void) +{ + _PyCrossInterpreterData *xid = PyMem_RawMalloc( + sizeof(_PyCrossInterpreterData)); + if (xid == NULL) { + PyErr_NoMemory(); + } + return xid; +} + +void +_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyCrossInterpreterData_Clear(interp, xid); + PyMem_RawFree(xid); +} + + +/* defining cross-interpreter data */ + +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + _PyCrossInterpreterData_INTERPID(data) = -1; +} + +static inline void +_xidata_clear(_PyCrossInterpreterData *data) +{ + // _PyCrossInterpreterData only has two members that need to be + // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. + // In both cases the original (owning) interpreter must be used, + // which is the caller's responsibility to ensure. + if (data->data != NULL) { + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + } + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + _PyCrossInterpreterData_INTERPID(data) = (interp != NULL) + ? PyInterpreterState_GetID(interp) + : -1; + data->new_object = new_object; +} + +int +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_RawMalloc(size); + if (data->data == NULL) { + return -1; + } + data->free = PyMem_RawFree; + return 0; +} + +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL + || _PyCrossInterpreterData_INTERPID(data) == -1 + || _PyCrossInterpreterData_INTERPID(data) == PyInterpreterState_GetID(interp)); + _xidata_clear(data); +} + + +/* using cross-interpreter data */ + +static int +_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) +{ + // data->data can be anything, including NULL, so we don't check it. + + // data->obj may be NULL, so we don't check it. + + if (_PyCrossInterpreterData_INTERPID(data) < 0) { + PyErr_SetString(PyExc_SystemError, "missing interp"); + return -1; + } + + if (data->new_object == NULL) { + PyErr_SetString(PyExc_SystemError, "missing new_object func"); + return -1; + } + + // data->free may be NULL, so we don't check it. + + return 0; +} + +static inline void +_set_xid_lookup_failure(PyInterpreterState *interp, + PyObject *obj, const char *msg) +{ + PyObject *exctype = _get_not_shareable_error_type(interp); + assert(exctype != NULL); + if (msg != NULL) { + assert(obj == NULL); + PyErr_SetString(exctype, msg); + } + else if (obj == NULL) { + PyErr_SetString(exctype, + "object does not support cross-interpreter data"); + } + else { + PyErr_Format(exctype, + "%S does not support cross-interpreter data", obj); + } +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + crossinterpdatafunc getdata = lookup_getdata(interp, obj); + if (getdata == NULL) { + if (!PyErr_Occurred()) { + _set_xid_lookup_failure(interp, obj, NULL); + } + return -1; + } + return 0; +} + +int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + PyThreadState *tstate = PyThreadState_Get(); + PyInterpreterState *interp = tstate->interp; + + // Reset data before re-populating. + *data = (_PyCrossInterpreterData){0}; + _PyCrossInterpreterData_INTERPID(data) = -1; + + // Call the "getdata" func for the object. + Py_INCREF(obj); + crossinterpdatafunc getdata = lookup_getdata(interp, obj); + if (getdata == NULL) { + Py_DECREF(obj); + if (!PyErr_Occurred()) { + _set_xid_lookup_failure(interp, obj, NULL); + } + return -1; + } + int res = getdata(tstate, obj, data); + Py_DECREF(obj); + if (res != 0) { + return -1; + } + + // Fill in the blanks and validate the result. + _PyCrossInterpreterData_INTERPID(data) = PyInterpreterState_GetID(interp); + if (_check_xidata(tstate, data) != 0) { + (void)_PyCrossInterpreterData_Release(data); + return -1; + } + + return 0; +} + +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +static int +_call_clear_xidata(void *data) +{ + _xidata_clear((_PyCrossInterpreterData *)data); + return 0; +} + +static int +_xidata_release(_PyCrossInterpreterData *data, int rawfree) +{ + if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { + // Nothing to release! + if (rawfree) { + PyMem_RawFree(data); + } + else { + data->data = NULL; + } + return 0; + } + + // Switch to the original interpreter. + PyInterpreterState *interp = _PyInterpreterState_LookUpID( + _PyCrossInterpreterData_INTERPID(data)); + if (interp == NULL) { + // The interpreter was already destroyed. + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + if (rawfree) { + PyMem_RawFree(data); + } + return -1; + } + + // "Release" the data and/or the object. + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); + } + else { + return _Py_CallInInterpreter(interp, _call_clear_xidata, data); + } +} + +int +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 0); +} + +int +_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 1); +} + + +/*************************/ +/* convenience utilities */ +/*************************/ + +static const char * +_copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) +{ + Py_ssize_t size = -1; + const char *str = PyUnicode_AsUTF8AndSize(strobj, &size); + if (str == NULL) { + return NULL; + } + + char *copied = PyMem_RawMalloc(size+1); + if (copied == NULL) { + PyErr_NoMemory(); + return NULL; + } + strcpy(copied, str); + if (p_size != NULL) { + *p_size = size; + } + return copied; +} + + +static int +_convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) +{ + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *create = NULL; + + // This is inspired by _PyErr_Display(). + PyObject *tbmod = PyImport_ImportModule("traceback"); + if (tbmod == NULL) { + return -1; + } + PyObject *tbexc_type = PyObject_GetAttrString(tbmod, "TracebackException"); + Py_DECREF(tbmod); + if (tbexc_type == NULL) { + return -1; + } + create = PyObject_GetAttrString(tbexc_type, "from_exception"); + Py_DECREF(tbexc_type); + if (create == NULL) { + return -1; + } + + args = PyTuple_Pack(1, exc); + if (args == NULL) { + goto error; + } + + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + if (PyDict_SetItemString(kwargs, "save_exc_type", Py_False) < 0) { + goto error; + } + if (PyDict_SetItemString(kwargs, "lookup_lines", Py_False) < 0) { + goto error; + } + + PyObject *tbexc = PyObject_Call(create, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(create); + if (tbexc == NULL) { + goto error; + } + + *p_tbexc = tbexc; + return 0; + +error: + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(create); + return -1; +} + +// We accommodate backports here. +#ifndef _Py_EMPTY_STR +# define _Py_EMPTY_STR &_Py_STR(empty) +#endif + +static const char * +_format_TracebackException(PyObject *tbexc) +{ + PyObject *lines = PyObject_CallMethod(tbexc, "format", NULL); + if (lines == NULL) { + return NULL; + } + assert(_Py_EMPTY_STR != NULL); + PyObject *formatted_obj = PyUnicode_Join(_Py_EMPTY_STR, lines); + Py_DECREF(lines); + if (formatted_obj == NULL) { + return NULL; + } + + Py_ssize_t size = -1; + const char *formatted = _copy_string_obj_raw(formatted_obj, &size); + Py_DECREF(formatted_obj); + // We remove trailing the newline added by TracebackException.format(). + assert(formatted[size-1] == '\n'); + ((char *)formatted)[size-1] = '\0'; + return formatted; +} + + +static int +_release_xid_data(_PyCrossInterpreterData *data, int rawfree) +{ + PyObject *exc = PyErr_GetRaisedException(); + int res = rawfree + ? _PyCrossInterpreterData_Release(data) + : _PyCrossInterpreterData_ReleaseAndRawFree(data); + if (res < 0) { + /* The owning interpreter is already destroyed. */ + _PyCrossInterpreterData_Clear(NULL, data); + // XXX Emit a warning? + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + return res; +} + + +/***********************/ +/* exception snapshots */ +/***********************/ + +static int +_excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc) +{ + /* Note that this copies directly rather than into an intermediate + struct and does not clear on error. If we need that then we + should have a separate function to wrap this one + and do all that there. */ + PyObject *strobj = NULL; + + PyTypeObject *type = Py_TYPE(exc); + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + assert(_Py_IsImmortal((PyObject *)type)); + info->builtin = type; + } + else { + // Only builtin types are preserved. + info->builtin = NULL; + } + + // __name__ + strobj = PyType_GetName(type); + if (strobj == NULL) { + return -1; + } + info->name = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->name == NULL) { + return -1; + } + + // __qualname__ + strobj = PyType_GetQualName(type); + if (strobj == NULL) { + return -1; + } + info->qualname = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->qualname == NULL) { + return -1; + } + + // __module__ + strobj = PyType_GetModuleName(type); + if (strobj == NULL) { + return -1; + } + info->module = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + + return 0; +} + +static int +_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype) +{ + PyObject *strobj = NULL; + + // __name__ + strobj = PyObject_GetAttrString(exctype, "__name__"); + if (strobj == NULL) { + return -1; + } + info->name = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->name == NULL) { + return -1; + } + + // __qualname__ + strobj = PyObject_GetAttrString(exctype, "__qualname__"); + if (strobj == NULL) { + return -1; + } + info->qualname = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->qualname == NULL) { + return -1; + } + + // __module__ + strobj = PyObject_GetAttrString(exctype, "__module__"); + if (strobj == NULL) { + return -1; + } + info->module = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + + return 0; +} + +static void +_excinfo_clear_type(struct _excinfo_type *info) +{ + if (info->builtin != NULL) { + assert(info->builtin->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(_Py_IsImmortal((PyObject *)info->builtin)); + } + if (info->name != NULL) { + PyMem_RawFree((void *)info->name); + } + if (info->qualname != NULL) { + PyMem_RawFree((void *)info->qualname); + } + if (info->module != NULL) { + PyMem_RawFree((void *)info->module); + } + *info = (struct _excinfo_type){NULL}; +} + +static void +_excinfo_normalize_type(struct _excinfo_type *info, + const char **p_module, const char **p_qualname) +{ + if (info->name == NULL) { + assert(info->builtin == NULL); + assert(info->qualname == NULL); + assert(info->module == NULL); + // This is inspired by TracebackException.format_exception_only(). + *p_module = NULL; + *p_qualname = NULL; + return; + } + + const char *module = info->module; + const char *qualname = info->qualname; + if (qualname == NULL) { + qualname = info->name; + } + assert(module != NULL); + if (strcmp(module, "builtins") == 0) { + module = NULL; + } + else if (strcmp(module, "__main__") == 0) { + module = NULL; + } + *p_qualname = qualname; + *p_module = module; +} + +static void +_PyXI_excinfo_Clear(_PyXI_excinfo *info) +{ + _excinfo_clear_type(&info->type); + if (info->msg != NULL) { + PyMem_RawFree((void *)info->msg); + } + if (info->errdisplay != NULL) { + PyMem_RawFree((void *)info->errdisplay); + } + *info = (_PyXI_excinfo){{NULL}}; +} + +PyObject * +_PyXI_excinfo_format(_PyXI_excinfo *info) +{ + const char *module, *qualname; + _excinfo_normalize_type(&info->type, &module, &qualname); + if (qualname != NULL) { + if (module != NULL) { + if (info->msg != NULL) { + return PyUnicode_FromFormat("%s.%s: %s", + module, qualname, info->msg); + } + else { + return PyUnicode_FromFormat("%s.%s", module, qualname); + } + } + else { + if (info->msg != NULL) { + return PyUnicode_FromFormat("%s: %s", qualname, info->msg); + } + else { + return PyUnicode_FromString(qualname); + } + } + } + else if (info->msg != NULL) { + return PyUnicode_FromString(info->msg); + } + else { + Py_RETURN_NONE; + } +} + +static const char * +_PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) +{ + assert(exc != NULL); + + if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) { + _PyXI_excinfo_Clear(info); + return NULL; + } + const char *failure = NULL; + + if (_excinfo_init_type_from_exception(&info->type, exc) < 0) { + failure = "error while initializing exception type snapshot"; + goto error; + } + + // Extract the exception message. + PyObject *msgobj = PyObject_Str(exc); + if (msgobj == NULL) { + failure = "error while formatting exception"; + goto error; + } + info->msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (info->msg == NULL) { + failure = "error while copying exception message"; + goto error; + } + + // Pickle a traceback.TracebackException. + PyObject *tbexc = NULL; + if (_convert_exc_to_TracebackException(exc, &tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while creating TracebackException"); +#endif + PyErr_Clear(); + } + else { + info->errdisplay = _format_TracebackException(tbexc); + Py_DECREF(tbexc); + if (info->errdisplay == NULL) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while formating TracebackException"); +#endif + PyErr_Clear(); + } + } + + return NULL; + +error: + assert(failure != NULL); + _PyXI_excinfo_Clear(info); + return failure; +} + +static const char * +_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) +{ + const char *failure = NULL; + + PyObject *exctype = PyObject_GetAttrString(obj, "type"); + if (exctype == NULL) { + failure = "exception snapshot missing 'type' attribute"; + goto error; + } + int res = _excinfo_init_type_from_object(&info->type, exctype); + Py_DECREF(exctype); + if (res < 0) { + failure = "error while initializing exception type snapshot"; + goto error; + } + + // Extract the exception message. + PyObject *msgobj = PyObject_GetAttrString(obj, "msg"); + if (msgobj == NULL) { + failure = "exception snapshot missing 'msg' attribute"; + goto error; + } + info->msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (info->msg == NULL) { + failure = "error while copying exception message"; + goto error; + } + + // Pickle a traceback.TracebackException. + PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay"); + if (errdisplay == NULL) { + failure = "exception snapshot missing 'errdisplay' attribute"; + goto error; + } + info->errdisplay = _copy_string_obj_raw(errdisplay, NULL); + Py_DECREF(errdisplay); + if (info->errdisplay == NULL) { + failure = "error while copying exception error display"; + goto error; + } + + return NULL; + +error: + assert(failure != NULL); + _PyXI_excinfo_Clear(info); + return failure; +} + +static void +_PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) +{ + PyObject *tbexc = NULL; + if (info->errdisplay != NULL) { + tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { + PyErr_Clear(); + } + } + + PyObject *formatted = _PyXI_excinfo_format(info); + PyErr_SetObject(exctype, formatted); + Py_DECREF(formatted); + + if (tbexc != NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyObject_SetAttrString(exc, "_errdisplay", tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored when setting _errdisplay"); +#endif + PyErr_Clear(); + } + Py_DECREF(tbexc); + PyErr_SetRaisedException(exc); + } +} + +static PyObject * +_PyXI_excinfo_TypeAsObject(_PyXI_excinfo *info) +{ + PyObject *ns = _PyNamespace_New(NULL); + if (ns == NULL) { + return NULL; + } + int empty = 1; + + if (info->type.name != NULL) { + PyObject *name = PyUnicode_FromString(info->type.name); + if (name == NULL) { + goto error; + } + int res = PyObject_SetAttrString(ns, "__name__", name); + Py_DECREF(name); + if (res < 0) { + goto error; + } + empty = 0; + } + + if (info->type.qualname != NULL) { + PyObject *qualname = PyUnicode_FromString(info->type.qualname); + if (qualname == NULL) { + goto error; + } + int res = PyObject_SetAttrString(ns, "__qualname__", qualname); + Py_DECREF(qualname); + if (res < 0) { + goto error; + } + empty = 0; + } + + if (info->type.module != NULL) { + PyObject *module = PyUnicode_FromString(info->type.module); + if (module == NULL) { + goto error; + } + int res = PyObject_SetAttrString(ns, "__module__", module); + Py_DECREF(module); + if (res < 0) { + goto error; + } + empty = 0; + } + + if (empty) { + Py_CLEAR(ns); + } + + return ns; + +error: + Py_DECREF(ns); + return NULL; +} + +static PyObject * +_PyXI_excinfo_AsObject(_PyXI_excinfo *info) +{ + PyObject *ns = _PyNamespace_New(NULL); + if (ns == NULL) { + return NULL; + } + int res; + + PyObject *type = _PyXI_excinfo_TypeAsObject(info); + if (type == NULL) { + if (PyErr_Occurred()) { + goto error; + } + type = Py_NewRef(Py_None); + } + res = PyObject_SetAttrString(ns, "type", type); + Py_DECREF(type); + if (res < 0) { + goto error; + } + + PyObject *msg = info->msg != NULL + ? PyUnicode_FromString(info->msg) + : Py_NewRef(Py_None); + if (msg == NULL) { + goto error; + } + res = PyObject_SetAttrString(ns, "msg", msg); + Py_DECREF(msg); + if (res < 0) { + goto error; + } + + PyObject *formatted = _PyXI_excinfo_format(info); + if (formatted == NULL) { + goto error; + } + res = PyObject_SetAttrString(ns, "formatted", formatted); + Py_DECREF(formatted); + if (res < 0) { + goto error; + } + + if (info->errdisplay != NULL) { + PyObject *tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { + PyErr_Clear(); + } + else { + res = PyObject_SetAttrString(ns, "errdisplay", tbexc); + Py_DECREF(tbexc); + if (res < 0) { + goto error; + } + } + } + + return ns; + +error: + Py_DECREF(ns); + return NULL; +} + + +int +_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +{ + assert(!PyErr_Occurred()); + if (exc == NULL || exc == Py_None) { + PyErr_SetString(PyExc_ValueError, "missing exc"); + return -1; + } + const char *failure; + if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { + failure = _PyXI_excinfo_InitFromException(info, exc); + } + else { + failure = _PyXI_excinfo_InitFromObject(info, exc); + } + if (failure != NULL) { + PyErr_SetString(PyExc_Exception, failure); + return -1; + } + return 0; +} + +PyObject * +_PyXI_FormatExcInfo(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_format(info); +} + +PyObject * +_PyXI_ExcInfoAsObject(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_AsObject(info); +} + +void +_PyXI_ClearExcInfo(_PyXI_excinfo *info) +{ + _PyXI_excinfo_Clear(info); +} + + +/***************************/ +/* short-term data sharing */ +/***************************/ + +/* error codes */ + +static int +_PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) +{ + assert(!PyErr_Occurred()); + switch (code) { + case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; + case _PyXI_ERR_UNCAUGHT_EXCEPTION: + // There is nothing to apply. +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif + return 0; + case _PyXI_ERR_OTHER: + // XXX msg? + PyErr_SetNone(PyExc_InterpreterError); + break; + case _PyXI_ERR_NO_MEMORY: + PyErr_NoMemory(); + break; + case _PyXI_ERR_ALREADY_RUNNING: + assert(interp != NULL); + assert(_PyInterpreterState_IsRunningMain(interp)); + _PyInterpreterState_FailIfRunningMain(interp); + break; + case _PyXI_ERR_MAIN_NS_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to get __main__ namespace"); + break; + case _PyXI_ERR_APPLY_NS_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to apply namespace to __main__"); + break; + case _PyXI_ERR_NOT_SHAREABLE: + _set_xid_lookup_failure(interp, NULL, NULL); + break; + default: +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#else + PyErr_Format(PyExc_RuntimeError, "unsupported error code %d", code); +#endif + } + assert(PyErr_Occurred()); + return -1; +} + +/* shared exceptions */ + +static const char * +_PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) +{ + if (error->interp == NULL) { + error->interp = PyInterpreterState_Get(); + } + + const char *failure = NULL; + if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // There is an unhandled exception we need to propagate. + failure = _PyXI_excinfo_InitFromException(&error->uncaught, excobj); + if (failure != NULL) { + // We failed to initialize error->uncaught. + // XXX Print the excobj/traceback? Emit a warning? + // XXX Print the current exception/traceback? + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + error->code = _PyXI_ERR_NO_MEMORY; + } + else { + error->code = _PyXI_ERR_OTHER; + } + PyErr_Clear(); + } + else { + error->code = code; + } + assert(error->code != _PyXI_ERR_NO_ERROR); + } + else { + // There is an error code we need to propagate. + assert(excobj == NULL); + assert(code != _PyXI_ERR_NO_ERROR); + error->code = code; + _PyXI_excinfo_Clear(&error->uncaught); + } + return failure; +} + +PyObject * +_PyXI_ApplyError(_PyXI_error *error) +{ + if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // Raise an exception that proxies the propagated exception. + return _PyXI_excinfo_AsObject(&error->uncaught); + } + else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { + // Propagate the exception directly. + _set_xid_lookup_failure(error->interp, NULL, error->uncaught.msg); + } + else { + // Raise an exception corresponding to the code. + assert(error->code != _PyXI_ERR_NO_ERROR); + (void)_PyXI_ApplyErrorCode(error->code, error->interp); + if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { + // __context__ will be set to a proxy of the propagated exception. + PyObject *exc = PyErr_GetRaisedException(); + _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); + PyObject *exc2 = PyErr_GetRaisedException(); + PyException_SetContext(exc, exc2); + PyErr_SetRaisedException(exc); + } + } + assert(PyErr_Occurred()); + return NULL; +} + +/* shared namespaces */ + +/* Shared namespaces are expected to have relatively short lifetimes. + This means dealloc of a shared namespace will normally happen "soon". + Namespace items hold cross-interpreter data, which must get released. + If the namespace/items are cleared in a different interpreter than + where the items' cross-interpreter data was set then that will cause + pending calls to be used to release the cross-interpreter data. + The tricky bit is that the pending calls can happen sufficiently + later that the namespace/items might already be deallocated. This is + a problem if the cross-interpreter data is allocated as part of a + namespace item. If that's the case then we must ensure the shared + namespace is only cleared/freed *after* that data has been released. */ + +typedef struct _sharednsitem { + const char *name; + _PyCrossInterpreterData *data; + // We could have a "PyCrossInterpreterData _data" field, so it would + // be allocated as part of the item and avoid an extra allocation. + // However, doing so adds a bunch of complexity because we must + // ensure the item isn't freed before a pending call might happen + // in a different interpreter to release the XI data. +} _PyXI_namespace_item; + +static int +_sharednsitem_is_initialized(_PyXI_namespace_item *item) +{ + if (item->name != NULL) { + return 1; + } + return 0; +} + +static int +_sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) +{ + item->name = _copy_string_obj_raw(key, NULL); + if (item->name == NULL) { + assert(!_sharednsitem_is_initialized(item)); + return -1; + } + item->data = NULL; + assert(_sharednsitem_is_initialized(item)); + return 0; +} + +static int +_sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) +{ + if (item->data == NULL) { + return 0; + } + if (p_interpid != NULL) { + *p_interpid = _PyCrossInterpreterData_INTERPID(item->data); + } + return 1; +} + +static int +_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) +{ + assert(_sharednsitem_is_initialized(item)); + assert(item->data == NULL); + item->data = PyMem_RawMalloc(sizeof(_PyCrossInterpreterData)); + if (item->data == NULL) { + PyErr_NoMemory(); + return -1; + } + if (_PyObject_GetCrossInterpreterData(value, item->data) != 0) { + PyMem_RawFree(item->data); + item->data = NULL; + // The caller may want to propagate PyExc_NotShareableError + // if currently switched between interpreters. + return -1; + } + return 0; +} + +static void +_sharednsitem_clear_value(_PyXI_namespace_item *item) +{ + _PyCrossInterpreterData *data = item->data; + if (data != NULL) { + item->data = NULL; + int rawfree = 1; + (void)_release_xid_data(data, rawfree); + } +} + +static void +_sharednsitem_clear(_PyXI_namespace_item *item) +{ + if (item->name != NULL) { + PyMem_RawFree((void *)item->name); + item->name = NULL; + } + _sharednsitem_clear_value(item); +} + +static int +_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) +{ + assert(item->name != NULL); + assert(item->data == NULL); + PyObject *value = PyDict_GetItemString(ns, item->name); // borrowed + if (value == NULL) { + if (PyErr_Occurred()) { + return -1; + } + // When applied, this item will be set to the default (or fail). + return 0; + } + if (_sharednsitem_set_value(item, value) < 0) { + return -1; + } + return 0; +} + +static int +_sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) +{ + PyObject *name = PyUnicode_FromString(item->name); + if (name == NULL) { + return -1; + } + PyObject *value; + if (item->data != NULL) { + value = _PyCrossInterpreterData_NewObject(item->data); + if (value == NULL) { + Py_DECREF(name); + return -1; + } + } + else { + value = Py_NewRef(dflt); + } + int res = PyDict_SetItem(ns, name, value); + Py_DECREF(name); + Py_DECREF(value); + return res; +} + +struct _sharedns { + Py_ssize_t len; + _PyXI_namespace_item *items; +}; + +static _PyXI_namespace * +_sharedns_new(void) +{ + _PyXI_namespace *ns = PyMem_RawCalloc(sizeof(_PyXI_namespace), 1); + if (ns == NULL) { + PyErr_NoMemory(); + return NULL; + } + *ns = (_PyXI_namespace){ 0 }; + return ns; +} + +static int +_sharedns_is_initialized(_PyXI_namespace *ns) +{ + if (ns->len == 0) { + assert(ns->items == NULL); + return 0; + } + + assert(ns->len > 0); + assert(ns->items != NULL); + assert(_sharednsitem_is_initialized(&ns->items[0])); + assert(ns->len == 1 + || _sharednsitem_is_initialized(&ns->items[ns->len - 1])); + return 1; +} + +#define HAS_COMPLETE_DATA 1 +#define HAS_PARTIAL_DATA 2 + +static int +_sharedns_has_xidata(_PyXI_namespace *ns, int64_t *p_interpid) +{ + // We expect _PyXI_namespace to always be initialized. + assert(_sharedns_is_initialized(ns)); + int res = 0; + _PyXI_namespace_item *item0 = &ns->items[0]; + if (!_sharednsitem_is_initialized(item0)) { + return 0; + } + int64_t interpid0 = -1; + if (!_sharednsitem_has_value(item0, &interpid0)) { + return 0; + } + if (ns->len > 1) { + // At this point we know it is has at least partial data. + _PyXI_namespace_item *itemN = &ns->items[ns->len-1]; + if (!_sharednsitem_is_initialized(itemN)) { + res = HAS_PARTIAL_DATA; + goto finally; + } + int64_t interpidN = -1; + if (!_sharednsitem_has_value(itemN, &interpidN)) { + res = HAS_PARTIAL_DATA; + goto finally; + } + assert(interpidN == interpid0); + } + res = HAS_COMPLETE_DATA; + *p_interpid = interpid0; + +finally: + return res; +} + +static void +_sharedns_clear(_PyXI_namespace *ns) +{ + if (!_sharedns_is_initialized(ns)) { + return; + } + + // If the cross-interpreter data were allocated as part of + // _PyXI_namespace_item (instead of dynamically), this is where + // we would need verify that we are clearing the items in the + // correct interpreter, to avoid a race with releasing the XI data + // via a pending call. See _sharedns_has_xidata(). + for (Py_ssize_t i=0; i < ns->len; i++) { + _sharednsitem_clear(&ns->items[i]); + } + PyMem_RawFree(ns->items); + ns->items = NULL; + ns->len = 0; +} + +static void +_sharedns_free(_PyXI_namespace *ns) +{ + _sharedns_clear(ns); + PyMem_RawFree(ns); +} + +static int +_sharedns_init(_PyXI_namespace *ns, PyObject *names) +{ + assert(!_sharedns_is_initialized(ns)); + assert(names != NULL); + Py_ssize_t len = PyDict_CheckExact(names) + ? PyDict_Size(names) + : PySequence_Size(names); + if (len < 0) { + return -1; + } + if (len == 0) { + PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); + return -1; + } + assert(len > 0); + + // Allocate the items. + _PyXI_namespace_item *items = + PyMem_RawCalloc(sizeof(struct _sharednsitem), len); + if (items == NULL) { + PyErr_NoMemory(); + return -1; + } + + // Fill in the names. + Py_ssize_t i = -1; + if (PyDict_CheckExact(names)) { + Py_ssize_t pos = 0; + for (i=0; i < len; i++) { + PyObject *key; + if (!PyDict_Next(names, &pos, &key, NULL)) { + // This should not be possible. + assert(0); + goto error; + } + if (_sharednsitem_init(&items[i], key) < 0) { + goto error; + } + } + } + else if (PySequence_Check(names)) { + for (i=0; i < len; i++) { + PyObject *key = PySequence_GetItem(names, i); + if (key == NULL) { + goto error; + } + int res = _sharednsitem_init(&items[i], key); + Py_DECREF(key); + if (res < 0) { + goto error; + } + } + } + else { + PyErr_SetString(PyExc_NotImplementedError, + "non-sequence namespace not supported"); + goto error; + } + + ns->items = items; + ns->len = len; + assert(_sharedns_is_initialized(ns)); + return 0; + +error: + for (Py_ssize_t j=0; j < i; j++) { + _sharednsitem_clear(&items[j]); + } + PyMem_RawFree(items); + assert(!_sharedns_is_initialized(ns)); + return -1; +} + +void +_PyXI_FreeNamespace(_PyXI_namespace *ns) +{ + if (!_sharedns_is_initialized(ns)) { + return; + } + + int64_t interpid = -1; + if (!_sharedns_has_xidata(ns, &interpid)) { + _sharedns_free(ns); + return; + } + + if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) { + _sharedns_free(ns); + } + else { + // If we weren't always dynamically allocating the cross-interpreter + // data in each item then we would need to using a pending call + // to call _sharedns_free(), to avoid the race between freeing + // the shared namespace and releasing the XI data. + _sharedns_free(ns); + } +} + +_PyXI_namespace * +_PyXI_NamespaceFromNames(PyObject *names) +{ + if (names == NULL || names == Py_None) { + return NULL; + } + + _PyXI_namespace *ns = _sharedns_new(); + if (ns == NULL) { + return NULL; + } + + if (_sharedns_init(ns, names) < 0) { + PyMem_RawFree(ns); + if (PySequence_Size(names) == 0) { + PyErr_Clear(); + } + return NULL; + } + + return ns; +} + +#ifndef NDEBUG +static int _session_is_active(_PyXI_session *); +#endif +static void _propagate_not_shareable_error(_PyXI_session *); + +int +_PyXI_FillNamespaceFromDict(_PyXI_namespace *ns, PyObject *nsobj, + _PyXI_session *session) +{ + // session must be entered already, if provided. + assert(session == NULL || _session_is_active(session)); + assert(_sharedns_is_initialized(ns)); + for (Py_ssize_t i=0; i < ns->len; i++) { + _PyXI_namespace_item *item = &ns->items[i]; + if (_sharednsitem_copy_from_ns(item, nsobj) < 0) { + _propagate_not_shareable_error(session); + // Clear out the ones we set so far. + for (Py_ssize_t j=0; j < i; j++) { + _sharednsitem_clear_value(&ns->items[j]); + } + return -1; + } + } + return 0; +} + +// All items are expected to be shareable. +static _PyXI_namespace * +_PyXI_NamespaceFromDict(PyObject *nsobj, _PyXI_session *session) +{ + // session must be entered already, if provided. + assert(session == NULL || _session_is_active(session)); + if (nsobj == NULL || nsobj == Py_None) { + return NULL; + } + if (!PyDict_CheckExact(nsobj)) { + PyErr_SetString(PyExc_TypeError, "expected a dict"); + return NULL; + } + + _PyXI_namespace *ns = _sharedns_new(); + if (ns == NULL) { + return NULL; + } + + if (_sharedns_init(ns, nsobj) < 0) { + if (PyDict_Size(nsobj) == 0) { + PyMem_RawFree(ns); + PyErr_Clear(); + return NULL; + } + goto error; + } + + if (_PyXI_FillNamespaceFromDict(ns, nsobj, session) < 0) { + goto error; + } + + return ns; + +error: + assert(PyErr_Occurred() + || (session != NULL && session->error_override != NULL)); + _sharedns_free(ns); + return NULL; +} + +int +_PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) +{ + for (Py_ssize_t i=0; i < ns->len; i++) { + if (_sharednsitem_apply(&ns->items[i], nsobj, dflt) != 0) { + return -1; + } + } + return 0; +} + + +/**********************/ +/* high-level helpers */ +/**********************/ + +/* enter/exit a cross-interpreter session */ + +static void +_enter_session(_PyXI_session *session, PyInterpreterState *interp) +{ + // Set here and cleared in _exit_session(). + assert(!session->own_init_tstate); + assert(session->init_tstate == NULL); + assert(session->prev_tstate == NULL); + // Set elsewhere and cleared in _exit_session(). + assert(!session->running); + assert(session->main_ns == NULL); + // Set elsewhere and cleared in _capture_current_exception(). + assert(session->error_override == NULL); + // Set elsewhere and cleared in _PyXI_ApplyCapturedException(). + assert(session->error == NULL); + + // Switch to interpreter. + PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *prev = tstate; + if (interp != tstate->interp) { + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); + // XXX Possible GILState issues? + session->prev_tstate = PyThreadState_Swap(tstate); + assert(session->prev_tstate == prev); + session->own_init_tstate = 1; + } + session->init_tstate = tstate; + session->prev_tstate = prev; +} + +static void +_exit_session(_PyXI_session *session) +{ + PyThreadState *tstate = session->init_tstate; + assert(tstate != NULL); + assert(PyThreadState_Get() == tstate); + + // Release any of the entered interpreters resources. + if (session->main_ns != NULL) { + Py_CLEAR(session->main_ns); + } + + // Ensure this thread no longer owns __main__. + if (session->running) { + _PyInterpreterState_SetNotRunningMain(tstate->interp); + assert(!PyErr_Occurred()); + session->running = 0; + } + + // Switch back. + assert(session->prev_tstate != NULL); + if (session->prev_tstate != session->init_tstate) { + assert(session->own_init_tstate); + session->own_init_tstate = 0; + PyThreadState_Clear(tstate); + PyThreadState_Swap(session->prev_tstate); + PyThreadState_Delete(tstate); + } + else { + assert(!session->own_init_tstate); + } + session->prev_tstate = NULL; + session->init_tstate = NULL; +} + +#ifndef NDEBUG +static int +_session_is_active(_PyXI_session *session) +{ + return (session->init_tstate != NULL); +} +#endif + +static void +_propagate_not_shareable_error(_PyXI_session *session) +{ + if (session == NULL) { + return; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + if (PyErr_ExceptionMatches(_get_not_shareable_error_type(interp))) { + // We want to propagate the exception directly. + session->_error_override = _PyXI_ERR_NOT_SHAREABLE; + session->error_override = &session->_error_override; + } +} + +static void +_capture_current_exception(_PyXI_session *session) +{ + assert(session->error == NULL); + if (!PyErr_Occurred()) { + assert(session->error_override == NULL); + return; + } + + // Handle the exception override. + _PyXI_errcode *override = session->error_override; + session->error_override = NULL; + _PyXI_errcode errcode = override != NULL + ? *override + : _PyXI_ERR_UNCAUGHT_EXCEPTION; + + // Pop the exception object. + PyObject *excval = NULL; + if (errcode == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We want to actually capture the current exception. + excval = PyErr_GetRaisedException(); + } + else if (errcode == _PyXI_ERR_ALREADY_RUNNING) { + // We don't need the exception info. + PyErr_Clear(); + } + else { + // We could do a variety of things here, depending on errcode. + // However, for now we simply capture the exception and save + // the errcode. + excval = PyErr_GetRaisedException(); + } + + // Capture the exception. + _PyXI_error *err = &session->_error; + *err = (_PyXI_error){ + .interp = session->init_tstate->interp, + }; + const char *failure; + if (excval == NULL) { + failure = _PyXI_InitError(err, NULL, errcode); + } + else { + failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); + Py_DECREF(excval); + if (failure == NULL && override != NULL) { + err->code = errcode; + } + } + + // Handle capture failure. + if (failure != NULL) { + // XXX Make this error message more generic. + fprintf(stderr, + "RunFailedError: script raised an uncaught exception (%s)", + failure); + err = NULL; + } + + // Finished! + assert(!PyErr_Occurred()); + session->error = err; +} + +PyObject * +_PyXI_ApplyCapturedException(_PyXI_session *session) +{ + assert(!PyErr_Occurred()); + assert(session->error != NULL); + PyObject *res = _PyXI_ApplyError(session->error); + assert((res == NULL) != (PyErr_Occurred() == NULL)); + session->error = NULL; + return res; +} + +int +_PyXI_HasCapturedException(_PyXI_session *session) +{ + return session->error != NULL; +} + +int +_PyXI_Enter(_PyXI_session *session, + PyInterpreterState *interp, PyObject *nsupdates) +{ + // Convert the attrs for cross-interpreter use. + _PyXI_namespace *sharedns = NULL; + if (nsupdates != NULL) { + sharedns = _PyXI_NamespaceFromDict(nsupdates, NULL); + if (sharedns == NULL && PyErr_Occurred()) { + assert(session->error == NULL); + return -1; + } + } + + // Switch to the requested interpreter (if necessary). + _enter_session(session, interp); + _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + + // Ensure this thread owns __main__. + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + // In the case where we didn't switch interpreters, it would + // be more efficient to leave the exception in place and return + // immediately. However, life is simpler if we don't. + errcode = _PyXI_ERR_ALREADY_RUNNING; + goto error; + } + session->running = 1; + + // Cache __main__.__dict__. + PyObject *main_mod = PyUnstable_InterpreterState_GetMainModule(interp); + if (main_mod == NULL) { + errcode = _PyXI_ERR_MAIN_NS_FAILURE; + goto error; + } + PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_DECREF(main_mod); + if (ns == NULL) { + errcode = _PyXI_ERR_MAIN_NS_FAILURE; + goto error; + } + session->main_ns = Py_NewRef(ns); + + // Apply the cross-interpreter data. + if (sharedns != NULL) { + if (_PyXI_ApplyNamespace(sharedns, ns, NULL) < 0) { + errcode = _PyXI_ERR_APPLY_NS_FAILURE; + goto error; + } + _PyXI_FreeNamespace(sharedns); + } + + errcode = _PyXI_ERR_NO_ERROR; + assert(!PyErr_Occurred()); + return 0; + +error: + assert(PyErr_Occurred()); + // We want to propagate all exceptions here directly (best effort). + assert(errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION); + session->error_override = &errcode; + _capture_current_exception(session); + _exit_session(session); + if (sharedns != NULL) { + _PyXI_FreeNamespace(sharedns); + } + return -1; +} + +void +_PyXI_Exit(_PyXI_session *session) +{ + _capture_current_exception(session); + _exit_session(session); +} + + +/*********************/ +/* runtime lifecycle */ +/*********************/ + +PyStatus +_PyXI_Init(PyInterpreterState *interp) +{ + // Initialize the XID lookup state (e.g. registry). + xid_lookup_init(interp); + + // Initialize exceptions (heap types). + if (_init_not_shareable_error_type(interp) < 0) { + return _PyStatus_ERR("failed to initialize NotShareableError"); + } + + return _PyStatus_OK(); +} + +// _PyXI_Fini() must be called before the interpreter is cleared, +// since we must clear some heap objects. + +void +_PyXI_Fini(PyInterpreterState *interp) +{ + // Finalize exceptions (heap types). + _fini_not_shareable_error_type(interp); + + // Finalize the XID lookup state (e.g. registry). + xid_lookup_fini(interp); +} + +PyStatus +_PyXI_InitTypes(PyInterpreterState *interp) +{ + if (init_exceptions(interp) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR("failed to initialize an exception type"); + } + return _PyStatus_OK(); +} + +void +_PyXI_FiniTypes(PyInterpreterState *interp) +{ + fini_exceptions(interp); +} + + +/*************/ +/* other API */ +/*************/ + +PyInterpreterState * +_PyXI_NewInterpreter(PyInterpreterConfig *config, long *maybe_whence, + PyThreadState **p_tstate, PyThreadState **p_save_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + assert(save_tstate != NULL); + + PyThreadState *tstate; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + if (PyStatus_Exception(status)) { + // Since no new thread state was created, there is no exception + // to propagate; raise a fresh one after swapping back in the + // old thread state. + PyThreadState_Swap(save_tstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + long whence = _PyInterpreterState_WHENCE_XI; + if (maybe_whence != NULL) { + whence = *maybe_whence; + } + _PyInterpreterState_SetWhence(interp, whence); + + if (p_tstate != NULL) { + // We leave the new thread state as the current one. + *p_tstate = tstate; + } + else { + // Throw away the initial tstate. + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + save_tstate = NULL; + } + if (p_save_tstate != NULL) { + *p_save_tstate = save_tstate; + } + return interp; +} + +void +_PyXI_EndInterpreter(PyInterpreterState *interp, + PyThreadState *tstate, PyThreadState **p_save_tstate) +{ +#ifndef NDEBUG + long whence = _PyInterpreterState_GetWhence(interp); +#endif + assert(whence != _PyInterpreterState_WHENCE_RUNTIME); + + if (!_PyInterpreterState_IsReady(interp)) { + assert(whence == _PyInterpreterState_WHENCE_UNKNOWN); + // PyInterpreterState_Clear() requires the GIL, + // which a not-ready does not have, so we don't clear it. + // That means there may be leaks here until clearing the + // interpreter is fixed. + PyInterpreterState_Delete(interp); + return; + } + assert(whence != _PyInterpreterState_WHENCE_UNKNOWN); + + PyThreadState *save_tstate = NULL; + PyThreadState *cur_tstate = PyThreadState_GET(); + if (tstate == NULL) { + if (PyThreadState_GetInterpreter(cur_tstate) == interp) { + tstate = cur_tstate; + } + else { + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + assert(tstate != NULL); + save_tstate = PyThreadState_Swap(tstate); + } + } + else { + assert(PyThreadState_GetInterpreter(tstate) == interp); + if (tstate != cur_tstate) { + assert(PyThreadState_GetInterpreter(cur_tstate) != interp); + save_tstate = PyThreadState_Swap(tstate); + } + } + + Py_EndInterpreter(tstate); + + if (p_save_tstate != NULL) { + save_tstate = *p_save_tstate; + } + PyThreadState_Swap(save_tstate); +} diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h new file mode 100644 index 00000000000000..863919ad42fb97 --- /dev/null +++ b/Python/crossinterp_data_lookup.h @@ -0,0 +1,594 @@ + +static crossinterpdatafunc _lookup_getdata_from_registry( + PyInterpreterState *, PyObject *); + +static crossinterpdatafunc +lookup_getdata(PyInterpreterState *interp, PyObject *obj) +{ + /* Cross-interpreter objects are looked up by exact match on the class. + We can reassess this policy when we move from a global registry to a + tp_* slot. */ + return _lookup_getdata_from_registry(interp, obj); +} + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + return lookup_getdata(interp, obj); +} + + +/***********************************************/ +/* a registry of {type -> crossinterpdatafunc} */ +/***********************************************/ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + + +/* registry lifecycle */ + +static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *); + +static void +_xidregistry_init(struct _xidregistry *registry) +{ + if (registry->initialized) { + return; + } + registry->initialized = 1; + + if (registry->global) { + // Registering the builtins is cheap so we don't bother doing it lazily. + assert(registry->head == NULL); + _register_builtins_for_crossinterpreter_data(registry); + } +} + +static void _xidregistry_clear(struct _xidregistry *); + +static void +_xidregistry_fini(struct _xidregistry *registry) +{ + if (!registry->initialized) { + return; + } + registry->initialized = 0; + + _xidregistry_clear(registry); +} + +static inline struct _xidregistry * _get_global_xidregistry(_PyRuntimeState *); +static inline struct _xidregistry * _get_xidregistry(PyInterpreterState *); + +static void +xid_lookup_init(PyInterpreterState *interp) +{ + if (_Py_IsMainInterpreter(interp)) { + _xidregistry_init(_get_global_xidregistry(interp->runtime)); + } + _xidregistry_init(_get_xidregistry(interp)); +} + +static void +xid_lookup_fini(PyInterpreterState *interp) +{ + _xidregistry_fini(_get_xidregistry(interp)); + if (_Py_IsMainInterpreter(interp)) { + _xidregistry_fini(_get_global_xidregistry(interp->runtime)); + } +} + + +/* registry thread safety */ + +static void +_xidregistry_lock(struct _xidregistry *registry) +{ + if (registry->global) { + PyMutex_Lock(®istry->mutex); + } + // else: Within an interpreter we rely on the GIL instead of a separate lock. +} + +static void +_xidregistry_unlock(struct _xidregistry *registry) +{ + if (registry->global) { + PyMutex_Unlock(®istry->mutex); + } +} + + +/* accessing the registry */ + +static inline struct _xidregistry * +_get_global_xidregistry(_PyRuntimeState *runtime) +{ + return &runtime->xi.registry; +} + +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp) +{ + return &interp->xi.registry; +} + +static inline struct _xidregistry * +_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *registry = _get_global_xidregistry(interp->runtime); + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + registry = _get_xidregistry(interp); + } + return registry; +} + +static struct _xidregitem * _xidregistry_remove_entry( + struct _xidregistry *, struct _xidregitem *); + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = _PyWeakref_GET_REF(cur->weakref); + if (registered == NULL) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; + } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_DECREF(registered); + } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; + } + return NULL; +} + +static crossinterpdatafunc +_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) +{ + PyTypeObject *cls = Py_TYPE(obj); + + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + + _xidregistry_unlock(xidregistry); + return func; +} + + +/* updating the registry */ + +static int +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) +{ + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) { + return -1; + } + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } + } + newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } + xidregistry->head = newhead; + return 0; +} + +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_XDECREF(entry->weakref); + PyMem_RawFree(entry); + return next; +} + +static void +_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + +int +_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; + } + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: + _xidregistry_unlock(xidregistry); + return res; +} + +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + _xidregistry_lock(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } + res = 1; + } + + _xidregistry_unlock(xidregistry); + return res; +} + + +/********************************************/ +/* cross-interpreter data for builtin types */ +/********************************************/ + +// bytes + +struct _shared_bytes_data { + char *bytes; + Py_ssize_t len; +}; + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); + return PyBytes_FromStringAndSize(shared->bytes, shared->len); +} + +static int +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; + if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); + return -1; + } + return 0; +} + +// str + +struct _shared_str_data { + int kind; + const void *buffer; + Py_ssize_t len; +}; + +static PyObject * +_new_str_object(_PyCrossInterpreterData *data) +{ + struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); + return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); +} + +static int +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; + shared->kind = PyUnicode_KIND(obj); + shared->buffer = PyUnicode_DATA(obj); + shared->len = PyUnicode_GET_LENGTH(obj); + return 0; +} + +// int + +static PyObject * +_new_long_object(_PyCrossInterpreterData *data) +{ + return PyLong_FromSsize_t((Py_ssize_t)(data->data)); +} + +static int +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + /* Note that this means the size of shareable ints is bounded by + * sys.maxsize. Hence on 32-bit architectures that is half the + * size of maximum shareable ints on 64-bit. + */ + Py_ssize_t value = PyLong_AsSsize_t(obj); + if (value == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); + } + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL + return 0; +} + +// float + +static PyObject * +_new_float_object(_PyCrossInterpreterData *data) +{ + double * value_ptr = data->data; + return PyFloat_FromDouble(*value_ptr); +} + +static int +_float_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(double), NULL, + _new_float_object + ) < 0) + { + return -1; + } + double *shared = (double *)data->data; + *shared = PyFloat_AsDouble(obj); + return 0; +} + +// None + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + return Py_NewRef(Py_None); +} + +static int +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL + return 0; +} + +// bool + +static PyObject * +_new_bool_object(_PyCrossInterpreterData *data) +{ + if (data->data){ + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static int +_bool_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, + (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL, + _new_bool_object); + // data->obj and data->free remain NULL + return 0; +} + +// tuple + +struct _shared_tuple_data { + Py_ssize_t len; + _PyCrossInterpreterData **data; +}; + +static PyObject * +_new_tuple_object(_PyCrossInterpreterData *data) +{ + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); + PyObject *tuple = PyTuple_New(shared->len); + if (tuple == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < shared->len; i++) { + PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + if (item == NULL){ + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} + +static void +_tuple_shared_free(void* data) +{ + struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data); +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET()); +#endif + for (Py_ssize_t i = 0; i < shared->len; i++) { + if (shared->data[i] != NULL) { + assert(_PyCrossInterpreterData_INTERPID(shared->data[i]) == interpid); + _PyCrossInterpreterData_Release(shared->data[i]); + PyMem_RawFree(shared->data[i]); + shared->data[i] = NULL; + } + } + PyMem_Free(shared->data); + PyMem_RawFree(shared); +} + +static int +_tuple_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_ssize_t len = PyTuple_GET_SIZE(obj); + if (len < 0) { + return -1; + } + struct _shared_tuple_data *shared = PyMem_RawMalloc(sizeof(struct _shared_tuple_data)); + if (shared == NULL){ + PyErr_NoMemory(); + return -1; + } + + shared->len = len; + shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); + if (shared->data == NULL) { + PyErr_NoMemory(); + return -1; + } + + for (Py_ssize_t i = 0; i < shared->len; i++) { + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + goto error; // PyErr_NoMemory already set + } + PyObject *item = PyTuple_GET_ITEM(obj, i); + + int res = -1; + if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { + res = _PyObject_GetCrossInterpreterData(item, data); + _Py_LeaveRecursiveCallTstate(tstate); + } + if (res < 0) { + PyMem_RawFree(data); + goto error; + } + shared->data[i] = data; + } + _PyCrossInterpreterData_Init( + data, tstate->interp, shared, obj, _new_tuple_object); + data->free = _tuple_shared_free; + return 0; + +error: + _tuple_shared_free(shared); + return -1; +} + +// registration + +static void +_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +{ + // None + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for cross-interpreter sharing"); + } + + // int + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + Py_FatalError("could not register int for cross-interpreter sharing"); + } + + // bytes + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for cross-interpreter sharing"); + } + + // str + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + Py_FatalError("could not register str for cross-interpreter sharing"); + } + + // bool + if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { + Py_FatalError("could not register bool for cross-interpreter sharing"); + } + + // float + if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { + Py_FatalError("could not register float for cross-interpreter sharing"); + } + + // tuple + if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + Py_FatalError("could not register tuple for cross-interpreter sharing"); + } +} diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h new file mode 100644 index 00000000000000..278511da615c75 --- /dev/null +++ b/Python/crossinterp_exceptions.h @@ -0,0 +1,95 @@ + +/* InterpreterError extends Exception */ + +static PyTypeObject _PyExc_InterpreterError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "interpreters.InterpreterError", + .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, + //.tp_base = (PyTypeObject *)PyExc_Exception, +}; +PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + +/* InterpreterNotFoundError extends InterpreterError */ + +static PyTypeObject _PyExc_InterpreterNotFoundError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "interpreters.InterpreterNotFoundError", + .tp_doc = PyDoc_STR("An interpreter was not found"), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, + .tp_base = &_PyExc_InterpreterError, +}; +PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; + +/* NotShareableError extends ValueError */ + +static int +_init_not_shareable_error_type(PyInterpreterState *interp) +{ + const char *name = "interpreters.NotShareableError"; + PyObject *base = PyExc_ValueError; + PyObject *ns = NULL; + PyObject *exctype = PyErr_NewException(name, base, ns); + if (exctype == NULL) { + return -1; + } + + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError = exctype; + return 0; +} + +static void +_fini_not_shareable_error_type(PyInterpreterState *interp) +{ + Py_CLEAR(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError); +} + +static PyObject * +_get_not_shareable_error_type(PyInterpreterState *interp) +{ + assert(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError != NULL); + return _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; +} + + +/* lifecycle */ + +static int +init_exceptions(PyInterpreterState *interp) +{ + PyTypeObject *base = (PyTypeObject *)PyExc_Exception; + + // builtin static types + + _PyExc_InterpreterError.tp_base = base; + _PyExc_InterpreterError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterError.tp_clear = base->tp_clear; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + return -1; + } + + _PyExc_InterpreterNotFoundError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterNotFoundError.tp_clear = base->tp_clear; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + return -1; + } + + // heap types + + // We would call _init_not_shareable_error_type() here too, + // but that leads to ref leaks + + return 0; +} + +static void +fini_exceptions(PyInterpreterState *interp) +{ + // Likewise with _fini_not_shareable_error_type(). + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterError); +} diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt deleted file mode 100644 index 276ab51143ab33..00000000000000 --- a/Python/deepfreeze/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains the generated .c files for all the deep-frozen -modules. Python/frozen.c depends on these files. - -None of these files are committed into the repo. - -See Tools/build/freeze_modules.py for more info. diff --git a/Python/dtoa.c b/Python/dtoa.c index 5dfc0e179cbc34..d0c89b2b468f75 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -72,7 +72,7 @@ /* Please send bug reports for the original dtoa.c code to David M. Gay (dmg * at acm dot org, with " at " changed at "@" and " dot " changed to "."). * Please report bugs for this modified version using the Python issue tracker - * (http://bugs.python.org). */ + * as detailed at (https://devguide.python.org/triage/issue-tracker/). */ /* On a machine with IEEE extended-precision registers, it is * necessary to specify double-precision (53-bit) rounding precision @@ -309,7 +309,7 @@ BCinfo { // struct Bigint is defined in pycore_dtoa.h. typedef struct Bigint Bigint; -#ifndef Py_USING_MEMORY_DEBUGGER +#if !defined(Py_GIL_DISABLED) && !defined(Py_USING_MEMORY_DEBUGGER) /* Memory management: memory is allocated from, and returned to, Kmax+1 pools of memory, where pool k (0 <= k <= Kmax) is for Bigints b with b->maxwds == @@ -428,7 +428,7 @@ Bfree(Bigint *v) } } -#endif /* Py_USING_MEMORY_DEBUGGER */ +#endif /* !defined(Py_GIL_DISABLED) && !defined(Py_USING_MEMORY_DEBUGGER) */ #define Bcopy(x,y) memcpy((char *)&x->sign, (char *)&y->sign, \ y->wds*sizeof(Long) + 2*sizeof(int)) @@ -673,10 +673,17 @@ mult(Bigint *a, Bigint *b) static Bigint * pow5mult(Bigint *b, int k) { - Bigint *b1, *p5, *p51; + Bigint *b1, *p5, **p5s; int i; static const int p05[3] = { 5, 25, 125 }; + // For double-to-string conversion, the maximum value of k is limited by + // DBL_MAX_10_EXP (308), the maximum decimal base-10 exponent for binary64. + // For string-to-double conversion, the extreme case is constrained by our + // hardcoded exponent limit before we underflow of -512, adjusted by + // STRTOD_DIGLIM-DBL_DIG-1, giving a maximum of k=535. + assert(0 <= k && k < 1024); + if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); if (b == NULL) @@ -686,18 +693,11 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; PyInterpreterState *interp = _PyInterpreterState_GET(); - p5 = interp->dtoa.p5s; - if (!p5) { - /* first time */ - p5 = i2b(625); - if (p5 == NULL) { - Bfree(b); - return NULL; - } - interp->dtoa.p5s = p5; - p5->next = 0; - } + p5s = interp->dtoa.p5s; for(;;) { + assert(p5s != interp->dtoa.p5s + Bigint_Pow5size); + p5 = *p5s; + p5s++; if (k & 1) { b1 = mult(b, p5); Bfree(b); @@ -707,17 +707,6 @@ pow5mult(Bigint *b, int k) } if (!(k >>= 1)) break; - p51 = p5->next; - if (!p51) { - p51 = mult(p5,p5); - if (p51 == NULL) { - Bfree(b); - return NULL; - } - p51->next = 0; - p5->next = p51; - } - p5 = p51; } return b; } @@ -1416,7 +1405,7 @@ _Py_dg_strtod(const char *s00, char **se) switch (c) { case '-': sign = 1; - /* fall through */ + _Py_FALLTHROUGH; case '+': c = *++s; } @@ -1485,7 +1474,7 @@ _Py_dg_strtod(const char *s00, char **se) switch (c) { case '-': esign = 1; - /* fall through */ + _Py_FALLTHROUGH; case '+': c = *++s; } @@ -2373,7 +2362,7 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, break; case 2: leftright = 0; - /* fall through */ + _Py_FALLTHROUGH; case 4: if (ndigits <= 0) ndigits = 1; @@ -2381,7 +2370,7 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, break; case 3: leftright = 0; - /* fall through */ + _Py_FALLTHROUGH; case 5: i = ndigits + k + 1; ilim = i; @@ -2811,3 +2800,42 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, } #endif // _PY_SHORT_FLOAT_REPR == 1 + +PyStatus +_PyDtoa_Init(PyInterpreterState *interp) +{ +#if _PY_SHORT_FLOAT_REPR == 1 && !defined(Py_USING_MEMORY_DEBUGGER) + Bigint **p5s = interp->dtoa.p5s; + + // 5**4 = 625 + Bigint *p5 = i2b(625); + if (p5 == NULL) { + return PyStatus_NoMemory(); + } + p5s[0] = p5; + + // compute 5**8, 5**16, 5**32, ..., 5**512 + for (Py_ssize_t i = 1; i < Bigint_Pow5size; i++) { + p5 = mult(p5, p5); + if (p5 == NULL) { + return PyStatus_NoMemory(); + } + p5s[i] = p5; + } + +#endif + return PyStatus_Ok(); +} + +void +_PyDtoa_Fini(PyInterpreterState *interp) +{ +#if _PY_SHORT_FLOAT_REPR == 1 && !defined(Py_USING_MEMORY_DEBUGGER) + Bigint **p5s = interp->dtoa.p5s; + for (Py_ssize_t i = 0; i < Bigint_Pow5size; i++) { + Bigint *p5 = p5s[i]; + p5s[i] = NULL; + Bfree(p5); + } +#endif +} diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 2a80ec4f18d757..960c6b4a2ef995 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -10,7 +10,17 @@ * https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/type-reflection/index.js */ EM_JS(int, _PyEM_detect_type_reflection, (), { - return "Function" in WebAssembly; + if (!("Function" in WebAssembly)) { + return false; + } + if (WebAssembly.Function.type) { + // Node v20 + Module.PyEM_CountArgs = (func) => WebAssembly.Function.type(wasmTable.get(func)).parameters.length; + } else { + // Node >= 22, v8-based browsers + Module.PyEM_CountArgs = (func) => wasmTable.get(func).type().parameters.length; + } + return true; }); void @@ -43,7 +53,7 @@ EM_JS(int, _PyEM_CountFuncParams, (PyCFunctionWithKeywords func), if (n !== undefined) { return n; } - n = WebAssembly.Function.type(wasmTable.get(func)).parameters.length; + n = Module.PyEM_CountArgs(func); _PyEM_CountFuncParams.cache.set(func, n); return n; } diff --git a/Python/errors.c b/Python/errors.c index 15af39b10dc07e..ad6b7dbef075cc 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -121,11 +121,11 @@ _PyErr_GetTopmostException(PyThreadState *tstate) _PyErr_StackItem *exc_info = tstate->exc_info; assert(exc_info); - while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && - exc_info->previous_item != NULL) + while (exc_info->exc_value == NULL && exc_info->previous_item != NULL) { exc_info = exc_info->previous_item; } + assert(!Py_IsNone(exc_info->exc_value)); return exc_info; } @@ -257,13 +257,14 @@ void _PyErr_SetKeyError(PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *tup = PyTuple_Pack(1, arg); - if (!tup) { + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, arg); + if (!exc) { /* caller will expect error to be set anyway */ return; } - _PyErr_SetObject(tstate, PyExc_KeyError, tup); - Py_DECREF(tup); + + _PyErr_SetObject(tstate, (PyObject*)Py_TYPE(exc), exc); + Py_DECREF(exc); } void @@ -592,7 +593,7 @@ PyErr_GetHandledException(void) void _PyErr_SetHandledException(PyThreadState *tstate, PyObject *exc) { - Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc)); + Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc == Py_None ? NULL : exc)); } void @@ -632,8 +633,8 @@ _PyErr_StackItemToExcInfoTuple(_PyErr_StackItem *err_info) PyObject *exc_type = get_exc_type(exc_value); PyObject *exc_traceback = get_exc_traceback(exc_value); - return Py_BuildValue( - "(OOO)", + return PyTuple_Pack( + 3, exc_type ? exc_type : Py_None, exc_value ? exc_value : Py_None, exc_traceback ? exc_traceback : Py_None); @@ -1569,14 +1570,16 @@ _PyErr_WriteUnraisableDefaultHook(PyObject *args) for Python to handle it. For example, when a destructor raises an exception or during garbage collection (gc.collect()). - If err_msg_str is non-NULL, the error message is formatted as: - "Exception ignored %s" % err_msg_str. Otherwise, use "Exception ignored in" - error message. + If format is non-NULL, the error message is formatted using format and + variable arguments as in PyUnicode_FromFormat(). + Otherwise, use "Exception ignored in" error message. An exception must be set when calling this function. */ -void -_PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) + +static void +format_unraisable_v(const char *format, va_list va, PyObject *obj) { + const char *err_msg_str; PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); @@ -1610,8 +1613,8 @@ _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) } } - if (err_msg_str != NULL) { - err_msg = PyUnicode_FromFormat("Exception ignored %s", err_msg_str); + if (format != NULL) { + err_msg = PyUnicode_FromFormatV(format, va); if (err_msg == NULL) { PyErr_Clear(); } @@ -1676,11 +1679,30 @@ _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) _PyErr_Clear(tstate); /* Just in case */ } +void +PyErr_FormatUnraisable(const char *format, ...) +{ + va_list va; + + va_start(va, format); + format_unraisable_v(format, va, NULL); + va_end(va); +} + +static void +format_unraisable(PyObject *obj, const char *format, ...) +{ + va_list va; + + va_start(va, format); + format_unraisable_v(format, va, obj); + va_end(va); +} void PyErr_WriteUnraisable(PyObject *obj) { - _PyErr_WriteUnraisableMsg(NULL, obj); + format_unraisable(obj, NULL); } diff --git a/Python/executor.c b/Python/executor.c deleted file mode 100644 index 1630f018626449..00000000000000 --- a/Python/executor.c +++ /dev/null @@ -1,147 +0,0 @@ -#include "Python.h" - -#include "opcode.h" - -#include "pycore_bitutils.h" -#include "pycore_call.h" -#include "pycore_ceval.h" -#include "pycore_dict.h" -#include "pycore_emscripten_signal.h" -#include "pycore_intrinsics.h" -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_opcode_metadata.h" -#include "pycore_opcode_utils.h" -#include "pycore_pyerrors.h" -#include "pycore_range.h" -#include "pycore_setobject.h" // _PySet_Update() -#include "pycore_sliceobject.h" -#include "pycore_uops.h" - -#define TIER_TWO 2 -#include "ceval_macros.h" - - -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - goto deoptimize; \ - } - -#ifdef Py_STATS -// Disable these macros that apply to Tier 1 stats when we are in Tier 2 -#undef STAT_INC -#define STAT_INC(opname, name) ((void)0) -#undef STAT_DEC -#define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) -#endif - -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 - - -_PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ -#ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } - #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { printf(__VA_ARGS__); } -#else - #define DPRINTF(level, ...) -#endif - - DPRINTF(3, - "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n", - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), - _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->prev_instr + 1 - - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); - - PyThreadState *tstate = _PyThreadState_GET(); - _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; - - CHECK_EVAL_BREAKER(); - - OPT_STAT_INC(traces_executed); - _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - int pc = 0; - int opcode; - int oparg; - uint64_t operand; -#ifdef Py_STATS - uint64_t trace_uop_execution_counter = 0; -#endif - - for (;;) { - opcode = self->trace[pc].opcode; - oparg = self->trace[pc].oparg; - operand = self->trace[pc].operand; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", - pc, - opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], - oparg, - operand, - (int)(stack_pointer - _PyFrame_Stackbase(frame))); - pc++; - OPT_STAT_INC(uops_executed); - UOP_EXE_INC(opcode); -#ifdef Py_STATS - trace_uop_execution_counter++; -#endif - - switch (opcode) { - -#include "executor_cases.c.h" - - default: - { - fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); - Py_FatalError("Unknown uop"); - } - - } - } - -unbound_local_error: - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from tstate->current_frame. - DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return NULL; - -deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->prev_instr--; // Back up to just before destination - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return frame; -} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 119e77b9369b13..8f6bc75b528d9b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1,770 +1,1171 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier2_generator.py // from: // Python/bytecodes.c // Do not edit! - case NOP: { +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 + + case _NOP: { break; } - case RESUME_CHECK: { -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + case _RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case LOAD_FAST_CHECK: { - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */ + + case _LOAD_FAST_CHECK: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_FAST: { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + case _LOAD_FAST_0: { + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_1: { + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_2: { + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_3: { + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_4: { + _PyStackRef value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_5: { + _PyStackRef value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_6: { + _PyStackRef value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_7: { + _PyStackRef value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_FAST_AND_CLEAR: { - PyObject *value; + case _LOAD_FAST_AND_CLEAR: { + _PyStackRef value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_CONST: { - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + case _LOAD_CONST: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_FAST: { - PyObject *value; + case _STORE_FAST_0: { + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case POP_TOP: { - PyObject *value; + case _STORE_FAST_1: { + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - Py_DECREF(value); - STACK_SHRINK(1); + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case PUSH_NULL: { - PyObject *res; - res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + case _STORE_FAST_2: { + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST_3: { + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST_4: { + _PyStackRef value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST_5: { + _PyStackRef value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST_6: { + _PyStackRef value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST_7: { + _PyStackRef value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case END_SEND: { - PyObject *value; - PyObject *receiver; + case _POP_TOP: { + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL: { + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_SEND: { + _PyStackRef value; + _PyStackRef receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + (void)receiver; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case UNARY_NEGATIVE: { - PyObject *value; - PyObject *res; + case _UNARY_NEGATIVE: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } - case UNARY_NOT: { - PyObject *value; - PyObject *res; + case _UNARY_NOT: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } - case TO_BOOL: { - PyObject *value; - PyObject *res; + case _TO_BOOL: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (err < 0) JUMP_TO_ERROR(); + res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } - case TO_BOOL_BOOL: { - PyObject *value; + case _TO_BOOL_BOOL: { + _PyStackRef value; value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + if (!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); break; } - case TO_BOOL_INT: { - PyObject *value; - PyObject *res; + case _TO_BOOL_INT: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } - case TO_BOOL_LIST: { - PyObject *value; - PyObject *res; + case _TO_BOOL_LIST: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyList_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); stack_pointer[-1] = res; break; } - case TO_BOOL_NONE: { - PyObject *value; - PyObject *res; + case _TO_BOOL_NONE: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + if (!PyStackRef_Is(value, PyStackRef_None)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; stack_pointer[-1] = res; break; } - case TO_BOOL_STR: { - PyObject *value; - PyObject *res; + case _TO_BOOL_STR: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } - case TO_BOOL_ALWAYS_TRUE: { - PyObject *value; - PyObject *res; + case _REPLACE_WITH_TRUE: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)operand; - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; stack_pointer[-1] = res; break; } - case UNARY_INVERT: { - PyObject *value; - PyObject *res; + case _UNARY_INVERT: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); - DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_INT: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_INT: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_ADD_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_BOTH_FLOAT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); - DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_FLOAT: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_FLOAT: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_ADD_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_BOTH_UNICODE: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); - DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_ADD_UNICODE: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case BINARY_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *res; + case _BINARY_SUBSCR: { + _PyStackRef sub; + _PyStackRef container; + _PyStackRef res; sub = stack_pointer[-1]; container = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case BINARY_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SLICE: { + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; + PyStackRef_CLOSE(container); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; + case _STORE_SLICE: { + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - STACK_SHRINK(4); - break; - } - - case BINARY_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) JUMP_TO_ERROR(); + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_LIST_INT: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case BINARY_SUBSCR_STR_INT: { - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_STR_INT: { + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(str)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_TUPLE_INT: { + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyTuple_CheckExact(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case BINARY_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_DICT: { + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (true) goto pop_2_error; - } - Py_INCREF(res); // Do this before DECREF'ing dict, sub - Py_DECREF(dict); - Py_DECREF(sub); - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) JUMP_TO_ERROR(); + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LIST_APPEND: { - PyObject *v; - PyObject *list; + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _LIST_APPEND: { + _PyStackRef v; + _PyStackRef list; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case SET_ADD: { - PyObject *v; - PyObject *set; + case _SET_ADD: { + _PyStackRef v; + _PyStackRef set; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *v; + case _STORE_SUBSCR: { + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; sub = stack_pointer[-1]; container = stack_pointer[-2]; v = stack_pointer[-3]; - #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; - STACK_SHRINK(3); - break; - } - - case STORE_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) JUMP_TO_ERROR(); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(3); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + case _STORE_SUBSCR_DICT: { + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - STACK_SHRINK(3); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); + if (err) JUMP_TO_ERROR(); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } - case DELETE_SUBSCR: { - PyObject *sub; - PyObject *container; + case _DELETE_SUBSCR: { + _PyStackRef sub; + _PyStackRef container; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - STACK_SHRINK(2); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) JUMP_TO_ERROR(); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case CALL_INTRINSIC_1: { - PyObject *value; - PyObject *res; + case _CALL_INTRINSIC_1: { + _PyStackRef value; + _PyStackRef res; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } - case CALL_INTRINSIC_2: { - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; + case _CALL_INTRINSIC_2: { + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; + oparg = CURRENT_OPARG(); + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case _POP_FRAME: { - PyObject *retval; + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_VALUE: { + _PyStackRef retval; + _PyStackRef res; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case GET_AITER: { - PyObject *obj; - PyObject *iter; + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ + + case _GET_AITER: { + _PyStackRef obj; + _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; + PyStackRef_CLOSE(obj); + if (true) JUMP_TO_ERROR(); } - - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + iter_o = (*getter)(obj_o); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) JUMP_TO_ERROR(); + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + if (true) JUMP_TO_ERROR(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } - case GET_ANEXT: { - PyObject *aiter; - PyObject *awaitable; + case _GET_ANEXT: { + _PyStackRef aiter; + _PyStackRef awaitable; aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - goto error; + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { + JUMP_TO_ERROR(); } } else { if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { - goto error; + JUMP_TO_ERROR(); } } else { @@ -772,91 +1173,160 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - goto error; + JUMP_TO_ERROR(); } - - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - goto error; + JUMP_TO_ERROR(); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case GET_AWAITABLE: { - PyObject *iterable; - PyObject *iter; + case _GET_AWAITABLE: { + _PyStackRef iterable; + _PyStackRef iter; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); } - - Py_DECREF(iterable); - - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + PyStackRef_CLOSE(iterable); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - - if (iter == NULL) goto pop_1_error; + if (iter_o == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } - case POP_EXCEPT: { - PyObject *exc_value; + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + + case _YIELD_VALUE: { + _PyStackRef retval; + _PyStackRef value; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_EXCEPT: { + _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + Py_XSETREF(exc_info->exc_value, + PyStackRef_Is(exc_value, PyStackRef_None) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_ASSERTION_ERROR: { - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + case _LOAD_COMMON_CONSTANT: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_BUILD_CLASS: { - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { + case _LOAD_BUILD_CLASS: { + _PyStackRef bc; + PyObject *bc_o; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) JUMP_TO_ERROR(); + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error; + if (true) JUMP_TO_ERROR(); } - STACK_GROW(1); - stack_pointer[-1] = bc; + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_NAME: { - PyObject *v; + case _STORE_NAME: { + _PyStackRef v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -864,455 +1334,445 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; + PyStackRef_CLOSE(v); + if (true) JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case DELETE_NAME: { + case _DELETE_NAME: { + oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; + JUMP_TO_ERROR(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - goto error; + NAME_ERROR_MSG, + name); + JUMP_TO_ERROR(); } break; } - case UNPACK_SEQUENCE: { - PyObject *seq; + case _UNPACK_SEQUENCE: { + _PyStackRef seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq; - PyObject **values; + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + PyStackRef_CLOSE(seq); + if (res == 0) JUMP_TO_ERROR(); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE: { + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case UNPACK_SEQUENCE_TUPLE: { - PyObject *seq; - PyObject **values; + case _UNPACK_SEQUENCE_TUPLE: { + _PyStackRef seq; + _PyStackRef *values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case UNPACK_SEQUENCE_LIST: { - PyObject *seq; - PyObject **values; + case _UNPACK_SEQUENCE_LIST: { + _PyStackRef seq; + _PyStackRef *values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyList_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case UNPACK_EX: { - PyObject *seq; + case _UNPACK_EX: { + _PyStackRef seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + PyStackRef_CLOSE(seq); + if (res == 0) JUMP_TO_ERROR(); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_ATTR: { - PyObject *owner; - PyObject *v; + case _STORE_ATTR: { + _PyStackRef owner; + _PyStackRef v; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; v = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; - STACK_SHRINK(2); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) JUMP_TO_ERROR(); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case DELETE_ATTR: { - PyObject *owner; + case _DELETE_ATTR: { + _PyStackRef owner; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - STACK_SHRINK(1); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (err) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_GLOBAL: { - PyObject *v; + case _STORE_GLOBAL: { + _PyStackRef v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case DELETE_GLOBAL: { + case _DELETE_GLOBAL: { + oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); + int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - goto error; + if (err < 0) { + JUMP_TO_ERROR(); + } + if (err == 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + JUMP_TO_ERROR(); } break; } - case LOAD_LOCALS: { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { + case _LOAD_LOCALS: { + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; + if (true) JUMP_TO_ERROR(); } - Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + locals = PyStackRef_FromPyObjectNew(l);; + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; - } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; - } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + + /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + + case _LOAD_GLOBAL: { + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } - if (v == NULL) { + if (true) JUMP_TO_ERROR(); + } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) JUMP_TO_ERROR(); + if (res_o == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) JUMP_TO_ERROR(); + if (res_o == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) JUMP_TO_ERROR(); } } } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - break; - } - - case LOAD_NAME: { - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; - } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { - goto error; - } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } - } - } - STACK_GROW(1); - stack_pointer[-1] = v; - break; - } - - case LOAD_GLOBAL: { - PyObject *res; - PyObject *null = NULL; - #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)operand; + uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } case _GUARD_BUILTINS_VERSION: { - uint16_t version = (uint16_t)operand; + uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } case _LOAD_GLOBAL_MODULE: { - PyObject *res; - PyObject *null = NULL; - uint16_t index = (uint16_t)operand; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS: { - PyObject *res; - PyObject *null = NULL; - uint16_t index = (uint16_t)operand; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_FAST: { + oparg = CURRENT_OPARG(); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR(); + } + SETLOCAL(oparg, PyStackRef_NULL); break; } - case DELETE_FAST: { - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + case _MAKE_CELL: { + oparg = CURRENT_OPARG(); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + JUMP_TO_ERROR(); + } + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); break; } - case DELETE_DEREF: { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + case _DELETE_DEREF: { + oparg = CURRENT_OPARG(); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + JUMP_TO_ERROR(); } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); break; } - case LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; + case _LOAD_FROM_DICT_OR_DEREF: { + _PyStackRef class_dict_st; + _PyStackRef value; + oparg = CURRENT_OPARG(); + class_dict_st = stack_pointer[-1]; + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - Py_DECREF(class_dict); - goto error; - } - Py_DECREF(class_dict); - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + JUMP_TO_ERROR(); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + JUMP_TO_ERROR(); } - Py_INCREF(value); } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[-1] = value; break; } - case LOAD_DEREF: { - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + case _LOAD_DEREF: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + if (true) JUMP_TO_ERROR(); } - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case STORE_DEREF: { - PyObject *v; + case _STORE_DEREF: { + _PyStackRef v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - STACK_SHRINK(1); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case COPY_FREE_VARS: { + case _COPY_FREE_VARS: { + oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyFunction_Check(frame->f_funcobj)); @@ -1321,331 +1781,368 @@ int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } break; } - case BUILD_STRING: { - PyObject **pieces; - PyObject *str; - pieces = stack_pointer - oparg; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + case _BUILD_STRING: { + _PyStackRef *pieces; + _PyStackRef str; + oparg = CURRENT_OPARG(); + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + PyStackRef_CLOSE(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; - break; - } - - case BUILD_TUPLE: { - PyObject **values; - PyObject *tup; - values = stack_pointer - oparg; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; - break; - } - - case BUILD_LIST: { - PyObject **values; - PyObject *list; - values = stack_pointer - oparg; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; - break; - } - - case LIST_EXTEND: { - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + if (str_o == NULL) JUMP_TO_ERROR(); + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_TUPLE: { + _PyStackRef *values; + _PyStackRef tup; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) JUMP_TO_ERROR(); + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_LIST: { + _PyStackRef *values; + _PyStackRef list; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (list_o == NULL) JUMP_TO_ERROR(); + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_EXTEND: { + _PyStackRef iterable_st; + _PyStackRef list_st; + oparg = CURRENT_OPARG(); + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); - if (true) goto pop_1_error; + PyStackRef_CLOSE(iterable_st); + if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - STACK_SHRINK(1); + PyStackRef_CLOSE(iterable_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case SET_UPDATE: { - PyObject *iterable; - PyObject *set; + case _SET_UPDATE: { + _PyStackRef iterable; + _PyStackRef set; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - STACK_SHRINK(1); - break; - } - - case BUILD_SET: { - PyObject **values; - PyObject *set; - values = stack_pointer - oparg; - set = PySet_New(NULL); - if (set == NULL) - goto error; - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + if (err < 0) JUMP_TO_ERROR(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + + case _BUILD_MAP: { + _PyStackRef *values; + _PyStackRef map; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; - break; - } - - case BUILD_MAP: { - PyObject **values; - PyObject *map; - values = stack_pointer - oparg*2; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map_o == NULL) JUMP_TO_ERROR(); + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); break; } - case SETUP_ANNOTATIONS: { + case _SETUP_ANNOTATIONS: { int err; PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error; + if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error; - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) JUMP_TO_ERROR(); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) JUMP_TO_ERROR(); } else { - /* do the same if locals() is not a dict */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } + Py_DECREF(ann_dict); } break; } - case BUILD_CONST_KEY_MAP: { - PyObject *keys; - PyObject **values; - PyObject *map; + case _BUILD_CONST_KEY_MAP: { + _PyStackRef keys; + _PyStackRef *values; + _PyStackRef map; + oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + values = &stack_pointer[-1 - oparg]; + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + PyStackRef_CLOSE(keys); + if (true) JUMP_TO_ERROR(); } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + PyStackRef_CLOSE(keys); + if (map_o == NULL) JUMP_TO_ERROR(); + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case DICT_UPDATE: { - PyObject *update; - PyObject *dict; + case _DICT_UPDATE: { + _PyStackRef update; + _PyStackRef dict; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); } - Py_DECREF(update); - if (true) goto pop_1_error; + PyStackRef_CLOSE(update); + if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); - STACK_SHRINK(1); + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case DICT_MERGE: { - PyObject *update; - PyObject *dict; - PyObject *callable; + case _DICT_MERGE: { + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + PyStackRef_CLOSE(update); + if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); - STACK_SHRINK(1); + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case MAP_ADD: { - PyObject *value; - PyObject *key; - PyObject *dict; + case _MAP_ADD: { + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); - break; - } - - case LOAD_SUPER_ATTR_ATTR: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) JUMP_TO_ERROR(); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ + + case _LOAD_SUPER_ATTR_ATTR: { + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; + oparg = CURRENT_OPARG(); + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; - break; - } - - case LOAD_SUPER_ATTR_METHOD: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) JUMP_TO_ERROR(); + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD: { + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; + oparg = CURRENT_OPARG(); + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); + if (true) JUMP_TO_ERROR(); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case LOAD_ATTR: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; + case _LOAD_ATTR: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. - meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -1653,529 +2150,828 @@ something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_TYPE_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - PyTypeObject *tp = Py_TYPE(owner); + uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner; + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner_o)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_0: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + case _LOAD_ATTR_INSTANCE_VALUE_1: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } + /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ + case _CHECK_ATTR_MODULE: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyModule_CheckExact(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + if (dict->ma_keys->dk_version != dict_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_ATTR_MODULE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); + PyObject *attr_o = ep->me_value; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_ATTR_WITH_HINT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(PyDict_CheckExact((PyObject *)dict)); break; } case _LOAD_ATTR_WITH_HINT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); + uint16_t hint = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); - attr = ep->me_value; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); - attr = ep->me_value; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + attr_o = ep->me_value; + } + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_SLOT_0: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_SLOT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + case _LOAD_ATTR_SLOT_1: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } + /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ + case _CHECK_ATTR_CLASS: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); + uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); + if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _LOAD_ATTR_CLASS_0: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_CLASS: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + case _LOAD_ATTR_CLASS_1: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _GUARD_DORV_VALUES: { - PyObject *owner; + /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ + + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _GUARD_DORV_NO_DICT: { + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (_PyObject_InlineValues(owner_o)->valid == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _STORE_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - Py_DECREF(owner); - STACK_SHRINK(2); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_WITH_HINT: { + _PyStackRef owner; + _PyStackRef value; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint16_t hint = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + old_value = ep->me_value; + if (old_value == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + old_value = ep->me_value; + if (old_value == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR_SLOT: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; - char *addr = (char *)owner + index; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); - STACK_SHRINK(2); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case COMPARE_OP: { - PyObject *right; - PyObject *left; - PyObject *res; + case _COMPARE_OP: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) JUMP_TO_ERROR(); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) JUMP_TO_ERROR(); + res = res_bool ? PyStackRef_True : PyStackRef_False; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case COMPARE_OP_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + case _COMPARE_OP_FLOAT: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case COMPARE_OP_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + case _COMPARE_OP_INT: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case COMPARE_OP_STR: { - PyObject *right; - PyObject *left; - PyObject *res; + case _COMPARE_OP_STR: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case IS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + case _IS_OP: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + #ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + #else + int res = PyStackRef_Is(left, right) ^ oparg; + #endif + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case CONTAINS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + case _CONTAINS_OP: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; - break; - } - - case CHECK_EG_MATCH: { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_SET: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + int res = _PySet_Contains((PySetObject *)right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyDict_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CONTAINS_OP, hit); + int res = PyDict_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_EG_MATCH: { + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (true) JUMP_TO_ERROR(); } - - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + &match_o, &rest_o); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) JUMP_TO_ERROR(); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) JUMP_TO_ERROR(); + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } - case CHECK_EXC_MATCH: { - PyObject *right; - PyObject *left; - PyObject *b; + case _CHECK_EXC_MATCH: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { + PyStackRef_CLOSE(right); + if (true) JUMP_TO_ERROR(); } - - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; + int res = PyErr_GivenExceptionMatches(left_o, right_o); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; break; } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ + case _IS_NONE: { - PyObject *value; - PyObject *b; + _PyStackRef value; + _PyStackRef b; value = stack_pointer[-1]; - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } stack_pointer[-1] = b; break; } - case GET_LEN: { - PyObject *obj; - PyObject *len_o; + case _GET_LEN: { + _PyStackRef obj; + _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; - break; - } - - case MATCH_CLASS: { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + if (len_i < 0) JUMP_TO_ERROR(); + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) JUMP_TO_ERROR(); + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_CLASS: { + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; + oparg = CURRENT_OPARG(); names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - attrs = Py_None; // Failure! + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); + // Error! + attrs = PyStackRef_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case MATCH_MAPPING: { - PyObject *subject; - PyObject *res; + case _MATCH_MAPPING: { + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case MATCH_SEQUENCE: { - PyObject *subject; - PyObject *res; + case _MATCH_SEQUENCE: { + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case MATCH_KEYS: { - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; + case _MATCH_KEYS: { + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + if (values_or_none_o == NULL) JUMP_TO_ERROR(); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case GET_ITER: { - PyObject *iterable; - PyObject *iter; + case _GET_ITER: { + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + PyStackRef_CLOSE(iterable); + if (PyStackRef_IsNull(iter)) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } - case GET_YIELD_FROM_ITER: { - PyObject *iterable; - PyObject *iter; + case _GET_YIELD_FROM_ITER: { + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -2183,993 +2979,1670 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - goto error; + JUMP_TO_ERROR(); } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - goto error; + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { + JUMP_TO_ERROR(); } - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; break; } + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ + + case _FOR_ITER_TIER_TWO: { + _PyStackRef iter; + _PyStackRef next; + iter = stack_pointer[-1]; + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + JUMP_TO_ERROR(); + } + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + /* The translator sets the deopt target just past the matching END_FOR */ + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ + case _ITER_CHECK_LIST: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _IS_ITER_EXHAUSTED_LIST: { - PyObject *iter; - PyObject *exhausted; + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_LIST: { + _PyStackRef iter; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; if (seq == NULL) { - exhausted = Py_True; - } - else if (it->it_index >= PyList_GET_SIZE(seq)) { - Py_DECREF(seq); - it->it_seq = NULL; - exhausted = Py_True; + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - else { - exhausted = Py_False; + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } - STACK_GROW(1); - stack_pointer[-1] = exhausted; break; } case _ITER_NEXT_LIST: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _ITER_CHECK_TUPLE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _IS_ITER_EXHAUSTED_TUPLE: { - PyObject *iter; - PyObject *exhausted; + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_TUPLE: { + _PyStackRef iter; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; if (seq == NULL) { - exhausted = Py_True; + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - else if (it->it_index >= PyTuple_GET_SIZE(seq)) { - Py_DECREF(seq); - it->it_seq = NULL; - exhausted = Py_True; - } - else { - exhausted = Py_False; + if (it->it_index >= PyTuple_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - STACK_GROW(1); - stack_pointer[-1] = exhausted; break; } case _ITER_NEXT_TUPLE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _ITER_CHECK_RANGE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _IS_ITER_EXHAUSTED_RANGE: { - PyObject *iter; - PyObject *exhausted; + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_RANGE: { + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - exhausted = r->len <= 0 ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = exhausted; + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _ITER_NEXT_RANGE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = next; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) JUMP_TO_ERROR(); + next = PyStackRef_FromPyObjectSteal(res); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _FOR_ITER_GEN_FRAME: { + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + oparg = CURRENT_OPARG(); + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); + stack_pointer[0].bits = (uintptr_t)gen_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SPECIAL: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + } + } + if (PyStackRef_IsNull(attr)) JUMP_TO_ERROR(); + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case WITH_EXCEPT_START: { - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; + case _WITH_EXCEPT_START: { + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case PUSH_EXC_INFO: { - PyObject *new_exc; - PyObject *prev_exc; + case _PUSH_EXC_INFO: { + _PyStackRef new_exc; + _PyStackRef prev_exc; new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner_o)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _GUARD_KEYS_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)operand; - PyTypeObject *owner_cls = Py_TYPE(owner); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_ATTR_METHOD_WITH_VALUES: { - PyObject *owner; - PyObject *attr; - PyObject *self; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _CHECK_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *null; - PyObject *callable; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - break; - } + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *callable; - PyObject *func; - PyObject *self; - callable = stack_pointer[-2 - oparg]; - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; - break; - } + /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); + case _CHECK_PERIODIC: { + CHECK_EVAL_BREAKER(); break; } - case _CHECK_FUNCTION_EXACT_ARGS: { - PyObject *self_or_null; - PyObject *callable; + case _PY_FRAME_GENERAL: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)operand; - DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION: { + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD: { + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_NON_PY_GENERAL: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _PyStackRef callable; + _PyStackRef func; + _PyStackRef self; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_PEP_523: { + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_STACK_SPACE: { - PyObject *callable; + _PyStackRef callable; + oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); - DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _INIT_CALL_PY_EXACT_ARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + case _INIT_CALL_PY_EXACT_ARGS_0: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; - args = stack_pointer - oparg; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_1: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case _PUSH_FRAME: { + case _INIT_CALL_PY_EXACT_ARGS_2: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; - STACK_SHRINK(1); - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } -#endif + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case CALL_TYPE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; + case _INIT_CALL_PY_EXACT_ARGS_3: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case CALL_STR_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; + case _INIT_CALL_PY_EXACT_ARGS_4: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case CALL_TUPLE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; + case _INIT_CALL_PY_EXACT_ARGS: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + break; + } + + case _CALL_TYPE_1: { + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_STR_1: { + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_TUPLE_1: { + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - case EXIT_INIT_CHECK: { - PyObject *should_be_none; + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _EXIT_INIT_CHECK: { + _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - goto error; + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + JUMP_TO_ERROR(); } - STACK_SHRINK(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + case _CALL_BUILTIN_CLASS: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_vectorcall == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_O: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_LEN: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LEN: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - goto error; + JUMP_TO_ERROR(); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case CALL_ISINSTANCE: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + GOTO_ERROR(error); + } + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_ISINSTANCE: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { - goto error; + JUMP_TO_ERROR(); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (meth->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + if (!Py_IS_TYPE(self, d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (meth->ml_flags != METH_NOARGS) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (meth->ml_flags != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } - case MAKE_FUNCTION: { - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); + /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ - Py_DECREF(codeobj); + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _MAKE_FUNCTION: { + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { - goto error; + JUMP_TO_ERROR(); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[-1] = func; break; } - case SET_FUNCTION_ATTRIBUTE: { - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; + case _SET_FUNCTION_ATTRIBUTE: { + _PyStackRef func_st; + _PyStackRef attr_st; + oparg = CURRENT_OPARG(); + func_st = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; - break; - } - - case BUILD_SLICE: { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + stack_pointer[-2] = func_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case CONVERT_VALUE: { - PyObject *value; - PyObject *result; + case _RETURN_GENERATOR: { + _PyStackRef res; + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + JUMP_TO_ERROR(); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + LOAD_SP(); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SLICE: { + _PyStackRef step = PyStackRef_NULL; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef slice; + oparg = CURRENT_OPARG(); + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) JUMP_TO_ERROR(); + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONVERT_VALUE: { + _PyStackRef value; + _PyStackRef result; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; + conv_fn = _PyEval_ConversionFuncs[oparg]; + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (result_o == NULL) JUMP_TO_ERROR(); + result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; break; } - case FORMAT_SIMPLE: { - PyObject *value; - PyObject *res; + case _FORMAT_SIMPLE: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); } else { res = value; @@ -3178,63 +4651,61 @@ break; } - case FORMAT_WITH_SPEC: { - PyObject *fmt_spec; - PyObject *value; - PyObject *res; + case _FORMAT_WITH_SPEC: { + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case COPY: { - PyObject *bottom; - PyObject *top; + case _COPY: { + _PyStackRef bottom; + _PyStackRef top; + oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); - top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case BINARY_OP: { - PyObject *rhs; - PyObject *lhs; - PyObject *res; + case _BINARY_OP: { + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; + oparg = CURRENT_OPARG(); rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case SWAP: { - PyObject *top; - PyObject *bottom; + case _SWAP: { + _PyStackRef top; + _PyStackRef bottom; + oparg = CURRENT_OPARG(); top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); @@ -3243,52 +4714,349 @@ break; } - case _POP_JUMP_IF_FALSE: { - PyObject *flag; + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ + + case _GUARD_IS_TRUE_POP: { + _PyStackRef flag; flag = stack_pointer[-1]; - if (Py_IsFalse(flag)) { - pc = oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (!PyStackRef_Is(flag, PyStackRef_True)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - STACK_SHRINK(1); + assert(PyStackRef_Is(flag, PyStackRef_True)); break; } - case _POP_JUMP_IF_TRUE: { - PyObject *flag; + case _GUARD_IS_FALSE_POP: { + _PyStackRef flag; flag = stack_pointer[-1]; - if (Py_IsTrue(flag)) { - pc = oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (!PyStackRef_Is(flag, PyStackRef_False)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_Is(flag, PyStackRef_False)); + break; + } + + case _GUARD_IS_NONE_POP: { + _PyStackRef val; + val = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (!PyStackRef_Is(val, PyStackRef_None)) { + PyStackRef_CLOSE(val); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } - STACK_SHRINK(1); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + _PyStackRef val; + val = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (PyStackRef_Is(val, PyStackRef_None)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyStackRef_CLOSE(val); break; } case _JUMP_TO_TOP: { - pc = 0; - CHECK_EVAL_BREAKER(); + JUMP_TO_JUMP_TARGET(); break; } case _SET_IP: { - TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + break; + } + + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)CURRENT_OPERAND(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _SAVE_RETURN_OFFSET: { + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif break; } case _EXIT_TRACE: { - frame->prev_instr--; // Back up to just before destination - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); + oparg = CURRENT_OPARG(); + _PyExitData *exit = ¤t_executor->exits[oparg]; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - return frame; + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + oparg, exit->temperature.as_counter, + (int)(target - _PyCode_CODE(code)), + _PyOpcode_OpName[target->op.code]); + } + #endif + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + if (optimized < 0) { + Py_DECREF(current_executor); + tstate->previous_executor = Py_None; + GOTO_UNWIND(); + } + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + } + exit->executor = executor; + } + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); break; } - case _INSERT: { - PyObject *top; - top = stack_pointer[-1]; - // Inserts TOS at position specified by oparg; - memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); - stack_pointer[-1 - oparg] = top; + case _CHECK_VALIDITY: { + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _LOAD_CONST_INLINE: { + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = PyStackRef_FromPyObjectNew(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = PyStackRef_FromPyObjectImmortal(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _PyStackRef pop; + _PyStackRef value; + pop = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); + stack_pointer[-1] = value; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + _PyStackRef value; + _PyStackRef null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = PyStackRef_FromPyObjectNew(ptr); + null = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + _PyStackRef value; + _PyStackRef null; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = PyStackRef_FromPyObjectImmortal(ptr); + null = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION: { + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + assert(PyFunction_Check(frame->f_funcobj)); + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + _PyStackRef opt; + opt = stack_pointer[-1]; + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); + exe->count++; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DYNAMIC_EXIT: { + oparg = CURRENT_OPARG(); + tstate->previous_executor = (PyObject *)current_executor; + _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _Py_CODEUNIT *target = frame->instr_ptr; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (lltrace >= 2) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + oparg, exit->temperature.as_counter, + (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), + _PyOpcode_OpName[target->op.code]); + } + #endif + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + if (!backoff_counter_triggers(exit->temperature)) { + exit->temperature = advance_backoff_counter(exit->temperature); + GOTO_TIER_ONE(target); + } + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + if (optimized < 0) { + Py_DECREF(current_executor); + tstate->previous_executor = Py_None; + GOTO_UNWIND(); + } + GOTO_TIER_ONE(target); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + GOTO_TIER_TWO(executor); + break; + } + + case _START_EXECUTOR: { + PyObject *executor = (PyObject *)CURRENT_OPERAND(); + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; + #ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; + #endif + if (!((_PyExecutorObject *)executor)->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _FATAL_ERROR: { + assert(0); + Py_FatalError("Fatal error uop executed."); + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + break; + } + + case _DEOPT: { + EXIT_TO_TIER1(); break; } + + case _ERROR_POP_N: { + oparg = CURRENT_OPARG(); + uint32_t target = (uint32_t)CURRENT_OPERAND(); + frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + GOTO_UNWIND(); + break; + } + + case _TIER2_RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + break; + } + +#undef TIER_TWO diff --git a/Python/fileutils.c b/Python/fileutils.c index 17a4ae56ef0528..c9ae1b3f54e167 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1239,6 +1239,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) BY_HANDLE_FILE_INFORMATION info; FILE_BASIC_INFO basicInfo; FILE_ID_INFO idInfo; + FILE_ID_INFO *pIdInfo = &idInfo; HANDLE h; int type; @@ -1271,15 +1272,19 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) } if (!GetFileInformationByHandle(h, &info) || - !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo)) || - !GetFileInformationByHandleEx(h, FileIdInfo, &idInfo, sizeof(idInfo))) { + !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo))) { /* The Win32 error is already set, but we also set errno for callers who expect it */ errno = winerror_to_errno(GetLastError()); return -1; } - _Py_attribute_data_to_stat(&info, 0, &basicInfo, &idInfo, status); + if (!GetFileInformationByHandleEx(h, FileIdInfo, &idInfo, sizeof(idInfo))) { + /* Failed to get FileIdInfo, so do not pass it along */ + pIdInfo = NULL; + } + + _Py_attribute_data_to_stat(&info, 0, &basicInfo, pIdInfo, status); return 0; #else return fstat(fd, status); @@ -1497,7 +1502,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) #else #if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) - if (ioctl_works != 0 && raise != 0) { + if (raise != 0 && _Py_atomic_load_int_relaxed(&ioctl_works) != 0) { /* fast-path: ioctl() only requires one syscall */ /* caveat: raise=0 is an indicator that we must be async-signal-safe * thus avoid using ioctl() so we skip the fast-path. */ @@ -1507,7 +1512,9 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) request = FIOCLEX; err = ioctl(fd, request, NULL); if (!err) { - ioctl_works = 1; + if (_Py_atomic_load_int_relaxed(&ioctl_works) == -1) { + _Py_atomic_store_int_relaxed(&ioctl_works, 1); + } return 0; } @@ -1534,7 +1541,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) with EACCES. While FIOCLEX is safe operation it may be unavailable because ioctl was denied altogether. This can be the case on Android. */ - ioctl_works = 0; + _Py_atomic_store_int_relaxed(&ioctl_works, 0); } /* fallback to fcntl() if ioctl() does not work */ } @@ -2290,6 +2297,99 @@ PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +void +_Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, + Py_ssize_t *rootsize) +{ + assert(drvsize); + assert(rootsize); +#ifndef MS_WINDOWS +#define IS_SEP(x) (*(x) == SEP) + *drvsize = 0; + if (!IS_SEP(&path[0])) { + // Relative path, e.g.: 'foo' + *rootsize = 0; + } + else if (!IS_SEP(&path[1]) || IS_SEP(&path[2])) { + // Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + *rootsize = 1; + } + else { + // Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + *rootsize = 2; + } +#undef IS_SEP +#else + const wchar_t *pEnd = size >= 0 ? &path[size] : NULL; +#define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) +#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) + if (IS_SEP(&path[0])) { + if (IS_SEP(&path[1])) { + // Device drives, e.g. \\.\device or \\?\device + // UNC drives, e.g. \\server\share or \\?\UNC\server\share + Py_ssize_t idx; + if (path[2] == L'?' && IS_SEP(&path[3]) && + (path[4] == L'U' || path[4] == L'u') && + (path[5] == L'N' || path[5] == L'n') && + (path[6] == L'C' || path[6] == L'c') && + IS_SEP(&path[7])) + { + idx = 8; + } + else { + idx = 2; + } + while (!SEP_OR_END(&path[idx])) { + idx++; + } + if (IS_END(&path[idx])) { + *drvsize = idx; + *rootsize = 0; + } + else { + idx++; + while (!SEP_OR_END(&path[idx])) { + idx++; + } + *drvsize = idx; + if (IS_END(&path[idx])) { + *rootsize = 0; + } + else { + *rootsize = 1; + } + } + } + else { + // Relative path with root, e.g. \Windows + *drvsize = 0; + *rootsize = 1; + } + } + else if (!IS_END(&path[0]) && path[1] == L':') { + *drvsize = 2; + if (IS_SEP(&path[2])) { + // Absolute drive-letter path, e.g. X:\Windows + *rootsize = 1; + } + else { + // Relative path with drive, e.g. X:Windows + *rootsize = 0; + } + } + else { + // Relative path, e.g. Windows + *drvsize = 0; + *rootsize = 0; + } +#undef SEP_OR_END +#undef IS_SEP +#undef IS_END +#endif +} + // The caller must ensure "buffer" is big enough. static int join_relfile(wchar_t *buffer, size_t bufsize, @@ -2406,49 +2506,39 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) - // Skip leading '.\' if (p1[0] == L'.' && IS_SEP(&p1[1])) { + // Skip leading '.\' path = &path[2]; - while (IS_SEP(path) && !IS_END(path)) { + while (IS_SEP(path)) { path++; } p1 = p2 = minP2 = path; lastC = SEP; } + else { + Py_ssize_t drvsize, rootsize; + _Py_skiproot(path, size, &drvsize, &rootsize); + if (drvsize || rootsize) { + // Skip past root and update minP2 + p1 = &path[drvsize + rootsize]; +#ifndef ALTSEP + p2 = p1; +#else + for (; p2 < p1; ++p2) { + if (*p2 == ALTSEP) { + *p2 = SEP; + } + } +#endif + minP2 = p2 - 1; + lastC = *minP2; #ifdef MS_WINDOWS - // Skip past drive segment and update minP2 - else if (p1[0] && p1[1] == L':') { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2; - lastC = L':'; - } - // Skip past all \\-prefixed paths, including \\?\, \\.\, - // and network paths, including the first segment. - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { - int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; - p1 += 2; - for (; !IS_END(p1) && sepCount; ++p1) { - if (IS_SEP(p1)) { - --sepCount; - *p2++ = lastC = SEP; - } else { - *p2++ = lastC = *p1; + if (lastC != SEP) { + minP2++; } +#endif } - minP2 = p2 - 1; } -#else - // Skip past two leading SEPs - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; - } -#endif /* MS_WINDOWS */ /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { @@ -2873,9 +2963,9 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, * non-opened fd in the middle. * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. */ -#ifdef __FreeBSD__ +#ifdef HAVE_CLOSEFROM # define USE_CLOSEFROM -#endif /* __FreeBSD__ */ +#endif /* HAVE_CLOSEFROM */ #ifdef HAVE_FDWALK # define USE_FDWALK @@ -2917,7 +3007,7 @@ _Py_closerange(int first, int last) #ifdef USE_CLOSEFROM if (last >= sysconf(_SC_OPEN_MAX)) { /* Any errors encountered while closing file descriptors are ignored */ - closefrom(first); + (void)closefrom(first); } else #endif /* USE_CLOSEFROM */ @@ -2938,3 +3028,76 @@ _Py_closerange(int first, int last) #endif /* USE_FDWALK */ _Py_END_SUPPRESS_IPH } + + +#ifndef MS_WINDOWS +// Ticks per second used by clock() and times() functions. +// See os.times() and time.process_time() implementations. +int +_Py_GetTicksPerSecond(long *ticks_per_second) +{ +#if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + long value = sysconf(_SC_CLK_TCK); + if (value < 1) { + return -1; + } + *ticks_per_second = value; +#elif defined(HZ) + assert(HZ >= 1); + *ticks_per_second = HZ; +#else + // Magic fallback value; may be bogus + *ticks_per_second = 60; +#endif + return 0; +} +#endif + + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +int +_Py_IsValidFD(int fd) +{ +/* dup() is faster than fstat(): fstat() can require input/output operations, + whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python + startup. Problem: dup() doesn't check if the file descriptor is valid on + some platforms. + + fcntl(fd, F_GETFD) is even faster, because it only checks the process table. + It is preferred over dup() when available, since it cannot fail with the + "too many open files" error (EMFILE). + + bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other + side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with + EBADF. FreeBSD has similar issue (bpo-32849). + + Only use dup() on Linux where dup() is enough to detect invalid FD + (bpo-32849). +*/ + if (fd < 0) { + return 0; + } +#if defined(F_GETFD) && ( \ + defined(__linux__) || \ + defined(__APPLE__) || \ + (defined(__wasm__) && !defined(__wasi__))) + return fcntl(fd, F_GETFD) >= 0; +#elif defined(__linux__) + int fd2 = dup(fd); + if (fd2 >= 0) { + close(fd2); + } + return (fd2 >= 0); +#elif defined(MS_WINDOWS) + HANDLE hfile; + _Py_BEGIN_SUPPRESS_IPH + hfile = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + return (hfile != INVALID_HANDLE_VALUE + && GetFileType(hfile) != FILE_TYPE_UNKNOWN); +#else + struct stat st; + return (fstat(fd, &st) == 0); +#endif +} diff --git a/Python/flowgraph.c b/Python/flowgraph.c index e89ad39b35719b..ec91b0e616c0a6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -22,13 +22,13 @@ #define DEFAULT_BLOCK_SIZE 16 -typedef _PyCompilerSrcLocation location; -typedef _PyCfgJumpTargetLabel jump_target_label; +typedef _Py_SourceLocation location; +typedef _PyJumpTargetLabel jump_target_label; typedef struct _PyCfgInstruction { int i_opcode; int i_oparg; - _PyCompilerSrcLocation i_loc; + _Py_SourceLocation i_loc; struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ } cfg_instr; @@ -40,7 +40,7 @@ typedef struct _PyCfgBasicblock { control flow. */ struct _PyCfgBasicblock *b_list; /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; + _PyJumpTargetLabel b_label; /* Exception stack at start of block, used by assembler to create the exception handling table */ struct _PyCfgExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ @@ -81,7 +81,7 @@ struct _PyCfgBuilder { /* pointer to the block currently being constructed */ struct _PyCfgBasicblock *g_curblock; /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; + _PyJumpTargetLabel g_current_label; }; typedef struct _PyCfgBuilder cfg_builder; @@ -92,11 +92,12 @@ static const jump_target_label NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) static inline int is_block_push(cfg_instr *i) { + assert(OPCODE_HAS_ARG(i->i_opcode) || !IS_BLOCK_PUSH_OPCODE(i->i_opcode)); return IS_BLOCK_PUSH_OPCODE(i->i_opcode); } @@ -144,6 +145,16 @@ basicblock_next_instr(basicblock *b) return b->b_iused++; } +static cfg_instr * +basicblock_last_instr(const basicblock *b) { + assert(b->b_iused >= 0); + if (b->b_iused > 0) { + assert(b->b_instr != NULL); + return &b->b_instr[b->b_iused - 1]; + } + return NULL; +} + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -151,7 +162,7 @@ basicblock_next_instr(basicblock *b) static basicblock * cfg_builder_new_block(cfg_builder *g) { - basicblock *b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock)); + basicblock *b = (basicblock *)PyMem_Calloc(1, sizeof(basicblock)); if (b == NULL) { PyErr_NoMemory(); return NULL; @@ -184,29 +195,35 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) return SUCCESS; } +static int +basicblock_add_jump(basicblock *b, int opcode, basicblock *target, location loc) +{ + cfg_instr *last = basicblock_last_instr(b); + if (last && is_jump(last)) { + return ERROR; + } + + RETURN_IF_ERROR( + basicblock_addop(b, opcode, target->b_label.id, loc)); + last = basicblock_last_instr(b); + assert(last && last->i_opcode == opcode); + last->i_target = target; + return SUCCESS; +} + static inline int -basicblock_append_instructions(basicblock *target, basicblock *source) +basicblock_append_instructions(basicblock *to, basicblock *from) { - for (int i = 0; i < source->b_iused; i++) { - int n = basicblock_next_instr(target); + for (int i = 0; i < from->b_iused; i++) { + int n = basicblock_next_instr(to); if (n < 0) { return ERROR; } - target->b_instr[n] = source->b_instr[i]; + to->b_instr[n] = from->b_instr[i]; } return SUCCESS; } -static cfg_instr * -basicblock_last_instr(const basicblock *b) { - assert(b->b_iused >= 0); - if (b->b_iused > 0) { - assert(b->b_instr != NULL); - return &b->b_instr[b->b_iused - 1]; - } - return NULL; -} - static inline int basicblock_nofallthrough(const basicblock *b) { cfg_instr *last = basicblock_last_instr(b); @@ -275,9 +292,9 @@ static void dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; - fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, %s\n", + fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, - b->b_startdepth, b_return); + b->b_startdepth, b->b_predecessors, b_return); if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { @@ -315,6 +332,16 @@ basicblock_exits_scope(const basicblock *b) { return last && IS_SCOPE_EXIT_OPCODE(last->i_opcode); } +static inline int +basicblock_has_eval_break(const basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (OPCODE_HAS_EVAL_BREAK(b->b_instr[i].i_opcode)) { + return true; + } + } + return false; +} + static bool cfg_builder_current_block_is_terminated(cfg_builder *g) { @@ -410,10 +437,10 @@ _PyCfgBuilder_Free(cfg_builder *g) basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { - PyObject_Free((void *)b->b_instr); + PyMem_Free((void *)b->b_instr); } basicblock *next = b->b_list; - PyObject_Free((void *)b); + PyMem_Free((void *)b); b = next; } PyMem_Free(g); @@ -448,39 +475,57 @@ _PyCfgBuilder_Addop(cfg_builder *g, int opcode, int oparg, location loc) } +static basicblock * +next_nonempty_block(basicblock *b) +{ + while (b && b->b_iused == 0) { + b = b->b_next; + } + return b; +} + /***** debugging helpers *****/ #ifndef NDEBUG -static int remove_redundant_nops(basicblock *bb); +static int remove_redundant_nops(cfg_builder *g); static bool no_redundant_nops(cfg_builder *g) { - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (remove_redundant_nops(b) != 0) { - return false; - } + if (remove_redundant_nops(g) != 0) { + return false; } return true; } static bool -no_empty_basic_blocks(cfg_builder *g) { +no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (b->b_iused == 0) { - return false; + cfg_instr *last = basicblock_last_instr(b); + if (last != NULL) { + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { + basicblock *next = next_nonempty_block(b->b_next); + basicblock *jump_target = next_nonempty_block(last->i_target); + if (jump_target == next) { + assert(next); + if (last->i_loc.lineno == next->b_instr[0].i_loc.lineno) { + assert(0); + return false; + } + } + } } } return true; } static bool -no_redundant_jumps(cfg_builder *g) { - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - cfg_instr *last = basicblock_last_instr(b); - if (last != NULL) { - if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - assert(last->i_target != b->b_next); - if (last->i_target == b->b_next) { +all_exits_have_lineno(basicblock *entryblock) { + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (instr->i_opcode == RETURN_VALUE) { + if (instr->i_loc.lineno < 0) { + assert(0); return false; } } @@ -488,7 +533,6 @@ no_redundant_jumps(cfg_builder *g) { } return true; } - #endif /***** CFG preprocessing (jump targets and exceptions) *****/ @@ -531,8 +575,8 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); - backwards_jump->b_instr[0].i_target = target; + RETURN_IF_ERROR( + basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; last->i_target = b->b_next; @@ -621,12 +665,6 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } -int -_PyCfg_JumpLabelsToTargets(cfg_builder *g) -{ - return translate_jump_labels_to_targets(g->g_entryblock); -} - static int mark_except_handlers(basicblock *entryblock) { #ifndef NDEBUG @@ -647,7 +685,7 @@ mark_except_handlers(basicblock *entryblock) { struct _PyCfgExceptStack { - basicblock *handlers[CO_MAXBLOCKS+1]; + basicblock *handlers[CO_MAXBLOCKS+2]; int depth; }; @@ -660,6 +698,7 @@ push_except_block(struct _PyCfgExceptStack *stack, cfg_instr *setup) { if (opcode == SETUP_WITH || opcode == SETUP_CLEANUP) { target->b_preserve_lasti = 1; } + assert(stack->depth <= CO_MAXBLOCKS); stack->handlers[++stack->depth] = target; return target; } @@ -712,6 +751,36 @@ make_cfg_traversal_stack(basicblock *entryblock) { return stack; } +/* Return the stack effect of opcode with argument oparg. + + Some opcodes have different stack effect when jump to the target and + when not jump. The 'jump' parameter specifies the case: + + * 0 -- when not jump + * 1 -- when jump + * -1 -- maximal + */ +Py_LOCAL(int) +stack_effect(int opcode, int oparg, int jump) +{ + if (opcode < 0) { + return PY_INVALID_STACK_EFFECT; + } + if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { + // Specialized instructions are not supported. + return PY_INVALID_STACK_EFFECT; + } + int popped = _PyOpcode_num_popped(opcode, oparg); + int pushed = _PyOpcode_num_pushed(opcode, oparg); + if (popped < 0 || pushed < 0) { + return PY_INVALID_STACK_EFFECT; + } + if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { + return 0; + } + return pushed - popped; +} + Py_LOCAL_INLINE(int) stackdepth_push(basicblock ***sp, basicblock *b, int depth) { @@ -756,8 +825,7 @@ calculate_stackdepth(cfg_builder *g) basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 0); + int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", @@ -774,8 +842,7 @@ calculate_stackdepth(cfg_builder *g) maxdepth = new_depth; } if (HAS_TARGET(instr->i_opcode)) { - effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 1); + effect = stack_effect(instr->i_opcode, instr->i_oparg, 1); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", @@ -840,6 +907,7 @@ label_exception_targets(basicblock *entryblock) { assert(except_stack != NULL); b->b_exceptstack = NULL; handler = except_stack_top(except_stack); + int last_yield_except_depth = -1; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { @@ -857,6 +925,7 @@ label_exception_targets(basicblock *entryblock) { } else if (instr->i_opcode == POP_BLOCK) { handler = pop_except_block(except_stack); + INSTR_SET_OP0(instr, NOP); } else if (is_jump(instr)) { instr->i_except = handler; @@ -878,10 +947,21 @@ label_exception_targets(basicblock *entryblock) { todo++; } } - else { - if (instr->i_opcode == YIELD_VALUE) { - instr->i_oparg = except_stack->depth; + else if (instr->i_opcode == YIELD_VALUE) { + instr->i_except = handler; + last_yield_except_depth = except_stack->depth; + } + else if (instr->i_opcode == RESUME) { + instr->i_except = handler; + if (instr->i_oparg != RESUME_AT_FUNC_START) { + assert(last_yield_except_depth >= 0); + if (last_yield_except_depth == 1) { + instr->i_oparg |= RESUME_OPARG_DEPTH1_MASK; + } + last_yield_except_depth = -1; } + } + else { instr->i_except = handler; } } @@ -912,7 +992,10 @@ label_exception_targets(basicblock *entryblock) { /***** CFG optimizations *****/ static int -mark_reachable(basicblock *entryblock) { +remove_unreachable(basicblock *entryblock) { + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + b->b_predecessors = 0; + } basicblock **stack = make_cfg_traversal_stack(entryblock); if (stack == NULL) { return ERROR; @@ -944,47 +1027,19 @@ mark_reachable(basicblock *entryblock) { } } PyMem_Free(stack); - return SUCCESS; -} -static void -eliminate_empty_basic_blocks(cfg_builder *g) { - /* Eliminate empty blocks */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - basicblock *next = b->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } - b->b_next = next; - } - while(g->g_entryblock && g->g_entryblock->b_iused == 0) { - g->g_entryblock = g->g_entryblock->b_next; - } - int next_lbl = get_max_label(g->g_entryblock) + 1; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - assert(b->b_iused > 0); - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (HAS_TARGET(instr->i_opcode)) { - basicblock *target = instr->i_target; - while (target->b_iused == 0) { - target = target->b_next; - } - if (instr->i_target != target) { - if (!IS_LABEL(target->b_label)) { - target->b_label.id = next_lbl++; - } - instr->i_target = target; - instr->i_oparg = target->b_label.id; - } - assert(instr->i_target && instr->i_target->b_iused > 0); - } - } + /* Delete unreachable instructions */ + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + if (b->b_predecessors == 0) { + b->b_iused = 0; + b->b_except_handler = 0; + } } + return SUCCESS; } static int -remove_redundant_nops(basicblock *bb) { +basicblock_remove_redundant_nops(basicblock *bb) { /* Remove NOPs when legal to do so. */ int dest = 0; int prev_lineno = -1; @@ -1011,10 +1066,7 @@ remove_redundant_nops(basicblock *bb) { } } else { - basicblock* next = bb->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } + basicblock *next = next_nonempty_block(bb->b_next); /* or if last instruction in BB and next BB has same line number */ if (next) { location next_loc = NO_LOCATION; @@ -1046,6 +1098,17 @@ remove_redundant_nops(basicblock *bb) { return num_removed; } +static int +remove_redundant_nops(cfg_builder *g) { + int changes = 0; + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + int change = basicblock_remove_redundant_nops(b); + RETURN_IF_ERROR(change); + changes += change; + } + return changes; +} + static int remove_redundant_nops_and_pairs(basicblock *entryblock) { @@ -1056,7 +1119,7 @@ remove_redundant_nops_and_pairs(basicblock *entryblock) cfg_instr *prev_instr = NULL; cfg_instr *instr = NULL; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); + RETURN_IF_ERROR(basicblock_remove_redundant_nops(b)); if (IS_LABEL(b->b_label)) { /* this block is a jump target, forget instr */ instr = NULL; @@ -1096,35 +1159,54 @@ remove_redundant_jumps(cfg_builder *g) { * non-empty block reached through normal flow control is the target * of that jump. If it is, then the jump instruction is redundant and * can be deleted. + * + * Return the number of changes applied, or -1 on error. */ - assert(no_empty_basic_blocks(g)); + + int changes = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - if (last->i_target == NULL) { + basicblock* jump_target = next_nonempty_block(last->i_target); + if (jump_target == NULL) { PyErr_SetString(PyExc_SystemError, "jump with NULL target"); return ERROR; } - if (last->i_target == b->b_next) { - assert(b->b_next->b_iused); + basicblock *next = next_nonempty_block(b->b_next); + if (jump_target == next) { + changes++; INSTR_SET_OP0(last, NOP); } } } - return SUCCESS; + + return changes; +} + +static inline bool +basicblock_has_no_lineno(basicblock *b) { + for (int i = 0; i < b->b_iused; i++) { + if (b->b_instr[i].i_loc.lineno >= 0) { + return false; + } + } + return true; } /* Maximum size of basic block that should be copied in optimizer */ #define MAX_COPY_SIZE 4 -/* If this block ends with an unconditional jump to a small exit block, then +/* If this block ends with an unconditional jump to a small exit block or + * a block that has no line numbers (and no fallthrough), then * remove the jump and extend this block with the target. * Returns 1 if extended, 0 if no change, and -1 on error. */ static int -inline_small_exit_blocks(basicblock *bb) { +basicblock_inline_small_or_no_lineno_blocks(basicblock *bb) { cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; @@ -1133,29 +1215,67 @@ inline_small_exit_blocks(basicblock *bb) { return 0; } basicblock *target = last->i_target; - if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { + bool small_exit_block = (basicblock_exits_scope(target) && + target->b_iused <= MAX_COPY_SIZE); + bool no_lineno_no_fallthrough = (basicblock_has_no_lineno(target) && + !BB_HAS_FALLTHROUGH(target)); + if (small_exit_block || no_lineno_no_fallthrough) { + assert(is_jump(last)); + int removed_jump_opcode = last->i_opcode; INSTR_SET_OP0(last, NOP); RETURN_IF_ERROR(basicblock_append_instructions(bb, target)); + if (no_lineno_no_fallthrough) { + last = basicblock_last_instr(bb); + if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode) && + removed_jump_opcode == JUMP) + { + /* Make sure we don't lose eval breaker checks */ + last->i_opcode = JUMP; + } + } + target->b_predecessors--; return 1; } return 0; } +static int +inline_small_or_no_lineno_blocks(basicblock *entryblock) { + bool changes; + do { + changes = false; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + int res = basicblock_inline_small_or_no_lineno_blocks(b); + RETURN_IF_ERROR(res); + if (res) { + changes = true; + } + } + } while(changes); /* every change removes a jump, ensuring convergence */ + return changes; +} + // Attempt to eliminate jumps to jumps by updating inst to jump to // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(cfg_instr *inst, cfg_instr *target, int opcode) +jump_thread(basicblock *bb, cfg_instr *inst, cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); + assert(inst == basicblock_last_instr(bb)); // bpo-45773: If inst->i_target == target->i_target, then nothing actually // changes (and we fall into an infinite loop): - if ((inst->i_loc.lineno == target->i_loc.lineno || target->i_loc.lineno == -1) && - inst->i_target != target->i_target) - { - inst->i_target = target->i_target; - inst->i_opcode = opcode; + if (inst->i_target != target->i_target) { + /* Change inst to NOP and append a jump to target->i_target. The + * NOP will be removed later if it's not needed for the lineno. + */ + INSTR_SET_OP0(inst, NOP); + + RETURN_IF_ERROR( + basicblock_add_jump( + bb, opcode, target->i_target, target->i_loc)); + return true; } return false; @@ -1448,16 +1568,12 @@ apply_static_swaps(basicblock *block, int i) } static int -optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject *consts) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - cfg_instr nop; - INSTR_SET_OP0(&nop, NOP); - cfg_instr *target = &nop; int opcode = 0; int oparg = 0; - int nextop = 0; for (int i = 0; i < bb->b_iused; i++) { cfg_instr *inst = &bb->b_instr[i]; bool is_copy_of_load_const = (opcode == LOAD_CONST && @@ -1466,118 +1582,148 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (! is_copy_of_load_const) { opcode = inst->i_opcode; oparg = inst->i_oparg; - if (HAS_TARGET(opcode)) { - assert(inst->i_target->b_iused > 0); - target = &inst->i_target->b_instr[0]; - assert(!IS_ASSEMBLER_OPCODE(target->i_opcode)); - } - else { - target = &nop; - } } - nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; assert(!IS_ASSEMBLER_OPCODE(opcode)); - switch (opcode) { - /* Remove LOAD_CONST const; conditional jump */ - case LOAD_CONST: + if (opcode != LOAD_CONST) { + continue; + } + int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; + switch(nextop) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: { - PyObject* cnt; - int is_true; - int jump_if_true; - switch(nextop) { - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - is_true = PyObject_IsTrue(cnt); - Py_DECREF(cnt); - if (is_true == -1) { - goto error; - } - INSTR_SET_OP0(inst, NOP); - jump_if_true = nextop == POP_JUMP_IF_TRUE; - if (is_true == jump_if_true) { - bb->b_instr[i+1].i_opcode = JUMP; - } - else { - INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); - } - break; - case IS_OP: - // Fold to POP_JUMP_IF_NONE: - // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_FALSE - // Fold to POP_JUMP_IF_NOT_NONE: - // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_TRUE - // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_FALSE - // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_TRUE - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - if (!Py_IsNone(cnt)) { - Py_DECREF(cnt); - break; - } - if (bb->b_iused <= i + 2) { - break; - } - cfg_instr *is_instr = &bb->b_instr[i + 1]; - cfg_instr *jump_instr = &bb->b_instr[i + 2]; - // Get rid of TO_BOOL regardless: - if (jump_instr->i_opcode == TO_BOOL) { - INSTR_SET_OP0(jump_instr, NOP); - if (bb->b_iused <= i + 3) { - break; - } - jump_instr = &bb->b_instr[i + 3]; - } - bool invert = is_instr->i_oparg; - if (jump_instr->i_opcode == POP_JUMP_IF_FALSE) { - invert = !invert; - } - else if (jump_instr->i_opcode != POP_JUMP_IF_TRUE) { - break; - } - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP0(is_instr, NOP); - jump_instr->i_opcode = invert ? POP_JUMP_IF_NOT_NONE - : POP_JUMP_IF_NONE; - break; - case RETURN_VALUE: - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); - break; - case TO_BOOL: - cnt = get_const_value(opcode, oparg, consts); - if (cnt == NULL) { - goto error; - } - is_true = PyObject_IsTrue(cnt); - Py_DECREF(cnt); - if (is_true == -1) { - goto error; - } - cnt = PyBool_FromLong(is_true); - int index = add_const(cnt, consts, const_cache); - if (index < 0) { - return ERROR; - } - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP1(&bb->b_instr[i + 1], LOAD_CONST, index); + /* Remove LOAD_CONST const; conditional jump */ + PyObject* cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + int is_true = PyObject_IsTrue(cnt); + Py_DECREF(cnt); + if (is_true == -1) { + return ERROR; + } + INSTR_SET_OP0(inst, NOP); + int jump_if_true = nextop == POP_JUMP_IF_TRUE; + if (is_true == jump_if_true) { + bb->b_instr[i+1].i_opcode = JUMP; + } + else { + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); + } + break; + } + case IS_OP: + { + // Fold to POP_JUMP_IF_NONE: + // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_FALSE + // Fold to POP_JUMP_IF_NOT_NONE: + // - LOAD_CONST(None) IS_OP(0) POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(1) POP_JUMP_IF_TRUE + // - LOAD_CONST(None) IS_OP(0) TO_BOOL POP_JUMP_IF_FALSE + // - LOAD_CONST(None) IS_OP(1) TO_BOOL POP_JUMP_IF_TRUE + PyObject *cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + if (!Py_IsNone(cnt)) { + Py_DECREF(cnt); + break; + } + if (bb->b_iused <= i + 2) { + break; + } + cfg_instr *is_instr = &bb->b_instr[i + 1]; + cfg_instr *jump_instr = &bb->b_instr[i + 2]; + // Get rid of TO_BOOL regardless: + if (jump_instr->i_opcode == TO_BOOL) { + INSTR_SET_OP0(jump_instr, NOP); + if (bb->b_iused <= i + 3) { break; + } + jump_instr = &bb->b_instr[i + 3]; + } + bool invert = is_instr->i_oparg; + if (jump_instr->i_opcode == POP_JUMP_IF_FALSE) { + invert = !invert; + } + else if (jump_instr->i_opcode != POP_JUMP_IF_TRUE) { + break; } + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(is_instr, NOP); + jump_instr->i_opcode = invert ? POP_JUMP_IF_NOT_NONE + : POP_JUMP_IF_NONE; break; } - /* Try to fold tuples of constants. - Skip over BUILD_TUPLE(1) UNPACK_SEQUENCE(1). - Replace BUILD_TUPLE(2) UNPACK_SEQUENCE(2) with SWAP(2). - Replace BUILD_TUPLE(3) UNPACK_SEQUENCE(3) with SWAP(3). */ + case RETURN_VALUE: + { + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); + break; + } + case TO_BOOL: + { + PyObject *cnt = get_const_value(opcode, oparg, consts); + if (cnt == NULL) { + return ERROR; + } + int is_true = PyObject_IsTrue(cnt); + Py_DECREF(cnt); + if (is_true == -1) { + return ERROR; + } + cnt = PyBool_FromLong(is_true); + int index = add_const(cnt, consts, const_cache); + if (index < 0) { + return ERROR; + } + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[i + 1], LOAD_CONST, index); + break; + } + } + } + return SUCCESS; +} + +static int +optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts) { + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts)); + } + return SUCCESS; +} + +static int +optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +{ + assert(PyDict_CheckExact(const_cache)); + assert(PyList_CheckExact(consts)); + cfg_instr nop; + INSTR_SET_OP0(&nop, NOP); + for (int i = 0; i < bb->b_iused; i++) { + cfg_instr *inst = &bb->b_instr[i]; + cfg_instr *target; + int opcode = inst->i_opcode; + int oparg = inst->i_oparg; + if (HAS_TARGET(opcode)) { + assert(inst->i_target->b_iused > 0); + target = &inst->i_target->b_instr[0]; + assert(!IS_ASSEMBLER_OPCODE(target->i_opcode)); + } + else { + target = &nop; + } + int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; + assert(!IS_ASSEMBLER_OPCODE(opcode)); + switch (opcode) { + /* Try to fold tuples of constants. + Skip over BUILD_TUPLE(1) UNPACK_SEQUENCE(1). + Replace BUILD_TUPLE(2) UNPACK_SEQUENCE(2) with SWAP(2). + Replace BUILD_TUPLE(3) UNPACK_SEQUENCE(3) with SWAP(3). */ case BUILD_TUPLE: if (nextop == UNPACK_SEQUENCE && oparg == bb->b_instr[i+1].i_oparg) { switch(oparg) { @@ -1602,25 +1748,30 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) case POP_JUMP_IF_NONE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, inst->i_opcode); + i -= jump_thread(bb, inst, target, inst->i_opcode); } break; case POP_JUMP_IF_FALSE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_FALSE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_FALSE); } break; case POP_JUMP_IF_TRUE: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, POP_JUMP_IF_TRUE); + i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; case JUMP: + case JUMP_NO_INTERRUPT: switch (target->i_opcode) { case JUMP: - i -= jump_thread(inst, target, JUMP); + i -= jump_thread(bb, inst, target, JUMP); + continue; + case JUMP_NO_INTERRUPT: + i -= jump_thread(bb, inst, target, opcode); + continue; } break; case FOR_ITER: @@ -1631,7 +1782,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) * of FOR_ITER. */ /* - i -= jump_thread(inst, target, FOR_ITER); + i -= jump_thread(bb, inst, target, FOR_ITER); */ } break; @@ -1687,9 +1838,6 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } break; - default: - /* All OPCODE_HAS_CONST opcodes should be handled with LOAD_CONST */ - assert (!OPCODE_HAS_CONST(inst->i_opcode)); } } @@ -1707,6 +1855,23 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) return ERROR; } +static int resolve_line_numbers(cfg_builder *g, int firstlineno); + +static int +remove_redundant_nops_and_jumps(cfg_builder *g) +{ + int removed_nops, removed_jumps; + do { + /* Convergence is guaranteed because the number of + * redundant jumps and nops only decreases. + */ + removed_nops = remove_redundant_nops(g); + RETURN_IF_ERROR(removed_nops); + removed_jumps = remove_redundant_jumps(g); + RETURN_IF_ERROR(removed_jumps); + } while(removed_nops + removed_jumps > 0); + return SUCCESS; +} /* Perform optimizations on a control flow graph. The consts object should still be in list form to allow new constants @@ -1716,37 +1881,21 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) NOPs. Later those NOPs are removed. */ static int -optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache) +optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstlineno) { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); - eliminate_empty_basic_blocks(g); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } - assert(no_empty_basic_blocks(g)); + RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); + RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); + RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); + RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); - assert(b->b_predecessors == 0); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(inline_small_exit_blocks(b)); - } - RETURN_IF_ERROR(mark_reachable(g->g_entryblock)); - - /* Delete unreachable instructions */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (b->b_predecessors == 0) { - b->b_iused = 0; - } - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } - eliminate_empty_basic_blocks(g); - assert(no_redundant_nops(g)); - RETURN_IF_ERROR(remove_redundant_jumps(g)); + RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); + RETURN_IF_ERROR(remove_redundant_nops_and_jumps(g)); + assert(no_redundant_jumps(g)); return SUCCESS; } @@ -1766,7 +1915,7 @@ make_super_instruction(cfg_instr *inst1, cfg_instr *inst2, int super_op) INSTR_SET_OP0(inst2, NOP); } -static void +static int insert_superinstructions(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -1793,11 +1942,9 @@ insert_superinstructions(cfg_builder *g) } } } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } - eliminate_empty_basic_blocks(g); + int res = remove_redundant_nops(g); assert(no_redundant_nops(g)); + return res; } // helper functions for add_checks_for_loads_of_unknown_variables @@ -1948,7 +2095,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* now index_map[i] == i if consts[i] is used, -1 otherwise */ /* condense consts */ Py_ssize_t n_used_consts = 0; - for (int i = 0; i < nconsts; i++) { + for (Py_ssize_t i = 0; i < nconsts; i++) { if (index_map[i] != -1) { assert(index_map[i] == i); index_map[n_used_consts++] = index_map[i]; @@ -2163,9 +2310,11 @@ push_cold_blocks_to_end(cfg_builder *g) { if (!IS_LABEL(b->b_next->b_label)) { b->b_next->b_label.id = next_lbl++; } - basicblock_addop(explicit_jump, JUMP, b->b_next->b_label.id, NO_LOCATION); + basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id, + NO_LOCATION); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; + explicit_jump->b_predecessors = 1; b->b_next = explicit_jump; /* set target */ @@ -2215,47 +2364,42 @@ push_cold_blocks_to_end(cfg_builder *g) { b->b_next = cold_blocks; if (cold_blocks != NULL) { - RETURN_IF_ERROR(remove_redundant_jumps(g)); + RETURN_IF_ERROR(remove_redundant_nops_and_jumps(g)); } return SUCCESS; } -static void -convert_pseudo_ops(basicblock *entryblock) +static int +convert_pseudo_ops(cfg_builder *g) { + basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { - assert(SAME_OPCODE_METADATA(instr->i_opcode, NOP)); + if (is_block_push(instr)) { INSTR_SET_OP0(instr, NOP); } else if (instr->i_opcode == LOAD_CLOSURE) { - assert(SAME_OPCODE_METADATA(LOAD_CLOSURE, LOAD_FAST)); + assert(is_pseudo_target(LOAD_CLOSURE, LOAD_FAST)); instr->i_opcode = LOAD_FAST; } else if (instr->i_opcode == STORE_FAST_MAYBE_NULL) { - assert(SAME_OPCODE_METADATA(STORE_FAST_MAYBE_NULL, STORE_FAST)); + assert(is_pseudo_target(STORE_FAST_MAYBE_NULL, STORE_FAST)); instr->i_opcode = STORE_FAST; } } } - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - remove_redundant_nops(b); - } + return remove_redundant_nops_and_jumps(g); } static inline bool -is_exit_without_lineno(basicblock *b) { - if (!basicblock_exits_scope(b)) { - return false; +is_exit_or_eval_check_without_lineno(basicblock *b) { + if (basicblock_exits_scope(b) || basicblock_has_eval_break(b)) { + return basicblock_has_no_lineno(b); } - for (int i = 0; i < b->b_iused; i++) { - if (b->b_instr[i].i_loc.lineno >= 0) { - return false; - } + else { + return false; } - return true; } @@ -2271,8 +2415,6 @@ is_exit_without_lineno(basicblock *b) { static int duplicate_exits_without_lineno(cfg_builder *g) { - assert(no_empty_basic_blocks(g)); - int next_lbl = get_max_label(g->g_entryblock) + 1; /* Copy all exit blocks without line number that are targets of a jump. @@ -2280,10 +2422,12 @@ duplicate_exits_without_lineno(cfg_builder *g) basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } if (is_jump(last)) { - basicblock *target = last->i_target; - if (is_exit_without_lineno(target) && target->b_predecessors > 1) { + basicblock *target = next_nonempty_block(last->i_target); + if (is_exit_or_eval_check_without_lineno(target) && target->b_predecessors > 1) { basicblock *new_target = copy_basicblock(g, target); if (new_target == NULL) { return ERROR; @@ -2303,7 +2447,7 @@ duplicate_exits_without_lineno(cfg_builder *g) * fall through, and thus can only have a single predecessor */ for (basicblock *b = entryblock; b != NULL; b = b->b_next) { if (BB_HAS_FALLTHROUGH(b) && b->b_next && b->b_iused > 0) { - if (is_exit_without_lineno(b->b_next)) { + if (is_exit_or_eval_check_without_lineno(b->b_next)) { cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); b->b_next->b_instr[0].i_loc = last->i_loc; @@ -2339,9 +2483,10 @@ propagate_line_numbers(basicblock *entryblock) { } } if (BB_HAS_FALLTHROUGH(b) && b->b_next->b_predecessors == 1) { - assert(b->b_next->b_iused); - if (b->b_next->b_instr[0].i_loc.lineno < 0) { - b->b_next->b_instr[0].i_loc = prev_location; + if (b->b_next->b_iused > 0) { + if (b->b_next->b_instr[0].i_loc.lineno < 0) { + b->b_next->b_instr[0].i_loc = prev_location; + } } } if (is_jump(last)) { @@ -2355,40 +2500,11 @@ propagate_line_numbers(basicblock *entryblock) { } } -/* Make sure that all returns have a line number, even if early passes - * have failed to propagate a correct line number. - * The resulting line number may not be correct according to PEP 626, - * but should be "good enough", and no worse than in older versions. */ -static void -guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { - int lineno = firstlineno; - assert(lineno > 0); - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - cfg_instr *last = basicblock_last_instr(b); - if (last == NULL) { - continue; - } - if (last->i_loc.lineno < 0) { - if (last->i_opcode == RETURN_VALUE) { - for (int i = 0; i < b->b_iused; i++) { - assert(b->b_instr[i].i_loc.lineno < 0); - - b->b_instr[i].i_loc.lineno = lineno; - } - } - } - else { - lineno = last->i_loc.lineno; - } - } -} - static int resolve_line_numbers(cfg_builder *g, int firstlineno) { RETURN_IF_ERROR(duplicate_exits_without_lineno(g)); propagate_line_numbers(g->g_entryblock); - guarantee_lineno_for_exits(g->g_entryblock, firstlineno); return SUCCESS; } @@ -2404,14 +2520,15 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache)); + RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache, firstlineno)); RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); RETURN_IF_ERROR( add_checks_for_loads_of_uninitialized_variables( g->g_entryblock, nlocals, nparams)); - insert_superinstructions(g); + RETURN_IF_ERROR(insert_superinstructions(g)); RETURN_IF_ERROR(push_cold_blocks_to_end(g)); + assert(all_exits_have_lineno(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); return SUCCESS; } @@ -2624,26 +2741,70 @@ prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_fl return nlocalsplus; } +cfg_builder * +_PyCfg_FromInstructionSequence(_PyInstructionSequence *seq) +{ + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return NULL; + } + cfg_builder *g = _PyCfgBuilder_New(); + if (g == NULL) { + return NULL; + } + for (int i = 0; i < seq->s_used; i++) { + seq->s_instrs[i].i_target = 0; + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg >= 0 && instr->i_oparg < seq->s_used); + seq->s_instrs[instr->i_oparg].i_target = 1; + } + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (instr->i_target) { + jump_target_label lbl_ = {i}; + if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { + goto error; + } + } + int opcode = instr->i_opcode; + int oparg = instr->i_oparg; + if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { + goto error; + } + } + if (_PyCfgBuilder_CheckSize(g) < 0) { + goto error; + } + return g; +error: + _PyCfgBuilder_Free(g); + return NULL; +} + int -_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq) +_PyCfg_ToInstructionSequence(cfg_builder *g, _PyInstructionSequence *seq) { int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; + lbl += 1; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode)) { + if (HAS_TARGET(instr->i_opcode)) { + /* Set oparg to the label id (it will later be mapped to an offset) */ instr->i_oparg = instr->i_target->b_label.id; } RETURN_IF_ERROR( - _PyCompile_InstructionSequence_Addop( + _PyInstructionSequence_Addop( seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); - _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + _PyExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; if (instr->i_except != NULL) { hi->h_label = instr->i_except->b_label.id; hi->h_startdepth = instr->i_except->b_startdepth; @@ -2654,6 +2815,9 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq } } } + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return ERROR; + } return SUCCESS; } @@ -2662,7 +2826,7 @@ int _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq) + _PyInstructionSequence *seq) { *stackdepth = calculate_stackdepth(g); if (*stackdepth < 0) { @@ -2683,7 +2847,7 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, return ERROR; } - convert_pseudo_ops(g->g_entryblock); + RETURN_IF_ERROR(convert_pseudo_ops(g)); /* Order of basic blocks must have been determined by now */ @@ -2697,3 +2861,77 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, return SUCCESS; } + +/* This is used by _PyCompile_Assemble to fill in the jump and exception + * targets in a synthetic CFG (which is not the output of the builtin compiler). + */ +int +_PyCfg_JumpLabelsToTargets(cfg_builder *g) +{ + RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); + RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); + return SUCCESS; +} + +/* Exported API functions */ + +int +PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) +{ + return stack_effect(opcode, oparg, jump); +} + +int +PyCompile_OpcodeStackEffect(int opcode, int oparg) +{ + return stack_effect(opcode, oparg, -1); +} + +/* Access to compiler optimizations for unit tests. + + * _PyCompile_OptimizeCfg takes an instruction list, constructs + * a CFG, optimizes it and converts back to an instruction list. + */ + +static PyObject * +cfg_to_instruction_sequence(cfg_builder *g) +{ + _PyInstructionSequence *seq = (_PyInstructionSequence *)_PyInstructionSequence_New(); + if (seq == NULL) { + return NULL; + } + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + PyInstructionSequence_Fini(seq); + return NULL; + } + return (PyObject*)seq; +} + +PyObject * +_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) +{ + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + PyObject *res = NULL; + cfg_builder *g = _PyCfg_FromInstructionSequence((_PyInstructionSequence*)seq); + if (g == NULL) { + goto error; + } + int nparams = 0, firstlineno = 1; + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, + nparams, firstlineno) < 0) { + goto error; + } + res = cfg_to_instruction_sequence(g); +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Free(g); + return res; +} diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 6af589f966a502..ebd67214f43042 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -320,7 +320,7 @@ parse_internal_render_format_spec(PyObject *obj, format->thousands_separators = LT_UNDER_FOUR_LOCALE; break; } - /* fall through */ + _Py_FALLTHROUGH; default: invalid_thousands_separator_type(format->thousands_separators, format->type); return 0; diff --git a/Python/frame.c b/Python/frame.c index b483903fdf3018..9c7e59601e6faf 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -16,11 +16,11 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ - PyObject **locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); int i = 0; /* locals and stack */ for (; i stacktop; i++) { - Py_VISIT(locals[i]); + Py_VISIT(PyStackRef_AsPyObjectBorrow(locals[i])); } return 0; } @@ -37,24 +37,15 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) return NULL; } PyErr_SetRaisedException(exc); - if (frame->frame_obj) { - // GH-97002: How did we get into this horrible situation? Most likely, - // allocating f triggered a GC collection, which ran some code that - // *also* created the same frame... while we were in the middle of - // creating it! See test_sneaky_frame_object in test_frame.py for a - // concrete example. - // - // Regardless, just throw f away and use that frame instead, since it's - // already been exposed to user code. It's actually a bit tricky to do - // this, since we aren't backed by a real _PyInterpreterFrame anymore. - // Just pretend that we have an owned, cleared frame so frame_dealloc - // doesn't make the situation worse: - f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_frame->owner = FRAME_CLEARED; - f->f_frame->frame_obj = f; - Py_DECREF(f); - return frame->frame_obj; - } + + // GH-97002: There was a time when a frame object could be created when we + // are allocating the new frame object f above, so frame->frame_obj would + // be assigned already. That path does not exist anymore. We won't call any + // Python code in this function and garbage collection will not run. + // Notice that _PyFrame_New_NoTrack() can potentially raise a MemoryError, + // but it won't allocate a traceback until the frame unwinds, so we are safe + // here. + assert(frame->frame_obj == NULL); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); f->f_frame = frame; @@ -62,18 +53,6 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) return f; } -void -_PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) -{ - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); - Py_ssize_t size = ((char*)&src->localsplus[src->stacktop]) - (char *)src; - memcpy(dest, src, size); - // Don't leave a dangling pointer to the old frame when creating generators - // and coroutines: - dest->previous = NULL; -} - - static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { @@ -90,7 +69,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: PyCodeObject *code = _PyFrame_GetCode(frame); - frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; } assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); @@ -115,13 +94,25 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) } } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame) +{ + assert(frame->stacktop >= 0); + int stacktop = frame->stacktop; + frame->stacktop = 0; + for (int i = 0; i < stacktop; i++) { + PyStackRef_XCLOSE(frame->localsplus[i]); + } + Py_CLEAR(frame->f_locals); +} + void _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { /* It is the responsibility of the owning generator/coroutine * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + _PyGen_GetGeneratorFromFrame(frame)->gi_frame_state == FRAME_CLEARED); // GH-99729: Clearing this frame can expose the stack (via finalizers). It's // crucial that this frame has been unlinked, and is no longer visible: assert(_PyThreadState_GET()->current_frame != frame); @@ -135,12 +126,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) } Py_DECREF(f); } - assert(frame->stacktop >= 0); - for (int i = 0; i < frame->stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); - } - Py_XDECREF(frame->frame_obj); - Py_XDECREF(frame->f_locals); + _PyFrame_ClearLocals(frame); Py_DECREF(frame->f_funcobj); } diff --git a/Python/frozen.c b/Python/frozen.c index 0fb38a11902f35..627f2ff9413562 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -66,40 +66,6 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#define GET_CODE(name) _Py_get_##name##_toplevel - -/* Start extern declarations */ -extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); -extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void); -extern PyObject *_Py_get_zipimport_toplevel(void); -extern PyObject *_Py_get_abc_toplevel(void); -extern PyObject *_Py_get_codecs_toplevel(void); -extern PyObject *_Py_get_io_toplevel(void); -extern PyObject *_Py_get__collections_abc_toplevel(void); -extern PyObject *_Py_get__sitebuiltins_toplevel(void); -extern PyObject *_Py_get_genericpath_toplevel(void); -extern PyObject *_Py_get_ntpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_os_toplevel(void); -extern PyObject *_Py_get_site_toplevel(void); -extern PyObject *_Py_get_stat_toplevel(void); -extern PyObject *_Py_get_importlib_util_toplevel(void); -extern PyObject *_Py_get_importlib_machinery_toplevel(void); -extern PyObject *_Py_get_runpy_toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_eggs_toplevel(void); -extern PyObject *_Py_get___phello___spam_toplevel(void); -extern PyObject *_Py_get_frozen_only_toplevel(void); -/* End extern declarations */ - static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 3ce9476c9ad46c..ec4566bd4f84bc 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -54,6 +54,12 @@ Py_FrozenMain(int argc, char **argv) Py_ExitStatusException(status); } + PyInterpreterState *interp = PyInterpreterState_Get(); + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + PyErr_Print(); + exit(1); + } + #ifdef MS_WINDOWS PyWinFreeze_ExeInit(); #endif @@ -83,6 +89,9 @@ Py_FrozenMain(int argc, char **argv) #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif + + _PyInterpreterState_SetNotRunningMain(interp); + if (Py_FinalizeEx() < 0) { sts = 120; } diff --git a/Python/future.c b/Python/future.c index 0dbc7ede20f324..8aeb541cb05107 100644 --- a/Python/future.c +++ b/Python/future.c @@ -1,13 +1,14 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_symtable.h" // _PyFutureFeatures #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" static int -future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) +future_check_features(_PyFutureFeatures *ff, stmt_ty s, PyObject *filename) { - int i; + Py_ssize_t i; assert(s->kind == ImportFrom_kind); @@ -53,7 +54,7 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) } static int -future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) +future_parse(_PyFutureFeatures *ff, mod_ty mod, PyObject *filename) { if (!(mod->kind == Module_kind || mod->kind == Interactive_kind)) { return 1; @@ -76,7 +77,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) * are another future statement and a doc string. */ - if (s->kind == ImportFrom_kind) { + if (s->kind == ImportFrom_kind && s->v.ImportFrom.level == 0) { identifier modname = s->v.ImportFrom.module; if (modname && _PyUnicode_EqualToASCIIString(modname, "__future__")) { @@ -98,10 +99,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) int -_PyFuture_FromAST(mod_ty mod, PyObject *filename, PyFutureFeatures *ff) +_PyFuture_FromAST(mod_ty mod, PyObject *filename, _PyFutureFeatures *ff) { ff->ff_features = 0; - ff->ff_location = (_PyCompilerSrcLocation){-1, -1, -1, -1}; + ff->ff_location = (_Py_SourceLocation){-1, -1, -1, -1}; if (!future_parse(ff, mod, filename)) { return 0; diff --git a/Python/gc.c b/Python/gc.c new file mode 100644 index 00000000000000..38a0da91a97510 --- /dev/null +++ b/Python/gc.c @@ -0,0 +1,2178 @@ +// This implements the reference cycle garbage collector. +// The Python module interface to the collector is in gcmodule.c. +// See https://devguide.python.org/internals/garbage-collector/ + +#include "Python.h" +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() +#include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_initconfig.h" +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_object.h" +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pydtrace.h" + +#ifndef Py_GIL_DISABLED + +typedef struct _gc_runtime_state GCState; + +#ifdef Py_DEBUG +# define GC_DEBUG +#endif + +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +// update_refs() set this bit for all objects in current generation. +// subtract_refs() and move_unreachable() uses this to distinguish +// visited object is in GCing or not. +// +// move_unreachable() removes this flag from reachable objects. +// Only unreachable objects have this flag. +// +// No objects in interpreter have this flag after GC ends. +#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING + +// Lowest bit of _gc_next is used for UNREACHABLE flag. +// +// This flag represents the object is in unreachable list in move_unreachable() +// +// Although this flag is used only in move_unreachable(), move_unreachable() +// doesn't clear this flag to skip unnecessary iteration. +// move_legacy_finalizers() removes this flag instead. +// Between them, unreachable list is not normal list and we can not use +// most gc_list_* functions for it. +#define NEXT_MASK_UNREACHABLE 2 + +#define AS_GC(op) _Py_AS_GC(op) +#define FROM_GC(gc) _Py_FROM_GC(gc) + +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +static inline int +gc_is_collecting(PyGC_Head *g) +{ + return (g->_gc_prev & PREV_MASK_COLLECTING) != 0; +} + +static inline void +gc_clear_collecting(PyGC_Head *g) +{ + g->_gc_prev &= ~PREV_MASK_COLLECTING; +} + +static inline Py_ssize_t +gc_get_refs(PyGC_Head *g) +{ + return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); +} + +static inline void +gc_set_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) + | PREV_MASK_COLLECTING + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_decref(PyGC_Head *g) +{ + _PyObject_ASSERT_WITH_MSG(FROM_GC(g), + gc_get_refs(g) > 0, + "refcount is too small"); + g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; +} + +static inline int +gc_old_space(PyGC_Head *g) +{ + return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline int +flip_old_space(int space) +{ + assert(space == 0 || space == 1); + return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline void +gc_flip_old_space(PyGC_Head *g) +{ + g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static PyGC_Head * +GEN_HEAD(GCState *gcstate, int n) +{ + assert((gcstate->visited_space & (~1)) == 0); + switch(n) { + case 0: + return &gcstate->young.head; + case 1: + return &gcstate->old[gcstate->visited_space].head; + case 2: + return &gcstate->old[gcstate->visited_space^1].head; + default: + Py_UNREACHABLE(); + } +} + +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + + +void +_PyGC_InitState(GCState *gcstate) +{ +#define INIT_HEAD(GEN) \ + do { \ + GEN.head._gc_next = (uintptr_t)&GEN.head; \ + GEN.head._gc_prev = (uintptr_t)&GEN.head; \ + } while (0) + + assert(gcstate->young.count == 0); + assert(gcstate->old[0].count == 0); + assert(gcstate->old[1].count == 0); + INIT_HEAD(gcstate->young); + INIT_HEAD(gcstate->old[0]); + INIT_HEAD(gcstate->old[1]); + INIT_HEAD(gcstate->permanent_generation); + +#undef INIT_HEAD +} + + +PyStatus +_PyGC_Init(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + + gcstate->garbage = PyList_New(0); + if (gcstate->garbage == NULL) { + return _PyStatus_NO_MEMORY(); + } + + gcstate->callbacks = PyList_New(0); + if (gcstate->callbacks == NULL) { + return _PyStatus_NO_MEMORY(); + } + gcstate->heap_size = 0; + + return _PyStatus_OK(); +} + + +/* +_gc_prev values +--------------- + +Between collections, _gc_prev is used for doubly linked list. + +Lowest two bits of _gc_prev are used for flags. +PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends +or _PyObject_GC_UNTRACK() is called. + +During a collection, _gc_prev is temporary used for gc_refs, and the gc list +is singly linked until _gc_prev is restored. + +gc_refs + At the start of a collection, update_refs() copies the true refcount + to gc_refs, for each object in the generation being collected. + subtract_refs() then adjusts gc_refs so that it equals the number of + times an object is referenced directly from outside the generation + being collected. + +PREV_MASK_COLLECTING + Objects in generation being collected are marked PREV_MASK_COLLECTING in + update_refs(). + + +_gc_next values +--------------- + +_gc_next takes these values: + +0 + The object is not tracked + +!= 0 + Pointer to the next object in the GC list. + Additionally, lowest bit is used temporary for + NEXT_MASK_UNREACHABLE flag described below. + +NEXT_MASK_UNREACHABLE + move_unreachable() then moves objects not reachable (whether directly or + indirectly) from outside the generation into an "unreachable" set and + set this flag. + + Objects that are found to be reachable have gc_refs set to 1. + When this flag is set for the reachable object, the object must be in + "unreachable" set. + The flag is unset and the object is moved back to "reachable" set. + + move_legacy_finalizers() will remove this flag from "unreachable" set. +*/ + +/*** list functions ***/ + +static inline void +gc_list_init(PyGC_Head *list) +{ + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; +} + +static inline int +gc_list_is_empty(PyGC_Head *list) +{ + return (list->_gc_next == (uintptr_t)list); +} + +/* Append `node` to `list`. */ +static inline void +gc_list_append(PyGC_Head *node, PyGC_Head *list) +{ + assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0); + PyGC_Head *last = (PyGC_Head *)list->_gc_prev; + + // last <-> node + _PyGCHead_SET_PREV(node, last); + _PyGCHead_SET_NEXT(last, node); + + // node <-> list + _PyGCHead_SET_NEXT(node, list); + list->_gc_prev = (uintptr_t)node; +} + +/* Remove `node` from the gc list it's currently in. */ +static inline void +gc_list_remove(PyGC_Head *node) +{ + PyGC_Head *prev = GC_PREV(node); + PyGC_Head *next = GC_NEXT(node); + + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); + + node->_gc_next = 0; /* object is not currently tracked */ +} + +/* Move `node` from the gc list it's currently in (which is not explicitly + * named here) to the end of `list`. This is semantically the same as + * gc_list_remove(node) followed by gc_list_append(node, list). + */ +static void +gc_list_move(PyGC_Head *node, PyGC_Head *list) +{ + /* Unlink from current list. */ + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + /* Relink at end of new list. */ + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); +} + +/* append list `from` onto list `to`; `from` becomes an empty list */ +static void +gc_list_merge(PyGC_Head *from, PyGC_Head *to) +{ + assert(from != to); + if (!gc_list_is_empty(from)) { + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + assert(gc_list_is_empty(to) || + gc_old_space(to_tail) == gc_old_space(from_tail)); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); + } + gc_list_init(from); +} + +static Py_ssize_t +gc_list_size(PyGC_Head *list) +{ + PyGC_Head *gc; + Py_ssize_t n = 0; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + n++; + } + return n; +} + +/* Walk the list and mark all objects as non-collecting */ +static inline void +gc_list_clear_collecting(PyGC_Head *collectable) +{ + PyGC_Head *gc; + for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { + gc_clear_collecting(gc); + } +} + +/* Append objects in a GC list to a Python list. + * Return 0 if all OK, < 0 if error (out of memory for list) + */ +static int +append_objects(PyObject *py_list, PyGC_Head *gc_list) +{ + PyGC_Head *gc; + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + if (op != py_list) { + if (PyList_Append(py_list, op)) { + return -1; /* exception */ + } + } + } + return 0; +} + +// Constants for validate_list's flags argument. +enum flagstates {collecting_clear_unreachable_clear, + collecting_clear_unreachable_set, + collecting_set_unreachable_clear, + collecting_set_unreachable_set}; + +#ifdef GC_DEBUG +// validate_list checks list consistency. And it works as document +// describing when flags are expected to be set / unset. +// `head` must be a doubly-linked gc list, although it's fine (expected!) if +// the prev and next pointers are "polluted" with flags. +// What's checked: +// - The `head` pointers are not polluted. +// - The objects' PREV_MASK_COLLECTING and NEXT_MASK_UNREACHABLE flags are all +// `set or clear, as specified by the 'flags' argument. +// - The prev and next pointers are mutually consistent. +static void +validate_list(PyGC_Head *head, enum flagstates flags) +{ + assert((head->_gc_prev & ~_PyGC_PREV_MASK) == 0); + assert((head->_gc_next & ~_PyGC_PREV_MASK) == 0); + uintptr_t prev_value = 0, next_value = 0; + switch (flags) { + case collecting_clear_unreachable_clear: + break; + case collecting_set_unreachable_clear: + prev_value = PREV_MASK_COLLECTING; + break; + case collecting_clear_unreachable_set: + next_value = NEXT_MASK_UNREACHABLE; + break; + case collecting_set_unreachable_set: + prev_value = PREV_MASK_COLLECTING; + next_value = NEXT_MASK_UNREACHABLE; + break; + default: + assert(! "bad internal flags argument"); + } + PyGC_Head *prev = head; + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + PyGC_Head *trueprev = GC_PREV(gc); + PyGC_Head *truenext = GC_NEXT(gc); + assert(truenext != NULL); + assert(trueprev == prev); + assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); + assert((gc->_gc_next & NEXT_MASK_UNREACHABLE) == next_value); + prev = gc; + gc = truenext; + } + assert(prev == GC_PREV(head)); +} + +static void +validate_old(GCState *gcstate) +{ + for (int space = 0; space < 2; space++) { + PyGC_Head *head = &gcstate->old[space].head; + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + PyGC_Head *next = GC_NEXT(gc); + assert(gc_old_space(gc) == space); + gc = next; + } + } +} + +static void +validate_consistent_old_space(PyGC_Head *head) +{ + PyGC_Head *prev = head; + PyGC_Head *gc = GC_NEXT(head); + if (gc == head) { + return; + } + int old_space = gc_old_space(gc); + while (gc != head) { + PyGC_Head *truenext = GC_NEXT(gc); + assert(truenext != NULL); + assert(gc_old_space(gc) == old_space); + prev = gc; + gc = truenext; + } + assert(prev == GC_PREV(head)); +} + +static void +gc_list_validate_space(PyGC_Head *head, int space) { + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + assert(gc_old_space(gc) == space); + gc = GC_NEXT(gc); + } +} + +#else +#define validate_list(x, y) do{}while(0) +#define validate_old(g) do{}while(0) +#define validate_consistent_old_space(l) do{}while(0) +#define gc_list_validate_space(l, s) do{}while(0) +#endif + +/*** end of list stuff ***/ + + +/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and + * PREV_MASK_COLLECTING bit is set for all objects in containers. + */ +static void +update_refs(PyGC_Head *containers) +{ + PyGC_Head *next; + PyGC_Head *gc = GC_NEXT(containers); + + while (gc != containers) { + next = GC_NEXT(gc); + PyObject *op = FROM_GC(gc); + if (_Py_IsImmortal(op)) { + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } + gc_reset_refs(gc, Py_REFCNT(op)); + /* Python's cyclic gc should never see an incoming refcount + * of 0: if something decref'ed to 0, it should have been + * deallocated immediately at that time. + * Possible cause (if the assert triggers): a tp_dealloc + * routine left a gc-aware object tracked during its teardown + * phase, and did something-- or allowed something to happen -- + * that called back into Python. gc can trigger then, and may + * see the still-tracked dying object. Before this assert + * was added, such mistakes went on to allow gc to try to + * delete the object again. In a debug build, that caused + * a mysterious segfault, when _Py_ForgetReference tried + * to remove the object from the doubly-linked list of all + * objects a second time. In a release build, an actual + * double deallocation occurred, which leads to corruption + * of the allocator's internal bookkeeping pointers. That's + * so serious that maybe this should be a release-build + * check instead of an assert? + */ + _PyObject_ASSERT(op, gc_get_refs(gc) != 0); + gc = next; + } +} + +/* A traversal callback for subtract_refs. */ +static int +visit_decref(PyObject *op, void *parent) +{ + OBJECT_STAT_INC(object_visits); + _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); + + if (_PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + /* We're only interested in gc_refs for objects in the + * generation being collected, which can be recognized + * because only they have positive gc_refs. + */ + if (gc_is_collecting(gc)) { + gc_decref(gc); + } + } + return 0; +} + +/* Subtract internal references from gc_refs. After this, gc_refs is >= 0 + * for all objects in containers, and is GC_REACHABLE for all tracked gc + * objects not in containers. The ones with gc_refs > 0 are directly + * reachable from outside containers, and so can't be collected. + */ +static void +subtract_refs(PyGC_Head *containers) +{ + traverseproc traverse; + PyGC_Head *gc = GC_NEXT(containers); + for (; gc != containers; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_decref, + op); + } +} + +/* A traversal callback for move_unreachable. */ +static int +visit_reachable(PyObject *op, void *arg) +{ + PyGC_Head *reachable = arg; + OBJECT_STAT_INC(object_visits); + if (!_PyObject_IS_GC(op)) { + return 0; + } + + PyGC_Head *gc = AS_GC(op); + const Py_ssize_t gc_refs = gc_get_refs(gc); + + // Ignore objects in other generation. + // This also skips objects "to the left" of the current position in + // move_unreachable's scan of the 'young' list - they've already been + // traversed, and no longer have the PREV_MASK_COLLECTING flag. + if (! gc_is_collecting(gc)) { + return 0; + } + // It would be a logic error elsewhere if the collecting flag were set on + // an untracked object. + _PyObject_ASSERT(op, gc->_gc_next != 0); + + if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { + /* This had gc_refs = 0 when move_unreachable got + * to it, but turns out it's reachable after all. + * Move it back to move_unreachable's 'young' list, + * and move_unreachable will eventually get to it + * again. + */ + // Manually unlink gc from unreachable list because the list functions + // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyObject_ASSERT(FROM_GC(prev), + prev->_gc_next & NEXT_MASK_UNREACHABLE); + _PyObject_ASSERT(FROM_GC(next), + next->_gc_next & NEXT_MASK_UNREACHABLE); + prev->_gc_next = gc->_gc_next; // copy flag bits + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + _PyGCHead_SET_PREV(next, prev); + + gc_list_append(gc, reachable); + gc_set_refs(gc, 1); + } + else if (gc_refs == 0) { + /* This is in move_unreachable's 'young' list, but + * the traversal hasn't yet gotten to it. All + * we need to do is tell move_unreachable that it's + * reachable. + */ + gc_set_refs(gc, 1); + } + /* Else there's nothing to do. + * If gc_refs > 0, it must be in move_unreachable's 'young' + * list, and move_unreachable will eventually get to it. + */ + else { + _PyObject_ASSERT_WITH_MSG(op, gc_refs > 0, "refcount is too small"); + } + return 0; +} + +/* Move the unreachable objects from young to unreachable. After this, + * all objects in young don't have PREV_MASK_COLLECTING flag and + * unreachable have the flag. + * All objects in young after this are directly or indirectly reachable + * from outside the original young; and all objects in unreachable are + * not. + * + * This function restores _gc_prev pointer. young and unreachable are + * doubly linked list after this function. + * But _gc_next in unreachable list has NEXT_MASK_UNREACHABLE flag. + * So we can not gc_list_* functions for unreachable until we remove the flag. + */ +static void +move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) +{ + // previous elem in the young list, used for restore gc_prev. + PyGC_Head *prev = young; + PyGC_Head *gc = GC_NEXT(young); + + /* Invariants: all objects "to the left" of us in young are reachable + * (directly or indirectly) from outside the young list as it was at entry. + * + * All other objects from the original young "to the left" of us are in + * unreachable now, and have NEXT_MASK_UNREACHABLE. All objects to the + * left of us in 'young' now have been scanned, and no objects here + * or to the right have been scanned yet. + */ + + validate_consistent_old_space(young); + /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ + uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); + while (gc != young) { + if (gc_get_refs(gc)) { + /* gc is definitely reachable from outside the + * original 'young'. Mark it as such, and traverse + * its pointers to find any other objects that may + * be directly reachable from it. Note that the + * call to tp_traverse may append objects to young, + * so we have to wait until it returns to determine + * the next object to visit. + */ + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) > 0, + "refcount is too small"); + // NOTE: visit_reachable may change gc->_gc_next when + // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! + (void) traverse(op, + visit_reachable, + (void *)young); + // relink gc_prev to prev element. + _PyGCHead_SET_PREV(gc, prev); + // gc is not COLLECTING state after here. + gc_clear_collecting(gc); + prev = gc; + } + else { + /* This *may* be unreachable. To make progress, + * assume it is. gc isn't directly reachable from + * any object we've already traversed, but may be + * reachable from an object we haven't gotten to yet. + * visit_reachable will eventually move gc back into + * young if that's so, and we'll see it again. + */ + // Move gc to unreachable. + // No need to gc->next->prev = prev because it is single linked. + prev->_gc_next = gc->_gc_next; + + // We can't use gc_list_append() here because we use + // NEXT_MASK_UNREACHABLE here. + PyGC_Head *last = GC_PREV(unreachable); + // NOTE: Since all objects in unreachable set has + // NEXT_MASK_UNREACHABLE flag, we set it unconditionally. + // But this may pollute the unreachable list head's 'next' pointer + // too. That's semantically senseless but expedient here - the + // damage is repaired when this function ends. + last->_gc_next = flags | (uintptr_t)gc; + _PyGCHead_SET_PREV(gc, last); + gc->_gc_next = flags | (uintptr_t)unreachable; + unreachable->_gc_prev = (uintptr_t)gc; + } + gc = _PyGCHead_NEXT(prev); + } + // young->_gc_prev must be last element remained in the list. + young->_gc_prev = (uintptr_t)prev; + young->_gc_next &= _PyGC_PREV_MASK; + // don't let the pollution of the list head's next pointer leak + unreachable->_gc_next &= _PyGC_PREV_MASK; +} + +static void +untrack_tuples(PyGC_Head *head) +{ + PyGC_Head *next, *gc = GC_NEXT(head); + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + } + gc = next; + } +} + +/* Try to untrack all currently tracked dictionaries */ +static void +untrack_dicts(PyGC_Head *head) +{ + PyGC_Head *next, *gc = GC_NEXT(head); + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + } + gc = next; + } +} + +/* Return true if object has a pre-PEP 442 finalization method. */ +static int +has_legacy_finalizer(PyObject *op) +{ + return Py_TYPE(op)->tp_del != NULL; +} + +/* Move the objects in unreachable with tp_del slots into `finalizers`. + * + * This function also removes NEXT_MASK_UNREACHABLE flag + * from _gc_next in unreachable. + */ +static void +move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) +{ + PyGC_Head *gc, *next; + _PyObject_ASSERT( + FROM_GC(unreachable), + (unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + + /* March over unreachable. Move objects with finalizers into + * `finalizers`. + */ + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyObject *op = FROM_GC(gc); + + _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + + if (has_legacy_finalizer(op)) { + gc_clear_collecting(gc); + gc_list_move(gc, finalizers); + } + } +} + +static inline void +clear_unreachable_mask(PyGC_Head *unreachable) +{ + /* Check that the list head does not have the unreachable bit set */ + _PyObject_ASSERT( + FROM_GC(unreachable), + ((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0); + _PyObject_ASSERT( + FROM_GC(unreachable), + (unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + + PyGC_Head *gc, *next; + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + } + validate_list(unreachable, collecting_set_unreachable_clear); +} + +/* A traversal callback for move_legacy_finalizer_reachable. */ +static int +visit_move(PyObject *op, void *arg) +{ + PyGC_Head *tolist = arg; + OBJECT_STAT_INC(object_visits); + if (_PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (gc_is_collecting(gc)) { + gc_list_move(gc, tolist); + gc_clear_collecting(gc); + } + } + return 0; +} + +/* Move objects that are reachable from finalizers, from the unreachable set + * into finalizers set. + */ +static void +move_legacy_finalizer_reachable(PyGC_Head *finalizers) +{ + traverseproc traverse; + PyGC_Head *gc = GC_NEXT(finalizers); + for (; gc != finalizers; gc = GC_NEXT(gc)) { + /* Note that the finalizers list may grow during this. */ + traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; + (void) traverse(FROM_GC(gc), + visit_move, + (void *)finalizers); + } +} + +/* Clear all weakrefs to unreachable objects, and if such a weakref has a + * callback, invoke it if necessary. Note that it's possible for such + * weakrefs to be outside the unreachable set -- indeed, those are precisely + * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for + * overview & some details. Some weakrefs with callbacks may be reclaimed + * directly by this routine; the number reclaimed is the return value. Other + * weakrefs with callbacks may be moved into the `old` generation. Objects + * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in + * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, + * no object in `unreachable` is weakly referenced anymore. + */ +static int +handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +{ + PyGC_Head *gc; + PyObject *op; /* generally FROM_GC(gc) */ + PyWeakReference *wr; /* generally a cast of op */ + PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ + PyGC_Head *next; + int num_freed = 0; + + gc_list_init(&wrcb_to_call); + + /* Clear all weakrefs to the objects in unreachable. If such a weakref + * also has a callback, move it into `wrcb_to_call` if the callback + * needs to be invoked. Note that we cannot invoke any callbacks until + * all weakrefs to unreachable objects are cleared, lest the callback + * resurrect an unreachable object via a still-active weakref. We + * make another pass over wrcb_to_call, invoking callbacks, after this + * pass completes. + */ + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyWeakReference **wrlist; + + op = FROM_GC(gc); + next = GC_NEXT(gc); + + if (PyWeakref_Check(op)) { + /* A weakref inside the unreachable set must be cleared. If we + * allow its callback to execute inside delete_garbage(), it + * could expose objects that have tp_clear already called on + * them. Or, it could resurrect unreachable objects. One way + * this can happen is if some container objects do not implement + * tp_traverse. Then, wr_object can be outside the unreachable + * set but can be deallocated as a result of breaking the + * reference cycle. If we don't clear the weakref, the callback + * will run and potentially cause a crash. See bpo-38006 for + * one example. + */ + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + /* It supports weakrefs. Does it have any? + * + * This is never triggered for static types so we can avoid the + * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + */ + wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + /* `op` may have some weakrefs. March over the list, clear + * all the weakrefs, and move the weakrefs with callbacks + * that must be called into wrcb_to_call. + */ + for (wr = *wrlist; wr != NULL; wr = *wrlist) { + PyGC_Head *wrasgc; /* AS_GC(wr) */ + + /* _PyWeakref_ClearRef clears the weakref but leaves + * the callback pointer intact. Obscure: it also + * changes *wrlist. + */ + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + if (wr->wr_callback == NULL) { + /* no callback */ + continue; + } + + /* Headache time. `op` is going away, and is weakly referenced by + * `wr`, which has a callback. Should the callback be invoked? If wr + * is also trash, no: + * + * 1. There's no need to call it. The object and the weakref are + * both going away, so it's legitimate to pretend the weakref is + * going away first. The user has to ensure a weakref outlives its + * referent if they want a guarantee that the wr callback will get + * invoked. + * + * 2. It may be catastrophic to call it. If the callback is also in + * cyclic trash (CT), then although the CT is unreachable from + * outside the current generation, CT may be reachable from the + * callback. Then the callback could resurrect insane objects. + * + * Since the callback is never needed and may be unsafe in this case, + * wr is simply left in the unreachable set. Note that because we + * already called _PyWeakref_ClearRef(wr), its callback will never + * trigger. + * + * OTOH, if wr isn't part of CT, we should invoke the callback: the + * weakref outlived the trash. Note that since wr isn't CT in this + * case, its callback can't be CT either -- wr acted as an external + * root to this generation, and therefore its callback did too. So + * nothing in CT is reachable from the callback either, so it's hard + * to imagine how calling it later could create a problem for us. wr + * is moved to wrcb_to_call in this case. + */ + if (gc_is_collecting(AS_GC((PyObject *)wr))) { + /* it should already have been cleared above */ + _PyObject_ASSERT((PyObject*)wr, wr->wr_object == Py_None); + continue; + } + + /* Create a new reference so that wr can't go away + * before we can process it again. + */ + Py_INCREF(wr); + + /* Move wr to wrcb_to_call, for the next pass. */ + wrasgc = AS_GC((PyObject *)wr); + // wrasgc is reachable, but next isn't, so they can't be the same + _PyObject_ASSERT((PyObject *)wr, wrasgc != next); + gc_list_move(wrasgc, &wrcb_to_call); + } + } + + /* Invoke the callbacks we decided to honor. It's safe to invoke them + * because they can't reference unreachable objects. + */ + int visited_space = get_gc_state()->visited_space; + while (! gc_list_is_empty(&wrcb_to_call)) { + PyObject *temp; + PyObject *callback; + + gc = (PyGC_Head*)wrcb_to_call._gc_next; + op = FROM_GC(gc); + _PyObject_ASSERT(op, PyWeakref_Check(op)); + wr = (PyWeakReference *)op; + callback = wr->wr_callback; + _PyObject_ASSERT(op, callback != NULL); + + /* copy-paste of weakrefobject.c's handle_callback() */ + temp = PyObject_CallOneArg(callback, (PyObject *)wr); + if (temp == NULL) { + PyErr_WriteUnraisable(callback); + } + else { + Py_DECREF(temp); + } + + /* Give up the reference we created in the first pass. When + * op's refcount hits 0 (which it may or may not do right now), + * op's tp_dealloc will decref op->wr_callback too. Note + * that the refcount probably will hit 0 now, and because this + * weakref was reachable to begin with, gc didn't already + * add it to its count of freed objects. Example: a reachable + * weak value dict maps some key to this reachable weakref. + * The callback removes this key->weakref mapping from the + * dict, leaving no other references to the weakref (excepting + * ours). + */ + Py_DECREF(op); + if (wrcb_to_call._gc_next == (uintptr_t)gc) { + /* object is still alive -- move it */ + gc_set_old_space(gc, visited_space); + gc_list_move(gc, old); + } + else { + ++num_freed; + } + } + + return num_freed; +} + +static void +debug_cycle(const char *msg, PyObject *op) +{ + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); +} + +/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable + * only from such cycles). + * If _PyGC_DEBUG_SAVEALL, all objects in finalizers are appended to the module + * garbage list (a Python list), else only the objects in finalizers with + * __del__ methods are appended to garbage. All objects in finalizers are + * merged into the old list regardless. + */ +static void +handle_legacy_finalizers(PyThreadState *tstate, + GCState *gcstate, + PyGC_Head *finalizers, PyGC_Head *old) +{ + assert(!_PyErr_Occurred(tstate)); + assert(gcstate->garbage != NULL); + + PyGC_Head *gc = GC_NEXT(finalizers); + for (; gc != finalizers; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + + if ((gcstate->debug & _PyGC_DEBUG_SAVEALL) || has_legacy_finalizer(op)) { + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + break; + } + } + } + + gc_list_merge(finalizers, old); +} + +/* Run first-time finalizers (if any) on all the objects in collectable. + * Note that this may remove some (or even all) of the objects from the + * list, due to refcounts falling to 0. + */ +static void +finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) +{ + destructor finalize; + PyGC_Head seen; + + /* While we're going through the loop, `finalize(op)` may cause op, or + * other objects, to be reclaimed via refcounts falling to zero. So + * there's little we can rely on about the structure of the input + * `collectable` list across iterations. For safety, we always take the + * first object in that list and move it to a temporary `seen` list. + * If objects vanish from the `collectable` and `seen` lists we don't + * care. + */ + gc_list_init(&seen); + + while (!gc_list_is_empty(collectable)) { + PyGC_Head *gc = GC_NEXT(collectable); + PyObject *op = FROM_GC(gc); + gc_list_move(gc, &seen); + if (!_PyGC_FINALIZED(op) && + (finalize = Py_TYPE(op)->tp_finalize) != NULL) + { + _PyGC_SET_FINALIZED(op); + Py_INCREF(op); + finalize(op); + assert(!_PyErr_Occurred(tstate)); + Py_DECREF(op); + } + } + gc_list_merge(&seen, collectable); +} + +/* Break reference cycles by clearing the containers involved. This is + * tricky business as the lists can be changing and we don't know which + * objects may be freed. It is possible I screwed something up here. + */ +static void +delete_garbage(PyThreadState *tstate, GCState *gcstate, + PyGC_Head *collectable, PyGC_Head *old) +{ + assert(!_PyErr_Occurred(tstate)); + + while (!gc_list_is_empty(collectable)) { + PyGC_Head *gc = GC_NEXT(collectable); + PyObject *op = FROM_GC(gc); + + _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, + "refcount is too small"); + + if (gcstate->debug & _PyGC_DEBUG_SAVEALL) { + assert(gcstate->garbage != NULL); + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + } + } + else { + inquiry clear; + if ((clear = Py_TYPE(op)->tp_clear) != NULL) { + Py_INCREF(op); + (void) clear(op); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", + Py_TYPE(op)->tp_name); + } + Py_DECREF(op); + } + } + if (GC_NEXT(collectable) == gc) { + /* object is still alive, move it, it may die later */ + gc_clear_collecting(gc); + gc_list_move(gc, old); + } + } +} + + +/* Deduce which objects among "base" are unreachable from outside the list + and move them to 'unreachable'. The process consist in the following steps: + +1. Copy all reference counts to a different field (gc_prev is used to hold + this copy to save memory). +2. Traverse all objects in "base" and visit all referred objects using + "tp_traverse" and for every visited object, subtract 1 to the reference + count (the one that we copied in the previous step). After this step, all + objects that can be reached directly from outside must have strictly positive + reference count, while all unreachable objects must have a count of exactly 0. +3. Identify all unreachable objects (the ones with 0 reference count) and move + them to the "unreachable" list. This step also needs to move back to "base" all + objects that were initially marked as unreachable but are referred transitively + by the reachable objects (the ones with strictly positive reference count). + +Contracts: + + * The "base" has to be a valid list with no mask set. + + * The "unreachable" list must be uninitialized (this function calls + gc_list_init over 'unreachable'). + +IMPORTANT: This function leaves 'unreachable' with the NEXT_MASK_UNREACHABLE +flag set but it does not clear it to skip unnecessary iteration. Before the +flag is cleared (for example, by using 'clear_unreachable_mask' function or +by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal +list and we can not use most gc_list_* functions for it. */ +static inline void +deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { + validate_list(base, collecting_clear_unreachable_clear); + /* Using ob_refcnt and gc_refs, calculate which objects in the + * container set are reachable from outside the set (i.e., have a + * refcount greater than 0 when all the references within the + * set are taken into account). + */ + update_refs(base); // gc_prev is used for gc_refs + subtract_refs(base); + + /* Leave everything reachable from outside base in base, and move + * everything else (in base) to unreachable. + * + * NOTE: This used to move the reachable objects into a reachable + * set instead. But most things usually turn out to be reachable, + * so it's more efficient to move the unreachable things. It "sounds slick" + * to move the unreachable objects, until you think about it - the reason it + * pays isn't actually obvious. + * + * Suppose we create objects A, B, C in that order. They appear in the young + * generation in the same order. If B points to A, and C to B, and C is + * reachable from outside, then the adjusted refcounts will be 0, 0, and 1 + * respectively. + * + * When move_unreachable finds A, A is moved to the unreachable list. The + * same for B when it's first encountered. Then C is traversed, B is moved + * _back_ to the reachable list. B is eventually traversed, and then A is + * moved back to the reachable list. + * + * So instead of not moving at all, the reachable objects B and A are moved + * twice each. Why is this a win? A straightforward algorithm to move the + * reachable objects instead would move A, B, and C once each. + * + * The key is that this dance leaves the objects in order C, B, A - it's + * reversed from the original order. On all _subsequent_ scans, none of + * them will move. Since most objects aren't in cycles, this can save an + * unbounded number of moves across an unbounded number of later collections. + * It can cost more only the first time the chain is scanned. + * + * Drawback: move_unreachable is also used to find out what's still trash + * after finalizers may resurrect objects. In _that_ case most unreachable + * objects will remain unreachable, so it would be more efficient to move + * the reachable objects instead. But this is a one-time cost, probably not + * worth complicating the code to speed just a little. + */ + move_unreachable(base, unreachable); // gc_prev is pointer again + validate_list(base, collecting_clear_unreachable_clear); + validate_list(unreachable, collecting_set_unreachable_set); +} + +/* Handle objects that may have resurrected after a call to 'finalize_garbage', moving + them to 'old_generation' and placing the rest on 'still_unreachable'. + + Contracts: + * After this function 'unreachable' must not be used anymore and 'still_unreachable' + will contain the objects that did not resurrect. + + * The "still_unreachable" list must be uninitialized (this function calls + gc_list_init over 'still_unreachable'). + +IMPORTANT: After a call to this function, the 'still_unreachable' set will have the +PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so +we can skip the expense of clearing the flag to avoid extra iteration. */ +static inline void +handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, + PyGC_Head *old_generation) +{ + // Remove the PREV_MASK_COLLECTING from unreachable + // to prepare it for a new call to 'deduce_unreachable' + gc_list_clear_collecting(unreachable); + + // After the call to deduce_unreachable, the 'still_unreachable' set will + // have the PREV_MARK_COLLECTING set, but the objects are going to be + // removed so we can skip the expense of clearing the flag. + PyGC_Head* resurrected = unreachable; + deduce_unreachable(resurrected, still_unreachable); + clear_unreachable_mask(still_unreachable); + + // Move the resurrected objects to the old generation for future collection. + gc_list_merge(resurrected, old_generation); +} + + +#define UNTRACK_TUPLES 1 +#define UNTRACK_DICTS 2 + +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + int untrack, + struct gc_collection_stats *stats); + +static inline Py_ssize_t +gc_list_set_space(PyGC_Head *list, int space) +{ + Py_ssize_t size = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + gc_set_old_space(gc, space); + size++; + } + return size; +} + +/* Making progress in the incremental collector + * In order to eventually collect all cycles + * the incremental collector must progress through the old + * space faster than objects are added to the old space. + * + * Each young or incremental collection adds a number of + * objects, S (for survivors) to the old space, and + * incremental collectors scan I objects from the old space. + * I > S must be true. We also want I > S * N to be where + * N > 1. Higher values of N mean that the old space is + * scanned more rapidly. + * The default incremental threshold of 10 translates to + * N == 1.4 (1 + 4/threshold) + */ + +/* Divide by 10, so that the default incremental threshold of 10 + * scans objects at 1% of the heap size */ +#define SCAN_RATE_DIVISOR 10 + +static void +add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) +{ + gcstate->generation_stats[gen].collected += stats->collected; + gcstate->generation_stats[gen].uncollectable += stats->uncollectable; + gcstate->generation_stats[gen].collections += 1; +} + +static void +gc_collect_young(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; +#ifdef Py_STATS + { + Py_ssize_t count = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { + count++; + } + } +#endif + + PyGC_Head survivors; + gc_list_init(&survivors); + gc_collect_region(tstate, young, &survivors, UNTRACK_TUPLES, stats); + Py_ssize_t survivor_count = 0; + if (gcstate->visited_space) { + /* objects in visited space have bit set, so we set it here */ + survivor_count = gc_list_set_space(&survivors, 1); + } + else { + PyGC_Head *gc; + for (gc = GC_NEXT(&survivors); gc != &survivors; gc = GC_NEXT(gc)) { +#ifdef GC_DEBUG + assert(gc_old_space(gc) == 0); +#endif + survivor_count++; + } + } + (void)survivor_count; // Silence compiler warning + gc_list_merge(&survivors, visited); + validate_old(gcstate); + gcstate->young.count = 0; + gcstate->old[gcstate->visited_space].count++; + Py_ssize_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 1) { + scale_factor = 1; + } + gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + add_stats(gcstate, 0, stats); +} + +#ifndef NDEBUG +static inline int +IS_IN_VISITED(PyGC_Head *gc, int visited_space) +{ + assert(visited_space == 0 || flip_old_space(visited_space) == 0); + return gc_old_space(gc) == visited_space; +} +#endif + +struct container_and_flag { + PyGC_Head *container; + int visited_space; + uintptr_t size; +}; + +/* A traversal callback for adding to container) */ +static int +visit_add_to_container(PyObject *op, void *arg) +{ + OBJECT_STAT_INC(object_visits); + struct container_and_flag *cf = (struct container_and_flag *)arg; + int visited = cf->visited_space; + assert(visited == get_gc_state()->visited_space); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited) { + gc_flip_old_space(gc); + gc_list_move(gc, cf->container); + cf->size++; + } + } + return 0; +} + +static uintptr_t +expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) +{ + validate_list(container, collecting_clear_unreachable_clear); + struct container_and_flag arg = { + .container = container, + .visited_space = gcstate->visited_space, + .size = 0 + }; + assert(GC_NEXT(gc) == container); + while (gc != container) { + /* Survivors will be moved to visited space, so they should + * have been marked as visited */ + assert(IS_IN_VISITED(gc, gcstate->visited_space)); + PyObject *op = FROM_GC(gc); + if (_Py_IsImmortal(op)) { + PyGC_Head *next = GC_NEXT(gc); + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + gc = GC_NEXT(gc); + } + return arg.size; +} + +/* Do bookkeeping for a completed GC cycle */ +static void +completed_cycle(GCState *gcstate) +{ +#ifdef Py_DEBUG + PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; + assert(gc_list_is_empty(not_visited)); +#endif + gcstate->visited_space = flip_old_space(gcstate->visited_space); + /* Make sure all young objects have old space bit set correctly */ + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *gc = GC_NEXT(young); + while (gc != young) { + PyGC_Head *next = GC_NEXT(gc); + gc_set_old_space(gc, gcstate->visited_space); + gc = next; + } + gcstate->work_to_do = 0; +} + +static void +gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) +{ + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + PyGC_Head increment; + gc_list_init(&increment); + Py_ssize_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 1) { + scale_factor = 1; + } + gc_list_merge(&gcstate->young.head, &increment); + gcstate->young.count = 0; + gc_list_validate_space(&increment, gcstate->visited_space); + Py_ssize_t increment_size = 0; + while (increment_size < gcstate->work_to_do) { + if (gc_list_is_empty(not_visited)) { + break; + } + PyGC_Head *gc = _PyGCHead_NEXT(not_visited); + gc_list_move(gc, &increment); + increment_size++; + gc_set_old_space(gc, gcstate->visited_space); + increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); + } + gc_list_validate_space(&increment, gcstate->visited_space); + PyGC_Head survivors; + gc_list_init(&survivors); + gc_collect_region(tstate, &increment, &survivors, UNTRACK_TUPLES, stats); + gc_list_validate_space(&survivors, gcstate->visited_space); + gc_list_merge(&survivors, visited); + assert(gc_list_is_empty(&increment)); + gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + gcstate->work_to_do -= increment_size; + + validate_old(gcstate); + add_stats(gcstate, 1, stats); + if (gc_list_is_empty(not_visited)) { + completed_cycle(gcstate); + } +} + + +static void +gc_collect_full(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ + GCState *gcstate = &tstate->interp->gc; + validate_old(gcstate); + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + /* merge all generations into visited */ + gc_list_validate_space(young, gcstate->visited_space); + gc_list_set_space(pending, gcstate->visited_space); + gc_list_merge(young, pending); + gcstate->young.count = 0; + gc_list_merge(pending, visited); + + gc_collect_region(tstate, visited, visited, + UNTRACK_TUPLES | UNTRACK_DICTS, + stats); + gcstate->young.count = 0; + gcstate->old[0].count = 0; + gcstate->old[1].count = 0; + + gcstate->work_to_do = - gcstate->young.threshold * 2; + _PyGC_ClearAllFreeLists(tstate->interp); + validate_old(gcstate); + add_stats(gcstate, 2, stats); +} + +/* This is the main function. Read this to understand how the + * collection process works. */ +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + int untrack, + struct gc_collection_stats *stats) +{ + PyGC_Head unreachable; /* non-problematic unreachable trash */ + PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ + PyGC_Head *gc; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + gc_list_init(&unreachable); + deduce_unreachable(from, &unreachable); + validate_consistent_old_space(from); + if (untrack & UNTRACK_TUPLES) { + untrack_tuples(from); + } + if (untrack & UNTRACK_DICTS) { + untrack_dicts(from); + } + validate_consistent_old_space(to); + if (from != to) { + gc_list_merge(from, to); + } + validate_consistent_old_space(to); + /* Move reachable objects to next generation. */ + + /* All objects in unreachable are trash, but objects reachable from + * legacy finalizers (e.g. tp_del) can't safely be deleted. + */ + gc_list_init(&finalizers); + // NEXT_MASK_UNREACHABLE is cleared here. + // After move_legacy_finalizers(), unreachable is normal list. + move_legacy_finalizers(&unreachable, &finalizers); + /* finalizers contains the unreachable objects with a legacy finalizer; + * unreachable objects reachable *from* those are also uncollectable, + * and we move those into the finalizers list too. + */ + move_legacy_finalizer_reachable(&finalizers); + validate_list(&finalizers, collecting_clear_unreachable_clear); + validate_list(&unreachable, collecting_set_unreachable_clear); + /* Print debugging information. */ + if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { + for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { + debug_cycle("collectable", FROM_GC(gc)); + } + } + + /* Clear weakrefs and invoke callbacks as necessary. */ + stats->collected += handle_weakrefs(&unreachable, to); + gc_list_validate_space(to, gcstate->visited_space); + validate_list(to, collecting_clear_unreachable_clear); + validate_list(&unreachable, collecting_set_unreachable_clear); + + /* Call tp_finalize on objects which have one. */ + finalize_garbage(tstate, &unreachable); + /* Handle any objects that may have resurrected after the call + * to 'finalize_garbage' and continue the collection with the + * objects that are still unreachable */ + PyGC_Head final_unreachable; + gc_list_init(&final_unreachable); + handle_resurrected_objects(&unreachable, &final_unreachable, to); + + /* Call tp_clear on objects in the final_unreachable set. This will cause + * the reference cycles to be broken. It may also cause some objects + * in finalizers to be freed. + */ + stats->collected += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, to); + + /* Collect statistics on uncollectable objects found and print + * debugging information. */ + Py_ssize_t n = 0; + for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { + n++; + if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) + debug_cycle("uncollectable", FROM_GC(gc)); + } + stats->uncollectable = n; + /* Append instances in the uncollectable set to a Python + * reachable list of garbage. The programmer has to deal with + * this if they insist on creating this type of structure. + */ + handle_legacy_finalizers(tstate, gcstate, &finalizers, to); + gc_list_validate_space(to, gcstate->visited_space); + validate_list(to, collecting_clear_unreachable_clear); +} + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +do_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + assert(!PyErr_Occurred()); + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", stats->collected, + "uncollectable", stats->uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!PyErr_Occurred()); +} + +static void +invoke_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + if (gcstate->callbacks == NULL) { + return; + } + do_gc_callback(gcstate, phase, generation, stats); +} + +static int +referrersvisit(PyObject* obj, void *arg) +{ + PyObject *objs = arg; + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(objs); i++) { + if (PyTuple_GET_ITEM(objs, i) == obj) { + return 1; + } + } + return 0; +} + +static int +gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) +{ + PyGC_Head *gc; + PyObject *obj; + traverseproc traverse; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + obj = FROM_GC(gc); + traverse = Py_TYPE(obj)->tp_traverse; + if (obj == objs || obj == resultlist) { + continue; + } + if (traverse(obj, referrersvisit, objs)) { + if (PyList_Append(resultlist, obj) < 0) { + return 0; /* error */ + } + } + } + return 1; /* no error */ +} + +PyObject * +_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + GCState *gcstate = &interp->gc; + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (!(gc_referrers_for(objs, GEN_HEAD(gcstate, i), result))) { + Py_DECREF(result); + return NULL; + } + } + return result; +} + +PyObject * +_PyGC_GetObjects(PyInterpreterState *interp, int generation) +{ + assert(generation >= -1 && generation < NUM_GENERATIONS); + GCState *gcstate = &interp->gc; + + PyObject *result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + if (generation == -1) { + /* If generation is -1, get all objects from all generations */ + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (append_objects(result, GEN_HEAD(gcstate, i))) { + goto error; + } + } + } + else { + if (append_objects(result, GEN_HEAD(gcstate, generation))) { + goto error; + } + } + + return result; +error: + Py_DECREF(result); + return NULL; +} + +void +_PyGC_Freeze(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + /* The permanent_generation has its old space bit set to zero */ + if (gcstate->visited_space) { + gc_list_set_space(&gcstate->young.head, 0); + } + gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); + gcstate->young.count = 0; + PyGC_Head*old0 = &gcstate->old[0].head; + PyGC_Head*old1 = &gcstate->old[1].head; + gc_list_merge(old0, &gcstate->permanent_generation.head); + gcstate->old[0].count = 0; + gc_list_set_space(old1, 0); + gc_list_merge(old1, &gcstate->permanent_generation.head); + gcstate->old[1].count = 0; + validate_old(gcstate); +} + +void +_PyGC_Unfreeze(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + gc_list_merge(&gcstate->permanent_generation.head, + &gcstate->old[0].head); + validate_old(gcstate); +} + +Py_ssize_t +_PyGC_GetFreezeCount(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + return gc_list_size(&gcstate->permanent_generation.head); +} + +/* C API for controlling the state of the garbage collector */ +int +PyGC_Enable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 1; + return old_state; +} + +int +PyGC_Disable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 0; + return old_state; +} + +int +PyGC_IsEnabled(void) +{ + GCState *gcstate = get_gc_state(); + return gcstate->enabled; +} + +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + GCState *gcstate = &tstate->interp->gc; + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + struct gc_collection_stats stats = { 0 }; + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "start", generation, &stats); + } + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + switch(generation) { + case 0: + gc_collect_young(tstate, &stats); + break; + case 1: + gc_collect_increment(tstate, &stats); + break; + case 2: + gc_collect_full(tstate, &stats); + break; + default: + Py_UNREACHABLE(); + } + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(stats.uncollectable + stats.collected); + } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "stop", generation, &stats); + } + _PyErr_SetRaisedException(tstate, exc); + GC_STAT_ADD(generation, objects_collected, stats.collected); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + validate_old(gcstate); + _Py_atomic_store_int(&gcstate->collecting, 0); + return stats.uncollectable + stats.collected; +} + +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) +{ + return _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_MANUAL); +} + +void +_PyGC_CollectNoFail(PyThreadState *tstate) +{ + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_SHUTDOWN); +} + +void +_PyGC_DumpShutdownStats(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (!(gcstate->debug & _PyGC_DEBUG_SAVEALL) + && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { + const char *message; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + message = "gc: %zd uncollectable objects at shutdown"; + } + else { + message = "gc: %zd uncollectable objects at shutdown; " \ + "use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; + } + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(gcstate->garbage))) + { + PyErr_WriteUnraisable(NULL); + } + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + PyObject *repr = NULL, *bytes = NULL; + repr = PyObject_Repr(gcstate->garbage); + if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) { + PyErr_WriteUnraisable(gcstate->garbage); + } + else { + PySys_WriteStderr( + " %s\n", + PyBytes_AS_STRING(bytes) + ); + } + Py_XDECREF(repr); + Py_XDECREF(bytes); + } + } +} + + +void +_PyGC_Fini(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + Py_CLEAR(gcstate->garbage); + Py_CLEAR(gcstate->callbacks); + + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ +} + +/* for debugging */ +void +_PyGC_Dump(PyGC_Head *g) +{ + _PyObject_Dump(FROM_GC(g)); +} + + +#ifdef Py_DEBUG +static int +visit_validate(PyObject *op, void *parent_raw) +{ + PyObject *parent = _PyObject_CAST(parent_raw); + if (_PyObject_IsFreed(op)) { + _PyObject_ASSERT_FAILED_MSG(parent, + "PyObject_GC_Track() object is not valid"); + } + return 0; +} +#endif + + +/* extension modules might be compiled with GC support so these + functions must always be available */ + +void +PyObject_GC_Track(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT_FAILED_MSG(op, + "object already tracked " + "by the garbage collector"); + } + _PyObject_GC_TRACK(op); + +#ifdef Py_DEBUG + /* Check that the object is valid: validate objects traversed + by tp_traverse() */ + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void)traverse(op, visit_validate, op); +#endif +} + +void +PyObject_GC_UnTrack(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + /* Obscure: the Py_TRASHCAN mechanism requires that we be able to + * call PyObject_GC_UnTrack twice on an object. + */ + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } +} + +int +PyObject_IS_GC(PyObject *obj) +{ + return _PyObject_IS_GC(obj); +} + +void +_Py_ScheduleGC(PyThreadState *tstate) +{ + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } +} + +void +_PyObject_GC_Link(PyObject *op) +{ + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); + + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + gc->_gc_next = 0; + gc->_gc_prev = 0; + gcstate->young.count++; /* number of allocated GC objects */ + gcstate->heap_size++; + if (gcstate->young.count > gcstate->young.threshold && + gcstate->enabled && + gcstate->young.threshold && + !_Py_atomic_load_int_relaxed(&gcstate->collecting) && + !_PyErr_Occurred(tstate)) + { + _Py_ScheduleGC(tstate); + } +} + +void +_Py_RunGC(PyThreadState *tstate) +{ + if (tstate->interp->gc.enabled) { + _PyGC_Collect(tstate, 1, _Py_GC_REASON_HEAP); + } +} + +static PyObject * +gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = _PyObject_MallocWithType(tp, size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + PyObject *op = (PyObject *)(mem + presize); + _PyObject_GC_Link(op); + return op; +} + + +PyObject * +_PyObject_GC_New(PyTypeObject *tp) +{ + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) +{ + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *)gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_InitVar(op, tp, nitems); + return op; +} + +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) +{ + const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); + const size_t presize = _PyType_PreHeaderSize(Py_TYPE(op)); + _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); + if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { + return (PyVarObject *)PyErr_NoMemory(); + } + char *mem = (char *)op - presize; + mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize); + if (mem == NULL) { + return (PyVarObject *)PyErr_NoMemory(); + } + op = (PyVarObject *) (mem + presize); + Py_SET_SIZE(op, nitems); + return op; +} + +void +PyObject_GC_Del(void *op) +{ + size_t presize = _PyType_PreHeaderSize(Py_TYPE(op)); + PyGC_Head *g = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op)) { + gc_list_remove(g); +#ifdef Py_DEBUG + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, "Object of type %s is not untracked before destruction", + Py_TYPE(op)->tp_name)) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); +#endif + } + GCState *gcstate = get_gc_state(); + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; + PyObject_Free(((char *)op)-presize); +} + +int +PyObject_GC_IsTracked(PyObject* obj) +{ + if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { + return 1; + } + return 0; +} + +int +PyObject_GC_IsFinalized(PyObject *obj) +{ + if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { + return 1; + } + return 0; +} + +static int +visit_generation(gcvisitobjects_t callback, void *arg, struct gc_generation *gen) +{ + PyGC_Head *gc_list, *gc; + gc_list = &gen->head; + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + return -1; + } + } + return 0; +} + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + GCState *gcstate = get_gc_state(); + int origenstate = gcstate->enabled; + gcstate->enabled = 0; + if (visit_generation(callback, arg, &gcstate->young)) { + goto done; + } + if (visit_generation(callback, arg, &gcstate->old[0])) { + goto done; + } + visit_generation(callback, arg, &gcstate->old[1]); +done: + gcstate->enabled = origenstate; +} + +#endif // Py_GIL_DISABLED diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c new file mode 100644 index 00000000000000..f19362c9573812 --- /dev/null +++ b/Python/gc_free_threading.c @@ -0,0 +1,1853 @@ +// Cyclic garbage collector implementation for free-threaded build. +#include "Python.h" +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit() +#include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_initconfig.h" +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_object.h" +#include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_object_stack.h" +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pydtrace.h" + +#ifdef Py_GIL_DISABLED + +typedef struct _gc_runtime_state GCState; + +#ifdef Py_DEBUG +# define GC_DEBUG +#endif + +// Each thread buffers the count of allocated objects in a thread-local +// variable up to +/- this amount to reduce the overhead of updating +// the global count. +#define LOCAL_ALLOC_COUNT_THRESHOLD 512 + +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +// A linked list of objects using the `ob_tid` field as the next pointer. +// The linked list pointers are distinct from any real thread ids, because the +// thread ids returned by _Py_ThreadId() are also pointers to distinct objects. +// No thread will confuse its own id with a linked list pointer. +struct worklist { + uintptr_t head; +}; + +struct worklist_iter { + uintptr_t *ptr; // pointer to current object + uintptr_t *next; // next value of ptr +}; + +struct visitor_args { + size_t offset; // offset of PyObject from start of block +}; + +// Per-collection state +struct collection_state { + struct visitor_args base; + PyInterpreterState *interp; + GCState *gcstate; + Py_ssize_t collected; + Py_ssize_t uncollectable; + Py_ssize_t long_lived_total; + struct worklist unreachable; + struct worklist legacy_finalizers; + struct worklist wrcb_to_call; + struct worklist objs_to_decref; +}; + +// iterate over a worklist +#define WORKSTACK_FOR_EACH(stack, op) \ + for ((op) = (PyObject *)(stack)->head; (op) != NULL; (op) = (PyObject *)(op)->ob_tid) + +// iterate over a worklist with support for removing the current object +#define WORKSTACK_FOR_EACH_ITER(stack, iter, op) \ + for (worklist_iter_init((iter), &(stack)->head), (op) = (PyObject *)(*(iter)->ptr); \ + (op) != NULL; \ + worklist_iter_init((iter), (iter)->next), (op) = (PyObject *)(*(iter)->ptr)) + +static void +worklist_push(struct worklist *worklist, PyObject *op) +{ + assert(op->ob_tid == 0); + op->ob_tid = worklist->head; + worklist->head = (uintptr_t)op; +} + +static PyObject * +worklist_pop(struct worklist *worklist) +{ + PyObject *op = (PyObject *)worklist->head; + if (op != NULL) { + worklist->head = op->ob_tid; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + } + return op; +} + +static void +worklist_iter_init(struct worklist_iter *iter, uintptr_t *next) +{ + iter->ptr = next; + PyObject *op = (PyObject *)*(iter->ptr); + if (op) { + iter->next = &op->ob_tid; + } +} + +static void +worklist_remove(struct worklist_iter *iter) +{ + PyObject *op = (PyObject *)*(iter->ptr); + *(iter->ptr) = op->ob_tid; + op->ob_tid = 0; + iter->next = iter->ptr; +} + +static inline int +gc_is_unreachable(PyObject *op) +{ + return (op->ob_gc_bits & _PyGC_BITS_UNREACHABLE) != 0; +} + +static void +gc_set_unreachable(PyObject *op) +{ + op->ob_gc_bits |= _PyGC_BITS_UNREACHABLE; +} + +static void +gc_clear_unreachable(PyObject *op) +{ + op->ob_gc_bits &= ~_PyGC_BITS_UNREACHABLE; +} + +// Initialize the `ob_tid` field to zero if the object is not already +// initialized as unreachable. +static void +gc_maybe_init_refs(PyObject *op) +{ + if (!gc_is_unreachable(op)) { + gc_set_unreachable(op); + op->ob_tid = 0; + } +} + +static inline Py_ssize_t +gc_get_refs(PyObject *op) +{ + return (Py_ssize_t)op->ob_tid; +} + +static inline void +gc_add_refs(PyObject *op, Py_ssize_t refs) +{ + assert(_PyObject_GC_IS_TRACKED(op)); + op->ob_tid += refs; +} + +static inline void +gc_decref(PyObject *op) +{ + op->ob_tid -= 1; +} + +static void +disable_deferred_refcounting(PyObject *op) +{ + if (_PyObject_HasDeferredRefcount(op)) { + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + op->ob_ref_shared -= (1 << _Py_REF_SHARED_SHIFT); + } +} + +static Py_ssize_t +merge_refcount(PyObject *op, Py_ssize_t extra) +{ + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + + Py_ssize_t refcount = Py_REFCNT(op); + refcount += extra; + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // No atomics necessary; all other threads in this interpreter are paused. + op->ob_tid = 0; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_SHARED(refcount, _Py_REF_MERGED); + return refcount; +} + +static void +gc_restore_tid(PyObject *op) +{ + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + mi_segment_t *segment = _mi_ptr_segment(op); + if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { + op->ob_tid = 0; + } + else { + // NOTE: may change ob_tid if the object was re-initialized by + // a different thread or its segment was abandoned and reclaimed. + // The segment thread id might be zero, in which case we should + // ensure the refcounts are now merged. + op->ob_tid = segment->thread_id; + if (op->ob_tid == 0) { + merge_refcount(op, 0); + } + } +} + +static void +gc_restore_refs(PyObject *op) +{ + if (gc_is_unreachable(op)) { + gc_restore_tid(op); + gc_clear_unreachable(op); + } +} + +// Given a mimalloc memory block return the PyObject stored in it or NULL if +// the block is not allocated or the object is not tracked or is immortal. +static PyObject * +op_from_block(void *block, void *arg, bool include_frozen) +{ + struct visitor_args *a = arg; + if (block == NULL) { + return NULL; + } + PyObject *op = (PyObject *)((char*)block + a->offset); + assert(PyObject_IS_GC(op)); + if (!_PyObject_GC_IS_TRACKED(op)) { + return NULL; + } + if (!include_frozen && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + return NULL; + } + return op; +} + +static int +gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Offset of PyObject header from start of memory block. + Py_ssize_t offset_base = 0; + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + offset_base += 2 * sizeof(size_t); + } + + // Objects with Py_TPFLAGS_PREHEADER have two extra fields + Py_ssize_t offset_pre = offset_base + 2 * sizeof(PyObject*); + + // visit each thread's heaps for GC objects + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + struct _mimalloc_thread_state *m = &((_PyThreadStateImpl *)p)->mimalloc; + if (!_Py_atomic_load_int(&m->initialized)) { + // The thread may not have called tstate_mimalloc_bind() yet. + continue; + } + + arg->offset = offset_base; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC], true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC_PRE], true, + visitor, arg)) { + return -1; + } + } + + // visit blocks in the per-interpreter abandoned pool (from dead threads) + mi_abandoned_pool_t *pool = &interp->mimalloc.abandoned_pool; + arg->offset = offset_base; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC, true, + visitor, arg)) { + return -1; + } + arg->offset = offset_pre; + if (!_mi_abandoned_pool_visit_blocks(pool, _Py_MIMALLOC_HEAP_GC_PRE, true, + visitor, arg)) { + return -1; + } + return 0; +} + +// Visits all GC objects in the interpreter's heaps. +// NOTE: It is not safe to allocate or free any mimalloc managed memory while +// this function is running. +static int +gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, + struct visitor_args *arg) +{ + // Other threads in the interpreter must be paused so that we can safely + // traverse their heaps. + assert(interp->stoptheworld.world_stopped); + + int err; + HEAD_LOCK(&_PyRuntime); + err = gc_visit_heaps_lock_held(interp, visitor, arg); + HEAD_UNLOCK(&_PyRuntime); + return err; +} + +static void +merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) +{ + struct _brc_thread_state *brc = &tstate->brc; + _PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge); + + PyObject *op; + while ((op = _PyObjectStack_Pop(&brc->local_objects_to_merge)) != NULL) { + // Subtract one when merging because the queue had a reference. + Py_ssize_t refcount = merge_refcount(op, -1); + + if (!_PyObject_GC_IS_TRACKED(op) && refcount == 0) { + // GC objects with zero refcount are handled subsequently by the + // GC as if they were cyclic trash, but we have to handle dead + // non-GC objects here. Add one to the refcount so that we can + // decref and deallocate the object once we start the world again. + op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + worklist_push(&state->objs_to_decref, op); + } + } +} + +static void +merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + merge_queued_objects((_PyThreadStateImpl *)p, state); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static void +process_delayed_frees(PyInterpreterState *interp) +{ + // In STW status, we can observe the latest write sequence by + // advancing the write sequence immediately. + _Py_qsbr_advance(&interp->qsbr); + _PyThreadStateImpl *current_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + _Py_qsbr_quiescent_state(current_tstate->qsbr); + HEAD_LOCK(&_PyRuntime); + PyThreadState *tstate = interp->threads.head; + while (tstate != NULL) { + _PyMem_ProcessDelayed(tstate); + tstate = (PyThreadState *)tstate->next; + } + HEAD_UNLOCK(&_PyRuntime); +} + +// Subtract an incoming reference from the computed "gc_refs" refcount. +static int +visit_decref(PyObject *op, void *arg) +{ + if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op)) { + // If update_refs hasn't reached this object yet, mark it + // as (tentatively) unreachable and initialize ob_tid to zero. + gc_maybe_init_refs(op); + gc_decref(op); + } + return 0; +} + +// Compute the number of external references to objects in the heap +// by subtracting internal references from the refcount. The difference is +// computed in the ob_tid field (we restore it later). +static bool +update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + // Exclude immortal objects from garbage collection + if (_Py_IsImmortal(op)) { + op->ob_tid = 0; + _PyObject_GC_UNTRACK(op); + gc_clear_unreachable(op); + return true; + } + + Py_ssize_t refcount = Py_REFCNT(op); + refcount -= _PyObject_HasDeferredRefcount(op); + _PyObject_ASSERT(op, refcount >= 0); + + if (refcount > 0 && !_PyObject_HasDeferredRefcount(op)) { + // Untrack tuples and dicts as necessary in this pass, but not objects + // with zero refcount, which we will want to collect. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + else if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } + } + } + + // We repurpose ob_tid to compute "gc_refs", the number of external + // references to the object (i.e., from outside the GC heaps). This means + // that ob_tid is no longer a valid thread id until it is restored by + // scan_heap_visitor(). Until then, we cannot use the standard reference + // counting functions or allow other threads to run Python code. + gc_maybe_init_refs(op); + + // Add the actual refcount to ob_tid. + gc_add_refs(op, refcount); + + // Subtract internal references from ob_tid. Objects with ob_tid > 0 + // are directly reachable from outside containers, and so can't be + // collected. + Py_TYPE(op)->tp_traverse(op, visit_decref, NULL); + return true; +} + +static int +visit_clear_unreachable(PyObject *op, _PyObjectStack *stack) +{ + if (gc_is_unreachable(op)) { + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + gc_clear_unreachable(op); + return _PyObjectStack_Push(stack, op); + } + return 0; +} + +// Transitively clear the unreachable bit on all objects reachable from op. +static int +mark_reachable(PyObject *op) +{ + _PyObjectStack stack = { NULL }; + do { + traverseproc traverse = Py_TYPE(op)->tp_traverse; + if (traverse(op, (visitproc)&visit_clear_unreachable, &stack) < 0) { + _PyObjectStack_Clear(&stack); + return -1; + } + op = _PyObjectStack_Pop(&stack); + } while (op != NULL); + return 0; +} + +#ifdef GC_DEBUG +static bool +validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, + "refcount is too small"); + return true; +} +#endif + +static bool +mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, + "refcount is too small"); + + if (gc_is_unreachable(op) && gc_get_refs(op) != 0) { + // Object is reachable but currently marked as unreachable. + // Mark it as reachable and traverse its pointers to find + // any other object that may be directly reachable from it. + gc_clear_unreachable(op); + + // Transitively mark reachable objects by clearing the unreachable flag. + if (mark_reachable(op) < 0) { + return false; + } + } + + return true; +} + +/* Return true if object has a pre-PEP 442 finalization method. */ +static int +has_legacy_finalizer(PyObject *op) +{ + return Py_TYPE(op)->tp_del != NULL; +} + +static bool +scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct collection_state *state = (struct collection_state *)args; + if (gc_is_unreachable(op)) { + // Disable deferred refcounting for unreachable objects so that they + // are collected immediately after finalization. + disable_deferred_refcounting(op); + + // Merge and add one to the refcount to prevent deallocation while we + // are holding on to it in a worklist. + merge_refcount(op, 1); + + if (has_legacy_finalizer(op)) { + // would be unreachable, but has legacy finalizer + gc_clear_unreachable(op); + worklist_push(&state->legacy_finalizers, op); + } + else { + worklist_push(&state->unreachable, op); + } + } + else { + // object is reachable, restore `ob_tid`; we're done with these objects + gc_restore_tid(op); + state->long_lived_total++; + } + + return true; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state); + +static int +deduce_unreachable_heap(PyInterpreterState *interp, + struct collection_state *state) +{ + // Identify objects that are directly reachable from outside the GC heap + // by computing the difference between the refcount and the number of + // incoming references. + gc_visit_heaps(interp, &update_refs, &state->base); + +#ifdef GC_DEBUG + // Check that all objects are marked as unreachable and that the computed + // reference count difference (stored in `ob_tid`) is non-negative. + gc_visit_heaps(interp, &validate_gc_objects, &state->base); +#endif + + // Transitively mark reachable objects by clearing the + // _PyGC_BITS_UNREACHABLE flag. + if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + return -1; + } + + // Identify remaining unreachable objects and push them onto a stack. + // Restores ob_tid for reachable objects. + gc_visit_heaps(interp, &scan_heap_visitor, &state->base); + + if (state->legacy_finalizers.head) { + // There may be objects reachable from legacy finalizers that are in + // the unreachable set. We need to mark them as reachable. + if (move_legacy_finalizer_reachable(state) < 0) { + return -1; + } + } + + return 0; +} + +static int +move_legacy_finalizer_reachable(struct collection_state *state) +{ + // Clear the reachable bit on all objects transitively reachable + // from the objects with legacy finalizers. + PyObject *op; + WORKSTACK_FOR_EACH(&state->legacy_finalizers, op) { + if (mark_reachable(op) < 0) { + return -1; + } + } + + // Move the reachable objects from the unreachable worklist to the legacy + // finalizer worklist. + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + worklist_remove(&iter); + worklist_push(&state->legacy_finalizers, op); + } + } + + return 0; +} + +// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are +// enqueued in `wrcb_to_call`, but not invoked yet. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable to ensure their + // callbacks will not be executed later from a `tp_clear()` + // inside delete_garbage(). That would be unsafe: it could + // resurrect a dead object or access a an already cleared object. + // See bpo-38006 for one example. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs, and enqueue the weakrefs with callbacks + // that must be called into wrcb_to_call. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + + // We do not invoke callbacks for weakrefs that are themselves + // unreachable. This is partly for historical reasons: weakrefs + // predate safe object finalization, and a weakref that is itself + // unreachable may have a callback that resurrects other + // unreachable objects. + if (wr->wr_callback == NULL || gc_is_unreachable((PyObject *)wr)) { + continue; + } + + // Create a new reference so that wr can't go away before we can + // process it again. + merge_refcount((PyObject *)wr, 1); + + // Enqueue weakref to be called later. + worklist_push(&state->wrcb_to_call, (PyObject *)wr); + } + } +} + +static void +call_weakref_callbacks(struct collection_state *state) +{ + // Invoke the callbacks we decided to honor. + PyObject *op; + while ((op = worklist_pop(&state->wrcb_to_call)) != NULL) { + _PyObject_ASSERT(op, PyWeakref_Check(op)); + + PyWeakReference *wr = (PyWeakReference *)op; + PyObject *callback = wr->wr_callback; + _PyObject_ASSERT(op, callback != NULL); + + /* copy-paste of weakrefobject.c's handle_callback() */ + PyObject *temp = PyObject_CallOneArg(callback, (PyObject *)wr); + if (temp == NULL) { + PyErr_WriteUnraisable(callback); + } + else { + Py_DECREF(temp); + } + + Py_DECREF(op); // drop worklist reference + } +} + + +static GCState * +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + + +void +_PyGC_InitState(GCState *gcstate) +{ + // TODO: move to pycore_runtime_init.h once the incremental GC lands. + gcstate->young.threshold = 2000; +} + + +PyStatus +_PyGC_Init(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + + // gh-117783: immortalize objects that would use deferred refcounting + // once the first non-main thread is created (but not in subinterpreters). + gcstate->immortalize = _Py_IsMainInterpreter(interp) ? 0 : -1; + + gcstate->garbage = PyList_New(0); + if (gcstate->garbage == NULL) { + return _PyStatus_NO_MEMORY(); + } + + gcstate->callbacks = PyList_New(0); + if (gcstate->callbacks == NULL) { + return _PyStatus_NO_MEMORY(); + } + + return _PyStatus_OK(); +} + +static void +debug_cycle(const char *msg, PyObject *op) +{ + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); +} + +/* Run first-time finalizers (if any) on all the objects in collectable. + * Note that this may remove some (or even all) of the objects from the + * list, due to refcounts falling to 0. + */ +static void +finalize_garbage(struct collection_state *state) +{ + // NOTE: the unreachable worklist holds a strong reference to the object + // to prevent it from being deallocated while we are holding on to it. + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (!_PyGC_FINALIZED(op)) { + destructor finalize = Py_TYPE(op)->tp_finalize; + if (finalize != NULL) { + _PyGC_SET_FINALIZED(op); + finalize(op); + assert(!_PyErr_Occurred(_PyThreadState_GET())); + } + } + } +} + +// Break reference cycles by clearing the containers involved. +static void +delete_garbage(struct collection_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = state->gcstate; + + assert(!_PyErr_Occurred(tstate)); + + PyObject *op; + while ((op = worklist_pop(&state->objs_to_decref)) != NULL) { + Py_DECREF(op); + } + + while ((op = worklist_pop(&state->unreachable)) != NULL) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + + // Clear the unreachable flag. + gc_clear_unreachable(op); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object might have been untracked by some other tp_clear() call. + Py_DECREF(op); // drop the reference from the worklist + continue; + } + + state->collected++; + + if (gcstate->debug & _PyGC_DEBUG_SAVEALL) { + assert(gcstate->garbage != NULL); + if (PyList_Append(gcstate->garbage, op) < 0) { + _PyErr_Clear(tstate); + } + } + else { + inquiry clear = Py_TYPE(op)->tp_clear; + if (clear != NULL) { + (void) clear(op); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in tp_clear of %s", + Py_TYPE(op)->tp_name); + } + } + } + + Py_DECREF(op); // drop the reference from the worklist + } +} + +static void +handle_legacy_finalizers(struct collection_state *state) +{ + GCState *gcstate = state->gcstate; + assert(gcstate->garbage != NULL); + + PyObject *op; + while ((op = worklist_pop(&state->legacy_finalizers)) != NULL) { + state->uncollectable++; + + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + debug_cycle("uncollectable", op); + } + + if ((gcstate->debug & _PyGC_DEBUG_SAVEALL) || has_legacy_finalizer(op)) { + if (PyList_Append(gcstate->garbage, op) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(op); // drop worklist reference + } +} + +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + // TODO +} + +// Traversal callback for handle_resurrected_objects. +static int +visit_decref_unreachable(PyObject *op, void *data) +{ + if (gc_is_unreachable(op) && _PyObject_GC_IS_TRACKED(op)) { + op->ob_ref_local -= 1; + } + return 0; +} + +// Handle objects that may have resurrected after a call to 'finalize_garbage'. +static int +handle_resurrected_objects(struct collection_state *state) +{ + // First, find externally reachable objects by computing the reference + // count difference in ob_ref_local. We can't use ob_tid here because + // that's already used to store the unreachable worklist. + PyObject *op; + struct worklist_iter iter; + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + assert(gc_is_unreachable(op)); + assert(_Py_REF_IS_MERGED(op->ob_ref_shared)); + + if (!_PyObject_GC_IS_TRACKED(op)) { + // Object was untracked by a finalizer. Schedule it for a Py_DECREF + // after we finish with the stop-the-world pause. + gc_clear_unreachable(op); + worklist_remove(&iter); + worklist_push(&state->objs_to_decref, op); + continue; + } + + Py_ssize_t refcount = (op->ob_ref_shared >> _Py_REF_SHARED_SHIFT); + if (refcount > INT32_MAX) { + // The refcount is too big to fit in `ob_ref_local`. Mark the + // object as immortal and bail out. + gc_clear_unreachable(op); + worklist_remove(&iter); + _Py_SetImmortal(op); + continue; + } + + op->ob_ref_local += (uint32_t)refcount; + + // Subtract one to account for the reference from the worklist. + op->ob_ref_local -= 1; + + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + (visitproc)visit_decref_unreachable, + NULL); + } + + // Find resurrected objects + bool any_resurrected = false; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + int32_t gc_refs = (int32_t)op->ob_ref_local; + op->ob_ref_local = 0; // restore ob_ref_local + + _PyObject_ASSERT(op, gc_refs >= 0); + + if (gc_is_unreachable(op) && gc_refs > 0) { + // Clear the unreachable flag on any transitively reachable objects + // from this one. + any_resurrected = true; + gc_clear_unreachable(op); + if (mark_reachable(op) < 0) { + return -1; + } + } + } + + if (any_resurrected) { + // Remove resurrected objects from the unreachable list. + WORKSTACK_FOR_EACH_ITER(&state->unreachable, &iter, op) { + if (!gc_is_unreachable(op)) { + _PyObject_ASSERT(op, Py_REFCNT(op) > 1); + worklist_remove(&iter); + merge_refcount(op, -1); // remove worklist reference + } + } + } + +#ifdef GC_DEBUG + WORKSTACK_FOR_EACH(&state->unreachable, op) { + _PyObject_ASSERT(op, gc_is_unreachable(op)); + _PyObject_ASSERT(op, _PyObject_GC_IS_TRACKED(op)); + _PyObject_ASSERT(op, op->ob_ref_local == 0); + _PyObject_ASSERT(op, _Py_REF_IS_MERGED(op->ob_ref_shared)); + } +#endif + + return 0; +} + + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + +static void +cleanup_worklist(struct worklist *worklist) +{ + PyObject *op; + while ((op = worklist_pop(worklist)) != NULL) { + gc_clear_unreachable(op); + Py_DECREF(op); + } +} + +static bool +gc_should_collect(GCState *gcstate) +{ + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int threshold = gcstate->young.threshold; + if (count <= threshold || threshold == 0 || !gcstate->enabled) { + return false; + } + // Avoid quadratic behavior by scaling threshold to the number of live + // objects. A few tests rely on immediate scheduling of the GC so we ignore + // the scaled threshold if generations[1].threshold is set to zero. + return (count > gcstate->long_lived_total / 4 || + gcstate->old[0].threshold == 0); +} + +static void +record_allocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + // We buffer the allocation count to avoid the overhead of atomic + // operations for every allocation. + gc->alloc_count++; + if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { + // TODO: Use Py_ssize_t for the generation count. + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + gc->alloc_count = 0; + + if (gc_should_collect(gcstate) && + !_Py_atomic_load_int_relaxed(&gcstate->collecting)) + { + _Py_ScheduleGC(tstate); + } + } +} + +static void +record_deallocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + gc->alloc_count--; + if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + gc->alloc_count = 0; + } +} + +static void +gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int generation) +{ + _PyEval_StopTheWorld(interp); + + // update collection and allocation counters + if (generation+1 < NUM_GENERATIONS) { + state->gcstate->old[generation].count += 1; + } + + state->gcstate->young.count = 0; + for (int i = 1; i <= generation; ++i) { + state->gcstate->old[i-1].count = 0; + } + + // merge refcounts for all queued objects + merge_all_queued_objects(interp, state); + process_delayed_frees(interp); + + // Find unreachable objects + int err = deduce_unreachable_heap(interp, state); + if (err < 0) { + _PyEval_StartTheWorld(interp); + goto error; + } + + // Print debugging information. + if (interp->gc.debug & _PyGC_DEBUG_COLLECTABLE) { + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + debug_cycle("collectable", op); + } + } + + // Record the number of live GC objects + interp->gc.long_lived_total = state->long_lived_total; + + // Clear weakrefs and enqueue callbacks (but do not call them). + clear_weakrefs(state); + _PyEval_StartTheWorld(interp); + + // Deallocate any object from the refcount merge step + cleanup_worklist(&state->objs_to_decref); + + // Call weakref callbacks and finalizers after unpausing other threads to + // avoid potential deadlocks. + call_weakref_callbacks(state); + finalize_garbage(state); + + // Handle any objects that may have resurrected after the finalization. + _PyEval_StopTheWorld(interp); + err = handle_resurrected_objects(state); + // Clear free lists in all threads + _PyGC_ClearAllFreeLists(interp); + _PyEval_StartTheWorld(interp); + + if (err < 0) { + goto error; + } + + // Call tp_clear on objects in the unreachable set. This will cause + // the reference cycles to be broken. It may also cause some objects + // to be freed. + delete_garbage(state); + + // Append objects with legacy finalizers to the "gc.garbage" list. + handle_legacy_finalizers(state); + return; + +error: + cleanup_worklist(&state->unreachable); + cleanup_worklist(&state->legacy_finalizers); + cleanup_worklist(&state->wrcb_to_call); + cleanup_worklist(&state->objs_to_decref); + PyErr_NoMemory(); + PyErr_FormatUnraisable("Out of memory during garbage collection"); +} + +/* This is the main function. Read this to understand how the + * collection process works. */ +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + Py_ssize_t m = 0; /* # objects collected */ + Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (reason == _Py_GC_REASON_HEAP && !gc_should_collect(gcstate)) { + // Don't collect if the threshold is not exceeded. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); + } + + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } + + PyInterpreterState *interp = tstate->interp; + + struct collection_state state = { + .interp = interp, + .gcstate = gcstate, + }; + + gc_collect_internal(interp, &state, generation); + + m = state.collected; + n = state.uncollectable; + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + double d = PyTime_AsSecondsDouble(t2 - t1); + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + n+m, n, d); + } + + // Clear the current thread's free-list again. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + _PyObject_ClearFreeLists(&tstate_impl->freelists, 0); + + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); + } + else { + PyErr_FormatUnraisable("Exception ignored in garbage collection"); + } + } + + /* Update stats */ + struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; + + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(n + m); + } + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); + } + + assert(!_PyErr_Occurred(tstate)); + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; +} + +struct get_referrers_args { + struct visitor_args base; + PyObject *objs; + struct worklist results; +}; + +static int +referrersvisit(PyObject* obj, void *arg) +{ + PyObject *objs = arg; + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(objs); i++) { + if (PyTuple_GET_ITEM(objs, i) == obj) { + return 1; + } + } + return 0; +} + +static bool +visit_get_referrers(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_referrers_args *arg = (struct get_referrers_args *)args; + if (Py_TYPE(op)->tp_traverse(op, referrersvisit, arg->objs)) { + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->results, op); + } + + return true; +} + +PyObject * +_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the PyListObject during + // gc_visit_heaps() because PyList_Append() may reclaim an abandoned + // mimalloc segments while we are traversing them. + struct get_referrers_args args = { .objs = objs }; + gc_visit_heaps(interp, &visit_get_referrers, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + if (op != objs && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.results)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +struct get_objects_args { + struct visitor_args base; + struct worklist objects; +}; + +static bool +visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op == NULL) { + return true; + } + + struct get_objects_args *arg = (struct get_objects_args *)args; + op->ob_tid = 0; // we will restore the refcount later + worklist_push(&arg->objects, op); + + return true; +} + +PyObject * +_PyGC_GetObjects(PyInterpreterState *interp, int generation) +{ + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + _PyEval_StopTheWorld(interp); + + // Append all objects to a worklist. This abuses ob_tid. We will restore + // it later. NOTE: We can't append to the list during gc_visit_heaps() + // because PyList_Append() may reclaim an abandoned mimalloc segment + // while we are traversing it. + struct get_objects_args args = { 0 }; + gc_visit_heaps(interp, &visit_get_objects, &args.base); + + bool error = false; + PyObject *op; + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + if (op != result && PyList_Append(result, op) < 0) { + error = true; + break; + } + } + + // In case of error, clear the remaining worklist + while ((op = worklist_pop(&args.objects)) != NULL) { + gc_restore_tid(op); + } + + _PyEval_StartTheWorld(interp); + + if (error) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static bool +visit_freeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits |= _PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Freeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_freeze, &args); + _PyEval_StartTheWorld(interp); +} + +static bool +visit_unfreeze(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL) { + op->ob_gc_bits &= ~_PyGC_BITS_FROZEN; + } + return true; +} + +void +_PyGC_Unfreeze(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_unfreeze, &args); + _PyEval_StartTheWorld(interp); +} + +struct count_frozen_args { + struct visitor_args base; + Py_ssize_t count; +}; + +static bool +visit_count_frozen(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, true); + if (op != NULL && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + struct count_frozen_args *arg = (struct count_frozen_args *)args; + arg->count++; + } + return true; +} + +Py_ssize_t +_PyGC_GetFreezeCount(PyInterpreterState *interp) +{ + struct count_frozen_args args = { .count = 0 }; + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &visit_count_frozen, &args.base); + _PyEval_StartTheWorld(interp); + return args.count; +} + +/* C API for controlling the state of the garbage collector */ +int +PyGC_Enable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 1; + return old_state; +} + +int +PyGC_Disable(void) +{ + GCState *gcstate = get_gc_state(); + int old_state = gcstate->enabled; + gcstate->enabled = 0; + return old_state; +} + +int +PyGC_IsEnabled(void) +{ + GCState *gcstate = get_gc_state(); + return gcstate->enabled; +} + +/* Public API to invoke gc.collect() from C */ +Py_ssize_t +PyGC_Collect(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + GCState *gcstate = &tstate->interp->gc; + + if (!gcstate->enabled) { + return 0; + } + + Py_ssize_t n; + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); + + return n; +} + +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +{ + return gc_collect_main(tstate, generation, reason); +} + +void +_PyGC_CollectNoFail(PyThreadState *tstate) +{ + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); +} + +void +_PyGC_DumpShutdownStats(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (!(gcstate->debug & _PyGC_DEBUG_SAVEALL) + && gcstate->garbage != NULL && PyList_GET_SIZE(gcstate->garbage) > 0) { + const char *message; + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + message = "gc: %zd uncollectable objects at shutdown"; + } + else { + message = "gc: %zd uncollectable objects at shutdown; " \ + "use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; + } + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(gcstate->garbage))) + { + PyErr_WriteUnraisable(NULL); + } + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + PyObject *repr = NULL, *bytes = NULL; + repr = PyObject_Repr(gcstate->garbage); + if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) { + PyErr_WriteUnraisable(gcstate->garbage); + } + else { + PySys_WriteStderr( + " %s\n", + PyBytes_AS_STRING(bytes) + ); + } + Py_XDECREF(repr); + Py_XDECREF(bytes); + } + } +} + + +void +_PyGC_Fini(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + Py_CLEAR(gcstate->garbage); + Py_CLEAR(gcstate->callbacks); + + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ +} + +/* for debugging */ + +#ifdef Py_DEBUG +static int +visit_validate(PyObject *op, void *parent_raw) +{ + PyObject *parent = _PyObject_CAST(parent_raw); + if (_PyObject_IsFreed(op)) { + _PyObject_ASSERT_FAILED_MSG(parent, + "PyObject_GC_Track() object is not valid"); + } + return 0; +} +#endif + + +/* extension modules might be compiled with GC support so these + functions must always be available */ + +void +PyObject_GC_Track(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT_FAILED_MSG(op, + "object already tracked " + "by the garbage collector"); + } + _PyObject_GC_TRACK(op); + +#ifdef Py_DEBUG + /* Check that the object is valid: validate objects traversed + by tp_traverse() */ + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void)traverse(op, visit_validate, op); +#endif +} + +void +PyObject_GC_UnTrack(void *op_raw) +{ + PyObject *op = _PyObject_CAST(op_raw); + /* Obscure: the Py_TRASHCAN mechanism requires that we be able to + * call PyObject_GC_UnTrack twice on an object. + */ + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } +} + +int +PyObject_IS_GC(PyObject *obj) +{ + return _PyObject_IS_GC(obj); +} + +void +_Py_ScheduleGC(PyThreadState *tstate) +{ + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } +} + +void +_PyObject_GC_Link(PyObject *op) +{ + record_allocation(_PyThreadState_GET()); +} + +void +_Py_RunGC(PyThreadState *tstate) +{ + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; + } + gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); +} + +static PyObject * +gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (basicsize > PY_SSIZE_T_MAX - presize) { + return _PyErr_NoMemory(tstate); + } + size_t size = presize + basicsize; + char *mem = _PyObject_MallocWithType(tp, size); + if (mem == NULL) { + return _PyErr_NoMemory(tstate); + } + if (presize) { + ((PyObject **)mem)[0] = NULL; + ((PyObject **)mem)[1] = NULL; + } + PyObject *op = (PyObject *)(mem + presize); + record_allocation(tstate); + return op; +} + +PyObject * +_PyObject_GC_New(PyTypeObject *tp) +{ + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) +{ + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size_t presize = _PyType_PreHeaderSize(tp); + size_t size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *)gc_alloc(tp, size, presize); + if (op == NULL) { + return NULL; + } + _PyObject_InitVar(op, tp, nitems); + return op; +} + +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + +PyVarObject * +_PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) +{ + const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); + const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); + if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { + return (PyVarObject *)PyErr_NoMemory(); + } + char *mem = (char *)op - presize; + mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize); + if (mem == NULL) { + return (PyVarObject *)PyErr_NoMemory(); + } + op = (PyVarObject *) (mem + presize); + Py_SET_SIZE(op, nitems); + return op; +} + +void +PyObject_GC_Del(void *op) +{ + size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); +#ifdef Py_DEBUG + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, "Object of type %s is not untracked before destruction", + ((PyObject*)op)->ob_type->tp_name)) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); +#endif + } + + record_deallocation(_PyThreadState_GET()); + PyObject *self = (PyObject *)op; + if (_PyObject_GC_IS_SHARED_INLINE(self)) { + _PyObject_FreeDelayed(((char *)op)-presize); + } + else { + PyObject_Free(((char *)op)-presize); + } +} + +int +PyObject_GC_IsTracked(PyObject* obj) +{ + return _PyObject_GC_IS_TRACKED(obj); +} + +int +PyObject_GC_IsFinalized(PyObject *obj) +{ + return _PyGC_FINALIZED(obj); +} + +struct custom_visitor_args { + struct visitor_args base; + gcvisitobjects_t callback; + void *arg; +}; + +static bool +custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + struct custom_visitor_args *wrapper = (struct custom_visitor_args *)args; + if (!wrapper->callback(op, wrapper->arg)) { + return false; + } + + return true; +} + +// gh-117783: Immortalize objects that use deferred reference counting to +// temporarily work around scaling bottlenecks. +static bool +immortalize_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op != NULL && _PyObject_HasDeferredRefcount(op)) { + _Py_SetImmortal(op); + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + } + return true; +} + +void +_PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp) +{ + struct visitor_args args; + _PyEval_StopTheWorld(interp); + if (interp->gc.immortalize == 0) { + gc_visit_heaps(interp, &immortalize_visitor, &args); + interp->gc.immortalize = 1; + } + _PyEval_StartTheWorld(interp); +} + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct custom_visitor_args wrapper = { + .callback = callback, + .arg = arg, + }; + + _PyEval_StopTheWorld(interp); + gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); + _PyEval_StartTheWorld(interp); +} + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + * Free-threading version: Since freelists are managed per thread, + * GC should clear all freelists by traversing all threads. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; + while (tstate != NULL) { + _PyObject_ClearFreeLists(&tstate->freelists, 0); + tstate = (_PyThreadStateImpl *)tstate->base.next; + } + HEAD_UNLOCK(&_PyRuntime); +} + +#endif // Py_GIL_DISABLED diff --git a/Python/gc_gil.c b/Python/gc_gil.c new file mode 100644 index 00000000000000..48646c7af86b7f --- /dev/null +++ b/Python/gc_gil.c @@ -0,0 +1,17 @@ +#include "Python.h" +#include "pycore_pystate.h" // _Py_ClearFreeLists() + +#ifndef Py_GIL_DISABLED + +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + */ +void +_PyGC_ClearAllFreeLists(PyInterpreterState *interp) +{ + _PyObject_ClearFreeLists(&interp->object_state.freelists, 0); +} + +#endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6fbe80fe03a128..61057221291c0a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,1476 +1,2800 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: // Python/bytecodes.c // Do not edit! - TARGET(NOP) { - DISPATCH(); - } - - TARGET(RESUME) { - PREDICTED(RESUME); - static_assert(0 == 0, "incorrect cache size"); - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr--; - } - else { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - next_instr[-1].op.code = RESUME_CHECK; - } - DISPATCH(); - } - - TARGET(RESUME_CHECK) { -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#ifdef TIER_TWO + #error "This file is for Tier 1 only" #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } +#define TIER_ONE 1 - TARGET(INSTRUMENTED_RESUME) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; + + TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); } - next_instr--; + STAT_INC(BINARY_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); } - else { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->prev_instr != next_instr-1) { - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - DISPATCH(); - } + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_FAST_CHECK) { - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_FAST) { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_FAST_AND_CLEAR) { - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_FAST_LOAD_FAST) { - PyObject *value1; - PyObject *value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; - DISPATCH(); - } - - TARGET(LOAD_CONST) { - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_FAST) { - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - STACK_SHRINK(1); + TARGET(BINARY_OP_ADD_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_FAST_LOAD_FAST) { - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; + TARGET(BINARY_OP_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_FAST_STORE_FAST) { - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(next_instr->op.code == STORE_FAST); + _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(POP_TOP) { - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); - STACK_SHRINK(1); + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(PUSH_NULL) { - PyObject *res; - res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(END_FOR) { - PyObject *value; - // POP_TOP - value = stack_pointer[-1]; + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - Py_DECREF(value); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - // POP_TOP - value = stack_pointer[-2]; + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_FLOAT { - Py_DECREF(value); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } - STACK_SHRINK(2); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_END_FOR) { - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; - } - PyErr_SetRaisedException(NULL); + TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - Py_DECREF(receiver); - Py_DECREF(value); - STACK_SHRINK(2); + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(END_SEND) { - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + } + PyStackRef_CLOSE(container); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_END_SEND) { - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { - goto error; + TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); } - PyErr_SetRaisedException(NULL); + STAT_INC(BINARY_SUBSCR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(UNARY_NEGATIVE) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNARY_NOT) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; + TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) goto pop_2_error; + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(TO_BOOL) { - PREDICTED(TO_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; - stack_pointer[-1] = res; - next_instr += 3; - DISPATCH(); - } - - TARGET(TO_BOOL_BOOL) { - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - next_instr += 3; - DISPATCH(); + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef container_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + container_st = stack_pointer[-2]; + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + STACK_SHRINK(2); + new_frame->localsplus[0] = container_st; + new_frame->localsplus[1] = sub_st; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(TO_BOOL_INT) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - Py_DECREF(value); - res = Py_True; - } - stack_pointer[-1] = res; - next_instr += 3; + TARGET(BINARY_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(TO_BOOL_LIST) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - next_instr += 3; + TARGET(BINARY_SUBSCR_STR_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(TO_BOOL_NONE) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; - next_instr += 3; + TARGET(BINARY_SUBSCR_TUPLE_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(TO_BOOL_STR) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + _PyStackRef keys; + _PyStackRef *values; + _PyStackRef map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + PyStackRef_CLOSE(keys); + if (true) { stack_pointer += -1 - oparg; goto error; } } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } - stack_pointer[-1] = res; - next_instr += 3; - DISPATCH(); - } - - TARGET(TO_BOOL_ALWAYS_TRUE) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&next_instr[1].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; - next_instr += 3; + PyStackRef_CLOSE(keys); + if (map_o == NULL) { stack_pointer += -1 - oparg; goto error; } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNARY_INVERT) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (list_o == NULL) { stack_pointer += -oparg; goto error; } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg*2; goto error; } } - // _BINARY_OP_MULTIPLY_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + if (map_o == NULL) { stack_pointer += -oparg*2; goto error; } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_ADD_INT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { + goto error; } - // _BINARY_OP_ADD_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } + Py_DECREF(item); } - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + if (err != 0) { + Py_DECREF(set_o); + if (true) { stack_pointer += -oparg; goto error; } + } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef step = PyStackRef_NULL; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } } - // _BINARY_OP_MULTIPLY_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); } - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + if (str_o == NULL) { stack_pointer += -oparg; goto error; } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) { stack_pointer += -oparg; goto error; } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CACHE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); DISPATCH(); } - TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE + TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; { - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; - DISPATCH(); - } - - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; + /* Skip 2 cache entries */ + // _CALL { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable_o) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - // _BINARY_OP_INPLACE_ADD_UNICODE + // _CHECK_PERIODIC { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); } - STACK_SHRINK(2); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR) { - PREDICTED(BINARY_SUBSCR); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; - PyObject *res; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef null; + _PyStackRef callable; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + goto error; + } + PyStackRef_CLOSE(callable); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + goto start_frame; + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef self; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; + } + // _CHECK_FUNCTION_VERSION + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } DISPATCH(); } - TARGET(BINARY_SLICE) { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; + TARGET(CALL_BOUND_METHOD_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + // _CHECK_METHOD_VERSION + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + } + // _EXPAND_METHOD + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); + } + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = self; + callable = method; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; DISPATCH(); } - TARGET(STORE_SLICE) { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _CHECK_PERIODIC + { } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - STACK_SHRINK(4); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef callargs_st; + _PyStackRef func_st; + _PyStackRef result; + if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } + callargs_st = stack_pointer[-1 - (oparg & 1)]; + func_st = stack_pointer[-3 - (oparg & 1)]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + goto error; + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + goto error; + } + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) goto error; + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (PyStackRef_IsNull(result)) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + PyStackRef_CLEAR(result); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + } + PyStackRef_CLOSE(func_st); + PyStackRef_CLOSE(callargs_st); + PyStackRef_XCLOSE(kwargs_st); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + if (PyStackRef_IsNull(result)) { stack_pointer += -3 - (oparg & 1); goto error; } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_STR_INT) { - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; next_instr += 1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - STACK_SHRINK(1); + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } - TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (true) goto pop_2_error; - } - Py_INCREF(res); // Do this before DECREF'ing dict, sub - Py_DECREF(dict); - Py_DECREF(sub); - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - - TARGET(LIST_APPEND) { - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + TARGET(CALL_ISINSTANCE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + if (retval < 0) { + goto error; + } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(SET_ADD) { - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); + TARGET(CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { stack_pointer += -3 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -3 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(STORE_SUBSCR) { - PREDICTED(STORE_SUBSCR); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; - PyObject *v; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - v = stack_pointer[-3]; - #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; - STACK_SHRINK(3); - next_instr += 1; + TARGET(CALL_LEN) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + goto error; + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + GOTO_ERROR(error); + } + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + TARGET(CALL_LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); STACK_SHRINK(3); - next_instr += 1; + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } - TARGET(STORE_SUBSCR_DICT) { - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - STACK_SHRINK(3); - next_instr += 1; + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DELETE_SUBSCR) { - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - STACK_SHRINK(2); + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_INTRINSIC_1) { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_INTRINSIC_2) { - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(RAISE_VARARGS) { - PyObject **args; - args = stack_pointer - oparg; - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; + TARGET(CALL_NON_PY_GENERAL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + callable = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - if (true) { STACK_SHRINK(oparg); goto error; } - } - - TARGET(INTERPRETER_EXIT) { - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); } - TARGET(RETURN_VALUE) { - PyObject *retval; - // _SAVE_CURRENT_IP + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + callable = stack_pointer[-2 - oparg]; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - // _POP_FRAME - retval = stack_pointer[-1]; - STACK_SHRINK(1); + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET { - assert(EMPTY()); #if TIER_ONE - assert(frame != &entry_frame); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif - STORE_SP(); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_VALUE) { - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - - TARGET(RETURN_CONST) { - PyObject *value; - PyObject *retval; - // LOAD_CONST + TARGET(CALL_PY_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + DEOPT_IF(tstate->interp->eval_frame, CALL); } - // _SAVE_CURRENT_IP + // _CHECK_FUNCTION_VERSION + callable = stack_pointer[-2 - oparg]; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - // _POP_FRAME - retval = value; + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET { - assert(EMPTY()); #if TIER_ONE - assert(frame != &entry_frame); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif - STORE_SP(); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_CONST) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - - TARGET(GET_AITER) { - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) goto pop_3_error; } - - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; + // _CHECK_PERIODIC + { } - stack_pointer[-1] = iter; + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(GET_ANEXT) { - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - goto error; - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - goto error; - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - goto error; - } - - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - - Py_DECREF(next_iter); - goto error; - } else { - Py_DECREF(next_iter); - } + TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) goto pop_3_error; + } + // _CHECK_PERIODIC + { } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(GET_AWAITABLE) { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (true) goto pop_2_error; } - - Py_DECREF(iterable); - - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match_o, &rest_o); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) goto pop_2_error; + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) goto pop_2_error; + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } - - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } - TARGET(SEND) { - PREDICTED(SEND); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; - PyObject *retval; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; - DISPATCH_INLINED(gen_frame); + TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { + PyStackRef_CLOSE(right); + if (true) goto pop_1_error; } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; + DISPATCH(); + } + + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef exc_value_st; + _PyStackRef last_sent_val_st; + _PyStackRef sub_iter_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + none = PyStackRef_None; } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, next_instr-1); + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); + STAT_INC(COMPARE_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? PyStackRef_True : PyStackRef_False; } else { - goto error; + res = PyStackRef_FromPyObjectSteal(res_o); } } - Py_DECREF(v); - stack_pointer[-1] = retval; - next_instr += 1; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(SEND_GEN) { - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; - DISPATCH_INLINED(gen_frame); - } - - TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - TARGET(YIELD_VALUE) { - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ - assert(frame != &entry_frame); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - TARGET(POP_EXCEPT) { - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(RERAISE) { - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - goto error; + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ContainsOp(right, next_instr); + DISPATCH_SAME_OPARG(); } + STAT_INC(CONTAINS_OP, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; + // _CONTAINS_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CONTAINS_OP_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + int res = PyDict_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CONTAINS_OP_SET) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + int res = _PySet_Contains((PySetObject *)right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - TARGET(END_ASYNC_FOR) { - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; - } - STACK_SHRINK(2); + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (result_o == NULL) goto pop_1_error; + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-1] = result; DISPATCH(); } - TARGET(CLEANUP_THROW) { - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; - } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); - goto exception_unwind; + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; DISPATCH(); } - TARGET(LOAD_ASSERTION_ERROR) { - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_BUILD_CLASS) { - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; + TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + goto error; } - STACK_GROW(1); - stack_pointer[-1] = bc; + Py_DECREF(oldobj); DISPATCH(); } - TARGET(STORE_NAME) { - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; + TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } + SETLOCAL(oparg, PyStackRef_NULL); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_Pop(GLOBALS(), name, NULL); + // Can't use ERROR_IF here. + if (err < 0) { + goto error; + } + if (err == 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1483,1129 +2807,1347 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); goto error; } DISPATCH(); } - TARGET(UNPACK_SEQUENCE) { - PREDICTED(UNPACK_SEQUENCE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - seq = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef sub; + _PyStackRef container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + PyStackRef_CLOSE(update); + if (true) goto pop_1_error; + } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef update; + _PyStackRef dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (PyDict_Update(dict_o, update_o) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + } + PyStackRef_CLOSE(update); + if (true) goto pop_1_error; + } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef exc_st; + _PyStackRef awaitable_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(END_FOR) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(UNPACK_EX) { - PyObject *seq; - seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef value; + _PyStackRef receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_ATTR) { - PREDICTED(STORE_ATTR); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *v; - owner = stack_pointer[-1]; - v = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + TARGET(ENTER_EXECUTOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; - STACK_SHRINK(2); - next_instr += 4; + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ DISPATCH(); } - TARGET(DELETE_ATTR) { - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - STACK_SHRINK(1); + TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + goto error; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_GLOBAL) { - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); + TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + if (PyStackRef_IsNull(res)) goto pop_1_error; + } + else { + res = value; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(DELETE_GLOBAL) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - goto error; + STAT_INC(FOR_ITER, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + next = PyStackRef_NULL; + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_LOCALS) { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; + TARGET(FOR_ITER_GEN) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + iter = stack_pointer[-1]; + { + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); + } + // _PUSH_FRAME + new_frame = gen_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); + // _ITER_JUMP_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + #endif + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); } - else if (_PyErr_Occurred(tstate)) { - goto error; + } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + PyStackRef_CLOSE(iter); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); + DISPATCH(); } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) goto error; + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); } + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); } } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_NAME) { - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; + TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + PyStackRef_CLOSE(obj); + if (true) goto pop_1_error; } - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + iter_o = (*getter)(obj_o); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) goto pop_1_error; + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + if (true) goto pop_1_error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter_o); + if (next_iter == NULL) { goto error; } } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + goto error; + } + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } } - STACK_GROW(1); - stack_pointer[-1] = v; + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_GLOBAL) { - PREDICTED(LOAD_GLOBAL); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } + TARGET(GET_AWAITABLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); + } + PyStackRef_CLOSE(iterable); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter_o); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ } } - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(LOAD_GLOBAL_MODULE) { - PyObject *res; - PyObject *null = NULL; - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&next_instr[1].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _LOAD_GLOBAL_MODULE - { - uint16_t index = read_u16(&next_instr[3].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; + TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + PyStackRef_CLOSE(iterable); + if (PyStackRef_IsNull(iter)) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *res; - PyObject *null = NULL; - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&next_instr[1].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + if (len_i < 0) goto error; + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + goto error; + } + iter = iterable; } - // _GUARD_BUILTINS_VERSION - { - uint16_t version = read_u16(&next_instr[2].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t index = read_u16(&next_instr[3].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + else { + /* `iterable` is not a generator. */ + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { + goto error; + } + PyStackRef_CLOSE(iterable); } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(DELETE_FAST) { - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(MAKE_CELL) { - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - goto resume_with_error; - } - SETLOCAL(oparg, cell); + TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = import_name(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(DELETE_DEREF) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - DISPATCH(); + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + /* Skip 3 cache entries */ + int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL); } - TARGET(LOAD_FROM_DICT_OR_DEREF) { - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - Py_DECREF(class_dict); - goto error; - } - Py_DECREF(class_dict); - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); + } + + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef value; + _PyStackRef receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { goto error; } - Py_INCREF(value); } - stack_pointer[-1] = value; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_DEREF) { - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef value; + _PyStackRef receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + goto error; + } } - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(STORE_DEREF) { - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - STACK_SHRINK(1); + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(COPY_FREE_VARS) { - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _Py_CODEUNIT *target; + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + PyStackRef_CLOSE(iter_stackref); + /* Skip END_FOR and POP_TOP */ + target = next_instr + oparg + 2; } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(BUILD_STRING) { - PyObject **pieces; - PyObject *str; - pieces = stack_pointer - oparg; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; - DISPATCH(); + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } - TARGET(BUILD_TUPLE) { - PyObject **values; - PyObject *tup; - values = stack_pointer - oparg; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(BUILD_LIST) { - PyObject **values; - PyObject *list; - values = stack_pointer - oparg; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(LIST_EXTEND) { - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - if (true) goto pop_1_error; - } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - STACK_SHRINK(1); - DISPATCH(); + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - TARGET(SET_UPDATE) { - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(BUILD_SET) { - PyObject **values; - PyObject *set; - values = stack_pointer - oparg; - set = PySet_New(NULL); - if (set == NULL) - goto error; - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int flag = PyStackRef_Is(value_stackref, PyStackRef_None); + int offset; + if (flag) { + offset = oparg; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; - DISPATCH(); - } - - TARGET(BUILD_MAP) { - PyObject **values; - PyObject *map; - values = stack_pointer - oparg*2; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + else { + PyStackRef_CLOSE(value_stackref); + offset = 0; } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(SETUP_ANNOTATIONS) { - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(LOCALS())) { - ann_dict = _PyDict_GetItemWithError(LOCALS(), - &_Py_ID(__annotations__)); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error; - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int offset; + int nflag = PyStackRef_Is(value_stackref, PyStackRef_None); + if (nflag) { + offset = 0; } else { - /* do the same if locals() is not a dict */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } + PyStackRef_CLOSE(value_stackref); + offset = oparg; } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(DICT_UPDATE) { - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version && tstate->tracing == 0) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + goto error; } - Py_DECREF(update); - if (true) goto pop_1_error; + next_instr = this_instr; } - Py_DECREF(update); - STACK_SHRINK(1); - DISPATCH(); - } - - TARGET(DICT_MERGE) { - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; + DISPATCH(); + } } - Py_DECREF(update); - STACK_SHRINK(1); DISPATCH(); } - TARGET(MAP_ADD) { - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); - DISPATCH(); + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(LOAD_SUPER_ATTR) { - PREDICTED(LOAD_SUPER_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *null = NULL; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; - #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); - if (err) goto pop_3_error; - } - - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + if (err) goto error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; } - TARGET(LOAD_SUPER_ATTR_ATTR) { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; next_instr += 1; - DISPATCH(); + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return PyStackRef_AsPyObjectSteal(retval); } - TARGET(LOAD_SUPER_ATTR_METHOD) { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; - } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; - } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + TARGET(IS_OP) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + #ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + #else + int res = PyStackRef_Is(left, right) ^ oparg; + #endif + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR) { - PREDICTED(LOAD_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - owner = stack_pointer[-1]; + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + /* Skip 1 cache entry */ + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); + if (optimized < 0) goto error; + if (optimized) { + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; + this_instr[1].counter = restart_backoff_counter(counter); } } else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } - next_instr += 9; + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ DISPATCH(); } - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&next_instr[3].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); } - TARGET(LOAD_ATTR_MODULE) { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&next_instr[3].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); DISPATCH(); } - TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&next_instr[3].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef v; + _PyStackRef list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef iterable_st; + _PyStackRef list_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(iterable_st); + if (true) goto pop_1_error; } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; + assert(Py_IsNone(none_val)); + PyStackRef_CLOSE(iterable_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_SLOT) { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_ATTR owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - // _LOAD_ATTR_SLOT + /* Skip 8 cache entries */ + // _LOAD_ATTR { - uint16_t index = read_u16(&next_instr[3].cache); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + } + else { + /* Classic, pushes one value. */ + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *fget = read_obj(&next_instr[5].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *getattribute = read_obj(&next_instr[5].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -2614,2518 +4156,2635 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner; - PyObject *value; + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _GUARD_DORV_VALUES + // _CHECK_MANAGED_OBJECT_HAS_VALUES { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; + // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&next_instr[3].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); - } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); - } - STACK_SHRINK(2); - next_instr += 4; - DISPATCH(); - } - - TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t hint = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - STACK_SHRINK(2); - next_instr += 4; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(STORE_ATTR_SLOT) { - PyObject *owner; - PyObject *value; + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; + // _CHECK_ATTR_METHOD_LAZY_DICT { - uint16_t index = read_u16(&next_instr[3].cache); - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - } - STACK_SHRINK(2); - next_instr += 4; - DISPATCH(); - } - - TARGET(COMPARE_OP) { - PREDICTED(COMPARE_OP); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); } - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT) { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; - DISPATCH(); - } - - TARGET(COMPARE_OP_INT) { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; - DISPATCH(); - } - - TARGET(COMPARE_OP_STR) { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 1; - DISPATCH(); - } - - TARGET(IS_OP) { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(CONTAINS_OP) { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CHECK_EG_MATCH) { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CHECK_EXC_MATCH) { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(IMPORT_NAME) { - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(IMPORT_FROM) { - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(JUMP_FORWARD) { - JUMPBY(oparg); + TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); + } + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(JUMP_BACKWARD) { - CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; - assert(oparg <= INSTR_OFFSET()); - JUMPBY(1-oparg); - #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && - // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; - } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - #endif /* ENABLE_SPECIALIZATION */ + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; DISPATCH(); } - TARGET(ENTER_EXECUTOR) { - CHECK_EVAL_BREAKER(); - - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; - Py_INCREF(executor); - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - goto resume_frame; - } - - TARGET(POP_JUMP_IF_FALSE) { - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - STACK_SHRINK(1); - next_instr += 1; + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); - next_instr += 1; - DISPATCH(); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(POP_JUMP_IF_NONE) { - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // POP_JUMP_IF_TRUE - cond = b; + // _LOAD_ATTR_SLOT { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); } - STACK_SHRINK(1); - next_instr += 1; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT { - if (Py_IsNone(value)) { - b = Py_True; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; } else { - b = Py_False; - Py_DECREF(value); + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; } + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } - // POP_JUMP_IF_FALSE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - STACK_SHRINK(1); + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error; + if (bc_o == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + TARGET(LOAD_COMMON_CONSTANT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(GET_LEN) { - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; + TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + _PyStackRef value; + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(MATCH_CLASS) { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - attrs = Py_None; // Failure! + TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(MATCH_MAPPING) { - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(MATCH_SEQUENCE) { - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(MATCH_KEYS) { - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(GET_ITER) { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - goto error; - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; + TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + goto error; } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } - Py_DECREF(iterable); } - stack_pointer[-1] = iter; + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(FOR_ITER) { - PREDICTED(FOR_ITER); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - iter = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + goto error; + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } goto error; } - monitor_raise(tstate, frame, next_instr-1); - _PyErr_Clear(tstate); } - /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - // Common case: no jump, leave it to the code generator - STACK_GROW(1); - stack_pointer[-1] = next; - next_instr += 1; + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error; + if (v_o == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error; + if (v_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto pop_1_error; + } + } + } + } + PyStackRef_CLOSE(mod_or_class_dict); + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; DISPATCH(); } - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = next_instr-1; - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; - } - else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; } - monitor_raise(tstate, frame, here); - _PyErr_Clear(tstate); } - /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) goto error; + if (res_o == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) goto error; + if (res_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } + } + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(FOR_ITER_LIST) { - PyObject *iter; - PyObject *next; - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; + TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_JUMP_LIST + // _GUARD_BUILTINS_VERSION { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_NEXT_LIST + // _LOAD_GLOBAL_BUILTINS { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + Py_INCREF(res_o); + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } - STACK_GROW(1); - stack_pointer[-1] = next; - next_instr += 1; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(FOR_ITER_TUPLE) { - PyObject *iter; - PyObject *next; - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_JUMP_TUPLE + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + Py_INCREF(res_o); + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } - // _ITER_NEXT_TUPLE - { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } - STACK_GROW(1); - stack_pointer[-1] = next; + locals = PyStackRef_FromPyObjectNew(l);; + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *v_o; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { + goto error; + } + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { + goto error; + } + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { + goto error; + } + if (v_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(FOR_ITER_RANGE) { - PyObject *iter; - PyObject *next; - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + TARGET(LOAD_SPECIAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + } } - // _ITER_JUMP_RANGE + if (PyStackRef_IsNull(attr)) goto pop_1_error; + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + DISPATCH_SAME_OPARG(); } + STAT_INC(LOAD_SUPER_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - // _ITER_NEXT_RANGE + // _LOAD_SUPER_ATTR + self_st = stack_pointer[-1]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); + Py_DECREF(super); + if (PyStackRef_IsNull(attr)) goto pop_3_error; + null = PyStackRef_NULL; } - STACK_GROW(1); - stack_pointer[-1] = next; - next_instr += 1; + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) goto pop_3_error; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(FOR_ITER_GEN) { - PyObject *iter; - iter = stack_pointer[-1]; - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; - DISPATCH_INLINED(gen_frame); - } - - TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - goto error; - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - goto error; + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); + if (true) goto pop_3_error; } - Py_DECREF(mgr); - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(BEFORE_WITH) { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } + TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { goto error; } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) { goto error; } - Py_DECREF(mgr); - res = _PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[-1] = func; DISPATCH(); } - TARGET(WITH_EXCEPT_START) { - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(PUSH_EXC_INFO) { - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - prev_exc = Py_None; + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = PyStackRef_None; // Failure! } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&next_instr[3].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; - } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; - next_instr += 9; + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + if (values_or_none_o == NULL) goto error; + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; - next_instr += 9; + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *owner; - PyObject *attr; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&next_instr[3].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&next_instr[5].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - next_instr += 9; + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *owner; - PyObject *attr; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&next_instr[5].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - next_instr += 9; + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&next_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&next_instr[5].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; - next_instr += 9; + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, + PyStackRef_Is(exc_value, PyStackRef_None) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_CALL) { - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); - if (err) goto error; - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - GO_TO_INSTRUCTION(CALL); + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - TARGET(CALL) { - PREDICTED(CALL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_Call(callable, next_instr, total_args); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + value = stack_pointer[-1]; + { + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _POP_JUMP_IF_TRUE + cond = b; + { + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *null; - PyObject *callable; - PyObject *self; - PyObject *self_or_null; - PyObject *func; - PyObject **args; - _PyInterpreterFrame *new_frame; - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + value = stack_pointer[-1]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } - // _SAVE_CURRENT_IP - next_instr += 3; - { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; - } - // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); + // _POP_JUMP_IF_FALSE + cond = b; { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_PY_EXACT_ARGS) { - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); + TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef new_exc; + _PyStackRef prev_exc; + new_exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + else { + prev_exc = PyStackRef_None; } - // _CHECK_STACK_SPACE - { - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = PyStackRef_AsPyObjectSteal(args[1]); + _Py_FALLTHROUGH; + case 1: + exc = PyStackRef_AsPyObjectSteal(args[0]); + _Py_FALLTHROUGH; + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; } - // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + if (true) { stack_pointer += -oparg; goto error; } + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef exc_st; + _PyStackRef *values; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + goto error; } } - // _SAVE_CURRENT_IP - next_instr += 3; - { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; - } - // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; + assert(frame == tstate->current_frame); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); + } + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } - #endif + } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); } - TARGET(CALL_TYPE_1) { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); + _PyStackRef value; + _PyStackRef retval; + _PyStackRef res; + // _LOAD_CONST + { + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + } + // _RETURN_VALUE + retval = value; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_STR_1) { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + goto error; + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + LOAD_SP(); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_TUPLE_1) { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - PyObject **args; - PyObject *null; - PyObject *callable; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - goto error; + TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _SEND + v = stack_pointer[-1]; + { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + } + else { + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + } + if (retval_o == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + goto error; + } + } + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + stack_pointer[-1] = retval; + DISPATCH(); } - TARGET(EXIT_INIT_CHECK) { - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - goto error; - } + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef v; + _PyStackRef receiver; + /* Skip 1 cache entry */ + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); - DISPATCH(); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); } - TARGET(CALL_BUILTIN_CLASS) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); } - Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_BUILTIN_O) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef v; + _PyStackRef set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef func_st; + _PyStackRef attr_st; + func_st = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; + default: + Py_UNREACHABLE(); } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + stack_pointer[-2] = func_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef iterable; + _PyStackRef set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_BUILTIN_FAST) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + /* Skip 3 cache entries */ + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR); + DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); + PyObject *old_value = values->values[index]; + values->values[index] = PyStackRef_AsPyObjectSteal(value); + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_LEN) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - goto error; + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_ISINSTANCE) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - goto error; + // _STORE_ATTR_WITH_HINT + value = stack_pointer[-2]; + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + PyStackRef_CLOSE(owner); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; + TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_LIST_APPEND) { - PyObject **args; - PyObject *self; - PyObject *callable; - args = stack_pointer - oparg; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_O) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + PyStackRef_CLOSE(v); + if (true) goto pop_1_error; } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + else + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + int err; + if (slice == NULL) { + err = 1; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + else { + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - next_instr += 3; - CHECK_EVAL_BREAKER(); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) goto pop_4_error; + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_KW) { - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); - } - - TARGET(CALL_KW) { - PREDICTED(CALL_KW); - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); } + STAT_INC(STORE_SUBSCR, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_3_error; } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef value; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); + if (err) goto pop_3_error; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - TARGET(CALL_FUNCTION_EX) { - PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - goto error; - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - goto error; - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); - if (err) goto error; - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); - } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; - CHECK_EVAL_BREAKER(); + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; DISPATCH(); } - TARGET(MAKE_FUNCTION) { - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - - Py_DECREF(codeobj); - if (func_obj == NULL) { - goto error; + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(TO_BOOL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; + /* Skip 2 cache entries */ + // _TO_BOOL + { + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (err < 0) goto pop_1_error; + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(SET_FUNCTION_ATTRIBUTE) { - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + // _REPLACE_WITH_TRUE + value = owner; + { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(RETURN_GENERATOR) { - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - goto error; + TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - goto resume_frame; + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); } - TARGET(BUILD_SLICE) { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(CONVERT_VALUE) { - PyObject *value; - PyObject *result; + TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!PyStackRef_Is(value, PyStackRef_None), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(FORMAT_SIMPLE) { - PyObject *value; - PyObject *res; + TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - res = value; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); } - TARGET(FORMAT_WITH_SPEC) { - PyObject *fmt_spec; - PyObject *value; - PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); + TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } - TARGET(COPY) { - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_OP) { - PREDICTED(BINARY_OP); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs; - PyObject *lhs; - PyObject *res; - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(SWAP) { - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + seq = stack_pointer[-1]; + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_INSTRUCTION) { - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); - if (next_opcode < 0) goto error; - next_instr--; - if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TARGET(INSTRUMENTED_JUMP_FORWARD) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + // _UNPACK_SEQUENCE + { + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_JUMP_BACKWARD) { - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; } else { - Py_DECREF(value); - offset = 0; + Py_DECREF(tb); } - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + if (PyStackRef_IsNull(res)) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; - } - #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - - TARGET(EXTENDED_ARG) { - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TARGET(CACHE) { - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - } - - TARGET(RESERVED) { - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); - } +#undef TIER_ONE diff --git a/Python/getargs.c b/Python/getargs.c index a0eef2ccee5c37..b96ce3a22dae7c 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,20 +1,24 @@ /* New getargs implementation */ +#define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() +#include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini +#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_pyerrors.h" // _Py_CalculateSuggestions() /* Export Stable ABIs (abi only) */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, - const char *, char **, ...); + const char *, const char * const *, ...); PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, - const char *, char **, va_list); + const char *, const char * const *, va_list); #define FLAG_COMPAT 1 @@ -46,14 +50,14 @@ static void seterror(Py_ssize_t, const char *, int *, const char *, const char * static const char *convertitem(PyObject *, const char **, va_list *, int, int *, char *, size_t, freelist_t *); static const char *converttuple(PyObject *, const char **, va_list *, int, - int *, char *, size_t, int, freelist_t *); + int *, char *, size_t, freelist_t *); static const char *convertsimple(PyObject *, const char **, va_list *, int, char *, size_t, freelist_t *); static Py_ssize_t convertbuffer(PyObject *, const void **p, const char **); static int getbuffer(PyObject *, Py_buffer *, const char**); static int vgetargskeywords(PyObject *, PyObject *, - const char *, char **, va_list *, int); + const char *, const char * const *, va_list *, int); static int vgetargskeywordsfast(PyObject *, PyObject *, struct _PyArg_Parser *, va_list *, int); static int vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, @@ -453,7 +457,7 @@ seterror(Py_ssize_t iarg, const char *msg, int *levels, const char *fname, static const char * converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, - int *levels, char *msgbuf, size_t bufsize, int toplevel, + int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist) { int level = 0; @@ -476,14 +480,13 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (c == ':' || c == ';' || c == '\0') break; - else if (level == 0 && Py_ISALPHA(c)) + else if (level == 0 && Py_ISALPHA(c) && c != 'e') n++; } if (!PySequence_Check(arg) || PyBytes_Check(arg)) { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - toplevel ? "expected %d arguments, not %.50s" : "must be %d-item sequence, not %.50s", n, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); @@ -493,18 +496,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, len = PySequence_Size(arg); if (len != n) { levels[0] = 0; - if (toplevel) { - PyOS_snprintf(msgbuf, bufsize, - "expected %d argument%s, not %zd", - n, - n == 1 ? "" : "s", - len); - } - else { - PyOS_snprintf(msgbuf, bufsize, - "must be sequence of length %d, not %zd", - n, len); - } + PyOS_snprintf(msgbuf, bufsize, + "must be sequence of length %d, not %zd", + n, len); return msgbuf; } @@ -547,7 +541,7 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format == '(' /* ')' */) { format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, - bufsize, 0, freelist); + bufsize, freelist); if (msg == NULL) format++; } @@ -595,6 +589,17 @@ converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) return msgbuf; } +static const char * +convertcharerr(const char *expected, const char *what, Py_ssize_t size, + char *msgbuf, size_t bufsize) +{ + assert(expected != NULL); + PyOS_snprintf(msgbuf, bufsize, + "must be %.50s, not %.50s of length %zd", + expected, what, size); + return msgbuf; +} + #define CONV_UNICODE "(unicode conversion error)" /* Convert a non-tuple argument. Return NULL if conversion went OK, @@ -619,7 +624,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, switch (c) { case 'b': { /* unsigned byte -- very short int */ - char *p = va_arg(*p_va, char *); + unsigned char *p = va_arg(*p_va, unsigned char *); long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -640,7 +645,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ - char *p = va_arg(*p_va, char *); + unsigned char *p = va_arg(*p_va, unsigned char *); unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -801,10 +806,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytes object", PyBytes_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyBytes_AS_STRING(arg)[0]; - else if (PyByteArray_Check(arg) && PyByteArray_Size(arg) == 1) + } + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytearray object", PyByteArray_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyByteArray_AS_STRING(arg)[0]; + } else return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; @@ -818,8 +835,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (!PyUnicode_Check(arg)) return converterr("a unicode character", arg, msgbuf, bufsize); - if (PyUnicode_GET_LENGTH(arg) != 1) - return converterr("a unicode character", arg, msgbuf, bufsize); + if (PyUnicode_GET_LENGTH(arg) != 1) { + return convertcharerr("a unicode character", + "a string", PyUnicode_GET_LENGTH(arg), + msgbuf, bufsize); + } kind = PyUnicode_KIND(arg); data = PyUnicode_DATA(arg); @@ -1186,17 +1206,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, arg, msgbuf, bufsize); format++; - /* Caller is interested in Py_buffer, and the object - supports it directly. */ + /* Caller is interested in Py_buffer, and the object supports it + directly. The request implicitly asks for PyBUF_SIMPLE, so the + result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } - if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) { - PyBuffer_Release((Py_buffer*)p); - return converterr("contiguous buffer", arg, msgbuf, bufsize); - } + assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( "(cleanup problem)", @@ -1241,15 +1259,12 @@ convertbuffer(PyObject *arg, const void **p, const char **errmsg) static int getbuffer(PyObject *arg, Py_buffer *view, const char **errmsg) { + /* PyBUF_SIMPLE implies C-contiguous */ if (PyObject_GetBuffer(arg, view, PyBUF_SIMPLE) != 0) { *errmsg = "bytes-like object"; return -1; } - if (!PyBuffer_IsContiguous(view, 'C')) { - PyBuffer_Release(view); - *errmsg = "contiguous buffer"; - return -1; - } + assert(PyBuffer_IsContiguous(view, 'C')); return 0; } @@ -1261,7 +1276,7 @@ int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, ...) + const char * const *kwlist, ...) { int retval; va_list va; @@ -1285,7 +1300,7 @@ int _PyArg_ParseTupleAndKeywords_SizeT(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, ...) + const char * const *kwlist, ...) { int retval; va_list va; @@ -1311,7 +1326,7 @@ int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, va_list va) + const char * const *kwlist, va_list va) { int retval; va_list lva; @@ -1336,7 +1351,7 @@ int _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, va_list va) + const char * const *kwlist, va_list va) { int retval; va_list lva; @@ -1437,12 +1452,31 @@ error_unexpected_keyword_arg(PyObject *kwargs, PyObject *kwnames, PyObject *kwtu int match = PySequence_Contains(kwtuple, keyword); if (match <= 0) { if (!match) { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword " - "argument for %.200s%s", - keyword, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *kwlist = PySequence_List(kwtuple); + if (!kwlist) { + return; + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(kwlist, keyword); + Py_DECREF(kwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + keyword); + } + } return; } @@ -1470,11 +1504,14 @@ PyArg_ValidateKeywordArguments(PyObject *kwargs) return 1; } +static PyObject * +new_kwtuple(const char * const *keywords, int total, int pos); + #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') static int vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, - char **kwlist, va_list *p_va, int flags) + const char * const *kwlist, va_list *p_va, int flags) { char msgbuf[512]; int levels[32]; @@ -1735,12 +1772,35 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, } } if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for %.200s%s", - key, - (fname == NULL) ? "this function" : fname, - (fname == NULL) ? "" : "()"); + PyObject *_pykwtuple = new_kwtuple(kwlist, len, pos); + if (!_pykwtuple) { + return cleanreturn(0, &freelist); + } + PyObject *pykwlist = PySequence_List(_pykwtuple); + Py_DECREF(_pykwtuple); + if (!pykwlist) { + return cleanreturn(0, &freelist); + } + PyObject *suggestion_keyword = _Py_CalculateSuggestions(pykwlist, key); + Py_DECREF(pykwlist); + + if (suggestion_keyword) { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'." + " Did you mean '%S'?", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key, + suggestion_keyword); + Py_DECREF(suggestion_keyword); + } + else { + PyErr_Format(PyExc_TypeError, + "%.200s%s got an unexpected keyword argument '%S'", + (fname == NULL) ? "this function" : fname, + (fname == NULL) ? "" : "()", + key); + } return cleanreturn(0, &freelist); } } @@ -1874,15 +1934,17 @@ new_kwtuple(const char * const *keywords, int total, int pos) Py_DECREF(kwtuple); return NULL; } - PyUnicode_InternInPlace(&str); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &str); PyTuple_SET_ITEM(kwtuple, i, str); } return kwtuple; } static int -_parser_init(struct _PyArg_Parser *parser) +_parser_init(void *arg) { + struct _PyArg_Parser *parser = (struct _PyArg_Parser *)arg; const char * const *keywords = parser->keywords; assert(keywords != NULL); assert(parser->pos == 0 && @@ -1893,7 +1955,7 @@ _parser_init(struct _PyArg_Parser *parser) int len, pos; if (scan_keywords(keywords, &len, &pos) < 0) { - return 0; + return -1; } const char *fname, *custommsg = NULL; @@ -1902,7 +1964,7 @@ _parser_init(struct _PyArg_Parser *parser) assert(parser->fname == NULL); if (parse_format(parser->format, len, pos, &fname, &custommsg, &min, &max) < 0) { - return 0; + return -1; } } else { @@ -1913,9 +1975,25 @@ _parser_init(struct _PyArg_Parser *parser) int owned; PyObject *kwtuple = parser->kwtuple; if (kwtuple == NULL) { + /* We may temporarily switch to the main interpreter to avoid + * creating a tuple that could outlive its owning interpreter. */ + PyThreadState *save_tstate = NULL; + PyThreadState *temp_tstate = NULL; + if (!_Py_IsMainInterpreter(PyInterpreterState_Get())) { + temp_tstate = PyThreadState_New(_PyInterpreterState_Main()); + if (temp_tstate == NULL) { + return -1; + } + save_tstate = PyThreadState_Swap(temp_tstate); + } kwtuple = new_kwtuple(keywords, len, pos); + if (temp_tstate != NULL) { + PyThreadState_Clear(temp_tstate); + (void)PyThreadState_Swap(save_tstate); + PyThreadState_Delete(temp_tstate); + } if (kwtuple == NULL) { - return 0; + return -1; } owned = 1; } @@ -1929,40 +2007,27 @@ _parser_init(struct _PyArg_Parser *parser) parser->min = min; parser->max = max; parser->kwtuple = kwtuple; - parser->initialized = owned ? 1 : -1; + parser->is_kwtuple_owned = owned; assert(parser->next == NULL); - parser->next = _PyRuntime.getargs.static_parsers; - _PyRuntime.getargs.static_parsers = parser; - return 1; + parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers); + do { + // compare-exchange updates parser->next on failure + } while (!_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, + &parser->next, parser)); + return 0; } static int parser_init(struct _PyArg_Parser *parser) { - // volatile as it can be modified by other threads - // and should not be optimized or reordered by compiler - if (*((volatile int *)&parser->initialized)) { - assert(parser->kwtuple != NULL); - return 1; - } - PyThread_acquire_lock(_PyRuntime.getargs.mutex, WAIT_LOCK); - // Check again if another thread initialized the parser - // while we were waiting for the lock. - if (*((volatile int *)&parser->initialized)) { - assert(parser->kwtuple != NULL); - PyThread_release_lock(_PyRuntime.getargs.mutex); - return 1; - } - int ret = _parser_init(parser); - PyThread_release_lock(_PyRuntime.getargs.mutex); - return ret; + return _PyOnceFlag_CallOnce(&parser->once, &_parser_init, parser); } static void parser_clear(struct _PyArg_Parser *parser) { - if (parser->initialized == 1) { + if (parser->is_kwtuple_owned) { Py_CLEAR(parser->kwtuple); } } @@ -2005,7 +2070,8 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, const char *format; const char *msg; PyObject *keyword; - int i, pos, len; + Py_ssize_t i; + int pos, len; Py_ssize_t nkwargs; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; @@ -2029,7 +2095,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, return 0; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return 0; } @@ -2262,7 +2328,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, args = buf; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return NULL; } @@ -2439,7 +2505,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, args = buf; } - if (!parser_init(parser)) { + if (parser_init(parser) < 0) { return NULL; } @@ -2522,7 +2588,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, * * Otherwise, we leave a place at `buf[vararg]` for vararg tuple * so the index is `i + 1`. */ - if (nargs < vararg) { + if (nargs < vararg && i != vararg) { buf[i] = current_arg; } else { @@ -2610,7 +2676,7 @@ skipitem(const char **p_format, va_list *p_va, int flags) goto err; format++; } - /* fall through */ + _Py_FALLTHROUGH; case 's': /* string */ case 'z': /* string or None */ @@ -2620,6 +2686,11 @@ skipitem(const char **p_format, va_list *p_va, int flags) if (p_va != NULL) { (void) va_arg(*p_va, char **); } + if (c == 'w' && *format != '*') + { + /* after 'w', only '*' is allowed */ + goto err; + } if (*format == '#') { if (p_va != NULL) { (void) va_arg(*p_va, Py_ssize_t *); diff --git a/Python/getcopyright.c b/Python/getcopyright.c index c1f1aad9b845b1..066c2ed66acddf 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,7 +4,7 @@ static const char cprt[] = "\ -Copyright (c) 2001-2023 Python Software Foundation.\n\ +Copyright (c) 2001-2024 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/Python/getversion.c b/Python/getversion.c index 5db836ab4bfd6d..226b2f999a6bfd 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -6,7 +6,7 @@ #include "patchlevel.h" static int initialized = 0; -static char version[250]; +static char version[300]; void _Py_InitVersion(void) { @@ -14,7 +14,12 @@ void _Py_InitVersion(void) return; } initialized = 1; - PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", +#ifdef Py_GIL_DISABLED + const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; +#else + const char *buildinfo_format = "%.80s (%.80s) %.80s"; +#endif + PyOS_snprintf(version, sizeof(version), buildinfo_format, PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); } diff --git a/Python/hashtable.c b/Python/hashtable.c index 8f5e8168ba1339..faf68fe4ff0bca 100644 --- a/Python/hashtable.c +++ b/Python/hashtable.c @@ -45,7 +45,7 @@ */ #include "Python.h" -#include "pycore_hashtable.h" +#include "pycore_hashtable.h" // export _Py_hashtable_new() #include "pycore_pyhash.h" // _Py_HashPointerRaw() #define HASHTABLE_MIN_SIZE 16 diff --git a/Python/import.c b/Python/import.c index cafdd834502224..40b7feac001d6e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,7 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_Pop() +#include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -34,6 +35,17 @@ module _imp #include "clinic/import.c.h" +#ifndef NDEBUG +static bool +is_interpreter_isolated(PyInterpreterState *interp) +{ + return !_Py_IsMainInterpreter(interp) + && !(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) + && interp->ceval.own_gil; +} +#endif + + /*******************************/ /* process-global import state */ /*******************************/ @@ -82,11 +94,7 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.import_func #define IMPORT_LOCK(interp) \ - (interp)->imports.lock.mutex -#define IMPORT_LOCK_THREAD(interp) \ - (interp)->imports.lock.thread -#define IMPORT_LOCK_LEVEL(interp) \ - (interp)->imports.lock.level + (interp)->imports.lock #define FIND_AND_LOAD(interp) \ (interp)->imports.find_and_load @@ -103,74 +111,14 @@ static struct _inittab *inittab_copy = NULL; void _PyImport_AcquireLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID) - return; /* Too bad */ - if (IMPORT_LOCK(interp) == NULL) { - IMPORT_LOCK(interp) = PyThread_allocate_lock(); - if (IMPORT_LOCK(interp) == NULL) - return; /* Nothing much we can do. */ - } - if (IMPORT_LOCK_THREAD(interp) == me) { - IMPORT_LOCK_LEVEL(interp)++; - return; - } - if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) - { - PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - PyEval_RestoreThread(tstate); - } - assert(IMPORT_LOCK_LEVEL(interp) == 0); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp) = 1; + _PyRecursiveMutex_Lock(&IMPORT_LOCK(interp)); } -int +void _PyImport_ReleaseLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) - return 0; /* Too bad */ - if (IMPORT_LOCK_THREAD(interp) != me) - return -1; - IMPORT_LOCK_LEVEL(interp)--; - assert(IMPORT_LOCK_LEVEL(interp) >= 0); - if (IMPORT_LOCK_LEVEL(interp) == 0) { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(IMPORT_LOCK(interp)); - } - return 1; -} - -#ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child() to ensure that newly - created child processes do not share locks with the parent. - We now acquire the import lock around fork() calls but on some platforms - (Solaris 9 and earlier? see isue7242) that still left us with problems. */ -PyStatus -_PyImport_ReInitLock(PyInterpreterState *interp) -{ - if (IMPORT_LOCK(interp) != NULL) { - if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { - return _PyStatus_ERR("failed to create a new lock"); - } - } - - if (IMPORT_LOCK_LEVEL(interp) > 1) { - /* Forked as a side effect of import */ - unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp)--; - } else { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - IMPORT_LOCK_LEVEL(interp) = 0; - } - return _PyStatus_OK(); + _PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp)); } -#endif /***************/ @@ -200,39 +148,54 @@ _PyImport_ClearModules(PyInterpreterState *interp) Py_SETREF(MODULES(interp), NULL); } +static inline PyObject * +get_modules_dict(PyThreadState *tstate, bool fatal) +{ + /* Technically, it would make sense to incref the dict, + * since sys.modules could be swapped out and decref'ed to 0 + * before the caller is done using it. However, that is highly + * unlikely, especially since we can rely on a global lock + * (i.e. the GIL) for thread-safety. */ + PyObject *modules = MODULES(tstate->interp); + if (modules == NULL) { + if (fatal) { + Py_FatalError("interpreter has no modules dictionary"); + } + _PyErr_SetString(tstate, PyExc_RuntimeError, + "unable to get sys.modules"); + return NULL; + } + return modules; +} + PyObject * PyImport_GetModuleDict(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (MODULES(interp) == NULL) { - Py_FatalError("interpreter has no modules dictionary"); - } - return MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + return get_modules_dict(tstate, true); } int _PyImport_SetModule(PyObject *name, PyObject *m) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *modules = get_modules_dict(tstate, true); return PyObject_SetItem(modules, name, m); } int _PyImport_SetModuleString(const char *name, PyObject *m) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *modules = get_modules_dict(tstate, true); return PyMapping_SetItemString(modules, name, m); } static PyObject * import_get_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, false); if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "unable to get sys.modules"); return NULL; } @@ -253,18 +216,21 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n NOTE: because of this, initializing must be set *before* stuffing the new module in sys.modules. */ - spec = PyObject_GetAttr(mod, &_Py_ID(__spec__)); - int busy = _PyModuleSpec_IsInitializing(spec); - Py_XDECREF(spec); - if (busy) { - /* Wait until module is done importing. */ - PyObject *value = PyObject_CallMethodOneArg( - IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); - if (value == NULL) { - return -1; - } - Py_DECREF(value); + int rc = PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec); + if (rc > 0) { + rc = _PyModuleSpec_IsInitializing(spec); + Py_DECREF(spec); + } + if (rc <= 0) { + return rc; + } + /* Wait until module is done importing. */ + PyObject *value = PyObject_CallMethodOneArg( + IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); + if (value == NULL) { + return -1; } + Py_DECREF(value); return 0; } @@ -294,10 +260,8 @@ PyImport_GetModule(PyObject *name) static PyObject * import_add_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, false); if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "no import module dictionary"); return NULL; } @@ -394,10 +358,10 @@ remove_module(PyThreadState *tstate, PyObject *name) { PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, true); if (PyDict_CheckExact(modules)) { - PyObject *mod = _PyDict_Pop(modules, name, Py_None); - Py_XDECREF(mod); + // Error is reported to the caller + (void)PyDict_Pop(modules, name, NULL); } else if (PyMapping_DelItem(modules, name) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { @@ -416,32 +380,65 @@ remove_module(PyThreadState *tstate, PyObject *name) Py_ssize_t _PyImport_GetNextModuleIndex(void) { - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); - LAST_MODULE_INDEX++; - Py_ssize_t index = LAST_MODULE_INDEX; - PyThread_release_lock(EXTENSIONS.mutex); + return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; +} + +#ifndef NDEBUG +struct extensions_cache_value; +static struct extensions_cache_value * _find_cached_def(PyModuleDef *); +static Py_ssize_t _get_cached_module_index(struct extensions_cache_value *); +#endif + +static Py_ssize_t +_get_module_index_from_def(PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; +#ifndef NDEBUG + struct extensions_cache_value *cached = _find_cached_def(def); + assert(cached == NULL || index == _get_cached_module_index(cached)); +#endif return index; } +static void +_set_module_index(PyModuleDef *def, Py_ssize_t index) +{ + assert(index > 0); + if (index == def->m_base.m_index) { + /* There's nothing to do. */ + } + else if (def->m_base.m_index == 0) { + /* It should have been initialized by PyModuleDef_Init(). + * We assert here to catch this in dev, but keep going otherwise. */ + assert(def->m_base.m_index != 0); + def->m_base.m_index = index; + } + else { + /* It was already set for a different module. + * We replace the old value. */ + assert(def->m_base.m_index > 0); + def->m_base.m_index = index; + } +} + static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { - if (index == 0) { + if (index <= 0) { return "invalid module index"; } if (MODULES_BY_INDEX(interp) == NULL) { return "Interpreters module-list not accessible."; } - if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + if (index >= PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { return "Module index out of bounds."; } return NULL; } static PyObject * -_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_get(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; if (_modules_by_index_check(interp, index) != NULL) { return NULL; } @@ -451,11 +448,9 @@ _modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) static int _modules_by_index_set(PyInterpreterState *interp, - PyModuleDef *def, PyObject *module) + Py_ssize_t index, PyObject *module) { - assert(def != NULL); - assert(def->m_slots == NULL); - assert(def->m_base.m_index > 0); + assert(index > 0); if (MODULES_BY_INDEX(interp) == NULL) { MODULES_BY_INDEX(interp) = PyList_New(0); @@ -464,7 +459,6 @@ _modules_by_index_set(PyInterpreterState *interp, } } - Py_ssize_t index = def->m_base.m_index; while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { return -1; @@ -475,9 +469,8 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); if (err != NULL) { Py_FatalError(err); @@ -494,7 +487,8 @@ PyState_FindModule(PyModuleDef* module) if (module->m_slots) { return NULL; } - return _modules_by_index_get(interp, module); + Py_ssize_t index = _get_module_index_from_def(module); + return _modules_by_index_get(interp, index); } /* _PyState_AddModule() has been completely removed from the C-API @@ -514,7 +508,9 @@ _PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - return _modules_by_index_set(tstate->interp, def, module); + assert(def->m_slots == NULL); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_set(tstate->interp, index, module); } int @@ -534,7 +530,7 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) } PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; + Py_ssize_t index = _get_module_index_from_def(def); if (MODULES_BY_INDEX(interp) && index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) @@ -543,7 +539,8 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) return -1; } - return _modules_by_index_set(interp, def, module); + assert(def->m_slots == NULL); + return _modules_by_index_set(interp, index, module); } int @@ -556,7 +553,8 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear_one(tstate->interp, def); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_clear_one(tstate->interp, index); } @@ -575,6 +573,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* cleanup the saved copy of module dicts */ PyModuleDef *md = PyModule_GetDef(m); if (md) { + // XXX Do this more carefully. The dict might be owned + // by another interpreter. Py_CLEAR(md->m_base.m_copy); } } @@ -585,7 +585,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) if (PyList_SetSlice(MODULES_BY_INDEX(interp), 0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)), NULL)) { - PyErr_WriteUnraisable(MODULES_BY_INDEX(interp)); + PyErr_FormatUnraisable("Exception ignored on clearing interpreters module list"); } } @@ -619,77 +619,100 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) ...for single-phase init modules, where m_size == -1: (6). first time (not found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() - 3. _PyImport_LoadDynamicModuleWithSpec(): load - 4. _PyImport_LoadDynamicModuleWithSpec(): call - 5. -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized() - 6. PyModule_CreateInitialized() -> PyModule_New() - 7. PyModule_CreateInitialized(): allocate mod->md_state - 8. PyModule_CreateInitialized() -> PyModule_AddFunctions() - 9. PyModule_CreateInitialized() -> PyModule_SetDocString() - 10. PyModule_CreateInitialized(): set mod->md_def - 11. : initialize the module - 12. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - 13. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init - 14. _PyImport_LoadDynamicModuleWithSpec(): set __file__ - 15. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject() - 16. _PyImport_FixupExtensionObject(): add it to interp->imports.modules_by_index - 17. _PyImport_FixupExtensionObject(): copy __dict__ into def->m_base.m_copy - 18. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + A. _imp_create_dynamic_impl() -> import_find_extension() + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> import_run_extension() + E. import_run_extension() -> _PyImport_RunModInitFunc() + F. _PyImport_RunModInitFunc(): call + G. -> PyModule_Create() -> PyModule_Create2() + -> PyModule_CreateInitialized() + H. PyModule_CreateInitialized() -> PyModule_New() + I. PyModule_CreateInitialized(): allocate mod->md_state + J. PyModule_CreateInitialized() -> PyModule_AddFunctions() + K. PyModule_CreateInitialized() -> PyModule_SetDocString() + L. PyModule_CreateInitialized(): set mod->md_def + M. : initialize the module, etc. + N. import_run_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + O. import_run_extension(): set __file__ + P. import_run_extension() -> update_global_state_for_extension() + Q. update_global_state_for_extension(): + copy __dict__ into def->m_base.m_copy + R. update_global_state_for_extension(): + add it to _PyRuntime.imports.extensions + S. import_run_extension() -> finish_singlephase_extension() + T. finish_singlephase_extension(): + add it to interp->imports.modules_by_index + U. finish_singlephase_extension(): add it to sys.modules + + Step (Q) is skipped for core modules (sys/builtins). (6). subsequent times (found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. import_find_extension() -> import_add_module() - 3. if name in sys.modules: use that module - 4. else: - 1. import_add_module() -> PyModule_NewObject() - 2. import_add_module(): set it on sys.modules - 5. import_find_extension(): copy the "m_copy" dict into __dict__ - 6. _imp_create_dynamic_impl() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + A. _imp_create_dynamic_impl() -> import_find_extension() + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension() -> import_add_module() + E. if name in sys.modules: use that module + F. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + G. reload_singlephase_extension(): copy the "m_copy" dict into __dict__ + H. reload_singlephase_extension(): add to modules_by_index (10). (every time): - 1. noop + A. noop ...for single-phase init modules, where m_size >= 0: (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): - 1-16. (same as for m_size == -1) + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R. (skipped) + S-U. (same as for m_size == -1) (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): - 1-16. (same as for m_size == -1) - 17. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R-U. (same as for m_size == -1) - (6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. import_find_extension(): call def->m_base.m_init - 3. import_find_extension(): add the module to sys.modules + (6). subsequent times (found in _PyRuntime.imports.extensions): + A. _imp_create_dynamic_impl() -> import_find_extension() + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension(): call def->m_base.m_init (see above) + E. reload_singlephase_extension(): add the module to sys.modules + F. reload_singlephase_extension(): add to modules_by_index (10). every time: - 1. noop + A. noop ...for multi-phase init modules: (6). every time: - 1. _imp_create_dynamic_impl() -> import_find_extension() (not found) - 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() - 3. _PyImport_LoadDynamicModuleWithSpec(): load module init func - 4. _PyImport_LoadDynamicModuleWithSpec(): call module init func - 5. _PyImport_LoadDynamicModuleWithSpec() -> PyModule_FromDefAndSpec() - 6. PyModule_FromDefAndSpec(): gather/check moduledef slots - 7. if there's a Py_mod_create slot: + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> import_run_extension() + E. import_run_extension() -> _PyImport_RunModInitFunc() + F. _PyImport_RunModInitFunc(): call + G. import_run_extension() -> PyModule_FromDefAndSpec() + H. PyModule_FromDefAndSpec(): gather/check moduledef slots + I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function - 8. else: + J. else: 1. PyModule_FromDefAndSpec() -> PyModule_NewObject() - 9: PyModule_FromDefAndSpec(): set mod->md_def - 10. PyModule_FromDefAndSpec() -> _add_methods_to_object() - 11. PyModule_FromDefAndSpec() -> PyModule_SetDocString() + K: PyModule_FromDefAndSpec(): set mod->md_def + L. PyModule_FromDefAndSpec() -> _add_methods_to_object() + M. PyModule_FromDefAndSpec() -> PyModule_SetDocString() (10). every time: - 1. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() - 2. if mod->md_state == NULL (including if m_size == 0): + A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() + B. if mod->md_state == NULL (including if m_size == 0): 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() 2. PyModule_ExecDef(): allocate mod->md_state 3. if there's a Py_mod_exec slot: @@ -880,22 +903,23 @@ gets even messier. static inline void extensions_lock_acquire(void) { - PyThread_acquire_lock(_PyRuntime.imports.extensions.mutex, WAIT_LOCK); + PyMutex_Lock(&_PyRuntime.imports.extensions.mutex); } static inline void extensions_lock_release(void) { - PyThread_release_lock(_PyRuntime.imports.extensions.mutex); + PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex); } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple (module name, module name) (for built-in modules) or by (filename, module name) (for dynamically loaded modules), containing these modules. A copy of the module's dictionary is stored by calling - _PyImport_FixupExtensionObject() immediately after the module initialization + fix_up_extension() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling import_find_extension(). @@ -905,6 +929,220 @@ extensions_lock_release(void) dictionary, to avoid loading shared libraries twice. */ +typedef struct cached_m_dict { + /* A shallow copy of the original module's __dict__. */ + PyObject *copied; + /* The interpreter that owns the copy. */ + int64_t interpid; +} *cached_m_dict_t; + +struct extensions_cache_value { + PyModuleDef *def; + + /* The function used to re-initialize the module. + This is only set for legacy (single-phase init) extension modules + and only used for those that support multiple initializations + (m_size >= 0). + It is set by update_global_state_for_extension(). */ + PyModInitFunction m_init; + + /* The module's index into its interpreter's modules_by_index cache. + This is set for all extension modules but only used for legacy ones. + (See PyInterpreterState.modules_by_index for more info.) */ + Py_ssize_t m_index; + + /* A copy of the module's __dict__ after the first time it was loaded. + This is only set/used for legacy modules that do not support + multiple initializations. + It is set exclusively by fixup_cached_def(). */ + cached_m_dict_t m_dict; + struct cached_m_dict _m_dict; + + _Py_ext_module_origin origin; + +#ifdef Py_GIL_DISABLED + /* The module's md_gil slot, for legacy modules that are reinitialized from + m_dict rather than calling their initialization function again. */ + void *md_gil; +#endif +}; + +static struct extensions_cache_value * +alloc_extensions_cache_value(void) +{ + struct extensions_cache_value *value + = PyMem_RawMalloc(sizeof(struct extensions_cache_value)); + if (value == NULL) { + PyErr_NoMemory(); + return NULL; + } + *value = (struct extensions_cache_value){0}; + return value; +} + +static void +free_extensions_cache_value(struct extensions_cache_value *value) +{ + PyMem_RawFree(value); +} + +static Py_ssize_t +_get_cached_module_index(struct extensions_cache_value *cached) +{ + assert(cached->m_index > 0); + return cached->m_index; +} + +static void +fixup_cached_def(struct extensions_cache_value *value) +{ + /* For the moment, the values in the def's m_base may belong + * to another module, and we're replacing them here. This can + * cause problems later if the old module is reloaded. + * + * Also, we don't decref any old cached values first when we + * replace them here, in case we need to restore them in the + * near future. Instead, the caller is responsible for wrapping + * this up by calling cleanup_old_cached_def() or + * restore_old_cached_def() if there was an error. */ + PyModuleDef *def = value->def; + assert(def != NULL); + + /* We assume that all module defs are statically allocated + and will never be freed. Otherwise, we would incref here. */ + _Py_SetImmortalUntracked((PyObject *)def); + + def->m_base.m_init = value->m_init; + + assert(value->m_index > 0); + _set_module_index(def, value->m_index); + + /* Different modules can share the same def, so we can't just + * expect m_copy to be NULL. */ + assert(def->m_base.m_copy == NULL + || def->m_base.m_init == NULL + || value->m_dict != NULL); + if (value->m_dict != NULL) { + assert(value->m_dict->copied != NULL); + /* As noted above, we don't first decref the old value, if any. */ + def->m_base.m_copy = Py_NewRef(value->m_dict->copied); + } +} + +static void +restore_old_cached_def(PyModuleDef *def, PyModuleDef_Base *oldbase) +{ + def->m_base = *oldbase; +} + +static void +cleanup_old_cached_def(PyModuleDef_Base *oldbase) +{ + Py_XDECREF(oldbase->m_copy); +} + +static void +del_cached_def(struct extensions_cache_value *value) +{ + /* If we hadn't made the stored defs immortal, we would decref here. + However, this decref would be problematic if the module def were + dynamically allocated, it were the last ref, and this function + were called with an interpreter other than the def's owner. */ + assert(value->def == NULL || _Py_IsImmortal(value->def)); + + Py_XDECREF(value->def->m_base.m_copy); + value->def->m_base.m_copy = NULL; +} + +static int +init_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict) +{ + assert(value != NULL); + /* This should only have been called without an m_dict already set. */ + assert(value->m_dict == NULL); + if (m_dict == NULL) { + return 0; + } + assert(PyDict_Check(m_dict)); + assert(value->origin != _Py_ext_module_origin_CORE); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!is_interpreter_isolated(interp)); + + /* XXX gh-88216: The copied dict is owned by the current + * interpreter. That's a problem if the interpreter has + * its own obmalloc state or if the module is successfully + * imported into such an interpreter. If the interpreter + * has its own GIL then there may be data races and + * PyImport_ClearModulesByIndex() can crash. Normally, + * a single-phase init module cannot be imported in an + * isolated interpreter, but there are ways around that. + * Hence, heere be dragons! Ideally we would instead do + * something like make a read-only, immortal copy of the + * dict using PyMem_RawMalloc() and store *that* in m_copy. + * Then we'd need to make sure to clear that when the + * runtime is finalized, rather than in + * PyImport_ClearModulesByIndex(). */ + PyObject *copied = PyDict_Copy(m_dict); + if (copied == NULL) { + /* We expect this can only be "out of memory". */ + return -1; + } + // XXX We may want to make the copy immortal. + + value->_m_dict = (struct cached_m_dict){ + .copied=copied, + .interpid=PyInterpreterState_GetID(interp), + }; + + value->m_dict = &value->_m_dict; + return 0; +} + +static void +del_cached_m_dict(struct extensions_cache_value *value) +{ + if (value->m_dict != NULL) { + assert(value->m_dict == &value->_m_dict); + assert(value->m_dict->copied != NULL); + /* In the future we can take advantage of m_dict->interpid + * to decref the dict using the owning interpreter. */ + Py_XDECREF(value->m_dict->copied); + value->m_dict = NULL; + } +} + +static PyObject * get_core_module_dict( + PyInterpreterState *interp, PyObject *name, PyObject *path); + +static PyObject * +get_cached_m_dict(struct extensions_cache_value *value, + PyObject *name, PyObject *path) +{ + assert(value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* It might be a core module (e.g. sys & builtins), + for which we don't cache m_dict. */ + if (value->origin == _Py_ext_module_origin_CORE) { + return get_core_module_dict(interp, name, path); + } + assert(value->def != NULL); + // XXX Switch to value->m_dict. + PyObject *m_dict = value->def->m_base.m_copy; + Py_XINCREF(m_dict); + return m_dict; +} + +static void +del_extensions_cache_value(struct extensions_cache_value *value) +{ + if (value != NULL) { + del_cached_m_dict(value); + del_cached_def(value); + free_extensions_cache_value(value); + } +} + static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { @@ -918,6 +1156,7 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; + // XXX Use a buffer if it's a temp value (every case but "set"). char *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); @@ -949,102 +1188,220 @@ hashtable_destroy_str(void *ptr) PyMem_RawFree(ptr); } -#define HTSEP ':' +#ifndef NDEBUG +struct hashtable_next_match_def_data { + PyModuleDef *def; + struct extensions_cache_value *matched; +}; -static PyModuleDef * -_extensions_cache_get(PyObject *filename, PyObject *name) +static int +hashtable_next_match_def(_Py_hashtable_t *ht, + const void *key, const void *value, void *user_data) { - PyModuleDef *def = NULL; - void *key = NULL; - extensions_lock_acquire(); + if (value == NULL) { + /* It was previously deleted. */ + return 0; + } + struct hashtable_next_match_def_data *data + = (struct hashtable_next_match_def_data *)user_data; + struct extensions_cache_value *cur + = (struct extensions_cache_value *)value; + if (cur->def == data->def) { + data->matched = cur; + return 1; + } + return 0; +} + +static struct extensions_cache_value * +_find_cached_def(PyModuleDef *def) +{ + struct hashtable_next_match_def_data data = {0}; + (void)_Py_hashtable_foreach( + EXTENSIONS.hashtable, hashtable_next_match_def, &data); + return data.matched; +} +#endif +#define HTSEP ':' + +static int +_extensions_cache_init(void) +{ + _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; + EXTENSIONS.hashtable = _Py_hashtable_new_full( + hashtable_hash_str, + hashtable_compare_str, + hashtable_destroy_str, // key + (_Py_hashtable_destroy_func)del_extensions_cache_value, // value + &alloc + ); if (EXTENSIONS.hashtable == NULL) { - goto finally; + PyErr_NoMemory(); + return -1; } + return 0; +} - key = hashtable_key_from_2_strings(filename, name, HTSEP); +static _Py_hashtable_entry_t * +_extensions_cache_find_unlocked(PyObject *path, PyObject *name, + void **p_key) +{ + if (EXTENSIONS.hashtable == NULL) { + return NULL; + } + void *key = hashtable_key_from_2_strings(path, name, HTSEP); if (key == NULL) { - goto finally; + return NULL; + } + _Py_hashtable_entry_t *entry = + _Py_hashtable_get_entry(EXTENSIONS.hashtable, key); + if (p_key != NULL) { + *p_key = key; + } + else { + hashtable_destroy_str(key); } - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + return entry; +} + +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_get(PyObject *path, PyObject *name) +{ + struct extensions_cache_value *value = NULL; + extensions_lock_acquire(); + + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { + /* It was never added. */ goto finally; } - def = (PyModuleDef *)entry->value; + value = (struct extensions_cache_value *)entry->value; finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } - return def; + return value; } -static int -_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_set(PyObject *path, PyObject *name, + PyModuleDef *def, PyModInitFunction m_init, + Py_ssize_t m_index, PyObject *m_dict, + _Py_ext_module_origin origin, void *md_gil) { - int res = -1; + struct extensions_cache_value *value = NULL; + void *key = NULL; + struct extensions_cache_value *newvalue = NULL; + PyModuleDef_Base olddefbase = def->m_base; + + assert(def != NULL); + assert(m_init == NULL || m_dict == NULL); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(def->m_base.m_init == NULL || def->m_base.m_init == m_init); + /* For now we don't worry about comparing value->m_copy. */ + assert(def->m_base.m_copy == NULL || m_dict != NULL); + assert((origin == _Py_ext_module_origin_DYNAMIC) == (name != path)); + assert(origin != _Py_ext_module_origin_CORE || m_dict == NULL); + extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { - _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; - EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key - /* There's no need to decref the def since it's immortal. */ - NULL, // value - &alloc - ); - if (EXTENSIONS.hashtable == NULL) { - PyErr_NoMemory(); + if (_extensions_cache_init() < 0) { goto finally; } } - void *key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { + /* Create a cached value to populate for the module. */ + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, &key); + value = entry == NULL + ? NULL + : (struct extensions_cache_value *)entry->value; + /* We should never be updating an existing cache value. */ + assert(value == NULL); + if (value != NULL) { + PyErr_Format(PyExc_SystemError, + "extension module %R is already cached", name); + goto finally; + } + newvalue = alloc_extensions_cache_value(); + if (newvalue == NULL) { goto finally; } - int already_set = 0; - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + /* Populate the new cache value data. */ + *newvalue = (struct extensions_cache_value){ + .def=def, + .m_init=m_init, + .m_index=m_index, + /* m_dict is set by set_cached_m_dict(). */ + .origin=origin, +#ifdef Py_GIL_DISABLED + .md_gil=md_gil, +#endif + }; +#ifndef Py_GIL_DISABLED + (void)md_gil; +#endif + if (init_cached_m_dict(newvalue, m_dict) < 0) { + goto finally; + } + fixup_cached_def(newvalue); + if (entry == NULL) { - if (_Py_hashtable_set(EXTENSIONS.hashtable, key, def) < 0) { - PyMem_RawFree(key); + /* It was never added. */ + if (_Py_hashtable_set(EXTENSIONS.hashtable, key, newvalue) < 0) { PyErr_NoMemory(); goto finally; } + /* The hashtable owns the key now. */ + key = NULL; + } + else if (value == NULL) { + /* It was previously deleted. */ + entry->value = newvalue; } else { - if (entry->value == NULL) { - entry->value = def; - } - else { - /* We expect it to be static, so it must be the same pointer. */ - assert((PyModuleDef *)entry->value == def); - already_set = 1; + /* We are updating the entry for an existing module. */ + /* We expect def to be static, so it must be the same pointer. */ + assert(value->def == def); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(value->m_init == m_init); + /* The same module can't switch between caching __dict__ and not. */ + assert((value->m_dict == NULL) == (m_dict == NULL)); + /* This shouldn't ever happen. */ + Py_UNREACHABLE(); + } + + value = newvalue; + +finally: + if (value == NULL) { + restore_old_cached_def(def, &olddefbase); + if (newvalue != NULL) { + del_extensions_cache_value(newvalue); } - PyMem_RawFree(key); } - if (!already_set) { - /* We assume that all module defs are statically allocated - and will never be freed. Otherwise, we would incref here. */ - _Py_SetImmortal(def); + else { + cleanup_old_cached_def(&olddefbase); } - res = 0; -finally: extensions_lock_release(); - return res; + if (key != NULL) { + hashtable_destroy_str(key); + } + + return value; } static void -_extensions_cache_delete(PyObject *filename, PyObject *name) +_extensions_cache_delete(PyObject *path, PyObject *name) { - void *key = NULL; extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { @@ -1052,13 +1409,8 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) goto finally; } - key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { - goto finally; - } - - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { /* It was never added. */ goto finally; @@ -1067,18 +1419,13 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) /* It was already removed. */ goto finally; } - /* If we hadn't made the stored defs immortal, we would decref here. - However, this decref would be problematic if the module def were - dynamically allocated, it were the last ref, and this function - were called with an interpreter other than the def's owner. */ - assert(_Py_IsImmortal(entry->value)); + struct extensions_cache_value *value = entry->value; entry->value = NULL; + del_extensions_cache_value(value); + finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } } static void @@ -1107,48 +1454,139 @@ check_multi_interp_extensions(PyInterpreterState *interp) interp, Py_RTFLAGS_MULTI_INTERP_EXTENSIONS)) { return true; } - return false; + return false; +} + +int +_PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (check_multi_interp_extensions(interp)) { + assert(!_Py_IsMainInterpreter(interp)); + PyErr_Format(PyExc_ImportError, + "module %s does not support loading in subinterpreters", + name); + return -1; + } + return 0; +} + +#ifdef Py_GIL_DISABLED +int +_PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (module == NULL) { + _PyEval_DisableGIL(tstate); + return 0; + } + + if (!PyModule_Check(module) || + ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + if (_PyEval_EnableGILPermanent(tstate)) { + int warn_result = PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "The global interpreter lock (GIL) has been enabled to load " + "module '%U', which has not declared that it can run safely " + "without the GIL. To override this behavior and keep the GIL " + "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", + module_name + ); + if (warn_result < 0) { + return warn_result; + } + } + + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { + PySys_FormatStderr("# loading module '%U', which requires the GIL\n", + module_name); + } + } + else { + _PyEval_DisableGIL(tstate); + } + + return 0; +} +#endif + +static PyThreadState * +switch_to_main_interpreter(PyThreadState *tstate) +{ + if (_Py_IsMainInterpreter(tstate->interp)) { + return tstate; + } + PyThreadState *main_tstate = _PyThreadState_NewBound( + _PyInterpreterState_Main(), _PyThreadState_WHENCE_EXEC); + if (main_tstate == NULL) { + return NULL; + } +#ifndef NDEBUG + PyThreadState *old_tstate = PyThreadState_Swap(main_tstate); + assert(old_tstate == tstate); +#else + (void)PyThreadState_Swap(main_tstate); +#endif + return main_tstate; } -int -_PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (check_multi_interp_extensions(interp)) { - assert(!_Py_IsMainInterpreter(interp)); - PyErr_Format(PyExc_ImportError, - "module %s does not support loading in subinterpreters", - name); - return -1; +static void +switch_back_from_main_interpreter(PyThreadState *tstate, + PyThreadState *main_tstate, + PyObject *tempobj) +{ + assert(main_tstate == PyThreadState_GET()); + assert(_Py_IsMainInterpreter(main_tstate->interp)); + assert(tstate->interp != main_tstate->interp); + + /* Handle any exceptions, which we cannot propagate directly + * to the subinterpreter. */ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* We trust it will be caught again soon. */ + PyErr_Clear(); + } + else { + /* Printing the exception should be sufficient. */ + PyErr_PrintEx(0); + } } - return 0; + + Py_XDECREF(tempobj); + + PyThreadState_Clear(main_tstate); + (void)PyThreadState_Swap(tstate); + PyThreadState_Delete(main_tstate); } static PyObject * get_core_module_dict(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { /* Only builtin modules are core. */ - if (filename == name) { + if (path == name) { assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { - return interp->sysdict_copy; + return Py_NewRef(interp->sysdict_copy); } assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { - return interp->builtins_copy; + return Py_NewRef(interp->builtins_copy); } assert(!PyErr_Occurred()); } return NULL; } +#ifndef NDEBUG static inline int -is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) +is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) { /* This might be called before the core dict copies are in place, so we can't rely on get_core_module_dict() here. */ - if (filename == name) { + if (path == name) { if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { return 1; } @@ -1159,171 +1597,576 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) return 0; } -static int -fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) + +static _Py_ext_module_kind +_get_extension_kind(PyModuleDef *def, bool check_size) { - if (mod == NULL || !PyModule_Check(mod)) { - PyErr_BadInternalCall(); - return -1; + _Py_ext_module_kind kind; + if (def == NULL) { + /* It must be a module created by reload_singlephase_extension() + * from m_copy. Ideally we'd do away with this case. */ + kind = _Py_ext_module_kind_SINGLEPHASE; } - - struct PyModuleDef *def = PyModule_GetDef(mod); - if (!def) { - PyErr_BadInternalCall(); - return -1; + else if (def->m_slots != NULL) { + kind = _Py_ext_module_kind_MULTIPHASE; } - - PyThreadState *tstate = _PyThreadState_GET(); - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { - return -1; + else if (check_size && def->m_size == -1) { + kind = _Py_ext_module_kind_SINGLEPHASE; } + else if (def->m_base.m_init != NULL) { + kind = _Py_ext_module_kind_SINGLEPHASE; + } + else { + // This is probably single-phase init, but a multi-phase + // module *can* have NULL m_slots. + kind = _Py_ext_module_kind_UNKNOWN; + } + return kind; +} + +/* The module might not be fully initialized yet + * and PyModule_FromDefAndSpec() checks m_size + * so we skip m_size. */ +#define assert_multiphase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, false); \ + assert(kind == _Py_ext_module_kind_MULTIPHASE \ + /* m_slots can be NULL. */ \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase(cached) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(cached->def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE); \ + } while (0) + +#else /* defined(NDEBUG) */ +#define assert_multiphase_def(def) +#define assert_singlephase_def(def) +#define assert_singlephase(cached) +#endif - // bpo-44050: Extensions and def->m_base.m_copy can be updated - // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1) { - if (!is_core_module(tstate->interp, name, filename)) { + +struct singlephase_global_update { + PyModInitFunction m_init; + Py_ssize_t m_index; + PyObject *m_dict; + _Py_ext_module_origin origin; + void *md_gil; +}; + +static struct extensions_cache_value * +update_global_state_for_extension(PyThreadState *tstate, + PyObject *path, PyObject *name, + PyModuleDef *def, + struct singlephase_global_update *singlephase) +{ + struct extensions_cache_value *cached = NULL; + PyModInitFunction m_init = NULL; + PyObject *m_dict = NULL; + + /* Set up for _extensions_cache_set(). */ + if (singlephase == NULL) { + assert(def->m_base.m_init == NULL); + assert(def->m_base.m_copy == NULL); + } + else { + if (singlephase->m_init != NULL) { + assert(singlephase->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + assert(def->m_size >= 0); + /* Remember pointer to module init function. */ + // XXX If two modules share a def then def->m_base will + // reflect the last one added (here) to the global cache. + // We should prevent this somehow. The simplest solution + // is probably to store m_copy/m_init in the cache along + // with the def, rather than within the def. + m_init = singlephase->m_init; + } + else if (singlephase->m_dict == NULL) { + /* It must be a core builtin module. */ + assert(is_core_module(tstate->interp, name, path)); + assert(def->m_size == -1); + assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_init == NULL); + } + else { + assert(PyDict_Check(singlephase->m_dict)); + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + assert(def->m_size == -1); + assert(!is_core_module(tstate->interp, name, path)); assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); - if (def->m_base.m_copy) { - /* Somebody already imported the module, - likely under a different name. - XXX this should really not happen. */ - Py_CLEAR(def->m_base.m_copy); - } - PyObject *dict = PyModule_GetDict(mod); - if (dict == NULL) { - return -1; - } - def->m_base.m_copy = PyDict_Copy(dict); - if (def->m_base.m_copy == NULL) { - return -1; - } + m_dict = singlephase->m_dict; } } + /* Add the module's def to the global cache. */ // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - if (_extensions_cache_set(filename, name, def) < 0) { - return -1; +#ifndef NDEBUG + cached = _extensions_cache_get(path, name); + assert(cached == NULL || cached->def == def); +#endif + cached = _extensions_cache_set( + path, name, def, m_init, singlephase->m_index, m_dict, + singlephase->origin, singlephase->md_gil); + if (cached == NULL) { + // XXX Ignore this error? Doing so would effectively + // mark the module as not loadable. + return NULL; } } - return 0; + return cached; } -int -_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) +/* For multi-phase init modules, the module is finished + * by PyModule_FromDefAndSpec(). */ +static int +finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, + struct extensions_cache_value *cached, + PyObject *name, PyObject *modules) { - if (PyObject_SetItem(modules, name, mod) < 0) { + assert(mod != NULL && PyModule_Check(mod)); + assert(cached->def == _PyModule_GetDef(mod)); + + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { return -1; } - if (fix_up_extension(mod, name, filename) < 0) { - PyMapping_DelItem(modules, name); - return -1; + + if (modules != NULL) { + if (PyObject_SetItem(modules, name, mod) < 0) { + return -1; + } } + return 0; } static PyObject * -import_find_extension(PyThreadState *tstate, PyObject *name, - PyObject *filename) +reload_singlephase_extension(PyThreadState *tstate, + struct extensions_cache_value *cached, + struct _Py_ext_module_loader_info *info) { - /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(filename, name); - if (def == NULL) { - return NULL; - } + PyModuleDef *def = cached->def; + assert(def != NULL); + assert_singlephase(cached); + PyObject *mod = NULL; /* It may have been successfully imported previously in an interpreter that allows legacy modules but is not allowed in the current interpreter. */ - const char *name_buf = PyUnicode_AsUTF8(name); + const char *name_buf = PyUnicode_AsUTF8(info->name); assert(name_buf != NULL); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { return NULL; } - PyObject *mod, *mdict; - PyObject *modules = MODULES(tstate->interp); - + PyObject *modules = get_modules_dict(tstate, true); if (def->m_size == -1) { - PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ + assert(cached->m_init == NULL); + assert(def->m_base.m_init == NULL); + // XXX Copying the cached dict may break interpreter isolation. + // We could solve this by temporarily acquiring the original + // interpreter's GIL. + PyObject *m_copy = get_cached_m_dict(cached, info->name, info->path); if (m_copy == NULL) { - /* It might be a core module (e.g. sys & builtins), - for which we don't set m_copy. */ - m_copy = get_core_module_dict(tstate->interp, name, filename); - if (m_copy == NULL) { - return NULL; - } + assert(!PyErr_Occurred()); + return NULL; } - mod = import_add_module(tstate, name); + mod = import_add_module(tstate, info->name); if (mod == NULL) { + Py_DECREF(m_copy); return NULL; } - mdict = PyModule_GetDict(mod); + PyObject *mdict = PyModule_GetDict(mod); if (mdict == NULL) { + Py_DECREF(m_copy); Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, m_copy)) { + int rc = PyDict_Update(mdict, m_copy); + Py_DECREF(m_copy); + if (rc < 0) { Py_DECREF(mod); return NULL; } +#ifdef Py_GIL_DISABLED + if (def->m_base.m_copy != NULL) { + // For non-core modules, fetch the GIL slot that was stored by + // import_run_extension(). + ((PyModuleObject *)mod)->md_gil = cached->md_gil; + } +#endif + /* We can't set mod->md_def if it's missing, + * because _PyImport_ClearModulesByIndex() might break + * due to violating interpreter isolation. + * See the note in set_cached_m_dict(). + * Until that is solved, we leave md_def set to NULL. */ + assert(_PyModule_GetDef(mod) == NULL + || _PyModule_GetDef(mod) == def); } else { - if (def->m_base.m_init == NULL) + assert(cached->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + // XXX Use cached->m_init. + PyModInitFunction p0 = def->m_base.m_init; + if (p0 == NULL) { + assert(!PyErr_Occurred()); return NULL; - mod = def->m_base.m_init(); - if (mod == NULL) + } + struct _Py_ext_module_loader_result res; + if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); return NULL; - if (PyObject_SetItem(modules, name, mod) == -1) { + } + assert(!PyErr_Occurred()); + assert(res.err == NULL); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + mod = res.module; + /* Tchnically, the init function could return a different module def. + * Then we would probably need to update the global cache. + * However, we don't expect anyone to change the def. */ + assert(res.def == def); + _Py_ext_module_loader_result_clear(&res); + + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + if (PyObject_SetItem(modules, info->name, mod) == -1) { Py_DECREF(mod); return NULL; } } - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { - PyMapping_DelItem(modules, name); + + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { + PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; } + return mod; +} + +static PyObject * +import_find_extension(PyThreadState *tstate, + struct _Py_ext_module_loader_info *info, + struct extensions_cache_value **p_cached) +{ + /* Only single-phase init modules will be in the cache. */ + struct extensions_cache_value *cached + = _extensions_cache_get(info->path, info->name); + if (cached == NULL) { + return NULL; + } + assert(cached->def != NULL); + assert_singlephase(cached); + *p_cached = cached; + + /* It may have been successfully imported previously + in an interpreter that allows legacy modules + but is not allowed in the current interpreter. */ + const char *name_buf = PyUnicode_AsUTF8(info->name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + return NULL; + } + + PyObject *mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + return NULL; + } + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", - name, filename); + info->name, info->path); + } + + return mod; +} + +static PyObject * +import_run_extension(PyThreadState *tstate, PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + PyObject *spec, PyObject *modules) +{ + /* Core modules go through _PyImport_FixupBuiltin(). */ + assert(!is_core_module(tstate->interp, info->name, info->path)); + + PyObject *mod = NULL; + PyModuleDef *def = NULL; + struct extensions_cache_value *cached = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + + /* We cannot know if the module is single-phase init or + * multi-phase init until after we call its init function. Even + * in isolated interpreters (that do not support single-phase init), + * the init function will run without restriction. For multi-phase + * init modules that isn't a problem because the init function only + * runs PyModuleDef_Init() on the module's def and then returns it. + * + * However, for single-phase init the module's init function will + * create the module, create other objects (and allocate other + * memory), populate it and its module state, and initialize static + * types. Some modules store other objects and data in global C + * variables and register callbacks with the runtime/stdlib or + * even external libraries (which is part of why we can't just + * dlclose() the module in the error case). That's a problem + * for isolated interpreters since all of the above happens + * and only then * will the import fail. Memory will leak, + * callbacks will still get used, and sometimes there + * will be crashes (memory access violations + * and use-after-free). + * + * To put it another way, if the module is single-phase init + * then the import will probably break interpreter isolation + * and should fail ASAP. However, the module's init function + * will still get run. That means it may still store state + * in the shared-object/DLL address space (which never gets + * closed/cleared), including objects (e.g. static types). + * This is a problem for isolated subinterpreters since each + * has its own object allocator. If the loaded shared-object + * still holds a reference to an object after the corresponding + * interpreter has finalized then either we must let it leak + * or else any later use of that object by another interpreter + * (or across multiple init-fini cycles) will crash the process. + * + * To avoid all of that, we make sure the module's init function + * is always run first with the main interpreter active. If it was + * already the main interpreter then we can continue loading the + * module like normal. Otherwise, right after the init function, + * we take care of some import state bookkeeping, switch back + * to the subinterpreter, check for single-phase init, + * and then continue loading like normal. */ + + bool switched = false; + /* We *could* leave in place a legacy interpreter here + * (one that shares obmalloc/GIL with main interp), + * but there isn't a big advantage, we anticipate + * such interpreters will be increasingly uncommon, + * and the code is a bit simpler if we always switch + * to the main interpreter. */ + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return NULL; + } + else if (main_tstate != tstate) { + switched = true; + /* In the switched case, we could play it safe + * by getting the main interpreter's import lock here. + * It's unlikely to matter though. */ + } + + struct _Py_ext_module_loader_result res; + int rc = _PyImport_RunModInitFunc(p0, info, &res); + if (rc < 0) { + /* We discard res.def. */ + assert(res.module == NULL); + } + else { + assert(!PyErr_Occurred()); + assert(res.err == NULL); + + mod = res.module; + res.module = NULL; + def = res.def; + assert(def != NULL); + + /* Do anything else that should be done + * while still using the main interpreter. */ + if (res.kind == _Py_ext_module_kind_SINGLEPHASE) { + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + PyObject *filename = NULL; + if (switched) { + // The original filename may be allocated by subinterpreter's + // obmalloc, so we create a copy here. + filename = _PyUnicode_Copy(info->filename); + if (filename == NULL) { + return NULL; + } + } else { + filename = Py_NewRef(info->filename); + } + // XXX There's a refleak somewhere with the filename. + // Until we can track it down, we immortalize it. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &filename); + + if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + /* Update global import state. */ + assert(def->m_base.m_index != 0); + struct singlephase_global_update singlephase = { + // XXX Modules that share a def should each get their own index, + // whereas currently they share (which means the per-interpreter + // cache is less reliable than it should be). + .m_index=def->m_base.m_index, + .origin=info->origin, +#ifdef Py_GIL_DISABLED + .md_gil=((PyModuleObject *)mod)->md_gil, +#endif + }; + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (def->m_size == -1) { + /* We will reload from m_copy. */ + assert(def->m_base.m_init == NULL); + singlephase.m_dict = PyModule_GetDict(mod); + assert(singlephase.m_dict != NULL); + } + else { + /* We will reload via the init function. */ + assert(def->m_size >= 0); + assert(def->m_base.m_copy == NULL); + singlephase.m_init = p0; + } + cached = update_global_state_for_extension( + tstate, info->path, info->name, def, &singlephase); + if (cached == NULL) { + assert(PyErr_Occurred()); + goto main_finally; + } + } + } + +main_finally: + /* Switch back to the subinterpreter. */ + if (switched) { + assert(main_tstate != tstate); + switch_back_from_main_interpreter(tstate, main_tstate, mod); + /* Any module we got from the init function will have to be + * reloaded in the subinterpreter. */ + mod = NULL; + } + + /*****************************************************************/ + /* At this point we are back to the interpreter we started with. */ + /*****************************************************************/ + + /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + goto error; + } + + if (res.kind == _Py_ext_module_kind_MULTIPHASE) { + assert_multiphase_def(def); + assert(mod == NULL); + /* Note that we cheat a little by not repeating the calls + * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ + mod = PyModule_FromDefAndSpec(def, spec); + if (mod == NULL) { + goto error; + } + } + else { + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + assert_singlephase_def(def); + + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + goto error; + } + assert(!PyErr_Occurred()); + + if (switched) { + /* We switched to the main interpreter to run the init + * function, so now we will "reload" the module from the + * cached data using the original subinterpreter. */ + assert(mod == NULL); + mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + goto error; + } + assert(!PyErr_Occurred()); + assert(PyModule_Check(mod)); + } + else { + assert(mod != NULL); + assert(PyModule_Check(mod)); + + /* Update per-interpreter import state. */ + PyObject *modules = get_modules_dict(tstate, true); + if (finish_singlephase_extension( + tstate, mod, cached, info->name, modules) < 0) + { + goto error; + } + } } + + _Py_ext_module_loader_result_clear(&res); return mod; + +error: + Py_XDECREF(mod); + _Py_ext_module_loader_result_clear(&res); + return NULL; } + static int clear_singlephase_extension(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { - PyModuleDef *def = _extensions_cache_get(filename, name); - if (def == NULL) { + struct extensions_cache_value *cached = _extensions_cache_get(path, name); + if (cached == NULL) { if (PyErr_Occurred()) { return -1; } return 0; } + PyModuleDef *def = cached->def; /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; Py_CLEAR(def->m_base.m_copy); - // We leave m_index alone since there's no reason to reset it. + def->m_base.m_index = 0; /* Clear the PyState_*Module() cache entry. */ - if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear_one(interp, def) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_check(interp, index) == NULL) { + if (_modules_by_index_clear_one(interp, index) < 0) { return -1; } } + /* We must use the main interpreter to clean up the cache. + * See the note in import_run_extension(). */ + PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return -1; + } + /* Clear the cached module def. */ - _extensions_cache_delete(filename, name); + _extensions_cache_delete(path, name); + + if (main_tstate != tstate) { + switch_back_from_main_interpreter(tstate, main_tstate, NULL); + } return 0; } @@ -1334,21 +2177,62 @@ clear_singlephase_extension(PyInterpreterState *interp, /*******************/ int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) +_PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, + PyObject *modules) { int res = -1; + assert(mod != NULL && PyModule_Check(mod)); + PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) { return -1; } - if (PyObject_SetItem(modules, nameobj, mod) < 0) { + + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + PyErr_BadInternalCall(); goto finally; } - if (fix_up_extension(mod, nameobj, nameobj) < 0) { - PyMapping_DelItem(modules, nameobj); + + /* We only use _PyImport_FixupBuiltin() for the core builtin modules + * (sys and builtins). These modules are single-phase init with no + * module state, but we also don't populate def->m_base.m_copy + * for them. */ + assert(is_core_module(tstate->interp, nameobj, nameobj)); + assert_singlephase_def(def); + assert(def->m_size == -1); + assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_index >= 0); + + /* We aren't using import_find_extension() for core modules, + * so we have to do the extra check to make sure the module + * isn't already in the global cache before calling + * update_global_state_for_extension(). */ + struct extensions_cache_value *cached + = _extensions_cache_get(nameobj, nameobj); + if (cached == NULL) { + struct singlephase_global_update singlephase = { + .m_index=def->m_base.m_index, + /* We don't want def->m_base.m_copy populated. */ + .m_dict=NULL, + .origin=_Py_ext_module_origin_CORE, +#ifdef Py_GIL_DISABLED + /* Unused when m_dict == NULL. */ + .md_gil=NULL, +#endif + }; + cached = update_global_state_for_extension( + tstate, nameobj, nameobj, def, &singlephase); + if (cached == NULL) { + goto finally; + } + } + + if (finish_singlephase_extension(tstate, mod, cached, nameobj, modules) < 0) { goto finally; } + res = 0; finally: @@ -1377,45 +2261,76 @@ is_builtin(PyObject *name) static PyObject* create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) { - PyObject *mod = import_find_extension(tstate, name, name); - if (mod || _PyErr_Occurred(tstate)) { - return mod; + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { + return NULL; } - PyObject *modules = MODULES(tstate->interp); - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - return import_add_module(tstate, name); - } - mod = (*p->initfunc)(); - if (mod == NULL) { - return NULL; - } + struct extensions_cache_value *cached = NULL; + PyObject *mod = import_find_extension(tstate, &info, &cached); + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); + goto finally; + } + else if (_PyErr_Occurred(tstate)) { + goto finally; + } - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } + struct _inittab *found = NULL; + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { + found = p; } } + if (found == NULL) { + // not found + mod = Py_NewRef(Py_None); + goto finally; + } - // not found - Py_RETURN_NONE; + PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } + +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif + /* Now load it. */ + mod = import_run_extension( + tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif + +finally: + _Py_ext_module_loader_info_clear(&info); + return mod; } @@ -2373,11 +3288,11 @@ get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, if (nhooks < 0) return NULL; /* Shouldn't happen */ - importer = PyDict_GetItemWithError(path_importer_cache, p); - if (importer != NULL || _PyErr_Occurred(tstate)) { - return Py_XNewRef(importer); + if (PyDict_GetItemRef(path_importer_cache, p, &importer) != 0) { + // found or error + return importer; } - + // not found /* set path_importer_cache[p] to None to avoid recursion */ if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) return NULL; @@ -2566,7 +3481,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level { PyObject *abs_name; PyObject *package = NULL; - PyObject *spec; + PyObject *spec = NULL; Py_ssize_t last_dot; PyObject *base; int level_up; @@ -2579,20 +3494,18 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level _PyErr_SetString(tstate, PyExc_TypeError, "globals must be a dict"); goto error; } - package = PyDict_GetItemWithError(globals, &_Py_ID(__package__)); + if (PyDict_GetItemRef(globals, &_Py_ID(__package__), &package) < 0) { + goto error; + } if (package == Py_None) { + Py_DECREF(package); package = NULL; } - else if (package == NULL && _PyErr_Occurred(tstate)) { - goto error; - } - spec = PyDict_GetItemWithError(globals, &_Py_ID(__spec__)); - if (spec == NULL && _PyErr_Occurred(tstate)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__spec__), &spec) < 0) { goto error; } if (package != NULL) { - Py_INCREF(package); if (!PyUnicode_Check(package)) { _PyErr_SetString(tstate, PyExc_TypeError, "package must be a string"); @@ -2636,16 +3549,15 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level goto error; } - package = PyDict_GetItemWithError(globals, &_Py_ID(__name__)); + if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &package) < 0) { + goto error; + } if (package == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_KeyError, - "'__name__' not in globals"); - } + _PyErr_SetString(tstate, PyExc_KeyError, + "'__name__' not in globals"); goto error; } - Py_INCREF(package); if (!PyUnicode_Check(package)) { _PyErr_SetString(tstate, PyExc_TypeError, "__name__ must be a string"); @@ -2693,6 +3605,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level } } + Py_XDECREF(spec); base = PyUnicode_Substring(package, 0, last_dot); Py_DECREF(package); if (base == NULL || PyUnicode_GET_LENGTH(name) == 0) { @@ -2709,6 +3622,7 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level "with no known parent package"); error: + Py_XDECREF(spec); Py_XDECREF(package); return NULL; } @@ -2722,7 +3636,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #define import_level FIND_AND_LOAD(interp).import_level #define accumulated FIND_AND_LOAD(interp).accumulated - _PyTime_t t1 = 0, accumulated_copy = accumulated; + PyTime_t t1 = 0, accumulated_copy = accumulated; PyObject *sys_path = PySys_GetObject("path"); PyObject *sys_meta_path = PySys_GetObject("meta_path"); @@ -2750,7 +3664,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_GetPerfCounter(); + // ignore error: don't block import if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); accumulated = 0; } @@ -2765,7 +3680,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - _PyTime_t cum = _PyTime_GetPerfCounter() - t1; + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + PyTime_t cum = t2 - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", @@ -3157,18 +4074,13 @@ _PyImport_FiniCore(PyInterpreterState *interp) int verbose = _PyInterpreterState_GetConfig(interp)->verbose; if (_PySys_ClearAttrString(interp, "meta_path", verbose) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing sys.meta_path"); } // XXX Pull in most of finalize_modules() in pylifecycle.c. if (_PySys_ClearAttrString(interp, "modules", verbose) < 0) { - PyErr_WriteUnraisable(NULL); - } - - if (IMPORT_LOCK(interp) != NULL) { - PyThread_free_lock(IMPORT_LOCK(interp)); - IMPORT_LOCK(interp) = NULL; + PyErr_FormatUnraisable("Exception ignored on clearing sys.modules"); } _PyImport_ClearCore(interp); @@ -3243,10 +4155,10 @@ _PyImport_FiniExternal(PyInterpreterState *interp) // XXX Uninstall importlib metapath importers here? if (_PySys_ClearAttrString(interp, "path_importer_cache", verbose) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing sys.path_importer_cache"); } if (_PySys_ClearAttrString(interp, "path_hooks", verbose) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing sys.path_hooks"); } } @@ -3303,8 +4215,7 @@ _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyBool_FromLong( - IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); + return PyBool_FromLong(PyMutex_IsLocked(&IMPORT_LOCK(interp).mutex)); } /*[clinic input] @@ -3338,11 +4249,12 @@ _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) < 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&IMPORT_LOCK(interp))) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } + _PyImport_ReleaseLock(interp); Py_RETURN_NONE; } @@ -3547,7 +4459,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, struct frozen_info info = {0}; Py_buffer buf = {0}; if (PyObject_CheckBuffer(dataobj)) { - if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) { + if (PyObject_GetBuffer(dataobj, &buf, PyBUF_SIMPLE) != 0) { return NULL; } info.data = (const char *)buf.buf; @@ -3698,9 +4610,16 @@ _imp__override_multi_interp_extensions_check_impl(PyObject *module, "cannot be used in the main interpreter"); return NULL; } +#ifdef Py_GIL_DISABLED + PyErr_SetString(PyExc_RuntimeError, + "_imp._override_multi_interp_extensions_check() " + "cannot be used in the free-threaded build"); + return NULL; +#else int oldvalue = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) = override; return PyLong_FromLong(oldvalue); +#endif } #ifdef HAVE_DYNAMIC_LOADING @@ -3719,44 +4638,88 @@ static PyObject * _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { - PyObject *mod, *name, *path; - FILE *fp; + PyObject *mod = NULL; + PyThreadState *tstate = _PyThreadState_GET(); - name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_init_from_spec(&info, spec) < 0) { return NULL; } - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) { - Py_DECREF(name); - return NULL; + struct extensions_cache_value *cached = NULL; + mod = import_find_extension(tstate, &info, &cached); + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); + goto finally; + } + else if (_PyErr_Occurred(tstate)) { + goto finally; } + /* Otherwise it must be multi-phase init or the first time it's loaded. */ - PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, name, path); - if (mod != NULL || _PyErr_Occurred(tstate)) { - assert(mod == NULL || !_PyErr_Occurred(tstate)); + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + + if (PySys_Audit("import", "OOOOO", info.name, info.filename, + Py_None, Py_None, Py_None) < 0) + { goto finally; } + /* We would move this (and the fclose() below) into + * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * code relies on fp still being open. */ + FILE *fp; if (file != NULL) { - fp = _Py_fopen_obj(path, "r"); + fp = _Py_fopen_obj(info.filename, "r"); if (fp == NULL) { goto finally; } } - else + else { fp = NULL; + } + + PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + if (p0 == NULL) { + goto finally; + } - mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif + mod = import_run_extension( + tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif - if (fp) + // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? + if (fp) { fclose(fp); + } finally: - Py_DECREF(name); - Py_DECREF(path); + _Py_ext_module_loader_info_clear(&info); return mod; } @@ -3867,6 +4830,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/importdl.c b/Python/importdl.c index 7dfd301d77efb4..964648338394c2 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -28,6 +28,11 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif + +/***********************************/ +/* module info to use when loading */ +/***********************************/ + static const char * const ascii_only_prefix = "PyInit"; static const char * const nonascii_prefix = "PyInitU"; @@ -93,59 +98,295 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { return NULL; } -PyObject * -_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) +void +_Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) { + Py_CLEAR(info->filename); #ifndef MS_WINDOWS - PyObject *pathbytes = NULL; + Py_CLEAR(info->filename_encoded); #endif - PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; - const char *name_buf, *hook_prefix; - const char *oldcontext, *newcontext; - dl_funcptr exportfunc; - PyModuleDef *def; - PyModInitFunction p0; + Py_CLEAR(info->name); + Py_CLEAR(info->name_encoded); +} - name_unicode = PyObject_GetAttrString(spec, "name"); - if (name_unicode == NULL) { - return NULL; - } - if (!PyUnicode_Check(name_unicode)) { +int +_Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, + PyObject *name, PyObject *filename, + _Py_ext_module_origin origin) +{ + struct _Py_ext_module_loader_info info = { + .origin=origin, + }; + + assert(name != NULL); + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, - "spec.name must be a string"); - goto error; + "module name must be a string"); + _Py_ext_module_loader_info_clear(&info); + return -1; } - newcontext = PyUnicode_AsUTF8(name_unicode); - if (newcontext == NULL) { - goto error; + assert(PyUnicode_GetLength(name) > 0); + info.name = Py_NewRef(name); + + info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + if (info.name_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; } - name = get_encoded_name(name_unicode, &hook_prefix); + info.newcontext = PyUnicode_AsUTF8(info.name); + if (info.newcontext == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } + + if (filename != NULL) { + if (!PyUnicode_Check(filename)) { + PyErr_SetString(PyExc_TypeError, + "module filename must be a string"); + _Py_ext_module_loader_info_clear(&info); + return -1; + } + info.filename = Py_NewRef(filename); + +#ifndef MS_WINDOWS + info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename); + if (info.filename_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } +#endif + + info.path = info.filename; + } + else { + info.path = info.name; + } + + *p_info = info; + return 0; +} + +int +_Py_ext_module_loader_info_init_for_builtin( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + assert(PyUnicode_Check(name)); + assert(PyUnicode_FindChar(name, '.', 0, PyUnicode_GetLength(name), -1) == -1); + assert(PyUnicode_GetLength(name) > 0); + + PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); + if (name_encoded == NULL) { + return -1; + } + + *info = (struct _Py_ext_module_loader_info){ + .name=Py_NewRef(name), + .name_encoded=name_encoded, + /* We won't need filename. */ + .path=name, + .origin=_Py_ext_module_origin_BUILTIN, + .hook_prefix=ascii_only_prefix, + .newcontext=NULL, + }; + return 0; +} + +int +_Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + if (_Py_ext_module_loader_info_init_for_builtin(info, name) < 0) { + return -1; + } + info->origin = _Py_ext_module_origin_CORE; + return 0; +} + +int +_Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *p_info, + PyObject *spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); if (name == NULL) { - goto error; + return -1; + } + PyObject *filename = PyObject_GetAttrString(spec, "origin"); + if (filename == NULL) { + Py_DECREF(name); + return -1; + } + /* We could also accommodate builtin modules here without much trouble. */ + _Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC; + int err = _Py_ext_module_loader_info_init(p_info, name, filename, origin); + Py_DECREF(name); + Py_DECREF(filename); + return err; +} + + +/********************************/ +/* module init function results */ +/********************************/ + +void +_Py_ext_module_loader_result_clear(struct _Py_ext_module_loader_result *res) +{ + /* Instead, the caller should have called + * _Py_ext_module_loader_result_apply_error(). */ + assert(res->err == NULL); + *res = (struct _Py_ext_module_loader_result){0}; +} + +static void +_Py_ext_module_loader_result_set_error( + struct _Py_ext_module_loader_result *res, + enum _Py_ext_module_loader_result_error_kind kind) +{ +#ifndef NDEBUG + switch (kind) { + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(PyErr_Occurred()); + break; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(!PyErr_Occurred()); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); } - name_buf = PyBytes_AS_STRING(name); +#endif - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) - goto error; + assert(res->err == NULL && res->_err.exc == NULL); + res->err = &res->_err; + *res->err = (struct _Py_ext_module_loader_result_error){ + .kind=kind, + .exc=PyErr_GetRaisedException(), + }; - if (PySys_Audit("import", "OOOOO", name_unicode, path, - Py_None, Py_None, Py_None) < 0) { - goto error; + /* For some kinds, we also set/check res->kind. */ + switch (kind) { + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + assert(res->kind == _Py_ext_module_kind_UNKNOWN); + res->kind = _Py_ext_module_kind_INVALID; + break; + /* None of the rest affect the result kind. */ + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); } +} +void +_Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name) +{ + assert(!PyErr_Occurred()); + assert(res->err != NULL && res->err == &res->_err); + struct _Py_ext_module_loader_result_error err = *res->err; + res->err = NULL; + + /* We're otherwise done with the result at this point. */ + _Py_ext_module_loader_result_clear(res); + +#ifndef NDEBUG + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(err.exc != NULL); + break; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(err.exc == NULL); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + const char *msg = NULL; + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: + break; + case _Py_ext_module_loader_result_ERR_MISSING: + msg = "initialization of %s failed without raising an exception"; + break; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + msg = "initialization of %s raised unreported exception"; + break; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + msg = "init function of %s returned uninitialized object"; + break; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: + msg = "initialization of %s did not return PyModuleDef"; + break; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: + msg = "initialization of %s did not return an extension module"; + break; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + msg = "initialization of %s did not return a valid extension module"; + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + PyErr_Format(PyExc_SystemError, + "loading %s failed due to init function", name); + return; + } + + if (err.exc != NULL) { + PyErr_SetRaisedException(err.exc); + err.exc = NULL; /* PyErr_SetRaisedException() stole our reference. */ + if (msg != NULL) { + _PyErr_FormatFromCause(PyExc_SystemError, msg, name); + } + } + else { + assert(msg != NULL); + PyErr_Format(PyExc_SystemError, msg, name); + } +} + + +/********************************************/ +/* getting/running the module init function */ +/********************************************/ + +PyModInitFunction +_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, + FILE *fp) +{ + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + dl_funcptr exportfunc; #ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf, - path, fp); + exportfunc = _PyImport_FindSharedFuncptrWindows( + info->hook_prefix, name_buf, info->filename, fp); #else - pathbytes = PyUnicode_EncodeFSDefault(path); - if (pathbytes == NULL) - goto error; - exportfunc = _PyImport_FindSharedFuncptr(hook_prefix, name_buf, - PyBytes_AS_STRING(pathbytes), - fp); - Py_DECREF(pathbytes); + { + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + exportfunc = _PyImport_FindSharedFuncptr( + info->hook_prefix, name_buf, path_buf, fp); + } #endif if (exportfunc == NULL) { @@ -154,101 +395,123 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) msg = PyUnicode_FromFormat( "dynamic module does not define " "module export function (%s_%s)", - hook_prefix, name_buf); - if (msg == NULL) - goto error; - PyErr_SetImportError(msg, name_unicode, path); - Py_DECREF(msg); + info->hook_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); + } } - goto error; + return NULL; } - p0 = (PyModInitFunction)exportfunc; + return (PyModInitFunction)exportfunc; +} + +int +_PyImport_RunModInitFunc(PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res) +{ + struct _Py_ext_module_loader_result res = { + .kind=_Py_ext_module_kind_UNKNOWN, + }; + + /* Call the module init function. */ /* Package context is needed for single-phase init */ - oldcontext = _PyImport_SwapPackageContext(newcontext); - m = p0(); + const char *oldcontext = _PyImport_SwapPackageContext(info->newcontext); + PyObject *m = p0(); _PyImport_SwapPackageContext(oldcontext); + /* Validate the result (and populate "res". */ + if (m == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "initialization of %s failed without raising an exception", - name_buf); + /* The init func for multi-phase init modules is expected + * to return a PyModuleDef after calling PyModuleDef_Init(). + * That function never raises an exception nor returns NULL, + * so at this point it must be a single-phase init modules. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + if (PyErr_Occurred()) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_EXCEPTION); + } + else { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING); } goto error; } else if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "initialization of %s raised unreported exception", - name_buf); + /* Likewise, we infer that this is a single-phase init module. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNREPORTED_EXC); + /* We would probably be correct to decref m here, + * but we weren't doing so before, + * so we stick with doing nothing. */ m = NULL; goto error; } + if (Py_IS_TYPE(m, NULL)) { /* This can happen when a PyModuleDef is returned without calling * PyModuleDef_Init on it */ - PyErr_Format(PyExc_SystemError, - "init function of %s returned uninitialized object", - name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNINITIALIZED); + /* Likewise, decref'ing here makes sense. However, the original + * code has a note about "prevent segfault in DECREF", + * so we play it safe and leave it alone. */ m = NULL; /* prevent segfault in DECREF */ goto error; } - if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(path); - return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); - } - - /* Fall back to single-phase init mechanism */ - if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { - goto error; + if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { + /* multi-phase init */ + res.kind = _Py_ext_module_kind_MULTIPHASE; + res.def = (PyModuleDef *)m; + /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - - if (hook_prefix == nonascii_prefix) { - /* don't allow legacy init for non-ASCII module names */ - PyErr_Format( - PyExc_SystemError, - "initialization of %s did not return PyModuleDef", - name_buf); + else if (info->hook_prefix == nonascii_prefix) { + /* Non-ASCII is only supported for multi-phase init. */ + res.kind = _Py_ext_module_kind_MULTIPHASE; + /* Don't allow legacy init for non-ASCII module names. */ + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE); goto error; } + else { + /* single-phase init (legacy) */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + res.module = m; - /* Remember pointer to module init function. */ - def = PyModule_GetDef(m); - if (def == NULL) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return an extension " - "module", name_buf); - goto error; - } - def->m_base.m_init = p0; + if (!PyModule_Check(m)) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NOT_MODULE); + goto error; + } - /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", path) < 0) { - PyErr_Clear(); /* Not important enough to report */ + res.def = _PyModule_GetDef(m); + if (res.def == NULL) { + PyErr_Clear(); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING_DEF); + goto error; + } } - PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) - goto error; - - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(path); - - return m; + assert(!PyErr_Occurred()); + assert(res.err == NULL); + *p_res = res; + return 0; error: - Py_DECREF(name_unicode); - Py_XDECREF(name); - Py_XDECREF(path); - Py_XDECREF(m); - return NULL; + assert(!PyErr_Occurred()); + assert(res.err != NULL); + Py_CLEAR(res.module); + res.def = NULL; + *p_res = res; + p_res->err = &p_res->_err; + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ diff --git a/Python/initconfig.c b/Python/initconfig.c index f7eb8535e98a6a..51897a2d0aef66 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,12 +24,16 @@ # endif #endif +#include "config_common.h" + + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { PyConfig_MEMBER_INT = 0, PyConfig_MEMBER_UINT = 1, PyConfig_MEMBER_ULONG = 2, + PyConfig_MEMBER_BOOL = 3, PyConfig_MEMBER_WSTR = 10, PyConfig_MEMBER_WSTR_OPT = 11, @@ -45,61 +49,65 @@ typedef struct { #define SPEC(MEMBER, TYPE) \ {#MEMBER, offsetof(PyConfig, MEMBER), PyConfig_MEMBER_##TYPE} +// Update _test_embed_set_config when adding new members static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(_config_init, UINT), - SPEC(isolated, UINT), - SPEC(use_environment, UINT), - SPEC(dev_mode, UINT), - SPEC(install_signal_handlers, UINT), - SPEC(use_hash_seed, UINT), + SPEC(isolated, BOOL), + SPEC(use_environment, BOOL), + SPEC(dev_mode, BOOL), + SPEC(install_signal_handlers, BOOL), + SPEC(use_hash_seed, BOOL), SPEC(hash_seed, ULONG), - SPEC(faulthandler, UINT), + SPEC(faulthandler, BOOL), SPEC(tracemalloc, UINT), SPEC(perf_profiling, UINT), - SPEC(import_time, UINT), - SPEC(code_debug_ranges, UINT), - SPEC(show_ref_count, UINT), - SPEC(dump_refs, UINT), + SPEC(import_time, BOOL), + SPEC(code_debug_ranges, BOOL), + SPEC(show_ref_count, BOOL), + SPEC(dump_refs, BOOL), SPEC(dump_refs_file, WSTR_OPT), - SPEC(malloc_stats, UINT), + SPEC(malloc_stats, BOOL), SPEC(filesystem_encoding, WSTR), SPEC(filesystem_errors, WSTR), SPEC(pycache_prefix, WSTR_OPT), - SPEC(parse_argv, UINT), + SPEC(parse_argv, BOOL), SPEC(orig_argv, WSTR_LIST), SPEC(argv, WSTR_LIST), SPEC(xoptions, WSTR_LIST), SPEC(warnoptions, WSTR_LIST), - SPEC(site_import, UINT), + SPEC(site_import, BOOL), SPEC(bytes_warning, UINT), - SPEC(warn_default_encoding, UINT), - SPEC(inspect, UINT), - SPEC(interactive, UINT), + SPEC(warn_default_encoding, BOOL), + SPEC(inspect, BOOL), + SPEC(interactive, BOOL), SPEC(optimization_level, UINT), - SPEC(parser_debug, UINT), - SPEC(write_bytecode, UINT), + SPEC(parser_debug, BOOL), + SPEC(write_bytecode, BOOL), SPEC(verbose, UINT), - SPEC(quiet, UINT), - SPEC(user_site_directory, UINT), - SPEC(configure_c_stdio, UINT), - SPEC(buffered_stdio, UINT), + SPEC(quiet, BOOL), + SPEC(user_site_directory, BOOL), + SPEC(configure_c_stdio, BOOL), + SPEC(buffered_stdio, BOOL), SPEC(stdio_encoding, WSTR), SPEC(stdio_errors, WSTR), #ifdef MS_WINDOWS - SPEC(legacy_windows_stdio, UINT), + SPEC(legacy_windows_stdio, BOOL), #endif SPEC(check_hash_pycs_mode, WSTR), - SPEC(use_frozen_modules, UINT), - SPEC(safe_path, UINT), + SPEC(use_frozen_modules, BOOL), + SPEC(safe_path, BOOL), SPEC(int_max_str_digits, INT), SPEC(cpu_count, INT), - SPEC(pathconfig_warnings, UINT), +#ifdef Py_GIL_DISABLED + SPEC(enable_gil, INT), +#endif + SPEC(pathconfig_warnings, BOOL), SPEC(program_name, WSTR), SPEC(pythonpath_env, WSTR_OPT), SPEC(home, WSTR_OPT), SPEC(platlibdir, WSTR), SPEC(sys_path_0, WSTR_OPT), - SPEC(module_search_paths_set, UINT), + SPEC(module_search_paths_set, BOOL), SPEC(module_search_paths, WSTR_LIST), SPEC(stdlib_dir, WSTR_OPT), SPEC(executable, WSTR_OPT), @@ -108,15 +116,18 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(base_prefix, WSTR_OPT), SPEC(exec_prefix, WSTR_OPT), SPEC(base_exec_prefix, WSTR_OPT), - SPEC(skip_source_first_line, UINT), + SPEC(skip_source_first_line, BOOL), SPEC(run_command, WSTR_OPT), SPEC(run_module, WSTR_OPT), SPEC(run_filename, WSTR_OPT), - SPEC(_install_importlib, UINT), - SPEC(_init_main, UINT), - SPEC(_is_python_build, UINT), + SPEC(_install_importlib, BOOL), + SPEC(_init_main, BOOL), + SPEC(_is_python_build, BOOL), #ifdef Py_STATS - SPEC(_pystats, UINT), + SPEC(_pystats, BOOL), +#endif +#ifdef Py_DEBUG + SPEC(run_presite, WSTR_OPT), #endif {NULL, 0, 0}, }; @@ -134,8 +145,8 @@ static const char usage_line[] = /* Lines sorted by option name; keep in sync with usage_envvars* below */ static const char usage_help[] = "\ Options (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-b : issue warnings about converting bytes/bytearray to str and comparing\n\ + bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ -B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : turn on parser debugging output (for experts only, only works on\n\ @@ -150,9 +161,10 @@ Options (and corresponding environment variables):\n\ .pyc extension; also PYTHONOPTIMIZE=x\n\ -OO : do -O changes and also discard docstrings; add .opt-2 before\n\ .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path; also PYTHONSAFEPATH\n\ +-P : don't prepend a potentially unsafe path to sys.path; also\n\ + PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x\n\ -S : don't imply 'import site' on initialization\n\ -u : force the stdout and stderr streams to be unbuffered;\n\ this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ @@ -166,9 +178,10 @@ Options (and corresponding environment variables):\n\ -X opt : set implementation-specific option\n\ --check-hash-based-pycs always|default|never:\n\ control how Python invalidates hash-based .pyc files\n\ ---help-env : print help about Python environment variables and exit\n\ ---help-xoptions : print help about implementation-specific -X options and exit\n\ ---help-all : print complete help information and exit\n\ +--help-env: print help about Python environment variables and exit\n\ +--help-xoptions: print help about implementation-specific -X options and exit\n\ +--help-all: print complete help information and exit\n\ +\n\ Arguments:\n\ file : program read from script file\n\ - : program read from stdin (default; interactive mode if a tty)\n\ @@ -177,126 +190,131 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ -\n\ --X faulthandler: enable faulthandler\n\ -\n\ --X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in the\n\ - interactive interpreter. This only works on debug builds\n\ -\n\ --X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored in a\n\ - traceback of a trace. Use -X tracemalloc=NFRAME to start tracing with a\n\ - traceback limit of NFRAME frames\n\ -\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in multi-threaded\n\ - application. Typical usage is python3 -X importtime -c 'import asyncio'\n\ -\n\ --X dev: enable CPython's \"development mode\", introducing additional runtime\n\ - checks which are too expensive to be enabled by default. Effect of the\n\ - developer mode:\n\ - * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks()\n\ - C function\n\ - * Enable the faulthandler module to dump the Python traceback on a crash\n\ - * Enable asyncio debug mode\n\ - * Set the dev_mode attribute of sys.flags to True\n\ - * io.IOBase destructor logs close() exceptions\n\ -\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the default\n\ - locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode (even when it would\n\ - otherwise activate automatically)\n\ -\n\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted at the\n\ - given directory instead of to the code tree\n\ -\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\n\ -\n\ --X no_debug_ranges: disable the inclusion of the tables mapping extra location \n\ - information (end line, start column offset and end column offset) to every \n\ - instruction in code objects. This is useful when smaller code objects and pyc \n\ - files are desired as well as suppressing the extra visual location indicators \n\ - when the interpreter displays tracebacks.\n\ -\n\ --X perf: activate support for the Linux \"perf\" profiler by activating the \"perf\"\n\ - trampoline. When this option is activated, the Linux \"perf\" profiler will be \n\ - able to report Python calls. This option is only available on some platforms and will \n\ - do nothing if is not supported on the current system. The default value is \"off\".\n\ -\n\ --X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ -\n\ --X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables.\n\ -\n\ --X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can help users who need\n\ - to limit resources in a container." - +-X cpu_count=N: override the return value of os.cpu_count();\n\ + -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ +-X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ +-X faulthandler: dump the Python traceback on fatal errors;\n\ + also PYTHONFAULTHANDLER\n\ +-X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ + for installed Python and \"off\" for a local build;\n\ + also PYTHON_FROZEN_MODULES\n\ +" +#ifdef Py_GIL_DISABLED +"-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" +#endif +"\ +-X importtime: show how long each import takes; also PYTHONPROFILEIMPORTTIME\n\ +-X int_max_str_digits=N: limit the size of int<->str conversions;\n\ + 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ +-X no_debug_ranges: don't include extra location information in code objects;\n\ + also PYTHONNODEBUGRANGES\n\ +-X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ +" +#ifdef Py_DEBUG +"-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" +#endif +"\ +-X pycache_prefix=PATH: write .pyc files to a parallel tree instead of to the\n\ + code tree; also PYTHONPYCACHEPREFIX\n\ +" #ifdef Py_STATS -"\n\ -\n\ --X pystats: Enable pystats collection at startup." +"-X pystats: enable pystats collection at startup; also PYTHONSTATS\n" #endif -; +"\ +-X showrefcount: output the total reference count and number of used\n\ + memory blocks when the program finishes or after each statement in\n\ + the interactive interpreter; only works on debug builds\n\ +-X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ + of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ +-X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ +-X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ + also PYTHONWARNDEFAULTENCODING\ +"; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = "Environment variables that change behavior:\n" -"PYTHONSTARTUP: file executed on interactive startup (no default)\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" -" default module search path. The result is sys.path.\n" -"PYTHONHOME : alternate directory (or %lc).\n" -" The default module search path uses %s.\n" -"PYTHONPLATLIBDIR : override sys.platlibdir.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONUTF8: if set to 1, enable the UTF-8 mode.\n" -"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" -"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str and bytes objects. It can also be set to an\n" -" integer in the range [0,4294967295] to get hash values with a\n" -" predictable seed.\n" -"PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int back to a str.\n" -" A value of 0 disables the limit. Conversions to or from bases 2, 4, 8,\n" -" 16, and 32 are never limited.\n" -"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n" +"PYTHONSTARTUP : file executed on interactive startup (no default)\n" +"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +" default module search path. The result is sys.path.\n" +"PYTHONHOME : alternate directory (or %lc).\n" +" The default module search path uses %s.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" +"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" +"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str and bytes objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash\n" +" values with a predictable seed.\n" +"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +" install debug hooks.\n" +"PYTHONMALLOCSTATS: print memory allocator statistics\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n" +" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" +" display of locale coercion and locale compatibility warnings\n" +" on stderr.\n" "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of choice.\n" -"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to a positive integer.\n" -"PYTHONDEVMODE: enable the development mode.\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" -"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" -"PYTHONNODEBUGRANGES: If this variable is set, it disables the inclusion of the \n" -" tables mapping extra location information (end line, start column offset \n" -" and end column offset) to every instruction in code objects. This is useful \n" -" when smaller code objects and pyc files are desired as well as suppressing the \n" -" extra visual location indicators when the interpreter displays tracebacks.\n" -"These variables have equivalent command-line parameters (see --help for details):\n" -"PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" -"PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" -" (-X int_max_str_digits=number)\n" -"PYTHONNOUSERSITE : disable user site directory (-s)\n" -"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" -"PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg : warning control (-W arg)\n" +" debugger. It can be set to the callable of your debugger of\n" +" choice.\n" +"PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" +" various kinds of output. Setting it to 0 deactivates\n" +" this behavior.\n" +"PYTHON_HISTORY : the location of a .python_history file.\n" +"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" +#ifdef Py_TRACE_REFS +"PYTHONDUMPREFS : dump objects and reference counts still alive after shutdown\n" +"PYTHONDUMPREFSFILE: dump objects and reference counts to the specified file\n" +#endif +#ifdef __APPLE__ +"PYTHONEXECUTABLE: set sys.argv[0] to this value (macOS only)\n" +#endif +#ifdef MS_WINDOWS +"PYTHONLEGACYWINDOWSFSENCODING: use legacy \"mbcs\" encoding for file system\n" +"PYTHONLEGACYWINDOWSSTDIO: use legacy Windows stdio\n" +#endif +"PYTHONUSERBASE : defines the user base directory (site.USER_BASE)\n" +"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" +"\n" +"These variables have equivalent command-line options (see --help for details):\n" +"PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" +"PYTHONDEBUG : enable parser debug mode (-d)\n" +"PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" +"PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" +"PYTHON_FROZEN_MODULES: whether to use frozen modules; the default is \"on\"\n" +" for installed Python and \"off\" for a local build\n" +" (-X frozen_modules)\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" +#endif +"PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS: limit the size of int<->str conversions;\n" +" 0 disables the limit (-X int_max_str_digits=N)\n" +"PYTHONNODEBUGRANGES: don't include extra location information in code objects\n" +" (-X no_debug_ranges)\n" +"PYTHONNOUSERSITE: disable user site directory (-s)\n" +"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" +"PYTHON_PERF_JIT_SUPPORT: enable Linux \"perf\" profiler support with JIT\n" +" (-X perf_jit)\n" +#ifdef Py_DEBUG +"PYTHON_PRESITE: import this module before site (-X presite)\n" +#endif +"PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" +"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" +" (-X pycache_prefix)\n" +"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering\n" +"PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif +"PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" +"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" +"PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" +" (-X warn_default_encoding)\n" +"PYTHONWARNINGS : warning control (-W)\n" ; #if defined(MS_WINDOWS) @@ -790,6 +808,9 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->run_module); CLEAR(config->run_filename); CLEAR(config->check_hash_pycs_mode); +#ifdef Py_DEBUG + CLEAR(config->run_presite); +#endif _PyWideStringList_Clear(&config->orig_argv); #undef CLEAR @@ -842,6 +863,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DEFAULT; +#endif } @@ -989,6 +1013,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) switch (spec->type) { case PyConfig_MEMBER_INT: case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: { *(int*)member = *(int*)member2; break; @@ -1044,6 +1069,12 @@ _PyConfig_AsDict(const PyConfig *config) obj = PyLong_FromLong(value); break; } + case PyConfig_MEMBER_BOOL: + { + int value = *(int*)member; + obj = PyBool_FromLong(value); + break; + } case PyConfig_MEMBER_ULONG: { unsigned long value = *(unsigned long*)member; @@ -1087,21 +1118,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; - } - if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1109,13 +1125,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { @@ -1267,19 +1276,20 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) char *member = (char *)config + spec->offset; switch (spec->type) { case PyConfig_MEMBER_INT: - if (config_dict_get_int(dict, spec->name, (int*)member) < 0) { - return -1; - } - break; case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: { int value; if (config_dict_get_int(dict, spec->name, &value) < 0) { return -1; } - if (value < 0) { - config_dict_invalid_value(spec->name); - return -1; + if (spec->type == PyConfig_MEMBER_BOOL + || spec->type == PyConfig_MEMBER_UINT) + { + if (value < 0) { + config_dict_invalid_value(spec->name); + return -1; + } } *(int*)member = value; break; @@ -1546,6 +1556,24 @@ config_wstr_to_int(const wchar_t *wstr, int *result) return 0; } +static PyStatus +config_read_gil(PyConfig *config, size_t len, wchar_t first_char) +{ +#ifdef Py_GIL_DISABLED + if (len == 1 && first_char == L'0') { + config->enable_gil = _PyConfig_GIL_DISABLE; + } + else if (len == 1 && first_char == L'1') { + config->enable_gil = _PyConfig_GIL_ENABLE; + } + else { + return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); + } + return _PyStatus_OK(); +#else + return _PyStatus_ERR("PYTHON_GIL / -X gil are not supported by this build"); +#endif +} static PyStatus config_read_env_vars(PyConfig *config) @@ -1624,6 +1652,15 @@ config_read_env_vars(PyConfig *config) config->safe_path = 1; } + const char *gil = config_get_env(config, "PYTHON_GIL"); + if (gil != NULL) { + size_t len = strlen(gil); + status = config_read_gil(config, len, gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + return _PyStatus_OK(); } @@ -1683,6 +1720,20 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } + env = config_get_env(config, "PYTHON_PERF_JIT_SUPPORT"); + if (env) { + if (_Py_str_to_int(env, &active) != 0) { + active = 0; + } + if (active) { + config->perf_profiling = 2; + } + } + xoption = config_get_xoption(config, L"perf_jit"); + if (xoption) { + config->perf_profiling = 2; + } + return _PyStatus_OK(); } @@ -1806,6 +1857,36 @@ config_init_pycache_prefix(PyConfig *config) } +#ifdef Py_DEBUG +static PyStatus +config_init_run_presite(PyConfig *config) +{ + assert(config->run_presite == NULL); + + const wchar_t *xoption = config_get_xoption(config, L"presite"); + if (xoption) { + const wchar_t *sep = wcschr(xoption, L'='); + if (sep && wcslen(sep) > 1) { + config->run_presite = _PyMem_RawWcsdup(sep + 1); + if (config->run_presite == NULL) { + return _PyStatus_NO_MEMORY(); + } + } + else { + // PYTHON_PRESITE env var ignored + // if "-X presite=" option is used + config->run_presite = NULL; + } + return _PyStatus_OK(); + } + + return CONFIG_GET_ENV_DUP(config, &config->run_presite, + L"PYTHON_PRESITE", + "PYTHON_PRESITE"); +} +#endif + + static PyStatus config_read_complex_options(PyConfig *config) { @@ -1861,6 +1942,16 @@ config_read_complex_options(PyConfig *config) return status; } } + +#ifdef Py_DEBUG + if (config->run_presite == NULL) { + status = config_init_run_presite(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif + return _PyStatus_OK(); } @@ -2078,6 +2169,19 @@ config_init_import(PyConfig *config, int compute_path_config) return status; } + const char *env = config_get_env(config, "PYTHON_FROZEN_MODULES"); + if (env == NULL) { + } + else if (strcmp(env, "on") == 0) { + config->use_frozen_modules = 1; + } + else if (strcmp(env, "off") == 0) { + config->use_frozen_modules = 0; + } else { + return PyStatus_Error("bad value for PYTHON_FROZEN_MODULES " + "(expected \"on\" or \"off\")"); + } + /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { @@ -2126,6 +2230,15 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } + const wchar_t *x_gil = config_get_xoption_value(config, L"gil"); + if (x_gil != NULL) { + size_t len = wcslen(x_gil); + status = config_read_gil(config, len, x_gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { config->_pystats = 1; @@ -2314,9 +2427,9 @@ static void config_complete_usage(const wchar_t* program) { config_usage(0, program); - puts("\n"); + putchar('\n'); config_envvars_usage(); - puts("\n"); + putchar('\n'); config_xoptions_usage(); } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c new file mode 100644 index 00000000000000..a3f85f754d71bb --- /dev/null +++ b/Python/instruction_sequence.c @@ -0,0 +1,453 @@ +/* + * This file implements a data structure representing a sequence of + * instructions, which is used by different parts of the compilation + * pipeline. + */ + + +#include + +#include "Python.h" + +#include "pycore_compile.h" // _PyCompile_EnsureArrayLargeEnough +#include "pycore_opcode_utils.h" +#include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; +typedef _Py_SourceLocation location; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +#include "clinic/instruction_sequence.c.h" + +#undef SUCCESS +#undef ERROR +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + if ((X) == -1) { \ + return ERROR; \ + } + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_allocated >= 0); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +_PyJumpTargetLabel +_PyInstructionSequence_NewLabel(instr_sequence *seq) +{ + _PyJumpTargetLabel lbl = {++seq->s_next_free_label}; + return lbl; +} + +int +_PyInstructionSequence_UseLabel(instr_sequence *seq, int lbl) +{ + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +int +_PyInstructionSequence_ApplyLabelMap(instr_sequence *instrs) +{ + if (instrs->s_labelmap == NULL) { + /* Already applied - nothing to do */ + return SUCCESS; + } + /* Replace labels by offsets in the code */ + for (int i=0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg < instrs->s_labelmap_size); + instr->i_oparg = instrs->s_labelmap[instr->i_oparg]; + } + _PyExceptHandlerInfo *hi = &instr->i_except_handler_info; + if (hi->h_label >= 0) { + assert(hi->h_label < instrs->s_labelmap_size); + hi->h_label = instrs->s_labelmap[hi->h_label]; + } + } + /* Clear label map so it's never used again */ + PyMem_Free(instrs->s_labelmap); + instrs->s_labelmap = NULL; + instrs->s_labelmap_size = 0; + return SUCCESS; +} + +#define MAX_OPCODE 511 + +int +_PyInstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, + location loc) +{ + assert(0 <= opcode && opcode <= MAX_OPCODE); + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +int +_PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +int +_PyInstructionSequence_AddNested(instr_sequence *seq, instr_sequence *nested) +{ + if (seq->s_nested == NULL) { + seq->s_nested = PyList_New(0); + if (seq->s_nested == NULL) { + return ERROR; + } + } + if (PyList_Append(seq->s_nested, (PyObject*)nested) < 0) { + return ERROR; + } + return SUCCESS; +} + +void +PyInstructionSequence_Fini(instr_sequence *seq) { + Py_XDECREF(seq->s_nested); + + PyMem_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyMem_Free(seq->s_instrs); + seq->s_instrs = NULL; +} + +/*[clinic input] +class InstructionSequenceType "_PyInstructionSequence *" "&_PyInstructionSequence_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=589963e07480390f]*/ + +static _PyInstructionSequence* +inst_seq_create(void) +{ + _PyInstructionSequence *seq; + seq = PyObject_GC_New(_PyInstructionSequence, &_PyInstructionSequence_Type); + if (seq == NULL) { + return NULL; + } + seq->s_instrs = NULL; + seq->s_allocated = 0; + seq->s_used = 0; + seq->s_next_free_label = 0; + seq->s_labelmap = NULL; + seq->s_labelmap_size = 0; + seq->s_nested = NULL; + + PyObject_GC_Track(seq); + return seq; +} + +PyObject* +_PyInstructionSequence_New(void) +{ + _PyInstructionSequence *seq = inst_seq_create(); + if (seq == NULL) { + return NULL; + } + return (PyObject*)seq; +} + +/*[clinic input] +@classmethod +InstructionSequenceType.__new__ as inst_seq_new + +Create a new InstructionSequence object. +[clinic start generated code]*/ + +static PyObject * +inst_seq_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=98881de92c8876f6 input=b393150146849c74]*/ +{ + return (PyObject*)inst_seq_create(); +} + +/*[clinic input] +InstructionSequenceType.use_label + + label: int + +Place label at current location. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label) +/*[clinic end generated code: output=4c06bbacb2854755 input=da55f49bb91841f3]*/ + +{ + if (_PyInstructionSequence_UseLabel(self, label) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.addop + + opcode: int + oparg: int + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + +Append an instruction. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset) +/*[clinic end generated code: output=af0cc22c048dfbf3 input=012762ac88198713]*/ +{ + _Py_SourceLocation loc = {lineno, col_offset, end_lineno, end_col_offset}; + if (_PyInstructionSequence_Addop(self, opcode, oparg, loc) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.new_label -> int + +Return a new label. +[clinic start generated code]*/ + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=dcb0589e4f5bf4bd input=c66040b9897bc327]*/ +{ + _PyJumpTargetLabel lbl = _PyInstructionSequence_NewLabel(self); + return lbl.id; +} + +/*[clinic input] +InstructionSequenceType.add_nested + + nested: object + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested) +/*[clinic end generated code: output=14540fad459f7971 input=f2c482568b3b3c0f]*/ +{ + if (!_PyInstructionSequence_Check(nested)) { + PyErr_Format(PyExc_TypeError, + "expected an instruction sequence, not %T", + Py_TYPE(nested)); + return NULL; + } + if (_PyInstructionSequence_AddNested(self, (_PyInstructionSequence*)nested) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.get_nested + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=f415112c292630cb input=e429e474c57b95b4]*/ +{ + if (self->s_nested == NULL) { + return PyList_New(0); + } + return Py_NewRef(self->s_nested); +} + +/*[clinic input] +InstructionSequenceType.get_instructions + +Return the instructions as a list of tuples or labels. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=23f4f3f894c301b3 input=fbadb5dadb611291]*/ +{ + if (_PyInstructionSequence_ApplyLabelMap(self) < 0) { + return NULL; + } + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < self->s_used; i++) { + instruction *instr = &self->s_instrs[i]; + location loc = instr->i_loc; + PyObject *inst_tuple; + + if (OPCODE_HAS_ARG(instr->i_opcode)) { + inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, instr->i_oparg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + else { + inst_tuple = Py_BuildValue( + "(iOiiii)", instr->i_opcode, Py_None, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + +static PyMethodDef inst_seq_methods[] = { + INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF + INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef inst_seq_memberlist[] = { + {NULL} /* Sentinel */ +}; + +static PyGetSetDef inst_seq_getsetters[] = { + {NULL} /* Sentinel */ +}; + +static void +inst_seq_dealloc(_PyInstructionSequence *seq) +{ + PyObject_GC_UnTrack(seq); + Py_TRASHCAN_BEGIN(seq, inst_seq_dealloc) + PyInstructionSequence_Fini(seq); + PyObject_GC_Del(seq); + Py_TRASHCAN_END +} + +static int +inst_seq_traverse(_PyInstructionSequence *seq, visitproc visit, void *arg) +{ + Py_VISIT(seq->s_nested); + return 0; +} + +static int +inst_seq_clear(_PyInstructionSequence *seq) +{ + Py_CLEAR(seq->s_nested); + return 0; +} + +PyTypeObject _PyInstructionSequence_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "InstructionSequence", + sizeof(_PyInstructionSequence), + 0, + (destructor)inst_seq_dealloc, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + inst_seq_new__doc__, /* tp_doc */ + (traverseproc)inst_seq_traverse, /* tp_traverse */ + (inquiry)inst_seq_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + inst_seq_methods, /* tp_methods */ + inst_seq_memberlist, /* tp_members */ + inst_seq_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + inst_seq_new, /* tp_new */ +}; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index eee1908e503e43..ae790a1441b933 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -6,6 +6,7 @@ #include "pycore_call.h" #include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS #include "pycore_code.h" // _PyCode_Clear_Executors() +#include "pycore_critical_section.h" #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_long.h" @@ -13,23 +14,46 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 -PyObject _PyInstrumentation_DISABLE = -{ - .ob_refcnt = _Py_IMMORTAL_REFCNT, - .ob_type = &PyBaseObject_Type -}; +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) -PyObject _PyInstrumentation_MISSING = -{ - .ob_refcnt = _Py_IMMORTAL_REFCNT, - .ob_type = &PyBaseObject_Type -}; +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \ + } +#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + +#else + +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) +#define ASSERT_WORLD_STOPPED() + +#endif + +#ifdef Py_GIL_DISABLED + +#define LOCK_CODE(code) \ + assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \ + Py_BEGIN_CRITICAL_SECTION(code) + +#define UNLOCK_CODE() Py_END_CRITICAL_SECTION() + +#else + +#define LOCK_CODE(code) +#define UNLOCK_CODE() + +#endif + +PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type); + +PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type); static const int8_t EVENT_FOR_OPCODE[256] = { [RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN, @@ -244,14 +268,15 @@ get_events(_Py_GlobalMonitors *m, int tool_id) * 8 bit value. * if line_delta == -128: * line = None # represented as -1 - * elif line_delta == -127: + * elif line_delta == -127 or line_delta == -126: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); * else: * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; */ #define NO_LINE -128 -#define COMPUTED_LINE -127 +#define COMPUTED_LINE_LINENO_CHANGE -127 +#define COMPUTED_LINE -126 #define OFFSET_SHIFT 4 @@ -278,7 +303,7 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) return -1; } - assert(line_delta == COMPUTED_LINE); + assert(line_delta == COMPUTED_LINE || line_delta == COMPUTED_LINE_LINENO_CHANGE); /* Look it up */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } @@ -286,6 +311,8 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) int _PyInstruction_GetLength(PyCodeObject *code, int offset) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); assert(opcode != RESERVED); @@ -457,6 +484,8 @@ valid_opcode(int opcode) static void sanity_check_instrumentation(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + _PyCoMonitoringData *data = code->_co_monitoring; if (data == NULL) { return; @@ -486,10 +515,6 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - if (data->per_instruction_tools) { - uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; - CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); - } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); @@ -596,9 +621,10 @@ de_instrument(PyCodeObject *code, int i, int event) return; } CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - *opcode_ptr = deinstrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); } } @@ -619,7 +645,7 @@ de_instrument_line(PyCodeObject *code, int i) CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); instr->op.code = original_opcode; if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } assert(instr->op.code != INSTRUMENTED_LINE); } @@ -642,12 +668,10 @@ de_instrument_per_instruction(PyCodeObject *code, int i) CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); *opcode_ptr = original_opcode; if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[i] = 0; } @@ -673,9 +697,11 @@ instrument(PyCodeObject *code, int i) int deopt = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - *opcode_ptr = instrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); + instr[1].counter = adaptive_counter_warmup(); } } } @@ -726,6 +752,7 @@ instrument_per_instruction(PyCodeObject *code, int i) static void remove_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); @@ -760,6 +787,8 @@ tools_is_subset_for_event(PyCodeObject * code, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(code->_co_monitoring); if (code->_co_monitoring->line_tools) { @@ -782,6 +811,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) static void add_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); @@ -802,6 +832,8 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->line_tools) { @@ -818,6 +850,8 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->per_instruction_tools) { @@ -834,6 +868,8 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(code->_co_monitoring); if (code->_co_monitoring->per_instruction_tools) { uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; @@ -857,7 +893,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, - Py_ssize_t nargsf, int8_t tool, int event) + size_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); @@ -899,23 +935,57 @@ static inline int most_significant_bit(uint8_t bits) { static uint32_t global_version(PyInterpreterState *interp) { - return interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK; + uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed( + &interp->ceval.instrumentation_version); +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uint32_t thread_version = + (uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK); + assert(thread_version == version); +#endif + return version; } +/* Atomically set the given version in the given location, without touching + anything in _PY_EVAL_EVENTS_MASK. */ static void -set_global_version(PyInterpreterState *interp, uint32_t version) +set_version_raw(uintptr_t *ptr, uint32_t version) { - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - intptr_t new; + uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr); + uintptr_t new; do { new = (old & _PY_EVAL_EVENTS_MASK) | version; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + } while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new)); +} + +static void +set_global_version(PyThreadState *tstate, uint32_t version) +{ + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + PyInterpreterState *interp = tstate->interp; + set_version_raw(&interp->ceval.instrumentation_version, version); + +#ifdef Py_GIL_DISABLED + // Set the version on all threads in free-threaded builds. + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + for (tstate = interp->threads.head; tstate; + tstate = PyThreadState_Next(tstate)) { + set_version_raw(&tstate->eval_breaker, version); + }; + HEAD_UNLOCK(runtime); +#else + // Normal builds take the current version from instrumentation_version when + // attaching a thread, so we only have to set the current thread's version. + set_version_raw(&tstate->eval_breaker, version); +#endif } static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); return global_version(interp) == code->_co_instrumentation_version; } @@ -923,11 +993,24 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _Py_LocalMonitors expected = local_union( interp->monitors, code->_co_monitoring->local_monitors); return monitors_equals(code->_co_monitoring->active_monitors, expected); } + +static int +debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code) +{ + int res; + LOCK_CODE(code); + res = is_version_up_to_date(code, interp) && + instrumentation_cross_checks(interp, code); + UNLOCK_CODE(); + return res; +} + #endif static inline uint8_t @@ -942,8 +1025,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event = PY_MONITORING_EVENT_CALL; } if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(is_version_up_to_date(code, interp)); - CHECK(instrumentation_cross_checks(interp, code)); + CHECK(debug_check_sanity(interp, code)); if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1002,7 +1084,7 @@ call_instrumentation_vector( args[2] = offset_obj; PyInterpreterState *interp = tstate->interp; uint8_t tools = get_tools_for_instruction(code, interp, offset, event); - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; while (tools) { @@ -1031,7 +1113,9 @@ call_instrumentation_vector( break; } else { + LOCK_CODE(code); remove_tools(code, offset, event, 1 << tool); + UNLOCK_CODE(); } } } @@ -1073,7 +1157,7 @@ _Py_call_instrumentation_jump( { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); - assert(frame->prev_instr == instr); + assert(frame->instr_ptr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1086,9 +1170,9 @@ _Py_call_instrumentation_jump( if (err) { return NULL; } - if (frame->prev_instr != instr) { + if (frame->instr_ptr != instr) { /* The callback has caused a jump (by setting the line number) */ - return frame->prev_instr; + return frame->instr_ptr; } return target; } @@ -1138,32 +1222,38 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { - assert(frame->prev_instr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(tstate->tracing == 0); + assert(debug_check_sanity(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; - int line = compute_line(code, i, line_delta); - assert(line >= 0); - int prev_index = (int)(prev - _PyCode_CODE(code)); - int prev_line = _Py_Instrumentation_GetLine(code, prev_index); - if (prev_line == line) { - int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; - /* RESUME and INSTRUMENTED_RESUME are needed for the operation of - * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. - */ - if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { - goto done; + int line = 0; + + if (line_delta == COMPUTED_LINE_LINENO_CHANGE) { + // We know the line number must have changed, don't need to calculate + // the line number for now because we might not need it. + line = -1; + } else { + line = compute_line(code, i, line_delta); + assert(line >= 0); + assert(prev != NULL); + int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_line = _Py_Instrumentation_GetLine(code, prev_index); + if (prev_line == line) { + int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + /* RESUME and INSTRUMENTED_RESUME are needed for the operation of + * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. + */ + if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { + goto done; + } } } + uint8_t tools = code->_co_monitoring->line_tools != NULL ? code->_co_monitoring->line_tools[i] : (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | @@ -1172,7 +1262,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, /* Special case sys.settrace to avoid boxing the line number, * only to immediately unbox it. */ if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) { - if (tstate->c_tracefunc != NULL && line >= 0) { + if (tstate->c_tracefunc != NULL) { PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame); if (frame_obj == NULL) { return -1; @@ -1185,6 +1275,12 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, tstate->tracing++; /* Call c_tracefunc directly, having set the line number. */ Py_INCREF(frame_obj); + if (line == -1 && line_delta > COMPUTED_LINE) { + /* Only assign f_lineno if it's easy to calculate, otherwise + * do lazy calculation by setting the f_lineno to 0. + */ + line = compute_line(code, i, line_delta); + } frame_obj->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None); frame_obj->f_lineno = 0; @@ -1201,6 +1297,11 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, if (tools == 0) { goto done; } + + if (line == -1) { + /* Need to calculate the line number now for monitoring events */ + line = compute_line(code, i, line_delta); + } PyObject *line_obj = PyLong_FromLong(line); if (line_obj == NULL) { return -1; @@ -1224,7 +1325,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ + LOCK_CODE(code); remove_line_tools(code, i, 1 << tool); + UNLOCK_CODE(); } } while (tools); Py_DECREF(line_obj); @@ -1241,8 +1344,7 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(debug_check_sanity(tstate->interp, code)); int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); @@ -1280,7 +1382,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ + LOCK_CODE(code); remove_per_instruction_tools(code, offset, 1 << tool); + UNLOCK_CODE(); } } Py_DECREF(offset_obj); @@ -1295,15 +1399,18 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) PyInterpreterState *is = _PyInterpreterState_GET(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); - PyObject *callback = is->monitoring_callables[tool_id][event_id]; - is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); + PyObject *callback = _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], + Py_XNewRef(obj)); + return callback; } static void initialize_tools(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t* tools = code->_co_monitoring->tools; + assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1359,7 +1466,9 @@ initialize_tools(PyCodeObject *code) static void initialize_lines(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + assert(line_data != NULL); int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; @@ -1393,6 +1502,13 @@ initialize_lines(PyCodeObject *code) */ if (line != current_line && line >= 0) { line_data[i].original_opcode = opcode; + if (line_data[i].line_delta == COMPUTED_LINE) { + /* Label this line as a line with a line number change + * which could help the monitoring callback to quickly + * identify the line number change. + */ + line_data[i].line_delta = COMPUTED_LINE_LINENO_CHANGE; + } } else { line_data[i].original_opcode = 0; @@ -1445,6 +1561,11 @@ initialize_lines(PyCodeObject *code) assert(target >= 0); if (line_data[target].line_delta != NO_LINE) { line_data[target].original_opcode = _Py_GetBaseOpcode(code, target); + if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) { + // If the line is a jump target, we are not sure if the line + // number changes, so we set it to COMPUTED_LINE. + line_data[target].line_delta = COMPUTED_LINE; + } } } /* Scan exception table */ @@ -1476,7 +1597,9 @@ initialize_lines(PyCodeObject *code) static void initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t *line_tools = code->_co_monitoring->line_tools; + assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1487,6 +1610,7 @@ initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) static int allocate_instrumentation_data(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); if (code->_co_monitoring == NULL) { code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); @@ -1508,6 +1632,8 @@ allocate_instrumentation_data(PyCodeObject *code) static int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + int code_len = (int)Py_SIZE(code); if (allocate_instrumentation_data(code)) { return -1; @@ -1549,9 +1675,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + // Initialize all of the instructions so if local events change while another thread is executing + // we know what the original opcode was. for (int i = 0; i < code_len; i++) { - code->_co_monitoring->per_instruction_opcodes[i] = 0; + int opcode = _PyCode_CODE(code)[i].op.code; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } } if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { @@ -1560,7 +1688,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { code->_co_monitoring->per_instruction_tools[i] = 0; } @@ -1569,27 +1696,23 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -int -_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +static int +force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) { - if (is_version_up_to_date(code, interp)) { - assert( - (interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK) == 0 || - instrumentation_cross_checks(interp, code) - ); - return 0; - } + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + +#ifdef _Py_TIER2 if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } + _Py_Executors_InvalidateDependency(interp, code, 1); +#endif int code_len = (int)Py_SIZE(code); - /* code->_co_firsttraceable >= code_len indicates - * that no instrumentation can be inserted. - * Exit early to avoid creating instrumentation + /* Exit early to avoid creating instrumentation * data for potential statically allocated code * objects. * See https://github.com/python/cpython/issues/108390 */ - if (code->_co_firsttraceable >= code_len) { + if (code->co_flags & CO_NO_MONITORING_EVENTS) { return 0; } if (update_instrumentation_data(code, interp)) { @@ -1612,12 +1735,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; - code->_co_instrumentation_version = global_version(interp); if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { -#ifdef INSTRUMENT_DEBUG - sanity_check_instrumentation(code); -#endif - return 0; + goto done; } /* Insert instrumentation */ for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { @@ -1648,7 +1767,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) // GH-103845: We need to remove both the line and instruction instrumentation before // adding new ones, otherwise we may remove the newly added instrumentation. - uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1656,9 +1774,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (removed_line_tools) { - remove_line_tools(code, i, removed_line_tools); - } + remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1670,15 +1786,14 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (removed_per_instruction_tools) { - remove_per_instruction_tools(code, i, removed_per_instruction_tools); - } + remove_per_instruction_tools(code, i, removed_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1686,9 +1801,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } + add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1700,18 +1813,47 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (new_per_instruction_tools) { - add_per_instruction_tools(code, i, new_per_instruction_tools); - } + add_per_instruction_tools(code, i, new_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } + +done: + FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version, + global_version(interp)); + #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif return 0; } +static int +instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + if (is_version_up_to_date(code, interp)) { + assert( + interp->ceval.instrumentation_version == 0 || + instrumentation_cross_checks(interp, code) + ); + return 0; + } + + return force_instrument_lock_held(code, interp); +} + +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + int res; + LOCK_CODE(code); + res = instrument_lock_held(code, interp); + UNLOCK_CODE(); + return res; +} + #define C_RETURN_EVENTS \ ((1 << PY_MONITORING_EVENT_C_RETURN) | \ (1 << PY_MONITORING_EVENT_C_RAISE)) @@ -1722,6 +1864,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int instrument_all_executing_code_objects(PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED(); + _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); PyThreadState* ts = PyInterpreterState_ThreadHead(interp); @@ -1730,7 +1874,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { - if (_Py_Instrument(_PyFrame_GetCode(frame), interp)) { + if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { return -1; } } @@ -1787,23 +1931,35 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; } + + int res; + _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - return -1; + res = -1; + goto done; } - set_global_version(interp, new_version); - return instrument_all_executing_code_objects(interp); + set_global_version(tstate, new_version); +#ifdef _Py_TIER2 + _Py_Executors_InvalidateAll(interp, 1); +#endif + res = instrument_all_executing_code_objects(interp); +done: + _PyEval_StartTheWorld(interp); + return res; } int @@ -1819,22 +1975,43 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent if (check_tool(interp, tool_id)) { return -1; } + + int res; + _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { - return -1; + res = -1; + goto done; } + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_local_events(local, tool_id, events); - if (is_version_up_to_date(code, interp)) { - /* Force instrumentation update */ - code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; - } - if (_Py_Instrument(code, interp)) { + + res = force_instrument_lock_held(code, interp); + +done: + _PyEval_StartTheWorld(interp); + return res; +} + +int +_PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (check_tool(interp, tool_id)) { return -1; } + if (code->_co_monitoring == NULL) { + *events = 0; + return 0; + } + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; + *events = get_local_events(local, tool_id); return 0; } @@ -2112,16 +2289,23 @@ monitoring_restart_events_impl(PyObject *module) * last restart version > instrumented version for all code objects * last restart version < current version */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; + + _PyEval_StopTheWorld(interp); uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; if (new_version <= MONITORING_VERSION_INCREMENT) { + _PyEval_StartTheWorld(interp); PyErr_Format(PyExc_OverflowError, "events set too many times"); return NULL; } interp->last_restart_version = restart_version; - set_global_version(interp, new_version); - if (instrument_all_executing_code_objects(interp)) { + set_global_version(tstate, new_version); + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + + if (res) { return NULL; } Py_RETURN_NONE; @@ -2239,3 +2423,309 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } + + +static int +capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject **args, Py_ssize_t nargs, int event) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; + + uint8_t tools = state->active; + assert(args[1] == NULL); + args[1] = codelike; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); + return -1; + } + if (event != PY_MONITORING_EVENT_LINE) { + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; + } + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + PyErr_Format(PyExc_ValueError, + "Cannot disable %s events. Callback removed.", + event_names[event]); + /* Clear tool to prevent infinite loop */ + Py_CLEAR(interp->monitoring_callables[tool][event]); + err = -1; + break; + } + else { + state->active &= ~(1 << tool); + } + } + } + return err; +} + +int +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (global_version(interp) == *version) { + return 0; + } + + _Py_GlobalMonitors *m = &interp->monitors; + for (Py_ssize_t i = 0; i < length; i++) { + int event = event_types[i]; + state_array[i].active = m->tools[event]; + } + *version = global_version(interp); + return 0; +} + +int +PyMonitoring_ExitScope(void) +{ + return 0; +} + +int +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_START); +} + +int +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_RESUME); +} + + + +int +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_RETURN); +} + +int +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_YIELD); +} + +int +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + assert(state->active); + PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 }; + return capi_call_instrumentation(state, codelike, offset, args, 4, + PY_MONITORING_EVENT_CALL); +} + +int +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + assert(state->active); + PyObject *lno = PyLong_FromLong(lineno); + if (lno == NULL) { + return -1; + } + PyObject *args[3] = { NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_LINE); + Py_DECREF(lno); + return res; +} + +int +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_JUMP); +} + +int +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH); +} + +int +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_C_RETURN); +} + +static inline int +exception_event_setup(PyObject **exc, int event) { + *exc = PyErr_GetRaisedException(); + if (*exc == NULL) { + PyErr_Format(PyExc_ValueError, + "Firing event %d with no exception set", + event); + return -1; + } + return 0; +} + + +static inline int +exception_event_teardown(int err, PyObject *exc) { + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_XDECREF(exc); + } + return err; +} + +int +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_THROW; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_C_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RERAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_UNWIND; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + int event = PY_MONITORING_EVENT_STOP_ITERATION; + assert(state->active); + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + Py_DECREF(exc); + return exception_event_teardown(err, NULL); +} diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000000..54e5dca284c215 --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,268 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(STR); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + Py_DECREF(item); + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + + Py_DECREF(dict); + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} diff --git a/Python/intrinsics.c b/Python/intrinsics.c index bbd79ec473f470..a6b2c108b67175 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -5,6 +5,7 @@ #include "pycore_frame.h" #include "pycore_function.h" #include "pycore_global_objects.h" +#include "pycore_compile.h" // _PyCompile_GetUnaryIntrinsicName, etc #include "pycore_intrinsics.h" // INTRINSIC_PRINT #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() @@ -122,18 +123,15 @@ static PyObject * import_star(PyThreadState* tstate, PyObject *from) { _PyInterpreterFrame *frame = tstate->current_frame; - if (_PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *locals = frame->f_locals; + PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); return NULL; } int err = import_all_from(tstate, locals, from); - _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(locals); if (err < 0) { return NULL; } @@ -264,12 +262,13 @@ _PyIntrinsics_BinaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_BOUND, make_typevar_with_bound) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, make_typevar_with_constraints) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_FUNCTION_TYPE_PARAMS, _Py_set_function_type_params) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_DEFAULT, _Py_set_typeparam_default) }; #undef INTRINSIC_FUNC_ENTRY PyObject* -PyUnstable_GetUnaryIntrinsicName(int index) +_PyCompile_GetUnaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_1) { return NULL; @@ -278,7 +277,7 @@ PyUnstable_GetUnaryIntrinsicName(int index) } PyObject* -PyUnstable_GetBinaryIntrinsicName(int index) +_PyCompile_GetBinaryIntrinsicName(int index) { if (index < 0 || index > MAX_INTRINSIC_2) { return NULL; diff --git a/Python/jit.c b/Python/jit.c new file mode 100644 index 00000000000000..33320761621c4c --- /dev/null +++ b/Python/jit.c @@ -0,0 +1,482 @@ +#ifdef _Py_JIT + +#include "Python.h" + +#include "pycore_abstract.h" +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_critical_section.h" +#include "pycore_dict.h" +#include "pycore_intrinsics.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyerrors.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" +#include "pycore_jit.h" + +// Memory management stuff: //////////////////////////////////////////////////// + +#ifndef MS_WINDOWS + #include +#endif + +static size_t +get_page_size(void) +{ +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +static void +jit_error(const char *message) +{ +#ifdef MS_WINDOWS + int hint = GetLastError(); +#else + int hint = errno; +#endif + PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); +} + +static unsigned char * +jit_alloc(size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int flags = MEM_COMMIT | MEM_RESERVE; + unsigned char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + int failed = memory == NULL; +#else + int flags = MAP_ANONYMOUS | MAP_PRIVATE; + unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int failed = memory == MAP_FAILED; +#endif + if (failed) { + jit_error("unable to allocate memory"); + return NULL; + } + return memory; +} + +static int +jit_free(unsigned char *memory, size_t size) +{ + assert(size); + assert(size % get_page_size() == 0); +#ifdef MS_WINDOWS + int failed = !VirtualFree(memory, 0, MEM_RELEASE); +#else + int failed = munmap(memory, size); +#endif + if (failed) { + jit_error("unable to free memory"); + return -1; + } + return 0; +} + +static int +mark_executable(unsigned char *memory, size_t size) +{ + if (size == 0) { + return 0; + } + assert(size % get_page_size() == 0); + // Do NOT ever leave the memory writable! Also, don't forget to flush the + // i-cache (I cannot begin to tell you how horrible that is to debug): +#ifdef MS_WINDOWS + if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) { + jit_error("unable to flush instruction cache"); + return -1; + } + int old; + int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); +#else + __builtin___clear_cache((char *)memory, (char *)memory + size); + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#endif + if (failed) { + jit_error("unable to protect executable memory"); + return -1; + } + return 0; +} + +// JIT compiler stuff: ///////////////////////////////////////////////////////// + +// Warning! AArch64 requires you to get your hands dirty. These are your gloves: + +// value[value_start : value_start + len] +static uint32_t +get_bits(uint64_t value, uint8_t value_start, uint8_t width) +{ + assert(width <= 32); + return (value >> value_start) & ((1ULL << width) - 1); +} + +// *loc[loc_start : loc_start + width] = value[value_start : value_start + width] +static void +set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, + uint8_t width) +{ + assert(loc_start + width <= 32); + // Clear the bits we're about to patch: + *loc &= ~(((1ULL << width) - 1) << loc_start); + assert(get_bits(*loc, loc_start, width) == 0); + // Patch the bits: + *loc |= get_bits(value, value_start, width) << loc_start; + assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width)); +} + +// See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions +// for instruction encodings: +#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) +#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) +#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) +#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) +#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) + +// LLD is a great reference for performing relocations... just keep in +// mind that Tools/jit/build.py does filtering and preprocessing for us! +// Here's a good place to start for each platform: +// - aarch64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h +// - aarch64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - aarch64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp +// - i686-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp +// - x86_64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + +// Many of these patches are "relaxing", meaning that they can rewrite the +// code they're patching to be more efficient (like turning a 64-bit memory +// load into a 32-bit immediate load). These patches have an "x" in their name. +// Relative patches have an "r" in their name. + +// 32-bit absolute address. +void +patch_32(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; +} + +// 32-bit relative address. +void +patch_32r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value -= (uintptr_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + *loc32 = (uint32_t)value; +} + +// 64-bit absolute address. +void +patch_64(unsigned char *location, uint64_t value) +{ + uint64_t *loc64 = (uint64_t *)location; + *loc64 = value; +} + +// 12-bit low part of an absolute address. Pairs nicely with patch_aarch64_21r +// (below). +void +patch_aarch64_12(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); + } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); +} + +// Relaxable 12-bit low part of an absolute address. Pairs nicely with +// patch_aarch64_21rx (below). +void +patch_aarch64_12x(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_21rx. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_12(location, value); +} + +// 16-bit low part of an absolute address. +void +patch_aarch64_16a(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); +} + +// 16-bit middle-low part of an absolute address. +void +patch_aarch64_16b(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); +} + +// 16-bit middle-high part of an absolute address. +void +patch_aarch64_16c(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); +} + +// 16-bit high part of an absolute address. +void +patch_aarch64_16d(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); +} + +// 21-bit count of pages between this page and an absolute address's page... I +// know, I know, it's weird. Pairs nicely with patch_aarch64_12 (above). +void +patch_aarch64_21r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value = (value >> 12) - ((uintptr_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); +} + +// Relaxable 21-bit count of pages between this page and an absolute address's +// page. Pairs nicely with patch_aarch64_12x (above). +void +patch_aarch64_21rx(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_12x. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_21r(location, value); +} + +// 28-bit relative branch. +void +patch_aarch64_26r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uintptr_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); +} + +// A pair of patch_aarch64_21rx and patch_aarch64_12x. +void +patch_aarch64_33rx(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Try to relax the pair of GOT loads into an immediate value: + assert(IS_AARCH64_ADRP(*loc32)); + unsigned char reg = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; + return; + } + relaxed = value - (uintptr_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + // Couldn't do it. Just patch the two instructions normally: + patch_aarch64_21rx(location, value); + patch_aarch64_12x(location + 4, value); +} + +// Relaxable 32-bit relative address. +void +patch_x86_64_32rx(unsigned char *location, uint64_t value) +{ + uint8_t *loc8 = (uint8_t *)location; + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + patch_32r(location, value); +} + +#include "jit_stencils.h" + +// Compiles executor in-place. Don't forget to call _PyJIT_Free later! +int +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], size_t length) +{ + const StencilGroup *group; + // Loop once to find the total compiled size: + uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + size_t code_size = 0; + size_t data_size = 0; + group = &trampoline; + code_size += group->code_size; + data_size += group->data_size; + for (size_t i = 0; i < length; i++) { + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + instruction_starts[i] = code_size; + code_size += group->code_size; + data_size += group->data_size; + } + group = &stencil_groups[_FATAL_ERROR]; + code_size += group->code_size; + data_size += group->data_size; + // Round up to the nearest page: + size_t page_size = get_page_size(); + assert((page_size & (page_size - 1)) == 0); + size_t padding = page_size - ((code_size + data_size) & (page_size - 1)); + size_t total_size = code_size + data_size + padding; + unsigned char *memory = jit_alloc(total_size); + if (memory == NULL) { + return -1; + } + // Update the offsets of each instruction: + for (size_t i = 0; i < length; i++) { + instruction_starts[i] += (uintptr_t)memory; + } + // Loop again to emit the code: + unsigned char *code = memory; + unsigned char *data = memory + code_size; + // Compile the trampoline, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). On platforms where + // we don't change calling conventions, the trampoline is empty and + // nothing is emitted here: + group = &trampoline; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; + assert(trace[0].opcode == _START_EXECUTOR); + for (size_t i = 0; i < length; i++) { + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + group->emit(code, data, executor, instruction, instruction_starts); + code += group->code_size; + data += group->data_size; + } + // Protect against accidental buffer overrun into data: + group = &stencil_groups[_FATAL_ERROR]; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; + assert(code == memory + code_size); + assert(data == memory + code_size + data_size); + if (mark_executable(memory, total_size)) { + jit_free(memory, total_size); + return -1; + } + executor->jit_code = memory; + executor->jit_side_entry = memory + trampoline.code_size; + executor->jit_size = total_size; + return 0; +} + +void +_PyJIT_Free(_PyExecutorObject *executor) +{ + unsigned char *memory = (unsigned char *)executor->jit_code; + size_t size = executor->jit_size; + if (memory) { + executor->jit_code = NULL; + executor->jit_side_entry = NULL; + executor->jit_size = 0; + if (jit_free(memory, size)) { + PyErr_WriteUnraisable(NULL); + } + } +} + +#endif // _Py_JIT diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 97d39a14ae43d0..74118030925e3e 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -16,6 +16,13 @@ typedef struct _PyLegacyEventHandler { int event; } _PyLegacyEventHandler; +#ifdef Py_GIL_DISABLED +#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#else +#define LOCK_SETUP() +#define UNLOCK_SETUP() +#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -46,7 +53,7 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) } static PyObject * -sys_profile_func2( +sys_profile_start( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -56,7 +63,17 @@ sys_profile_func2( } static PyObject * -sys_profile_func3( +sys_profile_throw( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + return call_profile_func(self, Py_None); +} + +static PyObject * +sys_profile_return( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -72,7 +89,7 @@ sys_profile_unwind( ) { assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); - return call_profile_func(self, Py_None); + return call_profile_func(self, NULL); } static PyObject * @@ -107,6 +124,35 @@ sys_profile_call_or_return( Py_RETURN_NONE; } +int +_PyEval_SetOpcodeTrace( + PyFrameObject *frame, + bool enable +) { + assert(frame != NULL); + assert(PyCode_Check(frame->f_frame->f_executable)); + + PyCodeObject *code = (PyCodeObject *)frame->f_frame->f_executable; + _PyMonitoringEventSet events = 0; + + if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { + return -1; + } + + if (enable) { + if (events & (1 << PY_MONITORING_EVENT_INSTRUCTION)) { + return 0; + } + events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); + } else { + if (!(events & (1 << PY_MONITORING_EVENT_INSTRUCTION))) { + return 0; + } + events &= (~(1 << PY_MONITORING_EVENT_INSTRUCTION)); + } + return _PyMonitoring_SetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, events); +} + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -120,8 +166,15 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) "Missing frame when calling trace function."); return NULL; } + if (frame->f_trace_opcodes) { + if (_PyEval_SetOpcodeTrace(frame, true) != 0) { + return NULL; + } + } + Py_INCREF(frame); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + frame->f_lineno = 0; Py_DECREF(frame); if (err) { return NULL; @@ -154,7 +207,7 @@ sys_trace_exception_func( } static PyObject * -sys_trace_func2( +sys_trace_start( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -164,7 +217,7 @@ sys_trace_func2( } static PyObject * -sys_trace_func3( +sys_trace_throw( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -173,6 +226,16 @@ sys_trace_func3( return call_trace_func(self, Py_None); } +static PyObject * +sys_trace_unwind( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + return call_trace_func(self, NULL); +} + static PyObject * sys_trace_return( _PyLegacyEventHandler *self, PyObject *const *args, @@ -210,11 +273,14 @@ sys_trace_instruction_func( "Missing frame when calling trace function."); return NULL; } - if (!frame->f_trace_opcodes) { + PyThreadState *tstate = _PyThreadState_GET(); + if (!tstate->c_tracefunc || !frame->f_trace_opcodes) { + if (_PyEval_SetOpcodeTrace(frame, false) != 0) { + return NULL; + } Py_RETURN_NONE; } Py_INCREF(frame); - PyThreadState *tstate = _PyThreadState_GET(); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); frame->f_lineno = 0; Py_DECREF(frame); @@ -356,34 +422,25 @@ is_tstate_valid(PyThreadState *tstate) } #endif -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +static Py_ssize_t +setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) { - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } + *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + (vectorcallfunc)sys_profile_start, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - (vectorcallfunc)sys_profile_func3, PyTrace_CALL, + (vectorcallfunc)sys_profile_throw, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + (vectorcallfunc)sys_profile_return, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { return -1; } @@ -411,25 +468,15 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; + *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); - Py_XDECREF(old_profileobj); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - - uint32_t events = 0; - if (tstate->interp->sys_profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); - } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + return tstate->interp->sys_profiling_threads; } int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) { assert(is_tstate_valid(tstate)); /* The caller must hold the GIL */ @@ -438,21 +485,42 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Call _PySys_Audit() in the context of the current thread state, even if tstate is not the current thread state. */ PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { return -1; } - assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_profileobj; + LOCK_SETUP(); + Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); + UNLOCK_SETUP(); + Py_XDECREF(old_profileobj); + + uint32_t events = 0; + if (profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); +} + +static Py_ssize_t +setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +{ + *old_traceobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_start, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func3, PyTrace_CALL, + (vectorcallfunc)sys_trace_throw, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } @@ -477,7 +545,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, + (vectorcallfunc)sys_trace_unwind, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } @@ -495,25 +563,55 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; + *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); - Py_XDECREF(old_traceobj); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); + return tstate->interp->sys_tracing_threads; +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_traceobj; + LOCK_SETUP(); + Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); + UNLOCK_SETUP(); + Py_XDECREF(old_traceobj); + if (tracing_threads < 0) { + return -1; + } uint32_t events = 0; - if (tstate->interp->sys_tracing_threads) { + if (tracing_threads) { events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | - (1 << PY_MONITORING_EVENT_STOP_ITERATION) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); - if (tstate->interp->f_opcode_trace_set) { - events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); + (1 << PY_MONITORING_EVENT_STOP_ITERATION); + + PyFrameObject* frame = PyEval_GetFrame(); + if (frame->f_trace_opcodes) { + int ret = _PyEval_SetOpcodeTrace(frame, true); + if (ret != 0) { + return ret; + } } } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); } diff --git a/Python/lock.c b/Python/lock.c index 3dad2aa93b5cc9..57675fe1873fa2 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,23 +5,24 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" +#include "pycore_time.h" // _PyTime_Add() #ifdef MS_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include // SwitchToThread() +# define WIN32_LEAN_AND_MEAN +# include // SwitchToThread() #elif defined(HAVE_SCHED_H) -#include // sched_yield() +# include // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then // the unlocking thread directly hands off ownership of the lock. This avoids // starvation. -static const _PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; +static const PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; // Spin for a bit before parking the thread. This is only enabled for // `--disable-gil` builds because it is unlikely to be helpful if the GIL is // enabled. -#if Py_NOGIL +#if Py_GIL_DISABLED static const int MAX_SPIN_COUNT = 40; #else static const int MAX_SPIN_COUNT = 0; @@ -30,7 +31,7 @@ static const int MAX_SPIN_COUNT = 0; struct mutex_entry { // The time after which the unlocking thread should hand off lock ownership // directly to the waiting thread. Written by the waiting thread. - _PyTime_t time_to_be_fair; + PyTime_t time_to_be_fair; // Set to 1 if the lock was handed off. Written by the unlocking thread. int handed_off; @@ -46,18 +47,12 @@ _Py_yield(void) #endif } -void -_PyMutex_LockSlow(PyMutex *m) -{ - _PyMutex_LockTimed(m, -1, _PY_LOCK_DETACH); -} - PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) { - uint8_t v = _Py_atomic_load_uint8_relaxed(&m->v); + uint8_t v = _Py_atomic_load_uint8_relaxed(&m->_bits); if ((v & _Py_LOCKED) == 0) { - if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v|_Py_LOCKED)) { return PY_LOCK_ACQUIRED; } } @@ -65,8 +60,10 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - _PyTime_t now = _PyTime_GetMonotonicClock(); - _PyTime_t endtime = 0; + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); } @@ -80,7 +77,7 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) for (;;) { if ((v & _Py_LOCKED) == 0) { // The lock is unlocked. Try to grab it. - if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v|_Py_LOCKED)) { return PY_LOCK_ACQUIRED; } continue; @@ -101,17 +98,17 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) if (!(v & _Py_HAS_PARKED)) { // We are the first waiter. Set the _Py_HAS_PARKED flag. newv = v | _Py_HAS_PARKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &v, newv)) { + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &v, newv)) { continue; } } - int ret = _PyParkingLot_Park(&m->v, &newv, sizeof(newv), timeout, + int ret = _PyParkingLot_Park(&m->_bits, &newv, sizeof(newv), timeout, &entry, (flags & _PY_LOCK_DETACH) != 0); if (ret == Py_PARK_OK) { if (entry.handed_off) { // We own the lock now. - assert(_Py_atomic_load_uint8_relaxed(&m->v) & _Py_LOCKED); + assert(_Py_atomic_load_uint8_relaxed(&m->_bits) & _Py_LOCKED); return PY_LOCK_ACQUIRED; } } @@ -133,7 +130,7 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) } } - v = _Py_atomic_load_uint8_relaxed(&m->v); + v = _Py_atomic_load_uint8_relaxed(&m->_bits); } } @@ -142,7 +139,9 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; @@ -153,13 +152,13 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) v |= _Py_HAS_PARKED; } } - _Py_atomic_store_uint8(&m->v, v); + _Py_atomic_store_uint8(&m->_bits, v); } int _PyMutex_TryUnlock(PyMutex *m) { - uint8_t v = _Py_atomic_load_uint8(&m->v); + uint8_t v = _Py_atomic_load_uint8(&m->_bits); for (;;) { if ((v & _Py_LOCKED) == 0) { // error: the mutex is not locked @@ -167,24 +166,16 @@ _PyMutex_TryUnlock(PyMutex *m) } else if ((v & _Py_HAS_PARKED)) { // wake up a single thread - _PyParkingLot_Unpark(&m->v, (_Py_unpark_fn_t *)mutex_unpark, m); + _PyParkingLot_Unpark(&m->_bits, (_Py_unpark_fn_t *)mutex_unpark, m); return 0; } - else if (_Py_atomic_compare_exchange_uint8(&m->v, &v, _Py_UNLOCKED)) { + else if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, _Py_UNLOCKED)) { // fast-path: no waiters return 0; } } } -void -_PyMutex_UnlockSlow(PyMutex *m) -{ - if (_PyMutex_TryUnlock(m) < 0) { - Py_FatalError("unlocking mutex that is not locked"); - } -} - // _PyRawMutex stores a linked list of `struct raw_mutex_entry`, one for each // thread waiting on the mutex, directly in the mutex itself. struct raw_mutex_entry { @@ -248,6 +239,13 @@ _PyRawMutex_UnlockSlow(_PyRawMutex *m) } } +int +_PyEvent_IsSet(PyEvent *evt) +{ + uint8_t v = _Py_atomic_load_uint8(&evt->v); + return v == _Py_LOCKED; +} + void _PyEvent_Notify(PyEvent *evt) { @@ -269,12 +267,12 @@ _PyEvent_Notify(PyEvent *evt) void PyEvent_Wait(PyEvent *evt) { - while (!PyEvent_WaitTimed(evt, -1)) + while (!PyEvent_WaitTimed(evt, -1, /*detach=*/1)) ; } int -PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -290,8 +288,305 @@ PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) uint8_t expected = _Py_HAS_PARKED; (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), - timeout_ns, NULL, 1); + timeout_ns, NULL, detach); return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } } + +static int +unlock_once(_PyOnceFlag *o, int res) +{ + // On success (res=0), we set the state to _Py_ONCE_INITIALIZED. + // On failure (res=-1), we reset the state to _Py_UNLOCKED. + uint8_t new_value; + switch (res) { + case -1: new_value = _Py_UNLOCKED; break; + case 0: new_value = _Py_ONCE_INITIALIZED; break; + default: { + Py_FatalError("invalid result from _PyOnceFlag_CallOnce"); + Py_UNREACHABLE(); + break; + } + } + + uint8_t old_value = _Py_atomic_exchange_uint8(&o->v, new_value); + if ((old_value & _Py_HAS_PARKED) != 0) { + // wake up anyone waiting on the once flag + _PyParkingLot_UnparkAll(&o->v); + } + return res; +} + +int +_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) +{ + uint8_t v = _Py_atomic_load_uint8(&flag->v); + for (;;) { + if (v == _Py_UNLOCKED) { + if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, _Py_LOCKED)) { + continue; + } + int res = fn(arg); + return unlock_once(flag, res); + } + + if (v == _Py_ONCE_INITIALIZED) { + return 0; + } + + // The once flag is initializing (locked). + assert((v & _Py_LOCKED)); + if (!(v & _Py_HAS_PARKED)) { + // We are the first waiter. Set the _Py_HAS_PARKED flag. + uint8_t new_value = v | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uint8(&flag->v, &v, new_value)) { + continue; + } + v = new_value; + } + + // Wait for initialization to finish. + _PyParkingLot_Park(&flag->v, &v, sizeof(v), -1, NULL, 1); + v = _Py_atomic_load_uint8(&flag->v); + } +} + +static int +recursive_mutex_is_owned_by(_PyRecursiveMutex *m, PyThread_ident_t tid) +{ + return _Py_atomic_load_ullong_relaxed(&m->thread) == tid; +} + +int +_PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m) +{ + return recursive_mutex_is_owned_by(m, PyThread_get_thread_ident_ex()); +} + +void +_PyRecursiveMutex_Lock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (recursive_mutex_is_owned_by(m, thread)) { + m->level++; + return; + } + PyMutex_Lock(&m->mutex); + _Py_atomic_store_ullong_relaxed(&m->thread, thread); + assert(m->level == 0); +} + +void +_PyRecursiveMutex_Unlock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (!recursive_mutex_is_owned_by(m, thread)) { + Py_FatalError("unlocking a recursive mutex that is not owned by the" + " current thread"); + } + if (m->level > 0) { + m->level--; + return; + } + assert(m->level == 0); + _Py_atomic_store_ullong_relaxed(&m->thread, 0); + PyMutex_Unlock(&m->mutex); +} + +#define _Py_WRITE_LOCKED 1 +#define _PyRWMutex_READER_SHIFT 2 +#define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT) + +static uintptr_t +rwmutex_set_parked_and_wait(_PyRWMutex *rwmutex, uintptr_t bits) +{ + // Set _Py_HAS_PARKED and wait until we are woken up. + if ((bits & _Py_HAS_PARKED) == 0) { + uintptr_t newval = bits | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + return bits; + } + bits = newval; + } + + _PyParkingLot_Park(&rwmutex->bits, &bits, sizeof(bits), -1, NULL, 1); + return _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); +} + +// The number of readers holding the lock +static uintptr_t +rwmutex_reader_count(uintptr_t bits) +{ + return bits >> _PyRWMutex_READER_SHIFT; +} + +void +_PyRWMutex_RLock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + if ((bits & _Py_WRITE_LOCKED)) { + // A writer already holds the lock. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else if ((bits & _Py_HAS_PARKED)) { + // Reader(s) hold the lock (or just gave up the lock), but there is + // at least one waiting writer. We can't grab the lock because we + // don't want to starve the writer. Instead, we park ourselves and + // wait for the writer to eventually wake us up. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else { + // The lock is unlocked or read-locked. Try to grab it. + assert(rwmutex_reader_count(bits) < _Py_RWMUTEX_MAX_READERS); + uintptr_t newval = bits + (1 << _PyRWMutex_READER_SHIFT); + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + continue; + } + return; + } + } +} + +void +_PyRWMutex_RUnlock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_add_uintptr(&rwmutex->bits, -(1 << _PyRWMutex_READER_SHIFT)); + assert(rwmutex_reader_count(bits) > 0 && "lock was not read-locked"); + bits -= (1 << _PyRWMutex_READER_SHIFT); + + if (rwmutex_reader_count(bits) == 0 && (bits & _Py_HAS_PARKED)) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + return; + } +} + +void +_PyRWMutex_Lock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + // If there are no active readers and it's not already write-locked, + // then we can grab the lock. + if ((bits & ~_Py_HAS_PARKED) == 0) { + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, + bits | _Py_WRITE_LOCKED)) { + continue; + } + return; + } + + // Otherwise, we have to wait. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + } +} + +void +_PyRWMutex_Unlock(_PyRWMutex *rwmutex) +{ + uintptr_t old_bits = _Py_atomic_exchange_uintptr(&rwmutex->bits, 0); + + assert((old_bits & _Py_WRITE_LOCKED) && "lock was not write-locked"); + assert(rwmutex_reader_count(old_bits) == 0 && "lock was read-locked"); + + if ((old_bits & _Py_HAS_PARKED) != 0) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + } +} + +#define SEQLOCK_IS_UPDATING(sequence) (sequence & 0x01) + +void _PySeqLock_LockWrite(_PySeqLock *seqlock) +{ + // lock by moving to an odd sequence number + uint32_t prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence); + while (1) { + if (SEQLOCK_IS_UPDATING(prev)) { + // Someone else is currently updating the cache + _Py_yield(); + prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence); + } + else if (_Py_atomic_compare_exchange_uint32(&seqlock->sequence, &prev, prev + 1)) { + // We've locked the cache + _Py_atomic_fence_release(); + break; + } + else { + _Py_yield(); + } + } +} + +void _PySeqLock_AbandonWrite(_PySeqLock *seqlock) +{ + uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) - 1; + assert(!SEQLOCK_IS_UPDATING(new_seq)); + _Py_atomic_store_uint32(&seqlock->sequence, new_seq); +} + +void _PySeqLock_UnlockWrite(_PySeqLock *seqlock) +{ + uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) + 1; + assert(!SEQLOCK_IS_UPDATING(new_seq)); + _Py_atomic_store_uint32(&seqlock->sequence, new_seq); +} + +uint32_t _PySeqLock_BeginRead(_PySeqLock *seqlock) +{ + uint32_t sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence); + while (SEQLOCK_IS_UPDATING(sequence)) { + _Py_yield(); + sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence); + } + + return sequence; +} + +int _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) +{ + // gh-121368: We need an explicit acquire fence here to ensure that + // this load of the sequence number is not reordered before any loads + // within the read lock. + _Py_atomic_fence_acquire(); + + if (_Py_atomic_load_uint32_relaxed(&seqlock->sequence) == previous) { + return 1; + } + + _Py_yield(); + return 0; +} + +int _PySeqLock_AfterFork(_PySeqLock *seqlock) +{ + // Synchronize again and validate that the entry hasn't been updated + // while we were readying the values. + if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { + seqlock->sequence = 0; + return 1; + } + + return 0; +} + +#undef PyMutex_Lock +void +PyMutex_Lock(PyMutex *m) +{ + _PyMutex_LockTimed(m, -1, _PY_LOCK_DETACH); +} + +#undef PyMutex_Unlock +void +PyMutex_Unlock(PyMutex *m) +{ + if (_PyMutex_TryUnlock(m) < 0) { + Py_FatalError("unlocking mutex that is not locked"); + } +} diff --git a/Python/marshal.c b/Python/marshal.c index 8940582c7f5328..fe97ccde2e5b13 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -7,12 +7,18 @@ and sharing. */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_code.h" // _PyCode_New() -#include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_long.h" // _PyLong_DigitCount -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "marshal.h" // Py_MARSHAL_VERSION +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_code.h" // _PyCode_New() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_long.h" // _PyLong_DigitCount +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "marshal.h" // Py_MARSHAL_VERSION +#include "pycore_pystate.h" // _PyInterpreterState_GET() + +#ifdef __APPLE__ +# include "TargetConditionals.h" +#endif /* __APPLE__ */ /*[clinic input] module marshal @@ -33,11 +39,15 @@ module marshal * #if defined(MS_WINDOWS) && defined(_DEBUG) */ #if defined(MS_WINDOWS) -#define MAX_MARSHAL_STACK_DEPTH 1000 +# define MAX_MARSHAL_STACK_DEPTH 1000 #elif defined(__wasi__) -#define MAX_MARSHAL_STACK_DEPTH 1500 +# define MAX_MARSHAL_STACK_DEPTH 1500 +// TARGET_OS_IPHONE covers any non-macOS Apple platform. +// It won't be defined on older macOS SDKs +#elif defined(__APPLE__) && defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +# define MAX_MARSHAL_STACK_DEPTH 1500 #else -#define MAX_MARSHAL_STACK_DEPTH 2000 +# define MAX_MARSHAL_STACK_DEPTH 2000 #endif #define TYPE_NULL '0' @@ -78,6 +88,7 @@ module marshal #define WFERR_UNMARSHALLABLE 1 #define WFERR_NESTEDTOODEEP 2 #define WFERR_NOMEMORY 3 +#define WFERR_CODE_NOT_ALLOWED 4 typedef struct { FILE *fp; @@ -89,6 +100,7 @@ typedef struct { char *buf; _Py_hashtable_t *hashtable; int version; + int allow_code; } WFILE; #define w_byte(c, p) do { \ @@ -225,6 +237,9 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p) w_byte((t) | flag, (p)); \ } while(0) +static PyObject * +_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code); + static void w_PyLong(const PyLongObject *ob, char flag, WFILE *p) { @@ -519,22 +534,29 @@ w_complex_object(PyObject *v, char flag, WFILE *p) return; } Py_ssize_t i = 0; - while (_PySet_NextEntry(v, &pos, &value, &hash)) { - PyObject *dump = PyMarshal_WriteObjectToString(value, p->version); + Py_BEGIN_CRITICAL_SECTION(v); + while (_PySet_NextEntryRef(v, &pos, &value, &hash)) { + PyObject *dump = _PyMarshal_WriteObjectToString(value, + p->version, p->allow_code); if (dump == NULL) { p->error = WFERR_UNMARSHALLABLE; - Py_DECREF(pairs); - return; + Py_DECREF(value); + break; } PyObject *pair = PyTuple_Pack(2, dump, value); Py_DECREF(dump); + Py_DECREF(value); if (pair == NULL) { p->error = WFERR_NOMEMORY; - Py_DECREF(pairs); - return; + break; } PyList_SET_ITEM(pairs, i++, pair); } + Py_END_CRITICAL_SECTION(); + if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) { + Py_DECREF(pairs); + return; + } assert(i == n); if (PyList_Sort(pairs)) { p->error = WFERR_NOMEMORY; @@ -549,6 +571,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) Py_DECREF(pairs); } else if (PyCode_Check(v)) { + if (!p->allow_code) { + p->error = WFERR_CODE_NOT_ALLOWED; + return; + } PyCodeObject *co = (PyCodeObject *)v; PyObject *co_code = _PyCode_GetCode(co); if (co_code == NULL) { @@ -657,6 +683,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.end = wf.ptr + sizeof(buf); wf.error = WFERR_OK; wf.version = version; + wf.allow_code = 1; if (w_init_refs(&wf, version)) { return; /* caller must check PyErr_Occurred() */ } @@ -674,6 +701,7 @@ typedef struct { char *buf; Py_ssize_t buf_size; PyObject *refs; /* a list */ + int allow_code; } RFILE; static const char * @@ -1128,7 +1156,7 @@ r_object(RFILE *p) case TYPE_ASCII_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_ASCII: n = r_long(p); if (n < 0 || n > SIZE32_MAX) { @@ -1142,7 +1170,7 @@ r_object(RFILE *p) case TYPE_SHORT_ASCII_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_SHORT_ASCII: n = r_byte(p); if (n == EOF) { @@ -1157,8 +1185,12 @@ r_object(RFILE *p) v = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, ptr, n); if (v == NULL) break; - if (is_interned) - PyUnicode_InternInPlace(&v); + if (is_interned) { + // marshal is meant to serialize .pyc files with code + // objects, and code-related strings are currently immortal. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &v); + } retval = v; R_REF(retval); break; @@ -1166,7 +1198,7 @@ r_object(RFILE *p) case TYPE_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_UNICODE: { const char *buffer; @@ -1190,8 +1222,12 @@ r_object(RFILE *p) } if (v == NULL) break; - if (is_interned) - PyUnicode_InternInPlace(&v); + if (is_interned) { + // marshal is meant to serialize .pyc files with code + // objects, and code-related strings are currently immortal. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &v); + } retval = v; R_REF(retval); break; @@ -1364,6 +1400,11 @@ r_object(RFILE *p) PyObject* linetable = NULL; PyObject *exceptiontable = NULL; + if (!p->allow_code) { + PyErr_SetString(PyExc_ValueError, + "unmarshalling code objects is disallowed"); + break; + } idx = r_ref_reserve(flag, p); if (idx < 0) break; @@ -1609,6 +1650,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) { RFILE rf; PyObject *result; + rf.allow_code = 1; rf.fp = fp; rf.readable = NULL; rf.depth = 0; @@ -1629,6 +1671,7 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) { RFILE rf; PyObject *result; + rf.allow_code = 1; rf.fp = NULL; rf.readable = NULL; rf.ptr = str; @@ -1645,8 +1688,8 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) return result; } -PyObject * -PyMarshal_WriteObjectToString(PyObject *x, int version) +static PyObject * +_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) { WFILE wf; @@ -1661,6 +1704,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.end = wf.ptr + PyBytes_GET_SIZE(wf.str); wf.error = WFERR_OK; wf.version = version; + wf.allow_code = allow_code; if (w_init_refs(&wf, version)) { Py_DECREF(wf.str); return NULL; @@ -1674,17 +1718,35 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) } if (wf.error != WFERR_OK) { Py_XDECREF(wf.str); - if (wf.error == WFERR_NOMEMORY) + switch (wf.error) { + case WFERR_NOMEMORY: PyErr_NoMemory(); - else + break; + case WFERR_NESTEDTOODEEP: + PyErr_SetString(PyExc_ValueError, + "object too deeply nested to marshal"); + break; + case WFERR_CODE_NOT_ALLOWED: + PyErr_SetString(PyExc_ValueError, + "marshalling code objects is disallowed"); + break; + default: + case WFERR_UNMARSHALLABLE: PyErr_SetString(PyExc_ValueError, - (wf.error==WFERR_UNMARSHALLABLE)?"unmarshallable object" - :"object too deeply nested to marshal"); + "unmarshallable object"); + break; + } return NULL; } return wf.str; } +PyObject * +PyMarshal_WriteObjectToString(PyObject *x, int version) +{ + return _PyMarshal_WriteObjectToString(x, version, 1); +} + /* And an interface for Python programs... */ /*[clinic input] marshal.dump @@ -1696,6 +1758,9 @@ marshal.dump version: int(c_default="Py_MARSHAL_VERSION") = version Indicates the data format that dump should use. / + * + allow_code: bool = True + Allow to write code objects. Write the value on the open file. @@ -1706,14 +1771,14 @@ to the file. The object will not be properly read back by load(). static PyObject * marshal_dump_impl(PyObject *module, PyObject *value, PyObject *file, - int version) -/*[clinic end generated code: output=aaee62c7028a7cb2 input=6c7a3c23c6fef556]*/ + int version, int allow_code) +/*[clinic end generated code: output=429e5fd61c2196b9 input=041f7f6669b0aafb]*/ { /* XXX Quick hack -- need to do this differently */ PyObject *s; PyObject *res; - s = PyMarshal_WriteObjectToString(value, version); + s = _PyMarshal_WriteObjectToString(value, version, allow_code); if (s == NULL) return NULL; res = PyObject_CallMethodOneArg(file, &_Py_ID(write), s); @@ -1727,6 +1792,9 @@ marshal.load file: object Must be readable binary file. / + * + allow_code: bool = True + Allow to load code objects. Read one value from the open file and return it. @@ -1739,8 +1807,8 @@ dump(), load() will substitute None for the unmarshallable type. [clinic start generated code]*/ static PyObject * -marshal_load(PyObject *module, PyObject *file) -/*[clinic end generated code: output=f8e5c33233566344 input=c85c2b594cd8124a]*/ +marshal_load_impl(PyObject *module, PyObject *file, int allow_code) +/*[clinic end generated code: output=0c1aaf3546ae3ed3 input=2dca7b570653b82f]*/ { PyObject *data, *result; RFILE rf; @@ -1762,6 +1830,7 @@ marshal_load(PyObject *module, PyObject *file) result = NULL; } else { + rf.allow_code = allow_code; rf.depth = 0; rf.fp = NULL; rf.readable = file; @@ -1787,6 +1856,9 @@ marshal.dumps version: int(c_default="Py_MARSHAL_VERSION") = version Indicates the data format that dumps should use. / + * + allow_code: bool = True + Allow to write code objects. Return the bytes object that would be written to a file by dump(value, file). @@ -1795,10 +1867,11 @@ unsupported type. [clinic start generated code]*/ static PyObject * -marshal_dumps_impl(PyObject *module, PyObject *value, int version) -/*[clinic end generated code: output=9c200f98d7256cad input=a2139ea8608e9b27]*/ +marshal_dumps_impl(PyObject *module, PyObject *value, int version, + int allow_code) +/*[clinic end generated code: output=115f90da518d1d49 input=167eaecceb63f0a8]*/ { - return PyMarshal_WriteObjectToString(value, version); + return _PyMarshal_WriteObjectToString(value, version, allow_code); } /*[clinic input] @@ -1806,6 +1879,9 @@ marshal.loads bytes: Py_buffer / + * + allow_code: bool = True + Allow to load code objects. Convert the bytes-like object to a value. @@ -1814,13 +1890,14 @@ bytes in the input are ignored. [clinic start generated code]*/ static PyObject * -marshal_loads_impl(PyObject *module, Py_buffer *bytes) -/*[clinic end generated code: output=9fc65985c93d1bb1 input=6f426518459c8495]*/ +marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) +/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ { RFILE rf; char *s = bytes->buf; Py_ssize_t n = bytes->len; PyObject* result; + rf.allow_code = allow_code; rf.fp = NULL; rf.readable = NULL; rf.ptr = s; @@ -1884,6 +1961,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/modsupport.c b/Python/modsupport.c index e9abf304e6502c..0fb7783345c78e 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -306,6 +306,7 @@ do_mkvalue(const char **p_format, va_list *p_va) return PyLong_FromSsize_t(va_arg(*p_va, Py_ssize_t)); #endif /* Fall through from 'n' to 'l' if Py_ssize_t is long */ + _Py_FALLTHROUGH; case 'l': return PyLong_FromLong(va_arg(*p_va, long)); diff --git a/Python/object_stack.c b/Python/object_stack.c new file mode 100644 index 00000000000000..bd9696822c8a9d --- /dev/null +++ b/Python/object_stack.c @@ -0,0 +1,108 @@ +// Stack of Python objects + +#include "Python.h" +#include "pycore_freelist.h" +#include "pycore_pystate.h" +#include "pycore_object_stack.h" + +extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); +extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); + +static struct _Py_object_stack_freelist * +get_object_stack_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->object_stacks; +} + +_PyObjectStackChunk * +_PyObjectStackChunk_New(void) +{ + _PyObjectStackChunk *buf; + struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); + if (obj_stack_freelist->numfree > 0) { + buf = obj_stack_freelist->items; + obj_stack_freelist->items = buf->prev; + obj_stack_freelist->numfree--; + } + else { + // NOTE: we use PyMem_RawMalloc() here because this is used by the GC + // during mimalloc heap traversal. In that context, it is not safe to + // allocate mimalloc memory, such as via PyMem_Malloc(). + buf = PyMem_RawMalloc(sizeof(_PyObjectStackChunk)); + if (buf == NULL) { + return NULL; + } + } + buf->prev = NULL; + buf->n = 0; + return buf; +} + +void +_PyObjectStackChunk_Free(_PyObjectStackChunk *buf) +{ + assert(buf->n == 0); + struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); + if (obj_stack_freelist->numfree >= 0 && + obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) + { + buf->prev = obj_stack_freelist->items; + obj_stack_freelist->items = buf; + obj_stack_freelist->numfree++; + } + else { + PyMem_RawFree(buf); + } +} + +void +_PyObjectStack_Clear(_PyObjectStack *queue) +{ + while (queue->head != NULL) { + _PyObjectStackChunk *buf = queue->head; + buf->n = 0; + queue->head = buf->prev; + _PyObjectStackChunk_Free(buf); + } +} + +void +_PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) +{ + if (src->head == NULL) { + return; + } + + if (dst->head != NULL) { + // First, append dst to the bottom of src + _PyObjectStackChunk *last = src->head; + while (last->prev != NULL) { + last = last->prev; + } + last->prev = dst->head; + } + + // Now that src has all the chunks, set dst to src + dst->head = src->head; + src->head = NULL; +} + +void +_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) +{ + if (!is_finalization) { + // Ignore requests to clear the free list during GC. We use object + // stacks during GC, so emptying the free-list is counterproductive. + return; + } + + struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; + while (freelist->numfree > 0) { + _PyObjectStackChunk *buf = freelist->items; + freelist->items = buf->prev; + freelist->numfree--; + PyMem_RawFree(buf); + } + freelist->numfree = -1; +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index bcd6ea7564f9b3..6097b249c0ad0b 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,10 +1,8 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, - &&TARGET_BEFORE_ASYNC_WITH, - &&TARGET_BEFORE_WITH, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_SLICE, &&TARGET_BINARY_SUBSCR, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -16,13 +14,12 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, - &&TARGET_RESERVED, &&TARGET_GET_ANEXT, &&TARGET_GET_ITER, + &&TARGET_RESERVED, &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_INTERPRETER_EXIT, - &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -82,6 +79,7 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_LIST_EXTEND, &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_COMMON_CONSTANT, &&TARGET_LOAD_CONST, &&TARGET_LOAD_DEREF, &&TARGET_LOAD_FAST, @@ -92,6 +90,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_GLOBAL, &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SPECIAL, &&TARGET_LOAD_SUPER_ATTR, &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, @@ -148,6 +147,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -163,6 +163,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -174,14 +175,17 @@ static void *opcode_targets[256] = { &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_PY_GENERAL, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, + &&TARGET_CONTAINS_OP_DICT, + &&TARGET_CONTAINS_OP_SET, &&TARGET_FOR_ITER_GEN, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, @@ -231,10 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, @@ -254,4 +254,5 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode}; + &&_unknown_opcode, +}; diff --git a/Python/optimizer.c b/Python/optimizer.c index 955ac812177ac4..561ec4efa4ee2a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,19 +1,41 @@ +#ifdef _Py_TIER2 + #include "Python.h" #include "opcode.h" #include "pycore_interp.h" +#include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() -#include "pycore_opcode_metadata.h" // _PyOpcode_OpName() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_uops.h" -#include "cpython/optimizer.h" +#include "pycore_uop_ids.h" +#include "pycore_jit.h" #include #include #include +#define NEED_OPCODE_METADATA +#include "pycore_uop_metadata.h" // Uop tables +#undef NEED_OPCODE_METADATA + #define MAX_EXECUTORS_SIZE 256 +#ifdef Py_DEBUG +static int +base_opcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_GetBaseOpcode(code, offset); + if (opcode == ENTER_EXECUTOR) { + int oparg = _PyCode_CODE(code)[offset].op.arg; + _PyExecutorObject *ex = code->co_executors->executors[oparg]; + return ex->vm_data.opcode; + } + return opcode; +} +#endif + static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { @@ -65,53 +87,37 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _PyExecutorObject *old = code->co_executors->executors[index]; - executor->vm_data.opcode = old->vm_data.opcode; - executor->vm_data.oparg = old->vm_data.oparg; - old->vm_data.opcode = 0; - code->co_executors->executors[index] = executor; - Py_DECREF(old); + _Py_ExecutorDetach(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); assert(code->co_executors->capacity > index); - executor->vm_data.opcode = instr->op.code; - executor->vm_data.oparg = instr->op.arg; - code->co_executors->executors[index] = executor; - assert(index < MAX_EXECUTORS_SIZE); - instr->op.code = ENTER_EXECUTOR; - instr->op.arg = index; code->co_executors->size++; } - return; + executor->vm_data.opcode = instr->op.code; + executor->vm_data.oparg = instr->op.arg; + executor->vm_data.code = code; + executor->vm_data.index = (int)(instr - _PyCode_CODE(code)); + code->co_executors->executors[index] = executor; + assert(index < MAX_EXECUTORS_SIZE); + instr->op.code = ENTER_EXECUTOR; + instr->op.arg = index; } -int -PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *new) -{ - if (instr->op.code != ENTER_EXECUTOR) { - PyErr_Format(PyExc_ValueError, "No executor to replace"); - return -1; - } - int index = instr->op.arg; - assert(index >= 0); - insert_executor(code, instr, index, new); - return 0; -} static int -error_optimize( +never_optimize( _PyOptimizerObject* self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return -1; + // This may be called if the optimizer is reset + return 0; } -static PyTypeObject DefaultOptimizer_Type = { +PyTypeObject _PyDefaultOptimizer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "noop_optimizer", .tp_basicsize = sizeof(_PyOptimizerObject), @@ -119,59 +125,72 @@ static PyTypeObject DefaultOptimizer_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, }; -_PyOptimizerObject _PyOptimizer_Default = { - PyObject_HEAD_INIT(&DefaultOptimizer_Type) - .optimize = error_optimize, - .resume_threshold = UINT16_MAX, - .backedge_threshold = UINT16_MAX, +static _PyOptimizerObject _PyOptimizer_Default = { + PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) + .optimize = never_optimize, }; _PyOptimizerObject * -PyUnstable_GetOptimizer(void) +_Py_GetOptimizer(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->optimizer == &_PyOptimizer_Default) { return NULL; } - assert(interp->optimizer_backedge_threshold == interp->optimizer->backedge_threshold); - assert(interp->optimizer_resume_threshold == interp->optimizer->resume_threshold); Py_INCREF(interp->optimizer); return interp->optimizer; } -void -PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) +static _PyExecutorObject * +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); + +static const _PyBloomFilter EMPTY_FILTER = { 0 }; + +_PyOptimizerObject * +_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { - PyInterpreterState *interp = _PyInterpreterState_GET(); if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } _PyOptimizerObject *old = interp->optimizer; + if (old == NULL) { + old = &_PyOptimizer_Default; + } Py_INCREF(optimizer); interp->optimizer = optimizer; - interp->optimizer_backedge_threshold = optimizer->backedge_threshold; - interp->optimizer_resume_threshold = optimizer->resume_threshold; - Py_DECREF(old); + return old; } int -_PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer) +_Py_SetTier2Optimizer(_PyOptimizerObject *optimizer) { - assert(src->op.code == JUMP_BACKWARD); - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer); + Py_XDECREF(old); + return old == NULL ? -1 : 0; +} + +/* Returns 1 if optimized, 0 if not optimized, and -1 for an error. + * If optimized, *executor_ptr contains a new reference to the executor + */ +int +_PyOptimizer_Optimize( + _PyInterpreterFrame *frame, _Py_CODEUNIT *start, + _PyStackRef *stack_pointer, _PyExecutorObject **executor_ptr) +{ + PyCodeObject *code = _PyFrame_GetCode(frame); assert(PyCode_Check(code)); PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_space_for_executor(code, src)) { + if (!has_space_for_executor(code, start)) { return 0; } _PyOptimizerObject *opt = interp->optimizer; - _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { - assert(executor == NULL); return err; } - int index = get_index_for_executor(code, src); + assert(*executor_ptr != NULL); + int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the * error will show up elsewhere. @@ -179,16 +198,16 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(executor); + Py_DECREF(*executor_ptr); return 0; } - insert_executor(code, src, index, executor); - Py_DECREF(executor); + insert_executor(code, start, index, *executor_ptr); + assert((*executor_ptr)->vm_data.valid); return 1; } _PyExecutorObject * -PyUnstable_GetExecutor(PyCodeObject *code, int offset) +_Py_GetExecutor(PyCodeObject *code, int offset) { int code_len = (int)Py_SIZE(code); for (int i = 0 ; i < code_len;) { @@ -204,131 +223,106 @@ PyUnstable_GetExecutor(PyCodeObject *code, int offset) return NULL; } -/** Test support **/ - - -typedef struct { - _PyOptimizerObject base; - int64_t count; -} _PyCounterOptimizerObject; - -typedef struct { - _PyExecutorObject executor; - _PyCounterOptimizerObject *optimizer; - _Py_CODEUNIT *next_instr; -} _PyCounterExecutorObject; - -static void -counter_dealloc(_PyCounterExecutorObject *self) { - Py_DECREF(self->optimizer); - PyObject_Free(self); -} - -static PyTypeObject CounterExecutor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "counting_executor", - .tp_basicsize = sizeof(_PyCounterExecutorObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_dealloc = (destructor)counter_dealloc, -}; - -static _PyInterpreterFrame * -counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject **stack_pointer) +static PyObject * +is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) { - ((_PyCounterExecutorObject *)self)->optimizer->count++; - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = ((_PyCounterExecutorObject *)self)->next_instr - 1; - Py_DECREF(self); - return frame; + return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); } -static int -counter_optimize( - _PyOptimizerObject* self, - PyCodeObject *code, - _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, - int Py_UNUSED(curr_stackentries) -) +static PyObject * +get_opcode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - _PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&CounterExecutor_Type); - if (executor == NULL) { - return -1; - } - executor->executor.execute = counter_execute; - Py_INCREF(self); - executor->optimizer = (_PyCounterOptimizerObject *)self; - executor->next_instr = instr; - *exec_ptr = (_PyExecutorObject *)executor; - return 1; + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.opcode); } static PyObject * -counter_get_counter(PyObject *self, PyObject *args) +get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg); } -static PyMethodDef counter_methods[] = { - { "get_count", counter_get_counter, METH_NOARGS, NULL }, +static PyMethodDef executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, { NULL, NULL }, }; -static PyTypeObject CounterOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "Counter optimizer", - .tp_basicsize = sizeof(_PyCounterOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_methods = counter_methods, - .tp_dealloc = (destructor)PyObject_Del, -}; - -PyObject * -PyUnstable_Optimizer_NewCounter(void) -{ - _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&CounterOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->base.optimize = counter_optimize; - opt->base.resume_threshold = UINT16_MAX; - opt->base.backedge_threshold = 0; - opt->count = 0; - return (PyObject *)opt; -} - ///////////////////// Experimental UOp Optimizer ///////////////////// +static int executor_clear(_PyExecutorObject *executor); +static void unlink_executor(_PyExecutorObject *executor); + static void -uop_dealloc(_PyUOpExecutorObject *self) { - PyObject_Free(self); +uop_dealloc(_PyExecutorObject *self) { + _PyObject_GC_UNTRACK(self); + assert(self->vm_data.code == NULL); + unlink_executor(self); +#ifdef _Py_JIT + _PyJIT_Free(self); +#endif + PyObject_GC_Del(self); } -static const char * -uop_name(int index) { - if (index <= MAX_REAL_OPCODE) { - return _PyOpcode_OpName[index]; +const char * +_PyUOpName(int index) +{ + if (index < 0 || index > MAX_UOP_ID) { + return NULL; } return _PyOpcode_uop_name[index]; } +#ifdef Py_DEBUG +void +_PyUOpPrint(const _PyUOpInstruction *uop) +{ + const char *name = _PyUOpName(uop->opcode); + if (name == NULL) { + printf("", uop->opcode); + } + else { + printf("%s", name); + } + switch(uop->format) { + case UOP_FORMAT_TARGET: + printf(" (%d, target=%d, operand=%#" PRIx64, + uop->oparg, + uop->target, + (uint64_t)uop->operand); + break; + case UOP_FORMAT_JUMP: + printf(" (%d, jump_target=%d, operand=%#" PRIx64, + uop->oparg, + uop->jump_target, + (uint64_t)uop->operand); + break; + default: + printf(" (%d, Unknown format)", uop->oparg); + } + if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) { + printf(", error_target=%d", uop->error_target); + } + + printf(")"); +} +#endif + static Py_ssize_t -uop_len(_PyUOpExecutorObject *self) +uop_len(_PyExecutorObject *self) { - return Py_SIZE(self); + return self->code_size; } static PyObject * -uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) +uop_item(_PyExecutorObject *self, Py_ssize_t index) { Py_ssize_t len = uop_len(self); if (index < 0 || index >= len) { PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = uop_name(self->trace[index].opcode); + const char *name = _PyUOpName(self->trace[index].opcode); if (name == NULL) { name = ""; } @@ -341,14 +335,21 @@ uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } + PyObject *target = PyLong_FromUnsignedLong(self->trace[index].target); + if (oparg == NULL) { + Py_DECREF(oparg); + Py_DECREF(oname); + return NULL; + } PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); if (operand == NULL) { + Py_DECREF(target); Py_DECREF(oparg); Py_DECREF(oname); return NULL; } - PyObject *args[3] = { oname, oparg, operand }; - return _PyTuple_FromArraySteal(args, 3); + PyObject *args[4] = { oname, oparg, target, operand }; + return _PyTuple_FromArraySteal(args, 4); } PySequenceMethods uop_as_sequence = { @@ -356,69 +357,91 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; -static PyTypeObject UOpExecutor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "uop_executor", - .tp_basicsize = sizeof(_PyUOpExecutorObject) - sizeof(_PyUOpInstruction), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_dealloc = (destructor)uop_dealloc, - .tp_as_sequence = &uop_as_sequence, -}; - static int -move_stubs( - _PyUOpInstruction *trace, - int trace_length, - int stubs_start, - int stubs_end -) +executor_traverse(PyObject *o, visitproc visit, void *arg) { - memmove(trace + trace_length, - trace + stubs_start, - (stubs_end - stubs_start) * sizeof(_PyUOpInstruction)); - // Patch up the jump targets - for (int i = 0; i < trace_length; i++) { - if (trace[i].opcode == _POP_JUMP_IF_FALSE || - trace[i].opcode == _POP_JUMP_IF_TRUE) - { - int target = trace[i].oparg; - if (target >= stubs_start) { - target += trace_length - stubs_start; - trace[i].oparg = target; - } - } + _PyExecutorObject *executor = (_PyExecutorObject *)o; + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_VISIT(executor->exits[i].executor); + } + return 0; +} + +static PyObject * +get_jit_code(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifndef _Py_JIT + PyErr_SetString(PyExc_RuntimeError, "JIT support not enabled."); + return NULL; +#else + _PyExecutorObject *executor = (_PyExecutorObject *)self; + if (executor->jit_code == NULL || executor->jit_size == 0) { + Py_RETURN_NONE; } - return trace_length + stubs_end - stubs_start; + return PyBytes_FromStringAndSize(executor->jit_code, executor->jit_size); +#endif } -#define TRACE_STACK_SIZE 5 +static PyMethodDef uop_executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_jit_code", get_jit_code, METH_NOARGS, NULL}, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, + { NULL, NULL }, +}; static int -translate_bytecode_to_trace( - PyCodeObject *code, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size) +executor_is_gc(PyObject *o) { - PyCodeObject *initial_code = code; - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - int max_length = buffer_size; - int reserved = 0; - struct { - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; + return !_Py_IsImmortal(o); +} -#ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } -#endif +PyTypeObject _PyUOpExecutor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "uop_executor", + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, + .tp_dealloc = (destructor)uop_dealloc, + .tp_as_sequence = &uop_as_sequence, + .tp_methods = uop_executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = (inquiry)executor_clear, + .tp_is_gc = executor_is_gc, +}; + +/* TO DO -- Generate these tables */ +static const uint16_t +_PyUOp_Replacements[MAX_UOP_ID + 1] = { + [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_FOR_ITER] = _FOR_ITER_TIER_TWO, +}; + +static const uint8_t +is_for_iter_test[MAX_UOP_ID + 1] = { + [_GUARD_NOT_EXHAUSTED_RANGE] = 1, + [_GUARD_NOT_EXHAUSTED_LIST] = 1, + [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, + [_FOR_ITER_TIER_TWO] = 1, +}; + +static const uint16_t +BRANCH_TO_GUARD[4][2] = { + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, +}; + + +#define CONFIDENCE_RANGE 1000 +#define CONFIDENCE_CUTOFF 333 #ifdef Py_DEBUG #define DPRINTF(level, ...) \ @@ -427,35 +450,42 @@ translate_bytecode_to_trace( #define DPRINTF(level, ...) #endif -#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND) \ - DPRINTF(2, \ - " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ - uop_name(OPCODE), \ - (OPARG), \ - (uint64_t)(OPERAND)); \ + +static inline int +add_to_trace( + _PyUOpInstruction *trace, + int trace_length, + uint16_t opcode, + uint16_t oparg, + uint64_t operand, + uint32_t target) +{ + trace[trace_length].opcode = opcode; + trace[trace_length].format = UOP_FORMAT_TARGET; + trace[trace_length].target = target; + trace[trace_length].oparg = oparg; + trace[trace_length].operand = operand; + return trace_length + 1; +} + +#ifdef Py_DEBUG +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ + assert(trace_length < max_length); \ + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ + if (lltrace >= 2) { \ + printf("%4d ADD_TO_TRACE: ", trace_length); \ + _PyUOpPrint(&trace[trace_length-1]); \ + printf("\n"); \ + } +#else +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - assert(reserved > 0); \ - reserved--; \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); +#endif #define INSTR_IP(INSTR, CODE) \ ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) -#define ADD_TO_STUB(INDEX, OPCODE, OPARG, OPERAND) \ - DPRINTF(2, " ADD_TO_STUB(%d, %s, %d, %" PRIu64 ")\n", \ - (INDEX), \ - uop_name(OPCODE), \ - (OPARG), \ - (uint64_t)(OPERAND)); \ - assert(reserved > 0); \ - reserved--; \ - trace[(INDEX)].opcode = (OPCODE); \ - trace[(INDEX)].oparg = (OPARG); \ - trace[(INDEX)].operand = (OPERAND); - // Reserve space for n uops #define RESERVE_RAW(n, opname) \ if (trace_length + (n) > max_length) { \ @@ -463,20 +493,21 @@ translate_bytecode_to_trace( (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ goto done; \ - } \ - reserved = (n); // Keep ADD_TO_TRACE / ADD_TO_STUB honest + } -// Reserve space for main+stub uops, plus 2 for _SET_IP and _EXIT_TRACE -#define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 2, uop_name(opcode)) +// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE +#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) +// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) #define TRACE_STACK_PUSH() \ if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ OPT_STAT_INC(trace_stack_overflow); \ - ADD_TO_TRACE(_SET_IP, 0, 0); \ + trace_length = 0; \ goto done; \ } \ + assert(func == NULL || func->func_code == (PyObject *)code); \ + trace_stack[trace_stack_depth].func = func; \ trace_stack[trace_stack_depth].code = code; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; @@ -485,93 +516,169 @@ translate_bytecode_to_trace( Py_FatalError("Trace stack underflow\n"); \ } \ trace_stack_depth--; \ + func = trace_stack[trace_stack_depth].func; \ code = trace_stack[trace_stack_depth].code; \ + assert(func == NULL || func->func_code == (PyObject *)code); \ instr = trace_stack[trace_stack_depth].instr; - DPRINTF(4, +/* Returns the length of the trace on success, + * 0 if it failed to produce a worthwhile trace, + * and -1 on an error. + */ +static int +translate_bytecode_to_trace( + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + _PyUOpInstruction *trace, + int buffer_size, + _PyBloomFilter *dependencies) +{ + bool progress_needed = true; + PyCodeObject *code = _PyFrame_GetCode(frame); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyFunction_Check(func)); + PyCodeObject *initial_code = code; + _Py_BloomFilter_Add(dependencies, initial_code); + _Py_CODEUNIT *initial_instr = instr; + int trace_length = 0; + // Leave space for possible trailing _EXIT_TRACE + int max_length = buffer_size-2; + struct { + PyFunctionObject *func; + PyCodeObject *code; + _Py_CODEUNIT *instr; + } trace_stack[TRACE_STACK_SIZE]; + int trace_stack_depth = 0; + int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions + +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } +#endif + + DPRINTF(2, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + uint32_t target = 0; top: // Jump here after _PUSH_FRAME or likely branches for (;;) { - RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP and _EXIT_TRACE - ADD_TO_TRACE(_SET_IP, INSTR_IP(instr, code), 0); + target = INSTR_IP(instr, code); + // Need space for _DEOPT + max_length--; uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extras = 0; - while (opcode == EXTENDED_ARG) { - instr++; - extras += 1; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - } + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *executor = - (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + assert(oparg < 256); + _PyExecutorObject *executor = code->co_executors->executors[oparg]; opcode = executor->vm_data.opcode; DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); - oparg = (oparg & 0xffffff00) | executor->vm_data.oparg; + oparg = executor->vm_data.oparg; } - switch (opcode) { - - case POP_JUMP_IF_NONE: - { - RESERVE(2, 2); - ADD_TO_TRACE(_IS_NONE, 0, 0); - opcode = POP_JUMP_IF_TRUE; - goto pop_jump_if_bool; + if (opcode == EXTENDED_ARG) { + instr++; + opcode = instr->op.code; + oparg = (oparg << 8) | instr->op.arg; + if (opcode == EXTENDED_ARG) { + instr--; + goto done; } - - case POP_JUMP_IF_NOT_NONE: - { - RESERVE(2, 2); - ADD_TO_TRACE(_IS_NONE, 0, 0); - opcode = POP_JUMP_IF_FALSE; - goto pop_jump_if_bool; + } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); + ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed) { + progress_needed = false; + if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) { + instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg; + initial_instr = instr; + if (opcode == JUMP_BACKWARD) { + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + } + continue; } + else { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + } + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for exit code + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error code + max_length--; + } + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { -pop_jump_if_bool: - RESERVE(1, 2); - max_length -= 2; // Really the start of the stubs + RESERVE(1); int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); - bool jump_likely = bitcount > 8; - bool jump_sense = opcode == POP_JUMP_IF_TRUE; - uint32_t uopcode = jump_sense ^ jump_likely ? - _POP_JUMP_IF_TRUE : _POP_JUMP_IF_FALSE; + int jump_likely = bitcount > 8; + /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. + If it's 16 or 0 (all or none were taken), adjust by 10% + (since the future is still somewhat uncertain). + For values in between, adjust proportionally. */ + if (jump_likely) { + confidence = confidence * (bitcount + 2) / 20; + } + else { + confidence = confidence * (18 - bitcount) / 20; + } + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; + DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + target, _PyOpcode_OpName[opcode], oparg, + counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); + if (confidence < CONFIDENCE_CUTOFF) { + DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); + OPT_STAT_INC(low_confidence); + goto done; + } _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; - _Py_CODEUNIT *stub_target = jump_likely ? next_instr : target_instr; - DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, sense=%d, uopcode=%s\n", - uop_name(opcode), oparg, - counter, bitcount, jump_likely, jump_sense, uop_name(uopcode)); - ADD_TO_TRACE(uopcode, max_length, 0); - ADD_TO_STUB(max_length, _SET_IP, INSTR_IP(stub_target, code), 0); - ADD_TO_STUB(max_length + 1, _EXIT_TRACE, 0, 0); if (jump_likely) { - DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", + DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); goto top; } + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); break; } case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: { - if (instr + 2 - oparg == initial_instr && code == initial_code) { - RESERVE(1, 0); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0); + _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[opcode] - (int)oparg; + if (target == initial_instr) { + /* We have looped round to the start */ + RESERVE(1); + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); } else { OPT_STAT_INC(inner_loop); @@ -582,51 +689,17 @@ translate_bytecode_to_trace( case JUMP_FORWARD: { - RESERVE(0, 0); + RESERVE(0); // This will emit two _SET_IP instructions; leave it to the optimizer instr += oparg; break; } - case FOR_ITER_LIST: - case FOR_ITER_TUPLE: - case FOR_ITER_RANGE: - { - RESERVE(4, 3); - int check_op, exhausted_op, next_op; - switch (opcode) { - case FOR_ITER_LIST: - check_op = _ITER_CHECK_LIST; - exhausted_op = _IS_ITER_EXHAUSTED_LIST; - next_op = _ITER_NEXT_LIST; - break; - case FOR_ITER_TUPLE: - check_op = _ITER_CHECK_TUPLE; - exhausted_op = _IS_ITER_EXHAUSTED_TUPLE; - next_op = _ITER_NEXT_TUPLE; - break; - case FOR_ITER_RANGE: - check_op = _ITER_CHECK_RANGE; - exhausted_op = _IS_ITER_EXHAUSTED_RANGE; - next_op = _ITER_NEXT_RANGE; - break; - default: - Py_UNREACHABLE(); - } - // Assume jump unlikely (can a for-loop exit be likely?) - _Py_CODEUNIT *target_instr = // +1 at the end skips over END_FOR - instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg + 1; - max_length -= 3; // Really the start of the stubs - ADD_TO_TRACE(check_op, 0, 0); - ADD_TO_TRACE(exhausted_op, 0, 0); - ADD_TO_TRACE(_POP_JUMP_IF_TRUE, max_length, 0); - ADD_TO_TRACE(next_op, 0, 0); - - ADD_TO_STUB(max_length + 0, POP_TOP, 0, 0); - ADD_TO_STUB(max_length + 1, _SET_IP, INSTR_IP(target_instr, code), 0); - ADD_TO_STUB(max_length + 2, _EXIT_TRACE, 0, 0); + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; - } default: { @@ -634,11 +707,12 @@ translate_bytecode_to_trace( if (expansion->nuops > 0) { // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; - RESERVE(nuops, 0); - if (expansion->uops[nuops-1].uop == _POP_FRAME) { + RESERVE(nuops + 1); /* One extra for exit */ + int16_t last_op = expansion->uops[nuops-1].uop; + if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of - // LOAD_CONST + _POP_FRAME. + // LOAD_CONST + _RETURN_VALUE. if (trace_stack_depth == 0) { DPRINTF(2, "Trace stack underflow\n"); OPT_STAT_INC(trace_stack_underflow); @@ -654,15 +728,7 @@ translate_bytecode_to_trace( int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { case OPARG_FULL: - if (extras && OPCODE_HAS_JUMP(opcode)) { - if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { - oparg -= extras; - } - else { - assert(opcode != JUMP_BACKWARD); - oparg += extras; - } - } + assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); break; case OPARG_CACHE_1: operand = read_u16(&instr[offset].cache); @@ -679,11 +745,23 @@ translate_bytecode_to_trace( case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; - case OPARG_SET_IP: // op==_SET_IP; oparg=next instr - oparg = INSTR_IP(instr + offset, code); - uop = _SET_IP; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); +#ifdef Py_DEBUG + { + uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + uint32_t jump_target = next_inst + oparg; + assert(base_opcode(code, jump_target) == END_FOR || + base_opcode(code, jump_target) == INSTRUMENTED_END_FOR); + assert(base_opcode(code, jump_target+1) == POP_TOP); + } +#endif break; - default: fprintf(stderr, "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", @@ -692,9 +770,22 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(uop, oparg, operand); - if (uop == _POP_FRAME) { + + if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { TRACE_STACK_POP(); + /* Set the operand to the function or code object returned to, + * to assist optimization passes. (See _PUSH_FRAME below.) + */ + if (func != NULL) { + operand = (uintptr_t)func; + } + else if (code != NULL) { + operand = (uintptr_t)code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -703,6 +794,7 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); int func_version_offset = @@ -710,10 +802,12 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); - DPRINTF(3, "Function object: %p\n", func); - if (func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + PyCodeObject *new_code = NULL; + PyFunctionObject *new_func = + _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); + DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", + (int)func_version, new_func, new_code); + if (new_code != NULL) { if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -721,7 +815,8 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(_SET_IP, 0, 0); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } if (new_code->co_version != func_version) { @@ -729,13 +824,38 @@ translate_bytecode_to_trace( // Perhaps it may happen again, so don't bother tracing. // TODO: Reason about this -- is it better to bail or not? DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(_SET_IP, 0, 0); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); + goto done; + } + if (opcode == FOR_ITER_GEN) { + DPRINTF(2, "Bailing due to dynamic target\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } // Increment IP to the return address instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); + _Py_BloomFilter_Add(dependencies, new_code); + /* Set the operand to the callee's function or code object, + * to assist optimization passes. + * We prefer setting it to the function (for remove_globals()) + * but if that's not available but the code is available, + * use the code, setting the low bit so the optimizer knows. + */ + if (new_func != NULL) { + operand = (uintptr_t)new_func; + } + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); code = new_code; + func = new_func; instr = _PyCode_CODE(code); DPRINTF(2, "Continuing in %s (%s:%d) at byte offset %d\n", @@ -745,13 +865,18 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } - ADD_TO_TRACE(_SET_IP, 0, 0); + DPRINTF(2, "Bail, new_code == NULL\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } + + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } break; } - DPRINTF(2, "Unsupported opcode %s\n", uop_name(opcode)); + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -769,153 +894,366 @@ translate_bytecode_to_trace( } assert(code == initial_code); // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE - if (trace_length > 3) { - ADD_TO_TRACE(_EXIT_TRACE, 0, 0); - DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d+%d\n", + if (progress_needed || trace_length < 5) { + OPT_STAT_INC(trace_too_short); + DPRINTF(2, + "No trace for %s (%s:%d) at byte offset %d (%s)\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code), - trace_length, - buffer_size - max_length); - if (max_length < buffer_size) { - // There are stubs - if (trace_length < max_length) { - // There's a gap before the stubs - // Move the stubs back to be immediately after the main trace - // (which ends at trace_length) - DPRINTF(2, - "Moving %d stub uops back by %d\n", - buffer_size - max_length, - max_length - trace_length); - trace_length = move_stubs(trace, trace_length, max_length, buffer_size); - } - else { - assert(trace_length == max_length); - // There's no gap - trace_length = buffer_size; - } - } - return trace_length; + progress_needed ? "no progress" : "too short"); + return 0; } - else { - OPT_STAT_INC(trace_too_short); - DPRINTF(4, - "No trace for %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + if (trace[trace_length-1].opcode != _JUMP_TO_TOP) { + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - return 0; + DPRINTF(1, + "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(initial_instr, code), + trace_length); + OPT_HIST(trace_length, trace_length_hist); + return trace_length; +} #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP #undef ADD_TO_TRACE #undef DPRINTF -} +#define UNSET_BIT(array, bit) (array[(bit)>>5] &= ~(1<<((bit)&31))) +#define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31))) +#define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31))) + +/* Count the number of unused uops and exits +*/ static int -remove_unneeded_uops(_PyUOpInstruction *trace, int trace_length) -{ - // Stage 1: Replace unneeded _SET_IP uops with NOP. - // Note that we don't enter stubs, those SET_IPs are needed. - int last_set_ip = -1; - int last_instr = 0; - bool need_ip = true; - for (int pc = 0; pc < trace_length; pc++) { - int opcode = trace[pc].opcode; - if (opcode == _SAVE_CURRENT_IP) { - // Special case: never remove preceding _SET_IP - last_set_ip = -1; +count_exits(_PyUOpInstruction *buffer, int length) +{ + int exit_count = 0; + for (int i = 0; i < length; i++) { + int opcode = buffer[i].opcode; + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { + exit_count++; } - else if (opcode == _SET_IP) { - if (!need_ip && last_set_ip >= 0) { - trace[last_set_ip].opcode = NOP; + } + return exit_count; +} + +static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +{ + inst->opcode = opcode; + inst->oparg = 0; + inst->operand = 0; + inst->format = UOP_FORMAT_TARGET; + inst->target = target; +} + +/* Convert implicit exits, errors and deopts + * into explicit ones. */ +static int +prepare_for_execution(_PyUOpInstruction *buffer, int length) +{ + int32_t current_jump = -1; + int32_t current_jump_target = -1; + int32_t current_error = -1; + int32_t current_error_target = -1; + int32_t current_popped = -1; + int32_t current_exit_op = -1; + /* Leaving in NOPs slows down the interpreter and messes up the stats */ + _PyUOpInstruction *copy_to = &buffer[0]; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + if (inst->opcode != _NOP) { + if (copy_to != inst) { + *copy_to = *inst; } - need_ip = false; - last_set_ip = pc; + copy_to++; } - else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { - last_instr = pc + 1; - break; - } - else { - // If opcode has ERROR or DEOPT, set need_up to true - if (_PyOpcode_opcode_metadata[opcode].flags & (HAS_ERROR_FLAG | HAS_DEOPT_FLAG)) { - need_ip = true; + } + length = (int)(copy_to - buffer); + int next_spare = length; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + int opcode = inst->opcode; + int32_t target = (int32_t)uop_get_target(inst); + if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? + _EXIT_TRACE : _DEOPT; + int32_t jump_target = target; + if (is_for_iter_test[opcode]) { + /* Target the POP_TOP immediately after the END_FOR, + * leaving only the iterator on the stack. */ + int extended_arg = inst->oparg > 255; + int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; + jump_target = next_inst + inst->oparg + 1; } + if (jump_target != current_jump_target || current_exit_op != exit_op) { + make_exit(&buffer[next_spare], exit_op, jump_target); + current_exit_op = exit_op; + current_jump_target = jump_target; + current_jump = next_spare; + next_spare++; + } + buffer[i].jump_target = current_jump; + buffer[i].format = UOP_FORMAT_JUMP; } - } - // Stage 2: Squash NOP opcodes (pre-existing or set above). - int dest = 0; - for (int pc = 0; pc < last_instr; pc++) { - int opcode = trace[pc].opcode; - if (opcode != NOP) { - if (pc != dest) { - trace[dest] = trace[pc]; + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? + 0 : _PyUop_num_popped(opcode, inst->oparg); + if (target != current_error_target || popped != current_popped) { + current_popped = popped; + current_error = next_spare; + current_error_target = target; + make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + buffer[next_spare].oparg = popped; + buffer[next_spare].operand = target; + next_spare++; } - dest++; + buffer[i].error_target = current_error; + if (buffer[i].format == UOP_FORMAT_TARGET) { + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 0; + } + } + if (opcode == _JUMP_TO_TOP) { + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 1; } } - // Stage 3: Move the stubs back. - if (dest < last_instr) { - int new_trace_length = move_stubs(trace, dest, last_instr, trace_length); + return next_spare; +} + +/* Executor side exits */ + +static _PyExecutorObject * +allocate_executor(int exit_count, int length) +{ + int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); + _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); + if (res == NULL) { + return NULL; + } + res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + res->code_size = length; + res->exit_count = exit_count; + return res; +} + #ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + +#define CHECK(PRED) \ +if (!(PRED)) { \ + printf(#PRED " at %d\n", i); \ + assert(0); \ +} + +static int +target_unused(int opcode) +{ + return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0; +} + +static void +sanity_check(_PyExecutorObject *executor) +{ + for (uint32_t i = 0; i < executor->exit_count; i++) { + _PyExitData *exit = &executor->exits[i]; + CHECK(exit->target < (1 << 25)); + } + bool ended = false; + uint32_t i = 0; + CHECK(executor->trace[0].opcode == _START_EXECUTOR); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK(opcode <= MAX_UOP_ID); + CHECK(_PyOpcode_uop_name[opcode] != NULL); + switch(inst->format) { + case UOP_FORMAT_TARGET: + CHECK(target_unused(opcode)); + break; + case UOP_FORMAT_JUMP: + CHECK(inst->jump_target < executor->code_size); + break; } - if (lltrace >= 2) { - printf("Optimized trace (length %d+%d = %d, saved %d):\n", - dest, trace_length - last_instr, new_trace_length, - trace_length - new_trace_length); - for (int pc = 0; pc < new_trace_length; pc++) { - printf("%4d: (%s, %d, %" PRIu64 ")\n", - pc, - uop_name(trace[pc].opcode), - (trace[pc].oparg), - (uint64_t)(trace[pc].operand)); - } + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + CHECK(inst->format == UOP_FORMAT_JUMP); + CHECK(inst->error_target < executor->code_size); } + if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + ended = true; + i++; + break; + } + } + CHECK(ended); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK( + opcode == _DEOPT || + opcode == _EXIT_TRACE || + opcode == _ERROR_POP_N); + } +} + +#undef CHECK #endif - trace_length = new_trace_length; + +/* Makes an executor from a buffer of uops. + * Account for the buffer having gaps and NOPs by computing a "used" + * bit vector and only copying the used uops. Here "used" means reachable + * and not a NOP. + */ +static _PyExecutorObject * +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +{ + int exit_count = count_exits(buffer, length); + _PyExecutorObject *executor = allocate_executor(exit_count, length); + if (executor == NULL) { + return NULL; } - return trace_length; + + /* Initialize exits */ + for (int i = 0; i < exit_count; i++) { + executor->exits[i].executor = NULL; + executor->exits[i].temperature = initial_temperature_backoff_counter(); + } + int next_exit = exit_count-1; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[0].operand = (uint64_t)executor; + for (int i = length-1; i >= 0; i--) { + int opcode = buffer[i].opcode; + dest--; + *dest = buffer[i]; + assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); + if (opcode == _EXIT_TRACE) { + executor->exits[next_exit].target = buffer[i].target; + dest->oparg = next_exit; + next_exit--; + } + if (opcode == _DYNAMIC_EXIT) { + executor->exits[next_exit].target = 0; + dest->oparg = next_exit; + next_exit--; + } + } + assert(next_exit == -1); + assert(dest == executor->trace); + assert(dest->opcode == _START_EXECUTOR); + _Py_ExecutorInit(executor, dependencies); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + if (lltrace >= 2) { + printf("Optimized trace (length %d):\n", length); + for (int i = 0; i < length; i++) { + printf("%4d OPTIMIZED: ", i); + _PyUOpPrint(&executor->trace[i]); + printf("\n"); + } + } + sanity_check(executor); +#endif +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_side_entry = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, length)) { + Py_DECREF(executor); + return NULL; + } +#endif + _PyObject_GC_TRACK(executor); + return executor; +} + +#ifdef Py_STATS +/* Returns the effective trace length. + * Ignores NOPs and trailing exit and error handling.*/ +int effective_trace_length(_PyUOpInstruction *buffer, int length) +{ + int nop_count = 0; + for (int i = 0; i < length; i++) { + int opcode = buffer[i].opcode; + if (opcode == _NOP) { + nop_count++; + } + if (opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP) { + return i+1-nop_count; + } + } + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); } +#endif static int uop_optimize( _PyOptimizerObject *self, - PyCodeObject *code, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries) { - _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; - int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH); - if (trace_length <= 0) { + _PyBloomFilter dependencies; + _Py_BloomFilter_Init(&dependencies); + _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; + OPT_STAT_INC(attempts); + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); + if (length <= 0) { // Error or nothing translated - return trace_length; + return length; } - OPT_HIST(trace_length, trace_length_hist); + assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); - char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); - if (uop_optimize != NULL && *uop_optimize > '0') { - trace_length = _Py_uop_analyze_and_optimize(code, trace, trace_length, curr_stackentries); + char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); + if (env_var == NULL || *env_var == '\0' || *env_var > '0') { + length = _Py_uop_analyze_and_optimize(frame, buffer, + length, + curr_stackentries, &dependencies); + if (length <= 0) { + return length; + } } - trace_length = remove_unneeded_uops(trace, trace_length); - _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); + assert(length < UOP_MAX_TRACE_LENGTH); + assert(length >= 1); + /* Fix up */ + for (int pc = 0; pc < length; pc++) { + int opcode = buffer[pc].opcode; + int oparg = buffer[pc].oparg; + if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { + buffer[pc].opcode = opcode + 1 + (oparg & 1); + } + else if (oparg < _PyUop_Replication[opcode]) { + buffer[pc].opcode = opcode + oparg + 1; + } + else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + break; + } + assert(_PyOpcode_uop_name[buffer[pc].opcode]); + assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); + } + OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); + length = prepare_for_execution(buffer, length); + assert(length <= UOP_MAX_TRACE_LENGTH); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; } - OPT_HIST(trace_length, optimized_trace_length_hist); - executor->base.execute = _PyUopExecute; - memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); - *exec_ptr = (_PyExecutorObject *)executor; + assert(length <= UOP_MAX_TRACE_LENGTH); + *exec_ptr = executor; return 1; } @@ -924,7 +1262,7 @@ uop_opt_dealloc(PyObject *self) { PyObject_Free(self); } -static PyTypeObject UOpOptimizer_Type = { +PyTypeObject _PyUOpOptimizer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_optimizer", .tp_basicsize = sizeof(_PyOptimizerObject), @@ -934,16 +1272,371 @@ static PyTypeObject UOpOptimizer_Type = { }; PyObject * -PyUnstable_Optimizer_NewUOpOptimizer(void) +_PyOptimizer_NewUOpOptimizer(void) { - _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &UOpOptimizer_Type); + _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); if (opt == NULL) { return NULL; } opt->optimize = uop_optimize; - opt->resume_threshold = UINT16_MAX; - // Need at least 3 iterations to settle specializations. - // A few lower bits of the counter are reserved for other flags. - opt->backedge_threshold = 16 << OPTIMIZER_BITS_IN_COUNTER; return (PyObject *)opt; } + +static void +counter_dealloc(_PyExecutorObject *self) { + /* The optimizer is the operand of the second uop. */ + PyObject *opt = (PyObject *)self->trace[1].operand; + Py_DECREF(opt); + uop_dealloc(self); +} + +PyTypeObject _PyCounterExecutor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "counting_executor", + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, + .tp_dealloc = (destructor)counter_dealloc, + .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = (inquiry)executor_clear, +}; + +static int +counter_optimize( + _PyOptimizerObject* self, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + _PyExecutorObject **exec_ptr, + int Py_UNUSED(curr_stackentries) +) +{ + PyCodeObject *code = _PyFrame_GetCode(frame); + int oparg = instr->op.arg; + while (instr->op.code == EXTENDED_ARG) { + instr++; + oparg = (oparg << 8) | instr->op.arg; + } + if (instr->op.code != JUMP_BACKWARD) { + /* Counter optimizer can only handle backward edges */ + return 0; + } + _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; + _PyUOpInstruction buffer[4] = { + { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, + { .opcode = _LOAD_CONST_INLINE, .operand = (uintptr_t)self }, + { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, + { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } + }; + _PyExecutorObject *executor = make_executor_from_uops(buffer, 4, &EMPTY_FILTER); + if (executor == NULL) { + return -1; + } + Py_INCREF(self); + Py_SET_TYPE(executor, &_PyCounterExecutor_Type); + *exec_ptr = executor; + return 1; +} + +static PyObject * +counter_get_counter(PyObject *self, PyObject *args) +{ + return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); +} + +static PyMethodDef counter_optimizer_methods[] = { + { "get_count", counter_get_counter, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + +PyTypeObject _PyCounterOptimizer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "Counter optimizer", + .tp_basicsize = sizeof(_PyCounterOptimizerObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_methods = counter_optimizer_methods, + .tp_dealloc = (destructor)PyObject_Del, +}; + +PyObject * +_PyOptimizer_NewCounter(void) +{ + _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); + if (opt == NULL) { + return NULL; + } + opt->base.optimize = counter_optimize; + opt->count = 0; + return (PyObject *)opt; +} + + +/***************************************** + * Executor management + ****************************************/ + +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigorous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define K 6 + +#define SEED 20221211 + +/* TO DO -- Use more modern hash functions with better distribution of bits */ +static uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +/* We want K hash functions that each set 1 bit. + * A hash function that sets 1 bit in M bits can be trivially + * derived from a log2(M) bit hash function. + * So we extract 8 (log2(256)) bits at a time from + * the 64bit hash. */ +void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(K <= 8); + for (int i = 0; i < K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> 5] |= (1 << (bits&31)); + hash >>= 8; + } +} + +static bool +bloom_filter_may_contain(_PyBloomFilter *bloom, _PyBloomFilter *hashes) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + +static void +link_executor(_PyExecutorObject *executor) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyExecutorLinkListNode *links = &executor->vm_data.links; + _PyExecutorObject *head = interp->executor_list_head; + if (head == NULL) { + interp->executor_list_head = executor; + links->previous = NULL; + links->next = NULL; + } + else { + assert(head->vm_data.links.previous == NULL); + links->previous = NULL; + links->next = head; + head->vm_data.links.previous = executor; + interp->executor_list_head = executor; + } + executor->vm_data.linked = true; + /* executor_list_head must be first in list */ + assert(interp->executor_list_head->vm_data.links.previous == NULL); +} + +static void +unlink_executor(_PyExecutorObject *executor) +{ + if (!executor->vm_data.linked) { + return; + } + _PyExecutorLinkListNode *links = &executor->vm_data.links; + assert(executor->vm_data.valid); + _PyExecutorObject *next = links->next; + _PyExecutorObject *prev = links->previous; + if (next != NULL) { + next->vm_data.links.previous = prev; + } + if (prev != NULL) { + prev->vm_data.links.next = next; + } + else { + // prev == NULL implies that executor is the list head + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp->executor_list_head == executor); + interp->executor_list_head = next; + } + executor->vm_data.linked = false; +} + +/* This must be called by optimizers before using the executor */ +void +_Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) +{ + executor->vm_data.valid = true; + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; + } + link_executor(executor); +} + +/* Detaches the executor from the code object (if any) that + * holds a reference to it */ +void +_Py_ExecutorDetach(_PyExecutorObject *executor) +{ + PyCodeObject *code = executor->vm_data.code; + if (code == NULL) { + return; + } + _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; + assert(instruction->op.code == ENTER_EXECUTOR); + int index = instruction->op.arg; + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; + executor->vm_data.code = NULL; + code->co_executors->executors[index] = NULL; + Py_DECREF(executor); +} + +static int +executor_clear(_PyExecutorObject *executor) +{ + if (!executor->vm_data.valid) { + return 0; + } + assert(executor->vm_data.valid == 1); + unlink_executor(executor); + executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference + * cycle with itself, so decref'ing a side exit could + * free the executor unless we hold a strong reference to it + */ + Py_INCREF(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + executor->exits[i].temperature = initial_unreachable_backoff_counter(); + Py_CLEAR(executor->exits[i].executor); + } + _Py_ExecutorDetach(executor); + Py_DECREF(executor); + return 0; +} + +void +_Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) +{ + assert(executor->vm_data.valid); + _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); +} + +/* Invalidate all executors that depend on `obj` + * May cause other executors to be invalidated as well + */ +void +_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + bool no_memory = false; + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + PyErr_Clear(); + no_memory = true; + } + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { + unlink_executor(exec); + if (no_memory) { + exec->vm_data.valid = 0; + } else { + if (PyList_Append(invalidate, (PyObject *)exec) < 0) { + PyErr_Clear(); + no_memory = true; + exec->vm_data.valid = 0; + } + } + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } + } + exec = next; + } + if (invalidate != NULL) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + } + return; +} + +/* Invalidate all executors */ +void +_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) +{ + while (interp->executor_list_head) { + _PyExecutorObject *executor = interp->executor_list_head; + assert(executor->vm_data.valid == 1 && executor->vm_data.linked == 1); + if (executor->vm_data.code) { + // Clear the entire code object so its co_executors array be freed: + _PyCode_Clear_Executors(executor->vm_data.code); + } + else { + executor_clear(executor); + } + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } + } +} + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 2d177f14ff268b..8c866417478128 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,25 +1,615 @@ +#ifdef _Py_TIER2 + +/* + * This file contains the support code for CPython's uops optimizer. + * It also performs some simple optimizations. + * It performs a traditional data-flow analysis[1] over the trace of uops. + * Using the information gained, it chooses to emit, or skip certain instructions + * if possible. + * + * [1] For information on data-flow analysis, please see + * https://clang.llvm.org/docs/DataFlowAnalysisIntro.html + * + * */ #include "Python.h" #include "opcode.h" +#include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_uops.h" +#include "pycore_uop_metadata.h" +#include "pycore_dict.h" #include "pycore_long.h" -#include "cpython/optimizer.h" +#include "pycore_optimizer.h" +#include "pycore_object.h" +#include "pycore_dict.h" +#include "pycore_function.h" +#include "pycore_uop_metadata.h" +#include "pycore_uop_ids.h" +#include "pycore_range.h" + +#include #include #include #include -#include "pycore_optimizer.h" +#ifdef Py_DEBUG + extern const char *_PyUOpName(int index); + extern void _PyUOpPrint(const _PyUOpInstruction *uop); + static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; + static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV(DEBUG_ENV); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; + } + #define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif -int -_Py_uop_analyze_and_optimize( + + +static inline bool +op_is_end(uint32_t opcode) +{ + return opcode == _EXIT_TRACE || opcode == _JUMP_TO_TOP; +} + +static int +get_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); +} + +static void +increment_mutations(PyObject* dict) { + assert(PyDict_CheckExact(dict)); + PyDictObject *d = (PyDictObject *)dict; + d->ma_version_tag += (1 << DICT_MAX_WATCHERS); +} + +/* The first two dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 +#define TYPE_WATCHER_ID 0 + +static int +globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, + PyObject* key, PyObject* new_value) +{ + RARE_EVENT_STAT_INC(watched_globals_modification); + assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1); + increment_mutations(dict); + PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); + return 0; +} + +static int +type_watcher_callback(PyTypeObject* type) +{ + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), type, 1); + PyType_Unwatch(TYPE_WATCHER_ID, (PyObject *)type); + return 0; +} + +static PyObject * +convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) +{ + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); + assert(PyDict_CheckExact(obj)); + PyDictObject *dict = (PyDictObject *)obj; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + assert(inst->operand <= UINT16_MAX); + if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + return NULL; + } + PyObject *res = entries[inst->operand].me_value; + if (res == NULL) { + return NULL; + } + if (_Py_IsImmortal(res)) { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + } + else { + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + } + inst->operand = (uint64_t)res; + return res; +} + +static int +incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +{ + if (!PyDict_CheckExact(obj)) { + return 1; + } + PyDictObject *dict = (PyDictObject *)obj; + if (dict->ma_keys->dk_version != inst->operand) { + return 1; + } + return 0; +} + +/* Returns 1 if successfully optimized + * 0 if the trace is not suitable for optimization (yet) + * -1 if there was an error. */ +static int +remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, + int buffer_size, _PyBloomFilter *dependencies) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = frame->f_builtins; + if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); + return 1; + } + PyObject *globals = frame->f_globals; + PyFunctionObject *function = (PyFunctionObject *)frame->f_funcobj; + assert(PyFunction_Check(function)); + assert(function->func_builtins == builtins); + assert(function->func_globals == globals); + uint32_t function_version = _PyFunction_GetVersionForCurrentState(function); + /* In order to treat globals as constants, we need to + * know that the globals dict is the one we expected, and + * that it hasn't changed + * In order to treat builtins as constants, we need to + * know that the builtins dict is the one we expected, and + * that it hasn't changed and that the global dictionary's + * keys have not changed */ + + /* These values represent stacks of booleans (one bool per bit). + * Pushing a frame shifts left, popping a frame shifts right. */ + uint32_t function_checked = 0; + uint32_t builtins_watched = 0; + uint32_t globals_watched = 0; + uint32_t prechecked_function_version = 0; + if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { + interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; + } + if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } + for (int pc = 0; pc < buffer_size; pc++) { + _PyUOpInstruction *inst = &buffer[pc]; + int opcode = inst->opcode; + switch(opcode) { + case _GUARD_BUILTINS_VERSION: + if (incorrect_keys(inst, builtins)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + return 0; + } + if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + continue; + } + if ((builtins_watched & 1) == 0) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + builtins_watched |= 1; + } + if (function_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_FUNCTION; + buffer[pc].operand = function_version; + function_checked |= 1; + } + break; + case _GUARD_GLOBALS_VERSION: + if (incorrect_keys(inst, globals)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + return 0; + } + uint64_t watched_mutations = get_mutations(globals); + if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + continue; + } + if ((globals_watched & 1) == 0) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + globals_watched |= 1; + } + if (function_checked & 1) { + buffer[pc].opcode = NOP; + } + else { + buffer[pc].opcode = _CHECK_FUNCTION; + buffer[pc].operand = function_version; + function_checked |= 1; + } + break; + case _LOAD_GLOBAL_BUILTINS: + if (function_checked & globals_watched & builtins_watched & 1) { + convert_global_to_const(inst, builtins); + } + break; + case _LOAD_GLOBAL_MODULE: + if (function_checked & globals_watched & 1) { + convert_global_to_const(inst, globals); + } + break; + case _PUSH_FRAME: + { + builtins_watched <<= 1; + globals_watched <<= 1; + function_checked <<= 1; + uint64_t operand = buffer[pc].operand; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; + if (func == NULL) { + return 1; + } + assert(PyFunction_Check(func)); + function_version = func->func_version; + if (prechecked_function_version == function_version) { + function_checked |= 1; + } + prechecked_function_version = 0; + globals = func->func_globals; + builtins = func->func_builtins; + if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); + return 1; + } + break; + } + case _RETURN_VALUE: + { + builtins_watched >>= 1; + globals_watched >>= 1; + function_checked >>= 1; + uint64_t operand = buffer[pc].operand; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; + if (func == NULL) { + return 1; + } + assert(PyFunction_Check(func)); + function_version = func->func_version; + globals = func->func_globals; + builtins = func->func_builtins; + break; + } + case _CHECK_FUNCTION_EXACT_ARGS: + prechecked_function_version = (uint32_t)buffer[pc].operand; + break; + default: + if (op_is_end(opcode)) { + return 1; + } + break; + } + } + return 0; +} + + + +#define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) +#define STACK_SIZE() ((int)(ctx->frame->stack_len)) + +#define WITHIN_STACK_BOUNDS() \ + (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + + +#define GETLOCAL(idx) ((ctx->frame->locals[idx])) + +#define REPLACE_OP(INST, OP, ARG, OPERAND) \ + INST->opcode = OP; \ + INST->oparg = ARG; \ + INST->operand = OPERAND; + +/* Shortened forms for convenience, used in optimizer_bytecodes.c */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_has_type _Py_uop_sym_has_type +#define sym_get_type _Py_uop_sym_get_type +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define sym_truthiness _Py_uop_sym_truthiness +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +static int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr) +{ + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + *result_ptr = value; + return 1; + } + int truthiness = sym_truthiness(value); + if (truthiness >= 0) { + PyObject *load = truthiness ? Py_True : Py_False; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + *result_ptr = sym_new_const(ctx, load); + return 1; + } + return 0; +} + +static void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +{ + REPLACE_OP(this_instr, _POP_TOP, 0, 0); + if (exit) { + REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); + this_instr[1].target = this_instr->target; + } +} + +/* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a + * PyCodeObject *. Retrieve the code object if possible. + */ +static PyCodeObject * +get_code(_PyUOpInstruction *op) +{ + assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); + PyCodeObject *co = NULL; + uint64_t operand = op->operand; + if (operand == 0) { + return NULL; + } + if (operand & 1) { + co = (PyCodeObject *)(operand & ~1); + } + else { + PyFunctionObject *func = (PyFunctionObject *)operand; + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + assert(PyCode_Check(co)); + return co; +} + +/* 1 for success, 0 for not ready, cannot error at the moment. */ +static int +optimize_uops( PyCodeObject *co, _PyUOpInstruction *trace, int trace_len, - int curr_stacklen + int curr_stacklen, + _PyBloomFilter *dependencies ) { + + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + uint32_t opcode = UINT16_MAX; + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; + + _Py_uop_abstractcontext_init(ctx); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; + + _PyUOpInstruction *this_instr = NULL; + for (int i = 0; !ctx->done; i++) { + assert(i < trace_len); + this_instr = &trace[i]; + + int oparg = this_instr->oparg; + opcode = this_instr->opcode; + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d abs: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + + switch (opcode) { + +#include "optimizer_cases.c.h" + + default: + DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); + Py_UNREACHABLE(); + } + assert(ctx->frame != NULL); + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + } + + /* Either reached the end or cannot optimize further, but there + * would be no benefit in retrying later */ + _Py_uop_abstractcontext_fini(ctx); + if (first_valid_check_stack != NULL) { + assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); + assert(max_space > 0); + assert(max_space <= INT_MAX); + assert(max_space <= INT32_MAX); + first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; + first_valid_check_stack->operand = max_space; + } return trace_len; + +error: + DPRINTF(3, "\n"); + DPRINTF(1, "Encountered error in abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } + _Py_uop_abstractcontext_fini(ctx); + return -1; + +} + + +static int +remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) +{ + /* Remove _SET_IP and _CHECK_VALIDITY where possible. + * _SET_IP is needed if the following instruction escapes or + * could error. _CHECK_VALIDITY is needed if the previous + * instruction could have escaped. */ + int last_set_ip = -1; + bool may_have_escaped = true; + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch (opcode) { + case _START_EXECUTOR: + may_have_escaped = false; + break; + case _SET_IP: + buffer[pc].opcode = _NOP; + last_set_ip = pc; + break; + case _CHECK_VALIDITY: + if (may_have_escaped) { + may_have_escaped = false; + } + else { + buffer[pc].opcode = _NOP; + } + break; + case _CHECK_VALIDITY_AND_SET_IP: + if (may_have_escaped) { + may_have_escaped = false; + buffer[pc].opcode = _CHECK_VALIDITY; + } + else { + buffer[pc].opcode = _NOP; + } + last_set_ip = pc; + break; + case _POP_TOP: + { + _PyUOpInstruction *last = &buffer[pc-1]; + while (last->opcode == _NOP) { + last--; + } + if (last->opcode == _LOAD_CONST_INLINE || + last->opcode == _LOAD_CONST_INLINE_BORROW || + last->opcode == _LOAD_FAST || + last->opcode == _COPY + ) { + last->opcode = _NOP; + buffer[pc].opcode = _NOP; + } + if (last->opcode == _REPLACE_WITH_TRUE) { + last->opcode = _NOP; + } + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return pc + 1; + default: + { + /* _PUSH_FRAME doesn't escape or error, but it + * does need the IP for the return address */ + bool needs_ip = opcode == _PUSH_FRAME; + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { + needs_ip = true; + may_have_escaped = true; + } + if (needs_ip && last_set_ip >= 0) { + if (buffer[last_set_ip].opcode == _CHECK_VALIDITY) { + buffer[last_set_ip].opcode = _CHECK_VALIDITY_AND_SET_IP; + } + else { + assert(buffer[last_set_ip].opcode == _NOP); + buffer[last_set_ip].opcode = _SET_IP; + } + last_set_ip = -1; + } + } + } + } + Py_UNREACHABLE(); } + +// 0 - failure, no error raised, just fall back to Tier 1 +// -1 - failure, and raise error +// > 0 - length of optimized trace +int +_Py_uop_analyze_and_optimize( + _PyInterpreterFrame *frame, + _PyUOpInstruction *buffer, + int length, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + OPT_STAT_INC(optimizer_attempts); + + int err = remove_globals(frame, buffer, length, dependencies); + if (err <= 0) { + return err; + } + + length = optimize_uops( + _PyFrame_GetCode(frame), buffer, + length, curr_stacklen, dependencies); + + if (length <= 0) { + return length; + } + + length = remove_unneeded_uops(buffer, length); + assert(length > 0); + + OPT_STAT_INC(optimizer_successes); + return length; +} + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c new file mode 100644 index 00000000000000..2ea839f5d6dc97 --- /dev/null +++ b/Python/optimizer_bytecodes.c @@ -0,0 +1,792 @@ +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); + +extern PyCodeObject *get_code(_PyUOpInstruction *op); + +static int +dummy_func(void) { + + PyCodeObject *co; + int oparg; + _Py_UopsSymbol *flag; + _Py_UopsSymbol *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame *new_frame; + _Py_UOpsContext *ctx; + _PyUOpInstruction *this_instr; + _PyBloomFilter *dependencies; + int modified; + int curr_space; + int max_space; + _PyUOpInstruction *first_valid_check_stack; + _PyUOpInstruction *corresponding_check_stack; + +// BEGIN BYTECODES // + + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + } + + op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + } + + op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp = sym_new_null(ctx); + GETLOCAL(oparg) = temp; + } + + op(_STORE_FAST, (value --)) { + GETLOCAL(oparg) = value; + } + + op(_PUSH_NULL, (-- res)) { + res = sym_new_null(ctx); + } + + op(_GUARD_BOTH_INT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + } + + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + + } + } + + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } + } + + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + } + + op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + } + + op(_BINARY_OP, (left, right -- res)) { + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + res = sym_new_type(ctx, &PyLong_Type); + } + else { + /* For any other op combining ints/floats the result is a float */ + res = sym_new_type(ctx, &PyFloat_Type); + } + } + res = sym_new_unknown(ctx); + } + + op(_BINARY_OP_ADD_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + } + + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + } + + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + } + + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + } + + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + } + + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + } + + op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + } + + op(_TO_BOOL, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + } + } + + op(_TO_BOOL_BOOL, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; + } + } + + op(_TO_BOOL_INT, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + } + + op(_TO_BOOL_LIST, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + } + + op(_TO_BOOL_NONE, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); + } + } + + op(_TO_BOOL_STR, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); + } + } + + op(_COMPARE_OP, (left, right -- res)) { + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + res = _Py_uop_sym_new_not_null(ctx); + } + } + + op(_COMPARE_OP_INT, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_COMPARE_OP_FLOAT, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_COMPARE_OP_STR, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_IS_OP, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_CONTAINS_OP, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_LOAD_CONST, (-- value)) { + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + } + + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + assert(oparg > 0); + top = bottom; + } + + op(_SWAP, (bottom, unused[oparg-2], top -- + top, unused[oparg-2], bottom)) { + } + + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)index; + (void)owner; + } + + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + } + + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg & 1) { + self_or_null = sym_new_unknown(ctx); + } + } + + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + (void)index; + null = sym_new_null(ctx); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + attr = sym_new_const(ctx, res); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)hint; + (void)owner; + } + + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)index; + (void)owner; + } + + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)descr; + (void)owner; + } + + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + } + + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + } + + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + } + + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + (void)callable; + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; + } + + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + int argcount = oparg; + + (void)callable; + + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uint64_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + ctx->done = true; + break; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + + assert(self_or_null != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + + } + } + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + new_frame = NULL; + ctx->done = true; + } + + op(_RETURN_VALUE, (retval -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_YIELD_VALUE, (unused -- res)) { + res = sym_new_unknown(ctx); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + ctx->done = true; + } + + op(_CHECK_STACK_SPACE, ( --)) { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; + } + + op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { + (void)framesize; + /* We should never see _CHECK_STACK_SPACE_OPERANDs. + * They are only created at the end of this pass. */ + Py_UNREACHABLE(); + } + + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; + } + + op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + next = sym_new_type(ctx, &PyLong_Type); + (void)iter; + } + + op(_GUARD_IS_TRUE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } + } + + op(_GUARD_IS_FALSE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } + } + + op(_GUARD_IS_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } + } + + op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } + } + + op(_CHECK_PEP_523, (--)) { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + + op(_LOAD_SPECIAL, (owner -- attr, self_or_null)) { + (void)owner; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); + } + + op(_JUMP_TO_TOP, (--)) { + ctx->done = true; + } + + op(_EXIT_TRACE, (--)) { + ctx->done = true; + } + +// END BYTECODES // + +} diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h new file mode 100644 index 00000000000000..978aa911b52efc --- /dev/null +++ b/Python/optimizer_cases.c.h @@ -0,0 +1,2212 @@ +// This file is generated by Tools/cases_generator/optimizer_generator.py +// from: +// Python/optimizer_bytecodes.c +// Do not edit! + + case _NOP: { + break; + } + + case _RESUME_CHECK: { + break; + } + + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_AND_CLEAR: { + _Py_UopsSymbol *value; + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp = sym_new_null(ctx); + GETLOCAL(oparg) = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST: { + _Py_UopsSymbol *value; + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST: { + _Py_UopsSymbol *value; + value = stack_pointer[-1]; + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL: { + _Py_UopsSymbol *res; + res = sym_new_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_SEND: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNARY_NEGATIVE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_NOT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_BOOL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_INT: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_LIST: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_NONE: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_STR: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); + } + stack_pointer[-1] = res; + break; + } + + case _REPLACE_WITH_TRUE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_INVERT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _GUARD_BOTH_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + break; + } + + case _GUARD_NOS_INT: { + break; + } + + case _GUARD_TOS_INT: { + break; + } + + case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } + } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + break; + } + + case _GUARD_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_FLOAT: { + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + break; + } + + case _BINARY_OP_ADD_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SLICE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SLICE: { + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_LIST_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_STR_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_TUPLE_INT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_DICT: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + + case _LIST_APPEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_ADD: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_DICT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_SUBSCR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_INTRINSIC_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _CALL_INTRINSIC_2: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_VALUE: { + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; + retval = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + + case _GET_AITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_ANEXT: { + _Py_UopsSymbol *awaitable; + awaitable = sym_new_not_null(ctx); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_AWAITABLE: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _SEND is not a viable micro-op for tier 2 */ + + /* _SEND_GEN is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + + case _YIELD_VALUE: { + _Py_UopsSymbol *res; + res = sym_new_unknown(ctx); + stack_pointer[-1] = res; + break; + } + + case _POP_EXCEPT: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_COMMON_CONSTANT: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_BUILD_CLASS: { + _Py_UopsSymbol *bc; + bc = sym_new_not_null(ctx); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_NAME: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_NAME: { + break; + } + + case _UNPACK_SEQUENCE: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE: { + _Py_UopsSymbol *val1; + _Py_UopsSymbol *val0; + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TUPLE: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_LIST: { + _Py_UopsSymbol **values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_EX: { + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_ATTR: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_GLOBAL: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + _Py_UopsSymbol *locals; + locals = sym_new_not_null(ctx); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + + /* _LOAD_NAME is not a viable micro-op for tier 2 */ + + case _LOAD_GLOBAL: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_FAST: { + break; + } + + case _MAKE_CELL: { + break; + } + + case _DELETE_DEREF: { + break; + } + + case _LOAD_FROM_DICT_OR_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_DEREF: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_DEREF: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY_FREE_VARS: { + break; + } + + case _BUILD_STRING: { + _Py_UopsSymbol *str; + str = sym_new_not_null(ctx); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_TUPLE: { + _Py_UopsSymbol *tup; + tup = sym_new_not_null(ctx); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_LIST: { + _Py_UopsSymbol *list; + list = sym_new_not_null(ctx); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_EXTEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _BUILD_SET is not a viable micro-op for tier 2 */ + + case _BUILD_MAP: { + _Py_UopsSymbol *map; + map = sym_new_not_null(ctx); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SETUP_ANNOTATIONS: { + break; + } + + case _BUILD_CONST_KEY_MAP: { + _Py_UopsSymbol *map; + map = sym_new_not_null(ctx); + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DICT_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DICT_MERGE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAP_ADD: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { + _Py_UopsSymbol *attr_st; + attr_st = sym_new_not_null(ctx); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD: { + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg & 1) { + self_or_null = sym_new_unknown(ctx); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_TYPE_VERSION: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)index; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_MODULE: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t dict_version = (uint32_t)this_instr->operand; + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + break; + } + + case _LOAD_ATTR_MODULE: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; + null = sym_new_null(ctx); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + attr = sym_new_const(ctx, res); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_WITH_HINT: { + break; + } + + case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)hint; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_SLOT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)index; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_CLASS: { + break; + } + + case _LOAD_ATTR_CLASS: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)descr; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + + case _GUARD_DORV_NO_DICT: { + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_WITH_HINT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_SLOT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + res = _Py_uop_sym_new_not_null(ctx); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_FLOAT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_INT: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_STR: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IS_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_SET: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_EG_MATCH: { + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + break; + } + + case _CHECK_EXC_MATCH: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + case _IS_NONE: { + _Py_UopsSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + case _GET_LEN: { + _Py_UopsSymbol *len; + len = sym_new_not_null(ctx); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_CLASS: { + _Py_UopsSymbol *attrs; + attrs = sym_new_not_null(ctx); + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_MAPPING: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_SEQUENCE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_KEYS: { + _Py_UopsSymbol *values_or_none; + values_or_none = sym_new_not_null(ctx); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_YIELD_FROM_ITER: { + _Py_UopsSymbol *iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 */ + + case _FOR_ITER_TIER_TWO: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + + case _ITER_CHECK_LIST: { + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_LIST: { + break; + } + + case _ITER_NEXT_LIST: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_TUPLE: { + break; + } + + case _ITER_NEXT_TUPLE: { + _Py_UopsSymbol *next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_RANGE: { + break; + } + + case _ITER_NEXT_RANGE: { + _Py_UopsSymbol *iter; + _Py_UopsSymbol *next; + iter = stack_pointer[-1]; + next = sym_new_type(ctx, &PyLong_Type); + (void)iter; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _FOR_ITER_GEN_FRAME: { + /* We are about to hit the end of the trace */ + ctx->done = true; + break; + } + + case _LOAD_SPECIAL: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _WITH_EXCEPT_START: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_EXC_INFO: { + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + break; + } + + case _GUARD_KEYS_VERSION: { + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + _Py_UopsSymbol *attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + _Py_UopsSymbol *attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + + /* _CALL is not a viable micro-op for tier 2 */ + + case _CHECK_PERIODIC: { + break; + } + + case _PY_FRAME_GENERAL: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + new_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; + callable = stack_pointer[-2 - oparg]; + (void)callable; + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_PEP_523: { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; + break; + } + + case _CHECK_STACK_SPACE: { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + (void)callable; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uint64_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + ctx->done = true; + break; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + assert(self_or_null != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + } + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_FRAME: { + _Py_UOpsAbstractFrame *new_frame; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; + break; + } + + case _CALL_TYPE_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_STR_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_TUPLE_1: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + + case _EXIT_INIT_CHECK: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_CLASS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_O: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LEN: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_ISINSTANCE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _CALL_KW is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { + _Py_UopsSymbol *func; + func = sym_new_not_null(ctx); + stack_pointer[-1] = func; + break; + } + + case _SET_FUNCTION_ATTRIBUTE: { + _Py_UopsSymbol *func_st; + func_st = sym_new_not_null(ctx); + stack_pointer[-2] = func_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_GENERATOR: { + _Py_UopsSymbol *res; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SLICE: { + _Py_UopsSymbol *slice; + slice = sym_new_not_null(ctx); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONVERT_VALUE: { + _Py_UopsSymbol *result; + result = sym_new_not_null(ctx); + stack_pointer[-1] = result; + break; + } + + case _FORMAT_SIMPLE: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _FORMAT_WITH_SPEC: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY: { + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = bottom; + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + res = sym_new_type(ctx, &PyLong_Type); + } + else { + /* For any other op combining ints/floats the result is a float */ + res = sym_new_type(ctx, &PyFloat_Type); + } + } + res = sym_new_unknown(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SWAP: { + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + break; + } + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + + case _GUARD_IS_TRUE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_FALSE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _JUMP_TO_TOP: { + ctx->done = true; + break; + } + + case _SET_IP: { + break; + } + + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)this_instr->operand; + (void)framesize; + /* We should never see _CHECK_STACK_SPACE_OPERANDs. + * They are only created at the end of this pass. */ + Py_UNREACHABLE(); + break; + } + + case _SAVE_RETURN_OFFSET: { + break; + } + + case _EXIT_TRACE: { + ctx->done = true; + break; + } + + case _CHECK_VALIDITY: { + break; + } + + case _LOAD_CONST_INLINE: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION: { + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DYNAMIC_EXIT: { + break; + } + + case _START_EXECUTOR: { + break; + } + + case _FATAL_ERROR: { + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + break; + } + + case _DEOPT: { + break; + } + + case _ERROR_POP_N: { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _TIER2_RESUME_CHECK: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c new file mode 100644 index 00000000000000..40cbf95e3d6d39 --- /dev/null +++ b/Python/optimizer_symbols.c @@ -0,0 +1,549 @@ +#ifdef _Py_TIER2 + +#include "Python.h" + +#include "pycore_code.h" +#include "pycore_frame.h" +#include "pycore_long.h" +#include "pycore_optimizer.h" + +#include +#include +#include + +/* Symbols + ======= + + See the diagram at + https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md + + We represent the nodes in the diagram as follows + (the flag bits are only defined in optimizer_symbols.c): + - Top: no flag bits, typ and const_val are NULL. + - NULL: IS_NULL flag set, type and const_val NULL. + - Not NULL: NOT_NULL flag set, type and const_val NULL. + - None/not None: not used. (None could be represented as any other constant.) + - Known type: NOT_NULL flag set and typ set; const_val is NULL. + - Known constant: NOT_NULL flag set, type set, const_val set. + - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. + */ + +// Flags for below. +#define IS_NULL 1 << 0 +#define NOT_NULL 1 << 1 +#define NO_SPACE 1 << 2 + +#ifdef Py_DEBUG +static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; +} +#define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else +#define DPRINTF(level, ...) +#endif + +static _Py_UopsSymbol NO_SPACE_SYMBOL = { + .flags = IS_NULL | NOT_NULL | NO_SPACE, + .typ = NULL, + .const_val = NULL, + .type_version = 0, +}; + +_Py_UopsSymbol * +out_of_space(_Py_UOpsContext *ctx) +{ + ctx->done = true; + ctx->out_of_space = true; + return &NO_SPACE_SYMBOL; +} + +static _Py_UopsSymbol * +sym_new(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + OPT_STAT_INC(optimizer_failure_reason_no_memory); + DPRINTF(1, "out of space for symbolic expression type\n"); + return NULL; + } + ctx->t_arena.ty_curr_number++; + self->flags = 0; + self->typ = NULL; + self->const_val = NULL; + self->type_version = 0; + + return self; +} + +static inline void +sym_set_flag(_Py_UopsSymbol *sym, int flag) +{ + sym->flags |= flag; +} + +static inline void +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL | NOT_NULL); + sym->typ = NULL; + Py_CLEAR(sym->const_val); + ctx->done = true; + ctx->contradiction = true; +} + +bool +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +{ + if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { + assert(sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->typ == NULL); + assert(sym->const_val == NULL); + return true; + } + return false; +} + +bool +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +{ + return sym->flags == NOT_NULL; +} + +bool +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +{ + return sym->flags == IS_NULL; +} + +bool +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +{ + return sym->const_val != NULL; +} + +PyObject * +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +{ + return sym->const_val; +} + +void +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (sym->flags & IS_NULL) { + sym_set_bottom(ctx, sym); + return; + } + if (sym->typ != NULL) { + if (sym->typ != typ) { + sym_set_bottom(ctx, sym); + return; + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + } +} + +bool +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +{ + // if the type version was already set, then it must be different and we should set it to bottom + if (sym->type_version) { + sym_set_bottom(ctx, sym); + return false; + } + sym->type_version = version; + return true; +} + +void +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) +{ + assert(const_val != NULL); + if (sym->flags & IS_NULL) { + sym_set_bottom(ctx, sym); + } + PyTypeObject *typ = Py_TYPE(const_val); + if (sym->typ != NULL && sym->typ != typ) { + sym_set_bottom(ctx, sym); + } + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { + // TODO: What if they're equal? + sym_set_bottom(ctx, sym); + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); + } +} + +void +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_not_null(sym)) { + sym_set_bottom(ctx, sym); + } + sym_set_flag(sym, IS_NULL); +} + +void +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_null(sym)) { + sym_set_bottom(ctx, sym); + } + sym_set_flag(sym, NOT_NULL); +} + + +_Py_UopsSymbol * +_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +{ + return sym_new(ctx); +} + +_Py_UopsSymbol * +_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + sym_set_flag(res, NOT_NULL); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +{ + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_type(ctx, res, typ); + return res; +} + +// Adds a new reference to const_val, owned by the symbol. +_Py_UopsSymbol * +_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_const(ctx, res, const_val); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { + return out_of_space(ctx); + } + _Py_uop_sym_set_null(ctx, null_sym); + return null_sym; +} + +PyTypeObject * +_Py_uop_sym_get_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return NULL; + } + return sym->typ; +} + +unsigned int +_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +{ + return sym->type_version; +} + +bool +_Py_uop_sym_has_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return false; + } + return sym->typ != NULL; +} + +bool +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + return _Py_uop_sym_get_type(sym) == typ; +} + +bool +_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +{ + return _Py_uop_sym_get_type_version(sym) == version; +} + + +int +_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +{ + /* There are some non-constant values for + * which `bool(val)` always evaluates to + * True or False, such as tuples with known + * length, but unknown contents, or bound-methods. + * This function will need updating + * should we support those values. + */ + if (_Py_uop_sym_is_bottom(sym)) { + return -1; + } + if (!_Py_uop_sym_is_const(sym)) { + return -1; + } + PyObject *value = _Py_uop_sym_get_const(sym); + if (value == Py_None) { + return 0; + } + /* Only handle a few known safe types */ + PyTypeObject *tp = Py_TYPE(value); + if (tp == &PyLong_Type) { + return !_PyLong_IsZero((PyLongObject *)value); + } + if (tp == &PyUnicode_Type) { + return value != &_Py_STR(empty); + } + if (tp == &PyBool_Type) { + return value == Py_True; + } + return -1; +} + +// 0 on success, -1 on error. +_Py_UOpsAbstractFrame * +_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len) +{ + assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + + frame->stack_len = co->co_stacksize; + frame->locals_len = co->co_nlocalsplus; + + frame->locals = ctx->n_consumed; + frame->stack = frame->locals + co->co_nlocalsplus; + frame->stack_pointer = frame->stack + curr_stackentries; + ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); + if (ctx->n_consumed >= ctx->limit) { + ctx->done = true; + ctx->out_of_space = true; + return NULL; + } + + // Initialize with the initial state of all local variables + for (int i = 0; i < arg_len; i++) { + frame->locals[i] = args[i]; + } + + for (int i = arg_len; i < co->co_nlocalsplus; i++) { + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + frame->locals[i] = local; + } + + + // Initialize the stack as well + for (int i = 0; i < curr_stackentries; i++) { + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + frame->stack[i] = stackvar; + } + + return frame; +} + +void +_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->curr_frame_depth = 0; + int tys = ctx->t_arena.ty_curr_number; + for (int i = 0; i < tys; i++) { + Py_CLEAR(ctx->t_arena.arena[i].const_val); + } +} + +void +_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +{ + ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; + ctx->n_consumed = ctx->locals_and_stack; +#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. + for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { + ctx->locals_and_stack[i] = NULL; + } +#endif + + // Setup the arena for sym expressions. + ctx->t_arena.ty_curr_number = 0; + ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + + // Frame setup + ctx->curr_frame_depth = 0; +} + +int +_Py_uop_frame_pop(_Py_UOpsContext *ctx) +{ + _Py_UOpsAbstractFrame *frame = ctx->frame; + ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; + assert(ctx->curr_frame_depth >= 1); + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + return 0; +} + +#define TEST_PREDICATE(PRED, MSG) \ +do { \ + if (!(PRED)) { \ + PyErr_SetString( \ + PyExc_AssertionError, \ + (MSG)); \ + goto fail; \ + } \ +} while (0) + +static _Py_UopsSymbol * +make_bottom(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_uop_sym_set_null(ctx, sym); + _Py_uop_sym_set_non_null(ctx, sym); + return sym; +} + +PyObject * +_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +{ + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + _Py_uop_abstractcontext_init(ctx); + PyObject *val_42 = NULL; + PyObject *val_43 = NULL; + + // Use a single 'sym' variable so copy-pasting tests is easier. + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + + sym = make_bottom(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + + val_42 = PyLong_FromLong(42); + assert(val_42 != NULL); + assert(_Py_IsImmortal(val_42)); + + val_43 = PyLong_FromLong(43); + assert(val_43 != NULL); + assert(_Py_IsImmortal(val_43)); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(ctx, sym, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(ctx, sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + + sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); + sym = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); + sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + + _Py_uop_abstractcontext_fini(ctx); + Py_DECREF(val_42); + Py_DECREF(val_43); + Py_RETURN_NONE; + +fail: + _Py_uop_abstractcontext_fini(ctx); + Py_XDECREF(val_42); + Py_XDECREF(val_43); + return NULL; +} + +#endif /* _Py_TIER2 */ diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 664e622cc17474..841b1d71ea16cb 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -1,11 +1,12 @@ #include "Python.h" #include "pycore_llist.h" -#include "pycore_lock.h" // _PyRawMutex +#include "pycore_lock.h" // _PyRawMutex #include "pycore_parking_lot.h" -#include "pycore_pyerrors.h" // _Py_FatalErrorFormat -#include "pycore_pystate.h" // _PyThreadState_GET -#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_time.h" // _PyTime_Add() #include @@ -91,7 +92,7 @@ _PySemaphore_Destroy(_PySemaphore *sema) } static int -_PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) +_PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) { int res; #if defined(MS_WINDOWS) @@ -118,10 +119,24 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (timeout >= 0) { struct timespec ts; - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); - _PyTime_AsTimespec(deadline, &ts); +#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) && !defined(_Py_THREAD_SANITIZER) + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); + _PyTime_AsTimespec_clamp(deadline, &ts); + + err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); +#else + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); + + _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_timedwait(&sema->platform_sem, &ts); +#endif } else { err = sem_wait(&sema->platform_sem); @@ -149,11 +164,17 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (sema->counter == 0) { if (timeout >= 0) { struct timespec ts; - - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); - _PyTime_AsTimespec(deadline, &ts); +#if defined(HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP) + _PyTime_AsTimespec_clamp(timeout, &ts); + err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); +#else + PyTime_t now; + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); + _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); +#endif // HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP } else { err = pthread_cond_wait(&sema->cond, &sema->mutex); @@ -175,19 +196,22 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) } int -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout, int detach) +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) { PyThreadState *tstate = NULL; if (detach) { tstate = _PyThreadState_GET(); - if (tstate) { + if (tstate && _Py_atomic_load_int_relaxed(&tstate->state) == + _Py_THREAD_ATTACHED) { + // Only detach if we are attached PyEval_ReleaseThread(tstate); } + else { + tstate = NULL; + } } - int res = _PySemaphore_PlatformWait(sema, timeout); - - if (detach && tstate) { + if (tstate) { PyEval_AcquireThread(tstate); } return res; @@ -231,6 +255,7 @@ dequeue(Bucket *bucket, const void *address) if (wait->addr == (uintptr_t)address) { llist_remove(node); --bucket->num_waiters; + wait->is_unparking = true; return wait; } } @@ -249,6 +274,7 @@ dequeue_all(Bucket *bucket, const void *address, struct llist_node *dst) llist_remove(node); llist_insert_tail(dst, node); --bucket->num_waiters; + wait->is_unparking = true; } } } @@ -268,7 +294,7 @@ atomic_memcmp(const void *addr, const void *expected, size_t addr_size) int _PyParkingLot_Park(const void *addr, const void *expected, size_t size, - _PyTime_t timeout_ns, void *park_arg, int detach) + PyTime_t timeout_ns, void *park_arg, int detach) { struct wait_entry wait = { .park_arg = park_arg, @@ -324,8 +350,6 @@ _PyParkingLot_Unpark(const void *addr, _Py_unpark_fn_t *fn, void *arg) _PyRawMutex_Lock(&bucket->mutex); struct wait_entry *waiter = dequeue(bucket, addr); if (waiter) { - waiter->is_unparking = true; - int has_more_waiters = (bucket->num_waiters > 0); fn(arg, waiter->park_arg, has_more_waiters); } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 50c60093cd4e32..33abaddc1b5df4 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -251,8 +251,7 @@ Py_SetPath(const wchar_t *path) } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void Py_SetPythonHome(const wchar_t *home) { int has_value = home && home[0]; @@ -275,8 +274,7 @@ Py_SetPythonHome(const wchar_t *home) } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void Py_SetProgramName(const wchar_t *program_name) { int has_value = program_name && program_name[0]; diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c new file mode 100644 index 00000000000000..0a8945958b4b3c --- /dev/null +++ b/Python/perf_jit_trampoline.c @@ -0,0 +1,616 @@ +#include "Python.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks +#include "pycore_frame.h" +#include "pycore_interp.h" + + +#ifdef PY_HAVE_PERF_TRAMPOLINE + +#include +#include +#include +#include // mmap() +#include +#include // sysconf() +#include // gettimeofday() +#include + +// ---------------------------------- +// Perf jitdump API +// ---------------------------------- + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; + int code_id; +} PerfMapJitState; + +static PerfMapJitState perf_jit_map_state; + +/* +Usually the binary and libraries are mapped in separate region like below: + + address -> + --+---------------------+--//--+---------------------+-- + | .text | .data | ... | | .text | .data | ... | + --+---------------------+--//--+---------------------+-- + myprog libc.so + +So it'd be easy and straight-forward to find a mapped binary or library from an +address. + +But for JIT code, the code arena only cares about the code section. But the +resulting DSOs (which is generated by perf inject -j) contain ELF headers and +unwind info too. Then it'd generate following address space with synthesized +MMAP events. Let's say it has a sample between address B and C. + + sample + | + address -> A B v C + --------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + --------------------------------------------------------------------------------------------------- + +If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see +the unwind info. If it maps both .text section and unwind sections, the sample +could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing +which one is right. So to make perf happy we have non-overlapping ranges for each +DSO: + + address -> + ------------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + ------------------------------------------------------------------------------------------------------- + +As the trampolines are constant, we add a constant padding but in general the padding needs to have the +size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + +#define PERF_JIT_CODE_PADDING 0x100 +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + PerfLoad = 0, + PerfMove = 1, + PerfDebugInfo = 2, + PerfClose = 3, + PerfUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +static const intptr_t nanoseconds_per_second = 1000000000; + +// Dwarf encoding constants + +static const uint8_t DwarfUData4 = 0x03; +static const uint8_t DwarfSData4 = 0x0b; +static const uint8_t DwarfPcRel = 0x10; +static const uint8_t DwarfDataRel = 0x30; +// static uint8_t DwarfOmit = 0xff; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= nanoseconds_per_second; + result += ts.tv_nsec; + return result; +} + +static int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + perf_jit_map_state.code_id = 0; + + trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + return &perf_jit_map_state; +} + +/* DWARF definitions. */ + +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 1); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 3); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + + // Create unwinding information (eh frame) + ELFObjectContext ctx; + char buffer[1024]; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + int eh_frame_size = ctx.p - ctx.startp; + + // Populate the unwind info event for perf + CodeUnwindingInfoEvent ev2; + ev2.base.event = PerfUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + // Ensure we have enough space between DSOs when perf maps them + assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + // Populate the eh Frame header + EhFrameHeader f; + f.version = 1; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; + f.fde_count_enc = DwarfUData4; + f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = PerfLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = syscall(SYS_gettid); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + perf_jit_map_state.code_id += 1; + ev.code_id = perf_jit_map_state.code_id; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + +#endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 209a23b6c1cbc7..f144f7d436fe68 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -133,7 +133,6 @@ any DWARF information available for them). #include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_frame.h" #include "pycore_interp.h" -#include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -144,6 +143,8 @@ any DWARF information available for them). #include // mmap() #include #include // sysconf() +#include // gettimeofday() + #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE @@ -188,12 +189,19 @@ struct code_arena_st { typedef struct code_arena_st code_arena_t; typedef struct trampoline_api_st trampoline_api_t; +enum perf_trampoline_type { + PERF_TRAMPOLINE_UNSET = 0, + PERF_TRAMPOLINE_TYPE_MAP = 1, + PERF_TRAMPOLINE_TYPE_JITDUMP = 2, +}; + #define perf_status _PyRuntime.ceval.perf.status #define extra_code_index _PyRuntime.ceval.perf.extra_code_index #define perf_code_arena _PyRuntime.ceval.perf.code_arena #define trampoline_api _PyRuntime.ceval.perf.trampoline_api #define perf_map_file _PyRuntime.ceval.perf.map_file - +#define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork +#define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type static void perf_map_write_entry(void *state, const void *code_addr, @@ -217,12 +225,52 @@ perf_map_write_entry(void *state, const void *code_addr, PyMem_RawFree(perf_map_entry); } +static void* +perf_map_init_state(void) +{ + PyUnstable_PerfMapState_Init(); + trampoline_api.code_padding = 0; + perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; + return NULL; +} + +static int +perf_map_free_state(void *state) +{ + PyUnstable_PerfMapState_Fini(); + return 0; +} + _PyPerf_Callbacks _Py_perfmap_callbacks = { - NULL, + &perf_map_init_state, &perf_map_write_entry, - NULL, + &perf_map_free_state, }; + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + +// TRAMPOLINE MANAGEMENT API + static int new_code_arena(void) { @@ -234,16 +282,16 @@ new_code_arena(void) mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, // fd (not used here) 0); // offset (not used here) - if (!memory) { + if (memory == MAP_FAILED) { PyErr_SetFromErrno(PyExc_OSError); - _PyErr_WriteUnraisableMsg( - "Failed to create new mmap for perf trampoline", NULL); + PyErr_FormatUnraisable("Failed to create new mmap for perf trampoline"); perf_status = PERF_STATUS_FAILED; return -1; } void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; + size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -252,18 +300,17 @@ new_code_arena(void) // measurable performance improvement by rounding trampolines up to 32-bit // or 64-bit alignment. - size_t n_copies = mem_size / code_size; + size_t n_copies = mem_size / chunk_size; for (size_t i = 0; i < n_copies; i++) { - memcpy(memory + i * code_size, start, code_size * sizeof(char)); + memcpy(memory + i * chunk_size, start, code_size * sizeof(char)); } // Some systems may prevent us from creating executable code on the fly. int res = mprotect(memory, mem_size, PROT_READ | PROT_EXEC); if (res == -1) { PyErr_SetFromErrno(PyExc_OSError); munmap(memory, mem_size); - _PyErr_WriteUnraisableMsg( - "Failed to set mmap for perf trampoline to PROT_READ | PROT_EXEC", - NULL); + PyErr_FormatUnraisable("Failed to set mmap for perf trampoline to " + "PROT_READ | PROT_EXEC"); return -1; } @@ -277,8 +324,7 @@ new_code_arena(void) if (new_arena == NULL) { PyErr_NoMemory(); munmap(memory, mem_size); - _PyErr_WriteUnraisableMsg("Failed to allocate new code arena struct", - NULL); + PyErr_FormatUnraisable("Failed to allocate new code arena struct for perf trampoline"); return -1; } @@ -310,16 +356,18 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= code_arena->code_size; - code_arena->current_addr += code_arena->code_size; + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + code_arena->size_left -= total_code_size; + code_arena->current_addr += total_code_size; return trampoline; } static inline py_trampoline compile_trampoline(void) { + size_t total_code_size = round_up(perf_code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= perf_code_arena->code_size)) { + (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { return NULL; } @@ -361,6 +409,26 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, } #endif // PY_HAVE_PERF_TRAMPOLINE +int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *co) +{ +#ifdef PY_HAVE_PERF_TRAMPOLINE + py_trampoline f = NULL; + assert(extra_code_index != -1); + int ret = _PyCode_GetExtra((PyObject *)co, extra_code_index, (void **)&f); + if (ret != 0 || f == NULL) { + py_trampoline new_trampoline = compile_trampoline(); + if (new_trampoline == NULL) { + return 0; + } + trampoline_api.write_state(trampoline_api.state, new_trampoline, + perf_code_arena->code_size, co); + return _PyCode_SetExtra((PyObject *)co, extra_code_index, + (void *)new_trampoline); + } +#endif // PY_HAVE_PERF_TRAMPOLINE + return 0; +} + int _PyIsPerfTrampolineActive(void) { @@ -399,7 +467,6 @@ _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *callbacks) trampoline_api.write_state = callbacks->write_state; trampoline_api.free_state = callbacks->free_state; trampoline_api.state = NULL; - perf_status = PERF_STATUS_OK; #endif return 0; } @@ -418,6 +485,7 @@ _PyPerfTrampoline_Init(int activate) } if (!activate) { tstate->interp->eval_frame = NULL; + perf_status = PERF_STATUS_NO_INIT; } else { tstate->interp->eval_frame = py_trampoline_evaluator; @@ -428,6 +496,9 @@ _PyPerfTrampoline_Init(int activate) if (extra_code_index == -1) { return -1; } + if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { + trampoline_api.state = trampoline_api.init_state(); + } perf_status = PERF_STATUS_OK; } #endif @@ -438,12 +509,35 @@ int _PyPerfTrampoline_Fini(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE + if (perf_status != PERF_STATUS_OK) { + return 0; + } PyThreadState *tstate = _PyThreadState_GET(); if (tstate->interp->eval_frame == py_trampoline_evaluator) { tstate->interp->eval_frame = NULL; } - free_code_arenas(); + if (perf_status == PERF_STATUS_OK) { + trampoline_api.free_state(trampoline_api.state); + perf_trampoline_type = PERF_TRAMPOLINE_UNSET; + } extra_code_index = -1; + perf_status = PERF_STATUS_NO_INIT; +#endif + return 0; +} + +void _PyPerfTrampoline_FreeArenas(void) { +#ifdef PY_HAVE_PERF_TRAMPOLINE + free_code_arenas(); +#endif + return; +} + +int +PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable){ +#ifdef PY_HAVE_PERF_TRAMPOLINE + persist_after_fork = enable; + return persist_after_fork; #endif return 0; } @@ -452,12 +546,24 @@ PyStatus _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE - // Restart trampoline in file in child. - int was_active = _PyIsPerfTrampolineActive(); - _PyPerfTrampoline_Fini(); - PyUnstable_PerfMapState_Fini(); - if (was_active) { - _PyPerfTrampoline_Init(1); + if (persist_after_fork) { + if (perf_trampoline_type != PERF_TRAMPOLINE_TYPE_MAP) { + return PyStatus_Error("Failed to copy perf map file as perf trampoline type is not type map."); + } + _PyPerfTrampoline_Fini(); + char filename[256]; + pid_t parent_pid = getppid(); + snprintf(filename, sizeof(filename), "/tmp/perf-%d.map", parent_pid); + if (PyUnstable_CopyPerfMapFile(filename) != 0) { + return PyStatus_Error("Failed to copy perf map file."); + } + } else { + // Restart trampoline in file in child. + int was_active = _PyIsPerfTrampolineActive(); + _PyPerfTrampoline_Fini(); + if (was_active) { + _PyPerfTrampoline_Init(1); + } } #endif return PyStatus_Ok(); diff --git a/Python/pyarena.c b/Python/pyarena.c index ead03370d153c3..7ab370163b2b93 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -6,9 +6,6 @@ Measurements with standard library modules suggest the average allocation is about 20 bytes and that most compiles use a single block. - - TODO(jhylton): Think about a realloc API, maybe just for the last - allocation? */ #define DEFAULT_BLOCK_SIZE 8192 @@ -108,7 +105,6 @@ block_alloc(block *b, size_t size) /* If we need to allocate more memory than will fit in the default block, allocate a one-off block that is exactly the right size. */ - /* TODO(jhylton): Think about space waste at end of block */ block *newbl = block_new( size < DEFAULT_BLOCK_SIZE ? DEFAULT_BLOCK_SIZE : size); diff --git a/Python/pyhash.c b/Python/pyhash.c index f9060b8003a0a7..1504fa201c9902 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -83,8 +83,6 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; */ -Py_hash_t _Py_HashPointer(const void *); - Py_hash_t _Py_HashDouble(PyObject *inst, double v) { @@ -92,11 +90,11 @@ _Py_HashDouble(PyObject *inst, double v) double m; Py_uhash_t x, y; - if (!Py_IS_FINITE(v)) { - if (Py_IS_INFINITY(v)) + if (!isfinite(v)) { + if (isinf(v)) return v > 0 ? _PyHASH_INF : -_PyHASH_INF; else - return _Py_HashPointer(inst); + return PyObject_GenericHash(inst); } m = frexp(v, &e); @@ -132,23 +130,19 @@ _Py_HashDouble(PyObject *inst, double v) } Py_hash_t -_Py_HashPointerRaw(const void *p) +Py_HashPointer(const void *ptr) { - size_t y = (size_t)p; - /* bottom 3 or 4 bits are likely to be 0; rotate y by 4 to avoid - excessive hash collisions for dicts and sets */ - y = (y >> 4) | (y << (8 * SIZEOF_VOID_P - 4)); - return (Py_hash_t)y; + Py_hash_t hash = _Py_HashPointerRaw(ptr); + if (hash == -1) { + hash = -2; + } + return hash; } Py_hash_t -_Py_HashPointer(const void *p) +PyObject_GenericHash(PyObject *obj) { - Py_hash_t x = _Py_HashPointerRaw(p); - if (x == -1) { - x = -2; - } - return x; + return Py_HashPointer(obj); } Py_hash_t @@ -176,12 +170,12 @@ _Py_HashBytes(const void *src, Py_ssize_t len) switch(len) { /* ((hash << 5) + hash) + *p == hash * 33 + *p */ - case 7: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 6: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 5: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 4: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 3: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 2: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ + case 7: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 6: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 5: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 4: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 3: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 2: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; case 1: hash = ((hash << 5) + hash) + *p++; break; default: Py_UNREACHABLE(); @@ -269,12 +263,12 @@ fnv(const void *src, Py_ssize_t len) x ^= (Py_uhash_t) *p << 7; while (blocks--) { PY_UHASH_CPY(block.bytes, p); - x = (_PyHASH_MULTIPLIER * x) ^ block.value; + x = (PyHASH_MULTIPLIER * x) ^ block.value; p += SIZEOF_PY_UHASH_T; } /* add remainder */ for (; remainder > 0; remainder--) - x = (_PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; + x = (PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; x ^= (Py_uhash_t) len; x ^= (Py_uhash_t) _Py_HashSecret.fnv.suffix; if (x == (Py_uhash_t) -1) { @@ -397,13 +391,13 @@ siphash13(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { t = 0; pt = (uint8_t *)&t; switch (src_sz) { - case 7: pt[6] = in[6]; /* fall through */ - case 6: pt[5] = in[5]; /* fall through */ - case 5: pt[4] = in[4]; /* fall through */ + case 7: pt[6] = in[6]; _Py_FALLTHROUGH; + case 6: pt[5] = in[5]; _Py_FALLTHROUGH; + case 5: pt[4] = in[4]; _Py_FALLTHROUGH; case 4: memcpy(pt, in, sizeof(uint32_t)); break; - case 3: pt[2] = in[2]; /* fall through */ - case 2: pt[1] = in[1]; /* fall through */ - case 1: pt[0] = in[0]; /* fall through */ + case 3: pt[2] = in[2]; _Py_FALLTHROUGH; + case 2: pt[1] = in[1]; _Py_FALLTHROUGH; + case 1: pt[0] = in[0]; break; } b |= _le64toh(t); @@ -448,13 +442,13 @@ siphash24(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { t = 0; pt = (uint8_t *)&t; switch (src_sz) { - case 7: pt[6] = in[6]; /* fall through */ - case 6: pt[5] = in[5]; /* fall through */ - case 5: pt[4] = in[4]; /* fall through */ + case 7: pt[6] = in[6]; _Py_FALLTHROUGH; + case 6: pt[5] = in[5]; _Py_FALLTHROUGH; + case 5: pt[4] = in[4]; _Py_FALLTHROUGH; case 4: memcpy(pt, in, sizeof(uint32_t)); break; - case 3: pt[2] = in[2]; /* fall through */ - case 2: pt[1] = in[1]; /* fall through */ - case 1: pt[0] = in[0]; /* fall through */ + case 3: pt[2] = in[2]; _Py_FALLTHROUGH; + case 2: pt[1] = in[1]; _Py_FALLTHROUGH; + case 1: pt[0] = in[0]; break; } b |= _le64toh(t); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 14033162377489..39eaa86098b2e5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -10,7 +10,6 @@ #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() #include "pycore_floatobject.h" // _PyFloat_InitTypes() -#include "pycore_genobject.h" // _PyAsyncGen_Fini() #include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -32,6 +31,7 @@ #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "opcode.h" @@ -69,7 +69,11 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ +static PyStatus init_android_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); +static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); /* The following places the `_PyRuntime` structure in a location that can be @@ -475,6 +479,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, if (interp == NULL) { return _PyStatus_ERR("can't make main interpreter"); } + assert(interp->_ready); status = _PyConfig_Write(config, runtime); if (_PyStatus_EXCEPTION(status)) { @@ -528,11 +533,6 @@ pycore_init_runtime(_PyRuntimeState *runtime, return status; } - status = _PyTime_Init(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = _PyImport_Init(); if (_PyStatus_EXCEPTION(status)) { return status; @@ -561,6 +561,15 @@ init_interp_settings(PyInterpreterState *interp, return _PyStatus_ERR("per-interpreter obmalloc does not support " "single-phase init extension modules"); } +#ifdef Py_GIL_DISABLED + if (!_Py_IsMainInterpreter(interp) && + !config->check_multi_interp_extensions) + { + return _PyStatus_ERR("The free-threaded build does not support " + "single-phase init extension modules in " + "subinterpreters"); + } +#endif if (config->allow_fork) { interp->feature_flags |= Py_RTFLAGS_FORK; @@ -581,46 +590,47 @@ init_interp_settings(PyInterpreterState *interp, interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; } - /* We check "gil" in init_interp_create_gil(). */ + switch (config->gil) { + case PyInterpreterConfig_DEFAULT_GIL: break; + case PyInterpreterConfig_SHARED_GIL: break; + case PyInterpreterConfig_OWN_GIL: break; + default: + return _PyStatus_ERR("invalid interpreter config 'gil' value"); + } return _PyStatus_OK(); } -static PyStatus +static void init_interp_create_gil(PyThreadState *tstate, int gil) { - PyStatus status; - /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ - status = _PyGILState_SetTstate(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + _PyGILState_SetTstate(tstate); - int own_gil; - switch (gil) { - case PyInterpreterConfig_DEFAULT_GIL: own_gil = 0; break; - case PyInterpreterConfig_SHARED_GIL: own_gil = 0; break; - case PyInterpreterConfig_OWN_GIL: own_gil = 1; break; - default: - return _PyStatus_ERR("invalid interpreter config 'gil' value"); - } + int own_gil = (gil == PyInterpreterConfig_OWN_GIL); /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate, own_gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - return _PyStatus_OK(); + _PyEval_InitGIL(tstate, own_gil); } +static int +builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef _Py_TIER2 + if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + _Py_Executors_InvalidateAll(interp, 1); + } +#endif + RARE_EVENT_INTERP_INC(interp, builtin_dict); + return 0; +} static PyStatus pycore_create_interpreter(_PyRuntimeState *runtime, @@ -635,6 +645,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); + _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); + interp->_ready = 1; status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -648,24 +660,31 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - // The main interpreter always has its own GIL. + // The main interpreter always has its own GIL and supports single-phase + // init extensions. config.gil = PyInterpreterConfig_OWN_GIL; + config.check_multi_interp_extensions = 0; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + return _PyStatus_NO_MEMORY(); + } + PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); + _PyThreadState_WHENCE_INIT); if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); } + runtime->main_tstate = tstate; _PyThreadState_Bind(tstate); - status = init_interp_create_gil(tstate, config.gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + init_interp_create_gil(tstate, config.gil); *tstate_p = tstate; return _PyStatus_OK(); @@ -686,6 +705,10 @@ pycore_init_global_objects(PyInterpreterState *interp) _PyUnicode_InitState(interp); + if (_Py_IsMainInterpreter(interp)) { + _Py_GetConstant_Init(); + } + return _PyStatus_OK(); } @@ -738,6 +761,12 @@ pycore_init_types(PyInterpreterState *interp) if (_PyStatus_EXCEPTION(status)) { return status; } + + status = _PyXI_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -752,7 +781,7 @@ pycore_init_builtins(PyThreadState *tstate) } PyObject *modules = _PyImport_GetModules(interp); - if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) { + if (_PyImport_FixupBuiltin(tstate, bimod, "builtins", modules) < 0) { goto error; } @@ -824,6 +853,16 @@ pycore_interp_init(PyThreadState *tstate) return status; } + status = _PyCode_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + status = _PyDtoa_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + // The GC must be initialized before the first GC collection. status = _PyGC_Init(interp); if (_PyStatus_EXCEPTION(status)) { @@ -854,6 +893,11 @@ pycore_interp_init(PyThreadState *tstate) goto done; } + status = _PyXI_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } + const PyConfig *config = _PyInterpreterState_GetConfig(interp); status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); @@ -1076,6 +1120,37 @@ pyinit_main_reconfigure(PyThreadState *tstate) } +#ifdef Py_DEBUG +static void +run_presite(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + + if (!config->run_presite) { + return; + } + + PyObject *presite_modname = PyUnicode_FromWideChar( + config->run_presite, + wcslen(config->run_presite) + ); + if (presite_modname == NULL) { + fprintf(stderr, "Could not convert pre-site module name to unicode\n"); + } + else { + PyObject *presite = PyImport_Import(presite_modname); + if (presite == NULL) { + fprintf(stderr, "pre-site import failed:\n"); + _PyErr_Print(tstate); + } + Py_XDECREF(presite); + Py_DECREF(presite_modname); + } +} +#endif + + static PyStatus init_interp_main(PyThreadState *tstate) { @@ -1139,7 +1214,14 @@ init_interp_main(PyThreadState *tstate) #ifdef PY_HAVE_PERF_TRAMPOLINE if (config->perf_profiling) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_callbacks) < 0 || + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } + else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || _PyPerfTrampoline_Init(config->perf_profiling) < 0) { return _PyStatus_ERR("can't initialize the perf trampoline"); } @@ -1157,6 +1239,17 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + +#ifdef Py_DEBUG + run_presite(tstate); +#endif + status = add_main_module(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1192,21 +1285,30 @@ init_interp_main(PyThreadState *tstate) } // Turn on experimental tier 2 (uops-based) optimizer + // This is also needed when the JIT is enabled +#ifdef _Py_TIER2 if (is_main_interp) { - char *envvar = Py_GETENV("PYTHONUOPS"); - int enabled = envvar != NULL && *envvar > '0'; - if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { - enabled = 1; + int enabled = 1; +#if _Py_TIER2 & 2 + enabled = 0; +#endif + char *env = Py_GETENV("PYTHON_JIT"); + if (env && *env != '\0') { + // PYTHON_JIT=0|1 overrides the default + enabled = *env != '0'; } if (enabled) { - PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); + PyObject *opt = _PyOptimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } - PyUnstable_SetOptimizer((_PyOptimizerObject *)opt); + if (_Py_SetTier2Optimizer((_PyOptimizerObject *)opt)) { + return _PyStatus_ERR("can't install optimizer"); + } Py_DECREF(opt); } } +#endif if (!is_main_interp) { // The main interpreter is handled in Py_Main(), for now. @@ -1233,6 +1335,12 @@ init_interp_main(PyThreadState *tstate) } } + + interp->dict_state.watchers[0] = &builtins_dict_watcher; + if (PyDict_Watch(0, interp->builtins) != 0) { + return _PyStatus_ERR("failed to set builtin dict watcher"); + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -1377,13 +1485,13 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) PySys_WriteStderr("# clear builtins._\n"); } if (PyDict_SetItemString(interp->builtins, "_", Py_None) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on setting builtin variable _"); } const char * const *p; for (p = sys_deletes; *p != NULL; p++) { if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing sys.%s", *p); } } for (p = sys_files; *p != NULL; p+=2) { @@ -1394,13 +1502,13 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) } PyObject *value; if (PyDict_GetItemStringRef(interp->sysdict, orig_name, &value) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on restoring sys.%s", name); } if (value == NULL) { value = Py_NewRef(Py_None); } if (PyDict_SetItemString(interp->sysdict, name, value) < 0) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on restoring sys.%s", name); } Py_DECREF(value); } @@ -1412,7 +1520,7 @@ finalize_remove_modules(PyObject *modules, int verbose) { PyObject *weaklist = PyList_New(0); if (weaklist == NULL) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on removing modules"); } #define STORE_MODULE_WEAKREF(name, mod) \ @@ -1421,13 +1529,13 @@ finalize_remove_modules(PyObject *modules, int verbose) if (wr) { \ PyObject *tup = PyTuple_Pack(2, name, wr); \ if (!tup || PyList_Append(weaklist, tup) < 0) { \ - PyErr_WriteUnraisable(NULL); \ + PyErr_FormatUnraisable("Exception ignored on removing modules"); \ } \ Py_XDECREF(tup); \ Py_DECREF(wr); \ } \ else { \ - PyErr_WriteUnraisable(NULL); \ + PyErr_FormatUnraisable("Exception ignored on removing modules"); \ } \ } @@ -1438,7 +1546,7 @@ finalize_remove_modules(PyObject *modules, int verbose) } \ STORE_MODULE_WEAKREF(name, mod); \ if (PyObject_SetItem(modules, name, Py_None) < 0) { \ - PyErr_WriteUnraisable(NULL); \ + PyErr_FormatUnraisable("Exception ignored on removing modules"); \ } \ } @@ -1452,14 +1560,14 @@ finalize_remove_modules(PyObject *modules, int verbose) else { PyObject *iterator = PyObject_GetIter(modules); if (iterator == NULL) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on removing modules"); } else { PyObject *key; while ((key = PyIter_Next(iterator))) { PyObject *value = PyObject_GetItem(modules, key); if (value == NULL) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on removing modules"); continue; } CLEAR_MODULE(key, value); @@ -1467,7 +1575,7 @@ finalize_remove_modules(PyObject *modules, int verbose) Py_DECREF(key); } if (PyErr_Occurred()) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on removing modules"); } Py_DECREF(iterator); } @@ -1487,7 +1595,7 @@ finalize_clear_modules_dict(PyObject *modules) } else { if (PyObject_CallMethodNoArgs(modules, &_Py_ID(clear)) == NULL) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on clearing sys.modules"); } } } @@ -1499,11 +1607,11 @@ finalize_restore_builtins(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; PyObject *dict = PyDict_Copy(interp->builtins); if (dict == NULL) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on restoring builtins"); } PyDict_Clear(interp->builtins); if (PyDict_Update(interp->builtins, interp->builtins_copy)) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on restoring builtins"); } Py_XDECREF(dict); } @@ -1559,6 +1667,17 @@ static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + +#ifdef _Py_TIER2 + // Invalidate all executors and turn off tier 2 optimizer + _Py_Executors_InvalidateAll(interp, 0); + _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + Py_XDECREF(old); +#endif + + // Stop watching __builtin__ modifications + PyDict_Unwatch(0, interp->builtins); + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -1665,7 +1784,7 @@ flush_std_files(void) if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { if (_PyFile_Flush(fout) < 0) { - PyErr_WriteUnraisable(fout); + PyErr_FormatUnraisable("Exception ignored on flushing sys.stdout"); status = -1; } } @@ -1698,11 +1817,11 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyTypes_FiniExtTypes(interp); _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); + _PyXI_FiniTypes(interp); _PyExc_Fini(interp); - _PyAsyncGen_Fini(interp); - _PyContext_Fini(interp); _PyFloat_FiniType(interp); _PyLong_FiniTypes(interp); _PyThread_FiniType(interp); @@ -1713,18 +1832,21 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_Fini(interp); + _PyCode_Fini(interp); + // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses // a dict internally. _PyUnicode_ClearInterned(interp); - _PyDict_Fini(interp); - _PyList_Fini(interp); - _PyTuple_Fini(interp); + _PyUnicode_Fini(interp); - _PySlice_Fini(interp); +#ifndef Py_GIL_DISABLED + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + _PyObject_ClearFreeLists(freelists, 1); +#endif - _PyUnicode_Fini(interp); - _PyFloat_Fini(interp); #ifdef Py_DEBUG _PyStaticObjects_CheckRefcnt(interp); #endif @@ -1736,8 +1858,10 @@ finalize_interp_clear(PyThreadState *tstate) { int is_main_interp = _Py_IsMainInterpreter(tstate->interp); + _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); + _PyDtoa_Fini(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1754,9 +1878,17 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); _PyPerfTrampoline_Fini(); + _PyPerfTrampoline_FreeArenas(); } finalize_interp_types(tstate->interp); + + /* Free any delayed free requests immediately */ + _PyMem_FiniDelayed(tstate->interp); + + /* finalize_interp_types may allocate Python objects so we may need to + abandon mimalloc segments again */ + _PyThreadState_ClearMimallocHeaps(tstate); } @@ -1776,20 +1908,73 @@ finalize_interp_delete(PyInterpreterState *interp) } -int -Py_FinalizeEx(void) +/* Conceptually, there isn't a good reason for Py_Finalize() + to be called in any other thread than the one where Py_Initialize() + was called. Consequently, it would make sense to fail if the thread + or thread state (or interpreter) don't match. However, such + constraints have never been enforced, and, as unlikely as it may be, + there may be users relying on the unconstrained behavior. Thus, + we do our best here to accommodate that possibility. */ + +static PyThreadState * +resolve_final_tstate(_PyRuntimeState *runtime) +{ + PyThreadState *main_tstate = runtime->main_tstate; + assert(main_tstate != NULL); + assert(main_tstate->thread_id == runtime->main_thread); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(main_tstate->interp == main_interp); + + PyThreadState *tstate = _PyThreadState_GET(); + if (_Py_IsMainThread()) { + if (tstate != main_tstate) { + /* This implies that Py_Finalize() was called while + a non-main interpreter was active or while the main + tstate was temporarily swapped out with another. + Neither case should be allowed, but, until we get around + to fixing that (and Py_Exit()), we're letting it go. */ + (void)PyThreadState_Swap(main_tstate); + } + } + else { + /* This is another unfortunate case where Py_Finalize() was + called when it shouldn't have been. We can't simply switch + over to the main thread. At the least, however, we can make + sure the main interpreter is active. */ + if (!_Py_IsMainInterpreter(tstate->interp)) { + /* We don't go to the trouble of updating runtime->main_tstate + since it will be dead soon anyway. */ + main_tstate = + _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); + if (main_tstate != NULL) { + _PyThreadState_Bind(main_tstate); + (void)PyThreadState_Swap(main_tstate); + } + else { + /* Fall back to the current tstate. It's better than nothing. */ + main_tstate = tstate; + } + } + } + assert(main_tstate != NULL); + + /* We might want to warn if main_tstate->current_frame != NULL. */ + + return main_tstate; +} + +static int +_Py_Finalize(_PyRuntimeState *runtime) { int status = 0; - _PyRuntimeState *runtime = &_PyRuntime; + /* Bail out early if already finalized (or never initialized). */ if (!runtime->initialized) { return status; } - /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyThreadState_GET(); - // XXX assert(_Py_IsMainInterpreter(tstate->interp)); - // XXX assert(_Py_IsMainThread()); + /* Get final thread state pointer. */ + PyThreadState *tstate = resolve_final_tstate(runtime); // Block some operations. tstate->interp->finalizing = 1; @@ -1811,7 +1996,8 @@ Py_FinalizeEx(void) */ _PyAtExit_Call(tstate->interp); - PyUnstable_PerfMapState_Fini(); + + assert(_PyThreadState_GET() == tstate); /* Copy the core config, PyInterpreterState_Delete() free the core config memory */ @@ -1826,6 +2012,9 @@ Py_FinalizeEx(void) int malloc_stats = tstate->interp->config.malloc_stats; #endif + /* Ensure that remaining threads are detached */ + _PyEval_StopTheWorldAll(runtime); + /* Remaining daemon threads will automatically exit when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(tstate->interp, tstate); @@ -1842,8 +2031,11 @@ Py_FinalizeEx(void) will be called in the current Python thread. Since _PyRuntimeState_SetFinalizing() has been called, no other Python thread can take the GIL at this point: if they try, they will exit - immediately. */ - _PyThreadState_DeleteExcept(tstate); + immediately. We start the world once we are the only thread state left, + before we call destructors. */ + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(runtime); + _PyThreadState_DeleteList(list); /* At this point no Python code should be running at all. The only thread state left should be the main thread of the main @@ -1887,6 +2079,9 @@ Py_FinalizeEx(void) _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); + /* Clean up any lingering subinterpreters. */ + finalize_subinterpreters(); + /* Print debug stats if any */ _PyEval_Fini(); @@ -1982,6 +2177,12 @@ Py_FinalizeEx(void) } #endif /* Py_TRACE_REFS */ +#ifdef WITH_PYMALLOC + if (malloc_stats) { + _PyObject_DebugMallocStats(stderr); + } +#endif + finalize_interp_delete(tstate->interp); #ifdef Py_REF_DEBUG @@ -1992,22 +2193,22 @@ Py_FinalizeEx(void) #endif _Py_FinalizeAllocatedBlocks(runtime); -#ifdef WITH_PYMALLOC - if (malloc_stats) { - _PyObject_DebugMallocStats(stderr); - } -#endif - call_ll_exitfuncs(runtime); _PyRuntime_Finalize(); return status; } +int +Py_FinalizeEx(void) +{ + return _Py_Finalize(&_PyRuntime); +} + void Py_Finalize(void) { - Py_FinalizeEx(); + (void)_Py_Finalize(&_PyRuntime); } @@ -2025,7 +2226,8 @@ Py_Finalize(void) */ static PyStatus -new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) +new_interpreter(PyThreadState **tstate_p, + const PyInterpreterConfig *config, long whence) { PyStatus status; @@ -2048,29 +2250,24 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) *tstate_p = NULL; return _PyStatus_OK(); } + _PyInterpreterState_SetWhence(interp, whence); + interp->_ready = 1; - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); - if (tstate == NULL) { - PyInterpreterState_Delete(interp); - *tstate_p = NULL; - return _PyStatus_OK(); - } - _PyThreadState_Bind(tstate); - + // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); - int has_gil = 0; + PyThreadState *tstate = NULL; /* From this point until the init_interp_create_gil() call, we must not do anything that requires that the GIL be held (or otherwise exist). That applies whether or not the new interpreter has its own GIL (e.g. the main interpreter). */ + if (save_tstate != NULL) { + _PyThreadState_Detach(save_tstate); + } /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { - // XXX Might new_interpreter() have been called without the GIL held? - _PyThreadState_Detach(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2092,11 +2289,22 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate, config->gil); - if (_PyStatus_EXCEPTION(status)) { + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } + + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); + if (tstate == NULL) { + status = _PyStatus_NO_MEMORY(); goto error; } - has_gil = 1; + + _PyThreadState_Bind(tstate); + init_interp_create_gil(tstate, config->gil); /* No objects have been created yet. */ @@ -2115,16 +2323,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) error: *tstate_p = NULL; - - /* Oops, it didn't work. Undo it all. */ - if (has_gil) { + if (tstate != NULL) { + PyThreadState_Clear(tstate); _PyThreadState_Detach(tstate); + PyThreadState_Delete(tstate); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyThreadState_Clear(tstate); - PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); return status; @@ -2134,15 +2340,17 @@ PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) { - return new_interpreter(tstate_p, config); + long whence = _PyInterpreterState_WHENCE_CAPI; + return new_interpreter(tstate_p, config, whence); } PyThreadState * Py_NewInterpreter(void) { PyThreadState *tstate = NULL; + long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI; const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = new_interpreter(&tstate, &config); + PyStatus status = new_interpreter(&tstate, &config, whence); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } @@ -2212,13 +2420,86 @@ _Py_IsInterpreterFinalizing(PyInterpreterState *interp) return finalizing != NULL; } +static void +finalize_subinterpreters(void) +{ + PyThreadState *final_tstate = _PyThreadState_GET(); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(final_tstate->interp == main_interp); + _PyRuntimeState *runtime = main_interp->runtime; + struct pyinterpreters *interpreters = &runtime->interpreters; + + /* Get the first interpreter in the list. */ + HEAD_LOCK(runtime); + PyInterpreterState *interp = interpreters->head; + if (interp == main_interp) { + interp = interp->next; + } + HEAD_UNLOCK(runtime); + + /* Bail out if there are no subinterpreters left. */ + if (interp == NULL) { + return; + } + + /* Warn the user if they forgot to clean up subinterpreters. */ + (void)PyErr_WarnEx( + PyExc_RuntimeWarning, + "remaining subinterpreters; " + "destroy them with _interpreters.destroy()", + 0); + + /* Swap out the current tstate, which we know must belong + to the main interpreter. */ + _PyThreadState_Detach(final_tstate); + + /* Clean up all remaining subinterpreters. */ + while (interp != NULL) { + assert(!_PyInterpreterState_IsRunningMain(interp)); + + /* Find the tstate to use for fini. We assume the interpreter + will have at most one tstate at this point. */ + PyThreadState *tstate = interp->threads.head; + if (tstate != NULL) { + /* Ideally we would be able to use tstate as-is, and rely + on it being in a ready state: no exception set, not + running anything (tstate->current_frame), matching the + current thread ID (tstate->thread_id). To play it safe, + we always delete it and use a fresh tstate instead. */ + assert(tstate != final_tstate); + _PyThreadState_Attach(tstate); + PyThreadState_Clear(tstate); + _PyThreadState_Detach(tstate); + PyThreadState_Delete(tstate); + } + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + + /* Destroy the subinterpreter. */ + _PyThreadState_Attach(tstate); + Py_EndInterpreter(tstate); + assert(_PyThreadState_GET() == NULL); + + /* Advance to the next interpreter. */ + HEAD_LOCK(runtime); + interp = interpreters->head; + if (interp == main_interp) { + interp = interp->next; + } + HEAD_UNLOCK(runtime); + } + + /* Switch back to the main interpreter. */ + _PyThreadState_Attach(final_tstate); +} + + /* Add the __main__ module */ static PyStatus add_main_module(PyInterpreterState *interp) { PyObject *m, *d, *ann_dict; - m = PyImport_AddModule("__main__"); + m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); @@ -2284,54 +2565,6 @@ init_import_site(void) return _PyStatus_OK(); } -/* Check if a file descriptor is valid or not. - Return 0 if the file descriptor is invalid, return non-zero otherwise. */ -static int -is_valid_fd(int fd) -{ -/* dup() is faster than fstat(): fstat() can require input/output operations, - whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python - startup. Problem: dup() doesn't check if the file descriptor is valid on - some platforms. - - fcntl(fd, F_GETFD) is even faster, because it only checks the process table. - It is preferred over dup() when available, since it cannot fail with the - "too many open files" error (EMFILE). - - bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other - side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with - EBADF. FreeBSD has similar issue (bpo-32849). - - Only use dup() on Linux where dup() is enough to detect invalid FD - (bpo-32849). -*/ - if (fd < 0) { - return 0; - } -#if defined(F_GETFD) && ( \ - defined(__linux__) || \ - defined(__APPLE__) || \ - defined(__wasm__)) - return fcntl(fd, F_GETFD) >= 0; -#elif defined(__linux__) - int fd2 = dup(fd); - if (fd2 >= 0) { - close(fd2); - } - return (fd2 >= 0); -#elif defined(MS_WINDOWS) - HANDLE hfile; - _Py_BEGIN_SUPPRESS_IPH - hfile = (HANDLE)_get_osfhandle(fd); - _Py_END_SUPPRESS_IPH - return (hfile != INVALID_HANDLE_VALUE - && GetFileType(hfile) != FILE_TYPE_UNKNOWN); -#else - struct stat st; - return (fstat(fd, &st) == 0); -#endif -} - /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(const PyConfig *config, PyObject* io, @@ -2345,8 +2578,9 @@ create_stdio(const PyConfig *config, PyObject* io, int buffering, isatty; const int buffered_stdio = config->buffered_stdio; - if (!is_valid_fd(fd)) + if (!_Py_IsValidFD(fd)) { Py_RETURN_NONE; + } /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -2462,9 +2696,9 @@ create_stdio(const PyConfig *config, PyObject* io, Py_XDECREF(text); Py_XDECREF(raw); - if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { /* Issue #24891: the file descriptor was closed after the first - is_valid_fd() check was called. Ignore the OSError and set the + _Py_IsValidFD() check was called. Ignore the OSError and set the stream to None. */ PyErr_Clear(); Py_RETURN_NONE; @@ -2603,6 +2837,73 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ +#include + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_android_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; +} + +#endif // __ANDROID__ + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) @@ -2734,6 +3035,30 @@ fatal_error_exit(int status) } } +static inline int +acquire_dict_lock_for_dump(PyObject *obj) +{ +#ifdef Py_GIL_DISABLED + PyMutex *mutex = &obj->ob_mutex; + if (_PyMutex_LockTimed(mutex, 0, 0) == PY_LOCK_ACQUIRED) { + return 1; + } + return 0; +#else + return 1; +#endif +} + +static inline void +release_dict_lock_for_dump(PyObject *obj) +{ +#ifdef Py_GIL_DISABLED + PyMutex *mutex = &obj->ob_mutex; + // We can not call PyMutex_Unlock because it's not async-signal-safe. + // So not to wake up other threads, we just use a simple atomic store in here. + _Py_atomic_store_uint8(&mutex->_bits, _Py_UNLOCKED); +#endif +} // Dump the list of extension modules of sys.modules, excluding stdlib modules // (sys.stdlib_module_names), into fd file descriptor. @@ -2761,13 +3086,18 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) PyObject *stdlib_module_names = NULL; if (interp->sysdict != NULL) { pos = 0; - while (PyDict_Next(interp->sysdict, &pos, &key, &value)) { + if (!acquire_dict_lock_for_dump(interp->sysdict)) { + // If we cannot acquire the lock, just don't dump the list of extension modules. + return; + } + while (_PyDict_Next(interp->sysdict, &pos, &key, &value, NULL)) { if (PyUnicode_Check(key) && PyUnicode_CompareWithASCIIString(key, "stdlib_module_names") == 0) { stdlib_module_names = value; break; } } + release_dict_lock_for_dump(interp->sysdict); } // If we failed to get sys.stdlib_module_names or it's not a frozenset, // don't exclude stdlib modules. @@ -2779,7 +3109,11 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) int header = 1; Py_ssize_t count = 0; pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { + if (!acquire_dict_lock_for_dump(modules)) { + // If we cannot acquire the lock, just don't dump the list of extension modules. + return; + } + while (_PyDict_Next(modules, &pos, &key, &value, NULL)) { if (!PyUnicode_Check(key)) { continue; } @@ -2794,6 +3128,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) Py_ssize_t i = 0; PyObject *item; Py_hash_t hash; + // if stdlib_module_names is not NULL, it is always a frozenset. while (_PySet_NextEntry(stdlib_module_names, &i, &item, &hash)) { if (PyUnicode_Check(item) && PyUnicode_Compare(key, item) == 0) @@ -2819,6 +3154,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) _Py_DumpASCII(fd, key); count++; } + release_dict_lock_for_dump(modules); if (count) { PUTS(fd, " (total: "); @@ -2994,14 +3330,14 @@ wait_for_thread_shutdown(PyThreadState *tstate) PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); if (threading == NULL) { if (_PyErr_Occurred(tstate)) { - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("Exception ignored on threading shutdown"); } /* else: threading not imported */ return; } result = PyObject_CallMethodNoArgs(threading, &_Py_ID(_shutdown)); if (result == NULL) { - PyErr_WriteUnraisable(threading); + PyErr_FormatUnraisable("Exception ignored on threading shutdown"); } else { Py_DECREF(result); @@ -3012,13 +3348,13 @@ wait_for_thread_shutdown(PyThreadState *tstate) int Py_AtExit(void (*func)(void)) { struct _atexit_runtime_state *state = &_PyRuntime.atexit; - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); if (state->ncallbacks >= NEXITFUNCS) { - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); return -1; } state->callbacks[state->ncallbacks++] = func; - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); return 0; } @@ -3028,18 +3364,18 @@ call_ll_exitfuncs(_PyRuntimeState *runtime) atexit_callbackfunc exitfunc; struct _atexit_runtime_state *state = &runtime->atexit; - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); while (state->ncallbacks > 0) { /* pop last function from the list */ state->ncallbacks--; exitfunc = state->callbacks[state->ncallbacks]; state->callbacks[state->ncallbacks] = NULL; - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); exitfunc(); - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); } - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); fflush(stdout); fflush(stderr); @@ -3048,7 +3384,11 @@ call_ll_exitfuncs(_PyRuntimeState *runtime) void _Py_NO_RETURN Py_Exit(int sts) { - if (Py_FinalizeEx() < 0) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { + _PyInterpreterState_SetNotRunningMain(tstate->interp); + } + if (_Py_Finalize(&_PyRuntime) < 0) { sts = 120; } diff --git a/Python/pystate.c b/Python/pystate.c index 92cf741f4ca4ed..602b13e18c71ae 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,13 +2,16 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" #include "pycore_code.h" // stats +#include "pycore_critical_section.h" // _PyCriticalSection_Resume() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_frame.h" #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() @@ -16,7 +19,7 @@ #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() /* -------------------------------------------------------------------------- CAUTION @@ -65,7 +68,7 @@ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL; #endif static inline PyThreadState * -current_fast_get(_PyRuntimeState *Py_UNUSED(runtime)) +current_fast_get(void) { #ifdef HAVE_THREAD_LOCAL return _Py_tss_tstate; @@ -99,14 +102,14 @@ current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) } #define tstate_verify_not_active(tstate) \ - if (tstate == current_fast_get((tstate)->interp->runtime)) { \ + if (tstate == current_fast_get()) { \ _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \ } PyThreadState * _PyThreadState_GetCurrent(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } @@ -236,6 +239,8 @@ tstate_is_bound(PyThreadState *tstate) static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); +static void tstate_mimalloc_bind(PyThreadState *); + static void bind_tstate(PyThreadState *tstate) { @@ -256,6 +261,15 @@ bind_tstate(PyThreadState *tstate) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + // Initialize biased reference counting inter-thread queue. Note that this + // needs to be initialized from the active thread. + _Py_brc_init_thread(tstate); +#endif + + // mimalloc state needs to be initialized from the active thread. + tstate_mimalloc_bind(tstate); + tstate->_status.bound = 1; } @@ -353,10 +367,9 @@ holds_gil(PyThreadState *tstate) // XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder // (and tstate->interp->runtime->ceval.gil.locked). assert(tstate != NULL); - _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - assert(tstate == gilstate_tss_get(runtime)); - return tstate == current_fast_get(runtime); + assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + return tstate == current_fast_get(); } @@ -379,50 +392,26 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP -#define NUMLOCKS 9 #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ - &(runtime)->xidregistry.mutex, \ - &(runtime)->getargs.mutex, \ - &(runtime)->unicode_state.ids.lock, \ + &(runtime)->xi.registry.mutex, \ + &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ - &(runtime)->ceval.pending_mainthread.lock, \ + &(runtime)->ceval.pending_mainthread.mutex, \ + &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ + &(runtime)->_main_interpreter.types.mutex, \ + &(runtime)->_main_interpreter.code_state.mutex, \ } -static int -alloc_for_runtime(PyThread_type_lock locks[NUMLOCKS]) -{ - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - for (int i = 0; i < NUMLOCKS; i++) { - PyThread_type_lock lock = PyThread_allocate_lock(); - if (lock == NULL) { - for (int j = 0; j < i; j++) { - PyThread_free_lock(locks[j]); - locks[j] = NULL; - } - break; - } - locks[i] = lock; - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return 0; -} - static void init_runtime(_PyRuntimeState *runtime, void *open_code_hook, void *open_code_userdata, _Py_AuditHookEntry *audit_hook_head, - Py_ssize_t unicode_next_index, - PyThread_type_lock locks[NUMLOCKS]) + Py_ssize_t unicode_next_index) { assert(!runtime->preinitializing); assert(!runtime->preinitialized); @@ -436,12 +425,6 @@ init_runtime(_PyRuntimeState *runtime, PyPreConfig_InitPythonConfig(&runtime->preconfig); - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - for (int i = 0; i < NUMLOCKS; i++) { - assert(locks[i] != NULL); - *lockptrs[i] = locks[i]; - } - // Set it to the ID of the main thread of the main interpreter. runtime->main_thread = PyThread_get_thread_ident(); @@ -467,11 +450,6 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // is called multiple times. Py_ssize_t unicode_next_index = runtime->unicode_state.ids.next_index; - PyThread_type_lock locks[NUMLOCKS]; - if (alloc_for_runtime(locks) != 0) { - return _PyStatus_NO_MEMORY(); - } - if (runtime->_initialized) { // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. @@ -490,13 +468,11 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) } init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head, - unicode_next_index, locks); + unicode_next_index); return _PyStatus_OK(); } -static void _xidregistry_clear(struct _xidregistry *); - void _PyRuntimeState_Fini(_PyRuntimeState *runtime) { @@ -505,8 +481,6 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif - _xidregistry_clear(&runtime->xidregistry); - if (gilstate_tss_initialized(runtime)) { gilstate_tss_fini(runtime); } @@ -514,23 +488,6 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) if (PyThread_tss_is_created(&runtime->trashTSSkey)) { PyThread_tss_delete(&runtime->trashTSSkey); } - - /* Force the allocator used by _PyRuntimeState_Init(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#define FREE_LOCK(LOCK) \ - if (LOCK != NULL) { \ - PyThread_free_lock(LOCK); \ - LOCK = NULL; \ - } - - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - for (int i = 0; i < NUMLOCKS; i++) { - FREE_LOCK(*lockptrs[i]); - } - -#undef FREE_LOCK - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } #ifdef HAVE_FORK @@ -542,33 +499,30 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) // This was initially set in _PyRuntimeState_Init(). runtime->main_thread = PyThread_get_thread_ident(); - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - int reinit_err = 0; - for (int i = 0; i < NUMLOCKS; i++) { - reinit_err += _PyThread_at_fork_reinit(lockptrs[i]); - } - /* PyOS_AfterFork_Child(), which calls this function, later calls - _PyInterpreterState_DeleteExceptMain(), so we only need to update - the main interpreter here. */ - assert(runtime->interpreters.main != NULL); - runtime->interpreters.main->xidregistry.mutex = runtime->xidregistry.mutex; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - // Clears the parking lot. Any waiting threads are dead. This must be // called before releasing any locks that use the parking lot. _PyParkingLot_AfterFork(); + // Re-initialize global locks + PyMutex *locks[] = LOCKS_INIT(runtime); + for (size_t i = 0; i < Py_ARRAY_LENGTH(locks); i++) { + _PyMutex_at_fork_reinit(locks[i]); + } +#ifdef Py_GIL_DISABLED + for (PyInterpreterState *interp = runtime->interpreters.head; + interp != NULL; interp = interp->next) + { + for (int i = 0; i < NUM_WEAKREF_LIST_LOCKS; i++) { + _PyMutex_at_fork_reinit(&interp->weakref_locks[i]); + } + } +#endif + + _PyTypes_AfterFork(); + /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ - reinit_err += _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); - - if (reinit_err < 0) { + if (_PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex) < 0) { return _PyStatus_ERR("Failed to reinitialize runtime locks"); } @@ -584,6 +538,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } + _PyThread_AfterFork(&runtime->threads); + return _PyStatus_OK(); } #endif @@ -604,24 +560,6 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) { struct pyinterpreters *interpreters = &runtime->interpreters; interpreters->next_id = 0; - - /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. - Create a new mutex if needed. */ - if (interpreters->mutex == NULL) { - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - interpreters->mutex = PyThread_allocate_lock(); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - if (interpreters->mutex == NULL) { - return _PyStatus_ERR("Can't initialize threads for interpreter"); - } - } - return _PyStatus_OK(); } @@ -638,10 +576,17 @@ free_interpreter(PyInterpreterState *interp) // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { + if (_PyMem_obmalloc_state_on_heap(interp)) { + // interpreter has its own obmalloc state, free it + PyMem_RawFree(interp->obmalloc); + interp->obmalloc = NULL; + } PyMem_RawFree(interp); } } - +#ifndef NDEBUG +static inline int check_interpreter_whence(long); +#endif /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -665,12 +610,16 @@ static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, PyInterpreterState *next, - PyThread_type_lock pending_lock) + long whence) { if (interp->_initialized) { return _PyStatus_ERR("interpreter already initialized"); } + assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; + assert(runtime != NULL); interp->runtime = runtime; @@ -681,23 +630,19 @@ init_interpreter(PyInterpreterState *interp, assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; - /* Initialize obmalloc, but only for subinterpreters, - since the main interpreter is initialized statically. */ - if (interp != &runtime->_main_interpreter) { - poolp temp[OBMALLOC_USED_POOLS_SIZE] = \ - _obmalloc_pools_INIT(interp->obmalloc.pools); - memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); - } - PyStatus status = _PyObject_InitState(interp); if (_PyStatus_EXCEPTION(status)) { return status; } - _PyEval_InitState(interp, pending_lock); + _PyEval_InitState(interp); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); +#ifdef Py_GIL_DISABLED + _Py_brc_init_state(interp); +#endif + llist_init(&interp->mem_free_queue.head); for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } @@ -709,18 +654,14 @@ init_interpreter(PyInterpreterState *interp, } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; - interp->optimizer = &_PyOptimizer_Default; - interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; - interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; - interp->next_func_version = 1; +#ifdef _Py_TIER2 + (void)_Py_SetOptimizer(interp, NULL); + interp->executor_list_head = NULL; +#endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } - interp->f_opcode_trace_set = false; - - assert(runtime->xidregistry.mutex != NULL); - interp->xidregistry.mutex = runtime->xidregistry.mutex; interp->_initialized = 1; return _PyStatus_OK(); @@ -743,11 +684,6 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) } } - PyThread_type_lock pending_lock = PyThread_allocate_lock(); - if (pending_lock == NULL) { - return _PyStatus_NO_MEMORY(); - } - /* We completely serialize creation of multiple interpreters, since it simplifies things here and blocking concurrent calls isn't a problem. Regardless, we must fully block subinterpreter creation until @@ -793,12 +729,12 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) } interpreters->head = interp; + long whence = _PyInterpreterState_WHENCE_UNKNOWN; status = init_interpreter(interp, runtime, - id, old_head, pending_lock); + id, old_head, whence); if (_PyStatus_EXCEPTION(status)) { goto error; } - pending_lock = NULL; HEAD_UNLOCK(runtime); @@ -809,9 +745,6 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) error: HEAD_UNLOCK(runtime); - if (pending_lock != NULL) { - PyThread_free_lock(pending_lock); - } if (interp != NULL) { free_interpreter(interp); } @@ -823,7 +756,7 @@ PyInterpreterState * PyInterpreterState_New(void) { // tstate can be NULL - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); PyInterpreterState *interp; PyStatus status = _PyInterpreterState_New(tstate, &interp); @@ -876,10 +809,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) tstate->_status.cleared = 0; } - Py_CLEAR(interp->optimizer); - interp->optimizer = &_PyOptimizer_Default; - interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; - interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; +#ifdef _Py_TIER2 + _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + assert(old != NULL); + Py_DECREF(old); +#endif /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state @@ -890,9 +824,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - // At this time, all the threads should be cleared so we don't need - // atomic operations for eval_breaker - interp->ceval.eval_breaker = 0; + // At this time, all the threads should be cleared so we don't need atomic + // operations for instrumentation_version or eval_breaker. + interp->ceval.instrumentation_version = 0; + tstate->eval_breaker = 0; for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; @@ -909,9 +844,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } PyConfig_Clear(&interp->config); - Py_CLEAR(interp->codec_search_path); - Py_CLEAR(interp->codec_search_cache); - Py_CLEAR(interp->codec_error_registry); + _PyCodec_Fini(interp); assert(interp->imports.modules == NULL); assert(interp->imports.modules_by_index == NULL); @@ -947,10 +880,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); - _xidregistry_clear(&interp->xidregistry); - /* The lock is owned by the runtime, so we don't free it here. */ - interp->xidregistry.mutex = NULL; - if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? @@ -974,7 +903,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; - interp->f_opcode_trace_set = false; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. @@ -987,7 +915,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // Use the current Python thread state to call audit hooks and to collect // garbage. It can be different than the current Python thread state // of 'interp'. - PyThreadState *current_tstate = current_fast_get(interp->runtime); + PyThreadState *current_tstate = current_fast_get(); _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -1002,7 +930,7 @@ _PyInterpreterState_Clear(PyThreadState *tstate) static inline void tstate_deactivate(PyThreadState *tstate); -static void tstate_set_detached(PyThreadState *tstate); +static void tstate_set_detached(PyThreadState *tstate, int detached_state); static void zapthreads(PyInterpreterState *interp); void @@ -1013,7 +941,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // XXX Clearing the "current" thread state should happen before // we start finalizing the interpreter (or the current thread state). - PyThreadState *tcur = current_fast_get(runtime); + PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); @@ -1021,8 +949,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) zapthreads(interp); - _PyEval_FiniState(&interp->ceval); - // XXX These two calls should be done at the end of clear_interpreter(), // but currently some objects get decref'ed after that. #ifdef Py_REF_DEBUG @@ -1057,6 +983,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyThread_free_lock(interp->id_mutex); } + _Py_qsbr_fini(interp); + _PyObject_FiniState(interp); free_interpreter(interp); @@ -1110,6 +1038,17 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) } #endif +static inline void +set_main_thread(PyInterpreterState *interp, PyThreadState *tstate) +{ + _Py_atomic_store_ptr_relaxed(&interp->threads.main, tstate); +} + +static inline PyThreadState * +get_main_thread(PyInterpreterState *interp) +{ + return _Py_atomic_load_ptr_relaxed(&interp->threads.main); +} int _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) @@ -1117,59 +1056,174 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { return -1; } - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); if (tstate->interp != interp) { PyErr_SetString(PyExc_RuntimeError, "current tstate has wrong interpreter"); return -1; } - interp->threads.main = tstate; + set_main_thread(interp, tstate); + return 0; } void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { - PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get(&_PyRuntime)); + assert(get_main_thread(interp) == current_fast_get()); + set_main_thread(interp, NULL); +} - if (tstate->on_delete != NULL) { - // The threading module was imported for the first time in this - // thread, so it was set as threading._main_thread. (See gh-75698.) - // The thread has finished running the Python program so we mark - // the thread object as finished. - assert(tstate->_whence != _PyThreadState_WHENCE_THREADING); - tstate->on_delete(tstate->on_delete_data); - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; +int +_PyInterpreterState_IsRunningMain(PyInterpreterState *interp) +{ + if (get_main_thread(interp) != NULL) { + return 1; } - - interp->threads.main = NULL; + // Embedders might not know to call _PyInterpreterState_SetRunningMain(), + // so their main thread wouldn't show it is running the main interpreter's + // program. (Py_Main() doesn't have this problem.) For now this isn't + // critical. If it were, we would need to infer "running main" from other + // information, like if it's the main interpreter. We used to do that + // but the naive approach led to some inconsistencies that caused problems. + return 0; } int -_PyInterpreterState_IsRunningMain(PyInterpreterState *interp) +_PyThreadState_IsRunningMain(PyThreadState *tstate) { - return (interp->threads.main != NULL); + PyInterpreterState *interp = tstate->interp; + // See the note in _PyInterpreterState_IsRunningMain() about + // possible false negatives here for embedders. + return get_main_thread(interp) == tstate; } int _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) { - if (interp->threads.main != NULL) { - PyErr_SetString(PyExc_RuntimeError, + if (get_main_thread(interp) != NULL) { + PyErr_SetString(PyExc_InterpreterError, "interpreter already running"); return -1; } return 0; } +void +_PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + if (get_main_thread(interp) != tstate) { + set_main_thread(interp, NULL); + } +} + //---------- // accessors //---------- +int +_PyInterpreterState_IsReady(PyInterpreterState *interp) +{ + return interp->_ready; +} + +#ifndef NDEBUG +static inline int +check_interpreter_whence(long whence) +{ + if(whence < 0) { + return -1; + } + if (whence > _PyInterpreterState_WHENCE_MAX) { + return -1; + } + return 0; +} +#endif + +long +_PyInterpreterState_GetWhence(PyInterpreterState *interp) +{ + assert(check_interpreter_whence(interp->_whence) == 0); + return interp->_whence; +} + +void +_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence) +{ + assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; +} + + +PyObject * +PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) +{ + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); + return NULL; + } + return PyMapping_GetItemString(modules, "__main__"); +} + + +PyObject * +PyInterpreterState_GetDict(PyInterpreterState *interp) +{ + if (interp->dict == NULL) { + interp->dict = PyDict_New(); + if (interp->dict == NULL) { + PyErr_Clear(); + } + } + /* Returning NULL means no per-interpreter dict is available. */ + return interp->dict; +} + + +//---------- +// interp ID +//---------- + +int64_t +_PyInterpreterState_ObjectToID(PyObject *idobj) +{ + if (!_PyIndex_Check(idobj)) { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(idobj)->tp_name); + return -1; + } + + // This may raise OverflowError. + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + long long id = PyLong_AsLongLong(idobj); + if (id == -1 && PyErr_Occurred()) { + return -1; + } + + if (id < 0) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } +#if LLONG_MAX > INT64_MAX + else if (id > INT64_MAX) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + return -1; + } +#endif + else { + return (int64_t)id; + } +} + int64_t PyInterpreterState_GetID(PyInterpreterState *interp) { @@ -1180,6 +1234,20 @@ PyInterpreterState_GetID(PyInterpreterState *interp) return interp->id; } +PyObject * +_PyInterpreterState_GetIDObject(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t interpid = interp->id; + if (interpid < 0) { + return NULL; + } + assert(interpid < LLONG_MAX); + return PyLong_FromLongLong(interpid); +} + int _PyInterpreterState_IDInitref(PyInterpreterState *interp) @@ -1225,9 +1293,8 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) PyThread_release_lock(interp->id_mutex); if (refcount == 0 && interp->requires_idref) { - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); - _PyThreadState_Bind(tstate); + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); // XXX Possible GILState issues? PyThreadState *save_tstate = _PyThreadState_Swap(runtime, tstate); @@ -1248,30 +1315,6 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) interp->requires_idref = required ? 1 : 0; } -PyObject * -PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) -{ - PyObject *modules = _PyImport_GetModules(interp); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); - return NULL; - } - return PyMapping_GetItemString(modules, "__main__"); -} - -PyObject * -PyInterpreterState_GetDict(PyInterpreterState *interp) -{ - if (interp->dict == NULL) { - interp->dict = PyDict_New(); - if (interp->dict == NULL) { - PyErr_Clear(); - } - } - /* Returning NULL means no per-interpreter dict is available. */ - return interp->dict; -} - //----------------------------- // look up an interpreter state @@ -1285,7 +1328,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) PyInterpreterState* PyInterpreterState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1327,12 +1370,22 @@ _PyInterpreterState_LookUpID(int64_t requested_id) HEAD_UNLOCK(runtime); } if (interp == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_InterpreterNotFoundError, "unrecognized interpreter ID %lld", requested_id); } return interp; } +PyInterpreterState * +_PyInterpreterState_LookUpIDObject(PyObject *requested_id) +{ + int64_t id = _PyInterpreterState_ObjectToID(requested_id); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + /********************************/ /* the per-thread runtime state */ @@ -1371,20 +1424,19 @@ allocate_chunk(int size_in_bytes, _PyStackChunk* previous) return res; } -static PyThreadState * +static _PyThreadStateImpl * alloc_threadstate(void) { - return PyMem_RawCalloc(1, sizeof(PyThreadState)); + return PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); } static void -free_threadstate(PyThreadState *tstate) +free_threadstate(_PyThreadStateImpl *tstate) { // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate == &tstate->interp->_initial_thread) { + if (tstate == &tstate->base.interp->_initial_thread) { // Restore to _PyThreadState_INIT. - tstate = &tstate->interp->_initial_thread; memcpy(tstate, &initial._main_interpreter._initial_thread, sizeof(*tstate)); @@ -1403,15 +1455,18 @@ free_threadstate(PyThreadState *tstate) */ static void -init_threadstate(PyThreadState *tstate, +init_threadstate(_PyThreadStateImpl *_tstate, PyInterpreterState *interp, uint64_t id, int whence) { + PyThreadState *tstate = (PyThreadState *)_tstate; if (tstate->_status.initialized) { Py_FatalError("thread state already initialized"); } assert(interp != NULL); tstate->interp = interp; + tstate->eval_breaker = + _Py_atomic_load_uintptr_relaxed(&interp->ceval.instrumentation_version); // next/prev are set in add_threadstate(). assert(tstate->next == NULL); @@ -1441,6 +1496,17 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->what_event = -1; + tstate->previous_executor = NULL; + tstate->dict_global_version = 0; + + tstate->delete_later = NULL; + + llist_init(&_tstate->mem_free_queue); + + if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { + // Start in the suspended state if there is an ongoing stop-the-world. + tstate->state = _Py_THREAD_SUSPENDED; + } tstate->_status.initialized = 1; } @@ -1462,17 +1528,25 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, static PyThreadState * new_threadstate(PyInterpreterState *interp, int whence) { - PyThreadState *tstate; + _PyThreadStateImpl *tstate; _PyRuntimeState *runtime = interp->runtime; // We don't need to allocate a thread state for the main interpreter // (the common case), but doing it later for the other case revealed a // reentrancy problem (deadlock). So for now we always allocate before // taking the interpreters lock. See GH-96071. - PyThreadState *new_tstate = alloc_threadstate(); + _PyThreadStateImpl *new_tstate = alloc_threadstate(); int used_newtstate; if (new_tstate == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); + if (qsbr_idx < 0) { + PyMem_RawFree(new_tstate); + return NULL; + } +#endif + /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(runtime); @@ -1500,21 +1574,41 @@ new_threadstate(PyInterpreterState *interp, int whence) } init_threadstate(tstate, interp, id, whence); - add_threadstate(interp, tstate, old_head); + add_threadstate(interp, (PyThreadState *)tstate, old_head); HEAD_UNLOCK(runtime); if (!used_newtstate) { // Must be called with lock unlocked to avoid re-entrancy deadlock. PyMem_RawFree(new_tstate); } - return tstate; + else { +#ifdef Py_GIL_DISABLED + if (_Py_atomic_load_int(&interp->gc.immortalize) == 0) { + // Immortalize objects marked as using deferred reference counting + // the first time a non-main thread is created. + _PyGC_ImmortalizeDeferredObjects(interp); + } +#endif + } + +#ifdef Py_GIL_DISABLED + // Must be called with lock unlocked to avoid lock ordering deadlocks. + _Py_qsbr_register(tstate, interp, qsbr_idx); +#endif + + return (PyThreadState *)tstate; } PyThreadState * PyThreadState_New(PyInterpreterState *interp) { - PyThreadState *tstate = new_threadstate(interp, - _PyThreadState_WHENCE_UNKNOWN); + return _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_UNKNOWN); +} + +PyThreadState * +_PyThreadState_NewBound(PyInterpreterState *interp, int whence) +{ + PyThreadState *tstate = new_threadstate(interp, whence); if (tstate) { bind_tstate(tstate); // This makes sure there's a gilstate tstate bound @@ -1565,6 +1659,8 @@ void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); + assert(current_fast_get()->interp == tstate->interp); + assert(!_PyThreadState_IsRunningMain(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1633,15 +1729,19 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); - if (tstate->on_delete != NULL) { - // For the "main" thread of each interpreter, this is meant - // to be done in _PyInterpreterState_SetNotRunningMain(). - // That leaves threads created by the threading module, - // and any threads killed by forking. - // However, we also accommodate "main" threads that still - // don't call _PyInterpreterState_SetNotRunningMain() yet. - tstate->on_delete(tstate->on_delete_data); - } +#ifdef Py_GIL_DISABLED + // Each thread should clear own freelists in free-threading builds. + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + _PyObject_ClearFreeLists(freelists, 1); + + // Remove ourself from the biased reference counting table of threads. + _Py_brc_remove_thread(tstate); +#endif + + // Merge our queue of pointers to be freed into the interpreter queue. + _PyMem_AbandonDelayed(tstate); + + _PyThreadState_ClearMimallocHeaps(tstate); tstate->_status.cleared = 1; @@ -1649,12 +1749,16 @@ PyThreadState_Clear(PyThreadState *tstate) // XXX Do it as early in the function as possible. } +static void +decrement_stoptheworld_countdown(struct _stoptheworld_state *stw); + /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); - assert(tstate->state != _Py_THREAD_ATTACHED); + tstate_verify_not_active(tstate); + assert(!_PyThreadState_IsRunningMain(tstate)); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1672,6 +1776,24 @@ tstate_delete_common(PyThreadState *tstate) if (tstate->next) { tstate->next->prev = tstate->prev; } + if (tstate->state != _Py_THREAD_SUSPENDED) { + // Any ongoing stop-the-world request should not wait for us because + // our thread is getting deleted. + if (interp->stoptheworld.requested) { + decrement_stoptheworld_countdown(&interp->stoptheworld); + } + if (runtime->stoptheworld.requested) { + decrement_stoptheworld_countdown(&runtime->stoptheworld); + } + } + +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + // Add our portion of the total refcount to the interpreter's total. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate->interp->object_state.reftotal += tstate_impl->reftotal; + tstate_impl->reftotal = 0; +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier @@ -1684,6 +1806,14 @@ tstate_delete_common(PyThreadState *tstate) // XXX Move to PyThreadState_Clear()? clear_datastack(tstate); + if (release_gil) { + _PyEval_ReleaseLock(tstate->interp, tstate, 1); + } + +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister(tstate); +#endif + tstate->_status.finalized = 1; } @@ -1695,8 +1825,8 @@ zapthreads(PyInterpreterState *interp) when the threads are all really dead (XXX famous last words). */ while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); - tstate_delete_common(tstate); - free_threadstate(tstate); + tstate_delete_common(tstate, 0); + free_threadstate((_PyThreadStateImpl *)tstate); } } @@ -1706,8 +1836,8 @@ PyThreadState_Delete(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); tstate_verify_not_active(tstate); - tstate_delete_common(tstate); - free_threadstate(tstate); + tstate_delete_common(tstate, 0); + free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1715,39 +1845,45 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - tstate_set_detached(tstate); - tstate_delete_common(tstate); +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif current_fast_clear(tstate->interp->runtime); - _PyEval_ReleaseLock(tstate->interp, NULL); - free_threadstate(tstate); + tstate_delete_common(tstate, 1); // release GIL as part of call + free_threadstate((_PyThreadStateImpl *)tstate); } void PyThreadState_DeleteCurrent(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _PyThreadState_DeleteCurrent(tstate); } -/* - * Delete all thread states except the one passed as argument. - * Note that, if there is a current thread state, it *must* be the one - * passed as argument. Also, this won't touch any other interpreters - * than the current one, since we don't know which thread state should - * be kept in those other interpreters. - */ -void -_PyThreadState_DeleteExcept(PyThreadState *tstate) +// Unlinks and removes all thread states from `tstate->interp`, with the +// exception of the one passed as an argument. However, it does not delete +// these thread states. Instead, it returns the removed thread states as a +// linked list. +// +// Note that if there is a current thread state, it *must* be the one +// passed as argument. Also, this won't touch any interpreters other +// than the current one, since we don't know which thread state should +// be kept in those other interpreters. +PyThreadState * +_PyThreadState_RemoveExcept(PyThreadState *tstate) { assert(tstate != NULL); PyInterpreterState *interp = tstate->interp; _PyRuntimeState *runtime = interp->runtime; +#ifdef Py_GIL_DISABLED + assert(runtime->stoptheworld.world_stopped); +#endif + HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of - thread states. This will allow calling PyThreadState_Clear() - without holding the lock. */ + thread states. */ PyThreadState *list = interp->threads.head; if (list == tstate) { list = tstate->next; @@ -1762,14 +1898,24 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) interp->threads.head = tstate; HEAD_UNLOCK(runtime); - /* Clear and deallocate all stale thread states. Even if this - executes Python code, we should be safe since it executes - in the current thread, not one of the stale threads. */ + return list; +} + +// Deletes the thread states in the linked list `list`. +// +// This is intended to be used in conjunction with _PyThreadState_RemoveExcept. +void +_PyThreadState_DeleteList(PyThreadState *list) +{ + // The world can't be stopped because we PyThreadState_Clear() can + // call destructors. + assert(!_PyRuntime.stoptheworld.world_stopped); + PyThreadState *p, *next; for (p = list; p; p = next) { next = p->next; PyThreadState_Clear(p); - free_threadstate(p); + free_threadstate((_PyThreadStateImpl *)p); } } @@ -1801,7 +1947,7 @@ _PyThreadState_GetDict(PyThreadState *tstate) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return NULL; } @@ -1875,15 +2021,11 @@ tstate_deactivate(PyThreadState *tstate) static int tstate_try_attach(PyThreadState *tstate) { -#ifdef Py_NOGIL +#ifdef Py_GIL_DISABLED int expected = _Py_THREAD_DETACHED; - if (_Py_atomic_compare_exchange_int( - &tstate->state, - &expected, - _Py_THREAD_ATTACHED)) { - return 1; - } - return 0; + return _Py_atomic_compare_exchange_int(&tstate->state, + &expected, + _Py_THREAD_ATTACHED); #else assert(tstate->state == _Py_THREAD_DETACHED); tstate->state = _Py_THREAD_ATTACHED; @@ -1892,16 +2034,30 @@ tstate_try_attach(PyThreadState *tstate) } static void -tstate_set_detached(PyThreadState *tstate) +tstate_set_detached(PyThreadState *tstate, int detached_state) { - assert(tstate->state == _Py_THREAD_ATTACHED); -#ifdef Py_NOGIL - _Py_atomic_store_int(&tstate->state, _Py_THREAD_DETACHED); + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); +#ifdef Py_GIL_DISABLED + _Py_atomic_store_int(&tstate->state, detached_state); #else - tstate->state = _Py_THREAD_DETACHED; + tstate->state = detached_state; #endif } +static void +tstate_wait_attach(PyThreadState *tstate) +{ + do { + int expected = _Py_THREAD_SUSPENDED; + + // Wait until we're switched out of SUSPENDED to DETACHED. + _PyParkingLot_Park(&tstate->state, &expected, sizeof(tstate->state), + /*timeout=*/-1, NULL, /*detach=*/0); + + // Once we're back in DETACHED we can re-attach + } while (!tstate_try_attach(tstate)); +} + void _PyThreadState_Attach(PyThreadState *tstate) { @@ -1912,21 +2068,42 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif _Py_EnsureTstateNotNULL(tstate); - if (current_fast_get(&_PyRuntime) != NULL) { + if (current_fast_get() != NULL) { Py_FatalError("non-NULL old thread state"); } - _PyEval_AcquireLock(tstate); - // XXX assert(tstate_is_alive(tstate)); - current_fast_set(&_PyRuntime, tstate); - tstate_activate(tstate); + while (1) { + _PyEval_AcquireLock(tstate); - if (!tstate_try_attach(tstate)) { - // TODO: Once stop-the-world GC is implemented for --disable-gil builds - // this will need to wait until the GC completes. For now, this case - // should never happen. - Py_FatalError("thread attach failed"); + // XXX assert(tstate_is_alive(tstate)); + current_fast_set(&_PyRuntime, tstate); + tstate_activate(tstate); + + if (!tstate_try_attach(tstate)) { + tstate_wait_attach(tstate); + } + +#ifdef Py_GIL_DISABLED + if (_PyEval_IsGILEnabled(tstate) && !tstate->_status.holds_gil) { + // The GIL was enabled between our call to _PyEval_AcquireLock() + // and when we attached (the GIL can't go from enabled to disabled + // here because only a thread holding the GIL can disable + // it). Detach and try again. + tstate_set_detached(tstate, _Py_THREAD_DETACHED); + tstate_deactivate(tstate); + current_fast_clear(&_PyRuntime); + continue; + } + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif + break; + } + + // Resume previous critical section. This acquires the lock(s) from the + // top-most critical section. + if (tstate->critical_section != 0) { + _PyCriticalSection_Resume(tstate); } #if defined(Py_DEBUG) @@ -1934,16 +2111,239 @@ _PyThreadState_Attach(PyThreadState *tstate) #endif } -void -_PyThreadState_Detach(PyThreadState *tstate) +static void +detach_thread(PyThreadState *tstate, int detached_state) { // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); - assert(tstate->state == _Py_THREAD_ATTACHED); - assert(tstate == current_fast_get(&_PyRuntime)); - tstate_set_detached(tstate); + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); + assert(tstate == current_fast_get()); + if (tstate->critical_section != 0) { + _PyCriticalSection_SuspendAll(tstate); + } +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_deactivate(tstate); + tstate_set_detached(tstate, detached_state); current_fast_clear(&_PyRuntime); - _PyEval_ReleaseLock(tstate->interp, tstate); + _PyEval_ReleaseLock(tstate->interp, tstate, 0); +} + +void +_PyThreadState_Detach(PyThreadState *tstate) +{ + detach_thread(tstate, _Py_THREAD_DETACHED); +} + +void +_PyThreadState_Suspend(PyThreadState *tstate) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); + + struct _stoptheworld_state *stw = NULL; + HEAD_LOCK(runtime); + if (runtime->stoptheworld.requested) { + stw = &runtime->stoptheworld; + } + else if (tstate->interp->stoptheworld.requested) { + stw = &tstate->interp->stoptheworld; + } + HEAD_UNLOCK(runtime); + + if (stw == NULL) { + // Switch directly to "detached" if there is no active stop-the-world + // request. + detach_thread(tstate, _Py_THREAD_DETACHED); + return; + } + + // Switch to "suspended" state. + detach_thread(tstate, _Py_THREAD_SUSPENDED); + + // Decrease the count of remaining threads needing to park. + HEAD_LOCK(runtime); + decrement_stoptheworld_countdown(stw); + HEAD_UNLOCK(runtime); +} + +// Decrease stop-the-world counter of remaining number of threads that need to +// pause. If we are the final thread to pause, notify the requesting thread. +static void +decrement_stoptheworld_countdown(struct _stoptheworld_state *stw) +{ + assert(stw->thread_countdown > 0); + if (--stw->thread_countdown == 0) { + _PyEvent_Notify(&stw->stop_event); + } +} + +#ifdef Py_GIL_DISABLED +// Interpreter for _Py_FOR_EACH_THREAD(). For global stop-the-world events, +// we start with the first interpreter and then iterate over all interpreters. +// For per-interpreter stop-the-world events, we only operate on the one +// interpreter. +static PyInterpreterState * +interp_for_stop_the_world(struct _stoptheworld_state *stw) +{ + return (stw->is_global + ? PyInterpreterState_Head() + : _Py_CONTAINER_OF(stw, PyInterpreterState, stoptheworld)); +} + +// Loops over threads for a stop-the-world event. +// For global: all threads in all interpreters +// For per-interpreter: all threads in the interpreter +#define _Py_FOR_EACH_THREAD(stw, i, t) \ + for (i = interp_for_stop_the_world((stw)); \ + i != NULL; i = ((stw->is_global) ? i->next : NULL)) \ + for (t = i->threads.head; t; t = t->next) + + +// Try to transition threads atomically from the "detached" state to the +// "gc stopped" state. Returns true if all threads are in the "gc stopped" +static bool +park_detached_threads(struct _stoptheworld_state *stw) +{ + int num_parked = 0; + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + int state = _Py_atomic_load_int_relaxed(&t->state); + if (state == _Py_THREAD_DETACHED) { + // Atomically transition to "suspended" if in "detached" state. + if (_Py_atomic_compare_exchange_int(&t->state, + &state, _Py_THREAD_SUSPENDED)) { + num_parked++; + } + } + else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { + _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT); + } + } + stw->thread_countdown -= num_parked; + assert(stw->thread_countdown >= 0); + return num_parked > 0 && stw->thread_countdown == 0; +} + +static void +stop_the_world(struct _stoptheworld_state *stw) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + PyMutex_Lock(&stw->mutex); + if (stw->is_global) { + _PyRWMutex_Lock(&runtime->stoptheworld_mutex); + } + else { + _PyRWMutex_RLock(&runtime->stoptheworld_mutex); + } + + HEAD_LOCK(runtime); + stw->requested = 1; + stw->thread_countdown = 0; + stw->stop_event = (PyEvent){0}; // zero-initialize (unset) + stw->requester = _PyThreadState_GET(); // may be NULL + + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + if (t != stw->requester) { + // Count all the other threads (we don't wait on ourself). + stw->thread_countdown++; + } + } + + if (stw->thread_countdown == 0) { + HEAD_UNLOCK(runtime); + stw->world_stopped = 1; + return; + } + + for (;;) { + // Switch threads that are detached to the GC stopped state + bool stopped_all_threads = park_detached_threads(stw); + HEAD_UNLOCK(runtime); + + if (stopped_all_threads) { + break; + } + + PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) + int detach = 0; + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) { + assert(stw->thread_countdown == 0); + break; + } + + HEAD_LOCK(runtime); + } + stw->world_stopped = 1; +} + +static void +start_the_world(struct _stoptheworld_state *stw) +{ + _PyRuntimeState *runtime = &_PyRuntime; + assert(PyMutex_IsLocked(&stw->mutex)); + + HEAD_LOCK(runtime); + stw->requested = 0; + stw->world_stopped = 0; + // Switch threads back to the detached state. + PyInterpreterState *i; + PyThreadState *t; + _Py_FOR_EACH_THREAD(stw, i, t) { + if (t != stw->requester) { + assert(_Py_atomic_load_int_relaxed(&t->state) == + _Py_THREAD_SUSPENDED); + _Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED); + _PyParkingLot_UnparkAll(&t->state); + } + } + stw->requester = NULL; + HEAD_UNLOCK(runtime); + if (stw->is_global) { + _PyRWMutex_Unlock(&runtime->stoptheworld_mutex); + } + else { + _PyRWMutex_RUnlock(&runtime->stoptheworld_mutex); + } + PyMutex_Unlock(&stw->mutex); +} +#endif // Py_GIL_DISABLED + +void +_PyEval_StopTheWorldAll(_PyRuntimeState *runtime) +{ +#ifdef Py_GIL_DISABLED + stop_the_world(&runtime->stoptheworld); +#endif +} + +void +_PyEval_StartTheWorldAll(_PyRuntimeState *runtime) +{ +#ifdef Py_GIL_DISABLED + start_the_world(&runtime->stoptheworld); +#endif +} + +void +_PyEval_StopTheWorld(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + stop_the_world(&interp->stoptheworld); +#endif +} + +void +_PyEval_StartTheWorld(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + start_the_world(&interp->stoptheworld); +#endif } //---------- @@ -1985,19 +2385,18 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) * deadlock, we need to release head_mutex before * the decref. */ - PyObject *old_exc = tstate->async_exc; - tstate->async_exc = Py_XNewRef(exc); + Py_XINCREF(exc); + PyObject *old_exc = _Py_atomic_exchange_ptr(&tstate->async_exc, exc); HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); - _PyEval_SignalAsyncExc(tstate->interp); + _Py_set_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); return 1; } HEAD_UNLOCK(runtime); return 0; } - //--------------------------------- // API for the current thread state //--------------------------------- @@ -2005,14 +2404,14 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) PyThreadState * PyThreadState_GetUnchecked(void) { - return current_fast_get(&_PyRuntime); + return current_fast_get(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = current_fast_get(&_PyRuntime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return tstate; } @@ -2020,7 +2419,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) { - PyThreadState *oldts = current_fast_get(runtime); + PyThreadState *oldts = current_fast_get(); if (oldts != NULL) { _PyThreadState_Detach(oldts); } @@ -2052,6 +2451,20 @@ _PyThreadState_Bind(PyThreadState *tstate) } } +#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +uintptr_t +_Py_GetThreadLocal_Addr(void) +{ +#ifdef HAVE_THREAD_LOCAL + // gh-112535: Use the address of the thread-local PyThreadState variable as + // a unique identifier for the current thread. Each thread has a unique + // _Py_tss_tstate variable with a unique address. + return (uintptr_t)&_Py_tss_tstate; +#else +# error "no supported thread-local variable storage classifier" +#endif +} +#endif /***********************************/ /* routines for advanced debuggers */ @@ -2101,7 +2514,7 @@ PyObject * _PyThread_CurrentFrames(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) { return NULL; } @@ -2117,6 +2530,7 @@ _PyThread_CurrentFrames(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { @@ -2150,6 +2564,7 @@ _PyThread_CurrentFrames(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } @@ -2162,7 +2577,7 @@ PyObject * _PyThread_CurrentExceptions(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); @@ -2181,6 +2596,7 @@ _PyThread_CurrentExceptions(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { @@ -2213,6 +2629,7 @@ _PyThread_CurrentExceptions(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } @@ -2252,7 +2669,7 @@ _PyGILState_Fini(PyInterpreterState *interp) // XXX Drop this. -PyStatus +void _PyGILState_SetTstate(PyThreadState *tstate) { /* must init with valid states */ @@ -2262,7 +2679,7 @@ _PyGILState_SetTstate(PyThreadState *tstate) if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, PyGILState is shared by all interpreters. The main * interpreter is responsible to initialize it. */ - return _PyStatus_OK(); + return; } #ifndef NDEBUG @@ -2272,8 +2689,6 @@ _PyGILState_SetTstate(PyThreadState *tstate) assert(gilstate_tss_get(runtime) == tstate); assert(tstate->gilstate_counter == 1); #endif - - return _PyStatus_OK(); } PyInterpreterState * @@ -2306,12 +2721,13 @@ PyGILState_Check(void) return 1; } - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return 0; } - return (tstate == gilstate_tss_get(runtime)); + PyThreadState *tcur = gilstate_tss_get(runtime); + return (tstate == tcur); } PyGILState_STATE @@ -2398,13 +2814,19 @@ PyGILState_Release(PyGILState_STATE oldstate) /* can't have been locked when we created it */ assert(oldstate == PyGILState_UNLOCKED); // XXX Unbind tstate here. + // gh-119585: `PyThreadState_Clear()` may call destructors that + // themselves use PyGILState_Ensure and PyGILState_Release, so make + // sure that gilstate_counter is not zero when calling it. + ++tstate->gilstate_counter; PyThreadState_Clear(tstate); + --tstate->gilstate_counter; /* Delete the thread-state. Note this releases the GIL too! * It's vital that the GIL be held here, to avoid shutdown * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ - assert(current_fast_get(runtime) == tstate); + assert(tstate->gilstate_counter == 0); + assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } /* Release the lock if necessary */ @@ -2414,593 +2836,6 @@ PyGILState_Release(PyGILState_STATE oldstate) } -/**************************/ -/* cross-interpreter data */ -/**************************/ - -/* cross-interpreter data */ - -static inline void -_xidata_init(_PyCrossInterpreterData *data) -{ - // If the value is being reused - // then _xidata_clear() should have been called already. - assert(data->data == NULL); - assert(data->obj == NULL); - *data = (_PyCrossInterpreterData){0}; - data->interp = -1; -} - -static inline void -_xidata_clear(_PyCrossInterpreterData *data) -{ - // _PyCrossInterpreterData only has two members that need to be - // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. - // In both cases the original (owning) interpreter must be used, - // which is the caller's responsibility to ensure. - if (data->data != NULL) { - if (data->free != NULL) { - data->free(data->data); - } - data->data = NULL; - } - Py_CLEAR(data->obj); -} - -void -_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - void *shared, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(data != NULL); - assert(new_object != NULL); - _xidata_init(data); - data->data = shared; - if (obj != NULL) { - assert(interp != NULL); - // released in _PyCrossInterpreterData_Clear() - data->obj = Py_NewRef(obj); - } - // Ideally every object would know its owning interpreter. - // Until then, we have to rely on the caller to identify it - // (but we don't need it in all cases). - data->interp = (interp != NULL) ? interp->id : -1; - data->new_object = new_object; -} - -int -_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - const size_t size, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(size > 0); - // For now we always free the shared data in the same interpreter - // where it was allocated, so the interpreter is required. - assert(interp != NULL); - _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); - data->data = PyMem_RawMalloc(size); - if (data->data == NULL) { - return -1; - } - data->free = PyMem_RawFree; - return 0; -} - -void -_PyCrossInterpreterData_Clear(PyInterpreterState *interp, - _PyCrossInterpreterData *data) -{ - assert(data != NULL); - // This must be called in the owning interpreter. - assert(interp == NULL || data->interp == interp->id); - _xidata_clear(data); -} - -static int -_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) -{ - // data->data can be anything, including NULL, so we don't check it. - - // data->obj may be NULL, so we don't check it. - - if (data->interp < 0) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); - return -1; - } - - if (data->new_object == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); - return -1; - } - - // data->free may be NULL, so we don't check it. - - return 0; -} - -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); - -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) -{ - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; -} - -int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - return -1; - } - return 0; -} - -int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) -{ - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); -#ifdef Py_DEBUG - // The caller must hold the GIL - _Py_EnsureTstateNotNULL(tstate); -#endif - PyInterpreterState *interp = tstate->interp; - - // Reset data before re-populating. - *data = (_PyCrossInterpreterData){0}; - data->interp = -1; - - // Call the "getdata" func for the object. - Py_INCREF(obj); - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - Py_DECREF(obj); - return -1; - } - int res = getdata(tstate, obj, data); - Py_DECREF(obj); - if (res != 0) { - return -1; - } - - // Fill in the blanks and validate the result. - data->interp = interp->id; - if (_check_xidata(tstate, data) != 0) { - (void)_PyCrossInterpreterData_Release(data); - return -1; - } - - return 0; -} - -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); -} - -int -_Py_CallInInterpreter(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - return func(arg); - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); - return 0; -} - -int -_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - int res = func(arg); - PyMem_RawFree(arg); - return res; - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); - return 0; -} - -static int -_call_clear_xidata(void *data) -{ - _xidata_clear((_PyCrossInterpreterData *)data); - return 0; -} - -static int -_xidata_release(_PyCrossInterpreterData *data, int rawfree) -{ - if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { - // Nothing to release! - if (rawfree) { - PyMem_RawFree(data); - } - else { - data->data = NULL; - } - return 0; - } - - // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); - if (interp == NULL) { - // The interpreter was already destroyed. - // This function shouldn't have been called. - // XXX Someone leaked some memory... - assert(PyErr_Occurred()); - if (rawfree) { - PyMem_RawFree(data); - } - return -1; - } - - // "Release" the data and/or the object. - if (rawfree) { - return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); - } - else { - return _Py_CallInInterpreter(interp, _call_clear_xidata, data); - } -} - -int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 0); -} - -int -_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 1); -} - -/* registry of {type -> crossinterpdatafunc} */ - -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) -{ - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) { - return -1; - } - *newhead = (struct _xidregitem){ - // We do not keep a reference, to avoid keeping the class alive. - .cls = cls, - .refcount = 1, - .getdata = getdata, - }; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - // XXX Assign a callback to clear the entry from the registry? - newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); - if (newhead->weakref == NULL) { - PyMem_RawFree(newhead); - return -1; - } - } - newhead->next = xidregistry->head; - if (newhead->next != NULL) { - newhead->next->prev = newhead; - } - xidregistry->head = newhead; - return 0; -} - -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) -{ - struct _xidregitem *next = entry->next; - if (entry->prev != NULL) { - assert(entry->prev->next == entry); - entry->prev->next = next; - } - else { - assert(xidregistry->head == entry); - xidregistry->head = next; - } - if (next != NULL) { - next->prev = entry->prev; - } - Py_XDECREF(entry->weakref); - PyMem_RawFree(entry); - return next; -} - -static void -_xidregistry_clear(struct _xidregistry *xidregistry) -{ - struct _xidregitem *cur = xidregistry->head; - xidregistry->head = NULL; - while (cur != NULL) { - struct _xidregitem *next = cur->next; - Py_XDECREF(cur->weakref); - PyMem_RawFree(cur); - cur = next; - } -} - -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) -{ - struct _xidregitem *cur = xidregistry->head; - while (cur != NULL) { - if (cur->weakref != NULL) { - // cur is/was a heap type. - PyObject *registered = _PyWeakref_GET_REF(cur->weakref); - if (registered == NULL) { - // The weakly ref'ed object was freed. - cur = _xidregistry_remove_entry(xidregistry, cur); - continue; - } - assert(PyType_Check(registered)); - assert(cur->cls == (PyTypeObject *)registered); - assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - Py_DECREF(registered); - } - if (cur->cls == cls) { - return cur; - } - cur = cur->next; - } - return NULL; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *xidregistry = &interp->runtime->xidregistry; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - assert(interp->xidregistry.mutex == xidregistry->mutex); - xidregistry = &interp->xidregistry; - } - return xidregistry; -} - -static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); - -static inline void -_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) -{ - if (xidregistry != &interp->xidregistry) { - assert(xidregistry == &interp->runtime->xidregistry); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); - } - } -} - -int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) -{ - if (!PyType_Check(cls)) { - PyErr_Format(PyExc_ValueError, "only classes may be registered"); - return -1; - } - if (getdata == NULL) { - PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); - return -1; - } - - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->getdata == getdata); - matched->refcount += 1; - goto finally; - } - - res = _xidregistry_add_type(xidregistry, cls, getdata); - -finally: - PyThread_release_lock(xidregistry->mutex); - return res; -} - -int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) -{ - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->refcount > 0); - matched->refcount -= 1; - if (matched->refcount == 0) { - (void)_xidregistry_remove_entry(xidregistry, matched); - } - res = 1; - } - - PyThread_release_lock(xidregistry->mutex); - return res; -} - - -/* Cross-interpreter objects are looked up by exact match on the class. - We can reassess this policy when we move from a global registry to a - tp_* slot. */ - -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) -{ - PyTypeObject *cls = Py_TYPE(obj); - - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; - - PyThread_release_lock(xidregistry->mutex); - return func; -} - -/* cross-interpreter data for builtin types */ - -struct _shared_bytes_data { - char *bytes; - Py_ssize_t len; -}; - -static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) -{ - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); - return PyBytes_FromStringAndSize(shared->bytes, shared->len); -} - -static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_bytes_data), obj, - _new_bytes_object - ) < 0) - { - return -1; - } - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; - if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); - return -1; - } - return 0; -} - -struct _shared_str_data { - int kind; - const void *buffer; - Py_ssize_t len; -}; - -static PyObject * -_new_str_object(_PyCrossInterpreterData *data) -{ - struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); - return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); -} - -static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_str_data), obj, - _new_str_object - ) < 0) - { - return -1; - } - struct _shared_str_data *shared = (struct _shared_str_data *)data->data; - shared->kind = PyUnicode_KIND(obj); - shared->buffer = PyUnicode_DATA(obj); - shared->len = PyUnicode_GET_LENGTH(obj); - return 0; -} - -static PyObject * -_new_long_object(_PyCrossInterpreterData *data) -{ - return PyLong_FromSsize_t((Py_ssize_t)(data->data)); -} - -static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - /* Note that this means the size of shareable ints is bounded by - * sys.maxsize. Hence on 32-bit architectures that is half the - * size of maximum shareable ints on 64-bit. - */ - Py_ssize_t value = PyLong_AsSsize_t(obj); - if (value == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); - } - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); - // data->obj and data->free remain NULL - return 0; -} - -static PyObject * -_new_none_object(_PyCrossInterpreterData *data) -{ - // XXX Singleton refcounts are problematic across interpreters... - return Py_NewRef(Py_None); -} - -static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); - // data->data, data->obj and data->free remain NULL - return 0; -} - -static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) -{ - // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { - Py_FatalError("could not register None for cross-interpreter sharing"); - } - - // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { - Py_FatalError("could not register int for cross-interpreter sharing"); - } - - // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { - Py_FatalError("could not register bytes for cross-interpreter sharing"); - } - - // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { - Py_FatalError("could not register str for cross-interpreter sharing"); - } -} - - /*************/ /* Other API */ /*************/ @@ -3020,11 +2855,18 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) { if (eval_frame == _PyEval_EvalFrameDefault) { - interp->eval_frame = NULL; + eval_frame = NULL; } - else { - interp->eval_frame = eval_frame; + if (eval_frame == interp->eval_frame) { + return; + } +#ifdef _Py_TIER2 + if (eval_frame != NULL) { + _Py_Executors_InvalidateAll(interp, 1); } +#endif + RARE_EVENT_INC(set_eval_frame_func); + interp->eval_frame = eval_frame; } @@ -3052,9 +2894,8 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config) const PyConfig* _Py_GetConfig(void) { - _PyRuntimeState *runtime = &_PyRuntime; assert(PyGILState_Check()); - PyThreadState *tstate = current_fast_get(runtime); + PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); } @@ -3185,3 +3026,80 @@ _PyThreadState_MustExit(PyThreadState *tstate) } return 1; } + +/********************/ +/* mimalloc support */ +/********************/ + +static void +tstate_mimalloc_bind(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state *mts = &((_PyThreadStateImpl*)tstate)->mimalloc; + + // Initialize the mimalloc thread state. This must be called from the + // same thread that will use the thread state. The "mem" heap doubles as + // the "backing" heap. + mi_tld_t *tld = &mts->tld; + _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]); + llist_init(&mts->page_list); + + // Exiting threads push any remaining in-use segments to the abandoned + // pool to be re-claimed later by other threads. We use per-interpreter + // pools to keep Python objects from different interpreters separate. + tld->segments.abandoned = &tstate->interp->mimalloc.abandoned_pool; + + // Don't fill in the first N bytes up to ob_type in debug builds. We may + // access ob_tid and the refcount fields in the dict and list lock-less + // accesses, so they must remain valid for a while after deallocation. + size_t base_offset = offsetof(PyObject, ob_type); + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + base_offset += 2 * sizeof(size_t); + } + size_t debug_offsets[_Py_MIMALLOC_HEAP_COUNT] = { + [_Py_MIMALLOC_HEAP_OBJECT] = base_offset, + [_Py_MIMALLOC_HEAP_GC] = base_offset, + [_Py_MIMALLOC_HEAP_GC_PRE] = base_offset + 2 * sizeof(PyObject *), + }; + + // Initialize each heap + for (uint8_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none(), false, i); + mts->heaps[i].debug_offset = (uint8_t)debug_offsets[i]; + } + + // Heaps that store Python objects should use QSBR to delay freeing + // mimalloc pages while there may be concurrent lock-free readers. + mts->heaps[_Py_MIMALLOC_HEAP_OBJECT].page_use_qsbr = true; + mts->heaps[_Py_MIMALLOC_HEAP_GC].page_use_qsbr = true; + mts->heaps[_Py_MIMALLOC_HEAP_GC_PRE].page_use_qsbr = true; + + // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. + // _PyObject_GC_New() and similar functions temporarily override this to + // use one of the GC heaps. + mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; + + _Py_atomic_store_int(&mts->initialized, 1); +#endif +} + +void +_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + if (!tstate->_status.bound) { + // The mimalloc heaps are only initialized when the thread is bound. + return; + } + + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + // Abandon all segments in use by this thread. This pushes them to + // a shared pool to later be reclaimed by other threads. It's important + // to do this before the thread state is destroyed so that objects + // remain visible to the GC. + _mi_heap_collect_abandon(&tstate_impl->mimalloc.heaps[i]); + } +#endif +} diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 16bf06f0e6cca2..2f2b588bd147d8 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -842,7 +842,7 @@ char * PyOS_double_to_string(double val, */ - if (Py_IS_NAN(val) || Py_IS_INFINITY(val)) + if (isnan(val) || isinf(val)) /* 3 for 'inf'/'nan', 1 for sign, 1 for '\0' */ bufsize = 5; else { @@ -860,10 +860,10 @@ char * PyOS_double_to_string(double val, } /* Handle nan and inf. */ - if (Py_IS_NAN(val)) { + if (isnan(val)) { strcpy(buf, "nan"); t = Py_DTST_NAN; - } else if (Py_IS_INFINITY(val)) { + } else if (isinf(val)) { if (copysign(1., val) == 1.) strcpy(buf, "inf"); else @@ -1234,7 +1234,7 @@ char * PyOS_double_to_string(double val, case 'E': float_strings = uc_float_strings; format_code = 'e'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'e': mode = 2; precision++; @@ -1244,7 +1244,7 @@ char * PyOS_double_to_string(double val, case 'F': float_strings = uc_float_strings; format_code = 'f'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'f': mode = 3; break; @@ -1253,7 +1253,7 @@ char * PyOS_double_to_string(double val, case 'G': float_strings = uc_float_strings; format_code = 'g'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'g': mode = 2; /* precision 0 makes no sense for 'g' format; interpret as 1 */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 74994295a9b23a..ce7f194e929c9c 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -40,14 +40,17 @@ /* Forward */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, - PyCompilerFlags *, PyArena *, PyObject*); + PyCompilerFlags *, PyArena *, PyObject*, int); static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *, PyCompilerFlags *); static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); static PyObject* pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags); - +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source); int _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, @@ -80,13 +83,11 @@ _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, return res; } - -/* Parse input from a file and execute it */ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) { - PyObject *filename_obj; + PyObject *filename_obj = NULL; if (filename != NULL) { filename_obj = PyUnicode_DecodeFSDefault(filename); if (filename_obj == NULL) { @@ -94,9 +95,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, return -1; } } - else { - filename_obj = NULL; - } int res = _PyRun_AnyFileObject(fp, filename_obj, closeit, flags); Py_XDECREF(filename_obj); return res; @@ -281,7 +279,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, } PyObject *main_dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src); + PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src, 1); _PyArena_Free(arena); Py_DECREF(main_module); if (res == NULL) { @@ -452,7 +450,7 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, v = run_pyc_file(pyc_fp, dict, dict, flags); } else { /* When running from stdin, leave __main__.__loader__ alone */ - if (PyUnicode_CompareWithASCIIString(filename, "") != 0 && + if ((!PyUnicode_Check(filename) || !PyUnicode_EqualToUTF8(filename, "")) && set_main_loader(dict, filename, "SourceFileLoader") < 0) { fprintf(stderr, "python: failed to set __main__.__loader__\n"); ret = -1; @@ -472,11 +470,11 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, done: if (set_file_name) { - if (PyDict_DelItemString(dict, "__file__")) { - PyErr_Clear(); + if (PyDict_PopString(dict, "__file__", NULL) < 0) { + PyErr_Print(); } - if (PyDict_DelItemString(dict, "__cached__")) { - PyErr_Clear(); + if (PyDict_PopString(dict, "__cached__", NULL) < 0) { + PyErr_Print(); } } Py_XDECREF(main_module); @@ -499,16 +497,25 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, int -PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) -{ +_PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags) { PyObject *main_module = PyImport_AddModuleRef("__main__"); if (main_module == NULL) { return -1; } PyObject *dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = PyRun_StringFlags(command, Py_file_input, - dict, dict, flags); + PyObject *res = NULL; + if (name == NULL) { + res = PyRun_StringFlags(command, Py_file_input, dict, dict, flags); + } else { + PyObject* the_name = PyUnicode_FromString(name); + if (!the_name) { + PyErr_Print(); + return -1; + } + res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0); + Py_DECREF(the_name); + } Py_DECREF(main_module); if (res == NULL) { PyErr_Print(); @@ -519,6 +526,12 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return 0; } +int +PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) +{ + return _PyRun_SimpleStringFlagsWithName(command, NULL, flags); +} + int _Py_HandleSystemExit(int *exitcode_p) { @@ -632,7 +645,7 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) PyErr_Clear(); goto done; } - _PyErr_WriteUnraisableMsg("in audit hook", NULL); + PyErr_FormatUnraisable("Exception ignored in audit hook"); } if (hook) { PyObject* args[3] = {typ, exc, tb}; @@ -1031,6 +1044,7 @@ print_exception_recursive(struct exception_print_context *ctx, PyObject *value) void _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) { + assert(value != NULL); assert(file != NULL && file != Py_None); if (PyExceptionInstance_Check(value) && tb != NULL && PyTraceBack_Check(tb)) { @@ -1047,10 +1061,6 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt; - if (!value || PyErr_GivenExceptionMatches(value, PyExc_MemoryError)) { - goto fallback; - } - // Try first with the stdlib traceback module PyObject *traceback_module = PyImport_ImportModule("traceback"); @@ -1078,7 +1088,8 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt; #ifdef Py_DEBUG if (PyErr_Occurred()) { - _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL); + PyErr_FormatUnraisable( + "Exception ignored in the internal traceback machinery"); } #endif PyErr_Clear(); @@ -1134,9 +1145,10 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } -PyObject * -PyRun_StringFlags(const char *str, int start, PyObject *globals, - PyObject *locals, PyCompilerFlags *flags) +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source) { PyObject *ret = NULL; mod_ty mod; @@ -1146,17 +1158,36 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; + PyObject* source = NULL; _Py_DECLARE_STR(anon_string, ""); - mod = _PyParser_ASTFromString( - str, &_Py_STR(anon_string), start, flags, arena); - if (mod != NULL) - ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena, NULL); + if (name) { + source = PyUnicode_FromString(str); + if (!source) { + PyErr_Clear(); + } + } else { + name = &_Py_STR(anon_string); + } + + mod = _PyParser_ASTFromString(str, name, start, flags, arena); + + if (mod != NULL) { + ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); + } + Py_XDECREF(source); _PyArena_Free(arena); return ret; } +PyObject * +PyRun_StringFlags(const char *str, int start, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { + + return _PyRun_StringFlagsWithName(str, NULL, start, globals, locals, flags, 0); +} + static PyObject * pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) @@ -1176,7 +1207,7 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *ret; if (mod != NULL) { - ret = run_mod(mod, filename, globals, locals, flags, arena, NULL); + ret = run_mod(mod, filename, globals, locals, flags, arena, NULL, 0); } else { ret = NULL; @@ -1242,17 +1273,20 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py _PyRuntime.signals.unhandled_keyboard_interrupt = 0; /* Set globals['__builtins__'] if it doesn't exist */ - if (globals != NULL) { - int has_builtins = PyDict_ContainsString(globals, "__builtins__"); - if (has_builtins < 0) { + if (!globals || !PyDict_Check(globals)) { + PyErr_SetString(PyExc_SystemError, "globals must be a real dict"); + return NULL; + } + int has_builtins = PyDict_ContainsString(globals, "__builtins__"); + if (has_builtins < 0) { + return NULL; + } + if (!has_builtins) { + if (PyDict_SetItemString(globals, "__builtins__", + tstate->interp->builtins) < 0) + { return NULL; } - if (!has_builtins) { - if (PyDict_SetItemString(globals, "__builtins__", - tstate->interp->builtins) < 0) { - return NULL; - } - } } v = PyEval_EvalCode((PyObject*)co, globals, locals); @@ -1264,15 +1298,19 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src) + PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src, + int generate_new_source) { PyThreadState *tstate = _PyThreadState_GET(); PyObject* interactive_filename = filename; if (interactive_src) { PyInterpreterState *interp = tstate->interp; - interactive_filename = PyUnicode_FromFormat( - "", interp->_interactive_src_count++ - ); + if (generate_new_source) { + interactive_filename = PyUnicode_FromFormat( + "%U-%d", filename, interp->_interactive_src_count++); + } else { + Py_INCREF(interactive_filename); + } if (interactive_filename == NULL) { return NULL; } @@ -1280,7 +1318,9 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); if (co == NULL) { - Py_DECREF(interactive_filename); + if (interactive_src) { + Py_DECREF(interactive_filename); + } return NULL; } diff --git a/Python/pytime.c b/Python/pytime.c index e4813d4a9c2a2a..cd76970718622f 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // PyTime_t #include // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -50,53 +50,82 @@ # error "time_t is not a two's complement integer type" #endif -#if _PyTime_MIN + _PyTime_MAX != -1 -# error "_PyTime_t is not a two's complement integer type" +#if PyTime_MIN + PyTime_MAX != -1 +# error "PyTime_t is not a two's complement integer type" #endif -static void -pytime_time_t_overflow(void) +#ifdef MS_WINDOWS +static _PyTimeFraction py_qpc_base = {0, 0}; + +// Forward declaration +static int py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc); +#endif + + +static PyTime_t +_PyTime_GCD(PyTime_t x, PyTime_t y) { - PyErr_SetString(PyExc_OverflowError, - "timestamp out of range for platform time_t"); + // Euclidean algorithm + assert(x >= 1); + assert(y >= 1); + while (y != 0) { + PyTime_t tmp = y; + y = x % y; + x = tmp; + } + assert(x >= 1); + return x; } -static void -pytime_overflow(void) +int +_PyTimeFraction_Set(_PyTimeFraction *frac, PyTime_t numer, PyTime_t denom) { - PyErr_SetString(PyExc_OverflowError, - "timestamp too large to convert to C _PyTime_t"); + if (numer < 1 || denom < 1) { + return -1; + } + + PyTime_t gcd = _PyTime_GCD(numer, denom); + frac->numer = numer / gcd; + frac->denom = denom / gcd; + return 0; } -static inline _PyTime_t -pytime_from_nanoseconds(_PyTime_t t) +double +_PyTimeFraction_Resolution(const _PyTimeFraction *frac) { - // _PyTime_t is a number of nanoseconds - return t; + return (double)frac->numer / (double)frac->denom / 1e9; } -static inline _PyTime_t -pytime_as_nanoseconds(_PyTime_t t) +static void +pytime_time_t_overflow(void) { - // _PyTime_t is a number of nanoseconds: see pytime_from_nanoseconds() - return t; + PyErr_SetString(PyExc_OverflowError, + "timestamp out of range for platform time_t"); +} + + +static void +pytime_overflow(void) +{ + PyErr_SetString(PyExc_OverflowError, + "timestamp too large to convert to C PyTime_t"); } -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +// Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_add(_PyTime_t *t1, _PyTime_t t2) +pytime_add(PyTime_t *t1, PyTime_t t2) { - if (t2 > 0 && *t1 > _PyTime_MAX - t2) { - *t1 = _PyTime_MAX; + if (t2 > 0 && *t1 > PyTime_MAX - t2) { + *t1 = PyTime_MAX; return -1; } - else if (t2 < 0 && *t1 < _PyTime_MIN - t2) { - *t1 = _PyTime_MIN; + else if (t2 < 0 && *t1 < PyTime_MIN - t2) { + *t1 = PyTime_MIN; return -1; } else { @@ -106,8 +135,8 @@ pytime_add(_PyTime_t *t1, _PyTime_t t2) } -_PyTime_t -_PyTime_Add(_PyTime_t t1, _PyTime_t t2) +PyTime_t +_PyTime_Add(PyTime_t t1, PyTime_t t2) { (void)pytime_add(&t1, t2); return t1; @@ -115,11 +144,11 @@ _PyTime_Add(_PyTime_t t1, _PyTime_t t2) static inline int -pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) +pytime_mul_check_overflow(PyTime_t a, PyTime_t b) { if (b != 0) { assert(b > 0); - return ((a < _PyTime_MIN / b) || (_PyTime_MAX / b < a)); + return ((a < PyTime_MIN / b) || (PyTime_MAX / b < a)); } else { return 0; @@ -127,13 +156,13 @@ pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) } -// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +// Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_mul(_PyTime_t *t, _PyTime_t k) +pytime_mul(PyTime_t *t, PyTime_t k) { assert(k >= 0); if (pytime_mul_check_overflow(*t, k)) { - *t = (*t >= 0) ? _PyTime_MAX : _PyTime_MIN; + *t = (*t >= 0) ? PyTime_MAX : PyTime_MIN; return -1; } else { @@ -143,26 +172,32 @@ pytime_mul(_PyTime_t *t, _PyTime_t k) } -// Compute t * k. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -static inline _PyTime_t -_PyTime_Mul(_PyTime_t t, _PyTime_t k) +// Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +static inline PyTime_t +_PyTime_Mul(PyTime_t t, PyTime_t k) { (void)pytime_mul(&t, k); return t; } +PyTime_t +_PyTimeFraction_Mul(PyTime_t ticks, const _PyTimeFraction *frac) +{ + const PyTime_t mul = frac->numer; + const PyTime_t div = frac->denom; + if (div == 1) { + // Fast-path taken by mach_absolute_time() with 1/1 time base. + return _PyTime_Mul(ticks, mul); + } -_PyTime_t -_PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, _PyTime_t div) -{ /* Compute (ticks * mul / div) in two parts to reduce the risk of integer overflow: compute the integer part, and then the remaining part. (ticks * mul) / div == (ticks / div) * mul + (ticks % div) * mul / div */ - _PyTime_t intpart, remaining; + PyTime_t intpart, remaining; intpart = ticks / div; ticks %= div; remaining = _PyTime_Mul(ticks, mul) / div; @@ -204,17 +239,17 @@ _PyLong_FromTime_t(time_t t) } -// Convert _PyTime_t to time_t. +// Convert PyTime_t to time_t. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsTime_t(_PyTime_t t, time_t *t2) +_PyTime_AsTime_t(PyTime_t t, time_t *t2) { #if SIZEOF_TIME_T < _SIZEOF_PYTIME_T - if ((_PyTime_t)PY_TIME_T_MAX < t) { + if ((PyTime_t)PY_TIME_T_MAX < t) { *t2 = PY_TIME_T_MAX; return -1; } - if (t < (_PyTime_t)PY_TIME_T_MIN) { + if (t < (PyTime_t)PY_TIME_T_MIN) { *t2 = PY_TIME_T_MIN; return -1; } @@ -225,17 +260,17 @@ _PyTime_AsTime_t(_PyTime_t t, time_t *t2) #ifdef MS_WINDOWS -// Convert _PyTime_t to long. +// Convert PyTime_t to long. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsLong(_PyTime_t t, long *t2) +_PyTime_AsCLong(PyTime_t t, long *t2) { #if SIZEOF_LONG < _SIZEOF_PYTIME_T - if ((_PyTime_t)LONG_MAX < t) { + if ((PyTime_t)LONG_MAX < t) { *t2 = LONG_MAX; return -1; } - if (t < (_PyTime_t)LONG_MIN) { + if (t < (PyTime_t)LONG_MIN) { *t2 = LONG_MIN; return -1; } @@ -340,7 +375,7 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, if (PyFloat_Check(obj)) { double d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { *numerator = 0; PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; @@ -368,7 +403,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } @@ -410,50 +445,42 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, } -_PyTime_t +PyTime_t _PyTime_FromSeconds(int seconds) { /* ensure that integer overflow cannot happen, int type should have 32 - bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 + bits, whereas PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 bits). */ - static_assert(INT_MAX <= _PyTime_MAX / SEC_TO_NS, "_PyTime_t overflow"); - static_assert(INT_MIN >= _PyTime_MIN / SEC_TO_NS, "_PyTime_t underflow"); + static_assert(INT_MAX <= PyTime_MAX / SEC_TO_NS, "PyTime_t overflow"); + static_assert(INT_MIN >= PyTime_MIN / SEC_TO_NS, "PyTime_t underflow"); - _PyTime_t t = (_PyTime_t)seconds; - assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) - || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); + PyTime_t t = (PyTime_t)seconds; + assert((t >= 0 && t <= PyTime_MAX / SEC_TO_NS) + || (t < 0 && t >= PyTime_MIN / SEC_TO_NS)); t *= SEC_TO_NS; - return pytime_from_nanoseconds(t); -} - - -_PyTime_t -_PyTime_FromNanoseconds(_PyTime_t ns) -{ - return pytime_from_nanoseconds(ns); + return t; } -_PyTime_t -_PyTime_FromMicrosecondsClamp(_PyTime_t us) +PyTime_t +_PyTime_FromMicrosecondsClamp(PyTime_t us) { - _PyTime_t ns = _PyTime_Mul(us, US_TO_NS); - return pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTime_Mul(us, US_TO_NS); + return ns; } int -_PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) +_PyTime_FromLong(PyTime_t *tp, PyObject *obj) { - if (!PyLong_Check(obj)) { PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); return -1; } - static_assert(sizeof(long long) == sizeof(_PyTime_t), - "_PyTime_t is not long long"); + static_assert(sizeof(long long) == sizeof(PyTime_t), + "PyTime_t is not long long"); long long nsec = PyLong_AsLongLong(obj); if (nsec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) { @@ -462,28 +489,28 @@ _PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) return -1; } - _PyTime_t t = (_PyTime_t)nsec; - *tp = pytime_from_nanoseconds(t); + PyTime_t t = (PyTime_t)nsec; + *tp = t; return 0; } #ifdef HAVE_CLOCK_GETTIME static int -pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) +pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) { - _PyTime_t t, tv_nsec; + PyTime_t t, tv_nsec; - static_assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t), - "timespec.tv_sec is larger than _PyTime_t"); - t = (_PyTime_t)ts->tv_sec; + static_assert(sizeof(ts->tv_sec) <= sizeof(PyTime_t), + "timespec.tv_sec is larger than PyTime_t"); + t = (PyTime_t)ts->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); tv_nsec = ts->tv_nsec; int res2 = pytime_add(&t, tv_nsec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -493,7 +520,7 @@ pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) } int -_PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) +_PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts) { return pytime_fromtimespec(tp, ts, 1); } @@ -502,18 +529,18 @@ _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) #ifndef MS_WINDOWS static int -pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) +pytime_fromtimeval(PyTime_t *tp, struct timeval *tv, int raise_exc) { - static_assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t), - "timeval.tv_sec is larger than _PyTime_t"); - _PyTime_t t = (_PyTime_t)tv->tv_sec; + static_assert(sizeof(tv->tv_sec) <= sizeof(PyTime_t), + "timeval.tv_sec is larger than PyTime_t"); + PyTime_t t = (PyTime_t)tv->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); - _PyTime_t usec = (_PyTime_t)tv->tv_usec * US_TO_NS; + PyTime_t usec = (PyTime_t)tv->tv_usec * US_TO_NS; int res2 = pytime_add(&t, usec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -524,7 +551,7 @@ pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) int -_PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) +_PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv) { return pytime_fromtimeval(tp, tv, 1); } @@ -532,7 +559,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) static int -pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, +pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, long unit_to_ns) { /* volatile avoids optimization changing how numbers are rounded */ @@ -544,25 +571,26 @@ pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, d = pytime_round(d, round); /* See comments in pytime_double_to_denominator */ - if (!((double)_PyTime_MIN <= d && d < -(double)_PyTime_MIN)) { + if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { pytime_time_t_overflow(); + *tp = 0; return -1; } - _PyTime_t ns = (_PyTime_t)d; + PyTime_t ns = (PyTime_t)d; - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } static int -pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, +pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, long unit_to_ns) { if (PyFloat_Check(obj)) { double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } @@ -577,45 +605,44 @@ pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, return -1; } - static_assert(sizeof(long long) <= sizeof(_PyTime_t), - "_PyTime_t is smaller than long long"); - _PyTime_t ns = (_PyTime_t)sec; + static_assert(sizeof(long long) <= sizeof(PyTime_t), + "PyTime_t is smaller than long long"); + PyTime_t ns = (PyTime_t)sec; if (pytime_mul(&ns, unit_to_ns) < 0) { pytime_overflow(); return -1; } - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } } int -_PyTime_FromSecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromSecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, SEC_TO_NS); } int -_PyTime_FromMillisecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromMillisecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, MS_TO_NS); } double -_PyTime_AsSecondsDouble(_PyTime_t t) +PyTime_AsSecondsDouble(PyTime_t ns) { /* volatile avoids optimization changing how numbers are rounded */ volatile double d; - _PyTime_t ns = pytime_as_nanoseconds(t); if (ns % SEC_TO_NS == 0) { /* Divide using integers to avoid rounding issues on the integer part. 1e-9 cannot be stored exactly in IEEE 64-bit. */ - _PyTime_t secs = ns / SEC_TO_NS; + PyTime_t secs = ns / SEC_TO_NS; d = (double)secs; } else { @@ -627,33 +654,28 @@ _PyTime_AsSecondsDouble(_PyTime_t t) PyObject * -_PyTime_AsNanosecondsObject(_PyTime_t t) +_PyTime_AsLong(PyTime_t ns) { - _PyTime_t ns = pytime_as_nanoseconds(t); - static_assert(sizeof(long long) >= sizeof(_PyTime_t), - "_PyTime_t is larger than long long"); + static_assert(sizeof(long long) >= sizeof(PyTime_t), + "PyTime_t is larger than long long"); return PyLong_FromLongLong((long long)ns); } -_PyTime_t -_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) +int +_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { - _PyTime_t tp; - if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { - return -1; - } - return tp; + return pytime_from_double(result, seconds, round, SEC_TO_NS); } -static _PyTime_t -pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) +static PyTime_t +pytime_divide_round_up(const PyTime_t t, const PyTime_t k) { assert(k > 1); if (t >= 0) { // Don't use (t + k - 1) / k to avoid integer overflow - // if t is equal to _PyTime_MAX - _PyTime_t q = t / k; + // if t is equal to PyTime_MAX + PyTime_t q = t / k; if (t % k) { q += 1; } @@ -661,8 +683,8 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) } else { // Don't use (t - (k - 1)) / k to avoid integer overflow - // if t is equals to _PyTime_MIN. - _PyTime_t q = t / k; + // if t is equals to PyTime_MIN. + PyTime_t q = t / k; if (t % k) { q -= 1; } @@ -671,15 +693,15 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) } -static _PyTime_t -pytime_divide(const _PyTime_t t, const _PyTime_t k, +static PyTime_t +pytime_divide(const PyTime_t t, const PyTime_t k, const _PyTime_round_t round) { assert(k > 1); if (round == _PyTime_ROUND_HALF_EVEN) { - _PyTime_t x = t / k; - _PyTime_t r = t % k; - _PyTime_t abs_r = Py_ABS(r); + PyTime_t x = t / k; + PyTime_t r = t % k; + PyTime_t abs_r = Py_ABS(r); if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) { x++; @@ -716,17 +738,17 @@ pytime_divide(const _PyTime_t t, const _PyTime_t k, // Compute (t / k, t % k) in (pq, pr). // Make sure that 0 <= pr < k. // Return 0 on success. -// Return -1 on underflow and store (_PyTime_MIN, 0) in (pq, pr). +// Return -1 on underflow and store (PyTime_MIN, 0) in (pq, pr). static int -pytime_divmod(const _PyTime_t t, const _PyTime_t k, - _PyTime_t *pq, _PyTime_t *pr) +pytime_divmod(const PyTime_t t, const PyTime_t k, + PyTime_t *pq, PyTime_t *pr) { assert(k > 1); - _PyTime_t q = t / k; - _PyTime_t r = t % k; + PyTime_t q = t / k; + PyTime_t r = t % k; if (r < 0) { - if (q == _PyTime_MIN) { - *pq = _PyTime_MIN; + if (q == PyTime_MIN) { + *pq = PyTime_MIN; *pr = 0; return -1; } @@ -741,47 +763,36 @@ pytime_divmod(const _PyTime_t t, const _PyTime_t k, } -_PyTime_t -_PyTime_AsNanoseconds(_PyTime_t t) -{ - return pytime_as_nanoseconds(t); -} - - #ifdef MS_WINDOWS -_PyTime_t -_PyTime_As100Nanoseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_As100Nanoseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_100NS, round); } #endif -_PyTime_t -_PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMicroseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_US, round); } -_PyTime_t -_PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMilliseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_MS, round); } static int -pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, +pytime_as_timeval(PyTime_t ns, PyTime_t *ptv_sec, int *ptv_usec, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t us = pytime_divide(ns, US_TO_NS, round); + PyTime_t us = pytime_divide(ns, US_TO_NS, round); - _PyTime_t tv_sec, tv_usec; + PyTime_t tv_sec, tv_usec; int res = pytime_divmod(us, SEC_TO_US, &tv_sec, &tv_usec); *ptv_sec = tv_sec; *ptv_usec = (int)tv_usec; @@ -790,16 +801,16 @@ pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, static int -pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, +pytime_as_timeval_struct(PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise_exc) { - _PyTime_t tv_sec; + PyTime_t tv_sec; int tv_usec; int res = pytime_as_timeval(t, &tv_sec, &tv_usec, round); int res2; #ifdef MS_WINDOWS // On Windows, timeval.tv_sec type is long - res2 = _PyTime_AsLong(tv_sec, &tv->tv_sec); + res2 = _PyTime_AsCLong(tv_sec, &tv->tv_sec); #else res2 = _PyTime_AsTime_t(tv_sec, &tv->tv_sec); #endif @@ -817,24 +828,24 @@ pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, int -_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { return pytime_as_timeval_struct(t, tv, round, 1); } void -_PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { (void)pytime_as_timeval_struct(t, tv, round, 0); } int -_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, +_PyTime_AsTimevalTime_t(PyTime_t t, time_t *p_secs, int *us, _PyTime_round_t round) { - _PyTime_t secs; + PyTime_t secs; if (pytime_as_timeval(t, &secs, us, round) < 0) { pytime_time_t_overflow(); return -1; @@ -850,10 +861,9 @@ _PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) static int -pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) +pytime_as_timespec(PyTime_t ns, struct timespec *ts, int raise_exc) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t tv_sec, tv_nsec; + PyTime_t tv_sec, tv_nsec; int res = pytime_divmod(ns, SEC_TO_NS, &tv_sec, &tv_nsec); int res2 = _PyTime_AsTime_t(tv_sec, &ts->tv_sec); @@ -870,49 +880,53 @@ pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) } void -_PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts) { (void)pytime_as_timespec(t, ts, 0); } int -_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec(PyTime_t t, struct timespec *ts) { return pytime_as_timespec(t, ts, 1); } #endif +// N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); + } #ifdef MS_WINDOWS FILETIME system_time; ULARGE_INTEGER large; - GetSystemTimeAsFileTime(&system_time); + GetSystemTimePreciseAsFileTime(&system_time); large.u.LowPart = system_time.dwLowDateTime; large.u.HighPart = system_time.dwHighDateTime; /* 11,644,473,600,000,000,000: number of nanoseconds between the 1st january 1601 and the 1st january 1970 (369 years + 89 leap days). */ - _PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; + *tp = ns; if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; + // GetSystemTimePreciseAsFileTime() is implemented using + // QueryPerformanceCounter() internally. + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } - info->implementation = "GetSystemTimeAsFileTime()"; + info->implementation = "GetSystemTimePreciseAsFileTime()"; info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->adjustable = 1; } @@ -994,148 +1008,175 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetSystemClock(void) +int +PyTime_Time(PyTime_t *result) { - _PyTime_t t; - if (py_get_system_clock(&t, NULL, 0) < 0) { - // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: - // silently ignore the failure and return 0. - t = 0; + if (py_get_system_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -_PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +PyTime_TimeRaw(PyTime_t *result) +{ + if (py_get_system_clock(result, NULL, 0) < 0) { + *result = 0; + return -1; + } + return 0; +} + + +int +_PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { return py_get_system_clock(t, info, 1); } -#ifdef __APPLE__ +#ifdef MS_WINDOWS static int -py_mach_timebase_info(_PyTime_t *pnumer, _PyTime_t *pdenom, int raise) +py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc) { - static mach_timebase_info_data_t timebase; - /* According to the Technical Q&A QA1398, mach_timebase_info() cannot - fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ - (void)mach_timebase_info(&timebase); + LARGE_INTEGER freq; + // Since Windows XP, the function cannot fail. + (void)QueryPerformanceFrequency(&freq); + LONGLONG frequency = freq.QuadPart; + + // Since Windows XP, frequency cannot be zero. + assert(frequency >= 1); + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); + PyTime_t denom = (PyTime_t)frequency; - /* Sanity check: should never occur in practice */ - if (timebase.numer < 1 || timebase.denom < 1) { - if (raise) { + // Known QueryPerformanceFrequency() values: + // + // * 10,000,000 (10 MHz): 100 ns resolution + // * 3,579,545 Hz (3.6 MHz): 279 ns resolution + if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { + if (raise_exc) { PyErr_SetString(PyExc_RuntimeError, - "invalid mach_timebase_info"); + "invalid QueryPerformanceFrequency"); } return -1; } + return 0; +} - /* Check that timebase.numer and timebase.denom can be casted to - _PyTime_t. In practice, timebase uses uint32_t, so casting cannot - overflow. At the end, only make sure that the type is uint32_t - (_PyTime_t is 64-bit long). */ - static_assert(sizeof(timebase.numer) <= sizeof(_PyTime_t), - "timebase.numer is larger than _PyTime_t"); - static_assert(sizeof(timebase.denom) <= sizeof(_PyTime_t), - "timebase.denom is larger than _PyTime_t"); - /* Make sure that _PyTime_MulDiv(ticks, timebase_numer, timebase_denom) - cannot overflow. +// N.B. If raise_exc=0, this may be called without the GIL. +static int +py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +{ + assert(info == NULL || raise_exc); - Known time bases: + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } - * (1, 1) on Intel - * (1000000000, 33333335) or (1000000000, 25000000) on PowerPC + if (info) { + info->implementation = "QueryPerformanceCounter()"; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); + info->monotonic = 1; + info->adjustable = 0; + } - None of these time bases can overflow with 64-bit _PyTime_t, but - check for overflow, just in case. */ - if ((_PyTime_t)timebase.numer > _PyTime_MAX / (_PyTime_t)timebase.denom) { - if (raise) { - PyErr_SetString(PyExc_OverflowError, - "mach_timebase_info is too large"); + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + LONGLONG ticksll = now.QuadPart; + + /* Make sure that casting LONGLONG to PyTime_t cannot overflow, + both types are signed */ + PyTime_t ticks; + static_assert(sizeof(ticksll) <= sizeof(ticks), + "LONGLONG is larger than PyTime_t"); + ticks = (PyTime_t)ticksll; + + *tp = _PyTimeFraction_Mul(ticks, &py_qpc_base); + return 0; +} +#endif // MS_WINDOWS + + +#ifdef __APPLE__ +static int +py_mach_timebase_info(_PyTimeFraction *base, int raise_exc) +{ + mach_timebase_info_data_t timebase; + // According to the Technical Q&A QA1398, mach_timebase_info() cannot + // fail: https://developer.apple.com/library/mac/#qa/qa1398/ + (void)mach_timebase_info(&timebase); + + // Check that timebase.numer and timebase.denom can be casted to + // PyTime_t. In practice, timebase uses uint32_t, so casting cannot + // overflow. At the end, only make sure that the type is uint32_t + // (PyTime_t is 64-bit long). + Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(PyTime_t)); + Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(PyTime_t)); + PyTime_t numer = (PyTime_t)timebase.numer; + PyTime_t denom = (PyTime_t)timebase.denom; + + // Known time bases: + // + // * (1, 1) on Intel: 1 ns + // * (1000000000, 33333335) on PowerPC: ~30 ns + // * (1000000000, 25000000) on PowerPC: 40 ns + if (_PyTimeFraction_Set(base, numer, denom) < 0) { + if (raise_exc) { + PyErr_SetString(PyExc_RuntimeError, + "invalid mach_timebase_info"); } return -1; } - - *pnumer = (_PyTime_t)timebase.numer; - *pdenom = (_PyTime_t)timebase.denom; return 0; } #endif +// N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); - -#if defined(MS_WINDOWS) - ULONGLONG ticks = GetTickCount64(); - static_assert(sizeof(ticks) <= sizeof(_PyTime_t), - "ULONGLONG is larger than _PyTime_t"); - _PyTime_t t; - if (ticks <= (ULONGLONG)_PyTime_MAX) { - t = (_PyTime_t)ticks; - } - else { - // GetTickCount64() maximum is larger than _PyTime_t maximum: - // ULONGLONG is unsigned, whereas _PyTime_t is signed. - t = _PyTime_MAX; + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); } - int res = pytime_mul(&t, MS_TO_NS); - *tp = t; - - if (raise_exc && res < 0) { - pytime_overflow(); +#if defined(MS_WINDOWS) + if (py_get_win_perf_counter(tp, info, raise_exc) < 0) { return -1; } - - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - info->implementation = "GetTickCount64()"; - info->monotonic = 1; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 0; - } - #elif defined(__APPLE__) - static _PyTime_t timebase_numer = 0; - static _PyTime_t timebase_denom = 0; - if (timebase_denom == 0) { - if (py_mach_timebase_info(&timebase_numer, &timebase_denom, raise_exc) < 0) { + static _PyTimeFraction base = {0, 0}; + if (base.denom == 0) { + if (py_mach_timebase_info(&base, raise_exc) < 0) { return -1; } } if (info) { info->implementation = "mach_absolute_time()"; - info->resolution = (double)timebase_numer / (double)timebase_denom * 1e-9; + info->resolution = _PyTimeFraction_Resolution(&base); info->monotonic = 1; info->adjustable = 0; } uint64_t uticks = mach_absolute_time(); // unsigned => signed - assert(uticks <= (uint64_t)_PyTime_MAX); - _PyTime_t ticks = (_PyTime_t)uticks; + assert(uticks <= (uint64_t)PyTime_MAX); + PyTime_t ticks = (PyTime_t)uticks; - _PyTime_t ns = _PyTime_MulDiv(ticks, timebase_numer, timebase_denom); - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); + *tp = ns; #elif defined(__hpux) - hrtime_t time; - - time = gethrtime(); + hrtime_t time = gethrtime(); if (time == -1) { if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); @@ -1143,7 +1184,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) return -1; } - *tp = pytime_from_nanoseconds(time); + *tp = time; if (info) { info->implementation = "gethrtime()"; @@ -1191,127 +1232,53 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetMonotonicClock(void) +int +PyTime_Monotonic(PyTime_t *result) { - _PyTime_t t; - if (py_get_monotonic_clock(&t, NULL, 0) < 0) { - // If mach_timebase_info(), clock_gettime() or gethrtime() fails: - // silently ignore the failure and return 0. - t = 0; + if (py_get_monotonic_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -_PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) -{ - return py_get_monotonic_clock(tp, info, 1); -} - - -#ifdef MS_WINDOWS -static int -py_win_perf_counter_frequency(LONGLONG *pfrequency, int raise) +PyTime_MonotonicRaw(PyTime_t *result) { - LONGLONG frequency; - - LARGE_INTEGER freq; - // Since Windows XP, the function cannot fail. - (void)QueryPerformanceFrequency(&freq); - frequency = freq.QuadPart; - - // Since Windows XP, frequency cannot be zero. - assert(frequency >= 1); - - /* Make also sure that (ticks * SEC_TO_NS) cannot overflow in - _PyTime_MulDiv(), with ticks < frequency. - - Known QueryPerformanceFrequency() values: - - * 10,000,000 (10 MHz): 100 ns resolution - * 3,579,545 Hz (3.6 MHz): 279 ns resolution - - None of these frequencies can overflow with 64-bit _PyTime_t, but - check for integer overflow just in case. */ - if (frequency > _PyTime_MAX / SEC_TO_NS) { - if (raise) { - PyErr_SetString(PyExc_OverflowError, - "QueryPerformanceFrequency is too large"); - } + if (py_get_monotonic_clock(result, NULL, 0) < 0) { + *result = 0; return -1; } - - *pfrequency = frequency; return 0; } -static int -py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +int +_PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { - assert(info == NULL || raise_exc); - - static LONGLONG frequency = 0; - if (frequency == 0) { - if (py_win_perf_counter_frequency(&frequency, raise_exc) < 0) { - return -1; - } - } - - if (info) { - info->implementation = "QueryPerformanceCounter()"; - info->resolution = 1.0 / (double)frequency; - info->monotonic = 1; - info->adjustable = 0; - } - - LARGE_INTEGER now; - QueryPerformanceCounter(&now); - LONGLONG ticksll = now.QuadPart; + return py_get_monotonic_clock(tp, info, 1); +} - /* Make sure that casting LONGLONG to _PyTime_t cannot overflow, - both types are signed */ - _PyTime_t ticks; - static_assert(sizeof(ticksll) <= sizeof(ticks), - "LONGLONG is larger than _PyTime_t"); - ticks = (_PyTime_t)ticksll; - _PyTime_t ns = _PyTime_MulDiv(ticks, SEC_TO_NS, (_PyTime_t)frequency); - *tp = pytime_from_nanoseconds(ns); - return 0; +int +_PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) +{ + return _PyTime_MonotonicWithInfo(t, info); } -#endif // MS_WINDOWS int -_PyTime_GetPerfCounterWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +PyTime_PerfCounter(PyTime_t *result) { -#ifdef MS_WINDOWS - return py_get_win_perf_counter(t, info, 1); -#else - return _PyTime_GetMonotonicClockWithInfo(t, info); -#endif + return PyTime_Monotonic(result); } -_PyTime_t -_PyTime_GetPerfCounter(void) +int +PyTime_PerfCounterRaw(PyTime_t *result) { - _PyTime_t t; - int res; -#ifdef MS_WINDOWS - res = py_get_win_perf_counter(&t, NULL, 0); -#else - res = py_get_monotonic_clock(&t, NULL, 0); -#endif - if (res < 0) { - // If py_win_perf_counter_frequency() or py_get_monotonic_clock() - // fails: silently ignore the failure and return 0. - t = 0; - } - return t; + return PyTime_MonotonicRaw(result); } @@ -1382,17 +1349,21 @@ _PyTime_gmtime(time_t t, struct tm *tm) } -_PyTime_t -_PyDeadline_Init(_PyTime_t timeout) +PyTime_t +_PyDeadline_Init(PyTime_t timeout) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return _PyTime_Add(now, timeout); } -_PyTime_t -_PyDeadline_Get(_PyTime_t deadline) +PyTime_t +_PyDeadline_Get(PyTime_t deadline) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return deadline - now; } diff --git a/Python/qsbr.c b/Python/qsbr.c new file mode 100644 index 00000000000000..a40219acfe2c29 --- /dev/null +++ b/Python/qsbr.c @@ -0,0 +1,289 @@ +/* + * Implementation of safe memory reclamation scheme using + * quiescent states. + * + * This is derived from the "GUS" safe memory reclamation technique + * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs + * in this code are likely due to the modifications. + * + * The original copyright is preserved below. + * + * Copyright (c) 2019,2020 Jeffrey Roberson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "Python.h" +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_Lock() +#include "pycore_qsbr.h" +#include "pycore_pystate.h" // _PyThreadState_GET() + + +// Starting size of the array of qsbr thread states +#define MIN_ARRAY_SIZE 8 + +// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing +// the write sequence. +#define QSBR_DEFERRED_LIMIT 10 + +// Allocate a QSBR thread state from the freelist +static struct _qsbr_thread_state * +qsbr_allocate(struct _qsbr_shared *shared) +{ + struct _qsbr_thread_state *qsbr = shared->freelist; + if (qsbr == NULL) { + return NULL; + } + shared->freelist = qsbr->freelist_next; + qsbr->freelist_next = NULL; + qsbr->shared = shared; + qsbr->allocated = true; + return qsbr; +} + +// Initialize (or reintialize) the freelist of QSBR thread states +static void +initialize_new_array(struct _qsbr_shared *shared) +{ + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr->tstate != NULL) { + // Update the thread state pointer to its QSBR state + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)qsbr->tstate; + tstate->qsbr = qsbr; + } + if (!qsbr->allocated) { + // Push to freelist + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} + +// Grow the array of QSBR thread states. Returns 0 on success, -1 on failure. +static int +grow_thread_array(struct _qsbr_shared *shared) +{ + Py_ssize_t new_size = shared->size * 2; + if (new_size < MIN_ARRAY_SIZE) { + new_size = MIN_ARRAY_SIZE; + } + + struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); + if (array == NULL) { + return -1; + } + + struct _qsbr_pad *old = shared->array; + if (old != NULL) { + memcpy(array, shared->array, shared->size * sizeof(*array)); + } + + shared->array = array; + shared->size = new_size; + shared->freelist = NULL; + initialize_new_array(shared); + + PyMem_RawFree(old); + return 0; +} + +uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared) +{ + // NOTE: with 64-bit sequence numbers, we don't have to worry too much + // about the wr_seq getting too far ahead of rd_seq, but if we ever use + // 32-bit sequence numbers, we'll need to be more careful. + return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; +} + +uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +{ + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { + return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; + } + qsbr->deferrals = 0; + return _Py_qsbr_advance(qsbr->shared); +} + +static uint64_t +qsbr_poll_scan(struct _qsbr_shared *shared) +{ + // Synchronize with store in _Py_qsbr_attach(). We need to ensure that + // the reads from each thread's sequence number are not reordered to see + // earlier "offline" states. + _Py_atomic_fence_seq_cst(); + + // Compute the minimum sequence number of all attached threads + uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq); + struct _qsbr_pad *array = shared->array; + for (Py_ssize_t i = 0, size = shared->size; i != size; i++) { + struct _qsbr_thread_state *qsbr = &array[i].qsbr; + + uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq); + if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) { + min_seq = seq; + } + } + + // Update the shared read sequence + uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq); + if (QSBR_LT(rd_seq, min_seq)) { + // It's okay if the compare-exchange failed: another thread updated it + (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq); + rd_seq = min_seq; + } + + return rd_seq; +} + +bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + assert(_Py_atomic_load_int_relaxed(&_PyThreadState_GET()->state) == _Py_THREAD_ATTACHED); + + if (_Py_qbsr_goal_reached(qsbr, goal)) { + return true; + } + + uint64_t rd_seq = qsbr_poll_scan(qsbr->shared); + return QSBR_LEQ(goal, rd_seq); +} + +void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq == 0 && "already attached"); + + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64(&qsbr->seq, seq); // needs seq_cst +} + +void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq != 0 && "already detached"); + + _Py_atomic_store_uint64_release(&qsbr->seq, QSBR_OFFLINE); +} + +Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + // Try allocating from our internal freelist + struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); + + // If there are no free entries, we pause all threads, grow the array, + // and update the pointers in PyThreadState to entries in the new array. + if (qsbr == NULL) { + _PyEval_StopTheWorld(interp); + if (grow_thread_array(shared) == 0) { + qsbr = qsbr_allocate(shared); + } + _PyEval_StartTheWorld(interp); + } + PyMutex_Unlock(&shared->mutex); + + if (qsbr == NULL) { + return -1; + } + + // Return an index rather than the pointer because the array may be + // resized and the pointer invalidated. + return (struct _qsbr_pad *)qsbr - shared->array; +} + +void +_Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, + Py_ssize_t index) +{ + // Associate the QSBR state with the thread state + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; + assert(qsbr->allocated && qsbr->tstate == NULL); + qsbr->tstate = (PyThreadState *)tstate; + tstate->qsbr = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_unregister(PyThreadState *tstate) +{ + struct _qsbr_shared *shared = &tstate->interp->qsbr; + struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; + + // gh-119369: GIL must be released (if held) to prevent deadlocks, because + // we might not have an active tstate, which means that blocking on PyMutex + // locks will not implicitly release the GIL. + assert(!tstate->_status.holds_gil); + + PyMutex_Lock(&shared->mutex); + // NOTE: we must load (or reload) the thread state's qbsr inside the mutex + // because the array may have been resized (changing tstate->qsbr) while + // we waited to acquire the mutex. + struct _qsbr_thread_state *qsbr = tstate_imp->qsbr; + + assert(qsbr->seq == 0 && "thread state must be detached"); + assert(qsbr->allocated && qsbr->tstate == tstate); + + tstate_imp->qsbr = NULL; + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_fini(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + PyMem_RawFree(shared->array); + shared->array = NULL; + shared->size = 0; + shared->freelist = NULL; +} + +void +_Py_qsbr_after_fork(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *this_qsbr = tstate->qsbr; + struct _qsbr_shared *shared = this_qsbr->shared; + + _PyMutex_at_fork_reinit(&shared->mutex); + + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr != this_qsbr && qsbr->allocated) { + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} diff --git a/Python/specialize.c b/Python/specialize.c index 49633b103b3815..497feca06e4c7f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -10,11 +10,15 @@ #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_uop_metadata.h" // _PyOpcode_uop_name +#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() #include // rand() +extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see * ./adaptive.md @@ -100,6 +104,7 @@ _Py_GetSpecializationStats(void) { return NULL; } int err = 0; + err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); @@ -134,7 +139,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i]) { + if (_PyOpcode_Caches[i] && i != JUMP_BACKWARD) { fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); } PRINT_STAT(i, specialization.success); @@ -184,7 +189,7 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); @@ -193,7 +198,6 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -211,6 +215,7 @@ print_gc_stats(FILE *out, GCStats *stats) } } +#ifdef _Py_TIER2 static void print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) { @@ -232,23 +237,27 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); - const char* const* names; - for (int i = 0; i < 512; i++) { - if (i < 256) { - names = _PyOpcode_OpName; - } else { - names = _PyOpcode_uop_name; - } + fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); + fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); + fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", + stats->optimizer_failure_reason_no_memory); + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); + for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", names[i], stats->opcode[i].execution_count); + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); + } + if (stats->opcode[i].miss) { + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } - for (int i = 0; i < 256; i++) { if (stats->unsupported_opcode[i]) { fprintf( @@ -259,6 +268,38 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) ); } } + + for (int i = 1; i <= MAX_UOP_ID; i++){ + for (int j = 1; j <= MAX_UOP_ID; j++) { + if (stats->opcode[i].pair_count[j]) { + fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); + } + } + } + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } +} +#endif + +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); } static void @@ -268,7 +309,10 @@ print_stats(FILE *out, PyStats *stats) print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 print_optimization_stats(out, &stats->optimization_stats); +#endif + print_rare_event_stats(out, &stats->rare_event_stats); } void @@ -386,22 +430,20 @@ _PyCode_Quicken(PyCodeObject *code) int caches = _PyOpcode_Caches[opcode]; if (caches) { // The initial value depends on the opcode - int initial_value; switch (opcode) { case JUMP_BACKWARD: - initial_value = 0; + instructions[i + 1].counter = initial_jump_backoff_counter(); break; case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: - initial_value = 0x5555; // Alternating 0, 1 bits + instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits break; default: - initial_value = adaptive_counter_warmup(); + instructions[i + 1].counter = adaptive_counter_warmup(); break; } - instructions[i + 1].cache = initial_value; i += caches; } } @@ -445,12 +487,11 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 - #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 + #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -523,6 +564,8 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 +#define SPEC_FAIL_CALL_METACLASS 31 +#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -577,9 +620,16 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_TO_BOOL_SET 17 #define SPEC_FAIL_TO_BOOL_TUPLE 18 +// CONTAINS_OP +#define SPEC_FAIL_CONTAINS_OP_STR 9 +#define SPEC_FAIL_CONTAINS_OP_TUPLE 10 +#define SPEC_FAIL_CONTAINS_OP_LIST 11 +#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12 + static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); +static uint32_t type_get_version(PyTypeObject *t, int opcode); static int specialize_module_load_attr( @@ -587,7 +637,7 @@ specialize_module_load_attr( ) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyModuleObject *m = (PyModuleObject *)owner; - assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); + assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); PyDictObject *dict = (PyDictObject *)m->md_dict; if (dict == NULL) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); @@ -629,7 +679,10 @@ specialize_module_load_attr( /* Attribute specialization */ void -_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) { +_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); @@ -738,7 +791,7 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto if (desc_cls == &PyMemberDescr_Type) { PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor; struct PyMemberDef *dmem = member->d_member; - if (dmem->type == Py_T_OBJECT_EX) { + if (dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT) { return OBJECT_SLOT; } return OTHER_SLOT; @@ -787,11 +840,7 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(*dorv) || - _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - // Virtual dictionary + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) { PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); @@ -808,7 +857,7 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; @@ -839,8 +888,10 @@ static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, P static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); void -_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); @@ -868,6 +919,9 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) PyObject *descr = NULL; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); + if (type_get_version(type, LOAD_ATTR) == 0) { + goto fail; + } switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -938,7 +992,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - assert(dmem->type == Py_T_OBJECT_EX); + assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); @@ -1032,8 +1086,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } void -_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +_Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); @@ -1051,6 +1107,9 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } PyObject *descr; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); + if (type_get_version(type, STORE_ATTR) == 0) { + goto fail; + } switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1078,7 +1137,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - assert(dmem->type == Py_T_OBJECT_EX); + assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); @@ -1177,6 +1236,9 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *descr = NULL; DescriptorClassification kind = 0; kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); + if (type_get_version((PyTypeObject *)owner, LOAD_ATTR) == 0) { + return -1; + } switch (kind) { case METHOD: case NON_DESCRIPTOR: @@ -1207,15 +1269,8 @@ PyObject *descr, DescriptorClassification kind, bool is_method) assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - return 0; - } Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1231,10 +1286,16 @@ PyObject *descr, DescriptorClassification kind, bool is_method) instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; } else { - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; - if (dictoffset < 0 || dictoffset > INT16_MAX) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); - return 0; + Py_ssize_t dictoffset; + if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dictoffset = MANAGED_DICT_OFFSET; + } + else { + dictoffset = owner_cls->tp_dictoffset; + if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } } if (dictoffset == 0) { instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT; @@ -1245,8 +1306,12 @@ PyObject *descr, DescriptorClassification kind, bool is_method) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - assert(owner_cls->tp_dictoffset > 0); - assert(owner_cls->tp_dictoffset <= INT16_MAX); + /* Cache entries must be unsigned values, so we offset the + * dictoffset by MANAGED_DICT_OFFSET. + * We do the reverse offset in LOAD_ATTR_METHOD_LAZY_DICT */ + dictoffset -= MANAGED_DICT_OFFSET; + assert(((uint16_t)dictoffset) == dictoffset); + cache->dict_offset = (uint16_t)dictoffset; instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; } else { @@ -1449,10 +1514,25 @@ function_get_version(PyObject *o, int opcode) return version; } +/* Returning 0 indicates a failure. */ +static uint32_t +type_get_version(PyTypeObject *t, int opcode) +{ + uint32_t version = t->tp_version_tag; + if (version == 0) { + SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); + return 0; + } + return version; +} + void _Py_Specialize_BinarySubscr( - PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) + _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -1551,8 +1631,11 @@ _Py_Specialize_BinarySubscr( } void -_Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) +_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(ENABLE_SPECIALIZATION); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); @@ -1666,8 +1749,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp) SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); return NULL; } if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { @@ -1718,8 +1801,14 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (Py_TYPE(tp) != &PyType_Type) { + goto generic; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (type_get_version(tp, CALL) == 0) { + return -1; + } if (init != NULL) { if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); @@ -1730,58 +1819,11 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); return 0; } - return -1; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); - return -1; -} - -#ifdef Py_STATS -static int -builtin_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_CFUNC_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_NOARGS: - return SPEC_FAIL_CALL_CFUNC_NOARGS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; - } -} - -static int -meth_descr_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_NOARGS: - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } +generic: + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } -#endif // Py_STATS static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, @@ -1824,8 +1866,8 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, return 0; } } - SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); - return -1; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } static int @@ -1840,36 +1882,25 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } - if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, kind); + int argcount = -1; + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; } - int argcount = code->co_argcount; - int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); - int min_args = argcount-defcount; - // GH-105840: min_args is negative when somebody sets too many __defaults__! - if (min_args < 0 || nargs > argcount || nargs < min_args) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; + if (kind == SIMPLE_FUNCTION) { + argcount = code->co_argcount; } - assert(nargs <= argcount && nargs >= min_args); - assert(min_args >= 0 && defcount >= 0); - assert(defcount == 0 || func->func_defaults != NULL); int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - if (argcount == nargs) { + if (argcount == nargs + bound_method) { instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } - else if (bound_method) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); - return -1; - } else { - instr->op.code = CALL_PY_WITH_DEFAULTS; + instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } return 0; } @@ -1878,6 +1909,7 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { if (PyCFunction_GET_FUNCTION(callable) == NULL) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } switch (PyCFunction_GET_FLAGS(callable) & @@ -1914,42 +1946,16 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return 0; } default: - SPECIALIZATION_FAIL(CALL, - builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); - return 1; - } -} - -#ifdef Py_STATS -static int -call_fail_kind(PyObject *callable) -{ - assert(!PyCFunction_CheckExact(callable)); - assert(!PyFunction_Check(callable)); - assert(!PyType_Check(callable)); - assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); - assert(!PyMethod_Check(callable)); - if (PyInstanceMethod_Check(callable)) { - return SPEC_FAIL_CALL_INSTANCE_METHOD; - } - // builtin method - else if (PyCMethod_Check(callable)) { - return SPEC_FAIL_CALL_CMETHOD; - } - else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { - return SPEC_FAIL_CALL_OPERATOR_WRAPPER; - } - else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) { - return SPEC_FAIL_CALL_METHOD_WRAPPER; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } - return SPEC_FAIL_OTHER; } -#endif // Py_STATS - void -_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); @@ -1970,7 +1976,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call((PyFunctionObject *)func, instr, nargs+1, true); + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true); } else { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); @@ -1978,8 +1984,8 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } } else { - SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); - fail = -1; + instr->op.code = CALL_NON_PY_GENERAL; + fail = 0; } if (fail) { STAT_INC(CALL, failure); @@ -2065,9 +2071,11 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) #endif // Py_STATS void -_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals) +_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals) { + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); @@ -2080,7 +2088,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, if (PyUnicode_CheckExact(lhs)) { _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; bool to_store = (next.op.code == STORE_FAST); - if (to_store && locals[next.op.arg] == lhs) { + if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) { instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE; goto success; } @@ -2172,9 +2180,12 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) #endif // Py_STATS void -_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, +_Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg) { + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); // All of these specializations compute boolean values, so they're all valid @@ -2235,8 +2246,10 @@ unpack_sequence_fail_kind(PyObject *seq) #endif // Py_STATS void -_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg) { + PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); @@ -2346,12 +2359,13 @@ int #endif // Py_STATS void -_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); - PyTypeObject *tp = Py_TYPE(iter); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyTypeObject *tp = Py_TYPE(iter_o); if (tp == &PyListIter_Type) { instr->op.code = FOR_ITER_LIST; goto success; @@ -2376,7 +2390,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) goto success; } SPECIALIZATION_FAIL(FOR_ITER, - _PySpecialization_ClassifyIterator(iter)); + _PySpecialization_ClassifyIterator(iter_o)); failure: STAT_INC(FOR_ITER, failure); instr->op.code = FOR_ITER; @@ -2388,8 +2402,10 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) } void -_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) +_Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { + PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); _PySendCache *cache = (_PySendCache *)(instr + 1); @@ -2415,11 +2431,12 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) } void -_Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) +_Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL); _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1); + PyObject *value = PyStackRef_AsPyObjectBorrow(value_o); if (PyBool_Check(value)) { instr->op.code = TO_BOOL_BOOL; goto success; @@ -2460,7 +2477,10 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OUT_OF_VERSIONS); goto failure; } - uint32_t version = Py_TYPE(value)->tp_version_tag; + uint32_t version = type_get_version(Py_TYPE(value), TO_BOOL); + if (version == 0) { + goto failure; + } instr->op.code = TO_BOOL_ALWAYS_TRUE; write_u32(cache->version, version); assert(version); @@ -2507,6 +2527,51 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) cache->counter = adaptive_counter_cooldown(); } +#ifdef Py_STATS +static int containsop_fail_kind(PyObject *value) { + if (PyUnicode_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_STR; + } + if (PyList_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_LIST; + } + if (PyTuple_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_TUPLE; + } + if (PyType_Check(value)) { + return SPEC_FAIL_CONTAINS_OP_USER_CLASS; + } + return SPEC_FAIL_OTHER; +} +#endif // Py_STATS + +void +_Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) +{ + PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); + _PyContainsOpCache *cache = (_PyContainsOpCache *)(instr + 1); + if (PyDict_CheckExact(value)) { + instr->op.code = CONTAINS_OP_DICT; + goto success; + } + if (PySet_CheckExact(value) || PyFrozenSet_CheckExact(value)) { + instr->op.code = CONTAINS_OP_SET; + goto success; + } + + SPECIALIZATION_FAIL(CONTAINS_OP, containsop_fail_kind(value)); + STAT_INC(CONTAINS_OP, failure); + instr->op.code = CONTAINS_OP; + cache->counter = adaptive_counter_backoff(cache->counter); + return; +success: + STAT_INC(CONTAINS_OP, success); + cache->counter = adaptive_counter_cooldown(); +} + /* Code init cleanup. * CALL_ALLOC_AND_ENTER_INIT will set up * the frame to execute the EXIT_INIT_CHECK @@ -2524,11 +2589,11 @@ static const PyBytesObject no_location = { }; const struct _PyCode_DEF(8) _Py_InitCleanup = { - _PyVarObject_HEAD_INIT(&PyCode_Type, 4) + _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), - .co_flags = CO_OPTIMIZED, + .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS, .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), .co_filename = &_Py_ID(__init__), @@ -2539,9 +2604,8 @@ const struct _PyCode_DEF(8) _Py_InitCleanup = { .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, .co_code_adaptive = { - NOP, 0, EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, - RESUME, 0, + RESUME, RESUME_AT_FUNC_START, } }; diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 701bfc35cc8182..9686d10563aa4d 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", "_ast", "_asyncio", "_bisect", @@ -19,6 +20,7 @@ static const char* _Py_stdlib_module_names[] = { "_codecs_tw", "_collections", "_collections_abc", +"_colorize", "_compat_pickle", "_compression", "_contextvars", @@ -37,7 +39,11 @@ static const char* _Py_stdlib_module_names[] = { "_hashlib", "_heapq", "_imp", +"_interpchannels", +"_interpqueues", +"_interpreters", "_io", +"_ios_support", "_json", "_locale", "_lsprof", @@ -59,6 +65,7 @@ static const char* _Py_stdlib_module_names[] = { "_pydecimal", "_pyio", "_pylong", +"_pyrepl", "_queue", "_random", "_scproxy", @@ -76,6 +83,7 @@ static const char* _Py_stdlib_module_names[] = { "_string", "_strptime", "_struct", +"_suggestions", "_symtable", "_sysconfig", "_thread", diff --git a/Python/structmember.c b/Python/structmember.c index 7a5a6a49d23116..d5e7ab83093dc8 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -2,8 +2,24 @@ /* Map C struct members to Python object attributes */ #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_object.h" // _Py_TryIncrefCompare(), FT_ATOMIC_*() +#include "pycore_critical_section.h" +static inline PyObject * +member_get_object(const char *addr, const char *obj_addr, PyMemberDef *l) +{ + PyObject *v = FT_ATOMIC_LOAD_PTR(*(PyObject **) addr); + if (v == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%T' object has no attribute '%s'", + (PyObject *)obj_addr, l->name); + } + return v; +} + PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { @@ -73,15 +89,19 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) Py_INCREF(v); break; case Py_T_OBJECT_EX: - v = *(PyObject **)addr; - if (v == NULL) { - PyObject *obj = (PyObject *)obj_addr; - PyTypeObject *tp = Py_TYPE(obj); - PyErr_Format(PyExc_AttributeError, - "'%.200s' object has no attribute '%s'", - tp->tp_name, l->name); - } + v = member_get_object(addr, obj_addr, l); +#ifndef Py_GIL_DISABLED Py_XINCREF(v); +#else + if (v != NULL) { + if (!_Py_TryIncrefCompare((PyObject **) addr, v)) { + Py_BEGIN_CRITICAL_SECTION((PyObject *) obj_addr); + v = member_get_object(addr, obj_addr, l); + Py_XINCREF(v); + Py_END_CRITICAL_SECTION(); + } + } +#endif break; case Py_T_LONGLONG: v = PyLong_FromLongLong(*(long long *)addr); @@ -90,6 +110,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); break; case _Py_T_NONE: + // doesn't require free-threading code path v = Py_NewRef(Py_None); break; default: @@ -116,6 +137,9 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } +#ifdef Py_GIL_DISABLED + PyObject *obj = (PyObject *) addr; +#endif addr += l->offset; if ((l->flags & Py_READONLY)) @@ -197,45 +221,67 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) WARN("Truncation of value to int"); break; } - case Py_T_UINT:{ - unsigned long ulong_val = PyLong_AsUnsignedLong(v); - if ((ulong_val == (unsigned long)-1) && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - ulong_val = PyLong_AsLong(v); - if ((ulong_val == (unsigned long)-1) && - PyErr_Occurred()) + case Py_T_UINT: { + /* XXX: For compatibility, accept negative int values + as well. */ + v = _PyNumber_Index(v); + if (v == NULL) { + return -1; + } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { return -1; - *(unsigned int *)addr = (unsigned int)ulong_val; + } + *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; WARN("Writing negative value into unsigned field"); - } else - *(unsigned int *)addr = (unsigned int)ulong_val; - if (ulong_val > UINT_MAX) - WARN("Truncation of value to unsigned int"); - break; } + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + Py_DECREF(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned int*)addr = (unsigned int)ulong_val; + if (ulong_val > UINT_MAX) { + WARN("Truncation of value to unsigned int"); + } + } + break; + } case Py_T_LONG:{ *(long*)addr = PyLong_AsLong(v); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; } - case Py_T_ULONG:{ - *(unsigned long*)addr = PyLong_AsUnsignedLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) { - /* XXX: For compatibility, accept negative int values - as well. */ - PyErr_Clear(); - *(unsigned long*)addr = PyLong_AsLong(v); - if ((*(unsigned long*)addr == (unsigned long)-1) - && PyErr_Occurred()) + case Py_T_ULONG: { + /* XXX: For compatibility, accept negative int values + as well. */ + v = _PyNumber_Index(v); + if (v == NULL) { + return -1; + } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { return -1; + } + *(unsigned long *)addr = (unsigned long)long_val; WARN("Writing negative value into unsigned field"); } - break; + else { + unsigned long ulong_val = PyLong_AsUnsignedLong(v); + Py_DECREF(v); + if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long*)addr = ulong_val; } + break; + } case Py_T_PYSSIZET:{ *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) @@ -257,8 +303,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; case _Py_T_OBJECT: case Py_T_OBJECT_EX: + Py_BEGIN_CRITICAL_SECTION(obj); oldv = *(PyObject **)addr; - *(PyObject **)addr = Py_XNewRef(v); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, Py_XNewRef(v)); + Py_END_CRITICAL_SECTION(); Py_XDECREF(oldv); break; case Py_T_CHAR: { @@ -284,18 +332,30 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; break; } - case Py_T_ULONGLONG:{ - unsigned long long value; - /* ??? PyLong_AsLongLong accepts an int, but PyLong_AsUnsignedLongLong - doesn't ??? */ - if (PyLong_Check(v)) - *(unsigned long long*)addr = value = PyLong_AsUnsignedLongLong(v); - else - *(unsigned long long*)addr = value = PyLong_AsLong(v); - if ((value == (unsigned long long)-1) && PyErr_Occurred()) + case Py_T_ULONGLONG: { + v = _PyNumber_Index(v); + if (v == NULL) { return -1; - break; } + if (_PyLong_IsNegative((PyLongObject *)v)) { + long long_val = PyLong_AsLong(v); + Py_DECREF(v); + if (long_val == -1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + WARN("Writing negative value into unsigned field"); + } + else { + unsigned long long ulonglong_val = PyLong_AsUnsignedLongLong(v); + Py_DECREF(v); + if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + *(unsigned long long*)addr = ulonglong_val; + } + break; + } default: PyErr_Format(PyExc_SystemError, "bad memberdescr type for %s", l->name); diff --git a/Python/suggestions.c b/Python/suggestions.c index 1ad359b18923f3..2ce0bcfd54e23f 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -4,8 +4,6 @@ #include "pycore_pyerrors.h" // export _Py_UTF8_Edit_Cost() #include "pycore_runtime.h" // _Py_ID() -#include "stdlib_module_names.h" // _Py_stdlib_module_names - #define MAX_CANDIDATE_ITEMS 750 #define MAX_STRING_SIZE 40 @@ -148,7 +146,7 @@ _Py_CalculateSuggestions(PyObject *dir, if (buffer == NULL) { return PyErr_NoMemory(); } - for (int i = 0; i < dir_size; ++i) { + for (Py_ssize_t i = 0; i < dir_size; ++i) { PyObject *item = PyList_GET_ITEM(dir, i); if (_PyUnicode_Equal(name, item)) { continue; @@ -178,225 +176,6 @@ _Py_CalculateSuggestions(PyObject *dir, return Py_XNewRef(suggestion); } -static PyObject * -get_suggestions_for_attribute_error(PyAttributeErrorObject *exc) -{ - PyObject *name = exc->name; // borrowed reference - PyObject *obj = exc->obj; // borrowed reference - - // Abort if we don't have an attribute name or we have an invalid one - if (name == NULL || obj == NULL || !PyUnicode_CheckExact(name)) { - return NULL; - } - - PyObject *dir = PyObject_Dir(obj); - if (dir == NULL) { - return NULL; - } - - PyObject *suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - return suggestions; -} - -static PyObject * -offer_suggestions_for_attribute_error(PyAttributeErrorObject *exc) -{ - PyObject* suggestion = get_suggestions_for_attribute_error(exc); - if (suggestion == NULL) { - return NULL; - } - // Add a trailer ". Did you mean: (...)?" - PyObject* result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - Py_DECREF(suggestion); - return result; -} - -static PyObject * -get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) -{ - PyCodeObject *code = PyFrame_GetCode(frame); - assert(code != NULL && code->co_localsplusnames != NULL); - - PyObject *varnames = _PyCode_GetVarnames(code); - Py_DECREF(code); - if (varnames == NULL) { - return NULL; - } - PyObject *dir = PySequence_List(varnames); - Py_DECREF(varnames); - if (dir == NULL) { - return NULL; - } - - // Are we inside a method and the instance has an attribute called 'name'? - int res = PySequence_Contains(dir, &_Py_ID(self)); - if (res < 0) { - goto error; - } - if (res > 0) { - PyObject* locals = PyFrame_GetLocals(frame); - if (!locals) { - goto error; - } - PyObject* self = PyDict_GetItemWithError(locals, &_Py_ID(self)); /* borrowed */ - if (!self) { - Py_DECREF(locals); - goto error; - } - - res = PyObject_HasAttrWithError(self, name); - Py_DECREF(locals); - if (res < 0) { - goto error; - } - if (res) { - Py_DECREF(dir); - return PyUnicode_FromFormat("self.%U", name); - } - } - - PyObject *suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (suggestions != NULL || PyErr_Occurred()) { - return suggestions; - } - - dir = PySequence_List(frame->f_frame->f_globals); - if (dir == NULL) { - return NULL; - } - suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (suggestions != NULL || PyErr_Occurred()) { - return suggestions; - } - - dir = PySequence_List(frame->f_frame->f_builtins); - if (dir == NULL) { - return NULL; - } - suggestions = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - - return suggestions; - -error: - Py_DECREF(dir); - return NULL; -} - -static bool -is_name_stdlib_module(PyObject* name) -{ - const char* the_name = PyUnicode_AsUTF8(name); - Py_ssize_t len = Py_ARRAY_LENGTH(_Py_stdlib_module_names); - for (Py_ssize_t i = 0; i < len; i++) { - if (strcmp(the_name, _Py_stdlib_module_names[i]) == 0) { - return 1; - } - } - return 0; -} - -static PyObject * -offer_suggestions_for_name_error(PyNameErrorObject *exc) -{ - PyObject *name = exc->name; // borrowed reference - PyTracebackObject *traceback = (PyTracebackObject *) exc->traceback; // borrowed reference - // Abort if we don't have a variable name or we have an invalid one - // or if we don't have a traceback to work with - if (name == NULL || !PyUnicode_CheckExact(name) || - traceback == NULL || !Py_IS_TYPE(traceback, &PyTraceBack_Type) - ) { - return NULL; - } - - // Move to the traceback of the exception - while (1) { - PyTracebackObject *next = traceback->tb_next; - if (next == NULL || !Py_IS_TYPE(next, &PyTraceBack_Type)) { - break; - } - else { - traceback = next; - } - } - - PyFrameObject *frame = traceback->tb_frame; - assert(frame != NULL); - - PyObject* suggestion = get_suggestions_for_name_error(name, frame); - if (suggestion == NULL && PyErr_Occurred()) { - return NULL; - } - - // Add a trailer ". Did you mean: (...)?" - PyObject* result = NULL; - if (!is_name_stdlib_module(name)) { - if (suggestion == NULL) { - return NULL; - } - result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - } else if (suggestion == NULL) { - result = PyUnicode_FromFormat(". Did you forget to import %R?", name); - } else { - result = PyUnicode_FromFormat(". Did you mean: %R? Or did you forget to import %R?", suggestion, name); - } - Py_XDECREF(suggestion); - return result; -} - -static PyObject * -offer_suggestions_for_import_error(PyImportErrorObject *exc) -{ - PyObject *mod_name = exc->name; // borrowed reference - PyObject *name = exc->name_from; // borrowed reference - if (name == NULL || mod_name == NULL || name == Py_None || - !PyUnicode_CheckExact(name) || !PyUnicode_CheckExact(mod_name)) { - return NULL; - } - - PyObject* mod = PyImport_GetModule(mod_name); - if (mod == NULL) { - return NULL; - } - - PyObject *dir = PyObject_Dir(mod); - Py_DECREF(mod); - if (dir == NULL) { - return NULL; - } - - PyObject *suggestion = _Py_CalculateSuggestions(dir, name); - Py_DECREF(dir); - if (!suggestion) { - return NULL; - } - - PyObject* result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); - Py_DECREF(suggestion); - return result; -} - -// Offer suggestions for a given exception. Returns a python string object containing the -// suggestions. This function returns NULL if no suggestion was found or if an exception happened, -// users must call PyErr_Occurred() to disambiguate. -PyObject * -_Py_Offer_Suggestions(PyObject *exception) -{ - PyObject *result = NULL; - assert(!PyErr_Occurred()); - if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_AttributeError)) { - result = offer_suggestions_for_attribute_error((PyAttributeErrorObject *) exception); - } else if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_NameError)) { - result = offer_suggestions_for_name_error((PyNameErrorObject *) exception); - } else if (Py_IS_TYPE(exception, (PyTypeObject*)PyExc_ImportError)) { - result = offer_suggestions_for_import_error((PyImportErrorObject *) exception); - } - return result; -} - Py_ssize_t _Py_UTF8_Edit_Cost(PyObject *a, PyObject *b, Py_ssize_t max_cost) { diff --git a/Python/symtable.c b/Python/symtable.c index 75ea9e902f4381..10103dbc2582a2 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -58,29 +58,33 @@ #define ANNOTATION_NOT_ALLOWED \ "%s cannot be used within an annotation" -#define TYPEVAR_BOUND_NOT_ALLOWED \ -"%s cannot be used within a TypeVar bound" +#define EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE \ +"%s cannot be used within %s" -#define TYPEALIAS_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_ALIAS \ "%s cannot be used within a type alias" -#define TYPEPARAM_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS \ "%s cannot be used within the definition of a generic" #define DUPLICATE_TYPE_PARAM \ "duplicate type parameter '%U'" +#define ASYNC_WITH_OUTISDE_ASYNC_FUNC \ +"'async with' outside async function" -#define LOCATION(x) \ - (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset +#define ASYNC_FOR_OUTISDE_ASYNC_FUNC \ +"'async for' outside async function" -#define ST_LOCATION(x) \ - (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset +#define LOCATION(x) SRC_LOCATION_FROM_AST(x) + +#define SET_ERROR_LOCATION(FNAME, L) \ + PyErr_RangedSyntaxLocationObject((FNAME), \ + (L).lineno, (L).col_offset + 1, (L).end_lineno, (L).end_col_offset + 1) static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, - void *key, int lineno, int col_offset, - int end_lineno, int end_col_offset) + void *key, _Py_SourceLocation loc) { PySTEntryObject *ste = NULL; PyObject *k = NULL; @@ -103,18 +107,17 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_children = NULL; ste->ste_directives = NULL; + ste->ste_mangled_names = NULL; ste->ste_type = block; + ste->ste_scope_info = NULL; + ste->ste_nested = 0; ste->ste_free = 0; ste->ste_varargs = 0; ste->ste_varkeywords = 0; - ste->ste_opt_lineno = 0; - ste->ste_opt_col_offset = 0; - ste->ste_lineno = lineno; - ste->ste_col_offset = col_offset; - ste->ste_end_lineno = end_lineno; - ste->ste_end_col_offset = end_col_offset; + ste->ste_annotations_used = 0; + ste->ste_loc = loc; if (st->st_cur != NULL && (st->st_cur->ste_nested || @@ -131,6 +134,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; ste->ste_needs_classdict = 0; + ste->ste_annotation_block = NULL; ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -153,7 +157,7 @@ static PyObject * ste_repr(PySTEntryObject *ste) { return PyUnicode_FromFormat("", - ste->ste_name, ste->ste_id, ste->ste_lineno); + ste->ste_name, ste->ste_id, ste->ste_loc.lineno); } static void @@ -166,6 +170,8 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_annotation_block); + Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -179,7 +185,7 @@ static PyMemberDef ste_memberlist[] = { {"children", _Py_T_OBJECT, OFF(ste_children), Py_READONLY}, {"nested", Py_T_INT, OFF(ste_nested), Py_READONLY}, {"type", Py_T_INT, OFF(ste_type), Py_READONLY}, - {"lineno", Py_T_INT, OFF(ste_lineno), Py_READONLY}, + {"lineno", Py_T_INT, OFF(ste_loc.lineno), Py_READONLY}, {NULL} }; @@ -226,9 +232,7 @@ PyTypeObject PySTEntry_Type = { static int symtable_analyze(struct symtable *st); static int symtable_enter_block(struct symtable *st, identifier name, - _Py_block_ty block, void *ast, - int lineno, int col_offset, - int end_lineno, int end_col_offset); + _Py_block_ty block, void *ast, _Py_SourceLocation loc); static int symtable_exit_block(struct symtable *st); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); @@ -243,14 +247,16 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); -static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, + struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); +static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty); /* For debugging purposes only */ @@ -263,9 +269,9 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) case ClassBlock: blocktype = "ClassBlock"; break; case ModuleBlock: blocktype = "ModuleBlock"; break; case AnnotationBlock: blocktype = "AnnotationBlock"; break; - case TypeVarBoundBlock: blocktype = "TypeVarBoundBlock"; break; + case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; - case TypeParamBlock: blocktype = "TypeParamBlock"; break; + case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { @@ -303,8 +309,8 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, - ste->ste_lineno, - ste->ste_col_offset, + ste->ste_loc.lineno, + ste->ste_loc.col_offset, prefix ); assert(msg != NULL); @@ -321,7 +327,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) if (flags & DEF_PARAM) printf(" DEF_PARAM"); if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL"); if (flags & USE) printf(" USE"); - if (flags & DEF_FREE) printf(" DEF_FREE"); if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS"); if (flags & DEF_IMPORT) printf(" DEF_IMPORT"); if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); @@ -386,17 +391,12 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; - int i; + Py_ssize_t i; PyThreadState *tstate; int starting_recursion_depth; @@ -417,12 +417,14 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ - if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { + + _Py_SourceLocation loc0 = {0, 0, 0, 0}; + if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, loc0)) { _PySymtable_Free(st); return NULL; } @@ -497,18 +499,29 @@ _PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItemWithError(st->st_blocks, k); - Py_DECREF(k); - - if (v) { - assert(PySTEntry_Check(v)); - } - else if (!PyErr_Occurred()) { + if (PyDict_GetItemRef(st->st_blocks, k, &v) == 0) { PyErr_SetString(PyExc_KeyError, "unknown symbol table entry"); } + Py_DECREF(k); - return (PySTEntryObject *)Py_XNewRef(v); + assert(v == NULL || PySTEntry_Check(v)); + return (PySTEntryObject *)v; +} + +int +_PySymtable_LookupOptional(struct symtable *st, void *key, + PySTEntryObject **out) +{ + PyObject *k = PyLong_FromVoidPtr(key); + if (k == NULL) { + *out = NULL; + return -1; + } + int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out); + Py_DECREF(k); + assert(*out == NULL || PySTEntry_Check(*out)); + return result; } long @@ -532,9 +545,10 @@ int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock - || ste->ste_type == TypeVarBoundBlock + || ste->ste_type == AnnotationBlock + || ste->ste_type == TypeVariableBlock || ste->ste_type == TypeAliasBlock - || ste->ste_type == TypeParamBlock; + || ste->ste_type == TypeParametersBlock; } static int @@ -608,16 +622,17 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) global: set of all symbol names explicitly declared as global */ -#define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyLong_FromLong(I); \ - if (!o) \ - return 0; \ - if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ +#define SET_SCOPE(DICT, NAME, I) \ + do { \ + PyObject *o = PyLong_FromLong(I); \ + if (!o) \ + return 0; \ + if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ + Py_DECREF(o); \ + return 0; \ + } \ Py_DECREF(o); \ - return 0; \ - } \ - Py_DECREF(o); \ -} + } while(0) /* Decide on scope of name, given flags. @@ -767,6 +782,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, { PyObject *k, *v; Py_ssize_t pos = 0; + int remove_dunder_class = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter long comp_flags = PyLong_AS_LONG(v); @@ -785,6 +802,16 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } + // __class__ is never allowed to be free through a class scope (see + // drop_class_free) + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + scope = GLOBAL_IMPLICIT; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); @@ -812,6 +839,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } } + comp->ste_free = PySet_Size(comp_free) > 0; + if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { + return 0; + } return 1; } @@ -981,6 +1012,12 @@ update_symbols(PyObject *symbols, PyObject *scopes, } Py_DECREF(name); } + + /* Check if loop ended because of exception in PyIter_Next */ + if (PyErr_Occurred()) { + goto error; + } + Py_DECREF(itr); Py_DECREF(v_free); return 1; @@ -1138,10 +1175,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions + // we inline all non-generator-expression comprehensions, + // except those in annotation scopes that are nested in classes int inline_comp = entry->ste_comprehension && - !entry->ste_generator; + !entry->ste_generator && + !ste->ste_can_see_class_scope; if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) @@ -1299,20 +1338,12 @@ symtable_exit_block(struct symtable *st) } static int -symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset, - int end_lineno, int end_col_offset) +symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { - PySTEntryObject *prev = NULL, *ste; - - ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset); - if (ste == NULL) - return 0; if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); return 0; } - prev = st->st_cur; + PySTEntryObject *prev = st->st_cur; /* bpo-37757: For now, disallow *all* assignment expressions in the * outermost iterator expression of a comprehension, even those inside * a nested comprehension or a lambda expression. @@ -1320,18 +1351,22 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, if (prev) { ste->ste_comp_iter_expr = prev->ste_comp_iter_expr; } + /* No need to inherit ste_mangled_names in classes, where all names + * are mangled. */ + if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) { + ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); + } /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); st->st_cur = ste; - /* Annotation blocks shouldn't have any affect on the symbol table since in - * the compilation stage, they will all be transformed to strings. They are - * only created if future 'annotations' feature is activated. */ - if (block == AnnotationBlock) { + /* If "from __future__ import annotations" is active, + * annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. */ + if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) { return 1; } - if (block == ModuleBlock) + if (ste->ste_type == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { @@ -1342,26 +1377,43 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 1; } +static int +symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, + void *ast, _Py_SourceLocation loc) +{ + PySTEntryObject *ste = ste_new(st, name, block, ast, loc); + if (ste == NULL) + return 0; + int result = symtable_enter_existing_block(st, ste); + Py_DECREF(ste); + return result; +} + static long -symtable_lookup(struct symtable *st, PyObject *name) +symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { - PyObject *mangled = _Py_Mangle(st->st_private, name); + PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name); if (!mangled) return 0; - long ret = _PyST_GetSymbol(st->st_cur, mangled); + long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); return ret; } +static long +symtable_lookup(struct symtable *st, PyObject *name) +{ + return symtable_lookup_entry(st, st->st_cur, name); +} + static int symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste, - int lineno, int col_offset, int end_lineno, int end_col_offset) + _Py_SourceLocation loc) { PyObject *o; PyObject *dict; long val; - PyObject *mangled = _Py_Mangle(st->st_private, name); - + PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; @@ -1371,16 +1423,12 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } if ((flag & DEF_TYPE_PARAM) && (val & DEF_TYPE_PARAM)) { PyErr_Format(PyExc_SyntaxError, DUPLICATE_TYPE_PARAM, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } val |= flag; @@ -1400,9 +1448,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } val |= DEF_COMP_ITER; @@ -1447,28 +1493,28 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s } static int -symtable_add_def(struct symtable *st, PyObject *name, int flag, - int lineno, int col_offset, int end_lineno, int end_col_offset) +symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc) { - return symtable_add_def_helper(st, name, flag, st->st_cur, - lineno, col_offset, end_lineno, end_col_offset); + if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) { + if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) { + return 0; + } + } + return symtable_add_def_helper(st, name, flag, st->st_cur, loc); } static int symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, - enum _stmt_kind kind, - int lineno, int col_offset, - int end_lineno, int end_col_offset) + enum _stmt_kind kind, _Py_SourceLocation loc) { _Py_block_ty current_type = st->st_cur->ste_type; - if(!symtable_enter_block(st, name, TypeParamBlock, ast, lineno, - col_offset, end_lineno, end_col_offset)) { + if(!symtable_enter_block(st, name, TypeParametersBlock, ast, loc)) { return 0; } if (current_type == ClassBlock) { st->st_cur->ste_can_see_class_scope = 1; - if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, loc)) { return 0; } } @@ -1476,37 +1522,30 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, _Py_DECLARE_STR(type_params, ".type_params"); // It gets "set" when we create the type params tuple and // "used" when we build up the bases. - if (!symtable_add_def(st, &_Py_STR(type_params), DEF_LOCAL, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(type_params), DEF_LOCAL, loc)) { return 0; } - if (!symtable_add_def(st, &_Py_STR(type_params), USE, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(type_params), USE, loc)) { return 0; } - st->st_private = name; // This is used for setting the generic base _Py_DECLARE_STR(generic_base, ".generic_base"); - if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL, loc)) { return 0; } - if (!symtable_add_def(st, &_Py_STR(generic_base), USE, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(generic_base), USE, loc)) { return 0; } } if (has_defaults) { _Py_DECLARE_STR(defaults, ".defaults"); - if (!symtable_add_def(st, &_Py_STR(defaults), DEF_PARAM, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(defaults), DEF_PARAM, loc)) { return 0; } } if (has_kwdefaults) { _Py_DECLARE_STR(kwdefaults, ".kwdefaults"); - if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM, loc)) { return 0; } } @@ -1528,43 +1567,48 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, return --(ST)->recursion_depth,(X) #define VISIT(ST, TYPE, V) \ - if (!symtable_visit_ ## TYPE((ST), (V))) \ - VISIT_QUIT((ST), 0); - -#define VISIT_SEQ(ST, TYPE, SEQ) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \ - int i = 0; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!elt) continue; /* can be NULL */ \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} + do { \ + if (!symtable_visit_ ## TYPE((ST), (V))) { \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ(ST, TYPE, SEQ) \ + do { \ + Py_ssize_t i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) \ + do { \ + Py_ssize_t i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) \ + do { \ + int i = 0; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!elt) continue; /* can be NULL */ \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) static int -symtable_record_directive(struct symtable *st, identifier name, int lineno, - int col_offset, int end_lineno, int end_col_offset) +symtable_record_directive(struct symtable *st, identifier name, _Py_SourceLocation loc) { PyObject *data, *mangled; int res; @@ -1573,10 +1617,11 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, if (!st->st_cur->ste_directives) return 0; } - mangled = _Py_Mangle(st->st_private, name); + mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; - data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset); + data = Py_BuildValue("(Niiii)", mangled, loc.lineno, loc.col_offset, + loc.end_lineno, loc.end_col_offset); if (!data) return 0; res = PyList_Append(st->st_cur->ste_directives, data); @@ -1596,6 +1641,35 @@ has_kwonlydefaults(asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) return 0; } +static int +check_import_from(struct symtable *st, stmt_ty s) +{ + assert(s->kind == ImportFrom_kind); + _Py_SourceLocation fut = st->st_future->ff_location; + if (s->v.ImportFrom.module && s->v.ImportFrom.level == 0 && + _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__") && + ((s->lineno > fut.lineno) || + ((s->lineno == fut.end_lineno) && (s->col_offset > fut.end_col_offset)))) + { + PyErr_SetString(PyExc_SyntaxError, + "from __future__ imports must occur " + "at the beginning of the file"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + return 1; +} + +static void +maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) +{ + if ((st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) && + (st->st_cur->ste_type == ModuleBlock)) + { + st->st_cur->ste_coroutine = 1; + } +} + static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { @@ -1605,7 +1679,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } switch (s->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.FunctionDef.args->defaults) @@ -1627,13 +1701,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) + s->v.FunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, - LOCATION(s))) + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) @@ -1643,12 +1726,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); + tmp = st->st_private; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, (void *)s->v.ClassDef.type_params, @@ -1656,15 +1741,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } + st->st_private = s->v.ClassDef.name; + st->st_cur->ste_mangled_names = PySet_New(NULL); + if (!st->st_cur->ste_mangled_names) { + VISIT_QUIT(st, 0); + } VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, - (void *)s, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + (void *)s, LOCATION(s))) { VISIT_QUIT(st, 0); - tmp = st->st_private; + } st->st_private = s->v.ClassDef.name; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_add_def(st, &_Py_ID(__type_params__), @@ -1678,13 +1767,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } } VISIT_SEQ(st, stmt, s->v.ClassDef.body); - st->st_private = tmp; if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } + st->st_private = tmp; break; } case TypeAlias_kind: { @@ -1704,8 +1793,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, type_param, s->v.TypeAlias.type_params); } if (!symtable_enter_block(st, name, TypeAliasBlock, - (void *)s, LOCATION(s))) + (void *)s, LOCATION(s))) { VISIT_QUIT(st, 0); + } st->st_cur->ste_can_see_class_scope = is_in_class; if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(s->v.TypeAlias.value))) { VISIT_QUIT(st, 0); @@ -1733,6 +1823,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assign.value); break; case AnnAssign_kind: + st->st_cur->ste_annotations_used = 1; if (s->v.AnnAssign.target->kind == Name_kind) { expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); @@ -1745,11 +1836,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) PyErr_Format(PyExc_SyntaxError, cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT, e_name->v.Name.id); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); VISIT_QUIT(st, 0); } if (s->v.AnnAssign.simple && @@ -1767,7 +1854,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) else { VISIT(st, expr, s->v.AnnAssign.target); } - if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation, + (void *)((uintptr_t)st->st_cur->ste_id + 1))) { VISIT_QUIT(st, 0); } @@ -1813,14 +1901,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Try_kind: VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case TryStar_kind: VISIT_SEQ(st, stmt, s->v.TryStar.body); - VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); + VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); break; case Assert_kind: @@ -1833,9 +1921,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case ImportFrom_kind: VISIT_SEQ(st, alias, s->v.ImportFrom.names); + if (!check_import_from(st, s)) { + VISIT_QUIT(st, 0); + } break; case Global_kind: { - int i; + Py_ssize_t i; asdl_identifier_seq *seq = s->v.Global.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); @@ -1855,23 +1946,20 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } PyErr_Format(PyExc_SyntaxError, msg, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); VISIT_QUIT(st, 0); } - if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s))) + if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s))) { VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + } + if (!symtable_record_directive(st, name, LOCATION(s))) { VISIT_QUIT(st, 0); + } } break; } case Nonlocal_kind: { - int i; + Py_ssize_t i; asdl_identifier_seq *seq = s->v.Nonlocal.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); @@ -1890,18 +1978,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) msg = NONLOCAL_AFTER_ASSIGN; } PyErr_Format(PyExc_SyntaxError, msg, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); VISIT_QUIT(st, 0); } if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s))) VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + if (!symtable_record_directive(st, name, LOCATION(s))) { VISIT_QUIT(st, 0); + } } break; } @@ -1917,7 +2001,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: + case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.AsyncFunctionDef.args->defaults) @@ -1940,14 +2024,23 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) + s->v.AsyncFunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, - s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); @@ -1958,11 +2051,20 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case AsyncWith_kind: + maybe_set_ste_coroutine_for_module(st, s); + if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTISDE_ASYNC_FUNC, LOCATION(s))) { + VISIT_QUIT(st, 0); + } VISIT_SEQ(st, withitem, s->v.AsyncWith.items); VISIT_SEQ(st, stmt, s->v.AsyncWith.body); break; case AsyncFor_kind: + maybe_set_ste_coroutine_for_module(st, s); + if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTISDE_ASYNC_FUNC, LOCATION(s))) { + VISIT_QUIT(st, 0); + } VISIT(st, expr, s->v.AsyncFor.target); VISIT(st, expr, s->v.AsyncFor.iter); VISIT_SEQ(st, stmt, s->v.AsyncFor.body); @@ -1993,15 +2095,11 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) * binding conflict with iteration variables, otherwise skip it */ if (ste->ste_comprehension) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); VISIT_QUIT(st, 0); } continue; @@ -2009,54 +2107,54 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) VISIT_QUIT(st, 0); } else { - if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e))) + if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e))) { VISIT_QUIT(st, 0); + } } - if (!symtable_record_directive(st, target_name, LOCATION(e))) + if (!symtable_record_directive(st, target_name, LOCATION(e))) { VISIT_QUIT(st, 0); + } return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e)); } /* If we find a ModuleBlock entry, add as GLOBAL */ if (ste->ste_type == ModuleBlock) { - if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) + if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) { VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, target_name, LOCATION(e))) + } + if (!symtable_record_directive(st, target_name, LOCATION(e))) { VISIT_QUIT(st, 0); + } return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e)); } /* Disallow usage in ClassBlock and type scopes */ if (ste->ste_type == ClassBlock || - ste->ste_type == TypeParamBlock || + ste->ste_type == TypeParametersBlock || ste->ste_type == TypeAliasBlock || - ste->ste_type == TypeVarBoundBlock) { + ste->ste_type == TypeVariableBlock) { switch (ste->ste_type) { case ClassBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS); break; - case TypeParamBlock: + case TypeParametersBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEPARAM); break; case TypeAliasBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEALIAS); break; - case TypeVarBoundBlock: + case TypeVariableBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEVAR_BOUND); break; default: Py_UNREACHABLE(); } - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); VISIT_QUIT(st, 0); } } @@ -2074,11 +2172,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) if (st->st_cur->ste_comp_iter_expr > 0) { /* Assignment isn't allowed in a comprehension iterable expression */ PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); return 0; } if (st->st_cur->ste_comprehension) { @@ -2118,26 +2212,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.UnaryOp.operand); break; case Lambda_kind: { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use lambda in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); if (e->v.Lambda.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); if (!symtable_enter_block(st, &_Py_ID(lambda), - FunctionBlock, (void *)e, - e->lineno, e->col_offset, - e->end_lineno, e->end_col_offset)) + FunctionBlock, (void *)e, LOCATION(e))) { VISIT_QUIT(st, 0); + } VISIT(st, arguments, e->v.Lambda.args); VISIT(st, expr, e->v.Lambda.body); if (!symtable_exit_block(st)) @@ -2233,11 +2315,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Slice_kind: if (e->v.Slice.lower) - VISIT(st, expr, e->v.Slice.lower) + VISIT(st, expr, e->v.Slice.lower); if (e->v.Slice.upper) - VISIT(st, expr, e->v.Slice.upper) + VISIT(st, expr, e->v.Slice.upper); if (e->v.Slice.step) - VISIT(st, expr, e->v.Slice.step) + VISIT(st, expr, e->v.Slice.step); break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, @@ -2262,6 +2344,33 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 1); } +static int +symtable_visit_type_param_bound_or_default( + struct symtable *st, expr_ty e, identifier name, + void *key, const char *ste_scope_info) +{ + if (e) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + if (!symtable_enter_block(st, name, TypeVariableBlock, key, LOCATION(e))) { + return 0; + } + + st->st_cur->ste_can_see_class_scope = is_in_class; + if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) { + VISIT_QUIT(st, 0); + } + + assert(ste_scope_info != NULL); + st->st_cur->ste_scope_info = ste_scope_info; + VISIT(st, expr, e); + + if (!symtable_exit_block(st)) { + return 0; + } + } + return 1; +} + static int symtable_visit_type_param(struct symtable *st, type_param_ty tp) { @@ -2274,28 +2383,48 @@ symtable_visit_type_param(struct symtable *st, type_param_ty tp) case TypeVar_kind: if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) VISIT_QUIT(st, 0); - if (tp->v.TypeVar.bound) { - int is_in_class = st->st_cur->ste_can_see_class_scope; - if (!symtable_enter_block(st, tp->v.TypeVar.name, - TypeVarBoundBlock, (void *)tp, - LOCATION(tp))) - VISIT_QUIT(st, 0); - st->st_cur->ste_can_see_class_scope = is_in_class; - if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(tp->v.TypeVar.bound))) { - VISIT_QUIT(st, 0); - } - VISIT(st, expr, tp->v.TypeVar.bound); - if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + + const char *ste_scope_info = NULL; + const expr_ty bound = tp->v.TypeVar.bound; + if (bound != NULL) { + ste_scope_info = bound->kind == Tuple_kind ? "a TypeVar constraint" : "a TypeVar bound"; + } + + // We must use a different key for the bound and default. The obvious choice would be to + // use the .bound and .default_value pointers, but that fails when the expression immediately + // inside the bound or default is a comprehension: we would reuse the same key for + // the comprehension scope. Therefore, use the address + 1 as the second key. + // The only requirement for the key is that it is unique and it matches the logic in + // compile.c where the scope is retrieved. + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name, + (void *)tp, ste_scope_info)) { + VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name, + (void *)((uintptr_t)tp + 1), "a TypeVar default")) { + VISIT_QUIT(st, 0); } break; case TypeVarTuple_kind: - if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { + VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name, + (void *)tp, "a TypeVarTuple default")) { VISIT_QUIT(st, 0); + } break; case ParamSpec_kind: - if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { + VISIT_QUIT(st, 0); + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name, + (void *)tp, "a ParamSpec default")) { VISIT_QUIT(st, 0); + } break; } VISIT_QUIT(st, 1); @@ -2357,7 +2486,7 @@ symtable_implicit_arg(struct symtable *st, int pos) PyObject *id = PyUnicode_FromFormat(".%d", pos); if (id == NULL) return 0; - if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) { + if (!symtable_add_def(st, id, DEF_PARAM, st->st_cur->ste_loc)) { Py_DECREF(id); return 0; } @@ -2368,7 +2497,7 @@ symtable_implicit_arg(struct symtable *st, int pos) static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args) { - int i; + Py_ssize_t i; if (!args) return -1; @@ -2383,18 +2512,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotation(struct symtable *st, expr_ty annotation) +symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)annotation, annotation->lineno, - annotation->col_offset, annotation->end_lineno, - annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + struct _symtable_entry *parent_ste = st->st_cur; + if (parent_ste->ste_annotation_block == NULL) { + _Py_block_ty current_type = parent_ste->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + key, LOCATION(annotation))) { + VISIT_QUIT(st, 0); + } + parent_ste->ste_annotation_block = + (struct _symtable_entry *)Py_NewRef(st->st_cur); + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (current_type == ClassBlock && !future_annotations) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { + return 0; + } + } + + _Py_DECLARE_STR(format, ".format"); + // The generated __annotate__ function takes a single parameter with the + // internal name ".format". + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, + LOCATION(annotation))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, + LOCATION(annotation))) { + return 0; + } + } + else { + if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { + VISIT_QUIT(st, 0); + } } VISIT(st, expr, annotation); - if (future_annotations && !symtable_exit_block(st)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2403,44 +2558,65 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation) static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { - int i; + Py_ssize_t i; if (!args) return -1; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) + if (arg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, arg->annotation); + } } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, + struct _symtable_entry *function_ste) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)o, o->lineno, o->col_offset, o->end_lineno, - o->end_col_offset)) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + _Py_block_ty current_type = st->st_cur->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + (void *)a, LOCATION(o))) { VISIT_QUIT(st, 0); } + if (is_in_class || current_type == ClassBlock) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) { + return 0; + } + } + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) { + return 0; + } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) return 0; - if (a->vararg && a->vararg->annotation) + if (a->vararg && a->vararg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) + } + if (a->kwarg && a->kwarg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->kwarg->annotation); + } if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (returns) { + st->st_cur->ste_annotations_used = 1; + VISIT(st, expr, returns); } - if (returns && !symtable_visit_annotation(st, returns)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2531,14 +2707,8 @@ symtable_visit_alias(struct symtable *st, alias_ty a) } else { if (st->st_cur->ste_type != ModuleBlock) { - int lineno = a->lineno; - int col_offset = a->col_offset; - int end_lineno = a->end_lineno; - int end_col_offset = a->end_col_offset; PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(a)); Py_DECREF(store_name); return 0; } @@ -2578,18 +2748,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty value) { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use comprehension in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } - int is_generator = (e->kind == GeneratorExp_kind); comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); @@ -2599,9 +2757,7 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_comp_iter_expr--; /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, - e->lineno, e->col_offset, - e->end_lineno, e->end_col_offset)) { + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, LOCATION(e))) { return 0; } switch(e->kind) { @@ -2684,23 +2840,28 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { - enum _block_type type = st->st_cur->ste_type; + _Py_block_ty type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); - else if (type == TypeVarBoundBlock) - PyErr_Format(PyExc_SyntaxError, TYPEVAR_BOUND_NOT_ALLOWED, name); - else if (type == TypeAliasBlock) - PyErr_Format(PyExc_SyntaxError, TYPEALIAS_NOT_ALLOWED, name); - else if (type == TypeParamBlock) - PyErr_Format(PyExc_SyntaxError, TYPEPARAM_NOT_ALLOWED, name); + else if (type == TypeVariableBlock) { + const char *info = st->st_cur->ste_scope_info; + assert(info != NULL); // e.g., info == "a ParamSpec default" + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE, name, info); + } + else if (type == TypeAliasBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_ALIAS, name); + } + else if (type == TypeParametersBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS, name); + } else return 1; - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); return 0; } @@ -2712,12 +2873,20 @@ symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) { (type == SetComprehension) ? "'yield' inside set comprehension" : (type == DictComprehension) ? "'yield' inside dict comprehension" : "'yield' inside generator expression"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, e->col_offset + 1, - e->end_lineno, e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); VISIT_QUIT(st, 0); } +static int +symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc) { + if (!st->st_cur->ste_coroutine) { + PyErr_SetString(PyExc_SyntaxError, msg); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + return 1; +} + struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags) @@ -2735,7 +2904,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { _PyArena_Free(arena); return NULL; @@ -2746,6 +2915,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, return st; } +PyObject * +_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name) +{ + /* Special case for type parameter blocks around generic classes: + * we want to mangle type parameter names (so a type param with a private + * name can be used inside the class body), but we don't want to mangle + * any other names that appear within the type parameter scope. + */ + if (ste->ste_mangled_names != NULL) { + int result = PySet_Contains(ste->ste_mangled_names, name); + if (result < 0) { + return NULL; + } + if (result == 0) { + return Py_NewRef(name); + } + } + return _Py_Mangle(privateobj, name); +} + PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 3debe7f7c139c6..1fff7e41767398 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -35,7 +35,6 @@ Data members: #include "pycore_sysmodule.h" // export _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() -#include "frameobject.h" // PyFrame_FastToLocalsWithError() #include "pydtrace.h" // PyDTrace_AUDIT() #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names @@ -110,6 +109,9 @@ PySys_GetObject(const char *name) PyObject *value = _PySys_GetObject(tstate->interp, name); /* XXX Suppress a new exception if it was raised and restore * the old one. */ + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable("Exception ignored in PySys_GetObject()"); + } _PyErr_SetRaisedException(tstate, exc); return value; } @@ -122,11 +124,9 @@ sys_set_object(PyInterpreterState *interp, PyObject *key, PyObject *v) } PyObject *sd = interp->sysdict; if (v == NULL) { - v = _PyDict_Pop(sd, key, Py_None); - if (v == NULL) { + if (PyDict_Pop(sd, key, NULL) < 0) { return -1; } - Py_DECREF(v); return 0; } else { @@ -450,15 +450,9 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) e->hookCFunction = (Py_AuditHookFunction)hook; e->userData = userData; - if (runtime->audit_hooks.mutex == NULL) { - /* The runtime must not be initialized yet. */ - add_audit_hook_entry_unlocked(runtime, e); - } - else { - PyThread_acquire_lock(runtime->audit_hooks.mutex, WAIT_LOCK); - add_audit_hook_entry_unlocked(runtime, e); - PyThread_release_lock(runtime->audit_hooks.mutex); - } + PyMutex_Lock(&runtime->audit_hooks.mutex); + add_audit_hook_entry_unlocked(runtime, e); + PyMutex_Unlock(&runtime->audit_hooks.mutex); return 0; } @@ -505,7 +499,8 @@ sys_addaudithook_impl(PyObject *module, PyObject *hook) } PyDoc_STRVAR(audit_doc, -"audit(event, *args)\n\ +"audit($module, event, /, *args)\n\ +--\n\ \n\ Passes the event to any audit hooks that are attached."); @@ -649,7 +644,8 @@ sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb } PyDoc_STRVAR(breakpointhook_doc, -"breakpointhook(*args, **kws)\n" +"breakpointhook($module, /, *args, **kwargs)\n" +"--\n" "\n" "This hook function is called by built-in breakpoint().\n" ); @@ -751,7 +747,7 @@ sys_displayhook(PyObject *module, PyObject *o) if (o == Py_None) { Py_RETURN_NONE; } - if (PyObject_SetAttr(builtins, &_Py_ID(_), Py_None) != 0) + if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), Py_None) != 0) return NULL; outf = _PySys_GetAttr(tstate, &_Py_ID(stdout)); if (outf == NULL || outf == Py_None) { @@ -773,10 +769,9 @@ sys_displayhook(PyObject *module, PyObject *o) return NULL; } } - _Py_DECLARE_STR(newline, "\n"); - if (PyFile_WriteObject(&_Py_STR(newline), outf, Py_PRINT_RAW) != 0) + if (PyFile_WriteObject(_Py_LATIN1_CHR('\n'), outf, Py_PRINT_RAW) != 0) return NULL; - if (PyObject_SetAttr(builtins, &_Py_ID(_), o) != 0) + if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), o) != 0) return NULL; Py_RETURN_NONE; } @@ -934,7 +929,7 @@ sys_getfilesystemencoding_impl(PyObject *module) if (u == NULL) { return NULL; } - _PyUnicode_InternInPlace(interp, &u); + _PyUnicode_InternImmortal(interp, &u); return u; } @@ -954,7 +949,7 @@ sys_getfilesystemencodeerrors_impl(PyObject *module) if (u == NULL) { return NULL; } - _PyUnicode_InternInPlace(interp, &u); + _PyUnicode_InternImmortal(interp, &u); return u; } @@ -976,8 +971,9 @@ sys_intern_impl(PyObject *module, PyObject *s) /*[clinic end generated code: output=be680c24f5c9e5d6 input=849483c006924e2f]*/ { if (PyUnicode_CheckExact(s)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_INCREF(s); - PyUnicode_InternInPlace(&s); + _PyUnicode_InternMortal(interp, &s); return s; } else { @@ -988,6 +984,23 @@ sys_intern_impl(PyObject *module, PyObject *s) } +/*[clinic input] +sys._is_interned -> bool + + string: unicode + / + +Return True if the given string is "interned". +[clinic start generated code]*/ + +static int +sys__is_interned_impl(PyObject *module, PyObject *string) +/*[clinic end generated code: output=c3678267b4e9d7ed input=039843e17883b606]*/ +{ + return PyUnicode_CHECK_INTERNED(string); +} + + /* * Cached interned string objects used for calling the profile and * trace functions. @@ -1008,13 +1021,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - /* Discard any previous modifications the frame's fast locals */ - if (frame->f_fast_as_locals) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - } - /* call the Python-level function */ if (arg == NULL) { arg = Py_None; @@ -1022,7 +1028,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, PyObject *args[3] = {(PyObject *)frame, whatstrings[what], arg}; PyObject *result = _PyObject_VectorcallTstate(tstate, callback, args, 3, NULL); - PyFrame_LocalsToFast(frame, 1); return result; } @@ -1073,34 +1078,40 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, return 0; } +/*[clinic input] +sys.settrace + + function: object + / + +Set the global debug tracing function. + +It will be called on each function call. See the debugger chapter +in the library manual. +[clinic start generated code]*/ + static PyObject * -sys_settrace(PyObject *self, PyObject *args) +sys_settrace(PyObject *module, PyObject *function) +/*[clinic end generated code: output=999d12e9d6ec4678 input=8107feb01c5f1c4e]*/ { PyThreadState *tstate = _PyThreadState_GET(); - if (args == Py_None) { + if (function == Py_None) { if (_PyEval_SetTrace(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetTrace(tstate, trace_trampoline, args) < 0) { + if (_PyEval_SetTrace(tstate, trace_trampoline, function) < 0) { return NULL; } } Py_RETURN_NONE; } -PyDoc_STRVAR(settrace_doc, -"settrace(function)\n\ -\n\ -Set the global debug tracing function. It will be called on each\n\ -function call. See the debugger chapter in the library manual." -); - /*[clinic input] sys._settraceallthreads - arg: object + function as arg: object / Set the global debug tracing function in all running threads belonging to the current interpreter. @@ -1111,7 +1122,7 @@ in the library manual. static PyObject * sys__settraceallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=161cca30207bf3ca input=5906aa1485a50289]*/ +/*[clinic end generated code: output=161cca30207bf3ca input=d4bde1f810d73675]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1147,45 +1158,51 @@ sys_gettrace_impl(PyObject *module) return Py_NewRef(temp); } +/*[clinic input] +sys.setprofile + + function: object + / + +Set the profiling function. + +It will be called on each function call and return. See the profiler +chapter in the library manual. +[clinic start generated code]*/ + static PyObject * -sys_setprofile(PyObject *self, PyObject *args) +sys_setprofile(PyObject *module, PyObject *function) +/*[clinic end generated code: output=1c3503105939db9c input=055d0d7961413a62]*/ { PyThreadState *tstate = _PyThreadState_GET(); - if (args == Py_None) { + if (function == Py_None) { if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetProfile(tstate, profile_trampoline, args) < 0) { + if (_PyEval_SetProfile(tstate, profile_trampoline, function) < 0) { return NULL; } } Py_RETURN_NONE; } -PyDoc_STRVAR(setprofile_doc, -"setprofile(function)\n\ -\n\ -Set the profiling function. It will be called on each function call\n\ -and return. See the profiler chapter in the library manual." -); - /*[clinic input] sys._setprofileallthreads - arg: object + function as arg: object / Set the profiling function in all running threads belonging to the current interpreter. -It will be called on each function call and return. See the profiler chapter -in the library manual. +It will be called on each function call and return. See the profiler +chapter in the library manual. [clinic start generated code]*/ static PyObject * sys__setprofileallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=2d61319e27b309fe input=d1a356d3f4f9060a]*/ +/*[clinic end generated code: output=2d61319e27b309fe input=a10589439ba20cee]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1381,12 +1398,6 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(finalizer)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { - return NULL; - } - } - else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { - return NULL; } if (firstiter && firstiter != Py_None) { @@ -1396,19 +1407,37 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(firstiter)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + } + + PyObject *cur_finalizer = _PyEval_GetAsyncGenFinalizer(); + + if (finalizer && finalizer != Py_None) { + if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { return NULL; } } - else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { return NULL; } + if (firstiter && firstiter != Py_None) { + if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + goto error; + } + } + else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + goto error; + } + Py_RETURN_NONE; + +error: + _PyEval_SetAsyncGenFinalizer(cur_finalizer); + return NULL; } PyDoc_STRVAR(set_asyncgen_hooks_doc, -"set_asyncgen_hooks(* [, firstiter] [, finalizer])\n\ +"set_asyncgen_hooks([firstiter] [, finalizer])\n\ \n\ Set a finalizer for async generators objects." ); @@ -1486,31 +1515,33 @@ get_hash_info(PyThreadState *tstate) int field = 0; PyHash_FuncDef *hashfunc; hash_info = PyStructSequence_New(&Hash_InfoType); - if (hash_info == NULL) - return NULL; - hashfunc = PyHash_GetFuncDef(); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(8*sizeof(Py_hash_t))); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromSsize_t(_PyHASH_MODULUS)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(_PyHASH_INF)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(0)); // This is no longer used - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(_PyHASH_IMAG)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyUnicode_FromString(hashfunc->name)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(hashfunc->hash_bits)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(hashfunc->seed_bits)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(Py_HASH_CUTOFF)); - if (_PyErr_Occurred(tstate)) { - Py_CLEAR(hash_info); + if (hash_info == NULL) { return NULL; } + hashfunc = PyHash_GetFuncDef(); + +#define SET_HASH_INFO_ITEM(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_CLEAR(hash_info); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(hash_info, field++, item); \ + } while(0) + + SET_HASH_INFO_ITEM(PyLong_FromLong(8 * sizeof(Py_hash_t))); + SET_HASH_INFO_ITEM(PyLong_FromSsize_t(_PyHASH_MODULUS)); + SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_INF)); + SET_HASH_INFO_ITEM(PyLong_FromLong(0)); // This is no longer used + SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_IMAG)); + SET_HASH_INFO_ITEM(PyUnicode_FromString(hashfunc->name)); + SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->hash_bits)); + SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->seed_bits)); + SET_HASH_INFO_ITEM(PyLong_FromLong(Py_HASH_CUTOFF)); + +#undef SET_HASH_INFO_ITEM + return hash_info; } /*[clinic input] @@ -1629,12 +1660,13 @@ sys_getwindowsversion_impl(PyObject *module) int pos = 0; OSVERSIONINFOEXW ver; - version = PyObject_GetAttrString(module, "_cached_windows_version"); + if (PyObject_GetOptionalAttrString(module, "_cached_windows_version", &version) < 0) { + return NULL; + }; if (version && PyObject_TypeCheck(version, &WindowsVersionType)) { return version; } Py_XDECREF(version); - PyErr_Clear(); ver.dwOSVersionInfoSize = sizeof(ver); if (!GetVersionExW((OSVERSIONINFOW*) &ver)) @@ -1644,15 +1676,24 @@ sys_getwindowsversion_impl(PyObject *module) if (version == NULL) return NULL; - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMajorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMinorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwBuildNumber)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwPlatformId)); - PyStructSequence_SET_ITEM(version, pos++, PyUnicode_FromWideChar(ver.szCSDVersion, -1)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMajor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMinor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wSuiteMask)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wProductType)); +#define SET_VERSION_INFO(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SET_ITEM(version, pos++, item); \ + } while(0) + + SET_VERSION_INFO(PyLong_FromLong(ver.dwMajorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwMinorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwBuildNumber)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwPlatformId)); + SET_VERSION_INFO(PyUnicode_FromWideChar(ver.szCSDVersion, -1)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMajor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMinor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wSuiteMask)); + SET_VERSION_INFO(PyLong_FromLong(ver.wProductType)); // GetVersion will lie if we are running in a compatibility mode. // We need to read the version info from a system file resource @@ -1660,6 +1701,10 @@ sys_getwindowsversion_impl(PyObject *module) // just return whatever GetVersion said. PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { + if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { + return NULL; + } + PyErr_Clear(); realVersion = Py_BuildValue("(kkk)", ver.dwMajorVersion, @@ -1668,21 +1713,19 @@ sys_getwindowsversion_impl(PyObject *module) ); } - if (realVersion) { - PyStructSequence_SET_ITEM(version, pos++, realVersion); - } + SET_VERSION_INFO(realVersion); - if (PyErr_Occurred()) { - Py_DECREF(version); - return NULL; - } +#undef SET_VERSION_INFO if (PyObject_SetAttrString(module, "_cached_windows_version", version) < 0) { - Py_DECREF(version); - return NULL; + goto error; } return version; + +error: + Py_DECREF(version); + return NULL; } #pragma warning(pop) @@ -1703,6 +1746,13 @@ static PyObject * sys__enablelegacywindowsfsencoding_impl(PyObject *module) /*[clinic end generated code: output=f5c3855b45e24fe9 input=2bfa931a20704492]*/ { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys._enablelegacywindowsfsencoding() is deprecated and will be " + "removed in Python 3.16. Use PYTHONLEGACYWINDOWSFSENCODING " + "instead.", 1)) + { + return NULL; + } if (_PyUnicode_EnableLegacyWindowsFSEncoding() < 0) { return NULL; } @@ -1859,7 +1909,15 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; } - return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o)); + size_t presize = 0; + if (!Py_IS_TYPE(o, &PyType_Type) || + PyType_HasFeature((PyTypeObject *)o, Py_TPFLAGS_HEAPTYPE)) + { + /* Add the size of the pre-header if "o" is not a static type */ + presize = _PyType_PreHeaderSize(Py_TYPE(o)); + } + + return (size_t)size + presize; } static PyObject * @@ -1949,14 +2007,22 @@ sys_getallocatedblocks_impl(PyObject *module) /*[clinic input] sys.getunicodeinternedsize -> Py_ssize_t + * + _only_immortal: bool = False + Return the number of elements of the unicode interned dictionary [clinic start generated code]*/ static Py_ssize_t -sys_getunicodeinternedsize_impl(PyObject *module) -/*[clinic end generated code: output=ad0e4c9738ed4129 input=726298eaa063347a]*/ +sys_getunicodeinternedsize_impl(PyObject *module, int _only_immortal) +/*[clinic end generated code: output=29a6377a94a14f70 input=0330b3408dd5bcc6]*/ { - return _PyUnicode_InternedSize(); + if (_only_immortal) { + return _PyUnicode_InternedSize_Immortal(); + } + else { + return _PyUnicode_InternedSize(); + } } /*[clinic input] @@ -2100,6 +2166,24 @@ sys__clear_type_cache_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._clear_internal_caches + +Clear all internal performance-related caches. +[clinic start generated code]*/ + +static PyObject * +sys__clear_internal_caches_impl(PyObject *module) +/*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ +{ +#ifdef _Py_TIER2 + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_Executors_InvalidateAll(interp, 0); +#endif + PyType_ClearCache(); + Py_RETURN_NONE; +} + /* Note that, for now, we do not have a per-interpreter equivalent for sys.is_finalizing(). */ @@ -2217,6 +2301,16 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; + } + } + } } else { PyErr_Format(PyExc_ValueError, "invalid backend: %s", backend); @@ -2320,6 +2414,39 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._baserepl + +Private function for getting the base REPL +[clinic start generated code]*/ + +static PyObject * +sys__baserepl_impl(PyObject *module) +/*[clinic end generated code: output=f19a36375ebe0a45 input=ade0ebb9fab56f3c]*/ +{ + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyRun_AnyFileExFlags(stdin, "", 0, &cf); + Py_RETURN_NONE; +} + +/*[clinic input] +sys._is_gil_enabled -> bool + +Return True if the GIL is currently enabled and False otherwise. +[clinic start generated code]*/ + +static int +sys__is_gil_enabled_impl(PyObject *module) +/*[clinic end generated code: output=57732cf53f5b9120 input=7e9c47f15a00e809]*/ +{ +#ifdef Py_GIL_DISABLED + return _PyEval_IsGILEnabled(_PyThreadState_GET()); +#else + return 1; +#endif +} + + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2361,7 +2488,7 @@ PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( #ifndef MS_WINDOWS if (perf_map_state.perf_map == NULL) { int ret = PyUnstable_PerfMapState_Init(); - if(ret != 0){ + if (ret != 0){ return ret; } } @@ -2388,6 +2515,41 @@ PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void) { #endif } +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { +#ifndef MS_WINDOWS + if (perf_map_state.perf_map == NULL) { + int ret = PyUnstable_PerfMapState_Init(); + if (ret != 0) { + return ret; + } + } + FILE* from = fopen(parent_filename, "r"); + if (!from) { + return -1; + } + char buf[4096]; + PyThread_acquire_lock(perf_map_state.map_lock, 1); + int fflush_result = 0, result = 0; + while (1) { + size_t bytes_read = fread(buf, 1, sizeof(buf), from); + size_t bytes_written = fwrite(buf, 1, bytes_read, perf_map_state.perf_map); + fflush_result = fflush(perf_map_state.perf_map); + if (fflush_result != 0 || bytes_read == 0 || bytes_written < bytes_read) { + result = -1; + goto close_and_release; + } + if (bytes_read < sizeof(buf) && feof(from)) { + goto close_and_release; + } + } +close_and_release: + fclose(from); + PyThread_release_lock(perf_map_state.map_lock); + return result; +#endif + return 0; +} + static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ @@ -2395,6 +2557,7 @@ static PyMethodDef sys_methods[] = { {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, + SYS__CLEAR_INTERNAL_CACHES_METHODDEF SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS__CURRENT_EXCEPTIONS_METHODDEF @@ -2422,16 +2585,17 @@ static PyMethodDef sys_methods[] = { SYS_GETWINDOWSVERSION_METHODDEF SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF SYS_INTERN_METHODDEF + SYS__IS_INTERNED_METHODDEF SYS_IS_FINALIZING_METHODDEF SYS_MDEBUG_METHODDEF SYS_SETSWITCHINTERVAL_METHODDEF SYS_GETSWITCHINTERVAL_METHODDEF SYS_SETDLOPENFLAGS_METHODDEF - {"setprofile", sys_setprofile, METH_O, setprofile_doc}, + SYS_SETPROFILE_METHODDEF SYS__SETPROFILEALLTHREADS_METHODDEF SYS_GETPROFILE_METHODDEF SYS_SETRECURSIONLIMIT_METHODDEF - {"settrace", sys_settrace, METH_O, settrace_doc}, + SYS_SETTRACE_METHODDEF SYS__SETTRACEALLTHREADS_METHODDEF SYS_GETTRACE_METHODDEF SYS_CALL_TRACING_METHODDEF @@ -2448,6 +2612,7 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF SYS__STATS_OFF_METHODDEF @@ -2455,6 +2620,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__IS_GIL_ENABLED_METHODDEF {NULL, NULL} // sentinel }; @@ -2950,6 +3116,7 @@ static PyStructSequence_Field flags_fields[] = { {"warn_default_encoding", "-X warn_default_encoding"}, {"safe_path", "-P"}, {"int_max_str_digits", "-X int_max_str_digits"}, + {"gil", "-X gil"}, {0} }; @@ -2999,6 +3166,16 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) SetFlag(config->warn_default_encoding); SetFlagObj(PyBool_FromLong(config->safe_path)); SetFlag(config->int_max_str_digits); +#ifdef Py_GIL_DISABLED + if (config->enable_gil == _PyConfig_GIL_DEFAULT) { + SetFlagObj(Py_NewRef(Py_None)); + } + else { + SetFlag(config->enable_gil); + } +#else + SetFlagObj(PyLong_FromLong(1)); +#endif #undef SetFlagObj #undef SetFlag return 0; @@ -3615,6 +3792,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (sysmod == NULL) { return _PyStatus_ERR("failed to create a module object"); } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(sysmod, Py_MOD_GIL_NOT_USED); +#endif PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { @@ -3641,7 +3821,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return status; } - if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) { + if (_PyImport_FixupBuiltin(tstate, sysmod, "sys", modules) < 0) { goto error; } @@ -3749,8 +3929,7 @@ make_sys_argv(int argc, wchar_t * const * argv) return list; } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) { wchar_t* empty_argv[1] = {L""}; @@ -3794,8 +3973,7 @@ PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) } } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void PySys_SetArgv(int argc, wchar_t **argv) { _Py_COMP_DIAG_PUSH diff --git a/Python/thread.c b/Python/thread.c index bf207cecb90505..b31d1dc5e770ef 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -6,6 +6,7 @@ Stuff shared by all thread_*.h files is collected here. */ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_MakePendingCalls() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_pythread.h" // _POSIX_THREADS @@ -19,8 +20,8 @@ // Define PY_TIMEOUT_MAX constant. #ifdef _POSIX_THREADS - // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - // convert microseconds to nanoseconds. + // PyThread_acquire_lock_timed() uses (us * 1000) to convert microseconds + // to nanoseconds. # define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) #elif defined (NT_THREADS) // WaitForSingleObject() accepts timeout in milliseconds in the range @@ -92,6 +93,89 @@ PyThread_set_stacksize(size_t size) } +int +PyThread_ParseTimeoutArg(PyObject *arg, int blocking, PY_TIMEOUT_T *timeout_p) +{ + assert(_PyTime_FromSeconds(-1) == PyThread_UNSET_TIMEOUT); + if (arg == NULL || arg == Py_None) { + *timeout_p = blocking ? PyThread_UNSET_TIMEOUT : 0; + return 0; + } + if (!blocking) { + PyErr_SetString(PyExc_ValueError, + "can't specify a timeout for a non-blocking call"); + return -1; + } + + PyTime_t timeout; + if (_PyTime_FromSecondsObject(&timeout, arg, _PyTime_ROUND_TIMEOUT) < 0) { + return -1; + } + if (timeout < 0) { + PyErr_SetString(PyExc_ValueError, + "timeout value must be a non-negative number"); + return -1; + } + + if (_PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_TIMEOUT) > PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "timeout value is too large"); + return -1; + } + *timeout_p = timeout; + return 0; +} + +PyLockStatus +PyThread_acquire_lock_timed_with_retries(PyThread_type_lock lock, + PY_TIMEOUT_T timeout) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyTime_t endtime = 0; + if (timeout > 0) { + endtime = _PyDeadline_Init(timeout); + } + + PyLockStatus r; + do { + PyTime_t microseconds; + microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); + + /* first a simple non-blocking try without releasing the GIL */ + r = PyThread_acquire_lock_timed(lock, 0, 0); + if (r == PY_LOCK_FAILURE && microseconds != 0) { + Py_BEGIN_ALLOW_THREADS + r = PyThread_acquire_lock_timed(lock, microseconds, 1); + Py_END_ALLOW_THREADS + } + + if (r == PY_LOCK_INTR) { + /* Run signal handlers if we were interrupted. Propagate + * exceptions from signal handlers, such as KeyboardInterrupt, by + * passing up PY_LOCK_INTR. */ + if (_PyEval_MakePendingCalls(tstate) < 0) { + return PY_LOCK_INTR; + } + + /* If we're using a timeout, recompute the timeout after processing + * signals, since those can take time. */ + if (timeout > 0) { + timeout = _PyDeadline_Get(endtime); + + /* Check for negative values, since those mean block forever. + */ + if (timeout < 0) { + r = PY_LOCK_FAILURE; + } + } + } + } while (r == PY_LOCK_INTR); /* Retry if we were interrupted. */ + + return r; +} + + /* Thread Specific Storage (TSS) API Cross-platform components of TSS API implementation. diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 26f441bd6d3c56..425658131c2fce 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_time.h" // _PyTime_AsMicroseconds() /* This code implemented by Dag.Gruneau@elsa.preseco.comm.se */ /* Fast NonRecursiveMutex support by Yakov Markovitch, markovitch@iso.ru */ @@ -76,17 +77,18 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - _PyTime_t nanoseconds = _PyTime_FromNanoseconds((_PyTime_t)milliseconds * 1000000); - _PyTime_t deadline = _PyTime_Add(_PyTime_GetPerfCounter(), nanoseconds); + PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyDeadline_Init(timeout); while (mutex->locked) { - _PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, - _PyTime_ROUND_TIMEOUT); + PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_GetPerfCounter(); - if (nanoseconds <= 0) { + + timeout = _PyDeadline_Get(deadline); + if (timeout <= 0) { break; } } @@ -182,9 +184,9 @@ bootstrap(void *call) return 0; } -unsigned long -PyThread_start_new_thread(void (*func)(void *), void *arg) -{ +int +PyThread_start_joinable_thread(void (*func)(void *), void *arg, + PyThread_ident_t* ident, PyThread_handle_t* handle) { HANDLE hThread; unsigned threadID; callobj *obj; @@ -194,7 +196,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) obj = (callobj*)HeapAlloc(GetProcessHeap(), 0, sizeof(*obj)); if (!obj) - return PYTHREAD_INVALID_THREAD_ID; + return -1; obj->func = func; obj->arg = arg; PyThreadState *tstate = _PyThreadState_GET(); @@ -207,22 +209,47 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) /* I've seen errno == EAGAIN here, which means "there are * too many threads". */ - int e = errno; - threadID = (unsigned)-1; HeapFree(GetProcessHeap(), 0, obj); + return -1; } - else { - CloseHandle(hThread); + *ident = threadID; + // The cast is safe since HANDLE is pointer-sized + *handle = (PyThread_handle_t) hThread; + return 0; +} + +unsigned long +PyThread_start_new_thread(void (*func)(void *), void *arg) { + PyThread_handle_t handle; + PyThread_ident_t ident; + if (PyThread_start_joinable_thread(func, arg, &ident, &handle)) { + return PYTHREAD_INVALID_THREAD_ID; } - return threadID; + CloseHandle((HANDLE) handle); + // The cast is safe since the ident is really an unsigned int + return (unsigned long) ident; +} + +int +PyThread_join_thread(PyThread_handle_t handle) { + HANDLE hThread = (HANDLE) handle; + int errored = (WaitForSingleObject(hThread, INFINITE) != WAIT_OBJECT_0); + CloseHandle(hThread); + return errored; +} + +int +PyThread_detach_thread(PyThread_handle_t handle) { + HANDLE hThread = (HANDLE) handle; + return (CloseHandle(hThread) == 0); } /* * Return the thread Id instead of a handle. The Id is said to uniquely identify the * thread in the system */ -unsigned long -PyThread_get_thread_ident(void) +PyThread_ident_t +PyThread_get_thread_ident_ex(void) { if (!initialized) PyThread_init_thread(); @@ -230,6 +257,13 @@ PyThread_get_thread_ident(void) return GetCurrentThreadId(); } +unsigned long +PyThread_get_thread_ident(void) +{ + return (unsigned long) PyThread_get_thread_ident_ex(); +} + + #ifdef PY_HAVE_THREAD_NATIVE_ID /* * Return the native Thread ID (TID) of the calling thread. @@ -408,16 +442,7 @@ PyThread_set_key_value(int key, void *value) void * PyThread_get_key_value(int key) { - /* because TLS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key); - SetLastError(error); - return result; + return TlsGetValue(key); } void @@ -489,14 +514,10 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - /* because TSS is used in the Py_END_ALLOW_THREAD macro, - * it is necessary to preserve the windows error state, because - * it is assumed to be preserved across the call to the macro. - * Ideally, the macro should be fixed, but it is simpler to - * do it here. - */ - DWORD error = GetLastError(); - void *result = TlsGetValue(key->_key); - SetLastError(error); - return result; + int err = GetLastError(); + void *r = TlsGetValue(key->_key); + if (r || !GetLastError()) { + SetLastError(err); + } + return r; } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 76a1f7763f23b9..f588b4620da0d3 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,5 +1,6 @@ #include "pycore_interp.h" // _PyInterpreterState.threads.stacksize #include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_FromMicrosecondsClamup() /* Posix threads interface */ @@ -20,6 +21,8 @@ # include /* syscall(SYS_gettid) */ #elif defined(__FreeBSD__) # include /* pthread_getthreadid_np() */ +#elif defined(__FreeBSD_kernel__) +# include /* syscall(SYS_thr_self) */ #elif defined(__OpenBSD__) # include /* getthrid() */ #elif defined(_AIX) @@ -92,6 +95,10 @@ #endif #endif +/* Thread sanitizer doesn't currently support sem_clockwait */ +#ifdef _Py_THREAD_SANITIZER +#undef HAVE_SEM_CLOCKWAIT +#endif /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: @@ -147,16 +154,18 @@ _PyThread_cond_init(PyCOND_T *cond) void _PyThread_cond_after(long long us, struct timespec *abs) { - _PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); - _PyTime_t t; + PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); + PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_GetMonotonicClock(); + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&t); } else #endif { - t = _PyTime_GetSystemClock(); + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&t); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -235,8 +244,8 @@ pythread_wrapper(void *arg) return NULL; } -unsigned long -PyThread_start_new_thread(void (*func)(void *), void *arg) +static int +do_start_joinable_thread(void (*func)(void *), void *arg, pthread_t* out_id) { pthread_t th; int status; @@ -252,7 +261,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) #if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED) if (pthread_attr_init(&attrs) != 0) - return PYTHREAD_INVALID_THREAD_ID; + return -1; #endif #if defined(THREAD_STACK_SIZE) PyThreadState *tstate = _PyThreadState_GET(); @@ -261,7 +270,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) if (tss != 0) { if (pthread_attr_setstacksize(&attrs, tss) != 0) { pthread_attr_destroy(&attrs); - return PYTHREAD_INVALID_THREAD_ID; + return -1; } } #endif @@ -272,7 +281,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) pythread_callback *callback = PyMem_RawMalloc(sizeof(pythread_callback)); if (callback == NULL) { - return PYTHREAD_INVALID_THREAD_ID; + return -1; } callback->func = func; @@ -292,11 +301,34 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) if (status != 0) { PyMem_RawFree(callback); - return PYTHREAD_INVALID_THREAD_ID; + return -1; } + *out_id = th; + return 0; +} - pthread_detach(th); +int +PyThread_start_joinable_thread(void (*func)(void *), void *arg, + PyThread_ident_t* ident, PyThread_handle_t* handle) { + pthread_t th = (pthread_t) 0; + if (do_start_joinable_thread(func, arg, &th)) { + return -1; + } + *ident = (PyThread_ident_t) th; + *handle = (PyThread_handle_t) th; + assert(th == (pthread_t) *ident); + assert(th == (pthread_t) *handle); + return 0; +} +unsigned long +PyThread_start_new_thread(void (*func)(void *), void *arg) +{ + pthread_t th = (pthread_t) 0; + if (do_start_joinable_thread(func, arg, &th)) { + return PYTHREAD_INVALID_THREAD_ID; + } + pthread_detach(th); #if SIZEOF_PTHREAD_T <= SIZEOF_LONG return (unsigned long) th; #else @@ -304,20 +336,36 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) #endif } +int +PyThread_join_thread(PyThread_handle_t th) { + return pthread_join((pthread_t) th, NULL); +} + +int +PyThread_detach_thread(PyThread_handle_t th) { + return pthread_detach((pthread_t) th); +} + /* XXX This implementation is considered (to quote Tim Peters) "inherently hosed" because: - It does not guarantee the promise that a non-zero integer is returned. - The cast to unsigned long is inherently unsafe. - It is not clear that the 'volatile' (for AIX?) are any longer necessary. */ -unsigned long -PyThread_get_thread_ident(void) -{ +PyThread_ident_t +PyThread_get_thread_ident_ex(void) { volatile pthread_t threadid; if (!initialized) PyThread_init_thread(); threadid = pthread_self(); - return (unsigned long) threadid; + assert(threadid == (pthread_t) (PyThread_ident_t) threadid); + return (PyThread_ident_t) threadid; +} + +unsigned long +PyThread_get_thread_ident(void) +{ + return (unsigned long) PyThread_get_thread_ident_ex(); } #ifdef PY_HAVE_THREAD_NATIVE_ID @@ -335,6 +383,9 @@ PyThread_get_thread_native_id(void) #elif defined(__FreeBSD__) int native_id; native_id = pthread_getthreadid_np(); +#elif defined(__FreeBSD_kernel__) + long native_id; + syscall(SYS_thr_self, &native_id); #elif defined(__OpenBSD__) pid_t native_id; native_id = getthrid(); @@ -437,31 +488,34 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, (void) error; /* silence unused-but-set-variable warning */ - _PyTime_t timeout; // relative timeout + PyTime_t timeout; // relative timeout if (microseconds >= 0) { // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout // overflow to the caller, so clamp the timeout to - // [_PyTime_MIN, _PyTime_MAX]. + // [PyTime_MIN, PyTime_MAX]. // - // _PyTime_MAX nanoseconds is around 292.3 years. + // PyTime_MAX nanoseconds is around 292.3 years. // // _thread.Lock.acquire() and _thread.RLock.acquire() raise an // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. timeout = _PyTime_FromMicrosecondsClamp(microseconds); } else { - timeout = _PyTime_FromNanoseconds(-1); + timeout = -1; } #ifdef HAVE_SEM_CLOCKWAIT struct timespec abs_timeout; // Local scope for deadline { - _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else - _PyTime_t deadline = 0; + PyTime_t deadline = 0; if (timeout > 0 && !intr_flag) { deadline = _PyDeadline_Init(timeout); } @@ -473,8 +527,11 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - _PyTime_t abs_time = _PyTime_Add(_PyTime_GetSystemClock(), - timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t abs_time = _PyTime_Add(now, timeout); + struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); status = fix_status(sem_timedwait(thelock, &ts)); diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 48bad36ec449ab..4741e594e52e65 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -94,6 +94,15 @@ pthread_detach(pthread_t thread) return 0; } +int +pthread_join(pthread_t thread, void** value_ptr) +{ + if (value_ptr) { + *value_ptr = NULL; + } + return 0; +} + PyAPI_FUNC(pthread_t) pthread_self(void) { return 0; diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md new file mode 100644 index 00000000000000..5ceda8e806045d --- /dev/null +++ b/Python/tier2_engine.md @@ -0,0 +1,150 @@ +# The tier 2 execution engine + +## General idea + +When execution in tier 1 becomes "hot", that is the counter for that point in +the code reaches some threshold, we create an executor and execute that +instead of the tier 1 bytecode. + +Since each executor must exit, we also track the "hotness" of those +exits and attach new executors to those exits. + +As the program executes, and the hot parts of the program get optimized, +a graph of executors forms. + +## Superblocks and Executors + +Once a point in the code has become hot enough, we want to optimize it. +Starting from that point we project the likely path of execution, +using information gathered by tier 1 to guide that projection to +form a "superblock", a mostly linear sequence of micro-ops. +Although mostly linear, it may include a single loop. + +We then optimize this superblock to form an optimized superblock, +which is equivalent but more efficient. + +A superblock is a representation of the code we want to execute, +but it is not in executable form. +The executable form is known as an executor. + +Executors are semantically equivalent to the superblock they are +created from, but are in a form that can be efficiently executable. + +There are two execution engines for executors, and two types of executors: +* The hardware which runs machine code executors created by the JIT compiler. +* The tier 2 interpreter runs bytecode executors. + +It would be very wasteful to support both a tier 2 interpreter and +JIT compiler in the same process. +For now, we will make the choice of engine a configuration option, +but we could make it a command line option in the future if that would prove useful. + + +### Tier 2 Interpreter + +For platforms without a JIT and for testing, we need an interpreter +for executors. It is similar in design to the tier 1 interpreter, but has a +different instruction set, and does not adapt. + +### JIT compiler + +The JIT compiler converts superblocks into machine code executors. +These have identical behavior to interpreted executors, except that +they consume more memory for the generated machine code and are a lot faster. + +## Transferring control + +There are three types of control transfer that we need to consider: +* Tier 1 to tier 2 +* Tier 2 to tier 1 +* One executor to another within tier 2 + +Since we expect the graph of executors to span most of the hot +part of the program, transfers from one executor to another should +be the most common. +Therefore, we want to make those transfers fast. + +### Tier 2 to tier 2 + +#### Cold exits + +All side exits start cold and most stay cold, but a few become +hot. We want to keep the memory consumption small for the many +cold exits, but those that become hot need to be fast. +However we cannot know in advance, which will be which. + +So that tier 2 to tier 2 transfers are fast for hot exits, +exits must be implemented as executors. In order to patch +executor exits when they get hot, a pointer to the current +executor must be passed to the exit executor. + +#### Handling reference counts + +There must be an implicit reference to the currently executing +executor, otherwise it might be freed. +Consequently, we must increment the reference count of an +executor just before executing it, and decrement it just after +executing it. + +We want to minimize the amount of data that is passed from +one executor to the next. In the JIT, this reduces the number +of arguments in the tailcall, freeing up registers for other uses. +It is less important in the interpreter, but following the same +design as the JIT simplifies debugging and is good for performance. + +Provided that we incref the new executor before executing it, we +can jump directly to the code of the executor, without needing +to pass a reference to that executor object. +However, we do need a reference to the previous executor, +so that it can be decref'd and for handling of cold exits. +To avoid messing up the JIT's register allocation, we pass a +reference to the previous executor in the thread state's +`previous_executor` field. + +#### The interpreter + +The tier 2 interpreter has a variable `current_executor` which +points to the currently live executor. When transferring from executor +`A` to executor `B` we do the following: +(Initially `current_executor` points to `A`, and the refcount of +`A` is elevated by one) + +1. Set the instruction pointer to start at the beginning of `B` +2. Increment the reference count of `B` +3. Start executing `B` + +We also make the first instruction in `B` do the following: +1. Set `current_executor` to point to `B` +2. Decrement the reference count of `A` (`A` is referenced by `tstate->previous_executor`) + +The net effect of the above is to safely decrement the refcount of `A`, +increment the refcount of `B` and set `current_executor` to point to `B`. + +#### In the JIT + +Transferring control from one executor to another is done via tailcalls. + +The compiled executor should do the same, except that there is no local +variable `current_executor`. + +### Tier 1 to tier 2 + +Since the executor doesn't know if the previous code was tier 1 or tier 2, +we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2 +transfer to the executor. + +We can then perform a tier 1 to tier 2 transfer by setting `current_executor` +to `None`, and then performing a tier 2 to tier 2 transfer as above. + +### Tier 2 to tier 1 + +Each micro-op that might exit to tier 1 contains a `target` value, +which is the offset of the tier 1 instruction to exit to in the +current code object. + +## Counters + +TO DO. +The implementation will change soon, so there is no point in +documenting it until then. + diff --git a/Python/traceback.c b/Python/traceback.c index 05d841e56ad7bd..47b77c9108dd9a 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -5,7 +5,6 @@ #include "pycore_ast.h" // asdl_seq_GET() #include "pycore_call.h" // _PyObject_CallMethodFormat() -#include "pycore_compile.h" // _PyAST_Optimize() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_frame.h" // _PyFrame_GetCode() #include "pycore_interp.h" // PyInterpreterState.gc @@ -16,7 +15,6 @@ #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // EXCEPTION_TB_HEADER -#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() #include "frameobject.h" // PyFrame_New() #include "osdefs.h" // SEP @@ -36,9 +34,9 @@ extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); /*[clinic input] -class TracebackType "PyTracebackObject *" "&PyTraceback_Type" +class traceback "PyTracebackObject *" "&PyTraceback_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=928fa06c10151120]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=cf96294b2bebc811]*/ #include "clinic/traceback.c.h" @@ -65,7 +63,7 @@ tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, /*[clinic input] @classmethod -TracebackType.__new__ as tb_new +traceback.__new__ as tb_new tb_next: object tb_frame: object(type='PyFrameObject *', subclass_of='&PyFrame_Type') @@ -78,7 +76,7 @@ Create a new traceback object. static PyObject * tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame, int tb_lasti, int tb_lineno) -/*[clinic end generated code: output=fa077debd72d861a input=01cbe8ec8783fca7]*/ +/*[clinic end generated code: output=fa077debd72d861a input=b88143145454cb59]*/ { if (tb_next == Py_None) { tb_next = NULL; @@ -109,6 +107,26 @@ tb_next_get(PyTracebackObject *self, void *Py_UNUSED(_)) return Py_NewRef(ret); } +static int +tb_get_lineno(PyTracebackObject* tb) { + _PyInterpreterFrame* frame = tb->tb_frame->f_frame; + assert(frame != NULL); + return PyCode_Addr2Line(_PyFrame_GetCode(frame), tb->tb_lasti); +} + +static PyObject * +tb_lineno_get(PyTracebackObject *self, void *Py_UNUSED(_)) +{ + int lineno = self->tb_lineno; + if (lineno == -1) { + lineno = tb_get_lineno(self); + if (lineno < 0) { + Py_RETURN_NONE; + } + } + return PyLong_FromLong(lineno); +} + static int tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_)) { @@ -152,12 +170,12 @@ static PyMethodDef tb_methods[] = { static PyMemberDef tb_memberlist[] = { {"tb_frame", _Py_T_OBJECT, OFF(tb_frame), Py_READONLY|Py_AUDIT_READ}, {"tb_lasti", Py_T_INT, OFF(tb_lasti), Py_READONLY}, - {"tb_lineno", Py_T_INT, OFF(tb_lineno), Py_READONLY}, {NULL} /* Sentinel */ }; static PyGetSetDef tb_getsetters[] = { {"tb_next", (getter)tb_next_get, (setter)tb_next_set, NULL, NULL}, + {"tb_lineno", (getter)tb_lineno_get, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -236,8 +254,7 @@ _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame) assert(tb_next == NULL || PyTraceBack_Check(tb_next)); assert(frame != NULL); int addr = _PyInterpreterFrame_LASTI(frame->f_frame) * sizeof(_Py_CODEUNIT); - return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, - PyFrame_GetLineNumber(frame)); + return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, -1); } @@ -600,44 +617,7 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen if (rc != 0 || !source_line) { /* ignore errors since we can't report them, can we? */ err = ignore_source_errors(); - goto done; - } - - int code_offset = tb->tb_lasti; - PyCodeObject* code = _PyFrame_GetCode(frame->f_frame); - const Py_ssize_t source_line_len = PyUnicode_GET_LENGTH(source_line); - - int start_line; - int end_line; - int start_col_byte_offset; - int end_col_byte_offset; - if (!PyCode_Addr2Location(code, code_offset, &start_line, &start_col_byte_offset, - &end_line, &end_col_byte_offset)) { - goto done; } - - if (start_line < 0 || end_line < 0 - || start_col_byte_offset < 0 - || end_col_byte_offset < 0) - { - goto done; - } - - // If this is a multi-line expression, then we will highlight until - // the last non-whitespace character. - const char *source_line_str = PyUnicode_AsUTF8(source_line); - if (!source_line_str) { - goto done; - } - - Py_ssize_t i = source_line_len; - while (--i >= 0) { - if (!IS_WHITESPACE(source_line_str[i])) { - break; - } - } - -done: Py_XDECREF(source_line); return err; } @@ -681,9 +661,13 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) } while (tb != NULL) { code = PyFrame_GetCode(tb->tb_frame); + int tb_lineno = tb->tb_lineno; + if (tb_lineno == -1) { + tb_lineno = tb_get_lineno(tb); + } if (last_file == NULL || code->co_filename != last_file || - last_line == -1 || tb->tb_lineno != last_line || + last_line == -1 || tb_lineno != last_line || last_name == NULL || code->co_name != last_name) { if (cnt > TB_RECURSIVE_CUTOFF) { if (tb_print_line_repeated(f, cnt) < 0) { @@ -691,13 +675,13 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) } } last_file = code->co_filename; - last_line = tb->tb_lineno; + last_line = tb_lineno; last_name = code->co_name; cnt = 0; } cnt++; if (cnt <= TB_RECURSIVE_CUTOFF) { - if (tb_displayline(tb, f, code->co_filename, tb->tb_lineno, + if (tb_displayline(tb, f, code->co_filename, tb_lineno, tb->tb_frame, code->co_name) < 0) { goto error; } @@ -980,7 +964,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { - PUTS(fd, " ...\n"); + if (MAX_FRAME_DEPTH < depth) { + PUTS(fd, "plus "); + _Py_DumpDecimal(fd, depth); + PUTS(fd, " frames\n"); + } break; } dump_frame(fd, frame); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 19b64c619feb6a..e58b60ddd5e484 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -312,7 +312,7 @@ traceback_hash(traceback_t *traceback) /* code based on tuplehash() of Objects/tupleobject.c */ Py_uhash_t x, y; /* Unsigned for defined overflow behavior. */ int len = traceback->nframe; - Py_uhash_t mult = _PyHASH_MULTIPLIER; + Py_uhash_t mult = PyHASH_MULTIPLIER; frame_t *frame; x = 0x345678UL; @@ -838,7 +838,7 @@ _PyTraceMalloc_Init(void) tracemalloc_tracebacks = hashtable_new(hashtable_hash_traceback, hashtable_compare_traceback, - NULL, raw_free); + raw_free, NULL); tracemalloc_traces = tracemalloc_create_traces_table(); tracemalloc_domains = tracemalloc_create_domains_table(); @@ -906,6 +906,10 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } + if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; @@ -1352,8 +1356,12 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ int -_PyTraceMalloc_NewReference(PyObject *op) +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) { + if (event != PyRefTracer_CREATE) { + return 0; + } + assert(PyGILState_Check()); if (!tracemalloc_config.tracing) { diff --git a/Python/vm-state.md b/Python/vm-state.md new file mode 100644 index 00000000000000..b3246557dbeea3 --- /dev/null +++ b/Python/vm-state.md @@ -0,0 +1,90 @@ +# Python VM State + +## Definition of Tiers + +- **Tier 1** is the classic Python bytecode interpreter. + This includes the specializing adaptive interpreter described in [PEP 659](https://peps.python.org/pep-0659/) and introduced in Python 3.11. +- **Tier 2**, also known as the micro-instruction ("uop") interpreter, is a new interpreter with a different instruction format. + It will be introduced in Python 3.13, and also forms the basis for a JIT using copy-and-patch technology that is likely to be introduced at the same time (but, unlike the Tier 2 interpreter, hasn't landed in the main branch yet). + +# Frame state + +Almost all interpreter state is nominally stored in the frame structure. +A pointer to the current frame is held in `frame`. It contains: + +- **local variables** (a.k.a. "fast locals") +- **evaluation stack** (tacked onto the end of the locals) +- **stack top** (an integer giving the top of the evaluation stack) +- **instruction pointer** +- **code object**, which holds things like the array of instructions, lists of constants and names referenced by certain instructions, the exception handling table, and the table that translates instruction offsets to line numbers +- **return offset**, only relevant during calls, telling the interpreter where to return + +There are some other fields in the frame structure of less importance; notably frames are linked together in a singly-linked list via the `previous` pointer, pointing from callee to caller. +The frame also holds a pointer to the current function, globals, builtins, and the locals converted to dict (used to support the `locals()` built-in). + +## Fast locals and evaluation stack + +The frame contains a single array of object pointers, `localsplus`, which contains both the fast locals and the stack. +The top of the stack, including the locals, is indicated by `stacktop`. +For example, in a function with three locals, if the stack contains one value, `frame->stacktop == 4`. + +The interpreters share an implementation which uses the same memory but caches the depth (as a pointer) in a C local, `stack_pointer`. +We aren't sure yet exactly how the JIT will implement the stack; likely some of the values near the top of the stack will be held in registers. + +## Instruction pointer + +The canonical, in-memory, representation of the instruction pointer is `frame->instr_ptr`. +It always points to an instruction in the bytecode array of the frame's code object. +Dispatching on `frame->instr_ptr` would be very inefficient, so in Tier 1 we cache the upcoming value of `frame->instr_ptr` in the C local `next_instr`. + +## Tier 2 + +- `stack_pointer` is the same as in Tier 1 (but may be different in the JIT). +- At runtime we do not need a cache representation of `frame->instr_ptr`, as all stores to `frame->instr_ptr` are explicit. +- During optimization we track the value of `frame->instr_ptr`, emitting `_SET_IP` whenever `frame->instr_ptr` would have been updated. + +The Tier 2 instruction pointer is strictly internal to the Tier 2 interpreter, so isn't visible to any other part of the code. + +## Unwinding + +Unwinding uses exception tables to find the next point at which normal execution can occur, or fail if there are no exception handlers. +During unwinding both the stack and the instruction pointer should be in their canonical, in-memory representation. + +## Jumps in bytecode + +The implementation of jumps within a single Tier 2 superblock/trace is just that, an implementation. +The implementation in the JIT and in the Tier 2 interpreter will necessarily be different. +What is in common is that representation in the Tier 2 optimizer. + +We need the following types of jumps: + +- Conditional branches within the superblock. These must only go forwards and be within the superblock. +- Terminal exits. These go back to the Tier 1 interpreter and cannot be modified. +- Loop end jumps. These go backwards, must be within the superblock, cannot be modified, and can only go to the start of the superblock. +- Patchable exits. These initially exit to code that tracks whether the exit is hot (presumably with a counter) and can be patched. + +Currently, we don't have patchable exits. +Patching exits should be fairly straightforward in the interpreter. +It will be more complex in the JIT. + +(We might also consider deoptimizations as a separate jump type.) + +# Thread state and interpreter state + +Another important piece of VM state is the **thread state**, held in `tstate`. +The current frame pointer, `frame`, is always equal to `tstate->current_frame`. +The thread state also holds the exception state (`tstate->exc_info`) and the recursion counters (`tstate->c_recursion_remaining` and `tstate->py_recursion_remaining`). + +The thread state is also used to access the **interpreter state** (`tstate->interp`), which is important since the "eval breaker" flags are stored there (`tstate->interp->ceval.eval_breaker`, an "atomic" variable), as well as the "PEP 523 function" (`tstate->interp->eval_frame`). +The interpreter state also holds the optimizer state (`optimizer` and some counters). +Note that the eval breaker may be moved to the thread state soon as part of the multicore (PEP 703) work. + +# Tier 2 IR format + +The tier 2 IR (Internal Representation) format is also the basis for the Tier 2 interpreter (though the two formats may eventually differ). This format is also used as the input to the machine code generator (the JIT compiler). + +Tier 2 IR entries are all the same size; there is no equivalent to `EXTENDED_ARG` or trailing inline cache entries. Each instruction is a struct with the following fields (all integers of varying sizes): + +- **opcode**: Sometimes the same as a Tier 1 opcode, sometimes a separate micro opcode. Tier 2 opcodes are 9 bits (as opposed to Tier 1 opcodes, which fit in 8 bits). By convention, Tier 2 opcode names start with `_`. +- **oparg**: The argument. Usually the same as the Tier 1 oparg after expansion of `EXTENDED_ARG` prefixes. Up to 32 bits. +- **operand**: An additional argument, Typically the value of *one* cache item from the Tier 1 inline cache, up to 64 bits. diff --git a/README.rst b/README.rst index 9ba06f76ba5da8..7dd3660b198784 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ -This is Python version 3.13.0 alpha 1 +This is Python version 3.14.0 alpha 0 ===================================== -.. image:: https://github.com/python/cpython/workflows/Tests/badge.svg +.. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions :target: https://github.com/python/cpython/actions @@ -14,7 +14,7 @@ This is Python version 3.13.0 alpha 1 :target: https://discuss.python.org/ -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. See the end of this file for further copyright and license information. @@ -136,7 +136,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.13 `_ document. For a more +3.14 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,7 +149,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.13 `_ is online, +`Documentation for Python 3.14 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -161,15 +161,6 @@ For information about building Python's documentation, refer to `Doc/README.rst `_. -Converting From Python 2.x to 3.x ---------------------------------- - -Significant backward incompatible changes were made for the release of Python -3.0, which may cause programs written for Python 2 to fail when run with Python -3. For more information about porting your code from Python 2 to Python 3, see -the `Porting HOWTO `_. - - Testing ------- @@ -206,25 +197,25 @@ directories installed using ``make altinstall`` contain the major and minor version and can thus live side-by-side. ``make install`` also creates ``${prefix}/bin/python3`` which refers to ``${prefix}/bin/python3.X``. If you intend to install multiple versions using the same prefix you must decide which -version (if any) is your "primary" version. Install that version using ``make -install``. Install all other versions using ``make altinstall``. +version (if any) is your "primary" version. Install that version using +``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the -primary version, you would execute ``make install`` in your 3.13 build directory +For example, if you want to install Python 2.7, 3.6, and 3.14 with 3.14 being the +primary version, you would execute ``make install`` in your 3.14 build directory and ``make altinstall`` in the others. Release Schedule ---------------- -See :pep:`719` for Python 3.13 release details. +See `PEP 745 `__ for Python 3.14 release details. Copyright and License Information --------------------------------- -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001-2024 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Tools/README b/Tools/README index 9c4b6d86e990ba..09bd6fb4798950 100644 --- a/Tools/README +++ b/Tools/README @@ -10,8 +10,6 @@ c-analyzer Tools to check no new global variables have been added. cases_generator Tooling to generate interpreters. -ccbench A Python threads-based concurrency benchmark. (*) - clinic A preprocessor for CPython C files in order to automate the boilerplate involved with writing argument parsing code for "builtins". @@ -28,8 +26,6 @@ i18n Tools for internationalization. pygettext.py importbench A set of micro-benchmarks for various import scenarios. -iobench Benchmark for the new Python I/O system. (*) - msi Support for packaging Python as an MSI package on Windows. nuget Files for the NuGet package manager for .NET. @@ -45,9 +41,6 @@ scripts A number of useful single-file programs, e.g. run_tests.py ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. -stringbench A suite of micro-benchmarks for various operations on - strings (both 8-bit and unicode). (*) - tz A script to dump timezone from /usr/share/zoneinfo. unicode Tools for generating unicodedata and codecs from unicode.org @@ -60,6 +53,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test wasm Config and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). -(*) A generic benchmark suite is maintained separately at https://github.com/python/performance - Note: The pynche color editor has moved to https://gitlab.com/warsaw/pynche diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index c3231a5a40c326..05633e3f77af49 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -21,7 +21,7 @@ verbose = False -# This must be kept in sync with Tools/cases_generator/generate_cases.py +# This must be kept in sync with Tools/cases_generator/analyzer.py RESUME = 149 def isprintable(b: bytes) -> bool: @@ -115,6 +115,7 @@ def __init__(self, file: TextIO) -> None: self.inits: list[str] = [] self.identifiers, self.strings = self.get_identifiers_and_strings() self.write('#include "Python.h"') + self.write('#include "internal/pycore_object.h"') self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_frame.h"') @@ -154,14 +155,10 @@ def block(self, prefix: str, suffix: str = "") -> None: self.write("}" + suffix) def object_head(self, typename: str) -> None: - with self.block(".ob_base =", ","): - self.write(f".ob_refcnt = _Py_IMMORTAL_REFCNT,") - self.write(f".ob_type = &{typename},") + self.write(f".ob_base = _PyObject_HEAD_INIT(&{typename}),") def object_var_head(self, typename: str, size: int) -> None: - with self.block(".ob_base =", ","): - self.object_head(typename) - self.write(f".ob_size = {size},") + self.write(f".ob_base = _PyVarObject_HEAD_INIT(&{typename}, {size}),") def field(self, obj: object, name: str) -> None: self.write(f".{name} = {getattr(obj, name)},") @@ -491,7 +488,10 @@ def generate(args: list[str], output: TextIO) -> None: parser = argparse.ArgumentParser() parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") -parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format") +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument("-f", "--file", help="read rule lines from a file") +group.add_argument('args', nargs="*", default=(), + help="Input file and module name (required) in file:modname format") @contextlib.contextmanager def report_time(label: str): @@ -509,9 +509,18 @@ def main() -> None: args = parser.parse_args() verbose = args.verbose output = args.output + + if args.file: + if verbose: + print(f"Reading targets from {args.file}") + with open(args.file, "rt", encoding="utf-8-sig") as fin: + rules = [x.strip() for x in fin] + else: + rules = args.args + with open(output, "w", encoding="utf-8") as file: with report_time("generate"): - generate(args.args, file) + generate(rules, file) if verbose: print(f"Wrote {os.path.getsize(output)} bytes to {output}") diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index a07f4d9786ea65..7f1dee18319749 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -5,10 +5,10 @@ from collections import namedtuple import hashlib -import os import ntpath +import os import posixpath -import argparse + from update_file import updating_file_with_tmpfile @@ -20,7 +20,6 @@ # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the # .gitattributes and .gitignore files needs to be updated. FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') @@ -232,7 +231,7 @@ def iter_subs(): ####################################### # frozen source files -class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')): +class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')): @classmethod def from_id(cls, frozenid, pyfile=None): @@ -240,8 +239,7 @@ def from_id(cls, frozenid, pyfile=None): pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py' #assert os.path.exists(pyfile), (frozenid, pyfile) frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR) - deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR) - return cls(frozenid, pyfile, frozenfile, deepfreezefile) + return cls(frozenid, pyfile, frozenfile) @property def frozenid(self): @@ -467,6 +465,17 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] +class UniqueList(list): + def __init__(self): + self._seen = set() + + def append(self, item): + if item in self._seen: + return + super().append(item) + self._seen.add(item) + + def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) @@ -476,7 +485,7 @@ def regen_frozen(modules): header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') - externlines = [] + externlines = UniqueList() bootstraplines = [] stdliblines = [] testlines = [] @@ -496,13 +505,6 @@ def regen_frozen(modules): lines.append(f'/* {mod.section} */') lastsection = mod.section - # Also add a extern declaration for the corresponding - # deepfreeze-generated function. - orig_name = mod.source.id - code_name = orig_name.replace(".", "_") - get_code_name = "_Py_get_%s_toplevel" % code_name - externlines.append("extern PyObject *%s(void);" % get_code_name) - pkg = 'true' if mod.ispkg else 'false' size = f"(int)sizeof({mod.symbol})" line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' @@ -537,13 +539,6 @@ def regen_frozen(modules): headerlines, FROZEN_FILE, ) - lines = replace_block( - lines, - "/* Start extern declarations */", - "/* End extern declarations */", - externlines, - FROZEN_FILE, - ) lines = replace_block( lines, "static const struct _frozen bootstrap_modules[] =", @@ -579,8 +574,6 @@ def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] - deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)", - "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') @@ -602,8 +595,6 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\") - deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -631,13 +622,6 @@ def regen_makefile(modules): rules, MAKEFILE, ) - lines = replace_block( - lines, - "# BEGIN: deepfreeze modules", - "# END: deepfreeze modules", - deepfreezerules, - MAKEFILE, - ) outfile.writelines(lines) @@ -645,7 +629,6 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezerules = ['\t') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') - projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(GeneratedFrozenModulesDir){header}') projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^') - deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' ) - - corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): @@ -675,16 +654,6 @@ def regen_pcbuild(modules): PCBUILD_PROJECT, ) outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezerules, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() @@ -696,17 +665,6 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) - print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') - with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - corelines, - PCBUILD_FILTERS, - ) - outfile.writelines(lines) ####################################### diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index ded19ee489e79b..882918fafb1edd 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -123,6 +123,14 @@ '__rdivmod__', '__buffer__', '__release_buffer__', + + #Workarounds for GH-108918 + 'alias', + 'args', + 'exc_type', + 'exc_value', + 'self', + 'traceback', ] NON_GENERATED_IMMORTAL_OBJECTS = [ @@ -362,9 +370,14 @@ def generate_static_strings_initializer(identifiers, strings): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') + for value, name in sorted(strings.items()): + printer.write(f'string = &_Py_STR({name});') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write(f'_PyUnicode_InternInPlace(interp, &string);') - # XXX What about "strings"? + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') printer.write(END) printer.write(after) @@ -406,15 +419,31 @@ def generate_global_object_finalizers(generated_immortal_objects): def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': identifiers = set(IDENTIFIERS) strings = {} + # Note that we store strings as they appear in C source, so the checks here + # can be defeated, e.g.: + # - "a" and "\0x61" won't be reported as duplicate. + # - "\n" appears as 2 characters. + # Probably not worth adding a C string parser. for name, string, *_ in iter_global_strings(): if string is None: if name not in IGNORED: identifiers.add(name) else: + if len(string) == 1 and ord(string) < 256: + # Give a nice message for common mistakes. + # To cover tricky cases (like "\n") we also generate C asserts. + raise ValueError( + 'do not use &_PyID or &_Py_STR for one-character latin-1 ' + + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name elif name != strings[string]: raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}') + overlap = identifiers & set(strings.keys()) + if overlap: + raise ValueError( + 'do not use both _PyID and _Py_DECLARE_STR for the same string: ' + + repr(overlap)) return identifiers, strings diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py new file mode 100644 index 00000000000000..c08568f2e00326 --- /dev/null +++ b/Tools/build/generate_sbom.py @@ -0,0 +1,345 @@ +"""Tool for generating Software Bill of Materials (SBOM) for Python's dependencies""" +import os +import re +import hashlib +import json +import glob +from pathlib import Path, PurePosixPath, PureWindowsPath +import subprocess +import sys +import urllib.request +import typing + +CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent + +# Before adding a new entry to this list, double check that +# the license expression is a valid SPDX license expression: +# See: https://spdx.org/licenses +ALLOWED_LICENSE_EXPRESSIONS = { + "Apache-2.0", + "Apache-2.0 OR BSD-2-Clause", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LGPL-2.1-only", + "MIT", + "MPL-2.0", + "Python-2.0.1", +} + +# Properties which are required for our purposes. +REQUIRED_PROPERTIES_PACKAGE = frozenset([ + "SPDXID", + "name", + "versionInfo", + "downloadLocation", + "checksums", + "licenseConcluded", + "externalRefs", + "primaryPackagePurpose", +]) + + +class PackageFiles(typing.NamedTuple): + """Structure for describing the files of a package""" + include: list[str] | None + exclude: list[str] | None = None + + +# SBOMS don't have a method to specify the sources of files +# so we need to do that external to the SBOM itself. Add new +# values to 'exclude' if we create new files within tracked +# directories that aren't sourced from third-party packages. +PACKAGE_TO_FILES = { + "mpdecimal": PackageFiles( + include=["Modules/_decimal/libmpdec/**"] + ), + "expat": PackageFiles( + include=["Modules/expat/**"], + exclude=[ + "Modules/expat/expat_config.h", + ] + ), + "macholib": PackageFiles( + include=["Lib/ctypes/macholib/**"], + exclude=[ + "Lib/ctypes/macholib/README.ctypes", + "Lib/ctypes/macholib/fetch_macholib", + "Lib/ctypes/macholib/fetch_macholib.bat", + ], + ), + "libb2": PackageFiles( + include=["Modules/_blake2/impl/**"] + ), + "hacl-star": PackageFiles( + include=["Modules/_hacl/**"], + exclude=[ + "Modules/_hacl/refresh.sh", + "Modules/_hacl/README.md", + "Modules/_hacl/python_hacl_namespace.h", + ] + ), +} + + +def spdx_id(value: str) -> str: + """Encode a value into characters that are valid in an SPDX ID""" + return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value) + + +def error_if(value: bool, error_message: str) -> None: + """Prints an error if a comparison fails along with a link to the devguide""" + if value: + print(error_message) + print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.") + sys.exit(1) + + +def filter_gitignored_paths(paths: list[str]) -> list[str]: + """ + Filter out paths excluded by the gitignore file. + The output of 'git check-ignore --non-matching --verbose' looks + like this for non-matching (included) files: + + '::' + + And looks like this for matching (excluded) files: + + '.gitignore:9:*.a Tools/lib.a' + """ + # Filter out files in gitignore. + # Non-matching files show up as '::' + git_check_ignore_proc = subprocess.run( + ["git", "check-ignore", "--verbose", "--non-matching", *paths], + cwd=CPYTHON_ROOT_DIR, + check=False, + stdout=subprocess.PIPE, + ) + # 1 means matches, 0 means no matches. + assert git_check_ignore_proc.returncode in (0, 1) + + # Paths may or may not be quoted, Windows quotes paths. + git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z") + + # Return the list of paths sorted + git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() + git_check_not_ignored = [] + for line in git_check_ignore_lines: + if match := git_check_ignore_re.fullmatch(line): + git_check_not_ignored.append(match.group(2) or match.group(3)) + return sorted(git_check_not_ignored) + + +def get_externals() -> list[str]: + """ + Parses 'PCbuild/get_externals.bat' for external libraries. + Returns a list of (git tag, name, version) tuples. + """ + get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat" + externals = re.findall( + r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s", + get_externals_bat_path.read_text() + ) + return externals + + +def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: + """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" + + for package in sbom_data["packages"]: + # Properties and ID must be properly formed. + error_if( + "name" not in package, + "Package is missing the 'name' field" + ) + + # Verify that the checksum matches the expected value + # and that the download URL is valid. + if "checksums" not in package or "CI" in os.environ: + download_location = package["downloadLocation"] + resp = urllib.request.urlopen(download_location) + error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") + + package["checksums"] = [{ + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(resp.read()).hexdigest() + }] + + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) + error_if( + bool(missing_required_keys), + f"Package '{package['name']}' is missing required fields: {missing_required_keys}", + ) + error_if( + package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"), + f"Package '{package['name']}' has a malformed SPDXID", + ) + + # Version must be in the download and external references. + version = package["versionInfo"] + error_if( + version not in package["downloadLocation"], + f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field", + ) + error_if( + any(version not in ref["referenceLocator"] for ref in package["externalRefs"]), + ( + f"Version '{version}' for package '{package['name']} not in " + f"all 'externalRefs[].referenceLocator' fields" + ), + ) + + # HACL* specifies its expected rev in a refresh script. + if package["name"] == "hacl-star": + hacl_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/_hacl/refresh.sh").read_text() + hacl_expected_rev_match = re.search( + r"expected_hacl_star_rev=([0-9a-f]{40})", + hacl_refresh_sh + ) + hacl_expected_rev = hacl_expected_rev_match and hacl_expected_rev_match.group(1) + + error_if( + hacl_expected_rev != version, + "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'" + ) + + # License must be on the approved list for SPDX. + license_concluded = package["licenseConcluded"] + error_if( + license_concluded != "NOASSERTION", + f"License identifier must be 'NOASSERTION'" + ) + + +def create_source_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + check_sbom_packages(sbom_data) + + # We call 'sorted()' here a lot to avoid filesystem scan order issues. + for name, files in sorted(PACKAGE_TO_FILES.items()): + package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") + exclude = files.exclude or () + for include in sorted(files.include or ()): + # Find all the paths and then filter them through .gitignore. + paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) + paths = filter_gitignored_paths(paths) + error_if( + len(paths) == 0, + f"No valid paths found at path '{include}' for package '{name}", + ) + + for path in paths: + + # Normalize the filename from any combination of slashes. + path = str(PurePosixPath(PureWindowsPath(path))) + + # Skip directories and excluded files + if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: + continue + + # SPDX requires SHA1 to be used for files, but we provide SHA256 too. + data = (CPYTHON_ROOT_DIR / path).read_bytes() + # We normalize line-endings for consistent checksums. + # This is a rudimentary check for binary files. + if b"\x00" not in data: + data = data.replace(b"\r\n", b"\n") + checksum_sha1 = hashlib.sha1(data).hexdigest() + checksum_sha256 = hashlib.sha256(data).hexdigest() + + file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}") + sbom_data["files"].append({ + "SPDXID": file_spdx_id, + "fileName": path, + "checksums": [ + {"algorithm": "SHA1", "checksumValue": checksum_sha1}, + {"algorithm": "SHA256", "checksumValue": checksum_sha256}, + ], + }) + + # Tie each file back to its respective package. + sbom_data["relationships"].append({ + "spdxElementId": package_spdx_id, + "relatedSpdxElement": file_spdx_id, + "relationshipType": "CONTAINS", + }) + + # Update the SBOM on disk + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +def create_externals_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + externals = get_externals() + externals_name_to_version = {} + externals_name_to_git_tag = {} + for git_tag in externals: + name, _, version = git_tag.rpartition("-") + externals_name_to_version[name] = version + externals_name_to_git_tag[name] = git_tag + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(externals_name_to_version) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + # Set the versionInfo and downloadLocation fields for all packages. + for package in sbom_data["packages"]: + package_version = externals_name_to_version[package["name"]] + + # Update the version information in all the locations. + package["versionInfo"] = package_version + for external_ref in package["externalRefs"]: + if external_ref["referenceType"] != "cpe23Type": + continue + # Version is the fifth field of a CPE. + cpe23ref = external_ref["referenceLocator"] + external_ref["referenceLocator"] = re.sub( + r"\A(cpe(?::[^:]+){4}):[^:]+:", + fr"\1:{package_version}:", + cpe23ref + ) + + download_location = ( + f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz" + ) + download_location_changed = download_location != package["downloadLocation"] + package["downloadLocation"] = download_location + + # If the download URL has changed we want one to get recalulated. + if download_location_changed: + package.pop("checksums", None) + + check_sbom_packages(sbom_data) + + # Update the SBOM on disk + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +def main() -> None: + create_source_sbom() + create_externals_sbom() + + +if __name__ == "__main__": + main() diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 766a85d3d6f39e..f9fd29509f3225 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -32,10 +32,10 @@ '_testconsole', '_testimportmultiple', '_testinternalcapi', + '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_xxsubinterpreters', - '_xxinterpchannels', + '_testexternalinspection', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini new file mode 100644 index 00000000000000..cf1dac7fde5ac5 --- /dev/null +++ b/Tools/build/mypy.ini @@ -0,0 +1,13 @@ +[mypy] +files = Tools/build/generate_sbom.py +pretty = True + +# Make sure Python can still be built +# using Python 3.10 for `PYTHON_FOR_REGEN`... +python_version = 3.10 + +# ...And be strict: +strict = True +strict_concatenate = True +enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined +warn_unreachable = True diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh new file mode 100755 index 00000000000000..e34a36c1a573e5 --- /dev/null +++ b/Tools/build/regen-configure.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e -x + +# The check_generated_files job of .github/workflows/build.yml must kept in +# sync with this script. Use the same container image than the job so the job +# doesn't need to run autoreconf in a container. +IMAGE="ubuntu:22.04" +DEPENDENCIES="autotools-dev autoconf autoconf-archive pkg-config" +AUTORECONF="autoreconf -ivf -Werror" + +WORK_DIR="/src" +SHELL_CMD="apt-get update && apt-get -yq install $DEPENDENCIES && cd $WORK_DIR && $AUTORECONF" + +abs_srcdir=$(cd $(dirname $0)/../..; pwd) + +if podman --version &>/dev/null; then + RUNTIME="podman" +elif docker --version &>/dev/null; then + RUNTIME="docker" +else + echo "$@ needs either Podman or Docker container runtime." >&2 + exit 1 +fi + +PATH_OPT="" +if command -v selinuxenabled >/dev/null && selinuxenabled; then + PATH_OPT=":Z" +fi + +"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" /usr/bin/bash -c "$SHELL_CMD" diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index ab345307ff9b64..7c534269c57a09 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -11,6 +11,11 @@ if sys.platform == 'darwin': ALLOWED_PREFIXES += ('__Py',) +# mimalloc doesn't use static, but it's symbols are not exported +# from the shared library. They do show up in the static library +# before its linked into an executable. +ALLOWED_STATIC_PREFIXES = ('mi_', '_mi_') + # "Legacy": some old symbols are prefixed by "PY_". EXCEPTIONS = frozenset({ 'PY_TIMEOUT_MAX', @@ -59,7 +64,7 @@ def get_exported_symbols(library, dynamic=False): return stdout -def get_smelly_symbols(stdout): +def get_smelly_symbols(stdout, dynamic=False): smelly_symbols = [] python_symbols = [] local_symbols = [] @@ -77,7 +82,9 @@ def get_smelly_symbols(stdout): symbol = parts[-1] result = '%s (type: %s)' % (symbol, symtype) - if symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS: + if (symbol.startswith(ALLOWED_PREFIXES) or + symbol in EXCEPTIONS or + (not dynamic and symbol.startswith(ALLOWED_STATIC_PREFIXES))): python_symbols.append(result) continue @@ -95,7 +102,7 @@ def get_smelly_symbols(stdout): def check_library(library, dynamic=False): nm_output = get_exported_symbols(library, dynamic) - smelly_symbols, python_symbols = get_smelly_symbols(nm_output) + smelly_symbols, python_symbols = get_smelly_symbols(nm_output, dynamic) if not smelly_symbols: print(f"OK: no smelly symbol found ({len(python_symbols)} Python symbols)") diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 7cba788ff33578..8b01c91e0d6bb3 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -275,9 +275,19 @@ def gen_ctypes_test(manifest, args, outfile): import sys import unittest from test.support.import_helper import import_module - from _testcapi import get_feature_macros + try: + from _testcapi import get_feature_macros + except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() + + # Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject + # layout differences. + # See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 + if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + ctypes_test = import_module('ctypes') class TestStableABIAvailability(unittest.TestCase): @@ -308,16 +318,11 @@ def test_windows_feature_macros(self): {'function', 'data'}, include_abi_only=True, ) - optional_items = {} + feature_macros = list(manifest.select({'feature_macro'})) + optional_items = {m.name: [] for m in feature_macros} for item in items: - if item.name in ( - # Some symbols aren't exported on all platforms. - # This is a bug: https://bugs.python.org/issue44133 - 'PyModule_Create2', 'PyModule_FromDefAndSpec2', - ): - continue if item.ifdef: - optional_items.setdefault(item.ifdef, []).append(item.name) + optional_items[item.ifdef].append(item.name) else: write(f' "{item.name}",') write(")") @@ -328,7 +333,6 @@ def test_windows_feature_macros(self): write(f" {name!r},") write(" )") write("") - feature_macros = list(manifest.select({'feature_macro'})) feature_names = sorted(m.name for m in feature_macros) write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})") @@ -521,7 +525,7 @@ def gcc_get_limited_api_macros(headers): api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16 - preprocesor_output_with_macros = subprocess.check_output( + preprocessor_output_with_macros = subprocess.check_output( sysconfig.get_config_var("CC").split() + [ # Prevent the expansion of the exported macros so we can @@ -540,7 +544,7 @@ def gcc_get_limited_api_macros(headers): return { target for target in re.findall( - r"#define (\w+)", preprocesor_output_with_macros + r"#define (\w+)", preprocessor_output_with_macros ) } @@ -561,7 +565,7 @@ def gcc_get_limited_api_definitions(headers): Requires Python built with a GCC-compatible compiler. (clang might work) """ api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16 - preprocesor_output = subprocess.check_output( + preprocessor_output = subprocess.check_output( sysconfig.get_config_var("CC").split() + [ # Prevent the expansion of the exported macros so we can capture @@ -581,13 +585,13 @@ def gcc_get_limited_api_definitions(headers): stderr=subprocess.DEVNULL, ) stable_functions = set( - re.findall(r"__PyAPI_FUNC\(.*?\)\s*(.*?)\s*\(", preprocesor_output) + re.findall(r"__PyAPI_FUNC\(.*?\)\s*(.*?)\s*\(", preprocessor_output) ) stable_exported_data = set( - re.findall(r"__EXPORT_DATA\((.*?)\)", preprocesor_output) + re.findall(r"__EXPORT_DATA\((.*?)\)", preprocessor_output) ) stable_data = set( - re.findall(r"__PyAPI_DATA\(.*?\)[\s\*\(]*([^);]*)\)?.*;", preprocesor_output) + re.findall(r"__PyAPI_DATA\(.*?\)[\s\*\(]*([^);]*)\)?.*;", preprocessor_output) ) return stable_data | stable_exported_data | stable_functions @@ -600,7 +604,7 @@ def check_private_names(manifest): if name.startswith('_') and not item.abi_only: raise ValueError( f'`{name}` is private (underscore-prefixed) and should be ' - + 'removed from the stable ABI list or or marked `abi_only`') + + 'removed from the stable ABI list or marked `abi_only`') def check_dump(manifest, filename): """Check that manifest.dump() corresponds to the data. diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index 29897425da6c03..a37da2f70757e5 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -14,7 +14,6 @@ from pathlib import Path from urllib.request import urlopen -PACKAGE_NAMES = ("pip",) ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip" WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled" ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8") @@ -97,8 +96,5 @@ def verify_wheel(package_name: str) -> bool: if __name__ == "__main__": - exit_status = 0 - for package_name in PACKAGE_NAMES: - if not verify_wheel(package_name): - exit_status = 1 + exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 781f9a4c8206c8..0c47470a0ecb7a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -7,17 +7,10 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args= set arm32_ssh= +set cmdline_args=%* +set cmdline_args=%cmdline_args:,=#COMMA#% -:CheckOpts -if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts -if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts +call:CheckOpts %cmdline_args% if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecution if "%arm32_ssh%"=="true" goto :Arm32Ssh @@ -49,3 +42,16 @@ echo The test worker should have the SSH agent running. echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh exit /b 127 + +:CheckOpts +set arg="%~1" +if %arg%=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts +if %arg%=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if %arg%=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if NOT %arg%=="" (set regrtest_args=%regrtest_args% %arg:#COMMA#=,%) & shift & goto CheckOpts +goto:eof diff --git a/Tools/c-analyzer/README b/Tools/c-analyzer/README index 86bf1e77e0bfea..41ea13280347a7 100644 --- a/Tools/c-analyzer/README +++ b/Tools/c-analyzer/README @@ -11,9 +11,8 @@ falls into one of several categories: * module state * Python runtime state -The ignored-globals.txt file is organized similarly. Of the different -categories, the last two are problematic and generally should not exist -in the codebase. +Of the different categories, the last two are problematic and +generally should not exist in the codebase. Globals that hold module state (i.e. in Modules/*.c) cause problems when multiple interpreters are in use. For more info, see PEP 3121, @@ -42,4 +41,3 @@ You can also use the more generic tool: If it reports any globals then they should be resolved. If the globals are runtime state then they should be folded into _PyRuntimeState. -Otherwise they should be added to ignored-globals.txt. diff --git a/Tools/c-analyzer/c_analyzer/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py index 171fa25102bffc..b83ffc087a08d8 100644 --- a/Tools/c-analyzer/c_analyzer/__init__.py +++ b/Tools/c-analyzer/c_analyzer/__init__.py @@ -18,7 +18,7 @@ def analyze(filenmes, **kwargs): - results = iter_analyis_results(filenames, **kwargs) + results = iter_analysis_results(filenames, **kwargs) return Analysis.from_results(results) diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index 5695daff67d6bd..c1a8ab3ad2f15d 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -72,6 +72,7 @@ def _ind(text, level=1, edges='both'): long | float | double | + _Complex | void | struct | @@ -121,6 +122,16 @@ def _ind(text, level=1, edges='both'): | (?: signed | unsigned ) # implies int | + (?: + (?: (?: float | double | long\s+double ) \s+ )? + _Complex + ) + | + (?: + _Complex + (?: \s+ (?: float | double | long\s+double ) )? + ) + | (?: (?: (?: signed | unsigned ) \s+ )? (?: (?: long | short ) \s+ )? diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 6ece70c77fd55c..bb5ec5ae087a74 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -7,6 +7,7 @@ FILES_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/ '_testcapimodule.c', + '_testlimitedcapi.c', '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', @@ -15,8 +16,8 @@ # C files in the fhe following directories must not be built with # Py_BUILD_CORE. DIRS_WITHOUT_INTERNAL_CAPI = frozenset(( - # Modules/_testcapi/ - '_testcapi', + '_testcapi', # Modules/_testcapi/ + '_testlimitedcapi', # Modules/_testlimitedcapi/ )) TOOL = 'gcc' diff --git a/Tools/c-analyzer/cpython/_files.py b/Tools/c-analyzer/cpython/_files.py index ee9e46f7e5e95f..a9bf0a1961222a 100644 --- a/Tools/c-analyzer/cpython/_files.py +++ b/Tools/c-analyzer/cpython/_files.py @@ -9,6 +9,7 @@ # Technically, this is covered by "Include/*.h": #'Include/cpython/*.h', 'Include/internal/*.h', + 'Include/internal/mimalloc/**/*.h', 'Modules/**/*.h', 'Modules/**/*.c', 'Objects/**/*.h', diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 4523b2ed5b9fdf..4623f2c8d671bd 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -70,7 +70,6 @@ def clean_lines(text): Python/thread_pthread_stubs.h # only huge constants (safe but parsing is slow) -Modules/_ssl_data.h Modules/_ssl_data_31.h Modules/_ssl_data_300.h Modules/_ssl_data_111.h @@ -84,10 +83,16 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h -Python/abstract_interp_cases.c.h +Python/optimizer_cases.c.h # not actually source Python/bytecodes.c +Python/optimizer_bytecodes.c + +# mimalloc +Objects/mimalloc/*.c +Include/internal/mimalloc/*.h +Include/internal/mimalloc/mimalloc/*.h # @end=conf@ ''') @@ -99,6 +104,7 @@ def clean_lines(text): # The problem with xmlparse.c is that something # has gone wrong where # we handle "maybe inline actual" # in Tools/c-analyzer/c_parser/parser/_global.py. +Modules/expat/internal.h Modules/expat/xmlparse.c ''') @@ -109,6 +115,7 @@ def clean_lines(text): * . * ./Include * ./Include/internal +* ./Include/internal/mimalloc Modules/_decimal/**/*.c Modules/_decimal/libmpdec Modules/_elementtree.c Modules/expat @@ -160,6 +167,7 @@ def clean_lines(text): Objects/stringlib/find.h Objects/stringlib/fastsearch.h Objects/stringlib/partition.h Objects/stringlib/fastsearch.h Objects/stringlib/replace.h Objects/stringlib/fastsearch.h +Objects/stringlib/repr.h Objects/stringlib/fastsearch.h Objects/stringlib/split.h Objects/stringlib/fastsearch.h # @end=tsv@ @@ -314,6 +322,7 @@ def clean_lines(text): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (35_000, 200), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (500_000, 5000), _abs('Python/pystate.c'): (500_000, 5000), @@ -328,7 +337,7 @@ def clean_lines(text): _abs('Python/stdlib_module_names.h'): (5_000, 500), # These large files are currently ignored (see above). - _abs('Modules/_ssl_data.h'): (80_000, 10_000), + _abs('Modules/_ssl_data_31.h'): (80_000, 10_000), _abs('Modules/_ssl_data_300.h'): (80_000, 10_000), _abs('Modules/_ssl_data_111.h'): (80_000, 10_000), _abs('Modules/cjkcodecs/mappings_*.h'): (160_000, 2_000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index aa8ce49ae86376..cb9750a69a632b 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -43,6 +43,7 @@ Objects/enumobject.c - PyReversed_Type - Objects/fileobject.c - PyStdPrinter_Type - Objects/floatobject.c - PyFloat_Type - Objects/frameobject.c - PyFrame_Type - +Objects/frameobject.c - PyFrameLocalsProxy_Type - Objects/funcobject.c - PyClassMethod_Type - Objects/funcobject.c - PyFunction_Type - Objects/funcobject.c - PyStaticMethod_Type - @@ -54,7 +55,6 @@ Objects/genobject.c - _PyAsyncGenASend_Type - Objects/genobject.c - _PyAsyncGenAThrow_Type - Objects/genobject.c - _PyAsyncGenWrappedValue_Type - Objects/genobject.c - _PyCoroWrapper_Type - -Objects/interpreteridobject.c - PyInterpreterID_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - @@ -93,6 +93,7 @@ Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - Objects/typevarobject.c - _PyTypeAlias_Type - +Objects/typevarobject.c - _PyNoDefault_Type - Objects/unicodeobject.c - PyUnicodeIter_Type - Objects/unicodeobject.c - PyUnicode_Type - Objects/weakrefobject.c - _PyWeakref_CallableProxyType - @@ -104,6 +105,7 @@ Python/bltinmodule.c - PyZip_Type - Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - +Python/instruction_sequence.c - _PyInstructionSequence_Type - Python/traceback.c - PyTraceBack_Type - ##----------------------- @@ -189,6 +191,7 @@ Objects/exceptions.c - _PyExc_ProcessLookupError - Objects/exceptions.c - _PyExc_TimeoutError - Objects/exceptions.c - _PyExc_EOFError - Objects/exceptions.c - _PyExc_RuntimeError - +Objects/exceptions.c - _PyExc_PythonFinalizationError - Objects/exceptions.c - _PyExc_RecursionError - Objects/exceptions.c - _PyExc_NotImplementedError - Objects/exceptions.c - _PyExc_NameError - @@ -197,6 +200,7 @@ Objects/exceptions.c - _PyExc_AttributeError - Objects/exceptions.c - _PyExc_SyntaxError - Objects/exceptions.c - _PyExc_IndentationError - Objects/exceptions.c - _PyExc_TabError - +Objects/exceptions.c - _PyExc_IncompleteInputError - Objects/exceptions.c - _PyExc_LookupError - Objects/exceptions.c - _PyExc_IndexError - Objects/exceptions.c - _PyExc_KeyError - @@ -253,6 +257,7 @@ Objects/exceptions.c - PyExc_ProcessLookupError - Objects/exceptions.c - PyExc_TimeoutError - Objects/exceptions.c - PyExc_EOFError - Objects/exceptions.c - PyExc_RuntimeError - +Objects/exceptions.c - PyExc_PythonFinalizationError - Objects/exceptions.c - PyExc_RecursionError - Objects/exceptions.c - PyExc_NotImplementedError - Objects/exceptions.c - PyExc_NameError - @@ -261,6 +266,7 @@ Objects/exceptions.c - PyExc_AttributeError - Objects/exceptions.c - PyExc_SyntaxError - Objects/exceptions.c - PyExc_IndentationError - Objects/exceptions.c - PyExc_TabError - +Objects/exceptions.c - PyExc_IncompleteInputError - Objects/exceptions.c - PyExc_LookupError - Objects/exceptions.c - PyExc_IndexError - Objects/exceptions.c - PyExc_KeyError - @@ -290,10 +296,18 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - +Python/crossinterp_exceptions.h - _PyExc_InterpreterError - +Python/crossinterp_exceptions.h - _PyExc_InterpreterNotFoundError - +Python/crossinterp_exceptions.h - PyExc_InterpreterError - +Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons +Modules/_datetimemodule.c - zero_delta - +Modules/_datetimemodule.c - utc_timezone - +Modules/_datetimemodule.c - capi - +Modules/_datetimemodule.c - _globals - Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - Objects/dictobject.c - empty_keys_struct - @@ -302,6 +316,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Objects/typevarobject.c - _Py_NoDefaultStruct - Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - @@ -328,6 +343,7 @@ Modules/_testclinic.c - DeprStarNew - Modules/_testclinic.c - DeprKwdInit - Modules/_testclinic.c - DeprKwdInitNoInline - Modules/_testclinic.c - DeprKwdNew - +Modules/_testclinic.c - TestClass - ################################## @@ -342,37 +358,6 @@ Modules/_testclinic.c - DeprKwdNew - ##----------------------- ## static types -Modules/_ctypes/_ctypes.c - PyCArrayType_Type - -Modules/_ctypes/_ctypes.c - PyCArray_Type - -Modules/_ctypes/_ctypes.c - PyCData_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtrType_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtr_Type - -Modules/_ctypes/_ctypes.c - PyCPointerType_Type - -Modules/_ctypes/_ctypes.c - PyCPointer_Type - -Modules/_ctypes/_ctypes.c - PyCSimpleType_Type - -Modules/_ctypes/_ctypes.c - PyCStructType_Type - -Modules/_ctypes/_ctypes.c - Simple_Type - -Modules/_ctypes/_ctypes.c - Struct_Type - -Modules/_ctypes/_ctypes.c - UnionType_Type - -Modules/_ctypes/_ctypes.c - Union_Type - -Modules/_ctypes/callproc.c - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArrayType_Type - -Modules/_ctypes/ctypes.h - PyCArray_Type - -Modules/_ctypes/ctypes.h - PyCData_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtrType_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtr_Type - -Modules/_ctypes/ctypes.h - PyCPointerType_Type - -Modules/_ctypes/ctypes.h - PyCPointer_Type - -Modules/_ctypes/ctypes.h - PyCSimpleType_Type - -Modules/_ctypes/ctypes.h - PyCStgDict_Type - -Modules/_ctypes/ctypes.h - PyCStructType_Type - -Modules/_ctypes/ctypes.h - PyExc_ArgError - -Modules/_ctypes/ctypes.h - _ctypes_conversion_encoding - -Modules/_ctypes/ctypes.h - _ctypes_conversion_errors - -Modules/_ctypes/ctypes.h - _ctypes_ptrtype_cache - -Modules/_ctypes/ctypes.h - basespec_string - -Modules/_ctypes/stgdict.c - PyCStgDict_Type - Modules/_cursesmodule.c - PyCursesWindow_Type - Modules/_datetimemodule.c - PyDateTime_DateTimeType - Modules/_datetimemodule.c - PyDateTime_DateType - @@ -386,6 +371,7 @@ Modules/xxmodule.c - Str_Type - Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - +Modules/_testcapi/monitoring.c - PyCodeLike_Type - ##----------------------- ## non-static types - initialized once @@ -397,32 +383,18 @@ Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - ## exception types -Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_tkinter.c - Tkinter_TclError - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - -##----------------------- -## cached - initialized once - -## manually cached PyUnicodeOjbect -Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - -Modules/_ctypes/_ctypes.c CreateSwappedType suffix - - ##----------------------- ## other ## initialized once -Modules/_ctypes/_ctypes.c - _unpickle - -Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype cache - Modules/_cursesmodule.c - ModDict - -Modules/_datetimemodule.c datetime_strptime module - ## state -Modules/_ctypes/_ctypes.c - _ctypes_ptrtype_cache - -Modules/_ctypes/_ctypes.c - global_state - -Modules/_ctypes/ctypes.h - global_state - Modules/_datetimemodule.c - _datetime_global_state - Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - @@ -473,3 +445,4 @@ Modules/readline.c - sigwinch_ohandler - Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - +Modules/rotatingtree.c - random_mutex - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 0ba7ab15ce89b8..63b640e465ac6b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -164,7 +164,8 @@ Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - # explicitly protected, internal-only -Modules/_xxinterpchannelsmodule.c - _globals - +Modules/_interpchannelsmodule.c - _globals - +Modules/_interpqueuesmodule.c - _globals - # set once during module init Modules/_decimal/_decimal.c - minalloc_is_set - @@ -197,6 +198,7 @@ Python/pystate.c - _Py_tss_tstate - Include/internal/pycore_blocks_output_buffer.h - BUFFER_BLOCK_SIZE - Modules/_csv.c - quote_styles - +Modules/_ctypes/_ctypes.c - _ctypesmodule - Modules/_ctypes/cfield.c - ffi_type_double - Modules/_ctypes/cfield.c - ffi_type_float - Modules/_ctypes/cfield.c - ffi_type_longdouble - @@ -215,6 +217,7 @@ Modules/_datetimemodule.c - max_fold_seconds - Modules/_datetimemodule.c datetime_isoformat specs - Modules/_datetimemodule.c parse_hh_mm_ss_ff correction - Modules/_datetimemodule.c time_isoformat specs - +Modules/_datetimemodule.c - capi_types - Modules/_decimal/_decimal.c - cond_map_template - Modules/_decimal/_decimal.c - dec_signal_string - Modules/_decimal/_decimal.c - dflt_ctx - @@ -244,11 +247,11 @@ Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - -Modules/_xxinterpchannelsmodule.c - _channelid_end_send - +Modules/_interpchannelsmodule.c - _channelid_end_recv - +Modules/_interpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - -Modules/_xxsubinterpretersmodule.c - no_exception - +Modules/_interpretersmodule.c - no_exception - Modules/arraymodule.c - descriptors - Modules/arraymodule.c - emptybuf - Modules/cjkcodecs/_codecs_cn.c - _mapping_list - @@ -324,7 +327,8 @@ Objects/obmalloc.c - _PyMem_Debug - Objects/obmalloc.c - _PyMem_Raw - Objects/obmalloc.c - _PyObject - Objects/obmalloc.c - last_final_leaks - -Objects/obmalloc.c - usedpools - +Objects/obmalloc.c - obmalloc_state_main - +Objects/obmalloc.c - obmalloc_state_initialized - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - Objects/unicodeobject.c - stripfuncnames - @@ -341,7 +345,7 @@ Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - Python/codecs.c - ucnhash_capi - -Python/codecs.c _PyCodecRegistry_Init methods - +Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LABEL - Python/compile.c - NO_LOCATION - Python/dynload_shlib.c - _PyImport_DynLoadFiletab - @@ -359,8 +363,11 @@ Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - +Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - @@ -374,18 +381,19 @@ Python/sysmodule.c - perf_map_state - Python/sysmodule.c - _PySys_ImplCacheTag - Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - -Python/optimizer.c - DefaultOptimizer_Type - -Python/optimizer.c - CounterExecutor_Type - -Python/optimizer.c - CounterOptimizer_Type - -Python/optimizer.c - UOpExecutor_Type - -Python/optimizer.c - UOpOptimizer_Type - +Python/optimizer.c - _PyDefaultOptimizer_Type - +Python/optimizer.c - _PyCounterExecutor_Type - +Python/optimizer.c - _PyCounterOptimizer_Type - +Python/optimizer.c - _PyUOpExecutor_Type - +Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - +Python/optimizer.c - _ColdExit_Type - +Python/optimizer.c - Py_FatalErrorExecutor - +Python/optimizer.c - EMPTY_FILTER - ##----------------------- ## test code -Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - -Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - Modules/_ctypes/_ctypes_test.c - _xxx_lib - Modules/_ctypes/_ctypes_test.c - an_integer - Modules/_ctypes/_ctypes_test.c - bottom - @@ -393,7 +401,6 @@ Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - Modules/_ctypes/_ctypes_test.c - left - -Modules/_ctypes/_ctypes_test.c - module_methods - Modules/_ctypes/_ctypes_test.c - my_eggs - Modules/_ctypes/_ctypes_test.c - my_spams - Modules/_ctypes/_ctypes_test.c - right - @@ -436,7 +443,6 @@ Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - -Modules/_testcapi/unicode.c - _testcapimodule - Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - @@ -530,7 +536,7 @@ Modules/_testcapimodule.c make_exception_with_doc kwlist - Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - Modules/_testcapimodule.c test_capsule buffer - -Modules/_testcapimodule.c test_empty_argparse kwlist - +Modules/_testcapimodule.c getargs_empty kwlist - Modules/_testcapimodule.c test_structmembers_new keywords - Modules/_testcapimodule.c getargs_s_hash_int keywords - Modules/_testcapimodule.c - g_dict_watch_events - @@ -565,7 +571,6 @@ Modules/_testmultiphase.c - def_nonascii_latin - Modules/_testmultiphase.c - def_nonmodule - Modules/_testmultiphase.c - def_nonmodule_with_exec_slots - Modules/_testmultiphase.c - def_nonmodule_with_methods - -Modules/_testmultiphase.c - imp_dummy_def - Modules/_testmultiphase.c - main_def - Modules/_testmultiphase.c - main_slots - Modules/_testmultiphase.c - meth_state_access_slots - @@ -589,6 +594,7 @@ Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - RE_FLAG_DEBUG - Modules/_xxtestfuzz/fuzzer.c - ast_literal_eval_method - +Modules/_xxtestfuzz/fuzzer.c - bytesio_type - Modules/_xxtestfuzz/fuzzer.c - compiled_patterns - Modules/_xxtestfuzz/fuzzer.c - csv_error - Modules/_xxtestfuzz/fuzzer.c - csv_module - @@ -598,12 +604,17 @@ Modules/_xxtestfuzz/fuzzer.c - re_compile_method - Modules/_xxtestfuzz/fuzzer.c - re_error_exception - Modules/_xxtestfuzz/fuzzer.c - struct_error - Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - +Modules/_xxtestfuzz/fuzzer.c - xmlparser_type - +Modules/_xxtestfuzz/fuzzer.c - pycompile_scratch - +Modules/_xxtestfuzz/fuzzer.c - start_vals - +Modules/_xxtestfuzz/fuzzer.c - optimize_vals - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_MATCH_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput STRUCT_UNPACK_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED - +Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput ELEMENTTREE_PARSEWHOLE_INITIALIZED - ##----------------------- ## the analyzer should have ignored these @@ -727,5 +738,7 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -Python/optimizer_analysis.c - _Py_PartitionRootNode_Type - -Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - +Modules/clinic/md5module.c.h _md5_md5 _keywords - +Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - +Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - +Objects/object.c - constants static PyObject*[] diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index ed802e44f31ad5..fb512c4646b851 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -5,16 +5,33 @@ Documentation for the instruction definitions in `Python/bytecodes.c` What's currently here: +- `analyzer.py`: code for converting `AST` generated by `Parser` + to more high-level structure for easier interaction - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parsing.py`: Parser for instruction definition DSL; main class `Parser` -- `generate_cases.py`: driver script to read `Python/bytecodes.c` and +- `parsing.py`: Parser for instruction definition DSL; main class: `Parser` +- `parser.py` helper for interactions with `parsing.py` +- `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `analysis.py`: `Analyzer` class used to read the input files -- `flags.py`: abstractions related to metadata flags for instructions -- `formatting.py`: `Formatter` class used to write the output files -- `instructions.py`: classes to analyze and write instructions -- `stacking.py`: code to handle generalized stack effects +- `optimizer_generator.py`: reads `Python/bytecodes.c` and + `Python/optimizer_bytecodes.c` and writes + `Python/optimizer_cases.c.h` +- `stack.py`: code to handle generalized stack effects +- `cwriter.py`: code which understands tokens and how to format C code; + main class: `CWriter` +- `generators_common.py`: helpers for generators +- `opcode_id_generator.py`: generate a list of opcodes and write them to + `Include/opcode_ids.h` +- `opcode_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_opcode_metadata.h` +- `py_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Lib/_opcode_metadata.py` +- `target_generator.py`: generate targets for computed goto dispatch and + write them to `Python/opcode_targets.h` +- `uop_id_generator.py`: generate a list of uop IDs and write them to + `Include/internal/pycore_uop_ids.h` +- `uop_metadata_generator.py`: reads the instruction definitions and + write the metadata to `Include/internal/pycore_uop_metadata.h` Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py deleted file mode 100644 index b2fa0205bea342..00000000000000 --- a/Tools/cases_generator/analysis.py +++ /dev/null @@ -1,481 +0,0 @@ -import re -import sys -import typing - -from _typing_backports import assert_never -from flags import InstructionFlags, variable_used -from formatting import prettify_filename, UNUSED -from instructions import ( - ActiveCacheEffect, - Component, - Instruction, - InstructionOrCacheEffect, - MacroInstruction, - MacroParts, - PseudoInstruction, -) -import parsing -from parsing import StackEffect - -BEGIN_MARKER = "// BEGIN BYTECODES //" -END_MARKER = "// END BYTECODES //" - -RESERVED_WORDS = { - "co_consts": "Use FRAME_CO_CONSTS.", - "co_names": "Use FRAME_CO_NAMES.", -} - -RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" - - -class Analyzer: - """Parse input, analyze it, and write to output.""" - - input_filenames: list[str] - errors: int = 0 - warnings: int = 0 - - def __init__(self, input_filenames: list[str]): - self.input_filenames = input_filenames - - def message(self, msg: str, node: parsing.Node) -> None: - lineno = 0 - filename = "" - if context := node.context: - filename = context.owner.filename - # Use line number of first non-comment in the node - for token in context.owner.tokens[context.begin : context.end]: - lineno = token.line - if token.kind != "COMMENT": - break - print(f"{filename}:{lineno}: {msg}", file=sys.stderr) - - def error(self, msg: str, node: parsing.Node) -> None: - self.message("error: " + msg, node) - self.errors += 1 - - def warning(self, msg: str, node: parsing.Node) -> None: - self.message("warning: " + msg, node) - self.warnings += 1 - - def note(self, msg: str, node: parsing.Node) -> None: - self.message("note: " + msg, node) - - everything: list[ - parsing.InstDef - | parsing.Macro - | parsing.Pseudo - ] - instrs: dict[str, Instruction] # Includes ops - macros: dict[str, parsing.Macro] - macro_instrs: dict[str, MacroInstruction] - families: dict[str, parsing.Family] - pseudos: dict[str, parsing.Pseudo] - pseudo_instrs: dict[str, PseudoInstruction] - - def parse(self) -> None: - """Parse the source text. - - We only want the parser to see the stuff between the - begin and end markers. - """ - - self.everything = [] - self.instrs = {} - self.macros = {} - self.families = {} - self.pseudos = {} - - instrs_idx: dict[str, int] = dict() - - for filename in self.input_filenames: - self.parse_file(filename, instrs_idx) - - files = " + ".join(self.input_filenames) - n_instrs = len(set(self.instrs) & set(self.macros)) - n_ops = len(self.instrs) - n_instrs - print( - f"Read {n_instrs} instructions, {n_ops} ops, " - f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " - f"and {len(self.families)} families from {files}", - file=sys.stderr, - ) - - def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: - with open(filename) as file: - src = file.read() - - psr = parsing.Parser(src, filename=prettify_filename(filename)) - - # Skip until begin marker - while tkn := psr.next(raw=True): - if tkn.text == BEGIN_MARKER: - break - else: - raise psr.make_syntax_error( - f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" - ) - start = psr.getpos() - - # Find end marker, then delete everything after it - while tkn := psr.next(raw=True): - if tkn.text == END_MARKER: - break - del psr.tokens[psr.getpos() - 1 :] - - # Parse from start - psr.setpos(start) - thing: parsing.Node | None - thing_first_token = psr.peek() - while thing := psr.definition(): - thing = typing.cast( - parsing.InstDef | parsing.Macro | parsing.Pseudo | parsing.Family, thing - ) - if ws := [w for w in RESERVED_WORDS if variable_used(thing, w)]: - self.error( - f"'{ws[0]}' is a reserved word. {RESERVED_WORDS[ws[0]]}", thing - ) - - match thing: - case parsing.InstDef(name=name): - macro: parsing.Macro | None = None - if thing.kind == "inst" and not thing.override: - macro = parsing.Macro(name, [parsing.OpName(name)]) - if name in self.instrs: - if not thing.override: - raise psr.make_syntax_error( - f"Duplicate definition of '{name}' @ {thing.context} " - f"previous definition @ {self.instrs[name].inst.context}", - thing_first_token, - ) - self.everything[instrs_idx[name]] = thing - if name not in self.instrs and thing.override: - raise psr.make_syntax_error( - f"Definition of '{name}' @ {thing.context} is supposed to be " - "an override but no previous definition exists.", - thing_first_token, - ) - self.instrs[name] = Instruction(thing) - instrs_idx[name] = len(self.everything) - self.everything.append(thing) - if macro is not None: - self.macros[macro.name] = macro - self.everything.append(macro) - case parsing.Macro(name): - self.macros[name] = thing - self.everything.append(thing) - case parsing.Family(name): - self.families[name] = thing - case parsing.Pseudo(name): - self.pseudos[name] = thing - self.everything.append(thing) - case _: - assert_never(thing) - if not psr.eof(): - raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") - - def analyze(self) -> None: - """Analyze the inputs. - - Raises SystemExit if there is an error. - """ - self.analyze_macros_and_pseudos() - self.map_families() - self.mark_predictions() - self.check_families() - - def mark_predictions(self) -> None: - """Mark the instructions that need PREDICTED() labels.""" - # Start with family heads - for family in self.families.values(): - if family.name in self.instrs: - self.instrs[family.name].predicted = True - if family.name in self.macro_instrs: - self.macro_instrs[family.name].predicted = True - # Also look for GO_TO_INSTRUCTION() calls - for instr in self.instrs.values(): - targets: set[str] = set() - for line in instr.block_text: - if m := re.match(RE_GO_TO_INSTR, line): - targets.add(m.group(1)) - for target in targets: - if target_instr := self.instrs.get(target): - target_instr.predicted = True - if target_macro := self.macro_instrs.get(target): - target_macro.predicted = True - if not target_instr and not target_macro: - self.error( - f"Unknown instruction {target!r} predicted in {instr.name!r}", - instr.inst, # TODO: Use better location - ) - - def map_families(self) -> None: - """Link instruction names back to their family, if they have one.""" - for family in self.families.values(): - for member in [family.name] + family.members: - if member_instr := self.instrs.get(member): - if ( - member_instr.family is not family - and member_instr.family is not None - ): - self.error( - f"Instruction {member} is a member of multiple families " - f"({member_instr.family.name}, {family.name}).", - family, - ) - else: - member_instr.family = family - if member_mac := self.macro_instrs.get(member): - assert member_mac.family is None, (member, member_mac.family.name) - member_mac.family = family - if not member_instr and not member_mac: - self.error( - f"Unknown instruction {member!r} referenced in family {family.name!r}", - family, - ) - # A sanctioned exception: - # This opcode is a member of the family but it doesn't pass the checks. - if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"): - mac.family = self.families.get("BINARY_OP") - - def check_families(self) -> None: - """Check each family: - - - Must have at least 2 members (including head) - - Head and all members must be known instructions - - Head and all members must have the same cache, input and output effects - """ - for family in self.families.values(): - if family.name not in self.macro_instrs and family.name not in self.instrs: - self.error( - f"Family {family.name!r} has unknown instruction {family.name!r}", - family, - ) - members = [ - member - for member in family.members - if member in self.instrs or member in self.macro_instrs - ] - if members != family.members: - unknown = set(family.members) - set(members) - self.error( - f"Family {family.name!r} has unknown members: {unknown}", family - ) - expected_effects = self.effect_counts(family.name) - for member in members: - member_effects = self.effect_counts(member) - if member_effects != expected_effects: - self.error( - f"Family {family.name!r} has inconsistent " - f"(cache, input, output) effects:\n" - f" {family.name} = {expected_effects}; " - f"{member} = {member_effects}", - family, - ) - - def effect_counts(self, name: str) -> tuple[int, int, int]: - if mac := self.macro_instrs.get(name): - cache = mac.cache_offset - input, output = 0, 0 - for part in mac.parts: - if isinstance(part, Component): - # A component may pop what the previous component pushed, - # so we offset the input/output counts by that. - delta_i = len(part.instr.input_effects) - delta_o = len(part.instr.output_effects) - offset = min(delta_i, output) - input += delta_i - offset - output += delta_o - offset - else: - assert False, f"Unknown instruction {name!r}" - return cache, input, output - - def analyze_macros_and_pseudos(self) -> None: - """Analyze each macro and pseudo instruction.""" - self.macro_instrs = {} - self.pseudo_instrs = {} - for name, macro in self.macros.items(): - self.macro_instrs[name] = mac = self.analyze_macro(macro) - self.check_macro_consistency(mac) - for name, pseudo in self.pseudos.items(): - self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) - - # TODO: Merge with similar code in stacking.py, write_components() - def check_macro_consistency(self, mac: MacroInstruction) -> None: - def get_var_names(instr: Instruction) -> dict[str, StackEffect]: - vars: dict[str, StackEffect] = {} - for eff in instr.input_effects + instr.output_effects: - if eff.name == UNUSED: - continue - if eff.name in vars: - if vars[eff.name] != eff: - self.error( - f"Instruction {instr.name!r} has " - f"inconsistent type/cond/size for variable " - f"{eff.name!r}: {vars[eff.name]} vs {eff}", - instr.inst, - ) - else: - vars[eff.name] = eff - return vars - - all_vars: dict[str, StackEffect] = {} - # print("Checking", mac.name) - prevop: Instruction | None = None - for part in mac.parts: - if not isinstance(part, Component): - continue - vars = get_var_names(part.instr) - # print(" //", part.instr.name, "//", vars) - for name, eff in vars.items(): - if name in all_vars: - if all_vars[name] != eff: - self.error( - f"Macro {mac.name!r} has " - f"inconsistent type/cond/size for variable " - f"{name!r}: " - f"{all_vars[name]} vs {eff} in {part.instr.name!r}", - mac.macro, - ) - else: - all_vars[name] = eff - if prevop is not None: - pushes = list(prevop.output_effects) - pops = list(reversed(part.instr.input_effects)) - copies: list[tuple[StackEffect, StackEffect]] = [] - while pushes and pops and pushes[-1] == pops[0]: - src, dst = pushes.pop(), pops.pop(0) - if src.name == dst.name or dst.name == UNUSED: - continue - copies.append((src, dst)) - reads = set(copy[0].name for copy in copies) - writes = set(copy[1].name for copy in copies) - if reads & writes: - self.error( - f"Macro {mac.name!r} has conflicting copies " - f"(source of one copy is destination of another): " - f"{reads & writes}", - mac.macro, - ) - prevop = part.instr - - def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: - components = self.check_macro_components(macro) - parts: MacroParts = [] - flags = InstructionFlags.newEmpty() - offset = 0 - for component in components: - match component: - case parsing.CacheEffect() as ceffect: - parts.append(ceffect) - offset += ceffect.size - case Instruction() as instr: - part, offset = self.analyze_instruction(instr, offset) - parts.append(part) - if instr.name != "_SET_IP": - # _SET_IP in a macro is a no-op in Tier 1 - flags.add(instr.instr_flags) - case _: - assert_never(component) - format = "IB" if flags.HAS_ARG_FLAG else "IX" - if offset: - format += "C" + "0" * (offset - 1) - return MacroInstruction(macro.name, format, flags, macro, parts, offset) - - def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: - targets = [self.instrs[target] for target in pseudo.targets] - assert targets - # Make sure the targets have the same fmt - fmts = list(set([t.instr_fmt for t in targets])) - assert len(fmts) == 1 - ignored_flags = {"HAS_EVAL_BREAK_FLAG", "HAS_DEOPT_FLAG", "HAS_ERROR_FLAG"} - assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1 - return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) - - def analyze_instruction( - self, instr: Instruction, offset: int - ) -> tuple[Component, int]: - active_effects: list[ActiveCacheEffect] = [] - for ceffect in instr.cache_effects: - if ceffect.name != UNUSED: - active_effects.append(ActiveCacheEffect(ceffect, offset)) - offset += ceffect.size - return ( - Component(instr, active_effects), - offset, - ) - - def check_macro_components( - self, macro: parsing.Macro - ) -> list[InstructionOrCacheEffect]: - components: list[InstructionOrCacheEffect] = [] - for uop in macro.uops: - match uop: - case parsing.OpName(name): - if name not in self.instrs: - self.error(f"Unknown instruction {name!r}", macro) - else: - components.append(self.instrs[name]) - case parsing.CacheEffect(): - components.append(uop) - case _: - assert_never(uop) - return components - - def report_non_viable_uops(self, jsonfile: str) -> None: - print("The following ops are not viable uops:") - skips = { - "CACHE", - "RESERVED", - "INTERPRETER_EXIT", - "JUMP_BACKWARD", - "LOAD_FAST_LOAD_FAST", - "LOAD_CONST_LOAD_FAST", - "STORE_FAST_STORE_FAST", - "_BINARY_OP_INPLACE_ADD_UNICODE", - "POP_JUMP_IF_TRUE", - "POP_JUMP_IF_FALSE", - "_ITER_JUMP_LIST", - "_ITER_JUMP_TUPLE", - "_ITER_JUMP_RANGE", - } - try: - # Secret feature: if bmraw.json exists, print and sort by execution count - counts = load_execution_counts(jsonfile) - except FileNotFoundError as err: - counts = {} - non_viable = [ - instr - for instr in self.instrs.values() - if instr.name not in skips - and not instr.name.startswith("INSTRUMENTED_") - and not instr.is_viable_uop() - ] - non_viable.sort(key=lambda instr: (-counts.get(instr.name, 0), instr.name)) - for instr in non_viable: - if instr.name in counts: - scount = f"{counts[instr.name]:,}" - else: - scount = "" - print(f" {scount:>15} {instr.name:<35}", end="") - if instr.name in self.families: - print(" (unspecialized)", end="") - elif instr.family is not None: - print(f" (specialization of {instr.family.name})", end="") - print() - - -def load_execution_counts(jsonfile: str) -> dict[str, int]: - import json - - with open(jsonfile) as f: - jsondata = json.load(f) - - # Look for keys like "opcode[LOAD_FAST].execution_count" - prefix = "opcode[" - suffix = "].execution_count" - res: dict[str, int] = {} - for key, value in jsondata.items(): - if key.startswith(prefix) and key.endswith(suffix): - res[key[len(prefix) : -len(suffix)]] = value - return res diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py new file mode 100644 index 00000000000000..ec365bad3992d5 --- /dev/null +++ b/Tools/cases_generator/analyzer.py @@ -0,0 +1,909 @@ +from dataclasses import dataclass +import lexer +import parser +import re +from typing import Optional + + +@dataclass +class Properties: + escapes: bool + error_with_pop: bool + error_without_pop: bool + deopts: bool + oparg: bool + jumps: bool + eval_breaker: bool + ends_with_eval_breaker: bool + needs_this: bool + always_exits: bool + stores_sp: bool + uses_co_consts: bool + uses_co_names: bool + uses_locals: bool + has_free: bool + side_exit: bool + pure: bool + tier: int | None = None + oparg_and_1: bool = False + const_oparg: int = -1 + + def dump(self, indent: str) -> None: + print(indent, end="") + text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + print(indent, text, sep="") + + @staticmethod + def from_list(properties: list["Properties"]) -> "Properties": + return Properties( + escapes=any(p.escapes for p in properties), + error_with_pop=any(p.error_with_pop for p in properties), + error_without_pop=any(p.error_without_pop for p in properties), + deopts=any(p.deopts for p in properties), + oparg=any(p.oparg for p in properties), + jumps=any(p.jumps for p in properties), + eval_breaker=any(p.eval_breaker for p in properties), + ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), + needs_this=any(p.needs_this for p in properties), + always_exits=any(p.always_exits for p in properties), + stores_sp=any(p.stores_sp for p in properties), + uses_co_consts=any(p.uses_co_consts for p in properties), + uses_co_names=any(p.uses_co_names for p in properties), + uses_locals=any(p.uses_locals for p in properties), + has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), + pure=all(p.pure for p in properties), + ) + + @property + def infallible(self) -> bool: + return not self.error_with_pop and not self.error_without_pop + + + +SKIP_PROPERTIES = Properties( + escapes=False, + error_with_pop=False, + error_without_pop=False, + deopts=False, + oparg=False, + jumps=False, + eval_breaker=False, + ends_with_eval_breaker=False, + needs_this=False, + always_exits=False, + stores_sp=False, + uses_co_consts=False, + uses_co_names=False, + uses_locals=False, + has_free=False, + side_exit=False, + pure=False, +) + + +@dataclass +class Skip: + "Unused cache entry" + size: int + + @property + def name(self) -> str: + return f"unused/{self.size}" + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + +@dataclass +class StackItem: + name: str + type: str | None + condition: str | None + size: str + peek: bool = False + + def __str__(self) -> str: + cond = f" if ({self.condition})" if self.condition else "" + size = f"[{self.size}]" if self.size else "" + type = "" if self.type is None else f"{self.type} " + return f"{type}{self.name}{size}{cond} {self.peek}" + + def is_array(self) -> bool: + return self.size != "" + + def get_size(self) -> str: + return self.size if self.size else "1" + +@dataclass +class StackEffect: + inputs: list[StackItem] + outputs: list[StackItem] + + def __str__(self) -> str: + return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + + +@dataclass +class CacheEntry: + name: str + size: int + + def __str__(self) -> str: + return f"{self.name}/{self.size}" + + +@dataclass +class Uop: + name: str + context: parser.Context | None + annotations: list[str] + stack: StackEffect + caches: list[CacheEntry] + body: list[lexer.Token] + properties: Properties + _size: int = -1 + implicitly_created: bool = False + replicated = 0 + replicates : "Uop | None" = None + + def dump(self, indent: str) -> None: + print( + indent, self.name, ", ".join(self.annotations) if self.annotations else "" + ) + print(indent, self.stack, ", ".join([str(c) for c in self.caches])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + if self._size < 0: + self._size = sum(c.size for c in self.caches) + return self._size + + def why_not_viable(self) -> str | None: + if self.name == "_SAVE_RETURN_OFFSET": + return None # Adjusts next_instr, but only in tier 1 code + if "INSTRUMENTED" in self.name: + return "is instrumented" + if "replaced" in self.annotations: + return "is replaced" + if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): + return "has tier 1 control flow" + if self.properties.needs_this: + return "uses the 'this_instr' variable" + if len([c for c in self.caches if c.name != "unused"]) > 1: + return "has unused cache entries" + if self.properties.error_with_pop and self.properties.error_without_pop: + return "has both popping and not-popping errors" + if self.properties.eval_breaker: + if self.properties.error_with_pop or self.properties.error_without_pop: + return "has error handling and eval-breaker check" + if self.properties.side_exit: + return "exits and eval-breaker check" + if self.properties.deopts: + return "deopts and eval-breaker check" + return None + + def is_viable(self) -> bool: + return self.why_not_viable() is None + + def is_super(self) -> bool: + for tkn in self.body: + if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1": + return True + return False + + +Part = Uop | Skip + + +@dataclass +class Instruction: + name: str + parts: list[Part] + _properties: Properties | None + is_target: bool = False + family: Optional["Family"] = None + opcode: int = -1 + + @property + def properties(self) -> Properties: + if self._properties is None: + self._properties = self._compute_properties() + return self._properties + + def _compute_properties(self) -> Properties: + return Properties.from_list([part.properties for part in self.parts]) + + def dump(self, indent: str) -> None: + print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + return 1 + sum(part.size for part in self.parts) + + def is_super(self) -> bool: + if len(self.parts) != 1: + return False + uop = self.parts[0] + if isinstance(uop, Uop): + return uop.is_super() + else: + return False + + +@dataclass +class PseudoInstruction: + name: str + stack: StackEffect + targets: list[Instruction] + flags: list[str] + opcode: int = -1 + + def dump(self, indent: str) -> None: + print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) + + @property + def properties(self) -> Properties: + return Properties.from_list([i.properties for i in self.targets]) + + +@dataclass +class Family: + name: str + size: str + members: list[Instruction] + + def dump(self, indent: str) -> None: + print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) + + +@dataclass +class Analysis: + instructions: dict[str, Instruction] + uops: dict[str, Uop] + families: dict[str, Family] + pseudos: dict[str, PseudoInstruction] + opmap: dict[str, int] + have_arg: int + min_instrumented: int + + +def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: + # To do -- support file and line output + # Construct a SyntaxError instance from message and token + return lexer.make_syntax_error(message, tkn.filename, tkn.line, tkn.column, "") + + +def override_error( + name: str, + context: parser.Context | None, + prev_context: parser.Context | None, + token: lexer.Token, +) -> SyntaxError: + return analysis_error( + f"Duplicate definition of '{name}' @ {context} " + f"previous definition @ {prev_context}", + token, + ) + + +def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: + cond = item.cond + if replace_op_arg_1 and OPARG_AND_1.match(item.cond): + cond = replace_op_arg_1 + return StackItem( + item.name, item.type, cond, item.size + ) + +def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None) -> StackEffect: + inputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] + for input, output in zip(inputs, outputs): + if input.name == output.name: + input.peek = output.peek = True + return StackEffect(inputs, outputs) + + +def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: + caches: list[parser.CacheEffect] = [ + i for i in inputs if isinstance(i, parser.CacheEffect) + ] + for cache in caches: + if cache.name == "unused": + raise analysis_error( + "Unused cache entry in op. Move to enclosing macro.", cache.tokens[0] + ) + return [CacheEntry(i.name, int(i.size)) for i in caches] + + +def variable_used(node: parser.InstDef, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + ) + +def tier_variable(node: parser.InstDef) -> int | None: + """Determine whether a tier variable is used in a node.""" + for token in node.tokens: + if token.kind == "ANNOTATION": + if token.text == "specializing": + return 1 + if re.fullmatch(r"tier\d", token.text): + return int(token.text[-1]) + return None + +def has_error_with_pop(op: parser.InstDef) -> bool: + return ( + variable_used(op, "ERROR_IF") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + +def has_error_without_pop(op: parser.InstDef) -> bool: + return ( + variable_used(op, "ERROR_NO_POP") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + + +NON_ESCAPING_FUNCTIONS = ( + "PyStackRef_FromPyObjectSteal", + "PyStackRef_AsPyObjectBorrow", + "PyStackRef_AsPyObjectSteal", + "PyStackRef_CLOSE", + "PyStackRef_DUP", + "PyStackRef_CLEAR", + "PyStackRef_IsNull", + "PyStackRef_TYPE", + "PyStackRef_False", + "PyStackRef_True", + "PyStackRef_None", + "PyStackRef_Is", + "PyStackRef_FromPyObjectNew", + "PyStackRef_AsPyObjectNew", + "PyStackRef_FromPyObjectImmortal", + "Py_INCREF", + "_PyManagedDictPointer_IsValues", + "_PyObject_GetManagedDict", + "_PyObject_ManagedDictPointer", + "_PyObject_InlineValues", + "_PyDictValues_AddToInsertionOrder", + "Py_DECREF", + "Py_XDECREF", + "_Py_DECREF_SPECIALIZED", + "DECREF_INPUTS_AND_REUSE_FLOAT", + "PyUnicode_Append", + "_PyLong_IsZero", + "Py_SIZE", + "Py_TYPE", + "PyList_GET_ITEM", + "PyList_SET_ITEM", + "PyTuple_GET_ITEM", + "PyList_GET_SIZE", + "PyTuple_GET_SIZE", + "Py_ARRAY_LENGTH", + "Py_Unicode_GET_LENGTH", + "PyUnicode_READ_CHAR", + "_Py_SINGLETON", + "PyUnicode_GET_LENGTH", + "_PyLong_IsCompact", + "_PyLong_IsNonNegativeCompact", + "_PyLong_CompactValue", + "_PyLong_DigitCount", + "_Py_NewRef", + "_Py_IsImmortal", + "PyLong_FromLong", + "_Py_STR", + "_PyLong_Add", + "_PyLong_Multiply", + "_PyLong_Subtract", + "Py_NewRef", + "_PyList_ITEMS", + "_PyTuple_ITEMS", + "_PyList_AppendTakeRef", + "_Py_atomic_load_uintptr_relaxed", + "_PyFrame_GetCode", + "_PyThreadState_HasStackSpace", + "_PyUnicode_Equal", + "_PyFrame_SetStackPointer", + "_PyType_HasFeature", + "PyUnicode_Concat", + "PySlice_New", + "_Py_LeaveRecursiveCallPy", + "CALL_STAT_INC", + "STAT_INC", + "maybe_lltrace_resume_frame", + "_PyUnicode_JoinArray", + "_PyEval_FrameClearAndPop", + "_PyFrame_StackPush", + "PyCell_New", + "PyFloat_AS_DOUBLE", + "_PyFrame_PushUnchecked", + "Py_FatalError", + "STACKREFS_TO_PYOBJECTS", + "STACKREFS_TO_PYOBJECTS_CLEANUP", + "CONVERSION_FAILED", + "_PyList_FromArraySteal", + "_PyTuple_FromArraySteal", + "_PyTuple_FromStackRefSteal", +) + +ESCAPING_FUNCTIONS = ( + "import_name", + "import_from", +) + + +def makes_escaping_api_call(instr: parser.InstDef) -> bool: + if "CALL_INTRINSIC" in instr.name: + return True + if instr.name == "_BINARY_OP": + return True + tkns = iter(instr.tokens) + for tkn in tkns: + if tkn.kind != lexer.IDENTIFIER: + continue + try: + next_tkn = next(tkns) + except StopIteration: + return False + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.text in ESCAPING_FUNCTIONS: + return True + if tkn.text == "tp_vectorcall": + return True + if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + return True + return False + + +EXITS = { + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +} + + +def eval_breaker_at_end(op: parser.InstDef) -> bool: + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + + +def always_exits(op: parser.InstDef) -> bool: + depth = 0 + tkn_iter = iter(op.tokens) + for tkn in tkn_iter: + if tkn.kind == "LBRACE": + depth += 1 + elif tkn.kind == "RBRACE": + depth -= 1 + elif depth > 1: + continue + elif tkn.kind == "GOTO" or tkn.kind == "RETURN": + return True + elif tkn.kind == "KEYWORD": + if tkn.text in EXITS: + return True + elif tkn.kind == "IDENTIFIER": + if tkn.text in EXITS: + return True + if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": + next(tkn_iter) # '(' + t = next(tkn_iter) + if t.text == "true": + return True + return False + + +def stack_effect_only_peeks(instr: parser.InstDef) -> bool: + stack_inputs = [s for s in instr.inputs if not isinstance(s, parser.CacheEffect)] + if len(stack_inputs) != len(instr.outputs): + return False + if len(stack_inputs) == 0: + return False + if any(s.cond for s in stack_inputs) or any(s.cond for s in instr.outputs): + return False + return all( + (s.name == other.name and s.type == other.type and s.size == other.size) + for s, other in zip(stack_inputs, instr.outputs) + ) + +OPARG_AND_1 = re.compile("\\(*oparg *& *1") + +def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: + for effect in op.inputs: + if isinstance(effect, parser.CacheEffect): + continue + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + for effect in op.outputs: + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + return False + +def compute_properties(op: parser.InstDef) -> Properties: + has_free = ( + variable_used(op, "PyCell_New") + or variable_used(op, "PyCell_GetRef") + or variable_used(op, "PyCell_SetTakeRef") + or variable_used(op, "PyCell_SwapTakeRef") + ) + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") + if deopts_if and exits_if: + tkn = op.tokens[0] + raise lexer.make_syntax_error( + "Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) + error_with_pop = has_error_with_pop(op) + error_without_pop = has_error_without_pop(op) + return Properties( + escapes=makes_escaping_api_call(op), + error_with_pop=error_with_pop, + error_without_pop=error_without_pop, + deopts=deopts_if, + side_exit=exits_if, + oparg=variable_used(op, "oparg"), + jumps=variable_used(op, "JUMPBY"), + eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), + ends_with_eval_breaker=eval_breaker_at_end(op), + needs_this=variable_used(op, "this_instr"), + always_exits=always_exits(op), + stores_sp=variable_used(op, "SYNC_SP"), + uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), + uses_co_names=variable_used(op, "FRAME_CO_NAMES"), + uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) + and not has_free, + has_free=has_free, + pure="pure" in op.annotations, + tier=tier_variable(op), + ) + + +def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: + result = Uop( + name=name, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=compute_properties(op), + ) + if effect_depends_on_oparg_1(op) and "split" in op.annotations: + result.properties.oparg_and_1 = True + for bit in ("0", "1"): + name_x = name + "_" + bit + properties = compute_properties(op) + if properties.oparg: + # May not need oparg anymore + properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op, bit), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + for anno in op.annotations: + if anno.startswith("replicate"): + result.replicated = int(anno[10:-1]) + break + else: + return result + for oparg in range(result.replicated): + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result + + +def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: + assert op.kind == "op" + if op.name in uops: + if "override" not in op.annotations: + raise override_error( + op.name, op.context, uops[op.name].context, op.tokens[0] + ) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) + + +def add_instruction( + name: str, parts: list[Part], instructions: dict[str, Instruction] +) -> None: + instructions[name] = Instruction(name, parts, None) + + +def desugar_inst( + inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + assert inst.kind == "inst" + name = inst.name + op_inputs: list[parser.InputEffect] = [] + parts: list[Part] = [] + uop_index = -1 + # Move unused cache entries to the Instruction, removing them from the Uop. + for input in inst.inputs: + if isinstance(input, parser.CacheEffect) and input.name == "unused": + parts.append(Skip(input.size)) + else: + op_inputs.append(input) + if uop_index < 0: + uop_index = len(parts) + # Place holder for the uop. + parts.append(Skip(0)) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) + uop.implicitly_created = True + uops[inst.name] = uop + if uop_index < 0: + parts.append(uop) + else: + parts[uop_index] = uop + add_instruction(name, parts, instructions) + + +def add_macro( + macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + parts: list[Uop | Skip] = [] + for part in macro.uops: + match part: + case parser.OpName(): + if part.name not in uops: + analysis_error(f"No Uop named {part.name}", macro.tokens[0]) + parts.append(uops[part.name]) + case parser.CacheEffect(): + parts.append(Skip(part.size)) + case _: + assert False + assert parts + add_instruction(macro.name, parts, instructions) + + +def add_family( + pfamily: parser.Family, + instructions: dict[str, Instruction], + families: dict[str, Family], +) -> None: + family = Family( + pfamily.name, + pfamily.size, + [instructions[member_name] for member_name in pfamily.members], + ) + for member in family.members: + member.family = family + # The head of the family is an implicit jump target for DEOPTs + instructions[family.name].is_target = True + families[family.name] = family + + +def add_pseudo( + pseudo: parser.Pseudo, + instructions: dict[str, Instruction], + pseudos: dict[str, PseudoInstruction], +) -> None: + pseudos[pseudo.name] = PseudoInstruction( + pseudo.name, + analyze_stack(pseudo), + [instructions[target] for target in pseudo.targets], + pseudo.flags, + ) + + +def assign_opcodes( + instructions: dict[str, Instruction], + families: dict[str, Family], + pseudos: dict[str, PseudoInstruction], +) -> tuple[dict[str, int], int, int]: + """Assigns opcodes, then returns the opmap, + have_arg and min_instrumented values""" + instmap: dict[str, int] = {} + + # 0 is reserved for cache entries. This helps debugging. + instmap["CACHE"] = 0 + + # 17 is reserved as it is the initial value for the specializing counter. + # This helps catch cases where we attempt to execute a cache. + instmap["RESERVED"] = 17 + + # 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py + instmap["RESUME"] = 149 + + # This is an historical oddity. + instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 + + instmap["INSTRUMENTED_LINE"] = 254 + + instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] + + # Special case: this instruction is implemented in ceval.c + # rather than bytecodes.c, so we need to add it explicitly + # here (at least until we add something to bytecodes.c to + # declare external instructions). + instrumented.append("INSTRUMENTED_LINE") + + specialized: set[str] = set() + no_arg: list[str] = [] + has_arg: list[str] = [] + + for family in families.values(): + specialized.update(inst.name for inst in family.members) + + for inst in instructions.values(): + name = inst.name + if name in specialized: + continue + if name in instrumented: + continue + if inst.properties.oparg: + has_arg.append(name) + else: + no_arg.append(name) + + # Specialized ops appear in their own section + # Instrumented opcodes are at the end of the valid range + min_internal = 150 + min_instrumented = 254 - (len(instrumented) - 1) + assert min_internal + len(specialized) < min_instrumented + + next_opcode = 1 + + def add_instruction(name: str) -> None: + nonlocal next_opcode + if name in instmap: + return # Pre-defined name + while next_opcode in instmap.values(): + next_opcode += 1 + instmap[name] = next_opcode + next_opcode += 1 + + for name in sorted(no_arg): + add_instruction(name) + for name in sorted(has_arg): + add_instruction(name) + # For compatibility + next_opcode = min_internal + for name in sorted(specialized): + add_instruction(name) + next_opcode = min_instrumented + for name in instrumented: + add_instruction(name) + + for name in instructions: + instructions[name].opcode = instmap[name] + + for op, name in enumerate(sorted(pseudos), 256): + instmap[name] = op + pseudos[name].opcode = op + + return instmap, len(no_arg), min_instrumented + + +def analyze_forest(forest: list[parser.AstNode]) -> Analysis: + instructions: dict[str, Instruction] = {} + uops: dict[str, Uop] = {} + families: dict[str, Family] = {} + pseudos: dict[str, PseudoInstruction] = {} + for node in forest: + match node: + case parser.InstDef(name): + if node.kind == "inst": + desugar_inst(node, instructions, uops) + else: + assert node.kind == "op" + add_op(node, uops) + case parser.Macro(): + pass + case parser.Family(): + pass + case parser.Pseudo(): + pass + case _: + assert False + for node in forest: + if isinstance(node, parser.Macro): + add_macro(node, instructions, uops) + for node in forest: + match node: + case parser.Family(): + add_family(node, instructions, families) + case parser.Pseudo(): + add_pseudo(node, instructions, pseudos) + case _: + pass + for uop in uops.values(): + tkn_iter = iter(uop.body) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text == "GO_TO_INSTRUCTION": + if next(tkn_iter).kind != "LPAREN": + continue + target = next(tkn_iter) + if target.kind != "IDENTIFIER": + continue + if target.text in instructions: + instructions[target.text].is_target = True + # Special case BINARY_OP_INPLACE_ADD_UNICODE + # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, + # as it is the wrong size, but we need it to maintain an + # historical optimization. + if "BINARY_OP_INPLACE_ADD_UNICODE" in instructions: + inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] + inst.family = families["BINARY_OP"] + families["BINARY_OP"].members.append(inst) + opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) + return Analysis( + instructions, uops, families, pseudos, opmap, first_arg, min_instrumented + ) + + +def analyze_files(filenames: list[str]) -> Analysis: + return analyze_forest(parser.parse_files(filenames)) + + +def dump_analysis(analysis: Analysis) -> None: + print("Uops:") + for u in analysis.uops.values(): + u.dump(" ") + print("Instructions:") + for i in analysis.instructions.values(): + i.dump(" ") + print("Families:") + for f in analysis.families.values(): + f.dump(" ") + print("Pseudos:") + for p in analysis.pseudos.values(): + p.dump(" ") + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("No input") + else: + filenames = sys.argv[1:] + dump_analysis(analyze_files(filenames)) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py new file mode 100644 index 00000000000000..069f0177a74018 --- /dev/null +++ b/Tools/cases_generator/cwriter.py @@ -0,0 +1,146 @@ +import contextlib +from lexer import Token +from typing import TextIO, Iterator + + +class CWriter: + "A writer that understands tokens and how to format C code" + + last_token: Token | None + + def __init__(self, out: TextIO, indent: int, line_directives: bool): + self.out = out + self.base_column = indent * 4 + self.indents = [i * 4 for i in range(indent + 1)] + self.line_directives = line_directives + self.last_token = None + self.newline = True + + def set_position(self, tkn: Token) -> None: + if self.last_token is not None: + if self.last_token.line < tkn.line: + self.out.write("\n") + if self.line_directives: + self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') + self.out.write(" " * self.indents[-1]) + else: + gap = tkn.column - self.last_token.end_column + self.out.write(" " * gap) + elif self.newline: + self.out.write(" " * self.indents[-1]) + self.last_token = tkn + self.newline = False + + def emit_at(self, txt: str, where: Token) -> None: + self.set_position(where) + self.out.write(txt) + + def maybe_dedent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens < 0: + self.indents.pop() + braces = txt.count("{") - txt.count("}") + if braces < 0 or is_label(txt): + self.indents.pop() + + def maybe_indent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens > 0: + if self.last_token: + offset = self.last_token.end_column - 1 + if offset <= self.indents[-1] or offset > 40: + offset = self.indents[-1] + 4 + else: + offset = self.indents[-1] + 4 + self.indents.append(offset) + if is_label(txt): + self.indents.append(self.indents[-1] + 4) + else: + braces = txt.count("{") - txt.count("}") + if braces > 0: + assert braces == 1 + if 'extern "C"' in txt: + self.indents.append(self.indents[-1]) + else: + self.indents.append(self.indents[-1] + 4) + + def emit_text(self, txt: str) -> None: + self.out.write(txt) + + def emit_multiline_comment(self, tkn: Token) -> None: + self.set_position(tkn) + lines = tkn.text.splitlines(True) + first = True + for line in lines: + text = line.lstrip() + if first: + spaces = 0 + else: + spaces = self.indents[-1] + if text.startswith("*"): + spaces += 1 + else: + spaces += 3 + first = False + self.out.write(" " * spaces) + self.out.write(text) + + def emit_token(self, tkn: Token) -> None: + if tkn.kind == "COMMENT" and "\n" in tkn.text: + return self.emit_multiline_comment(tkn) + self.maybe_dedent(tkn.text) + self.set_position(tkn) + self.emit_text(tkn.text) + self.maybe_indent(tkn.text) + + def emit_str(self, txt: str) -> None: + self.maybe_dedent(txt) + if self.newline and txt: + if txt[0] != "\n": + self.out.write(" " * self.indents[-1]) + self.newline = False + self.emit_text(txt) + if txt.endswith("\n"): + self.newline = True + self.maybe_indent(txt) + self.last_token = None + + def emit(self, txt: str | Token) -> None: + if isinstance(txt, Token): + self.emit_token(txt) + elif isinstance(txt, str): + self.emit_str(txt) + else: + assert False + + def start_line(self) -> None: + if not self.newline: + self.out.write("\n") + self.newline = True + self.last_token = None + + @contextlib.contextmanager + def header_guard(self, name: str) -> Iterator[None]: + self.out.write( + f""" +#ifndef {name} +#define {name} +#ifdef __cplusplus +extern "C" {{ +#endif + +""" + ) + yield + self.out.write( + f""" +#ifdef __cplusplus +}} +#endif +#endif /* !{name} */ +""" + ) + + +def is_label(txt: str) -> bool: + return not txt.startswith("//") and txt.endswith(":") diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py deleted file mode 100644 index 5241331bb97cdb..00000000000000 --- a/Tools/cases_generator/flags.py +++ /dev/null @@ -1,115 +0,0 @@ -import dataclasses - -from formatting import Formatter -import lexer as lx -import parsing -from typing import AbstractSet - - -@dataclasses.dataclass -class InstructionFlags: - """Construct and manipulate instruction flags""" - - HAS_ARG_FLAG: bool = False - HAS_CONST_FLAG: bool = False - HAS_NAME_FLAG: bool = False - HAS_JUMP_FLAG: bool = False - HAS_FREE_FLAG: bool = False - HAS_LOCAL_FLAG: bool = False - HAS_EVAL_BREAK_FLAG: bool = False - HAS_DEOPT_FLAG: bool = False - HAS_ERROR_FLAG: bool = False - - def __post_init__(self) -> None: - self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} - - @staticmethod - def fromInstruction(instr: parsing.Node) -> "InstructionFlags": - has_free = ( - variable_used(instr, "PyCell_New") - or variable_used(instr, "PyCell_GET") - or variable_used(instr, "PyCell_SET") - ) - - return InstructionFlags( - HAS_ARG_FLAG=variable_used(instr, "oparg"), - HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), - HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), - HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), - HAS_FREE_FLAG=has_free, - HAS_LOCAL_FLAG=( - variable_used(instr, "GETLOCAL") or variable_used(instr, "SETLOCAL") - ) - and not has_free, - HAS_EVAL_BREAK_FLAG=variable_used(instr, "CHECK_EVAL_BREAKER"), - HAS_DEOPT_FLAG=variable_used(instr, "DEOPT_IF"), - HAS_ERROR_FLAG=( - variable_used(instr, "ERROR_IF") - or variable_used(instr, "error") - or variable_used(instr, "pop_1_error") - or variable_used(instr, "exception_unwind") - or variable_used(instr, "resume_with_error") - ), - ) - - @staticmethod - def newEmpty() -> "InstructionFlags": - return InstructionFlags() - - def add(self, other: "InstructionFlags") -> None: - for name, value in dataclasses.asdict(other).items(): - if value: - setattr(self, name, value) - - def names(self, value: bool | None = None) -> list[str]: - if value is None: - return list(dataclasses.asdict(self).keys()) - return [n for n, v in dataclasses.asdict(self).items() if v == value] - - def bitmap(self, ignore: AbstractSet[str] = frozenset()) -> int: - flags = 0 - assert all(hasattr(self, name) for name in ignore) - for name in self.names(): - if getattr(self, name) and name not in ignore: - flags |= self.bitmask[name] - return flags - - @classmethod - def emit_macros(cls, out: Formatter) -> None: - flags = cls.newEmpty() - for name, value in flags.bitmask.items(): - out.emit(f"#define {name} ({value})") - - for name, value in flags.bitmask.items(): - out.emit( - f"#define OPCODE_{name[:-len('_FLAG')]}(OP) " - f"(_PyOpcode_opcode_metadata[OP].flags & ({name}))" - ) - - -def variable_used(node: parsing.Node, name: str) -> bool: - """Determine whether a variable with a given name is used in a node.""" - return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.tokens - ) - - -def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: - """Like variable_used(), but skips #if ENABLE_SPECIALIZATION blocks.""" - tokens: list[lx.Token] = [] - skipping = False - for i, token in enumerate(node.tokens): - if token.kind == "MACRO": - text = "".join(token.text.split()) - # TODO: Handle nested #if - if text == "#if": - if i + 1 < len(node.tokens) and node.tokens[i + 1].text in ( - "ENABLE_SPECIALIZATION", - "TIER_ONE", - ): - skipping = True - elif text in ("#else", "#endif"): - skipping = False - if not skipping: - tokens.append(token) - return any(token.kind == "IDENTIFIER" and token.text == name for token in tokens) diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py deleted file mode 100644 index 4fd9172d20c274..00000000000000 --- a/Tools/cases_generator/formatting.py +++ /dev/null @@ -1,206 +0,0 @@ -import contextlib -import re -import typing -from collections.abc import Iterator - -from parsing import StackEffect, Family - -UNUSED = "unused" - - -class Formatter: - """Wraps an output stream with the ability to indent etc.""" - - stream: typing.TextIO - prefix: str - emit_line_directives: bool = False - lineno: int # Next line number, 1-based - filename: str # Slightly improved stream.filename - nominal_lineno: int - nominal_filename: str - - def __init__( - self, - stream: typing.TextIO, - indent: int, - emit_line_directives: bool = False, - comment: str = "//", - ) -> None: - self.stream = stream - self.prefix = " " * indent - self.emit_line_directives = emit_line_directives - self.comment = comment - self.lineno = 1 - self.filename = prettify_filename(self.stream.name) - self.nominal_lineno = 1 - self.nominal_filename = self.filename - - def write_raw(self, s: str) -> None: - self.stream.write(s) - newlines = s.count("\n") - self.lineno += newlines - self.nominal_lineno += newlines - - def emit(self, arg: str) -> None: - if arg: - self.write_raw(f"{self.prefix}{arg}\n") - else: - self.write_raw("\n") - - def set_lineno(self, lineno: int, filename: str) -> None: - if self.emit_line_directives: - if lineno != self.nominal_lineno or filename != self.nominal_filename: - self.emit(f'#line {lineno} "{filename}"') - self.nominal_lineno = lineno - self.nominal_filename = filename - - def reset_lineno(self) -> None: - if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename: - self.set_lineno(self.lineno + 1, self.filename) - - @contextlib.contextmanager - def indent(self) -> Iterator[None]: - self.prefix += " " - yield - self.prefix = self.prefix[:-4] - - @contextlib.contextmanager - def block(self, head: str, tail: str = "") -> Iterator[None]: - if head: - self.emit(head + " {") - else: - self.emit("{") - with self.indent(): - yield - self.emit("}" + tail) - - def stack_adjust( - self, - input_effects: list[StackEffect], - output_effects: list[StackEffect], - ) -> None: - shrink, isym = list_effect_size(input_effects) - grow, osym = list_effect_size(output_effects) - diff = grow - shrink - if isym and isym != osym: - self.emit(f"STACK_SHRINK({isym});") - if diff < 0: - self.emit(f"STACK_SHRINK({-diff});") - if diff > 0: - self.emit(f"STACK_GROW({diff});") - if osym and osym != isym: - self.emit(f"STACK_GROW({osym});") - - def declare(self, dst: StackEffect, src: StackEffect | None) -> None: - if dst.name == UNUSED or dst.cond == "0": - return - typ = f"{dst.type}" if dst.type else "PyObject *" - if src: - cast = self.cast(dst, src) - initexpr = f"{cast}{src.name}" - if src.cond and src.cond != "1": - initexpr = f"{parenthesize_cond(src.cond)} ? {initexpr} : NULL" - init = f" = {initexpr}" - elif dst.cond and dst.cond != "1": - init = " = NULL" - else: - init = "" - sepa = "" if typ.endswith("*") else " " - self.emit(f"{typ}{sepa}{dst.name}{init};") - - def assign(self, dst: StackEffect, src: StackEffect) -> None: - if src.name == UNUSED or dst.name == UNUSED: - return - cast = self.cast(dst, src) - if re.match(r"^REG\(oparg(\d+)\)$", dst.name): - self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") - else: - stmt = f"{dst.name} = {cast}{src.name};" - if src.cond and src.cond != "1": - if src.cond == "0": - # It will not be executed - return - stmt = f"if ({src.cond}) {{ {stmt} }}" - self.emit(stmt) - - def cast(self, dst: StackEffect, src: StackEffect) -> str: - return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" - - def static_assert_family_size( - self, name: str, family: Family | None, cache_offset: int - ) -> None: - """Emit a static_assert for the size of a family, if known. - - This will fail at compile time if the cache size computed from - the instruction definition does not match the size of the struct - used by specialize.c. - """ - if family and name == family.name: - cache_size = family.size - if cache_size: - self.emit( - f"static_assert({cache_size} == {cache_offset}, " - f'"incorrect cache size");' - ) - - -def prettify_filename(filename: str) -> str: - # Make filename more user-friendly and less platform-specific, - # it is only used for error reporting at this point. - filename = filename.replace("\\", "/") - if filename.startswith("./"): - filename = filename[2:] - if filename.endswith(".new"): - filename = filename[:-4] - return filename - - -def list_effect_size(effects: list[StackEffect]) -> tuple[int, str]: - numeric = 0 - symbolic: list[str] = [] - for effect in effects: - diff, sym = effect_size(effect) - numeric += diff - if sym: - symbolic.append(maybe_parenthesize(sym)) - return numeric, " + ".join(symbolic) - - -def effect_size(effect: StackEffect) -> tuple[int, str]: - """Return the 'size' impact of a stack effect. - - Returns a tuple (numeric, symbolic) where: - - - numeric is an int giving the statically analyzable size of the effect - - symbolic is a string representing a variable effect (e.g. 'oparg*2') - - At most one of these will be non-zero / non-empty. - """ - if effect.size: - assert not effect.cond, "Array effects cannot have a condition" - return 0, effect.size - elif effect.cond: - if effect.cond in ("0", "1"): - return int(effect.cond), "" - return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - else: - return 1, "" - - -def maybe_parenthesize(sym: str) -> str: - """Add parentheses around a string if it contains an operator. - - An exception is made for '*' which is common and harmless - in the context where the symbolic size is used. - """ - if re.match(r"^[\s\w*]+$", sym): - return sym - else: - return f"({sym})" - - -def parenthesize_cond(cond: str) -> str: - """Parenthesize a condition, but only if it contains ?: itself.""" - if "?" in cond: - cond = f"({cond})" - return cond diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py deleted file mode 100644 index 04d617ba9f5173..00000000000000 --- a/Tools/cases_generator/generate_cases.py +++ /dev/null @@ -1,874 +0,0 @@ -"""Generate the main interpreter switch. -Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_cases.c.h, which is #included in ceval.c. -""" - -import argparse -import contextlib -import itertools -import os -import posixpath -import sys -import textwrap -import typing -from collections.abc import Iterator - -import stacking # Early import to avoid circular import -from _typing_backports import assert_never -from analysis import Analyzer -from formatting import Formatter, list_effect_size -from flags import InstructionFlags, variable_used -from instructions import ( - AnyInstruction, - AbstractInstruction, - Component, - Instruction, - MacroInstruction, - MacroParts, - PseudoInstruction, - TIER_ONE, - TIER_TWO, -) -import parsing -from parsing import StackEffect - - -HERE = os.path.dirname(__file__) -ROOT = os.path.join(HERE, "../..") -THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, posixpath.sep) - -DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) -DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) -DEFAULT_OPCODE_IDS_H_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Include/opcode_ids.h") -) -DEFAULT_OPCODE_TARGETS_H_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/opcode_targets.h") -) -DEFAULT_METADATA_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Include/internal/pycore_opcode_metadata.h") -) -DEFAULT_PYMETADATA_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Lib/_opcode_metadata.py") -) -DEFAULT_EXECUTOR_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/executor_cases.c.h") -) -DEFAULT_ABSTRACT_INTERPRETER_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/abstract_interp_cases.c.h") -) - -# Constants used instead of size for macro expansions. -# Note: 1, 2, 4 must match actual cache entry sizes. -OPARG_SIZES = { - "OPARG_FULL": 0, - "OPARG_CACHE_1": 1, - "OPARG_CACHE_2": 2, - "OPARG_CACHE_4": 4, - "OPARG_TOP": 5, - "OPARG_BOTTOM": 6, - "OPARG_SET_IP": 7, -} - -INSTR_FMT_PREFIX = "INSTR_FMT_" - -# TODO: generate all these after updating the DSL -SPECIALLY_HANDLED_ABSTRACT_INSTR = { - "LOAD_FAST", - "LOAD_FAST_CHECK", - "LOAD_FAST_AND_CLEAR", - "LOAD_CONST", - "STORE_FAST", - "STORE_FAST_MAYBE_NULL", - "COPY", - # Arithmetic - "_BINARY_OP_MULTIPLY_INT", - "_BINARY_OP_ADD_INT", - "_BINARY_OP_SUBTRACT_INT", -} - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-v", - "--verbose", - help="Print list of non-viable uops and exit", - action="store_true", -) -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) -arg_parser.add_argument( - "-n", - "--opcode_ids_h", - type=str, - help="Header file with opcode number definitions", - default=DEFAULT_OPCODE_IDS_H_OUTPUT, -) -arg_parser.add_argument( - "-t", - "--opcode_targets_h", - type=str, - help="File with opcode targets for computed gotos", - default=DEFAULT_OPCODE_TARGETS_H_OUTPUT, -) -arg_parser.add_argument( - "-m", - "--metadata", - type=str, - help="Generated C metadata", - default=DEFAULT_METADATA_OUTPUT, -) -arg_parser.add_argument( - "-p", - "--pymetadata", - type=str, - help="Generated Python metadata", - default=DEFAULT_PYMETADATA_OUTPUT, -) -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) -arg_parser.add_argument( - "-e", - "--executor-cases", - type=str, - help="Write executor cases to this file", - default=DEFAULT_EXECUTOR_OUTPUT, -) -arg_parser.add_argument( - "-a", - "--abstract-interpreter-cases", - type=str, - help="Write abstract interpreter cases to this file", - default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, -) - - -class Generator(Analyzer): - def get_stack_effect_info( - self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo - ) -> tuple[AnyInstruction | None, str, str]: - def effect_str(effects: list[StackEffect]) -> str: - n_effect, sym_effect = list_effect_size(effects) - if sym_effect: - return f"{sym_effect} + {n_effect}" if n_effect else sym_effect - return str(n_effect) - - instr: AnyInstruction | None - popped: str | None = None - pushed: str | None = None - match thing: - case parsing.InstDef(): - instr = self.instrs[thing.name] - popped = effect_str(instr.input_effects) - pushed = effect_str(instr.output_effects) - case parsing.Macro(): - instr = self.macro_instrs[thing.name] - popped, pushed = stacking.get_stack_effect_info_for_macro(instr) - case parsing.Pseudo(): - instr = self.pseudo_instrs[thing.name] - # Calculate stack effect, and check that it's the same - # for all targets. - for target in self.pseudos[thing.name].targets: - target_instr = self.instrs.get(target) - # Currently target is always an instr. This could change - # in the future, e.g., if we have a pseudo targetting a - # macro instruction. - assert target_instr - target_popped = effect_str(target_instr.input_effects) - target_pushed = effect_str(target_instr.output_effects) - if popped is None: - popped, pushed = target_popped, target_pushed - else: - assert popped == target_popped - assert pushed == target_pushed - case _: - assert_never(thing) - assert popped is not None and pushed is not None - return instr, popped, pushed - - @contextlib.contextmanager - def metadata_item(self, signature: str, open: str, close: str) -> Iterator[None]: - self.out.emit("") - self.out.emit(f"extern {signature};") - self.out.emit("#ifdef NEED_OPCODE_METADATA") - with self.out.block(f"{signature} {open}", close): - yield - self.out.emit("#endif // NEED_OPCODE_METADATA") - - def write_stack_effect_functions(self) -> None: - popped_data: list[tuple[AnyInstruction, str]] = [] - pushed_data: list[tuple[AnyInstruction, str]] = [] - for thing in self.everything: - if isinstance(thing, parsing.Macro) and thing.name in self.instrs: - continue - instr, popped, pushed = self.get_stack_effect_info(thing) - if instr is not None: - popped_data.append((instr, popped)) - pushed_data.append((instr, pushed)) - - def write_function( - direction: str, data: list[tuple[AnyInstruction, str]] - ) -> None: - with self.metadata_item( - f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", - "", - "", - ): - with self.out.block("switch(opcode)"): - for instr, effect in data: - self.out.emit(f"case {instr.name}:") - self.out.emit(f" return {effect};") - self.out.emit("default:") - self.out.emit(" return -1;") - - write_function("popped", popped_data) - write_function("pushed", pushed_data) - self.out.emit("") - - def from_source_files(self) -> str: - filenames = [] - for filename in self.input_filenames: - try: - filename = os.path.relpath(filename, ROOT) - except ValueError: - # May happen on Windows if root and temp on different volumes - pass - filenames.append(filename.replace(os.path.sep, posixpath.sep)) - paths = f"\n{self.out.comment} ".join(filenames) - return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" - - def write_provenance_header(self) -> None: - self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") - self.out.write_raw(self.from_source_files()) - self.out.write_raw(f"{self.out.comment} Do not edit!\n") - - def assign_opcode_ids(self) -> None: - """Assign IDs to opcodes""" - - ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode - instrumented_ops: list[str] = [] - - specialized_ops: set[str] = set() - for name, family in self.families.items(): - specialized_ops.update(family.members) - - for instr in self.macro_instrs.values(): - name = instr.name - if name in specialized_ops: - continue - if name.startswith("INSTRUMENTED_"): - instrumented_ops.append(name) - else: - ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) - - # Special case: this instruction is implemented in ceval.c - # rather than bytecodes.c, so we need to add it explicitly - # here (at least until we add something to bytecodes.c to - # declare external instructions). - instrumented_ops.append("INSTRUMENTED_LINE") - - # assert lists are unique - assert len(set(ops)) == len(ops) - assert len(set(instrumented_ops)) == len(instrumented_ops) - - opname: list[str | None] = [None] * 512 - opmap: dict[str, int] = {} - markers: dict[str, int] = {} - - def map_op(op: int, name: str) -> None: - assert op < len(opname) - assert opname[op] is None, (op, name) - assert name not in opmap - opname[op] = name - opmap[name] = op - - # 0 is reserved for cache entries. This helps debugging. - map_op(0, "CACHE") - - # 17 is reserved as it is the initial value for the specializing counter. - # This helps catch cases where we attempt to execute a cache. - map_op(17, "RESERVED") - - # 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - map_op(149, "RESUME") - - # Specialized ops appear in their own section - # Instrumented opcodes are at the end of the valid range - min_internal = 150 - min_instrumented = 254 - (len(instrumented_ops) - 1) - assert min_internal + len(specialized_ops) < min_instrumented - - next_opcode = 1 - for has_arg, name in sorted(ops): - if name in opmap: - continue # an anchored name, like CACHE - map_op(next_opcode, name) - if has_arg and "HAVE_ARGUMENT" not in markers: - markers["HAVE_ARGUMENT"] = next_opcode - - while opname[next_opcode] is not None: - next_opcode += 1 - - assert next_opcode < min_internal, next_opcode - - for i, op in enumerate(sorted(specialized_ops)): - map_op(min_internal + i, op) - - markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented - for i, op in enumerate(instrumented_ops): - map_op(min_instrumented + i, op) - - # Pseudo opcodes are after the valid range - for i, op in enumerate(sorted(self.pseudos)): - map_op(256 + i, op) - - assert 255 not in opmap.values() # 255 is reserved - self.opmap = opmap - self.markers = markers - - def write_opcode_ids( - self, opcode_ids_h_filename: str, opcode_targets_filename: str - ) -> None: - """Write header file that defined the opcode IDs""" - - with open(opcode_ids_h_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - self.write_provenance_header() - - self.out.emit("") - self.out.emit("#ifndef Py_OPCODE_IDS_H") - self.out.emit("#define Py_OPCODE_IDS_H") - self.out.emit("#ifdef __cplusplus") - self.out.emit('extern "C" {') - self.out.emit("#endif") - self.out.emit("") - self.out.emit("/* Instruction opcodes for compiled code */") - - def define(name: str, opcode: int) -> None: - self.out.emit(f"#define {name:<38} {opcode:>3}") - - all_pairs: list[tuple[int, int, str]] = [] - # the second item in the tuple sorts the markers before the ops - all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) - all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) - for i, _, name in sorted(all_pairs): - assert name is not None - define(name, i) - - self.out.emit("") - self.out.emit("#ifdef __cplusplus") - self.out.emit("}") - self.out.emit("#endif") - self.out.emit("#endif /* !Py_OPCODE_IDS_H */") - - with open(opcode_targets_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - with self.out.block("static void *opcode_targets[256] =", ";"): - targets = ["_unknown_opcode"] * 256 - for name, op in self.opmap.items(): - if op < 256: - targets[op] = f"TARGET_{name}" - f.write(",\n".join([f" &&{s}" for s in targets])) - - def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: - """Write instruction metadata to output file.""" - - # Compute the set of all instruction formats. - all_formats: set[str] = set() - for thing in self.everything: - format: str | None = None - match thing: - case parsing.InstDef(): - format = self.instrs[thing.name].instr_fmt - case parsing.Macro(): - format = self.macro_instrs[thing.name].instr_fmt - case parsing.Pseudo(): - for target in self.pseudos[thing.name].targets: - target_instr = self.instrs.get(target) - assert target_instr - if format is None: - format = target_instr.instr_fmt - else: - assert format == target_instr.instr_fmt - case _: - assert_never(thing) - assert format is not None - all_formats.add(format) - - # Turn it into a sorted list of enum values. - format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] - - with open(metadata_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - self.write_provenance_header() - - self.out.emit("") - self.out.emit("#ifndef Py_BUILD_CORE") - self.out.emit('# error "this header requires Py_BUILD_CORE define"') - self.out.emit("#endif") - self.out.emit("") - self.out.emit("#include // bool") - - self.write_pseudo_instrs() - - self.out.emit("") - self.write_uop_items(lambda name, counter: f"#define {name} {counter}") - - self.write_stack_effect_functions() - - # Write the enum definition for instruction formats. - with self.out.block("enum InstructionFormat", ";"): - for enum in format_enums: - self.out.emit(enum + ",") - - self.out.emit("") - self.out.emit( - "#define IS_VALID_OPCODE(OP) \\\n" - " (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \\\n" - " (_PyOpcode_opcode_metadata[(OP)].valid_entry))" - ) - - self.out.emit("") - InstructionFlags.emit_macros(self.out) - - self.out.emit("") - with self.out.block("struct opcode_metadata", ";"): - self.out.emit("bool valid_entry;") - self.out.emit("enum InstructionFormat instr_format;") - self.out.emit("int flags;") - self.out.emit("") - - with self.out.block("struct opcode_macro_expansion", ";"): - self.out.emit("int nuops;") - self.out.emit( - "struct { int16_t uop; int8_t size; int8_t offset; } uops[12];" - ) - self.out.emit("") - - for key, value in OPARG_SIZES.items(): - self.out.emit(f"#define {key} {value}") - self.out.emit("") - - self.out.emit( - "#define OPCODE_METADATA_FMT(OP) " - "(_PyOpcode_opcode_metadata[(OP)].instr_format)" - ) - self.out.emit("#define SAME_OPCODE_METADATA(OP1, OP2) \\") - self.out.emit( - " (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2))" - ) - self.out.emit("") - - # Write metadata array declaration - self.out.emit("#define OPCODE_METADATA_SIZE 512") - self.out.emit("#define OPCODE_UOP_NAME_SIZE 512") - self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") - - with self.metadata_item( - "const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", - "=", - ";", - ): - # Write metadata for each instruction - for thing in self.everything: - match thing: - case parsing.InstDef(): - self.write_metadata_for_inst(self.instrs[thing.name]) - case parsing.Macro(): - if thing.name not in self.instrs: - self.write_metadata_for_macro( - self.macro_instrs[thing.name] - ) - case parsing.Pseudo(): - self.write_metadata_for_pseudo( - self.pseudo_instrs[thing.name] - ) - case _: - assert_never(thing) - - with self.metadata_item( - "const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", - "=", - ";", - ): - # Write macro expansion for each non-pseudo instruction - for mac in self.macro_instrs.values(): - if is_super_instruction(mac): - # Special-case the heck out of super-instructions - self.write_super_expansions(mac.name) - else: - self.write_macro_expansions( - mac.name, mac.parts, mac.cache_offset - ) - - with self.metadata_item( - "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" - ): - self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') - - with self.metadata_item( - f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", - "=", - ";", - ): - for name in self.opmap: - self.out.emit(f'[{name}] = "{name}",') - - with self.metadata_item( - f"const uint8_t _PyOpcode_Caches[256]", - "=", - ";", - ): - family_member_names: set[str] = set() - for family in self.families.values(): - family_member_names.update(family.members) - for mac in self.macro_instrs.values(): - if ( - mac.cache_offset > 0 - and mac.name not in family_member_names - and not mac.name.startswith("INSTRUMENTED_") - ): - self.out.emit(f"[{mac.name}] = {mac.cache_offset},") - # Irregular case: - self.out.emit("[JUMP_BACKWARD] = 1,") - - deoptcodes = {} - for name, op in self.opmap.items(): - if op < 256: - deoptcodes[name] = name - for name, family in self.families.items(): - for m in family.members: - deoptcodes[m] = name - # special case: - deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP" - - with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"): - for opt, deopt in sorted(deoptcodes.items()): - self.out.emit(f"[{opt}] = {deopt},") - - self.out.emit("") - self.out.emit("#define EXTRA_CASES \\") - valid_opcodes = set(self.opmap.values()) - with self.out.indent(): - for op in range(256): - if op not in valid_opcodes: - self.out.emit(f"case {op}: \\") - self.out.emit(" ;\n") - - with open(pymetadata_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0, comment="#") - - self.write_provenance_header() - - # emit specializations - specialized_ops = set() - - self.out.emit("") - self.out.emit("_specializations = {") - for name, family in self.families.items(): - with self.out.indent(): - self.out.emit(f'"{family.name}": [') - with self.out.indent(): - for m in family.members: - self.out.emit(f'"{m}",') - specialized_ops.update(family.members) - self.out.emit(f"],") - self.out.emit("}") - - # Handle special case - self.out.emit("") - self.out.emit("# An irregular case:") - self.out.emit( - '_specializations["BINARY_OP"].append(' - '"BINARY_OP_INPLACE_ADD_UNICODE")' - ) - specialized_ops.add("BINARY_OP_INPLACE_ADD_UNICODE") - - ops = sorted((id, name) for (name, id) in self.opmap.items()) - # emit specialized opmap - self.out.emit("") - with self.out.block("_specialized_opmap ="): - for op, name in ops: - if name in specialized_ops: - self.out.emit(f"'{name}': {op},") - - # emit opmap - self.out.emit("") - with self.out.block("opmap ="): - for op, name in ops: - if name not in specialized_ops: - self.out.emit(f"'{name}': {op},") - - for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]: - self.out.emit(f"{name} = {self.markers[name]}") - - def write_pseudo_instrs(self) -> None: - """Write the IS_PSEUDO_INSTR macro""" - self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\") - for op in self.pseudos: - self.out.emit(f" ((OP) == {op}) || \\") - self.out.emit(f" 0)") - - def write_uop_items(self, make_text: typing.Callable[[str, int], str]) -> None: - """Write '#define XXX NNN' for each uop""" - counter = 300 # TODO: Avoid collision with pseudo instructions - seen = set() - - def add(name: str) -> None: - if name in seen: - return - nonlocal counter - self.out.emit(make_text(name, counter)) - counter += 1 - seen.add(name) - - # These two are first by convention - add("_EXIT_TRACE") - add("_SET_IP") - - for instr in self.instrs.values(): - # Skip ops that are also macros -- those are desugared inst()s - if instr.name not in self.macros: - add(instr.name) - - def write_macro_expansions( - self, name: str, parts: MacroParts, cache_offset: int - ) -> None: - """Write the macro expansions for a macro-instruction.""" - # TODO: Refactor to share code with write_cody(), is_viaible_uop(), etc. - offset = 0 # Cache effect offset - expansions: list[tuple[str, int, int]] = [] # [(name, size, offset), ...] - for part in parts: - if isinstance(part, Component): - # All component instructions must be viable uops - if not part.instr.is_viable_uop() and part.instr.name != "_SAVE_CURRENT_IP": - # This note just reminds us about macros that cannot - # be expanded to Tier 2 uops. It is not an error. - # It is sometimes emitted for macros that have a - # manual translation in translate_bytecode_to_trace() - # in Python/optimizer.c. - if len(parts) > 1 or part.instr.name != name: - self.note( - f"Part {part.instr.name} of {name} is not a viable uop", - part.instr.inst, - ) - return - if not part.active_caches: - if part.instr.name == "_SAVE_CURRENT_IP": - size, offset = OPARG_SIZES["OPARG_SET_IP"], cache_offset - 1 - else: - size, offset = OPARG_SIZES["OPARG_FULL"], 0 - else: - # If this assert triggers, is_viable_uops() lied - assert len(part.active_caches) == 1, (name, part.instr.name) - cache = part.active_caches[0] - size, offset = cache.effect.size, cache.offset - expansions.append((part.instr.name, size, offset)) - assert len(expansions) > 0, f"Macro {name} has empty expansion?!" - self.write_expansions(name, expansions) - - def write_super_expansions(self, name: str) -> None: - """Write special macro expansions for super-instructions. - - If you get an assertion failure here, you probably have accidentally - violated one of the assumptions here. - - - A super-instruction's name is of the form FIRST_SECOND where - FIRST and SECOND are regular instructions whose name has the - form FOO_BAR. Thus, there must be exactly 3 underscores. - Example: LOAD_CONST_STORE_FAST. - - - A super-instruction's body uses `oparg1 and `oparg2`, and no - other instruction's body uses those variable names. - - - A super-instruction has no active (used) cache entries. - - In the expansion, the first instruction's operand is all but the - bottom 4 bits of the super-instruction's oparg, and the second - instruction's operand is the bottom 4 bits. We use the special - size codes OPARG_TOP and OPARG_BOTTOM for these. - """ - pieces = name.split("_") - assert len(pieces) == 4, f"{name} doesn't look like a super-instr" - name1 = "_".join(pieces[:2]) - name2 = "_".join(pieces[2:]) - assert name1 in self.instrs, f"{name1} doesn't match any instr" - assert name2 in self.instrs, f"{name2} doesn't match any instr" - instr1 = self.instrs[name1] - instr2 = self.instrs[name2] - assert not instr1.active_caches, f"{name1} has active caches" - assert not instr2.active_caches, f"{name2} has active caches" - expansions: list[tuple[str, int, int]] = [ - (name1, OPARG_SIZES["OPARG_TOP"], 0), - (name2, OPARG_SIZES["OPARG_BOTTOM"], 0), - ] - self.write_expansions(name, expansions) - - def write_expansions( - self, name: str, expansions: list[tuple[str, int, int]] - ) -> None: - pieces = [ - f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions - ] - self.out.emit( - f"[{name}] = " - f"{{ .nuops = {len(pieces)}, .uops = {{ {', '.join(pieces)} }} }}," - ) - - def emit_metadata_entry(self, name: str, fmt: str, flags: InstructionFlags) -> None: - flag_names = flags.names(value=True) - if not flag_names: - flag_names.append("0") - self.out.emit( - f"[{name}] = {{ true, {INSTR_FMT_PREFIX}{fmt}," - f" {' | '.join(flag_names)} }}," - ) - - def write_metadata_for_inst(self, instr: Instruction) -> None: - """Write metadata for a single instruction.""" - self.emit_metadata_entry(instr.name, instr.instr_fmt, instr.instr_flags) - - def write_metadata_for_macro(self, mac: MacroInstruction) -> None: - """Write metadata for a macro-instruction.""" - self.emit_metadata_entry(mac.name, mac.instr_fmt, mac.instr_flags) - - def write_metadata_for_pseudo(self, ps: PseudoInstruction) -> None: - """Write metadata for a macro-instruction.""" - self.emit_metadata_entry(ps.name, ps.instr_fmt, ps.instr_flags) - - def write_instructions( - self, output_filename: str, emit_line_directives: bool - ) -> None: - """Write instructions to output file.""" - with open(output_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 8, emit_line_directives) - - self.write_provenance_header() - - # Write and count instructions of all kinds - n_macros = 0 - for thing in self.everything: - match thing: - case parsing.InstDef(): - pass - case parsing.Macro(): - n_macros += 1 - mac = self.macro_instrs[thing.name] - stacking.write_macro_instr(mac, self.out) - case parsing.Pseudo(): - pass - case _: - assert_never(thing) - - print( - f"Wrote {n_macros} cases to {output_filename}", - file=sys.stderr, - ) - - def write_executor_instructions( - self, executor_filename: str, emit_line_directives: bool - ) -> None: - """Generate cases for the Tier 2 interpreter.""" - n_uops = 0 - with open(executor_filename, "w") as f: - self.out = Formatter(f, 8, emit_line_directives) - self.write_provenance_header() - for instr in self.instrs.values(): - if instr.is_viable_uop(): - n_uops += 1 - self.out.emit("") - with self.out.block(f"case {instr.name}:"): - stacking.write_single_instr(instr, self.out, tier=TIER_TWO) - if instr.check_eval_breaker: - self.out.emit("CHECK_EVAL_BREAKER();") - self.out.emit("break;") - print( - f"Wrote {n_uops} cases to {executor_filename}", - file=sys.stderr, - ) - - def write_abstract_interpreter_instructions( - self, abstract_interpreter_filename: str, emit_line_directives: bool - ) -> None: - """Generate cases for the Tier 2 abstract interpreter/analzyer.""" - with open(abstract_interpreter_filename, "w") as f: - self.out = Formatter(f, 8, emit_line_directives) - self.write_provenance_header() - for instr in self.instrs.values(): - instr = AbstractInstruction(instr.inst) - if ( - instr.is_viable_uop() - and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR - ): - self.out.emit("") - with self.out.block(f"case {instr.name}:"): - instr.write(self.out, tier=TIER_TWO) - self.out.emit("break;") - print( - f"Wrote some stuff to {abstract_interpreter_filename}", - file=sys.stderr, - ) - - -def is_super_instruction(mac: MacroInstruction) -> bool: - if ( - len(mac.parts) == 1 - and isinstance(mac.parts[0], Component) - and variable_used(mac.parts[0].instr.inst, "oparg1") - ): - assert variable_used(mac.parts[0].instr.inst, "oparg2") - return True - else: - return False - - -def main() -> None: - """Parse command line, parse input, analyze, write output.""" - args = arg_parser.parse_args() # Prints message and sys.exit(2) on error - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - - # Raises OSError if input unreadable - a = Generator(args.input) - - a.parse() # Raises SyntaxError on failure - a.analyze() # Prints messages and sets a.errors on failure - if a.errors: - sys.exit(f"Found {a.errors} errors") - if args.verbose: - # Load execution counts from bmraw.json, if it exists - a.report_non_viable_uops("bmraw.json") - return - - # These raise OSError if output can't be written - a.write_instructions(args.output, args.emit_line_directives) - - a.assign_opcode_ids() - a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) - a.write_metadata(args.metadata, args.pymetadata) - a.write_executor_instructions(args.executor_cases, args.emit_line_directives) - a.write_abstract_interpreter_instructions( - args.abstract_interpreter_cases, args.emit_line_directives - ) - - -if __name__ == "__main__": - main() diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py new file mode 100644 index 00000000000000..9314bb9e79687f --- /dev/null +++ b/Tools/cases_generator/generators_common.py @@ -0,0 +1,250 @@ +from pathlib import Path +from typing import TextIO + +from analyzer import ( + Instruction, + Uop, + Properties, + StackItem, +) +from cwriter import CWriter +from typing import Callable, Mapping, TextIO, Iterator, Tuple +from lexer import Token +from stack import Stack + + +ROOT = Path(__file__).parent.parent.parent +DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix() + + +def root_relative_path(filename: str) -> str: + try: + return Path(filename).absolute().relative_to(ROOT).as_posix() + except ValueError: + # Not relative to root, just return original path. + return filename + + +def type_and_null(var: StackItem) -> Tuple[str, str]: + if var.type: + return var.type, "NULL" + elif var.is_array(): + return "_PyStackRef *", "NULL" + else: + return "_PyStackRef", "PyStackRef_NULL" + + +def write_header( + generator: str, sources: list[str], outfile: TextIO, comment: str = "//" +) -> None: + outfile.write( + f"""{comment} This file is generated by {root_relative_path(generator)} +{comment} from: +{comment} {", ".join(root_relative_path(src) for src in sources)} +{comment} Do not edit! +""" + ) + + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + + +def replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("DEOPT_IF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(", ") + assert inst is not None + assert inst.family is not None + out.emit(inst.family.name) + out.emit(");\n") + + +def replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + + +def replace_error_no_pop( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("goto error;", tkn) + + +def replace_decrefs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size: + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + if var.condition == "1": + out.emit(f"PyStackRef_CLOSE({var.name});\n") + elif var.condition != "0": + out.emit(f"PyStackRef_XCLOSE({var.name});\n") + else: + out.emit(f"PyStackRef_CLOSE({var.name});\n") + + +def replace_sync_sp( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + stack.flush(out) + + +def replace_check_eval_breaker( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit_at("CHECK_EVAL_BREAKER();", tkn) + + +REPLACEMENT_FUNCTIONS = { + "EXIT_IF": replace_deopt, + "DEOPT_IF": replace_deopt, + "ERROR_IF": replace_error, + "ERROR_NO_POP": replace_error_no_pop, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "SYNC_SP": replace_sync_sp, +} + +ReplacementFunctionType = Callable[ + [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None +] + + +def emit_tokens( + out: CWriter, + uop: Uop, + stack: Stack, + inst: Instruction | None, + replacement_functions: Mapping[ + str, ReplacementFunctionType + ] = REPLACEMENT_FUNCTIONS, +) -> None: + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + out.start_line() + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions: + replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst) + else: + out.emit(tkn) + + +def cflags(p: Properties) -> str: + flags: list[str] = [] + if p.oparg: + flags.append("HAS_ARG_FLAG") + if p.uses_co_consts: + flags.append("HAS_CONST_FLAG") + if p.uses_co_names: + flags.append("HAS_NAME_FLAG") + if p.jumps: + flags.append("HAS_JUMP_FLAG") + if p.has_free: + flags.append("HAS_FREE_FLAG") + if p.uses_locals: + flags.append("HAS_LOCAL_FLAG") + if p.eval_breaker: + flags.append("HAS_EVAL_BREAK_FLAG") + if p.deopts: + flags.append("HAS_DEOPT_FLAG") + if p.side_exit: + flags.append("HAS_EXIT_FLAG") + if not p.infallible: + flags.append("HAS_ERROR_FLAG") + if p.error_without_pop: + flags.append("HAS_ERROR_NO_POP_FLAG") + if p.escapes: + flags.append("HAS_ESCAPES_FLAG") + if p.pure: + flags.append("HAS_PURE_FLAG") + if p.oparg_and_1: + flags.append("HAS_OPARG_AND_1_FLAG") + if flags: + return " | ".join(flags) + else: + return "0" diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py deleted file mode 100644 index c6b551675e3e7e..00000000000000 --- a/Tools/cases_generator/instructions.py +++ /dev/null @@ -1,355 +0,0 @@ -import dataclasses -import re -import typing - -from flags import InstructionFlags, variable_used, variable_used_unspecialized -from formatting import ( - Formatter, - UNUSED, - list_effect_size, -) -import lexer as lx -import parsing -from parsing import StackEffect -import stacking - -BITS_PER_CODE_UNIT = 16 - - -@dataclasses.dataclass -class ActiveCacheEffect: - """Wraps a CacheEffect that is actually used, in context.""" - - effect: parsing.CacheEffect - offset: int - - -FORBIDDEN_NAMES_IN_UOPS = ( - "resume_with_error", - "kwnames", - "next_instr", - "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST - "JUMPBY", - "DISPATCH", - "INSTRUMENTED_JUMP", - "throwflag", - "exception_unwind", - "import_from", - "import_name", - "_PyObject_CallNoArgs", # Proxy for BEFORE_WITH - "TIER_ONE_ONLY", -) - - -# Interpreter tiers -TIER_ONE: typing.Final = 1 # Specializing adaptive interpreter (PEP 659) -TIER_TWO: typing.Final = 2 # Experimental tracing interpreter -Tiers: typing.TypeAlias = typing.Literal[1, 2] - - -@dataclasses.dataclass -class Instruction: - """An instruction with additional data and code.""" - - # Parts of the underlying instruction definition - inst: parsing.InstDef - name: str - block: parsing.Block - block_text: list[str] # Block.text, less curlies, less PREDICT() calls - block_line: int # First line of block in original code - - # Computed by constructor - always_exits: str # If the block always exits, its last line; else "" - has_deopt: bool - cache_offset: int - cache_effects: list[parsing.CacheEffect] - input_effects: list[StackEffect] - output_effects: list[StackEffect] - unmoved_names: frozenset[str] - instr_fmt: str - instr_flags: InstructionFlags - active_caches: list[ActiveCacheEffect] - - # Set later - family: parsing.Family | None = None - predicted: bool = False - - def __init__(self, inst: parsing.InstDef): - self.inst = inst - self.name = inst.name - self.block = inst.block - self.block_text, self.check_eval_breaker, self.block_line = extract_block_text( - self.block - ) - self.always_exits = always_exits(self.block_text) - self.has_deopt = variable_used(self.inst, "DEOPT_IF") - self.cache_effects = [ - effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) - ] - self.cache_offset = sum(c.size for c in self.cache_effects) - self.input_effects = [ - effect for effect in inst.inputs if isinstance(effect, StackEffect) - ] - self.output_effects = inst.outputs # For consistency/completeness - unmoved_names: set[str] = set() - for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect == oeffect and ieffect.name == oeffect.name: - unmoved_names.add(ieffect.name) - else: - break - self.unmoved_names = frozenset(unmoved_names) - - self.instr_flags = InstructionFlags.fromInstruction(inst) - - self.active_caches = [] - offset = 0 - for effect in self.cache_effects: - if effect.name != UNUSED: - self.active_caches.append(ActiveCacheEffect(effect, offset)) - offset += effect.size - - if self.instr_flags.HAS_ARG_FLAG: - fmt = "IB" - else: - fmt = "IX" - if offset: - fmt += "C" + "0" * (offset - 1) - self.instr_fmt = fmt - - def is_viable_uop(self) -> bool: - """Whether this instruction is viable as a uop.""" - dprint: typing.Callable[..., None] = lambda *args, **kwargs: None - if "FRAME" in self.name: - dprint = print - - if self.name == "_EXIT_TRACE": - return True # This has 'return frame' but it's okay - if self.always_exits: - dprint(f"Skipping {self.name} because it always exits: {self.always_exits}") - return False - if len(self.active_caches) > 1: - # print(f"Skipping {self.name} because it has >1 cache entries") - return False - res = True - for forbidden in FORBIDDEN_NAMES_IN_UOPS: - # NOTE: To disallow unspecialized uops, use - # if variable_used(self.inst, forbidden): - if variable_used_unspecialized(self.inst, forbidden): - dprint(f"Skipping {self.name} because it uses {forbidden}") - res = False - return res - - def write_body( - self, - out: Formatter, - dedent: int, - active_caches: list[ActiveCacheEffect], - tier: Tiers, - family: parsing.Family | None, - ) -> None: - """Write the instruction body.""" - # Write cache effect variable declarations and initializations - for active in active_caches: - ceffect = active.effect - bits = ceffect.size * BITS_PER_CODE_UNIT - if bits == 64: - # NOTE: We assume that 64-bit data in the cache - # is always an object pointer. - # If this becomes false, we need a way to specify - # syntactically what type the cache data is. - typ = "PyObject *" - func = "read_obj" - else: - typ = f"uint{bits}_t " - func = f"read_u{bits}" - if tier == TIER_ONE: - out.emit( - f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);" - ) - else: - out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") - - # Write the body, substituting a goto for ERROR_IF() and other stuff - assert dedent <= 0 - extra = " " * -dedent - names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) - offset = 0 - context = self.block.context - assert context is not None and context.owner is not None - filename = context.owner.filename - for line in self.block_text: - out.set_lineno(self.block_line + offset, filename) - offset += 1 - if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): - space, cond, label = m.groups() - space = extra + space - # ERROR_IF() must pop the inputs from the stack. - # The code block is responsible for DECREF()ing them. - # NOTE: If the label doesn't exist, just add it to ceval.c. - - # Don't pop common input/output effects at the bottom! - # These aren't DECREF'ed so they can stay. - ieffs = list(self.input_effects) - oeffs = list(self.output_effects) - while ( - ieffs - and oeffs - and ieffs[0] == oeffs[0] - and ieffs[0].name == oeffs[0].name - ): - ieffs.pop(0) - oeffs.pop(0) - ninputs, symbolic = list_effect_size(ieffs) - if ninputs: - label = f"pop_{ninputs}_{label}" - if symbolic: - out.write_raw( - f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" - ) - else: - out.write_raw(f"{space}if ({cond}) goto {label};\n") - elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): - space, cond = m.groups() - space = extra + space - target = family.name if family else self.name - out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") - elif "DEOPT" in line: - filename = context.owner.filename - lineno = context.owner.tokens[context.begin].line - print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") - out.write_raw(extra + line) - elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): - out.reset_lineno() - space = extra + m.group(1) - for ieff in self.input_effects: - if ieff.name in names_to_skip: - continue - if ieff.size: - out.write_raw( - f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" - ) - out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") - out.write_raw(f"{space}}}\n") - else: - decref = "XDECREF" if ieff.cond else "DECREF" - out.write_raw(f"{space}Py_{decref}({ieff.name});\n") - else: - out.write_raw(extra + line) - out.reset_lineno() - - -InstructionOrCacheEffect = Instruction | parsing.CacheEffect - - -# Instruction used for abstract interpretation. -class AbstractInstruction(Instruction): - def __init__(self, inst: parsing.InstDef): - super().__init__(inst) - - def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: - """Write one abstract instruction, sans prologue and epilogue.""" - stacking.write_single_instr_for_abstract_interp(self, out) - - def write_body( - self, - out: Formatter, - dedent: int, - active_caches: list[ActiveCacheEffect], - tier: Tiers, - family: parsing.Family | None, - ) -> None: - pass - - -@dataclasses.dataclass -class Component: - instr: Instruction - active_caches: list[ActiveCacheEffect] - - -MacroParts = list[Component | parsing.CacheEffect] - - -@dataclasses.dataclass -class MacroInstruction: - """A macro instruction.""" - - name: str - instr_fmt: str - instr_flags: InstructionFlags - macro: parsing.Macro - parts: MacroParts - cache_offset: int - # Set later - predicted: bool = False - family: parsing.Family | None = None - - -@dataclasses.dataclass -class PseudoInstruction: - """A pseudo instruction.""" - - name: str - targets: list[Instruction] - instr_fmt: str - instr_flags: InstructionFlags - - -AnyInstruction = Instruction | MacroInstruction | PseudoInstruction - - -def extract_block_text(block: parsing.Block) -> tuple[list[str], bool, int]: - # Get lines of text with proper dedent - blocklines = block.text.splitlines(True) - first_token: lx.Token = block.tokens[0] # IndexError means the context is broken - block_line = first_token.begin[0] - - # Remove blank lines from both ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - block_line += 1 - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Remove leading and trailing braces - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - block_line += 1 - - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Separate CHECK_EVAL_BREAKER() macro from end - check_eval_breaker = ( - blocklines != [] and blocklines[-1].strip() == "CHECK_EVAL_BREAKER();" - ) - if check_eval_breaker: - del blocklines[-1] - - return blocklines, check_eval_breaker, block_line - - -def always_exits(lines: list[str]) -> str: - """Determine whether a block always ends in a return/goto/etc.""" - if not lines: - return "" - line = lines[-1].rstrip() - # Indent must match exactly (TODO: Do something better) - if line[:12] != " " * 12: - return "" - line = line[12:] - if line.startswith( - ( - "goto ", - "return ", - "DISPATCH", - "GO_TO_", - "Py_UNREACHABLE()", - "ERROR_IF(true, ", - ) - ): - return line - return "" diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 5c4238756748a7..ba09931c541646 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -6,7 +6,7 @@ The CPython interpreter is defined in C, meaning that the semantics of the bytecode instructions, the dispatching mechanism, error handling, and tracing and instrumentation are all intermixed. -This document proposes defining a custom C-like DSL for defining the +This document proposes defining a custom C-like DSL for defining the instruction semantics and tools for generating the code deriving from the instruction definitions. @@ -15,6 +15,7 @@ These tools would be used to: * Generate the tier 2 interpreter * Generate documentation for instructions * Generate metadata about instructions, such as stack use (done). +* Generate the tier 2 optimizer's abstract interpreter. Having a single definition file ensures that there is a single source of truth for bytecode semantics. @@ -45,7 +46,7 @@ passes from the semantic definition, reducing errors. As we improve the performance of CPython, we need to optimize larger regions of code, use more complex optimizations and, ultimately, translate to machine -code. +code. All of these steps introduce the possibility of more bugs, and require more code to be written. One way to mitigate this is through the use of code generators. @@ -61,7 +62,7 @@ blocks as the instructions for the tier 1 (PEP 659) interpreter. Rewriting all the instructions is tedious and error-prone, and changing the instructions is a maintenance headache as both versions need to be kept in sync. -By using a code generator and using a common source for the instructions, or +By using a code generator and using a common source for the instructions, or parts of instructions, we can reduce the potential for errors considerably. @@ -74,7 +75,7 @@ We update it as the need arises. Each op definition has a kind, a name, a stack and instruction stream effect, and a piece of C code describing its semantics:: - + ``` file: (definition | family | pseudo)+ @@ -85,7 +86,7 @@ and a piece of C code describing its semantics:: "op" "(" NAME "," stack_effect ")" "{" C-code "}" | "macro" "(" NAME ")" "=" uop ("+" uop)* ";" - + stack_effect: "(" [inputs] "--" [outputs] ")" @@ -123,7 +124,13 @@ and a piece of C code describing its semantics:: "family" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" pseudo: - "pseudo" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" + "pseudo" "(" NAME "," stack_effect ["," "(" flags ")"]")" = "{" NAME ("," NAME)+ [","] "}" ";" + + flags: + flag ("|" flag)* + + flag: + HAS_ARG | HAS_DEOPT | etc.. ``` The following definitions may occur: @@ -140,6 +147,7 @@ The objects before the "--" are the objects on top of the stack at the start of the instruction. Those after the "--" are the objects on top of the stack at the end of the instruction. + An `inst` without `stack_effect` is a transitional form to allow the original C code definitions to be copied. It lacks information to generate anything other than the interpreter, but is useful for initial porting of code. @@ -158,6 +166,16 @@ By convention cache effects (`stream`) must precede the input effects. The name `oparg` is pre-defined as a 32 bit value fetched from the instruction stream. +### Special instruction annotations + +Instruction headers may be prefixed by one or more annotations. The non-exhaustive +list of annotations and their meanings are as follows: + +* `override`. For external use by other interpreter definitions to override the current + instruction definition. +* `pure`. This instruction has no side effects. +* 'tierN'. This instruction only used by tier N interpreter. + ### Special functions/macros The C code may include special functions that are understood by the tools as @@ -168,6 +186,7 @@ Those functions include: * `DEOPT_IF(cond, instruction)`. Deoptimize if `cond` is met. * `ERROR_IF(cond, label)`. Jump to error handler at `label` if `cond` is true. * `DECREF_INPUTS()`. Generate `Py_DECREF()` calls for the input stack effects. +* `SYNC_SP()`. Synchronizes the physical stack pointer with the stack effects. Note that the use of `DECREF_INPUTS()` is optional -- manual calls to `Py_DECREF()` or other approaches are also acceptable @@ -412,7 +431,7 @@ rather than popping and pushing, such that `LOAD_ATTR_SLOT` would look something stack_pointer += 1; } s1 = res; - } + } next_instr += (1 + 1 + 2 + 1 + 4); stack_pointer[-1] = s1; DISPATCH(); diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index a60f6c11a4c460..13aee94f2b957c 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -80,11 +80,12 @@ def choice(*opts: str) -> str: # Macros macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" -MACRO = "MACRO" +CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" IDENTIFIER = "IDENTIFIER" + suffix = r"([uU]?[lL]?[lL]?)" octal = r"0[0-7]+" + suffix hex = r"0[xX][0-9a-fA-F]+" @@ -111,7 +112,7 @@ def choice(*opts: str) -> str: char = r"\'.\'" # TODO: escape sequence CHARACTER = "CHARACTER" -comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +comment_re = r"(//.*)|/\*([^*]|\*[^/])*\*/" COMMENT = "COMMENT" newline = r"\n" @@ -173,10 +174,6 @@ def choice(*opts: str) -> str: kwds.append(INT) LONG = "LONG" kwds.append(LONG) -OVERRIDE = "OVERRIDE" -kwds.append(OVERRIDE) -REGISTER = "REGISTER" -kwds.append(REGISTER) OFFSETOF = "OFFSETOF" kwds.append(OFFSETOF) RESTRICT = "RESTRICT" @@ -207,8 +204,30 @@ def choice(*opts: str) -> str: kwds.append(VOLATILE) WHILE = "WHILE" kwds.append(WHILE) +# An instruction in the DSL +INST = "INST" +kwds.append(INST) +# A micro-op in the DSL +OP = "OP" +kwds.append(OP) +# A macro in the DSL +MACRO = "MACRO" +kwds.append(MACRO) keywords = {name.lower(): name for name in kwds} +ANNOTATION = "ANNOTATION" +annotations = { + "specializing", + "override", + "register", + "replaced", + "pure", + "split", + "replicate", + "tier1", + "tier2", +} + __all__ = [] __all__.extend(kwds) @@ -225,6 +244,7 @@ def make_syntax_error( @dataclass(slots=True) class Token: + filename: str kind: str text: str begin: tuple[int, int] @@ -252,7 +272,7 @@ def width(self) -> int: def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) - return Token(self.kind, txt, self.begin, self.end) + return Token(self.filename, self.kind, txt, self.begin, self.end) def __repr__(self) -> str: b0, b1 = self.begin @@ -263,13 +283,15 @@ def __repr__(self) -> str: return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: +def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() text = m.group(0) if text in keywords: kind = keywords[text] + elif text in annotations: + kind = ANNOTATION elif letter.match(text): kind = IDENTIFIER elif text == "...": @@ -289,7 +311,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T elif text[0] == "'": kind = CHARACTER elif text[0] == "#": - kind = MACRO + kind = CMACRO elif text[0] == "/" and text[1] in "/*": kind = COMMENT else: @@ -312,7 +334,9 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T else: begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start - linestart + len(text))) + yield Token( + filename, kind, text, begin, (line, start - linestart + len(text)) + ) def to_text(tkns: list[Token], dedent: int = 0) -> str: diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index e7175e263350b2..8e5a31851c596e 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -11,3 +11,5 @@ strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True +allow_redefinition = True +implicit_reexport = True diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py new file mode 100644 index 00000000000000..7932379b02dbff --- /dev/null +++ b/Tools/cases_generator/opcode_id_generator.py @@ -0,0 +1,62 @@ +"""Generate the list of opcode IDs. +Reads the instruction definitions from bytecodes.c. +Writes the IDs to opcode_ids.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h" + + +def generate_opcode_header( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_OPCODE_IDS_H"): + out.emit("/* Instruction opcodes for compiled code */\n") + + def write_define(name: str, op: int) -> None: + out.emit(f"#define {name:<38} {op:>3}\n") + + for op, name in sorted([(op, name) for (name, op) in analysis.opmap.items()]): + write_define(name, op) + + out.emit("\n") + write_define("HAVE_ARGUMENT", analysis.have_arg) + write_define("MIN_INSTRUMENTED_OPCODE", analysis.min_instrumented) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with all opcode IDs.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_opcode_header(args.input, data, outfile) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py new file mode 100644 index 00000000000000..0f5790dc4af40f --- /dev/null +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -0,0 +1,394 @@ +"""Generate opcode metadata. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to pycore_opcode_metadata.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + PseudoInstruction, + analyze_files, + Uop, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + cflags, +) +from cwriter import CWriter +from typing import TextIO +from stack import get_stack_effect + +# Constants used instead of size for macro expansions. +# Note: 1, 2, 4 must match actual cache entry sizes. +OPARG_KINDS = { + "OPARG_FULL": 0, + "OPARG_CACHE_1": 1, + "OPARG_CACHE_2": 2, + "OPARG_CACHE_4": 4, + "OPARG_TOP": 5, + "OPARG_BOTTOM": 6, + "OPARG_SAVE_RETURN_OFFSET": 7, + # Skip 8 as the other powers of 2 are sizes + "OPARG_REPLACED": 9, +} + +FLAGS = [ + "ARG", + "CONST", + "NAME", + "JUMP", + "FREE", + "LOCAL", + "EVAL_BREAK", + "DEOPT", + "ERROR", + "ESCAPES", + "EXIT", + "PURE", + "PASSTHROUGH", + "OPARG_AND_1", + "ERROR_NO_POP", +] + + +def generate_flag_macros(out: CWriter) -> None: + for i, flag in enumerate(FLAGS): + out.emit(f"#define HAS_{flag}_FLAG ({1< None: + for name, value in OPARG_KINDS.items(): + out.emit(f"#define {name} {value}\n") + out.emit("\n") + + +def emit_stack_effect_function( + out: CWriter, direction: str, data: list[tuple[str, str]] +) -> None: + out.emit(f"extern int _PyOpcode_num_{direction}(int opcode, int oparg);\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"int _PyOpcode_num_{direction}(int opcode, int oparg) {{\n") + out.emit("switch(opcode) {\n") + for name, effect in data: + out.emit(f"case {name}:\n") + out.emit(f" return {effect};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") + out.emit("#endif\n\n") + + +def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: + popped_data: list[tuple[str, str]] = [] + pushed_data: list[tuple[str, str]] = [] + def add(inst: Instruction | PseudoInstruction) -> None: + stack = get_stack_effect(inst) + popped = (-stack.base_offset).to_c() + pushed = (stack.top_offset - stack.base_offset).to_c() + popped_data.append((inst.name, popped)) + pushed_data.append((inst.name, pushed)) + + for inst in analysis.instructions.values(): + add(inst) + for pseudo in analysis.pseudos.values(): + add(pseudo) + + emit_stack_effect_function(out, "popped", sorted(popped_data)) + emit_stack_effect_function(out, "pushed", sorted(pushed_data)) + + +def generate_is_pseudo(analysis: Analysis, out: CWriter) -> None: + """Write the IS_PSEUDO_INSTR macro""" + out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\\n") + for op in analysis.pseudos: + out.emit(f"((OP) == {op}) || \\\n") + out.emit("0") + out.emit(")\n\n") + + +def get_format(inst: Instruction) -> str: + if inst.properties.oparg: + format = "INSTR_FMT_IB" + else: + format = "INSTR_FMT_IX" + if inst.size > 1: + format += "C" + format += "0" * (inst.size - 2) + return format + + +def generate_instruction_formats(analysis: Analysis, out: CWriter) -> None: + # Compute the set of all instruction formats. + formats: set[str] = set() + for inst in analysis.instructions.values(): + formats.add(get_format(inst)) + # Generate an enum for it + out.emit("enum InstructionFormat {\n") + next_id = 1 + for format in sorted(formats): + out.emit(f"{format} = {next_id},\n") + next_id += 1 + out.emit("};\n\n") + + +def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_Deopt[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_Deopt[256] = {\n") + deopts: list[tuple[str, str]] = [] + for inst in analysis.instructions.values(): + deopt = inst.name + if inst.family is not None: + deopt = inst.family.name + deopts.append((inst.name, deopt)) + deopts.append(("INSTRUMENTED_LINE", "INSTRUMENTED_LINE")) + for name, deopt in sorted(deopts): + out.emit(f"[{name}] = {deopt},\n") + out.emit("};\n\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def generate_cache_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_Caches[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_Caches[256] = {\n") + for inst in analysis.instructions.values(): + if inst.family and inst.family.name != inst.name: + continue + if inst.name.startswith("INSTRUMENTED"): + continue + if inst.size > 1: + out.emit(f"[{inst.name}] = {inst.size-1},\n") + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_name_table(analysis: Analysis, out: CWriter) -> None: + table_size = 256 + len(analysis.pseudos) + out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"const char *_PyOpcode_OpName[{table_size}] = {{\n") + names = list(analysis.instructions) + list(analysis.pseudos) + names.append("INSTRUMENTED_LINE") + for name in sorted(names): + out.emit(f'[{name}] = "{name}",\n') + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: + table_size = 256 + len(analysis.pseudos) + out.emit("struct opcode_metadata {\n") + out.emit("uint8_t valid_entry;\n") + out.emit("int8_t instr_format;\n") + out.emit("int16_t flags;\n") + out.emit("};\n\n") + out.emit( + f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit( + f"const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}] = {{\n" + ) + for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): + out.emit( + f"[{inst.name}] = {{ true, {get_format(inst)}, {cflags(inst.properties)} }},\n" + ) + for pseudo in sorted(analysis.pseudos.values(), key=lambda t: t.name): + flags = cflags(pseudo.properties) + for flag in pseudo.flags: + if flags == "0": + flags = f"{flag}_FLAG" + else: + flags += f" | {flag}_FLAG" + out.emit(f"[{pseudo.name}] = {{ true, -1, {flags} }},\n") + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: + expansions_table: dict[str, list[tuple[str, int, int]]] = {} + for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): + offset: int = 0 # Cache effect offset + expansions: list[tuple[str, int, int]] = [] # [(name, size, offset), ...] + if inst.is_super(): + pieces = inst.name.split("_") + assert len(pieces) == 4, f"{inst.name} doesn't look like a super-instr" + name1 = "_".join(pieces[:2]) + name2 = "_".join(pieces[2:]) + assert name1 in analysis.instructions, f"{name1} doesn't match any instr" + assert name2 in analysis.instructions, f"{name2} doesn't match any instr" + instr1 = analysis.instructions[name1] + instr2 = analysis.instructions[name2] + assert ( + len(instr1.parts) == 1 + ), f"{name1} is not a good superinstruction part" + assert ( + len(instr2.parts) == 1 + ), f"{name2} is not a good superinstruction part" + expansions.append((instr1.parts[0].name, OPARG_KINDS["OPARG_TOP"], 0)) + expansions.append((instr2.parts[0].name, OPARG_KINDS["OPARG_BOTTOM"], 0)) + elif not is_viable_expansion(inst): + continue + else: + for part in inst.parts: + size = part.size + if part.name == "_SAVE_RETURN_OFFSET": + size = OPARG_KINDS["OPARG_SAVE_RETURN_OFFSET"] + if isinstance(part, Uop): + # Skip specializations + if "specializing" in part.annotations: + continue + if "replaced" in part.annotations: + size = OPARG_KINDS["OPARG_REPLACED"] + expansions.append((part.name, size, offset if size else 0)) + offset += part.size + expansions_table[inst.name] = expansions + max_uops = max(len(ex) for ex in expansions_table.values()) + out.emit(f"#define MAX_UOP_PER_EXPANSION {max_uops}\n") + out.emit("struct opcode_macro_expansion {\n") + out.emit("int nuops;\n") + out.emit( + "struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION];\n" + ) + out.emit("};\n") + out.emit( + "extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256];\n\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const struct opcode_macro_expansion\n") + out.emit("_PyOpcode_macro_expansion[256] = {\n") + for inst_name, expansions in expansions_table.items(): + uops = [ + f"{{ {name}, {size}, {offset} }}" for (name, size, offset) in expansions + ] + out.emit( + f'[{inst_name}] = {{ .nuops = {len(expansions)}, .uops = {{ {", ".join(uops)} }} }},\n' + ) + out.emit("};\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def is_viable_expansion(inst: Instruction) -> bool: + "An instruction can be expanded if all its parts are viable for tier 2" + for part in inst.parts: + if isinstance(part, Uop): + # Skip specializing and replaced uops + if "specializing" in part.annotations: + continue + if "replaced" in part.annotations: + continue + if part.properties.tier == 1 or not part.is_viable(): + return False + return True + + +def generate_extra_cases(analysis: Analysis, out: CWriter) -> None: + out.emit("#define EXTRA_CASES \\\n") + valid_opcodes = set(analysis.opmap.values()) + for op in range(256): + if op not in valid_opcodes: + out.emit(f" case {op}: \\\n") + out.emit(" ;\n") + + +def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: + table_size = len(analysis.pseudos) + max_targets = max(len(pseudo.targets) for pseudo in analysis.pseudos.values()) + out.emit("struct pseudo_targets {\n") + out.emit(f"uint8_t targets[{max_targets + 1}];\n") + out.emit("};\n") + out.emit( + f"extern const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}];\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit( + f"const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}] = {{\n" + ) + for pseudo in analysis.pseudos.values(): + targets = ["0"] * (max_targets + 1) + for i, target in enumerate(pseudo.targets): + targets[i] = target.name + out.emit(f"[{pseudo.name}-256] = {{ {{ {', '.join(targets)} }} }},\n") + out.emit("};\n\n") + out.emit("#endif // NEED_OPCODE_METADATA\n") + out.emit("static inline bool\n") + out.emit("is_pseudo_target(int pseudo, int target) {\n") + out.emit(f"if (pseudo < 256 || pseudo >= {256+table_size}) {{\n") + out.emit(f"return false;\n") + out.emit("}\n") + out.emit( + f"for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) {{\n" + ) + out.emit( + f"if (_PyOpcode_PseudoTargets[pseudo-256].targets[i] == target) return true;\n" + ) + out.emit("}\n") + out.emit(f"return false;\n") + out.emit("}\n\n") + + +def generate_opcode_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_OPCODE_METADATA_H"): + out.emit("#ifndef Py_BUILD_CORE\n") + out.emit('# error "this header requires Py_BUILD_CORE define"\n') + out.emit("#endif\n\n") + out.emit("#include // bool\n") + out.emit('#include "opcode_ids.h"\n') + generate_is_pseudo(analysis, out) + out.emit('#include "pycore_uop_ids.h"\n') + generate_stack_effect_functions(analysis, out) + generate_instruction_formats(analysis, out) + table_size = 256 + len(analysis.pseudos) + out.emit("#define IS_VALID_OPCODE(OP) \\\n") + out.emit(f" (((OP) >= 0) && ((OP) < {table_size}) && \\\n") + out.emit(" (_PyOpcode_opcode_metadata[(OP)].valid_entry))\n\n") + generate_flag_macros(out) + generate_oparg_macros(out) + generate_metadata_table(analysis, out) + generate_expansion_table(analysis, out) + generate_name_table(analysis, out) + generate_cache_table(analysis, out) + generate_deopt_table(analysis, out) + generate_extra_cases(analysis, out) + generate_pseudo_targets(analysis, out) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with opcode metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_opcode_metadata.h" + + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_opcode_metadata(args.input, data, outfile) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py new file mode 100644 index 00000000000000..277521678434c2 --- /dev/null +++ b/Tools/cases_generator/optimizer_generator.py @@ -0,0 +1,232 @@ +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + analyze_files, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + replace_sync_sp, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Stack, SizeMismatch + +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() + + +def validate_uop(override: Uop, uop: Uop) -> None: + # To do + pass + + +def type_name(var: StackItem) -> str: + if var.is_array(): + return f"_Py_UopsSymbol **" + if var.type: + return var.type + return f"_Py_UopsSymbol *" + + +def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: + variables = {"unused"} + if not skip_inputs: + for var in reversed(uop.stack.inputs): + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + for var in uop.stack.outputs: + if var.peek: + continue + if var.name not in variables: + variables.add(var.name) + if var.condition: + out.emit(f"{type_name(var)}{var.name} = NULL;\n") + else: + out.emit(f"{type_name(var)}{var.name};\n") + + +def decref_inputs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + + +def emit_default(out: CWriter, uop: Uop) -> None: + for i, var in enumerate(uop.stack.outputs): + if var.name != "unused" and not var.peek: + if var.is_array(): + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") + elif var.name == "null": + out.emit(f"{var.name} = sym_new_null(ctx);\n") + else: + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + + +def write_uop( + override: Uop | None, + uop: Uop, + out: CWriter, + stack: Stack, + debug: bool, + skip_inputs: bool, +) -> None: + try: + prototype = override if override else uop + is_override = override is not None + out.start_line() + for var in reversed(prototype.stack.inputs): + res = stack.pop(var, extract_bits=True) + if not skip_inputs: + out.emit(res) + if not prototype.properties.stores_sp: + for i, var in enumerate(prototype.stack.outputs): + res = stack.push(var) + if not var.peek or is_override: + out.emit(res) + if debug: + args = [] + for var in prototype.stack.inputs: + if not var.peek or is_override: + args.append(var.name) + out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') + if override: + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") + if override: + replacement_funcs = { + "DECREF_INPUTS": decref_inputs, + "SYNC_SP": replace_sync_sp, + } + emit_tokens(out, override, stack, None, replacement_funcs) + else: + emit_default(out, uop) + + if prototype.properties.stores_sp: + for i, var in enumerate(prototype.stack.outputs): + if not var.peek or is_override: + out.emit(stack.push(var)) + out.start_line() + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_abstract_interpreter( + filenames: list[str], + abstract: Analysis, + base: Analysis, + outfile: TextIO, + debug: bool, +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + base_uop_names = set([uop.name for uop in base.uops.values()]) + for abstract_uop_name in abstract.uops: + assert abstract_uop_name in base_uop_names,\ + f"All abstract uops should override base uops, but {abstract_uop_name} is not." + + for uop in base.uops.values(): + override: Uop | None = None + if uop.name in abstract.uops: + override = abstract.uops[uop.name] + validate_uop(override, uop) + if uop.properties.tier == 1: + continue + if uop.replicates: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + if override: + declare_variables(override, out, skip_inputs=False) + else: + declare_variables(uop, out, skip_inputs=True) + stack = Stack() + write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + out.start_line() + out.emit("break;\n") + out.emit("}") + out.emit("\n\n") + + +def generate_tier2_abstract_from_files( + filenames: list[str], outfilename: str, debug: bool=False +) -> None: + assert len(filenames) == 2, "Need a base file and an abstract cases file." + base = analyze_files([filenames[0]]) + abstract = analyze_files([filenames[1]]) + with open(outfilename, "w") as outfile: + generate_abstract_interpreter(filenames, abstract, base, outfile, debug) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + + +arg_parser.add_argument("input", nargs='*', help="Abstract interpreter definition file") + +arg_parser.add_argument( + "base", nargs="*", help="The base instruction definition file(s)" +) + +arg_parser.add_argument("-d", "--debug", help="Insert debug calls", action="store_true") + +if __name__ == "__main__": + args = arg_parser.parse_args() + if not args.input: + args.base.append(DEFAULT_INPUT) + args.input.append(DEFAULT_ABSTRACT_INPUT) + else: + args.base.append(args.input[-1]) + args.input.pop() + abstract = analyze_files(args.input) + base = analyze_files(args.base) + with open(args.output, "w") as outfile: + generate_abstract_interpreter(args.input, abstract, base, outfile, args.debug) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py new file mode 100644 index 00000000000000..db672ad5501f15 --- /dev/null +++ b/Tools/cases_generator/parser.py @@ -0,0 +1,66 @@ +from parsing import ( # noqa: F401 + InstDef, + Macro, + Pseudo, + Family, + Parser, + Context, + CacheEffect, + StackEffect, + InputEffect, + OpName, + AstNode, +) + + +def prettify_filename(filename: str) -> str: + # Make filename more user-friendly and less platform-specific, + # it is only used for error reporting at this point. + filename = filename.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] + if filename.endswith(".new"): + filename = filename[:-4] + return filename + + +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" + + +def parse_files(filenames: list[str]) -> list[AstNode]: + result: list[AstNode] = [] + for filename in filenames: + with open(filename) as file: + src = file.read() + + psr = Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing_first_token = psr.peek() + while node := psr.definition(): + assert node is not None + result.append(node) # type: ignore[arg-type] + if not psr.eof(): + psr.backup() + raise psr.make_syntax_error( + f"Extra stuff at the end of {filename}", psr.next(True) + ) + return result diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 25be5ca3e0da5d..8957838f7a90a1 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -105,8 +105,7 @@ class OpName(Node): @dataclass class InstHeader(Node): - override: bool - register: bool + annotations: list[str] kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -115,8 +114,7 @@ class InstHeader(Node): @dataclass class InstDef(Node): - override: bool - register: bool + annotations: list[str] kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -140,20 +138,26 @@ class Family(Node): @dataclass class Pseudo(Node): name: str + inputs: list[InputEffect] + outputs: list[OutputEffect] + flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by +AstNode = InstDef | Macro | Pseudo | Family + + class Parser(PLexer): @contextual - def definition(self) -> InstDef | Macro | Pseudo | Family | None: - if inst := self.inst_def(): - return inst + def definition(self) -> AstNode | None: if macro := self.macro_def(): return macro if family := self.family_def(): return family if pseudo := self.pseudo_def(): return pseudo + if inst := self.inst_def(): + return inst return None @contextual @@ -161,8 +165,7 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.override, - hdr.register, + hdr.annotations, hdr.kind, hdr.name, hdr.inputs, @@ -174,13 +177,21 @@ def inst_def(self) -> InstDef | None: @contextual def inst_header(self) -> InstHeader | None: - # [override] inst(NAME) - # | [override] [register] inst(NAME, (inputs -- outputs)) - # | [override] [register] op(NAME, (inputs -- outputs)) - # TODO: Make INST a keyword in the lexer. - override = bool(self.expect(lx.OVERRIDE)) - register = bool(self.expect(lx.REGISTER)) - if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text in ("inst", "op"): + # annotation* inst(NAME, (inputs -- outputs)) + # | annotation* op(NAME, (inputs -- outputs)) + annotations = [] + while anno := self.expect(lx.ANNOTATION): + if anno.text == "replicate": + self.require(lx.LPAREN) + times = self.require(lx.NUMBER) + self.require(lx.RPAREN) + annotations.append(f"replicate({times.text})") + else: + annotations.append(anno.text) + tkn = self.expect(lx.INST) + if not tkn: + tkn = self.expect(lx.OP) + if tkn: kind = cast(Literal["inst", "op"], tkn.text) if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text @@ -188,7 +199,7 @@ def inst_header(self) -> InstHeader | None: inp, outp = self.io_effect() if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: - return InstHeader(override, register, kind, name, inp, outp) + return InstHeader(annotations, kind, name, inp, outp) return None def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: @@ -252,7 +263,7 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] + # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" @@ -274,7 +285,6 @@ def stack_effect(self) -> StackEffect | None: if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) - type_text = "PyObject **" size_text = size.text.strip() return StackEffect(tkn.text, type_text, cond_text, size_text) return None @@ -312,7 +322,7 @@ def op(self) -> OpName | None: @contextual def macro_def(self) -> Macro | None: - if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "macro": + if tkn := self.expect(lx.MACRO): if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.RPAREN): @@ -363,7 +373,9 @@ def family_def(self) -> Family | None: if self.expect(lx.COMMA): if not (size := self.expect(lx.IDENTIFIER)): if not (size := self.expect(lx.NUMBER)): - raise self.make_syntax_error("Expected identifier or number") + raise self.make_syntax_error( + "Expected identifier or number" + ) if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): @@ -375,19 +387,41 @@ def family_def(self) -> Family | None: ) return None + def flags(self) -> list[str]: + here = self.getpos() + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + flags = [tkn.text] + while self.expect(lx.COMMA): + if tkn := self.expect(lx.IDENTIFIER): + flags.append(tkn.text) + else: + break + if not self.expect(lx.RPAREN): + raise self.make_syntax_error("Expected comma or right paren") + return flags + self.setpos(here) + return [] + @contextual def pseudo_def(self) -> Pseudo | None: if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "pseudo": size = None if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): - if self.expect(lx.RPAREN): - if self.expect(lx.EQUALS): - if not self.expect(lx.LBRACE): - raise self.make_syntax_error("Expected {") - if members := self.members(): - if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo(tkn.text, members) + if self.expect(lx.COMMA): + inp, outp = self.io_effect() + if self.expect(lx.COMMA): + flags = self.flags() + else: + flags = [] + if self.expect(lx.RPAREN): + if self.expect(lx.EQUALS): + if not self.expect(lx.LBRACE): + raise self.make_syntax_error("Expected {") + if members := self.members(): + if self.expect(lx.RBRACE) and self.expect(lx.SEMI): + return Pseudo(tkn.text, inp, outp, flags, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py new file mode 100644 index 00000000000000..3f7ffbc5523fd0 --- /dev/null +++ b/Tools/cases_generator/py_metadata_generator.py @@ -0,0 +1,96 @@ +"""Generate opcode metadata for Python. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to _opcode_metadata.py by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + + +DEFAULT_OUTPUT = ROOT / "Lib/_opcode_metadata.py" + + +def get_specialized(analysis: Analysis) -> set[str]: + specialized: set[str] = set() + for family in analysis.families.values(): + for member in family.members: + specialized.add(member.name) + return specialized + + +def generate_specializations(analysis: Analysis, out: CWriter) -> None: + out.emit("_specializations = {\n") + for family in analysis.families.values(): + out.emit(f'"{family.name}": [\n') + for member in family.members: + out.emit(f' "{member.name}",\n') + out.emit("],\n") + out.emit("}\n\n") + + +def generate_specialized_opmap(analysis: Analysis, out: CWriter) -> None: + out.emit("_specialized_opmap = {\n") + names = [] + for family in analysis.families.values(): + for member in family.members: + if member.name == family.name: + continue + names.append(member.name) + for name in sorted(names): + out.emit(f"'{name}': {analysis.opmap[name]},\n") + out.emit("}\n\n") + + +def generate_opmap(analysis: Analysis, out: CWriter) -> None: + specialized = get_specialized(analysis) + out.emit("opmap = {\n") + for inst, op in analysis.opmap.items(): + if inst not in specialized: + out.emit(f"'{inst}': {analysis.opmap[inst]},\n") + out.emit("}\n\n") + + +def generate_py_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile, "#") + out = CWriter(outfile, 0, False) + generate_specializations(analysis, out) + generate_specialized_opmap(analysis, out) + generate_opmap(analysis, out) + out.emit(f"HAVE_ARGUMENT = {analysis.have_arg}\n") + out.emit(f"MIN_INSTRUMENTED_OPCODE = {analysis.min_instrumented}\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the Python file with opcode metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_py_metadata(args.input, data, outfile) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py new file mode 100644 index 00000000000000..ebe62df537f15f --- /dev/null +++ b/Tools/cases_generator/stack.py @@ -0,0 +1,232 @@ +import re +from analyzer import StackItem, StackEffect, Instruction, Uop, PseudoInstruction +from dataclasses import dataclass +from cwriter import CWriter +from typing import Iterator + +UNUSED = {"unused"} + + +def maybe_parenthesize(sym: str) -> str: + """Add parentheses around a string if it contains an operator + and is not already parenthesized. + + An exception is made for '*' which is common and harmless + in the context where the symbolic size is used. + """ + if sym.startswith("(") and sym.endswith(")"): + return sym + if re.match(r"^[\s\w*]+$", sym): + return sym + else: + return f"({sym})" + + +def var_size(var: StackItem) -> str: + if var.condition: + # Special case simplifications + if var.condition == "0": + return "0" + elif var.condition == "1": + return var.get_size() + elif var.condition == "oparg & 1" and not var.size: + return f"({var.condition})" + else: + return f"(({var.condition}) ? {var.get_size()} : 0)" + elif var.size: + return var.size + else: + return "1" + +@dataclass +class StackOffset: + "The stack offset of the virtual base of the stack from the physical stack pointer" + + popped: list[str] + pushed: list[str] + + @staticmethod + def empty() -> "StackOffset": + return StackOffset([], []) + + def pop(self, item: StackItem) -> None: + self.popped.append(var_size(item)) + + def push(self, item: StackItem) -> None: + self.pushed.append(var_size(item)) + + def __sub__(self, other: "StackOffset") -> "StackOffset": + return StackOffset(self.popped + other.pushed, self.pushed + other.popped) + + def __neg__(self) -> "StackOffset": + return StackOffset(self.pushed, self.popped) + + def simplify(self) -> None: + "Remove matching values from both the popped and pushed list" + if not self.popped or not self.pushed: + return + # Sort the list so the lexically largest element is last. + popped = sorted(self.popped) + pushed = sorted(self.pushed) + self.popped = [] + self.pushed = [] + while popped and pushed: + pop = popped.pop() + push = pushed.pop() + if pop == push: + pass + elif pop > push: + # if pop > push, there can be no element in pushed matching pop. + self.popped.append(pop) + pushed.append(push) + else: + self.pushed.append(push) + popped.append(pop) + self.popped.extend(popped) + self.pushed.extend(pushed) + + def to_c(self) -> str: + self.simplify() + int_offset = 0 + symbol_offset = "" + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + symbol_offset += f" - {maybe_parenthesize(item)}" + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + symbol_offset += f" + {maybe_parenthesize(item)}" + if symbol_offset and not int_offset: + res = symbol_offset + else: + res = f"{int_offset}{symbol_offset}" + if res.startswith(" + "): + res = res[3:] + if res.startswith(" - "): + res = "-" + res[3:] + return res + + def clear(self) -> None: + self.popped = [] + self.pushed = [] + + +class SizeMismatch(Exception): + pass + + +class Stack: + def __init__(self) -> None: + self.top_offset = StackOffset.empty() + self.base_offset = StackOffset.empty() + self.peek_offset = StackOffset.empty() + self.variables: list[StackItem] = [] + self.defined: set[str] = set() + + def pop(self, var: StackItem, extract_bits: bool = False) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch( + f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}" + ) + if popped.name == var.name: + return "" + elif popped.name in UNUSED: + self.defined.add(var.name) + return ( + f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + ) + elif var.name in UNUSED: + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name in UNUSED: + return "" + else: + self.defined.add(var.name) + cast = f"({var.type})" if (not indirect and var.type) else "" + bits = ".bits" if cast and not extract_bits else "" + assign = ( + f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" + ) + if var.condition: + if var.condition == "1": + return f"{assign}\n" + elif var.condition == "0": + return "" + else: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> str: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name not in UNUSED: + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: + out.start_line() + for var in self.variables: + if not var.peek: + cast = f"({cast_type})" if var.type else "" + bits = ".bits" if cast and not extract_bits else "" + if var.name not in UNUSED and not var.is_array(): + if var.condition: + if var.condition == "0": + continue + elif var.condition != "1": + out.emit(f"if ({var.condition}) ") + out.emit( + f"stack_pointer[{self.base_offset.to_c()}]{bits} = {cast}{var.name};\n" + ) + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + out.emit("assert(WITHIN_STACK_BOUNDS());\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + out.start_line() + + def as_comment(self) -> str: + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + + +def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: + stack = Stack() + def stacks(inst : Instruction | PseudoInstruction) -> Iterator[StackEffect]: + if isinstance(inst, Instruction): + for uop in inst.parts: + if isinstance(uop, Uop): + yield uop.stack + else: + assert isinstance(inst, PseudoInstruction) + yield inst.stack + + for s in stacks(inst): + for var in reversed(s.inputs): + stack.pop(var) + for var in s.outputs: + stack.push(var) + return stack diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py deleted file mode 100644 index bba2db8b059da8..00000000000000 --- a/Tools/cases_generator/stacking.py +++ /dev/null @@ -1,529 +0,0 @@ -import dataclasses -import typing - -from flags import variable_used_unspecialized -from formatting import ( - Formatter, - UNUSED, - maybe_parenthesize, - parenthesize_cond, -) -from instructions import ( - ActiveCacheEffect, - Instruction, - MacroInstruction, - Component, - Tiers, - TIER_ONE, -) -from parsing import StackEffect, CacheEffect, Family - - -@dataclasses.dataclass -class StackOffset: - """Represent the stack offset for a PEEK or POKE. - - - At stack_pointer[0], deep and high are both empty. - (Note that that is an invalid stack reference.) - - Below stack top, only deep is non-empty. - - Above stack top, only high is non-empty. - - In complex cases, both deep and high may be non-empty. - - All this would be much simpler if all stack entries were the same - size, but with conditional and array effects, they aren't. - The offsets are each represented by a list of StackEffect objects. - The name in the StackEffects is unused. - """ - - deep: list[StackEffect] = dataclasses.field(default_factory=list) - high: list[StackEffect] = dataclasses.field(default_factory=list) - - def clone(self) -> "StackOffset": - return StackOffset(list(self.deep), list(self.high)) - - def negate(self) -> "StackOffset": - return StackOffset(list(self.high), list(self.deep)) - - def deeper(self, eff: StackEffect) -> None: - if eff in self.high: - self.high.remove(eff) - else: - self.deep.append(eff) - - def higher(self, eff: StackEffect) -> None: - if eff in self.deep: - self.deep.remove(eff) - else: - self.high.append(eff) - - def as_terms(self) -> list[tuple[str, str]]: - num = 0 - terms: list[tuple[str, str]] = [] - for eff in self.deep: - if eff.size: - terms.append(("-", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond not in ("0", "1"): - terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) - elif eff.cond != "0": - num -= 1 - for eff in self.high: - if eff.size: - terms.append(("+", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond not in ("0", "1"): - terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) - elif eff.cond != "0": - num += 1 - if num < 0: - terms.insert(0, ("-", str(-num))) - elif num > 0: - terms.append(("+", str(num))) - return terms - - def as_index(self) -> str: - terms = self.as_terms() - return make_index(terms) - - def equivalent_to(self, other: "StackOffset") -> bool: - if self.deep == other.deep and self.high == other.high: - return True - deep = list(self.deep) - for x in other.deep: - try: - deep.remove(x) - except ValueError: - return False - if deep: - return False - high = list(self.high) - for x in other.high: - try: - high.remove(x) - except ValueError: - return False - if high: - return False - return True - - -def make_index(terms: list[tuple[str, str]]) -> str: - # Produce an index expression from the terms honoring PEP 8, - # surrounding binary ops with spaces but not unary minus - index = "" - for sign, term in terms: - if index: - index += f" {sign} {term}" - elif sign == "+": - index = term - else: - index = sign + term - return index or "0" - - -@dataclasses.dataclass -class StackItem: - offset: StackOffset - effect: StackEffect - - def as_variable(self, lax: bool = False) -> str: - """Return e.g. stack_pointer[-1].""" - terms = self.offset.as_terms() - if self.effect.size: - terms.insert(0, ("+", "stack_pointer")) - index = make_index(terms) - if self.effect.size: - res = index - else: - res = f"stack_pointer[{index}]" - if not lax: - # Check that we're not reading or writing above stack top. - # Skip this for output variable initialization (lax=True). - assert ( - self.effect in self.offset.deep and not self.offset.high - ), f"Push or pop above current stack level: {res}" - return res - - def as_stack_effect(self, lax: bool = False) -> StackEffect: - return StackEffect( - self.as_variable(lax=lax), - self.effect.type if self.effect.size else "", - self.effect.cond, - self.effect.size, - ) - - -@dataclasses.dataclass -class CopyItem: - src: StackItem - dst: StackItem - - -class EffectManager: - """Manage stack effects and offsets for an instruction.""" - - instr: Instruction - active_caches: list[ActiveCacheEffect] - peeks: list[StackItem] - pokes: list[StackItem] - copies: list[CopyItem] # See merge() - # Track offsets from stack pointer - min_offset: StackOffset - final_offset: StackOffset - # Link to previous manager - pred: "EffectManager | None" = None - - def __init__( - self, - instr: Instruction, - active_caches: list[ActiveCacheEffect], - pred: "EffectManager | None" = None, - ): - self.instr = instr - self.active_caches = active_caches - self.peeks = [] - self.pokes = [] - self.copies = [] - self.final_offset = pred.final_offset.clone() if pred else StackOffset() - for eff in reversed(instr.input_effects): - self.final_offset.deeper(eff) - self.peeks.append(StackItem(offset=self.final_offset.clone(), effect=eff)) - self.min_offset = self.final_offset.clone() - for eff in instr.output_effects: - self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) - self.final_offset.higher(eff) - - self.pred = pred - while pred: - # Replace push(x) + pop(y) with copy(x, y). - # Check that the sources and destinations are disjoint. - sources: set[str] = set() - destinations: set[str] = set() - while ( - pred.pokes - and self.peeks - and pred.pokes[-1].effect == self.peeks[0].effect - ): - src = pred.pokes.pop(-1) - dst = self.peeks.pop(0) - assert src.offset.equivalent_to(dst.offset), (src, dst) - pred.final_offset.deeper(src.effect) - if dst.effect.name != src.effect.name: - if dst.effect.name != UNUSED: - destinations.add(dst.effect.name) - if src.effect.name != UNUSED: - sources.add(src.effect.name) - self.copies.append(CopyItem(src, dst)) - # TODO: Turn this into an error (pass an Analyzer instance?) - assert sources & destinations == set(), ( - pred.instr.name, - self.instr.name, - sources, - destinations, - ) - # See if we can get more copies of a earlier predecessor. - if self.peeks and not pred.pokes and not pred.peeks: - pred = pred.pred - else: - pred = None # Break - - # Fix up patterns of copies through UNUSED, - # e.g. cp(a, UNUSED) + cp(UNUSED, b) -> cp(a, b). - if any(copy.src.effect.name == UNUSED for copy in self.copies): - pred = self.pred - while pred is not None: - for copy in self.copies: - if copy.src.effect.name == UNUSED: - for pred_copy in pred.copies: - if pred_copy.dst == copy.src: - copy.src = pred_copy.src - break - pred = pred.pred - - def adjust_deeper(self, eff: StackEffect) -> None: - for peek in self.peeks: - peek.offset.deeper(eff) - for poke in self.pokes: - poke.offset.deeper(eff) - for copy in self.copies: - copy.src.offset.deeper(eff) - copy.dst.offset.deeper(eff) - self.min_offset.deeper(eff) - self.final_offset.deeper(eff) - - def adjust_higher(self, eff: StackEffect) -> None: - for peek in self.peeks: - peek.offset.higher(eff) - for poke in self.pokes: - poke.offset.higher(eff) - for copy in self.copies: - copy.src.offset.higher(eff) - copy.dst.offset.higher(eff) - self.min_offset.higher(eff) - self.final_offset.higher(eff) - - def adjust(self, offset: StackOffset) -> None: - deep = list(offset.deep) - high = list(offset.high) - for down in deep: - self.adjust_deeper(down) - for up in high: - self.adjust_higher(up) - - def adjust_inverse(self, offset: StackOffset) -> None: - deep = list(offset.deep) - high = list(offset.high) - for down in deep: - self.adjust_higher(down) - for up in high: - self.adjust_deeper(up) - - def collect_vars(self) -> dict[str, StackEffect]: - """Collect all variables, skipping unused ones.""" - vars: dict[str, StackEffect] = {} - - def add(eff: StackEffect) -> None: - if eff.name != UNUSED: - if eff.name in vars: - # TODO: Make this an error - assert vars[eff.name] == eff, ( - self.instr.name, - eff.name, - vars[eff.name], - eff, - ) - else: - vars[eff.name] = eff - - for copy in self.copies: - add(copy.src.effect) - add(copy.dst.effect) - for peek in self.peeks: - add(peek.effect) - for poke in self.pokes: - add(poke.effect) - - return vars - - -def less_than(a: StackOffset, b: StackOffset) -> bool: - # TODO: Handle more cases - if a.high != b.high: - return False - return a.deep[: len(b.deep)] == b.deep - - -def get_managers(parts: list[Component]) -> list[EffectManager]: - managers: list[EffectManager] = [] - pred: EffectManager | None = None - for part in parts: - mgr = EffectManager(part.instr, part.active_caches, pred) - managers.append(mgr) - pred = mgr - return managers - - -def get_stack_effect_info_for_macro(mac: MacroInstruction) -> tuple[str, str]: - """Get the stack effect info for a macro instruction. - - Returns a tuple (popped, pushed) where each is a string giving a - symbolic expression for the number of values popped/pushed. - """ - parts = [part for part in mac.parts if isinstance(part, Component)] - managers = get_managers(parts) - popped = StackOffset() - for mgr in managers: - if less_than(mgr.min_offset, popped): - popped = mgr.min_offset.clone() - # Compute pushed = final - popped - pushed = managers[-1].final_offset.clone() - for effect in popped.deep: - pushed.higher(effect) - for effect in popped.high: - pushed.deeper(effect) - return popped.negate().as_index(), pushed.as_index() - - -def write_single_instr( - instr: Instruction, out: Formatter, tier: Tiers = TIER_ONE -) -> None: - try: - write_components( - [Component(instr, instr.active_caches)], - out, - tier, - 0, - instr.family, - ) - except AssertionError as err: - raise AssertionError(f"Error writing instruction {instr.name}") from err - - -def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: - parts = [ - part - for part in mac.parts - if isinstance(part, Component) and part.instr.name != "_SET_IP" - ] - out.emit("") - with out.block(f"TARGET({mac.name})"): - if mac.predicted: - out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) - try: - next_instr_is_set = write_components( - parts, out, TIER_ONE, mac.cache_offset, mac.family - ) - except AssertionError as err: - raise AssertionError(f"Error writing macro {mac.name}") from err - if not parts[-1].instr.always_exits: - if not next_instr_is_set and mac.cache_offset: - out.emit(f"next_instr += {mac.cache_offset};") - if parts[-1].instr.check_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();") - out.emit("DISPATCH();") - - -def write_components( - parts: list[Component], - out: Formatter, - tier: Tiers, - cache_offset: int, - family: Family | None, -) -> bool: - managers = get_managers(parts) - - all_vars: dict[str, StackEffect] = {} - for mgr in managers: - for name, eff in mgr.collect_vars().items(): - if name in all_vars: - # TODO: Turn this into an error -- variable conflict - assert all_vars[name] == eff, ( - name, - mgr.instr.name, - all_vars[name], - eff, - ) - else: - all_vars[name] = eff - - # Declare all variables - for name, eff in all_vars.items(): - out.declare(eff, None) - - next_instr_is_set = False - for mgr in managers: - if len(parts) > 1: - out.emit(f"// {mgr.instr.name}") - - for copy in mgr.copies: - copy_src_effect = copy.src.effect - if copy_src_effect.name != copy.dst.effect.name: - if copy_src_effect.name == UNUSED: - copy_src_effect = copy.src.as_stack_effect() - out.assign(copy.dst.effect, copy_src_effect) - for peek in mgr.peeks: - out.assign( - peek.effect, - peek.as_stack_effect(), - ) - # Initialize array outputs - for poke in mgr.pokes: - if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.assign( - poke.effect, - poke.as_stack_effect(lax=True), - ) - - if mgr.instr.name in ("_PUSH_FRAME", "_POP_FRAME"): - # Adjust stack to min_offset. - # This means that all input effects of this instruction - # are materialized, but not its output effects. - # That's as intended, since these two are so special. - out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) - # However, for tier 2, pretend the stack is at final offset. - mgr.adjust_inverse(mgr.final_offset) - if tier == TIER_ONE: - # TODO: Check in analyzer that _{PUSH,POP}_FRAME is last. - assert ( - mgr is managers[-1] - ), f"Expected {mgr.instr.name!r} to be the last uop" - assert_no_pokes(managers) - - if mgr.instr.name == "_SAVE_CURRENT_IP": - next_instr_is_set = True - if cache_offset: - out.emit(f"next_instr += {cache_offset};") - if tier == TIER_ONE: - assert_no_pokes(managers) - - if len(parts) == 1: - mgr.instr.write_body(out, 0, mgr.active_caches, tier, family) - else: - with out.block(""): - mgr.instr.write_body(out, -4, mgr.active_caches, tier, family) - - if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits: - # Adjust the stack to its final depth, *then* write the - # pokes for all preceding uops. - # Note that for array output effects we may still write - # past the stack top. - out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - write_all_pokes(mgr.final_offset, managers, out) - - return next_instr_is_set - - -def assert_no_pokes(managers: list[EffectManager]) -> None: - for mgr in managers: - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - assert ( - poke.effect.name == UNUSED - ), f"Unexpected poke of {poke.effect.name} in {mgr.instr.name!r}" - - -def write_all_pokes( - offset: StackOffset, managers: list[EffectManager], out: Formatter -) -> None: - # Emit all remaining pushes (pokes) - for m in managers: - m.adjust_inverse(offset) - write_pokes(m, out) - - -def write_pokes(mgr: EffectManager, out: Formatter) -> None: - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.assign( - poke.as_stack_effect(), - poke.effect, - ) - - -def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: - try: - _write_components_for_abstract_interp( - [Component(instr, instr.active_caches)], - out, - ) - except AssertionError as err: - raise AssertionError( - f"Error writing abstract instruction {instr.name}" - ) from err - - -def _write_components_for_abstract_interp( - parts: list[Component], - out: Formatter, -) -> None: - managers = get_managers(parts) - for mgr in managers: - if mgr is managers[-1]: - out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - mgr.adjust_inverse(mgr.final_offset) - # NULL out the output stack effects - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.emit( - f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" - f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);" - ) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py new file mode 100644 index 00000000000000..7f610bff6290e2 --- /dev/null +++ b/Tools/cases_generator/target_generator.py @@ -0,0 +1,53 @@ +"""Generate targets for computed goto dispatch +Reads the instruction definitions from bytecodes.c. +Writes the table to opcode_targets.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, +) +from cwriter import CWriter + + +DEFAULT_OUTPUT = ROOT / "Python/opcode_targets.h" + + +def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: + """Write header file that defines the jump target table""" + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&TARGET_{name},\n" + out.emit("static void *opcode_targets[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + +arg_parser = argparse.ArgumentParser( + description="Generate the file with dispatch targets.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + out = CWriter(outfile, 0, False) + write_opcode_targets(data, out) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py new file mode 100644 index 00000000000000..85be673b1c396c --- /dev/null +++ b/Tools/cases_generator/tier1_generator.py @@ -0,0 +1,205 @@ +"""Generate the main interpreter switch. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + analysis_error, + StackItem, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + type_and_null, +) +from cwriter import CWriter +from typing import TextIO +from stack import Stack, SizeMismatch + + +DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" + + +FOOTER = "#undef TIER_ONE\n" + + +def declare_variables(inst: Instruction, out: CWriter) -> None: + variables = {"unused"} + for uop in inst.parts: + if isinstance(uop, Uop): + for var in reversed(uop.stack.inputs): + if var.name not in variables: + variables.add(var.name) + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + + +def write_uop( + uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool +) -> int: + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + out.emit(f"/* Skip {uop.size} cache {entries} */\n") + return offset + uop.size + try: + out.start_line() + if braces: + out.emit(f"// {uop.name}\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if braces: + out.emit("{\n") + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + out.emit( + f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" + ) + if inst.family is None: + out.emit(f"(void){cache.name};\n") + offset += cache.size + emit_tokens(out, uop, stack, inst) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + if braces: + out.start_line() + out.emit("}\n") + # out.emit(stack.as_comment() + "\n") + return offset + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.parts: + if isinstance(uop, Skip): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + return False + + +def generate_tier1( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 +""" + ) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + needs_this = uses_this(inst) + out.emit("\n") + out.emit(f"TARGET({name}) {{\n") + unused_guard = "(void)this_instr;\n" if inst.family is None else "" + if needs_this and not inst.is_target: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + out.emit(unused_guard) + else: + out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED({name});\n") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + offset = write_uop(part, out, offset, stack, inst, insert_braces) + out.start_line() + if not inst.parts[-1].properties.always_exits: + stack.flush(out) + if inst.parts[-1].properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + outfile.write(FOOTER) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + + +def generate_tier1_from_files( + filenames: list[str], outfilename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_tier1(filenames, data, outfile, lines) + + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier1(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py new file mode 100644 index 00000000000000..7a69aa6e121fa7 --- /dev/null +++ b/Tools/cases_generator/tier2_generator.py @@ -0,0 +1,252 @@ +"""Generate the cases for the tier 2 interpreter. +Reads the instruction definitions from bytecodes.c. +Writes the cases to executor_cases.c.h, which is #included in ceval.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + analyze_files, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + emit_to, + REPLACEMENT_FUNCTIONS, + type_and_null, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import Stack, SizeMismatch + +DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" + + +def declare_variable( + var: StackItem, uop: Uop, variables: set[str], out: CWriter +) -> None: + if var.name in variables: + return + variables.add(var.name) + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + if uop.replicates: + # Replicas may not use all their conditional variables + # So avoid a compiler warning with a fake use + out.emit(f"(void){var.name};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + + +def declare_variables(uop: Uop, out: CWriter) -> None: + variables = {"unused"} + for var in reversed(uop.stack.inputs): + declare_variable(var, uop, variables, out) + for var in uop.stack.outputs: + declare_variable(var, uop, variables, out) + + +def tier2_replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") JUMP_TO_ERROR();\n") + + +def tier2_replace_error_no_pop( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("JUMP_TO_ERROR();", tkn) + +def tier2_replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n"); + out.emit("}\n") + + +def tier2_replace_exit_if( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n") + out.emit("}\n") + + +def tier2_replace_oparg( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + out.emit(tkn) + return + amp = next(tkn_iter) + if amp.text != "&": + out.emit(tkn) + out.emit(amp) + return + one = next(tkn_iter) + assert one.text == "1" + out.emit_at(uop.name[-1], tkn) + + +TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() +TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error +TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop +TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg +TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if + + +def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: + try: + out.start_line() + if uop.properties.oparg: + out.emit("oparg = CURRENT_OPARG();\n") + assert uop.properties.const_oparg < 0 + elif uop.properties.const_oparg >= 0: + out.emit(f"oparg = {uop.properties.const_oparg};\n") + out.emit(f"assert(oparg == CURRENT_OPARG());\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_tier2( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 +""" + ) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, uop in analysis.uops.items(): + if uop.properties.tier == 1: + continue + if uop.properties.oparg_and_1: + out.emit(f"/* {uop.name} is split on (oparg & 1) */\n\n") + continue + if uop.is_super(): + continue + why_not_viable = uop.why_not_viable() + if why_not_viable is not None: + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + declare_variables(uop, out) + stack = Stack() + write_uop(uop, out, stack) + out.start_line() + if not uop.properties.always_exits: + stack.flush(out) + if uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("break;\n") + out.start_line() + out.emit("}") + out.emit("\n\n") + outfile.write("#undef TIER_TWO\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier2(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py new file mode 100644 index 00000000000000..aae89faaa928e1 --- /dev/null +++ b/Tools/cases_generator/uop_id_generator.py @@ -0,0 +1,79 @@ +"""Generate the list of uop IDs. +Reads the instruction definitions from bytecodes.c. +Writes the IDs to pycore_uop_ids.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h" + + +def generate_uop_ids( + filenames: list[str], analysis: Analysis, outfile: TextIO, distinct_namespace: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_UOP_IDS_H"): + next_id = 1 if distinct_namespace else 300 + # These two are first by convention + out.emit(f"#define _EXIT_TRACE {next_id}\n") + next_id += 1 + out.emit(f"#define _SET_IP {next_id}\n") + next_id += 1 + PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} + + uops = [(uop.name, uop) for uop in analysis.uops.values()] + # Sort so that _BASE comes immediately before _BASE_0, etc. + for name, uop in sorted(uops): + if name in PRE_DEFINED: + continue + if uop.properties.tier == 1: + continue + if uop.implicitly_created and not distinct_namespace and not uop.replicated: + out.emit(f"#define {name} {name[1:]}\n") + else: + out.emit(f"#define {name} {next_id}\n") + next_id += 1 + + out.emit(f"#define MAX_UOP_ID {next_id-1}\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with all uop IDs.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) +arg_parser.add_argument( + "-n", + "--namespace", + help="Give uops a distinct namespace", + action="store_true", +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_uop_ids(args.input, data, outfile, args.namespace) diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py new file mode 100644 index 00000000000000..7b3325ada4a49f --- /dev/null +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -0,0 +1,95 @@ +"""Generate uop metadata. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to pycore_uop_metadata.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + cflags, +) +from stack import Stack +from cwriter import CWriter +from typing import TextIO + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" + + +def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") + out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1: + out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + + out.emit("};\n\n") + out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.replicated: + out.emit(f"[{uop.name}] = {uop.replicated},\n") + + out.emit("};\n\n") + out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") + for uop in sorted(analysis.uops.values(), key=lambda t: t.name): + if uop.is_viable() and uop.properties.tier != 1: + out.emit(f'[{uop.name}] = "{uop.name}",\n') + out.emit("};\n") + out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") + out.emit("switch(opcode) {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1: + stack = Stack() + for var in reversed(uop.stack.inputs): + stack.pop(var) + popped = (-stack.base_offset).to_c() + out.emit(f"case {uop.name}:\n") + out.emit(f" return {popped};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def generate_uop_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_UOP_METADATA_H"): + out.emit("#include \n") + out.emit('#include "pycore_uop_ids.h"\n') + generate_names_and_flags(analysis, out) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with uop metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_uop_metadata(args.input, data, outfile) diff --git a/Tools/ccbench/ccbench.py b/Tools/ccbench/ccbench.py deleted file mode 100644 index d52701a82948da..00000000000000 --- a/Tools/ccbench/ccbench.py +++ /dev/null @@ -1,606 +0,0 @@ -# This file should be kept compatible with both Python 2.6 and Python >= 3.0. - -from __future__ import division -from __future__ import print_function - -""" -ccbench, a Python concurrency benchmark. -""" - -import time -import os -import sys -import itertools -import threading -import subprocess -import socket -from optparse import OptionParser, SUPPRESS_HELP -import platform - -# Compatibility -try: - xrange -except NameError: - xrange = range - -try: - map = itertools.imap -except AttributeError: - pass - - -THROUGHPUT_DURATION = 2.0 - -LATENCY_PING_INTERVAL = 0.1 -LATENCY_DURATION = 2.0 - -BANDWIDTH_PACKET_SIZE = 1024 -BANDWIDTH_DURATION = 2.0 - - -def task_pidigits(): - """Pi calculation (Python)""" - _map = map - _count = itertools.count - _islice = itertools.islice - - def calc_ndigits(n): - # From http://shootout.alioth.debian.org/ - def gen_x(): - return _map(lambda k: (k, 4*k + 2, 0, 2*k + 1), _count(1)) - - def compose(a, b): - aq, ar, as_, at = a - bq, br, bs, bt = b - return (aq * bq, - aq * br + ar * bt, - as_ * bq + at * bs, - as_ * br + at * bt) - - def extract(z, j): - q, r, s, t = z - return (q*j + r) // (s*j + t) - - def pi_digits(): - z = (1, 0, 0, 1) - x = gen_x() - while 1: - y = extract(z, 3) - while y != extract(z, 4): - z = compose(z, next(x)) - y = extract(z, 3) - z = compose((10, -10*y, 0, 1), z) - yield y - - return list(_islice(pi_digits(), n)) - - return calc_ndigits, (50, ) - -def task_regex(): - """regular expression (C)""" - # XXX this task gives horrendous latency results. - import re - # Taken from the `inspect` module - pat = re.compile(r'^(\s*def\s)|(.*(? return the previous one. - if end_event: - return niters, duration - niters += step - duration = t2 - start_time - if duration >= min_duration: - end_event.append(None) - return niters, duration - if t2 - t1 < 0.01: - # Minimize interference of measurement on overall runtime - step = step * 3 // 2 - elif do_yield: - # OS scheduling of Python threads is sometimes so bad that we - # have to force thread switching ourselves, otherwise we get - # completely useless results. - _sleep(0.0001) - t1 = t2 - - -def run_throughput_test(func, args, nthreads): - assert nthreads >= 1 - - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - end_event = [] - - if nthreads == 1: - # Pure single-threaded performance, without any switching or - # synchronization overhead. - start_time = time.time() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=False)) - return results - - started = False - ready_cond = threading.Condition() - start_cond = threading.Condition() - ready = [] - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=True)) - - threads = [] - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # We don't want measurements to include thread startup overhead, - # so we arrange for timing to start after all threads are ready. - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - with start_cond: - start_time = time.time() - started = True - start_cond.notify(nthreads) - for t in threads: - t.join() - - return results - -def run_throughput_tests(max_threads): - for task in throughput_tasks: - print(task.__doc__) - print() - func, args = task() - nthreads = 1 - baseline_speed = None - while nthreads <= max_threads: - results = run_throughput_test(func, args, nthreads) - # Taking the max duration rather than average gives pessimistic - # results rather than optimistic. - speed = sum(r[0] for r in results) / max(r[1] for r in results) - print("threads=%d: %d" % (nthreads, speed), end="") - if baseline_speed is None: - print(" iterations/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -LAT_END = "END" - -def _sendto(sock, s, addr): - sock.sendto(s.encode('ascii'), addr) - -def _recv(sock, n): - return sock.recv(n).decode('ascii') - -def latency_client(addr, nb_pings, interval): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - _time = time.time - _sleep = time.sleep - def _ping(): - _sendto(sock, "%r\n" % _time(), addr) - # The first ping signals the parent process that we are ready. - _ping() - # We give the parent a bit of time to notice. - _sleep(1.0) - for i in range(nb_pings): - _sleep(interval) - _ping() - _sendto(sock, LAT_END + "\n", addr) - finally: - sock.close() - -def run_latency_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--latclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_latency_test(func, args, nthreads): - # Create a listening socket to receive the pings. We use UDP which should - # be painlessly cross-platform. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - interval = LATENCY_PING_INTERVAL - duration = LATENCY_DURATION - nb_pings = int(duration / interval) - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first ping(s) to arrive before - # unblocking the background threads. - chunks = [] - process = run_latency_client(addr=sock.getsockname(), - nb_pings=nb_pings, interval=interval) - s = _recv(sock, 4096) - _time = time.time - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - while LAT_END not in s: - s = _recv(sock, 4096) - t = _time() - chunks.append((t, s)) - - # Tell the background threads to stop. - end_event.append(None) - for t in threads: - t.join() - process.wait() - sock.close() - - for recv_time, chunk in chunks: - # NOTE: it is assumed that a line sent by a client wasn't received - # in two chunks because the lines are very small. - for line in chunk.splitlines(): - line = line.strip() - if line and line != LAT_END: - send_time = eval(line) - assert isinstance(send_time, float) - results.append((send_time, recv_time)) - - return results - -def run_latency_tests(max_threads): - for task in latency_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - while nthreads <= max_threads: - results = run_latency_test(func, args, nthreads) - n = len(results) - # We print out milliseconds - lats = [1000 * (t2 - t1) for (t1, t2) in results] - #print(list(map(int, lats))) - avg = sum(lats) / n - dev = (sum((x - avg) ** 2 for x in lats) / n) ** 0.5 - print("CPU threads=%d: %d ms. (std dev: %d ms.)" % (nthreads, avg, dev), end="") - print() - #print(" [... from %d samples]" % n) - nthreads += 1 - print() - - -BW_END = "END" - -def bandwidth_client(addr, packet_size, duration): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - local_addr = sock.getsockname() - _time = time.time - _sleep = time.sleep - def _send_chunk(msg): - _sendto(sock, ("%r#%s\n" % (local_addr, msg)).rjust(packet_size), addr) - # We give the parent some time to be ready. - _sleep(1.0) - try: - start_time = _time() - end_time = start_time + duration * 2.0 - i = 0 - while _time() < end_time: - _send_chunk(str(i)) - s = _recv(sock, packet_size) - assert len(s) == packet_size - i += 1 - _send_chunk(BW_END) - finally: - sock.close() - -def run_bandwidth_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--bwclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_bandwidth_test(func, args, nthreads): - # Create a listening socket to receive the packets. We use UDP which should - # be painlessly cross-platform. - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - duration = BANDWIDTH_DURATION - packet_size = BANDWIDTH_PACKET_SIZE - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first packet to arrive before - # unblocking the background threads. - process = run_bandwidth_client(addr=addr, - packet_size=packet_size, - duration=duration) - _time = time.time - # This will also wait for the parent to be ready - s = _recv(sock, packet_size) - remote_addr = eval(s.partition('#')[0]) - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - n = 0 - first_time = None - while not end_event and BW_END not in s: - _sendto(sock, s, remote_addr) - s = _recv(sock, packet_size) - if first_time is None: - first_time = _time() - n += 1 - end_time = _time() - - end_event.append(None) - for t in threads: - t.join() - process.kill() - - return (n - 1) / (end_time - first_time) - -def run_bandwidth_tests(max_threads): - for task in bandwidth_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - baseline_speed = None - while nthreads <= max_threads: - results = run_bandwidth_test(func, args, nthreads) - speed = results - #speed = len(results) * 1.0 / results[-1][0] - print("CPU threads=%d: %.1f" % (nthreads, speed), end="") - if baseline_speed is None: - print(" packets/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -def main(): - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--throughput", - action="store_true", dest="throughput", default=False, - help="run throughput tests") - parser.add_option("-l", "--latency", - action="store_true", dest="latency", default=False, - help="run latency tests") - parser.add_option("-b", "--bandwidth", - action="store_true", dest="bandwidth", default=False, - help="run I/O bandwidth tests") - parser.add_option("-i", "--interval", - action="store", type="int", dest="check_interval", default=None, - help="sys.setcheckinterval() value " - "(Python 3.8 and older)") - parser.add_option("-I", "--switch-interval", - action="store", type="float", dest="switch_interval", default=None, - help="sys.setswitchinterval() value " - "(Python 3.2 and newer)") - parser.add_option("-n", "--num-threads", - action="store", type="int", dest="nthreads", default=4, - help="max number of threads in tests") - - # Hidden option to run the pinging and bandwidth clients - parser.add_option("", "--latclient", - action="store", dest="latclient", default=None, - help=SUPPRESS_HELP) - parser.add_option("", "--bwclient", - action="store", dest="bwclient", default=None, - help=SUPPRESS_HELP) - - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - - if options.latclient: - kwargs = eval(options.latclient) - latency_client(**kwargs) - return - - if options.bwclient: - kwargs = eval(options.bwclient) - bandwidth_client(**kwargs) - return - - if not options.throughput and not options.latency and not options.bandwidth: - options.throughput = options.latency = options.bandwidth = True - if options.check_interval: - sys.setcheckinterval(options.check_interval) - if options.switch_interval: - sys.setswitchinterval(options.switch_interval) - - print("== %s %s (%s) ==" % ( - platform.python_implementation(), - platform.python_version(), - platform.python_build()[0], - )) - # Processor identification often has repeated spaces - cpu = ' '.join(platform.processor().split()) - print("== %s %s on '%s' ==" % ( - platform.machine(), - platform.system(), - cpu, - )) - print() - - if options.throughput: - print("--- Throughput ---") - print() - run_throughput_tests(options.nthreads) - - if options.latency: - print("--- Latency ---") - print() - run_latency_tests(options.nthreads) - - if options.bandwidth: - print("--- I/O bandwidth ---") - print() - run_bandwidth_tests(options.nthreads) - -if __name__ == "__main__": - main() diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index cbb3a9a8f3a8c2..c019572d0cb186 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -1,5 +1,7 @@ target-version = "py310" fix = true + +[lint] select = [ "F", # Enable all pyflakes rules "UP", # Enable all pyupgrade rules by default diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1bcc85537bf2da..362715d264620e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4,6528 +4,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. # -from __future__ import annotations - -import abc -import argparse -import ast -import builtins as bltns -import collections -import contextlib -import copy -import cpp -import dataclasses as dc -import enum -import functools -import hashlib -import inspect -import io -import itertools -import os -import pprint -import re -import shlex -import string -import sys -import textwrap - -from collections.abc import ( - Callable, - Iterable, - Iterator, - Sequence, -) -from operator import attrgetter -from types import FunctionType, NoneType -from typing import ( - TYPE_CHECKING, - Any, - Final, - Literal, - NamedTuple, - NoReturn, - Protocol, - TypeVar, - cast, - overload, -) - -# TODO: -# -# soon: -# -# * allow mixing any two of {positional-only, positional-or-keyword, -# keyword-only} -# * dict constructor uses positional-only and keyword-only -# * max and min use positional only with an optional group -# and keyword-only -# - -version = '1' - -NO_VARARG = "PY_SSIZE_T_MAX" -CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = { - "_keywords", - "_parser", - "args", - "argsbuf", - "fastargs", - "kwargs", - "kwnames", - "nargs", - "noptargs", - "return_value", -} - -# '#include "header.h" // reason': column of '//' comment -INCLUDE_COMMENT_COLUMN = 35 - -# match '#define Py_LIMITED_API' -LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') - - -class Sentinels(enum.Enum): - unspecified = "unspecified" - unknown = "unknown" - - def __repr__(self) -> str: - return f"<{self.value.capitalize()}>" - - -unspecified: Final = Sentinels.unspecified -unknown: Final = Sentinels.unknown - - -# This one needs to be a distinct class, unlike the other two -class Null: - def __repr__(self) -> str: - return '' - - -NULL = Null() - -sig_end_marker = '--' - -Appender = Callable[[str], None] -Outputter = Callable[[], str] -TemplateDict = dict[str, str] - -class _TextAccumulator(NamedTuple): - text: list[str] - append: Appender - output: Outputter - -def _text_accumulator() -> _TextAccumulator: - text: list[str] = [] - def output() -> str: - s = ''.join(text) - text.clear() - return s - return _TextAccumulator(text, text.append, output) - - -class TextAccumulator(NamedTuple): - append: Appender - output: Outputter - -def text_accumulator() -> TextAccumulator: - """ - Creates a simple text accumulator / joiner. - - Returns a pair of callables: - append, output - "append" appends a string to the accumulator. - "output" returns the contents of the accumulator - joined together (''.join(accumulator)) and - empties the accumulator. - """ - text, append, output = _text_accumulator() - return TextAccumulator(append, output) - - -@dc.dataclass -class ClinicError(Exception): - message: str - _: dc.KW_ONLY - lineno: int | None = None - filename: str | None = None - - def __post_init__(self) -> None: - super().__init__(self.message) - - def report(self, *, warn_only: bool = False) -> str: - msg = "Warning" if warn_only else "Error" - if self.filename is not None: - msg += f" in file {self.filename!r}" - if self.lineno is not None: - msg += f" on line {self.lineno}" - msg += ":\n" - msg += f"{self.message}\n" - return msg - - -@overload -def warn_or_fail( - *args: object, - fail: Literal[True], - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: ... - -@overload -def warn_or_fail( - *args: object, - fail: Literal[False] = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: ... - -def warn_or_fail( - *args: object, - fail: bool = False, - filename: str | None = None, - line_number: int | None = None, -) -> None: - joined = " ".join([str(a) for a in args]) - if clinic: - if filename is None: - filename = clinic.filename - if getattr(clinic, 'block_parser', None) and (line_number is None): - line_number = clinic.block_parser.line_number - error = ClinicError(joined, filename=filename, lineno=line_number) - if fail: - raise error - else: - print(error.report(warn_only=True)) - - -def warn( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> None: - return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) - -def fail( - *args: object, - filename: str | None = None, - line_number: int | None = None, -) -> NoReturn: - warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) - - -def quoted_for_c_string(s: str) -> str: - for old, new in ( - ('\\', '\\\\'), # must be first! - ('"', '\\"'), - ("'", "\\'"), - ): - s = s.replace(old, new) - return s - -def c_repr(s: str) -> str: - return '"' + s + '"' - - -def wrapped_c_string_literal( - text: str, - *, - width: int = 72, - suffix: str = '', - initial_indent: int = 0, - subsequent_indent: int = 4 -) -> str: - wrapped = textwrap.wrap(text, width=width, replace_whitespace=False, - drop_whitespace=False, break_on_hyphens=False) - separator = '"' + suffix + '\n' + subsequent_indent * ' ' + '"' - return initial_indent * ' ' + '"' + separator.join(wrapped) + '"' - - -is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match - -def is_legal_py_identifier(s: str) -> bool: - return all(is_legal_c_identifier(field) for field in s.split('.')) - -# identifiers that are okay in Python but aren't a good idea in C. -# so if they're used Argument Clinic will add "_value" to the end -# of the name in C. -c_keywords = set(""" -asm auto break case char const continue default do double -else enum extern float for goto if inline int long -register return short signed sizeof static struct switch -typedef typeof union unsigned void volatile while -""".strip().split()) - -def ensure_legal_c_identifier(s: str) -> str: - # for now, just complain if what we're given isn't legal - if not is_legal_c_identifier(s): - fail("Illegal C identifier:", s) - # but if we picked a C keyword, pick something else - if s in c_keywords: - return s + "_value" - return s - -def rstrip_lines(s: str) -> str: - text, add, output = _text_accumulator() - for line in s.split('\n'): - add(line.rstrip()) - add('\n') - text.pop() - return output() - -def format_escape(s: str) -> str: - # double up curly-braces, this string will be used - # as part of a format_map() template later - s = s.replace('{', '{{') - s = s.replace('}', '}}') - return s - -def linear_format(s: str, **kwargs: str) -> str: - """ - Perform str.format-like substitution, except: - * The strings substituted must be on lines by - themselves. (This line is the "source line".) - * If the substitution text is empty, the source line - is removed in the output. - * If the field is not recognized, the original line - is passed unmodified through to the output. - * If the substitution text is not empty: - * Each line of the substituted text is indented - by the indent of the source line. - * A newline will be added to the end. - """ - - add, output = text_accumulator() - for line in s.split('\n'): - indent, curly, trailing = line.partition('{') - if not curly: - add(line) - add('\n') - continue - - name, curly, trailing = trailing.partition('}') - if not curly or name not in kwargs: - add(line) - add('\n') - continue - - if trailing: - fail(f"Text found after {{{name}}} block marker! " - "It must be on a line by itself.") - if indent.strip(): - fail(f"Non-whitespace characters found before {{{name}}} block marker! " - "It must be on a line by itself.") - - value = kwargs[name] - if not value: - continue - - value = textwrap.indent(rstrip_lines(value), indent) - add(value) - add('\n') - - return output()[:-1] - -def indent_all_lines(s: str, prefix: str) -> str: - """ - Returns 's', with 'prefix' prepended to all lines. - - If the last line is empty, prefix is not prepended - to it. (If s is blank, returns s unchanged.) - - (textwrap.indent only adds to non-blank lines.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(prefix) - final.append(line) - final.append('\n') - if last: - final.append(prefix) - final.append(last) - return ''.join(final) - -def suffix_all_lines(s: str, suffix: str) -> str: - """ - Returns 's', with 'suffix' appended to all lines. - - If the last line is empty, suffix is not appended - to it. (If s is blank, returns s unchanged.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(line) - final.append(suffix) - final.append('\n') - if last: - final.append(last) - final.append(suffix) - return ''.join(final) - - -def pprint_words(items: list[str]) -> str: - if len(items) <= 2: - return " and ".join(items) - else: - return ", ".join(items[:-1]) + " and " + items[-1] - - -def version_splitter(s: str) -> tuple[int, ...]: - """Splits a version string into a tuple of integers. - - The following ASCII characters are allowed, and employ - the following conversions: - a -> -3 - b -> -2 - c -> -1 - (This permits Python-style version strings such as "1.4b3".) - """ - version: list[int] = [] - accumulator: list[str] = [] - def flush() -> None: - if not accumulator: - fail(f'Unsupported version string: {s!r}') - version.append(int(''.join(accumulator))) - accumulator.clear() - - for c in s: - if c.isdigit(): - accumulator.append(c) - elif c == '.': - flush() - elif c in 'abc': - flush() - version.append('abc'.index(c) - 3) - else: - fail(f'Illegal character {c!r} in version string {s!r}') - flush() - return tuple(version) - -def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: - iterator = itertools.zip_longest( - version_splitter(version1), version_splitter(version2), fillvalue=0 - ) - for a, b in iterator: - if a < b: - return -1 - if a > b: - return 1 - return 0 - - -class CRenderData: - def __init__(self) -> None: - - # The C statements to declare variables. - # Should be full lines with \n eol characters. - self.declarations: list[str] = [] - - # The C statements required to initialize the variables before the parse call. - # Should be full lines with \n eol characters. - self.initializers: list[str] = [] - - # The C statements needed to dynamically modify the values - # parsed by the parse call, before calling the impl. - self.modifications: list[str] = [] - - # The entries for the "keywords" array for PyArg_ParseTuple. - # Should be individual strings representing the names. - self.keywords: list[str] = [] - - # The "format units" for PyArg_ParseTuple. - # Should be individual strings that will get - self.format_units: list[str] = [] - - # The varargs arguments for PyArg_ParseTuple. - self.parse_arguments: list[str] = [] - - # The parameter declarations for the impl function. - self.impl_parameters: list[str] = [] - - # The arguments to the impl function at the time it's called. - self.impl_arguments: list[str] = [] - - # For return converters: the name of the variable that - # should receive the value returned by the impl. - self.return_value = "return_value" - - # For return converters: the code to convert the return - # value from the parse function. This is also where - # you should check the _return_value for errors, and - # "goto exit" if there are any. - self.return_conversion: list[str] = [] - self.converter_retval = "_return_value" - - # The C statements required to do some operations - # after the end of parsing but before cleaning up. - # These operations may be, for example, memory deallocations which - # can only be done without any error happening during argument parsing. - self.post_parsing: list[str] = [] - - # The C statements required to clean up after the impl call. - self.cleanup: list[str] = [] - - -class FormatCounterFormatter(string.Formatter): - """ - This counts how many instances of each formatter - "replacement string" appear in the format string. - - e.g. after evaluating "string {a}, {b}, {c}, {a}" - the counts dict would now look like - {'a': 2, 'b': 1, 'c': 1} - """ - def __init__(self) -> None: - self.counts = collections.Counter[str]() - - def get_value( - self, key: str, args: object, kwargs: object # type: ignore[override] - ) -> Literal['']: - self.counts[key] += 1 - return '' - -class Language(metaclass=abc.ABCMeta): - - start_line = "" - body_prefix = "" - stop_line = "" - checksum_line = "" - - def __init__(self, filename: str) -> None: - ... - - @abc.abstractmethod - def render( - self, - clinic: Clinic | None, - signatures: Iterable[Module | Class | Function] - ) -> str: - ... - - def parse_line(self, line: str) -> None: - ... - - def validate(self) -> None: - def assert_only_one( - attr: str, - *additional_fields: str - ) -> None: - """ - Ensures that the string found at getattr(self, attr) - contains exactly one formatter replacement string for - each valid field. The list of valid fields is - ['dsl_name'] extended by additional_fields. - - e.g. - self.fmt = "{dsl_name} {a} {b}" - - # this passes - self.assert_only_one('fmt', 'a', 'b') - - # this fails, the format string has a {b} in it - self.assert_only_one('fmt', 'a') - - # this fails, the format string doesn't have a {c} in it - self.assert_only_one('fmt', 'a', 'b', 'c') - - # this fails, the format string has two {a}s in it, - # it must contain exactly one - self.fmt2 = '{dsl_name} {a} {a}' - self.assert_only_one('fmt2', 'a') - - """ - fields = ['dsl_name'] - fields.extend(additional_fields) - line: str = getattr(self, attr) - fcf = FormatCounterFormatter() - fcf.format(line) - def local_fail(should_be_there_but_isnt: bool) -> None: - if should_be_there_but_isnt: - fail("{} {} must contain {{{}}} exactly once!".format( - self.__class__.__name__, attr, name)) - else: - fail("{} {} must not contain {{{}}}!".format( - self.__class__.__name__, attr, name)) - - for name, count in fcf.counts.items(): - if name in fields: - if count > 1: - local_fail(True) - else: - local_fail(False) - for name in fields: - if fcf.counts.get(name) != 1: - local_fail(True) - - assert_only_one('start_line') - assert_only_one('stop_line') - - field = "arguments" if "{arguments}" in self.checksum_line else "checksum" - assert_only_one('checksum_line', field) - - - -class PythonLanguage(Language): - - language = 'Python' - start_line = "#/*[{dsl_name} input]" - body_prefix = "#" - stop_line = "#[{dsl_name} start generated code]*/" - checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" - - -ParamTuple = tuple["Parameter", ...] - - -def permute_left_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (3,) - (2, 3) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in reversed(l): - accumulator = list(group) + accumulator - yield tuple(accumulator) - - -def permute_right_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (1,) - (1, 2) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in l: - accumulator.extend(group) - yield tuple(accumulator) - - -def permute_optional_groups( - left: Sequence[Iterable[Parameter]], - required: Iterable[Parameter], - right: Sequence[Iterable[Parameter]] -) -> tuple[ParamTuple, ...]: - """ - Generator function that computes the set of acceptable - argument lists for the provided iterables of - argument groups. (Actually it generates a tuple of tuples.) - - Algorithm: prefer left options over right options. - - If required is empty, left must also be empty. - """ - required = tuple(required) - if not required: - if left: - raise ValueError("required is empty but left is not") - - accumulator: list[ParamTuple] = [] - counts = set() - for r in permute_right_option_groups(right): - for l in permute_left_option_groups(left): - t = l + required + r - if len(t) in counts: - continue - counts.add(len(t)) - accumulator.append(t) - - accumulator.sort(key=len) - return tuple(accumulator) - - -def strip_leading_and_trailing_blank_lines(s: str) -> str: - lines = s.rstrip().split('\n') - while lines: - line = lines[0] - if line.strip(): - break - del lines[0] - return '\n'.join(lines) - -@functools.lru_cache() -def normalize_snippet( - s: str, - *, - indent: int = 0 -) -> str: - """ - Reformats s: - * removes leading and trailing blank lines - * ensures that it does not end with a newline - * dedents so the first nonwhite character on any line is at column "indent" - """ - s = strip_leading_and_trailing_blank_lines(s) - s = textwrap.dedent(s) - if indent: - s = textwrap.indent(s, ' ' * indent) - return s - - -def declare_parser( - f: Function, - *, - hasformat: bool = False, - clinic: Clinic, - limited_capi: bool, -) -> str: - """ - Generates the code template for a static local PyArg_Parser variable, - with an initializer. For core code (incl. builtin modules) the - kwtuple field is also statically initialized. Otherwise - it is initialized at runtime. - """ - if hasformat: - fname = '' - format_ = '.format = "{format_units}:{name}",' - else: - fname = '.fname = "{name}",' - format_ = '' - - num_keywords = len([ - p for p in f.parameters.values() - if not p.is_positional_only() and not p.is_vararg() - ]) - if limited_capi: - declarations = """ - #define KWTUPLE NULL - """ - elif num_keywords == 0: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - """ - else: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS %d - static struct {{ - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - }} _kwtuple = {{ - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = {{ {keywords_py} }}, - }}; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - """ % num_keywords - - condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' - clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) - clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) - - declarations += """ - static const char * const _keywords[] = {{{keywords_c} NULL}}; - static _PyArg_Parser _parser = {{ - .keywords = _keywords, - %s - .kwtuple = KWTUPLE, - }}; - #undef KWTUPLE - """ % (format_ or fname) - return normalize_snippet(declarations) - - -def wrap_declarations( - text: str, - length: int = 78 -) -> str: - """ - A simple-minded text wrapper for C function declarations. - - It views a declaration line as looking like this: - xxxxxxxx(xxxxxxxxx,xxxxxxxxx) - If called with length=30, it would wrap that line into - xxxxxxxx(xxxxxxxxx, - xxxxxxxxx) - (If the declaration has zero or one parameters, this - function won't wrap it.) - - If this doesn't work properly, it's probably better to - start from scratch with a more sophisticated algorithm, - rather than try and improve/debug this dumb little function. - """ - lines = [] - for line in text.split('\n'): - prefix, _, after_l_paren = line.partition('(') - if not after_l_paren: - lines.append(line) - continue - in_paren, _, after_r_paren = after_l_paren.partition(')') - if not _: - lines.append(line) - continue - if ',' not in in_paren: - lines.append(line) - continue - parameters = [x.strip() + ", " for x in in_paren.split(',')] - prefix += "(" - if len(prefix) < length: - spaces = " " * len(prefix) - else: - spaces = " " * 4 - - while parameters: - line = prefix - first = True - while parameters: - if (not first and - (len(line) + len(parameters[0]) > length)): - break - line += parameters.pop(0) - first = False - if not parameters: - line = line.rstrip(", ") + ")" + after_r_paren - lines.append(line.rstrip()) - prefix = spaces - return "\n".join(lines) - - -class CLanguage(Language): - - body_prefix = "#" - language = 'C' - start_line = "/*[{dsl_name} input]" - body_prefix = "" - stop_line = "[{dsl_name} start generated code]*/" - checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - - PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) - METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) - DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) - METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" - #define {methoddef_name} \ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) - METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - // Emit compiler warnings when we get to Python {major}.{minor}. - #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error {message} - #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 - # ifdef _MSC_VER - # pragma message ({message}) - # else - # warning {message} - # endif - #endif - """ - DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - if ({condition}) {{{{{errcheck} - if (PyErr_WarnEx(PyExc_DeprecationWarning, - {message}, 1)) - {{{{ - goto exit; - }}}} - }}}} - """ - - def __init__(self, filename: str) -> None: - super().__init__(filename) - self.cpp = cpp.Monitor(filename) - self.cpp.fail = fail # type: ignore[method-assign] - - def parse_line(self, line: str) -> None: - self.cpp.writeline(line) - - def render( - self, - clinic: Clinic | None, - signatures: Iterable[Module | Class | Function] - ) -> str: - function = None - for o in signatures: - if isinstance(o, Function): - if function: - fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) - function = o - return self.render_function(clinic, function) - - def compiler_deprecated_warning( - self, - func: Function, - parameters: list[Parameter], - ) -> str | None: - minversion: VersionTuple | None = None - for p in parameters: - for version in p.deprecated_positional, p.deprecated_keyword: - if version and (not minversion or minversion > version): - minversion = version - if not minversion: - return None - - # Format the preprocessor warning and error messages. - assert isinstance(self.cpp.filename, str) - message = f"Update the clinic input of {func.full_name!r}." - code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( - major=minversion[0], - minor=minversion[1], - message=c_repr(message), - ) - return normalize_snippet(code) - - def deprecate_positional_use( - self, - func: Function, - params: dict[int, Parameter], - ) -> str: - assert len(params) > 0 - first_pos = next(iter(params)) - last_pos = next(reversed(params)) - - # Format the deprecation message. - if len(params) == 1: - condition = f"nargs == {first_pos+1}" - amount = f"{first_pos+1} " if first_pos else "" - pl = "s" - else: - condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" - amount = f"more than {first_pos} " if first_pos else "" - pl = "s" if first_pos != 1 else "" - message = ( - f"Passing {amount}positional argument{pl} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_positional") - ): - names = [repr(p.name) for p in group] - pstr = pprint_words(names) - if len(names) == 1: - message += ( - f" Parameter {pstr} will become a keyword-only parameter " - f"in Python {major}.{minor}." - ) - else: - message += ( - f" Parameters {pstr} will become keyword-only parameters " - f"in Python {major}.{minor}." - ) - - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck="", - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return normalize_snippet(code, indent=4) - - def deprecate_keyword_use( - self, - func: Function, - params: dict[int, Parameter], - argname_fmt: str | None, - *, - fastcall: bool, - limited_capi: bool, - clinic: Clinic, - ) -> str: - assert len(params) > 0 - last_param = next(reversed(params.values())) - - # Format the deprecation message. - containscheck = "" - conditions = [] - for i, p in params.items(): - if p.is_optional(): - if argname_fmt: - conditions.append(f"nargs < {i+1} && {argname_fmt % i}") - elif fastcall: - conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") - containscheck = "PySequence_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") - containscheck = "PyDict_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions = [f"nargs < {i+1}"] - condition = ") || (".join(conditions) - if len(conditions) > 1: - condition = f"(({condition}))" - if last_param.is_optional(): - if fastcall: - if limited_capi: - condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" - else: - condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" - else: - if limited_capi: - condition = f"kwargs && PyDict_Size(kwargs) && {condition}" - else: - condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" - names = [repr(p.name) for p in params.values()] - pstr = pprint_words(names) - pl = 's' if len(params) != 1 else '' - message = ( - f"Passing keyword argument{pl} {pstr} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_keyword") - ): - names = [repr(p.name) for p in group] - pstr = pprint_words(names) - pl = 's' if len(names) != 1 else '' - message += ( - f" Parameter{pl} {pstr} will become positional-only " - f"in Python {major}.{minor}." - ) - - if containscheck: - errcheck = f""" - if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail - goto exit; - }}}}""" - else: - errcheck = "" - if argname_fmt: - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck=errcheck, - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return normalize_snippet(code, indent=4) - - def docstring_for_c_string( - self, - f: Function - ) -> str: - text, add, output = _text_accumulator() - # turn docstring into a properly quoted C string - for line in f.docstring.split('\n'): - add('"') - add(quoted_for_c_string(line)) - add('\\n"\n') - - if text[-2] == sig_end_marker: - # If we only have a signature, add the blank line that the - # __text_signature__ getter expects to be there. - add('"\\n"') - else: - text.pop() - add('"') - return ''.join(text) - - def output_templates( - self, - f: Function, - clinic: Clinic - ) -> dict[str, str]: - parameters = list(f.parameters.values()) - assert parameters - first_param = parameters.pop(0) - assert isinstance(first_param.converter, self_converter) - requires_defining_class = False - if parameters and isinstance(parameters[0].converter, defining_class_converter): - requires_defining_class = True - del parameters[0] - converters = [p.converter for p in parameters] - - # Copy includes from parameters to Clinic - for converter in converters: - include = converter.include - if include: - clinic.add_include(include.filename, include.reason, - condition=include.condition) - - has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - default_return_converter = f.return_converter.type == 'PyObject *' - new_or_init = f.kind.new_or_init - - vararg: int | str = NO_VARARG - pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 - for i, p in enumerate(parameters, 1): - if p.is_keyword_only(): - assert not p.is_positional_only() - if not p.is_optional(): - min_kw_only = i - max_pos - elif p.is_vararg(): - if vararg != NO_VARARG: - fail("Too many var args") - pseudo_args += 1 - vararg = i - 1 - else: - if vararg == NO_VARARG: - max_pos = i - if p.is_positional_only(): - pos_only = i - if not p.is_optional(): - min_pos = i - - meth_o = (len(parameters) == 1 and - parameters[0].is_positional_only() and - not converters[0].is_optional() and - not requires_defining_class and - not new_or_init) - - # we have to set these things before we're done: - # - # docstring_prototype - # docstring_definition - # impl_prototype - # methoddef_define - # parser_prototype - # parser_definition - # impl_definition - # cpp_if - # cpp_endif - # methoddef_ifndef - - return_value_declaration = "PyObject *return_value = NULL;" - methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE - if new_or_init and not f.docstring: - docstring_prototype = docstring_definition = '' - else: - docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR - docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR - impl_definition = self.IMPL_DEFINITION_PROTOTYPE - impl_prototype = parser_prototype = parser_definition = None - - # parser_body_fields remembers the fields passed in to the - # previous call to parser_body. this is used for an awful hack. - parser_body_fields: tuple[str, ...] = () - def parser_body( - prototype: str, - *fields: str, - declarations: str = '' - ) -> str: - nonlocal parser_body_fields - add, output = text_accumulator() - add(prototype) - parser_body_fields = fields - - preamble = normalize_snippet(""" - {{ - {return_value_declaration} - {parser_declarations} - {declarations} - {initializers} - """) + "\n" - finale = normalize_snippet(""" - {modifications} - {return_value} = {c_basename}_impl({impl_arguments}); - {return_conversion} - {post_parsing} - - {exit_label} - {cleanup} - return return_value; - }} - """) - for field in preamble, *fields, finale: - add('\n') - add(field) - return linear_format(output(), parser_declarations=declarations) - - fastcall = not new_or_init - limited_capi = clinic.limited_capi - if limited_capi and (requires_defining_class or pseudo_args or - (any(p.is_optional() for p in parameters) and - any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or - any(c.broken_limited_capi for c in converters)): - warn(f"Function {f.full_name} cannot use limited C API") - limited_capi = False - - parsearg: str | None - if not parameters: - parser_code: list[str] | None - if not requires_defining_class: - # no parameters, METH_NOARGS - flags = "METH_NOARGS" - parser_prototype = self.PARSER_PROTOTYPE_NOARGS - parser_code = [] - else: - assert fastcall - - flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - return_error = ('return NULL;' if default_return_converter - else 'goto exit;') - parser_code = [normalize_snippet(""" - if (nargs) {{ - PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); - %s - }} - """ % return_error, indent=4)] - - if default_return_converter: - parser_definition = '\n'.join([ - parser_prototype, - '{{', - *parser_code, - ' return {c_basename}_impl({impl_arguments});', - '}}']) - else: - parser_definition = parser_body(parser_prototype, *parser_code) - - elif meth_o: - flags = "METH_O" - - if (isinstance(converters[0], object_converter) and - converters[0].format_unit == 'O'): - meth_o_prototype = self.METH_O_PROTOTYPE - - if default_return_converter: - # maps perfectly to METH_O, doesn't need a return converter. - # so we skip making a parse function - # and call directly into the impl function. - impl_prototype = parser_prototype = parser_definition = '' - impl_definition = meth_o_prototype - else: - # SLIGHT HACK - # use impl_parameters for the parser here! - parser_prototype = meth_o_prototype - parser_definition = parser_body(parser_prototype) - - else: - argname = 'arg' - if parameters[0].name == argname: - argname += '_' - parser_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *%s) - """ % argname) - - displayname = parameters[0].get_displayname(0) - parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - parsearg = """ - if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - """ % argname - parser_definition = parser_body(parser_prototype, - normalize_snippet(parsearg, indent=4)) - - elif has_option_groups: - # positional parameters with option groups - # (we have to generate lots of PyArg_ParseTuple calls - # in a big switch statement) - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - - elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: - if fastcall: - # positional-only, but no option groups - # we only need one call to _PyArg_ParseStack - - flags = "METH_FASTCALL" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL - nargs = 'nargs' - argname_fmt = 'args[%d]' - else: - # positional-only, but no option groups - # we only need one call to PyArg_ParseTuple - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - if limited_capi: - nargs = 'PyTuple_Size(args)' - argname_fmt = 'PyTuple_GetItem(args, %d)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - - left_args = f"{nargs} - {max_pos}" - max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos - if limited_capi: - parser_code = [] - if nargs != 'nargs': - parser_code.append(normalize_snippet(f'Py_ssize_t nargs = {nargs};', indent=4)) - nargs = 'nargs' - if min_pos == max_args: - pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" - if ({nargs} != {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - if min_pos: - pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" - if ({nargs} < {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - if max_args != NO_VARARG: - pl = '' if max_args == 1 else 's' - parser_code.append(normalize_snippet(f""" - if ({nargs} > {max_args}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - parser_code = [normalize_snippet(f""" - if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ - goto exit; - }}}} - """, indent=4)] - - has_optional = False - for i, p in enumerate(parameters): - if p.is_vararg(): - if fastcall: - parser_code.append(normalize_snippet(""" - %s = PyTuple_New(%s); - if (!%s) {{ - goto exit; - }} - for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); - }} - """ % ( - p.converter.parser_name, - left_args, - p.converter.parser_name, - left_args, - p.converter.parser_name, - max_pos - ), indent=4)) - else: - parser_code.append(normalize_snippet(""" - %s = PyTuple_GetSlice(%d, -1); - """ % ( - p.converter.parser_name, - max_pos - ), indent=4)) - continue - - displayname = p.get_displayname(i+1) - argname = argname_fmt % i - parsearg = p.converter.parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if has_optional or p.is_optional(): - has_optional = True - parser_code.append(normalize_snippet(""" - if (%s < %d) {{ - goto skip_optional; - }} - """, indent=4) % (nargs, i + 1)) - parser_code.append(normalize_snippet(parsearg, indent=4)) - - if parser_code is not None: - if has_optional: - parser_code.append("skip_optional:") - else: - if limited_capi: - fastcall = False - if fastcall: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [normalize_snippet(""" - if (!PyArg_ParseTuple(args, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - parser_definition = parser_body(parser_prototype, *parser_code) - - else: - deprecated_positionals: dict[int, Parameter] = {} - deprecated_keywords: dict[int, Parameter] = {} - for i, p in enumerate(parameters): - if p.deprecated_positional: - deprecated_positionals[i] = p - if p.deprecated_keyword: - deprecated_keywords[i] = p - - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) - if vararg == NO_VARARG: - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only - ) - nargs = "nargs" - else: - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only, - vararg - ) - nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" - - if limited_capi: - parser_code = None - fastcall = False - - elif fastcall: - flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - else: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - declarations += "\nPyObject * const *fastargs;" - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - - if requires_defining_class: - flags = 'METH_METHOD|' + flags - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - - if parser_code is not None: - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - add_label: str | None = None - for i, p in enumerate(parameters): - if isinstance(p.converter, defining_class_converter): - raise ValueError("defining_class should be the first " - "parameter (after self)") - displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if add_label and (i == pos_only or i == max_pos): - parser_code.append("%s:" % add_label) - add_label = None - if not p.is_optional(): - parser_code.append(normalize_snippet(parsearg, indent=4)) - elif i < pos_only: - add_label = 'skip_optional_posonly' - parser_code.append(normalize_snippet(""" - if (nargs < %d) {{ - goto %s; - }} - """ % (i + 1, add_label), indent=4)) - if has_optional_kw: - parser_code.append(normalize_snippet(""" - noptargs--; - """, indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=4)) - else: - if i < max_pos: - label = 'skip_optional_pos' - first_opt = max(min_pos, pos_only) - else: - label = 'skip_optional_kwonly' - first_opt = max_pos + min_kw_only - if vararg != NO_VARARG: - first_opt += 1 - if i == first_opt: - add_label = label - parser_code.append(normalize_snippet(""" - if (!noptargs) {{ - goto %s; - }} - """ % add_label, indent=4)) - if i + 1 == len(parameters): - parser_code.append(normalize_snippet(parsearg, indent=4)) - else: - add_label = label - parser_code.append(normalize_snippet(""" - if (%s) {{ - """ % (argname_fmt % i), indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=8)) - parser_code.append(normalize_snippet(""" - if (!--noptargs) {{ - goto %s; - }} - }} - """ % add_label, indent=4)) - - if parser_code is not None: - if add_label: - parser_code.append("%s:" % add_label) - else: - declarations = declare_parser(f, clinic=clinic, - hasformat=True, - limited_capi=limited_capi) - if limited_capi: - # positional-or-keyword arguments - assert not fastcall - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [normalize_snippet(""" - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, - {parse_arguments})) - goto exit; - """, indent=4)] - declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" - - elif fastcall: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - parser_code = [normalize_snippet(""" - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - if deprecated_positionals: - code = self.deprecate_positional_use(f, deprecated_positionals) - # Insert the deprecation code before parameter parsing. - parser_code.insert(0, code) - - assert parser_prototype is not None - parser_definition = parser_body(parser_prototype, *parser_code, - declarations=declarations) - - - if new_or_init: - methoddef_define = '' - - if f.kind is METHOD_NEW: - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - else: - return_value_declaration = "int return_value = -1;" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ - - fields = list(parser_body_fields) - parses_positional = 'METH_NOARGS' not in flags - parses_keywords = 'METH_KEYWORDS' in flags - if parses_keywords: - assert parses_positional - - if requires_defining_class: - raise ValueError("Slot methods cannot access their defining class.") - - if not parses_keywords: - declarations = '{base_type_ptr}' - fields.insert(0, normalize_snippet(""" - if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ - goto exit; - }} - """, indent=4)) - if not parses_positional: - fields.insert(0, normalize_snippet(""" - if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ - goto exit; - }} - """, indent=4)) - - parser_definition = parser_body(parser_prototype, *fields, - declarations=declarations) - - - if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): - methoddef_cast = "(PyCFunction)" - methoddef_cast_end = "" - elif limited_capi: - methoddef_cast = "(PyCFunction)(void(*)(void))" - methoddef_cast_end = "" - else: - methoddef_cast = "_PyCFunction_CAST(" - methoddef_cast_end = ")" - - if f.methoddef_flags: - flags += '|' + f.methoddef_flags - - methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) - methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) - methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) - - methoddef_ifndef = '' - conditional = self.cpp.condition() - if not conditional: - cpp_if = cpp_endif = '' - else: - cpp_if = "#if " + conditional - cpp_endif = "#endif /* " + conditional + " */" - - if methoddef_define and f.full_name not in clinic.ifndef_symbols: - clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF - - # add ';' to the end of parser_prototype and impl_prototype - # (they mustn't be None, but they could be an empty string.) - assert parser_prototype is not None - if parser_prototype: - assert not parser_prototype.endswith(';') - parser_prototype += ';' - - if impl_prototype is None: - impl_prototype = impl_definition - if impl_prototype: - impl_prototype += ";" - - parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) - - compiler_warning = self.compiler_deprecated_warning(f, parameters) - if compiler_warning: - parser_definition = compiler_warning + "\n\n" + parser_definition - - d = { - "docstring_prototype" : docstring_prototype, - "docstring_definition" : docstring_definition, - "impl_prototype" : impl_prototype, - "methoddef_define" : methoddef_define, - "parser_prototype" : parser_prototype, - "parser_definition" : parser_definition, - "impl_definition" : impl_definition, - "cpp_if" : cpp_if, - "cpp_endif" : cpp_endif, - "methoddef_ifndef" : methoddef_ifndef, - } - - # make sure we didn't forget to assign something, - # and wrap each non-empty value in \n's - d2 = {} - for name, value in d.items(): - assert value is not None, "got a None value for template " + repr(name) - if value: - value = '\n' + value + '\n' - d2[name] = value - return d2 - - @staticmethod - def group_to_variable_name(group: int) -> str: - adjective = "left_" if group < 0 else "right_" - return "group_" + adjective + str(abs(group)) - - def render_option_group_parsing( - self, - f: Function, - template_dict: TemplateDict, - limited_capi: bool, - ) -> None: - # positional only, grouped, optional arguments! - # can be optional on the left or right. - # here's an example: - # - # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] - # - # Here group D are required, and all other groups are optional. - # (Group D's "group" is actually None.) - # We can figure out which sets of arguments we have based on - # how many arguments are in the tuple. - # - # Note that you need to count up on both sides. For example, - # you could have groups C+D, or C+D+E, or C+D+E+F. - # - # What if the number of arguments leads us to an ambiguous result? - # Clinic prefers groups on the left. So in the above example, - # five arguments would map to B+C, not C+D. - - add, output = text_accumulator() - parameters = list(f.parameters.values()) - if isinstance(parameters[0].converter, self_converter): - del parameters[0] - - group: list[Parameter] | None = None - left = [] - right = [] - required: list[Parameter] = [] - last: int | Literal[Sentinels.unspecified] = unspecified - - for p in parameters: - group_id = p.group - if group_id != last: - last = group_id - group = [] - if group_id < 0: - left.append(group) - elif group_id == 0: - group = required - else: - right.append(group) - assert group is not None - group.append(p) - - count_min = sys.maxsize - count_max = -1 - - if limited_capi: - nargs = 'PyTuple_Size(args)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - add(f"switch ({nargs}) {{\n") - for subset in permute_optional_groups(left, required, right): - count = len(subset) - count_min = min(count_min, count) - count_max = max(count_max, count) - - if count == 0: - add(""" case 0: - break; -""") - continue - - group_ids = {p.group for p in subset} # eliminate duplicates - d: dict[str, str | int] = {} - d['count'] = count - d['name'] = f.name - d['format_units'] = "".join(p.converter.format_unit for p in subset) - - parse_arguments: list[str] = [] - for p in subset: - p.converter.parse_argument(parse_arguments) - d['parse_arguments'] = ", ".join(parse_arguments) - - group_ids.discard(0) - lines = "\n".join([ - self.group_to_variable_name(g) + " = 1;" - for g in group_ids - ]) - - s = """\ - case {count}: - if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - {group_booleans} - break; -""" - s = linear_format(s, group_booleans=lines) - s = s.format_map(d) - add(s) - - add(" default:\n") - s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - add(s.format(f.full_name, count_min, count_max)) - add(' goto exit;\n') - add("}") - template_dict['option_group_parsing'] = format_escape(output()) - - def render_function( - self, - clinic: Clinic | None, - f: Function | None - ) -> str: - if f is None or clinic is None: - return "" - - add, output = text_accumulator() - data = CRenderData() - - assert f.parameters, "We should always have a 'self' at this point!" - parameters = f.render_parameters - converters = [p.converter for p in parameters] - - templates = self.output_templates(f, clinic) - - f_self = parameters[0] - selfless = parameters[1:] - assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" - - last_group = 0 - first_optional = len(selfless) - positional = selfless and selfless[-1].is_positional_only() - has_option_groups = False - - # offset i by -1 because first_optional needs to ignore self - for i, p in enumerate(parameters, -1): - c = p.converter - - if (i != -1) and (p.default is not unspecified): - first_optional = min(first_optional, i) - - if p.is_vararg(): - data.cleanup.append(f"Py_XDECREF({c.parser_name});") - - # insert group variable - group = p.group - if last_group != group: - last_group = group - if group: - group_name = self.group_to_variable_name(group) - data.impl_arguments.append(group_name) - data.declarations.append("int " + group_name + " = 0;") - data.impl_parameters.append("int " + group_name) - has_option_groups = True - - c.render(p, data) - - if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']') " - "unless all parameters are positional-only ('/').") - - # HACK - # when we're METH_O, but have a custom return converter, - # we use "impl_parameters" for the parsing function - # because that works better. but that means we must - # suppress actually declaring the impl's parameters - # as variables in the parsing function. but since it's - # METH_O, we have exactly one anyway, so we know exactly - # where it is. - if ("METH_O" in templates['methoddef_define'] and - '{impl_parameters}' in templates['parser_prototype']): - data.declarations.pop(0) - - full_name = f.full_name - template_dict = {'full_name': full_name} - template_dict['name'] = f.displayname - template_dict['c_basename'] = f.c_basename - template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" - - template_dict['docstring'] = self.docstring_for_c_string(f) - - template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' - for converter in converters: - converter.set_template_dict(template_dict) - - f.return_converter.render(f, data) - template_dict['impl_return_type'] = f.return_converter.type - - template_dict['declarations'] = format_escape("\n".join(data.declarations)) - template_dict['initializers'] = "\n\n".join(data.initializers) - template_dict['modifications'] = '\n\n'.join(data.modifications) - template_dict['keywords_c'] = ' '.join('"' + k + '",' - for k in data.keywords) - keywords = [k for k in data.keywords if k] - template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),' - for k in keywords) - template_dict['format_units'] = ''.join(data.format_units) - template_dict['parse_arguments'] = ', '.join(data.parse_arguments) - if data.parse_arguments: - template_dict['parse_arguments_comma'] = ','; - else: - template_dict['parse_arguments_comma'] = ''; - template_dict['impl_parameters'] = ", ".join(data.impl_parameters) - template_dict['impl_arguments'] = ", ".join(data.impl_arguments) - template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) - template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) - template_dict['cleanup'] = format_escape("".join(data.cleanup)) - template_dict['return_value'] = data.return_value - - # used by unpack tuple code generator - unpack_min = first_optional - unpack_max = len(selfless) - template_dict['unpack_min'] = str(unpack_min) - template_dict['unpack_max'] = str(unpack_max) - - if has_option_groups: - self.render_option_group_parsing(f, template_dict, - limited_capi=clinic.limited_capi) - - # buffers, not destination - for name, destination in clinic.destination_buffers.items(): - template = templates[name] - if has_option_groups: - template = linear_format(template, - option_group_parsing=template_dict['option_group_parsing']) - template = linear_format(template, - declarations=template_dict['declarations'], - return_conversion=template_dict['return_conversion'], - initializers=template_dict['initializers'], - modifications=template_dict['modifications'], - post_parsing=template_dict['post_parsing'], - cleanup=template_dict['cleanup'], - ) - - # Only generate the "exit:" label - # if we have any gotos - need_exit_label = "goto exit;" in template - template = linear_format(template, - exit_label="exit:" if need_exit_label else '' - ) - - s = template.format_map(template_dict) - - # mild hack: - # reflow long impl declarations - if name in {"impl_prototype", "impl_definition"}: - s = wrap_declarations(s) - - if clinic.line_prefix: - s = indent_all_lines(s, clinic.line_prefix) - if clinic.line_suffix: - s = suffix_all_lines(s, clinic.line_suffix) - - destination.append(s) - - return clinic.get_destination('block').dump() - - -def create_regex( - before: str, - after: str, - word: bool = True, - whole_line: bool = True -) -> re.Pattern[str]: - """Create an re object for matching marker lines.""" - group_re = r"\w+" if word else ".+" - pattern = r'{}({}){}' - if whole_line: - pattern = '^' + pattern + '$' - pattern = pattern.format(re.escape(before), group_re, re.escape(after)) - return re.compile(pattern) - - -@dc.dataclass(slots=True, repr=False) -class Block: - r""" - Represents a single block of text embedded in - another file. If dsl_name is None, the block represents - verbatim text, raw original text from the file, in - which case "input" will be the only non-false member. - If dsl_name is not None, the block represents a Clinic - block. - - input is always str, with embedded \n characters. - input represents the original text from the file; - if it's a Clinic block, it is the original text with - the body_prefix and redundant leading whitespace removed. - - dsl_name is either str or None. If str, it's the text - found on the start line of the block between the square - brackets. - - signatures is a list. - It may only contain clinic.Module, clinic.Class, and - clinic.Function objects. At the moment it should - contain at most one of each. - - output is either str or None. If str, it's the output - from this block, with embedded '\n' characters. - - indent is a str. It's the leading whitespace - that was found on every line of input. (If body_prefix is - not empty, this is the indent *after* removing the - body_prefix.) - - "indent" is different from the concept of "preindent" - (which is not stored as state on Block objects). - "preindent" is the whitespace that - was found in front of every line of input *before* the - "body_prefix" (see the Language object). If body_prefix - is empty, preindent must always be empty too. - - To illustrate the difference between "indent" and "preindent": - - Assume that '_' represents whitespace. - If the block processed was in a Python file, and looked like this: - ____#/*[python] - ____#__for a in range(20): - ____#____print(a) - ____#[python]*/ - "preindent" would be "____" and "indent" would be "__". - - """ - input: str - dsl_name: str | None = None - signatures: list[Module | Class | Function] = dc.field(default_factory=list) - output: Any = None # TODO: Very dynamic; probably untypeable in its current form? - indent: str = '' - - def __repr__(self) -> str: - dsl_name = self.dsl_name or "text" - def summarize(s: object) -> str: - s = repr(s) - if len(s) > 30: - return s[:26] + "..." + s[0] - return s - parts = ( - repr(dsl_name), - f"input={summarize(self.input)}", - f"output={summarize(self.output)}" - ) - return f"" - - -class BlockParser: - """ - Block-oriented parser for Argument Clinic. - Iterator, yields Block objects. - """ - - def __init__( - self, - input: str, - language: Language, - *, - verify: bool = True - ) -> None: - """ - "input" should be a str object - with embedded \n characters. - - "language" should be a Language object. - """ - language.validate() - - self.input = collections.deque(reversed(input.splitlines(keepends=True))) - self.block_start_line_number = self.line_number = 0 - - self.language = language - before, _, after = language.start_line.partition('{dsl_name}') - assert _ == '{dsl_name}' - self.find_start_re = create_regex(before, after, whole_line=False) - self.start_re = create_regex(before, after) - self.verify = verify - self.last_checksum_re: re.Pattern[str] | None = None - self.last_dsl_name: str | None = None - self.dsl_name: str | None = None - self.first_block = True - - def __iter__(self) -> BlockParser: - return self - - def __next__(self) -> Block: - while True: - if not self.input: - raise StopIteration - - if self.dsl_name: - return_value = self.parse_clinic_block(self.dsl_name) - self.dsl_name = None - self.first_block = False - return return_value - block = self.parse_verbatim_block() - if self.first_block and not block.input: - continue - self.first_block = False - return block - - - def is_start_line(self, line: str) -> str | None: - match = self.start_re.match(line.lstrip()) - return match.group(1) if match else None - - def _line(self, lookahead: bool = False) -> str: - self.line_number += 1 - line = self.input.pop() - if not lookahead: - self.language.parse_line(line) - return line - - def parse_verbatim_block(self) -> Block: - add, output = text_accumulator() - self.block_start_line_number = self.line_number - - while self.input: - line = self._line() - dsl_name = self.is_start_line(line) - if dsl_name: - self.dsl_name = dsl_name - break - add(line) - - return Block(output()) - - def parse_clinic_block(self, dsl_name: str) -> Block: - input_add, input_output = text_accumulator() - self.block_start_line_number = self.line_number + 1 - stop_line = self.language.stop_line.format(dsl_name=dsl_name) - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - - def is_stop_line(line: str) -> bool: - # make sure to recognize stop line even if it - # doesn't end with EOL (it could be the very end of the file) - if line.startswith(stop_line): - remainder = line.removeprefix(stop_line) - if remainder and not remainder.isspace(): - fail(f"Garbage after stop line: {remainder!r}") - return True - else: - # gh-92256: don't allow incorrectly formatted stop lines - if line.lstrip().startswith(stop_line): - fail(f"Whitespace is not allowed before the stop line: {line!r}") - return False - - # consume body of program - while self.input: - line = self._line() - if is_stop_line(line) or self.is_start_line(line): - break - if body_prefix: - line = line.lstrip() - assert line.startswith(body_prefix) - line = line.removeprefix(body_prefix) - input_add(line) - - # consume output and checksum line, if present. - if self.last_dsl_name == dsl_name: - checksum_re = self.last_checksum_re - else: - before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') - assert _ == '{arguments}' - checksum_re = create_regex(before, after, word=False) - self.last_dsl_name = dsl_name - self.last_checksum_re = checksum_re - assert checksum_re is not None - - # scan forward for checksum line - output_add, output_output = text_accumulator() - arguments = None - while self.input: - line = self._line(lookahead=True) - match = checksum_re.match(line.lstrip()) - arguments = match.group(1) if match else None - if arguments: - break - output_add(line) - if self.is_start_line(line): - break - - output: str | None - output = output_output() - if arguments: - d = {} - for field in shlex.split(arguments): - name, equals, value = field.partition('=') - if not equals: - fail(f"Mangled Argument Clinic marker line: {line!r}") - d[name.strip()] = value.strip() - - if self.verify: - if 'input' in d: - checksum = d['output'] - else: - checksum = d['checksum'] - - computed = compute_checksum(output, len(checksum)) - if checksum != computed: - fail("Checksum mismatch! " - f"Expected {checksum!r}, computed {computed!r}. " - "Suggested fix: remove all generated code including " - "the end marker, or use the '-f' option.") - else: - # put back output - output_lines = output.splitlines(keepends=True) - self.line_number -= len(output_lines) - self.input.extend(reversed(output_lines)) - output = None - - return Block(input_output(), dsl_name, output=output) - - -@dc.dataclass(slots=True, frozen=True) -class Include: - """ - An include like: #include "pycore_long.h" // _Py_ID() - """ - # Example: "pycore_long.h". - filename: str - - # Example: "_Py_ID()". - reason: str - - # None means unconditional include. - # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". - condition: str | None - - def sort_key(self) -> tuple[str, str]: - # order: '#if' comes before 'NO_CONDITION' - return (self.condition or 'NO_CONDITION', self.filename) - - -@dc.dataclass(slots=True) -class BlockPrinter: - language: Language - f: io.StringIO = dc.field(default_factory=io.StringIO) - - def print_block( - self, - block: Block, - *, - core_includes: bool = False, - limited_capi: bool, - header_includes: dict[str, Include], - ) -> None: - input = block.input - output = block.output - dsl_name = block.dsl_name - write = self.f.write - - assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) - - if not dsl_name: - write(input) - return - - write(self.language.start_line.format(dsl_name=dsl_name)) - write("\n") - - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - if not body_prefix: - write(input) - else: - for line in input.split('\n'): - write(body_prefix) - write(line) - write("\n") - - write(self.language.stop_line.format(dsl_name=dsl_name)) - write("\n") - - output = '' - if core_includes and header_includes: - # Emit optional "#include" directives for C headers - output += '\n' - - current_condition: str | None = None - includes = sorted(header_includes.values(), key=Include.sort_key) - for include in includes: - if include.condition != current_condition: - if current_condition: - output += '#endif\n' - current_condition = include.condition - if include.condition: - output += f'{include.condition}\n' - - if current_condition: - line = f'# include "{include.filename}"' - else: - line = f'#include "{include.filename}"' - if include.reason: - comment = f'// {include.reason}\n' - line = line.ljust(INCLUDE_COMMENT_COLUMN - 1) + comment - output += line - - if current_condition: - output += '#endif\n' - - input = ''.join(block.input) - output += ''.join(block.output) - if output: - if not output.endswith('\n'): - output += '\n' - write(output) - - arguments = "output={output} input={input}".format( - output=compute_checksum(output, 16), - input=compute_checksum(input, 16) - ) - write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) - write("\n") - - def write(self, text: str) -> None: - self.f.write(text) - - -class BufferSeries: - """ - Behaves like a "defaultlist". - When you ask for an index that doesn't exist yet, - the object grows the list until that item exists. - So o[n] will always work. - - Supports negative indices for actual items. - e.g. o[-1] is an element immediately preceding o[0]. - """ - - def __init__(self) -> None: - self._start = 0 - self._array: list[_TextAccumulator] = [] - self._constructor = _text_accumulator - - def __getitem__(self, i: int) -> _TextAccumulator: - i -= self._start - if i < 0: - self._start += i - prefix = [self._constructor() for x in range(-i)] - self._array = prefix + self._array - i = 0 - while i >= len(self._array): - self._array.append(self._constructor()) - return self._array[i] - - def clear(self) -> None: - for ta in self._array: - ta.text.clear() - - def dump(self) -> str: - texts = [ta.output() for ta in self._array] - return "".join(texts) - - -@dc.dataclass(slots=True, repr=False) -class Destination: - name: str - type: str - clinic: Clinic - buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) - filename: str = dc.field(init=False) # set in __post_init__ - - args: dc.InitVar[tuple[str, ...]] = () - - def __post_init__(self, args: tuple[str, ...]) -> None: - valid_types = ('buffer', 'file', 'suppress') - if self.type not in valid_types: - fail( - f"Invalid destination type {self.type!r} for {self.name}, " - f"must be {', '.join(valid_types)}" - ) - extra_arguments = 1 if self.type == "file" else 0 - if len(args) < extra_arguments: - fail(f"Not enough arguments for destination " - f"{self.name!r} new {self.type!r}") - if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") - if self.type =='file': - d = {} - filename = self.clinic.filename - d['path'] = filename - dirname, basename = os.path.split(filename) - if not dirname: - dirname = '.' - d['dirname'] = dirname - d['basename'] = basename - d['basename_root'], d['basename_extension'] = os.path.splitext(filename) - self.filename = args[0].format_map(d) - - def __repr__(self) -> str: - if self.type == 'file': - type_repr = f"type='file' file={self.filename!r}" - else: - type_repr = f"type={self.type!r}" - return f"" - - def clear(self) -> None: - if self.type != 'buffer': - fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") - self.buffers.clear() - - def dump(self) -> str: - return self.buffers.dump() - - -# "extensions" maps the file extension ("c", "py") to Language classes. -LangDict = dict[str, Callable[[str], Language]] -extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } -extensions['py'] = PythonLanguage - - -def write_file(filename: str, new_contents: str) -> None: - try: - with open(filename, encoding="utf-8") as fp: - old_contents = fp.read() - - if old_contents == new_contents: - # no change: avoid modifying the file modification time - return - except FileNotFoundError: - pass - # Atomic write using a temporary file and os.replace() - filename_new = f"{filename}.new" - with open(filename_new, "w", encoding="utf-8") as fp: - fp.write(new_contents) - try: - os.replace(filename_new, filename) - except: - os.unlink(filename_new) - raise - - -ClassDict = dict[str, "Class"] -DestinationDict = dict[str, Destination] -ModuleDict = dict[str, "Module"] - - -class Parser(Protocol): - def __init__(self, clinic: Clinic) -> None: ... - def parse(self, block: Block) -> None: ... - - -clinic: Clinic | None = None -class Clinic: - - presets_text = """ -preset block -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset original -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset file -everything file -methoddef_ifndef file 1 -docstring_prototype suppress -parser_prototype suppress -impl_definition block - -preset buffer -everything buffer -methoddef_ifndef buffer 1 -impl_definition block -docstring_prototype suppress -impl_prototype suppress -parser_prototype suppress - -preset partial-buffer -everything buffer -methoddef_ifndef buffer 1 -docstring_prototype block -impl_prototype suppress -methoddef_define block -parser_prototype block -impl_definition block - -""" - - def __init__( - self, - language: CLanguage, - printer: BlockPrinter | None = None, - *, - filename: str, - limited_capi: bool, - verify: bool = True, - ) -> None: - # maps strings to Parser objects. - # (instantiated from the "parsers" global.) - self.parsers: dict[str, Parser] = {} - self.language: CLanguage = language - if printer: - fail("Custom printers are broken right now") - self.printer = printer or BlockPrinter(language) - self.verify = verify - self.limited_capi = limited_capi - self.filename = filename - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - # dict: include name => Include instance - self.includes: dict[str, Include] = {} - - self.line_prefix = self.line_suffix = '' - - self.destinations: DestinationDict = {} - self.add_destination("block", "buffer") - self.add_destination("suppress", "suppress") - self.add_destination("buffer", "buffer") - if filename: - self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") - - d = self.get_destination_buffer - self.destination_buffers = { - 'cpp_if': d('file'), - 'docstring_prototype': d('suppress'), - 'docstring_definition': d('file'), - 'methoddef_define': d('file'), - 'impl_prototype': d('file'), - 'parser_prototype': d('suppress'), - 'parser_definition': d('file'), - 'cpp_endif': d('file'), - 'methoddef_ifndef': d('file', 1), - 'impl_definition': d('block'), - } - - DestBufferType = dict[str, _TextAccumulator] - DestBufferList = list[DestBufferType] - - self.destination_buffers_stack: DestBufferList = [] - self.ifndef_symbols: set[str] = set() - - self.presets: dict[str, dict[Any, Any]] = {} - preset = None - for line in self.presets_text.strip().split('\n'): - line = line.strip() - if not line: - continue - name, value, *options = line.split() - if name == 'preset': - self.presets[value] = preset = {} - continue - - if len(options): - index = int(options[0]) - else: - index = 0 - buffer = self.get_destination_buffer(value, index) - - if name == 'everything': - for name in self.destination_buffers: - preset[name] = buffer - continue - - assert name in self.destination_buffers - preset[name] = buffer - - global clinic - clinic = self - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - try: - existing = self.includes[name] - except KeyError: - pass - else: - if existing.condition and not condition: - # If the previous include has a condition and the new one is - # unconditional, override the include. - pass - else: - # Already included, do nothing. Only mention a single reason, - # no need to list all of them. - return - - self.includes[name] = Include(name, reason, condition) - - def add_destination( - self, - name: str, - type: str, - *args: str - ) -> None: - if name in self.destinations: - fail(f"Destination already exists: {name!r}") - self.destinations[name] = Destination(name, type, self, args) - - def get_destination(self, name: str) -> Destination: - d = self.destinations.get(name) - if not d: - fail(f"Destination does not exist: {name!r}") - return d - - def get_destination_buffer( - self, - name: str, - item: int = 0 - ) -> _TextAccumulator: - d = self.get_destination(name) - return d.buffers[item] - - def parse(self, input: str) -> str: - printer = self.printer - self.block_parser = BlockParser(input, self.language, verify=self.verify) - for block in self.block_parser: - dsl_name = block.dsl_name - if dsl_name: - if dsl_name not in self.parsers: - assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." - self.parsers[dsl_name] = parsers[dsl_name](self) - parser = self.parsers[dsl_name] - parser.parse(block) - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - - # these are destinations not buffers - for name, destination in self.destinations.items(): - if destination.type == 'suppress': - continue - output = destination.dump() - - if output: - block = Block("", dsl_name="clinic", output=output) - - if destination.type == 'buffer': - block.input = "dump " + name + "\n" - warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") - printer.write("\n") - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - continue - - if destination.type == 'file': - try: - dirname = os.path.dirname(destination.filename) - try: - os.makedirs(dirname) - except FileExistsError: - if not os.path.isdir(dirname): - fail(f"Can't write to destination " - f"{destination.filename!r}; " - f"can't make directory {dirname!r}!") - if self.verify: - with open(destination.filename) as f: - parser_2 = BlockParser(f.read(), language=self.language) - blocks = list(parser_2) - if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail(f"Modified destination file " - f"{destination.filename!r}; not overwriting!") - except FileNotFoundError: - pass - - block.input = 'preserve\n' - printer_2 = BlockPrinter(self.language) - printer_2.print_block(block, - core_includes=True, - limited_capi=self.limited_capi, - header_includes=self.includes) - write_file(destination.filename, printer_2.f.getvalue()) - continue - - return printer.f.getvalue() - - def _module_and_class( - self, fields: Sequence[str] - ) -> tuple[Module | Clinic, Class | None]: - """ - fields should be an iterable of field names. - returns a tuple of (module, class). - the module object could actually be self (a clinic object). - this function is only ever used to find the parent of where - a new class/module should go. - """ - parent: Clinic | Module | Class = self - module: Clinic | Module = self - cls: Class | None = None - - for idx, field in enumerate(fields): - if not isinstance(parent, Class): - if field in parent.modules: - parent = module = parent.modules[field] - continue - if field in parent.classes: - parent = cls = parent.classes[field] - else: - fullname = ".".join(fields[idx:]) - fail(f"Parent class or module {fullname!r} does not exist.") - - return module, cls - - def __repr__(self) -> str: - return "" - - -def parse_file( - filename: str, - *, - limited_capi: bool, - output: str | None = None, - verify: bool = True, -) -> None: - if not output: - output = filename - - extension = os.path.splitext(filename)[1][1:] - if not extension: - fail(f"Can't extract file type for file {filename!r}") - - try: - language = extensions[extension](filename) - except KeyError: - fail(f"Can't identify file type for file {filename!r}") - - with open(filename, encoding="utf-8") as f: - raw = f.read() - - # exit quickly if there are no clinic markers in the file - find_start_re = BlockParser("", language).find_start_re - if not find_start_re.search(raw): - return - - if LIMITED_CAPI_REGEX.search(raw): - limited_capi = True - - assert isinstance(language, CLanguage) - clinic = Clinic(language, - verify=verify, - filename=filename, - limited_capi=limited_capi) - cooked = clinic.parse(raw) - - write_file(output, cooked) - - -def compute_checksum( - input: str | None, - length: int | None = None -) -> str: - input = input or '' - s = hashlib.sha1(input.encode('utf-8')).hexdigest() - if length: - s = s[:length] - return s - - -class PythonParser: - def __init__(self, clinic: Clinic) -> None: - pass - - def parse(self, block: Block) -> None: - with contextlib.redirect_stdout(io.StringIO()) as s: - exec(block.input) - block.output = s.getvalue() - - -@dc.dataclass(repr=False) -class Module: - name: str - module: Module | Clinic - - def __post_init__(self) -> None: - self.parent = self.module - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -@dc.dataclass(repr=False) -class Class: - name: str - module: Module | Clinic - cls: Class | None - typedef: str - type_object: str - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -unsupported_special_methods: set[str] = set(""" - -__abs__ -__add__ -__and__ -__call__ -__delitem__ -__divmod__ -__eq__ -__float__ -__floordiv__ -__ge__ -__getattr__ -__getattribute__ -__getitem__ -__gt__ -__hash__ -__iadd__ -__iand__ -__ifloordiv__ -__ilshift__ -__imatmul__ -__imod__ -__imul__ -__index__ -__int__ -__invert__ -__ior__ -__ipow__ -__irshift__ -__isub__ -__iter__ -__itruediv__ -__ixor__ -__le__ -__len__ -__lshift__ -__lt__ -__matmul__ -__mod__ -__mul__ -__neg__ -__next__ -__or__ -__pos__ -__pow__ -__radd__ -__rand__ -__rdivmod__ -__repr__ -__rfloordiv__ -__rlshift__ -__rmatmul__ -__rmod__ -__rmul__ -__ror__ -__rpow__ -__rrshift__ -__rshift__ -__rsub__ -__rtruediv__ -__rxor__ -__setattr__ -__setitem__ -__str__ -__sub__ -__truediv__ -__xor__ - -""".strip().split()) - - -class FunctionKind(enum.Enum): - INVALID = enum.auto() - CALLABLE = enum.auto() - STATIC_METHOD = enum.auto() - CLASS_METHOD = enum.auto() - METHOD_INIT = enum.auto() - METHOD_NEW = enum.auto() - - @functools.cached_property - def new_or_init(self) -> bool: - return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} - - def __repr__(self) -> str: - return f"" - - -INVALID: Final = FunctionKind.INVALID -CALLABLE: Final = FunctionKind.CALLABLE -STATIC_METHOD: Final = FunctionKind.STATIC_METHOD -CLASS_METHOD: Final = FunctionKind.CLASS_METHOD -METHOD_INIT: Final = FunctionKind.METHOD_INIT -METHOD_NEW: Final = FunctionKind.METHOD_NEW - -ParamDict = dict[str, "Parameter"] -ReturnConverterType = Callable[..., "CReturnConverter"] - - -@dc.dataclass(repr=False) -class Function: - """ - Mutable duck type for inspect.Function. - - docstring - a str containing - * embedded line breaks - * text outdented to the left margin - * no trailing whitespace. - It will always be true that - (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) - """ - parameters: ParamDict = dc.field(default_factory=dict) - _: dc.KW_ONLY - name: str - module: Module | Clinic - cls: Class | None - c_basename: str - full_name: str - return_converter: CReturnConverter - kind: FunctionKind - coexist: bool - return_annotation: object = inspect.Signature.empty - docstring: str = '' - # docstring_only means "don't generate a machine-readable - # signature, just a normal docstring". it's True for - # functions with optional groups because we can't represent - # those accurately with inspect.Signature in 3.4. - docstring_only: bool = False - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.self_converter: self_converter | None = None - self.__render_parameters__: list[Parameter] | None = None - - @functools.cached_property - def displayname(self) -> str: - """Pretty-printable name.""" - if self.kind.new_or_init: - assert isinstance(self.cls, Class) - return self.cls.name - else: - return self.name - - @functools.cached_property - def fulldisplayname(self) -> str: - parent: Class | Module | Clinic | None - if self.kind.new_or_init: - parent = getattr(self.cls, "parent", None) - else: - parent = self.parent - name = self.displayname - while isinstance(parent, (Module, Class)): - name = f"{parent.name}.{name}" - parent = parent.parent - return name - - @property - def render_parameters(self) -> list[Parameter]: - if not self.__render_parameters__: - l: list[Parameter] = [] - self.__render_parameters__ = l - for p in self.parameters.values(): - p = p.copy() - p.converter.pre_render() - l.append(p) - return self.__render_parameters__ - - @property - def methoddef_flags(self) -> str | None: - if self.kind.new_or_init: - return None - flags = [] - match self.kind: - case FunctionKind.CLASS_METHOD: - flags.append('METH_CLASS') - case FunctionKind.STATIC_METHOD: - flags.append('METH_STATIC') - case _ as kind: - assert kind is FunctionKind.CALLABLE, f"unknown kind: {kind!r}" - if self.coexist: - flags.append('METH_COEXIST') - return '|'.join(flags) - - def __repr__(self) -> str: - return f'' - - def copy(self, **overrides: Any) -> Function: - f = dc.replace(self, **overrides) - f.parameters = { - name: value.copy(function=f) - for name, value in f.parameters.items() - } - return f - - -VersionTuple = tuple[int, int] - - -@dc.dataclass(repr=False, slots=True) -class Parameter: - """ - Mutable duck type of inspect.Parameter. - """ - name: str - kind: inspect._ParameterKind - _: dc.KW_ONLY - default: object = inspect.Parameter.empty - function: Function - converter: CConverter - annotation: object = inspect.Parameter.empty - docstring: str = '' - group: int = 0 - # (`None` signifies that there is no deprecation) - deprecated_positional: VersionTuple | None = None - deprecated_keyword: VersionTuple | None = None - right_bracket_count: int = dc.field(init=False, default=0) - - def __repr__(self) -> str: - return f'' - - def is_keyword_only(self) -> bool: - return self.kind == inspect.Parameter.KEYWORD_ONLY - - def is_positional_only(self) -> bool: - return self.kind == inspect.Parameter.POSITIONAL_ONLY - - def is_vararg(self) -> bool: - return self.kind == inspect.Parameter.VAR_POSITIONAL - - def is_optional(self) -> bool: - return not self.is_vararg() and (self.default is not unspecified) - - def copy( - self, - /, - *, - converter: CConverter | None = None, - function: Function | None = None, - **overrides: Any - ) -> Parameter: - function = function or self.function - if not converter: - converter = copy.copy(self.converter) - converter.function = function - return dc.replace(self, **overrides, function=function, converter=converter) - - def get_displayname(self, i: int) -> str: - if i == 0: - return 'argument' - if not self.is_positional_only(): - return f'argument {self.name!r}' - else: - return f'argument {i}' - - def render_docstring(self) -> str: - add, out = text_accumulator() - add(f" {self.name}\n") - for line in self.docstring.split("\n"): - add(f" {line}\n") - return out().rstrip() - - -CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) - -def add_c_converter( - f: CConverterClassT, - name: str | None = None -) -> CConverterClassT: - if not name: - name = f.__name__ - if not name.endswith('_converter'): - return f - name = name.removesuffix('_converter') - converters[name] = f - return f - -def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: - # automatically add converter for default format unit - # (but without stomping on the existing one if it's already - # set, in case you subclass) - if ((cls.format_unit not in ('O&', '')) and - (cls.format_unit not in legacy_converters)): - legacy_converters[cls.format_unit] = cls - return cls - -def add_legacy_c_converter( - format_unit: str, - **kwargs: Any -) -> Callable[[CConverterClassT], CConverterClassT]: - """ - Adds a legacy converter. - """ - def closure(f: CConverterClassT) -> CConverterClassT: - added_f: Callable[..., CConverter] - if not kwargs: - added_f = f - else: - # mypy's special-casing for functools.partial - # can't quite grapple with this code here - added_f = functools.partial(f, **kwargs) # type: ignore[arg-type] - if format_unit: - legacy_converters[format_unit] = added_f - return f - return closure - -class CConverterAutoRegister(type): - def __init__( - cls, name: str, bases: tuple[type, ...], classdict: dict[str, Any] - ) -> None: - converter_cls = cast(type["CConverter"], cls) - add_c_converter(converter_cls) - add_default_legacy_c_converter(converter_cls) - -class CConverter(metaclass=CConverterAutoRegister): - """ - For the init function, self, name, function, and default - must be keyword-or-positional parameters. All other - parameters must be keyword-only. - """ - - # The C name to use for this variable. - name: str - - # The Python name to use for this variable. - py_name: str - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type: str | None = None - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - # Or the magic value "unknown" if this value is a cannot be evaluated - # at Argument-Clinic-preprocessing time (but is presumed to be valid - # at runtime). - default: object = unspecified - - # If not None, default must be isinstance() of this type. - # (You can also specify a tuple of types.) - default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None - - # "default" converted into a C value, as a string. - # Or None if there is no default. - c_default: str | None = None - - # "default" converted into a Python value, as a string. - # Or None if there is no default. - py_default: str | None = None - - # The default value used to initialize the C variable when - # there is no default, but not specifying a default may - # result in an "uninitialized variable" warning. This can - # easily happen when using option groups--although - # properly-written code won't actually use the variable, - # the variable does get passed in to the _impl. (Ah, if - # only dataflow analysis could inline the static function!) - # - # This value is specified as a string. - # Every non-abstract subclass should supply a valid value. - c_ignored_default: str = 'NULL' - - # If true, wrap with Py_UNUSED. - unused = False - - # The C converter *function* to be used, if any. - # (If this is not None, format_unit must be 'O&'.) - converter: str | None = None - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into the _impl function? - impl_by_reference = False - - # Should Argument Clinic add a '&' before the name of - # the variable when passing it into PyArg_ParseTuple (AndKeywords)? - parse_by_reference = True - - ############################################################# - ############################################################# - ## You shouldn't need to read anything below this point to ## - ## write your own converter functions. ## - ############################################################# - ############################################################# - - # The "format unit" to specify for this variable when - # parsing arguments using PyArg_ParseTuple (AndKeywords). - # Custom converters should always use the default value of 'O&'. - format_unit = 'O&' - - # What encoding do we want for this variable? Only used - # by format units starting with 'e'. - encoding: str | None = None - - # Should this object be required to be a subclass of a specific type? - # If not None, should be a string representing a pointer to a - # PyTypeObject (e.g. "&PyUnicode_Type"). - # Only used by the 'O!' format unit (and the "object" converter). - subclass_of: str | None = None - - # See also the 'length_name' property. - # Only used by format units ending with '#'. - length = False - - # Should we show this parameter in the generated - # __text_signature__? This is *almost* always True. - # (It's only False for __new__, __init__, and METH_STATIC functions.) - show_in_signature = True - - # Overrides the name used in a text signature. - # The name used for a "self" parameter must be one of - # self, type, or module; however users can set their own. - # This lets the self_converter overrule the user-settable - # name, *just* for the text signature. - # Only set by self_converter. - signature_name: str | None = None - - include: Include | None = None - broken_limited_capi: bool = False - - # keep in sync with self_converter.__init__! - def __init__(self, - # Positional args: - name: str, - py_name: str, - function: Function, - default: object = unspecified, - *, # Keyword only args: - c_default: str | None = None, - py_default: str | None = None, - annotation: str | Literal[Sentinels.unspecified] = unspecified, - unused: bool = False, - **kwargs: Any - ) -> None: - self.name = ensure_legal_c_identifier(name) - self.py_name = py_name - self.unused = unused - - if default is not unspecified: - if (self.default_type - and default is not unknown - and not isinstance(default, self.default_type) - ): - if isinstance(self.default_type, type): - types_str = self.default_type.__name__ - else: - names = [cls.__name__ for cls in self.default_type] - types_str = ', '.join(names) - cls_name = self.__class__.__name__ - fail(f"{cls_name}: default value {default!r} for field " - f"{name!r} is not of type {types_str!r}") - self.default = default - - if c_default: - self.c_default = c_default - if py_default: - self.py_default = py_default - - if annotation is not unspecified: - fail("The 'annotation' parameter is not currently permitted.") - - # Make sure not to set self.function until after converter_init() has been called. - # This prevents you from caching information - # about the function in converter_init(). - # (That breaks if we get cloned.) - self.converter_init(**kwargs) - self.function = function - - # Add a custom __getattr__ method to improve the error message - # if somebody tries to access self.function in converter_init(). - # - # mypy will assume arbitrary access is okay for a class with a __getattr__ method, - # and that's not what we want, - # so put it inside an `if not TYPE_CHECKING` block - if not TYPE_CHECKING: - def __getattr__(self, attr): - if attr == "function": - fail( - f"{self.__class__.__name__!r} object has no attribute 'function'.\n" - f"Note: accessing self.function inside converter_init is disallowed!" - ) - return super().__getattr__(attr) - # this branch is just here for coverage reporting - else: # pragma: no cover - pass - - def converter_init(self) -> None: - pass - - def is_optional(self) -> bool: - return (self.default is not unspecified) - - def _render_self(self, parameter: Parameter, data: CRenderData) -> None: - self.parameter = parameter - name = self.parser_name - - # impl_arguments - s = ("&" if self.impl_by_reference else "") + name - data.impl_arguments.append(s) - if self.length: - data.impl_arguments.append(self.length_name) - - # impl_parameters - data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) - if self.length: - data.impl_parameters.append(f"Py_ssize_t {self.length_name}") - - def _render_non_self( - self, - parameter: Parameter, - data: CRenderData - ) -> None: - self.parameter = parameter - name = self.name - - # declarations - d = self.declaration(in_parser=True) - data.declarations.append(d) - - # initializers - initializers = self.initialize() - if initializers: - data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) - - # modifications - modifications = self.modify() - if modifications: - data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) - - # keywords - if parameter.is_vararg(): - pass - elif parameter.is_positional_only(): - data.keywords.append('') - else: - data.keywords.append(parameter.name) - - # format_units - if self.is_optional() and '|' not in data.format_units: - data.format_units.append('|') - if parameter.is_keyword_only() and '$' not in data.format_units: - data.format_units.append('$') - data.format_units.append(self.format_unit) - - # parse_arguments - self.parse_argument(data.parse_arguments) - - # post_parsing - if post_parsing := self.post_parsing(): - data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') - - # cleanup - cleanup = self.cleanup() - if cleanup: - data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - self._render_self(parameter, data) - self._render_non_self(parameter, data) - - @functools.cached_property - def length_name(self) -> str: - """Computes the name of the associated "length" variable.""" - assert self.length is not None - return self.parser_name + "_length" - - # Why is this one broken out separately? - # For "positional-only" function parsing, - # which generates a bunch of PyArg_ParseTuple calls. - def parse_argument(self, args: list[str]) -> None: - assert not (self.converter and self.encoding) - if self.format_unit == 'O&': - assert self.converter - args.append(self.converter) - - if self.encoding: - args.append(c_repr(self.encoding)) - elif self.subclass_of: - args.append(self.subclass_of) - - s = ("&" if self.parse_by_reference else "") + self.parser_name - args.append(s) - - if self.length: - args.append(f"&{self.length_name}") - - # - # All the functions after here are intended as extension points. - # - - def simple_declaration( - self, - by_reference: bool = False, - *, - in_parser: bool = False - ) -> str: - """ - Computes the basic declaration of the variable. - Used in computing the prototype declaration and the - variable declaration. - """ - assert isinstance(self.type, str) - prototype = [self.type] - if by_reference or not self.type.endswith('*'): - prototype.append(" ") - if by_reference: - prototype.append('*') - if in_parser: - name = self.parser_name - else: - name = self.name - if self.unused: - name = f"Py_UNUSED({name})" - prototype.append(name) - return "".join(prototype) - - def declaration(self, *, in_parser: bool = False) -> str: - """ - The C statement to declare this variable. - """ - declaration = [self.simple_declaration(in_parser=True)] - default = self.c_default - if not default and self.parameter.group: - default = self.c_ignored_default - if default: - declaration.append(" = ") - declaration.append(default) - declaration.append(";") - if self.length: - declaration.append('\n') - declaration.append(f"Py_ssize_t {self.length_name};") - return "".join(declaration) - - def initialize(self) -> str: - """ - The C statements required to set up this variable before parsing. - Returns a string containing this code indented at column 0. - If no initialization is necessary, returns an empty string. - """ - return "" - - def modify(self) -> str: - """ - The C statements required to modify this variable after parsing. - Returns a string containing this code indented at column 0. - If no modification is necessary, returns an empty string. - """ - return "" - - def post_parsing(self) -> str: - """ - The C statements required to do some operations after the end of parsing but before cleaning up. - Return a string containing this code indented at column 0. - If no operation is necessary, return an empty string. - """ - return "" - - def cleanup(self) -> str: - """ - The C statements required to clean up after this variable. - Returns a string containing this code indented at column 0. - If no cleanup is necessary, returns an empty string. - """ - return "" - - def pre_render(self) -> None: - """ - A second initialization function, like converter_init, - called just before rendering. - You are permitted to examine self.function here. - """ - pass - - def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: - assert '"' not in expected - if limited_capi: - if expected_literal: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be {expected}, not %.50s", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - return (f'PyErr_Format(PyExc_TypeError, ' - f'"{{{{name}}}}() {displayname} must be %.50s, not %.50s", ' - f'"{expected}", ' - f'{{argname}} == Py_None ? "None" : Py_TYPE({{argname}})->tp_name);') - else: - if expected_literal: - expected = f'"{expected}"' - return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' - - def format_code(self, fmt: str, *, - argname: str, - bad_argument: str | None = None, - bad_argument2: str | None = None, - **kwargs: Any) -> str: - if '{bad_argument}' in fmt: - if not bad_argument: - raise TypeError("required 'bad_argument' argument") - fmt = fmt.replace('{bad_argument}', bad_argument) - if '{bad_argument2}' in fmt: - if not bad_argument2: - raise TypeError("required 'bad_argument2' argument") - fmt = fmt.replace('{bad_argument2}', bad_argument2) - return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'O&': - return self.format_code(""" - if (!{converter}({argname}, &{paramname})) {{{{ - goto exit; - }}}} - """, - argname=argname, - converter=self.converter) - if self.format_unit == 'O!': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - if self.subclass_of in type_checks: - typecheck, typename = type_checks[self.subclass_of] - return self.format_code(""" - if (!{typecheck}({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), - typecheck=typecheck, typename=typename, cast=cast) - return self.format_code(""" - if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {cast}{argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', - expected_literal=False, limited_capi=limited_capi), - subclass_of=self.subclass_of, cast=cast) - if self.format_unit == 'O': - cast = '(%s)' % self.type if self.type != 'PyObject *' else '' - return self.format_code(""" - {paramname} = {cast}{argname}; - """, - argname=argname, cast=cast) - return None - - def set_template_dict(self, template_dict: TemplateDict) -> None: - pass - - @property - def parser_name(self) -> str: - if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 - return CLINIC_PREFIX + self.name - else: - return self.name - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - if self.include is not None: - raise ValueError("a converter only supports a single include") - self.include = Include(name, reason, condition) - -type_checks = { - '&PyLong_Type': ('PyLong_Check', 'int'), - '&PyTuple_Type': ('PyTuple_Check', 'tuple'), - '&PyList_Type': ('PyList_Check', 'list'), - '&PySet_Type': ('PySet_Check', 'set'), - '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), - '&PyDict_Type': ('PyDict_Check', 'dict'), - '&PyUnicode_Type': ('PyUnicode_Check', 'str'), - '&PyBytes_Type': ('PyBytes_Check', 'bytes'), - '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), -} - - -ConverterType = Callable[..., CConverter] -ConverterDict = dict[str, ConverterType] - -# maps strings to callables. -# these callables must be of the form: -# def foo(name, default, *, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CConverter object. -# The callable should not call builtins.print. -converters: ConverterDict = {} - -# maps strings to callables. -# these callables follow the same rules as those for "converters" above. -# note however that they will never be called with keyword-only parameters. -legacy_converters: ConverterDict = {} - -# maps strings to callables. -# these callables must be of the form: -# def foo(*, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CReturnConverter object. -# The callable should not call builtins.print. -ReturnConverterDict = dict[str, ReturnConverterType] -return_converters: ReturnConverterDict = {} - -TypeSet = set[bltns.type[Any]] - - -class bool_converter(CConverter): - type = 'int' - default_type = bool - format_unit = 'p' - c_ignored_default = '0' - - def converter_init(self, *, accept: TypeSet = {object}) -> None: - if accept == {int}: - self.format_unit = 'i' - elif accept != {object}: - fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified: - self.default = bool(self.default) - self.c_default = str(int(self.default)) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'p': - return self.format_code(""" - {paramname} = PyObject_IsTrue({argname}); - if ({paramname} < 0) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class defining_class_converter(CConverter): - """ - A special-case converter: - this is the default converter used for the defining class. - """ - type = 'PyTypeObject *' - format_unit = '' - show_in_signature = False - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - self._render_self(parameter, data) - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['defining_class_name'] = self.name - - -class char_converter(CConverter): - type = 'char' - default_type = (bytes, bytearray) - format_unit = 'c' - c_ignored_default = "'\0'" - - def converter_init(self) -> None: - if isinstance(self.default, self.default_type): - if len(self.default) != 1: - fail(f"char_converter: illegal default value {self.default!r}") - - self.c_default = repr(bytes(self.default))[1:] - if self.c_default == '"\'"': - self.c_default = r"'\''" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'c': - return self.format_code(""" - if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyBytes_AS_STRING({argname})[0]; - }}}} - else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyByteArray_AS_STRING({argname})[0]; - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -@add_legacy_c_converter('B', bitwise=True) -class unsigned_char_converter(CConverter): - type = 'unsigned char' - default_type = int - format_unit = 'b' - c_ignored_default = "'\0'" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'B' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'b': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < 0) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is less than minimum"); - goto exit; - }}}} - else if (ival > UCHAR_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - elif self.format_unit == 'B': - return self.format_code(""" - {{{{ - unsigned long ival = PyLong_AsUnsignedLongMask({argname}); - if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class byte_converter(unsigned_char_converter): pass - -class short_converter(CConverter): - type = 'short' - default_type = int - format_unit = 'h' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'h': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < SHRT_MIN) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is less than minimum"); - goto exit; - }}}} - else if (ival > SHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (short) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_short_converter(CConverter): - type = 'unsigned short' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'H' - else: - self.converter = '_PyLong_UnsignedShort_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'H': - return self.format_code(""" - {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > USHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned short"); - goto exit; - }}}} - {paramname} = (unsigned short) uval; - }}}} - """, - argname=argname) - -@add_legacy_c_converter('C', accept={str}) -class int_converter(CConverter): - type = 'int' - default_type = int - format_unit = 'i' - c_ignored_default = "0" - - def converter_init( - self, *, accept: TypeSet = {int}, type: str | None = None - ) -> None: - if accept == {str}: - self.format_unit = 'C' - elif accept != {int}: - fail(f"int_converter: illegal 'accept' argument {accept!r}") - if type is not None: - self.type = type - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'C': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_READ_CHAR({argname}, 0); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_int_converter(CConverter): - type = 'unsigned int' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'I' - else: - self.converter = '_PyLong_UnsignedInt_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'I': - return self.format_code(""" - {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > UINT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned int"); - goto exit; - }}}} - {paramname} = (unsigned int) uval; - }}}} - """, - argname=argname) - -class long_converter(CConverter): - type = 'long' - default_type = int - format_unit = 'l' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'l': - return self.format_code(""" - {paramname} = PyLong_AsLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_converter(CConverter): - type = 'unsigned long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'k' - else: - self.converter = '_PyLong_UnsignedLong_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'k': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLong({argname}); - if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class long_long_converter(CConverter): - type = 'long long' - default_type = int - format_unit = 'L' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'L': - return self.format_code(""" - {paramname} = PyLong_AsLongLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_long_converter(CConverter): - type = 'unsigned long long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'K' - else: - self.converter = '_PyLong_UnsignedLongLong_Converter' - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'K': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLongLong({argname}); - if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class Py_ssize_t_converter(CConverter): - type = 'Py_ssize_t' - c_ignored_default = "0" - - def converter_init(self, *, accept: TypeSet = {int}) -> None: - if accept == {int}: - self.format_unit = 'n' - self.default_type = int - self.add_include('pycore_abstract.h', '_PyNumber_Index()') - elif accept == {int, NoneType}: - self.converter = '_Py_convert_optional_to_ssize_t' - self.add_include('pycore_abstract.h', - '_Py_convert_optional_to_ssize_t()') - else: - fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - if limited_capi: - PyNumber_Index = 'PyNumber_Index' - else: - PyNumber_Index = '_PyNumber_Index' - return self.format_code(""" - {{{{ - Py_ssize_t ival = -1; - PyObject *iobj = {PyNumber_Index}({argname}); - if (iobj != NULL) {{{{ - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - }}}} - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - {paramname} = ival; - }}}} - """, - argname=argname, - PyNumber_Index=PyNumber_Index) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return self.format_code(""" - if ({argname} != Py_None) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), - ) - - -class slice_index_converter(CConverter): - type = 'Py_ssize_t' - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - if accept == {int}: - self.converter = '_PyEval_SliceIndexNotNone' - self.nullable = False - elif accept == {int, NoneType}: - self.converter = '_PyEval_SliceIndex' - self.nullable = True - else: - fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - if self.nullable: - return self.format_code(""" - if (!Py_IsNone({argname})) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - return 0; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "None or have an __index__ method"); - goto exit; - }}}} - }}}} - """, - argname=argname) - else: - return self.format_code(""" - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "have an __index__ method"); - goto exit; - }}}} - """, - argname=argname) - -class size_t_converter(CConverter): - type = 'size_t' - converter = '_PyLong_Size_t_Converter' - c_ignored_default = "0" - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - return self.format_code(""" - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsSize_t({argname}); - if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class fildes_converter(CConverter): - type = 'int' - converter = '_PyLong_FileDescriptor_Converter' - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - self.add_include('pycore_fileutils.h', - '_PyLong_FileDescriptor_Converter()') - - def _parse_arg(self, argname: str, displayname: str) -> str | None: - return self.format_code(""" - {paramname} = PyObject_AsFileDescriptor({argname}); - if ({paramname} == -1) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class float_converter(CConverter): - type = 'float' - default_type = float - format_unit = 'f' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'f': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); - }}}} - else - {{{{ - {paramname} = (float) PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class double_converter(CConverter): - type = 'double' - default_type = float - format_unit = 'd' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'd': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = PyFloat_AS_DOUBLE({argname}); - }}}} - else - {{{{ - {paramname} = PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class Py_complex_converter(CConverter): - type = 'Py_complex' - default_type = complex - format_unit = 'D' - c_ignored_default = "{0.0, 0.0}" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'D': - return self.format_code(""" - {paramname} = PyComplex_AsCComplex({argname}); - if (PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class object_converter(CConverter): - type = 'PyObject *' - format_unit = 'O' - - def converter_init( - self, *, - converter: str | None = None, - type: str | None = None, - subclass_of: str | None = None - ) -> None: - if converter: - if subclass_of: - fail("object: Cannot pass in both 'converter' and 'subclass_of'") - self.format_unit = 'O&' - self.converter = converter - elif subclass_of: - self.format_unit = 'O!' - self.subclass_of = subclass_of - - if type is not None: - self.type = type - - -# -# We define three conventions for buffer types in the 'accept' argument: -# -# buffer : any object supporting the buffer interface -# rwbuffer: any object supporting the buffer interface, but must be writeable -# robuffer: any object supporting the buffer interface, but must not be writeable -# - -class buffer: pass -class rwbuffer: pass -class robuffer: pass - -StrConverterKeyType = tuple[frozenset[type], bool, bool] - -def str_converter_key( - types: TypeSet, encoding: bool | str | None, zeroes: bool -) -> StrConverterKeyType: - return (frozenset(types), bool(encoding), bool(zeroes)) - -str_converter_argument_map: dict[StrConverterKeyType, str] = {} - -class str_converter(CConverter): - type = 'const char *' - default_type = (str, Null, NoneType) - format_unit = 's' - - def converter_init( - self, - *, - accept: TypeSet = {str}, - encoding: str | None = None, - zeroes: bool = False - ) -> None: - - key = str_converter_key(accept, encoding, zeroes) - format_unit = str_converter_argument_map.get(key) - if not format_unit: - fail("str_converter: illegal combination of arguments", key) - - self.format_unit = format_unit - self.length = bool(zeroes) - if encoding: - if self.default not in (Null, None, unspecified): - fail("str_converter: Argument Clinic doesn't support default values for encoded strings") - self.encoding = encoding - self.type = 'char *' - # sorry, clinic can't support preallocated buffers - # for es# and et# - self.c_default = "NULL" - if NoneType in accept and self.c_default == "Py_None": - self.c_default = "NULL" - - def post_parsing(self) -> str: - if self.encoding: - name = self.name - return f"PyMem_FREE({name});\n" - else: - return "" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 's': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - length_name=self.length_name) - if self.format_unit == 'z': - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - length_name=self.length_name) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -# -# This is the fourth or fifth rewrite of registering all the -# string converter format units. Previous approaches hid -# bugs--generally mismatches between the semantics of the format -# unit and the arguments necessary to represent those semantics -# properly. Hopefully with this approach we'll get it 100% right. -# -# The r() function (short for "register") both registers the -# mapping from arguments to format unit *and* registers the -# legacy C converter for that format unit. -# -ConverterKeywordDict = dict[str, TypeSet | bool] - -def r(format_unit: str, - *, - accept: TypeSet, - encoding: bool = False, - zeroes: bool = False -) -> None: - if not encoding and format_unit != 's': - # add the legacy c converters here too. - # - # note: add_legacy_c_converter can't work for - # es, es#, et, or et# - # because of their extra encoding argument - # - # also don't add the converter for 's' because - # the metaclass for CConverter adds it for us. - kwargs: ConverterKeywordDict = {} - if accept != {str}: - kwargs['accept'] = accept - if zeroes: - kwargs['zeroes'] = True - added_f = functools.partial(str_converter, **kwargs) - legacy_converters[format_unit] = added_f - - d = str_converter_argument_map - key = str_converter_key(accept, encoding, zeroes) - if key in d: - sys.exit("Duplicate keys specified for str_converter_argument_map!") - d[key] = format_unit - -r('es', encoding=True, accept={str}) -r('es#', encoding=True, zeroes=True, accept={str}) -r('et', encoding=True, accept={bytes, bytearray, str}) -r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) -r('s', accept={str}) -r('s#', zeroes=True, accept={robuffer, str}) -r('y', accept={robuffer}) -r('y#', zeroes=True, accept={robuffer}) -r('z', accept={str, NoneType}) -r('z#', zeroes=True, accept={robuffer, str, NoneType}) -del r - - -class PyBytesObject_converter(CConverter): - type = 'PyBytesObject *' - format_unit = 'S' - # accept = {bytes} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'S': - return self.format_code(""" - if (!PyBytes_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class PyByteArrayObject_converter(CConverter): - type = 'PyByteArrayObject *' - format_unit = 'Y' - # accept = {bytearray} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'Y': - return self.format_code(""" - if (!PyByteArray_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unicode_converter(CConverter): - type = 'PyObject *' - default_type = (str, Null, NoneType) - format_unit = 'U' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'U': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('u') -@add_legacy_c_converter('u#', zeroes=True) -@add_legacy_c_converter('Z', accept={str, NoneType}) -@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) -class Py_UNICODE_converter(CConverter): - type = 'const wchar_t *' - default_type = (str, Null, NoneType) - - def converter_init( - self, *, - accept: TypeSet = {str}, - zeroes: bool = False - ) -> None: - format_unit = 'Z' if accept=={str, NoneType} else 'u' - if zeroes: - format_unit += '#' - self.length = True - self.format_unit = format_unit - else: - self.accept = accept - if accept == {str}: - self.converter = '_PyUnicode_WideCharString_Converter' - elif accept == {str, NoneType}: - self.converter = '_PyUnicode_WideCharString_Opt_Converter' - else: - fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") - self.c_default = "NULL" - - def cleanup(self) -> str: - if self.length: - return "" - else: - return f"""PyMem_Free((void *){self.parser_name});\n""" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not self.length: - if self.accept == {str}: - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - elif self.accept == {str, NoneType}: - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('s*', accept={str, buffer}) -@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) -@add_legacy_c_converter('w*', accept={rwbuffer}) -class Py_buffer_converter(CConverter): - type = 'Py_buffer' - format_unit = 'y*' - impl_by_reference = True - c_ignored_default = "{NULL, NULL}" - - def converter_init(self, *, accept: TypeSet = {buffer}) -> None: - if self.default not in (unspecified, None): - fail("The only legal default value for Py_buffer is None.") - - self.c_default = self.c_ignored_default - - if accept == {str, buffer, NoneType}: - format_unit = 'z*' - elif accept == {str, buffer}: - format_unit = 's*' - elif accept == {buffer}: - format_unit = 'y*' - elif accept == {rwbuffer}: - format_unit = 'w*' - else: - fail("Py_buffer_converter: illegal combination of arguments") - - self.format_unit = format_unit - - def cleanup(self) -> str: - name = self.name - return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'y*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 's*': - return self.format_code(""" - if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t len; - const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); - if (ptr == NULL) {{{{ - goto exit; - }}}} - PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, 0); - }}}} - else {{{{ /* any bytes-like object */ - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument} - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 'w*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ - {bad_argument} - goto exit; - }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument2} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), - bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -def correct_name_for_self( - f: Function -) -> tuple[str, str]: - if f.kind in (CALLABLE, METHOD_INIT): - if f.cls: - return "PyObject *", "self" - return "PyObject *", "module" - if f.kind is STATIC_METHOD: - return "void *", "null" - if f.kind in (CLASS_METHOD, METHOD_NEW): - return "PyTypeObject *", "type" - raise AssertionError(f"Unhandled type of function f: {f.kind!r}") - -def required_type_for_self_for_parser( - f: Function -) -> str | None: - type, _ = correct_name_for_self(f) - if f.kind in (METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD): - return type - return None - - -class self_converter(CConverter): - """ - A special-case converter: - this is the default converter used for "self". - """ - type: str | None = None - format_unit = '' - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def pre_render(self) -> None: - f = self.function - default_type, default_name = correct_name_for_self(f) - self.signature_name = default_name - self.type = self.specified_type or self.type or default_type - - kind = self.function.kind - - if kind is STATIC_METHOD or kind.new_or_init: - self.show_in_signature = False - - # tp_new (METHOD_NEW) functions are of type newfunc: - # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); - # - # tp_init (METHOD_INIT) functions are of type initproc: - # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); - # - # All other functions generated by Argument Clinic are stored in - # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: - # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); - # However! We habitually cast these functions to PyCFunction, - # since functions that accept keyword arguments don't fit this signature - # but are stored there anyway. So strict type equality isn't important - # for these functions. - # - # So: - # - # * The name of the first parameter to the impl and the parsing function will always - # be self.name. - # - # * The type of the first parameter to the impl will always be of self.type. - # - # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is also self.type. - # This means that if you step into the parsing function, your "self" parameter - # is of the correct type, which may make debugging more pleasant. - # - # * Else if the function is tp_new (METHOD_NEW): - # * The type of the first parameter to the parsing function is "PyTypeObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyTypeObject *", we cast the first parameter to self.type - # in the impl call. - # - # * Else if the function is tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is "PyObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyObject *", we cast the first parameter to self.type - # in the impl call. - - @property - def parser_type(self) -> str: - assert self.type is not None - return required_type_for_self_for_parser(self.function) or self.type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - if self.function.kind is STATIC_METHOD: - return - - self._render_self(parameter, data) - - if self.type != self.parser_type: - # insert cast to impl_argument[0], aka self. - # we know we're in the first slot in all the CRenderData lists, - # because we render parameters in order, and self is always first. - assert len(data.impl_arguments) == 1 - assert data.impl_arguments[0] == self.name - assert self.type is not None - data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['self_name'] = self.name - template_dict['self_type'] = self.parser_type - kind = self.function.kind - cls = self.function.cls - - if kind.new_or_init and cls and cls.typedef: - if kind is METHOD_NEW: - type_check = ( - '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' - ).format(self.name) - else: - type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' - ' Py_TYPE({0})->tp_new == base_tp->tp_new)' - ).format(self.name) - - line = f'{type_check} &&\n ' - template_dict['self_type_check'] = line - - type_object = cls.type_object - type_ptr = f'PyTypeObject *base_tp = {type_object};' - template_dict['base_type_ptr'] = type_ptr - - -def add_c_return_converter( - f: ReturnConverterType, - name: str | None = None -) -> ReturnConverterType: - if not name: - name = f.__name__ - if not name.endswith('_return_converter'): - return f - name = name.removesuffix('_return_converter') - return_converters[name] = f - return f - - -class CReturnConverterAutoRegister(type): - def __init__( - cls: ReturnConverterType, - name: str, - bases: tuple[type, ...], - classdict: dict[str, Any] - ) -> None: - add_c_return_converter(cls) - - -class CReturnConverter(metaclass=CReturnConverterAutoRegister): - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type = 'PyObject *' - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - default: object = None - - def __init__( - self, - *, - py_default: str | None = None, - **kwargs: Any - ) -> None: - self.py_default = py_default - try: - self.return_converter_init(**kwargs) - except TypeError as e: - s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) - sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) - - def return_converter_init(self) -> None: ... - - def declare(self, data: CRenderData) -> None: - line: list[str] = [] - add = line.append - add(self.type) - if not self.type.endswith('*'): - add(' ') - add(data.converter_retval + ';') - data.declarations.append(''.join(line)) - data.return_value = data.converter_retval - - def err_occurred_if( - self, - expr: str, - data: CRenderData - ) -> None: - line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def err_occurred_if_null_pointer( - self, - variable: str, - data: CRenderData - ) -> None: - line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - -add_c_return_converter(CReturnConverter, 'object') - - -class bool_return_converter(CReturnConverter): - type = 'int' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1", data) - data.return_conversion.append( - f'return_value = PyBool_FromLong((long){data.converter_retval});\n' - ) - - -class long_return_converter(CReturnConverter): - type = 'long' - conversion_fn = 'PyLong_FromLong' - cast = '' - unsigned_cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) - data.return_conversion.append( - f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' - ) - - -class int_return_converter(long_return_converter): - type = 'int' - cast = '(long)' - - -class init_return_converter(long_return_converter): - """ - Special return converter for __init__ functions. - """ - type = 'int' - cast = '(long)' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - -class unsigned_long_return_converter(long_return_converter): - type = 'unsigned long' - conversion_fn = 'PyLong_FromUnsignedLong' - unsigned_cast = '(unsigned long)' - - -class unsigned_int_return_converter(unsigned_long_return_converter): - type = 'unsigned int' - cast = '(unsigned long)' - unsigned_cast = '(unsigned int)' - - -class Py_ssize_t_return_converter(long_return_converter): - type = 'Py_ssize_t' - conversion_fn = 'PyLong_FromSsize_t' - - -class size_t_return_converter(long_return_converter): - type = 'size_t' - conversion_fn = 'PyLong_FromSize_t' - unsigned_cast = '(size_t)' - - -class double_return_converter(CReturnConverter): - type = 'double' - cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1.0", data) - data.return_conversion.append( - f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' - ) - - -class float_return_converter(double_return_converter): - type = 'float' - cast = '(double)' - - -def eval_ast_expr( - node: ast.expr, - globals: dict[str, Any], - *, - filename: str = '-' -) -> Any: - """ - Takes an ast.Expr node. Compiles it into a function object, - then calls the function object with 0 arguments. - Returns the result of that function call. - - globals represents the globals dict the expression - should see. (There's no equivalent for "locals" here.) - """ - - if isinstance(node, ast.Expr): - node = node.value - - expr = ast.Expression(node) - co = compile(expr, filename, 'eval') - fn = FunctionType(co, globals) - return fn() - - -class IndentStack: - def __init__(self) -> None: - self.indents: list[int] = [] - self.margin: str | None = None - - def _ensure(self) -> None: - if not self.indents: - fail('IndentStack expected indents, but none are defined.') - - def measure(self, line: str) -> int: - """ - Returns the length of the line's margin. - """ - if '\t' in line: - fail('Tab characters are illegal in the Argument Clinic DSL.') - stripped = line.lstrip() - if not len(stripped): - # we can't tell anything from an empty line - # so just pretend it's indented like our current indent - self._ensure() - return self.indents[-1] - return len(line) - len(stripped) - - def infer(self, line: str) -> int: - """ - Infer what is now the current margin based on this line. - Returns: - 1 if we have indented (or this is the first margin) - 0 if the margin has not changed - -N if we have dedented N times - """ - indent = self.measure(line) - margin = ' ' * indent - if not self.indents: - self.indents.append(indent) - self.margin = margin - return 1 - current = self.indents[-1] - if indent == current: - return 0 - if indent > current: - self.indents.append(indent) - self.margin = margin - return 1 - # indent < current - if indent not in self.indents: - fail("Illegal outdent.") - outdent_count = 0 - while indent != current: - self.indents.pop() - current = self.indents[-1] - outdent_count -= 1 - self.margin = margin - return outdent_count - - @property - def depth(self) -> int: - """ - Returns how many margins are currently defined. - """ - return len(self.indents) - - def dedent(self, line: str) -> str: - """ - Dedents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .dedent() before calling .infer()" - margin = self.margin - indent = self.indents[-1] - if not line.startswith(margin): - fail('Cannot dedent; line does not start with the previous margin.') - return line[indent:] - - -StateKeeper = Callable[[str], None] -ConverterArgs = dict[str, Any] - -class ParamState(enum.IntEnum): - """Parameter parsing state. - - [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line - 01 2 3 4 5 6 <- state transitions - """ - # Before we've seen anything. - # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED - START = 0 - - # Left square backets before required params. - LEFT_SQUARE_BEFORE = 1 - - # In a group, before required params. - GROUP_BEFORE = 2 - - # Required params, positional-or-keyword or positional-only (we - # don't know yet). Renumber left groups! - REQUIRED = 3 - - # Positional-or-keyword or positional-only params that now must have - # default values. - OPTIONAL = 4 - - # In a group, after required params. - GROUP_AFTER = 5 - - # Right square brackets after required params. - RIGHT_SQUARE_AFTER = 6 - - -class FunctionNames(NamedTuple): - full_name: str - c_basename: str - - -class DSLParser: - function: Function | None - state: StateKeeper - keyword_only: bool - positional_only: bool - deprecated_positional: VersionTuple | None - deprecated_keyword: VersionTuple | None - group: int - parameter_state: ParamState - indent: IndentStack - kind: FunctionKind - coexist: bool - parameter_continuation: str - preserve_output: bool - from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') - - def __init__(self, clinic: Clinic) -> None: - self.clinic = clinic - - self.directives = {} - for name in dir(self): - # functions that start with directive_ are added to directives - _, s, key = name.partition("directive_") - if s: - self.directives[key] = getattr(self, name) - - # functions that start with at_ are too, with an @ in front - _, s, key = name.partition("at_") - if s: - self.directives['@' + key] = getattr(self, name) - - self.reset() - - def reset(self) -> None: - self.function = None - self.state = self.state_dsl_start - self.keyword_only = False - self.positional_only = False - self.deprecated_positional = None - self.deprecated_keyword = None - self.group = 0 - self.parameter_state: ParamState = ParamState.START - self.indent = IndentStack() - self.kind = CALLABLE - self.coexist = False - self.forced_text_signature: str | None = None - self.parameter_continuation = '' - self.preserve_output = False - - def directive_version(self, required: str) -> None: - global version - if version_comparitor(version, required) < 0: - fail("Insufficient Clinic version!\n" - f" Version: {version}\n" - f" Required: {required}") - - def directive_module(self, name: str) -> None: - fields = name.split('.')[:-1] - module, cls = self.clinic._module_and_class(fields) - if cls: - fail("Can't nest a module inside a class!") - - if name in module.modules: - fail(f"Already defined module {name!r}!") - - m = Module(name, module) - module.modules[name] = m - self.block.signatures.append(m) - - def directive_class( - self, - name: str, - typedef: str, - type_object: str - ) -> None: - fields = name.split('.') - name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - parent = cls or module - if name in parent.classes: - fail(f"Already defined class {name!r}!") - - c = Class(name, module, cls, typedef, type_object) - parent.classes[name] = c - self.block.signatures.append(c) - - def directive_set(self, name: str, value: str) -> None: - if name not in ("line_prefix", "line_suffix"): - fail(f"unknown variable {name!r}") - - value = value.format_map({ - 'block comment start': '/*', - 'block comment end': '*/', - }) - - self.clinic.__dict__[name] = value - - def directive_destination( - self, - name: str, - command: str, - *args: str - ) -> None: - match command: - case "new": - self.clinic.add_destination(name, *args) - case "clear": - self.clinic.get_destination(name).clear() - case _: - fail(f"unknown destination command {command!r}") - - - def directive_output( - self, - command_or_name: str, - destination: str = '' - ) -> None: - fd = self.clinic.destination_buffers - - if command_or_name == "preset": - preset = self.clinic.presets.get(destination) - if not preset: - fail(f"Unknown preset {destination!r}!") - fd.update(preset) - return - - if command_or_name == "push": - self.clinic.destination_buffers_stack.append(fd.copy()) - return - - if command_or_name == "pop": - if not self.clinic.destination_buffers_stack: - fail("Can't 'output pop', stack is empty!") - previous_fd = self.clinic.destination_buffers_stack.pop() - fd.update(previous_fd) - return - - # secret command for debugging! - if command_or_name == "print": - self.block.output.append(pprint.pformat(fd)) - self.block.output.append('\n') - return - - d = self.clinic.get_destination_buffer(destination) - - if command_or_name == "everything": - for name in list(fd): - fd[name] = d - return - - if command_or_name not in fd: - allowed = ["preset", "push", "pop", "print", "everything"] - allowed.extend(fd) - fail(f"Invalid command or destination name {command_or_name!r}. " - "Must be one of:\n -", - "\n - ".join([repr(word) for word in allowed])) - fd[command_or_name] = d - - def directive_dump(self, name: str) -> None: - self.block.output.append(self.clinic.get_destination(name).dump()) - - def directive_printout(self, *args: str) -> None: - self.block.output.append(' '.join(args)) - self.block.output.append('\n') - - def directive_preserve(self) -> None: - if self.preserve_output: - fail("Can't have 'preserve' twice in one block!") - self.preserve_output = True - - def at_classmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @classmethod, function is not a normal callable") - self.kind = CLASS_METHOD - - def at_staticmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @staticmethod, function is not a normal callable") - self.kind = STATIC_METHOD - - def at_coexist(self) -> None: - if self.coexist: - fail("Called @coexist twice!") - self.coexist = True - - def at_text_signature(self, text_signature: str) -> None: - if self.forced_text_signature: - fail("Called @text_signature twice!") - self.forced_text_signature = text_signature - - def parse(self, block: Block) -> None: - self.reset() - self.block = block - self.saved_output = self.block.output - block.output = [] - block_start = self.clinic.block_parser.line_number - lines = block.input.split('\n') - for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): - if '\t' in line: - fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', - line_number=block_start) - try: - self.state(line) - except ClinicError as exc: - exc.lineno = line_number - raise - - self.do_post_block_processing_cleanup(line_number) - block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) - - if self.preserve_output: - if block.output: - fail("'preserve' only works for blocks that don't produce any output!") - block.output = self.saved_output - - def in_docstring(self) -> bool: - """Return true if we are processing a docstring.""" - return self.state in { - self.state_parameter_docstring, - self.state_function_docstring, - } - - def valid_line(self, line: str) -> bool: - # ignore comment-only lines - if line.lstrip().startswith('#'): - return False - - # Ignore empty lines too - # (but not in docstring sections!) - if not self.in_docstring() and not line.strip(): - return False - - return True - - def next( - self, - state: StateKeeper, - line: str | None = None - ) -> None: - self.state = state - if line is not None: - self.state(line) - - def state_dsl_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # is it a directive? - fields = shlex.split(line) - directive_name = fields[0] - directive = self.directives.get(directive_name, None) - if directive: - try: - directive(*fields[1:]) - except TypeError as e: - fail(str(e)) - return - - self.next(self.state_modulename_name, line) - - @staticmethod - def parse_function_names(line: str) -> FunctionNames: - left, as_, right = line.partition(' as ') - full_name = left.strip() - c_basename = right.strip() - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") - if not c_basename: - fields = full_name.split(".") - if fields[-1] == '__new__': - fields.pop() - c_basename = "_".join(fields) - if not is_legal_py_identifier(full_name): - fail(f"Illegal function name: {full_name!r}") - if not is_legal_c_identifier(c_basename): - fail(f"Illegal C basename: {c_basename!r}") - return FunctionNames(full_name=full_name, c_basename=c_basename) - - def update_function_kind(self, fullname: str) -> None: - fields = fullname.split('.') - name = fields.pop() - _, cls = self.clinic._module_and_class(fields) - if name in unsupported_special_methods: - fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") - if name == '__new__': - if (self.kind is not CLASS_METHOD) or (not cls): - fail("'__new__' must be a class method!") - self.kind = METHOD_NEW - elif name == '__init__': - if (self.kind is not CALLABLE) or (not cls): - fail("'__init__' must be a normal method, not a class or static method!") - self.kind = METHOD_INIT - - def state_modulename_name(self, line: str) -> None: - # looking for declaration, which establishes the leftmost column - # line should be - # modulename.fnname [as c_basename] [-> return annotation] - # square brackets denote optional syntax. - # - # alternatively: - # modulename.fnname [as c_basename] = modulename.existing_fn_name - # clones the parameters and return converter from that - # function. you can't modify them. you must enter a - # new docstring. - # - # (but we might find a directive first!) - # - # this line is permitted to start with whitespace. - # we'll call this number of spaces F (for "function"). - - assert self.valid_line(line) - self.indent.infer(line) - - # are we cloning? - before, equals, existing = line.rpartition('=') - if equals: - full_name, c_basename = self.parse_function_names(before) - existing = existing.strip() - if is_legal_py_identifier(existing): - # we're cloning! - fields = [x.strip() for x in existing.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - for existing_function in (cls or module).functions: - if existing_function.name == function_name: - break - else: - print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) - print(f"{(cls or module).functions=}", file=sys.stderr) - fail(f"Couldn't find existing function {existing!r}!") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - self.update_function_kind(full_name) - overrides: dict[str, Any] = { - "name": function_name, - "full_name": full_name, - "module": module, - "cls": cls, - "c_basename": c_basename, - "docstring": "", - } - if not (existing_function.kind is self.kind and - existing_function.coexist == self.coexist): - # Allow __new__ or __init__ methods. - if existing_function.kind.new_or_init: - overrides["kind"] = self.kind - # Future enhancement: allow custom return converters - overrides["return_converter"] = CReturnConverter() - else: - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy(**overrides) - self.function = function - self.block.signatures.append(function) - (cls or module).functions.append(function) - self.next(self.state_function_docstring) - return - - line, _, returns = line.partition('->') - returns = returns.strip() - full_name, c_basename = self.parse_function_names(line) - - return_converter = None - if returns: - ast_input = f"def x() -> {returns}: pass" - try: - module_node = ast.parse(ast_input) - except SyntaxError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") - function_node = module_node.body[0] - assert isinstance(function_node, ast.FunctionDef) - try: - name, legacy, kwargs = self.parse_converter(function_node.returns) - if legacy: - fail(f"Legacy converter {name!r} not allowed as a return converter") - if name not in return_converters: - fail(f"No available return converter called {name!r}") - return_converter = return_converters[name](**kwargs) - except ValueError: - fail(f"Badly formed annotation for {full_name!r}: {returns!r}") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - self.update_function_kind(full_name) - if self.kind is METHOD_INIT and not return_converter: - return_converter = init_return_converter() - - if not return_converter: - return_converter = CReturnConverter() - - self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, - return_converter=return_converter, kind=self.kind, coexist=self.coexist) - self.block.signatures.append(self.function) - - # insert a self converter automatically - type, name = correct_name_for_self(self.function) - kwargs = {} - if cls and type == "PyObject *": - kwargs['type'] = cls.typedef - sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) - p_self = Parameter(name, inspect.Parameter.POSITIONAL_ONLY, - function=self.function, converter=sc) - self.function.parameters[name] = p_self - - (cls or module).functions.append(self.function) - self.next(self.state_parameters_start) - - # Now entering the parameters section. The rules, formally stated: - # - # * All lines must be indented with spaces only. - # * The first line must be a parameter declaration. - # * The first line must be indented. - # * This first line establishes the indent for parameters. - # * We'll call this number of spaces P (for "parameter"). - # * Thenceforth: - # * Lines indented with P spaces specify a parameter. - # * Lines indented with > P spaces are docstrings for the previous - # parameter. - # * We'll call this number of spaces D (for "docstring"). - # * All subsequent lines indented with >= D spaces are stored as - # part of the per-parameter docstring. - # * All lines will have the first D spaces of the indent stripped - # before they are stored. - # * It's illegal to have a line starting with a number of spaces X - # such that P < X < D. - # * A line with < P spaces is the first line of the function - # docstring, which ends processing for parameters and per-parameter - # docstrings. - # * The first line of the function docstring must be at the same - # indent as the function declaration. - # * It's illegal to have any line in the parameters section starting - # with X spaces such that F < X < P. (As before, F is the indent - # of the function declaration.) - # - # Also, currently Argument Clinic places the following restrictions on groups: - # * Each group must contain at least one parameter. - # * Each group may contain at most one group, which must be the furthest - # thing in the group from the required parameters. (The nested group - # must be the first in the group when it's before the required - # parameters, and the last thing in the group when after the required - # parameters.) - # * There may be at most one (top-level) group to the left or right of - # the required parameters. - # * You must specify a slash, and it must be after all parameters. - # (In other words: either all parameters are positional-only, - # or none are.) - # - # Said another way: - # * Each group must contain at least one parameter. - # * All left square brackets before the required parameters must be - # consecutive. (You can't have a left square bracket followed - # by a parameter, then another left square bracket. You can't - # have a left square bracket, a parameter, a right square bracket, - # and then a left square bracket.) - # * All right square brackets after the required parameters must be - # consecutive. - # - # These rules are enforced with a single state variable: - # "parameter_state". (Previously the code was a miasma of ifs and - # separate boolean state variables.) The states are defined in the - # ParamState class. - - def state_parameters_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # if this line is not indented, we have no parameters - if not self.indent.infer(line): - return self.next(self.state_function_docstring, line) - - self.parameter_continuation = '' - return self.next(self.state_parameter, line) - - - def to_required(self) -> None: - """ - Transition to the "required" parameter state. - """ - if self.parameter_state is not ParamState.REQUIRED: - self.parameter_state = ParamState.REQUIRED - assert self.function is not None - for p in self.function.parameters.values(): - p.group = -p.group - - def state_parameter(self, line: str) -> None: - assert isinstance(self.function, Function) - - if not self.valid_line(line): - return - - if self.parameter_continuation: - line = self.parameter_continuation + ' ' + line.lstrip() - self.parameter_continuation = '' - - assert self.indent.depth == 2 - indent = self.indent.infer(line) - if indent == -1: - # we outdented, must be to definition column - return self.next(self.state_function_docstring, line) - - if indent == 1: - # we indented, must be to new parameter docstring column - return self.next(self.state_parameter_docstring_start, line) - - line = line.rstrip() - if line.endswith('\\'): - self.parameter_continuation = line[:-1] - return - - line = line.lstrip() - version: VersionTuple | None = None - match = self.from_version_re.fullmatch(line) - if match: - line = match[1] - version = self.parse_version(match[2]) - - func = self.function - match line: - case '*': - self.parse_star(func, version) - case '[': - self.parse_opening_square_bracket(func) - case ']': - self.parse_closing_square_bracket(func) - case '/': - self.parse_slash(func, version) - case param: - self.parse_parameter(param) - - def parse_parameter(self, line: str) -> None: - assert self.function is not None - - match self.parameter_state: - case ParamState.START | ParamState.REQUIRED: - self.to_required() - case ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_BEFORE: - if not self.group: - self.to_required() - case ParamState.GROUP_AFTER | ParamState.OPTIONAL: - pass - case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") - - # handle "as" for parameters too - c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None - try: - ast_input = f"def x({base}): pass" - module = ast.parse(ast_input) - except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) - - function = module.body[0] - assert isinstance(function, ast.FunctionDef) - function_args = function.args - - if len(function_args.args) > 1: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") - if function_args.kwarg: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (**kwargs?): {line!r}") - - if function_args.vararg: - is_vararg = True - parameter = function_args.vararg - else: - is_vararg = False - parameter = function_args.args[0] - - parameter_name = parameter.arg - name, legacy, kwargs = self.parse_converter(parameter.annotation) - - if not default: - if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r}) " - "after a parameter with a default!") - value: Sentinels | Null - if is_vararg: - value = NULL - kwargs.setdefault('c_default', "NULL") - else: - value = unspecified - if 'py_default' in kwargs: - fail("You can't specify py_default without specifying a default value!") - else: - if is_vararg: - fail("Vararg can't take a default value!") - - if self.parameter_state is ParamState.REQUIRED: - self.parameter_state = ParamState.OPTIONAL - default = default.strip() - bad = False - ast_input = f"x = {default}" - try: - module = ast.parse(ast_input) - - if 'c_default' not in kwargs: - # we can only represent very simple data values in C. - # detect whether default is okay, via a denylist - # of disallowed ast nodes. - class DetectBadNodes(ast.NodeVisitor): - bad = False - def bad_node(self, node: ast.AST) -> None: - self.bad = True - - # inline function call - visit_Call = bad_node - # inline if statement ("x = 3 if y else z") - visit_IfExp = bad_node - - # comprehensions and generator expressions - visit_ListComp = visit_SetComp = bad_node - visit_DictComp = visit_GeneratorExp = bad_node - - # literals for advanced types - visit_Dict = visit_Set = bad_node - visit_List = visit_Tuple = bad_node - - # "starred": "a = [1, 2, 3]; *a" - visit_Starred = bad_node - - denylist = DetectBadNodes() - denylist.visit(module) - bad = denylist.bad - else: - # if they specify a c_default, we can be more lenient about the default value. - # but at least make an attempt at ensuring it's a valid expression. - try: - value = eval(default) - except NameError: - pass # probably a named constant - except Exception as e: - fail("Malformed expression given as default value " - f"{default!r} caused {e!r}") - else: - if value is unspecified: - fail("'unspecified' is not a legal default value!") - if bad: - fail(f"Unsupported expression as default value: {default!r}") - - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value - # mild hack: explicitly support NULL as a default value - c_default: str | None - if isinstance(expr, ast.Name) and expr.id == 'NULL': - value = NULL - py_default = '' - c_default = "NULL" - elif (isinstance(expr, ast.BinOp) or - (isinstance(expr, ast.UnaryOp) and - not (isinstance(expr.operand, ast.Constant) and - type(expr.operand.value) in {int, float, complex}) - )): - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify an expression ({default!r}) " - f"as your default value, " - f"you MUST specify a valid c_default.", - ast.dump(expr)) - py_default = default - value = unknown - elif isinstance(expr, ast.Attribute): - a = [] - n: ast.expr | ast.Attribute = expr - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - fail(f"Unsupported default value {default!r} " - "(looked like a Python constant)") - a.append(n.id) - py_default = ".".join(reversed(a)) - - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify a named constant ({py_default!r}) " - "as your default value, " - "you MUST specify a valid c_default.") - - try: - value = eval(py_default) - except NameError: - value = unknown - else: - value = ast.literal_eval(expr) - py_default = repr(value) - if isinstance(value, (bool, NoneType)): - c_default = "Py_" + py_default - elif isinstance(value, str): - c_default = c_repr(value) - else: - c_default = py_default - - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") - except (ValueError, AttributeError): - value = unknown - c_default = kwargs.get("c_default") - py_default = default - if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant " - f"({py_default!r}) as your default value, " - "you MUST specify a valid c_default.") - - kwargs.setdefault('c_default', c_default) - kwargs.setdefault('py_default', py_default) - - dict = legacy_converters if legacy else converters - legacy_str = "legacy " if legacy else "" - if name not in dict: - fail(f'{name!r} is not a valid {legacy_str}converter') - # if you use a c_name for the parameter, we just give that name to the converter - # but the parameter object gets the python name - converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) - - kind: inspect._ParameterKind - if is_vararg: - kind = inspect.Parameter.VAR_POSITIONAL - elif self.keyword_only: - kind = inspect.Parameter.KEYWORD_ONLY - else: - kind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if isinstance(converter, self_converter): - if len(self.function.parameters) == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'self' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'self' parameter cannot have a default value.") - if self.group: - fail("A 'self' parameter cannot be in an optional group.") - kind = inspect.Parameter.POSITIONAL_ONLY - self.parameter_state = ParamState.START - self.function.parameters.clear() - else: - fail("A 'self' parameter, if specified, must be the " - "very first thing in the parameter block.") - - if isinstance(converter, defining_class_converter): - _lp = len(self.function.parameters) - if _lp == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'defining_class' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'defining_class' parameter cannot have a default value.") - if self.group: - fail("A 'defining_class' parameter cannot be in an optional group.") - else: - fail("A 'defining_class' parameter, if specified, must either " - "be the first thing in the parameter block, or come just " - "after 'self'.") - - - p = Parameter(parameter_name, kind, function=self.function, - converter=converter, default=value, group=self.group, - deprecated_positional=self.deprecated_positional) - - names = [k.name for k in self.function.parameters.values()] - if parameter_name in names[1:]: - fail(f"You can't have two parameters named {parameter_name!r}!") - elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter {parameter_name!r} requires a custom C name") - - key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name - self.function.parameters[key] = p - - @staticmethod - def parse_converter( - annotation: ast.expr | None - ) -> tuple[str, bool, ConverterArgs]: - match annotation: - case ast.Constant(value=str() as value): - return value, True, {} - case ast.Name(name): - return name, False, {} - case ast.Call(func=ast.Name(name)): - symbols = globals() - kwargs: ConverterArgs = {} - for node in annotation.keywords: - if not isinstance(node.arg, str): - fail("Cannot use a kwarg splat in a function-call annotation") - kwargs[node.arg] = eval_ast_expr(node.value, symbols) - return name, False, kwargs - case _: - fail( - "Annotations must be either a name, a function call, or a string." - ) - - def parse_version(self, thenceforth: str) -> VersionTuple: - """Parse Python version in `[from ...]` marker.""" - assert isinstance(self.function, Function) - - try: - major, minor = thenceforth.split(".") - return int(major), int(minor) - except ValueError: - fail( - f"Function {self.function.name!r}: expected format '[from major.minor]' " - f"where 'major' and 'minor' are integers; got {thenceforth!r}" - ) - - def parse_star(self, function: Function, version: VersionTuple | None) -> None: - """Parse keyword-only parameter marker '*'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") - self.check_remaining_star() - self.keyword_only = True - else: - if self.keyword_only: - fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") - if self.deprecated_positional: - if self.deprecated_positional == version: - fail(f"Function {function.name!r} uses '* [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_positional < version: - fail(f"Function {function.name!r}: '* [from " - f"{version[0]}.{version[1]}]' must precede '* [from " - f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") - self.deprecated_positional = version - - def parse_opening_square_bracket(self, function: Function) -> None: - """Parse opening parameter group symbol '['.""" - match self.parameter_state: - case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.LEFT_SQUARE_BEFORE - case ParamState.REQUIRED | ParamState.GROUP_AFTER: - self.parameter_state = ParamState.GROUP_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.b)") - self.group += 1 - function.docstring_only = True - - def parse_closing_square_bracket(self, function: Function) -> None: - """Parse closing parameter group symbol ']'.""" - if not self.group: - fail(f"Function {function.name!r} has a ']' without a matching '['.") - if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name!r} has an empty group. " - "All groups must contain at least one parameter.") - self.group -= 1 - match self.parameter_state: - case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: - self.parameter_state = ParamState.RIGHT_SQUARE_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.c)") - - def parse_slash(self, function: Function, version: VersionTuple | None) -> None: - """Parse positional-only parameter marker '/'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.deprecated_keyword: - fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/' must precede '*'") - if self.positional_only: - fail(f"Function {function.name!r} uses '/' more than once.") - else: - if self.deprecated_keyword: - if self.deprecated_keyword == version: - fail(f"Function {function.name!r} uses '/ [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_keyword > version: - fail(f"Function {function.name!r}: '/ [from " - f"{version[0]}.{version[1]}]' must precede '/ [from " - f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") - self.positional_only = True - self.deprecated_keyword = version - if version is not None: - found = False - for p in reversed(function.parameters.values()): - found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - break - if not found: - fail(f"Function {function.name!r} specifies '/ [from ...]' " - f"without preceding parameters.") - # REQUIRED and OPTIONAL are allowed here, that allows positional-only - # without option groups to work (and have default values!) - allowed = { - ParamState.REQUIRED, - ParamState.OPTIONAL, - ParamState.RIGHT_SQUARE_AFTER, - ParamState.GROUP_BEFORE, - } - if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name!r} has an unsupported group configuration. " - f"(Unexpected state {self.parameter_state}.d)") - # fixup preceding parameters - for p in function.parameters.values(): - if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: - if version is None: - p.kind = inspect.Parameter.POSITIONAL_ONLY - elif p.deprecated_keyword is None: - p.deprecated_keyword = version - - def state_parameter_docstring_start(self, line: str) -> None: - assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" - self.parameter_docstring_indent = len(self.indent.margin) - assert self.indent.depth == 3 - return self.next(self.state_parameter_docstring, line) - - def docstring_append(self, obj: Function | Parameter, line: str) -> None: - """Add a rstripped line to the current docstring.""" - # gh-80282: We filter out non-ASCII characters from the docstring, - # since historically, some compilers may balk on non-ASCII input. - # If you're using Argument Clinic in an external project, - # you may not need to support the same array of platforms as CPython, - # so you may be able to remove this restriction. - matches = re.finditer(r'[^\x00-\x7F]', line) - if offending := ", ".join([repr(m[0]) for m in matches]): - warn("Non-ascii characters are not allowed in docstrings:", - offending) - - docstring = obj.docstring - if docstring: - docstring += "\n" - if stripped := line.rstrip(): - docstring += self.indent.dedent(stripped) - obj.docstring = docstring - - # every line of the docstring must start with at least F spaces, - # where F > P. - # these F spaces will be stripped. - def state_parameter_docstring(self, line: str) -> None: - if not self.valid_line(line): - return - - indent = self.indent.measure(line) - if indent < self.parameter_docstring_indent: - self.indent.infer(line) - assert self.indent.depth < 3 - if self.indent.depth == 2: - # back to a parameter - return self.next(self.state_parameter, line) - assert self.indent.depth == 1 - return self.next(self.state_function_docstring, line) - - assert self.function and self.function.parameters - last_param = next(reversed(self.function.parameters.values())) - self.docstring_append(last_param, line) - - # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str) -> None: - assert self.function is not None - - if self.group: - fail(f"Function {self.function.name!r} has a ']' without a matching '['.") - - if not self.valid_line(line): - return - - self.docstring_append(self.function, line) - - def format_docstring_signature( - self, f: Function, parameters: list[Parameter] - ) -> str: - text, add, output = _text_accumulator() - add(f.displayname) - if self.forced_text_signature: - add(self.forced_text_signature) - else: - add('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break - - - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) - s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') - - if p.is_vararg(): - p_add("*") - - name = p.converter.signature_name or p.name - p_add(name) - - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) - - if (p != last_p) or need_a_trailing_slash: - p_add(',') - - add_parameter(p_output()) - - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') - - # PEP 8 says: - # - # The Python standard library will not use function annotations - # as that would result in a premature commitment to a particular - # annotation style. Instead, the annotations are left for users - # to discover and experiment with useful annotation styles. - # - # therefore this is commented out: - # - # if f.return_converter.py_default: - # add(' -> ') - # add(f.return_converter.py_default) - - if not f.docstring_only: - add("\n" + sig_end_marker + "\n") - - signature_line = output() - - # now fix up the places where the brackets look wrong - return signature_line.replace(', ]', ',] ') - - @staticmethod - def format_docstring_parameters(params: list[Parameter]) -> str: - """Create substitution text for {parameters}""" - add, output = text_accumulator() - for p in params: - if p.docstring: - add(p.render_docstring()) - add('\n') - return output() - - def format_docstring(self) -> str: - assert self.function is not None - f = self.function - if f.kind.new_or_init and not f.docstring: - # don't render a docstring at all, no signature, nothing. - return f.docstring - - # Enforce the summary line! - # The first line of a docstring should be a summary of the function. - # It should fit on one line (80 columns? 79 maybe?) and be a paragraph - # by itself. - # - # Argument Clinic enforces the following rule: - # * either the docstring is empty, - # * or it must have a summary line. - # - # Guido said Clinic should enforce this: - # http://mail.python.org/pipermail/python-dev/2013-June/127110.html - - lines = f.docstring.split('\n') - if len(lines) >= 2: - if lines[1]: - fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" - "Every non-blank function docstring must start with " - "a single line summary followed by an empty line.") - elif len(lines) == 1: - # the docstring is only one line right now--the summary line. - # add an empty line after the summary line so we have space - # between it and the {parameters} we're about to add. - lines.append('') - - parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 - if parameters_marker_count > 1: - fail('You may not specify {parameters} more than once in a docstring!') - - # insert signature at front and params after the summary line - if not parameters_marker_count: - lines.insert(2, '{parameters}') - lines.insert(0, '{signature}') - - # finalize docstring - params = f.render_parameters - parameters = self.format_docstring_parameters(params) - signature = self.format_docstring_signature(f, params) - docstring = "\n".join(lines) - return linear_format(docstring, - signature=signature, - parameters=parameters).rstrip() - - def check_remaining_star(self, lineno: int | None = None) -> None: - assert isinstance(self.function, Function) - - if self.keyword_only: - symbol = '*' - elif self.deprecated_positional: - symbol = '* [from ...]' - else: - return - - for p in reversed(self.function.parameters.values()): - if self.keyword_only: - if p.kind == inspect.Parameter.KEYWORD_ONLY: - return - elif self.deprecated_positional: - if p.deprecated_positional == self.deprecated_positional: - return - break - - fail(f"Function {self.function.name!r} specifies {symbol!r} " - f"without following parameters.", line_number=lineno) - - def do_post_block_processing_cleanup(self, lineno: int) -> None: - """ - Called when processing the block is done. - """ - if not self.function: - return - - self.check_remaining_star(lineno) - self.function.docstring = self.format_docstring() - - - - -# maps strings to callables. -# the callable should return an object -# that implements the clinic parser -# interface (__init__ and parse). -# -# example parsers: -# "clinic", handles the Clinic DSL -# "python", handles running Python code -# -parsers: dict[str, Callable[[Clinic], Parser]] = { - 'clinic': DSLParser, - 'python': PythonParser, -} - - -def create_cli() -> argparse.ArgumentParser: - cmdline = argparse.ArgumentParser( - prog="clinic.py", - description="""Preprocessor for CPython C files. - -The purpose of the Argument Clinic is automating all the boilerplate involved -with writing argument parsing code for builtins and providing introspection -signatures ("docstrings") for CPython builtins. - -For more information see https://docs.python.org/3/howto/clinic.html""") - cmdline.add_argument("-f", "--force", action='store_true', - help="force output regeneration") - cmdline.add_argument("-o", "--output", type=str, - help="redirect file output to OUTPUT") - cmdline.add_argument("-v", "--verbose", action='store_true', - help="enable verbose mode") - cmdline.add_argument("--converters", action='store_true', - help=("print a list of all supported converters " - "and return converters")) - cmdline.add_argument("--make", action='store_true', - help="walk --srcdir to run over all relevant files") - cmdline.add_argument("--srcdir", type=str, default=os.curdir, - help="the directory tree to walk in --make mode") - cmdline.add_argument("--exclude", type=str, action="append", - help=("a file to exclude in --make mode; " - "can be given multiple times")) - cmdline.add_argument("--limited", dest="limited_capi", action='store_true', - help="use the Limited C API") - cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", - help="the list of files to process") - return cmdline - - -def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: - if ns.converters: - if ns.filename: - parser.error( - "can't specify --converters and a filename at the same time" - ) - converters: list[tuple[str, str]] = [] - return_converters: list[tuple[str, str]] = [] - ignored = set(""" - add_c_converter - add_c_return_converter - add_default_legacy_c_converter - add_legacy_c_converter - """.strip().split()) - module = globals() - for name in module: - for suffix, ids in ( - ("_return_converter", return_converters), - ("_converter", converters), - ): - if name in ignored: - continue - if name.endswith(suffix): - ids.append((name, name.removesuffix(suffix))) - break - print() - - print("Legacy converters:") - legacy = sorted(legacy_converters) - print(' ' + ' '.join(c for c in legacy if c[0].isupper())) - print(' ' + ' '.join(c for c in legacy if c[0].islower())) - print() - - for title, attribute, ids in ( - ("Converters", 'converter_init', converters), - ("Return converters", 'return_converter_init', return_converters), - ): - print(title + ":") - longest = -1 - for name, short_name in ids: - longest = max(longest, len(short_name)) - for name, short_name in sorted(ids, key=lambda x: x[1].lower()): - cls = module[name] - callable = getattr(cls, attribute, None) - if not callable: - continue - signature = inspect.signature(callable) - parameters = [] - for parameter_name, parameter in signature.parameters.items(): - if parameter.kind == inspect.Parameter.KEYWORD_ONLY: - if parameter.default != inspect.Parameter.empty: - s = f'{parameter_name}={parameter.default!r}' - else: - s = parameter_name - parameters.append(s) - print(' {}({})'.format(short_name, ', '.join(parameters))) - print() - print("All converters also accept (c_default=None, py_default=None, annotation=None).") - print("All return converters also accept (py_default=None).") - return - - if ns.make: - if ns.output or ns.filename: - parser.error("can't use -o or filenames with --make") - if not ns.srcdir: - parser.error("--srcdir must not be empty with --make") - if ns.exclude: - excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] - excludes = [os.path.normpath(f) for f in excludes] - else: - excludes = [] - for root, dirs, files in os.walk(ns.srcdir): - for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): - if rcs_dir in dirs: - dirs.remove(rcs_dir) - for filename in files: - # handle .c, .cpp and .h files - if not filename.endswith(('.c', '.cpp', '.h')): - continue - path = os.path.join(root, filename) - path = os.path.normpath(path) - if path in excludes: - continue - if ns.verbose: - print(path) - parse_file(path, - verify=not ns.force, limited_capi=ns.limited_capi) - return - - if not ns.filename: - parser.error("no input files") - - if ns.output and len(ns.filename) > 1: - parser.error("can't use -o with multiple filenames") - - for filename in ns.filename: - if ns.verbose: - print(filename) - parse_file(filename, output=ns.output, - verify=not ns.force, limited_capi=ns.limited_capi) - - -def main(argv: list[str] | None = None) -> NoReturn: - parser = create_cli() - args = parser.parse_args(argv) - try: - run_clinic(parser, args) - except ClinicError as exc: - sys.stderr.write(exc.report()) - sys.exit(1) - else: - sys.exit(0) +from libclinic.cli import main if __name__ == "__main__": diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py new file mode 100644 index 00000000000000..7c5cede2396677 --- /dev/null +++ b/Tools/clinic/libclinic/__init__.py @@ -0,0 +1,92 @@ +from typing import Final + +from .errors import ( + ClinicError, + warn, + fail, +) +from .formatting import ( + SIG_END_MARKER, + c_repr, + docstring_for_c_string, + format_escape, + indent_all_lines, + linear_format, + normalize_snippet, + pprint_words, + suffix_all_lines, + wrap_declarations, + wrapped_c_string_literal, +) +from .identifiers import ( + ensure_legal_c_identifier, + is_legal_c_identifier, + is_legal_py_identifier, +) +from .utils import ( + FormatCounterFormatter, + NULL, + Null, + Sentinels, + VersionTuple, + compute_checksum, + create_regex, + unknown, + unspecified, + write_file, +) + + +__all__ = [ + # Error handling + "ClinicError", + "warn", + "fail", + + # Formatting helpers + "SIG_END_MARKER", + "c_repr", + "docstring_for_c_string", + "format_escape", + "indent_all_lines", + "linear_format", + "normalize_snippet", + "pprint_words", + "suffix_all_lines", + "wrap_declarations", + "wrapped_c_string_literal", + + # Identifier helpers + "ensure_legal_c_identifier", + "is_legal_c_identifier", + "is_legal_py_identifier", + + # Utility functions + "FormatCounterFormatter", + "NULL", + "Null", + "Sentinels", + "VersionTuple", + "compute_checksum", + "create_regex", + "unknown", + "unspecified", + "write_file", +] + + +CLINIC_PREFIX: Final = "__clinic_" +CLINIC_PREFIXED_ARGS: Final = frozenset( + { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", + } +) diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py new file mode 100644 index 00000000000000..632bed3ce53dde --- /dev/null +++ b/Tools/clinic/libclinic/app.py @@ -0,0 +1,271 @@ +from __future__ import annotations +import os + +from collections.abc import Callable, Sequence +from typing import Any, TYPE_CHECKING + + +import libclinic +from libclinic import fail, warn +from libclinic.function import Class +from libclinic.block_parser import Block, BlockParser +from libclinic.codegen import BlockPrinter, Destination, CodeGen +from libclinic.parser import Parser, PythonParser +from libclinic.dsl_parser import DSLParser +if TYPE_CHECKING: + from libclinic.clanguage import CLanguage + from libclinic.function import ( + Module, Function, ClassDict, ModuleDict) + from libclinic.codegen import DestinationDict + + +# maps strings to callables. +# the callable should return an object +# that implements the clinic parser +# interface (__init__ and parse). +# +# example parsers: +# "clinic", handles the Clinic DSL +# "python", handles running Python code +# +parsers: dict[str, Callable[[Clinic], Parser]] = { + 'clinic': DSLParser, + 'python': PythonParser, +} + + +class Clinic: + + presets_text = """ +preset block +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset original +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset file +everything file +methoddef_ifndef file 1 +docstring_prototype suppress +parser_prototype suppress +impl_definition block + +preset buffer +everything buffer +methoddef_ifndef buffer 1 +impl_definition block +docstring_prototype suppress +impl_prototype suppress +parser_prototype suppress + +preset partial-buffer +everything buffer +methoddef_ifndef buffer 1 +docstring_prototype block +impl_prototype suppress +methoddef_define block +parser_prototype block +impl_definition block + +""" + + def __init__( + self, + language: CLanguage, + printer: BlockPrinter | None = None, + *, + filename: str, + limited_capi: bool, + verify: bool = True, + ) -> None: + # maps strings to Parser objects. + # (instantiated from the "parsers" global.) + self.parsers: dict[str, Parser] = {} + self.language: CLanguage = language + if printer: + fail("Custom printers are broken right now") + self.printer = printer or BlockPrinter(language) + self.verify = verify + self.limited_capi = limited_capi + self.filename = filename + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + self.codegen = CodeGen(self.limited_capi) + + self.line_prefix = self.line_suffix = '' + + self.destinations: DestinationDict = {} + self.add_destination("block", "buffer") + self.add_destination("suppress", "suppress") + self.add_destination("buffer", "buffer") + if filename: + self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") + + d = self.get_destination_buffer + self.destination_buffers = { + 'cpp_if': d('file'), + 'docstring_prototype': d('suppress'), + 'docstring_definition': d('file'), + 'methoddef_define': d('file'), + 'impl_prototype': d('file'), + 'parser_prototype': d('suppress'), + 'parser_definition': d('file'), + 'cpp_endif': d('file'), + 'methoddef_ifndef': d('file', 1), + 'impl_definition': d('block'), + } + + DestBufferType = dict[str, list[str]] + DestBufferList = list[DestBufferType] + + self.destination_buffers_stack: DestBufferList = [] + + self.presets: dict[str, dict[Any, Any]] = {} + preset = None + for line in self.presets_text.strip().split('\n'): + line = line.strip() + if not line: + continue + name, value, *options = line.split() + if name == 'preset': + self.presets[value] = preset = {} + continue + + if len(options): + index = int(options[0]) + else: + index = 0 + buffer = self.get_destination_buffer(value, index) + + if name == 'everything': + for name in self.destination_buffers: + preset[name] = buffer + continue + + assert name in self.destination_buffers + preset[name] = buffer + + def add_destination( + self, + name: str, + type: str, + *args: str + ) -> None: + if name in self.destinations: + fail(f"Destination already exists: {name!r}") + self.destinations[name] = Destination(name, type, self, args) + + def get_destination(self, name: str) -> Destination: + d = self.destinations.get(name) + if not d: + fail(f"Destination does not exist: {name!r}") + return d + + def get_destination_buffer( + self, + name: str, + item: int = 0 + ) -> list[str]: + d = self.get_destination(name) + return d.buffers[item] + + def parse(self, input: str) -> str: + printer = self.printer + self.block_parser = BlockParser(input, self.language, verify=self.verify) + for block in self.block_parser: + dsl_name = block.dsl_name + if dsl_name: + if dsl_name not in self.parsers: + assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." + self.parsers[dsl_name] = parsers[dsl_name](self) + parser = self.parsers[dsl_name] + parser.parse(block) + printer.print_block(block) + + # these are destinations not buffers + for name, destination in self.destinations.items(): + if destination.type == 'suppress': + continue + output = destination.dump() + + if output: + block = Block("", dsl_name="clinic", output=output) + + if destination.type == 'buffer': + block.input = "dump " + name + "\n" + warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") + printer.write("\n") + printer.print_block(block) + continue + + if destination.type == 'file': + try: + dirname = os.path.dirname(destination.filename) + try: + os.makedirs(dirname) + except FileExistsError: + if not os.path.isdir(dirname): + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") + if self.verify: + with open(destination.filename) as f: + parser_2 = BlockParser(f.read(), language=self.language) + blocks = list(parser_2) + if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") + except FileNotFoundError: + pass + + block.input = 'preserve\n' + includes = self.codegen.get_includes() + + printer_2 = BlockPrinter(self.language) + printer_2.print_block(block, header_includes=includes) + libclinic.write_file(destination.filename, + printer_2.f.getvalue()) + continue + + return printer.f.getvalue() + + def _module_and_class( + self, fields: Sequence[str] + ) -> tuple[Module | Clinic, Class | None]: + """ + fields should be an iterable of field names. + returns a tuple of (module, class). + the module object could actually be self (a clinic object). + this function is only ever used to find the parent of where + a new class/module should go. + """ + parent: Clinic | Module | Class = self + module: Clinic | Module = self + cls: Class | None = None + + for idx, field in enumerate(fields): + if not isinstance(parent, Class): + if field in parent.modules: + parent = module = parent.modules[field] + continue + if field in parent.classes: + parent = cls = parent.classes[field] + else: + fullname = ".".join(fields[idx:]) + fail(f"Parent class or module {fullname!r} does not exist.") + + return module, cls + + def __repr__(self) -> str: + return "" diff --git a/Tools/clinic/libclinic/block_parser.py b/Tools/clinic/libclinic/block_parser.py new file mode 100644 index 00000000000000..4c0198b53592a9 --- /dev/null +++ b/Tools/clinic/libclinic/block_parser.py @@ -0,0 +1,256 @@ +from __future__ import annotations +import collections +import dataclasses as dc +import re +import shlex +from typing import Any + +import libclinic +from libclinic import fail, ClinicError +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function) + + +@dc.dataclass(slots=True, repr=False) +class Block: + r""" + Represents a single block of text embedded in + another file. If dsl_name is None, the block represents + verbatim text, raw original text from the file, in + which case "input" will be the only non-false member. + If dsl_name is not None, the block represents a Clinic + block. + + input is always str, with embedded \n characters. + input represents the original text from the file; + if it's a Clinic block, it is the original text with + the body_prefix and redundant leading whitespace removed. + + dsl_name is either str or None. If str, it's the text + found on the start line of the block between the square + brackets. + + signatures is a list. + It may only contain clinic.Module, clinic.Class, and + clinic.Function objects. At the moment it should + contain at most one of each. + + output is either str or None. If str, it's the output + from this block, with embedded '\n' characters. + + indent is a str. It's the leading whitespace + that was found on every line of input. (If body_prefix is + not empty, this is the indent *after* removing the + body_prefix.) + + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that + was found in front of every line of input *before* the + "body_prefix" (see the Language object). If body_prefix + is empty, preindent must always be empty too. + + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: + ____#/*[python] + ____#__for a in range(20): + ____#____print(a) + ____#[python]*/ + "preindent" would be "____" and "indent" would be "__". + + """ + input: str + dsl_name: str | None = None + signatures: list[Module | Class | Function] = dc.field(default_factory=list) + output: Any = None # TODO: Very dynamic; probably untypeable in its current form? + indent: str = '' + + def __repr__(self) -> str: + dsl_name = self.dsl_name or "text" + def summarize(s: object) -> str: + s = repr(s) + if len(s) > 30: + return s[:26] + "..." + s[0] + return s + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" + + +class BlockParser: + """ + Block-oriented parser for Argument Clinic. + Iterator, yields Block objects. + """ + + def __init__( + self, + input: str, + language: Language, + *, + verify: bool = True + ) -> None: + """ + "input" should be a str object + with embedded \n characters. + + "language" should be a Language object. + """ + language.validate() + + self.input = collections.deque(reversed(input.splitlines(keepends=True))) + self.block_start_line_number = self.line_number = 0 + + self.language = language + before, _, after = language.start_line.partition('{dsl_name}') + assert _ == '{dsl_name}' + self.find_start_re = libclinic.create_regex(before, after, + whole_line=False) + self.start_re = libclinic.create_regex(before, after) + self.verify = verify + self.last_checksum_re: re.Pattern[str] | None = None + self.last_dsl_name: str | None = None + self.dsl_name: str | None = None + self.first_block = True + + def __iter__(self) -> BlockParser: + return self + + def __next__(self) -> Block: + while True: + if not self.input: + raise StopIteration + + if self.dsl_name: + try: + return_value = self.parse_clinic_block(self.dsl_name) + except ClinicError as exc: + exc.filename = self.language.filename + exc.lineno = self.line_number + raise + self.dsl_name = None + self.first_block = False + return return_value + block = self.parse_verbatim_block() + if self.first_block and not block.input: + continue + self.first_block = False + return block + + + def is_start_line(self, line: str) -> str | None: + match = self.start_re.match(line.lstrip()) + return match.group(1) if match else None + + def _line(self, lookahead: bool = False) -> str: + self.line_number += 1 + line = self.input.pop() + if not lookahead: + self.language.parse_line(line) + return line + + def parse_verbatim_block(self) -> Block: + lines = [] + self.block_start_line_number = self.line_number + + while self.input: + line = self._line() + dsl_name = self.is_start_line(line) + if dsl_name: + self.dsl_name = dsl_name + break + lines.append(line) + + return Block("".join(lines)) + + def parse_clinic_block(self, dsl_name: str) -> Block: + in_lines = [] + self.block_start_line_number = self.line_number + 1 + stop_line = self.language.stop_line.format(dsl_name=dsl_name) + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + + def is_stop_line(line: str) -> bool: + # make sure to recognize stop line even if it + # doesn't end with EOL (it could be the very end of the file) + if line.startswith(stop_line): + remainder = line.removeprefix(stop_line) + if remainder and not remainder.isspace(): + fail(f"Garbage after stop line: {remainder!r}") + return True + else: + # gh-92256: don't allow incorrectly formatted stop lines + if line.lstrip().startswith(stop_line): + fail(f"Whitespace is not allowed before the stop line: {line!r}") + return False + + # consume body of program + while self.input: + line = self._line() + if is_stop_line(line) or self.is_start_line(line): + break + if body_prefix: + line = line.lstrip() + assert line.startswith(body_prefix) + line = line.removeprefix(body_prefix) + in_lines.append(line) + + # consume output and checksum line, if present. + if self.last_dsl_name == dsl_name: + checksum_re = self.last_checksum_re + else: + before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') + assert _ == '{arguments}' + checksum_re = libclinic.create_regex(before, after, word=False) + self.last_dsl_name = dsl_name + self.last_checksum_re = checksum_re + assert checksum_re is not None + + # scan forward for checksum line + out_lines = [] + arguments = None + while self.input: + line = self._line(lookahead=True) + match = checksum_re.match(line.lstrip()) + arguments = match.group(1) if match else None + if arguments: + break + out_lines.append(line) + if self.is_start_line(line): + break + + output: str | None + output = "".join(out_lines) + if arguments: + d = {} + for field in shlex.split(arguments): + name, equals, value = field.partition('=') + if not equals: + fail(f"Mangled Argument Clinic marker line: {line!r}") + d[name.strip()] = value.strip() + + if self.verify: + if 'input' in d: + checksum = d['output'] + else: + checksum = d['checksum'] + + computed = libclinic.compute_checksum(output, len(checksum)) + if checksum != computed: + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " + "Suggested fix: remove all generated code including " + "the end marker, or use the '-f' option.") + else: + # put back output + output_lines = output.splitlines(keepends=True) + self.line_number -= len(output_lines) + self.input.extend(reversed(output_lines)) + output = None + + return Block("".join(in_lines), dsl_name, output=output) diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py new file mode 100644 index 00000000000000..73d47833d97294 --- /dev/null +++ b/Tools/clinic/libclinic/clanguage.py @@ -0,0 +1,537 @@ +from __future__ import annotations +import itertools +import sys +import textwrap +from typing import TYPE_CHECKING, Literal, Final +from operator import attrgetter +from collections.abc import Iterable + +import libclinic +from libclinic import ( + unspecified, fail, Sentinels, VersionTuple) +from libclinic.codegen import CRenderData, TemplateDict, CodeGen +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function, Parameter, + permute_optional_groups, + GETTER, SETTER, METHOD_INIT) +from libclinic.converters import self_converter +from libclinic.parse_args import ParseArgsCodeGen +if TYPE_CHECKING: + from libclinic.app import Clinic + + +def c_id(name: str) -> str: + if len(name) == 1 and ord(name) < 256: + if name.isalnum(): + return f"_Py_LATIN1_CHR('{name}')" + else: + return f'_Py_LATIN1_CHR({ord(name)})' + else: + return f'&_Py_ID({name})' + + +class CLanguage(Language): + + body_prefix = "#" + language = 'C' + start_line = "/*[{dsl_name} input]" + body_prefix = "" + stop_line = "[{dsl_name} start generated code]*/" + checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + + COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + // Emit compiler warnings when we get to Python {major}.{minor}. + #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 + # error {message} + #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 + # ifdef _MSC_VER + # pragma message ({message}) + # else + # warning {message} + # endif + #endif + """ + DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + if ({condition}) {{{{{errcheck} + if (PyErr_WarnEx(PyExc_DeprecationWarning, + {message}, 1)) + {{{{ + goto exit; + }}}} + }}}} + """ + + def __init__(self, filename: str) -> None: + super().__init__(filename) + self.cpp = libclinic.cpp.Monitor(filename) + + def parse_line(self, line: str) -> None: + self.cpp.writeline(line) + + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + function = None + for o in signatures: + if isinstance(o, Function): + if function: + fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) + function = o + return self.render_function(clinic, function) + + def compiler_deprecated_warning( + self, + func: Function, + parameters: list[Parameter], + ) -> str | None: + minversion: VersionTuple | None = None + for p in parameters: + for version in p.deprecated_positional, p.deprecated_keyword: + if version and (not minversion or minversion > version): + minversion = version + if not minversion: + return None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + message = f"Update the clinic input of {func.full_name!r}." + code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( + major=minversion[0], + minor=minversion[1], + message=libclinic.c_repr(message), + ) + return libclinic.normalize_snippet(code) + + def deprecate_positional_use( + self, + func: Function, + params: dict[int, Parameter], + ) -> str: + assert len(params) > 0 + first_pos = next(iter(params)) + last_pos = next(reversed(params)) + + # Format the deprecation message. + if len(params) == 1: + condition = f"nargs == {first_pos+1}" + amount = f"{first_pos+1} " if first_pos else "" + pl = "s" + else: + condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" + amount = f"more than {first_pos} " if first_pos else "" + pl = "s" if first_pos != 1 else "" + message = ( + f"Passing {amount}positional argument{pl} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_positional") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + if len(names) == 1: + message += ( + f" Parameter {pstr} will become a keyword-only parameter " + f"in Python {major}.{minor}." + ) + else: + message += ( + f" Parameters {pstr} will become keyword-only parameters " + f"in Python {major}.{minor}." + ) + + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck="", + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def deprecate_keyword_use( + self, + func: Function, + params: dict[int, Parameter], + argname_fmt: str | None = None, + *, + fastcall: bool, + codegen: CodeGen, + ) -> str: + assert len(params) > 0 + last_param = next(reversed(params.values())) + limited_capi = codegen.limited_capi + + # Format the deprecation message. + containscheck = "" + conditions = [] + for i, p in params.items(): + if p.is_optional(): + if argname_fmt: + conditions.append(f"nargs < {i+1} && {argname_fmt % i}") + elif fastcall: + conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, {c_id(p.name)})") + containscheck = "PySequence_Contains" + codegen.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, {c_id(p.name)})") + containscheck = "PyDict_Contains" + codegen.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions = [f"nargs < {i+1}"] + condition = ") || (".join(conditions) + if len(conditions) > 1: + condition = f"(({condition}))" + if last_param.is_optional(): + if fastcall: + if limited_capi: + condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + else: + if limited_capi: + condition = f"kwargs && PyDict_Size(kwargs) && {condition}" + else: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + names = [repr(p.name) for p in params.values()] + pstr = libclinic.pprint_words(names) + pl = 's' if len(params) != 1 else '' + message = ( + f"Passing keyword argument{pl} {pstr} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_keyword") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + pl = 's' if len(names) != 1 else '' + message += ( + f" Parameter{pl} {pstr} will become positional-only " + f"in Python {major}.{minor}." + ) + + if containscheck: + errcheck = f""" + if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail + goto exit; + }}}}""" + else: + errcheck = "" + if argname_fmt: + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck=errcheck, + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def output_templates( + self, + f: Function, + codegen: CodeGen, + ) -> dict[str, str]: + args = ParseArgsCodeGen(f, codegen) + return args.parse_args(self) + + @staticmethod + def group_to_variable_name(group: int) -> str: + adjective = "left_" if group < 0 else "right_" + return "group_" + adjective + str(abs(group)) + + def render_option_group_parsing( + self, + f: Function, + template_dict: TemplateDict, + limited_capi: bool, + ) -> None: + # positional only, grouped, optional arguments! + # can be optional on the left or right. + # here's an example: + # + # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] + # + # Here group D are required, and all other groups are optional. + # (Group D's "group" is actually None.) + # We can figure out which sets of arguments we have based on + # how many arguments are in the tuple. + # + # Note that you need to count up on both sides. For example, + # you could have groups C+D, or C+D+E, or C+D+E+F. + # + # What if the number of arguments leads us to an ambiguous result? + # Clinic prefers groups on the left. So in the above example, + # five arguments would map to B+C, not C+D. + + out = [] + parameters = list(f.parameters.values()) + if isinstance(parameters[0].converter, self_converter): + del parameters[0] + + group: list[Parameter] | None = None + left = [] + right = [] + required: list[Parameter] = [] + last: int | Literal[Sentinels.unspecified] = unspecified + + for p in parameters: + group_id = p.group + if group_id != last: + last = group_id + group = [] + if group_id < 0: + left.append(group) + elif group_id == 0: + group = required + else: + right.append(group) + assert group is not None + group.append(p) + + count_min = sys.maxsize + count_max = -1 + + if limited_capi: + nargs = 'PyTuple_Size(args)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + out.append(f"switch ({nargs}) {{\n") + for subset in permute_optional_groups(left, required, right): + count = len(subset) + count_min = min(count_min, count) + count_max = max(count_max, count) + + if count == 0: + out.append(""" case 0: + break; +""") + continue + + group_ids = {p.group for p in subset} # eliminate duplicates + d: dict[str, str | int] = {} + d['count'] = count + d['name'] = f.name + d['format_units'] = "".join(p.converter.format_unit for p in subset) + + parse_arguments: list[str] = [] + for p in subset: + p.converter.parse_argument(parse_arguments) + d['parse_arguments'] = ", ".join(parse_arguments) + + group_ids.discard(0) + lines = "\n".join([ + self.group_to_variable_name(g) + " = 1;" + for g in group_ids + ]) + + s = """\ + case {count}: + if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + {group_booleans} + break; +""" + s = libclinic.linear_format(s, group_booleans=lines) + s = s.format_map(d) + out.append(s) + + out.append(" default:\n") + s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' + out.append(s.format(f.full_name, count_min, count_max)) + out.append(' goto exit;\n') + out.append("}") + + template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) + + def render_function( + self, + clinic: Clinic, + f: Function | None + ) -> str: + if f is None: + return "" + + codegen = clinic.codegen + data = CRenderData() + + assert f.parameters, "We should always have a 'self' at this point!" + parameters = f.render_parameters + converters = [p.converter for p in parameters] + + templates = self.output_templates(f, codegen) + + f_self = parameters[0] + selfless = parameters[1:] + assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" + + if f.critical_section: + match len(f.target_critical_section): + case 0: + lock = 'Py_BEGIN_CRITICAL_SECTION({self_name});' + unlock = 'Py_END_CRITICAL_SECTION();' + case 1: + lock = 'Py_BEGIN_CRITICAL_SECTION({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION();' + case _: + lock = 'Py_BEGIN_CRITICAL_SECTION2({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION2();' + data.lock.append(lock) + data.unlock.append(unlock) + + last_group = 0 + first_optional = len(selfless) + positional = selfless and selfless[-1].is_positional_only() + has_option_groups = False + + # offset i by -1 because first_optional needs to ignore self + for i, p in enumerate(parameters, -1): + c = p.converter + + if (i != -1) and (p.default is not unspecified): + first_optional = min(first_optional, i) + + if p.is_vararg(): + data.cleanup.append(f"Py_XDECREF({c.parser_name});") + + # insert group variable + group = p.group + if last_group != group: + last_group = group + if group: + group_name = self.group_to_variable_name(group) + data.impl_arguments.append(group_name) + data.declarations.append("int " + group_name + " = 0;") + data.impl_parameters.append("int " + group_name) + has_option_groups = True + + c.render(p, data) + + if has_option_groups and (not positional): + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") + + # HACK + # when we're METH_O, but have a custom return converter, + # we use "impl_parameters" for the parsing function + # because that works better. but that means we must + # suppress actually declaring the impl's parameters + # as variables in the parsing function. but since it's + # METH_O, we have exactly one anyway, so we know exactly + # where it is. + if ("METH_O" in templates['methoddef_define'] and + '{impl_parameters}' in templates['parser_prototype']): + data.declarations.pop(0) + + full_name = f.full_name + template_dict = {'full_name': full_name} + template_dict['name'] = f.displayname + if f.kind in {GETTER, SETTER}: + template_dict['getset_name'] = f.c_basename.upper() + template_dict['getset_basename'] = f.c_basename + if f.kind is GETTER: + template_dict['c_basename'] = f.c_basename + "_get" + elif f.kind is SETTER: + template_dict['c_basename'] = f.c_basename + "_set" + # Implicitly add the setter value parameter. + data.impl_parameters.append("PyObject *value") + data.impl_arguments.append("value") + else: + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + + template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) + template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' + template_dict['target_critical_section'] = ', '.join(f.target_critical_section) + for converter in converters: + converter.set_template_dict(template_dict) + + if f.kind not in {SETTER, METHOD_INIT}: + f.return_converter.render(f, data) + template_dict['impl_return_type'] = f.return_converter.type + + template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) + template_dict['initializers'] = "\n\n".join(data.initializers) + template_dict['modifications'] = '\n\n'.join(data.modifications) + template_dict['keywords_c'] = ' '.join('"' + k + '",' + for k in data.keywords) + keywords = [k for k in data.keywords if k] + template_dict['keywords_py'] = ' '.join(c_id(k) + ',' + for k in keywords) + template_dict['format_units'] = ''.join(data.format_units) + template_dict['parse_arguments'] = ', '.join(data.parse_arguments) + if data.parse_arguments: + template_dict['parse_arguments_comma'] = ','; + else: + template_dict['parse_arguments_comma'] = ''; + template_dict['impl_parameters'] = ", ".join(data.impl_parameters) + template_dict['impl_arguments'] = ", ".join(data.impl_arguments) + + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) + template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) + + template_dict['return_value'] = data.return_value + template_dict['lock'] = "\n".join(data.lock) + template_dict['unlock'] = "\n".join(data.unlock) + + # used by unpack tuple code generator + unpack_min = first_optional + unpack_max = len(selfless) + template_dict['unpack_min'] = str(unpack_min) + template_dict['unpack_max'] = str(unpack_max) + + if has_option_groups: + self.render_option_group_parsing(f, template_dict, + limited_capi=codegen.limited_capi) + + # buffers, not destination + for name, destination in clinic.destination_buffers.items(): + template = templates[name] + if has_option_groups: + template = libclinic.linear_format(template, + option_group_parsing=template_dict['option_group_parsing']) + template = libclinic.linear_format(template, + declarations=template_dict['declarations'], + return_conversion=template_dict['return_conversion'], + initializers=template_dict['initializers'], + modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], + cleanup=template_dict['cleanup'], + lock=template_dict['lock'], + unlock=template_dict['unlock'], + ) + + # Only generate the "exit:" label + # if we have any gotos + label = "exit:" if "goto exit;" in template else "" + template = libclinic.linear_format(template, exit_label=label) + + s = template.format_map(template_dict) + + # mild hack: + # reflow long impl declarations + if name in {"impl_prototype", "impl_definition"}: + s = libclinic.wrap_declarations(s) + + if clinic.line_prefix: + s = libclinic.indent_all_lines(s, clinic.line_prefix) + if clinic.line_suffix: + s = libclinic.suffix_all_lines(s, clinic.line_suffix) + + destination.append(s) + + return clinic.get_destination('block').dump() diff --git a/Tools/clinic/libclinic/cli.py b/Tools/clinic/libclinic/cli.py new file mode 100644 index 00000000000000..f36c6d04efd383 --- /dev/null +++ b/Tools/clinic/libclinic/cli.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import argparse +import inspect +import os +import re +import sys +from collections.abc import Callable +from typing import NoReturn + + +# Local imports. +import libclinic +import libclinic.cpp +from libclinic import ClinicError +from libclinic.language import Language, PythonLanguage +from libclinic.block_parser import BlockParser +from libclinic.converter import ( + ConverterType, converters, legacy_converters) +from libclinic.return_converters import ( + return_converters, ReturnConverterType) +from libclinic.clanguage import CLanguage +from libclinic.app import Clinic + + +# TODO: +# +# soon: +# +# * allow mixing any two of {positional-only, positional-or-keyword, +# keyword-only} +# * dict constructor uses positional-only and keyword-only +# * max and min use positional only with an optional group +# and keyword-only +# + + +# Match '#define Py_LIMITED_API'. +# Match '# define Py_LIMITED_API 0x030d0000' (without the version). +LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') + + +# "extensions" maps the file extension ("c", "py") to Language classes. +LangDict = dict[str, Callable[[str], Language]] +extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } +extensions['py'] = PythonLanguage + + +def parse_file( + filename: str, + *, + limited_capi: bool, + output: str | None = None, + verify: bool = True, +) -> None: + if not output: + output = filename + + extension = os.path.splitext(filename)[1][1:] + if not extension: + raise ClinicError(f"Can't extract file type for file {filename!r}") + + try: + language = extensions[extension](filename) + except KeyError: + raise ClinicError(f"Can't identify file type for file {filename!r}") + + with open(filename, encoding="utf-8") as f: + raw = f.read() + + # exit quickly if there are no clinic markers in the file + find_start_re = BlockParser("", language).find_start_re + if not find_start_re.search(raw): + return + + if LIMITED_CAPI_REGEX.search(raw): + limited_capi = True + + assert isinstance(language, CLanguage) + clinic = Clinic(language, + verify=verify, + filename=filename, + limited_capi=limited_capi) + cooked = clinic.parse(raw) + + libclinic.write_file(output, cooked) + + +def create_cli() -> argparse.ArgumentParser: + cmdline = argparse.ArgumentParser( + prog="clinic.py", + description="""Preprocessor for CPython C files. + +The purpose of the Argument Clinic is automating all the boilerplate involved +with writing argument parsing code for builtins and providing introspection +signatures ("docstrings") for CPython builtins. + +For more information see https://devguide.python.org/development-tools/clinic/""") + cmdline.add_argument("-f", "--force", action='store_true', + help="force output regeneration") + cmdline.add_argument("-o", "--output", type=str, + help="redirect file output to OUTPUT") + cmdline.add_argument("-v", "--verbose", action='store_true', + help="enable verbose mode") + cmdline.add_argument("--converters", action='store_true', + help=("print a list of all supported converters " + "and return converters")) + cmdline.add_argument("--make", action='store_true', + help="walk --srcdir to run over all relevant files") + cmdline.add_argument("--srcdir", type=str, default=os.curdir, + help="the directory tree to walk in --make mode") + cmdline.add_argument("--exclude", type=str, action="append", + help=("a file to exclude in --make mode; " + "can be given multiple times")) + cmdline.add_argument("--limited", dest="limited_capi", action='store_true', + help="use the Limited C API") + cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", + help="the list of files to process") + return cmdline + + +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: + if ns.converters: + if ns.filename: + parser.error( + "can't specify --converters and a filename at the same time" + ) + AnyConverterType = ConverterType | ReturnConverterType + converter_list: list[tuple[str, AnyConverterType]] = [] + return_converter_list: list[tuple[str, AnyConverterType]] = [] + + for name, converter in converters.items(): + converter_list.append(( + name, + converter, + )) + for name, return_converter in return_converters.items(): + return_converter_list.append(( + name, + return_converter + )) + + print() + + print("Legacy converters:") + legacy = sorted(legacy_converters) + print(' ' + ' '.join(c for c in legacy if c[0].isupper())) + print(' ' + ' '.join(c for c in legacy if c[0].islower())) + print() + + for title, attribute, ids in ( + ("Converters", 'converter_init', converter_list), + ("Return converters", 'return_converter_init', return_converter_list), + ): + print(title + ":") + + ids.sort(key=lambda item: item[0].lower()) + longest = -1 + for name, _ in ids: + longest = max(longest, len(name)) + + for name, cls in ids: + callable = getattr(cls, attribute, None) + if not callable: + continue + signature = inspect.signature(callable) + parameters = [] + for parameter_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.KEYWORD_ONLY: + if parameter.default != inspect.Parameter.empty: + s = f'{parameter_name}={parameter.default!r}' + else: + s = parameter_name + parameters.append(s) + print(' {}({})'.format(name, ', '.join(parameters))) + print() + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") + return + + if ns.make: + if ns.output or ns.filename: + parser.error("can't use -o or filenames with --make") + if not ns.srcdir: + parser.error("--srcdir must not be empty with --make") + if ns.exclude: + excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] + excludes = [os.path.normpath(f) for f in excludes] + else: + excludes = [] + for root, dirs, files in os.walk(ns.srcdir): + for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): + if rcs_dir in dirs: + dirs.remove(rcs_dir) + for filename in files: + # handle .c, .cpp and .h files + if not filename.endswith(('.c', '.cpp', '.h')): + continue + path = os.path.join(root, filename) + path = os.path.normpath(path) + if path in excludes: + continue + if ns.verbose: + print(path) + parse_file(path, + verify=not ns.force, limited_capi=ns.limited_capi) + return + + if not ns.filename: + parser.error("no input files") + + if ns.output and len(ns.filename) > 1: + parser.error("can't use -o with multiple filenames") + + for filename in ns.filename: + if ns.verbose: + print(filename) + parse_file(filename, output=ns.output, + verify=not ns.force, limited_capi=ns.limited_capi) + + +def main(argv: list[str] | None = None) -> NoReturn: + parser = create_cli() + args = parser.parse_args(argv) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py new file mode 100644 index 00000000000000..b2f1db6f8ef8da --- /dev/null +++ b/Tools/clinic/libclinic/codegen.py @@ -0,0 +1,302 @@ +from __future__ import annotations +import dataclasses as dc +import io +import os +from typing import Final, TYPE_CHECKING + +import libclinic +from libclinic import fail +from libclinic.language import Language +from libclinic.block_parser import Block +if TYPE_CHECKING: + from libclinic.app import Clinic + + +TemplateDict = dict[str, str] + + +class CRenderData: + def __init__(self) -> None: + + # The C statements to declare variables. + # Should be full lines with \n eol characters. + self.declarations: list[str] = [] + + # The C statements required to initialize the variables before the parse call. + # Should be full lines with \n eol characters. + self.initializers: list[str] = [] + + # The C statements needed to dynamically modify the values + # parsed by the parse call, before calling the impl. + self.modifications: list[str] = [] + + # The entries for the "keywords" array for PyArg_ParseTuple. + # Should be individual strings representing the names. + self.keywords: list[str] = [] + + # The "format units" for PyArg_ParseTuple. + # Should be individual strings that will get + self.format_units: list[str] = [] + + # The varargs arguments for PyArg_ParseTuple. + self.parse_arguments: list[str] = [] + + # The parameter declarations for the impl function. + self.impl_parameters: list[str] = [] + + # The arguments to the impl function at the time it's called. + self.impl_arguments: list[str] = [] + + # For return converters: the name of the variable that + # should receive the value returned by the impl. + self.return_value = "return_value" + + # For return converters: the code to convert the return + # value from the parse function. This is also where + # you should check the _return_value for errors, and + # "goto exit" if there are any. + self.return_conversion: list[str] = [] + self.converter_retval = "_return_value" + + # The C statements required to do some operations + # after the end of parsing but before cleaning up. + # These operations may be, for example, memory deallocations which + # can only be done without any error happening during argument parsing. + self.post_parsing: list[str] = [] + + # The C statements required to clean up after the impl call. + self.cleanup: list[str] = [] + + # The C statements to generate critical sections (per-object locking). + self.lock: list[str] = [] + self.unlock: list[str] = [] + + +@dc.dataclass(slots=True, frozen=True) +class Include: + """ + An include like: #include "pycore_long.h" // _Py_ID() + """ + # Example: "pycore_long.h". + filename: str + + # Example: "_Py_ID()". + reason: str + + # None means unconditional include. + # Example: "#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)". + condition: str | None + + def sort_key(self) -> tuple[str, str]: + # order: '#if' comes before 'NO_CONDITION' + return (self.condition or 'NO_CONDITION', self.filename) + + +@dc.dataclass(slots=True) +class BlockPrinter: + language: Language + f: io.StringIO = dc.field(default_factory=io.StringIO) + + # '#include "header.h" // reason': column of '//' comment + INCLUDE_COMMENT_COLUMN: Final[int] = 35 + + def print_block( + self, + block: Block, + *, + header_includes: list[Include] | None = None, + ) -> None: + input = block.input + output = block.output + dsl_name = block.dsl_name + write = self.f.write + + assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) + + if not dsl_name: + write(input) + return + + write(self.language.start_line.format(dsl_name=dsl_name)) + write("\n") + + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + if not body_prefix: + write(input) + else: + for line in input.split('\n'): + write(body_prefix) + write(line) + write("\n") + + write(self.language.stop_line.format(dsl_name=dsl_name)) + write("\n") + + output = '' + if header_includes: + # Emit optional "#include" directives for C headers + output += '\n' + + current_condition: str | None = None + for include in header_includes: + if include.condition != current_condition: + if current_condition: + output += '#endif\n' + current_condition = include.condition + if include.condition: + output += f'{include.condition}\n' + + if current_condition: + line = f'# include "{include.filename}"' + else: + line = f'#include "{include.filename}"' + if include.reason: + comment = f'// {include.reason}\n' + line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment + output += line + + if current_condition: + output += '#endif\n' + + input = ''.join(block.input) + output += ''.join(block.output) + if output: + if not output.endswith('\n'): + output += '\n' + write(output) + + arguments = "output={output} input={input}".format( + output=libclinic.compute_checksum(output, 16), + input=libclinic.compute_checksum(input, 16) + ) + write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) + write("\n") + + def write(self, text: str) -> None: + self.f.write(text) + + +class BufferSeries: + """ + Behaves like a "defaultlist". + When you ask for an index that doesn't exist yet, + the object grows the list until that item exists. + So o[n] will always work. + + Supports negative indices for actual items. + e.g. o[-1] is an element immediately preceding o[0]. + """ + + def __init__(self) -> None: + self._start = 0 + self._array: list[list[str]] = [] + + def __getitem__(self, i: int) -> list[str]: + i -= self._start + if i < 0: + self._start += i + prefix: list[list[str]] = [[] for x in range(-i)] + self._array = prefix + self._array + i = 0 + while i >= len(self._array): + self._array.append([]) + return self._array[i] + + def clear(self) -> None: + for ta in self._array: + ta.clear() + + def dump(self) -> str: + texts = ["".join(ta) for ta in self._array] + self.clear() + return "".join(texts) + + +@dc.dataclass(slots=True, repr=False) +class Destination: + name: str + type: str + clinic: Clinic + buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) + filename: str = dc.field(init=False) # set in __post_init__ + + args: dc.InitVar[tuple[str, ...]] = () + + def __post_init__(self, args: tuple[str, ...]) -> None: + valid_types = ('buffer', 'file', 'suppress') + if self.type not in valid_types: + fail( + f"Invalid destination type {self.type!r} for {self.name}, " + f"must be {', '.join(valid_types)}" + ) + extra_arguments = 1 if self.type == "file" else 0 + if len(args) < extra_arguments: + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") + if len(args) > extra_arguments: + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") + if self.type =='file': + d = {} + filename = self.clinic.filename + d['path'] = filename + dirname, basename = os.path.split(filename) + if not dirname: + dirname = '.' + d['dirname'] = dirname + d['basename'] = basename + d['basename_root'], d['basename_extension'] = os.path.splitext(filename) + self.filename = args[0].format_map(d) + + def __repr__(self) -> str: + if self.type == 'file': + type_repr = f"type='file' file={self.filename!r}" + else: + type_repr = f"type={self.type!r}" + return f"" + + def clear(self) -> None: + if self.type != 'buffer': + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") + self.buffers.clear() + + def dump(self) -> str: + return self.buffers.dump() + + +DestinationDict = dict[str, Destination] + + +class CodeGen: + def __init__(self, limited_capi: bool) -> None: + self.limited_capi = limited_capi + self._ifndef_symbols: set[str] = set() + # dict: include name => Include instance + self._includes: dict[str, Include] = {} + + def add_ifndef_symbol(self, name: str) -> bool: + if name in self._ifndef_symbols: + return False + self._ifndef_symbols.add(name) + return True + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + try: + existing = self._includes[name] + except KeyError: + pass + else: + if existing.condition and not condition: + # If the previous include has a condition and the new one is + # unconditional, override the include. + pass + else: + # Already included, do nothing. Only mention a single reason, + # no need to list all of them. + return + + self._includes[name] = Include(name, reason, condition) + + def get_includes(self) -> list[Include]: + return sorted(self._includes.values(), + key=Include.sort_key) diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py new file mode 100644 index 00000000000000..86853bb4fba253 --- /dev/null +++ b/Tools/clinic/libclinic/converter.py @@ -0,0 +1,552 @@ +from __future__ import annotations +import builtins as bltns +import functools +from typing import Any, TypeVar, Literal, TYPE_CHECKING, cast +from collections.abc import Callable + +import libclinic +from libclinic import fail +from libclinic import Sentinels, unspecified, unknown +from libclinic.codegen import CRenderData, Include, TemplateDict +from libclinic.function import Function, Parameter + + +CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) + + +type_checks = { + '&PyLong_Type': ('PyLong_Check', 'int'), + '&PyTuple_Type': ('PyTuple_Check', 'tuple'), + '&PyList_Type': ('PyList_Check', 'list'), + '&PySet_Type': ('PySet_Check', 'set'), + '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), + '&PyDict_Type': ('PyDict_Check', 'dict'), + '&PyUnicode_Type': ('PyUnicode_Check', 'str'), + '&PyBytes_Type': ('PyBytes_Check', 'bytes'), + '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), +} + + +def add_c_converter( + f: CConverterClassT, + name: str | None = None +) -> CConverterClassT: + if not name: + name = f.__name__ + if not name.endswith('_converter'): + return f + name = name.removesuffix('_converter') + converters[name] = f + return f + + +def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT: + # automatically add converter for default format unit + # (but without stomping on the existing one if it's already + # set, in case you subclass) + if ((cls.format_unit not in ('O&', '')) and + (cls.format_unit not in legacy_converters)): + legacy_converters[cls.format_unit] = cls + return cls + + +class CConverterAutoRegister(type): + def __init__( + cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] + ) -> None: + converter_cls = cast(type["CConverter"], cls) + add_c_converter(converter_cls) + add_default_legacy_c_converter(converter_cls) + +class CConverter(metaclass=CConverterAutoRegister): + """ + For the init function, self, name, function, and default + must be keyword-or-positional parameters. All other + parameters must be keyword-only. + """ + + # The C name to use for this variable. + name: str + + # The Python name to use for this variable. + py_name: str + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type: str | None = None + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + # Or the magic value "unknown" if this value is a cannot be evaluated + # at Argument-Clinic-preprocessing time (but is presumed to be valid + # at runtime). + default: object = unspecified + + # If not None, default must be isinstance() of this type. + # (You can also specify a tuple of types.) + default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None + + # "default" converted into a C value, as a string. + # Or None if there is no default. + c_default: str | None = None + + # "default" converted into a Python value, as a string. + # Or None if there is no default. + py_default: str | None = None + + # The default value used to initialize the C variable when + # there is no default, but not specifying a default may + # result in an "uninitialized variable" warning. This can + # easily happen when using option groups--although + # properly-written code won't actually use the variable, + # the variable does get passed in to the _impl. (Ah, if + # only dataflow analysis could inline the static function!) + # + # This value is specified as a string. + # Every non-abstract subclass should supply a valid value. + c_ignored_default: str = 'NULL' + + # If true, wrap with Py_UNUSED. + unused = False + + # The C converter *function* to be used, if any. + # (If this is not None, format_unit must be 'O&'.) + converter: str | None = None + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into the _impl function? + impl_by_reference = False + + # Should Argument Clinic add a '&' before the name of + # the variable when passing it into PyArg_ParseTuple (AndKeywords)? + parse_by_reference = True + + ############################################################# + ############################################################# + ## You shouldn't need to read anything below this point to ## + ## write your own converter functions. ## + ############################################################# + ############################################################# + + # The "format unit" to specify for this variable when + # parsing arguments using PyArg_ParseTuple (AndKeywords). + # Custom converters should always use the default value of 'O&'. + format_unit = 'O&' + + # What encoding do we want for this variable? Only used + # by format units starting with 'e'. + encoding: str | None = None + + # Should this object be required to be a subclass of a specific type? + # If not None, should be a string representing a pointer to a + # PyTypeObject (e.g. "&PyUnicode_Type"). + # Only used by the 'O!' format unit (and the "object" converter). + subclass_of: str | None = None + + # See also the 'length_name' property. + # Only used by format units ending with '#'. + length = False + + # Should we show this parameter in the generated + # __text_signature__? This is *almost* always True. + # (It's only False for __new__, __init__, and METH_STATIC functions.) + show_in_signature = True + + # Overrides the name used in a text signature. + # The name used for a "self" parameter must be one of + # self, type, or module; however users can set their own. + # This lets the self_converter overrule the user-settable + # name, *just* for the text signature. + # Only set by self_converter. + signature_name: str | None = None + + broken_limited_capi: bool = False + + # keep in sync with self_converter.__init__! + def __init__(self, + # Positional args: + name: str, + py_name: str, + function: Function, + default: object = unspecified, + *, # Keyword only args: + c_default: str | None = None, + py_default: str | None = None, + annotation: str | Literal[Sentinels.unspecified] = unspecified, + unused: bool = False, + **kwargs: Any + ) -> None: + self.name = libclinic.ensure_legal_c_identifier(name) + self.py_name = py_name + self.unused = unused + self._includes: list[Include] = [] + + if default is not unspecified: + if (self.default_type + and default is not unknown + and not isinstance(default, self.default_type) + ): + if isinstance(self.default_type, type): + types_str = self.default_type.__name__ + else: + names = [cls.__name__ for cls in self.default_type] + types_str = ', '.join(names) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") + self.default = default + + if c_default: + self.c_default = c_default + if py_default: + self.py_default = py_default + + if annotation is not unspecified: + fail("The 'annotation' parameter is not currently permitted.") + + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) + self.converter_init(**kwargs) + self.function = function + + # Add a custom __getattr__ method to improve the error message + # if somebody tries to access self.function in converter_init(). + # + # mypy will assume arbitrary access is okay for a class with a __getattr__ method, + # and that's not what we want, + # so put it inside an `if not TYPE_CHECKING` block + if not TYPE_CHECKING: + def __getattr__(self, attr): + if attr == "function": + fail( + f"{self.__class__.__name__!r} object has no attribute 'function'.\n" + f"Note: accessing self.function inside converter_init is disallowed!" + ) + return super().__getattr__(attr) + # this branch is just here for coverage reporting + else: # pragma: no cover + pass + + def converter_init(self) -> None: + pass + + def is_optional(self) -> bool: + return (self.default is not unspecified) + + def _render_self(self, parameter: Parameter, data: CRenderData) -> None: + self.parameter = parameter + name = self.parser_name + + # impl_arguments + s = ("&" if self.impl_by_reference else "") + name + data.impl_arguments.append(s) + if self.length: + data.impl_arguments.append(self.length_name) + + # impl_parameters + data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) + if self.length: + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") + + def _render_non_self( + self, + parameter: Parameter, + data: CRenderData + ) -> None: + self.parameter = parameter + name = self.name + + # declarations + d = self.declaration(in_parser=True) + data.declarations.append(d) + + # initializers + initializers = self.initialize() + if initializers: + data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) + + # modifications + modifications = self.modify() + if modifications: + data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) + + # keywords + if parameter.is_vararg(): + pass + elif parameter.is_positional_only(): + data.keywords.append('') + else: + data.keywords.append(parameter.name) + + # format_units + if self.is_optional() and '|' not in data.format_units: + data.format_units.append('|') + if parameter.is_keyword_only() and '$' not in data.format_units: + data.format_units.append('$') + data.format_units.append(self.format_unit) + + # parse_arguments + self.parse_argument(data.parse_arguments) + + # post_parsing + if post_parsing := self.post_parsing(): + data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') + + # cleanup + cleanup = self.cleanup() + if cleanup: + data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + self._render_self(parameter, data) + self._render_non_self(parameter, data) + + @functools.cached_property + def length_name(self) -> str: + """Computes the name of the associated "length" variable.""" + assert self.length is not None + return self.parser_name + "_length" + + # Why is this one broken out separately? + # For "positional-only" function parsing, + # which generates a bunch of PyArg_ParseTuple calls. + def parse_argument(self, args: list[str]) -> None: + assert not (self.converter and self.encoding) + if self.format_unit == 'O&': + assert self.converter + args.append(self.converter) + + if self.encoding: + args.append(libclinic.c_repr(self.encoding)) + elif self.subclass_of: + args.append(self.subclass_of) + + s = ("&" if self.parse_by_reference else "") + self.parser_name + args.append(s) + + if self.length: + args.append(f"&{self.length_name}") + + # + # All the functions after here are intended as extension points. + # + + def simple_declaration( + self, + by_reference: bool = False, + *, + in_parser: bool = False + ) -> str: + """ + Computes the basic declaration of the variable. + Used in computing the prototype declaration and the + variable declaration. + """ + assert isinstance(self.type, str) + prototype = [self.type] + if by_reference or not self.type.endswith('*'): + prototype.append(" ") + if by_reference: + prototype.append('*') + if in_parser: + name = self.parser_name + else: + name = self.name + if self.unused: + name = f"Py_UNUSED({name})" + prototype.append(name) + return "".join(prototype) + + def declaration(self, *, in_parser: bool = False) -> str: + """ + The C statement to declare this variable. + """ + declaration = [self.simple_declaration(in_parser=True)] + default = self.c_default + if not default and self.parameter.group: + default = self.c_ignored_default + if default: + declaration.append(" = ") + declaration.append(default) + declaration.append(";") + if self.length: + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") + return "".join(declaration) + + def initialize(self) -> str: + """ + The C statements required to set up this variable before parsing. + Returns a string containing this code indented at column 0. + If no initialization is necessary, returns an empty string. + """ + return "" + + def modify(self) -> str: + """ + The C statements required to modify this variable after parsing. + Returns a string containing this code indented at column 0. + If no modification is necessary, returns an empty string. + """ + return "" + + def post_parsing(self) -> str: + """ + The C statements required to do some operations after the end of parsing but before cleaning up. + Return a string containing this code indented at column 0. + If no operation is necessary, return an empty string. + """ + return "" + + def cleanup(self) -> str: + """ + The C statements required to clean up after this variable. + Returns a string containing this code indented at column 0. + If no cleanup is necessary, returns an empty string. + """ + return "" + + def pre_render(self) -> None: + """ + A second initialization function, like converter_init, + called just before rendering. + You are permitted to examine self.function here. + """ + pass + + def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str: + assert '"' not in expected + if limited_capi: + if expected_literal: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be {expected}, not %T", ' + f'{{argname}});') + else: + return (f'PyErr_Format(PyExc_TypeError, ' + f'"{{{{name}}}}() {displayname} must be %s, not %T", ' + f'"{expected}", {{argname}});') + else: + if expected_literal: + expected = f'"{expected}"' + self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()') + return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});' + + def format_code(self, fmt: str, *, + argname: str, + bad_argument: str | None = None, + bad_argument2: str | None = None, + **kwargs: Any) -> str: + if '{bad_argument}' in fmt: + if not bad_argument: + raise TypeError("required 'bad_argument' argument") + fmt = fmt.replace('{bad_argument}', bad_argument) + if '{bad_argument2}' in fmt: + if not bad_argument2: + raise TypeError("required 'bad_argument2' argument") + fmt = fmt.replace('{bad_argument2}', bad_argument2) + return fmt.format(argname=argname, paramname=self.parser_name, **kwargs) + + def use_converter(self) -> None: + """Method called when self.converter is used to parse an argument.""" + pass + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'O&': + self.use_converter() + return self.format_code(""" + if (!{converter}({argname}, &{paramname})) {{{{ + goto exit; + }}}} + """, + argname=argname, + converter=self.converter) + if self.format_unit == 'O!': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + if self.subclass_of in type_checks: + typecheck, typename = type_checks[self.subclass_of] + return self.format_code(""" + if (!{typecheck}({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi), + typecheck=typecheck, typename=typename, cast=cast) + return self.format_code(""" + if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {cast}{argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name', + expected_literal=False, limited_capi=limited_capi), + subclass_of=self.subclass_of, cast=cast) + if self.format_unit == 'O': + cast = '(%s)' % self.type if self.type != 'PyObject *' else '' + return self.format_code(""" + {paramname} = {cast}{argname}; + """, + argname=argname, cast=cast) + return None + + def set_template_dict(self, template_dict: TemplateDict) -> None: + pass + + @property + def parser_name(self) -> str: + if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 + return libclinic.CLINIC_PREFIX + self.name + else: + return self.name + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + include = Include(name, reason, condition) + self._includes.append(include) + + def get_includes(self) -> list[Include]: + return self._includes + + +ConverterType = Callable[..., CConverter] +ConverterDict = dict[str, ConverterType] + +# maps strings to callables. +# these callables must be of the form: +# def foo(name, default, *, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CConverter object. +# The callable should not call builtins.print. +converters: ConverterDict = {} + +# maps strings to callables. +# these callables follow the same rules as those for "converters" above. +# note however that they will never be called with keyword-only parameters. +legacy_converters: ConverterDict = {} + + +def add_legacy_c_converter( + format_unit: str, + **kwargs: Any +) -> Callable[[CConverterClassT], CConverterClassT]: + def closure(f: CConverterClassT) -> CConverterClassT: + added_f: Callable[..., CConverter] + if not kwargs: + added_f = f + else: + added_f = functools.partial(f, **kwargs) + if format_unit: + legacy_converters[format_unit] = added_f + return f + return closure diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py new file mode 100644 index 00000000000000..bd5c2a2b73b94a --- /dev/null +++ b/Tools/clinic/libclinic/converters.py @@ -0,0 +1,1230 @@ +import builtins as bltns +import functools +import sys +from types import NoneType +from typing import Any + +from libclinic import fail, Null, unspecified, unknown +from libclinic.function import ( + Function, Parameter, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.codegen import CRenderData, TemplateDict +from libclinic.converter import ( + CConverter, legacy_converters, add_legacy_c_converter) + + +TypeSet = set[bltns.type[object]] + + +class bool_converter(CConverter): + type = 'int' + default_type = bool + format_unit = 'p' + c_ignored_default = '0' + + def converter_init(self, *, accept: TypeSet = {object}) -> None: + if accept == {int}: + self.format_unit = 'i' + elif accept != {object}: + fail(f"bool_converter: illegal 'accept' argument {accept!r}") + if self.default is not unspecified and self.default is not unknown: + self.default = bool(self.default) + self.c_default = str(int(self.default)) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'p': + return self.format_code(""" + {paramname} = PyObject_IsTrue({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class defining_class_converter(CConverter): + """ + A special-case converter: + this is the default converter used for the defining class. + """ + type = 'PyTypeObject *' + format_unit = '' + show_in_signature = False + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + self._render_self(parameter, data) + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['defining_class_name'] = self.name + + +class char_converter(CConverter): + type = 'char' + default_type = (bytes, bytearray) + format_unit = 'c' + c_ignored_default = "'\0'" + + def converter_init(self) -> None: + if isinstance(self.default, self.default_type): + if len(self.default) != 1: + fail(f"char_converter: illegal default value {self.default!r}") + + self.c_default = repr(bytes(self.default))[1:] + if self.c_default == '"\'"': + self.c_default = r"'\''" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'c': + return self.format_code(""" + if (PyBytes_Check({argname})) {{{{ + if (PyBytes_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE({argname})); + goto exit; + }}}} + {paramname} = PyBytes_AS_STRING({argname})[0]; + }}}} + else if (PyByteArray_Check({argname})) {{{{ + if (PyByteArray_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE({argname})); + goto exit; + }}}} + {paramname} = PyByteArray_AS_STRING({argname})[0]; + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + displayname=displayname, + bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('B', bitwise=True) +class unsigned_char_converter(CConverter): + type = 'unsigned char' + default_type = int + format_unit = 'b' + c_ignored_default = "'\0'" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'B' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'b': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < 0) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + }}}} + else if (ival > UCHAR_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + elif self.format_unit == 'B': + return self.format_code(""" + {{{{ + unsigned long ival = PyLong_AsUnsignedLongMask({argname}); + if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class byte_converter(unsigned_char_converter): + pass + + +class short_converter(CConverter): + type = 'short' + default_type = int + format_unit = 'h' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'h': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < SHRT_MIN) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + }}}} + else if (ival > SHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (short) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_short_converter(CConverter): + type = 'unsigned short' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'H' + else: + self.converter = '_PyLong_UnsignedShort_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedShort_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedShort_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'H': + return self.format_code(""" + {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > USHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned short"); + goto exit; + }}}} + {paramname} = (unsigned short) uval; + }}}} + """, + argname=argname) + + +@add_legacy_c_converter('C', accept={str}) +class int_converter(CConverter): + type = 'int' + default_type = int + format_unit = 'i' + c_ignored_default = "0" + + def converter_init( + self, *, accept: TypeSet = {int}, type: str | None = None + ) -> None: + if accept == {str}: + self.format_unit = 'C' + elif accept != {int}: + fail(f"int_converter: illegal 'accept' argument {accept!r}") + if type is not None: + self.type = type + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'C': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH({argname})); + goto exit; + }}}} + {paramname} = PyUnicode_READ_CHAR({argname}, 0); + """, + argname=argname, + displayname=displayname, + bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_int_converter(CConverter): + type = 'unsigned int' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'I' + else: + self.converter = '_PyLong_UnsignedInt_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedInt_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedInt_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'I': + return self.format_code(""" + {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > UINT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned int"); + goto exit; + }}}} + {paramname} = (unsigned int) uval; + }}}} + """, + argname=argname) + + +class long_converter(CConverter): + type = 'long' + default_type = int + format_unit = 'l' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'l': + return self.format_code(""" + {paramname} = PyLong_AsLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_converter(CConverter): + type = 'unsigned long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'k' + else: + self.converter = '_PyLong_UnsignedLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'k': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLong({argname}); + if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class long_long_converter(CConverter): + type = 'long long' + default_type = int + format_unit = 'L' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'L': + return self.format_code(""" + {paramname} = PyLong_AsLongLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_long_converter(CConverter): + type = 'unsigned long long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'K' + else: + self.converter = '_PyLong_UnsignedLongLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLongLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLongLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'K': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLongLong({argname}); + if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class Py_ssize_t_converter(CConverter): + type = 'Py_ssize_t' + c_ignored_default = "0" + + def converter_init(self, *, accept: TypeSet = {int}) -> None: + if accept == {int}: + self.format_unit = 'n' + self.default_type = int + elif accept == {int, NoneType}: + self.converter = '_Py_convert_optional_to_ssize_t' + else: + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") + + def use_converter(self) -> None: + if self.converter == '_Py_convert_optional_to_ssize_t': + self.add_include('pycore_abstract.h', + '_Py_convert_optional_to_ssize_t()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + if limited_capi: + PyNumber_Index = 'PyNumber_Index' + else: + PyNumber_Index = '_PyNumber_Index' + self.add_include('pycore_abstract.h', '_PyNumber_Index()') + return self.format_code(""" + {{{{ + Py_ssize_t ival = -1; + PyObject *iobj = {PyNumber_Index}({argname}); + if (iobj != NULL) {{{{ + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + }}}} + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + {paramname} = ival; + }}}} + """, + argname=argname, + PyNumber_Index=PyNumber_Index) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + if ({argname} != Py_None) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), + ) + + +class slice_index_converter(CConverter): + type = 'Py_ssize_t' + + def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + if accept == {int}: + self.converter = '_PyEval_SliceIndexNotNone' + self.nullable = False + elif accept == {int, NoneType}: + self.converter = '_PyEval_SliceIndex' + self.nullable = True + else: + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + if self.nullable: + return self.format_code(""" + if (!Py_IsNone({argname})) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + return 0; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "None or have an __index__ method"); + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + goto exit; + }}}} + """, + argname=argname) + + +class size_t_converter(CConverter): + type = 'size_t' + converter = '_PyLong_Size_t_Converter' + c_ignored_default = "0" + + def use_converter(self) -> None: + self.add_include('pycore_long.h', + '_PyLong_Size_t_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + return self.format_code(""" + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsSize_t({argname}); + if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class fildes_converter(CConverter): + type = 'int' + converter = '_PyLong_FileDescriptor_Converter' + + def use_converter(self) -> None: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + return self.format_code(""" + {paramname} = PyObject_AsFileDescriptor({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class float_converter(CConverter): + type = 'float' + default_type = float + format_unit = 'f' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'f': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); + }}}} + else + {{{{ + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class double_converter(CConverter): + type = 'double' + default_type = float + format_unit = 'd' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'd': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = PyFloat_AS_DOUBLE({argname}); + }}}} + else + {{{{ + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class Py_complex_converter(CConverter): + type = 'Py_complex' + default_type = complex + format_unit = 'D' + c_ignored_default = "{0.0, 0.0}" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'D': + return self.format_code(""" + {paramname} = PyComplex_AsCComplex({argname}); + if (PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class object_converter(CConverter): + type = 'PyObject *' + format_unit = 'O' + + def converter_init( + self, *, + converter: str | None = None, + type: str | None = None, + subclass_of: str | None = None + ) -> None: + if converter: + if subclass_of: + fail("object: Cannot pass in both 'converter' and 'subclass_of'") + self.format_unit = 'O&' + self.converter = converter + elif subclass_of: + self.format_unit = 'O!' + self.subclass_of = subclass_of + + if type is not None: + self.type = type + + +# +# We define three conventions for buffer types in the 'accept' argument: +# +# buffer : any object supporting the buffer interface +# rwbuffer: any object supporting the buffer interface, but must be writeable +# robuffer: any object supporting the buffer interface, but must not be writeable +# + +class buffer: + pass +class rwbuffer: + pass +class robuffer: + pass + + +StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] + +def str_converter_key( + types: TypeSet, encoding: bool | str | None, zeroes: bool +) -> StrConverterKeyType: + return (frozenset(types), bool(encoding), bool(zeroes)) + +str_converter_argument_map: dict[StrConverterKeyType, str] = {} + + +class str_converter(CConverter): + type = 'const char *' + default_type = (str, Null, NoneType) + format_unit = 's' + + def converter_init( + self, + *, + accept: TypeSet = {str}, + encoding: str | None = None, + zeroes: bool = False + ) -> None: + + key = str_converter_key(accept, encoding, zeroes) + format_unit = str_converter_argument_map.get(key) + if not format_unit: + fail("str_converter: illegal combination of arguments", key) + + self.format_unit = format_unit + self.length = bool(zeroes) + if encoding: + if self.default not in (Null, None, unspecified): + fail("str_converter: Argument Clinic doesn't support default values for encoded strings") + self.encoding = encoding + self.type = 'char *' + # sorry, clinic can't support preallocated buffers + # for es# and et# + self.c_default = "NULL" + if NoneType in accept and self.c_default == "Py_None": + self.c_default = "NULL" + + def post_parsing(self) -> str: + if self.encoding: + name = self.name + return f"PyMem_FREE({name});\n" + else: + return "" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 's': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + length_name=self.length_name) + if self.format_unit == 'z': + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + length_name=self.length_name) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + +# +# This is the fourth or fifth rewrite of registering all the +# string converter format units. Previous approaches hid +# bugs--generally mismatches between the semantics of the format +# unit and the arguments necessary to represent those semantics +# properly. Hopefully with this approach we'll get it 100% right. +# +# The r() function (short for "register") both registers the +# mapping from arguments to format unit *and* registers the +# legacy C converter for that format unit. +# +def r(format_unit: str, + *, + accept: TypeSet, + encoding: bool = False, + zeroes: bool = False +) -> None: + if not encoding and format_unit != 's': + # add the legacy c converters here too. + # + # note: add_legacy_c_converter can't work for + # es, es#, et, or et# + # because of their extra encoding argument + # + # also don't add the converter for 's' because + # the metaclass for CConverter adds it for us. + kwargs: dict[str, Any] = {} + if accept != {str}: + kwargs['accept'] = accept + if zeroes: + kwargs['zeroes'] = True + added_f = functools.partial(str_converter, **kwargs) + legacy_converters[format_unit] = added_f + + d = str_converter_argument_map + key = str_converter_key(accept, encoding, zeroes) + if key in d: + sys.exit("Duplicate keys specified for str_converter_argument_map!") + d[key] = format_unit + +r('es', encoding=True, accept={str}) +r('es#', encoding=True, zeroes=True, accept={str}) +r('et', encoding=True, accept={bytes, bytearray, str}) +r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) +r('s', accept={str}) +r('s#', zeroes=True, accept={robuffer, str}) +r('y', accept={robuffer}) +r('y#', zeroes=True, accept={robuffer}) +r('z', accept={str, NoneType}) +r('z#', zeroes=True, accept={robuffer, str, NoneType}) +del r + + +class PyBytesObject_converter(CConverter): + type = 'PyBytesObject *' + format_unit = 'S' + # accept = {bytes} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'S': + return self.format_code(""" + if (!PyBytes_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class PyByteArrayObject_converter(CConverter): + type = 'PyByteArrayObject *' + format_unit = 'Y' + # accept = {bytearray} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'Y': + return self.format_code(""" + if (!PyByteArray_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unicode_converter(CConverter): + type = 'PyObject *' + default_type = (str, Null, NoneType) + format_unit = 'U' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'U': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('u') +@add_legacy_c_converter('u#', zeroes=True) +@add_legacy_c_converter('Z', accept={str, NoneType}) +@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) +class Py_UNICODE_converter(CConverter): + type = 'const wchar_t *' + default_type = (str, Null, NoneType) + + def converter_init( + self, *, + accept: TypeSet = {str}, + zeroes: bool = False + ) -> None: + format_unit = 'Z' if accept=={str, NoneType} else 'u' + if zeroes: + format_unit += '#' + self.length = True + self.format_unit = format_unit + else: + self.accept = accept + if accept == {str}: + self.converter = '_PyUnicode_WideCharString_Converter' + elif accept == {str, NoneType}: + self.converter = '_PyUnicode_WideCharString_Opt_Converter' + else: + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") + self.c_default = "NULL" + + def cleanup(self) -> str: + if self.length: + return "" + else: + return f"""PyMem_Free((void *){self.parser_name});\n""" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not self.length: + if self.accept == {str}: + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + elif self.accept == {str, NoneType}: + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('s*', accept={str, buffer}) +@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) +@add_legacy_c_converter('w*', accept={rwbuffer}) +class Py_buffer_converter(CConverter): + type = 'Py_buffer' + format_unit = 'y*' + impl_by_reference = True + c_ignored_default = "{NULL, NULL}" + + def converter_init(self, *, accept: TypeSet = {buffer}) -> None: + if self.default not in (unspecified, None): + fail("The only legal default value for Py_buffer is None.") + + self.c_default = self.c_ignored_default + + if accept == {str, buffer, NoneType}: + format_unit = 'z*' + elif accept == {str, buffer}: + format_unit = 's*' + elif accept == {buffer}: + format_unit = 'y*' + elif accept == {rwbuffer}: + format_unit = 'w*' + else: + fail("Py_buffer_converter: illegal combination of arguments") + + self.format_unit = format_unit + + def cleanup(self) -> str: + name = self.name + return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. + if self.format_unit == 'y*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 's*': + return self.format_code(""" + if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t len; + const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); + if (ptr == NULL) {{{{ + goto exit; + }}}} + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} + }}}} + else {{{{ /* any bytes-like object */ + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 'w*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), + bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +def correct_name_for_self( + f: Function +) -> tuple[str, str]: + if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: + if f.cls: + return "PyObject *", "self" + return "PyObject *", "module" + if f.kind is STATIC_METHOD: + return "void *", "null" + if f.kind in (CLASS_METHOD, METHOD_NEW): + return "PyTypeObject *", "type" + raise AssertionError(f"Unhandled type of function f: {f.kind!r}") + + +class self_converter(CConverter): + """ + A special-case converter: + this is the default converter used for "self". + """ + type: str | None = None + format_unit = '' + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def pre_render(self) -> None: + f = self.function + default_type, default_name = correct_name_for_self(f) + self.signature_name = default_name + self.type = self.specified_type or self.type or default_type + + kind = self.function.kind + + if kind is STATIC_METHOD or kind.new_or_init: + self.show_in_signature = False + + # tp_new (METHOD_NEW) functions are of type newfunc: + # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); + # + # tp_init (METHOD_INIT) functions are of type initproc: + # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); + # + # All other functions generated by Argument Clinic are stored in + # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: + # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); + # However! We habitually cast these functions to PyCFunction, + # since functions that accept keyword arguments don't fit this signature + # but are stored there anyway. So strict type equality isn't important + # for these functions. + # + # So: + # + # * The name of the first parameter to the impl and the parsing function will always + # be self.name. + # + # * The type of the first parameter to the impl will always be of self.type. + # + # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is also self.type. + # This means that if you step into the parsing function, your "self" parameter + # is of the correct type, which may make debugging more pleasant. + # + # * Else if the function is tp_new (METHOD_NEW): + # * The type of the first parameter to the parsing function is "PyTypeObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyTypeObject *", we cast the first parameter to self.type + # in the impl call. + # + # * Else if the function is tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is "PyObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyObject *", we cast the first parameter to self.type + # in the impl call. + + @property + def parser_type(self) -> str: + assert self.type is not None + if self.function.kind in {METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD}: + tp, _ = correct_name_for_self(self.function) + return tp + return self.type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + if self.function.kind is STATIC_METHOD: + return + + self._render_self(parameter, data) + + if self.type != self.parser_type: + # insert cast to impl_argument[0], aka self. + # we know we're in the first slot in all the CRenderData lists, + # because we render parameters in order, and self is always first. + assert len(data.impl_arguments) == 1 + assert data.impl_arguments[0] == self.name + assert self.type is not None + data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['self_name'] = self.name + template_dict['self_type'] = self.parser_type + kind = self.function.kind + cls = self.function.cls + + if kind.new_or_init and cls and cls.typedef: + if kind is METHOD_NEW: + type_check = ( + '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' + ).format(self.name) + else: + type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' + ' Py_TYPE({0})->tp_new == base_tp->tp_new)' + ).format(self.name) + + line = f'{type_check} &&\n ' + template_dict['self_type_check'] = line + + type_object = cls.type_object + type_ptr = f'PyTypeObject *base_tp = {type_object};' + template_dict['base_type_ptr'] = type_ptr diff --git a/Tools/clinic/cpp.py b/Tools/clinic/libclinic/cpp.py similarity index 90% rename from Tools/clinic/cpp.py rename to Tools/clinic/libclinic/cpp.py index 16eee6fc399491..e115d65a88e1b6 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/libclinic/cpp.py @@ -3,6 +3,11 @@ import sys from typing import NoReturn +from .errors import ParseError + + +__all__ = ["Monitor"] + TokenAndCondition = tuple[str, str] TokenStack = list[TokenAndCondition] @@ -32,7 +37,7 @@ class Monitor: Anyway this implementation seems to work well enough for the CPython sources. """ - filename: str | None = None + filename: str _: dc.KW_ONLY verbose: bool = False @@ -59,14 +64,8 @@ def condition(self) -> str: """ return " && ".join(condition for token, condition in self.stack) - def fail(self, *a: object) -> NoReturn: - if self.filename: - filename = " " + self.filename - else: - filename = '' - print("Error at" + filename, "line", self.line_number, ":") - print(" ", ' '.join(str(x) for x in a)) - sys.exit(-1) + def fail(self, msg: str) -> NoReturn: + raise ParseError(msg, filename=self.filename, lineno=self.line_number) def writeline(self, line: str) -> None: self.line_number += 1 @@ -74,7 +73,7 @@ def writeline(self, line: str) -> None: def pop_stack() -> TokenAndCondition: if not self.stack: - self.fail("#" + token + " without matching #if / #ifdef / #ifndef!") + self.fail(f"#{token} without matching #if / #ifdef / #ifndef!") return self.stack.pop() if self.continuation: @@ -145,7 +144,7 @@ def pop_stack() -> TokenAndCondition: if token in {'if', 'ifdef', 'ifndef', 'elif'}: if not condition: - self.fail("Invalid format for #" + token + " line: no argument!") + self.fail(f"Invalid format for #{token} line: no argument!") if token in {'if', 'elif'}: if not is_a_simple_defined(condition): condition = "(" + condition + ")" @@ -155,7 +154,8 @@ def pop_stack() -> TokenAndCondition: else: fields = condition.split() if len(fields) != 1: - self.fail("Invalid format for #" + token + " line: should be exactly one argument!") + self.fail(f"Invalid format for #{token} line: " + "should be exactly one argument!") symbol = fields[0] condition = 'defined(' + symbol + ')' if token == 'ifndef': diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py new file mode 100644 index 00000000000000..ab9b586693d01c --- /dev/null +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -0,0 +1,1599 @@ +from __future__ import annotations +import ast +import enum +import inspect +import pprint +import re +import shlex +import sys +from collections.abc import Callable +from types import FunctionType, NoneType +from typing import TYPE_CHECKING, Any, NamedTuple + +import libclinic +from libclinic import ( + ClinicError, VersionTuple, + fail, warn, unspecified, unknown, NULL) +from libclinic.function import ( + Module, Class, Function, Parameter, + FunctionKind, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.converter import ( + converters, legacy_converters) +from libclinic.converters import ( + self_converter, defining_class_converter, + correct_name_for_self) +from libclinic.return_converters import ( + CReturnConverter, return_converters, + int_return_converter) +from libclinic.parser import create_parser_namespace +if TYPE_CHECKING: + from libclinic.block_parser import Block + from libclinic.app import Clinic + + +unsupported_special_methods: set[str] = set(""" + +__abs__ +__add__ +__and__ +__call__ +__delitem__ +__divmod__ +__eq__ +__float__ +__floordiv__ +__ge__ +__getattr__ +__getattribute__ +__getitem__ +__gt__ +__hash__ +__iadd__ +__iand__ +__ifloordiv__ +__ilshift__ +__imatmul__ +__imod__ +__imul__ +__index__ +__int__ +__invert__ +__ior__ +__ipow__ +__irshift__ +__isub__ +__iter__ +__itruediv__ +__ixor__ +__le__ +__len__ +__lshift__ +__lt__ +__matmul__ +__mod__ +__mul__ +__neg__ +__next__ +__or__ +__pos__ +__pow__ +__radd__ +__rand__ +__rdivmod__ +__repr__ +__rfloordiv__ +__rlshift__ +__rmatmul__ +__rmod__ +__rmul__ +__ror__ +__rpow__ +__rrshift__ +__rshift__ +__rsub__ +__rtruediv__ +__rxor__ +__setattr__ +__setitem__ +__str__ +__sub__ +__truediv__ +__xor__ + +""".strip().split()) + + +StateKeeper = Callable[[str], None] +ConverterArgs = dict[str, Any] + + +class ParamState(enum.IntEnum): + """Parameter parsing state. + + [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line + 01 2 3 4 5 6 <- state transitions + """ + # Before we've seen anything. + # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED + START = 0 + + # Left square brackets before required params. + LEFT_SQUARE_BEFORE = 1 + + # In a group, before required params. + GROUP_BEFORE = 2 + + # Required params, positional-or-keyword or positional-only (we + # don't know yet). Renumber left groups! + REQUIRED = 3 + + # Positional-or-keyword or positional-only params that now must have + # default values. + OPTIONAL = 4 + + # In a group, after required params. + GROUP_AFTER = 5 + + # Right square brackets after required params. + RIGHT_SQUARE_AFTER = 6 + + +class FunctionNames(NamedTuple): + full_name: str + c_basename: str + + +def eval_ast_expr( + node: ast.expr, + *, + filename: str = '-' +) -> Any: + """ + Takes an ast.Expr node. Compiles it into a function object, + then calls the function object with 0 arguments. + Returns the result of that function call. + + globals represents the globals dict the expression + should see. (There's no equivalent for "locals" here.) + """ + + if isinstance(node, ast.Expr): + node = node.value + + expr = ast.Expression(node) + namespace = create_parser_namespace() + co = compile(expr, filename, 'eval') + fn = FunctionType(co, namespace) + return fn() + + +class IndentStack: + def __init__(self) -> None: + self.indents: list[int] = [] + self.margin: str | None = None + + def _ensure(self) -> None: + if not self.indents: + fail('IndentStack expected indents, but none are defined.') + + def measure(self, line: str) -> int: + """ + Returns the length of the line's margin. + """ + if '\t' in line: + fail('Tab characters are illegal in the Argument Clinic DSL.') + stripped = line.lstrip() + if not len(stripped): + # we can't tell anything from an empty line + # so just pretend it's indented like our current indent + self._ensure() + return self.indents[-1] + return len(line) - len(stripped) + + def infer(self, line: str) -> int: + """ + Infer what is now the current margin based on this line. + Returns: + 1 if we have indented (or this is the first margin) + 0 if the margin has not changed + -N if we have dedented N times + """ + indent = self.measure(line) + margin = ' ' * indent + if not self.indents: + self.indents.append(indent) + self.margin = margin + return 1 + current = self.indents[-1] + if indent == current: + return 0 + if indent > current: + self.indents.append(indent) + self.margin = margin + return 1 + # indent < current + if indent not in self.indents: + fail("Illegal outdent.") + outdent_count = 0 + while indent != current: + self.indents.pop() + current = self.indents[-1] + outdent_count -= 1 + self.margin = margin + return outdent_count + + @property + def depth(self) -> int: + """ + Returns how many margins are currently defined. + """ + return len(self.indents) + + def dedent(self, line: str) -> str: + """ + Dedents a line by the currently defined margin. + """ + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" + margin = self.margin + indent = self.indents[-1] + if not line.startswith(margin): + fail('Cannot dedent; line does not start with the previous margin.') + return line[indent:] + + +class DSLParser: + function: Function | None + state: StateKeeper + keyword_only: bool + positional_only: bool + deprecated_positional: VersionTuple | None + deprecated_keyword: VersionTuple | None + group: int + parameter_state: ParamState + indent: IndentStack + kind: FunctionKind + coexist: bool + forced_text_signature: str | None + parameter_continuation: str + preserve_output: bool + critical_section: bool + target_critical_section: list[str] + from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') + + def __init__(self, clinic: Clinic) -> None: + self.clinic = clinic + + self.directives = {} + for name in dir(self): + # functions that start with directive_ are added to directives + _, s, key = name.partition("directive_") + if s: + self.directives[key] = getattr(self, name) + + # functions that start with at_ are too, with an @ in front + _, s, key = name.partition("at_") + if s: + self.directives['@' + key] = getattr(self, name) + + self.reset() + + def reset(self) -> None: + self.function = None + self.state = self.state_dsl_start + self.keyword_only = False + self.positional_only = False + self.deprecated_positional = None + self.deprecated_keyword = None + self.group = 0 + self.parameter_state: ParamState = ParamState.START + self.indent = IndentStack() + self.kind = CALLABLE + self.coexist = False + self.forced_text_signature = None + self.parameter_continuation = '' + self.preserve_output = False + self.critical_section = False + self.target_critical_section = [] + + def directive_module(self, name: str) -> None: + fields = name.split('.')[:-1] + module, cls = self.clinic._module_and_class(fields) + if cls: + fail("Can't nest a module inside a class!") + + if name in module.modules: + fail(f"Already defined module {name!r}!") + + m = Module(name, module) + module.modules[name] = m + self.block.signatures.append(m) + + def directive_class( + self, + name: str, + typedef: str, + type_object: str + ) -> None: + fields = name.split('.') + name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + parent = cls or module + if name in parent.classes: + fail(f"Already defined class {name!r}!") + + c = Class(name, module, cls, typedef, type_object) + parent.classes[name] = c + self.block.signatures.append(c) + + def directive_set(self, name: str, value: str) -> None: + if name not in ("line_prefix", "line_suffix"): + fail(f"unknown variable {name!r}") + + value = value.format_map({ + 'block comment start': '/*', + 'block comment end': '*/', + }) + + self.clinic.__dict__[name] = value + + def directive_destination( + self, + name: str, + command: str, + *args: str + ) -> None: + match command: + case "new": + self.clinic.add_destination(name, *args) + case "clear": + self.clinic.get_destination(name).clear() + case _: + fail(f"unknown destination command {command!r}") + + + def directive_output( + self, + command_or_name: str, + destination: str = '' + ) -> None: + fd = self.clinic.destination_buffers + + if command_or_name == "preset": + preset = self.clinic.presets.get(destination) + if not preset: + fail(f"Unknown preset {destination!r}!") + fd.update(preset) + return + + if command_or_name == "push": + self.clinic.destination_buffers_stack.append(fd.copy()) + return + + if command_or_name == "pop": + if not self.clinic.destination_buffers_stack: + fail("Can't 'output pop', stack is empty!") + previous_fd = self.clinic.destination_buffers_stack.pop() + fd.update(previous_fd) + return + + # secret command for debugging! + if command_or_name == "print": + self.block.output.append(pprint.pformat(fd)) + self.block.output.append('\n') + return + + d = self.clinic.get_destination_buffer(destination) + + if command_or_name == "everything": + for name in list(fd): + fd[name] = d + return + + if command_or_name not in fd: + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) + fd[command_or_name] = d + + def directive_dump(self, name: str) -> None: + self.block.output.append(self.clinic.get_destination(name).dump()) + + def directive_printout(self, *args: str) -> None: + self.block.output.append(' '.join(args)) + self.block.output.append('\n') + + def directive_preserve(self) -> None: + if self.preserve_output: + fail("Can't have 'preserve' twice in one block!") + self.preserve_output = True + + def at_classmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @classmethod, function is not a normal callable") + self.kind = CLASS_METHOD + + def at_critical_section(self, *args: str) -> None: + if len(args) > 2: + fail("Up to 2 critical section variables are supported") + self.target_critical_section.extend(args) + self.critical_section = True + + def at_getter(self) -> None: + match self.kind: + case FunctionKind.GETTER: + fail("Cannot apply @getter twice to the same function!") + case FunctionKind.SETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.GETTER + + def at_setter(self) -> None: + match self.kind: + case FunctionKind.SETTER: + fail("Cannot apply @setter twice to the same function!") + case FunctionKind.GETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.SETTER + + def at_staticmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @staticmethod, function is not a normal callable") + self.kind = STATIC_METHOD + + def at_coexist(self) -> None: + if self.coexist: + fail("Called @coexist twice!") + self.coexist = True + + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + + def parse(self, block: Block) -> None: + self.reset() + self.block = block + self.saved_output = self.block.output + block.output = [] + block_start = self.clinic.block_parser.line_number + lines = block.input.split('\n') + for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): + if '\t' in line: + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + exc.filename = self.clinic.filename + raise + + self.do_post_block_processing_cleanup(line_number) + block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) + + if self.preserve_output: + if block.output: + fail("'preserve' only works for blocks that don't produce any output!", + line_number=line_number) + block.output = self.saved_output + + def in_docstring(self) -> bool: + """Return true if we are processing a docstring.""" + return self.state in { + self.state_parameter_docstring, + self.state_function_docstring, + } + + def valid_line(self, line: str) -> bool: + # ignore comment-only lines + if line.lstrip().startswith('#'): + return False + + # Ignore empty lines too + # (but not in docstring sections!) + if not self.in_docstring() and not line.strip(): + return False + + return True + + def next( + self, + state: StateKeeper, + line: str | None = None + ) -> None: + self.state = state + if line is not None: + self.state(line) + + def state_dsl_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # is it a directive? + fields = shlex.split(line) + directive_name = fields[0] + directive = self.directives.get(directive_name, None) + if directive: + try: + directive(*fields[1:]) + except TypeError as e: + fail(str(e)) + return + + self.next(self.state_modulename_name, line) + + def parse_function_names(self, line: str) -> FunctionNames: + left, as_, right = line.partition(' as ') + full_name = left.strip() + c_basename = right.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") + if not c_basename: + fields = full_name.split(".") + if fields[-1] == '__new__': + fields.pop() + c_basename = "_".join(fields) + if not libclinic.is_legal_py_identifier(full_name): + fail(f"Illegal function name: {full_name!r}") + if not libclinic.is_legal_c_identifier(c_basename): + fail(f"Illegal C basename: {c_basename!r}") + names = FunctionNames(full_name=full_name, c_basename=c_basename) + self.normalize_function_kind(names.full_name) + return names + + def normalize_function_kind(self, fullname: str) -> None: + # Fetch the method name and possibly class. + fields = fullname.split('.') + name = fields.pop() + _, cls = self.clinic._module_and_class(fields) + + # Check special method requirements. + if name in unsupported_special_methods: + fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__init__' and (self.kind is not CALLABLE or not cls): + fail(f"{name!r} must be a normal method; got '{self.kind}'!") + if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): + fail("'__new__' must be a class method!") + if self.kind in {GETTER, SETTER} and not cls: + fail("@getter and @setter must be methods") + + # Normalise self.kind. + if name == '__new__': + self.kind = METHOD_NEW + elif name == '__init__': + self.kind = METHOD_INIT + + def resolve_return_converter( + self, full_name: str, forced_converter: str + ) -> CReturnConverter: + if forced_converter: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") + if self.kind is METHOD_INIT: + fail("__init__ methods cannot define a return type") + ast_input = f"def x() -> {forced_converter}: pass" + try: + module_node = ast.parse(ast_input) + except SyntaxError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + function_node = module_node.body[0] + assert isinstance(function_node, ast.FunctionDef) + try: + name, legacy, kwargs = self.parse_converter(function_node.returns) + if legacy: + fail(f"Legacy converter {name!r} not allowed as a return converter") + if name not in return_converters: + fail(f"No available return converter called {name!r}") + return return_converters[name](**kwargs) + except ValueError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + + if self.kind in {METHOD_INIT, SETTER}: + return int_return_converter() + return CReturnConverter() + + def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: + full_name, c_basename = names + fields = [x.strip() for x in existing.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + parent = cls or module + + for existing_function in parent.functions: + if existing_function.name == function_name: + break + else: + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) + fail(f"Couldn't find existing function {existing!r}!") + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() + else: + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) + self.function = function + self.block.signatures.append(function) + (cls or module).functions.append(function) + self.next(self.state_function_docstring) + + def state_modulename_name(self, line: str) -> None: + # looking for declaration, which establishes the leftmost column + # line should be + # modulename.fnname [as c_basename] [-> return annotation] + # square brackets denote optional syntax. + # + # alternatively: + # modulename.fnname [as c_basename] = modulename.existing_fn_name + # clones the parameters and return converter from that + # function. you can't modify them. you must enter a + # new docstring. + # + # (but we might find a directive first!) + # + # this line is permitted to start with whitespace. + # we'll call this number of spaces F (for "function"). + + assert self.valid_line(line) + self.indent.infer(line) + + # are we cloning? + before, equals, existing = line.rpartition('=') + if equals: + existing = existing.strip() + if libclinic.is_legal_py_identifier(existing): + if self.forced_text_signature: + fail("Cannot use @text_signature when cloning a function") + # we're cloning! + names = self.parse_function_names(before) + return self.parse_cloned_function(names, existing) + + line, _, returns = line.partition('->') + returns = returns.strip() + full_name, c_basename = self.parse_function_names(line) + return_converter = self.resolve_return_converter(full_name, returns) + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + func = Function( + name=function_name, + full_name=full_name, + module=module, + cls=cls, + c_basename=c_basename, + return_converter=return_converter, + kind=self.kind, + coexist=self.coexist, + critical_section=self.critical_section, + target_critical_section=self.target_critical_section, + forced_text_signature=self.forced_text_signature + ) + self.add_function(func) + + self.next(self.state_parameters_start) + + def add_function(self, func: Function) -> None: + # Insert a self converter automatically. + tp, name = correct_name_for_self(func) + if func.cls and tp == "PyObject *": + func.self_converter = self_converter(name, name, func, + type=func.cls.typedef) + else: + func.self_converter = self_converter(name, name, func) + func.parameters[name] = Parameter( + name, + inspect.Parameter.POSITIONAL_ONLY, + function=func, + converter=func.self_converter + ) + + self.block.signatures.append(func) + self.function = func + (func.cls or func.module).functions.append(func) + + # Now entering the parameters section. The rules, formally stated: + # + # * All lines must be indented with spaces only. + # * The first line must be a parameter declaration. + # * The first line must be indented. + # * This first line establishes the indent for parameters. + # * We'll call this number of spaces P (for "parameter"). + # * Thenceforth: + # * Lines indented with P spaces specify a parameter. + # * Lines indented with > P spaces are docstrings for the previous + # parameter. + # * We'll call this number of spaces D (for "docstring"). + # * All subsequent lines indented with >= D spaces are stored as + # part of the per-parameter docstring. + # * All lines will have the first D spaces of the indent stripped + # before they are stored. + # * It's illegal to have a line starting with a number of spaces X + # such that P < X < D. + # * A line with < P spaces is the first line of the function + # docstring, which ends processing for parameters and per-parameter + # docstrings. + # * The first line of the function docstring must be at the same + # indent as the function declaration. + # * It's illegal to have any line in the parameters section starting + # with X spaces such that F < X < P. (As before, F is the indent + # of the function declaration.) + # + # Also, currently Argument Clinic places the following restrictions on groups: + # * Each group must contain at least one parameter. + # * Each group may contain at most one group, which must be the furthest + # thing in the group from the required parameters. (The nested group + # must be the first in the group when it's before the required + # parameters, and the last thing in the group when after the required + # parameters.) + # * There may be at most one (top-level) group to the left or right of + # the required parameters. + # * You must specify a slash, and it must be after all parameters. + # (In other words: either all parameters are positional-only, + # or none are.) + # + # Said another way: + # * Each group must contain at least one parameter. + # * All left square brackets before the required parameters must be + # consecutive. (You can't have a left square bracket followed + # by a parameter, then another left square bracket. You can't + # have a left square bracket, a parameter, a right square bracket, + # and then a left square bracket.) + # * All right square brackets after the required parameters must be + # consecutive. + # + # These rules are enforced with a single state variable: + # "parameter_state". (Previously the code was a miasma of ifs and + # separate boolean state variables.) The states are defined in the + # ParamState class. + + def state_parameters_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # if this line is not indented, we have no parameters + if not self.indent.infer(line): + return self.next(self.state_function_docstring, line) + + assert self.function is not None + if self.function.kind in {GETTER, SETTER}: + getset = self.function.kind.name.lower() + fail(f"@{getset} methods cannot define parameters") + + self.parameter_continuation = '' + return self.next(self.state_parameter, line) + + + def to_required(self) -> None: + """ + Transition to the "required" parameter state. + """ + if self.parameter_state is not ParamState.REQUIRED: + self.parameter_state = ParamState.REQUIRED + assert self.function is not None + for p in self.function.parameters.values(): + p.group = -p.group + + def state_parameter(self, line: str) -> None: + assert isinstance(self.function, Function) + + if not self.valid_line(line): + return + + if self.parameter_continuation: + line = self.parameter_continuation + ' ' + line.lstrip() + self.parameter_continuation = '' + + assert self.indent.depth == 2 + indent = self.indent.infer(line) + if indent == -1: + # we outdented, must be to definition column + return self.next(self.state_function_docstring, line) + + if indent == 1: + # we indented, must be to new parameter docstring column + return self.next(self.state_parameter_docstring_start, line) + + line = line.rstrip() + if line.endswith('\\'): + self.parameter_continuation = line[:-1] + return + + line = line.lstrip() + version: VersionTuple | None = None + match = self.from_version_re.fullmatch(line) + if match: + line = match[1] + version = self.parse_version(match[2]) + + func = self.function + match line: + case '*': + self.parse_star(func, version) + case '[': + self.parse_opening_square_bracket(func) + case ']': + self.parse_closing_square_bracket(func) + case '/': + self.parse_slash(func, version) + case param: + self.parse_parameter(param) + + def parse_parameter(self, line: str) -> None: + assert self.function is not None + + match self.parameter_state: + case ParamState.START | ParamState.REQUIRED: + self.to_required() + case ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_BEFORE: + if not self.group: + self.to_required() + case ParamState.GROUP_AFTER | ParamState.OPTIONAL: + pass + case st: + fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") + + # handle "as" for parameters too + c_name = None + name, have_as_token, trailing = line.partition(' as ') + if have_as_token: + name = name.strip() + if ' ' not in name: + fields = trailing.strip().split(' ') + if not fields: + fail("Invalid 'as' clause!") + c_name = fields[0] + if c_name.endswith(':'): + name += ':' + c_name = c_name[:-1] + fields[0] = name + line = ' '.join(fields) + + default: str | None + base, equals, default = line.rpartition('=') + if not equals: + base = default + default = None + + module = None + try: + ast_input = f"def x({base}): pass" + module = ast.parse(ast_input) + except SyntaxError: + try: + # the last = was probably inside a function call, like + # c: int(accept={str}) + # so assume there was no actual default value. + default = None + ast_input = f"def x({line}): pass" + module = ast.parse(ast_input) + except SyntaxError: + pass + if not module: + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) + + function = module.body[0] + assert isinstance(function, ast.FunctionDef) + function_args = function.args + + if len(function_args.args) > 1: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") + if function_args.defaults or function_args.kw_defaults: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") + if function_args.kwarg: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") + + if function_args.vararg: + if any(p.is_vararg() for p in self.function.parameters.values()): + fail("Too many var args") + is_vararg = True + parameter = function_args.vararg + else: + is_vararg = False + parameter = function_args.args[0] + + parameter_name = parameter.arg + name, legacy, kwargs = self.parse_converter(parameter.annotation) + + value: object + if not default: + if self.parameter_state is ParamState.OPTIONAL: + fail(f"Can't have a parameter without a default ({parameter_name!r}) " + "after a parameter with a default!") + if is_vararg: + value = NULL + kwargs.setdefault('c_default', "NULL") + else: + value = unspecified + if 'py_default' in kwargs: + fail("You can't specify py_default without specifying a default value!") + else: + if is_vararg: + fail("Vararg can't take a default value!") + + if self.parameter_state is ParamState.REQUIRED: + self.parameter_state = ParamState.OPTIONAL + default = default.strip() + bad = False + ast_input = f"x = {default}" + try: + module = ast.parse(ast_input) + + if 'c_default' not in kwargs: + # we can only represent very simple data values in C. + # detect whether default is okay, via a denylist + # of disallowed ast nodes. + class DetectBadNodes(ast.NodeVisitor): + bad = False + def bad_node(self, node: ast.AST) -> None: + self.bad = True + + # inline function call + visit_Call = bad_node + # inline if statement ("x = 3 if y else z") + visit_IfExp = bad_node + + # comprehensions and generator expressions + visit_ListComp = visit_SetComp = bad_node + visit_DictComp = visit_GeneratorExp = bad_node + + # literals for advanced types + visit_Dict = visit_Set = bad_node + visit_List = visit_Tuple = bad_node + + # "starred": "a = [1, 2, 3]; *a" + visit_Starred = bad_node + + denylist = DetectBadNodes() + denylist.visit(module) + bad = denylist.bad + else: + # if they specify a c_default, we can be more lenient about the default value. + # but at least make an attempt at ensuring it's a valid expression. + try: + value = eval(default) + except NameError: + pass # probably a named constant + except Exception as e: + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") + else: + if value is unspecified: + fail("'unspecified' is not a legal default value!") + if bad: + fail(f"Unsupported expression as default value: {default!r}") + + assignment = module.body[0] + assert isinstance(assignment, ast.Assign) + expr = assignment.value + # mild hack: explicitly support NULL as a default value + c_default: str | None + if isinstance(expr, ast.Name) and expr.id == 'NULL': + value = NULL + py_default = '' + c_default = "NULL" + elif (isinstance(expr, ast.BinOp) or + (isinstance(expr, ast.UnaryOp) and + not (isinstance(expr.operand, ast.Constant) and + type(expr.operand.value) in {int, float, complex}) + )): + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) + py_default = default + value = unknown + elif isinstance(expr, ast.Attribute): + a = [] + n: ast.expr | ast.Attribute = expr + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") + a.append(n.id) + py_default = ".".join(reversed(a)) + + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") + + try: + value = eval(py_default) + except NameError: + value = unknown + else: + value = ast.literal_eval(expr) + py_default = repr(value) + if isinstance(value, (bool, NoneType)): + c_default = "Py_" + py_default + elif isinstance(value, str): + c_default = libclinic.c_repr(value) + else: + c_default = py_default + + except SyntaxError as e: + fail(f"Syntax error: {e.text!r}") + except (ValueError, AttributeError): + value = unknown + c_default = kwargs.get("c_default") + py_default = default + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") + + kwargs.setdefault('c_default', c_default) + kwargs.setdefault('py_default', py_default) + + dict = legacy_converters if legacy else converters + legacy_str = "legacy " if legacy else "" + if name not in dict: + fail(f'{name!r} is not a valid {legacy_str}converter') + # if you use a c_name for the parameter, we just give that name to the converter + # but the parameter object gets the python name + converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) + + kind: inspect._ParameterKind + if is_vararg: + kind = inspect.Parameter.VAR_POSITIONAL + elif self.keyword_only: + kind = inspect.Parameter.KEYWORD_ONLY + else: + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + + if isinstance(converter, self_converter): + if len(self.function.parameters) == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'self' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'self' parameter cannot have a default value.") + if self.group: + fail("A 'self' parameter cannot be in an optional group.") + kind = inspect.Parameter.POSITIONAL_ONLY + self.parameter_state = ParamState.START + self.function.parameters.clear() + else: + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") + + if isinstance(converter, defining_class_converter): + _lp = len(self.function.parameters) + if _lp == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'defining_class' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'defining_class' parameter cannot have a default value.") + if self.group: + fail("A 'defining_class' parameter cannot be in an optional group.") + if self.function.cls is None: + fail("A 'defining_class' parameter cannot be defined at module level.") + kind = inspect.Parameter.POSITIONAL_ONLY + else: + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") + + + p = Parameter(parameter_name, kind, function=self.function, + converter=converter, default=value, group=self.group, + deprecated_positional=self.deprecated_positional) + + names = [k.name for k in self.function.parameters.values()] + if parameter_name in names[1:]: + fail(f"You can't have two parameters named {parameter_name!r}!") + elif names and parameter_name == names[0] and c_name is None: + fail(f"Parameter {parameter_name!r} requires a custom C name") + + key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name + self.function.parameters[key] = p + + @staticmethod + def parse_converter( + annotation: ast.expr | None + ) -> tuple[str, bool, ConverterArgs]: + match annotation: + case ast.Constant(value=str() as value): + return value, True, {} + case ast.Name(name): + return name, False, {} + case ast.Call(func=ast.Name(name)): + kwargs: ConverterArgs = {} + for node in annotation.keywords: + if not isinstance(node.arg, str): + fail("Cannot use a kwarg splat in a function-call annotation") + kwargs[node.arg] = eval_ast_expr(node.value) + return name, False, kwargs + case _: + fail( + "Annotations must be either a name, a function call, or a string." + ) + + def parse_version(self, thenceforth: str) -> VersionTuple: + """Parse Python version in `[from ...]` marker.""" + assert isinstance(self.function, Function) + + try: + major, minor = thenceforth.split(".") + return int(major), int(minor) + except ValueError: + fail( + f"Function {self.function.name!r}: expected format '[from major.minor]' " + f"where 'major' and 'minor' are integers; got {thenceforth!r}" + ) + + def parse_star(self, function: Function, version: VersionTuple | None) -> None: + """Parse keyword-only parameter marker '*'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.keyword_only: + fail(f"Function {function.name!r} uses '*' more than once.") + self.check_previous_star() + self.check_remaining_star() + self.keyword_only = True + else: + if self.keyword_only: + fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") + if self.deprecated_positional: + if self.deprecated_positional == version: + fail(f"Function {function.name!r} uses '* [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_positional < version: + fail(f"Function {function.name!r}: '* [from " + f"{version[0]}.{version[1]}]' must precede '* [from " + f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") + self.deprecated_positional = version + + def parse_opening_square_bracket(self, function: Function) -> None: + """Parse opening parameter group symbol '['.""" + match self.parameter_state: + case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.LEFT_SQUARE_BEFORE + case ParamState.REQUIRED | ParamState.GROUP_AFTER: + self.parameter_state = ParamState.GROUP_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.b)") + self.group += 1 + function.docstring_only = True + + def parse_closing_square_bracket(self, function: Function) -> None: + """Parse closing parameter group symbol ']'.""" + if not self.group: + fail(f"Function {function.name!r} has a ']' without a matching '['.") + if not any(p.group == self.group for p in function.parameters.values()): + fail(f"Function {function.name!r} has an empty group. " + "All groups must contain at least one parameter.") + self.group -= 1 + match self.parameter_state: + case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: + self.parameter_state = ParamState.RIGHT_SQUARE_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.c)") + + def parse_slash(self, function: Function, version: VersionTuple | None) -> None: + """Parse positional-only parameter marker '/'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.deprecated_keyword: + fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/' must precede '*'") + if self.positional_only: + fail(f"Function {function.name!r} uses '/' more than once.") + else: + if self.deprecated_keyword: + if self.deprecated_keyword == version: + fail(f"Function {function.name!r} uses '/ [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_keyword > version: + fail(f"Function {function.name!r}: '/ [from " + f"{version[0]}.{version[1]}]' must precede '/ [from " + f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") + self.positional_only = True + self.deprecated_keyword = version + if version is not None: + found = False + for p in reversed(function.parameters.values()): + found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + break + if not found: + fail(f"Function {function.name!r} specifies '/ [from ...]' " + f"without preceding parameters.") + # REQUIRED and OPTIONAL are allowed here, that allows positional-only + # without option groups to work (and have default values!) + allowed = { + ParamState.REQUIRED, + ParamState.OPTIONAL, + ParamState.RIGHT_SQUARE_AFTER, + ParamState.GROUP_BEFORE, + } + if (self.parameter_state not in allowed) or self.group: + fail(f"Function {function.name!r} has an unsupported group configuration. " + f"(Unexpected state {self.parameter_state}.d)") + # fixup preceding parameters + for p in function.parameters.values(): + if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: + if version is None: + p.kind = inspect.Parameter.POSITIONAL_ONLY + elif p.deprecated_keyword is None: + p.deprecated_keyword = version + + def state_parameter_docstring_start(self, line: str) -> None: + assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" + self.parameter_docstring_indent = len(self.indent.margin) + assert self.indent.depth == 3 + return self.next(self.state_parameter_docstring, line) + + def docstring_append(self, obj: Function | Parameter, line: str) -> None: + """Add a rstripped line to the current docstring.""" + # gh-80282: We filter out non-ASCII characters from the docstring, + # since historically, some compilers may balk on non-ASCII input. + # If you're using Argument Clinic in an external project, + # you may not need to support the same array of platforms as CPython, + # so you may be able to remove this restriction. + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + + docstring = obj.docstring + if docstring: + docstring += "\n" + if stripped := line.rstrip(): + docstring += self.indent.dedent(stripped) + obj.docstring = docstring + + # every line of the docstring must start with at least F spaces, + # where F > P. + # these F spaces will be stripped. + def state_parameter_docstring(self, line: str) -> None: + if not self.valid_line(line): + return + + indent = self.indent.measure(line) + if indent < self.parameter_docstring_indent: + self.indent.infer(line) + assert self.indent.depth < 3 + if self.indent.depth == 2: + # back to a parameter + return self.next(self.state_parameter, line) + assert self.indent.depth == 1 + return self.next(self.state_function_docstring, line) + + assert self.function and self.function.parameters + last_param = next(reversed(self.function.parameters.values())) + self.docstring_append(last_param, line) + + # the final stanza of the DSL is the docstring. + def state_function_docstring(self, line: str) -> None: + assert self.function is not None + + if self.group: + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") + + if not self.valid_line(line): + return + + self.docstring_append(self.function, line) + + @staticmethod + def format_docstring_signature( + f: Function, parameters: list[Parameter] + ) -> str: + lines = [] + lines.append(f.displayname) + if f.forced_text_signature: + lines.append(f.forced_text_signature) + elif f.kind in {GETTER, SETTER}: + # @getter and @setter do not need signatures like a method or a function. + return '' + else: + lines.append('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired: int) -> str: + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break + + + added_star = False + + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(lines)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + lines.extend(["\n", indent]) + line_length = len(indent) + s = text + line_length += len(s) + lines.append(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name + + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue + + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_lines = [fix_right_bracket_count(p.right_bracket_count)] + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_lines.append('$') + + if p.is_vararg(): + p_lines.append("*") + + name = p.converter.signature_name or p.name + p_lines.append(name) + + if not p.is_vararg() and p.converter.is_optional(): + p_lines.append('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_lines.append(value) + + if (p != last_p) or need_a_trailing_slash: + p_lines.append(',') + + p_output = "".join(p_lines) + add_parameter(p_output) + + lines.append(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + lines.append(')') + + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) + + if not f.docstring_only: + lines.append("\n" + libclinic.SIG_END_MARKER + "\n") + + signature_line = "".join(lines) + + # now fix up the places where the brackets look wrong + return signature_line.replace(', ]', ',] ') + + @staticmethod + def format_docstring_parameters(params: list[Parameter]) -> str: + """Create substitution text for {parameters}""" + return "".join(p.render_docstring() + "\n" for p in params if p.docstring) + + def format_docstring(self) -> str: + assert self.function is not None + f = self.function + # For the following special cases, it does not make sense to render a docstring. + if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: + return f.docstring + + # Enforce the summary line! + # The first line of a docstring should be a summary of the function. + # It should fit on one line (80 columns? 79 maybe?) and be a paragraph + # by itself. + # + # Argument Clinic enforces the following rule: + # * either the docstring is empty, + # * or it must have a summary line. + # + # Guido said Clinic should enforce this: + # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + + lines = f.docstring.split('\n') + if len(lines) >= 2: + if lines[1]: + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") + elif len(lines) == 1: + # the docstring is only one line right now--the summary line. + # add an empty line after the summary line so we have space + # between it and the {parameters} we're about to add. + lines.append('') + + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 + if parameters_marker_count > 1: + fail('You may not specify {parameters} more than once in a docstring!') + + # insert signature at front and params after the summary line + if not parameters_marker_count: + lines.insert(2, '{parameters}') + lines.insert(0, '{signature}') + + # finalize docstring + params = f.render_parameters + parameters = self.format_docstring_parameters(params) + signature = self.format_docstring_signature(f, params) + docstring = "\n".join(lines) + return libclinic.linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() + + def check_remaining_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + symbol = '*' + elif self.deprecated_positional: + symbol = '* [from ...]' + else: + return + + for p in reversed(self.function.parameters.values()): + if self.keyword_only: + if p.kind == inspect.Parameter.KEYWORD_ONLY: + return + elif self.deprecated_positional: + if p.deprecated_positional == self.deprecated_positional: + return + break + + fail(f"Function {self.function.name!r} specifies {symbol!r} " + f"without following parameters.", line_number=lineno) + + def check_previous_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + for p in self.function.parameters.values(): + if p.kind == inspect.Parameter.VAR_POSITIONAL: + fail(f"Function {self.function.name!r} uses '*' more than once.") + + + def do_post_block_processing_cleanup(self, lineno: int) -> None: + """ + Called when processing the block is done. + """ + if not self.function: + return + + self.check_remaining_star(lineno) + try: + self.function.docstring = self.format_docstring() + except ClinicError as exc: + exc.lineno = lineno + exc.filename = self.clinic.filename + raise diff --git a/Tools/clinic/libclinic/errors.py b/Tools/clinic/libclinic/errors.py new file mode 100644 index 00000000000000..f06bdfbd864b2c --- /dev/null +++ b/Tools/clinic/libclinic/errors.py @@ -0,0 +1,72 @@ +import dataclasses as dc +from typing import Literal, NoReturn, overload + + +@dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + +class ParseError(ClinicError): + pass + + +@overload +def warn_or_fail( + *args: object, + fail: Literal[True], + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: ... + +@overload +def warn_or_fail( + *args: object, + fail: Literal[False] = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: ... + +def warn_or_fail( + *args: object, + fail: bool = False, + filename: str | None = None, + line_number: int | None = None, +) -> None: + joined = " ".join([str(a) for a in args]) + error = ClinicError(joined, filename=filename, lineno=line_number) + if fail: + raise error + else: + print(error.report(warn_only=True)) + + +def warn( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> None: + return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) + +def fail( + *args: object, + filename: str | None = None, + line_number: int | None = None, +) -> NoReturn: + warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py new file mode 100644 index 00000000000000..873ece6210017a --- /dev/null +++ b/Tools/clinic/libclinic/formatting.py @@ -0,0 +1,223 @@ +"""A collection of string formatting helpers.""" + +import functools +import textwrap +from typing import Final + +from libclinic import ClinicError + + +SIG_END_MARKER: Final = "--" + + +def docstring_for_c_string(docstring: str) -> str: + lines = [] + # Turn docstring into a properly quoted C string. + for line in docstring.split("\n"): + lines.append('"') + lines.append(_quoted_for_c_string(line)) + lines.append('\\n"\n') + + if lines[-2] == SIG_END_MARKER: + # If we only have a signature, add the blank line that the + # __text_signature__ getter expects to be there. + lines.append('"\\n"') + else: + lines.pop() + lines.append('"') + return "".join(lines) + + +def _quoted_for_c_string(text: str) -> str: + """Helper for docstring_for_c_string().""" + for old, new in ( + ("\\", "\\\\"), # must be first! + ('"', '\\"'), + ("'", "\\'"), + ): + text = text.replace(old, new) + return text + + +def c_repr(text: str) -> str: + return '"' + text + '"' + + +def wrapped_c_string_literal( + text: str, + *, + width: int = 72, + suffix: str = "", + initial_indent: int = 0, + subsequent_indent: int = 4 +) -> str: + wrapped = textwrap.wrap( + text, + width=width, + replace_whitespace=False, + drop_whitespace=False, + break_on_hyphens=False, + ) + separator = c_repr(suffix + "\n" + subsequent_indent * " ") + return initial_indent * " " + c_repr(separator.join(wrapped)) + + +def _add_prefix_and_suffix(text: str, *, prefix: str = "", suffix: str = "") -> str: + """Return 'text' with 'prefix' prepended and 'suffix' appended to all lines. + + If the last line is empty, it remains unchanged. + If text is blank, return text unchanged. + + (textwrap.indent only adds to non-blank lines.) + """ + *split, last = text.split("\n") + lines = [prefix + line + suffix + "\n" for line in split] + if last: + lines.append(prefix + last + suffix) + return "".join(lines) + + +def indent_all_lines(text: str, prefix: str) -> str: + return _add_prefix_and_suffix(text, prefix=prefix) + + +def suffix_all_lines(text: str, suffix: str) -> str: + return _add_prefix_and_suffix(text, suffix=suffix) + + +def pprint_words(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + return ", ".join(items[:-1]) + " and " + items[-1] + + +def _strip_leading_and_trailing_blank_lines(text: str) -> str: + lines = text.rstrip().split("\n") + while lines: + line = lines[0] + if line.strip(): + break + del lines[0] + return "\n".join(lines) + + +@functools.lru_cache() +def normalize_snippet(text: str, *, indent: int = 0) -> str: + """ + Reformats 'text': + * removes leading and trailing blank lines + * ensures that it does not end with a newline + * dedents so the first nonwhite character on any line is at column "indent" + """ + text = _strip_leading_and_trailing_blank_lines(text) + text = textwrap.dedent(text) + if indent: + text = textwrap.indent(text, " " * indent) + return text + + +def format_escape(text: str) -> str: + # double up curly-braces, this string will be used + # as part of a format_map() template later + text = text.replace("{", "{{") + text = text.replace("}", "}}") + return text + + +def wrap_declarations(text: str, length: int = 78) -> str: + """ + A simple-minded text wrapper for C function declarations. + + It views a declaration line as looking like this: + xxxxxxxx(xxxxxxxxx,xxxxxxxxx) + If called with length=30, it would wrap that line into + xxxxxxxx(xxxxxxxxx, + xxxxxxxxx) + (If the declaration has zero or one parameters, this + function won't wrap it.) + + If this doesn't work properly, it's probably better to + start from scratch with a more sophisticated algorithm, + rather than try and improve/debug this dumb little function. + """ + lines = [] + for line in text.split("\n"): + prefix, _, after_l_paren = line.partition("(") + if not after_l_paren: + lines.append(line) + continue + in_paren, _, after_r_paren = after_l_paren.partition(")") + if not _: + lines.append(line) + continue + if "," not in in_paren: + lines.append(line) + continue + parameters = [x.strip() + ", " for x in in_paren.split(",")] + prefix += "(" + if len(prefix) < length: + spaces = " " * len(prefix) + else: + spaces = " " * 4 + + while parameters: + line = prefix + first = True + while parameters: + if not first and (len(line) + len(parameters[0]) > length): + break + line += parameters.pop(0) + first = False + if not parameters: + line = line.rstrip(", ") + ")" + after_r_paren + lines.append(line.rstrip()) + prefix = spaces + return "\n".join(lines) + + +def linear_format(text: str, **kwargs: str) -> str: + """ + Perform str.format-like substitution, except: + * The strings substituted must be on lines by + themselves. (This line is the "source line".) + * If the substitution text is empty, the source line + is removed in the output. + * If the field is not recognized, the original line + is passed unmodified through to the output. + * If the substitution text is not empty: + * Each line of the substituted text is indented + by the indent of the source line. + * A newline will be added to the end. + """ + lines = [] + for line in text.split("\n"): + indent, curly, trailing = line.partition("{") + if not curly: + lines.extend([line, "\n"]) + continue + + name, curly, trailing = trailing.partition("}") + if not curly or name not in kwargs: + lines.extend([line, "\n"]) + continue + + if trailing: + raise ClinicError( + f"Text found after '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + if indent.strip(): + raise ClinicError( + f"Non-whitespace characters found before '{{{name}}}' block marker! " + "It must be on a line by itself." + ) + + value = kwargs[name] + if not value: + continue + + stripped = [line.rstrip() for line in value.split("\n")] + value = textwrap.indent("\n".join(stripped), indent) + lines.extend([value, "\n"]) + + return "".join(lines[:-1]) diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py new file mode 100644 index 00000000000000..93901263e44c04 --- /dev/null +++ b/Tools/clinic/libclinic/function.py @@ -0,0 +1,310 @@ +from __future__ import annotations +import dataclasses as dc +import copy +import enum +import functools +import inspect +from collections.abc import Iterable, Iterator, Sequence +from typing import Final, Any, TYPE_CHECKING +if TYPE_CHECKING: + from libclinic.converter import CConverter + from libclinic.converters import self_converter + from libclinic.return_converters import CReturnConverter + from libclinic.app import Clinic + +from libclinic import VersionTuple, unspecified + + +ClassDict = dict[str, "Class"] +ModuleDict = dict[str, "Module"] +ParamDict = dict[str, "Parameter"] + + +@dc.dataclass(repr=False) +class Module: + name: str + module: Module | Clinic + + def __post_init__(self) -> None: + self.parent = self.module + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +@dc.dataclass(repr=False) +class Class: + name: str + module: Module | Clinic + cls: Class | None + typedef: str + type_object: str + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +class FunctionKind(enum.Enum): + CALLABLE = enum.auto() + STATIC_METHOD = enum.auto() + CLASS_METHOD = enum.auto() + METHOD_INIT = enum.auto() + METHOD_NEW = enum.auto() + GETTER = enum.auto() + SETTER = enum.auto() + + @functools.cached_property + def new_or_init(self) -> bool: + return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} + + def __repr__(self) -> str: + return f"" + + +CALLABLE: Final = FunctionKind.CALLABLE +STATIC_METHOD: Final = FunctionKind.STATIC_METHOD +CLASS_METHOD: Final = FunctionKind.CLASS_METHOD +METHOD_INIT: Final = FunctionKind.METHOD_INIT +METHOD_NEW: Final = FunctionKind.METHOD_NEW +GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER + + +@dc.dataclass(repr=False) +class Function: + """ + Mutable duck type for inspect.Function. + + docstring - a str containing + * embedded line breaks + * text outdented to the left margin + * no trailing whitespace. + It will always be true that + (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) + """ + parameters: ParamDict = dc.field(default_factory=dict) + _: dc.KW_ONLY + name: str + module: Module | Clinic + cls: Class | None + c_basename: str + full_name: str + return_converter: CReturnConverter + kind: FunctionKind + coexist: bool + return_annotation: object = inspect.Signature.empty + docstring: str = '' + # docstring_only means "don't generate a machine-readable + # signature, just a normal docstring". it's True for + # functions with optional groups because we can't represent + # those accurately with inspect.Signature in 3.4. + docstring_only: bool = False + forced_text_signature: str | None = None + critical_section: bool = False + target_critical_section: list[str] = dc.field(default_factory=list) + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.self_converter: self_converter | None = None + self.__render_parameters__: list[Parameter] | None = None + + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name + + @property + def render_parameters(self) -> list[Parameter]: + if not self.__render_parameters__: + l: list[Parameter] = [] + self.__render_parameters__ = l + for p in self.parameters.values(): + p = p.copy() + p.converter.pre_render() + l.append(p) + return self.__render_parameters__ + + @property + def methoddef_flags(self) -> str | None: + if self.kind.new_or_init: + return None + flags = [] + match self.kind: + case FunctionKind.CLASS_METHOD: + flags.append('METH_CLASS') + case FunctionKind.STATIC_METHOD: + flags.append('METH_STATIC') + case _ as kind: + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} + assert kind in acceptable_kinds, f"unknown kind: {kind!r}" + if self.coexist: + flags.append('METH_COEXIST') + return '|'.join(flags) + + def __repr__(self) -> str: + return f'' + + def copy(self, **overrides: Any) -> Function: + f = dc.replace(self, **overrides) + f.parameters = { + name: value.copy(function=f) + for name, value in f.parameters.items() + } + return f + + +@dc.dataclass(repr=False, slots=True) +class Parameter: + """ + Mutable duck type of inspect.Parameter. + """ + name: str + kind: inspect._ParameterKind + _: dc.KW_ONLY + default: object = inspect.Parameter.empty + function: Function + converter: CConverter + annotation: object = inspect.Parameter.empty + docstring: str = '' + group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None + right_bracket_count: int = dc.field(init=False, default=0) + + def __repr__(self) -> str: + return f'' + + def is_keyword_only(self) -> bool: + return self.kind == inspect.Parameter.KEYWORD_ONLY + + def is_positional_only(self) -> bool: + return self.kind == inspect.Parameter.POSITIONAL_ONLY + + def is_vararg(self) -> bool: + return self.kind == inspect.Parameter.VAR_POSITIONAL + + def is_optional(self) -> bool: + return not self.is_vararg() and (self.default is not unspecified) + + def copy( + self, + /, + *, + converter: CConverter | None = None, + function: Function | None = None, + **overrides: Any + ) -> Parameter: + function = function or self.function + if not converter: + converter = copy.copy(self.converter) + converter.function = function + return dc.replace(self, **overrides, function=function, converter=converter) + + def get_displayname(self, i: int) -> str: + if i == 0: + return 'argument' + if not self.is_positional_only(): + return f'argument {self.name!r}' + else: + return f'argument {i}' + + def render_docstring(self) -> str: + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() + + +ParamTuple = tuple["Parameter", ...] + + +def permute_left_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (3,) + (2, 3) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in reversed(l): + accumulator = list(group) + accumulator + yield tuple(accumulator) + + +def permute_right_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (1,) + (1, 2) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in l: + accumulator.extend(group) + yield tuple(accumulator) + + +def permute_optional_groups( + left: Sequence[Iterable[Parameter]], + required: Iterable[Parameter], + right: Sequence[Iterable[Parameter]] +) -> tuple[ParamTuple, ...]: + """ + Generator function that computes the set of acceptable + argument lists for the provided iterables of + argument groups. (Actually it generates a tuple of tuples.) + + Algorithm: prefer left options over right options. + + If required is empty, left must also be empty. + """ + required = tuple(required) + if not required: + if left: + raise ValueError("required is empty but left is not") + + accumulator: list[ParamTuple] = [] + counts = set() + for r in permute_right_option_groups(right): + for l in permute_left_option_groups(left): + t = l + required + r + if len(t) in counts: + continue + counts.add(len(t)) + accumulator.append(t) + + accumulator.sort(key=len) + return tuple(accumulator) diff --git a/Tools/clinic/libclinic/identifiers.py b/Tools/clinic/libclinic/identifiers.py new file mode 100644 index 00000000000000..d3b80bbcef3b2b --- /dev/null +++ b/Tools/clinic/libclinic/identifiers.py @@ -0,0 +1,31 @@ +import re +from .errors import ClinicError + + +is_legal_c_identifier = re.compile("^[A-Za-z_][A-Za-z0-9_]*$").match + + +def is_legal_py_identifier(identifier: str) -> bool: + return all(is_legal_c_identifier(field) for field in identifier.split(".")) + + +# Identifiers that are okay in Python but aren't a good idea in C. +# So if they're used Argument Clinic will add "_value" to the end +# of the name in C. +_c_keywords = frozenset(""" +asm auto break case char const continue default do double +else enum extern float for goto if inline int long +register return short signed sizeof static struct switch +typedef typeof union unsigned void volatile while +""".strip().split() +) + + +def ensure_legal_c_identifier(identifier: str) -> str: + # For now, just complain if what we're given isn't legal. + if not is_legal_c_identifier(identifier): + raise ClinicError(f"Illegal C identifier: {identifier}") + # But if we picked a C keyword, pick something else. + if identifier in _c_keywords: + return identifier + "_value" + return identifier diff --git a/Tools/clinic/libclinic/language.py b/Tools/clinic/libclinic/language.py new file mode 100644 index 00000000000000..15975430c16022 --- /dev/null +++ b/Tools/clinic/libclinic/language.py @@ -0,0 +1,103 @@ +from __future__ import annotations +import abc +import typing +from collections.abc import ( + Iterable, +) + +import libclinic +from libclinic import fail +from libclinic.function import ( + Module, Class, Function) + +if typing.TYPE_CHECKING: + from libclinic.app import Clinic + + +class Language(metaclass=abc.ABCMeta): + + start_line = "" + body_prefix = "" + stop_line = "" + checksum_line = "" + + def __init__(self, filename: str) -> None: + self.filename = filename + + @abc.abstractmethod + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + ... + + def parse_line(self, line: str) -> None: + ... + + def validate(self) -> None: + def assert_only_one( + attr: str, + *additional_fields: str + ) -> None: + """ + Ensures that the string found at getattr(self, attr) + contains exactly one formatter replacement string for + each valid field. The list of valid fields is + ['dsl_name'] extended by additional_fields. + + e.g. + self.fmt = "{dsl_name} {a} {b}" + + # this passes + self.assert_only_one('fmt', 'a', 'b') + + # this fails, the format string has a {b} in it + self.assert_only_one('fmt', 'a') + + # this fails, the format string doesn't have a {c} in it + self.assert_only_one('fmt', 'a', 'b', 'c') + + # this fails, the format string has two {a}s in it, + # it must contain exactly one + self.fmt2 = '{dsl_name} {a} {a}' + self.assert_only_one('fmt2', 'a') + + """ + fields = ['dsl_name'] + fields.extend(additional_fields) + line: str = getattr(self, attr) + fcf = libclinic.FormatCounterFormatter() + fcf.format(line) + def local_fail(should_be_there_but_isnt: bool) -> None: + if should_be_there_but_isnt: + fail("{} {} must contain {{{}}} exactly once!".format( + self.__class__.__name__, attr, name)) + else: + fail("{} {} must not contain {{{}}}!".format( + self.__class__.__name__, attr, name)) + + for name, count in fcf.counts.items(): + if name in fields: + if count > 1: + local_fail(True) + else: + local_fail(False) + for name in fields: + if fcf.counts.get(name) != 1: + local_fail(True) + + assert_only_one('start_line') + assert_only_one('stop_line') + + field = "arguments" if "{arguments}" in self.checksum_line else "checksum" + assert_only_one('checksum_line', field) + + +class PythonLanguage(Language): + + language = 'Python' + start_line = "#/*[{dsl_name} input]" + body_prefix = "#" + stop_line = "#[{dsl_name} start generated code]*/" + checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py new file mode 100644 index 00000000000000..0f67901dd8609a --- /dev/null +++ b/Tools/clinic/libclinic/parse_args.py @@ -0,0 +1,948 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Final + +import libclinic +from libclinic import fail, warn +from libclinic.function import ( + Function, Parameter, + GETTER, SETTER, METHOD_NEW) +from libclinic.converter import CConverter +from libclinic.converters import ( + defining_class_converter, object_converter, self_converter) +if TYPE_CHECKING: + from libclinic.clanguage import CLanguage + from libclinic.codegen import CodeGen + + +def declare_parser( + f: Function, + *, + hasformat: bool = False, + codegen: CodeGen, +) -> str: + """ + Generates the code template for a static local PyArg_Parser variable, + with an initializer. For core code (incl. builtin modules) the + kwtuple field is also statically initialized. Otherwise + it is initialized at runtime. + """ + limited_capi = codegen.limited_capi + if hasformat: + fname = '' + format_ = '.format = "{format_units}:{name}",' + else: + fname = '.fname = "{name}",' + format_ = '' + + num_keywords = len([ + p for p in f.parameters.values() + if not p.is_positional_only() and not p.is_vararg() + ]) + + condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' + if limited_capi: + declarations = """ + #define KWTUPLE NULL + """ + elif num_keywords == 0: + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + """ + + codegen.add_include('pycore_runtime.h', '_Py_SINGLETON()', + condition=condition) + else: + # XXX Why do we not statically allocate the tuple + # for non-builtin modules? + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS %d + static struct {{ + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + }} _kwtuple = {{ + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = {{ {keywords_py} }}, + }}; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + """ % num_keywords + + codegen.add_include('pycore_gc.h', 'PyGC_Head', + condition=condition) + codegen.add_include('pycore_runtime.h', '_Py_ID()', + condition=condition) + + declarations += """ + static const char * const _keywords[] = {{{keywords_c} NULL}}; + static _PyArg_Parser _parser = {{ + .keywords = _keywords, + %s + .kwtuple = KWTUPLE, + }}; + #undef KWTUPLE + """ % (format_ or fname) + return libclinic.normalize_snippet(declarations) + + +NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" +PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) +""") +PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) +""") +PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) +""") +PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) +""") +PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +""") +PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +""") +PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) +""") +PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) +""") +PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) +""") +METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) +""") +DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); +""") +DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); +""") +GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({getset_basename}__doc__, + {docstring}); + #define {getset_basename}_HAS_DOCSTR +""") +IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) +""") +METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, +""") +GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_basename}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, + #endif +""") +SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_name}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, + #endif +""") +METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ +""") + + +class ParseArgsCodeGen: + func: Function + codegen: CodeGen + limited_capi: bool = False + + # Function parameters + parameters: list[Parameter] + converters: list[CConverter] + + # Is 'defining_class' used for the first parameter? + requires_defining_class: bool + + # Use METH_FASTCALL calling convention? + fastcall: bool + + # Declaration of the return variable (ex: "int return_value;") + return_value_declaration: str + + # Calling convention (ex: "METH_NOARGS") + flags: str + + # Variables declarations + declarations: str + + pos_only: int = 0 + min_pos: int = 0 + max_pos: int = 0 + min_kw_only: int = 0 + pseudo_args: int = 0 + vararg: int | str = NO_VARARG + + docstring_prototype: str + docstring_definition: str + impl_prototype: str | None + impl_definition: str + methoddef_define: str + parser_prototype: str + parser_definition: str + cpp_if: str + cpp_endif: str + methoddef_ifndef: str + + parser_body_fields: tuple[str, ...] + + def __init__(self, func: Function, codegen: CodeGen) -> None: + self.func = func + self.codegen = codegen + + self.parameters = list(self.func.parameters.values()) + first_param = self.parameters.pop(0) + if not isinstance(first_param.converter, self_converter): + raise ValueError("the first parameter must use self_converter") + + self.requires_defining_class = False + if self.parameters and isinstance(self.parameters[0].converter, defining_class_converter): + self.requires_defining_class = True + del self.parameters[0] + self.converters = [p.converter for p in self.parameters] + + if self.func.critical_section: + self.codegen.add_include('pycore_critical_section.h', + 'Py_BEGIN_CRITICAL_SECTION()') + self.fastcall = not self.is_new_or_init() + + self.pos_only = 0 + self.min_pos = 0 + self.max_pos = 0 + self.min_kw_only = 0 + self.pseudo_args = 0 + for i, p in enumerate(self.parameters, 1): + if p.is_keyword_only(): + assert not p.is_positional_only() + if not p.is_optional(): + self.min_kw_only = i - self.max_pos + elif p.is_vararg(): + self.pseudo_args += 1 + self.vararg = i - 1 + else: + if self.vararg == NO_VARARG: + self.max_pos = i + if p.is_positional_only(): + self.pos_only = i + if not p.is_optional(): + self.min_pos = i + + def is_new_or_init(self) -> bool: + return self.func.kind.new_or_init + + def has_option_groups(self) -> bool: + return (bool(self.parameters + and (self.parameters[0].group or self.parameters[-1].group))) + + def use_meth_o(self) -> bool: + return (len(self.parameters) == 1 + and self.parameters[0].is_positional_only() + and not self.converters[0].is_optional() + and not self.requires_defining_class + and not self.is_new_or_init()) + + def use_simple_return(self) -> bool: + return (self.func.return_converter.type == 'PyObject *' + and not self.func.critical_section) + + def select_prototypes(self) -> None: + self.docstring_prototype = '' + self.docstring_definition = '' + self.methoddef_define = METHODDEF_PROTOTYPE_DEFINE + self.return_value_declaration = "PyObject *return_value = NULL;" + + if self.is_new_or_init() and not self.func.docstring: + pass + elif self.func.kind is GETTER: + self.methoddef_define = GETTERDEF_PROTOTYPE_DEFINE + if self.func.docstring: + self.docstring_definition = GETSET_DOCSTRING_PROTOTYPE_STRVAR + elif self.func.kind is SETTER: + if self.func.docstring: + fail("docstrings are only supported for @getter, not @setter") + self.return_value_declaration = "int {return_value};" + self.methoddef_define = SETTERDEF_PROTOTYPE_DEFINE + else: + self.docstring_prototype = DOCSTRING_PROTOTYPE_VAR + self.docstring_definition = DOCSTRING_PROTOTYPE_STRVAR + + def init_limited_capi(self) -> None: + self.limited_capi = self.codegen.limited_capi + if self.limited_capi and (self.pseudo_args or + (any(p.is_optional() for p in self.parameters) and + any(p.is_keyword_only() and not p.is_optional() for p in self.parameters)) or + any(c.broken_limited_capi for c in self.converters)): + warn(f"Function {self.func.full_name} cannot use limited C API") + self.limited_capi = False + + def parser_body( + self, + *fields: str, + declarations: str = '' + ) -> None: + lines = [self.parser_prototype] + self.parser_body_fields = fields + + preamble = libclinic.normalize_snippet(""" + {{ + {return_value_declaration} + {parser_declarations} + {declarations} + {initializers} + """) + "\n" + finale = libclinic.normalize_snippet(""" + {modifications} + {lock} + {return_value} = {c_basename}_impl({impl_arguments}); + {unlock} + {return_conversion} + {post_parsing} + + {exit_label} + {cleanup} + return return_value; + }} + """) + for field in preamble, *fields, finale: + lines.append(field) + code = libclinic.linear_format("\n".join(lines), + parser_declarations=self.declarations) + self.parser_definition = code + + def parse_no_args(self) -> None: + parser_code: list[str] | None + simple_return = self.use_simple_return() + if self.func.kind is GETTER: + self.parser_prototype = PARSER_PROTOTYPE_GETTER + parser_code = [] + elif self.func.kind is SETTER: + self.parser_prototype = PARSER_PROTOTYPE_SETTER + parser_code = [] + elif not self.requires_defining_class: + # no self.parameters, METH_NOARGS + self.flags = "METH_NOARGS" + self.parser_prototype = PARSER_PROTOTYPE_NOARGS + parser_code = [] + else: + assert self.fastcall + + self.flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_DEF_CLASS + return_error = ('return NULL;' if simple_return + else 'goto exit;') + parser_code = [libclinic.normalize_snippet(""" + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ + PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); + %s + }} + """ % return_error, indent=4)] + + if simple_return: + self.parser_definition = '\n'.join([ + self.parser_prototype, + '{{', + *parser_code, + ' return {c_basename}_impl({impl_arguments});', + '}}']) + else: + self.parser_body(*parser_code) + + def parse_one_arg(self) -> None: + self.flags = "METH_O" + + if (isinstance(self.converters[0], object_converter) and + self.converters[0].format_unit == 'O'): + meth_o_prototype = METH_O_PROTOTYPE + + if self.use_simple_return(): + # maps perfectly to METH_O, doesn't need a return converter. + # so we skip making a parse function + # and call directly into the impl function. + self.impl_prototype = '' + self.impl_definition = meth_o_prototype + else: + # SLIGHT HACK + # use impl_parameters for the parser here! + self.parser_prototype = meth_o_prototype + self.parser_body() + + else: + argname = 'arg' + if self.parameters[0].name == argname: + argname += '_' + self.parser_prototype = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *%s) + """ % argname) + + displayname = self.parameters[0].get_displayname(0) + parsearg: str | None + parsearg = self.converters[0].parse_arg(argname, displayname, + limited_capi=self.limited_capi) + if parsearg is None: + self.converters[0].use_converter() + parsearg = """ + if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + """ % argname + + parser_code = libclinic.normalize_snippet(parsearg, indent=4) + self.parser_body(parser_code) + + def parse_option_groups(self) -> None: + # positional parameters with option groups + # (we have to generate lots of PyArg_ParseTuple calls + # in a big switch statement) + + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + parser_code = ' {option_group_parsing}' + self.parser_body(parser_code) + + def parse_pos_only(self) -> None: + if self.fastcall: + # positional-only, but no option groups + # we only need one call to _PyArg_ParseStack + + self.flags = "METH_FASTCALL" + self.parser_prototype = PARSER_PROTOTYPE_FASTCALL + nargs = 'nargs' + argname_fmt = 'args[%d]' + else: + # positional-only, but no option groups + # we only need one call to PyArg_ParseTuple + + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + if self.limited_capi: + nargs = 'PyTuple_Size(args)' + argname_fmt = 'PyTuple_GetItem(args, %d)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + argname_fmt = 'PyTuple_GET_ITEM(args, %d)' + + left_args = f"{nargs} - {self.max_pos}" + max_args = NO_VARARG if (self.vararg != NO_VARARG) else self.max_pos + if self.limited_capi: + parser_code = [] + if nargs != 'nargs': + nargs_def = f'Py_ssize_t nargs = {nargs};' + parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) + nargs = 'nargs' + if self.min_pos == max_args: + pl = '' if self.min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} != {self.min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected {self.min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + if self.min_pos: + pl = '' if self.min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} < {self.min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at least {self.min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + if max_args != NO_VARARG: + pl = '' if max_args == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} > {max_args}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_CheckPositional()') + parser_code = [libclinic.normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=4)] + + has_optional = False + use_parser_code = True + for i, p in enumerate(self.parameters): + if p.is_vararg(): + if self.fastcall: + parser_code.append(libclinic.normalize_snippet(""" + %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} + for (Py_ssize_t i = 0; i < %s; ++i) {{ + PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); + }} + """ % ( + p.converter.parser_name, + left_args, + p.converter.parser_name, + left_args, + p.converter.parser_name, + self.max_pos + ), indent=4)) + else: + parser_code.append(libclinic.normalize_snippet(""" + %s = PyTuple_GetSlice(%d, -1); + """ % ( + p.converter.parser_name, + self.max_pos + ), indent=4)) + continue + + displayname = p.get_displayname(i+1) + argname = argname_fmt % i + parsearg: str | None + parsearg = p.converter.parse_arg(argname, displayname, limited_capi=self.limited_capi) + if parsearg is None: + use_parser_code = False + parser_code = [] + break + if has_optional or p.is_optional(): + has_optional = True + parser_code.append(libclinic.normalize_snippet(""" + if (%s < %d) {{ + goto skip_optional; + }} + """, indent=4) % (nargs, i + 1)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + + if use_parser_code: + if has_optional: + parser_code.append("skip_optional:") + else: + for parameter in self.parameters: + parameter.converter.use_converter() + + if self.limited_capi: + self.fastcall = False + if self.fastcall: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseStack()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + self.flags = "METH_VARARGS" + self.parser_prototype = PARSER_PROTOTYPE_VARARGS + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTuple(args, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + self.parser_body(*parser_code) + + def parse_general(self, clang: CLanguage) -> None: + parsearg: str | None + deprecated_positionals: dict[int, Parameter] = {} + deprecated_keywords: dict[int, Parameter] = {} + for i, p in enumerate(self.parameters): + if p.deprecated_positional: + deprecated_positionals[i] = p + if p.deprecated_keyword: + deprecated_keywords[i] = p + + has_optional_kw = ( + max(self.pos_only, self.min_pos) + self.min_kw_only + < len(self.converters) - int(self.vararg != NO_VARARG) + ) + + use_parser_code = True + if self.limited_capi: + parser_code = [] + use_parser_code = False + self.fastcall = False + else: + if self.vararg == NO_VARARG: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + self.min_pos, + self.max_pos, + self.min_kw_only + ) + nargs = "nargs" + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywordsWithVararg()') + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + self.min_pos, + self.max_pos, + self.min_kw_only, + self.vararg + ) + nargs = f"Py_MIN(nargs, {self.max_pos})" if self.max_pos else "0" + + if self.fastcall: + self.flags = "METH_FASTCALL|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_FASTCALL_KEYWORDS + argname_fmt = 'args[%d]' + self.declarations = declare_parser(self.func, codegen=self.codegen) + self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + if has_optional_kw: + self.declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); + if (!args) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + else: + # positional-or-keyword arguments + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + argname_fmt = 'fastargs[%d]' + self.declarations = declare_parser(self.func, codegen=self.codegen) + self.declarations += "\nPyObject *argsbuf[%s];" % len(self.converters) + self.declarations += "\nPyObject * const *fastargs;" + self.declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if has_optional_kw: + self.declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, self.min_pos + self.min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); + if (!fastargs) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + + if self.requires_defining_class: + self.flags = 'METH_METHOD|' + self.flags + self.parser_prototype = PARSER_PROTOTYPE_DEF_CLASS + + if use_parser_code: + if deprecated_keywords: + code = clang.deprecate_keyword_use(self.func, deprecated_keywords, + argname_fmt, + codegen=self.codegen, + fastcall=self.fastcall) + parser_code.append(code) + + add_label: str | None = None + for i, p in enumerate(self.parameters): + if isinstance(p.converter, defining_class_converter): + raise ValueError("defining_class should be the first " + "parameter (after clang)") + displayname = p.get_displayname(i+1) + parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=self.limited_capi) + if parsearg is None: + parser_code = [] + use_parser_code = False + break + if add_label and (i == self.pos_only or i == self.max_pos): + parser_code.append("%s:" % add_label) + add_label = None + if not p.is_optional(): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + elif i < self.pos_only: + add_label = 'skip_optional_posonly' + parser_code.append(libclinic.normalize_snippet(""" + if (nargs < %d) {{ + goto %s; + }} + """ % (i + 1, add_label), indent=4)) + if has_optional_kw: + parser_code.append(libclinic.normalize_snippet(""" + noptargs--; + """, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + if i < self.max_pos: + label = 'skip_optional_pos' + first_opt = max(self.min_pos, self.pos_only) + else: + label = 'skip_optional_kwonly' + first_opt = self.max_pos + self.min_kw_only + if self.vararg != NO_VARARG: + first_opt += 1 + if i == first_opt: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (!noptargs) {{ + goto %s; + }} + """ % add_label, indent=4)) + if i + 1 == len(self.parameters): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (%s) {{ + """ % (argname_fmt % i), indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) + parser_code.append(libclinic.normalize_snippet(""" + if (!--noptargs) {{ + goto %s; + }} + }} + """ % add_label, indent=4)) + + if use_parser_code: + if add_label: + parser_code.append("%s:" % add_label) + else: + for parameter in self.parameters: + parameter.converter.use_converter() + + self.declarations = declare_parser(self.func, codegen=self.codegen, + hasformat=True) + if self.limited_capi: + # positional-or-keyword arguments + assert not self.fastcall + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, + {parse_arguments})) + goto exit; + """, indent=4)] + self.declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" + if deprecated_positionals or deprecated_keywords: + self.declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" + + elif self.fastcall: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseStackAndKeywords()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_ParseTupleAndKeywordsFast()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + if deprecated_positionals or deprecated_keywords: + self.declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = clang.deprecate_keyword_use(self.func, deprecated_keywords, + codegen=self.codegen, + fastcall=self.fastcall) + parser_code.append(code) + + if deprecated_positionals: + code = clang.deprecate_positional_use(self.func, deprecated_positionals) + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + + assert self.parser_prototype is not None + self.parser_body(*parser_code, declarations=self.declarations) + + def copy_includes(self) -> None: + # Copy includes from parameters to Clinic after parse_arg() + # has been called above. + for converter in self.converters: + for include in converter.get_includes(): + self.codegen.add_include( + include.filename, + include.reason, + condition=include.condition) + + def handle_new_or_init(self) -> None: + self.methoddef_define = '' + + if self.func.kind is METHOD_NEW: + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + else: + self.return_value_declaration = "int return_value = -1;" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD___INIT__ + + fields: list[str] = list(self.parser_body_fields) + parses_positional = 'METH_NOARGS' not in self.flags + parses_keywords = 'METH_KEYWORDS' in self.flags + if parses_keywords: + assert parses_positional + + if self.requires_defining_class: + raise ValueError("Slot methods cannot access their defining class.") + + if not parses_keywords: + self.declarations = '{base_type_ptr}' + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoKeywords()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ + goto exit; + }} + """, indent=4)) + if not parses_positional: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoPositional()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ + goto exit; + }} + """, indent=4)) + + self.parser_body(*fields, declarations=self.declarations) + + def process_methoddef(self, clang: CLanguage) -> None: + methoddef_cast_end = "" + if self.flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): + methoddef_cast = "(PyCFunction)" + elif self.func.kind is GETTER: + methoddef_cast = "" # This should end up unused + elif self.limited_capi: + methoddef_cast = "(PyCFunction)(void(*)(void))" + else: + methoddef_cast = "_PyCFunction_CAST(" + methoddef_cast_end = ")" + + if self.func.methoddef_flags: + self.flags += '|' + self.func.methoddef_flags + + self.methoddef_define = self.methoddef_define.replace('{methoddef_flags}', self.flags) + self.methoddef_define = self.methoddef_define.replace('{methoddef_cast}', methoddef_cast) + self.methoddef_define = self.methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) + + self.methoddef_ifndef = '' + conditional = clang.cpp.condition() + if not conditional: + self.cpp_if = self.cpp_endif = '' + else: + self.cpp_if = "#if " + conditional + self.cpp_endif = "#endif /* " + conditional + " */" + + if self.methoddef_define and self.codegen.add_ifndef_symbol(self.func.full_name): + self.methoddef_ifndef = METHODDEF_PROTOTYPE_IFNDEF + + def finalize(self, clang: CLanguage) -> None: + # add ';' to the end of self.parser_prototype and self.impl_prototype + # (they mustn't be None, but they could be an empty string.) + assert self.parser_prototype is not None + if self.parser_prototype: + assert not self.parser_prototype.endswith(';') + self.parser_prototype += ';' + + if self.impl_prototype is None: + self.impl_prototype = self.impl_definition + if self.impl_prototype: + self.impl_prototype += ";" + + self.parser_definition = self.parser_definition.replace("{return_value_declaration}", self.return_value_declaration) + + compiler_warning = clang.compiler_deprecated_warning(self.func, self.parameters) + if compiler_warning: + self.parser_definition = compiler_warning + "\n\n" + self.parser_definition + + def create_template_dict(self) -> dict[str, str]: + d = { + "docstring_prototype" : self.docstring_prototype, + "docstring_definition" : self.docstring_definition, + "impl_prototype" : self.impl_prototype, + "methoddef_define" : self.methoddef_define, + "parser_prototype" : self.parser_prototype, + "parser_definition" : self.parser_definition, + "impl_definition" : self.impl_definition, + "cpp_if" : self.cpp_if, + "cpp_endif" : self.cpp_endif, + "methoddef_ifndef" : self.methoddef_ifndef, + } + + # make sure we didn't forget to assign something, + # and wrap each non-empty value in \n's + d2 = {} + for name, value in d.items(): + assert value is not None, "got a None value for template " + repr(name) + if value: + value = '\n' + value + '\n' + d2[name] = value + return d2 + + def parse_args(self, clang: CLanguage) -> dict[str, str]: + self.select_prototypes() + self.init_limited_capi() + + self.flags = "" + self.declarations = "" + self.parser_prototype = "" + self.parser_definition = "" + self.impl_prototype = None + self.impl_definition = IMPL_DEFINITION_PROTOTYPE + + # parser_body_fields remembers the fields passed in to the + # previous call to parser_body. this is used for an awful hack. + self.parser_body_fields: tuple[str, ...] = () + + if not self.parameters: + self.parse_no_args() + elif self.use_meth_o(): + self.parse_one_arg() + elif self.has_option_groups(): + self.parse_option_groups() + elif (not self.requires_defining_class + and self.pos_only == len(self.parameters) - self.pseudo_args): + self.parse_pos_only() + else: + self.parse_general(clang) + + self.copy_includes() + if self.is_new_or_init(): + self.handle_new_or_init() + self.process_methoddef(clang) + self.finalize(clang) + + return self.create_template_dict() diff --git a/Tools/clinic/libclinic/parser.py b/Tools/clinic/libclinic/parser.py new file mode 100644 index 00000000000000..8135dd9ceb3937 --- /dev/null +++ b/Tools/clinic/libclinic/parser.py @@ -0,0 +1,53 @@ +from __future__ import annotations +import contextlib +import functools +import io +from types import NoneType +from typing import Any, Protocol, TYPE_CHECKING + +from libclinic import unspecified +from libclinic.block_parser import Block +from libclinic.converter import CConverter, converters +from libclinic.converters import buffer, robuffer, rwbuffer +from libclinic.return_converters import CReturnConverter, return_converters +if TYPE_CHECKING: + from libclinic.app import Clinic + + +class Parser(Protocol): + def __init__(self, clinic: Clinic) -> None: ... + def parse(self, block: Block) -> None: ... + + +@functools.cache +def _create_parser_base_namespace() -> dict[str, Any]: + ns = dict( + CConverter=CConverter, + CReturnConverter=CReturnConverter, + buffer=buffer, + robuffer=robuffer, + rwbuffer=rwbuffer, + unspecified=unspecified, + NoneType=NoneType, + ) + for name, converter in converters.items(): + ns[f'{name}_converter'] = converter + for name, return_converter in return_converters.items(): + ns[f'{name}_return_converter'] = return_converter + return ns + + +def create_parser_namespace() -> dict[str, Any]: + base_namespace = _create_parser_base_namespace() + return base_namespace.copy() + + +class PythonParser: + def __init__(self, clinic: Clinic) -> None: + pass + + def parse(self, block: Block) -> None: + namespace = create_parser_namespace() + with contextlib.redirect_stdout(io.StringIO()) as s: + exec(block.input, namespace) + block.output = s.getvalue() diff --git a/Tools/clinic/libclinic/return_converters.py b/Tools/clinic/libclinic/return_converters.py new file mode 100644 index 00000000000000..b41e053bae5f3a --- /dev/null +++ b/Tools/clinic/libclinic/return_converters.py @@ -0,0 +1,173 @@ +import sys +from collections.abc import Callable +from libclinic.codegen import CRenderData +from libclinic.function import Function +from typing import Any + + +ReturnConverterType = Callable[..., "CReturnConverter"] + + +# maps strings to callables. +# these callables must be of the form: +# def foo(*, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CReturnConverter object. +# The callable should not call builtins.print. +ReturnConverterDict = dict[str, ReturnConverterType] +return_converters: ReturnConverterDict = {} + + +def add_c_return_converter( + f: ReturnConverterType, + name: str | None = None +) -> ReturnConverterType: + if not name: + name = f.__name__ + if not name.endswith('_return_converter'): + return f + name = name.removesuffix('_return_converter') + return_converters[name] = f + return f + + +class CReturnConverterAutoRegister(type): + def __init__( + cls: ReturnConverterType, + name: str, + bases: tuple[type[object], ...], + classdict: dict[str, Any] + ) -> None: + add_c_return_converter(cls) + + +class CReturnConverter(metaclass=CReturnConverterAutoRegister): + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type = 'PyObject *' + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + default: object = None + + def __init__( + self, + *, + py_default: str | None = None, + **kwargs: Any + ) -> None: + self.py_default = py_default + try: + self.return_converter_init(**kwargs) + except TypeError as e: + s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) + sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) + + def return_converter_init(self) -> None: ... + + def declare(self, data: CRenderData) -> None: + line: list[str] = [] + add = line.append + add(self.type) + if not self.type.endswith('*'): + add(' ') + add(data.converter_retval + ';') + data.declarations.append(''.join(line)) + data.return_value = data.converter_retval + + def err_occurred_if( + self, + expr: str, + data: CRenderData + ) -> None: + line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def err_occurred_if_null_pointer( + self, + variable: str, + data: CRenderData + ) -> None: + line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + + +add_c_return_converter(CReturnConverter, 'object') + + +class bool_return_converter(CReturnConverter): + type = 'int' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) + + +class long_return_converter(CReturnConverter): + type = 'long' + conversion_fn = 'PyLong_FromLong' + cast = '' + unsigned_cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) + data.return_conversion.append( + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) + + +class int_return_converter(long_return_converter): + type = 'int' + cast = '(long)' + + +class unsigned_long_return_converter(long_return_converter): + type = 'unsigned long' + conversion_fn = 'PyLong_FromUnsignedLong' + unsigned_cast = '(unsigned long)' + + +class unsigned_int_return_converter(unsigned_long_return_converter): + type = 'unsigned int' + cast = '(unsigned long)' + unsigned_cast = '(unsigned int)' + + +class Py_ssize_t_return_converter(long_return_converter): + type = 'Py_ssize_t' + conversion_fn = 'PyLong_FromSsize_t' + + +class size_t_return_converter(long_return_converter): + type = 'size_t' + conversion_fn = 'PyLong_FromSize_t' + unsigned_cast = '(size_t)' + + +class double_return_converter(CReturnConverter): + type = 'double' + cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) + data.return_conversion.append( + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) + + +class float_return_converter(double_return_converter): + type = 'float' + cast = '(double)' diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py new file mode 100644 index 00000000000000..17e8f35be73bf4 --- /dev/null +++ b/Tools/clinic/libclinic/utils.py @@ -0,0 +1,93 @@ +import collections +import enum +import hashlib +import os +import re +import string +from typing import Literal, Final + + +def write_file(filename: str, new_contents: str) -> None: + """Write new content to file, iff the content changed.""" + try: + with open(filename, encoding="utf-8") as fp: + old_contents = fp.read() + + if old_contents == new_contents: + # no change: avoid modifying the file modification time + return + except FileNotFoundError: + pass + # Atomic write using a temporary file and os.replace() + filename_new = f"{filename}.new" + with open(filename_new, "w", encoding="utf-8") as fp: + fp.write(new_contents) + try: + os.replace(filename_new, filename) + except: + os.unlink(filename_new) + raise + + +def compute_checksum(input_: str, length: int | None = None) -> str: + checksum = hashlib.sha1(input_.encode("utf-8")).hexdigest() + if length: + checksum = checksum[:length] + return checksum + + +def create_regex( + before: str, after: str, word: bool = True, whole_line: bool = True +) -> re.Pattern[str]: + """Create a regex object for matching marker lines.""" + group_re = r"\w+" if word else ".+" + before = re.escape(before) + after = re.escape(after) + pattern = rf"{before}({group_re}){after}" + if whole_line: + pattern = rf"^{pattern}$" + return re.compile(pattern) + + +class FormatCounterFormatter(string.Formatter): + """ + This counts how many instances of each formatter + "replacement string" appear in the format string. + + e.g. after evaluating "string {a}, {b}, {c}, {a}" + the counts dict would now look like + {'a': 2, 'b': 1, 'c': 1} + """ + + def __init__(self) -> None: + self.counts = collections.Counter[str]() + + def get_value( + self, key: str, args: object, kwargs: object # type: ignore[override] + ) -> Literal[""]: + self.counts[key] += 1 + return "" + + +VersionTuple = tuple[int, int] + + +class Sentinels(enum.Enum): + unspecified = "unspecified" + unknown = "unknown" + + def __repr__(self) -> str: + return f"<{self.value.capitalize()}>" + + +unspecified: Final = Sentinels.unspecified +unknown: Final = Sentinels.unknown + + +# This one needs to be a distinct class, unlike the other two +class Null: + def __repr__(self) -> str: + return '' + + +NULL = Null() diff --git a/Tools/freeze/README b/Tools/freeze/README index 9b3ea1f2c723b1..516077bc7daa89 100644 --- a/Tools/freeze/README +++ b/Tools/freeze/README @@ -218,6 +218,11 @@ source tree). It is possible to create frozen programs that don't have a console window, by specifying the option '-s windows'. See the Usage below. +Usage under macOS +----------------- + +On macOS the freeze tool is not supported for framework builds. + Usage ----- diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index bc5e43f4853deb..de9772732cdb5d 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -136,6 +136,11 @@ def main(): makefile = 'Makefile' subsystem = 'console' + if sys.platform == "darwin" and sysconfig.get_config_var("PYTHONFRAMEWORK"): + print(f"{sys.argv[0]} cannot be used with framework builds of Python", file=sys.stderr) + sys.exit(1) + + # parse command line by first replacing any "-i" options with the # file contents. pos = 1 diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 78b0c08d25ab01..8aa74635aedb4f 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -66,11 +66,22 @@ def _type_unsigned_short_ptr(): def _type_unsigned_int_ptr(): return gdb.lookup_type('unsigned int').pointer() - def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _sizeof_pyobject(): + return gdb.lookup_type('PyObject').sizeof + +def _managed_dict_offset(): + # See pycore_object.h + pyobj = gdb.lookup_type("PyObject") + if any(field.name == "ob_ref_local" for field in pyobj.fields()): + return -1 * _sizeof_void_p() + else: + return -3 * _sizeof_void_p() + +Py_TPFLAGS_INLINE_VALUES = (1 << 2) Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) Py_TPFLAGS_LONG_SUBCLASS = (1 << 24) @@ -89,6 +100,8 @@ def _sizeof_void_p(): hexdigits = "0123456789abcdef" +USED_TAGS = 0b11 + ENCODING = locale.getpreferredencoding() FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' @@ -147,6 +160,8 @@ class PyObjectPtr(object): _typename = 'PyObject' def __init__(self, gdbval, cast_to=None): + # Clear the tagged pointer + gdbval = gdb.Value(int(gdbval) & (~USED_TAGS)).cast(gdbval.type) if cast_to: self._gdbval = gdbval.cast(cast_to) else: @@ -244,7 +259,7 @@ def proxyval(self, visited): Derived classes will override this. - For example, a PyIntObject* with ob_ival 42 in the inferior process + For example, a PyLongObjectPtr* with long_value 42 in the inferior process should result in an int(42) in this process. visited: a set of all gdb.Value pyobject pointers already visited @@ -457,7 +472,7 @@ def get_attr_dict(self): if dictoffset < 0: if int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT: assert dictoffset == -1 - dictoffset = -3 * _sizeof_void_p() + dictoffset = _managed_dict_offset() else: type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer() tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size']) @@ -485,12 +500,12 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - charptrptr_t = _type_char_ptr().pointer() - ptr = self._gdbval.cast(charptrptr_t) - 3 - char_ptr = ptr.dereference() - if (int(char_ptr) & 1) == 0: + obj_ptr = self._gdbval.cast(_type_char_ptr()) + dict_ptr_ptr = obj_ptr + _managed_dict_offset() + dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() + if int(dict_ptr): return None - char_ptr += 1 + char_ptr = obj_ptr + _sizeof_pyobject() values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values) @@ -856,7 +871,7 @@ class PyLongObjectPtr(PyObjectPtr): def proxyval(self, visited): ''' - Python's Include/longobjrep.h has this declaration: + Python's Include/longinterpr.h has this declaration: typedef struct _PyLongValue { uintptr_t lv_tag; /* Number of digits, sign and flags */ @@ -865,14 +880,18 @@ def proxyval(self, visited): struct _longobject { PyObject_HEAD - _PyLongValue long_value; + _PyLongValue long_value; }; with this description: The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + The sign of the value is stored in the lower 2 bits of lv_tag. + - 0: Positive + - 1: Zero + - 2: Negative + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. where SHIFT can be either: #define PyLong_SHIFT 30 @@ -1065,9 +1084,9 @@ def _f_nlocalsplus(self): def _f_lasti(self): codeunit_p = gdb.lookup_type("_Py_CODEUNIT").pointer() - prev_instr = self._gdbval["prev_instr"] + instr_ptr = self._gdbval["instr_ptr"] first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) - return int(prev_instr - first_instr) + return int(instr_ptr - first_instr) def is_shim(self): return self._f_special("owner", int) == FRAME_OWNED_BY_CSTACK @@ -1746,8 +1765,11 @@ def is_waiting_for_gil(self): return (name == 'take_gil') def is_gc_collect(self): - '''Is this frame gc_collect_main() within the garbage-collector?''' - return self._gdbframe.name() in ('collect', 'gc_collect_main') + '''Is this frame a collector within the garbage-collector?''' + return self._gdbframe.name() in ( + 'collect', 'gc_collect_full', 'gc_collect_main', + 'gc_collect_young', 'gc_collect_increment', + ) def get_pyop(self): try: diff --git a/Tools/importbench/importbench.py b/Tools/importbench/importbench.py index 0c4b3bc73517c5..eb101fe616c587 100644 --- a/Tools/importbench/importbench.py +++ b/Tools/importbench/importbench.py @@ -165,8 +165,8 @@ def using_bytecode_benchmark(seconds, repeat): def main(import_, options): if options.source_file: - with options.source_file: - prev_results = json.load(options.source_file) + with open(options.source_file, 'r', encoding='utf-8') as source_file: + prev_results = json.load(source_file) else: prev_results = {} __builtins__.__import__ = import_ @@ -218,8 +218,8 @@ def main(import_, options): new_result/old_result) print(benchmark_name, ':', result) if options.dest_file: - with options.dest_file: - json.dump(new_results, options.dest_file, indent=2) + with open(options.dest_file, 'w', encoding='utf-8') as dest_file: + json.dump(new_results, dest_file, indent=2) if __name__ == '__main__': @@ -229,11 +229,9 @@ def main(import_, options): parser.add_argument('-b', '--builtin', dest='builtin', action='store_true', default=False, help="use the built-in __import__") parser.add_argument('-r', '--read', dest='source_file', - type=argparse.FileType('r'), help='file to read benchmark data from to compare ' 'against') parser.add_argument('-w', '--write', dest='dest_file', - type=argparse.FileType('w'), help='file to write benchmark data to') parser.add_argument('--benchmark', dest='benchmark', help='specific benchmark to run') diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py deleted file mode 100644 index 4017149ec91630..00000000000000 --- a/Tools/iobench/iobench.py +++ /dev/null @@ -1,568 +0,0 @@ -import itertools -import os -import platform -import re -import sys -import time -from optparse import OptionParser - -out = sys.stdout - -TEXT_ENCODING = 'utf8' -NEWLINES = 'lf' - - -def text_open(fn, mode, encoding=None): - try: - return open(fn, mode, encoding=encoding or TEXT_ENCODING) - except TypeError: - return open(fn, mode) - - -def get_file_sizes(): - for s in ['20 KiB', '400 KiB', '10 MiB']: - size, unit = s.split() - size = int(size) * {'KiB': 1024, 'MiB': 1024 ** 2}[unit] - yield s.replace(' ', ''), size - - -def get_binary_files(): - return ((name + ".bin", size) for name, size in get_file_sizes()) - - -def get_text_files(): - return ((f"{name}-{TEXT_ENCODING}-{NEWLINES}.txt", size) - for name, size in get_file_sizes()) - - -def with_open_mode(mode): - def decorate(f): - f.file_open_mode = mode - return f - return decorate - - -def with_sizes(*sizes): - def decorate(f): - f.file_sizes = sizes - return f - return decorate - - -# Here begin the tests - -@with_open_mode("r") -@with_sizes("medium") -def read_bytewise(f): - """ read one unit at a time """ - f.seek(0) - while f.read(1): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_small_chunks(f): - """ read 20 units at a time """ - f.seek(0) - while f.read(20): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_big_chunks(f): - """ read 4096 units at a time """ - f.seek(0) - while f.read(4096): - pass - - -@with_open_mode("r") -@with_sizes("small", "medium", "large") -def read_whole_file(f): - """ read whole contents at once """ - f.seek(0) - while f.read(): - pass - - -@with_open_mode("rt") -@with_sizes("medium") -def read_lines(f): - """ read one line at a time """ - f.seek(0) - for line in f: - pass - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_bytewise(f): - """ seek forward one unit at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1): - f.seek(i, 0) - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_blockwise(f): - """ seek forward 1000 units at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1, 1000): - f.seek(i, 0) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_bytewise(f): - """ alternate read & seek one unit """ - f.seek(0) - while f.read(1): - f.seek(1, 1) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_blockwise(f): - """ alternate read & seek 1000 units """ - f.seek(0) - while f.read(1000): - f.seek(1000, 1) - - -@with_open_mode("w") -@with_sizes("small") -def write_bytewise(f, source): - """ write one unit at a time """ - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_small_chunks(f, source): - """ write 20 units at a time """ - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_medium_chunks(f, source): - """ write 4096 units at a time """ - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("w") -@with_sizes("large") -def write_large_chunks(f, source): - """ write 1e6 units at a time """ - for i in range(0, len(source), 1000000): - f.write(source[i:i+1000000]) - - -@with_open_mode("w+") -@with_sizes("small") -def modify_bytewise(f, source): - """ modify one unit at a time """ - f.seek(0) - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_small_chunks(f, source): - """ modify 20 units at a time """ - f.seek(0) - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_medium_chunks(f, source): - """ modify 4096 units at a time """ - f.seek(0) - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_bytewise(f, source): - """ alternate write & seek one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.write(source[i:i+1]) - f.seek(i+2) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_blockwise(f, source): - """ alternate write & seek 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.write(source[i:i+1000]) - f.seek(i+2000) - - -# XXX the 2 following tests don't work with py3k's text IO -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_bytewise(f, source): - """ alternate read & write one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.read(1) - f.write(source[i+1:i+2]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_blockwise(f, source): - """ alternate read & write 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.read(1000) - f.write(source[i+1000:i+2000]) - - -read_tests = [ - read_bytewise, read_small_chunks, read_lines, read_big_chunks, - None, read_whole_file, None, - seek_forward_bytewise, seek_forward_blockwise, - read_seek_bytewise, read_seek_blockwise, -] - -write_tests = [ - write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks, -] - -modify_tests = [ - modify_bytewise, modify_small_chunks, modify_medium_chunks, - None, - modify_seek_forward_bytewise, modify_seek_forward_blockwise, - read_modify_bytewise, read_modify_blockwise, -] - - -def run_during(duration, func): - _t = time.time - n = 0 - start = os.times() - start_timestamp = _t() - real_start = start[4] or start_timestamp - while True: - func() - n += 1 - if _t() - start_timestamp > duration: - break - end = os.times() - real = (end[4] if start[4] else time.time()) - real_start - return n, real, sum(end[0:2]) - sum(start[0:2]) - - -def warm_cache(filename): - with open(filename, "rb") as f: - f.read() - - -def run_all_tests(options): - def print_label(filename, func): - name = re.split(r'[-.]', filename)[0] - out.write( - f"[{name.center(7)}] {func.__doc__.strip()}... ".ljust(52)) - out.flush() - - def print_results(size, n, real, cpu): - bw = n * float(size) / 1024 ** 2 / real - bw = ("%4d MiB/s" if bw > 100 else "%.3g MiB/s") % bw - out.write(bw.rjust(12) + "\n") - if cpu < 0.90 * real: - out.write(" warning: test above used only " - f"{cpu / real:%} CPU, " - "result may be flawed!\n") - - def run_one_test(name, size, open_func, test_func, *args): - mode = test_func.file_open_mode - print_label(name, test_func) - if "w" not in mode or "+" in mode: - warm_cache(name) - with open_func(name) as f: - n, real, cpu = run_during(1.5, lambda: test_func(f, *args)) - print_results(size, n, real, cpu) - - def run_test_family(tests, mode_filter, files, open_func, *make_args): - for test_func in tests: - if test_func is None: - out.write("\n") - continue - if mode_filter in test_func.file_open_mode: - continue - for s in test_func.file_sizes: - name, size = files[size_names[s]] - #name += file_ext - args = tuple(f(name, size) for f in make_args) - run_one_test(name, size, - open_func, test_func, *args) - - size_names = { - "small": 0, - "medium": 1, - "large": 2, - } - - print(f"Python {sys.version}") - print("Unicode: PEP 393") - print(platform.platform()) - binary_files = list(get_binary_files()) - text_files = list(get_text_files()) - if "b" in options: - print("Binary unit = one byte") - if "t" in options: - print(f"Text unit = one character ({TEXT_ENCODING}-decoded)") - - # Binary reads - if "b" in options and "r" in options: - print("\n** Binary input **\n") - run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb")) - - # Text reads - if "t" in options and "r" in options: - print("\n** Text input **\n") - run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r")) - - # Binary writes - if "b" in options and "w" in options: - print("\n** Binary append **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(write_tests, "t", binary_files, - lambda fn: open(os.devnull, "wb"), make_test_source) - - # Text writes - if "t" in options and "w" in options: - print("\n** Text append **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(write_tests, "b", text_files, - lambda fn: text_open(os.devnull, "w"), make_test_source) - - # Binary overwrites - if "b" in options and "w" in options: - print("\n** Binary overwrite **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(modify_tests, "t", binary_files, - lambda fn: open(fn, "r+b"), make_test_source) - - # Text overwrites - if "t" in options and "w" in options: - print("\n** Text overwrite **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(modify_tests, "b", text_files, - lambda fn: text_open(fn, "r+"), make_test_source) - - -def prepare_files(): - print("Preparing files...") - # Binary files - for name, size in get_binary_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - with open(name, "wb") as f: - f.write(os.urandom(size)) - # Text files - chunk = [] - with text_open(__file__, "r", encoding='utf8') as f: - for line in f: - if line.startswith("# "): - break - else: - raise RuntimeError( - f"Couldn't find chunk marker in {__file__} !") - if NEWLINES == "all": - it = itertools.cycle(["\n", "\r", "\r\n"]) - else: - it = itertools.repeat( - {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES]) - chunk = "".join(line.replace("\n", next(it)) for line in f) - if isinstance(chunk, bytes): - chunk = chunk.decode('utf8') - chunk = chunk.encode(TEXT_ENCODING) - for name, size in get_text_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - head = chunk * (size // len(chunk)) - tail = chunk[:size % len(chunk)] - # Adjust tail to end on a character boundary - while True: - try: - tail.decode(TEXT_ENCODING) - break - except UnicodeDecodeError: - tail = tail[:-1] - with open(name, "wb") as f: - f.write(head) - f.write(tail) - - -def main(): - global TEXT_ENCODING, NEWLINES - - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-b", "--binary", - action="store_true", dest="binary", default=False, - help="run binary I/O tests") - parser.add_option("-t", "--text", - action="store_true", dest="text", default=False, - help="run text I/O tests") - parser.add_option("-r", "--read", - action="store_true", dest="read", default=False, - help="run read tests") - parser.add_option("-w", "--write", - action="store_true", dest="write", default=False, - help="run write & modify tests") - parser.add_option("-E", "--encoding", - action="store", dest="encoding", default=None, - help=f"encoding for text tests (default: {TEXT_ENCODING})") - parser.add_option("-N", "--newlines", - action="store", dest="newlines", default='lf', - help="line endings for text tests " - "(one of: {lf (default), cr, crlf, all})") - parser.add_option("-m", "--io-module", - action="store", dest="io_module", default=None, - help="io module to test (default: builtin open())") - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - NEWLINES = options.newlines.lower() - if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): - parser.error(f"invalid 'newlines' option: {NEWLINES!r}") - - test_options = "" - if options.read: - test_options += "r" - if options.write: - test_options += "w" - elif not options.read: - test_options += "rw" - if options.text: - test_options += "t" - if options.binary: - test_options += "b" - elif not options.text: - test_options += "tb" - - if options.encoding: - TEXT_ENCODING = options.encoding - - if options.io_module: - globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open - - prepare_files() - run_all_tests(test_options) - - -if __name__ == "__main__": - main() - - -# -- This part to exercise text reading. Don't change anything! -- -# - -""" -1. -Gáttir allar, -áðr gangi fram, -um skoðask skyli, -um skyggnast skyli, -því at óvíst er at vita, -hvar óvinir -sitja á fleti fyrir. - -2. -Gefendr heilir! -Gestr er inn kominn, -hvar skal sitja sjá? -Mjök er bráðr, -sá er á bröndum skal -síns of freista frama. - -3. -Elds er þörf, -þeims inn er kominn -ok á kné kalinn; -matar ok váða -er manni þörf, -þeim er hefr um fjall farit. - -4. -Vatns er þörf, -þeim er til verðar kemr, -þerru ok þjóðlaðar, -góðs of æðis, -ef sér geta mætti, -orðs ok endrþögu. - -5. -Vits er þörf, -þeim er víða ratar; -dælt er heima hvat; -at augabragði verðr, -sá er ekki kann -ok með snotrum sitr. - -6. -At hyggjandi sinni -skyli-t maðr hræsinn vera, -heldr gætinn at geði; -þá er horskr ok þögull -kemr heimisgarða til, -sjaldan verðr víti vörum, -því at óbrigðra vin -fær maðr aldregi -en mannvit mikit. - -7. -Inn vari gestr, -er til verðar kemr, -þunnu hljóði þegir, -eyrum hlýðir, -en augum skoðar; -svá nýsisk fróðra hverr fyrir. - -8. -Hinn er sæll, -er sér of getr -lof ok líknstafi; -ódælla er við þat, -er maðr eiga skal -annars brjóstum í. -""" - -""" -C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir. -Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous. -Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue. -Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit. -""" diff --git a/Tools/jit/README.md b/Tools/jit/README.md new file mode 100644 index 00000000000000..73d2deebbbc216 --- /dev/null +++ b/Tools/jit/README.md @@ -0,0 +1,62 @@ +The JIT Compiler +================ + +This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. + +## Installing LLVM + +The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). + +LLVM version 18 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-18`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +It's easy to install all of the required tools: + +### Linux + +Install LLVM 18 on Ubuntu/Debian: + +```sh +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 18 +``` + +Install LLVM 18 on Fedora Linux 40 or newer: + +```sh +sudo dnf install 'clang(major) = 18' 'llvm(major) = 18' +``` + +### macOS + +Install LLVM 18 with [Homebrew](https://brew.sh): + +```sh +brew install llvm@18 +``` + +Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. + +### Windows + +Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +Alternatively, you can use [chocolatey](https://chocolatey.org): + +```sh +choco install llvm --version=18.1.6 +``` + +### Dev Containers + +If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. + +## Building + +For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`. + +For all other builds, pass the new `--enable-experimental-jit` option to `configure`. + +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py new file mode 100644 index 00000000000000..606f280a14d974 --- /dev/null +++ b/Tools/jit/_llvm.py @@ -0,0 +1,100 @@ +"""Utilities for invoking LLVM tools.""" + +import asyncio +import functools +import os +import re +import shlex +import subprocess +import typing + +_LLVM_VERSION = 18 +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") + +_P = typing.ParamSpec("_P") +_R = typing.TypeVar("_R") +_C = typing.Callable[_P, typing.Awaitable[_R]] + + +def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: + cache = {} + lock = asyncio.Lock() + + @functools.wraps(f) + async def wrapper( + *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member + ) -> _R: + async with lock: + if args not in cache: + cache[args] = await f(*args, **kwargs) + return cache[args] + + return wrapper + + +_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + + +async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: + command = [tool, *args] + async with _CORES: + if echo: + print(shlex.join(command)) + try: + process = await asyncio.create_subprocess_exec( + *command, stdout=subprocess.PIPE + ) + except FileNotFoundError: + return None + out, _ = await process.communicate() + if process.returncode: + raise RuntimeError(f"{tool} exited with return code {process.returncode}") + return out.decode() + + +@_async_cache +async def _check_tool_version(name: str, *, echo: bool = False) -> bool: + output = await _run(name, ["--version"], echo=echo) + return bool(output and _LLVM_VERSION_PATTERN.search(output)) + + +@_async_cache +async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) + return output and output.removesuffix("\n") + + +@_async_cache +async def _find_tool(tool: str, *, echo: bool = False) -> str | None: + # Unversioned executables: + path = tool + if await _check_tool_version(path, echo=echo): + return path + # Versioned executables: + path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # Homebrew-installed executables: + prefix = await _get_brew_llvm_prefix(echo=echo) + if prefix is not None: + path = os.path.join(prefix, "bin", tool) + if await _check_tool_version(path, echo=echo): + return path + # Nothing found: + return None + + +async def maybe_run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str | None: + """Run an LLVM tool if it can be found. Otherwise, return None.""" + path = await _find_tool(tool, echo=echo) + return path and await _run(path, args, echo=echo) + + +async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: + """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" + output = await maybe_run(tool, args, echo=echo) + if output is None: + raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + return output diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py new file mode 100644 index 00000000000000..228fc389584dd7 --- /dev/null +++ b/Tools/jit/_schema.py @@ -0,0 +1,120 @@ +"""Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" + +import typing + +HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", + "ARM64_RELOC_GOT_LOAD_PAGE21", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_PAGE21", + "ARM64_RELOC_PAGEOFF12", + "ARM64_RELOC_UNSIGNED", + "IMAGE_REL_AMD64_REL32", + "IMAGE_REL_ARM64_BRANCH26", + "IMAGE_REL_ARM64_PAGEBASE_REL21", + "IMAGE_REL_ARM64_PAGEOFFSET_12A", + "IMAGE_REL_ARM64_PAGEOFFSET_12L", + "IMAGE_REL_I386_DIR32", + "IMAGE_REL_I386_REL32", + "R_AARCH64_ABS64", + "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", + "R_AARCH64_CALL26", + "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", + "R_AARCH64_LD64_GOT_LO12_NC", + "R_AARCH64_MOVW_UABS_G0_NC", + "R_AARCH64_MOVW_UABS_G1_NC", + "R_AARCH64_MOVW_UABS_G2_NC", + "R_AARCH64_MOVW_UABS_G3", + "R_X86_64_64", + "R_X86_64_GOTPCREL", + "R_X86_64_GOTPCRELX", + "R_X86_64_PC32", + "R_X86_64_REX_GOTPCRELX", + "X86_64_RELOC_BRANCH", + "X86_64_RELOC_GOT", + "X86_64_RELOC_GOT_LOAD", + "X86_64_RELOC_SIGNED", + "X86_64_RELOC_UNSIGNED", +] + + +class COFFRelocation(typing.TypedDict): + """A COFF object file relocation record.""" + + Type: dict[typing.Literal["Value"], HoleKind] + Symbol: str + Offset: int + + +class ELFRelocation(typing.TypedDict): + """An ELF object file relocation record.""" + + Addend: int + Offset: int + Symbol: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Value"], HoleKind] + + +class MachORelocation(typing.TypedDict): + """A Mach-O object file relocation record.""" + + Offset: int + Section: typing.NotRequired[dict[typing.Literal["Value"], str]] + Symbol: typing.NotRequired[dict[typing.Literal["Value"], str]] + Type: dict[typing.Literal["Value"], HoleKind] + + +class _COFFSymbol(typing.TypedDict): + Name: str + Value: int + + +class _ELFSymbol(typing.TypedDict): + Name: dict[typing.Literal["Name"], str] + Value: int + + +class _MachOSymbol(typing.TypedDict): + Name: dict[typing.Literal["Name"], str] + Value: int + + +class COFFSection(typing.TypedDict): + """A COFF object file section.""" + + Characteristics: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] + Number: int + RawDataSize: int + Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: list[dict[typing.Literal["Symbol"], _COFFSymbol]] + + +class ELFSection(typing.TypedDict): + """An ELF object file section.""" + + Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Info: int + Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] + SectionData: dict[typing.Literal["Bytes"], list[int]] + Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] + Type: dict[typing.Literal["Name"], str] + + +class MachOSection(typing.TypedDict): + """A Mach-O object file section.""" + + Address: int + Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Index: int + Name: dict[typing.Literal["Value"], str] + Relocations: typing.NotRequired[ + list[dict[typing.Literal["Relocation"], MachORelocation]] + ] + SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] + Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py new file mode 100644 index 00000000000000..1c6a9edb39840d --- /dev/null +++ b/Tools/jit/_stencils.py @@ -0,0 +1,375 @@ +"""Core data structures for compiled code templates.""" + +import dataclasses +import enum +import sys +import typing + +import _schema + + +@enum.unique +class HoleValue(enum.Enum): + """ + Different "base" values that can be patched into holes (usually combined with the + address of a symbol and/or an addend). + """ + + # The base address of the machine code for the current uop (exposed as _JIT_ENTRY): + CODE = enum.auto() + # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE): + CONTINUE = enum.auto() + # The base address of the read-only data for this uop: + DATA = enum.auto() + # The address of the current executor (exposed as _JIT_EXECUTOR): + EXECUTOR = enum.auto() + # The base address of the "global" offset table located in the read-only data. + # Shouldn't be present in the final stencils, since these are all replaced with + # equivalent DATA values: + GOT = enum.auto() + # The current uop's oparg (exposed as _JIT_OPARG): + OPARG = enum.auto() + # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): + OPERAND = enum.auto() + # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): + OPERAND_HI = enum.auto() + OPERAND_LO = enum.auto() + # The current uop's target (exposed as _JIT_TARGET): + TARGET = enum.auto() + # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): + JUMP_TARGET = enum.auto() + # The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET): + ERROR_TARGET = enum.auto() + # A hardcoded value of zero (used for symbol lookups): + ZERO = enum.auto() + + +# Map relocation types to our JIT's patch functions. "r" suffixes indicate that +# the patch function is relative. "x" suffixes indicate that they are "relaxing" +# (see comments in jit.c for more info): +_PATCH_FUNCS = { + # aarch64-apple-darwin: + "ARM64_RELOC_BRANCH26": "patch_aarch64_26r", + "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21rx", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12x", + "ARM64_RELOC_PAGE21": "patch_aarch64_21r", + "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", + "ARM64_RELOC_UNSIGNED": "patch_64", + # x86_64-pc-windows-msvc: + "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", + # aarch64-pc-windows-msvc: + "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", + "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", + "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", + "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12x", + # i686-pc-windows-msvc: + "IMAGE_REL_I386_DIR32": "patch_32", + "IMAGE_REL_I386_REL32": "patch_x86_64_32rx", + # aarch64-unknown-linux-gnu: + "R_AARCH64_ABS64": "patch_64", + "R_AARCH64_ADD_ABS_LO12_NC": "patch_aarch64_12", + "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", + "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", + "R_AARCH64_CALL26": "patch_aarch64_26r", + "R_AARCH64_JUMP26": "patch_aarch64_26r", + "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", + "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", + "R_AARCH64_MOVW_UABS_G1_NC": "patch_aarch64_16b", + "R_AARCH64_MOVW_UABS_G2_NC": "patch_aarch64_16c", + "R_AARCH64_MOVW_UABS_G3": "patch_aarch64_16d", + # x86_64-unknown-linux-gnu: + "R_X86_64_64": "patch_64", + "R_X86_64_GOTPCREL": "patch_32r", + "R_X86_64_GOTPCRELX": "patch_x86_64_32rx", + "R_X86_64_PC32": "patch_32r", + "R_X86_64_REX_GOTPCRELX": "patch_x86_64_32rx", + # x86_64-apple-darwin: + "X86_64_RELOC_BRANCH": "patch_32r", + "X86_64_RELOC_GOT": "patch_x86_64_32rx", + "X86_64_RELOC_GOT_LOAD": "patch_x86_64_32rx", + "X86_64_RELOC_SIGNED": "patch_32r", + "X86_64_RELOC_UNSIGNED": "patch_64", +} +# Translate HoleValues to C expressions: +_HOLE_EXPRS = { + HoleValue.CODE: "(uintptr_t)code", + HoleValue.CONTINUE: "(uintptr_t)code + sizeof(code_body)", + HoleValue.DATA: "(uintptr_t)data", + HoleValue.EXECUTOR: "(uintptr_t)executor", + # These should all have been turned into DATA values by process_relocations: + # HoleValue.GOT: "", + HoleValue.OPARG: "instruction->oparg", + HoleValue.OPERAND: "instruction->operand", + HoleValue.OPERAND_HI: "(instruction->operand >> 32)", + HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.TARGET: "instruction->target", + HoleValue.JUMP_TARGET: "instruction_starts[instruction->jump_target]", + HoleValue.ERROR_TARGET: "instruction_starts[instruction->error_target]", + HoleValue.ZERO: "", +} + + +@dataclasses.dataclass +class Hole: + """ + A "hole" in the stencil to be patched with a computed runtime value. + + Analogous to relocation records in an object file. + """ + + offset: int + kind: _schema.HoleKind + # Patch with this base value: + value: HoleValue + # ...plus the address of this symbol: + symbol: str | None + # ...plus this addend: + addend: int + func: str = dataclasses.field(init=False) + # Convenience method: + replace = dataclasses.replace + + def __post_init__(self) -> None: + self.func = _PATCH_FUNCS[self.kind] + + def fold(self, other: typing.Self) -> typing.Self | None: + """Combine two holes into a single hole, if possible.""" + if ( + self.offset + 4 == other.offset + and self.value == other.value + and self.symbol == other.symbol + and self.addend == other.addend + and self.func == "patch_aarch64_21rx" + and other.func == "patch_aarch64_12x" + ): + # These can *only* be properly relaxed when they appear together and + # patch the same value: + folded = self.replace() + folded.func = "patch_aarch64_33rx" + return folded + return None + + def as_c(self, where: str) -> str: + """Dump this hole as a call to a patch_* function.""" + location = f"{where} + {self.offset:#x}" + value = _HOLE_EXPRS[self.value] + if self.symbol: + if value: + value += " + " + value += f"(uintptr_t)&{self.symbol}" + if _signed(self.addend): + if value: + value += " + " + value += f"{_signed(self.addend):#x}" + return f"{self.func}({location}, {value});" + + +@dataclasses.dataclass +class Stencil: + """ + A contiguous block of machine code or data to be copied-and-patched. + + Analogous to a section or segment in an object file. + """ + + body: bytearray = dataclasses.field(default_factory=bytearray, init=False) + holes: list[Hole] = dataclasses.field(default_factory=list, init=False) + disassembly: list[str] = dataclasses.field(default_factory=list, init=False) + trampolines: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + + def pad(self, alignment: int) -> None: + """Pad the stencil to the given alignment.""" + offset = len(self.body) + padding = -offset % alignment + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + self.body.extend([0] * padding) + + def emit_aarch64_trampoline(self, hole: Hole, alignment: int) -> Hole: + """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" + assert hole.symbol is not None + reuse_trampoline = hole.symbol in self.trampolines + if reuse_trampoline: + # Re-use the base address of the previously created trampoline + base = self.trampolines[hole.symbol] + else: + self.pad(alignment) + base = len(self.body) + new_hole = hole.replace(addend=base, symbol=None, value=HoleValue.DATA) + + if reuse_trampoline: + return new_hole + + self.disassembly += [ + f"{base + 4 * 0:x}: 58000048 ldr x8, 8", + f"{base + 4 * 1:x}: d61f0100 br x8", + f"{base + 4 * 2:x}: 00000000", + f"{base + 4 * 2:016x}: R_AARCH64_ABS64 {hole.symbol}", + f"{base + 4 * 3:x}: 00000000", + ] + for code in [ + 0x58000048.to_bytes(4, sys.byteorder), + 0xD61F0100.to_bytes(4, sys.byteorder), + 0x00000000.to_bytes(4, sys.byteorder), + 0x00000000.to_bytes(4, sys.byteorder), + ]: + self.body.extend(code) + self.holes.append(hole.replace(offset=base + 8, kind="R_AARCH64_ABS64")) + self.trampolines[hole.symbol] = base + return new_hole + + def remove_jump(self, *, alignment: int = 1) -> None: + """Remove a zero-length continuation jump, if it exists.""" + hole = max(self.holes, key=lambda hole: hole.offset) + match hole: + case Hole( + offset=offset, + kind="IMAGE_REL_AMD64_REL32", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=-4, + ) as hole: + # jmp qword ptr [rip] + jump = b"\x48\xFF\x25\x00\x00\x00\x00" + offset -= 3 + case Hole( + offset=offset, + kind="IMAGE_REL_I386_REL32" | "X86_64_RELOC_BRANCH", + value=HoleValue.CONTINUE, + symbol=None, + addend=-4, + ) as hole: + # jmp 5 + jump = b"\xE9\x00\x00\x00\x00" + offset -= 1 + case Hole( + offset=offset, + kind="R_AARCH64_JUMP26", + value=HoleValue.CONTINUE, + symbol=None, + addend=0, + ) as hole: + # b #4 + jump = b"\x00\x00\x00\x14" + case Hole( + offset=offset, + kind="R_X86_64_GOTPCRELX", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=addend, + ) as hole: + assert _signed(addend) == -4 + # jmp qword ptr [rip] + jump = b"\xFF\x25\x00\x00\x00\x00" + offset -= 2 + case _: + return + if self.body[offset:] == jump and offset % alignment == 0: + self.body = self.body[:offset] + self.holes.remove(hole) + + +@dataclasses.dataclass +class StencilGroup: + """ + Code and data corresponding to a given micro-opcode. + + Analogous to an entire object file. + """ + + code: Stencil = dataclasses.field(default_factory=Stencil, init=False) + data: Stencil = dataclasses.field(default_factory=Stencil, init=False) + symbols: dict[int | str, tuple[HoleValue, int]] = dataclasses.field( + default_factory=dict, init=False + ) + _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) + + def process_relocations(self, *, alignment: int = 1) -> None: + """Fix up all GOT and internal relocations for this stencil group.""" + for hole in self.code.holes.copy(): + if ( + hole.kind + in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} + and hole.value is HoleValue.ZERO + ): + new_hole = self.data.emit_aarch64_trampoline(hole, alignment) + self.code.holes.remove(hole) + self.code.holes.append(new_hole) + self.code.remove_jump(alignment=alignment) + self.code.pad(alignment) + self.data.pad(8) + for stencil in [self.code, self.data]: + for hole in stencil.holes: + if hole.value is HoleValue.GOT: + assert hole.symbol is not None + hole.value = HoleValue.DATA + hole.addend += self._global_offset_table_lookup(hole.symbol) + hole.symbol = None + elif hole.symbol in self.symbols: + hole.value, addend = self.symbols[hole.symbol] + hole.addend += addend + hole.symbol = None + elif ( + hole.kind in {"IMAGE_REL_AMD64_REL32"} + and hole.value is HoleValue.ZERO + ): + raise ValueError( + f"Add PyAPI_FUNC(...) or PyAPI_DATA(...) to declaration of {hole.symbol}!" + ) + self._emit_global_offset_table() + self.code.holes.sort(key=lambda hole: hole.offset) + self.data.holes.sort(key=lambda hole: hole.offset) + + def _global_offset_table_lookup(self, symbol: str) -> int: + return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) + + def _emit_global_offset_table(self) -> None: + got = len(self.data.body) + for s, offset in self._got.items(): + if s in self.symbols: + value, addend = self.symbols[s] + symbol = None + else: + value, symbol = symbol_to_value(s) + addend = 0 + self.data.holes.append( + Hole(got + offset, "R_X86_64_64", value, symbol, addend) + ) + value_part = value.name if value is not HoleValue.ZERO else "" + if value_part and not symbol and not addend: + addend_part = "" + else: + signed = "+" if symbol is not None else "" + addend_part = f"&{symbol}" if symbol else "" + addend_part += f"{_signed(addend):{signed}#x}" + if value_part: + value_part += "+" + self.data.disassembly.append( + f"{len(self.data.body):x}: {value_part}{addend_part}" + ) + self.data.body.extend([0] * 8) + + def as_c(self, opname: str) -> str: + """Dump this hole as a StencilGroup initializer.""" + return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}}}" + + +def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: + """ + Convert a symbol name to a HoleValue and a symbol name. + + Some symbols (starting with "_JIT_") are special and are converted to their + own HoleValues. + """ + if symbol.startswith("_JIT_"): + try: + return HoleValue[symbol.removeprefix("_JIT_")], None + except KeyError: + pass + return HoleValue.ZERO, symbol + + +def _signed(value: int) -> int: + value %= 1 << 64 + if value & (1 << 63): + value -= 1 << 64 + return value diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py new file mode 100644 index 00000000000000..5604c429bcf8ad --- /dev/null +++ b/Tools/jit/_targets.py @@ -0,0 +1,525 @@ +"""Target-specific code generation, parsing, and processing.""" + +import asyncio +import dataclasses +import hashlib +import json +import os +import pathlib +import re +import sys +import tempfile +import typing + +import _llvm +import _schema +import _stencils +import _writer + +if sys.version_info < (3, 11): + raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") + +TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() +TOOLS_JIT = TOOLS_JIT_BUILD.parent +TOOLS = TOOLS_JIT.parent +CPYTHON = TOOLS.parent +PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" +TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" + + +_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) +_R = typing.TypeVar( + "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation +) + + +@dataclasses.dataclass +class _Target(typing.Generic[_S, _R]): + triple: str + _: dataclasses.KW_ONLY + alignment: int = 1 + args: typing.Sequence[str] = () + ghccc: bool = False + prefix: str = "" + stable: bool = False + debug: bool = False + verbose: bool = False + + def _compute_digest(self, out: pathlib.Path) -> str: + hasher = hashlib.sha256() + hasher.update(self.triple.encode()) + hasher.update(self.debug.to_bytes()) + # These dependencies are also reflected in _JITSources in regen.targets: + hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) + hasher.update((out / "pyconfig.h").read_bytes()) + for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): + for filename in filenames: + hasher.update(pathlib.Path(dirpath, filename).read_bytes()) + return hasher.hexdigest() + + async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: + group = _stencils.StencilGroup() + args = ["--disassemble", "--reloc", f"{path}"] + output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + if output is not None: + group.code.disassembly.extend( + line.expandtabs().strip() + for line in output.splitlines() + if not line.isspace() + ) + args = [ + "--elf-output-style=JSON", + "--expand-relocs", + # "--pretty-print", + "--section-data", + "--section-relocations", + "--section-symbols", + "--sections", + f"{path}", + ] + output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + # --elf-output-style=JSON is only *slightly* broken on Mach-O... + output = output.replace("PrivateExtern\n", "\n") + output = output.replace("Extern\n", "\n") + # ...and also COFF: + output = output[output.index("[", 1, None) :] + output = output[: output.rindex("]", None, -1) + 1] + sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) + for wrapped_section in sections: + self._handle_section(wrapped_section["Section"], group) + # The trampoline's entry point is just named "_ENTRY", since on some + # platforms we later assume that any function starting with "_JIT_" uses + # the GHC calling convention: + entry_symbol = "_JIT_ENTRY" if "_JIT_ENTRY" in group.symbols else "_ENTRY" + assert group.symbols[entry_symbol] == (_stencils.HoleValue.CODE, 0) + if group.data.body: + line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" + group.data.disassembly.append(line) + group.process_relocations(alignment=self.alignment) + return group + + def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: + raise NotImplementedError(type(self)) + + def _handle_relocation( + self, base: int, relocation: _R, raw: bytes + ) -> _stencils.Hole: + raise NotImplementedError(type(self)) + + async def _compile( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + # "Compile" the trampoline to an empty stencil group if it's not needed: + if opname == "trampoline" and not self.ghccc: + return _stencils.StencilGroup() + o = tempdir / f"{opname}.o" + args = [ + f"--target={self.triple}", + "-DPy_BUILD_CORE_MODULE", + "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-D_JIT_OPCODE={opname}", + "-D_PyJIT_ACTIVE", + "-D_Py_JIT", + "-I.", + f"-I{CPYTHON / 'Include'}", + f"-I{CPYTHON / 'Include' / 'internal'}", + f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", + f"-I{CPYTHON / 'Python'}", + "-O3", + "-c", + # This debug info isn't necessary, and bloats out the JIT'ed code. + # We *may* be able to re-enable this, process it, and JIT it for a + # nicer debugging experience... but that needs a lot more research: + "-fno-asynchronous-unwind-tables", + # Don't call built-in functions that we can't find or patch: + "-fno-builtin", + # Emit relaxable 64-bit calls/jumps, so we don't have to worry about + # about emitting in-range trampolines for out-of-range targets. + # We can probably remove this and emit trampolines in the future: + "-fno-plt", + # Don't call stack-smashing canaries that we can't find or patch: + "-fno-stack-protector", + "-std=c11", + *self.args, + ] + if self.ghccc: + # This is a bit of an ugly workaround, but it makes the code much + # smaller and faster, so it's worth it. We want to use the GHC + # calling convention, but Clang doesn't support it. So, we *first* + # compile the code to LLVM IR, perform some text replacements on the + # IR to change the calling convention(!), and then compile *that*. + # Once we have access to Clang 19, we can get rid of this and use + # __attribute__((preserve_none)) directly in the C code instead: + ll = tempdir / f"{opname}.ll" + args_ll = args + [ + # -fomit-frame-pointer is necessary because the GHC calling + # convention uses RBP to pass arguments: + "-S", + "-emit-llvm", + "-fomit-frame-pointer", + "-o", + f"{ll}", + f"{c}", + ] + await _llvm.run("clang", args_ll, echo=self.verbose) + ir = ll.read_text() + # This handles declarations, definitions, and calls to named symbols + # starting with "_JIT_": + ir = re.sub( + r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir + ) + # This handles calls to anonymous callees, since anything with + # "musttail" needs to use the same calling convention: + ir = ir.replace("musttail call", "musttail call ghccc") + # Sometimes *both* replacements happen at the same site, so fix it: + ir = ir.replace("ghccc ghccc", "ghccc") + ll.write_text(ir) + args_o = args + ["-Wno-unused-command-line-argument", "-o", f"{o}", f"{ll}"] + else: + args_o = args + ["-o", f"{o}", f"{c}"] + await _llvm.run("clang", args_o, echo=self.verbose) + return await self._parse(o) + + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: + generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() + opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) + tasks = [] + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + async with asyncio.TaskGroup() as group: + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) + return {task.get_name(): task.result() for task in tasks} + + def build( + self, out: pathlib.Path, *, comment: str = "", force: bool = False + ) -> None: + """Build jit_stencils.h in the given directory.""" + if not self.stable: + warning = f"JIT support for {self.triple} is still experimental!" + request = "Please report any issues you encounter.".center(len(warning)) + outline = "=" * len(warning) + print("\n".join(["", outline, warning, request, outline, ""])) + digest = f"// {self._compute_digest(out)}\n" + jit_stencils = out / "jit_stencils.h" + if ( + not force + and jit_stencils.exists() + and jit_stencils.read_text().startswith(digest) + ): + return + stencil_groups = asyncio.run(self._build_stencils()) + jit_stencils_new = out / "jit_stencils.h.new" + try: + with jit_stencils_new.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("\n") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + jit_stencils_new.replace(jit_stencils) + finally: + jit_stencils_new.unlink(missing_ok=True) + + +class _COFF( + _Target[_schema.COFFSection, _schema.COFFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.COFFSection, group: _stencils.StencilGroup + ) -> None: + flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} + if "SectionData" in section: + section_data_bytes = section["SectionData"]["Bytes"] + else: + # Zeroed BSS data, seen with printf debugging calls: + section_data_bytes = [0] * section["RawDataSize"] + if "IMAGE_SCN_MEM_EXECUTE" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + elif "IMAGE_SCN_MEM_READ" in flags: + value = _stencils.HoleValue.DATA + stencil = group.data + else: + return + base = len(stencil.body) + group.symbols[section["Number"]] = value, base + stencil.body.extend(section_data_bytes) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = base + symbol["Value"] + name = symbol["Name"] + name = name.removeprefix(self.prefix) + if name not in group.symbols: + group.symbols[name] = value, offset + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: + if name.startswith("__imp_"): + name = name.removeprefix("__imp_") + name = name.removeprefix(self.prefix) + return _stencils.HoleValue.GOT, name + name = name.removeprefix(self.prefix) + return _stencils.symbol_to_value(name) + + def _handle_relocation( + self, base: int, relocation: _schema.COFFRelocation, raw: bytes + ) -> _stencils.Hole: + match relocation: + case { + "Offset": offset, + "Symbol": s, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, + }: + offset += base + value, symbol = self._unwrap_dllimport(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") + case { + "Offset": offset, + "Symbol": s, + "Type": { + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + }, + }: + offset += base + value, symbol = self._unwrap_dllimport(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Symbol": s, + "Type": { + "Name": "IMAGE_REL_ARM64_BRANCH26" + | "IMAGE_REL_ARM64_PAGEBASE_REL21" + | "IMAGE_REL_ARM64_PAGEOFFSET_12A" + | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind + }, + }: + offset += base + value, symbol = self._unwrap_dllimport(s) + addend = 0 + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _ELF( + _Target[_schema.ELFSection, _schema.ELFRelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.ELFSection, group: _stencils.StencilGroup + ) -> None: + section_type = section["Type"]["Name"] + flags = {flag["Name"] for flag in section["Flags"]["Flags"]} + if section_type == "SHT_RELA": + assert "SHF_INFO_LINK" in flags, flags + assert not section["Symbols"] + value, base = group.symbols[section["Info"]] + if value is _stencils.HoleValue.CODE: + stencil = group.code + else: + assert value is _stencils.HoleValue.DATA + stencil = group.data + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + elif section_type == "SHT_PROGBITS": + if "SHF_ALLOC" not in flags: + return + if "SHF_EXECINSTR" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + else: + value = _stencils.HoleValue.DATA + stencil = group.data + group.symbols[section["Index"]] = value, len(stencil.body) + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = len(stencil.body) + symbol["Value"] + name = symbol["Name"]["Name"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + stencil.body.extend(section["SectionData"]["Bytes"]) + assert not section["Relocations"] + else: + assert section_type in { + "SHT_GROUP", + "SHT_LLVM_ADDRSIG", + "SHT_NOTE", + "SHT_NULL", + "SHT_STRTAB", + "SHT_SYMTAB", + }, section_type + + def _handle_relocation( + self, base: int, relocation: _schema.ELFRelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Name": s}, + "Type": { + "Name": "R_AARCH64_ADR_GOT_PAGE" + | "R_AARCH64_LD64_GOT_LO12_NC" + | "R_X86_64_GOTPCREL" + | "R_X86_64_GOTPCRELX" + | "R_X86_64_REX_GOTPCRELX" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +class _MachO( + _Target[_schema.MachOSection, _schema.MachORelocation] +): # pylint: disable = too-few-public-methods + def _handle_section( + self, section: _schema.MachOSection, group: _stencils.StencilGroup + ) -> None: + assert section["Address"] >= len(group.code.body) + assert "SectionData" in section + flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} + name = section["Name"]["Value"] + name = name.removeprefix(self.prefix) + if "Debug" in flags: + return + if "SomeInstructions" in flags: + value = _stencils.HoleValue.CODE + stencil = group.code + start_address = 0 + group.symbols[name] = value, section["Address"] - start_address + else: + value = _stencils.HoleValue.DATA + stencil = group.data + start_address = len(group.code.body) + group.symbols[name] = value, len(group.code.body) + base = section["Address"] - start_address + group.symbols[section["Index"]] = value, base + stencil.body.extend( + [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) + ) + stencil.body.extend(section["SectionData"]["Bytes"]) + assert "Symbols" in section + for wrapped_symbol in section["Symbols"]: + symbol = wrapped_symbol["Symbol"] + offset = symbol["Value"] - start_address + name = symbol["Name"]["Name"] + name = name.removeprefix(self.prefix) + group.symbols[name] = value, offset + assert "Relocations" in section + for wrapped_relocation in section["Relocations"]: + relocation = wrapped_relocation["Relocation"] + hole = self._handle_relocation(base, relocation, stencil.body) + stencil.holes.append(hole) + + def _handle_relocation( + self, base: int, relocation: _schema.MachORelocation, raw: bytes + ) -> _stencils.Hole: + symbol: str | None + match relocation: + case { + "Offset": offset, + "Symbol": {"Name": s}, + "Type": { + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" + | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = 0 + case { + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, + } | { + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Name": s}, + "Type": {"Name": kind}, + } | { + "Offset": offset, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = 0 + case _: + raise NotImplementedError(relocation) + return _stencils.Hole(offset, kind, value, symbol, addend) + + +def get_target(host: str) -> _COFF | _ELF | _MachO: + """Build a _Target for the given host "triple" and options.""" + # ghccc currently crashes Clang when combined with musttail on aarch64. :( + target: _COFF | _ELF | _MachO + if re.fullmatch(r"aarch64-apple-darwin.*", host): + target = _MachO(host, alignment=8, prefix="_") + elif re.fullmatch(r"aarch64-pc-windows-msvc", host): + args = ["-fms-runtime-lib=dll"] + target = _COFF(host, alignment=8, args=args) + elif re.fullmatch(r"aarch64-.*-linux-gnu", host): + args = ["-fpic"] + target = _ELF(host, alignment=8, args=args) + elif re.fullmatch(r"i686-pc-windows-msvc", host): + args = ["-DPy_NO_ENABLE_SHARED"] + target = _COFF(host, args=args, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-apple-darwin.*", host): + target = _MachO(host, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-pc-windows-msvc", host): + args = ["-fms-runtime-lib=dll"] + target = _COFF(host, args=args, ghccc=True) + elif re.fullmatch(r"x86_64-.*-linux-gnu", host): + args = ["-fpic"] + target = _ELF(host, args=args, ghccc=True) + else: + raise ValueError(host) + return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py new file mode 100644 index 00000000000000..9d11094f85c7ff --- /dev/null +++ b/Tools/jit/_writer.py @@ -0,0 +1,65 @@ +"""Utilities for writing StencilGroups out to a C header file.""" + +import itertools +import typing + +import _stencils + + +def _dump_footer(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: + yield "typedef struct {" + yield " void (*emit)(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[]);" + yield " size_t code_size;" + yield " size_t data_size;" + yield "} StencilGroup;" + yield "" + yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" + yield "" + yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + for opname, group in sorted(groups.items()): + if opname == "trampoline": + continue + yield f" [{opname}] = {group.as_c(opname)}," + yield "};" + + +def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: + yield "void" + yield f"emit_{opname}(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[])" + yield "{" + for part, stencil in [("code", group.code), ("data", group.data)]: + for line in stencil.disassembly: + yield f" // {line}" + if stencil.body: + yield f" const unsigned char {part}_body[{len(stencil.body)}] = {{" + for i in range(0, len(stencil.body), 8): + row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) + yield f" {row}" + yield " };" + # Data is written first (so relaxations in the code work properly): + for part, stencil in [("data", group.data), ("code", group.code)]: + if stencil.body: + yield f" memcpy({part}, {part}_body, sizeof({part}_body));" + skip = False + stencil.holes.sort(key=lambda hole: hole.offset) + for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): + if skip: + skip = False + continue + if pair and (folded := hole.fold(pair)): + skip = True + hole = folded + yield f" {hole.as_c(part)}" + yield "}" + yield "" + + +def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: + """Yield a JIT compiler line-by-line as a C header file.""" + for opname, group in sorted(groups.items()): + yield from _dump_stencil(opname, group) + yield from _dump_footer(groups) diff --git a/Tools/jit/build.py b/Tools/jit/build.py new file mode 100644 index 00000000000000..4a23c6f0afa74a --- /dev/null +++ b/Tools/jit/build.py @@ -0,0 +1,28 @@ +"""Build an experimental just-in-time compiler for CPython.""" + +import argparse +import pathlib +import shlex +import sys + +import _targets + +if __name__ == "__main__": + comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="compile for a debug build of Python" + ) + parser.add_argument( + "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="echo commands as they are run" + ) + args = parser.parse_args() + args.target.debug = args.debug + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt new file mode 100644 index 00000000000000..9e0f13f4050d79 --- /dev/null +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -0,0 +1,82 @@ +test_multiprocessing_fork +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess +test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling +test.test_cmd_line.CmdLineTest.test_no_std_streams +test.test_cmd_line.CmdLineTest.test_no_stdin +test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers +test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace +test.test_external_inspection.TestGetStackTrace.test_self_trace +test.test_faulthandler.FaultHandlerTests.test_enable_fd +test.test_faulthandler.FaultHandlerTests.test_enable_file +test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_logging.ConfigDictTest.test_111615 +test.test_logging.ConfigDictTest.test_config_queue_handler +test.test_logging.ConfigDictTest.test_multiprocessing_queues +test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists +test.test_os.TimerfdTests.test_timerfd_initval +test.test_os.TimerfdTests.test_timerfd_interval +test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME +test.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.PosixPathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root +test.test_posix.TestPosixSpawn.test_close_file +test.test_posix.TestPosixSpawnP.test_close_file +test.test_posixpath.PosixPathTest.test_ismount +test.test_signal.StressTest.test_stress_modifying_handlers +test.test_socket.BasicCANTest.testFilter +test.test_socket.BasicCANTest.testLoopback +test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm +test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCase.test_empty_env +test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCase.test_one_environment_variable +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env +test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable +test.test_venv.BasicTest.test_zippath_from_non_installed_posix diff --git a/Tools/jit/mypy.ini b/Tools/jit/mypy.ini new file mode 100644 index 00000000000000..768d0028516abd --- /dev/null +++ b/Tools/jit/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +files = Tools/jit +pretty = True +python_version = 3.11 +strict = True diff --git a/Tools/jit/template.c b/Tools/jit/template.c new file mode 100644 index 00000000000000..ec7d033e89deff --- /dev/null +++ b/Tools/jit/template.c @@ -0,0 +1,128 @@ +#include "Python.h" + +#include "pycore_backoff.h" +#include "pycore_call.h" +#include "pycore_ceval.h" +#include "pycore_cell.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_long.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_range.h" +#include "pycore_setobject.h" +#include "pycore_sliceobject.h" +#include "pycore_descrobject.h" +#include "pycore_stackref.h" + +#include "ceval_macros.h" + +#undef CURRENT_OPARG +#define CURRENT_OPARG() (_oparg) + +#undef CURRENT_OPERAND +#define CURRENT_OPERAND() (_operand) + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + do { \ + if ((COND)) { \ + goto deoptimize; \ + } \ + } while (0) + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION (0) + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) \ + do { \ + goto LABEL ## _tier_two; \ + } while (0) + +#undef GOTO_TIER_TWO +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + __attribute__((musttail)) \ + return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ +} while (0) + +#undef GOTO_TIER_ONE +#define GOTO_TIER_ONE(TARGET) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + return TARGET; \ +} while (0) + +#undef LOAD_IP +#define LOAD_IP(UNUSED) \ + do { \ + } while (0) + +#define PATCH_VALUE(TYPE, NAME, ALIAS) \ + PyAPI_DATA(void) ALIAS; \ + TYPE NAME = (TYPE)(uintptr_t)&ALIAS; + +#define PATCH_JUMP(ALIAS) \ +do { \ + PyAPI_DATA(void) ALIAS; \ + __attribute__((musttail)) \ + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ +} while (0) + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) + +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) + +#undef WITHIN_STACK_BOUNDS +#define WITHIN_STACK_BOUNDS() 1 + +_Py_CODEUNIT * +_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) +{ + // Locals that the instruction implementations expect to exist: + PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) + int oparg; + int uopcode = _JIT_OPCODE; + _Py_CODEUNIT *next_instr; + // Other stuff we need handy: + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) +#if SIZEOF_VOID_P == 8 + PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) +#else + assert(SIZEOF_VOID_P == 4); + PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) + PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) + uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; +#endif + PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + + // The actual instruction definitions (only one will be used): + switch (uopcode) { +#include "executor_cases.c.h" + default: + Py_UNREACHABLE(); + } + PATCH_JUMP(_JIT_CONTINUE); + // Labels that the instruction implementations expect to exist: + +error_tier_two: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(NULL); +exit_to_tier1: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); +exit_to_tier1_dynamic: + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(frame->instr_ptr); +} diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c new file mode 100644 index 00000000000000..a0a963f2a49656 --- /dev/null +++ b/Tools/jit/trampoline.c @@ -0,0 +1,25 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_jit.h" + +// This is where the calling convention changes, on platforms that require it. +// The actual change is patched in while the JIT compiler is being built, in +// Tools/jit/_targets.py. On other platforms, this function compiles to nothing. +_Py_CODEUNIT * +_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) +{ + // This is subtle. The actual trace will return to us once it exits, so we + // need to make sure that we stay alive until then. If our trace side-exits + // into another trace, and this trace is then invalidated, the code for + // *this function* will be freed and we'll crash upon return: + PyAPI_DATA(void) _JIT_EXECUTOR; + PyObject *executor = (PyObject *)(uintptr_t)&_JIT_EXECUTOR; + Py_INCREF(executor); + // Note that this is *not* a tail call: + PyAPI_DATA(void) _JIT_CONTINUE; + _Py_CODEUNIT *target = ((jit_func)&_JIT_CONTINUE)(frame, stack_pointer, tstate); + Py_SETREF(tstate->previous_executor, executor); + return target; +} diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index b9aab887c4939b..2fe8a475e7e3a3 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,6 +22,9 @@ if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts if "%~1" EQU "--test-marker" (set BUILDTEST=--test-marker) && shift && goto CheckOpts if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts +rem %IncludeFreethreaded% is recognised by the MSI build, but not the regular build. +rem We use it to build twice and then build the installer with its extra option +if /I "%~1" EQU "--disable-gil" (set IncludeFreethreaded=true) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM64 (set BUILDX86=1) && (set BUILDX64=1) @@ -44,6 +47,20 @@ if errorlevel 1 exit /B %ERRORLEVEL% if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% if errorlevel 1 exit /B %ERRORLEVEL% +if /I "%IncludeFreethreaded%"=="true" ( + rem Cannot "exit /B" inside an if block because %ERRORLEVEL% will be wrong. + rem We just skip everything after the first "errorlevel 1" and then exit after + if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% --disable-gil +) +if errorlevel 1 exit /B %ERRORLEVEL% + if defined BUILDDOC call "%PCBUILD%..\Doc\make.bat" html if errorlevel 1 exit /B %ERRORLEVEL% diff --git a/Tools/msi/bundle/Default.thm b/Tools/msi/bundle/Default.thm index d1b0f5bd9c1d4a..471d37acc33b3d 100644 --- a/Tools/msi/bundle/Default.thm +++ b/Tools/msi/bundle/Default.thm @@ -83,6 +83,7 @@ #(loc.PrecompileLabel) #(loc.Include_symbolsLabel) #(loc.Include_debugLabel) + #(loc.Include_freethreadedLabel) #(loc.CustomLocationLabel) diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 6f8befba3a2523..0014204e89d1bb 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -88,9 +88,11 @@ Select Customize to review current options. Install Python [ShortVersion] for &all users for &all users (requires admin privileges) Use admin privi&leges when installing py.exe + Python Launcher is already installed &Precompile standard library Download debugging &symbols Download debu&g binaries (requires VS 2017 or later) + Download &free-threaded binaries (experimental) [ActionLikeInstallation] Progress [ActionLikeInstalling]: diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 3a17ffbaa0b655..7cddda9b06555d 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -442,6 +442,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { ThemeControlElevates(_theme, ID_INSTALL_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_SIMPLE_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_UPGRADE_BUTTON, elevated); + + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { + LOC_STRING *pLocString = nullptr; + if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.ShortInstallLauncherBlockedLabel)", &pLocString)) && pLocString) { + ThemeSetTextControl(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, pLocString->wzText); + } + } } void Custom1Page_Show() { @@ -456,11 +464,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -718,25 +726,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in DWORD64 /*dw64Version*/, __in BOOTSTRAPPER_RELATED_OPERATION operation ) { - if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation && - (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1) || - CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1))) { - auto hr = LoadAssociateFilesStateFromKey(_engine, fPerMachine ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + // Only check launcher_AllUsers because we'll find the same packages + // twice if we check launcher_JustForMe as well. + if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected existing launcher install"); + + LONGLONG blockedLauncher, detectedLauncher; + if (FAILED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher))) { + blockedLauncher = 0; + } + + // Get the prior DetectedLauncher value so we can see if we've + // detected more than one, and then update the stored variable + // (we use the original value later on via the local). + if (FAILED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher))) { + detectedLauncher = 0; + } + if (!detectedLauncher) { + _engine->SetVariableNumeric(L"DetectedLauncher", 1); + } + + if (blockedLauncher) { + // Nothing else to do, we're already blocking + } + else if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) { + // Found a higher version, so we can't install ours. + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Higher version launcher has been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Launcher will not be installed"); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + else if (detectedLauncher) { + if (!blockedLauncher) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Multiple launcher installs have been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "No launcher will be installed or upgraded until one has been removed."); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } } + else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) { + // Found an older version, so let's run the equivalent as an upgrade + // This overrides "unknown" all users options, but will leave alone + // any that have already been set/detected. + // User can deselect the option to include the launcher, but cannot + // change it from the current per user/machine setting. + LONGLONG includeLauncher, includeLauncherAllUsers; + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } - LONGLONG includeLauncher; - if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - || includeLauncher == -1) { - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + if (includeLauncher < 0) { + _engine->SetVariableNumeric(L"Include_launcher", 1); + } + if (includeLauncherAllUsers < 0) { + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + } else if (includeLauncherAllUsers != fPerMachine ? 1 : 0) { + // Requested AllUsers option is inconsistent, so block + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } - _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } return CheckCanceled() ? IDCANCEL : IDNOACTION; } @@ -784,48 +834,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in LPCWSTR wzPackageId, __in HRESULT hrStatus, __in BOOTSTRAPPER_PACKAGE_STATE state - ) { - if (FAILED(hrStatus)) { - return; - } - - BOOL detectedLauncher = FALSE; - HKEY hkey = HKEY_LOCAL_MACHINE; - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 1); - } - } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); - } - } - - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher != -1) { - detectedLauncher = FALSE; - } - - if (detectedLauncher) { - /* When we detect the current version of the launcher. */ - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"DetectedLauncher", 1); - _engine->SetVariableString(L"Include_launcherState", L"disable"); - _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); - - auto hr = LoadAssociateFilesStateFromKey(_engine, hkey); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); - } - } - } + ) { } virtual STDMETHODIMP_(void) OnDetectComplete(__in HRESULT hrStatus) { @@ -835,19 +844,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } if (SUCCEEDED(hrStatus)) { - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher == -1) { - if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || - (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { - // When installing/downloading, we want to include the launcher - // by default. - _engine->SetVariableNumeric(L"Include_launcher", 1); - } else { - // Any other action, if we didn't detect the MSI then we want to - // keep it excluded - _engine->SetVariableNumeric(L"Include_launcher", 0); - _engine->SetVariableNumeric(L"AssociateFiles", 0); + // Update launcher install states + // If we didn't detect any existing installs, Include_launcher and + // InstallLauncherAllUsers will both be -1, so we will set to their + // defaults and leave the options enabled. + // Otherwise, if we detected an existing install, we disable the + // options so they remain fixed. + // The code in OnDetectRelatedMsiPackage is responsible for figuring + // out whether existing installs are compatible with the settings in + // place during detection. + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) + && blockedLauncher) { + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); + _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); + _engine->SetVariableString(L"Include_launcherState", L"disable"); + } + else { + LONGLONG includeLauncher, includeLauncherAllUsers, associateFiles; + + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } + if (FAILED(BalGetNumericVariable(L"AssociateFiles", &associateFiles))) { + associateFiles = -1; + } + + if (includeLauncherAllUsers < 0) { + includeLauncherAllUsers = 0; + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", includeLauncherAllUsers); + } + + if (includeLauncher < 0) { + if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || + (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { + // When installing/downloading, we include the launcher + // (though downloads should ignore this setting anyway) + _engine->SetVariableNumeric(L"Include_launcher", 1); + } else { + // Any other action, we should have detected an existing + // install (e.g. on remove/modify), so if we didn't, we + // assume it's not selected. + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"AssociateFiles", 0); + } + } + + if (associateFiles < 0) { + auto hr = LoadAssociateFilesStateFromKey( + _engine, + includeLauncherAllUsers ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER + ); + if (FAILED(hr)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + } else if (hr == S_OK) { + associateFiles = 1; + } + _engine->SetVariableNumeric(L"AssociateFiles", associateFiles); } } } @@ -2614,7 +2671,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets index 9c7410fe514d19..cb3effb4434843 100644 --- a/Tools/msi/bundle/bundle.targets +++ b/Tools/msi/bundle/bundle.targets @@ -72,6 +72,7 @@ + diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index 8b12baae31105e..abfeb88784890c 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -28,10 +28,11 @@ - + - + + @@ -82,12 +83,20 @@ + + + + + + + + - + @@ -104,6 +113,9 @@ + + + diff --git a/Tools/msi/bundle/packagegroups/freethreaded.wxs b/Tools/msi/bundle/packagegroups/freethreaded.wxs new file mode 100644 index 00000000000000..121ca34ab66157 --- /dev/null +++ b/Tools/msi/bundle/packagegroups/freethreaded.wxs @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14..080598a0a486ef 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be5..4357dc86d9d356 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/msi/freethreaded/freethreaded.wixproj b/Tools/msi/freethreaded/freethreaded.wixproj new file mode 100644 index 00000000000000..0b4bd055d77977 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wixproj @@ -0,0 +1,20 @@ + + + + {1B4502D5-B627-4F50-ABEA-4CC5A8E88265} + 2.0 + freethreaded + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wxs b/Tools/msi/freethreaded/freethreaded.wxs new file mode 100644 index 00000000000000..063aa28bf09fce --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wxs @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_d.wixproj b/Tools/msi/freethreaded/freethreaded_d.wixproj new file mode 100644 index 00000000000000..e1563d4f907126 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wixproj @@ -0,0 +1,20 @@ + + + + {D3677DCF-098A-4398-9FA5-8E74AC37E0DF} + 2.0 + freethreaded_d + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_d.wxs b/Tools/msi/freethreaded/freethreaded_d.wxs new file mode 100644 index 00000000000000..cddf22a6c803d3 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wxs @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_en-US.wxl_template b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template new file mode 100644 index 00000000000000..b9747eb256d24b --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template @@ -0,0 +1,8 @@ + + + Freethreaded Interpreter + freethreaded + Python {{ShortVersion}} ({{Bitness}}, freethreaded) + Launches the !(loc.ProductName) freethreaded interpreter. + https://www.python.org/ + diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs new file mode 100644 index 00000000000000..49ecb3429ad8f3 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -0,0 +1,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wixproj b/Tools/msi/freethreaded/freethreaded_pdb.wixproj new file mode 100644 index 00000000000000..789a4f55ae5191 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wixproj @@ -0,0 +1,20 @@ + + + + {E98E7539-64E7-4DCE-AACD-01E3ADE40EFD} + 2.0 + freethreaded_pdb + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wxs b/Tools/msi/freethreaded/freethreaded_pdb.wxs new file mode 100644 index 00000000000000..302ac416fe9275 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wxs @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index a82cad596d47a6..b8e16b5fe238a0 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -26,10 +26,10 @@ - + - + @@ -63,10 +63,10 @@ - + - + @@ -100,10 +100,10 @@ - + - + diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index cfb3ca9e76e24c..372c4823bce07f 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -24,14 +24,14 @@ This URI is used to generate the various GUIDs used by the installer. Installers built with the same URI will upgrade each other or block when attempting to downgrade. - + By default, this is the local computer name, which will produce installers that do not interfere with other installers. Products that intend to bundle Python should rebuild these modules with their own URI to avoid conflicting with the official releases. - + The official releases use "https://www.python.org/$(ArchName)" - + This is not the same as the DownloadUrl property used in the bundle projects. --> @@ -39,7 +39,7 @@ $(ReleaseUri)/ - + @@ -63,13 +63,17 @@ $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 - + + + false + + $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 - + 32-bit 64-bit @@ -91,9 +95,12 @@ PyDebugExt=$(PyDebugExt); PyArchExt=$(PyArchExt); PyTestExt=$(PyTestExt); + PydTag=$(PydTag); + FreethreadedPydTag=$(FreethreadedPydTag); OptionalFeatureName=$(OutputName); ssltag=$(OpenSSLDLLSuffix); Suffix32=$(PyArchExt); + IncludeFreethreaded=$(IncludeFreethreaded); $(DefineConstants);CRTRedist=$(CRTRedist); @@ -139,7 +146,7 @@ - + <_Uuid Include="CoreUpgradeCode"> upgradecode @@ -162,6 +169,12 @@ <_Uuid Include="PythonRegComponentGuid"> registry/$(OutputName) + <_Uuid Include="FreethreadedPythonExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/python.exe + + <_Uuid Include="FreethreadedPythonwExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/pythonw.exe + @(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) - + - + - + $(DefineConstants);@(_UuidValue,';'); diff --git a/Tools/msi/pip/pip.wxs b/Tools/msi/pip/pip.wxs index 1d8083cad91a56..627c4710a9fdfa 100644 --- a/Tools/msi/pip/pip.wxs +++ b/Tools/msi/pip/pip.wxs @@ -25,8 +25,8 @@ - - + + (&DefaultFeature=3) AND NOT (!DefaultFeature=3) diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs index bb9b258692a62f..6862a5899db2ba 100644 --- a/Tools/msi/test/test_files.wxs +++ b/Tools/msi/test/test_files.wxs @@ -1,6 +1,6 @@ - + diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index af1f0584bb5403..fc338f389ca6d9 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Check proposed changes for common issues.""" -import re import sys -import shutil import os.path import subprocess import sysconfig diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 262c8a6db68f6e..0b0b4b291c2b0e 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -107,7 +107,10 @@ def generate_python_code( help="Suppress code emission for rule actions", ) -python_parser = subparsers.add_parser("python", help="Generate Python code") +python_parser = subparsers.add_parser( + "python", + help="Generate Python code, needs grammar definition with Python actions", +) python_parser.set_defaults(func=generate_python_code) python_parser.add_argument("grammar_filename", help="Grammar description") python_parser.add_argument( diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 30bfb31471c7b2..67a7c0c4788e9d 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -139,10 +139,15 @@ def compile_c_extension( ] include_dirs = [ str(MOD_DIR.parent.parent.parent / "Include" / "internal"), + str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"), str(MOD_DIR.parent.parent.parent / "Parser"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), ] + if sys.platform == "win32": + # HACK: The location of pyconfig.h has moved within our build, and + # setuptools hasn't updated for it yet. So add the path manually for now + include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent) extension = Extension( extension_name, sources=[generated_source_path], @@ -215,6 +220,9 @@ def compile_c_extension( ) else: objects = compiler.object_filenames(extension.sources, output_dir=cmd.build_temp) + # The cmd.get_libraries() call needs a valid compiler attribute or we will + # get an incorrect library name on the free-threaded Windows build. + cmd.compiler = compiler # Now link the object files together into a "shared object" compiler.link_shared_object( objects, diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 301949bdae96fe..547c55dce130f7 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -38,7 +38,11 @@ #endif #ifdef __wasi__ -# define MAXSTACK 4000 +# ifdef Py_DEBUG +# define MAXSTACK 1000 +# else +# define MAXSTACK 4000 +# endif #else # define MAXSTACK 6000 #endif @@ -208,7 +212,7 @@ def visit_Rhs(self, node: Rhs) -> FunctionCall: if node.can_be_inlined: self.cache[node] = self.generate_call(node.alts[0].items[0]) else: - name = self.gen.artifical_rule_from_rhs(node) + name = self.gen.artificial_rule_from_rhs(node) self.cache[node] = FunctionCall( assigned_variable=f"{name}_var", function=f"{name}_rule", @@ -249,7 +253,7 @@ def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: else: return FunctionCall( function=f"_PyPegen_lookahead", - arguments=[positive, call.function, *call.arguments], + arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], return_type="int", ) @@ -327,7 +331,7 @@ def visit_Repeat1(self, node: Repeat1) -> FunctionCall: def visit_Gather(self, node: Gather) -> FunctionCall: if node in self.cache: return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) + name = self.gen.artificial_rule_from_gather(node) self.cache[node] = FunctionCall( assigned_variable=f"{name}_var", function=f"{name}_rule", @@ -641,7 +645,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.print("}") self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") - self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") + self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py index 82d717b72976e5..52611eae044e58 100644 --- a/Tools/peg_generator/pegen/keywordgen.py +++ b/Tools/peg_generator/pegen/keywordgen.py @@ -41,25 +41,24 @@ def main() -> None: description="Generate the Lib/keywords.py file from the grammar." ) parser.add_argument( - "grammar", type=str, help="The file with the grammar definition in PEG format" + "grammar", help="The file with the grammar definition in PEG format" ) parser.add_argument( - "tokens_file", type=argparse.FileType("r"), help="The file with the token definitions" + "tokens_file", help="The file with the token definitions" ) parser.add_argument( "keyword_file", - type=argparse.FileType("w"), help="The path to write the keyword definitions", ) args = parser.parse_args() grammar, _, _ = build_parser(args.grammar) - with args.tokens_file as tok_file: + with open(args.tokens_file) as tok_file: all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file) gen = CParserGenerator(grammar, all_tokens, exact_tok, non_exact_tok, file=None) gen.collect_rules() - with args.keyword_file as thefile: + with open(args.keyword_file, 'w') as thefile: all_keywords = sorted(list(gen.keywords.keys())) all_soft_keywords = sorted(gen.soft_keywords) diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index ad28f6c27dcb37..8cca7b6c39a5cc 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -41,7 +41,7 @@ class RuleCollectorVisitor(GrammarVisitor): - """Visitor that invokes a provieded callmaker visitor with just the NamedItem nodes""" + """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: self.rulses = rules @@ -167,7 +167,7 @@ def keyword_type(self) -> int: self.keyword_counter += 1 return self.keyword_counter - def artifical_rule_from_rhs(self, rhs: Rhs) -> str: + def artificial_rule_from_rhs(self, rhs: Rhs) -> str: self.counter += 1 name = f"_tmp_{self.counter}" # TODO: Pick a nicer name. self.all_rules[name] = Rule(name, None, rhs) @@ -183,7 +183,7 @@ def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str: self.all_rules[name] = Rule(name, None, Rhs([Alt([NamedItem(None, node)])])) return name - def artifical_rule_from_gather(self, node: Gather) -> str: + def artificial_rule_from_gather(self, node: Gather) -> str: self.counter += 1 name = f"_gather_{self.counter}" self.counter += 1 diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 4a2883eb4ee202..588d3d3f6ef8f8 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -116,7 +116,7 @@ def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: if len(node.alts) == 1 and len(node.alts[0].items) == 1: self.cache[node] = self.visit(node.alts[0].items[0]) else: - name = self.gen.artifical_rule_from_rhs(node) + name = self.gen.artificial_rule_from_rhs(node) self.cache[node] = name, f"self.{name}()" return self.cache[node] @@ -168,7 +168,7 @@ def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: def visit_Gather(self, node: Gather) -> Tuple[str, str]: if node in self.cache: return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) + name = self.gen.artificial_rule_from_gather(node) self.cache[node] = name, f"self.{name}()" # No trailing comma here either! return self.cache[node] diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index 1f0586bb2f7d6d..da4c9331625dd9 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -18,7 +18,7 @@ def strongly_connected_components( exactly once; vertices not part of a SCC are returned as singleton sets. - From http://code.activestate.com/recipes/578507/. + From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ identified: Set[str] = set() stack: List[str] = [] @@ -81,7 +81,7 @@ def topsort( {B, C} {A} - From http://code.activestate.com/recipes/577413/. + From https://code.activestate.com/recipes/577413-topological-sort/history/1/. """ # TODO: Use a faster algorithm? for k, v in data.items(): diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index c48a01eedf5d5c..4699d5712d9522 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -34,6 +34,18 @@ def check_intersection(self, first_alt: Alt, second_alt: Alt) -> None: ) +class RaiseRuleValidator(GrammarValidator): + def visit_Alt(self, node: Alt) -> None: + if self.rulename and self.rulename.startswith('invalid'): + # raising is allowed in invalid rules + return + if node.action and 'RAISE_SYNTAX_ERROR' in node.action: + raise ValidationError( + f"In {self.rulename!r} there is an alternative that contains " + f"RAISE_SYNTAX_ERROR; this is only allowed in invalid_ rules" + ) + + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index add28b1bb38183..de8496a17b85ef 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.5.1 +mypy==1.10.1 # needed for peg_generator: -types-psutil==5.9.5.16 -types-setuptools==68.2.0.0 +types-psutil==6.0.0.20240621 +types-setuptools==70.1.0.20240627 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index b95300a07dd2b4..ab3f39ac6ee087 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.87.1 +hypothesis==6.104.2 diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py new file mode 100644 index 00000000000000..1978a6c83dbe2b --- /dev/null +++ b/Tools/scripts/sortperf.py @@ -0,0 +1,197 @@ +""" +List sort performance test. + +To install `pyperf` you would need to: + + python3 -m pip install pyperf + +To run: + + python3 Tools/scripts/sortperf + +Options: + + * `benchmark` name to run + * `--rnd-seed` to set random seed + * `--size` to set the sorted list size + +Based on https://github.com/python/cpython/blob/963904335e579bfe39101adf3fd6a0cf705975ff/Lib/test/sortperf.py +""" + +from __future__ import annotations + +import argparse +import time +import random + + +# =============== +# Data generation +# =============== + +def _random_data(size: int, rand: random.Random) -> list[float]: + result = [rand.random() for _ in range(size)] + # Shuffle it a bit... + for i in range(10): + i = rand.randrange(size) + temp = result[:i] + del result[:i] + temp.reverse() + result.extend(temp) + del temp + assert len(result) == size + return result + + +def list_sort(size: int, rand: random.Random) -> list[float]: + return _random_data(size, rand) + + +def list_sort_descending(size: int, rand: random.Random) -> list[float]: + return list(reversed(list_sort_ascending(size, rand))) + + +def list_sort_ascending(size: int, rand: random.Random) -> list[float]: + return sorted(_random_data(size, rand)) + + +def list_sort_ascending_exchanged(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Do 3 random exchanges. + for _ in range(3): + i1 = rand.randrange(size) + i2 = rand.randrange(size) + result[i1], result[i2] = result[i2], result[i1] + return result + + +def list_sort_ascending_random(size: int, rand: random.Random) -> list[float]: + assert size >= 10, "This benchmark requires size to be >= 10" + result = list_sort_ascending(size, rand) + # Replace the last 10 with random floats. + result[-10:] = [rand.random() for _ in range(10)] + return result + + +def list_sort_ascending_one_percent(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Replace 1% of the elements at random. + for _ in range(size // 100): + result[rand.randrange(size)] = rand.random() + return result + + +def list_sort_duplicates(size: int, rand: random.Random) -> list[float]: + assert size >= 4 + result = list_sort_ascending(4, rand) + # Arrange for lots of duplicates. + result = result * (size // 4) + # Force the elements to be distinct objects, else timings can be + # artificially low. + return list(map(abs, result)) + + +def list_sort_equal(size: int, rand: random.Random) -> list[float]: + # All equal. Again, force the elements to be distinct objects. + return list(map(abs, [-0.519012] * size)) + + +def list_sort_worst_case(size: int, rand: random.Random) -> list[float]: + # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case + # for an older implementation of quicksort, which used the median + # of the first, last and middle elements as the pivot. + half = size // 2 + result = list(range(half - 1, -1, -1)) + result.extend(range(half)) + # Force to float, so that the timings are comparable. This is + # significantly faster if we leave them as ints. + return list(map(float, result)) + + +# ========= +# Benchmark +# ========= + +class Benchmark: + def __init__(self, name: str, size: int, seed: int) -> None: + self._name = name + self._size = size + self._seed = seed + self._random = random.Random(self._seed) + + def run(self, loops: int) -> float: + all_data = self._prepare_data(loops) + start = time.perf_counter() + + for data in all_data: + data.sort() # Benching this method! + + return time.perf_counter() - start + + def _prepare_data(self, loops: int) -> list[float]: + bench = BENCHMARKS[self._name] + data = bench(self._size, self._random) + return [data.copy() for _ in range(loops)] + + +def add_cmdline_args(cmd: list[str], args) -> None: + if args.benchmark: + cmd.append(args.benchmark) + cmd.append(f"--size={args.size}") + cmd.append(f"--rng-seed={args.rng_seed}") + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "benchmark", + choices=BENCHMARKS, + nargs="?", + help="Can be any of: {0}".format(", ".join(BENCHMARKS)), + ) + parser.add_argument( + "--size", + type=int, + default=DEFAULT_SIZE, + help=f"Size of the lists to sort (default: {DEFAULT_SIZE})", + ) + parser.add_argument( + "--rng-seed", + type=int, + default=DEFAULT_RANDOM_SEED, + help=f"Random number generator seed (default: {DEFAULT_RANDOM_SEED})", + ) + + +DEFAULT_SIZE = 1 << 14 +DEFAULT_RANDOM_SEED = 0 +BENCHMARKS = { + "list_sort": list_sort, + "list_sort_descending": list_sort_descending, + "list_sort_ascending": list_sort_ascending, + "list_sort_ascending_exchanged": list_sort_ascending_exchanged, + "list_sort_ascending_random": list_sort_ascending_random, + "list_sort_ascending_one_percent": list_sort_ascending_one_percent, + "list_sort_duplicates": list_sort_duplicates, + "list_sort_equal": list_sort_equal, + "list_sort_worst_case": list_sort_worst_case, +} + +if __name__ == "__main__": + # This needs `pyperf` 3rd party library: + import pyperf + + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + add_parser_args(runner.argparser) + args = runner.parse_args() + + runner.metadata["description"] = "Test `list.sort()` with different data" + runner.metadata["list_sort_size"] = args.size + runner.metadata["list_sort_random_seed"] = args.rng_seed + + if args.benchmark: + benchmarks = (args.benchmark,) + else: + benchmarks = sorted(BENCHMARKS) + for bench in benchmarks: + benchmark = Benchmark(bench, args.size, args.rng_seed) + runner.bench_time_func(bench, benchmark.run) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index bdca51df3dac53..ffbc40e6a37f3d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -2,910 +2,1412 @@ default stats folders. """ +from __future__ import annotations + # NOTE: Bytecode introspection modules (opcode, dis, etc.) should only -# happen when loading a single dataset. When comparing datasets, it +# be imported when loading a single dataset. When comparing datasets, it # could get it wrong, leading to subtle errors. import argparse import collections -import json -import os.path +from collections.abc import KeysView +from dataclasses import dataclass from datetime import date +import enum +import functools import itertools -import sys +import json +from operator import itemgetter +import os +from pathlib import Path import re +import sys +import textwrap +from typing import Any, Callable, TextIO, TypeAlias + + +RawData: TypeAlias = dict[str, Any] +Rows: TypeAlias = list[tuple] +Columns: TypeAlias = tuple[str, ...] +RowCalculator: TypeAlias = Callable[["Stats"], Rows] + + +# TODO: Check for parity + if os.name == "nt": DEFAULT_DIR = "c:\\temp\\py_stats\\" else: DEFAULT_DIR = "/tmp/py_stats/" + +SOURCE_DIR = Path(__file__).parents[2] + + TOTAL = "specialization.hit", "specialization.miss", "execution_count" -def format_ratio(num, den): - """ - Format a ratio as a percentage. When the denominator is 0, returns the empty - string. - """ - if den == 0: - return "" - else: - return f"{num/den:.01%}" +def pretty(name: str) -> str: + return name.replace("_", " ").lower() -def percentage_to_float(s): - """ - Converts a percentage string to a float. The empty string is returned as 0.0 - """ - if s == "": - return 0.0 - else: - assert s[-1] == "%" - return float(s[:-1]) +def _load_metadata_from_source(): + def get_defines(filepath: Path, prefix: str = "SPEC_FAIL"): + with open(SOURCE_DIR / filepath) as spec_src: + defines = collections.defaultdict(list) + start = "#define " + prefix + "_" + for line in spec_src: + line = line.strip() + if not line.startswith(start): + continue + line = line[len(start) :] + name, val = line.split() + defines[int(val.strip())].append(name.strip()) + return defines + + import opcode + + return { + "_specialized_instructions": [ + op for op in opcode._specialized_opmap.keys() if "__" not in op # type: ignore + ], + "_stats_defines": get_defines( + Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" + ), + "_defines": get_defines(Path("Python") / "specialize.c"), + } + + +def load_raw_data(input: Path) -> RawData: + if input.is_file(): + with open(input, "r") as fd: + data = json.load(fd) + data["_stats_defines"] = {int(k): v for k, v in data["_stats_defines"].items()} + data["_defines"] = {int(k): v for k, v in data["_defines"].items()} -def join_rows(a_rows, b_rows): - """ - Joins two tables together, side-by-side, where the first column in each is a - common key. - """ - if len(a_rows) == 0 and len(b_rows) == 0: - return [] + return data - if len(a_rows): - a_ncols = list(set(len(x) for x in a_rows)) - if len(a_ncols) != 1: - raise ValueError("Table a is ragged") + elif input.is_dir(): + stats = collections.Counter[str]() - if len(b_rows): - b_ncols = list(set(len(x) for x in b_rows)) - if len(b_ncols) != 1: - raise ValueError("Table b is ragged") + for filename in input.iterdir(): + with open(filename) as fd: + for line in fd: + try: + key, value = line.split(":") + except ValueError: + print( + f"Unparsable line: '{line.strip()}' in {filename}", + file=sys.stderr, + ) + continue + # Hack to handle older data files where some uops + # are missing an underscore prefix in their name + if key.startswith("uops[") and key[5:6] != "_": + key = "uops[_" + key[5:] + stats[key.strip()] += int(value) + stats["__nfiles__"] += 1 - if len(a_rows) and len(b_rows) and a_ncols[0] != b_ncols[0]: - raise ValueError("Tables have different widths") + data = dict(stats) + data.update(_load_metadata_from_source()) + return data - if len(a_rows): - ncols = a_ncols[0] else: - ncols = b_ncols[0] + raise ValueError(f"{input} is not a file or directory path") - default = [""] * (ncols - 1) - a_data = {x[0]: x[1:] for x in a_rows} - b_data = {x[0]: x[1:] for x in b_rows} - if len(a_data) != len(a_rows) or len(b_data) != len(b_rows): - raise ValueError("Duplicate keys") +def save_raw_data(data: RawData, json_output: TextIO): + json.dump(data, json_output) - # To preserve ordering, use A's keys as is and then add any in B that aren't - # in A - keys = list(a_data.keys()) + [k for k in b_data.keys() if k not in a_data] - return [(k, *a_data.get(k, default), *b_data.get(k, default)) for k in keys] +@dataclass(frozen=True) +class Doc: + text: str + doc: str -def calculate_specialization_stats(family_stats, total): - rows = [] - for key in sorted(family_stats): - if key.startswith("specialization.failure_kinds"): - continue - if key in ("specialization.hit", "specialization.miss"): - label = key[len("specialization.") :] - elif key == "execution_count": - continue - elif key in ( - "specialization.success", - "specialization.failure", - "specializable", - ): - continue - elif key.startswith("pair"): - continue + def markdown(self) -> str: + return textwrap.dedent( + f""" + {self.text} +
    + ⓘ + + {self.doc} +
    + """ + ) + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +@dataclass(frozen=True) +class Ratio: + num: int + den: int | None = None + percentage: bool = True + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num else: - label = key - rows.append( - ( - f"{label:>12}", - f"{family_stats[key]:>12}", - format_ratio(family_stats[key], total), - ) + return self.num / self.den + + def markdown(self) -> str: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" + + +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) + + +class OpcodeStats: + """ + Manages the data related to specific set of opcodes, e.g. tier1 (with prefix + "opcode") or tier2 (with prefix "uops"). + """ + + def __init__(self, data: dict[str, Any], defines, specialized_instructions): + self._data = data + self._defines = defines + self._specialized_instructions = specialized_instructions + + def get_opcode_names(self) -> KeysView[str]: + return self._data.keys() + + def get_pair_counts(self) -> dict[tuple[str, str], int]: + pair_counts = {} + for name_i, opcode_stat in self._data.items(): + for key, value in opcode_stat.items(): + if value and key.startswith("pair_count"): + name_j, _, _ = key[len("pair_count") + 1 :].partition("]") + pair_counts[(name_i, name_j)] = value + return pair_counts + + def get_total_execution_count(self) -> int: + return sum(x.get("execution_count", 0) for x in self._data.values()) + + def get_execution_counts(self) -> dict[str, tuple[int, int]]: + counts = {} + for name, opcode_stat in self._data.items(): + if "execution_count" in opcode_stat: + count = opcode_stat["execution_count"] + miss = 0 + if "specializable" not in opcode_stat: + miss = opcode_stat.get("specialization.miss", 0) + counts[name] = (count, miss) + return counts + + @functools.cache + def _get_pred_succ( + self, + ) -> tuple[dict[str, collections.Counter], dict[str, collections.Counter]]: + pair_counts = self.get_pair_counts() + + predecessors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter + ) + successors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter ) - return rows + for (first, second), count in pair_counts.items(): + if count: + predecessors[second][first] = count + successors[first][second] = count + + return predecessors, successors + + def get_predecessors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[0][opcode] + + def get_successors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[1][opcode] + def _get_stats_for_opcode(self, opcode: str) -> dict[str, int]: + return self._data[opcode] -def calculate_specialization_success_failure(family_stats): - total_attempts = 0 - for key in ("specialization.success", "specialization.failure"): - total_attempts += family_stats.get(key, 0) - rows = [] - if total_attempts: + def get_specialization_total(self, opcode: str) -> int: + family_stats = self._get_stats_for_opcode(opcode) + return sum(family_stats.get(kind, 0) for kind in TOTAL) + + def get_specialization_counts(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + + result = {} + for key, value in sorted(family_stats.items()): + if key.startswith("specialization."): + label = key[len("specialization.") :] + if label in ("success", "failure") or label.startswith("failure_kinds"): + continue + elif key in ( + "execution_count", + "specializable", + ) or key.startswith("pair"): + continue + else: + label = key + result[label] = value + + return result + + def get_specialization_success_failure(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + result = {} for key in ("specialization.success", "specialization.failure"): label = key[len("specialization.") :] - label = label[0].upper() + label[1:] val = family_stats.get(key, 0) - rows.append((label, val, format_ratio(val, total_attempts))) - return rows - - -def calculate_specialization_failure_kinds(name, family_stats, defines): - total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [0] * 40 - for key in family_stats: - if not key.startswith("specialization.failure_kind"): - continue - _, index = key[:-1].split("[") - index = int(index) - failure_kinds[index] = family_stats[key] - failures = [(value, index) for (index, value) in enumerate(failure_kinds)] - failures.sort(reverse=True) - rows = [] - for value, index in failures: - if not value: - continue - rows.append( - ( - kind_to_text(index, defines, name), - value, - format_ratio(value, total_failures), - ) + result[label] = val + return result + + def get_specialization_failure_total(self, opcode: str) -> int: + return self._get_stats_for_opcode(opcode).get("specialization.failure", 0) + + def get_specialization_failure_kinds(self, opcode: str) -> dict[str, int]: + def kind_to_text(kind: int, opcode: str): + if kind <= 8: + return pretty(self._defines[kind][0]) + if opcode == "LOAD_SUPER_ATTR": + opcode = "SUPER" + elif opcode.endswith("ATTR"): + opcode = "ATTR" + elif opcode in ("FOR_ITER", "SEND"): + opcode = "ITER" + elif opcode.endswith("SUBSCR"): + opcode = "SUBSCR" + for name in self._defines[kind]: + if name.startswith(opcode): + return pretty(name[len(opcode) + 1 :]) + return "kind " + str(kind) + + family_stats = self._get_stats_for_opcode(opcode) + failure_kinds = [0] * 40 + for key in family_stats: + if not key.startswith("specialization.failure_kind"): + continue + index = int(key[:-1].split("[")[1]) + failure_kinds[index] = family_stats[key] + return { + kind_to_text(index, opcode): value + for (index, value) in enumerate(failure_kinds) + if value + } + + def is_specializable(self, opcode: str) -> bool: + return "specializable" in self._get_stats_for_opcode(opcode) + + def get_specialized_total_counts(self) -> tuple[int, int, int]: + basic = 0 + specialized_hits = 0 + specialized_misses = 0 + not_specialized = 0 + for opcode, opcode_stat in self._data.items(): + if "execution_count" not in opcode_stat: + continue + count = opcode_stat["execution_count"] + if "specializable" in opcode_stat: + not_specialized += count + elif opcode in self._specialized_instructions: + miss = opcode_stat.get("specialization.miss", 0) + specialized_hits += count - miss + specialized_misses += miss + else: + basic += count + return basic, specialized_hits, specialized_misses, not_specialized + + def get_deferred_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.deferred", 0) + for opcode, opcode_stat in self._data.items() + if opcode != "RESUME" + } + + def get_misses_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.miss", 0) + for opcode, opcode_stat in self._data.items() + if not self.is_specializable(opcode) + } + + def get_opcode_counts(self) -> dict[str, int]: + counts = {} + for opcode, entry in self._data.items(): + count = entry.get("count", 0) + if count: + counts[opcode] = count + return counts + + +class Stats: + def __init__(self, data: RawData): + self._data = data + + def get(self, key: str) -> int: + return self._data.get(key, 0) + + @functools.cache + def get_opcode_stats(self, prefix: str) -> OpcodeStats: + opcode_stats = collections.defaultdict[str, dict](dict) + for key, value in self._data.items(): + if not key.startswith(prefix): + continue + name, _, rest = key[len(prefix) + 1 :].partition("]") + opcode_stats[name][rest.strip(".")] = value + return OpcodeStats( + opcode_stats, + self._data["_defines"], + self._data["_specialized_instructions"], ) - return rows - - -def print_specialization_stats(name, family_stats, defines): - if "specializable" not in family_stats: - return - total = sum(family_stats.get(kind, 0) for kind in TOTAL) - if total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - rows = calculate_specialization_stats(family_stats, total) - emit_table(("Kind", "Count", "Ratio"), rows) - rows = calculate_specialization_success_failure(family_stats) - if rows: - print_title("Specialization attempts", 4) - emit_table(("", "Count:", "Ratio:"), rows) - rows = calculate_specialization_failure_kinds(name, family_stats, defines) - emit_table(("Failure kind", "Count:", "Ratio:"), rows) - - -def print_comparative_specialization_stats( - name, base_family_stats, head_family_stats, defines -): - if "specializable" not in base_family_stats: - return - - base_total = sum(base_family_stats.get(kind, 0) for kind in TOTAL) - head_total = sum(head_family_stats.get(kind, 0) for kind in TOTAL) - if base_total + head_total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - base_rows = calculate_specialization_stats(base_family_stats, base_total) - head_rows = calculate_specialization_stats(head_family_stats, head_total) - emit_table( - ("Kind", "Base Count", "Base Ratio", "Head Count", "Head Ratio"), - join_rows(base_rows, head_rows), + + def get_call_stats(self) -> dict[str, int]: + defines = self._data["_stats_defines"] + result = {} + for key, value in sorted(self._data.items()): + if "Calls to" in key: + result[key] = value + elif key.startswith("Calls "): + name, index = key[:-1].split("[") + label = f"{name} ({pretty(defines[int(index)][0])})" + result[label] = value + + for key, value in sorted(self._data.items()): + if key.startswith("Frame"): + result[key] = value + + return result + + def get_object_stats(self) -> dict[str, tuple[int, int]]: + total_materializations = self._data.get("Object inline values", 0) + total_allocations = self._data.get("Object allocations", 0) + self._data.get( + "Object allocations from freelist", 0 ) - base_rows = calculate_specialization_success_failure(base_family_stats) - head_rows = calculate_specialization_success_failure(head_family_stats) - rows = join_rows(base_rows, head_rows) - if rows: - print_title("Specialization attempts", 4) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) - base_rows = calculate_specialization_failure_kinds( - name, base_family_stats, defines - ) - head_rows = calculate_specialization_failure_kinds( - name, head_family_stats, defines - ) - emit_table( - ( - "Failure kind", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", - ), - join_rows(base_rows, head_rows), - ) + total_increfs = self._data.get( + "Object interpreter increfs", 0 + ) + self._data.get("Object increfs", 0) + total_decrefs = self._data.get( + "Object interpreter decrefs", 0 + ) + self._data.get("Object decrefs", 0) + + result = {} + for key, value in self._data.items(): + if key.startswith("Object"): + if "materialize" in key: + den = total_materializations + elif "allocations" in key: + den = total_allocations + elif "increfs" in key: + den = total_increfs + elif "decrefs" in key: + den = total_decrefs + else: + den = None + label = key[6:].strip() + label = label[0].upper() + label[1:] + result[label] = (value, den) + return result + + def get_gc_stats(self) -> list[dict[str, int]]: + gc_stats: list[dict[str, int]] = [] + for key, value in self._data.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return gc_stats + + def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: + if "Optimization attempts" not in self._data: + return {} + + attempts = self._data["Optimization attempts"] + created = self._data["Optimization traces created"] + executed = self._data["Optimization traces executed"] + uops = self._data["Optimization uops executed"] + trace_stack_overflow = self._data["Optimization trace stack overflow"] + trace_stack_underflow = self._data["Optimization trace stack underflow"] + trace_too_long = self._data["Optimization trace too long"] + trace_too_short = self._data["Optimization trace too short"] + inner_loop = self._data["Optimization inner loop"] + recursive_call = self._data["Optimization recursive call"] + low_confidence = self._data["Optimization low confidence"] + executors_invalidated = self._data["Executors invalidated"] + + return { + Doc( + "Optimization attempts", + "The number of times a potential trace is identified. Specifically, this " + "occurs in the JUMP BACKWARD instruction when the counter reaches a " + "threshold.", + ): (attempts, None), + Doc( + "Traces created", "The number of traces that were successfully created." + ): (created, attempts), + Doc( + "Trace stack overflow", + "A trace is truncated because it would require more than 5 stack frames.", + ): (trace_stack_overflow, attempts), + Doc( + "Trace stack underflow", + "A potential trace is abandoned because it pops more frames than it pushes.", + ): (trace_stack_underflow, attempts), + Doc( + "Trace too long", + "A trace is truncated because it is longer than the instruction buffer.", + ): (trace_too_long, attempts), + Doc( + "Trace too short", + "A potential trace is abandoced because it it too short.", + ): (trace_too_short, attempts), + Doc( + "Inner loop found", "A trace is truncated because it has an inner loop" + ): (inner_loop, attempts), + Doc( + "Recursive call", + "A trace is truncated because it has a recursive call.", + ): (recursive_call, attempts), + Doc( + "Low confidence", + "A trace is abandoned because the likelihood of the jump to top being taken " + "is too low.", + ): (low_confidence, attempts), + Doc( + "Executors invalidated", + "The number of executors that were invalidated due to watched " + "dictionary changes.", + ): (executors_invalidated, created), + Doc("Traces executed", "The number of traces that were executed"): ( + executed, + None, + ), + Doc( + "Uops executed", + "The total number of uops (micro-operations) that were executed", + ): ( + uops, + executed, + ), + } + def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: + attempts = self._data["Optimization optimizer attempts"] + successes = self._data["Optimization optimizer successes"] + no_memory = self._data["Optimization optimizer failure no memory"] + builtins_changed = self._data["Optimizer remove globals builtins changed"] + incorrect_keys = self._data["Optimizer remove globals incorrect keys"] + + return { + Doc( + "Optimizer attempts", + "The number of times the trace optimizer (_Py_uop_analyze_and_optimize) was run.", + ): (attempts, None), + Doc( + "Optimizer successes", + "The number of traces that were successfully optimized.", + ): (successes, attempts), + Doc( + "Optimizer no memory", + "The number of optimizations that failed due to no memory.", + ): (no_memory, attempts), + Doc( + "Remove globals builtins changed", + "The builtins changed during optimization", + ): (builtins_changed, attempts), + Doc( + "Remove globals incorrect keys", + "The keys in the globals dictionary aren't what was expected", + ): (incorrect_keys, attempts), + } -def gather_stats(input): - # Note the output of this function must be JSON-serializable + def get_histogram(self, prefix: str) -> list[tuple[int, int]]: + rows = [] + for k, v in self._data.items(): + match = re.match(f"{prefix}\\[([0-9]+)\\]", k) + if match is not None: + entry = int(match.groups()[0]) + rows.append((entry, v)) + rows.sort() + return rows + + def get_rare_events(self) -> list[tuple[str, int]]: + prefix = "Rare event " + return [ + (key[len(prefix) + 1 : -1].replace("_", " "), val) + for key, val in self._data.items() + if key.startswith(prefix) + ] - if os.path.isfile(input): - with open(input, "r") as fd: - stats = json.load(fd) - stats["_stats_defines"] = { - int(k): v for k, v in stats["_stats_defines"].items() - } - stats["_defines"] = {int(k): v for k, v in stats["_defines"].items()} - return stats +class JoinMode(enum.Enum): + # Join using the first column as a key + SIMPLE = 0 + # Join using the first column as a key, and indicate the change in the + # second column of each input table as a new column + CHANGE = 1 + # Join using the first column as a key, indicating the change in the second + # column of each input table as a new column, and omit all other columns + CHANGE_ONE_COLUMN = 2 + # Join using the first column as a key, and indicate the change as a new + # column, but don't sort by the amount of change. + CHANGE_NO_SORT = 3 - elif os.path.isdir(input): - stats = collections.Counter() - for filename in os.listdir(input): - with open(os.path.join(input, filename)) as fd: - for line in fd: - try: - key, value = line.split(":") - except ValueError: - print( - f"Unparsable line: '{line.strip()}' in {filename}", - file=sys.stderr, - ) - continue - key = key.strip() - value = int(value) - stats[key] += value - stats["__nfiles__"] += 1 - import opcode +class Table: + """ + A Table defines how to convert a set of Stats into a specific set of rows + displaying some aspect of the data. + """ - stats["_specialized_instructions"] = [ - op for op in opcode._specialized_opmap.keys() if "__" not in op - ] - stats["_stats_defines"] = get_stats_defines() - stats["_defines"] = get_defines() + def __init__( + self, + column_names: Columns, + calc_rows: RowCalculator, + join_mode: JoinMode = JoinMode.SIMPLE, + ): + self.columns = column_names + self.calc_rows = calc_rows + self.join_mode = join_mode + + def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: + match self.join_mode: + case JoinMode.SIMPLE: + return (key, *row_a, *row_b) + case JoinMode.CHANGE | JoinMode.CHANGE_NO_SORT: + return (key, *row_a, *row_b, DiffRatio(row_a[0], row_b[0])) + case JoinMode.CHANGE_ONE_COLUMN: + return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) + + def join_columns(self, columns: Columns) -> Columns: + match self.join_mode: + case JoinMode.SIMPLE: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + case JoinMode.CHANGE | JoinMode.CHANGE_NO_SORT: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + ("Change:",) + case JoinMode.CHANGE_ONE_COLUMN: + return ( + columns[0], + "Base " + columns[1], + "Head " + columns[1], + "Change:", + ) - return stats - else: - raise ValueError(f"{input:r} is not a file or directory path") - - -def extract_opcode_stats(stats, prefix): - opcode_stats = collections.defaultdict(dict) - for key, value in stats.items(): - if not key.startswith(prefix): - continue - name, _, rest = key[len(prefix) + 1 :].partition("]") - opcode_stats[name][rest.strip(".")] = value - return opcode_stats - - -def parse_kinds(spec_src, prefix="SPEC_FAIL"): - defines = collections.defaultdict(list) - start = "#define " + prefix + "_" - for line in spec_src: - line = line.strip() - if not line.startswith(start): - continue - line = line[len(start) :] - name, val = line.split() - defines[int(val.strip())].append(name.strip()) - return defines - - -def pretty(defname): - return defname.replace("_", " ").lower() - - -def kind_to_text(kind, defines, opname): - if kind <= 8: - return pretty(defines[kind][0]) - if opname == "LOAD_SUPER_ATTR": - opname = "SUPER" - elif opname.endswith("ATTR"): - opname = "ATTR" - elif opname in ("FOR_ITER", "SEND"): - opname = "ITER" - elif opname.endswith("SUBSCR"): - opname = "SUBSCR" - for name in defines[kind]: - if name.startswith(opname): - return pretty(name[len(opname) + 1 :]) - return "kind " + str(kind) - - -def categorized_counts(opcode_stats, specialized_instructions): - basic = 0 - specialized = 0 - not_specialized = 0 - for name, opcode_stat in opcode_stats.items(): - if "execution_count" not in opcode_stat: - continue - count = opcode_stat["execution_count"] - if "specializable" in opcode_stat: - not_specialized += count - elif name in specialized_instructions: - miss = opcode_stat.get("specialization.miss", 0) - not_specialized += miss - specialized += count - miss - else: - basic += count - return basic, not_specialized, specialized + def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]: + ncols = len(self.columns) + default = ("",) * (ncols - 1) + data_a = {x[0]: x[1:] for x in rows_a} + data_b = {x[0]: x[1:] for x in rows_b} -def print_title(name, level=2): - print("#" * level, name) - print() + if len(data_a) != len(rows_a) or len(data_b) != len(rows_b): + raise ValueError("Duplicate keys") + + # To preserve ordering, use A's keys as is and then add any in B that + # aren't in A + keys = list(data_a.keys()) + [k for k in data_b.keys() if k not in data_a] + rows = [ + self.join_row(k, data_a.get(k, default), data_b.get(k, default)) + for k in keys + ] + if self.join_mode in (JoinMode.CHANGE, JoinMode.CHANGE_ONE_COLUMN): + rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) + + columns = self.join_columns(self.columns) + return columns, rows + + def get_table( + self, base_stats: Stats, head_stats: Stats | None = None + ) -> tuple[Columns, Rows]: + if head_stats is None: + rows = self.calc_rows(base_stats) + return self.columns, rows + else: + rows_a = self.calc_rows(base_stats) + rows_b = self.calc_rows(head_stats) + cols, rows = self.join_tables(rows_a, rows_b) + return cols, rows class Section: - def __init__(self, title, level=2, summary=None): + """ + A Section defines a section of the output document. + """ + + def __init__( + self, + title: str = "", + summary: str = "", + part_iter=None, + *, + comparative: bool = True, + doc: str = "", + ): self.title = title - self.level = level - if summary is None: + if not summary: self.summary = title.lower() else: self.summary = summary + self.doc = textwrap.dedent(doc) + if part_iter is None: + part_iter = [] + if isinstance(part_iter, list): - def __enter__(self): - print_title(self.title, self.level) - print("
    ") - print("", self.summary, "") - print() - return self + def iter_parts(base_stats: Stats, head_stats: Stats | None): + yield from part_iter - def __exit__(*args): - print() - print("
    ") - print() + self.part_iter = iter_parts + else: + self.part_iter = part_iter + self.comparative = comparative -def to_str(x): - if isinstance(x, int): - return format(x, ",d") - else: - return str(x) - - -def emit_table(header, rows): - width = len(header) - header_line = "|" - under_line = "|" - for item in header: - under = "---" - if item.endswith(":"): - item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line) - print(under_line) - for row in rows: - if width is not None and len(row) != width: - raise ValueError("Wrong number of elements in row '" + str(row) + "'") - print("|", " | ".join(to_str(i) for i in row), "|") - print() - - -def emit_histogram(title, stats, key, total): - rows = [] - for k, v in stats.items(): - if k.startswith(key): - entry = int(re.match(r".+\[([0-9]+)\]", k).groups()[0]) - rows.append((f"<= {entry}", int(v), format_ratio(int(v), total))) - # Don't include larger buckets with 0 entries - for j in range(len(rows) - 1, -1, -1): - if rows[j][1] != 0: - break - rows = rows[: j + 1] - - print(f"**{title}**\n") - emit_table(("Range", "Count:", "Ratio:"), rows) - - -def calculate_execution_counts(opcode_stats, total): - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - miss = 0 - if "specializable" not in opcode_stat: - miss = opcode_stat.get("specialization.miss") - counts.append((count, name, miss)) - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name, miss in counts: - cumulative += count - if miss: - miss = format_ratio(miss, count) - else: - miss = "" - rows.append( - ( - name, - count, - format_ratio(count, total), - format_ratio(cumulative, total), - miss, +def calc_execution_count_table(prefix: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats(prefix) + counts = opcode_stats.get_execution_counts() + total = opcode_stats.get_total_execution_count() + cumulative = 0 + rows: Rows = [] + for opcode, (count, miss) in sorted( + counts.items(), key=itemgetter(1), reverse=True + ): + cumulative += count + if miss: + miss_val = Ratio(miss, count) + else: + miss_val = None + rows.append( + ( + opcode, + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + miss_val, + ) ) - ) - return rows + return rows + return calc -def emit_execution_counts(opcode_stats, total): - with Section("Execution counts", summary="execution counts for all instructions"): - rows = calculate_execution_counts(opcode_stats, total) - emit_table(("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), rows) +def execution_count_section() -> Section: + return Section( + "Execution counts", + "Execution counts for Tier 1 instructions.", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("opcode"), + join_mode=JoinMode.CHANGE_ONE_COLUMN, + ) + ], + doc=""" + The "miss ratio" column shows the percentage of times the instruction + executed that it deoptimized. When this happens, the base unspecialized + instruction is not counted. + """, + ) -def _emit_comparative_execution_counts(base_rows, head_rows): - base_data = {x[0]: x[1:] for x in base_rows} - head_data = {x[0]: x[1:] for x in head_rows} - opcodes = base_data.keys() | head_data.keys() - rows = [] - default = [0, "0.0%", "0.0%", 0] - for opcode in opcodes: - base_entry = base_data.get(opcode, default) - head_entry = head_data.get(opcode, default) - if base_entry[0] == 0: - change = 1 - else: - change = (head_entry[0] - base_entry[0]) / base_entry[0] - rows.append((opcode, base_entry[0], head_entry[0], f"{change:0.1%}")) +def pair_count_section(prefix: str, title=None) -> Section: + def calc_pair_count_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats(prefix) + pair_counts = opcode_stats.get_pair_counts() + total = opcode_stats.get_total_execution_count() + + cumulative = 0 + rows: Rows = [] + for (opcode_i, opcode_j), count in itertools.islice( + sorted(pair_counts.items(), key=itemgetter(1), reverse=True), 100 + ): + cumulative += count + rows.append( + ( + f"{opcode_i} {opcode_j}", + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + ) + ) + return rows + + return Section( + "Pair counts", + f"Pair counts for top 100 {title if title else prefix} pairs", + [ + Table( + ("Pair", "Count:", "Self:", "Cumulative:"), + calc_pair_count_table, + ) + ], + comparative=False, + doc=""" + Pairs of specialized operations that deoptimize and are then followed by + the corresponding unspecialized instruction are not counted as pairs. + """, + ) - rows.sort(key=lambda x: abs(percentage_to_float(x[-1])), reverse=True) - emit_table(("Name", "Base Count:", "Head Count:", "Change:"), rows) +def pre_succ_pairs_section() -> Section: + def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None): + assert head_stats is None + opcode_stats = base_stats.get_opcode_stats("opcode") -def emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total, level=2 -): - with Section( - "Execution counts", summary="execution counts for all instructions", level=level - ): - base_rows = calculate_execution_counts(base_opcode_stats, base_total) - head_rows = calculate_execution_counts(head_opcode_stats, head_total) - _emit_comparative_execution_counts(base_rows, head_rows) + for opcode in opcode_stats.get_opcode_names(): + predecessors = opcode_stats.get_predecessors(opcode) + successors = opcode_stats.get_successors(opcode) + predecessors_total = predecessors.total() + successors_total = successors.total() + if predecessors_total == 0 and successors_total == 0: + continue + pred_rows = [ + (pred, Count(count), Ratio(count, predecessors_total)) + for (pred, count) in predecessors.most_common(5) + ] + succ_rows = [ + (succ, Count(count), Ratio(count, successors_total)) + for (succ, count) in successors.most_common(5) + ] + + yield Section( + opcode, + f"Successors and predecessors for {opcode}", + [ + Table( + ("Predecessors", "Count:", "Percentage:"), + lambda *_: pred_rows, # type: ignore + ), + Table( + ("Successors", "Count:", "Percentage:"), + lambda *_: succ_rows, # type: ignore + ), + ], + ) + return Section( + "Predecessor/Successor Pairs", + "Top 5 predecessors and successors of each Tier 1 opcode.", + iter_pre_succ_pairs_tables, + comparative=False, + doc=""" + This does not include the unspecialized instructions that occur after a + specialized instruction deoptimizes. + """, + ) -def get_defines(): - spec_path = os.path.join(os.path.dirname(__file__), "../../Python/specialize.c") - with open(spec_path) as spec_src: - defines = parse_kinds(spec_src) - return defines +def specialization_section() -> Section: + def calc_specialization_table(opcode: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + DOCS = { + "deferred": 'Lists the number of "deferred" (i.e. not specialized) instructions executed.', + "hit": "Specialized instructions that complete.", + "miss": "Specialized instructions that deopt.", + "deopt": "Specialized instructions that deopt.", + } -def emit_specialization_stats(opcode_stats, defines): - with Section("Specialization stats", summary="specialization stats by family"): - for name, opcode_stat in opcode_stats.items(): - print_specialization_stats(name, opcode_stat, defines) + opcode_stats = stats.get_opcode_stats("opcode") + total = opcode_stats.get_specialization_total(opcode) + specialization_counts = opcode_stats.get_specialization_counts(opcode) + return [ + ( + Doc(label, DOCS[label]), + Count(count), + Ratio(count, total), + ) + for label, count in specialization_counts.items() + ] + + return calc -def emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, defines -): - with Section("Specialization stats", summary="specialization stats by family"): - opcodes = set(base_opcode_stats.keys()) & set(head_opcode_stats.keys()) - for opcode in opcodes: - print_comparative_specialization_stats( - opcode, base_opcode_stats[opcode], head_opcode_stats[opcode], defines + def calc_specialization_success_failure_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + values = stats.get_opcode_stats( + "opcode" + ).get_specialization_success_failure(name) + total = sum(values.values()) + if total: + return [ + (label.capitalize(), Count(val), Ratio(val, total)) + for label, val in values.items() + ] + else: + return [] + + return calc + + def calc_specialization_failure_kind_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + failures = opcode_stats.get_specialization_failure_kinds(name) + total = opcode_stats.get_specialization_failure_total(name) + + return sorted( + [ + (label, Count(value), Ratio(value, total)) + for label, value in failures.items() + if value + ], + key=itemgetter(1), + reverse=True, ) + return calc + + def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): + opcode_base_stats = base_stats.get_opcode_stats("opcode") + names = opcode_base_stats.get_opcode_names() + if head_stats is not None: + opcode_head_stats = head_stats.get_opcode_stats("opcode") + names &= opcode_head_stats.get_opcode_names() # type: ignore + else: + opcode_head_stats = None -def calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions -): - basic, not_specialized, specialized = categorized_counts( - opcode_stats, specialized_instructions + for opcode in sorted(names): + if not opcode_base_stats.is_specializable(opcode): + continue + if opcode_base_stats.get_specialization_total(opcode) == 0 and ( + opcode_head_stats is None + or opcode_head_stats.get_specialization_total(opcode) == 0 + ): + continue + yield Section( + opcode, + f"specialization stats for {opcode} family", + [ + Table( + ("Kind", "Count:", "Ratio:"), + calc_specialization_table(opcode), + JoinMode.CHANGE, + ), + Table( + ("Success", "Count:", "Ratio:"), + calc_specialization_success_failure_table(opcode), + JoinMode.CHANGE, + ), + Table( + ("Failure kind", "Count:", "Ratio:"), + calc_specialization_failure_kind_table(opcode), + JoinMode.CHANGE, + ), + ], + ) + + return Section( + "Specialization stats", + "Specialization stats by family", + iter_specialization_tables, ) - return [ - ("Basic", basic, format_ratio(basic, total)), - ("Not specialized", not_specialized, format_ratio(not_specialized, total)), - ("Specialized", specialized, format_ratio(specialized, total)), - ] -def emit_specialization_overview(opcode_stats, total, specialized_instructions): - with Section("Specialization effectiveness"): - rows = calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions - ) - emit_table(("Instructions", "Count:", "Ratio:"), rows) - for title, field in ( - ("Deferred", "specialization.deferred"), - ("Misses", "specialization.miss"), - ): - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - # Avoid double counting misses - if title == "Misses" and "specializable" in opcode_stat: - continue - value = opcode_stat.get(field, 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - with Section(f"{title} by instruction", 3): - rows = [ - (name, count, format_ratio(count, total)) - for (count, name) in counts[:10] - ] - emit_table(("Name", "Count:", "Ratio:"), rows) - - -def emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - specialized_instructions, -): - with Section("Specialization effectiveness"): - base_rows = calculate_specialization_effectiveness( - base_opcode_stats, base_total, specialized_instructions - ) - head_rows = calculate_specialization_effectiveness( - head_opcode_stats, head_total, specialized_instructions - ) - emit_table( +def specialization_effectiveness_section() -> Section: + def calc_specialization_effectiveness_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = opcode_stats.get_total_execution_count() + + ( + basic, + specialized_hits, + specialized_misses, + not_specialized, + ) = opcode_stats.get_specialized_total_counts() + + return [ ( - "Instructions", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", + Doc( + "Basic", + "Instructions that are not and cannot be specialized, e.g. `LOAD_FAST`.", + ), + Count(basic), + Ratio(basic, total), ), - join_rows(base_rows, head_rows), - ) + ( + Doc( + "Not specialized", + "Instructions that could be specialized but aren't, e.g. `LOAD_ATTR`, `BINARY_SLICE`.", + ), + Count(not_specialized), + Ratio(not_specialized, total), + ), + ( + Doc( + "Specialized hits", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that complete.", + ), + Count(specialized_hits), + Ratio(specialized_hits, total), + ), + ( + Doc( + "Specialized misses", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that deopt.", + ), + Count(specialized_misses), + Ratio(specialized_misses, total), + ), + ] + + def calc_deferred_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + deferred_counts = opcode_stats.get_deferred_counts() + total = sum(deferred_counts.values()) + if total == 0: + return [] + + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + deferred_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] + def calc_misses_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + misses_counts = opcode_stats.get_misses_counts() + total = sum(misses_counts.values()) + if total == 0: + return [] + + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + misses_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] -def get_stats_defines(): - stats_path = os.path.join( - os.path.dirname(__file__), "../../Include/cpython/pystats.h" + return Section( + "Specialization effectiveness", + "", + [ + Table( + ("Instructions", "Count:", "Ratio:"), + calc_specialization_effectiveness_table, + JoinMode.CHANGE, + ), + Section( + "Deferred by instruction", + "Breakdown of deferred (not specialized) instruction counts by family", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_deferred_by_table, + JoinMode.CHANGE, + ) + ], + ), + Section( + "Misses by instruction", + "Breakdown of misses (specialized deopts) instruction counts by family", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_misses_by_table, + JoinMode.CHANGE, + ) + ], + ), + ], + doc=""" + All entries are execution counts. Should add up to the total number of + Tier 1 instructions executed. + """, ) - with open(stats_path) as stats_src: - defines = parse_kinds(stats_src, prefix="EVAL_CALL") - return defines - - -def calculate_call_stats(stats, defines): - total = 0 - for key, value in stats.items(): - if "Calls to" in key: - total += value - rows = [] - for key, value in stats.items(): - if "Calls to" in key: - rows.append((key, value, format_ratio(value, total))) - elif key.startswith("Calls "): - name, index = key[:-1].split("[") - index = int(index) - label = name + " (" + pretty(defines[index][0]) + ")" - rows.append((label, value, format_ratio(value, total))) - for key, value in stats.items(): - if key.startswith("Frame"): - rows.append((key, value, format_ratio(value, total))) - return rows - - -def emit_call_stats(stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - rows = calculate_call_stats(stats, defines) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_call_stats(base_stats, head_stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - base_rows = calculate_call_stats(base_stats, defines) - head_rows = calculate_call_stats(head_stats, defines) - rows = join_rows(base_rows, head_rows) - rows.sort(key=lambda x: -percentage_to_float(x[-1])) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) -def calculate_object_stats(stats): - total_materializations = stats.get("Object new values") - total_allocations = stats.get("Object allocations") + stats.get( - "Object allocations from freelist" - ) - total_increfs = stats.get("Object interpreter increfs") + stats.get( - "Object increfs" - ) - total_decrefs = stats.get("Object interpreter decrefs") + stats.get( - "Object decrefs" +def call_stats_section() -> Section: + def calc_call_stats_table(stats: Stats) -> Rows: + call_stats = stats.get_call_stats() + total = sum(v for k, v in call_stats.items() if "Calls to" in k) + return [ + (key, Count(value), Ratio(value, total)) + for key, value in call_stats.items() + ] + + return Section( + "Call stats", + "Inlined calls and frame stats", + [ + Table( + ("", "Count:", "Ratio:"), + calc_call_stats_table, + JoinMode.CHANGE, + ) + ], + doc=""" + This shows what fraction of calls to Python functions are inlined (i.e. + not having a call at the C level) and for those that are not, where the + call comes from. The various categories overlap. + + Also includes the count of frame objects created. + """, ) - rows = [] - for key, value in stats.items(): - if key.startswith("Object"): - if "materialize" in key: - ratio = format_ratio(value, total_materializations) - elif "allocations" in key: - ratio = format_ratio(value, total_allocations) - elif "increfs" in key: - ratio = format_ratio(value, total_increfs) - elif "decrefs" in key: - ratio = format_ratio(value, total_decrefs) - else: - ratio = "" - label = key[6:].strip() - label = label[0].upper() + label[1:] - rows.append((label, value, ratio)) - return rows - - -def calculate_gc_stats(stats): - gc_stats = [] - for key, value in stats.items(): - if not key.startswith("GC"): - continue - n, _, rest = key[3:].partition("]") - name = rest.strip() - gen_n = int(n) - while len(gc_stats) <= gen_n: - gc_stats.append({}) - gc_stats[gen_n][name] = value - return [ - (i, gen["collections"], gen["objects collected"], gen["object visits"]) - for (i, gen) in enumerate(gc_stats) - ] - - -def emit_object_stats(stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - rows = calculate_object_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_object_stats(base_stats, head_stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - base_rows = calculate_object_stats(base_stats) - head_rows = calculate_object_stats(head_stats) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - join_rows(base_rows, head_rows), - ) -def emit_gc_stats(stats): - with Section("GC stats", summary="GC collections and effectiveness"): - rows = calculate_gc_stats(stats) - emit_table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), - rows, - ) +def object_stats_section() -> Section: + def calc_object_stats_table(stats: Stats) -> Rows: + object_stats = stats.get_object_stats() + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in object_stats.items() + ] + return Section( + "Object stats", + "Allocations, frees and dict materializatons", + [ + Table( + ("", "Count:", "Ratio:"), + calc_object_stats_table, + JoinMode.CHANGE, + ) + ], + doc=""" + Below, "allocations" means "allocations that are not from a freelist". + Total allocations = "Allocations from freelist" + "Allocations". -def emit_comparative_gc_stats(base_stats, head_stats): - with Section("GC stats", summary="GC collections and effectiveness"): - base_rows = calculate_gc_stats(base_stats) - head_rows = calculate_gc_stats(head_stats) - emit_table( - ( - "Generation:", - "Base collections:", - "Head collections:", - "Base objects collected:", - "Head objects collected:", - "Base object visits:", - "Head object visits:", - ), - join_rows(base_rows, head_rows), - ) + "Inline values" is the number of values arrays inlined into objects. + The cache hit/miss numbers are for the MRO cache, split into dunder and + other names. + """, + ) -def get_total(opcode_stats): - total = 0 - for opcode_stat in opcode_stats.values(): - if "execution_count" in opcode_stat: - total += opcode_stat["execution_count"] - return total - - -def emit_pair_counts(opcode_stats, total): - pair_counts = [] - for name_i, opcode_stat in opcode_stats.items(): - for key, value in opcode_stat.items(): - if key.startswith("pair_count"): - name_j, _, _ = key[11:].partition("]") - if value: - pair_counts.append((value, (name_i, name_j))) - with Section("Pair counts", summary="Pair counts for top 100 pairs"): - pair_counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, pair in itertools.islice(pair_counts, 100): - name_i, name_j = pair - cumulative += count - rows.append( - ( - f"{name_i} {name_j}", - count, - format_ratio(count, total), - format_ratio(cumulative, total), - ) + +def gc_stats_section() -> Section: + def calc_gc_stats(stats: Stats) -> Rows: + gc_stats = stats.get_gc_stats() + + return [ + ( + Count(i), + Count(gen["collections"]), + Count(gen["objects collected"]), + Count(gen["object visits"]), ) - emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows) - with Section( - "Predecessor/Successor Pairs", - summary="Top 5 predecessors and successors of each opcode", - ): - predecessors = collections.defaultdict(collections.Counter) - successors = collections.defaultdict(collections.Counter) - total_predecessors = collections.Counter() - total_successors = collections.Counter() - for count, (first, second) in pair_counts: - if count: - predecessors[second][first] = count - successors[first][second] = count - total_predecessors[second] += count - total_successors[first] += count - for name in opcode_stats.keys(): - total1 = total_predecessors[name] - total2 = total_successors[name] - if total1 == 0 and total2 == 0: - continue - pred_rows = succ_rows = () - if total1: - pred_rows = [ - (pred, count, f"{count/total1:.1%}") - for (pred, count) in predecessors[name].most_common(5) - ] - if total2: - succ_rows = [ - (succ, count, f"{count/total2:.1%}") - for (succ, count) in successors[name].most_common(5) - ] - with Section(name, 3, f"Successors and predecessors for {name}"): - emit_table(("Predecessors", "Count:", "Percentage:"), pred_rows) - emit_table(("Successors", "Count:", "Percentage:"), succ_rows) - - -def calculate_optimization_stats(stats): - attempts = stats["Optimization attempts"] - created = stats["Optimization traces created"] - executed = stats["Optimization traces executed"] - uops = stats["Optimization uops executed"] - trace_stack_overflow = stats["Optimization trace stack overflow"] - trace_stack_underflow = stats["Optimization trace stack underflow"] - trace_too_long = stats["Optimization trace too long"] - trace_too_short = stats["Optimiztion trace too short"] - inner_loop = stats["Optimization inner loop"] - recursive_call = stats["Optimization recursive call"] - - return [ - ("Optimization attempts", attempts, ""), - ("Traces created", created, format_ratio(created, attempts)), - ("Traces executed", executed, ""), - ("Uops executed", uops, int(uops / (executed or 1))), - ("Trace stack overflow", trace_stack_overflow, ""), - ("Trace stack underflow", trace_stack_underflow, ""), - ("Trace too long", trace_too_long, ""), - ("Trace too short", trace_too_short, ""), - ("Inner loop found", inner_loop, ""), - ("Recursive call", recursive_call, ""), - ] - - -def calculate_uop_execution_counts(opcode_stats): - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - counts.append((count, name)) - total += count - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name in counts: - cumulative += count - rows.append( - (name, count, format_ratio(count, total), format_ratio(cumulative, total)) - ) - return rows + for (i, gen) in enumerate(gc_stats) + ] + return Section( + "GC stats", + "GC collections and effectiveness", + [ + Table( + ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + calc_gc_stats, + ) + ], + doc=""" + Collected/visits gives some measure of efficiency. + """, + ) -def emit_optimization_stats(stats): - if "Optimization attempts" not in stats: - return - uop_stats = extract_opcode_stats(stats, "uops") +def optimization_section() -> Section: + def calc_optimization_table(stats: Stats) -> Rows: + optimization_stats = stats.get_optimization_stats() - with Section( - "Optimization (Tier 2) stats", summary="statistics about the Tier 2 optimizer" - ): - with Section("Overall stats", level=3): - rows = calculate_optimization_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - emit_histogram( - "Trace length histogram", - stats, - "Trace length", - stats["Optimization traces created"], - ) - emit_histogram( - "Optimized trace length histogram", - stats, - "Optimized trace length", - stats["Optimization traces created"], - ) - emit_histogram( - "Trace run length histogram", - stats, - "Trace run length", - stats["Optimization traces executed"], - ) + return [ + ( + label, + Count(value), + Ratio(value, den, percentage=label != "Uops executed"), + ) + for label, (value, den) in optimization_stats.items() + ] - with Section("Uop stats", level=3): - rows = calculate_uop_execution_counts(uop_stats) - emit_table(("Uop", "Count:", "Self:", "Cumulative:"), rows) + def calc_optimizer_table(stats: Stats) -> Rows: + optimizer_stats = stats.get_optimizer_stats() - with Section("Unsupported opcodes", level=3): - unsupported_opcodes = extract_opcode_stats(stats, "unsupported_opcode") - data = [] - for opcode, entry in unsupported_opcodes.items(): - data.append((entry["count"], opcode)) - data.sort(reverse=True) - rows = [(x[1], x[0]) for x in data] - emit_table(("Opcode", "Count"), rows) + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in optimizer_stats.items() + ] + def calc_histogram_table(key: str, den: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + histogram = stats.get_histogram(key) + denominator = stats.get(den) + + rows: Rows = [] + last_non_zero = 0 + for k, v in histogram: + if v != 0: + last_non_zero = len(rows) + rows.append( + ( + f"<= {k:,d}", + Count(v), + Ratio(v, denominator), + ) + ) + # Don't include any zero entries at the end + rows = rows[: last_non_zero + 1] + return rows + + return calc + + def calc_unsupported_opcodes_table(stats: Stats) -> Rows: + unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in unsupported_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) -def emit_comparative_optimization_stats(base_stats, head_stats): - print("## Comparative optimization stats not implemented\n\n") + def calc_error_in_opcodes_table(stats: Stats) -> Rows: + error_in_opcodes = stats.get_opcode_stats("error_in_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in error_in_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): + if not base_stats.get_optimization_stats() or ( + head_stats is not None and not head_stats.get_optimization_stats() + ): + return + + yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + yield Table(("", "Count:", "Ratio:"), calc_optimizer_table, JoinMode.CHANGE) + for name, den in [ + ("Trace length", "Optimization traces created"), + ("Optimized trace length", "Optimization traces created"), + ("Trace run length", "Optimization traces executed"), + ]: + yield Section( + f"{name} histogram", + "", + [ + Table( + ("Range", "Count:", "Ratio:"), + calc_histogram_table(name, den), + JoinMode.CHANGE_NO_SORT, + ) + ], + ) + yield Section( + "Uop execution stats", + "", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("uops"), + JoinMode.CHANGE_ONE_COLUMN, + ) + ], + ) + yield pair_count_section(prefix="uop", title="Non-JIT uop") + yield Section( + "Unsupported opcodes", + "", + [ + Table( + ("Opcode", "Count:"), + calc_unsupported_opcodes_table, + JoinMode.CHANGE, + ) + ], + ) + yield Section( + "Optimizer errored out with opcode", + "Optimization stopped after encountering this opcode", + [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)], + ) -def output_single_stats(stats): - opcode_stats = extract_opcode_stats(stats, "opcode") - total = get_total(opcode_stats) - emit_execution_counts(opcode_stats, total) - emit_pair_counts(opcode_stats, total) - emit_specialization_stats(opcode_stats, stats["_defines"]) - emit_specialization_overview( - opcode_stats, total, stats["_specialized_instructions"] + return Section( + "Optimization (Tier 2) stats", + "statistics about the Tier 2 optimizer", + iter_optimization_tables, ) - emit_call_stats(stats, stats["_stats_defines"]) - emit_object_stats(stats) - emit_gc_stats(stats) - emit_optimization_stats(stats) - with Section("Meta stats", summary="Meta statistics"): - emit_table(("", "Count:"), [("Number of data files", stats["__nfiles__"])]) -def output_comparative_stats(base_stats, head_stats): - base_opcode_stats = extract_opcode_stats(base_stats, "opcode") - base_total = get_total(base_opcode_stats) - - head_opcode_stats = extract_opcode_stats(head_stats, "opcode") - head_total = get_total(head_opcode_stats) +def rare_event_section() -> Section: + def calc_rare_event_table(stats: Stats) -> Table: + DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + "watched dict modification": "A watched dict has been modified", + "watched globals modification": "A watched `globals()` dict has been modified", + } + return [(Doc(x, DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] - emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total - ) - emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, head_stats["_defines"] + return Section( + "Rare events", + "Counts of rare/unlikely events", + [Table(("Event", "Count:"), calc_rare_event_table, JoinMode.CHANGE)], ) - emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - head_stats["_specialized_instructions"], - ) - emit_comparative_call_stats(base_stats, head_stats, head_stats["_stats_defines"]) - emit_comparative_object_stats(base_stats, head_stats) - emit_comparative_gc_stats(base_stats, head_stats) - emit_comparative_optimization_stats(base_stats, head_stats) -def output_stats(inputs, json_output=None): - if len(inputs) == 1: - stats = gather_stats(inputs[0]) - if json_output is not None: - json.dump(stats, json_output) - output_single_stats(stats) - elif len(inputs) == 2: - if json_output is not None: - raise ValueError("Can not output to JSON when there are multiple inputs") +def meta_stats_section() -> Section: + def calc_rows(stats: Stats) -> Rows: + return [("Number of data files", Count(stats.get("__nfiles__")))] + + return Section( + "Meta stats", + "Meta statistics", + [Table(("", "Count:"), calc_rows, JoinMode.CHANGE)], + ) - base_stats = gather_stats(inputs[0]) - head_stats = gather_stats(inputs[1]) - output_comparative_stats(base_stats, head_stats) - print("---") - print("Stats gathered on:", date.today()) +LAYOUT = [ + execution_count_section(), + pair_count_section("opcode"), + pre_succ_pairs_section(), + specialization_section(), + specialization_effectiveness_section(), + call_stats_section(), + object_stats_section(), + gc_stats_section(), + optimization_section(), + rare_event_section(), + meta_stats_section(), +] + + +def output_markdown( + out: TextIO, + obj: Section | Table | list, + base_stats: Stats, + head_stats: Stats | None = None, + level: int = 2, +) -> None: + def to_markdown(x): + if hasattr(x, "markdown"): + return x.markdown() + elif isinstance(x, str): + return x + elif x is None: + return "" + else: + raise TypeError(f"Can't convert {x} to markdown") + + match obj: + case Section(): + if obj.title: + print("#" * level, obj.title, file=out) + print(file=out) + print("
    ", file=out) + print("", obj.summary, "", file=out) + print(file=out) + if obj.doc: + print(obj.doc, file=out) + + if head_stats is not None and obj.comparative is False: + print("Not included in comparative output.\n") + else: + for part in obj.part_iter(base_stats, head_stats): + output_markdown(out, part, base_stats, head_stats, level=level + 1) + print(file=out) + if obj.title: + print("
    ", file=out) + print(file=out) + + case Table(): + header, rows = obj.get_table(base_stats, head_stats) + if len(rows) == 0: + return + + alignments = [] + for item in header: + if item.endswith(":"): + alignments.append("right") + else: + alignments.append("left") + + print("", file=out) + print("", file=out) + print("", file=out) + for item, align in zip(header, alignments): + if item.endswith(":"): + item = item[:-1] + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("", file=out) + for row in rows: + if len(row) != len(header): + raise ValueError( + "Wrong number of elements in row '" + str(row) + "'" + ) + print("", file=out) + for col, align in zip(row, alignments): + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("
    {item}
    {to_markdown(col)}
    ", file=out) + print(file=out) + + case list(): + for part in obj: + output_markdown(out, part, base_stats, head_stats, level=level) + + print("---", file=out) + print("Stats gathered on:", date.today(), file=out) + + +def output_stats(inputs: list[Path], json_output=str | None): + match len(inputs): + case 1: + data = load_raw_data(Path(inputs[0])) + if json_output is not None: + with open(json_output, "w", encoding="utf-8") as f: + save_raw_data(data, f) # type: ignore + stats = Stats(data) + output_markdown(sys.stdout, LAYOUT, stats) + case 2: + if json_output is not None: + raise ValueError( + "Can not output to JSON when there are multiple inputs" + ) + base_data = load_raw_data(Path(inputs[0])) + head_data = load_raw_data(Path(inputs[1])) + base_stats = Stats(base_data) + head_stats = Stats(head_data) + output_markdown(sys.stdout, LAYOUT, base_stats, head_stats) def main(): @@ -929,7 +1431,6 @@ def main(): parser.add_argument( "--json-output", nargs="?", - type=argparse.FileType("w"), help="Output complete raw results to the given JSON file.", ) diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index ab1134ed8c4f77..d24e02210d489c 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -15,7 +15,6 @@ import operator import os import re -import sys parser = argparse.ArgumentParser( @@ -23,7 +22,7 @@ ) parser.add_argument("srcdir", help="OpenSSL source directory") parser.add_argument( - "output", nargs="?", type=argparse.FileType("w"), default=sys.stdout + "output", nargs="?", default=None ) @@ -126,8 +125,13 @@ def main(): lines.append("") lines.extend(gen_error_codes(args)) - for line in lines: - args.output.write(line + "\n") + if args.output is None: + for line in lines: + print(line) + else: + with open(args.output, 'w') as output: + for line in lines: + print(line, file=output) if __name__ == "__main__": diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 120e3883adc795..baa16102068aa0 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,8 +47,9 @@ OPENSSL_RECENT_VERSIONS = [ "1.1.1w", - "3.0.11", - "3.1.3", + "3.0.13", + "3.1.5", + "3.2.1", ] LIBRESSL_OLD_VERSIONS = [ diff --git a/Tools/stringbench/README b/Tools/stringbench/README deleted file mode 100644 index a271f12632afff..00000000000000 --- a/Tools/stringbench/README +++ /dev/null @@ -1,68 +0,0 @@ -stringbench is a set of performance tests comparing byte string -operations with unicode operations. The two string implementations -are loosely based on each other and sometimes the algorithm for one is -faster than the other. - -These test set was started at the Need For Speed sprint in Reykjavik -to identify which string methods could be sped up quickly and to -identify obvious places for improvement. - -Here is an example of a benchmark - - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -The bench decorator takes three parameters. The first is a short -description of how the code works. In most cases this is Python code -snippet. It is not the code which is actually run because the real -code is hand-optimized to focus on the method being tested. - -The second parameter is a group title. All benchmarks with the same -group title are listed together. This lets you compare different -implementations of the same algorithm, such as "t in s" -vs. "s.find(t)". - -The last is a count. Each benchmark loops over the algorithm either -100 or 1000 times, depending on the algorithm performance. The output -time is the time per benchmark call so the reader needs a way to know -how to scale the performance. - -These parameters become function attributes. - - -Here is an example of the output - - -========== count newlines -38.54 41.60 92.7 ...text.with.2000.newlines.count("\n") (*100) -========== early match, single character -1.14 1.18 96.8 ("A"*1000).find("A") (*1000) -0.44 0.41 105.6 "A" in "A"*1000 (*1000) -1.15 1.17 98.1 ("A"*1000).index("A") (*1000) - -The first column is the run time in milliseconds for byte strings. -The second is the run time for unicode strings. The third is a -percentage; byte time / unicode time. It's the percentage by which -unicode is faster than byte strings. - -The last column contains the code snippet and the repeat count for the -internal benchmark loop. - -The times are computed with 'timeit.py' which repeats the test more -and more times until the total time takes over 0.2 seconds, returning -the best time for a single iteration. - -The final line of the output is the cumulative time for byte and -unicode strings, and the overall performance of unicode relative to -bytes. For example - -4079.83 5432.25 75.1 TOTAL - -However, this has no meaning as it evenly weights every test. - diff --git a/Tools/stringbench/stringbench.py b/Tools/stringbench/stringbench.py deleted file mode 100644 index 5d2b4146378626..00000000000000 --- a/Tools/stringbench/stringbench.py +++ /dev/null @@ -1,1482 +0,0 @@ - -# Various microbenchmarks comparing unicode and byte string performance -# Please keep this file both 2.x and 3.x compatible! - -import timeit -import itertools -import operator -import re -import sys -import datetime -import optparse - -VERSION = '2.0' - -def p(*args): - sys.stdout.write(' '.join(str(s) for s in args) + '\n') - -if sys.version_info >= (3,): - BYTES = bytes_from_str = lambda x: x.encode('ascii') - UNICODE = unicode_from_str = lambda x: x -else: - BYTES = bytes_from_str = lambda x: x - UNICODE = unicode_from_str = lambda x: x.decode('ascii') - -class UnsupportedType(TypeError): - pass - - -p('stringbench v%s' % VERSION) -p(sys.version) -p(datetime.datetime.now()) - -REPEAT = 1 -REPEAT = 3 -#REPEAT = 7 - -if __name__ != "__main__": - raise SystemExit("Must run as main program") - -parser = optparse.OptionParser() -parser.add_option("-R", "--skip-re", dest="skip_re", - action="store_true", - help="skip regular expression tests") -parser.add_option("-8", "--8-bit", dest="bytes_only", - action="store_true", - help="only do 8-bit string benchmarks") -parser.add_option("-u", "--unicode", dest="unicode_only", - action="store_true", - help="only do Unicode string benchmarks") - - -_RANGE_1000 = list(range(1000)) -_RANGE_100 = list(range(100)) -_RANGE_10 = list(range(10)) - -dups = {} -def bench(s, group, repeat_count): - def blah(f): - if f.__name__ in dups: - raise AssertionError("Multiple functions with same name: %r" % - (f.__name__,)) - dups[f.__name__] = 1 - f.comment = s - f.is_bench = True - f.group = group - f.repeat_count = repeat_count - return f - return blah - -def uses_re(f): - f.uses_re = True - -####### 'in' comparisons - -@bench('"A" in "A"*1000', "early match, single character", 1000) -def in_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - for x in _RANGE_1000: - s2 in s1 - -@bench('"B" in "A"*1000', "no match, single character", 1000) -def in_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - for x in _RANGE_1000: - s2 in s1 - - -@bench('"AB" in "AB"*1000', "early match, two characters", 1000) -def in_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in "AB"*1000', "no match, two characters", 1000) -def in_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in ("AB"*300+"C")', "late match, two characters", 1000) -def in_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('s="ABC"*33; (s+"E") in ((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def in_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - for x in _RANGE_100: - s2 in s1 - -# Try with regex -@uses_re -@bench('s="ABC"*33; re.compile(s+"D").search((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def re_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - pat = re.compile(s2) - search = pat.search - for x in _RANGE_100: - search(s1) - - -#### same tests as 'in' but use 'find' - -@bench('("A"*1000).find("A")', "early match, single character", 1000) -def find_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("A"*1000).find("B")', "no match, single character", 1000) -def find_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - - -@bench('("AB"*1000).find("AB")', "early match, two characters", 1000) -def find_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("BC")', "no match, two characters", 1000) -def find_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("CA")', "no match, two characters", 1000) -def find_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"C").find("BC")', "late match, two characters", 1000) -def find_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"CA").find("CA")', "late match, two characters", 1000) -def find_test_slow_match_two_characters_bis(STR): - s1 = STR("AB" * 300+"CA") - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").find(s+"E")', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+"E"+s).find("E"+s)', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + e+m - s2 = e+m - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - - -#### Same tests for 'rfind' - -@bench('("A"*1000).rfind("A")', "early match, single character", 1000) -def rfind_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("A"*1000).rfind("B")', "no match, single character", 1000) -def rfind_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - - -@bench('("AB"*1000).rfind("AB")', "early match, two characters", 1000) -def rfind_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("BC")', "no match, two characters", 1000) -def rfind_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("CA")', "no match, two characters", 1000) -def rfind_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("C"+"AB"*300).rfind("CA")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("BC"+"AB"*300).rfind("BC")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters_bis(STR): - s1 = STR("BC" + "AB" * 300) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rfind("E"+s)', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e+m + (d+m)*500 - s2 = e+m - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - -@bench('s="ABC"*33; (s+"E"+("D"+s)*500).rfind(s+"E")', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = m+e + (d+m)*500 - s2 = m+e - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - - -#### Now with index. -# Skip the ones which fail because that would include exception overhead. - -@bench('("A"*1000).index("A")', "early match, single character", 1000) -def index_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*1000).index("AB")', "early match, two characters", 1000) -def index_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*300+"C").index("BC")', "late match, two characters", 1000) -def index_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").index(s+"E")', - "late match, 100 characters", 100) -def index_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_index = s1.index - for x in _RANGE_100: - s1_index(s2) - - -#### Same for rindex - -@bench('("A"*1000).rindex("A")', "early match, single character", 1000) -def rindex_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("AB"*1000).rindex("AB")', "early match, two characters", 1000) -def rindex_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("C"+"AB"*300).rindex("CA")', "late match, two characters", 1000) -def rindex_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rindex("E"+s)', - "late match, 100 characters", 100) -def rindex_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rindex = s1.rindex - for x in _RANGE_100: - s1_rindex(s2) - - -#### Same for partition - -@bench('("A"*1000).partition("A")', "early match, single character", 1000) -def partition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("A"*1000).partition("B")', "no match, single character", 1000) -def partition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - - -@bench('("AB"*1000).partition("AB")', "early match, two characters", 1000) -def partition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*1000).partition("BC")', "no match, two characters", 1000) -def partition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*300+"C").partition("BC")', "late match, two characters", 1000) -def partition_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").partition(s+"E")', - "late match, 100 characters", 100) -def partition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_partition = s1.partition - for x in _RANGE_100: - s1_partition(s2) - - -#### Same for rpartition - -@bench('("A"*1000).rpartition("A")', "early match, single character", 1000) -def rpartition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("A"*1000).rpartition("B")', "no match, single character", 1000) -def rpartition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - - -@bench('("AB"*1000).rpartition("AB")', "early match, two characters", 1000) -def rpartition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("AB"*1000).rpartition("BC")', "no match, two characters", 1000) -def rpartition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("C"+"AB"*300).rpartition("CA")', "late match, two characters", 1000) -def rpartition_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rpartition("E"+s)', - "late match, 100 characters", 100) -def rpartition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rpartition = s1.rpartition - for x in _RANGE_100: - s1_rpartition(s2) - - -#### Same for split(s, 1) - -@bench('("A"*1000).split("A", 1)', "early match, single character", 1000) -def split_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("A"*1000).split("B", 1)', "no match, single character", 1000) -def split_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - - -@bench('("AB"*1000).split("AB", 1)', "early match, two characters", 1000) -def split_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*1000).split("BC", 1)', "no match, two characters", 1000) -def split_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*300+"C").split("BC", 1)', "late match, two characters", 1000) -def split_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").split(s+"E", 1)', - "late match, 100 characters", 100) -def split_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_split = s1.split - for x in _RANGE_100: - s1_split(s2, 1) - - -#### Same for rsplit(s, 1) - -@bench('("A"*1000).rsplit("A", 1)', "early match, single character", 1000) -def rsplit_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("A"*1000).rsplit("B", 1)', "no match, single character", 1000) -def rsplit_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - - -@bench('("AB"*1000).rsplit("AB", 1)', "early match, two characters", 1000) -def rsplit_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("AB"*1000).rsplit("BC", 1)', "no match, two characters", 1000) -def rsplit_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("C"+"AB"*300).rsplit("CA", 1)', "late match, two characters", 1000) -def rsplit_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rsplit("E"+s, 1)', - "late match, 100 characters", 100) -def rsplit_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rsplit = s1.rsplit - for x in _RANGE_100: - s1_rsplit(s2, 1) - - -#### Benchmark the operator-based methods - -@bench('"A"*10', "repeat 1 character 10 times", 1000) -def repeat_single_10_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 10 - -@bench('"A"*1000', "repeat 1 character 1000 times", 1000) -def repeat_single_1000_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 1000 - -@bench('"ABCDE"*10', "repeat 5 characters 10 times", 1000) -def repeat_5_10_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 10 - -@bench('"ABCDE"*1000', "repeat 5 characters 1000 times", 1000) -def repeat_5_1000_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 1000 - -# + for concat - -@bench('"Andrew"+"Dalke"', "concat two strings", 1000) -def concat_two_strings(STR): - s1 = STR("Andrew") - s2 = STR("Dalke") - for x in _RANGE_1000: - s1+s2 - -@bench('s1+s2+s3+s4+...+s20', "concat 20 strings of words length 4 to 15", - 1000) -def concat_many_strings(STR): - s1=STR('TIXSGYNREDCVBHJ') - s2=STR('PUMTLXBZVDO') - s3=STR('FVZNJ') - s4=STR('OGDXUW') - s5=STR('WEIMRNCOYVGHKB') - s6=STR('FCQTNMXPUZH') - s7=STR('TICZJYRLBNVUEAK') - s8=STR('REYB') - s9=STR('PWUOQ') - s10=STR('EQHCMKBS') - s11=STR('AEVDFOH') - s12=STR('IFHVD') - s13=STR('JGTCNLXWOHQ') - s14=STR('ITSKEPYLROZAWXF') - s15=STR('THEK') - s16=STR('GHPZFBUYCKMNJIT') - s17=STR('JMUZ') - s18=STR('WLZQMTB') - s19=STR('KPADCBW') - s20=STR('TNJHZQAGBU') - for x in _RANGE_1000: - (s1 + s2+ s3+ s4+ s5+ s6+ s7+ s8+ s9+s10+ - s11+s12+s13+s14+s15+s16+s17+s18+s19+s20) - - -#### Benchmark join - -def get_bytes_yielding_seq(STR, arg): - if STR is BYTES and sys.version_info >= (3,): - raise UnsupportedType - return STR(arg) - -@bench('"A".join("")', - "join empty string, with 1 character sep", 100) -def join_empty_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"ABCDE".join("")', - "join empty string, with 5 character sep", 100) -def join_empty_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"A".join("ABC..Z")', - "join string with 26 characters, with 1 character sep", 1000) -def join_alphabet_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join("ABC..Z")', - "join string with 26 characters, with 5 character sep", 1000) -def join_alphabet_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(list("ABC..Z"))', - "join list of 26 characters, with 1 character sep", 1000) -def join_alphabet_list_single(STR): - sep = STR("A") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(list("ABC..Z"))', - "join list of 26 characters, with 5 character sep", 1000) -def join_alphabet_list_five(STR): - sep = STR("ABCDE") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(["Bob"]*100)', - "join list of 100 words, with 1 character sep", 1000) -def join_100_words_single(STR): - sep = STR("A") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(["Bob"]*100))', - "join list of 100 words, with 5 character sep", 1000) -def join_100_words_5(STR): - sep = STR("ABCDE") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -#### split tests - -@bench('("Here are some words. "*2).split()', "split whitespace (small)", 1000) -def whitespace_split(STR): - s = STR("Here are some words. "*2) - s_split = s.split - for x in _RANGE_1000: - s_split() - -@bench('("Here are some words. "*2).rsplit()', "split whitespace (small)", 1000) -def whitespace_rsplit(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit() - -@bench('("Here are some words. "*2).split(None, 1)', - "split 1 whitespace", 1000) -def whitespace_split_1(STR): - s = STR("Here are some words. "*2) - s_split = s.split - N = None - for x in _RANGE_1000: - s_split(N, 1) - -@bench('("Here are some words. "*2).rsplit(None, 1)', - "split 1 whitespace", 1000) -def whitespace_rsplit_1(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - N = None - for x in _RANGE_1000: - s_rsplit(N, 1) - -@bench('("Here are some words. "*2).partition(" ")', - "split 1 whitespace", 1000) -def whitespace_partition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_partition = s.partition - for x in _RANGE_1000: - s_partition(sep) - -@bench('("Here are some words. "*2).rpartition(" ")', - "split 1 whitespace", 1000) -def whitespace_rpartition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_rpartition = s.rpartition - for x in _RANGE_1000: - s_rpartition(sep) - -human_text = """\ -Python is a dynamic object-oriented programming language that can be -used for many kinds of software development. It offers strong support -for integration with other languages and tools, comes with extensive -standard libraries, and can be learned in a few days. Many Python -programmers report substantial productivity gains and feel the language -encourages the development of higher quality, more maintainable code. - -Python runs on Windows, Linux/Unix, Mac OS X, Amiga, Palm -Handhelds, and Nokia mobile phones. Python has also been ported to the -Java and .NET virtual machines. - -Python is distributed under an OSI-approved open source license that -makes it free to use, even for commercial products. -"""*25 -human_text_bytes = bytes_from_str(human_text) -human_text_unicode = unicode_from_str(human_text) -def _get_human_text(STR): - if STR is UNICODE: - return human_text_unicode - if STR is BYTES: - return human_text_bytes - raise AssertionError - -@bench('human_text.split()', "split whitespace (huge)", 10) -def whitespace_split_huge(STR): - s = _get_human_text(STR) - s_split = s.split - for x in _RANGE_10: - s_split() - -@bench('human_text.rsplit()', "split whitespace (huge)", 10) -def whitespace_rsplit_huge(STR): - s = _get_human_text(STR) - s_rsplit = s.rsplit - for x in _RANGE_10: - s_rsplit() - - - -@bench('"this\\nis\\na\\ntest\\n".split("\\n")', "split newlines", 1000) -def newlines_split(STR): - s = STR("this\nis\na\ntest\n") - s_split = s.split - nl = STR("\n") - for x in _RANGE_1000: - s_split(nl) - - -@bench('"this\\nis\\na\\ntest\\n".rsplit("\\n")', "split newlines", 1000) -def newlines_rsplit(STR): - s = STR("this\nis\na\ntest\n") - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_1000: - s_rsplit(nl) - -@bench('"this\\nis\\na\\ntest\\n".splitlines()', "split newlines", 1000) -def newlines_splitlines(STR): - s = STR("this\nis\na\ntest\n") - s_splitlines = s.splitlines - for x in _RANGE_1000: - s_splitlines() - -## split text with 2000 newlines - -def _make_2000_lines(): - import random - r = random.Random(100) - chars = list(map(chr, range(32, 128))) - i = 0 - while i < len(chars): - chars[i] = " " - i += r.randrange(9) - s = "".join(chars) - s = s*4 - words = [] - for i in range(2000): - start = r.randrange(96) - n = r.randint(5, 65) - words.append(s[start:start+n]) - return "\n".join(words)+"\n" - -_text_with_2000_lines = _make_2000_lines() -_text_with_2000_lines_bytes = bytes_from_str(_text_with_2000_lines) -_text_with_2000_lines_unicode = unicode_from_str(_text_with_2000_lines) -def _get_2000_lines(STR): - if STR is UNICODE: - return _text_with_2000_lines_unicode - if STR is BYTES: - return _text_with_2000_lines_bytes - raise AssertionError - - -@bench('"...text...".split("\\n")', "split 2000 newlines", 10) -def newlines_split_2000(STR): - s = _get_2000_lines(STR) - s_split = s.split - nl = STR("\n") - for x in _RANGE_10: - s_split(nl) - -@bench('"...text...".rsplit("\\n")', "split 2000 newlines", 10) -def newlines_rsplit_2000(STR): - s = _get_2000_lines(STR) - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_10: - s_rsplit(nl) - -@bench('"...text...".splitlines()', "split 2000 newlines", 10) -def newlines_splitlines_2000(STR): - s = _get_2000_lines(STR) - s_splitlines = s.splitlines - for x in _RANGE_10: - s_splitlines() - - -## split text on "--" characters -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".split("--")', - "split on multicharacter separator (small)", 1000) -def split_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_split = s.split - pat = STR("--") - for x in _RANGE_1000: - s_split(pat) -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".rsplit("--")', - "split on multicharacter separator (small)", 1000) -def rsplit_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_rsplit = s.rsplit - pat = STR("--") - for x in _RANGE_1000: - s_rsplit(pat) - -## split dna text on "ACTAT" characters -@bench('dna.split("ACTAT")', - "split on multicharacter separator (dna)", 10) -def split_multichar_sep_dna(STR): - s = _get_dna(STR) - s_split = s.split - pat = STR("ACTAT") - for x in _RANGE_10: - s_split(pat) - -@bench('dna.rsplit("ACTAT")', - "split on multicharacter separator (dna)", 10) -def rsplit_multichar_sep_dna(STR): - s = _get_dna(STR) - s_rsplit = s.rsplit - pat = STR("ACTAT") - for x in _RANGE_10: - s_rsplit(pat) - - - -## split with limits - -GFF3_example = "\t".join([ - "I", "Genomic_canonical", "region", "357208", "396183", ".", "+", ".", - "ID=Sequence:R119;note=Clone R119%3B Genbank AF063007;Name=R119"]) - -@bench('GFF3_example.split("\\t")', "tab split", 1000) -def tab_split_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep) - -@bench('GFF3_example.split("\\t", 8)', "tab split", 1000) -def tab_split_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep, 8) - -@bench('GFF3_example.rsplit("\\t")', "tab split", 1000) -def tab_rsplit_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep) - -@bench('GFF3_example.rsplit("\\t", 8)', "tab split", 1000) -def tab_rsplit_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep, 8) - -#### Count characters - -@bench('...text.with.2000.newlines.count("\\n")', - "count newlines", 10) -def count_newlines(STR): - s = _get_2000_lines(STR) - s_count = s.count - nl = STR("\n") - for x in _RANGE_10: - s_count(nl) - -# Orchid sequences concatenated, from Biopython -_dna = """ -CGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGGGTT -AATCTGGAGGATCTGTTTACTTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGAATTGCCATCG -AGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGCAGTTTTGCTCCAAGTCGTT -TGACACATAATTGGTGAAGGGGGTGGCATCCTTCCCTGACCCTCCCCCAACTATTTTTTTAACAACTCTC -AGCAACGGAGACTCAGTCTTCGGCAAATGCGATAAATGGTGTGAATTGCAGAATCCCGTGCACCATCGAG -TCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCATTGCGAGTCATAT -CTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCGGATGTGAGTTTGGCCCCTTGTTCTT -TGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAGGTGGACGAACTAT -GCTACAACAAAATTGTTGTGCAGAGGCCCCGGGTTGTCGTATTAGATGGGCCACCGTAATCTGAAGACCC -TTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGCGACCCCAGGTCAG -GTGAGCAACAGCTGTCGTAACAAGGTTTCCGTAGGGTGAACTGCGGAAGGATCATTGTTGAGATCACATA -ATAATTGATCGAGTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGAC -CTAGATTTGCCATCGAGCCTCCTTGGGAGCATCCTTGTTGGCGATATCTAAACCCTCAATTTTTCCCCCA -ATCAAATTACACAAAATTGGTGGAGGGGGTGGCATTCTTCCCTTACCCTCCCCCAAATATTTTTTTAACA -ACTCTCAGCAACGGATATCTCAGCTCTTGCATCGATGAAGAACCCACCGAAATGCGATAAATGGTGTGAA -TTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACG -CCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCG -GATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGATGCATGGGCTTTTGATGGTCCTAA -ATACGGCAAGAGGTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATAAG -ATGGGCCACCGATATCTGAAGACCCTTTTGGACCCCATTGGAGCCCATCAACCCATGTCAGTTGATGGCC -ATTCGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGA -GTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCA -TCGAGCCTCCTTGGGAGCTTTCTTGTTGGCGATATCTAAACCCTTGCCCGGCAGAGTTTTGGGAATCCCG -TGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCAT -TGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACACACCTGTTCAGCCGGTGCGGATGTGAGTTTG -GCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAG -GTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATTAGATGGGCCACCAT -AATCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGC -GACCCAGTCAGGTGAGGGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGAG -TTAATCTGGAGGATCTGTTTACTTTGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCAT -CGAGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTTGGCGCCAAGTCA -TATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAACAACTC -TCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGAATTGC -AGAATCCCGTGAACCATCGAGTCTTTGGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCT -GCCTGGGCATTGGGAATCATATCTCTCCCCTAACGAGGCTATCCAAACATACTGTTCATCCGGTGCGGAT -GTGAGTTTGGCCCCTTGTTCTTTGGTACCGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTCAAAA -CGGCAAGAGGTGGACGAACTATGCCACAACAAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTAGATG -GGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGACCA -TTTGTTGCGACCCCAGTCAGCTGAGCAACCCGCTGAGTGGAAGGTCATTGCCGATATCACATAATAATTG -ATCGAGTTAATCTGGAGGATCTGTTTACTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGATTT -GCCATCGAGCCTCCTTGGGAGTTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTGTGCGCCA -AGTCATATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAAC -AACTCTCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGA -ATTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCAC -GCCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCATCCGGTGC -GGATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTC -AAAACGGCAAGAGGTGGACGAACTATGCTACAACCAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTA -GATGGGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATG -ACCATGTGTTGCGACCCCAGTCAGCTGAGCAACGCGCTGAGCGTAACAAGGTTTCCGTAGGTGGACCTCC -GGGAGGATCATTGTTGAGATCACATAATAATTGATCGAGGTAATCTGGAGGATCTGCATATTTTGGTCAC -""" -_dna = "".join(_dna.splitlines()) -_dna = _dna * 25 -_dna_bytes = bytes_from_str(_dna) -_dna_unicode = unicode_from_str(_dna) - -def _get_dna(STR): - if STR is UNICODE: - return _dna_unicode - if STR is BYTES: - return _dna_bytes - raise AssertionError - -@bench('dna.count("AACT")', "count AACT substrings in DNA example", 10) -def count_aact(STR): - seq = _get_dna(STR) - seq_count = seq.count - needle = STR("AACT") - for x in _RANGE_10: - seq_count(needle) - -##### startswith and endswith - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Andrew")', 'startswith multiple characters', - 1000) -def startswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Anders")', - 'startswith multiple characters - not!', 1000) -def startswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - - -# endswith - -@bench('"Andrew".endswith("w")', 'endswith single character', 1000) -def endswith_single(STR): - s1 = STR("Andrew") - s2 = STR("w") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Andrew")', 'endswith multiple characters', 1000) -def endswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Anders")', - 'endswith multiple characters - not!', 1000) -def endswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -#### Strip - -@bench('"Hello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_right(STR): - s = STR("Hello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello!\\n".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_rstrip(STR): - s = STR("Hello!\n") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\nHello!".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_left(STR): - s = STR("\nHello!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_both(STR): - s = STR("\nHello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_lstrip(STR): - s = STR("\nHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - -@bench('s="Hello!\\n"; s[:-1] if s[-1]=="\\n" else s', - 'strip terminal newline', 1000) -def terminal_newline_if_else(STR): - s = STR("Hello!\n") - NL = STR("\n") - for x in _RANGE_1000: - s[:-1] if (s[-1] == NL) else s - - -# Strip multiple spaces or tabs - -@bench('"Hello\\t \\t".strip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_strip(STR): - s = STR("Hello\t \t!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello\\t \\t".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_rstrip(STR): - s = STR("Hello!\t \t") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\t \\tHello".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_lstrip(STR): - s = STR("\t \tHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - - -#### replace -@bench('"This is a test".replace(" ", "\\t")', 'replace single character', - 1000) -def replace_single_character(STR): - s = STR("This is a test!") - from_str = STR(" ") - to_str = STR("\t") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub(" ", "\\t", "This is a test"', 'replace single character', - 1000) -def replace_single_character_re(STR): - s = STR("This is a test!") - pat = re.compile(STR(" ")) - to_str = STR("\t") - pat_sub = pat.sub - for x in _RANGE_1000: - pat_sub(to_str, s) - -@bench('"...text.with.2000.lines...replace("\\n", " ")', - 'replace single character, big string', 10) -def replace_single_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR(" ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub("\\n", " ", "...text.with.2000.lines...")', - 'replace single character, big string', 10) -def replace_single_character_big_re(STR): - s = _get_2000_lines(STR) - pat = re.compile(STR("\n")) - to_str = STR(" ") - pat_sub = pat.sub - for x in _RANGE_10: - pat_sub(to_str, s) - - -@bench('dna.replace("ATC", "ATT")', - 'replace multiple characters, dna', 10) -def replace_multiple_characters_dna(STR): - seq = _get_dna(STR) - from_str = STR("ATC") - to_str = STR("ATT") - seq_replace = seq.replace - for x in _RANGE_10: - seq_replace(from_str, to_str) - -# This increases the character count -@bench('"...text.with.2000.newlines...replace("\\n", "\\r\\n")', - 'replace and expand multiple characters, big string', 10) -def replace_multiple_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR("\r\n") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - - -# This decreases the character count -@bench('"When shall we three meet again?".replace("ee", "")', - 'replace/remove multiple characters', 1000) -def replace_multiple_character_remove(STR): - s = STR("When shall we three meet again?") - from_str = STR("ee") - to_str = STR("") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - - -big_s = "A" + ("Z"*128*1024) -big_s_bytes = bytes_from_str(big_s) -big_s_unicode = unicode_from_str(big_s) -def _get_big_s(STR): - if STR is UNICODE: return big_s_unicode - if STR is BYTES: return big_s_bytes - raise AssertionError - -# The older replace implementation counted all matches in -# the string even when it only needed to make one replacement. -@bench('("A" + ("Z"*128*1024)).replace("A", "BB", 1)', - 'quick replace single character match', 10) -def quick_replace_single_match(STR): - s = _get_big_s(STR) - from_str = STR("A") - to_str = STR("BB") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - -@bench('("A" + ("Z"*128*1024)).replace("AZZ", "BBZZ", 1)', - 'quick replace multiple character match', 10) -def quick_replace_multiple_match(STR): - s = _get_big_s(STR) - from_str = STR("AZZ") - to_str = STR("BBZZ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - - -#### - -# CCP does a lot of this, for internationalisation of ingame messages. -_format = "The %(thing)s is %(place)s the %(location)s." -_format_dict = { "thing":"THING", "place":"PLACE", "location":"LOCATION", } -_format_bytes = bytes_from_str(_format) -_format_unicode = unicode_from_str(_format) -_format_dict_bytes = dict((bytes_from_str(k), bytes_from_str(v)) for (k,v) in _format_dict.items()) -_format_dict_unicode = dict((unicode_from_str(k), unicode_from_str(v)) for (k,v) in _format_dict.items()) - -def _get_format(STR): - if STR is UNICODE: - return _format_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_bytes - raise AssertionError - -def _get_format_dict(STR): - if STR is UNICODE: - return _format_dict_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_dict_bytes - raise AssertionError - -# Formatting. -@bench('"The %(k1)s is %(k2)s the %(k3)s."%{"k1":"x","k2":"y","k3":"z",}', - 'formatting a string type with a dict', 1000) -def format_with_dict(STR): - s = _get_format(STR) - d = _get_format_dict(STR) - for x in _RANGE_1000: - s % d - - -#### Upper- and lower- case conversion - -@bench('("Where in the world is Carmen San Deigo?"*10).lower()', - "case conversion -- rare", 1000) -def lower_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - -@bench('("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10).lower()', - "case conversion -- dense", 1000) -def lower_conversion_dense(STR): - s = STR("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - - -@bench('("wHERE IN THE WORLD IS cARMEN sAN dEIGO?"*10).upper()', - "case conversion -- rare", 1000) -def upper_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - -@bench('("where in the world is carmen san deigo?"*10).upper()', - "case conversion -- dense", 1000) -def upper_conversion_dense(STR): - s = STR("where in the world is carmen san deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - - -# end of benchmarks - -################# - -class BenchTimer(timeit.Timer): - def best(self, repeat=1): - for i in range(1, 10): - number = 10**i - x = self.timeit(number) - if x > 0.02: - break - times = [x] - for i in range(1, repeat): - times.append(self.timeit(number)) - return min(times) / number - -def main(): - (options, test_names) = parser.parse_args() - if options.bytes_only and options.unicode_only: - raise SystemExit("Only one of --8-bit and --unicode are allowed") - - bench_functions = [] - for (k,v) in globals().items(): - if hasattr(v, "is_bench"): - if test_names: - for name in test_names: - if name in v.group: - break - else: - # Not selected, ignore - continue - if options.skip_re and hasattr(v, "uses_re"): - continue - - bench_functions.append( (v.group, k, v) ) - bench_functions.sort() - - p("bytes\tunicode") - p("(in ms)\t(in ms)\t%\tcomment") - - bytes_total = uni_total = 0.0 - - for title, group in itertools.groupby(bench_functions, - operator.itemgetter(0)): - # Flush buffer before each group - sys.stdout.flush() - p("="*10, title) - for (_, k, v) in group: - if hasattr(v, "is_bench"): - bytes_time = 0.0 - bytes_time_s = " - " - if not options.unicode_only: - try: - bytes_time = BenchTimer("__main__.%s(__main__.BYTES)" % (k,), - "import __main__").best(REPEAT) - bytes_time_s = "%.2f" % (1000 * bytes_time) - bytes_total += bytes_time - except UnsupportedType: - bytes_time_s = "N/A" - uni_time = 0.0 - uni_time_s = " - " - if not options.bytes_only: - try: - uni_time = BenchTimer("__main__.%s(__main__.UNICODE)" % (k,), - "import __main__").best(REPEAT) - uni_time_s = "%.2f" % (1000 * uni_time) - uni_total += uni_time - except UnsupportedType: - uni_time_s = "N/A" - try: - average = bytes_time/uni_time - except (TypeError, ZeroDivisionError): - average = 0.0 - p("%s\t%s\t%.1f\t%s (*%d)" % ( - bytes_time_s, uni_time_s, 100.*average, - v.comment, v.repeat_count)) - - if bytes_total == uni_total == 0.0: - p("That was zippy!") - else: - try: - ratio = bytes_total/uni_total - except ZeroDivisionError: - ratio = 0.0 - p("%.2f\t%.2f\t%.1f\t%s" % ( - 1000*bytes_total, 1000*uni_total, 100.*ratio, - "TOTAL")) - -if __name__ == "__main__": - main() diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt new file mode 100644 index 00000000000000..0955387dfb8370 --- /dev/null +++ b/Tools/tsan/suppressions_free_threading.txt @@ -0,0 +1,52 @@ +# This file contains suppressions for the free-threaded build. It contains the +# suppressions for the default build and additional suppressions needed only in +# the free-threaded build. +# +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions + +## Default build suppresssions + +race:get_allocator_unlocked +race:set_allocator_unlocked + +## Free-threaded suppressions + + +# These entries are for warnings that trigger in a library function, as called +# by a CPython function. + +# https://gist.github.com/swtaarrs/8e0e365e1d9cecece3269a2fb2f2b8b8 +race:sock_recv_impl +# https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 +race:free_threadstate + + +# These warnings trigger directly in a CPython function. + +race_top:_PyEval_EvalFrameDefault +race_top:assign_version_tag +race_top:new_reference +race_top:_multiprocessing_SemLock_acquire_impl +race_top:list_get_item_ref +race_top:make_pending_calls +race_top:_Py_slot_tp_getattr_hook +race_top:add_threadstate +race_top:dump_traceback +race_top:fatal_error +race_top:_multiprocessing_SemLock_release_impl +race_top:_PyFrame_GetCode +race_top:_PyFrame_Initialize +race_top:PyInterpreterState_ThreadHead +race_top:_PyObject_TryGetInstanceAttribute +race_top:PyThreadState_Next +race_top:PyUnstable_InterpreterFrame_GetLine +race_top:tstate_delete_common +race_top:tstate_is_freed +race_top:type_modified_unlocked +race_top:write_thread_id +race_top:PyThreadState_Clear +# Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c +race_top:set_default_allocator_unlocked + +# https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 +thread:pthread_create diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt new file mode 100644 index 00000000000000..22ba9d6ba2ab4d --- /dev/null +++ b/Tools/tsan/supressions.txt @@ -0,0 +1,7 @@ +# This file contains suppressions for the default (with GIL) build. +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:get_allocator_unlocked +race:set_allocator_unlocked + +# https://gist.github.com/mpage/daaf32b39180c1989572957b943eb665 +thread:pthread_create diff --git a/Tools/unicode/dawg.py b/Tools/unicode/dawg.py new file mode 100644 index 00000000000000..b74d602f6e1d79 --- /dev/null +++ b/Tools/unicode/dawg.py @@ -0,0 +1,533 @@ +# Original Algorithm: +# By Steve Hanov, 2011. Released to the public domain. +# Please see http://stevehanov.ca/blog/index.php?id=115 for the accompanying article. +# +# Adapted for PyPy/CPython by Carl Friedrich Bolz-Tereick +# +# Based on Daciuk, Jan, et al. "Incremental construction of minimal acyclic finite-state automata." +# Computational linguistics 26.1 (2000): 3-16. +# +# Updated 2014 to use DAWG as a mapping; see +# Kowaltowski, T.; CL. Lucchesi (1993), "Applications of finite automata representing large vocabularies", +# Software-Practice and Experience 1993 + +from collections import defaultdict +from functools import cached_property + + +# This class represents a node in the directed acyclic word graph (DAWG). It +# has a list of edges to other nodes. It has functions for testing whether it +# is equivalent to another node. Nodes are equivalent if they have identical +# edges, and each identical edge leads to identical states. The __hash__ and +# __eq__ functions allow it to be used as a key in a python dictionary. + + +class DawgNode: + + def __init__(self, dawg): + self.id = dawg.next_id + dawg.next_id += 1 + self.final = False + self.edges = {} + + self.linear_edges = None # later: list of (string, next_state) + + def __str__(self): + if self.final: + arr = ["1"] + else: + arr = ["0"] + + for (label, node) in sorted(self.edges.items()): + arr.append(label) + arr.append(str(node.id)) + + return "_".join(arr) + __repr__ = __str__ + + def _as_tuple(self): + edges = sorted(self.edges.items()) + edge_tuple = tuple((label, node.id) for label, node in edges) + return (self.final, edge_tuple) + + def __hash__(self): + return hash(self._as_tuple()) + + def __eq__(self, other): + return self._as_tuple() == other._as_tuple() + + @cached_property + def num_reachable_linear(self): + # returns the number of different paths to final nodes reachable from + # this one + + count = 0 + # staying at self counts as a path if self is final + if self.final: + count += 1 + for label, node in self.linear_edges: + count += node.num_reachable_linear + + return count + + +class Dawg: + def __init__(self): + self.previous_word = "" + self.next_id = 0 + self.root = DawgNode(self) + + # Here is a list of nodes that have not been checked for duplication. + self.unchecked_nodes = [] + + # To deduplicate, maintain a dictionary with + # minimized_nodes[canonical_node] is canonical_node. + # Based on __hash__ and __eq__, minimized_nodes[n] is the + # canonical node equal to n. + # In other words, self.minimized_nodes[x] == x for all nodes found in + # the dict. + self.minimized_nodes = {} + + # word: value mapping + self.data = {} + # value: word mapping + self.inverse = {} + + def insert(self, word, value): + if not all(0 <= ord(c) < 128 for c in word): + raise ValueError("Use 7-bit ASCII characters only") + if word <= self.previous_word: + raise ValueError("Error: Words must be inserted in alphabetical order.") + if value in self.inverse: + raise ValueError(f"value {value} is duplicate, got it for word {self.inverse[value]} and now {word}") + + # find common prefix between word and previous word + common_prefix = 0 + for i in range(min(len(word), len(self.previous_word))): + if word[i] != self.previous_word[i]: + break + common_prefix += 1 + + # Check the unchecked_nodes for redundant nodes, proceeding from last + # one down to the common prefix size. Then truncate the list at that + # point. + self._minimize(common_prefix) + + self.data[word] = value + self.inverse[value] = word + + # add the suffix, starting from the correct node mid-way through the + # graph + if len(self.unchecked_nodes) == 0: + node = self.root + else: + node = self.unchecked_nodes[-1][2] + + for letter in word[common_prefix:]: + next_node = DawgNode(self) + node.edges[letter] = next_node + self.unchecked_nodes.append((node, letter, next_node)) + node = next_node + + node.final = True + self.previous_word = word + + def finish(self): + if not self.data: + raise ValueError("need at least one word in the dawg") + # minimize all unchecked_nodes + self._minimize(0) + + self._linearize_edges() + + topoorder, linear_data, inverse = self._topological_order() + return self.compute_packed(topoorder), linear_data, inverse + + def _minimize(self, down_to): + # proceed from the leaf up to a certain point + for i in range(len(self.unchecked_nodes) - 1, down_to - 1, -1): + (parent, letter, child) = self.unchecked_nodes[i] + if child in self.minimized_nodes: + # replace the child with the previously encountered one + parent.edges[letter] = self.minimized_nodes[child] + else: + # add the state to the minimized nodes. + self.minimized_nodes[child] = child + self.unchecked_nodes.pop() + + def _lookup(self, word): + """ Return an integer 0 <= k < number of strings in dawg + where word is the kth successful traversal of the dawg. """ + node = self.root + skipped = 0 # keep track of number of final nodes that we skipped + index = 0 + while index < len(word): + for label, child in node.linear_edges: + if word[index] == label[0]: + if word[index:index + len(label)] == label: + if node.final: + skipped += 1 + index += len(label) + node = child + break + else: + return None + skipped += child.num_reachable_linear + else: + return None + return skipped + + def enum_all_nodes(self): + stack = [self.root] + done = set() + while stack: + node = stack.pop() + if node.id in done: + continue + yield node + done.add(node.id) + for label, child in sorted(node.edges.items()): + stack.append(child) + + def prettyprint(self): + for node in sorted(self.enum_all_nodes(), key=lambda e: e.id): + s_final = " final" if node.final else "" + print(f"{node.id}: ({node}) {s_final}") + for label, child in sorted(node.edges.items()): + print(f" {label} goto {child.id}") + + def _inverse_lookup(self, number): + assert 0, "not working in the current form, but keep it as the pure python version of compact lookup" + result = [] + node = self.root + while 1: + if node.final: + if pos == 0: + return "".join(result) + pos -= 1 + for label, child in sorted(node.edges.items()): + nextpos = pos - child.num_reachable_linear + if nextpos < 0: + result.append(label) + node = child + break + else: + pos = nextpos + else: + assert 0 + + def _linearize_edges(self): + # compute "linear" edges. the idea is that long chains of edges without + # any of the intermediate states being final or any extra incoming or + # outgoing edges can be represented by having removing them, and + # instead using longer strings as edge labels (instead of single + # characters) + incoming = defaultdict(list) + nodes = sorted(self.enum_all_nodes(), key=lambda e: e.id) + for node in nodes: + for label, child in sorted(node.edges.items()): + incoming[child].append(node) + for node in nodes: + node.linear_edges = [] + for label, child in sorted(node.edges.items()): + s = [label] + while len(child.edges) == 1 and len(incoming[child]) == 1 and not child.final: + (c, child), = child.edges.items() + s.append(c) + node.linear_edges.append((''.join(s), child)) + + def _topological_order(self): + # compute reachable linear nodes, and the set of incoming edges for each node + order = [] + stack = [self.root] + seen = set() + while stack: + # depth first traversal + node = stack.pop() + if node.id in seen: + continue + seen.add(node.id) + order.append(node) + for label, child in node.linear_edges: + stack.append(child) + + # do a (slightly bad) topological sort + incoming = defaultdict(set) + for node in order: + for label, child in node.linear_edges: + incoming[child].add((label, node)) + no_incoming = [order[0]] + topoorder = [] + positions = {} + while no_incoming: + node = no_incoming.pop() + topoorder.append(node) + positions[node] = len(topoorder) + # use "reversed" to make sure that the linear_edges get reorderd + # from their alphabetical order as little as necessary (no_incoming + # is LIFO) + for label, child in reversed(node.linear_edges): + incoming[child].discard((label, node)) + if not incoming[child]: + no_incoming.append(child) + del incoming[child] + # check result + assert set(topoorder) == set(order) + assert len(set(topoorder)) == len(topoorder) + + for node in order: + node.linear_edges.sort(key=lambda element: positions[element[1]]) + + for node in order: + for label, child in node.linear_edges: + assert positions[child] > positions[node] + # number the nodes. afterwards every input string in the set has a + # unique number in the 0 <= number < len(data). We then put the data in + # self.data into a linear list using these numbers as indexes. + topoorder[0].num_reachable_linear + linear_data = [None] * len(self.data) + inverse = {} # maps value back to index + for word, value in self.data.items(): + index = self._lookup(word) + linear_data[index] = value + inverse[value] = index + + return topoorder, linear_data, inverse + + def compute_packed(self, order): + def compute_chunk(node, offsets): + """ compute the packed node/edge data for a node. result is a + list of bytes as long as order. the jump distance calculations use + the offsets dictionary to know where in the final big output + bytestring the individual nodes will end up. """ + result = bytearray() + offset = offsets[node] + encode_varint_unsigned(number_add_bits(node.num_reachable_linear, node.final), result) + if len(node.linear_edges) == 0: + assert node.final + encode_varint_unsigned(0, result) # add a 0 saying "done" + prev_child_offset = offset + len(result) + for edgeindex, (label, targetnode) in enumerate(node.linear_edges): + label = label.encode('ascii') + child_offset = offsets[targetnode] + child_offset_difference = child_offset - prev_child_offset + + info = number_add_bits(child_offset_difference, len(label) == 1, edgeindex == len(node.linear_edges) - 1) + if edgeindex == 0: + assert info != 0 + encode_varint_unsigned(info, result) + prev_child_offset = child_offset + if len(label) > 1: + encode_varint_unsigned(len(label), result) + result.extend(label) + return result + + def compute_new_offsets(chunks, offsets): + """ Given a list of chunks, compute the new offsets (by adding the + chunk lengths together). Also check if we cannot shrink the output + further because none of the node offsets are smaller now. if that's + the case return None. """ + new_offsets = {} + curr_offset = 0 + should_continue = False + for node, result in zip(order, chunks): + if curr_offset < offsets[node]: + # the new offset is below the current assumption, this + # means we can shrink the output more + should_continue = True + new_offsets[node] = curr_offset + curr_offset += len(result) + if not should_continue: + return None + return new_offsets + + # assign initial offsets to every node + offsets = {} + for i, node in enumerate(order): + # we don't know position of the edge yet, just use something big as + # the starting position. we'll have to do further iterations anyway, + # but the size is at least a lower limit then + offsets[node] = i * 2 ** 30 + + + # due to the variable integer width encoding of edge targets we need to + # run this to fixpoint. in the process we shrink the output more and + # more until we can't any more. at any point we can stop and use the + # output, but we might need padding zero bytes when joining the chunks + # to have the correct jump distances + last_offsets = None + while 1: + chunks = [compute_chunk(node, offsets) for node in order] + last_offsets = offsets + offsets = compute_new_offsets(chunks, offsets) + if offsets is None: # couldn't shrink + break + + # build the final packed string + total_result = bytearray() + for node, result in zip(order, chunks): + node_offset = last_offsets[node] + if node_offset > len(total_result): + # need to pad to get the offsets correct + padding = b"\x00" * (node_offset - len(total_result)) + total_result.extend(padding) + assert node_offset == len(total_result) + total_result.extend(result) + return bytes(total_result) + + +# ______________________________________________________________________ +# the following functions operate on the packed representation + +def number_add_bits(x, *bits): + for bit in bits: + assert bit == 0 or bit == 1 + x = (x << 1) | bit + return x + +def encode_varint_unsigned(i, res): + # https://en.wikipedia.org/wiki/LEB128 unsigned variant + more = True + startlen = len(res) + if i < 0: + raise ValueError("only positive numbers supported", i) + while more: + lowest7bits = i & 0b1111111 + i >>= 7 + if i == 0: + more = False + else: + lowest7bits |= 0b10000000 + res.append(lowest7bits) + return len(res) - startlen + +def number_split_bits(x, n, acc=()): + if n == 1: + return x >> 1, x & 1 + if n == 2: + return x >> 2, (x >> 1) & 1, x & 1 + assert 0, "implement me!" + +def decode_varint_unsigned(b, index=0): + res = 0 + shift = 0 + while True: + byte = b[index] + res = res | ((byte & 0b1111111) << shift) + index += 1 + shift += 7 + if not (byte & 0b10000000): + return res, index + +def decode_node(packed, node): + x, node = decode_varint_unsigned(packed, node) + node_count, final = number_split_bits(x, 1) + return node_count, final, node + +def decode_edge(packed, edgeindex, prev_child_offset, offset): + x, offset = decode_varint_unsigned(packed, offset) + if x == 0 and edgeindex == 0: + raise KeyError # trying to decode past a final node + child_offset_difference, len1, last_edge = number_split_bits(x, 2) + child_offset = prev_child_offset + child_offset_difference + if len1: + size = 1 + else: + size, offset = decode_varint_unsigned(packed, offset) + return child_offset, last_edge, size, offset + +def _match_edge(packed, s, size, node_offset, stringpos): + if size > 1 and stringpos + size > len(s): + # past the end of the string, can't match + return False + for i in range(size): + if packed[node_offset + i] != s[stringpos + i]: + # if a subsequent char of an edge doesn't match, the word isn't in + # the dawg + if i > 0: + raise KeyError + return False + return True + +def lookup(packed, data, s): + return data[_lookup(packed, s)] + +def _lookup(packed, s): + stringpos = 0 + node_offset = 0 + skipped = 0 # keep track of number of final nodes that we skipped + false = False + while stringpos < len(s): + #print(f"{node_offset=} {stringpos=}") + _, final, edge_offset = decode_node(packed, node_offset) + prev_child_offset = edge_offset + edgeindex = 0 + while 1: + child_offset, last_edge, size, edgelabel_chars_offset = decode_edge(packed, edgeindex, prev_child_offset, edge_offset) + #print(f" {edge_offset=} {child_offset=} {last_edge=} {size=} {edgelabel_chars_offset=}") + edgeindex += 1 + prev_child_offset = child_offset + if _match_edge(packed, s, size, edgelabel_chars_offset, stringpos): + # match + if final: + skipped += 1 + stringpos += size + node_offset = child_offset + break + if last_edge: + raise KeyError + descendant_count, _, _ = decode_node(packed, child_offset) + skipped += descendant_count + edge_offset = edgelabel_chars_offset + size + _, final, _ = decode_node(packed, node_offset) + if final: + return skipped + raise KeyError + +def inverse_lookup(packed, inverse, x): + pos = inverse[x] + return _inverse_lookup(packed, pos) + +def _inverse_lookup(packed, pos): + result = bytearray() + node_offset = 0 + while 1: + node_count, final, edge_offset = decode_node(packed, node_offset) + if final: + if pos == 0: + return bytes(result) + pos -= 1 + prev_child_offset = edge_offset + edgeindex = 0 + while 1: + child_offset, last_edge, size, edgelabel_chars_offset = decode_edge(packed, edgeindex, prev_child_offset, edge_offset) + edgeindex += 1 + prev_child_offset = child_offset + descendant_count, _, _ = decode_node(packed, child_offset) + nextpos = pos - descendant_count + if nextpos < 0: + assert edgelabel_chars_offset >= 0 + result.extend(packed[edgelabel_chars_offset: edgelabel_chars_offset + size]) + node_offset = child_offset + break + elif not last_edge: + pos = nextpos + edge_offset = edgelabel_chars_offset + size + else: + raise KeyError + else: + raise KeyError + + +def build_compression_dawg(ucdata): + d = Dawg() + ucdata.sort() + for name, value in ucdata: + d.insert(name, value) + packed, pos_to_code, reversedict = d.finish() + print("size of dawg [KiB]", round(len(packed) / 1024, 2)) + # check that lookup and inverse_lookup work correctly on the input data + for name, value in ucdata: + assert lookup(packed, pos_to_code, name.encode('ascii')) == value + assert inverse_lookup(packed, reversedict, value) == name.encode('ascii') + return packed, pos_to_code diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 6bf5274551c00a..a5ac09eb96f19a 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -623,120 +623,12 @@ def makeunicodetype(unicode, trace): # unicode name database def makeunicodename(unicode, trace): + from dawg import build_compression_dawg FILE = "Modules/unicodename_db.h" print("--- Preparing", FILE, "...") - # collect names - names = [None] * len(unicode.chars) - - for char in unicode.chars: - record = unicode.table[char] - if record: - name = record.name.strip() - if name and name[0] != "<": - names[char] = name + chr(0) - - print(len([n for n in names if n is not None]), "distinct names") - - # collect unique words from names (note that we differ between - # words inside a sentence, and words ending a sentence. the - # latter includes the trailing null byte. - - words = {} - n = b = 0 - for char in unicode.chars: - name = names[char] - if name: - w = name.split() - b = b + len(name) - n = n + len(w) - for w in w: - l = words.get(w) - if l: - l.append(None) - else: - words[w] = [len(words)] - - print(n, "words in text;", b, "bytes") - - wordlist = list(words.items()) - - # sort on falling frequency, then by name - def word_key(a): - aword, alist = a - return -len(alist), aword - wordlist.sort(key=word_key) - - # figure out how many phrasebook escapes we need - escapes = 0 - while escapes * 256 < len(wordlist): - escapes = escapes + 1 - print(escapes, "escapes") - - short = 256 - escapes - - assert short > 0 - - print(short, "short indexes in lexicon") - - # statistics - n = 0 - for i in range(short): - n = n + len(wordlist[i][1]) - print(n, "short indexes in phrasebook") - - # pick the most commonly used words, and sort the rest on falling - # length (to maximize overlap) - - wordlist, wordtail = wordlist[:short], wordlist[short:] - wordtail.sort(key=lambda a: a[0], reverse=True) - wordlist.extend(wordtail) - - # generate lexicon from words - - lexicon_offset = [0] - lexicon = "" - words = {} - - # build a lexicon string - offset = 0 - for w, x in wordlist: - # encoding: bit 7 indicates last character in word (chr(128) - # indicates the last character in an entire string) - ww = w[:-1] + chr(ord(w[-1])+128) - # reuse string tails, when possible - o = lexicon.find(ww) - if o < 0: - o = offset - lexicon = lexicon + ww - offset = offset + len(w) - words[w] = len(lexicon_offset) - lexicon_offset.append(o) - - lexicon = list(map(ord, lexicon)) - - # generate phrasebook from names and lexicon - phrasebook = [0] - phrasebook_offset = [0] * len(unicode.chars) - for char in unicode.chars: - name = names[char] - if name: - w = name.split() - phrasebook_offset[char] = len(phrasebook) - for w in w: - i = words[w] - if i < short: - phrasebook.append(i) - else: - # store as two bytes - phrasebook.append((i>>8) + short) - phrasebook.append(i&255) - - assert getsize(phrasebook) == 1 - - # # unicode name hash table # extract names @@ -748,12 +640,6 @@ def word_key(a): if name and name[0] != "<": data.append((name, char)) - # the magic number 47 was chosen to minimize the number of - # collisions on the current data set. if you like, change it - # and see what happens... - - codehash = Hash("code", data, 47) - print("--- Writing", FILE, "...") with open(FILE, "w") as fp: @@ -762,24 +648,22 @@ def word_key(a): fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) fprint() fprint("#define NAME_MAXLEN", 256) + assert max(len(x) for x in data) < 256 fprint() - fprint("/* lexicon */") - Array("lexicon", lexicon).dump(fp, trace) - Array("lexicon_offset", lexicon_offset).dump(fp, trace) - - # split decomposition index table - offset1, offset2, shift = splitbins(phrasebook_offset, trace) - - fprint("/* code->name phrasebook */") - fprint("#define phrasebook_shift", shift) - fprint("#define phrasebook_short", short) - - Array("phrasebook", phrasebook).dump(fp, trace) - Array("phrasebook_offset1", offset1).dump(fp, trace) - Array("phrasebook_offset2", offset2).dump(fp, trace) fprint("/* name->code dictionary */") - codehash.dump(fp, trace) + packed_dawg, pos_to_codepoint = build_compression_dawg(data) + notfound = len(pos_to_codepoint) + inverse_list = [notfound] * len(unicode.chars) + for pos, codepoint in enumerate(pos_to_codepoint): + inverse_list[codepoint] = pos + Array("packed_name_dawg", list(packed_dawg)).dump(fp, trace) + Array("dawg_pos_to_codepoint", pos_to_codepoint).dump(fp, trace) + index1, index2, shift = splitbins(inverse_list, trace) + fprint("#define DAWG_CODEPOINT_TO_POS_SHIFT", shift) + fprint("#define DAWG_CODEPOINT_TO_POS_NOTFOUND", notfound) + Array("dawg_codepoint_to_pos_index1", index1).dump(fp, trace) + Array("dawg_codepoint_to_pos_index2", index2).dump(fp, trace) fprint() fprint('static const unsigned int aliases_start = %#x;' % @@ -1188,94 +1072,6 @@ def uselatin1(self): self.chars = list(range(256)) -# hash table tools - -# this is a straight-forward reimplementation of Python's built-in -# dictionary type, using a static data structure, and a custom string -# hash algorithm. - -def myhash(s, magic): - h = 0 - for c in map(ord, s.upper()): - h = (h * magic) + c - ix = h & 0xff000000 - if ix: - h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff - return h - - -SIZES = [ - (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17), - (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3), - (65536,45), (131072,9), (262144,39), (524288,39), (1048576,9), - (2097152,5), (4194304,3), (8388608,33), (16777216,27) -] - - -class Hash: - def __init__(self, name, data, magic): - # turn a (key, value) list into a static hash table structure - - # determine table size - for size, poly in SIZES: - if size > len(data): - poly = size + poly - break - else: - raise AssertionError("ran out of polynomials") - - print(size, "slots in hash table") - - table = [None] * size - - mask = size-1 - - n = 0 - - hash = myhash - - # initialize hash table - for key, value in data: - h = hash(key, magic) - i = (~h) & mask - v = table[i] - if v is None: - table[i] = value - continue - incr = (h ^ (h >> 3)) & mask - if not incr: - incr = mask - while 1: - n = n + 1 - i = (i + incr) & mask - v = table[i] - if v is None: - table[i] = value - break - incr = incr << 1 - if incr > mask: - incr = incr ^ poly - - print(n, "collisions") - self.collisions = n - - for i in range(len(table)): - if table[i] is None: - table[i] = 0 - - self.data = Array(name + "_hash", table) - self.magic = magic - self.name = name - self.size = size - self.poly = poly - - def dump(self, file, trace): - # write data to file, as a C array - self.data.dump(file, trace) - file.write("#define %s_magic %d\n" % (self.name, self.magic)) - file.write("#define %s_size %d\n" % (self.name, self.size)) - file.write("#define %s_poly %d\n" % (self.name, self.poly)) - # stuff to deal with arrays of unsigned integers diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 8ef63c6dcd9ddc..bc3e4ba8bd5b76 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,6 +1,7 @@ # Python WebAssembly (WASM) build -**WARNING: WASM support is work-in-progress! Lots of features are not working yet.** +**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** +**Emscripten is NOT officially supported as of Python 3.13.** This directory contains configuration and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). Python supports Emscripten @@ -83,7 +84,7 @@ embuilder --pic build zlib bzip2 MINIMAL_PIC ``` -#### Compile a build Python interpreter +### Compile and build Python interpreter From within the container, run the following command: @@ -274,7 +275,7 @@ Node builds use ``NODERAWFS``. ### Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environents with cross-origin +browsers only provide the feature in secure environments with cross-origin isolation. The webserver must send cross-origin headers and correct MIME types for the JavaScript and WASM files. Otherwise the terminal will fail to load with an error message like ``Browsers disable shared array buffer``. @@ -298,102 +299,9 @@ AddType application/wasm wasm ## WASI (wasm32-wasi) -WASI builds require the [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+. -See `.devcontainer/Dockerfile` for an example of how to download and -install the WASI SDK. +See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). -### Build - -The script ``wasi-env`` sets necessary compiler and linker flags as well as -``pkg-config`` overrides. The script assumes that WASI-SDK is installed in -``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. - -There are two scripts you can use to do a WASI build from a source checkout. You can either use: - -```shell -./Tools/wasm/wasm_build.py wasi build -``` - -or: -```shell -./Tools/wasm/build_wasi.sh -``` - -The commands are equivalent to the following steps: - -- Make sure `Modules/Setup.local` exists -- Make sure the necessary build tools are installed: - - [WASI SDK](https://github.com/WebAssembly/wasi-sdk) (which ships with `clang`) - - `make` - - `pkg-config` (on Linux) -- Create the build Python - - `mkdir -p builddir/build` - - `pushd builddir/build` - - Get the build platform - - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` - - Shell: `../../config.guess` - - `../../configure -C` - - `make all` - - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` - - `popd` -- Create the host/WASI Python - - `mkdir builddir/wasi` - - `pushd builddir/wasi` - - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` - - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` - - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` - - Maps the source checkout to `/` in the WASI runtime - - Stdlib gets loaded from `/Lib` - - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` - - Set by `wasi-env` - - `WASI_SDK_PATH` - - `WASI_SYSROOT` - - `CC` - - `CPP` - - `CXX` - - `LDSHARED` - - `AR` - - `RANLIB` - - `CFLAGS` - - `LDFLAGS` - - `PKG_CONFIG_PATH` - - `PKG_CONFIG_LIBDIR` - - `PKG_CONFIG_SYSROOT_DIR` - - `PATH` - - `make all` - - -### Running - -If you followed the instructions above, you can run the interpreter via e.g., `wasmtime` from within the `Tools/wasi` directory (make sure to set/change `$PYTHON_VERSION` and do note the paths are relative to running in`builddir/wasi` for simplicity only): - -```shell -wasmtime run --mapdir /::../.. --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION python.wasm -- -``` - -There are also helpers provided by `Tools/wasm/wasm_build.py` as listed below. Also, if you used `Tools/wasm/build_wasi.sh`, a `run_wasi.sh` file will be created in `builddir/wasi` which will run the above command for you (it also uses absolute paths, so it can be executed from anywhere). - -#### REPL - -```shell -./Tools/wasm/wasm_build.py wasi repl -``` - -#### Tests - -```shell -./Tools/wasm/wasm_build.py wasi test -``` - -### Debugging - -* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The - feature is currently broken, see - https://github.com/bytecodealliance/wasmtime/issues/4669 . -* The environment variable ``RUST_LOG=wasi_common`` enables debug and - trace logging. - -## Detect WebAssembly builds +## Detecting WebAssembly builds ### Python code @@ -402,15 +310,17 @@ import os, sys if sys.platform == "emscripten": # Python on Emscripten + ... if sys.platform == "wasi": # Python on WASI + ... if os.name == "posix": # WASM platforms identify as POSIX-like. # Windows does not provide os.uname(). machine = os.uname().machine if machine.startswith("wasm"): - # WebAssembly (wasm32, wasm64 in the future) + # WebAssembly (wasm32, wasm64 potentially in the future) ``` ```python diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 5e98775400f6ea..4a1a466a4ab3f1 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -40,3 +40,12 @@ ac_cv_header_netpacket_packet_h=no # Disable int-conversion for wask-sdk as it triggers an error from version 17. ac_cv_disable_int_conversion=yes + +# preadv(), readv(), pwritev(), and writev() under wasmtime's WASI 0.2 support +# do not use more than the first buffer provided, failing under test_posix. +# Since wasmtime will not be changing this behaviour, disable the functions. +# https://github.com/bytecodealliance/wasmtime/issues/7830 +ac_cv_func_preadv=no +ac_cv_func_readv=no +ac_cv_func_pwritev=no +ac_cv_func_writev=no diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini index c62598f89eba69..4de0a30c260f5f 100644 --- a/Tools/wasm/mypy.ini +++ b/Tools/wasm/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = Tools/wasm +files = Tools/wasm/wasm_*.py pretty = True show_traceback = True @@ -9,6 +9,3 @@ python_version = 3.8 # Be strict... strict = True enable_error_code = truthy-bool,ignore-without-code - -# except for incomplete defs, which are useful for module authors: -disallow_incomplete_defs = False diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html index 17ffa0ea8bfeff..81a035a5c4cd93 100644 --- a/Tools/wasm/python.html +++ b/Tools/wasm/python.html @@ -35,11 +35,12 @@

    Simple REPL for Python WASM

    -
    +
    + +
    +
    The simple REPL provides a limited Python experience in the browser. diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js index 1b794608fffe7b..4ce4e16fc0fa19 100644 --- a/Tools/wasm/python.worker.js +++ b/Tools/wasm/python.worker.js @@ -19,18 +19,18 @@ class StdinBuffer { } stdin = () => { - if (this.numberOfCharacters + 1 === this.readIndex) { + while (this.numberOfCharacters + 1 === this.readIndex) { if (!this.sentNull) { // Must return null once to indicate we're done for now. this.sentNull = true return null } this.sentNull = false + // Prompt will reset this.readIndex to 1 this.prompt() } const char = this.buffer[this.readIndex] this.readIndex += 1 - // How do I send an EOF?? return char } } @@ -71,7 +71,11 @@ var Module = { onmessage = (event) => { if (event.data.type === 'run') { - // TODO: Set up files from event.data.files + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents) + } + } const ret = callMain(event.data.args) postMessage({ type: 'finished', diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 48908b02e60b96..95eda863cb62c6 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -55,7 +55,6 @@ if command -v ccache >/dev/null 2>&1; then CXX="ccache ${CXX}" fi -LDSHARED="${WASI_SDK_PATH}/bin/wasm-ld" AR="${WASI_SDK_PATH}/bin/llvm-ar" RANLIB="${WASI_SDK_PATH}/bin/ranlib" @@ -72,5 +71,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -# no exec, it makes arvg[0] path absolute. +# no exec, it makes argv[0] path absolute. "$@" diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py new file mode 100644 index 00000000000000..a14f58bdac0cb2 --- /dev/null +++ b/Tools/wasm/wasi.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + + +CHECKOUT = pathlib.Path(__file__).parent.parent.parent + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-wasi" +HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output(["tput", "cols"], + encoding="utf-8") + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("ðŸ“", working_dir) + if (clean_ok and getattr(context, "clean", False) and + working_dir.exists()): + print(f"🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("â¯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log") + stderr = subprocess.STDOUT + print(f"📠Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{BUILD_DIR}") + + return binary + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + print(f"👠{LOCAL_SETUP} exists ...") + else: + print(f"📠Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) + + binary = build_python_path() + cmd = [binary, "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +def find_wasi_sdk(): + """Find the path to wasi-sdk.""" + if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): + return pathlib.Path(wasi_sdk_path) + elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): + return default_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", + "AR": "llvm-ar", "RANLIB": "ranlib"} + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map(os.fsdecode, + [sysroot / "lib" / "pkgconfig", + sysroot / "share" / "pkgconfig"])) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"]]) + + return env + + +@subdir(HOST_DIR, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError("WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk") + + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + if pydebug: + sysconfig_data += "-pydebug" + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + args = {"GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm"} + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError("wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime") + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}"] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call(configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + print(f"ðŸƒâ€â™€ï¸ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call(["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [configure_build_python, make_build_python, configure_wasi_python, + make_wasi_python] + for step in steps: + step(context) + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"🧹 Deleting generated {LOCAL_SETUP} ...") + + +def main(): + default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Use WASI 0.2 primitives. + "--wasi preview2 " + # Enable thread support; causes use of preview1. + #"--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build-python", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build-python", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)") + make_host = subcommands.add_parser("make-host", + help="Run `make` for the host/WASI") + clean = subcommands.add_parser("clean", help="Delete files and directories " + "created by this script") + for subcommand in build, configure_build, make_build, configure_host, make_host: + subcommand.add_argument("--quiet", action="store_true", default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file") + for subcommand in configure_build, configure_host: + subcommand.add_argument("--clean", action="store_true", default=False, + dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + for subcommand in build, configure_host: + subcommand.add_argument("--wasi-sdk", type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") + subcommand.add_argument("--host-runner", action="store", + default=default_host_runner, dest="host_runner", + help="Command template for running the WASI host " + "(default designed for wasmtime 14 or newer: " + f"`{default_host_runner}`)") + + context = parser.parse_args() + + dispatch = {"configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents} + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index c0b9999a5dad03..bcb80212362b71 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -329,8 +329,10 @@ def _check_wasi() -> None: # workaround for https://github.com/python/cpython/issues/95952 "HOSTRUNNER": ( "wasmtime run " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " - "--mapdir /::{srcdir} --" + "--wasm max-wasm-stack=16777216 " + "--wasi preview2 " + "--dir {srcdir}::/ " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" ), "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, diff --git a/aclocal.m4 b/aclocal.m4 index da8ee95b9c7f6b..832aec19f48f17 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,4 +1,4 @@ -# generated automatically by aclocal 1.16.4 -*- Autoconf -*- +# generated automatically by aclocal 1.16.5 -*- Autoconf -*- # Copyright (C) 1996-2021 Free Software Foundation, Inc. @@ -150,6 +150,80 @@ AS_VAR_IF(CACHEVAR,yes, AS_VAR_POPDEF([CACHEVAR])dnl ])dnl AX_CHECK_COMPILE_FLAGS +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC + # =========================================================================== # https://www.gnu.org/software/autoconf-archive/ax_check_openssl.html # =========================================================================== @@ -276,7 +350,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ ]) # pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -# serial 11 (pkg-config-0.29.1) +# serial 12 (pkg-config-0.29.2) dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -318,7 +392,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29.1]) +[m4_define([PKG_MACROS_VERSION], [0.29.2]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ @@ -419,7 +493,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no -AC_MSG_CHECKING([for $1]) +AC_MSG_CHECKING([for $2]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) @@ -429,11 +503,11 @@ and $1[]_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details.]) if test $pkg_failed = yes; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` - else + else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs @@ -450,7 +524,7 @@ installed software in a non-standard prefix. _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full @@ -551,74 +625,6 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR -dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------ -dnl -dnl Prepare a "--with-" configure option using the lowercase -dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and -dnl PKG_CHECK_MODULES in a single macro. -AC_DEFUN([PKG_WITH_MODULES], -[ -m4_pushdef([with_arg], m4_tolower([$1])) - -m4_pushdef([description], - [m4_default([$5], [build with ]with_arg[ support])]) - -m4_pushdef([def_arg], [m4_default([$6], [auto])]) -m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) -m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) - -m4_case(def_arg, - [yes],[m4_pushdef([with_without], [--without-]with_arg)], - [m4_pushdef([with_without],[--with-]with_arg)]) - -AC_ARG_WITH(with_arg, - AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, - [AS_TR_SH([with_]with_arg)=def_arg]) - -AS_CASE([$AS_TR_SH([with_]with_arg)], - [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], - [auto],[PKG_CHECK_MODULES([$1],[$2], - [m4_n([def_action_if_found]) $3], - [m4_n([def_action_if_not_found]) $4])]) - -m4_popdef([with_arg]) -m4_popdef([description]) -m4_popdef([def_arg]) - -])dnl PKG_WITH_MODULES - -dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ----------------------------------------------- -dnl -dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES -dnl check._[VARIABLE-PREFIX] is exported as make variable. -AC_DEFUN([PKG_HAVE_WITH_MODULES], -[ -PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) - -AM_CONDITIONAL([HAVE_][$1], - [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) -])dnl PKG_HAVE_WITH_MODULES - -dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------------------ -dnl -dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after -dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make -dnl and preprocessor variable. -AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], -[ -PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) - -AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], - [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) -])dnl PKG_HAVE_DEFINE_WITH_MODULES - # AM_CONDITIONAL -*- Autoconf -*- # Copyright (C) 1997-2021 Free Software Foundation, Inc. diff --git a/config.sub b/config.sub index d74fb6deac942a..1bb6a05dc11026 100755 --- a/config.sub +++ b/config.sub @@ -1,14 +1,15 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2024 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-08-14' +# Patched 2024-02-03 to include support for arm64_32 and iOS/tvOS/watchOS simulators +timestamp='2024-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -76,13 +77,13 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2024 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -130,7 +131,7 @@ IFS=$saved_IFS # Separate into logical components for further validation case $1 in *-*-*-*-*) - echo Invalid configuration \`"$1"\': more than four components >&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) @@ -145,7 +146,8 @@ case $1 in nto-qnx* | linux-* | uclinux-uclibc* \ | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) + | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + | windows-* ) basic_machine=$field1 basic_os=$maybe_os ;; @@ -943,7 +945,7 @@ $basic_machine EOF IFS=$saved_IFS ;; - # We use `pc' rather than `unknown' + # We use 'pc' rather than 'unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) @@ -1020,6 +1022,11 @@ case $cpu-$vendor in ;; # Here we normalize CPU types with a missing or matching vendor + armh-unknown | armh-alt) + cpu=armv7l + vendor=alt + basic_os=${basic_os:-linux-gnueabihf} + ;; dpx20-unknown | dpx20-bull) cpu=rs6000 vendor=bull @@ -1070,7 +1077,7 @@ case $cpu-$vendor in pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) cpu=i586 ;; - pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + pentiumpro-* | p6-* | 6x86-* | athlon-* | athlon_*-*) cpu=i686 ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) @@ -1121,7 +1128,7 @@ case $cpu-$vendor in xscale-* | xscalee[bl]-*) cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; - arm64-*) + arm64-* | aarch64le-* | arm64_32-*) cpu=aarch64 ;; @@ -1175,7 +1182,7 @@ case $cpu-$vendor in case $cpu in 1750a | 580 \ | a29k \ - | aarch64 | aarch64_be \ + | aarch64 | aarch64_be | aarch64c | arm64ec \ | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ @@ -1194,50 +1201,29 @@ case $cpu-$vendor in | d10v | d30v | dlx | dsp16xx \ | e2k | elxsi | epiphany \ | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | javascript \ | h8300 | h8500 \ | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i*86 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ + | kvx \ | le32 | le64 \ | lm32 \ - | loongarch32 | loongarch64 | loongarchx32 \ + | loongarch32 | loongarch64 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ | m88110 | m88k | maxq | mb | mcore | mep | metag \ | microblaze | microblazeel \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64eb | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r3 | mipsisa32r3el \ - | mipsisa32r5 | mipsisa32r5el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r3 | mipsisa64r3el \ - | mipsisa64r5 | mipsisa64r5el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ + | mips* \ | mmix \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ + | nanomips* \ | nds32 | nds32le | nds32be \ | nfp \ | nios | nios2 | nios2eb | nios2el \ @@ -1269,6 +1255,7 @@ case $cpu-$vendor in | ubicom32 \ | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ | vax \ + | vc4 \ | visium \ | w65 \ | wasm32 | wasm64 \ @@ -1280,7 +1267,7 @@ case $cpu-$vendor in ;; *) - echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1301,11 +1288,12 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if test x$basic_os != x +if test x"$basic_os" != x then -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just +# First recognize some ad-hoc cases, or perhaps split kernel-os, or else just # set os. +obj= case $basic_os in gnu/linux*) kernel=linux @@ -1336,6 +1324,10 @@ EOF kernel=linux os=`echo "$basic_os" | sed -e 's|linux|gnu|'` ;; + managarm*) + kernel=managarm + os=`echo "$basic_os" | sed -e 's|managarm|mlibc|'` + ;; *) kernel= os=$basic_os @@ -1501,10 +1493,16 @@ case $os in os=eabi ;; *) - os=elf + os= + obj=elf ;; esac ;; + aout* | coff* | elf* | pe*) + # These are machine code file formats, not OSes + obj=$os + os= + ;; *) # No normalization, but not necessarily accepted, that comes below. ;; @@ -1523,12 +1521,15 @@ else # system, and we'll never get to this point. kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 @@ -1538,28 +1539,35 @@ case $cpu-$vendor in os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1581,19 +1589,24 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf ;; mips*-cisco) - os=elf + os= + obj=elf ;; - mips*-*) - os=elf + mips*-*|nanomips*-*) + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=sysv3 @@ -1602,7 +1615,8 @@ case $cpu-$vendor in os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1683,10 +1697,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1704,10 +1720,11 @@ esac fi -# Now, validate our (potentially fixed-up) OS. +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + case $os in # Sometimes we do "kernel-libc", so those need to count as OSes. - musl* | newlib* | relibc* | uclibc*) + llvm* | musl* | newlib* | relibc* | uclibc*) ;; # Likewise for "kernel-abi" eabi* | gnueabi*) @@ -1715,6 +1732,9 @@ case $os in # VxWorks passes extra cpu info in the 4th filed. simlinux | simwindows | spe) ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1723,7 +1743,7 @@ case $os in | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* \ + | os9* | macos* | osx* | ios* | tvos* | watchos* \ | mpw* | magic* | mmixware* | mon960* | lnews* \ | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ | aos* | aros* | cloudabi* | sortix* | twizzler* \ @@ -1732,11 +1752,11 @@ case $os in | mirbsd* | netbsd* | dicos* | openedition* | ose* \ | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ - | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | bosx* | nextstep* | cxux* | oabi* \ + | ptx* | ecoff* | winnt* | domain* | vsta* \ | udi* | lites* | ieee* | go32* | aux* | hcos* \ | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | cygwin* | msys* | moss* | proelf* | rtems* \ | midipix* | mingw32* | mingw64* | mint* \ | uxpv* | beos* | mpeix* | udk* | moxiebox* \ | interix* | uwin* | mks* | rhapsody* | darwin* \ @@ -1748,49 +1768,119 @@ case $os in | skyos* | haiku* | rdos* | toppers* | drops* | es* \ | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ - | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr*) + | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ + | fiwix* | mlibc* | cos* | mbr* | ironclad* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) # Don't forget version if it is 3.2v4 or newer. ;; + # This refers to builds using the UEFI calling convention + # (which depends on the architecture) and PE file format. + # Note that this is both a different calling convention and + # different file format than that of GNU-EFI + # (x86_64-w64-mingw32). + uefi) + ;; none) ;; + kernel* | msvc* ) + # Restricted further below + ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; *) - echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; + *) + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 exit 1 ;; esac # As a final step for OS-related things, validate the OS-kernel combination # (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* \ - | linux-musl* | linux-relibc* | linux-uclibc* ) +case $kernel-$os-$obj in + linux-gnu*- | linux-android*- | linux-dietlibc*- | linux-llvm*- \ + | linux-mlibc*- | linux-musl*- | linux-newlib*- \ + | linux-relibc*- | linux-uclibc*- ) + ;; + uclinux-uclibc*- ) ;; - uclinux-uclibc* ) + managarm-mlibc*- | managarm-kernel*- ) ;; - -dietlibc* | -newlib* | -musl* | -relibc* | -uclibc* ) + windows*-msvc*-) + ;; + -dietlibc*- | -llvm*- | -mlibc*- | -musl*- | -newlib*- | -relibc*- \ + | -uclibc*- ) # These are just libc implementations, not actual OSes, and thus # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 exit 1 ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 + exit 1 ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 + exit 1 ;; - nto-qnx*) + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 ;; - os2-emx) + kfreebsd*-gnu*- | kopensolaris*-gnu*-) ;; - *-eabi* | *-gnueabi*) + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; - -*) + nto-qnx*-) + ;; + os2-emx-) + ;; + *-eabi*- | *-gnueabi*-) + ;; + ios*-simulator- | tvos*-simulator- | watchos*-simulator- ) + ;; + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format + ;; + -*-) # Blank kernel with real OS is always fine. ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 exit 1 ;; esac @@ -1873,7 +1963,7 @@ case $vendor in ;; esac -echo "$cpu-$vendor-${kernel:+$kernel-}$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/configure b/configure index c87f518382f2ec..19786f18e61726 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for python 3.13. +# Generated by GNU Autoconf 2.71 for python 3.14. # # Report bugs to . # @@ -611,8 +611,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.13' -PACKAGE_STRING='python 3.13' +PACKAGE_VERSION='3.14' +PACKAGE_STRING='python 3.14' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -661,6 +661,10 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE +MODULE__TESTEXTERNALINSPECTION_FALSE +MODULE__TESTEXTERNALINSPECTION_TRUE +MODULE__TESTSINGLEPHASE_FALSE +MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE MODULE__TESTMULTIPHASE_TRUE MODULE__TESTIMPORTMULTIPLE_FALSE @@ -669,6 +673,8 @@ MODULE__TESTBUFFER_FALSE MODULE__TESTBUFFER_TRUE MODULE__TESTINTERNALCAPI_FALSE MODULE__TESTINTERNALCAPI_TRUE +MODULE__TESTLIMITEDCAPI_FALSE +MODULE__TESTLIMITEDCAPI_TRUE MODULE__TESTCLINIC_LIMITED_FALSE MODULE__TESTCLINIC_LIMITED_TRUE MODULE__TESTCLINIC_FALSE @@ -769,10 +775,12 @@ MODULE__MULTIPROCESSING_FALSE MODULE__MULTIPROCESSING_TRUE MODULE__ZONEINFO_FALSE MODULE__ZONEINFO_TRUE -MODULE__XXINTERPCHANNELS_FALSE -MODULE__XXINTERPCHANNELS_TRUE -MODULE__XXSUBINTERPRETERS_FALSE -MODULE__XXSUBINTERPRETERS_TRUE +MODULE__INTERPQUEUES_FALSE +MODULE__INTERPQUEUES_TRUE +MODULE__INTERPCHANNELS_FALSE +MODULE__INTERPCHANNELS_TRUE +MODULE__INTERPRETERS_FALSE +MODULE__INTERPRETERS_TRUE MODULE__TYPING_FALSE MODULE__TYPING_TRUE MODULE__STRUCT_FALSE @@ -787,8 +795,6 @@ MODULE__POSIXSUBPROCESS_FALSE MODULE__POSIXSUBPROCESS_TRUE MODULE__PICKLE_FALSE MODULE__PICKLE_TRUE -MODULE__OPCODE_FALSE -MODULE__OPCODE_TRUE MODULE__LSPROF_FALSE MODULE__LSPROF_TRUE MODULE__JSON_FALSE @@ -832,7 +838,8 @@ LIBPL PY_ENABLE_SHARED PLATLIBDIR BINLIBDEST -LIBPYTHON +MODULE_LDFLAGS +MODULE_DEPS_SHARED EXT_SUFFIX ALT_SOABI SOABI @@ -860,6 +867,8 @@ DTRACE_OBJS DTRACE_HEADERS DFLAGS DTRACE +INSTALL_MIMALLOC +MIMALLOC_HEADERS GDBM_LIBS GDBM_CFLAGS X11_LIBS @@ -869,6 +878,7 @@ TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS LIBMPDEC_INTERNAL +LIBMPDEC_LIBS LIBMPDEC_CFLAGS MODULE__CTYPES_MALLOC_CLOSURE LIBFFI_LIBS @@ -916,6 +926,9 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE +JIT_STENCILS_H +REGEN_JIT_COMMAND +ABI_THREAD ABIFLAGS LN MKDIR_P @@ -963,10 +976,14 @@ LDFLAGS CFLAGS CC HAS_XCRUN +IPHONEOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM -MACHDEP +APP_STORE_COMPLIANCE_PATCH_FLAGS +APP_STORE_COMPLIANCE_PATCH_TARGET +APP_STORE_COMPLIANCE_PATCH +INSTALLTARGETS FRAMEWORKINSTALLAPPSPREFIX FRAMEWORKUNIXTOOLSPREFIX FRAMEWORKPYTHONW @@ -974,6 +991,8 @@ FRAMEWORKALTINSTALLLAST FRAMEWORKALTINSTALLFIRST FRAMEWORKINSTALLLAST FRAMEWORKINSTALLFIRST +RESSRCDIR +PYTHONFRAMEWORKINSTALLNAMEPREFIX PYTHONFRAMEWORKINSTALLDIR PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKDIR @@ -983,6 +1002,7 @@ LIPO_INTEL64_FLAGS LIPO_32BIT_FLAGS ARCH_RUN_32BIT UNIVERSALSDK +MACHDEP PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1058,6 +1078,7 @@ enable_universalsdk with_universal_archs with_framework_name enable_framework +with_app_store_compliance with_emscripten_target enable_wasm_dynamic_linking enable_wasm_pthreads @@ -1070,6 +1091,7 @@ with_pydebug with_trace_refs enable_pystats with_assertions +enable_experimental_jit enable_optimizations with_lto enable_bolt @@ -1078,6 +1100,7 @@ with_dsymutil with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer +with_thread_sanitizer with_hash_algorithm with_tzpath with_libs @@ -1088,6 +1111,7 @@ enable_loadable_sqlite_extensions with_dbmliborder enable_ipv6 with_doc_strings +with_mimalloc with_pymalloc with_freelists with_c_locale_coercion @@ -1128,6 +1152,8 @@ LIBUUID_CFLAGS LIBUUID_LIBS LIBFFI_CFLAGS LIBFFI_LIBS +LIBMPDEC_CFLAGS +LIBMPDEC_LIBS LIBSQLITE3_CFLAGS LIBSQLITE3_LIBS TCLTK_CFLAGS @@ -1700,7 +1726,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.13 to adapt to many kinds of systems. +\`configure' configures python 3.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1766,7 +1792,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.13:";; + short | recursive ) echo "Configuration of python 3.14:";; esac cat <<\_ACEOF @@ -1795,6 +1821,9 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) + --enable-experimental-jit[=no|yes|yes-off|interpreter] + build the experimental just-in-time compiler + (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer @@ -1812,9 +1841,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.13 + --with-build-python=python3.14 path to build python binary for cross compiling - (default: _bootstrap_python or python3.13) + (default: _bootstrap_python or python3.14) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -1829,6 +1858,10 @@ Optional Packages: specify the name for the python framework on macOS only valid when --enable-framework is set. see Mac/README.rst (default is 'Python') + --with-app-store-compliance=[PATCH-FILE] + Enable any patches required for compiliance with app + stores. Optional PATCH-FILE specifies the custom + patch to apply. --with-emscripten-target=[browser|node] Emscripten platform --with-suffix=SUFFIX set executable suffix to SUFFIX (default is empty, @@ -1855,6 +1888,8 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) + --with-thread-sanitizer enable ThreadSanitizer data race detector, 'tsan' + (default is no) --with-hash-algorithm=[fnv|siphash13|siphash24] select hash algorithm for use in Python/pyhash.c (default is SipHash13) @@ -1864,8 +1899,9 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed libmpdec - library, see Doc/library/decimal.rst (default is no) + --with-system-libmpdec build _decimal module using an installed mpdecimal + library, see Doc/library/decimal.rst (default is + yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -1874,6 +1910,8 @@ Optional Packages: value is a colon separated string with the backend names `ndbm', `gdbm' and `bdb'. --with-doc-strings enable documentation strings (default is yes) + --with-mimalloc build with mimalloc memory allocator (default is yes + if C11 stdatomic.h is available.) --with-pymalloc enable specialized mallocs (default is yes) --with-freelists enable object freelists (default is yes) --with-c-locale-coercion @@ -1942,6 +1980,10 @@ Some influential environment variables: LIBFFI_CFLAGS C compiler flags for LIBFFI, overriding pkg-config LIBFFI_LIBS linker flags for LIBFFI, overriding pkg-config + LIBMPDEC_CFLAGS + C compiler flags for LIBMPDEC, overriding pkg-config + LIBMPDEC_LIBS + linker flags for LIBMPDEC, overriding pkg-config LIBSQLITE3_CFLAGS C compiler flags for LIBSQLITE3, overriding pkg-config LIBSQLITE3_LIBS @@ -2047,7 +2089,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.13 +python configure 3.14 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2704,7 +2746,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.13, which was +It was created by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -3791,7 +3833,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.13 +VERSION=3.14 # Version number of Python's own shared library file. @@ -3988,6 +4030,127 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi +# Set name for machine-dependent library files + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 +printf %s "checking MACHDEP... " >&6; } +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + as_fn_error $? "cross build not supported for $host" "$LINENO" 5 + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '/ ' | tr '[A-Z]' '[a-z]'` + ac_md_release=`echo $ac_sys_release | + tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` + +printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h + + fi +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 +printf "%s\n" "\"$MACHDEP\"" >&6; } + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios12.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-universalsdk" >&5 printf %s "checking for --enable-universalsdk... " >&6; } # Check whether --enable-universalsdk was given. @@ -4103,109 +4266,154 @@ then : enableval=$enable_framework; case $enableval in yes) - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + esac esac + case $enableval in no) - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + # Add files for Mac specific code to the list of output + # files: + ac_config_files="$ac_config_files Mac/Makefile" - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; - esac + ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" - # Add files for Mac specific code to the list of output - # files: - ac_config_files="$ac_config_files Mac/Makefile" + ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" - ac_config_files="$ac_config_files Mac/PythonLauncher/Makefile" + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" - ac_config_files="$ac_config_files Mac/Resources/framework/Info.plist" + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources - ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" + ac_config_files="$ac_config_files iOS/Resources/Info.plist" - esac + ;; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + ;; + esac + esac else $as_nop - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= - + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac fi @@ -4223,79 +4431,76 @@ fi -printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h -# Set name for machine-dependent library files -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 -printf %s "checking MACHDEP... " >&6; } -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - as_fn_error $? "cross build not supported for $host" "$LINENO" 5 - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '/ ' | tr '[A-Z]' '[a-z]'` - ac_md_release=`echo $ac_sys_release | - tr -d '/ ' | sed 's/^[A-Z]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" +printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-app-store-compliance" >&5 +printf %s "checking for --with-app-store-compliance... " >&6; } + +# Check whether --with-app_store_compliance was given. +if test ${with_app_store_compliance+y} +then : + withval=$with_app_store_compliance; + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + ;; + *) as_fn_error $? "no default app store compliance patch available for $ac_sys_system" "$LINENO" 5 ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying custom app store compliance patch" >&5 +printf "%s\n" "applying custom app store compliance patch" >&6; } + ;; esac - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\(0-9\)$!.0\1!g' | tr -d '.'` +else $as_nop -printf "%s\n" "#define Py_SUNOS_VERSION $SUNOS_VERSION" >>confdefs.h + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + Darwin) + # Always *check* the compliance patch on macOS + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS="--dry-run" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: checking (not applying) default app store compliance patch" >&5 +printf "%s\n" "checking (not applying) default app store compliance patch" >&6; } + ;; + *) + # No app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + APP_STORE_COMPLIANCE_PATCH_TARGET= + APP_STORE_COMPLIANCE_PATCH_FLAGS= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not patching for app store compliance" >&5 +printf "%s\n" "not patching for app store compliance" >&6; } + ;; + esac - fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 -printf "%s\n" "\"$MACHDEP\"" >&6; } + + + + if test "$cross_compiling" = yes; then @@ -4303,27 +4508,45 @@ if test "$cross_compiling" = yes; then *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" as_fn_error $? "cross build not supported for $host" "$LINENO" 5 esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -4390,6 +4613,9 @@ printf "%s\n" "#define _BSD_SOURCE 1" >>confdefs.h define_xopen_source=no;; Darwin/[12][0-9].*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -4452,6 +4678,9 @@ fi CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. + + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -4484,6 +4713,16 @@ case $host in #( ;; esac +case $ac_sys_system in #( + iOS) : + + as_fn_append CFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Extract the first word of "xcrun", so it can be a program name with args. @@ -5951,7 +6190,9 @@ then : else $as_nop cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` + if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then + ac_cv_cc_name="mpicc" + fi else ac_cv_cc_name="unknown" fi @@ -6153,6 +6397,34 @@ fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GCC compatible compiler" >&5 +printf %s "checking for GCC compatible compiler... " >&6; } +if test ${ac_cv_gcc_compat+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + ac_cv_gcc_compat=yes +else $as_nop + ac_cv_gcc_compat=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_compat" >&5 +printf "%s\n" "$ac_cv_gcc_compat" >&6; } + + preset_cxx="$CXX" if test -z "$CXX" @@ -6746,6 +7018,8 @@ printf %s "checking for multiarch... " >&6; } case $ac_sys_system in #( Darwin*) : MULTIARCH="" ;; #( + iOS) : + MULTIARCH="" ;; #( FreeBSD*) : MULTIARCH="" ;; #( *) : @@ -6753,8 +7027,6 @@ case $ac_sys_system in #( ;; esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 -printf "%s\n" "$MULTIARCH" >&6; } if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -6764,6 +7036,16 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MULTIARCH" >&5 +printf "%s\n" "$MULTIARCH" >&6; } + +case $ac_sys_system in #( + iOS) : + SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2` ;; #( + *) : + SOABI_PLATFORM=$PLATFORM_TRIPLET + ;; +esac if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -6777,18 +7059,20 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=1 ;; #( x86_64-apple-darwin*/clang) : PY_SUPPORT_TIER=1 ;; #( + aarch64-apple-darwin*/clang) : + PY_SUPPORT_TIER=1 ;; #( i686-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( x86_64-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( - aarch64-apple-darwin*/clang) : - PY_SUPPORT_TIER=2 ;; #( - aarch64-*-linux-gnu/gcc) : + aarch64-*-linux-gnu/gcc) : PY_SUPPORT_TIER=2 ;; #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : + PY_SUPPORT_TIER=2 ;; #( + wasm32-unknown-wasi/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6801,14 +7085,19 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( s390x-*-linux-gnu/gcc) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-emscripten/clang) : + x86_64-*-freebsd*/clang) : + PY_SUPPORT_TIER=3 ;; #( + aarch64-apple-ios*-simulator/clang) : PY_SUPPORT_TIER=3 ;; #( - wasm32-unknown-wasi/clang) : + aarch64-apple-ios*/clang) : PY_SUPPORT_TIER=3 ;; #( - x86_64-*-freebsd*/clang) : + aarch64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( + x86_64-*-linux-android/clang) : PY_SUPPORT_TIER=3 ;; #( *) : - PY_SUPPORT_TIER=0 + + PY_SUPPORT_TIER=0 ;; esac @@ -6892,6 +7181,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 @@ -7257,17 +7549,25 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking LDLIBRARY" >&5 printf %s "checking LDLIBRARY... " >&6; } -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + as_fn_error $? "Unknown platform for framework build" "$LINENO" 5;; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -7280,66 +7580,80 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 +printf "%s\n" "$LDLIBRARY" >&6; } if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +printf %s "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then @@ -7516,15 +7830,13 @@ then : fi ;; #( WASI/*) : - HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --' ;; #( + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -printf %s "checking HOSTRUNNER... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 printf "%s\n" "$HOSTRUNNER" >&6; } @@ -7532,9 +7844,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 -printf "%s\n" "$LDLIBRARY" >&6; } - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : @@ -7852,7 +8161,9 @@ fi # For calculating the .so ABI tag. + ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -7878,10 +8189,11 @@ printf "%s\n" "$disable_gil" >&6; } if test "$disable_gil" = "yes" then -printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h +printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -7987,6 +8299,48 @@ else printf "%s\n" "no" >&6; } fi +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac +if ${tier2_flags:+false} : +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " $tier2_flags" +fi +if ${jit_flags:+false} : +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " $jit_flags" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } + # Enable optimization flags @@ -8025,13 +8379,13 @@ if test "$Py_OPT" = 'true' ; then *gcc*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-semantic-interposition" >&5 printf %s "checking whether C compiler accepts -fno-semantic-interposition... " >&6; } -if test ${ax_cv_check_cflags___fno_semantic_interposition+y} +if test ${ax_cv_check_cflags__Werror__fno_semantic_interposition+y} then : printf %s "(cached) " >&6 else $as_nop ax_check_save_flags=$CFLAGS - CFLAGS="$CFLAGS -fno-semantic-interposition" + CFLAGS="$CFLAGS -Werror -fno-semantic-interposition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -8045,16 +8399,16 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ax_cv_check_cflags___fno_semantic_interposition=yes + ax_cv_check_cflags__Werror__fno_semantic_interposition=yes else $as_nop - ax_cv_check_cflags___fno_semantic_interposition=no + ax_cv_check_cflags__Werror__fno_semantic_interposition=no fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fno_semantic_interposition" >&5 -printf "%s\n" "$ax_cv_check_cflags___fno_semantic_interposition" >&6; } -if test "x$ax_cv_check_cflags___fno_semantic_interposition" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fno_semantic_interposition" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fno_semantic_interposition" >&6; } +if test "x$ax_cv_check_cflags__Werror__fno_semantic_interposition" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" @@ -8610,9 +8964,9 @@ case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -8626,9 +8980,9 @@ case "$CC_BASENAME" in case $ac_sys_system in Darwin*) PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test "${LLVM_PROF_FOUND}" = "not-found" then LLVM_PROF_ERR=yes @@ -9135,6 +9489,11 @@ then : PYDEBUG_CFLAGS="-Og" fi +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line @@ -9268,15 +9627,20 @@ then : # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. as_fn_append CFLAGS " -target wasm32-wasi-threads -pthread" as_fn_append CFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -Wl,--import-memory" + as_fn_append LDFLAGS_NODIST " -Wl,--export-memory" as_fn_append LDFLAGS_NODIST " -Wl,--max-memory=10485760" fi - as_fn_append LDFLAGS_NODIST " -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760" + as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( *) : @@ -9316,6 +9680,87 @@ else $as_nop BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" fi +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 +printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; } +if test ${ax_cv_check_cflags__Werror__fstack_protector_strong+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -fstack-protector-strong" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__fstack_protector_strong=yes +else $as_nop + ax_cv_check_cflags__Werror__fstack_protector_strong=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fstack_protector_strong" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fstack_protector_strong" >&6; } +if test "x$ax_cv_check_cflags__Werror__fstack_protector_strong" = xyes +then : + BASECFLAGS="$BASECFLAGS -fstack-protector-strong" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -fstack-protector-strong not supported" >&5 +printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 +printf %s "checking whether C compiler accepts -Wtrampolines... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wtrampolines+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wtrampolines" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wtrampolines=yes +else $as_nop + ax_cv_check_cflags__Werror__Wtrampolines=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wtrampolines" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wtrampolines" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wtrampolines" = xyes +then : + BASECFLAGS="$BASECFLAGS -Wtrampolines" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} +fi + + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -9330,7 +9775,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wextra -Werror" + as_fn_append CFLAGS " -Wextra -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9448,7 +9893,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-result -Werror" + as_fn_append CFLAGS " -Wunused-result -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9493,7 +9938,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-parameter -Werror" + as_fn_append CFLAGS " -Wunused-parameter -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9534,7 +9979,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wint-conversion -Werror" + as_fn_append CFLAGS " -Wint-conversion -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9575,7 +10020,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wmissing-field-initializers -Werror" + as_fn_append CFLAGS " -Wmissing-field-initializers -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9616,7 +10061,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wsign-compare -Werror" + as_fn_append CFLAGS " -Wsign-compare -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9657,7 +10102,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunreachable-code -Werror" + as_fn_append CFLAGS " -Wunreachable-code -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9709,7 +10154,7 @@ then : else $as_nop py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wstrict-prototypes -Werror" + as_fn_append CFLAGS " -Wstrict-prototypes -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -11136,42 +11581,22 @@ then : fi -# checks for typedefs - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_t in time.h" >&5 -printf %s "checking for clock_t in time.h... " >&6; } -if test ${ac_cv_clock_t_time_h+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "clock_t" >/dev/null 2>&1 +# Check for clock_t in time.h. +ac_fn_c_check_type "$LINENO" "clock_t" "ac_cv_type_clock_t" "#include +" +if test "x$ac_cv_type_clock_t" = xyes then : - ac_cv_clock_t_time_h=yes -else $as_nop - ac_cv_clock_t_time_h=no -fi -rm -rf conftest* +printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_clock_t_time_h" >&5 -printf "%s\n" "$ac_cv_clock_t_time_h" >&6; } -if test "x$ac_cv_clock_t_time_h" = xno -then : +else $as_nop printf "%s\n" "#define clock_t long" >>confdefs.h - fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for makedev" >&5 printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} @@ -11412,12 +11837,16 @@ then : printf "%s\n" "#define HAVE_SSIZE_T 1" >>confdefs.h + fi ac_fn_c_check_type "$LINENO" "__uint128_t" "ac_cv_type___uint128_t" "$ac_includes_default" if test "x$ac_cv_type___uint128_t" = xyes then : +printf "%s\n" "#define HAVE___UINT128_T 1" >>confdefs.h + + printf "%s\n" "#define HAVE_GCC_UINT128_T 1" >>confdefs.h fi @@ -12494,6 +12923,28 @@ with_ubsan="no" fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-thread-sanitizer" >&5 +printf %s "checking for --with-thread-sanitizer... " >&6; } + +# Check whether --with-thread_sanitizer was given. +if test ${with_thread_sanitizer+y} +then : + withval=$with_thread_sanitizer; +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +printf "%s\n" "$withval" >&6; } +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" + +else $as_nop + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +with_tsan="no" + +fi + + # Set info about shared libraries. @@ -12597,7 +13048,12 @@ then BLDSHARED="$LDSHARED" fi ;; - Emscripten|WASI) + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + BLDSHARED="$LDSHARED" + ;; + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -12685,7 +13141,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) if test "x$enable_wasm_dynamic_linking" = xyes @@ -12726,30 +13181,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi - - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -13030,8 +13489,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBUUID" >&5 -printf %s "checking for LIBUUID... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid >= 2.20" >&5 +printf %s "checking for uuid >= 2.20... " >&6; } if test -n "$LIBUUID_CFLAGS"; then pkg_cv_LIBUUID_CFLAGS="$LIBUUID_CFLAGS" @@ -13071,7 +13530,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -13211,7 +13670,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -13540,6 +13999,51 @@ printf "%s\n" "$AIX_BUILDDATE" >&6; } *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +if test "$cross_compiling" = yes +then : + ac_cv_c_complex_supported=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +} +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_c_complex_supported=yes +else $as_nop + ac_cv_c_complex_supported=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +if test "$ac_cv_c_complex_supported" = "yes"; then + +printf "%s\n" "#define Py_HAVE_C_COMPLEX 1" >>confdefs.h + +fi + # check for systems that require aligned memory access { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 printf %s "checking aligned memory access is required... " >&6; } @@ -13549,7 +14053,14 @@ then : else $as_nop if test "$cross_compiling" = yes then : + +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else ac_cv_aligned_required=yes +fi else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -13919,8 +14430,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBFFI" >&5 -printf %s "checking for LIBFFI... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libffi" >&5 +printf %s "checking for libffi... " >&6; } if test -n "$LIBFFI_CFLAGS"; then pkg_cv_LIBFFI_CFLAGS="$LIBFFI_CFLAGS" @@ -13960,7 +14471,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -14047,7 +14558,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -14138,6 +14649,10 @@ then : ctypes_malloc_closure=yes ;; #( + iOS) : + + ctypes_malloc_closure=yes + ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" ;; #( @@ -14303,47 +14818,174 @@ if test ${with_system_libmpdec+y} then : withval=$with_system_libmpdec; else $as_nop - with_system_libmpdec="no" + with_system_libmpdec="yes" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 printf "%s\n" "$with_system_libmpdec" >&6; } -if test "x$with_system_libmpdec" = xyes -then : - - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= - -else $as_nop - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - if test "x$with_pydebug" = xyes +if test "x$with_system_libmpdec" = xyes then : - as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 +printf %s "checking for libmpdec >= 2.5.0... " >&6; } +if test -n "$LIBMPDEC_CFLAGS"; then + pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi - + else + pkg_failed=untried +fi +if test -n "$LIBMPDEC_LIBS"; then + pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } -# Check whether _decimal should use a coroutine-local or thread-local context -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-decimal-contextvar" >&5 -printf %s "checking for --with-decimal-contextvar... " >&6; } - -# Check whether --with-decimal_contextvar was given. -if test ${with_decimal_contextvar+y} -then : - withval=$with_decimal_contextvar; -else $as_nop - with_decimal_contextvar="yes" +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + else + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBMPDEC_PKG_ERRORS" >&5 + + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +else + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +fi +else $as_nop + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi + +if test "x$with_system_libmpdec" = xyes +then : + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + +int +main (void) +{ +const char *x = mpd_version(); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + have_mpdec=yes +else $as_nop + have_mpdec=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} +fi + +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi + +# Disable forced inlining in debug builds, see GH-94847 +if test "x$with_pydebug" = xyes +then : + as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +fi + +# Check whether _decimal should use a coroutine-local or thread-local context +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-decimal-contextvar" >&5 +printf %s "checking for --with-decimal-contextvar... " >&6; } + +# Check whether --with-decimal_contextvar was given. +if test ${with_decimal_contextvar+y} +then : + withval=$with_decimal_contextvar; +else $as_nop + with_decimal_contextvar="yes" fi @@ -14357,51 +14999,53 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } -# Check for libmpdec machine flavor -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 +if test "x$with_system_libmpdec" = xno +then : + # Check for libmpdec machine flavor + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 printf %s "checking for decimal libmpdec machine... " >&6; } -case $ac_sys_system in #( + case $ac_sys_system in #( Darwin*) : libmpdec_system=Darwin ;; #( SunOS*) : libmpdec_system=sunos ;; #( *) : libmpdec_system=other - ;; + ;; esac -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $CC in #( + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + case $CC in #( *gcc*) : libmpdec_machine=ppro ;; #( *clang*) : libmpdec_machine=ppro ;; #( *) : libmpdec_machine=ansi32 - ;; + ;; esac - else - libmpdec_machine=ansi32 - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 + else + libmpdec_machine=ansi32 + fi + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 printf "%s\n" "$libmpdec_machine" >&6; } -case $libmpdec_machine in #( + case $libmpdec_machine in #( x64) : as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DASM=1" ;; #( uint128) : @@ -14418,8 +15062,9 @@ case $libmpdec_machine in #( as_fn_append LIBMPDEC_CFLAGS " -DUNIVERSAL=1" ;; #( *) : as_fn_error $? "_decimal: unsupported architecture" "$LINENO" 5 - ;; + ;; esac +fi if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -14438,6 +15083,9 @@ fi + + + if test "$ac_sys_system" = "Emscripten" -a -z "$LIBSQLITE3_CFLAGS" -a -z "$LIBSQLITE3_LIBS" then : @@ -14451,8 +15099,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBSQLITE3" >&5 -printf %s "checking for LIBSQLITE3... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3 >= 3.15.2" >&5 +printf %s "checking for sqlite3 >= 3.15.2... " >&6; } if test -n "$LIBSQLITE3_CFLAGS"; then pkg_cv_LIBSQLITE3_CFLAGS="$LIBSQLITE3_CFLAGS" @@ -14492,7 +15140,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -14514,7 +15162,7 @@ fi elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } LIBSQLITE3_CFLAGS=${LIBSQLITE3_CFLAGS-""} @@ -15215,8 +15863,8 @@ for _QUERY in \ pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for TCLTK" >&5 -printf %s "checking for TCLTK... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $_QUERY" >&5 +printf %s "checking for $_QUERY... " >&6; } if test -n "$TCLTK_CFLAGS"; then pkg_cv_TCLTK_CFLAGS="$TCLTK_CFLAGS" @@ -15256,7 +15904,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -15274,7 +15922,7 @@ fi found_tcltk=no elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } found_tcltk=no else @@ -15312,8 +15960,8 @@ case $ac_sys_system in #( pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for X11" >&5 -printf %s "checking for X11... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for x11" >&5 +printf %s "checking for x11... " >&6; } if test -n "$X11_CFLAGS"; then pkg_cv_X11_CFLAGS="$X11_CFLAGS" @@ -15353,7 +16001,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -15380,7 +16028,7 @@ Alternatively, you may set the environment variables X11_CFLAGS and X11_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details." "$LINENO" 5 elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} @@ -15984,24 +16632,47 @@ else # (e.g. gnu pth with pthread emulation) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS in unistd.h" >&5 printf %s "checking for _POSIX_THREADS in unistd.h... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS defined in unistd.h" >&5 +printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } +if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _POSIX_THREADS -yes -#endif +int +main (void) +{ + + #ifdef _POSIX_THREADS + int ok; + (void)ok; + #else + choke me + #endif + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__POSIX_THREADS_unistd_h=yes +else $as_nop + ac_cv_defined__POSIX_THREADS_unistd_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 +printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } +if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes else $as_nop unistd_defines_pthreads=no fi -rm -rf conftest* - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16501,66 +17172,136 @@ ipv6type=unknown ipv6lib=none ipv6trylibc=no -if test "$ipv6" = "yes"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 -printf %s "checking ipv6 stack type... " >&6; } +if test "$ipv6" = yes -a "$cross_compiling" = no; then for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for IPV6_INRIA_VERSION defined in netinet/in.h" >&5 +printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef IPV6_INRIA_VERSION -yes -#endif +int +main (void) +{ + + #ifdef IPV6_INRIA_VERSION + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes +else $as_nop + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } +if test $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h != "no" then : ipv6type=$i fi -rm -rf conftest* - ;; kame) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __KAME__ defined in netinet/in.h" >&5 +printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined___KAME___netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __KAME__ -yes -#endif +int +main (void) +{ + + #ifdef __KAME__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes + ac_cv_defined___KAME___netinet_in_h=yes +else $as_nop + ac_cv_defined___KAME___netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } +if test $ac_cv_defined___KAME___netinet_in_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-glibc) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __GLIBC__ defined in features.h" >&5 +printf %s "checking for __GLIBC__ defined in features.h... " >&6; } +if test ${ac_cv_defined___GLIBC___features_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -#endif +int +main (void) +{ + + #ifdef __GLIBC__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6trylibc=yes + ac_cv_defined___GLIBC___features_h=yes +else $as_nop + ac_cv_defined___GLIBC___features_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 +printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } +if test $ac_cv_defined___GLIBC___features_h != "no" +then : + ipv6type=$i + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-inet6) if test -d /usr/inet6; then @@ -16579,83 +17320,159 @@ rm -rf conftest* fi ;; toshiba) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _TOSHIBA_INET6 defined in sys/param.h" >&5 +printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _TOSHIBA_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _TOSHIBA_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__TOSHIBA_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; v6d) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h" >&5 +printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h... " >&6; } +if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __V6D__ -yes -#endif +int +main (void) +{ + + #ifdef __V6D__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes +else $as_nop + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no fi -rm -rf conftest* - - ;; - zeta) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 +printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } +if test $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h != "no" +then : + ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" +fi + ;; + zeta) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _ZETA_MINAMI_INET6 defined in sys/param.h" >&5 +printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _ZETA_MINAMI_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _ZETA_MINAMI_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; esac if test "$ipv6type" != "unknown"; then break fi done + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 +printf %s "checking ipv6 stack type... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ipv6type" >&5 printf "%s\n" "$ipv6type" >&6; } fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 library" >&5 +printf %s "checking ipv6 library... " >&6; } if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using lib$ipv6lib" >&5 -printf "%s\n" "$as_me: using lib$ipv6lib" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: lib$ipv6lib" >&5 +printf "%s\n" "lib$ipv6lib" >&6; } else if test "x$ipv6trylibc" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using libc" >&5 -printf "%s\n" "$as_me: using libc" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 +printf "%s\n" "libc" >&6; } else $as_nop @@ -16767,6 +17584,130 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_doc_strings" >&5 printf "%s\n" "$with_doc_strings" >&6; } +# Check for stdatomic.h, required for mimalloc. +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdatomic.h" >&5 +printf %s "checking for stdatomic.h... " >&6; } +if test ${ac_cv_header_stdatomic_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + atomic_int int_var; + atomic_uintptr_t uintptr_var; + int main() { + atomic_store_explicit(&int_var, 5, memory_order_relaxed); + atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); + int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); + return 0; + } + + +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_header_stdatomic_h=yes +else $as_nop + ac_cv_header_stdatomic_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdatomic_h" >&5 +printf "%s\n" "$ac_cv_header_stdatomic_h" >&6; } + +if test "x$ac_cv_header_stdatomic_h" = xyes +then : + + +printf "%s\n" "#define HAVE_STD_ATOMIC 1" >>confdefs.h + + +fi + +# Check for GCC >= 4.7 and clang __atomic builtin functions +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic_load_n and __atomic_store_n functions" >&5 +printf %s "checking for builtin __atomic_load_n and __atomic_store_n functions... " >&6; } +if test ${ac_cv_builtin_atomic+y} +then : + printf %s "(cached) " >&6 +else $as_nop + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + int val; + int main() { + __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); + (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); + return 0; + } + + +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_builtin_atomic=yes +else $as_nop + ac_cv_builtin_atomic=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_builtin_atomic" >&5 +printf "%s\n" "$ac_cv_builtin_atomic" >&6; } + +if test "x$ac_cv_builtin_atomic" = xyes +then : + + +printf "%s\n" "#define HAVE_BUILTIN_ATOMIC 1" >>confdefs.h + + +fi + +# --with-mimalloc +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-mimalloc" >&5 +printf %s "checking for --with-mimalloc... " >&6; } + +# Check whether --with-mimalloc was given. +if test ${with_mimalloc+y} +then : + withval=$with_mimalloc; +else $as_nop + with_mimalloc="$ac_cv_header_stdatomic_h" + +fi + + +if test "$with_mimalloc" != no; then + if test "$ac_cv_header_stdatomic_h" != yes; then + # mimalloc-atomic.h wants C11 stdatomic.h on POSIX + as_fn_error $? "mimalloc requires stdatomic.h, use --without-mimalloc to disable mimalloc." "$LINENO" 5 + fi + with_mimalloc=yes + +printf "%s\n" "#define WITH_MIMALLOC 1" >>confdefs.h + + MIMALLOC_HEADERS='$(MIMALLOC_HEADERS)' + +elif test "$disable_gil" = "yes"; then + as_fn_error $? "--disable-gil requires mimalloc memory allocator (--with-mimalloc)." "$LINENO" 5 +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5 +printf "%s\n" "$with_mimalloc" >&6; } +INSTALL_MIMALLOC=$with_mimalloc + + + # Check for Python-specific malloc support { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pymalloc" >&5 printf %s "checking for --with-pymalloc... " >&6; } @@ -17056,6 +17997,31 @@ else printf "%s\n" "$MACHDEP_OBJS" >&6; } fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi + + for name in $blocked_funcs; do + as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + + eval "$as_func_var=no" + + done +fi + # checks for library functions ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" if test "x$ac_cv_func_accept4" = xyes @@ -17092,6 +18058,12 @@ if test "x$ac_cv_func_clock" = xyes then : printf "%s\n" "#define HAVE_CLOCK 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "closefrom" "ac_cv_func_closefrom" +if test "x$ac_cv_func_closefrom" = xyes +then : + printf "%s\n" "#define HAVE_CLOSEFROM 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "close_range" "ac_cv_func_close_range" if test "x$ac_cv_func_close_range" = xyes @@ -17260,12 +18232,6 @@ if test "x$ac_cv_func_getegid" = xyes then : printf "%s\n" "#define HAVE_GETEGID 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" -if test "x$ac_cv_func_getentropy" = xyes -then : - printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "geteuid" "ac_cv_func_geteuid" if test "x$ac_cv_func_geteuid" = xyes @@ -17278,6 +18244,12 @@ if test "x$ac_cv_func_getgid" = xyes then : printf "%s\n" "#define HAVE_GETGID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getgrent" "ac_cv_func_getgrent" +if test "x$ac_cv_func_getgrent" = xyes +then : + printf "%s\n" "#define HAVE_GETGRENT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getgrgid" "ac_cv_func_getgrgid" if test "x$ac_cv_func_getgrgid" = xyes @@ -17302,12 +18274,6 @@ if test "x$ac_cv_func_getgrouplist" = xyes then : printf "%s\n" "#define HAVE_GETGROUPLIST 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" -if test "x$ac_cv_func_getgroups" = xyes -then : - printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "gethostname" "ac_cv_func_gethostname" if test "x$ac_cv_func_gethostname" = xyes @@ -17440,6 +18406,12 @@ if test "x$ac_cv_func_getwd" = xyes then : printf "%s\n" "#define HAVE_GETWD 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "grantpt" "ac_cv_func_grantpt" +if test "x$ac_cv_func_grantpt" = xyes +then : + printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex" if test "x$ac_cv_func_if_nameindex" = xyes @@ -17626,6 +18598,12 @@ if test "x$ac_cv_func_posix_fallocate" = xyes then : printf "%s\n" "#define HAVE_POSIX_FALLOCATE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_openpt" "ac_cv_func_posix_openpt" +if test "x$ac_cv_func_posix_openpt" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_OPENPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "posix_spawn" "ac_cv_func_posix_spawn" if test "x$ac_cv_func_posix_spawn" = xyes @@ -17638,6 +18616,12 @@ if test "x$ac_cv_func_posix_spawnp" = xyes then : printf "%s\n" "#define HAVE_POSIX_SPAWNP 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_spawn_file_actions_addclosefrom_np" "ac_cv_func_posix_spawn_file_actions_addclosefrom_np" +if test "x$ac_cv_func_posix_spawn_file_actions_addclosefrom_np" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pread" "ac_cv_func_pread" if test "x$ac_cv_func_pread" = xyes @@ -17656,6 +18640,18 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "process_vm_readv" "ac_cv_func_process_vm_readv" +if test "x$ac_cv_func_process_vm_readv" = xyes +then : + printf "%s\n" "#define HAVE_PROCESS_VM_READV 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" +if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_condattr_setclock" "ac_cv_func_pthread_condattr_setclock" if test "x$ac_cv_func_pthread_condattr_setclock" = xyes @@ -17674,6 +18670,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" +if test "x$ac_cv_func_ptsname" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "ptsname_r" "ac_cv_func_ptsname_r" +if test "x$ac_cv_func_ptsname_r" = xyes +then : + printf "%s\n" "#define HAVE_PTSNAME_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pwrite" "ac_cv_func_pwrite" if test "x$ac_cv_func_pwrite" = xyes @@ -17998,12 +19006,6 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" -if test "x$ac_cv_func_system" = xyes -then : - printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -18082,6 +19084,12 @@ if test "x$ac_cv_func_unlinkat" = xyes then : printf "%s\n" "#define HAVE_UNLINKAT 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "unlockpt" "ac_cv_func_unlockpt" +if test "x$ac_cv_func_unlockpt" = xyes +then : + printf "%s\n" "#define HAVE_UNLOCKPT 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "utimensat" "ac_cv_func_utimensat" if test "x$ac_cv_func_utimensat" = xyes @@ -18176,6 +19184,32 @@ fi fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" +if test "x$ac_cv_func_getentropy" = xyes +then : + printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" +if test "x$ac_cv_func_getgroups" = xyes +then : + printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" +if test "x$ac_cv_func_system" = xyes +then : + printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h + +fi + +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5 printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; } if test ${ac_cv_c_undeclared_builtin_options+y} @@ -19311,8 +20345,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZLIB" >&5 -printf %s "checking for ZLIB... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for zlib >= 1.2.0" >&5 +printf %s "checking for zlib >= 1.2.0... " >&6; } if test -n "$ZLIB_CFLAGS"; then pkg_cv_ZLIB_CFLAGS="$ZLIB_CFLAGS" @@ -19352,7 +20386,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -19495,7 +20529,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -19659,8 +20693,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for BZIP2" >&5 -printf %s "checking for BZIP2... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for bzip2" >&5 +printf %s "checking for bzip2... " >&6; } if test -n "$BZIP2_CFLAGS"; then pkg_cv_BZIP2_CFLAGS="$BZIP2_CFLAGS" @@ -19700,7 +20734,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -19796,7 +20830,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -19887,8 +20921,8 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBLZMA" >&5 -printf %s "checking for LIBLZMA... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for liblzma" >&5 +printf %s "checking for liblzma... " >&6; } if test -n "$LIBLZMA_CFLAGS"; then pkg_cv_LIBLZMA_CFLAGS="$LIBLZMA_CFLAGS" @@ -19928,7 +20962,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -20024,7 +21058,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -21471,6 +22505,11 @@ fi done +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then for ac_func in clock_settime do : @@ -21481,7 +22520,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : @@ -21519,7 +22558,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } if test "x$ac_cv_lib_rt_clock_settime" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h fi @@ -21528,7 +22567,12 @@ fi fi done +fi +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then for ac_func in clock_nanosleep do : @@ -21539,7 +22583,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 printf %s "checking for clock_nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_nanosleep+y} then : @@ -21577,7 +22621,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_nanosleep" >&6; } if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h fi @@ -21586,6 +22630,7 @@ fi fi done +fi for ac_func in nanosleep @@ -21658,6 +22703,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -21748,7 +22794,9 @@ else $as_nop if test "$cross_compiling" = yes then : -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -23651,7 +24699,7 @@ printf %s "checking ABIFLAGS... " >&6; } printf "%s\n" "$ABIFLAGS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking SOABI" >&5 printf %s "checking SOABI... " >&6; } -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $SOABI" >&5 printf "%s\n" "$SOABI" >&6; } @@ -23660,7 +24708,7 @@ printf "%s\n" "$SOABI" >&6; } if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} printf "%s\n" "#define ALT_SOABI \"${ALT_SOABI}\"" >>confdefs.h @@ -23675,21 +24723,30 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# On Android and Cygwin the shared libraries must be linked with libpython. +# Configure the flags and dependencies used when compiling shared modules + +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' + +# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" = "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) PLATLIBDIR="lib" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-platlibdir" >&5 @@ -23708,7 +24765,7 @@ then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -23722,9 +24779,9 @@ fi if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi @@ -23840,7 +24897,6 @@ fi - # Check whether --with-readline was given. if test ${with_readline+y} then : @@ -23863,29 +24919,13 @@ else $as_nop fi -# gh-105323: Need to handle the macOS editline as an alias of readline. -case $ac_sys_system/$ac_sys_release in #( - Darwin/*) : - ac_fn_c_check_type "$LINENO" "Function" "ac_cv_type_Function" "#include -" -if test "x$ac_cv_type_Function" = xyes -then : - printf "%s\n" "#define WITH_APPLE_EDITLINE 1" >>confdefs.h - -fi - ;; #( - *) : - - ;; -esac - if test "x$with_readline" = xreadline then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBREADLINE" >&5 -printf %s "checking for LIBREADLINE... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for readline" >&5 +printf %s "checking for readline... " >&6; } if test -n "$LIBREADLINE_CFLAGS"; then pkg_cv_LIBREADLINE_CFLAGS="$LIBREADLINE_CFLAGS" @@ -23925,7 +24965,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -24018,7 +25058,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -24115,8 +25155,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBEDIT" >&5 -printf %s "checking for LIBEDIT... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libedit" >&5 +printf %s "checking for libedit... " >&6; } if test -n "$LIBEDIT_CFLAGS"; then pkg_cv_LIBEDIT_CFLAGS="$LIBEDIT_CFLAGS" @@ -24156,7 +25196,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -24251,7 +25291,7 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } save_CFLAGS=$CFLAGS @@ -24693,15 +25733,66 @@ then : printf "%s\n" "#define HAVE_RL_COMPDISP_FUNC_T 1" >>confdefs.h + fi + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if rl_startup_hook takes arguments" >&5 +printf %s "checking if rl_startup_hook takes arguments... " >&6; } +if test ${ac_cv_readline_rl_startup_hook_takes_args+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS + #include /* Must be first for Gnu Readline */ + #ifdef WITH_EDITLINE + # include + #else + # include + # include + #endif + + extern int test_hook_func(const char *text, int state); +int +main (void) +{ +rl_startup_hook=test_hook_func; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_readline_rl_startup_hook_takes_args=yes +else $as_nop + ac_cv_readline_rl_startup_hook_takes_args=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_startup_hook_takes_args" >&5 +printf "%s\n" "$ac_cv_readline_rl_startup_hook_takes_args" >&6; } + if test "x$ac_cv_readline_rl_startup_hook_takes_args" = xyes +then : + + +printf "%s\n" "#define Py_RL_STARTUP_HOOK_TAKES_ARGS 1" >>confdefs.h + + +fi + + + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS fi @@ -24979,243 +26070,81 @@ printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h fi -have_curses=no -have_panel=no - - -ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" -if test "x$ac_cv_header_curses_h" = xyes -then : - printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h - -fi -ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_h" = xyes -then : - printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h - -fi - - -if test "x$ac_cv_header_ncurses_h" = xyes -then : - - if test "$ac_sys_system" != "Darwin"; then - -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CURSES" >&5 -printf %s "checking for CURSES... " >&6; } - -if test -n "$CURSES_CFLAGS"; then - pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncursesw" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$CURSES_LIBS"; then - pkg_cv_CURSES_LIBS="$CURSES_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncursesw" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` - else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether year with century should be normalized for strftime" >&5 +printf %s "checking whether year with century should be normalized for strftime... " >&6; } +if test ${ac_cv_normalize_century+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncursesw_initscr=yes -else $as_nop - ac_cv_lib_ncursesw_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} +if test "$cross_compiling" = yes then : - printf %s "(cached) " >&6 + ac_cv_normalize_century=yes else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) +#include +#include + +int main(void) { -return initscr (); - ; + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } return 0; } + _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_run "$LINENO" then : - ac_cv_lib_ncursesw_initscr=yes + ac_cv_normalize_century=yes else $as_nop - ac_cv_lib_ncursesw_initscr=no + ac_cv_normalize_century=no fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 +printf "%s\n" "$ac_cv_normalize_century" >&6; } +if test "$ac_cv_normalize_century" = yes +then +printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - +fi -else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +have_curses=no +have_panel=no - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - have_curses=ncursesw -fi - fi +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. - if test "x$have_curses" = xno -then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CURSES" >&5 -printf %s "checking for CURSES... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 +printf %s "checking for ncursesw... " >&6; } if test -n "$CURSES_CFLAGS"; then pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncursesw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -25227,12 +26156,12 @@ if test -n "$CURSES_LIBS"; then pkg_cv_CURSES_LIBS="$CURSES_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncursesw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -25244,7 +26173,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25253,193 +26182,31 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$CURSES_PKG_ERRORS" >&5 - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - + have_curses=no elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - + have_curses=no else CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_curses=ncurses - -fi - -fi - - -fi -CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') - -if test "$have_curses" = no -a "$ac_sys_system" = "Darwin"; then - - as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - -fi - - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking curses module flags" >&5 -printf %s "checking curses module flags... " >&6; } -if test "x$have_curses" = xno -then : - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&5 -printf "%s\n" "$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&6; } - -fi - -ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_panel_h" = xyes -then : - printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h - -fi - - -if test "x$ac_cv_header_panel_h" = xyes -then : - - - if test "$ac_sys_system" != "Darwin"; then - if test "x$have_curses" = xncursesw -then : +printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h + have_curses=yes pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PANEL" >&5 -printf %s "checking for PANEL... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 +printf %s "checking for panelw... " >&6; } if test -n "$PANEL_CFLAGS"; then pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" @@ -25479,7 +26246,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25495,147 +26262,105 @@ fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no + have_panel=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test "x$have_curses" = xno +then : -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + else + pkg_failed=untried fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_panel=panelw - -fi - -fi - fi - - if test "x$have_curses" = xncurses -then : +printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h + have_curses=yes pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PANEL" >&5 -printf %s "checking for PANEL... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 +printf %s "checking for panel... " >&6; } if test -n "$PANEL_CFLAGS"; then pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" @@ -25675,7 +26400,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -25691,83 +26416,165 @@ fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h - save_CFLAGS=$CFLAGS + have_panel=yes +fi +fi + + +fi + +save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + # Make sure we've got the header defines. + as_fn_append CPPFLAGS " $CURSES_CFLAGS $PANEL_CFLAGS" + ac_fn_c_check_header_compile "$LINENO" "ncursesw/curses.h" "ac_cv_header_ncursesw_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/ncurses.h" "ac_cv_header_ncursesw_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/panel.h" "ac_cv_header_ncursesw_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/curses.h" "ac_cv_header_ncurses_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/ncurses.h" "ac_cv_header_ncurses_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/panel.h" "ac_cv_header_ncurses_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +fi + + + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + as_fn_append LIBS " $CURSES_LIBS $PANEL_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing initscr" >&5 +printf %s "checking for library containing initscr... " >&6; } +if test ${ac_cv_search_initscr+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" + ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ -char update_panels (); +char initscr (); int main (void) { -return update_panels (); +return initscr (); ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' ncursesw ncurses +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_initscr=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_initscr+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_initscr+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else $as_nop + ac_cv_search_initscr=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 +printf "%s\n" "$ac_cv_search_initscr" >&6; } +ac_res=$ac_cv_search_initscr +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_curses" = xno +then : + have_curses=yes + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} +fi +else $as_nop + have_curses=no fi - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 +printf %s "checking for library containing update_panels... " >&6; } +if test ${ac_cv_search_update_panels+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" + ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25783,76 +26590,82 @@ return update_panels (); return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' panelw panel +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_update_panels=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_update_panels+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_update_panels+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else $as_nop + ac_cv_search_update_panels=no fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - - have_panel=panel - +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS fi - +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 +printf "%s\n" "$ac_cv_search_update_panels" >&6; } +ac_res=$ac_cv_search_update_panels +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_panel" = xno +then : + have_panel=yes + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} +fi +else $as_nop + have_panel=no fi -fi -PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking panel flags" >&5 -printf %s "checking panel flags... " >&6; } -if test "x$have_panel" = xno + +if test "have_curses" != "no" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -else $as_nop +if test "x$ac_sys_system" = xDarwin +then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&5 -printf "%s\n" "$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&6; } -fi + as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" fi +PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + # On Solaris, term.h requires curses.h ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " -#ifdef HAVE_CURSES_H -#include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif " @@ -25872,7 +26685,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -25902,10 +26730,6 @@ printf "%s\n" "#define MVWDELCH_IS_EXPRESSION 1" >>confdefs.h fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether WINDOW has _flags" >&5 printf %s "checking whether WINDOW has _flags... " >&6; } if test ${ac_cv_window_has_flags+y} @@ -25915,8 +26739,20 @@ else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define NCURSES_OPAQUE 0 - #include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif int main (void) @@ -25960,7 +26796,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26003,7 +26854,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26046,7 +26912,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26089,7 +26970,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26132,7 +27028,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26175,7 +27086,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26218,7 +27144,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26261,7 +27202,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26304,7 +27260,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26347,7 +27318,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26390,7 +27376,22 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26425,27 +27426,38 @@ fi CPPFLAGS=$ac_save_cppflags +fi +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : @@ -26466,12 +27478,12 @@ then : fi -if test "x$ac_cv_file__dev_ptmx" = xyes; then + if test "x$ac_cv_file__dev_ptmx" = xyes; then printf "%s\n" "#define HAVE_DEV_PTMX 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : @@ -26492,10 +27504,11 @@ then : fi -if test "x$ac_cv_file__dev_ptc" = xyes; then + if test "x$ac_cv_file__dev_ptc" = xyes; then printf "%s\n" "#define HAVE_DEV_PTC 1" >>confdefs.h + fi fi if test $ac_sys_system = Darwin @@ -26515,6 +27528,9 @@ ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" " if test "x$ac_cv_type_socklen_t" = xyes then : +printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h + + else $as_nop printf "%s\n" "#define socklen_t int" >>confdefs.h @@ -26674,17 +27690,19 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ Objects \ + Objects/mimalloc \ + Objects/mimalloc/prim \ Parser \ Parser/tokenizer \ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 printf %s "checking for build directories... " >&6; } for dir in $SRCDIRS; do @@ -26836,95 +27854,6 @@ printf "%s\n" "#define HAVE_IPA_PURE_CONST_BUG 1" >>confdefs.h esac fi -# Check for stdatomic.h -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdatomic.h" >&5 -printf %s "checking for stdatomic.h... " >&6; } -if test ${ac_cv_header_stdatomic_h+y} -then : - printf %s "(cached) " >&6 -else $as_nop - -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - - #include - atomic_int int_var; - atomic_uintptr_t uintptr_var; - int main(void) { - atomic_store_explicit(&int_var, 5, memory_order_relaxed); - atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); - int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); - return 0; - } - - -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_header_stdatomic_h=yes -else $as_nop - ac_cv_header_stdatomic_h=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdatomic_h" >&5 -printf "%s\n" "$ac_cv_header_stdatomic_h" >&6; } - -if test "x$ac_cv_header_stdatomic_h" = xyes -then : - - -printf "%s\n" "#define HAVE_STD_ATOMIC 1" >>confdefs.h - - -fi - -# Check for GCC >= 4.7 and clang __atomic builtin functions -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic_load_n and __atomic_store_n functions" >&5 -printf %s "checking for builtin __atomic_load_n and __atomic_store_n functions... " >&6; } -if test ${ac_cv_builtin_atomic+y} -then : - printf %s "(cached) " >&6 -else $as_nop - -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - - int val; - int main(void) { - __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); - (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); - return 0; - } - - -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_builtin_atomic=yes -else $as_nop - ac_cv_builtin_atomic=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_builtin_atomic" >&5 -printf "%s\n" "$ac_cv_builtin_atomic" >&6; } - -if test "x$ac_cv_builtin_atomic" = xyes -then : - - -printf "%s\n" "#define HAVE_BUILTIN_ATOMIC 1" >>confdefs.h - - -fi - # ensurepip option { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ensurepip" >&5 printf %s "checking for ensurepip... " >&6; } @@ -26940,6 +27869,8 @@ else $as_nop with_ensurepip=no ;; #( WASI) : with_ensurepip=no ;; #( + iOS) : + with_ensurepip=no ;; #( *) : with_ensurepip=upgrade ;; @@ -27435,7 +28366,12 @@ then : else $as_nop - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi fi @@ -27744,8 +28680,8 @@ then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for LIBB2" >&5 -printf %s "checking for LIBB2... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libb2" >&5 +printf %s "checking for libb2... " >&6; } if test -n "$LIBB2_CFLAGS"; then pkg_cv_LIBB2_CFLAGS="$LIBB2_CFLAGS" @@ -27785,7 +28721,7 @@ fi if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then @@ -27803,7 +28739,7 @@ fi have_libb2=no elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_libb2=no else @@ -27858,6 +28794,9 @@ printf "%s\n" "$TEST_MODULES" >&6; } # libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the # compiler flags. # +# gh-112779: On RISC-V, GCC 12 and earlier require libatomic support for 1-byte +# and 2-byte operations, but not for 8-byte operations. +# # Avoid #include or #include . The header # requires header which is only written below by AC_OUTPUT below. # If the check is done after AC_OUTPUT, modifying LIBS has no effect @@ -27897,12 +28836,19 @@ typedef intptr_t Py_ssize_t; int main() { - uint64_t byte; - _Py_atomic_store_uint64(&byte, 2); - if (_Py_atomic_or_uint64(&byte, 8) != 2) { + uint64_t value; + _Py_atomic_store_uint64(&value, 2); + if (_Py_atomic_or_uint64(&value, 8) != 2) { return 1; // error } - if (_Py_atomic_load_uint64(&byte) != 10) { + if (_Py_atomic_load_uint64(&value) != 10) { + return 1; // error + } + uint8_t byte = 0xb8; + if (_Py_atomic_or_uint8(&byte, 0x2d) != 0xb8) { + return 1; // error + } + if (_Py_atomic_load_uint8(&byte) != 0xbd) { return 1; // error } return 0; // all good @@ -27926,6 +28872,7 @@ printf "%s\n" "$ac_cv_libatomic_needed" >&6; } if test "x$ac_cv_libatomic_needed" = xyes then : LIBS="${LIBS} -latomic" + LIBATOMIC=${LIBATOMIC-"-latomic"} fi CPPFLAGS=$save_CPPFLAGS @@ -27949,6 +28896,27 @@ case $ac_sys_system in #( ;; #( Darwin) : ;; #( + iOS) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module_grp=n/a + py_cv_module_nis=n/a + py_cv_module_readline=n/a + py_cv_module_pwd=n/a + py_cv_module_spwd=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a + + ;; #( CYGWIN*) : @@ -27977,8 +28945,9 @@ case $ac_sys_system in #( py_cv_module__posixsubprocess=n/a py_cv_module__scproxy=n/a py_cv_module__tkinter=n/a - py_cv_module__xxsubinterpreters=n/a - py_cv_module__xxinterpchannels=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a py_cv_module_grp=n/a py_cv_module_pwd=n/a py_cv_module_resource=n/a @@ -28003,9 +28972,15 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a + py_cv_module__testexternalinspection=n/a + py_cv_module__testimportmultiple=n/a + py_cv_module__testmultiphase=n/a + py_cv_module__testsinglephase=n/a py_cv_module_fcntl=n/a py_cv_module_mmap=n/a py_cv_module_termios=n/a + py_cv_module_xxlimited=n/a + py_cv_module_xxlimited_35=n/a py_cv_module_=n/a @@ -28258,28 +29233,6 @@ then : -fi - - - if test "$py_cv_module__opcode" != "n/a" -then : - py_cv_module__opcode=yes -fi - if test "$py_cv_module__opcode" = yes; then - MODULE__OPCODE_TRUE= - MODULE__OPCODE_FALSE='#' -else - MODULE__OPCODE_TRUE='#' - MODULE__OPCODE_FALSE= -fi - - as_fn_append MODULE_BLOCK "MODULE__OPCODE_STATE=$py_cv_module__opcode$as_nl" - if test "x$py_cv_module__opcode" = xyes -then : - - - - fi @@ -28437,20 +29390,42 @@ then : fi - if test "$py_cv_module__xxsubinterpreters" != "n/a" + if test "$py_cv_module__interpreters" != "n/a" +then : + py_cv_module__interpreters=yes +fi + if test "$py_cv_module__interpreters" = yes; then + MODULE__INTERPRETERS_TRUE= + MODULE__INTERPRETERS_FALSE='#' +else + MODULE__INTERPRETERS_TRUE='#' + MODULE__INTERPRETERS_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE__INTERPRETERS_STATE=$py_cv_module__interpreters$as_nl" + if test "x$py_cv_module__interpreters" = xyes +then : + + + + +fi + + + if test "$py_cv_module__interpchannels" != "n/a" then : - py_cv_module__xxsubinterpreters=yes + py_cv_module__interpchannels=yes fi - if test "$py_cv_module__xxsubinterpreters" = yes; then - MODULE__XXSUBINTERPRETERS_TRUE= - MODULE__XXSUBINTERPRETERS_FALSE='#' + if test "$py_cv_module__interpchannels" = yes; then + MODULE__INTERPCHANNELS_TRUE= + MODULE__INTERPCHANNELS_FALSE='#' else - MODULE__XXSUBINTERPRETERS_TRUE='#' - MODULE__XXSUBINTERPRETERS_FALSE= + MODULE__INTERPCHANNELS_TRUE='#' + MODULE__INTERPCHANNELS_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXSUBINTERPRETERS_STATE=$py_cv_module__xxsubinterpreters$as_nl" - if test "x$py_cv_module__xxsubinterpreters" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPCHANNELS_STATE=$py_cv_module__interpchannels$as_nl" + if test "x$py_cv_module__interpchannels" = xyes then : @@ -28459,20 +29434,20 @@ then : fi - if test "$py_cv_module__xxinterpchannels" != "n/a" + if test "$py_cv_module__interpqueues" != "n/a" then : - py_cv_module__xxinterpchannels=yes + py_cv_module__interpqueues=yes fi - if test "$py_cv_module__xxinterpchannels" = yes; then - MODULE__XXINTERPCHANNELS_TRUE= - MODULE__XXINTERPCHANNELS_FALSE='#' + if test "$py_cv_module__interpqueues" = yes; then + MODULE__INTERPQUEUES_TRUE= + MODULE__INTERPQUEUES_FALSE='#' else - MODULE__XXINTERPCHANNELS_TRUE='#' - MODULE__XXINTERPCHANNELS_FALSE= + MODULE__INTERPQUEUES_TRUE='#' + MODULE__INTERPQUEUES_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXINTERPCHANNELS_STATE=$py_cv_module__xxinterpchannels$as_nl" - if test "x$py_cv_module__xxinterpchannels" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPQUEUES_STATE=$py_cv_module__interpqueues$as_nl" + if test "x$py_cv_module__interpqueues" = xyes then : @@ -28793,7 +29768,8 @@ then : if true then : - if test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes + if test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes else $as_nop @@ -29504,7 +30480,7 @@ then : if true then : - if test "$have_curses" != "no" + if test "$have_curses" = "yes" then : py_cv_module__curses=yes else $as_nop @@ -29543,7 +30519,7 @@ then : if true then : - if test "$have_panel" != "no" + if test "$have_curses" = "yes" && test "$have_panel" = "yes" then : py_cv_module__curses_panel=yes else $as_nop @@ -29582,7 +30558,7 @@ then : if true then : - if true + if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes else $as_nop @@ -29598,7 +30574,7 @@ fi then : as_fn_append MODULE_BLOCK "MODULE__DECIMAL_CFLAGS=$LIBMPDEC_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LIBS$as_nl" fi if test "$py_cv_module__decimal" = yes; then @@ -30079,7 +31055,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC$as_nl" fi if test "$py_cv_module__testcapi" = yes; then @@ -30170,6 +31146,44 @@ fi printf "%s\n" "$py_cv_module__testclinic_limited" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testlimitedcapi" >&5 +printf %s "checking for stdlib extension module _testlimitedcapi... " >&6; } + if test "$py_cv_module__testlimitedcapi" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testlimitedcapi=yes +else $as_nop + py_cv_module__testlimitedcapi=missing +fi +else $as_nop + py_cv_module__testlimitedcapi=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTLIMITEDCAPI_STATE=$py_cv_module__testlimitedcapi$as_nl" + if test "x$py_cv_module__testlimitedcapi" = xyes +then : + + + + +fi + if test "$py_cv_module__testlimitedcapi" = yes; then + MODULE__TESTLIMITEDCAPI_TRUE= + MODULE__TESTLIMITEDCAPI_FALSE='#' +else + MODULE__TESTLIMITEDCAPI_TRUE='#' + MODULE__TESTLIMITEDCAPI_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testlimitedcapi" >&5 +printf "%s\n" "$py_cv_module__testlimitedcapi" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 printf %s "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a" @@ -30322,6 +31336,82 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testsinglephase" >&5 +printf %s "checking for stdlib extension module _testsinglephase... " >&6; } + if test "$py_cv_module__testsinglephase" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module__testsinglephase=yes +else $as_nop + py_cv_module__testsinglephase=missing +fi +else $as_nop + py_cv_module__testsinglephase=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTSINGLEPHASE_STATE=$py_cv_module__testsinglephase$as_nl" + if test "x$py_cv_module__testsinglephase" = xyes +then : + + + + +fi + if test "$py_cv_module__testsinglephase" = yes; then + MODULE__TESTSINGLEPHASE_TRUE= + MODULE__TESTSINGLEPHASE_FALSE='#' +else + MODULE__TESTSINGLEPHASE_TRUE='#' + MODULE__TESTSINGLEPHASE_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testsinglephase" >&5 +printf "%s\n" "$py_cv_module__testsinglephase" >&6; } + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 +printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } + if test "$py_cv_module__testexternalinspection" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testexternalinspection=yes +else $as_nop + py_cv_module__testexternalinspection=missing +fi +else $as_nop + py_cv_module__testexternalinspection=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" + if test "x$py_cv_module__testexternalinspection" = xyes +then : + + + + +fi + if test "$py_cv_module__testexternalinspection" = yes; then + MODULE__TESTEXTERNALINSPECTION_TRUE= + MODULE__TESTEXTERNALINSPECTION_FALSE='#' +else + MODULE__TESTEXTERNALINSPECTION_TRUE='#' + MODULE__TESTEXTERNALINSPECTION_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 +printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 printf %s "checking for stdlib extension module xxsubtype... " >&6; } if test "$py_cv_module_xxsubtype" != "n/a" @@ -30420,8 +31510,8 @@ fi if test "x$py_cv_module__ctypes_test" = xyes then : - - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_CFLAGS=$LIBFFI_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBFFI_LIBS $LIBM$as_nl" fi if test "$py_cv_module__ctypes_test" = yes; then @@ -30442,7 +31532,7 @@ printf %s "checking for stdlib extension module xxlimited... " >&6; } if test "$py_cv_module_xxlimited" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : @@ -30480,7 +31570,7 @@ printf %s "checking for stdlib extension module xxlimited_35... " >&6; } if test "$py_cv_module_xxlimited_35" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : @@ -30674,10 +31764,6 @@ if test -z "${MODULE__LSPROF_TRUE}" && test -z "${MODULE__LSPROF_FALSE}"; then as_fn_error $? "conditional \"MODULE__LSPROF\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__OPCODE_TRUE}" && test -z "${MODULE__OPCODE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__OPCODE\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${MODULE__PICKLE_TRUE}" && test -z "${MODULE__PICKLE_FALSE}"; then as_fn_error $? "conditional \"MODULE__PICKLE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -30706,12 +31792,16 @@ if test -z "${MODULE__TYPING_TRUE}" && test -z "${MODULE__TYPING_FALSE}"; then as_fn_error $? "conditional \"MODULE__TYPING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXSUBINTERPRETERS_TRUE}" && test -z "${MODULE__XXSUBINTERPRETERS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXSUBINTERPRETERS\" was never defined. +if test -z "${MODULE__INTERPRETERS_TRUE}" && test -z "${MODULE__INTERPRETERS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPRETERS\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${MODULE__INTERPCHANNELS_TRUE}" && test -z "${MODULE__INTERPCHANNELS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPCHANNELS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHANNELS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined. +if test -z "${MODULE__INTERPQUEUES_TRUE}" && test -z "${MODULE__INTERPQUEUES_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPQUEUES\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then @@ -30914,6 +32004,10 @@ if test -z "${MODULE__TESTCLINIC_LIMITED_TRUE}" && test -z "${MODULE__TESTCLINIC as_fn_error $? "conditional \"MODULE__TESTCLINIC_LIMITED\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTLIMITEDCAPI_TRUE}" && test -z "${MODULE__TESTLIMITEDCAPI_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTLIMITEDCAPI\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -30930,6 +32024,14 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHASE_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi +if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then as_fn_error $? "conditional \"MODULE_XXSUBTYPE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31340,7 +32442,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.13, which was +This file was extended by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -31404,7 +32506,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.13 +python config.status 3.14 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" @@ -31532,6 +32634,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; + "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; @@ -32187,3 +33290,8 @@ CPython core team, see https://peps.python.org/pep-0011/ for more information. " >&2;} fi +if test "$ac_cv_header_stdatomic_h" != "yes"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 +printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} +fi + diff --git a/configure.ac b/configure.ac index cd69f0ede54496..df146cc9cf0b75 100644 --- a/configure.ac +++ b/configure.ac @@ -1,19 +1,16 @@ -dnl *************************************************** -dnl * Please run autoreconf -if to test your changes! * -dnl *************************************************** +dnl ************************************************************ +dnl * Please run autoreconf -ivf -Werror to test your changes! * +dnl ************************************************************ dnl dnl Python's configure script requires autoconf 2.71, autoconf-archive, -dnl pkgconf's m4 macros. +dnl aclocal 1.16, and pkg-config. dnl -dnl It is recommended to use a cpython_autoconf container to regenerate the -dnl configure script: -dnl -dnl podman run --rm --pull=always -v $(pwd):/src:Z quay.io/tiran/cpython_autoconf:271 -dnl docker run --rm --pull=always -v $(pwd):/src quay.io/tiran/cpython_autoconf:271 +dnl It is recommended to use the Tools/build/regen-configure.sh shell script +dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.13]) +m4_define([PYTHON_VERSION], [3.14]) AC_PREREQ([2.71]) @@ -310,6 +307,124 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +# Set name for machine-dependent library files +AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) +AC_MSG_CHECKING([MACHDEP]) +if test -z "$MACHDEP" +then + # avoid using uname for cross builds + if test "$cross_compiling" = yes; then + # ac_sys_system and ac_sys_release are used for setting + # a lot of different things including 'define_xopen_source' + # in the case statement below. + case "$host" in + *-*-linux-android*) + ac_sys_system=Linux-android + ;; + *-*-linux*) + ac_sys_system=Linux + ;; + *-*-cygwin*) + ac_sys_system=Cygwin + ;; + *-apple-ios*) + ac_sys_system=iOS + ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; + *-*-emscripten) + ac_sys_system=Emscripten + ;; + *-*-wasi) + ac_sys_system=WASI + ;; + *) + # for now, limit cross builds to known configurations + MACHDEP="unknown" + AC_MSG_ERROR([cross build not supported for $host]) + esac + ac_sys_release= + else + ac_sys_system=`uname -s` + if test "$ac_sys_system" = "AIX" \ + -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then + ac_sys_release=`uname -v` + else + ac_sys_release=`uname -r` + fi + fi + ac_md_system=`echo $ac_sys_system | + tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` + ac_md_release=`echo $ac_sys_release | + tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` + MACHDEP="$ac_md_system$ac_md_release" + + case $MACHDEP in + aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; + linux*) MACHDEP="linux";; + cygwin*) MACHDEP="cygwin";; + darwin*) MACHDEP="darwin";; + '') MACHDEP="unknown";; + esac + + if test "$ac_sys_system" = "SunOS"; then + # For Solaris, there isn't an OS version specific macro defined + # in most compilers, so we define one here. + SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` + AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], + [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) + fi +fi +AC_MSG_RESULT(["$MACHDEP"]) + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios12.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi + AC_MSG_CHECKING([for --enable-universalsdk]) AC_ARG_ENABLE([universalsdk], AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], @@ -419,109 +534,155 @@ AC_ARG_ENABLE([framework], [ case $enableval in yes) - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) AC_MSG_ERROR([Unknown platform for framework build]) + esac esac + case $enableval in no) - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " - FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" - FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" - FRAMEWORKPYTHONW="frameworkpythonw" - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - - case "${enableval}" in - /System*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - if test "${prefix}" = "NONE" ; then - # See below - FRAMEWORKUNIXTOOLSPREFIX="/usr" - fi - ;; + case $ac_sys_system in #( + Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" + FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" + FRAMEWORKPYTHONW="frameworkpythonw" + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + INSTALLTARGETS="commoninstall bininstall maninstall" + + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi - /Library*) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + case "${enableval}" in + /System*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + if test "${prefix}" = "NONE" ; then + # See below + FRAMEWORKUNIXTOOLSPREFIX="/usr" + fi + ;; + + /Library*) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + + */Library/Frameworks) + MDIR="`dirname "${enableval}"`" + MDIR="`dirname "${MDIR}"`" + FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" + + if test "${prefix}" = "NONE"; then + # User hasn't specified the + # --prefix option, but wants to install + # the framework in a non-default location, + # ensure that the compatibility links get + # installed relative to that prefix as well + # instead of in /usr/local. + FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" + fi + ;; - */Library/Frameworks) - MDIR="`dirname "${enableval}"`" - MDIR="`dirname "${MDIR}"`" - FRAMEWORKINSTALLAPPSPREFIX="${MDIR}/Applications" - - if test "${prefix}" = "NONE"; then - # User hasn't specified the - # --prefix option, but wants to install - # the framework in a non-default location, - # ensure that the compatibility links get - # installed relative to that prefix as well - # instead of in /usr/local. - FRAMEWORKUNIXTOOLSPREFIX="${MDIR}" - fi - ;; + *) + FRAMEWORKINSTALLAPPSPREFIX="/Applications" + ;; + esac - *) - FRAMEWORKINSTALLAPPSPREFIX="/Applications" - ;; + prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION + PYTHONFRAMEWORKINSTALLNAMEPREFIX=${prefix} + RESSRCDIR=Mac/Resources/framework + + # Add files for Mac specific code to the list of output + # files: + AC_CONFIG_FILES([Mac/Makefile]) + AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) + AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) + AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" + + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources + + AC_CONFIG_FILES([iOS/Resources/Info.plist]) + ;; + *) + AC_MSG_ERROR([Unknown platform for framework build]) + ;; + esac esac - - prefix=$PYTHONFRAMEWORKINSTALLDIR/Versions/$VERSION - - # Add files for Mac specific code to the list of output - # files: - AC_CONFIG_FILES([Mac/Makefile]) - AC_CONFIG_FILES([Mac/PythonLauncher/Makefile]) - AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) - AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) - esac ],[ - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= - + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ]) AC_SUBST([PYTHONFRAMEWORK]) AC_SUBST([PYTHONFRAMEWORKIDENTIFIER]) AC_SUBST([PYTHONFRAMEWORKDIR]) AC_SUBST([PYTHONFRAMEWORKPREFIX]) AC_SUBST([PYTHONFRAMEWORKINSTALLDIR]) +AC_SUBST([PYTHONFRAMEWORKINSTALLNAMEPREFIX]) +AC_SUBST([RESSRCDIR]) AC_SUBST([FRAMEWORKINSTALLFIRST]) AC_SUBST([FRAMEWORKINSTALLLAST]) AC_SUBST([FRAMEWORKALTINSTALLFIRST]) @@ -529,77 +690,68 @@ AC_SUBST([FRAMEWORKALTINSTALLLAST]) AC_SUBST([FRAMEWORKPYTHONW]) AC_SUBST([FRAMEWORKUNIXTOOLSPREFIX]) AC_SUBST([FRAMEWORKINSTALLAPPSPREFIX]) +AC_SUBST([INSTALLTARGETS]) AC_DEFINE_UNQUOTED([_PYTHONFRAMEWORK], ["${PYTHONFRAMEWORK}"], [framework name]) -# Set name for machine-dependent library files -AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) -AC_MSG_CHECKING([MACHDEP]) -if test -z "$MACHDEP" -then - # avoid using uname for cross builds - if test "$cross_compiling" = yes; then - # ac_sys_system and ac_sys_release are used for setting - # a lot of different things including 'define_xopen_source' - # in the case statement below. - case "$host" in - *-*-linux-android*) - ac_sys_system=Linux-android - ;; - *-*-linux*) - ac_sys_system=Linux - ;; - *-*-cygwin*) - ac_sys_system=Cygwin - ;; - *-*-vxworks*) - ac_sys_system=VxWorks - ;; - *-*-emscripten) - ac_sys_system=Emscripten - ;; - *-*-wasi) - ac_sys_system=WASI - ;; - *) - # for now, limit cross builds to known configurations - MACHDEP="unknown" - AC_MSG_ERROR([cross build not supported for $host]) - esac - ac_sys_release= - else - ac_sys_system=`uname -s` - if test "$ac_sys_system" = "AIX" \ - -o "$ac_sys_system" = "UnixWare" -o "$ac_sys_system" = "OpenUNIX"; then - ac_sys_release=`uname -v` - else - ac_sys_release=`uname -r` - fi - fi - ac_md_system=`echo $ac_sys_system | - tr -d '[/ ]' | tr '[[A-Z]]' '[[a-z]]'` - ac_md_release=`echo $ac_sys_release | - tr -d '[/ ]' | sed 's/^[[A-Z]]\.//' | sed 's/\..*//'` - MACHDEP="$ac_md_system$ac_md_release" - - case $MACHDEP in - aix*) MACHDEP="aix";; - linux*) MACHDEP="linux";; - cygwin*) MACHDEP="cygwin";; - darwin*) MACHDEP="darwin";; - '') MACHDEP="unknown";; +dnl quadrigraphs "@<:@" and "@:>@" produce "[" and "]" in the output +AC_MSG_CHECKING([for --with-app-store-compliance]) +AC_ARG_WITH( + [app_store_compliance], + [AS_HELP_STRING( + [--with-app-store-compliance=@<:@PATCH-FILE@:>@], + [Enable any patches required for compiliance with app stores. + Optional PATCH-FILE specifies the custom patch to apply.] + )],[ + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + ;; + *) AC_MSG_ERROR([no default app store compliance patch available for $ac_sys_system]) ;; + esac + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + AC_MSG_RESULT([applying custom app store compliance patch]) + ;; esac - - if test "$ac_sys_system" = "SunOS"; then - # For Solaris, there isn't an OS version specific macro defined - # in most compilers, so we define one here. - SUNOS_VERSION=`echo $ac_sys_release | sed -e 's!\.\([0-9]\)$!.0\1!g' | tr -d '.'` - AC_DEFINE_UNQUOTED([Py_SUNOS_VERSION], [$SUNOS_VERSION], - [The version of SunOS/Solaris as reported by `uname -r' without the dot.]) - fi -fi -AC_MSG_RESULT(["$MACHDEP"]) + ],[ + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS= + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + Darwin) + # Always *check* the compliance patch on macOS + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" + APP_STORE_COMPLIANCE_PATCH_FLAGS="--dry-run" + AC_MSG_RESULT([checking (not applying) default app store compliance patch]) + ;; + *) + # No app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + APP_STORE_COMPLIANCE_PATCH_TARGET= + APP_STORE_COMPLIANCE_PATCH_FLAGS= + AC_MSG_RESULT([not patching for app store compliance]) + ;; + esac +]) +AC_SUBST([APP_STORE_COMPLIANCE_PATCH]) +AC_SUBST([APP_STORE_COMPLIANCE_PATCH_TARGET]) +AC_SUBST([APP_STORE_COMPLIANCE_PATCH_FLAGS]) AC_SUBST([_PYTHON_HOST_PLATFORM]) if test "$cross_compiling" = yes; then @@ -607,27 +759,45 @@ if test "$cross_compiling" = yes; then *-*-linux*) case "$host_cpu" in arm*) - _host_cpu=arm + _host_ident=arm ;; *) - _host_cpu=$host_cpu + _host_ident=$host_cpu esac ;; *-*-cygwin*) - _host_cpu= + _host_ident= + ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac ;; *-*-vxworks*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; wasm32-*-* | wasm64-*-*) - _host_cpu=$host_cpu + _host_ident=$host_cpu ;; *) # for now, limit cross builds to known configurations MACHDEP="unknown" AC_MSG_ERROR([cross build not supported for $host]) esac - _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_ident:+-$_host_ident}" fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -693,6 +863,9 @@ case $ac_sys_system/$ac_sys_release in define_xopen_source=no;; Darwin/@<:@[12]@:>@@<:@0-9@:>@.*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -751,6 +924,9 @@ AC_SUBST([EXPORT_MACOSX_DEPLOYMENT_TARGET]) CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. +AC_SUBST([IPHONEOS_DEPLOYMENT_TARGET]) + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -783,6 +959,14 @@ AS_CASE([$host], ], ) +dnl Add the compiler flag for the iOS minimum supported OS version. +AS_CASE([$ac_sys_system], + [iOS], [ + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + ], +) + if test "$ac_sys_system" = "Darwin" then dnl look for SDKROOT @@ -866,7 +1050,9 @@ dnl check for GCC last, other compilers set __GNUC__, too. dnl msvc is listed for completeness. AC_CACHE_CHECK([for CC compiler name], [ac_cv_cc_name], [ cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` + if test $(expr "//$CC" : '.*/\(.*\)') = "mpicc"; then + ac_cv_cc_name="mpicc" + fi else ac_cv_cc_name="unknown" fi @@ -894,6 +1083,18 @@ rm -f conftest.c conftest.out # _POSIX_SOURCE, _POSIX_1_SOURCE, and more AC_USE_SYSTEM_EXTENSIONS +AC_CACHE_CHECK([for GCC compatible compiler], + [ac_cv_gcc_compat], + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([ + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + ], [])], + [ac_cv_gcc_compat=yes], + [ac_cv_gcc_compat=no])]) + AC_SUBST([CXX]) preset_cxx="$CXX" @@ -938,14 +1139,22 @@ else fi rm -f conftest.out +dnl On some platforms, using a true "triplet" for MULTIARCH would be redundant. +dnl For example, `arm64-apple-darwin` is redundant, because there isn't a +dnl non-Apple Darwin. Including the CPU architecture can also be potentially +dnl redundant - on macOS, for example, it's possible to do a single compile +dnl pass that includes multiple architectures, so it would be misleading for +dnl MULTIARCH (and thus the sysconfigdata module name) to include a single CPU +dnl architecture. PLATFORM_TRIPLET will be a pair or single value for these +dnl platforms. AC_MSG_CHECKING([for multiarch]) AS_CASE([$ac_sys_system], [Darwin*], [MULTIARCH=""], + [iOS], [MULTIARCH=""], [FreeBSD*], [MULTIARCH=""], [MULTIARCH=$($CC --print-multiarch 2>/dev/null)] ) AC_SUBST([MULTIARCH]) -AC_MSG_RESULT([$MULTIARCH]) if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then if test x$PLATFORM_TRIPLET != x$MULTIARCH; then @@ -955,6 +1164,17 @@ elif test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then MULTIARCH=$PLATFORM_TRIPLET fi AC_SUBST([PLATFORM_TRIPLET]) +AC_MSG_RESULT([$MULTIARCH]) + +dnl Even if we *do* include the CPU architecture in the MULTIARCH value, some +dnl platforms don't need the CPU architecture in the SOABI tag. These platforms +dnl will have multiple sysconfig modules (one for each CPU architecture), but +dnl use a single "fat" binary at runtime. SOABI_PLATFORM is the component of +dnl the PLATFORM_TRIPLET that will be used in binary module extensions. +AS_CASE([$ac_sys_system], + [iOS], [SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2`], + [SOABI_PLATFORM=$PLATFORM_TRIPLET] +) if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" @@ -969,22 +1189,26 @@ AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=2], dnl macOS on M1, any version [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [wasm32-unknown-emscripten/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly Emscripten - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=3], dnl WebAssembly System Interface [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [PY_SUPPORT_TIER=0] ) @@ -1029,6 +1253,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then @@ -1298,17 +1525,25 @@ fi AC_MSG_CHECKING([LDLIBRARY]) -# MacOSX framework builds need more magic. LDLIBRARY is the dynamic +# Apple framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then - LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' - RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} + case $ac_sys_system in + Darwin) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; + *) + AC_MSG_ERROR([Unknown platform for framework build]);; + esac BLDLIBRARY='' + RUNSHARED=DYLD_FRAMEWORK_PATH=`pwd`${DYLD_FRAMEWORK_PATH:+:${DYLD_FRAMEWORK_PATH}} else BLDLIBRARY='$(LDLIBRARY)' fi @@ -1320,66 +1555,78 @@ if test $enable_shared = "yes"; then [Defined if Python is built as a shared library.]) case $ac_sys_system in CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; + LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + DLLLIBRARY='libpython$(LDVERSION).dll' + ;; SunOS*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-Wl,-R,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + INSTSONAME="$LDLIBRARY".$SOVERSION + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; Linux*|GNU*|NetBSD*|FreeBSD*|DragonFly*|OpenBSD*|VxWorks*) - LDLIBRARY='libpython$(LDVERSION).so' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION - if test "$with_pydebug" != yes - then - PY3LIBRARY=libpython3.so - fi - ;; + LDLIBRARY='libpython$(LDVERSION).so' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + + if test "$with_pydebug" != yes + then + PY3LIBRARY=libpython3.so + fi + ;; hp*|HP*) - case `uname -m` in - ia64) - LDLIBRARY='libpython$(LDVERSION).so' - ;; - *) - LDLIBRARY='libpython$(LDVERSION).sl' - ;; - esac - BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' - RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} - ;; + case `uname -m` in + ia64) + LDLIBRARY='libpython$(LDVERSION).so' + ;; + *) + LDLIBRARY='libpython$(LDVERSION).sl' + ;; + esac + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; Darwin*) - LDLIBRARY='libpython$(LDVERSION).dylib' - BLDLIBRARY='-L. -lpython$(LDVERSION)' - RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} + ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) - LDLIBRARY='libpython$(LDVERSION).so' - RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} - ;; + LDLIBRARY='libpython$(LDVERSION).so' + RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} + ;; esac else # shared is disabled PY_ENABLE_SHARED=0 case $ac_sys_system in CYGWIN*) - BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' - ;; + BLDLIBRARY='$(LIBRARY)' + LDLIBRARY='libpython$(LDVERSION).dll.a' + ;; esac fi +AC_MSG_RESULT([$LDLIBRARY]) if test "$cross_compiling" = yes; then - RUNSHARED= + RUNSHARED= fi +AC_MSG_CHECKING([HOSTRUNNER]) AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then @@ -1420,12 +1667,11 @@ then dnl TODO: support other WASI runtimes dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --'], + [WASI/*], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi AC_SUBST([HOSTRUNNER]) -AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then @@ -1433,8 +1679,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -AC_MSG_RESULT([$LDLIBRARY]) - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], @@ -1495,7 +1739,9 @@ fi # For calculating the .so ABI tag. AC_SUBST([ABIFLAGS]) +AC_SUBST([ABI_THREAD]) ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -1508,10 +1754,11 @@ AC_MSG_RESULT([$disable_gil]) if test "$disable_gil" = "yes" then - AC_DEFINE([Py_NOGIL], [1], + AC_DEFINE([Py_GIL_DISABLED], [1], [Define if you want to disable the GIL]) # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -1582,6 +1829,41 @@ else AC_MSG_RESULT([no]) fi +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac +AS_VAR_IF([tier2_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) + # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) AC_SUBST([DEF_MAKE_RULE]) @@ -1614,7 +1896,7 @@ if test "$Py_OPT" = 'true' ; then AX_CHECK_COMPILE_FLAG([-fno-semantic-interposition],[ CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" - ]) + ], [], [-Werror]) ;; esac elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then @@ -1826,9 +2108,13 @@ case "$CC_BASENAME" in *clang*) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=m4_normalize(" + ${LLVM_PROFDATA} merge + -output=\"\$(shell pwd)/code.profclangd\" + \"\$(shell pwd)\"/*.profclangr + ") + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -1842,9 +2128,13 @@ case "$CC_BASENAME" in case $ac_sys_system in Darwin*) PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=m4_normalize(" + ${LLVM_PROFDATA} merge + -output=\"\$(shell pwd)/code.profclangd\" + \"\$(shell pwd)\"/*.profclangr + ") + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test "${LLVM_PROF_FOUND}" = "not-found" then LLVM_PROF_ERR=yes @@ -2060,6 +2350,11 @@ PYDEBUG_CFLAGS="-O0" AS_VAR_IF([ac_cv_cc_supports_og], [yes], [PYDEBUG_CFLAGS="-Og"]) +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST([OPT]) @@ -2162,16 +2457,22 @@ AS_CASE([$ac_sys_system], # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. AS_VAR_APPEND([CFLAGS], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([CFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--import-memory"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--export-memory"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--max-memory=10485760"]) ]) - dnl increase initial memory and stack size, move stack first + dnl gh-117645: Set the memory size to 40 MiB, the stack size to 16 MiB, + dnl and move the stack first. dnl https://github.com/WebAssembly/wasi-libc/issues/233 - AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040"]) ] ) @@ -2197,7 +2498,7 @@ AC_DEFUN([PY_CHECK_CC_WARNING], [ AS_VAR_PUSHDEF([py_var], [ac_cv_$1_]m4_normalize($2)[_warning]) AC_CACHE_CHECK([m4_ifblank([$3], [if we can $1 $CC $2 warning], [$3])], [py_var], [ AS_VAR_COPY([py_cflags], [CFLAGS]) - AS_VAR_APPEND([CFLAGS], ["-W$2 -Werror"]) + AS_VAR_APPEND([CFLAGS], [" -W$2 -Werror"]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])], [AS_VAR_SET([py_var], [yes])], [AS_VAR_SET([py_var], [no])]) @@ -2211,6 +2512,11 @@ AS_VAR_IF([with_strict_overflow], [yes], [BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS"], [BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS"]) +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. +AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [BASECFLAGS="$BASECFLAGS -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) +AX_CHECK_COMPILE_FLAG([-Wtrampolines], [BASECFLAGS="$BASECFLAGS -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -2760,15 +3066,11 @@ AC_CHECK_HEADERS( #endif ]) -# checks for typedefs -AC_CACHE_CHECK([for clock_t in time.h], [ac_cv_clock_t_time_h], [ - AC_EGREP_HEADER([clock_t], [time.h], [ac_cv_clock_t_time_h=yes], [ac_cv_clock_t_time_h=no]) -]) -dnl checks for "no" -AS_VAR_IF([ac_cv_clock_t_time_h], [no], [ - AC_DEFINE([clock_t], [long], - [Define to 'long' if doesn't define.]) -]) +# Check for clock_t in time.h. +AC_CHECK_TYPES([clock_t], [], + [AC_DEFINE([clock_t], [long], + [Define to 'long' if does not define clock_t.])], + [@%:@include ]) AC_CACHE_CHECK([for makedev], [ac_cv_func_makedev], [ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -2843,12 +3145,10 @@ AC_DEFINE_UNQUOTED([RETSIGTYPE],[void],[assume C89 semantics that RETSIGTYPE is AC_TYPE_SIZE_T AC_TYPE_UID_T -AC_CHECK_TYPE([ssize_t], - AC_DEFINE([HAVE_SSIZE_T], [1], - [Define if your compiler provides ssize_t]), [], []) -AC_CHECK_TYPE([__uint128_t], - AC_DEFINE([HAVE_GCC_UINT128_T], [1], - [Define if your compiler provides __uint128_t]), [], []) +AC_CHECK_TYPES([ssize_t]) +AC_CHECK_TYPES([__uint128_t], + [AC_DEFINE([HAVE_GCC_UINT128_T], [1], + [Define if your compiler provides __uint128_t])]) # Sizes and alignments of various common basic types # ANSI C requires sizeof(char) == 1, so no need to check it @@ -3065,6 +3365,24 @@ AC_MSG_RESULT([no]) with_ubsan="no" ]) +AC_MSG_CHECKING([for --with-thread-sanitizer]) +AC_ARG_WITH( + [thread_sanitizer], + [AS_HELP_STRING( + [--with-thread-sanitizer], + [enable ThreadSanitizer data race detector, 'tsan' (default is no)] + )], +[ +AC_MSG_RESULT([$withval]) +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" +], +[ +AC_MSG_RESULT([no]) +with_tsan="no" +]) + # Set info about shared libraries. AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) @@ -3165,7 +3483,12 @@ then BLDSHARED="$LDSHARED" fi ;; - Emscripten|WASI) + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + BLDSHARED="$LDSHARED" + ;; + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -3249,7 +3572,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ @@ -3285,30 +3607,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], + [0x$stack_size], + [Custom thread stack size depending on chosen sanitizer runtimes.]) - AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], - [0x$stack_size], - [Custom thread stack size depending on chosen sanitizer runtimes.]) + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -3502,6 +3828,35 @@ dnl The AIX_BUILDDATE is obtained from the kernel fileset - bos.mp64 *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +}]])], [ac_cv_c_complex_supported=yes], +[ac_cv_c_complex_supported=no], +[ac_cv_c_complex_supported=no]) +if test "$ac_cv_c_complex_supported" = "yes"; then + AC_DEFINE([Py_HAVE_C_COMPLEX], [1], + [Defined if _Complex C type is available.]) +fi + # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ @@ -3519,7 +3874,14 @@ int main(void) }]])], [ac_cv_aligned_required=no], [ac_cv_aligned_required=yes], -[ac_cv_aligned_required=yes]) +[ +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else + ac_cv_aligned_required=yes +fi]) ]) if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], @@ -3682,6 +4044,9 @@ AS_VAR_IF([have_libffi], [yes], [ dnl when do we need USING_APPLE_OS_LIBFFI? ctypes_malloc_closure=yes ], + [iOS], [ + ctypes_malloc_closure=yes + ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] ) AS_VAR_IF([ctypes_malloc_closure], [yes], [ @@ -3709,29 +4074,57 @@ AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], - [build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no)] + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], [], - [with_system_libmpdec="no"]) + [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) -AS_VAR_IF([with_system_libmpdec], [yes], [ - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= -], [ - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - - dnl Disable forced inlining in debug builds, see GH-94847 - AS_VAR_IF([with_pydebug], [yes], [ - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"]) - ]) -]) - -AC_SUBST([LIBMPDEC_CFLAGS]) -AC_SUBST([LIBMPDEC_INTERNAL]) +AC_DEFUN([USE_BUNDLED_LIBMPDEC], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no]) + +AS_VAR_IF( + [with_system_libmpdec], [yes], + [PKG_CHECK_MODULES( + [LIBMPDEC], [libmpdec >= 2.5.0], [], + [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL=])], + [USE_BUNDLED_LIBMPDEC()]) + +AS_VAR_IF([with_system_libmpdec], [yes], + [WITH_SAVE_ENV([ + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [const char *x = mpd_version();])], + [have_mpdec=yes], + [have_mpdec=no]) + ])], + [AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + consider using a system installed mpdecimal library.])])]) + +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([m4_normalize([ + no system libmpdecimal found; falling back to bundled libmpdecimal + (deprecated and scheduled for removal in Python 3.15)])]) + USE_BUNDLED_LIBMPDEC()]) + +# Disable forced inlining in debug builds, see GH-94847 +AS_VAR_IF( + [with_pydebug], [yes], + [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"])]) # Check whether _decimal should use a coroutine-local or thread-local context AC_MSG_CHECKING([for --with-decimal-contextvar]) @@ -3752,50 +4145,52 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) -# Check for libmpdec machine flavor -AC_MSG_CHECKING([for decimal libmpdec machine]) -AS_CASE([$ac_sys_system], - [Darwin*], [libmpdec_system=Darwin], - [SunOS*], [libmpdec_system=sunos], - [libmpdec_system=other] -) - -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$CC], - [*gcc*], [libmpdec_machine=ppro], - [*clang*], [libmpdec_machine=ppro], - [libmpdec_machine=ansi32] - ) - else - libmpdec_machine=ansi32 - fi -fi -AC_MSG_RESULT([$libmpdec_machine]) - -AS_CASE([$libmpdec_machine], - [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], - [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], - [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], - [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], - [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], - [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], - [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], - [AC_MSG_ERROR([_decimal: unsupported architecture])] -) +AS_VAR_IF( + [with_system_libmpdec], [no], + [# Check for libmpdec machine flavor + AC_MSG_CHECKING([for decimal libmpdec machine]) + AS_CASE([$ac_sys_system], + [Darwin*], [libmpdec_system=Darwin], + [SunOS*], [libmpdec_system=sunos], + [libmpdec_system=other] + ) + + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + AS_CASE([$CC], + [*gcc*], [libmpdec_machine=ppro], + [*clang*], [libmpdec_machine=ppro], + [libmpdec_machine=ansi32] + ) + else + libmpdec_machine=ansi32 + fi + fi + AC_MSG_RESULT([$libmpdec_machine]) + + AS_CASE([$libmpdec_machine], + [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], + [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], + [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], + [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], + [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], + [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], + [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], + [AC_MSG_ERROR([_decimal: unsupported architecture])] + )]) if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -3810,6 +4205,9 @@ if test "$have_glibc_memmove_bug" = yes; then AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -U_FORTIFY_SOURCE"]) fi +AC_SUBST([LIBMPDEC_CFLAGS]) +AC_SUBST([LIBMPDEC_INTERNAL]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -4168,13 +4566,9 @@ else # define _POSIX_THREADS in unistd.h. Some apparently don't # (e.g. gnu pth with pthread emulation) AC_MSG_CHECKING([for _POSIX_THREADS in unistd.h]) - AC_EGREP_CPP([yes], - [ -#include -#ifdef _POSIX_THREADS -yes -#endif - ], unistd_defines_pthreads=yes, unistd_defines_pthreads=no) + AX_CHECK_DEFINE([unistd.h], [_POSIX_THREADS], + [unistd_defines_pthreads=yes], + [unistd_defines_pthreads=no]) AC_MSG_RESULT([$unistd_defines_pthreads]) AC_DEFINE([_REENTRANT]) @@ -4347,41 +4741,27 @@ ipv6type=unknown ipv6lib=none ipv6trylibc=no -if test "$ipv6" = "yes"; then - AC_MSG_CHECKING([ipv6 stack type]) +if test "$ipv6" = yes -a "$cross_compiling" = no; then for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef IPV6_INRIA_VERSION -yes -@%:@endif], - [ipv6type=$i]) + AX_CHECK_DEFINE([netinet/in.h], [IPV6_INRIA_VERSION], [ipv6type=$i]) ;; kame) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef __KAME__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes]) + AX_CHECK_DEFINE([netinet/in.h], [__KAME__], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes]) ;; linux-glibc) - dnl http://www.v6.linux.or.jp/ - AC_EGREP_CPP([yes], [ -#include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -@%:@endif], - [ipv6type=$i; - ipv6trylibc=yes]) + dnl Advanced IPv6 support was added to glibc 2.1 in 1999. + AX_CHECK_DEFINE([features.h], [__GLIBC__], + [ipv6type=$i + ipv6trylibc=yes]) ;; linux-inet6) dnl http://www.v6.linux.or.jp/ @@ -4401,51 +4781,41 @@ yes fi ;; toshiba) - AC_EGREP_CPP([yes], [ -#include -#ifdef _TOSHIBA_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_TOSHIBA_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; v6d) - AC_EGREP_CPP([yes], [ -#include -#ifdef __V6D__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) + AX_CHECK_DEFINE([/usr/local/v6/include/sys/v6config.h], [__V6D__], + [ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) ;; zeta) - AC_EGREP_CPP([yes], [ -#include -#ifdef _ZETA_MINAMI_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_ZETA_MINAMI_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; esac if test "$ipv6type" != "unknown"; then break fi done + AC_MSG_CHECKING([ipv6 stack type]) AC_MSG_RESULT([$ipv6type]) fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + AC_MSG_CHECKING([ipv6 library]) if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - AC_MSG_NOTICE([using lib$ipv6lib]) + AC_MSG_RESULT([lib$ipv6lib]) else AS_VAR_IF([ipv6trylibc], [yes], [ - AC_MSG_NOTICE([using libc]) + AC_MSG_RESULT([libc]) ], [ AC_MSG_ERROR([m4_normalize([ No $ipv6lib library found; cannot continue. @@ -4497,6 +4867,73 @@ then fi AC_MSG_RESULT([$with_doc_strings]) +# Check for stdatomic.h, required for mimalloc. +AC_CACHE_CHECK([for stdatomic.h], [ac_cv_header_stdatomic_h], [ +AC_LINK_IFELSE( +[ + AC_LANG_SOURCE([[ + #include + atomic_int int_var; + atomic_uintptr_t uintptr_var; + int main() { + atomic_store_explicit(&int_var, 5, memory_order_relaxed); + atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); + int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); + return 0; + } + ]]) +],[ac_cv_header_stdatomic_h=yes],[ac_cv_header_stdatomic_h=no]) +]) + +AS_VAR_IF([ac_cv_header_stdatomic_h], [yes], [ + AC_DEFINE(HAVE_STD_ATOMIC, 1, + [Has stdatomic.h with atomic_int and atomic_uintptr_t]) +]) + +# Check for GCC >= 4.7 and clang __atomic builtin functions +AC_CACHE_CHECK([for builtin __atomic_load_n and __atomic_store_n functions], [ac_cv_builtin_atomic], [ +AC_LINK_IFELSE( +[ + AC_LANG_SOURCE([[ + int val; + int main() { + __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); + (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); + return 0; + } + ]]) +],[ac_cv_builtin_atomic=yes],[ac_cv_builtin_atomic=no]) +]) + +AS_VAR_IF([ac_cv_builtin_atomic], [yes], [ + AC_DEFINE(HAVE_BUILTIN_ATOMIC, 1, [Has builtin __atomic_load_n() and __atomic_store_n() functions]) +]) + +# --with-mimalloc +AC_MSG_CHECKING([for --with-mimalloc]) +AC_ARG_WITH([mimalloc], + [AS_HELP_STRING([--with-mimalloc], + [build with mimalloc memory allocator (default is yes if C11 stdatomic.h is available.)])], + [], + [with_mimalloc="$ac_cv_header_stdatomic_h"] +) + +if test "$with_mimalloc" != no; then + if test "$ac_cv_header_stdatomic_h" != yes; then + # mimalloc-atomic.h wants C11 stdatomic.h on POSIX + AC_MSG_ERROR([mimalloc requires stdatomic.h, use --without-mimalloc to disable mimalloc.]) + fi + with_mimalloc=yes + AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.]) + AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)']) +elif test "$disable_gil" = "yes"; then + AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).]) +fi + +AC_MSG_RESULT([$with_mimalloc]) +AC_SUBST([INSTALL_MIMALLOC], [$with_mimalloc]) +AC_SUBST([MIMALLOC_HEADERS]) + # Check for Python-specific malloc support AC_MSG_CHECKING([for --with-pymalloc]) AC_ARG_WITH( @@ -4674,22 +5111,47 @@ else AC_MSG_RESULT([$MACHDEP_OBJS]) fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi + + for name in $blocked_funcs; do + AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) + AS_VAR_SET([func_var], [no]) + AS_VAR_POPDEF([func_var]) + done +fi + # checks for library functions AC_CHECK_FUNCS([ \ - accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ + accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ - getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ + getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \ - pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \ - pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ + posix_spawn_file_actions_addclosefrom_np \ + pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -4697,8 +5159,8 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat utimensat utimes vfork \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -4709,6 +5171,14 @@ if test "$MACHDEP" != linux; then AC_CHECK_FUNCS([lchmod]) fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + AC_CHECK_FUNCS([getentropy getgroups system]) +fi + AC_CHECK_DECL([dirfd], [AC_DEFINE([HAVE_DIRFD], [1], [Define if you have the 'dirfd' function or macro.])], @@ -5009,17 +5479,28 @@ AC_CHECK_FUNCS([clock_getres], [], [ ]) ]) -AC_CHECK_FUNCS([clock_settime], [], [ - AC_CHECK_LIB([rt], [clock_settime], [ - AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) - ]) -]) +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then + AC_CHECK_FUNCS([clock_settime], [], [ + AC_CHECK_LIB([rt], [clock_settime], [ + AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) + ]) + ]) +fi -AC_CHECK_FUNCS([clock_nanosleep], [], [ - AC_CHECK_LIB([rt], [clock_nanosleep], [ - AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) - ]) -]) +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then + AC_CHECK_FUNCS([clock_nanosleep], [], [ + AC_CHECK_LIB([rt], [clock_nanosleep], [ + AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([nanosleep], [], [ AC_CHECK_LIB([rt], [nanosleep], [ @@ -5032,6 +5513,7 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -5158,7 +5640,9 @@ int main(void) [ac_cv_buggy_getaddrinfo=no], [ac_cv_buggy_getaddrinfo=yes], [ -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -5714,7 +6198,7 @@ AC_SUBST([SOABI]) AC_MSG_CHECKING([ABIFLAGS]) AC_MSG_RESULT([$ABIFLAGS]) AC_MSG_CHECKING([SOABI]) -SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} +SOABI='cpython-'`echo $VERSION | tr -d .`${ABIFLAGS}${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_MSG_RESULT([$SOABI]) # Release build, debug build (Py_DEBUG), and trace refs build (Py_TRACE_REFS) @@ -5722,7 +6206,7 @@ AC_MSG_RESULT([$SOABI]) if test "$Py_DEBUG" = 'true'; then # Similar to SOABI but remove "d" flag from ABIFLAGS AC_SUBST([ALT_SOABI]) - ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${PLATFORM_TRIPLET:+-$PLATFORM_TRIPLET} + ALT_SOABI='cpython-'`echo $VERSION | tr -d .``echo $ABIFLAGS | tr -d d`${SOABI_PLATFORM:+-$SOABI_PLATFORM} AC_DEFINE_UNQUOTED([ALT_SOABI], ["${ALT_SOABI}"], [Alternative SOABI used in debug build to load C extensions built in release mode]) fi @@ -5734,21 +6218,30 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) +# Configure the flags and dependencies used when compiling shared modules +AC_SUBST([MODULE_DEPS_SHARED]) +AC_SUBST([MODULE_LDFLAGS]) +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' + # On Android and Cygwin the shared libraries must be linked with libpython. -AC_SUBST([LIBPYTHON]) if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" = "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi AC_SUBST([BINLIBDEST]) -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) AC_SUBST([PLATLIBDIR]) PLATLIBDIR="lib" AC_MSG_CHECKING([for --with-platlibdir]) @@ -5767,7 +6260,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no then AC_MSG_RESULT([yes]) PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else AC_MSG_RESULT([no]) fi], @@ -5777,9 +6270,9 @@ fi], dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST([PY_ENABLE_SHARED]) if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi AC_SUBST([LIBPL]) @@ -5847,7 +6340,6 @@ dnl library (tinfo ncursesw ncurses termcap). We now assume that libreadline dnl or readline.pc provide correct linker information. AH_TEMPLATE([WITH_EDITLINE], [Define to build the readline module against libedit.]) -AH_TEMPLATE([WITH_APPLE_EDITLINE], [Define to build the readline module against Apple BSD editline.]) AC_ARG_WITH( [readline], @@ -5864,15 +6356,6 @@ AC_ARG_WITH( [with_readline=readline] ) -# gh-105323: Need to handle the macOS editline as an alias of readline. -AS_CASE([$ac_sys_system/$ac_sys_release], - [Darwin/*], [AC_CHECK_TYPE([Function], - [AC_DEFINE([WITH_APPLE_EDITLINE])], - [], - [@%:@include ])], - [] -) - AS_VAR_IF([with_readline], [readline], [ PKG_CHECK_MODULES([LIBREADLINE], [readline], [ LIBREADLINE=readline @@ -6008,11 +6491,22 @@ AS_VAR_IF([with_readline], [no], [ ]) # in readline as well as newer editline (April 2023) - AC_CHECK_TYPE([rl_compdisp_func_t], - [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1], - [Define if readline supports rl_compdisp_func_t])], - [], - [readline_includes]) + AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes]) + + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + AC_CACHE_CHECK([if rl_startup_hook takes arguments], [ac_cv_readline_rl_startup_hook_takes_args], [ + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([readline_includes] + [extern int test_hook_func(const char *text, int state);], + [rl_startup_hook=test_hook_func;])], + [ac_cv_readline_rl_startup_hook_takes_args=yes], + [ac_cv_readline_rl_startup_hook_takes_args=no] + ) + ]) + AS_VAR_IF([ac_cv_readline_rl_startup_hook_takes_args], [yes], [ + AC_DEFINE([Py_RL_STARTUP_HOOK_TAKES_ARGS], [1], [Define if rl_startup_hook takes arguments]) + ]) m4_undefine([readline_includes]) ])dnl WITH_SAVE_ENV() @@ -6172,138 +6666,136 @@ then [Define if you have struct stat.st_mtimensec]) fi -dnl check for ncurses/ncursesw and panel/panelw +AC_CACHE_CHECK([whether year with century should be normalized for strftime], [ac_cv_normalize_century], [ +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include + +int main(void) +{ + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } + return 0; +} +]])], +[ac_cv_normalize_century=yes], +[ac_cv_normalize_century=no], +[ac_cv_normalize_century=yes])]) +if test "$ac_cv_normalize_century" = yes +then + AC_DEFINE([Py_NORMALIZE_CENTURY], [1], + [Define if year with century should be normalized for strftime.]) +fi + +dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. -dnl have_curses=[no, ncursesw, ncurses] -dnl have_panel=[no, panelw, panel] +dnl have_curses=[no, yes] +dnl have_panel=[no, yes] have_curses=no have_panel=no -AH_TEMPLATE([HAVE_NCURSESW], [Define to 1 if you have the `ncursesw' library.]) -AC_CHECK_HEADERS([curses.h ncurses.h]) - -AS_VAR_IF([ac_cv_header_ncurses_h], [yes], [ - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - PKG_CHECK_MODULES([CURSES], [ncursesw], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncursesw], [initscr], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [no], [ - PKG_CHECK_MODULES([CURSES], [ncurses], [ - have_curses=ncurses - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncurses], [initscr], [ - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - ]) - ]) - ]) - ]) +dnl PY_CHECK_CURSES(LIBCURSES, LIBPANEL) +dnl Sets 'have_curses' and 'have_panel'. +dnl For the PKG_CHECK_MODULES() calls, we can safely reuse the first variable +dnl here, since we're only calling the macro a second time if the first call +dnl fails. +AC_DEFUN([PY_CHECK_CURSES], [dnl +AS_VAR_PUSHDEF([curses_var], [m4_toupper([$1])]) +AS_VAR_PUSHDEF([panel_var], [m4_toupper([$2])]) +PKG_CHECK_MODULES([CURSES], [$1], + [AC_DEFINE([HAVE_]curses_var, [1], [Define if you have the '$1' library]) + AS_VAR_SET([have_curses], [yes]) + PKG_CHECK_MODULES([PANEL], [$2], + [AC_DEFINE([HAVE_]panel_var, [1], [Define if you have the '$2' library]) + AS_VAR_SET([have_panel], [yes])], + [AS_VAR_SET([have_panel], [no])])], + [AS_VAR_SET([have_curses], [no])]) +AS_VAR_POPDEF([curses_var]) +AS_VAR_POPDEF([panel_var])]) + +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. +PY_CHECK_CURSES([ncursesw], [panelw]) +AS_VAR_IF([have_curses], [no], + [PY_CHECK_CURSES([ncurses], [panel])]) -])dnl ac_cv_header_ncurses_h = yes +WITH_SAVE_ENV([ + # Make sure we've got the header defines. + AS_VAR_APPEND([CPPFLAGS], [" $CURSES_CFLAGS $PANEL_CFLAGS"]) + AC_CHECK_HEADERS(m4_normalize([ + ncursesw/curses.h ncursesw/ncurses.h ncursesw/panel.h + ncurses/curses.h ncurses/ncurses.h ncurses/panel.h + curses.h ncurses.h panel.h + ])) + + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + AS_VAR_APPEND([LIBS], [" $CURSES_LIBS $PANEL_LIBS"]) + AC_SEARCH_LIBS([initscr], [ncursesw ncurses], + [AS_VAR_IF([have_curses], [no], + [AS_VAR_SET([have_curses], [yes]) + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"}])], + [AS_VAR_SET([have_curses], [no])]) + AC_SEARCH_LIBS([update_panels], [panelw panel], + [AS_VAR_IF([have_panel], [no], + [AS_VAR_SET([have_panel], [yes]) + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"}])], + [AS_VAR_SET([have_panel], [no])]) + +dnl Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque +dnl structs since version 5.7. If the macro is defined as zero before including +dnl [n]curses.h, ncurses will expose fields of the structs regardless of the +dnl configuration. +AC_DEFUN([_CURSES_INCLUDES],dnl +[ +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif +]) +AS_IF([test "have_curses" != "no"], [ dnl remove _XOPEN_SOURCE macro from curses cflags. pyconfig.h sets dnl the macro to 700. CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -if test "$have_curses" = no -a "$ac_sys_system" = "Darwin"; then +AS_VAR_IF([ac_sys_system], [Darwin], [ dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - dnl If we are here, we found a locally-supplied version of libncursesw. + dnl System-supplied ncurses combines libncurses/libpanel and supports wide + dnl characters, so we can use it like ncursesw. + dnl If a locally-supplied version of libncursesw is found, we will use that. dnl There should also be a libpanelw. dnl _XOPEN_SOURCE defines are usually excluded for macOS, but we need dnl _XOPEN_SOURCE_EXTENDED here for ncurses wide char support. AS_VAR_APPEND([CURSES_CFLAGS], [" -D_XOPEN_SOURCE_EXTENDED=1"]) - AC_DEFINE([HAVE_NCURSESW], [1]) -fi - -dnl TODO: detect "curses" and special cases tinfo, terminfo, or termcap - -AC_MSG_CHECKING([curses module flags]) -AS_VAR_IF([have_curses], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)]) ]) -dnl check for ncurses' panel/panelw library -AC_CHECK_HEADERS([panel.h]) - -AS_VAR_IF([ac_cv_header_panel_h], [yes], [ - - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - AS_VAR_IF([have_curses], [ncursesw], [ - PKG_CHECK_MODULES([PANEL], [panelw], [ - have_panel=panelw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panelw], [update_panels], [ - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - ]) - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [ncurses], [ - PKG_CHECK_MODULES([PANEL], [panel], [ - have_panel=panel - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panel], [update_panels], [ - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - ]) - ]) - ]) - ]) - -])dnl ac_cv_header_panel_h = yes - dnl pyconfig.h defines _XOPEN_SOURCE=700 PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -AC_MSG_CHECKING([panel flags]) -AS_VAR_IF([have_panel], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)]) -]) - -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" -fi - # On Solaris, term.h requires curses.h -AC_CHECK_HEADERS([term.h], [], [], [ -#ifdef HAVE_CURSES_H -#include -#endif -]) +AC_CHECK_HEADERS([term.h], [], [], _CURSES_INCLUDES) # On HP/UX 11.0, mvwdelch is a block with a return statement AC_CACHE_CHECK([whether mvwdelch is an expression], [ac_cv_mvwdelch_is_expression], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ int rtn; rtn = mvwdelch(0,0,0); ]])], @@ -6316,15 +6808,8 @@ then [Define if mvwdelch in curses.h is an expression.]) fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. AC_CACHE_CHECK([whether WINDOW has _flags], [ac_cv_window_has_flags], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #define NCURSES_OPAQUE 0 - #include -]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ WINDOW *w; w->_flags = 0; ]])], @@ -6346,8 +6831,7 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], [for curses function $1], [py_var], [AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM( - [@%:@include ], [ + [AC_LANG_PROGRAM(_CURSES_INCLUDES, [ #ifndef $1 void *x=$1 #endif @@ -6375,32 +6859,41 @@ PY_CHECK_CURSES_FUNC([has_key]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) CPPFLAGS=$ac_save_cppflags +])dnl have_curses != no +])dnl save env AC_MSG_NOTICE([checking for device files]) dnl NOTE: Inform user how to proceed with files when cross compiling. -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - AC_MSG_CHECKING([for /dev/ptmx]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - AC_MSG_CHECKING([for /dev/ptc]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) +dnl Some cross-compile builds are predictable; they won't ever +dnl have /dev/ptmx or /dev/ptc, so we can set them explicitly. +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + AC_MSG_CHECKING([for /dev/ptmx]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + AC_MSG_CHECKING([for /dev/ptc]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) + fi fi -fi -AC_CHECK_FILE([/dev/ptmx], [], []) -if test "x$ac_cv_file__dev_ptmx" = xyes; then - AC_DEFINE([HAVE_DEV_PTMX], [1], - [Define to 1 if you have the /dev/ptmx device file.]) -fi -AC_CHECK_FILE([/dev/ptc], [], []) -if test "x$ac_cv_file__dev_ptc" = xyes; then - AC_DEFINE([HAVE_DEV_PTC], [1], - [Define to 1 if you have the /dev/ptc device file.]) + AC_CHECK_FILE([/dev/ptmx], [], []) + if test "x$ac_cv_file__dev_ptmx" = xyes; then + AC_DEFINE([HAVE_DEV_PTMX], [1], + [Define to 1 if you have the /dev/ptmx device file.]) + fi + AC_CHECK_FILE([/dev/ptc], [], []) + if test "x$ac_cv_file__dev_ptc" = xyes; then + AC_DEFINE([HAVE_DEV_PTC], [1], + [Define to 1 if you have the /dev/ptc device file.]) + fi fi if test $ac_sys_system = Darwin @@ -6408,12 +6901,9 @@ then LIBS="$LIBS -framework CoreFoundation" fi -AC_CHECK_TYPE( - [socklen_t], [], - [AC_DEFINE( - [socklen_t], [int], - [Define to `int' if does not define.] - )], [ +AC_CHECK_TYPES([socklen_t], [], + [AC_DEFINE([socklen_t], [int], + [Define to 'int' if does not define.])], [ #ifdef HAVE_SYS_TYPES_H #include #endif @@ -6521,17 +7011,19 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ Objects \ + Objects/mimalloc \ + Objects/mimalloc/prim \ Parser \ Parser/tokenizer \ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then @@ -6622,49 +7114,6 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then esac fi -# Check for stdatomic.h -AC_CACHE_CHECK([for stdatomic.h], [ac_cv_header_stdatomic_h], [ -AC_LINK_IFELSE( -[ - AC_LANG_SOURCE([[ - #include - atomic_int int_var; - atomic_uintptr_t uintptr_var; - int main(void) { - atomic_store_explicit(&int_var, 5, memory_order_relaxed); - atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); - int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); - return 0; - } - ]]) -],[ac_cv_header_stdatomic_h=yes],[ac_cv_header_stdatomic_h=no]) -]) - -AS_VAR_IF([ac_cv_header_stdatomic_h], [yes], [ - AC_DEFINE([HAVE_STD_ATOMIC], [1], - [Has stdatomic.h with atomic_int and atomic_uintptr_t]) -]) - -# Check for GCC >= 4.7 and clang __atomic builtin functions -AC_CACHE_CHECK([for builtin __atomic_load_n and __atomic_store_n functions], [ac_cv_builtin_atomic], [ -AC_LINK_IFELSE( -[ - AC_LANG_SOURCE([[ - int val; - int main(void) { - __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); - (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); - return 0; - } - ]]) -],[ac_cv_builtin_atomic=yes],[ac_cv_builtin_atomic=no]) -]) - -AS_VAR_IF([ac_cv_builtin_atomic], [yes], [ - AC_DEFINE([HAVE_BUILTIN_ATOMIC], [1], - [Has builtin __atomic_load_n() and __atomic_store_n() functions]) -]) - # ensurepip option AC_MSG_CHECKING([for ensurepip]) AC_ARG_WITH([ensurepip], @@ -6675,6 +7124,7 @@ AC_ARG_WITH([ensurepip], AS_CASE([$ac_sys_system], [Emscripten], [with_ensurepip=no], [WASI], [with_ensurepip=no], + [iOS], [with_ensurepip=no], [with_ensurepip=upgrade] ) ]) @@ -6787,7 +7237,12 @@ AX_CHECK_OPENSSL([have_openssl=yes],[have_openssl=no]) AS_VAR_IF([GNULD], [yes], [ rpath_arg="-Wl,--enable-new-dtags,-rpath=" ], [ - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi ]) AC_MSG_CHECKING([for --with-openssl-rpath]) @@ -7005,6 +7460,9 @@ AC_SUBST([TEST_MODULES]) # libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the # compiler flags. # +# gh-112779: On RISC-V, GCC 12 and earlier require libatomic support for 1-byte +# and 2-byte operations, but not for 8-byte operations. +# # Avoid #include or #include . The header # requires header which is only written below by AC_OUTPUT below. # If the check is done after AC_OUTPUT, modifying LIBS has no effect @@ -7034,12 +7492,19 @@ typedef intptr_t Py_ssize_t; int main() { - uint64_t byte; - _Py_atomic_store_uint64(&byte, 2); - if (_Py_atomic_or_uint64(&byte, 8) != 2) { + uint64_t value; + _Py_atomic_store_uint64(&value, 2); + if (_Py_atomic_or_uint64(&value, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&value) != 10) { return 1; // error } - if (_Py_atomic_load_uint64(&byte) != 10) { + uint8_t byte = 0xb8; + if (_Py_atomic_or_uint8(&byte, 0x2d) != 0xb8) { + return 1; // error + } + if (_Py_atomic_load_uint8(&byte) != 0xbd) { return 1; // error } return 0; // all good @@ -7051,7 +7516,8 @@ int main() ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], - [LIBS="${LIBS} -latomic"]) + [LIBS="${LIBS} -latomic" + LIBATOMIC=${LIBATOMIC-"-latomic"}]) _RESTORE_VAR([CPPFLAGS]) @@ -7068,6 +7534,28 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [termios], [grp])], dnl The _scproxy module is available on macOS [Darwin], [], + [iOS], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl gdbm and nis aren't available + dnl Stub implementations are provided for pwd, grp etc APIs + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [grp], + [nis], + [readline], + [pwd], + [spwd], + [syslog], + ) + ], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], @@ -7087,8 +7575,9 @@ AS_CASE([$ac_sys_system], [_posixsubprocess], [_scproxy], [_tkinter], - [_xxsubinterpreters], - [_xxinterpchannels], + [_interpreters], + [_interpchannels], + [_interpqueues], [grp], [pwd], [resource], @@ -7106,11 +7595,19 @@ AS_CASE([$ac_sys_system], [Emscripten/node*], [], [WASI/*], [ dnl WASI SDK 15.0 does not support file locking, mmap, and more. + dnl Test modules that must be compiled as shared libraries are not supported + dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], + [_testexternalinspection], + [_testimportmultiple], + [_testmultiphase], + [_testsinglephase], [fcntl], [mmap], [termios], + [xxlimited], + [xxlimited_35], ) ] ) @@ -7195,7 +7692,6 @@ PY_STDLIB_MOD_SIMPLE([_csv]) PY_STDLIB_MOD_SIMPLE([_heapq]) PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) -PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) @@ -7203,8 +7699,9 @@ PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) -PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) -PY_STDLIB_MOD_SIMPLE([_xxinterpchannels]) +PY_STDLIB_MOD_SIMPLE([_interpreters]) +PY_STDLIB_MOD_SIMPLE([_interpchannels]) +PY_STDLIB_MOD_SIMPLE([_interpqueues]) PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules @@ -7235,7 +7732,9 @@ PY_STDLIB_MOD([_socket], -a "$ac_cv_header_netinet_in_h" = "yes"])) dnl platform specific extensions -PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) +PY_STDLIB_MOD([grp], [], + [test "$ac_cv_func_getgrent" = "yes" && + { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; }]) PY_STDLIB_MOD([pwd], [], [test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes]) PY_STDLIB_MOD([resource], [], [test "$ac_cv_header_sys_resource_h" = yes]) PY_STDLIB_MOD([_scproxy], @@ -7278,14 +7777,16 @@ PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) PY_STDLIB_MOD([_curses], - [], [test "$have_curses" != "no"], + [], [test "$have_curses" = "yes"], [$CURSES_CFLAGS], [$CURSES_LIBS] ) PY_STDLIB_MOD([_curses_panel], - [], [test "$have_panel" != "no"], + [], [test "$have_curses" = "yes" && test "$have_panel" = "yes"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS]) +PY_STDLIB_MOD([_decimal], + [], [test "$have_mpdec" = "yes"], + [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) @@ -7323,23 +7824,29 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules -PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testcapi], + [test "$TEST_MODULES" = yes], + dnl Modules/_testcapi needs -latomic for 32bit AIX build + [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testlimitedcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], - [], [$LIBM]) + [$LIBFFI_CFLAGS], [$LIBFFI_LIBS $LIBM]) dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) @@ -7392,3 +7899,10 @@ AS_VAR_IF([PY_SUPPORT_TIER], [0], [AC_MSG_WARN([ Platform "$host" with compiler "$ac_cv_cc_name" is not supported by the CPython core team, see https://peps.python.org/pep-0011/ for more information. ])]) + +if test "$ac_cv_header_stdatomic_h" != "yes"; then + AC_MSG_NOTICE(m4_normalize([ + Your compiler or platform does have a working C11 stdatomic.h. A future + version of Python may require stdatomic.h. + ])) +fi diff --git a/iOS/README.rst b/iOS/README.rst new file mode 100644 index 00000000000000..96cb00eb2e9bfe --- /dev/null +++ b/iOS/README.rst @@ -0,0 +1,385 @@ +==================== +Python on iOS README +==================== + +:Authors: + Russell Keith-Magee (2023-11) + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase +`__ and `Kivy's Buildozer +`__ will provide a much more approachable +user experience. + +Compilers for building on iOS +============================= + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +iOS specific arguments to configure +=================================== + +* ``--enable-framework[=DIR]`` + + This argument specifies the location where the Python.framework will be + installed. If ``DIR`` is not specified, the framework will be installed into + a subdirectory of the ``iOS/Frameworks`` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* ``--with-framework-name=NAME`` + + Specify the name for the Python framework; defaults to ``Python``. + + .. admonition:: Use this option with care! + + Unless you know what you're doing, changing the name of the Python + framework on iOS is not advised. If you use this option, you won't be able + to run the ``make testios`` target without making significant manual + alterations, and you won't be able to use any binary packages unless you + compile them yourself using your own framework name. + +Building Python on iOS +====================== + +ABIs and Architectures +---------------------- + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (``iphoneos``) or an +iOS simulator build (``iphonesimulator``). + +Apple uses the ``XCframework`` format to allow specifying a single dependency +that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +Building a single-architecture framework +---------------------------------------- + +The Python build system will create a ``Python.framework`` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. +The ``lib`` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +``Python.framework`` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the ``--enable-framework`` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like:: + + $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ ./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe + $ make + $ make install + +In this invocation: + +* ``iOS/Resources/bin`` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` + to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the + result passed to ``configure``, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, + it requires that compiler variables like ``CC`` include spaces, which can + cause significant problems with many C configuration systems which assume that + ``CC`` will be a single executable. + + To work around this problem, the ``iOS/Resources/bin`` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to ``xcrun``. This allows configure to use a ``CC`` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the ``bin`` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + ``xcode-select`` to set a new default Xcode globally, or you can use the + ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The + scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + ``IOS_SDK_VERSION`` environment variable. (e.g, setting + ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` + and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* ``--host`` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - ``arm64-apple-ios`` for ARM64 iOS devices. + - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple + Silicon devices. + - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel + devices. + +* ``--build`` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - ``arm64-apple-darwin`` for Apple Silicon devices. + - ``x86_64-apple-darwin`` for Intel devices. + +* ``/path/to/python.exe`` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The ``install`` target for iOS builds is slightly different to other + platforms. On most platforms, ``make install`` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the ``install`` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + ``--enable-framework`` will be the location where ``make install`` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). +This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, +``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` +environment variables, and the ``--with-openssl`` configure option. Versions of +these libraries pre-compiled for iOS can be found in `this repository +`__. LibFFI is +especially important, as many parts of the standard library (including the +``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the +``ctypes`` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 12.0. To specify a different deployment +target, provide the version number as part of the ``--host`` argument - for +example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 +simulator build with a deployment target of 15.4. + +Merge thin frameworks into fat frameworks +----------------------------------------- + +Once you've built a ``Python.framework`` for each ABI and and architecture, you +must produce a "fat" framework for each ABI that contains all the architectures +for that ABI. + +The ``iphoneos`` build only needs to support a single architecture, so it can be +used without modification. + +If you only want to support a single simulator architecture, (e.g., only support +ARM64 simulators), you can use a single architecture ``Python.framework`` build. +However, if you want to create ``Python.xcframework`` that supports *all* +architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and +x86_64 into a single "fat" framework. + +The "fat" framework can be constructed by performing a directory merge of the +content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and +``lib`` folders for each thin framework. When performing this merge: + +* The pure Python standard library content is identical for each architecture, + except for a handful of platform-specific files (such as the ``sysconfig`` + module). Ensure that the "fat" framework has the union of all standard library + files. + +* Any binary files in the standard library, plus the main + ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by + Xcode:: + + $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib + +* The header files will be identical on both architectures, except for + ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename + ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the + other architecture into the merged header folder as ``pyconfig-x86_64.h``. + Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into + the merged headers folder. This will allow the two Python architectures to + share a common ``pyconfig.h`` header file. + +At this point, you should have 2 Python.framework folders - one for ``iphoneos``, +and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. + +Merge frameworks into an XCframework +------------------------------------ + +Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those +frameworks into a single ``XCframework``. + +The initial skeleton of an ``XCframework`` is built using:: + + xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework + +Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of +the XCframework:: + + cp path/to/iphoneos/bin Python.xcframework/ios-arm64 + cp path/to/iphoneos/lib Python.xcframework/ios-arm64 + + cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator + cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator + +Note that the name of the architecture-specific slice for the simulator will +depend on the CPU architecture(s) that you build. + +You now have a Python.xcframework that can be used in a project. + +Testing Python on iOS +===================== + +The ``iOS/testbed`` folder that contains an Xcode project that is able to run +the iOS test suite. This project converts the Python test suite into a single +test case in Xcode's XCTest framework. The single XCTest passes if the test +suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` +), specifying a framework build (i.e. ``--enable-framework``). Ensure that your +``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and +exclude any non-iOS tools, then run:: + + $ make all + $ make install + $ make testios + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "iPhone SE (3rd generation)" simulator. + +While the test suite is running, Xcode does not display any console output. +After showing some Xcode build commands, the console output will print ``Testing +started``, and then appear to stop. It will remain in this state until the test +suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to boot and prepare the +iOS simulator. + +On success, the test suite will exit and report successful completion of the +test suite. No output of the Python test suite will be displayed. + +On failure, the output of the Python test suite *will* be displayed. This will +show the details of the tests that failed. + +Debugging test failures +----------------------- + +The easiest way to diagnose a single test failure is to open the testbed project +in Xcode and run the tests from there using the "Product > Test" menu item. + +To test in Xcode, you must ensure the testbed project has a copy of a compiled +framework. If you've configured your build with the default install location of +``iOS/Frameworks``, you can copy from that location into the test project. To +test on an ARM64 simulator, run:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* + $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +To test on an x86-64 simulator, run:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* + $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +To test on a physical device:: + + $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/* + $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64 + +Alternatively, you can configure your build to install directly into the +testbed project. For a simulator, use:: + + --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + +For a physical device, use:: + + --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64 + + +Testing on an iOS device +^^^^^^^^^^^^^^^^^^^^^^^^ + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. + +Running specific tests +^^^^^^^^^^^^^^^^^^^^^^ + +As the test suite is being executed on an iOS simulator, it is not possible to +pass in command line arguments to configure test suite operation. To work around +this limitation, the arguments that would normally be passed as command line +arguments are configured as a static string at the start of the XCTest method +``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test +suite, add a a string to the ``argv`` definition. These arguments will be passed +to the test suite as if they had been passed to ``python -m test`` at the +command line. + +Disabling automated breakpoints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Xcode will inserts an automatic breakpoint whenever a signal is +raised. The Python test suite raises many of these signals as part of normal +operation; unless you are trying to diagnose an issue with signals, the +automatic breakpoints can be inconvenient. However, they can be disabled by +creating a symbolic breakpoint that is triggered at the start of the test run. + +Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and +populate the new brewpoint with the following details: + +* **Name**: IgnoreSignals +* **Symbol**: UIApplicationMain +* **Action**: Add debugger commands for: + - ``process handle SIGINT -n true -p true -s false`` + - ``process handle SIGUSR1 -n true -p true -s false`` + - ``process handle SIGUSR2 -n true -p true -s false`` + - ``process handle SIGXFSZ -n true -p true -s false`` +* Check the "Automatically continue after evaluating" box. + +All other details can be left blank. When the process executes the +``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger +commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in new file mode 100644 index 00000000000000..c3e261ecd9eff7 --- /dev/null +++ b/iOS/Resources/Info.plist.in @@ -0,0 +1,34 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + Python + CFBundleGetInfoString + Python Runtime and Library + CFBundleIdentifier + @PYTHONFRAMEWORKIDENTIFIER@ + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Python + CFBundlePackageType + FMWK + CFBundleShortVersionString + @VERSION@ + CFBundleLongVersionString + %VERSION%, (c) 2001-2024 Python Software Foundation. + CFBundleSignature + ???? + CFBundleVersion + 1 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + @IPHONEOS_DEPLOYMENT_TARGET@ + + diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000000..8122332b9c1de0 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000000..4d525751eba798 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000000..891bb25bb4318c --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..32574cad284441 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..6aaf6fbe188c32 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..bcbe91f6061e16 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..e6a42d9b85dec7 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/dylib-Info-template.plist b/iOS/Resources/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/Resources/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/Resources/pyconfig.h b/iOS/Resources/pyconfig.h new file mode 100644 index 00000000000000..4acff2c6051637 --- /dev/null +++ b/iOS/Resources/pyconfig.h @@ -0,0 +1,7 @@ +#ifdef __arm64__ +#include "pyconfig-arm64.h" +#endif + +#ifdef __x86_64__ +#include "pyconfig-x86_64.h" +#endif diff --git a/iOS/testbed/Python.xcframework/Info.plist b/iOS/testbed/Python.xcframework/Info.plist new file mode 100644 index 00000000000000..c6418de6e74a4e --- /dev/null +++ b/iOS/testbed/Python.xcframework/Info.plist @@ -0,0 +1,44 @@ + + + + + AvailableLibraries + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64 + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64_x86_64-simulator + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/iOS/testbed/Python.xcframework/ios-arm64/README new file mode 100644 index 00000000000000..c1b076d12cddb7 --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS on-device +build for testing purposes. diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README new file mode 100644 index 00000000000000..ae334e5d769d9d --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS simulator +build for testing purposes (either x86_64 or ARM64). diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj new file mode 100644 index 00000000000000..d57cfc3dbe0304 --- /dev/null +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -0,0 +1,570 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66162B0EFA380010BFC8 /* AppDelegate.m */; }; + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; + 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 607A660A2B0EFA380010BFC8 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 607A66112B0EFA380010BFC8; + remoteInfo = iOSTestbed; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = iOSTestbed.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66152B0EFA380010BFC8 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; + 607A66162B0EFA380010BFC8 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = ""; }; + 607A66212B0EFA390010BFC8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 607A660F2B0EFA380010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662A2B0EFA3A0010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 607A66092B0EFA380010BFC8 = { + isa = PBXGroup; + children = ( + 607A664A2B0EFB310010BFC8 /* Python.xcframework */, + 607A66142B0EFA380010BFC8 /* iOSTestbed */, + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66132B0EFA380010BFC8 /* Products */, + 607A664F2B0EFFE00010BFC8 /* Frameworks */, + ); + sourceTree = ""; + }; + 607A66132B0EFA380010BFC8 /* Products */ = { + isa = PBXGroup; + children = ( + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */, + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 607A66142B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXGroup; + children = ( + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, + 607A66152B0EFA380010BFC8 /* AppDelegate.h */, + 607A66162B0EFA380010BFC8 /* AppDelegate.m */, + 607A66212B0EFA390010BFC8 /* Assets.xcassets */, + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */, + 607A66272B0EFA390010BFC8 /* main.m */, + ); + path = iOSTestbed; + sourceTree = ""; + }; + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXGroup; + children = ( + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + ); + path = iOSTestbedTests; + sourceTree = ""; + }; + 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { + isa = PBXGroup; + children = ( + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 607A66112B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */; + buildPhases = ( + 607A660E2B0EFA380010BFC8 /* Sources */, + 607A660F2B0EFA380010BFC8 /* Frameworks */, + 607A66102B0EFA380010BFC8 /* Resources */, + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = iOSTestbed; + productName = iOSTestbed; + productReference = 607A66122B0EFA380010BFC8 /* iOSTestbed.app */; + productType = "com.apple.product-type.application"; + }; + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */; + buildPhases = ( + 607A66292B0EFA3A0010BFC8 /* Sources */, + 607A662A2B0EFA3A0010BFC8 /* Frameworks */, + 607A662B2B0EFA3A0010BFC8 /* Resources */, + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */, + ); + name = iOSTestbedTests; + productName = iOSTestbedTests; + productReference = 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 607A660A2B0EFA380010BFC8 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1500; + TargetAttributes = { + 607A66112B0EFA380010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + }; + 607A662C2B0EFA3A0010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + TestTargetID = 607A66112B0EFA380010BFC8; + }; + }; + }; + buildConfigurationList = 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 607A66092B0EFA380010BFC8; + productRefGroup = 607A66132B0EFA380010BFC8 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 607A66112B0EFA380010BFC8 /* iOSTestbed */, + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 607A66102B0EFA380010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662B2B0EFA3A0010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Install Target Specific Python Standard Library"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + }; + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Prepare Python Binary Modules"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 607A660E2B0EFA380010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */, + 607A66282B0EFA390010BFC8 /* main.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66292B0EFA3A0010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 607A66112B0EFA380010BFC8 /* iOSTestbed */; + targetProxy = 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 607A66242B0EFA390010BFC8 /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 607A663F2B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 607A66402B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 607A66422B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 607A66432B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 607A66452B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Debug; + }; + 607A66462B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A663F2B0EFA3A0010BFC8 /* Debug */, + 607A66402B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66422B0EFA3A0010BFC8 /* Debug */, + 607A66432B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66452B0EFA3A0010BFC8 /* Debug */, + 607A66462B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 607A660A2B0EFA380010BFC8 /* Project object */; +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/iOS/testbed/iOSTestbed/AppDelegate.h new file mode 100644 index 00000000000000..f695b3b5efc08b --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.h @@ -0,0 +1,11 @@ +// +// AppDelegate.h +// iOSTestbed +// + +#import + +@interface AppDelegate : UIResponder + + +@end diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/iOS/testbed/iOSTestbed/AppDelegate.m new file mode 100644 index 00000000000000..e5085399d0ca5f --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.m @@ -0,0 +1,19 @@ +// +// AppDelegate.m +// iOSTestbed +// + +#import "AppDelegate.h" + +@interface AppDelegate () + +@end + +@implementation AppDelegate + + +- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + return YES; +} + +@end diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000000..eb878970081645 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000000..13613e3ee1a934 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json new file mode 100644 index 00000000000000..73c00596a7fca3 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 00000000000000..5daafe73a866b7 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/iOS/testbed/iOSTestbed/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/testbed/iOSTestbed/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist new file mode 100644 index 00000000000000..e2aa460b6fd5ee --- /dev/null +++ b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -0,0 +1,54 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + ${PRODUCT_NAME} + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + org.python.iOSTestbed + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + ${PRODUCT_NAME} + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIRequiresFullScreen + + UILaunchStoryboardName + Launch Screen + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + MainModule + ios + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + + + diff --git a/iOS/testbed/iOSTestbed/main.m b/iOS/testbed/iOSTestbed/main.m new file mode 100644 index 00000000000000..e32bd78c9b42ee --- /dev/null +++ b/iOS/testbed/iOSTestbed/main.m @@ -0,0 +1,16 @@ +// +// main.m +// iOSTestbed +// + +#import +#import "AppDelegate.h" + +int main(int argc, char * argv[]) { + NSString * appDelegateClassName; + @autoreleasepool { + appDelegateClassName = NSStringFromClass([AppDelegate class]); + + return UIApplicationMain(argc, argv, nil, appDelegateClassName); + } +} diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m new file mode 100644 index 00000000000000..e6a919c304ec8d --- /dev/null +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -0,0 +1,111 @@ +#import +#import + +@interface iOSTestbedTests : XCTestCase + +@end + +@implementation iOSTestbedTests + + +- (void)testPython { + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg + // will be handled as if it were an argument to `python -m test` + const char *argv[] = { + "iOSTestbed", // argv[0] is the process that is running. + "-uall", // Enable all resources + "-W", // Display test output on failure + // To run a subset of tests, add the test names below; e.g., + // "test_os", + // "test_sys", + }; + + // Start a Python interpreter. + int exit_code; + PyStatus status; + PyPreConfig preconfig; + PyConfig config; + NSString *python_home; + wchar_t *wtmp_str; + + NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + + // Generate an isolated Python configuration. + NSLog(@"Configuring isolated Python..."); + PyPreConfig_InitIsolatedConfig(&preconfig); + PyConfig_InitIsolatedConfig(&config); + + // Configure the Python interpreter: + // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. + // See https://docs.python.org/3/library/os.html#python-utf-8-mode. + preconfig.utf8_mode = 1; + // Don't buffer stdio. We want output to appears in the log immediately + config.buffered_stdio = 0; + // Don't write bytecode; we can't modify the app bundle + // after it has been signed. + config.write_bytecode = 0; + // Ensure that signal handlers are installed + config.install_signal_handlers = 1; + // Run the test module. + config.run_module = Py_DecodeLocale("test", NULL); + // For debugging - enable verbose mode. + // config.verbose = 1; + + NSLog(@"Pre-initializing Python runtime..."); + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to pre-initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Set the home for the Python interpreter + python_home = [NSString stringWithFormat:@"%@/python", resourcePath, nil]; + NSLog(@"PythonHome: %@", python_home); + wtmp_str = Py_DecodeLocale([python_home UTF8String], NULL); + status = PyConfig_SetString(&config, &config.home, wtmp_str); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to set PYTHONHOME: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + PyMem_RawFree(wtmp_str); + + // Read the site config + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to read site config: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Configure argc/argv..."); + status = PyConfig_SetBytesArgv(&config, sizeof(argv) / sizeof(char *), (char**) argv); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Initializing Python runtime..."); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Start the test suite. Print a separator to differentiate Python startup logs from app logs + NSLog(@"---------------------------------------------------------------------------"); + + exit_code = Py_RunMain(); + XCTAssertEqual(exit_code, 0, @"Python test suite did not pass"); + + NSLog(@"---------------------------------------------------------------------------"); + + Py_Finalize(); +} + + +@end diff --git a/pyconfig.h.in b/pyconfig.h.in index 4b8fc4a677c776..8fbba7ed3b949e 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,12 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if the system has the type `clock_t'. */ +#undef HAVE_CLOCK_T + +/* Define to 1 if you have the `closefrom' function. */ +#undef HAVE_CLOSEFROM + /* Define to 1 if you have the `close_range' function. */ #undef HAVE_CLOSE_RANGE @@ -471,6 +477,9 @@ /* Define to 1 if you have the `getgid' function. */ #undef HAVE_GETGID +/* Define to 1 if you have the `getgrent' function. */ +#undef HAVE_GETGRENT + /* Define to 1 if you have the `getgrgid' function. */ #undef HAVE_GETGRGID @@ -598,6 +607,9 @@ bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG +/* Define to 1 if you have the `grantpt' function. */ +#undef HAVE_GRANTPT + /* Define to 1 if you have the header file. */ #undef HAVE_GRP_H @@ -817,12 +829,33 @@ /* Define to 1 if you have the `nanosleep' function. */ #undef HAVE_NANOSLEEP -/* Define to 1 if you have the `ncursesw' library. */ +/* Define if you have the 'ncurses' library */ +#undef HAVE_NCURSES + +/* Define if you have the 'ncursesw' library */ #undef HAVE_NCURSESW +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_CURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_PANEL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_CURSES_H + /* Define to 1 if you have the header file. */ #undef HAVE_NCURSES_H +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_PANEL_H + /* Define to 1 if you have the header file. */ #undef HAVE_NDBM_H @@ -866,6 +899,12 @@ /* Define to 1 if you have the `openpty' function. */ #undef HAVE_OPENPTY +/* Define if you have the 'panel' library */ +#undef HAVE_PANEL + +/* Define if you have the 'panelw' library */ +#undef HAVE_PANELW + /* Define to 1 if you have the header file. */ #undef HAVE_PANEL_H @@ -896,12 +935,19 @@ /* Define to 1 if you have the `posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE +/* Define to 1 if you have the `posix_openpt' function. */ +#undef HAVE_POSIX_OPENPT + /* Define to 1 if you have the `posix_spawn' function. */ #undef HAVE_POSIX_SPAWN /* Define to 1 if you have the `posix_spawnp' function. */ #undef HAVE_POSIX_SPAWNP +/* Define to 1 if you have the `posix_spawn_file_actions_addclosefrom_np' + function. */ +#undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + /* Define to 1 if you have the `pread' function. */ #undef HAVE_PREAD @@ -917,12 +963,19 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H +/* Define to 1 if you have the `process_vm_readv' function. */ +#undef HAVE_PROCESS_VM_READV + /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES /* Define to 1 if you have the `pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK +/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. + */ +#undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR @@ -944,6 +997,12 @@ /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS +/* Define to 1 if you have the `ptsname' function. */ +#undef HAVE_PTSNAME + +/* Define to 1 if you have the `ptsname_r' function. */ +#undef HAVE_PTSNAME_R + /* Define to 1 if you have the header file. */ #undef HAVE_PTY_H @@ -983,7 +1042,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define if readline supports rl_compdisp_func_t */ +/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1169,13 +1228,16 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR +/* Define to 1 if the system has the type `socklen_t'. */ +#undef HAVE_SOCKLEN_T + /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H /* Define to 1 if you have the `splice' function. */ #undef HAVE_SPLICE -/* Define if your compiler provides ssize_t */ +/* Define to 1 if the system has the type `ssize_t'. */ #undef HAVE_SSIZE_T /* Define to 1 if you have the `statvfs' function. */ @@ -1452,6 +1514,9 @@ /* Define to 1 if you have the `unlinkat' function. */ #undef HAVE_UNLINKAT +/* Define to 1 if you have the `unlockpt' function. */ +#undef HAVE_UNLOCKPT + /* Define to 1 if you have the `unshare' function. */ #undef HAVE_UNSHARE @@ -1539,6 +1604,9 @@ /* Define to 1 if you have the `_getpty' function. */ #undef HAVE__GETPTY +/* Define to 1 if the system has the type `__uint128_t'. */ +#undef HAVE___UINT128_T + /* Define to 1 if `major', `minor', and `makedev' are declared in . */ #undef MAJOR_IN_MKDEV @@ -1611,12 +1679,21 @@ /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED +/* Define if you want to disable the GIL */ +#undef Py_GIL_DISABLED + /* Define hash algorithm for str, bytes and memoryview. SipHash24: 1, FNV: 2, SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM -/* Define if you want to disable the GIL */ -#undef Py_NOGIL +/* Defined if _Complex C type is available. */ +#undef Py_HAVE_C_COMPLEX + +/* Define if year with century should be normalized for strftime. */ +#undef Py_NORMALIZE_CENTURY + +/* Define if rl_startup_hook takes arguments */ +#undef Py_RL_STARTUP_HOOK_TAKES_ARGS /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS @@ -1800,9 +1877,6 @@ /* Define if WINDOW in curses.h offers a field _flags. */ #undef WINDOW_HAS_FLAGS -/* Define to build the readline module against Apple BSD editline. */ -#undef WITH_APPLE_EDITLINE - /* Define if you want build the _decimal module using a coroutine-local rather than a thread-local context */ #undef WITH_DECIMAL_CONTEXTVAR @@ -1827,6 +1901,9 @@ /* Define to 1 if libintl is needed for locale functions. */ #undef WITH_LIBINTL +/* Define if you want to compile in mimalloc memory allocator. */ +#undef WITH_MIMALLOC + /* Define if you want to produce an OpenStep/Rhapsody framework (shared library plus accessory files). */ #undef WITH_NEXT_FRAMEWORK @@ -1903,7 +1980,7 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 'long' if doesn't define. */ +/* Define to 'long' if does not define clock_t. */ #undef clock_t /* Define to empty if `const' does not conform to ANSI C. */ @@ -1927,7 +2004,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to 'int' if does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */